1 /* 2 * Tiny Code Generator for QEMU 3 * 4 * Copyright (c) 2008 Fabrice Bellard 5 * 6 * Permission is hereby granted, free of charge, to any person obtaining a copy 7 * of this software and associated documentation files (the "Software"), to deal 8 * in the Software without restriction, including without limitation the rights 9 * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 10 * copies of the Software, and to permit persons to whom the Software is 11 * furnished to do so, subject to the following conditions: 12 * 13 * The above copyright notice and this permission notice shall be included in 14 * all copies or substantial portions of the Software. 15 * 16 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 17 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 18 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL 19 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 20 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 21 * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN 22 * THE SOFTWARE. 23 */ 24 25 #include "qemu/osdep.h" 26 #include "qemu-common.h" 27 #include "cpu.h" 28 #include "exec/exec-all.h" 29 #include "tcg.h" 30 #include "tcg-op.h" 31 #include "tcg-mo.h" 32 #include "trace-tcg.h" 33 #include "trace/mem.h" 34 35 /* Reduce the number of ifdefs below. This assumes that all uses of 36 TCGV_HIGH and TCGV_LOW are properly protected by a conditional that 37 the compiler can eliminate. */ 38 #if TCG_TARGET_REG_BITS == 64 39 extern TCGv_i32 TCGV_LOW_link_error(TCGv_i64); 40 extern TCGv_i32 TCGV_HIGH_link_error(TCGv_i64); 41 #define TCGV_LOW TCGV_LOW_link_error 42 #define TCGV_HIGH TCGV_HIGH_link_error 43 #endif 44 45 void tcg_gen_op1(TCGOpcode opc, TCGArg a1) 46 { 47 TCGOp *op = tcg_emit_op(opc); 48 op->args[0] = a1; 49 } 50 51 void tcg_gen_op2(TCGOpcode opc, TCGArg a1, TCGArg a2) 52 { 53 TCGOp *op = tcg_emit_op(opc); 54 op->args[0] = a1; 55 op->args[1] = a2; 56 } 57 58 void tcg_gen_op3(TCGOpcode opc, TCGArg a1, TCGArg a2, TCGArg a3) 59 { 60 TCGOp *op = tcg_emit_op(opc); 61 op->args[0] = a1; 62 op->args[1] = a2; 63 op->args[2] = a3; 64 } 65 66 void tcg_gen_op4(TCGOpcode opc, TCGArg a1, TCGArg a2, TCGArg a3, TCGArg a4) 67 { 68 TCGOp *op = tcg_emit_op(opc); 69 op->args[0] = a1; 70 op->args[1] = a2; 71 op->args[2] = a3; 72 op->args[3] = a4; 73 } 74 75 void tcg_gen_op5(TCGOpcode opc, TCGArg a1, TCGArg a2, TCGArg a3, 76 TCGArg a4, TCGArg a5) 77 { 78 TCGOp *op = tcg_emit_op(opc); 79 op->args[0] = a1; 80 op->args[1] = a2; 81 op->args[2] = a3; 82 op->args[3] = a4; 83 op->args[4] = a5; 84 } 85 86 void tcg_gen_op6(TCGOpcode opc, TCGArg a1, TCGArg a2, TCGArg a3, 87 TCGArg a4, TCGArg a5, TCGArg a6) 88 { 89 TCGOp *op = tcg_emit_op(opc); 90 op->args[0] = a1; 91 op->args[1] = a2; 92 op->args[2] = a3; 93 op->args[3] = a4; 94 op->args[4] = a5; 95 op->args[5] = a6; 96 } 97 98 void tcg_gen_mb(TCGBar mb_type) 99 { 100 if (tcg_ctx->tb_cflags & CF_PARALLEL) { 101 tcg_gen_op1(INDEX_op_mb, mb_type); 102 } 103 } 104 105 /* 32 bit ops */ 106 107 void tcg_gen_addi_i32(TCGv_i32 ret, TCGv_i32 arg1, int32_t arg2) 108 { 109 /* some cases can be optimized here */ 110 if (arg2 == 0) { 111 tcg_gen_mov_i32(ret, arg1); 112 } else { 113 TCGv_i32 t0 = tcg_const_i32(arg2); 114 tcg_gen_add_i32(ret, arg1, t0); 115 tcg_temp_free_i32(t0); 116 } 117 } 118 119 void tcg_gen_subfi_i32(TCGv_i32 ret, int32_t arg1, TCGv_i32 arg2) 120 { 121 if (arg1 == 0 && TCG_TARGET_HAS_neg_i32) { 122 /* Don't recurse with tcg_gen_neg_i32. */ 123 tcg_gen_op2_i32(INDEX_op_neg_i32, ret, arg2); 124 } else { 125 TCGv_i32 t0 = tcg_const_i32(arg1); 126 tcg_gen_sub_i32(ret, t0, arg2); 127 tcg_temp_free_i32(t0); 128 } 129 } 130 131 void tcg_gen_subi_i32(TCGv_i32 ret, TCGv_i32 arg1, int32_t arg2) 132 { 133 /* some cases can be optimized here */ 134 if (arg2 == 0) { 135 tcg_gen_mov_i32(ret, arg1); 136 } else { 137 TCGv_i32 t0 = tcg_const_i32(arg2); 138 tcg_gen_sub_i32(ret, arg1, t0); 139 tcg_temp_free_i32(t0); 140 } 141 } 142 143 void tcg_gen_andi_i32(TCGv_i32 ret, TCGv_i32 arg1, int32_t arg2) 144 { 145 TCGv_i32 t0; 146 /* Some cases can be optimized here. */ 147 switch (arg2) { 148 case 0: 149 tcg_gen_movi_i32(ret, 0); 150 return; 151 case -1: 152 tcg_gen_mov_i32(ret, arg1); 153 return; 154 case 0xff: 155 /* Don't recurse with tcg_gen_ext8u_i32. */ 156 if (TCG_TARGET_HAS_ext8u_i32) { 157 tcg_gen_op2_i32(INDEX_op_ext8u_i32, ret, arg1); 158 return; 159 } 160 break; 161 case 0xffff: 162 if (TCG_TARGET_HAS_ext16u_i32) { 163 tcg_gen_op2_i32(INDEX_op_ext16u_i32, ret, arg1); 164 return; 165 } 166 break; 167 } 168 t0 = tcg_const_i32(arg2); 169 tcg_gen_and_i32(ret, arg1, t0); 170 tcg_temp_free_i32(t0); 171 } 172 173 void tcg_gen_ori_i32(TCGv_i32 ret, TCGv_i32 arg1, int32_t arg2) 174 { 175 /* Some cases can be optimized here. */ 176 if (arg2 == -1) { 177 tcg_gen_movi_i32(ret, -1); 178 } else if (arg2 == 0) { 179 tcg_gen_mov_i32(ret, arg1); 180 } else { 181 TCGv_i32 t0 = tcg_const_i32(arg2); 182 tcg_gen_or_i32(ret, arg1, t0); 183 tcg_temp_free_i32(t0); 184 } 185 } 186 187 void tcg_gen_xori_i32(TCGv_i32 ret, TCGv_i32 arg1, int32_t arg2) 188 { 189 /* Some cases can be optimized here. */ 190 if (arg2 == 0) { 191 tcg_gen_mov_i32(ret, arg1); 192 } else if (arg2 == -1 && TCG_TARGET_HAS_not_i32) { 193 /* Don't recurse with tcg_gen_not_i32. */ 194 tcg_gen_op2_i32(INDEX_op_not_i32, ret, arg1); 195 } else { 196 TCGv_i32 t0 = tcg_const_i32(arg2); 197 tcg_gen_xor_i32(ret, arg1, t0); 198 tcg_temp_free_i32(t0); 199 } 200 } 201 202 void tcg_gen_shli_i32(TCGv_i32 ret, TCGv_i32 arg1, int32_t arg2) 203 { 204 tcg_debug_assert(arg2 >= 0 && arg2 < 32); 205 if (arg2 == 0) { 206 tcg_gen_mov_i32(ret, arg1); 207 } else { 208 TCGv_i32 t0 = tcg_const_i32(arg2); 209 tcg_gen_shl_i32(ret, arg1, t0); 210 tcg_temp_free_i32(t0); 211 } 212 } 213 214 void tcg_gen_shri_i32(TCGv_i32 ret, TCGv_i32 arg1, int32_t arg2) 215 { 216 tcg_debug_assert(arg2 >= 0 && arg2 < 32); 217 if (arg2 == 0) { 218 tcg_gen_mov_i32(ret, arg1); 219 } else { 220 TCGv_i32 t0 = tcg_const_i32(arg2); 221 tcg_gen_shr_i32(ret, arg1, t0); 222 tcg_temp_free_i32(t0); 223 } 224 } 225 226 void tcg_gen_sari_i32(TCGv_i32 ret, TCGv_i32 arg1, int32_t arg2) 227 { 228 tcg_debug_assert(arg2 >= 0 && arg2 < 32); 229 if (arg2 == 0) { 230 tcg_gen_mov_i32(ret, arg1); 231 } else { 232 TCGv_i32 t0 = tcg_const_i32(arg2); 233 tcg_gen_sar_i32(ret, arg1, t0); 234 tcg_temp_free_i32(t0); 235 } 236 } 237 238 void tcg_gen_brcond_i32(TCGCond cond, TCGv_i32 arg1, TCGv_i32 arg2, TCGLabel *l) 239 { 240 if (cond == TCG_COND_ALWAYS) { 241 tcg_gen_br(l); 242 } else if (cond != TCG_COND_NEVER) { 243 l->refs++; 244 tcg_gen_op4ii_i32(INDEX_op_brcond_i32, arg1, arg2, cond, label_arg(l)); 245 } 246 } 247 248 void tcg_gen_brcondi_i32(TCGCond cond, TCGv_i32 arg1, int32_t arg2, TCGLabel *l) 249 { 250 if (cond == TCG_COND_ALWAYS) { 251 tcg_gen_br(l); 252 } else if (cond != TCG_COND_NEVER) { 253 TCGv_i32 t0 = tcg_const_i32(arg2); 254 tcg_gen_brcond_i32(cond, arg1, t0, l); 255 tcg_temp_free_i32(t0); 256 } 257 } 258 259 void tcg_gen_setcond_i32(TCGCond cond, TCGv_i32 ret, 260 TCGv_i32 arg1, TCGv_i32 arg2) 261 { 262 if (cond == TCG_COND_ALWAYS) { 263 tcg_gen_movi_i32(ret, 1); 264 } else if (cond == TCG_COND_NEVER) { 265 tcg_gen_movi_i32(ret, 0); 266 } else { 267 tcg_gen_op4i_i32(INDEX_op_setcond_i32, ret, arg1, arg2, cond); 268 } 269 } 270 271 void tcg_gen_setcondi_i32(TCGCond cond, TCGv_i32 ret, 272 TCGv_i32 arg1, int32_t arg2) 273 { 274 TCGv_i32 t0 = tcg_const_i32(arg2); 275 tcg_gen_setcond_i32(cond, ret, arg1, t0); 276 tcg_temp_free_i32(t0); 277 } 278 279 void tcg_gen_muli_i32(TCGv_i32 ret, TCGv_i32 arg1, int32_t arg2) 280 { 281 if (arg2 == 0) { 282 tcg_gen_movi_i32(ret, 0); 283 } else if (is_power_of_2(arg2)) { 284 tcg_gen_shli_i32(ret, arg1, ctz32(arg2)); 285 } else { 286 TCGv_i32 t0 = tcg_const_i32(arg2); 287 tcg_gen_mul_i32(ret, arg1, t0); 288 tcg_temp_free_i32(t0); 289 } 290 } 291 292 void tcg_gen_div_i32(TCGv_i32 ret, TCGv_i32 arg1, TCGv_i32 arg2) 293 { 294 if (TCG_TARGET_HAS_div_i32) { 295 tcg_gen_op3_i32(INDEX_op_div_i32, ret, arg1, arg2); 296 } else if (TCG_TARGET_HAS_div2_i32) { 297 TCGv_i32 t0 = tcg_temp_new_i32(); 298 tcg_gen_sari_i32(t0, arg1, 31); 299 tcg_gen_op5_i32(INDEX_op_div2_i32, ret, t0, arg1, t0, arg2); 300 tcg_temp_free_i32(t0); 301 } else { 302 gen_helper_div_i32(ret, arg1, arg2); 303 } 304 } 305 306 void tcg_gen_rem_i32(TCGv_i32 ret, TCGv_i32 arg1, TCGv_i32 arg2) 307 { 308 if (TCG_TARGET_HAS_rem_i32) { 309 tcg_gen_op3_i32(INDEX_op_rem_i32, ret, arg1, arg2); 310 } else if (TCG_TARGET_HAS_div_i32) { 311 TCGv_i32 t0 = tcg_temp_new_i32(); 312 tcg_gen_op3_i32(INDEX_op_div_i32, t0, arg1, arg2); 313 tcg_gen_mul_i32(t0, t0, arg2); 314 tcg_gen_sub_i32(ret, arg1, t0); 315 tcg_temp_free_i32(t0); 316 } else if (TCG_TARGET_HAS_div2_i32) { 317 TCGv_i32 t0 = tcg_temp_new_i32(); 318 tcg_gen_sari_i32(t0, arg1, 31); 319 tcg_gen_op5_i32(INDEX_op_div2_i32, t0, ret, arg1, t0, arg2); 320 tcg_temp_free_i32(t0); 321 } else { 322 gen_helper_rem_i32(ret, arg1, arg2); 323 } 324 } 325 326 void tcg_gen_divu_i32(TCGv_i32 ret, TCGv_i32 arg1, TCGv_i32 arg2) 327 { 328 if (TCG_TARGET_HAS_div_i32) { 329 tcg_gen_op3_i32(INDEX_op_divu_i32, ret, arg1, arg2); 330 } else if (TCG_TARGET_HAS_div2_i32) { 331 TCGv_i32 t0 = tcg_temp_new_i32(); 332 tcg_gen_movi_i32(t0, 0); 333 tcg_gen_op5_i32(INDEX_op_divu2_i32, ret, t0, arg1, t0, arg2); 334 tcg_temp_free_i32(t0); 335 } else { 336 gen_helper_divu_i32(ret, arg1, arg2); 337 } 338 } 339 340 void tcg_gen_remu_i32(TCGv_i32 ret, TCGv_i32 arg1, TCGv_i32 arg2) 341 { 342 if (TCG_TARGET_HAS_rem_i32) { 343 tcg_gen_op3_i32(INDEX_op_remu_i32, ret, arg1, arg2); 344 } else if (TCG_TARGET_HAS_div_i32) { 345 TCGv_i32 t0 = tcg_temp_new_i32(); 346 tcg_gen_op3_i32(INDEX_op_divu_i32, t0, arg1, arg2); 347 tcg_gen_mul_i32(t0, t0, arg2); 348 tcg_gen_sub_i32(ret, arg1, t0); 349 tcg_temp_free_i32(t0); 350 } else if (TCG_TARGET_HAS_div2_i32) { 351 TCGv_i32 t0 = tcg_temp_new_i32(); 352 tcg_gen_movi_i32(t0, 0); 353 tcg_gen_op5_i32(INDEX_op_divu2_i32, t0, ret, arg1, t0, arg2); 354 tcg_temp_free_i32(t0); 355 } else { 356 gen_helper_remu_i32(ret, arg1, arg2); 357 } 358 } 359 360 void tcg_gen_andc_i32(TCGv_i32 ret, TCGv_i32 arg1, TCGv_i32 arg2) 361 { 362 if (TCG_TARGET_HAS_andc_i32) { 363 tcg_gen_op3_i32(INDEX_op_andc_i32, ret, arg1, arg2); 364 } else { 365 TCGv_i32 t0 = tcg_temp_new_i32(); 366 tcg_gen_not_i32(t0, arg2); 367 tcg_gen_and_i32(ret, arg1, t0); 368 tcg_temp_free_i32(t0); 369 } 370 } 371 372 void tcg_gen_eqv_i32(TCGv_i32 ret, TCGv_i32 arg1, TCGv_i32 arg2) 373 { 374 if (TCG_TARGET_HAS_eqv_i32) { 375 tcg_gen_op3_i32(INDEX_op_eqv_i32, ret, arg1, arg2); 376 } else { 377 tcg_gen_xor_i32(ret, arg1, arg2); 378 tcg_gen_not_i32(ret, ret); 379 } 380 } 381 382 void tcg_gen_nand_i32(TCGv_i32 ret, TCGv_i32 arg1, TCGv_i32 arg2) 383 { 384 if (TCG_TARGET_HAS_nand_i32) { 385 tcg_gen_op3_i32(INDEX_op_nand_i32, ret, arg1, arg2); 386 } else { 387 tcg_gen_and_i32(ret, arg1, arg2); 388 tcg_gen_not_i32(ret, ret); 389 } 390 } 391 392 void tcg_gen_nor_i32(TCGv_i32 ret, TCGv_i32 arg1, TCGv_i32 arg2) 393 { 394 if (TCG_TARGET_HAS_nor_i32) { 395 tcg_gen_op3_i32(INDEX_op_nor_i32, ret, arg1, arg2); 396 } else { 397 tcg_gen_or_i32(ret, arg1, arg2); 398 tcg_gen_not_i32(ret, ret); 399 } 400 } 401 402 void tcg_gen_orc_i32(TCGv_i32 ret, TCGv_i32 arg1, TCGv_i32 arg2) 403 { 404 if (TCG_TARGET_HAS_orc_i32) { 405 tcg_gen_op3_i32(INDEX_op_orc_i32, ret, arg1, arg2); 406 } else { 407 TCGv_i32 t0 = tcg_temp_new_i32(); 408 tcg_gen_not_i32(t0, arg2); 409 tcg_gen_or_i32(ret, arg1, t0); 410 tcg_temp_free_i32(t0); 411 } 412 } 413 414 void tcg_gen_clz_i32(TCGv_i32 ret, TCGv_i32 arg1, TCGv_i32 arg2) 415 { 416 if (TCG_TARGET_HAS_clz_i32) { 417 tcg_gen_op3_i32(INDEX_op_clz_i32, ret, arg1, arg2); 418 } else if (TCG_TARGET_HAS_clz_i64) { 419 TCGv_i64 t1 = tcg_temp_new_i64(); 420 TCGv_i64 t2 = tcg_temp_new_i64(); 421 tcg_gen_extu_i32_i64(t1, arg1); 422 tcg_gen_extu_i32_i64(t2, arg2); 423 tcg_gen_addi_i64(t2, t2, 32); 424 tcg_gen_clz_i64(t1, t1, t2); 425 tcg_gen_extrl_i64_i32(ret, t1); 426 tcg_temp_free_i64(t1); 427 tcg_temp_free_i64(t2); 428 tcg_gen_subi_i32(ret, ret, 32); 429 } else { 430 gen_helper_clz_i32(ret, arg1, arg2); 431 } 432 } 433 434 void tcg_gen_clzi_i32(TCGv_i32 ret, TCGv_i32 arg1, uint32_t arg2) 435 { 436 TCGv_i32 t = tcg_const_i32(arg2); 437 tcg_gen_clz_i32(ret, arg1, t); 438 tcg_temp_free_i32(t); 439 } 440 441 void tcg_gen_ctz_i32(TCGv_i32 ret, TCGv_i32 arg1, TCGv_i32 arg2) 442 { 443 if (TCG_TARGET_HAS_ctz_i32) { 444 tcg_gen_op3_i32(INDEX_op_ctz_i32, ret, arg1, arg2); 445 } else if (TCG_TARGET_HAS_ctz_i64) { 446 TCGv_i64 t1 = tcg_temp_new_i64(); 447 TCGv_i64 t2 = tcg_temp_new_i64(); 448 tcg_gen_extu_i32_i64(t1, arg1); 449 tcg_gen_extu_i32_i64(t2, arg2); 450 tcg_gen_ctz_i64(t1, t1, t2); 451 tcg_gen_extrl_i64_i32(ret, t1); 452 tcg_temp_free_i64(t1); 453 tcg_temp_free_i64(t2); 454 } else if (TCG_TARGET_HAS_ctpop_i32 455 || TCG_TARGET_HAS_ctpop_i64 456 || TCG_TARGET_HAS_clz_i32 457 || TCG_TARGET_HAS_clz_i64) { 458 TCGv_i32 z, t = tcg_temp_new_i32(); 459 460 if (TCG_TARGET_HAS_ctpop_i32 || TCG_TARGET_HAS_ctpop_i64) { 461 tcg_gen_subi_i32(t, arg1, 1); 462 tcg_gen_andc_i32(t, t, arg1); 463 tcg_gen_ctpop_i32(t, t); 464 } else { 465 /* Since all non-x86 hosts have clz(0) == 32, don't fight it. */ 466 tcg_gen_neg_i32(t, arg1); 467 tcg_gen_and_i32(t, t, arg1); 468 tcg_gen_clzi_i32(t, t, 32); 469 tcg_gen_xori_i32(t, t, 31); 470 } 471 z = tcg_const_i32(0); 472 tcg_gen_movcond_i32(TCG_COND_EQ, ret, arg1, z, arg2, t); 473 tcg_temp_free_i32(t); 474 tcg_temp_free_i32(z); 475 } else { 476 gen_helper_ctz_i32(ret, arg1, arg2); 477 } 478 } 479 480 void tcg_gen_ctzi_i32(TCGv_i32 ret, TCGv_i32 arg1, uint32_t arg2) 481 { 482 if (!TCG_TARGET_HAS_ctz_i32 && TCG_TARGET_HAS_ctpop_i32 && arg2 == 32) { 483 /* This equivalence has the advantage of not requiring a fixup. */ 484 TCGv_i32 t = tcg_temp_new_i32(); 485 tcg_gen_subi_i32(t, arg1, 1); 486 tcg_gen_andc_i32(t, t, arg1); 487 tcg_gen_ctpop_i32(ret, t); 488 tcg_temp_free_i32(t); 489 } else { 490 TCGv_i32 t = tcg_const_i32(arg2); 491 tcg_gen_ctz_i32(ret, arg1, t); 492 tcg_temp_free_i32(t); 493 } 494 } 495 496 void tcg_gen_clrsb_i32(TCGv_i32 ret, TCGv_i32 arg) 497 { 498 if (TCG_TARGET_HAS_clz_i32) { 499 TCGv_i32 t = tcg_temp_new_i32(); 500 tcg_gen_sari_i32(t, arg, 31); 501 tcg_gen_xor_i32(t, t, arg); 502 tcg_gen_clzi_i32(t, t, 32); 503 tcg_gen_subi_i32(ret, t, 1); 504 tcg_temp_free_i32(t); 505 } else { 506 gen_helper_clrsb_i32(ret, arg); 507 } 508 } 509 510 void tcg_gen_ctpop_i32(TCGv_i32 ret, TCGv_i32 arg1) 511 { 512 if (TCG_TARGET_HAS_ctpop_i32) { 513 tcg_gen_op2_i32(INDEX_op_ctpop_i32, ret, arg1); 514 } else if (TCG_TARGET_HAS_ctpop_i64) { 515 TCGv_i64 t = tcg_temp_new_i64(); 516 tcg_gen_extu_i32_i64(t, arg1); 517 tcg_gen_ctpop_i64(t, t); 518 tcg_gen_extrl_i64_i32(ret, t); 519 tcg_temp_free_i64(t); 520 } else { 521 gen_helper_ctpop_i32(ret, arg1); 522 } 523 } 524 525 void tcg_gen_rotl_i32(TCGv_i32 ret, TCGv_i32 arg1, TCGv_i32 arg2) 526 { 527 if (TCG_TARGET_HAS_rot_i32) { 528 tcg_gen_op3_i32(INDEX_op_rotl_i32, ret, arg1, arg2); 529 } else { 530 TCGv_i32 t0, t1; 531 532 t0 = tcg_temp_new_i32(); 533 t1 = tcg_temp_new_i32(); 534 tcg_gen_shl_i32(t0, arg1, arg2); 535 tcg_gen_subfi_i32(t1, 32, arg2); 536 tcg_gen_shr_i32(t1, arg1, t1); 537 tcg_gen_or_i32(ret, t0, t1); 538 tcg_temp_free_i32(t0); 539 tcg_temp_free_i32(t1); 540 } 541 } 542 543 void tcg_gen_rotli_i32(TCGv_i32 ret, TCGv_i32 arg1, unsigned arg2) 544 { 545 tcg_debug_assert(arg2 < 32); 546 /* some cases can be optimized here */ 547 if (arg2 == 0) { 548 tcg_gen_mov_i32(ret, arg1); 549 } else if (TCG_TARGET_HAS_rot_i32) { 550 TCGv_i32 t0 = tcg_const_i32(arg2); 551 tcg_gen_rotl_i32(ret, arg1, t0); 552 tcg_temp_free_i32(t0); 553 } else { 554 TCGv_i32 t0, t1; 555 t0 = tcg_temp_new_i32(); 556 t1 = tcg_temp_new_i32(); 557 tcg_gen_shli_i32(t0, arg1, arg2); 558 tcg_gen_shri_i32(t1, arg1, 32 - arg2); 559 tcg_gen_or_i32(ret, t0, t1); 560 tcg_temp_free_i32(t0); 561 tcg_temp_free_i32(t1); 562 } 563 } 564 565 void tcg_gen_rotr_i32(TCGv_i32 ret, TCGv_i32 arg1, TCGv_i32 arg2) 566 { 567 if (TCG_TARGET_HAS_rot_i32) { 568 tcg_gen_op3_i32(INDEX_op_rotr_i32, ret, arg1, arg2); 569 } else { 570 TCGv_i32 t0, t1; 571 572 t0 = tcg_temp_new_i32(); 573 t1 = tcg_temp_new_i32(); 574 tcg_gen_shr_i32(t0, arg1, arg2); 575 tcg_gen_subfi_i32(t1, 32, arg2); 576 tcg_gen_shl_i32(t1, arg1, t1); 577 tcg_gen_or_i32(ret, t0, t1); 578 tcg_temp_free_i32(t0); 579 tcg_temp_free_i32(t1); 580 } 581 } 582 583 void tcg_gen_rotri_i32(TCGv_i32 ret, TCGv_i32 arg1, unsigned arg2) 584 { 585 tcg_debug_assert(arg2 < 32); 586 /* some cases can be optimized here */ 587 if (arg2 == 0) { 588 tcg_gen_mov_i32(ret, arg1); 589 } else { 590 tcg_gen_rotli_i32(ret, arg1, 32 - arg2); 591 } 592 } 593 594 void tcg_gen_deposit_i32(TCGv_i32 ret, TCGv_i32 arg1, TCGv_i32 arg2, 595 unsigned int ofs, unsigned int len) 596 { 597 uint32_t mask; 598 TCGv_i32 t1; 599 600 tcg_debug_assert(ofs < 32); 601 tcg_debug_assert(len > 0); 602 tcg_debug_assert(len <= 32); 603 tcg_debug_assert(ofs + len <= 32); 604 605 if (len == 32) { 606 tcg_gen_mov_i32(ret, arg2); 607 return; 608 } 609 if (TCG_TARGET_HAS_deposit_i32 && TCG_TARGET_deposit_i32_valid(ofs, len)) { 610 tcg_gen_op5ii_i32(INDEX_op_deposit_i32, ret, arg1, arg2, ofs, len); 611 return; 612 } 613 614 mask = (1u << len) - 1; 615 t1 = tcg_temp_new_i32(); 616 617 if (ofs + len < 32) { 618 tcg_gen_andi_i32(t1, arg2, mask); 619 tcg_gen_shli_i32(t1, t1, ofs); 620 } else { 621 tcg_gen_shli_i32(t1, arg2, ofs); 622 } 623 tcg_gen_andi_i32(ret, arg1, ~(mask << ofs)); 624 tcg_gen_or_i32(ret, ret, t1); 625 626 tcg_temp_free_i32(t1); 627 } 628 629 void tcg_gen_deposit_z_i32(TCGv_i32 ret, TCGv_i32 arg, 630 unsigned int ofs, unsigned int len) 631 { 632 tcg_debug_assert(ofs < 32); 633 tcg_debug_assert(len > 0); 634 tcg_debug_assert(len <= 32); 635 tcg_debug_assert(ofs + len <= 32); 636 637 if (ofs + len == 32) { 638 tcg_gen_shli_i32(ret, arg, ofs); 639 } else if (ofs == 0) { 640 tcg_gen_andi_i32(ret, arg, (1u << len) - 1); 641 } else if (TCG_TARGET_HAS_deposit_i32 642 && TCG_TARGET_deposit_i32_valid(ofs, len)) { 643 TCGv_i32 zero = tcg_const_i32(0); 644 tcg_gen_op5ii_i32(INDEX_op_deposit_i32, ret, zero, arg, ofs, len); 645 tcg_temp_free_i32(zero); 646 } else { 647 /* To help two-operand hosts we prefer to zero-extend first, 648 which allows ARG to stay live. */ 649 switch (len) { 650 case 16: 651 if (TCG_TARGET_HAS_ext16u_i32) { 652 tcg_gen_ext16u_i32(ret, arg); 653 tcg_gen_shli_i32(ret, ret, ofs); 654 return; 655 } 656 break; 657 case 8: 658 if (TCG_TARGET_HAS_ext8u_i32) { 659 tcg_gen_ext8u_i32(ret, arg); 660 tcg_gen_shli_i32(ret, ret, ofs); 661 return; 662 } 663 break; 664 } 665 /* Otherwise prefer zero-extension over AND for code size. */ 666 switch (ofs + len) { 667 case 16: 668 if (TCG_TARGET_HAS_ext16u_i32) { 669 tcg_gen_shli_i32(ret, arg, ofs); 670 tcg_gen_ext16u_i32(ret, ret); 671 return; 672 } 673 break; 674 case 8: 675 if (TCG_TARGET_HAS_ext8u_i32) { 676 tcg_gen_shli_i32(ret, arg, ofs); 677 tcg_gen_ext8u_i32(ret, ret); 678 return; 679 } 680 break; 681 } 682 tcg_gen_andi_i32(ret, arg, (1u << len) - 1); 683 tcg_gen_shli_i32(ret, ret, ofs); 684 } 685 } 686 687 void tcg_gen_extract_i32(TCGv_i32 ret, TCGv_i32 arg, 688 unsigned int ofs, unsigned int len) 689 { 690 tcg_debug_assert(ofs < 32); 691 tcg_debug_assert(len > 0); 692 tcg_debug_assert(len <= 32); 693 tcg_debug_assert(ofs + len <= 32); 694 695 /* Canonicalize certain special cases, even if extract is supported. */ 696 if (ofs + len == 32) { 697 tcg_gen_shri_i32(ret, arg, 32 - len); 698 return; 699 } 700 if (ofs == 0) { 701 tcg_gen_andi_i32(ret, arg, (1u << len) - 1); 702 return; 703 } 704 705 if (TCG_TARGET_HAS_extract_i32 706 && TCG_TARGET_extract_i32_valid(ofs, len)) { 707 tcg_gen_op4ii_i32(INDEX_op_extract_i32, ret, arg, ofs, len); 708 return; 709 } 710 711 /* Assume that zero-extension, if available, is cheaper than a shift. */ 712 switch (ofs + len) { 713 case 16: 714 if (TCG_TARGET_HAS_ext16u_i32) { 715 tcg_gen_ext16u_i32(ret, arg); 716 tcg_gen_shri_i32(ret, ret, ofs); 717 return; 718 } 719 break; 720 case 8: 721 if (TCG_TARGET_HAS_ext8u_i32) { 722 tcg_gen_ext8u_i32(ret, arg); 723 tcg_gen_shri_i32(ret, ret, ofs); 724 return; 725 } 726 break; 727 } 728 729 /* ??? Ideally we'd know what values are available for immediate AND. 730 Assume that 8 bits are available, plus the special case of 16, 731 so that we get ext8u, ext16u. */ 732 switch (len) { 733 case 1 ... 8: case 16: 734 tcg_gen_shri_i32(ret, arg, ofs); 735 tcg_gen_andi_i32(ret, ret, (1u << len) - 1); 736 break; 737 default: 738 tcg_gen_shli_i32(ret, arg, 32 - len - ofs); 739 tcg_gen_shri_i32(ret, ret, 32 - len); 740 break; 741 } 742 } 743 744 void tcg_gen_sextract_i32(TCGv_i32 ret, TCGv_i32 arg, 745 unsigned int ofs, unsigned int len) 746 { 747 tcg_debug_assert(ofs < 32); 748 tcg_debug_assert(len > 0); 749 tcg_debug_assert(len <= 32); 750 tcg_debug_assert(ofs + len <= 32); 751 752 /* Canonicalize certain special cases, even if extract is supported. */ 753 if (ofs + len == 32) { 754 tcg_gen_sari_i32(ret, arg, 32 - len); 755 return; 756 } 757 if (ofs == 0) { 758 switch (len) { 759 case 16: 760 tcg_gen_ext16s_i32(ret, arg); 761 return; 762 case 8: 763 tcg_gen_ext8s_i32(ret, arg); 764 return; 765 } 766 } 767 768 if (TCG_TARGET_HAS_sextract_i32 769 && TCG_TARGET_extract_i32_valid(ofs, len)) { 770 tcg_gen_op4ii_i32(INDEX_op_sextract_i32, ret, arg, ofs, len); 771 return; 772 } 773 774 /* Assume that sign-extension, if available, is cheaper than a shift. */ 775 switch (ofs + len) { 776 case 16: 777 if (TCG_TARGET_HAS_ext16s_i32) { 778 tcg_gen_ext16s_i32(ret, arg); 779 tcg_gen_sari_i32(ret, ret, ofs); 780 return; 781 } 782 break; 783 case 8: 784 if (TCG_TARGET_HAS_ext8s_i32) { 785 tcg_gen_ext8s_i32(ret, arg); 786 tcg_gen_sari_i32(ret, ret, ofs); 787 return; 788 } 789 break; 790 } 791 switch (len) { 792 case 16: 793 if (TCG_TARGET_HAS_ext16s_i32) { 794 tcg_gen_shri_i32(ret, arg, ofs); 795 tcg_gen_ext16s_i32(ret, ret); 796 return; 797 } 798 break; 799 case 8: 800 if (TCG_TARGET_HAS_ext8s_i32) { 801 tcg_gen_shri_i32(ret, arg, ofs); 802 tcg_gen_ext8s_i32(ret, ret); 803 return; 804 } 805 break; 806 } 807 808 tcg_gen_shli_i32(ret, arg, 32 - len - ofs); 809 tcg_gen_sari_i32(ret, ret, 32 - len); 810 } 811 812 void tcg_gen_movcond_i32(TCGCond cond, TCGv_i32 ret, TCGv_i32 c1, 813 TCGv_i32 c2, TCGv_i32 v1, TCGv_i32 v2) 814 { 815 if (cond == TCG_COND_ALWAYS) { 816 tcg_gen_mov_i32(ret, v1); 817 } else if (cond == TCG_COND_NEVER) { 818 tcg_gen_mov_i32(ret, v2); 819 } else if (TCG_TARGET_HAS_movcond_i32) { 820 tcg_gen_op6i_i32(INDEX_op_movcond_i32, ret, c1, c2, v1, v2, cond); 821 } else { 822 TCGv_i32 t0 = tcg_temp_new_i32(); 823 TCGv_i32 t1 = tcg_temp_new_i32(); 824 tcg_gen_setcond_i32(cond, t0, c1, c2); 825 tcg_gen_neg_i32(t0, t0); 826 tcg_gen_and_i32(t1, v1, t0); 827 tcg_gen_andc_i32(ret, v2, t0); 828 tcg_gen_or_i32(ret, ret, t1); 829 tcg_temp_free_i32(t0); 830 tcg_temp_free_i32(t1); 831 } 832 } 833 834 void tcg_gen_add2_i32(TCGv_i32 rl, TCGv_i32 rh, TCGv_i32 al, 835 TCGv_i32 ah, TCGv_i32 bl, TCGv_i32 bh) 836 { 837 if (TCG_TARGET_HAS_add2_i32) { 838 tcg_gen_op6_i32(INDEX_op_add2_i32, rl, rh, al, ah, bl, bh); 839 } else { 840 TCGv_i64 t0 = tcg_temp_new_i64(); 841 TCGv_i64 t1 = tcg_temp_new_i64(); 842 tcg_gen_concat_i32_i64(t0, al, ah); 843 tcg_gen_concat_i32_i64(t1, bl, bh); 844 tcg_gen_add_i64(t0, t0, t1); 845 tcg_gen_extr_i64_i32(rl, rh, t0); 846 tcg_temp_free_i64(t0); 847 tcg_temp_free_i64(t1); 848 } 849 } 850 851 void tcg_gen_sub2_i32(TCGv_i32 rl, TCGv_i32 rh, TCGv_i32 al, 852 TCGv_i32 ah, TCGv_i32 bl, TCGv_i32 bh) 853 { 854 if (TCG_TARGET_HAS_sub2_i32) { 855 tcg_gen_op6_i32(INDEX_op_sub2_i32, rl, rh, al, ah, bl, bh); 856 } else { 857 TCGv_i64 t0 = tcg_temp_new_i64(); 858 TCGv_i64 t1 = tcg_temp_new_i64(); 859 tcg_gen_concat_i32_i64(t0, al, ah); 860 tcg_gen_concat_i32_i64(t1, bl, bh); 861 tcg_gen_sub_i64(t0, t0, t1); 862 tcg_gen_extr_i64_i32(rl, rh, t0); 863 tcg_temp_free_i64(t0); 864 tcg_temp_free_i64(t1); 865 } 866 } 867 868 void tcg_gen_mulu2_i32(TCGv_i32 rl, TCGv_i32 rh, TCGv_i32 arg1, TCGv_i32 arg2) 869 { 870 if (TCG_TARGET_HAS_mulu2_i32) { 871 tcg_gen_op4_i32(INDEX_op_mulu2_i32, rl, rh, arg1, arg2); 872 } else if (TCG_TARGET_HAS_muluh_i32) { 873 TCGv_i32 t = tcg_temp_new_i32(); 874 tcg_gen_op3_i32(INDEX_op_mul_i32, t, arg1, arg2); 875 tcg_gen_op3_i32(INDEX_op_muluh_i32, rh, arg1, arg2); 876 tcg_gen_mov_i32(rl, t); 877 tcg_temp_free_i32(t); 878 } else { 879 TCGv_i64 t0 = tcg_temp_new_i64(); 880 TCGv_i64 t1 = tcg_temp_new_i64(); 881 tcg_gen_extu_i32_i64(t0, arg1); 882 tcg_gen_extu_i32_i64(t1, arg2); 883 tcg_gen_mul_i64(t0, t0, t1); 884 tcg_gen_extr_i64_i32(rl, rh, t0); 885 tcg_temp_free_i64(t0); 886 tcg_temp_free_i64(t1); 887 } 888 } 889 890 void tcg_gen_muls2_i32(TCGv_i32 rl, TCGv_i32 rh, TCGv_i32 arg1, TCGv_i32 arg2) 891 { 892 if (TCG_TARGET_HAS_muls2_i32) { 893 tcg_gen_op4_i32(INDEX_op_muls2_i32, rl, rh, arg1, arg2); 894 } else if (TCG_TARGET_HAS_mulsh_i32) { 895 TCGv_i32 t = tcg_temp_new_i32(); 896 tcg_gen_op3_i32(INDEX_op_mul_i32, t, arg1, arg2); 897 tcg_gen_op3_i32(INDEX_op_mulsh_i32, rh, arg1, arg2); 898 tcg_gen_mov_i32(rl, t); 899 tcg_temp_free_i32(t); 900 } else if (TCG_TARGET_REG_BITS == 32) { 901 TCGv_i32 t0 = tcg_temp_new_i32(); 902 TCGv_i32 t1 = tcg_temp_new_i32(); 903 TCGv_i32 t2 = tcg_temp_new_i32(); 904 TCGv_i32 t3 = tcg_temp_new_i32(); 905 tcg_gen_mulu2_i32(t0, t1, arg1, arg2); 906 /* Adjust for negative inputs. */ 907 tcg_gen_sari_i32(t2, arg1, 31); 908 tcg_gen_sari_i32(t3, arg2, 31); 909 tcg_gen_and_i32(t2, t2, arg2); 910 tcg_gen_and_i32(t3, t3, arg1); 911 tcg_gen_sub_i32(rh, t1, t2); 912 tcg_gen_sub_i32(rh, rh, t3); 913 tcg_gen_mov_i32(rl, t0); 914 tcg_temp_free_i32(t0); 915 tcg_temp_free_i32(t1); 916 tcg_temp_free_i32(t2); 917 tcg_temp_free_i32(t3); 918 } else { 919 TCGv_i64 t0 = tcg_temp_new_i64(); 920 TCGv_i64 t1 = tcg_temp_new_i64(); 921 tcg_gen_ext_i32_i64(t0, arg1); 922 tcg_gen_ext_i32_i64(t1, arg2); 923 tcg_gen_mul_i64(t0, t0, t1); 924 tcg_gen_extr_i64_i32(rl, rh, t0); 925 tcg_temp_free_i64(t0); 926 tcg_temp_free_i64(t1); 927 } 928 } 929 930 void tcg_gen_mulsu2_i32(TCGv_i32 rl, TCGv_i32 rh, TCGv_i32 arg1, TCGv_i32 arg2) 931 { 932 if (TCG_TARGET_REG_BITS == 32) { 933 TCGv_i32 t0 = tcg_temp_new_i32(); 934 TCGv_i32 t1 = tcg_temp_new_i32(); 935 TCGv_i32 t2 = tcg_temp_new_i32(); 936 tcg_gen_mulu2_i32(t0, t1, arg1, arg2); 937 /* Adjust for negative input for the signed arg1. */ 938 tcg_gen_sari_i32(t2, arg1, 31); 939 tcg_gen_and_i32(t2, t2, arg2); 940 tcg_gen_sub_i32(rh, t1, t2); 941 tcg_gen_mov_i32(rl, t0); 942 tcg_temp_free_i32(t0); 943 tcg_temp_free_i32(t1); 944 tcg_temp_free_i32(t2); 945 } else { 946 TCGv_i64 t0 = tcg_temp_new_i64(); 947 TCGv_i64 t1 = tcg_temp_new_i64(); 948 tcg_gen_ext_i32_i64(t0, arg1); 949 tcg_gen_extu_i32_i64(t1, arg2); 950 tcg_gen_mul_i64(t0, t0, t1); 951 tcg_gen_extr_i64_i32(rl, rh, t0); 952 tcg_temp_free_i64(t0); 953 tcg_temp_free_i64(t1); 954 } 955 } 956 957 void tcg_gen_ext8s_i32(TCGv_i32 ret, TCGv_i32 arg) 958 { 959 if (TCG_TARGET_HAS_ext8s_i32) { 960 tcg_gen_op2_i32(INDEX_op_ext8s_i32, ret, arg); 961 } else { 962 tcg_gen_shli_i32(ret, arg, 24); 963 tcg_gen_sari_i32(ret, ret, 24); 964 } 965 } 966 967 void tcg_gen_ext16s_i32(TCGv_i32 ret, TCGv_i32 arg) 968 { 969 if (TCG_TARGET_HAS_ext16s_i32) { 970 tcg_gen_op2_i32(INDEX_op_ext16s_i32, ret, arg); 971 } else { 972 tcg_gen_shli_i32(ret, arg, 16); 973 tcg_gen_sari_i32(ret, ret, 16); 974 } 975 } 976 977 void tcg_gen_ext8u_i32(TCGv_i32 ret, TCGv_i32 arg) 978 { 979 if (TCG_TARGET_HAS_ext8u_i32) { 980 tcg_gen_op2_i32(INDEX_op_ext8u_i32, ret, arg); 981 } else { 982 tcg_gen_andi_i32(ret, arg, 0xffu); 983 } 984 } 985 986 void tcg_gen_ext16u_i32(TCGv_i32 ret, TCGv_i32 arg) 987 { 988 if (TCG_TARGET_HAS_ext16u_i32) { 989 tcg_gen_op2_i32(INDEX_op_ext16u_i32, ret, arg); 990 } else { 991 tcg_gen_andi_i32(ret, arg, 0xffffu); 992 } 993 } 994 995 /* Note: we assume the two high bytes are set to zero */ 996 void tcg_gen_bswap16_i32(TCGv_i32 ret, TCGv_i32 arg) 997 { 998 if (TCG_TARGET_HAS_bswap16_i32) { 999 tcg_gen_op2_i32(INDEX_op_bswap16_i32, ret, arg); 1000 } else { 1001 TCGv_i32 t0 = tcg_temp_new_i32(); 1002 1003 tcg_gen_ext8u_i32(t0, arg); 1004 tcg_gen_shli_i32(t0, t0, 8); 1005 tcg_gen_shri_i32(ret, arg, 8); 1006 tcg_gen_or_i32(ret, ret, t0); 1007 tcg_temp_free_i32(t0); 1008 } 1009 } 1010 1011 void tcg_gen_bswap32_i32(TCGv_i32 ret, TCGv_i32 arg) 1012 { 1013 if (TCG_TARGET_HAS_bswap32_i32) { 1014 tcg_gen_op2_i32(INDEX_op_bswap32_i32, ret, arg); 1015 } else { 1016 TCGv_i32 t0 = tcg_temp_new_i32(); 1017 TCGv_i32 t1 = tcg_temp_new_i32(); 1018 TCGv_i32 t2 = tcg_const_i32(0x00ff00ff); 1019 1020 /* arg = abcd */ 1021 tcg_gen_shri_i32(t0, arg, 8); /* t0 = .abc */ 1022 tcg_gen_and_i32(t1, arg, t2); /* t1 = .b.d */ 1023 tcg_gen_and_i32(t0, t0, t2); /* t0 = .a.c */ 1024 tcg_gen_shli_i32(t1, t1, 8); /* t1 = b.d. */ 1025 tcg_gen_or_i32(ret, t0, t1); /* ret = badc */ 1026 1027 tcg_gen_shri_i32(t0, ret, 16); /* t0 = ..ba */ 1028 tcg_gen_shli_i32(t1, ret, 16); /* t1 = dc.. */ 1029 tcg_gen_or_i32(ret, t0, t1); /* ret = dcba */ 1030 1031 tcg_temp_free_i32(t0); 1032 tcg_temp_free_i32(t1); 1033 tcg_temp_free_i32(t2); 1034 } 1035 } 1036 1037 void tcg_gen_smin_i32(TCGv_i32 ret, TCGv_i32 a, TCGv_i32 b) 1038 { 1039 tcg_gen_movcond_i32(TCG_COND_LT, ret, a, b, a, b); 1040 } 1041 1042 void tcg_gen_umin_i32(TCGv_i32 ret, TCGv_i32 a, TCGv_i32 b) 1043 { 1044 tcg_gen_movcond_i32(TCG_COND_LTU, ret, a, b, a, b); 1045 } 1046 1047 void tcg_gen_smax_i32(TCGv_i32 ret, TCGv_i32 a, TCGv_i32 b) 1048 { 1049 tcg_gen_movcond_i32(TCG_COND_LT, ret, a, b, b, a); 1050 } 1051 1052 void tcg_gen_umax_i32(TCGv_i32 ret, TCGv_i32 a, TCGv_i32 b) 1053 { 1054 tcg_gen_movcond_i32(TCG_COND_LTU, ret, a, b, b, a); 1055 } 1056 1057 /* 64-bit ops */ 1058 1059 #if TCG_TARGET_REG_BITS == 32 1060 /* These are all inline for TCG_TARGET_REG_BITS == 64. */ 1061 1062 void tcg_gen_discard_i64(TCGv_i64 arg) 1063 { 1064 tcg_gen_discard_i32(TCGV_LOW(arg)); 1065 tcg_gen_discard_i32(TCGV_HIGH(arg)); 1066 } 1067 1068 void tcg_gen_mov_i64(TCGv_i64 ret, TCGv_i64 arg) 1069 { 1070 tcg_gen_mov_i32(TCGV_LOW(ret), TCGV_LOW(arg)); 1071 tcg_gen_mov_i32(TCGV_HIGH(ret), TCGV_HIGH(arg)); 1072 } 1073 1074 void tcg_gen_movi_i64(TCGv_i64 ret, int64_t arg) 1075 { 1076 tcg_gen_movi_i32(TCGV_LOW(ret), arg); 1077 tcg_gen_movi_i32(TCGV_HIGH(ret), arg >> 32); 1078 } 1079 1080 void tcg_gen_ld8u_i64(TCGv_i64 ret, TCGv_ptr arg2, tcg_target_long offset) 1081 { 1082 tcg_gen_ld8u_i32(TCGV_LOW(ret), arg2, offset); 1083 tcg_gen_movi_i32(TCGV_HIGH(ret), 0); 1084 } 1085 1086 void tcg_gen_ld8s_i64(TCGv_i64 ret, TCGv_ptr arg2, tcg_target_long offset) 1087 { 1088 tcg_gen_ld8s_i32(TCGV_LOW(ret), arg2, offset); 1089 tcg_gen_sari_i32(TCGV_HIGH(ret), TCGV_LOW(ret), 31); 1090 } 1091 1092 void tcg_gen_ld16u_i64(TCGv_i64 ret, TCGv_ptr arg2, tcg_target_long offset) 1093 { 1094 tcg_gen_ld16u_i32(TCGV_LOW(ret), arg2, offset); 1095 tcg_gen_movi_i32(TCGV_HIGH(ret), 0); 1096 } 1097 1098 void tcg_gen_ld16s_i64(TCGv_i64 ret, TCGv_ptr arg2, tcg_target_long offset) 1099 { 1100 tcg_gen_ld16s_i32(TCGV_LOW(ret), arg2, offset); 1101 tcg_gen_sari_i32(TCGV_HIGH(ret), TCGV_LOW(ret), 31); 1102 } 1103 1104 void tcg_gen_ld32u_i64(TCGv_i64 ret, TCGv_ptr arg2, tcg_target_long offset) 1105 { 1106 tcg_gen_ld_i32(TCGV_LOW(ret), arg2, offset); 1107 tcg_gen_movi_i32(TCGV_HIGH(ret), 0); 1108 } 1109 1110 void tcg_gen_ld32s_i64(TCGv_i64 ret, TCGv_ptr arg2, tcg_target_long offset) 1111 { 1112 tcg_gen_ld_i32(TCGV_LOW(ret), arg2, offset); 1113 tcg_gen_sari_i32(TCGV_HIGH(ret), TCGV_LOW(ret), 31); 1114 } 1115 1116 void tcg_gen_ld_i64(TCGv_i64 ret, TCGv_ptr arg2, tcg_target_long offset) 1117 { 1118 /* Since arg2 and ret have different types, 1119 they cannot be the same temporary */ 1120 #ifdef HOST_WORDS_BIGENDIAN 1121 tcg_gen_ld_i32(TCGV_HIGH(ret), arg2, offset); 1122 tcg_gen_ld_i32(TCGV_LOW(ret), arg2, offset + 4); 1123 #else 1124 tcg_gen_ld_i32(TCGV_LOW(ret), arg2, offset); 1125 tcg_gen_ld_i32(TCGV_HIGH(ret), arg2, offset + 4); 1126 #endif 1127 } 1128 1129 void tcg_gen_st_i64(TCGv_i64 arg1, TCGv_ptr arg2, tcg_target_long offset) 1130 { 1131 #ifdef HOST_WORDS_BIGENDIAN 1132 tcg_gen_st_i32(TCGV_HIGH(arg1), arg2, offset); 1133 tcg_gen_st_i32(TCGV_LOW(arg1), arg2, offset + 4); 1134 #else 1135 tcg_gen_st_i32(TCGV_LOW(arg1), arg2, offset); 1136 tcg_gen_st_i32(TCGV_HIGH(arg1), arg2, offset + 4); 1137 #endif 1138 } 1139 1140 void tcg_gen_and_i64(TCGv_i64 ret, TCGv_i64 arg1, TCGv_i64 arg2) 1141 { 1142 tcg_gen_and_i32(TCGV_LOW(ret), TCGV_LOW(arg1), TCGV_LOW(arg2)); 1143 tcg_gen_and_i32(TCGV_HIGH(ret), TCGV_HIGH(arg1), TCGV_HIGH(arg2)); 1144 } 1145 1146 void tcg_gen_or_i64(TCGv_i64 ret, TCGv_i64 arg1, TCGv_i64 arg2) 1147 { 1148 tcg_gen_or_i32(TCGV_LOW(ret), TCGV_LOW(arg1), TCGV_LOW(arg2)); 1149 tcg_gen_or_i32(TCGV_HIGH(ret), TCGV_HIGH(arg1), TCGV_HIGH(arg2)); 1150 } 1151 1152 void tcg_gen_xor_i64(TCGv_i64 ret, TCGv_i64 arg1, TCGv_i64 arg2) 1153 { 1154 tcg_gen_xor_i32(TCGV_LOW(ret), TCGV_LOW(arg1), TCGV_LOW(arg2)); 1155 tcg_gen_xor_i32(TCGV_HIGH(ret), TCGV_HIGH(arg1), TCGV_HIGH(arg2)); 1156 } 1157 1158 void tcg_gen_shl_i64(TCGv_i64 ret, TCGv_i64 arg1, TCGv_i64 arg2) 1159 { 1160 gen_helper_shl_i64(ret, arg1, arg2); 1161 } 1162 1163 void tcg_gen_shr_i64(TCGv_i64 ret, TCGv_i64 arg1, TCGv_i64 arg2) 1164 { 1165 gen_helper_shr_i64(ret, arg1, arg2); 1166 } 1167 1168 void tcg_gen_sar_i64(TCGv_i64 ret, TCGv_i64 arg1, TCGv_i64 arg2) 1169 { 1170 gen_helper_sar_i64(ret, arg1, arg2); 1171 } 1172 1173 void tcg_gen_mul_i64(TCGv_i64 ret, TCGv_i64 arg1, TCGv_i64 arg2) 1174 { 1175 TCGv_i64 t0; 1176 TCGv_i32 t1; 1177 1178 t0 = tcg_temp_new_i64(); 1179 t1 = tcg_temp_new_i32(); 1180 1181 tcg_gen_mulu2_i32(TCGV_LOW(t0), TCGV_HIGH(t0), 1182 TCGV_LOW(arg1), TCGV_LOW(arg2)); 1183 1184 tcg_gen_mul_i32(t1, TCGV_LOW(arg1), TCGV_HIGH(arg2)); 1185 tcg_gen_add_i32(TCGV_HIGH(t0), TCGV_HIGH(t0), t1); 1186 tcg_gen_mul_i32(t1, TCGV_HIGH(arg1), TCGV_LOW(arg2)); 1187 tcg_gen_add_i32(TCGV_HIGH(t0), TCGV_HIGH(t0), t1); 1188 1189 tcg_gen_mov_i64(ret, t0); 1190 tcg_temp_free_i64(t0); 1191 tcg_temp_free_i32(t1); 1192 } 1193 #endif /* TCG_TARGET_REG_SIZE == 32 */ 1194 1195 void tcg_gen_addi_i64(TCGv_i64 ret, TCGv_i64 arg1, int64_t arg2) 1196 { 1197 /* some cases can be optimized here */ 1198 if (arg2 == 0) { 1199 tcg_gen_mov_i64(ret, arg1); 1200 } else { 1201 TCGv_i64 t0 = tcg_const_i64(arg2); 1202 tcg_gen_add_i64(ret, arg1, t0); 1203 tcg_temp_free_i64(t0); 1204 } 1205 } 1206 1207 void tcg_gen_subfi_i64(TCGv_i64 ret, int64_t arg1, TCGv_i64 arg2) 1208 { 1209 if (arg1 == 0 && TCG_TARGET_HAS_neg_i64) { 1210 /* Don't recurse with tcg_gen_neg_i64. */ 1211 tcg_gen_op2_i64(INDEX_op_neg_i64, ret, arg2); 1212 } else { 1213 TCGv_i64 t0 = tcg_const_i64(arg1); 1214 tcg_gen_sub_i64(ret, t0, arg2); 1215 tcg_temp_free_i64(t0); 1216 } 1217 } 1218 1219 void tcg_gen_subi_i64(TCGv_i64 ret, TCGv_i64 arg1, int64_t arg2) 1220 { 1221 /* some cases can be optimized here */ 1222 if (arg2 == 0) { 1223 tcg_gen_mov_i64(ret, arg1); 1224 } else { 1225 TCGv_i64 t0 = tcg_const_i64(arg2); 1226 tcg_gen_sub_i64(ret, arg1, t0); 1227 tcg_temp_free_i64(t0); 1228 } 1229 } 1230 1231 void tcg_gen_andi_i64(TCGv_i64 ret, TCGv_i64 arg1, int64_t arg2) 1232 { 1233 TCGv_i64 t0; 1234 1235 if (TCG_TARGET_REG_BITS == 32) { 1236 tcg_gen_andi_i32(TCGV_LOW(ret), TCGV_LOW(arg1), arg2); 1237 tcg_gen_andi_i32(TCGV_HIGH(ret), TCGV_HIGH(arg1), arg2 >> 32); 1238 return; 1239 } 1240 1241 /* Some cases can be optimized here. */ 1242 switch (arg2) { 1243 case 0: 1244 tcg_gen_movi_i64(ret, 0); 1245 return; 1246 case -1: 1247 tcg_gen_mov_i64(ret, arg1); 1248 return; 1249 case 0xff: 1250 /* Don't recurse with tcg_gen_ext8u_i64. */ 1251 if (TCG_TARGET_HAS_ext8u_i64) { 1252 tcg_gen_op2_i64(INDEX_op_ext8u_i64, ret, arg1); 1253 return; 1254 } 1255 break; 1256 case 0xffff: 1257 if (TCG_TARGET_HAS_ext16u_i64) { 1258 tcg_gen_op2_i64(INDEX_op_ext16u_i64, ret, arg1); 1259 return; 1260 } 1261 break; 1262 case 0xffffffffu: 1263 if (TCG_TARGET_HAS_ext32u_i64) { 1264 tcg_gen_op2_i64(INDEX_op_ext32u_i64, ret, arg1); 1265 return; 1266 } 1267 break; 1268 } 1269 t0 = tcg_const_i64(arg2); 1270 tcg_gen_and_i64(ret, arg1, t0); 1271 tcg_temp_free_i64(t0); 1272 } 1273 1274 void tcg_gen_ori_i64(TCGv_i64 ret, TCGv_i64 arg1, int64_t arg2) 1275 { 1276 if (TCG_TARGET_REG_BITS == 32) { 1277 tcg_gen_ori_i32(TCGV_LOW(ret), TCGV_LOW(arg1), arg2); 1278 tcg_gen_ori_i32(TCGV_HIGH(ret), TCGV_HIGH(arg1), arg2 >> 32); 1279 return; 1280 } 1281 /* Some cases can be optimized here. */ 1282 if (arg2 == -1) { 1283 tcg_gen_movi_i64(ret, -1); 1284 } else if (arg2 == 0) { 1285 tcg_gen_mov_i64(ret, arg1); 1286 } else { 1287 TCGv_i64 t0 = tcg_const_i64(arg2); 1288 tcg_gen_or_i64(ret, arg1, t0); 1289 tcg_temp_free_i64(t0); 1290 } 1291 } 1292 1293 void tcg_gen_xori_i64(TCGv_i64 ret, TCGv_i64 arg1, int64_t arg2) 1294 { 1295 if (TCG_TARGET_REG_BITS == 32) { 1296 tcg_gen_xori_i32(TCGV_LOW(ret), TCGV_LOW(arg1), arg2); 1297 tcg_gen_xori_i32(TCGV_HIGH(ret), TCGV_HIGH(arg1), arg2 >> 32); 1298 return; 1299 } 1300 /* Some cases can be optimized here. */ 1301 if (arg2 == 0) { 1302 tcg_gen_mov_i64(ret, arg1); 1303 } else if (arg2 == -1 && TCG_TARGET_HAS_not_i64) { 1304 /* Don't recurse with tcg_gen_not_i64. */ 1305 tcg_gen_op2_i64(INDEX_op_not_i64, ret, arg1); 1306 } else { 1307 TCGv_i64 t0 = tcg_const_i64(arg2); 1308 tcg_gen_xor_i64(ret, arg1, t0); 1309 tcg_temp_free_i64(t0); 1310 } 1311 } 1312 1313 static inline void tcg_gen_shifti_i64(TCGv_i64 ret, TCGv_i64 arg1, 1314 unsigned c, bool right, bool arith) 1315 { 1316 tcg_debug_assert(c < 64); 1317 if (c == 0) { 1318 tcg_gen_mov_i32(TCGV_LOW(ret), TCGV_LOW(arg1)); 1319 tcg_gen_mov_i32(TCGV_HIGH(ret), TCGV_HIGH(arg1)); 1320 } else if (c >= 32) { 1321 c -= 32; 1322 if (right) { 1323 if (arith) { 1324 tcg_gen_sari_i32(TCGV_LOW(ret), TCGV_HIGH(arg1), c); 1325 tcg_gen_sari_i32(TCGV_HIGH(ret), TCGV_HIGH(arg1), 31); 1326 } else { 1327 tcg_gen_shri_i32(TCGV_LOW(ret), TCGV_HIGH(arg1), c); 1328 tcg_gen_movi_i32(TCGV_HIGH(ret), 0); 1329 } 1330 } else { 1331 tcg_gen_shli_i32(TCGV_HIGH(ret), TCGV_LOW(arg1), c); 1332 tcg_gen_movi_i32(TCGV_LOW(ret), 0); 1333 } 1334 } else { 1335 TCGv_i32 t0, t1; 1336 1337 t0 = tcg_temp_new_i32(); 1338 t1 = tcg_temp_new_i32(); 1339 if (right) { 1340 tcg_gen_shli_i32(t0, TCGV_HIGH(arg1), 32 - c); 1341 if (arith) { 1342 tcg_gen_sari_i32(t1, TCGV_HIGH(arg1), c); 1343 } else { 1344 tcg_gen_shri_i32(t1, TCGV_HIGH(arg1), c); 1345 } 1346 tcg_gen_shri_i32(TCGV_LOW(ret), TCGV_LOW(arg1), c); 1347 tcg_gen_or_i32(TCGV_LOW(ret), TCGV_LOW(ret), t0); 1348 tcg_gen_mov_i32(TCGV_HIGH(ret), t1); 1349 } else { 1350 tcg_gen_shri_i32(t0, TCGV_LOW(arg1), 32 - c); 1351 /* Note: ret can be the same as arg1, so we use t1 */ 1352 tcg_gen_shli_i32(t1, TCGV_LOW(arg1), c); 1353 tcg_gen_shli_i32(TCGV_HIGH(ret), TCGV_HIGH(arg1), c); 1354 tcg_gen_or_i32(TCGV_HIGH(ret), TCGV_HIGH(ret), t0); 1355 tcg_gen_mov_i32(TCGV_LOW(ret), t1); 1356 } 1357 tcg_temp_free_i32(t0); 1358 tcg_temp_free_i32(t1); 1359 } 1360 } 1361 1362 void tcg_gen_shli_i64(TCGv_i64 ret, TCGv_i64 arg1, int64_t arg2) 1363 { 1364 tcg_debug_assert(arg2 >= 0 && arg2 < 64); 1365 if (TCG_TARGET_REG_BITS == 32) { 1366 tcg_gen_shifti_i64(ret, arg1, arg2, 0, 0); 1367 } else if (arg2 == 0) { 1368 tcg_gen_mov_i64(ret, arg1); 1369 } else { 1370 TCGv_i64 t0 = tcg_const_i64(arg2); 1371 tcg_gen_shl_i64(ret, arg1, t0); 1372 tcg_temp_free_i64(t0); 1373 } 1374 } 1375 1376 void tcg_gen_shri_i64(TCGv_i64 ret, TCGv_i64 arg1, int64_t arg2) 1377 { 1378 tcg_debug_assert(arg2 >= 0 && arg2 < 64); 1379 if (TCG_TARGET_REG_BITS == 32) { 1380 tcg_gen_shifti_i64(ret, arg1, arg2, 1, 0); 1381 } else if (arg2 == 0) { 1382 tcg_gen_mov_i64(ret, arg1); 1383 } else { 1384 TCGv_i64 t0 = tcg_const_i64(arg2); 1385 tcg_gen_shr_i64(ret, arg1, t0); 1386 tcg_temp_free_i64(t0); 1387 } 1388 } 1389 1390 void tcg_gen_sari_i64(TCGv_i64 ret, TCGv_i64 arg1, int64_t arg2) 1391 { 1392 tcg_debug_assert(arg2 >= 0 && arg2 < 64); 1393 if (TCG_TARGET_REG_BITS == 32) { 1394 tcg_gen_shifti_i64(ret, arg1, arg2, 1, 1); 1395 } else if (arg2 == 0) { 1396 tcg_gen_mov_i64(ret, arg1); 1397 } else { 1398 TCGv_i64 t0 = tcg_const_i64(arg2); 1399 tcg_gen_sar_i64(ret, arg1, t0); 1400 tcg_temp_free_i64(t0); 1401 } 1402 } 1403 1404 void tcg_gen_brcond_i64(TCGCond cond, TCGv_i64 arg1, TCGv_i64 arg2, TCGLabel *l) 1405 { 1406 if (cond == TCG_COND_ALWAYS) { 1407 tcg_gen_br(l); 1408 } else if (cond != TCG_COND_NEVER) { 1409 l->refs++; 1410 if (TCG_TARGET_REG_BITS == 32) { 1411 tcg_gen_op6ii_i32(INDEX_op_brcond2_i32, TCGV_LOW(arg1), 1412 TCGV_HIGH(arg1), TCGV_LOW(arg2), 1413 TCGV_HIGH(arg2), cond, label_arg(l)); 1414 } else { 1415 tcg_gen_op4ii_i64(INDEX_op_brcond_i64, arg1, arg2, cond, 1416 label_arg(l)); 1417 } 1418 } 1419 } 1420 1421 void tcg_gen_brcondi_i64(TCGCond cond, TCGv_i64 arg1, int64_t arg2, TCGLabel *l) 1422 { 1423 if (cond == TCG_COND_ALWAYS) { 1424 tcg_gen_br(l); 1425 } else if (cond != TCG_COND_NEVER) { 1426 TCGv_i64 t0 = tcg_const_i64(arg2); 1427 tcg_gen_brcond_i64(cond, arg1, t0, l); 1428 tcg_temp_free_i64(t0); 1429 } 1430 } 1431 1432 void tcg_gen_setcond_i64(TCGCond cond, TCGv_i64 ret, 1433 TCGv_i64 arg1, TCGv_i64 arg2) 1434 { 1435 if (cond == TCG_COND_ALWAYS) { 1436 tcg_gen_movi_i64(ret, 1); 1437 } else if (cond == TCG_COND_NEVER) { 1438 tcg_gen_movi_i64(ret, 0); 1439 } else { 1440 if (TCG_TARGET_REG_BITS == 32) { 1441 tcg_gen_op6i_i32(INDEX_op_setcond2_i32, TCGV_LOW(ret), 1442 TCGV_LOW(arg1), TCGV_HIGH(arg1), 1443 TCGV_LOW(arg2), TCGV_HIGH(arg2), cond); 1444 tcg_gen_movi_i32(TCGV_HIGH(ret), 0); 1445 } else { 1446 tcg_gen_op4i_i64(INDEX_op_setcond_i64, ret, arg1, arg2, cond); 1447 } 1448 } 1449 } 1450 1451 void tcg_gen_setcondi_i64(TCGCond cond, TCGv_i64 ret, 1452 TCGv_i64 arg1, int64_t arg2) 1453 { 1454 TCGv_i64 t0 = tcg_const_i64(arg2); 1455 tcg_gen_setcond_i64(cond, ret, arg1, t0); 1456 tcg_temp_free_i64(t0); 1457 } 1458 1459 void tcg_gen_muli_i64(TCGv_i64 ret, TCGv_i64 arg1, int64_t arg2) 1460 { 1461 if (arg2 == 0) { 1462 tcg_gen_movi_i64(ret, 0); 1463 } else if (is_power_of_2(arg2)) { 1464 tcg_gen_shli_i64(ret, arg1, ctz64(arg2)); 1465 } else { 1466 TCGv_i64 t0 = tcg_const_i64(arg2); 1467 tcg_gen_mul_i64(ret, arg1, t0); 1468 tcg_temp_free_i64(t0); 1469 } 1470 } 1471 1472 void tcg_gen_div_i64(TCGv_i64 ret, TCGv_i64 arg1, TCGv_i64 arg2) 1473 { 1474 if (TCG_TARGET_HAS_div_i64) { 1475 tcg_gen_op3_i64(INDEX_op_div_i64, ret, arg1, arg2); 1476 } else if (TCG_TARGET_HAS_div2_i64) { 1477 TCGv_i64 t0 = tcg_temp_new_i64(); 1478 tcg_gen_sari_i64(t0, arg1, 63); 1479 tcg_gen_op5_i64(INDEX_op_div2_i64, ret, t0, arg1, t0, arg2); 1480 tcg_temp_free_i64(t0); 1481 } else { 1482 gen_helper_div_i64(ret, arg1, arg2); 1483 } 1484 } 1485 1486 void tcg_gen_rem_i64(TCGv_i64 ret, TCGv_i64 arg1, TCGv_i64 arg2) 1487 { 1488 if (TCG_TARGET_HAS_rem_i64) { 1489 tcg_gen_op3_i64(INDEX_op_rem_i64, ret, arg1, arg2); 1490 } else if (TCG_TARGET_HAS_div_i64) { 1491 TCGv_i64 t0 = tcg_temp_new_i64(); 1492 tcg_gen_op3_i64(INDEX_op_div_i64, t0, arg1, arg2); 1493 tcg_gen_mul_i64(t0, t0, arg2); 1494 tcg_gen_sub_i64(ret, arg1, t0); 1495 tcg_temp_free_i64(t0); 1496 } else if (TCG_TARGET_HAS_div2_i64) { 1497 TCGv_i64 t0 = tcg_temp_new_i64(); 1498 tcg_gen_sari_i64(t0, arg1, 63); 1499 tcg_gen_op5_i64(INDEX_op_div2_i64, t0, ret, arg1, t0, arg2); 1500 tcg_temp_free_i64(t0); 1501 } else { 1502 gen_helper_rem_i64(ret, arg1, arg2); 1503 } 1504 } 1505 1506 void tcg_gen_divu_i64(TCGv_i64 ret, TCGv_i64 arg1, TCGv_i64 arg2) 1507 { 1508 if (TCG_TARGET_HAS_div_i64) { 1509 tcg_gen_op3_i64(INDEX_op_divu_i64, ret, arg1, arg2); 1510 } else if (TCG_TARGET_HAS_div2_i64) { 1511 TCGv_i64 t0 = tcg_temp_new_i64(); 1512 tcg_gen_movi_i64(t0, 0); 1513 tcg_gen_op5_i64(INDEX_op_divu2_i64, ret, t0, arg1, t0, arg2); 1514 tcg_temp_free_i64(t0); 1515 } else { 1516 gen_helper_divu_i64(ret, arg1, arg2); 1517 } 1518 } 1519 1520 void tcg_gen_remu_i64(TCGv_i64 ret, TCGv_i64 arg1, TCGv_i64 arg2) 1521 { 1522 if (TCG_TARGET_HAS_rem_i64) { 1523 tcg_gen_op3_i64(INDEX_op_remu_i64, ret, arg1, arg2); 1524 } else if (TCG_TARGET_HAS_div_i64) { 1525 TCGv_i64 t0 = tcg_temp_new_i64(); 1526 tcg_gen_op3_i64(INDEX_op_divu_i64, t0, arg1, arg2); 1527 tcg_gen_mul_i64(t0, t0, arg2); 1528 tcg_gen_sub_i64(ret, arg1, t0); 1529 tcg_temp_free_i64(t0); 1530 } else if (TCG_TARGET_HAS_div2_i64) { 1531 TCGv_i64 t0 = tcg_temp_new_i64(); 1532 tcg_gen_movi_i64(t0, 0); 1533 tcg_gen_op5_i64(INDEX_op_divu2_i64, t0, ret, arg1, t0, arg2); 1534 tcg_temp_free_i64(t0); 1535 } else { 1536 gen_helper_remu_i64(ret, arg1, arg2); 1537 } 1538 } 1539 1540 void tcg_gen_ext8s_i64(TCGv_i64 ret, TCGv_i64 arg) 1541 { 1542 if (TCG_TARGET_REG_BITS == 32) { 1543 tcg_gen_ext8s_i32(TCGV_LOW(ret), TCGV_LOW(arg)); 1544 tcg_gen_sari_i32(TCGV_HIGH(ret), TCGV_LOW(ret), 31); 1545 } else if (TCG_TARGET_HAS_ext8s_i64) { 1546 tcg_gen_op2_i64(INDEX_op_ext8s_i64, ret, arg); 1547 } else { 1548 tcg_gen_shli_i64(ret, arg, 56); 1549 tcg_gen_sari_i64(ret, ret, 56); 1550 } 1551 } 1552 1553 void tcg_gen_ext16s_i64(TCGv_i64 ret, TCGv_i64 arg) 1554 { 1555 if (TCG_TARGET_REG_BITS == 32) { 1556 tcg_gen_ext16s_i32(TCGV_LOW(ret), TCGV_LOW(arg)); 1557 tcg_gen_sari_i32(TCGV_HIGH(ret), TCGV_LOW(ret), 31); 1558 } else if (TCG_TARGET_HAS_ext16s_i64) { 1559 tcg_gen_op2_i64(INDEX_op_ext16s_i64, ret, arg); 1560 } else { 1561 tcg_gen_shli_i64(ret, arg, 48); 1562 tcg_gen_sari_i64(ret, ret, 48); 1563 } 1564 } 1565 1566 void tcg_gen_ext32s_i64(TCGv_i64 ret, TCGv_i64 arg) 1567 { 1568 if (TCG_TARGET_REG_BITS == 32) { 1569 tcg_gen_mov_i32(TCGV_LOW(ret), TCGV_LOW(arg)); 1570 tcg_gen_sari_i32(TCGV_HIGH(ret), TCGV_LOW(ret), 31); 1571 } else if (TCG_TARGET_HAS_ext32s_i64) { 1572 tcg_gen_op2_i64(INDEX_op_ext32s_i64, ret, arg); 1573 } else { 1574 tcg_gen_shli_i64(ret, arg, 32); 1575 tcg_gen_sari_i64(ret, ret, 32); 1576 } 1577 } 1578 1579 void tcg_gen_ext8u_i64(TCGv_i64 ret, TCGv_i64 arg) 1580 { 1581 if (TCG_TARGET_REG_BITS == 32) { 1582 tcg_gen_ext8u_i32(TCGV_LOW(ret), TCGV_LOW(arg)); 1583 tcg_gen_movi_i32(TCGV_HIGH(ret), 0); 1584 } else if (TCG_TARGET_HAS_ext8u_i64) { 1585 tcg_gen_op2_i64(INDEX_op_ext8u_i64, ret, arg); 1586 } else { 1587 tcg_gen_andi_i64(ret, arg, 0xffu); 1588 } 1589 } 1590 1591 void tcg_gen_ext16u_i64(TCGv_i64 ret, TCGv_i64 arg) 1592 { 1593 if (TCG_TARGET_REG_BITS == 32) { 1594 tcg_gen_ext16u_i32(TCGV_LOW(ret), TCGV_LOW(arg)); 1595 tcg_gen_movi_i32(TCGV_HIGH(ret), 0); 1596 } else if (TCG_TARGET_HAS_ext16u_i64) { 1597 tcg_gen_op2_i64(INDEX_op_ext16u_i64, ret, arg); 1598 } else { 1599 tcg_gen_andi_i64(ret, arg, 0xffffu); 1600 } 1601 } 1602 1603 void tcg_gen_ext32u_i64(TCGv_i64 ret, TCGv_i64 arg) 1604 { 1605 if (TCG_TARGET_REG_BITS == 32) { 1606 tcg_gen_mov_i32(TCGV_LOW(ret), TCGV_LOW(arg)); 1607 tcg_gen_movi_i32(TCGV_HIGH(ret), 0); 1608 } else if (TCG_TARGET_HAS_ext32u_i64) { 1609 tcg_gen_op2_i64(INDEX_op_ext32u_i64, ret, arg); 1610 } else { 1611 tcg_gen_andi_i64(ret, arg, 0xffffffffu); 1612 } 1613 } 1614 1615 /* Note: we assume the six high bytes are set to zero */ 1616 void tcg_gen_bswap16_i64(TCGv_i64 ret, TCGv_i64 arg) 1617 { 1618 if (TCG_TARGET_REG_BITS == 32) { 1619 tcg_gen_bswap16_i32(TCGV_LOW(ret), TCGV_LOW(arg)); 1620 tcg_gen_movi_i32(TCGV_HIGH(ret), 0); 1621 } else if (TCG_TARGET_HAS_bswap16_i64) { 1622 tcg_gen_op2_i64(INDEX_op_bswap16_i64, ret, arg); 1623 } else { 1624 TCGv_i64 t0 = tcg_temp_new_i64(); 1625 1626 tcg_gen_ext8u_i64(t0, arg); 1627 tcg_gen_shli_i64(t0, t0, 8); 1628 tcg_gen_shri_i64(ret, arg, 8); 1629 tcg_gen_or_i64(ret, ret, t0); 1630 tcg_temp_free_i64(t0); 1631 } 1632 } 1633 1634 /* Note: we assume the four high bytes are set to zero */ 1635 void tcg_gen_bswap32_i64(TCGv_i64 ret, TCGv_i64 arg) 1636 { 1637 if (TCG_TARGET_REG_BITS == 32) { 1638 tcg_gen_bswap32_i32(TCGV_LOW(ret), TCGV_LOW(arg)); 1639 tcg_gen_movi_i32(TCGV_HIGH(ret), 0); 1640 } else if (TCG_TARGET_HAS_bswap32_i64) { 1641 tcg_gen_op2_i64(INDEX_op_bswap32_i64, ret, arg); 1642 } else { 1643 TCGv_i64 t0 = tcg_temp_new_i64(); 1644 TCGv_i64 t1 = tcg_temp_new_i64(); 1645 TCGv_i64 t2 = tcg_const_i64(0x00ff00ff); 1646 1647 /* arg = ....abcd */ 1648 tcg_gen_shri_i64(t0, arg, 8); /* t0 = .....abc */ 1649 tcg_gen_and_i64(t1, arg, t2); /* t1 = .....b.d */ 1650 tcg_gen_and_i64(t0, t0, t2); /* t0 = .....a.c */ 1651 tcg_gen_shli_i64(t1, t1, 8); /* t1 = ....b.d. */ 1652 tcg_gen_or_i64(ret, t0, t1); /* ret = ....badc */ 1653 1654 tcg_gen_shli_i64(t1, ret, 48); /* t1 = dc...... */ 1655 tcg_gen_shri_i64(t0, ret, 16); /* t0 = ......ba */ 1656 tcg_gen_shri_i64(t1, t1, 32); /* t1 = ....dc.. */ 1657 tcg_gen_or_i64(ret, t0, t1); /* ret = ....dcba */ 1658 1659 tcg_temp_free_i64(t0); 1660 tcg_temp_free_i64(t1); 1661 tcg_temp_free_i64(t2); 1662 } 1663 } 1664 1665 void tcg_gen_bswap64_i64(TCGv_i64 ret, TCGv_i64 arg) 1666 { 1667 if (TCG_TARGET_REG_BITS == 32) { 1668 TCGv_i32 t0, t1; 1669 t0 = tcg_temp_new_i32(); 1670 t1 = tcg_temp_new_i32(); 1671 1672 tcg_gen_bswap32_i32(t0, TCGV_LOW(arg)); 1673 tcg_gen_bswap32_i32(t1, TCGV_HIGH(arg)); 1674 tcg_gen_mov_i32(TCGV_LOW(ret), t1); 1675 tcg_gen_mov_i32(TCGV_HIGH(ret), t0); 1676 tcg_temp_free_i32(t0); 1677 tcg_temp_free_i32(t1); 1678 } else if (TCG_TARGET_HAS_bswap64_i64) { 1679 tcg_gen_op2_i64(INDEX_op_bswap64_i64, ret, arg); 1680 } else { 1681 TCGv_i64 t0 = tcg_temp_new_i64(); 1682 TCGv_i64 t1 = tcg_temp_new_i64(); 1683 TCGv_i64 t2 = tcg_temp_new_i64(); 1684 1685 /* arg = abcdefgh */ 1686 tcg_gen_movi_i64(t2, 0x00ff00ff00ff00ffull); 1687 tcg_gen_shri_i64(t0, arg, 8); /* t0 = .abcdefg */ 1688 tcg_gen_and_i64(t1, arg, t2); /* t1 = .b.d.f.h */ 1689 tcg_gen_and_i64(t0, t0, t2); /* t0 = .a.c.e.g */ 1690 tcg_gen_shli_i64(t1, t1, 8); /* t1 = b.d.f.h. */ 1691 tcg_gen_or_i64(ret, t0, t1); /* ret = badcfehg */ 1692 1693 tcg_gen_movi_i64(t2, 0x0000ffff0000ffffull); 1694 tcg_gen_shri_i64(t0, ret, 16); /* t0 = ..badcfe */ 1695 tcg_gen_and_i64(t1, ret, t2); /* t1 = ..dc..hg */ 1696 tcg_gen_and_i64(t0, t0, t2); /* t0 = ..ba..fe */ 1697 tcg_gen_shli_i64(t1, t1, 16); /* t1 = dc..hg.. */ 1698 tcg_gen_or_i64(ret, t0, t1); /* ret = dcbahgfe */ 1699 1700 tcg_gen_shri_i64(t0, ret, 32); /* t0 = ....dcba */ 1701 tcg_gen_shli_i64(t1, ret, 32); /* t1 = hgfe.... */ 1702 tcg_gen_or_i64(ret, t0, t1); /* ret = hgfedcba */ 1703 1704 tcg_temp_free_i64(t0); 1705 tcg_temp_free_i64(t1); 1706 tcg_temp_free_i64(t2); 1707 } 1708 } 1709 1710 void tcg_gen_not_i64(TCGv_i64 ret, TCGv_i64 arg) 1711 { 1712 if (TCG_TARGET_REG_BITS == 32) { 1713 tcg_gen_not_i32(TCGV_LOW(ret), TCGV_LOW(arg)); 1714 tcg_gen_not_i32(TCGV_HIGH(ret), TCGV_HIGH(arg)); 1715 } else if (TCG_TARGET_HAS_not_i64) { 1716 tcg_gen_op2_i64(INDEX_op_not_i64, ret, arg); 1717 } else { 1718 tcg_gen_xori_i64(ret, arg, -1); 1719 } 1720 } 1721 1722 void tcg_gen_andc_i64(TCGv_i64 ret, TCGv_i64 arg1, TCGv_i64 arg2) 1723 { 1724 if (TCG_TARGET_REG_BITS == 32) { 1725 tcg_gen_andc_i32(TCGV_LOW(ret), TCGV_LOW(arg1), TCGV_LOW(arg2)); 1726 tcg_gen_andc_i32(TCGV_HIGH(ret), TCGV_HIGH(arg1), TCGV_HIGH(arg2)); 1727 } else if (TCG_TARGET_HAS_andc_i64) { 1728 tcg_gen_op3_i64(INDEX_op_andc_i64, ret, arg1, arg2); 1729 } else { 1730 TCGv_i64 t0 = tcg_temp_new_i64(); 1731 tcg_gen_not_i64(t0, arg2); 1732 tcg_gen_and_i64(ret, arg1, t0); 1733 tcg_temp_free_i64(t0); 1734 } 1735 } 1736 1737 void tcg_gen_eqv_i64(TCGv_i64 ret, TCGv_i64 arg1, TCGv_i64 arg2) 1738 { 1739 if (TCG_TARGET_REG_BITS == 32) { 1740 tcg_gen_eqv_i32(TCGV_LOW(ret), TCGV_LOW(arg1), TCGV_LOW(arg2)); 1741 tcg_gen_eqv_i32(TCGV_HIGH(ret), TCGV_HIGH(arg1), TCGV_HIGH(arg2)); 1742 } else if (TCG_TARGET_HAS_eqv_i64) { 1743 tcg_gen_op3_i64(INDEX_op_eqv_i64, ret, arg1, arg2); 1744 } else { 1745 tcg_gen_xor_i64(ret, arg1, arg2); 1746 tcg_gen_not_i64(ret, ret); 1747 } 1748 } 1749 1750 void tcg_gen_nand_i64(TCGv_i64 ret, TCGv_i64 arg1, TCGv_i64 arg2) 1751 { 1752 if (TCG_TARGET_REG_BITS == 32) { 1753 tcg_gen_nand_i32(TCGV_LOW(ret), TCGV_LOW(arg1), TCGV_LOW(arg2)); 1754 tcg_gen_nand_i32(TCGV_HIGH(ret), TCGV_HIGH(arg1), TCGV_HIGH(arg2)); 1755 } else if (TCG_TARGET_HAS_nand_i64) { 1756 tcg_gen_op3_i64(INDEX_op_nand_i64, ret, arg1, arg2); 1757 } else { 1758 tcg_gen_and_i64(ret, arg1, arg2); 1759 tcg_gen_not_i64(ret, ret); 1760 } 1761 } 1762 1763 void tcg_gen_nor_i64(TCGv_i64 ret, TCGv_i64 arg1, TCGv_i64 arg2) 1764 { 1765 if (TCG_TARGET_REG_BITS == 32) { 1766 tcg_gen_nor_i32(TCGV_LOW(ret), TCGV_LOW(arg1), TCGV_LOW(arg2)); 1767 tcg_gen_nor_i32(TCGV_HIGH(ret), TCGV_HIGH(arg1), TCGV_HIGH(arg2)); 1768 } else if (TCG_TARGET_HAS_nor_i64) { 1769 tcg_gen_op3_i64(INDEX_op_nor_i64, ret, arg1, arg2); 1770 } else { 1771 tcg_gen_or_i64(ret, arg1, arg2); 1772 tcg_gen_not_i64(ret, ret); 1773 } 1774 } 1775 1776 void tcg_gen_orc_i64(TCGv_i64 ret, TCGv_i64 arg1, TCGv_i64 arg2) 1777 { 1778 if (TCG_TARGET_REG_BITS == 32) { 1779 tcg_gen_orc_i32(TCGV_LOW(ret), TCGV_LOW(arg1), TCGV_LOW(arg2)); 1780 tcg_gen_orc_i32(TCGV_HIGH(ret), TCGV_HIGH(arg1), TCGV_HIGH(arg2)); 1781 } else if (TCG_TARGET_HAS_orc_i64) { 1782 tcg_gen_op3_i64(INDEX_op_orc_i64, ret, arg1, arg2); 1783 } else { 1784 TCGv_i64 t0 = tcg_temp_new_i64(); 1785 tcg_gen_not_i64(t0, arg2); 1786 tcg_gen_or_i64(ret, arg1, t0); 1787 tcg_temp_free_i64(t0); 1788 } 1789 } 1790 1791 void tcg_gen_clz_i64(TCGv_i64 ret, TCGv_i64 arg1, TCGv_i64 arg2) 1792 { 1793 if (TCG_TARGET_HAS_clz_i64) { 1794 tcg_gen_op3_i64(INDEX_op_clz_i64, ret, arg1, arg2); 1795 } else { 1796 gen_helper_clz_i64(ret, arg1, arg2); 1797 } 1798 } 1799 1800 void tcg_gen_clzi_i64(TCGv_i64 ret, TCGv_i64 arg1, uint64_t arg2) 1801 { 1802 if (TCG_TARGET_REG_BITS == 32 1803 && TCG_TARGET_HAS_clz_i32 1804 && arg2 <= 0xffffffffu) { 1805 TCGv_i32 t = tcg_const_i32((uint32_t)arg2 - 32); 1806 tcg_gen_clz_i32(t, TCGV_LOW(arg1), t); 1807 tcg_gen_addi_i32(t, t, 32); 1808 tcg_gen_clz_i32(TCGV_LOW(ret), TCGV_HIGH(arg1), t); 1809 tcg_gen_movi_i32(TCGV_HIGH(ret), 0); 1810 tcg_temp_free_i32(t); 1811 } else { 1812 TCGv_i64 t = tcg_const_i64(arg2); 1813 tcg_gen_clz_i64(ret, arg1, t); 1814 tcg_temp_free_i64(t); 1815 } 1816 } 1817 1818 void tcg_gen_ctz_i64(TCGv_i64 ret, TCGv_i64 arg1, TCGv_i64 arg2) 1819 { 1820 if (TCG_TARGET_HAS_ctz_i64) { 1821 tcg_gen_op3_i64(INDEX_op_ctz_i64, ret, arg1, arg2); 1822 } else if (TCG_TARGET_HAS_ctpop_i64 || TCG_TARGET_HAS_clz_i64) { 1823 TCGv_i64 z, t = tcg_temp_new_i64(); 1824 1825 if (TCG_TARGET_HAS_ctpop_i64) { 1826 tcg_gen_subi_i64(t, arg1, 1); 1827 tcg_gen_andc_i64(t, t, arg1); 1828 tcg_gen_ctpop_i64(t, t); 1829 } else { 1830 /* Since all non-x86 hosts have clz(0) == 64, don't fight it. */ 1831 tcg_gen_neg_i64(t, arg1); 1832 tcg_gen_and_i64(t, t, arg1); 1833 tcg_gen_clzi_i64(t, t, 64); 1834 tcg_gen_xori_i64(t, t, 63); 1835 } 1836 z = tcg_const_i64(0); 1837 tcg_gen_movcond_i64(TCG_COND_EQ, ret, arg1, z, arg2, t); 1838 tcg_temp_free_i64(t); 1839 tcg_temp_free_i64(z); 1840 } else { 1841 gen_helper_ctz_i64(ret, arg1, arg2); 1842 } 1843 } 1844 1845 void tcg_gen_ctzi_i64(TCGv_i64 ret, TCGv_i64 arg1, uint64_t arg2) 1846 { 1847 if (TCG_TARGET_REG_BITS == 32 1848 && TCG_TARGET_HAS_ctz_i32 1849 && arg2 <= 0xffffffffu) { 1850 TCGv_i32 t32 = tcg_const_i32((uint32_t)arg2 - 32); 1851 tcg_gen_ctz_i32(t32, TCGV_HIGH(arg1), t32); 1852 tcg_gen_addi_i32(t32, t32, 32); 1853 tcg_gen_ctz_i32(TCGV_LOW(ret), TCGV_LOW(arg1), t32); 1854 tcg_gen_movi_i32(TCGV_HIGH(ret), 0); 1855 tcg_temp_free_i32(t32); 1856 } else if (!TCG_TARGET_HAS_ctz_i64 1857 && TCG_TARGET_HAS_ctpop_i64 1858 && arg2 == 64) { 1859 /* This equivalence has the advantage of not requiring a fixup. */ 1860 TCGv_i64 t = tcg_temp_new_i64(); 1861 tcg_gen_subi_i64(t, arg1, 1); 1862 tcg_gen_andc_i64(t, t, arg1); 1863 tcg_gen_ctpop_i64(ret, t); 1864 tcg_temp_free_i64(t); 1865 } else { 1866 TCGv_i64 t64 = tcg_const_i64(arg2); 1867 tcg_gen_ctz_i64(ret, arg1, t64); 1868 tcg_temp_free_i64(t64); 1869 } 1870 } 1871 1872 void tcg_gen_clrsb_i64(TCGv_i64 ret, TCGv_i64 arg) 1873 { 1874 if (TCG_TARGET_HAS_clz_i64 || TCG_TARGET_HAS_clz_i32) { 1875 TCGv_i64 t = tcg_temp_new_i64(); 1876 tcg_gen_sari_i64(t, arg, 63); 1877 tcg_gen_xor_i64(t, t, arg); 1878 tcg_gen_clzi_i64(t, t, 64); 1879 tcg_gen_subi_i64(ret, t, 1); 1880 tcg_temp_free_i64(t); 1881 } else { 1882 gen_helper_clrsb_i64(ret, arg); 1883 } 1884 } 1885 1886 void tcg_gen_ctpop_i64(TCGv_i64 ret, TCGv_i64 arg1) 1887 { 1888 if (TCG_TARGET_HAS_ctpop_i64) { 1889 tcg_gen_op2_i64(INDEX_op_ctpop_i64, ret, arg1); 1890 } else if (TCG_TARGET_REG_BITS == 32 && TCG_TARGET_HAS_ctpop_i32) { 1891 tcg_gen_ctpop_i32(TCGV_HIGH(ret), TCGV_HIGH(arg1)); 1892 tcg_gen_ctpop_i32(TCGV_LOW(ret), TCGV_LOW(arg1)); 1893 tcg_gen_add_i32(TCGV_LOW(ret), TCGV_LOW(ret), TCGV_HIGH(ret)); 1894 tcg_gen_movi_i32(TCGV_HIGH(ret), 0); 1895 } else { 1896 gen_helper_ctpop_i64(ret, arg1); 1897 } 1898 } 1899 1900 void tcg_gen_rotl_i64(TCGv_i64 ret, TCGv_i64 arg1, TCGv_i64 arg2) 1901 { 1902 if (TCG_TARGET_HAS_rot_i64) { 1903 tcg_gen_op3_i64(INDEX_op_rotl_i64, ret, arg1, arg2); 1904 } else { 1905 TCGv_i64 t0, t1; 1906 t0 = tcg_temp_new_i64(); 1907 t1 = tcg_temp_new_i64(); 1908 tcg_gen_shl_i64(t0, arg1, arg2); 1909 tcg_gen_subfi_i64(t1, 64, arg2); 1910 tcg_gen_shr_i64(t1, arg1, t1); 1911 tcg_gen_or_i64(ret, t0, t1); 1912 tcg_temp_free_i64(t0); 1913 tcg_temp_free_i64(t1); 1914 } 1915 } 1916 1917 void tcg_gen_rotli_i64(TCGv_i64 ret, TCGv_i64 arg1, unsigned arg2) 1918 { 1919 tcg_debug_assert(arg2 < 64); 1920 /* some cases can be optimized here */ 1921 if (arg2 == 0) { 1922 tcg_gen_mov_i64(ret, arg1); 1923 } else if (TCG_TARGET_HAS_rot_i64) { 1924 TCGv_i64 t0 = tcg_const_i64(arg2); 1925 tcg_gen_rotl_i64(ret, arg1, t0); 1926 tcg_temp_free_i64(t0); 1927 } else { 1928 TCGv_i64 t0, t1; 1929 t0 = tcg_temp_new_i64(); 1930 t1 = tcg_temp_new_i64(); 1931 tcg_gen_shli_i64(t0, arg1, arg2); 1932 tcg_gen_shri_i64(t1, arg1, 64 - arg2); 1933 tcg_gen_or_i64(ret, t0, t1); 1934 tcg_temp_free_i64(t0); 1935 tcg_temp_free_i64(t1); 1936 } 1937 } 1938 1939 void tcg_gen_rotr_i64(TCGv_i64 ret, TCGv_i64 arg1, TCGv_i64 arg2) 1940 { 1941 if (TCG_TARGET_HAS_rot_i64) { 1942 tcg_gen_op3_i64(INDEX_op_rotr_i64, ret, arg1, arg2); 1943 } else { 1944 TCGv_i64 t0, t1; 1945 t0 = tcg_temp_new_i64(); 1946 t1 = tcg_temp_new_i64(); 1947 tcg_gen_shr_i64(t0, arg1, arg2); 1948 tcg_gen_subfi_i64(t1, 64, arg2); 1949 tcg_gen_shl_i64(t1, arg1, t1); 1950 tcg_gen_or_i64(ret, t0, t1); 1951 tcg_temp_free_i64(t0); 1952 tcg_temp_free_i64(t1); 1953 } 1954 } 1955 1956 void tcg_gen_rotri_i64(TCGv_i64 ret, TCGv_i64 arg1, unsigned arg2) 1957 { 1958 tcg_debug_assert(arg2 < 64); 1959 /* some cases can be optimized here */ 1960 if (arg2 == 0) { 1961 tcg_gen_mov_i64(ret, arg1); 1962 } else { 1963 tcg_gen_rotli_i64(ret, arg1, 64 - arg2); 1964 } 1965 } 1966 1967 void tcg_gen_deposit_i64(TCGv_i64 ret, TCGv_i64 arg1, TCGv_i64 arg2, 1968 unsigned int ofs, unsigned int len) 1969 { 1970 uint64_t mask; 1971 TCGv_i64 t1; 1972 1973 tcg_debug_assert(ofs < 64); 1974 tcg_debug_assert(len > 0); 1975 tcg_debug_assert(len <= 64); 1976 tcg_debug_assert(ofs + len <= 64); 1977 1978 if (len == 64) { 1979 tcg_gen_mov_i64(ret, arg2); 1980 return; 1981 } 1982 if (TCG_TARGET_HAS_deposit_i64 && TCG_TARGET_deposit_i64_valid(ofs, len)) { 1983 tcg_gen_op5ii_i64(INDEX_op_deposit_i64, ret, arg1, arg2, ofs, len); 1984 return; 1985 } 1986 1987 if (TCG_TARGET_REG_BITS == 32) { 1988 if (ofs >= 32) { 1989 tcg_gen_deposit_i32(TCGV_HIGH(ret), TCGV_HIGH(arg1), 1990 TCGV_LOW(arg2), ofs - 32, len); 1991 tcg_gen_mov_i32(TCGV_LOW(ret), TCGV_LOW(arg1)); 1992 return; 1993 } 1994 if (ofs + len <= 32) { 1995 tcg_gen_deposit_i32(TCGV_LOW(ret), TCGV_LOW(arg1), 1996 TCGV_LOW(arg2), ofs, len); 1997 tcg_gen_mov_i32(TCGV_HIGH(ret), TCGV_HIGH(arg1)); 1998 return; 1999 } 2000 } 2001 2002 mask = (1ull << len) - 1; 2003 t1 = tcg_temp_new_i64(); 2004 2005 if (ofs + len < 64) { 2006 tcg_gen_andi_i64(t1, arg2, mask); 2007 tcg_gen_shli_i64(t1, t1, ofs); 2008 } else { 2009 tcg_gen_shli_i64(t1, arg2, ofs); 2010 } 2011 tcg_gen_andi_i64(ret, arg1, ~(mask << ofs)); 2012 tcg_gen_or_i64(ret, ret, t1); 2013 2014 tcg_temp_free_i64(t1); 2015 } 2016 2017 void tcg_gen_deposit_z_i64(TCGv_i64 ret, TCGv_i64 arg, 2018 unsigned int ofs, unsigned int len) 2019 { 2020 tcg_debug_assert(ofs < 64); 2021 tcg_debug_assert(len > 0); 2022 tcg_debug_assert(len <= 64); 2023 tcg_debug_assert(ofs + len <= 64); 2024 2025 if (ofs + len == 64) { 2026 tcg_gen_shli_i64(ret, arg, ofs); 2027 } else if (ofs == 0) { 2028 tcg_gen_andi_i64(ret, arg, (1ull << len) - 1); 2029 } else if (TCG_TARGET_HAS_deposit_i64 2030 && TCG_TARGET_deposit_i64_valid(ofs, len)) { 2031 TCGv_i64 zero = tcg_const_i64(0); 2032 tcg_gen_op5ii_i64(INDEX_op_deposit_i64, ret, zero, arg, ofs, len); 2033 tcg_temp_free_i64(zero); 2034 } else { 2035 if (TCG_TARGET_REG_BITS == 32) { 2036 if (ofs >= 32) { 2037 tcg_gen_deposit_z_i32(TCGV_HIGH(ret), TCGV_LOW(arg), 2038 ofs - 32, len); 2039 tcg_gen_movi_i32(TCGV_LOW(ret), 0); 2040 return; 2041 } 2042 if (ofs + len <= 32) { 2043 tcg_gen_deposit_z_i32(TCGV_LOW(ret), TCGV_LOW(arg), ofs, len); 2044 tcg_gen_movi_i32(TCGV_HIGH(ret), 0); 2045 return; 2046 } 2047 } 2048 /* To help two-operand hosts we prefer to zero-extend first, 2049 which allows ARG to stay live. */ 2050 switch (len) { 2051 case 32: 2052 if (TCG_TARGET_HAS_ext32u_i64) { 2053 tcg_gen_ext32u_i64(ret, arg); 2054 tcg_gen_shli_i64(ret, ret, ofs); 2055 return; 2056 } 2057 break; 2058 case 16: 2059 if (TCG_TARGET_HAS_ext16u_i64) { 2060 tcg_gen_ext16u_i64(ret, arg); 2061 tcg_gen_shli_i64(ret, ret, ofs); 2062 return; 2063 } 2064 break; 2065 case 8: 2066 if (TCG_TARGET_HAS_ext8u_i64) { 2067 tcg_gen_ext8u_i64(ret, arg); 2068 tcg_gen_shli_i64(ret, ret, ofs); 2069 return; 2070 } 2071 break; 2072 } 2073 /* Otherwise prefer zero-extension over AND for code size. */ 2074 switch (ofs + len) { 2075 case 32: 2076 if (TCG_TARGET_HAS_ext32u_i64) { 2077 tcg_gen_shli_i64(ret, arg, ofs); 2078 tcg_gen_ext32u_i64(ret, ret); 2079 return; 2080 } 2081 break; 2082 case 16: 2083 if (TCG_TARGET_HAS_ext16u_i64) { 2084 tcg_gen_shli_i64(ret, arg, ofs); 2085 tcg_gen_ext16u_i64(ret, ret); 2086 return; 2087 } 2088 break; 2089 case 8: 2090 if (TCG_TARGET_HAS_ext8u_i64) { 2091 tcg_gen_shli_i64(ret, arg, ofs); 2092 tcg_gen_ext8u_i64(ret, ret); 2093 return; 2094 } 2095 break; 2096 } 2097 tcg_gen_andi_i64(ret, arg, (1ull << len) - 1); 2098 tcg_gen_shli_i64(ret, ret, ofs); 2099 } 2100 } 2101 2102 void tcg_gen_extract_i64(TCGv_i64 ret, TCGv_i64 arg, 2103 unsigned int ofs, unsigned int len) 2104 { 2105 tcg_debug_assert(ofs < 64); 2106 tcg_debug_assert(len > 0); 2107 tcg_debug_assert(len <= 64); 2108 tcg_debug_assert(ofs + len <= 64); 2109 2110 /* Canonicalize certain special cases, even if extract is supported. */ 2111 if (ofs + len == 64) { 2112 tcg_gen_shri_i64(ret, arg, 64 - len); 2113 return; 2114 } 2115 if (ofs == 0) { 2116 tcg_gen_andi_i64(ret, arg, (1ull << len) - 1); 2117 return; 2118 } 2119 2120 if (TCG_TARGET_REG_BITS == 32) { 2121 /* Look for a 32-bit extract within one of the two words. */ 2122 if (ofs >= 32) { 2123 tcg_gen_extract_i32(TCGV_LOW(ret), TCGV_HIGH(arg), ofs - 32, len); 2124 tcg_gen_movi_i32(TCGV_HIGH(ret), 0); 2125 return; 2126 } 2127 if (ofs + len <= 32) { 2128 tcg_gen_extract_i32(TCGV_LOW(ret), TCGV_LOW(arg), ofs, len); 2129 tcg_gen_movi_i32(TCGV_HIGH(ret), 0); 2130 return; 2131 } 2132 /* The field is split across two words. One double-word 2133 shift is better than two double-word shifts. */ 2134 goto do_shift_and; 2135 } 2136 2137 if (TCG_TARGET_HAS_extract_i64 2138 && TCG_TARGET_extract_i64_valid(ofs, len)) { 2139 tcg_gen_op4ii_i64(INDEX_op_extract_i64, ret, arg, ofs, len); 2140 return; 2141 } 2142 2143 /* Assume that zero-extension, if available, is cheaper than a shift. */ 2144 switch (ofs + len) { 2145 case 32: 2146 if (TCG_TARGET_HAS_ext32u_i64) { 2147 tcg_gen_ext32u_i64(ret, arg); 2148 tcg_gen_shri_i64(ret, ret, ofs); 2149 return; 2150 } 2151 break; 2152 case 16: 2153 if (TCG_TARGET_HAS_ext16u_i64) { 2154 tcg_gen_ext16u_i64(ret, arg); 2155 tcg_gen_shri_i64(ret, ret, ofs); 2156 return; 2157 } 2158 break; 2159 case 8: 2160 if (TCG_TARGET_HAS_ext8u_i64) { 2161 tcg_gen_ext8u_i64(ret, arg); 2162 tcg_gen_shri_i64(ret, ret, ofs); 2163 return; 2164 } 2165 break; 2166 } 2167 2168 /* ??? Ideally we'd know what values are available for immediate AND. 2169 Assume that 8 bits are available, plus the special cases of 16 and 32, 2170 so that we get ext8u, ext16u, and ext32u. */ 2171 switch (len) { 2172 case 1 ... 8: case 16: case 32: 2173 do_shift_and: 2174 tcg_gen_shri_i64(ret, arg, ofs); 2175 tcg_gen_andi_i64(ret, ret, (1ull << len) - 1); 2176 break; 2177 default: 2178 tcg_gen_shli_i64(ret, arg, 64 - len - ofs); 2179 tcg_gen_shri_i64(ret, ret, 64 - len); 2180 break; 2181 } 2182 } 2183 2184 void tcg_gen_sextract_i64(TCGv_i64 ret, TCGv_i64 arg, 2185 unsigned int ofs, unsigned int len) 2186 { 2187 tcg_debug_assert(ofs < 64); 2188 tcg_debug_assert(len > 0); 2189 tcg_debug_assert(len <= 64); 2190 tcg_debug_assert(ofs + len <= 64); 2191 2192 /* Canonicalize certain special cases, even if sextract is supported. */ 2193 if (ofs + len == 64) { 2194 tcg_gen_sari_i64(ret, arg, 64 - len); 2195 return; 2196 } 2197 if (ofs == 0) { 2198 switch (len) { 2199 case 32: 2200 tcg_gen_ext32s_i64(ret, arg); 2201 return; 2202 case 16: 2203 tcg_gen_ext16s_i64(ret, arg); 2204 return; 2205 case 8: 2206 tcg_gen_ext8s_i64(ret, arg); 2207 return; 2208 } 2209 } 2210 2211 if (TCG_TARGET_REG_BITS == 32) { 2212 /* Look for a 32-bit extract within one of the two words. */ 2213 if (ofs >= 32) { 2214 tcg_gen_sextract_i32(TCGV_LOW(ret), TCGV_HIGH(arg), ofs - 32, len); 2215 } else if (ofs + len <= 32) { 2216 tcg_gen_sextract_i32(TCGV_LOW(ret), TCGV_LOW(arg), ofs, len); 2217 } else if (ofs == 0) { 2218 tcg_gen_mov_i32(TCGV_LOW(ret), TCGV_LOW(arg)); 2219 tcg_gen_sextract_i32(TCGV_HIGH(ret), TCGV_HIGH(arg), 0, len - 32); 2220 return; 2221 } else if (len > 32) { 2222 TCGv_i32 t = tcg_temp_new_i32(); 2223 /* Extract the bits for the high word normally. */ 2224 tcg_gen_sextract_i32(t, TCGV_HIGH(arg), ofs + 32, len - 32); 2225 /* Shift the field down for the low part. */ 2226 tcg_gen_shri_i64(ret, arg, ofs); 2227 /* Overwrite the shift into the high part. */ 2228 tcg_gen_mov_i32(TCGV_HIGH(ret), t); 2229 tcg_temp_free_i32(t); 2230 return; 2231 } else { 2232 /* Shift the field down for the low part, such that the 2233 field sits at the MSB. */ 2234 tcg_gen_shri_i64(ret, arg, ofs + len - 32); 2235 /* Shift the field down from the MSB, sign extending. */ 2236 tcg_gen_sari_i32(TCGV_LOW(ret), TCGV_LOW(ret), 32 - len); 2237 } 2238 /* Sign-extend the field from 32 bits. */ 2239 tcg_gen_sari_i32(TCGV_HIGH(ret), TCGV_LOW(ret), 31); 2240 return; 2241 } 2242 2243 if (TCG_TARGET_HAS_sextract_i64 2244 && TCG_TARGET_extract_i64_valid(ofs, len)) { 2245 tcg_gen_op4ii_i64(INDEX_op_sextract_i64, ret, arg, ofs, len); 2246 return; 2247 } 2248 2249 /* Assume that sign-extension, if available, is cheaper than a shift. */ 2250 switch (ofs + len) { 2251 case 32: 2252 if (TCG_TARGET_HAS_ext32s_i64) { 2253 tcg_gen_ext32s_i64(ret, arg); 2254 tcg_gen_sari_i64(ret, ret, ofs); 2255 return; 2256 } 2257 break; 2258 case 16: 2259 if (TCG_TARGET_HAS_ext16s_i64) { 2260 tcg_gen_ext16s_i64(ret, arg); 2261 tcg_gen_sari_i64(ret, ret, ofs); 2262 return; 2263 } 2264 break; 2265 case 8: 2266 if (TCG_TARGET_HAS_ext8s_i64) { 2267 tcg_gen_ext8s_i64(ret, arg); 2268 tcg_gen_sari_i64(ret, ret, ofs); 2269 return; 2270 } 2271 break; 2272 } 2273 switch (len) { 2274 case 32: 2275 if (TCG_TARGET_HAS_ext32s_i64) { 2276 tcg_gen_shri_i64(ret, arg, ofs); 2277 tcg_gen_ext32s_i64(ret, ret); 2278 return; 2279 } 2280 break; 2281 case 16: 2282 if (TCG_TARGET_HAS_ext16s_i64) { 2283 tcg_gen_shri_i64(ret, arg, ofs); 2284 tcg_gen_ext16s_i64(ret, ret); 2285 return; 2286 } 2287 break; 2288 case 8: 2289 if (TCG_TARGET_HAS_ext8s_i64) { 2290 tcg_gen_shri_i64(ret, arg, ofs); 2291 tcg_gen_ext8s_i64(ret, ret); 2292 return; 2293 } 2294 break; 2295 } 2296 tcg_gen_shli_i64(ret, arg, 64 - len - ofs); 2297 tcg_gen_sari_i64(ret, ret, 64 - len); 2298 } 2299 2300 void tcg_gen_movcond_i64(TCGCond cond, TCGv_i64 ret, TCGv_i64 c1, 2301 TCGv_i64 c2, TCGv_i64 v1, TCGv_i64 v2) 2302 { 2303 if (cond == TCG_COND_ALWAYS) { 2304 tcg_gen_mov_i64(ret, v1); 2305 } else if (cond == TCG_COND_NEVER) { 2306 tcg_gen_mov_i64(ret, v2); 2307 } else if (TCG_TARGET_REG_BITS == 32) { 2308 TCGv_i32 t0 = tcg_temp_new_i32(); 2309 TCGv_i32 t1 = tcg_temp_new_i32(); 2310 tcg_gen_op6i_i32(INDEX_op_setcond2_i32, t0, 2311 TCGV_LOW(c1), TCGV_HIGH(c1), 2312 TCGV_LOW(c2), TCGV_HIGH(c2), cond); 2313 2314 if (TCG_TARGET_HAS_movcond_i32) { 2315 tcg_gen_movi_i32(t1, 0); 2316 tcg_gen_movcond_i32(TCG_COND_NE, TCGV_LOW(ret), t0, t1, 2317 TCGV_LOW(v1), TCGV_LOW(v2)); 2318 tcg_gen_movcond_i32(TCG_COND_NE, TCGV_HIGH(ret), t0, t1, 2319 TCGV_HIGH(v1), TCGV_HIGH(v2)); 2320 } else { 2321 tcg_gen_neg_i32(t0, t0); 2322 2323 tcg_gen_and_i32(t1, TCGV_LOW(v1), t0); 2324 tcg_gen_andc_i32(TCGV_LOW(ret), TCGV_LOW(v2), t0); 2325 tcg_gen_or_i32(TCGV_LOW(ret), TCGV_LOW(ret), t1); 2326 2327 tcg_gen_and_i32(t1, TCGV_HIGH(v1), t0); 2328 tcg_gen_andc_i32(TCGV_HIGH(ret), TCGV_HIGH(v2), t0); 2329 tcg_gen_or_i32(TCGV_HIGH(ret), TCGV_HIGH(ret), t1); 2330 } 2331 tcg_temp_free_i32(t0); 2332 tcg_temp_free_i32(t1); 2333 } else if (TCG_TARGET_HAS_movcond_i64) { 2334 tcg_gen_op6i_i64(INDEX_op_movcond_i64, ret, c1, c2, v1, v2, cond); 2335 } else { 2336 TCGv_i64 t0 = tcg_temp_new_i64(); 2337 TCGv_i64 t1 = tcg_temp_new_i64(); 2338 tcg_gen_setcond_i64(cond, t0, c1, c2); 2339 tcg_gen_neg_i64(t0, t0); 2340 tcg_gen_and_i64(t1, v1, t0); 2341 tcg_gen_andc_i64(ret, v2, t0); 2342 tcg_gen_or_i64(ret, ret, t1); 2343 tcg_temp_free_i64(t0); 2344 tcg_temp_free_i64(t1); 2345 } 2346 } 2347 2348 void tcg_gen_add2_i64(TCGv_i64 rl, TCGv_i64 rh, TCGv_i64 al, 2349 TCGv_i64 ah, TCGv_i64 bl, TCGv_i64 bh) 2350 { 2351 if (TCG_TARGET_HAS_add2_i64) { 2352 tcg_gen_op6_i64(INDEX_op_add2_i64, rl, rh, al, ah, bl, bh); 2353 } else { 2354 TCGv_i64 t0 = tcg_temp_new_i64(); 2355 TCGv_i64 t1 = tcg_temp_new_i64(); 2356 tcg_gen_add_i64(t0, al, bl); 2357 tcg_gen_setcond_i64(TCG_COND_LTU, t1, t0, al); 2358 tcg_gen_add_i64(rh, ah, bh); 2359 tcg_gen_add_i64(rh, rh, t1); 2360 tcg_gen_mov_i64(rl, t0); 2361 tcg_temp_free_i64(t0); 2362 tcg_temp_free_i64(t1); 2363 } 2364 } 2365 2366 void tcg_gen_sub2_i64(TCGv_i64 rl, TCGv_i64 rh, TCGv_i64 al, 2367 TCGv_i64 ah, TCGv_i64 bl, TCGv_i64 bh) 2368 { 2369 if (TCG_TARGET_HAS_sub2_i64) { 2370 tcg_gen_op6_i64(INDEX_op_sub2_i64, rl, rh, al, ah, bl, bh); 2371 } else { 2372 TCGv_i64 t0 = tcg_temp_new_i64(); 2373 TCGv_i64 t1 = tcg_temp_new_i64(); 2374 tcg_gen_sub_i64(t0, al, bl); 2375 tcg_gen_setcond_i64(TCG_COND_LTU, t1, al, bl); 2376 tcg_gen_sub_i64(rh, ah, bh); 2377 tcg_gen_sub_i64(rh, rh, t1); 2378 tcg_gen_mov_i64(rl, t0); 2379 tcg_temp_free_i64(t0); 2380 tcg_temp_free_i64(t1); 2381 } 2382 } 2383 2384 void tcg_gen_mulu2_i64(TCGv_i64 rl, TCGv_i64 rh, TCGv_i64 arg1, TCGv_i64 arg2) 2385 { 2386 if (TCG_TARGET_HAS_mulu2_i64) { 2387 tcg_gen_op4_i64(INDEX_op_mulu2_i64, rl, rh, arg1, arg2); 2388 } else if (TCG_TARGET_HAS_muluh_i64) { 2389 TCGv_i64 t = tcg_temp_new_i64(); 2390 tcg_gen_op3_i64(INDEX_op_mul_i64, t, arg1, arg2); 2391 tcg_gen_op3_i64(INDEX_op_muluh_i64, rh, arg1, arg2); 2392 tcg_gen_mov_i64(rl, t); 2393 tcg_temp_free_i64(t); 2394 } else { 2395 TCGv_i64 t0 = tcg_temp_new_i64(); 2396 tcg_gen_mul_i64(t0, arg1, arg2); 2397 gen_helper_muluh_i64(rh, arg1, arg2); 2398 tcg_gen_mov_i64(rl, t0); 2399 tcg_temp_free_i64(t0); 2400 } 2401 } 2402 2403 void tcg_gen_muls2_i64(TCGv_i64 rl, TCGv_i64 rh, TCGv_i64 arg1, TCGv_i64 arg2) 2404 { 2405 if (TCG_TARGET_HAS_muls2_i64) { 2406 tcg_gen_op4_i64(INDEX_op_muls2_i64, rl, rh, arg1, arg2); 2407 } else if (TCG_TARGET_HAS_mulsh_i64) { 2408 TCGv_i64 t = tcg_temp_new_i64(); 2409 tcg_gen_op3_i64(INDEX_op_mul_i64, t, arg1, arg2); 2410 tcg_gen_op3_i64(INDEX_op_mulsh_i64, rh, arg1, arg2); 2411 tcg_gen_mov_i64(rl, t); 2412 tcg_temp_free_i64(t); 2413 } else if (TCG_TARGET_HAS_mulu2_i64 || TCG_TARGET_HAS_muluh_i64) { 2414 TCGv_i64 t0 = tcg_temp_new_i64(); 2415 TCGv_i64 t1 = tcg_temp_new_i64(); 2416 TCGv_i64 t2 = tcg_temp_new_i64(); 2417 TCGv_i64 t3 = tcg_temp_new_i64(); 2418 tcg_gen_mulu2_i64(t0, t1, arg1, arg2); 2419 /* Adjust for negative inputs. */ 2420 tcg_gen_sari_i64(t2, arg1, 63); 2421 tcg_gen_sari_i64(t3, arg2, 63); 2422 tcg_gen_and_i64(t2, t2, arg2); 2423 tcg_gen_and_i64(t3, t3, arg1); 2424 tcg_gen_sub_i64(rh, t1, t2); 2425 tcg_gen_sub_i64(rh, rh, t3); 2426 tcg_gen_mov_i64(rl, t0); 2427 tcg_temp_free_i64(t0); 2428 tcg_temp_free_i64(t1); 2429 tcg_temp_free_i64(t2); 2430 tcg_temp_free_i64(t3); 2431 } else { 2432 TCGv_i64 t0 = tcg_temp_new_i64(); 2433 tcg_gen_mul_i64(t0, arg1, arg2); 2434 gen_helper_mulsh_i64(rh, arg1, arg2); 2435 tcg_gen_mov_i64(rl, t0); 2436 tcg_temp_free_i64(t0); 2437 } 2438 } 2439 2440 void tcg_gen_mulsu2_i64(TCGv_i64 rl, TCGv_i64 rh, TCGv_i64 arg1, TCGv_i64 arg2) 2441 { 2442 TCGv_i64 t0 = tcg_temp_new_i64(); 2443 TCGv_i64 t1 = tcg_temp_new_i64(); 2444 TCGv_i64 t2 = tcg_temp_new_i64(); 2445 tcg_gen_mulu2_i64(t0, t1, arg1, arg2); 2446 /* Adjust for negative input for the signed arg1. */ 2447 tcg_gen_sari_i64(t2, arg1, 63); 2448 tcg_gen_and_i64(t2, t2, arg2); 2449 tcg_gen_sub_i64(rh, t1, t2); 2450 tcg_gen_mov_i64(rl, t0); 2451 tcg_temp_free_i64(t0); 2452 tcg_temp_free_i64(t1); 2453 tcg_temp_free_i64(t2); 2454 } 2455 2456 void tcg_gen_smin_i64(TCGv_i64 ret, TCGv_i64 a, TCGv_i64 b) 2457 { 2458 tcg_gen_movcond_i64(TCG_COND_LT, ret, a, b, a, b); 2459 } 2460 2461 void tcg_gen_umin_i64(TCGv_i64 ret, TCGv_i64 a, TCGv_i64 b) 2462 { 2463 tcg_gen_movcond_i64(TCG_COND_LTU, ret, a, b, a, b); 2464 } 2465 2466 void tcg_gen_smax_i64(TCGv_i64 ret, TCGv_i64 a, TCGv_i64 b) 2467 { 2468 tcg_gen_movcond_i64(TCG_COND_LT, ret, a, b, b, a); 2469 } 2470 2471 void tcg_gen_umax_i64(TCGv_i64 ret, TCGv_i64 a, TCGv_i64 b) 2472 { 2473 tcg_gen_movcond_i64(TCG_COND_LTU, ret, a, b, b, a); 2474 } 2475 2476 /* Size changing operations. */ 2477 2478 void tcg_gen_extrl_i64_i32(TCGv_i32 ret, TCGv_i64 arg) 2479 { 2480 if (TCG_TARGET_REG_BITS == 32) { 2481 tcg_gen_mov_i32(ret, TCGV_LOW(arg)); 2482 } else if (TCG_TARGET_HAS_extrl_i64_i32) { 2483 tcg_gen_op2(INDEX_op_extrl_i64_i32, 2484 tcgv_i32_arg(ret), tcgv_i64_arg(arg)); 2485 } else { 2486 tcg_gen_mov_i32(ret, (TCGv_i32)arg); 2487 } 2488 } 2489 2490 void tcg_gen_extrh_i64_i32(TCGv_i32 ret, TCGv_i64 arg) 2491 { 2492 if (TCG_TARGET_REG_BITS == 32) { 2493 tcg_gen_mov_i32(ret, TCGV_HIGH(arg)); 2494 } else if (TCG_TARGET_HAS_extrh_i64_i32) { 2495 tcg_gen_op2(INDEX_op_extrh_i64_i32, 2496 tcgv_i32_arg(ret), tcgv_i64_arg(arg)); 2497 } else { 2498 TCGv_i64 t = tcg_temp_new_i64(); 2499 tcg_gen_shri_i64(t, arg, 32); 2500 tcg_gen_mov_i32(ret, (TCGv_i32)t); 2501 tcg_temp_free_i64(t); 2502 } 2503 } 2504 2505 void tcg_gen_extu_i32_i64(TCGv_i64 ret, TCGv_i32 arg) 2506 { 2507 if (TCG_TARGET_REG_BITS == 32) { 2508 tcg_gen_mov_i32(TCGV_LOW(ret), arg); 2509 tcg_gen_movi_i32(TCGV_HIGH(ret), 0); 2510 } else { 2511 tcg_gen_op2(INDEX_op_extu_i32_i64, 2512 tcgv_i64_arg(ret), tcgv_i32_arg(arg)); 2513 } 2514 } 2515 2516 void tcg_gen_ext_i32_i64(TCGv_i64 ret, TCGv_i32 arg) 2517 { 2518 if (TCG_TARGET_REG_BITS == 32) { 2519 tcg_gen_mov_i32(TCGV_LOW(ret), arg); 2520 tcg_gen_sari_i32(TCGV_HIGH(ret), TCGV_LOW(ret), 31); 2521 } else { 2522 tcg_gen_op2(INDEX_op_ext_i32_i64, 2523 tcgv_i64_arg(ret), tcgv_i32_arg(arg)); 2524 } 2525 } 2526 2527 void tcg_gen_concat_i32_i64(TCGv_i64 dest, TCGv_i32 low, TCGv_i32 high) 2528 { 2529 TCGv_i64 tmp; 2530 2531 if (TCG_TARGET_REG_BITS == 32) { 2532 tcg_gen_mov_i32(TCGV_LOW(dest), low); 2533 tcg_gen_mov_i32(TCGV_HIGH(dest), high); 2534 return; 2535 } 2536 2537 tmp = tcg_temp_new_i64(); 2538 /* These extensions are only needed for type correctness. 2539 We may be able to do better given target specific information. */ 2540 tcg_gen_extu_i32_i64(tmp, high); 2541 tcg_gen_extu_i32_i64(dest, low); 2542 /* If deposit is available, use it. Otherwise use the extra 2543 knowledge that we have of the zero-extensions above. */ 2544 if (TCG_TARGET_HAS_deposit_i64 && TCG_TARGET_deposit_i64_valid(32, 32)) { 2545 tcg_gen_deposit_i64(dest, dest, tmp, 32, 32); 2546 } else { 2547 tcg_gen_shli_i64(tmp, tmp, 32); 2548 tcg_gen_or_i64(dest, dest, tmp); 2549 } 2550 tcg_temp_free_i64(tmp); 2551 } 2552 2553 void tcg_gen_extr_i64_i32(TCGv_i32 lo, TCGv_i32 hi, TCGv_i64 arg) 2554 { 2555 if (TCG_TARGET_REG_BITS == 32) { 2556 tcg_gen_mov_i32(lo, TCGV_LOW(arg)); 2557 tcg_gen_mov_i32(hi, TCGV_HIGH(arg)); 2558 } else { 2559 tcg_gen_extrl_i64_i32(lo, arg); 2560 tcg_gen_extrh_i64_i32(hi, arg); 2561 } 2562 } 2563 2564 void tcg_gen_extr32_i64(TCGv_i64 lo, TCGv_i64 hi, TCGv_i64 arg) 2565 { 2566 tcg_gen_ext32u_i64(lo, arg); 2567 tcg_gen_shri_i64(hi, arg, 32); 2568 } 2569 2570 /* QEMU specific operations. */ 2571 2572 void tcg_gen_exit_tb(TranslationBlock *tb, unsigned idx) 2573 { 2574 uintptr_t val = (uintptr_t)tb + idx; 2575 2576 if (tb == NULL) { 2577 tcg_debug_assert(idx == 0); 2578 } else if (idx <= TB_EXIT_IDXMAX) { 2579 #ifdef CONFIG_DEBUG_TCG 2580 /* This is an exit following a goto_tb. Verify that we have 2581 seen this numbered exit before, via tcg_gen_goto_tb. */ 2582 tcg_debug_assert(tcg_ctx->goto_tb_issue_mask & (1 << idx)); 2583 #endif 2584 /* When not chaining, exit without indicating a link. */ 2585 if (qemu_loglevel_mask(CPU_LOG_TB_NOCHAIN)) { 2586 val = 0; 2587 } 2588 } else { 2589 /* This is an exit via the exitreq label. */ 2590 tcg_debug_assert(idx == TB_EXIT_REQUESTED); 2591 } 2592 2593 tcg_gen_op1i(INDEX_op_exit_tb, val); 2594 } 2595 2596 void tcg_gen_goto_tb(unsigned idx) 2597 { 2598 /* We only support two chained exits. */ 2599 tcg_debug_assert(idx <= TB_EXIT_IDXMAX); 2600 #ifdef CONFIG_DEBUG_TCG 2601 /* Verify that we havn't seen this numbered exit before. */ 2602 tcg_debug_assert((tcg_ctx->goto_tb_issue_mask & (1 << idx)) == 0); 2603 tcg_ctx->goto_tb_issue_mask |= 1 << idx; 2604 #endif 2605 /* When not chaining, we simply fall through to the "fallback" exit. */ 2606 if (!qemu_loglevel_mask(CPU_LOG_TB_NOCHAIN)) { 2607 tcg_gen_op1i(INDEX_op_goto_tb, idx); 2608 } 2609 } 2610 2611 void tcg_gen_lookup_and_goto_ptr(void) 2612 { 2613 if (TCG_TARGET_HAS_goto_ptr && !qemu_loglevel_mask(CPU_LOG_TB_NOCHAIN)) { 2614 TCGv_ptr ptr = tcg_temp_new_ptr(); 2615 gen_helper_lookup_tb_ptr(ptr, cpu_env); 2616 tcg_gen_op1i(INDEX_op_goto_ptr, tcgv_ptr_arg(ptr)); 2617 tcg_temp_free_ptr(ptr); 2618 } else { 2619 tcg_gen_exit_tb(NULL, 0); 2620 } 2621 } 2622 2623 static inline TCGMemOp tcg_canonicalize_memop(TCGMemOp op, bool is64, bool st) 2624 { 2625 /* Trigger the asserts within as early as possible. */ 2626 (void)get_alignment_bits(op); 2627 2628 switch (op & MO_SIZE) { 2629 case MO_8: 2630 op &= ~MO_BSWAP; 2631 break; 2632 case MO_16: 2633 break; 2634 case MO_32: 2635 if (!is64) { 2636 op &= ~MO_SIGN; 2637 } 2638 break; 2639 case MO_64: 2640 if (!is64) { 2641 tcg_abort(); 2642 } 2643 break; 2644 } 2645 if (st) { 2646 op &= ~MO_SIGN; 2647 } 2648 return op; 2649 } 2650 2651 static void gen_ldst_i32(TCGOpcode opc, TCGv_i32 val, TCGv addr, 2652 TCGMemOp memop, TCGArg idx) 2653 { 2654 TCGMemOpIdx oi = make_memop_idx(memop, idx); 2655 #if TARGET_LONG_BITS == 32 2656 tcg_gen_op3i_i32(opc, val, addr, oi); 2657 #else 2658 if (TCG_TARGET_REG_BITS == 32) { 2659 tcg_gen_op4i_i32(opc, val, TCGV_LOW(addr), TCGV_HIGH(addr), oi); 2660 } else { 2661 tcg_gen_op3(opc, tcgv_i32_arg(val), tcgv_i64_arg(addr), oi); 2662 } 2663 #endif 2664 } 2665 2666 static void gen_ldst_i64(TCGOpcode opc, TCGv_i64 val, TCGv addr, 2667 TCGMemOp memop, TCGArg idx) 2668 { 2669 TCGMemOpIdx oi = make_memop_idx(memop, idx); 2670 #if TARGET_LONG_BITS == 32 2671 if (TCG_TARGET_REG_BITS == 32) { 2672 tcg_gen_op4i_i32(opc, TCGV_LOW(val), TCGV_HIGH(val), addr, oi); 2673 } else { 2674 tcg_gen_op3(opc, tcgv_i64_arg(val), tcgv_i32_arg(addr), oi); 2675 } 2676 #else 2677 if (TCG_TARGET_REG_BITS == 32) { 2678 tcg_gen_op5i_i32(opc, TCGV_LOW(val), TCGV_HIGH(val), 2679 TCGV_LOW(addr), TCGV_HIGH(addr), oi); 2680 } else { 2681 tcg_gen_op3i_i64(opc, val, addr, oi); 2682 } 2683 #endif 2684 } 2685 2686 static void tcg_gen_req_mo(TCGBar type) 2687 { 2688 #ifdef TCG_GUEST_DEFAULT_MO 2689 type &= TCG_GUEST_DEFAULT_MO; 2690 #endif 2691 type &= ~TCG_TARGET_DEFAULT_MO; 2692 if (type) { 2693 tcg_gen_mb(type | TCG_BAR_SC); 2694 } 2695 } 2696 2697 void tcg_gen_qemu_ld_i32(TCGv_i32 val, TCGv addr, TCGArg idx, TCGMemOp memop) 2698 { 2699 TCGMemOp orig_memop; 2700 2701 tcg_gen_req_mo(TCG_MO_LD_LD | TCG_MO_ST_LD); 2702 memop = tcg_canonicalize_memop(memop, 0, 0); 2703 trace_guest_mem_before_tcg(tcg_ctx->cpu, cpu_env, 2704 addr, trace_mem_get_info(memop, 0)); 2705 2706 orig_memop = memop; 2707 if (!TCG_TARGET_HAS_MEMORY_BSWAP && (memop & MO_BSWAP)) { 2708 memop &= ~MO_BSWAP; 2709 /* The bswap primitive requires zero-extended input. */ 2710 if ((memop & MO_SSIZE) == MO_SW) { 2711 memop &= ~MO_SIGN; 2712 } 2713 } 2714 2715 gen_ldst_i32(INDEX_op_qemu_ld_i32, val, addr, memop, idx); 2716 2717 if ((orig_memop ^ memop) & MO_BSWAP) { 2718 switch (orig_memop & MO_SIZE) { 2719 case MO_16: 2720 tcg_gen_bswap16_i32(val, val); 2721 if (orig_memop & MO_SIGN) { 2722 tcg_gen_ext16s_i32(val, val); 2723 } 2724 break; 2725 case MO_32: 2726 tcg_gen_bswap32_i32(val, val); 2727 break; 2728 default: 2729 g_assert_not_reached(); 2730 } 2731 } 2732 } 2733 2734 void tcg_gen_qemu_st_i32(TCGv_i32 val, TCGv addr, TCGArg idx, TCGMemOp memop) 2735 { 2736 TCGv_i32 swap = NULL; 2737 2738 tcg_gen_req_mo(TCG_MO_LD_ST | TCG_MO_ST_ST); 2739 memop = tcg_canonicalize_memop(memop, 0, 1); 2740 trace_guest_mem_before_tcg(tcg_ctx->cpu, cpu_env, 2741 addr, trace_mem_get_info(memop, 1)); 2742 2743 if (!TCG_TARGET_HAS_MEMORY_BSWAP && (memop & MO_BSWAP)) { 2744 swap = tcg_temp_new_i32(); 2745 switch (memop & MO_SIZE) { 2746 case MO_16: 2747 tcg_gen_ext16u_i32(swap, val); 2748 tcg_gen_bswap16_i32(swap, swap); 2749 break; 2750 case MO_32: 2751 tcg_gen_bswap32_i32(swap, val); 2752 break; 2753 default: 2754 g_assert_not_reached(); 2755 } 2756 val = swap; 2757 memop &= ~MO_BSWAP; 2758 } 2759 2760 gen_ldst_i32(INDEX_op_qemu_st_i32, val, addr, memop, idx); 2761 2762 if (swap) { 2763 tcg_temp_free_i32(swap); 2764 } 2765 } 2766 2767 void tcg_gen_qemu_ld_i64(TCGv_i64 val, TCGv addr, TCGArg idx, TCGMemOp memop) 2768 { 2769 TCGMemOp orig_memop; 2770 2771 if (TCG_TARGET_REG_BITS == 32 && (memop & MO_SIZE) < MO_64) { 2772 tcg_gen_qemu_ld_i32(TCGV_LOW(val), addr, idx, memop); 2773 if (memop & MO_SIGN) { 2774 tcg_gen_sari_i32(TCGV_HIGH(val), TCGV_LOW(val), 31); 2775 } else { 2776 tcg_gen_movi_i32(TCGV_HIGH(val), 0); 2777 } 2778 return; 2779 } 2780 2781 tcg_gen_req_mo(TCG_MO_LD_LD | TCG_MO_ST_LD); 2782 memop = tcg_canonicalize_memop(memop, 1, 0); 2783 trace_guest_mem_before_tcg(tcg_ctx->cpu, cpu_env, 2784 addr, trace_mem_get_info(memop, 0)); 2785 2786 orig_memop = memop; 2787 if (!TCG_TARGET_HAS_MEMORY_BSWAP && (memop & MO_BSWAP)) { 2788 memop &= ~MO_BSWAP; 2789 /* The bswap primitive requires zero-extended input. */ 2790 if ((memop & MO_SIGN) && (memop & MO_SIZE) < MO_64) { 2791 memop &= ~MO_SIGN; 2792 } 2793 } 2794 2795 gen_ldst_i64(INDEX_op_qemu_ld_i64, val, addr, memop, idx); 2796 2797 if ((orig_memop ^ memop) & MO_BSWAP) { 2798 switch (orig_memop & MO_SIZE) { 2799 case MO_16: 2800 tcg_gen_bswap16_i64(val, val); 2801 if (orig_memop & MO_SIGN) { 2802 tcg_gen_ext16s_i64(val, val); 2803 } 2804 break; 2805 case MO_32: 2806 tcg_gen_bswap32_i64(val, val); 2807 if (orig_memop & MO_SIGN) { 2808 tcg_gen_ext32s_i64(val, val); 2809 } 2810 break; 2811 case MO_64: 2812 tcg_gen_bswap64_i64(val, val); 2813 break; 2814 default: 2815 g_assert_not_reached(); 2816 } 2817 } 2818 } 2819 2820 void tcg_gen_qemu_st_i64(TCGv_i64 val, TCGv addr, TCGArg idx, TCGMemOp memop) 2821 { 2822 TCGv_i64 swap = NULL; 2823 2824 if (TCG_TARGET_REG_BITS == 32 && (memop & MO_SIZE) < MO_64) { 2825 tcg_gen_qemu_st_i32(TCGV_LOW(val), addr, idx, memop); 2826 return; 2827 } 2828 2829 tcg_gen_req_mo(TCG_MO_LD_ST | TCG_MO_ST_ST); 2830 memop = tcg_canonicalize_memop(memop, 1, 1); 2831 trace_guest_mem_before_tcg(tcg_ctx->cpu, cpu_env, 2832 addr, trace_mem_get_info(memop, 1)); 2833 2834 if (!TCG_TARGET_HAS_MEMORY_BSWAP && (memop & MO_BSWAP)) { 2835 swap = tcg_temp_new_i64(); 2836 switch (memop & MO_SIZE) { 2837 case MO_16: 2838 tcg_gen_ext16u_i64(swap, val); 2839 tcg_gen_bswap16_i64(swap, swap); 2840 break; 2841 case MO_32: 2842 tcg_gen_ext32u_i64(swap, val); 2843 tcg_gen_bswap32_i64(swap, swap); 2844 break; 2845 case MO_64: 2846 tcg_gen_bswap64_i64(swap, val); 2847 break; 2848 default: 2849 g_assert_not_reached(); 2850 } 2851 val = swap; 2852 memop &= ~MO_BSWAP; 2853 } 2854 2855 gen_ldst_i64(INDEX_op_qemu_st_i64, val, addr, memop, idx); 2856 2857 if (swap) { 2858 tcg_temp_free_i64(swap); 2859 } 2860 } 2861 2862 static void tcg_gen_ext_i32(TCGv_i32 ret, TCGv_i32 val, TCGMemOp opc) 2863 { 2864 switch (opc & MO_SSIZE) { 2865 case MO_SB: 2866 tcg_gen_ext8s_i32(ret, val); 2867 break; 2868 case MO_UB: 2869 tcg_gen_ext8u_i32(ret, val); 2870 break; 2871 case MO_SW: 2872 tcg_gen_ext16s_i32(ret, val); 2873 break; 2874 case MO_UW: 2875 tcg_gen_ext16u_i32(ret, val); 2876 break; 2877 default: 2878 tcg_gen_mov_i32(ret, val); 2879 break; 2880 } 2881 } 2882 2883 static void tcg_gen_ext_i64(TCGv_i64 ret, TCGv_i64 val, TCGMemOp opc) 2884 { 2885 switch (opc & MO_SSIZE) { 2886 case MO_SB: 2887 tcg_gen_ext8s_i64(ret, val); 2888 break; 2889 case MO_UB: 2890 tcg_gen_ext8u_i64(ret, val); 2891 break; 2892 case MO_SW: 2893 tcg_gen_ext16s_i64(ret, val); 2894 break; 2895 case MO_UW: 2896 tcg_gen_ext16u_i64(ret, val); 2897 break; 2898 case MO_SL: 2899 tcg_gen_ext32s_i64(ret, val); 2900 break; 2901 case MO_UL: 2902 tcg_gen_ext32u_i64(ret, val); 2903 break; 2904 default: 2905 tcg_gen_mov_i64(ret, val); 2906 break; 2907 } 2908 } 2909 2910 #ifdef CONFIG_SOFTMMU 2911 typedef void (*gen_atomic_cx_i32)(TCGv_i32, TCGv_env, TCGv, 2912 TCGv_i32, TCGv_i32, TCGv_i32); 2913 typedef void (*gen_atomic_cx_i64)(TCGv_i64, TCGv_env, TCGv, 2914 TCGv_i64, TCGv_i64, TCGv_i32); 2915 typedef void (*gen_atomic_op_i32)(TCGv_i32, TCGv_env, TCGv, 2916 TCGv_i32, TCGv_i32); 2917 typedef void (*gen_atomic_op_i64)(TCGv_i64, TCGv_env, TCGv, 2918 TCGv_i64, TCGv_i32); 2919 #else 2920 typedef void (*gen_atomic_cx_i32)(TCGv_i32, TCGv_env, TCGv, TCGv_i32, TCGv_i32); 2921 typedef void (*gen_atomic_cx_i64)(TCGv_i64, TCGv_env, TCGv, TCGv_i64, TCGv_i64); 2922 typedef void (*gen_atomic_op_i32)(TCGv_i32, TCGv_env, TCGv, TCGv_i32); 2923 typedef void (*gen_atomic_op_i64)(TCGv_i64, TCGv_env, TCGv, TCGv_i64); 2924 #endif 2925 2926 #ifdef CONFIG_ATOMIC64 2927 # define WITH_ATOMIC64(X) X, 2928 #else 2929 # define WITH_ATOMIC64(X) 2930 #endif 2931 2932 static void * const table_cmpxchg[16] = { 2933 [MO_8] = gen_helper_atomic_cmpxchgb, 2934 [MO_16 | MO_LE] = gen_helper_atomic_cmpxchgw_le, 2935 [MO_16 | MO_BE] = gen_helper_atomic_cmpxchgw_be, 2936 [MO_32 | MO_LE] = gen_helper_atomic_cmpxchgl_le, 2937 [MO_32 | MO_BE] = gen_helper_atomic_cmpxchgl_be, 2938 WITH_ATOMIC64([MO_64 | MO_LE] = gen_helper_atomic_cmpxchgq_le) 2939 WITH_ATOMIC64([MO_64 | MO_BE] = gen_helper_atomic_cmpxchgq_be) 2940 }; 2941 2942 void tcg_gen_atomic_cmpxchg_i32(TCGv_i32 retv, TCGv addr, TCGv_i32 cmpv, 2943 TCGv_i32 newv, TCGArg idx, TCGMemOp memop) 2944 { 2945 memop = tcg_canonicalize_memop(memop, 0, 0); 2946 2947 if (!(tcg_ctx->tb_cflags & CF_PARALLEL)) { 2948 TCGv_i32 t1 = tcg_temp_new_i32(); 2949 TCGv_i32 t2 = tcg_temp_new_i32(); 2950 2951 tcg_gen_ext_i32(t2, cmpv, memop & MO_SIZE); 2952 2953 tcg_gen_qemu_ld_i32(t1, addr, idx, memop & ~MO_SIGN); 2954 tcg_gen_movcond_i32(TCG_COND_EQ, t2, t1, t2, newv, t1); 2955 tcg_gen_qemu_st_i32(t2, addr, idx, memop); 2956 tcg_temp_free_i32(t2); 2957 2958 if (memop & MO_SIGN) { 2959 tcg_gen_ext_i32(retv, t1, memop); 2960 } else { 2961 tcg_gen_mov_i32(retv, t1); 2962 } 2963 tcg_temp_free_i32(t1); 2964 } else { 2965 gen_atomic_cx_i32 gen; 2966 2967 gen = table_cmpxchg[memop & (MO_SIZE | MO_BSWAP)]; 2968 tcg_debug_assert(gen != NULL); 2969 2970 #ifdef CONFIG_SOFTMMU 2971 { 2972 TCGv_i32 oi = tcg_const_i32(make_memop_idx(memop & ~MO_SIGN, idx)); 2973 gen(retv, cpu_env, addr, cmpv, newv, oi); 2974 tcg_temp_free_i32(oi); 2975 } 2976 #else 2977 gen(retv, cpu_env, addr, cmpv, newv); 2978 #endif 2979 2980 if (memop & MO_SIGN) { 2981 tcg_gen_ext_i32(retv, retv, memop); 2982 } 2983 } 2984 } 2985 2986 void tcg_gen_atomic_cmpxchg_i64(TCGv_i64 retv, TCGv addr, TCGv_i64 cmpv, 2987 TCGv_i64 newv, TCGArg idx, TCGMemOp memop) 2988 { 2989 memop = tcg_canonicalize_memop(memop, 1, 0); 2990 2991 if (!(tcg_ctx->tb_cflags & CF_PARALLEL)) { 2992 TCGv_i64 t1 = tcg_temp_new_i64(); 2993 TCGv_i64 t2 = tcg_temp_new_i64(); 2994 2995 tcg_gen_ext_i64(t2, cmpv, memop & MO_SIZE); 2996 2997 tcg_gen_qemu_ld_i64(t1, addr, idx, memop & ~MO_SIGN); 2998 tcg_gen_movcond_i64(TCG_COND_EQ, t2, t1, t2, newv, t1); 2999 tcg_gen_qemu_st_i64(t2, addr, idx, memop); 3000 tcg_temp_free_i64(t2); 3001 3002 if (memop & MO_SIGN) { 3003 tcg_gen_ext_i64(retv, t1, memop); 3004 } else { 3005 tcg_gen_mov_i64(retv, t1); 3006 } 3007 tcg_temp_free_i64(t1); 3008 } else if ((memop & MO_SIZE) == MO_64) { 3009 #ifdef CONFIG_ATOMIC64 3010 gen_atomic_cx_i64 gen; 3011 3012 gen = table_cmpxchg[memop & (MO_SIZE | MO_BSWAP)]; 3013 tcg_debug_assert(gen != NULL); 3014 3015 #ifdef CONFIG_SOFTMMU 3016 { 3017 TCGv_i32 oi = tcg_const_i32(make_memop_idx(memop, idx)); 3018 gen(retv, cpu_env, addr, cmpv, newv, oi); 3019 tcg_temp_free_i32(oi); 3020 } 3021 #else 3022 gen(retv, cpu_env, addr, cmpv, newv); 3023 #endif 3024 #else 3025 gen_helper_exit_atomic(cpu_env); 3026 /* Produce a result, so that we have a well-formed opcode stream 3027 with respect to uses of the result in the (dead) code following. */ 3028 tcg_gen_movi_i64(retv, 0); 3029 #endif /* CONFIG_ATOMIC64 */ 3030 } else { 3031 TCGv_i32 c32 = tcg_temp_new_i32(); 3032 TCGv_i32 n32 = tcg_temp_new_i32(); 3033 TCGv_i32 r32 = tcg_temp_new_i32(); 3034 3035 tcg_gen_extrl_i64_i32(c32, cmpv); 3036 tcg_gen_extrl_i64_i32(n32, newv); 3037 tcg_gen_atomic_cmpxchg_i32(r32, addr, c32, n32, idx, memop & ~MO_SIGN); 3038 tcg_temp_free_i32(c32); 3039 tcg_temp_free_i32(n32); 3040 3041 tcg_gen_extu_i32_i64(retv, r32); 3042 tcg_temp_free_i32(r32); 3043 3044 if (memop & MO_SIGN) { 3045 tcg_gen_ext_i64(retv, retv, memop); 3046 } 3047 } 3048 } 3049 3050 static void do_nonatomic_op_i32(TCGv_i32 ret, TCGv addr, TCGv_i32 val, 3051 TCGArg idx, TCGMemOp memop, bool new_val, 3052 void (*gen)(TCGv_i32, TCGv_i32, TCGv_i32)) 3053 { 3054 TCGv_i32 t1 = tcg_temp_new_i32(); 3055 TCGv_i32 t2 = tcg_temp_new_i32(); 3056 3057 memop = tcg_canonicalize_memop(memop, 0, 0); 3058 3059 tcg_gen_qemu_ld_i32(t1, addr, idx, memop & ~MO_SIGN); 3060 gen(t2, t1, val); 3061 tcg_gen_qemu_st_i32(t2, addr, idx, memop); 3062 3063 tcg_gen_ext_i32(ret, (new_val ? t2 : t1), memop); 3064 tcg_temp_free_i32(t1); 3065 tcg_temp_free_i32(t2); 3066 } 3067 3068 static void do_atomic_op_i32(TCGv_i32 ret, TCGv addr, TCGv_i32 val, 3069 TCGArg idx, TCGMemOp memop, void * const table[]) 3070 { 3071 gen_atomic_op_i32 gen; 3072 3073 memop = tcg_canonicalize_memop(memop, 0, 0); 3074 3075 gen = table[memop & (MO_SIZE | MO_BSWAP)]; 3076 tcg_debug_assert(gen != NULL); 3077 3078 #ifdef CONFIG_SOFTMMU 3079 { 3080 TCGv_i32 oi = tcg_const_i32(make_memop_idx(memop & ~MO_SIGN, idx)); 3081 gen(ret, cpu_env, addr, val, oi); 3082 tcg_temp_free_i32(oi); 3083 } 3084 #else 3085 gen(ret, cpu_env, addr, val); 3086 #endif 3087 3088 if (memop & MO_SIGN) { 3089 tcg_gen_ext_i32(ret, ret, memop); 3090 } 3091 } 3092 3093 static void do_nonatomic_op_i64(TCGv_i64 ret, TCGv addr, TCGv_i64 val, 3094 TCGArg idx, TCGMemOp memop, bool new_val, 3095 void (*gen)(TCGv_i64, TCGv_i64, TCGv_i64)) 3096 { 3097 TCGv_i64 t1 = tcg_temp_new_i64(); 3098 TCGv_i64 t2 = tcg_temp_new_i64(); 3099 3100 memop = tcg_canonicalize_memop(memop, 1, 0); 3101 3102 tcg_gen_qemu_ld_i64(t1, addr, idx, memop & ~MO_SIGN); 3103 gen(t2, t1, val); 3104 tcg_gen_qemu_st_i64(t2, addr, idx, memop); 3105 3106 tcg_gen_ext_i64(ret, (new_val ? t2 : t1), memop); 3107 tcg_temp_free_i64(t1); 3108 tcg_temp_free_i64(t2); 3109 } 3110 3111 static void do_atomic_op_i64(TCGv_i64 ret, TCGv addr, TCGv_i64 val, 3112 TCGArg idx, TCGMemOp memop, void * const table[]) 3113 { 3114 memop = tcg_canonicalize_memop(memop, 1, 0); 3115 3116 if ((memop & MO_SIZE) == MO_64) { 3117 #ifdef CONFIG_ATOMIC64 3118 gen_atomic_op_i64 gen; 3119 3120 gen = table[memop & (MO_SIZE | MO_BSWAP)]; 3121 tcg_debug_assert(gen != NULL); 3122 3123 #ifdef CONFIG_SOFTMMU 3124 { 3125 TCGv_i32 oi = tcg_const_i32(make_memop_idx(memop & ~MO_SIGN, idx)); 3126 gen(ret, cpu_env, addr, val, oi); 3127 tcg_temp_free_i32(oi); 3128 } 3129 #else 3130 gen(ret, cpu_env, addr, val); 3131 #endif 3132 #else 3133 gen_helper_exit_atomic(cpu_env); 3134 /* Produce a result, so that we have a well-formed opcode stream 3135 with respect to uses of the result in the (dead) code following. */ 3136 tcg_gen_movi_i64(ret, 0); 3137 #endif /* CONFIG_ATOMIC64 */ 3138 } else { 3139 TCGv_i32 v32 = tcg_temp_new_i32(); 3140 TCGv_i32 r32 = tcg_temp_new_i32(); 3141 3142 tcg_gen_extrl_i64_i32(v32, val); 3143 do_atomic_op_i32(r32, addr, v32, idx, memop & ~MO_SIGN, table); 3144 tcg_temp_free_i32(v32); 3145 3146 tcg_gen_extu_i32_i64(ret, r32); 3147 tcg_temp_free_i32(r32); 3148 3149 if (memop & MO_SIGN) { 3150 tcg_gen_ext_i64(ret, ret, memop); 3151 } 3152 } 3153 } 3154 3155 #define GEN_ATOMIC_HELPER(NAME, OP, NEW) \ 3156 static void * const table_##NAME[16] = { \ 3157 [MO_8] = gen_helper_atomic_##NAME##b, \ 3158 [MO_16 | MO_LE] = gen_helper_atomic_##NAME##w_le, \ 3159 [MO_16 | MO_BE] = gen_helper_atomic_##NAME##w_be, \ 3160 [MO_32 | MO_LE] = gen_helper_atomic_##NAME##l_le, \ 3161 [MO_32 | MO_BE] = gen_helper_atomic_##NAME##l_be, \ 3162 WITH_ATOMIC64([MO_64 | MO_LE] = gen_helper_atomic_##NAME##q_le) \ 3163 WITH_ATOMIC64([MO_64 | MO_BE] = gen_helper_atomic_##NAME##q_be) \ 3164 }; \ 3165 void tcg_gen_atomic_##NAME##_i32 \ 3166 (TCGv_i32 ret, TCGv addr, TCGv_i32 val, TCGArg idx, TCGMemOp memop) \ 3167 { \ 3168 if (tcg_ctx->tb_cflags & CF_PARALLEL) { \ 3169 do_atomic_op_i32(ret, addr, val, idx, memop, table_##NAME); \ 3170 } else { \ 3171 do_nonatomic_op_i32(ret, addr, val, idx, memop, NEW, \ 3172 tcg_gen_##OP##_i32); \ 3173 } \ 3174 } \ 3175 void tcg_gen_atomic_##NAME##_i64 \ 3176 (TCGv_i64 ret, TCGv addr, TCGv_i64 val, TCGArg idx, TCGMemOp memop) \ 3177 { \ 3178 if (tcg_ctx->tb_cflags & CF_PARALLEL) { \ 3179 do_atomic_op_i64(ret, addr, val, idx, memop, table_##NAME); \ 3180 } else { \ 3181 do_nonatomic_op_i64(ret, addr, val, idx, memop, NEW, \ 3182 tcg_gen_##OP##_i64); \ 3183 } \ 3184 } 3185 3186 GEN_ATOMIC_HELPER(fetch_add, add, 0) 3187 GEN_ATOMIC_HELPER(fetch_and, and, 0) 3188 GEN_ATOMIC_HELPER(fetch_or, or, 0) 3189 GEN_ATOMIC_HELPER(fetch_xor, xor, 0) 3190 GEN_ATOMIC_HELPER(fetch_smin, smin, 0) 3191 GEN_ATOMIC_HELPER(fetch_umin, umin, 0) 3192 GEN_ATOMIC_HELPER(fetch_smax, smax, 0) 3193 GEN_ATOMIC_HELPER(fetch_umax, umax, 0) 3194 3195 GEN_ATOMIC_HELPER(add_fetch, add, 1) 3196 GEN_ATOMIC_HELPER(and_fetch, and, 1) 3197 GEN_ATOMIC_HELPER(or_fetch, or, 1) 3198 GEN_ATOMIC_HELPER(xor_fetch, xor, 1) 3199 GEN_ATOMIC_HELPER(smin_fetch, smin, 1) 3200 GEN_ATOMIC_HELPER(umin_fetch, umin, 1) 3201 GEN_ATOMIC_HELPER(smax_fetch, smax, 1) 3202 GEN_ATOMIC_HELPER(umax_fetch, umax, 1) 3203 3204 static void tcg_gen_mov2_i32(TCGv_i32 r, TCGv_i32 a, TCGv_i32 b) 3205 { 3206 tcg_gen_mov_i32(r, b); 3207 } 3208 3209 static void tcg_gen_mov2_i64(TCGv_i64 r, TCGv_i64 a, TCGv_i64 b) 3210 { 3211 tcg_gen_mov_i64(r, b); 3212 } 3213 3214 GEN_ATOMIC_HELPER(xchg, mov2, 0) 3215 3216 #undef GEN_ATOMIC_HELPER 3217