1 /* 2 * Tiny Code Generator for QEMU 3 * 4 * Copyright (c) 2008 Fabrice Bellard 5 * 6 * Permission is hereby granted, free of charge, to any person obtaining a copy 7 * of this software and associated documentation files (the "Software"), to deal 8 * in the Software without restriction, including without limitation the rights 9 * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 10 * copies of the Software, and to permit persons to whom the Software is 11 * furnished to do so, subject to the following conditions: 12 * 13 * The above copyright notice and this permission notice shall be included in 14 * all copies or substantial portions of the Software. 15 * 16 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 17 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 18 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL 19 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 20 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 21 * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN 22 * THE SOFTWARE. 23 */ 24 25 #include "qemu/osdep.h" 26 #include "exec/exec-all.h" 27 #include "tcg/tcg.h" 28 #include "tcg/tcg-op.h" 29 #include "tcg/tcg-mo.h" 30 #include "exec/plugin-gen.h" 31 #include "tcg-internal.h" 32 33 34 void tcg_gen_op1(TCGOpcode opc, TCGArg a1) 35 { 36 TCGOp *op = tcg_emit_op(opc, 1); 37 op->args[0] = a1; 38 } 39 40 void tcg_gen_op2(TCGOpcode opc, TCGArg a1, TCGArg a2) 41 { 42 TCGOp *op = tcg_emit_op(opc, 2); 43 op->args[0] = a1; 44 op->args[1] = a2; 45 } 46 47 void tcg_gen_op3(TCGOpcode opc, TCGArg a1, TCGArg a2, TCGArg a3) 48 { 49 TCGOp *op = tcg_emit_op(opc, 3); 50 op->args[0] = a1; 51 op->args[1] = a2; 52 op->args[2] = a3; 53 } 54 55 void tcg_gen_op4(TCGOpcode opc, TCGArg a1, TCGArg a2, TCGArg a3, TCGArg a4) 56 { 57 TCGOp *op = tcg_emit_op(opc, 4); 58 op->args[0] = a1; 59 op->args[1] = a2; 60 op->args[2] = a3; 61 op->args[3] = a4; 62 } 63 64 void tcg_gen_op5(TCGOpcode opc, TCGArg a1, TCGArg a2, TCGArg a3, 65 TCGArg a4, TCGArg a5) 66 { 67 TCGOp *op = tcg_emit_op(opc, 5); 68 op->args[0] = a1; 69 op->args[1] = a2; 70 op->args[2] = a3; 71 op->args[3] = a4; 72 op->args[4] = a5; 73 } 74 75 void tcg_gen_op6(TCGOpcode opc, TCGArg a1, TCGArg a2, TCGArg a3, 76 TCGArg a4, TCGArg a5, TCGArg a6) 77 { 78 TCGOp *op = tcg_emit_op(opc, 6); 79 op->args[0] = a1; 80 op->args[1] = a2; 81 op->args[2] = a3; 82 op->args[3] = a4; 83 op->args[4] = a5; 84 op->args[5] = a6; 85 } 86 87 void tcg_gen_mb(TCGBar mb_type) 88 { 89 if (tcg_ctx->gen_tb->cflags & CF_PARALLEL) { 90 tcg_gen_op1(INDEX_op_mb, mb_type); 91 } 92 } 93 94 /* 32 bit ops */ 95 96 void tcg_gen_movi_i32(TCGv_i32 ret, int32_t arg) 97 { 98 tcg_gen_mov_i32(ret, tcg_constant_i32(arg)); 99 } 100 101 void tcg_gen_addi_i32(TCGv_i32 ret, TCGv_i32 arg1, int32_t arg2) 102 { 103 /* some cases can be optimized here */ 104 if (arg2 == 0) { 105 tcg_gen_mov_i32(ret, arg1); 106 } else { 107 tcg_gen_add_i32(ret, arg1, tcg_constant_i32(arg2)); 108 } 109 } 110 111 void tcg_gen_subfi_i32(TCGv_i32 ret, int32_t arg1, TCGv_i32 arg2) 112 { 113 if (arg1 == 0 && TCG_TARGET_HAS_neg_i32) { 114 /* Don't recurse with tcg_gen_neg_i32. */ 115 tcg_gen_op2_i32(INDEX_op_neg_i32, ret, arg2); 116 } else { 117 tcg_gen_sub_i32(ret, tcg_constant_i32(arg1), arg2); 118 } 119 } 120 121 void tcg_gen_subi_i32(TCGv_i32 ret, TCGv_i32 arg1, int32_t arg2) 122 { 123 /* some cases can be optimized here */ 124 if (arg2 == 0) { 125 tcg_gen_mov_i32(ret, arg1); 126 } else { 127 tcg_gen_sub_i32(ret, arg1, tcg_constant_i32(arg2)); 128 } 129 } 130 131 void tcg_gen_andi_i32(TCGv_i32 ret, TCGv_i32 arg1, int32_t arg2) 132 { 133 /* Some cases can be optimized here. */ 134 switch (arg2) { 135 case 0: 136 tcg_gen_movi_i32(ret, 0); 137 return; 138 case -1: 139 tcg_gen_mov_i32(ret, arg1); 140 return; 141 case 0xff: 142 /* Don't recurse with tcg_gen_ext8u_i32. */ 143 if (TCG_TARGET_HAS_ext8u_i32) { 144 tcg_gen_op2_i32(INDEX_op_ext8u_i32, ret, arg1); 145 return; 146 } 147 break; 148 case 0xffff: 149 if (TCG_TARGET_HAS_ext16u_i32) { 150 tcg_gen_op2_i32(INDEX_op_ext16u_i32, ret, arg1); 151 return; 152 } 153 break; 154 } 155 156 tcg_gen_and_i32(ret, arg1, tcg_constant_i32(arg2)); 157 } 158 159 void tcg_gen_ori_i32(TCGv_i32 ret, TCGv_i32 arg1, int32_t arg2) 160 { 161 /* Some cases can be optimized here. */ 162 if (arg2 == -1) { 163 tcg_gen_movi_i32(ret, -1); 164 } else if (arg2 == 0) { 165 tcg_gen_mov_i32(ret, arg1); 166 } else { 167 tcg_gen_or_i32(ret, arg1, tcg_constant_i32(arg2)); 168 } 169 } 170 171 void tcg_gen_xori_i32(TCGv_i32 ret, TCGv_i32 arg1, int32_t arg2) 172 { 173 /* Some cases can be optimized here. */ 174 if (arg2 == 0) { 175 tcg_gen_mov_i32(ret, arg1); 176 } else if (arg2 == -1 && TCG_TARGET_HAS_not_i32) { 177 /* Don't recurse with tcg_gen_not_i32. */ 178 tcg_gen_op2_i32(INDEX_op_not_i32, ret, arg1); 179 } else { 180 tcg_gen_xor_i32(ret, arg1, tcg_constant_i32(arg2)); 181 } 182 } 183 184 void tcg_gen_shli_i32(TCGv_i32 ret, TCGv_i32 arg1, int32_t arg2) 185 { 186 tcg_debug_assert(arg2 >= 0 && arg2 < 32); 187 if (arg2 == 0) { 188 tcg_gen_mov_i32(ret, arg1); 189 } else { 190 tcg_gen_shl_i32(ret, arg1, tcg_constant_i32(arg2)); 191 } 192 } 193 194 void tcg_gen_shri_i32(TCGv_i32 ret, TCGv_i32 arg1, int32_t arg2) 195 { 196 tcg_debug_assert(arg2 >= 0 && arg2 < 32); 197 if (arg2 == 0) { 198 tcg_gen_mov_i32(ret, arg1); 199 } else { 200 tcg_gen_shr_i32(ret, arg1, tcg_constant_i32(arg2)); 201 } 202 } 203 204 void tcg_gen_sari_i32(TCGv_i32 ret, TCGv_i32 arg1, int32_t arg2) 205 { 206 tcg_debug_assert(arg2 >= 0 && arg2 < 32); 207 if (arg2 == 0) { 208 tcg_gen_mov_i32(ret, arg1); 209 } else { 210 tcg_gen_sar_i32(ret, arg1, tcg_constant_i32(arg2)); 211 } 212 } 213 214 void tcg_gen_brcond_i32(TCGCond cond, TCGv_i32 arg1, TCGv_i32 arg2, TCGLabel *l) 215 { 216 if (cond == TCG_COND_ALWAYS) { 217 tcg_gen_br(l); 218 } else if (cond != TCG_COND_NEVER) { 219 l->refs++; 220 tcg_gen_op4ii_i32(INDEX_op_brcond_i32, arg1, arg2, cond, label_arg(l)); 221 } 222 } 223 224 void tcg_gen_brcondi_i32(TCGCond cond, TCGv_i32 arg1, int32_t arg2, TCGLabel *l) 225 { 226 if (cond == TCG_COND_ALWAYS) { 227 tcg_gen_br(l); 228 } else if (cond != TCG_COND_NEVER) { 229 tcg_gen_brcond_i32(cond, arg1, tcg_constant_i32(arg2), l); 230 } 231 } 232 233 void tcg_gen_setcond_i32(TCGCond cond, TCGv_i32 ret, 234 TCGv_i32 arg1, TCGv_i32 arg2) 235 { 236 if (cond == TCG_COND_ALWAYS) { 237 tcg_gen_movi_i32(ret, 1); 238 } else if (cond == TCG_COND_NEVER) { 239 tcg_gen_movi_i32(ret, 0); 240 } else { 241 tcg_gen_op4i_i32(INDEX_op_setcond_i32, ret, arg1, arg2, cond); 242 } 243 } 244 245 void tcg_gen_setcondi_i32(TCGCond cond, TCGv_i32 ret, 246 TCGv_i32 arg1, int32_t arg2) 247 { 248 tcg_gen_setcond_i32(cond, ret, arg1, tcg_constant_i32(arg2)); 249 } 250 251 void tcg_gen_muli_i32(TCGv_i32 ret, TCGv_i32 arg1, int32_t arg2) 252 { 253 if (arg2 == 0) { 254 tcg_gen_movi_i32(ret, 0); 255 } else if (is_power_of_2(arg2)) { 256 tcg_gen_shli_i32(ret, arg1, ctz32(arg2)); 257 } else { 258 tcg_gen_mul_i32(ret, arg1, tcg_constant_i32(arg2)); 259 } 260 } 261 262 void tcg_gen_div_i32(TCGv_i32 ret, TCGv_i32 arg1, TCGv_i32 arg2) 263 { 264 if (TCG_TARGET_HAS_div_i32) { 265 tcg_gen_op3_i32(INDEX_op_div_i32, ret, arg1, arg2); 266 } else if (TCG_TARGET_HAS_div2_i32) { 267 TCGv_i32 t0 = tcg_temp_ebb_new_i32(); 268 tcg_gen_sari_i32(t0, arg1, 31); 269 tcg_gen_op5_i32(INDEX_op_div2_i32, ret, t0, arg1, t0, arg2); 270 tcg_temp_free_i32(t0); 271 } else { 272 gen_helper_div_i32(ret, arg1, arg2); 273 } 274 } 275 276 void tcg_gen_rem_i32(TCGv_i32 ret, TCGv_i32 arg1, TCGv_i32 arg2) 277 { 278 if (TCG_TARGET_HAS_rem_i32) { 279 tcg_gen_op3_i32(INDEX_op_rem_i32, ret, arg1, arg2); 280 } else if (TCG_TARGET_HAS_div_i32) { 281 TCGv_i32 t0 = tcg_temp_ebb_new_i32(); 282 tcg_gen_op3_i32(INDEX_op_div_i32, t0, arg1, arg2); 283 tcg_gen_mul_i32(t0, t0, arg2); 284 tcg_gen_sub_i32(ret, arg1, t0); 285 tcg_temp_free_i32(t0); 286 } else if (TCG_TARGET_HAS_div2_i32) { 287 TCGv_i32 t0 = tcg_temp_ebb_new_i32(); 288 tcg_gen_sari_i32(t0, arg1, 31); 289 tcg_gen_op5_i32(INDEX_op_div2_i32, t0, ret, arg1, t0, arg2); 290 tcg_temp_free_i32(t0); 291 } else { 292 gen_helper_rem_i32(ret, arg1, arg2); 293 } 294 } 295 296 void tcg_gen_divu_i32(TCGv_i32 ret, TCGv_i32 arg1, TCGv_i32 arg2) 297 { 298 if (TCG_TARGET_HAS_div_i32) { 299 tcg_gen_op3_i32(INDEX_op_divu_i32, ret, arg1, arg2); 300 } else if (TCG_TARGET_HAS_div2_i32) { 301 TCGv_i32 t0 = tcg_temp_ebb_new_i32(); 302 tcg_gen_movi_i32(t0, 0); 303 tcg_gen_op5_i32(INDEX_op_divu2_i32, ret, t0, arg1, t0, arg2); 304 tcg_temp_free_i32(t0); 305 } else { 306 gen_helper_divu_i32(ret, arg1, arg2); 307 } 308 } 309 310 void tcg_gen_remu_i32(TCGv_i32 ret, TCGv_i32 arg1, TCGv_i32 arg2) 311 { 312 if (TCG_TARGET_HAS_rem_i32) { 313 tcg_gen_op3_i32(INDEX_op_remu_i32, ret, arg1, arg2); 314 } else if (TCG_TARGET_HAS_div_i32) { 315 TCGv_i32 t0 = tcg_temp_ebb_new_i32(); 316 tcg_gen_op3_i32(INDEX_op_divu_i32, t0, arg1, arg2); 317 tcg_gen_mul_i32(t0, t0, arg2); 318 tcg_gen_sub_i32(ret, arg1, t0); 319 tcg_temp_free_i32(t0); 320 } else if (TCG_TARGET_HAS_div2_i32) { 321 TCGv_i32 t0 = tcg_temp_ebb_new_i32(); 322 tcg_gen_movi_i32(t0, 0); 323 tcg_gen_op5_i32(INDEX_op_divu2_i32, t0, ret, arg1, t0, arg2); 324 tcg_temp_free_i32(t0); 325 } else { 326 gen_helper_remu_i32(ret, arg1, arg2); 327 } 328 } 329 330 void tcg_gen_andc_i32(TCGv_i32 ret, TCGv_i32 arg1, TCGv_i32 arg2) 331 { 332 if (TCG_TARGET_HAS_andc_i32) { 333 tcg_gen_op3_i32(INDEX_op_andc_i32, ret, arg1, arg2); 334 } else { 335 TCGv_i32 t0 = tcg_temp_ebb_new_i32(); 336 tcg_gen_not_i32(t0, arg2); 337 tcg_gen_and_i32(ret, arg1, t0); 338 tcg_temp_free_i32(t0); 339 } 340 } 341 342 void tcg_gen_eqv_i32(TCGv_i32 ret, TCGv_i32 arg1, TCGv_i32 arg2) 343 { 344 if (TCG_TARGET_HAS_eqv_i32) { 345 tcg_gen_op3_i32(INDEX_op_eqv_i32, ret, arg1, arg2); 346 } else { 347 tcg_gen_xor_i32(ret, arg1, arg2); 348 tcg_gen_not_i32(ret, ret); 349 } 350 } 351 352 void tcg_gen_nand_i32(TCGv_i32 ret, TCGv_i32 arg1, TCGv_i32 arg2) 353 { 354 if (TCG_TARGET_HAS_nand_i32) { 355 tcg_gen_op3_i32(INDEX_op_nand_i32, ret, arg1, arg2); 356 } else { 357 tcg_gen_and_i32(ret, arg1, arg2); 358 tcg_gen_not_i32(ret, ret); 359 } 360 } 361 362 void tcg_gen_nor_i32(TCGv_i32 ret, TCGv_i32 arg1, TCGv_i32 arg2) 363 { 364 if (TCG_TARGET_HAS_nor_i32) { 365 tcg_gen_op3_i32(INDEX_op_nor_i32, ret, arg1, arg2); 366 } else { 367 tcg_gen_or_i32(ret, arg1, arg2); 368 tcg_gen_not_i32(ret, ret); 369 } 370 } 371 372 void tcg_gen_orc_i32(TCGv_i32 ret, TCGv_i32 arg1, TCGv_i32 arg2) 373 { 374 if (TCG_TARGET_HAS_orc_i32) { 375 tcg_gen_op3_i32(INDEX_op_orc_i32, ret, arg1, arg2); 376 } else { 377 TCGv_i32 t0 = tcg_temp_ebb_new_i32(); 378 tcg_gen_not_i32(t0, arg2); 379 tcg_gen_or_i32(ret, arg1, t0); 380 tcg_temp_free_i32(t0); 381 } 382 } 383 384 void tcg_gen_clz_i32(TCGv_i32 ret, TCGv_i32 arg1, TCGv_i32 arg2) 385 { 386 if (TCG_TARGET_HAS_clz_i32) { 387 tcg_gen_op3_i32(INDEX_op_clz_i32, ret, arg1, arg2); 388 } else if (TCG_TARGET_HAS_clz_i64) { 389 TCGv_i64 t1 = tcg_temp_ebb_new_i64(); 390 TCGv_i64 t2 = tcg_temp_ebb_new_i64(); 391 tcg_gen_extu_i32_i64(t1, arg1); 392 tcg_gen_extu_i32_i64(t2, arg2); 393 tcg_gen_addi_i64(t2, t2, 32); 394 tcg_gen_clz_i64(t1, t1, t2); 395 tcg_gen_extrl_i64_i32(ret, t1); 396 tcg_temp_free_i64(t1); 397 tcg_temp_free_i64(t2); 398 tcg_gen_subi_i32(ret, ret, 32); 399 } else { 400 gen_helper_clz_i32(ret, arg1, arg2); 401 } 402 } 403 404 void tcg_gen_clzi_i32(TCGv_i32 ret, TCGv_i32 arg1, uint32_t arg2) 405 { 406 tcg_gen_clz_i32(ret, arg1, tcg_constant_i32(arg2)); 407 } 408 409 void tcg_gen_ctz_i32(TCGv_i32 ret, TCGv_i32 arg1, TCGv_i32 arg2) 410 { 411 if (TCG_TARGET_HAS_ctz_i32) { 412 tcg_gen_op3_i32(INDEX_op_ctz_i32, ret, arg1, arg2); 413 } else if (TCG_TARGET_HAS_ctz_i64) { 414 TCGv_i64 t1 = tcg_temp_ebb_new_i64(); 415 TCGv_i64 t2 = tcg_temp_ebb_new_i64(); 416 tcg_gen_extu_i32_i64(t1, arg1); 417 tcg_gen_extu_i32_i64(t2, arg2); 418 tcg_gen_ctz_i64(t1, t1, t2); 419 tcg_gen_extrl_i64_i32(ret, t1); 420 tcg_temp_free_i64(t1); 421 tcg_temp_free_i64(t2); 422 } else if (TCG_TARGET_HAS_ctpop_i32 423 || TCG_TARGET_HAS_ctpop_i64 424 || TCG_TARGET_HAS_clz_i32 425 || TCG_TARGET_HAS_clz_i64) { 426 TCGv_i32 z, t = tcg_temp_ebb_new_i32(); 427 428 if (TCG_TARGET_HAS_ctpop_i32 || TCG_TARGET_HAS_ctpop_i64) { 429 tcg_gen_subi_i32(t, arg1, 1); 430 tcg_gen_andc_i32(t, t, arg1); 431 tcg_gen_ctpop_i32(t, t); 432 } else { 433 /* Since all non-x86 hosts have clz(0) == 32, don't fight it. */ 434 tcg_gen_neg_i32(t, arg1); 435 tcg_gen_and_i32(t, t, arg1); 436 tcg_gen_clzi_i32(t, t, 32); 437 tcg_gen_xori_i32(t, t, 31); 438 } 439 z = tcg_constant_i32(0); 440 tcg_gen_movcond_i32(TCG_COND_EQ, ret, arg1, z, arg2, t); 441 tcg_temp_free_i32(t); 442 } else { 443 gen_helper_ctz_i32(ret, arg1, arg2); 444 } 445 } 446 447 void tcg_gen_ctzi_i32(TCGv_i32 ret, TCGv_i32 arg1, uint32_t arg2) 448 { 449 if (!TCG_TARGET_HAS_ctz_i32 && TCG_TARGET_HAS_ctpop_i32 && arg2 == 32) { 450 /* This equivalence has the advantage of not requiring a fixup. */ 451 TCGv_i32 t = tcg_temp_ebb_new_i32(); 452 tcg_gen_subi_i32(t, arg1, 1); 453 tcg_gen_andc_i32(t, t, arg1); 454 tcg_gen_ctpop_i32(ret, t); 455 tcg_temp_free_i32(t); 456 } else { 457 tcg_gen_ctz_i32(ret, arg1, tcg_constant_i32(arg2)); 458 } 459 } 460 461 void tcg_gen_clrsb_i32(TCGv_i32 ret, TCGv_i32 arg) 462 { 463 if (TCG_TARGET_HAS_clz_i32) { 464 TCGv_i32 t = tcg_temp_ebb_new_i32(); 465 tcg_gen_sari_i32(t, arg, 31); 466 tcg_gen_xor_i32(t, t, arg); 467 tcg_gen_clzi_i32(t, t, 32); 468 tcg_gen_subi_i32(ret, t, 1); 469 tcg_temp_free_i32(t); 470 } else { 471 gen_helper_clrsb_i32(ret, arg); 472 } 473 } 474 475 void tcg_gen_ctpop_i32(TCGv_i32 ret, TCGv_i32 arg1) 476 { 477 if (TCG_TARGET_HAS_ctpop_i32) { 478 tcg_gen_op2_i32(INDEX_op_ctpop_i32, ret, arg1); 479 } else if (TCG_TARGET_HAS_ctpop_i64) { 480 TCGv_i64 t = tcg_temp_ebb_new_i64(); 481 tcg_gen_extu_i32_i64(t, arg1); 482 tcg_gen_ctpop_i64(t, t); 483 tcg_gen_extrl_i64_i32(ret, t); 484 tcg_temp_free_i64(t); 485 } else { 486 gen_helper_ctpop_i32(ret, arg1); 487 } 488 } 489 490 void tcg_gen_rotl_i32(TCGv_i32 ret, TCGv_i32 arg1, TCGv_i32 arg2) 491 { 492 if (TCG_TARGET_HAS_rot_i32) { 493 tcg_gen_op3_i32(INDEX_op_rotl_i32, ret, arg1, arg2); 494 } else { 495 TCGv_i32 t0, t1; 496 497 t0 = tcg_temp_ebb_new_i32(); 498 t1 = tcg_temp_ebb_new_i32(); 499 tcg_gen_shl_i32(t0, arg1, arg2); 500 tcg_gen_subfi_i32(t1, 32, arg2); 501 tcg_gen_shr_i32(t1, arg1, t1); 502 tcg_gen_or_i32(ret, t0, t1); 503 tcg_temp_free_i32(t0); 504 tcg_temp_free_i32(t1); 505 } 506 } 507 508 void tcg_gen_rotli_i32(TCGv_i32 ret, TCGv_i32 arg1, int32_t arg2) 509 { 510 tcg_debug_assert(arg2 >= 0 && arg2 < 32); 511 /* some cases can be optimized here */ 512 if (arg2 == 0) { 513 tcg_gen_mov_i32(ret, arg1); 514 } else if (TCG_TARGET_HAS_rot_i32) { 515 tcg_gen_rotl_i32(ret, arg1, tcg_constant_i32(arg2)); 516 } else { 517 TCGv_i32 t0, t1; 518 t0 = tcg_temp_ebb_new_i32(); 519 t1 = tcg_temp_ebb_new_i32(); 520 tcg_gen_shli_i32(t0, arg1, arg2); 521 tcg_gen_shri_i32(t1, arg1, 32 - arg2); 522 tcg_gen_or_i32(ret, t0, t1); 523 tcg_temp_free_i32(t0); 524 tcg_temp_free_i32(t1); 525 } 526 } 527 528 void tcg_gen_rotr_i32(TCGv_i32 ret, TCGv_i32 arg1, TCGv_i32 arg2) 529 { 530 if (TCG_TARGET_HAS_rot_i32) { 531 tcg_gen_op3_i32(INDEX_op_rotr_i32, ret, arg1, arg2); 532 } else { 533 TCGv_i32 t0, t1; 534 535 t0 = tcg_temp_ebb_new_i32(); 536 t1 = tcg_temp_ebb_new_i32(); 537 tcg_gen_shr_i32(t0, arg1, arg2); 538 tcg_gen_subfi_i32(t1, 32, arg2); 539 tcg_gen_shl_i32(t1, arg1, t1); 540 tcg_gen_or_i32(ret, t0, t1); 541 tcg_temp_free_i32(t0); 542 tcg_temp_free_i32(t1); 543 } 544 } 545 546 void tcg_gen_rotri_i32(TCGv_i32 ret, TCGv_i32 arg1, int32_t arg2) 547 { 548 tcg_debug_assert(arg2 >= 0 && arg2 < 32); 549 /* some cases can be optimized here */ 550 if (arg2 == 0) { 551 tcg_gen_mov_i32(ret, arg1); 552 } else { 553 tcg_gen_rotli_i32(ret, arg1, 32 - arg2); 554 } 555 } 556 557 void tcg_gen_deposit_i32(TCGv_i32 ret, TCGv_i32 arg1, TCGv_i32 arg2, 558 unsigned int ofs, unsigned int len) 559 { 560 uint32_t mask; 561 TCGv_i32 t1; 562 563 tcg_debug_assert(ofs < 32); 564 tcg_debug_assert(len > 0); 565 tcg_debug_assert(len <= 32); 566 tcg_debug_assert(ofs + len <= 32); 567 568 if (len == 32) { 569 tcg_gen_mov_i32(ret, arg2); 570 return; 571 } 572 if (TCG_TARGET_HAS_deposit_i32 && TCG_TARGET_deposit_i32_valid(ofs, len)) { 573 tcg_gen_op5ii_i32(INDEX_op_deposit_i32, ret, arg1, arg2, ofs, len); 574 return; 575 } 576 577 t1 = tcg_temp_ebb_new_i32(); 578 579 if (TCG_TARGET_HAS_extract2_i32) { 580 if (ofs + len == 32) { 581 tcg_gen_shli_i32(t1, arg1, len); 582 tcg_gen_extract2_i32(ret, t1, arg2, len); 583 goto done; 584 } 585 if (ofs == 0) { 586 tcg_gen_extract2_i32(ret, arg1, arg2, len); 587 tcg_gen_rotli_i32(ret, ret, len); 588 goto done; 589 } 590 } 591 592 mask = (1u << len) - 1; 593 if (ofs + len < 32) { 594 tcg_gen_andi_i32(t1, arg2, mask); 595 tcg_gen_shli_i32(t1, t1, ofs); 596 } else { 597 tcg_gen_shli_i32(t1, arg2, ofs); 598 } 599 tcg_gen_andi_i32(ret, arg1, ~(mask << ofs)); 600 tcg_gen_or_i32(ret, ret, t1); 601 done: 602 tcg_temp_free_i32(t1); 603 } 604 605 void tcg_gen_deposit_z_i32(TCGv_i32 ret, TCGv_i32 arg, 606 unsigned int ofs, unsigned int len) 607 { 608 tcg_debug_assert(ofs < 32); 609 tcg_debug_assert(len > 0); 610 tcg_debug_assert(len <= 32); 611 tcg_debug_assert(ofs + len <= 32); 612 613 if (ofs + len == 32) { 614 tcg_gen_shli_i32(ret, arg, ofs); 615 } else if (ofs == 0) { 616 tcg_gen_andi_i32(ret, arg, (1u << len) - 1); 617 } else if (TCG_TARGET_HAS_deposit_i32 618 && TCG_TARGET_deposit_i32_valid(ofs, len)) { 619 TCGv_i32 zero = tcg_constant_i32(0); 620 tcg_gen_op5ii_i32(INDEX_op_deposit_i32, ret, zero, arg, ofs, len); 621 } else { 622 /* To help two-operand hosts we prefer to zero-extend first, 623 which allows ARG to stay live. */ 624 switch (len) { 625 case 16: 626 if (TCG_TARGET_HAS_ext16u_i32) { 627 tcg_gen_ext16u_i32(ret, arg); 628 tcg_gen_shli_i32(ret, ret, ofs); 629 return; 630 } 631 break; 632 case 8: 633 if (TCG_TARGET_HAS_ext8u_i32) { 634 tcg_gen_ext8u_i32(ret, arg); 635 tcg_gen_shli_i32(ret, ret, ofs); 636 return; 637 } 638 break; 639 } 640 /* Otherwise prefer zero-extension over AND for code size. */ 641 switch (ofs + len) { 642 case 16: 643 if (TCG_TARGET_HAS_ext16u_i32) { 644 tcg_gen_shli_i32(ret, arg, ofs); 645 tcg_gen_ext16u_i32(ret, ret); 646 return; 647 } 648 break; 649 case 8: 650 if (TCG_TARGET_HAS_ext8u_i32) { 651 tcg_gen_shli_i32(ret, arg, ofs); 652 tcg_gen_ext8u_i32(ret, ret); 653 return; 654 } 655 break; 656 } 657 tcg_gen_andi_i32(ret, arg, (1u << len) - 1); 658 tcg_gen_shli_i32(ret, ret, ofs); 659 } 660 } 661 662 void tcg_gen_extract_i32(TCGv_i32 ret, TCGv_i32 arg, 663 unsigned int ofs, unsigned int len) 664 { 665 tcg_debug_assert(ofs < 32); 666 tcg_debug_assert(len > 0); 667 tcg_debug_assert(len <= 32); 668 tcg_debug_assert(ofs + len <= 32); 669 670 /* Canonicalize certain special cases, even if extract is supported. */ 671 if (ofs + len == 32) { 672 tcg_gen_shri_i32(ret, arg, 32 - len); 673 return; 674 } 675 if (ofs == 0) { 676 tcg_gen_andi_i32(ret, arg, (1u << len) - 1); 677 return; 678 } 679 680 if (TCG_TARGET_HAS_extract_i32 681 && TCG_TARGET_extract_i32_valid(ofs, len)) { 682 tcg_gen_op4ii_i32(INDEX_op_extract_i32, ret, arg, ofs, len); 683 return; 684 } 685 686 /* Assume that zero-extension, if available, is cheaper than a shift. */ 687 switch (ofs + len) { 688 case 16: 689 if (TCG_TARGET_HAS_ext16u_i32) { 690 tcg_gen_ext16u_i32(ret, arg); 691 tcg_gen_shri_i32(ret, ret, ofs); 692 return; 693 } 694 break; 695 case 8: 696 if (TCG_TARGET_HAS_ext8u_i32) { 697 tcg_gen_ext8u_i32(ret, arg); 698 tcg_gen_shri_i32(ret, ret, ofs); 699 return; 700 } 701 break; 702 } 703 704 /* ??? Ideally we'd know what values are available for immediate AND. 705 Assume that 8 bits are available, plus the special case of 16, 706 so that we get ext8u, ext16u. */ 707 switch (len) { 708 case 1 ... 8: case 16: 709 tcg_gen_shri_i32(ret, arg, ofs); 710 tcg_gen_andi_i32(ret, ret, (1u << len) - 1); 711 break; 712 default: 713 tcg_gen_shli_i32(ret, arg, 32 - len - ofs); 714 tcg_gen_shri_i32(ret, ret, 32 - len); 715 break; 716 } 717 } 718 719 void tcg_gen_sextract_i32(TCGv_i32 ret, TCGv_i32 arg, 720 unsigned int ofs, unsigned int len) 721 { 722 tcg_debug_assert(ofs < 32); 723 tcg_debug_assert(len > 0); 724 tcg_debug_assert(len <= 32); 725 tcg_debug_assert(ofs + len <= 32); 726 727 /* Canonicalize certain special cases, even if extract is supported. */ 728 if (ofs + len == 32) { 729 tcg_gen_sari_i32(ret, arg, 32 - len); 730 return; 731 } 732 if (ofs == 0) { 733 switch (len) { 734 case 16: 735 tcg_gen_ext16s_i32(ret, arg); 736 return; 737 case 8: 738 tcg_gen_ext8s_i32(ret, arg); 739 return; 740 } 741 } 742 743 if (TCG_TARGET_HAS_sextract_i32 744 && TCG_TARGET_extract_i32_valid(ofs, len)) { 745 tcg_gen_op4ii_i32(INDEX_op_sextract_i32, ret, arg, ofs, len); 746 return; 747 } 748 749 /* Assume that sign-extension, if available, is cheaper than a shift. */ 750 switch (ofs + len) { 751 case 16: 752 if (TCG_TARGET_HAS_ext16s_i32) { 753 tcg_gen_ext16s_i32(ret, arg); 754 tcg_gen_sari_i32(ret, ret, ofs); 755 return; 756 } 757 break; 758 case 8: 759 if (TCG_TARGET_HAS_ext8s_i32) { 760 tcg_gen_ext8s_i32(ret, arg); 761 tcg_gen_sari_i32(ret, ret, ofs); 762 return; 763 } 764 break; 765 } 766 switch (len) { 767 case 16: 768 if (TCG_TARGET_HAS_ext16s_i32) { 769 tcg_gen_shri_i32(ret, arg, ofs); 770 tcg_gen_ext16s_i32(ret, ret); 771 return; 772 } 773 break; 774 case 8: 775 if (TCG_TARGET_HAS_ext8s_i32) { 776 tcg_gen_shri_i32(ret, arg, ofs); 777 tcg_gen_ext8s_i32(ret, ret); 778 return; 779 } 780 break; 781 } 782 783 tcg_gen_shli_i32(ret, arg, 32 - len - ofs); 784 tcg_gen_sari_i32(ret, ret, 32 - len); 785 } 786 787 /* 788 * Extract 32-bits from a 64-bit input, ah:al, starting from ofs. 789 * Unlike tcg_gen_extract_i32 above, len is fixed at 32. 790 */ 791 void tcg_gen_extract2_i32(TCGv_i32 ret, TCGv_i32 al, TCGv_i32 ah, 792 unsigned int ofs) 793 { 794 tcg_debug_assert(ofs <= 32); 795 if (ofs == 0) { 796 tcg_gen_mov_i32(ret, al); 797 } else if (ofs == 32) { 798 tcg_gen_mov_i32(ret, ah); 799 } else if (al == ah) { 800 tcg_gen_rotri_i32(ret, al, ofs); 801 } else if (TCG_TARGET_HAS_extract2_i32) { 802 tcg_gen_op4i_i32(INDEX_op_extract2_i32, ret, al, ah, ofs); 803 } else { 804 TCGv_i32 t0 = tcg_temp_ebb_new_i32(); 805 tcg_gen_shri_i32(t0, al, ofs); 806 tcg_gen_deposit_i32(ret, t0, ah, 32 - ofs, ofs); 807 tcg_temp_free_i32(t0); 808 } 809 } 810 811 void tcg_gen_movcond_i32(TCGCond cond, TCGv_i32 ret, TCGv_i32 c1, 812 TCGv_i32 c2, TCGv_i32 v1, TCGv_i32 v2) 813 { 814 if (cond == TCG_COND_ALWAYS) { 815 tcg_gen_mov_i32(ret, v1); 816 } else if (cond == TCG_COND_NEVER) { 817 tcg_gen_mov_i32(ret, v2); 818 } else if (TCG_TARGET_HAS_movcond_i32) { 819 tcg_gen_op6i_i32(INDEX_op_movcond_i32, ret, c1, c2, v1, v2, cond); 820 } else { 821 TCGv_i32 t0 = tcg_temp_ebb_new_i32(); 822 TCGv_i32 t1 = tcg_temp_ebb_new_i32(); 823 tcg_gen_setcond_i32(cond, t0, c1, c2); 824 tcg_gen_neg_i32(t0, t0); 825 tcg_gen_and_i32(t1, v1, t0); 826 tcg_gen_andc_i32(ret, v2, t0); 827 tcg_gen_or_i32(ret, ret, t1); 828 tcg_temp_free_i32(t0); 829 tcg_temp_free_i32(t1); 830 } 831 } 832 833 void tcg_gen_add2_i32(TCGv_i32 rl, TCGv_i32 rh, TCGv_i32 al, 834 TCGv_i32 ah, TCGv_i32 bl, TCGv_i32 bh) 835 { 836 if (TCG_TARGET_HAS_add2_i32) { 837 tcg_gen_op6_i32(INDEX_op_add2_i32, rl, rh, al, ah, bl, bh); 838 } else { 839 TCGv_i64 t0 = tcg_temp_ebb_new_i64(); 840 TCGv_i64 t1 = tcg_temp_ebb_new_i64(); 841 tcg_gen_concat_i32_i64(t0, al, ah); 842 tcg_gen_concat_i32_i64(t1, bl, bh); 843 tcg_gen_add_i64(t0, t0, t1); 844 tcg_gen_extr_i64_i32(rl, rh, t0); 845 tcg_temp_free_i64(t0); 846 tcg_temp_free_i64(t1); 847 } 848 } 849 850 void tcg_gen_sub2_i32(TCGv_i32 rl, TCGv_i32 rh, TCGv_i32 al, 851 TCGv_i32 ah, TCGv_i32 bl, TCGv_i32 bh) 852 { 853 if (TCG_TARGET_HAS_sub2_i32) { 854 tcg_gen_op6_i32(INDEX_op_sub2_i32, rl, rh, al, ah, bl, bh); 855 } else { 856 TCGv_i64 t0 = tcg_temp_ebb_new_i64(); 857 TCGv_i64 t1 = tcg_temp_ebb_new_i64(); 858 tcg_gen_concat_i32_i64(t0, al, ah); 859 tcg_gen_concat_i32_i64(t1, bl, bh); 860 tcg_gen_sub_i64(t0, t0, t1); 861 tcg_gen_extr_i64_i32(rl, rh, t0); 862 tcg_temp_free_i64(t0); 863 tcg_temp_free_i64(t1); 864 } 865 } 866 867 void tcg_gen_mulu2_i32(TCGv_i32 rl, TCGv_i32 rh, TCGv_i32 arg1, TCGv_i32 arg2) 868 { 869 if (TCG_TARGET_HAS_mulu2_i32) { 870 tcg_gen_op4_i32(INDEX_op_mulu2_i32, rl, rh, arg1, arg2); 871 } else if (TCG_TARGET_HAS_muluh_i32) { 872 TCGv_i32 t = tcg_temp_ebb_new_i32(); 873 tcg_gen_op3_i32(INDEX_op_mul_i32, t, arg1, arg2); 874 tcg_gen_op3_i32(INDEX_op_muluh_i32, rh, arg1, arg2); 875 tcg_gen_mov_i32(rl, t); 876 tcg_temp_free_i32(t); 877 } else if (TCG_TARGET_REG_BITS == 64) { 878 TCGv_i64 t0 = tcg_temp_ebb_new_i64(); 879 TCGv_i64 t1 = tcg_temp_ebb_new_i64(); 880 tcg_gen_extu_i32_i64(t0, arg1); 881 tcg_gen_extu_i32_i64(t1, arg2); 882 tcg_gen_mul_i64(t0, t0, t1); 883 tcg_gen_extr_i64_i32(rl, rh, t0); 884 tcg_temp_free_i64(t0); 885 tcg_temp_free_i64(t1); 886 } else { 887 qemu_build_not_reached(); 888 } 889 } 890 891 void tcg_gen_muls2_i32(TCGv_i32 rl, TCGv_i32 rh, TCGv_i32 arg1, TCGv_i32 arg2) 892 { 893 if (TCG_TARGET_HAS_muls2_i32) { 894 tcg_gen_op4_i32(INDEX_op_muls2_i32, rl, rh, arg1, arg2); 895 } else if (TCG_TARGET_HAS_mulsh_i32) { 896 TCGv_i32 t = tcg_temp_ebb_new_i32(); 897 tcg_gen_op3_i32(INDEX_op_mul_i32, t, arg1, arg2); 898 tcg_gen_op3_i32(INDEX_op_mulsh_i32, rh, arg1, arg2); 899 tcg_gen_mov_i32(rl, t); 900 tcg_temp_free_i32(t); 901 } else if (TCG_TARGET_REG_BITS == 32) { 902 TCGv_i32 t0 = tcg_temp_ebb_new_i32(); 903 TCGv_i32 t1 = tcg_temp_ebb_new_i32(); 904 TCGv_i32 t2 = tcg_temp_ebb_new_i32(); 905 TCGv_i32 t3 = tcg_temp_ebb_new_i32(); 906 tcg_gen_mulu2_i32(t0, t1, arg1, arg2); 907 /* Adjust for negative inputs. */ 908 tcg_gen_sari_i32(t2, arg1, 31); 909 tcg_gen_sari_i32(t3, arg2, 31); 910 tcg_gen_and_i32(t2, t2, arg2); 911 tcg_gen_and_i32(t3, t3, arg1); 912 tcg_gen_sub_i32(rh, t1, t2); 913 tcg_gen_sub_i32(rh, rh, t3); 914 tcg_gen_mov_i32(rl, t0); 915 tcg_temp_free_i32(t0); 916 tcg_temp_free_i32(t1); 917 tcg_temp_free_i32(t2); 918 tcg_temp_free_i32(t3); 919 } else { 920 TCGv_i64 t0 = tcg_temp_ebb_new_i64(); 921 TCGv_i64 t1 = tcg_temp_ebb_new_i64(); 922 tcg_gen_ext_i32_i64(t0, arg1); 923 tcg_gen_ext_i32_i64(t1, arg2); 924 tcg_gen_mul_i64(t0, t0, t1); 925 tcg_gen_extr_i64_i32(rl, rh, t0); 926 tcg_temp_free_i64(t0); 927 tcg_temp_free_i64(t1); 928 } 929 } 930 931 void tcg_gen_mulsu2_i32(TCGv_i32 rl, TCGv_i32 rh, TCGv_i32 arg1, TCGv_i32 arg2) 932 { 933 if (TCG_TARGET_REG_BITS == 32) { 934 TCGv_i32 t0 = tcg_temp_ebb_new_i32(); 935 TCGv_i32 t1 = tcg_temp_ebb_new_i32(); 936 TCGv_i32 t2 = tcg_temp_ebb_new_i32(); 937 tcg_gen_mulu2_i32(t0, t1, arg1, arg2); 938 /* Adjust for negative input for the signed arg1. */ 939 tcg_gen_sari_i32(t2, arg1, 31); 940 tcg_gen_and_i32(t2, t2, arg2); 941 tcg_gen_sub_i32(rh, t1, t2); 942 tcg_gen_mov_i32(rl, t0); 943 tcg_temp_free_i32(t0); 944 tcg_temp_free_i32(t1); 945 tcg_temp_free_i32(t2); 946 } else { 947 TCGv_i64 t0 = tcg_temp_ebb_new_i64(); 948 TCGv_i64 t1 = tcg_temp_ebb_new_i64(); 949 tcg_gen_ext_i32_i64(t0, arg1); 950 tcg_gen_extu_i32_i64(t1, arg2); 951 tcg_gen_mul_i64(t0, t0, t1); 952 tcg_gen_extr_i64_i32(rl, rh, t0); 953 tcg_temp_free_i64(t0); 954 tcg_temp_free_i64(t1); 955 } 956 } 957 958 void tcg_gen_ext8s_i32(TCGv_i32 ret, TCGv_i32 arg) 959 { 960 if (TCG_TARGET_HAS_ext8s_i32) { 961 tcg_gen_op2_i32(INDEX_op_ext8s_i32, ret, arg); 962 } else { 963 tcg_gen_shli_i32(ret, arg, 24); 964 tcg_gen_sari_i32(ret, ret, 24); 965 } 966 } 967 968 void tcg_gen_ext16s_i32(TCGv_i32 ret, TCGv_i32 arg) 969 { 970 if (TCG_TARGET_HAS_ext16s_i32) { 971 tcg_gen_op2_i32(INDEX_op_ext16s_i32, ret, arg); 972 } else { 973 tcg_gen_shli_i32(ret, arg, 16); 974 tcg_gen_sari_i32(ret, ret, 16); 975 } 976 } 977 978 void tcg_gen_ext8u_i32(TCGv_i32 ret, TCGv_i32 arg) 979 { 980 if (TCG_TARGET_HAS_ext8u_i32) { 981 tcg_gen_op2_i32(INDEX_op_ext8u_i32, ret, arg); 982 } else { 983 tcg_gen_andi_i32(ret, arg, 0xffu); 984 } 985 } 986 987 void tcg_gen_ext16u_i32(TCGv_i32 ret, TCGv_i32 arg) 988 { 989 if (TCG_TARGET_HAS_ext16u_i32) { 990 tcg_gen_op2_i32(INDEX_op_ext16u_i32, ret, arg); 991 } else { 992 tcg_gen_andi_i32(ret, arg, 0xffffu); 993 } 994 } 995 996 void tcg_gen_bswap16_i32(TCGv_i32 ret, TCGv_i32 arg, int flags) 997 { 998 /* Only one extension flag may be present. */ 999 tcg_debug_assert(!(flags & TCG_BSWAP_OS) || !(flags & TCG_BSWAP_OZ)); 1000 1001 if (TCG_TARGET_HAS_bswap16_i32) { 1002 tcg_gen_op3i_i32(INDEX_op_bswap16_i32, ret, arg, flags); 1003 } else { 1004 TCGv_i32 t0 = tcg_temp_ebb_new_i32(); 1005 TCGv_i32 t1 = tcg_temp_ebb_new_i32(); 1006 1007 tcg_gen_shri_i32(t0, arg, 8); 1008 if (!(flags & TCG_BSWAP_IZ)) { 1009 tcg_gen_ext8u_i32(t0, t0); 1010 } 1011 1012 if (flags & TCG_BSWAP_OS) { 1013 tcg_gen_shli_i32(t1, arg, 24); 1014 tcg_gen_sari_i32(t1, t1, 16); 1015 } else if (flags & TCG_BSWAP_OZ) { 1016 tcg_gen_ext8u_i32(t1, arg); 1017 tcg_gen_shli_i32(t1, t1, 8); 1018 } else { 1019 tcg_gen_shli_i32(t1, arg, 8); 1020 } 1021 1022 tcg_gen_or_i32(ret, t0, t1); 1023 tcg_temp_free_i32(t0); 1024 tcg_temp_free_i32(t1); 1025 } 1026 } 1027 1028 void tcg_gen_bswap32_i32(TCGv_i32 ret, TCGv_i32 arg) 1029 { 1030 if (TCG_TARGET_HAS_bswap32_i32) { 1031 tcg_gen_op3i_i32(INDEX_op_bswap32_i32, ret, arg, 0); 1032 } else { 1033 TCGv_i32 t0 = tcg_temp_ebb_new_i32(); 1034 TCGv_i32 t1 = tcg_temp_ebb_new_i32(); 1035 TCGv_i32 t2 = tcg_constant_i32(0x00ff00ff); 1036 1037 /* arg = abcd */ 1038 tcg_gen_shri_i32(t0, arg, 8); /* t0 = .abc */ 1039 tcg_gen_and_i32(t1, arg, t2); /* t1 = .b.d */ 1040 tcg_gen_and_i32(t0, t0, t2); /* t0 = .a.c */ 1041 tcg_gen_shli_i32(t1, t1, 8); /* t1 = b.d. */ 1042 tcg_gen_or_i32(ret, t0, t1); /* ret = badc */ 1043 1044 tcg_gen_shri_i32(t0, ret, 16); /* t0 = ..ba */ 1045 tcg_gen_shli_i32(t1, ret, 16); /* t1 = dc.. */ 1046 tcg_gen_or_i32(ret, t0, t1); /* ret = dcba */ 1047 1048 tcg_temp_free_i32(t0); 1049 tcg_temp_free_i32(t1); 1050 } 1051 } 1052 1053 void tcg_gen_hswap_i32(TCGv_i32 ret, TCGv_i32 arg) 1054 { 1055 /* Swapping 2 16-bit elements is a rotate. */ 1056 tcg_gen_rotli_i32(ret, arg, 16); 1057 } 1058 1059 void tcg_gen_smin_i32(TCGv_i32 ret, TCGv_i32 a, TCGv_i32 b) 1060 { 1061 tcg_gen_movcond_i32(TCG_COND_LT, ret, a, b, a, b); 1062 } 1063 1064 void tcg_gen_umin_i32(TCGv_i32 ret, TCGv_i32 a, TCGv_i32 b) 1065 { 1066 tcg_gen_movcond_i32(TCG_COND_LTU, ret, a, b, a, b); 1067 } 1068 1069 void tcg_gen_smax_i32(TCGv_i32 ret, TCGv_i32 a, TCGv_i32 b) 1070 { 1071 tcg_gen_movcond_i32(TCG_COND_LT, ret, a, b, b, a); 1072 } 1073 1074 void tcg_gen_umax_i32(TCGv_i32 ret, TCGv_i32 a, TCGv_i32 b) 1075 { 1076 tcg_gen_movcond_i32(TCG_COND_LTU, ret, a, b, b, a); 1077 } 1078 1079 void tcg_gen_abs_i32(TCGv_i32 ret, TCGv_i32 a) 1080 { 1081 TCGv_i32 t = tcg_temp_ebb_new_i32(); 1082 1083 tcg_gen_sari_i32(t, a, 31); 1084 tcg_gen_xor_i32(ret, a, t); 1085 tcg_gen_sub_i32(ret, ret, t); 1086 tcg_temp_free_i32(t); 1087 } 1088 1089 /* 64-bit ops */ 1090 1091 #if TCG_TARGET_REG_BITS == 32 1092 /* These are all inline for TCG_TARGET_REG_BITS == 64. */ 1093 1094 void tcg_gen_discard_i64(TCGv_i64 arg) 1095 { 1096 tcg_gen_discard_i32(TCGV_LOW(arg)); 1097 tcg_gen_discard_i32(TCGV_HIGH(arg)); 1098 } 1099 1100 void tcg_gen_mov_i64(TCGv_i64 ret, TCGv_i64 arg) 1101 { 1102 TCGTemp *ts = tcgv_i64_temp(arg); 1103 1104 /* Canonicalize TCGv_i64 TEMP_CONST into TCGv_i32 TEMP_CONST. */ 1105 if (ts->kind == TEMP_CONST) { 1106 tcg_gen_movi_i64(ret, ts->val); 1107 } else { 1108 tcg_gen_mov_i32(TCGV_LOW(ret), TCGV_LOW(arg)); 1109 tcg_gen_mov_i32(TCGV_HIGH(ret), TCGV_HIGH(arg)); 1110 } 1111 } 1112 1113 void tcg_gen_movi_i64(TCGv_i64 ret, int64_t arg) 1114 { 1115 tcg_gen_movi_i32(TCGV_LOW(ret), arg); 1116 tcg_gen_movi_i32(TCGV_HIGH(ret), arg >> 32); 1117 } 1118 1119 void tcg_gen_ld8u_i64(TCGv_i64 ret, TCGv_ptr arg2, tcg_target_long offset) 1120 { 1121 tcg_gen_ld8u_i32(TCGV_LOW(ret), arg2, offset); 1122 tcg_gen_movi_i32(TCGV_HIGH(ret), 0); 1123 } 1124 1125 void tcg_gen_ld8s_i64(TCGv_i64 ret, TCGv_ptr arg2, tcg_target_long offset) 1126 { 1127 tcg_gen_ld8s_i32(TCGV_LOW(ret), arg2, offset); 1128 tcg_gen_sari_i32(TCGV_HIGH(ret), TCGV_LOW(ret), 31); 1129 } 1130 1131 void tcg_gen_ld16u_i64(TCGv_i64 ret, TCGv_ptr arg2, tcg_target_long offset) 1132 { 1133 tcg_gen_ld16u_i32(TCGV_LOW(ret), arg2, offset); 1134 tcg_gen_movi_i32(TCGV_HIGH(ret), 0); 1135 } 1136 1137 void tcg_gen_ld16s_i64(TCGv_i64 ret, TCGv_ptr arg2, tcg_target_long offset) 1138 { 1139 tcg_gen_ld16s_i32(TCGV_LOW(ret), arg2, offset); 1140 tcg_gen_sari_i32(TCGV_HIGH(ret), TCGV_LOW(ret), 31); 1141 } 1142 1143 void tcg_gen_ld32u_i64(TCGv_i64 ret, TCGv_ptr arg2, tcg_target_long offset) 1144 { 1145 tcg_gen_ld_i32(TCGV_LOW(ret), arg2, offset); 1146 tcg_gen_movi_i32(TCGV_HIGH(ret), 0); 1147 } 1148 1149 void tcg_gen_ld32s_i64(TCGv_i64 ret, TCGv_ptr arg2, tcg_target_long offset) 1150 { 1151 tcg_gen_ld_i32(TCGV_LOW(ret), arg2, offset); 1152 tcg_gen_sari_i32(TCGV_HIGH(ret), TCGV_LOW(ret), 31); 1153 } 1154 1155 void tcg_gen_ld_i64(TCGv_i64 ret, TCGv_ptr arg2, tcg_target_long offset) 1156 { 1157 /* Since arg2 and ret have different types, 1158 they cannot be the same temporary */ 1159 #if HOST_BIG_ENDIAN 1160 tcg_gen_ld_i32(TCGV_HIGH(ret), arg2, offset); 1161 tcg_gen_ld_i32(TCGV_LOW(ret), arg2, offset + 4); 1162 #else 1163 tcg_gen_ld_i32(TCGV_LOW(ret), arg2, offset); 1164 tcg_gen_ld_i32(TCGV_HIGH(ret), arg2, offset + 4); 1165 #endif 1166 } 1167 1168 void tcg_gen_st8_i64(TCGv_i64 arg1, TCGv_ptr arg2, tcg_target_long offset) 1169 { 1170 tcg_gen_st8_i32(TCGV_LOW(arg1), arg2, offset); 1171 } 1172 1173 void tcg_gen_st16_i64(TCGv_i64 arg1, TCGv_ptr arg2, tcg_target_long offset) 1174 { 1175 tcg_gen_st16_i32(TCGV_LOW(arg1), arg2, offset); 1176 } 1177 1178 void tcg_gen_st32_i64(TCGv_i64 arg1, TCGv_ptr arg2, tcg_target_long offset) 1179 { 1180 tcg_gen_st_i32(TCGV_LOW(arg1), arg2, offset); 1181 } 1182 1183 void tcg_gen_st_i64(TCGv_i64 arg1, TCGv_ptr arg2, tcg_target_long offset) 1184 { 1185 #if HOST_BIG_ENDIAN 1186 tcg_gen_st_i32(TCGV_HIGH(arg1), arg2, offset); 1187 tcg_gen_st_i32(TCGV_LOW(arg1), arg2, offset + 4); 1188 #else 1189 tcg_gen_st_i32(TCGV_LOW(arg1), arg2, offset); 1190 tcg_gen_st_i32(TCGV_HIGH(arg1), arg2, offset + 4); 1191 #endif 1192 } 1193 1194 void tcg_gen_add_i64(TCGv_i64 ret, TCGv_i64 arg1, TCGv_i64 arg2) 1195 { 1196 tcg_gen_add2_i32(TCGV_LOW(ret), TCGV_HIGH(ret), TCGV_LOW(arg1), 1197 TCGV_HIGH(arg1), TCGV_LOW(arg2), TCGV_HIGH(arg2)); 1198 } 1199 1200 void tcg_gen_sub_i64(TCGv_i64 ret, TCGv_i64 arg1, TCGv_i64 arg2) 1201 { 1202 tcg_gen_sub2_i32(TCGV_LOW(ret), TCGV_HIGH(ret), TCGV_LOW(arg1), 1203 TCGV_HIGH(arg1), TCGV_LOW(arg2), TCGV_HIGH(arg2)); 1204 } 1205 1206 void tcg_gen_and_i64(TCGv_i64 ret, TCGv_i64 arg1, TCGv_i64 arg2) 1207 { 1208 tcg_gen_and_i32(TCGV_LOW(ret), TCGV_LOW(arg1), TCGV_LOW(arg2)); 1209 tcg_gen_and_i32(TCGV_HIGH(ret), TCGV_HIGH(arg1), TCGV_HIGH(arg2)); 1210 } 1211 1212 void tcg_gen_or_i64(TCGv_i64 ret, TCGv_i64 arg1, TCGv_i64 arg2) 1213 { 1214 tcg_gen_or_i32(TCGV_LOW(ret), TCGV_LOW(arg1), TCGV_LOW(arg2)); 1215 tcg_gen_or_i32(TCGV_HIGH(ret), TCGV_HIGH(arg1), TCGV_HIGH(arg2)); 1216 } 1217 1218 void tcg_gen_xor_i64(TCGv_i64 ret, TCGv_i64 arg1, TCGv_i64 arg2) 1219 { 1220 tcg_gen_xor_i32(TCGV_LOW(ret), TCGV_LOW(arg1), TCGV_LOW(arg2)); 1221 tcg_gen_xor_i32(TCGV_HIGH(ret), TCGV_HIGH(arg1), TCGV_HIGH(arg2)); 1222 } 1223 1224 void tcg_gen_shl_i64(TCGv_i64 ret, TCGv_i64 arg1, TCGv_i64 arg2) 1225 { 1226 gen_helper_shl_i64(ret, arg1, arg2); 1227 } 1228 1229 void tcg_gen_shr_i64(TCGv_i64 ret, TCGv_i64 arg1, TCGv_i64 arg2) 1230 { 1231 gen_helper_shr_i64(ret, arg1, arg2); 1232 } 1233 1234 void tcg_gen_sar_i64(TCGv_i64 ret, TCGv_i64 arg1, TCGv_i64 arg2) 1235 { 1236 gen_helper_sar_i64(ret, arg1, arg2); 1237 } 1238 1239 void tcg_gen_mul_i64(TCGv_i64 ret, TCGv_i64 arg1, TCGv_i64 arg2) 1240 { 1241 TCGv_i64 t0; 1242 TCGv_i32 t1; 1243 1244 t0 = tcg_temp_ebb_new_i64(); 1245 t1 = tcg_temp_ebb_new_i32(); 1246 1247 tcg_gen_mulu2_i32(TCGV_LOW(t0), TCGV_HIGH(t0), 1248 TCGV_LOW(arg1), TCGV_LOW(arg2)); 1249 1250 tcg_gen_mul_i32(t1, TCGV_LOW(arg1), TCGV_HIGH(arg2)); 1251 tcg_gen_add_i32(TCGV_HIGH(t0), TCGV_HIGH(t0), t1); 1252 tcg_gen_mul_i32(t1, TCGV_HIGH(arg1), TCGV_LOW(arg2)); 1253 tcg_gen_add_i32(TCGV_HIGH(t0), TCGV_HIGH(t0), t1); 1254 1255 tcg_gen_mov_i64(ret, t0); 1256 tcg_temp_free_i64(t0); 1257 tcg_temp_free_i32(t1); 1258 } 1259 1260 #else 1261 1262 void tcg_gen_movi_i64(TCGv_i64 ret, int64_t arg) 1263 { 1264 tcg_gen_mov_i64(ret, tcg_constant_i64(arg)); 1265 } 1266 1267 #endif /* TCG_TARGET_REG_SIZE == 32 */ 1268 1269 void tcg_gen_addi_i64(TCGv_i64 ret, TCGv_i64 arg1, int64_t arg2) 1270 { 1271 /* some cases can be optimized here */ 1272 if (arg2 == 0) { 1273 tcg_gen_mov_i64(ret, arg1); 1274 } else if (TCG_TARGET_REG_BITS == 64) { 1275 tcg_gen_add_i64(ret, arg1, tcg_constant_i64(arg2)); 1276 } else { 1277 tcg_gen_add2_i32(TCGV_LOW(ret), TCGV_HIGH(ret), 1278 TCGV_LOW(arg1), TCGV_HIGH(arg1), 1279 tcg_constant_i32(arg2), tcg_constant_i32(arg2 >> 32)); 1280 } 1281 } 1282 1283 void tcg_gen_subfi_i64(TCGv_i64 ret, int64_t arg1, TCGv_i64 arg2) 1284 { 1285 if (arg1 == 0 && TCG_TARGET_HAS_neg_i64) { 1286 /* Don't recurse with tcg_gen_neg_i64. */ 1287 tcg_gen_op2_i64(INDEX_op_neg_i64, ret, arg2); 1288 } else if (TCG_TARGET_REG_BITS == 64) { 1289 tcg_gen_sub_i64(ret, tcg_constant_i64(arg1), arg2); 1290 } else { 1291 tcg_gen_sub2_i32(TCGV_LOW(ret), TCGV_HIGH(ret), 1292 tcg_constant_i32(arg1), tcg_constant_i32(arg1 >> 32), 1293 TCGV_LOW(arg2), TCGV_HIGH(arg2)); 1294 } 1295 } 1296 1297 void tcg_gen_subi_i64(TCGv_i64 ret, TCGv_i64 arg1, int64_t arg2) 1298 { 1299 /* some cases can be optimized here */ 1300 if (arg2 == 0) { 1301 tcg_gen_mov_i64(ret, arg1); 1302 } else if (TCG_TARGET_REG_BITS == 64) { 1303 tcg_gen_sub_i64(ret, arg1, tcg_constant_i64(arg2)); 1304 } else { 1305 tcg_gen_sub2_i32(TCGV_LOW(ret), TCGV_HIGH(ret), 1306 TCGV_LOW(arg1), TCGV_HIGH(arg1), 1307 tcg_constant_i32(arg2), tcg_constant_i32(arg2 >> 32)); 1308 } 1309 } 1310 1311 void tcg_gen_andi_i64(TCGv_i64 ret, TCGv_i64 arg1, int64_t arg2) 1312 { 1313 if (TCG_TARGET_REG_BITS == 32) { 1314 tcg_gen_andi_i32(TCGV_LOW(ret), TCGV_LOW(arg1), arg2); 1315 tcg_gen_andi_i32(TCGV_HIGH(ret), TCGV_HIGH(arg1), arg2 >> 32); 1316 return; 1317 } 1318 1319 /* Some cases can be optimized here. */ 1320 switch (arg2) { 1321 case 0: 1322 tcg_gen_movi_i64(ret, 0); 1323 return; 1324 case -1: 1325 tcg_gen_mov_i64(ret, arg1); 1326 return; 1327 case 0xff: 1328 /* Don't recurse with tcg_gen_ext8u_i64. */ 1329 if (TCG_TARGET_HAS_ext8u_i64) { 1330 tcg_gen_op2_i64(INDEX_op_ext8u_i64, ret, arg1); 1331 return; 1332 } 1333 break; 1334 case 0xffff: 1335 if (TCG_TARGET_HAS_ext16u_i64) { 1336 tcg_gen_op2_i64(INDEX_op_ext16u_i64, ret, arg1); 1337 return; 1338 } 1339 break; 1340 case 0xffffffffu: 1341 if (TCG_TARGET_HAS_ext32u_i64) { 1342 tcg_gen_op2_i64(INDEX_op_ext32u_i64, ret, arg1); 1343 return; 1344 } 1345 break; 1346 } 1347 1348 tcg_gen_and_i64(ret, arg1, tcg_constant_i64(arg2)); 1349 } 1350 1351 void tcg_gen_ori_i64(TCGv_i64 ret, TCGv_i64 arg1, int64_t arg2) 1352 { 1353 if (TCG_TARGET_REG_BITS == 32) { 1354 tcg_gen_ori_i32(TCGV_LOW(ret), TCGV_LOW(arg1), arg2); 1355 tcg_gen_ori_i32(TCGV_HIGH(ret), TCGV_HIGH(arg1), arg2 >> 32); 1356 return; 1357 } 1358 /* Some cases can be optimized here. */ 1359 if (arg2 == -1) { 1360 tcg_gen_movi_i64(ret, -1); 1361 } else if (arg2 == 0) { 1362 tcg_gen_mov_i64(ret, arg1); 1363 } else { 1364 tcg_gen_or_i64(ret, arg1, tcg_constant_i64(arg2)); 1365 } 1366 } 1367 1368 void tcg_gen_xori_i64(TCGv_i64 ret, TCGv_i64 arg1, int64_t arg2) 1369 { 1370 if (TCG_TARGET_REG_BITS == 32) { 1371 tcg_gen_xori_i32(TCGV_LOW(ret), TCGV_LOW(arg1), arg2); 1372 tcg_gen_xori_i32(TCGV_HIGH(ret), TCGV_HIGH(arg1), arg2 >> 32); 1373 return; 1374 } 1375 /* Some cases can be optimized here. */ 1376 if (arg2 == 0) { 1377 tcg_gen_mov_i64(ret, arg1); 1378 } else if (arg2 == -1 && TCG_TARGET_HAS_not_i64) { 1379 /* Don't recurse with tcg_gen_not_i64. */ 1380 tcg_gen_op2_i64(INDEX_op_not_i64, ret, arg1); 1381 } else { 1382 tcg_gen_xor_i64(ret, arg1, tcg_constant_i64(arg2)); 1383 } 1384 } 1385 1386 static inline void tcg_gen_shifti_i64(TCGv_i64 ret, TCGv_i64 arg1, 1387 unsigned c, bool right, bool arith) 1388 { 1389 tcg_debug_assert(c < 64); 1390 if (c == 0) { 1391 tcg_gen_mov_i32(TCGV_LOW(ret), TCGV_LOW(arg1)); 1392 tcg_gen_mov_i32(TCGV_HIGH(ret), TCGV_HIGH(arg1)); 1393 } else if (c >= 32) { 1394 c -= 32; 1395 if (right) { 1396 if (arith) { 1397 tcg_gen_sari_i32(TCGV_LOW(ret), TCGV_HIGH(arg1), c); 1398 tcg_gen_sari_i32(TCGV_HIGH(ret), TCGV_HIGH(arg1), 31); 1399 } else { 1400 tcg_gen_shri_i32(TCGV_LOW(ret), TCGV_HIGH(arg1), c); 1401 tcg_gen_movi_i32(TCGV_HIGH(ret), 0); 1402 } 1403 } else { 1404 tcg_gen_shli_i32(TCGV_HIGH(ret), TCGV_LOW(arg1), c); 1405 tcg_gen_movi_i32(TCGV_LOW(ret), 0); 1406 } 1407 } else if (right) { 1408 if (TCG_TARGET_HAS_extract2_i32) { 1409 tcg_gen_extract2_i32(TCGV_LOW(ret), 1410 TCGV_LOW(arg1), TCGV_HIGH(arg1), c); 1411 } else { 1412 tcg_gen_shri_i32(TCGV_LOW(ret), TCGV_LOW(arg1), c); 1413 tcg_gen_deposit_i32(TCGV_LOW(ret), TCGV_LOW(ret), 1414 TCGV_HIGH(arg1), 32 - c, c); 1415 } 1416 if (arith) { 1417 tcg_gen_sari_i32(TCGV_HIGH(ret), TCGV_HIGH(arg1), c); 1418 } else { 1419 tcg_gen_shri_i32(TCGV_HIGH(ret), TCGV_HIGH(arg1), c); 1420 } 1421 } else { 1422 if (TCG_TARGET_HAS_extract2_i32) { 1423 tcg_gen_extract2_i32(TCGV_HIGH(ret), 1424 TCGV_LOW(arg1), TCGV_HIGH(arg1), 32 - c); 1425 } else { 1426 TCGv_i32 t0 = tcg_temp_ebb_new_i32(); 1427 tcg_gen_shri_i32(t0, TCGV_LOW(arg1), 32 - c); 1428 tcg_gen_deposit_i32(TCGV_HIGH(ret), t0, 1429 TCGV_HIGH(arg1), c, 32 - c); 1430 tcg_temp_free_i32(t0); 1431 } 1432 tcg_gen_shli_i32(TCGV_LOW(ret), TCGV_LOW(arg1), c); 1433 } 1434 } 1435 1436 void tcg_gen_shli_i64(TCGv_i64 ret, TCGv_i64 arg1, int64_t arg2) 1437 { 1438 tcg_debug_assert(arg2 >= 0 && arg2 < 64); 1439 if (TCG_TARGET_REG_BITS == 32) { 1440 tcg_gen_shifti_i64(ret, arg1, arg2, 0, 0); 1441 } else if (arg2 == 0) { 1442 tcg_gen_mov_i64(ret, arg1); 1443 } else { 1444 tcg_gen_shl_i64(ret, arg1, tcg_constant_i64(arg2)); 1445 } 1446 } 1447 1448 void tcg_gen_shri_i64(TCGv_i64 ret, TCGv_i64 arg1, int64_t arg2) 1449 { 1450 tcg_debug_assert(arg2 >= 0 && arg2 < 64); 1451 if (TCG_TARGET_REG_BITS == 32) { 1452 tcg_gen_shifti_i64(ret, arg1, arg2, 1, 0); 1453 } else if (arg2 == 0) { 1454 tcg_gen_mov_i64(ret, arg1); 1455 } else { 1456 tcg_gen_shr_i64(ret, arg1, tcg_constant_i64(arg2)); 1457 } 1458 } 1459 1460 void tcg_gen_sari_i64(TCGv_i64 ret, TCGv_i64 arg1, int64_t arg2) 1461 { 1462 tcg_debug_assert(arg2 >= 0 && arg2 < 64); 1463 if (TCG_TARGET_REG_BITS == 32) { 1464 tcg_gen_shifti_i64(ret, arg1, arg2, 1, 1); 1465 } else if (arg2 == 0) { 1466 tcg_gen_mov_i64(ret, arg1); 1467 } else { 1468 tcg_gen_sar_i64(ret, arg1, tcg_constant_i64(arg2)); 1469 } 1470 } 1471 1472 void tcg_gen_brcond_i64(TCGCond cond, TCGv_i64 arg1, TCGv_i64 arg2, TCGLabel *l) 1473 { 1474 if (cond == TCG_COND_ALWAYS) { 1475 tcg_gen_br(l); 1476 } else if (cond != TCG_COND_NEVER) { 1477 l->refs++; 1478 if (TCG_TARGET_REG_BITS == 32) { 1479 tcg_gen_op6ii_i32(INDEX_op_brcond2_i32, TCGV_LOW(arg1), 1480 TCGV_HIGH(arg1), TCGV_LOW(arg2), 1481 TCGV_HIGH(arg2), cond, label_arg(l)); 1482 } else { 1483 tcg_gen_op4ii_i64(INDEX_op_brcond_i64, arg1, arg2, cond, 1484 label_arg(l)); 1485 } 1486 } 1487 } 1488 1489 void tcg_gen_brcondi_i64(TCGCond cond, TCGv_i64 arg1, int64_t arg2, TCGLabel *l) 1490 { 1491 if (TCG_TARGET_REG_BITS == 64) { 1492 tcg_gen_brcond_i64(cond, arg1, tcg_constant_i64(arg2), l); 1493 } else if (cond == TCG_COND_ALWAYS) { 1494 tcg_gen_br(l); 1495 } else if (cond != TCG_COND_NEVER) { 1496 l->refs++; 1497 tcg_gen_op6ii_i32(INDEX_op_brcond2_i32, 1498 TCGV_LOW(arg1), TCGV_HIGH(arg1), 1499 tcg_constant_i32(arg2), 1500 tcg_constant_i32(arg2 >> 32), 1501 cond, label_arg(l)); 1502 } 1503 } 1504 1505 void tcg_gen_setcond_i64(TCGCond cond, TCGv_i64 ret, 1506 TCGv_i64 arg1, TCGv_i64 arg2) 1507 { 1508 if (cond == TCG_COND_ALWAYS) { 1509 tcg_gen_movi_i64(ret, 1); 1510 } else if (cond == TCG_COND_NEVER) { 1511 tcg_gen_movi_i64(ret, 0); 1512 } else { 1513 if (TCG_TARGET_REG_BITS == 32) { 1514 tcg_gen_op6i_i32(INDEX_op_setcond2_i32, TCGV_LOW(ret), 1515 TCGV_LOW(arg1), TCGV_HIGH(arg1), 1516 TCGV_LOW(arg2), TCGV_HIGH(arg2), cond); 1517 tcg_gen_movi_i32(TCGV_HIGH(ret), 0); 1518 } else { 1519 tcg_gen_op4i_i64(INDEX_op_setcond_i64, ret, arg1, arg2, cond); 1520 } 1521 } 1522 } 1523 1524 void tcg_gen_setcondi_i64(TCGCond cond, TCGv_i64 ret, 1525 TCGv_i64 arg1, int64_t arg2) 1526 { 1527 if (TCG_TARGET_REG_BITS == 64) { 1528 tcg_gen_setcond_i64(cond, ret, arg1, tcg_constant_i64(arg2)); 1529 } else if (cond == TCG_COND_ALWAYS) { 1530 tcg_gen_movi_i64(ret, 1); 1531 } else if (cond == TCG_COND_NEVER) { 1532 tcg_gen_movi_i64(ret, 0); 1533 } else { 1534 tcg_gen_op6i_i32(INDEX_op_setcond2_i32, TCGV_LOW(ret), 1535 TCGV_LOW(arg1), TCGV_HIGH(arg1), 1536 tcg_constant_i32(arg2), 1537 tcg_constant_i32(arg2 >> 32), cond); 1538 tcg_gen_movi_i32(TCGV_HIGH(ret), 0); 1539 } 1540 } 1541 1542 void tcg_gen_muli_i64(TCGv_i64 ret, TCGv_i64 arg1, int64_t arg2) 1543 { 1544 if (arg2 == 0) { 1545 tcg_gen_movi_i64(ret, 0); 1546 } else if (is_power_of_2(arg2)) { 1547 tcg_gen_shli_i64(ret, arg1, ctz64(arg2)); 1548 } else { 1549 TCGv_i64 t0 = tcg_const_i64(arg2); 1550 tcg_gen_mul_i64(ret, arg1, t0); 1551 tcg_temp_free_i64(t0); 1552 } 1553 } 1554 1555 void tcg_gen_div_i64(TCGv_i64 ret, TCGv_i64 arg1, TCGv_i64 arg2) 1556 { 1557 if (TCG_TARGET_HAS_div_i64) { 1558 tcg_gen_op3_i64(INDEX_op_div_i64, ret, arg1, arg2); 1559 } else if (TCG_TARGET_HAS_div2_i64) { 1560 TCGv_i64 t0 = tcg_temp_ebb_new_i64(); 1561 tcg_gen_sari_i64(t0, arg1, 63); 1562 tcg_gen_op5_i64(INDEX_op_div2_i64, ret, t0, arg1, t0, arg2); 1563 tcg_temp_free_i64(t0); 1564 } else { 1565 gen_helper_div_i64(ret, arg1, arg2); 1566 } 1567 } 1568 1569 void tcg_gen_rem_i64(TCGv_i64 ret, TCGv_i64 arg1, TCGv_i64 arg2) 1570 { 1571 if (TCG_TARGET_HAS_rem_i64) { 1572 tcg_gen_op3_i64(INDEX_op_rem_i64, ret, arg1, arg2); 1573 } else if (TCG_TARGET_HAS_div_i64) { 1574 TCGv_i64 t0 = tcg_temp_ebb_new_i64(); 1575 tcg_gen_op3_i64(INDEX_op_div_i64, t0, arg1, arg2); 1576 tcg_gen_mul_i64(t0, t0, arg2); 1577 tcg_gen_sub_i64(ret, arg1, t0); 1578 tcg_temp_free_i64(t0); 1579 } else if (TCG_TARGET_HAS_div2_i64) { 1580 TCGv_i64 t0 = tcg_temp_ebb_new_i64(); 1581 tcg_gen_sari_i64(t0, arg1, 63); 1582 tcg_gen_op5_i64(INDEX_op_div2_i64, t0, ret, arg1, t0, arg2); 1583 tcg_temp_free_i64(t0); 1584 } else { 1585 gen_helper_rem_i64(ret, arg1, arg2); 1586 } 1587 } 1588 1589 void tcg_gen_divu_i64(TCGv_i64 ret, TCGv_i64 arg1, TCGv_i64 arg2) 1590 { 1591 if (TCG_TARGET_HAS_div_i64) { 1592 tcg_gen_op3_i64(INDEX_op_divu_i64, ret, arg1, arg2); 1593 } else if (TCG_TARGET_HAS_div2_i64) { 1594 TCGv_i64 t0 = tcg_temp_ebb_new_i64(); 1595 tcg_gen_movi_i64(t0, 0); 1596 tcg_gen_op5_i64(INDEX_op_divu2_i64, ret, t0, arg1, t0, arg2); 1597 tcg_temp_free_i64(t0); 1598 } else { 1599 gen_helper_divu_i64(ret, arg1, arg2); 1600 } 1601 } 1602 1603 void tcg_gen_remu_i64(TCGv_i64 ret, TCGv_i64 arg1, TCGv_i64 arg2) 1604 { 1605 if (TCG_TARGET_HAS_rem_i64) { 1606 tcg_gen_op3_i64(INDEX_op_remu_i64, ret, arg1, arg2); 1607 } else if (TCG_TARGET_HAS_div_i64) { 1608 TCGv_i64 t0 = tcg_temp_ebb_new_i64(); 1609 tcg_gen_op3_i64(INDEX_op_divu_i64, t0, arg1, arg2); 1610 tcg_gen_mul_i64(t0, t0, arg2); 1611 tcg_gen_sub_i64(ret, arg1, t0); 1612 tcg_temp_free_i64(t0); 1613 } else if (TCG_TARGET_HAS_div2_i64) { 1614 TCGv_i64 t0 = tcg_temp_ebb_new_i64(); 1615 tcg_gen_movi_i64(t0, 0); 1616 tcg_gen_op5_i64(INDEX_op_divu2_i64, t0, ret, arg1, t0, arg2); 1617 tcg_temp_free_i64(t0); 1618 } else { 1619 gen_helper_remu_i64(ret, arg1, arg2); 1620 } 1621 } 1622 1623 void tcg_gen_ext8s_i64(TCGv_i64 ret, TCGv_i64 arg) 1624 { 1625 if (TCG_TARGET_REG_BITS == 32) { 1626 tcg_gen_ext8s_i32(TCGV_LOW(ret), TCGV_LOW(arg)); 1627 tcg_gen_sari_i32(TCGV_HIGH(ret), TCGV_LOW(ret), 31); 1628 } else if (TCG_TARGET_HAS_ext8s_i64) { 1629 tcg_gen_op2_i64(INDEX_op_ext8s_i64, ret, arg); 1630 } else { 1631 tcg_gen_shli_i64(ret, arg, 56); 1632 tcg_gen_sari_i64(ret, ret, 56); 1633 } 1634 } 1635 1636 void tcg_gen_ext16s_i64(TCGv_i64 ret, TCGv_i64 arg) 1637 { 1638 if (TCG_TARGET_REG_BITS == 32) { 1639 tcg_gen_ext16s_i32(TCGV_LOW(ret), TCGV_LOW(arg)); 1640 tcg_gen_sari_i32(TCGV_HIGH(ret), TCGV_LOW(ret), 31); 1641 } else if (TCG_TARGET_HAS_ext16s_i64) { 1642 tcg_gen_op2_i64(INDEX_op_ext16s_i64, ret, arg); 1643 } else { 1644 tcg_gen_shli_i64(ret, arg, 48); 1645 tcg_gen_sari_i64(ret, ret, 48); 1646 } 1647 } 1648 1649 void tcg_gen_ext32s_i64(TCGv_i64 ret, TCGv_i64 arg) 1650 { 1651 if (TCG_TARGET_REG_BITS == 32) { 1652 tcg_gen_mov_i32(TCGV_LOW(ret), TCGV_LOW(arg)); 1653 tcg_gen_sari_i32(TCGV_HIGH(ret), TCGV_LOW(ret), 31); 1654 } else if (TCG_TARGET_HAS_ext32s_i64) { 1655 tcg_gen_op2_i64(INDEX_op_ext32s_i64, ret, arg); 1656 } else { 1657 tcg_gen_shli_i64(ret, arg, 32); 1658 tcg_gen_sari_i64(ret, ret, 32); 1659 } 1660 } 1661 1662 void tcg_gen_ext8u_i64(TCGv_i64 ret, TCGv_i64 arg) 1663 { 1664 if (TCG_TARGET_REG_BITS == 32) { 1665 tcg_gen_ext8u_i32(TCGV_LOW(ret), TCGV_LOW(arg)); 1666 tcg_gen_movi_i32(TCGV_HIGH(ret), 0); 1667 } else if (TCG_TARGET_HAS_ext8u_i64) { 1668 tcg_gen_op2_i64(INDEX_op_ext8u_i64, ret, arg); 1669 } else { 1670 tcg_gen_andi_i64(ret, arg, 0xffu); 1671 } 1672 } 1673 1674 void tcg_gen_ext16u_i64(TCGv_i64 ret, TCGv_i64 arg) 1675 { 1676 if (TCG_TARGET_REG_BITS == 32) { 1677 tcg_gen_ext16u_i32(TCGV_LOW(ret), TCGV_LOW(arg)); 1678 tcg_gen_movi_i32(TCGV_HIGH(ret), 0); 1679 } else if (TCG_TARGET_HAS_ext16u_i64) { 1680 tcg_gen_op2_i64(INDEX_op_ext16u_i64, ret, arg); 1681 } else { 1682 tcg_gen_andi_i64(ret, arg, 0xffffu); 1683 } 1684 } 1685 1686 void tcg_gen_ext32u_i64(TCGv_i64 ret, TCGv_i64 arg) 1687 { 1688 if (TCG_TARGET_REG_BITS == 32) { 1689 tcg_gen_mov_i32(TCGV_LOW(ret), TCGV_LOW(arg)); 1690 tcg_gen_movi_i32(TCGV_HIGH(ret), 0); 1691 } else if (TCG_TARGET_HAS_ext32u_i64) { 1692 tcg_gen_op2_i64(INDEX_op_ext32u_i64, ret, arg); 1693 } else { 1694 tcg_gen_andi_i64(ret, arg, 0xffffffffu); 1695 } 1696 } 1697 1698 void tcg_gen_bswap16_i64(TCGv_i64 ret, TCGv_i64 arg, int flags) 1699 { 1700 /* Only one extension flag may be present. */ 1701 tcg_debug_assert(!(flags & TCG_BSWAP_OS) || !(flags & TCG_BSWAP_OZ)); 1702 1703 if (TCG_TARGET_REG_BITS == 32) { 1704 tcg_gen_bswap16_i32(TCGV_LOW(ret), TCGV_LOW(arg), flags); 1705 if (flags & TCG_BSWAP_OS) { 1706 tcg_gen_sari_i32(TCGV_HIGH(ret), TCGV_LOW(ret), 31); 1707 } else { 1708 tcg_gen_movi_i32(TCGV_HIGH(ret), 0); 1709 } 1710 } else if (TCG_TARGET_HAS_bswap16_i64) { 1711 tcg_gen_op3i_i64(INDEX_op_bswap16_i64, ret, arg, flags); 1712 } else { 1713 TCGv_i64 t0 = tcg_temp_ebb_new_i64(); 1714 TCGv_i64 t1 = tcg_temp_ebb_new_i64(); 1715 1716 tcg_gen_shri_i64(t0, arg, 8); 1717 if (!(flags & TCG_BSWAP_IZ)) { 1718 tcg_gen_ext8u_i64(t0, t0); 1719 } 1720 1721 if (flags & TCG_BSWAP_OS) { 1722 tcg_gen_shli_i64(t1, arg, 56); 1723 tcg_gen_sari_i64(t1, t1, 48); 1724 } else if (flags & TCG_BSWAP_OZ) { 1725 tcg_gen_ext8u_i64(t1, arg); 1726 tcg_gen_shli_i64(t1, t1, 8); 1727 } else { 1728 tcg_gen_shli_i64(t1, arg, 8); 1729 } 1730 1731 tcg_gen_or_i64(ret, t0, t1); 1732 tcg_temp_free_i64(t0); 1733 tcg_temp_free_i64(t1); 1734 } 1735 } 1736 1737 void tcg_gen_bswap32_i64(TCGv_i64 ret, TCGv_i64 arg, int flags) 1738 { 1739 /* Only one extension flag may be present. */ 1740 tcg_debug_assert(!(flags & TCG_BSWAP_OS) || !(flags & TCG_BSWAP_OZ)); 1741 1742 if (TCG_TARGET_REG_BITS == 32) { 1743 tcg_gen_bswap32_i32(TCGV_LOW(ret), TCGV_LOW(arg)); 1744 if (flags & TCG_BSWAP_OS) { 1745 tcg_gen_sari_i32(TCGV_HIGH(ret), TCGV_LOW(ret), 31); 1746 } else { 1747 tcg_gen_movi_i32(TCGV_HIGH(ret), 0); 1748 } 1749 } else if (TCG_TARGET_HAS_bswap32_i64) { 1750 tcg_gen_op3i_i64(INDEX_op_bswap32_i64, ret, arg, flags); 1751 } else { 1752 TCGv_i64 t0 = tcg_temp_ebb_new_i64(); 1753 TCGv_i64 t1 = tcg_temp_ebb_new_i64(); 1754 TCGv_i64 t2 = tcg_constant_i64(0x00ff00ff); 1755 1756 /* arg = xxxxabcd */ 1757 tcg_gen_shri_i64(t0, arg, 8); /* t0 = .xxxxabc */ 1758 tcg_gen_and_i64(t1, arg, t2); /* t1 = .....b.d */ 1759 tcg_gen_and_i64(t0, t0, t2); /* t0 = .....a.c */ 1760 tcg_gen_shli_i64(t1, t1, 8); /* t1 = ....b.d. */ 1761 tcg_gen_or_i64(ret, t0, t1); /* ret = ....badc */ 1762 1763 tcg_gen_shli_i64(t1, ret, 48); /* t1 = dc...... */ 1764 tcg_gen_shri_i64(t0, ret, 16); /* t0 = ......ba */ 1765 if (flags & TCG_BSWAP_OS) { 1766 tcg_gen_sari_i64(t1, t1, 32); /* t1 = ssssdc.. */ 1767 } else { 1768 tcg_gen_shri_i64(t1, t1, 32); /* t1 = ....dc.. */ 1769 } 1770 tcg_gen_or_i64(ret, t0, t1); /* ret = ssssdcba */ 1771 1772 tcg_temp_free_i64(t0); 1773 tcg_temp_free_i64(t1); 1774 } 1775 } 1776 1777 void tcg_gen_bswap64_i64(TCGv_i64 ret, TCGv_i64 arg) 1778 { 1779 if (TCG_TARGET_REG_BITS == 32) { 1780 TCGv_i32 t0, t1; 1781 t0 = tcg_temp_ebb_new_i32(); 1782 t1 = tcg_temp_ebb_new_i32(); 1783 1784 tcg_gen_bswap32_i32(t0, TCGV_LOW(arg)); 1785 tcg_gen_bswap32_i32(t1, TCGV_HIGH(arg)); 1786 tcg_gen_mov_i32(TCGV_LOW(ret), t1); 1787 tcg_gen_mov_i32(TCGV_HIGH(ret), t0); 1788 tcg_temp_free_i32(t0); 1789 tcg_temp_free_i32(t1); 1790 } else if (TCG_TARGET_HAS_bswap64_i64) { 1791 tcg_gen_op3i_i64(INDEX_op_bswap64_i64, ret, arg, 0); 1792 } else { 1793 TCGv_i64 t0 = tcg_temp_ebb_new_i64(); 1794 TCGv_i64 t1 = tcg_temp_ebb_new_i64(); 1795 TCGv_i64 t2 = tcg_temp_ebb_new_i64(); 1796 1797 /* arg = abcdefgh */ 1798 tcg_gen_movi_i64(t2, 0x00ff00ff00ff00ffull); 1799 tcg_gen_shri_i64(t0, arg, 8); /* t0 = .abcdefg */ 1800 tcg_gen_and_i64(t1, arg, t2); /* t1 = .b.d.f.h */ 1801 tcg_gen_and_i64(t0, t0, t2); /* t0 = .a.c.e.g */ 1802 tcg_gen_shli_i64(t1, t1, 8); /* t1 = b.d.f.h. */ 1803 tcg_gen_or_i64(ret, t0, t1); /* ret = badcfehg */ 1804 1805 tcg_gen_movi_i64(t2, 0x0000ffff0000ffffull); 1806 tcg_gen_shri_i64(t0, ret, 16); /* t0 = ..badcfe */ 1807 tcg_gen_and_i64(t1, ret, t2); /* t1 = ..dc..hg */ 1808 tcg_gen_and_i64(t0, t0, t2); /* t0 = ..ba..fe */ 1809 tcg_gen_shli_i64(t1, t1, 16); /* t1 = dc..hg.. */ 1810 tcg_gen_or_i64(ret, t0, t1); /* ret = dcbahgfe */ 1811 1812 tcg_gen_shri_i64(t0, ret, 32); /* t0 = ....dcba */ 1813 tcg_gen_shli_i64(t1, ret, 32); /* t1 = hgfe.... */ 1814 tcg_gen_or_i64(ret, t0, t1); /* ret = hgfedcba */ 1815 1816 tcg_temp_free_i64(t0); 1817 tcg_temp_free_i64(t1); 1818 tcg_temp_free_i64(t2); 1819 } 1820 } 1821 1822 void tcg_gen_hswap_i64(TCGv_i64 ret, TCGv_i64 arg) 1823 { 1824 uint64_t m = 0x0000ffff0000ffffull; 1825 TCGv_i64 t0 = tcg_temp_ebb_new_i64(); 1826 TCGv_i64 t1 = tcg_temp_ebb_new_i64(); 1827 1828 /* See include/qemu/bitops.h, hswap64. */ 1829 tcg_gen_rotli_i64(t1, arg, 32); 1830 tcg_gen_andi_i64(t0, t1, m); 1831 tcg_gen_shli_i64(t0, t0, 16); 1832 tcg_gen_shri_i64(t1, t1, 16); 1833 tcg_gen_andi_i64(t1, t1, m); 1834 tcg_gen_or_i64(ret, t0, t1); 1835 1836 tcg_temp_free_i64(t0); 1837 tcg_temp_free_i64(t1); 1838 } 1839 1840 void tcg_gen_wswap_i64(TCGv_i64 ret, TCGv_i64 arg) 1841 { 1842 /* Swapping 2 32-bit elements is a rotate. */ 1843 tcg_gen_rotli_i64(ret, arg, 32); 1844 } 1845 1846 void tcg_gen_not_i64(TCGv_i64 ret, TCGv_i64 arg) 1847 { 1848 if (TCG_TARGET_REG_BITS == 32) { 1849 tcg_gen_not_i32(TCGV_LOW(ret), TCGV_LOW(arg)); 1850 tcg_gen_not_i32(TCGV_HIGH(ret), TCGV_HIGH(arg)); 1851 } else if (TCG_TARGET_HAS_not_i64) { 1852 tcg_gen_op2_i64(INDEX_op_not_i64, ret, arg); 1853 } else { 1854 tcg_gen_xori_i64(ret, arg, -1); 1855 } 1856 } 1857 1858 void tcg_gen_andc_i64(TCGv_i64 ret, TCGv_i64 arg1, TCGv_i64 arg2) 1859 { 1860 if (TCG_TARGET_REG_BITS == 32) { 1861 tcg_gen_andc_i32(TCGV_LOW(ret), TCGV_LOW(arg1), TCGV_LOW(arg2)); 1862 tcg_gen_andc_i32(TCGV_HIGH(ret), TCGV_HIGH(arg1), TCGV_HIGH(arg2)); 1863 } else if (TCG_TARGET_HAS_andc_i64) { 1864 tcg_gen_op3_i64(INDEX_op_andc_i64, ret, arg1, arg2); 1865 } else { 1866 TCGv_i64 t0 = tcg_temp_ebb_new_i64(); 1867 tcg_gen_not_i64(t0, arg2); 1868 tcg_gen_and_i64(ret, arg1, t0); 1869 tcg_temp_free_i64(t0); 1870 } 1871 } 1872 1873 void tcg_gen_eqv_i64(TCGv_i64 ret, TCGv_i64 arg1, TCGv_i64 arg2) 1874 { 1875 if (TCG_TARGET_REG_BITS == 32) { 1876 tcg_gen_eqv_i32(TCGV_LOW(ret), TCGV_LOW(arg1), TCGV_LOW(arg2)); 1877 tcg_gen_eqv_i32(TCGV_HIGH(ret), TCGV_HIGH(arg1), TCGV_HIGH(arg2)); 1878 } else if (TCG_TARGET_HAS_eqv_i64) { 1879 tcg_gen_op3_i64(INDEX_op_eqv_i64, ret, arg1, arg2); 1880 } else { 1881 tcg_gen_xor_i64(ret, arg1, arg2); 1882 tcg_gen_not_i64(ret, ret); 1883 } 1884 } 1885 1886 void tcg_gen_nand_i64(TCGv_i64 ret, TCGv_i64 arg1, TCGv_i64 arg2) 1887 { 1888 if (TCG_TARGET_REG_BITS == 32) { 1889 tcg_gen_nand_i32(TCGV_LOW(ret), TCGV_LOW(arg1), TCGV_LOW(arg2)); 1890 tcg_gen_nand_i32(TCGV_HIGH(ret), TCGV_HIGH(arg1), TCGV_HIGH(arg2)); 1891 } else if (TCG_TARGET_HAS_nand_i64) { 1892 tcg_gen_op3_i64(INDEX_op_nand_i64, ret, arg1, arg2); 1893 } else { 1894 tcg_gen_and_i64(ret, arg1, arg2); 1895 tcg_gen_not_i64(ret, ret); 1896 } 1897 } 1898 1899 void tcg_gen_nor_i64(TCGv_i64 ret, TCGv_i64 arg1, TCGv_i64 arg2) 1900 { 1901 if (TCG_TARGET_REG_BITS == 32) { 1902 tcg_gen_nor_i32(TCGV_LOW(ret), TCGV_LOW(arg1), TCGV_LOW(arg2)); 1903 tcg_gen_nor_i32(TCGV_HIGH(ret), TCGV_HIGH(arg1), TCGV_HIGH(arg2)); 1904 } else if (TCG_TARGET_HAS_nor_i64) { 1905 tcg_gen_op3_i64(INDEX_op_nor_i64, ret, arg1, arg2); 1906 } else { 1907 tcg_gen_or_i64(ret, arg1, arg2); 1908 tcg_gen_not_i64(ret, ret); 1909 } 1910 } 1911 1912 void tcg_gen_orc_i64(TCGv_i64 ret, TCGv_i64 arg1, TCGv_i64 arg2) 1913 { 1914 if (TCG_TARGET_REG_BITS == 32) { 1915 tcg_gen_orc_i32(TCGV_LOW(ret), TCGV_LOW(arg1), TCGV_LOW(arg2)); 1916 tcg_gen_orc_i32(TCGV_HIGH(ret), TCGV_HIGH(arg1), TCGV_HIGH(arg2)); 1917 } else if (TCG_TARGET_HAS_orc_i64) { 1918 tcg_gen_op3_i64(INDEX_op_orc_i64, ret, arg1, arg2); 1919 } else { 1920 TCGv_i64 t0 = tcg_temp_ebb_new_i64(); 1921 tcg_gen_not_i64(t0, arg2); 1922 tcg_gen_or_i64(ret, arg1, t0); 1923 tcg_temp_free_i64(t0); 1924 } 1925 } 1926 1927 void tcg_gen_clz_i64(TCGv_i64 ret, TCGv_i64 arg1, TCGv_i64 arg2) 1928 { 1929 if (TCG_TARGET_HAS_clz_i64) { 1930 tcg_gen_op3_i64(INDEX_op_clz_i64, ret, arg1, arg2); 1931 } else { 1932 gen_helper_clz_i64(ret, arg1, arg2); 1933 } 1934 } 1935 1936 void tcg_gen_clzi_i64(TCGv_i64 ret, TCGv_i64 arg1, uint64_t arg2) 1937 { 1938 if (TCG_TARGET_REG_BITS == 32 1939 && TCG_TARGET_HAS_clz_i32 1940 && arg2 <= 0xffffffffu) { 1941 TCGv_i32 t = tcg_temp_ebb_new_i32(); 1942 tcg_gen_clzi_i32(t, TCGV_LOW(arg1), arg2 - 32); 1943 tcg_gen_addi_i32(t, t, 32); 1944 tcg_gen_clz_i32(TCGV_LOW(ret), TCGV_HIGH(arg1), t); 1945 tcg_gen_movi_i32(TCGV_HIGH(ret), 0); 1946 tcg_temp_free_i32(t); 1947 } else { 1948 TCGv_i64 t0 = tcg_const_i64(arg2); 1949 tcg_gen_clz_i64(ret, arg1, t0); 1950 tcg_temp_free_i64(t0); 1951 } 1952 } 1953 1954 void tcg_gen_ctz_i64(TCGv_i64 ret, TCGv_i64 arg1, TCGv_i64 arg2) 1955 { 1956 if (TCG_TARGET_HAS_ctz_i64) { 1957 tcg_gen_op3_i64(INDEX_op_ctz_i64, ret, arg1, arg2); 1958 } else if (TCG_TARGET_HAS_ctpop_i64 || TCG_TARGET_HAS_clz_i64) { 1959 TCGv_i64 z, t = tcg_temp_ebb_new_i64(); 1960 1961 if (TCG_TARGET_HAS_ctpop_i64) { 1962 tcg_gen_subi_i64(t, arg1, 1); 1963 tcg_gen_andc_i64(t, t, arg1); 1964 tcg_gen_ctpop_i64(t, t); 1965 } else { 1966 /* Since all non-x86 hosts have clz(0) == 64, don't fight it. */ 1967 tcg_gen_neg_i64(t, arg1); 1968 tcg_gen_and_i64(t, t, arg1); 1969 tcg_gen_clzi_i64(t, t, 64); 1970 tcg_gen_xori_i64(t, t, 63); 1971 } 1972 z = tcg_constant_i64(0); 1973 tcg_gen_movcond_i64(TCG_COND_EQ, ret, arg1, z, arg2, t); 1974 tcg_temp_free_i64(t); 1975 tcg_temp_free_i64(z); 1976 } else { 1977 gen_helper_ctz_i64(ret, arg1, arg2); 1978 } 1979 } 1980 1981 void tcg_gen_ctzi_i64(TCGv_i64 ret, TCGv_i64 arg1, uint64_t arg2) 1982 { 1983 if (TCG_TARGET_REG_BITS == 32 1984 && TCG_TARGET_HAS_ctz_i32 1985 && arg2 <= 0xffffffffu) { 1986 TCGv_i32 t32 = tcg_temp_ebb_new_i32(); 1987 tcg_gen_ctzi_i32(t32, TCGV_HIGH(arg1), arg2 - 32); 1988 tcg_gen_addi_i32(t32, t32, 32); 1989 tcg_gen_ctz_i32(TCGV_LOW(ret), TCGV_LOW(arg1), t32); 1990 tcg_gen_movi_i32(TCGV_HIGH(ret), 0); 1991 tcg_temp_free_i32(t32); 1992 } else if (!TCG_TARGET_HAS_ctz_i64 1993 && TCG_TARGET_HAS_ctpop_i64 1994 && arg2 == 64) { 1995 /* This equivalence has the advantage of not requiring a fixup. */ 1996 TCGv_i64 t = tcg_temp_ebb_new_i64(); 1997 tcg_gen_subi_i64(t, arg1, 1); 1998 tcg_gen_andc_i64(t, t, arg1); 1999 tcg_gen_ctpop_i64(ret, t); 2000 tcg_temp_free_i64(t); 2001 } else { 2002 TCGv_i64 t0 = tcg_const_i64(arg2); 2003 tcg_gen_ctz_i64(ret, arg1, t0); 2004 tcg_temp_free_i64(t0); 2005 } 2006 } 2007 2008 void tcg_gen_clrsb_i64(TCGv_i64 ret, TCGv_i64 arg) 2009 { 2010 if (TCG_TARGET_HAS_clz_i64 || TCG_TARGET_HAS_clz_i32) { 2011 TCGv_i64 t = tcg_temp_ebb_new_i64(); 2012 tcg_gen_sari_i64(t, arg, 63); 2013 tcg_gen_xor_i64(t, t, arg); 2014 tcg_gen_clzi_i64(t, t, 64); 2015 tcg_gen_subi_i64(ret, t, 1); 2016 tcg_temp_free_i64(t); 2017 } else { 2018 gen_helper_clrsb_i64(ret, arg); 2019 } 2020 } 2021 2022 void tcg_gen_ctpop_i64(TCGv_i64 ret, TCGv_i64 arg1) 2023 { 2024 if (TCG_TARGET_HAS_ctpop_i64) { 2025 tcg_gen_op2_i64(INDEX_op_ctpop_i64, ret, arg1); 2026 } else if (TCG_TARGET_REG_BITS == 32 && TCG_TARGET_HAS_ctpop_i32) { 2027 tcg_gen_ctpop_i32(TCGV_HIGH(ret), TCGV_HIGH(arg1)); 2028 tcg_gen_ctpop_i32(TCGV_LOW(ret), TCGV_LOW(arg1)); 2029 tcg_gen_add_i32(TCGV_LOW(ret), TCGV_LOW(ret), TCGV_HIGH(ret)); 2030 tcg_gen_movi_i32(TCGV_HIGH(ret), 0); 2031 } else { 2032 gen_helper_ctpop_i64(ret, arg1); 2033 } 2034 } 2035 2036 void tcg_gen_rotl_i64(TCGv_i64 ret, TCGv_i64 arg1, TCGv_i64 arg2) 2037 { 2038 if (TCG_TARGET_HAS_rot_i64) { 2039 tcg_gen_op3_i64(INDEX_op_rotl_i64, ret, arg1, arg2); 2040 } else { 2041 TCGv_i64 t0, t1; 2042 t0 = tcg_temp_ebb_new_i64(); 2043 t1 = tcg_temp_ebb_new_i64(); 2044 tcg_gen_shl_i64(t0, arg1, arg2); 2045 tcg_gen_subfi_i64(t1, 64, arg2); 2046 tcg_gen_shr_i64(t1, arg1, t1); 2047 tcg_gen_or_i64(ret, t0, t1); 2048 tcg_temp_free_i64(t0); 2049 tcg_temp_free_i64(t1); 2050 } 2051 } 2052 2053 void tcg_gen_rotli_i64(TCGv_i64 ret, TCGv_i64 arg1, int64_t arg2) 2054 { 2055 tcg_debug_assert(arg2 >= 0 && arg2 < 64); 2056 /* some cases can be optimized here */ 2057 if (arg2 == 0) { 2058 tcg_gen_mov_i64(ret, arg1); 2059 } else if (TCG_TARGET_HAS_rot_i64) { 2060 tcg_gen_rotl_i64(ret, arg1, tcg_constant_i64(arg2)); 2061 } else { 2062 TCGv_i64 t0, t1; 2063 t0 = tcg_temp_ebb_new_i64(); 2064 t1 = tcg_temp_ebb_new_i64(); 2065 tcg_gen_shli_i64(t0, arg1, arg2); 2066 tcg_gen_shri_i64(t1, arg1, 64 - arg2); 2067 tcg_gen_or_i64(ret, t0, t1); 2068 tcg_temp_free_i64(t0); 2069 tcg_temp_free_i64(t1); 2070 } 2071 } 2072 2073 void tcg_gen_rotr_i64(TCGv_i64 ret, TCGv_i64 arg1, TCGv_i64 arg2) 2074 { 2075 if (TCG_TARGET_HAS_rot_i64) { 2076 tcg_gen_op3_i64(INDEX_op_rotr_i64, ret, arg1, arg2); 2077 } else { 2078 TCGv_i64 t0, t1; 2079 t0 = tcg_temp_ebb_new_i64(); 2080 t1 = tcg_temp_ebb_new_i64(); 2081 tcg_gen_shr_i64(t0, arg1, arg2); 2082 tcg_gen_subfi_i64(t1, 64, arg2); 2083 tcg_gen_shl_i64(t1, arg1, t1); 2084 tcg_gen_or_i64(ret, t0, t1); 2085 tcg_temp_free_i64(t0); 2086 tcg_temp_free_i64(t1); 2087 } 2088 } 2089 2090 void tcg_gen_rotri_i64(TCGv_i64 ret, TCGv_i64 arg1, int64_t arg2) 2091 { 2092 tcg_debug_assert(arg2 >= 0 && arg2 < 64); 2093 /* some cases can be optimized here */ 2094 if (arg2 == 0) { 2095 tcg_gen_mov_i64(ret, arg1); 2096 } else { 2097 tcg_gen_rotli_i64(ret, arg1, 64 - arg2); 2098 } 2099 } 2100 2101 void tcg_gen_deposit_i64(TCGv_i64 ret, TCGv_i64 arg1, TCGv_i64 arg2, 2102 unsigned int ofs, unsigned int len) 2103 { 2104 uint64_t mask; 2105 TCGv_i64 t1; 2106 2107 tcg_debug_assert(ofs < 64); 2108 tcg_debug_assert(len > 0); 2109 tcg_debug_assert(len <= 64); 2110 tcg_debug_assert(ofs + len <= 64); 2111 2112 if (len == 64) { 2113 tcg_gen_mov_i64(ret, arg2); 2114 return; 2115 } 2116 if (TCG_TARGET_HAS_deposit_i64 && TCG_TARGET_deposit_i64_valid(ofs, len)) { 2117 tcg_gen_op5ii_i64(INDEX_op_deposit_i64, ret, arg1, arg2, ofs, len); 2118 return; 2119 } 2120 2121 if (TCG_TARGET_REG_BITS == 32) { 2122 if (ofs >= 32) { 2123 tcg_gen_deposit_i32(TCGV_HIGH(ret), TCGV_HIGH(arg1), 2124 TCGV_LOW(arg2), ofs - 32, len); 2125 tcg_gen_mov_i32(TCGV_LOW(ret), TCGV_LOW(arg1)); 2126 return; 2127 } 2128 if (ofs + len <= 32) { 2129 tcg_gen_deposit_i32(TCGV_LOW(ret), TCGV_LOW(arg1), 2130 TCGV_LOW(arg2), ofs, len); 2131 tcg_gen_mov_i32(TCGV_HIGH(ret), TCGV_HIGH(arg1)); 2132 return; 2133 } 2134 } 2135 2136 t1 = tcg_temp_ebb_new_i64(); 2137 2138 if (TCG_TARGET_HAS_extract2_i64) { 2139 if (ofs + len == 64) { 2140 tcg_gen_shli_i64(t1, arg1, len); 2141 tcg_gen_extract2_i64(ret, t1, arg2, len); 2142 goto done; 2143 } 2144 if (ofs == 0) { 2145 tcg_gen_extract2_i64(ret, arg1, arg2, len); 2146 tcg_gen_rotli_i64(ret, ret, len); 2147 goto done; 2148 } 2149 } 2150 2151 mask = (1ull << len) - 1; 2152 if (ofs + len < 64) { 2153 tcg_gen_andi_i64(t1, arg2, mask); 2154 tcg_gen_shli_i64(t1, t1, ofs); 2155 } else { 2156 tcg_gen_shli_i64(t1, arg2, ofs); 2157 } 2158 tcg_gen_andi_i64(ret, arg1, ~(mask << ofs)); 2159 tcg_gen_or_i64(ret, ret, t1); 2160 done: 2161 tcg_temp_free_i64(t1); 2162 } 2163 2164 void tcg_gen_deposit_z_i64(TCGv_i64 ret, TCGv_i64 arg, 2165 unsigned int ofs, unsigned int len) 2166 { 2167 tcg_debug_assert(ofs < 64); 2168 tcg_debug_assert(len > 0); 2169 tcg_debug_assert(len <= 64); 2170 tcg_debug_assert(ofs + len <= 64); 2171 2172 if (ofs + len == 64) { 2173 tcg_gen_shli_i64(ret, arg, ofs); 2174 } else if (ofs == 0) { 2175 tcg_gen_andi_i64(ret, arg, (1ull << len) - 1); 2176 } else if (TCG_TARGET_HAS_deposit_i64 2177 && TCG_TARGET_deposit_i64_valid(ofs, len)) { 2178 TCGv_i64 zero = tcg_constant_i64(0); 2179 tcg_gen_op5ii_i64(INDEX_op_deposit_i64, ret, zero, arg, ofs, len); 2180 } else { 2181 if (TCG_TARGET_REG_BITS == 32) { 2182 if (ofs >= 32) { 2183 tcg_gen_deposit_z_i32(TCGV_HIGH(ret), TCGV_LOW(arg), 2184 ofs - 32, len); 2185 tcg_gen_movi_i32(TCGV_LOW(ret), 0); 2186 return; 2187 } 2188 if (ofs + len <= 32) { 2189 tcg_gen_deposit_z_i32(TCGV_LOW(ret), TCGV_LOW(arg), ofs, len); 2190 tcg_gen_movi_i32(TCGV_HIGH(ret), 0); 2191 return; 2192 } 2193 } 2194 /* To help two-operand hosts we prefer to zero-extend first, 2195 which allows ARG to stay live. */ 2196 switch (len) { 2197 case 32: 2198 if (TCG_TARGET_HAS_ext32u_i64) { 2199 tcg_gen_ext32u_i64(ret, arg); 2200 tcg_gen_shli_i64(ret, ret, ofs); 2201 return; 2202 } 2203 break; 2204 case 16: 2205 if (TCG_TARGET_HAS_ext16u_i64) { 2206 tcg_gen_ext16u_i64(ret, arg); 2207 tcg_gen_shli_i64(ret, ret, ofs); 2208 return; 2209 } 2210 break; 2211 case 8: 2212 if (TCG_TARGET_HAS_ext8u_i64) { 2213 tcg_gen_ext8u_i64(ret, arg); 2214 tcg_gen_shli_i64(ret, ret, ofs); 2215 return; 2216 } 2217 break; 2218 } 2219 /* Otherwise prefer zero-extension over AND for code size. */ 2220 switch (ofs + len) { 2221 case 32: 2222 if (TCG_TARGET_HAS_ext32u_i64) { 2223 tcg_gen_shli_i64(ret, arg, ofs); 2224 tcg_gen_ext32u_i64(ret, ret); 2225 return; 2226 } 2227 break; 2228 case 16: 2229 if (TCG_TARGET_HAS_ext16u_i64) { 2230 tcg_gen_shli_i64(ret, arg, ofs); 2231 tcg_gen_ext16u_i64(ret, ret); 2232 return; 2233 } 2234 break; 2235 case 8: 2236 if (TCG_TARGET_HAS_ext8u_i64) { 2237 tcg_gen_shli_i64(ret, arg, ofs); 2238 tcg_gen_ext8u_i64(ret, ret); 2239 return; 2240 } 2241 break; 2242 } 2243 tcg_gen_andi_i64(ret, arg, (1ull << len) - 1); 2244 tcg_gen_shli_i64(ret, ret, ofs); 2245 } 2246 } 2247 2248 void tcg_gen_extract_i64(TCGv_i64 ret, TCGv_i64 arg, 2249 unsigned int ofs, unsigned int len) 2250 { 2251 tcg_debug_assert(ofs < 64); 2252 tcg_debug_assert(len > 0); 2253 tcg_debug_assert(len <= 64); 2254 tcg_debug_assert(ofs + len <= 64); 2255 2256 /* Canonicalize certain special cases, even if extract is supported. */ 2257 if (ofs + len == 64) { 2258 tcg_gen_shri_i64(ret, arg, 64 - len); 2259 return; 2260 } 2261 if (ofs == 0) { 2262 tcg_gen_andi_i64(ret, arg, (1ull << len) - 1); 2263 return; 2264 } 2265 2266 if (TCG_TARGET_REG_BITS == 32) { 2267 /* Look for a 32-bit extract within one of the two words. */ 2268 if (ofs >= 32) { 2269 tcg_gen_extract_i32(TCGV_LOW(ret), TCGV_HIGH(arg), ofs - 32, len); 2270 tcg_gen_movi_i32(TCGV_HIGH(ret), 0); 2271 return; 2272 } 2273 if (ofs + len <= 32) { 2274 tcg_gen_extract_i32(TCGV_LOW(ret), TCGV_LOW(arg), ofs, len); 2275 tcg_gen_movi_i32(TCGV_HIGH(ret), 0); 2276 return; 2277 } 2278 /* The field is split across two words. One double-word 2279 shift is better than two double-word shifts. */ 2280 goto do_shift_and; 2281 } 2282 2283 if (TCG_TARGET_HAS_extract_i64 2284 && TCG_TARGET_extract_i64_valid(ofs, len)) { 2285 tcg_gen_op4ii_i64(INDEX_op_extract_i64, ret, arg, ofs, len); 2286 return; 2287 } 2288 2289 /* Assume that zero-extension, if available, is cheaper than a shift. */ 2290 switch (ofs + len) { 2291 case 32: 2292 if (TCG_TARGET_HAS_ext32u_i64) { 2293 tcg_gen_ext32u_i64(ret, arg); 2294 tcg_gen_shri_i64(ret, ret, ofs); 2295 return; 2296 } 2297 break; 2298 case 16: 2299 if (TCG_TARGET_HAS_ext16u_i64) { 2300 tcg_gen_ext16u_i64(ret, arg); 2301 tcg_gen_shri_i64(ret, ret, ofs); 2302 return; 2303 } 2304 break; 2305 case 8: 2306 if (TCG_TARGET_HAS_ext8u_i64) { 2307 tcg_gen_ext8u_i64(ret, arg); 2308 tcg_gen_shri_i64(ret, ret, ofs); 2309 return; 2310 } 2311 break; 2312 } 2313 2314 /* ??? Ideally we'd know what values are available for immediate AND. 2315 Assume that 8 bits are available, plus the special cases of 16 and 32, 2316 so that we get ext8u, ext16u, and ext32u. */ 2317 switch (len) { 2318 case 1 ... 8: case 16: case 32: 2319 do_shift_and: 2320 tcg_gen_shri_i64(ret, arg, ofs); 2321 tcg_gen_andi_i64(ret, ret, (1ull << len) - 1); 2322 break; 2323 default: 2324 tcg_gen_shli_i64(ret, arg, 64 - len - ofs); 2325 tcg_gen_shri_i64(ret, ret, 64 - len); 2326 break; 2327 } 2328 } 2329 2330 void tcg_gen_sextract_i64(TCGv_i64 ret, TCGv_i64 arg, 2331 unsigned int ofs, unsigned int len) 2332 { 2333 tcg_debug_assert(ofs < 64); 2334 tcg_debug_assert(len > 0); 2335 tcg_debug_assert(len <= 64); 2336 tcg_debug_assert(ofs + len <= 64); 2337 2338 /* Canonicalize certain special cases, even if sextract is supported. */ 2339 if (ofs + len == 64) { 2340 tcg_gen_sari_i64(ret, arg, 64 - len); 2341 return; 2342 } 2343 if (ofs == 0) { 2344 switch (len) { 2345 case 32: 2346 tcg_gen_ext32s_i64(ret, arg); 2347 return; 2348 case 16: 2349 tcg_gen_ext16s_i64(ret, arg); 2350 return; 2351 case 8: 2352 tcg_gen_ext8s_i64(ret, arg); 2353 return; 2354 } 2355 } 2356 2357 if (TCG_TARGET_REG_BITS == 32) { 2358 /* Look for a 32-bit extract within one of the two words. */ 2359 if (ofs >= 32) { 2360 tcg_gen_sextract_i32(TCGV_LOW(ret), TCGV_HIGH(arg), ofs - 32, len); 2361 } else if (ofs + len <= 32) { 2362 tcg_gen_sextract_i32(TCGV_LOW(ret), TCGV_LOW(arg), ofs, len); 2363 } else if (ofs == 0) { 2364 tcg_gen_mov_i32(TCGV_LOW(ret), TCGV_LOW(arg)); 2365 tcg_gen_sextract_i32(TCGV_HIGH(ret), TCGV_HIGH(arg), 0, len - 32); 2366 return; 2367 } else if (len > 32) { 2368 TCGv_i32 t = tcg_temp_ebb_new_i32(); 2369 /* Extract the bits for the high word normally. */ 2370 tcg_gen_sextract_i32(t, TCGV_HIGH(arg), ofs + 32, len - 32); 2371 /* Shift the field down for the low part. */ 2372 tcg_gen_shri_i64(ret, arg, ofs); 2373 /* Overwrite the shift into the high part. */ 2374 tcg_gen_mov_i32(TCGV_HIGH(ret), t); 2375 tcg_temp_free_i32(t); 2376 return; 2377 } else { 2378 /* Shift the field down for the low part, such that the 2379 field sits at the MSB. */ 2380 tcg_gen_shri_i64(ret, arg, ofs + len - 32); 2381 /* Shift the field down from the MSB, sign extending. */ 2382 tcg_gen_sari_i32(TCGV_LOW(ret), TCGV_LOW(ret), 32 - len); 2383 } 2384 /* Sign-extend the field from 32 bits. */ 2385 tcg_gen_sari_i32(TCGV_HIGH(ret), TCGV_LOW(ret), 31); 2386 return; 2387 } 2388 2389 if (TCG_TARGET_HAS_sextract_i64 2390 && TCG_TARGET_extract_i64_valid(ofs, len)) { 2391 tcg_gen_op4ii_i64(INDEX_op_sextract_i64, ret, arg, ofs, len); 2392 return; 2393 } 2394 2395 /* Assume that sign-extension, if available, is cheaper than a shift. */ 2396 switch (ofs + len) { 2397 case 32: 2398 if (TCG_TARGET_HAS_ext32s_i64) { 2399 tcg_gen_ext32s_i64(ret, arg); 2400 tcg_gen_sari_i64(ret, ret, ofs); 2401 return; 2402 } 2403 break; 2404 case 16: 2405 if (TCG_TARGET_HAS_ext16s_i64) { 2406 tcg_gen_ext16s_i64(ret, arg); 2407 tcg_gen_sari_i64(ret, ret, ofs); 2408 return; 2409 } 2410 break; 2411 case 8: 2412 if (TCG_TARGET_HAS_ext8s_i64) { 2413 tcg_gen_ext8s_i64(ret, arg); 2414 tcg_gen_sari_i64(ret, ret, ofs); 2415 return; 2416 } 2417 break; 2418 } 2419 switch (len) { 2420 case 32: 2421 if (TCG_TARGET_HAS_ext32s_i64) { 2422 tcg_gen_shri_i64(ret, arg, ofs); 2423 tcg_gen_ext32s_i64(ret, ret); 2424 return; 2425 } 2426 break; 2427 case 16: 2428 if (TCG_TARGET_HAS_ext16s_i64) { 2429 tcg_gen_shri_i64(ret, arg, ofs); 2430 tcg_gen_ext16s_i64(ret, ret); 2431 return; 2432 } 2433 break; 2434 case 8: 2435 if (TCG_TARGET_HAS_ext8s_i64) { 2436 tcg_gen_shri_i64(ret, arg, ofs); 2437 tcg_gen_ext8s_i64(ret, ret); 2438 return; 2439 } 2440 break; 2441 } 2442 tcg_gen_shli_i64(ret, arg, 64 - len - ofs); 2443 tcg_gen_sari_i64(ret, ret, 64 - len); 2444 } 2445 2446 /* 2447 * Extract 64 bits from a 128-bit input, ah:al, starting from ofs. 2448 * Unlike tcg_gen_extract_i64 above, len is fixed at 64. 2449 */ 2450 void tcg_gen_extract2_i64(TCGv_i64 ret, TCGv_i64 al, TCGv_i64 ah, 2451 unsigned int ofs) 2452 { 2453 tcg_debug_assert(ofs <= 64); 2454 if (ofs == 0) { 2455 tcg_gen_mov_i64(ret, al); 2456 } else if (ofs == 64) { 2457 tcg_gen_mov_i64(ret, ah); 2458 } else if (al == ah) { 2459 tcg_gen_rotri_i64(ret, al, ofs); 2460 } else if (TCG_TARGET_HAS_extract2_i64) { 2461 tcg_gen_op4i_i64(INDEX_op_extract2_i64, ret, al, ah, ofs); 2462 } else { 2463 TCGv_i64 t0 = tcg_temp_ebb_new_i64(); 2464 tcg_gen_shri_i64(t0, al, ofs); 2465 tcg_gen_deposit_i64(ret, t0, ah, 64 - ofs, ofs); 2466 tcg_temp_free_i64(t0); 2467 } 2468 } 2469 2470 void tcg_gen_movcond_i64(TCGCond cond, TCGv_i64 ret, TCGv_i64 c1, 2471 TCGv_i64 c2, TCGv_i64 v1, TCGv_i64 v2) 2472 { 2473 if (cond == TCG_COND_ALWAYS) { 2474 tcg_gen_mov_i64(ret, v1); 2475 } else if (cond == TCG_COND_NEVER) { 2476 tcg_gen_mov_i64(ret, v2); 2477 } else if (TCG_TARGET_REG_BITS == 32) { 2478 TCGv_i32 t0 = tcg_temp_ebb_new_i32(); 2479 TCGv_i32 t1 = tcg_temp_ebb_new_i32(); 2480 tcg_gen_op6i_i32(INDEX_op_setcond2_i32, t0, 2481 TCGV_LOW(c1), TCGV_HIGH(c1), 2482 TCGV_LOW(c2), TCGV_HIGH(c2), cond); 2483 2484 if (TCG_TARGET_HAS_movcond_i32) { 2485 tcg_gen_movi_i32(t1, 0); 2486 tcg_gen_movcond_i32(TCG_COND_NE, TCGV_LOW(ret), t0, t1, 2487 TCGV_LOW(v1), TCGV_LOW(v2)); 2488 tcg_gen_movcond_i32(TCG_COND_NE, TCGV_HIGH(ret), t0, t1, 2489 TCGV_HIGH(v1), TCGV_HIGH(v2)); 2490 } else { 2491 tcg_gen_neg_i32(t0, t0); 2492 2493 tcg_gen_and_i32(t1, TCGV_LOW(v1), t0); 2494 tcg_gen_andc_i32(TCGV_LOW(ret), TCGV_LOW(v2), t0); 2495 tcg_gen_or_i32(TCGV_LOW(ret), TCGV_LOW(ret), t1); 2496 2497 tcg_gen_and_i32(t1, TCGV_HIGH(v1), t0); 2498 tcg_gen_andc_i32(TCGV_HIGH(ret), TCGV_HIGH(v2), t0); 2499 tcg_gen_or_i32(TCGV_HIGH(ret), TCGV_HIGH(ret), t1); 2500 } 2501 tcg_temp_free_i32(t0); 2502 tcg_temp_free_i32(t1); 2503 } else if (TCG_TARGET_HAS_movcond_i64) { 2504 tcg_gen_op6i_i64(INDEX_op_movcond_i64, ret, c1, c2, v1, v2, cond); 2505 } else { 2506 TCGv_i64 t0 = tcg_temp_ebb_new_i64(); 2507 TCGv_i64 t1 = tcg_temp_ebb_new_i64(); 2508 tcg_gen_setcond_i64(cond, t0, c1, c2); 2509 tcg_gen_neg_i64(t0, t0); 2510 tcg_gen_and_i64(t1, v1, t0); 2511 tcg_gen_andc_i64(ret, v2, t0); 2512 tcg_gen_or_i64(ret, ret, t1); 2513 tcg_temp_free_i64(t0); 2514 tcg_temp_free_i64(t1); 2515 } 2516 } 2517 2518 void tcg_gen_add2_i64(TCGv_i64 rl, TCGv_i64 rh, TCGv_i64 al, 2519 TCGv_i64 ah, TCGv_i64 bl, TCGv_i64 bh) 2520 { 2521 if (TCG_TARGET_HAS_add2_i64) { 2522 tcg_gen_op6_i64(INDEX_op_add2_i64, rl, rh, al, ah, bl, bh); 2523 } else { 2524 TCGv_i64 t0 = tcg_temp_ebb_new_i64(); 2525 TCGv_i64 t1 = tcg_temp_ebb_new_i64(); 2526 tcg_gen_add_i64(t0, al, bl); 2527 tcg_gen_setcond_i64(TCG_COND_LTU, t1, t0, al); 2528 tcg_gen_add_i64(rh, ah, bh); 2529 tcg_gen_add_i64(rh, rh, t1); 2530 tcg_gen_mov_i64(rl, t0); 2531 tcg_temp_free_i64(t0); 2532 tcg_temp_free_i64(t1); 2533 } 2534 } 2535 2536 void tcg_gen_sub2_i64(TCGv_i64 rl, TCGv_i64 rh, TCGv_i64 al, 2537 TCGv_i64 ah, TCGv_i64 bl, TCGv_i64 bh) 2538 { 2539 if (TCG_TARGET_HAS_sub2_i64) { 2540 tcg_gen_op6_i64(INDEX_op_sub2_i64, rl, rh, al, ah, bl, bh); 2541 } else { 2542 TCGv_i64 t0 = tcg_temp_ebb_new_i64(); 2543 TCGv_i64 t1 = tcg_temp_ebb_new_i64(); 2544 tcg_gen_sub_i64(t0, al, bl); 2545 tcg_gen_setcond_i64(TCG_COND_LTU, t1, al, bl); 2546 tcg_gen_sub_i64(rh, ah, bh); 2547 tcg_gen_sub_i64(rh, rh, t1); 2548 tcg_gen_mov_i64(rl, t0); 2549 tcg_temp_free_i64(t0); 2550 tcg_temp_free_i64(t1); 2551 } 2552 } 2553 2554 void tcg_gen_mulu2_i64(TCGv_i64 rl, TCGv_i64 rh, TCGv_i64 arg1, TCGv_i64 arg2) 2555 { 2556 if (TCG_TARGET_HAS_mulu2_i64) { 2557 tcg_gen_op4_i64(INDEX_op_mulu2_i64, rl, rh, arg1, arg2); 2558 } else if (TCG_TARGET_HAS_muluh_i64) { 2559 TCGv_i64 t = tcg_temp_ebb_new_i64(); 2560 tcg_gen_op3_i64(INDEX_op_mul_i64, t, arg1, arg2); 2561 tcg_gen_op3_i64(INDEX_op_muluh_i64, rh, arg1, arg2); 2562 tcg_gen_mov_i64(rl, t); 2563 tcg_temp_free_i64(t); 2564 } else { 2565 TCGv_i64 t0 = tcg_temp_ebb_new_i64(); 2566 tcg_gen_mul_i64(t0, arg1, arg2); 2567 gen_helper_muluh_i64(rh, arg1, arg2); 2568 tcg_gen_mov_i64(rl, t0); 2569 tcg_temp_free_i64(t0); 2570 } 2571 } 2572 2573 void tcg_gen_muls2_i64(TCGv_i64 rl, TCGv_i64 rh, TCGv_i64 arg1, TCGv_i64 arg2) 2574 { 2575 if (TCG_TARGET_HAS_muls2_i64) { 2576 tcg_gen_op4_i64(INDEX_op_muls2_i64, rl, rh, arg1, arg2); 2577 } else if (TCG_TARGET_HAS_mulsh_i64) { 2578 TCGv_i64 t = tcg_temp_ebb_new_i64(); 2579 tcg_gen_op3_i64(INDEX_op_mul_i64, t, arg1, arg2); 2580 tcg_gen_op3_i64(INDEX_op_mulsh_i64, rh, arg1, arg2); 2581 tcg_gen_mov_i64(rl, t); 2582 tcg_temp_free_i64(t); 2583 } else if (TCG_TARGET_HAS_mulu2_i64 || TCG_TARGET_HAS_muluh_i64) { 2584 TCGv_i64 t0 = tcg_temp_ebb_new_i64(); 2585 TCGv_i64 t1 = tcg_temp_ebb_new_i64(); 2586 TCGv_i64 t2 = tcg_temp_ebb_new_i64(); 2587 TCGv_i64 t3 = tcg_temp_ebb_new_i64(); 2588 tcg_gen_mulu2_i64(t0, t1, arg1, arg2); 2589 /* Adjust for negative inputs. */ 2590 tcg_gen_sari_i64(t2, arg1, 63); 2591 tcg_gen_sari_i64(t3, arg2, 63); 2592 tcg_gen_and_i64(t2, t2, arg2); 2593 tcg_gen_and_i64(t3, t3, arg1); 2594 tcg_gen_sub_i64(rh, t1, t2); 2595 tcg_gen_sub_i64(rh, rh, t3); 2596 tcg_gen_mov_i64(rl, t0); 2597 tcg_temp_free_i64(t0); 2598 tcg_temp_free_i64(t1); 2599 tcg_temp_free_i64(t2); 2600 tcg_temp_free_i64(t3); 2601 } else { 2602 TCGv_i64 t0 = tcg_temp_ebb_new_i64(); 2603 tcg_gen_mul_i64(t0, arg1, arg2); 2604 gen_helper_mulsh_i64(rh, arg1, arg2); 2605 tcg_gen_mov_i64(rl, t0); 2606 tcg_temp_free_i64(t0); 2607 } 2608 } 2609 2610 void tcg_gen_mulsu2_i64(TCGv_i64 rl, TCGv_i64 rh, TCGv_i64 arg1, TCGv_i64 arg2) 2611 { 2612 TCGv_i64 t0 = tcg_temp_ebb_new_i64(); 2613 TCGv_i64 t1 = tcg_temp_ebb_new_i64(); 2614 TCGv_i64 t2 = tcg_temp_ebb_new_i64(); 2615 tcg_gen_mulu2_i64(t0, t1, arg1, arg2); 2616 /* Adjust for negative input for the signed arg1. */ 2617 tcg_gen_sari_i64(t2, arg1, 63); 2618 tcg_gen_and_i64(t2, t2, arg2); 2619 tcg_gen_sub_i64(rh, t1, t2); 2620 tcg_gen_mov_i64(rl, t0); 2621 tcg_temp_free_i64(t0); 2622 tcg_temp_free_i64(t1); 2623 tcg_temp_free_i64(t2); 2624 } 2625 2626 void tcg_gen_smin_i64(TCGv_i64 ret, TCGv_i64 a, TCGv_i64 b) 2627 { 2628 tcg_gen_movcond_i64(TCG_COND_LT, ret, a, b, a, b); 2629 } 2630 2631 void tcg_gen_umin_i64(TCGv_i64 ret, TCGv_i64 a, TCGv_i64 b) 2632 { 2633 tcg_gen_movcond_i64(TCG_COND_LTU, ret, a, b, a, b); 2634 } 2635 2636 void tcg_gen_smax_i64(TCGv_i64 ret, TCGv_i64 a, TCGv_i64 b) 2637 { 2638 tcg_gen_movcond_i64(TCG_COND_LT, ret, a, b, b, a); 2639 } 2640 2641 void tcg_gen_umax_i64(TCGv_i64 ret, TCGv_i64 a, TCGv_i64 b) 2642 { 2643 tcg_gen_movcond_i64(TCG_COND_LTU, ret, a, b, b, a); 2644 } 2645 2646 void tcg_gen_abs_i64(TCGv_i64 ret, TCGv_i64 a) 2647 { 2648 TCGv_i64 t = tcg_temp_ebb_new_i64(); 2649 2650 tcg_gen_sari_i64(t, a, 63); 2651 tcg_gen_xor_i64(ret, a, t); 2652 tcg_gen_sub_i64(ret, ret, t); 2653 tcg_temp_free_i64(t); 2654 } 2655 2656 /* Size changing operations. */ 2657 2658 void tcg_gen_extrl_i64_i32(TCGv_i32 ret, TCGv_i64 arg) 2659 { 2660 if (TCG_TARGET_REG_BITS == 32) { 2661 tcg_gen_mov_i32(ret, TCGV_LOW(arg)); 2662 } else if (TCG_TARGET_HAS_extrl_i64_i32) { 2663 tcg_gen_op2(INDEX_op_extrl_i64_i32, 2664 tcgv_i32_arg(ret), tcgv_i64_arg(arg)); 2665 } else { 2666 tcg_gen_mov_i32(ret, (TCGv_i32)arg); 2667 } 2668 } 2669 2670 void tcg_gen_extrh_i64_i32(TCGv_i32 ret, TCGv_i64 arg) 2671 { 2672 if (TCG_TARGET_REG_BITS == 32) { 2673 tcg_gen_mov_i32(ret, TCGV_HIGH(arg)); 2674 } else if (TCG_TARGET_HAS_extrh_i64_i32) { 2675 tcg_gen_op2(INDEX_op_extrh_i64_i32, 2676 tcgv_i32_arg(ret), tcgv_i64_arg(arg)); 2677 } else { 2678 TCGv_i64 t = tcg_temp_ebb_new_i64(); 2679 tcg_gen_shri_i64(t, arg, 32); 2680 tcg_gen_mov_i32(ret, (TCGv_i32)t); 2681 tcg_temp_free_i64(t); 2682 } 2683 } 2684 2685 void tcg_gen_extu_i32_i64(TCGv_i64 ret, TCGv_i32 arg) 2686 { 2687 if (TCG_TARGET_REG_BITS == 32) { 2688 tcg_gen_mov_i32(TCGV_LOW(ret), arg); 2689 tcg_gen_movi_i32(TCGV_HIGH(ret), 0); 2690 } else { 2691 tcg_gen_op2(INDEX_op_extu_i32_i64, 2692 tcgv_i64_arg(ret), tcgv_i32_arg(arg)); 2693 } 2694 } 2695 2696 void tcg_gen_ext_i32_i64(TCGv_i64 ret, TCGv_i32 arg) 2697 { 2698 if (TCG_TARGET_REG_BITS == 32) { 2699 tcg_gen_mov_i32(TCGV_LOW(ret), arg); 2700 tcg_gen_sari_i32(TCGV_HIGH(ret), TCGV_LOW(ret), 31); 2701 } else { 2702 tcg_gen_op2(INDEX_op_ext_i32_i64, 2703 tcgv_i64_arg(ret), tcgv_i32_arg(arg)); 2704 } 2705 } 2706 2707 void tcg_gen_concat_i32_i64(TCGv_i64 dest, TCGv_i32 low, TCGv_i32 high) 2708 { 2709 TCGv_i64 tmp; 2710 2711 if (TCG_TARGET_REG_BITS == 32) { 2712 tcg_gen_mov_i32(TCGV_LOW(dest), low); 2713 tcg_gen_mov_i32(TCGV_HIGH(dest), high); 2714 return; 2715 } 2716 2717 tmp = tcg_temp_ebb_new_i64(); 2718 /* These extensions are only needed for type correctness. 2719 We may be able to do better given target specific information. */ 2720 tcg_gen_extu_i32_i64(tmp, high); 2721 tcg_gen_extu_i32_i64(dest, low); 2722 /* If deposit is available, use it. Otherwise use the extra 2723 knowledge that we have of the zero-extensions above. */ 2724 if (TCG_TARGET_HAS_deposit_i64 && TCG_TARGET_deposit_i64_valid(32, 32)) { 2725 tcg_gen_deposit_i64(dest, dest, tmp, 32, 32); 2726 } else { 2727 tcg_gen_shli_i64(tmp, tmp, 32); 2728 tcg_gen_or_i64(dest, dest, tmp); 2729 } 2730 tcg_temp_free_i64(tmp); 2731 } 2732 2733 void tcg_gen_extr_i64_i32(TCGv_i32 lo, TCGv_i32 hi, TCGv_i64 arg) 2734 { 2735 if (TCG_TARGET_REG_BITS == 32) { 2736 tcg_gen_mov_i32(lo, TCGV_LOW(arg)); 2737 tcg_gen_mov_i32(hi, TCGV_HIGH(arg)); 2738 } else { 2739 tcg_gen_extrl_i64_i32(lo, arg); 2740 tcg_gen_extrh_i64_i32(hi, arg); 2741 } 2742 } 2743 2744 void tcg_gen_extr32_i64(TCGv_i64 lo, TCGv_i64 hi, TCGv_i64 arg) 2745 { 2746 tcg_gen_ext32u_i64(lo, arg); 2747 tcg_gen_shri_i64(hi, arg, 32); 2748 } 2749 2750 void tcg_gen_extr_i128_i64(TCGv_i64 lo, TCGv_i64 hi, TCGv_i128 arg) 2751 { 2752 tcg_gen_mov_i64(lo, TCGV128_LOW(arg)); 2753 tcg_gen_mov_i64(hi, TCGV128_HIGH(arg)); 2754 } 2755 2756 void tcg_gen_concat_i64_i128(TCGv_i128 ret, TCGv_i64 lo, TCGv_i64 hi) 2757 { 2758 tcg_gen_mov_i64(TCGV128_LOW(ret), lo); 2759 tcg_gen_mov_i64(TCGV128_HIGH(ret), hi); 2760 } 2761 2762 void tcg_gen_mov_i128(TCGv_i128 dst, TCGv_i128 src) 2763 { 2764 if (dst != src) { 2765 tcg_gen_mov_i64(TCGV128_LOW(dst), TCGV128_LOW(src)); 2766 tcg_gen_mov_i64(TCGV128_HIGH(dst), TCGV128_HIGH(src)); 2767 } 2768 } 2769 2770 /* QEMU specific operations. */ 2771 2772 void tcg_gen_exit_tb(const TranslationBlock *tb, unsigned idx) 2773 { 2774 /* 2775 * Let the jit code return the read-only version of the 2776 * TranslationBlock, so that we minimize the pc-relative 2777 * distance of the address of the exit_tb code to TB. 2778 * This will improve utilization of pc-relative address loads. 2779 * 2780 * TODO: Move this to translator_loop, so that all const 2781 * TranslationBlock pointers refer to read-only memory. 2782 * This requires coordination with targets that do not use 2783 * the translator_loop. 2784 */ 2785 uintptr_t val = (uintptr_t)tcg_splitwx_to_rx((void *)tb) + idx; 2786 2787 if (tb == NULL) { 2788 tcg_debug_assert(idx == 0); 2789 } else if (idx <= TB_EXIT_IDXMAX) { 2790 #ifdef CONFIG_DEBUG_TCG 2791 /* This is an exit following a goto_tb. Verify that we have 2792 seen this numbered exit before, via tcg_gen_goto_tb. */ 2793 tcg_debug_assert(tcg_ctx->goto_tb_issue_mask & (1 << idx)); 2794 #endif 2795 } else { 2796 /* This is an exit via the exitreq label. */ 2797 tcg_debug_assert(idx == TB_EXIT_REQUESTED); 2798 } 2799 2800 plugin_gen_disable_mem_helpers(); 2801 tcg_gen_op1i(INDEX_op_exit_tb, val); 2802 } 2803 2804 void tcg_gen_goto_tb(unsigned idx) 2805 { 2806 /* We tested CF_NO_GOTO_TB in translator_use_goto_tb. */ 2807 tcg_debug_assert(!(tcg_ctx->gen_tb->cflags & CF_NO_GOTO_TB)); 2808 /* We only support two chained exits. */ 2809 tcg_debug_assert(idx <= TB_EXIT_IDXMAX); 2810 #ifdef CONFIG_DEBUG_TCG 2811 /* Verify that we haven't seen this numbered exit before. */ 2812 tcg_debug_assert((tcg_ctx->goto_tb_issue_mask & (1 << idx)) == 0); 2813 tcg_ctx->goto_tb_issue_mask |= 1 << idx; 2814 #endif 2815 plugin_gen_disable_mem_helpers(); 2816 tcg_gen_op1i(INDEX_op_goto_tb, idx); 2817 } 2818 2819 void tcg_gen_lookup_and_goto_ptr(void) 2820 { 2821 TCGv_ptr ptr; 2822 2823 if (tcg_ctx->gen_tb->cflags & CF_NO_GOTO_PTR) { 2824 tcg_gen_exit_tb(NULL, 0); 2825 return; 2826 } 2827 2828 plugin_gen_disable_mem_helpers(); 2829 ptr = tcg_temp_ebb_new_ptr(); 2830 gen_helper_lookup_tb_ptr(ptr, cpu_env); 2831 tcg_gen_op1i(INDEX_op_goto_ptr, tcgv_ptr_arg(ptr)); 2832 tcg_temp_free_ptr(ptr); 2833 } 2834 2835 static inline MemOp tcg_canonicalize_memop(MemOp op, bool is64, bool st) 2836 { 2837 /* Trigger the asserts within as early as possible. */ 2838 unsigned a_bits = get_alignment_bits(op); 2839 2840 /* Prefer MO_ALIGN+MO_XX over MO_ALIGN_XX+MO_XX */ 2841 if (a_bits == (op & MO_SIZE)) { 2842 op = (op & ~MO_AMASK) | MO_ALIGN; 2843 } 2844 2845 switch (op & MO_SIZE) { 2846 case MO_8: 2847 op &= ~MO_BSWAP; 2848 break; 2849 case MO_16: 2850 break; 2851 case MO_32: 2852 if (!is64) { 2853 op &= ~MO_SIGN; 2854 } 2855 break; 2856 case MO_64: 2857 if (is64) { 2858 op &= ~MO_SIGN; 2859 break; 2860 } 2861 /* fall through */ 2862 default: 2863 g_assert_not_reached(); 2864 } 2865 if (st) { 2866 op &= ~MO_SIGN; 2867 } 2868 return op; 2869 } 2870 2871 static void gen_ldst_i32(TCGOpcode opc, TCGv_i32 val, TCGv addr, 2872 MemOp memop, TCGArg idx) 2873 { 2874 MemOpIdx oi = make_memop_idx(memop, idx); 2875 #if TARGET_LONG_BITS == 32 2876 tcg_gen_op3i_i32(opc, val, addr, oi); 2877 #else 2878 if (TCG_TARGET_REG_BITS == 32) { 2879 tcg_gen_op4i_i32(opc, val, TCGV_LOW(addr), TCGV_HIGH(addr), oi); 2880 } else { 2881 tcg_gen_op3(opc, tcgv_i32_arg(val), tcgv_i64_arg(addr), oi); 2882 } 2883 #endif 2884 } 2885 2886 static void gen_ldst_i64(TCGOpcode opc, TCGv_i64 val, TCGv addr, 2887 MemOp memop, TCGArg idx) 2888 { 2889 MemOpIdx oi = make_memop_idx(memop, idx); 2890 #if TARGET_LONG_BITS == 32 2891 if (TCG_TARGET_REG_BITS == 32) { 2892 tcg_gen_op4i_i32(opc, TCGV_LOW(val), TCGV_HIGH(val), addr, oi); 2893 } else { 2894 tcg_gen_op3(opc, tcgv_i64_arg(val), tcgv_i32_arg(addr), oi); 2895 } 2896 #else 2897 if (TCG_TARGET_REG_BITS == 32) { 2898 tcg_gen_op5i_i32(opc, TCGV_LOW(val), TCGV_HIGH(val), 2899 TCGV_LOW(addr), TCGV_HIGH(addr), oi); 2900 } else { 2901 tcg_gen_op3i_i64(opc, val, addr, oi); 2902 } 2903 #endif 2904 } 2905 2906 static void tcg_gen_req_mo(TCGBar type) 2907 { 2908 #ifdef TCG_GUEST_DEFAULT_MO 2909 type &= TCG_GUEST_DEFAULT_MO; 2910 #endif 2911 type &= ~TCG_TARGET_DEFAULT_MO; 2912 if (type) { 2913 tcg_gen_mb(type | TCG_BAR_SC); 2914 } 2915 } 2916 2917 static inline TCGv plugin_prep_mem_callbacks(TCGv vaddr) 2918 { 2919 #ifdef CONFIG_PLUGIN 2920 if (tcg_ctx->plugin_insn != NULL) { 2921 /* Save a copy of the vaddr for use after a load. */ 2922 TCGv temp = tcg_temp_new(); 2923 tcg_gen_mov_tl(temp, vaddr); 2924 return temp; 2925 } 2926 #endif 2927 return vaddr; 2928 } 2929 2930 static void plugin_gen_mem_callbacks(TCGv vaddr, MemOpIdx oi, 2931 enum qemu_plugin_mem_rw rw) 2932 { 2933 #ifdef CONFIG_PLUGIN 2934 if (tcg_ctx->plugin_insn != NULL) { 2935 qemu_plugin_meminfo_t info = make_plugin_meminfo(oi, rw); 2936 plugin_gen_empty_mem_callback(vaddr, info); 2937 tcg_temp_free(vaddr); 2938 } 2939 #endif 2940 } 2941 2942 void tcg_gen_qemu_ld_i32(TCGv_i32 val, TCGv addr, TCGArg idx, MemOp memop) 2943 { 2944 MemOp orig_memop; 2945 MemOpIdx oi; 2946 2947 tcg_gen_req_mo(TCG_MO_LD_LD | TCG_MO_ST_LD); 2948 memop = tcg_canonicalize_memop(memop, 0, 0); 2949 oi = make_memop_idx(memop, idx); 2950 2951 orig_memop = memop; 2952 if (!TCG_TARGET_HAS_MEMORY_BSWAP && (memop & MO_BSWAP)) { 2953 memop &= ~MO_BSWAP; 2954 /* The bswap primitive benefits from zero-extended input. */ 2955 if ((memop & MO_SSIZE) == MO_SW) { 2956 memop &= ~MO_SIGN; 2957 } 2958 } 2959 2960 addr = plugin_prep_mem_callbacks(addr); 2961 gen_ldst_i32(INDEX_op_qemu_ld_i32, val, addr, memop, idx); 2962 plugin_gen_mem_callbacks(addr, oi, QEMU_PLUGIN_MEM_R); 2963 2964 if ((orig_memop ^ memop) & MO_BSWAP) { 2965 switch (orig_memop & MO_SIZE) { 2966 case MO_16: 2967 tcg_gen_bswap16_i32(val, val, (orig_memop & MO_SIGN 2968 ? TCG_BSWAP_IZ | TCG_BSWAP_OS 2969 : TCG_BSWAP_IZ | TCG_BSWAP_OZ)); 2970 break; 2971 case MO_32: 2972 tcg_gen_bswap32_i32(val, val); 2973 break; 2974 default: 2975 g_assert_not_reached(); 2976 } 2977 } 2978 } 2979 2980 void tcg_gen_qemu_st_i32(TCGv_i32 val, TCGv addr, TCGArg idx, MemOp memop) 2981 { 2982 TCGv_i32 swap = NULL; 2983 MemOpIdx oi; 2984 2985 tcg_gen_req_mo(TCG_MO_LD_ST | TCG_MO_ST_ST); 2986 memop = tcg_canonicalize_memop(memop, 0, 1); 2987 oi = make_memop_idx(memop, idx); 2988 2989 if (!TCG_TARGET_HAS_MEMORY_BSWAP && (memop & MO_BSWAP)) { 2990 swap = tcg_temp_ebb_new_i32(); 2991 switch (memop & MO_SIZE) { 2992 case MO_16: 2993 tcg_gen_bswap16_i32(swap, val, 0); 2994 break; 2995 case MO_32: 2996 tcg_gen_bswap32_i32(swap, val); 2997 break; 2998 default: 2999 g_assert_not_reached(); 3000 } 3001 val = swap; 3002 memop &= ~MO_BSWAP; 3003 } 3004 3005 addr = plugin_prep_mem_callbacks(addr); 3006 if (TCG_TARGET_HAS_qemu_st8_i32 && (memop & MO_SIZE) == MO_8) { 3007 gen_ldst_i32(INDEX_op_qemu_st8_i32, val, addr, memop, idx); 3008 } else { 3009 gen_ldst_i32(INDEX_op_qemu_st_i32, val, addr, memop, idx); 3010 } 3011 plugin_gen_mem_callbacks(addr, oi, QEMU_PLUGIN_MEM_W); 3012 3013 if (swap) { 3014 tcg_temp_free_i32(swap); 3015 } 3016 } 3017 3018 void tcg_gen_qemu_ld_i64(TCGv_i64 val, TCGv addr, TCGArg idx, MemOp memop) 3019 { 3020 MemOp orig_memop; 3021 MemOpIdx oi; 3022 3023 if (TCG_TARGET_REG_BITS == 32 && (memop & MO_SIZE) < MO_64) { 3024 tcg_gen_qemu_ld_i32(TCGV_LOW(val), addr, idx, memop); 3025 if (memop & MO_SIGN) { 3026 tcg_gen_sari_i32(TCGV_HIGH(val), TCGV_LOW(val), 31); 3027 } else { 3028 tcg_gen_movi_i32(TCGV_HIGH(val), 0); 3029 } 3030 return; 3031 } 3032 3033 tcg_gen_req_mo(TCG_MO_LD_LD | TCG_MO_ST_LD); 3034 memop = tcg_canonicalize_memop(memop, 1, 0); 3035 oi = make_memop_idx(memop, idx); 3036 3037 orig_memop = memop; 3038 if (!TCG_TARGET_HAS_MEMORY_BSWAP && (memop & MO_BSWAP)) { 3039 memop &= ~MO_BSWAP; 3040 /* The bswap primitive benefits from zero-extended input. */ 3041 if ((memop & MO_SIGN) && (memop & MO_SIZE) < MO_64) { 3042 memop &= ~MO_SIGN; 3043 } 3044 } 3045 3046 addr = plugin_prep_mem_callbacks(addr); 3047 gen_ldst_i64(INDEX_op_qemu_ld_i64, val, addr, memop, idx); 3048 plugin_gen_mem_callbacks(addr, oi, QEMU_PLUGIN_MEM_R); 3049 3050 if ((orig_memop ^ memop) & MO_BSWAP) { 3051 int flags = (orig_memop & MO_SIGN 3052 ? TCG_BSWAP_IZ | TCG_BSWAP_OS 3053 : TCG_BSWAP_IZ | TCG_BSWAP_OZ); 3054 switch (orig_memop & MO_SIZE) { 3055 case MO_16: 3056 tcg_gen_bswap16_i64(val, val, flags); 3057 break; 3058 case MO_32: 3059 tcg_gen_bswap32_i64(val, val, flags); 3060 break; 3061 case MO_64: 3062 tcg_gen_bswap64_i64(val, val); 3063 break; 3064 default: 3065 g_assert_not_reached(); 3066 } 3067 } 3068 } 3069 3070 void tcg_gen_qemu_st_i64(TCGv_i64 val, TCGv addr, TCGArg idx, MemOp memop) 3071 { 3072 TCGv_i64 swap = NULL; 3073 MemOpIdx oi; 3074 3075 if (TCG_TARGET_REG_BITS == 32 && (memop & MO_SIZE) < MO_64) { 3076 tcg_gen_qemu_st_i32(TCGV_LOW(val), addr, idx, memop); 3077 return; 3078 } 3079 3080 tcg_gen_req_mo(TCG_MO_LD_ST | TCG_MO_ST_ST); 3081 memop = tcg_canonicalize_memop(memop, 1, 1); 3082 oi = make_memop_idx(memop, idx); 3083 3084 if (!TCG_TARGET_HAS_MEMORY_BSWAP && (memop & MO_BSWAP)) { 3085 swap = tcg_temp_ebb_new_i64(); 3086 switch (memop & MO_SIZE) { 3087 case MO_16: 3088 tcg_gen_bswap16_i64(swap, val, 0); 3089 break; 3090 case MO_32: 3091 tcg_gen_bswap32_i64(swap, val, 0); 3092 break; 3093 case MO_64: 3094 tcg_gen_bswap64_i64(swap, val); 3095 break; 3096 default: 3097 g_assert_not_reached(); 3098 } 3099 val = swap; 3100 memop &= ~MO_BSWAP; 3101 } 3102 3103 addr = plugin_prep_mem_callbacks(addr); 3104 gen_ldst_i64(INDEX_op_qemu_st_i64, val, addr, memop, idx); 3105 plugin_gen_mem_callbacks(addr, oi, QEMU_PLUGIN_MEM_W); 3106 3107 if (swap) { 3108 tcg_temp_free_i64(swap); 3109 } 3110 } 3111 3112 static void canonicalize_memop_i128_as_i64(MemOp ret[2], MemOp orig) 3113 { 3114 MemOp mop_1 = orig, mop_2; 3115 3116 tcg_debug_assert((orig & MO_SIZE) == MO_128); 3117 tcg_debug_assert((orig & MO_SIGN) == 0); 3118 3119 /* Use a memory ordering implemented by the host. */ 3120 if (!TCG_TARGET_HAS_MEMORY_BSWAP && (orig & MO_BSWAP)) { 3121 mop_1 &= ~MO_BSWAP; 3122 } 3123 3124 /* Reduce the size to 64-bit. */ 3125 mop_1 = (mop_1 & ~MO_SIZE) | MO_64; 3126 3127 /* Retain the alignment constraints of the original. */ 3128 switch (orig & MO_AMASK) { 3129 case MO_UNALN: 3130 case MO_ALIGN_2: 3131 case MO_ALIGN_4: 3132 mop_2 = mop_1; 3133 break; 3134 case MO_ALIGN_8: 3135 /* Prefer MO_ALIGN+MO_64 to MO_ALIGN_8+MO_64. */ 3136 mop_1 = (mop_1 & ~MO_AMASK) | MO_ALIGN; 3137 mop_2 = mop_1; 3138 break; 3139 case MO_ALIGN: 3140 /* Second has 8-byte alignment; first has 16-byte alignment. */ 3141 mop_2 = mop_1; 3142 mop_1 = (mop_1 & ~MO_AMASK) | MO_ALIGN_16; 3143 break; 3144 case MO_ALIGN_16: 3145 case MO_ALIGN_32: 3146 case MO_ALIGN_64: 3147 /* Second has 8-byte alignment; first retains original. */ 3148 mop_2 = (mop_1 & ~MO_AMASK) | MO_ALIGN; 3149 break; 3150 default: 3151 g_assert_not_reached(); 3152 } 3153 ret[0] = mop_1; 3154 ret[1] = mop_2; 3155 } 3156 3157 void tcg_gen_qemu_ld_i128(TCGv_i128 val, TCGv addr, TCGArg idx, MemOp memop) 3158 { 3159 MemOp mop[2]; 3160 TCGv addr_p8; 3161 TCGv_i64 x, y; 3162 3163 canonicalize_memop_i128_as_i64(mop, memop); 3164 3165 tcg_gen_req_mo(TCG_MO_LD_LD | TCG_MO_ST_LD); 3166 addr = plugin_prep_mem_callbacks(addr); 3167 3168 /* TODO: respect atomicity of the operation. */ 3169 /* TODO: allow the tcg backend to see the whole operation. */ 3170 3171 /* 3172 * Since there are no global TCGv_i128, there is no visible state 3173 * changed if the second load faults. Load directly into the two 3174 * subwords. 3175 */ 3176 if ((memop & MO_BSWAP) == MO_LE) { 3177 x = TCGV128_LOW(val); 3178 y = TCGV128_HIGH(val); 3179 } else { 3180 x = TCGV128_HIGH(val); 3181 y = TCGV128_LOW(val); 3182 } 3183 3184 gen_ldst_i64(INDEX_op_qemu_ld_i64, x, addr, mop[0], idx); 3185 3186 if ((mop[0] ^ memop) & MO_BSWAP) { 3187 tcg_gen_bswap64_i64(x, x); 3188 } 3189 3190 addr_p8 = tcg_temp_new(); 3191 tcg_gen_addi_tl(addr_p8, addr, 8); 3192 gen_ldst_i64(INDEX_op_qemu_ld_i64, y, addr_p8, mop[1], idx); 3193 tcg_temp_free(addr_p8); 3194 3195 if ((mop[0] ^ memop) & MO_BSWAP) { 3196 tcg_gen_bswap64_i64(y, y); 3197 } 3198 3199 plugin_gen_mem_callbacks(addr, make_memop_idx(memop, idx), 3200 QEMU_PLUGIN_MEM_R); 3201 } 3202 3203 void tcg_gen_qemu_st_i128(TCGv_i128 val, TCGv addr, TCGArg idx, MemOp memop) 3204 { 3205 MemOp mop[2]; 3206 TCGv addr_p8; 3207 TCGv_i64 x, y; 3208 3209 canonicalize_memop_i128_as_i64(mop, memop); 3210 3211 tcg_gen_req_mo(TCG_MO_ST_LD | TCG_MO_ST_ST); 3212 addr = plugin_prep_mem_callbacks(addr); 3213 3214 /* TODO: respect atomicity of the operation. */ 3215 /* TODO: allow the tcg backend to see the whole operation. */ 3216 3217 if ((memop & MO_BSWAP) == MO_LE) { 3218 x = TCGV128_LOW(val); 3219 y = TCGV128_HIGH(val); 3220 } else { 3221 x = TCGV128_HIGH(val); 3222 y = TCGV128_LOW(val); 3223 } 3224 3225 addr_p8 = tcg_temp_new(); 3226 if ((mop[0] ^ memop) & MO_BSWAP) { 3227 TCGv_i64 t = tcg_temp_ebb_new_i64(); 3228 3229 tcg_gen_bswap64_i64(t, x); 3230 gen_ldst_i64(INDEX_op_qemu_st_i64, t, addr, mop[0], idx); 3231 tcg_gen_bswap64_i64(t, y); 3232 tcg_gen_addi_tl(addr_p8, addr, 8); 3233 gen_ldst_i64(INDEX_op_qemu_st_i64, t, addr_p8, mop[1], idx); 3234 tcg_temp_free_i64(t); 3235 } else { 3236 gen_ldst_i64(INDEX_op_qemu_st_i64, x, addr, mop[0], idx); 3237 tcg_gen_addi_tl(addr_p8, addr, 8); 3238 gen_ldst_i64(INDEX_op_qemu_st_i64, y, addr_p8, mop[1], idx); 3239 } 3240 tcg_temp_free(addr_p8); 3241 3242 plugin_gen_mem_callbacks(addr, make_memop_idx(memop, idx), 3243 QEMU_PLUGIN_MEM_W); 3244 } 3245 3246 static void tcg_gen_ext_i32(TCGv_i32 ret, TCGv_i32 val, MemOp opc) 3247 { 3248 switch (opc & MO_SSIZE) { 3249 case MO_SB: 3250 tcg_gen_ext8s_i32(ret, val); 3251 break; 3252 case MO_UB: 3253 tcg_gen_ext8u_i32(ret, val); 3254 break; 3255 case MO_SW: 3256 tcg_gen_ext16s_i32(ret, val); 3257 break; 3258 case MO_UW: 3259 tcg_gen_ext16u_i32(ret, val); 3260 break; 3261 default: 3262 tcg_gen_mov_i32(ret, val); 3263 break; 3264 } 3265 } 3266 3267 static void tcg_gen_ext_i64(TCGv_i64 ret, TCGv_i64 val, MemOp opc) 3268 { 3269 switch (opc & MO_SSIZE) { 3270 case MO_SB: 3271 tcg_gen_ext8s_i64(ret, val); 3272 break; 3273 case MO_UB: 3274 tcg_gen_ext8u_i64(ret, val); 3275 break; 3276 case MO_SW: 3277 tcg_gen_ext16s_i64(ret, val); 3278 break; 3279 case MO_UW: 3280 tcg_gen_ext16u_i64(ret, val); 3281 break; 3282 case MO_SL: 3283 tcg_gen_ext32s_i64(ret, val); 3284 break; 3285 case MO_UL: 3286 tcg_gen_ext32u_i64(ret, val); 3287 break; 3288 default: 3289 tcg_gen_mov_i64(ret, val); 3290 break; 3291 } 3292 } 3293 3294 typedef void (*gen_atomic_cx_i32)(TCGv_i32, TCGv_env, TCGv, 3295 TCGv_i32, TCGv_i32, TCGv_i32); 3296 typedef void (*gen_atomic_cx_i64)(TCGv_i64, TCGv_env, TCGv, 3297 TCGv_i64, TCGv_i64, TCGv_i32); 3298 typedef void (*gen_atomic_cx_i128)(TCGv_i128, TCGv_env, TCGv, 3299 TCGv_i128, TCGv_i128, TCGv_i32); 3300 typedef void (*gen_atomic_op_i32)(TCGv_i32, TCGv_env, TCGv, 3301 TCGv_i32, TCGv_i32); 3302 typedef void (*gen_atomic_op_i64)(TCGv_i64, TCGv_env, TCGv, 3303 TCGv_i64, TCGv_i32); 3304 3305 #ifdef CONFIG_ATOMIC64 3306 # define WITH_ATOMIC64(X) X, 3307 #else 3308 # define WITH_ATOMIC64(X) 3309 #endif 3310 #ifdef CONFIG_CMPXCHG128 3311 # define WITH_ATOMIC128(X) X, 3312 #else 3313 # define WITH_ATOMIC128(X) 3314 #endif 3315 3316 static void * const table_cmpxchg[(MO_SIZE | MO_BSWAP) + 1] = { 3317 [MO_8] = gen_helper_atomic_cmpxchgb, 3318 [MO_16 | MO_LE] = gen_helper_atomic_cmpxchgw_le, 3319 [MO_16 | MO_BE] = gen_helper_atomic_cmpxchgw_be, 3320 [MO_32 | MO_LE] = gen_helper_atomic_cmpxchgl_le, 3321 [MO_32 | MO_BE] = gen_helper_atomic_cmpxchgl_be, 3322 WITH_ATOMIC64([MO_64 | MO_LE] = gen_helper_atomic_cmpxchgq_le) 3323 WITH_ATOMIC64([MO_64 | MO_BE] = gen_helper_atomic_cmpxchgq_be) 3324 WITH_ATOMIC128([MO_128 | MO_LE] = gen_helper_atomic_cmpxchgo_le) 3325 WITH_ATOMIC128([MO_128 | MO_BE] = gen_helper_atomic_cmpxchgo_be) 3326 }; 3327 3328 void tcg_gen_nonatomic_cmpxchg_i32(TCGv_i32 retv, TCGv addr, TCGv_i32 cmpv, 3329 TCGv_i32 newv, TCGArg idx, MemOp memop) 3330 { 3331 TCGv_i32 t1 = tcg_temp_ebb_new_i32(); 3332 TCGv_i32 t2 = tcg_temp_ebb_new_i32(); 3333 3334 tcg_gen_ext_i32(t2, cmpv, memop & MO_SIZE); 3335 3336 tcg_gen_qemu_ld_i32(t1, addr, idx, memop & ~MO_SIGN); 3337 tcg_gen_movcond_i32(TCG_COND_EQ, t2, t1, t2, newv, t1); 3338 tcg_gen_qemu_st_i32(t2, addr, idx, memop); 3339 tcg_temp_free_i32(t2); 3340 3341 if (memop & MO_SIGN) { 3342 tcg_gen_ext_i32(retv, t1, memop); 3343 } else { 3344 tcg_gen_mov_i32(retv, t1); 3345 } 3346 tcg_temp_free_i32(t1); 3347 } 3348 3349 void tcg_gen_atomic_cmpxchg_i32(TCGv_i32 retv, TCGv addr, TCGv_i32 cmpv, 3350 TCGv_i32 newv, TCGArg idx, MemOp memop) 3351 { 3352 gen_atomic_cx_i32 gen; 3353 MemOpIdx oi; 3354 3355 if (!(tcg_ctx->gen_tb->cflags & CF_PARALLEL)) { 3356 tcg_gen_nonatomic_cmpxchg_i32(retv, addr, cmpv, newv, idx, memop); 3357 return; 3358 } 3359 3360 memop = tcg_canonicalize_memop(memop, 0, 0); 3361 gen = table_cmpxchg[memop & (MO_SIZE | MO_BSWAP)]; 3362 tcg_debug_assert(gen != NULL); 3363 3364 oi = make_memop_idx(memop & ~MO_SIGN, idx); 3365 gen(retv, cpu_env, addr, cmpv, newv, tcg_constant_i32(oi)); 3366 3367 if (memop & MO_SIGN) { 3368 tcg_gen_ext_i32(retv, retv, memop); 3369 } 3370 } 3371 3372 void tcg_gen_nonatomic_cmpxchg_i64(TCGv_i64 retv, TCGv addr, TCGv_i64 cmpv, 3373 TCGv_i64 newv, TCGArg idx, MemOp memop) 3374 { 3375 TCGv_i64 t1, t2; 3376 3377 if (TCG_TARGET_REG_BITS == 32 && (memop & MO_SIZE) < MO_64) { 3378 tcg_gen_nonatomic_cmpxchg_i32(TCGV_LOW(retv), addr, TCGV_LOW(cmpv), 3379 TCGV_LOW(newv), idx, memop); 3380 if (memop & MO_SIGN) { 3381 tcg_gen_sari_i32(TCGV_HIGH(retv), TCGV_LOW(retv), 31); 3382 } else { 3383 tcg_gen_movi_i32(TCGV_HIGH(retv), 0); 3384 } 3385 return; 3386 } 3387 3388 t1 = tcg_temp_ebb_new_i64(); 3389 t2 = tcg_temp_ebb_new_i64(); 3390 3391 tcg_gen_ext_i64(t2, cmpv, memop & MO_SIZE); 3392 3393 tcg_gen_qemu_ld_i64(t1, addr, idx, memop & ~MO_SIGN); 3394 tcg_gen_movcond_i64(TCG_COND_EQ, t2, t1, t2, newv, t1); 3395 tcg_gen_qemu_st_i64(t2, addr, idx, memop); 3396 tcg_temp_free_i64(t2); 3397 3398 if (memop & MO_SIGN) { 3399 tcg_gen_ext_i64(retv, t1, memop); 3400 } else { 3401 tcg_gen_mov_i64(retv, t1); 3402 } 3403 tcg_temp_free_i64(t1); 3404 } 3405 3406 void tcg_gen_atomic_cmpxchg_i64(TCGv_i64 retv, TCGv addr, TCGv_i64 cmpv, 3407 TCGv_i64 newv, TCGArg idx, MemOp memop) 3408 { 3409 if (!(tcg_ctx->gen_tb->cflags & CF_PARALLEL)) { 3410 tcg_gen_nonatomic_cmpxchg_i64(retv, addr, cmpv, newv, idx, memop); 3411 return; 3412 } 3413 3414 if ((memop & MO_SIZE) == MO_64) { 3415 gen_atomic_cx_i64 gen; 3416 3417 memop = tcg_canonicalize_memop(memop, 1, 0); 3418 gen = table_cmpxchg[memop & (MO_SIZE | MO_BSWAP)]; 3419 if (gen) { 3420 MemOpIdx oi = make_memop_idx(memop, idx); 3421 gen(retv, cpu_env, addr, cmpv, newv, tcg_constant_i32(oi)); 3422 return; 3423 } 3424 3425 gen_helper_exit_atomic(cpu_env); 3426 3427 /* 3428 * Produce a result for a well-formed opcode stream. This satisfies 3429 * liveness for set before used, which happens before this dead code 3430 * is removed. 3431 */ 3432 tcg_gen_movi_i64(retv, 0); 3433 return; 3434 } 3435 3436 if (TCG_TARGET_REG_BITS == 32) { 3437 tcg_gen_atomic_cmpxchg_i32(TCGV_LOW(retv), addr, TCGV_LOW(cmpv), 3438 TCGV_LOW(newv), idx, memop); 3439 if (memop & MO_SIGN) { 3440 tcg_gen_sari_i32(TCGV_HIGH(retv), TCGV_LOW(retv), 31); 3441 } else { 3442 tcg_gen_movi_i32(TCGV_HIGH(retv), 0); 3443 } 3444 } else { 3445 TCGv_i32 c32 = tcg_temp_ebb_new_i32(); 3446 TCGv_i32 n32 = tcg_temp_ebb_new_i32(); 3447 TCGv_i32 r32 = tcg_temp_ebb_new_i32(); 3448 3449 tcg_gen_extrl_i64_i32(c32, cmpv); 3450 tcg_gen_extrl_i64_i32(n32, newv); 3451 tcg_gen_atomic_cmpxchg_i32(r32, addr, c32, n32, idx, memop & ~MO_SIGN); 3452 tcg_temp_free_i32(c32); 3453 tcg_temp_free_i32(n32); 3454 3455 tcg_gen_extu_i32_i64(retv, r32); 3456 tcg_temp_free_i32(r32); 3457 3458 if (memop & MO_SIGN) { 3459 tcg_gen_ext_i64(retv, retv, memop); 3460 } 3461 } 3462 } 3463 3464 void tcg_gen_nonatomic_cmpxchg_i128(TCGv_i128 retv, TCGv addr, TCGv_i128 cmpv, 3465 TCGv_i128 newv, TCGArg idx, MemOp memop) 3466 { 3467 if (TCG_TARGET_REG_BITS == 32) { 3468 /* Inline expansion below is simply too large for 32-bit hosts. */ 3469 gen_atomic_cx_i128 gen = ((memop & MO_BSWAP) == MO_LE 3470 ? gen_helper_nonatomic_cmpxchgo_le 3471 : gen_helper_nonatomic_cmpxchgo_be); 3472 MemOpIdx oi = make_memop_idx(memop, idx); 3473 3474 tcg_debug_assert((memop & MO_SIZE) == MO_128); 3475 tcg_debug_assert((memop & MO_SIGN) == 0); 3476 3477 gen(retv, cpu_env, addr, cmpv, newv, tcg_constant_i32(oi)); 3478 } else { 3479 TCGv_i128 oldv = tcg_temp_ebb_new_i128(); 3480 TCGv_i128 tmpv = tcg_temp_ebb_new_i128(); 3481 TCGv_i64 t0 = tcg_temp_ebb_new_i64(); 3482 TCGv_i64 t1 = tcg_temp_ebb_new_i64(); 3483 TCGv_i64 z = tcg_constant_i64(0); 3484 3485 tcg_gen_qemu_ld_i128(oldv, addr, idx, memop); 3486 3487 /* Compare i128 */ 3488 tcg_gen_xor_i64(t0, TCGV128_LOW(oldv), TCGV128_LOW(cmpv)); 3489 tcg_gen_xor_i64(t1, TCGV128_HIGH(oldv), TCGV128_HIGH(cmpv)); 3490 tcg_gen_or_i64(t0, t0, t1); 3491 3492 /* tmpv = equal ? newv : oldv */ 3493 tcg_gen_movcond_i64(TCG_COND_EQ, TCGV128_LOW(tmpv), t0, z, 3494 TCGV128_LOW(newv), TCGV128_LOW(oldv)); 3495 tcg_gen_movcond_i64(TCG_COND_EQ, TCGV128_HIGH(tmpv), t0, z, 3496 TCGV128_HIGH(newv), TCGV128_HIGH(oldv)); 3497 3498 /* Unconditional writeback. */ 3499 tcg_gen_qemu_st_i128(tmpv, addr, idx, memop); 3500 tcg_gen_mov_i128(retv, oldv); 3501 3502 tcg_temp_free_i64(t0); 3503 tcg_temp_free_i64(t1); 3504 tcg_temp_free_i128(tmpv); 3505 tcg_temp_free_i128(oldv); 3506 } 3507 } 3508 3509 void tcg_gen_atomic_cmpxchg_i128(TCGv_i128 retv, TCGv addr, TCGv_i128 cmpv, 3510 TCGv_i128 newv, TCGArg idx, MemOp memop) 3511 { 3512 gen_atomic_cx_i128 gen; 3513 3514 if (!(tcg_ctx->gen_tb->cflags & CF_PARALLEL)) { 3515 tcg_gen_nonatomic_cmpxchg_i128(retv, addr, cmpv, newv, idx, memop); 3516 return; 3517 } 3518 3519 tcg_debug_assert((memop & MO_SIZE) == MO_128); 3520 tcg_debug_assert((memop & MO_SIGN) == 0); 3521 gen = table_cmpxchg[memop & (MO_SIZE | MO_BSWAP)]; 3522 3523 if (gen) { 3524 MemOpIdx oi = make_memop_idx(memop, idx); 3525 gen(retv, cpu_env, addr, cmpv, newv, tcg_constant_i32(oi)); 3526 return; 3527 } 3528 3529 gen_helper_exit_atomic(cpu_env); 3530 3531 /* 3532 * Produce a result for a well-formed opcode stream. This satisfies 3533 * liveness for set before used, which happens before this dead code 3534 * is removed. 3535 */ 3536 tcg_gen_movi_i64(TCGV128_LOW(retv), 0); 3537 tcg_gen_movi_i64(TCGV128_HIGH(retv), 0); 3538 } 3539 3540 static void do_nonatomic_op_i32(TCGv_i32 ret, TCGv addr, TCGv_i32 val, 3541 TCGArg idx, MemOp memop, bool new_val, 3542 void (*gen)(TCGv_i32, TCGv_i32, TCGv_i32)) 3543 { 3544 TCGv_i32 t1 = tcg_temp_ebb_new_i32(); 3545 TCGv_i32 t2 = tcg_temp_ebb_new_i32(); 3546 3547 memop = tcg_canonicalize_memop(memop, 0, 0); 3548 3549 tcg_gen_qemu_ld_i32(t1, addr, idx, memop); 3550 tcg_gen_ext_i32(t2, val, memop); 3551 gen(t2, t1, t2); 3552 tcg_gen_qemu_st_i32(t2, addr, idx, memop); 3553 3554 tcg_gen_ext_i32(ret, (new_val ? t2 : t1), memop); 3555 tcg_temp_free_i32(t1); 3556 tcg_temp_free_i32(t2); 3557 } 3558 3559 static void do_atomic_op_i32(TCGv_i32 ret, TCGv addr, TCGv_i32 val, 3560 TCGArg idx, MemOp memop, void * const table[]) 3561 { 3562 gen_atomic_op_i32 gen; 3563 MemOpIdx oi; 3564 3565 memop = tcg_canonicalize_memop(memop, 0, 0); 3566 3567 gen = table[memop & (MO_SIZE | MO_BSWAP)]; 3568 tcg_debug_assert(gen != NULL); 3569 3570 oi = make_memop_idx(memop & ~MO_SIGN, idx); 3571 gen(ret, cpu_env, addr, val, tcg_constant_i32(oi)); 3572 3573 if (memop & MO_SIGN) { 3574 tcg_gen_ext_i32(ret, ret, memop); 3575 } 3576 } 3577 3578 static void do_nonatomic_op_i64(TCGv_i64 ret, TCGv addr, TCGv_i64 val, 3579 TCGArg idx, MemOp memop, bool new_val, 3580 void (*gen)(TCGv_i64, TCGv_i64, TCGv_i64)) 3581 { 3582 TCGv_i64 t1 = tcg_temp_ebb_new_i64(); 3583 TCGv_i64 t2 = tcg_temp_ebb_new_i64(); 3584 3585 memop = tcg_canonicalize_memop(memop, 1, 0); 3586 3587 tcg_gen_qemu_ld_i64(t1, addr, idx, memop); 3588 tcg_gen_ext_i64(t2, val, memop); 3589 gen(t2, t1, t2); 3590 tcg_gen_qemu_st_i64(t2, addr, idx, memop); 3591 3592 tcg_gen_ext_i64(ret, (new_val ? t2 : t1), memop); 3593 tcg_temp_free_i64(t1); 3594 tcg_temp_free_i64(t2); 3595 } 3596 3597 static void do_atomic_op_i64(TCGv_i64 ret, TCGv addr, TCGv_i64 val, 3598 TCGArg idx, MemOp memop, void * const table[]) 3599 { 3600 memop = tcg_canonicalize_memop(memop, 1, 0); 3601 3602 if ((memop & MO_SIZE) == MO_64) { 3603 #ifdef CONFIG_ATOMIC64 3604 gen_atomic_op_i64 gen; 3605 MemOpIdx oi; 3606 3607 gen = table[memop & (MO_SIZE | MO_BSWAP)]; 3608 tcg_debug_assert(gen != NULL); 3609 3610 oi = make_memop_idx(memop & ~MO_SIGN, idx); 3611 gen(ret, cpu_env, addr, val, tcg_constant_i32(oi)); 3612 #else 3613 gen_helper_exit_atomic(cpu_env); 3614 /* Produce a result, so that we have a well-formed opcode stream 3615 with respect to uses of the result in the (dead) code following. */ 3616 tcg_gen_movi_i64(ret, 0); 3617 #endif /* CONFIG_ATOMIC64 */ 3618 } else { 3619 TCGv_i32 v32 = tcg_temp_ebb_new_i32(); 3620 TCGv_i32 r32 = tcg_temp_ebb_new_i32(); 3621 3622 tcg_gen_extrl_i64_i32(v32, val); 3623 do_atomic_op_i32(r32, addr, v32, idx, memop & ~MO_SIGN, table); 3624 tcg_temp_free_i32(v32); 3625 3626 tcg_gen_extu_i32_i64(ret, r32); 3627 tcg_temp_free_i32(r32); 3628 3629 if (memop & MO_SIGN) { 3630 tcg_gen_ext_i64(ret, ret, memop); 3631 } 3632 } 3633 } 3634 3635 #define GEN_ATOMIC_HELPER(NAME, OP, NEW) \ 3636 static void * const table_##NAME[(MO_SIZE | MO_BSWAP) + 1] = { \ 3637 [MO_8] = gen_helper_atomic_##NAME##b, \ 3638 [MO_16 | MO_LE] = gen_helper_atomic_##NAME##w_le, \ 3639 [MO_16 | MO_BE] = gen_helper_atomic_##NAME##w_be, \ 3640 [MO_32 | MO_LE] = gen_helper_atomic_##NAME##l_le, \ 3641 [MO_32 | MO_BE] = gen_helper_atomic_##NAME##l_be, \ 3642 WITH_ATOMIC64([MO_64 | MO_LE] = gen_helper_atomic_##NAME##q_le) \ 3643 WITH_ATOMIC64([MO_64 | MO_BE] = gen_helper_atomic_##NAME##q_be) \ 3644 }; \ 3645 void tcg_gen_atomic_##NAME##_i32 \ 3646 (TCGv_i32 ret, TCGv addr, TCGv_i32 val, TCGArg idx, MemOp memop) \ 3647 { \ 3648 if (tcg_ctx->gen_tb->cflags & CF_PARALLEL) { \ 3649 do_atomic_op_i32(ret, addr, val, idx, memop, table_##NAME); \ 3650 } else { \ 3651 do_nonatomic_op_i32(ret, addr, val, idx, memop, NEW, \ 3652 tcg_gen_##OP##_i32); \ 3653 } \ 3654 } \ 3655 void tcg_gen_atomic_##NAME##_i64 \ 3656 (TCGv_i64 ret, TCGv addr, TCGv_i64 val, TCGArg idx, MemOp memop) \ 3657 { \ 3658 if (tcg_ctx->gen_tb->cflags & CF_PARALLEL) { \ 3659 do_atomic_op_i64(ret, addr, val, idx, memop, table_##NAME); \ 3660 } else { \ 3661 do_nonatomic_op_i64(ret, addr, val, idx, memop, NEW, \ 3662 tcg_gen_##OP##_i64); \ 3663 } \ 3664 } 3665 3666 GEN_ATOMIC_HELPER(fetch_add, add, 0) 3667 GEN_ATOMIC_HELPER(fetch_and, and, 0) 3668 GEN_ATOMIC_HELPER(fetch_or, or, 0) 3669 GEN_ATOMIC_HELPER(fetch_xor, xor, 0) 3670 GEN_ATOMIC_HELPER(fetch_smin, smin, 0) 3671 GEN_ATOMIC_HELPER(fetch_umin, umin, 0) 3672 GEN_ATOMIC_HELPER(fetch_smax, smax, 0) 3673 GEN_ATOMIC_HELPER(fetch_umax, umax, 0) 3674 3675 GEN_ATOMIC_HELPER(add_fetch, add, 1) 3676 GEN_ATOMIC_HELPER(and_fetch, and, 1) 3677 GEN_ATOMIC_HELPER(or_fetch, or, 1) 3678 GEN_ATOMIC_HELPER(xor_fetch, xor, 1) 3679 GEN_ATOMIC_HELPER(smin_fetch, smin, 1) 3680 GEN_ATOMIC_HELPER(umin_fetch, umin, 1) 3681 GEN_ATOMIC_HELPER(smax_fetch, smax, 1) 3682 GEN_ATOMIC_HELPER(umax_fetch, umax, 1) 3683 3684 static void tcg_gen_mov2_i32(TCGv_i32 r, TCGv_i32 a, TCGv_i32 b) 3685 { 3686 tcg_gen_mov_i32(r, b); 3687 } 3688 3689 static void tcg_gen_mov2_i64(TCGv_i64 r, TCGv_i64 a, TCGv_i64 b) 3690 { 3691 tcg_gen_mov_i64(r, b); 3692 } 3693 3694 GEN_ATOMIC_HELPER(xchg, mov2, 0) 3695 3696 #undef GEN_ATOMIC_HELPER 3697