1 /* 2 * Tiny Code Generator for QEMU 3 * 4 * Copyright (c) 2008 Fabrice Bellard 5 * 6 * Permission is hereby granted, free of charge, to any person obtaining a copy 7 * of this software and associated documentation files (the "Software"), to deal 8 * in the Software without restriction, including without limitation the rights 9 * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 10 * copies of the Software, and to permit persons to whom the Software is 11 * furnished to do so, subject to the following conditions: 12 * 13 * The above copyright notice and this permission notice shall be included in 14 * all copies or substantial portions of the Software. 15 * 16 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 17 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 18 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL 19 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 20 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 21 * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN 22 * THE SOFTWARE. 23 */ 24 25 #include "qemu/osdep.h" 26 #include "exec/exec-all.h" 27 #include "tcg/tcg.h" 28 #include "tcg/tcg-op.h" 29 #include "tcg/tcg-mo.h" 30 #include "exec/plugin-gen.h" 31 #include "tcg-internal.h" 32 33 34 void tcg_gen_op1(TCGOpcode opc, TCGArg a1) 35 { 36 TCGOp *op = tcg_emit_op(opc, 1); 37 op->args[0] = a1; 38 } 39 40 void tcg_gen_op2(TCGOpcode opc, TCGArg a1, TCGArg a2) 41 { 42 TCGOp *op = tcg_emit_op(opc, 2); 43 op->args[0] = a1; 44 op->args[1] = a2; 45 } 46 47 void tcg_gen_op3(TCGOpcode opc, TCGArg a1, TCGArg a2, TCGArg a3) 48 { 49 TCGOp *op = tcg_emit_op(opc, 3); 50 op->args[0] = a1; 51 op->args[1] = a2; 52 op->args[2] = a3; 53 } 54 55 void tcg_gen_op4(TCGOpcode opc, TCGArg a1, TCGArg a2, TCGArg a3, TCGArg a4) 56 { 57 TCGOp *op = tcg_emit_op(opc, 4); 58 op->args[0] = a1; 59 op->args[1] = a2; 60 op->args[2] = a3; 61 op->args[3] = a4; 62 } 63 64 void tcg_gen_op5(TCGOpcode opc, TCGArg a1, TCGArg a2, TCGArg a3, 65 TCGArg a4, TCGArg a5) 66 { 67 TCGOp *op = tcg_emit_op(opc, 5); 68 op->args[0] = a1; 69 op->args[1] = a2; 70 op->args[2] = a3; 71 op->args[3] = a4; 72 op->args[4] = a5; 73 } 74 75 void tcg_gen_op6(TCGOpcode opc, TCGArg a1, TCGArg a2, TCGArg a3, 76 TCGArg a4, TCGArg a5, TCGArg a6) 77 { 78 TCGOp *op = tcg_emit_op(opc, 6); 79 op->args[0] = a1; 80 op->args[1] = a2; 81 op->args[2] = a3; 82 op->args[3] = a4; 83 op->args[4] = a5; 84 op->args[5] = a6; 85 } 86 87 void tcg_gen_mb(TCGBar mb_type) 88 { 89 if (tcg_ctx->gen_tb->cflags & CF_PARALLEL) { 90 tcg_gen_op1(INDEX_op_mb, mb_type); 91 } 92 } 93 94 /* 32 bit ops */ 95 96 void tcg_gen_movi_i32(TCGv_i32 ret, int32_t arg) 97 { 98 tcg_gen_mov_i32(ret, tcg_constant_i32(arg)); 99 } 100 101 void tcg_gen_addi_i32(TCGv_i32 ret, TCGv_i32 arg1, int32_t arg2) 102 { 103 /* some cases can be optimized here */ 104 if (arg2 == 0) { 105 tcg_gen_mov_i32(ret, arg1); 106 } else { 107 tcg_gen_add_i32(ret, arg1, tcg_constant_i32(arg2)); 108 } 109 } 110 111 void tcg_gen_subfi_i32(TCGv_i32 ret, int32_t arg1, TCGv_i32 arg2) 112 { 113 if (arg1 == 0 && TCG_TARGET_HAS_neg_i32) { 114 /* Don't recurse with tcg_gen_neg_i32. */ 115 tcg_gen_op2_i32(INDEX_op_neg_i32, ret, arg2); 116 } else { 117 tcg_gen_sub_i32(ret, tcg_constant_i32(arg1), arg2); 118 } 119 } 120 121 void tcg_gen_subi_i32(TCGv_i32 ret, TCGv_i32 arg1, int32_t arg2) 122 { 123 /* some cases can be optimized here */ 124 if (arg2 == 0) { 125 tcg_gen_mov_i32(ret, arg1); 126 } else { 127 tcg_gen_sub_i32(ret, arg1, tcg_constant_i32(arg2)); 128 } 129 } 130 131 void tcg_gen_andi_i32(TCGv_i32 ret, TCGv_i32 arg1, int32_t arg2) 132 { 133 /* Some cases can be optimized here. */ 134 switch (arg2) { 135 case 0: 136 tcg_gen_movi_i32(ret, 0); 137 return; 138 case -1: 139 tcg_gen_mov_i32(ret, arg1); 140 return; 141 case 0xff: 142 /* Don't recurse with tcg_gen_ext8u_i32. */ 143 if (TCG_TARGET_HAS_ext8u_i32) { 144 tcg_gen_op2_i32(INDEX_op_ext8u_i32, ret, arg1); 145 return; 146 } 147 break; 148 case 0xffff: 149 if (TCG_TARGET_HAS_ext16u_i32) { 150 tcg_gen_op2_i32(INDEX_op_ext16u_i32, ret, arg1); 151 return; 152 } 153 break; 154 } 155 156 tcg_gen_and_i32(ret, arg1, tcg_constant_i32(arg2)); 157 } 158 159 void tcg_gen_ori_i32(TCGv_i32 ret, TCGv_i32 arg1, int32_t arg2) 160 { 161 /* Some cases can be optimized here. */ 162 if (arg2 == -1) { 163 tcg_gen_movi_i32(ret, -1); 164 } else if (arg2 == 0) { 165 tcg_gen_mov_i32(ret, arg1); 166 } else { 167 tcg_gen_or_i32(ret, arg1, tcg_constant_i32(arg2)); 168 } 169 } 170 171 void tcg_gen_xori_i32(TCGv_i32 ret, TCGv_i32 arg1, int32_t arg2) 172 { 173 /* Some cases can be optimized here. */ 174 if (arg2 == 0) { 175 tcg_gen_mov_i32(ret, arg1); 176 } else if (arg2 == -1 && TCG_TARGET_HAS_not_i32) { 177 /* Don't recurse with tcg_gen_not_i32. */ 178 tcg_gen_op2_i32(INDEX_op_not_i32, ret, arg1); 179 } else { 180 tcg_gen_xor_i32(ret, arg1, tcg_constant_i32(arg2)); 181 } 182 } 183 184 void tcg_gen_shli_i32(TCGv_i32 ret, TCGv_i32 arg1, int32_t arg2) 185 { 186 tcg_debug_assert(arg2 >= 0 && arg2 < 32); 187 if (arg2 == 0) { 188 tcg_gen_mov_i32(ret, arg1); 189 } else { 190 tcg_gen_shl_i32(ret, arg1, tcg_constant_i32(arg2)); 191 } 192 } 193 194 void tcg_gen_shri_i32(TCGv_i32 ret, TCGv_i32 arg1, int32_t arg2) 195 { 196 tcg_debug_assert(arg2 >= 0 && arg2 < 32); 197 if (arg2 == 0) { 198 tcg_gen_mov_i32(ret, arg1); 199 } else { 200 tcg_gen_shr_i32(ret, arg1, tcg_constant_i32(arg2)); 201 } 202 } 203 204 void tcg_gen_sari_i32(TCGv_i32 ret, TCGv_i32 arg1, int32_t arg2) 205 { 206 tcg_debug_assert(arg2 >= 0 && arg2 < 32); 207 if (arg2 == 0) { 208 tcg_gen_mov_i32(ret, arg1); 209 } else { 210 tcg_gen_sar_i32(ret, arg1, tcg_constant_i32(arg2)); 211 } 212 } 213 214 void tcg_gen_brcond_i32(TCGCond cond, TCGv_i32 arg1, TCGv_i32 arg2, TCGLabel *l) 215 { 216 if (cond == TCG_COND_ALWAYS) { 217 tcg_gen_br(l); 218 } else if (cond != TCG_COND_NEVER) { 219 l->refs++; 220 tcg_gen_op4ii_i32(INDEX_op_brcond_i32, arg1, arg2, cond, label_arg(l)); 221 } 222 } 223 224 void tcg_gen_brcondi_i32(TCGCond cond, TCGv_i32 arg1, int32_t arg2, TCGLabel *l) 225 { 226 if (cond == TCG_COND_ALWAYS) { 227 tcg_gen_br(l); 228 } else if (cond != TCG_COND_NEVER) { 229 tcg_gen_brcond_i32(cond, arg1, tcg_constant_i32(arg2), l); 230 } 231 } 232 233 void tcg_gen_setcond_i32(TCGCond cond, TCGv_i32 ret, 234 TCGv_i32 arg1, TCGv_i32 arg2) 235 { 236 if (cond == TCG_COND_ALWAYS) { 237 tcg_gen_movi_i32(ret, 1); 238 } else if (cond == TCG_COND_NEVER) { 239 tcg_gen_movi_i32(ret, 0); 240 } else { 241 tcg_gen_op4i_i32(INDEX_op_setcond_i32, ret, arg1, arg2, cond); 242 } 243 } 244 245 void tcg_gen_setcondi_i32(TCGCond cond, TCGv_i32 ret, 246 TCGv_i32 arg1, int32_t arg2) 247 { 248 tcg_gen_setcond_i32(cond, ret, arg1, tcg_constant_i32(arg2)); 249 } 250 251 void tcg_gen_muli_i32(TCGv_i32 ret, TCGv_i32 arg1, int32_t arg2) 252 { 253 if (arg2 == 0) { 254 tcg_gen_movi_i32(ret, 0); 255 } else if (is_power_of_2(arg2)) { 256 tcg_gen_shli_i32(ret, arg1, ctz32(arg2)); 257 } else { 258 tcg_gen_mul_i32(ret, arg1, tcg_constant_i32(arg2)); 259 } 260 } 261 262 void tcg_gen_div_i32(TCGv_i32 ret, TCGv_i32 arg1, TCGv_i32 arg2) 263 { 264 if (TCG_TARGET_HAS_div_i32) { 265 tcg_gen_op3_i32(INDEX_op_div_i32, ret, arg1, arg2); 266 } else if (TCG_TARGET_HAS_div2_i32) { 267 TCGv_i32 t0 = tcg_temp_new_i32(); 268 tcg_gen_sari_i32(t0, arg1, 31); 269 tcg_gen_op5_i32(INDEX_op_div2_i32, ret, t0, arg1, t0, arg2); 270 tcg_temp_free_i32(t0); 271 } else { 272 gen_helper_div_i32(ret, arg1, arg2); 273 } 274 } 275 276 void tcg_gen_rem_i32(TCGv_i32 ret, TCGv_i32 arg1, TCGv_i32 arg2) 277 { 278 if (TCG_TARGET_HAS_rem_i32) { 279 tcg_gen_op3_i32(INDEX_op_rem_i32, ret, arg1, arg2); 280 } else if (TCG_TARGET_HAS_div_i32) { 281 TCGv_i32 t0 = tcg_temp_new_i32(); 282 tcg_gen_op3_i32(INDEX_op_div_i32, t0, arg1, arg2); 283 tcg_gen_mul_i32(t0, t0, arg2); 284 tcg_gen_sub_i32(ret, arg1, t0); 285 tcg_temp_free_i32(t0); 286 } else if (TCG_TARGET_HAS_div2_i32) { 287 TCGv_i32 t0 = tcg_temp_new_i32(); 288 tcg_gen_sari_i32(t0, arg1, 31); 289 tcg_gen_op5_i32(INDEX_op_div2_i32, t0, ret, arg1, t0, arg2); 290 tcg_temp_free_i32(t0); 291 } else { 292 gen_helper_rem_i32(ret, arg1, arg2); 293 } 294 } 295 296 void tcg_gen_divu_i32(TCGv_i32 ret, TCGv_i32 arg1, TCGv_i32 arg2) 297 { 298 if (TCG_TARGET_HAS_div_i32) { 299 tcg_gen_op3_i32(INDEX_op_divu_i32, ret, arg1, arg2); 300 } else if (TCG_TARGET_HAS_div2_i32) { 301 TCGv_i32 t0 = tcg_temp_new_i32(); 302 tcg_gen_movi_i32(t0, 0); 303 tcg_gen_op5_i32(INDEX_op_divu2_i32, ret, t0, arg1, t0, arg2); 304 tcg_temp_free_i32(t0); 305 } else { 306 gen_helper_divu_i32(ret, arg1, arg2); 307 } 308 } 309 310 void tcg_gen_remu_i32(TCGv_i32 ret, TCGv_i32 arg1, TCGv_i32 arg2) 311 { 312 if (TCG_TARGET_HAS_rem_i32) { 313 tcg_gen_op3_i32(INDEX_op_remu_i32, ret, arg1, arg2); 314 } else if (TCG_TARGET_HAS_div_i32) { 315 TCGv_i32 t0 = tcg_temp_new_i32(); 316 tcg_gen_op3_i32(INDEX_op_divu_i32, t0, arg1, arg2); 317 tcg_gen_mul_i32(t0, t0, arg2); 318 tcg_gen_sub_i32(ret, arg1, t0); 319 tcg_temp_free_i32(t0); 320 } else if (TCG_TARGET_HAS_div2_i32) { 321 TCGv_i32 t0 = tcg_temp_new_i32(); 322 tcg_gen_movi_i32(t0, 0); 323 tcg_gen_op5_i32(INDEX_op_divu2_i32, t0, ret, arg1, t0, arg2); 324 tcg_temp_free_i32(t0); 325 } else { 326 gen_helper_remu_i32(ret, arg1, arg2); 327 } 328 } 329 330 void tcg_gen_andc_i32(TCGv_i32 ret, TCGv_i32 arg1, TCGv_i32 arg2) 331 { 332 if (TCG_TARGET_HAS_andc_i32) { 333 tcg_gen_op3_i32(INDEX_op_andc_i32, ret, arg1, arg2); 334 } else { 335 TCGv_i32 t0 = tcg_temp_new_i32(); 336 tcg_gen_not_i32(t0, arg2); 337 tcg_gen_and_i32(ret, arg1, t0); 338 tcg_temp_free_i32(t0); 339 } 340 } 341 342 void tcg_gen_eqv_i32(TCGv_i32 ret, TCGv_i32 arg1, TCGv_i32 arg2) 343 { 344 if (TCG_TARGET_HAS_eqv_i32) { 345 tcg_gen_op3_i32(INDEX_op_eqv_i32, ret, arg1, arg2); 346 } else { 347 tcg_gen_xor_i32(ret, arg1, arg2); 348 tcg_gen_not_i32(ret, ret); 349 } 350 } 351 352 void tcg_gen_nand_i32(TCGv_i32 ret, TCGv_i32 arg1, TCGv_i32 arg2) 353 { 354 if (TCG_TARGET_HAS_nand_i32) { 355 tcg_gen_op3_i32(INDEX_op_nand_i32, ret, arg1, arg2); 356 } else { 357 tcg_gen_and_i32(ret, arg1, arg2); 358 tcg_gen_not_i32(ret, ret); 359 } 360 } 361 362 void tcg_gen_nor_i32(TCGv_i32 ret, TCGv_i32 arg1, TCGv_i32 arg2) 363 { 364 if (TCG_TARGET_HAS_nor_i32) { 365 tcg_gen_op3_i32(INDEX_op_nor_i32, ret, arg1, arg2); 366 } else { 367 tcg_gen_or_i32(ret, arg1, arg2); 368 tcg_gen_not_i32(ret, ret); 369 } 370 } 371 372 void tcg_gen_orc_i32(TCGv_i32 ret, TCGv_i32 arg1, TCGv_i32 arg2) 373 { 374 if (TCG_TARGET_HAS_orc_i32) { 375 tcg_gen_op3_i32(INDEX_op_orc_i32, ret, arg1, arg2); 376 } else { 377 TCGv_i32 t0 = tcg_temp_new_i32(); 378 tcg_gen_not_i32(t0, arg2); 379 tcg_gen_or_i32(ret, arg1, t0); 380 tcg_temp_free_i32(t0); 381 } 382 } 383 384 void tcg_gen_clz_i32(TCGv_i32 ret, TCGv_i32 arg1, TCGv_i32 arg2) 385 { 386 if (TCG_TARGET_HAS_clz_i32) { 387 tcg_gen_op3_i32(INDEX_op_clz_i32, ret, arg1, arg2); 388 } else if (TCG_TARGET_HAS_clz_i64) { 389 TCGv_i64 t1 = tcg_temp_new_i64(); 390 TCGv_i64 t2 = tcg_temp_new_i64(); 391 tcg_gen_extu_i32_i64(t1, arg1); 392 tcg_gen_extu_i32_i64(t2, arg2); 393 tcg_gen_addi_i64(t2, t2, 32); 394 tcg_gen_clz_i64(t1, t1, t2); 395 tcg_gen_extrl_i64_i32(ret, t1); 396 tcg_temp_free_i64(t1); 397 tcg_temp_free_i64(t2); 398 tcg_gen_subi_i32(ret, ret, 32); 399 } else { 400 gen_helper_clz_i32(ret, arg1, arg2); 401 } 402 } 403 404 void tcg_gen_clzi_i32(TCGv_i32 ret, TCGv_i32 arg1, uint32_t arg2) 405 { 406 tcg_gen_clz_i32(ret, arg1, tcg_constant_i32(arg2)); 407 } 408 409 void tcg_gen_ctz_i32(TCGv_i32 ret, TCGv_i32 arg1, TCGv_i32 arg2) 410 { 411 if (TCG_TARGET_HAS_ctz_i32) { 412 tcg_gen_op3_i32(INDEX_op_ctz_i32, ret, arg1, arg2); 413 } else if (TCG_TARGET_HAS_ctz_i64) { 414 TCGv_i64 t1 = tcg_temp_new_i64(); 415 TCGv_i64 t2 = tcg_temp_new_i64(); 416 tcg_gen_extu_i32_i64(t1, arg1); 417 tcg_gen_extu_i32_i64(t2, arg2); 418 tcg_gen_ctz_i64(t1, t1, t2); 419 tcg_gen_extrl_i64_i32(ret, t1); 420 tcg_temp_free_i64(t1); 421 tcg_temp_free_i64(t2); 422 } else if (TCG_TARGET_HAS_ctpop_i32 423 || TCG_TARGET_HAS_ctpop_i64 424 || TCG_TARGET_HAS_clz_i32 425 || TCG_TARGET_HAS_clz_i64) { 426 TCGv_i32 z, t = tcg_temp_new_i32(); 427 428 if (TCG_TARGET_HAS_ctpop_i32 || TCG_TARGET_HAS_ctpop_i64) { 429 tcg_gen_subi_i32(t, arg1, 1); 430 tcg_gen_andc_i32(t, t, arg1); 431 tcg_gen_ctpop_i32(t, t); 432 } else { 433 /* Since all non-x86 hosts have clz(0) == 32, don't fight it. */ 434 tcg_gen_neg_i32(t, arg1); 435 tcg_gen_and_i32(t, t, arg1); 436 tcg_gen_clzi_i32(t, t, 32); 437 tcg_gen_xori_i32(t, t, 31); 438 } 439 z = tcg_constant_i32(0); 440 tcg_gen_movcond_i32(TCG_COND_EQ, ret, arg1, z, arg2, t); 441 tcg_temp_free_i32(t); 442 } else { 443 gen_helper_ctz_i32(ret, arg1, arg2); 444 } 445 } 446 447 void tcg_gen_ctzi_i32(TCGv_i32 ret, TCGv_i32 arg1, uint32_t arg2) 448 { 449 if (!TCG_TARGET_HAS_ctz_i32 && TCG_TARGET_HAS_ctpop_i32 && arg2 == 32) { 450 /* This equivalence has the advantage of not requiring a fixup. */ 451 TCGv_i32 t = tcg_temp_new_i32(); 452 tcg_gen_subi_i32(t, arg1, 1); 453 tcg_gen_andc_i32(t, t, arg1); 454 tcg_gen_ctpop_i32(ret, t); 455 tcg_temp_free_i32(t); 456 } else { 457 tcg_gen_ctz_i32(ret, arg1, tcg_constant_i32(arg2)); 458 } 459 } 460 461 void tcg_gen_clrsb_i32(TCGv_i32 ret, TCGv_i32 arg) 462 { 463 if (TCG_TARGET_HAS_clz_i32) { 464 TCGv_i32 t = tcg_temp_new_i32(); 465 tcg_gen_sari_i32(t, arg, 31); 466 tcg_gen_xor_i32(t, t, arg); 467 tcg_gen_clzi_i32(t, t, 32); 468 tcg_gen_subi_i32(ret, t, 1); 469 tcg_temp_free_i32(t); 470 } else { 471 gen_helper_clrsb_i32(ret, arg); 472 } 473 } 474 475 void tcg_gen_ctpop_i32(TCGv_i32 ret, TCGv_i32 arg1) 476 { 477 if (TCG_TARGET_HAS_ctpop_i32) { 478 tcg_gen_op2_i32(INDEX_op_ctpop_i32, ret, arg1); 479 } else if (TCG_TARGET_HAS_ctpop_i64) { 480 TCGv_i64 t = tcg_temp_new_i64(); 481 tcg_gen_extu_i32_i64(t, arg1); 482 tcg_gen_ctpop_i64(t, t); 483 tcg_gen_extrl_i64_i32(ret, t); 484 tcg_temp_free_i64(t); 485 } else { 486 gen_helper_ctpop_i32(ret, arg1); 487 } 488 } 489 490 void tcg_gen_rotl_i32(TCGv_i32 ret, TCGv_i32 arg1, TCGv_i32 arg2) 491 { 492 if (TCG_TARGET_HAS_rot_i32) { 493 tcg_gen_op3_i32(INDEX_op_rotl_i32, ret, arg1, arg2); 494 } else { 495 TCGv_i32 t0, t1; 496 497 t0 = tcg_temp_new_i32(); 498 t1 = tcg_temp_new_i32(); 499 tcg_gen_shl_i32(t0, arg1, arg2); 500 tcg_gen_subfi_i32(t1, 32, arg2); 501 tcg_gen_shr_i32(t1, arg1, t1); 502 tcg_gen_or_i32(ret, t0, t1); 503 tcg_temp_free_i32(t0); 504 tcg_temp_free_i32(t1); 505 } 506 } 507 508 void tcg_gen_rotli_i32(TCGv_i32 ret, TCGv_i32 arg1, int32_t arg2) 509 { 510 tcg_debug_assert(arg2 >= 0 && arg2 < 32); 511 /* some cases can be optimized here */ 512 if (arg2 == 0) { 513 tcg_gen_mov_i32(ret, arg1); 514 } else if (TCG_TARGET_HAS_rot_i32) { 515 tcg_gen_rotl_i32(ret, arg1, tcg_constant_i32(arg2)); 516 } else { 517 TCGv_i32 t0, t1; 518 t0 = tcg_temp_new_i32(); 519 t1 = tcg_temp_new_i32(); 520 tcg_gen_shli_i32(t0, arg1, arg2); 521 tcg_gen_shri_i32(t1, arg1, 32 - arg2); 522 tcg_gen_or_i32(ret, t0, t1); 523 tcg_temp_free_i32(t0); 524 tcg_temp_free_i32(t1); 525 } 526 } 527 528 void tcg_gen_rotr_i32(TCGv_i32 ret, TCGv_i32 arg1, TCGv_i32 arg2) 529 { 530 if (TCG_TARGET_HAS_rot_i32) { 531 tcg_gen_op3_i32(INDEX_op_rotr_i32, ret, arg1, arg2); 532 } else { 533 TCGv_i32 t0, t1; 534 535 t0 = tcg_temp_new_i32(); 536 t1 = tcg_temp_new_i32(); 537 tcg_gen_shr_i32(t0, arg1, arg2); 538 tcg_gen_subfi_i32(t1, 32, arg2); 539 tcg_gen_shl_i32(t1, arg1, t1); 540 tcg_gen_or_i32(ret, t0, t1); 541 tcg_temp_free_i32(t0); 542 tcg_temp_free_i32(t1); 543 } 544 } 545 546 void tcg_gen_rotri_i32(TCGv_i32 ret, TCGv_i32 arg1, int32_t arg2) 547 { 548 tcg_debug_assert(arg2 >= 0 && arg2 < 32); 549 /* some cases can be optimized here */ 550 if (arg2 == 0) { 551 tcg_gen_mov_i32(ret, arg1); 552 } else { 553 tcg_gen_rotli_i32(ret, arg1, 32 - arg2); 554 } 555 } 556 557 void tcg_gen_deposit_i32(TCGv_i32 ret, TCGv_i32 arg1, TCGv_i32 arg2, 558 unsigned int ofs, unsigned int len) 559 { 560 uint32_t mask; 561 TCGv_i32 t1; 562 563 tcg_debug_assert(ofs < 32); 564 tcg_debug_assert(len > 0); 565 tcg_debug_assert(len <= 32); 566 tcg_debug_assert(ofs + len <= 32); 567 568 if (len == 32) { 569 tcg_gen_mov_i32(ret, arg2); 570 return; 571 } 572 if (TCG_TARGET_HAS_deposit_i32 && TCG_TARGET_deposit_i32_valid(ofs, len)) { 573 tcg_gen_op5ii_i32(INDEX_op_deposit_i32, ret, arg1, arg2, ofs, len); 574 return; 575 } 576 577 t1 = tcg_temp_new_i32(); 578 579 if (TCG_TARGET_HAS_extract2_i32) { 580 if (ofs + len == 32) { 581 tcg_gen_shli_i32(t1, arg1, len); 582 tcg_gen_extract2_i32(ret, t1, arg2, len); 583 goto done; 584 } 585 if (ofs == 0) { 586 tcg_gen_extract2_i32(ret, arg1, arg2, len); 587 tcg_gen_rotli_i32(ret, ret, len); 588 goto done; 589 } 590 } 591 592 mask = (1u << len) - 1; 593 if (ofs + len < 32) { 594 tcg_gen_andi_i32(t1, arg2, mask); 595 tcg_gen_shli_i32(t1, t1, ofs); 596 } else { 597 tcg_gen_shli_i32(t1, arg2, ofs); 598 } 599 tcg_gen_andi_i32(ret, arg1, ~(mask << ofs)); 600 tcg_gen_or_i32(ret, ret, t1); 601 done: 602 tcg_temp_free_i32(t1); 603 } 604 605 void tcg_gen_deposit_z_i32(TCGv_i32 ret, TCGv_i32 arg, 606 unsigned int ofs, unsigned int len) 607 { 608 tcg_debug_assert(ofs < 32); 609 tcg_debug_assert(len > 0); 610 tcg_debug_assert(len <= 32); 611 tcg_debug_assert(ofs + len <= 32); 612 613 if (ofs + len == 32) { 614 tcg_gen_shli_i32(ret, arg, ofs); 615 } else if (ofs == 0) { 616 tcg_gen_andi_i32(ret, arg, (1u << len) - 1); 617 } else if (TCG_TARGET_HAS_deposit_i32 618 && TCG_TARGET_deposit_i32_valid(ofs, len)) { 619 TCGv_i32 zero = tcg_constant_i32(0); 620 tcg_gen_op5ii_i32(INDEX_op_deposit_i32, ret, zero, arg, ofs, len); 621 } else { 622 /* To help two-operand hosts we prefer to zero-extend first, 623 which allows ARG to stay live. */ 624 switch (len) { 625 case 16: 626 if (TCG_TARGET_HAS_ext16u_i32) { 627 tcg_gen_ext16u_i32(ret, arg); 628 tcg_gen_shli_i32(ret, ret, ofs); 629 return; 630 } 631 break; 632 case 8: 633 if (TCG_TARGET_HAS_ext8u_i32) { 634 tcg_gen_ext8u_i32(ret, arg); 635 tcg_gen_shli_i32(ret, ret, ofs); 636 return; 637 } 638 break; 639 } 640 /* Otherwise prefer zero-extension over AND for code size. */ 641 switch (ofs + len) { 642 case 16: 643 if (TCG_TARGET_HAS_ext16u_i32) { 644 tcg_gen_shli_i32(ret, arg, ofs); 645 tcg_gen_ext16u_i32(ret, ret); 646 return; 647 } 648 break; 649 case 8: 650 if (TCG_TARGET_HAS_ext8u_i32) { 651 tcg_gen_shli_i32(ret, arg, ofs); 652 tcg_gen_ext8u_i32(ret, ret); 653 return; 654 } 655 break; 656 } 657 tcg_gen_andi_i32(ret, arg, (1u << len) - 1); 658 tcg_gen_shli_i32(ret, ret, ofs); 659 } 660 } 661 662 void tcg_gen_extract_i32(TCGv_i32 ret, TCGv_i32 arg, 663 unsigned int ofs, unsigned int len) 664 { 665 tcg_debug_assert(ofs < 32); 666 tcg_debug_assert(len > 0); 667 tcg_debug_assert(len <= 32); 668 tcg_debug_assert(ofs + len <= 32); 669 670 /* Canonicalize certain special cases, even if extract is supported. */ 671 if (ofs + len == 32) { 672 tcg_gen_shri_i32(ret, arg, 32 - len); 673 return; 674 } 675 if (ofs == 0) { 676 tcg_gen_andi_i32(ret, arg, (1u << len) - 1); 677 return; 678 } 679 680 if (TCG_TARGET_HAS_extract_i32 681 && TCG_TARGET_extract_i32_valid(ofs, len)) { 682 tcg_gen_op4ii_i32(INDEX_op_extract_i32, ret, arg, ofs, len); 683 return; 684 } 685 686 /* Assume that zero-extension, if available, is cheaper than a shift. */ 687 switch (ofs + len) { 688 case 16: 689 if (TCG_TARGET_HAS_ext16u_i32) { 690 tcg_gen_ext16u_i32(ret, arg); 691 tcg_gen_shri_i32(ret, ret, ofs); 692 return; 693 } 694 break; 695 case 8: 696 if (TCG_TARGET_HAS_ext8u_i32) { 697 tcg_gen_ext8u_i32(ret, arg); 698 tcg_gen_shri_i32(ret, ret, ofs); 699 return; 700 } 701 break; 702 } 703 704 /* ??? Ideally we'd know what values are available for immediate AND. 705 Assume that 8 bits are available, plus the special case of 16, 706 so that we get ext8u, ext16u. */ 707 switch (len) { 708 case 1 ... 8: case 16: 709 tcg_gen_shri_i32(ret, arg, ofs); 710 tcg_gen_andi_i32(ret, ret, (1u << len) - 1); 711 break; 712 default: 713 tcg_gen_shli_i32(ret, arg, 32 - len - ofs); 714 tcg_gen_shri_i32(ret, ret, 32 - len); 715 break; 716 } 717 } 718 719 void tcg_gen_sextract_i32(TCGv_i32 ret, TCGv_i32 arg, 720 unsigned int ofs, unsigned int len) 721 { 722 tcg_debug_assert(ofs < 32); 723 tcg_debug_assert(len > 0); 724 tcg_debug_assert(len <= 32); 725 tcg_debug_assert(ofs + len <= 32); 726 727 /* Canonicalize certain special cases, even if extract is supported. */ 728 if (ofs + len == 32) { 729 tcg_gen_sari_i32(ret, arg, 32 - len); 730 return; 731 } 732 if (ofs == 0) { 733 switch (len) { 734 case 16: 735 tcg_gen_ext16s_i32(ret, arg); 736 return; 737 case 8: 738 tcg_gen_ext8s_i32(ret, arg); 739 return; 740 } 741 } 742 743 if (TCG_TARGET_HAS_sextract_i32 744 && TCG_TARGET_extract_i32_valid(ofs, len)) { 745 tcg_gen_op4ii_i32(INDEX_op_sextract_i32, ret, arg, ofs, len); 746 return; 747 } 748 749 /* Assume that sign-extension, if available, is cheaper than a shift. */ 750 switch (ofs + len) { 751 case 16: 752 if (TCG_TARGET_HAS_ext16s_i32) { 753 tcg_gen_ext16s_i32(ret, arg); 754 tcg_gen_sari_i32(ret, ret, ofs); 755 return; 756 } 757 break; 758 case 8: 759 if (TCG_TARGET_HAS_ext8s_i32) { 760 tcg_gen_ext8s_i32(ret, arg); 761 tcg_gen_sari_i32(ret, ret, ofs); 762 return; 763 } 764 break; 765 } 766 switch (len) { 767 case 16: 768 if (TCG_TARGET_HAS_ext16s_i32) { 769 tcg_gen_shri_i32(ret, arg, ofs); 770 tcg_gen_ext16s_i32(ret, ret); 771 return; 772 } 773 break; 774 case 8: 775 if (TCG_TARGET_HAS_ext8s_i32) { 776 tcg_gen_shri_i32(ret, arg, ofs); 777 tcg_gen_ext8s_i32(ret, ret); 778 return; 779 } 780 break; 781 } 782 783 tcg_gen_shli_i32(ret, arg, 32 - len - ofs); 784 tcg_gen_sari_i32(ret, ret, 32 - len); 785 } 786 787 /* 788 * Extract 32-bits from a 64-bit input, ah:al, starting from ofs. 789 * Unlike tcg_gen_extract_i32 above, len is fixed at 32. 790 */ 791 void tcg_gen_extract2_i32(TCGv_i32 ret, TCGv_i32 al, TCGv_i32 ah, 792 unsigned int ofs) 793 { 794 tcg_debug_assert(ofs <= 32); 795 if (ofs == 0) { 796 tcg_gen_mov_i32(ret, al); 797 } else if (ofs == 32) { 798 tcg_gen_mov_i32(ret, ah); 799 } else if (al == ah) { 800 tcg_gen_rotri_i32(ret, al, ofs); 801 } else if (TCG_TARGET_HAS_extract2_i32) { 802 tcg_gen_op4i_i32(INDEX_op_extract2_i32, ret, al, ah, ofs); 803 } else { 804 TCGv_i32 t0 = tcg_temp_new_i32(); 805 tcg_gen_shri_i32(t0, al, ofs); 806 tcg_gen_deposit_i32(ret, t0, ah, 32 - ofs, ofs); 807 tcg_temp_free_i32(t0); 808 } 809 } 810 811 void tcg_gen_movcond_i32(TCGCond cond, TCGv_i32 ret, TCGv_i32 c1, 812 TCGv_i32 c2, TCGv_i32 v1, TCGv_i32 v2) 813 { 814 if (cond == TCG_COND_ALWAYS) { 815 tcg_gen_mov_i32(ret, v1); 816 } else if (cond == TCG_COND_NEVER) { 817 tcg_gen_mov_i32(ret, v2); 818 } else if (TCG_TARGET_HAS_movcond_i32) { 819 tcg_gen_op6i_i32(INDEX_op_movcond_i32, ret, c1, c2, v1, v2, cond); 820 } else { 821 TCGv_i32 t0 = tcg_temp_new_i32(); 822 TCGv_i32 t1 = tcg_temp_new_i32(); 823 tcg_gen_setcond_i32(cond, t0, c1, c2); 824 tcg_gen_neg_i32(t0, t0); 825 tcg_gen_and_i32(t1, v1, t0); 826 tcg_gen_andc_i32(ret, v2, t0); 827 tcg_gen_or_i32(ret, ret, t1); 828 tcg_temp_free_i32(t0); 829 tcg_temp_free_i32(t1); 830 } 831 } 832 833 void tcg_gen_add2_i32(TCGv_i32 rl, TCGv_i32 rh, TCGv_i32 al, 834 TCGv_i32 ah, TCGv_i32 bl, TCGv_i32 bh) 835 { 836 if (TCG_TARGET_HAS_add2_i32) { 837 tcg_gen_op6_i32(INDEX_op_add2_i32, rl, rh, al, ah, bl, bh); 838 } else { 839 TCGv_i64 t0 = tcg_temp_new_i64(); 840 TCGv_i64 t1 = tcg_temp_new_i64(); 841 tcg_gen_concat_i32_i64(t0, al, ah); 842 tcg_gen_concat_i32_i64(t1, bl, bh); 843 tcg_gen_add_i64(t0, t0, t1); 844 tcg_gen_extr_i64_i32(rl, rh, t0); 845 tcg_temp_free_i64(t0); 846 tcg_temp_free_i64(t1); 847 } 848 } 849 850 void tcg_gen_sub2_i32(TCGv_i32 rl, TCGv_i32 rh, TCGv_i32 al, 851 TCGv_i32 ah, TCGv_i32 bl, TCGv_i32 bh) 852 { 853 if (TCG_TARGET_HAS_sub2_i32) { 854 tcg_gen_op6_i32(INDEX_op_sub2_i32, rl, rh, al, ah, bl, bh); 855 } else { 856 TCGv_i64 t0 = tcg_temp_new_i64(); 857 TCGv_i64 t1 = tcg_temp_new_i64(); 858 tcg_gen_concat_i32_i64(t0, al, ah); 859 tcg_gen_concat_i32_i64(t1, bl, bh); 860 tcg_gen_sub_i64(t0, t0, t1); 861 tcg_gen_extr_i64_i32(rl, rh, t0); 862 tcg_temp_free_i64(t0); 863 tcg_temp_free_i64(t1); 864 } 865 } 866 867 void tcg_gen_mulu2_i32(TCGv_i32 rl, TCGv_i32 rh, TCGv_i32 arg1, TCGv_i32 arg2) 868 { 869 if (TCG_TARGET_HAS_mulu2_i32) { 870 tcg_gen_op4_i32(INDEX_op_mulu2_i32, rl, rh, arg1, arg2); 871 } else if (TCG_TARGET_HAS_muluh_i32) { 872 TCGv_i32 t = tcg_temp_new_i32(); 873 tcg_gen_op3_i32(INDEX_op_mul_i32, t, arg1, arg2); 874 tcg_gen_op3_i32(INDEX_op_muluh_i32, rh, arg1, arg2); 875 tcg_gen_mov_i32(rl, t); 876 tcg_temp_free_i32(t); 877 } else if (TCG_TARGET_REG_BITS == 64) { 878 TCGv_i64 t0 = tcg_temp_new_i64(); 879 TCGv_i64 t1 = tcg_temp_new_i64(); 880 tcg_gen_extu_i32_i64(t0, arg1); 881 tcg_gen_extu_i32_i64(t1, arg2); 882 tcg_gen_mul_i64(t0, t0, t1); 883 tcg_gen_extr_i64_i32(rl, rh, t0); 884 tcg_temp_free_i64(t0); 885 tcg_temp_free_i64(t1); 886 } else { 887 qemu_build_not_reached(); 888 } 889 } 890 891 void tcg_gen_muls2_i32(TCGv_i32 rl, TCGv_i32 rh, TCGv_i32 arg1, TCGv_i32 arg2) 892 { 893 if (TCG_TARGET_HAS_muls2_i32) { 894 tcg_gen_op4_i32(INDEX_op_muls2_i32, rl, rh, arg1, arg2); 895 } else if (TCG_TARGET_HAS_mulsh_i32) { 896 TCGv_i32 t = tcg_temp_new_i32(); 897 tcg_gen_op3_i32(INDEX_op_mul_i32, t, arg1, arg2); 898 tcg_gen_op3_i32(INDEX_op_mulsh_i32, rh, arg1, arg2); 899 tcg_gen_mov_i32(rl, t); 900 tcg_temp_free_i32(t); 901 } else if (TCG_TARGET_REG_BITS == 32) { 902 TCGv_i32 t0 = tcg_temp_new_i32(); 903 TCGv_i32 t1 = tcg_temp_new_i32(); 904 TCGv_i32 t2 = tcg_temp_new_i32(); 905 TCGv_i32 t3 = tcg_temp_new_i32(); 906 tcg_gen_mulu2_i32(t0, t1, arg1, arg2); 907 /* Adjust for negative inputs. */ 908 tcg_gen_sari_i32(t2, arg1, 31); 909 tcg_gen_sari_i32(t3, arg2, 31); 910 tcg_gen_and_i32(t2, t2, arg2); 911 tcg_gen_and_i32(t3, t3, arg1); 912 tcg_gen_sub_i32(rh, t1, t2); 913 tcg_gen_sub_i32(rh, rh, t3); 914 tcg_gen_mov_i32(rl, t0); 915 tcg_temp_free_i32(t0); 916 tcg_temp_free_i32(t1); 917 tcg_temp_free_i32(t2); 918 tcg_temp_free_i32(t3); 919 } else { 920 TCGv_i64 t0 = tcg_temp_new_i64(); 921 TCGv_i64 t1 = tcg_temp_new_i64(); 922 tcg_gen_ext_i32_i64(t0, arg1); 923 tcg_gen_ext_i32_i64(t1, arg2); 924 tcg_gen_mul_i64(t0, t0, t1); 925 tcg_gen_extr_i64_i32(rl, rh, t0); 926 tcg_temp_free_i64(t0); 927 tcg_temp_free_i64(t1); 928 } 929 } 930 931 void tcg_gen_mulsu2_i32(TCGv_i32 rl, TCGv_i32 rh, TCGv_i32 arg1, TCGv_i32 arg2) 932 { 933 if (TCG_TARGET_REG_BITS == 32) { 934 TCGv_i32 t0 = tcg_temp_new_i32(); 935 TCGv_i32 t1 = tcg_temp_new_i32(); 936 TCGv_i32 t2 = tcg_temp_new_i32(); 937 tcg_gen_mulu2_i32(t0, t1, arg1, arg2); 938 /* Adjust for negative input for the signed arg1. */ 939 tcg_gen_sari_i32(t2, arg1, 31); 940 tcg_gen_and_i32(t2, t2, arg2); 941 tcg_gen_sub_i32(rh, t1, t2); 942 tcg_gen_mov_i32(rl, t0); 943 tcg_temp_free_i32(t0); 944 tcg_temp_free_i32(t1); 945 tcg_temp_free_i32(t2); 946 } else { 947 TCGv_i64 t0 = tcg_temp_new_i64(); 948 TCGv_i64 t1 = tcg_temp_new_i64(); 949 tcg_gen_ext_i32_i64(t0, arg1); 950 tcg_gen_extu_i32_i64(t1, arg2); 951 tcg_gen_mul_i64(t0, t0, t1); 952 tcg_gen_extr_i64_i32(rl, rh, t0); 953 tcg_temp_free_i64(t0); 954 tcg_temp_free_i64(t1); 955 } 956 } 957 958 void tcg_gen_ext8s_i32(TCGv_i32 ret, TCGv_i32 arg) 959 { 960 if (TCG_TARGET_HAS_ext8s_i32) { 961 tcg_gen_op2_i32(INDEX_op_ext8s_i32, ret, arg); 962 } else { 963 tcg_gen_shli_i32(ret, arg, 24); 964 tcg_gen_sari_i32(ret, ret, 24); 965 } 966 } 967 968 void tcg_gen_ext16s_i32(TCGv_i32 ret, TCGv_i32 arg) 969 { 970 if (TCG_TARGET_HAS_ext16s_i32) { 971 tcg_gen_op2_i32(INDEX_op_ext16s_i32, ret, arg); 972 } else { 973 tcg_gen_shli_i32(ret, arg, 16); 974 tcg_gen_sari_i32(ret, ret, 16); 975 } 976 } 977 978 void tcg_gen_ext8u_i32(TCGv_i32 ret, TCGv_i32 arg) 979 { 980 if (TCG_TARGET_HAS_ext8u_i32) { 981 tcg_gen_op2_i32(INDEX_op_ext8u_i32, ret, arg); 982 } else { 983 tcg_gen_andi_i32(ret, arg, 0xffu); 984 } 985 } 986 987 void tcg_gen_ext16u_i32(TCGv_i32 ret, TCGv_i32 arg) 988 { 989 if (TCG_TARGET_HAS_ext16u_i32) { 990 tcg_gen_op2_i32(INDEX_op_ext16u_i32, ret, arg); 991 } else { 992 tcg_gen_andi_i32(ret, arg, 0xffffu); 993 } 994 } 995 996 void tcg_gen_bswap16_i32(TCGv_i32 ret, TCGv_i32 arg, int flags) 997 { 998 /* Only one extension flag may be present. */ 999 tcg_debug_assert(!(flags & TCG_BSWAP_OS) || !(flags & TCG_BSWAP_OZ)); 1000 1001 if (TCG_TARGET_HAS_bswap16_i32) { 1002 tcg_gen_op3i_i32(INDEX_op_bswap16_i32, ret, arg, flags); 1003 } else { 1004 TCGv_i32 t0 = tcg_temp_new_i32(); 1005 TCGv_i32 t1 = tcg_temp_new_i32(); 1006 1007 tcg_gen_shri_i32(t0, arg, 8); 1008 if (!(flags & TCG_BSWAP_IZ)) { 1009 tcg_gen_ext8u_i32(t0, t0); 1010 } 1011 1012 if (flags & TCG_BSWAP_OS) { 1013 tcg_gen_shli_i32(t1, arg, 24); 1014 tcg_gen_sari_i32(t1, t1, 16); 1015 } else if (flags & TCG_BSWAP_OZ) { 1016 tcg_gen_ext8u_i32(t1, arg); 1017 tcg_gen_shli_i32(t1, t1, 8); 1018 } else { 1019 tcg_gen_shli_i32(t1, arg, 8); 1020 } 1021 1022 tcg_gen_or_i32(ret, t0, t1); 1023 tcg_temp_free_i32(t0); 1024 tcg_temp_free_i32(t1); 1025 } 1026 } 1027 1028 void tcg_gen_bswap32_i32(TCGv_i32 ret, TCGv_i32 arg) 1029 { 1030 if (TCG_TARGET_HAS_bswap32_i32) { 1031 tcg_gen_op3i_i32(INDEX_op_bswap32_i32, ret, arg, 0); 1032 } else { 1033 TCGv_i32 t0 = tcg_temp_new_i32(); 1034 TCGv_i32 t1 = tcg_temp_new_i32(); 1035 TCGv_i32 t2 = tcg_constant_i32(0x00ff00ff); 1036 1037 /* arg = abcd */ 1038 tcg_gen_shri_i32(t0, arg, 8); /* t0 = .abc */ 1039 tcg_gen_and_i32(t1, arg, t2); /* t1 = .b.d */ 1040 tcg_gen_and_i32(t0, t0, t2); /* t0 = .a.c */ 1041 tcg_gen_shli_i32(t1, t1, 8); /* t1 = b.d. */ 1042 tcg_gen_or_i32(ret, t0, t1); /* ret = badc */ 1043 1044 tcg_gen_shri_i32(t0, ret, 16); /* t0 = ..ba */ 1045 tcg_gen_shli_i32(t1, ret, 16); /* t1 = dc.. */ 1046 tcg_gen_or_i32(ret, t0, t1); /* ret = dcba */ 1047 1048 tcg_temp_free_i32(t0); 1049 tcg_temp_free_i32(t1); 1050 } 1051 } 1052 1053 void tcg_gen_hswap_i32(TCGv_i32 ret, TCGv_i32 arg) 1054 { 1055 /* Swapping 2 16-bit elements is a rotate. */ 1056 tcg_gen_rotli_i32(ret, arg, 16); 1057 } 1058 1059 void tcg_gen_smin_i32(TCGv_i32 ret, TCGv_i32 a, TCGv_i32 b) 1060 { 1061 tcg_gen_movcond_i32(TCG_COND_LT, ret, a, b, a, b); 1062 } 1063 1064 void tcg_gen_umin_i32(TCGv_i32 ret, TCGv_i32 a, TCGv_i32 b) 1065 { 1066 tcg_gen_movcond_i32(TCG_COND_LTU, ret, a, b, a, b); 1067 } 1068 1069 void tcg_gen_smax_i32(TCGv_i32 ret, TCGv_i32 a, TCGv_i32 b) 1070 { 1071 tcg_gen_movcond_i32(TCG_COND_LT, ret, a, b, b, a); 1072 } 1073 1074 void tcg_gen_umax_i32(TCGv_i32 ret, TCGv_i32 a, TCGv_i32 b) 1075 { 1076 tcg_gen_movcond_i32(TCG_COND_LTU, ret, a, b, b, a); 1077 } 1078 1079 void tcg_gen_abs_i32(TCGv_i32 ret, TCGv_i32 a) 1080 { 1081 TCGv_i32 t = tcg_temp_new_i32(); 1082 1083 tcg_gen_sari_i32(t, a, 31); 1084 tcg_gen_xor_i32(ret, a, t); 1085 tcg_gen_sub_i32(ret, ret, t); 1086 tcg_temp_free_i32(t); 1087 } 1088 1089 /* 64-bit ops */ 1090 1091 #if TCG_TARGET_REG_BITS == 32 1092 /* These are all inline for TCG_TARGET_REG_BITS == 64. */ 1093 1094 void tcg_gen_discard_i64(TCGv_i64 arg) 1095 { 1096 tcg_gen_discard_i32(TCGV_LOW(arg)); 1097 tcg_gen_discard_i32(TCGV_HIGH(arg)); 1098 } 1099 1100 void tcg_gen_mov_i64(TCGv_i64 ret, TCGv_i64 arg) 1101 { 1102 TCGTemp *ts = tcgv_i64_temp(arg); 1103 1104 /* Canonicalize TCGv_i64 TEMP_CONST into TCGv_i32 TEMP_CONST. */ 1105 if (ts->kind == TEMP_CONST) { 1106 tcg_gen_movi_i64(ret, ts->val); 1107 } else { 1108 tcg_gen_mov_i32(TCGV_LOW(ret), TCGV_LOW(arg)); 1109 tcg_gen_mov_i32(TCGV_HIGH(ret), TCGV_HIGH(arg)); 1110 } 1111 } 1112 1113 void tcg_gen_movi_i64(TCGv_i64 ret, int64_t arg) 1114 { 1115 tcg_gen_movi_i32(TCGV_LOW(ret), arg); 1116 tcg_gen_movi_i32(TCGV_HIGH(ret), arg >> 32); 1117 } 1118 1119 void tcg_gen_ld8u_i64(TCGv_i64 ret, TCGv_ptr arg2, tcg_target_long offset) 1120 { 1121 tcg_gen_ld8u_i32(TCGV_LOW(ret), arg2, offset); 1122 tcg_gen_movi_i32(TCGV_HIGH(ret), 0); 1123 } 1124 1125 void tcg_gen_ld8s_i64(TCGv_i64 ret, TCGv_ptr arg2, tcg_target_long offset) 1126 { 1127 tcg_gen_ld8s_i32(TCGV_LOW(ret), arg2, offset); 1128 tcg_gen_sari_i32(TCGV_HIGH(ret), TCGV_LOW(ret), 31); 1129 } 1130 1131 void tcg_gen_ld16u_i64(TCGv_i64 ret, TCGv_ptr arg2, tcg_target_long offset) 1132 { 1133 tcg_gen_ld16u_i32(TCGV_LOW(ret), arg2, offset); 1134 tcg_gen_movi_i32(TCGV_HIGH(ret), 0); 1135 } 1136 1137 void tcg_gen_ld16s_i64(TCGv_i64 ret, TCGv_ptr arg2, tcg_target_long offset) 1138 { 1139 tcg_gen_ld16s_i32(TCGV_LOW(ret), arg2, offset); 1140 tcg_gen_sari_i32(TCGV_HIGH(ret), TCGV_LOW(ret), 31); 1141 } 1142 1143 void tcg_gen_ld32u_i64(TCGv_i64 ret, TCGv_ptr arg2, tcg_target_long offset) 1144 { 1145 tcg_gen_ld_i32(TCGV_LOW(ret), arg2, offset); 1146 tcg_gen_movi_i32(TCGV_HIGH(ret), 0); 1147 } 1148 1149 void tcg_gen_ld32s_i64(TCGv_i64 ret, TCGv_ptr arg2, tcg_target_long offset) 1150 { 1151 tcg_gen_ld_i32(TCGV_LOW(ret), arg2, offset); 1152 tcg_gen_sari_i32(TCGV_HIGH(ret), TCGV_LOW(ret), 31); 1153 } 1154 1155 void tcg_gen_ld_i64(TCGv_i64 ret, TCGv_ptr arg2, tcg_target_long offset) 1156 { 1157 /* Since arg2 and ret have different types, 1158 they cannot be the same temporary */ 1159 #if HOST_BIG_ENDIAN 1160 tcg_gen_ld_i32(TCGV_HIGH(ret), arg2, offset); 1161 tcg_gen_ld_i32(TCGV_LOW(ret), arg2, offset + 4); 1162 #else 1163 tcg_gen_ld_i32(TCGV_LOW(ret), arg2, offset); 1164 tcg_gen_ld_i32(TCGV_HIGH(ret), arg2, offset + 4); 1165 #endif 1166 } 1167 1168 void tcg_gen_st8_i64(TCGv_i64 arg1, TCGv_ptr arg2, tcg_target_long offset) 1169 { 1170 tcg_gen_st8_i32(TCGV_LOW(arg1), arg2, offset); 1171 } 1172 1173 void tcg_gen_st16_i64(TCGv_i64 arg1, TCGv_ptr arg2, tcg_target_long offset) 1174 { 1175 tcg_gen_st16_i32(TCGV_LOW(arg1), arg2, offset); 1176 } 1177 1178 void tcg_gen_st32_i64(TCGv_i64 arg1, TCGv_ptr arg2, tcg_target_long offset) 1179 { 1180 tcg_gen_st_i32(TCGV_LOW(arg1), arg2, offset); 1181 } 1182 1183 void tcg_gen_st_i64(TCGv_i64 arg1, TCGv_ptr arg2, tcg_target_long offset) 1184 { 1185 #if HOST_BIG_ENDIAN 1186 tcg_gen_st_i32(TCGV_HIGH(arg1), arg2, offset); 1187 tcg_gen_st_i32(TCGV_LOW(arg1), arg2, offset + 4); 1188 #else 1189 tcg_gen_st_i32(TCGV_LOW(arg1), arg2, offset); 1190 tcg_gen_st_i32(TCGV_HIGH(arg1), arg2, offset + 4); 1191 #endif 1192 } 1193 1194 void tcg_gen_add_i64(TCGv_i64 ret, TCGv_i64 arg1, TCGv_i64 arg2) 1195 { 1196 tcg_gen_add2_i32(TCGV_LOW(ret), TCGV_HIGH(ret), TCGV_LOW(arg1), 1197 TCGV_HIGH(arg1), TCGV_LOW(arg2), TCGV_HIGH(arg2)); 1198 } 1199 1200 void tcg_gen_sub_i64(TCGv_i64 ret, TCGv_i64 arg1, TCGv_i64 arg2) 1201 { 1202 tcg_gen_sub2_i32(TCGV_LOW(ret), TCGV_HIGH(ret), TCGV_LOW(arg1), 1203 TCGV_HIGH(arg1), TCGV_LOW(arg2), TCGV_HIGH(arg2)); 1204 } 1205 1206 void tcg_gen_and_i64(TCGv_i64 ret, TCGv_i64 arg1, TCGv_i64 arg2) 1207 { 1208 tcg_gen_and_i32(TCGV_LOW(ret), TCGV_LOW(arg1), TCGV_LOW(arg2)); 1209 tcg_gen_and_i32(TCGV_HIGH(ret), TCGV_HIGH(arg1), TCGV_HIGH(arg2)); 1210 } 1211 1212 void tcg_gen_or_i64(TCGv_i64 ret, TCGv_i64 arg1, TCGv_i64 arg2) 1213 { 1214 tcg_gen_or_i32(TCGV_LOW(ret), TCGV_LOW(arg1), TCGV_LOW(arg2)); 1215 tcg_gen_or_i32(TCGV_HIGH(ret), TCGV_HIGH(arg1), TCGV_HIGH(arg2)); 1216 } 1217 1218 void tcg_gen_xor_i64(TCGv_i64 ret, TCGv_i64 arg1, TCGv_i64 arg2) 1219 { 1220 tcg_gen_xor_i32(TCGV_LOW(ret), TCGV_LOW(arg1), TCGV_LOW(arg2)); 1221 tcg_gen_xor_i32(TCGV_HIGH(ret), TCGV_HIGH(arg1), TCGV_HIGH(arg2)); 1222 } 1223 1224 void tcg_gen_shl_i64(TCGv_i64 ret, TCGv_i64 arg1, TCGv_i64 arg2) 1225 { 1226 gen_helper_shl_i64(ret, arg1, arg2); 1227 } 1228 1229 void tcg_gen_shr_i64(TCGv_i64 ret, TCGv_i64 arg1, TCGv_i64 arg2) 1230 { 1231 gen_helper_shr_i64(ret, arg1, arg2); 1232 } 1233 1234 void tcg_gen_sar_i64(TCGv_i64 ret, TCGv_i64 arg1, TCGv_i64 arg2) 1235 { 1236 gen_helper_sar_i64(ret, arg1, arg2); 1237 } 1238 1239 void tcg_gen_mul_i64(TCGv_i64 ret, TCGv_i64 arg1, TCGv_i64 arg2) 1240 { 1241 TCGv_i64 t0; 1242 TCGv_i32 t1; 1243 1244 t0 = tcg_temp_new_i64(); 1245 t1 = tcg_temp_new_i32(); 1246 1247 tcg_gen_mulu2_i32(TCGV_LOW(t0), TCGV_HIGH(t0), 1248 TCGV_LOW(arg1), TCGV_LOW(arg2)); 1249 1250 tcg_gen_mul_i32(t1, TCGV_LOW(arg1), TCGV_HIGH(arg2)); 1251 tcg_gen_add_i32(TCGV_HIGH(t0), TCGV_HIGH(t0), t1); 1252 tcg_gen_mul_i32(t1, TCGV_HIGH(arg1), TCGV_LOW(arg2)); 1253 tcg_gen_add_i32(TCGV_HIGH(t0), TCGV_HIGH(t0), t1); 1254 1255 tcg_gen_mov_i64(ret, t0); 1256 tcg_temp_free_i64(t0); 1257 tcg_temp_free_i32(t1); 1258 } 1259 1260 #else 1261 1262 void tcg_gen_movi_i64(TCGv_i64 ret, int64_t arg) 1263 { 1264 tcg_gen_mov_i64(ret, tcg_constant_i64(arg)); 1265 } 1266 1267 #endif /* TCG_TARGET_REG_SIZE == 32 */ 1268 1269 void tcg_gen_addi_i64(TCGv_i64 ret, TCGv_i64 arg1, int64_t arg2) 1270 { 1271 /* some cases can be optimized here */ 1272 if (arg2 == 0) { 1273 tcg_gen_mov_i64(ret, arg1); 1274 } else if (TCG_TARGET_REG_BITS == 64) { 1275 tcg_gen_add_i64(ret, arg1, tcg_constant_i64(arg2)); 1276 } else { 1277 tcg_gen_add2_i32(TCGV_LOW(ret), TCGV_HIGH(ret), 1278 TCGV_LOW(arg1), TCGV_HIGH(arg1), 1279 tcg_constant_i32(arg2), tcg_constant_i32(arg2 >> 32)); 1280 } 1281 } 1282 1283 void tcg_gen_subfi_i64(TCGv_i64 ret, int64_t arg1, TCGv_i64 arg2) 1284 { 1285 if (arg1 == 0 && TCG_TARGET_HAS_neg_i64) { 1286 /* Don't recurse with tcg_gen_neg_i64. */ 1287 tcg_gen_op2_i64(INDEX_op_neg_i64, ret, arg2); 1288 } else if (TCG_TARGET_REG_BITS == 64) { 1289 tcg_gen_sub_i64(ret, tcg_constant_i64(arg1), arg2); 1290 } else { 1291 tcg_gen_sub2_i32(TCGV_LOW(ret), TCGV_HIGH(ret), 1292 tcg_constant_i32(arg1), tcg_constant_i32(arg1 >> 32), 1293 TCGV_LOW(arg2), TCGV_HIGH(arg2)); 1294 } 1295 } 1296 1297 void tcg_gen_subi_i64(TCGv_i64 ret, TCGv_i64 arg1, int64_t arg2) 1298 { 1299 /* some cases can be optimized here */ 1300 if (arg2 == 0) { 1301 tcg_gen_mov_i64(ret, arg1); 1302 } else if (TCG_TARGET_REG_BITS == 64) { 1303 tcg_gen_sub_i64(ret, arg1, tcg_constant_i64(arg2)); 1304 } else { 1305 tcg_gen_sub2_i32(TCGV_LOW(ret), TCGV_HIGH(ret), 1306 TCGV_LOW(arg1), TCGV_HIGH(arg1), 1307 tcg_constant_i32(arg2), tcg_constant_i32(arg2 >> 32)); 1308 } 1309 } 1310 1311 void tcg_gen_andi_i64(TCGv_i64 ret, TCGv_i64 arg1, int64_t arg2) 1312 { 1313 if (TCG_TARGET_REG_BITS == 32) { 1314 tcg_gen_andi_i32(TCGV_LOW(ret), TCGV_LOW(arg1), arg2); 1315 tcg_gen_andi_i32(TCGV_HIGH(ret), TCGV_HIGH(arg1), arg2 >> 32); 1316 return; 1317 } 1318 1319 /* Some cases can be optimized here. */ 1320 switch (arg2) { 1321 case 0: 1322 tcg_gen_movi_i64(ret, 0); 1323 return; 1324 case -1: 1325 tcg_gen_mov_i64(ret, arg1); 1326 return; 1327 case 0xff: 1328 /* Don't recurse with tcg_gen_ext8u_i64. */ 1329 if (TCG_TARGET_HAS_ext8u_i64) { 1330 tcg_gen_op2_i64(INDEX_op_ext8u_i64, ret, arg1); 1331 return; 1332 } 1333 break; 1334 case 0xffff: 1335 if (TCG_TARGET_HAS_ext16u_i64) { 1336 tcg_gen_op2_i64(INDEX_op_ext16u_i64, ret, arg1); 1337 return; 1338 } 1339 break; 1340 case 0xffffffffu: 1341 if (TCG_TARGET_HAS_ext32u_i64) { 1342 tcg_gen_op2_i64(INDEX_op_ext32u_i64, ret, arg1); 1343 return; 1344 } 1345 break; 1346 } 1347 1348 tcg_gen_and_i64(ret, arg1, tcg_constant_i64(arg2)); 1349 } 1350 1351 void tcg_gen_ori_i64(TCGv_i64 ret, TCGv_i64 arg1, int64_t arg2) 1352 { 1353 if (TCG_TARGET_REG_BITS == 32) { 1354 tcg_gen_ori_i32(TCGV_LOW(ret), TCGV_LOW(arg1), arg2); 1355 tcg_gen_ori_i32(TCGV_HIGH(ret), TCGV_HIGH(arg1), arg2 >> 32); 1356 return; 1357 } 1358 /* Some cases can be optimized here. */ 1359 if (arg2 == -1) { 1360 tcg_gen_movi_i64(ret, -1); 1361 } else if (arg2 == 0) { 1362 tcg_gen_mov_i64(ret, arg1); 1363 } else { 1364 tcg_gen_or_i64(ret, arg1, tcg_constant_i64(arg2)); 1365 } 1366 } 1367 1368 void tcg_gen_xori_i64(TCGv_i64 ret, TCGv_i64 arg1, int64_t arg2) 1369 { 1370 if (TCG_TARGET_REG_BITS == 32) { 1371 tcg_gen_xori_i32(TCGV_LOW(ret), TCGV_LOW(arg1), arg2); 1372 tcg_gen_xori_i32(TCGV_HIGH(ret), TCGV_HIGH(arg1), arg2 >> 32); 1373 return; 1374 } 1375 /* Some cases can be optimized here. */ 1376 if (arg2 == 0) { 1377 tcg_gen_mov_i64(ret, arg1); 1378 } else if (arg2 == -1 && TCG_TARGET_HAS_not_i64) { 1379 /* Don't recurse with tcg_gen_not_i64. */ 1380 tcg_gen_op2_i64(INDEX_op_not_i64, ret, arg1); 1381 } else { 1382 tcg_gen_xor_i64(ret, arg1, tcg_constant_i64(arg2)); 1383 } 1384 } 1385 1386 static inline void tcg_gen_shifti_i64(TCGv_i64 ret, TCGv_i64 arg1, 1387 unsigned c, bool right, bool arith) 1388 { 1389 tcg_debug_assert(c < 64); 1390 if (c == 0) { 1391 tcg_gen_mov_i32(TCGV_LOW(ret), TCGV_LOW(arg1)); 1392 tcg_gen_mov_i32(TCGV_HIGH(ret), TCGV_HIGH(arg1)); 1393 } else if (c >= 32) { 1394 c -= 32; 1395 if (right) { 1396 if (arith) { 1397 tcg_gen_sari_i32(TCGV_LOW(ret), TCGV_HIGH(arg1), c); 1398 tcg_gen_sari_i32(TCGV_HIGH(ret), TCGV_HIGH(arg1), 31); 1399 } else { 1400 tcg_gen_shri_i32(TCGV_LOW(ret), TCGV_HIGH(arg1), c); 1401 tcg_gen_movi_i32(TCGV_HIGH(ret), 0); 1402 } 1403 } else { 1404 tcg_gen_shli_i32(TCGV_HIGH(ret), TCGV_LOW(arg1), c); 1405 tcg_gen_movi_i32(TCGV_LOW(ret), 0); 1406 } 1407 } else if (right) { 1408 if (TCG_TARGET_HAS_extract2_i32) { 1409 tcg_gen_extract2_i32(TCGV_LOW(ret), 1410 TCGV_LOW(arg1), TCGV_HIGH(arg1), c); 1411 } else { 1412 tcg_gen_shri_i32(TCGV_LOW(ret), TCGV_LOW(arg1), c); 1413 tcg_gen_deposit_i32(TCGV_LOW(ret), TCGV_LOW(ret), 1414 TCGV_HIGH(arg1), 32 - c, c); 1415 } 1416 if (arith) { 1417 tcg_gen_sari_i32(TCGV_HIGH(ret), TCGV_HIGH(arg1), c); 1418 } else { 1419 tcg_gen_shri_i32(TCGV_HIGH(ret), TCGV_HIGH(arg1), c); 1420 } 1421 } else { 1422 if (TCG_TARGET_HAS_extract2_i32) { 1423 tcg_gen_extract2_i32(TCGV_HIGH(ret), 1424 TCGV_LOW(arg1), TCGV_HIGH(arg1), 32 - c); 1425 } else { 1426 TCGv_i32 t0 = tcg_temp_new_i32(); 1427 tcg_gen_shri_i32(t0, TCGV_LOW(arg1), 32 - c); 1428 tcg_gen_deposit_i32(TCGV_HIGH(ret), t0, 1429 TCGV_HIGH(arg1), c, 32 - c); 1430 tcg_temp_free_i32(t0); 1431 } 1432 tcg_gen_shli_i32(TCGV_LOW(ret), TCGV_LOW(arg1), c); 1433 } 1434 } 1435 1436 void tcg_gen_shli_i64(TCGv_i64 ret, TCGv_i64 arg1, int64_t arg2) 1437 { 1438 tcg_debug_assert(arg2 >= 0 && arg2 < 64); 1439 if (TCG_TARGET_REG_BITS == 32) { 1440 tcg_gen_shifti_i64(ret, arg1, arg2, 0, 0); 1441 } else if (arg2 == 0) { 1442 tcg_gen_mov_i64(ret, arg1); 1443 } else { 1444 tcg_gen_shl_i64(ret, arg1, tcg_constant_i64(arg2)); 1445 } 1446 } 1447 1448 void tcg_gen_shri_i64(TCGv_i64 ret, TCGv_i64 arg1, int64_t arg2) 1449 { 1450 tcg_debug_assert(arg2 >= 0 && arg2 < 64); 1451 if (TCG_TARGET_REG_BITS == 32) { 1452 tcg_gen_shifti_i64(ret, arg1, arg2, 1, 0); 1453 } else if (arg2 == 0) { 1454 tcg_gen_mov_i64(ret, arg1); 1455 } else { 1456 tcg_gen_shr_i64(ret, arg1, tcg_constant_i64(arg2)); 1457 } 1458 } 1459 1460 void tcg_gen_sari_i64(TCGv_i64 ret, TCGv_i64 arg1, int64_t arg2) 1461 { 1462 tcg_debug_assert(arg2 >= 0 && arg2 < 64); 1463 if (TCG_TARGET_REG_BITS == 32) { 1464 tcg_gen_shifti_i64(ret, arg1, arg2, 1, 1); 1465 } else if (arg2 == 0) { 1466 tcg_gen_mov_i64(ret, arg1); 1467 } else { 1468 tcg_gen_sar_i64(ret, arg1, tcg_constant_i64(arg2)); 1469 } 1470 } 1471 1472 void tcg_gen_brcond_i64(TCGCond cond, TCGv_i64 arg1, TCGv_i64 arg2, TCGLabel *l) 1473 { 1474 if (cond == TCG_COND_ALWAYS) { 1475 tcg_gen_br(l); 1476 } else if (cond != TCG_COND_NEVER) { 1477 l->refs++; 1478 if (TCG_TARGET_REG_BITS == 32) { 1479 tcg_gen_op6ii_i32(INDEX_op_brcond2_i32, TCGV_LOW(arg1), 1480 TCGV_HIGH(arg1), TCGV_LOW(arg2), 1481 TCGV_HIGH(arg2), cond, label_arg(l)); 1482 } else { 1483 tcg_gen_op4ii_i64(INDEX_op_brcond_i64, arg1, arg2, cond, 1484 label_arg(l)); 1485 } 1486 } 1487 } 1488 1489 void tcg_gen_brcondi_i64(TCGCond cond, TCGv_i64 arg1, int64_t arg2, TCGLabel *l) 1490 { 1491 if (TCG_TARGET_REG_BITS == 64) { 1492 tcg_gen_brcond_i64(cond, arg1, tcg_constant_i64(arg2), l); 1493 } else if (cond == TCG_COND_ALWAYS) { 1494 tcg_gen_br(l); 1495 } else if (cond != TCG_COND_NEVER) { 1496 l->refs++; 1497 tcg_gen_op6ii_i32(INDEX_op_brcond2_i32, 1498 TCGV_LOW(arg1), TCGV_HIGH(arg1), 1499 tcg_constant_i32(arg2), 1500 tcg_constant_i32(arg2 >> 32), 1501 cond, label_arg(l)); 1502 } 1503 } 1504 1505 void tcg_gen_setcond_i64(TCGCond cond, TCGv_i64 ret, 1506 TCGv_i64 arg1, TCGv_i64 arg2) 1507 { 1508 if (cond == TCG_COND_ALWAYS) { 1509 tcg_gen_movi_i64(ret, 1); 1510 } else if (cond == TCG_COND_NEVER) { 1511 tcg_gen_movi_i64(ret, 0); 1512 } else { 1513 if (TCG_TARGET_REG_BITS == 32) { 1514 tcg_gen_op6i_i32(INDEX_op_setcond2_i32, TCGV_LOW(ret), 1515 TCGV_LOW(arg1), TCGV_HIGH(arg1), 1516 TCGV_LOW(arg2), TCGV_HIGH(arg2), cond); 1517 tcg_gen_movi_i32(TCGV_HIGH(ret), 0); 1518 } else { 1519 tcg_gen_op4i_i64(INDEX_op_setcond_i64, ret, arg1, arg2, cond); 1520 } 1521 } 1522 } 1523 1524 void tcg_gen_setcondi_i64(TCGCond cond, TCGv_i64 ret, 1525 TCGv_i64 arg1, int64_t arg2) 1526 { 1527 if (TCG_TARGET_REG_BITS == 64) { 1528 tcg_gen_setcond_i64(cond, ret, arg1, tcg_constant_i64(arg2)); 1529 } else if (cond == TCG_COND_ALWAYS) { 1530 tcg_gen_movi_i64(ret, 1); 1531 } else if (cond == TCG_COND_NEVER) { 1532 tcg_gen_movi_i64(ret, 0); 1533 } else { 1534 tcg_gen_op6i_i32(INDEX_op_setcond2_i32, TCGV_LOW(ret), 1535 TCGV_LOW(arg1), TCGV_HIGH(arg1), 1536 tcg_constant_i32(arg2), 1537 tcg_constant_i32(arg2 >> 32), cond); 1538 tcg_gen_movi_i32(TCGV_HIGH(ret), 0); 1539 } 1540 } 1541 1542 void tcg_gen_muli_i64(TCGv_i64 ret, TCGv_i64 arg1, int64_t arg2) 1543 { 1544 if (arg2 == 0) { 1545 tcg_gen_movi_i64(ret, 0); 1546 } else if (is_power_of_2(arg2)) { 1547 tcg_gen_shli_i64(ret, arg1, ctz64(arg2)); 1548 } else { 1549 TCGv_i64 t0 = tcg_const_i64(arg2); 1550 tcg_gen_mul_i64(ret, arg1, t0); 1551 tcg_temp_free_i64(t0); 1552 } 1553 } 1554 1555 void tcg_gen_div_i64(TCGv_i64 ret, TCGv_i64 arg1, TCGv_i64 arg2) 1556 { 1557 if (TCG_TARGET_HAS_div_i64) { 1558 tcg_gen_op3_i64(INDEX_op_div_i64, ret, arg1, arg2); 1559 } else if (TCG_TARGET_HAS_div2_i64) { 1560 TCGv_i64 t0 = tcg_temp_new_i64(); 1561 tcg_gen_sari_i64(t0, arg1, 63); 1562 tcg_gen_op5_i64(INDEX_op_div2_i64, ret, t0, arg1, t0, arg2); 1563 tcg_temp_free_i64(t0); 1564 } else { 1565 gen_helper_div_i64(ret, arg1, arg2); 1566 } 1567 } 1568 1569 void tcg_gen_rem_i64(TCGv_i64 ret, TCGv_i64 arg1, TCGv_i64 arg2) 1570 { 1571 if (TCG_TARGET_HAS_rem_i64) { 1572 tcg_gen_op3_i64(INDEX_op_rem_i64, ret, arg1, arg2); 1573 } else if (TCG_TARGET_HAS_div_i64) { 1574 TCGv_i64 t0 = tcg_temp_new_i64(); 1575 tcg_gen_op3_i64(INDEX_op_div_i64, t0, arg1, arg2); 1576 tcg_gen_mul_i64(t0, t0, arg2); 1577 tcg_gen_sub_i64(ret, arg1, t0); 1578 tcg_temp_free_i64(t0); 1579 } else if (TCG_TARGET_HAS_div2_i64) { 1580 TCGv_i64 t0 = tcg_temp_new_i64(); 1581 tcg_gen_sari_i64(t0, arg1, 63); 1582 tcg_gen_op5_i64(INDEX_op_div2_i64, t0, ret, arg1, t0, arg2); 1583 tcg_temp_free_i64(t0); 1584 } else { 1585 gen_helper_rem_i64(ret, arg1, arg2); 1586 } 1587 } 1588 1589 void tcg_gen_divu_i64(TCGv_i64 ret, TCGv_i64 arg1, TCGv_i64 arg2) 1590 { 1591 if (TCG_TARGET_HAS_div_i64) { 1592 tcg_gen_op3_i64(INDEX_op_divu_i64, ret, arg1, arg2); 1593 } else if (TCG_TARGET_HAS_div2_i64) { 1594 TCGv_i64 t0 = tcg_temp_new_i64(); 1595 tcg_gen_movi_i64(t0, 0); 1596 tcg_gen_op5_i64(INDEX_op_divu2_i64, ret, t0, arg1, t0, arg2); 1597 tcg_temp_free_i64(t0); 1598 } else { 1599 gen_helper_divu_i64(ret, arg1, arg2); 1600 } 1601 } 1602 1603 void tcg_gen_remu_i64(TCGv_i64 ret, TCGv_i64 arg1, TCGv_i64 arg2) 1604 { 1605 if (TCG_TARGET_HAS_rem_i64) { 1606 tcg_gen_op3_i64(INDEX_op_remu_i64, ret, arg1, arg2); 1607 } else if (TCG_TARGET_HAS_div_i64) { 1608 TCGv_i64 t0 = tcg_temp_new_i64(); 1609 tcg_gen_op3_i64(INDEX_op_divu_i64, t0, arg1, arg2); 1610 tcg_gen_mul_i64(t0, t0, arg2); 1611 tcg_gen_sub_i64(ret, arg1, t0); 1612 tcg_temp_free_i64(t0); 1613 } else if (TCG_TARGET_HAS_div2_i64) { 1614 TCGv_i64 t0 = tcg_temp_new_i64(); 1615 tcg_gen_movi_i64(t0, 0); 1616 tcg_gen_op5_i64(INDEX_op_divu2_i64, t0, ret, arg1, t0, arg2); 1617 tcg_temp_free_i64(t0); 1618 } else { 1619 gen_helper_remu_i64(ret, arg1, arg2); 1620 } 1621 } 1622 1623 void tcg_gen_ext8s_i64(TCGv_i64 ret, TCGv_i64 arg) 1624 { 1625 if (TCG_TARGET_REG_BITS == 32) { 1626 tcg_gen_ext8s_i32(TCGV_LOW(ret), TCGV_LOW(arg)); 1627 tcg_gen_sari_i32(TCGV_HIGH(ret), TCGV_LOW(ret), 31); 1628 } else if (TCG_TARGET_HAS_ext8s_i64) { 1629 tcg_gen_op2_i64(INDEX_op_ext8s_i64, ret, arg); 1630 } else { 1631 tcg_gen_shli_i64(ret, arg, 56); 1632 tcg_gen_sari_i64(ret, ret, 56); 1633 } 1634 } 1635 1636 void tcg_gen_ext16s_i64(TCGv_i64 ret, TCGv_i64 arg) 1637 { 1638 if (TCG_TARGET_REG_BITS == 32) { 1639 tcg_gen_ext16s_i32(TCGV_LOW(ret), TCGV_LOW(arg)); 1640 tcg_gen_sari_i32(TCGV_HIGH(ret), TCGV_LOW(ret), 31); 1641 } else if (TCG_TARGET_HAS_ext16s_i64) { 1642 tcg_gen_op2_i64(INDEX_op_ext16s_i64, ret, arg); 1643 } else { 1644 tcg_gen_shli_i64(ret, arg, 48); 1645 tcg_gen_sari_i64(ret, ret, 48); 1646 } 1647 } 1648 1649 void tcg_gen_ext32s_i64(TCGv_i64 ret, TCGv_i64 arg) 1650 { 1651 if (TCG_TARGET_REG_BITS == 32) { 1652 tcg_gen_mov_i32(TCGV_LOW(ret), TCGV_LOW(arg)); 1653 tcg_gen_sari_i32(TCGV_HIGH(ret), TCGV_LOW(ret), 31); 1654 } else if (TCG_TARGET_HAS_ext32s_i64) { 1655 tcg_gen_op2_i64(INDEX_op_ext32s_i64, ret, arg); 1656 } else { 1657 tcg_gen_shli_i64(ret, arg, 32); 1658 tcg_gen_sari_i64(ret, ret, 32); 1659 } 1660 } 1661 1662 void tcg_gen_ext8u_i64(TCGv_i64 ret, TCGv_i64 arg) 1663 { 1664 if (TCG_TARGET_REG_BITS == 32) { 1665 tcg_gen_ext8u_i32(TCGV_LOW(ret), TCGV_LOW(arg)); 1666 tcg_gen_movi_i32(TCGV_HIGH(ret), 0); 1667 } else if (TCG_TARGET_HAS_ext8u_i64) { 1668 tcg_gen_op2_i64(INDEX_op_ext8u_i64, ret, arg); 1669 } else { 1670 tcg_gen_andi_i64(ret, arg, 0xffu); 1671 } 1672 } 1673 1674 void tcg_gen_ext16u_i64(TCGv_i64 ret, TCGv_i64 arg) 1675 { 1676 if (TCG_TARGET_REG_BITS == 32) { 1677 tcg_gen_ext16u_i32(TCGV_LOW(ret), TCGV_LOW(arg)); 1678 tcg_gen_movi_i32(TCGV_HIGH(ret), 0); 1679 } else if (TCG_TARGET_HAS_ext16u_i64) { 1680 tcg_gen_op2_i64(INDEX_op_ext16u_i64, ret, arg); 1681 } else { 1682 tcg_gen_andi_i64(ret, arg, 0xffffu); 1683 } 1684 } 1685 1686 void tcg_gen_ext32u_i64(TCGv_i64 ret, TCGv_i64 arg) 1687 { 1688 if (TCG_TARGET_REG_BITS == 32) { 1689 tcg_gen_mov_i32(TCGV_LOW(ret), TCGV_LOW(arg)); 1690 tcg_gen_movi_i32(TCGV_HIGH(ret), 0); 1691 } else if (TCG_TARGET_HAS_ext32u_i64) { 1692 tcg_gen_op2_i64(INDEX_op_ext32u_i64, ret, arg); 1693 } else { 1694 tcg_gen_andi_i64(ret, arg, 0xffffffffu); 1695 } 1696 } 1697 1698 void tcg_gen_bswap16_i64(TCGv_i64 ret, TCGv_i64 arg, int flags) 1699 { 1700 /* Only one extension flag may be present. */ 1701 tcg_debug_assert(!(flags & TCG_BSWAP_OS) || !(flags & TCG_BSWAP_OZ)); 1702 1703 if (TCG_TARGET_REG_BITS == 32) { 1704 tcg_gen_bswap16_i32(TCGV_LOW(ret), TCGV_LOW(arg), flags); 1705 if (flags & TCG_BSWAP_OS) { 1706 tcg_gen_sari_i32(TCGV_HIGH(ret), TCGV_LOW(ret), 31); 1707 } else { 1708 tcg_gen_movi_i32(TCGV_HIGH(ret), 0); 1709 } 1710 } else if (TCG_TARGET_HAS_bswap16_i64) { 1711 tcg_gen_op3i_i64(INDEX_op_bswap16_i64, ret, arg, flags); 1712 } else { 1713 TCGv_i64 t0 = tcg_temp_new_i64(); 1714 TCGv_i64 t1 = tcg_temp_new_i64(); 1715 1716 tcg_gen_shri_i64(t0, arg, 8); 1717 if (!(flags & TCG_BSWAP_IZ)) { 1718 tcg_gen_ext8u_i64(t0, t0); 1719 } 1720 1721 if (flags & TCG_BSWAP_OS) { 1722 tcg_gen_shli_i64(t1, arg, 56); 1723 tcg_gen_sari_i64(t1, t1, 48); 1724 } else if (flags & TCG_BSWAP_OZ) { 1725 tcg_gen_ext8u_i64(t1, arg); 1726 tcg_gen_shli_i64(t1, t1, 8); 1727 } else { 1728 tcg_gen_shli_i64(t1, arg, 8); 1729 } 1730 1731 tcg_gen_or_i64(ret, t0, t1); 1732 tcg_temp_free_i64(t0); 1733 tcg_temp_free_i64(t1); 1734 } 1735 } 1736 1737 void tcg_gen_bswap32_i64(TCGv_i64 ret, TCGv_i64 arg, int flags) 1738 { 1739 /* Only one extension flag may be present. */ 1740 tcg_debug_assert(!(flags & TCG_BSWAP_OS) || !(flags & TCG_BSWAP_OZ)); 1741 1742 if (TCG_TARGET_REG_BITS == 32) { 1743 tcg_gen_bswap32_i32(TCGV_LOW(ret), TCGV_LOW(arg)); 1744 if (flags & TCG_BSWAP_OS) { 1745 tcg_gen_sari_i32(TCGV_HIGH(ret), TCGV_LOW(ret), 31); 1746 } else { 1747 tcg_gen_movi_i32(TCGV_HIGH(ret), 0); 1748 } 1749 } else if (TCG_TARGET_HAS_bswap32_i64) { 1750 tcg_gen_op3i_i64(INDEX_op_bswap32_i64, ret, arg, flags); 1751 } else { 1752 TCGv_i64 t0 = tcg_temp_new_i64(); 1753 TCGv_i64 t1 = tcg_temp_new_i64(); 1754 TCGv_i64 t2 = tcg_constant_i64(0x00ff00ff); 1755 1756 /* arg = xxxxabcd */ 1757 tcg_gen_shri_i64(t0, arg, 8); /* t0 = .xxxxabc */ 1758 tcg_gen_and_i64(t1, arg, t2); /* t1 = .....b.d */ 1759 tcg_gen_and_i64(t0, t0, t2); /* t0 = .....a.c */ 1760 tcg_gen_shli_i64(t1, t1, 8); /* t1 = ....b.d. */ 1761 tcg_gen_or_i64(ret, t0, t1); /* ret = ....badc */ 1762 1763 tcg_gen_shli_i64(t1, ret, 48); /* t1 = dc...... */ 1764 tcg_gen_shri_i64(t0, ret, 16); /* t0 = ......ba */ 1765 if (flags & TCG_BSWAP_OS) { 1766 tcg_gen_sari_i64(t1, t1, 32); /* t1 = ssssdc.. */ 1767 } else { 1768 tcg_gen_shri_i64(t1, t1, 32); /* t1 = ....dc.. */ 1769 } 1770 tcg_gen_or_i64(ret, t0, t1); /* ret = ssssdcba */ 1771 1772 tcg_temp_free_i64(t0); 1773 tcg_temp_free_i64(t1); 1774 } 1775 } 1776 1777 void tcg_gen_bswap64_i64(TCGv_i64 ret, TCGv_i64 arg) 1778 { 1779 if (TCG_TARGET_REG_BITS == 32) { 1780 TCGv_i32 t0, t1; 1781 t0 = tcg_temp_new_i32(); 1782 t1 = tcg_temp_new_i32(); 1783 1784 tcg_gen_bswap32_i32(t0, TCGV_LOW(arg)); 1785 tcg_gen_bswap32_i32(t1, TCGV_HIGH(arg)); 1786 tcg_gen_mov_i32(TCGV_LOW(ret), t1); 1787 tcg_gen_mov_i32(TCGV_HIGH(ret), t0); 1788 tcg_temp_free_i32(t0); 1789 tcg_temp_free_i32(t1); 1790 } else if (TCG_TARGET_HAS_bswap64_i64) { 1791 tcg_gen_op3i_i64(INDEX_op_bswap64_i64, ret, arg, 0); 1792 } else { 1793 TCGv_i64 t0 = tcg_temp_new_i64(); 1794 TCGv_i64 t1 = tcg_temp_new_i64(); 1795 TCGv_i64 t2 = tcg_temp_new_i64(); 1796 1797 /* arg = abcdefgh */ 1798 tcg_gen_movi_i64(t2, 0x00ff00ff00ff00ffull); 1799 tcg_gen_shri_i64(t0, arg, 8); /* t0 = .abcdefg */ 1800 tcg_gen_and_i64(t1, arg, t2); /* t1 = .b.d.f.h */ 1801 tcg_gen_and_i64(t0, t0, t2); /* t0 = .a.c.e.g */ 1802 tcg_gen_shli_i64(t1, t1, 8); /* t1 = b.d.f.h. */ 1803 tcg_gen_or_i64(ret, t0, t1); /* ret = badcfehg */ 1804 1805 tcg_gen_movi_i64(t2, 0x0000ffff0000ffffull); 1806 tcg_gen_shri_i64(t0, ret, 16); /* t0 = ..badcfe */ 1807 tcg_gen_and_i64(t1, ret, t2); /* t1 = ..dc..hg */ 1808 tcg_gen_and_i64(t0, t0, t2); /* t0 = ..ba..fe */ 1809 tcg_gen_shli_i64(t1, t1, 16); /* t1 = dc..hg.. */ 1810 tcg_gen_or_i64(ret, t0, t1); /* ret = dcbahgfe */ 1811 1812 tcg_gen_shri_i64(t0, ret, 32); /* t0 = ....dcba */ 1813 tcg_gen_shli_i64(t1, ret, 32); /* t1 = hgfe.... */ 1814 tcg_gen_or_i64(ret, t0, t1); /* ret = hgfedcba */ 1815 1816 tcg_temp_free_i64(t0); 1817 tcg_temp_free_i64(t1); 1818 tcg_temp_free_i64(t2); 1819 } 1820 } 1821 1822 void tcg_gen_hswap_i64(TCGv_i64 ret, TCGv_i64 arg) 1823 { 1824 uint64_t m = 0x0000ffff0000ffffull; 1825 TCGv_i64 t0 = tcg_temp_new_i64(); 1826 TCGv_i64 t1 = tcg_temp_new_i64(); 1827 1828 /* See include/qemu/bitops.h, hswap64. */ 1829 tcg_gen_rotli_i64(t1, arg, 32); 1830 tcg_gen_andi_i64(t0, t1, m); 1831 tcg_gen_shli_i64(t0, t0, 16); 1832 tcg_gen_shri_i64(t1, t1, 16); 1833 tcg_gen_andi_i64(t1, t1, m); 1834 tcg_gen_or_i64(ret, t0, t1); 1835 1836 tcg_temp_free_i64(t0); 1837 tcg_temp_free_i64(t1); 1838 } 1839 1840 void tcg_gen_wswap_i64(TCGv_i64 ret, TCGv_i64 arg) 1841 { 1842 /* Swapping 2 32-bit elements is a rotate. */ 1843 tcg_gen_rotli_i64(ret, arg, 32); 1844 } 1845 1846 void tcg_gen_not_i64(TCGv_i64 ret, TCGv_i64 arg) 1847 { 1848 if (TCG_TARGET_REG_BITS == 32) { 1849 tcg_gen_not_i32(TCGV_LOW(ret), TCGV_LOW(arg)); 1850 tcg_gen_not_i32(TCGV_HIGH(ret), TCGV_HIGH(arg)); 1851 } else if (TCG_TARGET_HAS_not_i64) { 1852 tcg_gen_op2_i64(INDEX_op_not_i64, ret, arg); 1853 } else { 1854 tcg_gen_xori_i64(ret, arg, -1); 1855 } 1856 } 1857 1858 void tcg_gen_andc_i64(TCGv_i64 ret, TCGv_i64 arg1, TCGv_i64 arg2) 1859 { 1860 if (TCG_TARGET_REG_BITS == 32) { 1861 tcg_gen_andc_i32(TCGV_LOW(ret), TCGV_LOW(arg1), TCGV_LOW(arg2)); 1862 tcg_gen_andc_i32(TCGV_HIGH(ret), TCGV_HIGH(arg1), TCGV_HIGH(arg2)); 1863 } else if (TCG_TARGET_HAS_andc_i64) { 1864 tcg_gen_op3_i64(INDEX_op_andc_i64, ret, arg1, arg2); 1865 } else { 1866 TCGv_i64 t0 = tcg_temp_new_i64(); 1867 tcg_gen_not_i64(t0, arg2); 1868 tcg_gen_and_i64(ret, arg1, t0); 1869 tcg_temp_free_i64(t0); 1870 } 1871 } 1872 1873 void tcg_gen_eqv_i64(TCGv_i64 ret, TCGv_i64 arg1, TCGv_i64 arg2) 1874 { 1875 if (TCG_TARGET_REG_BITS == 32) { 1876 tcg_gen_eqv_i32(TCGV_LOW(ret), TCGV_LOW(arg1), TCGV_LOW(arg2)); 1877 tcg_gen_eqv_i32(TCGV_HIGH(ret), TCGV_HIGH(arg1), TCGV_HIGH(arg2)); 1878 } else if (TCG_TARGET_HAS_eqv_i64) { 1879 tcg_gen_op3_i64(INDEX_op_eqv_i64, ret, arg1, arg2); 1880 } else { 1881 tcg_gen_xor_i64(ret, arg1, arg2); 1882 tcg_gen_not_i64(ret, ret); 1883 } 1884 } 1885 1886 void tcg_gen_nand_i64(TCGv_i64 ret, TCGv_i64 arg1, TCGv_i64 arg2) 1887 { 1888 if (TCG_TARGET_REG_BITS == 32) { 1889 tcg_gen_nand_i32(TCGV_LOW(ret), TCGV_LOW(arg1), TCGV_LOW(arg2)); 1890 tcg_gen_nand_i32(TCGV_HIGH(ret), TCGV_HIGH(arg1), TCGV_HIGH(arg2)); 1891 } else if (TCG_TARGET_HAS_nand_i64) { 1892 tcg_gen_op3_i64(INDEX_op_nand_i64, ret, arg1, arg2); 1893 } else { 1894 tcg_gen_and_i64(ret, arg1, arg2); 1895 tcg_gen_not_i64(ret, ret); 1896 } 1897 } 1898 1899 void tcg_gen_nor_i64(TCGv_i64 ret, TCGv_i64 arg1, TCGv_i64 arg2) 1900 { 1901 if (TCG_TARGET_REG_BITS == 32) { 1902 tcg_gen_nor_i32(TCGV_LOW(ret), TCGV_LOW(arg1), TCGV_LOW(arg2)); 1903 tcg_gen_nor_i32(TCGV_HIGH(ret), TCGV_HIGH(arg1), TCGV_HIGH(arg2)); 1904 } else if (TCG_TARGET_HAS_nor_i64) { 1905 tcg_gen_op3_i64(INDEX_op_nor_i64, ret, arg1, arg2); 1906 } else { 1907 tcg_gen_or_i64(ret, arg1, arg2); 1908 tcg_gen_not_i64(ret, ret); 1909 } 1910 } 1911 1912 void tcg_gen_orc_i64(TCGv_i64 ret, TCGv_i64 arg1, TCGv_i64 arg2) 1913 { 1914 if (TCG_TARGET_REG_BITS == 32) { 1915 tcg_gen_orc_i32(TCGV_LOW(ret), TCGV_LOW(arg1), TCGV_LOW(arg2)); 1916 tcg_gen_orc_i32(TCGV_HIGH(ret), TCGV_HIGH(arg1), TCGV_HIGH(arg2)); 1917 } else if (TCG_TARGET_HAS_orc_i64) { 1918 tcg_gen_op3_i64(INDEX_op_orc_i64, ret, arg1, arg2); 1919 } else { 1920 TCGv_i64 t0 = tcg_temp_new_i64(); 1921 tcg_gen_not_i64(t0, arg2); 1922 tcg_gen_or_i64(ret, arg1, t0); 1923 tcg_temp_free_i64(t0); 1924 } 1925 } 1926 1927 void tcg_gen_clz_i64(TCGv_i64 ret, TCGv_i64 arg1, TCGv_i64 arg2) 1928 { 1929 if (TCG_TARGET_HAS_clz_i64) { 1930 tcg_gen_op3_i64(INDEX_op_clz_i64, ret, arg1, arg2); 1931 } else { 1932 gen_helper_clz_i64(ret, arg1, arg2); 1933 } 1934 } 1935 1936 void tcg_gen_clzi_i64(TCGv_i64 ret, TCGv_i64 arg1, uint64_t arg2) 1937 { 1938 if (TCG_TARGET_REG_BITS == 32 1939 && TCG_TARGET_HAS_clz_i32 1940 && arg2 <= 0xffffffffu) { 1941 TCGv_i32 t = tcg_temp_new_i32(); 1942 tcg_gen_clzi_i32(t, TCGV_LOW(arg1), arg2 - 32); 1943 tcg_gen_addi_i32(t, t, 32); 1944 tcg_gen_clz_i32(TCGV_LOW(ret), TCGV_HIGH(arg1), t); 1945 tcg_gen_movi_i32(TCGV_HIGH(ret), 0); 1946 tcg_temp_free_i32(t); 1947 } else { 1948 TCGv_i64 t0 = tcg_const_i64(arg2); 1949 tcg_gen_clz_i64(ret, arg1, t0); 1950 tcg_temp_free_i64(t0); 1951 } 1952 } 1953 1954 void tcg_gen_ctz_i64(TCGv_i64 ret, TCGv_i64 arg1, TCGv_i64 arg2) 1955 { 1956 if (TCG_TARGET_HAS_ctz_i64) { 1957 tcg_gen_op3_i64(INDEX_op_ctz_i64, ret, arg1, arg2); 1958 } else if (TCG_TARGET_HAS_ctpop_i64 || TCG_TARGET_HAS_clz_i64) { 1959 TCGv_i64 z, t = tcg_temp_new_i64(); 1960 1961 if (TCG_TARGET_HAS_ctpop_i64) { 1962 tcg_gen_subi_i64(t, arg1, 1); 1963 tcg_gen_andc_i64(t, t, arg1); 1964 tcg_gen_ctpop_i64(t, t); 1965 } else { 1966 /* Since all non-x86 hosts have clz(0) == 64, don't fight it. */ 1967 tcg_gen_neg_i64(t, arg1); 1968 tcg_gen_and_i64(t, t, arg1); 1969 tcg_gen_clzi_i64(t, t, 64); 1970 tcg_gen_xori_i64(t, t, 63); 1971 } 1972 z = tcg_constant_i64(0); 1973 tcg_gen_movcond_i64(TCG_COND_EQ, ret, arg1, z, arg2, t); 1974 tcg_temp_free_i64(t); 1975 tcg_temp_free_i64(z); 1976 } else { 1977 gen_helper_ctz_i64(ret, arg1, arg2); 1978 } 1979 } 1980 1981 void tcg_gen_ctzi_i64(TCGv_i64 ret, TCGv_i64 arg1, uint64_t arg2) 1982 { 1983 if (TCG_TARGET_REG_BITS == 32 1984 && TCG_TARGET_HAS_ctz_i32 1985 && arg2 <= 0xffffffffu) { 1986 TCGv_i32 t32 = tcg_temp_new_i32(); 1987 tcg_gen_ctzi_i32(t32, TCGV_HIGH(arg1), arg2 - 32); 1988 tcg_gen_addi_i32(t32, t32, 32); 1989 tcg_gen_ctz_i32(TCGV_LOW(ret), TCGV_LOW(arg1), t32); 1990 tcg_gen_movi_i32(TCGV_HIGH(ret), 0); 1991 tcg_temp_free_i32(t32); 1992 } else if (!TCG_TARGET_HAS_ctz_i64 1993 && TCG_TARGET_HAS_ctpop_i64 1994 && arg2 == 64) { 1995 /* This equivalence has the advantage of not requiring a fixup. */ 1996 TCGv_i64 t = tcg_temp_new_i64(); 1997 tcg_gen_subi_i64(t, arg1, 1); 1998 tcg_gen_andc_i64(t, t, arg1); 1999 tcg_gen_ctpop_i64(ret, t); 2000 tcg_temp_free_i64(t); 2001 } else { 2002 TCGv_i64 t0 = tcg_const_i64(arg2); 2003 tcg_gen_ctz_i64(ret, arg1, t0); 2004 tcg_temp_free_i64(t0); 2005 } 2006 } 2007 2008 void tcg_gen_clrsb_i64(TCGv_i64 ret, TCGv_i64 arg) 2009 { 2010 if (TCG_TARGET_HAS_clz_i64 || TCG_TARGET_HAS_clz_i32) { 2011 TCGv_i64 t = tcg_temp_new_i64(); 2012 tcg_gen_sari_i64(t, arg, 63); 2013 tcg_gen_xor_i64(t, t, arg); 2014 tcg_gen_clzi_i64(t, t, 64); 2015 tcg_gen_subi_i64(ret, t, 1); 2016 tcg_temp_free_i64(t); 2017 } else { 2018 gen_helper_clrsb_i64(ret, arg); 2019 } 2020 } 2021 2022 void tcg_gen_ctpop_i64(TCGv_i64 ret, TCGv_i64 arg1) 2023 { 2024 if (TCG_TARGET_HAS_ctpop_i64) { 2025 tcg_gen_op2_i64(INDEX_op_ctpop_i64, ret, arg1); 2026 } else if (TCG_TARGET_REG_BITS == 32 && TCG_TARGET_HAS_ctpop_i32) { 2027 tcg_gen_ctpop_i32(TCGV_HIGH(ret), TCGV_HIGH(arg1)); 2028 tcg_gen_ctpop_i32(TCGV_LOW(ret), TCGV_LOW(arg1)); 2029 tcg_gen_add_i32(TCGV_LOW(ret), TCGV_LOW(ret), TCGV_HIGH(ret)); 2030 tcg_gen_movi_i32(TCGV_HIGH(ret), 0); 2031 } else { 2032 gen_helper_ctpop_i64(ret, arg1); 2033 } 2034 } 2035 2036 void tcg_gen_rotl_i64(TCGv_i64 ret, TCGv_i64 arg1, TCGv_i64 arg2) 2037 { 2038 if (TCG_TARGET_HAS_rot_i64) { 2039 tcg_gen_op3_i64(INDEX_op_rotl_i64, ret, arg1, arg2); 2040 } else { 2041 TCGv_i64 t0, t1; 2042 t0 = tcg_temp_new_i64(); 2043 t1 = tcg_temp_new_i64(); 2044 tcg_gen_shl_i64(t0, arg1, arg2); 2045 tcg_gen_subfi_i64(t1, 64, arg2); 2046 tcg_gen_shr_i64(t1, arg1, t1); 2047 tcg_gen_or_i64(ret, t0, t1); 2048 tcg_temp_free_i64(t0); 2049 tcg_temp_free_i64(t1); 2050 } 2051 } 2052 2053 void tcg_gen_rotli_i64(TCGv_i64 ret, TCGv_i64 arg1, int64_t arg2) 2054 { 2055 tcg_debug_assert(arg2 >= 0 && arg2 < 64); 2056 /* some cases can be optimized here */ 2057 if (arg2 == 0) { 2058 tcg_gen_mov_i64(ret, arg1); 2059 } else if (TCG_TARGET_HAS_rot_i64) { 2060 tcg_gen_rotl_i64(ret, arg1, tcg_constant_i64(arg2)); 2061 } else { 2062 TCGv_i64 t0, t1; 2063 t0 = tcg_temp_new_i64(); 2064 t1 = tcg_temp_new_i64(); 2065 tcg_gen_shli_i64(t0, arg1, arg2); 2066 tcg_gen_shri_i64(t1, arg1, 64 - arg2); 2067 tcg_gen_or_i64(ret, t0, t1); 2068 tcg_temp_free_i64(t0); 2069 tcg_temp_free_i64(t1); 2070 } 2071 } 2072 2073 void tcg_gen_rotr_i64(TCGv_i64 ret, TCGv_i64 arg1, TCGv_i64 arg2) 2074 { 2075 if (TCG_TARGET_HAS_rot_i64) { 2076 tcg_gen_op3_i64(INDEX_op_rotr_i64, ret, arg1, arg2); 2077 } else { 2078 TCGv_i64 t0, t1; 2079 t0 = tcg_temp_new_i64(); 2080 t1 = tcg_temp_new_i64(); 2081 tcg_gen_shr_i64(t0, arg1, arg2); 2082 tcg_gen_subfi_i64(t1, 64, arg2); 2083 tcg_gen_shl_i64(t1, arg1, t1); 2084 tcg_gen_or_i64(ret, t0, t1); 2085 tcg_temp_free_i64(t0); 2086 tcg_temp_free_i64(t1); 2087 } 2088 } 2089 2090 void tcg_gen_rotri_i64(TCGv_i64 ret, TCGv_i64 arg1, int64_t arg2) 2091 { 2092 tcg_debug_assert(arg2 >= 0 && arg2 < 64); 2093 /* some cases can be optimized here */ 2094 if (arg2 == 0) { 2095 tcg_gen_mov_i64(ret, arg1); 2096 } else { 2097 tcg_gen_rotli_i64(ret, arg1, 64 - arg2); 2098 } 2099 } 2100 2101 void tcg_gen_deposit_i64(TCGv_i64 ret, TCGv_i64 arg1, TCGv_i64 arg2, 2102 unsigned int ofs, unsigned int len) 2103 { 2104 uint64_t mask; 2105 TCGv_i64 t1; 2106 2107 tcg_debug_assert(ofs < 64); 2108 tcg_debug_assert(len > 0); 2109 tcg_debug_assert(len <= 64); 2110 tcg_debug_assert(ofs + len <= 64); 2111 2112 if (len == 64) { 2113 tcg_gen_mov_i64(ret, arg2); 2114 return; 2115 } 2116 if (TCG_TARGET_HAS_deposit_i64 && TCG_TARGET_deposit_i64_valid(ofs, len)) { 2117 tcg_gen_op5ii_i64(INDEX_op_deposit_i64, ret, arg1, arg2, ofs, len); 2118 return; 2119 } 2120 2121 if (TCG_TARGET_REG_BITS == 32) { 2122 if (ofs >= 32) { 2123 tcg_gen_deposit_i32(TCGV_HIGH(ret), TCGV_HIGH(arg1), 2124 TCGV_LOW(arg2), ofs - 32, len); 2125 tcg_gen_mov_i32(TCGV_LOW(ret), TCGV_LOW(arg1)); 2126 return; 2127 } 2128 if (ofs + len <= 32) { 2129 tcg_gen_deposit_i32(TCGV_LOW(ret), TCGV_LOW(arg1), 2130 TCGV_LOW(arg2), ofs, len); 2131 tcg_gen_mov_i32(TCGV_HIGH(ret), TCGV_HIGH(arg1)); 2132 return; 2133 } 2134 } 2135 2136 t1 = tcg_temp_new_i64(); 2137 2138 if (TCG_TARGET_HAS_extract2_i64) { 2139 if (ofs + len == 64) { 2140 tcg_gen_shli_i64(t1, arg1, len); 2141 tcg_gen_extract2_i64(ret, t1, arg2, len); 2142 goto done; 2143 } 2144 if (ofs == 0) { 2145 tcg_gen_extract2_i64(ret, arg1, arg2, len); 2146 tcg_gen_rotli_i64(ret, ret, len); 2147 goto done; 2148 } 2149 } 2150 2151 mask = (1ull << len) - 1; 2152 if (ofs + len < 64) { 2153 tcg_gen_andi_i64(t1, arg2, mask); 2154 tcg_gen_shli_i64(t1, t1, ofs); 2155 } else { 2156 tcg_gen_shli_i64(t1, arg2, ofs); 2157 } 2158 tcg_gen_andi_i64(ret, arg1, ~(mask << ofs)); 2159 tcg_gen_or_i64(ret, ret, t1); 2160 done: 2161 tcg_temp_free_i64(t1); 2162 } 2163 2164 void tcg_gen_deposit_z_i64(TCGv_i64 ret, TCGv_i64 arg, 2165 unsigned int ofs, unsigned int len) 2166 { 2167 tcg_debug_assert(ofs < 64); 2168 tcg_debug_assert(len > 0); 2169 tcg_debug_assert(len <= 64); 2170 tcg_debug_assert(ofs + len <= 64); 2171 2172 if (ofs + len == 64) { 2173 tcg_gen_shli_i64(ret, arg, ofs); 2174 } else if (ofs == 0) { 2175 tcg_gen_andi_i64(ret, arg, (1ull << len) - 1); 2176 } else if (TCG_TARGET_HAS_deposit_i64 2177 && TCG_TARGET_deposit_i64_valid(ofs, len)) { 2178 TCGv_i64 zero = tcg_constant_i64(0); 2179 tcg_gen_op5ii_i64(INDEX_op_deposit_i64, ret, zero, arg, ofs, len); 2180 } else { 2181 if (TCG_TARGET_REG_BITS == 32) { 2182 if (ofs >= 32) { 2183 tcg_gen_deposit_z_i32(TCGV_HIGH(ret), TCGV_LOW(arg), 2184 ofs - 32, len); 2185 tcg_gen_movi_i32(TCGV_LOW(ret), 0); 2186 return; 2187 } 2188 if (ofs + len <= 32) { 2189 tcg_gen_deposit_z_i32(TCGV_LOW(ret), TCGV_LOW(arg), ofs, len); 2190 tcg_gen_movi_i32(TCGV_HIGH(ret), 0); 2191 return; 2192 } 2193 } 2194 /* To help two-operand hosts we prefer to zero-extend first, 2195 which allows ARG to stay live. */ 2196 switch (len) { 2197 case 32: 2198 if (TCG_TARGET_HAS_ext32u_i64) { 2199 tcg_gen_ext32u_i64(ret, arg); 2200 tcg_gen_shli_i64(ret, ret, ofs); 2201 return; 2202 } 2203 break; 2204 case 16: 2205 if (TCG_TARGET_HAS_ext16u_i64) { 2206 tcg_gen_ext16u_i64(ret, arg); 2207 tcg_gen_shli_i64(ret, ret, ofs); 2208 return; 2209 } 2210 break; 2211 case 8: 2212 if (TCG_TARGET_HAS_ext8u_i64) { 2213 tcg_gen_ext8u_i64(ret, arg); 2214 tcg_gen_shli_i64(ret, ret, ofs); 2215 return; 2216 } 2217 break; 2218 } 2219 /* Otherwise prefer zero-extension over AND for code size. */ 2220 switch (ofs + len) { 2221 case 32: 2222 if (TCG_TARGET_HAS_ext32u_i64) { 2223 tcg_gen_shli_i64(ret, arg, ofs); 2224 tcg_gen_ext32u_i64(ret, ret); 2225 return; 2226 } 2227 break; 2228 case 16: 2229 if (TCG_TARGET_HAS_ext16u_i64) { 2230 tcg_gen_shli_i64(ret, arg, ofs); 2231 tcg_gen_ext16u_i64(ret, ret); 2232 return; 2233 } 2234 break; 2235 case 8: 2236 if (TCG_TARGET_HAS_ext8u_i64) { 2237 tcg_gen_shli_i64(ret, arg, ofs); 2238 tcg_gen_ext8u_i64(ret, ret); 2239 return; 2240 } 2241 break; 2242 } 2243 tcg_gen_andi_i64(ret, arg, (1ull << len) - 1); 2244 tcg_gen_shli_i64(ret, ret, ofs); 2245 } 2246 } 2247 2248 void tcg_gen_extract_i64(TCGv_i64 ret, TCGv_i64 arg, 2249 unsigned int ofs, unsigned int len) 2250 { 2251 tcg_debug_assert(ofs < 64); 2252 tcg_debug_assert(len > 0); 2253 tcg_debug_assert(len <= 64); 2254 tcg_debug_assert(ofs + len <= 64); 2255 2256 /* Canonicalize certain special cases, even if extract is supported. */ 2257 if (ofs + len == 64) { 2258 tcg_gen_shri_i64(ret, arg, 64 - len); 2259 return; 2260 } 2261 if (ofs == 0) { 2262 tcg_gen_andi_i64(ret, arg, (1ull << len) - 1); 2263 return; 2264 } 2265 2266 if (TCG_TARGET_REG_BITS == 32) { 2267 /* Look for a 32-bit extract within one of the two words. */ 2268 if (ofs >= 32) { 2269 tcg_gen_extract_i32(TCGV_LOW(ret), TCGV_HIGH(arg), ofs - 32, len); 2270 tcg_gen_movi_i32(TCGV_HIGH(ret), 0); 2271 return; 2272 } 2273 if (ofs + len <= 32) { 2274 tcg_gen_extract_i32(TCGV_LOW(ret), TCGV_LOW(arg), ofs, len); 2275 tcg_gen_movi_i32(TCGV_HIGH(ret), 0); 2276 return; 2277 } 2278 /* The field is split across two words. One double-word 2279 shift is better than two double-word shifts. */ 2280 goto do_shift_and; 2281 } 2282 2283 if (TCG_TARGET_HAS_extract_i64 2284 && TCG_TARGET_extract_i64_valid(ofs, len)) { 2285 tcg_gen_op4ii_i64(INDEX_op_extract_i64, ret, arg, ofs, len); 2286 return; 2287 } 2288 2289 /* Assume that zero-extension, if available, is cheaper than a shift. */ 2290 switch (ofs + len) { 2291 case 32: 2292 if (TCG_TARGET_HAS_ext32u_i64) { 2293 tcg_gen_ext32u_i64(ret, arg); 2294 tcg_gen_shri_i64(ret, ret, ofs); 2295 return; 2296 } 2297 break; 2298 case 16: 2299 if (TCG_TARGET_HAS_ext16u_i64) { 2300 tcg_gen_ext16u_i64(ret, arg); 2301 tcg_gen_shri_i64(ret, ret, ofs); 2302 return; 2303 } 2304 break; 2305 case 8: 2306 if (TCG_TARGET_HAS_ext8u_i64) { 2307 tcg_gen_ext8u_i64(ret, arg); 2308 tcg_gen_shri_i64(ret, ret, ofs); 2309 return; 2310 } 2311 break; 2312 } 2313 2314 /* ??? Ideally we'd know what values are available for immediate AND. 2315 Assume that 8 bits are available, plus the special cases of 16 and 32, 2316 so that we get ext8u, ext16u, and ext32u. */ 2317 switch (len) { 2318 case 1 ... 8: case 16: case 32: 2319 do_shift_and: 2320 tcg_gen_shri_i64(ret, arg, ofs); 2321 tcg_gen_andi_i64(ret, ret, (1ull << len) - 1); 2322 break; 2323 default: 2324 tcg_gen_shli_i64(ret, arg, 64 - len - ofs); 2325 tcg_gen_shri_i64(ret, ret, 64 - len); 2326 break; 2327 } 2328 } 2329 2330 void tcg_gen_sextract_i64(TCGv_i64 ret, TCGv_i64 arg, 2331 unsigned int ofs, unsigned int len) 2332 { 2333 tcg_debug_assert(ofs < 64); 2334 tcg_debug_assert(len > 0); 2335 tcg_debug_assert(len <= 64); 2336 tcg_debug_assert(ofs + len <= 64); 2337 2338 /* Canonicalize certain special cases, even if sextract is supported. */ 2339 if (ofs + len == 64) { 2340 tcg_gen_sari_i64(ret, arg, 64 - len); 2341 return; 2342 } 2343 if (ofs == 0) { 2344 switch (len) { 2345 case 32: 2346 tcg_gen_ext32s_i64(ret, arg); 2347 return; 2348 case 16: 2349 tcg_gen_ext16s_i64(ret, arg); 2350 return; 2351 case 8: 2352 tcg_gen_ext8s_i64(ret, arg); 2353 return; 2354 } 2355 } 2356 2357 if (TCG_TARGET_REG_BITS == 32) { 2358 /* Look for a 32-bit extract within one of the two words. */ 2359 if (ofs >= 32) { 2360 tcg_gen_sextract_i32(TCGV_LOW(ret), TCGV_HIGH(arg), ofs - 32, len); 2361 } else if (ofs + len <= 32) { 2362 tcg_gen_sextract_i32(TCGV_LOW(ret), TCGV_LOW(arg), ofs, len); 2363 } else if (ofs == 0) { 2364 tcg_gen_mov_i32(TCGV_LOW(ret), TCGV_LOW(arg)); 2365 tcg_gen_sextract_i32(TCGV_HIGH(ret), TCGV_HIGH(arg), 0, len - 32); 2366 return; 2367 } else if (len > 32) { 2368 TCGv_i32 t = tcg_temp_new_i32(); 2369 /* Extract the bits for the high word normally. */ 2370 tcg_gen_sextract_i32(t, TCGV_HIGH(arg), ofs + 32, len - 32); 2371 /* Shift the field down for the low part. */ 2372 tcg_gen_shri_i64(ret, arg, ofs); 2373 /* Overwrite the shift into the high part. */ 2374 tcg_gen_mov_i32(TCGV_HIGH(ret), t); 2375 tcg_temp_free_i32(t); 2376 return; 2377 } else { 2378 /* Shift the field down for the low part, such that the 2379 field sits at the MSB. */ 2380 tcg_gen_shri_i64(ret, arg, ofs + len - 32); 2381 /* Shift the field down from the MSB, sign extending. */ 2382 tcg_gen_sari_i32(TCGV_LOW(ret), TCGV_LOW(ret), 32 - len); 2383 } 2384 /* Sign-extend the field from 32 bits. */ 2385 tcg_gen_sari_i32(TCGV_HIGH(ret), TCGV_LOW(ret), 31); 2386 return; 2387 } 2388 2389 if (TCG_TARGET_HAS_sextract_i64 2390 && TCG_TARGET_extract_i64_valid(ofs, len)) { 2391 tcg_gen_op4ii_i64(INDEX_op_sextract_i64, ret, arg, ofs, len); 2392 return; 2393 } 2394 2395 /* Assume that sign-extension, if available, is cheaper than a shift. */ 2396 switch (ofs + len) { 2397 case 32: 2398 if (TCG_TARGET_HAS_ext32s_i64) { 2399 tcg_gen_ext32s_i64(ret, arg); 2400 tcg_gen_sari_i64(ret, ret, ofs); 2401 return; 2402 } 2403 break; 2404 case 16: 2405 if (TCG_TARGET_HAS_ext16s_i64) { 2406 tcg_gen_ext16s_i64(ret, arg); 2407 tcg_gen_sari_i64(ret, ret, ofs); 2408 return; 2409 } 2410 break; 2411 case 8: 2412 if (TCG_TARGET_HAS_ext8s_i64) { 2413 tcg_gen_ext8s_i64(ret, arg); 2414 tcg_gen_sari_i64(ret, ret, ofs); 2415 return; 2416 } 2417 break; 2418 } 2419 switch (len) { 2420 case 32: 2421 if (TCG_TARGET_HAS_ext32s_i64) { 2422 tcg_gen_shri_i64(ret, arg, ofs); 2423 tcg_gen_ext32s_i64(ret, ret); 2424 return; 2425 } 2426 break; 2427 case 16: 2428 if (TCG_TARGET_HAS_ext16s_i64) { 2429 tcg_gen_shri_i64(ret, arg, ofs); 2430 tcg_gen_ext16s_i64(ret, ret); 2431 return; 2432 } 2433 break; 2434 case 8: 2435 if (TCG_TARGET_HAS_ext8s_i64) { 2436 tcg_gen_shri_i64(ret, arg, ofs); 2437 tcg_gen_ext8s_i64(ret, ret); 2438 return; 2439 } 2440 break; 2441 } 2442 tcg_gen_shli_i64(ret, arg, 64 - len - ofs); 2443 tcg_gen_sari_i64(ret, ret, 64 - len); 2444 } 2445 2446 /* 2447 * Extract 64 bits from a 128-bit input, ah:al, starting from ofs. 2448 * Unlike tcg_gen_extract_i64 above, len is fixed at 64. 2449 */ 2450 void tcg_gen_extract2_i64(TCGv_i64 ret, TCGv_i64 al, TCGv_i64 ah, 2451 unsigned int ofs) 2452 { 2453 tcg_debug_assert(ofs <= 64); 2454 if (ofs == 0) { 2455 tcg_gen_mov_i64(ret, al); 2456 } else if (ofs == 64) { 2457 tcg_gen_mov_i64(ret, ah); 2458 } else if (al == ah) { 2459 tcg_gen_rotri_i64(ret, al, ofs); 2460 } else if (TCG_TARGET_HAS_extract2_i64) { 2461 tcg_gen_op4i_i64(INDEX_op_extract2_i64, ret, al, ah, ofs); 2462 } else { 2463 TCGv_i64 t0 = tcg_temp_new_i64(); 2464 tcg_gen_shri_i64(t0, al, ofs); 2465 tcg_gen_deposit_i64(ret, t0, ah, 64 - ofs, ofs); 2466 tcg_temp_free_i64(t0); 2467 } 2468 } 2469 2470 void tcg_gen_movcond_i64(TCGCond cond, TCGv_i64 ret, TCGv_i64 c1, 2471 TCGv_i64 c2, TCGv_i64 v1, TCGv_i64 v2) 2472 { 2473 if (cond == TCG_COND_ALWAYS) { 2474 tcg_gen_mov_i64(ret, v1); 2475 } else if (cond == TCG_COND_NEVER) { 2476 tcg_gen_mov_i64(ret, v2); 2477 } else if (TCG_TARGET_REG_BITS == 32) { 2478 TCGv_i32 t0 = tcg_temp_new_i32(); 2479 TCGv_i32 t1 = tcg_temp_new_i32(); 2480 tcg_gen_op6i_i32(INDEX_op_setcond2_i32, t0, 2481 TCGV_LOW(c1), TCGV_HIGH(c1), 2482 TCGV_LOW(c2), TCGV_HIGH(c2), cond); 2483 2484 if (TCG_TARGET_HAS_movcond_i32) { 2485 tcg_gen_movi_i32(t1, 0); 2486 tcg_gen_movcond_i32(TCG_COND_NE, TCGV_LOW(ret), t0, t1, 2487 TCGV_LOW(v1), TCGV_LOW(v2)); 2488 tcg_gen_movcond_i32(TCG_COND_NE, TCGV_HIGH(ret), t0, t1, 2489 TCGV_HIGH(v1), TCGV_HIGH(v2)); 2490 } else { 2491 tcg_gen_neg_i32(t0, t0); 2492 2493 tcg_gen_and_i32(t1, TCGV_LOW(v1), t0); 2494 tcg_gen_andc_i32(TCGV_LOW(ret), TCGV_LOW(v2), t0); 2495 tcg_gen_or_i32(TCGV_LOW(ret), TCGV_LOW(ret), t1); 2496 2497 tcg_gen_and_i32(t1, TCGV_HIGH(v1), t0); 2498 tcg_gen_andc_i32(TCGV_HIGH(ret), TCGV_HIGH(v2), t0); 2499 tcg_gen_or_i32(TCGV_HIGH(ret), TCGV_HIGH(ret), t1); 2500 } 2501 tcg_temp_free_i32(t0); 2502 tcg_temp_free_i32(t1); 2503 } else if (TCG_TARGET_HAS_movcond_i64) { 2504 tcg_gen_op6i_i64(INDEX_op_movcond_i64, ret, c1, c2, v1, v2, cond); 2505 } else { 2506 TCGv_i64 t0 = tcg_temp_new_i64(); 2507 TCGv_i64 t1 = tcg_temp_new_i64(); 2508 tcg_gen_setcond_i64(cond, t0, c1, c2); 2509 tcg_gen_neg_i64(t0, t0); 2510 tcg_gen_and_i64(t1, v1, t0); 2511 tcg_gen_andc_i64(ret, v2, t0); 2512 tcg_gen_or_i64(ret, ret, t1); 2513 tcg_temp_free_i64(t0); 2514 tcg_temp_free_i64(t1); 2515 } 2516 } 2517 2518 void tcg_gen_add2_i64(TCGv_i64 rl, TCGv_i64 rh, TCGv_i64 al, 2519 TCGv_i64 ah, TCGv_i64 bl, TCGv_i64 bh) 2520 { 2521 if (TCG_TARGET_HAS_add2_i64) { 2522 tcg_gen_op6_i64(INDEX_op_add2_i64, rl, rh, al, ah, bl, bh); 2523 } else { 2524 TCGv_i64 t0 = tcg_temp_new_i64(); 2525 TCGv_i64 t1 = tcg_temp_new_i64(); 2526 tcg_gen_add_i64(t0, al, bl); 2527 tcg_gen_setcond_i64(TCG_COND_LTU, t1, t0, al); 2528 tcg_gen_add_i64(rh, ah, bh); 2529 tcg_gen_add_i64(rh, rh, t1); 2530 tcg_gen_mov_i64(rl, t0); 2531 tcg_temp_free_i64(t0); 2532 tcg_temp_free_i64(t1); 2533 } 2534 } 2535 2536 void tcg_gen_sub2_i64(TCGv_i64 rl, TCGv_i64 rh, TCGv_i64 al, 2537 TCGv_i64 ah, TCGv_i64 bl, TCGv_i64 bh) 2538 { 2539 if (TCG_TARGET_HAS_sub2_i64) { 2540 tcg_gen_op6_i64(INDEX_op_sub2_i64, rl, rh, al, ah, bl, bh); 2541 } else { 2542 TCGv_i64 t0 = tcg_temp_new_i64(); 2543 TCGv_i64 t1 = tcg_temp_new_i64(); 2544 tcg_gen_sub_i64(t0, al, bl); 2545 tcg_gen_setcond_i64(TCG_COND_LTU, t1, al, bl); 2546 tcg_gen_sub_i64(rh, ah, bh); 2547 tcg_gen_sub_i64(rh, rh, t1); 2548 tcg_gen_mov_i64(rl, t0); 2549 tcg_temp_free_i64(t0); 2550 tcg_temp_free_i64(t1); 2551 } 2552 } 2553 2554 void tcg_gen_mulu2_i64(TCGv_i64 rl, TCGv_i64 rh, TCGv_i64 arg1, TCGv_i64 arg2) 2555 { 2556 if (TCG_TARGET_HAS_mulu2_i64) { 2557 tcg_gen_op4_i64(INDEX_op_mulu2_i64, rl, rh, arg1, arg2); 2558 } else if (TCG_TARGET_HAS_muluh_i64) { 2559 TCGv_i64 t = tcg_temp_new_i64(); 2560 tcg_gen_op3_i64(INDEX_op_mul_i64, t, arg1, arg2); 2561 tcg_gen_op3_i64(INDEX_op_muluh_i64, rh, arg1, arg2); 2562 tcg_gen_mov_i64(rl, t); 2563 tcg_temp_free_i64(t); 2564 } else { 2565 TCGv_i64 t0 = tcg_temp_new_i64(); 2566 tcg_gen_mul_i64(t0, arg1, arg2); 2567 gen_helper_muluh_i64(rh, arg1, arg2); 2568 tcg_gen_mov_i64(rl, t0); 2569 tcg_temp_free_i64(t0); 2570 } 2571 } 2572 2573 void tcg_gen_muls2_i64(TCGv_i64 rl, TCGv_i64 rh, TCGv_i64 arg1, TCGv_i64 arg2) 2574 { 2575 if (TCG_TARGET_HAS_muls2_i64) { 2576 tcg_gen_op4_i64(INDEX_op_muls2_i64, rl, rh, arg1, arg2); 2577 } else if (TCG_TARGET_HAS_mulsh_i64) { 2578 TCGv_i64 t = tcg_temp_new_i64(); 2579 tcg_gen_op3_i64(INDEX_op_mul_i64, t, arg1, arg2); 2580 tcg_gen_op3_i64(INDEX_op_mulsh_i64, rh, arg1, arg2); 2581 tcg_gen_mov_i64(rl, t); 2582 tcg_temp_free_i64(t); 2583 } else if (TCG_TARGET_HAS_mulu2_i64 || TCG_TARGET_HAS_muluh_i64) { 2584 TCGv_i64 t0 = tcg_temp_new_i64(); 2585 TCGv_i64 t1 = tcg_temp_new_i64(); 2586 TCGv_i64 t2 = tcg_temp_new_i64(); 2587 TCGv_i64 t3 = tcg_temp_new_i64(); 2588 tcg_gen_mulu2_i64(t0, t1, arg1, arg2); 2589 /* Adjust for negative inputs. */ 2590 tcg_gen_sari_i64(t2, arg1, 63); 2591 tcg_gen_sari_i64(t3, arg2, 63); 2592 tcg_gen_and_i64(t2, t2, arg2); 2593 tcg_gen_and_i64(t3, t3, arg1); 2594 tcg_gen_sub_i64(rh, t1, t2); 2595 tcg_gen_sub_i64(rh, rh, t3); 2596 tcg_gen_mov_i64(rl, t0); 2597 tcg_temp_free_i64(t0); 2598 tcg_temp_free_i64(t1); 2599 tcg_temp_free_i64(t2); 2600 tcg_temp_free_i64(t3); 2601 } else { 2602 TCGv_i64 t0 = tcg_temp_new_i64(); 2603 tcg_gen_mul_i64(t0, arg1, arg2); 2604 gen_helper_mulsh_i64(rh, arg1, arg2); 2605 tcg_gen_mov_i64(rl, t0); 2606 tcg_temp_free_i64(t0); 2607 } 2608 } 2609 2610 void tcg_gen_mulsu2_i64(TCGv_i64 rl, TCGv_i64 rh, TCGv_i64 arg1, TCGv_i64 arg2) 2611 { 2612 TCGv_i64 t0 = tcg_temp_new_i64(); 2613 TCGv_i64 t1 = tcg_temp_new_i64(); 2614 TCGv_i64 t2 = tcg_temp_new_i64(); 2615 tcg_gen_mulu2_i64(t0, t1, arg1, arg2); 2616 /* Adjust for negative input for the signed arg1. */ 2617 tcg_gen_sari_i64(t2, arg1, 63); 2618 tcg_gen_and_i64(t2, t2, arg2); 2619 tcg_gen_sub_i64(rh, t1, t2); 2620 tcg_gen_mov_i64(rl, t0); 2621 tcg_temp_free_i64(t0); 2622 tcg_temp_free_i64(t1); 2623 tcg_temp_free_i64(t2); 2624 } 2625 2626 void tcg_gen_smin_i64(TCGv_i64 ret, TCGv_i64 a, TCGv_i64 b) 2627 { 2628 tcg_gen_movcond_i64(TCG_COND_LT, ret, a, b, a, b); 2629 } 2630 2631 void tcg_gen_umin_i64(TCGv_i64 ret, TCGv_i64 a, TCGv_i64 b) 2632 { 2633 tcg_gen_movcond_i64(TCG_COND_LTU, ret, a, b, a, b); 2634 } 2635 2636 void tcg_gen_smax_i64(TCGv_i64 ret, TCGv_i64 a, TCGv_i64 b) 2637 { 2638 tcg_gen_movcond_i64(TCG_COND_LT, ret, a, b, b, a); 2639 } 2640 2641 void tcg_gen_umax_i64(TCGv_i64 ret, TCGv_i64 a, TCGv_i64 b) 2642 { 2643 tcg_gen_movcond_i64(TCG_COND_LTU, ret, a, b, b, a); 2644 } 2645 2646 void tcg_gen_abs_i64(TCGv_i64 ret, TCGv_i64 a) 2647 { 2648 TCGv_i64 t = tcg_temp_new_i64(); 2649 2650 tcg_gen_sari_i64(t, a, 63); 2651 tcg_gen_xor_i64(ret, a, t); 2652 tcg_gen_sub_i64(ret, ret, t); 2653 tcg_temp_free_i64(t); 2654 } 2655 2656 /* Size changing operations. */ 2657 2658 void tcg_gen_extrl_i64_i32(TCGv_i32 ret, TCGv_i64 arg) 2659 { 2660 if (TCG_TARGET_REG_BITS == 32) { 2661 tcg_gen_mov_i32(ret, TCGV_LOW(arg)); 2662 } else if (TCG_TARGET_HAS_extrl_i64_i32) { 2663 tcg_gen_op2(INDEX_op_extrl_i64_i32, 2664 tcgv_i32_arg(ret), tcgv_i64_arg(arg)); 2665 } else { 2666 tcg_gen_mov_i32(ret, (TCGv_i32)arg); 2667 } 2668 } 2669 2670 void tcg_gen_extrh_i64_i32(TCGv_i32 ret, TCGv_i64 arg) 2671 { 2672 if (TCG_TARGET_REG_BITS == 32) { 2673 tcg_gen_mov_i32(ret, TCGV_HIGH(arg)); 2674 } else if (TCG_TARGET_HAS_extrh_i64_i32) { 2675 tcg_gen_op2(INDEX_op_extrh_i64_i32, 2676 tcgv_i32_arg(ret), tcgv_i64_arg(arg)); 2677 } else { 2678 TCGv_i64 t = tcg_temp_new_i64(); 2679 tcg_gen_shri_i64(t, arg, 32); 2680 tcg_gen_mov_i32(ret, (TCGv_i32)t); 2681 tcg_temp_free_i64(t); 2682 } 2683 } 2684 2685 void tcg_gen_extu_i32_i64(TCGv_i64 ret, TCGv_i32 arg) 2686 { 2687 if (TCG_TARGET_REG_BITS == 32) { 2688 tcg_gen_mov_i32(TCGV_LOW(ret), arg); 2689 tcg_gen_movi_i32(TCGV_HIGH(ret), 0); 2690 } else { 2691 tcg_gen_op2(INDEX_op_extu_i32_i64, 2692 tcgv_i64_arg(ret), tcgv_i32_arg(arg)); 2693 } 2694 } 2695 2696 void tcg_gen_ext_i32_i64(TCGv_i64 ret, TCGv_i32 arg) 2697 { 2698 if (TCG_TARGET_REG_BITS == 32) { 2699 tcg_gen_mov_i32(TCGV_LOW(ret), arg); 2700 tcg_gen_sari_i32(TCGV_HIGH(ret), TCGV_LOW(ret), 31); 2701 } else { 2702 tcg_gen_op2(INDEX_op_ext_i32_i64, 2703 tcgv_i64_arg(ret), tcgv_i32_arg(arg)); 2704 } 2705 } 2706 2707 void tcg_gen_concat_i32_i64(TCGv_i64 dest, TCGv_i32 low, TCGv_i32 high) 2708 { 2709 TCGv_i64 tmp; 2710 2711 if (TCG_TARGET_REG_BITS == 32) { 2712 tcg_gen_mov_i32(TCGV_LOW(dest), low); 2713 tcg_gen_mov_i32(TCGV_HIGH(dest), high); 2714 return; 2715 } 2716 2717 tmp = tcg_temp_new_i64(); 2718 /* These extensions are only needed for type correctness. 2719 We may be able to do better given target specific information. */ 2720 tcg_gen_extu_i32_i64(tmp, high); 2721 tcg_gen_extu_i32_i64(dest, low); 2722 /* If deposit is available, use it. Otherwise use the extra 2723 knowledge that we have of the zero-extensions above. */ 2724 if (TCG_TARGET_HAS_deposit_i64 && TCG_TARGET_deposit_i64_valid(32, 32)) { 2725 tcg_gen_deposit_i64(dest, dest, tmp, 32, 32); 2726 } else { 2727 tcg_gen_shli_i64(tmp, tmp, 32); 2728 tcg_gen_or_i64(dest, dest, tmp); 2729 } 2730 tcg_temp_free_i64(tmp); 2731 } 2732 2733 void tcg_gen_extr_i64_i32(TCGv_i32 lo, TCGv_i32 hi, TCGv_i64 arg) 2734 { 2735 if (TCG_TARGET_REG_BITS == 32) { 2736 tcg_gen_mov_i32(lo, TCGV_LOW(arg)); 2737 tcg_gen_mov_i32(hi, TCGV_HIGH(arg)); 2738 } else { 2739 tcg_gen_extrl_i64_i32(lo, arg); 2740 tcg_gen_extrh_i64_i32(hi, arg); 2741 } 2742 } 2743 2744 void tcg_gen_extr32_i64(TCGv_i64 lo, TCGv_i64 hi, TCGv_i64 arg) 2745 { 2746 tcg_gen_ext32u_i64(lo, arg); 2747 tcg_gen_shri_i64(hi, arg, 32); 2748 } 2749 2750 /* QEMU specific operations. */ 2751 2752 void tcg_gen_exit_tb(const TranslationBlock *tb, unsigned idx) 2753 { 2754 /* 2755 * Let the jit code return the read-only version of the 2756 * TranslationBlock, so that we minimize the pc-relative 2757 * distance of the address of the exit_tb code to TB. 2758 * This will improve utilization of pc-relative address loads. 2759 * 2760 * TODO: Move this to translator_loop, so that all const 2761 * TranslationBlock pointers refer to read-only memory. 2762 * This requires coordination with targets that do not use 2763 * the translator_loop. 2764 */ 2765 uintptr_t val = (uintptr_t)tcg_splitwx_to_rx((void *)tb) + idx; 2766 2767 if (tb == NULL) { 2768 tcg_debug_assert(idx == 0); 2769 } else if (idx <= TB_EXIT_IDXMAX) { 2770 #ifdef CONFIG_DEBUG_TCG 2771 /* This is an exit following a goto_tb. Verify that we have 2772 seen this numbered exit before, via tcg_gen_goto_tb. */ 2773 tcg_debug_assert(tcg_ctx->goto_tb_issue_mask & (1 << idx)); 2774 #endif 2775 } else { 2776 /* This is an exit via the exitreq label. */ 2777 tcg_debug_assert(idx == TB_EXIT_REQUESTED); 2778 } 2779 2780 plugin_gen_disable_mem_helpers(); 2781 tcg_gen_op1i(INDEX_op_exit_tb, val); 2782 } 2783 2784 void tcg_gen_goto_tb(unsigned idx) 2785 { 2786 /* We tested CF_NO_GOTO_TB in translator_use_goto_tb. */ 2787 tcg_debug_assert(!(tcg_ctx->gen_tb->cflags & CF_NO_GOTO_TB)); 2788 /* We only support two chained exits. */ 2789 tcg_debug_assert(idx <= TB_EXIT_IDXMAX); 2790 #ifdef CONFIG_DEBUG_TCG 2791 /* Verify that we haven't seen this numbered exit before. */ 2792 tcg_debug_assert((tcg_ctx->goto_tb_issue_mask & (1 << idx)) == 0); 2793 tcg_ctx->goto_tb_issue_mask |= 1 << idx; 2794 #endif 2795 plugin_gen_disable_mem_helpers(); 2796 tcg_gen_op1i(INDEX_op_goto_tb, idx); 2797 } 2798 2799 void tcg_gen_lookup_and_goto_ptr(void) 2800 { 2801 TCGv_ptr ptr; 2802 2803 if (tcg_ctx->gen_tb->cflags & CF_NO_GOTO_PTR) { 2804 tcg_gen_exit_tb(NULL, 0); 2805 return; 2806 } 2807 2808 plugin_gen_disable_mem_helpers(); 2809 ptr = tcg_temp_new_ptr(); 2810 gen_helper_lookup_tb_ptr(ptr, cpu_env); 2811 tcg_gen_op1i(INDEX_op_goto_ptr, tcgv_ptr_arg(ptr)); 2812 tcg_temp_free_ptr(ptr); 2813 } 2814 2815 static inline MemOp tcg_canonicalize_memop(MemOp op, bool is64, bool st) 2816 { 2817 /* Trigger the asserts within as early as possible. */ 2818 unsigned a_bits = get_alignment_bits(op); 2819 2820 /* Prefer MO_ALIGN+MO_XX over MO_ALIGN_XX+MO_XX */ 2821 if (a_bits == (op & MO_SIZE)) { 2822 op = (op & ~MO_AMASK) | MO_ALIGN; 2823 } 2824 2825 switch (op & MO_SIZE) { 2826 case MO_8: 2827 op &= ~MO_BSWAP; 2828 break; 2829 case MO_16: 2830 break; 2831 case MO_32: 2832 if (!is64) { 2833 op &= ~MO_SIGN; 2834 } 2835 break; 2836 case MO_64: 2837 if (is64) { 2838 op &= ~MO_SIGN; 2839 break; 2840 } 2841 /* fall through */ 2842 default: 2843 g_assert_not_reached(); 2844 } 2845 if (st) { 2846 op &= ~MO_SIGN; 2847 } 2848 return op; 2849 } 2850 2851 static void gen_ldst_i32(TCGOpcode opc, TCGv_i32 val, TCGv addr, 2852 MemOp memop, TCGArg idx) 2853 { 2854 MemOpIdx oi = make_memop_idx(memop, idx); 2855 #if TARGET_LONG_BITS == 32 2856 tcg_gen_op3i_i32(opc, val, addr, oi); 2857 #else 2858 if (TCG_TARGET_REG_BITS == 32) { 2859 tcg_gen_op4i_i32(opc, val, TCGV_LOW(addr), TCGV_HIGH(addr), oi); 2860 } else { 2861 tcg_gen_op3(opc, tcgv_i32_arg(val), tcgv_i64_arg(addr), oi); 2862 } 2863 #endif 2864 } 2865 2866 static void gen_ldst_i64(TCGOpcode opc, TCGv_i64 val, TCGv addr, 2867 MemOp memop, TCGArg idx) 2868 { 2869 MemOpIdx oi = make_memop_idx(memop, idx); 2870 #if TARGET_LONG_BITS == 32 2871 if (TCG_TARGET_REG_BITS == 32) { 2872 tcg_gen_op4i_i32(opc, TCGV_LOW(val), TCGV_HIGH(val), addr, oi); 2873 } else { 2874 tcg_gen_op3(opc, tcgv_i64_arg(val), tcgv_i32_arg(addr), oi); 2875 } 2876 #else 2877 if (TCG_TARGET_REG_BITS == 32) { 2878 tcg_gen_op5i_i32(opc, TCGV_LOW(val), TCGV_HIGH(val), 2879 TCGV_LOW(addr), TCGV_HIGH(addr), oi); 2880 } else { 2881 tcg_gen_op3i_i64(opc, val, addr, oi); 2882 } 2883 #endif 2884 } 2885 2886 static void tcg_gen_req_mo(TCGBar type) 2887 { 2888 #ifdef TCG_GUEST_DEFAULT_MO 2889 type &= TCG_GUEST_DEFAULT_MO; 2890 #endif 2891 type &= ~TCG_TARGET_DEFAULT_MO; 2892 if (type) { 2893 tcg_gen_mb(type | TCG_BAR_SC); 2894 } 2895 } 2896 2897 static inline TCGv plugin_prep_mem_callbacks(TCGv vaddr) 2898 { 2899 #ifdef CONFIG_PLUGIN 2900 if (tcg_ctx->plugin_insn != NULL) { 2901 /* Save a copy of the vaddr for use after a load. */ 2902 TCGv temp = tcg_temp_new(); 2903 tcg_gen_mov_tl(temp, vaddr); 2904 return temp; 2905 } 2906 #endif 2907 return vaddr; 2908 } 2909 2910 static void plugin_gen_mem_callbacks(TCGv vaddr, MemOpIdx oi, 2911 enum qemu_plugin_mem_rw rw) 2912 { 2913 #ifdef CONFIG_PLUGIN 2914 if (tcg_ctx->plugin_insn != NULL) { 2915 qemu_plugin_meminfo_t info = make_plugin_meminfo(oi, rw); 2916 plugin_gen_empty_mem_callback(vaddr, info); 2917 tcg_temp_free(vaddr); 2918 } 2919 #endif 2920 } 2921 2922 void tcg_gen_qemu_ld_i32(TCGv_i32 val, TCGv addr, TCGArg idx, MemOp memop) 2923 { 2924 MemOp orig_memop; 2925 MemOpIdx oi; 2926 2927 tcg_gen_req_mo(TCG_MO_LD_LD | TCG_MO_ST_LD); 2928 memop = tcg_canonicalize_memop(memop, 0, 0); 2929 oi = make_memop_idx(memop, idx); 2930 2931 orig_memop = memop; 2932 if (!TCG_TARGET_HAS_MEMORY_BSWAP && (memop & MO_BSWAP)) { 2933 memop &= ~MO_BSWAP; 2934 /* The bswap primitive benefits from zero-extended input. */ 2935 if ((memop & MO_SSIZE) == MO_SW) { 2936 memop &= ~MO_SIGN; 2937 } 2938 } 2939 2940 addr = plugin_prep_mem_callbacks(addr); 2941 gen_ldst_i32(INDEX_op_qemu_ld_i32, val, addr, memop, idx); 2942 plugin_gen_mem_callbacks(addr, oi, QEMU_PLUGIN_MEM_R); 2943 2944 if ((orig_memop ^ memop) & MO_BSWAP) { 2945 switch (orig_memop & MO_SIZE) { 2946 case MO_16: 2947 tcg_gen_bswap16_i32(val, val, (orig_memop & MO_SIGN 2948 ? TCG_BSWAP_IZ | TCG_BSWAP_OS 2949 : TCG_BSWAP_IZ | TCG_BSWAP_OZ)); 2950 break; 2951 case MO_32: 2952 tcg_gen_bswap32_i32(val, val); 2953 break; 2954 default: 2955 g_assert_not_reached(); 2956 } 2957 } 2958 } 2959 2960 void tcg_gen_qemu_st_i32(TCGv_i32 val, TCGv addr, TCGArg idx, MemOp memop) 2961 { 2962 TCGv_i32 swap = NULL; 2963 MemOpIdx oi; 2964 2965 tcg_gen_req_mo(TCG_MO_LD_ST | TCG_MO_ST_ST); 2966 memop = tcg_canonicalize_memop(memop, 0, 1); 2967 oi = make_memop_idx(memop, idx); 2968 2969 if (!TCG_TARGET_HAS_MEMORY_BSWAP && (memop & MO_BSWAP)) { 2970 swap = tcg_temp_new_i32(); 2971 switch (memop & MO_SIZE) { 2972 case MO_16: 2973 tcg_gen_bswap16_i32(swap, val, 0); 2974 break; 2975 case MO_32: 2976 tcg_gen_bswap32_i32(swap, val); 2977 break; 2978 default: 2979 g_assert_not_reached(); 2980 } 2981 val = swap; 2982 memop &= ~MO_BSWAP; 2983 } 2984 2985 addr = plugin_prep_mem_callbacks(addr); 2986 if (TCG_TARGET_HAS_qemu_st8_i32 && (memop & MO_SIZE) == MO_8) { 2987 gen_ldst_i32(INDEX_op_qemu_st8_i32, val, addr, memop, idx); 2988 } else { 2989 gen_ldst_i32(INDEX_op_qemu_st_i32, val, addr, memop, idx); 2990 } 2991 plugin_gen_mem_callbacks(addr, oi, QEMU_PLUGIN_MEM_W); 2992 2993 if (swap) { 2994 tcg_temp_free_i32(swap); 2995 } 2996 } 2997 2998 void tcg_gen_qemu_ld_i64(TCGv_i64 val, TCGv addr, TCGArg idx, MemOp memop) 2999 { 3000 MemOp orig_memop; 3001 MemOpIdx oi; 3002 3003 if (TCG_TARGET_REG_BITS == 32 && (memop & MO_SIZE) < MO_64) { 3004 tcg_gen_qemu_ld_i32(TCGV_LOW(val), addr, idx, memop); 3005 if (memop & MO_SIGN) { 3006 tcg_gen_sari_i32(TCGV_HIGH(val), TCGV_LOW(val), 31); 3007 } else { 3008 tcg_gen_movi_i32(TCGV_HIGH(val), 0); 3009 } 3010 return; 3011 } 3012 3013 tcg_gen_req_mo(TCG_MO_LD_LD | TCG_MO_ST_LD); 3014 memop = tcg_canonicalize_memop(memop, 1, 0); 3015 oi = make_memop_idx(memop, idx); 3016 3017 orig_memop = memop; 3018 if (!TCG_TARGET_HAS_MEMORY_BSWAP && (memop & MO_BSWAP)) { 3019 memop &= ~MO_BSWAP; 3020 /* The bswap primitive benefits from zero-extended input. */ 3021 if ((memop & MO_SIGN) && (memop & MO_SIZE) < MO_64) { 3022 memop &= ~MO_SIGN; 3023 } 3024 } 3025 3026 addr = plugin_prep_mem_callbacks(addr); 3027 gen_ldst_i64(INDEX_op_qemu_ld_i64, val, addr, memop, idx); 3028 plugin_gen_mem_callbacks(addr, oi, QEMU_PLUGIN_MEM_R); 3029 3030 if ((orig_memop ^ memop) & MO_BSWAP) { 3031 int flags = (orig_memop & MO_SIGN 3032 ? TCG_BSWAP_IZ | TCG_BSWAP_OS 3033 : TCG_BSWAP_IZ | TCG_BSWAP_OZ); 3034 switch (orig_memop & MO_SIZE) { 3035 case MO_16: 3036 tcg_gen_bswap16_i64(val, val, flags); 3037 break; 3038 case MO_32: 3039 tcg_gen_bswap32_i64(val, val, flags); 3040 break; 3041 case MO_64: 3042 tcg_gen_bswap64_i64(val, val); 3043 break; 3044 default: 3045 g_assert_not_reached(); 3046 } 3047 } 3048 } 3049 3050 void tcg_gen_qemu_st_i64(TCGv_i64 val, TCGv addr, TCGArg idx, MemOp memop) 3051 { 3052 TCGv_i64 swap = NULL; 3053 MemOpIdx oi; 3054 3055 if (TCG_TARGET_REG_BITS == 32 && (memop & MO_SIZE) < MO_64) { 3056 tcg_gen_qemu_st_i32(TCGV_LOW(val), addr, idx, memop); 3057 return; 3058 } 3059 3060 tcg_gen_req_mo(TCG_MO_LD_ST | TCG_MO_ST_ST); 3061 memop = tcg_canonicalize_memop(memop, 1, 1); 3062 oi = make_memop_idx(memop, idx); 3063 3064 if (!TCG_TARGET_HAS_MEMORY_BSWAP && (memop & MO_BSWAP)) { 3065 swap = tcg_temp_new_i64(); 3066 switch (memop & MO_SIZE) { 3067 case MO_16: 3068 tcg_gen_bswap16_i64(swap, val, 0); 3069 break; 3070 case MO_32: 3071 tcg_gen_bswap32_i64(swap, val, 0); 3072 break; 3073 case MO_64: 3074 tcg_gen_bswap64_i64(swap, val); 3075 break; 3076 default: 3077 g_assert_not_reached(); 3078 } 3079 val = swap; 3080 memop &= ~MO_BSWAP; 3081 } 3082 3083 addr = plugin_prep_mem_callbacks(addr); 3084 gen_ldst_i64(INDEX_op_qemu_st_i64, val, addr, memop, idx); 3085 plugin_gen_mem_callbacks(addr, oi, QEMU_PLUGIN_MEM_W); 3086 3087 if (swap) { 3088 tcg_temp_free_i64(swap); 3089 } 3090 } 3091 3092 static void tcg_gen_ext_i32(TCGv_i32 ret, TCGv_i32 val, MemOp opc) 3093 { 3094 switch (opc & MO_SSIZE) { 3095 case MO_SB: 3096 tcg_gen_ext8s_i32(ret, val); 3097 break; 3098 case MO_UB: 3099 tcg_gen_ext8u_i32(ret, val); 3100 break; 3101 case MO_SW: 3102 tcg_gen_ext16s_i32(ret, val); 3103 break; 3104 case MO_UW: 3105 tcg_gen_ext16u_i32(ret, val); 3106 break; 3107 default: 3108 tcg_gen_mov_i32(ret, val); 3109 break; 3110 } 3111 } 3112 3113 static void tcg_gen_ext_i64(TCGv_i64 ret, TCGv_i64 val, MemOp opc) 3114 { 3115 switch (opc & MO_SSIZE) { 3116 case MO_SB: 3117 tcg_gen_ext8s_i64(ret, val); 3118 break; 3119 case MO_UB: 3120 tcg_gen_ext8u_i64(ret, val); 3121 break; 3122 case MO_SW: 3123 tcg_gen_ext16s_i64(ret, val); 3124 break; 3125 case MO_UW: 3126 tcg_gen_ext16u_i64(ret, val); 3127 break; 3128 case MO_SL: 3129 tcg_gen_ext32s_i64(ret, val); 3130 break; 3131 case MO_UL: 3132 tcg_gen_ext32u_i64(ret, val); 3133 break; 3134 default: 3135 tcg_gen_mov_i64(ret, val); 3136 break; 3137 } 3138 } 3139 3140 typedef void (*gen_atomic_cx_i32)(TCGv_i32, TCGv_env, TCGv, 3141 TCGv_i32, TCGv_i32, TCGv_i32); 3142 typedef void (*gen_atomic_cx_i64)(TCGv_i64, TCGv_env, TCGv, 3143 TCGv_i64, TCGv_i64, TCGv_i32); 3144 typedef void (*gen_atomic_op_i32)(TCGv_i32, TCGv_env, TCGv, 3145 TCGv_i32, TCGv_i32); 3146 typedef void (*gen_atomic_op_i64)(TCGv_i64, TCGv_env, TCGv, 3147 TCGv_i64, TCGv_i32); 3148 3149 #ifdef CONFIG_ATOMIC64 3150 # define WITH_ATOMIC64(X) X, 3151 #else 3152 # define WITH_ATOMIC64(X) 3153 #endif 3154 3155 static void * const table_cmpxchg[(MO_SIZE | MO_BSWAP) + 1] = { 3156 [MO_8] = gen_helper_atomic_cmpxchgb, 3157 [MO_16 | MO_LE] = gen_helper_atomic_cmpxchgw_le, 3158 [MO_16 | MO_BE] = gen_helper_atomic_cmpxchgw_be, 3159 [MO_32 | MO_LE] = gen_helper_atomic_cmpxchgl_le, 3160 [MO_32 | MO_BE] = gen_helper_atomic_cmpxchgl_be, 3161 WITH_ATOMIC64([MO_64 | MO_LE] = gen_helper_atomic_cmpxchgq_le) 3162 WITH_ATOMIC64([MO_64 | MO_BE] = gen_helper_atomic_cmpxchgq_be) 3163 }; 3164 3165 void tcg_gen_atomic_cmpxchg_i32(TCGv_i32 retv, TCGv addr, TCGv_i32 cmpv, 3166 TCGv_i32 newv, TCGArg idx, MemOp memop) 3167 { 3168 memop = tcg_canonicalize_memop(memop, 0, 0); 3169 3170 if (!(tcg_ctx->gen_tb->cflags & CF_PARALLEL)) { 3171 TCGv_i32 t1 = tcg_temp_new_i32(); 3172 TCGv_i32 t2 = tcg_temp_new_i32(); 3173 3174 tcg_gen_ext_i32(t2, cmpv, memop & MO_SIZE); 3175 3176 tcg_gen_qemu_ld_i32(t1, addr, idx, memop & ~MO_SIGN); 3177 tcg_gen_movcond_i32(TCG_COND_EQ, t2, t1, t2, newv, t1); 3178 tcg_gen_qemu_st_i32(t2, addr, idx, memop); 3179 tcg_temp_free_i32(t2); 3180 3181 if (memop & MO_SIGN) { 3182 tcg_gen_ext_i32(retv, t1, memop); 3183 } else { 3184 tcg_gen_mov_i32(retv, t1); 3185 } 3186 tcg_temp_free_i32(t1); 3187 } else { 3188 gen_atomic_cx_i32 gen; 3189 MemOpIdx oi; 3190 3191 gen = table_cmpxchg[memop & (MO_SIZE | MO_BSWAP)]; 3192 tcg_debug_assert(gen != NULL); 3193 3194 oi = make_memop_idx(memop & ~MO_SIGN, idx); 3195 gen(retv, cpu_env, addr, cmpv, newv, tcg_constant_i32(oi)); 3196 3197 if (memop & MO_SIGN) { 3198 tcg_gen_ext_i32(retv, retv, memop); 3199 } 3200 } 3201 } 3202 3203 void tcg_gen_atomic_cmpxchg_i64(TCGv_i64 retv, TCGv addr, TCGv_i64 cmpv, 3204 TCGv_i64 newv, TCGArg idx, MemOp memop) 3205 { 3206 memop = tcg_canonicalize_memop(memop, 1, 0); 3207 3208 if (!(tcg_ctx->gen_tb->cflags & CF_PARALLEL)) { 3209 TCGv_i64 t1 = tcg_temp_new_i64(); 3210 TCGv_i64 t2 = tcg_temp_new_i64(); 3211 3212 tcg_gen_ext_i64(t2, cmpv, memop & MO_SIZE); 3213 3214 tcg_gen_qemu_ld_i64(t1, addr, idx, memop & ~MO_SIGN); 3215 tcg_gen_movcond_i64(TCG_COND_EQ, t2, t1, t2, newv, t1); 3216 tcg_gen_qemu_st_i64(t2, addr, idx, memop); 3217 tcg_temp_free_i64(t2); 3218 3219 if (memop & MO_SIGN) { 3220 tcg_gen_ext_i64(retv, t1, memop); 3221 } else { 3222 tcg_gen_mov_i64(retv, t1); 3223 } 3224 tcg_temp_free_i64(t1); 3225 } else if ((memop & MO_SIZE) == MO_64) { 3226 #ifdef CONFIG_ATOMIC64 3227 gen_atomic_cx_i64 gen; 3228 MemOpIdx oi; 3229 3230 gen = table_cmpxchg[memop & (MO_SIZE | MO_BSWAP)]; 3231 tcg_debug_assert(gen != NULL); 3232 3233 oi = make_memop_idx(memop, idx); 3234 gen(retv, cpu_env, addr, cmpv, newv, tcg_constant_i32(oi)); 3235 #else 3236 gen_helper_exit_atomic(cpu_env); 3237 /* Produce a result, so that we have a well-formed opcode stream 3238 with respect to uses of the result in the (dead) code following. */ 3239 tcg_gen_movi_i64(retv, 0); 3240 #endif /* CONFIG_ATOMIC64 */ 3241 } else { 3242 TCGv_i32 c32 = tcg_temp_new_i32(); 3243 TCGv_i32 n32 = tcg_temp_new_i32(); 3244 TCGv_i32 r32 = tcg_temp_new_i32(); 3245 3246 tcg_gen_extrl_i64_i32(c32, cmpv); 3247 tcg_gen_extrl_i64_i32(n32, newv); 3248 tcg_gen_atomic_cmpxchg_i32(r32, addr, c32, n32, idx, memop & ~MO_SIGN); 3249 tcg_temp_free_i32(c32); 3250 tcg_temp_free_i32(n32); 3251 3252 tcg_gen_extu_i32_i64(retv, r32); 3253 tcg_temp_free_i32(r32); 3254 3255 if (memop & MO_SIGN) { 3256 tcg_gen_ext_i64(retv, retv, memop); 3257 } 3258 } 3259 } 3260 3261 static void do_nonatomic_op_i32(TCGv_i32 ret, TCGv addr, TCGv_i32 val, 3262 TCGArg idx, MemOp memop, bool new_val, 3263 void (*gen)(TCGv_i32, TCGv_i32, TCGv_i32)) 3264 { 3265 TCGv_i32 t1 = tcg_temp_new_i32(); 3266 TCGv_i32 t2 = tcg_temp_new_i32(); 3267 3268 memop = tcg_canonicalize_memop(memop, 0, 0); 3269 3270 tcg_gen_qemu_ld_i32(t1, addr, idx, memop); 3271 tcg_gen_ext_i32(t2, val, memop); 3272 gen(t2, t1, t2); 3273 tcg_gen_qemu_st_i32(t2, addr, idx, memop); 3274 3275 tcg_gen_ext_i32(ret, (new_val ? t2 : t1), memop); 3276 tcg_temp_free_i32(t1); 3277 tcg_temp_free_i32(t2); 3278 } 3279 3280 static void do_atomic_op_i32(TCGv_i32 ret, TCGv addr, TCGv_i32 val, 3281 TCGArg idx, MemOp memop, void * const table[]) 3282 { 3283 gen_atomic_op_i32 gen; 3284 MemOpIdx oi; 3285 3286 memop = tcg_canonicalize_memop(memop, 0, 0); 3287 3288 gen = table[memop & (MO_SIZE | MO_BSWAP)]; 3289 tcg_debug_assert(gen != NULL); 3290 3291 oi = make_memop_idx(memop & ~MO_SIGN, idx); 3292 gen(ret, cpu_env, addr, val, tcg_constant_i32(oi)); 3293 3294 if (memop & MO_SIGN) { 3295 tcg_gen_ext_i32(ret, ret, memop); 3296 } 3297 } 3298 3299 static void do_nonatomic_op_i64(TCGv_i64 ret, TCGv addr, TCGv_i64 val, 3300 TCGArg idx, MemOp memop, bool new_val, 3301 void (*gen)(TCGv_i64, TCGv_i64, TCGv_i64)) 3302 { 3303 TCGv_i64 t1 = tcg_temp_new_i64(); 3304 TCGv_i64 t2 = tcg_temp_new_i64(); 3305 3306 memop = tcg_canonicalize_memop(memop, 1, 0); 3307 3308 tcg_gen_qemu_ld_i64(t1, addr, idx, memop); 3309 tcg_gen_ext_i64(t2, val, memop); 3310 gen(t2, t1, t2); 3311 tcg_gen_qemu_st_i64(t2, addr, idx, memop); 3312 3313 tcg_gen_ext_i64(ret, (new_val ? t2 : t1), memop); 3314 tcg_temp_free_i64(t1); 3315 tcg_temp_free_i64(t2); 3316 } 3317 3318 static void do_atomic_op_i64(TCGv_i64 ret, TCGv addr, TCGv_i64 val, 3319 TCGArg idx, MemOp memop, void * const table[]) 3320 { 3321 memop = tcg_canonicalize_memop(memop, 1, 0); 3322 3323 if ((memop & MO_SIZE) == MO_64) { 3324 #ifdef CONFIG_ATOMIC64 3325 gen_atomic_op_i64 gen; 3326 MemOpIdx oi; 3327 3328 gen = table[memop & (MO_SIZE | MO_BSWAP)]; 3329 tcg_debug_assert(gen != NULL); 3330 3331 oi = make_memop_idx(memop & ~MO_SIGN, idx); 3332 gen(ret, cpu_env, addr, val, tcg_constant_i32(oi)); 3333 #else 3334 gen_helper_exit_atomic(cpu_env); 3335 /* Produce a result, so that we have a well-formed opcode stream 3336 with respect to uses of the result in the (dead) code following. */ 3337 tcg_gen_movi_i64(ret, 0); 3338 #endif /* CONFIG_ATOMIC64 */ 3339 } else { 3340 TCGv_i32 v32 = tcg_temp_new_i32(); 3341 TCGv_i32 r32 = tcg_temp_new_i32(); 3342 3343 tcg_gen_extrl_i64_i32(v32, val); 3344 do_atomic_op_i32(r32, addr, v32, idx, memop & ~MO_SIGN, table); 3345 tcg_temp_free_i32(v32); 3346 3347 tcg_gen_extu_i32_i64(ret, r32); 3348 tcg_temp_free_i32(r32); 3349 3350 if (memop & MO_SIGN) { 3351 tcg_gen_ext_i64(ret, ret, memop); 3352 } 3353 } 3354 } 3355 3356 #define GEN_ATOMIC_HELPER(NAME, OP, NEW) \ 3357 static void * const table_##NAME[(MO_SIZE | MO_BSWAP) + 1] = { \ 3358 [MO_8] = gen_helper_atomic_##NAME##b, \ 3359 [MO_16 | MO_LE] = gen_helper_atomic_##NAME##w_le, \ 3360 [MO_16 | MO_BE] = gen_helper_atomic_##NAME##w_be, \ 3361 [MO_32 | MO_LE] = gen_helper_atomic_##NAME##l_le, \ 3362 [MO_32 | MO_BE] = gen_helper_atomic_##NAME##l_be, \ 3363 WITH_ATOMIC64([MO_64 | MO_LE] = gen_helper_atomic_##NAME##q_le) \ 3364 WITH_ATOMIC64([MO_64 | MO_BE] = gen_helper_atomic_##NAME##q_be) \ 3365 }; \ 3366 void tcg_gen_atomic_##NAME##_i32 \ 3367 (TCGv_i32 ret, TCGv addr, TCGv_i32 val, TCGArg idx, MemOp memop) \ 3368 { \ 3369 if (tcg_ctx->gen_tb->cflags & CF_PARALLEL) { \ 3370 do_atomic_op_i32(ret, addr, val, idx, memop, table_##NAME); \ 3371 } else { \ 3372 do_nonatomic_op_i32(ret, addr, val, idx, memop, NEW, \ 3373 tcg_gen_##OP##_i32); \ 3374 } \ 3375 } \ 3376 void tcg_gen_atomic_##NAME##_i64 \ 3377 (TCGv_i64 ret, TCGv addr, TCGv_i64 val, TCGArg idx, MemOp memop) \ 3378 { \ 3379 if (tcg_ctx->gen_tb->cflags & CF_PARALLEL) { \ 3380 do_atomic_op_i64(ret, addr, val, idx, memop, table_##NAME); \ 3381 } else { \ 3382 do_nonatomic_op_i64(ret, addr, val, idx, memop, NEW, \ 3383 tcg_gen_##OP##_i64); \ 3384 } \ 3385 } 3386 3387 GEN_ATOMIC_HELPER(fetch_add, add, 0) 3388 GEN_ATOMIC_HELPER(fetch_and, and, 0) 3389 GEN_ATOMIC_HELPER(fetch_or, or, 0) 3390 GEN_ATOMIC_HELPER(fetch_xor, xor, 0) 3391 GEN_ATOMIC_HELPER(fetch_smin, smin, 0) 3392 GEN_ATOMIC_HELPER(fetch_umin, umin, 0) 3393 GEN_ATOMIC_HELPER(fetch_smax, smax, 0) 3394 GEN_ATOMIC_HELPER(fetch_umax, umax, 0) 3395 3396 GEN_ATOMIC_HELPER(add_fetch, add, 1) 3397 GEN_ATOMIC_HELPER(and_fetch, and, 1) 3398 GEN_ATOMIC_HELPER(or_fetch, or, 1) 3399 GEN_ATOMIC_HELPER(xor_fetch, xor, 1) 3400 GEN_ATOMIC_HELPER(smin_fetch, smin, 1) 3401 GEN_ATOMIC_HELPER(umin_fetch, umin, 1) 3402 GEN_ATOMIC_HELPER(smax_fetch, smax, 1) 3403 GEN_ATOMIC_HELPER(umax_fetch, umax, 1) 3404 3405 static void tcg_gen_mov2_i32(TCGv_i32 r, TCGv_i32 a, TCGv_i32 b) 3406 { 3407 tcg_gen_mov_i32(r, b); 3408 } 3409 3410 static void tcg_gen_mov2_i64(TCGv_i64 r, TCGv_i64 a, TCGv_i64 b) 3411 { 3412 tcg_gen_mov_i64(r, b); 3413 } 3414 3415 GEN_ATOMIC_HELPER(xchg, mov2, 0) 3416 3417 #undef GEN_ATOMIC_HELPER 3418