1 /* 2 * Tiny Code Generator for QEMU 3 * 4 * Copyright (c) 2008 Fabrice Bellard 5 * 6 * Permission is hereby granted, free of charge, to any person obtaining a copy 7 * of this software and associated documentation files (the "Software"), to deal 8 * in the Software without restriction, including without limitation the rights 9 * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 10 * copies of the Software, and to permit persons to whom the Software is 11 * furnished to do so, subject to the following conditions: 12 * 13 * The above copyright notice and this permission notice shall be included in 14 * all copies or substantial portions of the Software. 15 * 16 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 17 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 18 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL 19 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 20 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 21 * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN 22 * THE SOFTWARE. 23 */ 24 25 #include "qemu/osdep.h" 26 #include "exec/exec-all.h" 27 #include "tcg/tcg.h" 28 #include "tcg/tcg-op.h" 29 #include "tcg/tcg-mo.h" 30 #include "exec/plugin-gen.h" 31 #include "tcg-internal.h" 32 33 34 void tcg_gen_op1(TCGOpcode opc, TCGArg a1) 35 { 36 TCGOp *op = tcg_emit_op(opc, 1); 37 op->args[0] = a1; 38 } 39 40 void tcg_gen_op2(TCGOpcode opc, TCGArg a1, TCGArg a2) 41 { 42 TCGOp *op = tcg_emit_op(opc, 2); 43 op->args[0] = a1; 44 op->args[1] = a2; 45 } 46 47 void tcg_gen_op3(TCGOpcode opc, TCGArg a1, TCGArg a2, TCGArg a3) 48 { 49 TCGOp *op = tcg_emit_op(opc, 3); 50 op->args[0] = a1; 51 op->args[1] = a2; 52 op->args[2] = a3; 53 } 54 55 void tcg_gen_op4(TCGOpcode opc, TCGArg a1, TCGArg a2, TCGArg a3, TCGArg a4) 56 { 57 TCGOp *op = tcg_emit_op(opc, 4); 58 op->args[0] = a1; 59 op->args[1] = a2; 60 op->args[2] = a3; 61 op->args[3] = a4; 62 } 63 64 void tcg_gen_op5(TCGOpcode opc, TCGArg a1, TCGArg a2, TCGArg a3, 65 TCGArg a4, TCGArg a5) 66 { 67 TCGOp *op = tcg_emit_op(opc, 5); 68 op->args[0] = a1; 69 op->args[1] = a2; 70 op->args[2] = a3; 71 op->args[3] = a4; 72 op->args[4] = a5; 73 } 74 75 void tcg_gen_op6(TCGOpcode opc, TCGArg a1, TCGArg a2, TCGArg a3, 76 TCGArg a4, TCGArg a5, TCGArg a6) 77 { 78 TCGOp *op = tcg_emit_op(opc, 6); 79 op->args[0] = a1; 80 op->args[1] = a2; 81 op->args[2] = a3; 82 op->args[3] = a4; 83 op->args[4] = a5; 84 op->args[5] = a6; 85 } 86 87 /* Generic ops. */ 88 89 static void add_last_as_label_use(TCGLabel *l) 90 { 91 TCGLabelUse *u = tcg_malloc(sizeof(TCGLabelUse)); 92 93 u->op = tcg_last_op(); 94 QSIMPLEQ_INSERT_TAIL(&l->branches, u, next); 95 } 96 97 void tcg_gen_br(TCGLabel *l) 98 { 99 tcg_gen_op1(INDEX_op_br, label_arg(l)); 100 add_last_as_label_use(l); 101 } 102 103 void tcg_gen_mb(TCGBar mb_type) 104 { 105 if (tcg_ctx->gen_tb->cflags & CF_PARALLEL) { 106 tcg_gen_op1(INDEX_op_mb, mb_type); 107 } 108 } 109 110 /* 32 bit ops */ 111 112 void tcg_gen_movi_i32(TCGv_i32 ret, int32_t arg) 113 { 114 tcg_gen_mov_i32(ret, tcg_constant_i32(arg)); 115 } 116 117 void tcg_gen_addi_i32(TCGv_i32 ret, TCGv_i32 arg1, int32_t arg2) 118 { 119 /* some cases can be optimized here */ 120 if (arg2 == 0) { 121 tcg_gen_mov_i32(ret, arg1); 122 } else { 123 tcg_gen_add_i32(ret, arg1, tcg_constant_i32(arg2)); 124 } 125 } 126 127 void tcg_gen_subfi_i32(TCGv_i32 ret, int32_t arg1, TCGv_i32 arg2) 128 { 129 if (arg1 == 0 && TCG_TARGET_HAS_neg_i32) { 130 /* Don't recurse with tcg_gen_neg_i32. */ 131 tcg_gen_op2_i32(INDEX_op_neg_i32, ret, arg2); 132 } else { 133 tcg_gen_sub_i32(ret, tcg_constant_i32(arg1), arg2); 134 } 135 } 136 137 void tcg_gen_subi_i32(TCGv_i32 ret, TCGv_i32 arg1, int32_t arg2) 138 { 139 /* some cases can be optimized here */ 140 if (arg2 == 0) { 141 tcg_gen_mov_i32(ret, arg1); 142 } else { 143 tcg_gen_sub_i32(ret, arg1, tcg_constant_i32(arg2)); 144 } 145 } 146 147 void tcg_gen_andi_i32(TCGv_i32 ret, TCGv_i32 arg1, int32_t arg2) 148 { 149 /* Some cases can be optimized here. */ 150 switch (arg2) { 151 case 0: 152 tcg_gen_movi_i32(ret, 0); 153 return; 154 case -1: 155 tcg_gen_mov_i32(ret, arg1); 156 return; 157 case 0xff: 158 /* Don't recurse with tcg_gen_ext8u_i32. */ 159 if (TCG_TARGET_HAS_ext8u_i32) { 160 tcg_gen_op2_i32(INDEX_op_ext8u_i32, ret, arg1); 161 return; 162 } 163 break; 164 case 0xffff: 165 if (TCG_TARGET_HAS_ext16u_i32) { 166 tcg_gen_op2_i32(INDEX_op_ext16u_i32, ret, arg1); 167 return; 168 } 169 break; 170 } 171 172 tcg_gen_and_i32(ret, arg1, tcg_constant_i32(arg2)); 173 } 174 175 void tcg_gen_ori_i32(TCGv_i32 ret, TCGv_i32 arg1, int32_t arg2) 176 { 177 /* Some cases can be optimized here. */ 178 if (arg2 == -1) { 179 tcg_gen_movi_i32(ret, -1); 180 } else if (arg2 == 0) { 181 tcg_gen_mov_i32(ret, arg1); 182 } else { 183 tcg_gen_or_i32(ret, arg1, tcg_constant_i32(arg2)); 184 } 185 } 186 187 void tcg_gen_xori_i32(TCGv_i32 ret, TCGv_i32 arg1, int32_t arg2) 188 { 189 /* Some cases can be optimized here. */ 190 if (arg2 == 0) { 191 tcg_gen_mov_i32(ret, arg1); 192 } else if (arg2 == -1 && TCG_TARGET_HAS_not_i32) { 193 /* Don't recurse with tcg_gen_not_i32. */ 194 tcg_gen_op2_i32(INDEX_op_not_i32, ret, arg1); 195 } else { 196 tcg_gen_xor_i32(ret, arg1, tcg_constant_i32(arg2)); 197 } 198 } 199 200 void tcg_gen_shli_i32(TCGv_i32 ret, TCGv_i32 arg1, int32_t arg2) 201 { 202 tcg_debug_assert(arg2 >= 0 && arg2 < 32); 203 if (arg2 == 0) { 204 tcg_gen_mov_i32(ret, arg1); 205 } else { 206 tcg_gen_shl_i32(ret, arg1, tcg_constant_i32(arg2)); 207 } 208 } 209 210 void tcg_gen_shri_i32(TCGv_i32 ret, TCGv_i32 arg1, int32_t arg2) 211 { 212 tcg_debug_assert(arg2 >= 0 && arg2 < 32); 213 if (arg2 == 0) { 214 tcg_gen_mov_i32(ret, arg1); 215 } else { 216 tcg_gen_shr_i32(ret, arg1, tcg_constant_i32(arg2)); 217 } 218 } 219 220 void tcg_gen_sari_i32(TCGv_i32 ret, TCGv_i32 arg1, int32_t arg2) 221 { 222 tcg_debug_assert(arg2 >= 0 && arg2 < 32); 223 if (arg2 == 0) { 224 tcg_gen_mov_i32(ret, arg1); 225 } else { 226 tcg_gen_sar_i32(ret, arg1, tcg_constant_i32(arg2)); 227 } 228 } 229 230 void tcg_gen_brcond_i32(TCGCond cond, TCGv_i32 arg1, TCGv_i32 arg2, TCGLabel *l) 231 { 232 if (cond == TCG_COND_ALWAYS) { 233 tcg_gen_br(l); 234 } else if (cond != TCG_COND_NEVER) { 235 tcg_gen_op4ii_i32(INDEX_op_brcond_i32, arg1, arg2, cond, label_arg(l)); 236 add_last_as_label_use(l); 237 } 238 } 239 240 void tcg_gen_brcondi_i32(TCGCond cond, TCGv_i32 arg1, int32_t arg2, TCGLabel *l) 241 { 242 if (cond == TCG_COND_ALWAYS) { 243 tcg_gen_br(l); 244 } else if (cond != TCG_COND_NEVER) { 245 tcg_gen_brcond_i32(cond, arg1, tcg_constant_i32(arg2), l); 246 } 247 } 248 249 void tcg_gen_setcond_i32(TCGCond cond, TCGv_i32 ret, 250 TCGv_i32 arg1, TCGv_i32 arg2) 251 { 252 if (cond == TCG_COND_ALWAYS) { 253 tcg_gen_movi_i32(ret, 1); 254 } else if (cond == TCG_COND_NEVER) { 255 tcg_gen_movi_i32(ret, 0); 256 } else { 257 tcg_gen_op4i_i32(INDEX_op_setcond_i32, ret, arg1, arg2, cond); 258 } 259 } 260 261 void tcg_gen_setcondi_i32(TCGCond cond, TCGv_i32 ret, 262 TCGv_i32 arg1, int32_t arg2) 263 { 264 tcg_gen_setcond_i32(cond, ret, arg1, tcg_constant_i32(arg2)); 265 } 266 267 void tcg_gen_muli_i32(TCGv_i32 ret, TCGv_i32 arg1, int32_t arg2) 268 { 269 if (arg2 == 0) { 270 tcg_gen_movi_i32(ret, 0); 271 } else if (is_power_of_2(arg2)) { 272 tcg_gen_shli_i32(ret, arg1, ctz32(arg2)); 273 } else { 274 tcg_gen_mul_i32(ret, arg1, tcg_constant_i32(arg2)); 275 } 276 } 277 278 void tcg_gen_div_i32(TCGv_i32 ret, TCGv_i32 arg1, TCGv_i32 arg2) 279 { 280 if (TCG_TARGET_HAS_div_i32) { 281 tcg_gen_op3_i32(INDEX_op_div_i32, ret, arg1, arg2); 282 } else if (TCG_TARGET_HAS_div2_i32) { 283 TCGv_i32 t0 = tcg_temp_ebb_new_i32(); 284 tcg_gen_sari_i32(t0, arg1, 31); 285 tcg_gen_op5_i32(INDEX_op_div2_i32, ret, t0, arg1, t0, arg2); 286 tcg_temp_free_i32(t0); 287 } else { 288 gen_helper_div_i32(ret, arg1, arg2); 289 } 290 } 291 292 void tcg_gen_rem_i32(TCGv_i32 ret, TCGv_i32 arg1, TCGv_i32 arg2) 293 { 294 if (TCG_TARGET_HAS_rem_i32) { 295 tcg_gen_op3_i32(INDEX_op_rem_i32, ret, arg1, arg2); 296 } else if (TCG_TARGET_HAS_div_i32) { 297 TCGv_i32 t0 = tcg_temp_ebb_new_i32(); 298 tcg_gen_op3_i32(INDEX_op_div_i32, t0, arg1, arg2); 299 tcg_gen_mul_i32(t0, t0, arg2); 300 tcg_gen_sub_i32(ret, arg1, t0); 301 tcg_temp_free_i32(t0); 302 } else if (TCG_TARGET_HAS_div2_i32) { 303 TCGv_i32 t0 = tcg_temp_ebb_new_i32(); 304 tcg_gen_sari_i32(t0, arg1, 31); 305 tcg_gen_op5_i32(INDEX_op_div2_i32, t0, ret, arg1, t0, arg2); 306 tcg_temp_free_i32(t0); 307 } else { 308 gen_helper_rem_i32(ret, arg1, arg2); 309 } 310 } 311 312 void tcg_gen_divu_i32(TCGv_i32 ret, TCGv_i32 arg1, TCGv_i32 arg2) 313 { 314 if (TCG_TARGET_HAS_div_i32) { 315 tcg_gen_op3_i32(INDEX_op_divu_i32, ret, arg1, arg2); 316 } else if (TCG_TARGET_HAS_div2_i32) { 317 TCGv_i32 t0 = tcg_temp_ebb_new_i32(); 318 tcg_gen_movi_i32(t0, 0); 319 tcg_gen_op5_i32(INDEX_op_divu2_i32, ret, t0, arg1, t0, arg2); 320 tcg_temp_free_i32(t0); 321 } else { 322 gen_helper_divu_i32(ret, arg1, arg2); 323 } 324 } 325 326 void tcg_gen_remu_i32(TCGv_i32 ret, TCGv_i32 arg1, TCGv_i32 arg2) 327 { 328 if (TCG_TARGET_HAS_rem_i32) { 329 tcg_gen_op3_i32(INDEX_op_remu_i32, ret, arg1, arg2); 330 } else if (TCG_TARGET_HAS_div_i32) { 331 TCGv_i32 t0 = tcg_temp_ebb_new_i32(); 332 tcg_gen_op3_i32(INDEX_op_divu_i32, t0, arg1, arg2); 333 tcg_gen_mul_i32(t0, t0, arg2); 334 tcg_gen_sub_i32(ret, arg1, t0); 335 tcg_temp_free_i32(t0); 336 } else if (TCG_TARGET_HAS_div2_i32) { 337 TCGv_i32 t0 = tcg_temp_ebb_new_i32(); 338 tcg_gen_movi_i32(t0, 0); 339 tcg_gen_op5_i32(INDEX_op_divu2_i32, t0, ret, arg1, t0, arg2); 340 tcg_temp_free_i32(t0); 341 } else { 342 gen_helper_remu_i32(ret, arg1, arg2); 343 } 344 } 345 346 void tcg_gen_andc_i32(TCGv_i32 ret, TCGv_i32 arg1, TCGv_i32 arg2) 347 { 348 if (TCG_TARGET_HAS_andc_i32) { 349 tcg_gen_op3_i32(INDEX_op_andc_i32, ret, arg1, arg2); 350 } else { 351 TCGv_i32 t0 = tcg_temp_ebb_new_i32(); 352 tcg_gen_not_i32(t0, arg2); 353 tcg_gen_and_i32(ret, arg1, t0); 354 tcg_temp_free_i32(t0); 355 } 356 } 357 358 void tcg_gen_eqv_i32(TCGv_i32 ret, TCGv_i32 arg1, TCGv_i32 arg2) 359 { 360 if (TCG_TARGET_HAS_eqv_i32) { 361 tcg_gen_op3_i32(INDEX_op_eqv_i32, ret, arg1, arg2); 362 } else { 363 tcg_gen_xor_i32(ret, arg1, arg2); 364 tcg_gen_not_i32(ret, ret); 365 } 366 } 367 368 void tcg_gen_nand_i32(TCGv_i32 ret, TCGv_i32 arg1, TCGv_i32 arg2) 369 { 370 if (TCG_TARGET_HAS_nand_i32) { 371 tcg_gen_op3_i32(INDEX_op_nand_i32, ret, arg1, arg2); 372 } else { 373 tcg_gen_and_i32(ret, arg1, arg2); 374 tcg_gen_not_i32(ret, ret); 375 } 376 } 377 378 void tcg_gen_nor_i32(TCGv_i32 ret, TCGv_i32 arg1, TCGv_i32 arg2) 379 { 380 if (TCG_TARGET_HAS_nor_i32) { 381 tcg_gen_op3_i32(INDEX_op_nor_i32, ret, arg1, arg2); 382 } else { 383 tcg_gen_or_i32(ret, arg1, arg2); 384 tcg_gen_not_i32(ret, ret); 385 } 386 } 387 388 void tcg_gen_orc_i32(TCGv_i32 ret, TCGv_i32 arg1, TCGv_i32 arg2) 389 { 390 if (TCG_TARGET_HAS_orc_i32) { 391 tcg_gen_op3_i32(INDEX_op_orc_i32, ret, arg1, arg2); 392 } else { 393 TCGv_i32 t0 = tcg_temp_ebb_new_i32(); 394 tcg_gen_not_i32(t0, arg2); 395 tcg_gen_or_i32(ret, arg1, t0); 396 tcg_temp_free_i32(t0); 397 } 398 } 399 400 void tcg_gen_clz_i32(TCGv_i32 ret, TCGv_i32 arg1, TCGv_i32 arg2) 401 { 402 if (TCG_TARGET_HAS_clz_i32) { 403 tcg_gen_op3_i32(INDEX_op_clz_i32, ret, arg1, arg2); 404 } else if (TCG_TARGET_HAS_clz_i64) { 405 TCGv_i64 t1 = tcg_temp_ebb_new_i64(); 406 TCGv_i64 t2 = tcg_temp_ebb_new_i64(); 407 tcg_gen_extu_i32_i64(t1, arg1); 408 tcg_gen_extu_i32_i64(t2, arg2); 409 tcg_gen_addi_i64(t2, t2, 32); 410 tcg_gen_clz_i64(t1, t1, t2); 411 tcg_gen_extrl_i64_i32(ret, t1); 412 tcg_temp_free_i64(t1); 413 tcg_temp_free_i64(t2); 414 tcg_gen_subi_i32(ret, ret, 32); 415 } else { 416 gen_helper_clz_i32(ret, arg1, arg2); 417 } 418 } 419 420 void tcg_gen_clzi_i32(TCGv_i32 ret, TCGv_i32 arg1, uint32_t arg2) 421 { 422 tcg_gen_clz_i32(ret, arg1, tcg_constant_i32(arg2)); 423 } 424 425 void tcg_gen_ctz_i32(TCGv_i32 ret, TCGv_i32 arg1, TCGv_i32 arg2) 426 { 427 if (TCG_TARGET_HAS_ctz_i32) { 428 tcg_gen_op3_i32(INDEX_op_ctz_i32, ret, arg1, arg2); 429 } else if (TCG_TARGET_HAS_ctz_i64) { 430 TCGv_i64 t1 = tcg_temp_ebb_new_i64(); 431 TCGv_i64 t2 = tcg_temp_ebb_new_i64(); 432 tcg_gen_extu_i32_i64(t1, arg1); 433 tcg_gen_extu_i32_i64(t2, arg2); 434 tcg_gen_ctz_i64(t1, t1, t2); 435 tcg_gen_extrl_i64_i32(ret, t1); 436 tcg_temp_free_i64(t1); 437 tcg_temp_free_i64(t2); 438 } else if (TCG_TARGET_HAS_ctpop_i32 439 || TCG_TARGET_HAS_ctpop_i64 440 || TCG_TARGET_HAS_clz_i32 441 || TCG_TARGET_HAS_clz_i64) { 442 TCGv_i32 z, t = tcg_temp_ebb_new_i32(); 443 444 if (TCG_TARGET_HAS_ctpop_i32 || TCG_TARGET_HAS_ctpop_i64) { 445 tcg_gen_subi_i32(t, arg1, 1); 446 tcg_gen_andc_i32(t, t, arg1); 447 tcg_gen_ctpop_i32(t, t); 448 } else { 449 /* Since all non-x86 hosts have clz(0) == 32, don't fight it. */ 450 tcg_gen_neg_i32(t, arg1); 451 tcg_gen_and_i32(t, t, arg1); 452 tcg_gen_clzi_i32(t, t, 32); 453 tcg_gen_xori_i32(t, t, 31); 454 } 455 z = tcg_constant_i32(0); 456 tcg_gen_movcond_i32(TCG_COND_EQ, ret, arg1, z, arg2, t); 457 tcg_temp_free_i32(t); 458 } else { 459 gen_helper_ctz_i32(ret, arg1, arg2); 460 } 461 } 462 463 void tcg_gen_ctzi_i32(TCGv_i32 ret, TCGv_i32 arg1, uint32_t arg2) 464 { 465 if (!TCG_TARGET_HAS_ctz_i32 && TCG_TARGET_HAS_ctpop_i32 && arg2 == 32) { 466 /* This equivalence has the advantage of not requiring a fixup. */ 467 TCGv_i32 t = tcg_temp_ebb_new_i32(); 468 tcg_gen_subi_i32(t, arg1, 1); 469 tcg_gen_andc_i32(t, t, arg1); 470 tcg_gen_ctpop_i32(ret, t); 471 tcg_temp_free_i32(t); 472 } else { 473 tcg_gen_ctz_i32(ret, arg1, tcg_constant_i32(arg2)); 474 } 475 } 476 477 void tcg_gen_clrsb_i32(TCGv_i32 ret, TCGv_i32 arg) 478 { 479 if (TCG_TARGET_HAS_clz_i32) { 480 TCGv_i32 t = tcg_temp_ebb_new_i32(); 481 tcg_gen_sari_i32(t, arg, 31); 482 tcg_gen_xor_i32(t, t, arg); 483 tcg_gen_clzi_i32(t, t, 32); 484 tcg_gen_subi_i32(ret, t, 1); 485 tcg_temp_free_i32(t); 486 } else { 487 gen_helper_clrsb_i32(ret, arg); 488 } 489 } 490 491 void tcg_gen_ctpop_i32(TCGv_i32 ret, TCGv_i32 arg1) 492 { 493 if (TCG_TARGET_HAS_ctpop_i32) { 494 tcg_gen_op2_i32(INDEX_op_ctpop_i32, ret, arg1); 495 } else if (TCG_TARGET_HAS_ctpop_i64) { 496 TCGv_i64 t = tcg_temp_ebb_new_i64(); 497 tcg_gen_extu_i32_i64(t, arg1); 498 tcg_gen_ctpop_i64(t, t); 499 tcg_gen_extrl_i64_i32(ret, t); 500 tcg_temp_free_i64(t); 501 } else { 502 gen_helper_ctpop_i32(ret, arg1); 503 } 504 } 505 506 void tcg_gen_rotl_i32(TCGv_i32 ret, TCGv_i32 arg1, TCGv_i32 arg2) 507 { 508 if (TCG_TARGET_HAS_rot_i32) { 509 tcg_gen_op3_i32(INDEX_op_rotl_i32, ret, arg1, arg2); 510 } else { 511 TCGv_i32 t0, t1; 512 513 t0 = tcg_temp_ebb_new_i32(); 514 t1 = tcg_temp_ebb_new_i32(); 515 tcg_gen_shl_i32(t0, arg1, arg2); 516 tcg_gen_subfi_i32(t1, 32, arg2); 517 tcg_gen_shr_i32(t1, arg1, t1); 518 tcg_gen_or_i32(ret, t0, t1); 519 tcg_temp_free_i32(t0); 520 tcg_temp_free_i32(t1); 521 } 522 } 523 524 void tcg_gen_rotli_i32(TCGv_i32 ret, TCGv_i32 arg1, int32_t arg2) 525 { 526 tcg_debug_assert(arg2 >= 0 && arg2 < 32); 527 /* some cases can be optimized here */ 528 if (arg2 == 0) { 529 tcg_gen_mov_i32(ret, arg1); 530 } else if (TCG_TARGET_HAS_rot_i32) { 531 tcg_gen_rotl_i32(ret, arg1, tcg_constant_i32(arg2)); 532 } else { 533 TCGv_i32 t0, t1; 534 t0 = tcg_temp_ebb_new_i32(); 535 t1 = tcg_temp_ebb_new_i32(); 536 tcg_gen_shli_i32(t0, arg1, arg2); 537 tcg_gen_shri_i32(t1, arg1, 32 - arg2); 538 tcg_gen_or_i32(ret, t0, t1); 539 tcg_temp_free_i32(t0); 540 tcg_temp_free_i32(t1); 541 } 542 } 543 544 void tcg_gen_rotr_i32(TCGv_i32 ret, TCGv_i32 arg1, TCGv_i32 arg2) 545 { 546 if (TCG_TARGET_HAS_rot_i32) { 547 tcg_gen_op3_i32(INDEX_op_rotr_i32, ret, arg1, arg2); 548 } else { 549 TCGv_i32 t0, t1; 550 551 t0 = tcg_temp_ebb_new_i32(); 552 t1 = tcg_temp_ebb_new_i32(); 553 tcg_gen_shr_i32(t0, arg1, arg2); 554 tcg_gen_subfi_i32(t1, 32, arg2); 555 tcg_gen_shl_i32(t1, arg1, t1); 556 tcg_gen_or_i32(ret, t0, t1); 557 tcg_temp_free_i32(t0); 558 tcg_temp_free_i32(t1); 559 } 560 } 561 562 void tcg_gen_rotri_i32(TCGv_i32 ret, TCGv_i32 arg1, int32_t arg2) 563 { 564 tcg_debug_assert(arg2 >= 0 && arg2 < 32); 565 /* some cases can be optimized here */ 566 if (arg2 == 0) { 567 tcg_gen_mov_i32(ret, arg1); 568 } else { 569 tcg_gen_rotli_i32(ret, arg1, 32 - arg2); 570 } 571 } 572 573 void tcg_gen_deposit_i32(TCGv_i32 ret, TCGv_i32 arg1, TCGv_i32 arg2, 574 unsigned int ofs, unsigned int len) 575 { 576 uint32_t mask; 577 TCGv_i32 t1; 578 579 tcg_debug_assert(ofs < 32); 580 tcg_debug_assert(len > 0); 581 tcg_debug_assert(len <= 32); 582 tcg_debug_assert(ofs + len <= 32); 583 584 if (len == 32) { 585 tcg_gen_mov_i32(ret, arg2); 586 return; 587 } 588 if (TCG_TARGET_HAS_deposit_i32 && TCG_TARGET_deposit_i32_valid(ofs, len)) { 589 tcg_gen_op5ii_i32(INDEX_op_deposit_i32, ret, arg1, arg2, ofs, len); 590 return; 591 } 592 593 t1 = tcg_temp_ebb_new_i32(); 594 595 if (TCG_TARGET_HAS_extract2_i32) { 596 if (ofs + len == 32) { 597 tcg_gen_shli_i32(t1, arg1, len); 598 tcg_gen_extract2_i32(ret, t1, arg2, len); 599 goto done; 600 } 601 if (ofs == 0) { 602 tcg_gen_extract2_i32(ret, arg1, arg2, len); 603 tcg_gen_rotli_i32(ret, ret, len); 604 goto done; 605 } 606 } 607 608 mask = (1u << len) - 1; 609 if (ofs + len < 32) { 610 tcg_gen_andi_i32(t1, arg2, mask); 611 tcg_gen_shli_i32(t1, t1, ofs); 612 } else { 613 tcg_gen_shli_i32(t1, arg2, ofs); 614 } 615 tcg_gen_andi_i32(ret, arg1, ~(mask << ofs)); 616 tcg_gen_or_i32(ret, ret, t1); 617 done: 618 tcg_temp_free_i32(t1); 619 } 620 621 void tcg_gen_deposit_z_i32(TCGv_i32 ret, TCGv_i32 arg, 622 unsigned int ofs, unsigned int len) 623 { 624 tcg_debug_assert(ofs < 32); 625 tcg_debug_assert(len > 0); 626 tcg_debug_assert(len <= 32); 627 tcg_debug_assert(ofs + len <= 32); 628 629 if (ofs + len == 32) { 630 tcg_gen_shli_i32(ret, arg, ofs); 631 } else if (ofs == 0) { 632 tcg_gen_andi_i32(ret, arg, (1u << len) - 1); 633 } else if (TCG_TARGET_HAS_deposit_i32 634 && TCG_TARGET_deposit_i32_valid(ofs, len)) { 635 TCGv_i32 zero = tcg_constant_i32(0); 636 tcg_gen_op5ii_i32(INDEX_op_deposit_i32, ret, zero, arg, ofs, len); 637 } else { 638 /* To help two-operand hosts we prefer to zero-extend first, 639 which allows ARG to stay live. */ 640 switch (len) { 641 case 16: 642 if (TCG_TARGET_HAS_ext16u_i32) { 643 tcg_gen_ext16u_i32(ret, arg); 644 tcg_gen_shli_i32(ret, ret, ofs); 645 return; 646 } 647 break; 648 case 8: 649 if (TCG_TARGET_HAS_ext8u_i32) { 650 tcg_gen_ext8u_i32(ret, arg); 651 tcg_gen_shli_i32(ret, ret, ofs); 652 return; 653 } 654 break; 655 } 656 /* Otherwise prefer zero-extension over AND for code size. */ 657 switch (ofs + len) { 658 case 16: 659 if (TCG_TARGET_HAS_ext16u_i32) { 660 tcg_gen_shli_i32(ret, arg, ofs); 661 tcg_gen_ext16u_i32(ret, ret); 662 return; 663 } 664 break; 665 case 8: 666 if (TCG_TARGET_HAS_ext8u_i32) { 667 tcg_gen_shli_i32(ret, arg, ofs); 668 tcg_gen_ext8u_i32(ret, ret); 669 return; 670 } 671 break; 672 } 673 tcg_gen_andi_i32(ret, arg, (1u << len) - 1); 674 tcg_gen_shli_i32(ret, ret, ofs); 675 } 676 } 677 678 void tcg_gen_extract_i32(TCGv_i32 ret, TCGv_i32 arg, 679 unsigned int ofs, unsigned int len) 680 { 681 tcg_debug_assert(ofs < 32); 682 tcg_debug_assert(len > 0); 683 tcg_debug_assert(len <= 32); 684 tcg_debug_assert(ofs + len <= 32); 685 686 /* Canonicalize certain special cases, even if extract is supported. */ 687 if (ofs + len == 32) { 688 tcg_gen_shri_i32(ret, arg, 32 - len); 689 return; 690 } 691 if (ofs == 0) { 692 tcg_gen_andi_i32(ret, arg, (1u << len) - 1); 693 return; 694 } 695 696 if (TCG_TARGET_HAS_extract_i32 697 && TCG_TARGET_extract_i32_valid(ofs, len)) { 698 tcg_gen_op4ii_i32(INDEX_op_extract_i32, ret, arg, ofs, len); 699 return; 700 } 701 702 /* Assume that zero-extension, if available, is cheaper than a shift. */ 703 switch (ofs + len) { 704 case 16: 705 if (TCG_TARGET_HAS_ext16u_i32) { 706 tcg_gen_ext16u_i32(ret, arg); 707 tcg_gen_shri_i32(ret, ret, ofs); 708 return; 709 } 710 break; 711 case 8: 712 if (TCG_TARGET_HAS_ext8u_i32) { 713 tcg_gen_ext8u_i32(ret, arg); 714 tcg_gen_shri_i32(ret, ret, ofs); 715 return; 716 } 717 break; 718 } 719 720 /* ??? Ideally we'd know what values are available for immediate AND. 721 Assume that 8 bits are available, plus the special case of 16, 722 so that we get ext8u, ext16u. */ 723 switch (len) { 724 case 1 ... 8: case 16: 725 tcg_gen_shri_i32(ret, arg, ofs); 726 tcg_gen_andi_i32(ret, ret, (1u << len) - 1); 727 break; 728 default: 729 tcg_gen_shli_i32(ret, arg, 32 - len - ofs); 730 tcg_gen_shri_i32(ret, ret, 32 - len); 731 break; 732 } 733 } 734 735 void tcg_gen_sextract_i32(TCGv_i32 ret, TCGv_i32 arg, 736 unsigned int ofs, unsigned int len) 737 { 738 tcg_debug_assert(ofs < 32); 739 tcg_debug_assert(len > 0); 740 tcg_debug_assert(len <= 32); 741 tcg_debug_assert(ofs + len <= 32); 742 743 /* Canonicalize certain special cases, even if extract is supported. */ 744 if (ofs + len == 32) { 745 tcg_gen_sari_i32(ret, arg, 32 - len); 746 return; 747 } 748 if (ofs == 0) { 749 switch (len) { 750 case 16: 751 tcg_gen_ext16s_i32(ret, arg); 752 return; 753 case 8: 754 tcg_gen_ext8s_i32(ret, arg); 755 return; 756 } 757 } 758 759 if (TCG_TARGET_HAS_sextract_i32 760 && TCG_TARGET_extract_i32_valid(ofs, len)) { 761 tcg_gen_op4ii_i32(INDEX_op_sextract_i32, ret, arg, ofs, len); 762 return; 763 } 764 765 /* Assume that sign-extension, if available, is cheaper than a shift. */ 766 switch (ofs + len) { 767 case 16: 768 if (TCG_TARGET_HAS_ext16s_i32) { 769 tcg_gen_ext16s_i32(ret, arg); 770 tcg_gen_sari_i32(ret, ret, ofs); 771 return; 772 } 773 break; 774 case 8: 775 if (TCG_TARGET_HAS_ext8s_i32) { 776 tcg_gen_ext8s_i32(ret, arg); 777 tcg_gen_sari_i32(ret, ret, ofs); 778 return; 779 } 780 break; 781 } 782 switch (len) { 783 case 16: 784 if (TCG_TARGET_HAS_ext16s_i32) { 785 tcg_gen_shri_i32(ret, arg, ofs); 786 tcg_gen_ext16s_i32(ret, ret); 787 return; 788 } 789 break; 790 case 8: 791 if (TCG_TARGET_HAS_ext8s_i32) { 792 tcg_gen_shri_i32(ret, arg, ofs); 793 tcg_gen_ext8s_i32(ret, ret); 794 return; 795 } 796 break; 797 } 798 799 tcg_gen_shli_i32(ret, arg, 32 - len - ofs); 800 tcg_gen_sari_i32(ret, ret, 32 - len); 801 } 802 803 /* 804 * Extract 32-bits from a 64-bit input, ah:al, starting from ofs. 805 * Unlike tcg_gen_extract_i32 above, len is fixed at 32. 806 */ 807 void tcg_gen_extract2_i32(TCGv_i32 ret, TCGv_i32 al, TCGv_i32 ah, 808 unsigned int ofs) 809 { 810 tcg_debug_assert(ofs <= 32); 811 if (ofs == 0) { 812 tcg_gen_mov_i32(ret, al); 813 } else if (ofs == 32) { 814 tcg_gen_mov_i32(ret, ah); 815 } else if (al == ah) { 816 tcg_gen_rotri_i32(ret, al, ofs); 817 } else if (TCG_TARGET_HAS_extract2_i32) { 818 tcg_gen_op4i_i32(INDEX_op_extract2_i32, ret, al, ah, ofs); 819 } else { 820 TCGv_i32 t0 = tcg_temp_ebb_new_i32(); 821 tcg_gen_shri_i32(t0, al, ofs); 822 tcg_gen_deposit_i32(ret, t0, ah, 32 - ofs, ofs); 823 tcg_temp_free_i32(t0); 824 } 825 } 826 827 void tcg_gen_movcond_i32(TCGCond cond, TCGv_i32 ret, TCGv_i32 c1, 828 TCGv_i32 c2, TCGv_i32 v1, TCGv_i32 v2) 829 { 830 if (cond == TCG_COND_ALWAYS) { 831 tcg_gen_mov_i32(ret, v1); 832 } else if (cond == TCG_COND_NEVER) { 833 tcg_gen_mov_i32(ret, v2); 834 } else if (TCG_TARGET_HAS_movcond_i32) { 835 tcg_gen_op6i_i32(INDEX_op_movcond_i32, ret, c1, c2, v1, v2, cond); 836 } else { 837 TCGv_i32 t0 = tcg_temp_ebb_new_i32(); 838 TCGv_i32 t1 = tcg_temp_ebb_new_i32(); 839 tcg_gen_setcond_i32(cond, t0, c1, c2); 840 tcg_gen_neg_i32(t0, t0); 841 tcg_gen_and_i32(t1, v1, t0); 842 tcg_gen_andc_i32(ret, v2, t0); 843 tcg_gen_or_i32(ret, ret, t1); 844 tcg_temp_free_i32(t0); 845 tcg_temp_free_i32(t1); 846 } 847 } 848 849 void tcg_gen_add2_i32(TCGv_i32 rl, TCGv_i32 rh, TCGv_i32 al, 850 TCGv_i32 ah, TCGv_i32 bl, TCGv_i32 bh) 851 { 852 if (TCG_TARGET_HAS_add2_i32) { 853 tcg_gen_op6_i32(INDEX_op_add2_i32, rl, rh, al, ah, bl, bh); 854 } else { 855 TCGv_i64 t0 = tcg_temp_ebb_new_i64(); 856 TCGv_i64 t1 = tcg_temp_ebb_new_i64(); 857 tcg_gen_concat_i32_i64(t0, al, ah); 858 tcg_gen_concat_i32_i64(t1, bl, bh); 859 tcg_gen_add_i64(t0, t0, t1); 860 tcg_gen_extr_i64_i32(rl, rh, t0); 861 tcg_temp_free_i64(t0); 862 tcg_temp_free_i64(t1); 863 } 864 } 865 866 void tcg_gen_sub2_i32(TCGv_i32 rl, TCGv_i32 rh, TCGv_i32 al, 867 TCGv_i32 ah, TCGv_i32 bl, TCGv_i32 bh) 868 { 869 if (TCG_TARGET_HAS_sub2_i32) { 870 tcg_gen_op6_i32(INDEX_op_sub2_i32, rl, rh, al, ah, bl, bh); 871 } else { 872 TCGv_i64 t0 = tcg_temp_ebb_new_i64(); 873 TCGv_i64 t1 = tcg_temp_ebb_new_i64(); 874 tcg_gen_concat_i32_i64(t0, al, ah); 875 tcg_gen_concat_i32_i64(t1, bl, bh); 876 tcg_gen_sub_i64(t0, t0, t1); 877 tcg_gen_extr_i64_i32(rl, rh, t0); 878 tcg_temp_free_i64(t0); 879 tcg_temp_free_i64(t1); 880 } 881 } 882 883 void tcg_gen_mulu2_i32(TCGv_i32 rl, TCGv_i32 rh, TCGv_i32 arg1, TCGv_i32 arg2) 884 { 885 if (TCG_TARGET_HAS_mulu2_i32) { 886 tcg_gen_op4_i32(INDEX_op_mulu2_i32, rl, rh, arg1, arg2); 887 } else if (TCG_TARGET_HAS_muluh_i32) { 888 TCGv_i32 t = tcg_temp_ebb_new_i32(); 889 tcg_gen_op3_i32(INDEX_op_mul_i32, t, arg1, arg2); 890 tcg_gen_op3_i32(INDEX_op_muluh_i32, rh, arg1, arg2); 891 tcg_gen_mov_i32(rl, t); 892 tcg_temp_free_i32(t); 893 } else if (TCG_TARGET_REG_BITS == 64) { 894 TCGv_i64 t0 = tcg_temp_ebb_new_i64(); 895 TCGv_i64 t1 = tcg_temp_ebb_new_i64(); 896 tcg_gen_extu_i32_i64(t0, arg1); 897 tcg_gen_extu_i32_i64(t1, arg2); 898 tcg_gen_mul_i64(t0, t0, t1); 899 tcg_gen_extr_i64_i32(rl, rh, t0); 900 tcg_temp_free_i64(t0); 901 tcg_temp_free_i64(t1); 902 } else { 903 qemu_build_not_reached(); 904 } 905 } 906 907 void tcg_gen_muls2_i32(TCGv_i32 rl, TCGv_i32 rh, TCGv_i32 arg1, TCGv_i32 arg2) 908 { 909 if (TCG_TARGET_HAS_muls2_i32) { 910 tcg_gen_op4_i32(INDEX_op_muls2_i32, rl, rh, arg1, arg2); 911 } else if (TCG_TARGET_HAS_mulsh_i32) { 912 TCGv_i32 t = tcg_temp_ebb_new_i32(); 913 tcg_gen_op3_i32(INDEX_op_mul_i32, t, arg1, arg2); 914 tcg_gen_op3_i32(INDEX_op_mulsh_i32, rh, arg1, arg2); 915 tcg_gen_mov_i32(rl, t); 916 tcg_temp_free_i32(t); 917 } else if (TCG_TARGET_REG_BITS == 32) { 918 TCGv_i32 t0 = tcg_temp_ebb_new_i32(); 919 TCGv_i32 t1 = tcg_temp_ebb_new_i32(); 920 TCGv_i32 t2 = tcg_temp_ebb_new_i32(); 921 TCGv_i32 t3 = tcg_temp_ebb_new_i32(); 922 tcg_gen_mulu2_i32(t0, t1, arg1, arg2); 923 /* Adjust for negative inputs. */ 924 tcg_gen_sari_i32(t2, arg1, 31); 925 tcg_gen_sari_i32(t3, arg2, 31); 926 tcg_gen_and_i32(t2, t2, arg2); 927 tcg_gen_and_i32(t3, t3, arg1); 928 tcg_gen_sub_i32(rh, t1, t2); 929 tcg_gen_sub_i32(rh, rh, t3); 930 tcg_gen_mov_i32(rl, t0); 931 tcg_temp_free_i32(t0); 932 tcg_temp_free_i32(t1); 933 tcg_temp_free_i32(t2); 934 tcg_temp_free_i32(t3); 935 } else { 936 TCGv_i64 t0 = tcg_temp_ebb_new_i64(); 937 TCGv_i64 t1 = tcg_temp_ebb_new_i64(); 938 tcg_gen_ext_i32_i64(t0, arg1); 939 tcg_gen_ext_i32_i64(t1, arg2); 940 tcg_gen_mul_i64(t0, t0, t1); 941 tcg_gen_extr_i64_i32(rl, rh, t0); 942 tcg_temp_free_i64(t0); 943 tcg_temp_free_i64(t1); 944 } 945 } 946 947 void tcg_gen_mulsu2_i32(TCGv_i32 rl, TCGv_i32 rh, TCGv_i32 arg1, TCGv_i32 arg2) 948 { 949 if (TCG_TARGET_REG_BITS == 32) { 950 TCGv_i32 t0 = tcg_temp_ebb_new_i32(); 951 TCGv_i32 t1 = tcg_temp_ebb_new_i32(); 952 TCGv_i32 t2 = tcg_temp_ebb_new_i32(); 953 tcg_gen_mulu2_i32(t0, t1, arg1, arg2); 954 /* Adjust for negative input for the signed arg1. */ 955 tcg_gen_sari_i32(t2, arg1, 31); 956 tcg_gen_and_i32(t2, t2, arg2); 957 tcg_gen_sub_i32(rh, t1, t2); 958 tcg_gen_mov_i32(rl, t0); 959 tcg_temp_free_i32(t0); 960 tcg_temp_free_i32(t1); 961 tcg_temp_free_i32(t2); 962 } else { 963 TCGv_i64 t0 = tcg_temp_ebb_new_i64(); 964 TCGv_i64 t1 = tcg_temp_ebb_new_i64(); 965 tcg_gen_ext_i32_i64(t0, arg1); 966 tcg_gen_extu_i32_i64(t1, arg2); 967 tcg_gen_mul_i64(t0, t0, t1); 968 tcg_gen_extr_i64_i32(rl, rh, t0); 969 tcg_temp_free_i64(t0); 970 tcg_temp_free_i64(t1); 971 } 972 } 973 974 void tcg_gen_ext8s_i32(TCGv_i32 ret, TCGv_i32 arg) 975 { 976 if (TCG_TARGET_HAS_ext8s_i32) { 977 tcg_gen_op2_i32(INDEX_op_ext8s_i32, ret, arg); 978 } else { 979 tcg_gen_shli_i32(ret, arg, 24); 980 tcg_gen_sari_i32(ret, ret, 24); 981 } 982 } 983 984 void tcg_gen_ext16s_i32(TCGv_i32 ret, TCGv_i32 arg) 985 { 986 if (TCG_TARGET_HAS_ext16s_i32) { 987 tcg_gen_op2_i32(INDEX_op_ext16s_i32, ret, arg); 988 } else { 989 tcg_gen_shli_i32(ret, arg, 16); 990 tcg_gen_sari_i32(ret, ret, 16); 991 } 992 } 993 994 void tcg_gen_ext8u_i32(TCGv_i32 ret, TCGv_i32 arg) 995 { 996 if (TCG_TARGET_HAS_ext8u_i32) { 997 tcg_gen_op2_i32(INDEX_op_ext8u_i32, ret, arg); 998 } else { 999 tcg_gen_andi_i32(ret, arg, 0xffu); 1000 } 1001 } 1002 1003 void tcg_gen_ext16u_i32(TCGv_i32 ret, TCGv_i32 arg) 1004 { 1005 if (TCG_TARGET_HAS_ext16u_i32) { 1006 tcg_gen_op2_i32(INDEX_op_ext16u_i32, ret, arg); 1007 } else { 1008 tcg_gen_andi_i32(ret, arg, 0xffffu); 1009 } 1010 } 1011 1012 void tcg_gen_bswap16_i32(TCGv_i32 ret, TCGv_i32 arg, int flags) 1013 { 1014 /* Only one extension flag may be present. */ 1015 tcg_debug_assert(!(flags & TCG_BSWAP_OS) || !(flags & TCG_BSWAP_OZ)); 1016 1017 if (TCG_TARGET_HAS_bswap16_i32) { 1018 tcg_gen_op3i_i32(INDEX_op_bswap16_i32, ret, arg, flags); 1019 } else { 1020 TCGv_i32 t0 = tcg_temp_ebb_new_i32(); 1021 TCGv_i32 t1 = tcg_temp_ebb_new_i32(); 1022 1023 tcg_gen_shri_i32(t0, arg, 8); 1024 if (!(flags & TCG_BSWAP_IZ)) { 1025 tcg_gen_ext8u_i32(t0, t0); 1026 } 1027 1028 if (flags & TCG_BSWAP_OS) { 1029 tcg_gen_shli_i32(t1, arg, 24); 1030 tcg_gen_sari_i32(t1, t1, 16); 1031 } else if (flags & TCG_BSWAP_OZ) { 1032 tcg_gen_ext8u_i32(t1, arg); 1033 tcg_gen_shli_i32(t1, t1, 8); 1034 } else { 1035 tcg_gen_shli_i32(t1, arg, 8); 1036 } 1037 1038 tcg_gen_or_i32(ret, t0, t1); 1039 tcg_temp_free_i32(t0); 1040 tcg_temp_free_i32(t1); 1041 } 1042 } 1043 1044 void tcg_gen_bswap32_i32(TCGv_i32 ret, TCGv_i32 arg) 1045 { 1046 if (TCG_TARGET_HAS_bswap32_i32) { 1047 tcg_gen_op3i_i32(INDEX_op_bswap32_i32, ret, arg, 0); 1048 } else { 1049 TCGv_i32 t0 = tcg_temp_ebb_new_i32(); 1050 TCGv_i32 t1 = tcg_temp_ebb_new_i32(); 1051 TCGv_i32 t2 = tcg_constant_i32(0x00ff00ff); 1052 1053 /* arg = abcd */ 1054 tcg_gen_shri_i32(t0, arg, 8); /* t0 = .abc */ 1055 tcg_gen_and_i32(t1, arg, t2); /* t1 = .b.d */ 1056 tcg_gen_and_i32(t0, t0, t2); /* t0 = .a.c */ 1057 tcg_gen_shli_i32(t1, t1, 8); /* t1 = b.d. */ 1058 tcg_gen_or_i32(ret, t0, t1); /* ret = badc */ 1059 1060 tcg_gen_shri_i32(t0, ret, 16); /* t0 = ..ba */ 1061 tcg_gen_shli_i32(t1, ret, 16); /* t1 = dc.. */ 1062 tcg_gen_or_i32(ret, t0, t1); /* ret = dcba */ 1063 1064 tcg_temp_free_i32(t0); 1065 tcg_temp_free_i32(t1); 1066 } 1067 } 1068 1069 void tcg_gen_hswap_i32(TCGv_i32 ret, TCGv_i32 arg) 1070 { 1071 /* Swapping 2 16-bit elements is a rotate. */ 1072 tcg_gen_rotli_i32(ret, arg, 16); 1073 } 1074 1075 void tcg_gen_smin_i32(TCGv_i32 ret, TCGv_i32 a, TCGv_i32 b) 1076 { 1077 tcg_gen_movcond_i32(TCG_COND_LT, ret, a, b, a, b); 1078 } 1079 1080 void tcg_gen_umin_i32(TCGv_i32 ret, TCGv_i32 a, TCGv_i32 b) 1081 { 1082 tcg_gen_movcond_i32(TCG_COND_LTU, ret, a, b, a, b); 1083 } 1084 1085 void tcg_gen_smax_i32(TCGv_i32 ret, TCGv_i32 a, TCGv_i32 b) 1086 { 1087 tcg_gen_movcond_i32(TCG_COND_LT, ret, a, b, b, a); 1088 } 1089 1090 void tcg_gen_umax_i32(TCGv_i32 ret, TCGv_i32 a, TCGv_i32 b) 1091 { 1092 tcg_gen_movcond_i32(TCG_COND_LTU, ret, a, b, b, a); 1093 } 1094 1095 void tcg_gen_abs_i32(TCGv_i32 ret, TCGv_i32 a) 1096 { 1097 TCGv_i32 t = tcg_temp_ebb_new_i32(); 1098 1099 tcg_gen_sari_i32(t, a, 31); 1100 tcg_gen_xor_i32(ret, a, t); 1101 tcg_gen_sub_i32(ret, ret, t); 1102 tcg_temp_free_i32(t); 1103 } 1104 1105 /* 64-bit ops */ 1106 1107 #if TCG_TARGET_REG_BITS == 32 1108 /* These are all inline for TCG_TARGET_REG_BITS == 64. */ 1109 1110 void tcg_gen_discard_i64(TCGv_i64 arg) 1111 { 1112 tcg_gen_discard_i32(TCGV_LOW(arg)); 1113 tcg_gen_discard_i32(TCGV_HIGH(arg)); 1114 } 1115 1116 void tcg_gen_mov_i64(TCGv_i64 ret, TCGv_i64 arg) 1117 { 1118 TCGTemp *ts = tcgv_i64_temp(arg); 1119 1120 /* Canonicalize TCGv_i64 TEMP_CONST into TCGv_i32 TEMP_CONST. */ 1121 if (ts->kind == TEMP_CONST) { 1122 tcg_gen_movi_i64(ret, ts->val); 1123 } else { 1124 tcg_gen_mov_i32(TCGV_LOW(ret), TCGV_LOW(arg)); 1125 tcg_gen_mov_i32(TCGV_HIGH(ret), TCGV_HIGH(arg)); 1126 } 1127 } 1128 1129 void tcg_gen_movi_i64(TCGv_i64 ret, int64_t arg) 1130 { 1131 tcg_gen_movi_i32(TCGV_LOW(ret), arg); 1132 tcg_gen_movi_i32(TCGV_HIGH(ret), arg >> 32); 1133 } 1134 1135 void tcg_gen_ld8u_i64(TCGv_i64 ret, TCGv_ptr arg2, tcg_target_long offset) 1136 { 1137 tcg_gen_ld8u_i32(TCGV_LOW(ret), arg2, offset); 1138 tcg_gen_movi_i32(TCGV_HIGH(ret), 0); 1139 } 1140 1141 void tcg_gen_ld8s_i64(TCGv_i64 ret, TCGv_ptr arg2, tcg_target_long offset) 1142 { 1143 tcg_gen_ld8s_i32(TCGV_LOW(ret), arg2, offset); 1144 tcg_gen_sari_i32(TCGV_HIGH(ret), TCGV_LOW(ret), 31); 1145 } 1146 1147 void tcg_gen_ld16u_i64(TCGv_i64 ret, TCGv_ptr arg2, tcg_target_long offset) 1148 { 1149 tcg_gen_ld16u_i32(TCGV_LOW(ret), arg2, offset); 1150 tcg_gen_movi_i32(TCGV_HIGH(ret), 0); 1151 } 1152 1153 void tcg_gen_ld16s_i64(TCGv_i64 ret, TCGv_ptr arg2, tcg_target_long offset) 1154 { 1155 tcg_gen_ld16s_i32(TCGV_LOW(ret), arg2, offset); 1156 tcg_gen_sari_i32(TCGV_HIGH(ret), TCGV_LOW(ret), 31); 1157 } 1158 1159 void tcg_gen_ld32u_i64(TCGv_i64 ret, TCGv_ptr arg2, tcg_target_long offset) 1160 { 1161 tcg_gen_ld_i32(TCGV_LOW(ret), arg2, offset); 1162 tcg_gen_movi_i32(TCGV_HIGH(ret), 0); 1163 } 1164 1165 void tcg_gen_ld32s_i64(TCGv_i64 ret, TCGv_ptr arg2, tcg_target_long offset) 1166 { 1167 tcg_gen_ld_i32(TCGV_LOW(ret), arg2, offset); 1168 tcg_gen_sari_i32(TCGV_HIGH(ret), TCGV_LOW(ret), 31); 1169 } 1170 1171 void tcg_gen_ld_i64(TCGv_i64 ret, TCGv_ptr arg2, tcg_target_long offset) 1172 { 1173 /* Since arg2 and ret have different types, 1174 they cannot be the same temporary */ 1175 #if HOST_BIG_ENDIAN 1176 tcg_gen_ld_i32(TCGV_HIGH(ret), arg2, offset); 1177 tcg_gen_ld_i32(TCGV_LOW(ret), arg2, offset + 4); 1178 #else 1179 tcg_gen_ld_i32(TCGV_LOW(ret), arg2, offset); 1180 tcg_gen_ld_i32(TCGV_HIGH(ret), arg2, offset + 4); 1181 #endif 1182 } 1183 1184 void tcg_gen_st8_i64(TCGv_i64 arg1, TCGv_ptr arg2, tcg_target_long offset) 1185 { 1186 tcg_gen_st8_i32(TCGV_LOW(arg1), arg2, offset); 1187 } 1188 1189 void tcg_gen_st16_i64(TCGv_i64 arg1, TCGv_ptr arg2, tcg_target_long offset) 1190 { 1191 tcg_gen_st16_i32(TCGV_LOW(arg1), arg2, offset); 1192 } 1193 1194 void tcg_gen_st32_i64(TCGv_i64 arg1, TCGv_ptr arg2, tcg_target_long offset) 1195 { 1196 tcg_gen_st_i32(TCGV_LOW(arg1), arg2, offset); 1197 } 1198 1199 void tcg_gen_st_i64(TCGv_i64 arg1, TCGv_ptr arg2, tcg_target_long offset) 1200 { 1201 #if HOST_BIG_ENDIAN 1202 tcg_gen_st_i32(TCGV_HIGH(arg1), arg2, offset); 1203 tcg_gen_st_i32(TCGV_LOW(arg1), arg2, offset + 4); 1204 #else 1205 tcg_gen_st_i32(TCGV_LOW(arg1), arg2, offset); 1206 tcg_gen_st_i32(TCGV_HIGH(arg1), arg2, offset + 4); 1207 #endif 1208 } 1209 1210 void tcg_gen_add_i64(TCGv_i64 ret, TCGv_i64 arg1, TCGv_i64 arg2) 1211 { 1212 tcg_gen_add2_i32(TCGV_LOW(ret), TCGV_HIGH(ret), TCGV_LOW(arg1), 1213 TCGV_HIGH(arg1), TCGV_LOW(arg2), TCGV_HIGH(arg2)); 1214 } 1215 1216 void tcg_gen_sub_i64(TCGv_i64 ret, TCGv_i64 arg1, TCGv_i64 arg2) 1217 { 1218 tcg_gen_sub2_i32(TCGV_LOW(ret), TCGV_HIGH(ret), TCGV_LOW(arg1), 1219 TCGV_HIGH(arg1), TCGV_LOW(arg2), TCGV_HIGH(arg2)); 1220 } 1221 1222 void tcg_gen_and_i64(TCGv_i64 ret, TCGv_i64 arg1, TCGv_i64 arg2) 1223 { 1224 tcg_gen_and_i32(TCGV_LOW(ret), TCGV_LOW(arg1), TCGV_LOW(arg2)); 1225 tcg_gen_and_i32(TCGV_HIGH(ret), TCGV_HIGH(arg1), TCGV_HIGH(arg2)); 1226 } 1227 1228 void tcg_gen_or_i64(TCGv_i64 ret, TCGv_i64 arg1, TCGv_i64 arg2) 1229 { 1230 tcg_gen_or_i32(TCGV_LOW(ret), TCGV_LOW(arg1), TCGV_LOW(arg2)); 1231 tcg_gen_or_i32(TCGV_HIGH(ret), TCGV_HIGH(arg1), TCGV_HIGH(arg2)); 1232 } 1233 1234 void tcg_gen_xor_i64(TCGv_i64 ret, TCGv_i64 arg1, TCGv_i64 arg2) 1235 { 1236 tcg_gen_xor_i32(TCGV_LOW(ret), TCGV_LOW(arg1), TCGV_LOW(arg2)); 1237 tcg_gen_xor_i32(TCGV_HIGH(ret), TCGV_HIGH(arg1), TCGV_HIGH(arg2)); 1238 } 1239 1240 void tcg_gen_shl_i64(TCGv_i64 ret, TCGv_i64 arg1, TCGv_i64 arg2) 1241 { 1242 gen_helper_shl_i64(ret, arg1, arg2); 1243 } 1244 1245 void tcg_gen_shr_i64(TCGv_i64 ret, TCGv_i64 arg1, TCGv_i64 arg2) 1246 { 1247 gen_helper_shr_i64(ret, arg1, arg2); 1248 } 1249 1250 void tcg_gen_sar_i64(TCGv_i64 ret, TCGv_i64 arg1, TCGv_i64 arg2) 1251 { 1252 gen_helper_sar_i64(ret, arg1, arg2); 1253 } 1254 1255 void tcg_gen_mul_i64(TCGv_i64 ret, TCGv_i64 arg1, TCGv_i64 arg2) 1256 { 1257 TCGv_i64 t0; 1258 TCGv_i32 t1; 1259 1260 t0 = tcg_temp_ebb_new_i64(); 1261 t1 = tcg_temp_ebb_new_i32(); 1262 1263 tcg_gen_mulu2_i32(TCGV_LOW(t0), TCGV_HIGH(t0), 1264 TCGV_LOW(arg1), TCGV_LOW(arg2)); 1265 1266 tcg_gen_mul_i32(t1, TCGV_LOW(arg1), TCGV_HIGH(arg2)); 1267 tcg_gen_add_i32(TCGV_HIGH(t0), TCGV_HIGH(t0), t1); 1268 tcg_gen_mul_i32(t1, TCGV_HIGH(arg1), TCGV_LOW(arg2)); 1269 tcg_gen_add_i32(TCGV_HIGH(t0), TCGV_HIGH(t0), t1); 1270 1271 tcg_gen_mov_i64(ret, t0); 1272 tcg_temp_free_i64(t0); 1273 tcg_temp_free_i32(t1); 1274 } 1275 1276 #else 1277 1278 void tcg_gen_movi_i64(TCGv_i64 ret, int64_t arg) 1279 { 1280 tcg_gen_mov_i64(ret, tcg_constant_i64(arg)); 1281 } 1282 1283 #endif /* TCG_TARGET_REG_SIZE == 32 */ 1284 1285 void tcg_gen_addi_i64(TCGv_i64 ret, TCGv_i64 arg1, int64_t arg2) 1286 { 1287 /* some cases can be optimized here */ 1288 if (arg2 == 0) { 1289 tcg_gen_mov_i64(ret, arg1); 1290 } else if (TCG_TARGET_REG_BITS == 64) { 1291 tcg_gen_add_i64(ret, arg1, tcg_constant_i64(arg2)); 1292 } else { 1293 tcg_gen_add2_i32(TCGV_LOW(ret), TCGV_HIGH(ret), 1294 TCGV_LOW(arg1), TCGV_HIGH(arg1), 1295 tcg_constant_i32(arg2), tcg_constant_i32(arg2 >> 32)); 1296 } 1297 } 1298 1299 void tcg_gen_subfi_i64(TCGv_i64 ret, int64_t arg1, TCGv_i64 arg2) 1300 { 1301 if (arg1 == 0 && TCG_TARGET_HAS_neg_i64) { 1302 /* Don't recurse with tcg_gen_neg_i64. */ 1303 tcg_gen_op2_i64(INDEX_op_neg_i64, ret, arg2); 1304 } else if (TCG_TARGET_REG_BITS == 64) { 1305 tcg_gen_sub_i64(ret, tcg_constant_i64(arg1), arg2); 1306 } else { 1307 tcg_gen_sub2_i32(TCGV_LOW(ret), TCGV_HIGH(ret), 1308 tcg_constant_i32(arg1), tcg_constant_i32(arg1 >> 32), 1309 TCGV_LOW(arg2), TCGV_HIGH(arg2)); 1310 } 1311 } 1312 1313 void tcg_gen_subi_i64(TCGv_i64 ret, TCGv_i64 arg1, int64_t arg2) 1314 { 1315 /* some cases can be optimized here */ 1316 if (arg2 == 0) { 1317 tcg_gen_mov_i64(ret, arg1); 1318 } else if (TCG_TARGET_REG_BITS == 64) { 1319 tcg_gen_sub_i64(ret, arg1, tcg_constant_i64(arg2)); 1320 } else { 1321 tcg_gen_sub2_i32(TCGV_LOW(ret), TCGV_HIGH(ret), 1322 TCGV_LOW(arg1), TCGV_HIGH(arg1), 1323 tcg_constant_i32(arg2), tcg_constant_i32(arg2 >> 32)); 1324 } 1325 } 1326 1327 void tcg_gen_andi_i64(TCGv_i64 ret, TCGv_i64 arg1, int64_t arg2) 1328 { 1329 if (TCG_TARGET_REG_BITS == 32) { 1330 tcg_gen_andi_i32(TCGV_LOW(ret), TCGV_LOW(arg1), arg2); 1331 tcg_gen_andi_i32(TCGV_HIGH(ret), TCGV_HIGH(arg1), arg2 >> 32); 1332 return; 1333 } 1334 1335 /* Some cases can be optimized here. */ 1336 switch (arg2) { 1337 case 0: 1338 tcg_gen_movi_i64(ret, 0); 1339 return; 1340 case -1: 1341 tcg_gen_mov_i64(ret, arg1); 1342 return; 1343 case 0xff: 1344 /* Don't recurse with tcg_gen_ext8u_i64. */ 1345 if (TCG_TARGET_HAS_ext8u_i64) { 1346 tcg_gen_op2_i64(INDEX_op_ext8u_i64, ret, arg1); 1347 return; 1348 } 1349 break; 1350 case 0xffff: 1351 if (TCG_TARGET_HAS_ext16u_i64) { 1352 tcg_gen_op2_i64(INDEX_op_ext16u_i64, ret, arg1); 1353 return; 1354 } 1355 break; 1356 case 0xffffffffu: 1357 if (TCG_TARGET_HAS_ext32u_i64) { 1358 tcg_gen_op2_i64(INDEX_op_ext32u_i64, ret, arg1); 1359 return; 1360 } 1361 break; 1362 } 1363 1364 tcg_gen_and_i64(ret, arg1, tcg_constant_i64(arg2)); 1365 } 1366 1367 void tcg_gen_ori_i64(TCGv_i64 ret, TCGv_i64 arg1, int64_t arg2) 1368 { 1369 if (TCG_TARGET_REG_BITS == 32) { 1370 tcg_gen_ori_i32(TCGV_LOW(ret), TCGV_LOW(arg1), arg2); 1371 tcg_gen_ori_i32(TCGV_HIGH(ret), TCGV_HIGH(arg1), arg2 >> 32); 1372 return; 1373 } 1374 /* Some cases can be optimized here. */ 1375 if (arg2 == -1) { 1376 tcg_gen_movi_i64(ret, -1); 1377 } else if (arg2 == 0) { 1378 tcg_gen_mov_i64(ret, arg1); 1379 } else { 1380 tcg_gen_or_i64(ret, arg1, tcg_constant_i64(arg2)); 1381 } 1382 } 1383 1384 void tcg_gen_xori_i64(TCGv_i64 ret, TCGv_i64 arg1, int64_t arg2) 1385 { 1386 if (TCG_TARGET_REG_BITS == 32) { 1387 tcg_gen_xori_i32(TCGV_LOW(ret), TCGV_LOW(arg1), arg2); 1388 tcg_gen_xori_i32(TCGV_HIGH(ret), TCGV_HIGH(arg1), arg2 >> 32); 1389 return; 1390 } 1391 /* Some cases can be optimized here. */ 1392 if (arg2 == 0) { 1393 tcg_gen_mov_i64(ret, arg1); 1394 } else if (arg2 == -1 && TCG_TARGET_HAS_not_i64) { 1395 /* Don't recurse with tcg_gen_not_i64. */ 1396 tcg_gen_op2_i64(INDEX_op_not_i64, ret, arg1); 1397 } else { 1398 tcg_gen_xor_i64(ret, arg1, tcg_constant_i64(arg2)); 1399 } 1400 } 1401 1402 static inline void tcg_gen_shifti_i64(TCGv_i64 ret, TCGv_i64 arg1, 1403 unsigned c, bool right, bool arith) 1404 { 1405 tcg_debug_assert(c < 64); 1406 if (c == 0) { 1407 tcg_gen_mov_i32(TCGV_LOW(ret), TCGV_LOW(arg1)); 1408 tcg_gen_mov_i32(TCGV_HIGH(ret), TCGV_HIGH(arg1)); 1409 } else if (c >= 32) { 1410 c -= 32; 1411 if (right) { 1412 if (arith) { 1413 tcg_gen_sari_i32(TCGV_LOW(ret), TCGV_HIGH(arg1), c); 1414 tcg_gen_sari_i32(TCGV_HIGH(ret), TCGV_HIGH(arg1), 31); 1415 } else { 1416 tcg_gen_shri_i32(TCGV_LOW(ret), TCGV_HIGH(arg1), c); 1417 tcg_gen_movi_i32(TCGV_HIGH(ret), 0); 1418 } 1419 } else { 1420 tcg_gen_shli_i32(TCGV_HIGH(ret), TCGV_LOW(arg1), c); 1421 tcg_gen_movi_i32(TCGV_LOW(ret), 0); 1422 } 1423 } else if (right) { 1424 if (TCG_TARGET_HAS_extract2_i32) { 1425 tcg_gen_extract2_i32(TCGV_LOW(ret), 1426 TCGV_LOW(arg1), TCGV_HIGH(arg1), c); 1427 } else { 1428 tcg_gen_shri_i32(TCGV_LOW(ret), TCGV_LOW(arg1), c); 1429 tcg_gen_deposit_i32(TCGV_LOW(ret), TCGV_LOW(ret), 1430 TCGV_HIGH(arg1), 32 - c, c); 1431 } 1432 if (arith) { 1433 tcg_gen_sari_i32(TCGV_HIGH(ret), TCGV_HIGH(arg1), c); 1434 } else { 1435 tcg_gen_shri_i32(TCGV_HIGH(ret), TCGV_HIGH(arg1), c); 1436 } 1437 } else { 1438 if (TCG_TARGET_HAS_extract2_i32) { 1439 tcg_gen_extract2_i32(TCGV_HIGH(ret), 1440 TCGV_LOW(arg1), TCGV_HIGH(arg1), 32 - c); 1441 } else { 1442 TCGv_i32 t0 = tcg_temp_ebb_new_i32(); 1443 tcg_gen_shri_i32(t0, TCGV_LOW(arg1), 32 - c); 1444 tcg_gen_deposit_i32(TCGV_HIGH(ret), t0, 1445 TCGV_HIGH(arg1), c, 32 - c); 1446 tcg_temp_free_i32(t0); 1447 } 1448 tcg_gen_shli_i32(TCGV_LOW(ret), TCGV_LOW(arg1), c); 1449 } 1450 } 1451 1452 void tcg_gen_shli_i64(TCGv_i64 ret, TCGv_i64 arg1, int64_t arg2) 1453 { 1454 tcg_debug_assert(arg2 >= 0 && arg2 < 64); 1455 if (TCG_TARGET_REG_BITS == 32) { 1456 tcg_gen_shifti_i64(ret, arg1, arg2, 0, 0); 1457 } else if (arg2 == 0) { 1458 tcg_gen_mov_i64(ret, arg1); 1459 } else { 1460 tcg_gen_shl_i64(ret, arg1, tcg_constant_i64(arg2)); 1461 } 1462 } 1463 1464 void tcg_gen_shri_i64(TCGv_i64 ret, TCGv_i64 arg1, int64_t arg2) 1465 { 1466 tcg_debug_assert(arg2 >= 0 && arg2 < 64); 1467 if (TCG_TARGET_REG_BITS == 32) { 1468 tcg_gen_shifti_i64(ret, arg1, arg2, 1, 0); 1469 } else if (arg2 == 0) { 1470 tcg_gen_mov_i64(ret, arg1); 1471 } else { 1472 tcg_gen_shr_i64(ret, arg1, tcg_constant_i64(arg2)); 1473 } 1474 } 1475 1476 void tcg_gen_sari_i64(TCGv_i64 ret, TCGv_i64 arg1, int64_t arg2) 1477 { 1478 tcg_debug_assert(arg2 >= 0 && arg2 < 64); 1479 if (TCG_TARGET_REG_BITS == 32) { 1480 tcg_gen_shifti_i64(ret, arg1, arg2, 1, 1); 1481 } else if (arg2 == 0) { 1482 tcg_gen_mov_i64(ret, arg1); 1483 } else { 1484 tcg_gen_sar_i64(ret, arg1, tcg_constant_i64(arg2)); 1485 } 1486 } 1487 1488 void tcg_gen_brcond_i64(TCGCond cond, TCGv_i64 arg1, TCGv_i64 arg2, TCGLabel *l) 1489 { 1490 if (cond == TCG_COND_ALWAYS) { 1491 tcg_gen_br(l); 1492 } else if (cond != TCG_COND_NEVER) { 1493 if (TCG_TARGET_REG_BITS == 32) { 1494 tcg_gen_op6ii_i32(INDEX_op_brcond2_i32, TCGV_LOW(arg1), 1495 TCGV_HIGH(arg1), TCGV_LOW(arg2), 1496 TCGV_HIGH(arg2), cond, label_arg(l)); 1497 } else { 1498 tcg_gen_op4ii_i64(INDEX_op_brcond_i64, arg1, arg2, cond, 1499 label_arg(l)); 1500 } 1501 add_last_as_label_use(l); 1502 } 1503 } 1504 1505 void tcg_gen_brcondi_i64(TCGCond cond, TCGv_i64 arg1, int64_t arg2, TCGLabel *l) 1506 { 1507 if (TCG_TARGET_REG_BITS == 64) { 1508 tcg_gen_brcond_i64(cond, arg1, tcg_constant_i64(arg2), l); 1509 } else if (cond == TCG_COND_ALWAYS) { 1510 tcg_gen_br(l); 1511 } else if (cond != TCG_COND_NEVER) { 1512 tcg_gen_op6ii_i32(INDEX_op_brcond2_i32, 1513 TCGV_LOW(arg1), TCGV_HIGH(arg1), 1514 tcg_constant_i32(arg2), 1515 tcg_constant_i32(arg2 >> 32), 1516 cond, label_arg(l)); 1517 add_last_as_label_use(l); 1518 } 1519 } 1520 1521 void tcg_gen_setcond_i64(TCGCond cond, TCGv_i64 ret, 1522 TCGv_i64 arg1, TCGv_i64 arg2) 1523 { 1524 if (cond == TCG_COND_ALWAYS) { 1525 tcg_gen_movi_i64(ret, 1); 1526 } else if (cond == TCG_COND_NEVER) { 1527 tcg_gen_movi_i64(ret, 0); 1528 } else { 1529 if (TCG_TARGET_REG_BITS == 32) { 1530 tcg_gen_op6i_i32(INDEX_op_setcond2_i32, TCGV_LOW(ret), 1531 TCGV_LOW(arg1), TCGV_HIGH(arg1), 1532 TCGV_LOW(arg2), TCGV_HIGH(arg2), cond); 1533 tcg_gen_movi_i32(TCGV_HIGH(ret), 0); 1534 } else { 1535 tcg_gen_op4i_i64(INDEX_op_setcond_i64, ret, arg1, arg2, cond); 1536 } 1537 } 1538 } 1539 1540 void tcg_gen_setcondi_i64(TCGCond cond, TCGv_i64 ret, 1541 TCGv_i64 arg1, int64_t arg2) 1542 { 1543 if (TCG_TARGET_REG_BITS == 64) { 1544 tcg_gen_setcond_i64(cond, ret, arg1, tcg_constant_i64(arg2)); 1545 } else if (cond == TCG_COND_ALWAYS) { 1546 tcg_gen_movi_i64(ret, 1); 1547 } else if (cond == TCG_COND_NEVER) { 1548 tcg_gen_movi_i64(ret, 0); 1549 } else { 1550 tcg_gen_op6i_i32(INDEX_op_setcond2_i32, TCGV_LOW(ret), 1551 TCGV_LOW(arg1), TCGV_HIGH(arg1), 1552 tcg_constant_i32(arg2), 1553 tcg_constant_i32(arg2 >> 32), cond); 1554 tcg_gen_movi_i32(TCGV_HIGH(ret), 0); 1555 } 1556 } 1557 1558 void tcg_gen_muli_i64(TCGv_i64 ret, TCGv_i64 arg1, int64_t arg2) 1559 { 1560 if (arg2 == 0) { 1561 tcg_gen_movi_i64(ret, 0); 1562 } else if (is_power_of_2(arg2)) { 1563 tcg_gen_shli_i64(ret, arg1, ctz64(arg2)); 1564 } else { 1565 TCGv_i64 t0 = tcg_const_i64(arg2); 1566 tcg_gen_mul_i64(ret, arg1, t0); 1567 tcg_temp_free_i64(t0); 1568 } 1569 } 1570 1571 void tcg_gen_div_i64(TCGv_i64 ret, TCGv_i64 arg1, TCGv_i64 arg2) 1572 { 1573 if (TCG_TARGET_HAS_div_i64) { 1574 tcg_gen_op3_i64(INDEX_op_div_i64, ret, arg1, arg2); 1575 } else if (TCG_TARGET_HAS_div2_i64) { 1576 TCGv_i64 t0 = tcg_temp_ebb_new_i64(); 1577 tcg_gen_sari_i64(t0, arg1, 63); 1578 tcg_gen_op5_i64(INDEX_op_div2_i64, ret, t0, arg1, t0, arg2); 1579 tcg_temp_free_i64(t0); 1580 } else { 1581 gen_helper_div_i64(ret, arg1, arg2); 1582 } 1583 } 1584 1585 void tcg_gen_rem_i64(TCGv_i64 ret, TCGv_i64 arg1, TCGv_i64 arg2) 1586 { 1587 if (TCG_TARGET_HAS_rem_i64) { 1588 tcg_gen_op3_i64(INDEX_op_rem_i64, ret, arg1, arg2); 1589 } else if (TCG_TARGET_HAS_div_i64) { 1590 TCGv_i64 t0 = tcg_temp_ebb_new_i64(); 1591 tcg_gen_op3_i64(INDEX_op_div_i64, t0, arg1, arg2); 1592 tcg_gen_mul_i64(t0, t0, arg2); 1593 tcg_gen_sub_i64(ret, arg1, t0); 1594 tcg_temp_free_i64(t0); 1595 } else if (TCG_TARGET_HAS_div2_i64) { 1596 TCGv_i64 t0 = tcg_temp_ebb_new_i64(); 1597 tcg_gen_sari_i64(t0, arg1, 63); 1598 tcg_gen_op5_i64(INDEX_op_div2_i64, t0, ret, arg1, t0, arg2); 1599 tcg_temp_free_i64(t0); 1600 } else { 1601 gen_helper_rem_i64(ret, arg1, arg2); 1602 } 1603 } 1604 1605 void tcg_gen_divu_i64(TCGv_i64 ret, TCGv_i64 arg1, TCGv_i64 arg2) 1606 { 1607 if (TCG_TARGET_HAS_div_i64) { 1608 tcg_gen_op3_i64(INDEX_op_divu_i64, ret, arg1, arg2); 1609 } else if (TCG_TARGET_HAS_div2_i64) { 1610 TCGv_i64 t0 = tcg_temp_ebb_new_i64(); 1611 tcg_gen_movi_i64(t0, 0); 1612 tcg_gen_op5_i64(INDEX_op_divu2_i64, ret, t0, arg1, t0, arg2); 1613 tcg_temp_free_i64(t0); 1614 } else { 1615 gen_helper_divu_i64(ret, arg1, arg2); 1616 } 1617 } 1618 1619 void tcg_gen_remu_i64(TCGv_i64 ret, TCGv_i64 arg1, TCGv_i64 arg2) 1620 { 1621 if (TCG_TARGET_HAS_rem_i64) { 1622 tcg_gen_op3_i64(INDEX_op_remu_i64, ret, arg1, arg2); 1623 } else if (TCG_TARGET_HAS_div_i64) { 1624 TCGv_i64 t0 = tcg_temp_ebb_new_i64(); 1625 tcg_gen_op3_i64(INDEX_op_divu_i64, t0, arg1, arg2); 1626 tcg_gen_mul_i64(t0, t0, arg2); 1627 tcg_gen_sub_i64(ret, arg1, t0); 1628 tcg_temp_free_i64(t0); 1629 } else if (TCG_TARGET_HAS_div2_i64) { 1630 TCGv_i64 t0 = tcg_temp_ebb_new_i64(); 1631 tcg_gen_movi_i64(t0, 0); 1632 tcg_gen_op5_i64(INDEX_op_divu2_i64, t0, ret, arg1, t0, arg2); 1633 tcg_temp_free_i64(t0); 1634 } else { 1635 gen_helper_remu_i64(ret, arg1, arg2); 1636 } 1637 } 1638 1639 void tcg_gen_ext8s_i64(TCGv_i64 ret, TCGv_i64 arg) 1640 { 1641 if (TCG_TARGET_REG_BITS == 32) { 1642 tcg_gen_ext8s_i32(TCGV_LOW(ret), TCGV_LOW(arg)); 1643 tcg_gen_sari_i32(TCGV_HIGH(ret), TCGV_LOW(ret), 31); 1644 } else if (TCG_TARGET_HAS_ext8s_i64) { 1645 tcg_gen_op2_i64(INDEX_op_ext8s_i64, ret, arg); 1646 } else { 1647 tcg_gen_shli_i64(ret, arg, 56); 1648 tcg_gen_sari_i64(ret, ret, 56); 1649 } 1650 } 1651 1652 void tcg_gen_ext16s_i64(TCGv_i64 ret, TCGv_i64 arg) 1653 { 1654 if (TCG_TARGET_REG_BITS == 32) { 1655 tcg_gen_ext16s_i32(TCGV_LOW(ret), TCGV_LOW(arg)); 1656 tcg_gen_sari_i32(TCGV_HIGH(ret), TCGV_LOW(ret), 31); 1657 } else if (TCG_TARGET_HAS_ext16s_i64) { 1658 tcg_gen_op2_i64(INDEX_op_ext16s_i64, ret, arg); 1659 } else { 1660 tcg_gen_shli_i64(ret, arg, 48); 1661 tcg_gen_sari_i64(ret, ret, 48); 1662 } 1663 } 1664 1665 void tcg_gen_ext32s_i64(TCGv_i64 ret, TCGv_i64 arg) 1666 { 1667 if (TCG_TARGET_REG_BITS == 32) { 1668 tcg_gen_mov_i32(TCGV_LOW(ret), TCGV_LOW(arg)); 1669 tcg_gen_sari_i32(TCGV_HIGH(ret), TCGV_LOW(ret), 31); 1670 } else if (TCG_TARGET_HAS_ext32s_i64) { 1671 tcg_gen_op2_i64(INDEX_op_ext32s_i64, ret, arg); 1672 } else { 1673 tcg_gen_shli_i64(ret, arg, 32); 1674 tcg_gen_sari_i64(ret, ret, 32); 1675 } 1676 } 1677 1678 void tcg_gen_ext8u_i64(TCGv_i64 ret, TCGv_i64 arg) 1679 { 1680 if (TCG_TARGET_REG_BITS == 32) { 1681 tcg_gen_ext8u_i32(TCGV_LOW(ret), TCGV_LOW(arg)); 1682 tcg_gen_movi_i32(TCGV_HIGH(ret), 0); 1683 } else if (TCG_TARGET_HAS_ext8u_i64) { 1684 tcg_gen_op2_i64(INDEX_op_ext8u_i64, ret, arg); 1685 } else { 1686 tcg_gen_andi_i64(ret, arg, 0xffu); 1687 } 1688 } 1689 1690 void tcg_gen_ext16u_i64(TCGv_i64 ret, TCGv_i64 arg) 1691 { 1692 if (TCG_TARGET_REG_BITS == 32) { 1693 tcg_gen_ext16u_i32(TCGV_LOW(ret), TCGV_LOW(arg)); 1694 tcg_gen_movi_i32(TCGV_HIGH(ret), 0); 1695 } else if (TCG_TARGET_HAS_ext16u_i64) { 1696 tcg_gen_op2_i64(INDEX_op_ext16u_i64, ret, arg); 1697 } else { 1698 tcg_gen_andi_i64(ret, arg, 0xffffu); 1699 } 1700 } 1701 1702 void tcg_gen_ext32u_i64(TCGv_i64 ret, TCGv_i64 arg) 1703 { 1704 if (TCG_TARGET_REG_BITS == 32) { 1705 tcg_gen_mov_i32(TCGV_LOW(ret), TCGV_LOW(arg)); 1706 tcg_gen_movi_i32(TCGV_HIGH(ret), 0); 1707 } else if (TCG_TARGET_HAS_ext32u_i64) { 1708 tcg_gen_op2_i64(INDEX_op_ext32u_i64, ret, arg); 1709 } else { 1710 tcg_gen_andi_i64(ret, arg, 0xffffffffu); 1711 } 1712 } 1713 1714 void tcg_gen_bswap16_i64(TCGv_i64 ret, TCGv_i64 arg, int flags) 1715 { 1716 /* Only one extension flag may be present. */ 1717 tcg_debug_assert(!(flags & TCG_BSWAP_OS) || !(flags & TCG_BSWAP_OZ)); 1718 1719 if (TCG_TARGET_REG_BITS == 32) { 1720 tcg_gen_bswap16_i32(TCGV_LOW(ret), TCGV_LOW(arg), flags); 1721 if (flags & TCG_BSWAP_OS) { 1722 tcg_gen_sari_i32(TCGV_HIGH(ret), TCGV_LOW(ret), 31); 1723 } else { 1724 tcg_gen_movi_i32(TCGV_HIGH(ret), 0); 1725 } 1726 } else if (TCG_TARGET_HAS_bswap16_i64) { 1727 tcg_gen_op3i_i64(INDEX_op_bswap16_i64, ret, arg, flags); 1728 } else { 1729 TCGv_i64 t0 = tcg_temp_ebb_new_i64(); 1730 TCGv_i64 t1 = tcg_temp_ebb_new_i64(); 1731 1732 tcg_gen_shri_i64(t0, arg, 8); 1733 if (!(flags & TCG_BSWAP_IZ)) { 1734 tcg_gen_ext8u_i64(t0, t0); 1735 } 1736 1737 if (flags & TCG_BSWAP_OS) { 1738 tcg_gen_shli_i64(t1, arg, 56); 1739 tcg_gen_sari_i64(t1, t1, 48); 1740 } else if (flags & TCG_BSWAP_OZ) { 1741 tcg_gen_ext8u_i64(t1, arg); 1742 tcg_gen_shli_i64(t1, t1, 8); 1743 } else { 1744 tcg_gen_shli_i64(t1, arg, 8); 1745 } 1746 1747 tcg_gen_or_i64(ret, t0, t1); 1748 tcg_temp_free_i64(t0); 1749 tcg_temp_free_i64(t1); 1750 } 1751 } 1752 1753 void tcg_gen_bswap32_i64(TCGv_i64 ret, TCGv_i64 arg, int flags) 1754 { 1755 /* Only one extension flag may be present. */ 1756 tcg_debug_assert(!(flags & TCG_BSWAP_OS) || !(flags & TCG_BSWAP_OZ)); 1757 1758 if (TCG_TARGET_REG_BITS == 32) { 1759 tcg_gen_bswap32_i32(TCGV_LOW(ret), TCGV_LOW(arg)); 1760 if (flags & TCG_BSWAP_OS) { 1761 tcg_gen_sari_i32(TCGV_HIGH(ret), TCGV_LOW(ret), 31); 1762 } else { 1763 tcg_gen_movi_i32(TCGV_HIGH(ret), 0); 1764 } 1765 } else if (TCG_TARGET_HAS_bswap32_i64) { 1766 tcg_gen_op3i_i64(INDEX_op_bswap32_i64, ret, arg, flags); 1767 } else { 1768 TCGv_i64 t0 = tcg_temp_ebb_new_i64(); 1769 TCGv_i64 t1 = tcg_temp_ebb_new_i64(); 1770 TCGv_i64 t2 = tcg_constant_i64(0x00ff00ff); 1771 1772 /* arg = xxxxabcd */ 1773 tcg_gen_shri_i64(t0, arg, 8); /* t0 = .xxxxabc */ 1774 tcg_gen_and_i64(t1, arg, t2); /* t1 = .....b.d */ 1775 tcg_gen_and_i64(t0, t0, t2); /* t0 = .....a.c */ 1776 tcg_gen_shli_i64(t1, t1, 8); /* t1 = ....b.d. */ 1777 tcg_gen_or_i64(ret, t0, t1); /* ret = ....badc */ 1778 1779 tcg_gen_shli_i64(t1, ret, 48); /* t1 = dc...... */ 1780 tcg_gen_shri_i64(t0, ret, 16); /* t0 = ......ba */ 1781 if (flags & TCG_BSWAP_OS) { 1782 tcg_gen_sari_i64(t1, t1, 32); /* t1 = ssssdc.. */ 1783 } else { 1784 tcg_gen_shri_i64(t1, t1, 32); /* t1 = ....dc.. */ 1785 } 1786 tcg_gen_or_i64(ret, t0, t1); /* ret = ssssdcba */ 1787 1788 tcg_temp_free_i64(t0); 1789 tcg_temp_free_i64(t1); 1790 } 1791 } 1792 1793 void tcg_gen_bswap64_i64(TCGv_i64 ret, TCGv_i64 arg) 1794 { 1795 if (TCG_TARGET_REG_BITS == 32) { 1796 TCGv_i32 t0, t1; 1797 t0 = tcg_temp_ebb_new_i32(); 1798 t1 = tcg_temp_ebb_new_i32(); 1799 1800 tcg_gen_bswap32_i32(t0, TCGV_LOW(arg)); 1801 tcg_gen_bswap32_i32(t1, TCGV_HIGH(arg)); 1802 tcg_gen_mov_i32(TCGV_LOW(ret), t1); 1803 tcg_gen_mov_i32(TCGV_HIGH(ret), t0); 1804 tcg_temp_free_i32(t0); 1805 tcg_temp_free_i32(t1); 1806 } else if (TCG_TARGET_HAS_bswap64_i64) { 1807 tcg_gen_op3i_i64(INDEX_op_bswap64_i64, ret, arg, 0); 1808 } else { 1809 TCGv_i64 t0 = tcg_temp_ebb_new_i64(); 1810 TCGv_i64 t1 = tcg_temp_ebb_new_i64(); 1811 TCGv_i64 t2 = tcg_temp_ebb_new_i64(); 1812 1813 /* arg = abcdefgh */ 1814 tcg_gen_movi_i64(t2, 0x00ff00ff00ff00ffull); 1815 tcg_gen_shri_i64(t0, arg, 8); /* t0 = .abcdefg */ 1816 tcg_gen_and_i64(t1, arg, t2); /* t1 = .b.d.f.h */ 1817 tcg_gen_and_i64(t0, t0, t2); /* t0 = .a.c.e.g */ 1818 tcg_gen_shli_i64(t1, t1, 8); /* t1 = b.d.f.h. */ 1819 tcg_gen_or_i64(ret, t0, t1); /* ret = badcfehg */ 1820 1821 tcg_gen_movi_i64(t2, 0x0000ffff0000ffffull); 1822 tcg_gen_shri_i64(t0, ret, 16); /* t0 = ..badcfe */ 1823 tcg_gen_and_i64(t1, ret, t2); /* t1 = ..dc..hg */ 1824 tcg_gen_and_i64(t0, t0, t2); /* t0 = ..ba..fe */ 1825 tcg_gen_shli_i64(t1, t1, 16); /* t1 = dc..hg.. */ 1826 tcg_gen_or_i64(ret, t0, t1); /* ret = dcbahgfe */ 1827 1828 tcg_gen_shri_i64(t0, ret, 32); /* t0 = ....dcba */ 1829 tcg_gen_shli_i64(t1, ret, 32); /* t1 = hgfe.... */ 1830 tcg_gen_or_i64(ret, t0, t1); /* ret = hgfedcba */ 1831 1832 tcg_temp_free_i64(t0); 1833 tcg_temp_free_i64(t1); 1834 tcg_temp_free_i64(t2); 1835 } 1836 } 1837 1838 void tcg_gen_hswap_i64(TCGv_i64 ret, TCGv_i64 arg) 1839 { 1840 uint64_t m = 0x0000ffff0000ffffull; 1841 TCGv_i64 t0 = tcg_temp_ebb_new_i64(); 1842 TCGv_i64 t1 = tcg_temp_ebb_new_i64(); 1843 1844 /* See include/qemu/bitops.h, hswap64. */ 1845 tcg_gen_rotli_i64(t1, arg, 32); 1846 tcg_gen_andi_i64(t0, t1, m); 1847 tcg_gen_shli_i64(t0, t0, 16); 1848 tcg_gen_shri_i64(t1, t1, 16); 1849 tcg_gen_andi_i64(t1, t1, m); 1850 tcg_gen_or_i64(ret, t0, t1); 1851 1852 tcg_temp_free_i64(t0); 1853 tcg_temp_free_i64(t1); 1854 } 1855 1856 void tcg_gen_wswap_i64(TCGv_i64 ret, TCGv_i64 arg) 1857 { 1858 /* Swapping 2 32-bit elements is a rotate. */ 1859 tcg_gen_rotli_i64(ret, arg, 32); 1860 } 1861 1862 void tcg_gen_not_i64(TCGv_i64 ret, TCGv_i64 arg) 1863 { 1864 if (TCG_TARGET_REG_BITS == 32) { 1865 tcg_gen_not_i32(TCGV_LOW(ret), TCGV_LOW(arg)); 1866 tcg_gen_not_i32(TCGV_HIGH(ret), TCGV_HIGH(arg)); 1867 } else if (TCG_TARGET_HAS_not_i64) { 1868 tcg_gen_op2_i64(INDEX_op_not_i64, ret, arg); 1869 } else { 1870 tcg_gen_xori_i64(ret, arg, -1); 1871 } 1872 } 1873 1874 void tcg_gen_andc_i64(TCGv_i64 ret, TCGv_i64 arg1, TCGv_i64 arg2) 1875 { 1876 if (TCG_TARGET_REG_BITS == 32) { 1877 tcg_gen_andc_i32(TCGV_LOW(ret), TCGV_LOW(arg1), TCGV_LOW(arg2)); 1878 tcg_gen_andc_i32(TCGV_HIGH(ret), TCGV_HIGH(arg1), TCGV_HIGH(arg2)); 1879 } else if (TCG_TARGET_HAS_andc_i64) { 1880 tcg_gen_op3_i64(INDEX_op_andc_i64, ret, arg1, arg2); 1881 } else { 1882 TCGv_i64 t0 = tcg_temp_ebb_new_i64(); 1883 tcg_gen_not_i64(t0, arg2); 1884 tcg_gen_and_i64(ret, arg1, t0); 1885 tcg_temp_free_i64(t0); 1886 } 1887 } 1888 1889 void tcg_gen_eqv_i64(TCGv_i64 ret, TCGv_i64 arg1, TCGv_i64 arg2) 1890 { 1891 if (TCG_TARGET_REG_BITS == 32) { 1892 tcg_gen_eqv_i32(TCGV_LOW(ret), TCGV_LOW(arg1), TCGV_LOW(arg2)); 1893 tcg_gen_eqv_i32(TCGV_HIGH(ret), TCGV_HIGH(arg1), TCGV_HIGH(arg2)); 1894 } else if (TCG_TARGET_HAS_eqv_i64) { 1895 tcg_gen_op3_i64(INDEX_op_eqv_i64, ret, arg1, arg2); 1896 } else { 1897 tcg_gen_xor_i64(ret, arg1, arg2); 1898 tcg_gen_not_i64(ret, ret); 1899 } 1900 } 1901 1902 void tcg_gen_nand_i64(TCGv_i64 ret, TCGv_i64 arg1, TCGv_i64 arg2) 1903 { 1904 if (TCG_TARGET_REG_BITS == 32) { 1905 tcg_gen_nand_i32(TCGV_LOW(ret), TCGV_LOW(arg1), TCGV_LOW(arg2)); 1906 tcg_gen_nand_i32(TCGV_HIGH(ret), TCGV_HIGH(arg1), TCGV_HIGH(arg2)); 1907 } else if (TCG_TARGET_HAS_nand_i64) { 1908 tcg_gen_op3_i64(INDEX_op_nand_i64, ret, arg1, arg2); 1909 } else { 1910 tcg_gen_and_i64(ret, arg1, arg2); 1911 tcg_gen_not_i64(ret, ret); 1912 } 1913 } 1914 1915 void tcg_gen_nor_i64(TCGv_i64 ret, TCGv_i64 arg1, TCGv_i64 arg2) 1916 { 1917 if (TCG_TARGET_REG_BITS == 32) { 1918 tcg_gen_nor_i32(TCGV_LOW(ret), TCGV_LOW(arg1), TCGV_LOW(arg2)); 1919 tcg_gen_nor_i32(TCGV_HIGH(ret), TCGV_HIGH(arg1), TCGV_HIGH(arg2)); 1920 } else if (TCG_TARGET_HAS_nor_i64) { 1921 tcg_gen_op3_i64(INDEX_op_nor_i64, ret, arg1, arg2); 1922 } else { 1923 tcg_gen_or_i64(ret, arg1, arg2); 1924 tcg_gen_not_i64(ret, ret); 1925 } 1926 } 1927 1928 void tcg_gen_orc_i64(TCGv_i64 ret, TCGv_i64 arg1, TCGv_i64 arg2) 1929 { 1930 if (TCG_TARGET_REG_BITS == 32) { 1931 tcg_gen_orc_i32(TCGV_LOW(ret), TCGV_LOW(arg1), TCGV_LOW(arg2)); 1932 tcg_gen_orc_i32(TCGV_HIGH(ret), TCGV_HIGH(arg1), TCGV_HIGH(arg2)); 1933 } else if (TCG_TARGET_HAS_orc_i64) { 1934 tcg_gen_op3_i64(INDEX_op_orc_i64, ret, arg1, arg2); 1935 } else { 1936 TCGv_i64 t0 = tcg_temp_ebb_new_i64(); 1937 tcg_gen_not_i64(t0, arg2); 1938 tcg_gen_or_i64(ret, arg1, t0); 1939 tcg_temp_free_i64(t0); 1940 } 1941 } 1942 1943 void tcg_gen_clz_i64(TCGv_i64 ret, TCGv_i64 arg1, TCGv_i64 arg2) 1944 { 1945 if (TCG_TARGET_HAS_clz_i64) { 1946 tcg_gen_op3_i64(INDEX_op_clz_i64, ret, arg1, arg2); 1947 } else { 1948 gen_helper_clz_i64(ret, arg1, arg2); 1949 } 1950 } 1951 1952 void tcg_gen_clzi_i64(TCGv_i64 ret, TCGv_i64 arg1, uint64_t arg2) 1953 { 1954 if (TCG_TARGET_REG_BITS == 32 1955 && TCG_TARGET_HAS_clz_i32 1956 && arg2 <= 0xffffffffu) { 1957 TCGv_i32 t = tcg_temp_ebb_new_i32(); 1958 tcg_gen_clzi_i32(t, TCGV_LOW(arg1), arg2 - 32); 1959 tcg_gen_addi_i32(t, t, 32); 1960 tcg_gen_clz_i32(TCGV_LOW(ret), TCGV_HIGH(arg1), t); 1961 tcg_gen_movi_i32(TCGV_HIGH(ret), 0); 1962 tcg_temp_free_i32(t); 1963 } else { 1964 TCGv_i64 t0 = tcg_const_i64(arg2); 1965 tcg_gen_clz_i64(ret, arg1, t0); 1966 tcg_temp_free_i64(t0); 1967 } 1968 } 1969 1970 void tcg_gen_ctz_i64(TCGv_i64 ret, TCGv_i64 arg1, TCGv_i64 arg2) 1971 { 1972 if (TCG_TARGET_HAS_ctz_i64) { 1973 tcg_gen_op3_i64(INDEX_op_ctz_i64, ret, arg1, arg2); 1974 } else if (TCG_TARGET_HAS_ctpop_i64 || TCG_TARGET_HAS_clz_i64) { 1975 TCGv_i64 z, t = tcg_temp_ebb_new_i64(); 1976 1977 if (TCG_TARGET_HAS_ctpop_i64) { 1978 tcg_gen_subi_i64(t, arg1, 1); 1979 tcg_gen_andc_i64(t, t, arg1); 1980 tcg_gen_ctpop_i64(t, t); 1981 } else { 1982 /* Since all non-x86 hosts have clz(0) == 64, don't fight it. */ 1983 tcg_gen_neg_i64(t, arg1); 1984 tcg_gen_and_i64(t, t, arg1); 1985 tcg_gen_clzi_i64(t, t, 64); 1986 tcg_gen_xori_i64(t, t, 63); 1987 } 1988 z = tcg_constant_i64(0); 1989 tcg_gen_movcond_i64(TCG_COND_EQ, ret, arg1, z, arg2, t); 1990 tcg_temp_free_i64(t); 1991 tcg_temp_free_i64(z); 1992 } else { 1993 gen_helper_ctz_i64(ret, arg1, arg2); 1994 } 1995 } 1996 1997 void tcg_gen_ctzi_i64(TCGv_i64 ret, TCGv_i64 arg1, uint64_t arg2) 1998 { 1999 if (TCG_TARGET_REG_BITS == 32 2000 && TCG_TARGET_HAS_ctz_i32 2001 && arg2 <= 0xffffffffu) { 2002 TCGv_i32 t32 = tcg_temp_ebb_new_i32(); 2003 tcg_gen_ctzi_i32(t32, TCGV_HIGH(arg1), arg2 - 32); 2004 tcg_gen_addi_i32(t32, t32, 32); 2005 tcg_gen_ctz_i32(TCGV_LOW(ret), TCGV_LOW(arg1), t32); 2006 tcg_gen_movi_i32(TCGV_HIGH(ret), 0); 2007 tcg_temp_free_i32(t32); 2008 } else if (!TCG_TARGET_HAS_ctz_i64 2009 && TCG_TARGET_HAS_ctpop_i64 2010 && arg2 == 64) { 2011 /* This equivalence has the advantage of not requiring a fixup. */ 2012 TCGv_i64 t = tcg_temp_ebb_new_i64(); 2013 tcg_gen_subi_i64(t, arg1, 1); 2014 tcg_gen_andc_i64(t, t, arg1); 2015 tcg_gen_ctpop_i64(ret, t); 2016 tcg_temp_free_i64(t); 2017 } else { 2018 TCGv_i64 t0 = tcg_const_i64(arg2); 2019 tcg_gen_ctz_i64(ret, arg1, t0); 2020 tcg_temp_free_i64(t0); 2021 } 2022 } 2023 2024 void tcg_gen_clrsb_i64(TCGv_i64 ret, TCGv_i64 arg) 2025 { 2026 if (TCG_TARGET_HAS_clz_i64 || TCG_TARGET_HAS_clz_i32) { 2027 TCGv_i64 t = tcg_temp_ebb_new_i64(); 2028 tcg_gen_sari_i64(t, arg, 63); 2029 tcg_gen_xor_i64(t, t, arg); 2030 tcg_gen_clzi_i64(t, t, 64); 2031 tcg_gen_subi_i64(ret, t, 1); 2032 tcg_temp_free_i64(t); 2033 } else { 2034 gen_helper_clrsb_i64(ret, arg); 2035 } 2036 } 2037 2038 void tcg_gen_ctpop_i64(TCGv_i64 ret, TCGv_i64 arg1) 2039 { 2040 if (TCG_TARGET_HAS_ctpop_i64) { 2041 tcg_gen_op2_i64(INDEX_op_ctpop_i64, ret, arg1); 2042 } else if (TCG_TARGET_REG_BITS == 32 && TCG_TARGET_HAS_ctpop_i32) { 2043 tcg_gen_ctpop_i32(TCGV_HIGH(ret), TCGV_HIGH(arg1)); 2044 tcg_gen_ctpop_i32(TCGV_LOW(ret), TCGV_LOW(arg1)); 2045 tcg_gen_add_i32(TCGV_LOW(ret), TCGV_LOW(ret), TCGV_HIGH(ret)); 2046 tcg_gen_movi_i32(TCGV_HIGH(ret), 0); 2047 } else { 2048 gen_helper_ctpop_i64(ret, arg1); 2049 } 2050 } 2051 2052 void tcg_gen_rotl_i64(TCGv_i64 ret, TCGv_i64 arg1, TCGv_i64 arg2) 2053 { 2054 if (TCG_TARGET_HAS_rot_i64) { 2055 tcg_gen_op3_i64(INDEX_op_rotl_i64, ret, arg1, arg2); 2056 } else { 2057 TCGv_i64 t0, t1; 2058 t0 = tcg_temp_ebb_new_i64(); 2059 t1 = tcg_temp_ebb_new_i64(); 2060 tcg_gen_shl_i64(t0, arg1, arg2); 2061 tcg_gen_subfi_i64(t1, 64, arg2); 2062 tcg_gen_shr_i64(t1, arg1, t1); 2063 tcg_gen_or_i64(ret, t0, t1); 2064 tcg_temp_free_i64(t0); 2065 tcg_temp_free_i64(t1); 2066 } 2067 } 2068 2069 void tcg_gen_rotli_i64(TCGv_i64 ret, TCGv_i64 arg1, int64_t arg2) 2070 { 2071 tcg_debug_assert(arg2 >= 0 && arg2 < 64); 2072 /* some cases can be optimized here */ 2073 if (arg2 == 0) { 2074 tcg_gen_mov_i64(ret, arg1); 2075 } else if (TCG_TARGET_HAS_rot_i64) { 2076 tcg_gen_rotl_i64(ret, arg1, tcg_constant_i64(arg2)); 2077 } else { 2078 TCGv_i64 t0, t1; 2079 t0 = tcg_temp_ebb_new_i64(); 2080 t1 = tcg_temp_ebb_new_i64(); 2081 tcg_gen_shli_i64(t0, arg1, arg2); 2082 tcg_gen_shri_i64(t1, arg1, 64 - arg2); 2083 tcg_gen_or_i64(ret, t0, t1); 2084 tcg_temp_free_i64(t0); 2085 tcg_temp_free_i64(t1); 2086 } 2087 } 2088 2089 void tcg_gen_rotr_i64(TCGv_i64 ret, TCGv_i64 arg1, TCGv_i64 arg2) 2090 { 2091 if (TCG_TARGET_HAS_rot_i64) { 2092 tcg_gen_op3_i64(INDEX_op_rotr_i64, ret, arg1, arg2); 2093 } else { 2094 TCGv_i64 t0, t1; 2095 t0 = tcg_temp_ebb_new_i64(); 2096 t1 = tcg_temp_ebb_new_i64(); 2097 tcg_gen_shr_i64(t0, arg1, arg2); 2098 tcg_gen_subfi_i64(t1, 64, arg2); 2099 tcg_gen_shl_i64(t1, arg1, t1); 2100 tcg_gen_or_i64(ret, t0, t1); 2101 tcg_temp_free_i64(t0); 2102 tcg_temp_free_i64(t1); 2103 } 2104 } 2105 2106 void tcg_gen_rotri_i64(TCGv_i64 ret, TCGv_i64 arg1, int64_t arg2) 2107 { 2108 tcg_debug_assert(arg2 >= 0 && arg2 < 64); 2109 /* some cases can be optimized here */ 2110 if (arg2 == 0) { 2111 tcg_gen_mov_i64(ret, arg1); 2112 } else { 2113 tcg_gen_rotli_i64(ret, arg1, 64 - arg2); 2114 } 2115 } 2116 2117 void tcg_gen_deposit_i64(TCGv_i64 ret, TCGv_i64 arg1, TCGv_i64 arg2, 2118 unsigned int ofs, unsigned int len) 2119 { 2120 uint64_t mask; 2121 TCGv_i64 t1; 2122 2123 tcg_debug_assert(ofs < 64); 2124 tcg_debug_assert(len > 0); 2125 tcg_debug_assert(len <= 64); 2126 tcg_debug_assert(ofs + len <= 64); 2127 2128 if (len == 64) { 2129 tcg_gen_mov_i64(ret, arg2); 2130 return; 2131 } 2132 if (TCG_TARGET_HAS_deposit_i64 && TCG_TARGET_deposit_i64_valid(ofs, len)) { 2133 tcg_gen_op5ii_i64(INDEX_op_deposit_i64, ret, arg1, arg2, ofs, len); 2134 return; 2135 } 2136 2137 if (TCG_TARGET_REG_BITS == 32) { 2138 if (ofs >= 32) { 2139 tcg_gen_deposit_i32(TCGV_HIGH(ret), TCGV_HIGH(arg1), 2140 TCGV_LOW(arg2), ofs - 32, len); 2141 tcg_gen_mov_i32(TCGV_LOW(ret), TCGV_LOW(arg1)); 2142 return; 2143 } 2144 if (ofs + len <= 32) { 2145 tcg_gen_deposit_i32(TCGV_LOW(ret), TCGV_LOW(arg1), 2146 TCGV_LOW(arg2), ofs, len); 2147 tcg_gen_mov_i32(TCGV_HIGH(ret), TCGV_HIGH(arg1)); 2148 return; 2149 } 2150 } 2151 2152 t1 = tcg_temp_ebb_new_i64(); 2153 2154 if (TCG_TARGET_HAS_extract2_i64) { 2155 if (ofs + len == 64) { 2156 tcg_gen_shli_i64(t1, arg1, len); 2157 tcg_gen_extract2_i64(ret, t1, arg2, len); 2158 goto done; 2159 } 2160 if (ofs == 0) { 2161 tcg_gen_extract2_i64(ret, arg1, arg2, len); 2162 tcg_gen_rotli_i64(ret, ret, len); 2163 goto done; 2164 } 2165 } 2166 2167 mask = (1ull << len) - 1; 2168 if (ofs + len < 64) { 2169 tcg_gen_andi_i64(t1, arg2, mask); 2170 tcg_gen_shli_i64(t1, t1, ofs); 2171 } else { 2172 tcg_gen_shli_i64(t1, arg2, ofs); 2173 } 2174 tcg_gen_andi_i64(ret, arg1, ~(mask << ofs)); 2175 tcg_gen_or_i64(ret, ret, t1); 2176 done: 2177 tcg_temp_free_i64(t1); 2178 } 2179 2180 void tcg_gen_deposit_z_i64(TCGv_i64 ret, TCGv_i64 arg, 2181 unsigned int ofs, unsigned int len) 2182 { 2183 tcg_debug_assert(ofs < 64); 2184 tcg_debug_assert(len > 0); 2185 tcg_debug_assert(len <= 64); 2186 tcg_debug_assert(ofs + len <= 64); 2187 2188 if (ofs + len == 64) { 2189 tcg_gen_shli_i64(ret, arg, ofs); 2190 } else if (ofs == 0) { 2191 tcg_gen_andi_i64(ret, arg, (1ull << len) - 1); 2192 } else if (TCG_TARGET_HAS_deposit_i64 2193 && TCG_TARGET_deposit_i64_valid(ofs, len)) { 2194 TCGv_i64 zero = tcg_constant_i64(0); 2195 tcg_gen_op5ii_i64(INDEX_op_deposit_i64, ret, zero, arg, ofs, len); 2196 } else { 2197 if (TCG_TARGET_REG_BITS == 32) { 2198 if (ofs >= 32) { 2199 tcg_gen_deposit_z_i32(TCGV_HIGH(ret), TCGV_LOW(arg), 2200 ofs - 32, len); 2201 tcg_gen_movi_i32(TCGV_LOW(ret), 0); 2202 return; 2203 } 2204 if (ofs + len <= 32) { 2205 tcg_gen_deposit_z_i32(TCGV_LOW(ret), TCGV_LOW(arg), ofs, len); 2206 tcg_gen_movi_i32(TCGV_HIGH(ret), 0); 2207 return; 2208 } 2209 } 2210 /* To help two-operand hosts we prefer to zero-extend first, 2211 which allows ARG to stay live. */ 2212 switch (len) { 2213 case 32: 2214 if (TCG_TARGET_HAS_ext32u_i64) { 2215 tcg_gen_ext32u_i64(ret, arg); 2216 tcg_gen_shli_i64(ret, ret, ofs); 2217 return; 2218 } 2219 break; 2220 case 16: 2221 if (TCG_TARGET_HAS_ext16u_i64) { 2222 tcg_gen_ext16u_i64(ret, arg); 2223 tcg_gen_shli_i64(ret, ret, ofs); 2224 return; 2225 } 2226 break; 2227 case 8: 2228 if (TCG_TARGET_HAS_ext8u_i64) { 2229 tcg_gen_ext8u_i64(ret, arg); 2230 tcg_gen_shli_i64(ret, ret, ofs); 2231 return; 2232 } 2233 break; 2234 } 2235 /* Otherwise prefer zero-extension over AND for code size. */ 2236 switch (ofs + len) { 2237 case 32: 2238 if (TCG_TARGET_HAS_ext32u_i64) { 2239 tcg_gen_shli_i64(ret, arg, ofs); 2240 tcg_gen_ext32u_i64(ret, ret); 2241 return; 2242 } 2243 break; 2244 case 16: 2245 if (TCG_TARGET_HAS_ext16u_i64) { 2246 tcg_gen_shli_i64(ret, arg, ofs); 2247 tcg_gen_ext16u_i64(ret, ret); 2248 return; 2249 } 2250 break; 2251 case 8: 2252 if (TCG_TARGET_HAS_ext8u_i64) { 2253 tcg_gen_shli_i64(ret, arg, ofs); 2254 tcg_gen_ext8u_i64(ret, ret); 2255 return; 2256 } 2257 break; 2258 } 2259 tcg_gen_andi_i64(ret, arg, (1ull << len) - 1); 2260 tcg_gen_shli_i64(ret, ret, ofs); 2261 } 2262 } 2263 2264 void tcg_gen_extract_i64(TCGv_i64 ret, TCGv_i64 arg, 2265 unsigned int ofs, unsigned int len) 2266 { 2267 tcg_debug_assert(ofs < 64); 2268 tcg_debug_assert(len > 0); 2269 tcg_debug_assert(len <= 64); 2270 tcg_debug_assert(ofs + len <= 64); 2271 2272 /* Canonicalize certain special cases, even if extract is supported. */ 2273 if (ofs + len == 64) { 2274 tcg_gen_shri_i64(ret, arg, 64 - len); 2275 return; 2276 } 2277 if (ofs == 0) { 2278 tcg_gen_andi_i64(ret, arg, (1ull << len) - 1); 2279 return; 2280 } 2281 2282 if (TCG_TARGET_REG_BITS == 32) { 2283 /* Look for a 32-bit extract within one of the two words. */ 2284 if (ofs >= 32) { 2285 tcg_gen_extract_i32(TCGV_LOW(ret), TCGV_HIGH(arg), ofs - 32, len); 2286 tcg_gen_movi_i32(TCGV_HIGH(ret), 0); 2287 return; 2288 } 2289 if (ofs + len <= 32) { 2290 tcg_gen_extract_i32(TCGV_LOW(ret), TCGV_LOW(arg), ofs, len); 2291 tcg_gen_movi_i32(TCGV_HIGH(ret), 0); 2292 return; 2293 } 2294 /* The field is split across two words. One double-word 2295 shift is better than two double-word shifts. */ 2296 goto do_shift_and; 2297 } 2298 2299 if (TCG_TARGET_HAS_extract_i64 2300 && TCG_TARGET_extract_i64_valid(ofs, len)) { 2301 tcg_gen_op4ii_i64(INDEX_op_extract_i64, ret, arg, ofs, len); 2302 return; 2303 } 2304 2305 /* Assume that zero-extension, if available, is cheaper than a shift. */ 2306 switch (ofs + len) { 2307 case 32: 2308 if (TCG_TARGET_HAS_ext32u_i64) { 2309 tcg_gen_ext32u_i64(ret, arg); 2310 tcg_gen_shri_i64(ret, ret, ofs); 2311 return; 2312 } 2313 break; 2314 case 16: 2315 if (TCG_TARGET_HAS_ext16u_i64) { 2316 tcg_gen_ext16u_i64(ret, arg); 2317 tcg_gen_shri_i64(ret, ret, ofs); 2318 return; 2319 } 2320 break; 2321 case 8: 2322 if (TCG_TARGET_HAS_ext8u_i64) { 2323 tcg_gen_ext8u_i64(ret, arg); 2324 tcg_gen_shri_i64(ret, ret, ofs); 2325 return; 2326 } 2327 break; 2328 } 2329 2330 /* ??? Ideally we'd know what values are available for immediate AND. 2331 Assume that 8 bits are available, plus the special cases of 16 and 32, 2332 so that we get ext8u, ext16u, and ext32u. */ 2333 switch (len) { 2334 case 1 ... 8: case 16: case 32: 2335 do_shift_and: 2336 tcg_gen_shri_i64(ret, arg, ofs); 2337 tcg_gen_andi_i64(ret, ret, (1ull << len) - 1); 2338 break; 2339 default: 2340 tcg_gen_shli_i64(ret, arg, 64 - len - ofs); 2341 tcg_gen_shri_i64(ret, ret, 64 - len); 2342 break; 2343 } 2344 } 2345 2346 void tcg_gen_sextract_i64(TCGv_i64 ret, TCGv_i64 arg, 2347 unsigned int ofs, unsigned int len) 2348 { 2349 tcg_debug_assert(ofs < 64); 2350 tcg_debug_assert(len > 0); 2351 tcg_debug_assert(len <= 64); 2352 tcg_debug_assert(ofs + len <= 64); 2353 2354 /* Canonicalize certain special cases, even if sextract is supported. */ 2355 if (ofs + len == 64) { 2356 tcg_gen_sari_i64(ret, arg, 64 - len); 2357 return; 2358 } 2359 if (ofs == 0) { 2360 switch (len) { 2361 case 32: 2362 tcg_gen_ext32s_i64(ret, arg); 2363 return; 2364 case 16: 2365 tcg_gen_ext16s_i64(ret, arg); 2366 return; 2367 case 8: 2368 tcg_gen_ext8s_i64(ret, arg); 2369 return; 2370 } 2371 } 2372 2373 if (TCG_TARGET_REG_BITS == 32) { 2374 /* Look for a 32-bit extract within one of the two words. */ 2375 if (ofs >= 32) { 2376 tcg_gen_sextract_i32(TCGV_LOW(ret), TCGV_HIGH(arg), ofs - 32, len); 2377 } else if (ofs + len <= 32) { 2378 tcg_gen_sextract_i32(TCGV_LOW(ret), TCGV_LOW(arg), ofs, len); 2379 } else if (ofs == 0) { 2380 tcg_gen_mov_i32(TCGV_LOW(ret), TCGV_LOW(arg)); 2381 tcg_gen_sextract_i32(TCGV_HIGH(ret), TCGV_HIGH(arg), 0, len - 32); 2382 return; 2383 } else if (len > 32) { 2384 TCGv_i32 t = tcg_temp_ebb_new_i32(); 2385 /* Extract the bits for the high word normally. */ 2386 tcg_gen_sextract_i32(t, TCGV_HIGH(arg), ofs + 32, len - 32); 2387 /* Shift the field down for the low part. */ 2388 tcg_gen_shri_i64(ret, arg, ofs); 2389 /* Overwrite the shift into the high part. */ 2390 tcg_gen_mov_i32(TCGV_HIGH(ret), t); 2391 tcg_temp_free_i32(t); 2392 return; 2393 } else { 2394 /* Shift the field down for the low part, such that the 2395 field sits at the MSB. */ 2396 tcg_gen_shri_i64(ret, arg, ofs + len - 32); 2397 /* Shift the field down from the MSB, sign extending. */ 2398 tcg_gen_sari_i32(TCGV_LOW(ret), TCGV_LOW(ret), 32 - len); 2399 } 2400 /* Sign-extend the field from 32 bits. */ 2401 tcg_gen_sari_i32(TCGV_HIGH(ret), TCGV_LOW(ret), 31); 2402 return; 2403 } 2404 2405 if (TCG_TARGET_HAS_sextract_i64 2406 && TCG_TARGET_extract_i64_valid(ofs, len)) { 2407 tcg_gen_op4ii_i64(INDEX_op_sextract_i64, ret, arg, ofs, len); 2408 return; 2409 } 2410 2411 /* Assume that sign-extension, if available, is cheaper than a shift. */ 2412 switch (ofs + len) { 2413 case 32: 2414 if (TCG_TARGET_HAS_ext32s_i64) { 2415 tcg_gen_ext32s_i64(ret, arg); 2416 tcg_gen_sari_i64(ret, ret, ofs); 2417 return; 2418 } 2419 break; 2420 case 16: 2421 if (TCG_TARGET_HAS_ext16s_i64) { 2422 tcg_gen_ext16s_i64(ret, arg); 2423 tcg_gen_sari_i64(ret, ret, ofs); 2424 return; 2425 } 2426 break; 2427 case 8: 2428 if (TCG_TARGET_HAS_ext8s_i64) { 2429 tcg_gen_ext8s_i64(ret, arg); 2430 tcg_gen_sari_i64(ret, ret, ofs); 2431 return; 2432 } 2433 break; 2434 } 2435 switch (len) { 2436 case 32: 2437 if (TCG_TARGET_HAS_ext32s_i64) { 2438 tcg_gen_shri_i64(ret, arg, ofs); 2439 tcg_gen_ext32s_i64(ret, ret); 2440 return; 2441 } 2442 break; 2443 case 16: 2444 if (TCG_TARGET_HAS_ext16s_i64) { 2445 tcg_gen_shri_i64(ret, arg, ofs); 2446 tcg_gen_ext16s_i64(ret, ret); 2447 return; 2448 } 2449 break; 2450 case 8: 2451 if (TCG_TARGET_HAS_ext8s_i64) { 2452 tcg_gen_shri_i64(ret, arg, ofs); 2453 tcg_gen_ext8s_i64(ret, ret); 2454 return; 2455 } 2456 break; 2457 } 2458 tcg_gen_shli_i64(ret, arg, 64 - len - ofs); 2459 tcg_gen_sari_i64(ret, ret, 64 - len); 2460 } 2461 2462 /* 2463 * Extract 64 bits from a 128-bit input, ah:al, starting from ofs. 2464 * Unlike tcg_gen_extract_i64 above, len is fixed at 64. 2465 */ 2466 void tcg_gen_extract2_i64(TCGv_i64 ret, TCGv_i64 al, TCGv_i64 ah, 2467 unsigned int ofs) 2468 { 2469 tcg_debug_assert(ofs <= 64); 2470 if (ofs == 0) { 2471 tcg_gen_mov_i64(ret, al); 2472 } else if (ofs == 64) { 2473 tcg_gen_mov_i64(ret, ah); 2474 } else if (al == ah) { 2475 tcg_gen_rotri_i64(ret, al, ofs); 2476 } else if (TCG_TARGET_HAS_extract2_i64) { 2477 tcg_gen_op4i_i64(INDEX_op_extract2_i64, ret, al, ah, ofs); 2478 } else { 2479 TCGv_i64 t0 = tcg_temp_ebb_new_i64(); 2480 tcg_gen_shri_i64(t0, al, ofs); 2481 tcg_gen_deposit_i64(ret, t0, ah, 64 - ofs, ofs); 2482 tcg_temp_free_i64(t0); 2483 } 2484 } 2485 2486 void tcg_gen_movcond_i64(TCGCond cond, TCGv_i64 ret, TCGv_i64 c1, 2487 TCGv_i64 c2, TCGv_i64 v1, TCGv_i64 v2) 2488 { 2489 if (cond == TCG_COND_ALWAYS) { 2490 tcg_gen_mov_i64(ret, v1); 2491 } else if (cond == TCG_COND_NEVER) { 2492 tcg_gen_mov_i64(ret, v2); 2493 } else if (TCG_TARGET_REG_BITS == 32) { 2494 TCGv_i32 t0 = tcg_temp_ebb_new_i32(); 2495 TCGv_i32 t1 = tcg_temp_ebb_new_i32(); 2496 tcg_gen_op6i_i32(INDEX_op_setcond2_i32, t0, 2497 TCGV_LOW(c1), TCGV_HIGH(c1), 2498 TCGV_LOW(c2), TCGV_HIGH(c2), cond); 2499 2500 if (TCG_TARGET_HAS_movcond_i32) { 2501 tcg_gen_movi_i32(t1, 0); 2502 tcg_gen_movcond_i32(TCG_COND_NE, TCGV_LOW(ret), t0, t1, 2503 TCGV_LOW(v1), TCGV_LOW(v2)); 2504 tcg_gen_movcond_i32(TCG_COND_NE, TCGV_HIGH(ret), t0, t1, 2505 TCGV_HIGH(v1), TCGV_HIGH(v2)); 2506 } else { 2507 tcg_gen_neg_i32(t0, t0); 2508 2509 tcg_gen_and_i32(t1, TCGV_LOW(v1), t0); 2510 tcg_gen_andc_i32(TCGV_LOW(ret), TCGV_LOW(v2), t0); 2511 tcg_gen_or_i32(TCGV_LOW(ret), TCGV_LOW(ret), t1); 2512 2513 tcg_gen_and_i32(t1, TCGV_HIGH(v1), t0); 2514 tcg_gen_andc_i32(TCGV_HIGH(ret), TCGV_HIGH(v2), t0); 2515 tcg_gen_or_i32(TCGV_HIGH(ret), TCGV_HIGH(ret), t1); 2516 } 2517 tcg_temp_free_i32(t0); 2518 tcg_temp_free_i32(t1); 2519 } else if (TCG_TARGET_HAS_movcond_i64) { 2520 tcg_gen_op6i_i64(INDEX_op_movcond_i64, ret, c1, c2, v1, v2, cond); 2521 } else { 2522 TCGv_i64 t0 = tcg_temp_ebb_new_i64(); 2523 TCGv_i64 t1 = tcg_temp_ebb_new_i64(); 2524 tcg_gen_setcond_i64(cond, t0, c1, c2); 2525 tcg_gen_neg_i64(t0, t0); 2526 tcg_gen_and_i64(t1, v1, t0); 2527 tcg_gen_andc_i64(ret, v2, t0); 2528 tcg_gen_or_i64(ret, ret, t1); 2529 tcg_temp_free_i64(t0); 2530 tcg_temp_free_i64(t1); 2531 } 2532 } 2533 2534 void tcg_gen_add2_i64(TCGv_i64 rl, TCGv_i64 rh, TCGv_i64 al, 2535 TCGv_i64 ah, TCGv_i64 bl, TCGv_i64 bh) 2536 { 2537 if (TCG_TARGET_HAS_add2_i64) { 2538 tcg_gen_op6_i64(INDEX_op_add2_i64, rl, rh, al, ah, bl, bh); 2539 } else { 2540 TCGv_i64 t0 = tcg_temp_ebb_new_i64(); 2541 TCGv_i64 t1 = tcg_temp_ebb_new_i64(); 2542 tcg_gen_add_i64(t0, al, bl); 2543 tcg_gen_setcond_i64(TCG_COND_LTU, t1, t0, al); 2544 tcg_gen_add_i64(rh, ah, bh); 2545 tcg_gen_add_i64(rh, rh, t1); 2546 tcg_gen_mov_i64(rl, t0); 2547 tcg_temp_free_i64(t0); 2548 tcg_temp_free_i64(t1); 2549 } 2550 } 2551 2552 void tcg_gen_sub2_i64(TCGv_i64 rl, TCGv_i64 rh, TCGv_i64 al, 2553 TCGv_i64 ah, TCGv_i64 bl, TCGv_i64 bh) 2554 { 2555 if (TCG_TARGET_HAS_sub2_i64) { 2556 tcg_gen_op6_i64(INDEX_op_sub2_i64, rl, rh, al, ah, bl, bh); 2557 } else { 2558 TCGv_i64 t0 = tcg_temp_ebb_new_i64(); 2559 TCGv_i64 t1 = tcg_temp_ebb_new_i64(); 2560 tcg_gen_sub_i64(t0, al, bl); 2561 tcg_gen_setcond_i64(TCG_COND_LTU, t1, al, bl); 2562 tcg_gen_sub_i64(rh, ah, bh); 2563 tcg_gen_sub_i64(rh, rh, t1); 2564 tcg_gen_mov_i64(rl, t0); 2565 tcg_temp_free_i64(t0); 2566 tcg_temp_free_i64(t1); 2567 } 2568 } 2569 2570 void tcg_gen_mulu2_i64(TCGv_i64 rl, TCGv_i64 rh, TCGv_i64 arg1, TCGv_i64 arg2) 2571 { 2572 if (TCG_TARGET_HAS_mulu2_i64) { 2573 tcg_gen_op4_i64(INDEX_op_mulu2_i64, rl, rh, arg1, arg2); 2574 } else if (TCG_TARGET_HAS_muluh_i64) { 2575 TCGv_i64 t = tcg_temp_ebb_new_i64(); 2576 tcg_gen_op3_i64(INDEX_op_mul_i64, t, arg1, arg2); 2577 tcg_gen_op3_i64(INDEX_op_muluh_i64, rh, arg1, arg2); 2578 tcg_gen_mov_i64(rl, t); 2579 tcg_temp_free_i64(t); 2580 } else { 2581 TCGv_i64 t0 = tcg_temp_ebb_new_i64(); 2582 tcg_gen_mul_i64(t0, arg1, arg2); 2583 gen_helper_muluh_i64(rh, arg1, arg2); 2584 tcg_gen_mov_i64(rl, t0); 2585 tcg_temp_free_i64(t0); 2586 } 2587 } 2588 2589 void tcg_gen_muls2_i64(TCGv_i64 rl, TCGv_i64 rh, TCGv_i64 arg1, TCGv_i64 arg2) 2590 { 2591 if (TCG_TARGET_HAS_muls2_i64) { 2592 tcg_gen_op4_i64(INDEX_op_muls2_i64, rl, rh, arg1, arg2); 2593 } else if (TCG_TARGET_HAS_mulsh_i64) { 2594 TCGv_i64 t = tcg_temp_ebb_new_i64(); 2595 tcg_gen_op3_i64(INDEX_op_mul_i64, t, arg1, arg2); 2596 tcg_gen_op3_i64(INDEX_op_mulsh_i64, rh, arg1, arg2); 2597 tcg_gen_mov_i64(rl, t); 2598 tcg_temp_free_i64(t); 2599 } else if (TCG_TARGET_HAS_mulu2_i64 || TCG_TARGET_HAS_muluh_i64) { 2600 TCGv_i64 t0 = tcg_temp_ebb_new_i64(); 2601 TCGv_i64 t1 = tcg_temp_ebb_new_i64(); 2602 TCGv_i64 t2 = tcg_temp_ebb_new_i64(); 2603 TCGv_i64 t3 = tcg_temp_ebb_new_i64(); 2604 tcg_gen_mulu2_i64(t0, t1, arg1, arg2); 2605 /* Adjust for negative inputs. */ 2606 tcg_gen_sari_i64(t2, arg1, 63); 2607 tcg_gen_sari_i64(t3, arg2, 63); 2608 tcg_gen_and_i64(t2, t2, arg2); 2609 tcg_gen_and_i64(t3, t3, arg1); 2610 tcg_gen_sub_i64(rh, t1, t2); 2611 tcg_gen_sub_i64(rh, rh, t3); 2612 tcg_gen_mov_i64(rl, t0); 2613 tcg_temp_free_i64(t0); 2614 tcg_temp_free_i64(t1); 2615 tcg_temp_free_i64(t2); 2616 tcg_temp_free_i64(t3); 2617 } else { 2618 TCGv_i64 t0 = tcg_temp_ebb_new_i64(); 2619 tcg_gen_mul_i64(t0, arg1, arg2); 2620 gen_helper_mulsh_i64(rh, arg1, arg2); 2621 tcg_gen_mov_i64(rl, t0); 2622 tcg_temp_free_i64(t0); 2623 } 2624 } 2625 2626 void tcg_gen_mulsu2_i64(TCGv_i64 rl, TCGv_i64 rh, TCGv_i64 arg1, TCGv_i64 arg2) 2627 { 2628 TCGv_i64 t0 = tcg_temp_ebb_new_i64(); 2629 TCGv_i64 t1 = tcg_temp_ebb_new_i64(); 2630 TCGv_i64 t2 = tcg_temp_ebb_new_i64(); 2631 tcg_gen_mulu2_i64(t0, t1, arg1, arg2); 2632 /* Adjust for negative input for the signed arg1. */ 2633 tcg_gen_sari_i64(t2, arg1, 63); 2634 tcg_gen_and_i64(t2, t2, arg2); 2635 tcg_gen_sub_i64(rh, t1, t2); 2636 tcg_gen_mov_i64(rl, t0); 2637 tcg_temp_free_i64(t0); 2638 tcg_temp_free_i64(t1); 2639 tcg_temp_free_i64(t2); 2640 } 2641 2642 void tcg_gen_smin_i64(TCGv_i64 ret, TCGv_i64 a, TCGv_i64 b) 2643 { 2644 tcg_gen_movcond_i64(TCG_COND_LT, ret, a, b, a, b); 2645 } 2646 2647 void tcg_gen_umin_i64(TCGv_i64 ret, TCGv_i64 a, TCGv_i64 b) 2648 { 2649 tcg_gen_movcond_i64(TCG_COND_LTU, ret, a, b, a, b); 2650 } 2651 2652 void tcg_gen_smax_i64(TCGv_i64 ret, TCGv_i64 a, TCGv_i64 b) 2653 { 2654 tcg_gen_movcond_i64(TCG_COND_LT, ret, a, b, b, a); 2655 } 2656 2657 void tcg_gen_umax_i64(TCGv_i64 ret, TCGv_i64 a, TCGv_i64 b) 2658 { 2659 tcg_gen_movcond_i64(TCG_COND_LTU, ret, a, b, b, a); 2660 } 2661 2662 void tcg_gen_abs_i64(TCGv_i64 ret, TCGv_i64 a) 2663 { 2664 TCGv_i64 t = tcg_temp_ebb_new_i64(); 2665 2666 tcg_gen_sari_i64(t, a, 63); 2667 tcg_gen_xor_i64(ret, a, t); 2668 tcg_gen_sub_i64(ret, ret, t); 2669 tcg_temp_free_i64(t); 2670 } 2671 2672 /* Size changing operations. */ 2673 2674 void tcg_gen_extrl_i64_i32(TCGv_i32 ret, TCGv_i64 arg) 2675 { 2676 if (TCG_TARGET_REG_BITS == 32) { 2677 tcg_gen_mov_i32(ret, TCGV_LOW(arg)); 2678 } else if (TCG_TARGET_HAS_extrl_i64_i32) { 2679 tcg_gen_op2(INDEX_op_extrl_i64_i32, 2680 tcgv_i32_arg(ret), tcgv_i64_arg(arg)); 2681 } else { 2682 tcg_gen_mov_i32(ret, (TCGv_i32)arg); 2683 } 2684 } 2685 2686 void tcg_gen_extrh_i64_i32(TCGv_i32 ret, TCGv_i64 arg) 2687 { 2688 if (TCG_TARGET_REG_BITS == 32) { 2689 tcg_gen_mov_i32(ret, TCGV_HIGH(arg)); 2690 } else if (TCG_TARGET_HAS_extrh_i64_i32) { 2691 tcg_gen_op2(INDEX_op_extrh_i64_i32, 2692 tcgv_i32_arg(ret), tcgv_i64_arg(arg)); 2693 } else { 2694 TCGv_i64 t = tcg_temp_ebb_new_i64(); 2695 tcg_gen_shri_i64(t, arg, 32); 2696 tcg_gen_mov_i32(ret, (TCGv_i32)t); 2697 tcg_temp_free_i64(t); 2698 } 2699 } 2700 2701 void tcg_gen_extu_i32_i64(TCGv_i64 ret, TCGv_i32 arg) 2702 { 2703 if (TCG_TARGET_REG_BITS == 32) { 2704 tcg_gen_mov_i32(TCGV_LOW(ret), arg); 2705 tcg_gen_movi_i32(TCGV_HIGH(ret), 0); 2706 } else { 2707 tcg_gen_op2(INDEX_op_extu_i32_i64, 2708 tcgv_i64_arg(ret), tcgv_i32_arg(arg)); 2709 } 2710 } 2711 2712 void tcg_gen_ext_i32_i64(TCGv_i64 ret, TCGv_i32 arg) 2713 { 2714 if (TCG_TARGET_REG_BITS == 32) { 2715 tcg_gen_mov_i32(TCGV_LOW(ret), arg); 2716 tcg_gen_sari_i32(TCGV_HIGH(ret), TCGV_LOW(ret), 31); 2717 } else { 2718 tcg_gen_op2(INDEX_op_ext_i32_i64, 2719 tcgv_i64_arg(ret), tcgv_i32_arg(arg)); 2720 } 2721 } 2722 2723 void tcg_gen_concat_i32_i64(TCGv_i64 dest, TCGv_i32 low, TCGv_i32 high) 2724 { 2725 TCGv_i64 tmp; 2726 2727 if (TCG_TARGET_REG_BITS == 32) { 2728 tcg_gen_mov_i32(TCGV_LOW(dest), low); 2729 tcg_gen_mov_i32(TCGV_HIGH(dest), high); 2730 return; 2731 } 2732 2733 tmp = tcg_temp_ebb_new_i64(); 2734 /* These extensions are only needed for type correctness. 2735 We may be able to do better given target specific information. */ 2736 tcg_gen_extu_i32_i64(tmp, high); 2737 tcg_gen_extu_i32_i64(dest, low); 2738 /* If deposit is available, use it. Otherwise use the extra 2739 knowledge that we have of the zero-extensions above. */ 2740 if (TCG_TARGET_HAS_deposit_i64 && TCG_TARGET_deposit_i64_valid(32, 32)) { 2741 tcg_gen_deposit_i64(dest, dest, tmp, 32, 32); 2742 } else { 2743 tcg_gen_shli_i64(tmp, tmp, 32); 2744 tcg_gen_or_i64(dest, dest, tmp); 2745 } 2746 tcg_temp_free_i64(tmp); 2747 } 2748 2749 void tcg_gen_extr_i64_i32(TCGv_i32 lo, TCGv_i32 hi, TCGv_i64 arg) 2750 { 2751 if (TCG_TARGET_REG_BITS == 32) { 2752 tcg_gen_mov_i32(lo, TCGV_LOW(arg)); 2753 tcg_gen_mov_i32(hi, TCGV_HIGH(arg)); 2754 } else { 2755 tcg_gen_extrl_i64_i32(lo, arg); 2756 tcg_gen_extrh_i64_i32(hi, arg); 2757 } 2758 } 2759 2760 void tcg_gen_extr32_i64(TCGv_i64 lo, TCGv_i64 hi, TCGv_i64 arg) 2761 { 2762 tcg_gen_ext32u_i64(lo, arg); 2763 tcg_gen_shri_i64(hi, arg, 32); 2764 } 2765 2766 void tcg_gen_extr_i128_i64(TCGv_i64 lo, TCGv_i64 hi, TCGv_i128 arg) 2767 { 2768 tcg_gen_mov_i64(lo, TCGV128_LOW(arg)); 2769 tcg_gen_mov_i64(hi, TCGV128_HIGH(arg)); 2770 } 2771 2772 void tcg_gen_concat_i64_i128(TCGv_i128 ret, TCGv_i64 lo, TCGv_i64 hi) 2773 { 2774 tcg_gen_mov_i64(TCGV128_LOW(ret), lo); 2775 tcg_gen_mov_i64(TCGV128_HIGH(ret), hi); 2776 } 2777 2778 void tcg_gen_mov_i128(TCGv_i128 dst, TCGv_i128 src) 2779 { 2780 if (dst != src) { 2781 tcg_gen_mov_i64(TCGV128_LOW(dst), TCGV128_LOW(src)); 2782 tcg_gen_mov_i64(TCGV128_HIGH(dst), TCGV128_HIGH(src)); 2783 } 2784 } 2785 2786 /* QEMU specific operations. */ 2787 2788 void tcg_gen_exit_tb(const TranslationBlock *tb, unsigned idx) 2789 { 2790 /* 2791 * Let the jit code return the read-only version of the 2792 * TranslationBlock, so that we minimize the pc-relative 2793 * distance of the address of the exit_tb code to TB. 2794 * This will improve utilization of pc-relative address loads. 2795 * 2796 * TODO: Move this to translator_loop, so that all const 2797 * TranslationBlock pointers refer to read-only memory. 2798 * This requires coordination with targets that do not use 2799 * the translator_loop. 2800 */ 2801 uintptr_t val = (uintptr_t)tcg_splitwx_to_rx((void *)tb) + idx; 2802 2803 if (tb == NULL) { 2804 tcg_debug_assert(idx == 0); 2805 } else if (idx <= TB_EXIT_IDXMAX) { 2806 #ifdef CONFIG_DEBUG_TCG 2807 /* This is an exit following a goto_tb. Verify that we have 2808 seen this numbered exit before, via tcg_gen_goto_tb. */ 2809 tcg_debug_assert(tcg_ctx->goto_tb_issue_mask & (1 << idx)); 2810 #endif 2811 } else { 2812 /* This is an exit via the exitreq label. */ 2813 tcg_debug_assert(idx == TB_EXIT_REQUESTED); 2814 } 2815 2816 plugin_gen_disable_mem_helpers(); 2817 tcg_gen_op1i(INDEX_op_exit_tb, val); 2818 } 2819 2820 void tcg_gen_goto_tb(unsigned idx) 2821 { 2822 /* We tested CF_NO_GOTO_TB in translator_use_goto_tb. */ 2823 tcg_debug_assert(!(tcg_ctx->gen_tb->cflags & CF_NO_GOTO_TB)); 2824 /* We only support two chained exits. */ 2825 tcg_debug_assert(idx <= TB_EXIT_IDXMAX); 2826 #ifdef CONFIG_DEBUG_TCG 2827 /* Verify that we haven't seen this numbered exit before. */ 2828 tcg_debug_assert((tcg_ctx->goto_tb_issue_mask & (1 << idx)) == 0); 2829 tcg_ctx->goto_tb_issue_mask |= 1 << idx; 2830 #endif 2831 plugin_gen_disable_mem_helpers(); 2832 tcg_gen_op1i(INDEX_op_goto_tb, idx); 2833 } 2834 2835 void tcg_gen_lookup_and_goto_ptr(void) 2836 { 2837 TCGv_ptr ptr; 2838 2839 if (tcg_ctx->gen_tb->cflags & CF_NO_GOTO_PTR) { 2840 tcg_gen_exit_tb(NULL, 0); 2841 return; 2842 } 2843 2844 plugin_gen_disable_mem_helpers(); 2845 ptr = tcg_temp_ebb_new_ptr(); 2846 gen_helper_lookup_tb_ptr(ptr, cpu_env); 2847 tcg_gen_op1i(INDEX_op_goto_ptr, tcgv_ptr_arg(ptr)); 2848 tcg_temp_free_ptr(ptr); 2849 } 2850 2851 static inline MemOp tcg_canonicalize_memop(MemOp op, bool is64, bool st) 2852 { 2853 /* Trigger the asserts within as early as possible. */ 2854 unsigned a_bits = get_alignment_bits(op); 2855 2856 /* Prefer MO_ALIGN+MO_XX over MO_ALIGN_XX+MO_XX */ 2857 if (a_bits == (op & MO_SIZE)) { 2858 op = (op & ~MO_AMASK) | MO_ALIGN; 2859 } 2860 2861 switch (op & MO_SIZE) { 2862 case MO_8: 2863 op &= ~MO_BSWAP; 2864 break; 2865 case MO_16: 2866 break; 2867 case MO_32: 2868 if (!is64) { 2869 op &= ~MO_SIGN; 2870 } 2871 break; 2872 case MO_64: 2873 if (is64) { 2874 op &= ~MO_SIGN; 2875 break; 2876 } 2877 /* fall through */ 2878 default: 2879 g_assert_not_reached(); 2880 } 2881 if (st) { 2882 op &= ~MO_SIGN; 2883 } 2884 return op; 2885 } 2886 2887 static void gen_ldst_i32(TCGOpcode opc, TCGv_i32 val, TCGv addr, 2888 MemOp memop, TCGArg idx) 2889 { 2890 MemOpIdx oi = make_memop_idx(memop, idx); 2891 #if TARGET_LONG_BITS == 32 2892 tcg_gen_op3i_i32(opc, val, addr, oi); 2893 #else 2894 if (TCG_TARGET_REG_BITS == 32) { 2895 tcg_gen_op4i_i32(opc, val, TCGV_LOW(addr), TCGV_HIGH(addr), oi); 2896 } else { 2897 tcg_gen_op3(opc, tcgv_i32_arg(val), tcgv_i64_arg(addr), oi); 2898 } 2899 #endif 2900 } 2901 2902 static void gen_ldst_i64(TCGOpcode opc, TCGv_i64 val, TCGv addr, 2903 MemOp memop, TCGArg idx) 2904 { 2905 MemOpIdx oi = make_memop_idx(memop, idx); 2906 #if TARGET_LONG_BITS == 32 2907 if (TCG_TARGET_REG_BITS == 32) { 2908 tcg_gen_op4i_i32(opc, TCGV_LOW(val), TCGV_HIGH(val), addr, oi); 2909 } else { 2910 tcg_gen_op3(opc, tcgv_i64_arg(val), tcgv_i32_arg(addr), oi); 2911 } 2912 #else 2913 if (TCG_TARGET_REG_BITS == 32) { 2914 tcg_gen_op5i_i32(opc, TCGV_LOW(val), TCGV_HIGH(val), 2915 TCGV_LOW(addr), TCGV_HIGH(addr), oi); 2916 } else { 2917 tcg_gen_op3i_i64(opc, val, addr, oi); 2918 } 2919 #endif 2920 } 2921 2922 static void tcg_gen_req_mo(TCGBar type) 2923 { 2924 #ifdef TCG_GUEST_DEFAULT_MO 2925 type &= TCG_GUEST_DEFAULT_MO; 2926 #endif 2927 type &= ~TCG_TARGET_DEFAULT_MO; 2928 if (type) { 2929 tcg_gen_mb(type | TCG_BAR_SC); 2930 } 2931 } 2932 2933 static inline TCGv plugin_prep_mem_callbacks(TCGv vaddr) 2934 { 2935 #ifdef CONFIG_PLUGIN 2936 if (tcg_ctx->plugin_insn != NULL) { 2937 /* Save a copy of the vaddr for use after a load. */ 2938 TCGv temp = tcg_temp_new(); 2939 tcg_gen_mov_tl(temp, vaddr); 2940 return temp; 2941 } 2942 #endif 2943 return vaddr; 2944 } 2945 2946 static void plugin_gen_mem_callbacks(TCGv vaddr, MemOpIdx oi, 2947 enum qemu_plugin_mem_rw rw) 2948 { 2949 #ifdef CONFIG_PLUGIN 2950 if (tcg_ctx->plugin_insn != NULL) { 2951 qemu_plugin_meminfo_t info = make_plugin_meminfo(oi, rw); 2952 plugin_gen_empty_mem_callback(vaddr, info); 2953 tcg_temp_free(vaddr); 2954 } 2955 #endif 2956 } 2957 2958 void tcg_gen_qemu_ld_i32(TCGv_i32 val, TCGv addr, TCGArg idx, MemOp memop) 2959 { 2960 MemOp orig_memop; 2961 MemOpIdx oi; 2962 2963 tcg_gen_req_mo(TCG_MO_LD_LD | TCG_MO_ST_LD); 2964 memop = tcg_canonicalize_memop(memop, 0, 0); 2965 oi = make_memop_idx(memop, idx); 2966 2967 orig_memop = memop; 2968 if (!TCG_TARGET_HAS_MEMORY_BSWAP && (memop & MO_BSWAP)) { 2969 memop &= ~MO_BSWAP; 2970 /* The bswap primitive benefits from zero-extended input. */ 2971 if ((memop & MO_SSIZE) == MO_SW) { 2972 memop &= ~MO_SIGN; 2973 } 2974 } 2975 2976 addr = plugin_prep_mem_callbacks(addr); 2977 gen_ldst_i32(INDEX_op_qemu_ld_i32, val, addr, memop, idx); 2978 plugin_gen_mem_callbacks(addr, oi, QEMU_PLUGIN_MEM_R); 2979 2980 if ((orig_memop ^ memop) & MO_BSWAP) { 2981 switch (orig_memop & MO_SIZE) { 2982 case MO_16: 2983 tcg_gen_bswap16_i32(val, val, (orig_memop & MO_SIGN 2984 ? TCG_BSWAP_IZ | TCG_BSWAP_OS 2985 : TCG_BSWAP_IZ | TCG_BSWAP_OZ)); 2986 break; 2987 case MO_32: 2988 tcg_gen_bswap32_i32(val, val); 2989 break; 2990 default: 2991 g_assert_not_reached(); 2992 } 2993 } 2994 } 2995 2996 void tcg_gen_qemu_st_i32(TCGv_i32 val, TCGv addr, TCGArg idx, MemOp memop) 2997 { 2998 TCGv_i32 swap = NULL; 2999 MemOpIdx oi; 3000 3001 tcg_gen_req_mo(TCG_MO_LD_ST | TCG_MO_ST_ST); 3002 memop = tcg_canonicalize_memop(memop, 0, 1); 3003 oi = make_memop_idx(memop, idx); 3004 3005 if (!TCG_TARGET_HAS_MEMORY_BSWAP && (memop & MO_BSWAP)) { 3006 swap = tcg_temp_ebb_new_i32(); 3007 switch (memop & MO_SIZE) { 3008 case MO_16: 3009 tcg_gen_bswap16_i32(swap, val, 0); 3010 break; 3011 case MO_32: 3012 tcg_gen_bswap32_i32(swap, val); 3013 break; 3014 default: 3015 g_assert_not_reached(); 3016 } 3017 val = swap; 3018 memop &= ~MO_BSWAP; 3019 } 3020 3021 addr = plugin_prep_mem_callbacks(addr); 3022 if (TCG_TARGET_HAS_qemu_st8_i32 && (memop & MO_SIZE) == MO_8) { 3023 gen_ldst_i32(INDEX_op_qemu_st8_i32, val, addr, memop, idx); 3024 } else { 3025 gen_ldst_i32(INDEX_op_qemu_st_i32, val, addr, memop, idx); 3026 } 3027 plugin_gen_mem_callbacks(addr, oi, QEMU_PLUGIN_MEM_W); 3028 3029 if (swap) { 3030 tcg_temp_free_i32(swap); 3031 } 3032 } 3033 3034 void tcg_gen_qemu_ld_i64(TCGv_i64 val, TCGv addr, TCGArg idx, MemOp memop) 3035 { 3036 MemOp orig_memop; 3037 MemOpIdx oi; 3038 3039 if (TCG_TARGET_REG_BITS == 32 && (memop & MO_SIZE) < MO_64) { 3040 tcg_gen_qemu_ld_i32(TCGV_LOW(val), addr, idx, memop); 3041 if (memop & MO_SIGN) { 3042 tcg_gen_sari_i32(TCGV_HIGH(val), TCGV_LOW(val), 31); 3043 } else { 3044 tcg_gen_movi_i32(TCGV_HIGH(val), 0); 3045 } 3046 return; 3047 } 3048 3049 tcg_gen_req_mo(TCG_MO_LD_LD | TCG_MO_ST_LD); 3050 memop = tcg_canonicalize_memop(memop, 1, 0); 3051 oi = make_memop_idx(memop, idx); 3052 3053 orig_memop = memop; 3054 if (!TCG_TARGET_HAS_MEMORY_BSWAP && (memop & MO_BSWAP)) { 3055 memop &= ~MO_BSWAP; 3056 /* The bswap primitive benefits from zero-extended input. */ 3057 if ((memop & MO_SIGN) && (memop & MO_SIZE) < MO_64) { 3058 memop &= ~MO_SIGN; 3059 } 3060 } 3061 3062 addr = plugin_prep_mem_callbacks(addr); 3063 gen_ldst_i64(INDEX_op_qemu_ld_i64, val, addr, memop, idx); 3064 plugin_gen_mem_callbacks(addr, oi, QEMU_PLUGIN_MEM_R); 3065 3066 if ((orig_memop ^ memop) & MO_BSWAP) { 3067 int flags = (orig_memop & MO_SIGN 3068 ? TCG_BSWAP_IZ | TCG_BSWAP_OS 3069 : TCG_BSWAP_IZ | TCG_BSWAP_OZ); 3070 switch (orig_memop & MO_SIZE) { 3071 case MO_16: 3072 tcg_gen_bswap16_i64(val, val, flags); 3073 break; 3074 case MO_32: 3075 tcg_gen_bswap32_i64(val, val, flags); 3076 break; 3077 case MO_64: 3078 tcg_gen_bswap64_i64(val, val); 3079 break; 3080 default: 3081 g_assert_not_reached(); 3082 } 3083 } 3084 } 3085 3086 void tcg_gen_qemu_st_i64(TCGv_i64 val, TCGv addr, TCGArg idx, MemOp memop) 3087 { 3088 TCGv_i64 swap = NULL; 3089 MemOpIdx oi; 3090 3091 if (TCG_TARGET_REG_BITS == 32 && (memop & MO_SIZE) < MO_64) { 3092 tcg_gen_qemu_st_i32(TCGV_LOW(val), addr, idx, memop); 3093 return; 3094 } 3095 3096 tcg_gen_req_mo(TCG_MO_LD_ST | TCG_MO_ST_ST); 3097 memop = tcg_canonicalize_memop(memop, 1, 1); 3098 oi = make_memop_idx(memop, idx); 3099 3100 if (!TCG_TARGET_HAS_MEMORY_BSWAP && (memop & MO_BSWAP)) { 3101 swap = tcg_temp_ebb_new_i64(); 3102 switch (memop & MO_SIZE) { 3103 case MO_16: 3104 tcg_gen_bswap16_i64(swap, val, 0); 3105 break; 3106 case MO_32: 3107 tcg_gen_bswap32_i64(swap, val, 0); 3108 break; 3109 case MO_64: 3110 tcg_gen_bswap64_i64(swap, val); 3111 break; 3112 default: 3113 g_assert_not_reached(); 3114 } 3115 val = swap; 3116 memop &= ~MO_BSWAP; 3117 } 3118 3119 addr = plugin_prep_mem_callbacks(addr); 3120 gen_ldst_i64(INDEX_op_qemu_st_i64, val, addr, memop, idx); 3121 plugin_gen_mem_callbacks(addr, oi, QEMU_PLUGIN_MEM_W); 3122 3123 if (swap) { 3124 tcg_temp_free_i64(swap); 3125 } 3126 } 3127 3128 static void canonicalize_memop_i128_as_i64(MemOp ret[2], MemOp orig) 3129 { 3130 MemOp mop_1 = orig, mop_2; 3131 3132 tcg_debug_assert((orig & MO_SIZE) == MO_128); 3133 tcg_debug_assert((orig & MO_SIGN) == 0); 3134 3135 /* Use a memory ordering implemented by the host. */ 3136 if (!TCG_TARGET_HAS_MEMORY_BSWAP && (orig & MO_BSWAP)) { 3137 mop_1 &= ~MO_BSWAP; 3138 } 3139 3140 /* Reduce the size to 64-bit. */ 3141 mop_1 = (mop_1 & ~MO_SIZE) | MO_64; 3142 3143 /* Retain the alignment constraints of the original. */ 3144 switch (orig & MO_AMASK) { 3145 case MO_UNALN: 3146 case MO_ALIGN_2: 3147 case MO_ALIGN_4: 3148 mop_2 = mop_1; 3149 break; 3150 case MO_ALIGN_8: 3151 /* Prefer MO_ALIGN+MO_64 to MO_ALIGN_8+MO_64. */ 3152 mop_1 = (mop_1 & ~MO_AMASK) | MO_ALIGN; 3153 mop_2 = mop_1; 3154 break; 3155 case MO_ALIGN: 3156 /* Second has 8-byte alignment; first has 16-byte alignment. */ 3157 mop_2 = mop_1; 3158 mop_1 = (mop_1 & ~MO_AMASK) | MO_ALIGN_16; 3159 break; 3160 case MO_ALIGN_16: 3161 case MO_ALIGN_32: 3162 case MO_ALIGN_64: 3163 /* Second has 8-byte alignment; first retains original. */ 3164 mop_2 = (mop_1 & ~MO_AMASK) | MO_ALIGN; 3165 break; 3166 default: 3167 g_assert_not_reached(); 3168 } 3169 ret[0] = mop_1; 3170 ret[1] = mop_2; 3171 } 3172 3173 void tcg_gen_qemu_ld_i128(TCGv_i128 val, TCGv addr, TCGArg idx, MemOp memop) 3174 { 3175 MemOp mop[2]; 3176 TCGv addr_p8; 3177 TCGv_i64 x, y; 3178 3179 canonicalize_memop_i128_as_i64(mop, memop); 3180 3181 tcg_gen_req_mo(TCG_MO_LD_LD | TCG_MO_ST_LD); 3182 addr = plugin_prep_mem_callbacks(addr); 3183 3184 /* TODO: respect atomicity of the operation. */ 3185 /* TODO: allow the tcg backend to see the whole operation. */ 3186 3187 /* 3188 * Since there are no global TCGv_i128, there is no visible state 3189 * changed if the second load faults. Load directly into the two 3190 * subwords. 3191 */ 3192 if ((memop & MO_BSWAP) == MO_LE) { 3193 x = TCGV128_LOW(val); 3194 y = TCGV128_HIGH(val); 3195 } else { 3196 x = TCGV128_HIGH(val); 3197 y = TCGV128_LOW(val); 3198 } 3199 3200 gen_ldst_i64(INDEX_op_qemu_ld_i64, x, addr, mop[0], idx); 3201 3202 if ((mop[0] ^ memop) & MO_BSWAP) { 3203 tcg_gen_bswap64_i64(x, x); 3204 } 3205 3206 addr_p8 = tcg_temp_new(); 3207 tcg_gen_addi_tl(addr_p8, addr, 8); 3208 gen_ldst_i64(INDEX_op_qemu_ld_i64, y, addr_p8, mop[1], idx); 3209 tcg_temp_free(addr_p8); 3210 3211 if ((mop[0] ^ memop) & MO_BSWAP) { 3212 tcg_gen_bswap64_i64(y, y); 3213 } 3214 3215 plugin_gen_mem_callbacks(addr, make_memop_idx(memop, idx), 3216 QEMU_PLUGIN_MEM_R); 3217 } 3218 3219 void tcg_gen_qemu_st_i128(TCGv_i128 val, TCGv addr, TCGArg idx, MemOp memop) 3220 { 3221 MemOp mop[2]; 3222 TCGv addr_p8; 3223 TCGv_i64 x, y; 3224 3225 canonicalize_memop_i128_as_i64(mop, memop); 3226 3227 tcg_gen_req_mo(TCG_MO_ST_LD | TCG_MO_ST_ST); 3228 addr = plugin_prep_mem_callbacks(addr); 3229 3230 /* TODO: respect atomicity of the operation. */ 3231 /* TODO: allow the tcg backend to see the whole operation. */ 3232 3233 if ((memop & MO_BSWAP) == MO_LE) { 3234 x = TCGV128_LOW(val); 3235 y = TCGV128_HIGH(val); 3236 } else { 3237 x = TCGV128_HIGH(val); 3238 y = TCGV128_LOW(val); 3239 } 3240 3241 addr_p8 = tcg_temp_new(); 3242 if ((mop[0] ^ memop) & MO_BSWAP) { 3243 TCGv_i64 t = tcg_temp_ebb_new_i64(); 3244 3245 tcg_gen_bswap64_i64(t, x); 3246 gen_ldst_i64(INDEX_op_qemu_st_i64, t, addr, mop[0], idx); 3247 tcg_gen_bswap64_i64(t, y); 3248 tcg_gen_addi_tl(addr_p8, addr, 8); 3249 gen_ldst_i64(INDEX_op_qemu_st_i64, t, addr_p8, mop[1], idx); 3250 tcg_temp_free_i64(t); 3251 } else { 3252 gen_ldst_i64(INDEX_op_qemu_st_i64, x, addr, mop[0], idx); 3253 tcg_gen_addi_tl(addr_p8, addr, 8); 3254 gen_ldst_i64(INDEX_op_qemu_st_i64, y, addr_p8, mop[1], idx); 3255 } 3256 tcg_temp_free(addr_p8); 3257 3258 plugin_gen_mem_callbacks(addr, make_memop_idx(memop, idx), 3259 QEMU_PLUGIN_MEM_W); 3260 } 3261 3262 static void tcg_gen_ext_i32(TCGv_i32 ret, TCGv_i32 val, MemOp opc) 3263 { 3264 switch (opc & MO_SSIZE) { 3265 case MO_SB: 3266 tcg_gen_ext8s_i32(ret, val); 3267 break; 3268 case MO_UB: 3269 tcg_gen_ext8u_i32(ret, val); 3270 break; 3271 case MO_SW: 3272 tcg_gen_ext16s_i32(ret, val); 3273 break; 3274 case MO_UW: 3275 tcg_gen_ext16u_i32(ret, val); 3276 break; 3277 default: 3278 tcg_gen_mov_i32(ret, val); 3279 break; 3280 } 3281 } 3282 3283 static void tcg_gen_ext_i64(TCGv_i64 ret, TCGv_i64 val, MemOp opc) 3284 { 3285 switch (opc & MO_SSIZE) { 3286 case MO_SB: 3287 tcg_gen_ext8s_i64(ret, val); 3288 break; 3289 case MO_UB: 3290 tcg_gen_ext8u_i64(ret, val); 3291 break; 3292 case MO_SW: 3293 tcg_gen_ext16s_i64(ret, val); 3294 break; 3295 case MO_UW: 3296 tcg_gen_ext16u_i64(ret, val); 3297 break; 3298 case MO_SL: 3299 tcg_gen_ext32s_i64(ret, val); 3300 break; 3301 case MO_UL: 3302 tcg_gen_ext32u_i64(ret, val); 3303 break; 3304 default: 3305 tcg_gen_mov_i64(ret, val); 3306 break; 3307 } 3308 } 3309 3310 typedef void (*gen_atomic_cx_i32)(TCGv_i32, TCGv_env, TCGv, 3311 TCGv_i32, TCGv_i32, TCGv_i32); 3312 typedef void (*gen_atomic_cx_i64)(TCGv_i64, TCGv_env, TCGv, 3313 TCGv_i64, TCGv_i64, TCGv_i32); 3314 typedef void (*gen_atomic_cx_i128)(TCGv_i128, TCGv_env, TCGv, 3315 TCGv_i128, TCGv_i128, TCGv_i32); 3316 typedef void (*gen_atomic_op_i32)(TCGv_i32, TCGv_env, TCGv, 3317 TCGv_i32, TCGv_i32); 3318 typedef void (*gen_atomic_op_i64)(TCGv_i64, TCGv_env, TCGv, 3319 TCGv_i64, TCGv_i32); 3320 3321 #ifdef CONFIG_ATOMIC64 3322 # define WITH_ATOMIC64(X) X, 3323 #else 3324 # define WITH_ATOMIC64(X) 3325 #endif 3326 #ifdef CONFIG_CMPXCHG128 3327 # define WITH_ATOMIC128(X) X, 3328 #else 3329 # define WITH_ATOMIC128(X) 3330 #endif 3331 3332 static void * const table_cmpxchg[(MO_SIZE | MO_BSWAP) + 1] = { 3333 [MO_8] = gen_helper_atomic_cmpxchgb, 3334 [MO_16 | MO_LE] = gen_helper_atomic_cmpxchgw_le, 3335 [MO_16 | MO_BE] = gen_helper_atomic_cmpxchgw_be, 3336 [MO_32 | MO_LE] = gen_helper_atomic_cmpxchgl_le, 3337 [MO_32 | MO_BE] = gen_helper_atomic_cmpxchgl_be, 3338 WITH_ATOMIC64([MO_64 | MO_LE] = gen_helper_atomic_cmpxchgq_le) 3339 WITH_ATOMIC64([MO_64 | MO_BE] = gen_helper_atomic_cmpxchgq_be) 3340 WITH_ATOMIC128([MO_128 | MO_LE] = gen_helper_atomic_cmpxchgo_le) 3341 WITH_ATOMIC128([MO_128 | MO_BE] = gen_helper_atomic_cmpxchgo_be) 3342 }; 3343 3344 void tcg_gen_nonatomic_cmpxchg_i32(TCGv_i32 retv, TCGv addr, TCGv_i32 cmpv, 3345 TCGv_i32 newv, TCGArg idx, MemOp memop) 3346 { 3347 TCGv_i32 t1 = tcg_temp_ebb_new_i32(); 3348 TCGv_i32 t2 = tcg_temp_ebb_new_i32(); 3349 3350 tcg_gen_ext_i32(t2, cmpv, memop & MO_SIZE); 3351 3352 tcg_gen_qemu_ld_i32(t1, addr, idx, memop & ~MO_SIGN); 3353 tcg_gen_movcond_i32(TCG_COND_EQ, t2, t1, t2, newv, t1); 3354 tcg_gen_qemu_st_i32(t2, addr, idx, memop); 3355 tcg_temp_free_i32(t2); 3356 3357 if (memop & MO_SIGN) { 3358 tcg_gen_ext_i32(retv, t1, memop); 3359 } else { 3360 tcg_gen_mov_i32(retv, t1); 3361 } 3362 tcg_temp_free_i32(t1); 3363 } 3364 3365 void tcg_gen_atomic_cmpxchg_i32(TCGv_i32 retv, TCGv addr, TCGv_i32 cmpv, 3366 TCGv_i32 newv, TCGArg idx, MemOp memop) 3367 { 3368 gen_atomic_cx_i32 gen; 3369 MemOpIdx oi; 3370 3371 if (!(tcg_ctx->gen_tb->cflags & CF_PARALLEL)) { 3372 tcg_gen_nonatomic_cmpxchg_i32(retv, addr, cmpv, newv, idx, memop); 3373 return; 3374 } 3375 3376 memop = tcg_canonicalize_memop(memop, 0, 0); 3377 gen = table_cmpxchg[memop & (MO_SIZE | MO_BSWAP)]; 3378 tcg_debug_assert(gen != NULL); 3379 3380 oi = make_memop_idx(memop & ~MO_SIGN, idx); 3381 gen(retv, cpu_env, addr, cmpv, newv, tcg_constant_i32(oi)); 3382 3383 if (memop & MO_SIGN) { 3384 tcg_gen_ext_i32(retv, retv, memop); 3385 } 3386 } 3387 3388 void tcg_gen_nonatomic_cmpxchg_i64(TCGv_i64 retv, TCGv addr, TCGv_i64 cmpv, 3389 TCGv_i64 newv, TCGArg idx, MemOp memop) 3390 { 3391 TCGv_i64 t1, t2; 3392 3393 if (TCG_TARGET_REG_BITS == 32 && (memop & MO_SIZE) < MO_64) { 3394 tcg_gen_nonatomic_cmpxchg_i32(TCGV_LOW(retv), addr, TCGV_LOW(cmpv), 3395 TCGV_LOW(newv), idx, memop); 3396 if (memop & MO_SIGN) { 3397 tcg_gen_sari_i32(TCGV_HIGH(retv), TCGV_LOW(retv), 31); 3398 } else { 3399 tcg_gen_movi_i32(TCGV_HIGH(retv), 0); 3400 } 3401 return; 3402 } 3403 3404 t1 = tcg_temp_ebb_new_i64(); 3405 t2 = tcg_temp_ebb_new_i64(); 3406 3407 tcg_gen_ext_i64(t2, cmpv, memop & MO_SIZE); 3408 3409 tcg_gen_qemu_ld_i64(t1, addr, idx, memop & ~MO_SIGN); 3410 tcg_gen_movcond_i64(TCG_COND_EQ, t2, t1, t2, newv, t1); 3411 tcg_gen_qemu_st_i64(t2, addr, idx, memop); 3412 tcg_temp_free_i64(t2); 3413 3414 if (memop & MO_SIGN) { 3415 tcg_gen_ext_i64(retv, t1, memop); 3416 } else { 3417 tcg_gen_mov_i64(retv, t1); 3418 } 3419 tcg_temp_free_i64(t1); 3420 } 3421 3422 void tcg_gen_atomic_cmpxchg_i64(TCGv_i64 retv, TCGv addr, TCGv_i64 cmpv, 3423 TCGv_i64 newv, TCGArg idx, MemOp memop) 3424 { 3425 if (!(tcg_ctx->gen_tb->cflags & CF_PARALLEL)) { 3426 tcg_gen_nonatomic_cmpxchg_i64(retv, addr, cmpv, newv, idx, memop); 3427 return; 3428 } 3429 3430 if ((memop & MO_SIZE) == MO_64) { 3431 gen_atomic_cx_i64 gen; 3432 3433 memop = tcg_canonicalize_memop(memop, 1, 0); 3434 gen = table_cmpxchg[memop & (MO_SIZE | MO_BSWAP)]; 3435 if (gen) { 3436 MemOpIdx oi = make_memop_idx(memop, idx); 3437 gen(retv, cpu_env, addr, cmpv, newv, tcg_constant_i32(oi)); 3438 return; 3439 } 3440 3441 gen_helper_exit_atomic(cpu_env); 3442 3443 /* 3444 * Produce a result for a well-formed opcode stream. This satisfies 3445 * liveness for set before used, which happens before this dead code 3446 * is removed. 3447 */ 3448 tcg_gen_movi_i64(retv, 0); 3449 return; 3450 } 3451 3452 if (TCG_TARGET_REG_BITS == 32) { 3453 tcg_gen_atomic_cmpxchg_i32(TCGV_LOW(retv), addr, TCGV_LOW(cmpv), 3454 TCGV_LOW(newv), idx, memop); 3455 if (memop & MO_SIGN) { 3456 tcg_gen_sari_i32(TCGV_HIGH(retv), TCGV_LOW(retv), 31); 3457 } else { 3458 tcg_gen_movi_i32(TCGV_HIGH(retv), 0); 3459 } 3460 } else { 3461 TCGv_i32 c32 = tcg_temp_ebb_new_i32(); 3462 TCGv_i32 n32 = tcg_temp_ebb_new_i32(); 3463 TCGv_i32 r32 = tcg_temp_ebb_new_i32(); 3464 3465 tcg_gen_extrl_i64_i32(c32, cmpv); 3466 tcg_gen_extrl_i64_i32(n32, newv); 3467 tcg_gen_atomic_cmpxchg_i32(r32, addr, c32, n32, idx, memop & ~MO_SIGN); 3468 tcg_temp_free_i32(c32); 3469 tcg_temp_free_i32(n32); 3470 3471 tcg_gen_extu_i32_i64(retv, r32); 3472 tcg_temp_free_i32(r32); 3473 3474 if (memop & MO_SIGN) { 3475 tcg_gen_ext_i64(retv, retv, memop); 3476 } 3477 } 3478 } 3479 3480 void tcg_gen_nonatomic_cmpxchg_i128(TCGv_i128 retv, TCGv addr, TCGv_i128 cmpv, 3481 TCGv_i128 newv, TCGArg idx, MemOp memop) 3482 { 3483 if (TCG_TARGET_REG_BITS == 32) { 3484 /* Inline expansion below is simply too large for 32-bit hosts. */ 3485 gen_atomic_cx_i128 gen = ((memop & MO_BSWAP) == MO_LE 3486 ? gen_helper_nonatomic_cmpxchgo_le 3487 : gen_helper_nonatomic_cmpxchgo_be); 3488 MemOpIdx oi = make_memop_idx(memop, idx); 3489 3490 tcg_debug_assert((memop & MO_SIZE) == MO_128); 3491 tcg_debug_assert((memop & MO_SIGN) == 0); 3492 3493 gen(retv, cpu_env, addr, cmpv, newv, tcg_constant_i32(oi)); 3494 } else { 3495 TCGv_i128 oldv = tcg_temp_ebb_new_i128(); 3496 TCGv_i128 tmpv = tcg_temp_ebb_new_i128(); 3497 TCGv_i64 t0 = tcg_temp_ebb_new_i64(); 3498 TCGv_i64 t1 = tcg_temp_ebb_new_i64(); 3499 TCGv_i64 z = tcg_constant_i64(0); 3500 3501 tcg_gen_qemu_ld_i128(oldv, addr, idx, memop); 3502 3503 /* Compare i128 */ 3504 tcg_gen_xor_i64(t0, TCGV128_LOW(oldv), TCGV128_LOW(cmpv)); 3505 tcg_gen_xor_i64(t1, TCGV128_HIGH(oldv), TCGV128_HIGH(cmpv)); 3506 tcg_gen_or_i64(t0, t0, t1); 3507 3508 /* tmpv = equal ? newv : oldv */ 3509 tcg_gen_movcond_i64(TCG_COND_EQ, TCGV128_LOW(tmpv), t0, z, 3510 TCGV128_LOW(newv), TCGV128_LOW(oldv)); 3511 tcg_gen_movcond_i64(TCG_COND_EQ, TCGV128_HIGH(tmpv), t0, z, 3512 TCGV128_HIGH(newv), TCGV128_HIGH(oldv)); 3513 3514 /* Unconditional writeback. */ 3515 tcg_gen_qemu_st_i128(tmpv, addr, idx, memop); 3516 tcg_gen_mov_i128(retv, oldv); 3517 3518 tcg_temp_free_i64(t0); 3519 tcg_temp_free_i64(t1); 3520 tcg_temp_free_i128(tmpv); 3521 tcg_temp_free_i128(oldv); 3522 } 3523 } 3524 3525 void tcg_gen_atomic_cmpxchg_i128(TCGv_i128 retv, TCGv addr, TCGv_i128 cmpv, 3526 TCGv_i128 newv, TCGArg idx, MemOp memop) 3527 { 3528 gen_atomic_cx_i128 gen; 3529 3530 if (!(tcg_ctx->gen_tb->cflags & CF_PARALLEL)) { 3531 tcg_gen_nonatomic_cmpxchg_i128(retv, addr, cmpv, newv, idx, memop); 3532 return; 3533 } 3534 3535 tcg_debug_assert((memop & MO_SIZE) == MO_128); 3536 tcg_debug_assert((memop & MO_SIGN) == 0); 3537 gen = table_cmpxchg[memop & (MO_SIZE | MO_BSWAP)]; 3538 3539 if (gen) { 3540 MemOpIdx oi = make_memop_idx(memop, idx); 3541 gen(retv, cpu_env, addr, cmpv, newv, tcg_constant_i32(oi)); 3542 return; 3543 } 3544 3545 gen_helper_exit_atomic(cpu_env); 3546 3547 /* 3548 * Produce a result for a well-formed opcode stream. This satisfies 3549 * liveness for set before used, which happens before this dead code 3550 * is removed. 3551 */ 3552 tcg_gen_movi_i64(TCGV128_LOW(retv), 0); 3553 tcg_gen_movi_i64(TCGV128_HIGH(retv), 0); 3554 } 3555 3556 static void do_nonatomic_op_i32(TCGv_i32 ret, TCGv addr, TCGv_i32 val, 3557 TCGArg idx, MemOp memop, bool new_val, 3558 void (*gen)(TCGv_i32, TCGv_i32, TCGv_i32)) 3559 { 3560 TCGv_i32 t1 = tcg_temp_ebb_new_i32(); 3561 TCGv_i32 t2 = tcg_temp_ebb_new_i32(); 3562 3563 memop = tcg_canonicalize_memop(memop, 0, 0); 3564 3565 tcg_gen_qemu_ld_i32(t1, addr, idx, memop); 3566 tcg_gen_ext_i32(t2, val, memop); 3567 gen(t2, t1, t2); 3568 tcg_gen_qemu_st_i32(t2, addr, idx, memop); 3569 3570 tcg_gen_ext_i32(ret, (new_val ? t2 : t1), memop); 3571 tcg_temp_free_i32(t1); 3572 tcg_temp_free_i32(t2); 3573 } 3574 3575 static void do_atomic_op_i32(TCGv_i32 ret, TCGv addr, TCGv_i32 val, 3576 TCGArg idx, MemOp memop, void * const table[]) 3577 { 3578 gen_atomic_op_i32 gen; 3579 MemOpIdx oi; 3580 3581 memop = tcg_canonicalize_memop(memop, 0, 0); 3582 3583 gen = table[memop & (MO_SIZE | MO_BSWAP)]; 3584 tcg_debug_assert(gen != NULL); 3585 3586 oi = make_memop_idx(memop & ~MO_SIGN, idx); 3587 gen(ret, cpu_env, addr, val, tcg_constant_i32(oi)); 3588 3589 if (memop & MO_SIGN) { 3590 tcg_gen_ext_i32(ret, ret, memop); 3591 } 3592 } 3593 3594 static void do_nonatomic_op_i64(TCGv_i64 ret, TCGv addr, TCGv_i64 val, 3595 TCGArg idx, MemOp memop, bool new_val, 3596 void (*gen)(TCGv_i64, TCGv_i64, TCGv_i64)) 3597 { 3598 TCGv_i64 t1 = tcg_temp_ebb_new_i64(); 3599 TCGv_i64 t2 = tcg_temp_ebb_new_i64(); 3600 3601 memop = tcg_canonicalize_memop(memop, 1, 0); 3602 3603 tcg_gen_qemu_ld_i64(t1, addr, idx, memop); 3604 tcg_gen_ext_i64(t2, val, memop); 3605 gen(t2, t1, t2); 3606 tcg_gen_qemu_st_i64(t2, addr, idx, memop); 3607 3608 tcg_gen_ext_i64(ret, (new_val ? t2 : t1), memop); 3609 tcg_temp_free_i64(t1); 3610 tcg_temp_free_i64(t2); 3611 } 3612 3613 static void do_atomic_op_i64(TCGv_i64 ret, TCGv addr, TCGv_i64 val, 3614 TCGArg idx, MemOp memop, void * const table[]) 3615 { 3616 memop = tcg_canonicalize_memop(memop, 1, 0); 3617 3618 if ((memop & MO_SIZE) == MO_64) { 3619 #ifdef CONFIG_ATOMIC64 3620 gen_atomic_op_i64 gen; 3621 MemOpIdx oi; 3622 3623 gen = table[memop & (MO_SIZE | MO_BSWAP)]; 3624 tcg_debug_assert(gen != NULL); 3625 3626 oi = make_memop_idx(memop & ~MO_SIGN, idx); 3627 gen(ret, cpu_env, addr, val, tcg_constant_i32(oi)); 3628 #else 3629 gen_helper_exit_atomic(cpu_env); 3630 /* Produce a result, so that we have a well-formed opcode stream 3631 with respect to uses of the result in the (dead) code following. */ 3632 tcg_gen_movi_i64(ret, 0); 3633 #endif /* CONFIG_ATOMIC64 */ 3634 } else { 3635 TCGv_i32 v32 = tcg_temp_ebb_new_i32(); 3636 TCGv_i32 r32 = tcg_temp_ebb_new_i32(); 3637 3638 tcg_gen_extrl_i64_i32(v32, val); 3639 do_atomic_op_i32(r32, addr, v32, idx, memop & ~MO_SIGN, table); 3640 tcg_temp_free_i32(v32); 3641 3642 tcg_gen_extu_i32_i64(ret, r32); 3643 tcg_temp_free_i32(r32); 3644 3645 if (memop & MO_SIGN) { 3646 tcg_gen_ext_i64(ret, ret, memop); 3647 } 3648 } 3649 } 3650 3651 #define GEN_ATOMIC_HELPER(NAME, OP, NEW) \ 3652 static void * const table_##NAME[(MO_SIZE | MO_BSWAP) + 1] = { \ 3653 [MO_8] = gen_helper_atomic_##NAME##b, \ 3654 [MO_16 | MO_LE] = gen_helper_atomic_##NAME##w_le, \ 3655 [MO_16 | MO_BE] = gen_helper_atomic_##NAME##w_be, \ 3656 [MO_32 | MO_LE] = gen_helper_atomic_##NAME##l_le, \ 3657 [MO_32 | MO_BE] = gen_helper_atomic_##NAME##l_be, \ 3658 WITH_ATOMIC64([MO_64 | MO_LE] = gen_helper_atomic_##NAME##q_le) \ 3659 WITH_ATOMIC64([MO_64 | MO_BE] = gen_helper_atomic_##NAME##q_be) \ 3660 }; \ 3661 void tcg_gen_atomic_##NAME##_i32 \ 3662 (TCGv_i32 ret, TCGv addr, TCGv_i32 val, TCGArg idx, MemOp memop) \ 3663 { \ 3664 if (tcg_ctx->gen_tb->cflags & CF_PARALLEL) { \ 3665 do_atomic_op_i32(ret, addr, val, idx, memop, table_##NAME); \ 3666 } else { \ 3667 do_nonatomic_op_i32(ret, addr, val, idx, memop, NEW, \ 3668 tcg_gen_##OP##_i32); \ 3669 } \ 3670 } \ 3671 void tcg_gen_atomic_##NAME##_i64 \ 3672 (TCGv_i64 ret, TCGv addr, TCGv_i64 val, TCGArg idx, MemOp memop) \ 3673 { \ 3674 if (tcg_ctx->gen_tb->cflags & CF_PARALLEL) { \ 3675 do_atomic_op_i64(ret, addr, val, idx, memop, table_##NAME); \ 3676 } else { \ 3677 do_nonatomic_op_i64(ret, addr, val, idx, memop, NEW, \ 3678 tcg_gen_##OP##_i64); \ 3679 } \ 3680 } 3681 3682 GEN_ATOMIC_HELPER(fetch_add, add, 0) 3683 GEN_ATOMIC_HELPER(fetch_and, and, 0) 3684 GEN_ATOMIC_HELPER(fetch_or, or, 0) 3685 GEN_ATOMIC_HELPER(fetch_xor, xor, 0) 3686 GEN_ATOMIC_HELPER(fetch_smin, smin, 0) 3687 GEN_ATOMIC_HELPER(fetch_umin, umin, 0) 3688 GEN_ATOMIC_HELPER(fetch_smax, smax, 0) 3689 GEN_ATOMIC_HELPER(fetch_umax, umax, 0) 3690 3691 GEN_ATOMIC_HELPER(add_fetch, add, 1) 3692 GEN_ATOMIC_HELPER(and_fetch, and, 1) 3693 GEN_ATOMIC_HELPER(or_fetch, or, 1) 3694 GEN_ATOMIC_HELPER(xor_fetch, xor, 1) 3695 GEN_ATOMIC_HELPER(smin_fetch, smin, 1) 3696 GEN_ATOMIC_HELPER(umin_fetch, umin, 1) 3697 GEN_ATOMIC_HELPER(smax_fetch, smax, 1) 3698 GEN_ATOMIC_HELPER(umax_fetch, umax, 1) 3699 3700 static void tcg_gen_mov2_i32(TCGv_i32 r, TCGv_i32 a, TCGv_i32 b) 3701 { 3702 tcg_gen_mov_i32(r, b); 3703 } 3704 3705 static void tcg_gen_mov2_i64(TCGv_i64 r, TCGv_i64 a, TCGv_i64 b) 3706 { 3707 tcg_gen_mov_i64(r, b); 3708 } 3709 3710 GEN_ATOMIC_HELPER(xchg, mov2, 0) 3711 3712 #undef GEN_ATOMIC_HELPER 3713