1 /* 2 * Tiny Code Generator for QEMU 3 * 4 * Copyright (c) 2008 Fabrice Bellard 5 * 6 * Permission is hereby granted, free of charge, to any person obtaining a copy 7 * of this software and associated documentation files (the "Software"), to deal 8 * in the Software without restriction, including without limitation the rights 9 * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 10 * copies of the Software, and to permit persons to whom the Software is 11 * furnished to do so, subject to the following conditions: 12 * 13 * The above copyright notice and this permission notice shall be included in 14 * all copies or substantial portions of the Software. 15 * 16 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 17 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 18 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL 19 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 20 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 21 * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN 22 * THE SOFTWARE. 23 */ 24 25 #include "qemu/osdep.h" 26 #include "tcg/tcg.h" 27 #include "tcg/tcg-temp-internal.h" 28 #include "tcg/tcg-op-common.h" 29 #include "exec/translation-block.h" 30 #include "exec/plugin-gen.h" 31 #include "tcg-internal.h" 32 33 34 /* 35 * Encourage the compiler to tail-call to a function, rather than inlining. 36 * Minimizes code size across 99 bottles of beer on the wall. 37 */ 38 #define NI __attribute__((noinline)) 39 40 void NI tcg_gen_op1(TCGOpcode opc, TCGArg a1) 41 { 42 TCGOp *op = tcg_emit_op(opc, 1); 43 op->args[0] = a1; 44 } 45 46 void NI tcg_gen_op2(TCGOpcode opc, TCGArg a1, TCGArg a2) 47 { 48 TCGOp *op = tcg_emit_op(opc, 2); 49 op->args[0] = a1; 50 op->args[1] = a2; 51 } 52 53 void NI tcg_gen_op3(TCGOpcode opc, TCGArg a1, TCGArg a2, TCGArg a3) 54 { 55 TCGOp *op = tcg_emit_op(opc, 3); 56 op->args[0] = a1; 57 op->args[1] = a2; 58 op->args[2] = a3; 59 } 60 61 void NI tcg_gen_op4(TCGOpcode opc, TCGArg a1, TCGArg a2, TCGArg a3, TCGArg a4) 62 { 63 TCGOp *op = tcg_emit_op(opc, 4); 64 op->args[0] = a1; 65 op->args[1] = a2; 66 op->args[2] = a3; 67 op->args[3] = a4; 68 } 69 70 void NI tcg_gen_op5(TCGOpcode opc, TCGArg a1, TCGArg a2, TCGArg a3, 71 TCGArg a4, TCGArg a5) 72 { 73 TCGOp *op = tcg_emit_op(opc, 5); 74 op->args[0] = a1; 75 op->args[1] = a2; 76 op->args[2] = a3; 77 op->args[3] = a4; 78 op->args[4] = a5; 79 } 80 81 void NI tcg_gen_op6(TCGOpcode opc, TCGArg a1, TCGArg a2, TCGArg a3, 82 TCGArg a4, TCGArg a5, TCGArg a6) 83 { 84 TCGOp *op = tcg_emit_op(opc, 6); 85 op->args[0] = a1; 86 op->args[1] = a2; 87 op->args[2] = a3; 88 op->args[3] = a4; 89 op->args[4] = a5; 90 op->args[5] = a6; 91 } 92 93 /* 94 * With CONFIG_DEBUG_TCG, tcgv_*_tmp via tcgv_*_arg, is an out-of-line 95 * assertion check. Force tail calls to avoid too much code expansion. 96 */ 97 #ifdef CONFIG_DEBUG_TCG 98 # define DNI NI 99 #else 100 # define DNI 101 #endif 102 103 static void DNI tcg_gen_op1_i32(TCGOpcode opc, TCGv_i32 a1) 104 { 105 tcg_gen_op1(opc, tcgv_i32_arg(a1)); 106 } 107 108 static void DNI tcg_gen_op1_i64(TCGOpcode opc, TCGv_i64 a1) 109 { 110 tcg_gen_op1(opc, tcgv_i64_arg(a1)); 111 } 112 113 static void DNI tcg_gen_op1i(TCGOpcode opc, TCGArg a1) 114 { 115 tcg_gen_op1(opc, a1); 116 } 117 118 static void DNI tcg_gen_op2_i32(TCGOpcode opc, TCGv_i32 a1, TCGv_i32 a2) 119 { 120 tcg_gen_op2(opc, tcgv_i32_arg(a1), tcgv_i32_arg(a2)); 121 } 122 123 static void DNI tcg_gen_op2_i64(TCGOpcode opc, TCGv_i64 a1, TCGv_i64 a2) 124 { 125 tcg_gen_op2(opc, tcgv_i64_arg(a1), tcgv_i64_arg(a2)); 126 } 127 128 static void DNI tcg_gen_op3_i32(TCGOpcode opc, TCGv_i32 a1, 129 TCGv_i32 a2, TCGv_i32 a3) 130 { 131 tcg_gen_op3(opc, tcgv_i32_arg(a1), tcgv_i32_arg(a2), tcgv_i32_arg(a3)); 132 } 133 134 static void DNI tcg_gen_op3_i64(TCGOpcode opc, TCGv_i64 a1, 135 TCGv_i64 a2, TCGv_i64 a3) 136 { 137 tcg_gen_op3(opc, tcgv_i64_arg(a1), tcgv_i64_arg(a2), tcgv_i64_arg(a3)); 138 } 139 140 static void DNI tcg_gen_op3i_i32(TCGOpcode opc, TCGv_i32 a1, 141 TCGv_i32 a2, TCGArg a3) 142 { 143 tcg_gen_op3(opc, tcgv_i32_arg(a1), tcgv_i32_arg(a2), a3); 144 } 145 146 static void DNI tcg_gen_op3i_i64(TCGOpcode opc, TCGv_i64 a1, 147 TCGv_i64 a2, TCGArg a3) 148 { 149 tcg_gen_op3(opc, tcgv_i64_arg(a1), tcgv_i64_arg(a2), a3); 150 } 151 152 static void DNI tcg_gen_ldst_op_i32(TCGOpcode opc, TCGv_i32 val, 153 TCGv_ptr base, TCGArg offset) 154 { 155 tcg_gen_op3(opc, tcgv_i32_arg(val), tcgv_ptr_arg(base), offset); 156 } 157 158 static void DNI tcg_gen_ldst_op_i64(TCGOpcode opc, TCGv_i64 val, 159 TCGv_ptr base, TCGArg offset) 160 { 161 tcg_gen_op3(opc, tcgv_i64_arg(val), tcgv_ptr_arg(base), offset); 162 } 163 164 static void DNI tcg_gen_op4_i32(TCGOpcode opc, TCGv_i32 a1, TCGv_i32 a2, 165 TCGv_i32 a3, TCGv_i32 a4) 166 { 167 tcg_gen_op4(opc, tcgv_i32_arg(a1), tcgv_i32_arg(a2), 168 tcgv_i32_arg(a3), tcgv_i32_arg(a4)); 169 } 170 171 static void DNI tcg_gen_op4_i64(TCGOpcode opc, TCGv_i64 a1, TCGv_i64 a2, 172 TCGv_i64 a3, TCGv_i64 a4) 173 { 174 tcg_gen_op4(opc, tcgv_i64_arg(a1), tcgv_i64_arg(a2), 175 tcgv_i64_arg(a3), tcgv_i64_arg(a4)); 176 } 177 178 static void DNI tcg_gen_op4i_i32(TCGOpcode opc, TCGv_i32 a1, TCGv_i32 a2, 179 TCGv_i32 a3, TCGArg a4) 180 { 181 tcg_gen_op4(opc, tcgv_i32_arg(a1), tcgv_i32_arg(a2), 182 tcgv_i32_arg(a3), a4); 183 } 184 185 static void DNI tcg_gen_op4i_i64(TCGOpcode opc, TCGv_i64 a1, TCGv_i64 a2, 186 TCGv_i64 a3, TCGArg a4) 187 { 188 tcg_gen_op4(opc, tcgv_i64_arg(a1), tcgv_i64_arg(a2), 189 tcgv_i64_arg(a3), a4); 190 } 191 192 static void DNI tcg_gen_op4ii_i32(TCGOpcode opc, TCGv_i32 a1, TCGv_i32 a2, 193 TCGArg a3, TCGArg a4) 194 { 195 tcg_gen_op4(opc, tcgv_i32_arg(a1), tcgv_i32_arg(a2), a3, a4); 196 } 197 198 static void DNI tcg_gen_op4ii_i64(TCGOpcode opc, TCGv_i64 a1, TCGv_i64 a2, 199 TCGArg a3, TCGArg a4) 200 { 201 tcg_gen_op4(opc, tcgv_i64_arg(a1), tcgv_i64_arg(a2), a3, a4); 202 } 203 204 static void DNI tcg_gen_op5_i32(TCGOpcode opc, TCGv_i32 a1, TCGv_i32 a2, 205 TCGv_i32 a3, TCGv_i32 a4, TCGv_i32 a5) 206 { 207 tcg_gen_op5(opc, tcgv_i32_arg(a1), tcgv_i32_arg(a2), 208 tcgv_i32_arg(a3), tcgv_i32_arg(a4), tcgv_i32_arg(a5)); 209 } 210 211 static void DNI tcg_gen_op5_i64(TCGOpcode opc, TCGv_i64 a1, TCGv_i64 a2, 212 TCGv_i64 a3, TCGv_i64 a4, TCGv_i64 a5) 213 { 214 tcg_gen_op5(opc, tcgv_i64_arg(a1), tcgv_i64_arg(a2), 215 tcgv_i64_arg(a3), tcgv_i64_arg(a4), tcgv_i64_arg(a5)); 216 } 217 218 static void DNI tcg_gen_op5ii_i32(TCGOpcode opc, TCGv_i32 a1, TCGv_i32 a2, 219 TCGv_i32 a3, TCGArg a4, TCGArg a5) 220 { 221 tcg_gen_op5(opc, tcgv_i32_arg(a1), tcgv_i32_arg(a2), 222 tcgv_i32_arg(a3), a4, a5); 223 } 224 225 static void DNI tcg_gen_op5ii_i64(TCGOpcode opc, TCGv_i64 a1, TCGv_i64 a2, 226 TCGv_i64 a3, TCGArg a4, TCGArg a5) 227 { 228 tcg_gen_op5(opc, tcgv_i64_arg(a1), tcgv_i64_arg(a2), 229 tcgv_i64_arg(a3), a4, a5); 230 } 231 232 static void DNI tcg_gen_op6_i32(TCGOpcode opc, TCGv_i32 a1, TCGv_i32 a2, 233 TCGv_i32 a3, TCGv_i32 a4, 234 TCGv_i32 a5, TCGv_i32 a6) 235 { 236 tcg_gen_op6(opc, tcgv_i32_arg(a1), tcgv_i32_arg(a2), 237 tcgv_i32_arg(a3), tcgv_i32_arg(a4), tcgv_i32_arg(a5), 238 tcgv_i32_arg(a6)); 239 } 240 241 static void DNI tcg_gen_op6_i64(TCGOpcode opc, TCGv_i64 a1, TCGv_i64 a2, 242 TCGv_i64 a3, TCGv_i64 a4, 243 TCGv_i64 a5, TCGv_i64 a6) 244 { 245 tcg_gen_op6(opc, tcgv_i64_arg(a1), tcgv_i64_arg(a2), 246 tcgv_i64_arg(a3), tcgv_i64_arg(a4), tcgv_i64_arg(a5), 247 tcgv_i64_arg(a6)); 248 } 249 250 static void DNI tcg_gen_op6i_i32(TCGOpcode opc, TCGv_i32 a1, TCGv_i32 a2, 251 TCGv_i32 a3, TCGv_i32 a4, 252 TCGv_i32 a5, TCGArg a6) 253 { 254 tcg_gen_op6(opc, tcgv_i32_arg(a1), tcgv_i32_arg(a2), 255 tcgv_i32_arg(a3), tcgv_i32_arg(a4), tcgv_i32_arg(a5), a6); 256 } 257 258 static void DNI tcg_gen_op6i_i64(TCGOpcode opc, TCGv_i64 a1, TCGv_i64 a2, 259 TCGv_i64 a3, TCGv_i64 a4, 260 TCGv_i64 a5, TCGArg a6) 261 { 262 tcg_gen_op6(opc, tcgv_i64_arg(a1), tcgv_i64_arg(a2), 263 tcgv_i64_arg(a3), tcgv_i64_arg(a4), tcgv_i64_arg(a5), a6); 264 } 265 266 static void DNI tcg_gen_op6ii_i32(TCGOpcode opc, TCGv_i32 a1, TCGv_i32 a2, 267 TCGv_i32 a3, TCGv_i32 a4, 268 TCGArg a5, TCGArg a6) 269 { 270 tcg_gen_op6(opc, tcgv_i32_arg(a1), tcgv_i32_arg(a2), 271 tcgv_i32_arg(a3), tcgv_i32_arg(a4), a5, a6); 272 } 273 274 /* Generic ops. */ 275 276 void gen_set_label(TCGLabel *l) 277 { 278 l->present = 1; 279 tcg_gen_op1(INDEX_op_set_label, label_arg(l)); 280 } 281 282 static void add_last_as_label_use(TCGLabel *l) 283 { 284 TCGLabelUse *u = tcg_malloc(sizeof(TCGLabelUse)); 285 286 u->op = tcg_last_op(); 287 QSIMPLEQ_INSERT_TAIL(&l->branches, u, next); 288 } 289 290 void tcg_gen_br(TCGLabel *l) 291 { 292 tcg_gen_op1(INDEX_op_br, label_arg(l)); 293 add_last_as_label_use(l); 294 } 295 296 void tcg_gen_mb(TCGBar mb_type) 297 { 298 #ifdef CONFIG_USER_ONLY 299 bool parallel = tcg_ctx->gen_tb->cflags & CF_PARALLEL; 300 #else 301 /* 302 * It is tempting to elide the barrier in a uniprocessor context. 303 * However, even with a single cpu we have i/o threads running in 304 * parallel, and lack of memory order can result in e.g. virtio 305 * queue entries being read incorrectly. 306 */ 307 bool parallel = true; 308 #endif 309 310 if (parallel) { 311 tcg_gen_op1(INDEX_op_mb, mb_type); 312 } 313 } 314 315 void tcg_gen_plugin_cb_start(unsigned from, unsigned type, unsigned wr) 316 { 317 tcg_gen_op3(INDEX_op_plugin_cb_start, from, type, wr); 318 } 319 320 void tcg_gen_plugin_cb_end(void) 321 { 322 tcg_emit_op(INDEX_op_plugin_cb_end, 0); 323 } 324 325 /* 32 bit ops */ 326 327 void tcg_gen_discard_i32(TCGv_i32 arg) 328 { 329 tcg_gen_op1_i32(INDEX_op_discard, arg); 330 } 331 332 void tcg_gen_mov_i32(TCGv_i32 ret, TCGv_i32 arg) 333 { 334 if (ret != arg) { 335 tcg_gen_op2_i32(INDEX_op_mov_i32, ret, arg); 336 } 337 } 338 339 void tcg_gen_movi_i32(TCGv_i32 ret, int32_t arg) 340 { 341 tcg_gen_mov_i32(ret, tcg_constant_i32(arg)); 342 } 343 344 void tcg_gen_add_i32(TCGv_i32 ret, TCGv_i32 arg1, TCGv_i32 arg2) 345 { 346 tcg_gen_op3_i32(INDEX_op_add_i32, ret, arg1, arg2); 347 } 348 349 void tcg_gen_addi_i32(TCGv_i32 ret, TCGv_i32 arg1, int32_t arg2) 350 { 351 /* some cases can be optimized here */ 352 if (arg2 == 0) { 353 tcg_gen_mov_i32(ret, arg1); 354 } else { 355 tcg_gen_add_i32(ret, arg1, tcg_constant_i32(arg2)); 356 } 357 } 358 359 void tcg_gen_sub_i32(TCGv_i32 ret, TCGv_i32 arg1, TCGv_i32 arg2) 360 { 361 tcg_gen_op3_i32(INDEX_op_sub_i32, ret, arg1, arg2); 362 } 363 364 void tcg_gen_subfi_i32(TCGv_i32 ret, int32_t arg1, TCGv_i32 arg2) 365 { 366 if (arg1 == 0) { 367 tcg_gen_neg_i32(ret, arg2); 368 } else { 369 tcg_gen_sub_i32(ret, tcg_constant_i32(arg1), arg2); 370 } 371 } 372 373 void tcg_gen_subi_i32(TCGv_i32 ret, TCGv_i32 arg1, int32_t arg2) 374 { 375 tcg_gen_addi_i32(ret, arg1, -arg2); 376 } 377 378 void tcg_gen_neg_i32(TCGv_i32 ret, TCGv_i32 arg) 379 { 380 tcg_gen_op2_i32(INDEX_op_neg_i32, ret, arg); 381 } 382 383 void tcg_gen_and_i32(TCGv_i32 ret, TCGv_i32 arg1, TCGv_i32 arg2) 384 { 385 tcg_gen_op3_i32(INDEX_op_and_i32, ret, arg1, arg2); 386 } 387 388 void tcg_gen_andi_i32(TCGv_i32 ret, TCGv_i32 arg1, int32_t arg2) 389 { 390 /* Some cases can be optimized here. */ 391 switch (arg2) { 392 case 0: 393 tcg_gen_movi_i32(ret, 0); 394 return; 395 case -1: 396 tcg_gen_mov_i32(ret, arg1); 397 return; 398 case 0xff: 399 /* Don't recurse with tcg_gen_ext8u_i32. */ 400 if (TCG_TARGET_HAS_ext8u_i32) { 401 tcg_gen_op2_i32(INDEX_op_ext8u_i32, ret, arg1); 402 return; 403 } 404 break; 405 case 0xffff: 406 if (TCG_TARGET_HAS_ext16u_i32) { 407 tcg_gen_op2_i32(INDEX_op_ext16u_i32, ret, arg1); 408 return; 409 } 410 break; 411 } 412 413 tcg_gen_and_i32(ret, arg1, tcg_constant_i32(arg2)); 414 } 415 416 void tcg_gen_or_i32(TCGv_i32 ret, TCGv_i32 arg1, TCGv_i32 arg2) 417 { 418 tcg_gen_op3_i32(INDEX_op_or_i32, ret, arg1, arg2); 419 } 420 421 void tcg_gen_ori_i32(TCGv_i32 ret, TCGv_i32 arg1, int32_t arg2) 422 { 423 /* Some cases can be optimized here. */ 424 if (arg2 == -1) { 425 tcg_gen_movi_i32(ret, -1); 426 } else if (arg2 == 0) { 427 tcg_gen_mov_i32(ret, arg1); 428 } else { 429 tcg_gen_or_i32(ret, arg1, tcg_constant_i32(arg2)); 430 } 431 } 432 433 void tcg_gen_xor_i32(TCGv_i32 ret, TCGv_i32 arg1, TCGv_i32 arg2) 434 { 435 tcg_gen_op3_i32(INDEX_op_xor_i32, ret, arg1, arg2); 436 } 437 438 void tcg_gen_xori_i32(TCGv_i32 ret, TCGv_i32 arg1, int32_t arg2) 439 { 440 /* Some cases can be optimized here. */ 441 if (arg2 == 0) { 442 tcg_gen_mov_i32(ret, arg1); 443 } else if (arg2 == -1 && TCG_TARGET_HAS_not_i32) { 444 /* Don't recurse with tcg_gen_not_i32. */ 445 tcg_gen_op2_i32(INDEX_op_not_i32, ret, arg1); 446 } else { 447 tcg_gen_xor_i32(ret, arg1, tcg_constant_i32(arg2)); 448 } 449 } 450 451 void tcg_gen_not_i32(TCGv_i32 ret, TCGv_i32 arg) 452 { 453 if (TCG_TARGET_HAS_not_i32) { 454 tcg_gen_op2_i32(INDEX_op_not_i32, ret, arg); 455 } else { 456 tcg_gen_xori_i32(ret, arg, -1); 457 } 458 } 459 460 void tcg_gen_shl_i32(TCGv_i32 ret, TCGv_i32 arg1, TCGv_i32 arg2) 461 { 462 tcg_gen_op3_i32(INDEX_op_shl_i32, ret, arg1, arg2); 463 } 464 465 void tcg_gen_shli_i32(TCGv_i32 ret, TCGv_i32 arg1, int32_t arg2) 466 { 467 tcg_debug_assert(arg2 >= 0 && arg2 < 32); 468 if (arg2 == 0) { 469 tcg_gen_mov_i32(ret, arg1); 470 } else { 471 tcg_gen_shl_i32(ret, arg1, tcg_constant_i32(arg2)); 472 } 473 } 474 475 void tcg_gen_shr_i32(TCGv_i32 ret, TCGv_i32 arg1, TCGv_i32 arg2) 476 { 477 tcg_gen_op3_i32(INDEX_op_shr_i32, ret, arg1, arg2); 478 } 479 480 void tcg_gen_shri_i32(TCGv_i32 ret, TCGv_i32 arg1, int32_t arg2) 481 { 482 tcg_debug_assert(arg2 >= 0 && arg2 < 32); 483 if (arg2 == 0) { 484 tcg_gen_mov_i32(ret, arg1); 485 } else { 486 tcg_gen_shr_i32(ret, arg1, tcg_constant_i32(arg2)); 487 } 488 } 489 490 void tcg_gen_sar_i32(TCGv_i32 ret, TCGv_i32 arg1, TCGv_i32 arg2) 491 { 492 tcg_gen_op3_i32(INDEX_op_sar_i32, ret, arg1, arg2); 493 } 494 495 void tcg_gen_sari_i32(TCGv_i32 ret, TCGv_i32 arg1, int32_t arg2) 496 { 497 tcg_debug_assert(arg2 >= 0 && arg2 < 32); 498 if (arg2 == 0) { 499 tcg_gen_mov_i32(ret, arg1); 500 } else { 501 tcg_gen_sar_i32(ret, arg1, tcg_constant_i32(arg2)); 502 } 503 } 504 505 void tcg_gen_brcond_i32(TCGCond cond, TCGv_i32 arg1, TCGv_i32 arg2, TCGLabel *l) 506 { 507 if (cond == TCG_COND_ALWAYS) { 508 tcg_gen_br(l); 509 } else if (cond != TCG_COND_NEVER) { 510 tcg_gen_op4ii_i32(INDEX_op_brcond_i32, arg1, arg2, cond, label_arg(l)); 511 add_last_as_label_use(l); 512 } 513 } 514 515 void tcg_gen_brcondi_i32(TCGCond cond, TCGv_i32 arg1, int32_t arg2, TCGLabel *l) 516 { 517 if (cond == TCG_COND_ALWAYS) { 518 tcg_gen_br(l); 519 } else if (cond != TCG_COND_NEVER) { 520 tcg_gen_brcond_i32(cond, arg1, tcg_constant_i32(arg2), l); 521 } 522 } 523 524 void tcg_gen_setcond_i32(TCGCond cond, TCGv_i32 ret, 525 TCGv_i32 arg1, TCGv_i32 arg2) 526 { 527 if (cond == TCG_COND_ALWAYS) { 528 tcg_gen_movi_i32(ret, 1); 529 } else if (cond == TCG_COND_NEVER) { 530 tcg_gen_movi_i32(ret, 0); 531 } else { 532 tcg_gen_op4i_i32(INDEX_op_setcond_i32, ret, arg1, arg2, cond); 533 } 534 } 535 536 void tcg_gen_setcondi_i32(TCGCond cond, TCGv_i32 ret, 537 TCGv_i32 arg1, int32_t arg2) 538 { 539 tcg_gen_setcond_i32(cond, ret, arg1, tcg_constant_i32(arg2)); 540 } 541 542 void tcg_gen_negsetcond_i32(TCGCond cond, TCGv_i32 ret, 543 TCGv_i32 arg1, TCGv_i32 arg2) 544 { 545 if (cond == TCG_COND_ALWAYS) { 546 tcg_gen_movi_i32(ret, -1); 547 } else if (cond == TCG_COND_NEVER) { 548 tcg_gen_movi_i32(ret, 0); 549 } else if (TCG_TARGET_HAS_negsetcond_i32) { 550 tcg_gen_op4i_i32(INDEX_op_negsetcond_i32, ret, arg1, arg2, cond); 551 } else { 552 tcg_gen_setcond_i32(cond, ret, arg1, arg2); 553 tcg_gen_neg_i32(ret, ret); 554 } 555 } 556 557 void tcg_gen_negsetcondi_i32(TCGCond cond, TCGv_i32 ret, 558 TCGv_i32 arg1, int32_t arg2) 559 { 560 tcg_gen_negsetcond_i32(cond, ret, arg1, tcg_constant_i32(arg2)); 561 } 562 563 void tcg_gen_mul_i32(TCGv_i32 ret, TCGv_i32 arg1, TCGv_i32 arg2) 564 { 565 tcg_gen_op3_i32(INDEX_op_mul_i32, ret, arg1, arg2); 566 } 567 568 void tcg_gen_muli_i32(TCGv_i32 ret, TCGv_i32 arg1, int32_t arg2) 569 { 570 if (arg2 == 0) { 571 tcg_gen_movi_i32(ret, 0); 572 } else if (is_power_of_2(arg2)) { 573 tcg_gen_shli_i32(ret, arg1, ctz32(arg2)); 574 } else { 575 tcg_gen_mul_i32(ret, arg1, tcg_constant_i32(arg2)); 576 } 577 } 578 579 void tcg_gen_div_i32(TCGv_i32 ret, TCGv_i32 arg1, TCGv_i32 arg2) 580 { 581 if (TCG_TARGET_HAS_div_i32) { 582 tcg_gen_op3_i32(INDEX_op_div_i32, ret, arg1, arg2); 583 } else if (TCG_TARGET_HAS_div2_i32) { 584 TCGv_i32 t0 = tcg_temp_ebb_new_i32(); 585 tcg_gen_sari_i32(t0, arg1, 31); 586 tcg_gen_op5_i32(INDEX_op_div2_i32, ret, t0, arg1, t0, arg2); 587 tcg_temp_free_i32(t0); 588 } else { 589 gen_helper_div_i32(ret, arg1, arg2); 590 } 591 } 592 593 void tcg_gen_rem_i32(TCGv_i32 ret, TCGv_i32 arg1, TCGv_i32 arg2) 594 { 595 if (TCG_TARGET_HAS_rem_i32) { 596 tcg_gen_op3_i32(INDEX_op_rem_i32, ret, arg1, arg2); 597 } else if (TCG_TARGET_HAS_div_i32) { 598 TCGv_i32 t0 = tcg_temp_ebb_new_i32(); 599 tcg_gen_op3_i32(INDEX_op_div_i32, t0, arg1, arg2); 600 tcg_gen_mul_i32(t0, t0, arg2); 601 tcg_gen_sub_i32(ret, arg1, t0); 602 tcg_temp_free_i32(t0); 603 } else if (TCG_TARGET_HAS_div2_i32) { 604 TCGv_i32 t0 = tcg_temp_ebb_new_i32(); 605 tcg_gen_sari_i32(t0, arg1, 31); 606 tcg_gen_op5_i32(INDEX_op_div2_i32, t0, ret, arg1, t0, arg2); 607 tcg_temp_free_i32(t0); 608 } else { 609 gen_helper_rem_i32(ret, arg1, arg2); 610 } 611 } 612 613 void tcg_gen_divu_i32(TCGv_i32 ret, TCGv_i32 arg1, TCGv_i32 arg2) 614 { 615 if (TCG_TARGET_HAS_div_i32) { 616 tcg_gen_op3_i32(INDEX_op_divu_i32, ret, arg1, arg2); 617 } else if (TCG_TARGET_HAS_div2_i32) { 618 TCGv_i32 t0 = tcg_temp_ebb_new_i32(); 619 TCGv_i32 zero = tcg_constant_i32(0); 620 tcg_gen_op5_i32(INDEX_op_divu2_i32, ret, t0, arg1, zero, arg2); 621 tcg_temp_free_i32(t0); 622 } else { 623 gen_helper_divu_i32(ret, arg1, arg2); 624 } 625 } 626 627 void tcg_gen_remu_i32(TCGv_i32 ret, TCGv_i32 arg1, TCGv_i32 arg2) 628 { 629 if (TCG_TARGET_HAS_rem_i32) { 630 tcg_gen_op3_i32(INDEX_op_remu_i32, ret, arg1, arg2); 631 } else if (TCG_TARGET_HAS_div_i32) { 632 TCGv_i32 t0 = tcg_temp_ebb_new_i32(); 633 tcg_gen_op3_i32(INDEX_op_divu_i32, t0, arg1, arg2); 634 tcg_gen_mul_i32(t0, t0, arg2); 635 tcg_gen_sub_i32(ret, arg1, t0); 636 tcg_temp_free_i32(t0); 637 } else if (TCG_TARGET_HAS_div2_i32) { 638 TCGv_i32 t0 = tcg_temp_ebb_new_i32(); 639 TCGv_i32 zero = tcg_constant_i32(0); 640 tcg_gen_op5_i32(INDEX_op_divu2_i32, t0, ret, arg1, zero, arg2); 641 tcg_temp_free_i32(t0); 642 } else { 643 gen_helper_remu_i32(ret, arg1, arg2); 644 } 645 } 646 647 void tcg_gen_andc_i32(TCGv_i32 ret, TCGv_i32 arg1, TCGv_i32 arg2) 648 { 649 if (TCG_TARGET_HAS_andc_i32) { 650 tcg_gen_op3_i32(INDEX_op_andc_i32, ret, arg1, arg2); 651 } else { 652 TCGv_i32 t0 = tcg_temp_ebb_new_i32(); 653 tcg_gen_not_i32(t0, arg2); 654 tcg_gen_and_i32(ret, arg1, t0); 655 tcg_temp_free_i32(t0); 656 } 657 } 658 659 void tcg_gen_eqv_i32(TCGv_i32 ret, TCGv_i32 arg1, TCGv_i32 arg2) 660 { 661 if (TCG_TARGET_HAS_eqv_i32) { 662 tcg_gen_op3_i32(INDEX_op_eqv_i32, ret, arg1, arg2); 663 } else { 664 tcg_gen_xor_i32(ret, arg1, arg2); 665 tcg_gen_not_i32(ret, ret); 666 } 667 } 668 669 void tcg_gen_nand_i32(TCGv_i32 ret, TCGv_i32 arg1, TCGv_i32 arg2) 670 { 671 if (TCG_TARGET_HAS_nand_i32) { 672 tcg_gen_op3_i32(INDEX_op_nand_i32, ret, arg1, arg2); 673 } else { 674 tcg_gen_and_i32(ret, arg1, arg2); 675 tcg_gen_not_i32(ret, ret); 676 } 677 } 678 679 void tcg_gen_nor_i32(TCGv_i32 ret, TCGv_i32 arg1, TCGv_i32 arg2) 680 { 681 if (TCG_TARGET_HAS_nor_i32) { 682 tcg_gen_op3_i32(INDEX_op_nor_i32, ret, arg1, arg2); 683 } else { 684 tcg_gen_or_i32(ret, arg1, arg2); 685 tcg_gen_not_i32(ret, ret); 686 } 687 } 688 689 void tcg_gen_orc_i32(TCGv_i32 ret, TCGv_i32 arg1, TCGv_i32 arg2) 690 { 691 if (TCG_TARGET_HAS_orc_i32) { 692 tcg_gen_op3_i32(INDEX_op_orc_i32, ret, arg1, arg2); 693 } else { 694 TCGv_i32 t0 = tcg_temp_ebb_new_i32(); 695 tcg_gen_not_i32(t0, arg2); 696 tcg_gen_or_i32(ret, arg1, t0); 697 tcg_temp_free_i32(t0); 698 } 699 } 700 701 void tcg_gen_clz_i32(TCGv_i32 ret, TCGv_i32 arg1, TCGv_i32 arg2) 702 { 703 if (TCG_TARGET_HAS_clz_i32) { 704 tcg_gen_op3_i32(INDEX_op_clz_i32, ret, arg1, arg2); 705 } else if (TCG_TARGET_HAS_clz_i64) { 706 TCGv_i64 t1 = tcg_temp_ebb_new_i64(); 707 TCGv_i64 t2 = tcg_temp_ebb_new_i64(); 708 tcg_gen_extu_i32_i64(t1, arg1); 709 tcg_gen_extu_i32_i64(t2, arg2); 710 tcg_gen_addi_i64(t2, t2, 32); 711 tcg_gen_clz_i64(t1, t1, t2); 712 tcg_gen_extrl_i64_i32(ret, t1); 713 tcg_temp_free_i64(t1); 714 tcg_temp_free_i64(t2); 715 tcg_gen_subi_i32(ret, ret, 32); 716 } else { 717 gen_helper_clz_i32(ret, arg1, arg2); 718 } 719 } 720 721 void tcg_gen_clzi_i32(TCGv_i32 ret, TCGv_i32 arg1, uint32_t arg2) 722 { 723 tcg_gen_clz_i32(ret, arg1, tcg_constant_i32(arg2)); 724 } 725 726 void tcg_gen_ctz_i32(TCGv_i32 ret, TCGv_i32 arg1, TCGv_i32 arg2) 727 { 728 if (TCG_TARGET_HAS_ctz_i32) { 729 tcg_gen_op3_i32(INDEX_op_ctz_i32, ret, arg1, arg2); 730 } else if (TCG_TARGET_HAS_ctz_i64) { 731 TCGv_i64 t1 = tcg_temp_ebb_new_i64(); 732 TCGv_i64 t2 = tcg_temp_ebb_new_i64(); 733 tcg_gen_extu_i32_i64(t1, arg1); 734 tcg_gen_extu_i32_i64(t2, arg2); 735 tcg_gen_ctz_i64(t1, t1, t2); 736 tcg_gen_extrl_i64_i32(ret, t1); 737 tcg_temp_free_i64(t1); 738 tcg_temp_free_i64(t2); 739 } else if (TCG_TARGET_HAS_ctpop_i32 740 || TCG_TARGET_HAS_ctpop_i64 741 || TCG_TARGET_HAS_clz_i32 742 || TCG_TARGET_HAS_clz_i64) { 743 TCGv_i32 z, t = tcg_temp_ebb_new_i32(); 744 745 if (TCG_TARGET_HAS_ctpop_i32 || TCG_TARGET_HAS_ctpop_i64) { 746 tcg_gen_subi_i32(t, arg1, 1); 747 tcg_gen_andc_i32(t, t, arg1); 748 tcg_gen_ctpop_i32(t, t); 749 } else { 750 /* Since all non-x86 hosts have clz(0) == 32, don't fight it. */ 751 tcg_gen_neg_i32(t, arg1); 752 tcg_gen_and_i32(t, t, arg1); 753 tcg_gen_clzi_i32(t, t, 32); 754 tcg_gen_xori_i32(t, t, 31); 755 } 756 z = tcg_constant_i32(0); 757 tcg_gen_movcond_i32(TCG_COND_EQ, ret, arg1, z, arg2, t); 758 tcg_temp_free_i32(t); 759 } else { 760 gen_helper_ctz_i32(ret, arg1, arg2); 761 } 762 } 763 764 void tcg_gen_ctzi_i32(TCGv_i32 ret, TCGv_i32 arg1, uint32_t arg2) 765 { 766 if (!TCG_TARGET_HAS_ctz_i32 && TCG_TARGET_HAS_ctpop_i32 && arg2 == 32) { 767 /* This equivalence has the advantage of not requiring a fixup. */ 768 TCGv_i32 t = tcg_temp_ebb_new_i32(); 769 tcg_gen_subi_i32(t, arg1, 1); 770 tcg_gen_andc_i32(t, t, arg1); 771 tcg_gen_ctpop_i32(ret, t); 772 tcg_temp_free_i32(t); 773 } else { 774 tcg_gen_ctz_i32(ret, arg1, tcg_constant_i32(arg2)); 775 } 776 } 777 778 void tcg_gen_clrsb_i32(TCGv_i32 ret, TCGv_i32 arg) 779 { 780 if (TCG_TARGET_HAS_clz_i32) { 781 TCGv_i32 t = tcg_temp_ebb_new_i32(); 782 tcg_gen_sari_i32(t, arg, 31); 783 tcg_gen_xor_i32(t, t, arg); 784 tcg_gen_clzi_i32(t, t, 32); 785 tcg_gen_subi_i32(ret, t, 1); 786 tcg_temp_free_i32(t); 787 } else { 788 gen_helper_clrsb_i32(ret, arg); 789 } 790 } 791 792 void tcg_gen_ctpop_i32(TCGv_i32 ret, TCGv_i32 arg1) 793 { 794 if (TCG_TARGET_HAS_ctpop_i32) { 795 tcg_gen_op2_i32(INDEX_op_ctpop_i32, ret, arg1); 796 } else if (TCG_TARGET_HAS_ctpop_i64) { 797 TCGv_i64 t = tcg_temp_ebb_new_i64(); 798 tcg_gen_extu_i32_i64(t, arg1); 799 tcg_gen_ctpop_i64(t, t); 800 tcg_gen_extrl_i64_i32(ret, t); 801 tcg_temp_free_i64(t); 802 } else { 803 gen_helper_ctpop_i32(ret, arg1); 804 } 805 } 806 807 void tcg_gen_rotl_i32(TCGv_i32 ret, TCGv_i32 arg1, TCGv_i32 arg2) 808 { 809 if (TCG_TARGET_HAS_rot_i32) { 810 tcg_gen_op3_i32(INDEX_op_rotl_i32, ret, arg1, arg2); 811 } else { 812 TCGv_i32 t0, t1; 813 814 t0 = tcg_temp_ebb_new_i32(); 815 t1 = tcg_temp_ebb_new_i32(); 816 tcg_gen_shl_i32(t0, arg1, arg2); 817 tcg_gen_subfi_i32(t1, 32, arg2); 818 tcg_gen_shr_i32(t1, arg1, t1); 819 tcg_gen_or_i32(ret, t0, t1); 820 tcg_temp_free_i32(t0); 821 tcg_temp_free_i32(t1); 822 } 823 } 824 825 void tcg_gen_rotli_i32(TCGv_i32 ret, TCGv_i32 arg1, int32_t arg2) 826 { 827 tcg_debug_assert(arg2 >= 0 && arg2 < 32); 828 /* some cases can be optimized here */ 829 if (arg2 == 0) { 830 tcg_gen_mov_i32(ret, arg1); 831 } else if (TCG_TARGET_HAS_rot_i32) { 832 tcg_gen_rotl_i32(ret, arg1, tcg_constant_i32(arg2)); 833 } else { 834 TCGv_i32 t0, t1; 835 t0 = tcg_temp_ebb_new_i32(); 836 t1 = tcg_temp_ebb_new_i32(); 837 tcg_gen_shli_i32(t0, arg1, arg2); 838 tcg_gen_shri_i32(t1, arg1, 32 - arg2); 839 tcg_gen_or_i32(ret, t0, t1); 840 tcg_temp_free_i32(t0); 841 tcg_temp_free_i32(t1); 842 } 843 } 844 845 void tcg_gen_rotr_i32(TCGv_i32 ret, TCGv_i32 arg1, TCGv_i32 arg2) 846 { 847 if (TCG_TARGET_HAS_rot_i32) { 848 tcg_gen_op3_i32(INDEX_op_rotr_i32, ret, arg1, arg2); 849 } else { 850 TCGv_i32 t0, t1; 851 852 t0 = tcg_temp_ebb_new_i32(); 853 t1 = tcg_temp_ebb_new_i32(); 854 tcg_gen_shr_i32(t0, arg1, arg2); 855 tcg_gen_subfi_i32(t1, 32, arg2); 856 tcg_gen_shl_i32(t1, arg1, t1); 857 tcg_gen_or_i32(ret, t0, t1); 858 tcg_temp_free_i32(t0); 859 tcg_temp_free_i32(t1); 860 } 861 } 862 863 void tcg_gen_rotri_i32(TCGv_i32 ret, TCGv_i32 arg1, int32_t arg2) 864 { 865 tcg_debug_assert(arg2 >= 0 && arg2 < 32); 866 /* some cases can be optimized here */ 867 if (arg2 == 0) { 868 tcg_gen_mov_i32(ret, arg1); 869 } else { 870 tcg_gen_rotli_i32(ret, arg1, 32 - arg2); 871 } 872 } 873 874 void tcg_gen_deposit_i32(TCGv_i32 ret, TCGv_i32 arg1, TCGv_i32 arg2, 875 unsigned int ofs, unsigned int len) 876 { 877 uint32_t mask; 878 TCGv_i32 t1; 879 880 tcg_debug_assert(ofs < 32); 881 tcg_debug_assert(len > 0); 882 tcg_debug_assert(len <= 32); 883 tcg_debug_assert(ofs + len <= 32); 884 885 if (len == 32) { 886 tcg_gen_mov_i32(ret, arg2); 887 return; 888 } 889 if (TCG_TARGET_HAS_deposit_i32 && TCG_TARGET_deposit_i32_valid(ofs, len)) { 890 tcg_gen_op5ii_i32(INDEX_op_deposit_i32, ret, arg1, arg2, ofs, len); 891 return; 892 } 893 894 t1 = tcg_temp_ebb_new_i32(); 895 896 if (TCG_TARGET_HAS_extract2_i32) { 897 if (ofs + len == 32) { 898 tcg_gen_shli_i32(t1, arg1, len); 899 tcg_gen_extract2_i32(ret, t1, arg2, len); 900 goto done; 901 } 902 if (ofs == 0) { 903 tcg_gen_extract2_i32(ret, arg1, arg2, len); 904 tcg_gen_rotli_i32(ret, ret, len); 905 goto done; 906 } 907 } 908 909 mask = (1u << len) - 1; 910 if (ofs + len < 32) { 911 tcg_gen_andi_i32(t1, arg2, mask); 912 tcg_gen_shli_i32(t1, t1, ofs); 913 } else { 914 tcg_gen_shli_i32(t1, arg2, ofs); 915 } 916 tcg_gen_andi_i32(ret, arg1, ~(mask << ofs)); 917 tcg_gen_or_i32(ret, ret, t1); 918 done: 919 tcg_temp_free_i32(t1); 920 } 921 922 void tcg_gen_deposit_z_i32(TCGv_i32 ret, TCGv_i32 arg, 923 unsigned int ofs, unsigned int len) 924 { 925 tcg_debug_assert(ofs < 32); 926 tcg_debug_assert(len > 0); 927 tcg_debug_assert(len <= 32); 928 tcg_debug_assert(ofs + len <= 32); 929 930 if (ofs + len == 32) { 931 tcg_gen_shli_i32(ret, arg, ofs); 932 } else if (ofs == 0) { 933 tcg_gen_andi_i32(ret, arg, (1u << len) - 1); 934 } else if (TCG_TARGET_HAS_deposit_i32 935 && TCG_TARGET_deposit_i32_valid(ofs, len)) { 936 TCGv_i32 zero = tcg_constant_i32(0); 937 tcg_gen_op5ii_i32(INDEX_op_deposit_i32, ret, zero, arg, ofs, len); 938 } else { 939 /* To help two-operand hosts we prefer to zero-extend first, 940 which allows ARG to stay live. */ 941 switch (len) { 942 case 16: 943 if (TCG_TARGET_HAS_ext16u_i32) { 944 tcg_gen_ext16u_i32(ret, arg); 945 tcg_gen_shli_i32(ret, ret, ofs); 946 return; 947 } 948 break; 949 case 8: 950 if (TCG_TARGET_HAS_ext8u_i32) { 951 tcg_gen_ext8u_i32(ret, arg); 952 tcg_gen_shli_i32(ret, ret, ofs); 953 return; 954 } 955 break; 956 } 957 /* Otherwise prefer zero-extension over AND for code size. */ 958 switch (ofs + len) { 959 case 16: 960 if (TCG_TARGET_HAS_ext16u_i32) { 961 tcg_gen_shli_i32(ret, arg, ofs); 962 tcg_gen_ext16u_i32(ret, ret); 963 return; 964 } 965 break; 966 case 8: 967 if (TCG_TARGET_HAS_ext8u_i32) { 968 tcg_gen_shli_i32(ret, arg, ofs); 969 tcg_gen_ext8u_i32(ret, ret); 970 return; 971 } 972 break; 973 } 974 tcg_gen_andi_i32(ret, arg, (1u << len) - 1); 975 tcg_gen_shli_i32(ret, ret, ofs); 976 } 977 } 978 979 void tcg_gen_extract_i32(TCGv_i32 ret, TCGv_i32 arg, 980 unsigned int ofs, unsigned int len) 981 { 982 tcg_debug_assert(ofs < 32); 983 tcg_debug_assert(len > 0); 984 tcg_debug_assert(len <= 32); 985 tcg_debug_assert(ofs + len <= 32); 986 987 /* Canonicalize certain special cases, even if extract is supported. */ 988 if (ofs + len == 32) { 989 tcg_gen_shri_i32(ret, arg, 32 - len); 990 return; 991 } 992 if (ofs == 0) { 993 tcg_gen_andi_i32(ret, arg, (1u << len) - 1); 994 return; 995 } 996 997 if (TCG_TARGET_HAS_extract_i32 998 && TCG_TARGET_extract_i32_valid(ofs, len)) { 999 tcg_gen_op4ii_i32(INDEX_op_extract_i32, ret, arg, ofs, len); 1000 return; 1001 } 1002 1003 /* Assume that zero-extension, if available, is cheaper than a shift. */ 1004 switch (ofs + len) { 1005 case 16: 1006 if (TCG_TARGET_HAS_ext16u_i32) { 1007 tcg_gen_ext16u_i32(ret, arg); 1008 tcg_gen_shri_i32(ret, ret, ofs); 1009 return; 1010 } 1011 break; 1012 case 8: 1013 if (TCG_TARGET_HAS_ext8u_i32) { 1014 tcg_gen_ext8u_i32(ret, arg); 1015 tcg_gen_shri_i32(ret, ret, ofs); 1016 return; 1017 } 1018 break; 1019 } 1020 1021 /* ??? Ideally we'd know what values are available for immediate AND. 1022 Assume that 8 bits are available, plus the special case of 16, 1023 so that we get ext8u, ext16u. */ 1024 switch (len) { 1025 case 1 ... 8: case 16: 1026 tcg_gen_shri_i32(ret, arg, ofs); 1027 tcg_gen_andi_i32(ret, ret, (1u << len) - 1); 1028 break; 1029 default: 1030 tcg_gen_shli_i32(ret, arg, 32 - len - ofs); 1031 tcg_gen_shri_i32(ret, ret, 32 - len); 1032 break; 1033 } 1034 } 1035 1036 void tcg_gen_sextract_i32(TCGv_i32 ret, TCGv_i32 arg, 1037 unsigned int ofs, unsigned int len) 1038 { 1039 tcg_debug_assert(ofs < 32); 1040 tcg_debug_assert(len > 0); 1041 tcg_debug_assert(len <= 32); 1042 tcg_debug_assert(ofs + len <= 32); 1043 1044 /* Canonicalize certain special cases, even if extract is supported. */ 1045 if (ofs + len == 32) { 1046 tcg_gen_sari_i32(ret, arg, 32 - len); 1047 return; 1048 } 1049 if (ofs == 0) { 1050 switch (len) { 1051 case 16: 1052 tcg_gen_ext16s_i32(ret, arg); 1053 return; 1054 case 8: 1055 tcg_gen_ext8s_i32(ret, arg); 1056 return; 1057 } 1058 } 1059 1060 if (TCG_TARGET_HAS_sextract_i32 1061 && TCG_TARGET_extract_i32_valid(ofs, len)) { 1062 tcg_gen_op4ii_i32(INDEX_op_sextract_i32, ret, arg, ofs, len); 1063 return; 1064 } 1065 1066 /* Assume that sign-extension, if available, is cheaper than a shift. */ 1067 switch (ofs + len) { 1068 case 16: 1069 if (TCG_TARGET_HAS_ext16s_i32) { 1070 tcg_gen_ext16s_i32(ret, arg); 1071 tcg_gen_sari_i32(ret, ret, ofs); 1072 return; 1073 } 1074 break; 1075 case 8: 1076 if (TCG_TARGET_HAS_ext8s_i32) { 1077 tcg_gen_ext8s_i32(ret, arg); 1078 tcg_gen_sari_i32(ret, ret, ofs); 1079 return; 1080 } 1081 break; 1082 } 1083 switch (len) { 1084 case 16: 1085 if (TCG_TARGET_HAS_ext16s_i32) { 1086 tcg_gen_shri_i32(ret, arg, ofs); 1087 tcg_gen_ext16s_i32(ret, ret); 1088 return; 1089 } 1090 break; 1091 case 8: 1092 if (TCG_TARGET_HAS_ext8s_i32) { 1093 tcg_gen_shri_i32(ret, arg, ofs); 1094 tcg_gen_ext8s_i32(ret, ret); 1095 return; 1096 } 1097 break; 1098 } 1099 1100 tcg_gen_shli_i32(ret, arg, 32 - len - ofs); 1101 tcg_gen_sari_i32(ret, ret, 32 - len); 1102 } 1103 1104 /* 1105 * Extract 32-bits from a 64-bit input, ah:al, starting from ofs. 1106 * Unlike tcg_gen_extract_i32 above, len is fixed at 32. 1107 */ 1108 void tcg_gen_extract2_i32(TCGv_i32 ret, TCGv_i32 al, TCGv_i32 ah, 1109 unsigned int ofs) 1110 { 1111 tcg_debug_assert(ofs <= 32); 1112 if (ofs == 0) { 1113 tcg_gen_mov_i32(ret, al); 1114 } else if (ofs == 32) { 1115 tcg_gen_mov_i32(ret, ah); 1116 } else if (al == ah) { 1117 tcg_gen_rotri_i32(ret, al, ofs); 1118 } else if (TCG_TARGET_HAS_extract2_i32) { 1119 tcg_gen_op4i_i32(INDEX_op_extract2_i32, ret, al, ah, ofs); 1120 } else { 1121 TCGv_i32 t0 = tcg_temp_ebb_new_i32(); 1122 tcg_gen_shri_i32(t0, al, ofs); 1123 tcg_gen_deposit_i32(ret, t0, ah, 32 - ofs, ofs); 1124 tcg_temp_free_i32(t0); 1125 } 1126 } 1127 1128 void tcg_gen_movcond_i32(TCGCond cond, TCGv_i32 ret, TCGv_i32 c1, 1129 TCGv_i32 c2, TCGv_i32 v1, TCGv_i32 v2) 1130 { 1131 if (cond == TCG_COND_ALWAYS) { 1132 tcg_gen_mov_i32(ret, v1); 1133 } else if (cond == TCG_COND_NEVER) { 1134 tcg_gen_mov_i32(ret, v2); 1135 } else { 1136 tcg_gen_op6i_i32(INDEX_op_movcond_i32, ret, c1, c2, v1, v2, cond); 1137 } 1138 } 1139 1140 void tcg_gen_add2_i32(TCGv_i32 rl, TCGv_i32 rh, TCGv_i32 al, 1141 TCGv_i32 ah, TCGv_i32 bl, TCGv_i32 bh) 1142 { 1143 if (TCG_TARGET_HAS_add2_i32) { 1144 tcg_gen_op6_i32(INDEX_op_add2_i32, rl, rh, al, ah, bl, bh); 1145 } else { 1146 TCGv_i64 t0 = tcg_temp_ebb_new_i64(); 1147 TCGv_i64 t1 = tcg_temp_ebb_new_i64(); 1148 tcg_gen_concat_i32_i64(t0, al, ah); 1149 tcg_gen_concat_i32_i64(t1, bl, bh); 1150 tcg_gen_add_i64(t0, t0, t1); 1151 tcg_gen_extr_i64_i32(rl, rh, t0); 1152 tcg_temp_free_i64(t0); 1153 tcg_temp_free_i64(t1); 1154 } 1155 } 1156 1157 void tcg_gen_sub2_i32(TCGv_i32 rl, TCGv_i32 rh, TCGv_i32 al, 1158 TCGv_i32 ah, TCGv_i32 bl, TCGv_i32 bh) 1159 { 1160 if (TCG_TARGET_HAS_sub2_i32) { 1161 tcg_gen_op6_i32(INDEX_op_sub2_i32, rl, rh, al, ah, bl, bh); 1162 } else { 1163 TCGv_i64 t0 = tcg_temp_ebb_new_i64(); 1164 TCGv_i64 t1 = tcg_temp_ebb_new_i64(); 1165 tcg_gen_concat_i32_i64(t0, al, ah); 1166 tcg_gen_concat_i32_i64(t1, bl, bh); 1167 tcg_gen_sub_i64(t0, t0, t1); 1168 tcg_gen_extr_i64_i32(rl, rh, t0); 1169 tcg_temp_free_i64(t0); 1170 tcg_temp_free_i64(t1); 1171 } 1172 } 1173 1174 void tcg_gen_mulu2_i32(TCGv_i32 rl, TCGv_i32 rh, TCGv_i32 arg1, TCGv_i32 arg2) 1175 { 1176 if (TCG_TARGET_HAS_mulu2_i32) { 1177 tcg_gen_op4_i32(INDEX_op_mulu2_i32, rl, rh, arg1, arg2); 1178 } else if (TCG_TARGET_HAS_muluh_i32) { 1179 TCGv_i32 t = tcg_temp_ebb_new_i32(); 1180 tcg_gen_op3_i32(INDEX_op_mul_i32, t, arg1, arg2); 1181 tcg_gen_op3_i32(INDEX_op_muluh_i32, rh, arg1, arg2); 1182 tcg_gen_mov_i32(rl, t); 1183 tcg_temp_free_i32(t); 1184 } else if (TCG_TARGET_REG_BITS == 64) { 1185 TCGv_i64 t0 = tcg_temp_ebb_new_i64(); 1186 TCGv_i64 t1 = tcg_temp_ebb_new_i64(); 1187 tcg_gen_extu_i32_i64(t0, arg1); 1188 tcg_gen_extu_i32_i64(t1, arg2); 1189 tcg_gen_mul_i64(t0, t0, t1); 1190 tcg_gen_extr_i64_i32(rl, rh, t0); 1191 tcg_temp_free_i64(t0); 1192 tcg_temp_free_i64(t1); 1193 } else { 1194 qemu_build_not_reached(); 1195 } 1196 } 1197 1198 void tcg_gen_muls2_i32(TCGv_i32 rl, TCGv_i32 rh, TCGv_i32 arg1, TCGv_i32 arg2) 1199 { 1200 if (TCG_TARGET_HAS_muls2_i32) { 1201 tcg_gen_op4_i32(INDEX_op_muls2_i32, rl, rh, arg1, arg2); 1202 } else if (TCG_TARGET_HAS_mulsh_i32) { 1203 TCGv_i32 t = tcg_temp_ebb_new_i32(); 1204 tcg_gen_op3_i32(INDEX_op_mul_i32, t, arg1, arg2); 1205 tcg_gen_op3_i32(INDEX_op_mulsh_i32, rh, arg1, arg2); 1206 tcg_gen_mov_i32(rl, t); 1207 tcg_temp_free_i32(t); 1208 } else if (TCG_TARGET_REG_BITS == 32) { 1209 TCGv_i32 t0 = tcg_temp_ebb_new_i32(); 1210 TCGv_i32 t1 = tcg_temp_ebb_new_i32(); 1211 TCGv_i32 t2 = tcg_temp_ebb_new_i32(); 1212 TCGv_i32 t3 = tcg_temp_ebb_new_i32(); 1213 tcg_gen_mulu2_i32(t0, t1, arg1, arg2); 1214 /* Adjust for negative inputs. */ 1215 tcg_gen_sari_i32(t2, arg1, 31); 1216 tcg_gen_sari_i32(t3, arg2, 31); 1217 tcg_gen_and_i32(t2, t2, arg2); 1218 tcg_gen_and_i32(t3, t3, arg1); 1219 tcg_gen_sub_i32(rh, t1, t2); 1220 tcg_gen_sub_i32(rh, rh, t3); 1221 tcg_gen_mov_i32(rl, t0); 1222 tcg_temp_free_i32(t0); 1223 tcg_temp_free_i32(t1); 1224 tcg_temp_free_i32(t2); 1225 tcg_temp_free_i32(t3); 1226 } else { 1227 TCGv_i64 t0 = tcg_temp_ebb_new_i64(); 1228 TCGv_i64 t1 = tcg_temp_ebb_new_i64(); 1229 tcg_gen_ext_i32_i64(t0, arg1); 1230 tcg_gen_ext_i32_i64(t1, arg2); 1231 tcg_gen_mul_i64(t0, t0, t1); 1232 tcg_gen_extr_i64_i32(rl, rh, t0); 1233 tcg_temp_free_i64(t0); 1234 tcg_temp_free_i64(t1); 1235 } 1236 } 1237 1238 void tcg_gen_mulsu2_i32(TCGv_i32 rl, TCGv_i32 rh, TCGv_i32 arg1, TCGv_i32 arg2) 1239 { 1240 if (TCG_TARGET_REG_BITS == 32) { 1241 TCGv_i32 t0 = tcg_temp_ebb_new_i32(); 1242 TCGv_i32 t1 = tcg_temp_ebb_new_i32(); 1243 TCGv_i32 t2 = tcg_temp_ebb_new_i32(); 1244 tcg_gen_mulu2_i32(t0, t1, arg1, arg2); 1245 /* Adjust for negative input for the signed arg1. */ 1246 tcg_gen_sari_i32(t2, arg1, 31); 1247 tcg_gen_and_i32(t2, t2, arg2); 1248 tcg_gen_sub_i32(rh, t1, t2); 1249 tcg_gen_mov_i32(rl, t0); 1250 tcg_temp_free_i32(t0); 1251 tcg_temp_free_i32(t1); 1252 tcg_temp_free_i32(t2); 1253 } else { 1254 TCGv_i64 t0 = tcg_temp_ebb_new_i64(); 1255 TCGv_i64 t1 = tcg_temp_ebb_new_i64(); 1256 tcg_gen_ext_i32_i64(t0, arg1); 1257 tcg_gen_extu_i32_i64(t1, arg2); 1258 tcg_gen_mul_i64(t0, t0, t1); 1259 tcg_gen_extr_i64_i32(rl, rh, t0); 1260 tcg_temp_free_i64(t0); 1261 tcg_temp_free_i64(t1); 1262 } 1263 } 1264 1265 void tcg_gen_ext8s_i32(TCGv_i32 ret, TCGv_i32 arg) 1266 { 1267 if (TCG_TARGET_HAS_ext8s_i32) { 1268 tcg_gen_op2_i32(INDEX_op_ext8s_i32, ret, arg); 1269 } else { 1270 tcg_gen_shli_i32(ret, arg, 24); 1271 tcg_gen_sari_i32(ret, ret, 24); 1272 } 1273 } 1274 1275 void tcg_gen_ext16s_i32(TCGv_i32 ret, TCGv_i32 arg) 1276 { 1277 if (TCG_TARGET_HAS_ext16s_i32) { 1278 tcg_gen_op2_i32(INDEX_op_ext16s_i32, ret, arg); 1279 } else { 1280 tcg_gen_shli_i32(ret, arg, 16); 1281 tcg_gen_sari_i32(ret, ret, 16); 1282 } 1283 } 1284 1285 void tcg_gen_ext8u_i32(TCGv_i32 ret, TCGv_i32 arg) 1286 { 1287 if (TCG_TARGET_HAS_ext8u_i32) { 1288 tcg_gen_op2_i32(INDEX_op_ext8u_i32, ret, arg); 1289 } else { 1290 tcg_gen_andi_i32(ret, arg, 0xffu); 1291 } 1292 } 1293 1294 void tcg_gen_ext16u_i32(TCGv_i32 ret, TCGv_i32 arg) 1295 { 1296 if (TCG_TARGET_HAS_ext16u_i32) { 1297 tcg_gen_op2_i32(INDEX_op_ext16u_i32, ret, arg); 1298 } else { 1299 tcg_gen_andi_i32(ret, arg, 0xffffu); 1300 } 1301 } 1302 1303 /* 1304 * bswap16_i32: 16-bit byte swap on the low bits of a 32-bit value. 1305 * 1306 * Byte pattern: xxab -> yyba 1307 * 1308 * With TCG_BSWAP_IZ, x == zero, else undefined. 1309 * With TCG_BSWAP_OZ, y == zero, with TCG_BSWAP_OS y == sign, else undefined. 1310 */ 1311 void tcg_gen_bswap16_i32(TCGv_i32 ret, TCGv_i32 arg, int flags) 1312 { 1313 /* Only one extension flag may be present. */ 1314 tcg_debug_assert(!(flags & TCG_BSWAP_OS) || !(flags & TCG_BSWAP_OZ)); 1315 1316 if (TCG_TARGET_HAS_bswap16_i32) { 1317 tcg_gen_op3i_i32(INDEX_op_bswap16_i32, ret, arg, flags); 1318 } else { 1319 TCGv_i32 t0 = tcg_temp_ebb_new_i32(); 1320 TCGv_i32 t1 = tcg_temp_ebb_new_i32(); 1321 1322 /* arg = ..ab (IZ) xxab (!IZ) */ 1323 tcg_gen_shri_i32(t0, arg, 8); /* t0 = ...a (IZ) .xxa (!IZ) */ 1324 if (!(flags & TCG_BSWAP_IZ)) { 1325 tcg_gen_ext8u_i32(t0, t0); /* t0 = ...a */ 1326 } 1327 1328 if (flags & TCG_BSWAP_OS) { 1329 tcg_gen_shli_i32(t1, arg, 24); /* t1 = b... */ 1330 tcg_gen_sari_i32(t1, t1, 16); /* t1 = ssb. */ 1331 } else if (flags & TCG_BSWAP_OZ) { 1332 tcg_gen_ext8u_i32(t1, arg); /* t1 = ...b */ 1333 tcg_gen_shli_i32(t1, t1, 8); /* t1 = ..b. */ 1334 } else { 1335 tcg_gen_shli_i32(t1, arg, 8); /* t1 = xab. */ 1336 } 1337 1338 tcg_gen_or_i32(ret, t0, t1); /* ret = ..ba (OZ) */ 1339 /* = ssba (OS) */ 1340 /* = xaba (no flag) */ 1341 tcg_temp_free_i32(t0); 1342 tcg_temp_free_i32(t1); 1343 } 1344 } 1345 1346 /* 1347 * bswap32_i32: 32-bit byte swap on a 32-bit value. 1348 * 1349 * Byte pattern: abcd -> dcba 1350 */ 1351 void tcg_gen_bswap32_i32(TCGv_i32 ret, TCGv_i32 arg) 1352 { 1353 if (TCG_TARGET_HAS_bswap32_i32) { 1354 tcg_gen_op3i_i32(INDEX_op_bswap32_i32, ret, arg, 0); 1355 } else { 1356 TCGv_i32 t0 = tcg_temp_ebb_new_i32(); 1357 TCGv_i32 t1 = tcg_temp_ebb_new_i32(); 1358 TCGv_i32 t2 = tcg_constant_i32(0x00ff00ff); 1359 1360 /* arg = abcd */ 1361 tcg_gen_shri_i32(t0, arg, 8); /* t0 = .abc */ 1362 tcg_gen_and_i32(t1, arg, t2); /* t1 = .b.d */ 1363 tcg_gen_and_i32(t0, t0, t2); /* t0 = .a.c */ 1364 tcg_gen_shli_i32(t1, t1, 8); /* t1 = b.d. */ 1365 tcg_gen_or_i32(ret, t0, t1); /* ret = badc */ 1366 1367 tcg_gen_shri_i32(t0, ret, 16); /* t0 = ..ba */ 1368 tcg_gen_shli_i32(t1, ret, 16); /* t1 = dc.. */ 1369 tcg_gen_or_i32(ret, t0, t1); /* ret = dcba */ 1370 1371 tcg_temp_free_i32(t0); 1372 tcg_temp_free_i32(t1); 1373 } 1374 } 1375 1376 /* 1377 * hswap_i32: Swap 16-bit halfwords within a 32-bit value. 1378 * 1379 * Byte pattern: abcd -> cdab 1380 */ 1381 void tcg_gen_hswap_i32(TCGv_i32 ret, TCGv_i32 arg) 1382 { 1383 /* Swapping 2 16-bit elements is a rotate. */ 1384 tcg_gen_rotli_i32(ret, arg, 16); 1385 } 1386 1387 void tcg_gen_smin_i32(TCGv_i32 ret, TCGv_i32 a, TCGv_i32 b) 1388 { 1389 tcg_gen_movcond_i32(TCG_COND_LT, ret, a, b, a, b); 1390 } 1391 1392 void tcg_gen_umin_i32(TCGv_i32 ret, TCGv_i32 a, TCGv_i32 b) 1393 { 1394 tcg_gen_movcond_i32(TCG_COND_LTU, ret, a, b, a, b); 1395 } 1396 1397 void tcg_gen_smax_i32(TCGv_i32 ret, TCGv_i32 a, TCGv_i32 b) 1398 { 1399 tcg_gen_movcond_i32(TCG_COND_LT, ret, a, b, b, a); 1400 } 1401 1402 void tcg_gen_umax_i32(TCGv_i32 ret, TCGv_i32 a, TCGv_i32 b) 1403 { 1404 tcg_gen_movcond_i32(TCG_COND_LTU, ret, a, b, b, a); 1405 } 1406 1407 void tcg_gen_abs_i32(TCGv_i32 ret, TCGv_i32 a) 1408 { 1409 TCGv_i32 t = tcg_temp_ebb_new_i32(); 1410 1411 tcg_gen_sari_i32(t, a, 31); 1412 tcg_gen_xor_i32(ret, a, t); 1413 tcg_gen_sub_i32(ret, ret, t); 1414 tcg_temp_free_i32(t); 1415 } 1416 1417 void tcg_gen_ld8u_i32(TCGv_i32 ret, TCGv_ptr arg2, tcg_target_long offset) 1418 { 1419 tcg_gen_ldst_op_i32(INDEX_op_ld8u_i32, ret, arg2, offset); 1420 } 1421 1422 void tcg_gen_ld8s_i32(TCGv_i32 ret, TCGv_ptr arg2, tcg_target_long offset) 1423 { 1424 tcg_gen_ldst_op_i32(INDEX_op_ld8s_i32, ret, arg2, offset); 1425 } 1426 1427 void tcg_gen_ld16u_i32(TCGv_i32 ret, TCGv_ptr arg2, tcg_target_long offset) 1428 { 1429 tcg_gen_ldst_op_i32(INDEX_op_ld16u_i32, ret, arg2, offset); 1430 } 1431 1432 void tcg_gen_ld16s_i32(TCGv_i32 ret, TCGv_ptr arg2, tcg_target_long offset) 1433 { 1434 tcg_gen_ldst_op_i32(INDEX_op_ld16s_i32, ret, arg2, offset); 1435 } 1436 1437 void tcg_gen_ld_i32(TCGv_i32 ret, TCGv_ptr arg2, tcg_target_long offset) 1438 { 1439 tcg_gen_ldst_op_i32(INDEX_op_ld_i32, ret, arg2, offset); 1440 } 1441 1442 void tcg_gen_st8_i32(TCGv_i32 arg1, TCGv_ptr arg2, tcg_target_long offset) 1443 { 1444 tcg_gen_ldst_op_i32(INDEX_op_st8_i32, arg1, arg2, offset); 1445 } 1446 1447 void tcg_gen_st16_i32(TCGv_i32 arg1, TCGv_ptr arg2, tcg_target_long offset) 1448 { 1449 tcg_gen_ldst_op_i32(INDEX_op_st16_i32, arg1, arg2, offset); 1450 } 1451 1452 void tcg_gen_st_i32(TCGv_i32 arg1, TCGv_ptr arg2, tcg_target_long offset) 1453 { 1454 tcg_gen_ldst_op_i32(INDEX_op_st_i32, arg1, arg2, offset); 1455 } 1456 1457 1458 /* 64-bit ops */ 1459 1460 void tcg_gen_discard_i64(TCGv_i64 arg) 1461 { 1462 if (TCG_TARGET_REG_BITS == 64) { 1463 tcg_gen_op1_i64(INDEX_op_discard, arg); 1464 } else { 1465 tcg_gen_discard_i32(TCGV_LOW(arg)); 1466 tcg_gen_discard_i32(TCGV_HIGH(arg)); 1467 } 1468 } 1469 1470 void tcg_gen_mov_i64(TCGv_i64 ret, TCGv_i64 arg) 1471 { 1472 if (ret == arg) { 1473 return; 1474 } 1475 if (TCG_TARGET_REG_BITS == 64) { 1476 tcg_gen_op2_i64(INDEX_op_mov_i64, ret, arg); 1477 } else { 1478 TCGTemp *ts = tcgv_i64_temp(arg); 1479 1480 /* Canonicalize TCGv_i64 TEMP_CONST into TCGv_i32 TEMP_CONST. */ 1481 if (ts->kind == TEMP_CONST) { 1482 tcg_gen_movi_i64(ret, ts->val); 1483 } else { 1484 tcg_gen_mov_i32(TCGV_LOW(ret), TCGV_LOW(arg)); 1485 tcg_gen_mov_i32(TCGV_HIGH(ret), TCGV_HIGH(arg)); 1486 } 1487 } 1488 } 1489 1490 void tcg_gen_movi_i64(TCGv_i64 ret, int64_t arg) 1491 { 1492 if (TCG_TARGET_REG_BITS == 64) { 1493 tcg_gen_mov_i64(ret, tcg_constant_i64(arg)); 1494 } else { 1495 tcg_gen_movi_i32(TCGV_LOW(ret), arg); 1496 tcg_gen_movi_i32(TCGV_HIGH(ret), arg >> 32); 1497 } 1498 } 1499 1500 void tcg_gen_ld8u_i64(TCGv_i64 ret, TCGv_ptr arg2, tcg_target_long offset) 1501 { 1502 if (TCG_TARGET_REG_BITS == 64) { 1503 tcg_gen_ldst_op_i64(INDEX_op_ld8u_i64, ret, arg2, offset); 1504 } else { 1505 tcg_gen_ld8u_i32(TCGV_LOW(ret), arg2, offset); 1506 tcg_gen_movi_i32(TCGV_HIGH(ret), 0); 1507 } 1508 } 1509 1510 void tcg_gen_ld8s_i64(TCGv_i64 ret, TCGv_ptr arg2, tcg_target_long offset) 1511 { 1512 if (TCG_TARGET_REG_BITS == 64) { 1513 tcg_gen_ldst_op_i64(INDEX_op_ld8s_i64, ret, arg2, offset); 1514 } else { 1515 tcg_gen_ld8s_i32(TCGV_LOW(ret), arg2, offset); 1516 tcg_gen_sari_i32(TCGV_HIGH(ret), TCGV_LOW(ret), 31); 1517 } 1518 } 1519 1520 void tcg_gen_ld16u_i64(TCGv_i64 ret, TCGv_ptr arg2, tcg_target_long offset) 1521 { 1522 if (TCG_TARGET_REG_BITS == 64) { 1523 tcg_gen_ldst_op_i64(INDEX_op_ld16u_i64, ret, arg2, offset); 1524 } else { 1525 tcg_gen_ld16u_i32(TCGV_LOW(ret), arg2, offset); 1526 tcg_gen_movi_i32(TCGV_HIGH(ret), 0); 1527 } 1528 } 1529 1530 void tcg_gen_ld16s_i64(TCGv_i64 ret, TCGv_ptr arg2, tcg_target_long offset) 1531 { 1532 if (TCG_TARGET_REG_BITS == 64) { 1533 tcg_gen_ldst_op_i64(INDEX_op_ld16s_i64, ret, arg2, offset); 1534 } else { 1535 tcg_gen_ld16s_i32(TCGV_LOW(ret), arg2, offset); 1536 tcg_gen_sari_i32(TCGV_HIGH(ret), TCGV_LOW(ret), 31); 1537 } 1538 } 1539 1540 void tcg_gen_ld32u_i64(TCGv_i64 ret, TCGv_ptr arg2, tcg_target_long offset) 1541 { 1542 if (TCG_TARGET_REG_BITS == 64) { 1543 tcg_gen_ldst_op_i64(INDEX_op_ld32u_i64, ret, arg2, offset); 1544 } else { 1545 tcg_gen_ld_i32(TCGV_LOW(ret), arg2, offset); 1546 tcg_gen_movi_i32(TCGV_HIGH(ret), 0); 1547 } 1548 } 1549 1550 void tcg_gen_ld32s_i64(TCGv_i64 ret, TCGv_ptr arg2, tcg_target_long offset) 1551 { 1552 if (TCG_TARGET_REG_BITS == 64) { 1553 tcg_gen_ldst_op_i64(INDEX_op_ld32s_i64, ret, arg2, offset); 1554 } else { 1555 tcg_gen_ld_i32(TCGV_LOW(ret), arg2, offset); 1556 tcg_gen_sari_i32(TCGV_HIGH(ret), TCGV_LOW(ret), 31); 1557 } 1558 } 1559 1560 void tcg_gen_ld_i64(TCGv_i64 ret, TCGv_ptr arg2, tcg_target_long offset) 1561 { 1562 /* 1563 * For 32-bit host, since arg2 and ret have different types, 1564 * they cannot be the same temporary -- no chance of overlap. 1565 */ 1566 if (TCG_TARGET_REG_BITS == 64) { 1567 tcg_gen_ldst_op_i64(INDEX_op_ld_i64, ret, arg2, offset); 1568 } else if (HOST_BIG_ENDIAN) { 1569 tcg_gen_ld_i32(TCGV_HIGH(ret), arg2, offset); 1570 tcg_gen_ld_i32(TCGV_LOW(ret), arg2, offset + 4); 1571 } else { 1572 tcg_gen_ld_i32(TCGV_LOW(ret), arg2, offset); 1573 tcg_gen_ld_i32(TCGV_HIGH(ret), arg2, offset + 4); 1574 } 1575 } 1576 1577 void tcg_gen_st8_i64(TCGv_i64 arg1, TCGv_ptr arg2, tcg_target_long offset) 1578 { 1579 if (TCG_TARGET_REG_BITS == 64) { 1580 tcg_gen_ldst_op_i64(INDEX_op_st8_i64, arg1, arg2, offset); 1581 } else { 1582 tcg_gen_st8_i32(TCGV_LOW(arg1), arg2, offset); 1583 } 1584 } 1585 1586 void tcg_gen_st16_i64(TCGv_i64 arg1, TCGv_ptr arg2, tcg_target_long offset) 1587 { 1588 if (TCG_TARGET_REG_BITS == 64) { 1589 tcg_gen_ldst_op_i64(INDEX_op_st16_i64, arg1, arg2, offset); 1590 } else { 1591 tcg_gen_st16_i32(TCGV_LOW(arg1), arg2, offset); 1592 } 1593 } 1594 1595 void tcg_gen_st32_i64(TCGv_i64 arg1, TCGv_ptr arg2, tcg_target_long offset) 1596 { 1597 if (TCG_TARGET_REG_BITS == 64) { 1598 tcg_gen_ldst_op_i64(INDEX_op_st32_i64, arg1, arg2, offset); 1599 } else { 1600 tcg_gen_st_i32(TCGV_LOW(arg1), arg2, offset); 1601 } 1602 } 1603 1604 void tcg_gen_st_i64(TCGv_i64 arg1, TCGv_ptr arg2, tcg_target_long offset) 1605 { 1606 if (TCG_TARGET_REG_BITS == 64) { 1607 tcg_gen_ldst_op_i64(INDEX_op_st_i64, arg1, arg2, offset); 1608 } else if (HOST_BIG_ENDIAN) { 1609 tcg_gen_st_i32(TCGV_HIGH(arg1), arg2, offset); 1610 tcg_gen_st_i32(TCGV_LOW(arg1), arg2, offset + 4); 1611 } else { 1612 tcg_gen_st_i32(TCGV_LOW(arg1), arg2, offset); 1613 tcg_gen_st_i32(TCGV_HIGH(arg1), arg2, offset + 4); 1614 } 1615 } 1616 1617 void tcg_gen_add_i64(TCGv_i64 ret, TCGv_i64 arg1, TCGv_i64 arg2) 1618 { 1619 if (TCG_TARGET_REG_BITS == 64) { 1620 tcg_gen_op3_i64(INDEX_op_add_i64, ret, arg1, arg2); 1621 } else { 1622 tcg_gen_add2_i32(TCGV_LOW(ret), TCGV_HIGH(ret), TCGV_LOW(arg1), 1623 TCGV_HIGH(arg1), TCGV_LOW(arg2), TCGV_HIGH(arg2)); 1624 } 1625 } 1626 1627 void tcg_gen_sub_i64(TCGv_i64 ret, TCGv_i64 arg1, TCGv_i64 arg2) 1628 { 1629 if (TCG_TARGET_REG_BITS == 64) { 1630 tcg_gen_op3_i64(INDEX_op_sub_i64, ret, arg1, arg2); 1631 } else { 1632 tcg_gen_sub2_i32(TCGV_LOW(ret), TCGV_HIGH(ret), TCGV_LOW(arg1), 1633 TCGV_HIGH(arg1), TCGV_LOW(arg2), TCGV_HIGH(arg2)); 1634 } 1635 } 1636 1637 void tcg_gen_and_i64(TCGv_i64 ret, TCGv_i64 arg1, TCGv_i64 arg2) 1638 { 1639 if (TCG_TARGET_REG_BITS == 64) { 1640 tcg_gen_op3_i64(INDEX_op_and_i64, ret, arg1, arg2); 1641 } else { 1642 tcg_gen_and_i32(TCGV_LOW(ret), TCGV_LOW(arg1), TCGV_LOW(arg2)); 1643 tcg_gen_and_i32(TCGV_HIGH(ret), TCGV_HIGH(arg1), TCGV_HIGH(arg2)); 1644 } 1645 } 1646 1647 void tcg_gen_or_i64(TCGv_i64 ret, TCGv_i64 arg1, TCGv_i64 arg2) 1648 { 1649 if (TCG_TARGET_REG_BITS == 64) { 1650 tcg_gen_op3_i64(INDEX_op_or_i64, ret, arg1, arg2); 1651 } else { 1652 tcg_gen_or_i32(TCGV_LOW(ret), TCGV_LOW(arg1), TCGV_LOW(arg2)); 1653 tcg_gen_or_i32(TCGV_HIGH(ret), TCGV_HIGH(arg1), TCGV_HIGH(arg2)); 1654 } 1655 } 1656 1657 void tcg_gen_xor_i64(TCGv_i64 ret, TCGv_i64 arg1, TCGv_i64 arg2) 1658 { 1659 if (TCG_TARGET_REG_BITS == 64) { 1660 tcg_gen_op3_i64(INDEX_op_xor_i64, ret, arg1, arg2); 1661 } else { 1662 tcg_gen_xor_i32(TCGV_LOW(ret), TCGV_LOW(arg1), TCGV_LOW(arg2)); 1663 tcg_gen_xor_i32(TCGV_HIGH(ret), TCGV_HIGH(arg1), TCGV_HIGH(arg2)); 1664 } 1665 } 1666 1667 void tcg_gen_shl_i64(TCGv_i64 ret, TCGv_i64 arg1, TCGv_i64 arg2) 1668 { 1669 if (TCG_TARGET_REG_BITS == 64) { 1670 tcg_gen_op3_i64(INDEX_op_shl_i64, ret, arg1, arg2); 1671 } else { 1672 gen_helper_shl_i64(ret, arg1, arg2); 1673 } 1674 } 1675 1676 void tcg_gen_shr_i64(TCGv_i64 ret, TCGv_i64 arg1, TCGv_i64 arg2) 1677 { 1678 if (TCG_TARGET_REG_BITS == 64) { 1679 tcg_gen_op3_i64(INDEX_op_shr_i64, ret, arg1, arg2); 1680 } else { 1681 gen_helper_shr_i64(ret, arg1, arg2); 1682 } 1683 } 1684 1685 void tcg_gen_sar_i64(TCGv_i64 ret, TCGv_i64 arg1, TCGv_i64 arg2) 1686 { 1687 if (TCG_TARGET_REG_BITS == 64) { 1688 tcg_gen_op3_i64(INDEX_op_sar_i64, ret, arg1, arg2); 1689 } else { 1690 gen_helper_sar_i64(ret, arg1, arg2); 1691 } 1692 } 1693 1694 void tcg_gen_mul_i64(TCGv_i64 ret, TCGv_i64 arg1, TCGv_i64 arg2) 1695 { 1696 TCGv_i64 t0; 1697 TCGv_i32 t1; 1698 1699 if (TCG_TARGET_REG_BITS == 64) { 1700 tcg_gen_op3_i64(INDEX_op_mul_i64, ret, arg1, arg2); 1701 return; 1702 } 1703 1704 1705 t0 = tcg_temp_ebb_new_i64(); 1706 t1 = tcg_temp_ebb_new_i32(); 1707 1708 tcg_gen_mulu2_i32(TCGV_LOW(t0), TCGV_HIGH(t0), 1709 TCGV_LOW(arg1), TCGV_LOW(arg2)); 1710 1711 tcg_gen_mul_i32(t1, TCGV_LOW(arg1), TCGV_HIGH(arg2)); 1712 tcg_gen_add_i32(TCGV_HIGH(t0), TCGV_HIGH(t0), t1); 1713 tcg_gen_mul_i32(t1, TCGV_HIGH(arg1), TCGV_LOW(arg2)); 1714 tcg_gen_add_i32(TCGV_HIGH(t0), TCGV_HIGH(t0), t1); 1715 1716 tcg_gen_mov_i64(ret, t0); 1717 tcg_temp_free_i64(t0); 1718 tcg_temp_free_i32(t1); 1719 } 1720 1721 void tcg_gen_addi_i64(TCGv_i64 ret, TCGv_i64 arg1, int64_t arg2) 1722 { 1723 /* some cases can be optimized here */ 1724 if (arg2 == 0) { 1725 tcg_gen_mov_i64(ret, arg1); 1726 } else if (TCG_TARGET_REG_BITS == 64) { 1727 tcg_gen_add_i64(ret, arg1, tcg_constant_i64(arg2)); 1728 } else { 1729 tcg_gen_add2_i32(TCGV_LOW(ret), TCGV_HIGH(ret), 1730 TCGV_LOW(arg1), TCGV_HIGH(arg1), 1731 tcg_constant_i32(arg2), tcg_constant_i32(arg2 >> 32)); 1732 } 1733 } 1734 1735 void tcg_gen_subfi_i64(TCGv_i64 ret, int64_t arg1, TCGv_i64 arg2) 1736 { 1737 if (arg1 == 0) { 1738 tcg_gen_neg_i64(ret, arg2); 1739 } else if (TCG_TARGET_REG_BITS == 64) { 1740 tcg_gen_sub_i64(ret, tcg_constant_i64(arg1), arg2); 1741 } else { 1742 tcg_gen_sub2_i32(TCGV_LOW(ret), TCGV_HIGH(ret), 1743 tcg_constant_i32(arg1), tcg_constant_i32(arg1 >> 32), 1744 TCGV_LOW(arg2), TCGV_HIGH(arg2)); 1745 } 1746 } 1747 1748 void tcg_gen_subi_i64(TCGv_i64 ret, TCGv_i64 arg1, int64_t arg2) 1749 { 1750 tcg_gen_addi_i64(ret, arg1, -arg2); 1751 } 1752 1753 void tcg_gen_neg_i64(TCGv_i64 ret, TCGv_i64 arg) 1754 { 1755 if (TCG_TARGET_REG_BITS == 64) { 1756 tcg_gen_op2_i64(INDEX_op_neg_i64, ret, arg); 1757 } else { 1758 TCGv_i32 zero = tcg_constant_i32(0); 1759 tcg_gen_sub2_i32(TCGV_LOW(ret), TCGV_HIGH(ret), 1760 zero, zero, TCGV_LOW(arg), TCGV_HIGH(arg)); 1761 } 1762 } 1763 1764 void tcg_gen_andi_i64(TCGv_i64 ret, TCGv_i64 arg1, int64_t arg2) 1765 { 1766 if (TCG_TARGET_REG_BITS == 32) { 1767 tcg_gen_andi_i32(TCGV_LOW(ret), TCGV_LOW(arg1), arg2); 1768 tcg_gen_andi_i32(TCGV_HIGH(ret), TCGV_HIGH(arg1), arg2 >> 32); 1769 return; 1770 } 1771 1772 /* Some cases can be optimized here. */ 1773 switch (arg2) { 1774 case 0: 1775 tcg_gen_movi_i64(ret, 0); 1776 return; 1777 case -1: 1778 tcg_gen_mov_i64(ret, arg1); 1779 return; 1780 case 0xff: 1781 /* Don't recurse with tcg_gen_ext8u_i64. */ 1782 if (TCG_TARGET_HAS_ext8u_i64) { 1783 tcg_gen_op2_i64(INDEX_op_ext8u_i64, ret, arg1); 1784 return; 1785 } 1786 break; 1787 case 0xffff: 1788 if (TCG_TARGET_HAS_ext16u_i64) { 1789 tcg_gen_op2_i64(INDEX_op_ext16u_i64, ret, arg1); 1790 return; 1791 } 1792 break; 1793 case 0xffffffffu: 1794 if (TCG_TARGET_HAS_ext32u_i64) { 1795 tcg_gen_op2_i64(INDEX_op_ext32u_i64, ret, arg1); 1796 return; 1797 } 1798 break; 1799 } 1800 1801 tcg_gen_and_i64(ret, arg1, tcg_constant_i64(arg2)); 1802 } 1803 1804 void tcg_gen_ori_i64(TCGv_i64 ret, TCGv_i64 arg1, int64_t arg2) 1805 { 1806 if (TCG_TARGET_REG_BITS == 32) { 1807 tcg_gen_ori_i32(TCGV_LOW(ret), TCGV_LOW(arg1), arg2); 1808 tcg_gen_ori_i32(TCGV_HIGH(ret), TCGV_HIGH(arg1), arg2 >> 32); 1809 return; 1810 } 1811 /* Some cases can be optimized here. */ 1812 if (arg2 == -1) { 1813 tcg_gen_movi_i64(ret, -1); 1814 } else if (arg2 == 0) { 1815 tcg_gen_mov_i64(ret, arg1); 1816 } else { 1817 tcg_gen_or_i64(ret, arg1, tcg_constant_i64(arg2)); 1818 } 1819 } 1820 1821 void tcg_gen_xori_i64(TCGv_i64 ret, TCGv_i64 arg1, int64_t arg2) 1822 { 1823 if (TCG_TARGET_REG_BITS == 32) { 1824 tcg_gen_xori_i32(TCGV_LOW(ret), TCGV_LOW(arg1), arg2); 1825 tcg_gen_xori_i32(TCGV_HIGH(ret), TCGV_HIGH(arg1), arg2 >> 32); 1826 return; 1827 } 1828 /* Some cases can be optimized here. */ 1829 if (arg2 == 0) { 1830 tcg_gen_mov_i64(ret, arg1); 1831 } else if (arg2 == -1 && TCG_TARGET_HAS_not_i64) { 1832 /* Don't recurse with tcg_gen_not_i64. */ 1833 tcg_gen_op2_i64(INDEX_op_not_i64, ret, arg1); 1834 } else { 1835 tcg_gen_xor_i64(ret, arg1, tcg_constant_i64(arg2)); 1836 } 1837 } 1838 1839 static inline void tcg_gen_shifti_i64(TCGv_i64 ret, TCGv_i64 arg1, 1840 unsigned c, bool right, bool arith) 1841 { 1842 tcg_debug_assert(c < 64); 1843 if (c == 0) { 1844 tcg_gen_mov_i32(TCGV_LOW(ret), TCGV_LOW(arg1)); 1845 tcg_gen_mov_i32(TCGV_HIGH(ret), TCGV_HIGH(arg1)); 1846 } else if (c >= 32) { 1847 c -= 32; 1848 if (right) { 1849 if (arith) { 1850 tcg_gen_sari_i32(TCGV_LOW(ret), TCGV_HIGH(arg1), c); 1851 tcg_gen_sari_i32(TCGV_HIGH(ret), TCGV_HIGH(arg1), 31); 1852 } else { 1853 tcg_gen_shri_i32(TCGV_LOW(ret), TCGV_HIGH(arg1), c); 1854 tcg_gen_movi_i32(TCGV_HIGH(ret), 0); 1855 } 1856 } else { 1857 tcg_gen_shli_i32(TCGV_HIGH(ret), TCGV_LOW(arg1), c); 1858 tcg_gen_movi_i32(TCGV_LOW(ret), 0); 1859 } 1860 } else if (right) { 1861 if (TCG_TARGET_HAS_extract2_i32) { 1862 tcg_gen_extract2_i32(TCGV_LOW(ret), 1863 TCGV_LOW(arg1), TCGV_HIGH(arg1), c); 1864 } else { 1865 tcg_gen_shri_i32(TCGV_LOW(ret), TCGV_LOW(arg1), c); 1866 tcg_gen_deposit_i32(TCGV_LOW(ret), TCGV_LOW(ret), 1867 TCGV_HIGH(arg1), 32 - c, c); 1868 } 1869 if (arith) { 1870 tcg_gen_sari_i32(TCGV_HIGH(ret), TCGV_HIGH(arg1), c); 1871 } else { 1872 tcg_gen_shri_i32(TCGV_HIGH(ret), TCGV_HIGH(arg1), c); 1873 } 1874 } else { 1875 if (TCG_TARGET_HAS_extract2_i32) { 1876 tcg_gen_extract2_i32(TCGV_HIGH(ret), 1877 TCGV_LOW(arg1), TCGV_HIGH(arg1), 32 - c); 1878 } else { 1879 TCGv_i32 t0 = tcg_temp_ebb_new_i32(); 1880 tcg_gen_shri_i32(t0, TCGV_LOW(arg1), 32 - c); 1881 tcg_gen_deposit_i32(TCGV_HIGH(ret), t0, 1882 TCGV_HIGH(arg1), c, 32 - c); 1883 tcg_temp_free_i32(t0); 1884 } 1885 tcg_gen_shli_i32(TCGV_LOW(ret), TCGV_LOW(arg1), c); 1886 } 1887 } 1888 1889 void tcg_gen_shli_i64(TCGv_i64 ret, TCGv_i64 arg1, int64_t arg2) 1890 { 1891 tcg_debug_assert(arg2 >= 0 && arg2 < 64); 1892 if (TCG_TARGET_REG_BITS == 32) { 1893 tcg_gen_shifti_i64(ret, arg1, arg2, 0, 0); 1894 } else if (arg2 == 0) { 1895 tcg_gen_mov_i64(ret, arg1); 1896 } else { 1897 tcg_gen_shl_i64(ret, arg1, tcg_constant_i64(arg2)); 1898 } 1899 } 1900 1901 void tcg_gen_shri_i64(TCGv_i64 ret, TCGv_i64 arg1, int64_t arg2) 1902 { 1903 tcg_debug_assert(arg2 >= 0 && arg2 < 64); 1904 if (TCG_TARGET_REG_BITS == 32) { 1905 tcg_gen_shifti_i64(ret, arg1, arg2, 1, 0); 1906 } else if (arg2 == 0) { 1907 tcg_gen_mov_i64(ret, arg1); 1908 } else { 1909 tcg_gen_shr_i64(ret, arg1, tcg_constant_i64(arg2)); 1910 } 1911 } 1912 1913 void tcg_gen_sari_i64(TCGv_i64 ret, TCGv_i64 arg1, int64_t arg2) 1914 { 1915 tcg_debug_assert(arg2 >= 0 && arg2 < 64); 1916 if (TCG_TARGET_REG_BITS == 32) { 1917 tcg_gen_shifti_i64(ret, arg1, arg2, 1, 1); 1918 } else if (arg2 == 0) { 1919 tcg_gen_mov_i64(ret, arg1); 1920 } else { 1921 tcg_gen_sar_i64(ret, arg1, tcg_constant_i64(arg2)); 1922 } 1923 } 1924 1925 void tcg_gen_brcond_i64(TCGCond cond, TCGv_i64 arg1, TCGv_i64 arg2, TCGLabel *l) 1926 { 1927 if (cond == TCG_COND_ALWAYS) { 1928 tcg_gen_br(l); 1929 } else if (cond != TCG_COND_NEVER) { 1930 if (TCG_TARGET_REG_BITS == 32) { 1931 tcg_gen_op6ii_i32(INDEX_op_brcond2_i32, TCGV_LOW(arg1), 1932 TCGV_HIGH(arg1), TCGV_LOW(arg2), 1933 TCGV_HIGH(arg2), cond, label_arg(l)); 1934 } else { 1935 tcg_gen_op4ii_i64(INDEX_op_brcond_i64, arg1, arg2, cond, 1936 label_arg(l)); 1937 } 1938 add_last_as_label_use(l); 1939 } 1940 } 1941 1942 void tcg_gen_brcondi_i64(TCGCond cond, TCGv_i64 arg1, int64_t arg2, TCGLabel *l) 1943 { 1944 if (TCG_TARGET_REG_BITS == 64) { 1945 tcg_gen_brcond_i64(cond, arg1, tcg_constant_i64(arg2), l); 1946 } else if (cond == TCG_COND_ALWAYS) { 1947 tcg_gen_br(l); 1948 } else if (cond != TCG_COND_NEVER) { 1949 tcg_gen_op6ii_i32(INDEX_op_brcond2_i32, 1950 TCGV_LOW(arg1), TCGV_HIGH(arg1), 1951 tcg_constant_i32(arg2), 1952 tcg_constant_i32(arg2 >> 32), 1953 cond, label_arg(l)); 1954 add_last_as_label_use(l); 1955 } 1956 } 1957 1958 void tcg_gen_setcond_i64(TCGCond cond, TCGv_i64 ret, 1959 TCGv_i64 arg1, TCGv_i64 arg2) 1960 { 1961 if (cond == TCG_COND_ALWAYS) { 1962 tcg_gen_movi_i64(ret, 1); 1963 } else if (cond == TCG_COND_NEVER) { 1964 tcg_gen_movi_i64(ret, 0); 1965 } else { 1966 if (TCG_TARGET_REG_BITS == 32) { 1967 tcg_gen_op6i_i32(INDEX_op_setcond2_i32, TCGV_LOW(ret), 1968 TCGV_LOW(arg1), TCGV_HIGH(arg1), 1969 TCGV_LOW(arg2), TCGV_HIGH(arg2), cond); 1970 tcg_gen_movi_i32(TCGV_HIGH(ret), 0); 1971 } else { 1972 tcg_gen_op4i_i64(INDEX_op_setcond_i64, ret, arg1, arg2, cond); 1973 } 1974 } 1975 } 1976 1977 void tcg_gen_setcondi_i64(TCGCond cond, TCGv_i64 ret, 1978 TCGv_i64 arg1, int64_t arg2) 1979 { 1980 if (TCG_TARGET_REG_BITS == 64) { 1981 tcg_gen_setcond_i64(cond, ret, arg1, tcg_constant_i64(arg2)); 1982 } else if (cond == TCG_COND_ALWAYS) { 1983 tcg_gen_movi_i64(ret, 1); 1984 } else if (cond == TCG_COND_NEVER) { 1985 tcg_gen_movi_i64(ret, 0); 1986 } else { 1987 tcg_gen_op6i_i32(INDEX_op_setcond2_i32, TCGV_LOW(ret), 1988 TCGV_LOW(arg1), TCGV_HIGH(arg1), 1989 tcg_constant_i32(arg2), 1990 tcg_constant_i32(arg2 >> 32), cond); 1991 tcg_gen_movi_i32(TCGV_HIGH(ret), 0); 1992 } 1993 } 1994 1995 void tcg_gen_negsetcondi_i64(TCGCond cond, TCGv_i64 ret, 1996 TCGv_i64 arg1, int64_t arg2) 1997 { 1998 tcg_gen_negsetcond_i64(cond, ret, arg1, tcg_constant_i64(arg2)); 1999 } 2000 2001 void tcg_gen_negsetcond_i64(TCGCond cond, TCGv_i64 ret, 2002 TCGv_i64 arg1, TCGv_i64 arg2) 2003 { 2004 if (cond == TCG_COND_ALWAYS) { 2005 tcg_gen_movi_i64(ret, -1); 2006 } else if (cond == TCG_COND_NEVER) { 2007 tcg_gen_movi_i64(ret, 0); 2008 } else if (TCG_TARGET_HAS_negsetcond_i64) { 2009 tcg_gen_op4i_i64(INDEX_op_negsetcond_i64, ret, arg1, arg2, cond); 2010 } else if (TCG_TARGET_REG_BITS == 32) { 2011 tcg_gen_op6i_i32(INDEX_op_setcond2_i32, TCGV_LOW(ret), 2012 TCGV_LOW(arg1), TCGV_HIGH(arg1), 2013 TCGV_LOW(arg2), TCGV_HIGH(arg2), cond); 2014 tcg_gen_neg_i32(TCGV_LOW(ret), TCGV_LOW(ret)); 2015 tcg_gen_mov_i32(TCGV_HIGH(ret), TCGV_LOW(ret)); 2016 } else { 2017 tcg_gen_setcond_i64(cond, ret, arg1, arg2); 2018 tcg_gen_neg_i64(ret, ret); 2019 } 2020 } 2021 2022 void tcg_gen_muli_i64(TCGv_i64 ret, TCGv_i64 arg1, int64_t arg2) 2023 { 2024 if (arg2 == 0) { 2025 tcg_gen_movi_i64(ret, 0); 2026 } else if (is_power_of_2(arg2)) { 2027 tcg_gen_shli_i64(ret, arg1, ctz64(arg2)); 2028 } else { 2029 tcg_gen_mul_i64(ret, arg1, tcg_constant_i64(arg2)); 2030 } 2031 } 2032 2033 void tcg_gen_div_i64(TCGv_i64 ret, TCGv_i64 arg1, TCGv_i64 arg2) 2034 { 2035 if (TCG_TARGET_HAS_div_i64) { 2036 tcg_gen_op3_i64(INDEX_op_div_i64, ret, arg1, arg2); 2037 } else if (TCG_TARGET_HAS_div2_i64) { 2038 TCGv_i64 t0 = tcg_temp_ebb_new_i64(); 2039 tcg_gen_sari_i64(t0, arg1, 63); 2040 tcg_gen_op5_i64(INDEX_op_div2_i64, ret, t0, arg1, t0, arg2); 2041 tcg_temp_free_i64(t0); 2042 } else { 2043 gen_helper_div_i64(ret, arg1, arg2); 2044 } 2045 } 2046 2047 void tcg_gen_rem_i64(TCGv_i64 ret, TCGv_i64 arg1, TCGv_i64 arg2) 2048 { 2049 if (TCG_TARGET_HAS_rem_i64) { 2050 tcg_gen_op3_i64(INDEX_op_rem_i64, ret, arg1, arg2); 2051 } else if (TCG_TARGET_HAS_div_i64) { 2052 TCGv_i64 t0 = tcg_temp_ebb_new_i64(); 2053 tcg_gen_op3_i64(INDEX_op_div_i64, t0, arg1, arg2); 2054 tcg_gen_mul_i64(t0, t0, arg2); 2055 tcg_gen_sub_i64(ret, arg1, t0); 2056 tcg_temp_free_i64(t0); 2057 } else if (TCG_TARGET_HAS_div2_i64) { 2058 TCGv_i64 t0 = tcg_temp_ebb_new_i64(); 2059 tcg_gen_sari_i64(t0, arg1, 63); 2060 tcg_gen_op5_i64(INDEX_op_div2_i64, t0, ret, arg1, t0, arg2); 2061 tcg_temp_free_i64(t0); 2062 } else { 2063 gen_helper_rem_i64(ret, arg1, arg2); 2064 } 2065 } 2066 2067 void tcg_gen_divu_i64(TCGv_i64 ret, TCGv_i64 arg1, TCGv_i64 arg2) 2068 { 2069 if (TCG_TARGET_HAS_div_i64) { 2070 tcg_gen_op3_i64(INDEX_op_divu_i64, ret, arg1, arg2); 2071 } else if (TCG_TARGET_HAS_div2_i64) { 2072 TCGv_i64 t0 = tcg_temp_ebb_new_i64(); 2073 TCGv_i64 zero = tcg_constant_i64(0); 2074 tcg_gen_op5_i64(INDEX_op_divu2_i64, ret, t0, arg1, zero, arg2); 2075 tcg_temp_free_i64(t0); 2076 } else { 2077 gen_helper_divu_i64(ret, arg1, arg2); 2078 } 2079 } 2080 2081 void tcg_gen_remu_i64(TCGv_i64 ret, TCGv_i64 arg1, TCGv_i64 arg2) 2082 { 2083 if (TCG_TARGET_HAS_rem_i64) { 2084 tcg_gen_op3_i64(INDEX_op_remu_i64, ret, arg1, arg2); 2085 } else if (TCG_TARGET_HAS_div_i64) { 2086 TCGv_i64 t0 = tcg_temp_ebb_new_i64(); 2087 tcg_gen_op3_i64(INDEX_op_divu_i64, t0, arg1, arg2); 2088 tcg_gen_mul_i64(t0, t0, arg2); 2089 tcg_gen_sub_i64(ret, arg1, t0); 2090 tcg_temp_free_i64(t0); 2091 } else if (TCG_TARGET_HAS_div2_i64) { 2092 TCGv_i64 t0 = tcg_temp_ebb_new_i64(); 2093 TCGv_i64 zero = tcg_constant_i64(0); 2094 tcg_gen_op5_i64(INDEX_op_divu2_i64, t0, ret, arg1, zero, arg2); 2095 tcg_temp_free_i64(t0); 2096 } else { 2097 gen_helper_remu_i64(ret, arg1, arg2); 2098 } 2099 } 2100 2101 void tcg_gen_ext8s_i64(TCGv_i64 ret, TCGv_i64 arg) 2102 { 2103 if (TCG_TARGET_REG_BITS == 32) { 2104 tcg_gen_ext8s_i32(TCGV_LOW(ret), TCGV_LOW(arg)); 2105 tcg_gen_sari_i32(TCGV_HIGH(ret), TCGV_LOW(ret), 31); 2106 } else if (TCG_TARGET_HAS_ext8s_i64) { 2107 tcg_gen_op2_i64(INDEX_op_ext8s_i64, ret, arg); 2108 } else { 2109 tcg_gen_shli_i64(ret, arg, 56); 2110 tcg_gen_sari_i64(ret, ret, 56); 2111 } 2112 } 2113 2114 void tcg_gen_ext16s_i64(TCGv_i64 ret, TCGv_i64 arg) 2115 { 2116 if (TCG_TARGET_REG_BITS == 32) { 2117 tcg_gen_ext16s_i32(TCGV_LOW(ret), TCGV_LOW(arg)); 2118 tcg_gen_sari_i32(TCGV_HIGH(ret), TCGV_LOW(ret), 31); 2119 } else if (TCG_TARGET_HAS_ext16s_i64) { 2120 tcg_gen_op2_i64(INDEX_op_ext16s_i64, ret, arg); 2121 } else { 2122 tcg_gen_shli_i64(ret, arg, 48); 2123 tcg_gen_sari_i64(ret, ret, 48); 2124 } 2125 } 2126 2127 void tcg_gen_ext32s_i64(TCGv_i64 ret, TCGv_i64 arg) 2128 { 2129 if (TCG_TARGET_REG_BITS == 32) { 2130 tcg_gen_mov_i32(TCGV_LOW(ret), TCGV_LOW(arg)); 2131 tcg_gen_sari_i32(TCGV_HIGH(ret), TCGV_LOW(ret), 31); 2132 } else if (TCG_TARGET_HAS_ext32s_i64) { 2133 tcg_gen_op2_i64(INDEX_op_ext32s_i64, ret, arg); 2134 } else { 2135 tcg_gen_shli_i64(ret, arg, 32); 2136 tcg_gen_sari_i64(ret, ret, 32); 2137 } 2138 } 2139 2140 void tcg_gen_ext8u_i64(TCGv_i64 ret, TCGv_i64 arg) 2141 { 2142 if (TCG_TARGET_REG_BITS == 32) { 2143 tcg_gen_ext8u_i32(TCGV_LOW(ret), TCGV_LOW(arg)); 2144 tcg_gen_movi_i32(TCGV_HIGH(ret), 0); 2145 } else if (TCG_TARGET_HAS_ext8u_i64) { 2146 tcg_gen_op2_i64(INDEX_op_ext8u_i64, ret, arg); 2147 } else { 2148 tcg_gen_andi_i64(ret, arg, 0xffu); 2149 } 2150 } 2151 2152 void tcg_gen_ext16u_i64(TCGv_i64 ret, TCGv_i64 arg) 2153 { 2154 if (TCG_TARGET_REG_BITS == 32) { 2155 tcg_gen_ext16u_i32(TCGV_LOW(ret), TCGV_LOW(arg)); 2156 tcg_gen_movi_i32(TCGV_HIGH(ret), 0); 2157 } else if (TCG_TARGET_HAS_ext16u_i64) { 2158 tcg_gen_op2_i64(INDEX_op_ext16u_i64, ret, arg); 2159 } else { 2160 tcg_gen_andi_i64(ret, arg, 0xffffu); 2161 } 2162 } 2163 2164 void tcg_gen_ext32u_i64(TCGv_i64 ret, TCGv_i64 arg) 2165 { 2166 if (TCG_TARGET_REG_BITS == 32) { 2167 tcg_gen_mov_i32(TCGV_LOW(ret), TCGV_LOW(arg)); 2168 tcg_gen_movi_i32(TCGV_HIGH(ret), 0); 2169 } else if (TCG_TARGET_HAS_ext32u_i64) { 2170 tcg_gen_op2_i64(INDEX_op_ext32u_i64, ret, arg); 2171 } else { 2172 tcg_gen_andi_i64(ret, arg, 0xffffffffu); 2173 } 2174 } 2175 2176 /* 2177 * bswap16_i64: 16-bit byte swap on the low bits of a 64-bit value. 2178 * 2179 * Byte pattern: xxxxxxxxab -> yyyyyyyyba 2180 * 2181 * With TCG_BSWAP_IZ, x == zero, else undefined. 2182 * With TCG_BSWAP_OZ, y == zero, with TCG_BSWAP_OS y == sign, else undefined. 2183 */ 2184 void tcg_gen_bswap16_i64(TCGv_i64 ret, TCGv_i64 arg, int flags) 2185 { 2186 /* Only one extension flag may be present. */ 2187 tcg_debug_assert(!(flags & TCG_BSWAP_OS) || !(flags & TCG_BSWAP_OZ)); 2188 2189 if (TCG_TARGET_REG_BITS == 32) { 2190 tcg_gen_bswap16_i32(TCGV_LOW(ret), TCGV_LOW(arg), flags); 2191 if (flags & TCG_BSWAP_OS) { 2192 tcg_gen_sari_i32(TCGV_HIGH(ret), TCGV_LOW(ret), 31); 2193 } else { 2194 tcg_gen_movi_i32(TCGV_HIGH(ret), 0); 2195 } 2196 } else if (TCG_TARGET_HAS_bswap16_i64) { 2197 tcg_gen_op3i_i64(INDEX_op_bswap16_i64, ret, arg, flags); 2198 } else { 2199 TCGv_i64 t0 = tcg_temp_ebb_new_i64(); 2200 TCGv_i64 t1 = tcg_temp_ebb_new_i64(); 2201 2202 /* arg = ......ab or xxxxxxab */ 2203 tcg_gen_shri_i64(t0, arg, 8); /* t0 = .......a or .xxxxxxa */ 2204 if (!(flags & TCG_BSWAP_IZ)) { 2205 tcg_gen_ext8u_i64(t0, t0); /* t0 = .......a */ 2206 } 2207 2208 if (flags & TCG_BSWAP_OS) { 2209 tcg_gen_shli_i64(t1, arg, 56); /* t1 = b....... */ 2210 tcg_gen_sari_i64(t1, t1, 48); /* t1 = ssssssb. */ 2211 } else if (flags & TCG_BSWAP_OZ) { 2212 tcg_gen_ext8u_i64(t1, arg); /* t1 = .......b */ 2213 tcg_gen_shli_i64(t1, t1, 8); /* t1 = ......b. */ 2214 } else { 2215 tcg_gen_shli_i64(t1, arg, 8); /* t1 = xxxxxab. */ 2216 } 2217 2218 tcg_gen_or_i64(ret, t0, t1); /* ret = ......ba (OZ) */ 2219 /* ssssssba (OS) */ 2220 /* xxxxxaba (no flag) */ 2221 tcg_temp_free_i64(t0); 2222 tcg_temp_free_i64(t1); 2223 } 2224 } 2225 2226 /* 2227 * bswap32_i64: 32-bit byte swap on the low bits of a 64-bit value. 2228 * 2229 * Byte pattern: xxxxabcd -> yyyydcba 2230 * 2231 * With TCG_BSWAP_IZ, x == zero, else undefined. 2232 * With TCG_BSWAP_OZ, y == zero, with TCG_BSWAP_OS y == sign, else undefined. 2233 */ 2234 void tcg_gen_bswap32_i64(TCGv_i64 ret, TCGv_i64 arg, int flags) 2235 { 2236 /* Only one extension flag may be present. */ 2237 tcg_debug_assert(!(flags & TCG_BSWAP_OS) || !(flags & TCG_BSWAP_OZ)); 2238 2239 if (TCG_TARGET_REG_BITS == 32) { 2240 tcg_gen_bswap32_i32(TCGV_LOW(ret), TCGV_LOW(arg)); 2241 if (flags & TCG_BSWAP_OS) { 2242 tcg_gen_sari_i32(TCGV_HIGH(ret), TCGV_LOW(ret), 31); 2243 } else { 2244 tcg_gen_movi_i32(TCGV_HIGH(ret), 0); 2245 } 2246 } else if (TCG_TARGET_HAS_bswap32_i64) { 2247 tcg_gen_op3i_i64(INDEX_op_bswap32_i64, ret, arg, flags); 2248 } else { 2249 TCGv_i64 t0 = tcg_temp_ebb_new_i64(); 2250 TCGv_i64 t1 = tcg_temp_ebb_new_i64(); 2251 TCGv_i64 t2 = tcg_constant_i64(0x00ff00ff); 2252 2253 /* arg = xxxxabcd */ 2254 tcg_gen_shri_i64(t0, arg, 8); /* t0 = .xxxxabc */ 2255 tcg_gen_and_i64(t1, arg, t2); /* t1 = .....b.d */ 2256 tcg_gen_and_i64(t0, t0, t2); /* t0 = .....a.c */ 2257 tcg_gen_shli_i64(t1, t1, 8); /* t1 = ....b.d. */ 2258 tcg_gen_or_i64(ret, t0, t1); /* ret = ....badc */ 2259 2260 tcg_gen_shli_i64(t1, ret, 48); /* t1 = dc...... */ 2261 tcg_gen_shri_i64(t0, ret, 16); /* t0 = ......ba */ 2262 if (flags & TCG_BSWAP_OS) { 2263 tcg_gen_sari_i64(t1, t1, 32); /* t1 = ssssdc.. */ 2264 } else { 2265 tcg_gen_shri_i64(t1, t1, 32); /* t1 = ....dc.. */ 2266 } 2267 tcg_gen_or_i64(ret, t0, t1); /* ret = ssssdcba (OS) */ 2268 /* ....dcba (else) */ 2269 2270 tcg_temp_free_i64(t0); 2271 tcg_temp_free_i64(t1); 2272 } 2273 } 2274 2275 /* 2276 * bswap64_i64: 64-bit byte swap on a 64-bit value. 2277 * 2278 * Byte pattern: abcdefgh -> hgfedcba 2279 */ 2280 void tcg_gen_bswap64_i64(TCGv_i64 ret, TCGv_i64 arg) 2281 { 2282 if (TCG_TARGET_REG_BITS == 32) { 2283 TCGv_i32 t0, t1; 2284 t0 = tcg_temp_ebb_new_i32(); 2285 t1 = tcg_temp_ebb_new_i32(); 2286 2287 tcg_gen_bswap32_i32(t0, TCGV_LOW(arg)); 2288 tcg_gen_bswap32_i32(t1, TCGV_HIGH(arg)); 2289 tcg_gen_mov_i32(TCGV_LOW(ret), t1); 2290 tcg_gen_mov_i32(TCGV_HIGH(ret), t0); 2291 tcg_temp_free_i32(t0); 2292 tcg_temp_free_i32(t1); 2293 } else if (TCG_TARGET_HAS_bswap64_i64) { 2294 tcg_gen_op3i_i64(INDEX_op_bswap64_i64, ret, arg, 0); 2295 } else { 2296 TCGv_i64 t0 = tcg_temp_ebb_new_i64(); 2297 TCGv_i64 t1 = tcg_temp_ebb_new_i64(); 2298 TCGv_i64 t2 = tcg_temp_ebb_new_i64(); 2299 2300 /* arg = abcdefgh */ 2301 tcg_gen_movi_i64(t2, 0x00ff00ff00ff00ffull); 2302 tcg_gen_shri_i64(t0, arg, 8); /* t0 = .abcdefg */ 2303 tcg_gen_and_i64(t1, arg, t2); /* t1 = .b.d.f.h */ 2304 tcg_gen_and_i64(t0, t0, t2); /* t0 = .a.c.e.g */ 2305 tcg_gen_shli_i64(t1, t1, 8); /* t1 = b.d.f.h. */ 2306 tcg_gen_or_i64(ret, t0, t1); /* ret = badcfehg */ 2307 2308 tcg_gen_movi_i64(t2, 0x0000ffff0000ffffull); 2309 tcg_gen_shri_i64(t0, ret, 16); /* t0 = ..badcfe */ 2310 tcg_gen_and_i64(t1, ret, t2); /* t1 = ..dc..hg */ 2311 tcg_gen_and_i64(t0, t0, t2); /* t0 = ..ba..fe */ 2312 tcg_gen_shli_i64(t1, t1, 16); /* t1 = dc..hg.. */ 2313 tcg_gen_or_i64(ret, t0, t1); /* ret = dcbahgfe */ 2314 2315 tcg_gen_shri_i64(t0, ret, 32); /* t0 = ....dcba */ 2316 tcg_gen_shli_i64(t1, ret, 32); /* t1 = hgfe.... */ 2317 tcg_gen_or_i64(ret, t0, t1); /* ret = hgfedcba */ 2318 2319 tcg_temp_free_i64(t0); 2320 tcg_temp_free_i64(t1); 2321 tcg_temp_free_i64(t2); 2322 } 2323 } 2324 2325 /* 2326 * hswap_i64: Swap 16-bit halfwords within a 64-bit value. 2327 * See also include/qemu/bitops.h, hswap64. 2328 * 2329 * Byte pattern: abcdefgh -> ghefcdab 2330 */ 2331 void tcg_gen_hswap_i64(TCGv_i64 ret, TCGv_i64 arg) 2332 { 2333 uint64_t m = 0x0000ffff0000ffffull; 2334 TCGv_i64 t0 = tcg_temp_ebb_new_i64(); 2335 TCGv_i64 t1 = tcg_temp_ebb_new_i64(); 2336 2337 /* arg = abcdefgh */ 2338 tcg_gen_rotli_i64(t1, arg, 32); /* t1 = efghabcd */ 2339 tcg_gen_andi_i64(t0, t1, m); /* t0 = ..gh..cd */ 2340 tcg_gen_shli_i64(t0, t0, 16); /* t0 = gh..cd.. */ 2341 tcg_gen_shri_i64(t1, t1, 16); /* t1 = ..efghab */ 2342 tcg_gen_andi_i64(t1, t1, m); /* t1 = ..ef..ab */ 2343 tcg_gen_or_i64(ret, t0, t1); /* ret = ghefcdab */ 2344 2345 tcg_temp_free_i64(t0); 2346 tcg_temp_free_i64(t1); 2347 } 2348 2349 /* 2350 * wswap_i64: Swap 32-bit words within a 64-bit value. 2351 * 2352 * Byte pattern: abcdefgh -> efghabcd 2353 */ 2354 void tcg_gen_wswap_i64(TCGv_i64 ret, TCGv_i64 arg) 2355 { 2356 /* Swapping 2 32-bit elements is a rotate. */ 2357 tcg_gen_rotli_i64(ret, arg, 32); 2358 } 2359 2360 void tcg_gen_not_i64(TCGv_i64 ret, TCGv_i64 arg) 2361 { 2362 if (TCG_TARGET_REG_BITS == 32) { 2363 tcg_gen_not_i32(TCGV_LOW(ret), TCGV_LOW(arg)); 2364 tcg_gen_not_i32(TCGV_HIGH(ret), TCGV_HIGH(arg)); 2365 } else if (TCG_TARGET_HAS_not_i64) { 2366 tcg_gen_op2_i64(INDEX_op_not_i64, ret, arg); 2367 } else { 2368 tcg_gen_xori_i64(ret, arg, -1); 2369 } 2370 } 2371 2372 void tcg_gen_andc_i64(TCGv_i64 ret, TCGv_i64 arg1, TCGv_i64 arg2) 2373 { 2374 if (TCG_TARGET_REG_BITS == 32) { 2375 tcg_gen_andc_i32(TCGV_LOW(ret), TCGV_LOW(arg1), TCGV_LOW(arg2)); 2376 tcg_gen_andc_i32(TCGV_HIGH(ret), TCGV_HIGH(arg1), TCGV_HIGH(arg2)); 2377 } else if (TCG_TARGET_HAS_andc_i64) { 2378 tcg_gen_op3_i64(INDEX_op_andc_i64, ret, arg1, arg2); 2379 } else { 2380 TCGv_i64 t0 = tcg_temp_ebb_new_i64(); 2381 tcg_gen_not_i64(t0, arg2); 2382 tcg_gen_and_i64(ret, arg1, t0); 2383 tcg_temp_free_i64(t0); 2384 } 2385 } 2386 2387 void tcg_gen_eqv_i64(TCGv_i64 ret, TCGv_i64 arg1, TCGv_i64 arg2) 2388 { 2389 if (TCG_TARGET_REG_BITS == 32) { 2390 tcg_gen_eqv_i32(TCGV_LOW(ret), TCGV_LOW(arg1), TCGV_LOW(arg2)); 2391 tcg_gen_eqv_i32(TCGV_HIGH(ret), TCGV_HIGH(arg1), TCGV_HIGH(arg2)); 2392 } else if (TCG_TARGET_HAS_eqv_i64) { 2393 tcg_gen_op3_i64(INDEX_op_eqv_i64, ret, arg1, arg2); 2394 } else { 2395 tcg_gen_xor_i64(ret, arg1, arg2); 2396 tcg_gen_not_i64(ret, ret); 2397 } 2398 } 2399 2400 void tcg_gen_nand_i64(TCGv_i64 ret, TCGv_i64 arg1, TCGv_i64 arg2) 2401 { 2402 if (TCG_TARGET_REG_BITS == 32) { 2403 tcg_gen_nand_i32(TCGV_LOW(ret), TCGV_LOW(arg1), TCGV_LOW(arg2)); 2404 tcg_gen_nand_i32(TCGV_HIGH(ret), TCGV_HIGH(arg1), TCGV_HIGH(arg2)); 2405 } else if (TCG_TARGET_HAS_nand_i64) { 2406 tcg_gen_op3_i64(INDEX_op_nand_i64, ret, arg1, arg2); 2407 } else { 2408 tcg_gen_and_i64(ret, arg1, arg2); 2409 tcg_gen_not_i64(ret, ret); 2410 } 2411 } 2412 2413 void tcg_gen_nor_i64(TCGv_i64 ret, TCGv_i64 arg1, TCGv_i64 arg2) 2414 { 2415 if (TCG_TARGET_REG_BITS == 32) { 2416 tcg_gen_nor_i32(TCGV_LOW(ret), TCGV_LOW(arg1), TCGV_LOW(arg2)); 2417 tcg_gen_nor_i32(TCGV_HIGH(ret), TCGV_HIGH(arg1), TCGV_HIGH(arg2)); 2418 } else if (TCG_TARGET_HAS_nor_i64) { 2419 tcg_gen_op3_i64(INDEX_op_nor_i64, ret, arg1, arg2); 2420 } else { 2421 tcg_gen_or_i64(ret, arg1, arg2); 2422 tcg_gen_not_i64(ret, ret); 2423 } 2424 } 2425 2426 void tcg_gen_orc_i64(TCGv_i64 ret, TCGv_i64 arg1, TCGv_i64 arg2) 2427 { 2428 if (TCG_TARGET_REG_BITS == 32) { 2429 tcg_gen_orc_i32(TCGV_LOW(ret), TCGV_LOW(arg1), TCGV_LOW(arg2)); 2430 tcg_gen_orc_i32(TCGV_HIGH(ret), TCGV_HIGH(arg1), TCGV_HIGH(arg2)); 2431 } else if (TCG_TARGET_HAS_orc_i64) { 2432 tcg_gen_op3_i64(INDEX_op_orc_i64, ret, arg1, arg2); 2433 } else { 2434 TCGv_i64 t0 = tcg_temp_ebb_new_i64(); 2435 tcg_gen_not_i64(t0, arg2); 2436 tcg_gen_or_i64(ret, arg1, t0); 2437 tcg_temp_free_i64(t0); 2438 } 2439 } 2440 2441 void tcg_gen_clz_i64(TCGv_i64 ret, TCGv_i64 arg1, TCGv_i64 arg2) 2442 { 2443 if (TCG_TARGET_HAS_clz_i64) { 2444 tcg_gen_op3_i64(INDEX_op_clz_i64, ret, arg1, arg2); 2445 } else { 2446 gen_helper_clz_i64(ret, arg1, arg2); 2447 } 2448 } 2449 2450 void tcg_gen_clzi_i64(TCGv_i64 ret, TCGv_i64 arg1, uint64_t arg2) 2451 { 2452 if (TCG_TARGET_REG_BITS == 32 2453 && TCG_TARGET_HAS_clz_i32 2454 && arg2 <= 0xffffffffu) { 2455 TCGv_i32 t = tcg_temp_ebb_new_i32(); 2456 tcg_gen_clzi_i32(t, TCGV_LOW(arg1), arg2 - 32); 2457 tcg_gen_addi_i32(t, t, 32); 2458 tcg_gen_clz_i32(TCGV_LOW(ret), TCGV_HIGH(arg1), t); 2459 tcg_gen_movi_i32(TCGV_HIGH(ret), 0); 2460 tcg_temp_free_i32(t); 2461 } else { 2462 tcg_gen_clz_i64(ret, arg1, tcg_constant_i64(arg2)); 2463 } 2464 } 2465 2466 void tcg_gen_ctz_i64(TCGv_i64 ret, TCGv_i64 arg1, TCGv_i64 arg2) 2467 { 2468 if (TCG_TARGET_HAS_ctz_i64) { 2469 tcg_gen_op3_i64(INDEX_op_ctz_i64, ret, arg1, arg2); 2470 } else if (TCG_TARGET_HAS_ctpop_i64 || TCG_TARGET_HAS_clz_i64) { 2471 TCGv_i64 z, t = tcg_temp_ebb_new_i64(); 2472 2473 if (TCG_TARGET_HAS_ctpop_i64) { 2474 tcg_gen_subi_i64(t, arg1, 1); 2475 tcg_gen_andc_i64(t, t, arg1); 2476 tcg_gen_ctpop_i64(t, t); 2477 } else { 2478 /* Since all non-x86 hosts have clz(0) == 64, don't fight it. */ 2479 tcg_gen_neg_i64(t, arg1); 2480 tcg_gen_and_i64(t, t, arg1); 2481 tcg_gen_clzi_i64(t, t, 64); 2482 tcg_gen_xori_i64(t, t, 63); 2483 } 2484 z = tcg_constant_i64(0); 2485 tcg_gen_movcond_i64(TCG_COND_EQ, ret, arg1, z, arg2, t); 2486 tcg_temp_free_i64(t); 2487 tcg_temp_free_i64(z); 2488 } else { 2489 gen_helper_ctz_i64(ret, arg1, arg2); 2490 } 2491 } 2492 2493 void tcg_gen_ctzi_i64(TCGv_i64 ret, TCGv_i64 arg1, uint64_t arg2) 2494 { 2495 if (TCG_TARGET_REG_BITS == 32 2496 && TCG_TARGET_HAS_ctz_i32 2497 && arg2 <= 0xffffffffu) { 2498 TCGv_i32 t32 = tcg_temp_ebb_new_i32(); 2499 tcg_gen_ctzi_i32(t32, TCGV_HIGH(arg1), arg2 - 32); 2500 tcg_gen_addi_i32(t32, t32, 32); 2501 tcg_gen_ctz_i32(TCGV_LOW(ret), TCGV_LOW(arg1), t32); 2502 tcg_gen_movi_i32(TCGV_HIGH(ret), 0); 2503 tcg_temp_free_i32(t32); 2504 } else if (!TCG_TARGET_HAS_ctz_i64 2505 && TCG_TARGET_HAS_ctpop_i64 2506 && arg2 == 64) { 2507 /* This equivalence has the advantage of not requiring a fixup. */ 2508 TCGv_i64 t = tcg_temp_ebb_new_i64(); 2509 tcg_gen_subi_i64(t, arg1, 1); 2510 tcg_gen_andc_i64(t, t, arg1); 2511 tcg_gen_ctpop_i64(ret, t); 2512 tcg_temp_free_i64(t); 2513 } else { 2514 tcg_gen_ctz_i64(ret, arg1, tcg_constant_i64(arg2)); 2515 } 2516 } 2517 2518 void tcg_gen_clrsb_i64(TCGv_i64 ret, TCGv_i64 arg) 2519 { 2520 if (TCG_TARGET_HAS_clz_i64 || TCG_TARGET_HAS_clz_i32) { 2521 TCGv_i64 t = tcg_temp_ebb_new_i64(); 2522 tcg_gen_sari_i64(t, arg, 63); 2523 tcg_gen_xor_i64(t, t, arg); 2524 tcg_gen_clzi_i64(t, t, 64); 2525 tcg_gen_subi_i64(ret, t, 1); 2526 tcg_temp_free_i64(t); 2527 } else { 2528 gen_helper_clrsb_i64(ret, arg); 2529 } 2530 } 2531 2532 void tcg_gen_ctpop_i64(TCGv_i64 ret, TCGv_i64 arg1) 2533 { 2534 if (TCG_TARGET_HAS_ctpop_i64) { 2535 tcg_gen_op2_i64(INDEX_op_ctpop_i64, ret, arg1); 2536 } else if (TCG_TARGET_REG_BITS == 32 && TCG_TARGET_HAS_ctpop_i32) { 2537 tcg_gen_ctpop_i32(TCGV_HIGH(ret), TCGV_HIGH(arg1)); 2538 tcg_gen_ctpop_i32(TCGV_LOW(ret), TCGV_LOW(arg1)); 2539 tcg_gen_add_i32(TCGV_LOW(ret), TCGV_LOW(ret), TCGV_HIGH(ret)); 2540 tcg_gen_movi_i32(TCGV_HIGH(ret), 0); 2541 } else { 2542 gen_helper_ctpop_i64(ret, arg1); 2543 } 2544 } 2545 2546 void tcg_gen_rotl_i64(TCGv_i64 ret, TCGv_i64 arg1, TCGv_i64 arg2) 2547 { 2548 if (TCG_TARGET_HAS_rot_i64) { 2549 tcg_gen_op3_i64(INDEX_op_rotl_i64, ret, arg1, arg2); 2550 } else { 2551 TCGv_i64 t0, t1; 2552 t0 = tcg_temp_ebb_new_i64(); 2553 t1 = tcg_temp_ebb_new_i64(); 2554 tcg_gen_shl_i64(t0, arg1, arg2); 2555 tcg_gen_subfi_i64(t1, 64, arg2); 2556 tcg_gen_shr_i64(t1, arg1, t1); 2557 tcg_gen_or_i64(ret, t0, t1); 2558 tcg_temp_free_i64(t0); 2559 tcg_temp_free_i64(t1); 2560 } 2561 } 2562 2563 void tcg_gen_rotli_i64(TCGv_i64 ret, TCGv_i64 arg1, int64_t arg2) 2564 { 2565 tcg_debug_assert(arg2 >= 0 && arg2 < 64); 2566 /* some cases can be optimized here */ 2567 if (arg2 == 0) { 2568 tcg_gen_mov_i64(ret, arg1); 2569 } else if (TCG_TARGET_HAS_rot_i64) { 2570 tcg_gen_rotl_i64(ret, arg1, tcg_constant_i64(arg2)); 2571 } else { 2572 TCGv_i64 t0, t1; 2573 t0 = tcg_temp_ebb_new_i64(); 2574 t1 = tcg_temp_ebb_new_i64(); 2575 tcg_gen_shli_i64(t0, arg1, arg2); 2576 tcg_gen_shri_i64(t1, arg1, 64 - arg2); 2577 tcg_gen_or_i64(ret, t0, t1); 2578 tcg_temp_free_i64(t0); 2579 tcg_temp_free_i64(t1); 2580 } 2581 } 2582 2583 void tcg_gen_rotr_i64(TCGv_i64 ret, TCGv_i64 arg1, TCGv_i64 arg2) 2584 { 2585 if (TCG_TARGET_HAS_rot_i64) { 2586 tcg_gen_op3_i64(INDEX_op_rotr_i64, ret, arg1, arg2); 2587 } else { 2588 TCGv_i64 t0, t1; 2589 t0 = tcg_temp_ebb_new_i64(); 2590 t1 = tcg_temp_ebb_new_i64(); 2591 tcg_gen_shr_i64(t0, arg1, arg2); 2592 tcg_gen_subfi_i64(t1, 64, arg2); 2593 tcg_gen_shl_i64(t1, arg1, t1); 2594 tcg_gen_or_i64(ret, t0, t1); 2595 tcg_temp_free_i64(t0); 2596 tcg_temp_free_i64(t1); 2597 } 2598 } 2599 2600 void tcg_gen_rotri_i64(TCGv_i64 ret, TCGv_i64 arg1, int64_t arg2) 2601 { 2602 tcg_debug_assert(arg2 >= 0 && arg2 < 64); 2603 /* some cases can be optimized here */ 2604 if (arg2 == 0) { 2605 tcg_gen_mov_i64(ret, arg1); 2606 } else { 2607 tcg_gen_rotli_i64(ret, arg1, 64 - arg2); 2608 } 2609 } 2610 2611 void tcg_gen_deposit_i64(TCGv_i64 ret, TCGv_i64 arg1, TCGv_i64 arg2, 2612 unsigned int ofs, unsigned int len) 2613 { 2614 uint64_t mask; 2615 TCGv_i64 t1; 2616 2617 tcg_debug_assert(ofs < 64); 2618 tcg_debug_assert(len > 0); 2619 tcg_debug_assert(len <= 64); 2620 tcg_debug_assert(ofs + len <= 64); 2621 2622 if (len == 64) { 2623 tcg_gen_mov_i64(ret, arg2); 2624 return; 2625 } 2626 if (TCG_TARGET_HAS_deposit_i64 && TCG_TARGET_deposit_i64_valid(ofs, len)) { 2627 tcg_gen_op5ii_i64(INDEX_op_deposit_i64, ret, arg1, arg2, ofs, len); 2628 return; 2629 } 2630 2631 if (TCG_TARGET_REG_BITS == 32) { 2632 if (ofs >= 32) { 2633 tcg_gen_deposit_i32(TCGV_HIGH(ret), TCGV_HIGH(arg1), 2634 TCGV_LOW(arg2), ofs - 32, len); 2635 tcg_gen_mov_i32(TCGV_LOW(ret), TCGV_LOW(arg1)); 2636 return; 2637 } 2638 if (ofs + len <= 32) { 2639 tcg_gen_deposit_i32(TCGV_LOW(ret), TCGV_LOW(arg1), 2640 TCGV_LOW(arg2), ofs, len); 2641 tcg_gen_mov_i32(TCGV_HIGH(ret), TCGV_HIGH(arg1)); 2642 return; 2643 } 2644 } 2645 2646 t1 = tcg_temp_ebb_new_i64(); 2647 2648 if (TCG_TARGET_HAS_extract2_i64) { 2649 if (ofs + len == 64) { 2650 tcg_gen_shli_i64(t1, arg1, len); 2651 tcg_gen_extract2_i64(ret, t1, arg2, len); 2652 goto done; 2653 } 2654 if (ofs == 0) { 2655 tcg_gen_extract2_i64(ret, arg1, arg2, len); 2656 tcg_gen_rotli_i64(ret, ret, len); 2657 goto done; 2658 } 2659 } 2660 2661 mask = (1ull << len) - 1; 2662 if (ofs + len < 64) { 2663 tcg_gen_andi_i64(t1, arg2, mask); 2664 tcg_gen_shli_i64(t1, t1, ofs); 2665 } else { 2666 tcg_gen_shli_i64(t1, arg2, ofs); 2667 } 2668 tcg_gen_andi_i64(ret, arg1, ~(mask << ofs)); 2669 tcg_gen_or_i64(ret, ret, t1); 2670 done: 2671 tcg_temp_free_i64(t1); 2672 } 2673 2674 void tcg_gen_deposit_z_i64(TCGv_i64 ret, TCGv_i64 arg, 2675 unsigned int ofs, unsigned int len) 2676 { 2677 tcg_debug_assert(ofs < 64); 2678 tcg_debug_assert(len > 0); 2679 tcg_debug_assert(len <= 64); 2680 tcg_debug_assert(ofs + len <= 64); 2681 2682 if (ofs + len == 64) { 2683 tcg_gen_shli_i64(ret, arg, ofs); 2684 } else if (ofs == 0) { 2685 tcg_gen_andi_i64(ret, arg, (1ull << len) - 1); 2686 } else if (TCG_TARGET_HAS_deposit_i64 2687 && TCG_TARGET_deposit_i64_valid(ofs, len)) { 2688 TCGv_i64 zero = tcg_constant_i64(0); 2689 tcg_gen_op5ii_i64(INDEX_op_deposit_i64, ret, zero, arg, ofs, len); 2690 } else { 2691 if (TCG_TARGET_REG_BITS == 32) { 2692 if (ofs >= 32) { 2693 tcg_gen_deposit_z_i32(TCGV_HIGH(ret), TCGV_LOW(arg), 2694 ofs - 32, len); 2695 tcg_gen_movi_i32(TCGV_LOW(ret), 0); 2696 return; 2697 } 2698 if (ofs + len <= 32) { 2699 tcg_gen_deposit_z_i32(TCGV_LOW(ret), TCGV_LOW(arg), ofs, len); 2700 tcg_gen_movi_i32(TCGV_HIGH(ret), 0); 2701 return; 2702 } 2703 } 2704 /* To help two-operand hosts we prefer to zero-extend first, 2705 which allows ARG to stay live. */ 2706 switch (len) { 2707 case 32: 2708 if (TCG_TARGET_HAS_ext32u_i64) { 2709 tcg_gen_ext32u_i64(ret, arg); 2710 tcg_gen_shli_i64(ret, ret, ofs); 2711 return; 2712 } 2713 break; 2714 case 16: 2715 if (TCG_TARGET_HAS_ext16u_i64) { 2716 tcg_gen_ext16u_i64(ret, arg); 2717 tcg_gen_shli_i64(ret, ret, ofs); 2718 return; 2719 } 2720 break; 2721 case 8: 2722 if (TCG_TARGET_HAS_ext8u_i64) { 2723 tcg_gen_ext8u_i64(ret, arg); 2724 tcg_gen_shli_i64(ret, ret, ofs); 2725 return; 2726 } 2727 break; 2728 } 2729 /* Otherwise prefer zero-extension over AND for code size. */ 2730 switch (ofs + len) { 2731 case 32: 2732 if (TCG_TARGET_HAS_ext32u_i64) { 2733 tcg_gen_shli_i64(ret, arg, ofs); 2734 tcg_gen_ext32u_i64(ret, ret); 2735 return; 2736 } 2737 break; 2738 case 16: 2739 if (TCG_TARGET_HAS_ext16u_i64) { 2740 tcg_gen_shli_i64(ret, arg, ofs); 2741 tcg_gen_ext16u_i64(ret, ret); 2742 return; 2743 } 2744 break; 2745 case 8: 2746 if (TCG_TARGET_HAS_ext8u_i64) { 2747 tcg_gen_shli_i64(ret, arg, ofs); 2748 tcg_gen_ext8u_i64(ret, ret); 2749 return; 2750 } 2751 break; 2752 } 2753 tcg_gen_andi_i64(ret, arg, (1ull << len) - 1); 2754 tcg_gen_shli_i64(ret, ret, ofs); 2755 } 2756 } 2757 2758 void tcg_gen_extract_i64(TCGv_i64 ret, TCGv_i64 arg, 2759 unsigned int ofs, unsigned int len) 2760 { 2761 tcg_debug_assert(ofs < 64); 2762 tcg_debug_assert(len > 0); 2763 tcg_debug_assert(len <= 64); 2764 tcg_debug_assert(ofs + len <= 64); 2765 2766 /* Canonicalize certain special cases, even if extract is supported. */ 2767 if (ofs + len == 64) { 2768 tcg_gen_shri_i64(ret, arg, 64 - len); 2769 return; 2770 } 2771 if (ofs == 0) { 2772 tcg_gen_andi_i64(ret, arg, (1ull << len) - 1); 2773 return; 2774 } 2775 2776 if (TCG_TARGET_REG_BITS == 32) { 2777 /* Look for a 32-bit extract within one of the two words. */ 2778 if (ofs >= 32) { 2779 tcg_gen_extract_i32(TCGV_LOW(ret), TCGV_HIGH(arg), ofs - 32, len); 2780 tcg_gen_movi_i32(TCGV_HIGH(ret), 0); 2781 return; 2782 } 2783 if (ofs + len <= 32) { 2784 tcg_gen_extract_i32(TCGV_LOW(ret), TCGV_LOW(arg), ofs, len); 2785 tcg_gen_movi_i32(TCGV_HIGH(ret), 0); 2786 return; 2787 } 2788 /* The field is split across two words. One double-word 2789 shift is better than two double-word shifts. */ 2790 goto do_shift_and; 2791 } 2792 2793 if (TCG_TARGET_HAS_extract_i64 2794 && TCG_TARGET_extract_i64_valid(ofs, len)) { 2795 tcg_gen_op4ii_i64(INDEX_op_extract_i64, ret, arg, ofs, len); 2796 return; 2797 } 2798 2799 /* Assume that zero-extension, if available, is cheaper than a shift. */ 2800 switch (ofs + len) { 2801 case 32: 2802 if (TCG_TARGET_HAS_ext32u_i64) { 2803 tcg_gen_ext32u_i64(ret, arg); 2804 tcg_gen_shri_i64(ret, ret, ofs); 2805 return; 2806 } 2807 break; 2808 case 16: 2809 if (TCG_TARGET_HAS_ext16u_i64) { 2810 tcg_gen_ext16u_i64(ret, arg); 2811 tcg_gen_shri_i64(ret, ret, ofs); 2812 return; 2813 } 2814 break; 2815 case 8: 2816 if (TCG_TARGET_HAS_ext8u_i64) { 2817 tcg_gen_ext8u_i64(ret, arg); 2818 tcg_gen_shri_i64(ret, ret, ofs); 2819 return; 2820 } 2821 break; 2822 } 2823 2824 /* ??? Ideally we'd know what values are available for immediate AND. 2825 Assume that 8 bits are available, plus the special cases of 16 and 32, 2826 so that we get ext8u, ext16u, and ext32u. */ 2827 switch (len) { 2828 case 1 ... 8: case 16: case 32: 2829 do_shift_and: 2830 tcg_gen_shri_i64(ret, arg, ofs); 2831 tcg_gen_andi_i64(ret, ret, (1ull << len) - 1); 2832 break; 2833 default: 2834 tcg_gen_shli_i64(ret, arg, 64 - len - ofs); 2835 tcg_gen_shri_i64(ret, ret, 64 - len); 2836 break; 2837 } 2838 } 2839 2840 void tcg_gen_sextract_i64(TCGv_i64 ret, TCGv_i64 arg, 2841 unsigned int ofs, unsigned int len) 2842 { 2843 tcg_debug_assert(ofs < 64); 2844 tcg_debug_assert(len > 0); 2845 tcg_debug_assert(len <= 64); 2846 tcg_debug_assert(ofs + len <= 64); 2847 2848 /* Canonicalize certain special cases, even if sextract is supported. */ 2849 if (ofs + len == 64) { 2850 tcg_gen_sari_i64(ret, arg, 64 - len); 2851 return; 2852 } 2853 if (ofs == 0) { 2854 switch (len) { 2855 case 32: 2856 tcg_gen_ext32s_i64(ret, arg); 2857 return; 2858 case 16: 2859 tcg_gen_ext16s_i64(ret, arg); 2860 return; 2861 case 8: 2862 tcg_gen_ext8s_i64(ret, arg); 2863 return; 2864 } 2865 } 2866 2867 if (TCG_TARGET_REG_BITS == 32) { 2868 /* Look for a 32-bit extract within one of the two words. */ 2869 if (ofs >= 32) { 2870 tcg_gen_sextract_i32(TCGV_LOW(ret), TCGV_HIGH(arg), ofs - 32, len); 2871 } else if (ofs + len <= 32) { 2872 tcg_gen_sextract_i32(TCGV_LOW(ret), TCGV_LOW(arg), ofs, len); 2873 } else if (ofs == 0) { 2874 tcg_gen_mov_i32(TCGV_LOW(ret), TCGV_LOW(arg)); 2875 tcg_gen_sextract_i32(TCGV_HIGH(ret), TCGV_HIGH(arg), 0, len - 32); 2876 return; 2877 } else if (len > 32) { 2878 TCGv_i32 t = tcg_temp_ebb_new_i32(); 2879 /* Extract the bits for the high word normally. */ 2880 tcg_gen_sextract_i32(t, TCGV_HIGH(arg), ofs + 32, len - 32); 2881 /* Shift the field down for the low part. */ 2882 tcg_gen_shri_i64(ret, arg, ofs); 2883 /* Overwrite the shift into the high part. */ 2884 tcg_gen_mov_i32(TCGV_HIGH(ret), t); 2885 tcg_temp_free_i32(t); 2886 return; 2887 } else { 2888 /* Shift the field down for the low part, such that the 2889 field sits at the MSB. */ 2890 tcg_gen_shri_i64(ret, arg, ofs + len - 32); 2891 /* Shift the field down from the MSB, sign extending. */ 2892 tcg_gen_sari_i32(TCGV_LOW(ret), TCGV_LOW(ret), 32 - len); 2893 } 2894 /* Sign-extend the field from 32 bits. */ 2895 tcg_gen_sari_i32(TCGV_HIGH(ret), TCGV_LOW(ret), 31); 2896 return; 2897 } 2898 2899 if (TCG_TARGET_HAS_sextract_i64 2900 && TCG_TARGET_extract_i64_valid(ofs, len)) { 2901 tcg_gen_op4ii_i64(INDEX_op_sextract_i64, ret, arg, ofs, len); 2902 return; 2903 } 2904 2905 /* Assume that sign-extension, if available, is cheaper than a shift. */ 2906 switch (ofs + len) { 2907 case 32: 2908 if (TCG_TARGET_HAS_ext32s_i64) { 2909 tcg_gen_ext32s_i64(ret, arg); 2910 tcg_gen_sari_i64(ret, ret, ofs); 2911 return; 2912 } 2913 break; 2914 case 16: 2915 if (TCG_TARGET_HAS_ext16s_i64) { 2916 tcg_gen_ext16s_i64(ret, arg); 2917 tcg_gen_sari_i64(ret, ret, ofs); 2918 return; 2919 } 2920 break; 2921 case 8: 2922 if (TCG_TARGET_HAS_ext8s_i64) { 2923 tcg_gen_ext8s_i64(ret, arg); 2924 tcg_gen_sari_i64(ret, ret, ofs); 2925 return; 2926 } 2927 break; 2928 } 2929 switch (len) { 2930 case 32: 2931 if (TCG_TARGET_HAS_ext32s_i64) { 2932 tcg_gen_shri_i64(ret, arg, ofs); 2933 tcg_gen_ext32s_i64(ret, ret); 2934 return; 2935 } 2936 break; 2937 case 16: 2938 if (TCG_TARGET_HAS_ext16s_i64) { 2939 tcg_gen_shri_i64(ret, arg, ofs); 2940 tcg_gen_ext16s_i64(ret, ret); 2941 return; 2942 } 2943 break; 2944 case 8: 2945 if (TCG_TARGET_HAS_ext8s_i64) { 2946 tcg_gen_shri_i64(ret, arg, ofs); 2947 tcg_gen_ext8s_i64(ret, ret); 2948 return; 2949 } 2950 break; 2951 } 2952 tcg_gen_shli_i64(ret, arg, 64 - len - ofs); 2953 tcg_gen_sari_i64(ret, ret, 64 - len); 2954 } 2955 2956 /* 2957 * Extract 64 bits from a 128-bit input, ah:al, starting from ofs. 2958 * Unlike tcg_gen_extract_i64 above, len is fixed at 64. 2959 */ 2960 void tcg_gen_extract2_i64(TCGv_i64 ret, TCGv_i64 al, TCGv_i64 ah, 2961 unsigned int ofs) 2962 { 2963 tcg_debug_assert(ofs <= 64); 2964 if (ofs == 0) { 2965 tcg_gen_mov_i64(ret, al); 2966 } else if (ofs == 64) { 2967 tcg_gen_mov_i64(ret, ah); 2968 } else if (al == ah) { 2969 tcg_gen_rotri_i64(ret, al, ofs); 2970 } else if (TCG_TARGET_HAS_extract2_i64) { 2971 tcg_gen_op4i_i64(INDEX_op_extract2_i64, ret, al, ah, ofs); 2972 } else { 2973 TCGv_i64 t0 = tcg_temp_ebb_new_i64(); 2974 tcg_gen_shri_i64(t0, al, ofs); 2975 tcg_gen_deposit_i64(ret, t0, ah, 64 - ofs, ofs); 2976 tcg_temp_free_i64(t0); 2977 } 2978 } 2979 2980 void tcg_gen_movcond_i64(TCGCond cond, TCGv_i64 ret, TCGv_i64 c1, 2981 TCGv_i64 c2, TCGv_i64 v1, TCGv_i64 v2) 2982 { 2983 if (cond == TCG_COND_ALWAYS) { 2984 tcg_gen_mov_i64(ret, v1); 2985 } else if (cond == TCG_COND_NEVER) { 2986 tcg_gen_mov_i64(ret, v2); 2987 } else if (TCG_TARGET_REG_BITS == 64) { 2988 tcg_gen_op6i_i64(INDEX_op_movcond_i64, ret, c1, c2, v1, v2, cond); 2989 } else { 2990 TCGv_i32 t0 = tcg_temp_ebb_new_i32(); 2991 TCGv_i32 zero = tcg_constant_i32(0); 2992 2993 tcg_gen_op6i_i32(INDEX_op_setcond2_i32, t0, 2994 TCGV_LOW(c1), TCGV_HIGH(c1), 2995 TCGV_LOW(c2), TCGV_HIGH(c2), cond); 2996 2997 tcg_gen_movcond_i32(TCG_COND_NE, TCGV_LOW(ret), t0, zero, 2998 TCGV_LOW(v1), TCGV_LOW(v2)); 2999 tcg_gen_movcond_i32(TCG_COND_NE, TCGV_HIGH(ret), t0, zero, 3000 TCGV_HIGH(v1), TCGV_HIGH(v2)); 3001 3002 tcg_temp_free_i32(t0); 3003 } 3004 } 3005 3006 void tcg_gen_add2_i64(TCGv_i64 rl, TCGv_i64 rh, TCGv_i64 al, 3007 TCGv_i64 ah, TCGv_i64 bl, TCGv_i64 bh) 3008 { 3009 if (TCG_TARGET_HAS_add2_i64) { 3010 tcg_gen_op6_i64(INDEX_op_add2_i64, rl, rh, al, ah, bl, bh); 3011 } else { 3012 TCGv_i64 t0 = tcg_temp_ebb_new_i64(); 3013 TCGv_i64 t1 = tcg_temp_ebb_new_i64(); 3014 tcg_gen_add_i64(t0, al, bl); 3015 tcg_gen_setcond_i64(TCG_COND_LTU, t1, t0, al); 3016 tcg_gen_add_i64(rh, ah, bh); 3017 tcg_gen_add_i64(rh, rh, t1); 3018 tcg_gen_mov_i64(rl, t0); 3019 tcg_temp_free_i64(t0); 3020 tcg_temp_free_i64(t1); 3021 } 3022 } 3023 3024 void tcg_gen_sub2_i64(TCGv_i64 rl, TCGv_i64 rh, TCGv_i64 al, 3025 TCGv_i64 ah, TCGv_i64 bl, TCGv_i64 bh) 3026 { 3027 if (TCG_TARGET_HAS_sub2_i64) { 3028 tcg_gen_op6_i64(INDEX_op_sub2_i64, rl, rh, al, ah, bl, bh); 3029 } else { 3030 TCGv_i64 t0 = tcg_temp_ebb_new_i64(); 3031 TCGv_i64 t1 = tcg_temp_ebb_new_i64(); 3032 tcg_gen_sub_i64(t0, al, bl); 3033 tcg_gen_setcond_i64(TCG_COND_LTU, t1, al, bl); 3034 tcg_gen_sub_i64(rh, ah, bh); 3035 tcg_gen_sub_i64(rh, rh, t1); 3036 tcg_gen_mov_i64(rl, t0); 3037 tcg_temp_free_i64(t0); 3038 tcg_temp_free_i64(t1); 3039 } 3040 } 3041 3042 void tcg_gen_mulu2_i64(TCGv_i64 rl, TCGv_i64 rh, TCGv_i64 arg1, TCGv_i64 arg2) 3043 { 3044 if (TCG_TARGET_HAS_mulu2_i64) { 3045 tcg_gen_op4_i64(INDEX_op_mulu2_i64, rl, rh, arg1, arg2); 3046 } else if (TCG_TARGET_HAS_muluh_i64) { 3047 TCGv_i64 t = tcg_temp_ebb_new_i64(); 3048 tcg_gen_op3_i64(INDEX_op_mul_i64, t, arg1, arg2); 3049 tcg_gen_op3_i64(INDEX_op_muluh_i64, rh, arg1, arg2); 3050 tcg_gen_mov_i64(rl, t); 3051 tcg_temp_free_i64(t); 3052 } else { 3053 TCGv_i64 t0 = tcg_temp_ebb_new_i64(); 3054 tcg_gen_mul_i64(t0, arg1, arg2); 3055 gen_helper_muluh_i64(rh, arg1, arg2); 3056 tcg_gen_mov_i64(rl, t0); 3057 tcg_temp_free_i64(t0); 3058 } 3059 } 3060 3061 void tcg_gen_muls2_i64(TCGv_i64 rl, TCGv_i64 rh, TCGv_i64 arg1, TCGv_i64 arg2) 3062 { 3063 if (TCG_TARGET_HAS_muls2_i64) { 3064 tcg_gen_op4_i64(INDEX_op_muls2_i64, rl, rh, arg1, arg2); 3065 } else if (TCG_TARGET_HAS_mulsh_i64) { 3066 TCGv_i64 t = tcg_temp_ebb_new_i64(); 3067 tcg_gen_op3_i64(INDEX_op_mul_i64, t, arg1, arg2); 3068 tcg_gen_op3_i64(INDEX_op_mulsh_i64, rh, arg1, arg2); 3069 tcg_gen_mov_i64(rl, t); 3070 tcg_temp_free_i64(t); 3071 } else if (TCG_TARGET_HAS_mulu2_i64 || TCG_TARGET_HAS_muluh_i64) { 3072 TCGv_i64 t0 = tcg_temp_ebb_new_i64(); 3073 TCGv_i64 t1 = tcg_temp_ebb_new_i64(); 3074 TCGv_i64 t2 = tcg_temp_ebb_new_i64(); 3075 TCGv_i64 t3 = tcg_temp_ebb_new_i64(); 3076 tcg_gen_mulu2_i64(t0, t1, arg1, arg2); 3077 /* Adjust for negative inputs. */ 3078 tcg_gen_sari_i64(t2, arg1, 63); 3079 tcg_gen_sari_i64(t3, arg2, 63); 3080 tcg_gen_and_i64(t2, t2, arg2); 3081 tcg_gen_and_i64(t3, t3, arg1); 3082 tcg_gen_sub_i64(rh, t1, t2); 3083 tcg_gen_sub_i64(rh, rh, t3); 3084 tcg_gen_mov_i64(rl, t0); 3085 tcg_temp_free_i64(t0); 3086 tcg_temp_free_i64(t1); 3087 tcg_temp_free_i64(t2); 3088 tcg_temp_free_i64(t3); 3089 } else { 3090 TCGv_i64 t0 = tcg_temp_ebb_new_i64(); 3091 tcg_gen_mul_i64(t0, arg1, arg2); 3092 gen_helper_mulsh_i64(rh, arg1, arg2); 3093 tcg_gen_mov_i64(rl, t0); 3094 tcg_temp_free_i64(t0); 3095 } 3096 } 3097 3098 void tcg_gen_mulsu2_i64(TCGv_i64 rl, TCGv_i64 rh, TCGv_i64 arg1, TCGv_i64 arg2) 3099 { 3100 TCGv_i64 t0 = tcg_temp_ebb_new_i64(); 3101 TCGv_i64 t1 = tcg_temp_ebb_new_i64(); 3102 TCGv_i64 t2 = tcg_temp_ebb_new_i64(); 3103 tcg_gen_mulu2_i64(t0, t1, arg1, arg2); 3104 /* Adjust for negative input for the signed arg1. */ 3105 tcg_gen_sari_i64(t2, arg1, 63); 3106 tcg_gen_and_i64(t2, t2, arg2); 3107 tcg_gen_sub_i64(rh, t1, t2); 3108 tcg_gen_mov_i64(rl, t0); 3109 tcg_temp_free_i64(t0); 3110 tcg_temp_free_i64(t1); 3111 tcg_temp_free_i64(t2); 3112 } 3113 3114 void tcg_gen_smin_i64(TCGv_i64 ret, TCGv_i64 a, TCGv_i64 b) 3115 { 3116 tcg_gen_movcond_i64(TCG_COND_LT, ret, a, b, a, b); 3117 } 3118 3119 void tcg_gen_umin_i64(TCGv_i64 ret, TCGv_i64 a, TCGv_i64 b) 3120 { 3121 tcg_gen_movcond_i64(TCG_COND_LTU, ret, a, b, a, b); 3122 } 3123 3124 void tcg_gen_smax_i64(TCGv_i64 ret, TCGv_i64 a, TCGv_i64 b) 3125 { 3126 tcg_gen_movcond_i64(TCG_COND_LT, ret, a, b, b, a); 3127 } 3128 3129 void tcg_gen_umax_i64(TCGv_i64 ret, TCGv_i64 a, TCGv_i64 b) 3130 { 3131 tcg_gen_movcond_i64(TCG_COND_LTU, ret, a, b, b, a); 3132 } 3133 3134 void tcg_gen_abs_i64(TCGv_i64 ret, TCGv_i64 a) 3135 { 3136 TCGv_i64 t = tcg_temp_ebb_new_i64(); 3137 3138 tcg_gen_sari_i64(t, a, 63); 3139 tcg_gen_xor_i64(ret, a, t); 3140 tcg_gen_sub_i64(ret, ret, t); 3141 tcg_temp_free_i64(t); 3142 } 3143 3144 /* Size changing operations. */ 3145 3146 void tcg_gen_extrl_i64_i32(TCGv_i32 ret, TCGv_i64 arg) 3147 { 3148 if (TCG_TARGET_REG_BITS == 32) { 3149 tcg_gen_mov_i32(ret, TCGV_LOW(arg)); 3150 } else if (TCG_TARGET_HAS_extr_i64_i32) { 3151 tcg_gen_op2(INDEX_op_extrl_i64_i32, 3152 tcgv_i32_arg(ret), tcgv_i64_arg(arg)); 3153 } else { 3154 tcg_gen_mov_i32(ret, (TCGv_i32)arg); 3155 } 3156 } 3157 3158 void tcg_gen_extrh_i64_i32(TCGv_i32 ret, TCGv_i64 arg) 3159 { 3160 if (TCG_TARGET_REG_BITS == 32) { 3161 tcg_gen_mov_i32(ret, TCGV_HIGH(arg)); 3162 } else if (TCG_TARGET_HAS_extr_i64_i32) { 3163 tcg_gen_op2(INDEX_op_extrh_i64_i32, 3164 tcgv_i32_arg(ret), tcgv_i64_arg(arg)); 3165 } else { 3166 TCGv_i64 t = tcg_temp_ebb_new_i64(); 3167 tcg_gen_shri_i64(t, arg, 32); 3168 tcg_gen_mov_i32(ret, (TCGv_i32)t); 3169 tcg_temp_free_i64(t); 3170 } 3171 } 3172 3173 void tcg_gen_extu_i32_i64(TCGv_i64 ret, TCGv_i32 arg) 3174 { 3175 if (TCG_TARGET_REG_BITS == 32) { 3176 tcg_gen_mov_i32(TCGV_LOW(ret), arg); 3177 tcg_gen_movi_i32(TCGV_HIGH(ret), 0); 3178 } else { 3179 tcg_gen_op2(INDEX_op_extu_i32_i64, 3180 tcgv_i64_arg(ret), tcgv_i32_arg(arg)); 3181 } 3182 } 3183 3184 void tcg_gen_ext_i32_i64(TCGv_i64 ret, TCGv_i32 arg) 3185 { 3186 if (TCG_TARGET_REG_BITS == 32) { 3187 tcg_gen_mov_i32(TCGV_LOW(ret), arg); 3188 tcg_gen_sari_i32(TCGV_HIGH(ret), TCGV_LOW(ret), 31); 3189 } else { 3190 tcg_gen_op2(INDEX_op_ext_i32_i64, 3191 tcgv_i64_arg(ret), tcgv_i32_arg(arg)); 3192 } 3193 } 3194 3195 void tcg_gen_concat_i32_i64(TCGv_i64 dest, TCGv_i32 low, TCGv_i32 high) 3196 { 3197 TCGv_i64 tmp; 3198 3199 if (TCG_TARGET_REG_BITS == 32) { 3200 tcg_gen_mov_i32(TCGV_LOW(dest), low); 3201 tcg_gen_mov_i32(TCGV_HIGH(dest), high); 3202 return; 3203 } 3204 3205 tmp = tcg_temp_ebb_new_i64(); 3206 /* These extensions are only needed for type correctness. 3207 We may be able to do better given target specific information. */ 3208 tcg_gen_extu_i32_i64(tmp, high); 3209 tcg_gen_extu_i32_i64(dest, low); 3210 /* If deposit is available, use it. Otherwise use the extra 3211 knowledge that we have of the zero-extensions above. */ 3212 if (TCG_TARGET_HAS_deposit_i64 && TCG_TARGET_deposit_i64_valid(32, 32)) { 3213 tcg_gen_deposit_i64(dest, dest, tmp, 32, 32); 3214 } else { 3215 tcg_gen_shli_i64(tmp, tmp, 32); 3216 tcg_gen_or_i64(dest, dest, tmp); 3217 } 3218 tcg_temp_free_i64(tmp); 3219 } 3220 3221 void tcg_gen_extr_i64_i32(TCGv_i32 lo, TCGv_i32 hi, TCGv_i64 arg) 3222 { 3223 if (TCG_TARGET_REG_BITS == 32) { 3224 tcg_gen_mov_i32(lo, TCGV_LOW(arg)); 3225 tcg_gen_mov_i32(hi, TCGV_HIGH(arg)); 3226 } else { 3227 tcg_gen_extrl_i64_i32(lo, arg); 3228 tcg_gen_extrh_i64_i32(hi, arg); 3229 } 3230 } 3231 3232 void tcg_gen_extr32_i64(TCGv_i64 lo, TCGv_i64 hi, TCGv_i64 arg) 3233 { 3234 tcg_gen_ext32u_i64(lo, arg); 3235 tcg_gen_shri_i64(hi, arg, 32); 3236 } 3237 3238 void tcg_gen_concat32_i64(TCGv_i64 ret, TCGv_i64 lo, TCGv_i64 hi) 3239 { 3240 tcg_gen_deposit_i64(ret, lo, hi, 32, 32); 3241 } 3242 3243 void tcg_gen_extr_i128_i64(TCGv_i64 lo, TCGv_i64 hi, TCGv_i128 arg) 3244 { 3245 tcg_gen_mov_i64(lo, TCGV128_LOW(arg)); 3246 tcg_gen_mov_i64(hi, TCGV128_HIGH(arg)); 3247 } 3248 3249 void tcg_gen_concat_i64_i128(TCGv_i128 ret, TCGv_i64 lo, TCGv_i64 hi) 3250 { 3251 tcg_gen_mov_i64(TCGV128_LOW(ret), lo); 3252 tcg_gen_mov_i64(TCGV128_HIGH(ret), hi); 3253 } 3254 3255 void tcg_gen_mov_i128(TCGv_i128 dst, TCGv_i128 src) 3256 { 3257 if (dst != src) { 3258 tcg_gen_mov_i64(TCGV128_LOW(dst), TCGV128_LOW(src)); 3259 tcg_gen_mov_i64(TCGV128_HIGH(dst), TCGV128_HIGH(src)); 3260 } 3261 } 3262 3263 void tcg_gen_ld_i128(TCGv_i128 ret, TCGv_ptr base, tcg_target_long offset) 3264 { 3265 if (HOST_BIG_ENDIAN) { 3266 tcg_gen_ld_i64(TCGV128_HIGH(ret), base, offset); 3267 tcg_gen_ld_i64(TCGV128_LOW(ret), base, offset + 8); 3268 } else { 3269 tcg_gen_ld_i64(TCGV128_LOW(ret), base, offset); 3270 tcg_gen_ld_i64(TCGV128_HIGH(ret), base, offset + 8); 3271 } 3272 } 3273 3274 void tcg_gen_st_i128(TCGv_i128 val, TCGv_ptr base, tcg_target_long offset) 3275 { 3276 if (HOST_BIG_ENDIAN) { 3277 tcg_gen_st_i64(TCGV128_HIGH(val), base, offset); 3278 tcg_gen_st_i64(TCGV128_LOW(val), base, offset + 8); 3279 } else { 3280 tcg_gen_st_i64(TCGV128_LOW(val), base, offset); 3281 tcg_gen_st_i64(TCGV128_HIGH(val), base, offset + 8); 3282 } 3283 } 3284 3285 /* QEMU specific operations. */ 3286 3287 void tcg_gen_exit_tb(const TranslationBlock *tb, unsigned idx) 3288 { 3289 /* 3290 * Let the jit code return the read-only version of the 3291 * TranslationBlock, so that we minimize the pc-relative 3292 * distance of the address of the exit_tb code to TB. 3293 * This will improve utilization of pc-relative address loads. 3294 * 3295 * TODO: Move this to translator_loop, so that all const 3296 * TranslationBlock pointers refer to read-only memory. 3297 * This requires coordination with targets that do not use 3298 * the translator_loop. 3299 */ 3300 uintptr_t val = (uintptr_t)tcg_splitwx_to_rx((void *)tb) + idx; 3301 3302 if (tb == NULL) { 3303 tcg_debug_assert(idx == 0); 3304 } else if (idx <= TB_EXIT_IDXMAX) { 3305 #ifdef CONFIG_DEBUG_TCG 3306 /* This is an exit following a goto_tb. Verify that we have 3307 seen this numbered exit before, via tcg_gen_goto_tb. */ 3308 tcg_debug_assert(tcg_ctx->goto_tb_issue_mask & (1 << idx)); 3309 #endif 3310 } else { 3311 /* This is an exit via the exitreq label. */ 3312 tcg_debug_assert(idx == TB_EXIT_REQUESTED); 3313 } 3314 3315 tcg_gen_op1i(INDEX_op_exit_tb, val); 3316 } 3317 3318 void tcg_gen_goto_tb(unsigned idx) 3319 { 3320 /* We tested CF_NO_GOTO_TB in translator_use_goto_tb. */ 3321 tcg_debug_assert(!(tcg_ctx->gen_tb->cflags & CF_NO_GOTO_TB)); 3322 /* We only support two chained exits. */ 3323 tcg_debug_assert(idx <= TB_EXIT_IDXMAX); 3324 #ifdef CONFIG_DEBUG_TCG 3325 /* Verify that we haven't seen this numbered exit before. */ 3326 tcg_debug_assert((tcg_ctx->goto_tb_issue_mask & (1 << idx)) == 0); 3327 tcg_ctx->goto_tb_issue_mask |= 1 << idx; 3328 #endif 3329 plugin_gen_disable_mem_helpers(); 3330 tcg_gen_op1i(INDEX_op_goto_tb, idx); 3331 } 3332 3333 void tcg_gen_lookup_and_goto_ptr(void) 3334 { 3335 TCGv_ptr ptr; 3336 3337 if (tcg_ctx->gen_tb->cflags & CF_NO_GOTO_PTR) { 3338 tcg_gen_exit_tb(NULL, 0); 3339 return; 3340 } 3341 3342 plugin_gen_disable_mem_helpers(); 3343 ptr = tcg_temp_ebb_new_ptr(); 3344 gen_helper_lookup_tb_ptr(ptr, tcg_env); 3345 tcg_gen_op1i(INDEX_op_goto_ptr, tcgv_ptr_arg(ptr)); 3346 tcg_temp_free_ptr(ptr); 3347 } 3348