1 /* 2 * Tiny Code Generator for QEMU 3 * 4 * Copyright (c) 2008 Fabrice Bellard 5 * 6 * Permission is hereby granted, free of charge, to any person obtaining a copy 7 * of this software and associated documentation files (the "Software"), to deal 8 * in the Software without restriction, including without limitation the rights 9 * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 10 * copies of the Software, and to permit persons to whom the Software is 11 * furnished to do so, subject to the following conditions: 12 * 13 * The above copyright notice and this permission notice shall be included in 14 * all copies or substantial portions of the Software. 15 * 16 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 17 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 18 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL 19 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 20 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 21 * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN 22 * THE SOFTWARE. 23 */ 24 25 #include "qemu/osdep.h" 26 #include "tcg/tcg.h" 27 #include "tcg/tcg-temp-internal.h" 28 #include "tcg/tcg-op-common.h" 29 #include "exec/translation-block.h" 30 #include "exec/plugin-gen.h" 31 #include "tcg-internal.h" 32 #include "tcg-has.h" 33 34 /* 35 * Encourage the compiler to tail-call to a function, rather than inlining. 36 * Minimizes code size across 99 bottles of beer on the wall. 37 */ 38 #define NI __attribute__((noinline)) 39 40 TCGOp * NI tcg_gen_op1(TCGOpcode opc, TCGType type, TCGArg a1) 41 { 42 TCGOp *op = tcg_emit_op(opc, 1); 43 TCGOP_TYPE(op) = type; 44 op->args[0] = a1; 45 return op; 46 } 47 48 TCGOp * NI tcg_gen_op2(TCGOpcode opc, TCGType type, TCGArg a1, TCGArg a2) 49 { 50 TCGOp *op = tcg_emit_op(opc, 2); 51 TCGOP_TYPE(op) = type; 52 op->args[0] = a1; 53 op->args[1] = a2; 54 return op; 55 } 56 57 TCGOp * NI tcg_gen_op3(TCGOpcode opc, TCGType type, TCGArg a1, 58 TCGArg a2, TCGArg a3) 59 { 60 TCGOp *op = tcg_emit_op(opc, 3); 61 TCGOP_TYPE(op) = type; 62 op->args[0] = a1; 63 op->args[1] = a2; 64 op->args[2] = a3; 65 return op; 66 } 67 68 TCGOp * NI tcg_gen_op4(TCGOpcode opc, TCGType type, TCGArg a1, TCGArg a2, 69 TCGArg a3, TCGArg a4) 70 { 71 TCGOp *op = tcg_emit_op(opc, 4); 72 TCGOP_TYPE(op) = type; 73 op->args[0] = a1; 74 op->args[1] = a2; 75 op->args[2] = a3; 76 op->args[3] = a4; 77 return op; 78 } 79 80 TCGOp * NI tcg_gen_op5(TCGOpcode opc, TCGType type, TCGArg a1, TCGArg a2, 81 TCGArg a3, TCGArg a4, TCGArg a5) 82 { 83 TCGOp *op = tcg_emit_op(opc, 5); 84 TCGOP_TYPE(op) = type; 85 op->args[0] = a1; 86 op->args[1] = a2; 87 op->args[2] = a3; 88 op->args[3] = a4; 89 op->args[4] = a5; 90 return op; 91 } 92 93 TCGOp * NI tcg_gen_op6(TCGOpcode opc, TCGType type, TCGArg a1, TCGArg a2, 94 TCGArg a3, TCGArg a4, TCGArg a5, TCGArg a6) 95 { 96 TCGOp *op = tcg_emit_op(opc, 6); 97 TCGOP_TYPE(op) = type; 98 op->args[0] = a1; 99 op->args[1] = a2; 100 op->args[2] = a3; 101 op->args[3] = a4; 102 op->args[4] = a5; 103 op->args[5] = a6; 104 return op; 105 } 106 107 /* 108 * With CONFIG_DEBUG_TCG, tcgv_*_tmp via tcgv_*_arg, is an out-of-line 109 * assertion check. Force tail calls to avoid too much code expansion. 110 */ 111 #ifdef CONFIG_DEBUG_TCG 112 # define DNI NI 113 #else 114 # define DNI 115 #endif 116 117 static void DNI tcg_gen_op1_i32(TCGOpcode opc, TCGType type, TCGv_i32 a1) 118 { 119 tcg_gen_op1(opc, type, tcgv_i32_arg(a1)); 120 } 121 122 static void DNI tcg_gen_op1_i64(TCGOpcode opc, TCGType type, TCGv_i64 a1) 123 { 124 tcg_gen_op1(opc, type, tcgv_i64_arg(a1)); 125 } 126 127 static TCGOp * DNI tcg_gen_op1i(TCGOpcode opc, TCGType type, TCGArg a1) 128 { 129 return tcg_gen_op1(opc, type, a1); 130 } 131 132 static void DNI tcg_gen_op2_i32(TCGOpcode opc, TCGv_i32 a1, TCGv_i32 a2) 133 { 134 tcg_gen_op2(opc, TCG_TYPE_I32, tcgv_i32_arg(a1), tcgv_i32_arg(a2)); 135 } 136 137 static void DNI tcg_gen_op2_i64(TCGOpcode opc, TCGv_i64 a1, TCGv_i64 a2) 138 { 139 tcg_gen_op2(opc, TCG_TYPE_I64, tcgv_i64_arg(a1), tcgv_i64_arg(a2)); 140 } 141 142 static void DNI tcg_gen_op3_i32(TCGOpcode opc, TCGv_i32 a1, 143 TCGv_i32 a2, TCGv_i32 a3) 144 { 145 tcg_gen_op3(opc, TCG_TYPE_I32, tcgv_i32_arg(a1), 146 tcgv_i32_arg(a2), tcgv_i32_arg(a3)); 147 } 148 149 static void DNI tcg_gen_op3_i64(TCGOpcode opc, TCGv_i64 a1, 150 TCGv_i64 a2, TCGv_i64 a3) 151 { 152 tcg_gen_op3(opc, TCG_TYPE_I64, tcgv_i64_arg(a1), 153 tcgv_i64_arg(a2), tcgv_i64_arg(a3)); 154 } 155 156 static void DNI tcg_gen_op3i_i32(TCGOpcode opc, TCGv_i32 a1, 157 TCGv_i32 a2, TCGArg a3) 158 { 159 tcg_gen_op3(opc, TCG_TYPE_I32, tcgv_i32_arg(a1), tcgv_i32_arg(a2), a3); 160 } 161 162 static void DNI tcg_gen_op3i_i64(TCGOpcode opc, TCGv_i64 a1, 163 TCGv_i64 a2, TCGArg a3) 164 { 165 tcg_gen_op3(opc, TCG_TYPE_I64, tcgv_i64_arg(a1), tcgv_i64_arg(a2), a3); 166 } 167 168 static void DNI tcg_gen_ldst_op_i32(TCGOpcode opc, TCGv_i32 val, 169 TCGv_ptr base, TCGArg offset) 170 { 171 tcg_gen_op3(opc, TCG_TYPE_I32, tcgv_i32_arg(val), 172 tcgv_ptr_arg(base), offset); 173 } 174 175 static void DNI tcg_gen_ldst_op_i64(TCGOpcode opc, TCGv_i64 val, 176 TCGv_ptr base, TCGArg offset) 177 { 178 tcg_gen_op3(opc, TCG_TYPE_I64, tcgv_i64_arg(val), 179 tcgv_ptr_arg(base), offset); 180 } 181 182 static void DNI tcg_gen_op4_i32(TCGOpcode opc, TCGv_i32 a1, TCGv_i32 a2, 183 TCGv_i32 a3, TCGv_i32 a4) 184 { 185 tcg_gen_op4(opc, TCG_TYPE_I32, tcgv_i32_arg(a1), tcgv_i32_arg(a2), 186 tcgv_i32_arg(a3), tcgv_i32_arg(a4)); 187 } 188 189 static void DNI tcg_gen_op4_i64(TCGOpcode opc, TCGv_i64 a1, TCGv_i64 a2, 190 TCGv_i64 a3, TCGv_i64 a4) 191 { 192 tcg_gen_op4(opc, TCG_TYPE_I64, tcgv_i64_arg(a1), tcgv_i64_arg(a2), 193 tcgv_i64_arg(a3), tcgv_i64_arg(a4)); 194 } 195 196 static void DNI tcg_gen_op4i_i32(TCGOpcode opc, TCGv_i32 a1, TCGv_i32 a2, 197 TCGv_i32 a3, TCGArg a4) 198 { 199 tcg_gen_op4(opc, TCG_TYPE_I32, tcgv_i32_arg(a1), tcgv_i32_arg(a2), 200 tcgv_i32_arg(a3), a4); 201 } 202 203 static void DNI tcg_gen_op4i_i64(TCGOpcode opc, TCGv_i64 a1, TCGv_i64 a2, 204 TCGv_i64 a3, TCGArg a4) 205 { 206 tcg_gen_op4(opc, TCG_TYPE_I64, tcgv_i64_arg(a1), tcgv_i64_arg(a2), 207 tcgv_i64_arg(a3), a4); 208 } 209 210 static TCGOp * DNI tcg_gen_op4ii_i32(TCGOpcode opc, TCGv_i32 a1, TCGv_i32 a2, 211 TCGArg a3, TCGArg a4) 212 { 213 return tcg_gen_op4(opc, TCG_TYPE_I32, 214 tcgv_i32_arg(a1), tcgv_i32_arg(a2), a3, a4); 215 } 216 217 static TCGOp * DNI tcg_gen_op4ii_i64(TCGOpcode opc, TCGv_i64 a1, TCGv_i64 a2, 218 TCGArg a3, TCGArg a4) 219 { 220 return tcg_gen_op4(opc, TCG_TYPE_I64, 221 tcgv_i64_arg(a1), tcgv_i64_arg(a2), a3, a4); 222 } 223 224 static void DNI tcg_gen_op5_i32(TCGOpcode opc, TCGv_i32 a1, TCGv_i32 a2, 225 TCGv_i32 a3, TCGv_i32 a4, TCGv_i32 a5) 226 { 227 tcg_gen_op5(opc, TCG_TYPE_I32, tcgv_i32_arg(a1), tcgv_i32_arg(a2), 228 tcgv_i32_arg(a3), tcgv_i32_arg(a4), tcgv_i32_arg(a5)); 229 } 230 231 static void DNI tcg_gen_op5_i64(TCGOpcode opc, TCGv_i64 a1, TCGv_i64 a2, 232 TCGv_i64 a3, TCGv_i64 a4, TCGv_i64 a5) 233 { 234 tcg_gen_op5(opc, TCG_TYPE_I64, tcgv_i64_arg(a1), tcgv_i64_arg(a2), 235 tcgv_i64_arg(a3), tcgv_i64_arg(a4), tcgv_i64_arg(a5)); 236 } 237 238 static void DNI tcg_gen_op5ii_i32(TCGOpcode opc, TCGv_i32 a1, TCGv_i32 a2, 239 TCGv_i32 a3, TCGArg a4, TCGArg a5) 240 { 241 tcg_gen_op5(opc, TCG_TYPE_I32, tcgv_i32_arg(a1), tcgv_i32_arg(a2), 242 tcgv_i32_arg(a3), a4, a5); 243 } 244 245 static void DNI tcg_gen_op5ii_i64(TCGOpcode opc, TCGv_i64 a1, TCGv_i64 a2, 246 TCGv_i64 a3, TCGArg a4, TCGArg a5) 247 { 248 tcg_gen_op5(opc, TCG_TYPE_I64, tcgv_i64_arg(a1), tcgv_i64_arg(a2), 249 tcgv_i64_arg(a3), a4, a5); 250 } 251 252 static void DNI tcg_gen_op6_i32(TCGOpcode opc, TCGv_i32 a1, TCGv_i32 a2, 253 TCGv_i32 a3, TCGv_i32 a4, 254 TCGv_i32 a5, TCGv_i32 a6) 255 { 256 tcg_gen_op6(opc, TCG_TYPE_I32, tcgv_i32_arg(a1), tcgv_i32_arg(a2), 257 tcgv_i32_arg(a3), tcgv_i32_arg(a4), tcgv_i32_arg(a5), 258 tcgv_i32_arg(a6)); 259 } 260 261 static void DNI tcg_gen_op6_i64(TCGOpcode opc, TCGv_i64 a1, TCGv_i64 a2, 262 TCGv_i64 a3, TCGv_i64 a4, 263 TCGv_i64 a5, TCGv_i64 a6) 264 { 265 tcg_gen_op6(opc, TCG_TYPE_I64, tcgv_i64_arg(a1), tcgv_i64_arg(a2), 266 tcgv_i64_arg(a3), tcgv_i64_arg(a4), tcgv_i64_arg(a5), 267 tcgv_i64_arg(a6)); 268 } 269 270 static void DNI tcg_gen_op6i_i32(TCGOpcode opc, TCGv_i32 a1, TCGv_i32 a2, 271 TCGv_i32 a3, TCGv_i32 a4, 272 TCGv_i32 a5, TCGArg a6) 273 { 274 tcg_gen_op6(opc, TCG_TYPE_I32, tcgv_i32_arg(a1), tcgv_i32_arg(a2), 275 tcgv_i32_arg(a3), tcgv_i32_arg(a4), tcgv_i32_arg(a5), a6); 276 } 277 278 static void DNI tcg_gen_op6i_i64(TCGOpcode opc, TCGv_i64 a1, TCGv_i64 a2, 279 TCGv_i64 a3, TCGv_i64 a4, 280 TCGv_i64 a5, TCGArg a6) 281 { 282 tcg_gen_op6(opc, TCG_TYPE_I64, tcgv_i64_arg(a1), tcgv_i64_arg(a2), 283 tcgv_i64_arg(a3), tcgv_i64_arg(a4), tcgv_i64_arg(a5), a6); 284 } 285 286 static TCGOp * DNI tcg_gen_op6ii_i32(TCGOpcode opc, TCGv_i32 a1, TCGv_i32 a2, 287 TCGv_i32 a3, TCGv_i32 a4, 288 TCGArg a5, TCGArg a6) 289 { 290 return tcg_gen_op6(opc, TCG_TYPE_I32, tcgv_i32_arg(a1), tcgv_i32_arg(a2), 291 tcgv_i32_arg(a3), tcgv_i32_arg(a4), a5, a6); 292 } 293 294 /* Generic ops. */ 295 296 void gen_set_label(TCGLabel *l) 297 { 298 l->present = 1; 299 tcg_gen_op1(INDEX_op_set_label, 0, label_arg(l)); 300 } 301 302 static void add_as_label_use(TCGLabel *l, TCGOp *op) 303 { 304 TCGLabelUse *u = tcg_malloc(sizeof(TCGLabelUse)); 305 306 u->op = op; 307 QSIMPLEQ_INSERT_TAIL(&l->branches, u, next); 308 } 309 310 void tcg_gen_br(TCGLabel *l) 311 { 312 add_as_label_use(l, tcg_gen_op1(INDEX_op_br, 0, label_arg(l))); 313 } 314 315 void tcg_gen_mb(TCGBar mb_type) 316 { 317 #ifdef CONFIG_USER_ONLY 318 bool parallel = tcg_ctx->gen_tb->cflags & CF_PARALLEL; 319 #else 320 /* 321 * It is tempting to elide the barrier in a uniprocessor context. 322 * However, even with a single cpu we have i/o threads running in 323 * parallel, and lack of memory order can result in e.g. virtio 324 * queue entries being read incorrectly. 325 */ 326 bool parallel = true; 327 #endif 328 329 if (parallel) { 330 tcg_gen_op1(INDEX_op_mb, 0, mb_type); 331 } 332 } 333 334 void tcg_gen_plugin_cb(unsigned from) 335 { 336 tcg_gen_op1(INDEX_op_plugin_cb, 0, from); 337 } 338 339 void tcg_gen_plugin_mem_cb(TCGv_i64 addr, unsigned meminfo) 340 { 341 tcg_gen_op2(INDEX_op_plugin_mem_cb, 0, tcgv_i64_arg(addr), meminfo); 342 } 343 344 /* 32 bit ops */ 345 346 void tcg_gen_discard_i32(TCGv_i32 arg) 347 { 348 tcg_gen_op1_i32(INDEX_op_discard, TCG_TYPE_I32, arg); 349 } 350 351 void tcg_gen_mov_i32(TCGv_i32 ret, TCGv_i32 arg) 352 { 353 if (ret != arg) { 354 tcg_gen_op2_i32(INDEX_op_mov, ret, arg); 355 } 356 } 357 358 void tcg_gen_movi_i32(TCGv_i32 ret, int32_t arg) 359 { 360 tcg_gen_mov_i32(ret, tcg_constant_i32(arg)); 361 } 362 363 void tcg_gen_add_i32(TCGv_i32 ret, TCGv_i32 arg1, TCGv_i32 arg2) 364 { 365 tcg_gen_op3_i32(INDEX_op_add, ret, arg1, arg2); 366 } 367 368 void tcg_gen_addi_i32(TCGv_i32 ret, TCGv_i32 arg1, int32_t arg2) 369 { 370 /* some cases can be optimized here */ 371 if (arg2 == 0) { 372 tcg_gen_mov_i32(ret, arg1); 373 } else { 374 tcg_gen_add_i32(ret, arg1, tcg_constant_i32(arg2)); 375 } 376 } 377 378 void tcg_gen_sub_i32(TCGv_i32 ret, TCGv_i32 arg1, TCGv_i32 arg2) 379 { 380 tcg_gen_op3_i32(INDEX_op_sub, ret, arg1, arg2); 381 } 382 383 void tcg_gen_subfi_i32(TCGv_i32 ret, int32_t arg1, TCGv_i32 arg2) 384 { 385 if (arg1 == 0) { 386 tcg_gen_neg_i32(ret, arg2); 387 } else { 388 tcg_gen_sub_i32(ret, tcg_constant_i32(arg1), arg2); 389 } 390 } 391 392 void tcg_gen_subi_i32(TCGv_i32 ret, TCGv_i32 arg1, int32_t arg2) 393 { 394 tcg_gen_addi_i32(ret, arg1, -arg2); 395 } 396 397 void tcg_gen_neg_i32(TCGv_i32 ret, TCGv_i32 arg) 398 { 399 tcg_gen_op2_i32(INDEX_op_neg, ret, arg); 400 } 401 402 void tcg_gen_and_i32(TCGv_i32 ret, TCGv_i32 arg1, TCGv_i32 arg2) 403 { 404 tcg_gen_op3_i32(INDEX_op_and, ret, arg1, arg2); 405 } 406 407 void tcg_gen_andi_i32(TCGv_i32 ret, TCGv_i32 arg1, int32_t arg2) 408 { 409 /* Some cases can be optimized here. */ 410 switch (arg2) { 411 case 0: 412 tcg_gen_movi_i32(ret, 0); 413 return; 414 case -1: 415 tcg_gen_mov_i32(ret, arg1); 416 return; 417 default: 418 /* 419 * Canonicalize on extract, if valid. This aids x86 with its 420 * 2 operand MOVZBL and 2 operand AND, selecting the TCGOpcode 421 * which does not require matching operands. Other backends can 422 * trivially expand the extract to AND during code generation. 423 */ 424 if (!(arg2 & (arg2 + 1))) { 425 unsigned len = ctz32(~arg2); 426 if (TCG_TARGET_extract_valid(TCG_TYPE_I32, 0, len)) { 427 tcg_gen_extract_i32(ret, arg1, 0, len); 428 return; 429 } 430 } 431 break; 432 } 433 434 tcg_gen_and_i32(ret, arg1, tcg_constant_i32(arg2)); 435 } 436 437 void tcg_gen_or_i32(TCGv_i32 ret, TCGv_i32 arg1, TCGv_i32 arg2) 438 { 439 tcg_gen_op3_i32(INDEX_op_or, ret, arg1, arg2); 440 } 441 442 void tcg_gen_ori_i32(TCGv_i32 ret, TCGv_i32 arg1, int32_t arg2) 443 { 444 /* Some cases can be optimized here. */ 445 if (arg2 == -1) { 446 tcg_gen_movi_i32(ret, -1); 447 } else if (arg2 == 0) { 448 tcg_gen_mov_i32(ret, arg1); 449 } else { 450 tcg_gen_or_i32(ret, arg1, tcg_constant_i32(arg2)); 451 } 452 } 453 454 void tcg_gen_xor_i32(TCGv_i32 ret, TCGv_i32 arg1, TCGv_i32 arg2) 455 { 456 tcg_gen_op3_i32(INDEX_op_xor, ret, arg1, arg2); 457 } 458 459 void tcg_gen_xori_i32(TCGv_i32 ret, TCGv_i32 arg1, int32_t arg2) 460 { 461 /* Some cases can be optimized here. */ 462 if (arg2 == 0) { 463 tcg_gen_mov_i32(ret, arg1); 464 } else if (arg2 == -1 && 465 tcg_op_supported(INDEX_op_not, TCG_TYPE_I32, 0)) { 466 /* Don't recurse with tcg_gen_not_i32. */ 467 tcg_gen_op2_i32(INDEX_op_not, ret, arg1); 468 } else { 469 tcg_gen_xor_i32(ret, arg1, tcg_constant_i32(arg2)); 470 } 471 } 472 473 void tcg_gen_not_i32(TCGv_i32 ret, TCGv_i32 arg) 474 { 475 if (tcg_op_supported(INDEX_op_not, TCG_TYPE_I32, 0)) { 476 tcg_gen_op2_i32(INDEX_op_not, ret, arg); 477 } else { 478 tcg_gen_xori_i32(ret, arg, -1); 479 } 480 } 481 482 void tcg_gen_shl_i32(TCGv_i32 ret, TCGv_i32 arg1, TCGv_i32 arg2) 483 { 484 tcg_gen_op3_i32(INDEX_op_shl, ret, arg1, arg2); 485 } 486 487 void tcg_gen_shli_i32(TCGv_i32 ret, TCGv_i32 arg1, int32_t arg2) 488 { 489 tcg_debug_assert(arg2 >= 0 && arg2 < 32); 490 if (arg2 == 0) { 491 tcg_gen_mov_i32(ret, arg1); 492 } else { 493 tcg_gen_shl_i32(ret, arg1, tcg_constant_i32(arg2)); 494 } 495 } 496 497 void tcg_gen_shr_i32(TCGv_i32 ret, TCGv_i32 arg1, TCGv_i32 arg2) 498 { 499 tcg_gen_op3_i32(INDEX_op_shr, ret, arg1, arg2); 500 } 501 502 void tcg_gen_shri_i32(TCGv_i32 ret, TCGv_i32 arg1, int32_t arg2) 503 { 504 tcg_debug_assert(arg2 >= 0 && arg2 < 32); 505 if (arg2 == 0) { 506 tcg_gen_mov_i32(ret, arg1); 507 } else { 508 tcg_gen_shr_i32(ret, arg1, tcg_constant_i32(arg2)); 509 } 510 } 511 512 void tcg_gen_sar_i32(TCGv_i32 ret, TCGv_i32 arg1, TCGv_i32 arg2) 513 { 514 tcg_gen_op3_i32(INDEX_op_sar, ret, arg1, arg2); 515 } 516 517 void tcg_gen_sari_i32(TCGv_i32 ret, TCGv_i32 arg1, int32_t arg2) 518 { 519 tcg_debug_assert(arg2 >= 0 && arg2 < 32); 520 if (arg2 == 0) { 521 tcg_gen_mov_i32(ret, arg1); 522 } else { 523 tcg_gen_sar_i32(ret, arg1, tcg_constant_i32(arg2)); 524 } 525 } 526 527 void tcg_gen_brcond_i32(TCGCond cond, TCGv_i32 arg1, TCGv_i32 arg2, TCGLabel *l) 528 { 529 if (cond == TCG_COND_ALWAYS) { 530 tcg_gen_br(l); 531 } else if (cond != TCG_COND_NEVER) { 532 TCGOp *op = tcg_gen_op4ii_i32(INDEX_op_brcond_i32, 533 arg1, arg2, cond, label_arg(l)); 534 add_as_label_use(l, op); 535 } 536 } 537 538 void tcg_gen_brcondi_i32(TCGCond cond, TCGv_i32 arg1, int32_t arg2, TCGLabel *l) 539 { 540 if (cond == TCG_COND_ALWAYS) { 541 tcg_gen_br(l); 542 } else if (cond != TCG_COND_NEVER) { 543 tcg_gen_brcond_i32(cond, arg1, tcg_constant_i32(arg2), l); 544 } 545 } 546 547 void tcg_gen_setcond_i32(TCGCond cond, TCGv_i32 ret, 548 TCGv_i32 arg1, TCGv_i32 arg2) 549 { 550 if (cond == TCG_COND_ALWAYS) { 551 tcg_gen_movi_i32(ret, 1); 552 } else if (cond == TCG_COND_NEVER) { 553 tcg_gen_movi_i32(ret, 0); 554 } else { 555 tcg_gen_op4i_i32(INDEX_op_setcond_i32, ret, arg1, arg2, cond); 556 } 557 } 558 559 void tcg_gen_setcondi_i32(TCGCond cond, TCGv_i32 ret, 560 TCGv_i32 arg1, int32_t arg2) 561 { 562 tcg_gen_setcond_i32(cond, ret, arg1, tcg_constant_i32(arg2)); 563 } 564 565 void tcg_gen_negsetcond_i32(TCGCond cond, TCGv_i32 ret, 566 TCGv_i32 arg1, TCGv_i32 arg2) 567 { 568 if (cond == TCG_COND_ALWAYS) { 569 tcg_gen_movi_i32(ret, -1); 570 } else if (cond == TCG_COND_NEVER) { 571 tcg_gen_movi_i32(ret, 0); 572 } else if (TCG_TARGET_HAS_negsetcond_i32) { 573 tcg_gen_op4i_i32(INDEX_op_negsetcond_i32, ret, arg1, arg2, cond); 574 } else { 575 tcg_gen_setcond_i32(cond, ret, arg1, arg2); 576 tcg_gen_neg_i32(ret, ret); 577 } 578 } 579 580 void tcg_gen_negsetcondi_i32(TCGCond cond, TCGv_i32 ret, 581 TCGv_i32 arg1, int32_t arg2) 582 { 583 tcg_gen_negsetcond_i32(cond, ret, arg1, tcg_constant_i32(arg2)); 584 } 585 586 void tcg_gen_mul_i32(TCGv_i32 ret, TCGv_i32 arg1, TCGv_i32 arg2) 587 { 588 tcg_gen_op3_i32(INDEX_op_mul, ret, arg1, arg2); 589 } 590 591 void tcg_gen_muli_i32(TCGv_i32 ret, TCGv_i32 arg1, int32_t arg2) 592 { 593 if (arg2 == 0) { 594 tcg_gen_movi_i32(ret, 0); 595 } else if (is_power_of_2(arg2)) { 596 tcg_gen_shli_i32(ret, arg1, ctz32(arg2)); 597 } else { 598 tcg_gen_mul_i32(ret, arg1, tcg_constant_i32(arg2)); 599 } 600 } 601 602 void tcg_gen_div_i32(TCGv_i32 ret, TCGv_i32 arg1, TCGv_i32 arg2) 603 { 604 if (tcg_op_supported(INDEX_op_divs, TCG_TYPE_I32, 0)) { 605 tcg_gen_op3_i32(INDEX_op_divs, ret, arg1, arg2); 606 } else if (tcg_op_supported(INDEX_op_divs2, TCG_TYPE_I32, 0)) { 607 TCGv_i32 t0 = tcg_temp_ebb_new_i32(); 608 tcg_gen_sari_i32(t0, arg1, 31); 609 tcg_gen_op5_i32(INDEX_op_divs2, ret, t0, arg1, t0, arg2); 610 tcg_temp_free_i32(t0); 611 } else { 612 gen_helper_div_i32(ret, arg1, arg2); 613 } 614 } 615 616 void tcg_gen_rem_i32(TCGv_i32 ret, TCGv_i32 arg1, TCGv_i32 arg2) 617 { 618 if (tcg_op_supported(INDEX_op_rems, TCG_TYPE_I32, 0)) { 619 tcg_gen_op3_i32(INDEX_op_rems, ret, arg1, arg2); 620 } else if (tcg_op_supported(INDEX_op_divs, TCG_TYPE_I32, 0)) { 621 TCGv_i32 t0 = tcg_temp_ebb_new_i32(); 622 tcg_gen_op3_i32(INDEX_op_divs, t0, arg1, arg2); 623 tcg_gen_mul_i32(t0, t0, arg2); 624 tcg_gen_sub_i32(ret, arg1, t0); 625 tcg_temp_free_i32(t0); 626 } else if (tcg_op_supported(INDEX_op_divs2, TCG_TYPE_I32, 0)) { 627 TCGv_i32 t0 = tcg_temp_ebb_new_i32(); 628 tcg_gen_sari_i32(t0, arg1, 31); 629 tcg_gen_op5_i32(INDEX_op_divs2, t0, ret, arg1, t0, arg2); 630 tcg_temp_free_i32(t0); 631 } else { 632 gen_helper_rem_i32(ret, arg1, arg2); 633 } 634 } 635 636 void tcg_gen_divu_i32(TCGv_i32 ret, TCGv_i32 arg1, TCGv_i32 arg2) 637 { 638 if (tcg_op_supported(INDEX_op_divu, TCG_TYPE_I32, 0)) { 639 tcg_gen_op3_i32(INDEX_op_divu, ret, arg1, arg2); 640 } else if (tcg_op_supported(INDEX_op_divu2, TCG_TYPE_I32, 0)) { 641 TCGv_i32 t0 = tcg_temp_ebb_new_i32(); 642 TCGv_i32 zero = tcg_constant_i32(0); 643 tcg_gen_op5_i32(INDEX_op_divu2, ret, t0, arg1, zero, arg2); 644 tcg_temp_free_i32(t0); 645 } else { 646 gen_helper_divu_i32(ret, arg1, arg2); 647 } 648 } 649 650 void tcg_gen_remu_i32(TCGv_i32 ret, TCGv_i32 arg1, TCGv_i32 arg2) 651 { 652 if (tcg_op_supported(INDEX_op_remu, TCG_TYPE_I32, 0)) { 653 tcg_gen_op3_i32(INDEX_op_remu, ret, arg1, arg2); 654 } else if (tcg_op_supported(INDEX_op_divu, TCG_TYPE_I32, 0)) { 655 TCGv_i32 t0 = tcg_temp_ebb_new_i32(); 656 tcg_gen_op3_i32(INDEX_op_divu, t0, arg1, arg2); 657 tcg_gen_mul_i32(t0, t0, arg2); 658 tcg_gen_sub_i32(ret, arg1, t0); 659 tcg_temp_free_i32(t0); 660 } else if (tcg_op_supported(INDEX_op_divu2, TCG_TYPE_I32, 0)) { 661 TCGv_i32 t0 = tcg_temp_ebb_new_i32(); 662 TCGv_i32 zero = tcg_constant_i32(0); 663 tcg_gen_op5_i32(INDEX_op_divu2, t0, ret, arg1, zero, arg2); 664 tcg_temp_free_i32(t0); 665 } else { 666 gen_helper_remu_i32(ret, arg1, arg2); 667 } 668 } 669 670 void tcg_gen_andc_i32(TCGv_i32 ret, TCGv_i32 arg1, TCGv_i32 arg2) 671 { 672 if (tcg_op_supported(INDEX_op_andc, TCG_TYPE_I32, 0)) { 673 tcg_gen_op3_i32(INDEX_op_andc, ret, arg1, arg2); 674 } else { 675 TCGv_i32 t0 = tcg_temp_ebb_new_i32(); 676 tcg_gen_not_i32(t0, arg2); 677 tcg_gen_and_i32(ret, arg1, t0); 678 tcg_temp_free_i32(t0); 679 } 680 } 681 682 void tcg_gen_eqv_i32(TCGv_i32 ret, TCGv_i32 arg1, TCGv_i32 arg2) 683 { 684 if (tcg_op_supported(INDEX_op_eqv, TCG_TYPE_I32, 0)) { 685 tcg_gen_op3_i32(INDEX_op_eqv, ret, arg1, arg2); 686 } else { 687 tcg_gen_xor_i32(ret, arg1, arg2); 688 tcg_gen_not_i32(ret, ret); 689 } 690 } 691 692 void tcg_gen_nand_i32(TCGv_i32 ret, TCGv_i32 arg1, TCGv_i32 arg2) 693 { 694 if (tcg_op_supported(INDEX_op_nand, TCG_TYPE_I32, 0)) { 695 tcg_gen_op3_i32(INDEX_op_nand, ret, arg1, arg2); 696 } else { 697 tcg_gen_and_i32(ret, arg1, arg2); 698 tcg_gen_not_i32(ret, ret); 699 } 700 } 701 702 void tcg_gen_nor_i32(TCGv_i32 ret, TCGv_i32 arg1, TCGv_i32 arg2) 703 { 704 if (tcg_op_supported(INDEX_op_nor, TCG_TYPE_I32, 0)) { 705 tcg_gen_op3_i32(INDEX_op_nor, ret, arg1, arg2); 706 } else { 707 tcg_gen_or_i32(ret, arg1, arg2); 708 tcg_gen_not_i32(ret, ret); 709 } 710 } 711 712 void tcg_gen_orc_i32(TCGv_i32 ret, TCGv_i32 arg1, TCGv_i32 arg2) 713 { 714 if (tcg_op_supported(INDEX_op_orc, TCG_TYPE_I32, 0)) { 715 tcg_gen_op3_i32(INDEX_op_orc, ret, arg1, arg2); 716 } else { 717 TCGv_i32 t0 = tcg_temp_ebb_new_i32(); 718 tcg_gen_not_i32(t0, arg2); 719 tcg_gen_or_i32(ret, arg1, t0); 720 tcg_temp_free_i32(t0); 721 } 722 } 723 724 void tcg_gen_clz_i32(TCGv_i32 ret, TCGv_i32 arg1, TCGv_i32 arg2) 725 { 726 if (TCG_TARGET_HAS_clz_i32) { 727 tcg_gen_op3_i32(INDEX_op_clz_i32, ret, arg1, arg2); 728 } else if (TCG_TARGET_HAS_clz_i64) { 729 TCGv_i64 t1 = tcg_temp_ebb_new_i64(); 730 TCGv_i64 t2 = tcg_temp_ebb_new_i64(); 731 tcg_gen_extu_i32_i64(t1, arg1); 732 tcg_gen_extu_i32_i64(t2, arg2); 733 tcg_gen_addi_i64(t2, t2, 32); 734 tcg_gen_clz_i64(t1, t1, t2); 735 tcg_gen_extrl_i64_i32(ret, t1); 736 tcg_temp_free_i64(t1); 737 tcg_temp_free_i64(t2); 738 tcg_gen_subi_i32(ret, ret, 32); 739 } else { 740 gen_helper_clz_i32(ret, arg1, arg2); 741 } 742 } 743 744 void tcg_gen_clzi_i32(TCGv_i32 ret, TCGv_i32 arg1, uint32_t arg2) 745 { 746 tcg_gen_clz_i32(ret, arg1, tcg_constant_i32(arg2)); 747 } 748 749 void tcg_gen_ctz_i32(TCGv_i32 ret, TCGv_i32 arg1, TCGv_i32 arg2) 750 { 751 if (TCG_TARGET_HAS_ctz_i32) { 752 tcg_gen_op3_i32(INDEX_op_ctz_i32, ret, arg1, arg2); 753 } else if (TCG_TARGET_HAS_ctz_i64) { 754 TCGv_i64 t1 = tcg_temp_ebb_new_i64(); 755 TCGv_i64 t2 = tcg_temp_ebb_new_i64(); 756 tcg_gen_extu_i32_i64(t1, arg1); 757 tcg_gen_extu_i32_i64(t2, arg2); 758 tcg_gen_ctz_i64(t1, t1, t2); 759 tcg_gen_extrl_i64_i32(ret, t1); 760 tcg_temp_free_i64(t1); 761 tcg_temp_free_i64(t2); 762 } else if (TCG_TARGET_HAS_ctpop_i32 763 || TCG_TARGET_HAS_ctpop_i64 764 || TCG_TARGET_HAS_clz_i32 765 || TCG_TARGET_HAS_clz_i64) { 766 TCGv_i32 z, t = tcg_temp_ebb_new_i32(); 767 768 if (TCG_TARGET_HAS_ctpop_i32 || TCG_TARGET_HAS_ctpop_i64) { 769 tcg_gen_subi_i32(t, arg1, 1); 770 tcg_gen_andc_i32(t, t, arg1); 771 tcg_gen_ctpop_i32(t, t); 772 } else { 773 /* Since all non-x86 hosts have clz(0) == 32, don't fight it. */ 774 tcg_gen_neg_i32(t, arg1); 775 tcg_gen_and_i32(t, t, arg1); 776 tcg_gen_clzi_i32(t, t, 32); 777 tcg_gen_xori_i32(t, t, 31); 778 } 779 z = tcg_constant_i32(0); 780 tcg_gen_movcond_i32(TCG_COND_EQ, ret, arg1, z, arg2, t); 781 tcg_temp_free_i32(t); 782 } else { 783 gen_helper_ctz_i32(ret, arg1, arg2); 784 } 785 } 786 787 void tcg_gen_ctzi_i32(TCGv_i32 ret, TCGv_i32 arg1, uint32_t arg2) 788 { 789 if (!TCG_TARGET_HAS_ctz_i32 && TCG_TARGET_HAS_ctpop_i32 && arg2 == 32) { 790 /* This equivalence has the advantage of not requiring a fixup. */ 791 TCGv_i32 t = tcg_temp_ebb_new_i32(); 792 tcg_gen_subi_i32(t, arg1, 1); 793 tcg_gen_andc_i32(t, t, arg1); 794 tcg_gen_ctpop_i32(ret, t); 795 tcg_temp_free_i32(t); 796 } else { 797 tcg_gen_ctz_i32(ret, arg1, tcg_constant_i32(arg2)); 798 } 799 } 800 801 void tcg_gen_clrsb_i32(TCGv_i32 ret, TCGv_i32 arg) 802 { 803 if (TCG_TARGET_HAS_clz_i32) { 804 TCGv_i32 t = tcg_temp_ebb_new_i32(); 805 tcg_gen_sari_i32(t, arg, 31); 806 tcg_gen_xor_i32(t, t, arg); 807 tcg_gen_clzi_i32(t, t, 32); 808 tcg_gen_subi_i32(ret, t, 1); 809 tcg_temp_free_i32(t); 810 } else { 811 gen_helper_clrsb_i32(ret, arg); 812 } 813 } 814 815 void tcg_gen_ctpop_i32(TCGv_i32 ret, TCGv_i32 arg1) 816 { 817 if (TCG_TARGET_HAS_ctpop_i32) { 818 tcg_gen_op2_i32(INDEX_op_ctpop_i32, ret, arg1); 819 } else if (TCG_TARGET_HAS_ctpop_i64) { 820 TCGv_i64 t = tcg_temp_ebb_new_i64(); 821 tcg_gen_extu_i32_i64(t, arg1); 822 tcg_gen_ctpop_i64(t, t); 823 tcg_gen_extrl_i64_i32(ret, t); 824 tcg_temp_free_i64(t); 825 } else { 826 gen_helper_ctpop_i32(ret, arg1); 827 } 828 } 829 830 void tcg_gen_rotl_i32(TCGv_i32 ret, TCGv_i32 arg1, TCGv_i32 arg2) 831 { 832 if (tcg_op_supported(INDEX_op_rotl, TCG_TYPE_I32, 0)) { 833 tcg_gen_op3_i32(INDEX_op_rotl, ret, arg1, arg2); 834 } else if (tcg_op_supported(INDEX_op_rotr, TCG_TYPE_I32, 0)) { 835 TCGv_i32 t0 = tcg_temp_ebb_new_i32(); 836 tcg_gen_neg_i32(t0, arg2); 837 tcg_gen_op3_i32(INDEX_op_rotr, ret, arg1, t0); 838 tcg_temp_free_i32(t0); 839 } else { 840 TCGv_i32 t0 = tcg_temp_ebb_new_i32(); 841 TCGv_i32 t1 = tcg_temp_ebb_new_i32(); 842 tcg_gen_shl_i32(t0, arg1, arg2); 843 tcg_gen_neg_i32(t1, arg2); 844 tcg_gen_shr_i32(t1, arg1, t1); 845 tcg_gen_or_i32(ret, t0, t1); 846 tcg_temp_free_i32(t0); 847 tcg_temp_free_i32(t1); 848 } 849 } 850 851 void tcg_gen_rotli_i32(TCGv_i32 ret, TCGv_i32 arg1, int32_t arg2) 852 { 853 tcg_debug_assert(arg2 >= 0 && arg2 < 32); 854 /* some cases can be optimized here */ 855 if (arg2 == 0) { 856 tcg_gen_mov_i32(ret, arg1); 857 } else if (tcg_op_supported(INDEX_op_rotl, TCG_TYPE_I32, 0)) { 858 TCGv_i32 t0 = tcg_constant_i32(arg2); 859 tcg_gen_op3_i32(INDEX_op_rotl, ret, arg1, t0); 860 } else if (tcg_op_supported(INDEX_op_rotr, TCG_TYPE_I32, 0)) { 861 TCGv_i32 t0 = tcg_constant_i32(32 - arg2); 862 tcg_gen_op3_i32(INDEX_op_rotr, ret, arg1, t0); 863 } else { 864 TCGv_i32 t0 = tcg_temp_ebb_new_i32(); 865 TCGv_i32 t1 = tcg_temp_ebb_new_i32(); 866 tcg_gen_shli_i32(t0, arg1, arg2); 867 tcg_gen_shri_i32(t1, arg1, 32 - arg2); 868 tcg_gen_or_i32(ret, t0, t1); 869 tcg_temp_free_i32(t0); 870 tcg_temp_free_i32(t1); 871 } 872 } 873 874 void tcg_gen_rotr_i32(TCGv_i32 ret, TCGv_i32 arg1, TCGv_i32 arg2) 875 { 876 if (tcg_op_supported(INDEX_op_rotr, TCG_TYPE_I32, 0)) { 877 tcg_gen_op3_i32(INDEX_op_rotr, ret, arg1, arg2); 878 } else if (tcg_op_supported(INDEX_op_rotl, TCG_TYPE_I32, 0)) { 879 TCGv_i32 t0 = tcg_temp_ebb_new_i32(); 880 tcg_gen_neg_i32(t0, arg2); 881 tcg_gen_op3_i32(INDEX_op_rotl, ret, arg1, t0); 882 tcg_temp_free_i32(t0); 883 } else { 884 TCGv_i32 t0 = tcg_temp_ebb_new_i32(); 885 TCGv_i32 t1 = tcg_temp_ebb_new_i32(); 886 tcg_gen_shr_i32(t0, arg1, arg2); 887 tcg_gen_neg_i32(t1, arg2); 888 tcg_gen_shl_i32(t1, arg1, t1); 889 tcg_gen_or_i32(ret, t0, t1); 890 tcg_temp_free_i32(t0); 891 tcg_temp_free_i32(t1); 892 } 893 } 894 895 void tcg_gen_rotri_i32(TCGv_i32 ret, TCGv_i32 arg1, int32_t arg2) 896 { 897 tcg_debug_assert(arg2 >= 0 && arg2 < 32); 898 tcg_gen_rotli_i32(ret, arg1, -arg2 & 31); 899 } 900 901 void tcg_gen_deposit_i32(TCGv_i32 ret, TCGv_i32 arg1, TCGv_i32 arg2, 902 unsigned int ofs, unsigned int len) 903 { 904 uint32_t mask; 905 TCGv_i32 t1; 906 907 tcg_debug_assert(ofs < 32); 908 tcg_debug_assert(len > 0); 909 tcg_debug_assert(len <= 32); 910 tcg_debug_assert(ofs + len <= 32); 911 912 if (len == 32) { 913 tcg_gen_mov_i32(ret, arg2); 914 return; 915 } 916 if (TCG_TARGET_deposit_valid(TCG_TYPE_I32, ofs, len)) { 917 tcg_gen_op5ii_i32(INDEX_op_deposit_i32, ret, arg1, arg2, ofs, len); 918 return; 919 } 920 921 t1 = tcg_temp_ebb_new_i32(); 922 923 if (TCG_TARGET_HAS_extract2_i32) { 924 if (ofs + len == 32) { 925 tcg_gen_shli_i32(t1, arg1, len); 926 tcg_gen_extract2_i32(ret, t1, arg2, len); 927 goto done; 928 } 929 if (ofs == 0) { 930 tcg_gen_extract2_i32(ret, arg1, arg2, len); 931 tcg_gen_rotli_i32(ret, ret, len); 932 goto done; 933 } 934 } 935 936 mask = (1u << len) - 1; 937 if (ofs + len < 32) { 938 tcg_gen_andi_i32(t1, arg2, mask); 939 tcg_gen_shli_i32(t1, t1, ofs); 940 } else { 941 tcg_gen_shli_i32(t1, arg2, ofs); 942 } 943 tcg_gen_andi_i32(ret, arg1, ~(mask << ofs)); 944 tcg_gen_or_i32(ret, ret, t1); 945 done: 946 tcg_temp_free_i32(t1); 947 } 948 949 void tcg_gen_deposit_z_i32(TCGv_i32 ret, TCGv_i32 arg, 950 unsigned int ofs, unsigned int len) 951 { 952 tcg_debug_assert(ofs < 32); 953 tcg_debug_assert(len > 0); 954 tcg_debug_assert(len <= 32); 955 tcg_debug_assert(ofs + len <= 32); 956 957 if (ofs + len == 32) { 958 tcg_gen_shli_i32(ret, arg, ofs); 959 } else if (ofs == 0) { 960 tcg_gen_andi_i32(ret, arg, (1u << len) - 1); 961 } else if (TCG_TARGET_deposit_valid(TCG_TYPE_I32, ofs, len)) { 962 TCGv_i32 zero = tcg_constant_i32(0); 963 tcg_gen_op5ii_i32(INDEX_op_deposit_i32, ret, zero, arg, ofs, len); 964 } else { 965 /* 966 * To help two-operand hosts we prefer to zero-extend first, 967 * which allows ARG to stay live. 968 */ 969 if (TCG_TARGET_extract_valid(TCG_TYPE_I32, 0, len)) { 970 tcg_gen_extract_i32(ret, arg, 0, len); 971 tcg_gen_shli_i32(ret, ret, ofs); 972 return; 973 } 974 /* Otherwise prefer zero-extension over AND for code size. */ 975 if (TCG_TARGET_extract_valid(TCG_TYPE_I32, 0, ofs + len)) { 976 tcg_gen_shli_i32(ret, arg, ofs); 977 tcg_gen_extract_i32(ret, ret, 0, ofs + len); 978 return; 979 } 980 tcg_gen_andi_i32(ret, arg, (1u << len) - 1); 981 tcg_gen_shli_i32(ret, ret, ofs); 982 } 983 } 984 985 void tcg_gen_extract_i32(TCGv_i32 ret, TCGv_i32 arg, 986 unsigned int ofs, unsigned int len) 987 { 988 tcg_debug_assert(ofs < 32); 989 tcg_debug_assert(len > 0); 990 tcg_debug_assert(len <= 32); 991 tcg_debug_assert(ofs + len <= 32); 992 993 /* Canonicalize certain special cases, even if extract is supported. */ 994 if (ofs + len == 32) { 995 tcg_gen_shri_i32(ret, arg, 32 - len); 996 return; 997 } 998 999 if (TCG_TARGET_extract_valid(TCG_TYPE_I32, ofs, len)) { 1000 tcg_gen_op4ii_i32(INDEX_op_extract_i32, ret, arg, ofs, len); 1001 return; 1002 } 1003 if (ofs == 0) { 1004 tcg_gen_andi_i32(ret, arg, (1u << len) - 1); 1005 return; 1006 } 1007 1008 /* Assume that zero-extension, if available, is cheaper than a shift. */ 1009 if (TCG_TARGET_extract_valid(TCG_TYPE_I32, 0, ofs + len)) { 1010 tcg_gen_op4ii_i32(INDEX_op_extract_i32, ret, arg, 0, ofs + len); 1011 tcg_gen_shri_i32(ret, ret, ofs); 1012 return; 1013 } 1014 1015 /* ??? Ideally we'd know what values are available for immediate AND. 1016 Assume that 8 bits are available, plus the special case of 16, 1017 so that we get ext8u, ext16u. */ 1018 switch (len) { 1019 case 1 ... 8: case 16: 1020 tcg_gen_shri_i32(ret, arg, ofs); 1021 tcg_gen_andi_i32(ret, ret, (1u << len) - 1); 1022 break; 1023 default: 1024 tcg_gen_shli_i32(ret, arg, 32 - len - ofs); 1025 tcg_gen_shri_i32(ret, ret, 32 - len); 1026 break; 1027 } 1028 } 1029 1030 void tcg_gen_sextract_i32(TCGv_i32 ret, TCGv_i32 arg, 1031 unsigned int ofs, unsigned int len) 1032 { 1033 tcg_debug_assert(ofs < 32); 1034 tcg_debug_assert(len > 0); 1035 tcg_debug_assert(len <= 32); 1036 tcg_debug_assert(ofs + len <= 32); 1037 1038 /* Canonicalize certain special cases, even if extract is supported. */ 1039 if (ofs + len == 32) { 1040 tcg_gen_sari_i32(ret, arg, 32 - len); 1041 return; 1042 } 1043 1044 if (TCG_TARGET_sextract_valid(TCG_TYPE_I32, ofs, len)) { 1045 tcg_gen_op4ii_i32(INDEX_op_sextract_i32, ret, arg, ofs, len); 1046 return; 1047 } 1048 1049 /* Assume that sign-extension, if available, is cheaper than a shift. */ 1050 if (TCG_TARGET_sextract_valid(TCG_TYPE_I32, 0, ofs + len)) { 1051 tcg_gen_op4ii_i32(INDEX_op_sextract_i32, ret, arg, 0, ofs + len); 1052 tcg_gen_sari_i32(ret, ret, ofs); 1053 return; 1054 } 1055 if (TCG_TARGET_sextract_valid(TCG_TYPE_I32, 0, len)) { 1056 tcg_gen_shri_i32(ret, arg, ofs); 1057 tcg_gen_op4ii_i32(INDEX_op_sextract_i32, ret, ret, 0, len); 1058 return; 1059 } 1060 1061 tcg_gen_shli_i32(ret, arg, 32 - len - ofs); 1062 tcg_gen_sari_i32(ret, ret, 32 - len); 1063 } 1064 1065 /* 1066 * Extract 32-bits from a 64-bit input, ah:al, starting from ofs. 1067 * Unlike tcg_gen_extract_i32 above, len is fixed at 32. 1068 */ 1069 void tcg_gen_extract2_i32(TCGv_i32 ret, TCGv_i32 al, TCGv_i32 ah, 1070 unsigned int ofs) 1071 { 1072 tcg_debug_assert(ofs <= 32); 1073 if (ofs == 0) { 1074 tcg_gen_mov_i32(ret, al); 1075 } else if (ofs == 32) { 1076 tcg_gen_mov_i32(ret, ah); 1077 } else if (al == ah) { 1078 tcg_gen_rotri_i32(ret, al, ofs); 1079 } else if (TCG_TARGET_HAS_extract2_i32) { 1080 tcg_gen_op4i_i32(INDEX_op_extract2_i32, ret, al, ah, ofs); 1081 } else { 1082 TCGv_i32 t0 = tcg_temp_ebb_new_i32(); 1083 tcg_gen_shri_i32(t0, al, ofs); 1084 tcg_gen_deposit_i32(ret, t0, ah, 32 - ofs, ofs); 1085 tcg_temp_free_i32(t0); 1086 } 1087 } 1088 1089 void tcg_gen_movcond_i32(TCGCond cond, TCGv_i32 ret, TCGv_i32 c1, 1090 TCGv_i32 c2, TCGv_i32 v1, TCGv_i32 v2) 1091 { 1092 if (cond == TCG_COND_ALWAYS) { 1093 tcg_gen_mov_i32(ret, v1); 1094 } else if (cond == TCG_COND_NEVER) { 1095 tcg_gen_mov_i32(ret, v2); 1096 } else { 1097 tcg_gen_op6i_i32(INDEX_op_movcond_i32, ret, c1, c2, v1, v2, cond); 1098 } 1099 } 1100 1101 void tcg_gen_add2_i32(TCGv_i32 rl, TCGv_i32 rh, TCGv_i32 al, 1102 TCGv_i32 ah, TCGv_i32 bl, TCGv_i32 bh) 1103 { 1104 if (TCG_TARGET_HAS_add2_i32) { 1105 tcg_gen_op6_i32(INDEX_op_add2_i32, rl, rh, al, ah, bl, bh); 1106 } else { 1107 TCGv_i64 t0 = tcg_temp_ebb_new_i64(); 1108 TCGv_i64 t1 = tcg_temp_ebb_new_i64(); 1109 tcg_gen_concat_i32_i64(t0, al, ah); 1110 tcg_gen_concat_i32_i64(t1, bl, bh); 1111 tcg_gen_add_i64(t0, t0, t1); 1112 tcg_gen_extr_i64_i32(rl, rh, t0); 1113 tcg_temp_free_i64(t0); 1114 tcg_temp_free_i64(t1); 1115 } 1116 } 1117 1118 void tcg_gen_sub2_i32(TCGv_i32 rl, TCGv_i32 rh, TCGv_i32 al, 1119 TCGv_i32 ah, TCGv_i32 bl, TCGv_i32 bh) 1120 { 1121 if (TCG_TARGET_HAS_sub2_i32) { 1122 tcg_gen_op6_i32(INDEX_op_sub2_i32, rl, rh, al, ah, bl, bh); 1123 } else { 1124 TCGv_i64 t0 = tcg_temp_ebb_new_i64(); 1125 TCGv_i64 t1 = tcg_temp_ebb_new_i64(); 1126 tcg_gen_concat_i32_i64(t0, al, ah); 1127 tcg_gen_concat_i32_i64(t1, bl, bh); 1128 tcg_gen_sub_i64(t0, t0, t1); 1129 tcg_gen_extr_i64_i32(rl, rh, t0); 1130 tcg_temp_free_i64(t0); 1131 tcg_temp_free_i64(t1); 1132 } 1133 } 1134 1135 void tcg_gen_mulu2_i32(TCGv_i32 rl, TCGv_i32 rh, TCGv_i32 arg1, TCGv_i32 arg2) 1136 { 1137 if (TCG_TARGET_HAS_mulu2_i32) { 1138 tcg_gen_op4_i32(INDEX_op_mulu2_i32, rl, rh, arg1, arg2); 1139 } else if (tcg_op_supported(INDEX_op_muluh, TCG_TYPE_I32, 0)) { 1140 TCGv_i32 t = tcg_temp_ebb_new_i32(); 1141 tcg_gen_op3_i32(INDEX_op_mul, t, arg1, arg2); 1142 tcg_gen_op3_i32(INDEX_op_muluh, rh, arg1, arg2); 1143 tcg_gen_mov_i32(rl, t); 1144 tcg_temp_free_i32(t); 1145 } else if (TCG_TARGET_REG_BITS == 64) { 1146 TCGv_i64 t0 = tcg_temp_ebb_new_i64(); 1147 TCGv_i64 t1 = tcg_temp_ebb_new_i64(); 1148 tcg_gen_extu_i32_i64(t0, arg1); 1149 tcg_gen_extu_i32_i64(t1, arg2); 1150 tcg_gen_mul_i64(t0, t0, t1); 1151 tcg_gen_extr_i64_i32(rl, rh, t0); 1152 tcg_temp_free_i64(t0); 1153 tcg_temp_free_i64(t1); 1154 } else { 1155 qemu_build_not_reached(); 1156 } 1157 } 1158 1159 void tcg_gen_muls2_i32(TCGv_i32 rl, TCGv_i32 rh, TCGv_i32 arg1, TCGv_i32 arg2) 1160 { 1161 if (TCG_TARGET_HAS_muls2_i32) { 1162 tcg_gen_op4_i32(INDEX_op_muls2_i32, rl, rh, arg1, arg2); 1163 } else if (tcg_op_supported(INDEX_op_mulsh, TCG_TYPE_I32, 0)) { 1164 TCGv_i32 t = tcg_temp_ebb_new_i32(); 1165 tcg_gen_op3_i32(INDEX_op_mul, t, arg1, arg2); 1166 tcg_gen_op3_i32(INDEX_op_mulsh, rh, arg1, arg2); 1167 tcg_gen_mov_i32(rl, t); 1168 tcg_temp_free_i32(t); 1169 } else if (TCG_TARGET_REG_BITS == 32) { 1170 TCGv_i32 t0 = tcg_temp_ebb_new_i32(); 1171 TCGv_i32 t1 = tcg_temp_ebb_new_i32(); 1172 TCGv_i32 t2 = tcg_temp_ebb_new_i32(); 1173 TCGv_i32 t3 = tcg_temp_ebb_new_i32(); 1174 tcg_gen_mulu2_i32(t0, t1, arg1, arg2); 1175 /* Adjust for negative inputs. */ 1176 tcg_gen_sari_i32(t2, arg1, 31); 1177 tcg_gen_sari_i32(t3, arg2, 31); 1178 tcg_gen_and_i32(t2, t2, arg2); 1179 tcg_gen_and_i32(t3, t3, arg1); 1180 tcg_gen_sub_i32(rh, t1, t2); 1181 tcg_gen_sub_i32(rh, rh, t3); 1182 tcg_gen_mov_i32(rl, t0); 1183 tcg_temp_free_i32(t0); 1184 tcg_temp_free_i32(t1); 1185 tcg_temp_free_i32(t2); 1186 tcg_temp_free_i32(t3); 1187 } else { 1188 TCGv_i64 t0 = tcg_temp_ebb_new_i64(); 1189 TCGv_i64 t1 = tcg_temp_ebb_new_i64(); 1190 tcg_gen_ext_i32_i64(t0, arg1); 1191 tcg_gen_ext_i32_i64(t1, arg2); 1192 tcg_gen_mul_i64(t0, t0, t1); 1193 tcg_gen_extr_i64_i32(rl, rh, t0); 1194 tcg_temp_free_i64(t0); 1195 tcg_temp_free_i64(t1); 1196 } 1197 } 1198 1199 void tcg_gen_mulsu2_i32(TCGv_i32 rl, TCGv_i32 rh, TCGv_i32 arg1, TCGv_i32 arg2) 1200 { 1201 if (TCG_TARGET_REG_BITS == 32) { 1202 TCGv_i32 t0 = tcg_temp_ebb_new_i32(); 1203 TCGv_i32 t1 = tcg_temp_ebb_new_i32(); 1204 TCGv_i32 t2 = tcg_temp_ebb_new_i32(); 1205 tcg_gen_mulu2_i32(t0, t1, arg1, arg2); 1206 /* Adjust for negative input for the signed arg1. */ 1207 tcg_gen_sari_i32(t2, arg1, 31); 1208 tcg_gen_and_i32(t2, t2, arg2); 1209 tcg_gen_sub_i32(rh, t1, t2); 1210 tcg_gen_mov_i32(rl, t0); 1211 tcg_temp_free_i32(t0); 1212 tcg_temp_free_i32(t1); 1213 tcg_temp_free_i32(t2); 1214 } else { 1215 TCGv_i64 t0 = tcg_temp_ebb_new_i64(); 1216 TCGv_i64 t1 = tcg_temp_ebb_new_i64(); 1217 tcg_gen_ext_i32_i64(t0, arg1); 1218 tcg_gen_extu_i32_i64(t1, arg2); 1219 tcg_gen_mul_i64(t0, t0, t1); 1220 tcg_gen_extr_i64_i32(rl, rh, t0); 1221 tcg_temp_free_i64(t0); 1222 tcg_temp_free_i64(t1); 1223 } 1224 } 1225 1226 void tcg_gen_ext8s_i32(TCGv_i32 ret, TCGv_i32 arg) 1227 { 1228 tcg_gen_sextract_i32(ret, arg, 0, 8); 1229 } 1230 1231 void tcg_gen_ext16s_i32(TCGv_i32 ret, TCGv_i32 arg) 1232 { 1233 tcg_gen_sextract_i32(ret, arg, 0, 16); 1234 } 1235 1236 void tcg_gen_ext8u_i32(TCGv_i32 ret, TCGv_i32 arg) 1237 { 1238 tcg_gen_extract_i32(ret, arg, 0, 8); 1239 } 1240 1241 void tcg_gen_ext16u_i32(TCGv_i32 ret, TCGv_i32 arg) 1242 { 1243 tcg_gen_extract_i32(ret, arg, 0, 16); 1244 } 1245 1246 /* 1247 * bswap16_i32: 16-bit byte swap on the low bits of a 32-bit value. 1248 * 1249 * Byte pattern: xxab -> yyba 1250 * 1251 * With TCG_BSWAP_IZ, x == zero, else undefined. 1252 * With TCG_BSWAP_OZ, y == zero, with TCG_BSWAP_OS y == sign, else undefined. 1253 */ 1254 void tcg_gen_bswap16_i32(TCGv_i32 ret, TCGv_i32 arg, int flags) 1255 { 1256 /* Only one extension flag may be present. */ 1257 tcg_debug_assert(!(flags & TCG_BSWAP_OS) || !(flags & TCG_BSWAP_OZ)); 1258 1259 if (TCG_TARGET_HAS_bswap16_i32) { 1260 tcg_gen_op3i_i32(INDEX_op_bswap16_i32, ret, arg, flags); 1261 } else { 1262 TCGv_i32 t0 = tcg_temp_ebb_new_i32(); 1263 TCGv_i32 t1 = tcg_temp_ebb_new_i32(); 1264 1265 /* arg = ..ab (IZ) xxab (!IZ) */ 1266 tcg_gen_shri_i32(t0, arg, 8); /* t0 = ...a (IZ) .xxa (!IZ) */ 1267 if (!(flags & TCG_BSWAP_IZ)) { 1268 tcg_gen_ext8u_i32(t0, t0); /* t0 = ...a */ 1269 } 1270 1271 if (flags & TCG_BSWAP_OS) { 1272 tcg_gen_shli_i32(t1, arg, 24); /* t1 = b... */ 1273 tcg_gen_sari_i32(t1, t1, 16); /* t1 = ssb. */ 1274 } else if (flags & TCG_BSWAP_OZ) { 1275 tcg_gen_ext8u_i32(t1, arg); /* t1 = ...b */ 1276 tcg_gen_shli_i32(t1, t1, 8); /* t1 = ..b. */ 1277 } else { 1278 tcg_gen_shli_i32(t1, arg, 8); /* t1 = xab. */ 1279 } 1280 1281 tcg_gen_or_i32(ret, t0, t1); /* ret = ..ba (OZ) */ 1282 /* = ssba (OS) */ 1283 /* = xaba (no flag) */ 1284 tcg_temp_free_i32(t0); 1285 tcg_temp_free_i32(t1); 1286 } 1287 } 1288 1289 /* 1290 * bswap32_i32: 32-bit byte swap on a 32-bit value. 1291 * 1292 * Byte pattern: abcd -> dcba 1293 */ 1294 void tcg_gen_bswap32_i32(TCGv_i32 ret, TCGv_i32 arg) 1295 { 1296 if (TCG_TARGET_HAS_bswap32_i32) { 1297 tcg_gen_op3i_i32(INDEX_op_bswap32_i32, ret, arg, 0); 1298 } else { 1299 TCGv_i32 t0 = tcg_temp_ebb_new_i32(); 1300 TCGv_i32 t1 = tcg_temp_ebb_new_i32(); 1301 TCGv_i32 t2 = tcg_constant_i32(0x00ff00ff); 1302 1303 /* arg = abcd */ 1304 tcg_gen_shri_i32(t0, arg, 8); /* t0 = .abc */ 1305 tcg_gen_and_i32(t1, arg, t2); /* t1 = .b.d */ 1306 tcg_gen_and_i32(t0, t0, t2); /* t0 = .a.c */ 1307 tcg_gen_shli_i32(t1, t1, 8); /* t1 = b.d. */ 1308 tcg_gen_or_i32(ret, t0, t1); /* ret = badc */ 1309 1310 tcg_gen_shri_i32(t0, ret, 16); /* t0 = ..ba */ 1311 tcg_gen_shli_i32(t1, ret, 16); /* t1 = dc.. */ 1312 tcg_gen_or_i32(ret, t0, t1); /* ret = dcba */ 1313 1314 tcg_temp_free_i32(t0); 1315 tcg_temp_free_i32(t1); 1316 } 1317 } 1318 1319 /* 1320 * hswap_i32: Swap 16-bit halfwords within a 32-bit value. 1321 * 1322 * Byte pattern: abcd -> cdab 1323 */ 1324 void tcg_gen_hswap_i32(TCGv_i32 ret, TCGv_i32 arg) 1325 { 1326 /* Swapping 2 16-bit elements is a rotate. */ 1327 tcg_gen_rotli_i32(ret, arg, 16); 1328 } 1329 1330 void tcg_gen_smin_i32(TCGv_i32 ret, TCGv_i32 a, TCGv_i32 b) 1331 { 1332 tcg_gen_movcond_i32(TCG_COND_LT, ret, a, b, a, b); 1333 } 1334 1335 void tcg_gen_umin_i32(TCGv_i32 ret, TCGv_i32 a, TCGv_i32 b) 1336 { 1337 tcg_gen_movcond_i32(TCG_COND_LTU, ret, a, b, a, b); 1338 } 1339 1340 void tcg_gen_smax_i32(TCGv_i32 ret, TCGv_i32 a, TCGv_i32 b) 1341 { 1342 tcg_gen_movcond_i32(TCG_COND_LT, ret, a, b, b, a); 1343 } 1344 1345 void tcg_gen_umax_i32(TCGv_i32 ret, TCGv_i32 a, TCGv_i32 b) 1346 { 1347 tcg_gen_movcond_i32(TCG_COND_LTU, ret, a, b, b, a); 1348 } 1349 1350 void tcg_gen_abs_i32(TCGv_i32 ret, TCGv_i32 a) 1351 { 1352 TCGv_i32 t = tcg_temp_ebb_new_i32(); 1353 1354 tcg_gen_sari_i32(t, a, 31); 1355 tcg_gen_xor_i32(ret, a, t); 1356 tcg_gen_sub_i32(ret, ret, t); 1357 tcg_temp_free_i32(t); 1358 } 1359 1360 void tcg_gen_ld8u_i32(TCGv_i32 ret, TCGv_ptr arg2, tcg_target_long offset) 1361 { 1362 tcg_gen_ldst_op_i32(INDEX_op_ld8u_i32, ret, arg2, offset); 1363 } 1364 1365 void tcg_gen_ld8s_i32(TCGv_i32 ret, TCGv_ptr arg2, tcg_target_long offset) 1366 { 1367 tcg_gen_ldst_op_i32(INDEX_op_ld8s_i32, ret, arg2, offset); 1368 } 1369 1370 void tcg_gen_ld16u_i32(TCGv_i32 ret, TCGv_ptr arg2, tcg_target_long offset) 1371 { 1372 tcg_gen_ldst_op_i32(INDEX_op_ld16u_i32, ret, arg2, offset); 1373 } 1374 1375 void tcg_gen_ld16s_i32(TCGv_i32 ret, TCGv_ptr arg2, tcg_target_long offset) 1376 { 1377 tcg_gen_ldst_op_i32(INDEX_op_ld16s_i32, ret, arg2, offset); 1378 } 1379 1380 void tcg_gen_ld_i32(TCGv_i32 ret, TCGv_ptr arg2, tcg_target_long offset) 1381 { 1382 tcg_gen_ldst_op_i32(INDEX_op_ld_i32, ret, arg2, offset); 1383 } 1384 1385 void tcg_gen_st8_i32(TCGv_i32 arg1, TCGv_ptr arg2, tcg_target_long offset) 1386 { 1387 tcg_gen_ldst_op_i32(INDEX_op_st8_i32, arg1, arg2, offset); 1388 } 1389 1390 void tcg_gen_st16_i32(TCGv_i32 arg1, TCGv_ptr arg2, tcg_target_long offset) 1391 { 1392 tcg_gen_ldst_op_i32(INDEX_op_st16_i32, arg1, arg2, offset); 1393 } 1394 1395 void tcg_gen_st_i32(TCGv_i32 arg1, TCGv_ptr arg2, tcg_target_long offset) 1396 { 1397 tcg_gen_ldst_op_i32(INDEX_op_st_i32, arg1, arg2, offset); 1398 } 1399 1400 1401 /* 64-bit ops */ 1402 1403 void tcg_gen_discard_i64(TCGv_i64 arg) 1404 { 1405 if (TCG_TARGET_REG_BITS == 64) { 1406 tcg_gen_op1_i64(INDEX_op_discard, TCG_TYPE_I64, arg); 1407 } else { 1408 tcg_gen_discard_i32(TCGV_LOW(arg)); 1409 tcg_gen_discard_i32(TCGV_HIGH(arg)); 1410 } 1411 } 1412 1413 void tcg_gen_mov_i64(TCGv_i64 ret, TCGv_i64 arg) 1414 { 1415 if (ret == arg) { 1416 return; 1417 } 1418 if (TCG_TARGET_REG_BITS == 64) { 1419 tcg_gen_op2_i64(INDEX_op_mov, ret, arg); 1420 } else { 1421 TCGTemp *ts = tcgv_i64_temp(arg); 1422 1423 /* Canonicalize TCGv_i64 TEMP_CONST into TCGv_i32 TEMP_CONST. */ 1424 if (ts->kind == TEMP_CONST) { 1425 tcg_gen_movi_i64(ret, ts->val); 1426 } else { 1427 tcg_gen_mov_i32(TCGV_LOW(ret), TCGV_LOW(arg)); 1428 tcg_gen_mov_i32(TCGV_HIGH(ret), TCGV_HIGH(arg)); 1429 } 1430 } 1431 } 1432 1433 void tcg_gen_movi_i64(TCGv_i64 ret, int64_t arg) 1434 { 1435 if (TCG_TARGET_REG_BITS == 64) { 1436 tcg_gen_mov_i64(ret, tcg_constant_i64(arg)); 1437 } else { 1438 tcg_gen_movi_i32(TCGV_LOW(ret), arg); 1439 tcg_gen_movi_i32(TCGV_HIGH(ret), arg >> 32); 1440 } 1441 } 1442 1443 void tcg_gen_ld8u_i64(TCGv_i64 ret, TCGv_ptr arg2, tcg_target_long offset) 1444 { 1445 if (TCG_TARGET_REG_BITS == 64) { 1446 tcg_gen_ldst_op_i64(INDEX_op_ld8u_i64, ret, arg2, offset); 1447 } else { 1448 tcg_gen_ld8u_i32(TCGV_LOW(ret), arg2, offset); 1449 tcg_gen_movi_i32(TCGV_HIGH(ret), 0); 1450 } 1451 } 1452 1453 void tcg_gen_ld8s_i64(TCGv_i64 ret, TCGv_ptr arg2, tcg_target_long offset) 1454 { 1455 if (TCG_TARGET_REG_BITS == 64) { 1456 tcg_gen_ldst_op_i64(INDEX_op_ld8s_i64, ret, arg2, offset); 1457 } else { 1458 tcg_gen_ld8s_i32(TCGV_LOW(ret), arg2, offset); 1459 tcg_gen_sari_i32(TCGV_HIGH(ret), TCGV_LOW(ret), 31); 1460 } 1461 } 1462 1463 void tcg_gen_ld16u_i64(TCGv_i64 ret, TCGv_ptr arg2, tcg_target_long offset) 1464 { 1465 if (TCG_TARGET_REG_BITS == 64) { 1466 tcg_gen_ldst_op_i64(INDEX_op_ld16u_i64, ret, arg2, offset); 1467 } else { 1468 tcg_gen_ld16u_i32(TCGV_LOW(ret), arg2, offset); 1469 tcg_gen_movi_i32(TCGV_HIGH(ret), 0); 1470 } 1471 } 1472 1473 void tcg_gen_ld16s_i64(TCGv_i64 ret, TCGv_ptr arg2, tcg_target_long offset) 1474 { 1475 if (TCG_TARGET_REG_BITS == 64) { 1476 tcg_gen_ldst_op_i64(INDEX_op_ld16s_i64, ret, arg2, offset); 1477 } else { 1478 tcg_gen_ld16s_i32(TCGV_LOW(ret), arg2, offset); 1479 tcg_gen_sari_i32(TCGV_HIGH(ret), TCGV_LOW(ret), 31); 1480 } 1481 } 1482 1483 void tcg_gen_ld32u_i64(TCGv_i64 ret, TCGv_ptr arg2, tcg_target_long offset) 1484 { 1485 if (TCG_TARGET_REG_BITS == 64) { 1486 tcg_gen_ldst_op_i64(INDEX_op_ld32u_i64, ret, arg2, offset); 1487 } else { 1488 tcg_gen_ld_i32(TCGV_LOW(ret), arg2, offset); 1489 tcg_gen_movi_i32(TCGV_HIGH(ret), 0); 1490 } 1491 } 1492 1493 void tcg_gen_ld32s_i64(TCGv_i64 ret, TCGv_ptr arg2, tcg_target_long offset) 1494 { 1495 if (TCG_TARGET_REG_BITS == 64) { 1496 tcg_gen_ldst_op_i64(INDEX_op_ld32s_i64, ret, arg2, offset); 1497 } else { 1498 tcg_gen_ld_i32(TCGV_LOW(ret), arg2, offset); 1499 tcg_gen_sari_i32(TCGV_HIGH(ret), TCGV_LOW(ret), 31); 1500 } 1501 } 1502 1503 void tcg_gen_ld_i64(TCGv_i64 ret, TCGv_ptr arg2, tcg_target_long offset) 1504 { 1505 /* 1506 * For 32-bit host, since arg2 and ret have different types, 1507 * they cannot be the same temporary -- no chance of overlap. 1508 */ 1509 if (TCG_TARGET_REG_BITS == 64) { 1510 tcg_gen_ldst_op_i64(INDEX_op_ld_i64, ret, arg2, offset); 1511 } else if (HOST_BIG_ENDIAN) { 1512 tcg_gen_ld_i32(TCGV_HIGH(ret), arg2, offset); 1513 tcg_gen_ld_i32(TCGV_LOW(ret), arg2, offset + 4); 1514 } else { 1515 tcg_gen_ld_i32(TCGV_LOW(ret), arg2, offset); 1516 tcg_gen_ld_i32(TCGV_HIGH(ret), arg2, offset + 4); 1517 } 1518 } 1519 1520 void tcg_gen_st8_i64(TCGv_i64 arg1, TCGv_ptr arg2, tcg_target_long offset) 1521 { 1522 if (TCG_TARGET_REG_BITS == 64) { 1523 tcg_gen_ldst_op_i64(INDEX_op_st8_i64, arg1, arg2, offset); 1524 } else { 1525 tcg_gen_st8_i32(TCGV_LOW(arg1), arg2, offset); 1526 } 1527 } 1528 1529 void tcg_gen_st16_i64(TCGv_i64 arg1, TCGv_ptr arg2, tcg_target_long offset) 1530 { 1531 if (TCG_TARGET_REG_BITS == 64) { 1532 tcg_gen_ldst_op_i64(INDEX_op_st16_i64, arg1, arg2, offset); 1533 } else { 1534 tcg_gen_st16_i32(TCGV_LOW(arg1), arg2, offset); 1535 } 1536 } 1537 1538 void tcg_gen_st32_i64(TCGv_i64 arg1, TCGv_ptr arg2, tcg_target_long offset) 1539 { 1540 if (TCG_TARGET_REG_BITS == 64) { 1541 tcg_gen_ldst_op_i64(INDEX_op_st32_i64, arg1, arg2, offset); 1542 } else { 1543 tcg_gen_st_i32(TCGV_LOW(arg1), arg2, offset); 1544 } 1545 } 1546 1547 void tcg_gen_st_i64(TCGv_i64 arg1, TCGv_ptr arg2, tcg_target_long offset) 1548 { 1549 if (TCG_TARGET_REG_BITS == 64) { 1550 tcg_gen_ldst_op_i64(INDEX_op_st_i64, arg1, arg2, offset); 1551 } else if (HOST_BIG_ENDIAN) { 1552 tcg_gen_st_i32(TCGV_HIGH(arg1), arg2, offset); 1553 tcg_gen_st_i32(TCGV_LOW(arg1), arg2, offset + 4); 1554 } else { 1555 tcg_gen_st_i32(TCGV_LOW(arg1), arg2, offset); 1556 tcg_gen_st_i32(TCGV_HIGH(arg1), arg2, offset + 4); 1557 } 1558 } 1559 1560 void tcg_gen_add_i64(TCGv_i64 ret, TCGv_i64 arg1, TCGv_i64 arg2) 1561 { 1562 if (TCG_TARGET_REG_BITS == 64) { 1563 tcg_gen_op3_i64(INDEX_op_add, ret, arg1, arg2); 1564 } else { 1565 tcg_gen_add2_i32(TCGV_LOW(ret), TCGV_HIGH(ret), TCGV_LOW(arg1), 1566 TCGV_HIGH(arg1), TCGV_LOW(arg2), TCGV_HIGH(arg2)); 1567 } 1568 } 1569 1570 void tcg_gen_sub_i64(TCGv_i64 ret, TCGv_i64 arg1, TCGv_i64 arg2) 1571 { 1572 if (TCG_TARGET_REG_BITS == 64) { 1573 tcg_gen_op3_i64(INDEX_op_sub, ret, arg1, arg2); 1574 } else { 1575 tcg_gen_sub2_i32(TCGV_LOW(ret), TCGV_HIGH(ret), TCGV_LOW(arg1), 1576 TCGV_HIGH(arg1), TCGV_LOW(arg2), TCGV_HIGH(arg2)); 1577 } 1578 } 1579 1580 void tcg_gen_and_i64(TCGv_i64 ret, TCGv_i64 arg1, TCGv_i64 arg2) 1581 { 1582 if (TCG_TARGET_REG_BITS == 64) { 1583 tcg_gen_op3_i64(INDEX_op_and, ret, arg1, arg2); 1584 } else { 1585 tcg_gen_and_i32(TCGV_LOW(ret), TCGV_LOW(arg1), TCGV_LOW(arg2)); 1586 tcg_gen_and_i32(TCGV_HIGH(ret), TCGV_HIGH(arg1), TCGV_HIGH(arg2)); 1587 } 1588 } 1589 1590 void tcg_gen_or_i64(TCGv_i64 ret, TCGv_i64 arg1, TCGv_i64 arg2) 1591 { 1592 if (TCG_TARGET_REG_BITS == 64) { 1593 tcg_gen_op3_i64(INDEX_op_or, ret, arg1, arg2); 1594 } else { 1595 tcg_gen_or_i32(TCGV_LOW(ret), TCGV_LOW(arg1), TCGV_LOW(arg2)); 1596 tcg_gen_or_i32(TCGV_HIGH(ret), TCGV_HIGH(arg1), TCGV_HIGH(arg2)); 1597 } 1598 } 1599 1600 void tcg_gen_xor_i64(TCGv_i64 ret, TCGv_i64 arg1, TCGv_i64 arg2) 1601 { 1602 if (TCG_TARGET_REG_BITS == 64) { 1603 tcg_gen_op3_i64(INDEX_op_xor, ret, arg1, arg2); 1604 } else { 1605 tcg_gen_xor_i32(TCGV_LOW(ret), TCGV_LOW(arg1), TCGV_LOW(arg2)); 1606 tcg_gen_xor_i32(TCGV_HIGH(ret), TCGV_HIGH(arg1), TCGV_HIGH(arg2)); 1607 } 1608 } 1609 1610 void tcg_gen_shl_i64(TCGv_i64 ret, TCGv_i64 arg1, TCGv_i64 arg2) 1611 { 1612 if (TCG_TARGET_REG_BITS == 64) { 1613 tcg_gen_op3_i64(INDEX_op_shl, ret, arg1, arg2); 1614 } else { 1615 gen_helper_shl_i64(ret, arg1, arg2); 1616 } 1617 } 1618 1619 void tcg_gen_shr_i64(TCGv_i64 ret, TCGv_i64 arg1, TCGv_i64 arg2) 1620 { 1621 if (TCG_TARGET_REG_BITS == 64) { 1622 tcg_gen_op3_i64(INDEX_op_shr, ret, arg1, arg2); 1623 } else { 1624 gen_helper_shr_i64(ret, arg1, arg2); 1625 } 1626 } 1627 1628 void tcg_gen_sar_i64(TCGv_i64 ret, TCGv_i64 arg1, TCGv_i64 arg2) 1629 { 1630 if (TCG_TARGET_REG_BITS == 64) { 1631 tcg_gen_op3_i64(INDEX_op_sar, ret, arg1, arg2); 1632 } else { 1633 gen_helper_sar_i64(ret, arg1, arg2); 1634 } 1635 } 1636 1637 void tcg_gen_mul_i64(TCGv_i64 ret, TCGv_i64 arg1, TCGv_i64 arg2) 1638 { 1639 TCGv_i64 t0; 1640 TCGv_i32 t1; 1641 1642 if (TCG_TARGET_REG_BITS == 64) { 1643 tcg_gen_op3_i64(INDEX_op_mul, ret, arg1, arg2); 1644 return; 1645 } 1646 1647 1648 t0 = tcg_temp_ebb_new_i64(); 1649 t1 = tcg_temp_ebb_new_i32(); 1650 1651 tcg_gen_mulu2_i32(TCGV_LOW(t0), TCGV_HIGH(t0), 1652 TCGV_LOW(arg1), TCGV_LOW(arg2)); 1653 1654 tcg_gen_mul_i32(t1, TCGV_LOW(arg1), TCGV_HIGH(arg2)); 1655 tcg_gen_add_i32(TCGV_HIGH(t0), TCGV_HIGH(t0), t1); 1656 tcg_gen_mul_i32(t1, TCGV_HIGH(arg1), TCGV_LOW(arg2)); 1657 tcg_gen_add_i32(TCGV_HIGH(t0), TCGV_HIGH(t0), t1); 1658 1659 tcg_gen_mov_i64(ret, t0); 1660 tcg_temp_free_i64(t0); 1661 tcg_temp_free_i32(t1); 1662 } 1663 1664 void tcg_gen_addi_i64(TCGv_i64 ret, TCGv_i64 arg1, int64_t arg2) 1665 { 1666 /* some cases can be optimized here */ 1667 if (arg2 == 0) { 1668 tcg_gen_mov_i64(ret, arg1); 1669 } else if (TCG_TARGET_REG_BITS == 64) { 1670 tcg_gen_add_i64(ret, arg1, tcg_constant_i64(arg2)); 1671 } else { 1672 tcg_gen_add2_i32(TCGV_LOW(ret), TCGV_HIGH(ret), 1673 TCGV_LOW(arg1), TCGV_HIGH(arg1), 1674 tcg_constant_i32(arg2), tcg_constant_i32(arg2 >> 32)); 1675 } 1676 } 1677 1678 void tcg_gen_subfi_i64(TCGv_i64 ret, int64_t arg1, TCGv_i64 arg2) 1679 { 1680 if (arg1 == 0) { 1681 tcg_gen_neg_i64(ret, arg2); 1682 } else if (TCG_TARGET_REG_BITS == 64) { 1683 tcg_gen_sub_i64(ret, tcg_constant_i64(arg1), arg2); 1684 } else { 1685 tcg_gen_sub2_i32(TCGV_LOW(ret), TCGV_HIGH(ret), 1686 tcg_constant_i32(arg1), tcg_constant_i32(arg1 >> 32), 1687 TCGV_LOW(arg2), TCGV_HIGH(arg2)); 1688 } 1689 } 1690 1691 void tcg_gen_subi_i64(TCGv_i64 ret, TCGv_i64 arg1, int64_t arg2) 1692 { 1693 tcg_gen_addi_i64(ret, arg1, -arg2); 1694 } 1695 1696 void tcg_gen_neg_i64(TCGv_i64 ret, TCGv_i64 arg) 1697 { 1698 if (TCG_TARGET_REG_BITS == 64) { 1699 tcg_gen_op2_i64(INDEX_op_neg, ret, arg); 1700 } else { 1701 TCGv_i32 zero = tcg_constant_i32(0); 1702 tcg_gen_sub2_i32(TCGV_LOW(ret), TCGV_HIGH(ret), 1703 zero, zero, TCGV_LOW(arg), TCGV_HIGH(arg)); 1704 } 1705 } 1706 1707 void tcg_gen_andi_i64(TCGv_i64 ret, TCGv_i64 arg1, int64_t arg2) 1708 { 1709 if (TCG_TARGET_REG_BITS == 32) { 1710 tcg_gen_andi_i32(TCGV_LOW(ret), TCGV_LOW(arg1), arg2); 1711 tcg_gen_andi_i32(TCGV_HIGH(ret), TCGV_HIGH(arg1), arg2 >> 32); 1712 return; 1713 } 1714 1715 /* Some cases can be optimized here. */ 1716 switch (arg2) { 1717 case 0: 1718 tcg_gen_movi_i64(ret, 0); 1719 return; 1720 case -1: 1721 tcg_gen_mov_i64(ret, arg1); 1722 return; 1723 default: 1724 /* 1725 * Canonicalize on extract, if valid. This aids x86 with its 1726 * 2 operand MOVZBL and 2 operand AND, selecting the TCGOpcode 1727 * which does not require matching operands. Other backends can 1728 * trivially expand the extract to AND during code generation. 1729 */ 1730 if (!(arg2 & (arg2 + 1))) { 1731 unsigned len = ctz64(~arg2); 1732 if (TCG_TARGET_extract_valid(TCG_TYPE_I64, 0, len)) { 1733 tcg_gen_extract_i64(ret, arg1, 0, len); 1734 return; 1735 } 1736 } 1737 break; 1738 } 1739 1740 tcg_gen_and_i64(ret, arg1, tcg_constant_i64(arg2)); 1741 } 1742 1743 void tcg_gen_ori_i64(TCGv_i64 ret, TCGv_i64 arg1, int64_t arg2) 1744 { 1745 if (TCG_TARGET_REG_BITS == 32) { 1746 tcg_gen_ori_i32(TCGV_LOW(ret), TCGV_LOW(arg1), arg2); 1747 tcg_gen_ori_i32(TCGV_HIGH(ret), TCGV_HIGH(arg1), arg2 >> 32); 1748 return; 1749 } 1750 /* Some cases can be optimized here. */ 1751 if (arg2 == -1) { 1752 tcg_gen_movi_i64(ret, -1); 1753 } else if (arg2 == 0) { 1754 tcg_gen_mov_i64(ret, arg1); 1755 } else { 1756 tcg_gen_or_i64(ret, arg1, tcg_constant_i64(arg2)); 1757 } 1758 } 1759 1760 void tcg_gen_xori_i64(TCGv_i64 ret, TCGv_i64 arg1, int64_t arg2) 1761 { 1762 if (TCG_TARGET_REG_BITS == 32) { 1763 tcg_gen_xori_i32(TCGV_LOW(ret), TCGV_LOW(arg1), arg2); 1764 tcg_gen_xori_i32(TCGV_HIGH(ret), TCGV_HIGH(arg1), arg2 >> 32); 1765 return; 1766 } 1767 /* Some cases can be optimized here. */ 1768 if (arg2 == 0) { 1769 tcg_gen_mov_i64(ret, arg1); 1770 } else if (arg2 == -1 && 1771 tcg_op_supported(INDEX_op_not, TCG_TYPE_I64, 0)) { 1772 /* Don't recurse with tcg_gen_not_i64. */ 1773 tcg_gen_op2_i64(INDEX_op_not, ret, arg1); 1774 } else { 1775 tcg_gen_xor_i64(ret, arg1, tcg_constant_i64(arg2)); 1776 } 1777 } 1778 1779 static inline void tcg_gen_shifti_i64(TCGv_i64 ret, TCGv_i64 arg1, 1780 unsigned c, bool right, bool arith) 1781 { 1782 tcg_debug_assert(c < 64); 1783 if (c == 0) { 1784 tcg_gen_mov_i32(TCGV_LOW(ret), TCGV_LOW(arg1)); 1785 tcg_gen_mov_i32(TCGV_HIGH(ret), TCGV_HIGH(arg1)); 1786 } else if (c >= 32) { 1787 c -= 32; 1788 if (right) { 1789 if (arith) { 1790 tcg_gen_sari_i32(TCGV_LOW(ret), TCGV_HIGH(arg1), c); 1791 tcg_gen_sari_i32(TCGV_HIGH(ret), TCGV_HIGH(arg1), 31); 1792 } else { 1793 tcg_gen_shri_i32(TCGV_LOW(ret), TCGV_HIGH(arg1), c); 1794 tcg_gen_movi_i32(TCGV_HIGH(ret), 0); 1795 } 1796 } else { 1797 tcg_gen_shli_i32(TCGV_HIGH(ret), TCGV_LOW(arg1), c); 1798 tcg_gen_movi_i32(TCGV_LOW(ret), 0); 1799 } 1800 } else if (right) { 1801 if (TCG_TARGET_HAS_extract2_i32) { 1802 tcg_gen_extract2_i32(TCGV_LOW(ret), 1803 TCGV_LOW(arg1), TCGV_HIGH(arg1), c); 1804 } else { 1805 tcg_gen_shri_i32(TCGV_LOW(ret), TCGV_LOW(arg1), c); 1806 tcg_gen_deposit_i32(TCGV_LOW(ret), TCGV_LOW(ret), 1807 TCGV_HIGH(arg1), 32 - c, c); 1808 } 1809 if (arith) { 1810 tcg_gen_sari_i32(TCGV_HIGH(ret), TCGV_HIGH(arg1), c); 1811 } else { 1812 tcg_gen_shri_i32(TCGV_HIGH(ret), TCGV_HIGH(arg1), c); 1813 } 1814 } else { 1815 if (TCG_TARGET_HAS_extract2_i32) { 1816 tcg_gen_extract2_i32(TCGV_HIGH(ret), 1817 TCGV_LOW(arg1), TCGV_HIGH(arg1), 32 - c); 1818 } else { 1819 TCGv_i32 t0 = tcg_temp_ebb_new_i32(); 1820 tcg_gen_shri_i32(t0, TCGV_LOW(arg1), 32 - c); 1821 tcg_gen_deposit_i32(TCGV_HIGH(ret), t0, 1822 TCGV_HIGH(arg1), c, 32 - c); 1823 tcg_temp_free_i32(t0); 1824 } 1825 tcg_gen_shli_i32(TCGV_LOW(ret), TCGV_LOW(arg1), c); 1826 } 1827 } 1828 1829 void tcg_gen_shli_i64(TCGv_i64 ret, TCGv_i64 arg1, int64_t arg2) 1830 { 1831 tcg_debug_assert(arg2 >= 0 && arg2 < 64); 1832 if (TCG_TARGET_REG_BITS == 32) { 1833 tcg_gen_shifti_i64(ret, arg1, arg2, 0, 0); 1834 } else if (arg2 == 0) { 1835 tcg_gen_mov_i64(ret, arg1); 1836 } else { 1837 tcg_gen_shl_i64(ret, arg1, tcg_constant_i64(arg2)); 1838 } 1839 } 1840 1841 void tcg_gen_shri_i64(TCGv_i64 ret, TCGv_i64 arg1, int64_t arg2) 1842 { 1843 tcg_debug_assert(arg2 >= 0 && arg2 < 64); 1844 if (TCG_TARGET_REG_BITS == 32) { 1845 tcg_gen_shifti_i64(ret, arg1, arg2, 1, 0); 1846 } else if (arg2 == 0) { 1847 tcg_gen_mov_i64(ret, arg1); 1848 } else { 1849 tcg_gen_shr_i64(ret, arg1, tcg_constant_i64(arg2)); 1850 } 1851 } 1852 1853 void tcg_gen_sari_i64(TCGv_i64 ret, TCGv_i64 arg1, int64_t arg2) 1854 { 1855 tcg_debug_assert(arg2 >= 0 && arg2 < 64); 1856 if (TCG_TARGET_REG_BITS == 32) { 1857 tcg_gen_shifti_i64(ret, arg1, arg2, 1, 1); 1858 } else if (arg2 == 0) { 1859 tcg_gen_mov_i64(ret, arg1); 1860 } else { 1861 tcg_gen_sar_i64(ret, arg1, tcg_constant_i64(arg2)); 1862 } 1863 } 1864 1865 void tcg_gen_brcond_i64(TCGCond cond, TCGv_i64 arg1, TCGv_i64 arg2, TCGLabel *l) 1866 { 1867 if (cond == TCG_COND_ALWAYS) { 1868 tcg_gen_br(l); 1869 } else if (cond != TCG_COND_NEVER) { 1870 TCGOp *op; 1871 if (TCG_TARGET_REG_BITS == 32) { 1872 op = tcg_gen_op6ii_i32(INDEX_op_brcond2_i32, TCGV_LOW(arg1), 1873 TCGV_HIGH(arg1), TCGV_LOW(arg2), 1874 TCGV_HIGH(arg2), cond, label_arg(l)); 1875 } else { 1876 op = tcg_gen_op4ii_i64(INDEX_op_brcond_i64, arg1, arg2, cond, 1877 label_arg(l)); 1878 } 1879 add_as_label_use(l, op); 1880 } 1881 } 1882 1883 void tcg_gen_brcondi_i64(TCGCond cond, TCGv_i64 arg1, int64_t arg2, TCGLabel *l) 1884 { 1885 if (TCG_TARGET_REG_BITS == 64) { 1886 tcg_gen_brcond_i64(cond, arg1, tcg_constant_i64(arg2), l); 1887 } else if (cond == TCG_COND_ALWAYS) { 1888 tcg_gen_br(l); 1889 } else if (cond != TCG_COND_NEVER) { 1890 TCGOp *op = tcg_gen_op6ii_i32(INDEX_op_brcond2_i32, 1891 TCGV_LOW(arg1), TCGV_HIGH(arg1), 1892 tcg_constant_i32(arg2), 1893 tcg_constant_i32(arg2 >> 32), 1894 cond, label_arg(l)); 1895 add_as_label_use(l, op); 1896 } 1897 } 1898 1899 void tcg_gen_setcond_i64(TCGCond cond, TCGv_i64 ret, 1900 TCGv_i64 arg1, TCGv_i64 arg2) 1901 { 1902 if (cond == TCG_COND_ALWAYS) { 1903 tcg_gen_movi_i64(ret, 1); 1904 } else if (cond == TCG_COND_NEVER) { 1905 tcg_gen_movi_i64(ret, 0); 1906 } else { 1907 if (TCG_TARGET_REG_BITS == 32) { 1908 tcg_gen_op6i_i32(INDEX_op_setcond2_i32, TCGV_LOW(ret), 1909 TCGV_LOW(arg1), TCGV_HIGH(arg1), 1910 TCGV_LOW(arg2), TCGV_HIGH(arg2), cond); 1911 tcg_gen_movi_i32(TCGV_HIGH(ret), 0); 1912 } else { 1913 tcg_gen_op4i_i64(INDEX_op_setcond_i64, ret, arg1, arg2, cond); 1914 } 1915 } 1916 } 1917 1918 void tcg_gen_setcondi_i64(TCGCond cond, TCGv_i64 ret, 1919 TCGv_i64 arg1, int64_t arg2) 1920 { 1921 if (TCG_TARGET_REG_BITS == 64) { 1922 tcg_gen_setcond_i64(cond, ret, arg1, tcg_constant_i64(arg2)); 1923 } else if (cond == TCG_COND_ALWAYS) { 1924 tcg_gen_movi_i64(ret, 1); 1925 } else if (cond == TCG_COND_NEVER) { 1926 tcg_gen_movi_i64(ret, 0); 1927 } else { 1928 tcg_gen_op6i_i32(INDEX_op_setcond2_i32, TCGV_LOW(ret), 1929 TCGV_LOW(arg1), TCGV_HIGH(arg1), 1930 tcg_constant_i32(arg2), 1931 tcg_constant_i32(arg2 >> 32), cond); 1932 tcg_gen_movi_i32(TCGV_HIGH(ret), 0); 1933 } 1934 } 1935 1936 void tcg_gen_negsetcondi_i64(TCGCond cond, TCGv_i64 ret, 1937 TCGv_i64 arg1, int64_t arg2) 1938 { 1939 tcg_gen_negsetcond_i64(cond, ret, arg1, tcg_constant_i64(arg2)); 1940 } 1941 1942 void tcg_gen_negsetcond_i64(TCGCond cond, TCGv_i64 ret, 1943 TCGv_i64 arg1, TCGv_i64 arg2) 1944 { 1945 if (cond == TCG_COND_ALWAYS) { 1946 tcg_gen_movi_i64(ret, -1); 1947 } else if (cond == TCG_COND_NEVER) { 1948 tcg_gen_movi_i64(ret, 0); 1949 } else if (TCG_TARGET_HAS_negsetcond_i64) { 1950 tcg_gen_op4i_i64(INDEX_op_negsetcond_i64, ret, arg1, arg2, cond); 1951 } else if (TCG_TARGET_REG_BITS == 32) { 1952 tcg_gen_op6i_i32(INDEX_op_setcond2_i32, TCGV_LOW(ret), 1953 TCGV_LOW(arg1), TCGV_HIGH(arg1), 1954 TCGV_LOW(arg2), TCGV_HIGH(arg2), cond); 1955 tcg_gen_neg_i32(TCGV_LOW(ret), TCGV_LOW(ret)); 1956 tcg_gen_mov_i32(TCGV_HIGH(ret), TCGV_LOW(ret)); 1957 } else { 1958 tcg_gen_setcond_i64(cond, ret, arg1, arg2); 1959 tcg_gen_neg_i64(ret, ret); 1960 } 1961 } 1962 1963 void tcg_gen_muli_i64(TCGv_i64 ret, TCGv_i64 arg1, int64_t arg2) 1964 { 1965 if (arg2 == 0) { 1966 tcg_gen_movi_i64(ret, 0); 1967 } else if (is_power_of_2(arg2)) { 1968 tcg_gen_shli_i64(ret, arg1, ctz64(arg2)); 1969 } else { 1970 tcg_gen_mul_i64(ret, arg1, tcg_constant_i64(arg2)); 1971 } 1972 } 1973 1974 void tcg_gen_div_i64(TCGv_i64 ret, TCGv_i64 arg1, TCGv_i64 arg2) 1975 { 1976 if (tcg_op_supported(INDEX_op_divs, TCG_TYPE_I64, 0)) { 1977 tcg_gen_op3_i64(INDEX_op_divs, ret, arg1, arg2); 1978 } else if (tcg_op_supported(INDEX_op_divs2, TCG_TYPE_I64, 0)) { 1979 TCGv_i64 t0 = tcg_temp_ebb_new_i64(); 1980 tcg_gen_sari_i64(t0, arg1, 63); 1981 tcg_gen_op5_i64(INDEX_op_divs2, ret, t0, arg1, t0, arg2); 1982 tcg_temp_free_i64(t0); 1983 } else { 1984 gen_helper_div_i64(ret, arg1, arg2); 1985 } 1986 } 1987 1988 void tcg_gen_rem_i64(TCGv_i64 ret, TCGv_i64 arg1, TCGv_i64 arg2) 1989 { 1990 if (tcg_op_supported(INDEX_op_rems, TCG_TYPE_I64, 0)) { 1991 tcg_gen_op3_i64(INDEX_op_rems, ret, arg1, arg2); 1992 } else if (tcg_op_supported(INDEX_op_divs, TCG_TYPE_I64, 0)) { 1993 TCGv_i64 t0 = tcg_temp_ebb_new_i64(); 1994 tcg_gen_op3_i64(INDEX_op_divs, t0, arg1, arg2); 1995 tcg_gen_mul_i64(t0, t0, arg2); 1996 tcg_gen_sub_i64(ret, arg1, t0); 1997 tcg_temp_free_i64(t0); 1998 } else if (tcg_op_supported(INDEX_op_divs2, TCG_TYPE_I64, 0)) { 1999 TCGv_i64 t0 = tcg_temp_ebb_new_i64(); 2000 tcg_gen_sari_i64(t0, arg1, 63); 2001 tcg_gen_op5_i64(INDEX_op_divs2, t0, ret, arg1, t0, arg2); 2002 tcg_temp_free_i64(t0); 2003 } else { 2004 gen_helper_rem_i64(ret, arg1, arg2); 2005 } 2006 } 2007 2008 void tcg_gen_divu_i64(TCGv_i64 ret, TCGv_i64 arg1, TCGv_i64 arg2) 2009 { 2010 if (tcg_op_supported(INDEX_op_divu, TCG_TYPE_I64, 0)) { 2011 tcg_gen_op3_i64(INDEX_op_divu, ret, arg1, arg2); 2012 } else if (tcg_op_supported(INDEX_op_divu2, TCG_TYPE_I64, 0)) { 2013 TCGv_i64 t0 = tcg_temp_ebb_new_i64(); 2014 TCGv_i64 zero = tcg_constant_i64(0); 2015 tcg_gen_op5_i64(INDEX_op_divu2, ret, t0, arg1, zero, arg2); 2016 tcg_temp_free_i64(t0); 2017 } else { 2018 gen_helper_divu_i64(ret, arg1, arg2); 2019 } 2020 } 2021 2022 void tcg_gen_remu_i64(TCGv_i64 ret, TCGv_i64 arg1, TCGv_i64 arg2) 2023 { 2024 if (tcg_op_supported(INDEX_op_remu, TCG_TYPE_I64, 0)) { 2025 tcg_gen_op3_i64(INDEX_op_remu, ret, arg1, arg2); 2026 } else if (tcg_op_supported(INDEX_op_divu, TCG_TYPE_I64, 0)) { 2027 TCGv_i64 t0 = tcg_temp_ebb_new_i64(); 2028 tcg_gen_op3_i64(INDEX_op_divu, t0, arg1, arg2); 2029 tcg_gen_mul_i64(t0, t0, arg2); 2030 tcg_gen_sub_i64(ret, arg1, t0); 2031 tcg_temp_free_i64(t0); 2032 } else if (tcg_op_supported(INDEX_op_divu2, TCG_TYPE_I64, 0)) { 2033 TCGv_i64 t0 = tcg_temp_ebb_new_i64(); 2034 TCGv_i64 zero = tcg_constant_i64(0); 2035 tcg_gen_op5_i64(INDEX_op_divu2, t0, ret, arg1, zero, arg2); 2036 tcg_temp_free_i64(t0); 2037 } else { 2038 gen_helper_remu_i64(ret, arg1, arg2); 2039 } 2040 } 2041 2042 void tcg_gen_ext8s_i64(TCGv_i64 ret, TCGv_i64 arg) 2043 { 2044 tcg_gen_sextract_i64(ret, arg, 0, 8); 2045 } 2046 2047 void tcg_gen_ext16s_i64(TCGv_i64 ret, TCGv_i64 arg) 2048 { 2049 tcg_gen_sextract_i64(ret, arg, 0, 16); 2050 } 2051 2052 void tcg_gen_ext32s_i64(TCGv_i64 ret, TCGv_i64 arg) 2053 { 2054 tcg_gen_sextract_i64(ret, arg, 0, 32); 2055 } 2056 2057 void tcg_gen_ext8u_i64(TCGv_i64 ret, TCGv_i64 arg) 2058 { 2059 tcg_gen_extract_i64(ret, arg, 0, 8); 2060 } 2061 2062 void tcg_gen_ext16u_i64(TCGv_i64 ret, TCGv_i64 arg) 2063 { 2064 tcg_gen_extract_i64(ret, arg, 0, 16); 2065 } 2066 2067 void tcg_gen_ext32u_i64(TCGv_i64 ret, TCGv_i64 arg) 2068 { 2069 tcg_gen_extract_i64(ret, arg, 0, 32); 2070 } 2071 2072 /* 2073 * bswap16_i64: 16-bit byte swap on the low bits of a 64-bit value. 2074 * 2075 * Byte pattern: xxxxxxxxab -> yyyyyyyyba 2076 * 2077 * With TCG_BSWAP_IZ, x == zero, else undefined. 2078 * With TCG_BSWAP_OZ, y == zero, with TCG_BSWAP_OS y == sign, else undefined. 2079 */ 2080 void tcg_gen_bswap16_i64(TCGv_i64 ret, TCGv_i64 arg, int flags) 2081 { 2082 /* Only one extension flag may be present. */ 2083 tcg_debug_assert(!(flags & TCG_BSWAP_OS) || !(flags & TCG_BSWAP_OZ)); 2084 2085 if (TCG_TARGET_REG_BITS == 32) { 2086 tcg_gen_bswap16_i32(TCGV_LOW(ret), TCGV_LOW(arg), flags); 2087 if (flags & TCG_BSWAP_OS) { 2088 tcg_gen_sari_i32(TCGV_HIGH(ret), TCGV_LOW(ret), 31); 2089 } else { 2090 tcg_gen_movi_i32(TCGV_HIGH(ret), 0); 2091 } 2092 } else if (TCG_TARGET_HAS_bswap16_i64) { 2093 tcg_gen_op3i_i64(INDEX_op_bswap16_i64, ret, arg, flags); 2094 } else { 2095 TCGv_i64 t0 = tcg_temp_ebb_new_i64(); 2096 TCGv_i64 t1 = tcg_temp_ebb_new_i64(); 2097 2098 /* arg = ......ab or xxxxxxab */ 2099 tcg_gen_shri_i64(t0, arg, 8); /* t0 = .......a or .xxxxxxa */ 2100 if (!(flags & TCG_BSWAP_IZ)) { 2101 tcg_gen_ext8u_i64(t0, t0); /* t0 = .......a */ 2102 } 2103 2104 if (flags & TCG_BSWAP_OS) { 2105 tcg_gen_shli_i64(t1, arg, 56); /* t1 = b....... */ 2106 tcg_gen_sari_i64(t1, t1, 48); /* t1 = ssssssb. */ 2107 } else if (flags & TCG_BSWAP_OZ) { 2108 tcg_gen_ext8u_i64(t1, arg); /* t1 = .......b */ 2109 tcg_gen_shli_i64(t1, t1, 8); /* t1 = ......b. */ 2110 } else { 2111 tcg_gen_shli_i64(t1, arg, 8); /* t1 = xxxxxab. */ 2112 } 2113 2114 tcg_gen_or_i64(ret, t0, t1); /* ret = ......ba (OZ) */ 2115 /* ssssssba (OS) */ 2116 /* xxxxxaba (no flag) */ 2117 tcg_temp_free_i64(t0); 2118 tcg_temp_free_i64(t1); 2119 } 2120 } 2121 2122 /* 2123 * bswap32_i64: 32-bit byte swap on the low bits of a 64-bit value. 2124 * 2125 * Byte pattern: xxxxabcd -> yyyydcba 2126 * 2127 * With TCG_BSWAP_IZ, x == zero, else undefined. 2128 * With TCG_BSWAP_OZ, y == zero, with TCG_BSWAP_OS y == sign, else undefined. 2129 */ 2130 void tcg_gen_bswap32_i64(TCGv_i64 ret, TCGv_i64 arg, int flags) 2131 { 2132 /* Only one extension flag may be present. */ 2133 tcg_debug_assert(!(flags & TCG_BSWAP_OS) || !(flags & TCG_BSWAP_OZ)); 2134 2135 if (TCG_TARGET_REG_BITS == 32) { 2136 tcg_gen_bswap32_i32(TCGV_LOW(ret), TCGV_LOW(arg)); 2137 if (flags & TCG_BSWAP_OS) { 2138 tcg_gen_sari_i32(TCGV_HIGH(ret), TCGV_LOW(ret), 31); 2139 } else { 2140 tcg_gen_movi_i32(TCGV_HIGH(ret), 0); 2141 } 2142 } else if (TCG_TARGET_HAS_bswap32_i64) { 2143 tcg_gen_op3i_i64(INDEX_op_bswap32_i64, ret, arg, flags); 2144 } else { 2145 TCGv_i64 t0 = tcg_temp_ebb_new_i64(); 2146 TCGv_i64 t1 = tcg_temp_ebb_new_i64(); 2147 TCGv_i64 t2 = tcg_constant_i64(0x00ff00ff); 2148 2149 /* arg = xxxxabcd */ 2150 tcg_gen_shri_i64(t0, arg, 8); /* t0 = .xxxxabc */ 2151 tcg_gen_and_i64(t1, arg, t2); /* t1 = .....b.d */ 2152 tcg_gen_and_i64(t0, t0, t2); /* t0 = .....a.c */ 2153 tcg_gen_shli_i64(t1, t1, 8); /* t1 = ....b.d. */ 2154 tcg_gen_or_i64(ret, t0, t1); /* ret = ....badc */ 2155 2156 tcg_gen_shli_i64(t1, ret, 48); /* t1 = dc...... */ 2157 tcg_gen_shri_i64(t0, ret, 16); /* t0 = ......ba */ 2158 if (flags & TCG_BSWAP_OS) { 2159 tcg_gen_sari_i64(t1, t1, 32); /* t1 = ssssdc.. */ 2160 } else { 2161 tcg_gen_shri_i64(t1, t1, 32); /* t1 = ....dc.. */ 2162 } 2163 tcg_gen_or_i64(ret, t0, t1); /* ret = ssssdcba (OS) */ 2164 /* ....dcba (else) */ 2165 2166 tcg_temp_free_i64(t0); 2167 tcg_temp_free_i64(t1); 2168 } 2169 } 2170 2171 /* 2172 * bswap64_i64: 64-bit byte swap on a 64-bit value. 2173 * 2174 * Byte pattern: abcdefgh -> hgfedcba 2175 */ 2176 void tcg_gen_bswap64_i64(TCGv_i64 ret, TCGv_i64 arg) 2177 { 2178 if (TCG_TARGET_REG_BITS == 32) { 2179 TCGv_i32 t0, t1; 2180 t0 = tcg_temp_ebb_new_i32(); 2181 t1 = tcg_temp_ebb_new_i32(); 2182 2183 tcg_gen_bswap32_i32(t0, TCGV_LOW(arg)); 2184 tcg_gen_bswap32_i32(t1, TCGV_HIGH(arg)); 2185 tcg_gen_mov_i32(TCGV_LOW(ret), t1); 2186 tcg_gen_mov_i32(TCGV_HIGH(ret), t0); 2187 tcg_temp_free_i32(t0); 2188 tcg_temp_free_i32(t1); 2189 } else if (TCG_TARGET_HAS_bswap64_i64) { 2190 tcg_gen_op3i_i64(INDEX_op_bswap64_i64, ret, arg, 0); 2191 } else { 2192 TCGv_i64 t0 = tcg_temp_ebb_new_i64(); 2193 TCGv_i64 t1 = tcg_temp_ebb_new_i64(); 2194 TCGv_i64 t2 = tcg_temp_ebb_new_i64(); 2195 2196 /* arg = abcdefgh */ 2197 tcg_gen_movi_i64(t2, 0x00ff00ff00ff00ffull); 2198 tcg_gen_shri_i64(t0, arg, 8); /* t0 = .abcdefg */ 2199 tcg_gen_and_i64(t1, arg, t2); /* t1 = .b.d.f.h */ 2200 tcg_gen_and_i64(t0, t0, t2); /* t0 = .a.c.e.g */ 2201 tcg_gen_shli_i64(t1, t1, 8); /* t1 = b.d.f.h. */ 2202 tcg_gen_or_i64(ret, t0, t1); /* ret = badcfehg */ 2203 2204 tcg_gen_movi_i64(t2, 0x0000ffff0000ffffull); 2205 tcg_gen_shri_i64(t0, ret, 16); /* t0 = ..badcfe */ 2206 tcg_gen_and_i64(t1, ret, t2); /* t1 = ..dc..hg */ 2207 tcg_gen_and_i64(t0, t0, t2); /* t0 = ..ba..fe */ 2208 tcg_gen_shli_i64(t1, t1, 16); /* t1 = dc..hg.. */ 2209 tcg_gen_or_i64(ret, t0, t1); /* ret = dcbahgfe */ 2210 2211 tcg_gen_shri_i64(t0, ret, 32); /* t0 = ....dcba */ 2212 tcg_gen_shli_i64(t1, ret, 32); /* t1 = hgfe.... */ 2213 tcg_gen_or_i64(ret, t0, t1); /* ret = hgfedcba */ 2214 2215 tcg_temp_free_i64(t0); 2216 tcg_temp_free_i64(t1); 2217 tcg_temp_free_i64(t2); 2218 } 2219 } 2220 2221 /* 2222 * hswap_i64: Swap 16-bit halfwords within a 64-bit value. 2223 * See also include/qemu/bitops.h, hswap64. 2224 * 2225 * Byte pattern: abcdefgh -> ghefcdab 2226 */ 2227 void tcg_gen_hswap_i64(TCGv_i64 ret, TCGv_i64 arg) 2228 { 2229 uint64_t m = 0x0000ffff0000ffffull; 2230 TCGv_i64 t0 = tcg_temp_ebb_new_i64(); 2231 TCGv_i64 t1 = tcg_temp_ebb_new_i64(); 2232 2233 /* arg = abcdefgh */ 2234 tcg_gen_rotli_i64(t1, arg, 32); /* t1 = efghabcd */ 2235 tcg_gen_andi_i64(t0, t1, m); /* t0 = ..gh..cd */ 2236 tcg_gen_shli_i64(t0, t0, 16); /* t0 = gh..cd.. */ 2237 tcg_gen_shri_i64(t1, t1, 16); /* t1 = ..efghab */ 2238 tcg_gen_andi_i64(t1, t1, m); /* t1 = ..ef..ab */ 2239 tcg_gen_or_i64(ret, t0, t1); /* ret = ghefcdab */ 2240 2241 tcg_temp_free_i64(t0); 2242 tcg_temp_free_i64(t1); 2243 } 2244 2245 /* 2246 * wswap_i64: Swap 32-bit words within a 64-bit value. 2247 * 2248 * Byte pattern: abcdefgh -> efghabcd 2249 */ 2250 void tcg_gen_wswap_i64(TCGv_i64 ret, TCGv_i64 arg) 2251 { 2252 /* Swapping 2 32-bit elements is a rotate. */ 2253 tcg_gen_rotli_i64(ret, arg, 32); 2254 } 2255 2256 void tcg_gen_not_i64(TCGv_i64 ret, TCGv_i64 arg) 2257 { 2258 if (TCG_TARGET_REG_BITS == 32) { 2259 tcg_gen_not_i32(TCGV_LOW(ret), TCGV_LOW(arg)); 2260 tcg_gen_not_i32(TCGV_HIGH(ret), TCGV_HIGH(arg)); 2261 } else if (tcg_op_supported(INDEX_op_not, TCG_TYPE_I64, 0)) { 2262 tcg_gen_op2_i64(INDEX_op_not, ret, arg); 2263 } else { 2264 tcg_gen_xori_i64(ret, arg, -1); 2265 } 2266 } 2267 2268 void tcg_gen_andc_i64(TCGv_i64 ret, TCGv_i64 arg1, TCGv_i64 arg2) 2269 { 2270 if (TCG_TARGET_REG_BITS == 32) { 2271 tcg_gen_andc_i32(TCGV_LOW(ret), TCGV_LOW(arg1), TCGV_LOW(arg2)); 2272 tcg_gen_andc_i32(TCGV_HIGH(ret), TCGV_HIGH(arg1), TCGV_HIGH(arg2)); 2273 } else if (tcg_op_supported(INDEX_op_andc, TCG_TYPE_I64, 0)) { 2274 tcg_gen_op3_i64(INDEX_op_andc, ret, arg1, arg2); 2275 } else { 2276 TCGv_i64 t0 = tcg_temp_ebb_new_i64(); 2277 tcg_gen_not_i64(t0, arg2); 2278 tcg_gen_and_i64(ret, arg1, t0); 2279 tcg_temp_free_i64(t0); 2280 } 2281 } 2282 2283 void tcg_gen_eqv_i64(TCGv_i64 ret, TCGv_i64 arg1, TCGv_i64 arg2) 2284 { 2285 if (TCG_TARGET_REG_BITS == 32) { 2286 tcg_gen_eqv_i32(TCGV_LOW(ret), TCGV_LOW(arg1), TCGV_LOW(arg2)); 2287 tcg_gen_eqv_i32(TCGV_HIGH(ret), TCGV_HIGH(arg1), TCGV_HIGH(arg2)); 2288 } else if (tcg_op_supported(INDEX_op_eqv, TCG_TYPE_I64, 0)) { 2289 tcg_gen_op3_i64(INDEX_op_eqv, ret, arg1, arg2); 2290 } else { 2291 tcg_gen_xor_i64(ret, arg1, arg2); 2292 tcg_gen_not_i64(ret, ret); 2293 } 2294 } 2295 2296 void tcg_gen_nand_i64(TCGv_i64 ret, TCGv_i64 arg1, TCGv_i64 arg2) 2297 { 2298 if (TCG_TARGET_REG_BITS == 32) { 2299 tcg_gen_nand_i32(TCGV_LOW(ret), TCGV_LOW(arg1), TCGV_LOW(arg2)); 2300 tcg_gen_nand_i32(TCGV_HIGH(ret), TCGV_HIGH(arg1), TCGV_HIGH(arg2)); 2301 } else if (tcg_op_supported(INDEX_op_nand, TCG_TYPE_I64, 0)) { 2302 tcg_gen_op3_i64(INDEX_op_nand, ret, arg1, arg2); 2303 } else { 2304 tcg_gen_and_i64(ret, arg1, arg2); 2305 tcg_gen_not_i64(ret, ret); 2306 } 2307 } 2308 2309 void tcg_gen_nor_i64(TCGv_i64 ret, TCGv_i64 arg1, TCGv_i64 arg2) 2310 { 2311 if (TCG_TARGET_REG_BITS == 32) { 2312 tcg_gen_nor_i32(TCGV_LOW(ret), TCGV_LOW(arg1), TCGV_LOW(arg2)); 2313 tcg_gen_nor_i32(TCGV_HIGH(ret), TCGV_HIGH(arg1), TCGV_HIGH(arg2)); 2314 } else if (tcg_op_supported(INDEX_op_nor, TCG_TYPE_I64, 0)) { 2315 tcg_gen_op3_i64(INDEX_op_nor, ret, arg1, arg2); 2316 } else { 2317 tcg_gen_or_i64(ret, arg1, arg2); 2318 tcg_gen_not_i64(ret, ret); 2319 } 2320 } 2321 2322 void tcg_gen_orc_i64(TCGv_i64 ret, TCGv_i64 arg1, TCGv_i64 arg2) 2323 { 2324 if (TCG_TARGET_REG_BITS == 32) { 2325 tcg_gen_orc_i32(TCGV_LOW(ret), TCGV_LOW(arg1), TCGV_LOW(arg2)); 2326 tcg_gen_orc_i32(TCGV_HIGH(ret), TCGV_HIGH(arg1), TCGV_HIGH(arg2)); 2327 } else if (tcg_op_supported(INDEX_op_orc, TCG_TYPE_I64, 0)) { 2328 tcg_gen_op3_i64(INDEX_op_orc, ret, arg1, arg2); 2329 } else { 2330 TCGv_i64 t0 = tcg_temp_ebb_new_i64(); 2331 tcg_gen_not_i64(t0, arg2); 2332 tcg_gen_or_i64(ret, arg1, t0); 2333 tcg_temp_free_i64(t0); 2334 } 2335 } 2336 2337 void tcg_gen_clz_i64(TCGv_i64 ret, TCGv_i64 arg1, TCGv_i64 arg2) 2338 { 2339 if (TCG_TARGET_HAS_clz_i64) { 2340 tcg_gen_op3_i64(INDEX_op_clz_i64, ret, arg1, arg2); 2341 } else { 2342 gen_helper_clz_i64(ret, arg1, arg2); 2343 } 2344 } 2345 2346 void tcg_gen_clzi_i64(TCGv_i64 ret, TCGv_i64 arg1, uint64_t arg2) 2347 { 2348 if (TCG_TARGET_REG_BITS == 32 2349 && TCG_TARGET_HAS_clz_i32 2350 && arg2 <= 0xffffffffu) { 2351 TCGv_i32 t = tcg_temp_ebb_new_i32(); 2352 tcg_gen_clzi_i32(t, TCGV_LOW(arg1), arg2 - 32); 2353 tcg_gen_addi_i32(t, t, 32); 2354 tcg_gen_clz_i32(TCGV_LOW(ret), TCGV_HIGH(arg1), t); 2355 tcg_gen_movi_i32(TCGV_HIGH(ret), 0); 2356 tcg_temp_free_i32(t); 2357 } else { 2358 tcg_gen_clz_i64(ret, arg1, tcg_constant_i64(arg2)); 2359 } 2360 } 2361 2362 void tcg_gen_ctz_i64(TCGv_i64 ret, TCGv_i64 arg1, TCGv_i64 arg2) 2363 { 2364 if (TCG_TARGET_HAS_ctz_i64) { 2365 tcg_gen_op3_i64(INDEX_op_ctz_i64, ret, arg1, arg2); 2366 } else if (TCG_TARGET_HAS_ctpop_i64 || TCG_TARGET_HAS_clz_i64) { 2367 TCGv_i64 z, t = tcg_temp_ebb_new_i64(); 2368 2369 if (TCG_TARGET_HAS_ctpop_i64) { 2370 tcg_gen_subi_i64(t, arg1, 1); 2371 tcg_gen_andc_i64(t, t, arg1); 2372 tcg_gen_ctpop_i64(t, t); 2373 } else { 2374 /* Since all non-x86 hosts have clz(0) == 64, don't fight it. */ 2375 tcg_gen_neg_i64(t, arg1); 2376 tcg_gen_and_i64(t, t, arg1); 2377 tcg_gen_clzi_i64(t, t, 64); 2378 tcg_gen_xori_i64(t, t, 63); 2379 } 2380 z = tcg_constant_i64(0); 2381 tcg_gen_movcond_i64(TCG_COND_EQ, ret, arg1, z, arg2, t); 2382 tcg_temp_free_i64(t); 2383 tcg_temp_free_i64(z); 2384 } else { 2385 gen_helper_ctz_i64(ret, arg1, arg2); 2386 } 2387 } 2388 2389 void tcg_gen_ctzi_i64(TCGv_i64 ret, TCGv_i64 arg1, uint64_t arg2) 2390 { 2391 if (TCG_TARGET_REG_BITS == 32 2392 && TCG_TARGET_HAS_ctz_i32 2393 && arg2 <= 0xffffffffu) { 2394 TCGv_i32 t32 = tcg_temp_ebb_new_i32(); 2395 tcg_gen_ctzi_i32(t32, TCGV_HIGH(arg1), arg2 - 32); 2396 tcg_gen_addi_i32(t32, t32, 32); 2397 tcg_gen_ctz_i32(TCGV_LOW(ret), TCGV_LOW(arg1), t32); 2398 tcg_gen_movi_i32(TCGV_HIGH(ret), 0); 2399 tcg_temp_free_i32(t32); 2400 } else if (!TCG_TARGET_HAS_ctz_i64 2401 && TCG_TARGET_HAS_ctpop_i64 2402 && arg2 == 64) { 2403 /* This equivalence has the advantage of not requiring a fixup. */ 2404 TCGv_i64 t = tcg_temp_ebb_new_i64(); 2405 tcg_gen_subi_i64(t, arg1, 1); 2406 tcg_gen_andc_i64(t, t, arg1); 2407 tcg_gen_ctpop_i64(ret, t); 2408 tcg_temp_free_i64(t); 2409 } else { 2410 tcg_gen_ctz_i64(ret, arg1, tcg_constant_i64(arg2)); 2411 } 2412 } 2413 2414 void tcg_gen_clrsb_i64(TCGv_i64 ret, TCGv_i64 arg) 2415 { 2416 if (TCG_TARGET_HAS_clz_i64 || TCG_TARGET_HAS_clz_i32) { 2417 TCGv_i64 t = tcg_temp_ebb_new_i64(); 2418 tcg_gen_sari_i64(t, arg, 63); 2419 tcg_gen_xor_i64(t, t, arg); 2420 tcg_gen_clzi_i64(t, t, 64); 2421 tcg_gen_subi_i64(ret, t, 1); 2422 tcg_temp_free_i64(t); 2423 } else { 2424 gen_helper_clrsb_i64(ret, arg); 2425 } 2426 } 2427 2428 void tcg_gen_ctpop_i64(TCGv_i64 ret, TCGv_i64 arg1) 2429 { 2430 if (TCG_TARGET_HAS_ctpop_i64) { 2431 tcg_gen_op2_i64(INDEX_op_ctpop_i64, ret, arg1); 2432 } else if (TCG_TARGET_REG_BITS == 32 && TCG_TARGET_HAS_ctpop_i32) { 2433 tcg_gen_ctpop_i32(TCGV_HIGH(ret), TCGV_HIGH(arg1)); 2434 tcg_gen_ctpop_i32(TCGV_LOW(ret), TCGV_LOW(arg1)); 2435 tcg_gen_add_i32(TCGV_LOW(ret), TCGV_LOW(ret), TCGV_HIGH(ret)); 2436 tcg_gen_movi_i32(TCGV_HIGH(ret), 0); 2437 } else { 2438 gen_helper_ctpop_i64(ret, arg1); 2439 } 2440 } 2441 2442 void tcg_gen_rotl_i64(TCGv_i64 ret, TCGv_i64 arg1, TCGv_i64 arg2) 2443 { 2444 if (tcg_op_supported(INDEX_op_rotl, TCG_TYPE_I64, 0)) { 2445 tcg_gen_op3_i64(INDEX_op_rotl, ret, arg1, arg2); 2446 } else if (tcg_op_supported(INDEX_op_rotl, TCG_TYPE_I64, 0)) { 2447 TCGv_i64 t0 = tcg_temp_ebb_new_i64(); 2448 tcg_gen_neg_i64(t0, arg2); 2449 tcg_gen_op3_i64(INDEX_op_rotr, ret, arg1, t0); 2450 tcg_temp_free_i64(t0); 2451 } else { 2452 TCGv_i64 t0 = tcg_temp_ebb_new_i64(); 2453 TCGv_i64 t1 = tcg_temp_ebb_new_i64(); 2454 tcg_gen_shl_i64(t0, arg1, arg2); 2455 tcg_gen_neg_i64(t1, arg2); 2456 tcg_gen_shr_i64(t1, arg1, t1); 2457 tcg_gen_or_i64(ret, t0, t1); 2458 tcg_temp_free_i64(t0); 2459 tcg_temp_free_i64(t1); 2460 } 2461 } 2462 2463 void tcg_gen_rotli_i64(TCGv_i64 ret, TCGv_i64 arg1, int64_t arg2) 2464 { 2465 tcg_debug_assert(arg2 >= 0 && arg2 < 64); 2466 /* some cases can be optimized here */ 2467 if (arg2 == 0) { 2468 tcg_gen_mov_i64(ret, arg1); 2469 } else if (tcg_op_supported(INDEX_op_rotl, TCG_TYPE_I64, 0)) { 2470 TCGv_i64 t0 = tcg_constant_i64(arg2); 2471 tcg_gen_op3_i64(INDEX_op_rotl, ret, arg1, t0); 2472 } else if (tcg_op_supported(INDEX_op_rotr, TCG_TYPE_I64, 0)) { 2473 TCGv_i64 t0 = tcg_constant_i64(64 - arg2); 2474 tcg_gen_op3_i64(INDEX_op_rotr, ret, arg1, t0); 2475 } else { 2476 TCGv_i64 t0 = tcg_temp_ebb_new_i64(); 2477 TCGv_i64 t1 = tcg_temp_ebb_new_i64(); 2478 tcg_gen_shli_i64(t0, arg1, arg2); 2479 tcg_gen_shri_i64(t1, arg1, 64 - arg2); 2480 tcg_gen_or_i64(ret, t0, t1); 2481 tcg_temp_free_i64(t0); 2482 tcg_temp_free_i64(t1); 2483 } 2484 } 2485 2486 void tcg_gen_rotr_i64(TCGv_i64 ret, TCGv_i64 arg1, TCGv_i64 arg2) 2487 { 2488 if (tcg_op_supported(INDEX_op_rotr, TCG_TYPE_I64, 0)) { 2489 tcg_gen_op3_i64(INDEX_op_rotr, ret, arg1, arg2); 2490 } else if (tcg_op_supported(INDEX_op_rotl, TCG_TYPE_I64, 0)) { 2491 TCGv_i64 t0 = tcg_temp_ebb_new_i64(); 2492 tcg_gen_neg_i64(t0, arg2); 2493 tcg_gen_op3_i64(INDEX_op_rotl, ret, arg1, t0); 2494 tcg_temp_free_i64(t0); 2495 } else { 2496 TCGv_i64 t0 = tcg_temp_ebb_new_i64(); 2497 TCGv_i64 t1 = tcg_temp_ebb_new_i64(); 2498 tcg_gen_shr_i64(t0, arg1, arg2); 2499 tcg_gen_neg_i64(t1, arg2); 2500 tcg_gen_shl_i64(t1, arg1, t1); 2501 tcg_gen_or_i64(ret, t0, t1); 2502 tcg_temp_free_i64(t0); 2503 tcg_temp_free_i64(t1); 2504 } 2505 } 2506 2507 void tcg_gen_rotri_i64(TCGv_i64 ret, TCGv_i64 arg1, int64_t arg2) 2508 { 2509 tcg_debug_assert(arg2 >= 0 && arg2 < 64); 2510 tcg_gen_rotli_i64(ret, arg1, -arg2 & 63); 2511 } 2512 2513 void tcg_gen_deposit_i64(TCGv_i64 ret, TCGv_i64 arg1, TCGv_i64 arg2, 2514 unsigned int ofs, unsigned int len) 2515 { 2516 uint64_t mask; 2517 TCGv_i64 t1; 2518 2519 tcg_debug_assert(ofs < 64); 2520 tcg_debug_assert(len > 0); 2521 tcg_debug_assert(len <= 64); 2522 tcg_debug_assert(ofs + len <= 64); 2523 2524 if (len == 64) { 2525 tcg_gen_mov_i64(ret, arg2); 2526 return; 2527 } 2528 2529 if (TCG_TARGET_REG_BITS == 64) { 2530 if (TCG_TARGET_deposit_valid(TCG_TYPE_I64, ofs, len)) { 2531 tcg_gen_op5ii_i64(INDEX_op_deposit_i64, ret, arg1, arg2, ofs, len); 2532 return; 2533 } 2534 } else { 2535 if (ofs >= 32) { 2536 tcg_gen_deposit_i32(TCGV_HIGH(ret), TCGV_HIGH(arg1), 2537 TCGV_LOW(arg2), ofs - 32, len); 2538 tcg_gen_mov_i32(TCGV_LOW(ret), TCGV_LOW(arg1)); 2539 return; 2540 } 2541 if (ofs + len <= 32) { 2542 tcg_gen_deposit_i32(TCGV_LOW(ret), TCGV_LOW(arg1), 2543 TCGV_LOW(arg2), ofs, len); 2544 tcg_gen_mov_i32(TCGV_HIGH(ret), TCGV_HIGH(arg1)); 2545 return; 2546 } 2547 } 2548 2549 t1 = tcg_temp_ebb_new_i64(); 2550 2551 if (TCG_TARGET_HAS_extract2_i64) { 2552 if (ofs + len == 64) { 2553 tcg_gen_shli_i64(t1, arg1, len); 2554 tcg_gen_extract2_i64(ret, t1, arg2, len); 2555 goto done; 2556 } 2557 if (ofs == 0) { 2558 tcg_gen_extract2_i64(ret, arg1, arg2, len); 2559 tcg_gen_rotli_i64(ret, ret, len); 2560 goto done; 2561 } 2562 } 2563 2564 mask = (1ull << len) - 1; 2565 if (ofs + len < 64) { 2566 tcg_gen_andi_i64(t1, arg2, mask); 2567 tcg_gen_shli_i64(t1, t1, ofs); 2568 } else { 2569 tcg_gen_shli_i64(t1, arg2, ofs); 2570 } 2571 tcg_gen_andi_i64(ret, arg1, ~(mask << ofs)); 2572 tcg_gen_or_i64(ret, ret, t1); 2573 done: 2574 tcg_temp_free_i64(t1); 2575 } 2576 2577 void tcg_gen_deposit_z_i64(TCGv_i64 ret, TCGv_i64 arg, 2578 unsigned int ofs, unsigned int len) 2579 { 2580 tcg_debug_assert(ofs < 64); 2581 tcg_debug_assert(len > 0); 2582 tcg_debug_assert(len <= 64); 2583 tcg_debug_assert(ofs + len <= 64); 2584 2585 if (ofs + len == 64) { 2586 tcg_gen_shli_i64(ret, arg, ofs); 2587 } else if (ofs == 0) { 2588 tcg_gen_andi_i64(ret, arg, (1ull << len) - 1); 2589 } else if (TCG_TARGET_REG_BITS == 64 && 2590 TCG_TARGET_deposit_valid(TCG_TYPE_I64, ofs, len)) { 2591 TCGv_i64 zero = tcg_constant_i64(0); 2592 tcg_gen_op5ii_i64(INDEX_op_deposit_i64, ret, zero, arg, ofs, len); 2593 } else { 2594 if (TCG_TARGET_REG_BITS == 32) { 2595 if (ofs >= 32) { 2596 tcg_gen_deposit_z_i32(TCGV_HIGH(ret), TCGV_LOW(arg), 2597 ofs - 32, len); 2598 tcg_gen_movi_i32(TCGV_LOW(ret), 0); 2599 return; 2600 } 2601 if (ofs + len <= 32) { 2602 tcg_gen_deposit_z_i32(TCGV_LOW(ret), TCGV_LOW(arg), ofs, len); 2603 tcg_gen_movi_i32(TCGV_HIGH(ret), 0); 2604 return; 2605 } 2606 } 2607 /* 2608 * To help two-operand hosts we prefer to zero-extend first, 2609 * which allows ARG to stay live. 2610 */ 2611 if (TCG_TARGET_extract_valid(TCG_TYPE_I64, 0, len)) { 2612 tcg_gen_extract_i64(ret, arg, 0, len); 2613 tcg_gen_shli_i64(ret, ret, ofs); 2614 return; 2615 } 2616 /* Otherwise prefer zero-extension over AND for code size. */ 2617 if (TCG_TARGET_extract_valid(TCG_TYPE_I64, 0, ofs + len)) { 2618 tcg_gen_shli_i64(ret, arg, ofs); 2619 tcg_gen_extract_i64(ret, ret, 0, ofs + len); 2620 return; 2621 } 2622 tcg_gen_andi_i64(ret, arg, (1ull << len) - 1); 2623 tcg_gen_shli_i64(ret, ret, ofs); 2624 } 2625 } 2626 2627 void tcg_gen_extract_i64(TCGv_i64 ret, TCGv_i64 arg, 2628 unsigned int ofs, unsigned int len) 2629 { 2630 tcg_debug_assert(ofs < 64); 2631 tcg_debug_assert(len > 0); 2632 tcg_debug_assert(len <= 64); 2633 tcg_debug_assert(ofs + len <= 64); 2634 2635 /* Canonicalize certain special cases, even if extract is supported. */ 2636 if (ofs + len == 64) { 2637 tcg_gen_shri_i64(ret, arg, 64 - len); 2638 return; 2639 } 2640 2641 if (TCG_TARGET_REG_BITS == 32) { 2642 /* Look for a 32-bit extract within one of the two words. */ 2643 if (ofs >= 32) { 2644 tcg_gen_extract_i32(TCGV_LOW(ret), TCGV_HIGH(arg), ofs - 32, len); 2645 tcg_gen_movi_i32(TCGV_HIGH(ret), 0); 2646 return; 2647 } 2648 if (ofs + len <= 32) { 2649 tcg_gen_extract_i32(TCGV_LOW(ret), TCGV_LOW(arg), ofs, len); 2650 tcg_gen_movi_i32(TCGV_HIGH(ret), 0); 2651 return; 2652 } 2653 2654 /* The field is split across two words. */ 2655 tcg_gen_extract2_i32(TCGV_LOW(ret), TCGV_LOW(arg), 2656 TCGV_HIGH(arg), ofs); 2657 if (len <= 32) { 2658 tcg_gen_extract_i32(TCGV_LOW(ret), TCGV_LOW(ret), 0, len); 2659 tcg_gen_movi_i32(TCGV_HIGH(ret), 0); 2660 } else { 2661 tcg_gen_extract_i32(TCGV_HIGH(ret), TCGV_HIGH(arg), 2662 ofs, len - 32); 2663 } 2664 return; 2665 } 2666 2667 if (TCG_TARGET_extract_valid(TCG_TYPE_I64, ofs, len)) { 2668 tcg_gen_op4ii_i64(INDEX_op_extract_i64, ret, arg, ofs, len); 2669 return; 2670 } 2671 if (ofs == 0) { 2672 tcg_gen_andi_i64(ret, arg, (1ull << len) - 1); 2673 return; 2674 } 2675 2676 /* Assume that zero-extension, if available, is cheaper than a shift. */ 2677 if (TCG_TARGET_extract_valid(TCG_TYPE_I64, 0, ofs + len)) { 2678 tcg_gen_op4ii_i64(INDEX_op_extract_i64, ret, arg, 0, ofs + len); 2679 tcg_gen_shri_i64(ret, ret, ofs); 2680 return; 2681 } 2682 2683 /* ??? Ideally we'd know what values are available for immediate AND. 2684 Assume that 8 bits are available, plus the special cases of 16 and 32, 2685 so that we get ext8u, ext16u, and ext32u. */ 2686 switch (len) { 2687 case 1 ... 8: case 16: case 32: 2688 tcg_gen_shri_i64(ret, arg, ofs); 2689 tcg_gen_andi_i64(ret, ret, (1ull << len) - 1); 2690 break; 2691 default: 2692 tcg_gen_shli_i64(ret, arg, 64 - len - ofs); 2693 tcg_gen_shri_i64(ret, ret, 64 - len); 2694 break; 2695 } 2696 } 2697 2698 void tcg_gen_sextract_i64(TCGv_i64 ret, TCGv_i64 arg, 2699 unsigned int ofs, unsigned int len) 2700 { 2701 tcg_debug_assert(ofs < 64); 2702 tcg_debug_assert(len > 0); 2703 tcg_debug_assert(len <= 64); 2704 tcg_debug_assert(ofs + len <= 64); 2705 2706 /* Canonicalize certain special cases, even if sextract is supported. */ 2707 if (ofs + len == 64) { 2708 tcg_gen_sari_i64(ret, arg, 64 - len); 2709 return; 2710 } 2711 2712 if (TCG_TARGET_REG_BITS == 32) { 2713 /* Look for a 32-bit extract within one of the two words. */ 2714 if (ofs >= 32) { 2715 tcg_gen_sextract_i32(TCGV_LOW(ret), TCGV_HIGH(arg), ofs - 32, len); 2716 } else if (ofs + len <= 32) { 2717 tcg_gen_sextract_i32(TCGV_LOW(ret), TCGV_LOW(arg), ofs, len); 2718 } else if (ofs == 0) { 2719 tcg_gen_mov_i32(TCGV_LOW(ret), TCGV_LOW(arg)); 2720 tcg_gen_sextract_i32(TCGV_HIGH(ret), TCGV_HIGH(arg), 0, len - 32); 2721 return; 2722 } else if (len > 32) { 2723 TCGv_i32 t = tcg_temp_ebb_new_i32(); 2724 /* Extract the bits for the high word normally. */ 2725 tcg_gen_sextract_i32(t, TCGV_HIGH(arg), ofs + 32, len - 32); 2726 /* Shift the field down for the low part. */ 2727 tcg_gen_shri_i64(ret, arg, ofs); 2728 /* Overwrite the shift into the high part. */ 2729 tcg_gen_mov_i32(TCGV_HIGH(ret), t); 2730 tcg_temp_free_i32(t); 2731 return; 2732 } else { 2733 /* Shift the field down for the low part, such that the 2734 field sits at the MSB. */ 2735 tcg_gen_shri_i64(ret, arg, ofs + len - 32); 2736 /* Shift the field down from the MSB, sign extending. */ 2737 tcg_gen_sari_i32(TCGV_LOW(ret), TCGV_LOW(ret), 32 - len); 2738 } 2739 /* Sign-extend the field from 32 bits. */ 2740 tcg_gen_sari_i32(TCGV_HIGH(ret), TCGV_LOW(ret), 31); 2741 return; 2742 } 2743 2744 if (TCG_TARGET_sextract_valid(TCG_TYPE_I64, ofs, len)) { 2745 tcg_gen_op4ii_i64(INDEX_op_sextract_i64, ret, arg, ofs, len); 2746 return; 2747 } 2748 2749 /* Assume that sign-extension, if available, is cheaper than a shift. */ 2750 if (TCG_TARGET_sextract_valid(TCG_TYPE_I64, 0, ofs + len)) { 2751 tcg_gen_op4ii_i64(INDEX_op_sextract_i64, ret, arg, 0, ofs + len); 2752 tcg_gen_sari_i64(ret, ret, ofs); 2753 return; 2754 } 2755 if (TCG_TARGET_sextract_valid(TCG_TYPE_I64, 0, len)) { 2756 tcg_gen_shri_i64(ret, arg, ofs); 2757 tcg_gen_op4ii_i64(INDEX_op_sextract_i64, ret, ret, 0, len); 2758 return; 2759 } 2760 2761 tcg_gen_shli_i64(ret, arg, 64 - len - ofs); 2762 tcg_gen_sari_i64(ret, ret, 64 - len); 2763 } 2764 2765 /* 2766 * Extract 64 bits from a 128-bit input, ah:al, starting from ofs. 2767 * Unlike tcg_gen_extract_i64 above, len is fixed at 64. 2768 */ 2769 void tcg_gen_extract2_i64(TCGv_i64 ret, TCGv_i64 al, TCGv_i64 ah, 2770 unsigned int ofs) 2771 { 2772 tcg_debug_assert(ofs <= 64); 2773 if (ofs == 0) { 2774 tcg_gen_mov_i64(ret, al); 2775 } else if (ofs == 64) { 2776 tcg_gen_mov_i64(ret, ah); 2777 } else if (al == ah) { 2778 tcg_gen_rotri_i64(ret, al, ofs); 2779 } else if (TCG_TARGET_HAS_extract2_i64) { 2780 tcg_gen_op4i_i64(INDEX_op_extract2_i64, ret, al, ah, ofs); 2781 } else { 2782 TCGv_i64 t0 = tcg_temp_ebb_new_i64(); 2783 tcg_gen_shri_i64(t0, al, ofs); 2784 tcg_gen_deposit_i64(ret, t0, ah, 64 - ofs, ofs); 2785 tcg_temp_free_i64(t0); 2786 } 2787 } 2788 2789 void tcg_gen_movcond_i64(TCGCond cond, TCGv_i64 ret, TCGv_i64 c1, 2790 TCGv_i64 c2, TCGv_i64 v1, TCGv_i64 v2) 2791 { 2792 if (cond == TCG_COND_ALWAYS) { 2793 tcg_gen_mov_i64(ret, v1); 2794 } else if (cond == TCG_COND_NEVER) { 2795 tcg_gen_mov_i64(ret, v2); 2796 } else if (TCG_TARGET_REG_BITS == 64) { 2797 tcg_gen_op6i_i64(INDEX_op_movcond_i64, ret, c1, c2, v1, v2, cond); 2798 } else { 2799 TCGv_i32 t0 = tcg_temp_ebb_new_i32(); 2800 TCGv_i32 zero = tcg_constant_i32(0); 2801 2802 tcg_gen_op6i_i32(INDEX_op_setcond2_i32, t0, 2803 TCGV_LOW(c1), TCGV_HIGH(c1), 2804 TCGV_LOW(c2), TCGV_HIGH(c2), cond); 2805 2806 tcg_gen_movcond_i32(TCG_COND_NE, TCGV_LOW(ret), t0, zero, 2807 TCGV_LOW(v1), TCGV_LOW(v2)); 2808 tcg_gen_movcond_i32(TCG_COND_NE, TCGV_HIGH(ret), t0, zero, 2809 TCGV_HIGH(v1), TCGV_HIGH(v2)); 2810 2811 tcg_temp_free_i32(t0); 2812 } 2813 } 2814 2815 void tcg_gen_add2_i64(TCGv_i64 rl, TCGv_i64 rh, TCGv_i64 al, 2816 TCGv_i64 ah, TCGv_i64 bl, TCGv_i64 bh) 2817 { 2818 if (TCG_TARGET_HAS_add2_i64) { 2819 tcg_gen_op6_i64(INDEX_op_add2_i64, rl, rh, al, ah, bl, bh); 2820 } else { 2821 TCGv_i64 t0 = tcg_temp_ebb_new_i64(); 2822 TCGv_i64 t1 = tcg_temp_ebb_new_i64(); 2823 tcg_gen_add_i64(t0, al, bl); 2824 tcg_gen_setcond_i64(TCG_COND_LTU, t1, t0, al); 2825 tcg_gen_add_i64(rh, ah, bh); 2826 tcg_gen_add_i64(rh, rh, t1); 2827 tcg_gen_mov_i64(rl, t0); 2828 tcg_temp_free_i64(t0); 2829 tcg_temp_free_i64(t1); 2830 } 2831 } 2832 2833 void tcg_gen_sub2_i64(TCGv_i64 rl, TCGv_i64 rh, TCGv_i64 al, 2834 TCGv_i64 ah, TCGv_i64 bl, TCGv_i64 bh) 2835 { 2836 if (TCG_TARGET_HAS_sub2_i64) { 2837 tcg_gen_op6_i64(INDEX_op_sub2_i64, rl, rh, al, ah, bl, bh); 2838 } else { 2839 TCGv_i64 t0 = tcg_temp_ebb_new_i64(); 2840 TCGv_i64 t1 = tcg_temp_ebb_new_i64(); 2841 tcg_gen_sub_i64(t0, al, bl); 2842 tcg_gen_setcond_i64(TCG_COND_LTU, t1, al, bl); 2843 tcg_gen_sub_i64(rh, ah, bh); 2844 tcg_gen_sub_i64(rh, rh, t1); 2845 tcg_gen_mov_i64(rl, t0); 2846 tcg_temp_free_i64(t0); 2847 tcg_temp_free_i64(t1); 2848 } 2849 } 2850 2851 void tcg_gen_mulu2_i64(TCGv_i64 rl, TCGv_i64 rh, TCGv_i64 arg1, TCGv_i64 arg2) 2852 { 2853 if (TCG_TARGET_HAS_mulu2_i64) { 2854 tcg_gen_op4_i64(INDEX_op_mulu2_i64, rl, rh, arg1, arg2); 2855 } else if (tcg_op_supported(INDEX_op_muluh, TCG_TYPE_I64, 0)) { 2856 TCGv_i64 t = tcg_temp_ebb_new_i64(); 2857 tcg_gen_op3_i64(INDEX_op_mul, t, arg1, arg2); 2858 tcg_gen_op3_i64(INDEX_op_muluh, rh, arg1, arg2); 2859 tcg_gen_mov_i64(rl, t); 2860 tcg_temp_free_i64(t); 2861 } else { 2862 TCGv_i64 t0 = tcg_temp_ebb_new_i64(); 2863 tcg_gen_mul_i64(t0, arg1, arg2); 2864 gen_helper_muluh_i64(rh, arg1, arg2); 2865 tcg_gen_mov_i64(rl, t0); 2866 tcg_temp_free_i64(t0); 2867 } 2868 } 2869 2870 void tcg_gen_muls2_i64(TCGv_i64 rl, TCGv_i64 rh, TCGv_i64 arg1, TCGv_i64 arg2) 2871 { 2872 if (TCG_TARGET_HAS_muls2_i64) { 2873 tcg_gen_op4_i64(INDEX_op_muls2_i64, rl, rh, arg1, arg2); 2874 } else if (tcg_op_supported(INDEX_op_mulsh, TCG_TYPE_I64, 0)) { 2875 TCGv_i64 t = tcg_temp_ebb_new_i64(); 2876 tcg_gen_op3_i64(INDEX_op_mul, t, arg1, arg2); 2877 tcg_gen_op3_i64(INDEX_op_mulsh, rh, arg1, arg2); 2878 tcg_gen_mov_i64(rl, t); 2879 tcg_temp_free_i64(t); 2880 } else if (TCG_TARGET_HAS_mulu2_i64 || 2881 tcg_op_supported(INDEX_op_muluh, TCG_TYPE_I64, 0)) { 2882 TCGv_i64 t0 = tcg_temp_ebb_new_i64(); 2883 TCGv_i64 t1 = tcg_temp_ebb_new_i64(); 2884 TCGv_i64 t2 = tcg_temp_ebb_new_i64(); 2885 TCGv_i64 t3 = tcg_temp_ebb_new_i64(); 2886 tcg_gen_mulu2_i64(t0, t1, arg1, arg2); 2887 /* Adjust for negative inputs. */ 2888 tcg_gen_sari_i64(t2, arg1, 63); 2889 tcg_gen_sari_i64(t3, arg2, 63); 2890 tcg_gen_and_i64(t2, t2, arg2); 2891 tcg_gen_and_i64(t3, t3, arg1); 2892 tcg_gen_sub_i64(rh, t1, t2); 2893 tcg_gen_sub_i64(rh, rh, t3); 2894 tcg_gen_mov_i64(rl, t0); 2895 tcg_temp_free_i64(t0); 2896 tcg_temp_free_i64(t1); 2897 tcg_temp_free_i64(t2); 2898 tcg_temp_free_i64(t3); 2899 } else { 2900 TCGv_i64 t0 = tcg_temp_ebb_new_i64(); 2901 tcg_gen_mul_i64(t0, arg1, arg2); 2902 gen_helper_mulsh_i64(rh, arg1, arg2); 2903 tcg_gen_mov_i64(rl, t0); 2904 tcg_temp_free_i64(t0); 2905 } 2906 } 2907 2908 void tcg_gen_mulsu2_i64(TCGv_i64 rl, TCGv_i64 rh, TCGv_i64 arg1, TCGv_i64 arg2) 2909 { 2910 TCGv_i64 t0 = tcg_temp_ebb_new_i64(); 2911 TCGv_i64 t1 = tcg_temp_ebb_new_i64(); 2912 TCGv_i64 t2 = tcg_temp_ebb_new_i64(); 2913 tcg_gen_mulu2_i64(t0, t1, arg1, arg2); 2914 /* Adjust for negative input for the signed arg1. */ 2915 tcg_gen_sari_i64(t2, arg1, 63); 2916 tcg_gen_and_i64(t2, t2, arg2); 2917 tcg_gen_sub_i64(rh, t1, t2); 2918 tcg_gen_mov_i64(rl, t0); 2919 tcg_temp_free_i64(t0); 2920 tcg_temp_free_i64(t1); 2921 tcg_temp_free_i64(t2); 2922 } 2923 2924 void tcg_gen_smin_i64(TCGv_i64 ret, TCGv_i64 a, TCGv_i64 b) 2925 { 2926 tcg_gen_movcond_i64(TCG_COND_LT, ret, a, b, a, b); 2927 } 2928 2929 void tcg_gen_umin_i64(TCGv_i64 ret, TCGv_i64 a, TCGv_i64 b) 2930 { 2931 tcg_gen_movcond_i64(TCG_COND_LTU, ret, a, b, a, b); 2932 } 2933 2934 void tcg_gen_smax_i64(TCGv_i64 ret, TCGv_i64 a, TCGv_i64 b) 2935 { 2936 tcg_gen_movcond_i64(TCG_COND_LT, ret, a, b, b, a); 2937 } 2938 2939 void tcg_gen_umax_i64(TCGv_i64 ret, TCGv_i64 a, TCGv_i64 b) 2940 { 2941 tcg_gen_movcond_i64(TCG_COND_LTU, ret, a, b, b, a); 2942 } 2943 2944 void tcg_gen_abs_i64(TCGv_i64 ret, TCGv_i64 a) 2945 { 2946 TCGv_i64 t = tcg_temp_ebb_new_i64(); 2947 2948 tcg_gen_sari_i64(t, a, 63); 2949 tcg_gen_xor_i64(ret, a, t); 2950 tcg_gen_sub_i64(ret, ret, t); 2951 tcg_temp_free_i64(t); 2952 } 2953 2954 /* Size changing operations. */ 2955 2956 void tcg_gen_extrl_i64_i32(TCGv_i32 ret, TCGv_i64 arg) 2957 { 2958 if (TCG_TARGET_REG_BITS == 32) { 2959 tcg_gen_mov_i32(ret, TCGV_LOW(arg)); 2960 } else if (TCG_TARGET_HAS_extr_i64_i32) { 2961 tcg_gen_op2(INDEX_op_extrl_i64_i32, TCG_TYPE_I32, 2962 tcgv_i32_arg(ret), tcgv_i64_arg(arg)); 2963 } else { 2964 tcg_gen_mov_i32(ret, (TCGv_i32)arg); 2965 } 2966 } 2967 2968 void tcg_gen_extrh_i64_i32(TCGv_i32 ret, TCGv_i64 arg) 2969 { 2970 if (TCG_TARGET_REG_BITS == 32) { 2971 tcg_gen_mov_i32(ret, TCGV_HIGH(arg)); 2972 } else if (TCG_TARGET_HAS_extr_i64_i32) { 2973 tcg_gen_op2(INDEX_op_extrh_i64_i32, TCG_TYPE_I32, 2974 tcgv_i32_arg(ret), tcgv_i64_arg(arg)); 2975 } else { 2976 TCGv_i64 t = tcg_temp_ebb_new_i64(); 2977 tcg_gen_shri_i64(t, arg, 32); 2978 tcg_gen_mov_i32(ret, (TCGv_i32)t); 2979 tcg_temp_free_i64(t); 2980 } 2981 } 2982 2983 void tcg_gen_extu_i32_i64(TCGv_i64 ret, TCGv_i32 arg) 2984 { 2985 if (TCG_TARGET_REG_BITS == 32) { 2986 tcg_gen_mov_i32(TCGV_LOW(ret), arg); 2987 tcg_gen_movi_i32(TCGV_HIGH(ret), 0); 2988 } else { 2989 tcg_gen_op2(INDEX_op_extu_i32_i64, TCG_TYPE_I64, 2990 tcgv_i64_arg(ret), tcgv_i32_arg(arg)); 2991 } 2992 } 2993 2994 void tcg_gen_ext_i32_i64(TCGv_i64 ret, TCGv_i32 arg) 2995 { 2996 if (TCG_TARGET_REG_BITS == 32) { 2997 tcg_gen_mov_i32(TCGV_LOW(ret), arg); 2998 tcg_gen_sari_i32(TCGV_HIGH(ret), TCGV_LOW(ret), 31); 2999 } else { 3000 tcg_gen_op2(INDEX_op_ext_i32_i64, TCG_TYPE_I64, 3001 tcgv_i64_arg(ret), tcgv_i32_arg(arg)); 3002 } 3003 } 3004 3005 void tcg_gen_concat_i32_i64(TCGv_i64 dest, TCGv_i32 low, TCGv_i32 high) 3006 { 3007 TCGv_i64 tmp; 3008 3009 if (TCG_TARGET_REG_BITS == 32) { 3010 tcg_gen_mov_i32(TCGV_LOW(dest), low); 3011 tcg_gen_mov_i32(TCGV_HIGH(dest), high); 3012 return; 3013 } 3014 3015 tmp = tcg_temp_ebb_new_i64(); 3016 /* These extensions are only needed for type correctness. 3017 We may be able to do better given target specific information. */ 3018 tcg_gen_extu_i32_i64(tmp, high); 3019 tcg_gen_extu_i32_i64(dest, low); 3020 /* If deposit is available, use it. Otherwise use the extra 3021 knowledge that we have of the zero-extensions above. */ 3022 if (TCG_TARGET_deposit_valid(TCG_TYPE_I64, 32, 32)) { 3023 tcg_gen_deposit_i64(dest, dest, tmp, 32, 32); 3024 } else { 3025 tcg_gen_shli_i64(tmp, tmp, 32); 3026 tcg_gen_or_i64(dest, dest, tmp); 3027 } 3028 tcg_temp_free_i64(tmp); 3029 } 3030 3031 void tcg_gen_extr_i64_i32(TCGv_i32 lo, TCGv_i32 hi, TCGv_i64 arg) 3032 { 3033 if (TCG_TARGET_REG_BITS == 32) { 3034 tcg_gen_mov_i32(lo, TCGV_LOW(arg)); 3035 tcg_gen_mov_i32(hi, TCGV_HIGH(arg)); 3036 } else { 3037 tcg_gen_extrl_i64_i32(lo, arg); 3038 tcg_gen_extrh_i64_i32(hi, arg); 3039 } 3040 } 3041 3042 void tcg_gen_extr32_i64(TCGv_i64 lo, TCGv_i64 hi, TCGv_i64 arg) 3043 { 3044 tcg_gen_ext32u_i64(lo, arg); 3045 tcg_gen_shri_i64(hi, arg, 32); 3046 } 3047 3048 void tcg_gen_concat32_i64(TCGv_i64 ret, TCGv_i64 lo, TCGv_i64 hi) 3049 { 3050 tcg_gen_deposit_i64(ret, lo, hi, 32, 32); 3051 } 3052 3053 void tcg_gen_extr_i128_i64(TCGv_i64 lo, TCGv_i64 hi, TCGv_i128 arg) 3054 { 3055 tcg_gen_mov_i64(lo, TCGV128_LOW(arg)); 3056 tcg_gen_mov_i64(hi, TCGV128_HIGH(arg)); 3057 } 3058 3059 void tcg_gen_concat_i64_i128(TCGv_i128 ret, TCGv_i64 lo, TCGv_i64 hi) 3060 { 3061 tcg_gen_mov_i64(TCGV128_LOW(ret), lo); 3062 tcg_gen_mov_i64(TCGV128_HIGH(ret), hi); 3063 } 3064 3065 void tcg_gen_mov_i128(TCGv_i128 dst, TCGv_i128 src) 3066 { 3067 if (dst != src) { 3068 tcg_gen_mov_i64(TCGV128_LOW(dst), TCGV128_LOW(src)); 3069 tcg_gen_mov_i64(TCGV128_HIGH(dst), TCGV128_HIGH(src)); 3070 } 3071 } 3072 3073 void tcg_gen_ld_i128(TCGv_i128 ret, TCGv_ptr base, tcg_target_long offset) 3074 { 3075 if (HOST_BIG_ENDIAN) { 3076 tcg_gen_ld_i64(TCGV128_HIGH(ret), base, offset); 3077 tcg_gen_ld_i64(TCGV128_LOW(ret), base, offset + 8); 3078 } else { 3079 tcg_gen_ld_i64(TCGV128_LOW(ret), base, offset); 3080 tcg_gen_ld_i64(TCGV128_HIGH(ret), base, offset + 8); 3081 } 3082 } 3083 3084 void tcg_gen_st_i128(TCGv_i128 val, TCGv_ptr base, tcg_target_long offset) 3085 { 3086 if (HOST_BIG_ENDIAN) { 3087 tcg_gen_st_i64(TCGV128_HIGH(val), base, offset); 3088 tcg_gen_st_i64(TCGV128_LOW(val), base, offset + 8); 3089 } else { 3090 tcg_gen_st_i64(TCGV128_LOW(val), base, offset); 3091 tcg_gen_st_i64(TCGV128_HIGH(val), base, offset + 8); 3092 } 3093 } 3094 3095 /* QEMU specific operations. */ 3096 3097 void tcg_gen_exit_tb(const TranslationBlock *tb, unsigned idx) 3098 { 3099 /* 3100 * Let the jit code return the read-only version of the 3101 * TranslationBlock, so that we minimize the pc-relative 3102 * distance of the address of the exit_tb code to TB. 3103 * This will improve utilization of pc-relative address loads. 3104 * 3105 * TODO: Move this to translator_loop, so that all const 3106 * TranslationBlock pointers refer to read-only memory. 3107 * This requires coordination with targets that do not use 3108 * the translator_loop. 3109 */ 3110 uintptr_t val = (uintptr_t)tcg_splitwx_to_rx((void *)tb) + idx; 3111 3112 if (tb == NULL) { 3113 tcg_debug_assert(idx == 0); 3114 } else if (idx <= TB_EXIT_IDXMAX) { 3115 #ifdef CONFIG_DEBUG_TCG 3116 /* This is an exit following a goto_tb. Verify that we have 3117 seen this numbered exit before, via tcg_gen_goto_tb. */ 3118 tcg_debug_assert(tcg_ctx->goto_tb_issue_mask & (1 << idx)); 3119 #endif 3120 } else { 3121 /* This is an exit via the exitreq label. */ 3122 tcg_debug_assert(idx == TB_EXIT_REQUESTED); 3123 } 3124 3125 tcg_gen_op1i(INDEX_op_exit_tb, 0, val); 3126 } 3127 3128 void tcg_gen_goto_tb(unsigned idx) 3129 { 3130 /* We tested CF_NO_GOTO_TB in translator_use_goto_tb. */ 3131 tcg_debug_assert(!(tcg_ctx->gen_tb->cflags & CF_NO_GOTO_TB)); 3132 /* We only support two chained exits. */ 3133 tcg_debug_assert(idx <= TB_EXIT_IDXMAX); 3134 #ifdef CONFIG_DEBUG_TCG 3135 /* Verify that we haven't seen this numbered exit before. */ 3136 tcg_debug_assert((tcg_ctx->goto_tb_issue_mask & (1 << idx)) == 0); 3137 tcg_ctx->goto_tb_issue_mask |= 1 << idx; 3138 #endif 3139 plugin_gen_disable_mem_helpers(); 3140 tcg_gen_op1i(INDEX_op_goto_tb, 0, idx); 3141 } 3142 3143 void tcg_gen_lookup_and_goto_ptr(void) 3144 { 3145 TCGv_ptr ptr; 3146 3147 if (tcg_ctx->gen_tb->cflags & CF_NO_GOTO_PTR) { 3148 tcg_gen_exit_tb(NULL, 0); 3149 return; 3150 } 3151 3152 plugin_gen_disable_mem_helpers(); 3153 ptr = tcg_temp_ebb_new_ptr(); 3154 gen_helper_lookup_tb_ptr(ptr, tcg_env); 3155 tcg_gen_op1i(INDEX_op_goto_ptr, TCG_TYPE_PTR, tcgv_ptr_arg(ptr)); 3156 tcg_temp_free_ptr(ptr); 3157 } 3158