1 /* 2 * Tiny Code Generator for QEMU 3 * 4 * Copyright (c) 2008 Fabrice Bellard 5 * 6 * Permission is hereby granted, free of charge, to any person obtaining a copy 7 * of this software and associated documentation files (the "Software"), to deal 8 * in the Software without restriction, including without limitation the rights 9 * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 10 * copies of the Software, and to permit persons to whom the Software is 11 * furnished to do so, subject to the following conditions: 12 * 13 * The above copyright notice and this permission notice shall be included in 14 * all copies or substantial portions of the Software. 15 * 16 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 17 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 18 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL 19 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 20 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 21 * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN 22 * THE SOFTWARE. 23 */ 24 25 #include "qemu/osdep.h" 26 #include "tcg/tcg.h" 27 #include "tcg/tcg-temp-internal.h" 28 #include "tcg/tcg-op-common.h" 29 #include "exec/translation-block.h" 30 #include "exec/plugin-gen.h" 31 #include "tcg-internal.h" 32 #include "tcg-has.h" 33 34 /* 35 * Encourage the compiler to tail-call to a function, rather than inlining. 36 * Minimizes code size across 99 bottles of beer on the wall. 37 */ 38 #define NI __attribute__((noinline)) 39 40 TCGOp * NI tcg_gen_op1(TCGOpcode opc, TCGType type, TCGArg a1) 41 { 42 TCGOp *op = tcg_emit_op(opc, 1); 43 TCGOP_TYPE(op) = type; 44 op->args[0] = a1; 45 return op; 46 } 47 48 TCGOp * NI tcg_gen_op2(TCGOpcode opc, TCGType type, TCGArg a1, TCGArg a2) 49 { 50 TCGOp *op = tcg_emit_op(opc, 2); 51 TCGOP_TYPE(op) = type; 52 op->args[0] = a1; 53 op->args[1] = a2; 54 return op; 55 } 56 57 TCGOp * NI tcg_gen_op3(TCGOpcode opc, TCGType type, TCGArg a1, 58 TCGArg a2, TCGArg a3) 59 { 60 TCGOp *op = tcg_emit_op(opc, 3); 61 TCGOP_TYPE(op) = type; 62 op->args[0] = a1; 63 op->args[1] = a2; 64 op->args[2] = a3; 65 return op; 66 } 67 68 TCGOp * NI tcg_gen_op4(TCGOpcode opc, TCGType type, TCGArg a1, TCGArg a2, 69 TCGArg a3, TCGArg a4) 70 { 71 TCGOp *op = tcg_emit_op(opc, 4); 72 TCGOP_TYPE(op) = type; 73 op->args[0] = a1; 74 op->args[1] = a2; 75 op->args[2] = a3; 76 op->args[3] = a4; 77 return op; 78 } 79 80 TCGOp * NI tcg_gen_op5(TCGOpcode opc, TCGType type, TCGArg a1, TCGArg a2, 81 TCGArg a3, TCGArg a4, TCGArg a5) 82 { 83 TCGOp *op = tcg_emit_op(opc, 5); 84 TCGOP_TYPE(op) = type; 85 op->args[0] = a1; 86 op->args[1] = a2; 87 op->args[2] = a3; 88 op->args[3] = a4; 89 op->args[4] = a5; 90 return op; 91 } 92 93 TCGOp * NI tcg_gen_op6(TCGOpcode opc, TCGType type, TCGArg a1, TCGArg a2, 94 TCGArg a3, TCGArg a4, TCGArg a5, TCGArg a6) 95 { 96 TCGOp *op = tcg_emit_op(opc, 6); 97 TCGOP_TYPE(op) = type; 98 op->args[0] = a1; 99 op->args[1] = a2; 100 op->args[2] = a3; 101 op->args[3] = a4; 102 op->args[4] = a5; 103 op->args[5] = a6; 104 return op; 105 } 106 107 /* 108 * With CONFIG_DEBUG_TCG, tcgv_*_tmp via tcgv_*_arg, is an out-of-line 109 * assertion check. Force tail calls to avoid too much code expansion. 110 */ 111 #ifdef CONFIG_DEBUG_TCG 112 # define DNI NI 113 #else 114 # define DNI 115 #endif 116 117 static void DNI tcg_gen_op1_i32(TCGOpcode opc, TCGType type, TCGv_i32 a1) 118 { 119 tcg_gen_op1(opc, type, tcgv_i32_arg(a1)); 120 } 121 122 static void DNI tcg_gen_op1_i64(TCGOpcode opc, TCGType type, TCGv_i64 a1) 123 { 124 tcg_gen_op1(opc, type, tcgv_i64_arg(a1)); 125 } 126 127 static TCGOp * DNI tcg_gen_op1i(TCGOpcode opc, TCGType type, TCGArg a1) 128 { 129 return tcg_gen_op1(opc, type, a1); 130 } 131 132 static void DNI tcg_gen_op2_i32(TCGOpcode opc, TCGv_i32 a1, TCGv_i32 a2) 133 { 134 tcg_gen_op2(opc, TCG_TYPE_I32, tcgv_i32_arg(a1), tcgv_i32_arg(a2)); 135 } 136 137 static void DNI tcg_gen_op2_i64(TCGOpcode opc, TCGv_i64 a1, TCGv_i64 a2) 138 { 139 tcg_gen_op2(opc, TCG_TYPE_I64, tcgv_i64_arg(a1), tcgv_i64_arg(a2)); 140 } 141 142 static void DNI tcg_gen_op3_i32(TCGOpcode opc, TCGv_i32 a1, 143 TCGv_i32 a2, TCGv_i32 a3) 144 { 145 tcg_gen_op3(opc, TCG_TYPE_I32, tcgv_i32_arg(a1), 146 tcgv_i32_arg(a2), tcgv_i32_arg(a3)); 147 } 148 149 static void DNI tcg_gen_op3_i64(TCGOpcode opc, TCGv_i64 a1, 150 TCGv_i64 a2, TCGv_i64 a3) 151 { 152 tcg_gen_op3(opc, TCG_TYPE_I64, tcgv_i64_arg(a1), 153 tcgv_i64_arg(a2), tcgv_i64_arg(a3)); 154 } 155 156 static void DNI tcg_gen_op3i_i32(TCGOpcode opc, TCGv_i32 a1, 157 TCGv_i32 a2, TCGArg a3) 158 { 159 tcg_gen_op3(opc, TCG_TYPE_I32, tcgv_i32_arg(a1), tcgv_i32_arg(a2), a3); 160 } 161 162 static void DNI tcg_gen_op3i_i64(TCGOpcode opc, TCGv_i64 a1, 163 TCGv_i64 a2, TCGArg a3) 164 { 165 tcg_gen_op3(opc, TCG_TYPE_I64, tcgv_i64_arg(a1), tcgv_i64_arg(a2), a3); 166 } 167 168 static void DNI tcg_gen_ldst_op_i32(TCGOpcode opc, TCGv_i32 val, 169 TCGv_ptr base, TCGArg offset) 170 { 171 tcg_gen_op3(opc, TCG_TYPE_I32, tcgv_i32_arg(val), 172 tcgv_ptr_arg(base), offset); 173 } 174 175 static void DNI tcg_gen_ldst_op_i64(TCGOpcode opc, TCGv_i64 val, 176 TCGv_ptr base, TCGArg offset) 177 { 178 tcg_gen_op3(opc, TCG_TYPE_I64, tcgv_i64_arg(val), 179 tcgv_ptr_arg(base), offset); 180 } 181 182 static void DNI tcg_gen_op4_i32(TCGOpcode opc, TCGv_i32 a1, TCGv_i32 a2, 183 TCGv_i32 a3, TCGv_i32 a4) 184 { 185 tcg_gen_op4(opc, TCG_TYPE_I32, tcgv_i32_arg(a1), tcgv_i32_arg(a2), 186 tcgv_i32_arg(a3), tcgv_i32_arg(a4)); 187 } 188 189 static void DNI tcg_gen_op4_i64(TCGOpcode opc, TCGv_i64 a1, TCGv_i64 a2, 190 TCGv_i64 a3, TCGv_i64 a4) 191 { 192 tcg_gen_op4(opc, TCG_TYPE_I64, tcgv_i64_arg(a1), tcgv_i64_arg(a2), 193 tcgv_i64_arg(a3), tcgv_i64_arg(a4)); 194 } 195 196 static void DNI tcg_gen_op4i_i32(TCGOpcode opc, TCGv_i32 a1, TCGv_i32 a2, 197 TCGv_i32 a3, TCGArg a4) 198 { 199 tcg_gen_op4(opc, TCG_TYPE_I32, tcgv_i32_arg(a1), tcgv_i32_arg(a2), 200 tcgv_i32_arg(a3), a4); 201 } 202 203 static void DNI tcg_gen_op4i_i64(TCGOpcode opc, TCGv_i64 a1, TCGv_i64 a2, 204 TCGv_i64 a3, TCGArg a4) 205 { 206 tcg_gen_op4(opc, TCG_TYPE_I64, tcgv_i64_arg(a1), tcgv_i64_arg(a2), 207 tcgv_i64_arg(a3), a4); 208 } 209 210 static TCGOp * DNI tcg_gen_op4ii_i32(TCGOpcode opc, TCGv_i32 a1, TCGv_i32 a2, 211 TCGArg a3, TCGArg a4) 212 { 213 return tcg_gen_op4(opc, TCG_TYPE_I32, 214 tcgv_i32_arg(a1), tcgv_i32_arg(a2), a3, a4); 215 } 216 217 static TCGOp * DNI tcg_gen_op4ii_i64(TCGOpcode opc, TCGv_i64 a1, TCGv_i64 a2, 218 TCGArg a3, TCGArg a4) 219 { 220 return tcg_gen_op4(opc, TCG_TYPE_I64, 221 tcgv_i64_arg(a1), tcgv_i64_arg(a2), a3, a4); 222 } 223 224 static void DNI tcg_gen_op5_i32(TCGOpcode opc, TCGv_i32 a1, TCGv_i32 a2, 225 TCGv_i32 a3, TCGv_i32 a4, TCGv_i32 a5) 226 { 227 tcg_gen_op5(opc, TCG_TYPE_I32, tcgv_i32_arg(a1), tcgv_i32_arg(a2), 228 tcgv_i32_arg(a3), tcgv_i32_arg(a4), tcgv_i32_arg(a5)); 229 } 230 231 static void DNI tcg_gen_op5_i64(TCGOpcode opc, TCGv_i64 a1, TCGv_i64 a2, 232 TCGv_i64 a3, TCGv_i64 a4, TCGv_i64 a5) 233 { 234 tcg_gen_op5(opc, TCG_TYPE_I64, tcgv_i64_arg(a1), tcgv_i64_arg(a2), 235 tcgv_i64_arg(a3), tcgv_i64_arg(a4), tcgv_i64_arg(a5)); 236 } 237 238 static void DNI tcg_gen_op5ii_i32(TCGOpcode opc, TCGv_i32 a1, TCGv_i32 a2, 239 TCGv_i32 a3, TCGArg a4, TCGArg a5) 240 { 241 tcg_gen_op5(opc, TCG_TYPE_I32, tcgv_i32_arg(a1), tcgv_i32_arg(a2), 242 tcgv_i32_arg(a3), a4, a5); 243 } 244 245 static void DNI tcg_gen_op5ii_i64(TCGOpcode opc, TCGv_i64 a1, TCGv_i64 a2, 246 TCGv_i64 a3, TCGArg a4, TCGArg a5) 247 { 248 tcg_gen_op5(opc, TCG_TYPE_I64, tcgv_i64_arg(a1), tcgv_i64_arg(a2), 249 tcgv_i64_arg(a3), a4, a5); 250 } 251 252 static void DNI tcg_gen_op6_i32(TCGOpcode opc, TCGv_i32 a1, TCGv_i32 a2, 253 TCGv_i32 a3, TCGv_i32 a4, 254 TCGv_i32 a5, TCGv_i32 a6) 255 { 256 tcg_gen_op6(opc, TCG_TYPE_I32, tcgv_i32_arg(a1), tcgv_i32_arg(a2), 257 tcgv_i32_arg(a3), tcgv_i32_arg(a4), tcgv_i32_arg(a5), 258 tcgv_i32_arg(a6)); 259 } 260 261 static void DNI tcg_gen_op6_i64(TCGOpcode opc, TCGv_i64 a1, TCGv_i64 a2, 262 TCGv_i64 a3, TCGv_i64 a4, 263 TCGv_i64 a5, TCGv_i64 a6) 264 { 265 tcg_gen_op6(opc, TCG_TYPE_I64, tcgv_i64_arg(a1), tcgv_i64_arg(a2), 266 tcgv_i64_arg(a3), tcgv_i64_arg(a4), tcgv_i64_arg(a5), 267 tcgv_i64_arg(a6)); 268 } 269 270 static void DNI tcg_gen_op6i_i32(TCGOpcode opc, TCGv_i32 a1, TCGv_i32 a2, 271 TCGv_i32 a3, TCGv_i32 a4, 272 TCGv_i32 a5, TCGArg a6) 273 { 274 tcg_gen_op6(opc, TCG_TYPE_I32, tcgv_i32_arg(a1), tcgv_i32_arg(a2), 275 tcgv_i32_arg(a3), tcgv_i32_arg(a4), tcgv_i32_arg(a5), a6); 276 } 277 278 static void DNI tcg_gen_op6i_i64(TCGOpcode opc, TCGv_i64 a1, TCGv_i64 a2, 279 TCGv_i64 a3, TCGv_i64 a4, 280 TCGv_i64 a5, TCGArg a6) 281 { 282 tcg_gen_op6(opc, TCG_TYPE_I64, tcgv_i64_arg(a1), tcgv_i64_arg(a2), 283 tcgv_i64_arg(a3), tcgv_i64_arg(a4), tcgv_i64_arg(a5), a6); 284 } 285 286 static TCGOp * DNI tcg_gen_op6ii_i32(TCGOpcode opc, TCGv_i32 a1, TCGv_i32 a2, 287 TCGv_i32 a3, TCGv_i32 a4, 288 TCGArg a5, TCGArg a6) 289 { 290 return tcg_gen_op6(opc, TCG_TYPE_I32, tcgv_i32_arg(a1), tcgv_i32_arg(a2), 291 tcgv_i32_arg(a3), tcgv_i32_arg(a4), a5, a6); 292 } 293 294 /* Generic ops. */ 295 296 void gen_set_label(TCGLabel *l) 297 { 298 l->present = 1; 299 tcg_gen_op1(INDEX_op_set_label, 0, label_arg(l)); 300 } 301 302 static void add_as_label_use(TCGLabel *l, TCGOp *op) 303 { 304 TCGLabelUse *u = tcg_malloc(sizeof(TCGLabelUse)); 305 306 u->op = op; 307 QSIMPLEQ_INSERT_TAIL(&l->branches, u, next); 308 } 309 310 void tcg_gen_br(TCGLabel *l) 311 { 312 add_as_label_use(l, tcg_gen_op1(INDEX_op_br, 0, label_arg(l))); 313 } 314 315 void tcg_gen_mb(TCGBar mb_type) 316 { 317 #ifdef CONFIG_USER_ONLY 318 bool parallel = tcg_ctx->gen_tb->cflags & CF_PARALLEL; 319 #else 320 /* 321 * It is tempting to elide the barrier in a uniprocessor context. 322 * However, even with a single cpu we have i/o threads running in 323 * parallel, and lack of memory order can result in e.g. virtio 324 * queue entries being read incorrectly. 325 */ 326 bool parallel = true; 327 #endif 328 329 if (parallel) { 330 tcg_gen_op1(INDEX_op_mb, 0, mb_type); 331 } 332 } 333 334 void tcg_gen_plugin_cb(unsigned from) 335 { 336 tcg_gen_op1(INDEX_op_plugin_cb, 0, from); 337 } 338 339 void tcg_gen_plugin_mem_cb(TCGv_i64 addr, unsigned meminfo) 340 { 341 tcg_gen_op2(INDEX_op_plugin_mem_cb, 0, tcgv_i64_arg(addr), meminfo); 342 } 343 344 /* 32 bit ops */ 345 346 void tcg_gen_discard_i32(TCGv_i32 arg) 347 { 348 tcg_gen_op1_i32(INDEX_op_discard, TCG_TYPE_I32, arg); 349 } 350 351 void tcg_gen_mov_i32(TCGv_i32 ret, TCGv_i32 arg) 352 { 353 if (ret != arg) { 354 tcg_gen_op2_i32(INDEX_op_mov, ret, arg); 355 } 356 } 357 358 void tcg_gen_movi_i32(TCGv_i32 ret, int32_t arg) 359 { 360 tcg_gen_mov_i32(ret, tcg_constant_i32(arg)); 361 } 362 363 void tcg_gen_add_i32(TCGv_i32 ret, TCGv_i32 arg1, TCGv_i32 arg2) 364 { 365 tcg_gen_op3_i32(INDEX_op_add, ret, arg1, arg2); 366 } 367 368 void tcg_gen_addi_i32(TCGv_i32 ret, TCGv_i32 arg1, int32_t arg2) 369 { 370 /* some cases can be optimized here */ 371 if (arg2 == 0) { 372 tcg_gen_mov_i32(ret, arg1); 373 } else { 374 tcg_gen_add_i32(ret, arg1, tcg_constant_i32(arg2)); 375 } 376 } 377 378 void tcg_gen_sub_i32(TCGv_i32 ret, TCGv_i32 arg1, TCGv_i32 arg2) 379 { 380 tcg_gen_op3_i32(INDEX_op_sub, ret, arg1, arg2); 381 } 382 383 void tcg_gen_subfi_i32(TCGv_i32 ret, int32_t arg1, TCGv_i32 arg2) 384 { 385 if (arg1 == 0) { 386 tcg_gen_neg_i32(ret, arg2); 387 } else { 388 tcg_gen_sub_i32(ret, tcg_constant_i32(arg1), arg2); 389 } 390 } 391 392 void tcg_gen_subi_i32(TCGv_i32 ret, TCGv_i32 arg1, int32_t arg2) 393 { 394 tcg_gen_addi_i32(ret, arg1, -arg2); 395 } 396 397 void tcg_gen_neg_i32(TCGv_i32 ret, TCGv_i32 arg) 398 { 399 tcg_gen_op2_i32(INDEX_op_neg, ret, arg); 400 } 401 402 void tcg_gen_and_i32(TCGv_i32 ret, TCGv_i32 arg1, TCGv_i32 arg2) 403 { 404 tcg_gen_op3_i32(INDEX_op_and, ret, arg1, arg2); 405 } 406 407 void tcg_gen_andi_i32(TCGv_i32 ret, TCGv_i32 arg1, int32_t arg2) 408 { 409 /* Some cases can be optimized here. */ 410 switch (arg2) { 411 case 0: 412 tcg_gen_movi_i32(ret, 0); 413 return; 414 case -1: 415 tcg_gen_mov_i32(ret, arg1); 416 return; 417 default: 418 /* 419 * Canonicalize on extract, if valid. This aids x86 with its 420 * 2 operand MOVZBL and 2 operand AND, selecting the TCGOpcode 421 * which does not require matching operands. Other backends can 422 * trivially expand the extract to AND during code generation. 423 */ 424 if (!(arg2 & (arg2 + 1))) { 425 unsigned len = ctz32(~arg2); 426 if (TCG_TARGET_extract_valid(TCG_TYPE_I32, 0, len)) { 427 tcg_gen_extract_i32(ret, arg1, 0, len); 428 return; 429 } 430 } 431 break; 432 } 433 434 tcg_gen_and_i32(ret, arg1, tcg_constant_i32(arg2)); 435 } 436 437 void tcg_gen_or_i32(TCGv_i32 ret, TCGv_i32 arg1, TCGv_i32 arg2) 438 { 439 tcg_gen_op3_i32(INDEX_op_or, ret, arg1, arg2); 440 } 441 442 void tcg_gen_ori_i32(TCGv_i32 ret, TCGv_i32 arg1, int32_t arg2) 443 { 444 /* Some cases can be optimized here. */ 445 if (arg2 == -1) { 446 tcg_gen_movi_i32(ret, -1); 447 } else if (arg2 == 0) { 448 tcg_gen_mov_i32(ret, arg1); 449 } else { 450 tcg_gen_or_i32(ret, arg1, tcg_constant_i32(arg2)); 451 } 452 } 453 454 void tcg_gen_xor_i32(TCGv_i32 ret, TCGv_i32 arg1, TCGv_i32 arg2) 455 { 456 tcg_gen_op3_i32(INDEX_op_xor, ret, arg1, arg2); 457 } 458 459 void tcg_gen_xori_i32(TCGv_i32 ret, TCGv_i32 arg1, int32_t arg2) 460 { 461 /* Some cases can be optimized here. */ 462 if (arg2 == 0) { 463 tcg_gen_mov_i32(ret, arg1); 464 } else if (arg2 == -1 && 465 tcg_op_supported(INDEX_op_not, TCG_TYPE_I32, 0)) { 466 /* Don't recurse with tcg_gen_not_i32. */ 467 tcg_gen_op2_i32(INDEX_op_not, ret, arg1); 468 } else { 469 tcg_gen_xor_i32(ret, arg1, tcg_constant_i32(arg2)); 470 } 471 } 472 473 void tcg_gen_not_i32(TCGv_i32 ret, TCGv_i32 arg) 474 { 475 if (tcg_op_supported(INDEX_op_not, TCG_TYPE_I32, 0)) { 476 tcg_gen_op2_i32(INDEX_op_not, ret, arg); 477 } else { 478 tcg_gen_xori_i32(ret, arg, -1); 479 } 480 } 481 482 void tcg_gen_shl_i32(TCGv_i32 ret, TCGv_i32 arg1, TCGv_i32 arg2) 483 { 484 tcg_gen_op3_i32(INDEX_op_shl, ret, arg1, arg2); 485 } 486 487 void tcg_gen_shli_i32(TCGv_i32 ret, TCGv_i32 arg1, int32_t arg2) 488 { 489 tcg_debug_assert(arg2 >= 0 && arg2 < 32); 490 if (arg2 == 0) { 491 tcg_gen_mov_i32(ret, arg1); 492 } else { 493 tcg_gen_shl_i32(ret, arg1, tcg_constant_i32(arg2)); 494 } 495 } 496 497 void tcg_gen_shr_i32(TCGv_i32 ret, TCGv_i32 arg1, TCGv_i32 arg2) 498 { 499 tcg_gen_op3_i32(INDEX_op_shr, ret, arg1, arg2); 500 } 501 502 void tcg_gen_shri_i32(TCGv_i32 ret, TCGv_i32 arg1, int32_t arg2) 503 { 504 tcg_debug_assert(arg2 >= 0 && arg2 < 32); 505 if (arg2 == 0) { 506 tcg_gen_mov_i32(ret, arg1); 507 } else { 508 tcg_gen_shr_i32(ret, arg1, tcg_constant_i32(arg2)); 509 } 510 } 511 512 void tcg_gen_sar_i32(TCGv_i32 ret, TCGv_i32 arg1, TCGv_i32 arg2) 513 { 514 tcg_gen_op3_i32(INDEX_op_sar, ret, arg1, arg2); 515 } 516 517 void tcg_gen_sari_i32(TCGv_i32 ret, TCGv_i32 arg1, int32_t arg2) 518 { 519 tcg_debug_assert(arg2 >= 0 && arg2 < 32); 520 if (arg2 == 0) { 521 tcg_gen_mov_i32(ret, arg1); 522 } else { 523 tcg_gen_sar_i32(ret, arg1, tcg_constant_i32(arg2)); 524 } 525 } 526 527 void tcg_gen_brcond_i32(TCGCond cond, TCGv_i32 arg1, TCGv_i32 arg2, TCGLabel *l) 528 { 529 if (cond == TCG_COND_ALWAYS) { 530 tcg_gen_br(l); 531 } else if (cond != TCG_COND_NEVER) { 532 TCGOp *op = tcg_gen_op4ii_i32(INDEX_op_brcond_i32, 533 arg1, arg2, cond, label_arg(l)); 534 add_as_label_use(l, op); 535 } 536 } 537 538 void tcg_gen_brcondi_i32(TCGCond cond, TCGv_i32 arg1, int32_t arg2, TCGLabel *l) 539 { 540 if (cond == TCG_COND_ALWAYS) { 541 tcg_gen_br(l); 542 } else if (cond != TCG_COND_NEVER) { 543 tcg_gen_brcond_i32(cond, arg1, tcg_constant_i32(arg2), l); 544 } 545 } 546 547 void tcg_gen_setcond_i32(TCGCond cond, TCGv_i32 ret, 548 TCGv_i32 arg1, TCGv_i32 arg2) 549 { 550 if (cond == TCG_COND_ALWAYS) { 551 tcg_gen_movi_i32(ret, 1); 552 } else if (cond == TCG_COND_NEVER) { 553 tcg_gen_movi_i32(ret, 0); 554 } else { 555 tcg_gen_op4i_i32(INDEX_op_setcond_i32, ret, arg1, arg2, cond); 556 } 557 } 558 559 void tcg_gen_setcondi_i32(TCGCond cond, TCGv_i32 ret, 560 TCGv_i32 arg1, int32_t arg2) 561 { 562 tcg_gen_setcond_i32(cond, ret, arg1, tcg_constant_i32(arg2)); 563 } 564 565 void tcg_gen_negsetcond_i32(TCGCond cond, TCGv_i32 ret, 566 TCGv_i32 arg1, TCGv_i32 arg2) 567 { 568 if (cond == TCG_COND_ALWAYS) { 569 tcg_gen_movi_i32(ret, -1); 570 } else if (cond == TCG_COND_NEVER) { 571 tcg_gen_movi_i32(ret, 0); 572 } else if (TCG_TARGET_HAS_negsetcond_i32) { 573 tcg_gen_op4i_i32(INDEX_op_negsetcond_i32, ret, arg1, arg2, cond); 574 } else { 575 tcg_gen_setcond_i32(cond, ret, arg1, arg2); 576 tcg_gen_neg_i32(ret, ret); 577 } 578 } 579 580 void tcg_gen_negsetcondi_i32(TCGCond cond, TCGv_i32 ret, 581 TCGv_i32 arg1, int32_t arg2) 582 { 583 tcg_gen_negsetcond_i32(cond, ret, arg1, tcg_constant_i32(arg2)); 584 } 585 586 void tcg_gen_mul_i32(TCGv_i32 ret, TCGv_i32 arg1, TCGv_i32 arg2) 587 { 588 tcg_gen_op3_i32(INDEX_op_mul, ret, arg1, arg2); 589 } 590 591 void tcg_gen_muli_i32(TCGv_i32 ret, TCGv_i32 arg1, int32_t arg2) 592 { 593 if (arg2 == 0) { 594 tcg_gen_movi_i32(ret, 0); 595 } else if (is_power_of_2(arg2)) { 596 tcg_gen_shli_i32(ret, arg1, ctz32(arg2)); 597 } else { 598 tcg_gen_mul_i32(ret, arg1, tcg_constant_i32(arg2)); 599 } 600 } 601 602 void tcg_gen_div_i32(TCGv_i32 ret, TCGv_i32 arg1, TCGv_i32 arg2) 603 { 604 if (tcg_op_supported(INDEX_op_divs, TCG_TYPE_I32, 0)) { 605 tcg_gen_op3_i32(INDEX_op_divs, ret, arg1, arg2); 606 } else if (tcg_op_supported(INDEX_op_divs2, TCG_TYPE_I32, 0)) { 607 TCGv_i32 t0 = tcg_temp_ebb_new_i32(); 608 tcg_gen_sari_i32(t0, arg1, 31); 609 tcg_gen_op5_i32(INDEX_op_divs2, ret, t0, arg1, t0, arg2); 610 tcg_temp_free_i32(t0); 611 } else { 612 gen_helper_div_i32(ret, arg1, arg2); 613 } 614 } 615 616 void tcg_gen_rem_i32(TCGv_i32 ret, TCGv_i32 arg1, TCGv_i32 arg2) 617 { 618 if (tcg_op_supported(INDEX_op_rems, TCG_TYPE_I32, 0)) { 619 tcg_gen_op3_i32(INDEX_op_rems, ret, arg1, arg2); 620 } else if (tcg_op_supported(INDEX_op_divs, TCG_TYPE_I32, 0)) { 621 TCGv_i32 t0 = tcg_temp_ebb_new_i32(); 622 tcg_gen_op3_i32(INDEX_op_divs, t0, arg1, arg2); 623 tcg_gen_mul_i32(t0, t0, arg2); 624 tcg_gen_sub_i32(ret, arg1, t0); 625 tcg_temp_free_i32(t0); 626 } else if (tcg_op_supported(INDEX_op_divs2, TCG_TYPE_I32, 0)) { 627 TCGv_i32 t0 = tcg_temp_ebb_new_i32(); 628 tcg_gen_sari_i32(t0, arg1, 31); 629 tcg_gen_op5_i32(INDEX_op_divs2, t0, ret, arg1, t0, arg2); 630 tcg_temp_free_i32(t0); 631 } else { 632 gen_helper_rem_i32(ret, arg1, arg2); 633 } 634 } 635 636 void tcg_gen_divu_i32(TCGv_i32 ret, TCGv_i32 arg1, TCGv_i32 arg2) 637 { 638 if (tcg_op_supported(INDEX_op_divu, TCG_TYPE_I32, 0)) { 639 tcg_gen_op3_i32(INDEX_op_divu, ret, arg1, arg2); 640 } else if (tcg_op_supported(INDEX_op_divu2, TCG_TYPE_I32, 0)) { 641 TCGv_i32 t0 = tcg_temp_ebb_new_i32(); 642 TCGv_i32 zero = tcg_constant_i32(0); 643 tcg_gen_op5_i32(INDEX_op_divu2, ret, t0, arg1, zero, arg2); 644 tcg_temp_free_i32(t0); 645 } else { 646 gen_helper_divu_i32(ret, arg1, arg2); 647 } 648 } 649 650 void tcg_gen_remu_i32(TCGv_i32 ret, TCGv_i32 arg1, TCGv_i32 arg2) 651 { 652 if (tcg_op_supported(INDEX_op_remu, TCG_TYPE_I32, 0)) { 653 tcg_gen_op3_i32(INDEX_op_remu, ret, arg1, arg2); 654 } else if (tcg_op_supported(INDEX_op_divu, TCG_TYPE_I32, 0)) { 655 TCGv_i32 t0 = tcg_temp_ebb_new_i32(); 656 tcg_gen_op3_i32(INDEX_op_divu, t0, arg1, arg2); 657 tcg_gen_mul_i32(t0, t0, arg2); 658 tcg_gen_sub_i32(ret, arg1, t0); 659 tcg_temp_free_i32(t0); 660 } else if (tcg_op_supported(INDEX_op_divu2, TCG_TYPE_I32, 0)) { 661 TCGv_i32 t0 = tcg_temp_ebb_new_i32(); 662 TCGv_i32 zero = tcg_constant_i32(0); 663 tcg_gen_op5_i32(INDEX_op_divu2, t0, ret, arg1, zero, arg2); 664 tcg_temp_free_i32(t0); 665 } else { 666 gen_helper_remu_i32(ret, arg1, arg2); 667 } 668 } 669 670 void tcg_gen_andc_i32(TCGv_i32 ret, TCGv_i32 arg1, TCGv_i32 arg2) 671 { 672 if (tcg_op_supported(INDEX_op_andc, TCG_TYPE_I32, 0)) { 673 tcg_gen_op3_i32(INDEX_op_andc, ret, arg1, arg2); 674 } else { 675 TCGv_i32 t0 = tcg_temp_ebb_new_i32(); 676 tcg_gen_not_i32(t0, arg2); 677 tcg_gen_and_i32(ret, arg1, t0); 678 tcg_temp_free_i32(t0); 679 } 680 } 681 682 void tcg_gen_eqv_i32(TCGv_i32 ret, TCGv_i32 arg1, TCGv_i32 arg2) 683 { 684 if (tcg_op_supported(INDEX_op_eqv, TCG_TYPE_I32, 0)) { 685 tcg_gen_op3_i32(INDEX_op_eqv, ret, arg1, arg2); 686 } else { 687 tcg_gen_xor_i32(ret, arg1, arg2); 688 tcg_gen_not_i32(ret, ret); 689 } 690 } 691 692 void tcg_gen_nand_i32(TCGv_i32 ret, TCGv_i32 arg1, TCGv_i32 arg2) 693 { 694 if (tcg_op_supported(INDEX_op_nand, TCG_TYPE_I32, 0)) { 695 tcg_gen_op3_i32(INDEX_op_nand, ret, arg1, arg2); 696 } else { 697 tcg_gen_and_i32(ret, arg1, arg2); 698 tcg_gen_not_i32(ret, ret); 699 } 700 } 701 702 void tcg_gen_nor_i32(TCGv_i32 ret, TCGv_i32 arg1, TCGv_i32 arg2) 703 { 704 if (tcg_op_supported(INDEX_op_nor, TCG_TYPE_I32, 0)) { 705 tcg_gen_op3_i32(INDEX_op_nor, ret, arg1, arg2); 706 } else { 707 tcg_gen_or_i32(ret, arg1, arg2); 708 tcg_gen_not_i32(ret, ret); 709 } 710 } 711 712 void tcg_gen_orc_i32(TCGv_i32 ret, TCGv_i32 arg1, TCGv_i32 arg2) 713 { 714 if (tcg_op_supported(INDEX_op_orc, TCG_TYPE_I32, 0)) { 715 tcg_gen_op3_i32(INDEX_op_orc, ret, arg1, arg2); 716 } else { 717 TCGv_i32 t0 = tcg_temp_ebb_new_i32(); 718 tcg_gen_not_i32(t0, arg2); 719 tcg_gen_or_i32(ret, arg1, t0); 720 tcg_temp_free_i32(t0); 721 } 722 } 723 724 void tcg_gen_clz_i32(TCGv_i32 ret, TCGv_i32 arg1, TCGv_i32 arg2) 725 { 726 if (tcg_op_supported(INDEX_op_clz, TCG_TYPE_I32, 0)) { 727 tcg_gen_op3_i32(INDEX_op_clz, ret, arg1, arg2); 728 } else if (tcg_op_supported(INDEX_op_clz, TCG_TYPE_I64, 0)) { 729 TCGv_i64 t1 = tcg_temp_ebb_new_i64(); 730 TCGv_i64 t2 = tcg_temp_ebb_new_i64(); 731 tcg_gen_extu_i32_i64(t1, arg1); 732 tcg_gen_extu_i32_i64(t2, arg2); 733 tcg_gen_addi_i64(t2, t2, 32); 734 tcg_gen_clz_i64(t1, t1, t2); 735 tcg_gen_extrl_i64_i32(ret, t1); 736 tcg_temp_free_i64(t1); 737 tcg_temp_free_i64(t2); 738 tcg_gen_subi_i32(ret, ret, 32); 739 } else { 740 gen_helper_clz_i32(ret, arg1, arg2); 741 } 742 } 743 744 void tcg_gen_clzi_i32(TCGv_i32 ret, TCGv_i32 arg1, uint32_t arg2) 745 { 746 tcg_gen_clz_i32(ret, arg1, tcg_constant_i32(arg2)); 747 } 748 749 void tcg_gen_ctz_i32(TCGv_i32 ret, TCGv_i32 arg1, TCGv_i32 arg2) 750 { 751 TCGv_i32 z, t; 752 753 if (TCG_TARGET_HAS_ctz_i32) { 754 tcg_gen_op3_i32(INDEX_op_ctz_i32, ret, arg1, arg2); 755 return; 756 } 757 if (TCG_TARGET_HAS_ctz_i64) { 758 TCGv_i64 t1 = tcg_temp_ebb_new_i64(); 759 TCGv_i64 t2 = tcg_temp_ebb_new_i64(); 760 tcg_gen_extu_i32_i64(t1, arg1); 761 tcg_gen_extu_i32_i64(t2, arg2); 762 tcg_gen_ctz_i64(t1, t1, t2); 763 tcg_gen_extrl_i64_i32(ret, t1); 764 tcg_temp_free_i64(t1); 765 tcg_temp_free_i64(t2); 766 return; 767 } 768 if (TCG_TARGET_HAS_ctpop_i32 || TCG_TARGET_HAS_ctpop_i64) { 769 t = tcg_temp_ebb_new_i32(); 770 tcg_gen_subi_i32(t, arg1, 1); 771 tcg_gen_andc_i32(t, t, arg1); 772 tcg_gen_ctpop_i32(t, t); 773 } else if (tcg_op_supported(INDEX_op_clz, TCG_TYPE_REG, 0)) { 774 t = tcg_temp_ebb_new_i32(); 775 tcg_gen_neg_i32(t, arg1); 776 tcg_gen_and_i32(t, t, arg1); 777 tcg_gen_clzi_i32(t, t, 32); 778 tcg_gen_xori_i32(t, t, 31); 779 } else { 780 gen_helper_ctz_i32(ret, arg1, arg2); 781 return; 782 } 783 784 z = tcg_constant_i32(0); 785 tcg_gen_movcond_i32(TCG_COND_EQ, ret, arg1, z, arg2, t); 786 tcg_temp_free_i32(t); 787 } 788 789 void tcg_gen_ctzi_i32(TCGv_i32 ret, TCGv_i32 arg1, uint32_t arg2) 790 { 791 if (!TCG_TARGET_HAS_ctz_i32 && TCG_TARGET_HAS_ctpop_i32 && arg2 == 32) { 792 /* This equivalence has the advantage of not requiring a fixup. */ 793 TCGv_i32 t = tcg_temp_ebb_new_i32(); 794 tcg_gen_subi_i32(t, arg1, 1); 795 tcg_gen_andc_i32(t, t, arg1); 796 tcg_gen_ctpop_i32(ret, t); 797 tcg_temp_free_i32(t); 798 } else { 799 tcg_gen_ctz_i32(ret, arg1, tcg_constant_i32(arg2)); 800 } 801 } 802 803 void tcg_gen_clrsb_i32(TCGv_i32 ret, TCGv_i32 arg) 804 { 805 if (tcg_op_supported(INDEX_op_clz, TCG_TYPE_REG, 0)) { 806 TCGv_i32 t = tcg_temp_ebb_new_i32(); 807 tcg_gen_sari_i32(t, arg, 31); 808 tcg_gen_xor_i32(t, t, arg); 809 tcg_gen_clzi_i32(t, t, 32); 810 tcg_gen_subi_i32(ret, t, 1); 811 tcg_temp_free_i32(t); 812 } else { 813 gen_helper_clrsb_i32(ret, arg); 814 } 815 } 816 817 void tcg_gen_ctpop_i32(TCGv_i32 ret, TCGv_i32 arg1) 818 { 819 if (TCG_TARGET_HAS_ctpop_i32) { 820 tcg_gen_op2_i32(INDEX_op_ctpop_i32, ret, arg1); 821 } else if (TCG_TARGET_HAS_ctpop_i64) { 822 TCGv_i64 t = tcg_temp_ebb_new_i64(); 823 tcg_gen_extu_i32_i64(t, arg1); 824 tcg_gen_ctpop_i64(t, t); 825 tcg_gen_extrl_i64_i32(ret, t); 826 tcg_temp_free_i64(t); 827 } else { 828 gen_helper_ctpop_i32(ret, arg1); 829 } 830 } 831 832 void tcg_gen_rotl_i32(TCGv_i32 ret, TCGv_i32 arg1, TCGv_i32 arg2) 833 { 834 if (tcg_op_supported(INDEX_op_rotl, TCG_TYPE_I32, 0)) { 835 tcg_gen_op3_i32(INDEX_op_rotl, ret, arg1, arg2); 836 } else if (tcg_op_supported(INDEX_op_rotr, TCG_TYPE_I32, 0)) { 837 TCGv_i32 t0 = tcg_temp_ebb_new_i32(); 838 tcg_gen_neg_i32(t0, arg2); 839 tcg_gen_op3_i32(INDEX_op_rotr, ret, arg1, t0); 840 tcg_temp_free_i32(t0); 841 } else { 842 TCGv_i32 t0 = tcg_temp_ebb_new_i32(); 843 TCGv_i32 t1 = tcg_temp_ebb_new_i32(); 844 tcg_gen_shl_i32(t0, arg1, arg2); 845 tcg_gen_neg_i32(t1, arg2); 846 tcg_gen_shr_i32(t1, arg1, t1); 847 tcg_gen_or_i32(ret, t0, t1); 848 tcg_temp_free_i32(t0); 849 tcg_temp_free_i32(t1); 850 } 851 } 852 853 void tcg_gen_rotli_i32(TCGv_i32 ret, TCGv_i32 arg1, int32_t arg2) 854 { 855 tcg_debug_assert(arg2 >= 0 && arg2 < 32); 856 /* some cases can be optimized here */ 857 if (arg2 == 0) { 858 tcg_gen_mov_i32(ret, arg1); 859 } else if (tcg_op_supported(INDEX_op_rotl, TCG_TYPE_I32, 0)) { 860 TCGv_i32 t0 = tcg_constant_i32(arg2); 861 tcg_gen_op3_i32(INDEX_op_rotl, ret, arg1, t0); 862 } else if (tcg_op_supported(INDEX_op_rotr, TCG_TYPE_I32, 0)) { 863 TCGv_i32 t0 = tcg_constant_i32(32 - arg2); 864 tcg_gen_op3_i32(INDEX_op_rotr, ret, arg1, t0); 865 } else { 866 TCGv_i32 t0 = tcg_temp_ebb_new_i32(); 867 TCGv_i32 t1 = tcg_temp_ebb_new_i32(); 868 tcg_gen_shli_i32(t0, arg1, arg2); 869 tcg_gen_shri_i32(t1, arg1, 32 - arg2); 870 tcg_gen_or_i32(ret, t0, t1); 871 tcg_temp_free_i32(t0); 872 tcg_temp_free_i32(t1); 873 } 874 } 875 876 void tcg_gen_rotr_i32(TCGv_i32 ret, TCGv_i32 arg1, TCGv_i32 arg2) 877 { 878 if (tcg_op_supported(INDEX_op_rotr, TCG_TYPE_I32, 0)) { 879 tcg_gen_op3_i32(INDEX_op_rotr, ret, arg1, arg2); 880 } else if (tcg_op_supported(INDEX_op_rotl, TCG_TYPE_I32, 0)) { 881 TCGv_i32 t0 = tcg_temp_ebb_new_i32(); 882 tcg_gen_neg_i32(t0, arg2); 883 tcg_gen_op3_i32(INDEX_op_rotl, ret, arg1, t0); 884 tcg_temp_free_i32(t0); 885 } else { 886 TCGv_i32 t0 = tcg_temp_ebb_new_i32(); 887 TCGv_i32 t1 = tcg_temp_ebb_new_i32(); 888 tcg_gen_shr_i32(t0, arg1, arg2); 889 tcg_gen_neg_i32(t1, arg2); 890 tcg_gen_shl_i32(t1, arg1, t1); 891 tcg_gen_or_i32(ret, t0, t1); 892 tcg_temp_free_i32(t0); 893 tcg_temp_free_i32(t1); 894 } 895 } 896 897 void tcg_gen_rotri_i32(TCGv_i32 ret, TCGv_i32 arg1, int32_t arg2) 898 { 899 tcg_debug_assert(arg2 >= 0 && arg2 < 32); 900 tcg_gen_rotli_i32(ret, arg1, -arg2 & 31); 901 } 902 903 void tcg_gen_deposit_i32(TCGv_i32 ret, TCGv_i32 arg1, TCGv_i32 arg2, 904 unsigned int ofs, unsigned int len) 905 { 906 uint32_t mask; 907 TCGv_i32 t1; 908 909 tcg_debug_assert(ofs < 32); 910 tcg_debug_assert(len > 0); 911 tcg_debug_assert(len <= 32); 912 tcg_debug_assert(ofs + len <= 32); 913 914 if (len == 32) { 915 tcg_gen_mov_i32(ret, arg2); 916 return; 917 } 918 if (TCG_TARGET_deposit_valid(TCG_TYPE_I32, ofs, len)) { 919 tcg_gen_op5ii_i32(INDEX_op_deposit_i32, ret, arg1, arg2, ofs, len); 920 return; 921 } 922 923 t1 = tcg_temp_ebb_new_i32(); 924 925 if (TCG_TARGET_HAS_extract2_i32) { 926 if (ofs + len == 32) { 927 tcg_gen_shli_i32(t1, arg1, len); 928 tcg_gen_extract2_i32(ret, t1, arg2, len); 929 goto done; 930 } 931 if (ofs == 0) { 932 tcg_gen_extract2_i32(ret, arg1, arg2, len); 933 tcg_gen_rotli_i32(ret, ret, len); 934 goto done; 935 } 936 } 937 938 mask = (1u << len) - 1; 939 if (ofs + len < 32) { 940 tcg_gen_andi_i32(t1, arg2, mask); 941 tcg_gen_shli_i32(t1, t1, ofs); 942 } else { 943 tcg_gen_shli_i32(t1, arg2, ofs); 944 } 945 tcg_gen_andi_i32(ret, arg1, ~(mask << ofs)); 946 tcg_gen_or_i32(ret, ret, t1); 947 done: 948 tcg_temp_free_i32(t1); 949 } 950 951 void tcg_gen_deposit_z_i32(TCGv_i32 ret, TCGv_i32 arg, 952 unsigned int ofs, unsigned int len) 953 { 954 tcg_debug_assert(ofs < 32); 955 tcg_debug_assert(len > 0); 956 tcg_debug_assert(len <= 32); 957 tcg_debug_assert(ofs + len <= 32); 958 959 if (ofs + len == 32) { 960 tcg_gen_shli_i32(ret, arg, ofs); 961 } else if (ofs == 0) { 962 tcg_gen_andi_i32(ret, arg, (1u << len) - 1); 963 } else if (TCG_TARGET_deposit_valid(TCG_TYPE_I32, ofs, len)) { 964 TCGv_i32 zero = tcg_constant_i32(0); 965 tcg_gen_op5ii_i32(INDEX_op_deposit_i32, ret, zero, arg, ofs, len); 966 } else { 967 /* 968 * To help two-operand hosts we prefer to zero-extend first, 969 * which allows ARG to stay live. 970 */ 971 if (TCG_TARGET_extract_valid(TCG_TYPE_I32, 0, len)) { 972 tcg_gen_extract_i32(ret, arg, 0, len); 973 tcg_gen_shli_i32(ret, ret, ofs); 974 return; 975 } 976 /* Otherwise prefer zero-extension over AND for code size. */ 977 if (TCG_TARGET_extract_valid(TCG_TYPE_I32, 0, ofs + len)) { 978 tcg_gen_shli_i32(ret, arg, ofs); 979 tcg_gen_extract_i32(ret, ret, 0, ofs + len); 980 return; 981 } 982 tcg_gen_andi_i32(ret, arg, (1u << len) - 1); 983 tcg_gen_shli_i32(ret, ret, ofs); 984 } 985 } 986 987 void tcg_gen_extract_i32(TCGv_i32 ret, TCGv_i32 arg, 988 unsigned int ofs, unsigned int len) 989 { 990 tcg_debug_assert(ofs < 32); 991 tcg_debug_assert(len > 0); 992 tcg_debug_assert(len <= 32); 993 tcg_debug_assert(ofs + len <= 32); 994 995 /* Canonicalize certain special cases, even if extract is supported. */ 996 if (ofs + len == 32) { 997 tcg_gen_shri_i32(ret, arg, 32 - len); 998 return; 999 } 1000 1001 if (TCG_TARGET_extract_valid(TCG_TYPE_I32, ofs, len)) { 1002 tcg_gen_op4ii_i32(INDEX_op_extract_i32, ret, arg, ofs, len); 1003 return; 1004 } 1005 if (ofs == 0) { 1006 tcg_gen_andi_i32(ret, arg, (1u << len) - 1); 1007 return; 1008 } 1009 1010 /* Assume that zero-extension, if available, is cheaper than a shift. */ 1011 if (TCG_TARGET_extract_valid(TCG_TYPE_I32, 0, ofs + len)) { 1012 tcg_gen_op4ii_i32(INDEX_op_extract_i32, ret, arg, 0, ofs + len); 1013 tcg_gen_shri_i32(ret, ret, ofs); 1014 return; 1015 } 1016 1017 /* ??? Ideally we'd know what values are available for immediate AND. 1018 Assume that 8 bits are available, plus the special case of 16, 1019 so that we get ext8u, ext16u. */ 1020 switch (len) { 1021 case 1 ... 8: case 16: 1022 tcg_gen_shri_i32(ret, arg, ofs); 1023 tcg_gen_andi_i32(ret, ret, (1u << len) - 1); 1024 break; 1025 default: 1026 tcg_gen_shli_i32(ret, arg, 32 - len - ofs); 1027 tcg_gen_shri_i32(ret, ret, 32 - len); 1028 break; 1029 } 1030 } 1031 1032 void tcg_gen_sextract_i32(TCGv_i32 ret, TCGv_i32 arg, 1033 unsigned int ofs, unsigned int len) 1034 { 1035 tcg_debug_assert(ofs < 32); 1036 tcg_debug_assert(len > 0); 1037 tcg_debug_assert(len <= 32); 1038 tcg_debug_assert(ofs + len <= 32); 1039 1040 /* Canonicalize certain special cases, even if extract is supported. */ 1041 if (ofs + len == 32) { 1042 tcg_gen_sari_i32(ret, arg, 32 - len); 1043 return; 1044 } 1045 1046 if (TCG_TARGET_sextract_valid(TCG_TYPE_I32, ofs, len)) { 1047 tcg_gen_op4ii_i32(INDEX_op_sextract_i32, ret, arg, ofs, len); 1048 return; 1049 } 1050 1051 /* Assume that sign-extension, if available, is cheaper than a shift. */ 1052 if (TCG_TARGET_sextract_valid(TCG_TYPE_I32, 0, ofs + len)) { 1053 tcg_gen_op4ii_i32(INDEX_op_sextract_i32, ret, arg, 0, ofs + len); 1054 tcg_gen_sari_i32(ret, ret, ofs); 1055 return; 1056 } 1057 if (TCG_TARGET_sextract_valid(TCG_TYPE_I32, 0, len)) { 1058 tcg_gen_shri_i32(ret, arg, ofs); 1059 tcg_gen_op4ii_i32(INDEX_op_sextract_i32, ret, ret, 0, len); 1060 return; 1061 } 1062 1063 tcg_gen_shli_i32(ret, arg, 32 - len - ofs); 1064 tcg_gen_sari_i32(ret, ret, 32 - len); 1065 } 1066 1067 /* 1068 * Extract 32-bits from a 64-bit input, ah:al, starting from ofs. 1069 * Unlike tcg_gen_extract_i32 above, len is fixed at 32. 1070 */ 1071 void tcg_gen_extract2_i32(TCGv_i32 ret, TCGv_i32 al, TCGv_i32 ah, 1072 unsigned int ofs) 1073 { 1074 tcg_debug_assert(ofs <= 32); 1075 if (ofs == 0) { 1076 tcg_gen_mov_i32(ret, al); 1077 } else if (ofs == 32) { 1078 tcg_gen_mov_i32(ret, ah); 1079 } else if (al == ah) { 1080 tcg_gen_rotri_i32(ret, al, ofs); 1081 } else if (TCG_TARGET_HAS_extract2_i32) { 1082 tcg_gen_op4i_i32(INDEX_op_extract2_i32, ret, al, ah, ofs); 1083 } else { 1084 TCGv_i32 t0 = tcg_temp_ebb_new_i32(); 1085 tcg_gen_shri_i32(t0, al, ofs); 1086 tcg_gen_deposit_i32(ret, t0, ah, 32 - ofs, ofs); 1087 tcg_temp_free_i32(t0); 1088 } 1089 } 1090 1091 void tcg_gen_movcond_i32(TCGCond cond, TCGv_i32 ret, TCGv_i32 c1, 1092 TCGv_i32 c2, TCGv_i32 v1, TCGv_i32 v2) 1093 { 1094 if (cond == TCG_COND_ALWAYS) { 1095 tcg_gen_mov_i32(ret, v1); 1096 } else if (cond == TCG_COND_NEVER) { 1097 tcg_gen_mov_i32(ret, v2); 1098 } else { 1099 tcg_gen_op6i_i32(INDEX_op_movcond_i32, ret, c1, c2, v1, v2, cond); 1100 } 1101 } 1102 1103 void tcg_gen_add2_i32(TCGv_i32 rl, TCGv_i32 rh, TCGv_i32 al, 1104 TCGv_i32 ah, TCGv_i32 bl, TCGv_i32 bh) 1105 { 1106 if (TCG_TARGET_HAS_add2_i32) { 1107 tcg_gen_op6_i32(INDEX_op_add2_i32, rl, rh, al, ah, bl, bh); 1108 } else { 1109 TCGv_i64 t0 = tcg_temp_ebb_new_i64(); 1110 TCGv_i64 t1 = tcg_temp_ebb_new_i64(); 1111 tcg_gen_concat_i32_i64(t0, al, ah); 1112 tcg_gen_concat_i32_i64(t1, bl, bh); 1113 tcg_gen_add_i64(t0, t0, t1); 1114 tcg_gen_extr_i64_i32(rl, rh, t0); 1115 tcg_temp_free_i64(t0); 1116 tcg_temp_free_i64(t1); 1117 } 1118 } 1119 1120 void tcg_gen_sub2_i32(TCGv_i32 rl, TCGv_i32 rh, TCGv_i32 al, 1121 TCGv_i32 ah, TCGv_i32 bl, TCGv_i32 bh) 1122 { 1123 if (TCG_TARGET_HAS_sub2_i32) { 1124 tcg_gen_op6_i32(INDEX_op_sub2_i32, rl, rh, al, ah, bl, bh); 1125 } else { 1126 TCGv_i64 t0 = tcg_temp_ebb_new_i64(); 1127 TCGv_i64 t1 = tcg_temp_ebb_new_i64(); 1128 tcg_gen_concat_i32_i64(t0, al, ah); 1129 tcg_gen_concat_i32_i64(t1, bl, bh); 1130 tcg_gen_sub_i64(t0, t0, t1); 1131 tcg_gen_extr_i64_i32(rl, rh, t0); 1132 tcg_temp_free_i64(t0); 1133 tcg_temp_free_i64(t1); 1134 } 1135 } 1136 1137 void tcg_gen_mulu2_i32(TCGv_i32 rl, TCGv_i32 rh, TCGv_i32 arg1, TCGv_i32 arg2) 1138 { 1139 if (TCG_TARGET_HAS_mulu2_i32) { 1140 tcg_gen_op4_i32(INDEX_op_mulu2_i32, rl, rh, arg1, arg2); 1141 } else if (tcg_op_supported(INDEX_op_muluh, TCG_TYPE_I32, 0)) { 1142 TCGv_i32 t = tcg_temp_ebb_new_i32(); 1143 tcg_gen_op3_i32(INDEX_op_mul, t, arg1, arg2); 1144 tcg_gen_op3_i32(INDEX_op_muluh, rh, arg1, arg2); 1145 tcg_gen_mov_i32(rl, t); 1146 tcg_temp_free_i32(t); 1147 } else if (TCG_TARGET_REG_BITS == 64) { 1148 TCGv_i64 t0 = tcg_temp_ebb_new_i64(); 1149 TCGv_i64 t1 = tcg_temp_ebb_new_i64(); 1150 tcg_gen_extu_i32_i64(t0, arg1); 1151 tcg_gen_extu_i32_i64(t1, arg2); 1152 tcg_gen_mul_i64(t0, t0, t1); 1153 tcg_gen_extr_i64_i32(rl, rh, t0); 1154 tcg_temp_free_i64(t0); 1155 tcg_temp_free_i64(t1); 1156 } else { 1157 qemu_build_not_reached(); 1158 } 1159 } 1160 1161 void tcg_gen_muls2_i32(TCGv_i32 rl, TCGv_i32 rh, TCGv_i32 arg1, TCGv_i32 arg2) 1162 { 1163 if (TCG_TARGET_HAS_muls2_i32) { 1164 tcg_gen_op4_i32(INDEX_op_muls2_i32, rl, rh, arg1, arg2); 1165 } else if (tcg_op_supported(INDEX_op_mulsh, TCG_TYPE_I32, 0)) { 1166 TCGv_i32 t = tcg_temp_ebb_new_i32(); 1167 tcg_gen_op3_i32(INDEX_op_mul, t, arg1, arg2); 1168 tcg_gen_op3_i32(INDEX_op_mulsh, rh, arg1, arg2); 1169 tcg_gen_mov_i32(rl, t); 1170 tcg_temp_free_i32(t); 1171 } else if (TCG_TARGET_REG_BITS == 32) { 1172 TCGv_i32 t0 = tcg_temp_ebb_new_i32(); 1173 TCGv_i32 t1 = tcg_temp_ebb_new_i32(); 1174 TCGv_i32 t2 = tcg_temp_ebb_new_i32(); 1175 TCGv_i32 t3 = tcg_temp_ebb_new_i32(); 1176 tcg_gen_mulu2_i32(t0, t1, arg1, arg2); 1177 /* Adjust for negative inputs. */ 1178 tcg_gen_sari_i32(t2, arg1, 31); 1179 tcg_gen_sari_i32(t3, arg2, 31); 1180 tcg_gen_and_i32(t2, t2, arg2); 1181 tcg_gen_and_i32(t3, t3, arg1); 1182 tcg_gen_sub_i32(rh, t1, t2); 1183 tcg_gen_sub_i32(rh, rh, t3); 1184 tcg_gen_mov_i32(rl, t0); 1185 tcg_temp_free_i32(t0); 1186 tcg_temp_free_i32(t1); 1187 tcg_temp_free_i32(t2); 1188 tcg_temp_free_i32(t3); 1189 } else { 1190 TCGv_i64 t0 = tcg_temp_ebb_new_i64(); 1191 TCGv_i64 t1 = tcg_temp_ebb_new_i64(); 1192 tcg_gen_ext_i32_i64(t0, arg1); 1193 tcg_gen_ext_i32_i64(t1, arg2); 1194 tcg_gen_mul_i64(t0, t0, t1); 1195 tcg_gen_extr_i64_i32(rl, rh, t0); 1196 tcg_temp_free_i64(t0); 1197 tcg_temp_free_i64(t1); 1198 } 1199 } 1200 1201 void tcg_gen_mulsu2_i32(TCGv_i32 rl, TCGv_i32 rh, TCGv_i32 arg1, TCGv_i32 arg2) 1202 { 1203 if (TCG_TARGET_REG_BITS == 32) { 1204 TCGv_i32 t0 = tcg_temp_ebb_new_i32(); 1205 TCGv_i32 t1 = tcg_temp_ebb_new_i32(); 1206 TCGv_i32 t2 = tcg_temp_ebb_new_i32(); 1207 tcg_gen_mulu2_i32(t0, t1, arg1, arg2); 1208 /* Adjust for negative input for the signed arg1. */ 1209 tcg_gen_sari_i32(t2, arg1, 31); 1210 tcg_gen_and_i32(t2, t2, arg2); 1211 tcg_gen_sub_i32(rh, t1, t2); 1212 tcg_gen_mov_i32(rl, t0); 1213 tcg_temp_free_i32(t0); 1214 tcg_temp_free_i32(t1); 1215 tcg_temp_free_i32(t2); 1216 } else { 1217 TCGv_i64 t0 = tcg_temp_ebb_new_i64(); 1218 TCGv_i64 t1 = tcg_temp_ebb_new_i64(); 1219 tcg_gen_ext_i32_i64(t0, arg1); 1220 tcg_gen_extu_i32_i64(t1, arg2); 1221 tcg_gen_mul_i64(t0, t0, t1); 1222 tcg_gen_extr_i64_i32(rl, rh, t0); 1223 tcg_temp_free_i64(t0); 1224 tcg_temp_free_i64(t1); 1225 } 1226 } 1227 1228 void tcg_gen_ext8s_i32(TCGv_i32 ret, TCGv_i32 arg) 1229 { 1230 tcg_gen_sextract_i32(ret, arg, 0, 8); 1231 } 1232 1233 void tcg_gen_ext16s_i32(TCGv_i32 ret, TCGv_i32 arg) 1234 { 1235 tcg_gen_sextract_i32(ret, arg, 0, 16); 1236 } 1237 1238 void tcg_gen_ext8u_i32(TCGv_i32 ret, TCGv_i32 arg) 1239 { 1240 tcg_gen_extract_i32(ret, arg, 0, 8); 1241 } 1242 1243 void tcg_gen_ext16u_i32(TCGv_i32 ret, TCGv_i32 arg) 1244 { 1245 tcg_gen_extract_i32(ret, arg, 0, 16); 1246 } 1247 1248 /* 1249 * bswap16_i32: 16-bit byte swap on the low bits of a 32-bit value. 1250 * 1251 * Byte pattern: xxab -> yyba 1252 * 1253 * With TCG_BSWAP_IZ, x == zero, else undefined. 1254 * With TCG_BSWAP_OZ, y == zero, with TCG_BSWAP_OS y == sign, else undefined. 1255 */ 1256 void tcg_gen_bswap16_i32(TCGv_i32 ret, TCGv_i32 arg, int flags) 1257 { 1258 /* Only one extension flag may be present. */ 1259 tcg_debug_assert(!(flags & TCG_BSWAP_OS) || !(flags & TCG_BSWAP_OZ)); 1260 1261 if (TCG_TARGET_HAS_bswap16_i32) { 1262 tcg_gen_op3i_i32(INDEX_op_bswap16_i32, ret, arg, flags); 1263 } else { 1264 TCGv_i32 t0 = tcg_temp_ebb_new_i32(); 1265 TCGv_i32 t1 = tcg_temp_ebb_new_i32(); 1266 1267 /* arg = ..ab (IZ) xxab (!IZ) */ 1268 tcg_gen_shri_i32(t0, arg, 8); /* t0 = ...a (IZ) .xxa (!IZ) */ 1269 if (!(flags & TCG_BSWAP_IZ)) { 1270 tcg_gen_ext8u_i32(t0, t0); /* t0 = ...a */ 1271 } 1272 1273 if (flags & TCG_BSWAP_OS) { 1274 tcg_gen_shli_i32(t1, arg, 24); /* t1 = b... */ 1275 tcg_gen_sari_i32(t1, t1, 16); /* t1 = ssb. */ 1276 } else if (flags & TCG_BSWAP_OZ) { 1277 tcg_gen_ext8u_i32(t1, arg); /* t1 = ...b */ 1278 tcg_gen_shli_i32(t1, t1, 8); /* t1 = ..b. */ 1279 } else { 1280 tcg_gen_shli_i32(t1, arg, 8); /* t1 = xab. */ 1281 } 1282 1283 tcg_gen_or_i32(ret, t0, t1); /* ret = ..ba (OZ) */ 1284 /* = ssba (OS) */ 1285 /* = xaba (no flag) */ 1286 tcg_temp_free_i32(t0); 1287 tcg_temp_free_i32(t1); 1288 } 1289 } 1290 1291 /* 1292 * bswap32_i32: 32-bit byte swap on a 32-bit value. 1293 * 1294 * Byte pattern: abcd -> dcba 1295 */ 1296 void tcg_gen_bswap32_i32(TCGv_i32 ret, TCGv_i32 arg) 1297 { 1298 if (TCG_TARGET_HAS_bswap32_i32) { 1299 tcg_gen_op3i_i32(INDEX_op_bswap32_i32, ret, arg, 0); 1300 } else { 1301 TCGv_i32 t0 = tcg_temp_ebb_new_i32(); 1302 TCGv_i32 t1 = tcg_temp_ebb_new_i32(); 1303 TCGv_i32 t2 = tcg_constant_i32(0x00ff00ff); 1304 1305 /* arg = abcd */ 1306 tcg_gen_shri_i32(t0, arg, 8); /* t0 = .abc */ 1307 tcg_gen_and_i32(t1, arg, t2); /* t1 = .b.d */ 1308 tcg_gen_and_i32(t0, t0, t2); /* t0 = .a.c */ 1309 tcg_gen_shli_i32(t1, t1, 8); /* t1 = b.d. */ 1310 tcg_gen_or_i32(ret, t0, t1); /* ret = badc */ 1311 1312 tcg_gen_shri_i32(t0, ret, 16); /* t0 = ..ba */ 1313 tcg_gen_shli_i32(t1, ret, 16); /* t1 = dc.. */ 1314 tcg_gen_or_i32(ret, t0, t1); /* ret = dcba */ 1315 1316 tcg_temp_free_i32(t0); 1317 tcg_temp_free_i32(t1); 1318 } 1319 } 1320 1321 /* 1322 * hswap_i32: Swap 16-bit halfwords within a 32-bit value. 1323 * 1324 * Byte pattern: abcd -> cdab 1325 */ 1326 void tcg_gen_hswap_i32(TCGv_i32 ret, TCGv_i32 arg) 1327 { 1328 /* Swapping 2 16-bit elements is a rotate. */ 1329 tcg_gen_rotli_i32(ret, arg, 16); 1330 } 1331 1332 void tcg_gen_smin_i32(TCGv_i32 ret, TCGv_i32 a, TCGv_i32 b) 1333 { 1334 tcg_gen_movcond_i32(TCG_COND_LT, ret, a, b, a, b); 1335 } 1336 1337 void tcg_gen_umin_i32(TCGv_i32 ret, TCGv_i32 a, TCGv_i32 b) 1338 { 1339 tcg_gen_movcond_i32(TCG_COND_LTU, ret, a, b, a, b); 1340 } 1341 1342 void tcg_gen_smax_i32(TCGv_i32 ret, TCGv_i32 a, TCGv_i32 b) 1343 { 1344 tcg_gen_movcond_i32(TCG_COND_LT, ret, a, b, b, a); 1345 } 1346 1347 void tcg_gen_umax_i32(TCGv_i32 ret, TCGv_i32 a, TCGv_i32 b) 1348 { 1349 tcg_gen_movcond_i32(TCG_COND_LTU, ret, a, b, b, a); 1350 } 1351 1352 void tcg_gen_abs_i32(TCGv_i32 ret, TCGv_i32 a) 1353 { 1354 TCGv_i32 t = tcg_temp_ebb_new_i32(); 1355 1356 tcg_gen_sari_i32(t, a, 31); 1357 tcg_gen_xor_i32(ret, a, t); 1358 tcg_gen_sub_i32(ret, ret, t); 1359 tcg_temp_free_i32(t); 1360 } 1361 1362 void tcg_gen_ld8u_i32(TCGv_i32 ret, TCGv_ptr arg2, tcg_target_long offset) 1363 { 1364 tcg_gen_ldst_op_i32(INDEX_op_ld8u_i32, ret, arg2, offset); 1365 } 1366 1367 void tcg_gen_ld8s_i32(TCGv_i32 ret, TCGv_ptr arg2, tcg_target_long offset) 1368 { 1369 tcg_gen_ldst_op_i32(INDEX_op_ld8s_i32, ret, arg2, offset); 1370 } 1371 1372 void tcg_gen_ld16u_i32(TCGv_i32 ret, TCGv_ptr arg2, tcg_target_long offset) 1373 { 1374 tcg_gen_ldst_op_i32(INDEX_op_ld16u_i32, ret, arg2, offset); 1375 } 1376 1377 void tcg_gen_ld16s_i32(TCGv_i32 ret, TCGv_ptr arg2, tcg_target_long offset) 1378 { 1379 tcg_gen_ldst_op_i32(INDEX_op_ld16s_i32, ret, arg2, offset); 1380 } 1381 1382 void tcg_gen_ld_i32(TCGv_i32 ret, TCGv_ptr arg2, tcg_target_long offset) 1383 { 1384 tcg_gen_ldst_op_i32(INDEX_op_ld_i32, ret, arg2, offset); 1385 } 1386 1387 void tcg_gen_st8_i32(TCGv_i32 arg1, TCGv_ptr arg2, tcg_target_long offset) 1388 { 1389 tcg_gen_ldst_op_i32(INDEX_op_st8_i32, arg1, arg2, offset); 1390 } 1391 1392 void tcg_gen_st16_i32(TCGv_i32 arg1, TCGv_ptr arg2, tcg_target_long offset) 1393 { 1394 tcg_gen_ldst_op_i32(INDEX_op_st16_i32, arg1, arg2, offset); 1395 } 1396 1397 void tcg_gen_st_i32(TCGv_i32 arg1, TCGv_ptr arg2, tcg_target_long offset) 1398 { 1399 tcg_gen_ldst_op_i32(INDEX_op_st_i32, arg1, arg2, offset); 1400 } 1401 1402 1403 /* 64-bit ops */ 1404 1405 void tcg_gen_discard_i64(TCGv_i64 arg) 1406 { 1407 if (TCG_TARGET_REG_BITS == 64) { 1408 tcg_gen_op1_i64(INDEX_op_discard, TCG_TYPE_I64, arg); 1409 } else { 1410 tcg_gen_discard_i32(TCGV_LOW(arg)); 1411 tcg_gen_discard_i32(TCGV_HIGH(arg)); 1412 } 1413 } 1414 1415 void tcg_gen_mov_i64(TCGv_i64 ret, TCGv_i64 arg) 1416 { 1417 if (ret == arg) { 1418 return; 1419 } 1420 if (TCG_TARGET_REG_BITS == 64) { 1421 tcg_gen_op2_i64(INDEX_op_mov, ret, arg); 1422 } else { 1423 TCGTemp *ts = tcgv_i64_temp(arg); 1424 1425 /* Canonicalize TCGv_i64 TEMP_CONST into TCGv_i32 TEMP_CONST. */ 1426 if (ts->kind == TEMP_CONST) { 1427 tcg_gen_movi_i64(ret, ts->val); 1428 } else { 1429 tcg_gen_mov_i32(TCGV_LOW(ret), TCGV_LOW(arg)); 1430 tcg_gen_mov_i32(TCGV_HIGH(ret), TCGV_HIGH(arg)); 1431 } 1432 } 1433 } 1434 1435 void tcg_gen_movi_i64(TCGv_i64 ret, int64_t arg) 1436 { 1437 if (TCG_TARGET_REG_BITS == 64) { 1438 tcg_gen_mov_i64(ret, tcg_constant_i64(arg)); 1439 } else { 1440 tcg_gen_movi_i32(TCGV_LOW(ret), arg); 1441 tcg_gen_movi_i32(TCGV_HIGH(ret), arg >> 32); 1442 } 1443 } 1444 1445 void tcg_gen_ld8u_i64(TCGv_i64 ret, TCGv_ptr arg2, tcg_target_long offset) 1446 { 1447 if (TCG_TARGET_REG_BITS == 64) { 1448 tcg_gen_ldst_op_i64(INDEX_op_ld8u_i64, ret, arg2, offset); 1449 } else { 1450 tcg_gen_ld8u_i32(TCGV_LOW(ret), arg2, offset); 1451 tcg_gen_movi_i32(TCGV_HIGH(ret), 0); 1452 } 1453 } 1454 1455 void tcg_gen_ld8s_i64(TCGv_i64 ret, TCGv_ptr arg2, tcg_target_long offset) 1456 { 1457 if (TCG_TARGET_REG_BITS == 64) { 1458 tcg_gen_ldst_op_i64(INDEX_op_ld8s_i64, ret, arg2, offset); 1459 } else { 1460 tcg_gen_ld8s_i32(TCGV_LOW(ret), arg2, offset); 1461 tcg_gen_sari_i32(TCGV_HIGH(ret), TCGV_LOW(ret), 31); 1462 } 1463 } 1464 1465 void tcg_gen_ld16u_i64(TCGv_i64 ret, TCGv_ptr arg2, tcg_target_long offset) 1466 { 1467 if (TCG_TARGET_REG_BITS == 64) { 1468 tcg_gen_ldst_op_i64(INDEX_op_ld16u_i64, ret, arg2, offset); 1469 } else { 1470 tcg_gen_ld16u_i32(TCGV_LOW(ret), arg2, offset); 1471 tcg_gen_movi_i32(TCGV_HIGH(ret), 0); 1472 } 1473 } 1474 1475 void tcg_gen_ld16s_i64(TCGv_i64 ret, TCGv_ptr arg2, tcg_target_long offset) 1476 { 1477 if (TCG_TARGET_REG_BITS == 64) { 1478 tcg_gen_ldst_op_i64(INDEX_op_ld16s_i64, ret, arg2, offset); 1479 } else { 1480 tcg_gen_ld16s_i32(TCGV_LOW(ret), arg2, offset); 1481 tcg_gen_sari_i32(TCGV_HIGH(ret), TCGV_LOW(ret), 31); 1482 } 1483 } 1484 1485 void tcg_gen_ld32u_i64(TCGv_i64 ret, TCGv_ptr arg2, tcg_target_long offset) 1486 { 1487 if (TCG_TARGET_REG_BITS == 64) { 1488 tcg_gen_ldst_op_i64(INDEX_op_ld32u_i64, ret, arg2, offset); 1489 } else { 1490 tcg_gen_ld_i32(TCGV_LOW(ret), arg2, offset); 1491 tcg_gen_movi_i32(TCGV_HIGH(ret), 0); 1492 } 1493 } 1494 1495 void tcg_gen_ld32s_i64(TCGv_i64 ret, TCGv_ptr arg2, tcg_target_long offset) 1496 { 1497 if (TCG_TARGET_REG_BITS == 64) { 1498 tcg_gen_ldst_op_i64(INDEX_op_ld32s_i64, ret, arg2, offset); 1499 } else { 1500 tcg_gen_ld_i32(TCGV_LOW(ret), arg2, offset); 1501 tcg_gen_sari_i32(TCGV_HIGH(ret), TCGV_LOW(ret), 31); 1502 } 1503 } 1504 1505 void tcg_gen_ld_i64(TCGv_i64 ret, TCGv_ptr arg2, tcg_target_long offset) 1506 { 1507 /* 1508 * For 32-bit host, since arg2 and ret have different types, 1509 * they cannot be the same temporary -- no chance of overlap. 1510 */ 1511 if (TCG_TARGET_REG_BITS == 64) { 1512 tcg_gen_ldst_op_i64(INDEX_op_ld_i64, ret, arg2, offset); 1513 } else if (HOST_BIG_ENDIAN) { 1514 tcg_gen_ld_i32(TCGV_HIGH(ret), arg2, offset); 1515 tcg_gen_ld_i32(TCGV_LOW(ret), arg2, offset + 4); 1516 } else { 1517 tcg_gen_ld_i32(TCGV_LOW(ret), arg2, offset); 1518 tcg_gen_ld_i32(TCGV_HIGH(ret), arg2, offset + 4); 1519 } 1520 } 1521 1522 void tcg_gen_st8_i64(TCGv_i64 arg1, TCGv_ptr arg2, tcg_target_long offset) 1523 { 1524 if (TCG_TARGET_REG_BITS == 64) { 1525 tcg_gen_ldst_op_i64(INDEX_op_st8_i64, arg1, arg2, offset); 1526 } else { 1527 tcg_gen_st8_i32(TCGV_LOW(arg1), arg2, offset); 1528 } 1529 } 1530 1531 void tcg_gen_st16_i64(TCGv_i64 arg1, TCGv_ptr arg2, tcg_target_long offset) 1532 { 1533 if (TCG_TARGET_REG_BITS == 64) { 1534 tcg_gen_ldst_op_i64(INDEX_op_st16_i64, arg1, arg2, offset); 1535 } else { 1536 tcg_gen_st16_i32(TCGV_LOW(arg1), arg2, offset); 1537 } 1538 } 1539 1540 void tcg_gen_st32_i64(TCGv_i64 arg1, TCGv_ptr arg2, tcg_target_long offset) 1541 { 1542 if (TCG_TARGET_REG_BITS == 64) { 1543 tcg_gen_ldst_op_i64(INDEX_op_st32_i64, arg1, arg2, offset); 1544 } else { 1545 tcg_gen_st_i32(TCGV_LOW(arg1), arg2, offset); 1546 } 1547 } 1548 1549 void tcg_gen_st_i64(TCGv_i64 arg1, TCGv_ptr arg2, tcg_target_long offset) 1550 { 1551 if (TCG_TARGET_REG_BITS == 64) { 1552 tcg_gen_ldst_op_i64(INDEX_op_st_i64, arg1, arg2, offset); 1553 } else if (HOST_BIG_ENDIAN) { 1554 tcg_gen_st_i32(TCGV_HIGH(arg1), arg2, offset); 1555 tcg_gen_st_i32(TCGV_LOW(arg1), arg2, offset + 4); 1556 } else { 1557 tcg_gen_st_i32(TCGV_LOW(arg1), arg2, offset); 1558 tcg_gen_st_i32(TCGV_HIGH(arg1), arg2, offset + 4); 1559 } 1560 } 1561 1562 void tcg_gen_add_i64(TCGv_i64 ret, TCGv_i64 arg1, TCGv_i64 arg2) 1563 { 1564 if (TCG_TARGET_REG_BITS == 64) { 1565 tcg_gen_op3_i64(INDEX_op_add, ret, arg1, arg2); 1566 } else { 1567 tcg_gen_add2_i32(TCGV_LOW(ret), TCGV_HIGH(ret), TCGV_LOW(arg1), 1568 TCGV_HIGH(arg1), TCGV_LOW(arg2), TCGV_HIGH(arg2)); 1569 } 1570 } 1571 1572 void tcg_gen_sub_i64(TCGv_i64 ret, TCGv_i64 arg1, TCGv_i64 arg2) 1573 { 1574 if (TCG_TARGET_REG_BITS == 64) { 1575 tcg_gen_op3_i64(INDEX_op_sub, ret, arg1, arg2); 1576 } else { 1577 tcg_gen_sub2_i32(TCGV_LOW(ret), TCGV_HIGH(ret), TCGV_LOW(arg1), 1578 TCGV_HIGH(arg1), TCGV_LOW(arg2), TCGV_HIGH(arg2)); 1579 } 1580 } 1581 1582 void tcg_gen_and_i64(TCGv_i64 ret, TCGv_i64 arg1, TCGv_i64 arg2) 1583 { 1584 if (TCG_TARGET_REG_BITS == 64) { 1585 tcg_gen_op3_i64(INDEX_op_and, ret, arg1, arg2); 1586 } else { 1587 tcg_gen_and_i32(TCGV_LOW(ret), TCGV_LOW(arg1), TCGV_LOW(arg2)); 1588 tcg_gen_and_i32(TCGV_HIGH(ret), TCGV_HIGH(arg1), TCGV_HIGH(arg2)); 1589 } 1590 } 1591 1592 void tcg_gen_or_i64(TCGv_i64 ret, TCGv_i64 arg1, TCGv_i64 arg2) 1593 { 1594 if (TCG_TARGET_REG_BITS == 64) { 1595 tcg_gen_op3_i64(INDEX_op_or, ret, arg1, arg2); 1596 } else { 1597 tcg_gen_or_i32(TCGV_LOW(ret), TCGV_LOW(arg1), TCGV_LOW(arg2)); 1598 tcg_gen_or_i32(TCGV_HIGH(ret), TCGV_HIGH(arg1), TCGV_HIGH(arg2)); 1599 } 1600 } 1601 1602 void tcg_gen_xor_i64(TCGv_i64 ret, TCGv_i64 arg1, TCGv_i64 arg2) 1603 { 1604 if (TCG_TARGET_REG_BITS == 64) { 1605 tcg_gen_op3_i64(INDEX_op_xor, ret, arg1, arg2); 1606 } else { 1607 tcg_gen_xor_i32(TCGV_LOW(ret), TCGV_LOW(arg1), TCGV_LOW(arg2)); 1608 tcg_gen_xor_i32(TCGV_HIGH(ret), TCGV_HIGH(arg1), TCGV_HIGH(arg2)); 1609 } 1610 } 1611 1612 void tcg_gen_shl_i64(TCGv_i64 ret, TCGv_i64 arg1, TCGv_i64 arg2) 1613 { 1614 if (TCG_TARGET_REG_BITS == 64) { 1615 tcg_gen_op3_i64(INDEX_op_shl, ret, arg1, arg2); 1616 } else { 1617 gen_helper_shl_i64(ret, arg1, arg2); 1618 } 1619 } 1620 1621 void tcg_gen_shr_i64(TCGv_i64 ret, TCGv_i64 arg1, TCGv_i64 arg2) 1622 { 1623 if (TCG_TARGET_REG_BITS == 64) { 1624 tcg_gen_op3_i64(INDEX_op_shr, ret, arg1, arg2); 1625 } else { 1626 gen_helper_shr_i64(ret, arg1, arg2); 1627 } 1628 } 1629 1630 void tcg_gen_sar_i64(TCGv_i64 ret, TCGv_i64 arg1, TCGv_i64 arg2) 1631 { 1632 if (TCG_TARGET_REG_BITS == 64) { 1633 tcg_gen_op3_i64(INDEX_op_sar, ret, arg1, arg2); 1634 } else { 1635 gen_helper_sar_i64(ret, arg1, arg2); 1636 } 1637 } 1638 1639 void tcg_gen_mul_i64(TCGv_i64 ret, TCGv_i64 arg1, TCGv_i64 arg2) 1640 { 1641 TCGv_i64 t0; 1642 TCGv_i32 t1; 1643 1644 if (TCG_TARGET_REG_BITS == 64) { 1645 tcg_gen_op3_i64(INDEX_op_mul, ret, arg1, arg2); 1646 return; 1647 } 1648 1649 1650 t0 = tcg_temp_ebb_new_i64(); 1651 t1 = tcg_temp_ebb_new_i32(); 1652 1653 tcg_gen_mulu2_i32(TCGV_LOW(t0), TCGV_HIGH(t0), 1654 TCGV_LOW(arg1), TCGV_LOW(arg2)); 1655 1656 tcg_gen_mul_i32(t1, TCGV_LOW(arg1), TCGV_HIGH(arg2)); 1657 tcg_gen_add_i32(TCGV_HIGH(t0), TCGV_HIGH(t0), t1); 1658 tcg_gen_mul_i32(t1, TCGV_HIGH(arg1), TCGV_LOW(arg2)); 1659 tcg_gen_add_i32(TCGV_HIGH(t0), TCGV_HIGH(t0), t1); 1660 1661 tcg_gen_mov_i64(ret, t0); 1662 tcg_temp_free_i64(t0); 1663 tcg_temp_free_i32(t1); 1664 } 1665 1666 void tcg_gen_addi_i64(TCGv_i64 ret, TCGv_i64 arg1, int64_t arg2) 1667 { 1668 /* some cases can be optimized here */ 1669 if (arg2 == 0) { 1670 tcg_gen_mov_i64(ret, arg1); 1671 } else if (TCG_TARGET_REG_BITS == 64) { 1672 tcg_gen_add_i64(ret, arg1, tcg_constant_i64(arg2)); 1673 } else { 1674 tcg_gen_add2_i32(TCGV_LOW(ret), TCGV_HIGH(ret), 1675 TCGV_LOW(arg1), TCGV_HIGH(arg1), 1676 tcg_constant_i32(arg2), tcg_constant_i32(arg2 >> 32)); 1677 } 1678 } 1679 1680 void tcg_gen_subfi_i64(TCGv_i64 ret, int64_t arg1, TCGv_i64 arg2) 1681 { 1682 if (arg1 == 0) { 1683 tcg_gen_neg_i64(ret, arg2); 1684 } else if (TCG_TARGET_REG_BITS == 64) { 1685 tcg_gen_sub_i64(ret, tcg_constant_i64(arg1), arg2); 1686 } else { 1687 tcg_gen_sub2_i32(TCGV_LOW(ret), TCGV_HIGH(ret), 1688 tcg_constant_i32(arg1), tcg_constant_i32(arg1 >> 32), 1689 TCGV_LOW(arg2), TCGV_HIGH(arg2)); 1690 } 1691 } 1692 1693 void tcg_gen_subi_i64(TCGv_i64 ret, TCGv_i64 arg1, int64_t arg2) 1694 { 1695 tcg_gen_addi_i64(ret, arg1, -arg2); 1696 } 1697 1698 void tcg_gen_neg_i64(TCGv_i64 ret, TCGv_i64 arg) 1699 { 1700 if (TCG_TARGET_REG_BITS == 64) { 1701 tcg_gen_op2_i64(INDEX_op_neg, ret, arg); 1702 } else { 1703 TCGv_i32 zero = tcg_constant_i32(0); 1704 tcg_gen_sub2_i32(TCGV_LOW(ret), TCGV_HIGH(ret), 1705 zero, zero, TCGV_LOW(arg), TCGV_HIGH(arg)); 1706 } 1707 } 1708 1709 void tcg_gen_andi_i64(TCGv_i64 ret, TCGv_i64 arg1, int64_t arg2) 1710 { 1711 if (TCG_TARGET_REG_BITS == 32) { 1712 tcg_gen_andi_i32(TCGV_LOW(ret), TCGV_LOW(arg1), arg2); 1713 tcg_gen_andi_i32(TCGV_HIGH(ret), TCGV_HIGH(arg1), arg2 >> 32); 1714 return; 1715 } 1716 1717 /* Some cases can be optimized here. */ 1718 switch (arg2) { 1719 case 0: 1720 tcg_gen_movi_i64(ret, 0); 1721 return; 1722 case -1: 1723 tcg_gen_mov_i64(ret, arg1); 1724 return; 1725 default: 1726 /* 1727 * Canonicalize on extract, if valid. This aids x86 with its 1728 * 2 operand MOVZBL and 2 operand AND, selecting the TCGOpcode 1729 * which does not require matching operands. Other backends can 1730 * trivially expand the extract to AND during code generation. 1731 */ 1732 if (!(arg2 & (arg2 + 1))) { 1733 unsigned len = ctz64(~arg2); 1734 if (TCG_TARGET_extract_valid(TCG_TYPE_I64, 0, len)) { 1735 tcg_gen_extract_i64(ret, arg1, 0, len); 1736 return; 1737 } 1738 } 1739 break; 1740 } 1741 1742 tcg_gen_and_i64(ret, arg1, tcg_constant_i64(arg2)); 1743 } 1744 1745 void tcg_gen_ori_i64(TCGv_i64 ret, TCGv_i64 arg1, int64_t arg2) 1746 { 1747 if (TCG_TARGET_REG_BITS == 32) { 1748 tcg_gen_ori_i32(TCGV_LOW(ret), TCGV_LOW(arg1), arg2); 1749 tcg_gen_ori_i32(TCGV_HIGH(ret), TCGV_HIGH(arg1), arg2 >> 32); 1750 return; 1751 } 1752 /* Some cases can be optimized here. */ 1753 if (arg2 == -1) { 1754 tcg_gen_movi_i64(ret, -1); 1755 } else if (arg2 == 0) { 1756 tcg_gen_mov_i64(ret, arg1); 1757 } else { 1758 tcg_gen_or_i64(ret, arg1, tcg_constant_i64(arg2)); 1759 } 1760 } 1761 1762 void tcg_gen_xori_i64(TCGv_i64 ret, TCGv_i64 arg1, int64_t arg2) 1763 { 1764 if (TCG_TARGET_REG_BITS == 32) { 1765 tcg_gen_xori_i32(TCGV_LOW(ret), TCGV_LOW(arg1), arg2); 1766 tcg_gen_xori_i32(TCGV_HIGH(ret), TCGV_HIGH(arg1), arg2 >> 32); 1767 return; 1768 } 1769 /* Some cases can be optimized here. */ 1770 if (arg2 == 0) { 1771 tcg_gen_mov_i64(ret, arg1); 1772 } else if (arg2 == -1 && 1773 tcg_op_supported(INDEX_op_not, TCG_TYPE_I64, 0)) { 1774 /* Don't recurse with tcg_gen_not_i64. */ 1775 tcg_gen_op2_i64(INDEX_op_not, ret, arg1); 1776 } else { 1777 tcg_gen_xor_i64(ret, arg1, tcg_constant_i64(arg2)); 1778 } 1779 } 1780 1781 static inline void tcg_gen_shifti_i64(TCGv_i64 ret, TCGv_i64 arg1, 1782 unsigned c, bool right, bool arith) 1783 { 1784 tcg_debug_assert(c < 64); 1785 if (c == 0) { 1786 tcg_gen_mov_i32(TCGV_LOW(ret), TCGV_LOW(arg1)); 1787 tcg_gen_mov_i32(TCGV_HIGH(ret), TCGV_HIGH(arg1)); 1788 } else if (c >= 32) { 1789 c -= 32; 1790 if (right) { 1791 if (arith) { 1792 tcg_gen_sari_i32(TCGV_LOW(ret), TCGV_HIGH(arg1), c); 1793 tcg_gen_sari_i32(TCGV_HIGH(ret), TCGV_HIGH(arg1), 31); 1794 } else { 1795 tcg_gen_shri_i32(TCGV_LOW(ret), TCGV_HIGH(arg1), c); 1796 tcg_gen_movi_i32(TCGV_HIGH(ret), 0); 1797 } 1798 } else { 1799 tcg_gen_shli_i32(TCGV_HIGH(ret), TCGV_LOW(arg1), c); 1800 tcg_gen_movi_i32(TCGV_LOW(ret), 0); 1801 } 1802 } else if (right) { 1803 if (TCG_TARGET_HAS_extract2_i32) { 1804 tcg_gen_extract2_i32(TCGV_LOW(ret), 1805 TCGV_LOW(arg1), TCGV_HIGH(arg1), c); 1806 } else { 1807 tcg_gen_shri_i32(TCGV_LOW(ret), TCGV_LOW(arg1), c); 1808 tcg_gen_deposit_i32(TCGV_LOW(ret), TCGV_LOW(ret), 1809 TCGV_HIGH(arg1), 32 - c, c); 1810 } 1811 if (arith) { 1812 tcg_gen_sari_i32(TCGV_HIGH(ret), TCGV_HIGH(arg1), c); 1813 } else { 1814 tcg_gen_shri_i32(TCGV_HIGH(ret), TCGV_HIGH(arg1), c); 1815 } 1816 } else { 1817 if (TCG_TARGET_HAS_extract2_i32) { 1818 tcg_gen_extract2_i32(TCGV_HIGH(ret), 1819 TCGV_LOW(arg1), TCGV_HIGH(arg1), 32 - c); 1820 } else { 1821 TCGv_i32 t0 = tcg_temp_ebb_new_i32(); 1822 tcg_gen_shri_i32(t0, TCGV_LOW(arg1), 32 - c); 1823 tcg_gen_deposit_i32(TCGV_HIGH(ret), t0, 1824 TCGV_HIGH(arg1), c, 32 - c); 1825 tcg_temp_free_i32(t0); 1826 } 1827 tcg_gen_shli_i32(TCGV_LOW(ret), TCGV_LOW(arg1), c); 1828 } 1829 } 1830 1831 void tcg_gen_shli_i64(TCGv_i64 ret, TCGv_i64 arg1, int64_t arg2) 1832 { 1833 tcg_debug_assert(arg2 >= 0 && arg2 < 64); 1834 if (TCG_TARGET_REG_BITS == 32) { 1835 tcg_gen_shifti_i64(ret, arg1, arg2, 0, 0); 1836 } else if (arg2 == 0) { 1837 tcg_gen_mov_i64(ret, arg1); 1838 } else { 1839 tcg_gen_shl_i64(ret, arg1, tcg_constant_i64(arg2)); 1840 } 1841 } 1842 1843 void tcg_gen_shri_i64(TCGv_i64 ret, TCGv_i64 arg1, int64_t arg2) 1844 { 1845 tcg_debug_assert(arg2 >= 0 && arg2 < 64); 1846 if (TCG_TARGET_REG_BITS == 32) { 1847 tcg_gen_shifti_i64(ret, arg1, arg2, 1, 0); 1848 } else if (arg2 == 0) { 1849 tcg_gen_mov_i64(ret, arg1); 1850 } else { 1851 tcg_gen_shr_i64(ret, arg1, tcg_constant_i64(arg2)); 1852 } 1853 } 1854 1855 void tcg_gen_sari_i64(TCGv_i64 ret, TCGv_i64 arg1, int64_t arg2) 1856 { 1857 tcg_debug_assert(arg2 >= 0 && arg2 < 64); 1858 if (TCG_TARGET_REG_BITS == 32) { 1859 tcg_gen_shifti_i64(ret, arg1, arg2, 1, 1); 1860 } else if (arg2 == 0) { 1861 tcg_gen_mov_i64(ret, arg1); 1862 } else { 1863 tcg_gen_sar_i64(ret, arg1, tcg_constant_i64(arg2)); 1864 } 1865 } 1866 1867 void tcg_gen_brcond_i64(TCGCond cond, TCGv_i64 arg1, TCGv_i64 arg2, TCGLabel *l) 1868 { 1869 if (cond == TCG_COND_ALWAYS) { 1870 tcg_gen_br(l); 1871 } else if (cond != TCG_COND_NEVER) { 1872 TCGOp *op; 1873 if (TCG_TARGET_REG_BITS == 32) { 1874 op = tcg_gen_op6ii_i32(INDEX_op_brcond2_i32, TCGV_LOW(arg1), 1875 TCGV_HIGH(arg1), TCGV_LOW(arg2), 1876 TCGV_HIGH(arg2), cond, label_arg(l)); 1877 } else { 1878 op = tcg_gen_op4ii_i64(INDEX_op_brcond_i64, arg1, arg2, cond, 1879 label_arg(l)); 1880 } 1881 add_as_label_use(l, op); 1882 } 1883 } 1884 1885 void tcg_gen_brcondi_i64(TCGCond cond, TCGv_i64 arg1, int64_t arg2, TCGLabel *l) 1886 { 1887 if (TCG_TARGET_REG_BITS == 64) { 1888 tcg_gen_brcond_i64(cond, arg1, tcg_constant_i64(arg2), l); 1889 } else if (cond == TCG_COND_ALWAYS) { 1890 tcg_gen_br(l); 1891 } else if (cond != TCG_COND_NEVER) { 1892 TCGOp *op = tcg_gen_op6ii_i32(INDEX_op_brcond2_i32, 1893 TCGV_LOW(arg1), TCGV_HIGH(arg1), 1894 tcg_constant_i32(arg2), 1895 tcg_constant_i32(arg2 >> 32), 1896 cond, label_arg(l)); 1897 add_as_label_use(l, op); 1898 } 1899 } 1900 1901 void tcg_gen_setcond_i64(TCGCond cond, TCGv_i64 ret, 1902 TCGv_i64 arg1, TCGv_i64 arg2) 1903 { 1904 if (cond == TCG_COND_ALWAYS) { 1905 tcg_gen_movi_i64(ret, 1); 1906 } else if (cond == TCG_COND_NEVER) { 1907 tcg_gen_movi_i64(ret, 0); 1908 } else { 1909 if (TCG_TARGET_REG_BITS == 32) { 1910 tcg_gen_op6i_i32(INDEX_op_setcond2_i32, TCGV_LOW(ret), 1911 TCGV_LOW(arg1), TCGV_HIGH(arg1), 1912 TCGV_LOW(arg2), TCGV_HIGH(arg2), cond); 1913 tcg_gen_movi_i32(TCGV_HIGH(ret), 0); 1914 } else { 1915 tcg_gen_op4i_i64(INDEX_op_setcond_i64, ret, arg1, arg2, cond); 1916 } 1917 } 1918 } 1919 1920 void tcg_gen_setcondi_i64(TCGCond cond, TCGv_i64 ret, 1921 TCGv_i64 arg1, int64_t arg2) 1922 { 1923 if (TCG_TARGET_REG_BITS == 64) { 1924 tcg_gen_setcond_i64(cond, ret, arg1, tcg_constant_i64(arg2)); 1925 } else if (cond == TCG_COND_ALWAYS) { 1926 tcg_gen_movi_i64(ret, 1); 1927 } else if (cond == TCG_COND_NEVER) { 1928 tcg_gen_movi_i64(ret, 0); 1929 } else { 1930 tcg_gen_op6i_i32(INDEX_op_setcond2_i32, TCGV_LOW(ret), 1931 TCGV_LOW(arg1), TCGV_HIGH(arg1), 1932 tcg_constant_i32(arg2), 1933 tcg_constant_i32(arg2 >> 32), cond); 1934 tcg_gen_movi_i32(TCGV_HIGH(ret), 0); 1935 } 1936 } 1937 1938 void tcg_gen_negsetcondi_i64(TCGCond cond, TCGv_i64 ret, 1939 TCGv_i64 arg1, int64_t arg2) 1940 { 1941 tcg_gen_negsetcond_i64(cond, ret, arg1, tcg_constant_i64(arg2)); 1942 } 1943 1944 void tcg_gen_negsetcond_i64(TCGCond cond, TCGv_i64 ret, 1945 TCGv_i64 arg1, TCGv_i64 arg2) 1946 { 1947 if (cond == TCG_COND_ALWAYS) { 1948 tcg_gen_movi_i64(ret, -1); 1949 } else if (cond == TCG_COND_NEVER) { 1950 tcg_gen_movi_i64(ret, 0); 1951 } else if (TCG_TARGET_HAS_negsetcond_i64) { 1952 tcg_gen_op4i_i64(INDEX_op_negsetcond_i64, ret, arg1, arg2, cond); 1953 } else if (TCG_TARGET_REG_BITS == 32) { 1954 tcg_gen_op6i_i32(INDEX_op_setcond2_i32, TCGV_LOW(ret), 1955 TCGV_LOW(arg1), TCGV_HIGH(arg1), 1956 TCGV_LOW(arg2), TCGV_HIGH(arg2), cond); 1957 tcg_gen_neg_i32(TCGV_LOW(ret), TCGV_LOW(ret)); 1958 tcg_gen_mov_i32(TCGV_HIGH(ret), TCGV_LOW(ret)); 1959 } else { 1960 tcg_gen_setcond_i64(cond, ret, arg1, arg2); 1961 tcg_gen_neg_i64(ret, ret); 1962 } 1963 } 1964 1965 void tcg_gen_muli_i64(TCGv_i64 ret, TCGv_i64 arg1, int64_t arg2) 1966 { 1967 if (arg2 == 0) { 1968 tcg_gen_movi_i64(ret, 0); 1969 } else if (is_power_of_2(arg2)) { 1970 tcg_gen_shli_i64(ret, arg1, ctz64(arg2)); 1971 } else { 1972 tcg_gen_mul_i64(ret, arg1, tcg_constant_i64(arg2)); 1973 } 1974 } 1975 1976 void tcg_gen_div_i64(TCGv_i64 ret, TCGv_i64 arg1, TCGv_i64 arg2) 1977 { 1978 if (tcg_op_supported(INDEX_op_divs, TCG_TYPE_I64, 0)) { 1979 tcg_gen_op3_i64(INDEX_op_divs, ret, arg1, arg2); 1980 } else if (tcg_op_supported(INDEX_op_divs2, TCG_TYPE_I64, 0)) { 1981 TCGv_i64 t0 = tcg_temp_ebb_new_i64(); 1982 tcg_gen_sari_i64(t0, arg1, 63); 1983 tcg_gen_op5_i64(INDEX_op_divs2, ret, t0, arg1, t0, arg2); 1984 tcg_temp_free_i64(t0); 1985 } else { 1986 gen_helper_div_i64(ret, arg1, arg2); 1987 } 1988 } 1989 1990 void tcg_gen_rem_i64(TCGv_i64 ret, TCGv_i64 arg1, TCGv_i64 arg2) 1991 { 1992 if (tcg_op_supported(INDEX_op_rems, TCG_TYPE_I64, 0)) { 1993 tcg_gen_op3_i64(INDEX_op_rems, ret, arg1, arg2); 1994 } else if (tcg_op_supported(INDEX_op_divs, TCG_TYPE_I64, 0)) { 1995 TCGv_i64 t0 = tcg_temp_ebb_new_i64(); 1996 tcg_gen_op3_i64(INDEX_op_divs, t0, arg1, arg2); 1997 tcg_gen_mul_i64(t0, t0, arg2); 1998 tcg_gen_sub_i64(ret, arg1, t0); 1999 tcg_temp_free_i64(t0); 2000 } else if (tcg_op_supported(INDEX_op_divs2, TCG_TYPE_I64, 0)) { 2001 TCGv_i64 t0 = tcg_temp_ebb_new_i64(); 2002 tcg_gen_sari_i64(t0, arg1, 63); 2003 tcg_gen_op5_i64(INDEX_op_divs2, t0, ret, arg1, t0, arg2); 2004 tcg_temp_free_i64(t0); 2005 } else { 2006 gen_helper_rem_i64(ret, arg1, arg2); 2007 } 2008 } 2009 2010 void tcg_gen_divu_i64(TCGv_i64 ret, TCGv_i64 arg1, TCGv_i64 arg2) 2011 { 2012 if (tcg_op_supported(INDEX_op_divu, TCG_TYPE_I64, 0)) { 2013 tcg_gen_op3_i64(INDEX_op_divu, ret, arg1, arg2); 2014 } else if (tcg_op_supported(INDEX_op_divu2, TCG_TYPE_I64, 0)) { 2015 TCGv_i64 t0 = tcg_temp_ebb_new_i64(); 2016 TCGv_i64 zero = tcg_constant_i64(0); 2017 tcg_gen_op5_i64(INDEX_op_divu2, ret, t0, arg1, zero, arg2); 2018 tcg_temp_free_i64(t0); 2019 } else { 2020 gen_helper_divu_i64(ret, arg1, arg2); 2021 } 2022 } 2023 2024 void tcg_gen_remu_i64(TCGv_i64 ret, TCGv_i64 arg1, TCGv_i64 arg2) 2025 { 2026 if (tcg_op_supported(INDEX_op_remu, TCG_TYPE_I64, 0)) { 2027 tcg_gen_op3_i64(INDEX_op_remu, ret, arg1, arg2); 2028 } else if (tcg_op_supported(INDEX_op_divu, TCG_TYPE_I64, 0)) { 2029 TCGv_i64 t0 = tcg_temp_ebb_new_i64(); 2030 tcg_gen_op3_i64(INDEX_op_divu, t0, arg1, arg2); 2031 tcg_gen_mul_i64(t0, t0, arg2); 2032 tcg_gen_sub_i64(ret, arg1, t0); 2033 tcg_temp_free_i64(t0); 2034 } else if (tcg_op_supported(INDEX_op_divu2, TCG_TYPE_I64, 0)) { 2035 TCGv_i64 t0 = tcg_temp_ebb_new_i64(); 2036 TCGv_i64 zero = tcg_constant_i64(0); 2037 tcg_gen_op5_i64(INDEX_op_divu2, t0, ret, arg1, zero, arg2); 2038 tcg_temp_free_i64(t0); 2039 } else { 2040 gen_helper_remu_i64(ret, arg1, arg2); 2041 } 2042 } 2043 2044 void tcg_gen_ext8s_i64(TCGv_i64 ret, TCGv_i64 arg) 2045 { 2046 tcg_gen_sextract_i64(ret, arg, 0, 8); 2047 } 2048 2049 void tcg_gen_ext16s_i64(TCGv_i64 ret, TCGv_i64 arg) 2050 { 2051 tcg_gen_sextract_i64(ret, arg, 0, 16); 2052 } 2053 2054 void tcg_gen_ext32s_i64(TCGv_i64 ret, TCGv_i64 arg) 2055 { 2056 tcg_gen_sextract_i64(ret, arg, 0, 32); 2057 } 2058 2059 void tcg_gen_ext8u_i64(TCGv_i64 ret, TCGv_i64 arg) 2060 { 2061 tcg_gen_extract_i64(ret, arg, 0, 8); 2062 } 2063 2064 void tcg_gen_ext16u_i64(TCGv_i64 ret, TCGv_i64 arg) 2065 { 2066 tcg_gen_extract_i64(ret, arg, 0, 16); 2067 } 2068 2069 void tcg_gen_ext32u_i64(TCGv_i64 ret, TCGv_i64 arg) 2070 { 2071 tcg_gen_extract_i64(ret, arg, 0, 32); 2072 } 2073 2074 /* 2075 * bswap16_i64: 16-bit byte swap on the low bits of a 64-bit value. 2076 * 2077 * Byte pattern: xxxxxxxxab -> yyyyyyyyba 2078 * 2079 * With TCG_BSWAP_IZ, x == zero, else undefined. 2080 * With TCG_BSWAP_OZ, y == zero, with TCG_BSWAP_OS y == sign, else undefined. 2081 */ 2082 void tcg_gen_bswap16_i64(TCGv_i64 ret, TCGv_i64 arg, int flags) 2083 { 2084 /* Only one extension flag may be present. */ 2085 tcg_debug_assert(!(flags & TCG_BSWAP_OS) || !(flags & TCG_BSWAP_OZ)); 2086 2087 if (TCG_TARGET_REG_BITS == 32) { 2088 tcg_gen_bswap16_i32(TCGV_LOW(ret), TCGV_LOW(arg), flags); 2089 if (flags & TCG_BSWAP_OS) { 2090 tcg_gen_sari_i32(TCGV_HIGH(ret), TCGV_LOW(ret), 31); 2091 } else { 2092 tcg_gen_movi_i32(TCGV_HIGH(ret), 0); 2093 } 2094 } else if (TCG_TARGET_HAS_bswap16_i64) { 2095 tcg_gen_op3i_i64(INDEX_op_bswap16_i64, ret, arg, flags); 2096 } else { 2097 TCGv_i64 t0 = tcg_temp_ebb_new_i64(); 2098 TCGv_i64 t1 = tcg_temp_ebb_new_i64(); 2099 2100 /* arg = ......ab or xxxxxxab */ 2101 tcg_gen_shri_i64(t0, arg, 8); /* t0 = .......a or .xxxxxxa */ 2102 if (!(flags & TCG_BSWAP_IZ)) { 2103 tcg_gen_ext8u_i64(t0, t0); /* t0 = .......a */ 2104 } 2105 2106 if (flags & TCG_BSWAP_OS) { 2107 tcg_gen_shli_i64(t1, arg, 56); /* t1 = b....... */ 2108 tcg_gen_sari_i64(t1, t1, 48); /* t1 = ssssssb. */ 2109 } else if (flags & TCG_BSWAP_OZ) { 2110 tcg_gen_ext8u_i64(t1, arg); /* t1 = .......b */ 2111 tcg_gen_shli_i64(t1, t1, 8); /* t1 = ......b. */ 2112 } else { 2113 tcg_gen_shli_i64(t1, arg, 8); /* t1 = xxxxxab. */ 2114 } 2115 2116 tcg_gen_or_i64(ret, t0, t1); /* ret = ......ba (OZ) */ 2117 /* ssssssba (OS) */ 2118 /* xxxxxaba (no flag) */ 2119 tcg_temp_free_i64(t0); 2120 tcg_temp_free_i64(t1); 2121 } 2122 } 2123 2124 /* 2125 * bswap32_i64: 32-bit byte swap on the low bits of a 64-bit value. 2126 * 2127 * Byte pattern: xxxxabcd -> yyyydcba 2128 * 2129 * With TCG_BSWAP_IZ, x == zero, else undefined. 2130 * With TCG_BSWAP_OZ, y == zero, with TCG_BSWAP_OS y == sign, else undefined. 2131 */ 2132 void tcg_gen_bswap32_i64(TCGv_i64 ret, TCGv_i64 arg, int flags) 2133 { 2134 /* Only one extension flag may be present. */ 2135 tcg_debug_assert(!(flags & TCG_BSWAP_OS) || !(flags & TCG_BSWAP_OZ)); 2136 2137 if (TCG_TARGET_REG_BITS == 32) { 2138 tcg_gen_bswap32_i32(TCGV_LOW(ret), TCGV_LOW(arg)); 2139 if (flags & TCG_BSWAP_OS) { 2140 tcg_gen_sari_i32(TCGV_HIGH(ret), TCGV_LOW(ret), 31); 2141 } else { 2142 tcg_gen_movi_i32(TCGV_HIGH(ret), 0); 2143 } 2144 } else if (TCG_TARGET_HAS_bswap32_i64) { 2145 tcg_gen_op3i_i64(INDEX_op_bswap32_i64, ret, arg, flags); 2146 } else { 2147 TCGv_i64 t0 = tcg_temp_ebb_new_i64(); 2148 TCGv_i64 t1 = tcg_temp_ebb_new_i64(); 2149 TCGv_i64 t2 = tcg_constant_i64(0x00ff00ff); 2150 2151 /* arg = xxxxabcd */ 2152 tcg_gen_shri_i64(t0, arg, 8); /* t0 = .xxxxabc */ 2153 tcg_gen_and_i64(t1, arg, t2); /* t1 = .....b.d */ 2154 tcg_gen_and_i64(t0, t0, t2); /* t0 = .....a.c */ 2155 tcg_gen_shli_i64(t1, t1, 8); /* t1 = ....b.d. */ 2156 tcg_gen_or_i64(ret, t0, t1); /* ret = ....badc */ 2157 2158 tcg_gen_shli_i64(t1, ret, 48); /* t1 = dc...... */ 2159 tcg_gen_shri_i64(t0, ret, 16); /* t0 = ......ba */ 2160 if (flags & TCG_BSWAP_OS) { 2161 tcg_gen_sari_i64(t1, t1, 32); /* t1 = ssssdc.. */ 2162 } else { 2163 tcg_gen_shri_i64(t1, t1, 32); /* t1 = ....dc.. */ 2164 } 2165 tcg_gen_or_i64(ret, t0, t1); /* ret = ssssdcba (OS) */ 2166 /* ....dcba (else) */ 2167 2168 tcg_temp_free_i64(t0); 2169 tcg_temp_free_i64(t1); 2170 } 2171 } 2172 2173 /* 2174 * bswap64_i64: 64-bit byte swap on a 64-bit value. 2175 * 2176 * Byte pattern: abcdefgh -> hgfedcba 2177 */ 2178 void tcg_gen_bswap64_i64(TCGv_i64 ret, TCGv_i64 arg) 2179 { 2180 if (TCG_TARGET_REG_BITS == 32) { 2181 TCGv_i32 t0, t1; 2182 t0 = tcg_temp_ebb_new_i32(); 2183 t1 = tcg_temp_ebb_new_i32(); 2184 2185 tcg_gen_bswap32_i32(t0, TCGV_LOW(arg)); 2186 tcg_gen_bswap32_i32(t1, TCGV_HIGH(arg)); 2187 tcg_gen_mov_i32(TCGV_LOW(ret), t1); 2188 tcg_gen_mov_i32(TCGV_HIGH(ret), t0); 2189 tcg_temp_free_i32(t0); 2190 tcg_temp_free_i32(t1); 2191 } else if (TCG_TARGET_HAS_bswap64_i64) { 2192 tcg_gen_op3i_i64(INDEX_op_bswap64_i64, ret, arg, 0); 2193 } else { 2194 TCGv_i64 t0 = tcg_temp_ebb_new_i64(); 2195 TCGv_i64 t1 = tcg_temp_ebb_new_i64(); 2196 TCGv_i64 t2 = tcg_temp_ebb_new_i64(); 2197 2198 /* arg = abcdefgh */ 2199 tcg_gen_movi_i64(t2, 0x00ff00ff00ff00ffull); 2200 tcg_gen_shri_i64(t0, arg, 8); /* t0 = .abcdefg */ 2201 tcg_gen_and_i64(t1, arg, t2); /* t1 = .b.d.f.h */ 2202 tcg_gen_and_i64(t0, t0, t2); /* t0 = .a.c.e.g */ 2203 tcg_gen_shli_i64(t1, t1, 8); /* t1 = b.d.f.h. */ 2204 tcg_gen_or_i64(ret, t0, t1); /* ret = badcfehg */ 2205 2206 tcg_gen_movi_i64(t2, 0x0000ffff0000ffffull); 2207 tcg_gen_shri_i64(t0, ret, 16); /* t0 = ..badcfe */ 2208 tcg_gen_and_i64(t1, ret, t2); /* t1 = ..dc..hg */ 2209 tcg_gen_and_i64(t0, t0, t2); /* t0 = ..ba..fe */ 2210 tcg_gen_shli_i64(t1, t1, 16); /* t1 = dc..hg.. */ 2211 tcg_gen_or_i64(ret, t0, t1); /* ret = dcbahgfe */ 2212 2213 tcg_gen_shri_i64(t0, ret, 32); /* t0 = ....dcba */ 2214 tcg_gen_shli_i64(t1, ret, 32); /* t1 = hgfe.... */ 2215 tcg_gen_or_i64(ret, t0, t1); /* ret = hgfedcba */ 2216 2217 tcg_temp_free_i64(t0); 2218 tcg_temp_free_i64(t1); 2219 tcg_temp_free_i64(t2); 2220 } 2221 } 2222 2223 /* 2224 * hswap_i64: Swap 16-bit halfwords within a 64-bit value. 2225 * See also include/qemu/bitops.h, hswap64. 2226 * 2227 * Byte pattern: abcdefgh -> ghefcdab 2228 */ 2229 void tcg_gen_hswap_i64(TCGv_i64 ret, TCGv_i64 arg) 2230 { 2231 uint64_t m = 0x0000ffff0000ffffull; 2232 TCGv_i64 t0 = tcg_temp_ebb_new_i64(); 2233 TCGv_i64 t1 = tcg_temp_ebb_new_i64(); 2234 2235 /* arg = abcdefgh */ 2236 tcg_gen_rotli_i64(t1, arg, 32); /* t1 = efghabcd */ 2237 tcg_gen_andi_i64(t0, t1, m); /* t0 = ..gh..cd */ 2238 tcg_gen_shli_i64(t0, t0, 16); /* t0 = gh..cd.. */ 2239 tcg_gen_shri_i64(t1, t1, 16); /* t1 = ..efghab */ 2240 tcg_gen_andi_i64(t1, t1, m); /* t1 = ..ef..ab */ 2241 tcg_gen_or_i64(ret, t0, t1); /* ret = ghefcdab */ 2242 2243 tcg_temp_free_i64(t0); 2244 tcg_temp_free_i64(t1); 2245 } 2246 2247 /* 2248 * wswap_i64: Swap 32-bit words within a 64-bit value. 2249 * 2250 * Byte pattern: abcdefgh -> efghabcd 2251 */ 2252 void tcg_gen_wswap_i64(TCGv_i64 ret, TCGv_i64 arg) 2253 { 2254 /* Swapping 2 32-bit elements is a rotate. */ 2255 tcg_gen_rotli_i64(ret, arg, 32); 2256 } 2257 2258 void tcg_gen_not_i64(TCGv_i64 ret, TCGv_i64 arg) 2259 { 2260 if (TCG_TARGET_REG_BITS == 32) { 2261 tcg_gen_not_i32(TCGV_LOW(ret), TCGV_LOW(arg)); 2262 tcg_gen_not_i32(TCGV_HIGH(ret), TCGV_HIGH(arg)); 2263 } else if (tcg_op_supported(INDEX_op_not, TCG_TYPE_I64, 0)) { 2264 tcg_gen_op2_i64(INDEX_op_not, ret, arg); 2265 } else { 2266 tcg_gen_xori_i64(ret, arg, -1); 2267 } 2268 } 2269 2270 void tcg_gen_andc_i64(TCGv_i64 ret, TCGv_i64 arg1, TCGv_i64 arg2) 2271 { 2272 if (TCG_TARGET_REG_BITS == 32) { 2273 tcg_gen_andc_i32(TCGV_LOW(ret), TCGV_LOW(arg1), TCGV_LOW(arg2)); 2274 tcg_gen_andc_i32(TCGV_HIGH(ret), TCGV_HIGH(arg1), TCGV_HIGH(arg2)); 2275 } else if (tcg_op_supported(INDEX_op_andc, TCG_TYPE_I64, 0)) { 2276 tcg_gen_op3_i64(INDEX_op_andc, ret, arg1, arg2); 2277 } else { 2278 TCGv_i64 t0 = tcg_temp_ebb_new_i64(); 2279 tcg_gen_not_i64(t0, arg2); 2280 tcg_gen_and_i64(ret, arg1, t0); 2281 tcg_temp_free_i64(t0); 2282 } 2283 } 2284 2285 void tcg_gen_eqv_i64(TCGv_i64 ret, TCGv_i64 arg1, TCGv_i64 arg2) 2286 { 2287 if (TCG_TARGET_REG_BITS == 32) { 2288 tcg_gen_eqv_i32(TCGV_LOW(ret), TCGV_LOW(arg1), TCGV_LOW(arg2)); 2289 tcg_gen_eqv_i32(TCGV_HIGH(ret), TCGV_HIGH(arg1), TCGV_HIGH(arg2)); 2290 } else if (tcg_op_supported(INDEX_op_eqv, TCG_TYPE_I64, 0)) { 2291 tcg_gen_op3_i64(INDEX_op_eqv, ret, arg1, arg2); 2292 } else { 2293 tcg_gen_xor_i64(ret, arg1, arg2); 2294 tcg_gen_not_i64(ret, ret); 2295 } 2296 } 2297 2298 void tcg_gen_nand_i64(TCGv_i64 ret, TCGv_i64 arg1, TCGv_i64 arg2) 2299 { 2300 if (TCG_TARGET_REG_BITS == 32) { 2301 tcg_gen_nand_i32(TCGV_LOW(ret), TCGV_LOW(arg1), TCGV_LOW(arg2)); 2302 tcg_gen_nand_i32(TCGV_HIGH(ret), TCGV_HIGH(arg1), TCGV_HIGH(arg2)); 2303 } else if (tcg_op_supported(INDEX_op_nand, TCG_TYPE_I64, 0)) { 2304 tcg_gen_op3_i64(INDEX_op_nand, ret, arg1, arg2); 2305 } else { 2306 tcg_gen_and_i64(ret, arg1, arg2); 2307 tcg_gen_not_i64(ret, ret); 2308 } 2309 } 2310 2311 void tcg_gen_nor_i64(TCGv_i64 ret, TCGv_i64 arg1, TCGv_i64 arg2) 2312 { 2313 if (TCG_TARGET_REG_BITS == 32) { 2314 tcg_gen_nor_i32(TCGV_LOW(ret), TCGV_LOW(arg1), TCGV_LOW(arg2)); 2315 tcg_gen_nor_i32(TCGV_HIGH(ret), TCGV_HIGH(arg1), TCGV_HIGH(arg2)); 2316 } else if (tcg_op_supported(INDEX_op_nor, TCG_TYPE_I64, 0)) { 2317 tcg_gen_op3_i64(INDEX_op_nor, ret, arg1, arg2); 2318 } else { 2319 tcg_gen_or_i64(ret, arg1, arg2); 2320 tcg_gen_not_i64(ret, ret); 2321 } 2322 } 2323 2324 void tcg_gen_orc_i64(TCGv_i64 ret, TCGv_i64 arg1, TCGv_i64 arg2) 2325 { 2326 if (TCG_TARGET_REG_BITS == 32) { 2327 tcg_gen_orc_i32(TCGV_LOW(ret), TCGV_LOW(arg1), TCGV_LOW(arg2)); 2328 tcg_gen_orc_i32(TCGV_HIGH(ret), TCGV_HIGH(arg1), TCGV_HIGH(arg2)); 2329 } else if (tcg_op_supported(INDEX_op_orc, TCG_TYPE_I64, 0)) { 2330 tcg_gen_op3_i64(INDEX_op_orc, ret, arg1, arg2); 2331 } else { 2332 TCGv_i64 t0 = tcg_temp_ebb_new_i64(); 2333 tcg_gen_not_i64(t0, arg2); 2334 tcg_gen_or_i64(ret, arg1, t0); 2335 tcg_temp_free_i64(t0); 2336 } 2337 } 2338 2339 void tcg_gen_clz_i64(TCGv_i64 ret, TCGv_i64 arg1, TCGv_i64 arg2) 2340 { 2341 if (tcg_op_supported(INDEX_op_clz, TCG_TYPE_I64, 0)) { 2342 tcg_gen_op3_i64(INDEX_op_clz, ret, arg1, arg2); 2343 } else { 2344 gen_helper_clz_i64(ret, arg1, arg2); 2345 } 2346 } 2347 2348 void tcg_gen_clzi_i64(TCGv_i64 ret, TCGv_i64 arg1, uint64_t arg2) 2349 { 2350 if (TCG_TARGET_REG_BITS == 32 2351 && arg2 <= 0xffffffffu 2352 && tcg_op_supported(INDEX_op_clz, TCG_TYPE_I32, 0)) { 2353 TCGv_i32 t = tcg_temp_ebb_new_i32(); 2354 tcg_gen_clzi_i32(t, TCGV_LOW(arg1), arg2 - 32); 2355 tcg_gen_addi_i32(t, t, 32); 2356 tcg_gen_clz_i32(TCGV_LOW(ret), TCGV_HIGH(arg1), t); 2357 tcg_gen_movi_i32(TCGV_HIGH(ret), 0); 2358 tcg_temp_free_i32(t); 2359 } else { 2360 tcg_gen_clz_i64(ret, arg1, tcg_constant_i64(arg2)); 2361 } 2362 } 2363 2364 void tcg_gen_ctz_i64(TCGv_i64 ret, TCGv_i64 arg1, TCGv_i64 arg2) 2365 { 2366 TCGv_i64 z, t; 2367 2368 if (TCG_TARGET_HAS_ctz_i64) { 2369 tcg_gen_op3_i64(INDEX_op_ctz_i64, ret, arg1, arg2); 2370 return; 2371 } 2372 if (TCG_TARGET_HAS_ctpop_i64) { 2373 t = tcg_temp_ebb_new_i64(); 2374 tcg_gen_subi_i64(t, arg1, 1); 2375 tcg_gen_andc_i64(t, t, arg1); 2376 tcg_gen_ctpop_i64(t, t); 2377 } else if (tcg_op_supported(INDEX_op_clz, TCG_TYPE_I64, 0)) { 2378 t = tcg_temp_ebb_new_i64(); 2379 tcg_gen_neg_i64(t, arg1); 2380 tcg_gen_and_i64(t, t, arg1); 2381 tcg_gen_clzi_i64(t, t, 64); 2382 tcg_gen_xori_i64(t, t, 63); 2383 } else { 2384 gen_helper_ctz_i64(ret, arg1, arg2); 2385 return; 2386 } 2387 2388 z = tcg_constant_i64(0); 2389 tcg_gen_movcond_i64(TCG_COND_EQ, ret, arg1, z, arg2, t); 2390 tcg_temp_free_i64(t); 2391 } 2392 2393 void tcg_gen_ctzi_i64(TCGv_i64 ret, TCGv_i64 arg1, uint64_t arg2) 2394 { 2395 if (TCG_TARGET_REG_BITS == 32 2396 && arg2 <= 0xffffffffu 2397 && tcg_op_supported(INDEX_op_ctz_i32, TCG_TYPE_I32, 0)) { 2398 TCGv_i32 t32 = tcg_temp_ebb_new_i32(); 2399 tcg_gen_ctzi_i32(t32, TCGV_HIGH(arg1), arg2 - 32); 2400 tcg_gen_addi_i32(t32, t32, 32); 2401 tcg_gen_ctz_i32(TCGV_LOW(ret), TCGV_LOW(arg1), t32); 2402 tcg_gen_movi_i32(TCGV_HIGH(ret), 0); 2403 tcg_temp_free_i32(t32); 2404 } else if (arg2 == 64 2405 && !tcg_op_supported(INDEX_op_ctz_i64, TCG_TYPE_I64, 0) 2406 && TCG_TARGET_HAS_ctpop_i64) { 2407 /* This equivalence has the advantage of not requiring a fixup. */ 2408 TCGv_i64 t = tcg_temp_ebb_new_i64(); 2409 tcg_gen_subi_i64(t, arg1, 1); 2410 tcg_gen_andc_i64(t, t, arg1); 2411 tcg_gen_ctpop_i64(ret, t); 2412 tcg_temp_free_i64(t); 2413 } else { 2414 tcg_gen_ctz_i64(ret, arg1, tcg_constant_i64(arg2)); 2415 } 2416 } 2417 2418 void tcg_gen_clrsb_i64(TCGv_i64 ret, TCGv_i64 arg) 2419 { 2420 if (tcg_op_supported(INDEX_op_clz, TCG_TYPE_I64, 0)) { 2421 TCGv_i64 t = tcg_temp_ebb_new_i64(); 2422 tcg_gen_sari_i64(t, arg, 63); 2423 tcg_gen_xor_i64(t, t, arg); 2424 tcg_gen_clzi_i64(t, t, 64); 2425 tcg_gen_subi_i64(ret, t, 1); 2426 tcg_temp_free_i64(t); 2427 } else { 2428 gen_helper_clrsb_i64(ret, arg); 2429 } 2430 } 2431 2432 void tcg_gen_ctpop_i64(TCGv_i64 ret, TCGv_i64 arg1) 2433 { 2434 if (TCG_TARGET_HAS_ctpop_i64) { 2435 tcg_gen_op2_i64(INDEX_op_ctpop_i64, ret, arg1); 2436 } else if (TCG_TARGET_REG_BITS == 32 && TCG_TARGET_HAS_ctpop_i32) { 2437 tcg_gen_ctpop_i32(TCGV_HIGH(ret), TCGV_HIGH(arg1)); 2438 tcg_gen_ctpop_i32(TCGV_LOW(ret), TCGV_LOW(arg1)); 2439 tcg_gen_add_i32(TCGV_LOW(ret), TCGV_LOW(ret), TCGV_HIGH(ret)); 2440 tcg_gen_movi_i32(TCGV_HIGH(ret), 0); 2441 } else { 2442 gen_helper_ctpop_i64(ret, arg1); 2443 } 2444 } 2445 2446 void tcg_gen_rotl_i64(TCGv_i64 ret, TCGv_i64 arg1, TCGv_i64 arg2) 2447 { 2448 if (tcg_op_supported(INDEX_op_rotl, TCG_TYPE_I64, 0)) { 2449 tcg_gen_op3_i64(INDEX_op_rotl, ret, arg1, arg2); 2450 } else if (tcg_op_supported(INDEX_op_rotl, TCG_TYPE_I64, 0)) { 2451 TCGv_i64 t0 = tcg_temp_ebb_new_i64(); 2452 tcg_gen_neg_i64(t0, arg2); 2453 tcg_gen_op3_i64(INDEX_op_rotr, ret, arg1, t0); 2454 tcg_temp_free_i64(t0); 2455 } else { 2456 TCGv_i64 t0 = tcg_temp_ebb_new_i64(); 2457 TCGv_i64 t1 = tcg_temp_ebb_new_i64(); 2458 tcg_gen_shl_i64(t0, arg1, arg2); 2459 tcg_gen_neg_i64(t1, arg2); 2460 tcg_gen_shr_i64(t1, arg1, t1); 2461 tcg_gen_or_i64(ret, t0, t1); 2462 tcg_temp_free_i64(t0); 2463 tcg_temp_free_i64(t1); 2464 } 2465 } 2466 2467 void tcg_gen_rotli_i64(TCGv_i64 ret, TCGv_i64 arg1, int64_t arg2) 2468 { 2469 tcg_debug_assert(arg2 >= 0 && arg2 < 64); 2470 /* some cases can be optimized here */ 2471 if (arg2 == 0) { 2472 tcg_gen_mov_i64(ret, arg1); 2473 } else if (tcg_op_supported(INDEX_op_rotl, TCG_TYPE_I64, 0)) { 2474 TCGv_i64 t0 = tcg_constant_i64(arg2); 2475 tcg_gen_op3_i64(INDEX_op_rotl, ret, arg1, t0); 2476 } else if (tcg_op_supported(INDEX_op_rotr, TCG_TYPE_I64, 0)) { 2477 TCGv_i64 t0 = tcg_constant_i64(64 - arg2); 2478 tcg_gen_op3_i64(INDEX_op_rotr, ret, arg1, t0); 2479 } else { 2480 TCGv_i64 t0 = tcg_temp_ebb_new_i64(); 2481 TCGv_i64 t1 = tcg_temp_ebb_new_i64(); 2482 tcg_gen_shli_i64(t0, arg1, arg2); 2483 tcg_gen_shri_i64(t1, arg1, 64 - arg2); 2484 tcg_gen_or_i64(ret, t0, t1); 2485 tcg_temp_free_i64(t0); 2486 tcg_temp_free_i64(t1); 2487 } 2488 } 2489 2490 void tcg_gen_rotr_i64(TCGv_i64 ret, TCGv_i64 arg1, TCGv_i64 arg2) 2491 { 2492 if (tcg_op_supported(INDEX_op_rotr, TCG_TYPE_I64, 0)) { 2493 tcg_gen_op3_i64(INDEX_op_rotr, ret, arg1, arg2); 2494 } else if (tcg_op_supported(INDEX_op_rotl, TCG_TYPE_I64, 0)) { 2495 TCGv_i64 t0 = tcg_temp_ebb_new_i64(); 2496 tcg_gen_neg_i64(t0, arg2); 2497 tcg_gen_op3_i64(INDEX_op_rotl, ret, arg1, t0); 2498 tcg_temp_free_i64(t0); 2499 } else { 2500 TCGv_i64 t0 = tcg_temp_ebb_new_i64(); 2501 TCGv_i64 t1 = tcg_temp_ebb_new_i64(); 2502 tcg_gen_shr_i64(t0, arg1, arg2); 2503 tcg_gen_neg_i64(t1, arg2); 2504 tcg_gen_shl_i64(t1, arg1, t1); 2505 tcg_gen_or_i64(ret, t0, t1); 2506 tcg_temp_free_i64(t0); 2507 tcg_temp_free_i64(t1); 2508 } 2509 } 2510 2511 void tcg_gen_rotri_i64(TCGv_i64 ret, TCGv_i64 arg1, int64_t arg2) 2512 { 2513 tcg_debug_assert(arg2 >= 0 && arg2 < 64); 2514 tcg_gen_rotli_i64(ret, arg1, -arg2 & 63); 2515 } 2516 2517 void tcg_gen_deposit_i64(TCGv_i64 ret, TCGv_i64 arg1, TCGv_i64 arg2, 2518 unsigned int ofs, unsigned int len) 2519 { 2520 uint64_t mask; 2521 TCGv_i64 t1; 2522 2523 tcg_debug_assert(ofs < 64); 2524 tcg_debug_assert(len > 0); 2525 tcg_debug_assert(len <= 64); 2526 tcg_debug_assert(ofs + len <= 64); 2527 2528 if (len == 64) { 2529 tcg_gen_mov_i64(ret, arg2); 2530 return; 2531 } 2532 2533 if (TCG_TARGET_REG_BITS == 64) { 2534 if (TCG_TARGET_deposit_valid(TCG_TYPE_I64, ofs, len)) { 2535 tcg_gen_op5ii_i64(INDEX_op_deposit_i64, ret, arg1, arg2, ofs, len); 2536 return; 2537 } 2538 } else { 2539 if (ofs >= 32) { 2540 tcg_gen_deposit_i32(TCGV_HIGH(ret), TCGV_HIGH(arg1), 2541 TCGV_LOW(arg2), ofs - 32, len); 2542 tcg_gen_mov_i32(TCGV_LOW(ret), TCGV_LOW(arg1)); 2543 return; 2544 } 2545 if (ofs + len <= 32) { 2546 tcg_gen_deposit_i32(TCGV_LOW(ret), TCGV_LOW(arg1), 2547 TCGV_LOW(arg2), ofs, len); 2548 tcg_gen_mov_i32(TCGV_HIGH(ret), TCGV_HIGH(arg1)); 2549 return; 2550 } 2551 } 2552 2553 t1 = tcg_temp_ebb_new_i64(); 2554 2555 if (TCG_TARGET_HAS_extract2_i64) { 2556 if (ofs + len == 64) { 2557 tcg_gen_shli_i64(t1, arg1, len); 2558 tcg_gen_extract2_i64(ret, t1, arg2, len); 2559 goto done; 2560 } 2561 if (ofs == 0) { 2562 tcg_gen_extract2_i64(ret, arg1, arg2, len); 2563 tcg_gen_rotli_i64(ret, ret, len); 2564 goto done; 2565 } 2566 } 2567 2568 mask = (1ull << len) - 1; 2569 if (ofs + len < 64) { 2570 tcg_gen_andi_i64(t1, arg2, mask); 2571 tcg_gen_shli_i64(t1, t1, ofs); 2572 } else { 2573 tcg_gen_shli_i64(t1, arg2, ofs); 2574 } 2575 tcg_gen_andi_i64(ret, arg1, ~(mask << ofs)); 2576 tcg_gen_or_i64(ret, ret, t1); 2577 done: 2578 tcg_temp_free_i64(t1); 2579 } 2580 2581 void tcg_gen_deposit_z_i64(TCGv_i64 ret, TCGv_i64 arg, 2582 unsigned int ofs, unsigned int len) 2583 { 2584 tcg_debug_assert(ofs < 64); 2585 tcg_debug_assert(len > 0); 2586 tcg_debug_assert(len <= 64); 2587 tcg_debug_assert(ofs + len <= 64); 2588 2589 if (ofs + len == 64) { 2590 tcg_gen_shli_i64(ret, arg, ofs); 2591 } else if (ofs == 0) { 2592 tcg_gen_andi_i64(ret, arg, (1ull << len) - 1); 2593 } else if (TCG_TARGET_REG_BITS == 64 && 2594 TCG_TARGET_deposit_valid(TCG_TYPE_I64, ofs, len)) { 2595 TCGv_i64 zero = tcg_constant_i64(0); 2596 tcg_gen_op5ii_i64(INDEX_op_deposit_i64, ret, zero, arg, ofs, len); 2597 } else { 2598 if (TCG_TARGET_REG_BITS == 32) { 2599 if (ofs >= 32) { 2600 tcg_gen_deposit_z_i32(TCGV_HIGH(ret), TCGV_LOW(arg), 2601 ofs - 32, len); 2602 tcg_gen_movi_i32(TCGV_LOW(ret), 0); 2603 return; 2604 } 2605 if (ofs + len <= 32) { 2606 tcg_gen_deposit_z_i32(TCGV_LOW(ret), TCGV_LOW(arg), ofs, len); 2607 tcg_gen_movi_i32(TCGV_HIGH(ret), 0); 2608 return; 2609 } 2610 } 2611 /* 2612 * To help two-operand hosts we prefer to zero-extend first, 2613 * which allows ARG to stay live. 2614 */ 2615 if (TCG_TARGET_extract_valid(TCG_TYPE_I64, 0, len)) { 2616 tcg_gen_extract_i64(ret, arg, 0, len); 2617 tcg_gen_shli_i64(ret, ret, ofs); 2618 return; 2619 } 2620 /* Otherwise prefer zero-extension over AND for code size. */ 2621 if (TCG_TARGET_extract_valid(TCG_TYPE_I64, 0, ofs + len)) { 2622 tcg_gen_shli_i64(ret, arg, ofs); 2623 tcg_gen_extract_i64(ret, ret, 0, ofs + len); 2624 return; 2625 } 2626 tcg_gen_andi_i64(ret, arg, (1ull << len) - 1); 2627 tcg_gen_shli_i64(ret, ret, ofs); 2628 } 2629 } 2630 2631 void tcg_gen_extract_i64(TCGv_i64 ret, TCGv_i64 arg, 2632 unsigned int ofs, unsigned int len) 2633 { 2634 tcg_debug_assert(ofs < 64); 2635 tcg_debug_assert(len > 0); 2636 tcg_debug_assert(len <= 64); 2637 tcg_debug_assert(ofs + len <= 64); 2638 2639 /* Canonicalize certain special cases, even if extract is supported. */ 2640 if (ofs + len == 64) { 2641 tcg_gen_shri_i64(ret, arg, 64 - len); 2642 return; 2643 } 2644 2645 if (TCG_TARGET_REG_BITS == 32) { 2646 /* Look for a 32-bit extract within one of the two words. */ 2647 if (ofs >= 32) { 2648 tcg_gen_extract_i32(TCGV_LOW(ret), TCGV_HIGH(arg), ofs - 32, len); 2649 tcg_gen_movi_i32(TCGV_HIGH(ret), 0); 2650 return; 2651 } 2652 if (ofs + len <= 32) { 2653 tcg_gen_extract_i32(TCGV_LOW(ret), TCGV_LOW(arg), ofs, len); 2654 tcg_gen_movi_i32(TCGV_HIGH(ret), 0); 2655 return; 2656 } 2657 2658 /* The field is split across two words. */ 2659 tcg_gen_extract2_i32(TCGV_LOW(ret), TCGV_LOW(arg), 2660 TCGV_HIGH(arg), ofs); 2661 if (len <= 32) { 2662 tcg_gen_extract_i32(TCGV_LOW(ret), TCGV_LOW(ret), 0, len); 2663 tcg_gen_movi_i32(TCGV_HIGH(ret), 0); 2664 } else { 2665 tcg_gen_extract_i32(TCGV_HIGH(ret), TCGV_HIGH(arg), 2666 ofs, len - 32); 2667 } 2668 return; 2669 } 2670 2671 if (TCG_TARGET_extract_valid(TCG_TYPE_I64, ofs, len)) { 2672 tcg_gen_op4ii_i64(INDEX_op_extract_i64, ret, arg, ofs, len); 2673 return; 2674 } 2675 if (ofs == 0) { 2676 tcg_gen_andi_i64(ret, arg, (1ull << len) - 1); 2677 return; 2678 } 2679 2680 /* Assume that zero-extension, if available, is cheaper than a shift. */ 2681 if (TCG_TARGET_extract_valid(TCG_TYPE_I64, 0, ofs + len)) { 2682 tcg_gen_op4ii_i64(INDEX_op_extract_i64, ret, arg, 0, ofs + len); 2683 tcg_gen_shri_i64(ret, ret, ofs); 2684 return; 2685 } 2686 2687 /* ??? Ideally we'd know what values are available for immediate AND. 2688 Assume that 8 bits are available, plus the special cases of 16 and 32, 2689 so that we get ext8u, ext16u, and ext32u. */ 2690 switch (len) { 2691 case 1 ... 8: case 16: case 32: 2692 tcg_gen_shri_i64(ret, arg, ofs); 2693 tcg_gen_andi_i64(ret, ret, (1ull << len) - 1); 2694 break; 2695 default: 2696 tcg_gen_shli_i64(ret, arg, 64 - len - ofs); 2697 tcg_gen_shri_i64(ret, ret, 64 - len); 2698 break; 2699 } 2700 } 2701 2702 void tcg_gen_sextract_i64(TCGv_i64 ret, TCGv_i64 arg, 2703 unsigned int ofs, unsigned int len) 2704 { 2705 tcg_debug_assert(ofs < 64); 2706 tcg_debug_assert(len > 0); 2707 tcg_debug_assert(len <= 64); 2708 tcg_debug_assert(ofs + len <= 64); 2709 2710 /* Canonicalize certain special cases, even if sextract is supported. */ 2711 if (ofs + len == 64) { 2712 tcg_gen_sari_i64(ret, arg, 64 - len); 2713 return; 2714 } 2715 2716 if (TCG_TARGET_REG_BITS == 32) { 2717 /* Look for a 32-bit extract within one of the two words. */ 2718 if (ofs >= 32) { 2719 tcg_gen_sextract_i32(TCGV_LOW(ret), TCGV_HIGH(arg), ofs - 32, len); 2720 } else if (ofs + len <= 32) { 2721 tcg_gen_sextract_i32(TCGV_LOW(ret), TCGV_LOW(arg), ofs, len); 2722 } else if (ofs == 0) { 2723 tcg_gen_mov_i32(TCGV_LOW(ret), TCGV_LOW(arg)); 2724 tcg_gen_sextract_i32(TCGV_HIGH(ret), TCGV_HIGH(arg), 0, len - 32); 2725 return; 2726 } else if (len > 32) { 2727 TCGv_i32 t = tcg_temp_ebb_new_i32(); 2728 /* Extract the bits for the high word normally. */ 2729 tcg_gen_sextract_i32(t, TCGV_HIGH(arg), ofs + 32, len - 32); 2730 /* Shift the field down for the low part. */ 2731 tcg_gen_shri_i64(ret, arg, ofs); 2732 /* Overwrite the shift into the high part. */ 2733 tcg_gen_mov_i32(TCGV_HIGH(ret), t); 2734 tcg_temp_free_i32(t); 2735 return; 2736 } else { 2737 /* Shift the field down for the low part, such that the 2738 field sits at the MSB. */ 2739 tcg_gen_shri_i64(ret, arg, ofs + len - 32); 2740 /* Shift the field down from the MSB, sign extending. */ 2741 tcg_gen_sari_i32(TCGV_LOW(ret), TCGV_LOW(ret), 32 - len); 2742 } 2743 /* Sign-extend the field from 32 bits. */ 2744 tcg_gen_sari_i32(TCGV_HIGH(ret), TCGV_LOW(ret), 31); 2745 return; 2746 } 2747 2748 if (TCG_TARGET_sextract_valid(TCG_TYPE_I64, ofs, len)) { 2749 tcg_gen_op4ii_i64(INDEX_op_sextract_i64, ret, arg, ofs, len); 2750 return; 2751 } 2752 2753 /* Assume that sign-extension, if available, is cheaper than a shift. */ 2754 if (TCG_TARGET_sextract_valid(TCG_TYPE_I64, 0, ofs + len)) { 2755 tcg_gen_op4ii_i64(INDEX_op_sextract_i64, ret, arg, 0, ofs + len); 2756 tcg_gen_sari_i64(ret, ret, ofs); 2757 return; 2758 } 2759 if (TCG_TARGET_sextract_valid(TCG_TYPE_I64, 0, len)) { 2760 tcg_gen_shri_i64(ret, arg, ofs); 2761 tcg_gen_op4ii_i64(INDEX_op_sextract_i64, ret, ret, 0, len); 2762 return; 2763 } 2764 2765 tcg_gen_shli_i64(ret, arg, 64 - len - ofs); 2766 tcg_gen_sari_i64(ret, ret, 64 - len); 2767 } 2768 2769 /* 2770 * Extract 64 bits from a 128-bit input, ah:al, starting from ofs. 2771 * Unlike tcg_gen_extract_i64 above, len is fixed at 64. 2772 */ 2773 void tcg_gen_extract2_i64(TCGv_i64 ret, TCGv_i64 al, TCGv_i64 ah, 2774 unsigned int ofs) 2775 { 2776 tcg_debug_assert(ofs <= 64); 2777 if (ofs == 0) { 2778 tcg_gen_mov_i64(ret, al); 2779 } else if (ofs == 64) { 2780 tcg_gen_mov_i64(ret, ah); 2781 } else if (al == ah) { 2782 tcg_gen_rotri_i64(ret, al, ofs); 2783 } else if (TCG_TARGET_HAS_extract2_i64) { 2784 tcg_gen_op4i_i64(INDEX_op_extract2_i64, ret, al, ah, ofs); 2785 } else { 2786 TCGv_i64 t0 = tcg_temp_ebb_new_i64(); 2787 tcg_gen_shri_i64(t0, al, ofs); 2788 tcg_gen_deposit_i64(ret, t0, ah, 64 - ofs, ofs); 2789 tcg_temp_free_i64(t0); 2790 } 2791 } 2792 2793 void tcg_gen_movcond_i64(TCGCond cond, TCGv_i64 ret, TCGv_i64 c1, 2794 TCGv_i64 c2, TCGv_i64 v1, TCGv_i64 v2) 2795 { 2796 if (cond == TCG_COND_ALWAYS) { 2797 tcg_gen_mov_i64(ret, v1); 2798 } else if (cond == TCG_COND_NEVER) { 2799 tcg_gen_mov_i64(ret, v2); 2800 } else if (TCG_TARGET_REG_BITS == 64) { 2801 tcg_gen_op6i_i64(INDEX_op_movcond_i64, ret, c1, c2, v1, v2, cond); 2802 } else { 2803 TCGv_i32 t0 = tcg_temp_ebb_new_i32(); 2804 TCGv_i32 zero = tcg_constant_i32(0); 2805 2806 tcg_gen_op6i_i32(INDEX_op_setcond2_i32, t0, 2807 TCGV_LOW(c1), TCGV_HIGH(c1), 2808 TCGV_LOW(c2), TCGV_HIGH(c2), cond); 2809 2810 tcg_gen_movcond_i32(TCG_COND_NE, TCGV_LOW(ret), t0, zero, 2811 TCGV_LOW(v1), TCGV_LOW(v2)); 2812 tcg_gen_movcond_i32(TCG_COND_NE, TCGV_HIGH(ret), t0, zero, 2813 TCGV_HIGH(v1), TCGV_HIGH(v2)); 2814 2815 tcg_temp_free_i32(t0); 2816 } 2817 } 2818 2819 void tcg_gen_add2_i64(TCGv_i64 rl, TCGv_i64 rh, TCGv_i64 al, 2820 TCGv_i64 ah, TCGv_i64 bl, TCGv_i64 bh) 2821 { 2822 if (TCG_TARGET_HAS_add2_i64) { 2823 tcg_gen_op6_i64(INDEX_op_add2_i64, rl, rh, al, ah, bl, bh); 2824 } else { 2825 TCGv_i64 t0 = tcg_temp_ebb_new_i64(); 2826 TCGv_i64 t1 = tcg_temp_ebb_new_i64(); 2827 tcg_gen_add_i64(t0, al, bl); 2828 tcg_gen_setcond_i64(TCG_COND_LTU, t1, t0, al); 2829 tcg_gen_add_i64(rh, ah, bh); 2830 tcg_gen_add_i64(rh, rh, t1); 2831 tcg_gen_mov_i64(rl, t0); 2832 tcg_temp_free_i64(t0); 2833 tcg_temp_free_i64(t1); 2834 } 2835 } 2836 2837 void tcg_gen_sub2_i64(TCGv_i64 rl, TCGv_i64 rh, TCGv_i64 al, 2838 TCGv_i64 ah, TCGv_i64 bl, TCGv_i64 bh) 2839 { 2840 if (TCG_TARGET_HAS_sub2_i64) { 2841 tcg_gen_op6_i64(INDEX_op_sub2_i64, rl, rh, al, ah, bl, bh); 2842 } else { 2843 TCGv_i64 t0 = tcg_temp_ebb_new_i64(); 2844 TCGv_i64 t1 = tcg_temp_ebb_new_i64(); 2845 tcg_gen_sub_i64(t0, al, bl); 2846 tcg_gen_setcond_i64(TCG_COND_LTU, t1, al, bl); 2847 tcg_gen_sub_i64(rh, ah, bh); 2848 tcg_gen_sub_i64(rh, rh, t1); 2849 tcg_gen_mov_i64(rl, t0); 2850 tcg_temp_free_i64(t0); 2851 tcg_temp_free_i64(t1); 2852 } 2853 } 2854 2855 void tcg_gen_mulu2_i64(TCGv_i64 rl, TCGv_i64 rh, TCGv_i64 arg1, TCGv_i64 arg2) 2856 { 2857 if (TCG_TARGET_HAS_mulu2_i64) { 2858 tcg_gen_op4_i64(INDEX_op_mulu2_i64, rl, rh, arg1, arg2); 2859 } else if (tcg_op_supported(INDEX_op_muluh, TCG_TYPE_I64, 0)) { 2860 TCGv_i64 t = tcg_temp_ebb_new_i64(); 2861 tcg_gen_op3_i64(INDEX_op_mul, t, arg1, arg2); 2862 tcg_gen_op3_i64(INDEX_op_muluh, rh, arg1, arg2); 2863 tcg_gen_mov_i64(rl, t); 2864 tcg_temp_free_i64(t); 2865 } else { 2866 TCGv_i64 t0 = tcg_temp_ebb_new_i64(); 2867 tcg_gen_mul_i64(t0, arg1, arg2); 2868 gen_helper_muluh_i64(rh, arg1, arg2); 2869 tcg_gen_mov_i64(rl, t0); 2870 tcg_temp_free_i64(t0); 2871 } 2872 } 2873 2874 void tcg_gen_muls2_i64(TCGv_i64 rl, TCGv_i64 rh, TCGv_i64 arg1, TCGv_i64 arg2) 2875 { 2876 if (TCG_TARGET_HAS_muls2_i64) { 2877 tcg_gen_op4_i64(INDEX_op_muls2_i64, rl, rh, arg1, arg2); 2878 } else if (tcg_op_supported(INDEX_op_mulsh, TCG_TYPE_I64, 0)) { 2879 TCGv_i64 t = tcg_temp_ebb_new_i64(); 2880 tcg_gen_op3_i64(INDEX_op_mul, t, arg1, arg2); 2881 tcg_gen_op3_i64(INDEX_op_mulsh, rh, arg1, arg2); 2882 tcg_gen_mov_i64(rl, t); 2883 tcg_temp_free_i64(t); 2884 } else if (TCG_TARGET_HAS_mulu2_i64 || 2885 tcg_op_supported(INDEX_op_muluh, TCG_TYPE_I64, 0)) { 2886 TCGv_i64 t0 = tcg_temp_ebb_new_i64(); 2887 TCGv_i64 t1 = tcg_temp_ebb_new_i64(); 2888 TCGv_i64 t2 = tcg_temp_ebb_new_i64(); 2889 TCGv_i64 t3 = tcg_temp_ebb_new_i64(); 2890 tcg_gen_mulu2_i64(t0, t1, arg1, arg2); 2891 /* Adjust for negative inputs. */ 2892 tcg_gen_sari_i64(t2, arg1, 63); 2893 tcg_gen_sari_i64(t3, arg2, 63); 2894 tcg_gen_and_i64(t2, t2, arg2); 2895 tcg_gen_and_i64(t3, t3, arg1); 2896 tcg_gen_sub_i64(rh, t1, t2); 2897 tcg_gen_sub_i64(rh, rh, t3); 2898 tcg_gen_mov_i64(rl, t0); 2899 tcg_temp_free_i64(t0); 2900 tcg_temp_free_i64(t1); 2901 tcg_temp_free_i64(t2); 2902 tcg_temp_free_i64(t3); 2903 } else { 2904 TCGv_i64 t0 = tcg_temp_ebb_new_i64(); 2905 tcg_gen_mul_i64(t0, arg1, arg2); 2906 gen_helper_mulsh_i64(rh, arg1, arg2); 2907 tcg_gen_mov_i64(rl, t0); 2908 tcg_temp_free_i64(t0); 2909 } 2910 } 2911 2912 void tcg_gen_mulsu2_i64(TCGv_i64 rl, TCGv_i64 rh, TCGv_i64 arg1, TCGv_i64 arg2) 2913 { 2914 TCGv_i64 t0 = tcg_temp_ebb_new_i64(); 2915 TCGv_i64 t1 = tcg_temp_ebb_new_i64(); 2916 TCGv_i64 t2 = tcg_temp_ebb_new_i64(); 2917 tcg_gen_mulu2_i64(t0, t1, arg1, arg2); 2918 /* Adjust for negative input for the signed arg1. */ 2919 tcg_gen_sari_i64(t2, arg1, 63); 2920 tcg_gen_and_i64(t2, t2, arg2); 2921 tcg_gen_sub_i64(rh, t1, t2); 2922 tcg_gen_mov_i64(rl, t0); 2923 tcg_temp_free_i64(t0); 2924 tcg_temp_free_i64(t1); 2925 tcg_temp_free_i64(t2); 2926 } 2927 2928 void tcg_gen_smin_i64(TCGv_i64 ret, TCGv_i64 a, TCGv_i64 b) 2929 { 2930 tcg_gen_movcond_i64(TCG_COND_LT, ret, a, b, a, b); 2931 } 2932 2933 void tcg_gen_umin_i64(TCGv_i64 ret, TCGv_i64 a, TCGv_i64 b) 2934 { 2935 tcg_gen_movcond_i64(TCG_COND_LTU, ret, a, b, a, b); 2936 } 2937 2938 void tcg_gen_smax_i64(TCGv_i64 ret, TCGv_i64 a, TCGv_i64 b) 2939 { 2940 tcg_gen_movcond_i64(TCG_COND_LT, ret, a, b, b, a); 2941 } 2942 2943 void tcg_gen_umax_i64(TCGv_i64 ret, TCGv_i64 a, TCGv_i64 b) 2944 { 2945 tcg_gen_movcond_i64(TCG_COND_LTU, ret, a, b, b, a); 2946 } 2947 2948 void tcg_gen_abs_i64(TCGv_i64 ret, TCGv_i64 a) 2949 { 2950 TCGv_i64 t = tcg_temp_ebb_new_i64(); 2951 2952 tcg_gen_sari_i64(t, a, 63); 2953 tcg_gen_xor_i64(ret, a, t); 2954 tcg_gen_sub_i64(ret, ret, t); 2955 tcg_temp_free_i64(t); 2956 } 2957 2958 /* Size changing operations. */ 2959 2960 void tcg_gen_extrl_i64_i32(TCGv_i32 ret, TCGv_i64 arg) 2961 { 2962 if (TCG_TARGET_REG_BITS == 32) { 2963 tcg_gen_mov_i32(ret, TCGV_LOW(arg)); 2964 } else if (TCG_TARGET_HAS_extr_i64_i32) { 2965 tcg_gen_op2(INDEX_op_extrl_i64_i32, TCG_TYPE_I32, 2966 tcgv_i32_arg(ret), tcgv_i64_arg(arg)); 2967 } else { 2968 tcg_gen_mov_i32(ret, (TCGv_i32)arg); 2969 } 2970 } 2971 2972 void tcg_gen_extrh_i64_i32(TCGv_i32 ret, TCGv_i64 arg) 2973 { 2974 if (TCG_TARGET_REG_BITS == 32) { 2975 tcg_gen_mov_i32(ret, TCGV_HIGH(arg)); 2976 } else if (TCG_TARGET_HAS_extr_i64_i32) { 2977 tcg_gen_op2(INDEX_op_extrh_i64_i32, TCG_TYPE_I32, 2978 tcgv_i32_arg(ret), tcgv_i64_arg(arg)); 2979 } else { 2980 TCGv_i64 t = tcg_temp_ebb_new_i64(); 2981 tcg_gen_shri_i64(t, arg, 32); 2982 tcg_gen_mov_i32(ret, (TCGv_i32)t); 2983 tcg_temp_free_i64(t); 2984 } 2985 } 2986 2987 void tcg_gen_extu_i32_i64(TCGv_i64 ret, TCGv_i32 arg) 2988 { 2989 if (TCG_TARGET_REG_BITS == 32) { 2990 tcg_gen_mov_i32(TCGV_LOW(ret), arg); 2991 tcg_gen_movi_i32(TCGV_HIGH(ret), 0); 2992 } else { 2993 tcg_gen_op2(INDEX_op_extu_i32_i64, TCG_TYPE_I64, 2994 tcgv_i64_arg(ret), tcgv_i32_arg(arg)); 2995 } 2996 } 2997 2998 void tcg_gen_ext_i32_i64(TCGv_i64 ret, TCGv_i32 arg) 2999 { 3000 if (TCG_TARGET_REG_BITS == 32) { 3001 tcg_gen_mov_i32(TCGV_LOW(ret), arg); 3002 tcg_gen_sari_i32(TCGV_HIGH(ret), TCGV_LOW(ret), 31); 3003 } else { 3004 tcg_gen_op2(INDEX_op_ext_i32_i64, TCG_TYPE_I64, 3005 tcgv_i64_arg(ret), tcgv_i32_arg(arg)); 3006 } 3007 } 3008 3009 void tcg_gen_concat_i32_i64(TCGv_i64 dest, TCGv_i32 low, TCGv_i32 high) 3010 { 3011 TCGv_i64 tmp; 3012 3013 if (TCG_TARGET_REG_BITS == 32) { 3014 tcg_gen_mov_i32(TCGV_LOW(dest), low); 3015 tcg_gen_mov_i32(TCGV_HIGH(dest), high); 3016 return; 3017 } 3018 3019 tmp = tcg_temp_ebb_new_i64(); 3020 /* These extensions are only needed for type correctness. 3021 We may be able to do better given target specific information. */ 3022 tcg_gen_extu_i32_i64(tmp, high); 3023 tcg_gen_extu_i32_i64(dest, low); 3024 /* If deposit is available, use it. Otherwise use the extra 3025 knowledge that we have of the zero-extensions above. */ 3026 if (TCG_TARGET_deposit_valid(TCG_TYPE_I64, 32, 32)) { 3027 tcg_gen_deposit_i64(dest, dest, tmp, 32, 32); 3028 } else { 3029 tcg_gen_shli_i64(tmp, tmp, 32); 3030 tcg_gen_or_i64(dest, dest, tmp); 3031 } 3032 tcg_temp_free_i64(tmp); 3033 } 3034 3035 void tcg_gen_extr_i64_i32(TCGv_i32 lo, TCGv_i32 hi, TCGv_i64 arg) 3036 { 3037 if (TCG_TARGET_REG_BITS == 32) { 3038 tcg_gen_mov_i32(lo, TCGV_LOW(arg)); 3039 tcg_gen_mov_i32(hi, TCGV_HIGH(arg)); 3040 } else { 3041 tcg_gen_extrl_i64_i32(lo, arg); 3042 tcg_gen_extrh_i64_i32(hi, arg); 3043 } 3044 } 3045 3046 void tcg_gen_extr32_i64(TCGv_i64 lo, TCGv_i64 hi, TCGv_i64 arg) 3047 { 3048 tcg_gen_ext32u_i64(lo, arg); 3049 tcg_gen_shri_i64(hi, arg, 32); 3050 } 3051 3052 void tcg_gen_concat32_i64(TCGv_i64 ret, TCGv_i64 lo, TCGv_i64 hi) 3053 { 3054 tcg_gen_deposit_i64(ret, lo, hi, 32, 32); 3055 } 3056 3057 void tcg_gen_extr_i128_i64(TCGv_i64 lo, TCGv_i64 hi, TCGv_i128 arg) 3058 { 3059 tcg_gen_mov_i64(lo, TCGV128_LOW(arg)); 3060 tcg_gen_mov_i64(hi, TCGV128_HIGH(arg)); 3061 } 3062 3063 void tcg_gen_concat_i64_i128(TCGv_i128 ret, TCGv_i64 lo, TCGv_i64 hi) 3064 { 3065 tcg_gen_mov_i64(TCGV128_LOW(ret), lo); 3066 tcg_gen_mov_i64(TCGV128_HIGH(ret), hi); 3067 } 3068 3069 void tcg_gen_mov_i128(TCGv_i128 dst, TCGv_i128 src) 3070 { 3071 if (dst != src) { 3072 tcg_gen_mov_i64(TCGV128_LOW(dst), TCGV128_LOW(src)); 3073 tcg_gen_mov_i64(TCGV128_HIGH(dst), TCGV128_HIGH(src)); 3074 } 3075 } 3076 3077 void tcg_gen_ld_i128(TCGv_i128 ret, TCGv_ptr base, tcg_target_long offset) 3078 { 3079 if (HOST_BIG_ENDIAN) { 3080 tcg_gen_ld_i64(TCGV128_HIGH(ret), base, offset); 3081 tcg_gen_ld_i64(TCGV128_LOW(ret), base, offset + 8); 3082 } else { 3083 tcg_gen_ld_i64(TCGV128_LOW(ret), base, offset); 3084 tcg_gen_ld_i64(TCGV128_HIGH(ret), base, offset + 8); 3085 } 3086 } 3087 3088 void tcg_gen_st_i128(TCGv_i128 val, TCGv_ptr base, tcg_target_long offset) 3089 { 3090 if (HOST_BIG_ENDIAN) { 3091 tcg_gen_st_i64(TCGV128_HIGH(val), base, offset); 3092 tcg_gen_st_i64(TCGV128_LOW(val), base, offset + 8); 3093 } else { 3094 tcg_gen_st_i64(TCGV128_LOW(val), base, offset); 3095 tcg_gen_st_i64(TCGV128_HIGH(val), base, offset + 8); 3096 } 3097 } 3098 3099 /* QEMU specific operations. */ 3100 3101 void tcg_gen_exit_tb(const TranslationBlock *tb, unsigned idx) 3102 { 3103 /* 3104 * Let the jit code return the read-only version of the 3105 * TranslationBlock, so that we minimize the pc-relative 3106 * distance of the address of the exit_tb code to TB. 3107 * This will improve utilization of pc-relative address loads. 3108 * 3109 * TODO: Move this to translator_loop, so that all const 3110 * TranslationBlock pointers refer to read-only memory. 3111 * This requires coordination with targets that do not use 3112 * the translator_loop. 3113 */ 3114 uintptr_t val = (uintptr_t)tcg_splitwx_to_rx((void *)tb) + idx; 3115 3116 if (tb == NULL) { 3117 tcg_debug_assert(idx == 0); 3118 } else if (idx <= TB_EXIT_IDXMAX) { 3119 #ifdef CONFIG_DEBUG_TCG 3120 /* This is an exit following a goto_tb. Verify that we have 3121 seen this numbered exit before, via tcg_gen_goto_tb. */ 3122 tcg_debug_assert(tcg_ctx->goto_tb_issue_mask & (1 << idx)); 3123 #endif 3124 } else { 3125 /* This is an exit via the exitreq label. */ 3126 tcg_debug_assert(idx == TB_EXIT_REQUESTED); 3127 } 3128 3129 tcg_gen_op1i(INDEX_op_exit_tb, 0, val); 3130 } 3131 3132 void tcg_gen_goto_tb(unsigned idx) 3133 { 3134 /* We tested CF_NO_GOTO_TB in translator_use_goto_tb. */ 3135 tcg_debug_assert(!(tcg_ctx->gen_tb->cflags & CF_NO_GOTO_TB)); 3136 /* We only support two chained exits. */ 3137 tcg_debug_assert(idx <= TB_EXIT_IDXMAX); 3138 #ifdef CONFIG_DEBUG_TCG 3139 /* Verify that we haven't seen this numbered exit before. */ 3140 tcg_debug_assert((tcg_ctx->goto_tb_issue_mask & (1 << idx)) == 0); 3141 tcg_ctx->goto_tb_issue_mask |= 1 << idx; 3142 #endif 3143 plugin_gen_disable_mem_helpers(); 3144 tcg_gen_op1i(INDEX_op_goto_tb, 0, idx); 3145 } 3146 3147 void tcg_gen_lookup_and_goto_ptr(void) 3148 { 3149 TCGv_ptr ptr; 3150 3151 if (tcg_ctx->gen_tb->cflags & CF_NO_GOTO_PTR) { 3152 tcg_gen_exit_tb(NULL, 0); 3153 return; 3154 } 3155 3156 plugin_gen_disable_mem_helpers(); 3157 ptr = tcg_temp_ebb_new_ptr(); 3158 gen_helper_lookup_tb_ptr(ptr, tcg_env); 3159 tcg_gen_op1i(INDEX_op_goto_ptr, TCG_TYPE_PTR, tcgv_ptr_arg(ptr)); 3160 tcg_temp_free_ptr(ptr); 3161 } 3162