1 /* 2 * Tiny Code Generator for QEMU 3 * 4 * Copyright (c) 2008 Fabrice Bellard 5 * 6 * Permission is hereby granted, free of charge, to any person obtaining a copy 7 * of this software and associated documentation files (the "Software"), to deal 8 * in the Software without restriction, including without limitation the rights 9 * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 10 * copies of the Software, and to permit persons to whom the Software is 11 * furnished to do so, subject to the following conditions: 12 * 13 * The above copyright notice and this permission notice shall be included in 14 * all copies or substantial portions of the Software. 15 * 16 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 17 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 18 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL 19 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 20 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 21 * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN 22 * THE SOFTWARE. 23 */ 24 25 #include "qemu/osdep.h" 26 #include "tcg/tcg.h" 27 #include "tcg/tcg-temp-internal.h" 28 #include "tcg/tcg-op-common.h" 29 #include "exec/translation-block.h" 30 #include "exec/plugin-gen.h" 31 #include "tcg-internal.h" 32 #include "tcg-has.h" 33 34 /* 35 * Encourage the compiler to tail-call to a function, rather than inlining. 36 * Minimizes code size across 99 bottles of beer on the wall. 37 */ 38 #define NI __attribute__((noinline)) 39 40 TCGOp * NI tcg_gen_op1(TCGOpcode opc, TCGType type, TCGArg a1) 41 { 42 TCGOp *op = tcg_emit_op(opc, 1); 43 TCGOP_TYPE(op) = type; 44 op->args[0] = a1; 45 return op; 46 } 47 48 TCGOp * NI tcg_gen_op2(TCGOpcode opc, TCGType type, TCGArg a1, TCGArg a2) 49 { 50 TCGOp *op = tcg_emit_op(opc, 2); 51 TCGOP_TYPE(op) = type; 52 op->args[0] = a1; 53 op->args[1] = a2; 54 return op; 55 } 56 57 TCGOp * NI tcg_gen_op3(TCGOpcode opc, TCGType type, TCGArg a1, 58 TCGArg a2, TCGArg a3) 59 { 60 TCGOp *op = tcg_emit_op(opc, 3); 61 TCGOP_TYPE(op) = type; 62 op->args[0] = a1; 63 op->args[1] = a2; 64 op->args[2] = a3; 65 return op; 66 } 67 68 TCGOp * NI tcg_gen_op4(TCGOpcode opc, TCGType type, TCGArg a1, TCGArg a2, 69 TCGArg a3, TCGArg a4) 70 { 71 TCGOp *op = tcg_emit_op(opc, 4); 72 TCGOP_TYPE(op) = type; 73 op->args[0] = a1; 74 op->args[1] = a2; 75 op->args[2] = a3; 76 op->args[3] = a4; 77 return op; 78 } 79 80 TCGOp * NI tcg_gen_op5(TCGOpcode opc, TCGType type, TCGArg a1, TCGArg a2, 81 TCGArg a3, TCGArg a4, TCGArg a5) 82 { 83 TCGOp *op = tcg_emit_op(opc, 5); 84 TCGOP_TYPE(op) = type; 85 op->args[0] = a1; 86 op->args[1] = a2; 87 op->args[2] = a3; 88 op->args[3] = a4; 89 op->args[4] = a5; 90 return op; 91 } 92 93 TCGOp * NI tcg_gen_op6(TCGOpcode opc, TCGType type, TCGArg a1, TCGArg a2, 94 TCGArg a3, TCGArg a4, TCGArg a5, TCGArg a6) 95 { 96 TCGOp *op = tcg_emit_op(opc, 6); 97 TCGOP_TYPE(op) = type; 98 op->args[0] = a1; 99 op->args[1] = a2; 100 op->args[2] = a3; 101 op->args[3] = a4; 102 op->args[4] = a5; 103 op->args[5] = a6; 104 return op; 105 } 106 107 /* 108 * With CONFIG_DEBUG_TCG, tcgv_*_tmp via tcgv_*_arg, is an out-of-line 109 * assertion check. Force tail calls to avoid too much code expansion. 110 */ 111 #ifdef CONFIG_DEBUG_TCG 112 # define DNI NI 113 #else 114 # define DNI 115 #endif 116 117 static void DNI tcg_gen_op1_i32(TCGOpcode opc, TCGType type, TCGv_i32 a1) 118 { 119 tcg_gen_op1(opc, type, tcgv_i32_arg(a1)); 120 } 121 122 static void DNI tcg_gen_op1_i64(TCGOpcode opc, TCGType type, TCGv_i64 a1) 123 { 124 tcg_gen_op1(opc, type, tcgv_i64_arg(a1)); 125 } 126 127 static TCGOp * DNI tcg_gen_op1i(TCGOpcode opc, TCGType type, TCGArg a1) 128 { 129 return tcg_gen_op1(opc, type, a1); 130 } 131 132 static void DNI tcg_gen_op2_i32(TCGOpcode opc, TCGv_i32 a1, TCGv_i32 a2) 133 { 134 tcg_gen_op2(opc, TCG_TYPE_I32, tcgv_i32_arg(a1), tcgv_i32_arg(a2)); 135 } 136 137 static void DNI tcg_gen_op2_i64(TCGOpcode opc, TCGv_i64 a1, TCGv_i64 a2) 138 { 139 tcg_gen_op2(opc, TCG_TYPE_I64, tcgv_i64_arg(a1), tcgv_i64_arg(a2)); 140 } 141 142 static void DNI tcg_gen_op3_i32(TCGOpcode opc, TCGv_i32 a1, 143 TCGv_i32 a2, TCGv_i32 a3) 144 { 145 tcg_gen_op3(opc, TCG_TYPE_I32, tcgv_i32_arg(a1), 146 tcgv_i32_arg(a2), tcgv_i32_arg(a3)); 147 } 148 149 static void DNI tcg_gen_op3_i64(TCGOpcode opc, TCGv_i64 a1, 150 TCGv_i64 a2, TCGv_i64 a3) 151 { 152 tcg_gen_op3(opc, TCG_TYPE_I64, tcgv_i64_arg(a1), 153 tcgv_i64_arg(a2), tcgv_i64_arg(a3)); 154 } 155 156 static void DNI tcg_gen_op3i_i32(TCGOpcode opc, TCGv_i32 a1, 157 TCGv_i32 a2, TCGArg a3) 158 { 159 tcg_gen_op3(opc, TCG_TYPE_I32, tcgv_i32_arg(a1), tcgv_i32_arg(a2), a3); 160 } 161 162 static void DNI tcg_gen_op3i_i64(TCGOpcode opc, TCGv_i64 a1, 163 TCGv_i64 a2, TCGArg a3) 164 { 165 tcg_gen_op3(opc, TCG_TYPE_I64, tcgv_i64_arg(a1), tcgv_i64_arg(a2), a3); 166 } 167 168 static void DNI tcg_gen_ldst_op_i32(TCGOpcode opc, TCGv_i32 val, 169 TCGv_ptr base, TCGArg offset) 170 { 171 tcg_gen_op3(opc, TCG_TYPE_I32, tcgv_i32_arg(val), 172 tcgv_ptr_arg(base), offset); 173 } 174 175 static void DNI tcg_gen_ldst_op_i64(TCGOpcode opc, TCGv_i64 val, 176 TCGv_ptr base, TCGArg offset) 177 { 178 tcg_gen_op3(opc, TCG_TYPE_I64, tcgv_i64_arg(val), 179 tcgv_ptr_arg(base), offset); 180 } 181 182 static void DNI tcg_gen_op4_i32(TCGOpcode opc, TCGv_i32 a1, TCGv_i32 a2, 183 TCGv_i32 a3, TCGv_i32 a4) 184 { 185 tcg_gen_op4(opc, TCG_TYPE_I32, tcgv_i32_arg(a1), tcgv_i32_arg(a2), 186 tcgv_i32_arg(a3), tcgv_i32_arg(a4)); 187 } 188 189 static void DNI tcg_gen_op4_i64(TCGOpcode opc, TCGv_i64 a1, TCGv_i64 a2, 190 TCGv_i64 a3, TCGv_i64 a4) 191 { 192 tcg_gen_op4(opc, TCG_TYPE_I64, tcgv_i64_arg(a1), tcgv_i64_arg(a2), 193 tcgv_i64_arg(a3), tcgv_i64_arg(a4)); 194 } 195 196 static void DNI tcg_gen_op4i_i32(TCGOpcode opc, TCGv_i32 a1, TCGv_i32 a2, 197 TCGv_i32 a3, TCGArg a4) 198 { 199 tcg_gen_op4(opc, TCG_TYPE_I32, tcgv_i32_arg(a1), tcgv_i32_arg(a2), 200 tcgv_i32_arg(a3), a4); 201 } 202 203 static void DNI tcg_gen_op4i_i64(TCGOpcode opc, TCGv_i64 a1, TCGv_i64 a2, 204 TCGv_i64 a3, TCGArg a4) 205 { 206 tcg_gen_op4(opc, TCG_TYPE_I64, tcgv_i64_arg(a1), tcgv_i64_arg(a2), 207 tcgv_i64_arg(a3), a4); 208 } 209 210 static TCGOp * DNI tcg_gen_op4ii_i32(TCGOpcode opc, TCGv_i32 a1, TCGv_i32 a2, 211 TCGArg a3, TCGArg a4) 212 { 213 return tcg_gen_op4(opc, TCG_TYPE_I32, 214 tcgv_i32_arg(a1), tcgv_i32_arg(a2), a3, a4); 215 } 216 217 static TCGOp * DNI tcg_gen_op4ii_i64(TCGOpcode opc, TCGv_i64 a1, TCGv_i64 a2, 218 TCGArg a3, TCGArg a4) 219 { 220 return tcg_gen_op4(opc, TCG_TYPE_I64, 221 tcgv_i64_arg(a1), tcgv_i64_arg(a2), a3, a4); 222 } 223 224 static void DNI tcg_gen_op5_i32(TCGOpcode opc, TCGv_i32 a1, TCGv_i32 a2, 225 TCGv_i32 a3, TCGv_i32 a4, TCGv_i32 a5) 226 { 227 tcg_gen_op5(opc, TCG_TYPE_I32, tcgv_i32_arg(a1), tcgv_i32_arg(a2), 228 tcgv_i32_arg(a3), tcgv_i32_arg(a4), tcgv_i32_arg(a5)); 229 } 230 231 static void DNI tcg_gen_op5_i64(TCGOpcode opc, TCGv_i64 a1, TCGv_i64 a2, 232 TCGv_i64 a3, TCGv_i64 a4, TCGv_i64 a5) 233 { 234 tcg_gen_op5(opc, TCG_TYPE_I64, tcgv_i64_arg(a1), tcgv_i64_arg(a2), 235 tcgv_i64_arg(a3), tcgv_i64_arg(a4), tcgv_i64_arg(a5)); 236 } 237 238 static void DNI tcg_gen_op5ii_i32(TCGOpcode opc, TCGv_i32 a1, TCGv_i32 a2, 239 TCGv_i32 a3, TCGArg a4, TCGArg a5) 240 { 241 tcg_gen_op5(opc, TCG_TYPE_I32, tcgv_i32_arg(a1), tcgv_i32_arg(a2), 242 tcgv_i32_arg(a3), a4, a5); 243 } 244 245 static void DNI tcg_gen_op5ii_i64(TCGOpcode opc, TCGv_i64 a1, TCGv_i64 a2, 246 TCGv_i64 a3, TCGArg a4, TCGArg a5) 247 { 248 tcg_gen_op5(opc, TCG_TYPE_I64, tcgv_i64_arg(a1), tcgv_i64_arg(a2), 249 tcgv_i64_arg(a3), a4, a5); 250 } 251 252 static void DNI tcg_gen_op6_i32(TCGOpcode opc, TCGv_i32 a1, TCGv_i32 a2, 253 TCGv_i32 a3, TCGv_i32 a4, 254 TCGv_i32 a5, TCGv_i32 a6) 255 { 256 tcg_gen_op6(opc, TCG_TYPE_I32, tcgv_i32_arg(a1), tcgv_i32_arg(a2), 257 tcgv_i32_arg(a3), tcgv_i32_arg(a4), tcgv_i32_arg(a5), 258 tcgv_i32_arg(a6)); 259 } 260 261 static void DNI tcg_gen_op6_i64(TCGOpcode opc, TCGv_i64 a1, TCGv_i64 a2, 262 TCGv_i64 a3, TCGv_i64 a4, 263 TCGv_i64 a5, TCGv_i64 a6) 264 { 265 tcg_gen_op6(opc, TCG_TYPE_I64, tcgv_i64_arg(a1), tcgv_i64_arg(a2), 266 tcgv_i64_arg(a3), tcgv_i64_arg(a4), tcgv_i64_arg(a5), 267 tcgv_i64_arg(a6)); 268 } 269 270 static void DNI tcg_gen_op6i_i32(TCGOpcode opc, TCGv_i32 a1, TCGv_i32 a2, 271 TCGv_i32 a3, TCGv_i32 a4, 272 TCGv_i32 a5, TCGArg a6) 273 { 274 tcg_gen_op6(opc, TCG_TYPE_I32, tcgv_i32_arg(a1), tcgv_i32_arg(a2), 275 tcgv_i32_arg(a3), tcgv_i32_arg(a4), tcgv_i32_arg(a5), a6); 276 } 277 278 static void DNI tcg_gen_op6i_i64(TCGOpcode opc, TCGv_i64 a1, TCGv_i64 a2, 279 TCGv_i64 a3, TCGv_i64 a4, 280 TCGv_i64 a5, TCGArg a6) 281 { 282 tcg_gen_op6(opc, TCG_TYPE_I64, tcgv_i64_arg(a1), tcgv_i64_arg(a2), 283 tcgv_i64_arg(a3), tcgv_i64_arg(a4), tcgv_i64_arg(a5), a6); 284 } 285 286 static TCGOp * DNI tcg_gen_op6ii_i32(TCGOpcode opc, TCGv_i32 a1, TCGv_i32 a2, 287 TCGv_i32 a3, TCGv_i32 a4, 288 TCGArg a5, TCGArg a6) 289 { 290 return tcg_gen_op6(opc, TCG_TYPE_I32, tcgv_i32_arg(a1), tcgv_i32_arg(a2), 291 tcgv_i32_arg(a3), tcgv_i32_arg(a4), a5, a6); 292 } 293 294 /* Generic ops. */ 295 296 void gen_set_label(TCGLabel *l) 297 { 298 l->present = 1; 299 tcg_gen_op1(INDEX_op_set_label, 0, label_arg(l)); 300 } 301 302 static void add_as_label_use(TCGLabel *l, TCGOp *op) 303 { 304 TCGLabelUse *u = tcg_malloc(sizeof(TCGLabelUse)); 305 306 u->op = op; 307 QSIMPLEQ_INSERT_TAIL(&l->branches, u, next); 308 } 309 310 void tcg_gen_br(TCGLabel *l) 311 { 312 add_as_label_use(l, tcg_gen_op1(INDEX_op_br, 0, label_arg(l))); 313 } 314 315 void tcg_gen_mb(TCGBar mb_type) 316 { 317 #ifdef CONFIG_USER_ONLY 318 bool parallel = tcg_ctx->gen_tb->cflags & CF_PARALLEL; 319 #else 320 /* 321 * It is tempting to elide the barrier in a uniprocessor context. 322 * However, even with a single cpu we have i/o threads running in 323 * parallel, and lack of memory order can result in e.g. virtio 324 * queue entries being read incorrectly. 325 */ 326 bool parallel = true; 327 #endif 328 329 if (parallel) { 330 tcg_gen_op1(INDEX_op_mb, 0, mb_type); 331 } 332 } 333 334 void tcg_gen_plugin_cb(unsigned from) 335 { 336 tcg_gen_op1(INDEX_op_plugin_cb, 0, from); 337 } 338 339 void tcg_gen_plugin_mem_cb(TCGv_i64 addr, unsigned meminfo) 340 { 341 tcg_gen_op2(INDEX_op_plugin_mem_cb, 0, tcgv_i64_arg(addr), meminfo); 342 } 343 344 /* 32 bit ops */ 345 346 void tcg_gen_discard_i32(TCGv_i32 arg) 347 { 348 tcg_gen_op1_i32(INDEX_op_discard, TCG_TYPE_I32, arg); 349 } 350 351 void tcg_gen_mov_i32(TCGv_i32 ret, TCGv_i32 arg) 352 { 353 if (ret != arg) { 354 tcg_gen_op2_i32(INDEX_op_mov, ret, arg); 355 } 356 } 357 358 void tcg_gen_movi_i32(TCGv_i32 ret, int32_t arg) 359 { 360 tcg_gen_mov_i32(ret, tcg_constant_i32(arg)); 361 } 362 363 void tcg_gen_add_i32(TCGv_i32 ret, TCGv_i32 arg1, TCGv_i32 arg2) 364 { 365 tcg_gen_op3_i32(INDEX_op_add, ret, arg1, arg2); 366 } 367 368 void tcg_gen_addi_i32(TCGv_i32 ret, TCGv_i32 arg1, int32_t arg2) 369 { 370 /* some cases can be optimized here */ 371 if (arg2 == 0) { 372 tcg_gen_mov_i32(ret, arg1); 373 } else { 374 tcg_gen_add_i32(ret, arg1, tcg_constant_i32(arg2)); 375 } 376 } 377 378 void tcg_gen_sub_i32(TCGv_i32 ret, TCGv_i32 arg1, TCGv_i32 arg2) 379 { 380 tcg_gen_op3_i32(INDEX_op_sub, ret, arg1, arg2); 381 } 382 383 void tcg_gen_subfi_i32(TCGv_i32 ret, int32_t arg1, TCGv_i32 arg2) 384 { 385 if (arg1 == 0) { 386 tcg_gen_neg_i32(ret, arg2); 387 } else { 388 tcg_gen_sub_i32(ret, tcg_constant_i32(arg1), arg2); 389 } 390 } 391 392 void tcg_gen_subi_i32(TCGv_i32 ret, TCGv_i32 arg1, int32_t arg2) 393 { 394 tcg_gen_addi_i32(ret, arg1, -arg2); 395 } 396 397 void tcg_gen_neg_i32(TCGv_i32 ret, TCGv_i32 arg) 398 { 399 tcg_gen_op2_i32(INDEX_op_neg, ret, arg); 400 } 401 402 void tcg_gen_and_i32(TCGv_i32 ret, TCGv_i32 arg1, TCGv_i32 arg2) 403 { 404 tcg_gen_op3_i32(INDEX_op_and, ret, arg1, arg2); 405 } 406 407 void tcg_gen_andi_i32(TCGv_i32 ret, TCGv_i32 arg1, int32_t arg2) 408 { 409 /* Some cases can be optimized here. */ 410 switch (arg2) { 411 case 0: 412 tcg_gen_movi_i32(ret, 0); 413 return; 414 case -1: 415 tcg_gen_mov_i32(ret, arg1); 416 return; 417 default: 418 /* 419 * Canonicalize on extract, if valid. This aids x86 with its 420 * 2 operand MOVZBL and 2 operand AND, selecting the TCGOpcode 421 * which does not require matching operands. Other backends can 422 * trivially expand the extract to AND during code generation. 423 */ 424 if (!(arg2 & (arg2 + 1))) { 425 unsigned len = ctz32(~arg2); 426 if (TCG_TARGET_extract_valid(TCG_TYPE_I32, 0, len)) { 427 tcg_gen_extract_i32(ret, arg1, 0, len); 428 return; 429 } 430 } 431 break; 432 } 433 434 tcg_gen_and_i32(ret, arg1, tcg_constant_i32(arg2)); 435 } 436 437 void tcg_gen_or_i32(TCGv_i32 ret, TCGv_i32 arg1, TCGv_i32 arg2) 438 { 439 tcg_gen_op3_i32(INDEX_op_or, ret, arg1, arg2); 440 } 441 442 void tcg_gen_ori_i32(TCGv_i32 ret, TCGv_i32 arg1, int32_t arg2) 443 { 444 /* Some cases can be optimized here. */ 445 if (arg2 == -1) { 446 tcg_gen_movi_i32(ret, -1); 447 } else if (arg2 == 0) { 448 tcg_gen_mov_i32(ret, arg1); 449 } else { 450 tcg_gen_or_i32(ret, arg1, tcg_constant_i32(arg2)); 451 } 452 } 453 454 void tcg_gen_xor_i32(TCGv_i32 ret, TCGv_i32 arg1, TCGv_i32 arg2) 455 { 456 tcg_gen_op3_i32(INDEX_op_xor, ret, arg1, arg2); 457 } 458 459 void tcg_gen_xori_i32(TCGv_i32 ret, TCGv_i32 arg1, int32_t arg2) 460 { 461 /* Some cases can be optimized here. */ 462 if (arg2 == 0) { 463 tcg_gen_mov_i32(ret, arg1); 464 } else if (arg2 == -1 && 465 tcg_op_supported(INDEX_op_not, TCG_TYPE_I32, 0)) { 466 /* Don't recurse with tcg_gen_not_i32. */ 467 tcg_gen_op2_i32(INDEX_op_not, ret, arg1); 468 } else { 469 tcg_gen_xor_i32(ret, arg1, tcg_constant_i32(arg2)); 470 } 471 } 472 473 void tcg_gen_not_i32(TCGv_i32 ret, TCGv_i32 arg) 474 { 475 if (tcg_op_supported(INDEX_op_not, TCG_TYPE_I32, 0)) { 476 tcg_gen_op2_i32(INDEX_op_not, ret, arg); 477 } else { 478 tcg_gen_xori_i32(ret, arg, -1); 479 } 480 } 481 482 void tcg_gen_shl_i32(TCGv_i32 ret, TCGv_i32 arg1, TCGv_i32 arg2) 483 { 484 tcg_gen_op3_i32(INDEX_op_shl, ret, arg1, arg2); 485 } 486 487 void tcg_gen_shli_i32(TCGv_i32 ret, TCGv_i32 arg1, int32_t arg2) 488 { 489 tcg_debug_assert(arg2 >= 0 && arg2 < 32); 490 if (arg2 == 0) { 491 tcg_gen_mov_i32(ret, arg1); 492 } else { 493 tcg_gen_shl_i32(ret, arg1, tcg_constant_i32(arg2)); 494 } 495 } 496 497 void tcg_gen_shr_i32(TCGv_i32 ret, TCGv_i32 arg1, TCGv_i32 arg2) 498 { 499 tcg_gen_op3_i32(INDEX_op_shr, ret, arg1, arg2); 500 } 501 502 void tcg_gen_shri_i32(TCGv_i32 ret, TCGv_i32 arg1, int32_t arg2) 503 { 504 tcg_debug_assert(arg2 >= 0 && arg2 < 32); 505 if (arg2 == 0) { 506 tcg_gen_mov_i32(ret, arg1); 507 } else { 508 tcg_gen_shr_i32(ret, arg1, tcg_constant_i32(arg2)); 509 } 510 } 511 512 void tcg_gen_sar_i32(TCGv_i32 ret, TCGv_i32 arg1, TCGv_i32 arg2) 513 { 514 tcg_gen_op3_i32(INDEX_op_sar, ret, arg1, arg2); 515 } 516 517 void tcg_gen_sari_i32(TCGv_i32 ret, TCGv_i32 arg1, int32_t arg2) 518 { 519 tcg_debug_assert(arg2 >= 0 && arg2 < 32); 520 if (arg2 == 0) { 521 tcg_gen_mov_i32(ret, arg1); 522 } else { 523 tcg_gen_sar_i32(ret, arg1, tcg_constant_i32(arg2)); 524 } 525 } 526 527 void tcg_gen_brcond_i32(TCGCond cond, TCGv_i32 arg1, TCGv_i32 arg2, TCGLabel *l) 528 { 529 if (cond == TCG_COND_ALWAYS) { 530 tcg_gen_br(l); 531 } else if (cond != TCG_COND_NEVER) { 532 TCGOp *op = tcg_gen_op4ii_i32(INDEX_op_brcond_i32, 533 arg1, arg2, cond, label_arg(l)); 534 add_as_label_use(l, op); 535 } 536 } 537 538 void tcg_gen_brcondi_i32(TCGCond cond, TCGv_i32 arg1, int32_t arg2, TCGLabel *l) 539 { 540 if (cond == TCG_COND_ALWAYS) { 541 tcg_gen_br(l); 542 } else if (cond != TCG_COND_NEVER) { 543 tcg_gen_brcond_i32(cond, arg1, tcg_constant_i32(arg2), l); 544 } 545 } 546 547 void tcg_gen_setcond_i32(TCGCond cond, TCGv_i32 ret, 548 TCGv_i32 arg1, TCGv_i32 arg2) 549 { 550 if (cond == TCG_COND_ALWAYS) { 551 tcg_gen_movi_i32(ret, 1); 552 } else if (cond == TCG_COND_NEVER) { 553 tcg_gen_movi_i32(ret, 0); 554 } else { 555 tcg_gen_op4i_i32(INDEX_op_setcond_i32, ret, arg1, arg2, cond); 556 } 557 } 558 559 void tcg_gen_setcondi_i32(TCGCond cond, TCGv_i32 ret, 560 TCGv_i32 arg1, int32_t arg2) 561 { 562 tcg_gen_setcond_i32(cond, ret, arg1, tcg_constant_i32(arg2)); 563 } 564 565 void tcg_gen_negsetcond_i32(TCGCond cond, TCGv_i32 ret, 566 TCGv_i32 arg1, TCGv_i32 arg2) 567 { 568 if (cond == TCG_COND_ALWAYS) { 569 tcg_gen_movi_i32(ret, -1); 570 } else if (cond == TCG_COND_NEVER) { 571 tcg_gen_movi_i32(ret, 0); 572 } else if (TCG_TARGET_HAS_negsetcond_i32) { 573 tcg_gen_op4i_i32(INDEX_op_negsetcond_i32, ret, arg1, arg2, cond); 574 } else { 575 tcg_gen_setcond_i32(cond, ret, arg1, arg2); 576 tcg_gen_neg_i32(ret, ret); 577 } 578 } 579 580 void tcg_gen_negsetcondi_i32(TCGCond cond, TCGv_i32 ret, 581 TCGv_i32 arg1, int32_t arg2) 582 { 583 tcg_gen_negsetcond_i32(cond, ret, arg1, tcg_constant_i32(arg2)); 584 } 585 586 void tcg_gen_mul_i32(TCGv_i32 ret, TCGv_i32 arg1, TCGv_i32 arg2) 587 { 588 tcg_gen_op3_i32(INDEX_op_mul, ret, arg1, arg2); 589 } 590 591 void tcg_gen_muli_i32(TCGv_i32 ret, TCGv_i32 arg1, int32_t arg2) 592 { 593 if (arg2 == 0) { 594 tcg_gen_movi_i32(ret, 0); 595 } else if (is_power_of_2(arg2)) { 596 tcg_gen_shli_i32(ret, arg1, ctz32(arg2)); 597 } else { 598 tcg_gen_mul_i32(ret, arg1, tcg_constant_i32(arg2)); 599 } 600 } 601 602 void tcg_gen_div_i32(TCGv_i32 ret, TCGv_i32 arg1, TCGv_i32 arg2) 603 { 604 if (tcg_op_supported(INDEX_op_divs, TCG_TYPE_I32, 0)) { 605 tcg_gen_op3_i32(INDEX_op_divs, ret, arg1, arg2); 606 } else if (tcg_op_supported(INDEX_op_divs2, TCG_TYPE_I32, 0)) { 607 TCGv_i32 t0 = tcg_temp_ebb_new_i32(); 608 tcg_gen_sari_i32(t0, arg1, 31); 609 tcg_gen_op5_i32(INDEX_op_divs2, ret, t0, arg1, t0, arg2); 610 tcg_temp_free_i32(t0); 611 } else { 612 gen_helper_div_i32(ret, arg1, arg2); 613 } 614 } 615 616 void tcg_gen_rem_i32(TCGv_i32 ret, TCGv_i32 arg1, TCGv_i32 arg2) 617 { 618 if (tcg_op_supported(INDEX_op_rems, TCG_TYPE_I32, 0)) { 619 tcg_gen_op3_i32(INDEX_op_rems, ret, arg1, arg2); 620 } else if (tcg_op_supported(INDEX_op_divs, TCG_TYPE_I32, 0)) { 621 TCGv_i32 t0 = tcg_temp_ebb_new_i32(); 622 tcg_gen_op3_i32(INDEX_op_divs, t0, arg1, arg2); 623 tcg_gen_mul_i32(t0, t0, arg2); 624 tcg_gen_sub_i32(ret, arg1, t0); 625 tcg_temp_free_i32(t0); 626 } else if (tcg_op_supported(INDEX_op_divs2, TCG_TYPE_I32, 0)) { 627 TCGv_i32 t0 = tcg_temp_ebb_new_i32(); 628 tcg_gen_sari_i32(t0, arg1, 31); 629 tcg_gen_op5_i32(INDEX_op_divs2, t0, ret, arg1, t0, arg2); 630 tcg_temp_free_i32(t0); 631 } else { 632 gen_helper_rem_i32(ret, arg1, arg2); 633 } 634 } 635 636 void tcg_gen_divu_i32(TCGv_i32 ret, TCGv_i32 arg1, TCGv_i32 arg2) 637 { 638 if (tcg_op_supported(INDEX_op_divu, TCG_TYPE_I32, 0)) { 639 tcg_gen_op3_i32(INDEX_op_divu, ret, arg1, arg2); 640 } else if (tcg_op_supported(INDEX_op_divu2, TCG_TYPE_I32, 0)) { 641 TCGv_i32 t0 = tcg_temp_ebb_new_i32(); 642 TCGv_i32 zero = tcg_constant_i32(0); 643 tcg_gen_op5_i32(INDEX_op_divu2, ret, t0, arg1, zero, arg2); 644 tcg_temp_free_i32(t0); 645 } else { 646 gen_helper_divu_i32(ret, arg1, arg2); 647 } 648 } 649 650 void tcg_gen_remu_i32(TCGv_i32 ret, TCGv_i32 arg1, TCGv_i32 arg2) 651 { 652 if (tcg_op_supported(INDEX_op_remu, TCG_TYPE_I32, 0)) { 653 tcg_gen_op3_i32(INDEX_op_remu, ret, arg1, arg2); 654 } else if (tcg_op_supported(INDEX_op_divu, TCG_TYPE_I32, 0)) { 655 TCGv_i32 t0 = tcg_temp_ebb_new_i32(); 656 tcg_gen_op3_i32(INDEX_op_divu, t0, arg1, arg2); 657 tcg_gen_mul_i32(t0, t0, arg2); 658 tcg_gen_sub_i32(ret, arg1, t0); 659 tcg_temp_free_i32(t0); 660 } else if (tcg_op_supported(INDEX_op_divu2, TCG_TYPE_I32, 0)) { 661 TCGv_i32 t0 = tcg_temp_ebb_new_i32(); 662 TCGv_i32 zero = tcg_constant_i32(0); 663 tcg_gen_op5_i32(INDEX_op_divu2, t0, ret, arg1, zero, arg2); 664 tcg_temp_free_i32(t0); 665 } else { 666 gen_helper_remu_i32(ret, arg1, arg2); 667 } 668 } 669 670 void tcg_gen_andc_i32(TCGv_i32 ret, TCGv_i32 arg1, TCGv_i32 arg2) 671 { 672 if (tcg_op_supported(INDEX_op_andc, TCG_TYPE_I32, 0)) { 673 tcg_gen_op3_i32(INDEX_op_andc, ret, arg1, arg2); 674 } else { 675 TCGv_i32 t0 = tcg_temp_ebb_new_i32(); 676 tcg_gen_not_i32(t0, arg2); 677 tcg_gen_and_i32(ret, arg1, t0); 678 tcg_temp_free_i32(t0); 679 } 680 } 681 682 void tcg_gen_eqv_i32(TCGv_i32 ret, TCGv_i32 arg1, TCGv_i32 arg2) 683 { 684 if (tcg_op_supported(INDEX_op_eqv, TCG_TYPE_I32, 0)) { 685 tcg_gen_op3_i32(INDEX_op_eqv, ret, arg1, arg2); 686 } else { 687 tcg_gen_xor_i32(ret, arg1, arg2); 688 tcg_gen_not_i32(ret, ret); 689 } 690 } 691 692 void tcg_gen_nand_i32(TCGv_i32 ret, TCGv_i32 arg1, TCGv_i32 arg2) 693 { 694 if (tcg_op_supported(INDEX_op_nand, TCG_TYPE_I32, 0)) { 695 tcg_gen_op3_i32(INDEX_op_nand, ret, arg1, arg2); 696 } else { 697 tcg_gen_and_i32(ret, arg1, arg2); 698 tcg_gen_not_i32(ret, ret); 699 } 700 } 701 702 void tcg_gen_nor_i32(TCGv_i32 ret, TCGv_i32 arg1, TCGv_i32 arg2) 703 { 704 if (tcg_op_supported(INDEX_op_nor, TCG_TYPE_I32, 0)) { 705 tcg_gen_op3_i32(INDEX_op_nor, ret, arg1, arg2); 706 } else { 707 tcg_gen_or_i32(ret, arg1, arg2); 708 tcg_gen_not_i32(ret, ret); 709 } 710 } 711 712 void tcg_gen_orc_i32(TCGv_i32 ret, TCGv_i32 arg1, TCGv_i32 arg2) 713 { 714 if (tcg_op_supported(INDEX_op_orc, TCG_TYPE_I32, 0)) { 715 tcg_gen_op3_i32(INDEX_op_orc, ret, arg1, arg2); 716 } else { 717 TCGv_i32 t0 = tcg_temp_ebb_new_i32(); 718 tcg_gen_not_i32(t0, arg2); 719 tcg_gen_or_i32(ret, arg1, t0); 720 tcg_temp_free_i32(t0); 721 } 722 } 723 724 void tcg_gen_clz_i32(TCGv_i32 ret, TCGv_i32 arg1, TCGv_i32 arg2) 725 { 726 if (tcg_op_supported(INDEX_op_clz, TCG_TYPE_I32, 0)) { 727 tcg_gen_op3_i32(INDEX_op_clz, ret, arg1, arg2); 728 } else if (tcg_op_supported(INDEX_op_clz, TCG_TYPE_I64, 0)) { 729 TCGv_i64 t1 = tcg_temp_ebb_new_i64(); 730 TCGv_i64 t2 = tcg_temp_ebb_new_i64(); 731 tcg_gen_extu_i32_i64(t1, arg1); 732 tcg_gen_extu_i32_i64(t2, arg2); 733 tcg_gen_addi_i64(t2, t2, 32); 734 tcg_gen_clz_i64(t1, t1, t2); 735 tcg_gen_extrl_i64_i32(ret, t1); 736 tcg_temp_free_i64(t1); 737 tcg_temp_free_i64(t2); 738 tcg_gen_subi_i32(ret, ret, 32); 739 } else { 740 gen_helper_clz_i32(ret, arg1, arg2); 741 } 742 } 743 744 void tcg_gen_clzi_i32(TCGv_i32 ret, TCGv_i32 arg1, uint32_t arg2) 745 { 746 tcg_gen_clz_i32(ret, arg1, tcg_constant_i32(arg2)); 747 } 748 749 void tcg_gen_ctz_i32(TCGv_i32 ret, TCGv_i32 arg1, TCGv_i32 arg2) 750 { 751 TCGv_i32 z, t; 752 753 if (tcg_op_supported(INDEX_op_ctz, TCG_TYPE_I32, 0)) { 754 tcg_gen_op3_i32(INDEX_op_ctz, ret, arg1, arg2); 755 return; 756 } 757 if (tcg_op_supported(INDEX_op_ctz, TCG_TYPE_I64, 0)) { 758 TCGv_i64 t1 = tcg_temp_ebb_new_i64(); 759 TCGv_i64 t2 = tcg_temp_ebb_new_i64(); 760 tcg_gen_extu_i32_i64(t1, arg1); 761 tcg_gen_extu_i32_i64(t2, arg2); 762 tcg_gen_ctz_i64(t1, t1, t2); 763 tcg_gen_extrl_i64_i32(ret, t1); 764 tcg_temp_free_i64(t1); 765 tcg_temp_free_i64(t2); 766 return; 767 } 768 if (tcg_op_supported(INDEX_op_ctpop, TCG_TYPE_REG, 0)) { 769 t = tcg_temp_ebb_new_i32(); 770 tcg_gen_subi_i32(t, arg1, 1); 771 tcg_gen_andc_i32(t, t, arg1); 772 tcg_gen_ctpop_i32(t, t); 773 } else if (tcg_op_supported(INDEX_op_clz, TCG_TYPE_REG, 0)) { 774 t = tcg_temp_ebb_new_i32(); 775 tcg_gen_neg_i32(t, arg1); 776 tcg_gen_and_i32(t, t, arg1); 777 tcg_gen_clzi_i32(t, t, 32); 778 tcg_gen_xori_i32(t, t, 31); 779 } else { 780 gen_helper_ctz_i32(ret, arg1, arg2); 781 return; 782 } 783 784 z = tcg_constant_i32(0); 785 tcg_gen_movcond_i32(TCG_COND_EQ, ret, arg1, z, arg2, t); 786 tcg_temp_free_i32(t); 787 } 788 789 void tcg_gen_ctzi_i32(TCGv_i32 ret, TCGv_i32 arg1, uint32_t arg2) 790 { 791 if (arg2 == 32 792 && !tcg_op_supported(INDEX_op_ctz, TCG_TYPE_I32, 0) 793 && tcg_op_supported(INDEX_op_ctpop, TCG_TYPE_REG, 0)) { 794 /* This equivalence has the advantage of not requiring a fixup. */ 795 TCGv_i32 t = tcg_temp_ebb_new_i32(); 796 tcg_gen_subi_i32(t, arg1, 1); 797 tcg_gen_andc_i32(t, t, arg1); 798 tcg_gen_ctpop_i32(ret, t); 799 tcg_temp_free_i32(t); 800 } else { 801 tcg_gen_ctz_i32(ret, arg1, tcg_constant_i32(arg2)); 802 } 803 } 804 805 void tcg_gen_clrsb_i32(TCGv_i32 ret, TCGv_i32 arg) 806 { 807 if (tcg_op_supported(INDEX_op_clz, TCG_TYPE_REG, 0)) { 808 TCGv_i32 t = tcg_temp_ebb_new_i32(); 809 tcg_gen_sari_i32(t, arg, 31); 810 tcg_gen_xor_i32(t, t, arg); 811 tcg_gen_clzi_i32(t, t, 32); 812 tcg_gen_subi_i32(ret, t, 1); 813 tcg_temp_free_i32(t); 814 } else { 815 gen_helper_clrsb_i32(ret, arg); 816 } 817 } 818 819 void tcg_gen_ctpop_i32(TCGv_i32 ret, TCGv_i32 arg1) 820 { 821 if (tcg_op_supported(INDEX_op_ctpop, TCG_TYPE_I32, 0)) { 822 tcg_gen_op2_i32(INDEX_op_ctpop, ret, arg1); 823 } else if (tcg_op_supported(INDEX_op_ctpop, TCG_TYPE_I64, 0)) { 824 TCGv_i64 t = tcg_temp_ebb_new_i64(); 825 tcg_gen_extu_i32_i64(t, arg1); 826 tcg_gen_ctpop_i64(t, t); 827 tcg_gen_extrl_i64_i32(ret, t); 828 tcg_temp_free_i64(t); 829 } else { 830 gen_helper_ctpop_i32(ret, arg1); 831 } 832 } 833 834 void tcg_gen_rotl_i32(TCGv_i32 ret, TCGv_i32 arg1, TCGv_i32 arg2) 835 { 836 if (tcg_op_supported(INDEX_op_rotl, TCG_TYPE_I32, 0)) { 837 tcg_gen_op3_i32(INDEX_op_rotl, ret, arg1, arg2); 838 } else if (tcg_op_supported(INDEX_op_rotr, TCG_TYPE_I32, 0)) { 839 TCGv_i32 t0 = tcg_temp_ebb_new_i32(); 840 tcg_gen_neg_i32(t0, arg2); 841 tcg_gen_op3_i32(INDEX_op_rotr, ret, arg1, t0); 842 tcg_temp_free_i32(t0); 843 } else { 844 TCGv_i32 t0 = tcg_temp_ebb_new_i32(); 845 TCGv_i32 t1 = tcg_temp_ebb_new_i32(); 846 tcg_gen_shl_i32(t0, arg1, arg2); 847 tcg_gen_neg_i32(t1, arg2); 848 tcg_gen_shr_i32(t1, arg1, t1); 849 tcg_gen_or_i32(ret, t0, t1); 850 tcg_temp_free_i32(t0); 851 tcg_temp_free_i32(t1); 852 } 853 } 854 855 void tcg_gen_rotli_i32(TCGv_i32 ret, TCGv_i32 arg1, int32_t arg2) 856 { 857 tcg_debug_assert(arg2 >= 0 && arg2 < 32); 858 /* some cases can be optimized here */ 859 if (arg2 == 0) { 860 tcg_gen_mov_i32(ret, arg1); 861 } else if (tcg_op_supported(INDEX_op_rotl, TCG_TYPE_I32, 0)) { 862 TCGv_i32 t0 = tcg_constant_i32(arg2); 863 tcg_gen_op3_i32(INDEX_op_rotl, ret, arg1, t0); 864 } else if (tcg_op_supported(INDEX_op_rotr, TCG_TYPE_I32, 0)) { 865 TCGv_i32 t0 = tcg_constant_i32(32 - arg2); 866 tcg_gen_op3_i32(INDEX_op_rotr, ret, arg1, t0); 867 } else { 868 TCGv_i32 t0 = tcg_temp_ebb_new_i32(); 869 TCGv_i32 t1 = tcg_temp_ebb_new_i32(); 870 tcg_gen_shli_i32(t0, arg1, arg2); 871 tcg_gen_shri_i32(t1, arg1, 32 - arg2); 872 tcg_gen_or_i32(ret, t0, t1); 873 tcg_temp_free_i32(t0); 874 tcg_temp_free_i32(t1); 875 } 876 } 877 878 void tcg_gen_rotr_i32(TCGv_i32 ret, TCGv_i32 arg1, TCGv_i32 arg2) 879 { 880 if (tcg_op_supported(INDEX_op_rotr, TCG_TYPE_I32, 0)) { 881 tcg_gen_op3_i32(INDEX_op_rotr, ret, arg1, arg2); 882 } else if (tcg_op_supported(INDEX_op_rotl, TCG_TYPE_I32, 0)) { 883 TCGv_i32 t0 = tcg_temp_ebb_new_i32(); 884 tcg_gen_neg_i32(t0, arg2); 885 tcg_gen_op3_i32(INDEX_op_rotl, ret, arg1, t0); 886 tcg_temp_free_i32(t0); 887 } else { 888 TCGv_i32 t0 = tcg_temp_ebb_new_i32(); 889 TCGv_i32 t1 = tcg_temp_ebb_new_i32(); 890 tcg_gen_shr_i32(t0, arg1, arg2); 891 tcg_gen_neg_i32(t1, arg2); 892 tcg_gen_shl_i32(t1, arg1, t1); 893 tcg_gen_or_i32(ret, t0, t1); 894 tcg_temp_free_i32(t0); 895 tcg_temp_free_i32(t1); 896 } 897 } 898 899 void tcg_gen_rotri_i32(TCGv_i32 ret, TCGv_i32 arg1, int32_t arg2) 900 { 901 tcg_debug_assert(arg2 >= 0 && arg2 < 32); 902 tcg_gen_rotli_i32(ret, arg1, -arg2 & 31); 903 } 904 905 void tcg_gen_deposit_i32(TCGv_i32 ret, TCGv_i32 arg1, TCGv_i32 arg2, 906 unsigned int ofs, unsigned int len) 907 { 908 uint32_t mask; 909 TCGv_i32 t1; 910 911 tcg_debug_assert(ofs < 32); 912 tcg_debug_assert(len > 0); 913 tcg_debug_assert(len <= 32); 914 tcg_debug_assert(ofs + len <= 32); 915 916 if (len == 32) { 917 tcg_gen_mov_i32(ret, arg2); 918 return; 919 } 920 if (TCG_TARGET_deposit_valid(TCG_TYPE_I32, ofs, len)) { 921 tcg_gen_op5ii_i32(INDEX_op_deposit_i32, ret, arg1, arg2, ofs, len); 922 return; 923 } 924 925 t1 = tcg_temp_ebb_new_i32(); 926 927 if (TCG_TARGET_HAS_extract2_i32) { 928 if (ofs + len == 32) { 929 tcg_gen_shli_i32(t1, arg1, len); 930 tcg_gen_extract2_i32(ret, t1, arg2, len); 931 goto done; 932 } 933 if (ofs == 0) { 934 tcg_gen_extract2_i32(ret, arg1, arg2, len); 935 tcg_gen_rotli_i32(ret, ret, len); 936 goto done; 937 } 938 } 939 940 mask = (1u << len) - 1; 941 if (ofs + len < 32) { 942 tcg_gen_andi_i32(t1, arg2, mask); 943 tcg_gen_shli_i32(t1, t1, ofs); 944 } else { 945 tcg_gen_shli_i32(t1, arg2, ofs); 946 } 947 tcg_gen_andi_i32(ret, arg1, ~(mask << ofs)); 948 tcg_gen_or_i32(ret, ret, t1); 949 done: 950 tcg_temp_free_i32(t1); 951 } 952 953 void tcg_gen_deposit_z_i32(TCGv_i32 ret, TCGv_i32 arg, 954 unsigned int ofs, unsigned int len) 955 { 956 tcg_debug_assert(ofs < 32); 957 tcg_debug_assert(len > 0); 958 tcg_debug_assert(len <= 32); 959 tcg_debug_assert(ofs + len <= 32); 960 961 if (ofs + len == 32) { 962 tcg_gen_shli_i32(ret, arg, ofs); 963 } else if (ofs == 0) { 964 tcg_gen_andi_i32(ret, arg, (1u << len) - 1); 965 } else if (TCG_TARGET_deposit_valid(TCG_TYPE_I32, ofs, len)) { 966 TCGv_i32 zero = tcg_constant_i32(0); 967 tcg_gen_op5ii_i32(INDEX_op_deposit_i32, ret, zero, arg, ofs, len); 968 } else { 969 /* 970 * To help two-operand hosts we prefer to zero-extend first, 971 * which allows ARG to stay live. 972 */ 973 if (TCG_TARGET_extract_valid(TCG_TYPE_I32, 0, len)) { 974 tcg_gen_extract_i32(ret, arg, 0, len); 975 tcg_gen_shli_i32(ret, ret, ofs); 976 return; 977 } 978 /* Otherwise prefer zero-extension over AND for code size. */ 979 if (TCG_TARGET_extract_valid(TCG_TYPE_I32, 0, ofs + len)) { 980 tcg_gen_shli_i32(ret, arg, ofs); 981 tcg_gen_extract_i32(ret, ret, 0, ofs + len); 982 return; 983 } 984 tcg_gen_andi_i32(ret, arg, (1u << len) - 1); 985 tcg_gen_shli_i32(ret, ret, ofs); 986 } 987 } 988 989 void tcg_gen_extract_i32(TCGv_i32 ret, TCGv_i32 arg, 990 unsigned int ofs, unsigned int len) 991 { 992 tcg_debug_assert(ofs < 32); 993 tcg_debug_assert(len > 0); 994 tcg_debug_assert(len <= 32); 995 tcg_debug_assert(ofs + len <= 32); 996 997 /* Canonicalize certain special cases, even if extract is supported. */ 998 if (ofs + len == 32) { 999 tcg_gen_shri_i32(ret, arg, 32 - len); 1000 return; 1001 } 1002 1003 if (TCG_TARGET_extract_valid(TCG_TYPE_I32, ofs, len)) { 1004 tcg_gen_op4ii_i32(INDEX_op_extract_i32, ret, arg, ofs, len); 1005 return; 1006 } 1007 if (ofs == 0) { 1008 tcg_gen_andi_i32(ret, arg, (1u << len) - 1); 1009 return; 1010 } 1011 1012 /* Assume that zero-extension, if available, is cheaper than a shift. */ 1013 if (TCG_TARGET_extract_valid(TCG_TYPE_I32, 0, ofs + len)) { 1014 tcg_gen_op4ii_i32(INDEX_op_extract_i32, ret, arg, 0, ofs + len); 1015 tcg_gen_shri_i32(ret, ret, ofs); 1016 return; 1017 } 1018 1019 /* ??? Ideally we'd know what values are available for immediate AND. 1020 Assume that 8 bits are available, plus the special case of 16, 1021 so that we get ext8u, ext16u. */ 1022 switch (len) { 1023 case 1 ... 8: case 16: 1024 tcg_gen_shri_i32(ret, arg, ofs); 1025 tcg_gen_andi_i32(ret, ret, (1u << len) - 1); 1026 break; 1027 default: 1028 tcg_gen_shli_i32(ret, arg, 32 - len - ofs); 1029 tcg_gen_shri_i32(ret, ret, 32 - len); 1030 break; 1031 } 1032 } 1033 1034 void tcg_gen_sextract_i32(TCGv_i32 ret, TCGv_i32 arg, 1035 unsigned int ofs, unsigned int len) 1036 { 1037 tcg_debug_assert(ofs < 32); 1038 tcg_debug_assert(len > 0); 1039 tcg_debug_assert(len <= 32); 1040 tcg_debug_assert(ofs + len <= 32); 1041 1042 /* Canonicalize certain special cases, even if extract is supported. */ 1043 if (ofs + len == 32) { 1044 tcg_gen_sari_i32(ret, arg, 32 - len); 1045 return; 1046 } 1047 1048 if (TCG_TARGET_sextract_valid(TCG_TYPE_I32, ofs, len)) { 1049 tcg_gen_op4ii_i32(INDEX_op_sextract_i32, ret, arg, ofs, len); 1050 return; 1051 } 1052 1053 /* Assume that sign-extension, if available, is cheaper than a shift. */ 1054 if (TCG_TARGET_sextract_valid(TCG_TYPE_I32, 0, ofs + len)) { 1055 tcg_gen_op4ii_i32(INDEX_op_sextract_i32, ret, arg, 0, ofs + len); 1056 tcg_gen_sari_i32(ret, ret, ofs); 1057 return; 1058 } 1059 if (TCG_TARGET_sextract_valid(TCG_TYPE_I32, 0, len)) { 1060 tcg_gen_shri_i32(ret, arg, ofs); 1061 tcg_gen_op4ii_i32(INDEX_op_sextract_i32, ret, ret, 0, len); 1062 return; 1063 } 1064 1065 tcg_gen_shli_i32(ret, arg, 32 - len - ofs); 1066 tcg_gen_sari_i32(ret, ret, 32 - len); 1067 } 1068 1069 /* 1070 * Extract 32-bits from a 64-bit input, ah:al, starting from ofs. 1071 * Unlike tcg_gen_extract_i32 above, len is fixed at 32. 1072 */ 1073 void tcg_gen_extract2_i32(TCGv_i32 ret, TCGv_i32 al, TCGv_i32 ah, 1074 unsigned int ofs) 1075 { 1076 tcg_debug_assert(ofs <= 32); 1077 if (ofs == 0) { 1078 tcg_gen_mov_i32(ret, al); 1079 } else if (ofs == 32) { 1080 tcg_gen_mov_i32(ret, ah); 1081 } else if (al == ah) { 1082 tcg_gen_rotri_i32(ret, al, ofs); 1083 } else if (TCG_TARGET_HAS_extract2_i32) { 1084 tcg_gen_op4i_i32(INDEX_op_extract2_i32, ret, al, ah, ofs); 1085 } else { 1086 TCGv_i32 t0 = tcg_temp_ebb_new_i32(); 1087 tcg_gen_shri_i32(t0, al, ofs); 1088 tcg_gen_deposit_i32(ret, t0, ah, 32 - ofs, ofs); 1089 tcg_temp_free_i32(t0); 1090 } 1091 } 1092 1093 void tcg_gen_movcond_i32(TCGCond cond, TCGv_i32 ret, TCGv_i32 c1, 1094 TCGv_i32 c2, TCGv_i32 v1, TCGv_i32 v2) 1095 { 1096 if (cond == TCG_COND_ALWAYS) { 1097 tcg_gen_mov_i32(ret, v1); 1098 } else if (cond == TCG_COND_NEVER) { 1099 tcg_gen_mov_i32(ret, v2); 1100 } else { 1101 tcg_gen_op6i_i32(INDEX_op_movcond_i32, ret, c1, c2, v1, v2, cond); 1102 } 1103 } 1104 1105 void tcg_gen_add2_i32(TCGv_i32 rl, TCGv_i32 rh, TCGv_i32 al, 1106 TCGv_i32 ah, TCGv_i32 bl, TCGv_i32 bh) 1107 { 1108 if (TCG_TARGET_HAS_add2_i32) { 1109 tcg_gen_op6_i32(INDEX_op_add2_i32, rl, rh, al, ah, bl, bh); 1110 } else { 1111 TCGv_i64 t0 = tcg_temp_ebb_new_i64(); 1112 TCGv_i64 t1 = tcg_temp_ebb_new_i64(); 1113 tcg_gen_concat_i32_i64(t0, al, ah); 1114 tcg_gen_concat_i32_i64(t1, bl, bh); 1115 tcg_gen_add_i64(t0, t0, t1); 1116 tcg_gen_extr_i64_i32(rl, rh, t0); 1117 tcg_temp_free_i64(t0); 1118 tcg_temp_free_i64(t1); 1119 } 1120 } 1121 1122 void tcg_gen_sub2_i32(TCGv_i32 rl, TCGv_i32 rh, TCGv_i32 al, 1123 TCGv_i32 ah, TCGv_i32 bl, TCGv_i32 bh) 1124 { 1125 if (TCG_TARGET_HAS_sub2_i32) { 1126 tcg_gen_op6_i32(INDEX_op_sub2_i32, rl, rh, al, ah, bl, bh); 1127 } else { 1128 TCGv_i64 t0 = tcg_temp_ebb_new_i64(); 1129 TCGv_i64 t1 = tcg_temp_ebb_new_i64(); 1130 tcg_gen_concat_i32_i64(t0, al, ah); 1131 tcg_gen_concat_i32_i64(t1, bl, bh); 1132 tcg_gen_sub_i64(t0, t0, t1); 1133 tcg_gen_extr_i64_i32(rl, rh, t0); 1134 tcg_temp_free_i64(t0); 1135 tcg_temp_free_i64(t1); 1136 } 1137 } 1138 1139 void tcg_gen_mulu2_i32(TCGv_i32 rl, TCGv_i32 rh, TCGv_i32 arg1, TCGv_i32 arg2) 1140 { 1141 if (TCG_TARGET_HAS_mulu2_i32) { 1142 tcg_gen_op4_i32(INDEX_op_mulu2_i32, rl, rh, arg1, arg2); 1143 } else if (tcg_op_supported(INDEX_op_muluh, TCG_TYPE_I32, 0)) { 1144 TCGv_i32 t = tcg_temp_ebb_new_i32(); 1145 tcg_gen_op3_i32(INDEX_op_mul, t, arg1, arg2); 1146 tcg_gen_op3_i32(INDEX_op_muluh, rh, arg1, arg2); 1147 tcg_gen_mov_i32(rl, t); 1148 tcg_temp_free_i32(t); 1149 } else if (TCG_TARGET_REG_BITS == 64) { 1150 TCGv_i64 t0 = tcg_temp_ebb_new_i64(); 1151 TCGv_i64 t1 = tcg_temp_ebb_new_i64(); 1152 tcg_gen_extu_i32_i64(t0, arg1); 1153 tcg_gen_extu_i32_i64(t1, arg2); 1154 tcg_gen_mul_i64(t0, t0, t1); 1155 tcg_gen_extr_i64_i32(rl, rh, t0); 1156 tcg_temp_free_i64(t0); 1157 tcg_temp_free_i64(t1); 1158 } else { 1159 qemu_build_not_reached(); 1160 } 1161 } 1162 1163 void tcg_gen_muls2_i32(TCGv_i32 rl, TCGv_i32 rh, TCGv_i32 arg1, TCGv_i32 arg2) 1164 { 1165 if (tcg_op_supported(INDEX_op_muls2, TCG_TYPE_I32, 0)) { 1166 tcg_gen_op4_i32(INDEX_op_muls2, rl, rh, arg1, arg2); 1167 } else if (tcg_op_supported(INDEX_op_mulsh, TCG_TYPE_I32, 0)) { 1168 TCGv_i32 t = tcg_temp_ebb_new_i32(); 1169 tcg_gen_op3_i32(INDEX_op_mul, t, arg1, arg2); 1170 tcg_gen_op3_i32(INDEX_op_mulsh, rh, arg1, arg2); 1171 tcg_gen_mov_i32(rl, t); 1172 tcg_temp_free_i32(t); 1173 } else if (TCG_TARGET_REG_BITS == 32) { 1174 TCGv_i32 t0 = tcg_temp_ebb_new_i32(); 1175 TCGv_i32 t1 = tcg_temp_ebb_new_i32(); 1176 TCGv_i32 t2 = tcg_temp_ebb_new_i32(); 1177 TCGv_i32 t3 = tcg_temp_ebb_new_i32(); 1178 tcg_gen_mulu2_i32(t0, t1, arg1, arg2); 1179 /* Adjust for negative inputs. */ 1180 tcg_gen_sari_i32(t2, arg1, 31); 1181 tcg_gen_sari_i32(t3, arg2, 31); 1182 tcg_gen_and_i32(t2, t2, arg2); 1183 tcg_gen_and_i32(t3, t3, arg1); 1184 tcg_gen_sub_i32(rh, t1, t2); 1185 tcg_gen_sub_i32(rh, rh, t3); 1186 tcg_gen_mov_i32(rl, t0); 1187 tcg_temp_free_i32(t0); 1188 tcg_temp_free_i32(t1); 1189 tcg_temp_free_i32(t2); 1190 tcg_temp_free_i32(t3); 1191 } else { 1192 TCGv_i64 t0 = tcg_temp_ebb_new_i64(); 1193 TCGv_i64 t1 = tcg_temp_ebb_new_i64(); 1194 tcg_gen_ext_i32_i64(t0, arg1); 1195 tcg_gen_ext_i32_i64(t1, arg2); 1196 tcg_gen_mul_i64(t0, t0, t1); 1197 tcg_gen_extr_i64_i32(rl, rh, t0); 1198 tcg_temp_free_i64(t0); 1199 tcg_temp_free_i64(t1); 1200 } 1201 } 1202 1203 void tcg_gen_mulsu2_i32(TCGv_i32 rl, TCGv_i32 rh, TCGv_i32 arg1, TCGv_i32 arg2) 1204 { 1205 if (TCG_TARGET_REG_BITS == 32) { 1206 TCGv_i32 t0 = tcg_temp_ebb_new_i32(); 1207 TCGv_i32 t1 = tcg_temp_ebb_new_i32(); 1208 TCGv_i32 t2 = tcg_temp_ebb_new_i32(); 1209 tcg_gen_mulu2_i32(t0, t1, arg1, arg2); 1210 /* Adjust for negative input for the signed arg1. */ 1211 tcg_gen_sari_i32(t2, arg1, 31); 1212 tcg_gen_and_i32(t2, t2, arg2); 1213 tcg_gen_sub_i32(rh, t1, t2); 1214 tcg_gen_mov_i32(rl, t0); 1215 tcg_temp_free_i32(t0); 1216 tcg_temp_free_i32(t1); 1217 tcg_temp_free_i32(t2); 1218 } else { 1219 TCGv_i64 t0 = tcg_temp_ebb_new_i64(); 1220 TCGv_i64 t1 = tcg_temp_ebb_new_i64(); 1221 tcg_gen_ext_i32_i64(t0, arg1); 1222 tcg_gen_extu_i32_i64(t1, arg2); 1223 tcg_gen_mul_i64(t0, t0, t1); 1224 tcg_gen_extr_i64_i32(rl, rh, t0); 1225 tcg_temp_free_i64(t0); 1226 tcg_temp_free_i64(t1); 1227 } 1228 } 1229 1230 void tcg_gen_ext8s_i32(TCGv_i32 ret, TCGv_i32 arg) 1231 { 1232 tcg_gen_sextract_i32(ret, arg, 0, 8); 1233 } 1234 1235 void tcg_gen_ext16s_i32(TCGv_i32 ret, TCGv_i32 arg) 1236 { 1237 tcg_gen_sextract_i32(ret, arg, 0, 16); 1238 } 1239 1240 void tcg_gen_ext8u_i32(TCGv_i32 ret, TCGv_i32 arg) 1241 { 1242 tcg_gen_extract_i32(ret, arg, 0, 8); 1243 } 1244 1245 void tcg_gen_ext16u_i32(TCGv_i32 ret, TCGv_i32 arg) 1246 { 1247 tcg_gen_extract_i32(ret, arg, 0, 16); 1248 } 1249 1250 /* 1251 * bswap16_i32: 16-bit byte swap on the low bits of a 32-bit value. 1252 * 1253 * Byte pattern: xxab -> yyba 1254 * 1255 * With TCG_BSWAP_IZ, x == zero, else undefined. 1256 * With TCG_BSWAP_OZ, y == zero, with TCG_BSWAP_OS y == sign, else undefined. 1257 */ 1258 void tcg_gen_bswap16_i32(TCGv_i32 ret, TCGv_i32 arg, int flags) 1259 { 1260 /* Only one extension flag may be present. */ 1261 tcg_debug_assert(!(flags & TCG_BSWAP_OS) || !(flags & TCG_BSWAP_OZ)); 1262 1263 if (TCG_TARGET_HAS_bswap16_i32) { 1264 tcg_gen_op3i_i32(INDEX_op_bswap16_i32, ret, arg, flags); 1265 } else { 1266 TCGv_i32 t0 = tcg_temp_ebb_new_i32(); 1267 TCGv_i32 t1 = tcg_temp_ebb_new_i32(); 1268 1269 /* arg = ..ab (IZ) xxab (!IZ) */ 1270 tcg_gen_shri_i32(t0, arg, 8); /* t0 = ...a (IZ) .xxa (!IZ) */ 1271 if (!(flags & TCG_BSWAP_IZ)) { 1272 tcg_gen_ext8u_i32(t0, t0); /* t0 = ...a */ 1273 } 1274 1275 if (flags & TCG_BSWAP_OS) { 1276 tcg_gen_shli_i32(t1, arg, 24); /* t1 = b... */ 1277 tcg_gen_sari_i32(t1, t1, 16); /* t1 = ssb. */ 1278 } else if (flags & TCG_BSWAP_OZ) { 1279 tcg_gen_ext8u_i32(t1, arg); /* t1 = ...b */ 1280 tcg_gen_shli_i32(t1, t1, 8); /* t1 = ..b. */ 1281 } else { 1282 tcg_gen_shli_i32(t1, arg, 8); /* t1 = xab. */ 1283 } 1284 1285 tcg_gen_or_i32(ret, t0, t1); /* ret = ..ba (OZ) */ 1286 /* = ssba (OS) */ 1287 /* = xaba (no flag) */ 1288 tcg_temp_free_i32(t0); 1289 tcg_temp_free_i32(t1); 1290 } 1291 } 1292 1293 /* 1294 * bswap32_i32: 32-bit byte swap on a 32-bit value. 1295 * 1296 * Byte pattern: abcd -> dcba 1297 */ 1298 void tcg_gen_bswap32_i32(TCGv_i32 ret, TCGv_i32 arg) 1299 { 1300 if (TCG_TARGET_HAS_bswap32_i32) { 1301 tcg_gen_op3i_i32(INDEX_op_bswap32_i32, ret, arg, 0); 1302 } else { 1303 TCGv_i32 t0 = tcg_temp_ebb_new_i32(); 1304 TCGv_i32 t1 = tcg_temp_ebb_new_i32(); 1305 TCGv_i32 t2 = tcg_constant_i32(0x00ff00ff); 1306 1307 /* arg = abcd */ 1308 tcg_gen_shri_i32(t0, arg, 8); /* t0 = .abc */ 1309 tcg_gen_and_i32(t1, arg, t2); /* t1 = .b.d */ 1310 tcg_gen_and_i32(t0, t0, t2); /* t0 = .a.c */ 1311 tcg_gen_shli_i32(t1, t1, 8); /* t1 = b.d. */ 1312 tcg_gen_or_i32(ret, t0, t1); /* ret = badc */ 1313 1314 tcg_gen_shri_i32(t0, ret, 16); /* t0 = ..ba */ 1315 tcg_gen_shli_i32(t1, ret, 16); /* t1 = dc.. */ 1316 tcg_gen_or_i32(ret, t0, t1); /* ret = dcba */ 1317 1318 tcg_temp_free_i32(t0); 1319 tcg_temp_free_i32(t1); 1320 } 1321 } 1322 1323 /* 1324 * hswap_i32: Swap 16-bit halfwords within a 32-bit value. 1325 * 1326 * Byte pattern: abcd -> cdab 1327 */ 1328 void tcg_gen_hswap_i32(TCGv_i32 ret, TCGv_i32 arg) 1329 { 1330 /* Swapping 2 16-bit elements is a rotate. */ 1331 tcg_gen_rotli_i32(ret, arg, 16); 1332 } 1333 1334 void tcg_gen_smin_i32(TCGv_i32 ret, TCGv_i32 a, TCGv_i32 b) 1335 { 1336 tcg_gen_movcond_i32(TCG_COND_LT, ret, a, b, a, b); 1337 } 1338 1339 void tcg_gen_umin_i32(TCGv_i32 ret, TCGv_i32 a, TCGv_i32 b) 1340 { 1341 tcg_gen_movcond_i32(TCG_COND_LTU, ret, a, b, a, b); 1342 } 1343 1344 void tcg_gen_smax_i32(TCGv_i32 ret, TCGv_i32 a, TCGv_i32 b) 1345 { 1346 tcg_gen_movcond_i32(TCG_COND_LT, ret, a, b, b, a); 1347 } 1348 1349 void tcg_gen_umax_i32(TCGv_i32 ret, TCGv_i32 a, TCGv_i32 b) 1350 { 1351 tcg_gen_movcond_i32(TCG_COND_LTU, ret, a, b, b, a); 1352 } 1353 1354 void tcg_gen_abs_i32(TCGv_i32 ret, TCGv_i32 a) 1355 { 1356 TCGv_i32 t = tcg_temp_ebb_new_i32(); 1357 1358 tcg_gen_sari_i32(t, a, 31); 1359 tcg_gen_xor_i32(ret, a, t); 1360 tcg_gen_sub_i32(ret, ret, t); 1361 tcg_temp_free_i32(t); 1362 } 1363 1364 void tcg_gen_ld8u_i32(TCGv_i32 ret, TCGv_ptr arg2, tcg_target_long offset) 1365 { 1366 tcg_gen_ldst_op_i32(INDEX_op_ld8u_i32, ret, arg2, offset); 1367 } 1368 1369 void tcg_gen_ld8s_i32(TCGv_i32 ret, TCGv_ptr arg2, tcg_target_long offset) 1370 { 1371 tcg_gen_ldst_op_i32(INDEX_op_ld8s_i32, ret, arg2, offset); 1372 } 1373 1374 void tcg_gen_ld16u_i32(TCGv_i32 ret, TCGv_ptr arg2, tcg_target_long offset) 1375 { 1376 tcg_gen_ldst_op_i32(INDEX_op_ld16u_i32, ret, arg2, offset); 1377 } 1378 1379 void tcg_gen_ld16s_i32(TCGv_i32 ret, TCGv_ptr arg2, tcg_target_long offset) 1380 { 1381 tcg_gen_ldst_op_i32(INDEX_op_ld16s_i32, ret, arg2, offset); 1382 } 1383 1384 void tcg_gen_ld_i32(TCGv_i32 ret, TCGv_ptr arg2, tcg_target_long offset) 1385 { 1386 tcg_gen_ldst_op_i32(INDEX_op_ld_i32, ret, arg2, offset); 1387 } 1388 1389 void tcg_gen_st8_i32(TCGv_i32 arg1, TCGv_ptr arg2, tcg_target_long offset) 1390 { 1391 tcg_gen_ldst_op_i32(INDEX_op_st8_i32, arg1, arg2, offset); 1392 } 1393 1394 void tcg_gen_st16_i32(TCGv_i32 arg1, TCGv_ptr arg2, tcg_target_long offset) 1395 { 1396 tcg_gen_ldst_op_i32(INDEX_op_st16_i32, arg1, arg2, offset); 1397 } 1398 1399 void tcg_gen_st_i32(TCGv_i32 arg1, TCGv_ptr arg2, tcg_target_long offset) 1400 { 1401 tcg_gen_ldst_op_i32(INDEX_op_st_i32, arg1, arg2, offset); 1402 } 1403 1404 1405 /* 64-bit ops */ 1406 1407 void tcg_gen_discard_i64(TCGv_i64 arg) 1408 { 1409 if (TCG_TARGET_REG_BITS == 64) { 1410 tcg_gen_op1_i64(INDEX_op_discard, TCG_TYPE_I64, arg); 1411 } else { 1412 tcg_gen_discard_i32(TCGV_LOW(arg)); 1413 tcg_gen_discard_i32(TCGV_HIGH(arg)); 1414 } 1415 } 1416 1417 void tcg_gen_mov_i64(TCGv_i64 ret, TCGv_i64 arg) 1418 { 1419 if (ret == arg) { 1420 return; 1421 } 1422 if (TCG_TARGET_REG_BITS == 64) { 1423 tcg_gen_op2_i64(INDEX_op_mov, ret, arg); 1424 } else { 1425 TCGTemp *ts = tcgv_i64_temp(arg); 1426 1427 /* Canonicalize TCGv_i64 TEMP_CONST into TCGv_i32 TEMP_CONST. */ 1428 if (ts->kind == TEMP_CONST) { 1429 tcg_gen_movi_i64(ret, ts->val); 1430 } else { 1431 tcg_gen_mov_i32(TCGV_LOW(ret), TCGV_LOW(arg)); 1432 tcg_gen_mov_i32(TCGV_HIGH(ret), TCGV_HIGH(arg)); 1433 } 1434 } 1435 } 1436 1437 void tcg_gen_movi_i64(TCGv_i64 ret, int64_t arg) 1438 { 1439 if (TCG_TARGET_REG_BITS == 64) { 1440 tcg_gen_mov_i64(ret, tcg_constant_i64(arg)); 1441 } else { 1442 tcg_gen_movi_i32(TCGV_LOW(ret), arg); 1443 tcg_gen_movi_i32(TCGV_HIGH(ret), arg >> 32); 1444 } 1445 } 1446 1447 void tcg_gen_ld8u_i64(TCGv_i64 ret, TCGv_ptr arg2, tcg_target_long offset) 1448 { 1449 if (TCG_TARGET_REG_BITS == 64) { 1450 tcg_gen_ldst_op_i64(INDEX_op_ld8u_i64, ret, arg2, offset); 1451 } else { 1452 tcg_gen_ld8u_i32(TCGV_LOW(ret), arg2, offset); 1453 tcg_gen_movi_i32(TCGV_HIGH(ret), 0); 1454 } 1455 } 1456 1457 void tcg_gen_ld8s_i64(TCGv_i64 ret, TCGv_ptr arg2, tcg_target_long offset) 1458 { 1459 if (TCG_TARGET_REG_BITS == 64) { 1460 tcg_gen_ldst_op_i64(INDEX_op_ld8s_i64, ret, arg2, offset); 1461 } else { 1462 tcg_gen_ld8s_i32(TCGV_LOW(ret), arg2, offset); 1463 tcg_gen_sari_i32(TCGV_HIGH(ret), TCGV_LOW(ret), 31); 1464 } 1465 } 1466 1467 void tcg_gen_ld16u_i64(TCGv_i64 ret, TCGv_ptr arg2, tcg_target_long offset) 1468 { 1469 if (TCG_TARGET_REG_BITS == 64) { 1470 tcg_gen_ldst_op_i64(INDEX_op_ld16u_i64, ret, arg2, offset); 1471 } else { 1472 tcg_gen_ld16u_i32(TCGV_LOW(ret), arg2, offset); 1473 tcg_gen_movi_i32(TCGV_HIGH(ret), 0); 1474 } 1475 } 1476 1477 void tcg_gen_ld16s_i64(TCGv_i64 ret, TCGv_ptr arg2, tcg_target_long offset) 1478 { 1479 if (TCG_TARGET_REG_BITS == 64) { 1480 tcg_gen_ldst_op_i64(INDEX_op_ld16s_i64, ret, arg2, offset); 1481 } else { 1482 tcg_gen_ld16s_i32(TCGV_LOW(ret), arg2, offset); 1483 tcg_gen_sari_i32(TCGV_HIGH(ret), TCGV_LOW(ret), 31); 1484 } 1485 } 1486 1487 void tcg_gen_ld32u_i64(TCGv_i64 ret, TCGv_ptr arg2, tcg_target_long offset) 1488 { 1489 if (TCG_TARGET_REG_BITS == 64) { 1490 tcg_gen_ldst_op_i64(INDEX_op_ld32u_i64, ret, arg2, offset); 1491 } else { 1492 tcg_gen_ld_i32(TCGV_LOW(ret), arg2, offset); 1493 tcg_gen_movi_i32(TCGV_HIGH(ret), 0); 1494 } 1495 } 1496 1497 void tcg_gen_ld32s_i64(TCGv_i64 ret, TCGv_ptr arg2, tcg_target_long offset) 1498 { 1499 if (TCG_TARGET_REG_BITS == 64) { 1500 tcg_gen_ldst_op_i64(INDEX_op_ld32s_i64, ret, arg2, offset); 1501 } else { 1502 tcg_gen_ld_i32(TCGV_LOW(ret), arg2, offset); 1503 tcg_gen_sari_i32(TCGV_HIGH(ret), TCGV_LOW(ret), 31); 1504 } 1505 } 1506 1507 void tcg_gen_ld_i64(TCGv_i64 ret, TCGv_ptr arg2, tcg_target_long offset) 1508 { 1509 /* 1510 * For 32-bit host, since arg2 and ret have different types, 1511 * they cannot be the same temporary -- no chance of overlap. 1512 */ 1513 if (TCG_TARGET_REG_BITS == 64) { 1514 tcg_gen_ldst_op_i64(INDEX_op_ld_i64, ret, arg2, offset); 1515 } else if (HOST_BIG_ENDIAN) { 1516 tcg_gen_ld_i32(TCGV_HIGH(ret), arg2, offset); 1517 tcg_gen_ld_i32(TCGV_LOW(ret), arg2, offset + 4); 1518 } else { 1519 tcg_gen_ld_i32(TCGV_LOW(ret), arg2, offset); 1520 tcg_gen_ld_i32(TCGV_HIGH(ret), arg2, offset + 4); 1521 } 1522 } 1523 1524 void tcg_gen_st8_i64(TCGv_i64 arg1, TCGv_ptr arg2, tcg_target_long offset) 1525 { 1526 if (TCG_TARGET_REG_BITS == 64) { 1527 tcg_gen_ldst_op_i64(INDEX_op_st8_i64, arg1, arg2, offset); 1528 } else { 1529 tcg_gen_st8_i32(TCGV_LOW(arg1), arg2, offset); 1530 } 1531 } 1532 1533 void tcg_gen_st16_i64(TCGv_i64 arg1, TCGv_ptr arg2, tcg_target_long offset) 1534 { 1535 if (TCG_TARGET_REG_BITS == 64) { 1536 tcg_gen_ldst_op_i64(INDEX_op_st16_i64, arg1, arg2, offset); 1537 } else { 1538 tcg_gen_st16_i32(TCGV_LOW(arg1), arg2, offset); 1539 } 1540 } 1541 1542 void tcg_gen_st32_i64(TCGv_i64 arg1, TCGv_ptr arg2, tcg_target_long offset) 1543 { 1544 if (TCG_TARGET_REG_BITS == 64) { 1545 tcg_gen_ldst_op_i64(INDEX_op_st32_i64, arg1, arg2, offset); 1546 } else { 1547 tcg_gen_st_i32(TCGV_LOW(arg1), arg2, offset); 1548 } 1549 } 1550 1551 void tcg_gen_st_i64(TCGv_i64 arg1, TCGv_ptr arg2, tcg_target_long offset) 1552 { 1553 if (TCG_TARGET_REG_BITS == 64) { 1554 tcg_gen_ldst_op_i64(INDEX_op_st_i64, arg1, arg2, offset); 1555 } else if (HOST_BIG_ENDIAN) { 1556 tcg_gen_st_i32(TCGV_HIGH(arg1), arg2, offset); 1557 tcg_gen_st_i32(TCGV_LOW(arg1), arg2, offset + 4); 1558 } else { 1559 tcg_gen_st_i32(TCGV_LOW(arg1), arg2, offset); 1560 tcg_gen_st_i32(TCGV_HIGH(arg1), arg2, offset + 4); 1561 } 1562 } 1563 1564 void tcg_gen_add_i64(TCGv_i64 ret, TCGv_i64 arg1, TCGv_i64 arg2) 1565 { 1566 if (TCG_TARGET_REG_BITS == 64) { 1567 tcg_gen_op3_i64(INDEX_op_add, ret, arg1, arg2); 1568 } else { 1569 tcg_gen_add2_i32(TCGV_LOW(ret), TCGV_HIGH(ret), TCGV_LOW(arg1), 1570 TCGV_HIGH(arg1), TCGV_LOW(arg2), TCGV_HIGH(arg2)); 1571 } 1572 } 1573 1574 void tcg_gen_sub_i64(TCGv_i64 ret, TCGv_i64 arg1, TCGv_i64 arg2) 1575 { 1576 if (TCG_TARGET_REG_BITS == 64) { 1577 tcg_gen_op3_i64(INDEX_op_sub, ret, arg1, arg2); 1578 } else { 1579 tcg_gen_sub2_i32(TCGV_LOW(ret), TCGV_HIGH(ret), TCGV_LOW(arg1), 1580 TCGV_HIGH(arg1), TCGV_LOW(arg2), TCGV_HIGH(arg2)); 1581 } 1582 } 1583 1584 void tcg_gen_and_i64(TCGv_i64 ret, TCGv_i64 arg1, TCGv_i64 arg2) 1585 { 1586 if (TCG_TARGET_REG_BITS == 64) { 1587 tcg_gen_op3_i64(INDEX_op_and, ret, arg1, arg2); 1588 } else { 1589 tcg_gen_and_i32(TCGV_LOW(ret), TCGV_LOW(arg1), TCGV_LOW(arg2)); 1590 tcg_gen_and_i32(TCGV_HIGH(ret), TCGV_HIGH(arg1), TCGV_HIGH(arg2)); 1591 } 1592 } 1593 1594 void tcg_gen_or_i64(TCGv_i64 ret, TCGv_i64 arg1, TCGv_i64 arg2) 1595 { 1596 if (TCG_TARGET_REG_BITS == 64) { 1597 tcg_gen_op3_i64(INDEX_op_or, ret, arg1, arg2); 1598 } else { 1599 tcg_gen_or_i32(TCGV_LOW(ret), TCGV_LOW(arg1), TCGV_LOW(arg2)); 1600 tcg_gen_or_i32(TCGV_HIGH(ret), TCGV_HIGH(arg1), TCGV_HIGH(arg2)); 1601 } 1602 } 1603 1604 void tcg_gen_xor_i64(TCGv_i64 ret, TCGv_i64 arg1, TCGv_i64 arg2) 1605 { 1606 if (TCG_TARGET_REG_BITS == 64) { 1607 tcg_gen_op3_i64(INDEX_op_xor, ret, arg1, arg2); 1608 } else { 1609 tcg_gen_xor_i32(TCGV_LOW(ret), TCGV_LOW(arg1), TCGV_LOW(arg2)); 1610 tcg_gen_xor_i32(TCGV_HIGH(ret), TCGV_HIGH(arg1), TCGV_HIGH(arg2)); 1611 } 1612 } 1613 1614 void tcg_gen_shl_i64(TCGv_i64 ret, TCGv_i64 arg1, TCGv_i64 arg2) 1615 { 1616 if (TCG_TARGET_REG_BITS == 64) { 1617 tcg_gen_op3_i64(INDEX_op_shl, ret, arg1, arg2); 1618 } else { 1619 gen_helper_shl_i64(ret, arg1, arg2); 1620 } 1621 } 1622 1623 void tcg_gen_shr_i64(TCGv_i64 ret, TCGv_i64 arg1, TCGv_i64 arg2) 1624 { 1625 if (TCG_TARGET_REG_BITS == 64) { 1626 tcg_gen_op3_i64(INDEX_op_shr, ret, arg1, arg2); 1627 } else { 1628 gen_helper_shr_i64(ret, arg1, arg2); 1629 } 1630 } 1631 1632 void tcg_gen_sar_i64(TCGv_i64 ret, TCGv_i64 arg1, TCGv_i64 arg2) 1633 { 1634 if (TCG_TARGET_REG_BITS == 64) { 1635 tcg_gen_op3_i64(INDEX_op_sar, ret, arg1, arg2); 1636 } else { 1637 gen_helper_sar_i64(ret, arg1, arg2); 1638 } 1639 } 1640 1641 void tcg_gen_mul_i64(TCGv_i64 ret, TCGv_i64 arg1, TCGv_i64 arg2) 1642 { 1643 TCGv_i64 t0; 1644 TCGv_i32 t1; 1645 1646 if (TCG_TARGET_REG_BITS == 64) { 1647 tcg_gen_op3_i64(INDEX_op_mul, ret, arg1, arg2); 1648 return; 1649 } 1650 1651 1652 t0 = tcg_temp_ebb_new_i64(); 1653 t1 = tcg_temp_ebb_new_i32(); 1654 1655 tcg_gen_mulu2_i32(TCGV_LOW(t0), TCGV_HIGH(t0), 1656 TCGV_LOW(arg1), TCGV_LOW(arg2)); 1657 1658 tcg_gen_mul_i32(t1, TCGV_LOW(arg1), TCGV_HIGH(arg2)); 1659 tcg_gen_add_i32(TCGV_HIGH(t0), TCGV_HIGH(t0), t1); 1660 tcg_gen_mul_i32(t1, TCGV_HIGH(arg1), TCGV_LOW(arg2)); 1661 tcg_gen_add_i32(TCGV_HIGH(t0), TCGV_HIGH(t0), t1); 1662 1663 tcg_gen_mov_i64(ret, t0); 1664 tcg_temp_free_i64(t0); 1665 tcg_temp_free_i32(t1); 1666 } 1667 1668 void tcg_gen_addi_i64(TCGv_i64 ret, TCGv_i64 arg1, int64_t arg2) 1669 { 1670 /* some cases can be optimized here */ 1671 if (arg2 == 0) { 1672 tcg_gen_mov_i64(ret, arg1); 1673 } else if (TCG_TARGET_REG_BITS == 64) { 1674 tcg_gen_add_i64(ret, arg1, tcg_constant_i64(arg2)); 1675 } else { 1676 tcg_gen_add2_i32(TCGV_LOW(ret), TCGV_HIGH(ret), 1677 TCGV_LOW(arg1), TCGV_HIGH(arg1), 1678 tcg_constant_i32(arg2), tcg_constant_i32(arg2 >> 32)); 1679 } 1680 } 1681 1682 void tcg_gen_subfi_i64(TCGv_i64 ret, int64_t arg1, TCGv_i64 arg2) 1683 { 1684 if (arg1 == 0) { 1685 tcg_gen_neg_i64(ret, arg2); 1686 } else if (TCG_TARGET_REG_BITS == 64) { 1687 tcg_gen_sub_i64(ret, tcg_constant_i64(arg1), arg2); 1688 } else { 1689 tcg_gen_sub2_i32(TCGV_LOW(ret), TCGV_HIGH(ret), 1690 tcg_constant_i32(arg1), tcg_constant_i32(arg1 >> 32), 1691 TCGV_LOW(arg2), TCGV_HIGH(arg2)); 1692 } 1693 } 1694 1695 void tcg_gen_subi_i64(TCGv_i64 ret, TCGv_i64 arg1, int64_t arg2) 1696 { 1697 tcg_gen_addi_i64(ret, arg1, -arg2); 1698 } 1699 1700 void tcg_gen_neg_i64(TCGv_i64 ret, TCGv_i64 arg) 1701 { 1702 if (TCG_TARGET_REG_BITS == 64) { 1703 tcg_gen_op2_i64(INDEX_op_neg, ret, arg); 1704 } else { 1705 TCGv_i32 zero = tcg_constant_i32(0); 1706 tcg_gen_sub2_i32(TCGV_LOW(ret), TCGV_HIGH(ret), 1707 zero, zero, TCGV_LOW(arg), TCGV_HIGH(arg)); 1708 } 1709 } 1710 1711 void tcg_gen_andi_i64(TCGv_i64 ret, TCGv_i64 arg1, int64_t arg2) 1712 { 1713 if (TCG_TARGET_REG_BITS == 32) { 1714 tcg_gen_andi_i32(TCGV_LOW(ret), TCGV_LOW(arg1), arg2); 1715 tcg_gen_andi_i32(TCGV_HIGH(ret), TCGV_HIGH(arg1), arg2 >> 32); 1716 return; 1717 } 1718 1719 /* Some cases can be optimized here. */ 1720 switch (arg2) { 1721 case 0: 1722 tcg_gen_movi_i64(ret, 0); 1723 return; 1724 case -1: 1725 tcg_gen_mov_i64(ret, arg1); 1726 return; 1727 default: 1728 /* 1729 * Canonicalize on extract, if valid. This aids x86 with its 1730 * 2 operand MOVZBL and 2 operand AND, selecting the TCGOpcode 1731 * which does not require matching operands. Other backends can 1732 * trivially expand the extract to AND during code generation. 1733 */ 1734 if (!(arg2 & (arg2 + 1))) { 1735 unsigned len = ctz64(~arg2); 1736 if (TCG_TARGET_extract_valid(TCG_TYPE_I64, 0, len)) { 1737 tcg_gen_extract_i64(ret, arg1, 0, len); 1738 return; 1739 } 1740 } 1741 break; 1742 } 1743 1744 tcg_gen_and_i64(ret, arg1, tcg_constant_i64(arg2)); 1745 } 1746 1747 void tcg_gen_ori_i64(TCGv_i64 ret, TCGv_i64 arg1, int64_t arg2) 1748 { 1749 if (TCG_TARGET_REG_BITS == 32) { 1750 tcg_gen_ori_i32(TCGV_LOW(ret), TCGV_LOW(arg1), arg2); 1751 tcg_gen_ori_i32(TCGV_HIGH(ret), TCGV_HIGH(arg1), arg2 >> 32); 1752 return; 1753 } 1754 /* Some cases can be optimized here. */ 1755 if (arg2 == -1) { 1756 tcg_gen_movi_i64(ret, -1); 1757 } else if (arg2 == 0) { 1758 tcg_gen_mov_i64(ret, arg1); 1759 } else { 1760 tcg_gen_or_i64(ret, arg1, tcg_constant_i64(arg2)); 1761 } 1762 } 1763 1764 void tcg_gen_xori_i64(TCGv_i64 ret, TCGv_i64 arg1, int64_t arg2) 1765 { 1766 if (TCG_TARGET_REG_BITS == 32) { 1767 tcg_gen_xori_i32(TCGV_LOW(ret), TCGV_LOW(arg1), arg2); 1768 tcg_gen_xori_i32(TCGV_HIGH(ret), TCGV_HIGH(arg1), arg2 >> 32); 1769 return; 1770 } 1771 /* Some cases can be optimized here. */ 1772 if (arg2 == 0) { 1773 tcg_gen_mov_i64(ret, arg1); 1774 } else if (arg2 == -1 && 1775 tcg_op_supported(INDEX_op_not, TCG_TYPE_I64, 0)) { 1776 /* Don't recurse with tcg_gen_not_i64. */ 1777 tcg_gen_op2_i64(INDEX_op_not, ret, arg1); 1778 } else { 1779 tcg_gen_xor_i64(ret, arg1, tcg_constant_i64(arg2)); 1780 } 1781 } 1782 1783 static inline void tcg_gen_shifti_i64(TCGv_i64 ret, TCGv_i64 arg1, 1784 unsigned c, bool right, bool arith) 1785 { 1786 tcg_debug_assert(c < 64); 1787 if (c == 0) { 1788 tcg_gen_mov_i32(TCGV_LOW(ret), TCGV_LOW(arg1)); 1789 tcg_gen_mov_i32(TCGV_HIGH(ret), TCGV_HIGH(arg1)); 1790 } else if (c >= 32) { 1791 c -= 32; 1792 if (right) { 1793 if (arith) { 1794 tcg_gen_sari_i32(TCGV_LOW(ret), TCGV_HIGH(arg1), c); 1795 tcg_gen_sari_i32(TCGV_HIGH(ret), TCGV_HIGH(arg1), 31); 1796 } else { 1797 tcg_gen_shri_i32(TCGV_LOW(ret), TCGV_HIGH(arg1), c); 1798 tcg_gen_movi_i32(TCGV_HIGH(ret), 0); 1799 } 1800 } else { 1801 tcg_gen_shli_i32(TCGV_HIGH(ret), TCGV_LOW(arg1), c); 1802 tcg_gen_movi_i32(TCGV_LOW(ret), 0); 1803 } 1804 } else if (right) { 1805 if (TCG_TARGET_HAS_extract2_i32) { 1806 tcg_gen_extract2_i32(TCGV_LOW(ret), 1807 TCGV_LOW(arg1), TCGV_HIGH(arg1), c); 1808 } else { 1809 tcg_gen_shri_i32(TCGV_LOW(ret), TCGV_LOW(arg1), c); 1810 tcg_gen_deposit_i32(TCGV_LOW(ret), TCGV_LOW(ret), 1811 TCGV_HIGH(arg1), 32 - c, c); 1812 } 1813 if (arith) { 1814 tcg_gen_sari_i32(TCGV_HIGH(ret), TCGV_HIGH(arg1), c); 1815 } else { 1816 tcg_gen_shri_i32(TCGV_HIGH(ret), TCGV_HIGH(arg1), c); 1817 } 1818 } else { 1819 if (TCG_TARGET_HAS_extract2_i32) { 1820 tcg_gen_extract2_i32(TCGV_HIGH(ret), 1821 TCGV_LOW(arg1), TCGV_HIGH(arg1), 32 - c); 1822 } else { 1823 TCGv_i32 t0 = tcg_temp_ebb_new_i32(); 1824 tcg_gen_shri_i32(t0, TCGV_LOW(arg1), 32 - c); 1825 tcg_gen_deposit_i32(TCGV_HIGH(ret), t0, 1826 TCGV_HIGH(arg1), c, 32 - c); 1827 tcg_temp_free_i32(t0); 1828 } 1829 tcg_gen_shli_i32(TCGV_LOW(ret), TCGV_LOW(arg1), c); 1830 } 1831 } 1832 1833 void tcg_gen_shli_i64(TCGv_i64 ret, TCGv_i64 arg1, int64_t arg2) 1834 { 1835 tcg_debug_assert(arg2 >= 0 && arg2 < 64); 1836 if (TCG_TARGET_REG_BITS == 32) { 1837 tcg_gen_shifti_i64(ret, arg1, arg2, 0, 0); 1838 } else if (arg2 == 0) { 1839 tcg_gen_mov_i64(ret, arg1); 1840 } else { 1841 tcg_gen_shl_i64(ret, arg1, tcg_constant_i64(arg2)); 1842 } 1843 } 1844 1845 void tcg_gen_shri_i64(TCGv_i64 ret, TCGv_i64 arg1, int64_t arg2) 1846 { 1847 tcg_debug_assert(arg2 >= 0 && arg2 < 64); 1848 if (TCG_TARGET_REG_BITS == 32) { 1849 tcg_gen_shifti_i64(ret, arg1, arg2, 1, 0); 1850 } else if (arg2 == 0) { 1851 tcg_gen_mov_i64(ret, arg1); 1852 } else { 1853 tcg_gen_shr_i64(ret, arg1, tcg_constant_i64(arg2)); 1854 } 1855 } 1856 1857 void tcg_gen_sari_i64(TCGv_i64 ret, TCGv_i64 arg1, int64_t arg2) 1858 { 1859 tcg_debug_assert(arg2 >= 0 && arg2 < 64); 1860 if (TCG_TARGET_REG_BITS == 32) { 1861 tcg_gen_shifti_i64(ret, arg1, arg2, 1, 1); 1862 } else if (arg2 == 0) { 1863 tcg_gen_mov_i64(ret, arg1); 1864 } else { 1865 tcg_gen_sar_i64(ret, arg1, tcg_constant_i64(arg2)); 1866 } 1867 } 1868 1869 void tcg_gen_brcond_i64(TCGCond cond, TCGv_i64 arg1, TCGv_i64 arg2, TCGLabel *l) 1870 { 1871 if (cond == TCG_COND_ALWAYS) { 1872 tcg_gen_br(l); 1873 } else if (cond != TCG_COND_NEVER) { 1874 TCGOp *op; 1875 if (TCG_TARGET_REG_BITS == 32) { 1876 op = tcg_gen_op6ii_i32(INDEX_op_brcond2_i32, TCGV_LOW(arg1), 1877 TCGV_HIGH(arg1), TCGV_LOW(arg2), 1878 TCGV_HIGH(arg2), cond, label_arg(l)); 1879 } else { 1880 op = tcg_gen_op4ii_i64(INDEX_op_brcond_i64, arg1, arg2, cond, 1881 label_arg(l)); 1882 } 1883 add_as_label_use(l, op); 1884 } 1885 } 1886 1887 void tcg_gen_brcondi_i64(TCGCond cond, TCGv_i64 arg1, int64_t arg2, TCGLabel *l) 1888 { 1889 if (TCG_TARGET_REG_BITS == 64) { 1890 tcg_gen_brcond_i64(cond, arg1, tcg_constant_i64(arg2), l); 1891 } else if (cond == TCG_COND_ALWAYS) { 1892 tcg_gen_br(l); 1893 } else if (cond != TCG_COND_NEVER) { 1894 TCGOp *op = tcg_gen_op6ii_i32(INDEX_op_brcond2_i32, 1895 TCGV_LOW(arg1), TCGV_HIGH(arg1), 1896 tcg_constant_i32(arg2), 1897 tcg_constant_i32(arg2 >> 32), 1898 cond, label_arg(l)); 1899 add_as_label_use(l, op); 1900 } 1901 } 1902 1903 void tcg_gen_setcond_i64(TCGCond cond, TCGv_i64 ret, 1904 TCGv_i64 arg1, TCGv_i64 arg2) 1905 { 1906 if (cond == TCG_COND_ALWAYS) { 1907 tcg_gen_movi_i64(ret, 1); 1908 } else if (cond == TCG_COND_NEVER) { 1909 tcg_gen_movi_i64(ret, 0); 1910 } else { 1911 if (TCG_TARGET_REG_BITS == 32) { 1912 tcg_gen_op6i_i32(INDEX_op_setcond2_i32, TCGV_LOW(ret), 1913 TCGV_LOW(arg1), TCGV_HIGH(arg1), 1914 TCGV_LOW(arg2), TCGV_HIGH(arg2), cond); 1915 tcg_gen_movi_i32(TCGV_HIGH(ret), 0); 1916 } else { 1917 tcg_gen_op4i_i64(INDEX_op_setcond_i64, ret, arg1, arg2, cond); 1918 } 1919 } 1920 } 1921 1922 void tcg_gen_setcondi_i64(TCGCond cond, TCGv_i64 ret, 1923 TCGv_i64 arg1, int64_t arg2) 1924 { 1925 if (TCG_TARGET_REG_BITS == 64) { 1926 tcg_gen_setcond_i64(cond, ret, arg1, tcg_constant_i64(arg2)); 1927 } else if (cond == TCG_COND_ALWAYS) { 1928 tcg_gen_movi_i64(ret, 1); 1929 } else if (cond == TCG_COND_NEVER) { 1930 tcg_gen_movi_i64(ret, 0); 1931 } else { 1932 tcg_gen_op6i_i32(INDEX_op_setcond2_i32, TCGV_LOW(ret), 1933 TCGV_LOW(arg1), TCGV_HIGH(arg1), 1934 tcg_constant_i32(arg2), 1935 tcg_constant_i32(arg2 >> 32), cond); 1936 tcg_gen_movi_i32(TCGV_HIGH(ret), 0); 1937 } 1938 } 1939 1940 void tcg_gen_negsetcondi_i64(TCGCond cond, TCGv_i64 ret, 1941 TCGv_i64 arg1, int64_t arg2) 1942 { 1943 tcg_gen_negsetcond_i64(cond, ret, arg1, tcg_constant_i64(arg2)); 1944 } 1945 1946 void tcg_gen_negsetcond_i64(TCGCond cond, TCGv_i64 ret, 1947 TCGv_i64 arg1, TCGv_i64 arg2) 1948 { 1949 if (cond == TCG_COND_ALWAYS) { 1950 tcg_gen_movi_i64(ret, -1); 1951 } else if (cond == TCG_COND_NEVER) { 1952 tcg_gen_movi_i64(ret, 0); 1953 } else if (TCG_TARGET_HAS_negsetcond_i64) { 1954 tcg_gen_op4i_i64(INDEX_op_negsetcond_i64, ret, arg1, arg2, cond); 1955 } else if (TCG_TARGET_REG_BITS == 32) { 1956 tcg_gen_op6i_i32(INDEX_op_setcond2_i32, TCGV_LOW(ret), 1957 TCGV_LOW(arg1), TCGV_HIGH(arg1), 1958 TCGV_LOW(arg2), TCGV_HIGH(arg2), cond); 1959 tcg_gen_neg_i32(TCGV_LOW(ret), TCGV_LOW(ret)); 1960 tcg_gen_mov_i32(TCGV_HIGH(ret), TCGV_LOW(ret)); 1961 } else { 1962 tcg_gen_setcond_i64(cond, ret, arg1, arg2); 1963 tcg_gen_neg_i64(ret, ret); 1964 } 1965 } 1966 1967 void tcg_gen_muli_i64(TCGv_i64 ret, TCGv_i64 arg1, int64_t arg2) 1968 { 1969 if (arg2 == 0) { 1970 tcg_gen_movi_i64(ret, 0); 1971 } else if (is_power_of_2(arg2)) { 1972 tcg_gen_shli_i64(ret, arg1, ctz64(arg2)); 1973 } else { 1974 tcg_gen_mul_i64(ret, arg1, tcg_constant_i64(arg2)); 1975 } 1976 } 1977 1978 void tcg_gen_div_i64(TCGv_i64 ret, TCGv_i64 arg1, TCGv_i64 arg2) 1979 { 1980 if (tcg_op_supported(INDEX_op_divs, TCG_TYPE_I64, 0)) { 1981 tcg_gen_op3_i64(INDEX_op_divs, ret, arg1, arg2); 1982 } else if (tcg_op_supported(INDEX_op_divs2, TCG_TYPE_I64, 0)) { 1983 TCGv_i64 t0 = tcg_temp_ebb_new_i64(); 1984 tcg_gen_sari_i64(t0, arg1, 63); 1985 tcg_gen_op5_i64(INDEX_op_divs2, ret, t0, arg1, t0, arg2); 1986 tcg_temp_free_i64(t0); 1987 } else { 1988 gen_helper_div_i64(ret, arg1, arg2); 1989 } 1990 } 1991 1992 void tcg_gen_rem_i64(TCGv_i64 ret, TCGv_i64 arg1, TCGv_i64 arg2) 1993 { 1994 if (tcg_op_supported(INDEX_op_rems, TCG_TYPE_I64, 0)) { 1995 tcg_gen_op3_i64(INDEX_op_rems, ret, arg1, arg2); 1996 } else if (tcg_op_supported(INDEX_op_divs, TCG_TYPE_I64, 0)) { 1997 TCGv_i64 t0 = tcg_temp_ebb_new_i64(); 1998 tcg_gen_op3_i64(INDEX_op_divs, t0, arg1, arg2); 1999 tcg_gen_mul_i64(t0, t0, arg2); 2000 tcg_gen_sub_i64(ret, arg1, t0); 2001 tcg_temp_free_i64(t0); 2002 } else if (tcg_op_supported(INDEX_op_divs2, TCG_TYPE_I64, 0)) { 2003 TCGv_i64 t0 = tcg_temp_ebb_new_i64(); 2004 tcg_gen_sari_i64(t0, arg1, 63); 2005 tcg_gen_op5_i64(INDEX_op_divs2, t0, ret, arg1, t0, arg2); 2006 tcg_temp_free_i64(t0); 2007 } else { 2008 gen_helper_rem_i64(ret, arg1, arg2); 2009 } 2010 } 2011 2012 void tcg_gen_divu_i64(TCGv_i64 ret, TCGv_i64 arg1, TCGv_i64 arg2) 2013 { 2014 if (tcg_op_supported(INDEX_op_divu, TCG_TYPE_I64, 0)) { 2015 tcg_gen_op3_i64(INDEX_op_divu, ret, arg1, arg2); 2016 } else if (tcg_op_supported(INDEX_op_divu2, TCG_TYPE_I64, 0)) { 2017 TCGv_i64 t0 = tcg_temp_ebb_new_i64(); 2018 TCGv_i64 zero = tcg_constant_i64(0); 2019 tcg_gen_op5_i64(INDEX_op_divu2, ret, t0, arg1, zero, arg2); 2020 tcg_temp_free_i64(t0); 2021 } else { 2022 gen_helper_divu_i64(ret, arg1, arg2); 2023 } 2024 } 2025 2026 void tcg_gen_remu_i64(TCGv_i64 ret, TCGv_i64 arg1, TCGv_i64 arg2) 2027 { 2028 if (tcg_op_supported(INDEX_op_remu, TCG_TYPE_I64, 0)) { 2029 tcg_gen_op3_i64(INDEX_op_remu, ret, arg1, arg2); 2030 } else if (tcg_op_supported(INDEX_op_divu, TCG_TYPE_I64, 0)) { 2031 TCGv_i64 t0 = tcg_temp_ebb_new_i64(); 2032 tcg_gen_op3_i64(INDEX_op_divu, t0, arg1, arg2); 2033 tcg_gen_mul_i64(t0, t0, arg2); 2034 tcg_gen_sub_i64(ret, arg1, t0); 2035 tcg_temp_free_i64(t0); 2036 } else if (tcg_op_supported(INDEX_op_divu2, TCG_TYPE_I64, 0)) { 2037 TCGv_i64 t0 = tcg_temp_ebb_new_i64(); 2038 TCGv_i64 zero = tcg_constant_i64(0); 2039 tcg_gen_op5_i64(INDEX_op_divu2, t0, ret, arg1, zero, arg2); 2040 tcg_temp_free_i64(t0); 2041 } else { 2042 gen_helper_remu_i64(ret, arg1, arg2); 2043 } 2044 } 2045 2046 void tcg_gen_ext8s_i64(TCGv_i64 ret, TCGv_i64 arg) 2047 { 2048 tcg_gen_sextract_i64(ret, arg, 0, 8); 2049 } 2050 2051 void tcg_gen_ext16s_i64(TCGv_i64 ret, TCGv_i64 arg) 2052 { 2053 tcg_gen_sextract_i64(ret, arg, 0, 16); 2054 } 2055 2056 void tcg_gen_ext32s_i64(TCGv_i64 ret, TCGv_i64 arg) 2057 { 2058 tcg_gen_sextract_i64(ret, arg, 0, 32); 2059 } 2060 2061 void tcg_gen_ext8u_i64(TCGv_i64 ret, TCGv_i64 arg) 2062 { 2063 tcg_gen_extract_i64(ret, arg, 0, 8); 2064 } 2065 2066 void tcg_gen_ext16u_i64(TCGv_i64 ret, TCGv_i64 arg) 2067 { 2068 tcg_gen_extract_i64(ret, arg, 0, 16); 2069 } 2070 2071 void tcg_gen_ext32u_i64(TCGv_i64 ret, TCGv_i64 arg) 2072 { 2073 tcg_gen_extract_i64(ret, arg, 0, 32); 2074 } 2075 2076 /* 2077 * bswap16_i64: 16-bit byte swap on the low bits of a 64-bit value. 2078 * 2079 * Byte pattern: xxxxxxxxab -> yyyyyyyyba 2080 * 2081 * With TCG_BSWAP_IZ, x == zero, else undefined. 2082 * With TCG_BSWAP_OZ, y == zero, with TCG_BSWAP_OS y == sign, else undefined. 2083 */ 2084 void tcg_gen_bswap16_i64(TCGv_i64 ret, TCGv_i64 arg, int flags) 2085 { 2086 /* Only one extension flag may be present. */ 2087 tcg_debug_assert(!(flags & TCG_BSWAP_OS) || !(flags & TCG_BSWAP_OZ)); 2088 2089 if (TCG_TARGET_REG_BITS == 32) { 2090 tcg_gen_bswap16_i32(TCGV_LOW(ret), TCGV_LOW(arg), flags); 2091 if (flags & TCG_BSWAP_OS) { 2092 tcg_gen_sari_i32(TCGV_HIGH(ret), TCGV_LOW(ret), 31); 2093 } else { 2094 tcg_gen_movi_i32(TCGV_HIGH(ret), 0); 2095 } 2096 } else if (TCG_TARGET_HAS_bswap16_i64) { 2097 tcg_gen_op3i_i64(INDEX_op_bswap16_i64, ret, arg, flags); 2098 } else { 2099 TCGv_i64 t0 = tcg_temp_ebb_new_i64(); 2100 TCGv_i64 t1 = tcg_temp_ebb_new_i64(); 2101 2102 /* arg = ......ab or xxxxxxab */ 2103 tcg_gen_shri_i64(t0, arg, 8); /* t0 = .......a or .xxxxxxa */ 2104 if (!(flags & TCG_BSWAP_IZ)) { 2105 tcg_gen_ext8u_i64(t0, t0); /* t0 = .......a */ 2106 } 2107 2108 if (flags & TCG_BSWAP_OS) { 2109 tcg_gen_shli_i64(t1, arg, 56); /* t1 = b....... */ 2110 tcg_gen_sari_i64(t1, t1, 48); /* t1 = ssssssb. */ 2111 } else if (flags & TCG_BSWAP_OZ) { 2112 tcg_gen_ext8u_i64(t1, arg); /* t1 = .......b */ 2113 tcg_gen_shli_i64(t1, t1, 8); /* t1 = ......b. */ 2114 } else { 2115 tcg_gen_shli_i64(t1, arg, 8); /* t1 = xxxxxab. */ 2116 } 2117 2118 tcg_gen_or_i64(ret, t0, t1); /* ret = ......ba (OZ) */ 2119 /* ssssssba (OS) */ 2120 /* xxxxxaba (no flag) */ 2121 tcg_temp_free_i64(t0); 2122 tcg_temp_free_i64(t1); 2123 } 2124 } 2125 2126 /* 2127 * bswap32_i64: 32-bit byte swap on the low bits of a 64-bit value. 2128 * 2129 * Byte pattern: xxxxabcd -> yyyydcba 2130 * 2131 * With TCG_BSWAP_IZ, x == zero, else undefined. 2132 * With TCG_BSWAP_OZ, y == zero, with TCG_BSWAP_OS y == sign, else undefined. 2133 */ 2134 void tcg_gen_bswap32_i64(TCGv_i64 ret, TCGv_i64 arg, int flags) 2135 { 2136 /* Only one extension flag may be present. */ 2137 tcg_debug_assert(!(flags & TCG_BSWAP_OS) || !(flags & TCG_BSWAP_OZ)); 2138 2139 if (TCG_TARGET_REG_BITS == 32) { 2140 tcg_gen_bswap32_i32(TCGV_LOW(ret), TCGV_LOW(arg)); 2141 if (flags & TCG_BSWAP_OS) { 2142 tcg_gen_sari_i32(TCGV_HIGH(ret), TCGV_LOW(ret), 31); 2143 } else { 2144 tcg_gen_movi_i32(TCGV_HIGH(ret), 0); 2145 } 2146 } else if (TCG_TARGET_HAS_bswap32_i64) { 2147 tcg_gen_op3i_i64(INDEX_op_bswap32_i64, ret, arg, flags); 2148 } else { 2149 TCGv_i64 t0 = tcg_temp_ebb_new_i64(); 2150 TCGv_i64 t1 = tcg_temp_ebb_new_i64(); 2151 TCGv_i64 t2 = tcg_constant_i64(0x00ff00ff); 2152 2153 /* arg = xxxxabcd */ 2154 tcg_gen_shri_i64(t0, arg, 8); /* t0 = .xxxxabc */ 2155 tcg_gen_and_i64(t1, arg, t2); /* t1 = .....b.d */ 2156 tcg_gen_and_i64(t0, t0, t2); /* t0 = .....a.c */ 2157 tcg_gen_shli_i64(t1, t1, 8); /* t1 = ....b.d. */ 2158 tcg_gen_or_i64(ret, t0, t1); /* ret = ....badc */ 2159 2160 tcg_gen_shli_i64(t1, ret, 48); /* t1 = dc...... */ 2161 tcg_gen_shri_i64(t0, ret, 16); /* t0 = ......ba */ 2162 if (flags & TCG_BSWAP_OS) { 2163 tcg_gen_sari_i64(t1, t1, 32); /* t1 = ssssdc.. */ 2164 } else { 2165 tcg_gen_shri_i64(t1, t1, 32); /* t1 = ....dc.. */ 2166 } 2167 tcg_gen_or_i64(ret, t0, t1); /* ret = ssssdcba (OS) */ 2168 /* ....dcba (else) */ 2169 2170 tcg_temp_free_i64(t0); 2171 tcg_temp_free_i64(t1); 2172 } 2173 } 2174 2175 /* 2176 * bswap64_i64: 64-bit byte swap on a 64-bit value. 2177 * 2178 * Byte pattern: abcdefgh -> hgfedcba 2179 */ 2180 void tcg_gen_bswap64_i64(TCGv_i64 ret, TCGv_i64 arg) 2181 { 2182 if (TCG_TARGET_REG_BITS == 32) { 2183 TCGv_i32 t0, t1; 2184 t0 = tcg_temp_ebb_new_i32(); 2185 t1 = tcg_temp_ebb_new_i32(); 2186 2187 tcg_gen_bswap32_i32(t0, TCGV_LOW(arg)); 2188 tcg_gen_bswap32_i32(t1, TCGV_HIGH(arg)); 2189 tcg_gen_mov_i32(TCGV_LOW(ret), t1); 2190 tcg_gen_mov_i32(TCGV_HIGH(ret), t0); 2191 tcg_temp_free_i32(t0); 2192 tcg_temp_free_i32(t1); 2193 } else if (TCG_TARGET_HAS_bswap64_i64) { 2194 tcg_gen_op3i_i64(INDEX_op_bswap64_i64, ret, arg, 0); 2195 } else { 2196 TCGv_i64 t0 = tcg_temp_ebb_new_i64(); 2197 TCGv_i64 t1 = tcg_temp_ebb_new_i64(); 2198 TCGv_i64 t2 = tcg_temp_ebb_new_i64(); 2199 2200 /* arg = abcdefgh */ 2201 tcg_gen_movi_i64(t2, 0x00ff00ff00ff00ffull); 2202 tcg_gen_shri_i64(t0, arg, 8); /* t0 = .abcdefg */ 2203 tcg_gen_and_i64(t1, arg, t2); /* t1 = .b.d.f.h */ 2204 tcg_gen_and_i64(t0, t0, t2); /* t0 = .a.c.e.g */ 2205 tcg_gen_shli_i64(t1, t1, 8); /* t1 = b.d.f.h. */ 2206 tcg_gen_or_i64(ret, t0, t1); /* ret = badcfehg */ 2207 2208 tcg_gen_movi_i64(t2, 0x0000ffff0000ffffull); 2209 tcg_gen_shri_i64(t0, ret, 16); /* t0 = ..badcfe */ 2210 tcg_gen_and_i64(t1, ret, t2); /* t1 = ..dc..hg */ 2211 tcg_gen_and_i64(t0, t0, t2); /* t0 = ..ba..fe */ 2212 tcg_gen_shli_i64(t1, t1, 16); /* t1 = dc..hg.. */ 2213 tcg_gen_or_i64(ret, t0, t1); /* ret = dcbahgfe */ 2214 2215 tcg_gen_shri_i64(t0, ret, 32); /* t0 = ....dcba */ 2216 tcg_gen_shli_i64(t1, ret, 32); /* t1 = hgfe.... */ 2217 tcg_gen_or_i64(ret, t0, t1); /* ret = hgfedcba */ 2218 2219 tcg_temp_free_i64(t0); 2220 tcg_temp_free_i64(t1); 2221 tcg_temp_free_i64(t2); 2222 } 2223 } 2224 2225 /* 2226 * hswap_i64: Swap 16-bit halfwords within a 64-bit value. 2227 * See also include/qemu/bitops.h, hswap64. 2228 * 2229 * Byte pattern: abcdefgh -> ghefcdab 2230 */ 2231 void tcg_gen_hswap_i64(TCGv_i64 ret, TCGv_i64 arg) 2232 { 2233 uint64_t m = 0x0000ffff0000ffffull; 2234 TCGv_i64 t0 = tcg_temp_ebb_new_i64(); 2235 TCGv_i64 t1 = tcg_temp_ebb_new_i64(); 2236 2237 /* arg = abcdefgh */ 2238 tcg_gen_rotli_i64(t1, arg, 32); /* t1 = efghabcd */ 2239 tcg_gen_andi_i64(t0, t1, m); /* t0 = ..gh..cd */ 2240 tcg_gen_shli_i64(t0, t0, 16); /* t0 = gh..cd.. */ 2241 tcg_gen_shri_i64(t1, t1, 16); /* t1 = ..efghab */ 2242 tcg_gen_andi_i64(t1, t1, m); /* t1 = ..ef..ab */ 2243 tcg_gen_or_i64(ret, t0, t1); /* ret = ghefcdab */ 2244 2245 tcg_temp_free_i64(t0); 2246 tcg_temp_free_i64(t1); 2247 } 2248 2249 /* 2250 * wswap_i64: Swap 32-bit words within a 64-bit value. 2251 * 2252 * Byte pattern: abcdefgh -> efghabcd 2253 */ 2254 void tcg_gen_wswap_i64(TCGv_i64 ret, TCGv_i64 arg) 2255 { 2256 /* Swapping 2 32-bit elements is a rotate. */ 2257 tcg_gen_rotli_i64(ret, arg, 32); 2258 } 2259 2260 void tcg_gen_not_i64(TCGv_i64 ret, TCGv_i64 arg) 2261 { 2262 if (TCG_TARGET_REG_BITS == 32) { 2263 tcg_gen_not_i32(TCGV_LOW(ret), TCGV_LOW(arg)); 2264 tcg_gen_not_i32(TCGV_HIGH(ret), TCGV_HIGH(arg)); 2265 } else if (tcg_op_supported(INDEX_op_not, TCG_TYPE_I64, 0)) { 2266 tcg_gen_op2_i64(INDEX_op_not, ret, arg); 2267 } else { 2268 tcg_gen_xori_i64(ret, arg, -1); 2269 } 2270 } 2271 2272 void tcg_gen_andc_i64(TCGv_i64 ret, TCGv_i64 arg1, TCGv_i64 arg2) 2273 { 2274 if (TCG_TARGET_REG_BITS == 32) { 2275 tcg_gen_andc_i32(TCGV_LOW(ret), TCGV_LOW(arg1), TCGV_LOW(arg2)); 2276 tcg_gen_andc_i32(TCGV_HIGH(ret), TCGV_HIGH(arg1), TCGV_HIGH(arg2)); 2277 } else if (tcg_op_supported(INDEX_op_andc, TCG_TYPE_I64, 0)) { 2278 tcg_gen_op3_i64(INDEX_op_andc, ret, arg1, arg2); 2279 } else { 2280 TCGv_i64 t0 = tcg_temp_ebb_new_i64(); 2281 tcg_gen_not_i64(t0, arg2); 2282 tcg_gen_and_i64(ret, arg1, t0); 2283 tcg_temp_free_i64(t0); 2284 } 2285 } 2286 2287 void tcg_gen_eqv_i64(TCGv_i64 ret, TCGv_i64 arg1, TCGv_i64 arg2) 2288 { 2289 if (TCG_TARGET_REG_BITS == 32) { 2290 tcg_gen_eqv_i32(TCGV_LOW(ret), TCGV_LOW(arg1), TCGV_LOW(arg2)); 2291 tcg_gen_eqv_i32(TCGV_HIGH(ret), TCGV_HIGH(arg1), TCGV_HIGH(arg2)); 2292 } else if (tcg_op_supported(INDEX_op_eqv, TCG_TYPE_I64, 0)) { 2293 tcg_gen_op3_i64(INDEX_op_eqv, ret, arg1, arg2); 2294 } else { 2295 tcg_gen_xor_i64(ret, arg1, arg2); 2296 tcg_gen_not_i64(ret, ret); 2297 } 2298 } 2299 2300 void tcg_gen_nand_i64(TCGv_i64 ret, TCGv_i64 arg1, TCGv_i64 arg2) 2301 { 2302 if (TCG_TARGET_REG_BITS == 32) { 2303 tcg_gen_nand_i32(TCGV_LOW(ret), TCGV_LOW(arg1), TCGV_LOW(arg2)); 2304 tcg_gen_nand_i32(TCGV_HIGH(ret), TCGV_HIGH(arg1), TCGV_HIGH(arg2)); 2305 } else if (tcg_op_supported(INDEX_op_nand, TCG_TYPE_I64, 0)) { 2306 tcg_gen_op3_i64(INDEX_op_nand, ret, arg1, arg2); 2307 } else { 2308 tcg_gen_and_i64(ret, arg1, arg2); 2309 tcg_gen_not_i64(ret, ret); 2310 } 2311 } 2312 2313 void tcg_gen_nor_i64(TCGv_i64 ret, TCGv_i64 arg1, TCGv_i64 arg2) 2314 { 2315 if (TCG_TARGET_REG_BITS == 32) { 2316 tcg_gen_nor_i32(TCGV_LOW(ret), TCGV_LOW(arg1), TCGV_LOW(arg2)); 2317 tcg_gen_nor_i32(TCGV_HIGH(ret), TCGV_HIGH(arg1), TCGV_HIGH(arg2)); 2318 } else if (tcg_op_supported(INDEX_op_nor, TCG_TYPE_I64, 0)) { 2319 tcg_gen_op3_i64(INDEX_op_nor, ret, arg1, arg2); 2320 } else { 2321 tcg_gen_or_i64(ret, arg1, arg2); 2322 tcg_gen_not_i64(ret, ret); 2323 } 2324 } 2325 2326 void tcg_gen_orc_i64(TCGv_i64 ret, TCGv_i64 arg1, TCGv_i64 arg2) 2327 { 2328 if (TCG_TARGET_REG_BITS == 32) { 2329 tcg_gen_orc_i32(TCGV_LOW(ret), TCGV_LOW(arg1), TCGV_LOW(arg2)); 2330 tcg_gen_orc_i32(TCGV_HIGH(ret), TCGV_HIGH(arg1), TCGV_HIGH(arg2)); 2331 } else if (tcg_op_supported(INDEX_op_orc, TCG_TYPE_I64, 0)) { 2332 tcg_gen_op3_i64(INDEX_op_orc, ret, arg1, arg2); 2333 } else { 2334 TCGv_i64 t0 = tcg_temp_ebb_new_i64(); 2335 tcg_gen_not_i64(t0, arg2); 2336 tcg_gen_or_i64(ret, arg1, t0); 2337 tcg_temp_free_i64(t0); 2338 } 2339 } 2340 2341 void tcg_gen_clz_i64(TCGv_i64 ret, TCGv_i64 arg1, TCGv_i64 arg2) 2342 { 2343 if (tcg_op_supported(INDEX_op_clz, TCG_TYPE_I64, 0)) { 2344 tcg_gen_op3_i64(INDEX_op_clz, ret, arg1, arg2); 2345 } else { 2346 gen_helper_clz_i64(ret, arg1, arg2); 2347 } 2348 } 2349 2350 void tcg_gen_clzi_i64(TCGv_i64 ret, TCGv_i64 arg1, uint64_t arg2) 2351 { 2352 if (TCG_TARGET_REG_BITS == 32 2353 && arg2 <= 0xffffffffu 2354 && tcg_op_supported(INDEX_op_clz, TCG_TYPE_I32, 0)) { 2355 TCGv_i32 t = tcg_temp_ebb_new_i32(); 2356 tcg_gen_clzi_i32(t, TCGV_LOW(arg1), arg2 - 32); 2357 tcg_gen_addi_i32(t, t, 32); 2358 tcg_gen_clz_i32(TCGV_LOW(ret), TCGV_HIGH(arg1), t); 2359 tcg_gen_movi_i32(TCGV_HIGH(ret), 0); 2360 tcg_temp_free_i32(t); 2361 } else { 2362 tcg_gen_clz_i64(ret, arg1, tcg_constant_i64(arg2)); 2363 } 2364 } 2365 2366 void tcg_gen_ctz_i64(TCGv_i64 ret, TCGv_i64 arg1, TCGv_i64 arg2) 2367 { 2368 TCGv_i64 z, t; 2369 2370 if (tcg_op_supported(INDEX_op_ctz, TCG_TYPE_I64, 0)) { 2371 tcg_gen_op3_i64(INDEX_op_ctz, ret, arg1, arg2); 2372 return; 2373 } 2374 if (tcg_op_supported(INDEX_op_ctpop, TCG_TYPE_I64, 0)) { 2375 t = tcg_temp_ebb_new_i64(); 2376 tcg_gen_subi_i64(t, arg1, 1); 2377 tcg_gen_andc_i64(t, t, arg1); 2378 tcg_gen_ctpop_i64(t, t); 2379 } else if (tcg_op_supported(INDEX_op_clz, TCG_TYPE_I64, 0)) { 2380 t = tcg_temp_ebb_new_i64(); 2381 tcg_gen_neg_i64(t, arg1); 2382 tcg_gen_and_i64(t, t, arg1); 2383 tcg_gen_clzi_i64(t, t, 64); 2384 tcg_gen_xori_i64(t, t, 63); 2385 } else { 2386 gen_helper_ctz_i64(ret, arg1, arg2); 2387 return; 2388 } 2389 2390 z = tcg_constant_i64(0); 2391 tcg_gen_movcond_i64(TCG_COND_EQ, ret, arg1, z, arg2, t); 2392 tcg_temp_free_i64(t); 2393 } 2394 2395 void tcg_gen_ctzi_i64(TCGv_i64 ret, TCGv_i64 arg1, uint64_t arg2) 2396 { 2397 if (TCG_TARGET_REG_BITS == 32 2398 && arg2 <= 0xffffffffu 2399 && tcg_op_supported(INDEX_op_ctz, TCG_TYPE_I32, 0)) { 2400 TCGv_i32 t32 = tcg_temp_ebb_new_i32(); 2401 tcg_gen_ctzi_i32(t32, TCGV_HIGH(arg1), arg2 - 32); 2402 tcg_gen_addi_i32(t32, t32, 32); 2403 tcg_gen_ctz_i32(TCGV_LOW(ret), TCGV_LOW(arg1), t32); 2404 tcg_gen_movi_i32(TCGV_HIGH(ret), 0); 2405 tcg_temp_free_i32(t32); 2406 } else if (arg2 == 64 2407 && !tcg_op_supported(INDEX_op_ctz, TCG_TYPE_I64, 0) 2408 && tcg_op_supported(INDEX_op_ctpop, TCG_TYPE_I64, 0)) { 2409 /* This equivalence has the advantage of not requiring a fixup. */ 2410 TCGv_i64 t = tcg_temp_ebb_new_i64(); 2411 tcg_gen_subi_i64(t, arg1, 1); 2412 tcg_gen_andc_i64(t, t, arg1); 2413 tcg_gen_ctpop_i64(ret, t); 2414 tcg_temp_free_i64(t); 2415 } else { 2416 tcg_gen_ctz_i64(ret, arg1, tcg_constant_i64(arg2)); 2417 } 2418 } 2419 2420 void tcg_gen_clrsb_i64(TCGv_i64 ret, TCGv_i64 arg) 2421 { 2422 if (tcg_op_supported(INDEX_op_clz, TCG_TYPE_I64, 0)) { 2423 TCGv_i64 t = tcg_temp_ebb_new_i64(); 2424 tcg_gen_sari_i64(t, arg, 63); 2425 tcg_gen_xor_i64(t, t, arg); 2426 tcg_gen_clzi_i64(t, t, 64); 2427 tcg_gen_subi_i64(ret, t, 1); 2428 tcg_temp_free_i64(t); 2429 } else { 2430 gen_helper_clrsb_i64(ret, arg); 2431 } 2432 } 2433 2434 void tcg_gen_ctpop_i64(TCGv_i64 ret, TCGv_i64 arg1) 2435 { 2436 if (TCG_TARGET_REG_BITS == 64) { 2437 if (tcg_op_supported(INDEX_op_ctpop, TCG_TYPE_I64, 0)) { 2438 tcg_gen_op2_i64(INDEX_op_ctpop, ret, arg1); 2439 return; 2440 } 2441 } else { 2442 if (tcg_op_supported(INDEX_op_ctpop, TCG_TYPE_I32, 0)) { 2443 tcg_gen_ctpop_i32(TCGV_HIGH(ret), TCGV_HIGH(arg1)); 2444 tcg_gen_ctpop_i32(TCGV_LOW(ret), TCGV_LOW(arg1)); 2445 tcg_gen_add_i32(TCGV_LOW(ret), TCGV_LOW(ret), TCGV_HIGH(ret)); 2446 tcg_gen_movi_i32(TCGV_HIGH(ret), 0); 2447 return; 2448 } 2449 } 2450 gen_helper_ctpop_i64(ret, arg1); 2451 } 2452 2453 void tcg_gen_rotl_i64(TCGv_i64 ret, TCGv_i64 arg1, TCGv_i64 arg2) 2454 { 2455 if (tcg_op_supported(INDEX_op_rotl, TCG_TYPE_I64, 0)) { 2456 tcg_gen_op3_i64(INDEX_op_rotl, ret, arg1, arg2); 2457 } else if (tcg_op_supported(INDEX_op_rotl, TCG_TYPE_I64, 0)) { 2458 TCGv_i64 t0 = tcg_temp_ebb_new_i64(); 2459 tcg_gen_neg_i64(t0, arg2); 2460 tcg_gen_op3_i64(INDEX_op_rotr, ret, arg1, t0); 2461 tcg_temp_free_i64(t0); 2462 } else { 2463 TCGv_i64 t0 = tcg_temp_ebb_new_i64(); 2464 TCGv_i64 t1 = tcg_temp_ebb_new_i64(); 2465 tcg_gen_shl_i64(t0, arg1, arg2); 2466 tcg_gen_neg_i64(t1, arg2); 2467 tcg_gen_shr_i64(t1, arg1, t1); 2468 tcg_gen_or_i64(ret, t0, t1); 2469 tcg_temp_free_i64(t0); 2470 tcg_temp_free_i64(t1); 2471 } 2472 } 2473 2474 void tcg_gen_rotli_i64(TCGv_i64 ret, TCGv_i64 arg1, int64_t arg2) 2475 { 2476 tcg_debug_assert(arg2 >= 0 && arg2 < 64); 2477 /* some cases can be optimized here */ 2478 if (arg2 == 0) { 2479 tcg_gen_mov_i64(ret, arg1); 2480 } else if (tcg_op_supported(INDEX_op_rotl, TCG_TYPE_I64, 0)) { 2481 TCGv_i64 t0 = tcg_constant_i64(arg2); 2482 tcg_gen_op3_i64(INDEX_op_rotl, ret, arg1, t0); 2483 } else if (tcg_op_supported(INDEX_op_rotr, TCG_TYPE_I64, 0)) { 2484 TCGv_i64 t0 = tcg_constant_i64(64 - arg2); 2485 tcg_gen_op3_i64(INDEX_op_rotr, ret, arg1, t0); 2486 } else { 2487 TCGv_i64 t0 = tcg_temp_ebb_new_i64(); 2488 TCGv_i64 t1 = tcg_temp_ebb_new_i64(); 2489 tcg_gen_shli_i64(t0, arg1, arg2); 2490 tcg_gen_shri_i64(t1, arg1, 64 - arg2); 2491 tcg_gen_or_i64(ret, t0, t1); 2492 tcg_temp_free_i64(t0); 2493 tcg_temp_free_i64(t1); 2494 } 2495 } 2496 2497 void tcg_gen_rotr_i64(TCGv_i64 ret, TCGv_i64 arg1, TCGv_i64 arg2) 2498 { 2499 if (tcg_op_supported(INDEX_op_rotr, TCG_TYPE_I64, 0)) { 2500 tcg_gen_op3_i64(INDEX_op_rotr, ret, arg1, arg2); 2501 } else if (tcg_op_supported(INDEX_op_rotl, TCG_TYPE_I64, 0)) { 2502 TCGv_i64 t0 = tcg_temp_ebb_new_i64(); 2503 tcg_gen_neg_i64(t0, arg2); 2504 tcg_gen_op3_i64(INDEX_op_rotl, ret, arg1, t0); 2505 tcg_temp_free_i64(t0); 2506 } else { 2507 TCGv_i64 t0 = tcg_temp_ebb_new_i64(); 2508 TCGv_i64 t1 = tcg_temp_ebb_new_i64(); 2509 tcg_gen_shr_i64(t0, arg1, arg2); 2510 tcg_gen_neg_i64(t1, arg2); 2511 tcg_gen_shl_i64(t1, arg1, t1); 2512 tcg_gen_or_i64(ret, t0, t1); 2513 tcg_temp_free_i64(t0); 2514 tcg_temp_free_i64(t1); 2515 } 2516 } 2517 2518 void tcg_gen_rotri_i64(TCGv_i64 ret, TCGv_i64 arg1, int64_t arg2) 2519 { 2520 tcg_debug_assert(arg2 >= 0 && arg2 < 64); 2521 tcg_gen_rotli_i64(ret, arg1, -arg2 & 63); 2522 } 2523 2524 void tcg_gen_deposit_i64(TCGv_i64 ret, TCGv_i64 arg1, TCGv_i64 arg2, 2525 unsigned int ofs, unsigned int len) 2526 { 2527 uint64_t mask; 2528 TCGv_i64 t1; 2529 2530 tcg_debug_assert(ofs < 64); 2531 tcg_debug_assert(len > 0); 2532 tcg_debug_assert(len <= 64); 2533 tcg_debug_assert(ofs + len <= 64); 2534 2535 if (len == 64) { 2536 tcg_gen_mov_i64(ret, arg2); 2537 return; 2538 } 2539 2540 if (TCG_TARGET_REG_BITS == 64) { 2541 if (TCG_TARGET_deposit_valid(TCG_TYPE_I64, ofs, len)) { 2542 tcg_gen_op5ii_i64(INDEX_op_deposit_i64, ret, arg1, arg2, ofs, len); 2543 return; 2544 } 2545 } else { 2546 if (ofs >= 32) { 2547 tcg_gen_deposit_i32(TCGV_HIGH(ret), TCGV_HIGH(arg1), 2548 TCGV_LOW(arg2), ofs - 32, len); 2549 tcg_gen_mov_i32(TCGV_LOW(ret), TCGV_LOW(arg1)); 2550 return; 2551 } 2552 if (ofs + len <= 32) { 2553 tcg_gen_deposit_i32(TCGV_LOW(ret), TCGV_LOW(arg1), 2554 TCGV_LOW(arg2), ofs, len); 2555 tcg_gen_mov_i32(TCGV_HIGH(ret), TCGV_HIGH(arg1)); 2556 return; 2557 } 2558 } 2559 2560 t1 = tcg_temp_ebb_new_i64(); 2561 2562 if (TCG_TARGET_HAS_extract2_i64) { 2563 if (ofs + len == 64) { 2564 tcg_gen_shli_i64(t1, arg1, len); 2565 tcg_gen_extract2_i64(ret, t1, arg2, len); 2566 goto done; 2567 } 2568 if (ofs == 0) { 2569 tcg_gen_extract2_i64(ret, arg1, arg2, len); 2570 tcg_gen_rotli_i64(ret, ret, len); 2571 goto done; 2572 } 2573 } 2574 2575 mask = (1ull << len) - 1; 2576 if (ofs + len < 64) { 2577 tcg_gen_andi_i64(t1, arg2, mask); 2578 tcg_gen_shli_i64(t1, t1, ofs); 2579 } else { 2580 tcg_gen_shli_i64(t1, arg2, ofs); 2581 } 2582 tcg_gen_andi_i64(ret, arg1, ~(mask << ofs)); 2583 tcg_gen_or_i64(ret, ret, t1); 2584 done: 2585 tcg_temp_free_i64(t1); 2586 } 2587 2588 void tcg_gen_deposit_z_i64(TCGv_i64 ret, TCGv_i64 arg, 2589 unsigned int ofs, unsigned int len) 2590 { 2591 tcg_debug_assert(ofs < 64); 2592 tcg_debug_assert(len > 0); 2593 tcg_debug_assert(len <= 64); 2594 tcg_debug_assert(ofs + len <= 64); 2595 2596 if (ofs + len == 64) { 2597 tcg_gen_shli_i64(ret, arg, ofs); 2598 } else if (ofs == 0) { 2599 tcg_gen_andi_i64(ret, arg, (1ull << len) - 1); 2600 } else if (TCG_TARGET_REG_BITS == 64 && 2601 TCG_TARGET_deposit_valid(TCG_TYPE_I64, ofs, len)) { 2602 TCGv_i64 zero = tcg_constant_i64(0); 2603 tcg_gen_op5ii_i64(INDEX_op_deposit_i64, ret, zero, arg, ofs, len); 2604 } else { 2605 if (TCG_TARGET_REG_BITS == 32) { 2606 if (ofs >= 32) { 2607 tcg_gen_deposit_z_i32(TCGV_HIGH(ret), TCGV_LOW(arg), 2608 ofs - 32, len); 2609 tcg_gen_movi_i32(TCGV_LOW(ret), 0); 2610 return; 2611 } 2612 if (ofs + len <= 32) { 2613 tcg_gen_deposit_z_i32(TCGV_LOW(ret), TCGV_LOW(arg), ofs, len); 2614 tcg_gen_movi_i32(TCGV_HIGH(ret), 0); 2615 return; 2616 } 2617 } 2618 /* 2619 * To help two-operand hosts we prefer to zero-extend first, 2620 * which allows ARG to stay live. 2621 */ 2622 if (TCG_TARGET_extract_valid(TCG_TYPE_I64, 0, len)) { 2623 tcg_gen_extract_i64(ret, arg, 0, len); 2624 tcg_gen_shli_i64(ret, ret, ofs); 2625 return; 2626 } 2627 /* Otherwise prefer zero-extension over AND for code size. */ 2628 if (TCG_TARGET_extract_valid(TCG_TYPE_I64, 0, ofs + len)) { 2629 tcg_gen_shli_i64(ret, arg, ofs); 2630 tcg_gen_extract_i64(ret, ret, 0, ofs + len); 2631 return; 2632 } 2633 tcg_gen_andi_i64(ret, arg, (1ull << len) - 1); 2634 tcg_gen_shli_i64(ret, ret, ofs); 2635 } 2636 } 2637 2638 void tcg_gen_extract_i64(TCGv_i64 ret, TCGv_i64 arg, 2639 unsigned int ofs, unsigned int len) 2640 { 2641 tcg_debug_assert(ofs < 64); 2642 tcg_debug_assert(len > 0); 2643 tcg_debug_assert(len <= 64); 2644 tcg_debug_assert(ofs + len <= 64); 2645 2646 /* Canonicalize certain special cases, even if extract is supported. */ 2647 if (ofs + len == 64) { 2648 tcg_gen_shri_i64(ret, arg, 64 - len); 2649 return; 2650 } 2651 2652 if (TCG_TARGET_REG_BITS == 32) { 2653 /* Look for a 32-bit extract within one of the two words. */ 2654 if (ofs >= 32) { 2655 tcg_gen_extract_i32(TCGV_LOW(ret), TCGV_HIGH(arg), ofs - 32, len); 2656 tcg_gen_movi_i32(TCGV_HIGH(ret), 0); 2657 return; 2658 } 2659 if (ofs + len <= 32) { 2660 tcg_gen_extract_i32(TCGV_LOW(ret), TCGV_LOW(arg), ofs, len); 2661 tcg_gen_movi_i32(TCGV_HIGH(ret), 0); 2662 return; 2663 } 2664 2665 /* The field is split across two words. */ 2666 tcg_gen_extract2_i32(TCGV_LOW(ret), TCGV_LOW(arg), 2667 TCGV_HIGH(arg), ofs); 2668 if (len <= 32) { 2669 tcg_gen_extract_i32(TCGV_LOW(ret), TCGV_LOW(ret), 0, len); 2670 tcg_gen_movi_i32(TCGV_HIGH(ret), 0); 2671 } else { 2672 tcg_gen_extract_i32(TCGV_HIGH(ret), TCGV_HIGH(arg), 2673 ofs, len - 32); 2674 } 2675 return; 2676 } 2677 2678 if (TCG_TARGET_extract_valid(TCG_TYPE_I64, ofs, len)) { 2679 tcg_gen_op4ii_i64(INDEX_op_extract_i64, ret, arg, ofs, len); 2680 return; 2681 } 2682 if (ofs == 0) { 2683 tcg_gen_andi_i64(ret, arg, (1ull << len) - 1); 2684 return; 2685 } 2686 2687 /* Assume that zero-extension, if available, is cheaper than a shift. */ 2688 if (TCG_TARGET_extract_valid(TCG_TYPE_I64, 0, ofs + len)) { 2689 tcg_gen_op4ii_i64(INDEX_op_extract_i64, ret, arg, 0, ofs + len); 2690 tcg_gen_shri_i64(ret, ret, ofs); 2691 return; 2692 } 2693 2694 /* ??? Ideally we'd know what values are available for immediate AND. 2695 Assume that 8 bits are available, plus the special cases of 16 and 32, 2696 so that we get ext8u, ext16u, and ext32u. */ 2697 switch (len) { 2698 case 1 ... 8: case 16: case 32: 2699 tcg_gen_shri_i64(ret, arg, ofs); 2700 tcg_gen_andi_i64(ret, ret, (1ull << len) - 1); 2701 break; 2702 default: 2703 tcg_gen_shli_i64(ret, arg, 64 - len - ofs); 2704 tcg_gen_shri_i64(ret, ret, 64 - len); 2705 break; 2706 } 2707 } 2708 2709 void tcg_gen_sextract_i64(TCGv_i64 ret, TCGv_i64 arg, 2710 unsigned int ofs, unsigned int len) 2711 { 2712 tcg_debug_assert(ofs < 64); 2713 tcg_debug_assert(len > 0); 2714 tcg_debug_assert(len <= 64); 2715 tcg_debug_assert(ofs + len <= 64); 2716 2717 /* Canonicalize certain special cases, even if sextract is supported. */ 2718 if (ofs + len == 64) { 2719 tcg_gen_sari_i64(ret, arg, 64 - len); 2720 return; 2721 } 2722 2723 if (TCG_TARGET_REG_BITS == 32) { 2724 /* Look for a 32-bit extract within one of the two words. */ 2725 if (ofs >= 32) { 2726 tcg_gen_sextract_i32(TCGV_LOW(ret), TCGV_HIGH(arg), ofs - 32, len); 2727 } else if (ofs + len <= 32) { 2728 tcg_gen_sextract_i32(TCGV_LOW(ret), TCGV_LOW(arg), ofs, len); 2729 } else if (ofs == 0) { 2730 tcg_gen_mov_i32(TCGV_LOW(ret), TCGV_LOW(arg)); 2731 tcg_gen_sextract_i32(TCGV_HIGH(ret), TCGV_HIGH(arg), 0, len - 32); 2732 return; 2733 } else if (len > 32) { 2734 TCGv_i32 t = tcg_temp_ebb_new_i32(); 2735 /* Extract the bits for the high word normally. */ 2736 tcg_gen_sextract_i32(t, TCGV_HIGH(arg), ofs + 32, len - 32); 2737 /* Shift the field down for the low part. */ 2738 tcg_gen_shri_i64(ret, arg, ofs); 2739 /* Overwrite the shift into the high part. */ 2740 tcg_gen_mov_i32(TCGV_HIGH(ret), t); 2741 tcg_temp_free_i32(t); 2742 return; 2743 } else { 2744 /* Shift the field down for the low part, such that the 2745 field sits at the MSB. */ 2746 tcg_gen_shri_i64(ret, arg, ofs + len - 32); 2747 /* Shift the field down from the MSB, sign extending. */ 2748 tcg_gen_sari_i32(TCGV_LOW(ret), TCGV_LOW(ret), 32 - len); 2749 } 2750 /* Sign-extend the field from 32 bits. */ 2751 tcg_gen_sari_i32(TCGV_HIGH(ret), TCGV_LOW(ret), 31); 2752 return; 2753 } 2754 2755 if (TCG_TARGET_sextract_valid(TCG_TYPE_I64, ofs, len)) { 2756 tcg_gen_op4ii_i64(INDEX_op_sextract_i64, ret, arg, ofs, len); 2757 return; 2758 } 2759 2760 /* Assume that sign-extension, if available, is cheaper than a shift. */ 2761 if (TCG_TARGET_sextract_valid(TCG_TYPE_I64, 0, ofs + len)) { 2762 tcg_gen_op4ii_i64(INDEX_op_sextract_i64, ret, arg, 0, ofs + len); 2763 tcg_gen_sari_i64(ret, ret, ofs); 2764 return; 2765 } 2766 if (TCG_TARGET_sextract_valid(TCG_TYPE_I64, 0, len)) { 2767 tcg_gen_shri_i64(ret, arg, ofs); 2768 tcg_gen_op4ii_i64(INDEX_op_sextract_i64, ret, ret, 0, len); 2769 return; 2770 } 2771 2772 tcg_gen_shli_i64(ret, arg, 64 - len - ofs); 2773 tcg_gen_sari_i64(ret, ret, 64 - len); 2774 } 2775 2776 /* 2777 * Extract 64 bits from a 128-bit input, ah:al, starting from ofs. 2778 * Unlike tcg_gen_extract_i64 above, len is fixed at 64. 2779 */ 2780 void tcg_gen_extract2_i64(TCGv_i64 ret, TCGv_i64 al, TCGv_i64 ah, 2781 unsigned int ofs) 2782 { 2783 tcg_debug_assert(ofs <= 64); 2784 if (ofs == 0) { 2785 tcg_gen_mov_i64(ret, al); 2786 } else if (ofs == 64) { 2787 tcg_gen_mov_i64(ret, ah); 2788 } else if (al == ah) { 2789 tcg_gen_rotri_i64(ret, al, ofs); 2790 } else if (TCG_TARGET_HAS_extract2_i64) { 2791 tcg_gen_op4i_i64(INDEX_op_extract2_i64, ret, al, ah, ofs); 2792 } else { 2793 TCGv_i64 t0 = tcg_temp_ebb_new_i64(); 2794 tcg_gen_shri_i64(t0, al, ofs); 2795 tcg_gen_deposit_i64(ret, t0, ah, 64 - ofs, ofs); 2796 tcg_temp_free_i64(t0); 2797 } 2798 } 2799 2800 void tcg_gen_movcond_i64(TCGCond cond, TCGv_i64 ret, TCGv_i64 c1, 2801 TCGv_i64 c2, TCGv_i64 v1, TCGv_i64 v2) 2802 { 2803 if (cond == TCG_COND_ALWAYS) { 2804 tcg_gen_mov_i64(ret, v1); 2805 } else if (cond == TCG_COND_NEVER) { 2806 tcg_gen_mov_i64(ret, v2); 2807 } else if (TCG_TARGET_REG_BITS == 64) { 2808 tcg_gen_op6i_i64(INDEX_op_movcond_i64, ret, c1, c2, v1, v2, cond); 2809 } else { 2810 TCGv_i32 t0 = tcg_temp_ebb_new_i32(); 2811 TCGv_i32 zero = tcg_constant_i32(0); 2812 2813 tcg_gen_op6i_i32(INDEX_op_setcond2_i32, t0, 2814 TCGV_LOW(c1), TCGV_HIGH(c1), 2815 TCGV_LOW(c2), TCGV_HIGH(c2), cond); 2816 2817 tcg_gen_movcond_i32(TCG_COND_NE, TCGV_LOW(ret), t0, zero, 2818 TCGV_LOW(v1), TCGV_LOW(v2)); 2819 tcg_gen_movcond_i32(TCG_COND_NE, TCGV_HIGH(ret), t0, zero, 2820 TCGV_HIGH(v1), TCGV_HIGH(v2)); 2821 2822 tcg_temp_free_i32(t0); 2823 } 2824 } 2825 2826 void tcg_gen_add2_i64(TCGv_i64 rl, TCGv_i64 rh, TCGv_i64 al, 2827 TCGv_i64 ah, TCGv_i64 bl, TCGv_i64 bh) 2828 { 2829 if (TCG_TARGET_HAS_add2_i64) { 2830 tcg_gen_op6_i64(INDEX_op_add2_i64, rl, rh, al, ah, bl, bh); 2831 } else { 2832 TCGv_i64 t0 = tcg_temp_ebb_new_i64(); 2833 TCGv_i64 t1 = tcg_temp_ebb_new_i64(); 2834 tcg_gen_add_i64(t0, al, bl); 2835 tcg_gen_setcond_i64(TCG_COND_LTU, t1, t0, al); 2836 tcg_gen_add_i64(rh, ah, bh); 2837 tcg_gen_add_i64(rh, rh, t1); 2838 tcg_gen_mov_i64(rl, t0); 2839 tcg_temp_free_i64(t0); 2840 tcg_temp_free_i64(t1); 2841 } 2842 } 2843 2844 void tcg_gen_sub2_i64(TCGv_i64 rl, TCGv_i64 rh, TCGv_i64 al, 2845 TCGv_i64 ah, TCGv_i64 bl, TCGv_i64 bh) 2846 { 2847 if (TCG_TARGET_HAS_sub2_i64) { 2848 tcg_gen_op6_i64(INDEX_op_sub2_i64, rl, rh, al, ah, bl, bh); 2849 } else { 2850 TCGv_i64 t0 = tcg_temp_ebb_new_i64(); 2851 TCGv_i64 t1 = tcg_temp_ebb_new_i64(); 2852 tcg_gen_sub_i64(t0, al, bl); 2853 tcg_gen_setcond_i64(TCG_COND_LTU, t1, al, bl); 2854 tcg_gen_sub_i64(rh, ah, bh); 2855 tcg_gen_sub_i64(rh, rh, t1); 2856 tcg_gen_mov_i64(rl, t0); 2857 tcg_temp_free_i64(t0); 2858 tcg_temp_free_i64(t1); 2859 } 2860 } 2861 2862 void tcg_gen_mulu2_i64(TCGv_i64 rl, TCGv_i64 rh, TCGv_i64 arg1, TCGv_i64 arg2) 2863 { 2864 if (TCG_TARGET_HAS_mulu2_i64) { 2865 tcg_gen_op4_i64(INDEX_op_mulu2_i64, rl, rh, arg1, arg2); 2866 } else if (tcg_op_supported(INDEX_op_muluh, TCG_TYPE_I64, 0)) { 2867 TCGv_i64 t = tcg_temp_ebb_new_i64(); 2868 tcg_gen_op3_i64(INDEX_op_mul, t, arg1, arg2); 2869 tcg_gen_op3_i64(INDEX_op_muluh, rh, arg1, arg2); 2870 tcg_gen_mov_i64(rl, t); 2871 tcg_temp_free_i64(t); 2872 } else { 2873 TCGv_i64 t0 = tcg_temp_ebb_new_i64(); 2874 tcg_gen_mul_i64(t0, arg1, arg2); 2875 gen_helper_muluh_i64(rh, arg1, arg2); 2876 tcg_gen_mov_i64(rl, t0); 2877 tcg_temp_free_i64(t0); 2878 } 2879 } 2880 2881 void tcg_gen_muls2_i64(TCGv_i64 rl, TCGv_i64 rh, TCGv_i64 arg1, TCGv_i64 arg2) 2882 { 2883 if (tcg_op_supported(INDEX_op_muls2, TCG_TYPE_I64, 0)) { 2884 tcg_gen_op4_i64(INDEX_op_muls2, rl, rh, arg1, arg2); 2885 } else if (tcg_op_supported(INDEX_op_mulsh, TCG_TYPE_I64, 0)) { 2886 TCGv_i64 t = tcg_temp_ebb_new_i64(); 2887 tcg_gen_op3_i64(INDEX_op_mul, t, arg1, arg2); 2888 tcg_gen_op3_i64(INDEX_op_mulsh, rh, arg1, arg2); 2889 tcg_gen_mov_i64(rl, t); 2890 tcg_temp_free_i64(t); 2891 } else if (TCG_TARGET_HAS_mulu2_i64 || 2892 tcg_op_supported(INDEX_op_muluh, TCG_TYPE_I64, 0)) { 2893 TCGv_i64 t0 = tcg_temp_ebb_new_i64(); 2894 TCGv_i64 t1 = tcg_temp_ebb_new_i64(); 2895 TCGv_i64 t2 = tcg_temp_ebb_new_i64(); 2896 TCGv_i64 t3 = tcg_temp_ebb_new_i64(); 2897 tcg_gen_mulu2_i64(t0, t1, arg1, arg2); 2898 /* Adjust for negative inputs. */ 2899 tcg_gen_sari_i64(t2, arg1, 63); 2900 tcg_gen_sari_i64(t3, arg2, 63); 2901 tcg_gen_and_i64(t2, t2, arg2); 2902 tcg_gen_and_i64(t3, t3, arg1); 2903 tcg_gen_sub_i64(rh, t1, t2); 2904 tcg_gen_sub_i64(rh, rh, t3); 2905 tcg_gen_mov_i64(rl, t0); 2906 tcg_temp_free_i64(t0); 2907 tcg_temp_free_i64(t1); 2908 tcg_temp_free_i64(t2); 2909 tcg_temp_free_i64(t3); 2910 } else { 2911 TCGv_i64 t0 = tcg_temp_ebb_new_i64(); 2912 tcg_gen_mul_i64(t0, arg1, arg2); 2913 gen_helper_mulsh_i64(rh, arg1, arg2); 2914 tcg_gen_mov_i64(rl, t0); 2915 tcg_temp_free_i64(t0); 2916 } 2917 } 2918 2919 void tcg_gen_mulsu2_i64(TCGv_i64 rl, TCGv_i64 rh, TCGv_i64 arg1, TCGv_i64 arg2) 2920 { 2921 TCGv_i64 t0 = tcg_temp_ebb_new_i64(); 2922 TCGv_i64 t1 = tcg_temp_ebb_new_i64(); 2923 TCGv_i64 t2 = tcg_temp_ebb_new_i64(); 2924 tcg_gen_mulu2_i64(t0, t1, arg1, arg2); 2925 /* Adjust for negative input for the signed arg1. */ 2926 tcg_gen_sari_i64(t2, arg1, 63); 2927 tcg_gen_and_i64(t2, t2, arg2); 2928 tcg_gen_sub_i64(rh, t1, t2); 2929 tcg_gen_mov_i64(rl, t0); 2930 tcg_temp_free_i64(t0); 2931 tcg_temp_free_i64(t1); 2932 tcg_temp_free_i64(t2); 2933 } 2934 2935 void tcg_gen_smin_i64(TCGv_i64 ret, TCGv_i64 a, TCGv_i64 b) 2936 { 2937 tcg_gen_movcond_i64(TCG_COND_LT, ret, a, b, a, b); 2938 } 2939 2940 void tcg_gen_umin_i64(TCGv_i64 ret, TCGv_i64 a, TCGv_i64 b) 2941 { 2942 tcg_gen_movcond_i64(TCG_COND_LTU, ret, a, b, a, b); 2943 } 2944 2945 void tcg_gen_smax_i64(TCGv_i64 ret, TCGv_i64 a, TCGv_i64 b) 2946 { 2947 tcg_gen_movcond_i64(TCG_COND_LT, ret, a, b, b, a); 2948 } 2949 2950 void tcg_gen_umax_i64(TCGv_i64 ret, TCGv_i64 a, TCGv_i64 b) 2951 { 2952 tcg_gen_movcond_i64(TCG_COND_LTU, ret, a, b, b, a); 2953 } 2954 2955 void tcg_gen_abs_i64(TCGv_i64 ret, TCGv_i64 a) 2956 { 2957 TCGv_i64 t = tcg_temp_ebb_new_i64(); 2958 2959 tcg_gen_sari_i64(t, a, 63); 2960 tcg_gen_xor_i64(ret, a, t); 2961 tcg_gen_sub_i64(ret, ret, t); 2962 tcg_temp_free_i64(t); 2963 } 2964 2965 /* Size changing operations. */ 2966 2967 void tcg_gen_extrl_i64_i32(TCGv_i32 ret, TCGv_i64 arg) 2968 { 2969 if (TCG_TARGET_REG_BITS == 32) { 2970 tcg_gen_mov_i32(ret, TCGV_LOW(arg)); 2971 } else if (TCG_TARGET_HAS_extr_i64_i32) { 2972 tcg_gen_op2(INDEX_op_extrl_i64_i32, TCG_TYPE_I32, 2973 tcgv_i32_arg(ret), tcgv_i64_arg(arg)); 2974 } else { 2975 tcg_gen_mov_i32(ret, (TCGv_i32)arg); 2976 } 2977 } 2978 2979 void tcg_gen_extrh_i64_i32(TCGv_i32 ret, TCGv_i64 arg) 2980 { 2981 if (TCG_TARGET_REG_BITS == 32) { 2982 tcg_gen_mov_i32(ret, TCGV_HIGH(arg)); 2983 } else if (TCG_TARGET_HAS_extr_i64_i32) { 2984 tcg_gen_op2(INDEX_op_extrh_i64_i32, TCG_TYPE_I32, 2985 tcgv_i32_arg(ret), tcgv_i64_arg(arg)); 2986 } else { 2987 TCGv_i64 t = tcg_temp_ebb_new_i64(); 2988 tcg_gen_shri_i64(t, arg, 32); 2989 tcg_gen_mov_i32(ret, (TCGv_i32)t); 2990 tcg_temp_free_i64(t); 2991 } 2992 } 2993 2994 void tcg_gen_extu_i32_i64(TCGv_i64 ret, TCGv_i32 arg) 2995 { 2996 if (TCG_TARGET_REG_BITS == 32) { 2997 tcg_gen_mov_i32(TCGV_LOW(ret), arg); 2998 tcg_gen_movi_i32(TCGV_HIGH(ret), 0); 2999 } else { 3000 tcg_gen_op2(INDEX_op_extu_i32_i64, TCG_TYPE_I64, 3001 tcgv_i64_arg(ret), tcgv_i32_arg(arg)); 3002 } 3003 } 3004 3005 void tcg_gen_ext_i32_i64(TCGv_i64 ret, TCGv_i32 arg) 3006 { 3007 if (TCG_TARGET_REG_BITS == 32) { 3008 tcg_gen_mov_i32(TCGV_LOW(ret), arg); 3009 tcg_gen_sari_i32(TCGV_HIGH(ret), TCGV_LOW(ret), 31); 3010 } else { 3011 tcg_gen_op2(INDEX_op_ext_i32_i64, TCG_TYPE_I64, 3012 tcgv_i64_arg(ret), tcgv_i32_arg(arg)); 3013 } 3014 } 3015 3016 void tcg_gen_concat_i32_i64(TCGv_i64 dest, TCGv_i32 low, TCGv_i32 high) 3017 { 3018 TCGv_i64 tmp; 3019 3020 if (TCG_TARGET_REG_BITS == 32) { 3021 tcg_gen_mov_i32(TCGV_LOW(dest), low); 3022 tcg_gen_mov_i32(TCGV_HIGH(dest), high); 3023 return; 3024 } 3025 3026 tmp = tcg_temp_ebb_new_i64(); 3027 /* These extensions are only needed for type correctness. 3028 We may be able to do better given target specific information. */ 3029 tcg_gen_extu_i32_i64(tmp, high); 3030 tcg_gen_extu_i32_i64(dest, low); 3031 /* If deposit is available, use it. Otherwise use the extra 3032 knowledge that we have of the zero-extensions above. */ 3033 if (TCG_TARGET_deposit_valid(TCG_TYPE_I64, 32, 32)) { 3034 tcg_gen_deposit_i64(dest, dest, tmp, 32, 32); 3035 } else { 3036 tcg_gen_shli_i64(tmp, tmp, 32); 3037 tcg_gen_or_i64(dest, dest, tmp); 3038 } 3039 tcg_temp_free_i64(tmp); 3040 } 3041 3042 void tcg_gen_extr_i64_i32(TCGv_i32 lo, TCGv_i32 hi, TCGv_i64 arg) 3043 { 3044 if (TCG_TARGET_REG_BITS == 32) { 3045 tcg_gen_mov_i32(lo, TCGV_LOW(arg)); 3046 tcg_gen_mov_i32(hi, TCGV_HIGH(arg)); 3047 } else { 3048 tcg_gen_extrl_i64_i32(lo, arg); 3049 tcg_gen_extrh_i64_i32(hi, arg); 3050 } 3051 } 3052 3053 void tcg_gen_extr32_i64(TCGv_i64 lo, TCGv_i64 hi, TCGv_i64 arg) 3054 { 3055 tcg_gen_ext32u_i64(lo, arg); 3056 tcg_gen_shri_i64(hi, arg, 32); 3057 } 3058 3059 void tcg_gen_concat32_i64(TCGv_i64 ret, TCGv_i64 lo, TCGv_i64 hi) 3060 { 3061 tcg_gen_deposit_i64(ret, lo, hi, 32, 32); 3062 } 3063 3064 void tcg_gen_extr_i128_i64(TCGv_i64 lo, TCGv_i64 hi, TCGv_i128 arg) 3065 { 3066 tcg_gen_mov_i64(lo, TCGV128_LOW(arg)); 3067 tcg_gen_mov_i64(hi, TCGV128_HIGH(arg)); 3068 } 3069 3070 void tcg_gen_concat_i64_i128(TCGv_i128 ret, TCGv_i64 lo, TCGv_i64 hi) 3071 { 3072 tcg_gen_mov_i64(TCGV128_LOW(ret), lo); 3073 tcg_gen_mov_i64(TCGV128_HIGH(ret), hi); 3074 } 3075 3076 void tcg_gen_mov_i128(TCGv_i128 dst, TCGv_i128 src) 3077 { 3078 if (dst != src) { 3079 tcg_gen_mov_i64(TCGV128_LOW(dst), TCGV128_LOW(src)); 3080 tcg_gen_mov_i64(TCGV128_HIGH(dst), TCGV128_HIGH(src)); 3081 } 3082 } 3083 3084 void tcg_gen_ld_i128(TCGv_i128 ret, TCGv_ptr base, tcg_target_long offset) 3085 { 3086 if (HOST_BIG_ENDIAN) { 3087 tcg_gen_ld_i64(TCGV128_HIGH(ret), base, offset); 3088 tcg_gen_ld_i64(TCGV128_LOW(ret), base, offset + 8); 3089 } else { 3090 tcg_gen_ld_i64(TCGV128_LOW(ret), base, offset); 3091 tcg_gen_ld_i64(TCGV128_HIGH(ret), base, offset + 8); 3092 } 3093 } 3094 3095 void tcg_gen_st_i128(TCGv_i128 val, TCGv_ptr base, tcg_target_long offset) 3096 { 3097 if (HOST_BIG_ENDIAN) { 3098 tcg_gen_st_i64(TCGV128_HIGH(val), base, offset); 3099 tcg_gen_st_i64(TCGV128_LOW(val), base, offset + 8); 3100 } else { 3101 tcg_gen_st_i64(TCGV128_LOW(val), base, offset); 3102 tcg_gen_st_i64(TCGV128_HIGH(val), base, offset + 8); 3103 } 3104 } 3105 3106 /* QEMU specific operations. */ 3107 3108 void tcg_gen_exit_tb(const TranslationBlock *tb, unsigned idx) 3109 { 3110 /* 3111 * Let the jit code return the read-only version of the 3112 * TranslationBlock, so that we minimize the pc-relative 3113 * distance of the address of the exit_tb code to TB. 3114 * This will improve utilization of pc-relative address loads. 3115 * 3116 * TODO: Move this to translator_loop, so that all const 3117 * TranslationBlock pointers refer to read-only memory. 3118 * This requires coordination with targets that do not use 3119 * the translator_loop. 3120 */ 3121 uintptr_t val = (uintptr_t)tcg_splitwx_to_rx((void *)tb) + idx; 3122 3123 if (tb == NULL) { 3124 tcg_debug_assert(idx == 0); 3125 } else if (idx <= TB_EXIT_IDXMAX) { 3126 #ifdef CONFIG_DEBUG_TCG 3127 /* This is an exit following a goto_tb. Verify that we have 3128 seen this numbered exit before, via tcg_gen_goto_tb. */ 3129 tcg_debug_assert(tcg_ctx->goto_tb_issue_mask & (1 << idx)); 3130 #endif 3131 } else { 3132 /* This is an exit via the exitreq label. */ 3133 tcg_debug_assert(idx == TB_EXIT_REQUESTED); 3134 } 3135 3136 tcg_gen_op1i(INDEX_op_exit_tb, 0, val); 3137 } 3138 3139 void tcg_gen_goto_tb(unsigned idx) 3140 { 3141 /* We tested CF_NO_GOTO_TB in translator_use_goto_tb. */ 3142 tcg_debug_assert(!(tcg_ctx->gen_tb->cflags & CF_NO_GOTO_TB)); 3143 /* We only support two chained exits. */ 3144 tcg_debug_assert(idx <= TB_EXIT_IDXMAX); 3145 #ifdef CONFIG_DEBUG_TCG 3146 /* Verify that we haven't seen this numbered exit before. */ 3147 tcg_debug_assert((tcg_ctx->goto_tb_issue_mask & (1 << idx)) == 0); 3148 tcg_ctx->goto_tb_issue_mask |= 1 << idx; 3149 #endif 3150 plugin_gen_disable_mem_helpers(); 3151 tcg_gen_op1i(INDEX_op_goto_tb, 0, idx); 3152 } 3153 3154 void tcg_gen_lookup_and_goto_ptr(void) 3155 { 3156 TCGv_ptr ptr; 3157 3158 if (tcg_ctx->gen_tb->cflags & CF_NO_GOTO_PTR) { 3159 tcg_gen_exit_tb(NULL, 0); 3160 return; 3161 } 3162 3163 plugin_gen_disable_mem_helpers(); 3164 ptr = tcg_temp_ebb_new_ptr(); 3165 gen_helper_lookup_tb_ptr(ptr, tcg_env); 3166 tcg_gen_op1i(INDEX_op_goto_ptr, TCG_TYPE_PTR, tcgv_ptr_arg(ptr)); 3167 tcg_temp_free_ptr(ptr); 3168 } 3169