1 /* 2 * Tiny Code Generator for QEMU 3 * 4 * Copyright (c) 2008 Fabrice Bellard 5 * 6 * Permission is hereby granted, free of charge, to any person obtaining a copy 7 * of this software and associated documentation files (the "Software"), to deal 8 * in the Software without restriction, including without limitation the rights 9 * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 10 * copies of the Software, and to permit persons to whom the Software is 11 * furnished to do so, subject to the following conditions: 12 * 13 * The above copyright notice and this permission notice shall be included in 14 * all copies or substantial portions of the Software. 15 * 16 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 17 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 18 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL 19 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 20 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 21 * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN 22 * THE SOFTWARE. 23 */ 24 25 #include "qemu/osdep.h" 26 #include "tcg/tcg.h" 27 #include "tcg/tcg-temp-internal.h" 28 #include "tcg/tcg-op-common.h" 29 #include "exec/translation-block.h" 30 #include "exec/plugin-gen.h" 31 #include "tcg-internal.h" 32 #include "tcg-has.h" 33 34 /* 35 * Encourage the compiler to tail-call to a function, rather than inlining. 36 * Minimizes code size across 99 bottles of beer on the wall. 37 */ 38 #define NI __attribute__((noinline)) 39 40 TCGOp * NI tcg_gen_op1(TCGOpcode opc, TCGType type, TCGArg a1) 41 { 42 TCGOp *op = tcg_emit_op(opc, 1); 43 TCGOP_TYPE(op) = type; 44 op->args[0] = a1; 45 return op; 46 } 47 48 TCGOp * NI tcg_gen_op2(TCGOpcode opc, TCGType type, TCGArg a1, TCGArg a2) 49 { 50 TCGOp *op = tcg_emit_op(opc, 2); 51 TCGOP_TYPE(op) = type; 52 op->args[0] = a1; 53 op->args[1] = a2; 54 return op; 55 } 56 57 TCGOp * NI tcg_gen_op3(TCGOpcode opc, TCGType type, TCGArg a1, 58 TCGArg a2, TCGArg a3) 59 { 60 TCGOp *op = tcg_emit_op(opc, 3); 61 TCGOP_TYPE(op) = type; 62 op->args[0] = a1; 63 op->args[1] = a2; 64 op->args[2] = a3; 65 return op; 66 } 67 68 TCGOp * NI tcg_gen_op4(TCGOpcode opc, TCGType type, TCGArg a1, TCGArg a2, 69 TCGArg a3, TCGArg a4) 70 { 71 TCGOp *op = tcg_emit_op(opc, 4); 72 TCGOP_TYPE(op) = type; 73 op->args[0] = a1; 74 op->args[1] = a2; 75 op->args[2] = a3; 76 op->args[3] = a4; 77 return op; 78 } 79 80 TCGOp * NI tcg_gen_op5(TCGOpcode opc, TCGType type, TCGArg a1, TCGArg a2, 81 TCGArg a3, TCGArg a4, TCGArg a5) 82 { 83 TCGOp *op = tcg_emit_op(opc, 5); 84 TCGOP_TYPE(op) = type; 85 op->args[0] = a1; 86 op->args[1] = a2; 87 op->args[2] = a3; 88 op->args[3] = a4; 89 op->args[4] = a5; 90 return op; 91 } 92 93 TCGOp * NI tcg_gen_op6(TCGOpcode opc, TCGType type, TCGArg a1, TCGArg a2, 94 TCGArg a3, TCGArg a4, TCGArg a5, TCGArg a6) 95 { 96 TCGOp *op = tcg_emit_op(opc, 6); 97 TCGOP_TYPE(op) = type; 98 op->args[0] = a1; 99 op->args[1] = a2; 100 op->args[2] = a3; 101 op->args[3] = a4; 102 op->args[4] = a5; 103 op->args[5] = a6; 104 return op; 105 } 106 107 /* 108 * With CONFIG_DEBUG_TCG, tcgv_*_tmp via tcgv_*_arg, is an out-of-line 109 * assertion check. Force tail calls to avoid too much code expansion. 110 */ 111 #ifdef CONFIG_DEBUG_TCG 112 # define DNI NI 113 #else 114 # define DNI 115 #endif 116 117 static void DNI tcg_gen_op1_i32(TCGOpcode opc, TCGType type, TCGv_i32 a1) 118 { 119 tcg_gen_op1(opc, type, tcgv_i32_arg(a1)); 120 } 121 122 static void DNI tcg_gen_op1_i64(TCGOpcode opc, TCGType type, TCGv_i64 a1) 123 { 124 tcg_gen_op1(opc, type, tcgv_i64_arg(a1)); 125 } 126 127 static TCGOp * DNI tcg_gen_op1i(TCGOpcode opc, TCGType type, TCGArg a1) 128 { 129 return tcg_gen_op1(opc, type, a1); 130 } 131 132 static void DNI tcg_gen_op2_i32(TCGOpcode opc, TCGv_i32 a1, TCGv_i32 a2) 133 { 134 tcg_gen_op2(opc, TCG_TYPE_I32, tcgv_i32_arg(a1), tcgv_i32_arg(a2)); 135 } 136 137 static void DNI tcg_gen_op2_i64(TCGOpcode opc, TCGv_i64 a1, TCGv_i64 a2) 138 { 139 tcg_gen_op2(opc, TCG_TYPE_I64, tcgv_i64_arg(a1), tcgv_i64_arg(a2)); 140 } 141 142 static void DNI tcg_gen_op3_i32(TCGOpcode opc, TCGv_i32 a1, 143 TCGv_i32 a2, TCGv_i32 a3) 144 { 145 tcg_gen_op3(opc, TCG_TYPE_I32, tcgv_i32_arg(a1), 146 tcgv_i32_arg(a2), tcgv_i32_arg(a3)); 147 } 148 149 static void DNI tcg_gen_op3_i64(TCGOpcode opc, TCGv_i64 a1, 150 TCGv_i64 a2, TCGv_i64 a3) 151 { 152 tcg_gen_op3(opc, TCG_TYPE_I64, tcgv_i64_arg(a1), 153 tcgv_i64_arg(a2), tcgv_i64_arg(a3)); 154 } 155 156 static void DNI tcg_gen_op3i_i32(TCGOpcode opc, TCGv_i32 a1, 157 TCGv_i32 a2, TCGArg a3) 158 { 159 tcg_gen_op3(opc, TCG_TYPE_I32, tcgv_i32_arg(a1), tcgv_i32_arg(a2), a3); 160 } 161 162 static void DNI tcg_gen_op3i_i64(TCGOpcode opc, TCGv_i64 a1, 163 TCGv_i64 a2, TCGArg a3) 164 { 165 tcg_gen_op3(opc, TCG_TYPE_I64, tcgv_i64_arg(a1), tcgv_i64_arg(a2), a3); 166 } 167 168 static void DNI tcg_gen_ldst_op_i32(TCGOpcode opc, TCGv_i32 val, 169 TCGv_ptr base, TCGArg offset) 170 { 171 tcg_gen_op3(opc, TCG_TYPE_I32, tcgv_i32_arg(val), 172 tcgv_ptr_arg(base), offset); 173 } 174 175 static void DNI tcg_gen_ldst_op_i64(TCGOpcode opc, TCGv_i64 val, 176 TCGv_ptr base, TCGArg offset) 177 { 178 tcg_gen_op3(opc, TCG_TYPE_I64, tcgv_i64_arg(val), 179 tcgv_ptr_arg(base), offset); 180 } 181 182 static void DNI tcg_gen_op4_i32(TCGOpcode opc, TCGv_i32 a1, TCGv_i32 a2, 183 TCGv_i32 a3, TCGv_i32 a4) 184 { 185 tcg_gen_op4(opc, TCG_TYPE_I32, tcgv_i32_arg(a1), tcgv_i32_arg(a2), 186 tcgv_i32_arg(a3), tcgv_i32_arg(a4)); 187 } 188 189 static void DNI tcg_gen_op4_i64(TCGOpcode opc, TCGv_i64 a1, TCGv_i64 a2, 190 TCGv_i64 a3, TCGv_i64 a4) 191 { 192 tcg_gen_op4(opc, TCG_TYPE_I64, tcgv_i64_arg(a1), tcgv_i64_arg(a2), 193 tcgv_i64_arg(a3), tcgv_i64_arg(a4)); 194 } 195 196 static void DNI tcg_gen_op4i_i32(TCGOpcode opc, TCGv_i32 a1, TCGv_i32 a2, 197 TCGv_i32 a3, TCGArg a4) 198 { 199 tcg_gen_op4(opc, TCG_TYPE_I32, tcgv_i32_arg(a1), tcgv_i32_arg(a2), 200 tcgv_i32_arg(a3), a4); 201 } 202 203 static void DNI tcg_gen_op4i_i64(TCGOpcode opc, TCGv_i64 a1, TCGv_i64 a2, 204 TCGv_i64 a3, TCGArg a4) 205 { 206 tcg_gen_op4(opc, TCG_TYPE_I64, tcgv_i64_arg(a1), tcgv_i64_arg(a2), 207 tcgv_i64_arg(a3), a4); 208 } 209 210 static TCGOp * DNI tcg_gen_op4ii_i32(TCGOpcode opc, TCGv_i32 a1, TCGv_i32 a2, 211 TCGArg a3, TCGArg a4) 212 { 213 return tcg_gen_op4(opc, TCG_TYPE_I32, 214 tcgv_i32_arg(a1), tcgv_i32_arg(a2), a3, a4); 215 } 216 217 static TCGOp * DNI tcg_gen_op4ii_i64(TCGOpcode opc, TCGv_i64 a1, TCGv_i64 a2, 218 TCGArg a3, TCGArg a4) 219 { 220 return tcg_gen_op4(opc, TCG_TYPE_I64, 221 tcgv_i64_arg(a1), tcgv_i64_arg(a2), a3, a4); 222 } 223 224 static void DNI tcg_gen_op5_i32(TCGOpcode opc, TCGv_i32 a1, TCGv_i32 a2, 225 TCGv_i32 a3, TCGv_i32 a4, TCGv_i32 a5) 226 { 227 tcg_gen_op5(opc, TCG_TYPE_I32, tcgv_i32_arg(a1), tcgv_i32_arg(a2), 228 tcgv_i32_arg(a3), tcgv_i32_arg(a4), tcgv_i32_arg(a5)); 229 } 230 231 static void DNI tcg_gen_op5_i64(TCGOpcode opc, TCGv_i64 a1, TCGv_i64 a2, 232 TCGv_i64 a3, TCGv_i64 a4, TCGv_i64 a5) 233 { 234 tcg_gen_op5(opc, TCG_TYPE_I64, tcgv_i64_arg(a1), tcgv_i64_arg(a2), 235 tcgv_i64_arg(a3), tcgv_i64_arg(a4), tcgv_i64_arg(a5)); 236 } 237 238 static void DNI tcg_gen_op5ii_i32(TCGOpcode opc, TCGv_i32 a1, TCGv_i32 a2, 239 TCGv_i32 a3, TCGArg a4, TCGArg a5) 240 { 241 tcg_gen_op5(opc, TCG_TYPE_I32, tcgv_i32_arg(a1), tcgv_i32_arg(a2), 242 tcgv_i32_arg(a3), a4, a5); 243 } 244 245 static void DNI tcg_gen_op5ii_i64(TCGOpcode opc, TCGv_i64 a1, TCGv_i64 a2, 246 TCGv_i64 a3, TCGArg a4, TCGArg a5) 247 { 248 tcg_gen_op5(opc, TCG_TYPE_I64, tcgv_i64_arg(a1), tcgv_i64_arg(a2), 249 tcgv_i64_arg(a3), a4, a5); 250 } 251 252 static void DNI tcg_gen_op6_i32(TCGOpcode opc, TCGv_i32 a1, TCGv_i32 a2, 253 TCGv_i32 a3, TCGv_i32 a4, 254 TCGv_i32 a5, TCGv_i32 a6) 255 { 256 tcg_gen_op6(opc, TCG_TYPE_I32, tcgv_i32_arg(a1), tcgv_i32_arg(a2), 257 tcgv_i32_arg(a3), tcgv_i32_arg(a4), tcgv_i32_arg(a5), 258 tcgv_i32_arg(a6)); 259 } 260 261 static void DNI tcg_gen_op6_i64(TCGOpcode opc, TCGv_i64 a1, TCGv_i64 a2, 262 TCGv_i64 a3, TCGv_i64 a4, 263 TCGv_i64 a5, TCGv_i64 a6) 264 { 265 tcg_gen_op6(opc, TCG_TYPE_I64, tcgv_i64_arg(a1), tcgv_i64_arg(a2), 266 tcgv_i64_arg(a3), tcgv_i64_arg(a4), tcgv_i64_arg(a5), 267 tcgv_i64_arg(a6)); 268 } 269 270 static void DNI tcg_gen_op6i_i32(TCGOpcode opc, TCGv_i32 a1, TCGv_i32 a2, 271 TCGv_i32 a3, TCGv_i32 a4, 272 TCGv_i32 a5, TCGArg a6) 273 { 274 tcg_gen_op6(opc, TCG_TYPE_I32, tcgv_i32_arg(a1), tcgv_i32_arg(a2), 275 tcgv_i32_arg(a3), tcgv_i32_arg(a4), tcgv_i32_arg(a5), a6); 276 } 277 278 static void DNI tcg_gen_op6i_i64(TCGOpcode opc, TCGv_i64 a1, TCGv_i64 a2, 279 TCGv_i64 a3, TCGv_i64 a4, 280 TCGv_i64 a5, TCGArg a6) 281 { 282 tcg_gen_op6(opc, TCG_TYPE_I64, tcgv_i64_arg(a1), tcgv_i64_arg(a2), 283 tcgv_i64_arg(a3), tcgv_i64_arg(a4), tcgv_i64_arg(a5), a6); 284 } 285 286 static TCGOp * DNI tcg_gen_op6ii_i32(TCGOpcode opc, TCGv_i32 a1, TCGv_i32 a2, 287 TCGv_i32 a3, TCGv_i32 a4, 288 TCGArg a5, TCGArg a6) 289 { 290 return tcg_gen_op6(opc, TCG_TYPE_I32, tcgv_i32_arg(a1), tcgv_i32_arg(a2), 291 tcgv_i32_arg(a3), tcgv_i32_arg(a4), a5, a6); 292 } 293 294 /* Generic ops. */ 295 296 void gen_set_label(TCGLabel *l) 297 { 298 l->present = 1; 299 tcg_gen_op1(INDEX_op_set_label, 0, label_arg(l)); 300 } 301 302 static void add_as_label_use(TCGLabel *l, TCGOp *op) 303 { 304 TCGLabelUse *u = tcg_malloc(sizeof(TCGLabelUse)); 305 306 u->op = op; 307 QSIMPLEQ_INSERT_TAIL(&l->branches, u, next); 308 } 309 310 void tcg_gen_br(TCGLabel *l) 311 { 312 add_as_label_use(l, tcg_gen_op1(INDEX_op_br, 0, label_arg(l))); 313 } 314 315 void tcg_gen_mb(TCGBar mb_type) 316 { 317 #ifdef CONFIG_USER_ONLY 318 bool parallel = tcg_ctx->gen_tb->cflags & CF_PARALLEL; 319 #else 320 /* 321 * It is tempting to elide the barrier in a uniprocessor context. 322 * However, even with a single cpu we have i/o threads running in 323 * parallel, and lack of memory order can result in e.g. virtio 324 * queue entries being read incorrectly. 325 */ 326 bool parallel = true; 327 #endif 328 329 if (parallel) { 330 tcg_gen_op1(INDEX_op_mb, 0, mb_type); 331 } 332 } 333 334 void tcg_gen_plugin_cb(unsigned from) 335 { 336 tcg_gen_op1(INDEX_op_plugin_cb, 0, from); 337 } 338 339 void tcg_gen_plugin_mem_cb(TCGv_i64 addr, unsigned meminfo) 340 { 341 tcg_gen_op2(INDEX_op_plugin_mem_cb, 0, tcgv_i64_arg(addr), meminfo); 342 } 343 344 /* 32 bit ops */ 345 346 void tcg_gen_discard_i32(TCGv_i32 arg) 347 { 348 tcg_gen_op1_i32(INDEX_op_discard, TCG_TYPE_I32, arg); 349 } 350 351 void tcg_gen_mov_i32(TCGv_i32 ret, TCGv_i32 arg) 352 { 353 if (ret != arg) { 354 tcg_gen_op2_i32(INDEX_op_mov, ret, arg); 355 } 356 } 357 358 void tcg_gen_movi_i32(TCGv_i32 ret, int32_t arg) 359 { 360 tcg_gen_mov_i32(ret, tcg_constant_i32(arg)); 361 } 362 363 void tcg_gen_add_i32(TCGv_i32 ret, TCGv_i32 arg1, TCGv_i32 arg2) 364 { 365 tcg_gen_op3_i32(INDEX_op_add, ret, arg1, arg2); 366 } 367 368 void tcg_gen_addi_i32(TCGv_i32 ret, TCGv_i32 arg1, int32_t arg2) 369 { 370 /* some cases can be optimized here */ 371 if (arg2 == 0) { 372 tcg_gen_mov_i32(ret, arg1); 373 } else { 374 tcg_gen_add_i32(ret, arg1, tcg_constant_i32(arg2)); 375 } 376 } 377 378 void tcg_gen_sub_i32(TCGv_i32 ret, TCGv_i32 arg1, TCGv_i32 arg2) 379 { 380 tcg_gen_op3_i32(INDEX_op_sub, ret, arg1, arg2); 381 } 382 383 void tcg_gen_subfi_i32(TCGv_i32 ret, int32_t arg1, TCGv_i32 arg2) 384 { 385 if (arg1 == 0) { 386 tcg_gen_neg_i32(ret, arg2); 387 } else { 388 tcg_gen_sub_i32(ret, tcg_constant_i32(arg1), arg2); 389 } 390 } 391 392 void tcg_gen_subi_i32(TCGv_i32 ret, TCGv_i32 arg1, int32_t arg2) 393 { 394 tcg_gen_addi_i32(ret, arg1, -arg2); 395 } 396 397 void tcg_gen_neg_i32(TCGv_i32 ret, TCGv_i32 arg) 398 { 399 tcg_gen_op2_i32(INDEX_op_neg, ret, arg); 400 } 401 402 void tcg_gen_and_i32(TCGv_i32 ret, TCGv_i32 arg1, TCGv_i32 arg2) 403 { 404 tcg_gen_op3_i32(INDEX_op_and, ret, arg1, arg2); 405 } 406 407 void tcg_gen_andi_i32(TCGv_i32 ret, TCGv_i32 arg1, int32_t arg2) 408 { 409 /* Some cases can be optimized here. */ 410 switch (arg2) { 411 case 0: 412 tcg_gen_movi_i32(ret, 0); 413 return; 414 case -1: 415 tcg_gen_mov_i32(ret, arg1); 416 return; 417 default: 418 /* 419 * Canonicalize on extract, if valid. This aids x86 with its 420 * 2 operand MOVZBL and 2 operand AND, selecting the TCGOpcode 421 * which does not require matching operands. Other backends can 422 * trivially expand the extract to AND during code generation. 423 */ 424 if (!(arg2 & (arg2 + 1))) { 425 unsigned len = ctz32(~arg2); 426 if (TCG_TARGET_extract_valid(TCG_TYPE_I32, 0, len)) { 427 tcg_gen_extract_i32(ret, arg1, 0, len); 428 return; 429 } 430 } 431 break; 432 } 433 434 tcg_gen_and_i32(ret, arg1, tcg_constant_i32(arg2)); 435 } 436 437 void tcg_gen_or_i32(TCGv_i32 ret, TCGv_i32 arg1, TCGv_i32 arg2) 438 { 439 tcg_gen_op3_i32(INDEX_op_or, ret, arg1, arg2); 440 } 441 442 void tcg_gen_ori_i32(TCGv_i32 ret, TCGv_i32 arg1, int32_t arg2) 443 { 444 /* Some cases can be optimized here. */ 445 if (arg2 == -1) { 446 tcg_gen_movi_i32(ret, -1); 447 } else if (arg2 == 0) { 448 tcg_gen_mov_i32(ret, arg1); 449 } else { 450 tcg_gen_or_i32(ret, arg1, tcg_constant_i32(arg2)); 451 } 452 } 453 454 void tcg_gen_xor_i32(TCGv_i32 ret, TCGv_i32 arg1, TCGv_i32 arg2) 455 { 456 tcg_gen_op3_i32(INDEX_op_xor, ret, arg1, arg2); 457 } 458 459 void tcg_gen_xori_i32(TCGv_i32 ret, TCGv_i32 arg1, int32_t arg2) 460 { 461 /* Some cases can be optimized here. */ 462 if (arg2 == 0) { 463 tcg_gen_mov_i32(ret, arg1); 464 } else if (arg2 == -1 && 465 tcg_op_supported(INDEX_op_not, TCG_TYPE_I32, 0)) { 466 /* Don't recurse with tcg_gen_not_i32. */ 467 tcg_gen_op2_i32(INDEX_op_not, ret, arg1); 468 } else { 469 tcg_gen_xor_i32(ret, arg1, tcg_constant_i32(arg2)); 470 } 471 } 472 473 void tcg_gen_not_i32(TCGv_i32 ret, TCGv_i32 arg) 474 { 475 if (tcg_op_supported(INDEX_op_not, TCG_TYPE_I32, 0)) { 476 tcg_gen_op2_i32(INDEX_op_not, ret, arg); 477 } else { 478 tcg_gen_xori_i32(ret, arg, -1); 479 } 480 } 481 482 void tcg_gen_shl_i32(TCGv_i32 ret, TCGv_i32 arg1, TCGv_i32 arg2) 483 { 484 tcg_gen_op3_i32(INDEX_op_shl, ret, arg1, arg2); 485 } 486 487 void tcg_gen_shli_i32(TCGv_i32 ret, TCGv_i32 arg1, int32_t arg2) 488 { 489 tcg_debug_assert(arg2 >= 0 && arg2 < 32); 490 if (arg2 == 0) { 491 tcg_gen_mov_i32(ret, arg1); 492 } else { 493 tcg_gen_shl_i32(ret, arg1, tcg_constant_i32(arg2)); 494 } 495 } 496 497 void tcg_gen_shr_i32(TCGv_i32 ret, TCGv_i32 arg1, TCGv_i32 arg2) 498 { 499 tcg_gen_op3_i32(INDEX_op_shr, ret, arg1, arg2); 500 } 501 502 void tcg_gen_shri_i32(TCGv_i32 ret, TCGv_i32 arg1, int32_t arg2) 503 { 504 tcg_debug_assert(arg2 >= 0 && arg2 < 32); 505 if (arg2 == 0) { 506 tcg_gen_mov_i32(ret, arg1); 507 } else { 508 tcg_gen_shr_i32(ret, arg1, tcg_constant_i32(arg2)); 509 } 510 } 511 512 void tcg_gen_sar_i32(TCGv_i32 ret, TCGv_i32 arg1, TCGv_i32 arg2) 513 { 514 tcg_gen_op3_i32(INDEX_op_sar, ret, arg1, arg2); 515 } 516 517 void tcg_gen_sari_i32(TCGv_i32 ret, TCGv_i32 arg1, int32_t arg2) 518 { 519 tcg_debug_assert(arg2 >= 0 && arg2 < 32); 520 if (arg2 == 0) { 521 tcg_gen_mov_i32(ret, arg1); 522 } else { 523 tcg_gen_sar_i32(ret, arg1, tcg_constant_i32(arg2)); 524 } 525 } 526 527 void tcg_gen_brcond_i32(TCGCond cond, TCGv_i32 arg1, TCGv_i32 arg2, TCGLabel *l) 528 { 529 if (cond == TCG_COND_ALWAYS) { 530 tcg_gen_br(l); 531 } else if (cond != TCG_COND_NEVER) { 532 TCGOp *op = tcg_gen_op4ii_i32(INDEX_op_brcond_i32, 533 arg1, arg2, cond, label_arg(l)); 534 add_as_label_use(l, op); 535 } 536 } 537 538 void tcg_gen_brcondi_i32(TCGCond cond, TCGv_i32 arg1, int32_t arg2, TCGLabel *l) 539 { 540 if (cond == TCG_COND_ALWAYS) { 541 tcg_gen_br(l); 542 } else if (cond != TCG_COND_NEVER) { 543 tcg_gen_brcond_i32(cond, arg1, tcg_constant_i32(arg2), l); 544 } 545 } 546 547 void tcg_gen_setcond_i32(TCGCond cond, TCGv_i32 ret, 548 TCGv_i32 arg1, TCGv_i32 arg2) 549 { 550 if (cond == TCG_COND_ALWAYS) { 551 tcg_gen_movi_i32(ret, 1); 552 } else if (cond == TCG_COND_NEVER) { 553 tcg_gen_movi_i32(ret, 0); 554 } else { 555 tcg_gen_op4i_i32(INDEX_op_setcond_i32, ret, arg1, arg2, cond); 556 } 557 } 558 559 void tcg_gen_setcondi_i32(TCGCond cond, TCGv_i32 ret, 560 TCGv_i32 arg1, int32_t arg2) 561 { 562 tcg_gen_setcond_i32(cond, ret, arg1, tcg_constant_i32(arg2)); 563 } 564 565 void tcg_gen_negsetcond_i32(TCGCond cond, TCGv_i32 ret, 566 TCGv_i32 arg1, TCGv_i32 arg2) 567 { 568 if (cond == TCG_COND_ALWAYS) { 569 tcg_gen_movi_i32(ret, -1); 570 } else if (cond == TCG_COND_NEVER) { 571 tcg_gen_movi_i32(ret, 0); 572 } else if (TCG_TARGET_HAS_negsetcond_i32) { 573 tcg_gen_op4i_i32(INDEX_op_negsetcond_i32, ret, arg1, arg2, cond); 574 } else { 575 tcg_gen_setcond_i32(cond, ret, arg1, arg2); 576 tcg_gen_neg_i32(ret, ret); 577 } 578 } 579 580 void tcg_gen_negsetcondi_i32(TCGCond cond, TCGv_i32 ret, 581 TCGv_i32 arg1, int32_t arg2) 582 { 583 tcg_gen_negsetcond_i32(cond, ret, arg1, tcg_constant_i32(arg2)); 584 } 585 586 void tcg_gen_mul_i32(TCGv_i32 ret, TCGv_i32 arg1, TCGv_i32 arg2) 587 { 588 tcg_gen_op3_i32(INDEX_op_mul, ret, arg1, arg2); 589 } 590 591 void tcg_gen_muli_i32(TCGv_i32 ret, TCGv_i32 arg1, int32_t arg2) 592 { 593 if (arg2 == 0) { 594 tcg_gen_movi_i32(ret, 0); 595 } else if (is_power_of_2(arg2)) { 596 tcg_gen_shli_i32(ret, arg1, ctz32(arg2)); 597 } else { 598 tcg_gen_mul_i32(ret, arg1, tcg_constant_i32(arg2)); 599 } 600 } 601 602 void tcg_gen_div_i32(TCGv_i32 ret, TCGv_i32 arg1, TCGv_i32 arg2) 603 { 604 if (tcg_op_supported(INDEX_op_divs, TCG_TYPE_I32, 0)) { 605 tcg_gen_op3_i32(INDEX_op_divs, ret, arg1, arg2); 606 } else if (tcg_op_supported(INDEX_op_divs2, TCG_TYPE_I32, 0)) { 607 TCGv_i32 t0 = tcg_temp_ebb_new_i32(); 608 tcg_gen_sari_i32(t0, arg1, 31); 609 tcg_gen_op5_i32(INDEX_op_divs2, ret, t0, arg1, t0, arg2); 610 tcg_temp_free_i32(t0); 611 } else { 612 gen_helper_div_i32(ret, arg1, arg2); 613 } 614 } 615 616 void tcg_gen_rem_i32(TCGv_i32 ret, TCGv_i32 arg1, TCGv_i32 arg2) 617 { 618 if (tcg_op_supported(INDEX_op_rems, TCG_TYPE_I32, 0)) { 619 tcg_gen_op3_i32(INDEX_op_rems, ret, arg1, arg2); 620 } else if (tcg_op_supported(INDEX_op_divs, TCG_TYPE_I32, 0)) { 621 TCGv_i32 t0 = tcg_temp_ebb_new_i32(); 622 tcg_gen_op3_i32(INDEX_op_divs, t0, arg1, arg2); 623 tcg_gen_mul_i32(t0, t0, arg2); 624 tcg_gen_sub_i32(ret, arg1, t0); 625 tcg_temp_free_i32(t0); 626 } else if (tcg_op_supported(INDEX_op_divs2, TCG_TYPE_I32, 0)) { 627 TCGv_i32 t0 = tcg_temp_ebb_new_i32(); 628 tcg_gen_sari_i32(t0, arg1, 31); 629 tcg_gen_op5_i32(INDEX_op_divs2, t0, ret, arg1, t0, arg2); 630 tcg_temp_free_i32(t0); 631 } else { 632 gen_helper_rem_i32(ret, arg1, arg2); 633 } 634 } 635 636 void tcg_gen_divu_i32(TCGv_i32 ret, TCGv_i32 arg1, TCGv_i32 arg2) 637 { 638 if (tcg_op_supported(INDEX_op_divu, TCG_TYPE_I32, 0)) { 639 tcg_gen_op3_i32(INDEX_op_divu, ret, arg1, arg2); 640 } else if (tcg_op_supported(INDEX_op_divu2, TCG_TYPE_I32, 0)) { 641 TCGv_i32 t0 = tcg_temp_ebb_new_i32(); 642 TCGv_i32 zero = tcg_constant_i32(0); 643 tcg_gen_op5_i32(INDEX_op_divu2, ret, t0, arg1, zero, arg2); 644 tcg_temp_free_i32(t0); 645 } else { 646 gen_helper_divu_i32(ret, arg1, arg2); 647 } 648 } 649 650 void tcg_gen_remu_i32(TCGv_i32 ret, TCGv_i32 arg1, TCGv_i32 arg2) 651 { 652 if (tcg_op_supported(INDEX_op_remu, TCG_TYPE_I32, 0)) { 653 tcg_gen_op3_i32(INDEX_op_remu, ret, arg1, arg2); 654 } else if (tcg_op_supported(INDEX_op_divu, TCG_TYPE_I32, 0)) { 655 TCGv_i32 t0 = tcg_temp_ebb_new_i32(); 656 tcg_gen_op3_i32(INDEX_op_divu, t0, arg1, arg2); 657 tcg_gen_mul_i32(t0, t0, arg2); 658 tcg_gen_sub_i32(ret, arg1, t0); 659 tcg_temp_free_i32(t0); 660 } else if (tcg_op_supported(INDEX_op_divu2, TCG_TYPE_I32, 0)) { 661 TCGv_i32 t0 = tcg_temp_ebb_new_i32(); 662 TCGv_i32 zero = tcg_constant_i32(0); 663 tcg_gen_op5_i32(INDEX_op_divu2, t0, ret, arg1, zero, arg2); 664 tcg_temp_free_i32(t0); 665 } else { 666 gen_helper_remu_i32(ret, arg1, arg2); 667 } 668 } 669 670 void tcg_gen_andc_i32(TCGv_i32 ret, TCGv_i32 arg1, TCGv_i32 arg2) 671 { 672 if (tcg_op_supported(INDEX_op_andc, TCG_TYPE_I32, 0)) { 673 tcg_gen_op3_i32(INDEX_op_andc, ret, arg1, arg2); 674 } else { 675 TCGv_i32 t0 = tcg_temp_ebb_new_i32(); 676 tcg_gen_not_i32(t0, arg2); 677 tcg_gen_and_i32(ret, arg1, t0); 678 tcg_temp_free_i32(t0); 679 } 680 } 681 682 void tcg_gen_eqv_i32(TCGv_i32 ret, TCGv_i32 arg1, TCGv_i32 arg2) 683 { 684 if (tcg_op_supported(INDEX_op_eqv, TCG_TYPE_I32, 0)) { 685 tcg_gen_op3_i32(INDEX_op_eqv, ret, arg1, arg2); 686 } else { 687 tcg_gen_xor_i32(ret, arg1, arg2); 688 tcg_gen_not_i32(ret, ret); 689 } 690 } 691 692 void tcg_gen_nand_i32(TCGv_i32 ret, TCGv_i32 arg1, TCGv_i32 arg2) 693 { 694 if (tcg_op_supported(INDEX_op_nand, TCG_TYPE_I32, 0)) { 695 tcg_gen_op3_i32(INDEX_op_nand, ret, arg1, arg2); 696 } else { 697 tcg_gen_and_i32(ret, arg1, arg2); 698 tcg_gen_not_i32(ret, ret); 699 } 700 } 701 702 void tcg_gen_nor_i32(TCGv_i32 ret, TCGv_i32 arg1, TCGv_i32 arg2) 703 { 704 if (tcg_op_supported(INDEX_op_nor, TCG_TYPE_I32, 0)) { 705 tcg_gen_op3_i32(INDEX_op_nor, ret, arg1, arg2); 706 } else { 707 tcg_gen_or_i32(ret, arg1, arg2); 708 tcg_gen_not_i32(ret, ret); 709 } 710 } 711 712 void tcg_gen_orc_i32(TCGv_i32 ret, TCGv_i32 arg1, TCGv_i32 arg2) 713 { 714 if (tcg_op_supported(INDEX_op_orc, TCG_TYPE_I32, 0)) { 715 tcg_gen_op3_i32(INDEX_op_orc, ret, arg1, arg2); 716 } else { 717 TCGv_i32 t0 = tcg_temp_ebb_new_i32(); 718 tcg_gen_not_i32(t0, arg2); 719 tcg_gen_or_i32(ret, arg1, t0); 720 tcg_temp_free_i32(t0); 721 } 722 } 723 724 void tcg_gen_clz_i32(TCGv_i32 ret, TCGv_i32 arg1, TCGv_i32 arg2) 725 { 726 if (tcg_op_supported(INDEX_op_clz, TCG_TYPE_I32, 0)) { 727 tcg_gen_op3_i32(INDEX_op_clz, ret, arg1, arg2); 728 } else if (tcg_op_supported(INDEX_op_clz, TCG_TYPE_I64, 0)) { 729 TCGv_i64 t1 = tcg_temp_ebb_new_i64(); 730 TCGv_i64 t2 = tcg_temp_ebb_new_i64(); 731 tcg_gen_extu_i32_i64(t1, arg1); 732 tcg_gen_extu_i32_i64(t2, arg2); 733 tcg_gen_addi_i64(t2, t2, 32); 734 tcg_gen_clz_i64(t1, t1, t2); 735 tcg_gen_extrl_i64_i32(ret, t1); 736 tcg_temp_free_i64(t1); 737 tcg_temp_free_i64(t2); 738 tcg_gen_subi_i32(ret, ret, 32); 739 } else { 740 gen_helper_clz_i32(ret, arg1, arg2); 741 } 742 } 743 744 void tcg_gen_clzi_i32(TCGv_i32 ret, TCGv_i32 arg1, uint32_t arg2) 745 { 746 tcg_gen_clz_i32(ret, arg1, tcg_constant_i32(arg2)); 747 } 748 749 void tcg_gen_ctz_i32(TCGv_i32 ret, TCGv_i32 arg1, TCGv_i32 arg2) 750 { 751 TCGv_i32 z, t; 752 753 if (tcg_op_supported(INDEX_op_ctz, TCG_TYPE_I32, 0)) { 754 tcg_gen_op3_i32(INDEX_op_ctz, ret, arg1, arg2); 755 return; 756 } 757 if (tcg_op_supported(INDEX_op_ctz, TCG_TYPE_I64, 0)) { 758 TCGv_i64 t1 = tcg_temp_ebb_new_i64(); 759 TCGv_i64 t2 = tcg_temp_ebb_new_i64(); 760 tcg_gen_extu_i32_i64(t1, arg1); 761 tcg_gen_extu_i32_i64(t2, arg2); 762 tcg_gen_ctz_i64(t1, t1, t2); 763 tcg_gen_extrl_i64_i32(ret, t1); 764 tcg_temp_free_i64(t1); 765 tcg_temp_free_i64(t2); 766 return; 767 } 768 if (TCG_TARGET_HAS_ctpop_i32 || TCG_TARGET_HAS_ctpop_i64) { 769 t = tcg_temp_ebb_new_i32(); 770 tcg_gen_subi_i32(t, arg1, 1); 771 tcg_gen_andc_i32(t, t, arg1); 772 tcg_gen_ctpop_i32(t, t); 773 } else if (tcg_op_supported(INDEX_op_clz, TCG_TYPE_REG, 0)) { 774 t = tcg_temp_ebb_new_i32(); 775 tcg_gen_neg_i32(t, arg1); 776 tcg_gen_and_i32(t, t, arg1); 777 tcg_gen_clzi_i32(t, t, 32); 778 tcg_gen_xori_i32(t, t, 31); 779 } else { 780 gen_helper_ctz_i32(ret, arg1, arg2); 781 return; 782 } 783 784 z = tcg_constant_i32(0); 785 tcg_gen_movcond_i32(TCG_COND_EQ, ret, arg1, z, arg2, t); 786 tcg_temp_free_i32(t); 787 } 788 789 void tcg_gen_ctzi_i32(TCGv_i32 ret, TCGv_i32 arg1, uint32_t arg2) 790 { 791 if (!tcg_op_supported(INDEX_op_ctz, TCG_TYPE_I32, 0) 792 && TCG_TARGET_HAS_ctpop_i32 && arg2 == 32) { 793 /* This equivalence has the advantage of not requiring a fixup. */ 794 TCGv_i32 t = tcg_temp_ebb_new_i32(); 795 tcg_gen_subi_i32(t, arg1, 1); 796 tcg_gen_andc_i32(t, t, arg1); 797 tcg_gen_ctpop_i32(ret, t); 798 tcg_temp_free_i32(t); 799 } else { 800 tcg_gen_ctz_i32(ret, arg1, tcg_constant_i32(arg2)); 801 } 802 } 803 804 void tcg_gen_clrsb_i32(TCGv_i32 ret, TCGv_i32 arg) 805 { 806 if (tcg_op_supported(INDEX_op_clz, TCG_TYPE_REG, 0)) { 807 TCGv_i32 t = tcg_temp_ebb_new_i32(); 808 tcg_gen_sari_i32(t, arg, 31); 809 tcg_gen_xor_i32(t, t, arg); 810 tcg_gen_clzi_i32(t, t, 32); 811 tcg_gen_subi_i32(ret, t, 1); 812 tcg_temp_free_i32(t); 813 } else { 814 gen_helper_clrsb_i32(ret, arg); 815 } 816 } 817 818 void tcg_gen_ctpop_i32(TCGv_i32 ret, TCGv_i32 arg1) 819 { 820 if (TCG_TARGET_HAS_ctpop_i32) { 821 tcg_gen_op2_i32(INDEX_op_ctpop_i32, ret, arg1); 822 } else if (TCG_TARGET_HAS_ctpop_i64) { 823 TCGv_i64 t = tcg_temp_ebb_new_i64(); 824 tcg_gen_extu_i32_i64(t, arg1); 825 tcg_gen_ctpop_i64(t, t); 826 tcg_gen_extrl_i64_i32(ret, t); 827 tcg_temp_free_i64(t); 828 } else { 829 gen_helper_ctpop_i32(ret, arg1); 830 } 831 } 832 833 void tcg_gen_rotl_i32(TCGv_i32 ret, TCGv_i32 arg1, TCGv_i32 arg2) 834 { 835 if (tcg_op_supported(INDEX_op_rotl, TCG_TYPE_I32, 0)) { 836 tcg_gen_op3_i32(INDEX_op_rotl, ret, arg1, arg2); 837 } else if (tcg_op_supported(INDEX_op_rotr, TCG_TYPE_I32, 0)) { 838 TCGv_i32 t0 = tcg_temp_ebb_new_i32(); 839 tcg_gen_neg_i32(t0, arg2); 840 tcg_gen_op3_i32(INDEX_op_rotr, ret, arg1, t0); 841 tcg_temp_free_i32(t0); 842 } else { 843 TCGv_i32 t0 = tcg_temp_ebb_new_i32(); 844 TCGv_i32 t1 = tcg_temp_ebb_new_i32(); 845 tcg_gen_shl_i32(t0, arg1, arg2); 846 tcg_gen_neg_i32(t1, arg2); 847 tcg_gen_shr_i32(t1, arg1, t1); 848 tcg_gen_or_i32(ret, t0, t1); 849 tcg_temp_free_i32(t0); 850 tcg_temp_free_i32(t1); 851 } 852 } 853 854 void tcg_gen_rotli_i32(TCGv_i32 ret, TCGv_i32 arg1, int32_t arg2) 855 { 856 tcg_debug_assert(arg2 >= 0 && arg2 < 32); 857 /* some cases can be optimized here */ 858 if (arg2 == 0) { 859 tcg_gen_mov_i32(ret, arg1); 860 } else if (tcg_op_supported(INDEX_op_rotl, TCG_TYPE_I32, 0)) { 861 TCGv_i32 t0 = tcg_constant_i32(arg2); 862 tcg_gen_op3_i32(INDEX_op_rotl, ret, arg1, t0); 863 } else if (tcg_op_supported(INDEX_op_rotr, TCG_TYPE_I32, 0)) { 864 TCGv_i32 t0 = tcg_constant_i32(32 - arg2); 865 tcg_gen_op3_i32(INDEX_op_rotr, ret, arg1, t0); 866 } else { 867 TCGv_i32 t0 = tcg_temp_ebb_new_i32(); 868 TCGv_i32 t1 = tcg_temp_ebb_new_i32(); 869 tcg_gen_shli_i32(t0, arg1, arg2); 870 tcg_gen_shri_i32(t1, arg1, 32 - arg2); 871 tcg_gen_or_i32(ret, t0, t1); 872 tcg_temp_free_i32(t0); 873 tcg_temp_free_i32(t1); 874 } 875 } 876 877 void tcg_gen_rotr_i32(TCGv_i32 ret, TCGv_i32 arg1, TCGv_i32 arg2) 878 { 879 if (tcg_op_supported(INDEX_op_rotr, TCG_TYPE_I32, 0)) { 880 tcg_gen_op3_i32(INDEX_op_rotr, ret, arg1, arg2); 881 } else if (tcg_op_supported(INDEX_op_rotl, TCG_TYPE_I32, 0)) { 882 TCGv_i32 t0 = tcg_temp_ebb_new_i32(); 883 tcg_gen_neg_i32(t0, arg2); 884 tcg_gen_op3_i32(INDEX_op_rotl, ret, arg1, t0); 885 tcg_temp_free_i32(t0); 886 } else { 887 TCGv_i32 t0 = tcg_temp_ebb_new_i32(); 888 TCGv_i32 t1 = tcg_temp_ebb_new_i32(); 889 tcg_gen_shr_i32(t0, arg1, arg2); 890 tcg_gen_neg_i32(t1, arg2); 891 tcg_gen_shl_i32(t1, arg1, t1); 892 tcg_gen_or_i32(ret, t0, t1); 893 tcg_temp_free_i32(t0); 894 tcg_temp_free_i32(t1); 895 } 896 } 897 898 void tcg_gen_rotri_i32(TCGv_i32 ret, TCGv_i32 arg1, int32_t arg2) 899 { 900 tcg_debug_assert(arg2 >= 0 && arg2 < 32); 901 tcg_gen_rotli_i32(ret, arg1, -arg2 & 31); 902 } 903 904 void tcg_gen_deposit_i32(TCGv_i32 ret, TCGv_i32 arg1, TCGv_i32 arg2, 905 unsigned int ofs, unsigned int len) 906 { 907 uint32_t mask; 908 TCGv_i32 t1; 909 910 tcg_debug_assert(ofs < 32); 911 tcg_debug_assert(len > 0); 912 tcg_debug_assert(len <= 32); 913 tcg_debug_assert(ofs + len <= 32); 914 915 if (len == 32) { 916 tcg_gen_mov_i32(ret, arg2); 917 return; 918 } 919 if (TCG_TARGET_deposit_valid(TCG_TYPE_I32, ofs, len)) { 920 tcg_gen_op5ii_i32(INDEX_op_deposit_i32, ret, arg1, arg2, ofs, len); 921 return; 922 } 923 924 t1 = tcg_temp_ebb_new_i32(); 925 926 if (TCG_TARGET_HAS_extract2_i32) { 927 if (ofs + len == 32) { 928 tcg_gen_shli_i32(t1, arg1, len); 929 tcg_gen_extract2_i32(ret, t1, arg2, len); 930 goto done; 931 } 932 if (ofs == 0) { 933 tcg_gen_extract2_i32(ret, arg1, arg2, len); 934 tcg_gen_rotli_i32(ret, ret, len); 935 goto done; 936 } 937 } 938 939 mask = (1u << len) - 1; 940 if (ofs + len < 32) { 941 tcg_gen_andi_i32(t1, arg2, mask); 942 tcg_gen_shli_i32(t1, t1, ofs); 943 } else { 944 tcg_gen_shli_i32(t1, arg2, ofs); 945 } 946 tcg_gen_andi_i32(ret, arg1, ~(mask << ofs)); 947 tcg_gen_or_i32(ret, ret, t1); 948 done: 949 tcg_temp_free_i32(t1); 950 } 951 952 void tcg_gen_deposit_z_i32(TCGv_i32 ret, TCGv_i32 arg, 953 unsigned int ofs, unsigned int len) 954 { 955 tcg_debug_assert(ofs < 32); 956 tcg_debug_assert(len > 0); 957 tcg_debug_assert(len <= 32); 958 tcg_debug_assert(ofs + len <= 32); 959 960 if (ofs + len == 32) { 961 tcg_gen_shli_i32(ret, arg, ofs); 962 } else if (ofs == 0) { 963 tcg_gen_andi_i32(ret, arg, (1u << len) - 1); 964 } else if (TCG_TARGET_deposit_valid(TCG_TYPE_I32, ofs, len)) { 965 TCGv_i32 zero = tcg_constant_i32(0); 966 tcg_gen_op5ii_i32(INDEX_op_deposit_i32, ret, zero, arg, ofs, len); 967 } else { 968 /* 969 * To help two-operand hosts we prefer to zero-extend first, 970 * which allows ARG to stay live. 971 */ 972 if (TCG_TARGET_extract_valid(TCG_TYPE_I32, 0, len)) { 973 tcg_gen_extract_i32(ret, arg, 0, len); 974 tcg_gen_shli_i32(ret, ret, ofs); 975 return; 976 } 977 /* Otherwise prefer zero-extension over AND for code size. */ 978 if (TCG_TARGET_extract_valid(TCG_TYPE_I32, 0, ofs + len)) { 979 tcg_gen_shli_i32(ret, arg, ofs); 980 tcg_gen_extract_i32(ret, ret, 0, ofs + len); 981 return; 982 } 983 tcg_gen_andi_i32(ret, arg, (1u << len) - 1); 984 tcg_gen_shli_i32(ret, ret, ofs); 985 } 986 } 987 988 void tcg_gen_extract_i32(TCGv_i32 ret, TCGv_i32 arg, 989 unsigned int ofs, unsigned int len) 990 { 991 tcg_debug_assert(ofs < 32); 992 tcg_debug_assert(len > 0); 993 tcg_debug_assert(len <= 32); 994 tcg_debug_assert(ofs + len <= 32); 995 996 /* Canonicalize certain special cases, even if extract is supported. */ 997 if (ofs + len == 32) { 998 tcg_gen_shri_i32(ret, arg, 32 - len); 999 return; 1000 } 1001 1002 if (TCG_TARGET_extract_valid(TCG_TYPE_I32, ofs, len)) { 1003 tcg_gen_op4ii_i32(INDEX_op_extract_i32, ret, arg, ofs, len); 1004 return; 1005 } 1006 if (ofs == 0) { 1007 tcg_gen_andi_i32(ret, arg, (1u << len) - 1); 1008 return; 1009 } 1010 1011 /* Assume that zero-extension, if available, is cheaper than a shift. */ 1012 if (TCG_TARGET_extract_valid(TCG_TYPE_I32, 0, ofs + len)) { 1013 tcg_gen_op4ii_i32(INDEX_op_extract_i32, ret, arg, 0, ofs + len); 1014 tcg_gen_shri_i32(ret, ret, ofs); 1015 return; 1016 } 1017 1018 /* ??? Ideally we'd know what values are available for immediate AND. 1019 Assume that 8 bits are available, plus the special case of 16, 1020 so that we get ext8u, ext16u. */ 1021 switch (len) { 1022 case 1 ... 8: case 16: 1023 tcg_gen_shri_i32(ret, arg, ofs); 1024 tcg_gen_andi_i32(ret, ret, (1u << len) - 1); 1025 break; 1026 default: 1027 tcg_gen_shli_i32(ret, arg, 32 - len - ofs); 1028 tcg_gen_shri_i32(ret, ret, 32 - len); 1029 break; 1030 } 1031 } 1032 1033 void tcg_gen_sextract_i32(TCGv_i32 ret, TCGv_i32 arg, 1034 unsigned int ofs, unsigned int len) 1035 { 1036 tcg_debug_assert(ofs < 32); 1037 tcg_debug_assert(len > 0); 1038 tcg_debug_assert(len <= 32); 1039 tcg_debug_assert(ofs + len <= 32); 1040 1041 /* Canonicalize certain special cases, even if extract is supported. */ 1042 if (ofs + len == 32) { 1043 tcg_gen_sari_i32(ret, arg, 32 - len); 1044 return; 1045 } 1046 1047 if (TCG_TARGET_sextract_valid(TCG_TYPE_I32, ofs, len)) { 1048 tcg_gen_op4ii_i32(INDEX_op_sextract_i32, ret, arg, ofs, len); 1049 return; 1050 } 1051 1052 /* Assume that sign-extension, if available, is cheaper than a shift. */ 1053 if (TCG_TARGET_sextract_valid(TCG_TYPE_I32, 0, ofs + len)) { 1054 tcg_gen_op4ii_i32(INDEX_op_sextract_i32, ret, arg, 0, ofs + len); 1055 tcg_gen_sari_i32(ret, ret, ofs); 1056 return; 1057 } 1058 if (TCG_TARGET_sextract_valid(TCG_TYPE_I32, 0, len)) { 1059 tcg_gen_shri_i32(ret, arg, ofs); 1060 tcg_gen_op4ii_i32(INDEX_op_sextract_i32, ret, ret, 0, len); 1061 return; 1062 } 1063 1064 tcg_gen_shli_i32(ret, arg, 32 - len - ofs); 1065 tcg_gen_sari_i32(ret, ret, 32 - len); 1066 } 1067 1068 /* 1069 * Extract 32-bits from a 64-bit input, ah:al, starting from ofs. 1070 * Unlike tcg_gen_extract_i32 above, len is fixed at 32. 1071 */ 1072 void tcg_gen_extract2_i32(TCGv_i32 ret, TCGv_i32 al, TCGv_i32 ah, 1073 unsigned int ofs) 1074 { 1075 tcg_debug_assert(ofs <= 32); 1076 if (ofs == 0) { 1077 tcg_gen_mov_i32(ret, al); 1078 } else if (ofs == 32) { 1079 tcg_gen_mov_i32(ret, ah); 1080 } else if (al == ah) { 1081 tcg_gen_rotri_i32(ret, al, ofs); 1082 } else if (TCG_TARGET_HAS_extract2_i32) { 1083 tcg_gen_op4i_i32(INDEX_op_extract2_i32, ret, al, ah, ofs); 1084 } else { 1085 TCGv_i32 t0 = tcg_temp_ebb_new_i32(); 1086 tcg_gen_shri_i32(t0, al, ofs); 1087 tcg_gen_deposit_i32(ret, t0, ah, 32 - ofs, ofs); 1088 tcg_temp_free_i32(t0); 1089 } 1090 } 1091 1092 void tcg_gen_movcond_i32(TCGCond cond, TCGv_i32 ret, TCGv_i32 c1, 1093 TCGv_i32 c2, TCGv_i32 v1, TCGv_i32 v2) 1094 { 1095 if (cond == TCG_COND_ALWAYS) { 1096 tcg_gen_mov_i32(ret, v1); 1097 } else if (cond == TCG_COND_NEVER) { 1098 tcg_gen_mov_i32(ret, v2); 1099 } else { 1100 tcg_gen_op6i_i32(INDEX_op_movcond_i32, ret, c1, c2, v1, v2, cond); 1101 } 1102 } 1103 1104 void tcg_gen_add2_i32(TCGv_i32 rl, TCGv_i32 rh, TCGv_i32 al, 1105 TCGv_i32 ah, TCGv_i32 bl, TCGv_i32 bh) 1106 { 1107 if (TCG_TARGET_HAS_add2_i32) { 1108 tcg_gen_op6_i32(INDEX_op_add2_i32, rl, rh, al, ah, bl, bh); 1109 } else { 1110 TCGv_i64 t0 = tcg_temp_ebb_new_i64(); 1111 TCGv_i64 t1 = tcg_temp_ebb_new_i64(); 1112 tcg_gen_concat_i32_i64(t0, al, ah); 1113 tcg_gen_concat_i32_i64(t1, bl, bh); 1114 tcg_gen_add_i64(t0, t0, t1); 1115 tcg_gen_extr_i64_i32(rl, rh, t0); 1116 tcg_temp_free_i64(t0); 1117 tcg_temp_free_i64(t1); 1118 } 1119 } 1120 1121 void tcg_gen_sub2_i32(TCGv_i32 rl, TCGv_i32 rh, TCGv_i32 al, 1122 TCGv_i32 ah, TCGv_i32 bl, TCGv_i32 bh) 1123 { 1124 if (TCG_TARGET_HAS_sub2_i32) { 1125 tcg_gen_op6_i32(INDEX_op_sub2_i32, rl, rh, al, ah, bl, bh); 1126 } else { 1127 TCGv_i64 t0 = tcg_temp_ebb_new_i64(); 1128 TCGv_i64 t1 = tcg_temp_ebb_new_i64(); 1129 tcg_gen_concat_i32_i64(t0, al, ah); 1130 tcg_gen_concat_i32_i64(t1, bl, bh); 1131 tcg_gen_sub_i64(t0, t0, t1); 1132 tcg_gen_extr_i64_i32(rl, rh, t0); 1133 tcg_temp_free_i64(t0); 1134 tcg_temp_free_i64(t1); 1135 } 1136 } 1137 1138 void tcg_gen_mulu2_i32(TCGv_i32 rl, TCGv_i32 rh, TCGv_i32 arg1, TCGv_i32 arg2) 1139 { 1140 if (TCG_TARGET_HAS_mulu2_i32) { 1141 tcg_gen_op4_i32(INDEX_op_mulu2_i32, rl, rh, arg1, arg2); 1142 } else if (tcg_op_supported(INDEX_op_muluh, TCG_TYPE_I32, 0)) { 1143 TCGv_i32 t = tcg_temp_ebb_new_i32(); 1144 tcg_gen_op3_i32(INDEX_op_mul, t, arg1, arg2); 1145 tcg_gen_op3_i32(INDEX_op_muluh, rh, arg1, arg2); 1146 tcg_gen_mov_i32(rl, t); 1147 tcg_temp_free_i32(t); 1148 } else if (TCG_TARGET_REG_BITS == 64) { 1149 TCGv_i64 t0 = tcg_temp_ebb_new_i64(); 1150 TCGv_i64 t1 = tcg_temp_ebb_new_i64(); 1151 tcg_gen_extu_i32_i64(t0, arg1); 1152 tcg_gen_extu_i32_i64(t1, arg2); 1153 tcg_gen_mul_i64(t0, t0, t1); 1154 tcg_gen_extr_i64_i32(rl, rh, t0); 1155 tcg_temp_free_i64(t0); 1156 tcg_temp_free_i64(t1); 1157 } else { 1158 qemu_build_not_reached(); 1159 } 1160 } 1161 1162 void tcg_gen_muls2_i32(TCGv_i32 rl, TCGv_i32 rh, TCGv_i32 arg1, TCGv_i32 arg2) 1163 { 1164 if (TCG_TARGET_HAS_muls2_i32) { 1165 tcg_gen_op4_i32(INDEX_op_muls2_i32, rl, rh, arg1, arg2); 1166 } else if (tcg_op_supported(INDEX_op_mulsh, TCG_TYPE_I32, 0)) { 1167 TCGv_i32 t = tcg_temp_ebb_new_i32(); 1168 tcg_gen_op3_i32(INDEX_op_mul, t, arg1, arg2); 1169 tcg_gen_op3_i32(INDEX_op_mulsh, rh, arg1, arg2); 1170 tcg_gen_mov_i32(rl, t); 1171 tcg_temp_free_i32(t); 1172 } else if (TCG_TARGET_REG_BITS == 32) { 1173 TCGv_i32 t0 = tcg_temp_ebb_new_i32(); 1174 TCGv_i32 t1 = tcg_temp_ebb_new_i32(); 1175 TCGv_i32 t2 = tcg_temp_ebb_new_i32(); 1176 TCGv_i32 t3 = tcg_temp_ebb_new_i32(); 1177 tcg_gen_mulu2_i32(t0, t1, arg1, arg2); 1178 /* Adjust for negative inputs. */ 1179 tcg_gen_sari_i32(t2, arg1, 31); 1180 tcg_gen_sari_i32(t3, arg2, 31); 1181 tcg_gen_and_i32(t2, t2, arg2); 1182 tcg_gen_and_i32(t3, t3, arg1); 1183 tcg_gen_sub_i32(rh, t1, t2); 1184 tcg_gen_sub_i32(rh, rh, t3); 1185 tcg_gen_mov_i32(rl, t0); 1186 tcg_temp_free_i32(t0); 1187 tcg_temp_free_i32(t1); 1188 tcg_temp_free_i32(t2); 1189 tcg_temp_free_i32(t3); 1190 } else { 1191 TCGv_i64 t0 = tcg_temp_ebb_new_i64(); 1192 TCGv_i64 t1 = tcg_temp_ebb_new_i64(); 1193 tcg_gen_ext_i32_i64(t0, arg1); 1194 tcg_gen_ext_i32_i64(t1, arg2); 1195 tcg_gen_mul_i64(t0, t0, t1); 1196 tcg_gen_extr_i64_i32(rl, rh, t0); 1197 tcg_temp_free_i64(t0); 1198 tcg_temp_free_i64(t1); 1199 } 1200 } 1201 1202 void tcg_gen_mulsu2_i32(TCGv_i32 rl, TCGv_i32 rh, TCGv_i32 arg1, TCGv_i32 arg2) 1203 { 1204 if (TCG_TARGET_REG_BITS == 32) { 1205 TCGv_i32 t0 = tcg_temp_ebb_new_i32(); 1206 TCGv_i32 t1 = tcg_temp_ebb_new_i32(); 1207 TCGv_i32 t2 = tcg_temp_ebb_new_i32(); 1208 tcg_gen_mulu2_i32(t0, t1, arg1, arg2); 1209 /* Adjust for negative input for the signed arg1. */ 1210 tcg_gen_sari_i32(t2, arg1, 31); 1211 tcg_gen_and_i32(t2, t2, arg2); 1212 tcg_gen_sub_i32(rh, t1, t2); 1213 tcg_gen_mov_i32(rl, t0); 1214 tcg_temp_free_i32(t0); 1215 tcg_temp_free_i32(t1); 1216 tcg_temp_free_i32(t2); 1217 } else { 1218 TCGv_i64 t0 = tcg_temp_ebb_new_i64(); 1219 TCGv_i64 t1 = tcg_temp_ebb_new_i64(); 1220 tcg_gen_ext_i32_i64(t0, arg1); 1221 tcg_gen_extu_i32_i64(t1, arg2); 1222 tcg_gen_mul_i64(t0, t0, t1); 1223 tcg_gen_extr_i64_i32(rl, rh, t0); 1224 tcg_temp_free_i64(t0); 1225 tcg_temp_free_i64(t1); 1226 } 1227 } 1228 1229 void tcg_gen_ext8s_i32(TCGv_i32 ret, TCGv_i32 arg) 1230 { 1231 tcg_gen_sextract_i32(ret, arg, 0, 8); 1232 } 1233 1234 void tcg_gen_ext16s_i32(TCGv_i32 ret, TCGv_i32 arg) 1235 { 1236 tcg_gen_sextract_i32(ret, arg, 0, 16); 1237 } 1238 1239 void tcg_gen_ext8u_i32(TCGv_i32 ret, TCGv_i32 arg) 1240 { 1241 tcg_gen_extract_i32(ret, arg, 0, 8); 1242 } 1243 1244 void tcg_gen_ext16u_i32(TCGv_i32 ret, TCGv_i32 arg) 1245 { 1246 tcg_gen_extract_i32(ret, arg, 0, 16); 1247 } 1248 1249 /* 1250 * bswap16_i32: 16-bit byte swap on the low bits of a 32-bit value. 1251 * 1252 * Byte pattern: xxab -> yyba 1253 * 1254 * With TCG_BSWAP_IZ, x == zero, else undefined. 1255 * With TCG_BSWAP_OZ, y == zero, with TCG_BSWAP_OS y == sign, else undefined. 1256 */ 1257 void tcg_gen_bswap16_i32(TCGv_i32 ret, TCGv_i32 arg, int flags) 1258 { 1259 /* Only one extension flag may be present. */ 1260 tcg_debug_assert(!(flags & TCG_BSWAP_OS) || !(flags & TCG_BSWAP_OZ)); 1261 1262 if (TCG_TARGET_HAS_bswap16_i32) { 1263 tcg_gen_op3i_i32(INDEX_op_bswap16_i32, ret, arg, flags); 1264 } else { 1265 TCGv_i32 t0 = tcg_temp_ebb_new_i32(); 1266 TCGv_i32 t1 = tcg_temp_ebb_new_i32(); 1267 1268 /* arg = ..ab (IZ) xxab (!IZ) */ 1269 tcg_gen_shri_i32(t0, arg, 8); /* t0 = ...a (IZ) .xxa (!IZ) */ 1270 if (!(flags & TCG_BSWAP_IZ)) { 1271 tcg_gen_ext8u_i32(t0, t0); /* t0 = ...a */ 1272 } 1273 1274 if (flags & TCG_BSWAP_OS) { 1275 tcg_gen_shli_i32(t1, arg, 24); /* t1 = b... */ 1276 tcg_gen_sari_i32(t1, t1, 16); /* t1 = ssb. */ 1277 } else if (flags & TCG_BSWAP_OZ) { 1278 tcg_gen_ext8u_i32(t1, arg); /* t1 = ...b */ 1279 tcg_gen_shli_i32(t1, t1, 8); /* t1 = ..b. */ 1280 } else { 1281 tcg_gen_shli_i32(t1, arg, 8); /* t1 = xab. */ 1282 } 1283 1284 tcg_gen_or_i32(ret, t0, t1); /* ret = ..ba (OZ) */ 1285 /* = ssba (OS) */ 1286 /* = xaba (no flag) */ 1287 tcg_temp_free_i32(t0); 1288 tcg_temp_free_i32(t1); 1289 } 1290 } 1291 1292 /* 1293 * bswap32_i32: 32-bit byte swap on a 32-bit value. 1294 * 1295 * Byte pattern: abcd -> dcba 1296 */ 1297 void tcg_gen_bswap32_i32(TCGv_i32 ret, TCGv_i32 arg) 1298 { 1299 if (TCG_TARGET_HAS_bswap32_i32) { 1300 tcg_gen_op3i_i32(INDEX_op_bswap32_i32, ret, arg, 0); 1301 } else { 1302 TCGv_i32 t0 = tcg_temp_ebb_new_i32(); 1303 TCGv_i32 t1 = tcg_temp_ebb_new_i32(); 1304 TCGv_i32 t2 = tcg_constant_i32(0x00ff00ff); 1305 1306 /* arg = abcd */ 1307 tcg_gen_shri_i32(t0, arg, 8); /* t0 = .abc */ 1308 tcg_gen_and_i32(t1, arg, t2); /* t1 = .b.d */ 1309 tcg_gen_and_i32(t0, t0, t2); /* t0 = .a.c */ 1310 tcg_gen_shli_i32(t1, t1, 8); /* t1 = b.d. */ 1311 tcg_gen_or_i32(ret, t0, t1); /* ret = badc */ 1312 1313 tcg_gen_shri_i32(t0, ret, 16); /* t0 = ..ba */ 1314 tcg_gen_shli_i32(t1, ret, 16); /* t1 = dc.. */ 1315 tcg_gen_or_i32(ret, t0, t1); /* ret = dcba */ 1316 1317 tcg_temp_free_i32(t0); 1318 tcg_temp_free_i32(t1); 1319 } 1320 } 1321 1322 /* 1323 * hswap_i32: Swap 16-bit halfwords within a 32-bit value. 1324 * 1325 * Byte pattern: abcd -> cdab 1326 */ 1327 void tcg_gen_hswap_i32(TCGv_i32 ret, TCGv_i32 arg) 1328 { 1329 /* Swapping 2 16-bit elements is a rotate. */ 1330 tcg_gen_rotli_i32(ret, arg, 16); 1331 } 1332 1333 void tcg_gen_smin_i32(TCGv_i32 ret, TCGv_i32 a, TCGv_i32 b) 1334 { 1335 tcg_gen_movcond_i32(TCG_COND_LT, ret, a, b, a, b); 1336 } 1337 1338 void tcg_gen_umin_i32(TCGv_i32 ret, TCGv_i32 a, TCGv_i32 b) 1339 { 1340 tcg_gen_movcond_i32(TCG_COND_LTU, ret, a, b, a, b); 1341 } 1342 1343 void tcg_gen_smax_i32(TCGv_i32 ret, TCGv_i32 a, TCGv_i32 b) 1344 { 1345 tcg_gen_movcond_i32(TCG_COND_LT, ret, a, b, b, a); 1346 } 1347 1348 void tcg_gen_umax_i32(TCGv_i32 ret, TCGv_i32 a, TCGv_i32 b) 1349 { 1350 tcg_gen_movcond_i32(TCG_COND_LTU, ret, a, b, b, a); 1351 } 1352 1353 void tcg_gen_abs_i32(TCGv_i32 ret, TCGv_i32 a) 1354 { 1355 TCGv_i32 t = tcg_temp_ebb_new_i32(); 1356 1357 tcg_gen_sari_i32(t, a, 31); 1358 tcg_gen_xor_i32(ret, a, t); 1359 tcg_gen_sub_i32(ret, ret, t); 1360 tcg_temp_free_i32(t); 1361 } 1362 1363 void tcg_gen_ld8u_i32(TCGv_i32 ret, TCGv_ptr arg2, tcg_target_long offset) 1364 { 1365 tcg_gen_ldst_op_i32(INDEX_op_ld8u_i32, ret, arg2, offset); 1366 } 1367 1368 void tcg_gen_ld8s_i32(TCGv_i32 ret, TCGv_ptr arg2, tcg_target_long offset) 1369 { 1370 tcg_gen_ldst_op_i32(INDEX_op_ld8s_i32, ret, arg2, offset); 1371 } 1372 1373 void tcg_gen_ld16u_i32(TCGv_i32 ret, TCGv_ptr arg2, tcg_target_long offset) 1374 { 1375 tcg_gen_ldst_op_i32(INDEX_op_ld16u_i32, ret, arg2, offset); 1376 } 1377 1378 void tcg_gen_ld16s_i32(TCGv_i32 ret, TCGv_ptr arg2, tcg_target_long offset) 1379 { 1380 tcg_gen_ldst_op_i32(INDEX_op_ld16s_i32, ret, arg2, offset); 1381 } 1382 1383 void tcg_gen_ld_i32(TCGv_i32 ret, TCGv_ptr arg2, tcg_target_long offset) 1384 { 1385 tcg_gen_ldst_op_i32(INDEX_op_ld_i32, ret, arg2, offset); 1386 } 1387 1388 void tcg_gen_st8_i32(TCGv_i32 arg1, TCGv_ptr arg2, tcg_target_long offset) 1389 { 1390 tcg_gen_ldst_op_i32(INDEX_op_st8_i32, arg1, arg2, offset); 1391 } 1392 1393 void tcg_gen_st16_i32(TCGv_i32 arg1, TCGv_ptr arg2, tcg_target_long offset) 1394 { 1395 tcg_gen_ldst_op_i32(INDEX_op_st16_i32, arg1, arg2, offset); 1396 } 1397 1398 void tcg_gen_st_i32(TCGv_i32 arg1, TCGv_ptr arg2, tcg_target_long offset) 1399 { 1400 tcg_gen_ldst_op_i32(INDEX_op_st_i32, arg1, arg2, offset); 1401 } 1402 1403 1404 /* 64-bit ops */ 1405 1406 void tcg_gen_discard_i64(TCGv_i64 arg) 1407 { 1408 if (TCG_TARGET_REG_BITS == 64) { 1409 tcg_gen_op1_i64(INDEX_op_discard, TCG_TYPE_I64, arg); 1410 } else { 1411 tcg_gen_discard_i32(TCGV_LOW(arg)); 1412 tcg_gen_discard_i32(TCGV_HIGH(arg)); 1413 } 1414 } 1415 1416 void tcg_gen_mov_i64(TCGv_i64 ret, TCGv_i64 arg) 1417 { 1418 if (ret == arg) { 1419 return; 1420 } 1421 if (TCG_TARGET_REG_BITS == 64) { 1422 tcg_gen_op2_i64(INDEX_op_mov, ret, arg); 1423 } else { 1424 TCGTemp *ts = tcgv_i64_temp(arg); 1425 1426 /* Canonicalize TCGv_i64 TEMP_CONST into TCGv_i32 TEMP_CONST. */ 1427 if (ts->kind == TEMP_CONST) { 1428 tcg_gen_movi_i64(ret, ts->val); 1429 } else { 1430 tcg_gen_mov_i32(TCGV_LOW(ret), TCGV_LOW(arg)); 1431 tcg_gen_mov_i32(TCGV_HIGH(ret), TCGV_HIGH(arg)); 1432 } 1433 } 1434 } 1435 1436 void tcg_gen_movi_i64(TCGv_i64 ret, int64_t arg) 1437 { 1438 if (TCG_TARGET_REG_BITS == 64) { 1439 tcg_gen_mov_i64(ret, tcg_constant_i64(arg)); 1440 } else { 1441 tcg_gen_movi_i32(TCGV_LOW(ret), arg); 1442 tcg_gen_movi_i32(TCGV_HIGH(ret), arg >> 32); 1443 } 1444 } 1445 1446 void tcg_gen_ld8u_i64(TCGv_i64 ret, TCGv_ptr arg2, tcg_target_long offset) 1447 { 1448 if (TCG_TARGET_REG_BITS == 64) { 1449 tcg_gen_ldst_op_i64(INDEX_op_ld8u_i64, ret, arg2, offset); 1450 } else { 1451 tcg_gen_ld8u_i32(TCGV_LOW(ret), arg2, offset); 1452 tcg_gen_movi_i32(TCGV_HIGH(ret), 0); 1453 } 1454 } 1455 1456 void tcg_gen_ld8s_i64(TCGv_i64 ret, TCGv_ptr arg2, tcg_target_long offset) 1457 { 1458 if (TCG_TARGET_REG_BITS == 64) { 1459 tcg_gen_ldst_op_i64(INDEX_op_ld8s_i64, ret, arg2, offset); 1460 } else { 1461 tcg_gen_ld8s_i32(TCGV_LOW(ret), arg2, offset); 1462 tcg_gen_sari_i32(TCGV_HIGH(ret), TCGV_LOW(ret), 31); 1463 } 1464 } 1465 1466 void tcg_gen_ld16u_i64(TCGv_i64 ret, TCGv_ptr arg2, tcg_target_long offset) 1467 { 1468 if (TCG_TARGET_REG_BITS == 64) { 1469 tcg_gen_ldst_op_i64(INDEX_op_ld16u_i64, ret, arg2, offset); 1470 } else { 1471 tcg_gen_ld16u_i32(TCGV_LOW(ret), arg2, offset); 1472 tcg_gen_movi_i32(TCGV_HIGH(ret), 0); 1473 } 1474 } 1475 1476 void tcg_gen_ld16s_i64(TCGv_i64 ret, TCGv_ptr arg2, tcg_target_long offset) 1477 { 1478 if (TCG_TARGET_REG_BITS == 64) { 1479 tcg_gen_ldst_op_i64(INDEX_op_ld16s_i64, ret, arg2, offset); 1480 } else { 1481 tcg_gen_ld16s_i32(TCGV_LOW(ret), arg2, offset); 1482 tcg_gen_sari_i32(TCGV_HIGH(ret), TCGV_LOW(ret), 31); 1483 } 1484 } 1485 1486 void tcg_gen_ld32u_i64(TCGv_i64 ret, TCGv_ptr arg2, tcg_target_long offset) 1487 { 1488 if (TCG_TARGET_REG_BITS == 64) { 1489 tcg_gen_ldst_op_i64(INDEX_op_ld32u_i64, ret, arg2, offset); 1490 } else { 1491 tcg_gen_ld_i32(TCGV_LOW(ret), arg2, offset); 1492 tcg_gen_movi_i32(TCGV_HIGH(ret), 0); 1493 } 1494 } 1495 1496 void tcg_gen_ld32s_i64(TCGv_i64 ret, TCGv_ptr arg2, tcg_target_long offset) 1497 { 1498 if (TCG_TARGET_REG_BITS == 64) { 1499 tcg_gen_ldst_op_i64(INDEX_op_ld32s_i64, ret, arg2, offset); 1500 } else { 1501 tcg_gen_ld_i32(TCGV_LOW(ret), arg2, offset); 1502 tcg_gen_sari_i32(TCGV_HIGH(ret), TCGV_LOW(ret), 31); 1503 } 1504 } 1505 1506 void tcg_gen_ld_i64(TCGv_i64 ret, TCGv_ptr arg2, tcg_target_long offset) 1507 { 1508 /* 1509 * For 32-bit host, since arg2 and ret have different types, 1510 * they cannot be the same temporary -- no chance of overlap. 1511 */ 1512 if (TCG_TARGET_REG_BITS == 64) { 1513 tcg_gen_ldst_op_i64(INDEX_op_ld_i64, ret, arg2, offset); 1514 } else if (HOST_BIG_ENDIAN) { 1515 tcg_gen_ld_i32(TCGV_HIGH(ret), arg2, offset); 1516 tcg_gen_ld_i32(TCGV_LOW(ret), arg2, offset + 4); 1517 } else { 1518 tcg_gen_ld_i32(TCGV_LOW(ret), arg2, offset); 1519 tcg_gen_ld_i32(TCGV_HIGH(ret), arg2, offset + 4); 1520 } 1521 } 1522 1523 void tcg_gen_st8_i64(TCGv_i64 arg1, TCGv_ptr arg2, tcg_target_long offset) 1524 { 1525 if (TCG_TARGET_REG_BITS == 64) { 1526 tcg_gen_ldst_op_i64(INDEX_op_st8_i64, arg1, arg2, offset); 1527 } else { 1528 tcg_gen_st8_i32(TCGV_LOW(arg1), arg2, offset); 1529 } 1530 } 1531 1532 void tcg_gen_st16_i64(TCGv_i64 arg1, TCGv_ptr arg2, tcg_target_long offset) 1533 { 1534 if (TCG_TARGET_REG_BITS == 64) { 1535 tcg_gen_ldst_op_i64(INDEX_op_st16_i64, arg1, arg2, offset); 1536 } else { 1537 tcg_gen_st16_i32(TCGV_LOW(arg1), arg2, offset); 1538 } 1539 } 1540 1541 void tcg_gen_st32_i64(TCGv_i64 arg1, TCGv_ptr arg2, tcg_target_long offset) 1542 { 1543 if (TCG_TARGET_REG_BITS == 64) { 1544 tcg_gen_ldst_op_i64(INDEX_op_st32_i64, arg1, arg2, offset); 1545 } else { 1546 tcg_gen_st_i32(TCGV_LOW(arg1), arg2, offset); 1547 } 1548 } 1549 1550 void tcg_gen_st_i64(TCGv_i64 arg1, TCGv_ptr arg2, tcg_target_long offset) 1551 { 1552 if (TCG_TARGET_REG_BITS == 64) { 1553 tcg_gen_ldst_op_i64(INDEX_op_st_i64, arg1, arg2, offset); 1554 } else if (HOST_BIG_ENDIAN) { 1555 tcg_gen_st_i32(TCGV_HIGH(arg1), arg2, offset); 1556 tcg_gen_st_i32(TCGV_LOW(arg1), arg2, offset + 4); 1557 } else { 1558 tcg_gen_st_i32(TCGV_LOW(arg1), arg2, offset); 1559 tcg_gen_st_i32(TCGV_HIGH(arg1), arg2, offset + 4); 1560 } 1561 } 1562 1563 void tcg_gen_add_i64(TCGv_i64 ret, TCGv_i64 arg1, TCGv_i64 arg2) 1564 { 1565 if (TCG_TARGET_REG_BITS == 64) { 1566 tcg_gen_op3_i64(INDEX_op_add, ret, arg1, arg2); 1567 } else { 1568 tcg_gen_add2_i32(TCGV_LOW(ret), TCGV_HIGH(ret), TCGV_LOW(arg1), 1569 TCGV_HIGH(arg1), TCGV_LOW(arg2), TCGV_HIGH(arg2)); 1570 } 1571 } 1572 1573 void tcg_gen_sub_i64(TCGv_i64 ret, TCGv_i64 arg1, TCGv_i64 arg2) 1574 { 1575 if (TCG_TARGET_REG_BITS == 64) { 1576 tcg_gen_op3_i64(INDEX_op_sub, ret, arg1, arg2); 1577 } else { 1578 tcg_gen_sub2_i32(TCGV_LOW(ret), TCGV_HIGH(ret), TCGV_LOW(arg1), 1579 TCGV_HIGH(arg1), TCGV_LOW(arg2), TCGV_HIGH(arg2)); 1580 } 1581 } 1582 1583 void tcg_gen_and_i64(TCGv_i64 ret, TCGv_i64 arg1, TCGv_i64 arg2) 1584 { 1585 if (TCG_TARGET_REG_BITS == 64) { 1586 tcg_gen_op3_i64(INDEX_op_and, ret, arg1, arg2); 1587 } else { 1588 tcg_gen_and_i32(TCGV_LOW(ret), TCGV_LOW(arg1), TCGV_LOW(arg2)); 1589 tcg_gen_and_i32(TCGV_HIGH(ret), TCGV_HIGH(arg1), TCGV_HIGH(arg2)); 1590 } 1591 } 1592 1593 void tcg_gen_or_i64(TCGv_i64 ret, TCGv_i64 arg1, TCGv_i64 arg2) 1594 { 1595 if (TCG_TARGET_REG_BITS == 64) { 1596 tcg_gen_op3_i64(INDEX_op_or, ret, arg1, arg2); 1597 } else { 1598 tcg_gen_or_i32(TCGV_LOW(ret), TCGV_LOW(arg1), TCGV_LOW(arg2)); 1599 tcg_gen_or_i32(TCGV_HIGH(ret), TCGV_HIGH(arg1), TCGV_HIGH(arg2)); 1600 } 1601 } 1602 1603 void tcg_gen_xor_i64(TCGv_i64 ret, TCGv_i64 arg1, TCGv_i64 arg2) 1604 { 1605 if (TCG_TARGET_REG_BITS == 64) { 1606 tcg_gen_op3_i64(INDEX_op_xor, ret, arg1, arg2); 1607 } else { 1608 tcg_gen_xor_i32(TCGV_LOW(ret), TCGV_LOW(arg1), TCGV_LOW(arg2)); 1609 tcg_gen_xor_i32(TCGV_HIGH(ret), TCGV_HIGH(arg1), TCGV_HIGH(arg2)); 1610 } 1611 } 1612 1613 void tcg_gen_shl_i64(TCGv_i64 ret, TCGv_i64 arg1, TCGv_i64 arg2) 1614 { 1615 if (TCG_TARGET_REG_BITS == 64) { 1616 tcg_gen_op3_i64(INDEX_op_shl, ret, arg1, arg2); 1617 } else { 1618 gen_helper_shl_i64(ret, arg1, arg2); 1619 } 1620 } 1621 1622 void tcg_gen_shr_i64(TCGv_i64 ret, TCGv_i64 arg1, TCGv_i64 arg2) 1623 { 1624 if (TCG_TARGET_REG_BITS == 64) { 1625 tcg_gen_op3_i64(INDEX_op_shr, ret, arg1, arg2); 1626 } else { 1627 gen_helper_shr_i64(ret, arg1, arg2); 1628 } 1629 } 1630 1631 void tcg_gen_sar_i64(TCGv_i64 ret, TCGv_i64 arg1, TCGv_i64 arg2) 1632 { 1633 if (TCG_TARGET_REG_BITS == 64) { 1634 tcg_gen_op3_i64(INDEX_op_sar, ret, arg1, arg2); 1635 } else { 1636 gen_helper_sar_i64(ret, arg1, arg2); 1637 } 1638 } 1639 1640 void tcg_gen_mul_i64(TCGv_i64 ret, TCGv_i64 arg1, TCGv_i64 arg2) 1641 { 1642 TCGv_i64 t0; 1643 TCGv_i32 t1; 1644 1645 if (TCG_TARGET_REG_BITS == 64) { 1646 tcg_gen_op3_i64(INDEX_op_mul, ret, arg1, arg2); 1647 return; 1648 } 1649 1650 1651 t0 = tcg_temp_ebb_new_i64(); 1652 t1 = tcg_temp_ebb_new_i32(); 1653 1654 tcg_gen_mulu2_i32(TCGV_LOW(t0), TCGV_HIGH(t0), 1655 TCGV_LOW(arg1), TCGV_LOW(arg2)); 1656 1657 tcg_gen_mul_i32(t1, TCGV_LOW(arg1), TCGV_HIGH(arg2)); 1658 tcg_gen_add_i32(TCGV_HIGH(t0), TCGV_HIGH(t0), t1); 1659 tcg_gen_mul_i32(t1, TCGV_HIGH(arg1), TCGV_LOW(arg2)); 1660 tcg_gen_add_i32(TCGV_HIGH(t0), TCGV_HIGH(t0), t1); 1661 1662 tcg_gen_mov_i64(ret, t0); 1663 tcg_temp_free_i64(t0); 1664 tcg_temp_free_i32(t1); 1665 } 1666 1667 void tcg_gen_addi_i64(TCGv_i64 ret, TCGv_i64 arg1, int64_t arg2) 1668 { 1669 /* some cases can be optimized here */ 1670 if (arg2 == 0) { 1671 tcg_gen_mov_i64(ret, arg1); 1672 } else if (TCG_TARGET_REG_BITS == 64) { 1673 tcg_gen_add_i64(ret, arg1, tcg_constant_i64(arg2)); 1674 } else { 1675 tcg_gen_add2_i32(TCGV_LOW(ret), TCGV_HIGH(ret), 1676 TCGV_LOW(arg1), TCGV_HIGH(arg1), 1677 tcg_constant_i32(arg2), tcg_constant_i32(arg2 >> 32)); 1678 } 1679 } 1680 1681 void tcg_gen_subfi_i64(TCGv_i64 ret, int64_t arg1, TCGv_i64 arg2) 1682 { 1683 if (arg1 == 0) { 1684 tcg_gen_neg_i64(ret, arg2); 1685 } else if (TCG_TARGET_REG_BITS == 64) { 1686 tcg_gen_sub_i64(ret, tcg_constant_i64(arg1), arg2); 1687 } else { 1688 tcg_gen_sub2_i32(TCGV_LOW(ret), TCGV_HIGH(ret), 1689 tcg_constant_i32(arg1), tcg_constant_i32(arg1 >> 32), 1690 TCGV_LOW(arg2), TCGV_HIGH(arg2)); 1691 } 1692 } 1693 1694 void tcg_gen_subi_i64(TCGv_i64 ret, TCGv_i64 arg1, int64_t arg2) 1695 { 1696 tcg_gen_addi_i64(ret, arg1, -arg2); 1697 } 1698 1699 void tcg_gen_neg_i64(TCGv_i64 ret, TCGv_i64 arg) 1700 { 1701 if (TCG_TARGET_REG_BITS == 64) { 1702 tcg_gen_op2_i64(INDEX_op_neg, ret, arg); 1703 } else { 1704 TCGv_i32 zero = tcg_constant_i32(0); 1705 tcg_gen_sub2_i32(TCGV_LOW(ret), TCGV_HIGH(ret), 1706 zero, zero, TCGV_LOW(arg), TCGV_HIGH(arg)); 1707 } 1708 } 1709 1710 void tcg_gen_andi_i64(TCGv_i64 ret, TCGv_i64 arg1, int64_t arg2) 1711 { 1712 if (TCG_TARGET_REG_BITS == 32) { 1713 tcg_gen_andi_i32(TCGV_LOW(ret), TCGV_LOW(arg1), arg2); 1714 tcg_gen_andi_i32(TCGV_HIGH(ret), TCGV_HIGH(arg1), arg2 >> 32); 1715 return; 1716 } 1717 1718 /* Some cases can be optimized here. */ 1719 switch (arg2) { 1720 case 0: 1721 tcg_gen_movi_i64(ret, 0); 1722 return; 1723 case -1: 1724 tcg_gen_mov_i64(ret, arg1); 1725 return; 1726 default: 1727 /* 1728 * Canonicalize on extract, if valid. This aids x86 with its 1729 * 2 operand MOVZBL and 2 operand AND, selecting the TCGOpcode 1730 * which does not require matching operands. Other backends can 1731 * trivially expand the extract to AND during code generation. 1732 */ 1733 if (!(arg2 & (arg2 + 1))) { 1734 unsigned len = ctz64(~arg2); 1735 if (TCG_TARGET_extract_valid(TCG_TYPE_I64, 0, len)) { 1736 tcg_gen_extract_i64(ret, arg1, 0, len); 1737 return; 1738 } 1739 } 1740 break; 1741 } 1742 1743 tcg_gen_and_i64(ret, arg1, tcg_constant_i64(arg2)); 1744 } 1745 1746 void tcg_gen_ori_i64(TCGv_i64 ret, TCGv_i64 arg1, int64_t arg2) 1747 { 1748 if (TCG_TARGET_REG_BITS == 32) { 1749 tcg_gen_ori_i32(TCGV_LOW(ret), TCGV_LOW(arg1), arg2); 1750 tcg_gen_ori_i32(TCGV_HIGH(ret), TCGV_HIGH(arg1), arg2 >> 32); 1751 return; 1752 } 1753 /* Some cases can be optimized here. */ 1754 if (arg2 == -1) { 1755 tcg_gen_movi_i64(ret, -1); 1756 } else if (arg2 == 0) { 1757 tcg_gen_mov_i64(ret, arg1); 1758 } else { 1759 tcg_gen_or_i64(ret, arg1, tcg_constant_i64(arg2)); 1760 } 1761 } 1762 1763 void tcg_gen_xori_i64(TCGv_i64 ret, TCGv_i64 arg1, int64_t arg2) 1764 { 1765 if (TCG_TARGET_REG_BITS == 32) { 1766 tcg_gen_xori_i32(TCGV_LOW(ret), TCGV_LOW(arg1), arg2); 1767 tcg_gen_xori_i32(TCGV_HIGH(ret), TCGV_HIGH(arg1), arg2 >> 32); 1768 return; 1769 } 1770 /* Some cases can be optimized here. */ 1771 if (arg2 == 0) { 1772 tcg_gen_mov_i64(ret, arg1); 1773 } else if (arg2 == -1 && 1774 tcg_op_supported(INDEX_op_not, TCG_TYPE_I64, 0)) { 1775 /* Don't recurse with tcg_gen_not_i64. */ 1776 tcg_gen_op2_i64(INDEX_op_not, ret, arg1); 1777 } else { 1778 tcg_gen_xor_i64(ret, arg1, tcg_constant_i64(arg2)); 1779 } 1780 } 1781 1782 static inline void tcg_gen_shifti_i64(TCGv_i64 ret, TCGv_i64 arg1, 1783 unsigned c, bool right, bool arith) 1784 { 1785 tcg_debug_assert(c < 64); 1786 if (c == 0) { 1787 tcg_gen_mov_i32(TCGV_LOW(ret), TCGV_LOW(arg1)); 1788 tcg_gen_mov_i32(TCGV_HIGH(ret), TCGV_HIGH(arg1)); 1789 } else if (c >= 32) { 1790 c -= 32; 1791 if (right) { 1792 if (arith) { 1793 tcg_gen_sari_i32(TCGV_LOW(ret), TCGV_HIGH(arg1), c); 1794 tcg_gen_sari_i32(TCGV_HIGH(ret), TCGV_HIGH(arg1), 31); 1795 } else { 1796 tcg_gen_shri_i32(TCGV_LOW(ret), TCGV_HIGH(arg1), c); 1797 tcg_gen_movi_i32(TCGV_HIGH(ret), 0); 1798 } 1799 } else { 1800 tcg_gen_shli_i32(TCGV_HIGH(ret), TCGV_LOW(arg1), c); 1801 tcg_gen_movi_i32(TCGV_LOW(ret), 0); 1802 } 1803 } else if (right) { 1804 if (TCG_TARGET_HAS_extract2_i32) { 1805 tcg_gen_extract2_i32(TCGV_LOW(ret), 1806 TCGV_LOW(arg1), TCGV_HIGH(arg1), c); 1807 } else { 1808 tcg_gen_shri_i32(TCGV_LOW(ret), TCGV_LOW(arg1), c); 1809 tcg_gen_deposit_i32(TCGV_LOW(ret), TCGV_LOW(ret), 1810 TCGV_HIGH(arg1), 32 - c, c); 1811 } 1812 if (arith) { 1813 tcg_gen_sari_i32(TCGV_HIGH(ret), TCGV_HIGH(arg1), c); 1814 } else { 1815 tcg_gen_shri_i32(TCGV_HIGH(ret), TCGV_HIGH(arg1), c); 1816 } 1817 } else { 1818 if (TCG_TARGET_HAS_extract2_i32) { 1819 tcg_gen_extract2_i32(TCGV_HIGH(ret), 1820 TCGV_LOW(arg1), TCGV_HIGH(arg1), 32 - c); 1821 } else { 1822 TCGv_i32 t0 = tcg_temp_ebb_new_i32(); 1823 tcg_gen_shri_i32(t0, TCGV_LOW(arg1), 32 - c); 1824 tcg_gen_deposit_i32(TCGV_HIGH(ret), t0, 1825 TCGV_HIGH(arg1), c, 32 - c); 1826 tcg_temp_free_i32(t0); 1827 } 1828 tcg_gen_shli_i32(TCGV_LOW(ret), TCGV_LOW(arg1), c); 1829 } 1830 } 1831 1832 void tcg_gen_shli_i64(TCGv_i64 ret, TCGv_i64 arg1, int64_t arg2) 1833 { 1834 tcg_debug_assert(arg2 >= 0 && arg2 < 64); 1835 if (TCG_TARGET_REG_BITS == 32) { 1836 tcg_gen_shifti_i64(ret, arg1, arg2, 0, 0); 1837 } else if (arg2 == 0) { 1838 tcg_gen_mov_i64(ret, arg1); 1839 } else { 1840 tcg_gen_shl_i64(ret, arg1, tcg_constant_i64(arg2)); 1841 } 1842 } 1843 1844 void tcg_gen_shri_i64(TCGv_i64 ret, TCGv_i64 arg1, int64_t arg2) 1845 { 1846 tcg_debug_assert(arg2 >= 0 && arg2 < 64); 1847 if (TCG_TARGET_REG_BITS == 32) { 1848 tcg_gen_shifti_i64(ret, arg1, arg2, 1, 0); 1849 } else if (arg2 == 0) { 1850 tcg_gen_mov_i64(ret, arg1); 1851 } else { 1852 tcg_gen_shr_i64(ret, arg1, tcg_constant_i64(arg2)); 1853 } 1854 } 1855 1856 void tcg_gen_sari_i64(TCGv_i64 ret, TCGv_i64 arg1, int64_t arg2) 1857 { 1858 tcg_debug_assert(arg2 >= 0 && arg2 < 64); 1859 if (TCG_TARGET_REG_BITS == 32) { 1860 tcg_gen_shifti_i64(ret, arg1, arg2, 1, 1); 1861 } else if (arg2 == 0) { 1862 tcg_gen_mov_i64(ret, arg1); 1863 } else { 1864 tcg_gen_sar_i64(ret, arg1, tcg_constant_i64(arg2)); 1865 } 1866 } 1867 1868 void tcg_gen_brcond_i64(TCGCond cond, TCGv_i64 arg1, TCGv_i64 arg2, TCGLabel *l) 1869 { 1870 if (cond == TCG_COND_ALWAYS) { 1871 tcg_gen_br(l); 1872 } else if (cond != TCG_COND_NEVER) { 1873 TCGOp *op; 1874 if (TCG_TARGET_REG_BITS == 32) { 1875 op = tcg_gen_op6ii_i32(INDEX_op_brcond2_i32, TCGV_LOW(arg1), 1876 TCGV_HIGH(arg1), TCGV_LOW(arg2), 1877 TCGV_HIGH(arg2), cond, label_arg(l)); 1878 } else { 1879 op = tcg_gen_op4ii_i64(INDEX_op_brcond_i64, arg1, arg2, cond, 1880 label_arg(l)); 1881 } 1882 add_as_label_use(l, op); 1883 } 1884 } 1885 1886 void tcg_gen_brcondi_i64(TCGCond cond, TCGv_i64 arg1, int64_t arg2, TCGLabel *l) 1887 { 1888 if (TCG_TARGET_REG_BITS == 64) { 1889 tcg_gen_brcond_i64(cond, arg1, tcg_constant_i64(arg2), l); 1890 } else if (cond == TCG_COND_ALWAYS) { 1891 tcg_gen_br(l); 1892 } else if (cond != TCG_COND_NEVER) { 1893 TCGOp *op = tcg_gen_op6ii_i32(INDEX_op_brcond2_i32, 1894 TCGV_LOW(arg1), TCGV_HIGH(arg1), 1895 tcg_constant_i32(arg2), 1896 tcg_constant_i32(arg2 >> 32), 1897 cond, label_arg(l)); 1898 add_as_label_use(l, op); 1899 } 1900 } 1901 1902 void tcg_gen_setcond_i64(TCGCond cond, TCGv_i64 ret, 1903 TCGv_i64 arg1, TCGv_i64 arg2) 1904 { 1905 if (cond == TCG_COND_ALWAYS) { 1906 tcg_gen_movi_i64(ret, 1); 1907 } else if (cond == TCG_COND_NEVER) { 1908 tcg_gen_movi_i64(ret, 0); 1909 } else { 1910 if (TCG_TARGET_REG_BITS == 32) { 1911 tcg_gen_op6i_i32(INDEX_op_setcond2_i32, TCGV_LOW(ret), 1912 TCGV_LOW(arg1), TCGV_HIGH(arg1), 1913 TCGV_LOW(arg2), TCGV_HIGH(arg2), cond); 1914 tcg_gen_movi_i32(TCGV_HIGH(ret), 0); 1915 } else { 1916 tcg_gen_op4i_i64(INDEX_op_setcond_i64, ret, arg1, arg2, cond); 1917 } 1918 } 1919 } 1920 1921 void tcg_gen_setcondi_i64(TCGCond cond, TCGv_i64 ret, 1922 TCGv_i64 arg1, int64_t arg2) 1923 { 1924 if (TCG_TARGET_REG_BITS == 64) { 1925 tcg_gen_setcond_i64(cond, ret, arg1, tcg_constant_i64(arg2)); 1926 } else if (cond == TCG_COND_ALWAYS) { 1927 tcg_gen_movi_i64(ret, 1); 1928 } else if (cond == TCG_COND_NEVER) { 1929 tcg_gen_movi_i64(ret, 0); 1930 } else { 1931 tcg_gen_op6i_i32(INDEX_op_setcond2_i32, TCGV_LOW(ret), 1932 TCGV_LOW(arg1), TCGV_HIGH(arg1), 1933 tcg_constant_i32(arg2), 1934 tcg_constant_i32(arg2 >> 32), cond); 1935 tcg_gen_movi_i32(TCGV_HIGH(ret), 0); 1936 } 1937 } 1938 1939 void tcg_gen_negsetcondi_i64(TCGCond cond, TCGv_i64 ret, 1940 TCGv_i64 arg1, int64_t arg2) 1941 { 1942 tcg_gen_negsetcond_i64(cond, ret, arg1, tcg_constant_i64(arg2)); 1943 } 1944 1945 void tcg_gen_negsetcond_i64(TCGCond cond, TCGv_i64 ret, 1946 TCGv_i64 arg1, TCGv_i64 arg2) 1947 { 1948 if (cond == TCG_COND_ALWAYS) { 1949 tcg_gen_movi_i64(ret, -1); 1950 } else if (cond == TCG_COND_NEVER) { 1951 tcg_gen_movi_i64(ret, 0); 1952 } else if (TCG_TARGET_HAS_negsetcond_i64) { 1953 tcg_gen_op4i_i64(INDEX_op_negsetcond_i64, ret, arg1, arg2, cond); 1954 } else if (TCG_TARGET_REG_BITS == 32) { 1955 tcg_gen_op6i_i32(INDEX_op_setcond2_i32, TCGV_LOW(ret), 1956 TCGV_LOW(arg1), TCGV_HIGH(arg1), 1957 TCGV_LOW(arg2), TCGV_HIGH(arg2), cond); 1958 tcg_gen_neg_i32(TCGV_LOW(ret), TCGV_LOW(ret)); 1959 tcg_gen_mov_i32(TCGV_HIGH(ret), TCGV_LOW(ret)); 1960 } else { 1961 tcg_gen_setcond_i64(cond, ret, arg1, arg2); 1962 tcg_gen_neg_i64(ret, ret); 1963 } 1964 } 1965 1966 void tcg_gen_muli_i64(TCGv_i64 ret, TCGv_i64 arg1, int64_t arg2) 1967 { 1968 if (arg2 == 0) { 1969 tcg_gen_movi_i64(ret, 0); 1970 } else if (is_power_of_2(arg2)) { 1971 tcg_gen_shli_i64(ret, arg1, ctz64(arg2)); 1972 } else { 1973 tcg_gen_mul_i64(ret, arg1, tcg_constant_i64(arg2)); 1974 } 1975 } 1976 1977 void tcg_gen_div_i64(TCGv_i64 ret, TCGv_i64 arg1, TCGv_i64 arg2) 1978 { 1979 if (tcg_op_supported(INDEX_op_divs, TCG_TYPE_I64, 0)) { 1980 tcg_gen_op3_i64(INDEX_op_divs, ret, arg1, arg2); 1981 } else if (tcg_op_supported(INDEX_op_divs2, TCG_TYPE_I64, 0)) { 1982 TCGv_i64 t0 = tcg_temp_ebb_new_i64(); 1983 tcg_gen_sari_i64(t0, arg1, 63); 1984 tcg_gen_op5_i64(INDEX_op_divs2, ret, t0, arg1, t0, arg2); 1985 tcg_temp_free_i64(t0); 1986 } else { 1987 gen_helper_div_i64(ret, arg1, arg2); 1988 } 1989 } 1990 1991 void tcg_gen_rem_i64(TCGv_i64 ret, TCGv_i64 arg1, TCGv_i64 arg2) 1992 { 1993 if (tcg_op_supported(INDEX_op_rems, TCG_TYPE_I64, 0)) { 1994 tcg_gen_op3_i64(INDEX_op_rems, ret, arg1, arg2); 1995 } else if (tcg_op_supported(INDEX_op_divs, TCG_TYPE_I64, 0)) { 1996 TCGv_i64 t0 = tcg_temp_ebb_new_i64(); 1997 tcg_gen_op3_i64(INDEX_op_divs, t0, arg1, arg2); 1998 tcg_gen_mul_i64(t0, t0, arg2); 1999 tcg_gen_sub_i64(ret, arg1, t0); 2000 tcg_temp_free_i64(t0); 2001 } else if (tcg_op_supported(INDEX_op_divs2, TCG_TYPE_I64, 0)) { 2002 TCGv_i64 t0 = tcg_temp_ebb_new_i64(); 2003 tcg_gen_sari_i64(t0, arg1, 63); 2004 tcg_gen_op5_i64(INDEX_op_divs2, t0, ret, arg1, t0, arg2); 2005 tcg_temp_free_i64(t0); 2006 } else { 2007 gen_helper_rem_i64(ret, arg1, arg2); 2008 } 2009 } 2010 2011 void tcg_gen_divu_i64(TCGv_i64 ret, TCGv_i64 arg1, TCGv_i64 arg2) 2012 { 2013 if (tcg_op_supported(INDEX_op_divu, TCG_TYPE_I64, 0)) { 2014 tcg_gen_op3_i64(INDEX_op_divu, ret, arg1, arg2); 2015 } else if (tcg_op_supported(INDEX_op_divu2, TCG_TYPE_I64, 0)) { 2016 TCGv_i64 t0 = tcg_temp_ebb_new_i64(); 2017 TCGv_i64 zero = tcg_constant_i64(0); 2018 tcg_gen_op5_i64(INDEX_op_divu2, ret, t0, arg1, zero, arg2); 2019 tcg_temp_free_i64(t0); 2020 } else { 2021 gen_helper_divu_i64(ret, arg1, arg2); 2022 } 2023 } 2024 2025 void tcg_gen_remu_i64(TCGv_i64 ret, TCGv_i64 arg1, TCGv_i64 arg2) 2026 { 2027 if (tcg_op_supported(INDEX_op_remu, TCG_TYPE_I64, 0)) { 2028 tcg_gen_op3_i64(INDEX_op_remu, ret, arg1, arg2); 2029 } else if (tcg_op_supported(INDEX_op_divu, TCG_TYPE_I64, 0)) { 2030 TCGv_i64 t0 = tcg_temp_ebb_new_i64(); 2031 tcg_gen_op3_i64(INDEX_op_divu, t0, arg1, arg2); 2032 tcg_gen_mul_i64(t0, t0, arg2); 2033 tcg_gen_sub_i64(ret, arg1, t0); 2034 tcg_temp_free_i64(t0); 2035 } else if (tcg_op_supported(INDEX_op_divu2, TCG_TYPE_I64, 0)) { 2036 TCGv_i64 t0 = tcg_temp_ebb_new_i64(); 2037 TCGv_i64 zero = tcg_constant_i64(0); 2038 tcg_gen_op5_i64(INDEX_op_divu2, t0, ret, arg1, zero, arg2); 2039 tcg_temp_free_i64(t0); 2040 } else { 2041 gen_helper_remu_i64(ret, arg1, arg2); 2042 } 2043 } 2044 2045 void tcg_gen_ext8s_i64(TCGv_i64 ret, TCGv_i64 arg) 2046 { 2047 tcg_gen_sextract_i64(ret, arg, 0, 8); 2048 } 2049 2050 void tcg_gen_ext16s_i64(TCGv_i64 ret, TCGv_i64 arg) 2051 { 2052 tcg_gen_sextract_i64(ret, arg, 0, 16); 2053 } 2054 2055 void tcg_gen_ext32s_i64(TCGv_i64 ret, TCGv_i64 arg) 2056 { 2057 tcg_gen_sextract_i64(ret, arg, 0, 32); 2058 } 2059 2060 void tcg_gen_ext8u_i64(TCGv_i64 ret, TCGv_i64 arg) 2061 { 2062 tcg_gen_extract_i64(ret, arg, 0, 8); 2063 } 2064 2065 void tcg_gen_ext16u_i64(TCGv_i64 ret, TCGv_i64 arg) 2066 { 2067 tcg_gen_extract_i64(ret, arg, 0, 16); 2068 } 2069 2070 void tcg_gen_ext32u_i64(TCGv_i64 ret, TCGv_i64 arg) 2071 { 2072 tcg_gen_extract_i64(ret, arg, 0, 32); 2073 } 2074 2075 /* 2076 * bswap16_i64: 16-bit byte swap on the low bits of a 64-bit value. 2077 * 2078 * Byte pattern: xxxxxxxxab -> yyyyyyyyba 2079 * 2080 * With TCG_BSWAP_IZ, x == zero, else undefined. 2081 * With TCG_BSWAP_OZ, y == zero, with TCG_BSWAP_OS y == sign, else undefined. 2082 */ 2083 void tcg_gen_bswap16_i64(TCGv_i64 ret, TCGv_i64 arg, int flags) 2084 { 2085 /* Only one extension flag may be present. */ 2086 tcg_debug_assert(!(flags & TCG_BSWAP_OS) || !(flags & TCG_BSWAP_OZ)); 2087 2088 if (TCG_TARGET_REG_BITS == 32) { 2089 tcg_gen_bswap16_i32(TCGV_LOW(ret), TCGV_LOW(arg), flags); 2090 if (flags & TCG_BSWAP_OS) { 2091 tcg_gen_sari_i32(TCGV_HIGH(ret), TCGV_LOW(ret), 31); 2092 } else { 2093 tcg_gen_movi_i32(TCGV_HIGH(ret), 0); 2094 } 2095 } else if (TCG_TARGET_HAS_bswap16_i64) { 2096 tcg_gen_op3i_i64(INDEX_op_bswap16_i64, ret, arg, flags); 2097 } else { 2098 TCGv_i64 t0 = tcg_temp_ebb_new_i64(); 2099 TCGv_i64 t1 = tcg_temp_ebb_new_i64(); 2100 2101 /* arg = ......ab or xxxxxxab */ 2102 tcg_gen_shri_i64(t0, arg, 8); /* t0 = .......a or .xxxxxxa */ 2103 if (!(flags & TCG_BSWAP_IZ)) { 2104 tcg_gen_ext8u_i64(t0, t0); /* t0 = .......a */ 2105 } 2106 2107 if (flags & TCG_BSWAP_OS) { 2108 tcg_gen_shli_i64(t1, arg, 56); /* t1 = b....... */ 2109 tcg_gen_sari_i64(t1, t1, 48); /* t1 = ssssssb. */ 2110 } else if (flags & TCG_BSWAP_OZ) { 2111 tcg_gen_ext8u_i64(t1, arg); /* t1 = .......b */ 2112 tcg_gen_shli_i64(t1, t1, 8); /* t1 = ......b. */ 2113 } else { 2114 tcg_gen_shli_i64(t1, arg, 8); /* t1 = xxxxxab. */ 2115 } 2116 2117 tcg_gen_or_i64(ret, t0, t1); /* ret = ......ba (OZ) */ 2118 /* ssssssba (OS) */ 2119 /* xxxxxaba (no flag) */ 2120 tcg_temp_free_i64(t0); 2121 tcg_temp_free_i64(t1); 2122 } 2123 } 2124 2125 /* 2126 * bswap32_i64: 32-bit byte swap on the low bits of a 64-bit value. 2127 * 2128 * Byte pattern: xxxxabcd -> yyyydcba 2129 * 2130 * With TCG_BSWAP_IZ, x == zero, else undefined. 2131 * With TCG_BSWAP_OZ, y == zero, with TCG_BSWAP_OS y == sign, else undefined. 2132 */ 2133 void tcg_gen_bswap32_i64(TCGv_i64 ret, TCGv_i64 arg, int flags) 2134 { 2135 /* Only one extension flag may be present. */ 2136 tcg_debug_assert(!(flags & TCG_BSWAP_OS) || !(flags & TCG_BSWAP_OZ)); 2137 2138 if (TCG_TARGET_REG_BITS == 32) { 2139 tcg_gen_bswap32_i32(TCGV_LOW(ret), TCGV_LOW(arg)); 2140 if (flags & TCG_BSWAP_OS) { 2141 tcg_gen_sari_i32(TCGV_HIGH(ret), TCGV_LOW(ret), 31); 2142 } else { 2143 tcg_gen_movi_i32(TCGV_HIGH(ret), 0); 2144 } 2145 } else if (TCG_TARGET_HAS_bswap32_i64) { 2146 tcg_gen_op3i_i64(INDEX_op_bswap32_i64, ret, arg, flags); 2147 } else { 2148 TCGv_i64 t0 = tcg_temp_ebb_new_i64(); 2149 TCGv_i64 t1 = tcg_temp_ebb_new_i64(); 2150 TCGv_i64 t2 = tcg_constant_i64(0x00ff00ff); 2151 2152 /* arg = xxxxabcd */ 2153 tcg_gen_shri_i64(t0, arg, 8); /* t0 = .xxxxabc */ 2154 tcg_gen_and_i64(t1, arg, t2); /* t1 = .....b.d */ 2155 tcg_gen_and_i64(t0, t0, t2); /* t0 = .....a.c */ 2156 tcg_gen_shli_i64(t1, t1, 8); /* t1 = ....b.d. */ 2157 tcg_gen_or_i64(ret, t0, t1); /* ret = ....badc */ 2158 2159 tcg_gen_shli_i64(t1, ret, 48); /* t1 = dc...... */ 2160 tcg_gen_shri_i64(t0, ret, 16); /* t0 = ......ba */ 2161 if (flags & TCG_BSWAP_OS) { 2162 tcg_gen_sari_i64(t1, t1, 32); /* t1 = ssssdc.. */ 2163 } else { 2164 tcg_gen_shri_i64(t1, t1, 32); /* t1 = ....dc.. */ 2165 } 2166 tcg_gen_or_i64(ret, t0, t1); /* ret = ssssdcba (OS) */ 2167 /* ....dcba (else) */ 2168 2169 tcg_temp_free_i64(t0); 2170 tcg_temp_free_i64(t1); 2171 } 2172 } 2173 2174 /* 2175 * bswap64_i64: 64-bit byte swap on a 64-bit value. 2176 * 2177 * Byte pattern: abcdefgh -> hgfedcba 2178 */ 2179 void tcg_gen_bswap64_i64(TCGv_i64 ret, TCGv_i64 arg) 2180 { 2181 if (TCG_TARGET_REG_BITS == 32) { 2182 TCGv_i32 t0, t1; 2183 t0 = tcg_temp_ebb_new_i32(); 2184 t1 = tcg_temp_ebb_new_i32(); 2185 2186 tcg_gen_bswap32_i32(t0, TCGV_LOW(arg)); 2187 tcg_gen_bswap32_i32(t1, TCGV_HIGH(arg)); 2188 tcg_gen_mov_i32(TCGV_LOW(ret), t1); 2189 tcg_gen_mov_i32(TCGV_HIGH(ret), t0); 2190 tcg_temp_free_i32(t0); 2191 tcg_temp_free_i32(t1); 2192 } else if (TCG_TARGET_HAS_bswap64_i64) { 2193 tcg_gen_op3i_i64(INDEX_op_bswap64_i64, ret, arg, 0); 2194 } else { 2195 TCGv_i64 t0 = tcg_temp_ebb_new_i64(); 2196 TCGv_i64 t1 = tcg_temp_ebb_new_i64(); 2197 TCGv_i64 t2 = tcg_temp_ebb_new_i64(); 2198 2199 /* arg = abcdefgh */ 2200 tcg_gen_movi_i64(t2, 0x00ff00ff00ff00ffull); 2201 tcg_gen_shri_i64(t0, arg, 8); /* t0 = .abcdefg */ 2202 tcg_gen_and_i64(t1, arg, t2); /* t1 = .b.d.f.h */ 2203 tcg_gen_and_i64(t0, t0, t2); /* t0 = .a.c.e.g */ 2204 tcg_gen_shli_i64(t1, t1, 8); /* t1 = b.d.f.h. */ 2205 tcg_gen_or_i64(ret, t0, t1); /* ret = badcfehg */ 2206 2207 tcg_gen_movi_i64(t2, 0x0000ffff0000ffffull); 2208 tcg_gen_shri_i64(t0, ret, 16); /* t0 = ..badcfe */ 2209 tcg_gen_and_i64(t1, ret, t2); /* t1 = ..dc..hg */ 2210 tcg_gen_and_i64(t0, t0, t2); /* t0 = ..ba..fe */ 2211 tcg_gen_shli_i64(t1, t1, 16); /* t1 = dc..hg.. */ 2212 tcg_gen_or_i64(ret, t0, t1); /* ret = dcbahgfe */ 2213 2214 tcg_gen_shri_i64(t0, ret, 32); /* t0 = ....dcba */ 2215 tcg_gen_shli_i64(t1, ret, 32); /* t1 = hgfe.... */ 2216 tcg_gen_or_i64(ret, t0, t1); /* ret = hgfedcba */ 2217 2218 tcg_temp_free_i64(t0); 2219 tcg_temp_free_i64(t1); 2220 tcg_temp_free_i64(t2); 2221 } 2222 } 2223 2224 /* 2225 * hswap_i64: Swap 16-bit halfwords within a 64-bit value. 2226 * See also include/qemu/bitops.h, hswap64. 2227 * 2228 * Byte pattern: abcdefgh -> ghefcdab 2229 */ 2230 void tcg_gen_hswap_i64(TCGv_i64 ret, TCGv_i64 arg) 2231 { 2232 uint64_t m = 0x0000ffff0000ffffull; 2233 TCGv_i64 t0 = tcg_temp_ebb_new_i64(); 2234 TCGv_i64 t1 = tcg_temp_ebb_new_i64(); 2235 2236 /* arg = abcdefgh */ 2237 tcg_gen_rotli_i64(t1, arg, 32); /* t1 = efghabcd */ 2238 tcg_gen_andi_i64(t0, t1, m); /* t0 = ..gh..cd */ 2239 tcg_gen_shli_i64(t0, t0, 16); /* t0 = gh..cd.. */ 2240 tcg_gen_shri_i64(t1, t1, 16); /* t1 = ..efghab */ 2241 tcg_gen_andi_i64(t1, t1, m); /* t1 = ..ef..ab */ 2242 tcg_gen_or_i64(ret, t0, t1); /* ret = ghefcdab */ 2243 2244 tcg_temp_free_i64(t0); 2245 tcg_temp_free_i64(t1); 2246 } 2247 2248 /* 2249 * wswap_i64: Swap 32-bit words within a 64-bit value. 2250 * 2251 * Byte pattern: abcdefgh -> efghabcd 2252 */ 2253 void tcg_gen_wswap_i64(TCGv_i64 ret, TCGv_i64 arg) 2254 { 2255 /* Swapping 2 32-bit elements is a rotate. */ 2256 tcg_gen_rotli_i64(ret, arg, 32); 2257 } 2258 2259 void tcg_gen_not_i64(TCGv_i64 ret, TCGv_i64 arg) 2260 { 2261 if (TCG_TARGET_REG_BITS == 32) { 2262 tcg_gen_not_i32(TCGV_LOW(ret), TCGV_LOW(arg)); 2263 tcg_gen_not_i32(TCGV_HIGH(ret), TCGV_HIGH(arg)); 2264 } else if (tcg_op_supported(INDEX_op_not, TCG_TYPE_I64, 0)) { 2265 tcg_gen_op2_i64(INDEX_op_not, ret, arg); 2266 } else { 2267 tcg_gen_xori_i64(ret, arg, -1); 2268 } 2269 } 2270 2271 void tcg_gen_andc_i64(TCGv_i64 ret, TCGv_i64 arg1, TCGv_i64 arg2) 2272 { 2273 if (TCG_TARGET_REG_BITS == 32) { 2274 tcg_gen_andc_i32(TCGV_LOW(ret), TCGV_LOW(arg1), TCGV_LOW(arg2)); 2275 tcg_gen_andc_i32(TCGV_HIGH(ret), TCGV_HIGH(arg1), TCGV_HIGH(arg2)); 2276 } else if (tcg_op_supported(INDEX_op_andc, TCG_TYPE_I64, 0)) { 2277 tcg_gen_op3_i64(INDEX_op_andc, ret, arg1, arg2); 2278 } else { 2279 TCGv_i64 t0 = tcg_temp_ebb_new_i64(); 2280 tcg_gen_not_i64(t0, arg2); 2281 tcg_gen_and_i64(ret, arg1, t0); 2282 tcg_temp_free_i64(t0); 2283 } 2284 } 2285 2286 void tcg_gen_eqv_i64(TCGv_i64 ret, TCGv_i64 arg1, TCGv_i64 arg2) 2287 { 2288 if (TCG_TARGET_REG_BITS == 32) { 2289 tcg_gen_eqv_i32(TCGV_LOW(ret), TCGV_LOW(arg1), TCGV_LOW(arg2)); 2290 tcg_gen_eqv_i32(TCGV_HIGH(ret), TCGV_HIGH(arg1), TCGV_HIGH(arg2)); 2291 } else if (tcg_op_supported(INDEX_op_eqv, TCG_TYPE_I64, 0)) { 2292 tcg_gen_op3_i64(INDEX_op_eqv, ret, arg1, arg2); 2293 } else { 2294 tcg_gen_xor_i64(ret, arg1, arg2); 2295 tcg_gen_not_i64(ret, ret); 2296 } 2297 } 2298 2299 void tcg_gen_nand_i64(TCGv_i64 ret, TCGv_i64 arg1, TCGv_i64 arg2) 2300 { 2301 if (TCG_TARGET_REG_BITS == 32) { 2302 tcg_gen_nand_i32(TCGV_LOW(ret), TCGV_LOW(arg1), TCGV_LOW(arg2)); 2303 tcg_gen_nand_i32(TCGV_HIGH(ret), TCGV_HIGH(arg1), TCGV_HIGH(arg2)); 2304 } else if (tcg_op_supported(INDEX_op_nand, TCG_TYPE_I64, 0)) { 2305 tcg_gen_op3_i64(INDEX_op_nand, ret, arg1, arg2); 2306 } else { 2307 tcg_gen_and_i64(ret, arg1, arg2); 2308 tcg_gen_not_i64(ret, ret); 2309 } 2310 } 2311 2312 void tcg_gen_nor_i64(TCGv_i64 ret, TCGv_i64 arg1, TCGv_i64 arg2) 2313 { 2314 if (TCG_TARGET_REG_BITS == 32) { 2315 tcg_gen_nor_i32(TCGV_LOW(ret), TCGV_LOW(arg1), TCGV_LOW(arg2)); 2316 tcg_gen_nor_i32(TCGV_HIGH(ret), TCGV_HIGH(arg1), TCGV_HIGH(arg2)); 2317 } else if (tcg_op_supported(INDEX_op_nor, TCG_TYPE_I64, 0)) { 2318 tcg_gen_op3_i64(INDEX_op_nor, ret, arg1, arg2); 2319 } else { 2320 tcg_gen_or_i64(ret, arg1, arg2); 2321 tcg_gen_not_i64(ret, ret); 2322 } 2323 } 2324 2325 void tcg_gen_orc_i64(TCGv_i64 ret, TCGv_i64 arg1, TCGv_i64 arg2) 2326 { 2327 if (TCG_TARGET_REG_BITS == 32) { 2328 tcg_gen_orc_i32(TCGV_LOW(ret), TCGV_LOW(arg1), TCGV_LOW(arg2)); 2329 tcg_gen_orc_i32(TCGV_HIGH(ret), TCGV_HIGH(arg1), TCGV_HIGH(arg2)); 2330 } else if (tcg_op_supported(INDEX_op_orc, TCG_TYPE_I64, 0)) { 2331 tcg_gen_op3_i64(INDEX_op_orc, ret, arg1, arg2); 2332 } else { 2333 TCGv_i64 t0 = tcg_temp_ebb_new_i64(); 2334 tcg_gen_not_i64(t0, arg2); 2335 tcg_gen_or_i64(ret, arg1, t0); 2336 tcg_temp_free_i64(t0); 2337 } 2338 } 2339 2340 void tcg_gen_clz_i64(TCGv_i64 ret, TCGv_i64 arg1, TCGv_i64 arg2) 2341 { 2342 if (tcg_op_supported(INDEX_op_clz, TCG_TYPE_I64, 0)) { 2343 tcg_gen_op3_i64(INDEX_op_clz, ret, arg1, arg2); 2344 } else { 2345 gen_helper_clz_i64(ret, arg1, arg2); 2346 } 2347 } 2348 2349 void tcg_gen_clzi_i64(TCGv_i64 ret, TCGv_i64 arg1, uint64_t arg2) 2350 { 2351 if (TCG_TARGET_REG_BITS == 32 2352 && arg2 <= 0xffffffffu 2353 && tcg_op_supported(INDEX_op_clz, TCG_TYPE_I32, 0)) { 2354 TCGv_i32 t = tcg_temp_ebb_new_i32(); 2355 tcg_gen_clzi_i32(t, TCGV_LOW(arg1), arg2 - 32); 2356 tcg_gen_addi_i32(t, t, 32); 2357 tcg_gen_clz_i32(TCGV_LOW(ret), TCGV_HIGH(arg1), t); 2358 tcg_gen_movi_i32(TCGV_HIGH(ret), 0); 2359 tcg_temp_free_i32(t); 2360 } else { 2361 tcg_gen_clz_i64(ret, arg1, tcg_constant_i64(arg2)); 2362 } 2363 } 2364 2365 void tcg_gen_ctz_i64(TCGv_i64 ret, TCGv_i64 arg1, TCGv_i64 arg2) 2366 { 2367 TCGv_i64 z, t; 2368 2369 if (tcg_op_supported(INDEX_op_ctz, TCG_TYPE_I64, 0)) { 2370 tcg_gen_op3_i64(INDEX_op_ctz, ret, arg1, arg2); 2371 return; 2372 } 2373 if (TCG_TARGET_HAS_ctpop_i64) { 2374 t = tcg_temp_ebb_new_i64(); 2375 tcg_gen_subi_i64(t, arg1, 1); 2376 tcg_gen_andc_i64(t, t, arg1); 2377 tcg_gen_ctpop_i64(t, t); 2378 } else if (tcg_op_supported(INDEX_op_clz, TCG_TYPE_I64, 0)) { 2379 t = tcg_temp_ebb_new_i64(); 2380 tcg_gen_neg_i64(t, arg1); 2381 tcg_gen_and_i64(t, t, arg1); 2382 tcg_gen_clzi_i64(t, t, 64); 2383 tcg_gen_xori_i64(t, t, 63); 2384 } else { 2385 gen_helper_ctz_i64(ret, arg1, arg2); 2386 return; 2387 } 2388 2389 z = tcg_constant_i64(0); 2390 tcg_gen_movcond_i64(TCG_COND_EQ, ret, arg1, z, arg2, t); 2391 tcg_temp_free_i64(t); 2392 } 2393 2394 void tcg_gen_ctzi_i64(TCGv_i64 ret, TCGv_i64 arg1, uint64_t arg2) 2395 { 2396 if (TCG_TARGET_REG_BITS == 32 2397 && arg2 <= 0xffffffffu 2398 && tcg_op_supported(INDEX_op_ctz, TCG_TYPE_I32, 0)) { 2399 TCGv_i32 t32 = tcg_temp_ebb_new_i32(); 2400 tcg_gen_ctzi_i32(t32, TCGV_HIGH(arg1), arg2 - 32); 2401 tcg_gen_addi_i32(t32, t32, 32); 2402 tcg_gen_ctz_i32(TCGV_LOW(ret), TCGV_LOW(arg1), t32); 2403 tcg_gen_movi_i32(TCGV_HIGH(ret), 0); 2404 tcg_temp_free_i32(t32); 2405 } else if (arg2 == 64 2406 && !tcg_op_supported(INDEX_op_ctz, TCG_TYPE_I64, 0) 2407 && TCG_TARGET_HAS_ctpop_i64) { 2408 /* This equivalence has the advantage of not requiring a fixup. */ 2409 TCGv_i64 t = tcg_temp_ebb_new_i64(); 2410 tcg_gen_subi_i64(t, arg1, 1); 2411 tcg_gen_andc_i64(t, t, arg1); 2412 tcg_gen_ctpop_i64(ret, t); 2413 tcg_temp_free_i64(t); 2414 } else { 2415 tcg_gen_ctz_i64(ret, arg1, tcg_constant_i64(arg2)); 2416 } 2417 } 2418 2419 void tcg_gen_clrsb_i64(TCGv_i64 ret, TCGv_i64 arg) 2420 { 2421 if (tcg_op_supported(INDEX_op_clz, TCG_TYPE_I64, 0)) { 2422 TCGv_i64 t = tcg_temp_ebb_new_i64(); 2423 tcg_gen_sari_i64(t, arg, 63); 2424 tcg_gen_xor_i64(t, t, arg); 2425 tcg_gen_clzi_i64(t, t, 64); 2426 tcg_gen_subi_i64(ret, t, 1); 2427 tcg_temp_free_i64(t); 2428 } else { 2429 gen_helper_clrsb_i64(ret, arg); 2430 } 2431 } 2432 2433 void tcg_gen_ctpop_i64(TCGv_i64 ret, TCGv_i64 arg1) 2434 { 2435 if (TCG_TARGET_HAS_ctpop_i64) { 2436 tcg_gen_op2_i64(INDEX_op_ctpop_i64, ret, arg1); 2437 } else if (TCG_TARGET_REG_BITS == 32 && TCG_TARGET_HAS_ctpop_i32) { 2438 tcg_gen_ctpop_i32(TCGV_HIGH(ret), TCGV_HIGH(arg1)); 2439 tcg_gen_ctpop_i32(TCGV_LOW(ret), TCGV_LOW(arg1)); 2440 tcg_gen_add_i32(TCGV_LOW(ret), TCGV_LOW(ret), TCGV_HIGH(ret)); 2441 tcg_gen_movi_i32(TCGV_HIGH(ret), 0); 2442 } else { 2443 gen_helper_ctpop_i64(ret, arg1); 2444 } 2445 } 2446 2447 void tcg_gen_rotl_i64(TCGv_i64 ret, TCGv_i64 arg1, TCGv_i64 arg2) 2448 { 2449 if (tcg_op_supported(INDEX_op_rotl, TCG_TYPE_I64, 0)) { 2450 tcg_gen_op3_i64(INDEX_op_rotl, ret, arg1, arg2); 2451 } else if (tcg_op_supported(INDEX_op_rotl, TCG_TYPE_I64, 0)) { 2452 TCGv_i64 t0 = tcg_temp_ebb_new_i64(); 2453 tcg_gen_neg_i64(t0, arg2); 2454 tcg_gen_op3_i64(INDEX_op_rotr, ret, arg1, t0); 2455 tcg_temp_free_i64(t0); 2456 } else { 2457 TCGv_i64 t0 = tcg_temp_ebb_new_i64(); 2458 TCGv_i64 t1 = tcg_temp_ebb_new_i64(); 2459 tcg_gen_shl_i64(t0, arg1, arg2); 2460 tcg_gen_neg_i64(t1, arg2); 2461 tcg_gen_shr_i64(t1, arg1, t1); 2462 tcg_gen_or_i64(ret, t0, t1); 2463 tcg_temp_free_i64(t0); 2464 tcg_temp_free_i64(t1); 2465 } 2466 } 2467 2468 void tcg_gen_rotli_i64(TCGv_i64 ret, TCGv_i64 arg1, int64_t arg2) 2469 { 2470 tcg_debug_assert(arg2 >= 0 && arg2 < 64); 2471 /* some cases can be optimized here */ 2472 if (arg2 == 0) { 2473 tcg_gen_mov_i64(ret, arg1); 2474 } else if (tcg_op_supported(INDEX_op_rotl, TCG_TYPE_I64, 0)) { 2475 TCGv_i64 t0 = tcg_constant_i64(arg2); 2476 tcg_gen_op3_i64(INDEX_op_rotl, ret, arg1, t0); 2477 } else if (tcg_op_supported(INDEX_op_rotr, TCG_TYPE_I64, 0)) { 2478 TCGv_i64 t0 = tcg_constant_i64(64 - arg2); 2479 tcg_gen_op3_i64(INDEX_op_rotr, ret, arg1, t0); 2480 } else { 2481 TCGv_i64 t0 = tcg_temp_ebb_new_i64(); 2482 TCGv_i64 t1 = tcg_temp_ebb_new_i64(); 2483 tcg_gen_shli_i64(t0, arg1, arg2); 2484 tcg_gen_shri_i64(t1, arg1, 64 - arg2); 2485 tcg_gen_or_i64(ret, t0, t1); 2486 tcg_temp_free_i64(t0); 2487 tcg_temp_free_i64(t1); 2488 } 2489 } 2490 2491 void tcg_gen_rotr_i64(TCGv_i64 ret, TCGv_i64 arg1, TCGv_i64 arg2) 2492 { 2493 if (tcg_op_supported(INDEX_op_rotr, TCG_TYPE_I64, 0)) { 2494 tcg_gen_op3_i64(INDEX_op_rotr, ret, arg1, arg2); 2495 } else if (tcg_op_supported(INDEX_op_rotl, TCG_TYPE_I64, 0)) { 2496 TCGv_i64 t0 = tcg_temp_ebb_new_i64(); 2497 tcg_gen_neg_i64(t0, arg2); 2498 tcg_gen_op3_i64(INDEX_op_rotl, ret, arg1, t0); 2499 tcg_temp_free_i64(t0); 2500 } else { 2501 TCGv_i64 t0 = tcg_temp_ebb_new_i64(); 2502 TCGv_i64 t1 = tcg_temp_ebb_new_i64(); 2503 tcg_gen_shr_i64(t0, arg1, arg2); 2504 tcg_gen_neg_i64(t1, arg2); 2505 tcg_gen_shl_i64(t1, arg1, t1); 2506 tcg_gen_or_i64(ret, t0, t1); 2507 tcg_temp_free_i64(t0); 2508 tcg_temp_free_i64(t1); 2509 } 2510 } 2511 2512 void tcg_gen_rotri_i64(TCGv_i64 ret, TCGv_i64 arg1, int64_t arg2) 2513 { 2514 tcg_debug_assert(arg2 >= 0 && arg2 < 64); 2515 tcg_gen_rotli_i64(ret, arg1, -arg2 & 63); 2516 } 2517 2518 void tcg_gen_deposit_i64(TCGv_i64 ret, TCGv_i64 arg1, TCGv_i64 arg2, 2519 unsigned int ofs, unsigned int len) 2520 { 2521 uint64_t mask; 2522 TCGv_i64 t1; 2523 2524 tcg_debug_assert(ofs < 64); 2525 tcg_debug_assert(len > 0); 2526 tcg_debug_assert(len <= 64); 2527 tcg_debug_assert(ofs + len <= 64); 2528 2529 if (len == 64) { 2530 tcg_gen_mov_i64(ret, arg2); 2531 return; 2532 } 2533 2534 if (TCG_TARGET_REG_BITS == 64) { 2535 if (TCG_TARGET_deposit_valid(TCG_TYPE_I64, ofs, len)) { 2536 tcg_gen_op5ii_i64(INDEX_op_deposit_i64, ret, arg1, arg2, ofs, len); 2537 return; 2538 } 2539 } else { 2540 if (ofs >= 32) { 2541 tcg_gen_deposit_i32(TCGV_HIGH(ret), TCGV_HIGH(arg1), 2542 TCGV_LOW(arg2), ofs - 32, len); 2543 tcg_gen_mov_i32(TCGV_LOW(ret), TCGV_LOW(arg1)); 2544 return; 2545 } 2546 if (ofs + len <= 32) { 2547 tcg_gen_deposit_i32(TCGV_LOW(ret), TCGV_LOW(arg1), 2548 TCGV_LOW(arg2), ofs, len); 2549 tcg_gen_mov_i32(TCGV_HIGH(ret), TCGV_HIGH(arg1)); 2550 return; 2551 } 2552 } 2553 2554 t1 = tcg_temp_ebb_new_i64(); 2555 2556 if (TCG_TARGET_HAS_extract2_i64) { 2557 if (ofs + len == 64) { 2558 tcg_gen_shli_i64(t1, arg1, len); 2559 tcg_gen_extract2_i64(ret, t1, arg2, len); 2560 goto done; 2561 } 2562 if (ofs == 0) { 2563 tcg_gen_extract2_i64(ret, arg1, arg2, len); 2564 tcg_gen_rotli_i64(ret, ret, len); 2565 goto done; 2566 } 2567 } 2568 2569 mask = (1ull << len) - 1; 2570 if (ofs + len < 64) { 2571 tcg_gen_andi_i64(t1, arg2, mask); 2572 tcg_gen_shli_i64(t1, t1, ofs); 2573 } else { 2574 tcg_gen_shli_i64(t1, arg2, ofs); 2575 } 2576 tcg_gen_andi_i64(ret, arg1, ~(mask << ofs)); 2577 tcg_gen_or_i64(ret, ret, t1); 2578 done: 2579 tcg_temp_free_i64(t1); 2580 } 2581 2582 void tcg_gen_deposit_z_i64(TCGv_i64 ret, TCGv_i64 arg, 2583 unsigned int ofs, unsigned int len) 2584 { 2585 tcg_debug_assert(ofs < 64); 2586 tcg_debug_assert(len > 0); 2587 tcg_debug_assert(len <= 64); 2588 tcg_debug_assert(ofs + len <= 64); 2589 2590 if (ofs + len == 64) { 2591 tcg_gen_shli_i64(ret, arg, ofs); 2592 } else if (ofs == 0) { 2593 tcg_gen_andi_i64(ret, arg, (1ull << len) - 1); 2594 } else if (TCG_TARGET_REG_BITS == 64 && 2595 TCG_TARGET_deposit_valid(TCG_TYPE_I64, ofs, len)) { 2596 TCGv_i64 zero = tcg_constant_i64(0); 2597 tcg_gen_op5ii_i64(INDEX_op_deposit_i64, ret, zero, arg, ofs, len); 2598 } else { 2599 if (TCG_TARGET_REG_BITS == 32) { 2600 if (ofs >= 32) { 2601 tcg_gen_deposit_z_i32(TCGV_HIGH(ret), TCGV_LOW(arg), 2602 ofs - 32, len); 2603 tcg_gen_movi_i32(TCGV_LOW(ret), 0); 2604 return; 2605 } 2606 if (ofs + len <= 32) { 2607 tcg_gen_deposit_z_i32(TCGV_LOW(ret), TCGV_LOW(arg), ofs, len); 2608 tcg_gen_movi_i32(TCGV_HIGH(ret), 0); 2609 return; 2610 } 2611 } 2612 /* 2613 * To help two-operand hosts we prefer to zero-extend first, 2614 * which allows ARG to stay live. 2615 */ 2616 if (TCG_TARGET_extract_valid(TCG_TYPE_I64, 0, len)) { 2617 tcg_gen_extract_i64(ret, arg, 0, len); 2618 tcg_gen_shli_i64(ret, ret, ofs); 2619 return; 2620 } 2621 /* Otherwise prefer zero-extension over AND for code size. */ 2622 if (TCG_TARGET_extract_valid(TCG_TYPE_I64, 0, ofs + len)) { 2623 tcg_gen_shli_i64(ret, arg, ofs); 2624 tcg_gen_extract_i64(ret, ret, 0, ofs + len); 2625 return; 2626 } 2627 tcg_gen_andi_i64(ret, arg, (1ull << len) - 1); 2628 tcg_gen_shli_i64(ret, ret, ofs); 2629 } 2630 } 2631 2632 void tcg_gen_extract_i64(TCGv_i64 ret, TCGv_i64 arg, 2633 unsigned int ofs, unsigned int len) 2634 { 2635 tcg_debug_assert(ofs < 64); 2636 tcg_debug_assert(len > 0); 2637 tcg_debug_assert(len <= 64); 2638 tcg_debug_assert(ofs + len <= 64); 2639 2640 /* Canonicalize certain special cases, even if extract is supported. */ 2641 if (ofs + len == 64) { 2642 tcg_gen_shri_i64(ret, arg, 64 - len); 2643 return; 2644 } 2645 2646 if (TCG_TARGET_REG_BITS == 32) { 2647 /* Look for a 32-bit extract within one of the two words. */ 2648 if (ofs >= 32) { 2649 tcg_gen_extract_i32(TCGV_LOW(ret), TCGV_HIGH(arg), ofs - 32, len); 2650 tcg_gen_movi_i32(TCGV_HIGH(ret), 0); 2651 return; 2652 } 2653 if (ofs + len <= 32) { 2654 tcg_gen_extract_i32(TCGV_LOW(ret), TCGV_LOW(arg), ofs, len); 2655 tcg_gen_movi_i32(TCGV_HIGH(ret), 0); 2656 return; 2657 } 2658 2659 /* The field is split across two words. */ 2660 tcg_gen_extract2_i32(TCGV_LOW(ret), TCGV_LOW(arg), 2661 TCGV_HIGH(arg), ofs); 2662 if (len <= 32) { 2663 tcg_gen_extract_i32(TCGV_LOW(ret), TCGV_LOW(ret), 0, len); 2664 tcg_gen_movi_i32(TCGV_HIGH(ret), 0); 2665 } else { 2666 tcg_gen_extract_i32(TCGV_HIGH(ret), TCGV_HIGH(arg), 2667 ofs, len - 32); 2668 } 2669 return; 2670 } 2671 2672 if (TCG_TARGET_extract_valid(TCG_TYPE_I64, ofs, len)) { 2673 tcg_gen_op4ii_i64(INDEX_op_extract_i64, ret, arg, ofs, len); 2674 return; 2675 } 2676 if (ofs == 0) { 2677 tcg_gen_andi_i64(ret, arg, (1ull << len) - 1); 2678 return; 2679 } 2680 2681 /* Assume that zero-extension, if available, is cheaper than a shift. */ 2682 if (TCG_TARGET_extract_valid(TCG_TYPE_I64, 0, ofs + len)) { 2683 tcg_gen_op4ii_i64(INDEX_op_extract_i64, ret, arg, 0, ofs + len); 2684 tcg_gen_shri_i64(ret, ret, ofs); 2685 return; 2686 } 2687 2688 /* ??? Ideally we'd know what values are available for immediate AND. 2689 Assume that 8 bits are available, plus the special cases of 16 and 32, 2690 so that we get ext8u, ext16u, and ext32u. */ 2691 switch (len) { 2692 case 1 ... 8: case 16: case 32: 2693 tcg_gen_shri_i64(ret, arg, ofs); 2694 tcg_gen_andi_i64(ret, ret, (1ull << len) - 1); 2695 break; 2696 default: 2697 tcg_gen_shli_i64(ret, arg, 64 - len - ofs); 2698 tcg_gen_shri_i64(ret, ret, 64 - len); 2699 break; 2700 } 2701 } 2702 2703 void tcg_gen_sextract_i64(TCGv_i64 ret, TCGv_i64 arg, 2704 unsigned int ofs, unsigned int len) 2705 { 2706 tcg_debug_assert(ofs < 64); 2707 tcg_debug_assert(len > 0); 2708 tcg_debug_assert(len <= 64); 2709 tcg_debug_assert(ofs + len <= 64); 2710 2711 /* Canonicalize certain special cases, even if sextract is supported. */ 2712 if (ofs + len == 64) { 2713 tcg_gen_sari_i64(ret, arg, 64 - len); 2714 return; 2715 } 2716 2717 if (TCG_TARGET_REG_BITS == 32) { 2718 /* Look for a 32-bit extract within one of the two words. */ 2719 if (ofs >= 32) { 2720 tcg_gen_sextract_i32(TCGV_LOW(ret), TCGV_HIGH(arg), ofs - 32, len); 2721 } else if (ofs + len <= 32) { 2722 tcg_gen_sextract_i32(TCGV_LOW(ret), TCGV_LOW(arg), ofs, len); 2723 } else if (ofs == 0) { 2724 tcg_gen_mov_i32(TCGV_LOW(ret), TCGV_LOW(arg)); 2725 tcg_gen_sextract_i32(TCGV_HIGH(ret), TCGV_HIGH(arg), 0, len - 32); 2726 return; 2727 } else if (len > 32) { 2728 TCGv_i32 t = tcg_temp_ebb_new_i32(); 2729 /* Extract the bits for the high word normally. */ 2730 tcg_gen_sextract_i32(t, TCGV_HIGH(arg), ofs + 32, len - 32); 2731 /* Shift the field down for the low part. */ 2732 tcg_gen_shri_i64(ret, arg, ofs); 2733 /* Overwrite the shift into the high part. */ 2734 tcg_gen_mov_i32(TCGV_HIGH(ret), t); 2735 tcg_temp_free_i32(t); 2736 return; 2737 } else { 2738 /* Shift the field down for the low part, such that the 2739 field sits at the MSB. */ 2740 tcg_gen_shri_i64(ret, arg, ofs + len - 32); 2741 /* Shift the field down from the MSB, sign extending. */ 2742 tcg_gen_sari_i32(TCGV_LOW(ret), TCGV_LOW(ret), 32 - len); 2743 } 2744 /* Sign-extend the field from 32 bits. */ 2745 tcg_gen_sari_i32(TCGV_HIGH(ret), TCGV_LOW(ret), 31); 2746 return; 2747 } 2748 2749 if (TCG_TARGET_sextract_valid(TCG_TYPE_I64, ofs, len)) { 2750 tcg_gen_op4ii_i64(INDEX_op_sextract_i64, ret, arg, ofs, len); 2751 return; 2752 } 2753 2754 /* Assume that sign-extension, if available, is cheaper than a shift. */ 2755 if (TCG_TARGET_sextract_valid(TCG_TYPE_I64, 0, ofs + len)) { 2756 tcg_gen_op4ii_i64(INDEX_op_sextract_i64, ret, arg, 0, ofs + len); 2757 tcg_gen_sari_i64(ret, ret, ofs); 2758 return; 2759 } 2760 if (TCG_TARGET_sextract_valid(TCG_TYPE_I64, 0, len)) { 2761 tcg_gen_shri_i64(ret, arg, ofs); 2762 tcg_gen_op4ii_i64(INDEX_op_sextract_i64, ret, ret, 0, len); 2763 return; 2764 } 2765 2766 tcg_gen_shli_i64(ret, arg, 64 - len - ofs); 2767 tcg_gen_sari_i64(ret, ret, 64 - len); 2768 } 2769 2770 /* 2771 * Extract 64 bits from a 128-bit input, ah:al, starting from ofs. 2772 * Unlike tcg_gen_extract_i64 above, len is fixed at 64. 2773 */ 2774 void tcg_gen_extract2_i64(TCGv_i64 ret, TCGv_i64 al, TCGv_i64 ah, 2775 unsigned int ofs) 2776 { 2777 tcg_debug_assert(ofs <= 64); 2778 if (ofs == 0) { 2779 tcg_gen_mov_i64(ret, al); 2780 } else if (ofs == 64) { 2781 tcg_gen_mov_i64(ret, ah); 2782 } else if (al == ah) { 2783 tcg_gen_rotri_i64(ret, al, ofs); 2784 } else if (TCG_TARGET_HAS_extract2_i64) { 2785 tcg_gen_op4i_i64(INDEX_op_extract2_i64, ret, al, ah, ofs); 2786 } else { 2787 TCGv_i64 t0 = tcg_temp_ebb_new_i64(); 2788 tcg_gen_shri_i64(t0, al, ofs); 2789 tcg_gen_deposit_i64(ret, t0, ah, 64 - ofs, ofs); 2790 tcg_temp_free_i64(t0); 2791 } 2792 } 2793 2794 void tcg_gen_movcond_i64(TCGCond cond, TCGv_i64 ret, TCGv_i64 c1, 2795 TCGv_i64 c2, TCGv_i64 v1, TCGv_i64 v2) 2796 { 2797 if (cond == TCG_COND_ALWAYS) { 2798 tcg_gen_mov_i64(ret, v1); 2799 } else if (cond == TCG_COND_NEVER) { 2800 tcg_gen_mov_i64(ret, v2); 2801 } else if (TCG_TARGET_REG_BITS == 64) { 2802 tcg_gen_op6i_i64(INDEX_op_movcond_i64, ret, c1, c2, v1, v2, cond); 2803 } else { 2804 TCGv_i32 t0 = tcg_temp_ebb_new_i32(); 2805 TCGv_i32 zero = tcg_constant_i32(0); 2806 2807 tcg_gen_op6i_i32(INDEX_op_setcond2_i32, t0, 2808 TCGV_LOW(c1), TCGV_HIGH(c1), 2809 TCGV_LOW(c2), TCGV_HIGH(c2), cond); 2810 2811 tcg_gen_movcond_i32(TCG_COND_NE, TCGV_LOW(ret), t0, zero, 2812 TCGV_LOW(v1), TCGV_LOW(v2)); 2813 tcg_gen_movcond_i32(TCG_COND_NE, TCGV_HIGH(ret), t0, zero, 2814 TCGV_HIGH(v1), TCGV_HIGH(v2)); 2815 2816 tcg_temp_free_i32(t0); 2817 } 2818 } 2819 2820 void tcg_gen_add2_i64(TCGv_i64 rl, TCGv_i64 rh, TCGv_i64 al, 2821 TCGv_i64 ah, TCGv_i64 bl, TCGv_i64 bh) 2822 { 2823 if (TCG_TARGET_HAS_add2_i64) { 2824 tcg_gen_op6_i64(INDEX_op_add2_i64, rl, rh, al, ah, bl, bh); 2825 } else { 2826 TCGv_i64 t0 = tcg_temp_ebb_new_i64(); 2827 TCGv_i64 t1 = tcg_temp_ebb_new_i64(); 2828 tcg_gen_add_i64(t0, al, bl); 2829 tcg_gen_setcond_i64(TCG_COND_LTU, t1, t0, al); 2830 tcg_gen_add_i64(rh, ah, bh); 2831 tcg_gen_add_i64(rh, rh, t1); 2832 tcg_gen_mov_i64(rl, t0); 2833 tcg_temp_free_i64(t0); 2834 tcg_temp_free_i64(t1); 2835 } 2836 } 2837 2838 void tcg_gen_sub2_i64(TCGv_i64 rl, TCGv_i64 rh, TCGv_i64 al, 2839 TCGv_i64 ah, TCGv_i64 bl, TCGv_i64 bh) 2840 { 2841 if (TCG_TARGET_HAS_sub2_i64) { 2842 tcg_gen_op6_i64(INDEX_op_sub2_i64, rl, rh, al, ah, bl, bh); 2843 } else { 2844 TCGv_i64 t0 = tcg_temp_ebb_new_i64(); 2845 TCGv_i64 t1 = tcg_temp_ebb_new_i64(); 2846 tcg_gen_sub_i64(t0, al, bl); 2847 tcg_gen_setcond_i64(TCG_COND_LTU, t1, al, bl); 2848 tcg_gen_sub_i64(rh, ah, bh); 2849 tcg_gen_sub_i64(rh, rh, t1); 2850 tcg_gen_mov_i64(rl, t0); 2851 tcg_temp_free_i64(t0); 2852 tcg_temp_free_i64(t1); 2853 } 2854 } 2855 2856 void tcg_gen_mulu2_i64(TCGv_i64 rl, TCGv_i64 rh, TCGv_i64 arg1, TCGv_i64 arg2) 2857 { 2858 if (TCG_TARGET_HAS_mulu2_i64) { 2859 tcg_gen_op4_i64(INDEX_op_mulu2_i64, rl, rh, arg1, arg2); 2860 } else if (tcg_op_supported(INDEX_op_muluh, TCG_TYPE_I64, 0)) { 2861 TCGv_i64 t = tcg_temp_ebb_new_i64(); 2862 tcg_gen_op3_i64(INDEX_op_mul, t, arg1, arg2); 2863 tcg_gen_op3_i64(INDEX_op_muluh, rh, arg1, arg2); 2864 tcg_gen_mov_i64(rl, t); 2865 tcg_temp_free_i64(t); 2866 } else { 2867 TCGv_i64 t0 = tcg_temp_ebb_new_i64(); 2868 tcg_gen_mul_i64(t0, arg1, arg2); 2869 gen_helper_muluh_i64(rh, arg1, arg2); 2870 tcg_gen_mov_i64(rl, t0); 2871 tcg_temp_free_i64(t0); 2872 } 2873 } 2874 2875 void tcg_gen_muls2_i64(TCGv_i64 rl, TCGv_i64 rh, TCGv_i64 arg1, TCGv_i64 arg2) 2876 { 2877 if (TCG_TARGET_HAS_muls2_i64) { 2878 tcg_gen_op4_i64(INDEX_op_muls2_i64, rl, rh, arg1, arg2); 2879 } else if (tcg_op_supported(INDEX_op_mulsh, TCG_TYPE_I64, 0)) { 2880 TCGv_i64 t = tcg_temp_ebb_new_i64(); 2881 tcg_gen_op3_i64(INDEX_op_mul, t, arg1, arg2); 2882 tcg_gen_op3_i64(INDEX_op_mulsh, rh, arg1, arg2); 2883 tcg_gen_mov_i64(rl, t); 2884 tcg_temp_free_i64(t); 2885 } else if (TCG_TARGET_HAS_mulu2_i64 || 2886 tcg_op_supported(INDEX_op_muluh, TCG_TYPE_I64, 0)) { 2887 TCGv_i64 t0 = tcg_temp_ebb_new_i64(); 2888 TCGv_i64 t1 = tcg_temp_ebb_new_i64(); 2889 TCGv_i64 t2 = tcg_temp_ebb_new_i64(); 2890 TCGv_i64 t3 = tcg_temp_ebb_new_i64(); 2891 tcg_gen_mulu2_i64(t0, t1, arg1, arg2); 2892 /* Adjust for negative inputs. */ 2893 tcg_gen_sari_i64(t2, arg1, 63); 2894 tcg_gen_sari_i64(t3, arg2, 63); 2895 tcg_gen_and_i64(t2, t2, arg2); 2896 tcg_gen_and_i64(t3, t3, arg1); 2897 tcg_gen_sub_i64(rh, t1, t2); 2898 tcg_gen_sub_i64(rh, rh, t3); 2899 tcg_gen_mov_i64(rl, t0); 2900 tcg_temp_free_i64(t0); 2901 tcg_temp_free_i64(t1); 2902 tcg_temp_free_i64(t2); 2903 tcg_temp_free_i64(t3); 2904 } else { 2905 TCGv_i64 t0 = tcg_temp_ebb_new_i64(); 2906 tcg_gen_mul_i64(t0, arg1, arg2); 2907 gen_helper_mulsh_i64(rh, arg1, arg2); 2908 tcg_gen_mov_i64(rl, t0); 2909 tcg_temp_free_i64(t0); 2910 } 2911 } 2912 2913 void tcg_gen_mulsu2_i64(TCGv_i64 rl, TCGv_i64 rh, TCGv_i64 arg1, TCGv_i64 arg2) 2914 { 2915 TCGv_i64 t0 = tcg_temp_ebb_new_i64(); 2916 TCGv_i64 t1 = tcg_temp_ebb_new_i64(); 2917 TCGv_i64 t2 = tcg_temp_ebb_new_i64(); 2918 tcg_gen_mulu2_i64(t0, t1, arg1, arg2); 2919 /* Adjust for negative input for the signed arg1. */ 2920 tcg_gen_sari_i64(t2, arg1, 63); 2921 tcg_gen_and_i64(t2, t2, arg2); 2922 tcg_gen_sub_i64(rh, t1, t2); 2923 tcg_gen_mov_i64(rl, t0); 2924 tcg_temp_free_i64(t0); 2925 tcg_temp_free_i64(t1); 2926 tcg_temp_free_i64(t2); 2927 } 2928 2929 void tcg_gen_smin_i64(TCGv_i64 ret, TCGv_i64 a, TCGv_i64 b) 2930 { 2931 tcg_gen_movcond_i64(TCG_COND_LT, ret, a, b, a, b); 2932 } 2933 2934 void tcg_gen_umin_i64(TCGv_i64 ret, TCGv_i64 a, TCGv_i64 b) 2935 { 2936 tcg_gen_movcond_i64(TCG_COND_LTU, ret, a, b, a, b); 2937 } 2938 2939 void tcg_gen_smax_i64(TCGv_i64 ret, TCGv_i64 a, TCGv_i64 b) 2940 { 2941 tcg_gen_movcond_i64(TCG_COND_LT, ret, a, b, b, a); 2942 } 2943 2944 void tcg_gen_umax_i64(TCGv_i64 ret, TCGv_i64 a, TCGv_i64 b) 2945 { 2946 tcg_gen_movcond_i64(TCG_COND_LTU, ret, a, b, b, a); 2947 } 2948 2949 void tcg_gen_abs_i64(TCGv_i64 ret, TCGv_i64 a) 2950 { 2951 TCGv_i64 t = tcg_temp_ebb_new_i64(); 2952 2953 tcg_gen_sari_i64(t, a, 63); 2954 tcg_gen_xor_i64(ret, a, t); 2955 tcg_gen_sub_i64(ret, ret, t); 2956 tcg_temp_free_i64(t); 2957 } 2958 2959 /* Size changing operations. */ 2960 2961 void tcg_gen_extrl_i64_i32(TCGv_i32 ret, TCGv_i64 arg) 2962 { 2963 if (TCG_TARGET_REG_BITS == 32) { 2964 tcg_gen_mov_i32(ret, TCGV_LOW(arg)); 2965 } else if (TCG_TARGET_HAS_extr_i64_i32) { 2966 tcg_gen_op2(INDEX_op_extrl_i64_i32, TCG_TYPE_I32, 2967 tcgv_i32_arg(ret), tcgv_i64_arg(arg)); 2968 } else { 2969 tcg_gen_mov_i32(ret, (TCGv_i32)arg); 2970 } 2971 } 2972 2973 void tcg_gen_extrh_i64_i32(TCGv_i32 ret, TCGv_i64 arg) 2974 { 2975 if (TCG_TARGET_REG_BITS == 32) { 2976 tcg_gen_mov_i32(ret, TCGV_HIGH(arg)); 2977 } else if (TCG_TARGET_HAS_extr_i64_i32) { 2978 tcg_gen_op2(INDEX_op_extrh_i64_i32, TCG_TYPE_I32, 2979 tcgv_i32_arg(ret), tcgv_i64_arg(arg)); 2980 } else { 2981 TCGv_i64 t = tcg_temp_ebb_new_i64(); 2982 tcg_gen_shri_i64(t, arg, 32); 2983 tcg_gen_mov_i32(ret, (TCGv_i32)t); 2984 tcg_temp_free_i64(t); 2985 } 2986 } 2987 2988 void tcg_gen_extu_i32_i64(TCGv_i64 ret, TCGv_i32 arg) 2989 { 2990 if (TCG_TARGET_REG_BITS == 32) { 2991 tcg_gen_mov_i32(TCGV_LOW(ret), arg); 2992 tcg_gen_movi_i32(TCGV_HIGH(ret), 0); 2993 } else { 2994 tcg_gen_op2(INDEX_op_extu_i32_i64, TCG_TYPE_I64, 2995 tcgv_i64_arg(ret), tcgv_i32_arg(arg)); 2996 } 2997 } 2998 2999 void tcg_gen_ext_i32_i64(TCGv_i64 ret, TCGv_i32 arg) 3000 { 3001 if (TCG_TARGET_REG_BITS == 32) { 3002 tcg_gen_mov_i32(TCGV_LOW(ret), arg); 3003 tcg_gen_sari_i32(TCGV_HIGH(ret), TCGV_LOW(ret), 31); 3004 } else { 3005 tcg_gen_op2(INDEX_op_ext_i32_i64, TCG_TYPE_I64, 3006 tcgv_i64_arg(ret), tcgv_i32_arg(arg)); 3007 } 3008 } 3009 3010 void tcg_gen_concat_i32_i64(TCGv_i64 dest, TCGv_i32 low, TCGv_i32 high) 3011 { 3012 TCGv_i64 tmp; 3013 3014 if (TCG_TARGET_REG_BITS == 32) { 3015 tcg_gen_mov_i32(TCGV_LOW(dest), low); 3016 tcg_gen_mov_i32(TCGV_HIGH(dest), high); 3017 return; 3018 } 3019 3020 tmp = tcg_temp_ebb_new_i64(); 3021 /* These extensions are only needed for type correctness. 3022 We may be able to do better given target specific information. */ 3023 tcg_gen_extu_i32_i64(tmp, high); 3024 tcg_gen_extu_i32_i64(dest, low); 3025 /* If deposit is available, use it. Otherwise use the extra 3026 knowledge that we have of the zero-extensions above. */ 3027 if (TCG_TARGET_deposit_valid(TCG_TYPE_I64, 32, 32)) { 3028 tcg_gen_deposit_i64(dest, dest, tmp, 32, 32); 3029 } else { 3030 tcg_gen_shli_i64(tmp, tmp, 32); 3031 tcg_gen_or_i64(dest, dest, tmp); 3032 } 3033 tcg_temp_free_i64(tmp); 3034 } 3035 3036 void tcg_gen_extr_i64_i32(TCGv_i32 lo, TCGv_i32 hi, TCGv_i64 arg) 3037 { 3038 if (TCG_TARGET_REG_BITS == 32) { 3039 tcg_gen_mov_i32(lo, TCGV_LOW(arg)); 3040 tcg_gen_mov_i32(hi, TCGV_HIGH(arg)); 3041 } else { 3042 tcg_gen_extrl_i64_i32(lo, arg); 3043 tcg_gen_extrh_i64_i32(hi, arg); 3044 } 3045 } 3046 3047 void tcg_gen_extr32_i64(TCGv_i64 lo, TCGv_i64 hi, TCGv_i64 arg) 3048 { 3049 tcg_gen_ext32u_i64(lo, arg); 3050 tcg_gen_shri_i64(hi, arg, 32); 3051 } 3052 3053 void tcg_gen_concat32_i64(TCGv_i64 ret, TCGv_i64 lo, TCGv_i64 hi) 3054 { 3055 tcg_gen_deposit_i64(ret, lo, hi, 32, 32); 3056 } 3057 3058 void tcg_gen_extr_i128_i64(TCGv_i64 lo, TCGv_i64 hi, TCGv_i128 arg) 3059 { 3060 tcg_gen_mov_i64(lo, TCGV128_LOW(arg)); 3061 tcg_gen_mov_i64(hi, TCGV128_HIGH(arg)); 3062 } 3063 3064 void tcg_gen_concat_i64_i128(TCGv_i128 ret, TCGv_i64 lo, TCGv_i64 hi) 3065 { 3066 tcg_gen_mov_i64(TCGV128_LOW(ret), lo); 3067 tcg_gen_mov_i64(TCGV128_HIGH(ret), hi); 3068 } 3069 3070 void tcg_gen_mov_i128(TCGv_i128 dst, TCGv_i128 src) 3071 { 3072 if (dst != src) { 3073 tcg_gen_mov_i64(TCGV128_LOW(dst), TCGV128_LOW(src)); 3074 tcg_gen_mov_i64(TCGV128_HIGH(dst), TCGV128_HIGH(src)); 3075 } 3076 } 3077 3078 void tcg_gen_ld_i128(TCGv_i128 ret, TCGv_ptr base, tcg_target_long offset) 3079 { 3080 if (HOST_BIG_ENDIAN) { 3081 tcg_gen_ld_i64(TCGV128_HIGH(ret), base, offset); 3082 tcg_gen_ld_i64(TCGV128_LOW(ret), base, offset + 8); 3083 } else { 3084 tcg_gen_ld_i64(TCGV128_LOW(ret), base, offset); 3085 tcg_gen_ld_i64(TCGV128_HIGH(ret), base, offset + 8); 3086 } 3087 } 3088 3089 void tcg_gen_st_i128(TCGv_i128 val, TCGv_ptr base, tcg_target_long offset) 3090 { 3091 if (HOST_BIG_ENDIAN) { 3092 tcg_gen_st_i64(TCGV128_HIGH(val), base, offset); 3093 tcg_gen_st_i64(TCGV128_LOW(val), base, offset + 8); 3094 } else { 3095 tcg_gen_st_i64(TCGV128_LOW(val), base, offset); 3096 tcg_gen_st_i64(TCGV128_HIGH(val), base, offset + 8); 3097 } 3098 } 3099 3100 /* QEMU specific operations. */ 3101 3102 void tcg_gen_exit_tb(const TranslationBlock *tb, unsigned idx) 3103 { 3104 /* 3105 * Let the jit code return the read-only version of the 3106 * TranslationBlock, so that we minimize the pc-relative 3107 * distance of the address of the exit_tb code to TB. 3108 * This will improve utilization of pc-relative address loads. 3109 * 3110 * TODO: Move this to translator_loop, so that all const 3111 * TranslationBlock pointers refer to read-only memory. 3112 * This requires coordination with targets that do not use 3113 * the translator_loop. 3114 */ 3115 uintptr_t val = (uintptr_t)tcg_splitwx_to_rx((void *)tb) + idx; 3116 3117 if (tb == NULL) { 3118 tcg_debug_assert(idx == 0); 3119 } else if (idx <= TB_EXIT_IDXMAX) { 3120 #ifdef CONFIG_DEBUG_TCG 3121 /* This is an exit following a goto_tb. Verify that we have 3122 seen this numbered exit before, via tcg_gen_goto_tb. */ 3123 tcg_debug_assert(tcg_ctx->goto_tb_issue_mask & (1 << idx)); 3124 #endif 3125 } else { 3126 /* This is an exit via the exitreq label. */ 3127 tcg_debug_assert(idx == TB_EXIT_REQUESTED); 3128 } 3129 3130 tcg_gen_op1i(INDEX_op_exit_tb, 0, val); 3131 } 3132 3133 void tcg_gen_goto_tb(unsigned idx) 3134 { 3135 /* We tested CF_NO_GOTO_TB in translator_use_goto_tb. */ 3136 tcg_debug_assert(!(tcg_ctx->gen_tb->cflags & CF_NO_GOTO_TB)); 3137 /* We only support two chained exits. */ 3138 tcg_debug_assert(idx <= TB_EXIT_IDXMAX); 3139 #ifdef CONFIG_DEBUG_TCG 3140 /* Verify that we haven't seen this numbered exit before. */ 3141 tcg_debug_assert((tcg_ctx->goto_tb_issue_mask & (1 << idx)) == 0); 3142 tcg_ctx->goto_tb_issue_mask |= 1 << idx; 3143 #endif 3144 plugin_gen_disable_mem_helpers(); 3145 tcg_gen_op1i(INDEX_op_goto_tb, 0, idx); 3146 } 3147 3148 void tcg_gen_lookup_and_goto_ptr(void) 3149 { 3150 TCGv_ptr ptr; 3151 3152 if (tcg_ctx->gen_tb->cflags & CF_NO_GOTO_PTR) { 3153 tcg_gen_exit_tb(NULL, 0); 3154 return; 3155 } 3156 3157 plugin_gen_disable_mem_helpers(); 3158 ptr = tcg_temp_ebb_new_ptr(); 3159 gen_helper_lookup_tb_ptr(ptr, tcg_env); 3160 tcg_gen_op1i(INDEX_op_goto_ptr, TCG_TYPE_PTR, tcgv_ptr_arg(ptr)); 3161 tcg_temp_free_ptr(ptr); 3162 } 3163