1 /* 2 * Tiny Code Generator for QEMU 3 * 4 * Copyright (c) 2008 Fabrice Bellard 5 * 6 * Permission is hereby granted, free of charge, to any person obtaining a copy 7 * of this software and associated documentation files (the "Software"), to deal 8 * in the Software without restriction, including without limitation the rights 9 * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 10 * copies of the Software, and to permit persons to whom the Software is 11 * furnished to do so, subject to the following conditions: 12 * 13 * The above copyright notice and this permission notice shall be included in 14 * all copies or substantial portions of the Software. 15 * 16 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 17 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 18 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL 19 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 20 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 21 * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN 22 * THE SOFTWARE. 23 */ 24 25 #include "qemu/osdep.h" 26 #include "tcg/tcg.h" 27 #include "tcg/tcg-temp-internal.h" 28 #include "tcg/tcg-op-common.h" 29 #include "exec/translation-block.h" 30 #include "exec/plugin-gen.h" 31 #include "tcg-internal.h" 32 #include "tcg-has.h" 33 34 /* 35 * Encourage the compiler to tail-call to a function, rather than inlining. 36 * Minimizes code size across 99 bottles of beer on the wall. 37 */ 38 #define NI __attribute__((noinline)) 39 40 TCGOp * NI tcg_gen_op1(TCGOpcode opc, TCGType type, TCGArg a1) 41 { 42 TCGOp *op = tcg_emit_op(opc, 1); 43 TCGOP_TYPE(op) = type; 44 op->args[0] = a1; 45 return op; 46 } 47 48 TCGOp * NI tcg_gen_op2(TCGOpcode opc, TCGType type, TCGArg a1, TCGArg a2) 49 { 50 TCGOp *op = tcg_emit_op(opc, 2); 51 TCGOP_TYPE(op) = type; 52 op->args[0] = a1; 53 op->args[1] = a2; 54 return op; 55 } 56 57 TCGOp * NI tcg_gen_op3(TCGOpcode opc, TCGType type, TCGArg a1, 58 TCGArg a2, TCGArg a3) 59 { 60 TCGOp *op = tcg_emit_op(opc, 3); 61 TCGOP_TYPE(op) = type; 62 op->args[0] = a1; 63 op->args[1] = a2; 64 op->args[2] = a3; 65 return op; 66 } 67 68 TCGOp * NI tcg_gen_op4(TCGOpcode opc, TCGType type, TCGArg a1, TCGArg a2, 69 TCGArg a3, TCGArg a4) 70 { 71 TCGOp *op = tcg_emit_op(opc, 4); 72 TCGOP_TYPE(op) = type; 73 op->args[0] = a1; 74 op->args[1] = a2; 75 op->args[2] = a3; 76 op->args[3] = a4; 77 return op; 78 } 79 80 TCGOp * NI tcg_gen_op5(TCGOpcode opc, TCGType type, TCGArg a1, TCGArg a2, 81 TCGArg a3, TCGArg a4, TCGArg a5) 82 { 83 TCGOp *op = tcg_emit_op(opc, 5); 84 TCGOP_TYPE(op) = type; 85 op->args[0] = a1; 86 op->args[1] = a2; 87 op->args[2] = a3; 88 op->args[3] = a4; 89 op->args[4] = a5; 90 return op; 91 } 92 93 TCGOp * NI tcg_gen_op6(TCGOpcode opc, TCGType type, TCGArg a1, TCGArg a2, 94 TCGArg a3, TCGArg a4, TCGArg a5, TCGArg a6) 95 { 96 TCGOp *op = tcg_emit_op(opc, 6); 97 TCGOP_TYPE(op) = type; 98 op->args[0] = a1; 99 op->args[1] = a2; 100 op->args[2] = a3; 101 op->args[3] = a4; 102 op->args[4] = a5; 103 op->args[5] = a6; 104 return op; 105 } 106 107 /* 108 * With CONFIG_DEBUG_TCG, tcgv_*_tmp via tcgv_*_arg, is an out-of-line 109 * assertion check. Force tail calls to avoid too much code expansion. 110 */ 111 #ifdef CONFIG_DEBUG_TCG 112 # define DNI NI 113 #else 114 # define DNI 115 #endif 116 117 static void DNI tcg_gen_op1_i32(TCGOpcode opc, TCGType type, TCGv_i32 a1) 118 { 119 tcg_gen_op1(opc, type, tcgv_i32_arg(a1)); 120 } 121 122 static void DNI tcg_gen_op1_i64(TCGOpcode opc, TCGType type, TCGv_i64 a1) 123 { 124 tcg_gen_op1(opc, type, tcgv_i64_arg(a1)); 125 } 126 127 static TCGOp * DNI tcg_gen_op1i(TCGOpcode opc, TCGType type, TCGArg a1) 128 { 129 return tcg_gen_op1(opc, type, a1); 130 } 131 132 static void DNI tcg_gen_op2_i32(TCGOpcode opc, TCGv_i32 a1, TCGv_i32 a2) 133 { 134 tcg_gen_op2(opc, TCG_TYPE_I32, tcgv_i32_arg(a1), tcgv_i32_arg(a2)); 135 } 136 137 static void DNI tcg_gen_op2_i64(TCGOpcode opc, TCGv_i64 a1, TCGv_i64 a2) 138 { 139 tcg_gen_op2(opc, TCG_TYPE_I64, tcgv_i64_arg(a1), tcgv_i64_arg(a2)); 140 } 141 142 static void DNI tcg_gen_op3_i32(TCGOpcode opc, TCGv_i32 a1, 143 TCGv_i32 a2, TCGv_i32 a3) 144 { 145 tcg_gen_op3(opc, TCG_TYPE_I32, tcgv_i32_arg(a1), 146 tcgv_i32_arg(a2), tcgv_i32_arg(a3)); 147 } 148 149 static void DNI tcg_gen_op3_i64(TCGOpcode opc, TCGv_i64 a1, 150 TCGv_i64 a2, TCGv_i64 a3) 151 { 152 tcg_gen_op3(opc, TCG_TYPE_I64, tcgv_i64_arg(a1), 153 tcgv_i64_arg(a2), tcgv_i64_arg(a3)); 154 } 155 156 static void DNI tcg_gen_op3i_i32(TCGOpcode opc, TCGv_i32 a1, 157 TCGv_i32 a2, TCGArg a3) 158 { 159 tcg_gen_op3(opc, TCG_TYPE_I32, tcgv_i32_arg(a1), tcgv_i32_arg(a2), a3); 160 } 161 162 static void DNI tcg_gen_op3i_i64(TCGOpcode opc, TCGv_i64 a1, 163 TCGv_i64 a2, TCGArg a3) 164 { 165 tcg_gen_op3(opc, TCG_TYPE_I64, tcgv_i64_arg(a1), tcgv_i64_arg(a2), a3); 166 } 167 168 static void DNI tcg_gen_ldst_op_i32(TCGOpcode opc, TCGv_i32 val, 169 TCGv_ptr base, TCGArg offset) 170 { 171 tcg_gen_op3(opc, TCG_TYPE_I32, tcgv_i32_arg(val), 172 tcgv_ptr_arg(base), offset); 173 } 174 175 static void DNI tcg_gen_ldst_op_i64(TCGOpcode opc, TCGv_i64 val, 176 TCGv_ptr base, TCGArg offset) 177 { 178 tcg_gen_op3(opc, TCG_TYPE_I64, tcgv_i64_arg(val), 179 tcgv_ptr_arg(base), offset); 180 } 181 182 static void DNI tcg_gen_op4_i32(TCGOpcode opc, TCGv_i32 a1, TCGv_i32 a2, 183 TCGv_i32 a3, TCGv_i32 a4) 184 { 185 tcg_gen_op4(opc, TCG_TYPE_I32, tcgv_i32_arg(a1), tcgv_i32_arg(a2), 186 tcgv_i32_arg(a3), tcgv_i32_arg(a4)); 187 } 188 189 static void DNI tcg_gen_op4_i64(TCGOpcode opc, TCGv_i64 a1, TCGv_i64 a2, 190 TCGv_i64 a3, TCGv_i64 a4) 191 { 192 tcg_gen_op4(opc, TCG_TYPE_I64, tcgv_i64_arg(a1), tcgv_i64_arg(a2), 193 tcgv_i64_arg(a3), tcgv_i64_arg(a4)); 194 } 195 196 static void DNI tcg_gen_op4i_i32(TCGOpcode opc, TCGv_i32 a1, TCGv_i32 a2, 197 TCGv_i32 a3, TCGArg a4) 198 { 199 tcg_gen_op4(opc, TCG_TYPE_I32, tcgv_i32_arg(a1), tcgv_i32_arg(a2), 200 tcgv_i32_arg(a3), a4); 201 } 202 203 static void DNI tcg_gen_op4i_i64(TCGOpcode opc, TCGv_i64 a1, TCGv_i64 a2, 204 TCGv_i64 a3, TCGArg a4) 205 { 206 tcg_gen_op4(opc, TCG_TYPE_I64, tcgv_i64_arg(a1), tcgv_i64_arg(a2), 207 tcgv_i64_arg(a3), a4); 208 } 209 210 static TCGOp * DNI tcg_gen_op4ii_i32(TCGOpcode opc, TCGv_i32 a1, TCGv_i32 a2, 211 TCGArg a3, TCGArg a4) 212 { 213 return tcg_gen_op4(opc, TCG_TYPE_I32, 214 tcgv_i32_arg(a1), tcgv_i32_arg(a2), a3, a4); 215 } 216 217 static TCGOp * DNI tcg_gen_op4ii_i64(TCGOpcode opc, TCGv_i64 a1, TCGv_i64 a2, 218 TCGArg a3, TCGArg a4) 219 { 220 return tcg_gen_op4(opc, TCG_TYPE_I64, 221 tcgv_i64_arg(a1), tcgv_i64_arg(a2), a3, a4); 222 } 223 224 static void DNI tcg_gen_op5_i32(TCGOpcode opc, TCGv_i32 a1, TCGv_i32 a2, 225 TCGv_i32 a3, TCGv_i32 a4, TCGv_i32 a5) 226 { 227 tcg_gen_op5(opc, TCG_TYPE_I32, tcgv_i32_arg(a1), tcgv_i32_arg(a2), 228 tcgv_i32_arg(a3), tcgv_i32_arg(a4), tcgv_i32_arg(a5)); 229 } 230 231 static void DNI tcg_gen_op5_i64(TCGOpcode opc, TCGv_i64 a1, TCGv_i64 a2, 232 TCGv_i64 a3, TCGv_i64 a4, TCGv_i64 a5) 233 { 234 tcg_gen_op5(opc, TCG_TYPE_I64, tcgv_i64_arg(a1), tcgv_i64_arg(a2), 235 tcgv_i64_arg(a3), tcgv_i64_arg(a4), tcgv_i64_arg(a5)); 236 } 237 238 static void DNI tcg_gen_op5ii_i32(TCGOpcode opc, TCGv_i32 a1, TCGv_i32 a2, 239 TCGv_i32 a3, TCGArg a4, TCGArg a5) 240 { 241 tcg_gen_op5(opc, TCG_TYPE_I32, tcgv_i32_arg(a1), tcgv_i32_arg(a2), 242 tcgv_i32_arg(a3), a4, a5); 243 } 244 245 static void DNI tcg_gen_op5ii_i64(TCGOpcode opc, TCGv_i64 a1, TCGv_i64 a2, 246 TCGv_i64 a3, TCGArg a4, TCGArg a5) 247 { 248 tcg_gen_op5(opc, TCG_TYPE_I64, tcgv_i64_arg(a1), tcgv_i64_arg(a2), 249 tcgv_i64_arg(a3), a4, a5); 250 } 251 252 static void DNI tcg_gen_op6_i32(TCGOpcode opc, TCGv_i32 a1, TCGv_i32 a2, 253 TCGv_i32 a3, TCGv_i32 a4, 254 TCGv_i32 a5, TCGv_i32 a6) 255 { 256 tcg_gen_op6(opc, TCG_TYPE_I32, tcgv_i32_arg(a1), tcgv_i32_arg(a2), 257 tcgv_i32_arg(a3), tcgv_i32_arg(a4), tcgv_i32_arg(a5), 258 tcgv_i32_arg(a6)); 259 } 260 261 static void DNI tcg_gen_op6_i64(TCGOpcode opc, TCGv_i64 a1, TCGv_i64 a2, 262 TCGv_i64 a3, TCGv_i64 a4, 263 TCGv_i64 a5, TCGv_i64 a6) 264 { 265 tcg_gen_op6(opc, TCG_TYPE_I64, tcgv_i64_arg(a1), tcgv_i64_arg(a2), 266 tcgv_i64_arg(a3), tcgv_i64_arg(a4), tcgv_i64_arg(a5), 267 tcgv_i64_arg(a6)); 268 } 269 270 static void DNI tcg_gen_op6i_i32(TCGOpcode opc, TCGv_i32 a1, TCGv_i32 a2, 271 TCGv_i32 a3, TCGv_i32 a4, 272 TCGv_i32 a5, TCGArg a6) 273 { 274 tcg_gen_op6(opc, TCG_TYPE_I32, tcgv_i32_arg(a1), tcgv_i32_arg(a2), 275 tcgv_i32_arg(a3), tcgv_i32_arg(a4), tcgv_i32_arg(a5), a6); 276 } 277 278 static void DNI tcg_gen_op6i_i64(TCGOpcode opc, TCGv_i64 a1, TCGv_i64 a2, 279 TCGv_i64 a3, TCGv_i64 a4, 280 TCGv_i64 a5, TCGArg a6) 281 { 282 tcg_gen_op6(opc, TCG_TYPE_I64, tcgv_i64_arg(a1), tcgv_i64_arg(a2), 283 tcgv_i64_arg(a3), tcgv_i64_arg(a4), tcgv_i64_arg(a5), a6); 284 } 285 286 static TCGOp * DNI tcg_gen_op6ii_i32(TCGOpcode opc, TCGv_i32 a1, TCGv_i32 a2, 287 TCGv_i32 a3, TCGv_i32 a4, 288 TCGArg a5, TCGArg a6) 289 { 290 return tcg_gen_op6(opc, TCG_TYPE_I32, tcgv_i32_arg(a1), tcgv_i32_arg(a2), 291 tcgv_i32_arg(a3), tcgv_i32_arg(a4), a5, a6); 292 } 293 294 /* Generic ops. */ 295 296 void gen_set_label(TCGLabel *l) 297 { 298 l->present = 1; 299 tcg_gen_op1(INDEX_op_set_label, 0, label_arg(l)); 300 } 301 302 static void add_as_label_use(TCGLabel *l, TCGOp *op) 303 { 304 TCGLabelUse *u = tcg_malloc(sizeof(TCGLabelUse)); 305 306 u->op = op; 307 QSIMPLEQ_INSERT_TAIL(&l->branches, u, next); 308 } 309 310 void tcg_gen_br(TCGLabel *l) 311 { 312 add_as_label_use(l, tcg_gen_op1(INDEX_op_br, 0, label_arg(l))); 313 } 314 315 void tcg_gen_mb(TCGBar mb_type) 316 { 317 #ifdef CONFIG_USER_ONLY 318 bool parallel = tcg_ctx->gen_tb->cflags & CF_PARALLEL; 319 #else 320 /* 321 * It is tempting to elide the barrier in a uniprocessor context. 322 * However, even with a single cpu we have i/o threads running in 323 * parallel, and lack of memory order can result in e.g. virtio 324 * queue entries being read incorrectly. 325 */ 326 bool parallel = true; 327 #endif 328 329 if (parallel) { 330 tcg_gen_op1(INDEX_op_mb, 0, mb_type); 331 } 332 } 333 334 void tcg_gen_plugin_cb(unsigned from) 335 { 336 tcg_gen_op1(INDEX_op_plugin_cb, 0, from); 337 } 338 339 void tcg_gen_plugin_mem_cb(TCGv_i64 addr, unsigned meminfo) 340 { 341 tcg_gen_op2(INDEX_op_plugin_mem_cb, 0, tcgv_i64_arg(addr), meminfo); 342 } 343 344 /* 32 bit ops */ 345 346 void tcg_gen_discard_i32(TCGv_i32 arg) 347 { 348 tcg_gen_op1_i32(INDEX_op_discard, TCG_TYPE_I32, arg); 349 } 350 351 void tcg_gen_mov_i32(TCGv_i32 ret, TCGv_i32 arg) 352 { 353 if (ret != arg) { 354 tcg_gen_op2_i32(INDEX_op_mov, ret, arg); 355 } 356 } 357 358 void tcg_gen_movi_i32(TCGv_i32 ret, int32_t arg) 359 { 360 tcg_gen_mov_i32(ret, tcg_constant_i32(arg)); 361 } 362 363 void tcg_gen_add_i32(TCGv_i32 ret, TCGv_i32 arg1, TCGv_i32 arg2) 364 { 365 tcg_gen_op3_i32(INDEX_op_add, ret, arg1, arg2); 366 } 367 368 void tcg_gen_addi_i32(TCGv_i32 ret, TCGv_i32 arg1, int32_t arg2) 369 { 370 /* some cases can be optimized here */ 371 if (arg2 == 0) { 372 tcg_gen_mov_i32(ret, arg1); 373 } else { 374 tcg_gen_add_i32(ret, arg1, tcg_constant_i32(arg2)); 375 } 376 } 377 378 void tcg_gen_sub_i32(TCGv_i32 ret, TCGv_i32 arg1, TCGv_i32 arg2) 379 { 380 tcg_gen_op3_i32(INDEX_op_sub, ret, arg1, arg2); 381 } 382 383 void tcg_gen_subfi_i32(TCGv_i32 ret, int32_t arg1, TCGv_i32 arg2) 384 { 385 if (arg1 == 0) { 386 tcg_gen_neg_i32(ret, arg2); 387 } else { 388 tcg_gen_sub_i32(ret, tcg_constant_i32(arg1), arg2); 389 } 390 } 391 392 void tcg_gen_subi_i32(TCGv_i32 ret, TCGv_i32 arg1, int32_t arg2) 393 { 394 tcg_gen_addi_i32(ret, arg1, -arg2); 395 } 396 397 void tcg_gen_neg_i32(TCGv_i32 ret, TCGv_i32 arg) 398 { 399 tcg_gen_op2_i32(INDEX_op_neg, ret, arg); 400 } 401 402 void tcg_gen_and_i32(TCGv_i32 ret, TCGv_i32 arg1, TCGv_i32 arg2) 403 { 404 tcg_gen_op3_i32(INDEX_op_and, ret, arg1, arg2); 405 } 406 407 void tcg_gen_andi_i32(TCGv_i32 ret, TCGv_i32 arg1, int32_t arg2) 408 { 409 /* Some cases can be optimized here. */ 410 switch (arg2) { 411 case 0: 412 tcg_gen_movi_i32(ret, 0); 413 return; 414 case -1: 415 tcg_gen_mov_i32(ret, arg1); 416 return; 417 default: 418 /* 419 * Canonicalize on extract, if valid. This aids x86 with its 420 * 2 operand MOVZBL and 2 operand AND, selecting the TCGOpcode 421 * which does not require matching operands. Other backends can 422 * trivially expand the extract to AND during code generation. 423 */ 424 if (!(arg2 & (arg2 + 1))) { 425 unsigned len = ctz32(~arg2); 426 if (TCG_TARGET_extract_valid(TCG_TYPE_I32, 0, len)) { 427 tcg_gen_extract_i32(ret, arg1, 0, len); 428 return; 429 } 430 } 431 break; 432 } 433 434 tcg_gen_and_i32(ret, arg1, tcg_constant_i32(arg2)); 435 } 436 437 void tcg_gen_or_i32(TCGv_i32 ret, TCGv_i32 arg1, TCGv_i32 arg2) 438 { 439 tcg_gen_op3_i32(INDEX_op_or, ret, arg1, arg2); 440 } 441 442 void tcg_gen_ori_i32(TCGv_i32 ret, TCGv_i32 arg1, int32_t arg2) 443 { 444 /* Some cases can be optimized here. */ 445 if (arg2 == -1) { 446 tcg_gen_movi_i32(ret, -1); 447 } else if (arg2 == 0) { 448 tcg_gen_mov_i32(ret, arg1); 449 } else { 450 tcg_gen_or_i32(ret, arg1, tcg_constant_i32(arg2)); 451 } 452 } 453 454 void tcg_gen_xor_i32(TCGv_i32 ret, TCGv_i32 arg1, TCGv_i32 arg2) 455 { 456 tcg_gen_op3_i32(INDEX_op_xor, ret, arg1, arg2); 457 } 458 459 void tcg_gen_xori_i32(TCGv_i32 ret, TCGv_i32 arg1, int32_t arg2) 460 { 461 /* Some cases can be optimized here. */ 462 if (arg2 == 0) { 463 tcg_gen_mov_i32(ret, arg1); 464 } else if (arg2 == -1 && 465 tcg_op_supported(INDEX_op_not, TCG_TYPE_I32, 0)) { 466 /* Don't recurse with tcg_gen_not_i32. */ 467 tcg_gen_op2_i32(INDEX_op_not, ret, arg1); 468 } else { 469 tcg_gen_xor_i32(ret, arg1, tcg_constant_i32(arg2)); 470 } 471 } 472 473 void tcg_gen_not_i32(TCGv_i32 ret, TCGv_i32 arg) 474 { 475 if (tcg_op_supported(INDEX_op_not, TCG_TYPE_I32, 0)) { 476 tcg_gen_op2_i32(INDEX_op_not, ret, arg); 477 } else { 478 tcg_gen_xori_i32(ret, arg, -1); 479 } 480 } 481 482 void tcg_gen_shl_i32(TCGv_i32 ret, TCGv_i32 arg1, TCGv_i32 arg2) 483 { 484 tcg_gen_op3_i32(INDEX_op_shl_i32, ret, arg1, arg2); 485 } 486 487 void tcg_gen_shli_i32(TCGv_i32 ret, TCGv_i32 arg1, int32_t arg2) 488 { 489 tcg_debug_assert(arg2 >= 0 && arg2 < 32); 490 if (arg2 == 0) { 491 tcg_gen_mov_i32(ret, arg1); 492 } else { 493 tcg_gen_shl_i32(ret, arg1, tcg_constant_i32(arg2)); 494 } 495 } 496 497 void tcg_gen_shr_i32(TCGv_i32 ret, TCGv_i32 arg1, TCGv_i32 arg2) 498 { 499 tcg_gen_op3_i32(INDEX_op_shr_i32, ret, arg1, arg2); 500 } 501 502 void tcg_gen_shri_i32(TCGv_i32 ret, TCGv_i32 arg1, int32_t arg2) 503 { 504 tcg_debug_assert(arg2 >= 0 && arg2 < 32); 505 if (arg2 == 0) { 506 tcg_gen_mov_i32(ret, arg1); 507 } else { 508 tcg_gen_shr_i32(ret, arg1, tcg_constant_i32(arg2)); 509 } 510 } 511 512 void tcg_gen_sar_i32(TCGv_i32 ret, TCGv_i32 arg1, TCGv_i32 arg2) 513 { 514 tcg_gen_op3_i32(INDEX_op_sar_i32, ret, arg1, arg2); 515 } 516 517 void tcg_gen_sari_i32(TCGv_i32 ret, TCGv_i32 arg1, int32_t arg2) 518 { 519 tcg_debug_assert(arg2 >= 0 && arg2 < 32); 520 if (arg2 == 0) { 521 tcg_gen_mov_i32(ret, arg1); 522 } else { 523 tcg_gen_sar_i32(ret, arg1, tcg_constant_i32(arg2)); 524 } 525 } 526 527 void tcg_gen_brcond_i32(TCGCond cond, TCGv_i32 arg1, TCGv_i32 arg2, TCGLabel *l) 528 { 529 if (cond == TCG_COND_ALWAYS) { 530 tcg_gen_br(l); 531 } else if (cond != TCG_COND_NEVER) { 532 TCGOp *op = tcg_gen_op4ii_i32(INDEX_op_brcond_i32, 533 arg1, arg2, cond, label_arg(l)); 534 add_as_label_use(l, op); 535 } 536 } 537 538 void tcg_gen_brcondi_i32(TCGCond cond, TCGv_i32 arg1, int32_t arg2, TCGLabel *l) 539 { 540 if (cond == TCG_COND_ALWAYS) { 541 tcg_gen_br(l); 542 } else if (cond != TCG_COND_NEVER) { 543 tcg_gen_brcond_i32(cond, arg1, tcg_constant_i32(arg2), l); 544 } 545 } 546 547 void tcg_gen_setcond_i32(TCGCond cond, TCGv_i32 ret, 548 TCGv_i32 arg1, TCGv_i32 arg2) 549 { 550 if (cond == TCG_COND_ALWAYS) { 551 tcg_gen_movi_i32(ret, 1); 552 } else if (cond == TCG_COND_NEVER) { 553 tcg_gen_movi_i32(ret, 0); 554 } else { 555 tcg_gen_op4i_i32(INDEX_op_setcond_i32, ret, arg1, arg2, cond); 556 } 557 } 558 559 void tcg_gen_setcondi_i32(TCGCond cond, TCGv_i32 ret, 560 TCGv_i32 arg1, int32_t arg2) 561 { 562 tcg_gen_setcond_i32(cond, ret, arg1, tcg_constant_i32(arg2)); 563 } 564 565 void tcg_gen_negsetcond_i32(TCGCond cond, TCGv_i32 ret, 566 TCGv_i32 arg1, TCGv_i32 arg2) 567 { 568 if (cond == TCG_COND_ALWAYS) { 569 tcg_gen_movi_i32(ret, -1); 570 } else if (cond == TCG_COND_NEVER) { 571 tcg_gen_movi_i32(ret, 0); 572 } else if (TCG_TARGET_HAS_negsetcond_i32) { 573 tcg_gen_op4i_i32(INDEX_op_negsetcond_i32, ret, arg1, arg2, cond); 574 } else { 575 tcg_gen_setcond_i32(cond, ret, arg1, arg2); 576 tcg_gen_neg_i32(ret, ret); 577 } 578 } 579 580 void tcg_gen_negsetcondi_i32(TCGCond cond, TCGv_i32 ret, 581 TCGv_i32 arg1, int32_t arg2) 582 { 583 tcg_gen_negsetcond_i32(cond, ret, arg1, tcg_constant_i32(arg2)); 584 } 585 586 void tcg_gen_mul_i32(TCGv_i32 ret, TCGv_i32 arg1, TCGv_i32 arg2) 587 { 588 tcg_gen_op3_i32(INDEX_op_mul, ret, arg1, arg2); 589 } 590 591 void tcg_gen_muli_i32(TCGv_i32 ret, TCGv_i32 arg1, int32_t arg2) 592 { 593 if (arg2 == 0) { 594 tcg_gen_movi_i32(ret, 0); 595 } else if (is_power_of_2(arg2)) { 596 tcg_gen_shli_i32(ret, arg1, ctz32(arg2)); 597 } else { 598 tcg_gen_mul_i32(ret, arg1, tcg_constant_i32(arg2)); 599 } 600 } 601 602 void tcg_gen_div_i32(TCGv_i32 ret, TCGv_i32 arg1, TCGv_i32 arg2) 603 { 604 if (TCG_TARGET_HAS_div_i32) { 605 tcg_gen_op3_i32(INDEX_op_div_i32, ret, arg1, arg2); 606 } else if (TCG_TARGET_HAS_div2_i32) { 607 TCGv_i32 t0 = tcg_temp_ebb_new_i32(); 608 tcg_gen_sari_i32(t0, arg1, 31); 609 tcg_gen_op5_i32(INDEX_op_div2_i32, ret, t0, arg1, t0, arg2); 610 tcg_temp_free_i32(t0); 611 } else { 612 gen_helper_div_i32(ret, arg1, arg2); 613 } 614 } 615 616 void tcg_gen_rem_i32(TCGv_i32 ret, TCGv_i32 arg1, TCGv_i32 arg2) 617 { 618 if (TCG_TARGET_HAS_rem_i32) { 619 tcg_gen_op3_i32(INDEX_op_rem_i32, ret, arg1, arg2); 620 } else if (TCG_TARGET_HAS_div_i32) { 621 TCGv_i32 t0 = tcg_temp_ebb_new_i32(); 622 tcg_gen_op3_i32(INDEX_op_div_i32, t0, arg1, arg2); 623 tcg_gen_mul_i32(t0, t0, arg2); 624 tcg_gen_sub_i32(ret, arg1, t0); 625 tcg_temp_free_i32(t0); 626 } else if (TCG_TARGET_HAS_div2_i32) { 627 TCGv_i32 t0 = tcg_temp_ebb_new_i32(); 628 tcg_gen_sari_i32(t0, arg1, 31); 629 tcg_gen_op5_i32(INDEX_op_div2_i32, t0, ret, arg1, t0, arg2); 630 tcg_temp_free_i32(t0); 631 } else { 632 gen_helper_rem_i32(ret, arg1, arg2); 633 } 634 } 635 636 void tcg_gen_divu_i32(TCGv_i32 ret, TCGv_i32 arg1, TCGv_i32 arg2) 637 { 638 if (TCG_TARGET_HAS_div_i32) { 639 tcg_gen_op3_i32(INDEX_op_divu_i32, ret, arg1, arg2); 640 } else if (TCG_TARGET_HAS_div2_i32) { 641 TCGv_i32 t0 = tcg_temp_ebb_new_i32(); 642 TCGv_i32 zero = tcg_constant_i32(0); 643 tcg_gen_op5_i32(INDEX_op_divu2_i32, ret, t0, arg1, zero, arg2); 644 tcg_temp_free_i32(t0); 645 } else { 646 gen_helper_divu_i32(ret, arg1, arg2); 647 } 648 } 649 650 void tcg_gen_remu_i32(TCGv_i32 ret, TCGv_i32 arg1, TCGv_i32 arg2) 651 { 652 if (TCG_TARGET_HAS_rem_i32) { 653 tcg_gen_op3_i32(INDEX_op_remu_i32, ret, arg1, arg2); 654 } else if (TCG_TARGET_HAS_div_i32) { 655 TCGv_i32 t0 = tcg_temp_ebb_new_i32(); 656 tcg_gen_op3_i32(INDEX_op_divu_i32, t0, arg1, arg2); 657 tcg_gen_mul_i32(t0, t0, arg2); 658 tcg_gen_sub_i32(ret, arg1, t0); 659 tcg_temp_free_i32(t0); 660 } else if (TCG_TARGET_HAS_div2_i32) { 661 TCGv_i32 t0 = tcg_temp_ebb_new_i32(); 662 TCGv_i32 zero = tcg_constant_i32(0); 663 tcg_gen_op5_i32(INDEX_op_divu2_i32, t0, ret, arg1, zero, arg2); 664 tcg_temp_free_i32(t0); 665 } else { 666 gen_helper_remu_i32(ret, arg1, arg2); 667 } 668 } 669 670 void tcg_gen_andc_i32(TCGv_i32 ret, TCGv_i32 arg1, TCGv_i32 arg2) 671 { 672 if (tcg_op_supported(INDEX_op_andc, TCG_TYPE_I32, 0)) { 673 tcg_gen_op3_i32(INDEX_op_andc, ret, arg1, arg2); 674 } else { 675 TCGv_i32 t0 = tcg_temp_ebb_new_i32(); 676 tcg_gen_not_i32(t0, arg2); 677 tcg_gen_and_i32(ret, arg1, t0); 678 tcg_temp_free_i32(t0); 679 } 680 } 681 682 void tcg_gen_eqv_i32(TCGv_i32 ret, TCGv_i32 arg1, TCGv_i32 arg2) 683 { 684 if (tcg_op_supported(INDEX_op_eqv, TCG_TYPE_I32, 0)) { 685 tcg_gen_op3_i32(INDEX_op_eqv, ret, arg1, arg2); 686 } else { 687 tcg_gen_xor_i32(ret, arg1, arg2); 688 tcg_gen_not_i32(ret, ret); 689 } 690 } 691 692 void tcg_gen_nand_i32(TCGv_i32 ret, TCGv_i32 arg1, TCGv_i32 arg2) 693 { 694 if (tcg_op_supported(INDEX_op_nand, TCG_TYPE_I32, 0)) { 695 tcg_gen_op3_i32(INDEX_op_nand, ret, arg1, arg2); 696 } else { 697 tcg_gen_and_i32(ret, arg1, arg2); 698 tcg_gen_not_i32(ret, ret); 699 } 700 } 701 702 void tcg_gen_nor_i32(TCGv_i32 ret, TCGv_i32 arg1, TCGv_i32 arg2) 703 { 704 if (tcg_op_supported(INDEX_op_nor, TCG_TYPE_I32, 0)) { 705 tcg_gen_op3_i32(INDEX_op_nor, ret, arg1, arg2); 706 } else { 707 tcg_gen_or_i32(ret, arg1, arg2); 708 tcg_gen_not_i32(ret, ret); 709 } 710 } 711 712 void tcg_gen_orc_i32(TCGv_i32 ret, TCGv_i32 arg1, TCGv_i32 arg2) 713 { 714 if (tcg_op_supported(INDEX_op_orc, TCG_TYPE_I32, 0)) { 715 tcg_gen_op3_i32(INDEX_op_orc, ret, arg1, arg2); 716 } else { 717 TCGv_i32 t0 = tcg_temp_ebb_new_i32(); 718 tcg_gen_not_i32(t0, arg2); 719 tcg_gen_or_i32(ret, arg1, t0); 720 tcg_temp_free_i32(t0); 721 } 722 } 723 724 void tcg_gen_clz_i32(TCGv_i32 ret, TCGv_i32 arg1, TCGv_i32 arg2) 725 { 726 if (TCG_TARGET_HAS_clz_i32) { 727 tcg_gen_op3_i32(INDEX_op_clz_i32, ret, arg1, arg2); 728 } else if (TCG_TARGET_HAS_clz_i64) { 729 TCGv_i64 t1 = tcg_temp_ebb_new_i64(); 730 TCGv_i64 t2 = tcg_temp_ebb_new_i64(); 731 tcg_gen_extu_i32_i64(t1, arg1); 732 tcg_gen_extu_i32_i64(t2, arg2); 733 tcg_gen_addi_i64(t2, t2, 32); 734 tcg_gen_clz_i64(t1, t1, t2); 735 tcg_gen_extrl_i64_i32(ret, t1); 736 tcg_temp_free_i64(t1); 737 tcg_temp_free_i64(t2); 738 tcg_gen_subi_i32(ret, ret, 32); 739 } else { 740 gen_helper_clz_i32(ret, arg1, arg2); 741 } 742 } 743 744 void tcg_gen_clzi_i32(TCGv_i32 ret, TCGv_i32 arg1, uint32_t arg2) 745 { 746 tcg_gen_clz_i32(ret, arg1, tcg_constant_i32(arg2)); 747 } 748 749 void tcg_gen_ctz_i32(TCGv_i32 ret, TCGv_i32 arg1, TCGv_i32 arg2) 750 { 751 if (TCG_TARGET_HAS_ctz_i32) { 752 tcg_gen_op3_i32(INDEX_op_ctz_i32, ret, arg1, arg2); 753 } else if (TCG_TARGET_HAS_ctz_i64) { 754 TCGv_i64 t1 = tcg_temp_ebb_new_i64(); 755 TCGv_i64 t2 = tcg_temp_ebb_new_i64(); 756 tcg_gen_extu_i32_i64(t1, arg1); 757 tcg_gen_extu_i32_i64(t2, arg2); 758 tcg_gen_ctz_i64(t1, t1, t2); 759 tcg_gen_extrl_i64_i32(ret, t1); 760 tcg_temp_free_i64(t1); 761 tcg_temp_free_i64(t2); 762 } else if (TCG_TARGET_HAS_ctpop_i32 763 || TCG_TARGET_HAS_ctpop_i64 764 || TCG_TARGET_HAS_clz_i32 765 || TCG_TARGET_HAS_clz_i64) { 766 TCGv_i32 z, t = tcg_temp_ebb_new_i32(); 767 768 if (TCG_TARGET_HAS_ctpop_i32 || TCG_TARGET_HAS_ctpop_i64) { 769 tcg_gen_subi_i32(t, arg1, 1); 770 tcg_gen_andc_i32(t, t, arg1); 771 tcg_gen_ctpop_i32(t, t); 772 } else { 773 /* Since all non-x86 hosts have clz(0) == 32, don't fight it. */ 774 tcg_gen_neg_i32(t, arg1); 775 tcg_gen_and_i32(t, t, arg1); 776 tcg_gen_clzi_i32(t, t, 32); 777 tcg_gen_xori_i32(t, t, 31); 778 } 779 z = tcg_constant_i32(0); 780 tcg_gen_movcond_i32(TCG_COND_EQ, ret, arg1, z, arg2, t); 781 tcg_temp_free_i32(t); 782 } else { 783 gen_helper_ctz_i32(ret, arg1, arg2); 784 } 785 } 786 787 void tcg_gen_ctzi_i32(TCGv_i32 ret, TCGv_i32 arg1, uint32_t arg2) 788 { 789 if (!TCG_TARGET_HAS_ctz_i32 && TCG_TARGET_HAS_ctpop_i32 && arg2 == 32) { 790 /* This equivalence has the advantage of not requiring a fixup. */ 791 TCGv_i32 t = tcg_temp_ebb_new_i32(); 792 tcg_gen_subi_i32(t, arg1, 1); 793 tcg_gen_andc_i32(t, t, arg1); 794 tcg_gen_ctpop_i32(ret, t); 795 tcg_temp_free_i32(t); 796 } else { 797 tcg_gen_ctz_i32(ret, arg1, tcg_constant_i32(arg2)); 798 } 799 } 800 801 void tcg_gen_clrsb_i32(TCGv_i32 ret, TCGv_i32 arg) 802 { 803 if (TCG_TARGET_HAS_clz_i32) { 804 TCGv_i32 t = tcg_temp_ebb_new_i32(); 805 tcg_gen_sari_i32(t, arg, 31); 806 tcg_gen_xor_i32(t, t, arg); 807 tcg_gen_clzi_i32(t, t, 32); 808 tcg_gen_subi_i32(ret, t, 1); 809 tcg_temp_free_i32(t); 810 } else { 811 gen_helper_clrsb_i32(ret, arg); 812 } 813 } 814 815 void tcg_gen_ctpop_i32(TCGv_i32 ret, TCGv_i32 arg1) 816 { 817 if (TCG_TARGET_HAS_ctpop_i32) { 818 tcg_gen_op2_i32(INDEX_op_ctpop_i32, ret, arg1); 819 } else if (TCG_TARGET_HAS_ctpop_i64) { 820 TCGv_i64 t = tcg_temp_ebb_new_i64(); 821 tcg_gen_extu_i32_i64(t, arg1); 822 tcg_gen_ctpop_i64(t, t); 823 tcg_gen_extrl_i64_i32(ret, t); 824 tcg_temp_free_i64(t); 825 } else { 826 gen_helper_ctpop_i32(ret, arg1); 827 } 828 } 829 830 void tcg_gen_rotl_i32(TCGv_i32 ret, TCGv_i32 arg1, TCGv_i32 arg2) 831 { 832 if (TCG_TARGET_HAS_rot_i32) { 833 tcg_gen_op3_i32(INDEX_op_rotl_i32, ret, arg1, arg2); 834 } else { 835 TCGv_i32 t0, t1; 836 837 t0 = tcg_temp_ebb_new_i32(); 838 t1 = tcg_temp_ebb_new_i32(); 839 tcg_gen_shl_i32(t0, arg1, arg2); 840 tcg_gen_subfi_i32(t1, 32, arg2); 841 tcg_gen_shr_i32(t1, arg1, t1); 842 tcg_gen_or_i32(ret, t0, t1); 843 tcg_temp_free_i32(t0); 844 tcg_temp_free_i32(t1); 845 } 846 } 847 848 void tcg_gen_rotli_i32(TCGv_i32 ret, TCGv_i32 arg1, int32_t arg2) 849 { 850 tcg_debug_assert(arg2 >= 0 && arg2 < 32); 851 /* some cases can be optimized here */ 852 if (arg2 == 0) { 853 tcg_gen_mov_i32(ret, arg1); 854 } else if (TCG_TARGET_HAS_rot_i32) { 855 tcg_gen_rotl_i32(ret, arg1, tcg_constant_i32(arg2)); 856 } else { 857 TCGv_i32 t0, t1; 858 t0 = tcg_temp_ebb_new_i32(); 859 t1 = tcg_temp_ebb_new_i32(); 860 tcg_gen_shli_i32(t0, arg1, arg2); 861 tcg_gen_shri_i32(t1, arg1, 32 - arg2); 862 tcg_gen_or_i32(ret, t0, t1); 863 tcg_temp_free_i32(t0); 864 tcg_temp_free_i32(t1); 865 } 866 } 867 868 void tcg_gen_rotr_i32(TCGv_i32 ret, TCGv_i32 arg1, TCGv_i32 arg2) 869 { 870 if (TCG_TARGET_HAS_rot_i32) { 871 tcg_gen_op3_i32(INDEX_op_rotr_i32, ret, arg1, arg2); 872 } else { 873 TCGv_i32 t0, t1; 874 875 t0 = tcg_temp_ebb_new_i32(); 876 t1 = tcg_temp_ebb_new_i32(); 877 tcg_gen_shr_i32(t0, arg1, arg2); 878 tcg_gen_subfi_i32(t1, 32, arg2); 879 tcg_gen_shl_i32(t1, arg1, t1); 880 tcg_gen_or_i32(ret, t0, t1); 881 tcg_temp_free_i32(t0); 882 tcg_temp_free_i32(t1); 883 } 884 } 885 886 void tcg_gen_rotri_i32(TCGv_i32 ret, TCGv_i32 arg1, int32_t arg2) 887 { 888 tcg_debug_assert(arg2 >= 0 && arg2 < 32); 889 /* some cases can be optimized here */ 890 if (arg2 == 0) { 891 tcg_gen_mov_i32(ret, arg1); 892 } else { 893 tcg_gen_rotli_i32(ret, arg1, 32 - arg2); 894 } 895 } 896 897 void tcg_gen_deposit_i32(TCGv_i32 ret, TCGv_i32 arg1, TCGv_i32 arg2, 898 unsigned int ofs, unsigned int len) 899 { 900 uint32_t mask; 901 TCGv_i32 t1; 902 903 tcg_debug_assert(ofs < 32); 904 tcg_debug_assert(len > 0); 905 tcg_debug_assert(len <= 32); 906 tcg_debug_assert(ofs + len <= 32); 907 908 if (len == 32) { 909 tcg_gen_mov_i32(ret, arg2); 910 return; 911 } 912 if (TCG_TARGET_deposit_valid(TCG_TYPE_I32, ofs, len)) { 913 tcg_gen_op5ii_i32(INDEX_op_deposit_i32, ret, arg1, arg2, ofs, len); 914 return; 915 } 916 917 t1 = tcg_temp_ebb_new_i32(); 918 919 if (TCG_TARGET_HAS_extract2_i32) { 920 if (ofs + len == 32) { 921 tcg_gen_shli_i32(t1, arg1, len); 922 tcg_gen_extract2_i32(ret, t1, arg2, len); 923 goto done; 924 } 925 if (ofs == 0) { 926 tcg_gen_extract2_i32(ret, arg1, arg2, len); 927 tcg_gen_rotli_i32(ret, ret, len); 928 goto done; 929 } 930 } 931 932 mask = (1u << len) - 1; 933 if (ofs + len < 32) { 934 tcg_gen_andi_i32(t1, arg2, mask); 935 tcg_gen_shli_i32(t1, t1, ofs); 936 } else { 937 tcg_gen_shli_i32(t1, arg2, ofs); 938 } 939 tcg_gen_andi_i32(ret, arg1, ~(mask << ofs)); 940 tcg_gen_or_i32(ret, ret, t1); 941 done: 942 tcg_temp_free_i32(t1); 943 } 944 945 void tcg_gen_deposit_z_i32(TCGv_i32 ret, TCGv_i32 arg, 946 unsigned int ofs, unsigned int len) 947 { 948 tcg_debug_assert(ofs < 32); 949 tcg_debug_assert(len > 0); 950 tcg_debug_assert(len <= 32); 951 tcg_debug_assert(ofs + len <= 32); 952 953 if (ofs + len == 32) { 954 tcg_gen_shli_i32(ret, arg, ofs); 955 } else if (ofs == 0) { 956 tcg_gen_andi_i32(ret, arg, (1u << len) - 1); 957 } else if (TCG_TARGET_deposit_valid(TCG_TYPE_I32, ofs, len)) { 958 TCGv_i32 zero = tcg_constant_i32(0); 959 tcg_gen_op5ii_i32(INDEX_op_deposit_i32, ret, zero, arg, ofs, len); 960 } else { 961 /* 962 * To help two-operand hosts we prefer to zero-extend first, 963 * which allows ARG to stay live. 964 */ 965 if (TCG_TARGET_extract_valid(TCG_TYPE_I32, 0, len)) { 966 tcg_gen_extract_i32(ret, arg, 0, len); 967 tcg_gen_shli_i32(ret, ret, ofs); 968 return; 969 } 970 /* Otherwise prefer zero-extension over AND for code size. */ 971 if (TCG_TARGET_extract_valid(TCG_TYPE_I32, 0, ofs + len)) { 972 tcg_gen_shli_i32(ret, arg, ofs); 973 tcg_gen_extract_i32(ret, ret, 0, ofs + len); 974 return; 975 } 976 tcg_gen_andi_i32(ret, arg, (1u << len) - 1); 977 tcg_gen_shli_i32(ret, ret, ofs); 978 } 979 } 980 981 void tcg_gen_extract_i32(TCGv_i32 ret, TCGv_i32 arg, 982 unsigned int ofs, unsigned int len) 983 { 984 tcg_debug_assert(ofs < 32); 985 tcg_debug_assert(len > 0); 986 tcg_debug_assert(len <= 32); 987 tcg_debug_assert(ofs + len <= 32); 988 989 /* Canonicalize certain special cases, even if extract is supported. */ 990 if (ofs + len == 32) { 991 tcg_gen_shri_i32(ret, arg, 32 - len); 992 return; 993 } 994 995 if (TCG_TARGET_extract_valid(TCG_TYPE_I32, ofs, len)) { 996 tcg_gen_op4ii_i32(INDEX_op_extract_i32, ret, arg, ofs, len); 997 return; 998 } 999 if (ofs == 0) { 1000 tcg_gen_andi_i32(ret, arg, (1u << len) - 1); 1001 return; 1002 } 1003 1004 /* Assume that zero-extension, if available, is cheaper than a shift. */ 1005 if (TCG_TARGET_extract_valid(TCG_TYPE_I32, 0, ofs + len)) { 1006 tcg_gen_op4ii_i32(INDEX_op_extract_i32, ret, arg, 0, ofs + len); 1007 tcg_gen_shri_i32(ret, ret, ofs); 1008 return; 1009 } 1010 1011 /* ??? Ideally we'd know what values are available for immediate AND. 1012 Assume that 8 bits are available, plus the special case of 16, 1013 so that we get ext8u, ext16u. */ 1014 switch (len) { 1015 case 1 ... 8: case 16: 1016 tcg_gen_shri_i32(ret, arg, ofs); 1017 tcg_gen_andi_i32(ret, ret, (1u << len) - 1); 1018 break; 1019 default: 1020 tcg_gen_shli_i32(ret, arg, 32 - len - ofs); 1021 tcg_gen_shri_i32(ret, ret, 32 - len); 1022 break; 1023 } 1024 } 1025 1026 void tcg_gen_sextract_i32(TCGv_i32 ret, TCGv_i32 arg, 1027 unsigned int ofs, unsigned int len) 1028 { 1029 tcg_debug_assert(ofs < 32); 1030 tcg_debug_assert(len > 0); 1031 tcg_debug_assert(len <= 32); 1032 tcg_debug_assert(ofs + len <= 32); 1033 1034 /* Canonicalize certain special cases, even if extract is supported. */ 1035 if (ofs + len == 32) { 1036 tcg_gen_sari_i32(ret, arg, 32 - len); 1037 return; 1038 } 1039 1040 if (TCG_TARGET_sextract_valid(TCG_TYPE_I32, ofs, len)) { 1041 tcg_gen_op4ii_i32(INDEX_op_sextract_i32, ret, arg, ofs, len); 1042 return; 1043 } 1044 1045 /* Assume that sign-extension, if available, is cheaper than a shift. */ 1046 if (TCG_TARGET_sextract_valid(TCG_TYPE_I32, 0, ofs + len)) { 1047 tcg_gen_op4ii_i32(INDEX_op_sextract_i32, ret, arg, 0, ofs + len); 1048 tcg_gen_sari_i32(ret, ret, ofs); 1049 return; 1050 } 1051 if (TCG_TARGET_sextract_valid(TCG_TYPE_I32, 0, len)) { 1052 tcg_gen_shri_i32(ret, arg, ofs); 1053 tcg_gen_op4ii_i32(INDEX_op_sextract_i32, ret, ret, 0, len); 1054 return; 1055 } 1056 1057 tcg_gen_shli_i32(ret, arg, 32 - len - ofs); 1058 tcg_gen_sari_i32(ret, ret, 32 - len); 1059 } 1060 1061 /* 1062 * Extract 32-bits from a 64-bit input, ah:al, starting from ofs. 1063 * Unlike tcg_gen_extract_i32 above, len is fixed at 32. 1064 */ 1065 void tcg_gen_extract2_i32(TCGv_i32 ret, TCGv_i32 al, TCGv_i32 ah, 1066 unsigned int ofs) 1067 { 1068 tcg_debug_assert(ofs <= 32); 1069 if (ofs == 0) { 1070 tcg_gen_mov_i32(ret, al); 1071 } else if (ofs == 32) { 1072 tcg_gen_mov_i32(ret, ah); 1073 } else if (al == ah) { 1074 tcg_gen_rotri_i32(ret, al, ofs); 1075 } else if (TCG_TARGET_HAS_extract2_i32) { 1076 tcg_gen_op4i_i32(INDEX_op_extract2_i32, ret, al, ah, ofs); 1077 } else { 1078 TCGv_i32 t0 = tcg_temp_ebb_new_i32(); 1079 tcg_gen_shri_i32(t0, al, ofs); 1080 tcg_gen_deposit_i32(ret, t0, ah, 32 - ofs, ofs); 1081 tcg_temp_free_i32(t0); 1082 } 1083 } 1084 1085 void tcg_gen_movcond_i32(TCGCond cond, TCGv_i32 ret, TCGv_i32 c1, 1086 TCGv_i32 c2, TCGv_i32 v1, TCGv_i32 v2) 1087 { 1088 if (cond == TCG_COND_ALWAYS) { 1089 tcg_gen_mov_i32(ret, v1); 1090 } else if (cond == TCG_COND_NEVER) { 1091 tcg_gen_mov_i32(ret, v2); 1092 } else { 1093 tcg_gen_op6i_i32(INDEX_op_movcond_i32, ret, c1, c2, v1, v2, cond); 1094 } 1095 } 1096 1097 void tcg_gen_add2_i32(TCGv_i32 rl, TCGv_i32 rh, TCGv_i32 al, 1098 TCGv_i32 ah, TCGv_i32 bl, TCGv_i32 bh) 1099 { 1100 if (TCG_TARGET_HAS_add2_i32) { 1101 tcg_gen_op6_i32(INDEX_op_add2_i32, rl, rh, al, ah, bl, bh); 1102 } else { 1103 TCGv_i64 t0 = tcg_temp_ebb_new_i64(); 1104 TCGv_i64 t1 = tcg_temp_ebb_new_i64(); 1105 tcg_gen_concat_i32_i64(t0, al, ah); 1106 tcg_gen_concat_i32_i64(t1, bl, bh); 1107 tcg_gen_add_i64(t0, t0, t1); 1108 tcg_gen_extr_i64_i32(rl, rh, t0); 1109 tcg_temp_free_i64(t0); 1110 tcg_temp_free_i64(t1); 1111 } 1112 } 1113 1114 void tcg_gen_sub2_i32(TCGv_i32 rl, TCGv_i32 rh, TCGv_i32 al, 1115 TCGv_i32 ah, TCGv_i32 bl, TCGv_i32 bh) 1116 { 1117 if (TCG_TARGET_HAS_sub2_i32) { 1118 tcg_gen_op6_i32(INDEX_op_sub2_i32, rl, rh, al, ah, bl, bh); 1119 } else { 1120 TCGv_i64 t0 = tcg_temp_ebb_new_i64(); 1121 TCGv_i64 t1 = tcg_temp_ebb_new_i64(); 1122 tcg_gen_concat_i32_i64(t0, al, ah); 1123 tcg_gen_concat_i32_i64(t1, bl, bh); 1124 tcg_gen_sub_i64(t0, t0, t1); 1125 tcg_gen_extr_i64_i32(rl, rh, t0); 1126 tcg_temp_free_i64(t0); 1127 tcg_temp_free_i64(t1); 1128 } 1129 } 1130 1131 void tcg_gen_mulu2_i32(TCGv_i32 rl, TCGv_i32 rh, TCGv_i32 arg1, TCGv_i32 arg2) 1132 { 1133 if (TCG_TARGET_HAS_mulu2_i32) { 1134 tcg_gen_op4_i32(INDEX_op_mulu2_i32, rl, rh, arg1, arg2); 1135 } else if (tcg_op_supported(INDEX_op_muluh, TCG_TYPE_I32, 0)) { 1136 TCGv_i32 t = tcg_temp_ebb_new_i32(); 1137 tcg_gen_op3_i32(INDEX_op_mul, t, arg1, arg2); 1138 tcg_gen_op3_i32(INDEX_op_muluh, rh, arg1, arg2); 1139 tcg_gen_mov_i32(rl, t); 1140 tcg_temp_free_i32(t); 1141 } else if (TCG_TARGET_REG_BITS == 64) { 1142 TCGv_i64 t0 = tcg_temp_ebb_new_i64(); 1143 TCGv_i64 t1 = tcg_temp_ebb_new_i64(); 1144 tcg_gen_extu_i32_i64(t0, arg1); 1145 tcg_gen_extu_i32_i64(t1, arg2); 1146 tcg_gen_mul_i64(t0, t0, t1); 1147 tcg_gen_extr_i64_i32(rl, rh, t0); 1148 tcg_temp_free_i64(t0); 1149 tcg_temp_free_i64(t1); 1150 } else { 1151 qemu_build_not_reached(); 1152 } 1153 } 1154 1155 void tcg_gen_muls2_i32(TCGv_i32 rl, TCGv_i32 rh, TCGv_i32 arg1, TCGv_i32 arg2) 1156 { 1157 if (TCG_TARGET_HAS_muls2_i32) { 1158 tcg_gen_op4_i32(INDEX_op_muls2_i32, rl, rh, arg1, arg2); 1159 } else if (TCG_TARGET_HAS_mulsh_i32) { 1160 TCGv_i32 t = tcg_temp_ebb_new_i32(); 1161 tcg_gen_op3_i32(INDEX_op_mul, t, arg1, arg2); 1162 tcg_gen_op3_i32(INDEX_op_mulsh_i32, rh, arg1, arg2); 1163 tcg_gen_mov_i32(rl, t); 1164 tcg_temp_free_i32(t); 1165 } else if (TCG_TARGET_REG_BITS == 32) { 1166 TCGv_i32 t0 = tcg_temp_ebb_new_i32(); 1167 TCGv_i32 t1 = tcg_temp_ebb_new_i32(); 1168 TCGv_i32 t2 = tcg_temp_ebb_new_i32(); 1169 TCGv_i32 t3 = tcg_temp_ebb_new_i32(); 1170 tcg_gen_mulu2_i32(t0, t1, arg1, arg2); 1171 /* Adjust for negative inputs. */ 1172 tcg_gen_sari_i32(t2, arg1, 31); 1173 tcg_gen_sari_i32(t3, arg2, 31); 1174 tcg_gen_and_i32(t2, t2, arg2); 1175 tcg_gen_and_i32(t3, t3, arg1); 1176 tcg_gen_sub_i32(rh, t1, t2); 1177 tcg_gen_sub_i32(rh, rh, t3); 1178 tcg_gen_mov_i32(rl, t0); 1179 tcg_temp_free_i32(t0); 1180 tcg_temp_free_i32(t1); 1181 tcg_temp_free_i32(t2); 1182 tcg_temp_free_i32(t3); 1183 } else { 1184 TCGv_i64 t0 = tcg_temp_ebb_new_i64(); 1185 TCGv_i64 t1 = tcg_temp_ebb_new_i64(); 1186 tcg_gen_ext_i32_i64(t0, arg1); 1187 tcg_gen_ext_i32_i64(t1, arg2); 1188 tcg_gen_mul_i64(t0, t0, t1); 1189 tcg_gen_extr_i64_i32(rl, rh, t0); 1190 tcg_temp_free_i64(t0); 1191 tcg_temp_free_i64(t1); 1192 } 1193 } 1194 1195 void tcg_gen_mulsu2_i32(TCGv_i32 rl, TCGv_i32 rh, TCGv_i32 arg1, TCGv_i32 arg2) 1196 { 1197 if (TCG_TARGET_REG_BITS == 32) { 1198 TCGv_i32 t0 = tcg_temp_ebb_new_i32(); 1199 TCGv_i32 t1 = tcg_temp_ebb_new_i32(); 1200 TCGv_i32 t2 = tcg_temp_ebb_new_i32(); 1201 tcg_gen_mulu2_i32(t0, t1, arg1, arg2); 1202 /* Adjust for negative input for the signed arg1. */ 1203 tcg_gen_sari_i32(t2, arg1, 31); 1204 tcg_gen_and_i32(t2, t2, arg2); 1205 tcg_gen_sub_i32(rh, t1, t2); 1206 tcg_gen_mov_i32(rl, t0); 1207 tcg_temp_free_i32(t0); 1208 tcg_temp_free_i32(t1); 1209 tcg_temp_free_i32(t2); 1210 } else { 1211 TCGv_i64 t0 = tcg_temp_ebb_new_i64(); 1212 TCGv_i64 t1 = tcg_temp_ebb_new_i64(); 1213 tcg_gen_ext_i32_i64(t0, arg1); 1214 tcg_gen_extu_i32_i64(t1, arg2); 1215 tcg_gen_mul_i64(t0, t0, t1); 1216 tcg_gen_extr_i64_i32(rl, rh, t0); 1217 tcg_temp_free_i64(t0); 1218 tcg_temp_free_i64(t1); 1219 } 1220 } 1221 1222 void tcg_gen_ext8s_i32(TCGv_i32 ret, TCGv_i32 arg) 1223 { 1224 tcg_gen_sextract_i32(ret, arg, 0, 8); 1225 } 1226 1227 void tcg_gen_ext16s_i32(TCGv_i32 ret, TCGv_i32 arg) 1228 { 1229 tcg_gen_sextract_i32(ret, arg, 0, 16); 1230 } 1231 1232 void tcg_gen_ext8u_i32(TCGv_i32 ret, TCGv_i32 arg) 1233 { 1234 tcg_gen_extract_i32(ret, arg, 0, 8); 1235 } 1236 1237 void tcg_gen_ext16u_i32(TCGv_i32 ret, TCGv_i32 arg) 1238 { 1239 tcg_gen_extract_i32(ret, arg, 0, 16); 1240 } 1241 1242 /* 1243 * bswap16_i32: 16-bit byte swap on the low bits of a 32-bit value. 1244 * 1245 * Byte pattern: xxab -> yyba 1246 * 1247 * With TCG_BSWAP_IZ, x == zero, else undefined. 1248 * With TCG_BSWAP_OZ, y == zero, with TCG_BSWAP_OS y == sign, else undefined. 1249 */ 1250 void tcg_gen_bswap16_i32(TCGv_i32 ret, TCGv_i32 arg, int flags) 1251 { 1252 /* Only one extension flag may be present. */ 1253 tcg_debug_assert(!(flags & TCG_BSWAP_OS) || !(flags & TCG_BSWAP_OZ)); 1254 1255 if (TCG_TARGET_HAS_bswap16_i32) { 1256 tcg_gen_op3i_i32(INDEX_op_bswap16_i32, ret, arg, flags); 1257 } else { 1258 TCGv_i32 t0 = tcg_temp_ebb_new_i32(); 1259 TCGv_i32 t1 = tcg_temp_ebb_new_i32(); 1260 1261 /* arg = ..ab (IZ) xxab (!IZ) */ 1262 tcg_gen_shri_i32(t0, arg, 8); /* t0 = ...a (IZ) .xxa (!IZ) */ 1263 if (!(flags & TCG_BSWAP_IZ)) { 1264 tcg_gen_ext8u_i32(t0, t0); /* t0 = ...a */ 1265 } 1266 1267 if (flags & TCG_BSWAP_OS) { 1268 tcg_gen_shli_i32(t1, arg, 24); /* t1 = b... */ 1269 tcg_gen_sari_i32(t1, t1, 16); /* t1 = ssb. */ 1270 } else if (flags & TCG_BSWAP_OZ) { 1271 tcg_gen_ext8u_i32(t1, arg); /* t1 = ...b */ 1272 tcg_gen_shli_i32(t1, t1, 8); /* t1 = ..b. */ 1273 } else { 1274 tcg_gen_shli_i32(t1, arg, 8); /* t1 = xab. */ 1275 } 1276 1277 tcg_gen_or_i32(ret, t0, t1); /* ret = ..ba (OZ) */ 1278 /* = ssba (OS) */ 1279 /* = xaba (no flag) */ 1280 tcg_temp_free_i32(t0); 1281 tcg_temp_free_i32(t1); 1282 } 1283 } 1284 1285 /* 1286 * bswap32_i32: 32-bit byte swap on a 32-bit value. 1287 * 1288 * Byte pattern: abcd -> dcba 1289 */ 1290 void tcg_gen_bswap32_i32(TCGv_i32 ret, TCGv_i32 arg) 1291 { 1292 if (TCG_TARGET_HAS_bswap32_i32) { 1293 tcg_gen_op3i_i32(INDEX_op_bswap32_i32, ret, arg, 0); 1294 } else { 1295 TCGv_i32 t0 = tcg_temp_ebb_new_i32(); 1296 TCGv_i32 t1 = tcg_temp_ebb_new_i32(); 1297 TCGv_i32 t2 = tcg_constant_i32(0x00ff00ff); 1298 1299 /* arg = abcd */ 1300 tcg_gen_shri_i32(t0, arg, 8); /* t0 = .abc */ 1301 tcg_gen_and_i32(t1, arg, t2); /* t1 = .b.d */ 1302 tcg_gen_and_i32(t0, t0, t2); /* t0 = .a.c */ 1303 tcg_gen_shli_i32(t1, t1, 8); /* t1 = b.d. */ 1304 tcg_gen_or_i32(ret, t0, t1); /* ret = badc */ 1305 1306 tcg_gen_shri_i32(t0, ret, 16); /* t0 = ..ba */ 1307 tcg_gen_shli_i32(t1, ret, 16); /* t1 = dc.. */ 1308 tcg_gen_or_i32(ret, t0, t1); /* ret = dcba */ 1309 1310 tcg_temp_free_i32(t0); 1311 tcg_temp_free_i32(t1); 1312 } 1313 } 1314 1315 /* 1316 * hswap_i32: Swap 16-bit halfwords within a 32-bit value. 1317 * 1318 * Byte pattern: abcd -> cdab 1319 */ 1320 void tcg_gen_hswap_i32(TCGv_i32 ret, TCGv_i32 arg) 1321 { 1322 /* Swapping 2 16-bit elements is a rotate. */ 1323 tcg_gen_rotli_i32(ret, arg, 16); 1324 } 1325 1326 void tcg_gen_smin_i32(TCGv_i32 ret, TCGv_i32 a, TCGv_i32 b) 1327 { 1328 tcg_gen_movcond_i32(TCG_COND_LT, ret, a, b, a, b); 1329 } 1330 1331 void tcg_gen_umin_i32(TCGv_i32 ret, TCGv_i32 a, TCGv_i32 b) 1332 { 1333 tcg_gen_movcond_i32(TCG_COND_LTU, ret, a, b, a, b); 1334 } 1335 1336 void tcg_gen_smax_i32(TCGv_i32 ret, TCGv_i32 a, TCGv_i32 b) 1337 { 1338 tcg_gen_movcond_i32(TCG_COND_LT, ret, a, b, b, a); 1339 } 1340 1341 void tcg_gen_umax_i32(TCGv_i32 ret, TCGv_i32 a, TCGv_i32 b) 1342 { 1343 tcg_gen_movcond_i32(TCG_COND_LTU, ret, a, b, b, a); 1344 } 1345 1346 void tcg_gen_abs_i32(TCGv_i32 ret, TCGv_i32 a) 1347 { 1348 TCGv_i32 t = tcg_temp_ebb_new_i32(); 1349 1350 tcg_gen_sari_i32(t, a, 31); 1351 tcg_gen_xor_i32(ret, a, t); 1352 tcg_gen_sub_i32(ret, ret, t); 1353 tcg_temp_free_i32(t); 1354 } 1355 1356 void tcg_gen_ld8u_i32(TCGv_i32 ret, TCGv_ptr arg2, tcg_target_long offset) 1357 { 1358 tcg_gen_ldst_op_i32(INDEX_op_ld8u_i32, ret, arg2, offset); 1359 } 1360 1361 void tcg_gen_ld8s_i32(TCGv_i32 ret, TCGv_ptr arg2, tcg_target_long offset) 1362 { 1363 tcg_gen_ldst_op_i32(INDEX_op_ld8s_i32, ret, arg2, offset); 1364 } 1365 1366 void tcg_gen_ld16u_i32(TCGv_i32 ret, TCGv_ptr arg2, tcg_target_long offset) 1367 { 1368 tcg_gen_ldst_op_i32(INDEX_op_ld16u_i32, ret, arg2, offset); 1369 } 1370 1371 void tcg_gen_ld16s_i32(TCGv_i32 ret, TCGv_ptr arg2, tcg_target_long offset) 1372 { 1373 tcg_gen_ldst_op_i32(INDEX_op_ld16s_i32, ret, arg2, offset); 1374 } 1375 1376 void tcg_gen_ld_i32(TCGv_i32 ret, TCGv_ptr arg2, tcg_target_long offset) 1377 { 1378 tcg_gen_ldst_op_i32(INDEX_op_ld_i32, ret, arg2, offset); 1379 } 1380 1381 void tcg_gen_st8_i32(TCGv_i32 arg1, TCGv_ptr arg2, tcg_target_long offset) 1382 { 1383 tcg_gen_ldst_op_i32(INDEX_op_st8_i32, arg1, arg2, offset); 1384 } 1385 1386 void tcg_gen_st16_i32(TCGv_i32 arg1, TCGv_ptr arg2, tcg_target_long offset) 1387 { 1388 tcg_gen_ldst_op_i32(INDEX_op_st16_i32, arg1, arg2, offset); 1389 } 1390 1391 void tcg_gen_st_i32(TCGv_i32 arg1, TCGv_ptr arg2, tcg_target_long offset) 1392 { 1393 tcg_gen_ldst_op_i32(INDEX_op_st_i32, arg1, arg2, offset); 1394 } 1395 1396 1397 /* 64-bit ops */ 1398 1399 void tcg_gen_discard_i64(TCGv_i64 arg) 1400 { 1401 if (TCG_TARGET_REG_BITS == 64) { 1402 tcg_gen_op1_i64(INDEX_op_discard, TCG_TYPE_I64, arg); 1403 } else { 1404 tcg_gen_discard_i32(TCGV_LOW(arg)); 1405 tcg_gen_discard_i32(TCGV_HIGH(arg)); 1406 } 1407 } 1408 1409 void tcg_gen_mov_i64(TCGv_i64 ret, TCGv_i64 arg) 1410 { 1411 if (ret == arg) { 1412 return; 1413 } 1414 if (TCG_TARGET_REG_BITS == 64) { 1415 tcg_gen_op2_i64(INDEX_op_mov, ret, arg); 1416 } else { 1417 TCGTemp *ts = tcgv_i64_temp(arg); 1418 1419 /* Canonicalize TCGv_i64 TEMP_CONST into TCGv_i32 TEMP_CONST. */ 1420 if (ts->kind == TEMP_CONST) { 1421 tcg_gen_movi_i64(ret, ts->val); 1422 } else { 1423 tcg_gen_mov_i32(TCGV_LOW(ret), TCGV_LOW(arg)); 1424 tcg_gen_mov_i32(TCGV_HIGH(ret), TCGV_HIGH(arg)); 1425 } 1426 } 1427 } 1428 1429 void tcg_gen_movi_i64(TCGv_i64 ret, int64_t arg) 1430 { 1431 if (TCG_TARGET_REG_BITS == 64) { 1432 tcg_gen_mov_i64(ret, tcg_constant_i64(arg)); 1433 } else { 1434 tcg_gen_movi_i32(TCGV_LOW(ret), arg); 1435 tcg_gen_movi_i32(TCGV_HIGH(ret), arg >> 32); 1436 } 1437 } 1438 1439 void tcg_gen_ld8u_i64(TCGv_i64 ret, TCGv_ptr arg2, tcg_target_long offset) 1440 { 1441 if (TCG_TARGET_REG_BITS == 64) { 1442 tcg_gen_ldst_op_i64(INDEX_op_ld8u_i64, ret, arg2, offset); 1443 } else { 1444 tcg_gen_ld8u_i32(TCGV_LOW(ret), arg2, offset); 1445 tcg_gen_movi_i32(TCGV_HIGH(ret), 0); 1446 } 1447 } 1448 1449 void tcg_gen_ld8s_i64(TCGv_i64 ret, TCGv_ptr arg2, tcg_target_long offset) 1450 { 1451 if (TCG_TARGET_REG_BITS == 64) { 1452 tcg_gen_ldst_op_i64(INDEX_op_ld8s_i64, ret, arg2, offset); 1453 } else { 1454 tcg_gen_ld8s_i32(TCGV_LOW(ret), arg2, offset); 1455 tcg_gen_sari_i32(TCGV_HIGH(ret), TCGV_LOW(ret), 31); 1456 } 1457 } 1458 1459 void tcg_gen_ld16u_i64(TCGv_i64 ret, TCGv_ptr arg2, tcg_target_long offset) 1460 { 1461 if (TCG_TARGET_REG_BITS == 64) { 1462 tcg_gen_ldst_op_i64(INDEX_op_ld16u_i64, ret, arg2, offset); 1463 } else { 1464 tcg_gen_ld16u_i32(TCGV_LOW(ret), arg2, offset); 1465 tcg_gen_movi_i32(TCGV_HIGH(ret), 0); 1466 } 1467 } 1468 1469 void tcg_gen_ld16s_i64(TCGv_i64 ret, TCGv_ptr arg2, tcg_target_long offset) 1470 { 1471 if (TCG_TARGET_REG_BITS == 64) { 1472 tcg_gen_ldst_op_i64(INDEX_op_ld16s_i64, ret, arg2, offset); 1473 } else { 1474 tcg_gen_ld16s_i32(TCGV_LOW(ret), arg2, offset); 1475 tcg_gen_sari_i32(TCGV_HIGH(ret), TCGV_LOW(ret), 31); 1476 } 1477 } 1478 1479 void tcg_gen_ld32u_i64(TCGv_i64 ret, TCGv_ptr arg2, tcg_target_long offset) 1480 { 1481 if (TCG_TARGET_REG_BITS == 64) { 1482 tcg_gen_ldst_op_i64(INDEX_op_ld32u_i64, ret, arg2, offset); 1483 } else { 1484 tcg_gen_ld_i32(TCGV_LOW(ret), arg2, offset); 1485 tcg_gen_movi_i32(TCGV_HIGH(ret), 0); 1486 } 1487 } 1488 1489 void tcg_gen_ld32s_i64(TCGv_i64 ret, TCGv_ptr arg2, tcg_target_long offset) 1490 { 1491 if (TCG_TARGET_REG_BITS == 64) { 1492 tcg_gen_ldst_op_i64(INDEX_op_ld32s_i64, ret, arg2, offset); 1493 } else { 1494 tcg_gen_ld_i32(TCGV_LOW(ret), arg2, offset); 1495 tcg_gen_sari_i32(TCGV_HIGH(ret), TCGV_LOW(ret), 31); 1496 } 1497 } 1498 1499 void tcg_gen_ld_i64(TCGv_i64 ret, TCGv_ptr arg2, tcg_target_long offset) 1500 { 1501 /* 1502 * For 32-bit host, since arg2 and ret have different types, 1503 * they cannot be the same temporary -- no chance of overlap. 1504 */ 1505 if (TCG_TARGET_REG_BITS == 64) { 1506 tcg_gen_ldst_op_i64(INDEX_op_ld_i64, ret, arg2, offset); 1507 } else if (HOST_BIG_ENDIAN) { 1508 tcg_gen_ld_i32(TCGV_HIGH(ret), arg2, offset); 1509 tcg_gen_ld_i32(TCGV_LOW(ret), arg2, offset + 4); 1510 } else { 1511 tcg_gen_ld_i32(TCGV_LOW(ret), arg2, offset); 1512 tcg_gen_ld_i32(TCGV_HIGH(ret), arg2, offset + 4); 1513 } 1514 } 1515 1516 void tcg_gen_st8_i64(TCGv_i64 arg1, TCGv_ptr arg2, tcg_target_long offset) 1517 { 1518 if (TCG_TARGET_REG_BITS == 64) { 1519 tcg_gen_ldst_op_i64(INDEX_op_st8_i64, arg1, arg2, offset); 1520 } else { 1521 tcg_gen_st8_i32(TCGV_LOW(arg1), arg2, offset); 1522 } 1523 } 1524 1525 void tcg_gen_st16_i64(TCGv_i64 arg1, TCGv_ptr arg2, tcg_target_long offset) 1526 { 1527 if (TCG_TARGET_REG_BITS == 64) { 1528 tcg_gen_ldst_op_i64(INDEX_op_st16_i64, arg1, arg2, offset); 1529 } else { 1530 tcg_gen_st16_i32(TCGV_LOW(arg1), arg2, offset); 1531 } 1532 } 1533 1534 void tcg_gen_st32_i64(TCGv_i64 arg1, TCGv_ptr arg2, tcg_target_long offset) 1535 { 1536 if (TCG_TARGET_REG_BITS == 64) { 1537 tcg_gen_ldst_op_i64(INDEX_op_st32_i64, arg1, arg2, offset); 1538 } else { 1539 tcg_gen_st_i32(TCGV_LOW(arg1), arg2, offset); 1540 } 1541 } 1542 1543 void tcg_gen_st_i64(TCGv_i64 arg1, TCGv_ptr arg2, tcg_target_long offset) 1544 { 1545 if (TCG_TARGET_REG_BITS == 64) { 1546 tcg_gen_ldst_op_i64(INDEX_op_st_i64, arg1, arg2, offset); 1547 } else if (HOST_BIG_ENDIAN) { 1548 tcg_gen_st_i32(TCGV_HIGH(arg1), arg2, offset); 1549 tcg_gen_st_i32(TCGV_LOW(arg1), arg2, offset + 4); 1550 } else { 1551 tcg_gen_st_i32(TCGV_LOW(arg1), arg2, offset); 1552 tcg_gen_st_i32(TCGV_HIGH(arg1), arg2, offset + 4); 1553 } 1554 } 1555 1556 void tcg_gen_add_i64(TCGv_i64 ret, TCGv_i64 arg1, TCGv_i64 arg2) 1557 { 1558 if (TCG_TARGET_REG_BITS == 64) { 1559 tcg_gen_op3_i64(INDEX_op_add, ret, arg1, arg2); 1560 } else { 1561 tcg_gen_add2_i32(TCGV_LOW(ret), TCGV_HIGH(ret), TCGV_LOW(arg1), 1562 TCGV_HIGH(arg1), TCGV_LOW(arg2), TCGV_HIGH(arg2)); 1563 } 1564 } 1565 1566 void tcg_gen_sub_i64(TCGv_i64 ret, TCGv_i64 arg1, TCGv_i64 arg2) 1567 { 1568 if (TCG_TARGET_REG_BITS == 64) { 1569 tcg_gen_op3_i64(INDEX_op_sub, ret, arg1, arg2); 1570 } else { 1571 tcg_gen_sub2_i32(TCGV_LOW(ret), TCGV_HIGH(ret), TCGV_LOW(arg1), 1572 TCGV_HIGH(arg1), TCGV_LOW(arg2), TCGV_HIGH(arg2)); 1573 } 1574 } 1575 1576 void tcg_gen_and_i64(TCGv_i64 ret, TCGv_i64 arg1, TCGv_i64 arg2) 1577 { 1578 if (TCG_TARGET_REG_BITS == 64) { 1579 tcg_gen_op3_i64(INDEX_op_and, ret, arg1, arg2); 1580 } else { 1581 tcg_gen_and_i32(TCGV_LOW(ret), TCGV_LOW(arg1), TCGV_LOW(arg2)); 1582 tcg_gen_and_i32(TCGV_HIGH(ret), TCGV_HIGH(arg1), TCGV_HIGH(arg2)); 1583 } 1584 } 1585 1586 void tcg_gen_or_i64(TCGv_i64 ret, TCGv_i64 arg1, TCGv_i64 arg2) 1587 { 1588 if (TCG_TARGET_REG_BITS == 64) { 1589 tcg_gen_op3_i64(INDEX_op_or, ret, arg1, arg2); 1590 } else { 1591 tcg_gen_or_i32(TCGV_LOW(ret), TCGV_LOW(arg1), TCGV_LOW(arg2)); 1592 tcg_gen_or_i32(TCGV_HIGH(ret), TCGV_HIGH(arg1), TCGV_HIGH(arg2)); 1593 } 1594 } 1595 1596 void tcg_gen_xor_i64(TCGv_i64 ret, TCGv_i64 arg1, TCGv_i64 arg2) 1597 { 1598 if (TCG_TARGET_REG_BITS == 64) { 1599 tcg_gen_op3_i64(INDEX_op_xor, ret, arg1, arg2); 1600 } else { 1601 tcg_gen_xor_i32(TCGV_LOW(ret), TCGV_LOW(arg1), TCGV_LOW(arg2)); 1602 tcg_gen_xor_i32(TCGV_HIGH(ret), TCGV_HIGH(arg1), TCGV_HIGH(arg2)); 1603 } 1604 } 1605 1606 void tcg_gen_shl_i64(TCGv_i64 ret, TCGv_i64 arg1, TCGv_i64 arg2) 1607 { 1608 if (TCG_TARGET_REG_BITS == 64) { 1609 tcg_gen_op3_i64(INDEX_op_shl_i64, ret, arg1, arg2); 1610 } else { 1611 gen_helper_shl_i64(ret, arg1, arg2); 1612 } 1613 } 1614 1615 void tcg_gen_shr_i64(TCGv_i64 ret, TCGv_i64 arg1, TCGv_i64 arg2) 1616 { 1617 if (TCG_TARGET_REG_BITS == 64) { 1618 tcg_gen_op3_i64(INDEX_op_shr_i64, ret, arg1, arg2); 1619 } else { 1620 gen_helper_shr_i64(ret, arg1, arg2); 1621 } 1622 } 1623 1624 void tcg_gen_sar_i64(TCGv_i64 ret, TCGv_i64 arg1, TCGv_i64 arg2) 1625 { 1626 if (TCG_TARGET_REG_BITS == 64) { 1627 tcg_gen_op3_i64(INDEX_op_sar_i64, ret, arg1, arg2); 1628 } else { 1629 gen_helper_sar_i64(ret, arg1, arg2); 1630 } 1631 } 1632 1633 void tcg_gen_mul_i64(TCGv_i64 ret, TCGv_i64 arg1, TCGv_i64 arg2) 1634 { 1635 TCGv_i64 t0; 1636 TCGv_i32 t1; 1637 1638 if (TCG_TARGET_REG_BITS == 64) { 1639 tcg_gen_op3_i64(INDEX_op_mul, ret, arg1, arg2); 1640 return; 1641 } 1642 1643 1644 t0 = tcg_temp_ebb_new_i64(); 1645 t1 = tcg_temp_ebb_new_i32(); 1646 1647 tcg_gen_mulu2_i32(TCGV_LOW(t0), TCGV_HIGH(t0), 1648 TCGV_LOW(arg1), TCGV_LOW(arg2)); 1649 1650 tcg_gen_mul_i32(t1, TCGV_LOW(arg1), TCGV_HIGH(arg2)); 1651 tcg_gen_add_i32(TCGV_HIGH(t0), TCGV_HIGH(t0), t1); 1652 tcg_gen_mul_i32(t1, TCGV_HIGH(arg1), TCGV_LOW(arg2)); 1653 tcg_gen_add_i32(TCGV_HIGH(t0), TCGV_HIGH(t0), t1); 1654 1655 tcg_gen_mov_i64(ret, t0); 1656 tcg_temp_free_i64(t0); 1657 tcg_temp_free_i32(t1); 1658 } 1659 1660 void tcg_gen_addi_i64(TCGv_i64 ret, TCGv_i64 arg1, int64_t arg2) 1661 { 1662 /* some cases can be optimized here */ 1663 if (arg2 == 0) { 1664 tcg_gen_mov_i64(ret, arg1); 1665 } else if (TCG_TARGET_REG_BITS == 64) { 1666 tcg_gen_add_i64(ret, arg1, tcg_constant_i64(arg2)); 1667 } else { 1668 tcg_gen_add2_i32(TCGV_LOW(ret), TCGV_HIGH(ret), 1669 TCGV_LOW(arg1), TCGV_HIGH(arg1), 1670 tcg_constant_i32(arg2), tcg_constant_i32(arg2 >> 32)); 1671 } 1672 } 1673 1674 void tcg_gen_subfi_i64(TCGv_i64 ret, int64_t arg1, TCGv_i64 arg2) 1675 { 1676 if (arg1 == 0) { 1677 tcg_gen_neg_i64(ret, arg2); 1678 } else if (TCG_TARGET_REG_BITS == 64) { 1679 tcg_gen_sub_i64(ret, tcg_constant_i64(arg1), arg2); 1680 } else { 1681 tcg_gen_sub2_i32(TCGV_LOW(ret), TCGV_HIGH(ret), 1682 tcg_constant_i32(arg1), tcg_constant_i32(arg1 >> 32), 1683 TCGV_LOW(arg2), TCGV_HIGH(arg2)); 1684 } 1685 } 1686 1687 void tcg_gen_subi_i64(TCGv_i64 ret, TCGv_i64 arg1, int64_t arg2) 1688 { 1689 tcg_gen_addi_i64(ret, arg1, -arg2); 1690 } 1691 1692 void tcg_gen_neg_i64(TCGv_i64 ret, TCGv_i64 arg) 1693 { 1694 if (TCG_TARGET_REG_BITS == 64) { 1695 tcg_gen_op2_i64(INDEX_op_neg, ret, arg); 1696 } else { 1697 TCGv_i32 zero = tcg_constant_i32(0); 1698 tcg_gen_sub2_i32(TCGV_LOW(ret), TCGV_HIGH(ret), 1699 zero, zero, TCGV_LOW(arg), TCGV_HIGH(arg)); 1700 } 1701 } 1702 1703 void tcg_gen_andi_i64(TCGv_i64 ret, TCGv_i64 arg1, int64_t arg2) 1704 { 1705 if (TCG_TARGET_REG_BITS == 32) { 1706 tcg_gen_andi_i32(TCGV_LOW(ret), TCGV_LOW(arg1), arg2); 1707 tcg_gen_andi_i32(TCGV_HIGH(ret), TCGV_HIGH(arg1), arg2 >> 32); 1708 return; 1709 } 1710 1711 /* Some cases can be optimized here. */ 1712 switch (arg2) { 1713 case 0: 1714 tcg_gen_movi_i64(ret, 0); 1715 return; 1716 case -1: 1717 tcg_gen_mov_i64(ret, arg1); 1718 return; 1719 default: 1720 /* 1721 * Canonicalize on extract, if valid. This aids x86 with its 1722 * 2 operand MOVZBL and 2 operand AND, selecting the TCGOpcode 1723 * which does not require matching operands. Other backends can 1724 * trivially expand the extract to AND during code generation. 1725 */ 1726 if (!(arg2 & (arg2 + 1))) { 1727 unsigned len = ctz64(~arg2); 1728 if (TCG_TARGET_extract_valid(TCG_TYPE_I64, 0, len)) { 1729 tcg_gen_extract_i64(ret, arg1, 0, len); 1730 return; 1731 } 1732 } 1733 break; 1734 } 1735 1736 tcg_gen_and_i64(ret, arg1, tcg_constant_i64(arg2)); 1737 } 1738 1739 void tcg_gen_ori_i64(TCGv_i64 ret, TCGv_i64 arg1, int64_t arg2) 1740 { 1741 if (TCG_TARGET_REG_BITS == 32) { 1742 tcg_gen_ori_i32(TCGV_LOW(ret), TCGV_LOW(arg1), arg2); 1743 tcg_gen_ori_i32(TCGV_HIGH(ret), TCGV_HIGH(arg1), arg2 >> 32); 1744 return; 1745 } 1746 /* Some cases can be optimized here. */ 1747 if (arg2 == -1) { 1748 tcg_gen_movi_i64(ret, -1); 1749 } else if (arg2 == 0) { 1750 tcg_gen_mov_i64(ret, arg1); 1751 } else { 1752 tcg_gen_or_i64(ret, arg1, tcg_constant_i64(arg2)); 1753 } 1754 } 1755 1756 void tcg_gen_xori_i64(TCGv_i64 ret, TCGv_i64 arg1, int64_t arg2) 1757 { 1758 if (TCG_TARGET_REG_BITS == 32) { 1759 tcg_gen_xori_i32(TCGV_LOW(ret), TCGV_LOW(arg1), arg2); 1760 tcg_gen_xori_i32(TCGV_HIGH(ret), TCGV_HIGH(arg1), arg2 >> 32); 1761 return; 1762 } 1763 /* Some cases can be optimized here. */ 1764 if (arg2 == 0) { 1765 tcg_gen_mov_i64(ret, arg1); 1766 } else if (arg2 == -1 && 1767 tcg_op_supported(INDEX_op_not, TCG_TYPE_I64, 0)) { 1768 /* Don't recurse with tcg_gen_not_i64. */ 1769 tcg_gen_op2_i64(INDEX_op_not, ret, arg1); 1770 } else { 1771 tcg_gen_xor_i64(ret, arg1, tcg_constant_i64(arg2)); 1772 } 1773 } 1774 1775 static inline void tcg_gen_shifti_i64(TCGv_i64 ret, TCGv_i64 arg1, 1776 unsigned c, bool right, bool arith) 1777 { 1778 tcg_debug_assert(c < 64); 1779 if (c == 0) { 1780 tcg_gen_mov_i32(TCGV_LOW(ret), TCGV_LOW(arg1)); 1781 tcg_gen_mov_i32(TCGV_HIGH(ret), TCGV_HIGH(arg1)); 1782 } else if (c >= 32) { 1783 c -= 32; 1784 if (right) { 1785 if (arith) { 1786 tcg_gen_sari_i32(TCGV_LOW(ret), TCGV_HIGH(arg1), c); 1787 tcg_gen_sari_i32(TCGV_HIGH(ret), TCGV_HIGH(arg1), 31); 1788 } else { 1789 tcg_gen_shri_i32(TCGV_LOW(ret), TCGV_HIGH(arg1), c); 1790 tcg_gen_movi_i32(TCGV_HIGH(ret), 0); 1791 } 1792 } else { 1793 tcg_gen_shli_i32(TCGV_HIGH(ret), TCGV_LOW(arg1), c); 1794 tcg_gen_movi_i32(TCGV_LOW(ret), 0); 1795 } 1796 } else if (right) { 1797 if (TCG_TARGET_HAS_extract2_i32) { 1798 tcg_gen_extract2_i32(TCGV_LOW(ret), 1799 TCGV_LOW(arg1), TCGV_HIGH(arg1), c); 1800 } else { 1801 tcg_gen_shri_i32(TCGV_LOW(ret), TCGV_LOW(arg1), c); 1802 tcg_gen_deposit_i32(TCGV_LOW(ret), TCGV_LOW(ret), 1803 TCGV_HIGH(arg1), 32 - c, c); 1804 } 1805 if (arith) { 1806 tcg_gen_sari_i32(TCGV_HIGH(ret), TCGV_HIGH(arg1), c); 1807 } else { 1808 tcg_gen_shri_i32(TCGV_HIGH(ret), TCGV_HIGH(arg1), c); 1809 } 1810 } else { 1811 if (TCG_TARGET_HAS_extract2_i32) { 1812 tcg_gen_extract2_i32(TCGV_HIGH(ret), 1813 TCGV_LOW(arg1), TCGV_HIGH(arg1), 32 - c); 1814 } else { 1815 TCGv_i32 t0 = tcg_temp_ebb_new_i32(); 1816 tcg_gen_shri_i32(t0, TCGV_LOW(arg1), 32 - c); 1817 tcg_gen_deposit_i32(TCGV_HIGH(ret), t0, 1818 TCGV_HIGH(arg1), c, 32 - c); 1819 tcg_temp_free_i32(t0); 1820 } 1821 tcg_gen_shli_i32(TCGV_LOW(ret), TCGV_LOW(arg1), c); 1822 } 1823 } 1824 1825 void tcg_gen_shli_i64(TCGv_i64 ret, TCGv_i64 arg1, int64_t arg2) 1826 { 1827 tcg_debug_assert(arg2 >= 0 && arg2 < 64); 1828 if (TCG_TARGET_REG_BITS == 32) { 1829 tcg_gen_shifti_i64(ret, arg1, arg2, 0, 0); 1830 } else if (arg2 == 0) { 1831 tcg_gen_mov_i64(ret, arg1); 1832 } else { 1833 tcg_gen_shl_i64(ret, arg1, tcg_constant_i64(arg2)); 1834 } 1835 } 1836 1837 void tcg_gen_shri_i64(TCGv_i64 ret, TCGv_i64 arg1, int64_t arg2) 1838 { 1839 tcg_debug_assert(arg2 >= 0 && arg2 < 64); 1840 if (TCG_TARGET_REG_BITS == 32) { 1841 tcg_gen_shifti_i64(ret, arg1, arg2, 1, 0); 1842 } else if (arg2 == 0) { 1843 tcg_gen_mov_i64(ret, arg1); 1844 } else { 1845 tcg_gen_shr_i64(ret, arg1, tcg_constant_i64(arg2)); 1846 } 1847 } 1848 1849 void tcg_gen_sari_i64(TCGv_i64 ret, TCGv_i64 arg1, int64_t arg2) 1850 { 1851 tcg_debug_assert(arg2 >= 0 && arg2 < 64); 1852 if (TCG_TARGET_REG_BITS == 32) { 1853 tcg_gen_shifti_i64(ret, arg1, arg2, 1, 1); 1854 } else if (arg2 == 0) { 1855 tcg_gen_mov_i64(ret, arg1); 1856 } else { 1857 tcg_gen_sar_i64(ret, arg1, tcg_constant_i64(arg2)); 1858 } 1859 } 1860 1861 void tcg_gen_brcond_i64(TCGCond cond, TCGv_i64 arg1, TCGv_i64 arg2, TCGLabel *l) 1862 { 1863 if (cond == TCG_COND_ALWAYS) { 1864 tcg_gen_br(l); 1865 } else if (cond != TCG_COND_NEVER) { 1866 TCGOp *op; 1867 if (TCG_TARGET_REG_BITS == 32) { 1868 op = tcg_gen_op6ii_i32(INDEX_op_brcond2_i32, TCGV_LOW(arg1), 1869 TCGV_HIGH(arg1), TCGV_LOW(arg2), 1870 TCGV_HIGH(arg2), cond, label_arg(l)); 1871 } else { 1872 op = tcg_gen_op4ii_i64(INDEX_op_brcond_i64, arg1, arg2, cond, 1873 label_arg(l)); 1874 } 1875 add_as_label_use(l, op); 1876 } 1877 } 1878 1879 void tcg_gen_brcondi_i64(TCGCond cond, TCGv_i64 arg1, int64_t arg2, TCGLabel *l) 1880 { 1881 if (TCG_TARGET_REG_BITS == 64) { 1882 tcg_gen_brcond_i64(cond, arg1, tcg_constant_i64(arg2), l); 1883 } else if (cond == TCG_COND_ALWAYS) { 1884 tcg_gen_br(l); 1885 } else if (cond != TCG_COND_NEVER) { 1886 TCGOp *op = tcg_gen_op6ii_i32(INDEX_op_brcond2_i32, 1887 TCGV_LOW(arg1), TCGV_HIGH(arg1), 1888 tcg_constant_i32(arg2), 1889 tcg_constant_i32(arg2 >> 32), 1890 cond, label_arg(l)); 1891 add_as_label_use(l, op); 1892 } 1893 } 1894 1895 void tcg_gen_setcond_i64(TCGCond cond, TCGv_i64 ret, 1896 TCGv_i64 arg1, TCGv_i64 arg2) 1897 { 1898 if (cond == TCG_COND_ALWAYS) { 1899 tcg_gen_movi_i64(ret, 1); 1900 } else if (cond == TCG_COND_NEVER) { 1901 tcg_gen_movi_i64(ret, 0); 1902 } else { 1903 if (TCG_TARGET_REG_BITS == 32) { 1904 tcg_gen_op6i_i32(INDEX_op_setcond2_i32, TCGV_LOW(ret), 1905 TCGV_LOW(arg1), TCGV_HIGH(arg1), 1906 TCGV_LOW(arg2), TCGV_HIGH(arg2), cond); 1907 tcg_gen_movi_i32(TCGV_HIGH(ret), 0); 1908 } else { 1909 tcg_gen_op4i_i64(INDEX_op_setcond_i64, ret, arg1, arg2, cond); 1910 } 1911 } 1912 } 1913 1914 void tcg_gen_setcondi_i64(TCGCond cond, TCGv_i64 ret, 1915 TCGv_i64 arg1, int64_t arg2) 1916 { 1917 if (TCG_TARGET_REG_BITS == 64) { 1918 tcg_gen_setcond_i64(cond, ret, arg1, tcg_constant_i64(arg2)); 1919 } else if (cond == TCG_COND_ALWAYS) { 1920 tcg_gen_movi_i64(ret, 1); 1921 } else if (cond == TCG_COND_NEVER) { 1922 tcg_gen_movi_i64(ret, 0); 1923 } else { 1924 tcg_gen_op6i_i32(INDEX_op_setcond2_i32, TCGV_LOW(ret), 1925 TCGV_LOW(arg1), TCGV_HIGH(arg1), 1926 tcg_constant_i32(arg2), 1927 tcg_constant_i32(arg2 >> 32), cond); 1928 tcg_gen_movi_i32(TCGV_HIGH(ret), 0); 1929 } 1930 } 1931 1932 void tcg_gen_negsetcondi_i64(TCGCond cond, TCGv_i64 ret, 1933 TCGv_i64 arg1, int64_t arg2) 1934 { 1935 tcg_gen_negsetcond_i64(cond, ret, arg1, tcg_constant_i64(arg2)); 1936 } 1937 1938 void tcg_gen_negsetcond_i64(TCGCond cond, TCGv_i64 ret, 1939 TCGv_i64 arg1, TCGv_i64 arg2) 1940 { 1941 if (cond == TCG_COND_ALWAYS) { 1942 tcg_gen_movi_i64(ret, -1); 1943 } else if (cond == TCG_COND_NEVER) { 1944 tcg_gen_movi_i64(ret, 0); 1945 } else if (TCG_TARGET_HAS_negsetcond_i64) { 1946 tcg_gen_op4i_i64(INDEX_op_negsetcond_i64, ret, arg1, arg2, cond); 1947 } else if (TCG_TARGET_REG_BITS == 32) { 1948 tcg_gen_op6i_i32(INDEX_op_setcond2_i32, TCGV_LOW(ret), 1949 TCGV_LOW(arg1), TCGV_HIGH(arg1), 1950 TCGV_LOW(arg2), TCGV_HIGH(arg2), cond); 1951 tcg_gen_neg_i32(TCGV_LOW(ret), TCGV_LOW(ret)); 1952 tcg_gen_mov_i32(TCGV_HIGH(ret), TCGV_LOW(ret)); 1953 } else { 1954 tcg_gen_setcond_i64(cond, ret, arg1, arg2); 1955 tcg_gen_neg_i64(ret, ret); 1956 } 1957 } 1958 1959 void tcg_gen_muli_i64(TCGv_i64 ret, TCGv_i64 arg1, int64_t arg2) 1960 { 1961 if (arg2 == 0) { 1962 tcg_gen_movi_i64(ret, 0); 1963 } else if (is_power_of_2(arg2)) { 1964 tcg_gen_shli_i64(ret, arg1, ctz64(arg2)); 1965 } else { 1966 tcg_gen_mul_i64(ret, arg1, tcg_constant_i64(arg2)); 1967 } 1968 } 1969 1970 void tcg_gen_div_i64(TCGv_i64 ret, TCGv_i64 arg1, TCGv_i64 arg2) 1971 { 1972 if (TCG_TARGET_HAS_div_i64) { 1973 tcg_gen_op3_i64(INDEX_op_div_i64, ret, arg1, arg2); 1974 } else if (TCG_TARGET_HAS_div2_i64) { 1975 TCGv_i64 t0 = tcg_temp_ebb_new_i64(); 1976 tcg_gen_sari_i64(t0, arg1, 63); 1977 tcg_gen_op5_i64(INDEX_op_div2_i64, ret, t0, arg1, t0, arg2); 1978 tcg_temp_free_i64(t0); 1979 } else { 1980 gen_helper_div_i64(ret, arg1, arg2); 1981 } 1982 } 1983 1984 void tcg_gen_rem_i64(TCGv_i64 ret, TCGv_i64 arg1, TCGv_i64 arg2) 1985 { 1986 if (TCG_TARGET_HAS_rem_i64) { 1987 tcg_gen_op3_i64(INDEX_op_rem_i64, ret, arg1, arg2); 1988 } else if (TCG_TARGET_HAS_div_i64) { 1989 TCGv_i64 t0 = tcg_temp_ebb_new_i64(); 1990 tcg_gen_op3_i64(INDEX_op_div_i64, t0, arg1, arg2); 1991 tcg_gen_mul_i64(t0, t0, arg2); 1992 tcg_gen_sub_i64(ret, arg1, t0); 1993 tcg_temp_free_i64(t0); 1994 } else if (TCG_TARGET_HAS_div2_i64) { 1995 TCGv_i64 t0 = tcg_temp_ebb_new_i64(); 1996 tcg_gen_sari_i64(t0, arg1, 63); 1997 tcg_gen_op5_i64(INDEX_op_div2_i64, t0, ret, arg1, t0, arg2); 1998 tcg_temp_free_i64(t0); 1999 } else { 2000 gen_helper_rem_i64(ret, arg1, arg2); 2001 } 2002 } 2003 2004 void tcg_gen_divu_i64(TCGv_i64 ret, TCGv_i64 arg1, TCGv_i64 arg2) 2005 { 2006 if (TCG_TARGET_HAS_div_i64) { 2007 tcg_gen_op3_i64(INDEX_op_divu_i64, ret, arg1, arg2); 2008 } else if (TCG_TARGET_HAS_div2_i64) { 2009 TCGv_i64 t0 = tcg_temp_ebb_new_i64(); 2010 TCGv_i64 zero = tcg_constant_i64(0); 2011 tcg_gen_op5_i64(INDEX_op_divu2_i64, ret, t0, arg1, zero, arg2); 2012 tcg_temp_free_i64(t0); 2013 } else { 2014 gen_helper_divu_i64(ret, arg1, arg2); 2015 } 2016 } 2017 2018 void tcg_gen_remu_i64(TCGv_i64 ret, TCGv_i64 arg1, TCGv_i64 arg2) 2019 { 2020 if (TCG_TARGET_HAS_rem_i64) { 2021 tcg_gen_op3_i64(INDEX_op_remu_i64, ret, arg1, arg2); 2022 } else if (TCG_TARGET_HAS_div_i64) { 2023 TCGv_i64 t0 = tcg_temp_ebb_new_i64(); 2024 tcg_gen_op3_i64(INDEX_op_divu_i64, t0, arg1, arg2); 2025 tcg_gen_mul_i64(t0, t0, arg2); 2026 tcg_gen_sub_i64(ret, arg1, t0); 2027 tcg_temp_free_i64(t0); 2028 } else if (TCG_TARGET_HAS_div2_i64) { 2029 TCGv_i64 t0 = tcg_temp_ebb_new_i64(); 2030 TCGv_i64 zero = tcg_constant_i64(0); 2031 tcg_gen_op5_i64(INDEX_op_divu2_i64, t0, ret, arg1, zero, arg2); 2032 tcg_temp_free_i64(t0); 2033 } else { 2034 gen_helper_remu_i64(ret, arg1, arg2); 2035 } 2036 } 2037 2038 void tcg_gen_ext8s_i64(TCGv_i64 ret, TCGv_i64 arg) 2039 { 2040 tcg_gen_sextract_i64(ret, arg, 0, 8); 2041 } 2042 2043 void tcg_gen_ext16s_i64(TCGv_i64 ret, TCGv_i64 arg) 2044 { 2045 tcg_gen_sextract_i64(ret, arg, 0, 16); 2046 } 2047 2048 void tcg_gen_ext32s_i64(TCGv_i64 ret, TCGv_i64 arg) 2049 { 2050 tcg_gen_sextract_i64(ret, arg, 0, 32); 2051 } 2052 2053 void tcg_gen_ext8u_i64(TCGv_i64 ret, TCGv_i64 arg) 2054 { 2055 tcg_gen_extract_i64(ret, arg, 0, 8); 2056 } 2057 2058 void tcg_gen_ext16u_i64(TCGv_i64 ret, TCGv_i64 arg) 2059 { 2060 tcg_gen_extract_i64(ret, arg, 0, 16); 2061 } 2062 2063 void tcg_gen_ext32u_i64(TCGv_i64 ret, TCGv_i64 arg) 2064 { 2065 tcg_gen_extract_i64(ret, arg, 0, 32); 2066 } 2067 2068 /* 2069 * bswap16_i64: 16-bit byte swap on the low bits of a 64-bit value. 2070 * 2071 * Byte pattern: xxxxxxxxab -> yyyyyyyyba 2072 * 2073 * With TCG_BSWAP_IZ, x == zero, else undefined. 2074 * With TCG_BSWAP_OZ, y == zero, with TCG_BSWAP_OS y == sign, else undefined. 2075 */ 2076 void tcg_gen_bswap16_i64(TCGv_i64 ret, TCGv_i64 arg, int flags) 2077 { 2078 /* Only one extension flag may be present. */ 2079 tcg_debug_assert(!(flags & TCG_BSWAP_OS) || !(flags & TCG_BSWAP_OZ)); 2080 2081 if (TCG_TARGET_REG_BITS == 32) { 2082 tcg_gen_bswap16_i32(TCGV_LOW(ret), TCGV_LOW(arg), flags); 2083 if (flags & TCG_BSWAP_OS) { 2084 tcg_gen_sari_i32(TCGV_HIGH(ret), TCGV_LOW(ret), 31); 2085 } else { 2086 tcg_gen_movi_i32(TCGV_HIGH(ret), 0); 2087 } 2088 } else if (TCG_TARGET_HAS_bswap16_i64) { 2089 tcg_gen_op3i_i64(INDEX_op_bswap16_i64, ret, arg, flags); 2090 } else { 2091 TCGv_i64 t0 = tcg_temp_ebb_new_i64(); 2092 TCGv_i64 t1 = tcg_temp_ebb_new_i64(); 2093 2094 /* arg = ......ab or xxxxxxab */ 2095 tcg_gen_shri_i64(t0, arg, 8); /* t0 = .......a or .xxxxxxa */ 2096 if (!(flags & TCG_BSWAP_IZ)) { 2097 tcg_gen_ext8u_i64(t0, t0); /* t0 = .......a */ 2098 } 2099 2100 if (flags & TCG_BSWAP_OS) { 2101 tcg_gen_shli_i64(t1, arg, 56); /* t1 = b....... */ 2102 tcg_gen_sari_i64(t1, t1, 48); /* t1 = ssssssb. */ 2103 } else if (flags & TCG_BSWAP_OZ) { 2104 tcg_gen_ext8u_i64(t1, arg); /* t1 = .......b */ 2105 tcg_gen_shli_i64(t1, t1, 8); /* t1 = ......b. */ 2106 } else { 2107 tcg_gen_shli_i64(t1, arg, 8); /* t1 = xxxxxab. */ 2108 } 2109 2110 tcg_gen_or_i64(ret, t0, t1); /* ret = ......ba (OZ) */ 2111 /* ssssssba (OS) */ 2112 /* xxxxxaba (no flag) */ 2113 tcg_temp_free_i64(t0); 2114 tcg_temp_free_i64(t1); 2115 } 2116 } 2117 2118 /* 2119 * bswap32_i64: 32-bit byte swap on the low bits of a 64-bit value. 2120 * 2121 * Byte pattern: xxxxabcd -> yyyydcba 2122 * 2123 * With TCG_BSWAP_IZ, x == zero, else undefined. 2124 * With TCG_BSWAP_OZ, y == zero, with TCG_BSWAP_OS y == sign, else undefined. 2125 */ 2126 void tcg_gen_bswap32_i64(TCGv_i64 ret, TCGv_i64 arg, int flags) 2127 { 2128 /* Only one extension flag may be present. */ 2129 tcg_debug_assert(!(flags & TCG_BSWAP_OS) || !(flags & TCG_BSWAP_OZ)); 2130 2131 if (TCG_TARGET_REG_BITS == 32) { 2132 tcg_gen_bswap32_i32(TCGV_LOW(ret), TCGV_LOW(arg)); 2133 if (flags & TCG_BSWAP_OS) { 2134 tcg_gen_sari_i32(TCGV_HIGH(ret), TCGV_LOW(ret), 31); 2135 } else { 2136 tcg_gen_movi_i32(TCGV_HIGH(ret), 0); 2137 } 2138 } else if (TCG_TARGET_HAS_bswap32_i64) { 2139 tcg_gen_op3i_i64(INDEX_op_bswap32_i64, ret, arg, flags); 2140 } else { 2141 TCGv_i64 t0 = tcg_temp_ebb_new_i64(); 2142 TCGv_i64 t1 = tcg_temp_ebb_new_i64(); 2143 TCGv_i64 t2 = tcg_constant_i64(0x00ff00ff); 2144 2145 /* arg = xxxxabcd */ 2146 tcg_gen_shri_i64(t0, arg, 8); /* t0 = .xxxxabc */ 2147 tcg_gen_and_i64(t1, arg, t2); /* t1 = .....b.d */ 2148 tcg_gen_and_i64(t0, t0, t2); /* t0 = .....a.c */ 2149 tcg_gen_shli_i64(t1, t1, 8); /* t1 = ....b.d. */ 2150 tcg_gen_or_i64(ret, t0, t1); /* ret = ....badc */ 2151 2152 tcg_gen_shli_i64(t1, ret, 48); /* t1 = dc...... */ 2153 tcg_gen_shri_i64(t0, ret, 16); /* t0 = ......ba */ 2154 if (flags & TCG_BSWAP_OS) { 2155 tcg_gen_sari_i64(t1, t1, 32); /* t1 = ssssdc.. */ 2156 } else { 2157 tcg_gen_shri_i64(t1, t1, 32); /* t1 = ....dc.. */ 2158 } 2159 tcg_gen_or_i64(ret, t0, t1); /* ret = ssssdcba (OS) */ 2160 /* ....dcba (else) */ 2161 2162 tcg_temp_free_i64(t0); 2163 tcg_temp_free_i64(t1); 2164 } 2165 } 2166 2167 /* 2168 * bswap64_i64: 64-bit byte swap on a 64-bit value. 2169 * 2170 * Byte pattern: abcdefgh -> hgfedcba 2171 */ 2172 void tcg_gen_bswap64_i64(TCGv_i64 ret, TCGv_i64 arg) 2173 { 2174 if (TCG_TARGET_REG_BITS == 32) { 2175 TCGv_i32 t0, t1; 2176 t0 = tcg_temp_ebb_new_i32(); 2177 t1 = tcg_temp_ebb_new_i32(); 2178 2179 tcg_gen_bswap32_i32(t0, TCGV_LOW(arg)); 2180 tcg_gen_bswap32_i32(t1, TCGV_HIGH(arg)); 2181 tcg_gen_mov_i32(TCGV_LOW(ret), t1); 2182 tcg_gen_mov_i32(TCGV_HIGH(ret), t0); 2183 tcg_temp_free_i32(t0); 2184 tcg_temp_free_i32(t1); 2185 } else if (TCG_TARGET_HAS_bswap64_i64) { 2186 tcg_gen_op3i_i64(INDEX_op_bswap64_i64, ret, arg, 0); 2187 } else { 2188 TCGv_i64 t0 = tcg_temp_ebb_new_i64(); 2189 TCGv_i64 t1 = tcg_temp_ebb_new_i64(); 2190 TCGv_i64 t2 = tcg_temp_ebb_new_i64(); 2191 2192 /* arg = abcdefgh */ 2193 tcg_gen_movi_i64(t2, 0x00ff00ff00ff00ffull); 2194 tcg_gen_shri_i64(t0, arg, 8); /* t0 = .abcdefg */ 2195 tcg_gen_and_i64(t1, arg, t2); /* t1 = .b.d.f.h */ 2196 tcg_gen_and_i64(t0, t0, t2); /* t0 = .a.c.e.g */ 2197 tcg_gen_shli_i64(t1, t1, 8); /* t1 = b.d.f.h. */ 2198 tcg_gen_or_i64(ret, t0, t1); /* ret = badcfehg */ 2199 2200 tcg_gen_movi_i64(t2, 0x0000ffff0000ffffull); 2201 tcg_gen_shri_i64(t0, ret, 16); /* t0 = ..badcfe */ 2202 tcg_gen_and_i64(t1, ret, t2); /* t1 = ..dc..hg */ 2203 tcg_gen_and_i64(t0, t0, t2); /* t0 = ..ba..fe */ 2204 tcg_gen_shli_i64(t1, t1, 16); /* t1 = dc..hg.. */ 2205 tcg_gen_or_i64(ret, t0, t1); /* ret = dcbahgfe */ 2206 2207 tcg_gen_shri_i64(t0, ret, 32); /* t0 = ....dcba */ 2208 tcg_gen_shli_i64(t1, ret, 32); /* t1 = hgfe.... */ 2209 tcg_gen_or_i64(ret, t0, t1); /* ret = hgfedcba */ 2210 2211 tcg_temp_free_i64(t0); 2212 tcg_temp_free_i64(t1); 2213 tcg_temp_free_i64(t2); 2214 } 2215 } 2216 2217 /* 2218 * hswap_i64: Swap 16-bit halfwords within a 64-bit value. 2219 * See also include/qemu/bitops.h, hswap64. 2220 * 2221 * Byte pattern: abcdefgh -> ghefcdab 2222 */ 2223 void tcg_gen_hswap_i64(TCGv_i64 ret, TCGv_i64 arg) 2224 { 2225 uint64_t m = 0x0000ffff0000ffffull; 2226 TCGv_i64 t0 = tcg_temp_ebb_new_i64(); 2227 TCGv_i64 t1 = tcg_temp_ebb_new_i64(); 2228 2229 /* arg = abcdefgh */ 2230 tcg_gen_rotli_i64(t1, arg, 32); /* t1 = efghabcd */ 2231 tcg_gen_andi_i64(t0, t1, m); /* t0 = ..gh..cd */ 2232 tcg_gen_shli_i64(t0, t0, 16); /* t0 = gh..cd.. */ 2233 tcg_gen_shri_i64(t1, t1, 16); /* t1 = ..efghab */ 2234 tcg_gen_andi_i64(t1, t1, m); /* t1 = ..ef..ab */ 2235 tcg_gen_or_i64(ret, t0, t1); /* ret = ghefcdab */ 2236 2237 tcg_temp_free_i64(t0); 2238 tcg_temp_free_i64(t1); 2239 } 2240 2241 /* 2242 * wswap_i64: Swap 32-bit words within a 64-bit value. 2243 * 2244 * Byte pattern: abcdefgh -> efghabcd 2245 */ 2246 void tcg_gen_wswap_i64(TCGv_i64 ret, TCGv_i64 arg) 2247 { 2248 /* Swapping 2 32-bit elements is a rotate. */ 2249 tcg_gen_rotli_i64(ret, arg, 32); 2250 } 2251 2252 void tcg_gen_not_i64(TCGv_i64 ret, TCGv_i64 arg) 2253 { 2254 if (TCG_TARGET_REG_BITS == 32) { 2255 tcg_gen_not_i32(TCGV_LOW(ret), TCGV_LOW(arg)); 2256 tcg_gen_not_i32(TCGV_HIGH(ret), TCGV_HIGH(arg)); 2257 } else if (tcg_op_supported(INDEX_op_not, TCG_TYPE_I64, 0)) { 2258 tcg_gen_op2_i64(INDEX_op_not, ret, arg); 2259 } else { 2260 tcg_gen_xori_i64(ret, arg, -1); 2261 } 2262 } 2263 2264 void tcg_gen_andc_i64(TCGv_i64 ret, TCGv_i64 arg1, TCGv_i64 arg2) 2265 { 2266 if (TCG_TARGET_REG_BITS == 32) { 2267 tcg_gen_andc_i32(TCGV_LOW(ret), TCGV_LOW(arg1), TCGV_LOW(arg2)); 2268 tcg_gen_andc_i32(TCGV_HIGH(ret), TCGV_HIGH(arg1), TCGV_HIGH(arg2)); 2269 } else if (tcg_op_supported(INDEX_op_andc, TCG_TYPE_I64, 0)) { 2270 tcg_gen_op3_i64(INDEX_op_andc, ret, arg1, arg2); 2271 } else { 2272 TCGv_i64 t0 = tcg_temp_ebb_new_i64(); 2273 tcg_gen_not_i64(t0, arg2); 2274 tcg_gen_and_i64(ret, arg1, t0); 2275 tcg_temp_free_i64(t0); 2276 } 2277 } 2278 2279 void tcg_gen_eqv_i64(TCGv_i64 ret, TCGv_i64 arg1, TCGv_i64 arg2) 2280 { 2281 if (TCG_TARGET_REG_BITS == 32) { 2282 tcg_gen_eqv_i32(TCGV_LOW(ret), TCGV_LOW(arg1), TCGV_LOW(arg2)); 2283 tcg_gen_eqv_i32(TCGV_HIGH(ret), TCGV_HIGH(arg1), TCGV_HIGH(arg2)); 2284 } else if (tcg_op_supported(INDEX_op_eqv, TCG_TYPE_I64, 0)) { 2285 tcg_gen_op3_i64(INDEX_op_eqv, ret, arg1, arg2); 2286 } else { 2287 tcg_gen_xor_i64(ret, arg1, arg2); 2288 tcg_gen_not_i64(ret, ret); 2289 } 2290 } 2291 2292 void tcg_gen_nand_i64(TCGv_i64 ret, TCGv_i64 arg1, TCGv_i64 arg2) 2293 { 2294 if (TCG_TARGET_REG_BITS == 32) { 2295 tcg_gen_nand_i32(TCGV_LOW(ret), TCGV_LOW(arg1), TCGV_LOW(arg2)); 2296 tcg_gen_nand_i32(TCGV_HIGH(ret), TCGV_HIGH(arg1), TCGV_HIGH(arg2)); 2297 } else if (tcg_op_supported(INDEX_op_nand, TCG_TYPE_I64, 0)) { 2298 tcg_gen_op3_i64(INDEX_op_nand, ret, arg1, arg2); 2299 } else { 2300 tcg_gen_and_i64(ret, arg1, arg2); 2301 tcg_gen_not_i64(ret, ret); 2302 } 2303 } 2304 2305 void tcg_gen_nor_i64(TCGv_i64 ret, TCGv_i64 arg1, TCGv_i64 arg2) 2306 { 2307 if (TCG_TARGET_REG_BITS == 32) { 2308 tcg_gen_nor_i32(TCGV_LOW(ret), TCGV_LOW(arg1), TCGV_LOW(arg2)); 2309 tcg_gen_nor_i32(TCGV_HIGH(ret), TCGV_HIGH(arg1), TCGV_HIGH(arg2)); 2310 } else if (tcg_op_supported(INDEX_op_nor, TCG_TYPE_I64, 0)) { 2311 tcg_gen_op3_i64(INDEX_op_nor, ret, arg1, arg2); 2312 } else { 2313 tcg_gen_or_i64(ret, arg1, arg2); 2314 tcg_gen_not_i64(ret, ret); 2315 } 2316 } 2317 2318 void tcg_gen_orc_i64(TCGv_i64 ret, TCGv_i64 arg1, TCGv_i64 arg2) 2319 { 2320 if (TCG_TARGET_REG_BITS == 32) { 2321 tcg_gen_orc_i32(TCGV_LOW(ret), TCGV_LOW(arg1), TCGV_LOW(arg2)); 2322 tcg_gen_orc_i32(TCGV_HIGH(ret), TCGV_HIGH(arg1), TCGV_HIGH(arg2)); 2323 } else if (tcg_op_supported(INDEX_op_orc, TCG_TYPE_I64, 0)) { 2324 tcg_gen_op3_i64(INDEX_op_orc, ret, arg1, arg2); 2325 } else { 2326 TCGv_i64 t0 = tcg_temp_ebb_new_i64(); 2327 tcg_gen_not_i64(t0, arg2); 2328 tcg_gen_or_i64(ret, arg1, t0); 2329 tcg_temp_free_i64(t0); 2330 } 2331 } 2332 2333 void tcg_gen_clz_i64(TCGv_i64 ret, TCGv_i64 arg1, TCGv_i64 arg2) 2334 { 2335 if (TCG_TARGET_HAS_clz_i64) { 2336 tcg_gen_op3_i64(INDEX_op_clz_i64, ret, arg1, arg2); 2337 } else { 2338 gen_helper_clz_i64(ret, arg1, arg2); 2339 } 2340 } 2341 2342 void tcg_gen_clzi_i64(TCGv_i64 ret, TCGv_i64 arg1, uint64_t arg2) 2343 { 2344 if (TCG_TARGET_REG_BITS == 32 2345 && TCG_TARGET_HAS_clz_i32 2346 && arg2 <= 0xffffffffu) { 2347 TCGv_i32 t = tcg_temp_ebb_new_i32(); 2348 tcg_gen_clzi_i32(t, TCGV_LOW(arg1), arg2 - 32); 2349 tcg_gen_addi_i32(t, t, 32); 2350 tcg_gen_clz_i32(TCGV_LOW(ret), TCGV_HIGH(arg1), t); 2351 tcg_gen_movi_i32(TCGV_HIGH(ret), 0); 2352 tcg_temp_free_i32(t); 2353 } else { 2354 tcg_gen_clz_i64(ret, arg1, tcg_constant_i64(arg2)); 2355 } 2356 } 2357 2358 void tcg_gen_ctz_i64(TCGv_i64 ret, TCGv_i64 arg1, TCGv_i64 arg2) 2359 { 2360 if (TCG_TARGET_HAS_ctz_i64) { 2361 tcg_gen_op3_i64(INDEX_op_ctz_i64, ret, arg1, arg2); 2362 } else if (TCG_TARGET_HAS_ctpop_i64 || TCG_TARGET_HAS_clz_i64) { 2363 TCGv_i64 z, t = tcg_temp_ebb_new_i64(); 2364 2365 if (TCG_TARGET_HAS_ctpop_i64) { 2366 tcg_gen_subi_i64(t, arg1, 1); 2367 tcg_gen_andc_i64(t, t, arg1); 2368 tcg_gen_ctpop_i64(t, t); 2369 } else { 2370 /* Since all non-x86 hosts have clz(0) == 64, don't fight it. */ 2371 tcg_gen_neg_i64(t, arg1); 2372 tcg_gen_and_i64(t, t, arg1); 2373 tcg_gen_clzi_i64(t, t, 64); 2374 tcg_gen_xori_i64(t, t, 63); 2375 } 2376 z = tcg_constant_i64(0); 2377 tcg_gen_movcond_i64(TCG_COND_EQ, ret, arg1, z, arg2, t); 2378 tcg_temp_free_i64(t); 2379 tcg_temp_free_i64(z); 2380 } else { 2381 gen_helper_ctz_i64(ret, arg1, arg2); 2382 } 2383 } 2384 2385 void tcg_gen_ctzi_i64(TCGv_i64 ret, TCGv_i64 arg1, uint64_t arg2) 2386 { 2387 if (TCG_TARGET_REG_BITS == 32 2388 && TCG_TARGET_HAS_ctz_i32 2389 && arg2 <= 0xffffffffu) { 2390 TCGv_i32 t32 = tcg_temp_ebb_new_i32(); 2391 tcg_gen_ctzi_i32(t32, TCGV_HIGH(arg1), arg2 - 32); 2392 tcg_gen_addi_i32(t32, t32, 32); 2393 tcg_gen_ctz_i32(TCGV_LOW(ret), TCGV_LOW(arg1), t32); 2394 tcg_gen_movi_i32(TCGV_HIGH(ret), 0); 2395 tcg_temp_free_i32(t32); 2396 } else if (!TCG_TARGET_HAS_ctz_i64 2397 && TCG_TARGET_HAS_ctpop_i64 2398 && arg2 == 64) { 2399 /* This equivalence has the advantage of not requiring a fixup. */ 2400 TCGv_i64 t = tcg_temp_ebb_new_i64(); 2401 tcg_gen_subi_i64(t, arg1, 1); 2402 tcg_gen_andc_i64(t, t, arg1); 2403 tcg_gen_ctpop_i64(ret, t); 2404 tcg_temp_free_i64(t); 2405 } else { 2406 tcg_gen_ctz_i64(ret, arg1, tcg_constant_i64(arg2)); 2407 } 2408 } 2409 2410 void tcg_gen_clrsb_i64(TCGv_i64 ret, TCGv_i64 arg) 2411 { 2412 if (TCG_TARGET_HAS_clz_i64 || TCG_TARGET_HAS_clz_i32) { 2413 TCGv_i64 t = tcg_temp_ebb_new_i64(); 2414 tcg_gen_sari_i64(t, arg, 63); 2415 tcg_gen_xor_i64(t, t, arg); 2416 tcg_gen_clzi_i64(t, t, 64); 2417 tcg_gen_subi_i64(ret, t, 1); 2418 tcg_temp_free_i64(t); 2419 } else { 2420 gen_helper_clrsb_i64(ret, arg); 2421 } 2422 } 2423 2424 void tcg_gen_ctpop_i64(TCGv_i64 ret, TCGv_i64 arg1) 2425 { 2426 if (TCG_TARGET_HAS_ctpop_i64) { 2427 tcg_gen_op2_i64(INDEX_op_ctpop_i64, ret, arg1); 2428 } else if (TCG_TARGET_REG_BITS == 32 && TCG_TARGET_HAS_ctpop_i32) { 2429 tcg_gen_ctpop_i32(TCGV_HIGH(ret), TCGV_HIGH(arg1)); 2430 tcg_gen_ctpop_i32(TCGV_LOW(ret), TCGV_LOW(arg1)); 2431 tcg_gen_add_i32(TCGV_LOW(ret), TCGV_LOW(ret), TCGV_HIGH(ret)); 2432 tcg_gen_movi_i32(TCGV_HIGH(ret), 0); 2433 } else { 2434 gen_helper_ctpop_i64(ret, arg1); 2435 } 2436 } 2437 2438 void tcg_gen_rotl_i64(TCGv_i64 ret, TCGv_i64 arg1, TCGv_i64 arg2) 2439 { 2440 if (TCG_TARGET_HAS_rot_i64) { 2441 tcg_gen_op3_i64(INDEX_op_rotl_i64, ret, arg1, arg2); 2442 } else { 2443 TCGv_i64 t0, t1; 2444 t0 = tcg_temp_ebb_new_i64(); 2445 t1 = tcg_temp_ebb_new_i64(); 2446 tcg_gen_shl_i64(t0, arg1, arg2); 2447 tcg_gen_subfi_i64(t1, 64, arg2); 2448 tcg_gen_shr_i64(t1, arg1, t1); 2449 tcg_gen_or_i64(ret, t0, t1); 2450 tcg_temp_free_i64(t0); 2451 tcg_temp_free_i64(t1); 2452 } 2453 } 2454 2455 void tcg_gen_rotli_i64(TCGv_i64 ret, TCGv_i64 arg1, int64_t arg2) 2456 { 2457 tcg_debug_assert(arg2 >= 0 && arg2 < 64); 2458 /* some cases can be optimized here */ 2459 if (arg2 == 0) { 2460 tcg_gen_mov_i64(ret, arg1); 2461 } else if (TCG_TARGET_HAS_rot_i64) { 2462 tcg_gen_rotl_i64(ret, arg1, tcg_constant_i64(arg2)); 2463 } else { 2464 TCGv_i64 t0, t1; 2465 t0 = tcg_temp_ebb_new_i64(); 2466 t1 = tcg_temp_ebb_new_i64(); 2467 tcg_gen_shli_i64(t0, arg1, arg2); 2468 tcg_gen_shri_i64(t1, arg1, 64 - arg2); 2469 tcg_gen_or_i64(ret, t0, t1); 2470 tcg_temp_free_i64(t0); 2471 tcg_temp_free_i64(t1); 2472 } 2473 } 2474 2475 void tcg_gen_rotr_i64(TCGv_i64 ret, TCGv_i64 arg1, TCGv_i64 arg2) 2476 { 2477 if (TCG_TARGET_HAS_rot_i64) { 2478 tcg_gen_op3_i64(INDEX_op_rotr_i64, ret, arg1, arg2); 2479 } else { 2480 TCGv_i64 t0, t1; 2481 t0 = tcg_temp_ebb_new_i64(); 2482 t1 = tcg_temp_ebb_new_i64(); 2483 tcg_gen_shr_i64(t0, arg1, arg2); 2484 tcg_gen_subfi_i64(t1, 64, arg2); 2485 tcg_gen_shl_i64(t1, arg1, t1); 2486 tcg_gen_or_i64(ret, t0, t1); 2487 tcg_temp_free_i64(t0); 2488 tcg_temp_free_i64(t1); 2489 } 2490 } 2491 2492 void tcg_gen_rotri_i64(TCGv_i64 ret, TCGv_i64 arg1, int64_t arg2) 2493 { 2494 tcg_debug_assert(arg2 >= 0 && arg2 < 64); 2495 /* some cases can be optimized here */ 2496 if (arg2 == 0) { 2497 tcg_gen_mov_i64(ret, arg1); 2498 } else { 2499 tcg_gen_rotli_i64(ret, arg1, 64 - arg2); 2500 } 2501 } 2502 2503 void tcg_gen_deposit_i64(TCGv_i64 ret, TCGv_i64 arg1, TCGv_i64 arg2, 2504 unsigned int ofs, unsigned int len) 2505 { 2506 uint64_t mask; 2507 TCGv_i64 t1; 2508 2509 tcg_debug_assert(ofs < 64); 2510 tcg_debug_assert(len > 0); 2511 tcg_debug_assert(len <= 64); 2512 tcg_debug_assert(ofs + len <= 64); 2513 2514 if (len == 64) { 2515 tcg_gen_mov_i64(ret, arg2); 2516 return; 2517 } 2518 2519 if (TCG_TARGET_REG_BITS == 64) { 2520 if (TCG_TARGET_deposit_valid(TCG_TYPE_I64, ofs, len)) { 2521 tcg_gen_op5ii_i64(INDEX_op_deposit_i64, ret, arg1, arg2, ofs, len); 2522 return; 2523 } 2524 } else { 2525 if (ofs >= 32) { 2526 tcg_gen_deposit_i32(TCGV_HIGH(ret), TCGV_HIGH(arg1), 2527 TCGV_LOW(arg2), ofs - 32, len); 2528 tcg_gen_mov_i32(TCGV_LOW(ret), TCGV_LOW(arg1)); 2529 return; 2530 } 2531 if (ofs + len <= 32) { 2532 tcg_gen_deposit_i32(TCGV_LOW(ret), TCGV_LOW(arg1), 2533 TCGV_LOW(arg2), ofs, len); 2534 tcg_gen_mov_i32(TCGV_HIGH(ret), TCGV_HIGH(arg1)); 2535 return; 2536 } 2537 } 2538 2539 t1 = tcg_temp_ebb_new_i64(); 2540 2541 if (TCG_TARGET_HAS_extract2_i64) { 2542 if (ofs + len == 64) { 2543 tcg_gen_shli_i64(t1, arg1, len); 2544 tcg_gen_extract2_i64(ret, t1, arg2, len); 2545 goto done; 2546 } 2547 if (ofs == 0) { 2548 tcg_gen_extract2_i64(ret, arg1, arg2, len); 2549 tcg_gen_rotli_i64(ret, ret, len); 2550 goto done; 2551 } 2552 } 2553 2554 mask = (1ull << len) - 1; 2555 if (ofs + len < 64) { 2556 tcg_gen_andi_i64(t1, arg2, mask); 2557 tcg_gen_shli_i64(t1, t1, ofs); 2558 } else { 2559 tcg_gen_shli_i64(t1, arg2, ofs); 2560 } 2561 tcg_gen_andi_i64(ret, arg1, ~(mask << ofs)); 2562 tcg_gen_or_i64(ret, ret, t1); 2563 done: 2564 tcg_temp_free_i64(t1); 2565 } 2566 2567 void tcg_gen_deposit_z_i64(TCGv_i64 ret, TCGv_i64 arg, 2568 unsigned int ofs, unsigned int len) 2569 { 2570 tcg_debug_assert(ofs < 64); 2571 tcg_debug_assert(len > 0); 2572 tcg_debug_assert(len <= 64); 2573 tcg_debug_assert(ofs + len <= 64); 2574 2575 if (ofs + len == 64) { 2576 tcg_gen_shli_i64(ret, arg, ofs); 2577 } else if (ofs == 0) { 2578 tcg_gen_andi_i64(ret, arg, (1ull << len) - 1); 2579 } else if (TCG_TARGET_REG_BITS == 64 && 2580 TCG_TARGET_deposit_valid(TCG_TYPE_I64, ofs, len)) { 2581 TCGv_i64 zero = tcg_constant_i64(0); 2582 tcg_gen_op5ii_i64(INDEX_op_deposit_i64, ret, zero, arg, ofs, len); 2583 } else { 2584 if (TCG_TARGET_REG_BITS == 32) { 2585 if (ofs >= 32) { 2586 tcg_gen_deposit_z_i32(TCGV_HIGH(ret), TCGV_LOW(arg), 2587 ofs - 32, len); 2588 tcg_gen_movi_i32(TCGV_LOW(ret), 0); 2589 return; 2590 } 2591 if (ofs + len <= 32) { 2592 tcg_gen_deposit_z_i32(TCGV_LOW(ret), TCGV_LOW(arg), ofs, len); 2593 tcg_gen_movi_i32(TCGV_HIGH(ret), 0); 2594 return; 2595 } 2596 } 2597 /* 2598 * To help two-operand hosts we prefer to zero-extend first, 2599 * which allows ARG to stay live. 2600 */ 2601 if (TCG_TARGET_extract_valid(TCG_TYPE_I64, 0, len)) { 2602 tcg_gen_extract_i64(ret, arg, 0, len); 2603 tcg_gen_shli_i64(ret, ret, ofs); 2604 return; 2605 } 2606 /* Otherwise prefer zero-extension over AND for code size. */ 2607 if (TCG_TARGET_extract_valid(TCG_TYPE_I64, 0, ofs + len)) { 2608 tcg_gen_shli_i64(ret, arg, ofs); 2609 tcg_gen_extract_i64(ret, ret, 0, ofs + len); 2610 return; 2611 } 2612 tcg_gen_andi_i64(ret, arg, (1ull << len) - 1); 2613 tcg_gen_shli_i64(ret, ret, ofs); 2614 } 2615 } 2616 2617 void tcg_gen_extract_i64(TCGv_i64 ret, TCGv_i64 arg, 2618 unsigned int ofs, unsigned int len) 2619 { 2620 tcg_debug_assert(ofs < 64); 2621 tcg_debug_assert(len > 0); 2622 tcg_debug_assert(len <= 64); 2623 tcg_debug_assert(ofs + len <= 64); 2624 2625 /* Canonicalize certain special cases, even if extract is supported. */ 2626 if (ofs + len == 64) { 2627 tcg_gen_shri_i64(ret, arg, 64 - len); 2628 return; 2629 } 2630 2631 if (TCG_TARGET_REG_BITS == 32) { 2632 /* Look for a 32-bit extract within one of the two words. */ 2633 if (ofs >= 32) { 2634 tcg_gen_extract_i32(TCGV_LOW(ret), TCGV_HIGH(arg), ofs - 32, len); 2635 tcg_gen_movi_i32(TCGV_HIGH(ret), 0); 2636 return; 2637 } 2638 if (ofs + len <= 32) { 2639 tcg_gen_extract_i32(TCGV_LOW(ret), TCGV_LOW(arg), ofs, len); 2640 tcg_gen_movi_i32(TCGV_HIGH(ret), 0); 2641 return; 2642 } 2643 2644 /* The field is split across two words. */ 2645 tcg_gen_extract2_i32(TCGV_LOW(ret), TCGV_LOW(arg), 2646 TCGV_HIGH(arg), ofs); 2647 if (len <= 32) { 2648 tcg_gen_extract_i32(TCGV_LOW(ret), TCGV_LOW(ret), 0, len); 2649 tcg_gen_movi_i32(TCGV_HIGH(ret), 0); 2650 } else { 2651 tcg_gen_extract_i32(TCGV_HIGH(ret), TCGV_HIGH(arg), 2652 ofs, len - 32); 2653 } 2654 return; 2655 } 2656 2657 if (TCG_TARGET_extract_valid(TCG_TYPE_I64, ofs, len)) { 2658 tcg_gen_op4ii_i64(INDEX_op_extract_i64, ret, arg, ofs, len); 2659 return; 2660 } 2661 if (ofs == 0) { 2662 tcg_gen_andi_i64(ret, arg, (1ull << len) - 1); 2663 return; 2664 } 2665 2666 /* Assume that zero-extension, if available, is cheaper than a shift. */ 2667 if (TCG_TARGET_extract_valid(TCG_TYPE_I64, 0, ofs + len)) { 2668 tcg_gen_op4ii_i64(INDEX_op_extract_i64, ret, arg, 0, ofs + len); 2669 tcg_gen_shri_i64(ret, ret, ofs); 2670 return; 2671 } 2672 2673 /* ??? Ideally we'd know what values are available for immediate AND. 2674 Assume that 8 bits are available, plus the special cases of 16 and 32, 2675 so that we get ext8u, ext16u, and ext32u. */ 2676 switch (len) { 2677 case 1 ... 8: case 16: case 32: 2678 tcg_gen_shri_i64(ret, arg, ofs); 2679 tcg_gen_andi_i64(ret, ret, (1ull << len) - 1); 2680 break; 2681 default: 2682 tcg_gen_shli_i64(ret, arg, 64 - len - ofs); 2683 tcg_gen_shri_i64(ret, ret, 64 - len); 2684 break; 2685 } 2686 } 2687 2688 void tcg_gen_sextract_i64(TCGv_i64 ret, TCGv_i64 arg, 2689 unsigned int ofs, unsigned int len) 2690 { 2691 tcg_debug_assert(ofs < 64); 2692 tcg_debug_assert(len > 0); 2693 tcg_debug_assert(len <= 64); 2694 tcg_debug_assert(ofs + len <= 64); 2695 2696 /* Canonicalize certain special cases, even if sextract is supported. */ 2697 if (ofs + len == 64) { 2698 tcg_gen_sari_i64(ret, arg, 64 - len); 2699 return; 2700 } 2701 2702 if (TCG_TARGET_REG_BITS == 32) { 2703 /* Look for a 32-bit extract within one of the two words. */ 2704 if (ofs >= 32) { 2705 tcg_gen_sextract_i32(TCGV_LOW(ret), TCGV_HIGH(arg), ofs - 32, len); 2706 } else if (ofs + len <= 32) { 2707 tcg_gen_sextract_i32(TCGV_LOW(ret), TCGV_LOW(arg), ofs, len); 2708 } else if (ofs == 0) { 2709 tcg_gen_mov_i32(TCGV_LOW(ret), TCGV_LOW(arg)); 2710 tcg_gen_sextract_i32(TCGV_HIGH(ret), TCGV_HIGH(arg), 0, len - 32); 2711 return; 2712 } else if (len > 32) { 2713 TCGv_i32 t = tcg_temp_ebb_new_i32(); 2714 /* Extract the bits for the high word normally. */ 2715 tcg_gen_sextract_i32(t, TCGV_HIGH(arg), ofs + 32, len - 32); 2716 /* Shift the field down for the low part. */ 2717 tcg_gen_shri_i64(ret, arg, ofs); 2718 /* Overwrite the shift into the high part. */ 2719 tcg_gen_mov_i32(TCGV_HIGH(ret), t); 2720 tcg_temp_free_i32(t); 2721 return; 2722 } else { 2723 /* Shift the field down for the low part, such that the 2724 field sits at the MSB. */ 2725 tcg_gen_shri_i64(ret, arg, ofs + len - 32); 2726 /* Shift the field down from the MSB, sign extending. */ 2727 tcg_gen_sari_i32(TCGV_LOW(ret), TCGV_LOW(ret), 32 - len); 2728 } 2729 /* Sign-extend the field from 32 bits. */ 2730 tcg_gen_sari_i32(TCGV_HIGH(ret), TCGV_LOW(ret), 31); 2731 return; 2732 } 2733 2734 if (TCG_TARGET_sextract_valid(TCG_TYPE_I64, ofs, len)) { 2735 tcg_gen_op4ii_i64(INDEX_op_sextract_i64, ret, arg, ofs, len); 2736 return; 2737 } 2738 2739 /* Assume that sign-extension, if available, is cheaper than a shift. */ 2740 if (TCG_TARGET_sextract_valid(TCG_TYPE_I64, 0, ofs + len)) { 2741 tcg_gen_op4ii_i64(INDEX_op_sextract_i64, ret, arg, 0, ofs + len); 2742 tcg_gen_sari_i64(ret, ret, ofs); 2743 return; 2744 } 2745 if (TCG_TARGET_sextract_valid(TCG_TYPE_I64, 0, len)) { 2746 tcg_gen_shri_i64(ret, arg, ofs); 2747 tcg_gen_op4ii_i64(INDEX_op_sextract_i64, ret, ret, 0, len); 2748 return; 2749 } 2750 2751 tcg_gen_shli_i64(ret, arg, 64 - len - ofs); 2752 tcg_gen_sari_i64(ret, ret, 64 - len); 2753 } 2754 2755 /* 2756 * Extract 64 bits from a 128-bit input, ah:al, starting from ofs. 2757 * Unlike tcg_gen_extract_i64 above, len is fixed at 64. 2758 */ 2759 void tcg_gen_extract2_i64(TCGv_i64 ret, TCGv_i64 al, TCGv_i64 ah, 2760 unsigned int ofs) 2761 { 2762 tcg_debug_assert(ofs <= 64); 2763 if (ofs == 0) { 2764 tcg_gen_mov_i64(ret, al); 2765 } else if (ofs == 64) { 2766 tcg_gen_mov_i64(ret, ah); 2767 } else if (al == ah) { 2768 tcg_gen_rotri_i64(ret, al, ofs); 2769 } else if (TCG_TARGET_HAS_extract2_i64) { 2770 tcg_gen_op4i_i64(INDEX_op_extract2_i64, ret, al, ah, ofs); 2771 } else { 2772 TCGv_i64 t0 = tcg_temp_ebb_new_i64(); 2773 tcg_gen_shri_i64(t0, al, ofs); 2774 tcg_gen_deposit_i64(ret, t0, ah, 64 - ofs, ofs); 2775 tcg_temp_free_i64(t0); 2776 } 2777 } 2778 2779 void tcg_gen_movcond_i64(TCGCond cond, TCGv_i64 ret, TCGv_i64 c1, 2780 TCGv_i64 c2, TCGv_i64 v1, TCGv_i64 v2) 2781 { 2782 if (cond == TCG_COND_ALWAYS) { 2783 tcg_gen_mov_i64(ret, v1); 2784 } else if (cond == TCG_COND_NEVER) { 2785 tcg_gen_mov_i64(ret, v2); 2786 } else if (TCG_TARGET_REG_BITS == 64) { 2787 tcg_gen_op6i_i64(INDEX_op_movcond_i64, ret, c1, c2, v1, v2, cond); 2788 } else { 2789 TCGv_i32 t0 = tcg_temp_ebb_new_i32(); 2790 TCGv_i32 zero = tcg_constant_i32(0); 2791 2792 tcg_gen_op6i_i32(INDEX_op_setcond2_i32, t0, 2793 TCGV_LOW(c1), TCGV_HIGH(c1), 2794 TCGV_LOW(c2), TCGV_HIGH(c2), cond); 2795 2796 tcg_gen_movcond_i32(TCG_COND_NE, TCGV_LOW(ret), t0, zero, 2797 TCGV_LOW(v1), TCGV_LOW(v2)); 2798 tcg_gen_movcond_i32(TCG_COND_NE, TCGV_HIGH(ret), t0, zero, 2799 TCGV_HIGH(v1), TCGV_HIGH(v2)); 2800 2801 tcg_temp_free_i32(t0); 2802 } 2803 } 2804 2805 void tcg_gen_add2_i64(TCGv_i64 rl, TCGv_i64 rh, TCGv_i64 al, 2806 TCGv_i64 ah, TCGv_i64 bl, TCGv_i64 bh) 2807 { 2808 if (TCG_TARGET_HAS_add2_i64) { 2809 tcg_gen_op6_i64(INDEX_op_add2_i64, rl, rh, al, ah, bl, bh); 2810 } else { 2811 TCGv_i64 t0 = tcg_temp_ebb_new_i64(); 2812 TCGv_i64 t1 = tcg_temp_ebb_new_i64(); 2813 tcg_gen_add_i64(t0, al, bl); 2814 tcg_gen_setcond_i64(TCG_COND_LTU, t1, t0, al); 2815 tcg_gen_add_i64(rh, ah, bh); 2816 tcg_gen_add_i64(rh, rh, t1); 2817 tcg_gen_mov_i64(rl, t0); 2818 tcg_temp_free_i64(t0); 2819 tcg_temp_free_i64(t1); 2820 } 2821 } 2822 2823 void tcg_gen_sub2_i64(TCGv_i64 rl, TCGv_i64 rh, TCGv_i64 al, 2824 TCGv_i64 ah, TCGv_i64 bl, TCGv_i64 bh) 2825 { 2826 if (TCG_TARGET_HAS_sub2_i64) { 2827 tcg_gen_op6_i64(INDEX_op_sub2_i64, rl, rh, al, ah, bl, bh); 2828 } else { 2829 TCGv_i64 t0 = tcg_temp_ebb_new_i64(); 2830 TCGv_i64 t1 = tcg_temp_ebb_new_i64(); 2831 tcg_gen_sub_i64(t0, al, bl); 2832 tcg_gen_setcond_i64(TCG_COND_LTU, t1, al, bl); 2833 tcg_gen_sub_i64(rh, ah, bh); 2834 tcg_gen_sub_i64(rh, rh, t1); 2835 tcg_gen_mov_i64(rl, t0); 2836 tcg_temp_free_i64(t0); 2837 tcg_temp_free_i64(t1); 2838 } 2839 } 2840 2841 void tcg_gen_mulu2_i64(TCGv_i64 rl, TCGv_i64 rh, TCGv_i64 arg1, TCGv_i64 arg2) 2842 { 2843 if (TCG_TARGET_HAS_mulu2_i64) { 2844 tcg_gen_op4_i64(INDEX_op_mulu2_i64, rl, rh, arg1, arg2); 2845 } else if (tcg_op_supported(INDEX_op_muluh, TCG_TYPE_I64, 0)) { 2846 TCGv_i64 t = tcg_temp_ebb_new_i64(); 2847 tcg_gen_op3_i64(INDEX_op_mul, t, arg1, arg2); 2848 tcg_gen_op3_i64(INDEX_op_muluh, rh, arg1, arg2); 2849 tcg_gen_mov_i64(rl, t); 2850 tcg_temp_free_i64(t); 2851 } else { 2852 TCGv_i64 t0 = tcg_temp_ebb_new_i64(); 2853 tcg_gen_mul_i64(t0, arg1, arg2); 2854 gen_helper_muluh_i64(rh, arg1, arg2); 2855 tcg_gen_mov_i64(rl, t0); 2856 tcg_temp_free_i64(t0); 2857 } 2858 } 2859 2860 void tcg_gen_muls2_i64(TCGv_i64 rl, TCGv_i64 rh, TCGv_i64 arg1, TCGv_i64 arg2) 2861 { 2862 if (TCG_TARGET_HAS_muls2_i64) { 2863 tcg_gen_op4_i64(INDEX_op_muls2_i64, rl, rh, arg1, arg2); 2864 } else if (TCG_TARGET_HAS_mulsh_i64) { 2865 TCGv_i64 t = tcg_temp_ebb_new_i64(); 2866 tcg_gen_op3_i64(INDEX_op_mul, t, arg1, arg2); 2867 tcg_gen_op3_i64(INDEX_op_mulsh_i64, rh, arg1, arg2); 2868 tcg_gen_mov_i64(rl, t); 2869 tcg_temp_free_i64(t); 2870 } else if (TCG_TARGET_HAS_mulu2_i64 || 2871 tcg_op_supported(INDEX_op_muluh, TCG_TYPE_I64, 0)) { 2872 TCGv_i64 t0 = tcg_temp_ebb_new_i64(); 2873 TCGv_i64 t1 = tcg_temp_ebb_new_i64(); 2874 TCGv_i64 t2 = tcg_temp_ebb_new_i64(); 2875 TCGv_i64 t3 = tcg_temp_ebb_new_i64(); 2876 tcg_gen_mulu2_i64(t0, t1, arg1, arg2); 2877 /* Adjust for negative inputs. */ 2878 tcg_gen_sari_i64(t2, arg1, 63); 2879 tcg_gen_sari_i64(t3, arg2, 63); 2880 tcg_gen_and_i64(t2, t2, arg2); 2881 tcg_gen_and_i64(t3, t3, arg1); 2882 tcg_gen_sub_i64(rh, t1, t2); 2883 tcg_gen_sub_i64(rh, rh, t3); 2884 tcg_gen_mov_i64(rl, t0); 2885 tcg_temp_free_i64(t0); 2886 tcg_temp_free_i64(t1); 2887 tcg_temp_free_i64(t2); 2888 tcg_temp_free_i64(t3); 2889 } else { 2890 TCGv_i64 t0 = tcg_temp_ebb_new_i64(); 2891 tcg_gen_mul_i64(t0, arg1, arg2); 2892 gen_helper_mulsh_i64(rh, arg1, arg2); 2893 tcg_gen_mov_i64(rl, t0); 2894 tcg_temp_free_i64(t0); 2895 } 2896 } 2897 2898 void tcg_gen_mulsu2_i64(TCGv_i64 rl, TCGv_i64 rh, TCGv_i64 arg1, TCGv_i64 arg2) 2899 { 2900 TCGv_i64 t0 = tcg_temp_ebb_new_i64(); 2901 TCGv_i64 t1 = tcg_temp_ebb_new_i64(); 2902 TCGv_i64 t2 = tcg_temp_ebb_new_i64(); 2903 tcg_gen_mulu2_i64(t0, t1, arg1, arg2); 2904 /* Adjust for negative input for the signed arg1. */ 2905 tcg_gen_sari_i64(t2, arg1, 63); 2906 tcg_gen_and_i64(t2, t2, arg2); 2907 tcg_gen_sub_i64(rh, t1, t2); 2908 tcg_gen_mov_i64(rl, t0); 2909 tcg_temp_free_i64(t0); 2910 tcg_temp_free_i64(t1); 2911 tcg_temp_free_i64(t2); 2912 } 2913 2914 void tcg_gen_smin_i64(TCGv_i64 ret, TCGv_i64 a, TCGv_i64 b) 2915 { 2916 tcg_gen_movcond_i64(TCG_COND_LT, ret, a, b, a, b); 2917 } 2918 2919 void tcg_gen_umin_i64(TCGv_i64 ret, TCGv_i64 a, TCGv_i64 b) 2920 { 2921 tcg_gen_movcond_i64(TCG_COND_LTU, ret, a, b, a, b); 2922 } 2923 2924 void tcg_gen_smax_i64(TCGv_i64 ret, TCGv_i64 a, TCGv_i64 b) 2925 { 2926 tcg_gen_movcond_i64(TCG_COND_LT, ret, a, b, b, a); 2927 } 2928 2929 void tcg_gen_umax_i64(TCGv_i64 ret, TCGv_i64 a, TCGv_i64 b) 2930 { 2931 tcg_gen_movcond_i64(TCG_COND_LTU, ret, a, b, b, a); 2932 } 2933 2934 void tcg_gen_abs_i64(TCGv_i64 ret, TCGv_i64 a) 2935 { 2936 TCGv_i64 t = tcg_temp_ebb_new_i64(); 2937 2938 tcg_gen_sari_i64(t, a, 63); 2939 tcg_gen_xor_i64(ret, a, t); 2940 tcg_gen_sub_i64(ret, ret, t); 2941 tcg_temp_free_i64(t); 2942 } 2943 2944 /* Size changing operations. */ 2945 2946 void tcg_gen_extrl_i64_i32(TCGv_i32 ret, TCGv_i64 arg) 2947 { 2948 if (TCG_TARGET_REG_BITS == 32) { 2949 tcg_gen_mov_i32(ret, TCGV_LOW(arg)); 2950 } else if (TCG_TARGET_HAS_extr_i64_i32) { 2951 tcg_gen_op2(INDEX_op_extrl_i64_i32, TCG_TYPE_I32, 2952 tcgv_i32_arg(ret), tcgv_i64_arg(arg)); 2953 } else { 2954 tcg_gen_mov_i32(ret, (TCGv_i32)arg); 2955 } 2956 } 2957 2958 void tcg_gen_extrh_i64_i32(TCGv_i32 ret, TCGv_i64 arg) 2959 { 2960 if (TCG_TARGET_REG_BITS == 32) { 2961 tcg_gen_mov_i32(ret, TCGV_HIGH(arg)); 2962 } else if (TCG_TARGET_HAS_extr_i64_i32) { 2963 tcg_gen_op2(INDEX_op_extrh_i64_i32, TCG_TYPE_I32, 2964 tcgv_i32_arg(ret), tcgv_i64_arg(arg)); 2965 } else { 2966 TCGv_i64 t = tcg_temp_ebb_new_i64(); 2967 tcg_gen_shri_i64(t, arg, 32); 2968 tcg_gen_mov_i32(ret, (TCGv_i32)t); 2969 tcg_temp_free_i64(t); 2970 } 2971 } 2972 2973 void tcg_gen_extu_i32_i64(TCGv_i64 ret, TCGv_i32 arg) 2974 { 2975 if (TCG_TARGET_REG_BITS == 32) { 2976 tcg_gen_mov_i32(TCGV_LOW(ret), arg); 2977 tcg_gen_movi_i32(TCGV_HIGH(ret), 0); 2978 } else { 2979 tcg_gen_op2(INDEX_op_extu_i32_i64, TCG_TYPE_I64, 2980 tcgv_i64_arg(ret), tcgv_i32_arg(arg)); 2981 } 2982 } 2983 2984 void tcg_gen_ext_i32_i64(TCGv_i64 ret, TCGv_i32 arg) 2985 { 2986 if (TCG_TARGET_REG_BITS == 32) { 2987 tcg_gen_mov_i32(TCGV_LOW(ret), arg); 2988 tcg_gen_sari_i32(TCGV_HIGH(ret), TCGV_LOW(ret), 31); 2989 } else { 2990 tcg_gen_op2(INDEX_op_ext_i32_i64, TCG_TYPE_I64, 2991 tcgv_i64_arg(ret), tcgv_i32_arg(arg)); 2992 } 2993 } 2994 2995 void tcg_gen_concat_i32_i64(TCGv_i64 dest, TCGv_i32 low, TCGv_i32 high) 2996 { 2997 TCGv_i64 tmp; 2998 2999 if (TCG_TARGET_REG_BITS == 32) { 3000 tcg_gen_mov_i32(TCGV_LOW(dest), low); 3001 tcg_gen_mov_i32(TCGV_HIGH(dest), high); 3002 return; 3003 } 3004 3005 tmp = tcg_temp_ebb_new_i64(); 3006 /* These extensions are only needed for type correctness. 3007 We may be able to do better given target specific information. */ 3008 tcg_gen_extu_i32_i64(tmp, high); 3009 tcg_gen_extu_i32_i64(dest, low); 3010 /* If deposit is available, use it. Otherwise use the extra 3011 knowledge that we have of the zero-extensions above. */ 3012 if (TCG_TARGET_deposit_valid(TCG_TYPE_I64, 32, 32)) { 3013 tcg_gen_deposit_i64(dest, dest, tmp, 32, 32); 3014 } else { 3015 tcg_gen_shli_i64(tmp, tmp, 32); 3016 tcg_gen_or_i64(dest, dest, tmp); 3017 } 3018 tcg_temp_free_i64(tmp); 3019 } 3020 3021 void tcg_gen_extr_i64_i32(TCGv_i32 lo, TCGv_i32 hi, TCGv_i64 arg) 3022 { 3023 if (TCG_TARGET_REG_BITS == 32) { 3024 tcg_gen_mov_i32(lo, TCGV_LOW(arg)); 3025 tcg_gen_mov_i32(hi, TCGV_HIGH(arg)); 3026 } else { 3027 tcg_gen_extrl_i64_i32(lo, arg); 3028 tcg_gen_extrh_i64_i32(hi, arg); 3029 } 3030 } 3031 3032 void tcg_gen_extr32_i64(TCGv_i64 lo, TCGv_i64 hi, TCGv_i64 arg) 3033 { 3034 tcg_gen_ext32u_i64(lo, arg); 3035 tcg_gen_shri_i64(hi, arg, 32); 3036 } 3037 3038 void tcg_gen_concat32_i64(TCGv_i64 ret, TCGv_i64 lo, TCGv_i64 hi) 3039 { 3040 tcg_gen_deposit_i64(ret, lo, hi, 32, 32); 3041 } 3042 3043 void tcg_gen_extr_i128_i64(TCGv_i64 lo, TCGv_i64 hi, TCGv_i128 arg) 3044 { 3045 tcg_gen_mov_i64(lo, TCGV128_LOW(arg)); 3046 tcg_gen_mov_i64(hi, TCGV128_HIGH(arg)); 3047 } 3048 3049 void tcg_gen_concat_i64_i128(TCGv_i128 ret, TCGv_i64 lo, TCGv_i64 hi) 3050 { 3051 tcg_gen_mov_i64(TCGV128_LOW(ret), lo); 3052 tcg_gen_mov_i64(TCGV128_HIGH(ret), hi); 3053 } 3054 3055 void tcg_gen_mov_i128(TCGv_i128 dst, TCGv_i128 src) 3056 { 3057 if (dst != src) { 3058 tcg_gen_mov_i64(TCGV128_LOW(dst), TCGV128_LOW(src)); 3059 tcg_gen_mov_i64(TCGV128_HIGH(dst), TCGV128_HIGH(src)); 3060 } 3061 } 3062 3063 void tcg_gen_ld_i128(TCGv_i128 ret, TCGv_ptr base, tcg_target_long offset) 3064 { 3065 if (HOST_BIG_ENDIAN) { 3066 tcg_gen_ld_i64(TCGV128_HIGH(ret), base, offset); 3067 tcg_gen_ld_i64(TCGV128_LOW(ret), base, offset + 8); 3068 } else { 3069 tcg_gen_ld_i64(TCGV128_LOW(ret), base, offset); 3070 tcg_gen_ld_i64(TCGV128_HIGH(ret), base, offset + 8); 3071 } 3072 } 3073 3074 void tcg_gen_st_i128(TCGv_i128 val, TCGv_ptr base, tcg_target_long offset) 3075 { 3076 if (HOST_BIG_ENDIAN) { 3077 tcg_gen_st_i64(TCGV128_HIGH(val), base, offset); 3078 tcg_gen_st_i64(TCGV128_LOW(val), base, offset + 8); 3079 } else { 3080 tcg_gen_st_i64(TCGV128_LOW(val), base, offset); 3081 tcg_gen_st_i64(TCGV128_HIGH(val), base, offset + 8); 3082 } 3083 } 3084 3085 /* QEMU specific operations. */ 3086 3087 void tcg_gen_exit_tb(const TranslationBlock *tb, unsigned idx) 3088 { 3089 /* 3090 * Let the jit code return the read-only version of the 3091 * TranslationBlock, so that we minimize the pc-relative 3092 * distance of the address of the exit_tb code to TB. 3093 * This will improve utilization of pc-relative address loads. 3094 * 3095 * TODO: Move this to translator_loop, so that all const 3096 * TranslationBlock pointers refer to read-only memory. 3097 * This requires coordination with targets that do not use 3098 * the translator_loop. 3099 */ 3100 uintptr_t val = (uintptr_t)tcg_splitwx_to_rx((void *)tb) + idx; 3101 3102 if (tb == NULL) { 3103 tcg_debug_assert(idx == 0); 3104 } else if (idx <= TB_EXIT_IDXMAX) { 3105 #ifdef CONFIG_DEBUG_TCG 3106 /* This is an exit following a goto_tb. Verify that we have 3107 seen this numbered exit before, via tcg_gen_goto_tb. */ 3108 tcg_debug_assert(tcg_ctx->goto_tb_issue_mask & (1 << idx)); 3109 #endif 3110 } else { 3111 /* This is an exit via the exitreq label. */ 3112 tcg_debug_assert(idx == TB_EXIT_REQUESTED); 3113 } 3114 3115 tcg_gen_op1i(INDEX_op_exit_tb, 0, val); 3116 } 3117 3118 void tcg_gen_goto_tb(unsigned idx) 3119 { 3120 /* We tested CF_NO_GOTO_TB in translator_use_goto_tb. */ 3121 tcg_debug_assert(!(tcg_ctx->gen_tb->cflags & CF_NO_GOTO_TB)); 3122 /* We only support two chained exits. */ 3123 tcg_debug_assert(idx <= TB_EXIT_IDXMAX); 3124 #ifdef CONFIG_DEBUG_TCG 3125 /* Verify that we haven't seen this numbered exit before. */ 3126 tcg_debug_assert((tcg_ctx->goto_tb_issue_mask & (1 << idx)) == 0); 3127 tcg_ctx->goto_tb_issue_mask |= 1 << idx; 3128 #endif 3129 plugin_gen_disable_mem_helpers(); 3130 tcg_gen_op1i(INDEX_op_goto_tb, 0, idx); 3131 } 3132 3133 void tcg_gen_lookup_and_goto_ptr(void) 3134 { 3135 TCGv_ptr ptr; 3136 3137 if (tcg_ctx->gen_tb->cflags & CF_NO_GOTO_PTR) { 3138 tcg_gen_exit_tb(NULL, 0); 3139 return; 3140 } 3141 3142 plugin_gen_disable_mem_helpers(); 3143 ptr = tcg_temp_ebb_new_ptr(); 3144 gen_helper_lookup_tb_ptr(ptr, tcg_env); 3145 tcg_gen_op1i(INDEX_op_goto_ptr, TCG_TYPE_PTR, tcgv_ptr_arg(ptr)); 3146 tcg_temp_free_ptr(ptr); 3147 } 3148