1 /* 2 * Tiny Code Generator for QEMU 3 * 4 * Copyright (c) 2018 Linaro, Inc. 5 * 6 * This library is free software; you can redistribute it and/or 7 * modify it under the terms of the GNU Lesser General Public 8 * License as published by the Free Software Foundation; either 9 * version 2.1 of the License, or (at your option) any later version. 10 * 11 * This library is distributed in the hope that it will be useful, 12 * but WITHOUT ANY WARRANTY; without even the implied warranty of 13 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU 14 * Lesser General Public License for more details. 15 * 16 * You should have received a copy of the GNU Lesser General Public 17 * License along with this library; if not, see <http://www.gnu.org/licenses/>. 18 */ 19 20 #include "qemu/osdep.h" 21 #include "tcg/tcg.h" 22 #include "tcg/tcg-temp-internal.h" 23 #include "tcg/tcg-op-common.h" 24 #include "tcg/tcg-mo.h" 25 #include "tcg-internal.h" 26 27 /* 28 * Vector optional opcode tracking. 29 * Except for the basic logical operations (and, or, xor), and 30 * data movement (mov, ld, st, dupi), many vector opcodes are 31 * optional and may not be supported on the host. Thank Intel 32 * for the irregularity in their instruction set. 33 * 34 * The gvec expanders allow custom vector operations to be composed, 35 * generally via the .fniv callback in the GVecGen* structures. At 36 * the same time, in deciding whether to use this hook we need to 37 * know if the host supports the required operations. This is 38 * presented as an array of opcodes, terminated by 0. Each opcode 39 * is assumed to be expanded with the given VECE. 40 * 41 * For debugging, we want to validate this array. Therefore, when 42 * tcg_ctx->vec_opt_opc is non-NULL, the tcg_gen_*_vec expanders 43 * will validate that their opcode is present in the list. 44 */ 45 static void tcg_assert_listed_vecop(TCGOpcode op) 46 { 47 #ifdef CONFIG_DEBUG_TCG 48 const TCGOpcode *p = tcg_ctx->vecop_list; 49 if (p) { 50 for (; *p; ++p) { 51 if (*p == op) { 52 return; 53 } 54 } 55 g_assert_not_reached(); 56 } 57 #endif 58 } 59 60 bool tcg_can_emit_vecop_list(const TCGOpcode *list, 61 TCGType type, unsigned vece) 62 { 63 if (list == NULL) { 64 return true; 65 } 66 67 for (; *list; ++list) { 68 TCGOpcode opc = *list; 69 70 #ifdef CONFIG_DEBUG_TCG 71 switch (opc) { 72 case INDEX_op_and_vec: 73 case INDEX_op_or_vec: 74 case INDEX_op_xor_vec: 75 case INDEX_op_mov_vec: 76 case INDEX_op_dup_vec: 77 case INDEX_op_dup2_vec: 78 case INDEX_op_ld_vec: 79 case INDEX_op_st_vec: 80 case INDEX_op_bitsel_vec: 81 /* These opcodes are mandatory and should not be listed. */ 82 g_assert_not_reached(); 83 case INDEX_op_not_vec: 84 /* These opcodes have generic expansions using the above. */ 85 g_assert_not_reached(); 86 default: 87 break; 88 } 89 #endif 90 91 if (tcg_can_emit_vec_op(opc, type, vece)) { 92 continue; 93 } 94 95 /* 96 * The opcode list is created by front ends based on what they 97 * actually invoke. We must mirror the logic in the routines 98 * below for generic expansions using other opcodes. 99 */ 100 switch (opc) { 101 case INDEX_op_neg_vec: 102 if (tcg_can_emit_vec_op(INDEX_op_sub_vec, type, vece)) { 103 continue; 104 } 105 break; 106 case INDEX_op_abs_vec: 107 if (tcg_can_emit_vec_op(INDEX_op_sub_vec, type, vece) 108 && (tcg_can_emit_vec_op(INDEX_op_smax_vec, type, vece) > 0 109 || tcg_can_emit_vec_op(INDEX_op_sari_vec, type, vece) > 0 110 || tcg_can_emit_vec_op(INDEX_op_cmp_vec, type, vece))) { 111 continue; 112 } 113 break; 114 case INDEX_op_usadd_vec: 115 if (tcg_can_emit_vec_op(INDEX_op_umin_vec, type, vece) || 116 tcg_can_emit_vec_op(INDEX_op_cmp_vec, type, vece)) { 117 continue; 118 } 119 break; 120 case INDEX_op_ussub_vec: 121 if (tcg_can_emit_vec_op(INDEX_op_umax_vec, type, vece) || 122 tcg_can_emit_vec_op(INDEX_op_cmp_vec, type, vece)) { 123 continue; 124 } 125 break; 126 case INDEX_op_cmpsel_vec: 127 case INDEX_op_smin_vec: 128 case INDEX_op_smax_vec: 129 case INDEX_op_umin_vec: 130 case INDEX_op_umax_vec: 131 if (tcg_can_emit_vec_op(INDEX_op_cmp_vec, type, vece)) { 132 continue; 133 } 134 break; 135 default: 136 break; 137 } 138 return false; 139 } 140 return true; 141 } 142 143 void vec_gen_2(TCGOpcode opc, TCGType type, unsigned vece, TCGArg r, TCGArg a) 144 { 145 TCGOp *op = tcg_emit_op(opc, 2); 146 TCGOP_VECL(op) = type - TCG_TYPE_V64; 147 TCGOP_VECE(op) = vece; 148 op->args[0] = r; 149 op->args[1] = a; 150 } 151 152 void vec_gen_3(TCGOpcode opc, TCGType type, unsigned vece, 153 TCGArg r, TCGArg a, TCGArg b) 154 { 155 TCGOp *op = tcg_emit_op(opc, 3); 156 TCGOP_VECL(op) = type - TCG_TYPE_V64; 157 TCGOP_VECE(op) = vece; 158 op->args[0] = r; 159 op->args[1] = a; 160 op->args[2] = b; 161 } 162 163 void vec_gen_4(TCGOpcode opc, TCGType type, unsigned vece, 164 TCGArg r, TCGArg a, TCGArg b, TCGArg c) 165 { 166 TCGOp *op = tcg_emit_op(opc, 4); 167 TCGOP_VECL(op) = type - TCG_TYPE_V64; 168 TCGOP_VECE(op) = vece; 169 op->args[0] = r; 170 op->args[1] = a; 171 op->args[2] = b; 172 op->args[3] = c; 173 } 174 175 static void vec_gen_6(TCGOpcode opc, TCGType type, unsigned vece, TCGArg r, 176 TCGArg a, TCGArg b, TCGArg c, TCGArg d, TCGArg e) 177 { 178 TCGOp *op = tcg_emit_op(opc, 6); 179 TCGOP_VECL(op) = type - TCG_TYPE_V64; 180 TCGOP_VECE(op) = vece; 181 op->args[0] = r; 182 op->args[1] = a; 183 op->args[2] = b; 184 op->args[3] = c; 185 op->args[4] = d; 186 op->args[5] = e; 187 } 188 189 static void vec_gen_op2(TCGOpcode opc, unsigned vece, TCGv_vec r, TCGv_vec a) 190 { 191 TCGTemp *rt = tcgv_vec_temp(r); 192 TCGTemp *at = tcgv_vec_temp(a); 193 TCGType type = rt->base_type; 194 195 /* Must enough inputs for the output. */ 196 tcg_debug_assert(at->base_type >= type); 197 vec_gen_2(opc, type, vece, temp_arg(rt), temp_arg(at)); 198 } 199 200 static void vec_gen_op3(TCGOpcode opc, unsigned vece, 201 TCGv_vec r, TCGv_vec a, TCGv_vec b) 202 { 203 TCGTemp *rt = tcgv_vec_temp(r); 204 TCGTemp *at = tcgv_vec_temp(a); 205 TCGTemp *bt = tcgv_vec_temp(b); 206 TCGType type = rt->base_type; 207 208 /* Must enough inputs for the output. */ 209 tcg_debug_assert(at->base_type >= type); 210 tcg_debug_assert(bt->base_type >= type); 211 vec_gen_3(opc, type, vece, temp_arg(rt), temp_arg(at), temp_arg(bt)); 212 } 213 214 void tcg_gen_mov_vec(TCGv_vec r, TCGv_vec a) 215 { 216 if (r != a) { 217 vec_gen_op2(INDEX_op_mov_vec, 0, r, a); 218 } 219 } 220 221 void tcg_gen_dupi_vec(unsigned vece, TCGv_vec r, uint64_t a) 222 { 223 TCGTemp *rt = tcgv_vec_temp(r); 224 tcg_gen_mov_vec(r, tcg_constant_vec(rt->base_type, vece, a)); 225 } 226 227 void tcg_gen_dup_i64_vec(unsigned vece, TCGv_vec r, TCGv_i64 a) 228 { 229 TCGArg ri = tcgv_vec_arg(r); 230 TCGTemp *rt = arg_temp(ri); 231 TCGType type = rt->base_type; 232 233 if (TCG_TARGET_REG_BITS == 64) { 234 TCGArg ai = tcgv_i64_arg(a); 235 vec_gen_2(INDEX_op_dup_vec, type, vece, ri, ai); 236 } else if (vece == MO_64) { 237 TCGArg al = tcgv_i32_arg(TCGV_LOW(a)); 238 TCGArg ah = tcgv_i32_arg(TCGV_HIGH(a)); 239 vec_gen_3(INDEX_op_dup2_vec, type, MO_64, ri, al, ah); 240 } else { 241 TCGArg ai = tcgv_i32_arg(TCGV_LOW(a)); 242 vec_gen_2(INDEX_op_dup_vec, type, vece, ri, ai); 243 } 244 } 245 246 void tcg_gen_dup_i32_vec(unsigned vece, TCGv_vec r, TCGv_i32 a) 247 { 248 TCGArg ri = tcgv_vec_arg(r); 249 TCGArg ai = tcgv_i32_arg(a); 250 TCGTemp *rt = arg_temp(ri); 251 TCGType type = rt->base_type; 252 253 vec_gen_2(INDEX_op_dup_vec, type, vece, ri, ai); 254 } 255 256 void tcg_gen_dup_mem_vec(unsigned vece, TCGv_vec r, TCGv_ptr b, 257 tcg_target_long ofs) 258 { 259 TCGArg ri = tcgv_vec_arg(r); 260 TCGArg bi = tcgv_ptr_arg(b); 261 TCGTemp *rt = arg_temp(ri); 262 TCGType type = rt->base_type; 263 264 vec_gen_3(INDEX_op_dupm_vec, type, vece, ri, bi, ofs); 265 } 266 267 static void vec_gen_ldst(TCGOpcode opc, TCGv_vec r, TCGv_ptr b, TCGArg o) 268 { 269 TCGArg ri = tcgv_vec_arg(r); 270 TCGArg bi = tcgv_ptr_arg(b); 271 TCGTemp *rt = arg_temp(ri); 272 TCGType type = rt->base_type; 273 274 vec_gen_3(opc, type, 0, ri, bi, o); 275 } 276 277 void tcg_gen_ld_vec(TCGv_vec r, TCGv_ptr b, TCGArg o) 278 { 279 vec_gen_ldst(INDEX_op_ld_vec, r, b, o); 280 } 281 282 void tcg_gen_st_vec(TCGv_vec r, TCGv_ptr b, TCGArg o) 283 { 284 vec_gen_ldst(INDEX_op_st_vec, r, b, o); 285 } 286 287 void tcg_gen_stl_vec(TCGv_vec r, TCGv_ptr b, TCGArg o, TCGType low_type) 288 { 289 TCGArg ri = tcgv_vec_arg(r); 290 TCGArg bi = tcgv_ptr_arg(b); 291 TCGTemp *rt = arg_temp(ri); 292 TCGType type = rt->base_type; 293 294 tcg_debug_assert(low_type >= TCG_TYPE_V64); 295 tcg_debug_assert(low_type <= type); 296 vec_gen_3(INDEX_op_st_vec, low_type, 0, ri, bi, o); 297 } 298 299 void tcg_gen_and_vec(unsigned vece, TCGv_vec r, TCGv_vec a, TCGv_vec b) 300 { 301 vec_gen_op3(INDEX_op_and_vec, 0, r, a, b); 302 } 303 304 void tcg_gen_or_vec(unsigned vece, TCGv_vec r, TCGv_vec a, TCGv_vec b) 305 { 306 vec_gen_op3(INDEX_op_or_vec, 0, r, a, b); 307 } 308 309 void tcg_gen_xor_vec(unsigned vece, TCGv_vec r, TCGv_vec a, TCGv_vec b) 310 { 311 vec_gen_op3(INDEX_op_xor_vec, 0, r, a, b); 312 } 313 314 void tcg_gen_andc_vec(unsigned vece, TCGv_vec r, TCGv_vec a, TCGv_vec b) 315 { 316 if (TCG_TARGET_HAS_andc_vec) { 317 vec_gen_op3(INDEX_op_andc_vec, 0, r, a, b); 318 } else { 319 TCGv_vec t = tcg_temp_new_vec_matching(r); 320 tcg_gen_not_vec(0, t, b); 321 tcg_gen_and_vec(0, r, a, t); 322 tcg_temp_free_vec(t); 323 } 324 } 325 326 void tcg_gen_orc_vec(unsigned vece, TCGv_vec r, TCGv_vec a, TCGv_vec b) 327 { 328 if (TCG_TARGET_HAS_orc_vec) { 329 vec_gen_op3(INDEX_op_orc_vec, 0, r, a, b); 330 } else { 331 TCGv_vec t = tcg_temp_new_vec_matching(r); 332 tcg_gen_not_vec(0, t, b); 333 tcg_gen_or_vec(0, r, a, t); 334 tcg_temp_free_vec(t); 335 } 336 } 337 338 void tcg_gen_nand_vec(unsigned vece, TCGv_vec r, TCGv_vec a, TCGv_vec b) 339 { 340 if (TCG_TARGET_HAS_nand_vec) { 341 vec_gen_op3(INDEX_op_nand_vec, 0, r, a, b); 342 } else { 343 tcg_gen_and_vec(0, r, a, b); 344 tcg_gen_not_vec(0, r, r); 345 } 346 } 347 348 void tcg_gen_nor_vec(unsigned vece, TCGv_vec r, TCGv_vec a, TCGv_vec b) 349 { 350 if (TCG_TARGET_HAS_nor_vec) { 351 vec_gen_op3(INDEX_op_nor_vec, 0, r, a, b); 352 } else { 353 tcg_gen_or_vec(0, r, a, b); 354 tcg_gen_not_vec(0, r, r); 355 } 356 } 357 358 void tcg_gen_eqv_vec(unsigned vece, TCGv_vec r, TCGv_vec a, TCGv_vec b) 359 { 360 if (TCG_TARGET_HAS_eqv_vec) { 361 vec_gen_op3(INDEX_op_eqv_vec, 0, r, a, b); 362 } else { 363 tcg_gen_xor_vec(0, r, a, b); 364 tcg_gen_not_vec(0, r, r); 365 } 366 } 367 368 static bool do_op2(unsigned vece, TCGv_vec r, TCGv_vec a, TCGOpcode opc) 369 { 370 TCGTemp *rt = tcgv_vec_temp(r); 371 TCGTemp *at = tcgv_vec_temp(a); 372 TCGArg ri = temp_arg(rt); 373 TCGArg ai = temp_arg(at); 374 TCGType type = rt->base_type; 375 int can; 376 377 tcg_debug_assert(at->base_type >= type); 378 tcg_assert_listed_vecop(opc); 379 can = tcg_can_emit_vec_op(opc, type, vece); 380 if (can > 0) { 381 vec_gen_2(opc, type, vece, ri, ai); 382 } else if (can < 0) { 383 const TCGOpcode *hold_list = tcg_swap_vecop_list(NULL); 384 tcg_expand_vec_op(opc, type, vece, ri, ai); 385 tcg_swap_vecop_list(hold_list); 386 } else { 387 return false; 388 } 389 return true; 390 } 391 392 void tcg_gen_not_vec(unsigned vece, TCGv_vec r, TCGv_vec a) 393 { 394 if (TCG_TARGET_HAS_not_vec) { 395 vec_gen_op2(INDEX_op_not_vec, 0, r, a); 396 } else { 397 tcg_gen_xor_vec(0, r, a, tcg_constant_vec_matching(r, 0, -1)); 398 } 399 } 400 401 void tcg_gen_neg_vec(unsigned vece, TCGv_vec r, TCGv_vec a) 402 { 403 const TCGOpcode *hold_list; 404 405 tcg_assert_listed_vecop(INDEX_op_neg_vec); 406 hold_list = tcg_swap_vecop_list(NULL); 407 408 if (!TCG_TARGET_HAS_neg_vec || !do_op2(vece, r, a, INDEX_op_neg_vec)) { 409 tcg_gen_sub_vec(vece, r, tcg_constant_vec_matching(r, vece, 0), a); 410 } 411 tcg_swap_vecop_list(hold_list); 412 } 413 414 void tcg_gen_abs_vec(unsigned vece, TCGv_vec r, TCGv_vec a) 415 { 416 const TCGOpcode *hold_list; 417 418 tcg_assert_listed_vecop(INDEX_op_abs_vec); 419 hold_list = tcg_swap_vecop_list(NULL); 420 421 if (!do_op2(vece, r, a, INDEX_op_abs_vec)) { 422 TCGType type = tcgv_vec_temp(r)->base_type; 423 TCGv_vec t = tcg_temp_new_vec(type); 424 425 tcg_debug_assert(tcg_can_emit_vec_op(INDEX_op_sub_vec, type, vece)); 426 if (tcg_can_emit_vec_op(INDEX_op_smax_vec, type, vece) > 0) { 427 tcg_gen_neg_vec(vece, t, a); 428 tcg_gen_smax_vec(vece, r, a, t); 429 } else { 430 if (tcg_can_emit_vec_op(INDEX_op_sari_vec, type, vece) > 0) { 431 tcg_gen_sari_vec(vece, t, a, (8 << vece) - 1); 432 } else { 433 tcg_gen_cmp_vec(TCG_COND_LT, vece, t, a, 434 tcg_constant_vec(type, vece, 0)); 435 } 436 tcg_gen_xor_vec(vece, r, a, t); 437 tcg_gen_sub_vec(vece, r, r, t); 438 } 439 440 tcg_temp_free_vec(t); 441 } 442 tcg_swap_vecop_list(hold_list); 443 } 444 445 static void do_shifti(TCGOpcode opc, unsigned vece, 446 TCGv_vec r, TCGv_vec a, int64_t i) 447 { 448 TCGTemp *rt = tcgv_vec_temp(r); 449 TCGTemp *at = tcgv_vec_temp(a); 450 TCGArg ri = temp_arg(rt); 451 TCGArg ai = temp_arg(at); 452 TCGType type = rt->base_type; 453 int can; 454 455 tcg_debug_assert(at->base_type == type); 456 tcg_debug_assert(i >= 0 && i < (8 << vece)); 457 tcg_assert_listed_vecop(opc); 458 459 if (i == 0) { 460 tcg_gen_mov_vec(r, a); 461 return; 462 } 463 464 can = tcg_can_emit_vec_op(opc, type, vece); 465 if (can > 0) { 466 vec_gen_3(opc, type, vece, ri, ai, i); 467 } else { 468 /* We leave the choice of expansion via scalar or vector shift 469 to the target. Often, but not always, dupi can feed a vector 470 shift easier than a scalar. */ 471 const TCGOpcode *hold_list = tcg_swap_vecop_list(NULL); 472 tcg_debug_assert(can < 0); 473 tcg_expand_vec_op(opc, type, vece, ri, ai, i); 474 tcg_swap_vecop_list(hold_list); 475 } 476 } 477 478 void tcg_gen_shli_vec(unsigned vece, TCGv_vec r, TCGv_vec a, int64_t i) 479 { 480 do_shifti(INDEX_op_shli_vec, vece, r, a, i); 481 } 482 483 void tcg_gen_shri_vec(unsigned vece, TCGv_vec r, TCGv_vec a, int64_t i) 484 { 485 do_shifti(INDEX_op_shri_vec, vece, r, a, i); 486 } 487 488 void tcg_gen_sari_vec(unsigned vece, TCGv_vec r, TCGv_vec a, int64_t i) 489 { 490 do_shifti(INDEX_op_sari_vec, vece, r, a, i); 491 } 492 493 void tcg_gen_rotli_vec(unsigned vece, TCGv_vec r, TCGv_vec a, int64_t i) 494 { 495 do_shifti(INDEX_op_rotli_vec, vece, r, a, i); 496 } 497 498 void tcg_gen_rotri_vec(unsigned vece, TCGv_vec r, TCGv_vec a, int64_t i) 499 { 500 int bits = 8 << vece; 501 tcg_debug_assert(i >= 0 && i < bits); 502 do_shifti(INDEX_op_rotli_vec, vece, r, a, -i & (bits - 1)); 503 } 504 505 void tcg_gen_cmp_vec(TCGCond cond, unsigned vece, 506 TCGv_vec r, TCGv_vec a, TCGv_vec b) 507 { 508 TCGTemp *rt = tcgv_vec_temp(r); 509 TCGTemp *at = tcgv_vec_temp(a); 510 TCGTemp *bt = tcgv_vec_temp(b); 511 TCGTemp *tt = NULL; 512 TCGArg ri = temp_arg(rt); 513 TCGArg ai = temp_arg(at); 514 TCGArg bi = temp_arg(bt); 515 TCGArg ti; 516 TCGType type = rt->base_type; 517 int can; 518 519 tcg_debug_assert(at->base_type >= type); 520 tcg_debug_assert(bt->base_type >= type); 521 tcg_assert_listed_vecop(INDEX_op_cmp_vec); 522 can = tcg_can_emit_vec_op(INDEX_op_cmp_vec, type, vece); 523 524 if (!TCG_TARGET_HAS_tst_vec && is_tst_cond(cond)) { 525 tt = tcg_temp_new_internal(type, TEMP_EBB); 526 ti = temp_arg(tt); 527 vec_gen_3(INDEX_op_and_vec, type, 0, ti, ai, bi); 528 at = tt; 529 ai = ti; 530 bt = tcg_constant_internal(type, 0); 531 bi = temp_arg(bt); 532 cond = tcg_tst_eqne_cond(cond); 533 } 534 535 if (can > 0) { 536 vec_gen_4(INDEX_op_cmp_vec, type, vece, ri, ai, bi, cond); 537 } else { 538 const TCGOpcode *hold_list = tcg_swap_vecop_list(NULL); 539 tcg_debug_assert(can < 0); 540 tcg_expand_vec_op(INDEX_op_cmp_vec, type, vece, ri, ai, bi, cond); 541 tcg_swap_vecop_list(hold_list); 542 } 543 544 if (tt) { 545 tcg_temp_free_internal(tt); 546 } 547 } 548 549 static bool do_op3(unsigned vece, TCGv_vec r, TCGv_vec a, 550 TCGv_vec b, TCGOpcode opc) 551 { 552 TCGTemp *rt = tcgv_vec_temp(r); 553 TCGTemp *at = tcgv_vec_temp(a); 554 TCGTemp *bt = tcgv_vec_temp(b); 555 TCGArg ri = temp_arg(rt); 556 TCGArg ai = temp_arg(at); 557 TCGArg bi = temp_arg(bt); 558 TCGType type = rt->base_type; 559 int can; 560 561 tcg_debug_assert(at->base_type >= type); 562 tcg_debug_assert(bt->base_type >= type); 563 tcg_assert_listed_vecop(opc); 564 can = tcg_can_emit_vec_op(opc, type, vece); 565 if (can > 0) { 566 vec_gen_3(opc, type, vece, ri, ai, bi); 567 } else if (can < 0) { 568 const TCGOpcode *hold_list = tcg_swap_vecop_list(NULL); 569 tcg_expand_vec_op(opc, type, vece, ri, ai, bi); 570 tcg_swap_vecop_list(hold_list); 571 } else { 572 return false; 573 } 574 return true; 575 } 576 577 static void do_op3_nofail(unsigned vece, TCGv_vec r, TCGv_vec a, 578 TCGv_vec b, TCGOpcode opc) 579 { 580 bool ok = do_op3(vece, r, a, b, opc); 581 tcg_debug_assert(ok); 582 } 583 584 void tcg_gen_add_vec(unsigned vece, TCGv_vec r, TCGv_vec a, TCGv_vec b) 585 { 586 do_op3_nofail(vece, r, a, b, INDEX_op_add_vec); 587 } 588 589 void tcg_gen_sub_vec(unsigned vece, TCGv_vec r, TCGv_vec a, TCGv_vec b) 590 { 591 do_op3_nofail(vece, r, a, b, INDEX_op_sub_vec); 592 } 593 594 void tcg_gen_mul_vec(unsigned vece, TCGv_vec r, TCGv_vec a, TCGv_vec b) 595 { 596 do_op3_nofail(vece, r, a, b, INDEX_op_mul_vec); 597 } 598 599 void tcg_gen_ssadd_vec(unsigned vece, TCGv_vec r, TCGv_vec a, TCGv_vec b) 600 { 601 do_op3_nofail(vece, r, a, b, INDEX_op_ssadd_vec); 602 } 603 604 void tcg_gen_usadd_vec(unsigned vece, TCGv_vec r, TCGv_vec a, TCGv_vec b) 605 { 606 if (!do_op3(vece, r, a, b, INDEX_op_usadd_vec)) { 607 const TCGOpcode *hold_list = tcg_swap_vecop_list(NULL); 608 TCGv_vec t = tcg_temp_new_vec_matching(r); 609 610 /* usadd(a, b) = min(a, ~b) + b */ 611 tcg_gen_not_vec(vece, t, b); 612 tcg_gen_umin_vec(vece, t, t, a); 613 tcg_gen_add_vec(vece, r, t, b); 614 615 tcg_temp_free_vec(t); 616 tcg_swap_vecop_list(hold_list); 617 } 618 } 619 620 void tcg_gen_sssub_vec(unsigned vece, TCGv_vec r, TCGv_vec a, TCGv_vec b) 621 { 622 do_op3_nofail(vece, r, a, b, INDEX_op_sssub_vec); 623 } 624 625 void tcg_gen_ussub_vec(unsigned vece, TCGv_vec r, TCGv_vec a, TCGv_vec b) 626 { 627 if (!do_op3(vece, r, a, b, INDEX_op_ussub_vec)) { 628 const TCGOpcode *hold_list = tcg_swap_vecop_list(NULL); 629 TCGv_vec t = tcg_temp_new_vec_matching(r); 630 631 /* ussub(a, b) = max(a, b) - b */ 632 tcg_gen_umax_vec(vece, t, a, b); 633 tcg_gen_sub_vec(vece, r, t, b); 634 635 tcg_temp_free_vec(t); 636 tcg_swap_vecop_list(hold_list); 637 } 638 } 639 640 static void do_minmax(unsigned vece, TCGv_vec r, TCGv_vec a, 641 TCGv_vec b, TCGOpcode opc, TCGCond cond) 642 { 643 if (!do_op3(vece, r, a, b, opc)) { 644 const TCGOpcode *hold_list = tcg_swap_vecop_list(NULL); 645 tcg_gen_cmpsel_vec(cond, vece, r, a, b, a, b); 646 tcg_swap_vecop_list(hold_list); 647 } 648 } 649 650 void tcg_gen_smin_vec(unsigned vece, TCGv_vec r, TCGv_vec a, TCGv_vec b) 651 { 652 do_minmax(vece, r, a, b, INDEX_op_smin_vec, TCG_COND_LT); 653 } 654 655 void tcg_gen_umin_vec(unsigned vece, TCGv_vec r, TCGv_vec a, TCGv_vec b) 656 { 657 do_minmax(vece, r, a, b, INDEX_op_umin_vec, TCG_COND_LTU); 658 } 659 660 void tcg_gen_smax_vec(unsigned vece, TCGv_vec r, TCGv_vec a, TCGv_vec b) 661 { 662 do_minmax(vece, r, a, b, INDEX_op_smax_vec, TCG_COND_GT); 663 } 664 665 void tcg_gen_umax_vec(unsigned vece, TCGv_vec r, TCGv_vec a, TCGv_vec b) 666 { 667 do_minmax(vece, r, a, b, INDEX_op_umax_vec, TCG_COND_GTU); 668 } 669 670 void tcg_gen_shlv_vec(unsigned vece, TCGv_vec r, TCGv_vec a, TCGv_vec b) 671 { 672 do_op3_nofail(vece, r, a, b, INDEX_op_shlv_vec); 673 } 674 675 void tcg_gen_shrv_vec(unsigned vece, TCGv_vec r, TCGv_vec a, TCGv_vec b) 676 { 677 do_op3_nofail(vece, r, a, b, INDEX_op_shrv_vec); 678 } 679 680 void tcg_gen_sarv_vec(unsigned vece, TCGv_vec r, TCGv_vec a, TCGv_vec b) 681 { 682 do_op3_nofail(vece, r, a, b, INDEX_op_sarv_vec); 683 } 684 685 void tcg_gen_rotlv_vec(unsigned vece, TCGv_vec r, TCGv_vec a, TCGv_vec b) 686 { 687 do_op3_nofail(vece, r, a, b, INDEX_op_rotlv_vec); 688 } 689 690 void tcg_gen_rotrv_vec(unsigned vece, TCGv_vec r, TCGv_vec a, TCGv_vec b) 691 { 692 do_op3_nofail(vece, r, a, b, INDEX_op_rotrv_vec); 693 } 694 695 static void do_shifts(unsigned vece, TCGv_vec r, TCGv_vec a, 696 TCGv_i32 s, TCGOpcode opc) 697 { 698 TCGTemp *rt = tcgv_vec_temp(r); 699 TCGTemp *at = tcgv_vec_temp(a); 700 TCGTemp *st = tcgv_i32_temp(s); 701 TCGArg ri = temp_arg(rt); 702 TCGArg ai = temp_arg(at); 703 TCGArg si = temp_arg(st); 704 TCGType type = rt->base_type; 705 int can; 706 707 tcg_debug_assert(at->base_type >= type); 708 tcg_assert_listed_vecop(opc); 709 can = tcg_can_emit_vec_op(opc, type, vece); 710 if (can > 0) { 711 vec_gen_3(opc, type, vece, ri, ai, si); 712 } else if (can < 0) { 713 const TCGOpcode *hold_list = tcg_swap_vecop_list(NULL); 714 tcg_expand_vec_op(opc, type, vece, ri, ai, si); 715 tcg_swap_vecop_list(hold_list); 716 } else { 717 g_assert_not_reached(); 718 } 719 } 720 721 void tcg_gen_shls_vec(unsigned vece, TCGv_vec r, TCGv_vec a, TCGv_i32 b) 722 { 723 do_shifts(vece, r, a, b, INDEX_op_shls_vec); 724 } 725 726 void tcg_gen_shrs_vec(unsigned vece, TCGv_vec r, TCGv_vec a, TCGv_i32 b) 727 { 728 do_shifts(vece, r, a, b, INDEX_op_shrs_vec); 729 } 730 731 void tcg_gen_sars_vec(unsigned vece, TCGv_vec r, TCGv_vec a, TCGv_i32 b) 732 { 733 do_shifts(vece, r, a, b, INDEX_op_sars_vec); 734 } 735 736 void tcg_gen_rotls_vec(unsigned vece, TCGv_vec r, TCGv_vec a, TCGv_i32 s) 737 { 738 do_shifts(vece, r, a, s, INDEX_op_rotls_vec); 739 } 740 741 void tcg_gen_bitsel_vec(unsigned vece, TCGv_vec r, TCGv_vec a, 742 TCGv_vec b, TCGv_vec c) 743 { 744 TCGTemp *rt = tcgv_vec_temp(r); 745 TCGTemp *at = tcgv_vec_temp(a); 746 TCGTemp *bt = tcgv_vec_temp(b); 747 TCGTemp *ct = tcgv_vec_temp(c); 748 TCGType type = rt->base_type; 749 750 tcg_debug_assert(at->base_type >= type); 751 tcg_debug_assert(bt->base_type >= type); 752 tcg_debug_assert(ct->base_type >= type); 753 754 if (TCG_TARGET_HAS_bitsel_vec) { 755 vec_gen_4(INDEX_op_bitsel_vec, type, MO_8, 756 temp_arg(rt), temp_arg(at), temp_arg(bt), temp_arg(ct)); 757 } else { 758 TCGv_vec t = tcg_temp_new_vec(type); 759 tcg_gen_and_vec(MO_8, t, a, b); 760 tcg_gen_andc_vec(MO_8, r, c, a); 761 tcg_gen_or_vec(MO_8, r, r, t); 762 tcg_temp_free_vec(t); 763 } 764 } 765 766 void tcg_gen_cmpsel_vec(TCGCond cond, unsigned vece, TCGv_vec r, 767 TCGv_vec a, TCGv_vec b, TCGv_vec c, TCGv_vec d) 768 { 769 TCGTemp *rt = tcgv_vec_temp(r); 770 TCGTemp *at = tcgv_vec_temp(a); 771 TCGTemp *bt = tcgv_vec_temp(b); 772 TCGTemp *ct = tcgv_vec_temp(c); 773 TCGTemp *dt = tcgv_vec_temp(d); 774 TCGArg ri = temp_arg(rt); 775 TCGArg ai = temp_arg(at); 776 TCGArg bi = temp_arg(bt); 777 TCGArg ci = temp_arg(ct); 778 TCGArg di = temp_arg(dt); 779 TCGType type = rt->base_type; 780 const TCGOpcode *hold_list; 781 int can; 782 783 tcg_debug_assert(at->base_type >= type); 784 tcg_debug_assert(bt->base_type >= type); 785 tcg_debug_assert(ct->base_type >= type); 786 tcg_debug_assert(dt->base_type >= type); 787 788 tcg_assert_listed_vecop(INDEX_op_cmpsel_vec); 789 hold_list = tcg_swap_vecop_list(NULL); 790 can = tcg_can_emit_vec_op(INDEX_op_cmpsel_vec, type, vece); 791 792 if (can > 0) { 793 vec_gen_6(INDEX_op_cmpsel_vec, type, vece, ri, ai, bi, ci, di, cond); 794 } else if (can < 0) { 795 tcg_expand_vec_op(INDEX_op_cmpsel_vec, type, vece, 796 ri, ai, bi, ci, di, cond); 797 } else { 798 TCGv_vec t = tcg_temp_new_vec(type); 799 tcg_gen_cmp_vec(cond, vece, t, a, b); 800 tcg_gen_bitsel_vec(vece, r, t, c, d); 801 tcg_temp_free_vec(t); 802 } 803 tcg_swap_vecop_list(hold_list); 804 } 805