1 /* 2 * Tiny Code Generator for QEMU 3 * 4 * Copyright (c) 2018 Linaro, Inc. 5 * 6 * This library is free software; you can redistribute it and/or 7 * modify it under the terms of the GNU Lesser General Public 8 * License as published by the Free Software Foundation; either 9 * version 2.1 of the License, or (at your option) any later version. 10 * 11 * This library is distributed in the hope that it will be useful, 12 * but WITHOUT ANY WARRANTY; without even the implied warranty of 13 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU 14 * Lesser General Public License for more details. 15 * 16 * You should have received a copy of the GNU Lesser General Public 17 * License along with this library; if not, see <http://www.gnu.org/licenses/>. 18 */ 19 20 #include "qemu/osdep.h" 21 #include "tcg/tcg.h" 22 #include "tcg/tcg-temp-internal.h" 23 #include "tcg/tcg-op-common.h" 24 #include "tcg/tcg-mo.h" 25 #include "tcg-internal.h" 26 27 /* 28 * Vector optional opcode tracking. 29 * Except for the basic logical operations (and, or, xor), and 30 * data movement (mov, ld, st, dupi), many vector opcodes are 31 * optional and may not be supported on the host. Thank Intel 32 * for the irregularity in their instruction set. 33 * 34 * The gvec expanders allow custom vector operations to be composed, 35 * generally via the .fniv callback in the GVecGen* structures. At 36 * the same time, in deciding whether to use this hook we need to 37 * know if the host supports the required operations. This is 38 * presented as an array of opcodes, terminated by 0. Each opcode 39 * is assumed to be expanded with the given VECE. 40 * 41 * For debugging, we want to validate this array. Therefore, when 42 * tcg_ctx->vec_opt_opc is non-NULL, the tcg_gen_*_vec expanders 43 * will validate that their opcode is present in the list. 44 */ 45 static void tcg_assert_listed_vecop(TCGOpcode op) 46 { 47 #ifdef CONFIG_DEBUG_TCG 48 const TCGOpcode *p = tcg_ctx->vecop_list; 49 if (p) { 50 for (; *p; ++p) { 51 if (*p == op) { 52 return; 53 } 54 } 55 g_assert_not_reached(); 56 } 57 #endif 58 } 59 60 bool tcg_can_emit_vecop_list(const TCGOpcode *list, 61 TCGType type, unsigned vece) 62 { 63 if (list == NULL) { 64 return true; 65 } 66 67 for (; *list; ++list) { 68 TCGOpcode opc = *list; 69 70 #ifdef CONFIG_DEBUG_TCG 71 switch (opc) { 72 case INDEX_op_and_vec: 73 case INDEX_op_or_vec: 74 case INDEX_op_xor_vec: 75 case INDEX_op_mov_vec: 76 case INDEX_op_dup_vec: 77 case INDEX_op_dup2_vec: 78 case INDEX_op_ld_vec: 79 case INDEX_op_st_vec: 80 case INDEX_op_bitsel_vec: 81 /* These opcodes are mandatory and should not be listed. */ 82 g_assert_not_reached(); 83 case INDEX_op_not_vec: 84 /* These opcodes have generic expansions using the above. */ 85 g_assert_not_reached(); 86 default: 87 break; 88 } 89 #endif 90 91 if (tcg_can_emit_vec_op(opc, type, vece)) { 92 continue; 93 } 94 95 /* 96 * The opcode list is created by front ends based on what they 97 * actually invoke. We must mirror the logic in the routines 98 * below for generic expansions using other opcodes. 99 */ 100 switch (opc) { 101 case INDEX_op_neg_vec: 102 if (tcg_can_emit_vec_op(INDEX_op_sub_vec, type, vece)) { 103 continue; 104 } 105 break; 106 case INDEX_op_abs_vec: 107 if (tcg_can_emit_vec_op(INDEX_op_sub_vec, type, vece) 108 && (tcg_can_emit_vec_op(INDEX_op_smax_vec, type, vece) > 0 109 || tcg_can_emit_vec_op(INDEX_op_sari_vec, type, vece) > 0 110 || tcg_can_emit_vec_op(INDEX_op_cmp_vec, type, vece))) { 111 continue; 112 } 113 break; 114 case INDEX_op_usadd_vec: 115 if (tcg_can_emit_vec_op(INDEX_op_umin_vec, type, vece) || 116 tcg_can_emit_vec_op(INDEX_op_cmp_vec, type, vece)) { 117 continue; 118 } 119 break; 120 case INDEX_op_ussub_vec: 121 if (tcg_can_emit_vec_op(INDEX_op_umax_vec, type, vece) || 122 tcg_can_emit_vec_op(INDEX_op_cmp_vec, type, vece)) { 123 continue; 124 } 125 break; 126 case INDEX_op_cmpsel_vec: 127 case INDEX_op_smin_vec: 128 case INDEX_op_smax_vec: 129 case INDEX_op_umin_vec: 130 case INDEX_op_umax_vec: 131 if (tcg_can_emit_vec_op(INDEX_op_cmp_vec, type, vece)) { 132 continue; 133 } 134 break; 135 default: 136 break; 137 } 138 return false; 139 } 140 return true; 141 } 142 143 void vec_gen_2(TCGOpcode opc, TCGType type, unsigned vece, TCGArg r, TCGArg a) 144 { 145 TCGOp *op = tcg_emit_op(opc, 2); 146 TCGOP_VECL(op) = type - TCG_TYPE_V64; 147 TCGOP_VECE(op) = vece; 148 op->args[0] = r; 149 op->args[1] = a; 150 } 151 152 void vec_gen_3(TCGOpcode opc, TCGType type, unsigned vece, 153 TCGArg r, TCGArg a, TCGArg b) 154 { 155 TCGOp *op = tcg_emit_op(opc, 3); 156 TCGOP_VECL(op) = type - TCG_TYPE_V64; 157 TCGOP_VECE(op) = vece; 158 op->args[0] = r; 159 op->args[1] = a; 160 op->args[2] = b; 161 } 162 163 void vec_gen_4(TCGOpcode opc, TCGType type, unsigned vece, 164 TCGArg r, TCGArg a, TCGArg b, TCGArg c) 165 { 166 TCGOp *op = tcg_emit_op(opc, 4); 167 TCGOP_VECL(op) = type - TCG_TYPE_V64; 168 TCGOP_VECE(op) = vece; 169 op->args[0] = r; 170 op->args[1] = a; 171 op->args[2] = b; 172 op->args[3] = c; 173 } 174 175 static void vec_gen_6(TCGOpcode opc, TCGType type, unsigned vece, TCGArg r, 176 TCGArg a, TCGArg b, TCGArg c, TCGArg d, TCGArg e) 177 { 178 TCGOp *op = tcg_emit_op(opc, 6); 179 TCGOP_VECL(op) = type - TCG_TYPE_V64; 180 TCGOP_VECE(op) = vece; 181 op->args[0] = r; 182 op->args[1] = a; 183 op->args[2] = b; 184 op->args[3] = c; 185 op->args[4] = d; 186 op->args[5] = e; 187 } 188 189 static void vec_gen_op2(TCGOpcode opc, unsigned vece, TCGv_vec r, TCGv_vec a) 190 { 191 TCGTemp *rt = tcgv_vec_temp(r); 192 TCGTemp *at = tcgv_vec_temp(a); 193 TCGType type = rt->base_type; 194 195 /* Must enough inputs for the output. */ 196 tcg_debug_assert(at->base_type >= type); 197 vec_gen_2(opc, type, vece, temp_arg(rt), temp_arg(at)); 198 } 199 200 static void vec_gen_op3(TCGOpcode opc, unsigned vece, 201 TCGv_vec r, TCGv_vec a, TCGv_vec b) 202 { 203 TCGTemp *rt = tcgv_vec_temp(r); 204 TCGTemp *at = tcgv_vec_temp(a); 205 TCGTemp *bt = tcgv_vec_temp(b); 206 TCGType type = rt->base_type; 207 208 /* Must enough inputs for the output. */ 209 tcg_debug_assert(at->base_type >= type); 210 tcg_debug_assert(bt->base_type >= type); 211 vec_gen_3(opc, type, vece, temp_arg(rt), temp_arg(at), temp_arg(bt)); 212 } 213 214 void tcg_gen_mov_vec(TCGv_vec r, TCGv_vec a) 215 { 216 if (r != a) { 217 vec_gen_op2(INDEX_op_mov_vec, 0, r, a); 218 } 219 } 220 221 void tcg_gen_dupi_vec(unsigned vece, TCGv_vec r, uint64_t a) 222 { 223 TCGTemp *rt = tcgv_vec_temp(r); 224 tcg_gen_mov_vec(r, tcg_constant_vec(rt->base_type, vece, a)); 225 } 226 227 void tcg_gen_dup_i64_vec(unsigned vece, TCGv_vec r, TCGv_i64 a) 228 { 229 TCGArg ri = tcgv_vec_arg(r); 230 TCGTemp *rt = arg_temp(ri); 231 TCGType type = rt->base_type; 232 233 if (TCG_TARGET_REG_BITS == 64) { 234 TCGArg ai = tcgv_i64_arg(a); 235 vec_gen_2(INDEX_op_dup_vec, type, vece, ri, ai); 236 } else if (vece == MO_64) { 237 TCGArg al = tcgv_i32_arg(TCGV_LOW(a)); 238 TCGArg ah = tcgv_i32_arg(TCGV_HIGH(a)); 239 vec_gen_3(INDEX_op_dup2_vec, type, MO_64, ri, al, ah); 240 } else { 241 TCGArg ai = tcgv_i32_arg(TCGV_LOW(a)); 242 vec_gen_2(INDEX_op_dup_vec, type, vece, ri, ai); 243 } 244 } 245 246 void tcg_gen_dup_i32_vec(unsigned vece, TCGv_vec r, TCGv_i32 a) 247 { 248 TCGArg ri = tcgv_vec_arg(r); 249 TCGArg ai = tcgv_i32_arg(a); 250 TCGTemp *rt = arg_temp(ri); 251 TCGType type = rt->base_type; 252 253 vec_gen_2(INDEX_op_dup_vec, type, vece, ri, ai); 254 } 255 256 void tcg_gen_dup_mem_vec(unsigned vece, TCGv_vec r, TCGv_ptr b, 257 tcg_target_long ofs) 258 { 259 TCGArg ri = tcgv_vec_arg(r); 260 TCGArg bi = tcgv_ptr_arg(b); 261 TCGTemp *rt = arg_temp(ri); 262 TCGType type = rt->base_type; 263 264 vec_gen_3(INDEX_op_dupm_vec, type, vece, ri, bi, ofs); 265 } 266 267 static void vec_gen_ldst(TCGOpcode opc, TCGv_vec r, TCGv_ptr b, TCGArg o) 268 { 269 TCGArg ri = tcgv_vec_arg(r); 270 TCGArg bi = tcgv_ptr_arg(b); 271 TCGTemp *rt = arg_temp(ri); 272 TCGType type = rt->base_type; 273 274 vec_gen_3(opc, type, 0, ri, bi, o); 275 } 276 277 void tcg_gen_ld_vec(TCGv_vec r, TCGv_ptr b, TCGArg o) 278 { 279 vec_gen_ldst(INDEX_op_ld_vec, r, b, o); 280 } 281 282 void tcg_gen_st_vec(TCGv_vec r, TCGv_ptr b, TCGArg o) 283 { 284 vec_gen_ldst(INDEX_op_st_vec, r, b, o); 285 } 286 287 void tcg_gen_stl_vec(TCGv_vec r, TCGv_ptr b, TCGArg o, TCGType low_type) 288 { 289 TCGArg ri = tcgv_vec_arg(r); 290 TCGArg bi = tcgv_ptr_arg(b); 291 TCGTemp *rt = arg_temp(ri); 292 TCGType type = rt->base_type; 293 294 tcg_debug_assert(low_type >= TCG_TYPE_V64); 295 tcg_debug_assert(low_type <= type); 296 vec_gen_3(INDEX_op_st_vec, low_type, 0, ri, bi, o); 297 } 298 299 void tcg_gen_and_vec(unsigned vece, TCGv_vec r, TCGv_vec a, TCGv_vec b) 300 { 301 vec_gen_op3(INDEX_op_and_vec, 0, r, a, b); 302 } 303 304 void tcg_gen_or_vec(unsigned vece, TCGv_vec r, TCGv_vec a, TCGv_vec b) 305 { 306 vec_gen_op3(INDEX_op_or_vec, 0, r, a, b); 307 } 308 309 void tcg_gen_xor_vec(unsigned vece, TCGv_vec r, TCGv_vec a, TCGv_vec b) 310 { 311 vec_gen_op3(INDEX_op_xor_vec, 0, r, a, b); 312 } 313 314 void tcg_gen_andc_vec(unsigned vece, TCGv_vec r, TCGv_vec a, TCGv_vec b) 315 { 316 if (TCG_TARGET_HAS_andc_vec) { 317 vec_gen_op3(INDEX_op_andc_vec, 0, r, a, b); 318 } else { 319 TCGv_vec t = tcg_temp_new_vec_matching(r); 320 tcg_gen_not_vec(0, t, b); 321 tcg_gen_and_vec(0, r, a, t); 322 tcg_temp_free_vec(t); 323 } 324 } 325 326 void tcg_gen_orc_vec(unsigned vece, TCGv_vec r, TCGv_vec a, TCGv_vec b) 327 { 328 if (TCG_TARGET_HAS_orc_vec) { 329 vec_gen_op3(INDEX_op_orc_vec, 0, r, a, b); 330 } else { 331 TCGv_vec t = tcg_temp_new_vec_matching(r); 332 tcg_gen_not_vec(0, t, b); 333 tcg_gen_or_vec(0, r, a, t); 334 tcg_temp_free_vec(t); 335 } 336 } 337 338 void tcg_gen_nand_vec(unsigned vece, TCGv_vec r, TCGv_vec a, TCGv_vec b) 339 { 340 if (TCG_TARGET_HAS_nand_vec) { 341 vec_gen_op3(INDEX_op_nand_vec, 0, r, a, b); 342 } else { 343 tcg_gen_and_vec(0, r, a, b); 344 tcg_gen_not_vec(0, r, r); 345 } 346 } 347 348 void tcg_gen_nor_vec(unsigned vece, TCGv_vec r, TCGv_vec a, TCGv_vec b) 349 { 350 if (TCG_TARGET_HAS_nor_vec) { 351 vec_gen_op3(INDEX_op_nor_vec, 0, r, a, b); 352 } else { 353 tcg_gen_or_vec(0, r, a, b); 354 tcg_gen_not_vec(0, r, r); 355 } 356 } 357 358 void tcg_gen_eqv_vec(unsigned vece, TCGv_vec r, TCGv_vec a, TCGv_vec b) 359 { 360 if (TCG_TARGET_HAS_eqv_vec) { 361 vec_gen_op3(INDEX_op_eqv_vec, 0, r, a, b); 362 } else { 363 tcg_gen_xor_vec(0, r, a, b); 364 tcg_gen_not_vec(0, r, r); 365 } 366 } 367 368 static bool do_op2(unsigned vece, TCGv_vec r, TCGv_vec a, TCGOpcode opc) 369 { 370 TCGTemp *rt = tcgv_vec_temp(r); 371 TCGTemp *at = tcgv_vec_temp(a); 372 TCGArg ri = temp_arg(rt); 373 TCGArg ai = temp_arg(at); 374 TCGType type = rt->base_type; 375 int can; 376 377 tcg_debug_assert(at->base_type >= type); 378 tcg_assert_listed_vecop(opc); 379 can = tcg_can_emit_vec_op(opc, type, vece); 380 if (can > 0) { 381 vec_gen_2(opc, type, vece, ri, ai); 382 } else if (can < 0) { 383 const TCGOpcode *hold_list = tcg_swap_vecop_list(NULL); 384 tcg_expand_vec_op(opc, type, vece, ri, ai); 385 tcg_swap_vecop_list(hold_list); 386 } else { 387 return false; 388 } 389 return true; 390 } 391 392 void tcg_gen_not_vec(unsigned vece, TCGv_vec r, TCGv_vec a) 393 { 394 if (TCG_TARGET_HAS_not_vec) { 395 vec_gen_op2(INDEX_op_not_vec, 0, r, a); 396 } else { 397 tcg_gen_xor_vec(0, r, a, tcg_constant_vec_matching(r, 0, -1)); 398 } 399 } 400 401 void tcg_gen_neg_vec(unsigned vece, TCGv_vec r, TCGv_vec a) 402 { 403 const TCGOpcode *hold_list; 404 405 tcg_assert_listed_vecop(INDEX_op_neg_vec); 406 hold_list = tcg_swap_vecop_list(NULL); 407 408 if (!TCG_TARGET_HAS_neg_vec || !do_op2(vece, r, a, INDEX_op_neg_vec)) { 409 tcg_gen_sub_vec(vece, r, tcg_constant_vec_matching(r, vece, 0), a); 410 } 411 tcg_swap_vecop_list(hold_list); 412 } 413 414 void tcg_gen_abs_vec(unsigned vece, TCGv_vec r, TCGv_vec a) 415 { 416 const TCGOpcode *hold_list; 417 418 tcg_assert_listed_vecop(INDEX_op_abs_vec); 419 hold_list = tcg_swap_vecop_list(NULL); 420 421 if (!do_op2(vece, r, a, INDEX_op_abs_vec)) { 422 TCGType type = tcgv_vec_temp(r)->base_type; 423 TCGv_vec t = tcg_temp_new_vec(type); 424 425 tcg_debug_assert(tcg_can_emit_vec_op(INDEX_op_sub_vec, type, vece)); 426 if (tcg_can_emit_vec_op(INDEX_op_smax_vec, type, vece) > 0) { 427 tcg_gen_neg_vec(vece, t, a); 428 tcg_gen_smax_vec(vece, r, a, t); 429 } else { 430 if (tcg_can_emit_vec_op(INDEX_op_sari_vec, type, vece) > 0) { 431 tcg_gen_sari_vec(vece, t, a, (8 << vece) - 1); 432 } else { 433 tcg_gen_cmp_vec(TCG_COND_LT, vece, t, a, 434 tcg_constant_vec(type, vece, 0)); 435 } 436 tcg_gen_xor_vec(vece, r, a, t); 437 tcg_gen_sub_vec(vece, r, r, t); 438 } 439 440 tcg_temp_free_vec(t); 441 } 442 tcg_swap_vecop_list(hold_list); 443 } 444 445 static void do_shifti(TCGOpcode opc, unsigned vece, 446 TCGv_vec r, TCGv_vec a, int64_t i) 447 { 448 TCGTemp *rt = tcgv_vec_temp(r); 449 TCGTemp *at = tcgv_vec_temp(a); 450 TCGArg ri = temp_arg(rt); 451 TCGArg ai = temp_arg(at); 452 TCGType type = rt->base_type; 453 int can; 454 455 tcg_debug_assert(at->base_type == type); 456 tcg_debug_assert(i >= 0 && i < (8 << vece)); 457 tcg_assert_listed_vecop(opc); 458 459 if (i == 0) { 460 tcg_gen_mov_vec(r, a); 461 return; 462 } 463 464 can = tcg_can_emit_vec_op(opc, type, vece); 465 if (can > 0) { 466 vec_gen_3(opc, type, vece, ri, ai, i); 467 } else { 468 /* We leave the choice of expansion via scalar or vector shift 469 to the target. Often, but not always, dupi can feed a vector 470 shift easier than a scalar. */ 471 const TCGOpcode *hold_list = tcg_swap_vecop_list(NULL); 472 tcg_debug_assert(can < 0); 473 tcg_expand_vec_op(opc, type, vece, ri, ai, i); 474 tcg_swap_vecop_list(hold_list); 475 } 476 } 477 478 void tcg_gen_shli_vec(unsigned vece, TCGv_vec r, TCGv_vec a, int64_t i) 479 { 480 do_shifti(INDEX_op_shli_vec, vece, r, a, i); 481 } 482 483 void tcg_gen_shri_vec(unsigned vece, TCGv_vec r, TCGv_vec a, int64_t i) 484 { 485 do_shifti(INDEX_op_shri_vec, vece, r, a, i); 486 } 487 488 void tcg_gen_sari_vec(unsigned vece, TCGv_vec r, TCGv_vec a, int64_t i) 489 { 490 do_shifti(INDEX_op_sari_vec, vece, r, a, i); 491 } 492 493 void tcg_gen_rotli_vec(unsigned vece, TCGv_vec r, TCGv_vec a, int64_t i) 494 { 495 do_shifti(INDEX_op_rotli_vec, vece, r, a, i); 496 } 497 498 void tcg_gen_rotri_vec(unsigned vece, TCGv_vec r, TCGv_vec a, int64_t i) 499 { 500 int bits = 8 << vece; 501 tcg_debug_assert(i >= 0 && i < bits); 502 do_shifti(INDEX_op_rotli_vec, vece, r, a, -i & (bits - 1)); 503 } 504 505 void tcg_gen_cmp_vec(TCGCond cond, unsigned vece, 506 TCGv_vec r, TCGv_vec a, TCGv_vec b) 507 { 508 TCGTemp *rt = tcgv_vec_temp(r); 509 TCGTemp *at = tcgv_vec_temp(a); 510 TCGTemp *bt = tcgv_vec_temp(b); 511 TCGArg ri = temp_arg(rt); 512 TCGArg ai = temp_arg(at); 513 TCGArg bi = temp_arg(bt); 514 TCGType type = rt->base_type; 515 int can; 516 517 tcg_debug_assert(at->base_type >= type); 518 tcg_debug_assert(bt->base_type >= type); 519 tcg_assert_listed_vecop(INDEX_op_cmp_vec); 520 can = tcg_can_emit_vec_op(INDEX_op_cmp_vec, type, vece); 521 if (can > 0) { 522 vec_gen_4(INDEX_op_cmp_vec, type, vece, ri, ai, bi, cond); 523 } else { 524 const TCGOpcode *hold_list = tcg_swap_vecop_list(NULL); 525 tcg_debug_assert(can < 0); 526 tcg_expand_vec_op(INDEX_op_cmp_vec, type, vece, ri, ai, bi, cond); 527 tcg_swap_vecop_list(hold_list); 528 } 529 } 530 531 static bool do_op3(unsigned vece, TCGv_vec r, TCGv_vec a, 532 TCGv_vec b, TCGOpcode opc) 533 { 534 TCGTemp *rt = tcgv_vec_temp(r); 535 TCGTemp *at = tcgv_vec_temp(a); 536 TCGTemp *bt = tcgv_vec_temp(b); 537 TCGArg ri = temp_arg(rt); 538 TCGArg ai = temp_arg(at); 539 TCGArg bi = temp_arg(bt); 540 TCGType type = rt->base_type; 541 int can; 542 543 tcg_debug_assert(at->base_type >= type); 544 tcg_debug_assert(bt->base_type >= type); 545 tcg_assert_listed_vecop(opc); 546 can = tcg_can_emit_vec_op(opc, type, vece); 547 if (can > 0) { 548 vec_gen_3(opc, type, vece, ri, ai, bi); 549 } else if (can < 0) { 550 const TCGOpcode *hold_list = tcg_swap_vecop_list(NULL); 551 tcg_expand_vec_op(opc, type, vece, ri, ai, bi); 552 tcg_swap_vecop_list(hold_list); 553 } else { 554 return false; 555 } 556 return true; 557 } 558 559 static void do_op3_nofail(unsigned vece, TCGv_vec r, TCGv_vec a, 560 TCGv_vec b, TCGOpcode opc) 561 { 562 bool ok = do_op3(vece, r, a, b, opc); 563 tcg_debug_assert(ok); 564 } 565 566 void tcg_gen_add_vec(unsigned vece, TCGv_vec r, TCGv_vec a, TCGv_vec b) 567 { 568 do_op3_nofail(vece, r, a, b, INDEX_op_add_vec); 569 } 570 571 void tcg_gen_sub_vec(unsigned vece, TCGv_vec r, TCGv_vec a, TCGv_vec b) 572 { 573 do_op3_nofail(vece, r, a, b, INDEX_op_sub_vec); 574 } 575 576 void tcg_gen_mul_vec(unsigned vece, TCGv_vec r, TCGv_vec a, TCGv_vec b) 577 { 578 do_op3_nofail(vece, r, a, b, INDEX_op_mul_vec); 579 } 580 581 void tcg_gen_ssadd_vec(unsigned vece, TCGv_vec r, TCGv_vec a, TCGv_vec b) 582 { 583 do_op3_nofail(vece, r, a, b, INDEX_op_ssadd_vec); 584 } 585 586 void tcg_gen_usadd_vec(unsigned vece, TCGv_vec r, TCGv_vec a, TCGv_vec b) 587 { 588 if (!do_op3(vece, r, a, b, INDEX_op_usadd_vec)) { 589 const TCGOpcode *hold_list = tcg_swap_vecop_list(NULL); 590 TCGv_vec t = tcg_temp_new_vec_matching(r); 591 592 /* usadd(a, b) = min(a, ~b) + b */ 593 tcg_gen_not_vec(vece, t, b); 594 tcg_gen_umin_vec(vece, t, t, a); 595 tcg_gen_add_vec(vece, r, t, b); 596 597 tcg_temp_free_vec(t); 598 tcg_swap_vecop_list(hold_list); 599 } 600 } 601 602 void tcg_gen_sssub_vec(unsigned vece, TCGv_vec r, TCGv_vec a, TCGv_vec b) 603 { 604 do_op3_nofail(vece, r, a, b, INDEX_op_sssub_vec); 605 } 606 607 void tcg_gen_ussub_vec(unsigned vece, TCGv_vec r, TCGv_vec a, TCGv_vec b) 608 { 609 if (!do_op3(vece, r, a, b, INDEX_op_ussub_vec)) { 610 const TCGOpcode *hold_list = tcg_swap_vecop_list(NULL); 611 TCGv_vec t = tcg_temp_new_vec_matching(r); 612 613 /* ussub(a, b) = max(a, b) - b */ 614 tcg_gen_umax_vec(vece, t, a, b); 615 tcg_gen_sub_vec(vece, r, t, b); 616 617 tcg_temp_free_vec(t); 618 tcg_swap_vecop_list(hold_list); 619 } 620 } 621 622 static void do_minmax(unsigned vece, TCGv_vec r, TCGv_vec a, 623 TCGv_vec b, TCGOpcode opc, TCGCond cond) 624 { 625 if (!do_op3(vece, r, a, b, opc)) { 626 const TCGOpcode *hold_list = tcg_swap_vecop_list(NULL); 627 tcg_gen_cmpsel_vec(cond, vece, r, a, b, a, b); 628 tcg_swap_vecop_list(hold_list); 629 } 630 } 631 632 void tcg_gen_smin_vec(unsigned vece, TCGv_vec r, TCGv_vec a, TCGv_vec b) 633 { 634 do_minmax(vece, r, a, b, INDEX_op_smin_vec, TCG_COND_LT); 635 } 636 637 void tcg_gen_umin_vec(unsigned vece, TCGv_vec r, TCGv_vec a, TCGv_vec b) 638 { 639 do_minmax(vece, r, a, b, INDEX_op_umin_vec, TCG_COND_LTU); 640 } 641 642 void tcg_gen_smax_vec(unsigned vece, TCGv_vec r, TCGv_vec a, TCGv_vec b) 643 { 644 do_minmax(vece, r, a, b, INDEX_op_smax_vec, TCG_COND_GT); 645 } 646 647 void tcg_gen_umax_vec(unsigned vece, TCGv_vec r, TCGv_vec a, TCGv_vec b) 648 { 649 do_minmax(vece, r, a, b, INDEX_op_umax_vec, TCG_COND_GTU); 650 } 651 652 void tcg_gen_shlv_vec(unsigned vece, TCGv_vec r, TCGv_vec a, TCGv_vec b) 653 { 654 do_op3_nofail(vece, r, a, b, INDEX_op_shlv_vec); 655 } 656 657 void tcg_gen_shrv_vec(unsigned vece, TCGv_vec r, TCGv_vec a, TCGv_vec b) 658 { 659 do_op3_nofail(vece, r, a, b, INDEX_op_shrv_vec); 660 } 661 662 void tcg_gen_sarv_vec(unsigned vece, TCGv_vec r, TCGv_vec a, TCGv_vec b) 663 { 664 do_op3_nofail(vece, r, a, b, INDEX_op_sarv_vec); 665 } 666 667 void tcg_gen_rotlv_vec(unsigned vece, TCGv_vec r, TCGv_vec a, TCGv_vec b) 668 { 669 do_op3_nofail(vece, r, a, b, INDEX_op_rotlv_vec); 670 } 671 672 void tcg_gen_rotrv_vec(unsigned vece, TCGv_vec r, TCGv_vec a, TCGv_vec b) 673 { 674 do_op3_nofail(vece, r, a, b, INDEX_op_rotrv_vec); 675 } 676 677 static void do_shifts(unsigned vece, TCGv_vec r, TCGv_vec a, 678 TCGv_i32 s, TCGOpcode opc) 679 { 680 TCGTemp *rt = tcgv_vec_temp(r); 681 TCGTemp *at = tcgv_vec_temp(a); 682 TCGTemp *st = tcgv_i32_temp(s); 683 TCGArg ri = temp_arg(rt); 684 TCGArg ai = temp_arg(at); 685 TCGArg si = temp_arg(st); 686 TCGType type = rt->base_type; 687 int can; 688 689 tcg_debug_assert(at->base_type >= type); 690 tcg_assert_listed_vecop(opc); 691 can = tcg_can_emit_vec_op(opc, type, vece); 692 if (can > 0) { 693 vec_gen_3(opc, type, vece, ri, ai, si); 694 } else if (can < 0) { 695 const TCGOpcode *hold_list = tcg_swap_vecop_list(NULL); 696 tcg_expand_vec_op(opc, type, vece, ri, ai, si); 697 tcg_swap_vecop_list(hold_list); 698 } else { 699 g_assert_not_reached(); 700 } 701 } 702 703 void tcg_gen_shls_vec(unsigned vece, TCGv_vec r, TCGv_vec a, TCGv_i32 b) 704 { 705 do_shifts(vece, r, a, b, INDEX_op_shls_vec); 706 } 707 708 void tcg_gen_shrs_vec(unsigned vece, TCGv_vec r, TCGv_vec a, TCGv_i32 b) 709 { 710 do_shifts(vece, r, a, b, INDEX_op_shrs_vec); 711 } 712 713 void tcg_gen_sars_vec(unsigned vece, TCGv_vec r, TCGv_vec a, TCGv_i32 b) 714 { 715 do_shifts(vece, r, a, b, INDEX_op_sars_vec); 716 } 717 718 void tcg_gen_rotls_vec(unsigned vece, TCGv_vec r, TCGv_vec a, TCGv_i32 s) 719 { 720 do_shifts(vece, r, a, s, INDEX_op_rotls_vec); 721 } 722 723 void tcg_gen_bitsel_vec(unsigned vece, TCGv_vec r, TCGv_vec a, 724 TCGv_vec b, TCGv_vec c) 725 { 726 TCGTemp *rt = tcgv_vec_temp(r); 727 TCGTemp *at = tcgv_vec_temp(a); 728 TCGTemp *bt = tcgv_vec_temp(b); 729 TCGTemp *ct = tcgv_vec_temp(c); 730 TCGType type = rt->base_type; 731 732 tcg_debug_assert(at->base_type >= type); 733 tcg_debug_assert(bt->base_type >= type); 734 tcg_debug_assert(ct->base_type >= type); 735 736 if (TCG_TARGET_HAS_bitsel_vec) { 737 vec_gen_4(INDEX_op_bitsel_vec, type, MO_8, 738 temp_arg(rt), temp_arg(at), temp_arg(bt), temp_arg(ct)); 739 } else { 740 TCGv_vec t = tcg_temp_new_vec(type); 741 tcg_gen_and_vec(MO_8, t, a, b); 742 tcg_gen_andc_vec(MO_8, r, c, a); 743 tcg_gen_or_vec(MO_8, r, r, t); 744 tcg_temp_free_vec(t); 745 } 746 } 747 748 void tcg_gen_cmpsel_vec(TCGCond cond, unsigned vece, TCGv_vec r, 749 TCGv_vec a, TCGv_vec b, TCGv_vec c, TCGv_vec d) 750 { 751 TCGTemp *rt = tcgv_vec_temp(r); 752 TCGTemp *at = tcgv_vec_temp(a); 753 TCGTemp *bt = tcgv_vec_temp(b); 754 TCGTemp *ct = tcgv_vec_temp(c); 755 TCGTemp *dt = tcgv_vec_temp(d); 756 TCGArg ri = temp_arg(rt); 757 TCGArg ai = temp_arg(at); 758 TCGArg bi = temp_arg(bt); 759 TCGArg ci = temp_arg(ct); 760 TCGArg di = temp_arg(dt); 761 TCGType type = rt->base_type; 762 const TCGOpcode *hold_list; 763 int can; 764 765 tcg_debug_assert(at->base_type >= type); 766 tcg_debug_assert(bt->base_type >= type); 767 tcg_debug_assert(ct->base_type >= type); 768 tcg_debug_assert(dt->base_type >= type); 769 770 tcg_assert_listed_vecop(INDEX_op_cmpsel_vec); 771 hold_list = tcg_swap_vecop_list(NULL); 772 can = tcg_can_emit_vec_op(INDEX_op_cmpsel_vec, type, vece); 773 774 if (can > 0) { 775 vec_gen_6(INDEX_op_cmpsel_vec, type, vece, ri, ai, bi, ci, di, cond); 776 } else if (can < 0) { 777 tcg_expand_vec_op(INDEX_op_cmpsel_vec, type, vece, 778 ri, ai, bi, ci, di, cond); 779 } else { 780 TCGv_vec t = tcg_temp_new_vec(type); 781 tcg_gen_cmp_vec(cond, vece, t, a, b); 782 tcg_gen_bitsel_vec(vece, r, t, c, d); 783 tcg_temp_free_vec(t); 784 } 785 tcg_swap_vecop_list(hold_list); 786 } 787