1 /* 2 * Tiny Code Generator for QEMU 3 * 4 * Copyright (c) 2018 Linaro, Inc. 5 * 6 * This library is free software; you can redistribute it and/or 7 * modify it under the terms of the GNU Lesser General Public 8 * License as published by the Free Software Foundation; either 9 * version 2.1 of the License, or (at your option) any later version. 10 * 11 * This library is distributed in the hope that it will be useful, 12 * but WITHOUT ANY WARRANTY; without even the implied warranty of 13 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU 14 * Lesser General Public License for more details. 15 * 16 * You should have received a copy of the GNU Lesser General Public 17 * License along with this library; if not, see <http://www.gnu.org/licenses/>. 18 */ 19 20 #include "qemu/osdep.h" 21 #include "tcg/tcg.h" 22 #include "tcg/tcg-op.h" 23 #include "tcg/tcg-mo.h" 24 25 /* Reduce the number of ifdefs below. This assumes that all uses of 26 TCGV_HIGH and TCGV_LOW are properly protected by a conditional that 27 the compiler can eliminate. */ 28 #if TCG_TARGET_REG_BITS == 64 29 extern TCGv_i32 TCGV_LOW_link_error(TCGv_i64); 30 extern TCGv_i32 TCGV_HIGH_link_error(TCGv_i64); 31 #define TCGV_LOW TCGV_LOW_link_error 32 #define TCGV_HIGH TCGV_HIGH_link_error 33 #endif 34 35 /* 36 * Vector optional opcode tracking. 37 * Except for the basic logical operations (and, or, xor), and 38 * data movement (mov, ld, st, dupi), many vector opcodes are 39 * optional and may not be supported on the host. Thank Intel 40 * for the irregularity in their instruction set. 41 * 42 * The gvec expanders allow custom vector operations to be composed, 43 * generally via the .fniv callback in the GVecGen* structures. At 44 * the same time, in deciding whether to use this hook we need to 45 * know if the host supports the required operations. This is 46 * presented as an array of opcodes, terminated by 0. Each opcode 47 * is assumed to be expanded with the given VECE. 48 * 49 * For debugging, we want to validate this array. Therefore, when 50 * tcg_ctx->vec_opt_opc is non-NULL, the tcg_gen_*_vec expanders 51 * will validate that their opcode is present in the list. 52 */ 53 #ifdef CONFIG_DEBUG_TCG 54 void tcg_assert_listed_vecop(TCGOpcode op) 55 { 56 const TCGOpcode *p = tcg_ctx->vecop_list; 57 if (p) { 58 for (; *p; ++p) { 59 if (*p == op) { 60 return; 61 } 62 } 63 g_assert_not_reached(); 64 } 65 } 66 #endif 67 68 bool tcg_can_emit_vecop_list(const TCGOpcode *list, 69 TCGType type, unsigned vece) 70 { 71 if (list == NULL) { 72 return true; 73 } 74 75 for (; *list; ++list) { 76 TCGOpcode opc = *list; 77 78 #ifdef CONFIG_DEBUG_TCG 79 switch (opc) { 80 case INDEX_op_and_vec: 81 case INDEX_op_or_vec: 82 case INDEX_op_xor_vec: 83 case INDEX_op_mov_vec: 84 case INDEX_op_dup_vec: 85 case INDEX_op_dup2_vec: 86 case INDEX_op_ld_vec: 87 case INDEX_op_st_vec: 88 case INDEX_op_bitsel_vec: 89 /* These opcodes are mandatory and should not be listed. */ 90 g_assert_not_reached(); 91 case INDEX_op_not_vec: 92 /* These opcodes have generic expansions using the above. */ 93 g_assert_not_reached(); 94 default: 95 break; 96 } 97 #endif 98 99 if (tcg_can_emit_vec_op(opc, type, vece)) { 100 continue; 101 } 102 103 /* 104 * The opcode list is created by front ends based on what they 105 * actually invoke. We must mirror the logic in the routines 106 * below for generic expansions using other opcodes. 107 */ 108 switch (opc) { 109 case INDEX_op_neg_vec: 110 if (tcg_can_emit_vec_op(INDEX_op_sub_vec, type, vece)) { 111 continue; 112 } 113 break; 114 case INDEX_op_abs_vec: 115 if (tcg_can_emit_vec_op(INDEX_op_sub_vec, type, vece) 116 && (tcg_can_emit_vec_op(INDEX_op_smax_vec, type, vece) > 0 117 || tcg_can_emit_vec_op(INDEX_op_sari_vec, type, vece) > 0 118 || tcg_can_emit_vec_op(INDEX_op_cmp_vec, type, vece))) { 119 continue; 120 } 121 break; 122 case INDEX_op_cmpsel_vec: 123 case INDEX_op_smin_vec: 124 case INDEX_op_smax_vec: 125 case INDEX_op_umin_vec: 126 case INDEX_op_umax_vec: 127 if (tcg_can_emit_vec_op(INDEX_op_cmp_vec, type, vece)) { 128 continue; 129 } 130 break; 131 default: 132 break; 133 } 134 return false; 135 } 136 return true; 137 } 138 139 void vec_gen_2(TCGOpcode opc, TCGType type, unsigned vece, TCGArg r, TCGArg a) 140 { 141 TCGOp *op = tcg_emit_op(opc); 142 TCGOP_VECL(op) = type - TCG_TYPE_V64; 143 TCGOP_VECE(op) = vece; 144 op->args[0] = r; 145 op->args[1] = a; 146 } 147 148 void vec_gen_3(TCGOpcode opc, TCGType type, unsigned vece, 149 TCGArg r, TCGArg a, TCGArg b) 150 { 151 TCGOp *op = tcg_emit_op(opc); 152 TCGOP_VECL(op) = type - TCG_TYPE_V64; 153 TCGOP_VECE(op) = vece; 154 op->args[0] = r; 155 op->args[1] = a; 156 op->args[2] = b; 157 } 158 159 void vec_gen_4(TCGOpcode opc, TCGType type, unsigned vece, 160 TCGArg r, TCGArg a, TCGArg b, TCGArg c) 161 { 162 TCGOp *op = tcg_emit_op(opc); 163 TCGOP_VECL(op) = type - TCG_TYPE_V64; 164 TCGOP_VECE(op) = vece; 165 op->args[0] = r; 166 op->args[1] = a; 167 op->args[2] = b; 168 op->args[3] = c; 169 } 170 171 static void vec_gen_6(TCGOpcode opc, TCGType type, unsigned vece, TCGArg r, 172 TCGArg a, TCGArg b, TCGArg c, TCGArg d, TCGArg e) 173 { 174 TCGOp *op = tcg_emit_op(opc); 175 TCGOP_VECL(op) = type - TCG_TYPE_V64; 176 TCGOP_VECE(op) = vece; 177 op->args[0] = r; 178 op->args[1] = a; 179 op->args[2] = b; 180 op->args[3] = c; 181 op->args[4] = d; 182 op->args[5] = e; 183 } 184 185 static void vec_gen_op2(TCGOpcode opc, unsigned vece, TCGv_vec r, TCGv_vec a) 186 { 187 TCGTemp *rt = tcgv_vec_temp(r); 188 TCGTemp *at = tcgv_vec_temp(a); 189 TCGType type = rt->base_type; 190 191 /* Must enough inputs for the output. */ 192 tcg_debug_assert(at->base_type >= type); 193 vec_gen_2(opc, type, vece, temp_arg(rt), temp_arg(at)); 194 } 195 196 static void vec_gen_op3(TCGOpcode opc, unsigned vece, 197 TCGv_vec r, TCGv_vec a, TCGv_vec b) 198 { 199 TCGTemp *rt = tcgv_vec_temp(r); 200 TCGTemp *at = tcgv_vec_temp(a); 201 TCGTemp *bt = tcgv_vec_temp(b); 202 TCGType type = rt->base_type; 203 204 /* Must enough inputs for the output. */ 205 tcg_debug_assert(at->base_type >= type); 206 tcg_debug_assert(bt->base_type >= type); 207 vec_gen_3(opc, type, vece, temp_arg(rt), temp_arg(at), temp_arg(bt)); 208 } 209 210 void tcg_gen_mov_vec(TCGv_vec r, TCGv_vec a) 211 { 212 if (r != a) { 213 vec_gen_op2(INDEX_op_mov_vec, 0, r, a); 214 } 215 } 216 217 TCGv_vec tcg_const_zeros_vec(TCGType type) 218 { 219 TCGv_vec ret = tcg_temp_new_vec(type); 220 tcg_gen_dupi_vec(MO_64, ret, 0); 221 return ret; 222 } 223 224 TCGv_vec tcg_const_ones_vec(TCGType type) 225 { 226 TCGv_vec ret = tcg_temp_new_vec(type); 227 tcg_gen_dupi_vec(MO_64, ret, -1); 228 return ret; 229 } 230 231 TCGv_vec tcg_const_zeros_vec_matching(TCGv_vec m) 232 { 233 TCGTemp *t = tcgv_vec_temp(m); 234 return tcg_const_zeros_vec(t->base_type); 235 } 236 237 TCGv_vec tcg_const_ones_vec_matching(TCGv_vec m) 238 { 239 TCGTemp *t = tcgv_vec_temp(m); 240 return tcg_const_ones_vec(t->base_type); 241 } 242 243 void tcg_gen_dupi_vec(unsigned vece, TCGv_vec r, uint64_t a) 244 { 245 TCGTemp *rt = tcgv_vec_temp(r); 246 tcg_gen_mov_vec(r, tcg_constant_vec(rt->base_type, vece, a)); 247 } 248 249 void tcg_gen_dup_i64_vec(unsigned vece, TCGv_vec r, TCGv_i64 a) 250 { 251 TCGArg ri = tcgv_vec_arg(r); 252 TCGTemp *rt = arg_temp(ri); 253 TCGType type = rt->base_type; 254 255 if (TCG_TARGET_REG_BITS == 64) { 256 TCGArg ai = tcgv_i64_arg(a); 257 vec_gen_2(INDEX_op_dup_vec, type, vece, ri, ai); 258 } else if (vece == MO_64) { 259 TCGArg al = tcgv_i32_arg(TCGV_LOW(a)); 260 TCGArg ah = tcgv_i32_arg(TCGV_HIGH(a)); 261 vec_gen_3(INDEX_op_dup2_vec, type, MO_64, ri, al, ah); 262 } else { 263 TCGArg ai = tcgv_i32_arg(TCGV_LOW(a)); 264 vec_gen_2(INDEX_op_dup_vec, type, vece, ri, ai); 265 } 266 } 267 268 void tcg_gen_dup_i32_vec(unsigned vece, TCGv_vec r, TCGv_i32 a) 269 { 270 TCGArg ri = tcgv_vec_arg(r); 271 TCGArg ai = tcgv_i32_arg(a); 272 TCGTemp *rt = arg_temp(ri); 273 TCGType type = rt->base_type; 274 275 vec_gen_2(INDEX_op_dup_vec, type, vece, ri, ai); 276 } 277 278 void tcg_gen_dup_mem_vec(unsigned vece, TCGv_vec r, TCGv_ptr b, 279 tcg_target_long ofs) 280 { 281 TCGArg ri = tcgv_vec_arg(r); 282 TCGArg bi = tcgv_ptr_arg(b); 283 TCGTemp *rt = arg_temp(ri); 284 TCGType type = rt->base_type; 285 286 vec_gen_3(INDEX_op_dupm_vec, type, vece, ri, bi, ofs); 287 } 288 289 static void vec_gen_ldst(TCGOpcode opc, TCGv_vec r, TCGv_ptr b, TCGArg o) 290 { 291 TCGArg ri = tcgv_vec_arg(r); 292 TCGArg bi = tcgv_ptr_arg(b); 293 TCGTemp *rt = arg_temp(ri); 294 TCGType type = rt->base_type; 295 296 vec_gen_3(opc, type, 0, ri, bi, o); 297 } 298 299 void tcg_gen_ld_vec(TCGv_vec r, TCGv_ptr b, TCGArg o) 300 { 301 vec_gen_ldst(INDEX_op_ld_vec, r, b, o); 302 } 303 304 void tcg_gen_st_vec(TCGv_vec r, TCGv_ptr b, TCGArg o) 305 { 306 vec_gen_ldst(INDEX_op_st_vec, r, b, o); 307 } 308 309 void tcg_gen_stl_vec(TCGv_vec r, TCGv_ptr b, TCGArg o, TCGType low_type) 310 { 311 TCGArg ri = tcgv_vec_arg(r); 312 TCGArg bi = tcgv_ptr_arg(b); 313 TCGTemp *rt = arg_temp(ri); 314 TCGType type = rt->base_type; 315 316 tcg_debug_assert(low_type >= TCG_TYPE_V64); 317 tcg_debug_assert(low_type <= type); 318 vec_gen_3(INDEX_op_st_vec, low_type, 0, ri, bi, o); 319 } 320 321 void tcg_gen_and_vec(unsigned vece, TCGv_vec r, TCGv_vec a, TCGv_vec b) 322 { 323 vec_gen_op3(INDEX_op_and_vec, 0, r, a, b); 324 } 325 326 void tcg_gen_or_vec(unsigned vece, TCGv_vec r, TCGv_vec a, TCGv_vec b) 327 { 328 vec_gen_op3(INDEX_op_or_vec, 0, r, a, b); 329 } 330 331 void tcg_gen_xor_vec(unsigned vece, TCGv_vec r, TCGv_vec a, TCGv_vec b) 332 { 333 vec_gen_op3(INDEX_op_xor_vec, 0, r, a, b); 334 } 335 336 void tcg_gen_andc_vec(unsigned vece, TCGv_vec r, TCGv_vec a, TCGv_vec b) 337 { 338 if (TCG_TARGET_HAS_andc_vec) { 339 vec_gen_op3(INDEX_op_andc_vec, 0, r, a, b); 340 } else { 341 TCGv_vec t = tcg_temp_new_vec_matching(r); 342 tcg_gen_not_vec(0, t, b); 343 tcg_gen_and_vec(0, r, a, t); 344 tcg_temp_free_vec(t); 345 } 346 } 347 348 void tcg_gen_orc_vec(unsigned vece, TCGv_vec r, TCGv_vec a, TCGv_vec b) 349 { 350 if (TCG_TARGET_HAS_orc_vec) { 351 vec_gen_op3(INDEX_op_orc_vec, 0, r, a, b); 352 } else { 353 TCGv_vec t = tcg_temp_new_vec_matching(r); 354 tcg_gen_not_vec(0, t, b); 355 tcg_gen_or_vec(0, r, a, t); 356 tcg_temp_free_vec(t); 357 } 358 } 359 360 void tcg_gen_nand_vec(unsigned vece, TCGv_vec r, TCGv_vec a, TCGv_vec b) 361 { 362 /* TODO: Add TCG_TARGET_HAS_nand_vec when adding a backend supports it. */ 363 tcg_gen_and_vec(0, r, a, b); 364 tcg_gen_not_vec(0, r, r); 365 } 366 367 void tcg_gen_nor_vec(unsigned vece, TCGv_vec r, TCGv_vec a, TCGv_vec b) 368 { 369 /* TODO: Add TCG_TARGET_HAS_nor_vec when adding a backend supports it. */ 370 tcg_gen_or_vec(0, r, a, b); 371 tcg_gen_not_vec(0, r, r); 372 } 373 374 void tcg_gen_eqv_vec(unsigned vece, TCGv_vec r, TCGv_vec a, TCGv_vec b) 375 { 376 /* TODO: Add TCG_TARGET_HAS_eqv_vec when adding a backend supports it. */ 377 tcg_gen_xor_vec(0, r, a, b); 378 tcg_gen_not_vec(0, r, r); 379 } 380 381 static bool do_op2(unsigned vece, TCGv_vec r, TCGv_vec a, TCGOpcode opc) 382 { 383 TCGTemp *rt = tcgv_vec_temp(r); 384 TCGTemp *at = tcgv_vec_temp(a); 385 TCGArg ri = temp_arg(rt); 386 TCGArg ai = temp_arg(at); 387 TCGType type = rt->base_type; 388 int can; 389 390 tcg_debug_assert(at->base_type >= type); 391 tcg_assert_listed_vecop(opc); 392 can = tcg_can_emit_vec_op(opc, type, vece); 393 if (can > 0) { 394 vec_gen_2(opc, type, vece, ri, ai); 395 } else if (can < 0) { 396 const TCGOpcode *hold_list = tcg_swap_vecop_list(NULL); 397 tcg_expand_vec_op(opc, type, vece, ri, ai); 398 tcg_swap_vecop_list(hold_list); 399 } else { 400 return false; 401 } 402 return true; 403 } 404 405 void tcg_gen_not_vec(unsigned vece, TCGv_vec r, TCGv_vec a) 406 { 407 const TCGOpcode *hold_list = tcg_swap_vecop_list(NULL); 408 409 if (!TCG_TARGET_HAS_not_vec || !do_op2(vece, r, a, INDEX_op_not_vec)) { 410 TCGv_vec t = tcg_const_ones_vec_matching(r); 411 tcg_gen_xor_vec(0, r, a, t); 412 tcg_temp_free_vec(t); 413 } 414 tcg_swap_vecop_list(hold_list); 415 } 416 417 void tcg_gen_neg_vec(unsigned vece, TCGv_vec r, TCGv_vec a) 418 { 419 const TCGOpcode *hold_list; 420 421 tcg_assert_listed_vecop(INDEX_op_neg_vec); 422 hold_list = tcg_swap_vecop_list(NULL); 423 424 if (!TCG_TARGET_HAS_neg_vec || !do_op2(vece, r, a, INDEX_op_neg_vec)) { 425 TCGv_vec t = tcg_const_zeros_vec_matching(r); 426 tcg_gen_sub_vec(vece, r, t, a); 427 tcg_temp_free_vec(t); 428 } 429 tcg_swap_vecop_list(hold_list); 430 } 431 432 void tcg_gen_abs_vec(unsigned vece, TCGv_vec r, TCGv_vec a) 433 { 434 const TCGOpcode *hold_list; 435 436 tcg_assert_listed_vecop(INDEX_op_abs_vec); 437 hold_list = tcg_swap_vecop_list(NULL); 438 439 if (!do_op2(vece, r, a, INDEX_op_abs_vec)) { 440 TCGType type = tcgv_vec_temp(r)->base_type; 441 TCGv_vec t = tcg_temp_new_vec(type); 442 443 tcg_debug_assert(tcg_can_emit_vec_op(INDEX_op_sub_vec, type, vece)); 444 if (tcg_can_emit_vec_op(INDEX_op_smax_vec, type, vece) > 0) { 445 tcg_gen_neg_vec(vece, t, a); 446 tcg_gen_smax_vec(vece, r, a, t); 447 } else { 448 if (tcg_can_emit_vec_op(INDEX_op_sari_vec, type, vece) > 0) { 449 tcg_gen_sari_vec(vece, t, a, (8 << vece) - 1); 450 } else { 451 tcg_gen_cmp_vec(TCG_COND_LT, vece, t, a, 452 tcg_constant_vec(type, vece, 0)); 453 } 454 tcg_gen_xor_vec(vece, r, a, t); 455 tcg_gen_sub_vec(vece, r, r, t); 456 } 457 458 tcg_temp_free_vec(t); 459 } 460 tcg_swap_vecop_list(hold_list); 461 } 462 463 static void do_shifti(TCGOpcode opc, unsigned vece, 464 TCGv_vec r, TCGv_vec a, int64_t i) 465 { 466 TCGTemp *rt = tcgv_vec_temp(r); 467 TCGTemp *at = tcgv_vec_temp(a); 468 TCGArg ri = temp_arg(rt); 469 TCGArg ai = temp_arg(at); 470 TCGType type = rt->base_type; 471 int can; 472 473 tcg_debug_assert(at->base_type == type); 474 tcg_debug_assert(i >= 0 && i < (8 << vece)); 475 tcg_assert_listed_vecop(opc); 476 477 if (i == 0) { 478 tcg_gen_mov_vec(r, a); 479 return; 480 } 481 482 can = tcg_can_emit_vec_op(opc, type, vece); 483 if (can > 0) { 484 vec_gen_3(opc, type, vece, ri, ai, i); 485 } else { 486 /* We leave the choice of expansion via scalar or vector shift 487 to the target. Often, but not always, dupi can feed a vector 488 shift easier than a scalar. */ 489 const TCGOpcode *hold_list = tcg_swap_vecop_list(NULL); 490 tcg_debug_assert(can < 0); 491 tcg_expand_vec_op(opc, type, vece, ri, ai, i); 492 tcg_swap_vecop_list(hold_list); 493 } 494 } 495 496 void tcg_gen_shli_vec(unsigned vece, TCGv_vec r, TCGv_vec a, int64_t i) 497 { 498 do_shifti(INDEX_op_shli_vec, vece, r, a, i); 499 } 500 501 void tcg_gen_shri_vec(unsigned vece, TCGv_vec r, TCGv_vec a, int64_t i) 502 { 503 do_shifti(INDEX_op_shri_vec, vece, r, a, i); 504 } 505 506 void tcg_gen_sari_vec(unsigned vece, TCGv_vec r, TCGv_vec a, int64_t i) 507 { 508 do_shifti(INDEX_op_sari_vec, vece, r, a, i); 509 } 510 511 void tcg_gen_rotli_vec(unsigned vece, TCGv_vec r, TCGv_vec a, int64_t i) 512 { 513 do_shifti(INDEX_op_rotli_vec, vece, r, a, i); 514 } 515 516 void tcg_gen_rotri_vec(unsigned vece, TCGv_vec r, TCGv_vec a, int64_t i) 517 { 518 int bits = 8 << vece; 519 tcg_debug_assert(i >= 0 && i < bits); 520 do_shifti(INDEX_op_rotli_vec, vece, r, a, -i & (bits - 1)); 521 } 522 523 void tcg_gen_cmp_vec(TCGCond cond, unsigned vece, 524 TCGv_vec r, TCGv_vec a, TCGv_vec b) 525 { 526 TCGTemp *rt = tcgv_vec_temp(r); 527 TCGTemp *at = tcgv_vec_temp(a); 528 TCGTemp *bt = tcgv_vec_temp(b); 529 TCGArg ri = temp_arg(rt); 530 TCGArg ai = temp_arg(at); 531 TCGArg bi = temp_arg(bt); 532 TCGType type = rt->base_type; 533 int can; 534 535 tcg_debug_assert(at->base_type >= type); 536 tcg_debug_assert(bt->base_type >= type); 537 tcg_assert_listed_vecop(INDEX_op_cmp_vec); 538 can = tcg_can_emit_vec_op(INDEX_op_cmp_vec, type, vece); 539 if (can > 0) { 540 vec_gen_4(INDEX_op_cmp_vec, type, vece, ri, ai, bi, cond); 541 } else { 542 const TCGOpcode *hold_list = tcg_swap_vecop_list(NULL); 543 tcg_debug_assert(can < 0); 544 tcg_expand_vec_op(INDEX_op_cmp_vec, type, vece, ri, ai, bi, cond); 545 tcg_swap_vecop_list(hold_list); 546 } 547 } 548 549 static bool do_op3(unsigned vece, TCGv_vec r, TCGv_vec a, 550 TCGv_vec b, TCGOpcode opc) 551 { 552 TCGTemp *rt = tcgv_vec_temp(r); 553 TCGTemp *at = tcgv_vec_temp(a); 554 TCGTemp *bt = tcgv_vec_temp(b); 555 TCGArg ri = temp_arg(rt); 556 TCGArg ai = temp_arg(at); 557 TCGArg bi = temp_arg(bt); 558 TCGType type = rt->base_type; 559 int can; 560 561 tcg_debug_assert(at->base_type >= type); 562 tcg_debug_assert(bt->base_type >= type); 563 tcg_assert_listed_vecop(opc); 564 can = tcg_can_emit_vec_op(opc, type, vece); 565 if (can > 0) { 566 vec_gen_3(opc, type, vece, ri, ai, bi); 567 } else if (can < 0) { 568 const TCGOpcode *hold_list = tcg_swap_vecop_list(NULL); 569 tcg_expand_vec_op(opc, type, vece, ri, ai, bi); 570 tcg_swap_vecop_list(hold_list); 571 } else { 572 return false; 573 } 574 return true; 575 } 576 577 static void do_op3_nofail(unsigned vece, TCGv_vec r, TCGv_vec a, 578 TCGv_vec b, TCGOpcode opc) 579 { 580 bool ok = do_op3(vece, r, a, b, opc); 581 tcg_debug_assert(ok); 582 } 583 584 void tcg_gen_add_vec(unsigned vece, TCGv_vec r, TCGv_vec a, TCGv_vec b) 585 { 586 do_op3_nofail(vece, r, a, b, INDEX_op_add_vec); 587 } 588 589 void tcg_gen_sub_vec(unsigned vece, TCGv_vec r, TCGv_vec a, TCGv_vec b) 590 { 591 do_op3_nofail(vece, r, a, b, INDEX_op_sub_vec); 592 } 593 594 void tcg_gen_mul_vec(unsigned vece, TCGv_vec r, TCGv_vec a, TCGv_vec b) 595 { 596 do_op3_nofail(vece, r, a, b, INDEX_op_mul_vec); 597 } 598 599 void tcg_gen_ssadd_vec(unsigned vece, TCGv_vec r, TCGv_vec a, TCGv_vec b) 600 { 601 do_op3_nofail(vece, r, a, b, INDEX_op_ssadd_vec); 602 } 603 604 void tcg_gen_usadd_vec(unsigned vece, TCGv_vec r, TCGv_vec a, TCGv_vec b) 605 { 606 do_op3_nofail(vece, r, a, b, INDEX_op_usadd_vec); 607 } 608 609 void tcg_gen_sssub_vec(unsigned vece, TCGv_vec r, TCGv_vec a, TCGv_vec b) 610 { 611 do_op3_nofail(vece, r, a, b, INDEX_op_sssub_vec); 612 } 613 614 void tcg_gen_ussub_vec(unsigned vece, TCGv_vec r, TCGv_vec a, TCGv_vec b) 615 { 616 do_op3_nofail(vece, r, a, b, INDEX_op_ussub_vec); 617 } 618 619 static void do_minmax(unsigned vece, TCGv_vec r, TCGv_vec a, 620 TCGv_vec b, TCGOpcode opc, TCGCond cond) 621 { 622 if (!do_op3(vece, r, a, b, opc)) { 623 const TCGOpcode *hold_list = tcg_swap_vecop_list(NULL); 624 tcg_gen_cmpsel_vec(cond, vece, r, a, b, a, b); 625 tcg_swap_vecop_list(hold_list); 626 } 627 } 628 629 void tcg_gen_smin_vec(unsigned vece, TCGv_vec r, TCGv_vec a, TCGv_vec b) 630 { 631 do_minmax(vece, r, a, b, INDEX_op_smin_vec, TCG_COND_LT); 632 } 633 634 void tcg_gen_umin_vec(unsigned vece, TCGv_vec r, TCGv_vec a, TCGv_vec b) 635 { 636 do_minmax(vece, r, a, b, INDEX_op_umin_vec, TCG_COND_LTU); 637 } 638 639 void tcg_gen_smax_vec(unsigned vece, TCGv_vec r, TCGv_vec a, TCGv_vec b) 640 { 641 do_minmax(vece, r, a, b, INDEX_op_smax_vec, TCG_COND_GT); 642 } 643 644 void tcg_gen_umax_vec(unsigned vece, TCGv_vec r, TCGv_vec a, TCGv_vec b) 645 { 646 do_minmax(vece, r, a, b, INDEX_op_umax_vec, TCG_COND_GTU); 647 } 648 649 void tcg_gen_shlv_vec(unsigned vece, TCGv_vec r, TCGv_vec a, TCGv_vec b) 650 { 651 do_op3_nofail(vece, r, a, b, INDEX_op_shlv_vec); 652 } 653 654 void tcg_gen_shrv_vec(unsigned vece, TCGv_vec r, TCGv_vec a, TCGv_vec b) 655 { 656 do_op3_nofail(vece, r, a, b, INDEX_op_shrv_vec); 657 } 658 659 void tcg_gen_sarv_vec(unsigned vece, TCGv_vec r, TCGv_vec a, TCGv_vec b) 660 { 661 do_op3_nofail(vece, r, a, b, INDEX_op_sarv_vec); 662 } 663 664 void tcg_gen_rotlv_vec(unsigned vece, TCGv_vec r, TCGv_vec a, TCGv_vec b) 665 { 666 do_op3_nofail(vece, r, a, b, INDEX_op_rotlv_vec); 667 } 668 669 void tcg_gen_rotrv_vec(unsigned vece, TCGv_vec r, TCGv_vec a, TCGv_vec b) 670 { 671 do_op3_nofail(vece, r, a, b, INDEX_op_rotrv_vec); 672 } 673 674 static void do_shifts(unsigned vece, TCGv_vec r, TCGv_vec a, 675 TCGv_i32 s, TCGOpcode opc) 676 { 677 TCGTemp *rt = tcgv_vec_temp(r); 678 TCGTemp *at = tcgv_vec_temp(a); 679 TCGTemp *st = tcgv_i32_temp(s); 680 TCGArg ri = temp_arg(rt); 681 TCGArg ai = temp_arg(at); 682 TCGArg si = temp_arg(st); 683 TCGType type = rt->base_type; 684 int can; 685 686 tcg_debug_assert(at->base_type >= type); 687 tcg_assert_listed_vecop(opc); 688 can = tcg_can_emit_vec_op(opc, type, vece); 689 if (can > 0) { 690 vec_gen_3(opc, type, vece, ri, ai, si); 691 } else if (can < 0) { 692 const TCGOpcode *hold_list = tcg_swap_vecop_list(NULL); 693 tcg_expand_vec_op(opc, type, vece, ri, ai, si); 694 tcg_swap_vecop_list(hold_list); 695 } else { 696 g_assert_not_reached(); 697 } 698 } 699 700 void tcg_gen_shls_vec(unsigned vece, TCGv_vec r, TCGv_vec a, TCGv_i32 b) 701 { 702 do_shifts(vece, r, a, b, INDEX_op_shls_vec); 703 } 704 705 void tcg_gen_shrs_vec(unsigned vece, TCGv_vec r, TCGv_vec a, TCGv_i32 b) 706 { 707 do_shifts(vece, r, a, b, INDEX_op_shrs_vec); 708 } 709 710 void tcg_gen_sars_vec(unsigned vece, TCGv_vec r, TCGv_vec a, TCGv_i32 b) 711 { 712 do_shifts(vece, r, a, b, INDEX_op_sars_vec); 713 } 714 715 void tcg_gen_rotls_vec(unsigned vece, TCGv_vec r, TCGv_vec a, TCGv_i32 s) 716 { 717 do_shifts(vece, r, a, s, INDEX_op_rotls_vec); 718 } 719 720 void tcg_gen_bitsel_vec(unsigned vece, TCGv_vec r, TCGv_vec a, 721 TCGv_vec b, TCGv_vec c) 722 { 723 TCGTemp *rt = tcgv_vec_temp(r); 724 TCGTemp *at = tcgv_vec_temp(a); 725 TCGTemp *bt = tcgv_vec_temp(b); 726 TCGTemp *ct = tcgv_vec_temp(c); 727 TCGType type = rt->base_type; 728 729 tcg_debug_assert(at->base_type >= type); 730 tcg_debug_assert(bt->base_type >= type); 731 tcg_debug_assert(ct->base_type >= type); 732 733 if (TCG_TARGET_HAS_bitsel_vec) { 734 vec_gen_4(INDEX_op_bitsel_vec, type, MO_8, 735 temp_arg(rt), temp_arg(at), temp_arg(bt), temp_arg(ct)); 736 } else { 737 TCGv_vec t = tcg_temp_new_vec(type); 738 tcg_gen_and_vec(MO_8, t, a, b); 739 tcg_gen_andc_vec(MO_8, r, c, a); 740 tcg_gen_or_vec(MO_8, r, r, t); 741 tcg_temp_free_vec(t); 742 } 743 } 744 745 void tcg_gen_cmpsel_vec(TCGCond cond, unsigned vece, TCGv_vec r, 746 TCGv_vec a, TCGv_vec b, TCGv_vec c, TCGv_vec d) 747 { 748 TCGTemp *rt = tcgv_vec_temp(r); 749 TCGTemp *at = tcgv_vec_temp(a); 750 TCGTemp *bt = tcgv_vec_temp(b); 751 TCGTemp *ct = tcgv_vec_temp(c); 752 TCGTemp *dt = tcgv_vec_temp(d); 753 TCGArg ri = temp_arg(rt); 754 TCGArg ai = temp_arg(at); 755 TCGArg bi = temp_arg(bt); 756 TCGArg ci = temp_arg(ct); 757 TCGArg di = temp_arg(dt); 758 TCGType type = rt->base_type; 759 const TCGOpcode *hold_list; 760 int can; 761 762 tcg_debug_assert(at->base_type >= type); 763 tcg_debug_assert(bt->base_type >= type); 764 tcg_debug_assert(ct->base_type >= type); 765 tcg_debug_assert(dt->base_type >= type); 766 767 tcg_assert_listed_vecop(INDEX_op_cmpsel_vec); 768 hold_list = tcg_swap_vecop_list(NULL); 769 can = tcg_can_emit_vec_op(INDEX_op_cmpsel_vec, type, vece); 770 771 if (can > 0) { 772 vec_gen_6(INDEX_op_cmpsel_vec, type, vece, ri, ai, bi, ci, di, cond); 773 } else if (can < 0) { 774 tcg_expand_vec_op(INDEX_op_cmpsel_vec, type, vece, 775 ri, ai, bi, ci, di, cond); 776 } else { 777 TCGv_vec t = tcg_temp_new_vec(type); 778 tcg_gen_cmp_vec(cond, vece, t, a, b); 779 tcg_gen_bitsel_vec(vece, r, t, c, d); 780 tcg_temp_free_vec(t); 781 } 782 tcg_swap_vecop_list(hold_list); 783 } 784