1 /* 2 * Tiny Code Generator for QEMU 3 * 4 * Copyright (c) 2018 Linaro, Inc. 5 * 6 * This library is free software; you can redistribute it and/or 7 * modify it under the terms of the GNU Lesser General Public 8 * License as published by the Free Software Foundation; either 9 * version 2.1 of the License, or (at your option) any later version. 10 * 11 * This library is distributed in the hope that it will be useful, 12 * but WITHOUT ANY WARRANTY; without even the implied warranty of 13 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU 14 * Lesser General Public License for more details. 15 * 16 * You should have received a copy of the GNU Lesser General Public 17 * License along with this library; if not, see <http://www.gnu.org/licenses/>. 18 */ 19 20 #include "qemu/osdep.h" 21 #include "qemu-common.h" 22 #include "cpu.h" 23 #include "tcg.h" 24 #include "tcg-op.h" 25 #include "tcg-mo.h" 26 27 /* Reduce the number of ifdefs below. This assumes that all uses of 28 TCGV_HIGH and TCGV_LOW are properly protected by a conditional that 29 the compiler can eliminate. */ 30 #if TCG_TARGET_REG_BITS == 64 31 extern TCGv_i32 TCGV_LOW_link_error(TCGv_i64); 32 extern TCGv_i32 TCGV_HIGH_link_error(TCGv_i64); 33 #define TCGV_LOW TCGV_LOW_link_error 34 #define TCGV_HIGH TCGV_HIGH_link_error 35 #endif 36 37 /* 38 * Vector optional opcode tracking. 39 * Except for the basic logical operations (and, or, xor), and 40 * data movement (mov, ld, st, dupi), many vector opcodes are 41 * optional and may not be supported on the host. Thank Intel 42 * for the irregularity in their instruction set. 43 * 44 * The gvec expanders allow custom vector operations to be composed, 45 * generally via the .fniv callback in the GVecGen* structures. At 46 * the same time, in deciding whether to use this hook we need to 47 * know if the host supports the required operations. This is 48 * presented as an array of opcodes, terminated by 0. Each opcode 49 * is assumed to be expanded with the given VECE. 50 * 51 * For debugging, we want to validate this array. Therefore, when 52 * tcg_ctx->vec_opt_opc is non-NULL, the tcg_gen_*_vec expanders 53 * will validate that their opcode is present in the list. 54 */ 55 #ifdef CONFIG_DEBUG_TCG 56 void tcg_assert_listed_vecop(TCGOpcode op) 57 { 58 const TCGOpcode *p = tcg_ctx->vecop_list; 59 if (p) { 60 for (; *p; ++p) { 61 if (*p == op) { 62 return; 63 } 64 } 65 g_assert_not_reached(); 66 } 67 } 68 #endif 69 70 bool tcg_can_emit_vecop_list(const TCGOpcode *list, 71 TCGType type, unsigned vece) 72 { 73 if (list == NULL) { 74 return true; 75 } 76 77 for (; *list; ++list) { 78 TCGOpcode opc = *list; 79 80 #ifdef CONFIG_DEBUG_TCG 81 switch (opc) { 82 case INDEX_op_and_vec: 83 case INDEX_op_or_vec: 84 case INDEX_op_xor_vec: 85 case INDEX_op_mov_vec: 86 case INDEX_op_dup_vec: 87 case INDEX_op_dupi_vec: 88 case INDEX_op_dup2_vec: 89 case INDEX_op_ld_vec: 90 case INDEX_op_st_vec: 91 case INDEX_op_bitsel_vec: 92 /* These opcodes are mandatory and should not be listed. */ 93 g_assert_not_reached(); 94 default: 95 break; 96 } 97 #endif 98 99 if (tcg_can_emit_vec_op(opc, type, vece)) { 100 continue; 101 } 102 103 /* 104 * The opcode list is created by front ends based on what they 105 * actually invoke. We must mirror the logic in the routines 106 * below for generic expansions using other opcodes. 107 */ 108 switch (opc) { 109 case INDEX_op_neg_vec: 110 if (tcg_can_emit_vec_op(INDEX_op_sub_vec, type, vece)) { 111 continue; 112 } 113 break; 114 case INDEX_op_abs_vec: 115 if (tcg_can_emit_vec_op(INDEX_op_sub_vec, type, vece) 116 && (tcg_can_emit_vec_op(INDEX_op_smax_vec, type, vece) > 0 117 || tcg_can_emit_vec_op(INDEX_op_sari_vec, type, vece) > 0 118 || tcg_can_emit_vec_op(INDEX_op_cmp_vec, type, vece))) { 119 continue; 120 } 121 break; 122 default: 123 break; 124 } 125 return false; 126 } 127 return true; 128 } 129 130 void vec_gen_2(TCGOpcode opc, TCGType type, unsigned vece, TCGArg r, TCGArg a) 131 { 132 TCGOp *op = tcg_emit_op(opc); 133 TCGOP_VECL(op) = type - TCG_TYPE_V64; 134 TCGOP_VECE(op) = vece; 135 op->args[0] = r; 136 op->args[1] = a; 137 } 138 139 void vec_gen_3(TCGOpcode opc, TCGType type, unsigned vece, 140 TCGArg r, TCGArg a, TCGArg b) 141 { 142 TCGOp *op = tcg_emit_op(opc); 143 TCGOP_VECL(op) = type - TCG_TYPE_V64; 144 TCGOP_VECE(op) = vece; 145 op->args[0] = r; 146 op->args[1] = a; 147 op->args[2] = b; 148 } 149 150 void vec_gen_4(TCGOpcode opc, TCGType type, unsigned vece, 151 TCGArg r, TCGArg a, TCGArg b, TCGArg c) 152 { 153 TCGOp *op = tcg_emit_op(opc); 154 TCGOP_VECL(op) = type - TCG_TYPE_V64; 155 TCGOP_VECE(op) = vece; 156 op->args[0] = r; 157 op->args[1] = a; 158 op->args[2] = b; 159 op->args[3] = c; 160 } 161 162 static void vec_gen_op2(TCGOpcode opc, unsigned vece, TCGv_vec r, TCGv_vec a) 163 { 164 TCGTemp *rt = tcgv_vec_temp(r); 165 TCGTemp *at = tcgv_vec_temp(a); 166 TCGType type = rt->base_type; 167 168 /* Must enough inputs for the output. */ 169 tcg_debug_assert(at->base_type >= type); 170 vec_gen_2(opc, type, vece, temp_arg(rt), temp_arg(at)); 171 } 172 173 static void vec_gen_op3(TCGOpcode opc, unsigned vece, 174 TCGv_vec r, TCGv_vec a, TCGv_vec b) 175 { 176 TCGTemp *rt = tcgv_vec_temp(r); 177 TCGTemp *at = tcgv_vec_temp(a); 178 TCGTemp *bt = tcgv_vec_temp(b); 179 TCGType type = rt->base_type; 180 181 /* Must enough inputs for the output. */ 182 tcg_debug_assert(at->base_type >= type); 183 tcg_debug_assert(bt->base_type >= type); 184 vec_gen_3(opc, type, vece, temp_arg(rt), temp_arg(at), temp_arg(bt)); 185 } 186 187 void tcg_gen_mov_vec(TCGv_vec r, TCGv_vec a) 188 { 189 if (r != a) { 190 vec_gen_op2(INDEX_op_mov_vec, 0, r, a); 191 } 192 } 193 194 #define MO_REG (TCG_TARGET_REG_BITS == 64 ? MO_64 : MO_32) 195 196 static void do_dupi_vec(TCGv_vec r, unsigned vece, TCGArg a) 197 { 198 TCGTemp *rt = tcgv_vec_temp(r); 199 vec_gen_2(INDEX_op_dupi_vec, rt->base_type, vece, temp_arg(rt), a); 200 } 201 202 TCGv_vec tcg_const_zeros_vec(TCGType type) 203 { 204 TCGv_vec ret = tcg_temp_new_vec(type); 205 do_dupi_vec(ret, MO_REG, 0); 206 return ret; 207 } 208 209 TCGv_vec tcg_const_ones_vec(TCGType type) 210 { 211 TCGv_vec ret = tcg_temp_new_vec(type); 212 do_dupi_vec(ret, MO_REG, -1); 213 return ret; 214 } 215 216 TCGv_vec tcg_const_zeros_vec_matching(TCGv_vec m) 217 { 218 TCGTemp *t = tcgv_vec_temp(m); 219 return tcg_const_zeros_vec(t->base_type); 220 } 221 222 TCGv_vec tcg_const_ones_vec_matching(TCGv_vec m) 223 { 224 TCGTemp *t = tcgv_vec_temp(m); 225 return tcg_const_ones_vec(t->base_type); 226 } 227 228 void tcg_gen_dup64i_vec(TCGv_vec r, uint64_t a) 229 { 230 if (TCG_TARGET_REG_BITS == 32 && a == deposit64(a, 32, 32, a)) { 231 do_dupi_vec(r, MO_32, a); 232 } else if (TCG_TARGET_REG_BITS == 64 || a == (uint64_t)(int32_t)a) { 233 do_dupi_vec(r, MO_64, a); 234 } else { 235 TCGv_i64 c = tcg_const_i64(a); 236 tcg_gen_dup_i64_vec(MO_64, r, c); 237 tcg_temp_free_i64(c); 238 } 239 } 240 241 void tcg_gen_dup32i_vec(TCGv_vec r, uint32_t a) 242 { 243 do_dupi_vec(r, MO_REG, dup_const(MO_32, a)); 244 } 245 246 void tcg_gen_dup16i_vec(TCGv_vec r, uint32_t a) 247 { 248 do_dupi_vec(r, MO_REG, dup_const(MO_16, a)); 249 } 250 251 void tcg_gen_dup8i_vec(TCGv_vec r, uint32_t a) 252 { 253 do_dupi_vec(r, MO_REG, dup_const(MO_8, a)); 254 } 255 256 void tcg_gen_dupi_vec(unsigned vece, TCGv_vec r, uint64_t a) 257 { 258 do_dupi_vec(r, MO_REG, dup_const(vece, a)); 259 } 260 261 void tcg_gen_dup_i64_vec(unsigned vece, TCGv_vec r, TCGv_i64 a) 262 { 263 TCGArg ri = tcgv_vec_arg(r); 264 TCGTemp *rt = arg_temp(ri); 265 TCGType type = rt->base_type; 266 267 if (TCG_TARGET_REG_BITS == 64) { 268 TCGArg ai = tcgv_i64_arg(a); 269 vec_gen_2(INDEX_op_dup_vec, type, vece, ri, ai); 270 } else if (vece == MO_64) { 271 TCGArg al = tcgv_i32_arg(TCGV_LOW(a)); 272 TCGArg ah = tcgv_i32_arg(TCGV_HIGH(a)); 273 vec_gen_3(INDEX_op_dup2_vec, type, MO_64, ri, al, ah); 274 } else { 275 TCGArg ai = tcgv_i32_arg(TCGV_LOW(a)); 276 vec_gen_2(INDEX_op_dup_vec, type, vece, ri, ai); 277 } 278 } 279 280 void tcg_gen_dup_i32_vec(unsigned vece, TCGv_vec r, TCGv_i32 a) 281 { 282 TCGArg ri = tcgv_vec_arg(r); 283 TCGArg ai = tcgv_i32_arg(a); 284 TCGTemp *rt = arg_temp(ri); 285 TCGType type = rt->base_type; 286 287 vec_gen_2(INDEX_op_dup_vec, type, vece, ri, ai); 288 } 289 290 void tcg_gen_dup_mem_vec(unsigned vece, TCGv_vec r, TCGv_ptr b, 291 tcg_target_long ofs) 292 { 293 TCGArg ri = tcgv_vec_arg(r); 294 TCGArg bi = tcgv_ptr_arg(b); 295 TCGTemp *rt = arg_temp(ri); 296 TCGType type = rt->base_type; 297 298 vec_gen_3(INDEX_op_dupm_vec, type, vece, ri, bi, ofs); 299 } 300 301 static void vec_gen_ldst(TCGOpcode opc, TCGv_vec r, TCGv_ptr b, TCGArg o) 302 { 303 TCGArg ri = tcgv_vec_arg(r); 304 TCGArg bi = tcgv_ptr_arg(b); 305 TCGTemp *rt = arg_temp(ri); 306 TCGType type = rt->base_type; 307 308 vec_gen_3(opc, type, 0, ri, bi, o); 309 } 310 311 void tcg_gen_ld_vec(TCGv_vec r, TCGv_ptr b, TCGArg o) 312 { 313 vec_gen_ldst(INDEX_op_ld_vec, r, b, o); 314 } 315 316 void tcg_gen_st_vec(TCGv_vec r, TCGv_ptr b, TCGArg o) 317 { 318 vec_gen_ldst(INDEX_op_st_vec, r, b, o); 319 } 320 321 void tcg_gen_stl_vec(TCGv_vec r, TCGv_ptr b, TCGArg o, TCGType low_type) 322 { 323 TCGArg ri = tcgv_vec_arg(r); 324 TCGArg bi = tcgv_ptr_arg(b); 325 TCGTemp *rt = arg_temp(ri); 326 TCGType type = rt->base_type; 327 328 tcg_debug_assert(low_type >= TCG_TYPE_V64); 329 tcg_debug_assert(low_type <= type); 330 vec_gen_3(INDEX_op_st_vec, low_type, 0, ri, bi, o); 331 } 332 333 void tcg_gen_and_vec(unsigned vece, TCGv_vec r, TCGv_vec a, TCGv_vec b) 334 { 335 vec_gen_op3(INDEX_op_and_vec, 0, r, a, b); 336 } 337 338 void tcg_gen_or_vec(unsigned vece, TCGv_vec r, TCGv_vec a, TCGv_vec b) 339 { 340 vec_gen_op3(INDEX_op_or_vec, 0, r, a, b); 341 } 342 343 void tcg_gen_xor_vec(unsigned vece, TCGv_vec r, TCGv_vec a, TCGv_vec b) 344 { 345 vec_gen_op3(INDEX_op_xor_vec, 0, r, a, b); 346 } 347 348 void tcg_gen_andc_vec(unsigned vece, TCGv_vec r, TCGv_vec a, TCGv_vec b) 349 { 350 if (TCG_TARGET_HAS_andc_vec) { 351 vec_gen_op3(INDEX_op_andc_vec, 0, r, a, b); 352 } else { 353 TCGv_vec t = tcg_temp_new_vec_matching(r); 354 tcg_gen_not_vec(0, t, b); 355 tcg_gen_and_vec(0, r, a, t); 356 tcg_temp_free_vec(t); 357 } 358 } 359 360 void tcg_gen_orc_vec(unsigned vece, TCGv_vec r, TCGv_vec a, TCGv_vec b) 361 { 362 if (TCG_TARGET_HAS_orc_vec) { 363 vec_gen_op3(INDEX_op_orc_vec, 0, r, a, b); 364 } else { 365 TCGv_vec t = tcg_temp_new_vec_matching(r); 366 tcg_gen_not_vec(0, t, b); 367 tcg_gen_or_vec(0, r, a, t); 368 tcg_temp_free_vec(t); 369 } 370 } 371 372 void tcg_gen_nand_vec(unsigned vece, TCGv_vec r, TCGv_vec a, TCGv_vec b) 373 { 374 /* TODO: Add TCG_TARGET_HAS_nand_vec when adding a backend supports it. */ 375 tcg_gen_and_vec(0, r, a, b); 376 tcg_gen_not_vec(0, r, r); 377 } 378 379 void tcg_gen_nor_vec(unsigned vece, TCGv_vec r, TCGv_vec a, TCGv_vec b) 380 { 381 /* TODO: Add TCG_TARGET_HAS_nor_vec when adding a backend supports it. */ 382 tcg_gen_or_vec(0, r, a, b); 383 tcg_gen_not_vec(0, r, r); 384 } 385 386 void tcg_gen_eqv_vec(unsigned vece, TCGv_vec r, TCGv_vec a, TCGv_vec b) 387 { 388 /* TODO: Add TCG_TARGET_HAS_eqv_vec when adding a backend supports it. */ 389 tcg_gen_xor_vec(0, r, a, b); 390 tcg_gen_not_vec(0, r, r); 391 } 392 393 static bool do_op2(unsigned vece, TCGv_vec r, TCGv_vec a, TCGOpcode opc) 394 { 395 TCGTemp *rt = tcgv_vec_temp(r); 396 TCGTemp *at = tcgv_vec_temp(a); 397 TCGArg ri = temp_arg(rt); 398 TCGArg ai = temp_arg(at); 399 TCGType type = rt->base_type; 400 int can; 401 402 tcg_debug_assert(at->base_type >= type); 403 tcg_assert_listed_vecop(opc); 404 can = tcg_can_emit_vec_op(opc, type, vece); 405 if (can > 0) { 406 vec_gen_2(opc, type, vece, ri, ai); 407 } else if (can < 0) { 408 const TCGOpcode *hold_list = tcg_swap_vecop_list(NULL); 409 tcg_expand_vec_op(opc, type, vece, ri, ai); 410 tcg_swap_vecop_list(hold_list); 411 } else { 412 return false; 413 } 414 return true; 415 } 416 417 void tcg_gen_not_vec(unsigned vece, TCGv_vec r, TCGv_vec a) 418 { 419 if (!TCG_TARGET_HAS_not_vec || !do_op2(vece, r, a, INDEX_op_not_vec)) { 420 TCGv_vec t = tcg_const_ones_vec_matching(r); 421 tcg_gen_xor_vec(0, r, a, t); 422 tcg_temp_free_vec(t); 423 } 424 } 425 426 void tcg_gen_neg_vec(unsigned vece, TCGv_vec r, TCGv_vec a) 427 { 428 const TCGOpcode *hold_list; 429 430 tcg_assert_listed_vecop(INDEX_op_neg_vec); 431 hold_list = tcg_swap_vecop_list(NULL); 432 433 if (!TCG_TARGET_HAS_neg_vec || !do_op2(vece, r, a, INDEX_op_neg_vec)) { 434 TCGv_vec t = tcg_const_zeros_vec_matching(r); 435 tcg_gen_sub_vec(vece, r, t, a); 436 tcg_temp_free_vec(t); 437 } 438 tcg_swap_vecop_list(hold_list); 439 } 440 441 void tcg_gen_abs_vec(unsigned vece, TCGv_vec r, TCGv_vec a) 442 { 443 const TCGOpcode *hold_list; 444 445 tcg_assert_listed_vecop(INDEX_op_abs_vec); 446 hold_list = tcg_swap_vecop_list(NULL); 447 448 if (!do_op2(vece, r, a, INDEX_op_abs_vec)) { 449 TCGType type = tcgv_vec_temp(r)->base_type; 450 TCGv_vec t = tcg_temp_new_vec(type); 451 452 tcg_debug_assert(tcg_can_emit_vec_op(INDEX_op_sub_vec, type, vece)); 453 if (tcg_can_emit_vec_op(INDEX_op_smax_vec, type, vece) > 0) { 454 tcg_gen_neg_vec(vece, t, a); 455 tcg_gen_smax_vec(vece, r, a, t); 456 } else { 457 if (tcg_can_emit_vec_op(INDEX_op_sari_vec, type, vece) > 0) { 458 tcg_gen_sari_vec(vece, t, a, (8 << vece) - 1); 459 } else { 460 do_dupi_vec(t, MO_REG, 0); 461 tcg_gen_cmp_vec(TCG_COND_LT, vece, t, a, t); 462 } 463 tcg_gen_xor_vec(vece, r, a, t); 464 tcg_gen_sub_vec(vece, r, r, t); 465 } 466 467 tcg_temp_free_vec(t); 468 } 469 tcg_swap_vecop_list(hold_list); 470 } 471 472 static void do_shifti(TCGOpcode opc, unsigned vece, 473 TCGv_vec r, TCGv_vec a, int64_t i) 474 { 475 TCGTemp *rt = tcgv_vec_temp(r); 476 TCGTemp *at = tcgv_vec_temp(a); 477 TCGArg ri = temp_arg(rt); 478 TCGArg ai = temp_arg(at); 479 TCGType type = rt->base_type; 480 int can; 481 482 tcg_debug_assert(at->base_type == type); 483 tcg_debug_assert(i >= 0 && i < (8 << vece)); 484 tcg_assert_listed_vecop(opc); 485 486 if (i == 0) { 487 tcg_gen_mov_vec(r, a); 488 return; 489 } 490 491 can = tcg_can_emit_vec_op(opc, type, vece); 492 if (can > 0) { 493 vec_gen_3(opc, type, vece, ri, ai, i); 494 } else { 495 /* We leave the choice of expansion via scalar or vector shift 496 to the target. Often, but not always, dupi can feed a vector 497 shift easier than a scalar. */ 498 const TCGOpcode *hold_list = tcg_swap_vecop_list(NULL); 499 tcg_debug_assert(can < 0); 500 tcg_expand_vec_op(opc, type, vece, ri, ai, i); 501 tcg_swap_vecop_list(hold_list); 502 } 503 } 504 505 void tcg_gen_shli_vec(unsigned vece, TCGv_vec r, TCGv_vec a, int64_t i) 506 { 507 do_shifti(INDEX_op_shli_vec, vece, r, a, i); 508 } 509 510 void tcg_gen_shri_vec(unsigned vece, TCGv_vec r, TCGv_vec a, int64_t i) 511 { 512 do_shifti(INDEX_op_shri_vec, vece, r, a, i); 513 } 514 515 void tcg_gen_sari_vec(unsigned vece, TCGv_vec r, TCGv_vec a, int64_t i) 516 { 517 do_shifti(INDEX_op_sari_vec, vece, r, a, i); 518 } 519 520 void tcg_gen_cmp_vec(TCGCond cond, unsigned vece, 521 TCGv_vec r, TCGv_vec a, TCGv_vec b) 522 { 523 TCGTemp *rt = tcgv_vec_temp(r); 524 TCGTemp *at = tcgv_vec_temp(a); 525 TCGTemp *bt = tcgv_vec_temp(b); 526 TCGArg ri = temp_arg(rt); 527 TCGArg ai = temp_arg(at); 528 TCGArg bi = temp_arg(bt); 529 TCGType type = rt->base_type; 530 int can; 531 532 tcg_debug_assert(at->base_type >= type); 533 tcg_debug_assert(bt->base_type >= type); 534 tcg_assert_listed_vecop(INDEX_op_cmp_vec); 535 can = tcg_can_emit_vec_op(INDEX_op_cmp_vec, type, vece); 536 if (can > 0) { 537 vec_gen_4(INDEX_op_cmp_vec, type, vece, ri, ai, bi, cond); 538 } else { 539 const TCGOpcode *hold_list = tcg_swap_vecop_list(NULL); 540 tcg_debug_assert(can < 0); 541 tcg_expand_vec_op(INDEX_op_cmp_vec, type, vece, ri, ai, bi, cond); 542 tcg_swap_vecop_list(hold_list); 543 } 544 } 545 546 static void do_op3(unsigned vece, TCGv_vec r, TCGv_vec a, 547 TCGv_vec b, TCGOpcode opc) 548 { 549 TCGTemp *rt = tcgv_vec_temp(r); 550 TCGTemp *at = tcgv_vec_temp(a); 551 TCGTemp *bt = tcgv_vec_temp(b); 552 TCGArg ri = temp_arg(rt); 553 TCGArg ai = temp_arg(at); 554 TCGArg bi = temp_arg(bt); 555 TCGType type = rt->base_type; 556 int can; 557 558 tcg_debug_assert(at->base_type >= type); 559 tcg_debug_assert(bt->base_type >= type); 560 tcg_assert_listed_vecop(opc); 561 can = tcg_can_emit_vec_op(opc, type, vece); 562 if (can > 0) { 563 vec_gen_3(opc, type, vece, ri, ai, bi); 564 } else { 565 const TCGOpcode *hold_list = tcg_swap_vecop_list(NULL); 566 tcg_debug_assert(can < 0); 567 tcg_expand_vec_op(opc, type, vece, ri, ai, bi); 568 tcg_swap_vecop_list(hold_list); 569 } 570 } 571 572 void tcg_gen_add_vec(unsigned vece, TCGv_vec r, TCGv_vec a, TCGv_vec b) 573 { 574 do_op3(vece, r, a, b, INDEX_op_add_vec); 575 } 576 577 void tcg_gen_sub_vec(unsigned vece, TCGv_vec r, TCGv_vec a, TCGv_vec b) 578 { 579 do_op3(vece, r, a, b, INDEX_op_sub_vec); 580 } 581 582 void tcg_gen_mul_vec(unsigned vece, TCGv_vec r, TCGv_vec a, TCGv_vec b) 583 { 584 do_op3(vece, r, a, b, INDEX_op_mul_vec); 585 } 586 587 void tcg_gen_ssadd_vec(unsigned vece, TCGv_vec r, TCGv_vec a, TCGv_vec b) 588 { 589 do_op3(vece, r, a, b, INDEX_op_ssadd_vec); 590 } 591 592 void tcg_gen_usadd_vec(unsigned vece, TCGv_vec r, TCGv_vec a, TCGv_vec b) 593 { 594 do_op3(vece, r, a, b, INDEX_op_usadd_vec); 595 } 596 597 void tcg_gen_sssub_vec(unsigned vece, TCGv_vec r, TCGv_vec a, TCGv_vec b) 598 { 599 do_op3(vece, r, a, b, INDEX_op_sssub_vec); 600 } 601 602 void tcg_gen_ussub_vec(unsigned vece, TCGv_vec r, TCGv_vec a, TCGv_vec b) 603 { 604 do_op3(vece, r, a, b, INDEX_op_ussub_vec); 605 } 606 607 void tcg_gen_smin_vec(unsigned vece, TCGv_vec r, TCGv_vec a, TCGv_vec b) 608 { 609 do_op3(vece, r, a, b, INDEX_op_smin_vec); 610 } 611 612 void tcg_gen_umin_vec(unsigned vece, TCGv_vec r, TCGv_vec a, TCGv_vec b) 613 { 614 do_op3(vece, r, a, b, INDEX_op_umin_vec); 615 } 616 617 void tcg_gen_smax_vec(unsigned vece, TCGv_vec r, TCGv_vec a, TCGv_vec b) 618 { 619 do_op3(vece, r, a, b, INDEX_op_smax_vec); 620 } 621 622 void tcg_gen_umax_vec(unsigned vece, TCGv_vec r, TCGv_vec a, TCGv_vec b) 623 { 624 do_op3(vece, r, a, b, INDEX_op_umax_vec); 625 } 626 627 void tcg_gen_shlv_vec(unsigned vece, TCGv_vec r, TCGv_vec a, TCGv_vec b) 628 { 629 do_op3(vece, r, a, b, INDEX_op_shlv_vec); 630 } 631 632 void tcg_gen_shrv_vec(unsigned vece, TCGv_vec r, TCGv_vec a, TCGv_vec b) 633 { 634 do_op3(vece, r, a, b, INDEX_op_shrv_vec); 635 } 636 637 void tcg_gen_sarv_vec(unsigned vece, TCGv_vec r, TCGv_vec a, TCGv_vec b) 638 { 639 do_op3(vece, r, a, b, INDEX_op_sarv_vec); 640 } 641 642 static void do_shifts(unsigned vece, TCGv_vec r, TCGv_vec a, 643 TCGv_i32 s, TCGOpcode opc_s, TCGOpcode opc_v) 644 { 645 TCGTemp *rt = tcgv_vec_temp(r); 646 TCGTemp *at = tcgv_vec_temp(a); 647 TCGTemp *st = tcgv_i32_temp(s); 648 TCGArg ri = temp_arg(rt); 649 TCGArg ai = temp_arg(at); 650 TCGArg si = temp_arg(st); 651 TCGType type = rt->base_type; 652 const TCGOpcode *hold_list; 653 int can; 654 655 tcg_debug_assert(at->base_type >= type); 656 tcg_assert_listed_vecop(opc_s); 657 hold_list = tcg_swap_vecop_list(NULL); 658 659 can = tcg_can_emit_vec_op(opc_s, type, vece); 660 if (can > 0) { 661 vec_gen_3(opc_s, type, vece, ri, ai, si); 662 } else if (can < 0) { 663 tcg_expand_vec_op(opc_s, type, vece, ri, ai, si); 664 } else { 665 TCGv_vec vec_s = tcg_temp_new_vec(type); 666 667 if (vece == MO_64) { 668 TCGv_i64 s64 = tcg_temp_new_i64(); 669 tcg_gen_extu_i32_i64(s64, s); 670 tcg_gen_dup_i64_vec(MO_64, vec_s, s64); 671 tcg_temp_free_i64(s64); 672 } else { 673 tcg_gen_dup_i32_vec(vece, vec_s, s); 674 } 675 do_op3(vece, r, a, vec_s, opc_v); 676 tcg_temp_free_vec(vec_s); 677 } 678 tcg_swap_vecop_list(hold_list); 679 } 680 681 void tcg_gen_shls_vec(unsigned vece, TCGv_vec r, TCGv_vec a, TCGv_i32 b) 682 { 683 do_shifts(vece, r, a, b, INDEX_op_shls_vec, INDEX_op_shlv_vec); 684 } 685 686 void tcg_gen_shrs_vec(unsigned vece, TCGv_vec r, TCGv_vec a, TCGv_i32 b) 687 { 688 do_shifts(vece, r, a, b, INDEX_op_shrs_vec, INDEX_op_shrv_vec); 689 } 690 691 void tcg_gen_sars_vec(unsigned vece, TCGv_vec r, TCGv_vec a, TCGv_i32 b) 692 { 693 do_shifts(vece, r, a, b, INDEX_op_sars_vec, INDEX_op_sarv_vec); 694 } 695 696 void tcg_gen_bitsel_vec(unsigned vece, TCGv_vec r, TCGv_vec a, 697 TCGv_vec b, TCGv_vec c) 698 { 699 TCGTemp *rt = tcgv_vec_temp(r); 700 TCGTemp *at = tcgv_vec_temp(a); 701 TCGTemp *bt = tcgv_vec_temp(b); 702 TCGTemp *ct = tcgv_vec_temp(c); 703 TCGType type = rt->base_type; 704 705 tcg_debug_assert(at->base_type >= type); 706 tcg_debug_assert(bt->base_type >= type); 707 tcg_debug_assert(ct->base_type >= type); 708 709 if (TCG_TARGET_HAS_bitsel_vec) { 710 vec_gen_4(INDEX_op_bitsel_vec, type, MO_8, 711 temp_arg(rt), temp_arg(at), temp_arg(bt), temp_arg(ct)); 712 } else { 713 TCGv_vec t = tcg_temp_new_vec(type); 714 tcg_gen_and_vec(MO_8, t, a, b); 715 tcg_gen_andc_vec(MO_8, r, c, a); 716 tcg_gen_or_vec(MO_8, r, r, t); 717 tcg_temp_free_vec(t); 718 } 719 } 720