1 /* 2 * Tiny Code Generator for QEMU 3 * 4 * Copyright (c) 2018 Linaro, Inc. 5 * 6 * This library is free software; you can redistribute it and/or 7 * modify it under the terms of the GNU Lesser General Public 8 * License as published by the Free Software Foundation; either 9 * version 2.1 of the License, or (at your option) any later version. 10 * 11 * This library is distributed in the hope that it will be useful, 12 * but WITHOUT ANY WARRANTY; without even the implied warranty of 13 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU 14 * Lesser General Public License for more details. 15 * 16 * You should have received a copy of the GNU Lesser General Public 17 * License along with this library; if not, see <http://www.gnu.org/licenses/>. 18 */ 19 20 #include "qemu/osdep.h" 21 #include "qemu-common.h" 22 #include "cpu.h" 23 #include "tcg.h" 24 #include "tcg-op.h" 25 #include "tcg-mo.h" 26 27 /* Reduce the number of ifdefs below. This assumes that all uses of 28 TCGV_HIGH and TCGV_LOW are properly protected by a conditional that 29 the compiler can eliminate. */ 30 #if TCG_TARGET_REG_BITS == 64 31 extern TCGv_i32 TCGV_LOW_link_error(TCGv_i64); 32 extern TCGv_i32 TCGV_HIGH_link_error(TCGv_i64); 33 #define TCGV_LOW TCGV_LOW_link_error 34 #define TCGV_HIGH TCGV_HIGH_link_error 35 #endif 36 37 /* 38 * Vector optional opcode tracking. 39 * Except for the basic logical operations (and, or, xor), and 40 * data movement (mov, ld, st, dupi), many vector opcodes are 41 * optional and may not be supported on the host. Thank Intel 42 * for the irregularity in their instruction set. 43 * 44 * The gvec expanders allow custom vector operations to be composed, 45 * generally via the .fniv callback in the GVecGen* structures. At 46 * the same time, in deciding whether to use this hook we need to 47 * know if the host supports the required operations. This is 48 * presented as an array of opcodes, terminated by 0. Each opcode 49 * is assumed to be expanded with the given VECE. 50 * 51 * For debugging, we want to validate this array. Therefore, when 52 * tcg_ctx->vec_opt_opc is non-NULL, the tcg_gen_*_vec expanders 53 * will validate that their opcode is present in the list. 54 */ 55 #ifdef CONFIG_DEBUG_TCG 56 void tcg_assert_listed_vecop(TCGOpcode op) 57 { 58 const TCGOpcode *p = tcg_ctx->vecop_list; 59 if (p) { 60 for (; *p; ++p) { 61 if (*p == op) { 62 return; 63 } 64 } 65 g_assert_not_reached(); 66 } 67 } 68 #endif 69 70 bool tcg_can_emit_vecop_list(const TCGOpcode *list, 71 TCGType type, unsigned vece) 72 { 73 if (list == NULL) { 74 return true; 75 } 76 77 for (; *list; ++list) { 78 TCGOpcode opc = *list; 79 80 #ifdef CONFIG_DEBUG_TCG 81 switch (opc) { 82 case INDEX_op_and_vec: 83 case INDEX_op_or_vec: 84 case INDEX_op_xor_vec: 85 case INDEX_op_mov_vec: 86 case INDEX_op_dup_vec: 87 case INDEX_op_dupi_vec: 88 case INDEX_op_dup2_vec: 89 case INDEX_op_ld_vec: 90 case INDEX_op_st_vec: 91 /* These opcodes are mandatory and should not be listed. */ 92 g_assert_not_reached(); 93 default: 94 break; 95 } 96 #endif 97 98 if (tcg_can_emit_vec_op(opc, type, vece)) { 99 continue; 100 } 101 102 /* 103 * The opcode list is created by front ends based on what they 104 * actually invoke. We must mirror the logic in the routines 105 * below for generic expansions using other opcodes. 106 */ 107 switch (opc) { 108 case INDEX_op_neg_vec: 109 if (tcg_can_emit_vec_op(INDEX_op_sub_vec, type, vece)) { 110 continue; 111 } 112 break; 113 case INDEX_op_abs_vec: 114 if (tcg_can_emit_vec_op(INDEX_op_sub_vec, type, vece) 115 && (tcg_can_emit_vec_op(INDEX_op_smax_vec, type, vece) > 0 116 || tcg_can_emit_vec_op(INDEX_op_sari_vec, type, vece) > 0 117 || tcg_can_emit_vec_op(INDEX_op_cmp_vec, type, vece))) { 118 continue; 119 } 120 break; 121 default: 122 break; 123 } 124 return false; 125 } 126 return true; 127 } 128 129 void vec_gen_2(TCGOpcode opc, TCGType type, unsigned vece, TCGArg r, TCGArg a) 130 { 131 TCGOp *op = tcg_emit_op(opc); 132 TCGOP_VECL(op) = type - TCG_TYPE_V64; 133 TCGOP_VECE(op) = vece; 134 op->args[0] = r; 135 op->args[1] = a; 136 } 137 138 void vec_gen_3(TCGOpcode opc, TCGType type, unsigned vece, 139 TCGArg r, TCGArg a, TCGArg b) 140 { 141 TCGOp *op = tcg_emit_op(opc); 142 TCGOP_VECL(op) = type - TCG_TYPE_V64; 143 TCGOP_VECE(op) = vece; 144 op->args[0] = r; 145 op->args[1] = a; 146 op->args[2] = b; 147 } 148 149 void vec_gen_4(TCGOpcode opc, TCGType type, unsigned vece, 150 TCGArg r, TCGArg a, TCGArg b, TCGArg c) 151 { 152 TCGOp *op = tcg_emit_op(opc); 153 TCGOP_VECL(op) = type - TCG_TYPE_V64; 154 TCGOP_VECE(op) = vece; 155 op->args[0] = r; 156 op->args[1] = a; 157 op->args[2] = b; 158 op->args[3] = c; 159 } 160 161 static void vec_gen_op2(TCGOpcode opc, unsigned vece, TCGv_vec r, TCGv_vec a) 162 { 163 TCGTemp *rt = tcgv_vec_temp(r); 164 TCGTemp *at = tcgv_vec_temp(a); 165 TCGType type = rt->base_type; 166 167 /* Must enough inputs for the output. */ 168 tcg_debug_assert(at->base_type >= type); 169 vec_gen_2(opc, type, vece, temp_arg(rt), temp_arg(at)); 170 } 171 172 static void vec_gen_op3(TCGOpcode opc, unsigned vece, 173 TCGv_vec r, TCGv_vec a, TCGv_vec b) 174 { 175 TCGTemp *rt = tcgv_vec_temp(r); 176 TCGTemp *at = tcgv_vec_temp(a); 177 TCGTemp *bt = tcgv_vec_temp(b); 178 TCGType type = rt->base_type; 179 180 /* Must enough inputs for the output. */ 181 tcg_debug_assert(at->base_type >= type); 182 tcg_debug_assert(bt->base_type >= type); 183 vec_gen_3(opc, type, vece, temp_arg(rt), temp_arg(at), temp_arg(bt)); 184 } 185 186 void tcg_gen_mov_vec(TCGv_vec r, TCGv_vec a) 187 { 188 if (r != a) { 189 vec_gen_op2(INDEX_op_mov_vec, 0, r, a); 190 } 191 } 192 193 #define MO_REG (TCG_TARGET_REG_BITS == 64 ? MO_64 : MO_32) 194 195 static void do_dupi_vec(TCGv_vec r, unsigned vece, TCGArg a) 196 { 197 TCGTemp *rt = tcgv_vec_temp(r); 198 vec_gen_2(INDEX_op_dupi_vec, rt->base_type, vece, temp_arg(rt), a); 199 } 200 201 TCGv_vec tcg_const_zeros_vec(TCGType type) 202 { 203 TCGv_vec ret = tcg_temp_new_vec(type); 204 do_dupi_vec(ret, MO_REG, 0); 205 return ret; 206 } 207 208 TCGv_vec tcg_const_ones_vec(TCGType type) 209 { 210 TCGv_vec ret = tcg_temp_new_vec(type); 211 do_dupi_vec(ret, MO_REG, -1); 212 return ret; 213 } 214 215 TCGv_vec tcg_const_zeros_vec_matching(TCGv_vec m) 216 { 217 TCGTemp *t = tcgv_vec_temp(m); 218 return tcg_const_zeros_vec(t->base_type); 219 } 220 221 TCGv_vec tcg_const_ones_vec_matching(TCGv_vec m) 222 { 223 TCGTemp *t = tcgv_vec_temp(m); 224 return tcg_const_ones_vec(t->base_type); 225 } 226 227 void tcg_gen_dup64i_vec(TCGv_vec r, uint64_t a) 228 { 229 if (TCG_TARGET_REG_BITS == 32 && a == deposit64(a, 32, 32, a)) { 230 do_dupi_vec(r, MO_32, a); 231 } else if (TCG_TARGET_REG_BITS == 64 || a == (uint64_t)(int32_t)a) { 232 do_dupi_vec(r, MO_64, a); 233 } else { 234 TCGv_i64 c = tcg_const_i64(a); 235 tcg_gen_dup_i64_vec(MO_64, r, c); 236 tcg_temp_free_i64(c); 237 } 238 } 239 240 void tcg_gen_dup32i_vec(TCGv_vec r, uint32_t a) 241 { 242 do_dupi_vec(r, MO_REG, dup_const(MO_32, a)); 243 } 244 245 void tcg_gen_dup16i_vec(TCGv_vec r, uint32_t a) 246 { 247 do_dupi_vec(r, MO_REG, dup_const(MO_16, a)); 248 } 249 250 void tcg_gen_dup8i_vec(TCGv_vec r, uint32_t a) 251 { 252 do_dupi_vec(r, MO_REG, dup_const(MO_8, a)); 253 } 254 255 void tcg_gen_dupi_vec(unsigned vece, TCGv_vec r, uint64_t a) 256 { 257 do_dupi_vec(r, MO_REG, dup_const(vece, a)); 258 } 259 260 void tcg_gen_dup_i64_vec(unsigned vece, TCGv_vec r, TCGv_i64 a) 261 { 262 TCGArg ri = tcgv_vec_arg(r); 263 TCGTemp *rt = arg_temp(ri); 264 TCGType type = rt->base_type; 265 266 if (TCG_TARGET_REG_BITS == 64) { 267 TCGArg ai = tcgv_i64_arg(a); 268 vec_gen_2(INDEX_op_dup_vec, type, vece, ri, ai); 269 } else if (vece == MO_64) { 270 TCGArg al = tcgv_i32_arg(TCGV_LOW(a)); 271 TCGArg ah = tcgv_i32_arg(TCGV_HIGH(a)); 272 vec_gen_3(INDEX_op_dup2_vec, type, MO_64, ri, al, ah); 273 } else { 274 TCGArg ai = tcgv_i32_arg(TCGV_LOW(a)); 275 vec_gen_2(INDEX_op_dup_vec, type, vece, ri, ai); 276 } 277 } 278 279 void tcg_gen_dup_i32_vec(unsigned vece, TCGv_vec r, TCGv_i32 a) 280 { 281 TCGArg ri = tcgv_vec_arg(r); 282 TCGArg ai = tcgv_i32_arg(a); 283 TCGTemp *rt = arg_temp(ri); 284 TCGType type = rt->base_type; 285 286 vec_gen_2(INDEX_op_dup_vec, type, vece, ri, ai); 287 } 288 289 void tcg_gen_dup_mem_vec(unsigned vece, TCGv_vec r, TCGv_ptr b, 290 tcg_target_long ofs) 291 { 292 TCGArg ri = tcgv_vec_arg(r); 293 TCGArg bi = tcgv_ptr_arg(b); 294 TCGTemp *rt = arg_temp(ri); 295 TCGType type = rt->base_type; 296 297 vec_gen_3(INDEX_op_dupm_vec, type, vece, ri, bi, ofs); 298 } 299 300 static void vec_gen_ldst(TCGOpcode opc, TCGv_vec r, TCGv_ptr b, TCGArg o) 301 { 302 TCGArg ri = tcgv_vec_arg(r); 303 TCGArg bi = tcgv_ptr_arg(b); 304 TCGTemp *rt = arg_temp(ri); 305 TCGType type = rt->base_type; 306 307 vec_gen_3(opc, type, 0, ri, bi, o); 308 } 309 310 void tcg_gen_ld_vec(TCGv_vec r, TCGv_ptr b, TCGArg o) 311 { 312 vec_gen_ldst(INDEX_op_ld_vec, r, b, o); 313 } 314 315 void tcg_gen_st_vec(TCGv_vec r, TCGv_ptr b, TCGArg o) 316 { 317 vec_gen_ldst(INDEX_op_st_vec, r, b, o); 318 } 319 320 void tcg_gen_stl_vec(TCGv_vec r, TCGv_ptr b, TCGArg o, TCGType low_type) 321 { 322 TCGArg ri = tcgv_vec_arg(r); 323 TCGArg bi = tcgv_ptr_arg(b); 324 TCGTemp *rt = arg_temp(ri); 325 TCGType type = rt->base_type; 326 327 tcg_debug_assert(low_type >= TCG_TYPE_V64); 328 tcg_debug_assert(low_type <= type); 329 vec_gen_3(INDEX_op_st_vec, low_type, 0, ri, bi, o); 330 } 331 332 void tcg_gen_and_vec(unsigned vece, TCGv_vec r, TCGv_vec a, TCGv_vec b) 333 { 334 vec_gen_op3(INDEX_op_and_vec, 0, r, a, b); 335 } 336 337 void tcg_gen_or_vec(unsigned vece, TCGv_vec r, TCGv_vec a, TCGv_vec b) 338 { 339 vec_gen_op3(INDEX_op_or_vec, 0, r, a, b); 340 } 341 342 void tcg_gen_xor_vec(unsigned vece, TCGv_vec r, TCGv_vec a, TCGv_vec b) 343 { 344 vec_gen_op3(INDEX_op_xor_vec, 0, r, a, b); 345 } 346 347 void tcg_gen_andc_vec(unsigned vece, TCGv_vec r, TCGv_vec a, TCGv_vec b) 348 { 349 if (TCG_TARGET_HAS_andc_vec) { 350 vec_gen_op3(INDEX_op_andc_vec, 0, r, a, b); 351 } else { 352 TCGv_vec t = tcg_temp_new_vec_matching(r); 353 tcg_gen_not_vec(0, t, b); 354 tcg_gen_and_vec(0, r, a, t); 355 tcg_temp_free_vec(t); 356 } 357 } 358 359 void tcg_gen_orc_vec(unsigned vece, TCGv_vec r, TCGv_vec a, TCGv_vec b) 360 { 361 if (TCG_TARGET_HAS_orc_vec) { 362 vec_gen_op3(INDEX_op_orc_vec, 0, r, a, b); 363 } else { 364 TCGv_vec t = tcg_temp_new_vec_matching(r); 365 tcg_gen_not_vec(0, t, b); 366 tcg_gen_or_vec(0, r, a, t); 367 tcg_temp_free_vec(t); 368 } 369 } 370 371 void tcg_gen_nand_vec(unsigned vece, TCGv_vec r, TCGv_vec a, TCGv_vec b) 372 { 373 /* TODO: Add TCG_TARGET_HAS_nand_vec when adding a backend supports it. */ 374 tcg_gen_and_vec(0, r, a, b); 375 tcg_gen_not_vec(0, r, r); 376 } 377 378 void tcg_gen_nor_vec(unsigned vece, TCGv_vec r, TCGv_vec a, TCGv_vec b) 379 { 380 /* TODO: Add TCG_TARGET_HAS_nor_vec when adding a backend supports it. */ 381 tcg_gen_or_vec(0, r, a, b); 382 tcg_gen_not_vec(0, r, r); 383 } 384 385 void tcg_gen_eqv_vec(unsigned vece, TCGv_vec r, TCGv_vec a, TCGv_vec b) 386 { 387 /* TODO: Add TCG_TARGET_HAS_eqv_vec when adding a backend supports it. */ 388 tcg_gen_xor_vec(0, r, a, b); 389 tcg_gen_not_vec(0, r, r); 390 } 391 392 static bool do_op2(unsigned vece, TCGv_vec r, TCGv_vec a, TCGOpcode opc) 393 { 394 TCGTemp *rt = tcgv_vec_temp(r); 395 TCGTemp *at = tcgv_vec_temp(a); 396 TCGArg ri = temp_arg(rt); 397 TCGArg ai = temp_arg(at); 398 TCGType type = rt->base_type; 399 int can; 400 401 tcg_debug_assert(at->base_type >= type); 402 tcg_assert_listed_vecop(opc); 403 can = tcg_can_emit_vec_op(opc, type, vece); 404 if (can > 0) { 405 vec_gen_2(opc, type, vece, ri, ai); 406 } else if (can < 0) { 407 const TCGOpcode *hold_list = tcg_swap_vecop_list(NULL); 408 tcg_expand_vec_op(opc, type, vece, ri, ai); 409 tcg_swap_vecop_list(hold_list); 410 } else { 411 return false; 412 } 413 return true; 414 } 415 416 void tcg_gen_not_vec(unsigned vece, TCGv_vec r, TCGv_vec a) 417 { 418 if (!TCG_TARGET_HAS_not_vec || !do_op2(vece, r, a, INDEX_op_not_vec)) { 419 TCGv_vec t = tcg_const_ones_vec_matching(r); 420 tcg_gen_xor_vec(0, r, a, t); 421 tcg_temp_free_vec(t); 422 } 423 } 424 425 void tcg_gen_neg_vec(unsigned vece, TCGv_vec r, TCGv_vec a) 426 { 427 const TCGOpcode *hold_list; 428 429 tcg_assert_listed_vecop(INDEX_op_neg_vec); 430 hold_list = tcg_swap_vecop_list(NULL); 431 432 if (!TCG_TARGET_HAS_neg_vec || !do_op2(vece, r, a, INDEX_op_neg_vec)) { 433 TCGv_vec t = tcg_const_zeros_vec_matching(r); 434 tcg_gen_sub_vec(vece, r, t, a); 435 tcg_temp_free_vec(t); 436 } 437 tcg_swap_vecop_list(hold_list); 438 } 439 440 void tcg_gen_abs_vec(unsigned vece, TCGv_vec r, TCGv_vec a) 441 { 442 const TCGOpcode *hold_list; 443 444 tcg_assert_listed_vecop(INDEX_op_abs_vec); 445 hold_list = tcg_swap_vecop_list(NULL); 446 447 if (!do_op2(vece, r, a, INDEX_op_abs_vec)) { 448 TCGType type = tcgv_vec_temp(r)->base_type; 449 TCGv_vec t = tcg_temp_new_vec(type); 450 451 tcg_debug_assert(tcg_can_emit_vec_op(INDEX_op_sub_vec, type, vece)); 452 if (tcg_can_emit_vec_op(INDEX_op_smax_vec, type, vece) > 0) { 453 tcg_gen_neg_vec(vece, t, a); 454 tcg_gen_smax_vec(vece, r, a, t); 455 } else { 456 if (tcg_can_emit_vec_op(INDEX_op_sari_vec, type, vece) > 0) { 457 tcg_gen_sari_vec(vece, t, a, (8 << vece) - 1); 458 } else { 459 do_dupi_vec(t, MO_REG, 0); 460 tcg_gen_cmp_vec(TCG_COND_LT, vece, t, a, t); 461 } 462 tcg_gen_xor_vec(vece, r, a, t); 463 tcg_gen_sub_vec(vece, r, r, t); 464 } 465 466 tcg_temp_free_vec(t); 467 } 468 tcg_swap_vecop_list(hold_list); 469 } 470 471 static void do_shifti(TCGOpcode opc, unsigned vece, 472 TCGv_vec r, TCGv_vec a, int64_t i) 473 { 474 TCGTemp *rt = tcgv_vec_temp(r); 475 TCGTemp *at = tcgv_vec_temp(a); 476 TCGArg ri = temp_arg(rt); 477 TCGArg ai = temp_arg(at); 478 TCGType type = rt->base_type; 479 int can; 480 481 tcg_debug_assert(at->base_type == type); 482 tcg_debug_assert(i >= 0 && i < (8 << vece)); 483 tcg_assert_listed_vecop(opc); 484 485 if (i == 0) { 486 tcg_gen_mov_vec(r, a); 487 return; 488 } 489 490 can = tcg_can_emit_vec_op(opc, type, vece); 491 if (can > 0) { 492 vec_gen_3(opc, type, vece, ri, ai, i); 493 } else { 494 /* We leave the choice of expansion via scalar or vector shift 495 to the target. Often, but not always, dupi can feed a vector 496 shift easier than a scalar. */ 497 const TCGOpcode *hold_list = tcg_swap_vecop_list(NULL); 498 tcg_debug_assert(can < 0); 499 tcg_expand_vec_op(opc, type, vece, ri, ai, i); 500 tcg_swap_vecop_list(hold_list); 501 } 502 } 503 504 void tcg_gen_shli_vec(unsigned vece, TCGv_vec r, TCGv_vec a, int64_t i) 505 { 506 do_shifti(INDEX_op_shli_vec, vece, r, a, i); 507 } 508 509 void tcg_gen_shri_vec(unsigned vece, TCGv_vec r, TCGv_vec a, int64_t i) 510 { 511 do_shifti(INDEX_op_shri_vec, vece, r, a, i); 512 } 513 514 void tcg_gen_sari_vec(unsigned vece, TCGv_vec r, TCGv_vec a, int64_t i) 515 { 516 do_shifti(INDEX_op_sari_vec, vece, r, a, i); 517 } 518 519 void tcg_gen_cmp_vec(TCGCond cond, unsigned vece, 520 TCGv_vec r, TCGv_vec a, TCGv_vec b) 521 { 522 TCGTemp *rt = tcgv_vec_temp(r); 523 TCGTemp *at = tcgv_vec_temp(a); 524 TCGTemp *bt = tcgv_vec_temp(b); 525 TCGArg ri = temp_arg(rt); 526 TCGArg ai = temp_arg(at); 527 TCGArg bi = temp_arg(bt); 528 TCGType type = rt->base_type; 529 int can; 530 531 tcg_debug_assert(at->base_type >= type); 532 tcg_debug_assert(bt->base_type >= type); 533 tcg_assert_listed_vecop(INDEX_op_cmp_vec); 534 can = tcg_can_emit_vec_op(INDEX_op_cmp_vec, type, vece); 535 if (can > 0) { 536 vec_gen_4(INDEX_op_cmp_vec, type, vece, ri, ai, bi, cond); 537 } else { 538 const TCGOpcode *hold_list = tcg_swap_vecop_list(NULL); 539 tcg_debug_assert(can < 0); 540 tcg_expand_vec_op(INDEX_op_cmp_vec, type, vece, ri, ai, bi, cond); 541 tcg_swap_vecop_list(hold_list); 542 } 543 } 544 545 static void do_op3(unsigned vece, TCGv_vec r, TCGv_vec a, 546 TCGv_vec b, TCGOpcode opc) 547 { 548 TCGTemp *rt = tcgv_vec_temp(r); 549 TCGTemp *at = tcgv_vec_temp(a); 550 TCGTemp *bt = tcgv_vec_temp(b); 551 TCGArg ri = temp_arg(rt); 552 TCGArg ai = temp_arg(at); 553 TCGArg bi = temp_arg(bt); 554 TCGType type = rt->base_type; 555 int can; 556 557 tcg_debug_assert(at->base_type >= type); 558 tcg_debug_assert(bt->base_type >= type); 559 tcg_assert_listed_vecop(opc); 560 can = tcg_can_emit_vec_op(opc, type, vece); 561 if (can > 0) { 562 vec_gen_3(opc, type, vece, ri, ai, bi); 563 } else { 564 const TCGOpcode *hold_list = tcg_swap_vecop_list(NULL); 565 tcg_debug_assert(can < 0); 566 tcg_expand_vec_op(opc, type, vece, ri, ai, bi); 567 tcg_swap_vecop_list(hold_list); 568 } 569 } 570 571 void tcg_gen_add_vec(unsigned vece, TCGv_vec r, TCGv_vec a, TCGv_vec b) 572 { 573 do_op3(vece, r, a, b, INDEX_op_add_vec); 574 } 575 576 void tcg_gen_sub_vec(unsigned vece, TCGv_vec r, TCGv_vec a, TCGv_vec b) 577 { 578 do_op3(vece, r, a, b, INDEX_op_sub_vec); 579 } 580 581 void tcg_gen_mul_vec(unsigned vece, TCGv_vec r, TCGv_vec a, TCGv_vec b) 582 { 583 do_op3(vece, r, a, b, INDEX_op_mul_vec); 584 } 585 586 void tcg_gen_ssadd_vec(unsigned vece, TCGv_vec r, TCGv_vec a, TCGv_vec b) 587 { 588 do_op3(vece, r, a, b, INDEX_op_ssadd_vec); 589 } 590 591 void tcg_gen_usadd_vec(unsigned vece, TCGv_vec r, TCGv_vec a, TCGv_vec b) 592 { 593 do_op3(vece, r, a, b, INDEX_op_usadd_vec); 594 } 595 596 void tcg_gen_sssub_vec(unsigned vece, TCGv_vec r, TCGv_vec a, TCGv_vec b) 597 { 598 do_op3(vece, r, a, b, INDEX_op_sssub_vec); 599 } 600 601 void tcg_gen_ussub_vec(unsigned vece, TCGv_vec r, TCGv_vec a, TCGv_vec b) 602 { 603 do_op3(vece, r, a, b, INDEX_op_ussub_vec); 604 } 605 606 void tcg_gen_smin_vec(unsigned vece, TCGv_vec r, TCGv_vec a, TCGv_vec b) 607 { 608 do_op3(vece, r, a, b, INDEX_op_smin_vec); 609 } 610 611 void tcg_gen_umin_vec(unsigned vece, TCGv_vec r, TCGv_vec a, TCGv_vec b) 612 { 613 do_op3(vece, r, a, b, INDEX_op_umin_vec); 614 } 615 616 void tcg_gen_smax_vec(unsigned vece, TCGv_vec r, TCGv_vec a, TCGv_vec b) 617 { 618 do_op3(vece, r, a, b, INDEX_op_smax_vec); 619 } 620 621 void tcg_gen_umax_vec(unsigned vece, TCGv_vec r, TCGv_vec a, TCGv_vec b) 622 { 623 do_op3(vece, r, a, b, INDEX_op_umax_vec); 624 } 625 626 void tcg_gen_shlv_vec(unsigned vece, TCGv_vec r, TCGv_vec a, TCGv_vec b) 627 { 628 do_op3(vece, r, a, b, INDEX_op_shlv_vec); 629 } 630 631 void tcg_gen_shrv_vec(unsigned vece, TCGv_vec r, TCGv_vec a, TCGv_vec b) 632 { 633 do_op3(vece, r, a, b, INDEX_op_shrv_vec); 634 } 635 636 void tcg_gen_sarv_vec(unsigned vece, TCGv_vec r, TCGv_vec a, TCGv_vec b) 637 { 638 do_op3(vece, r, a, b, INDEX_op_sarv_vec); 639 } 640 641 static void do_shifts(unsigned vece, TCGv_vec r, TCGv_vec a, 642 TCGv_i32 s, TCGOpcode opc_s, TCGOpcode opc_v) 643 { 644 TCGTemp *rt = tcgv_vec_temp(r); 645 TCGTemp *at = tcgv_vec_temp(a); 646 TCGTemp *st = tcgv_i32_temp(s); 647 TCGArg ri = temp_arg(rt); 648 TCGArg ai = temp_arg(at); 649 TCGArg si = temp_arg(st); 650 TCGType type = rt->base_type; 651 const TCGOpcode *hold_list; 652 int can; 653 654 tcg_debug_assert(at->base_type >= type); 655 tcg_assert_listed_vecop(opc_s); 656 hold_list = tcg_swap_vecop_list(NULL); 657 658 can = tcg_can_emit_vec_op(opc_s, type, vece); 659 if (can > 0) { 660 vec_gen_3(opc_s, type, vece, ri, ai, si); 661 } else if (can < 0) { 662 tcg_expand_vec_op(opc_s, type, vece, ri, ai, si); 663 } else { 664 TCGv_vec vec_s = tcg_temp_new_vec(type); 665 666 if (vece == MO_64) { 667 TCGv_i64 s64 = tcg_temp_new_i64(); 668 tcg_gen_extu_i32_i64(s64, s); 669 tcg_gen_dup_i64_vec(MO_64, vec_s, s64); 670 tcg_temp_free_i64(s64); 671 } else { 672 tcg_gen_dup_i32_vec(vece, vec_s, s); 673 } 674 do_op3(vece, r, a, vec_s, opc_v); 675 tcg_temp_free_vec(vec_s); 676 } 677 tcg_swap_vecop_list(hold_list); 678 } 679 680 void tcg_gen_shls_vec(unsigned vece, TCGv_vec r, TCGv_vec a, TCGv_i32 b) 681 { 682 do_shifts(vece, r, a, b, INDEX_op_shls_vec, INDEX_op_shlv_vec); 683 } 684 685 void tcg_gen_shrs_vec(unsigned vece, TCGv_vec r, TCGv_vec a, TCGv_i32 b) 686 { 687 do_shifts(vece, r, a, b, INDEX_op_shrs_vec, INDEX_op_shrv_vec); 688 } 689 690 void tcg_gen_sars_vec(unsigned vece, TCGv_vec r, TCGv_vec a, TCGv_i32 b) 691 { 692 do_shifts(vece, r, a, b, INDEX_op_sars_vec, INDEX_op_sarv_vec); 693 } 694