xref: /openbmc/qemu/tcg/tcg-op-vec.c (revision dcc99bd833840c6c4e909d391df17b71e47dea62)
1  /*
2   * Tiny Code Generator for QEMU
3   *
4   * Copyright (c) 2018 Linaro, Inc.
5   *
6   * This library is free software; you can redistribute it and/or
7   * modify it under the terms of the GNU Lesser General Public
8   * License as published by the Free Software Foundation; either
9   * version 2.1 of the License, or (at your option) any later version.
10   *
11   * This library is distributed in the hope that it will be useful,
12   * but WITHOUT ANY WARRANTY; without even the implied warranty of
13   * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
14   * Lesser General Public License for more details.
15   *
16   * You should have received a copy of the GNU Lesser General Public
17   * License along with this library; if not, see <http://www.gnu.org/licenses/>.
18   */
19  
20  #include "qemu/osdep.h"
21  #include "tcg/tcg.h"
22  #include "tcg/tcg-op.h"
23  #include "tcg/tcg-mo.h"
24  
25  /* Reduce the number of ifdefs below.  This assumes that all uses of
26     TCGV_HIGH and TCGV_LOW are properly protected by a conditional that
27     the compiler can eliminate.  */
28  #if TCG_TARGET_REG_BITS == 64
29  extern TCGv_i32 TCGV_LOW_link_error(TCGv_i64);
30  extern TCGv_i32 TCGV_HIGH_link_error(TCGv_i64);
31  #define TCGV_LOW  TCGV_LOW_link_error
32  #define TCGV_HIGH TCGV_HIGH_link_error
33  #endif
34  
35  /*
36   * Vector optional opcode tracking.
37   * Except for the basic logical operations (and, or, xor), and
38   * data movement (mov, ld, st, dupi), many vector opcodes are
39   * optional and may not be supported on the host.  Thank Intel
40   * for the irregularity in their instruction set.
41   *
42   * The gvec expanders allow custom vector operations to be composed,
43   * generally via the .fniv callback in the GVecGen* structures.  At
44   * the same time, in deciding whether to use this hook we need to
45   * know if the host supports the required operations.  This is
46   * presented as an array of opcodes, terminated by 0.  Each opcode
47   * is assumed to be expanded with the given VECE.
48   *
49   * For debugging, we want to validate this array.  Therefore, when
50   * tcg_ctx->vec_opt_opc is non-NULL, the tcg_gen_*_vec expanders
51   * will validate that their opcode is present in the list.
52   */
53  #ifdef CONFIG_DEBUG_TCG
54  void tcg_assert_listed_vecop(TCGOpcode op)
55  {
56      const TCGOpcode *p = tcg_ctx->vecop_list;
57      if (p) {
58          for (; *p; ++p) {
59              if (*p == op) {
60                  return;
61              }
62          }
63          g_assert_not_reached();
64      }
65  }
66  #endif
67  
68  bool tcg_can_emit_vecop_list(const TCGOpcode *list,
69                               TCGType type, unsigned vece)
70  {
71      if (list == NULL) {
72          return true;
73      }
74  
75      for (; *list; ++list) {
76          TCGOpcode opc = *list;
77  
78  #ifdef CONFIG_DEBUG_TCG
79          switch (opc) {
80          case INDEX_op_and_vec:
81          case INDEX_op_or_vec:
82          case INDEX_op_xor_vec:
83          case INDEX_op_mov_vec:
84          case INDEX_op_dup_vec:
85          case INDEX_op_dup2_vec:
86          case INDEX_op_ld_vec:
87          case INDEX_op_st_vec:
88          case INDEX_op_bitsel_vec:
89              /* These opcodes are mandatory and should not be listed.  */
90              g_assert_not_reached();
91          case INDEX_op_not_vec:
92              /* These opcodes have generic expansions using the above.  */
93              g_assert_not_reached();
94          default:
95              break;
96          }
97  #endif
98  
99          if (tcg_can_emit_vec_op(opc, type, vece)) {
100              continue;
101          }
102  
103          /*
104           * The opcode list is created by front ends based on what they
105           * actually invoke.  We must mirror the logic in the routines
106           * below for generic expansions using other opcodes.
107           */
108          switch (opc) {
109          case INDEX_op_neg_vec:
110              if (tcg_can_emit_vec_op(INDEX_op_sub_vec, type, vece)) {
111                  continue;
112              }
113              break;
114          case INDEX_op_abs_vec:
115              if (tcg_can_emit_vec_op(INDEX_op_sub_vec, type, vece)
116                  && (tcg_can_emit_vec_op(INDEX_op_smax_vec, type, vece) > 0
117                      || tcg_can_emit_vec_op(INDEX_op_sari_vec, type, vece) > 0
118                      || tcg_can_emit_vec_op(INDEX_op_cmp_vec, type, vece))) {
119                  continue;
120              }
121              break;
122          case INDEX_op_usadd_vec:
123              if (tcg_can_emit_vec_op(INDEX_op_umin_vec, type, vece) ||
124                  tcg_can_emit_vec_op(INDEX_op_cmp_vec, type, vece)) {
125                  continue;
126              }
127              break;
128          case INDEX_op_ussub_vec:
129              if (tcg_can_emit_vec_op(INDEX_op_umax_vec, type, vece) ||
130                  tcg_can_emit_vec_op(INDEX_op_cmp_vec, type, vece)) {
131                  continue;
132              }
133              break;
134          case INDEX_op_cmpsel_vec:
135          case INDEX_op_smin_vec:
136          case INDEX_op_smax_vec:
137          case INDEX_op_umin_vec:
138          case INDEX_op_umax_vec:
139              if (tcg_can_emit_vec_op(INDEX_op_cmp_vec, type, vece)) {
140                  continue;
141              }
142              break;
143          default:
144              break;
145          }
146          return false;
147      }
148      return true;
149  }
150  
151  void vec_gen_2(TCGOpcode opc, TCGType type, unsigned vece, TCGArg r, TCGArg a)
152  {
153      TCGOp *op = tcg_emit_op(opc);
154      TCGOP_VECL(op) = type - TCG_TYPE_V64;
155      TCGOP_VECE(op) = vece;
156      op->args[0] = r;
157      op->args[1] = a;
158  }
159  
160  void vec_gen_3(TCGOpcode opc, TCGType type, unsigned vece,
161                 TCGArg r, TCGArg a, TCGArg b)
162  {
163      TCGOp *op = tcg_emit_op(opc);
164      TCGOP_VECL(op) = type - TCG_TYPE_V64;
165      TCGOP_VECE(op) = vece;
166      op->args[0] = r;
167      op->args[1] = a;
168      op->args[2] = b;
169  }
170  
171  void vec_gen_4(TCGOpcode opc, TCGType type, unsigned vece,
172                 TCGArg r, TCGArg a, TCGArg b, TCGArg c)
173  {
174      TCGOp *op = tcg_emit_op(opc);
175      TCGOP_VECL(op) = type - TCG_TYPE_V64;
176      TCGOP_VECE(op) = vece;
177      op->args[0] = r;
178      op->args[1] = a;
179      op->args[2] = b;
180      op->args[3] = c;
181  }
182  
183  static void vec_gen_6(TCGOpcode opc, TCGType type, unsigned vece, TCGArg r,
184                        TCGArg a, TCGArg b, TCGArg c, TCGArg d, TCGArg e)
185  {
186      TCGOp *op = tcg_emit_op(opc);
187      TCGOP_VECL(op) = type - TCG_TYPE_V64;
188      TCGOP_VECE(op) = vece;
189      op->args[0] = r;
190      op->args[1] = a;
191      op->args[2] = b;
192      op->args[3] = c;
193      op->args[4] = d;
194      op->args[5] = e;
195  }
196  
197  static void vec_gen_op2(TCGOpcode opc, unsigned vece, TCGv_vec r, TCGv_vec a)
198  {
199      TCGTemp *rt = tcgv_vec_temp(r);
200      TCGTemp *at = tcgv_vec_temp(a);
201      TCGType type = rt->base_type;
202  
203      /* Must enough inputs for the output.  */
204      tcg_debug_assert(at->base_type >= type);
205      vec_gen_2(opc, type, vece, temp_arg(rt), temp_arg(at));
206  }
207  
208  static void vec_gen_op3(TCGOpcode opc, unsigned vece,
209                          TCGv_vec r, TCGv_vec a, TCGv_vec b)
210  {
211      TCGTemp *rt = tcgv_vec_temp(r);
212      TCGTemp *at = tcgv_vec_temp(a);
213      TCGTemp *bt = tcgv_vec_temp(b);
214      TCGType type = rt->base_type;
215  
216      /* Must enough inputs for the output.  */
217      tcg_debug_assert(at->base_type >= type);
218      tcg_debug_assert(bt->base_type >= type);
219      vec_gen_3(opc, type, vece, temp_arg(rt), temp_arg(at), temp_arg(bt));
220  }
221  
222  void tcg_gen_mov_vec(TCGv_vec r, TCGv_vec a)
223  {
224      if (r != a) {
225          vec_gen_op2(INDEX_op_mov_vec, 0, r, a);
226      }
227  }
228  
229  TCGv_vec tcg_const_zeros_vec(TCGType type)
230  {
231      TCGv_vec ret = tcg_temp_new_vec(type);
232      tcg_gen_dupi_vec(MO_64, ret, 0);
233      return ret;
234  }
235  
236  TCGv_vec tcg_const_ones_vec(TCGType type)
237  {
238      TCGv_vec ret = tcg_temp_new_vec(type);
239      tcg_gen_dupi_vec(MO_64, ret, -1);
240      return ret;
241  }
242  
243  TCGv_vec tcg_const_zeros_vec_matching(TCGv_vec m)
244  {
245      TCGTemp *t = tcgv_vec_temp(m);
246      return tcg_const_zeros_vec(t->base_type);
247  }
248  
249  TCGv_vec tcg_const_ones_vec_matching(TCGv_vec m)
250  {
251      TCGTemp *t = tcgv_vec_temp(m);
252      return tcg_const_ones_vec(t->base_type);
253  }
254  
255  void tcg_gen_dupi_vec(unsigned vece, TCGv_vec r, uint64_t a)
256  {
257      TCGTemp *rt = tcgv_vec_temp(r);
258      tcg_gen_mov_vec(r, tcg_constant_vec(rt->base_type, vece, a));
259  }
260  
261  void tcg_gen_dup_i64_vec(unsigned vece, TCGv_vec r, TCGv_i64 a)
262  {
263      TCGArg ri = tcgv_vec_arg(r);
264      TCGTemp *rt = arg_temp(ri);
265      TCGType type = rt->base_type;
266  
267      if (TCG_TARGET_REG_BITS == 64) {
268          TCGArg ai = tcgv_i64_arg(a);
269          vec_gen_2(INDEX_op_dup_vec, type, vece, ri, ai);
270      } else if (vece == MO_64) {
271          TCGArg al = tcgv_i32_arg(TCGV_LOW(a));
272          TCGArg ah = tcgv_i32_arg(TCGV_HIGH(a));
273          vec_gen_3(INDEX_op_dup2_vec, type, MO_64, ri, al, ah);
274      } else {
275          TCGArg ai = tcgv_i32_arg(TCGV_LOW(a));
276          vec_gen_2(INDEX_op_dup_vec, type, vece, ri, ai);
277      }
278  }
279  
280  void tcg_gen_dup_i32_vec(unsigned vece, TCGv_vec r, TCGv_i32 a)
281  {
282      TCGArg ri = tcgv_vec_arg(r);
283      TCGArg ai = tcgv_i32_arg(a);
284      TCGTemp *rt = arg_temp(ri);
285      TCGType type = rt->base_type;
286  
287      vec_gen_2(INDEX_op_dup_vec, type, vece, ri, ai);
288  }
289  
290  void tcg_gen_dup_mem_vec(unsigned vece, TCGv_vec r, TCGv_ptr b,
291                           tcg_target_long ofs)
292  {
293      TCGArg ri = tcgv_vec_arg(r);
294      TCGArg bi = tcgv_ptr_arg(b);
295      TCGTemp *rt = arg_temp(ri);
296      TCGType type = rt->base_type;
297  
298      vec_gen_3(INDEX_op_dupm_vec, type, vece, ri, bi, ofs);
299  }
300  
301  static void vec_gen_ldst(TCGOpcode opc, TCGv_vec r, TCGv_ptr b, TCGArg o)
302  {
303      TCGArg ri = tcgv_vec_arg(r);
304      TCGArg bi = tcgv_ptr_arg(b);
305      TCGTemp *rt = arg_temp(ri);
306      TCGType type = rt->base_type;
307  
308      vec_gen_3(opc, type, 0, ri, bi, o);
309  }
310  
311  void tcg_gen_ld_vec(TCGv_vec r, TCGv_ptr b, TCGArg o)
312  {
313      vec_gen_ldst(INDEX_op_ld_vec, r, b, o);
314  }
315  
316  void tcg_gen_st_vec(TCGv_vec r, TCGv_ptr b, TCGArg o)
317  {
318      vec_gen_ldst(INDEX_op_st_vec, r, b, o);
319  }
320  
321  void tcg_gen_stl_vec(TCGv_vec r, TCGv_ptr b, TCGArg o, TCGType low_type)
322  {
323      TCGArg ri = tcgv_vec_arg(r);
324      TCGArg bi = tcgv_ptr_arg(b);
325      TCGTemp *rt = arg_temp(ri);
326      TCGType type = rt->base_type;
327  
328      tcg_debug_assert(low_type >= TCG_TYPE_V64);
329      tcg_debug_assert(low_type <= type);
330      vec_gen_3(INDEX_op_st_vec, low_type, 0, ri, bi, o);
331  }
332  
333  void tcg_gen_and_vec(unsigned vece, TCGv_vec r, TCGv_vec a, TCGv_vec b)
334  {
335      vec_gen_op3(INDEX_op_and_vec, 0, r, a, b);
336  }
337  
338  void tcg_gen_or_vec(unsigned vece, TCGv_vec r, TCGv_vec a, TCGv_vec b)
339  {
340      vec_gen_op3(INDEX_op_or_vec, 0, r, a, b);
341  }
342  
343  void tcg_gen_xor_vec(unsigned vece, TCGv_vec r, TCGv_vec a, TCGv_vec b)
344  {
345      vec_gen_op3(INDEX_op_xor_vec, 0, r, a, b);
346  }
347  
348  void tcg_gen_andc_vec(unsigned vece, TCGv_vec r, TCGv_vec a, TCGv_vec b)
349  {
350      if (TCG_TARGET_HAS_andc_vec) {
351          vec_gen_op3(INDEX_op_andc_vec, 0, r, a, b);
352      } else {
353          TCGv_vec t = tcg_temp_new_vec_matching(r);
354          tcg_gen_not_vec(0, t, b);
355          tcg_gen_and_vec(0, r, a, t);
356          tcg_temp_free_vec(t);
357      }
358  }
359  
360  void tcg_gen_orc_vec(unsigned vece, TCGv_vec r, TCGv_vec a, TCGv_vec b)
361  {
362      if (TCG_TARGET_HAS_orc_vec) {
363          vec_gen_op3(INDEX_op_orc_vec, 0, r, a, b);
364      } else {
365          TCGv_vec t = tcg_temp_new_vec_matching(r);
366          tcg_gen_not_vec(0, t, b);
367          tcg_gen_or_vec(0, r, a, t);
368          tcg_temp_free_vec(t);
369      }
370  }
371  
372  void tcg_gen_nand_vec(unsigned vece, TCGv_vec r, TCGv_vec a, TCGv_vec b)
373  {
374      /* TODO: Add TCG_TARGET_HAS_nand_vec when adding a backend supports it. */
375      tcg_gen_and_vec(0, r, a, b);
376      tcg_gen_not_vec(0, r, r);
377  }
378  
379  void tcg_gen_nor_vec(unsigned vece, TCGv_vec r, TCGv_vec a, TCGv_vec b)
380  {
381      /* TODO: Add TCG_TARGET_HAS_nor_vec when adding a backend supports it. */
382      tcg_gen_or_vec(0, r, a, b);
383      tcg_gen_not_vec(0, r, r);
384  }
385  
386  void tcg_gen_eqv_vec(unsigned vece, TCGv_vec r, TCGv_vec a, TCGv_vec b)
387  {
388      /* TODO: Add TCG_TARGET_HAS_eqv_vec when adding a backend supports it. */
389      tcg_gen_xor_vec(0, r, a, b);
390      tcg_gen_not_vec(0, r, r);
391  }
392  
393  static bool do_op2(unsigned vece, TCGv_vec r, TCGv_vec a, TCGOpcode opc)
394  {
395      TCGTemp *rt = tcgv_vec_temp(r);
396      TCGTemp *at = tcgv_vec_temp(a);
397      TCGArg ri = temp_arg(rt);
398      TCGArg ai = temp_arg(at);
399      TCGType type = rt->base_type;
400      int can;
401  
402      tcg_debug_assert(at->base_type >= type);
403      tcg_assert_listed_vecop(opc);
404      can = tcg_can_emit_vec_op(opc, type, vece);
405      if (can > 0) {
406          vec_gen_2(opc, type, vece, ri, ai);
407      } else if (can < 0) {
408          const TCGOpcode *hold_list = tcg_swap_vecop_list(NULL);
409          tcg_expand_vec_op(opc, type, vece, ri, ai);
410          tcg_swap_vecop_list(hold_list);
411      } else {
412          return false;
413      }
414      return true;
415  }
416  
417  void tcg_gen_not_vec(unsigned vece, TCGv_vec r, TCGv_vec a)
418  {
419      const TCGOpcode *hold_list = tcg_swap_vecop_list(NULL);
420  
421      if (!TCG_TARGET_HAS_not_vec || !do_op2(vece, r, a, INDEX_op_not_vec)) {
422          TCGv_vec t = tcg_const_ones_vec_matching(r);
423          tcg_gen_xor_vec(0, r, a, t);
424          tcg_temp_free_vec(t);
425      }
426      tcg_swap_vecop_list(hold_list);
427  }
428  
429  void tcg_gen_neg_vec(unsigned vece, TCGv_vec r, TCGv_vec a)
430  {
431      const TCGOpcode *hold_list;
432  
433      tcg_assert_listed_vecop(INDEX_op_neg_vec);
434      hold_list = tcg_swap_vecop_list(NULL);
435  
436      if (!TCG_TARGET_HAS_neg_vec || !do_op2(vece, r, a, INDEX_op_neg_vec)) {
437          TCGv_vec t = tcg_const_zeros_vec_matching(r);
438          tcg_gen_sub_vec(vece, r, t, a);
439          tcg_temp_free_vec(t);
440      }
441      tcg_swap_vecop_list(hold_list);
442  }
443  
444  void tcg_gen_abs_vec(unsigned vece, TCGv_vec r, TCGv_vec a)
445  {
446      const TCGOpcode *hold_list;
447  
448      tcg_assert_listed_vecop(INDEX_op_abs_vec);
449      hold_list = tcg_swap_vecop_list(NULL);
450  
451      if (!do_op2(vece, r, a, INDEX_op_abs_vec)) {
452          TCGType type = tcgv_vec_temp(r)->base_type;
453          TCGv_vec t = tcg_temp_new_vec(type);
454  
455          tcg_debug_assert(tcg_can_emit_vec_op(INDEX_op_sub_vec, type, vece));
456          if (tcg_can_emit_vec_op(INDEX_op_smax_vec, type, vece) > 0) {
457              tcg_gen_neg_vec(vece, t, a);
458              tcg_gen_smax_vec(vece, r, a, t);
459          } else {
460              if (tcg_can_emit_vec_op(INDEX_op_sari_vec, type, vece) > 0) {
461                  tcg_gen_sari_vec(vece, t, a, (8 << vece) - 1);
462              } else {
463                  tcg_gen_cmp_vec(TCG_COND_LT, vece, t, a,
464                                  tcg_constant_vec(type, vece, 0));
465              }
466              tcg_gen_xor_vec(vece, r, a, t);
467              tcg_gen_sub_vec(vece, r, r, t);
468          }
469  
470          tcg_temp_free_vec(t);
471      }
472      tcg_swap_vecop_list(hold_list);
473  }
474  
475  static void do_shifti(TCGOpcode opc, unsigned vece,
476                        TCGv_vec r, TCGv_vec a, int64_t i)
477  {
478      TCGTemp *rt = tcgv_vec_temp(r);
479      TCGTemp *at = tcgv_vec_temp(a);
480      TCGArg ri = temp_arg(rt);
481      TCGArg ai = temp_arg(at);
482      TCGType type = rt->base_type;
483      int can;
484  
485      tcg_debug_assert(at->base_type == type);
486      tcg_debug_assert(i >= 0 && i < (8 << vece));
487      tcg_assert_listed_vecop(opc);
488  
489      if (i == 0) {
490          tcg_gen_mov_vec(r, a);
491          return;
492      }
493  
494      can = tcg_can_emit_vec_op(opc, type, vece);
495      if (can > 0) {
496          vec_gen_3(opc, type, vece, ri, ai, i);
497      } else {
498          /* We leave the choice of expansion via scalar or vector shift
499             to the target.  Often, but not always, dupi can feed a vector
500             shift easier than a scalar.  */
501          const TCGOpcode *hold_list = tcg_swap_vecop_list(NULL);
502          tcg_debug_assert(can < 0);
503          tcg_expand_vec_op(opc, type, vece, ri, ai, i);
504          tcg_swap_vecop_list(hold_list);
505      }
506  }
507  
508  void tcg_gen_shli_vec(unsigned vece, TCGv_vec r, TCGv_vec a, int64_t i)
509  {
510      do_shifti(INDEX_op_shli_vec, vece, r, a, i);
511  }
512  
513  void tcg_gen_shri_vec(unsigned vece, TCGv_vec r, TCGv_vec a, int64_t i)
514  {
515      do_shifti(INDEX_op_shri_vec, vece, r, a, i);
516  }
517  
518  void tcg_gen_sari_vec(unsigned vece, TCGv_vec r, TCGv_vec a, int64_t i)
519  {
520      do_shifti(INDEX_op_sari_vec, vece, r, a, i);
521  }
522  
523  void tcg_gen_rotli_vec(unsigned vece, TCGv_vec r, TCGv_vec a, int64_t i)
524  {
525      do_shifti(INDEX_op_rotli_vec, vece, r, a, i);
526  }
527  
528  void tcg_gen_rotri_vec(unsigned vece, TCGv_vec r, TCGv_vec a, int64_t i)
529  {
530      int bits = 8 << vece;
531      tcg_debug_assert(i >= 0 && i < bits);
532      do_shifti(INDEX_op_rotli_vec, vece, r, a, -i & (bits - 1));
533  }
534  
535  void tcg_gen_cmp_vec(TCGCond cond, unsigned vece,
536                       TCGv_vec r, TCGv_vec a, TCGv_vec b)
537  {
538      TCGTemp *rt = tcgv_vec_temp(r);
539      TCGTemp *at = tcgv_vec_temp(a);
540      TCGTemp *bt = tcgv_vec_temp(b);
541      TCGArg ri = temp_arg(rt);
542      TCGArg ai = temp_arg(at);
543      TCGArg bi = temp_arg(bt);
544      TCGType type = rt->base_type;
545      int can;
546  
547      tcg_debug_assert(at->base_type >= type);
548      tcg_debug_assert(bt->base_type >= type);
549      tcg_assert_listed_vecop(INDEX_op_cmp_vec);
550      can = tcg_can_emit_vec_op(INDEX_op_cmp_vec, type, vece);
551      if (can > 0) {
552          vec_gen_4(INDEX_op_cmp_vec, type, vece, ri, ai, bi, cond);
553      } else {
554          const TCGOpcode *hold_list = tcg_swap_vecop_list(NULL);
555          tcg_debug_assert(can < 0);
556          tcg_expand_vec_op(INDEX_op_cmp_vec, type, vece, ri, ai, bi, cond);
557          tcg_swap_vecop_list(hold_list);
558      }
559  }
560  
561  static bool do_op3(unsigned vece, TCGv_vec r, TCGv_vec a,
562                     TCGv_vec b, TCGOpcode opc)
563  {
564      TCGTemp *rt = tcgv_vec_temp(r);
565      TCGTemp *at = tcgv_vec_temp(a);
566      TCGTemp *bt = tcgv_vec_temp(b);
567      TCGArg ri = temp_arg(rt);
568      TCGArg ai = temp_arg(at);
569      TCGArg bi = temp_arg(bt);
570      TCGType type = rt->base_type;
571      int can;
572  
573      tcg_debug_assert(at->base_type >= type);
574      tcg_debug_assert(bt->base_type >= type);
575      tcg_assert_listed_vecop(opc);
576      can = tcg_can_emit_vec_op(opc, type, vece);
577      if (can > 0) {
578          vec_gen_3(opc, type, vece, ri, ai, bi);
579      } else if (can < 0) {
580          const TCGOpcode *hold_list = tcg_swap_vecop_list(NULL);
581          tcg_expand_vec_op(opc, type, vece, ri, ai, bi);
582          tcg_swap_vecop_list(hold_list);
583      } else {
584          return false;
585      }
586      return true;
587  }
588  
589  static void do_op3_nofail(unsigned vece, TCGv_vec r, TCGv_vec a,
590                            TCGv_vec b, TCGOpcode opc)
591  {
592      bool ok = do_op3(vece, r, a, b, opc);
593      tcg_debug_assert(ok);
594  }
595  
596  void tcg_gen_add_vec(unsigned vece, TCGv_vec r, TCGv_vec a, TCGv_vec b)
597  {
598      do_op3_nofail(vece, r, a, b, INDEX_op_add_vec);
599  }
600  
601  void tcg_gen_sub_vec(unsigned vece, TCGv_vec r, TCGv_vec a, TCGv_vec b)
602  {
603      do_op3_nofail(vece, r, a, b, INDEX_op_sub_vec);
604  }
605  
606  void tcg_gen_mul_vec(unsigned vece, TCGv_vec r, TCGv_vec a, TCGv_vec b)
607  {
608      do_op3_nofail(vece, r, a, b, INDEX_op_mul_vec);
609  }
610  
611  void tcg_gen_ssadd_vec(unsigned vece, TCGv_vec r, TCGv_vec a, TCGv_vec b)
612  {
613      do_op3_nofail(vece, r, a, b, INDEX_op_ssadd_vec);
614  }
615  
616  void tcg_gen_usadd_vec(unsigned vece, TCGv_vec r, TCGv_vec a, TCGv_vec b)
617  {
618      if (!do_op3(vece, r, a, b, INDEX_op_usadd_vec)) {
619          const TCGOpcode *hold_list = tcg_swap_vecop_list(NULL);
620          TCGv_vec t = tcg_temp_new_vec_matching(r);
621  
622          /* usadd(a, b) = min(a, ~b) + b */
623          tcg_gen_not_vec(vece, t, b);
624          tcg_gen_umin_vec(vece, t, t, a);
625          tcg_gen_add_vec(vece, r, t, b);
626  
627          tcg_temp_free_vec(t);
628          tcg_swap_vecop_list(hold_list);
629      }
630  }
631  
632  void tcg_gen_sssub_vec(unsigned vece, TCGv_vec r, TCGv_vec a, TCGv_vec b)
633  {
634      do_op3_nofail(vece, r, a, b, INDEX_op_sssub_vec);
635  }
636  
637  void tcg_gen_ussub_vec(unsigned vece, TCGv_vec r, TCGv_vec a, TCGv_vec b)
638  {
639      if (!do_op3(vece, r, a, b, INDEX_op_ussub_vec)) {
640          const TCGOpcode *hold_list = tcg_swap_vecop_list(NULL);
641          TCGv_vec t = tcg_temp_new_vec_matching(r);
642  
643          /* ussub(a, b) = max(a, b) - b */
644          tcg_gen_umax_vec(vece, t, a, b);
645          tcg_gen_sub_vec(vece, r, t, b);
646  
647          tcg_temp_free_vec(t);
648          tcg_swap_vecop_list(hold_list);
649      }
650  }
651  
652  static void do_minmax(unsigned vece, TCGv_vec r, TCGv_vec a,
653                        TCGv_vec b, TCGOpcode opc, TCGCond cond)
654  {
655      if (!do_op3(vece, r, a, b, opc)) {
656          const TCGOpcode *hold_list = tcg_swap_vecop_list(NULL);
657          tcg_gen_cmpsel_vec(cond, vece, r, a, b, a, b);
658          tcg_swap_vecop_list(hold_list);
659      }
660  }
661  
662  void tcg_gen_smin_vec(unsigned vece, TCGv_vec r, TCGv_vec a, TCGv_vec b)
663  {
664      do_minmax(vece, r, a, b, INDEX_op_smin_vec, TCG_COND_LT);
665  }
666  
667  void tcg_gen_umin_vec(unsigned vece, TCGv_vec r, TCGv_vec a, TCGv_vec b)
668  {
669      do_minmax(vece, r, a, b, INDEX_op_umin_vec, TCG_COND_LTU);
670  }
671  
672  void tcg_gen_smax_vec(unsigned vece, TCGv_vec r, TCGv_vec a, TCGv_vec b)
673  {
674      do_minmax(vece, r, a, b, INDEX_op_smax_vec, TCG_COND_GT);
675  }
676  
677  void tcg_gen_umax_vec(unsigned vece, TCGv_vec r, TCGv_vec a, TCGv_vec b)
678  {
679      do_minmax(vece, r, a, b, INDEX_op_umax_vec, TCG_COND_GTU);
680  }
681  
682  void tcg_gen_shlv_vec(unsigned vece, TCGv_vec r, TCGv_vec a, TCGv_vec b)
683  {
684      do_op3_nofail(vece, r, a, b, INDEX_op_shlv_vec);
685  }
686  
687  void tcg_gen_shrv_vec(unsigned vece, TCGv_vec r, TCGv_vec a, TCGv_vec b)
688  {
689      do_op3_nofail(vece, r, a, b, INDEX_op_shrv_vec);
690  }
691  
692  void tcg_gen_sarv_vec(unsigned vece, TCGv_vec r, TCGv_vec a, TCGv_vec b)
693  {
694      do_op3_nofail(vece, r, a, b, INDEX_op_sarv_vec);
695  }
696  
697  void tcg_gen_rotlv_vec(unsigned vece, TCGv_vec r, TCGv_vec a, TCGv_vec b)
698  {
699      do_op3_nofail(vece, r, a, b, INDEX_op_rotlv_vec);
700  }
701  
702  void tcg_gen_rotrv_vec(unsigned vece, TCGv_vec r, TCGv_vec a, TCGv_vec b)
703  {
704      do_op3_nofail(vece, r, a, b, INDEX_op_rotrv_vec);
705  }
706  
707  static void do_shifts(unsigned vece, TCGv_vec r, TCGv_vec a,
708                        TCGv_i32 s, TCGOpcode opc)
709  {
710      TCGTemp *rt = tcgv_vec_temp(r);
711      TCGTemp *at = tcgv_vec_temp(a);
712      TCGTemp *st = tcgv_i32_temp(s);
713      TCGArg ri = temp_arg(rt);
714      TCGArg ai = temp_arg(at);
715      TCGArg si = temp_arg(st);
716      TCGType type = rt->base_type;
717      int can;
718  
719      tcg_debug_assert(at->base_type >= type);
720      tcg_assert_listed_vecop(opc);
721      can = tcg_can_emit_vec_op(opc, type, vece);
722      if (can > 0) {
723          vec_gen_3(opc, type, vece, ri, ai, si);
724      } else if (can < 0) {
725          const TCGOpcode *hold_list = tcg_swap_vecop_list(NULL);
726          tcg_expand_vec_op(opc, type, vece, ri, ai, si);
727          tcg_swap_vecop_list(hold_list);
728      } else {
729          g_assert_not_reached();
730      }
731  }
732  
733  void tcg_gen_shls_vec(unsigned vece, TCGv_vec r, TCGv_vec a, TCGv_i32 b)
734  {
735      do_shifts(vece, r, a, b, INDEX_op_shls_vec);
736  }
737  
738  void tcg_gen_shrs_vec(unsigned vece, TCGv_vec r, TCGv_vec a, TCGv_i32 b)
739  {
740      do_shifts(vece, r, a, b, INDEX_op_shrs_vec);
741  }
742  
743  void tcg_gen_sars_vec(unsigned vece, TCGv_vec r, TCGv_vec a, TCGv_i32 b)
744  {
745      do_shifts(vece, r, a, b, INDEX_op_sars_vec);
746  }
747  
748  void tcg_gen_rotls_vec(unsigned vece, TCGv_vec r, TCGv_vec a, TCGv_i32 s)
749  {
750      do_shifts(vece, r, a, s, INDEX_op_rotls_vec);
751  }
752  
753  void tcg_gen_bitsel_vec(unsigned vece, TCGv_vec r, TCGv_vec a,
754                          TCGv_vec b, TCGv_vec c)
755  {
756      TCGTemp *rt = tcgv_vec_temp(r);
757      TCGTemp *at = tcgv_vec_temp(a);
758      TCGTemp *bt = tcgv_vec_temp(b);
759      TCGTemp *ct = tcgv_vec_temp(c);
760      TCGType type = rt->base_type;
761  
762      tcg_debug_assert(at->base_type >= type);
763      tcg_debug_assert(bt->base_type >= type);
764      tcg_debug_assert(ct->base_type >= type);
765  
766      if (TCG_TARGET_HAS_bitsel_vec) {
767          vec_gen_4(INDEX_op_bitsel_vec, type, MO_8,
768                    temp_arg(rt), temp_arg(at), temp_arg(bt), temp_arg(ct));
769      } else {
770          TCGv_vec t = tcg_temp_new_vec(type);
771          tcg_gen_and_vec(MO_8, t, a, b);
772          tcg_gen_andc_vec(MO_8, r, c, a);
773          tcg_gen_or_vec(MO_8, r, r, t);
774          tcg_temp_free_vec(t);
775      }
776  }
777  
778  void tcg_gen_cmpsel_vec(TCGCond cond, unsigned vece, TCGv_vec r,
779                          TCGv_vec a, TCGv_vec b, TCGv_vec c, TCGv_vec d)
780  {
781      TCGTemp *rt = tcgv_vec_temp(r);
782      TCGTemp *at = tcgv_vec_temp(a);
783      TCGTemp *bt = tcgv_vec_temp(b);
784      TCGTemp *ct = tcgv_vec_temp(c);
785      TCGTemp *dt = tcgv_vec_temp(d);
786      TCGArg ri = temp_arg(rt);
787      TCGArg ai = temp_arg(at);
788      TCGArg bi = temp_arg(bt);
789      TCGArg ci = temp_arg(ct);
790      TCGArg di = temp_arg(dt);
791      TCGType type = rt->base_type;
792      const TCGOpcode *hold_list;
793      int can;
794  
795      tcg_debug_assert(at->base_type >= type);
796      tcg_debug_assert(bt->base_type >= type);
797      tcg_debug_assert(ct->base_type >= type);
798      tcg_debug_assert(dt->base_type >= type);
799  
800      tcg_assert_listed_vecop(INDEX_op_cmpsel_vec);
801      hold_list = tcg_swap_vecop_list(NULL);
802      can = tcg_can_emit_vec_op(INDEX_op_cmpsel_vec, type, vece);
803  
804      if (can > 0) {
805          vec_gen_6(INDEX_op_cmpsel_vec, type, vece, ri, ai, bi, ci, di, cond);
806      } else if (can < 0) {
807          tcg_expand_vec_op(INDEX_op_cmpsel_vec, type, vece,
808                            ri, ai, bi, ci, di, cond);
809      } else {
810          TCGv_vec t = tcg_temp_new_vec(type);
811          tcg_gen_cmp_vec(cond, vece, t, a, b);
812          tcg_gen_bitsel_vec(vece, r, t, c, d);
813          tcg_temp_free_vec(t);
814      }
815      tcg_swap_vecop_list(hold_list);
816  }
817