22 #include "tcg/tcg-temp-internal.h"
23 #include "tcg/tcg-op-common.h"
24 #include "tcg/tcg-op-gvec-common.h"
25 #include "tcg/tcg-gvec-desc.h"
44     case 16:  in check_size_align()
54     max_align = maxsz >= 16 ? 15 : 7;  in check_size_align()
99      * values from the caller will not be detected, e.g. if the  in simd_desc()
101      * incorrectly passes us 1 << (SIMD_DATA_BITS - 1).  in simd_desc()
106     oprsz = (oprsz / 8) - 1;  in simd_desc()
107     maxsz = (maxsz / 8) - 1;  in simd_desc()
111      * oprsz is {8,16,32} or matches maxsz.  Encode the final  in simd_desc()
125 /* Generate a call to a gvec-style helper with two vector operands.  */
145 /* Generate a call to a gvec-style helper with two vector operands
166 /* Generate a call to a gvec-style helper with three vector operands.  */
189 /* Generate a call to a gvec-style helper with four vector operands.  */
215 /* Generate a call to a gvec-style helper with five vector operands.  */
244 /* Generate a call to a gvec-style helper with three vector operands
265 /* Generate a call to a gvec-style helper with three vector operands
289 /* Generate a call to a gvec-style helper with four vector operands
317 /* Generate a call to a gvec-style helper with five vector operands
362     if (lnsz < 16) {  in check_size_impl()
363         /* For sizes below 16, accept no remainder. */  in check_size_impl()
370          * power of 2, but always a multiple of 16.  The intent is  in check_size_impl()
371          * that e.g. size == 80 would be expanded with 2x32 + 1x16.  in check_size_impl()
410         tcg_gen_deposit_i32(out, in, in, 16, 16);  in tcg_gen_dup_i32()
453      * power of 2, but always a multiple of 16.  The intent is  in choose_vector_type()
454      * that e.g. size == 80 would be expanded with 2x32 + 1x16.  in choose_vector_type()
462         (!(size & 16) ||  in choose_vector_type()
471         check_size_impl(size, 16) &&  in choose_vector_type()
493      * This may be expand_clr for the tail of an operation, e.g.  in do_dup_store()
506          * power of 2, but always a multiple of 16.  The intent is  in do_dup_store()
507          * that e.g. size == 80 would be expanded with 2x32 + 1x16.  in do_dup_store()
514         for (; i + 16 <= oprsz; i += 16) {  in do_dup_store()
528         expand_clr(dofs + oprsz, maxsz - oprsz);  in do_dup_store()
561      * Prefer integer when 64-bit host and no variable dup.  in do_dup()
586             /* We are given a 32-bit variable input.  For a 64-bit host,  in do_dup()
587                use a 64-bit operation unless the 32-bit operation would  in do_dup()
599             /* We are given a 64-bit variable input.  */  in do_dup()
604             /* For 64-bit hosts, use 64-bit constants for "simple" constants  in do_dup()
605                or when we'd need too many 32-bit stores, or when a 64-bit  in do_dup()
609                     && (in_c == 0 || in_c == -1  in do_dup()
639      * This may be expand_clr for the tail of an operation, e.g.  in do_dup()
705         expand_clr(dofs + oprsz, maxsz - oprsz);  in do_dup()
715 /* Expand OPSZ bytes worth of two-operand operations using i32 elements.  */
776 /* Expand OPSZ bytes worth of three-operand operations using i32 elements.  */
827 /* Expand OPSZ bytes worth of three-operand operations using i32 elements.  */
878 /* Expand OPSZ bytes worth of two-operand operations using i64 elements.  */
939 /* Expand OPSZ bytes worth of three-operand operations using i64 elements.  */
990 /* Expand OPSZ bytes worth of three-operand operations using i64 elements.  */
1041 /* Expand OPSZ bytes worth of two-operand operations using host vectors.  */
1060 /* Expand OPSZ bytes worth of two-vector operands and an immediate operand
1099 /* Expand OPSZ bytes worth of three-operand operations using host vectors.  */
1121  * Expand OPSZ bytes worth of three-vector operands and an immediate operand
1149 /* Expand OPSZ bytes worth of four-operand operations using host vectors.  */
1174  * Expand OPSZ bytes worth of four-vector operands and an immediate operand
1197 /* Expand a vector two-operand operation.  */
1199                     uint32_t oprsz, uint32_t maxsz, const GVecGen2 *g)  in tcg_gen_gvec_2()  argument
1201     const TCGOpcode *this_list = g->opt_opc ? : vecop_list_empty;  in tcg_gen_gvec_2()
1210     if (g->fniv) {  in tcg_gen_gvec_2()
1211         type = choose_vector_type(g->opt_opc, g->vece, oprsz, g->prefer_i64);  in tcg_gen_gvec_2()
1216          * power of 2, but always a multiple of 16.  The intent is  in tcg_gen_gvec_2()
1217          * that e.g. size == 80 would be expanded with 2x32 + 1x16.  in tcg_gen_gvec_2()
1220         expand_2_vec(g->vece, dofs, aofs, some, 32, TCG_TYPE_V256,  in tcg_gen_gvec_2()
1221                      g->load_dest, g->fniv);  in tcg_gen_gvec_2()
1227         oprsz -= some;  in tcg_gen_gvec_2()
1228         maxsz -= some;  in tcg_gen_gvec_2()
1231         expand_2_vec(g->vece, dofs, aofs, oprsz, 16, TCG_TYPE_V128,  in tcg_gen_gvec_2()
1232                      g->load_dest, g->fniv);  in tcg_gen_gvec_2()
1235         expand_2_vec(g->vece, dofs, aofs, oprsz, 8, TCG_TYPE_V64,  in tcg_gen_gvec_2()
1236                      g->load_dest, g->fniv);  in tcg_gen_gvec_2()
1240         if (g->fni8 && check_size_impl(oprsz, 8)) {  in tcg_gen_gvec_2()
1241             expand_2_i64(dofs, aofs, oprsz, g->load_dest, g->fni8);  in tcg_gen_gvec_2()
1242         } else if (g->fni4 && check_size_impl(oprsz, 4)) {  in tcg_gen_gvec_2()
1243             expand_2_i32(dofs, aofs, oprsz, g->load_dest, g->fni4);  in tcg_gen_gvec_2()
1245             assert(g->fno != NULL);  in tcg_gen_gvec_2()
1246             tcg_gen_gvec_2_ool(dofs, aofs, oprsz, maxsz, g->data, g->fno);  in tcg_gen_gvec_2()
1257         expand_clr(dofs + oprsz, maxsz - oprsz);  in tcg_gen_gvec_2()
1263                      uint32_t maxsz, int64_t c, const GVecGen2i *g)  in tcg_gen_gvec_2i()  argument
1265     const TCGOpcode *this_list = g->opt_opc ? : vecop_list_empty;  in tcg_gen_gvec_2i()
1274     if (g->fniv) {  in tcg_gen_gvec_2i()
1275         type = choose_vector_type(g->opt_opc, g->vece, oprsz, g->prefer_i64);  in tcg_gen_gvec_2i()
1280          * power of 2, but always a multiple of 16.  The intent is  in tcg_gen_gvec_2i()
1281          * that e.g. size == 80 would be expanded with 2x32 + 1x16.  in tcg_gen_gvec_2i()
1284         expand_2i_vec(g->vece, dofs, aofs, some, 32, TCG_TYPE_V256,  in tcg_gen_gvec_2i()
1285                       c, g->load_dest, g->fniv);  in tcg_gen_gvec_2i()
1291         oprsz -= some;  in tcg_gen_gvec_2i()
1292         maxsz -= some;  in tcg_gen_gvec_2i()
1295         expand_2i_vec(g->vece, dofs, aofs, oprsz, 16, TCG_TYPE_V128,  in tcg_gen_gvec_2i()
1296                       c, g->load_dest, g->fniv);  in tcg_gen_gvec_2i()
1299         expand_2i_vec(g->vece, dofs, aofs, oprsz, 8, TCG_TYPE_V64,  in tcg_gen_gvec_2i()
1300                       c, g->load_dest, g->fniv);  in tcg_gen_gvec_2i()
1304         if (g->fni8 && check_size_impl(oprsz, 8)) {  in tcg_gen_gvec_2i()
1305             expand_2i_i64(dofs, aofs, oprsz, c, g->load_dest, g->fni8);  in tcg_gen_gvec_2i()
1306         } else if (g->fni4 && check_size_impl(oprsz, 4)) {  in tcg_gen_gvec_2i()
1307             expand_2i_i32(dofs, aofs, oprsz, c, g->load_dest, g->fni4);  in tcg_gen_gvec_2i()
1309             if (g->fno) {  in tcg_gen_gvec_2i()
1310                 tcg_gen_gvec_2_ool(dofs, aofs, oprsz, maxsz, c, g->fno);  in tcg_gen_gvec_2i()
1314                                     maxsz, c, g->fnoi);  in tcg_gen_gvec_2i()
1326         expand_clr(dofs + oprsz, maxsz - oprsz);  in tcg_gen_gvec_2i()
1332                      uint32_t maxsz, TCGv_i64 c, const GVecGen2s *g)  in tcg_gen_gvec_2s()  argument
1340     if (g->fniv) {  in tcg_gen_gvec_2s()
1341         type = choose_vector_type(g->opt_opc, g->vece, oprsz, g->prefer_i64);  in tcg_gen_gvec_2s()
1344         const TCGOpcode *this_list = g->opt_opc ? : vecop_list_empty;  in tcg_gen_gvec_2s()
1349         tcg_gen_dup_i64_vec(g->vece, t_vec, c);  in tcg_gen_gvec_2s()
1354              * power of 2, but always a multiple of 16.  The intent is  in tcg_gen_gvec_2s()
1355              * that e.g. size == 80 would be expanded with 2x32 + 1x16.  in tcg_gen_gvec_2s()
1358             expand_2s_vec(g->vece, dofs, aofs, some, 32, TCG_TYPE_V256,  in tcg_gen_gvec_2s()
1359                           t_vec, g->scalar_first, g->fniv);  in tcg_gen_gvec_2s()
1365             oprsz -= some;  in tcg_gen_gvec_2s()
1366             maxsz -= some;  in tcg_gen_gvec_2s()
1370             expand_2s_vec(g->vece, dofs, aofs, oprsz, 16, TCG_TYPE_V128,  in tcg_gen_gvec_2s()
1371                           t_vec, g->scalar_first, g->fniv);  in tcg_gen_gvec_2s()
1375             expand_2s_vec(g->vece, dofs, aofs, oprsz, 8, TCG_TYPE_V64,  in tcg_gen_gvec_2s()
1376                           t_vec, g->scalar_first, g->fniv);  in tcg_gen_gvec_2s()
1384     } else if (g->fni8 && check_size_impl(oprsz, 8)) {  in tcg_gen_gvec_2s()
1387         tcg_gen_dup_i64(g->vece, t64, c);  in tcg_gen_gvec_2s()
1388         expand_2s_i64(dofs, aofs, oprsz, t64, g->scalar_first, g->fni8);  in tcg_gen_gvec_2s()
1390     } else if (g->fni4 && check_size_impl(oprsz, 4)) {  in tcg_gen_gvec_2s()
1394         tcg_gen_dup_i32(g->vece, t32, t32);  in tcg_gen_gvec_2s()
1395         expand_2s_i32(dofs, aofs, oprsz, t32, g->scalar_first, g->fni4);  in tcg_gen_gvec_2s()
1398         tcg_gen_gvec_2i_ool(dofs, aofs, c, oprsz, maxsz, 0, g->fno);  in tcg_gen_gvec_2s()
1403         expand_clr(dofs + oprsz, maxsz - oprsz);  in tcg_gen_gvec_2s()
1407 /* Expand a vector three-operand operation.  */
1409                     uint32_t oprsz, uint32_t maxsz, const GVecGen3 *g)  in tcg_gen_gvec_3()  argument
1411     const TCGOpcode *this_list = g->opt_opc ? : vecop_list_empty;  in tcg_gen_gvec_3()
1420     if (g->fniv) {  in tcg_gen_gvec_3()
1421         type = choose_vector_type(g->opt_opc, g->vece, oprsz, g->prefer_i64);  in tcg_gen_gvec_3()
1426          * power of 2, but always a multiple of 16.  The intent is  in tcg_gen_gvec_3()
1427          * that e.g. size == 80 would be expanded with 2x32 + 1x16.  in tcg_gen_gvec_3()
1430         expand_3_vec(g->vece, dofs, aofs, bofs, some, 32, TCG_TYPE_V256,  in tcg_gen_gvec_3()
1431                      g->load_dest, g->fniv);  in tcg_gen_gvec_3()
1438         oprsz -= some;  in tcg_gen_gvec_3()
1439         maxsz -= some;  in tcg_gen_gvec_3()
1442         expand_3_vec(g->vece, dofs, aofs, bofs, oprsz, 16, TCG_TYPE_V128,  in tcg_gen_gvec_3()
1443                      g->load_dest, g->fniv);  in tcg_gen_gvec_3()
1446         expand_3_vec(g->vece, dofs, aofs, bofs, oprsz, 8, TCG_TYPE_V64,  in tcg_gen_gvec_3()
1447                      g->load_dest, g->fniv);  in tcg_gen_gvec_3()
1451         if (g->fni8 && check_size_impl(oprsz, 8)) {  in tcg_gen_gvec_3()
1452             expand_3_i64(dofs, aofs, bofs, oprsz, g->load_dest, g->fni8);  in tcg_gen_gvec_3()
1453         } else if (g->fni4 && check_size_impl(oprsz, 4)) {  in tcg_gen_gvec_3()
1454             expand_3_i32(dofs, aofs, bofs, oprsz, g->load_dest, g->fni4);  in tcg_gen_gvec_3()
1456             assert(g->fno != NULL);  in tcg_gen_gvec_3()
1458                                maxsz, g->data, g->fno);  in tcg_gen_gvec_3()
1469         expand_clr(dofs + oprsz, maxsz - oprsz);  in tcg_gen_gvec_3()
1476                      const GVecGen3i *g)  in tcg_gen_gvec_3i()  argument
1478     const TCGOpcode *this_list = g->opt_opc ? : vecop_list_empty;  in tcg_gen_gvec_3i()
1487     if (g->fniv) {  in tcg_gen_gvec_3i()
1488         type = choose_vector_type(g->opt_opc, g->vece, oprsz, g->prefer_i64);  in tcg_gen_gvec_3i()
1494          * power of 2, but always a multiple of 16.  The intent is  in tcg_gen_gvec_3i()
1495          * that e.g. size == 80 would be expanded with 2x32 + 1x16.  in tcg_gen_gvec_3i()
1498         expand_3i_vec(g->vece, dofs, aofs, bofs, some, 32, TCG_TYPE_V256,  in tcg_gen_gvec_3i()
1499                       c, g->load_dest, g->write_aofs, g->fniv);  in tcg_gen_gvec_3i()
1506         oprsz -= some;  in tcg_gen_gvec_3i()
1507         maxsz -= some;  in tcg_gen_gvec_3i()
1510         expand_3i_vec(g->vece, dofs, aofs, bofs, oprsz, 16, TCG_TYPE_V128,  in tcg_gen_gvec_3i()
1511                       c, g->load_dest, g->write_aofs, g->fniv);  in tcg_gen_gvec_3i()
1514         expand_3i_vec(g->vece, dofs, aofs, bofs, oprsz, 8, TCG_TYPE_V64,  in tcg_gen_gvec_3i()
1515                       c, g->load_dest, g->write_aofs, g->fniv);  in tcg_gen_gvec_3i()
1519         if (g->fni8 && check_size_impl(oprsz, 8)) {  in tcg_gen_gvec_3i()
1521                           g->load_dest, g->write_aofs, g->fni8);  in tcg_gen_gvec_3i()
1522         } else if (g->fni4 && check_size_impl(oprsz, 4)) {  in tcg_gen_gvec_3i()
1524                           g->load_dest, g->write_aofs, g->fni4);  in tcg_gen_gvec_3i()
1526             assert(g->fno != NULL);  in tcg_gen_gvec_3i()
1527             tcg_gen_gvec_3_ool(dofs, aofs, bofs, oprsz, maxsz, c, g->fno);  in tcg_gen_gvec_3i()
1538         expand_clr(dofs + oprsz, maxsz - oprsz);  in tcg_gen_gvec_3i()
1542 /* Expand a vector four-operand operation.  */
1544                     uint32_t oprsz, uint32_t maxsz, const GVecGen4 *g)  in tcg_gen_gvec_4()  argument
1546     const TCGOpcode *this_list = g->opt_opc ? : vecop_list_empty;  in tcg_gen_gvec_4()
1555     if (g->fniv) {  in tcg_gen_gvec_4()
1556         type = choose_vector_type(g->opt_opc, g->vece, oprsz, g->prefer_i64);  in tcg_gen_gvec_4()
1561          * power of 2, but always a multiple of 16.  The intent is  in tcg_gen_gvec_4()
1562          * that e.g. size == 80 would be expanded with 2x32 + 1x16.  in tcg_gen_gvec_4()
1565         expand_4_vec(g->vece, dofs, aofs, bofs, cofs, some,  in tcg_gen_gvec_4()
1566                      32, TCG_TYPE_V256, g->write_aofs, g->fniv);  in tcg_gen_gvec_4()
1574         oprsz -= some;  in tcg_gen_gvec_4()
1575         maxsz -= some;  in tcg_gen_gvec_4()
1578         expand_4_vec(g->vece, dofs, aofs, bofs, cofs, oprsz,  in tcg_gen_gvec_4()
1579                      16, TCG_TYPE_V128, g->write_aofs, g->fniv);  in tcg_gen_gvec_4()
1582         expand_4_vec(g->vece, dofs, aofs, bofs, cofs, oprsz,  in tcg_gen_gvec_4()
1583                      8, TCG_TYPE_V64, g->write_aofs, g->fniv);  in tcg_gen_gvec_4()
1587         if (g->fni8 && check_size_impl(oprsz, 8)) {  in tcg_gen_gvec_4()
1589                          g->write_aofs, g->fni8);  in tcg_gen_gvec_4()
1590         } else if (g->fni4 && check_size_impl(oprsz, 4)) {  in tcg_gen_gvec_4()
1592                          g->write_aofs, g->fni4);  in tcg_gen_gvec_4()
1594             assert(g->fno != NULL);  in tcg_gen_gvec_4()
1596                                oprsz, maxsz, g->data, g->fno);  in tcg_gen_gvec_4()
1607         expand_clr(dofs + oprsz, maxsz - oprsz);  in tcg_gen_gvec_4()
1611 /* Expand a vector four-operand operation.  */
1614                      const GVecGen4i *g)  in tcg_gen_gvec_4i()  argument
1616     const TCGOpcode *this_list = g->opt_opc ? : vecop_list_empty;  in tcg_gen_gvec_4i()
1625     if (g->fniv) {  in tcg_gen_gvec_4i()
1626         type = choose_vector_type(g->opt_opc, g->vece, oprsz, g->prefer_i64);  in tcg_gen_gvec_4i()
1632          * power of 2, but always a multiple of 16.  The intent is  in tcg_gen_gvec_4i()
1633          * that e.g. size == 80 would be expanded with 2x32 + 1x16.  in tcg_gen_gvec_4i()
1636         expand_4i_vec(g->vece, dofs, aofs, bofs, cofs, some,  in tcg_gen_gvec_4i()
1637                       32, TCG_TYPE_V256, c, g->fniv);  in tcg_gen_gvec_4i()
1645         oprsz -= some;  in tcg_gen_gvec_4i()
1646         maxsz -= some;  in tcg_gen_gvec_4i()
1649         expand_4i_vec(g->vece, dofs, aofs, bofs, cofs, oprsz,  in tcg_gen_gvec_4i()
1650                        16, TCG_TYPE_V128, c, g->fniv);  in tcg_gen_gvec_4i()
1653         expand_4i_vec(g->vece, dofs, aofs, bofs, cofs, oprsz,  in tcg_gen_gvec_4i()
1654                       8, TCG_TYPE_V64, c, g->fniv);  in tcg_gen_gvec_4i()
1658         if (g->fni8 && check_size_impl(oprsz, 8)) {  in tcg_gen_gvec_4i()
1659             expand_4i_i64(dofs, aofs, bofs, cofs, oprsz, c, g->fni8);  in tcg_gen_gvec_4i()
1660         } else if (g->fni4 && check_size_impl(oprsz, 4)) {  in tcg_gen_gvec_4i()
1661             expand_4i_i32(dofs, aofs, bofs, cofs, oprsz, c, g->fni4);  in tcg_gen_gvec_4i()
1663             assert(g->fno != NULL);  in tcg_gen_gvec_4i()
1665                                oprsz, maxsz, c, g->fno);  in tcg_gen_gvec_4i()
1676         expand_clr(dofs + oprsz, maxsz - oprsz);  in tcg_gen_gvec_4i()
1692     static const GVecGen2 g = {  in tcg_gen_gvec_mov()  local
1699         tcg_gen_gvec_2(dofs, aofs, oprsz, maxsz, &g);  in tcg_gen_gvec_mov()
1703             expand_clr(dofs + oprsz, maxsz - oprsz);  in tcg_gen_gvec_mov()
1756         /* 128-bit duplicate.  */  in tcg_gen_gvec_dup_mem()
1759         tcg_debug_assert(oprsz >= 16);  in tcg_gen_gvec_dup_mem()
1764             for (i = (aofs == dofs) * 16; i < oprsz; i += 16) {  in tcg_gen_gvec_dup_mem()
1773             for (i = (aofs == dofs) * 16; i < oprsz; i += 16) {  in tcg_gen_gvec_dup_mem()
1781             expand_clr(dofs + oprsz, maxsz - oprsz);  in tcg_gen_gvec_dup_mem()
1784         /* 256-bit duplicate.  */  in tcg_gen_gvec_dup_mem()
1801             tcg_gen_ld_vec(in1, tcg_env, aofs + 16);  in tcg_gen_gvec_dup_mem()
1804                 tcg_gen_st_vec(in1, tcg_env, dofs + i + 16);  in tcg_gen_gvec_dup_mem()
1824             expand_clr(dofs + oprsz, maxsz - oprsz);  in tcg_gen_gvec_dup_mem()
1841     static const GVecGen2 g = {  in tcg_gen_gvec_not()  local
1847     tcg_gen_gvec_2(dofs, aofs, oprsz, maxsz, &g);  in tcg_gen_gvec_not()
1851    should be the sign bit of each lane.  This 6-operation form is more
1853    the 64-bit operation.  */
1911     tcg_gen_deposit_i32(d, t1, t2, 0, 16);  in tcg_gen_vec_add16_i32()
1936     static const GVecGen3 g[4] = {  in tcg_gen_gvec_add()  local
1961     tcg_gen_gvec_3(dofs, aofs, bofs, oprsz, maxsz, &g[vece]);  in tcg_gen_gvec_add()
1967     static const GVecGen2s g[4] = {  in tcg_gen_gvec_adds()  local
1992     tcg_gen_gvec_2s(dofs, aofs, oprsz, maxsz, c, &g[vece]);  in tcg_gen_gvec_adds()
2007     static const GVecGen2s g[4] = {  in tcg_gen_gvec_subs()  local
2032     tcg_gen_gvec_2s(dofs, aofs, oprsz, maxsz, c, &g[vece]);  in tcg_gen_gvec_subs()
2094     tcg_gen_deposit_i32(d, t1, t2, 0, 16);  in tcg_gen_vec_sub16_i32()
2117     static const GVecGen3 g[4] = {  in tcg_gen_gvec_sub()  local
2142     tcg_gen_gvec_3(dofs, aofs, bofs, oprsz, maxsz, &g[vece]);  in tcg_gen_gvec_sub()
2150     static const GVecGen3 g[4] = {  in tcg_gen_gvec_mul()  local
2173     tcg_gen_gvec_3(dofs, aofs, bofs, oprsz, maxsz, &g[vece]);  in tcg_gen_gvec_mul()
2179     static const GVecGen2s g[4] = {  in tcg_gen_gvec_muls()  local
2202     tcg_gen_gvec_2s(dofs, aofs, oprsz, maxsz, c, &g[vece]);  in tcg_gen_gvec_muls()
2216     static const GVecGen3 g[4] = {  in tcg_gen_gvec_ssadd()  local
2235     tcg_gen_gvec_3(dofs, aofs, bofs, oprsz, maxsz, &g[vece]);  in tcg_gen_gvec_ssadd()
2242     static const GVecGen3 g[4] = {  in tcg_gen_gvec_sssub()  local
2261     tcg_gen_gvec_3(dofs, aofs, bofs, oprsz, maxsz, &g[vece]);  in tcg_gen_gvec_sssub()
2266     TCGv_i32 max = tcg_constant_i32(-1);  in tcg_gen_usadd_i32()
2273     TCGv_i64 max = tcg_constant_i64(-1);  in tcg_gen_usadd_i64()
2282     static const GVecGen3 g[4] = {  in tcg_gen_gvec_usadd()  local
2303     tcg_gen_gvec_3(dofs, aofs, bofs, oprsz, maxsz, &g[vece]);  in tcg_gen_gvec_usadd()
2324     static const GVecGen3 g[4] = {  in tcg_gen_gvec_ussub()  local
2345     tcg_gen_gvec_3(dofs, aofs, bofs, oprsz, maxsz, &g[vece]);  in tcg_gen_gvec_ussub()
2352     static const GVecGen3 g[4] = {  in tcg_gen_gvec_smin()  local
2373     tcg_gen_gvec_3(dofs, aofs, bofs, oprsz, maxsz, &g[vece]);  in tcg_gen_gvec_smin()
2380     static const GVecGen3 g[4] = {  in tcg_gen_gvec_umin()  local
2401     tcg_gen_gvec_3(dofs, aofs, bofs, oprsz, maxsz, &g[vece]);  in tcg_gen_gvec_umin()
2408     static const GVecGen3 g[4] = {  in tcg_gen_gvec_smax()  local
2429     tcg_gen_gvec_3(dofs, aofs, bofs, oprsz, maxsz, &g[vece]);  in tcg_gen_gvec_smax()
2436     static const GVecGen3 g[4] = {  in tcg_gen_gvec_umax()  local
2457     tcg_gen_gvec_3(dofs, aofs, bofs, oprsz, maxsz, &g[vece]);  in tcg_gen_gvec_umax()
2506     static const GVecGen2 g[4] = {  in tcg_gen_gvec_neg()  local
2531     tcg_gen_gvec_2(dofs, aofs, oprsz, maxsz, &g[vece]);  in tcg_gen_gvec_neg()
2539     /* Create -1 for each negative element.  */  in gen_absv_mask()
2540     tcg_gen_shri_i64(t, b, nbit - 1);  in gen_absv_mask()
2542     tcg_gen_muli_i64(t, t, (1 << nbit) - 1);  in gen_absv_mask()
2545      * Invert (via xor -1) and add one.  in gen_absv_mask()
2570     static const GVecGen2 g[4] = {  in tcg_gen_gvec_abs()  local
2595     tcg_gen_gvec_2(dofs, aofs, oprsz, maxsz, &g[vece]);  in tcg_gen_gvec_abs()
2601     static const GVecGen3 g = {  in tcg_gen_gvec_and()  local
2611         tcg_gen_gvec_3(dofs, aofs, bofs, oprsz, maxsz, &g);  in tcg_gen_gvec_and()
2618     static const GVecGen3 g = {  in tcg_gen_gvec_or()  local
2628         tcg_gen_gvec_3(dofs, aofs, bofs, oprsz, maxsz, &g);  in tcg_gen_gvec_or()
2635     static const GVecGen3 g = {  in tcg_gen_gvec_xor()  local
2645         tcg_gen_gvec_3(dofs, aofs, bofs, oprsz, maxsz, &g);  in tcg_gen_gvec_xor()
2652     static const GVecGen3 g = {  in tcg_gen_gvec_andc()  local
2662         tcg_gen_gvec_3(dofs, aofs, bofs, oprsz, maxsz, &g);  in tcg_gen_gvec_andc()
2669     static const GVecGen3 g = {  in tcg_gen_gvec_orc()  local
2677         tcg_gen_gvec_dup_imm(MO_64, dofs, oprsz, maxsz, -1);  in tcg_gen_gvec_orc()
2679         tcg_gen_gvec_3(dofs, aofs, bofs, oprsz, maxsz, &g);  in tcg_gen_gvec_orc()
2686     static const GVecGen3 g = {  in tcg_gen_gvec_nand()  local
2696         tcg_gen_gvec_3(dofs, aofs, bofs, oprsz, maxsz, &g);  in tcg_gen_gvec_nand()
2703     static const GVecGen3 g = {  in tcg_gen_gvec_nor()  local
2713         tcg_gen_gvec_3(dofs, aofs, bofs, oprsz, maxsz, &g);  in tcg_gen_gvec_nor()
2720     static const GVecGen3 g = {  in tcg_gen_gvec_eqv()  local
2728         tcg_gen_gvec_dup_imm(MO_64, dofs, oprsz, maxsz, -1);  in tcg_gen_gvec_eqv()
2730         tcg_gen_gvec_3(dofs, aofs, bofs, oprsz, maxsz, &g);  in tcg_gen_gvec_eqv()
2761     static GVecGen2s g = {  in tcg_gen_gvec_andcs()  local
2771     tcg_gen_gvec_2s(dofs, aofs, oprsz, maxsz, tmp, &g);  in tcg_gen_gvec_andcs()
2855     static const GVecGen2i g[4] = {  in tcg_gen_gvec_shli()  local
2884         tcg_gen_gvec_2i(dofs, aofs, oprsz, maxsz, shift, &g[vece]);  in tcg_gen_gvec_shli()
2920     static const GVecGen2i g[4] = {  in tcg_gen_gvec_shri()  local
2949         tcg_gen_gvec_2i(dofs, aofs, oprsz, maxsz, shift, &g[vece]);  in tcg_gen_gvec_shri()
2961     tcg_gen_muli_i64(s, s, (2 << c) - 2); /* replicate isolated signs */  in tcg_gen_vec_sar8i_i64()
2976     tcg_gen_muli_i64(s, s, (2 << c) - 2); /* replicate isolated signs */  in tcg_gen_vec_sar16i_i64()
2989     tcg_gen_muli_i32(s, s, (2 << c) - 2); /* replicate isolated signs */  in tcg_gen_vec_sar8i_i32()
3004     tcg_gen_muli_i32(s, s, (2 << c) - 2); /* replicate isolated signs */  in tcg_gen_vec_sar16i_i32()
3013     static const GVecGen2i g[4] = {  in tcg_gen_gvec_sari()  local
3042         tcg_gen_gvec_2i(dofs, aofs, oprsz, maxsz, shift, &g[vece]);  in tcg_gen_gvec_sari()
3051     tcg_gen_shri_i64(a, a, 8 - c);  in tcg_gen_vec_rotl8i_i64()
3062     tcg_gen_shri_i64(a, a, 16 - c);  in tcg_gen_vec_rotl16i_i64()
3072     static const GVecGen2i g[4] = {  in tcg_gen_gvec_rotli()  local
3101         tcg_gen_gvec_2i(dofs, aofs, oprsz, maxsz, shift, &g[vece]);  in tcg_gen_gvec_rotli()
3110     tcg_gen_gvec_rotli(vece, dofs, aofs, -shift & ((8 << vece) - 1),  in tcg_gen_gvec_rotri()
3115  * Specialized generation vector shifts by a non-constant scalar.
3145                uint32_t oprsz, uint32_t maxsz, const GVecGen2sh *g)  in do_gvec_shifts()  argument
3154     type = choose_vector_type(g->s_list, vece, oprsz, vece == MO_64);  in do_gvec_shifts()
3161                            TCG_TYPE_V256, shift, g->fniv_s);  in do_gvec_shifts()
3167             oprsz -= some;  in do_gvec_shifts()
3168             maxsz -= some;  in do_gvec_shifts()
3171             expand_2sh_vec(vece, dofs, aofs, oprsz, 16,  in do_gvec_shifts()
3172                            TCG_TYPE_V128, shift, g->fniv_s);  in do_gvec_shifts()
3176                            TCG_TYPE_V64, shift, g->fniv_s);  in do_gvec_shifts()
3186     type = choose_vector_type(g->v_list, vece, oprsz, vece == MO_64);  in do_gvec_shifts()
3204                           v_shift, false, g->fniv_v);  in do_gvec_shifts()
3210             oprsz -= some;  in do_gvec_shifts()
3211             maxsz -= some;  in do_gvec_shifts()
3214             expand_2s_vec(vece, dofs, aofs, oprsz, 16, TCG_TYPE_V128,  in do_gvec_shifts()
3215                           v_shift, false, g->fniv_v);  in do_gvec_shifts()
3219                           v_shift, false, g->fniv_v);  in do_gvec_shifts()
3231         expand_2s_i32(dofs, aofs, oprsz, shift, false, g->fni4);  in do_gvec_shifts()
3235         expand_2s_i64(dofs, aofs, oprsz, sh64, false, g->fni8);  in do_gvec_shifts()
3247         g->fno[vece](a0, a1, desc);  in do_gvec_shifts()
3257         expand_clr(dofs + oprsz, maxsz - oprsz);  in do_gvec_shifts()
3264     static const GVecGen2sh g = {  in tcg_gen_gvec_shls()  local
3280     do_gvec_shifts(vece, dofs, aofs, shift, oprsz, maxsz, &g);  in tcg_gen_gvec_shls()
3286     static const GVecGen2sh g = {  in tcg_gen_gvec_shrs()  local
3302     do_gvec_shifts(vece, dofs, aofs, shift, oprsz, maxsz, &g);  in tcg_gen_gvec_shrs()
3308     static const GVecGen2sh g = {  in tcg_gen_gvec_sars()  local
3324     do_gvec_shifts(vece, dofs, aofs, shift, oprsz, maxsz, &g);  in tcg_gen_gvec_sars()
3330     static const GVecGen2sh g = {  in tcg_gen_gvec_rotls()  local
3346     do_gvec_shifts(vece, dofs, aofs, shift, oprsz, maxsz, &g);  in tcg_gen_gvec_rotls()
3355     tcg_gen_andi_i32(tmp, tmp, (8 << vece) - 1);  in tcg_gen_gvec_rotrs()
3366  * If the target has some other behaviour from out-of-range shifts,
3374     TCGv_vec m = tcg_constant_vec_matching(d, vece, (8 << vece) - 1);  in tcg_gen_shlv_mod_vec()
3403     static const GVecGen3 g[4] = {  in tcg_gen_gvec_shlv()  local
3426     tcg_gen_gvec_3(dofs, aofs, bofs, oprsz, maxsz, &g[vece]);  in tcg_gen_gvec_shlv()
3437     TCGv_vec m = tcg_constant_vec_matching(d, vece, (8 << vece) - 1);  in tcg_gen_shrv_mod_vec()
3466     static const GVecGen3 g[4] = {  in tcg_gen_gvec_shrv()  local
3489     tcg_gen_gvec_3(dofs, aofs, bofs, oprsz, maxsz, &g[vece]);  in tcg_gen_gvec_shrv()
3500     TCGv_vec m = tcg_constant_vec_matching(d, vece, (8 << vece) - 1);  in tcg_gen_sarv_mod_vec()
3529     static const GVecGen3 g[4] = {  in tcg_gen_gvec_sarv()  local
3552     tcg_gen_gvec_3(dofs, aofs, bofs, oprsz, maxsz, &g[vece]);  in tcg_gen_gvec_sarv()
3563     TCGv_vec m = tcg_constant_vec_matching(d, vece, (8 << vece) - 1);  in tcg_gen_rotlv_mod_vec()
3592     static const GVecGen3 g[4] = {  in tcg_gen_gvec_rotlv()  local
3615     tcg_gen_gvec_3(dofs, aofs, bofs, oprsz, maxsz, &g[vece]);  in tcg_gen_gvec_rotlv()
3622     TCGv_vec m = tcg_constant_vec_matching(d, vece, (8 << vece) - 1);  in tcg_gen_rotrv_mod_vec()
3651     static const GVecGen3 g[4] = {  in tcg_gen_gvec_rotrv()  local
3674     tcg_gen_gvec_3(dofs, aofs, bofs, oprsz, maxsz, &g[vece]);  in tcg_gen_gvec_rotrv()
3677 /* Expand OPSZ bytes worth of three-operand operations using i32 elements.  */
3757     static gen_helper_gvec_3 * const * const fns[16] = {  in tcg_gen_gvec_cmp()
3775                NULL, NULL, -(cond == TCG_COND_ALWAYS));  in tcg_gen_gvec_cmp()
3781      * Prefer integer when 64-bit host and 64-bit comparison.  in tcg_gen_gvec_cmp()
3789          * power of 2, but always a multiple of 16.  The intent is  in tcg_gen_gvec_cmp()
3790          * that e.g. size == 80 would be expanded with 2x32 + 1x16.  in tcg_gen_gvec_cmp()
3800         oprsz -= some;  in tcg_gen_gvec_cmp()
3801         maxsz -= some;  in tcg_gen_gvec_cmp()
3804         expand_cmp_vec(vece, dofs, aofs, bofs, oprsz, 16, TCG_TYPE_V128, cond);  in tcg_gen_gvec_cmp()
3836         expand_clr(dofs + oprsz, maxsz - oprsz);  in tcg_gen_gvec_cmp()
3880     static gen_helper_gvec_2i * const * const fns[16] = {  in tcg_gen_gvec_cmps()
3895                NULL, NULL, -(cond == TCG_COND_ALWAYS));  in tcg_gen_gvec_cmps()
3901      * Prefer integer when 64-bit host and 64-bit comparison.  in tcg_gen_gvec_cmps()
3918             oprsz -= some;  in tcg_gen_gvec_cmps()
3919             maxsz -= some;  in tcg_gen_gvec_cmps()
3923             some = QEMU_ALIGN_DOWN(oprsz, 16);  in tcg_gen_gvec_cmps()
3924             expand_cmps_vec(vece, dofs, aofs, some, 16,  in tcg_gen_gvec_cmps()
3977         expand_clr(dofs + oprsz, maxsz - oprsz);  in tcg_gen_gvec_cmps()
4003     static const GVecGen4 g = {  in tcg_gen_gvec_bitsel()  local
4009     tcg_gen_gvec_4(dofs, aofs, bofs, cofs, oprsz, maxsz, &g);  in tcg_gen_gvec_bitsel()