554     int32_t *d = vd, *n = vn, *m = (int32_t *)vm + H4(idx);  in HELPER()
572     int32_t *d = vd, *n = vn, *m = (int32_t *)vm + H4(idx);  in HELPER()
590     int32_t *d = vd, *n = vn, *m = (int32_t *)vm + H4(idx);  in HELPER()
608     int32_t *d = vd, *n = vn, *m = (int32_t *)vm + H4(idx);  in HELPER()
671     int32_t *d = vd, *n = vn, *m = (int32_t *)vm + H4(idx);  in HELPER()
686     int32_t *d = vd, *n = vn, *m = (int32_t *)vm + H4(idx);  in HELPER()
868 DO_DOT_IDX(gvec_sdot_idx_b, int32_t, int8_t, int8_t, H4)
869 DO_DOT_IDX(gvec_udot_idx_b, uint32_t, uint8_t, uint8_t, H4)
870 DO_DOT_IDX(gvec_sudot_idx_b, int32_t, int8_t, uint8_t, H4)
871 DO_DOT_IDX(gvec_usdot_idx_b, int32_t, uint8_t, int8_t, H4)
920         float32 e0 = n[H4(i)];  in HELPER()
921         float32 e1 = m[H4(i + 1)] ^ neg_imag;  in HELPER()
922         float32 e2 = n[H4(i + 1)];  in HELPER()
923         float32 e3 = m[H4(i)] ^ neg_real;  in HELPER()
925         d[H4(i)] = float32_add(e0, e1, fpst);  in HELPER()
926         d[H4(i + 1)] = float32_add(e2, e3, fpst);  in HELPER()
1037         float32 e2 = n[H4(i + flip)];  in HELPER()
1038         float32 e1 = m[H4(i + flip)] ^ neg_real;  in HELPER()
1040         float32 e3 = m[H4(i + 1 - flip)] ^ neg_imag;  in HELPER()
1042         d[H4(i)] = float32_muladd(e2, e1, a[H4(i)], 0, fpst);  in HELPER()
1043         d[H4(i + 1)] = float32_muladd(e4, e3, a[H4(i + 1)], 0, fpst);  in HELPER()
1067         float32 mr = m[H4(i + 2 * index + 0)];  in HELPER()
1068         float32 mi = m[H4(i + 2 * index + 1)];  in HELPER()
1073             float32 e2 = n[H4(j + flip)];  in HELPER()
1076             d[H4(j)] = float32_muladd(e2, e1, a[H4(j)], 0, fpst);  in HELPER()
1077             d[H4(j + 1)] = float32_muladd(e4, e3, a[H4(j + 1)], 0, fpst);  in HELPER()
1569 DO_MUL_IDX(gvec_mul_idx_s, uint32_t, H4)
1591 DO_MLA_IDX(gvec_mla_idx_s, uint32_t, +, H4)
1595 DO_MLA_IDX(gvec_mls_idx_s, uint32_t, -, H4)
1619 DO_FMUL_IDX(gvec_fmul_idx_s, nop, float32_mul, float32, H4)
1625 DO_FMUL_IDX(gvec_fmulx_idx_s, nop, helper_vfp_mulxs, float32, H4)
1637 DO_FMUL_IDX(gvec_fmla_nf_idx_s, float32_add, float32_mul, float32, H4)
1639 DO_FMUL_IDX(gvec_fmls_nf_idx_s, float32_sub, float32_mul, float32, H4)
1664 DO_FMLA_IDX(gvec_fmla_idx_s, float32, H4)
2059         d[H4(i)] = float32_muladd(n_1, m_1, d[H4(i)], 0, fpst);  in do_fmlal()
2124         d[H4(i)] = float32_muladd(n_1, m_1, d[H4(i)], 0, fpst);  in do_fmlal_idx()
2326     intptr_t sel = H4(simd_data(desc));  in HELPER()
2430 DO_3OP_PAIR(gvec_faddp_s, float32_add, float32, H4)
2434 DO_3OP_PAIR(gvec_fmaxp_s, float32_max, float32, H4)
2438 DO_3OP_PAIR(gvec_fminp_s, float32_min, float32, H4)
2442 DO_3OP_PAIR(gvec_fmaxnump_s, float32_maxnum, float32, H4)
2446 DO_3OP_PAIR(gvec_fminnump_s, float32_minnum, float32, H4)
2473 DO_3OP_PAIR(gvec_addp_s, ADD, uint32_t, H4)
2479 DO_3OP_PAIR(gvec_smaxp_s, MAX, int32_t, H4)
2483 DO_3OP_PAIR(gvec_umaxp_s, MAX, uint32_t, H4)
2487 DO_3OP_PAIR(gvec_sminp_s, MIN, int32_t, H4)
2491 DO_3OP_PAIR(gvec_uminp_s, MIN, uint32_t, H4)
2771         sum0 = a[H4(0 + 0)];  in do_mmla_b()
2773         sum1 = a[H4(0 + 1)];  in do_mmla_b()
2775         sum2 = a[H4(2 + 0)];  in do_mmla_b()
2777         sum3 = a[H4(2 + 1)];  in do_mmla_b()
2780         d[H4(0)] = sum0;  in do_mmla_b()
2781         d[H4(1)] = sum1;  in do_mmla_b()
2782         d[H4(2)] = sum2;  in do_mmla_b()
2783         d[H4(3)] = sum3;  in do_mmla_b()
2920             uint32_t m_idx = m[i + H4(index)];  in HELPER()
2928             uint32_t m_idx = m[i + H4(index)];  in HELPER()
2957             sum00 = a[s + H4(0 + 0)];  in HELPER()
2958             sum00 = bfdotadd_ebf(sum00, n[s + H4(0 + 0)], m[s + H4(0 + 0)], &fpst, &fpst_odd);  in HELPER()
2959             sum00 = bfdotadd_ebf(sum00, n[s + H4(0 + 1)], m[s + H4(0 + 1)], &fpst, &fpst_odd);  in HELPER()
2961             sum01 = a[s + H4(0 + 1)];  in HELPER()
2962             sum01 = bfdotadd_ebf(sum01, n[s + H4(0 + 0)], m[s + H4(2 + 0)], &fpst, &fpst_odd);  in HELPER()
2963             sum01 = bfdotadd_ebf(sum01, n[s + H4(0 + 1)], m[s + H4(2 + 1)], &fpst, &fpst_odd);  in HELPER()
2965             sum10 = a[s + H4(2 + 0)];  in HELPER()
2966             sum10 = bfdotadd_ebf(sum10, n[s + H4(2 + 0)], m[s + H4(0 + 0)], &fpst, &fpst_odd);  in HELPER()
2967             sum10 = bfdotadd_ebf(sum10, n[s + H4(2 + 1)], m[s + H4(0 + 1)], &fpst, &fpst_odd);  in HELPER()
2969             sum11 = a[s + H4(2 + 1)];  in HELPER()
2970             sum11 = bfdotadd_ebf(sum11, n[s + H4(2 + 0)], m[s + H4(2 + 0)], &fpst, &fpst_odd);  in HELPER()
2971             sum11 = bfdotadd_ebf(sum11, n[s + H4(2 + 1)], m[s + H4(2 + 1)], &fpst, &fpst_odd);  in HELPER()
2973             d[s + H4(0 + 0)] = sum00;  in HELPER()
2974             d[s + H4(0 + 1)] = sum01;  in HELPER()
2975             d[s + H4(2 + 0)] = sum10;  in HELPER()
2976             d[s + H4(2 + 1)] = sum11;  in HELPER()
2989             sum00 = a[s + H4(0 + 0)];  in HELPER()
2990             sum00 = bfdotadd(sum00, n[s + H4(0 + 0)], m[s + H4(0 + 0)], &fpst);  in HELPER()
2991             sum00 = bfdotadd(sum00, n[s + H4(0 + 1)], m[s + H4(0 + 1)], &fpst);  in HELPER()
2993             sum01 = a[s + H4(0 + 1)];  in HELPER()
2994             sum01 = bfdotadd(sum01, n[s + H4(0 + 0)], m[s + H4(2 + 0)], &fpst);  in HELPER()
2995             sum01 = bfdotadd(sum01, n[s + H4(0 + 1)], m[s + H4(2 + 1)], &fpst);  in HELPER()
2997             sum10 = a[s + H4(2 + 0)];  in HELPER()
2998             sum10 = bfdotadd(sum10, n[s + H4(2 + 0)], m[s + H4(0 + 0)], &fpst);  in HELPER()
2999             sum10 = bfdotadd(sum10, n[s + H4(2 + 1)], m[s + H4(0 + 1)], &fpst);  in HELPER()
3001             sum11 = a[s + H4(2 + 1)];  in HELPER()
3002             sum11 = bfdotadd(sum11, n[s + H4(2 + 0)], m[s + H4(2 + 0)], &fpst);  in HELPER()
3003             sum11 = bfdotadd(sum11, n[s + H4(2 + 1)], m[s + H4(2 + 1)], &fpst);  in HELPER()
3005             d[s + H4(0 + 0)] = sum00;  in HELPER()
3006             d[s + H4(0 + 1)] = sum01;  in HELPER()
3007             d[s + H4(2 + 0)] = sum10;  in HELPER()
3008             d[s + H4(2 + 1)] = sum11;  in HELPER()
3025         d[H4(i)] = float32_muladd(nn, mm, a[H4(i)], 0, stat);  in HELPER()
3046             d[H4(j)] = float32_muladd(n_j, m_idx, a[H4(j)], 0, stat);  in HELPER()