1*a3ef070eSClaudio Fontana /*
2*a3ef070eSClaudio Fontana * ARM AdvSIMD / SVE Vector Helpers
3*a3ef070eSClaudio Fontana *
4*a3ef070eSClaudio Fontana * Copyright (c) 2020 Linaro
5*a3ef070eSClaudio Fontana *
6*a3ef070eSClaudio Fontana * This library is free software; you can redistribute it and/or
7*a3ef070eSClaudio Fontana * modify it under the terms of the GNU Lesser General Public
8*a3ef070eSClaudio Fontana * License as published by the Free Software Foundation; either
9*a3ef070eSClaudio Fontana * version 2.1 of the License, or (at your option) any later version.
10*a3ef070eSClaudio Fontana *
11*a3ef070eSClaudio Fontana * This library is distributed in the hope that it will be useful,
12*a3ef070eSClaudio Fontana * but WITHOUT ANY WARRANTY; without even the implied warranty of
13*a3ef070eSClaudio Fontana * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
14*a3ef070eSClaudio Fontana * Lesser General Public License for more details.
15*a3ef070eSClaudio Fontana *
16*a3ef070eSClaudio Fontana * You should have received a copy of the GNU Lesser General Public
17*a3ef070eSClaudio Fontana * License along with this library; if not, see <http://www.gnu.org/licenses/>.
18*a3ef070eSClaudio Fontana */
19*a3ef070eSClaudio Fontana
20*a3ef070eSClaudio Fontana #ifndef TARGET_ARM_VEC_INTERNAL_H
21*a3ef070eSClaudio Fontana #define TARGET_ARM_VEC_INTERNAL_H
22*a3ef070eSClaudio Fontana
23*a3ef070eSClaudio Fontana /*
24*a3ef070eSClaudio Fontana * Note that vector data is stored in host-endian 64-bit chunks,
25*a3ef070eSClaudio Fontana * so addressing units smaller than that needs a host-endian fixup.
26*a3ef070eSClaudio Fontana *
27*a3ef070eSClaudio Fontana * The H<N> macros are used when indexing an array of elements of size N.
28*a3ef070eSClaudio Fontana *
29*a3ef070eSClaudio Fontana * The H1_<N> macros are used when performing byte arithmetic and then
30*a3ef070eSClaudio Fontana * casting the final pointer to a type of size N.
31*a3ef070eSClaudio Fontana */
32*a3ef070eSClaudio Fontana #if HOST_BIG_ENDIAN
33*a3ef070eSClaudio Fontana #define H1(x) ((x) ^ 7)
34*a3ef070eSClaudio Fontana #define H1_2(x) ((x) ^ 6)
35*a3ef070eSClaudio Fontana #define H1_4(x) ((x) ^ 4)
36*a3ef070eSClaudio Fontana #define H2(x) ((x) ^ 3)
37*a3ef070eSClaudio Fontana #define H4(x) ((x) ^ 1)
38*a3ef070eSClaudio Fontana #else
39*a3ef070eSClaudio Fontana #define H1(x) (x)
40*a3ef070eSClaudio Fontana #define H1_2(x) (x)
41*a3ef070eSClaudio Fontana #define H1_4(x) (x)
42*a3ef070eSClaudio Fontana #define H2(x) (x)
43*a3ef070eSClaudio Fontana #define H4(x) (x)
44*a3ef070eSClaudio Fontana #endif
45*a3ef070eSClaudio Fontana /*
46*a3ef070eSClaudio Fontana * Access to 64-bit elements isn't host-endian dependent; we provide H8
47*a3ef070eSClaudio Fontana * and H1_8 so that when a function is being generated from a macro we
48*a3ef070eSClaudio Fontana * can pass these rather than an empty macro argument, for clarity.
49*a3ef070eSClaudio Fontana */
50*a3ef070eSClaudio Fontana #define H8(x) (x)
51*a3ef070eSClaudio Fontana #define H1_8(x) (x)
52*a3ef070eSClaudio Fontana
53*a3ef070eSClaudio Fontana /*
54*a3ef070eSClaudio Fontana * Expand active predicate bits to bytes, for byte elements.
55*a3ef070eSClaudio Fontana */
56*a3ef070eSClaudio Fontana extern const uint64_t expand_pred_b_data[256];
expand_pred_b(uint8_t byte)57*a3ef070eSClaudio Fontana static inline uint64_t expand_pred_b(uint8_t byte)
58*a3ef070eSClaudio Fontana {
59*a3ef070eSClaudio Fontana return expand_pred_b_data[byte];
60*a3ef070eSClaudio Fontana }
61*a3ef070eSClaudio Fontana
62*a3ef070eSClaudio Fontana /* Similarly for half-word elements. */
63*a3ef070eSClaudio Fontana extern const uint64_t expand_pred_h_data[0x55 + 1];
expand_pred_h(uint8_t byte)64*a3ef070eSClaudio Fontana static inline uint64_t expand_pred_h(uint8_t byte)
65*a3ef070eSClaudio Fontana {
66*a3ef070eSClaudio Fontana return expand_pred_h_data[byte & 0x55];
67*a3ef070eSClaudio Fontana }
68*a3ef070eSClaudio Fontana
clear_tail(void * vd,uintptr_t opr_sz,uintptr_t max_sz)69*a3ef070eSClaudio Fontana static inline void clear_tail(void *vd, uintptr_t opr_sz, uintptr_t max_sz)
70*a3ef070eSClaudio Fontana {
71*a3ef070eSClaudio Fontana uint64_t *d = vd + opr_sz;
72*a3ef070eSClaudio Fontana uintptr_t i;
73*a3ef070eSClaudio Fontana
74*a3ef070eSClaudio Fontana for (i = opr_sz; i < max_sz; i += 8) {
75*a3ef070eSClaudio Fontana *d++ = 0;
76*a3ef070eSClaudio Fontana }
77*a3ef070eSClaudio Fontana }
78*a3ef070eSClaudio Fontana
do_sqrshl_bhs(int32_t src,int32_t shift,int bits,bool round,uint32_t * sat)79*a3ef070eSClaudio Fontana static inline int32_t do_sqrshl_bhs(int32_t src, int32_t shift, int bits,
80*a3ef070eSClaudio Fontana bool round, uint32_t *sat)
81*a3ef070eSClaudio Fontana {
82*a3ef070eSClaudio Fontana if (shift <= -bits) {
83*a3ef070eSClaudio Fontana /* Rounding the sign bit always produces 0. */
84*a3ef070eSClaudio Fontana if (round) {
85*a3ef070eSClaudio Fontana return 0;
86*a3ef070eSClaudio Fontana }
87*a3ef070eSClaudio Fontana return src >> 31;
88*a3ef070eSClaudio Fontana } else if (shift < 0) {
89*a3ef070eSClaudio Fontana if (round) {
90*a3ef070eSClaudio Fontana src >>= -shift - 1;
91*a3ef070eSClaudio Fontana return (src >> 1) + (src & 1);
92*a3ef070eSClaudio Fontana }
93*a3ef070eSClaudio Fontana return src >> -shift;
94*a3ef070eSClaudio Fontana } else if (shift < bits) {
95*a3ef070eSClaudio Fontana int32_t val = src << shift;
96*a3ef070eSClaudio Fontana if (bits == 32) {
97*a3ef070eSClaudio Fontana if (!sat || val >> shift == src) {
98*a3ef070eSClaudio Fontana return val;
99*a3ef070eSClaudio Fontana }
100*a3ef070eSClaudio Fontana } else {
101*a3ef070eSClaudio Fontana int32_t extval = sextract32(val, 0, bits);
102*a3ef070eSClaudio Fontana if (!sat || val == extval) {
103*a3ef070eSClaudio Fontana return extval;
104*a3ef070eSClaudio Fontana }
105*a3ef070eSClaudio Fontana }
106*a3ef070eSClaudio Fontana } else if (!sat || src == 0) {
107*a3ef070eSClaudio Fontana return 0;
108*a3ef070eSClaudio Fontana }
109*a3ef070eSClaudio Fontana
110*a3ef070eSClaudio Fontana *sat = 1;
111*a3ef070eSClaudio Fontana return (1u << (bits - 1)) - (src >= 0);
112*a3ef070eSClaudio Fontana }
113*a3ef070eSClaudio Fontana
do_uqrshl_bhs(uint32_t src,int32_t shift,int bits,bool round,uint32_t * sat)114*a3ef070eSClaudio Fontana static inline uint32_t do_uqrshl_bhs(uint32_t src, int32_t shift, int bits,
115*a3ef070eSClaudio Fontana bool round, uint32_t *sat)
116*a3ef070eSClaudio Fontana {
117*a3ef070eSClaudio Fontana if (shift <= -(bits + round)) {
118*a3ef070eSClaudio Fontana return 0;
119*a3ef070eSClaudio Fontana } else if (shift < 0) {
120*a3ef070eSClaudio Fontana if (round) {
121*a3ef070eSClaudio Fontana src >>= -shift - 1;
122*a3ef070eSClaudio Fontana return (src >> 1) + (src & 1);
123*a3ef070eSClaudio Fontana }
124*a3ef070eSClaudio Fontana return src >> -shift;
125*a3ef070eSClaudio Fontana } else if (shift < bits) {
126*a3ef070eSClaudio Fontana uint32_t val = src << shift;
127*a3ef070eSClaudio Fontana if (bits == 32) {
128*a3ef070eSClaudio Fontana if (!sat || val >> shift == src) {
129*a3ef070eSClaudio Fontana return val;
130*a3ef070eSClaudio Fontana }
131*a3ef070eSClaudio Fontana } else {
132*a3ef070eSClaudio Fontana uint32_t extval = extract32(val, 0, bits);
133*a3ef070eSClaudio Fontana if (!sat || val == extval) {
134*a3ef070eSClaudio Fontana return extval;
135*a3ef070eSClaudio Fontana }
136*a3ef070eSClaudio Fontana }
137*a3ef070eSClaudio Fontana } else if (!sat || src == 0) {
138*a3ef070eSClaudio Fontana return 0;
139*a3ef070eSClaudio Fontana }
140*a3ef070eSClaudio Fontana
141*a3ef070eSClaudio Fontana *sat = 1;
142*a3ef070eSClaudio Fontana return MAKE_64BIT_MASK(0, bits);
143*a3ef070eSClaudio Fontana }
144*a3ef070eSClaudio Fontana
do_suqrshl_bhs(int32_t src,int32_t shift,int bits,bool round,uint32_t * sat)145*a3ef070eSClaudio Fontana static inline int32_t do_suqrshl_bhs(int32_t src, int32_t shift, int bits,
146*a3ef070eSClaudio Fontana bool round, uint32_t *sat)
147*a3ef070eSClaudio Fontana {
148*a3ef070eSClaudio Fontana if (sat && src < 0) {
149*a3ef070eSClaudio Fontana *sat = 1;
150*a3ef070eSClaudio Fontana return 0;
151*a3ef070eSClaudio Fontana }
152*a3ef070eSClaudio Fontana return do_uqrshl_bhs(src, shift, bits, round, sat);
153*a3ef070eSClaudio Fontana }
154*a3ef070eSClaudio Fontana
do_sqrshl_d(int64_t src,int64_t shift,bool round,uint32_t * sat)155*a3ef070eSClaudio Fontana static inline int64_t do_sqrshl_d(int64_t src, int64_t shift,
156*a3ef070eSClaudio Fontana bool round, uint32_t *sat)
157*a3ef070eSClaudio Fontana {
158*a3ef070eSClaudio Fontana if (shift <= -64) {
159*a3ef070eSClaudio Fontana /* Rounding the sign bit always produces 0. */
160*a3ef070eSClaudio Fontana if (round) {
161*a3ef070eSClaudio Fontana return 0;
162*a3ef070eSClaudio Fontana }
163*a3ef070eSClaudio Fontana return src >> 63;
164*a3ef070eSClaudio Fontana } else if (shift < 0) {
165*a3ef070eSClaudio Fontana if (round) {
166*a3ef070eSClaudio Fontana src >>= -shift - 1;
167*a3ef070eSClaudio Fontana return (src >> 1) + (src & 1);
168*a3ef070eSClaudio Fontana }
169*a3ef070eSClaudio Fontana return src >> -shift;
170*a3ef070eSClaudio Fontana } else if (shift < 64) {
171*a3ef070eSClaudio Fontana int64_t val = src << shift;
172*a3ef070eSClaudio Fontana if (!sat || val >> shift == src) {
173*a3ef070eSClaudio Fontana return val;
174*a3ef070eSClaudio Fontana }
175*a3ef070eSClaudio Fontana } else if (!sat || src == 0) {
176*a3ef070eSClaudio Fontana return 0;
177*a3ef070eSClaudio Fontana }
178*a3ef070eSClaudio Fontana
179*a3ef070eSClaudio Fontana *sat = 1;
180*a3ef070eSClaudio Fontana return src < 0 ? INT64_MIN : INT64_MAX;
181*a3ef070eSClaudio Fontana }
182*a3ef070eSClaudio Fontana
do_uqrshl_d(uint64_t src,int64_t shift,bool round,uint32_t * sat)183*a3ef070eSClaudio Fontana static inline uint64_t do_uqrshl_d(uint64_t src, int64_t shift,
184*a3ef070eSClaudio Fontana bool round, uint32_t *sat)
185*a3ef070eSClaudio Fontana {
186*a3ef070eSClaudio Fontana if (shift <= -(64 + round)) {
187*a3ef070eSClaudio Fontana return 0;
188*a3ef070eSClaudio Fontana } else if (shift < 0) {
189*a3ef070eSClaudio Fontana if (round) {
190*a3ef070eSClaudio Fontana src >>= -shift - 1;
191*a3ef070eSClaudio Fontana return (src >> 1) + (src & 1);
192*a3ef070eSClaudio Fontana }
193*a3ef070eSClaudio Fontana return src >> -shift;
194*a3ef070eSClaudio Fontana } else if (shift < 64) {
195*a3ef070eSClaudio Fontana uint64_t val = src << shift;
196*a3ef070eSClaudio Fontana if (!sat || val >> shift == src) {
197*a3ef070eSClaudio Fontana return val;
198*a3ef070eSClaudio Fontana }
199*a3ef070eSClaudio Fontana } else if (!sat || src == 0) {
200*a3ef070eSClaudio Fontana return 0;
201*a3ef070eSClaudio Fontana }
202*a3ef070eSClaudio Fontana
203*a3ef070eSClaudio Fontana *sat = 1;
204*a3ef070eSClaudio Fontana return UINT64_MAX;
205*a3ef070eSClaudio Fontana }
206*a3ef070eSClaudio Fontana
do_suqrshl_d(int64_t src,int64_t shift,bool round,uint32_t * sat)207*a3ef070eSClaudio Fontana static inline int64_t do_suqrshl_d(int64_t src, int64_t shift,
208*a3ef070eSClaudio Fontana bool round, uint32_t *sat)
209*a3ef070eSClaudio Fontana {
210*a3ef070eSClaudio Fontana if (sat && src < 0) {
211*a3ef070eSClaudio Fontana *sat = 1;
212*a3ef070eSClaudio Fontana return 0;
213*a3ef070eSClaudio Fontana }
214*a3ef070eSClaudio Fontana return do_uqrshl_d(src, shift, round, sat);
215*a3ef070eSClaudio Fontana }
216*a3ef070eSClaudio Fontana
217*a3ef070eSClaudio Fontana int8_t do_sqrdmlah_b(int8_t, int8_t, int8_t, bool, bool);
218*a3ef070eSClaudio Fontana int16_t do_sqrdmlah_h(int16_t, int16_t, int16_t, bool, bool, uint32_t *);
219*a3ef070eSClaudio Fontana int32_t do_sqrdmlah_s(int32_t, int32_t, int32_t, bool, bool, uint32_t *);
220*a3ef070eSClaudio Fontana int64_t do_sqrdmlah_d(int64_t, int64_t, int64_t, bool, bool);
221*a3ef070eSClaudio Fontana
222*a3ef070eSClaudio Fontana /**
223*a3ef070eSClaudio Fontana * bfdotadd:
224*a3ef070eSClaudio Fontana * @sum: addend
225*a3ef070eSClaudio Fontana * @e1, @e2: multiplicand vectors
226*a3ef070eSClaudio Fontana *
227*a3ef070eSClaudio Fontana * BFloat16 2-way dot product of @e1 & @e2, accumulating with @sum.
228*a3ef070eSClaudio Fontana * The @e1 and @e2 operands correspond to the 32-bit source vector
229*a3ef070eSClaudio Fontana * slots and contain two Bfloat16 values each.
230*a3ef070eSClaudio Fontana *
231*a3ef070eSClaudio Fontana * Corresponds to the ARM pseudocode function BFDotAdd.
232*a3ef070eSClaudio Fontana */
233*a3ef070eSClaudio Fontana float32 bfdotadd(float32 sum, uint32_t e1, uint32_t e2);
234*a3ef070eSClaudio Fontana
235*a3ef070eSClaudio Fontana #endif /* TARGET_ARM_VEC_INTERNAL_H */
236