1 /* 2 * ARM AdvSIMD / SVE Vector Helpers 3 * 4 * Copyright (c) 2020 Linaro 5 * 6 * This library is free software; you can redistribute it and/or 7 * modify it under the terms of the GNU Lesser General Public 8 * License as published by the Free Software Foundation; either 9 * version 2.1 of the License, or (at your option) any later version. 10 * 11 * This library is distributed in the hope that it will be useful, 12 * but WITHOUT ANY WARRANTY; without even the implied warranty of 13 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU 14 * Lesser General Public License for more details. 15 * 16 * You should have received a copy of the GNU Lesser General Public 17 * License along with this library; if not, see <http://www.gnu.org/licenses/>. 18 */ 19 20 #ifndef TARGET_ARM_VEC_INTERNAL_H 21 #define TARGET_ARM_VEC_INTERNAL_H 22 23 /* 24 * Note that vector data is stored in host-endian 64-bit chunks, 25 * so addressing units smaller than that needs a host-endian fixup. 26 * 27 * The H<N> macros are used when indexing an array of elements of size N. 28 * 29 * The H1_<N> macros are used when performing byte arithmetic and then 30 * casting the final pointer to a type of size N. 31 */ 32 #if HOST_BIG_ENDIAN 33 #define H1(x) ((x) ^ 7) 34 #define H1_2(x) ((x) ^ 6) 35 #define H1_4(x) ((x) ^ 4) 36 #define H2(x) ((x) ^ 3) 37 #define H4(x) ((x) ^ 1) 38 #else 39 #define H1(x) (x) 40 #define H1_2(x) (x) 41 #define H1_4(x) (x) 42 #define H2(x) (x) 43 #define H4(x) (x) 44 #endif 45 /* 46 * Access to 64-bit elements isn't host-endian dependent; we provide H8 47 * and H1_8 so that when a function is being generated from a macro we 48 * can pass these rather than an empty macro argument, for clarity. 49 */ 50 #define H8(x) (x) 51 #define H1_8(x) (x) 52 53 /* 54 * Expand active predicate bits to bytes, for byte elements. 55 */ 56 extern const uint64_t expand_pred_b_data[256]; 57 static inline uint64_t expand_pred_b(uint8_t byte) 58 { 59 return expand_pred_b_data[byte]; 60 } 61 62 /* Similarly for half-word elements. */ 63 extern const uint64_t expand_pred_h_data[0x55 + 1]; 64 static inline uint64_t expand_pred_h(uint8_t byte) 65 { 66 return expand_pred_h_data[byte & 0x55]; 67 } 68 69 static inline void clear_tail(void *vd, uintptr_t opr_sz, uintptr_t max_sz) 70 { 71 uint64_t *d = vd + opr_sz; 72 uintptr_t i; 73 74 for (i = opr_sz; i < max_sz; i += 8) { 75 *d++ = 0; 76 } 77 } 78 79 static inline int32_t do_sqrshl_bhs(int32_t src, int32_t shift, int bits, 80 bool round, uint32_t *sat) 81 { 82 if (shift <= -bits) { 83 /* Rounding the sign bit always produces 0. */ 84 if (round) { 85 return 0; 86 } 87 return src >> 31; 88 } else if (shift < 0) { 89 if (round) { 90 src >>= -shift - 1; 91 return (src >> 1) + (src & 1); 92 } 93 return src >> -shift; 94 } else if (shift < bits) { 95 int32_t val = src << shift; 96 if (bits == 32) { 97 if (!sat || val >> shift == src) { 98 return val; 99 } 100 } else { 101 int32_t extval = sextract32(val, 0, bits); 102 if (!sat || val == extval) { 103 return extval; 104 } 105 } 106 } else if (!sat || src == 0) { 107 return 0; 108 } 109 110 *sat = 1; 111 return (1u << (bits - 1)) - (src >= 0); 112 } 113 114 static inline uint32_t do_uqrshl_bhs(uint32_t src, int32_t shift, int bits, 115 bool round, uint32_t *sat) 116 { 117 if (shift <= -(bits + round)) { 118 return 0; 119 } else if (shift < 0) { 120 if (round) { 121 src >>= -shift - 1; 122 return (src >> 1) + (src & 1); 123 } 124 return src >> -shift; 125 } else if (shift < bits) { 126 uint32_t val = src << shift; 127 if (bits == 32) { 128 if (!sat || val >> shift == src) { 129 return val; 130 } 131 } else { 132 uint32_t extval = extract32(val, 0, bits); 133 if (!sat || val == extval) { 134 return extval; 135 } 136 } 137 } else if (!sat || src == 0) { 138 return 0; 139 } 140 141 *sat = 1; 142 return MAKE_64BIT_MASK(0, bits); 143 } 144 145 static inline int32_t do_suqrshl_bhs(int32_t src, int32_t shift, int bits, 146 bool round, uint32_t *sat) 147 { 148 if (sat && src < 0) { 149 *sat = 1; 150 return 0; 151 } 152 return do_uqrshl_bhs(src, shift, bits, round, sat); 153 } 154 155 static inline int64_t do_sqrshl_d(int64_t src, int64_t shift, 156 bool round, uint32_t *sat) 157 { 158 if (shift <= -64) { 159 /* Rounding the sign bit always produces 0. */ 160 if (round) { 161 return 0; 162 } 163 return src >> 63; 164 } else if (shift < 0) { 165 if (round) { 166 src >>= -shift - 1; 167 return (src >> 1) + (src & 1); 168 } 169 return src >> -shift; 170 } else if (shift < 64) { 171 int64_t val = src << shift; 172 if (!sat || val >> shift == src) { 173 return val; 174 } 175 } else if (!sat || src == 0) { 176 return 0; 177 } 178 179 *sat = 1; 180 return src < 0 ? INT64_MIN : INT64_MAX; 181 } 182 183 static inline uint64_t do_uqrshl_d(uint64_t src, int64_t shift, 184 bool round, uint32_t *sat) 185 { 186 if (shift <= -(64 + round)) { 187 return 0; 188 } else if (shift < 0) { 189 if (round) { 190 src >>= -shift - 1; 191 return (src >> 1) + (src & 1); 192 } 193 return src >> -shift; 194 } else if (shift < 64) { 195 uint64_t val = src << shift; 196 if (!sat || val >> shift == src) { 197 return val; 198 } 199 } else if (!sat || src == 0) { 200 return 0; 201 } 202 203 *sat = 1; 204 return UINT64_MAX; 205 } 206 207 static inline int64_t do_suqrshl_d(int64_t src, int64_t shift, 208 bool round, uint32_t *sat) 209 { 210 if (sat && src < 0) { 211 *sat = 1; 212 return 0; 213 } 214 return do_uqrshl_d(src, shift, round, sat); 215 } 216 217 int8_t do_sqrdmlah_b(int8_t, int8_t, int8_t, bool, bool); 218 int16_t do_sqrdmlah_h(int16_t, int16_t, int16_t, bool, bool, uint32_t *); 219 int32_t do_sqrdmlah_s(int32_t, int32_t, int32_t, bool, bool, uint32_t *); 220 int64_t do_sqrdmlah_d(int64_t, int64_t, int64_t, bool, bool); 221 222 /** 223 * bfdotadd: 224 * @sum: addend 225 * @e1, @e2: multiplicand vectors 226 * @fpst: floating-point status to use 227 * 228 * BFloat16 2-way dot product of @e1 & @e2, accumulating with @sum. 229 * The @e1 and @e2 operands correspond to the 32-bit source vector 230 * slots and contain two Bfloat16 values each. 231 * 232 * Corresponds to the ARM pseudocode function BFDotAdd, specialized 233 * for the FPCR.EBF == 0 case. 234 */ 235 float32 bfdotadd(float32 sum, uint32_t e1, uint32_t e2, float_status *fpst); 236 /** 237 * bfdotadd_ebf: 238 * @sum: addend 239 * @e1, @e2: multiplicand vectors 240 * @fpst: floating-point status to use 241 * @fpst_odd: floating-point status to use for round-to-odd operations 242 * 243 * BFloat16 2-way dot product of @e1 & @e2, accumulating with @sum. 244 * The @e1 and @e2 operands correspond to the 32-bit source vector 245 * slots and contain two Bfloat16 values each. 246 * 247 * Corresponds to the ARM pseudocode function BFDotAdd, specialized 248 * for the FPCR.EBF == 1 case. 249 */ 250 float32 bfdotadd_ebf(float32 sum, uint32_t e1, uint32_t e2, 251 float_status *fpst, float_status *fpst_odd); 252 253 /** 254 * is_ebf: 255 * @env: CPU state 256 * @statusp: pointer to floating point status to fill in 257 * @oddstatusp: pointer to floating point status to fill in for round-to-odd 258 * 259 * Determine whether a BFDotAdd operation should use FPCR.EBF = 0 260 * or FPCR.EBF = 1 semantics. On return, has initialized *statusp 261 * and *oddstatusp to suitable float_status arguments to use with either 262 * bfdotadd() or bfdotadd_ebf(). 263 * Returns true for EBF = 1, false for EBF = 0. (The caller should use this 264 * to decide whether to call bfdotadd() or bfdotadd_ebf().) 265 */ 266 bool is_ebf(CPUARMState *env, float_status *statusp, float_status *oddstatusp); 267 268 #endif /* TARGET_ARM_VEC_INTERNAL_H */ 269