1 /* 2 * Generic vector operation expansion 3 * 4 * Copyright (c) 2018 Linaro 5 * 6 * This library is free software; you can redistribute it and/or 7 * modify it under the terms of the GNU Lesser General Public 8 * License as published by the Free Software Foundation; either 9 * version 2.1 of the License, or (at your option) any later version. 10 * 11 * This library is distributed in the hope that it will be useful, 12 * but WITHOUT ANY WARRANTY; without even the implied warranty of 13 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU 14 * Lesser General Public License for more details. 15 * 16 * You should have received a copy of the GNU Lesser General Public 17 * License along with this library; if not, see <http://www.gnu.org/licenses/>. 18 */ 19 20 #ifndef TCG_TCG_OP_GVEC_H 21 #define TCG_TCG_OP_GVEC_H 22 23 /* 24 * "Generic" vectors. All operands are given as offsets from ENV, 25 * and therefore cannot also be allocated via tcg_global_mem_new_*. 26 * OPRSZ is the byte size of the vector upon which the operation is performed. 27 * MAXSZ is the byte size of the full vector; bytes beyond OPSZ are cleared. 28 * 29 * All sizes must be 8 or any multiple of 16. 30 * When OPRSZ is 8, the alignment may be 8, otherwise must be 16. 31 * Operands may completely, but not partially, overlap. 32 */ 33 34 /* Expand a call to a gvec-style helper, with pointers to two vector 35 operands, and a descriptor (see tcg-gvec-desc.h). */ 36 typedef void gen_helper_gvec_2(TCGv_ptr, TCGv_ptr, TCGv_i32); 37 void tcg_gen_gvec_2_ool(uint32_t dofs, uint32_t aofs, 38 uint32_t oprsz, uint32_t maxsz, int32_t data, 39 gen_helper_gvec_2 *fn); 40 41 /* Similarly, passing an extra data value. */ 42 typedef void gen_helper_gvec_2i(TCGv_ptr, TCGv_ptr, TCGv_i64, TCGv_i32); 43 void tcg_gen_gvec_2i_ool(uint32_t dofs, uint32_t aofs, TCGv_i64 c, 44 uint32_t oprsz, uint32_t maxsz, int32_t data, 45 gen_helper_gvec_2i *fn); 46 47 /* Similarly, passing an extra pointer (e.g. env or float_status). */ 48 typedef void gen_helper_gvec_2_ptr(TCGv_ptr, TCGv_ptr, TCGv_ptr, TCGv_i32); 49 void tcg_gen_gvec_2_ptr(uint32_t dofs, uint32_t aofs, 50 TCGv_ptr ptr, uint32_t oprsz, uint32_t maxsz, 51 int32_t data, gen_helper_gvec_2_ptr *fn); 52 53 /* Similarly, with three vector operands. */ 54 typedef void gen_helper_gvec_3(TCGv_ptr, TCGv_ptr, TCGv_ptr, TCGv_i32); 55 void tcg_gen_gvec_3_ool(uint32_t dofs, uint32_t aofs, uint32_t bofs, 56 uint32_t oprsz, uint32_t maxsz, int32_t data, 57 gen_helper_gvec_3 *fn); 58 59 /* Similarly, with four vector operands. */ 60 typedef void gen_helper_gvec_4(TCGv_ptr, TCGv_ptr, TCGv_ptr, 61 TCGv_ptr, TCGv_i32); 62 void tcg_gen_gvec_4_ool(uint32_t dofs, uint32_t aofs, uint32_t bofs, 63 uint32_t cofs, uint32_t oprsz, uint32_t maxsz, 64 int32_t data, gen_helper_gvec_4 *fn); 65 66 /* Similarly, with five vector operands. */ 67 typedef void gen_helper_gvec_5(TCGv_ptr, TCGv_ptr, TCGv_ptr, TCGv_ptr, 68 TCGv_ptr, TCGv_i32); 69 void tcg_gen_gvec_5_ool(uint32_t dofs, uint32_t aofs, uint32_t bofs, 70 uint32_t cofs, uint32_t xofs, uint32_t oprsz, 71 uint32_t maxsz, int32_t data, gen_helper_gvec_5 *fn); 72 73 typedef void gen_helper_gvec_3_ptr(TCGv_ptr, TCGv_ptr, TCGv_ptr, 74 TCGv_ptr, TCGv_i32); 75 void tcg_gen_gvec_3_ptr(uint32_t dofs, uint32_t aofs, uint32_t bofs, 76 TCGv_ptr ptr, uint32_t oprsz, uint32_t maxsz, 77 int32_t data, gen_helper_gvec_3_ptr *fn); 78 79 typedef void gen_helper_gvec_4_ptr(TCGv_ptr, TCGv_ptr, TCGv_ptr, 80 TCGv_ptr, TCGv_ptr, TCGv_i32); 81 void tcg_gen_gvec_4_ptr(uint32_t dofs, uint32_t aofs, uint32_t bofs, 82 uint32_t cofs, TCGv_ptr ptr, uint32_t oprsz, 83 uint32_t maxsz, int32_t data, 84 gen_helper_gvec_4_ptr *fn); 85 86 typedef void gen_helper_gvec_5_ptr(TCGv_ptr, TCGv_ptr, TCGv_ptr, TCGv_ptr, 87 TCGv_ptr, TCGv_ptr, TCGv_i32); 88 void tcg_gen_gvec_5_ptr(uint32_t dofs, uint32_t aofs, uint32_t bofs, 89 uint32_t cofs, uint32_t eofs, TCGv_ptr ptr, 90 uint32_t oprsz, uint32_t maxsz, int32_t data, 91 gen_helper_gvec_5_ptr *fn); 92 93 /* Expand a gvec operation. Either inline or out-of-line depending on 94 the actual vector size and the operations supported by the host. */ 95 typedef struct { 96 /* Expand inline as a 64-bit or 32-bit integer. 97 Only one of these will be non-NULL. */ 98 void (*fni8)(TCGv_i64, TCGv_i64); 99 void (*fni4)(TCGv_i32, TCGv_i32); 100 /* Expand inline with a host vector type. */ 101 void (*fniv)(unsigned, TCGv_vec, TCGv_vec); 102 /* Expand out-of-line helper w/descriptor. */ 103 gen_helper_gvec_2 *fno; 104 /* The optional opcodes, if any, utilized by .fniv. */ 105 const TCGOpcode *opt_opc; 106 /* The data argument to the out-of-line helper. */ 107 int32_t data; 108 /* The vector element size, if applicable. */ 109 uint8_t vece; 110 /* Prefer i64 to v64. */ 111 bool prefer_i64; 112 /* Load dest as a 2nd source operand. */ 113 bool load_dest; 114 } GVecGen2; 115 116 typedef struct { 117 /* Expand inline as a 64-bit or 32-bit integer. 118 Only one of these will be non-NULL. */ 119 void (*fni8)(TCGv_i64, TCGv_i64, int64_t); 120 void (*fni4)(TCGv_i32, TCGv_i32, int32_t); 121 /* Expand inline with a host vector type. */ 122 void (*fniv)(unsigned, TCGv_vec, TCGv_vec, int64_t); 123 /* Expand out-of-line helper w/descriptor, data in descriptor. */ 124 gen_helper_gvec_2 *fno; 125 /* Expand out-of-line helper w/descriptor, data as argument. */ 126 gen_helper_gvec_2i *fnoi; 127 /* The optional opcodes, if any, utilized by .fniv. */ 128 const TCGOpcode *opt_opc; 129 /* The vector element size, if applicable. */ 130 uint8_t vece; 131 /* Prefer i64 to v64. */ 132 bool prefer_i64; 133 /* Load dest as a 3rd source operand. */ 134 bool load_dest; 135 } GVecGen2i; 136 137 typedef struct { 138 /* Expand inline as a 64-bit or 32-bit integer. 139 Only one of these will be non-NULL. */ 140 void (*fni8)(TCGv_i64, TCGv_i64, TCGv_i64); 141 void (*fni4)(TCGv_i32, TCGv_i32, TCGv_i32); 142 /* Expand inline with a host vector type. */ 143 void (*fniv)(unsigned, TCGv_vec, TCGv_vec, TCGv_vec); 144 /* Expand out-of-line helper w/descriptor. */ 145 gen_helper_gvec_2i *fno; 146 /* The optional opcodes, if any, utilized by .fniv. */ 147 const TCGOpcode *opt_opc; 148 /* The data argument to the out-of-line helper. */ 149 uint32_t data; 150 /* The vector element size, if applicable. */ 151 uint8_t vece; 152 /* Prefer i64 to v64. */ 153 bool prefer_i64; 154 /* Load scalar as 1st source operand. */ 155 bool scalar_first; 156 } GVecGen2s; 157 158 typedef struct { 159 /* Expand inline as a 64-bit or 32-bit integer. 160 Only one of these will be non-NULL. */ 161 void (*fni8)(TCGv_i64, TCGv_i64, TCGv_i64); 162 void (*fni4)(TCGv_i32, TCGv_i32, TCGv_i32); 163 /* Expand inline with a host vector type. */ 164 void (*fniv)(unsigned, TCGv_vec, TCGv_vec, TCGv_vec); 165 /* Expand out-of-line helper w/descriptor. */ 166 gen_helper_gvec_3 *fno; 167 /* The optional opcodes, if any, utilized by .fniv. */ 168 const TCGOpcode *opt_opc; 169 /* The data argument to the out-of-line helper. */ 170 int32_t data; 171 /* The vector element size, if applicable. */ 172 uint8_t vece; 173 /* Prefer i64 to v64. */ 174 bool prefer_i64; 175 /* Load dest as a 3rd source operand. */ 176 bool load_dest; 177 } GVecGen3; 178 179 typedef struct { 180 /* 181 * Expand inline as a 64-bit or 32-bit integer. Only one of these will be 182 * non-NULL. 183 */ 184 void (*fni8)(TCGv_i64, TCGv_i64, TCGv_i64, int64_t); 185 void (*fni4)(TCGv_i32, TCGv_i32, TCGv_i32, int32_t); 186 /* Expand inline with a host vector type. */ 187 void (*fniv)(unsigned, TCGv_vec, TCGv_vec, TCGv_vec, int64_t); 188 /* Expand out-of-line helper w/descriptor, data in descriptor. */ 189 gen_helper_gvec_3 *fno; 190 /* The optional opcodes, if any, utilized by .fniv. */ 191 const TCGOpcode *opt_opc; 192 /* The vector element size, if applicable. */ 193 uint8_t vece; 194 /* Prefer i64 to v64. */ 195 bool prefer_i64; 196 /* Load dest as a 3rd source operand. */ 197 bool load_dest; 198 } GVecGen3i; 199 200 typedef struct { 201 /* Expand inline as a 64-bit or 32-bit integer. 202 Only one of these will be non-NULL. */ 203 void (*fni8)(TCGv_i64, TCGv_i64, TCGv_i64, TCGv_i64); 204 void (*fni4)(TCGv_i32, TCGv_i32, TCGv_i32, TCGv_i32); 205 /* Expand inline with a host vector type. */ 206 void (*fniv)(unsigned, TCGv_vec, TCGv_vec, TCGv_vec, TCGv_vec); 207 /* Expand out-of-line helper w/descriptor. */ 208 gen_helper_gvec_4 *fno; 209 /* The optional opcodes, if any, utilized by .fniv. */ 210 const TCGOpcode *opt_opc; 211 /* The data argument to the out-of-line helper. */ 212 int32_t data; 213 /* The vector element size, if applicable. */ 214 uint8_t vece; 215 /* Prefer i64 to v64. */ 216 bool prefer_i64; 217 /* Write aofs as a 2nd dest operand. */ 218 bool write_aofs; 219 } GVecGen4; 220 221 void tcg_gen_gvec_2(uint32_t dofs, uint32_t aofs, 222 uint32_t oprsz, uint32_t maxsz, const GVecGen2 *); 223 void tcg_gen_gvec_2i(uint32_t dofs, uint32_t aofs, uint32_t oprsz, 224 uint32_t maxsz, int64_t c, const GVecGen2i *); 225 void tcg_gen_gvec_2s(uint32_t dofs, uint32_t aofs, uint32_t oprsz, 226 uint32_t maxsz, TCGv_i64 c, const GVecGen2s *); 227 void tcg_gen_gvec_3(uint32_t dofs, uint32_t aofs, uint32_t bofs, 228 uint32_t oprsz, uint32_t maxsz, const GVecGen3 *); 229 void tcg_gen_gvec_3i(uint32_t dofs, uint32_t aofs, uint32_t bofs, 230 uint32_t oprsz, uint32_t maxsz, int64_t c, 231 const GVecGen3i *); 232 void tcg_gen_gvec_4(uint32_t dofs, uint32_t aofs, uint32_t bofs, uint32_t cofs, 233 uint32_t oprsz, uint32_t maxsz, const GVecGen4 *); 234 235 /* Expand a specific vector operation. */ 236 237 void tcg_gen_gvec_mov(unsigned vece, uint32_t dofs, uint32_t aofs, 238 uint32_t oprsz, uint32_t maxsz); 239 void tcg_gen_gvec_not(unsigned vece, uint32_t dofs, uint32_t aofs, 240 uint32_t oprsz, uint32_t maxsz); 241 void tcg_gen_gvec_neg(unsigned vece, uint32_t dofs, uint32_t aofs, 242 uint32_t oprsz, uint32_t maxsz); 243 void tcg_gen_gvec_abs(unsigned vece, uint32_t dofs, uint32_t aofs, 244 uint32_t oprsz, uint32_t maxsz); 245 246 void tcg_gen_gvec_add(unsigned vece, uint32_t dofs, uint32_t aofs, 247 uint32_t bofs, uint32_t oprsz, uint32_t maxsz); 248 void tcg_gen_gvec_sub(unsigned vece, uint32_t dofs, uint32_t aofs, 249 uint32_t bofs, uint32_t oprsz, uint32_t maxsz); 250 void tcg_gen_gvec_mul(unsigned vece, uint32_t dofs, uint32_t aofs, 251 uint32_t bofs, uint32_t oprsz, uint32_t maxsz); 252 253 void tcg_gen_gvec_addi(unsigned vece, uint32_t dofs, uint32_t aofs, 254 int64_t c, uint32_t oprsz, uint32_t maxsz); 255 void tcg_gen_gvec_muli(unsigned vece, uint32_t dofs, uint32_t aofs, 256 int64_t c, uint32_t oprsz, uint32_t maxsz); 257 258 void tcg_gen_gvec_adds(unsigned vece, uint32_t dofs, uint32_t aofs, 259 TCGv_i64 c, uint32_t oprsz, uint32_t maxsz); 260 void tcg_gen_gvec_subs(unsigned vece, uint32_t dofs, uint32_t aofs, 261 TCGv_i64 c, uint32_t oprsz, uint32_t maxsz); 262 void tcg_gen_gvec_muls(unsigned vece, uint32_t dofs, uint32_t aofs, 263 TCGv_i64 c, uint32_t oprsz, uint32_t maxsz); 264 265 /* Saturated arithmetic. */ 266 void tcg_gen_gvec_ssadd(unsigned vece, uint32_t dofs, uint32_t aofs, 267 uint32_t bofs, uint32_t oprsz, uint32_t maxsz); 268 void tcg_gen_gvec_sssub(unsigned vece, uint32_t dofs, uint32_t aofs, 269 uint32_t bofs, uint32_t oprsz, uint32_t maxsz); 270 void tcg_gen_gvec_usadd(unsigned vece, uint32_t dofs, uint32_t aofs, 271 uint32_t bofs, uint32_t oprsz, uint32_t maxsz); 272 void tcg_gen_gvec_ussub(unsigned vece, uint32_t dofs, uint32_t aofs, 273 uint32_t bofs, uint32_t oprsz, uint32_t maxsz); 274 275 /* Min/max. */ 276 void tcg_gen_gvec_smin(unsigned vece, uint32_t dofs, uint32_t aofs, 277 uint32_t bofs, uint32_t oprsz, uint32_t maxsz); 278 void tcg_gen_gvec_umin(unsigned vece, uint32_t dofs, uint32_t aofs, 279 uint32_t bofs, uint32_t oprsz, uint32_t maxsz); 280 void tcg_gen_gvec_smax(unsigned vece, uint32_t dofs, uint32_t aofs, 281 uint32_t bofs, uint32_t oprsz, uint32_t maxsz); 282 void tcg_gen_gvec_umax(unsigned vece, uint32_t dofs, uint32_t aofs, 283 uint32_t bofs, uint32_t oprsz, uint32_t maxsz); 284 285 void tcg_gen_gvec_and(unsigned vece, uint32_t dofs, uint32_t aofs, 286 uint32_t bofs, uint32_t oprsz, uint32_t maxsz); 287 void tcg_gen_gvec_or(unsigned vece, uint32_t dofs, uint32_t aofs, 288 uint32_t bofs, uint32_t oprsz, uint32_t maxsz); 289 void tcg_gen_gvec_xor(unsigned vece, uint32_t dofs, uint32_t aofs, 290 uint32_t bofs, uint32_t oprsz, uint32_t maxsz); 291 void tcg_gen_gvec_andc(unsigned vece, uint32_t dofs, uint32_t aofs, 292 uint32_t bofs, uint32_t oprsz, uint32_t maxsz); 293 void tcg_gen_gvec_orc(unsigned vece, uint32_t dofs, uint32_t aofs, 294 uint32_t bofs, uint32_t oprsz, uint32_t maxsz); 295 void tcg_gen_gvec_nand(unsigned vece, uint32_t dofs, uint32_t aofs, 296 uint32_t bofs, uint32_t oprsz, uint32_t maxsz); 297 void tcg_gen_gvec_nor(unsigned vece, uint32_t dofs, uint32_t aofs, 298 uint32_t bofs, uint32_t oprsz, uint32_t maxsz); 299 void tcg_gen_gvec_eqv(unsigned vece, uint32_t dofs, uint32_t aofs, 300 uint32_t bofs, uint32_t oprsz, uint32_t maxsz); 301 302 void tcg_gen_gvec_andi(unsigned vece, uint32_t dofs, uint32_t aofs, 303 int64_t c, uint32_t oprsz, uint32_t maxsz); 304 void tcg_gen_gvec_xori(unsigned vece, uint32_t dofs, uint32_t aofs, 305 int64_t c, uint32_t oprsz, uint32_t maxsz); 306 void tcg_gen_gvec_ori(unsigned vece, uint32_t dofs, uint32_t aofs, 307 int64_t c, uint32_t oprsz, uint32_t maxsz); 308 309 void tcg_gen_gvec_ands(unsigned vece, uint32_t dofs, uint32_t aofs, 310 TCGv_i64 c, uint32_t oprsz, uint32_t maxsz); 311 void tcg_gen_gvec_xors(unsigned vece, uint32_t dofs, uint32_t aofs, 312 TCGv_i64 c, uint32_t oprsz, uint32_t maxsz); 313 void tcg_gen_gvec_ors(unsigned vece, uint32_t dofs, uint32_t aofs, 314 TCGv_i64 c, uint32_t oprsz, uint32_t maxsz); 315 316 void tcg_gen_gvec_dup_mem(unsigned vece, uint32_t dofs, uint32_t aofs, 317 uint32_t s, uint32_t m); 318 void tcg_gen_gvec_dup_imm(unsigned vece, uint32_t dofs, uint32_t s, 319 uint32_t m, uint64_t imm); 320 void tcg_gen_gvec_dup_i32(unsigned vece, uint32_t dofs, uint32_t s, 321 uint32_t m, TCGv_i32); 322 void tcg_gen_gvec_dup_i64(unsigned vece, uint32_t dofs, uint32_t s, 323 uint32_t m, TCGv_i64); 324 325 #if TARGET_LONG_BITS == 64 326 # define tcg_gen_gvec_dup_tl tcg_gen_gvec_dup_i64 327 #else 328 # define tcg_gen_gvec_dup_tl tcg_gen_gvec_dup_i32 329 #endif 330 331 void tcg_gen_gvec_shli(unsigned vece, uint32_t dofs, uint32_t aofs, 332 int64_t shift, uint32_t oprsz, uint32_t maxsz); 333 void tcg_gen_gvec_shri(unsigned vece, uint32_t dofs, uint32_t aofs, 334 int64_t shift, uint32_t oprsz, uint32_t maxsz); 335 void tcg_gen_gvec_sari(unsigned vece, uint32_t dofs, uint32_t aofs, 336 int64_t shift, uint32_t oprsz, uint32_t maxsz); 337 void tcg_gen_gvec_rotli(unsigned vece, uint32_t dofs, uint32_t aofs, 338 int64_t shift, uint32_t oprsz, uint32_t maxsz); 339 void tcg_gen_gvec_rotri(unsigned vece, uint32_t dofs, uint32_t aofs, 340 int64_t shift, uint32_t oprsz, uint32_t maxsz); 341 342 void tcg_gen_gvec_shls(unsigned vece, uint32_t dofs, uint32_t aofs, 343 TCGv_i32 shift, uint32_t oprsz, uint32_t maxsz); 344 void tcg_gen_gvec_shrs(unsigned vece, uint32_t dofs, uint32_t aofs, 345 TCGv_i32 shift, uint32_t oprsz, uint32_t maxsz); 346 void tcg_gen_gvec_sars(unsigned vece, uint32_t dofs, uint32_t aofs, 347 TCGv_i32 shift, uint32_t oprsz, uint32_t maxsz); 348 349 /* 350 * Perform vector shift by vector element, modulo the element size. 351 * E.g. D[i] = A[i] << (B[i] % (8 << vece)). 352 */ 353 void tcg_gen_gvec_shlv(unsigned vece, uint32_t dofs, uint32_t aofs, 354 uint32_t bofs, uint32_t oprsz, uint32_t maxsz); 355 void tcg_gen_gvec_shrv(unsigned vece, uint32_t dofs, uint32_t aofs, 356 uint32_t bofs, uint32_t oprsz, uint32_t maxsz); 357 void tcg_gen_gvec_sarv(unsigned vece, uint32_t dofs, uint32_t aofs, 358 uint32_t bofs, uint32_t oprsz, uint32_t maxsz); 359 360 void tcg_gen_gvec_cmp(TCGCond cond, unsigned vece, uint32_t dofs, 361 uint32_t aofs, uint32_t bofs, 362 uint32_t oprsz, uint32_t maxsz); 363 364 /* 365 * Perform vector bit select: d = (b & a) | (c & ~a). 366 */ 367 void tcg_gen_gvec_bitsel(unsigned vece, uint32_t dofs, uint32_t aofs, 368 uint32_t bofs, uint32_t cofs, 369 uint32_t oprsz, uint32_t maxsz); 370 371 /* 372 * 64-bit vector operations. Use these when the register has been allocated 373 * with tcg_global_mem_new_i64, and so we cannot also address it via pointer. 374 * OPRSZ = MAXSZ = 8. 375 */ 376 377 void tcg_gen_vec_neg8_i64(TCGv_i64 d, TCGv_i64 a); 378 void tcg_gen_vec_neg16_i64(TCGv_i64 d, TCGv_i64 a); 379 void tcg_gen_vec_neg32_i64(TCGv_i64 d, TCGv_i64 a); 380 381 void tcg_gen_vec_add8_i64(TCGv_i64 d, TCGv_i64 a, TCGv_i64 b); 382 void tcg_gen_vec_add16_i64(TCGv_i64 d, TCGv_i64 a, TCGv_i64 b); 383 void tcg_gen_vec_add32_i64(TCGv_i64 d, TCGv_i64 a, TCGv_i64 b); 384 385 void tcg_gen_vec_sub8_i64(TCGv_i64 d, TCGv_i64 a, TCGv_i64 b); 386 void tcg_gen_vec_sub16_i64(TCGv_i64 d, TCGv_i64 a, TCGv_i64 b); 387 void tcg_gen_vec_sub32_i64(TCGv_i64 d, TCGv_i64 a, TCGv_i64 b); 388 389 void tcg_gen_vec_shl8i_i64(TCGv_i64 d, TCGv_i64 a, int64_t); 390 void tcg_gen_vec_shl16i_i64(TCGv_i64 d, TCGv_i64 a, int64_t); 391 void tcg_gen_vec_shr8i_i64(TCGv_i64 d, TCGv_i64 a, int64_t); 392 void tcg_gen_vec_shr16i_i64(TCGv_i64 d, TCGv_i64 a, int64_t); 393 void tcg_gen_vec_sar8i_i64(TCGv_i64 d, TCGv_i64 a, int64_t); 394 void tcg_gen_vec_sar16i_i64(TCGv_i64 d, TCGv_i64 a, int64_t); 395 void tcg_gen_vec_rotl8i_i64(TCGv_i64 d, TCGv_i64 a, int64_t c); 396 void tcg_gen_vec_rotl16i_i64(TCGv_i64 d, TCGv_i64 a, int64_t c); 397 398 #endif 399