1 /* 2 * MMX/3DNow!/SSE/SSE2/SSE3/SSSE3/SSE4/PNI support 3 * 4 * Copyright (c) 2005 Fabrice Bellard 5 * Copyright (c) 2008 Intel Corporation <andrew.zaborowski@intel.com> 6 * 7 * This library is free software; you can redistribute it and/or 8 * modify it under the terms of the GNU Lesser General Public 9 * License as published by the Free Software Foundation; either 10 * version 2.1 of the License, or (at your option) any later version. 11 * 12 * This library is distributed in the hope that it will be useful, 13 * but WITHOUT ANY WARRANTY; without even the implied warranty of 14 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU 15 * Lesser General Public License for more details. 16 * 17 * You should have received a copy of the GNU Lesser General Public 18 * License along with this library; if not, see <http://www.gnu.org/licenses/>. 19 */ 20 21 #include "crypto/aes.h" 22 23 #if SHIFT == 0 24 #define Reg MMXReg 25 #define XMM_ONLY(...) 26 #define B(n) MMX_B(n) 27 #define W(n) MMX_W(n) 28 #define L(n) MMX_L(n) 29 #define Q(n) MMX_Q(n) 30 #define SUFFIX _mmx 31 #else 32 #define Reg ZMMReg 33 #define XMM_ONLY(...) __VA_ARGS__ 34 #define B(n) ZMM_B(n) 35 #define W(n) ZMM_W(n) 36 #define L(n) ZMM_L(n) 37 #define Q(n) ZMM_Q(n) 38 #if SHIFT == 1 39 #define SUFFIX _xmm 40 #else 41 #define SUFFIX _ymm 42 #endif 43 #endif 44 45 #define LANE_WIDTH (SHIFT ? 16 : 8) 46 #define PACK_WIDTH (LANE_WIDTH / 2) 47 48 #if SHIFT == 0 49 #define FPSRL(x, c) ((x) >> shift) 50 #define FPSRAW(x, c) ((int16_t)(x) >> shift) 51 #define FPSRAL(x, c) ((int32_t)(x) >> shift) 52 #define FPSLL(x, c) ((x) << shift) 53 #endif 54 55 void glue(helper_psrlw, SUFFIX)(CPUX86State *env, Reg *d, Reg *s, Reg *c) 56 { 57 int shift; 58 if (c->Q(0) > 15) { 59 for (int i = 0; i < 1 << SHIFT; i++) { 60 d->Q(i) = 0; 61 } 62 } else { 63 shift = c->B(0); 64 for (int i = 0; i < 4 << SHIFT; i++) { 65 d->W(i) = FPSRL(s->W(i), shift); 66 } 67 } 68 } 69 70 void glue(helper_psllw, SUFFIX)(CPUX86State *env, Reg *d, Reg *s, Reg *c) 71 { 72 int shift; 73 if (c->Q(0) > 15) { 74 for (int i = 0; i < 1 << SHIFT; i++) { 75 d->Q(i) = 0; 76 } 77 } else { 78 shift = c->B(0); 79 for (int i = 0; i < 4 << SHIFT; i++) { 80 d->W(i) = FPSLL(s->W(i), shift); 81 } 82 } 83 } 84 85 void glue(helper_psraw, SUFFIX)(CPUX86State *env, Reg *d, Reg *s, Reg *c) 86 { 87 int shift; 88 if (c->Q(0) > 15) { 89 shift = 15; 90 } else { 91 shift = c->B(0); 92 } 93 for (int i = 0; i < 4 << SHIFT; i++) { 94 d->W(i) = FPSRAW(s->W(i), shift); 95 } 96 } 97 98 void glue(helper_psrld, SUFFIX)(CPUX86State *env, Reg *d, Reg *s, Reg *c) 99 { 100 int shift; 101 if (c->Q(0) > 31) { 102 for (int i = 0; i < 1 << SHIFT; i++) { 103 d->Q(i) = 0; 104 } 105 } else { 106 shift = c->B(0); 107 for (int i = 0; i < 2 << SHIFT; i++) { 108 d->L(i) = FPSRL(s->L(i), shift); 109 } 110 } 111 } 112 113 void glue(helper_pslld, SUFFIX)(CPUX86State *env, Reg *d, Reg *s, Reg *c) 114 { 115 int shift; 116 if (c->Q(0) > 31) { 117 for (int i = 0; i < 1 << SHIFT; i++) { 118 d->Q(i) = 0; 119 } 120 } else { 121 shift = c->B(0); 122 for (int i = 0; i < 2 << SHIFT; i++) { 123 d->L(i) = FPSLL(s->L(i), shift); 124 } 125 } 126 } 127 128 void glue(helper_psrad, SUFFIX)(CPUX86State *env, Reg *d, Reg *s, Reg *c) 129 { 130 int shift; 131 if (c->Q(0) > 31) { 132 shift = 31; 133 } else { 134 shift = c->B(0); 135 } 136 for (int i = 0; i < 2 << SHIFT; i++) { 137 d->L(i) = FPSRAL(s->L(i), shift); 138 } 139 } 140 141 void glue(helper_psrlq, SUFFIX)(CPUX86State *env, Reg *d, Reg *s, Reg *c) 142 { 143 int shift; 144 if (c->Q(0) > 63) { 145 for (int i = 0; i < 1 << SHIFT; i++) { 146 d->Q(i) = 0; 147 } 148 } else { 149 shift = c->B(0); 150 for (int i = 0; i < 1 << SHIFT; i++) { 151 d->Q(i) = FPSRL(s->Q(i), shift); 152 } 153 } 154 } 155 156 void glue(helper_psllq, SUFFIX)(CPUX86State *env, Reg *d, Reg *s, Reg *c) 157 { 158 int shift; 159 if (c->Q(0) > 63) { 160 for (int i = 0; i < 1 << SHIFT; i++) { 161 d->Q(i) = 0; 162 } 163 } else { 164 shift = c->B(0); 165 for (int i = 0; i < 1 << SHIFT; i++) { 166 d->Q(i) = FPSLL(s->Q(i), shift); 167 } 168 } 169 } 170 171 #if SHIFT >= 1 172 void glue(helper_psrldq, SUFFIX)(CPUX86State *env, Reg *d, Reg *s, Reg *c) 173 { 174 int shift, i, j; 175 176 shift = c->L(0); 177 if (shift > 16) { 178 shift = 16; 179 } 180 for (j = 0; j < 8 << SHIFT; j += LANE_WIDTH) { 181 for (i = 0; i < 16 - shift; i++) { 182 d->B(j + i) = s->B(j + i + shift); 183 } 184 for (i = 16 - shift; i < 16; i++) { 185 d->B(j + i) = 0; 186 } 187 } 188 } 189 190 void glue(helper_pslldq, SUFFIX)(CPUX86State *env, Reg *d, Reg *s, Reg *c) 191 { 192 int shift, i, j; 193 194 shift = c->L(0); 195 if (shift > 16) { 196 shift = 16; 197 } 198 for (j = 0; j < 8 << SHIFT; j += LANE_WIDTH) { 199 for (i = 15; i >= shift; i--) { 200 d->B(j + i) = s->B(j + i - shift); 201 } 202 for (i = 0; i < shift; i++) { 203 d->B(j + i) = 0; 204 } 205 } 206 } 207 #endif 208 209 #define SSE_HELPER_1(name, elem, num, F) \ 210 void glue(name, SUFFIX)(CPUX86State *env, Reg *d, Reg *s) \ 211 { \ 212 int n = num; \ 213 for (int i = 0; i < n; i++) { \ 214 d->elem(i) = F(s->elem(i)); \ 215 } \ 216 } 217 218 #define SSE_HELPER_2(name, elem, num, F) \ 219 void glue(name, SUFFIX)(CPUX86State *env, Reg *d, Reg *v, Reg *s) \ 220 { \ 221 int n = num; \ 222 for (int i = 0; i < n; i++) { \ 223 d->elem(i) = F(v->elem(i), s->elem(i)); \ 224 } \ 225 } 226 227 #define SSE_HELPER_B(name, F) \ 228 SSE_HELPER_2(name, B, 8 << SHIFT, F) 229 230 #define SSE_HELPER_W(name, F) \ 231 SSE_HELPER_2(name, W, 4 << SHIFT, F) 232 233 #define SSE_HELPER_L(name, F) \ 234 SSE_HELPER_2(name, L, 2 << SHIFT, F) 235 236 #define SSE_HELPER_Q(name, F) \ 237 SSE_HELPER_2(name, Q, 1 << SHIFT, F) 238 239 #if SHIFT == 0 240 static inline int satub(int x) 241 { 242 if (x < 0) { 243 return 0; 244 } else if (x > 255) { 245 return 255; 246 } else { 247 return x; 248 } 249 } 250 251 static inline int satuw(int x) 252 { 253 if (x < 0) { 254 return 0; 255 } else if (x > 65535) { 256 return 65535; 257 } else { 258 return x; 259 } 260 } 261 262 static inline int satsb(int x) 263 { 264 if (x < -128) { 265 return -128; 266 } else if (x > 127) { 267 return 127; 268 } else { 269 return x; 270 } 271 } 272 273 static inline int satsw(int x) 274 { 275 if (x < -32768) { 276 return -32768; 277 } else if (x > 32767) { 278 return 32767; 279 } else { 280 return x; 281 } 282 } 283 284 #define FADD(a, b) ((a) + (b)) 285 #define FADDUB(a, b) satub((a) + (b)) 286 #define FADDUW(a, b) satuw((a) + (b)) 287 #define FADDSB(a, b) satsb((int8_t)(a) + (int8_t)(b)) 288 #define FADDSW(a, b) satsw((int16_t)(a) + (int16_t)(b)) 289 290 #define FSUB(a, b) ((a) - (b)) 291 #define FSUBUB(a, b) satub((a) - (b)) 292 #define FSUBUW(a, b) satuw((a) - (b)) 293 #define FSUBSB(a, b) satsb((int8_t)(a) - (int8_t)(b)) 294 #define FSUBSW(a, b) satsw((int16_t)(a) - (int16_t)(b)) 295 #define FMINUB(a, b) ((a) < (b)) ? (a) : (b) 296 #define FMINSW(a, b) ((int16_t)(a) < (int16_t)(b)) ? (a) : (b) 297 #define FMAXUB(a, b) ((a) > (b)) ? (a) : (b) 298 #define FMAXSW(a, b) ((int16_t)(a) > (int16_t)(b)) ? (a) : (b) 299 300 #define FMULHRW(a, b) (((int16_t)(a) * (int16_t)(b) + 0x8000) >> 16) 301 #define FMULHUW(a, b) ((a) * (b) >> 16) 302 #define FMULHW(a, b) ((int16_t)(a) * (int16_t)(b) >> 16) 303 304 #define FAVG(a, b) (((a) + (b) + 1) >> 1) 305 #endif 306 307 SSE_HELPER_W(helper_pmulhuw, FMULHUW) 308 SSE_HELPER_W(helper_pmulhw, FMULHW) 309 310 #if SHIFT == 0 311 void glue(helper_pmulhrw, SUFFIX)(CPUX86State *env, Reg *d, Reg *s) 312 { 313 d->W(0) = FMULHRW(d->W(0), s->W(0)); 314 d->W(1) = FMULHRW(d->W(1), s->W(1)); 315 d->W(2) = FMULHRW(d->W(2), s->W(2)); 316 d->W(3) = FMULHRW(d->W(3), s->W(3)); 317 } 318 #endif 319 320 SSE_HELPER_B(helper_pavgb, FAVG) 321 SSE_HELPER_W(helper_pavgw, FAVG) 322 323 void glue(helper_pmuludq, SUFFIX)(CPUX86State *env, Reg *d, Reg *v, Reg *s) 324 { 325 int i; 326 327 for (i = 0; i < (1 << SHIFT); i++) { 328 d->Q(i) = (uint64_t)s->L(i * 2) * (uint64_t)v->L(i * 2); 329 } 330 } 331 332 void glue(helper_pmaddwd, SUFFIX)(CPUX86State *env, Reg *d, Reg *v, Reg *s) 333 { 334 int i; 335 336 for (i = 0; i < (2 << SHIFT); i++) { 337 d->L(i) = (int16_t)s->W(2 * i) * (int16_t)v->W(2 * i) + 338 (int16_t)s->W(2 * i + 1) * (int16_t)v->W(2 * i + 1); 339 } 340 } 341 342 #if SHIFT == 0 343 static inline int abs1(int a) 344 { 345 if (a < 0) { 346 return -a; 347 } else { 348 return a; 349 } 350 } 351 #endif 352 void glue(helper_psadbw, SUFFIX)(CPUX86State *env, Reg *d, Reg *v, Reg *s) 353 { 354 int i; 355 356 for (i = 0; i < (1 << SHIFT); i++) { 357 unsigned int val = 0; 358 val += abs1(v->B(8 * i + 0) - s->B(8 * i + 0)); 359 val += abs1(v->B(8 * i + 1) - s->B(8 * i + 1)); 360 val += abs1(v->B(8 * i + 2) - s->B(8 * i + 2)); 361 val += abs1(v->B(8 * i + 3) - s->B(8 * i + 3)); 362 val += abs1(v->B(8 * i + 4) - s->B(8 * i + 4)); 363 val += abs1(v->B(8 * i + 5) - s->B(8 * i + 5)); 364 val += abs1(v->B(8 * i + 6) - s->B(8 * i + 6)); 365 val += abs1(v->B(8 * i + 7) - s->B(8 * i + 7)); 366 d->Q(i) = val; 367 } 368 } 369 370 #if SHIFT < 2 371 void glue(helper_maskmov, SUFFIX)(CPUX86State *env, Reg *d, Reg *s, 372 target_ulong a0) 373 { 374 int i; 375 376 for (i = 0; i < (8 << SHIFT); i++) { 377 if (s->B(i) & 0x80) { 378 cpu_stb_data_ra(env, a0 + i, d->B(i), GETPC()); 379 } 380 } 381 } 382 #endif 383 384 #define SHUFFLE4(F, a, b, offset) do { \ 385 r0 = a->F((order & 3) + offset); \ 386 r1 = a->F(((order >> 2) & 3) + offset); \ 387 r2 = b->F(((order >> 4) & 3) + offset); \ 388 r3 = b->F(((order >> 6) & 3) + offset); \ 389 d->F(offset) = r0; \ 390 d->F(offset + 1) = r1; \ 391 d->F(offset + 2) = r2; \ 392 d->F(offset + 3) = r3; \ 393 } while (0) 394 395 #if SHIFT == 0 396 void glue(helper_pshufw, SUFFIX)(Reg *d, Reg *s, int order) 397 { 398 uint16_t r0, r1, r2, r3; 399 400 SHUFFLE4(W, s, s, 0); 401 } 402 #else 403 void glue(helper_shufps, SUFFIX)(Reg *d, Reg *v, Reg *s, int order) 404 { 405 uint32_t r0, r1, r2, r3; 406 int i; 407 408 for (i = 0; i < 2 << SHIFT; i += 4) { 409 SHUFFLE4(L, v, s, i); 410 } 411 } 412 413 void glue(helper_shufpd, SUFFIX)(Reg *d, Reg *v, Reg *s, int order) 414 { 415 uint64_t r0, r1; 416 int i; 417 418 for (i = 0; i < 1 << SHIFT; i += 2) { 419 r0 = v->Q(((order & 1) & 1) + i); 420 r1 = s->Q(((order >> 1) & 1) + i); 421 d->Q(i) = r0; 422 d->Q(i + 1) = r1; 423 order >>= 2; 424 } 425 } 426 427 void glue(helper_pshufd, SUFFIX)(Reg *d, Reg *s, int order) 428 { 429 uint32_t r0, r1, r2, r3; 430 int i; 431 432 for (i = 0; i < 2 << SHIFT; i += 4) { 433 SHUFFLE4(L, s, s, i); 434 } 435 } 436 437 void glue(helper_pshuflw, SUFFIX)(Reg *d, Reg *s, int order) 438 { 439 uint16_t r0, r1, r2, r3; 440 int i, j; 441 442 for (i = 0, j = 1; j < 1 << SHIFT; i += 8, j += 2) { 443 SHUFFLE4(W, s, s, i); 444 d->Q(j) = s->Q(j); 445 } 446 } 447 448 void glue(helper_pshufhw, SUFFIX)(Reg *d, Reg *s, int order) 449 { 450 uint16_t r0, r1, r2, r3; 451 int i, j; 452 453 for (i = 4, j = 0; j < 1 << SHIFT; i += 8, j += 2) { 454 d->Q(j) = s->Q(j); 455 SHUFFLE4(W, s, s, i); 456 } 457 } 458 #endif 459 460 #if SHIFT >= 1 461 /* FPU ops */ 462 /* XXX: not accurate */ 463 464 #define SSE_HELPER_P(name, F) \ 465 void glue(helper_ ## name ## ps, SUFFIX)(CPUX86State *env, \ 466 Reg *d, Reg *v, Reg *s) \ 467 { \ 468 int i; \ 469 for (i = 0; i < 2 << SHIFT; i++) { \ 470 d->ZMM_S(i) = F(32, v->ZMM_S(i), s->ZMM_S(i)); \ 471 } \ 472 } \ 473 \ 474 void glue(helper_ ## name ## pd, SUFFIX)(CPUX86State *env, \ 475 Reg *d, Reg *v, Reg *s) \ 476 { \ 477 int i; \ 478 for (i = 0; i < 1 << SHIFT; i++) { \ 479 d->ZMM_D(i) = F(64, v->ZMM_D(i), s->ZMM_D(i)); \ 480 } \ 481 } 482 483 #if SHIFT == 1 484 485 #define SSE_HELPER_S(name, F) \ 486 SSE_HELPER_P(name, F) \ 487 \ 488 void helper_ ## name ## ss(CPUX86State *env, Reg *d, Reg *v, Reg *s)\ 489 { \ 490 int i; \ 491 d->ZMM_S(0) = F(32, v->ZMM_S(0), s->ZMM_S(0)); \ 492 for (i = 1; i < 2 << SHIFT; i++) { \ 493 d->ZMM_L(i) = v->ZMM_L(i); \ 494 } \ 495 } \ 496 \ 497 void helper_ ## name ## sd(CPUX86State *env, Reg *d, Reg *v, Reg *s)\ 498 { \ 499 int i; \ 500 d->ZMM_D(0) = F(64, v->ZMM_D(0), s->ZMM_D(0)); \ 501 for (i = 1; i < 1 << SHIFT; i++) { \ 502 d->ZMM_Q(i) = v->ZMM_Q(i); \ 503 } \ 504 } 505 506 #else 507 508 #define SSE_HELPER_S(name, F) SSE_HELPER_P(name, F) 509 510 #endif 511 512 #define FPU_ADD(size, a, b) float ## size ## _add(a, b, &env->sse_status) 513 #define FPU_SUB(size, a, b) float ## size ## _sub(a, b, &env->sse_status) 514 #define FPU_MUL(size, a, b) float ## size ## _mul(a, b, &env->sse_status) 515 #define FPU_DIV(size, a, b) float ## size ## _div(a, b, &env->sse_status) 516 517 /* Note that the choice of comparison op here is important to get the 518 * special cases right: for min and max Intel specifies that (-0,0), 519 * (NaN, anything) and (anything, NaN) return the second argument. 520 */ 521 #define FPU_MIN(size, a, b) \ 522 (float ## size ## _lt(a, b, &env->sse_status) ? (a) : (b)) 523 #define FPU_MAX(size, a, b) \ 524 (float ## size ## _lt(b, a, &env->sse_status) ? (a) : (b)) 525 526 SSE_HELPER_S(add, FPU_ADD) 527 SSE_HELPER_S(sub, FPU_SUB) 528 SSE_HELPER_S(mul, FPU_MUL) 529 SSE_HELPER_S(div, FPU_DIV) 530 SSE_HELPER_S(min, FPU_MIN) 531 SSE_HELPER_S(max, FPU_MAX) 532 533 void glue(helper_sqrtps, SUFFIX)(CPUX86State *env, Reg *d, Reg *s) 534 { 535 int i; 536 for (i = 0; i < 2 << SHIFT; i++) { 537 d->ZMM_S(i) = float32_sqrt(s->ZMM_S(i), &env->sse_status); 538 } 539 } 540 541 void glue(helper_sqrtpd, SUFFIX)(CPUX86State *env, Reg *d, Reg *s) 542 { 543 int i; 544 for (i = 0; i < 1 << SHIFT; i++) { 545 d->ZMM_D(i) = float64_sqrt(s->ZMM_D(i), &env->sse_status); 546 } 547 } 548 549 #if SHIFT == 1 550 void helper_sqrtss(CPUX86State *env, Reg *d, Reg *v, Reg *s) 551 { 552 int i; 553 d->ZMM_S(0) = float32_sqrt(s->ZMM_S(0), &env->sse_status); 554 for (i = 1; i < 2 << SHIFT; i++) { 555 d->ZMM_L(i) = v->ZMM_L(i); 556 } 557 } 558 559 void helper_sqrtsd(CPUX86State *env, Reg *d, Reg *v, Reg *s) 560 { 561 int i; 562 d->ZMM_D(0) = float64_sqrt(s->ZMM_D(0), &env->sse_status); 563 for (i = 1; i < 1 << SHIFT; i++) { 564 d->ZMM_Q(i) = v->ZMM_Q(i); 565 } 566 } 567 #endif 568 569 /* float to float conversions */ 570 void glue(helper_cvtps2pd, SUFFIX)(CPUX86State *env, Reg *d, Reg *s) 571 { 572 int i; 573 for (i = 1 << SHIFT; --i >= 0; ) { 574 d->ZMM_D(i) = float32_to_float64(s->ZMM_S(i), &env->sse_status); 575 } 576 } 577 578 void glue(helper_cvtpd2ps, SUFFIX)(CPUX86State *env, Reg *d, Reg *s) 579 { 580 int i; 581 for (i = 0; i < 1 << SHIFT; i++) { 582 d->ZMM_S(i) = float64_to_float32(s->ZMM_D(i), &env->sse_status); 583 } 584 for (i >>= 1; i < 1 << SHIFT; i++) { 585 d->Q(i) = 0; 586 } 587 } 588 589 #if SHIFT >= 1 590 void glue(helper_cvtph2ps, SUFFIX)(CPUX86State *env, Reg *d, Reg *s) 591 { 592 int i; 593 594 for (i = 2 << SHIFT; --i >= 0; ) { 595 d->ZMM_S(i) = float16_to_float32(s->ZMM_H(i), true, &env->sse_status); 596 } 597 } 598 599 void glue(helper_cvtps2ph, SUFFIX)(CPUX86State *env, Reg *d, Reg *s, int mode) 600 { 601 int i; 602 FloatRoundMode prev_rounding_mode = env->sse_status.float_rounding_mode; 603 if (!(mode & (1 << 2))) { 604 set_x86_rounding_mode(mode & 3, &env->sse_status); 605 } 606 607 for (i = 0; i < 2 << SHIFT; i++) { 608 d->ZMM_H(i) = float32_to_float16(s->ZMM_S(i), true, &env->sse_status); 609 } 610 for (i >>= 2; i < 1 << SHIFT; i++) { 611 d->Q(i) = 0; 612 } 613 614 env->sse_status.float_rounding_mode = prev_rounding_mode; 615 } 616 #endif 617 618 #if SHIFT == 1 619 void helper_cvtss2sd(CPUX86State *env, Reg *d, Reg *v, Reg *s) 620 { 621 int i; 622 d->ZMM_D(0) = float32_to_float64(s->ZMM_S(0), &env->sse_status); 623 for (i = 1; i < 1 << SHIFT; i++) { 624 d->ZMM_Q(i) = v->ZMM_Q(i); 625 } 626 } 627 628 void helper_cvtsd2ss(CPUX86State *env, Reg *d, Reg *v, Reg *s) 629 { 630 int i; 631 d->ZMM_S(0) = float64_to_float32(s->ZMM_D(0), &env->sse_status); 632 for (i = 1; i < 2 << SHIFT; i++) { 633 d->ZMM_L(i) = v->ZMM_L(i); 634 } 635 } 636 #endif 637 638 /* integer to float */ 639 void glue(helper_cvtdq2ps, SUFFIX)(CPUX86State *env, Reg *d, Reg *s) 640 { 641 int i; 642 for (i = 0; i < 2 << SHIFT; i++) { 643 d->ZMM_S(i) = int32_to_float32(s->ZMM_L(i), &env->sse_status); 644 } 645 } 646 647 void glue(helper_cvtdq2pd, SUFFIX)(CPUX86State *env, Reg *d, Reg *s) 648 { 649 int i; 650 for (i = 1 << SHIFT; --i >= 0; ) { 651 int32_t l = s->ZMM_L(i); 652 d->ZMM_D(i) = int32_to_float64(l, &env->sse_status); 653 } 654 } 655 656 #if SHIFT == 1 657 void helper_cvtpi2ps(CPUX86State *env, ZMMReg *d, MMXReg *s) 658 { 659 d->ZMM_S(0) = int32_to_float32(s->MMX_L(0), &env->sse_status); 660 d->ZMM_S(1) = int32_to_float32(s->MMX_L(1), &env->sse_status); 661 } 662 663 void helper_cvtpi2pd(CPUX86State *env, ZMMReg *d, MMXReg *s) 664 { 665 d->ZMM_D(0) = int32_to_float64(s->MMX_L(0), &env->sse_status); 666 d->ZMM_D(1) = int32_to_float64(s->MMX_L(1), &env->sse_status); 667 } 668 669 void helper_cvtsi2ss(CPUX86State *env, ZMMReg *d, uint32_t val) 670 { 671 d->ZMM_S(0) = int32_to_float32(val, &env->sse_status); 672 } 673 674 void helper_cvtsi2sd(CPUX86State *env, ZMMReg *d, uint32_t val) 675 { 676 d->ZMM_D(0) = int32_to_float64(val, &env->sse_status); 677 } 678 679 #ifdef TARGET_X86_64 680 void helper_cvtsq2ss(CPUX86State *env, ZMMReg *d, uint64_t val) 681 { 682 d->ZMM_S(0) = int64_to_float32(val, &env->sse_status); 683 } 684 685 void helper_cvtsq2sd(CPUX86State *env, ZMMReg *d, uint64_t val) 686 { 687 d->ZMM_D(0) = int64_to_float64(val, &env->sse_status); 688 } 689 #endif 690 691 #endif 692 693 /* float to integer */ 694 695 #if SHIFT == 1 696 /* 697 * x86 mandates that we return the indefinite integer value for the result 698 * of any float-to-integer conversion that raises the 'invalid' exception. 699 * Wrap the softfloat functions to get this behaviour. 700 */ 701 #define WRAP_FLOATCONV(RETTYPE, FN, FLOATTYPE, INDEFVALUE) \ 702 static inline RETTYPE x86_##FN(FLOATTYPE a, float_status *s) \ 703 { \ 704 int oldflags, newflags; \ 705 RETTYPE r; \ 706 \ 707 oldflags = get_float_exception_flags(s); \ 708 set_float_exception_flags(0, s); \ 709 r = FN(a, s); \ 710 newflags = get_float_exception_flags(s); \ 711 if (newflags & float_flag_invalid) { \ 712 r = INDEFVALUE; \ 713 } \ 714 set_float_exception_flags(newflags | oldflags, s); \ 715 return r; \ 716 } 717 718 WRAP_FLOATCONV(int32_t, float32_to_int32, float32, INT32_MIN) 719 WRAP_FLOATCONV(int32_t, float32_to_int32_round_to_zero, float32, INT32_MIN) 720 WRAP_FLOATCONV(int32_t, float64_to_int32, float64, INT32_MIN) 721 WRAP_FLOATCONV(int32_t, float64_to_int32_round_to_zero, float64, INT32_MIN) 722 WRAP_FLOATCONV(int64_t, float32_to_int64, float32, INT64_MIN) 723 WRAP_FLOATCONV(int64_t, float32_to_int64_round_to_zero, float32, INT64_MIN) 724 WRAP_FLOATCONV(int64_t, float64_to_int64, float64, INT64_MIN) 725 WRAP_FLOATCONV(int64_t, float64_to_int64_round_to_zero, float64, INT64_MIN) 726 #endif 727 728 void glue(helper_cvtps2dq, SUFFIX)(CPUX86State *env, ZMMReg *d, ZMMReg *s) 729 { 730 int i; 731 for (i = 0; i < 2 << SHIFT; i++) { 732 d->ZMM_L(i) = x86_float32_to_int32(s->ZMM_S(i), &env->sse_status); 733 } 734 } 735 736 void glue(helper_cvtpd2dq, SUFFIX)(CPUX86State *env, ZMMReg *d, ZMMReg *s) 737 { 738 int i; 739 for (i = 0; i < 1 << SHIFT; i++) { 740 d->ZMM_L(i) = x86_float64_to_int32(s->ZMM_D(i), &env->sse_status); 741 } 742 for (i >>= 1; i < 1 << SHIFT; i++) { 743 d->Q(i) = 0; 744 } 745 } 746 747 #if SHIFT == 1 748 void helper_cvtps2pi(CPUX86State *env, MMXReg *d, ZMMReg *s) 749 { 750 d->MMX_L(0) = x86_float32_to_int32(s->ZMM_S(0), &env->sse_status); 751 d->MMX_L(1) = x86_float32_to_int32(s->ZMM_S(1), &env->sse_status); 752 } 753 754 void helper_cvtpd2pi(CPUX86State *env, MMXReg *d, ZMMReg *s) 755 { 756 d->MMX_L(0) = x86_float64_to_int32(s->ZMM_D(0), &env->sse_status); 757 d->MMX_L(1) = x86_float64_to_int32(s->ZMM_D(1), &env->sse_status); 758 } 759 760 int32_t helper_cvtss2si(CPUX86State *env, ZMMReg *s) 761 { 762 return x86_float32_to_int32(s->ZMM_S(0), &env->sse_status); 763 } 764 765 int32_t helper_cvtsd2si(CPUX86State *env, ZMMReg *s) 766 { 767 return x86_float64_to_int32(s->ZMM_D(0), &env->sse_status); 768 } 769 770 #ifdef TARGET_X86_64 771 int64_t helper_cvtss2sq(CPUX86State *env, ZMMReg *s) 772 { 773 return x86_float32_to_int64(s->ZMM_S(0), &env->sse_status); 774 } 775 776 int64_t helper_cvtsd2sq(CPUX86State *env, ZMMReg *s) 777 { 778 return x86_float64_to_int64(s->ZMM_D(0), &env->sse_status); 779 } 780 #endif 781 #endif 782 783 /* float to integer truncated */ 784 void glue(helper_cvttps2dq, SUFFIX)(CPUX86State *env, ZMMReg *d, ZMMReg *s) 785 { 786 int i; 787 for (i = 0; i < 2 << SHIFT; i++) { 788 d->ZMM_L(i) = x86_float32_to_int32_round_to_zero(s->ZMM_S(i), 789 &env->sse_status); 790 } 791 } 792 793 void glue(helper_cvttpd2dq, SUFFIX)(CPUX86State *env, ZMMReg *d, ZMMReg *s) 794 { 795 int i; 796 for (i = 0; i < 1 << SHIFT; i++) { 797 d->ZMM_L(i) = x86_float64_to_int32_round_to_zero(s->ZMM_D(i), 798 &env->sse_status); 799 } 800 for (i >>= 1; i < 1 << SHIFT; i++) { 801 d->Q(i) = 0; 802 } 803 } 804 805 #if SHIFT == 1 806 void helper_cvttps2pi(CPUX86State *env, MMXReg *d, ZMMReg *s) 807 { 808 d->MMX_L(0) = x86_float32_to_int32_round_to_zero(s->ZMM_S(0), &env->sse_status); 809 d->MMX_L(1) = x86_float32_to_int32_round_to_zero(s->ZMM_S(1), &env->sse_status); 810 } 811 812 void helper_cvttpd2pi(CPUX86State *env, MMXReg *d, ZMMReg *s) 813 { 814 d->MMX_L(0) = x86_float64_to_int32_round_to_zero(s->ZMM_D(0), &env->sse_status); 815 d->MMX_L(1) = x86_float64_to_int32_round_to_zero(s->ZMM_D(1), &env->sse_status); 816 } 817 818 int32_t helper_cvttss2si(CPUX86State *env, ZMMReg *s) 819 { 820 return x86_float32_to_int32_round_to_zero(s->ZMM_S(0), &env->sse_status); 821 } 822 823 int32_t helper_cvttsd2si(CPUX86State *env, ZMMReg *s) 824 { 825 return x86_float64_to_int32_round_to_zero(s->ZMM_D(0), &env->sse_status); 826 } 827 828 #ifdef TARGET_X86_64 829 int64_t helper_cvttss2sq(CPUX86State *env, ZMMReg *s) 830 { 831 return x86_float32_to_int64_round_to_zero(s->ZMM_S(0), &env->sse_status); 832 } 833 834 int64_t helper_cvttsd2sq(CPUX86State *env, ZMMReg *s) 835 { 836 return x86_float64_to_int64_round_to_zero(s->ZMM_D(0), &env->sse_status); 837 } 838 #endif 839 #endif 840 841 void glue(helper_rsqrtps, SUFFIX)(CPUX86State *env, ZMMReg *d, ZMMReg *s) 842 { 843 uint8_t old_flags = get_float_exception_flags(&env->sse_status); 844 int i; 845 for (i = 0; i < 2 << SHIFT; i++) { 846 d->ZMM_S(i) = float32_div(float32_one, 847 float32_sqrt(s->ZMM_S(i), &env->sse_status), 848 &env->sse_status); 849 } 850 set_float_exception_flags(old_flags, &env->sse_status); 851 } 852 853 #if SHIFT == 1 854 void helper_rsqrtss(CPUX86State *env, ZMMReg *d, ZMMReg *v, ZMMReg *s) 855 { 856 uint8_t old_flags = get_float_exception_flags(&env->sse_status); 857 int i; 858 d->ZMM_S(0) = float32_div(float32_one, 859 float32_sqrt(s->ZMM_S(0), &env->sse_status), 860 &env->sse_status); 861 set_float_exception_flags(old_flags, &env->sse_status); 862 for (i = 1; i < 2 << SHIFT; i++) { 863 d->ZMM_L(i) = v->ZMM_L(i); 864 } 865 } 866 #endif 867 868 void glue(helper_rcpps, SUFFIX)(CPUX86State *env, ZMMReg *d, ZMMReg *s) 869 { 870 uint8_t old_flags = get_float_exception_flags(&env->sse_status); 871 int i; 872 for (i = 0; i < 2 << SHIFT; i++) { 873 d->ZMM_S(i) = float32_div(float32_one, s->ZMM_S(i), &env->sse_status); 874 } 875 set_float_exception_flags(old_flags, &env->sse_status); 876 } 877 878 #if SHIFT == 1 879 void helper_rcpss(CPUX86State *env, ZMMReg *d, ZMMReg *v, ZMMReg *s) 880 { 881 uint8_t old_flags = get_float_exception_flags(&env->sse_status); 882 int i; 883 d->ZMM_S(0) = float32_div(float32_one, s->ZMM_S(0), &env->sse_status); 884 for (i = 1; i < 2 << SHIFT; i++) { 885 d->ZMM_L(i) = v->ZMM_L(i); 886 } 887 set_float_exception_flags(old_flags, &env->sse_status); 888 } 889 #endif 890 891 #if SHIFT == 1 892 static inline uint64_t helper_extrq(uint64_t src, int shift, int len) 893 { 894 uint64_t mask; 895 896 if (len == 0) { 897 mask = ~0LL; 898 } else { 899 mask = (1ULL << len) - 1; 900 } 901 return (src >> shift) & mask; 902 } 903 904 void helper_extrq_r(CPUX86State *env, ZMMReg *d, ZMMReg *s) 905 { 906 d->ZMM_Q(0) = helper_extrq(d->ZMM_Q(0), s->ZMM_B(1) & 63, s->ZMM_B(0) & 63); 907 } 908 909 void helper_extrq_i(CPUX86State *env, ZMMReg *d, int index, int length) 910 { 911 d->ZMM_Q(0) = helper_extrq(d->ZMM_Q(0), index, length); 912 } 913 914 static inline uint64_t helper_insertq(uint64_t dest, uint64_t src, int shift, int len) 915 { 916 uint64_t mask; 917 918 if (len == 0) { 919 mask = ~0ULL; 920 } else { 921 mask = (1ULL << len) - 1; 922 } 923 return (dest & ~(mask << shift)) | ((src & mask) << shift); 924 } 925 926 void helper_insertq_r(CPUX86State *env, ZMMReg *d, ZMMReg *s) 927 { 928 d->ZMM_Q(0) = helper_insertq(d->ZMM_Q(0), s->ZMM_Q(0), s->ZMM_B(9) & 63, s->ZMM_B(8) & 63); 929 } 930 931 void helper_insertq_i(CPUX86State *env, ZMMReg *d, ZMMReg *s, int index, int length) 932 { 933 d->ZMM_Q(0) = helper_insertq(d->ZMM_Q(0), s->ZMM_Q(0), index, length); 934 } 935 #endif 936 937 #define SSE_HELPER_HPS(name, F) \ 938 void glue(helper_ ## name, SUFFIX)(CPUX86State *env, Reg *d, Reg *v, Reg *s) \ 939 { \ 940 float32 r[2 << SHIFT]; \ 941 int i, j, k; \ 942 for (k = 0; k < 2 << SHIFT; k += LANE_WIDTH / 4) { \ 943 for (i = j = 0; j < 4; i++, j += 2) { \ 944 r[i + k] = F(v->ZMM_S(j + k), v->ZMM_S(j + k + 1), &env->sse_status); \ 945 } \ 946 for (j = 0; j < 4; i++, j += 2) { \ 947 r[i + k] = F(s->ZMM_S(j + k), s->ZMM_S(j + k + 1), &env->sse_status); \ 948 } \ 949 } \ 950 for (i = 0; i < 2 << SHIFT; i++) { \ 951 d->ZMM_S(i) = r[i]; \ 952 } \ 953 } 954 955 SSE_HELPER_HPS(haddps, float32_add) 956 SSE_HELPER_HPS(hsubps, float32_sub) 957 958 #define SSE_HELPER_HPD(name, F) \ 959 void glue(helper_ ## name, SUFFIX)(CPUX86State *env, Reg *d, Reg *v, Reg *s) \ 960 { \ 961 float64 r[1 << SHIFT]; \ 962 int i, j, k; \ 963 for (k = 0; k < 1 << SHIFT; k += LANE_WIDTH / 8) { \ 964 for (i = j = 0; j < 2; i++, j += 2) { \ 965 r[i + k] = F(v->ZMM_D(j + k), v->ZMM_D(j + k + 1), &env->sse_status); \ 966 } \ 967 for (j = 0; j < 2; i++, j += 2) { \ 968 r[i + k] = F(s->ZMM_D(j + k), s->ZMM_D(j + k + 1), &env->sse_status); \ 969 } \ 970 } \ 971 for (i = 0; i < 1 << SHIFT; i++) { \ 972 d->ZMM_D(i) = r[i]; \ 973 } \ 974 } 975 976 SSE_HELPER_HPD(haddpd, float64_add) 977 SSE_HELPER_HPD(hsubpd, float64_sub) 978 979 void glue(helper_addsubps, SUFFIX)(CPUX86State *env, Reg *d, Reg *v, Reg *s) 980 { 981 int i; 982 for (i = 0; i < 2 << SHIFT; i += 2) { 983 d->ZMM_S(i) = float32_sub(v->ZMM_S(i), s->ZMM_S(i), &env->sse_status); 984 d->ZMM_S(i+1) = float32_add(v->ZMM_S(i+1), s->ZMM_S(i+1), &env->sse_status); 985 } 986 } 987 988 void glue(helper_addsubpd, SUFFIX)(CPUX86State *env, Reg *d, Reg *v, Reg *s) 989 { 990 int i; 991 for (i = 0; i < 1 << SHIFT; i += 2) { 992 d->ZMM_D(i) = float64_sub(v->ZMM_D(i), s->ZMM_D(i), &env->sse_status); 993 d->ZMM_D(i+1) = float64_add(v->ZMM_D(i+1), s->ZMM_D(i+1), &env->sse_status); 994 } 995 } 996 997 #define SSE_HELPER_CMP_P(name, F, C) \ 998 void glue(helper_ ## name ## ps, SUFFIX)(CPUX86State *env, \ 999 Reg *d, Reg *v, Reg *s) \ 1000 { \ 1001 int i; \ 1002 for (i = 0; i < 2 << SHIFT; i++) { \ 1003 d->ZMM_L(i) = C(F(32, v->ZMM_S(i), s->ZMM_S(i))) ? -1 : 0; \ 1004 } \ 1005 } \ 1006 \ 1007 void glue(helper_ ## name ## pd, SUFFIX)(CPUX86State *env, \ 1008 Reg *d, Reg *v, Reg *s) \ 1009 { \ 1010 int i; \ 1011 for (i = 0; i < 1 << SHIFT; i++) { \ 1012 d->ZMM_Q(i) = C(F(64, v->ZMM_D(i), s->ZMM_D(i))) ? -1 : 0; \ 1013 } \ 1014 } 1015 1016 #if SHIFT == 1 1017 #define SSE_HELPER_CMP(name, F, C) \ 1018 SSE_HELPER_CMP_P(name, F, C) \ 1019 void helper_ ## name ## ss(CPUX86State *env, Reg *d, Reg *v, Reg *s) \ 1020 { \ 1021 int i; \ 1022 d->ZMM_L(0) = C(F(32, v->ZMM_S(0), s->ZMM_S(0))) ? -1 : 0; \ 1023 for (i = 1; i < 2 << SHIFT; i++) { \ 1024 d->ZMM_L(i) = v->ZMM_L(i); \ 1025 } \ 1026 } \ 1027 \ 1028 void helper_ ## name ## sd(CPUX86State *env, Reg *d, Reg *v, Reg *s) \ 1029 { \ 1030 int i; \ 1031 d->ZMM_Q(0) = C(F(64, v->ZMM_D(0), s->ZMM_D(0))) ? -1 : 0; \ 1032 for (i = 1; i < 1 << SHIFT; i++) { \ 1033 d->ZMM_Q(i) = v->ZMM_Q(i); \ 1034 } \ 1035 } 1036 1037 static inline bool FPU_EQU(FloatRelation x) 1038 { 1039 return (x == float_relation_equal || x == float_relation_unordered); 1040 } 1041 static inline bool FPU_GE(FloatRelation x) 1042 { 1043 return (x == float_relation_equal || x == float_relation_greater); 1044 } 1045 #define FPU_EQ(x) (x == float_relation_equal) 1046 #define FPU_LT(x) (x == float_relation_less) 1047 #define FPU_LE(x) (x <= float_relation_equal) 1048 #define FPU_GT(x) (x == float_relation_greater) 1049 #define FPU_UNORD(x) (x == float_relation_unordered) 1050 /* We must make sure we evaluate the argument in case it is a signalling NAN */ 1051 #define FPU_FALSE(x) (x == float_relation_equal && 0) 1052 1053 #define FPU_CMPQ(size, a, b) \ 1054 float ## size ## _compare_quiet(a, b, &env->sse_status) 1055 #define FPU_CMPS(size, a, b) \ 1056 float ## size ## _compare(a, b, &env->sse_status) 1057 1058 #else 1059 #define SSE_HELPER_CMP(name, F, C) SSE_HELPER_CMP_P(name, F, C) 1060 #endif 1061 1062 SSE_HELPER_CMP(cmpeq, FPU_CMPQ, FPU_EQ) 1063 SSE_HELPER_CMP(cmplt, FPU_CMPS, FPU_LT) 1064 SSE_HELPER_CMP(cmple, FPU_CMPS, FPU_LE) 1065 SSE_HELPER_CMP(cmpunord, FPU_CMPQ, FPU_UNORD) 1066 SSE_HELPER_CMP(cmpneq, FPU_CMPQ, !FPU_EQ) 1067 SSE_HELPER_CMP(cmpnlt, FPU_CMPS, !FPU_LT) 1068 SSE_HELPER_CMP(cmpnle, FPU_CMPS, !FPU_LE) 1069 SSE_HELPER_CMP(cmpord, FPU_CMPQ, !FPU_UNORD) 1070 1071 SSE_HELPER_CMP(cmpequ, FPU_CMPQ, FPU_EQU) 1072 SSE_HELPER_CMP(cmpnge, FPU_CMPS, !FPU_GE) 1073 SSE_HELPER_CMP(cmpngt, FPU_CMPS, !FPU_GT) 1074 SSE_HELPER_CMP(cmpfalse, FPU_CMPQ, FPU_FALSE) 1075 SSE_HELPER_CMP(cmpnequ, FPU_CMPQ, !FPU_EQU) 1076 SSE_HELPER_CMP(cmpge, FPU_CMPS, FPU_GE) 1077 SSE_HELPER_CMP(cmpgt, FPU_CMPS, FPU_GT) 1078 SSE_HELPER_CMP(cmptrue, FPU_CMPQ, !FPU_FALSE) 1079 1080 SSE_HELPER_CMP(cmpeqs, FPU_CMPS, FPU_EQ) 1081 SSE_HELPER_CMP(cmpltq, FPU_CMPQ, FPU_LT) 1082 SSE_HELPER_CMP(cmpleq, FPU_CMPQ, FPU_LE) 1083 SSE_HELPER_CMP(cmpunords, FPU_CMPS, FPU_UNORD) 1084 SSE_HELPER_CMP(cmpneqq, FPU_CMPS, !FPU_EQ) 1085 SSE_HELPER_CMP(cmpnltq, FPU_CMPQ, !FPU_LT) 1086 SSE_HELPER_CMP(cmpnleq, FPU_CMPQ, !FPU_LE) 1087 SSE_HELPER_CMP(cmpords, FPU_CMPS, !FPU_UNORD) 1088 1089 SSE_HELPER_CMP(cmpequs, FPU_CMPS, FPU_EQU) 1090 SSE_HELPER_CMP(cmpngeq, FPU_CMPQ, !FPU_GE) 1091 SSE_HELPER_CMP(cmpngtq, FPU_CMPQ, !FPU_GT) 1092 SSE_HELPER_CMP(cmpfalses, FPU_CMPS, FPU_FALSE) 1093 SSE_HELPER_CMP(cmpnequs, FPU_CMPS, !FPU_EQU) 1094 SSE_HELPER_CMP(cmpgeq, FPU_CMPQ, FPU_GE) 1095 SSE_HELPER_CMP(cmpgtq, FPU_CMPQ, FPU_GT) 1096 SSE_HELPER_CMP(cmptrues, FPU_CMPS, !FPU_FALSE) 1097 1098 #undef SSE_HELPER_CMP 1099 1100 #if SHIFT == 1 1101 static const int comis_eflags[4] = {CC_C, CC_Z, 0, CC_Z | CC_P | CC_C}; 1102 1103 void helper_ucomiss(CPUX86State *env, Reg *d, Reg *s) 1104 { 1105 FloatRelation ret; 1106 float32 s0, s1; 1107 1108 s0 = d->ZMM_S(0); 1109 s1 = s->ZMM_S(0); 1110 ret = float32_compare_quiet(s0, s1, &env->sse_status); 1111 CC_SRC = comis_eflags[ret + 1]; 1112 } 1113 1114 void helper_comiss(CPUX86State *env, Reg *d, Reg *s) 1115 { 1116 FloatRelation ret; 1117 float32 s0, s1; 1118 1119 s0 = d->ZMM_S(0); 1120 s1 = s->ZMM_S(0); 1121 ret = float32_compare(s0, s1, &env->sse_status); 1122 CC_SRC = comis_eflags[ret + 1]; 1123 } 1124 1125 void helper_ucomisd(CPUX86State *env, Reg *d, Reg *s) 1126 { 1127 FloatRelation ret; 1128 float64 d0, d1; 1129 1130 d0 = d->ZMM_D(0); 1131 d1 = s->ZMM_D(0); 1132 ret = float64_compare_quiet(d0, d1, &env->sse_status); 1133 CC_SRC = comis_eflags[ret + 1]; 1134 } 1135 1136 void helper_comisd(CPUX86State *env, Reg *d, Reg *s) 1137 { 1138 FloatRelation ret; 1139 float64 d0, d1; 1140 1141 d0 = d->ZMM_D(0); 1142 d1 = s->ZMM_D(0); 1143 ret = float64_compare(d0, d1, &env->sse_status); 1144 CC_SRC = comis_eflags[ret + 1]; 1145 } 1146 #endif 1147 1148 uint32_t glue(helper_movmskps, SUFFIX)(CPUX86State *env, Reg *s) 1149 { 1150 uint32_t mask; 1151 int i; 1152 1153 mask = 0; 1154 for (i = 0; i < 2 << SHIFT; i++) { 1155 mask |= (s->ZMM_L(i) >> (31 - i)) & (1 << i); 1156 } 1157 return mask; 1158 } 1159 1160 uint32_t glue(helper_movmskpd, SUFFIX)(CPUX86State *env, Reg *s) 1161 { 1162 uint32_t mask; 1163 int i; 1164 1165 mask = 0; 1166 for (i = 0; i < 1 << SHIFT; i++) { 1167 mask |= (s->ZMM_Q(i) >> (63 - i)) & (1 << i); 1168 } 1169 return mask; 1170 } 1171 1172 #endif 1173 1174 #define PACK_HELPER_B(name, F) \ 1175 void glue(helper_pack ## name, SUFFIX)(CPUX86State *env, \ 1176 Reg *d, Reg *v, Reg *s) \ 1177 { \ 1178 uint8_t r[PACK_WIDTH * 2]; \ 1179 int j, k; \ 1180 for (j = 0; j < 4 << SHIFT; j += PACK_WIDTH) { \ 1181 for (k = 0; k < PACK_WIDTH; k++) { \ 1182 r[k] = F((int16_t)v->W(j + k)); \ 1183 } \ 1184 for (k = 0; k < PACK_WIDTH; k++) { \ 1185 r[PACK_WIDTH + k] = F((int16_t)s->W(j + k)); \ 1186 } \ 1187 for (k = 0; k < PACK_WIDTH * 2; k++) { \ 1188 d->B(2 * j + k) = r[k]; \ 1189 } \ 1190 } \ 1191 } 1192 1193 PACK_HELPER_B(sswb, satsb) 1194 PACK_HELPER_B(uswb, satub) 1195 1196 void glue(helper_packssdw, SUFFIX)(CPUX86State *env, Reg *d, Reg *v, Reg *s) 1197 { 1198 uint16_t r[PACK_WIDTH]; 1199 int j, k; 1200 1201 for (j = 0; j < 2 << SHIFT; j += PACK_WIDTH / 2) { 1202 for (k = 0; k < PACK_WIDTH / 2; k++) { 1203 r[k] = satsw(v->L(j + k)); 1204 } 1205 for (k = 0; k < PACK_WIDTH / 2; k++) { 1206 r[PACK_WIDTH / 2 + k] = satsw(s->L(j + k)); 1207 } 1208 for (k = 0; k < PACK_WIDTH; k++) { 1209 d->W(2 * j + k) = r[k]; 1210 } 1211 } 1212 } 1213 1214 #define UNPCK_OP(base_name, base) \ 1215 \ 1216 void glue(helper_punpck ## base_name ## bw, SUFFIX)(CPUX86State *env,\ 1217 Reg *d, Reg *v, Reg *s) \ 1218 { \ 1219 uint8_t r[PACK_WIDTH * 2]; \ 1220 int j, i; \ 1221 \ 1222 for (j = 0; j < 8 << SHIFT; ) { \ 1223 int k = j + base * PACK_WIDTH; \ 1224 for (i = 0; i < PACK_WIDTH; i++) { \ 1225 r[2 * i] = v->B(k + i); \ 1226 r[2 * i + 1] = s->B(k + i); \ 1227 } \ 1228 for (i = 0; i < PACK_WIDTH * 2; i++, j++) { \ 1229 d->B(j) = r[i]; \ 1230 } \ 1231 } \ 1232 } \ 1233 \ 1234 void glue(helper_punpck ## base_name ## wd, SUFFIX)(CPUX86State *env,\ 1235 Reg *d, Reg *v, Reg *s) \ 1236 { \ 1237 uint16_t r[PACK_WIDTH]; \ 1238 int j, i; \ 1239 \ 1240 for (j = 0; j < 4 << SHIFT; ) { \ 1241 int k = j + base * PACK_WIDTH / 2; \ 1242 for (i = 0; i < PACK_WIDTH / 2; i++) { \ 1243 r[2 * i] = v->W(k + i); \ 1244 r[2 * i + 1] = s->W(k + i); \ 1245 } \ 1246 for (i = 0; i < PACK_WIDTH; i++, j++) { \ 1247 d->W(j) = r[i]; \ 1248 } \ 1249 } \ 1250 } \ 1251 \ 1252 void glue(helper_punpck ## base_name ## dq, SUFFIX)(CPUX86State *env,\ 1253 Reg *d, Reg *v, Reg *s) \ 1254 { \ 1255 uint32_t r[PACK_WIDTH / 2]; \ 1256 int j, i; \ 1257 \ 1258 for (j = 0; j < 2 << SHIFT; ) { \ 1259 int k = j + base * PACK_WIDTH / 4; \ 1260 for (i = 0; i < PACK_WIDTH / 4; i++) { \ 1261 r[2 * i] = v->L(k + i); \ 1262 r[2 * i + 1] = s->L(k + i); \ 1263 } \ 1264 for (i = 0; i < PACK_WIDTH / 2; i++, j++) { \ 1265 d->L(j) = r[i]; \ 1266 } \ 1267 } \ 1268 } \ 1269 \ 1270 XMM_ONLY( \ 1271 void glue(helper_punpck ## base_name ## qdq, SUFFIX)( \ 1272 CPUX86State *env, Reg *d, Reg *v, Reg *s) \ 1273 { \ 1274 uint64_t r[2]; \ 1275 int i; \ 1276 \ 1277 for (i = 0; i < 1 << SHIFT; i += 2) { \ 1278 r[0] = v->Q(base + i); \ 1279 r[1] = s->Q(base + i); \ 1280 d->Q(i) = r[0]; \ 1281 d->Q(i + 1) = r[1]; \ 1282 } \ 1283 } \ 1284 ) 1285 1286 UNPCK_OP(l, 0) 1287 UNPCK_OP(h, 1) 1288 1289 #undef PACK_WIDTH 1290 #undef PACK_HELPER_B 1291 #undef UNPCK_OP 1292 1293 1294 /* 3DNow! float ops */ 1295 #if SHIFT == 0 1296 void helper_pi2fd(CPUX86State *env, MMXReg *d, MMXReg *s) 1297 { 1298 d->MMX_S(0) = int32_to_float32(s->MMX_L(0), &env->mmx_status); 1299 d->MMX_S(1) = int32_to_float32(s->MMX_L(1), &env->mmx_status); 1300 } 1301 1302 void helper_pi2fw(CPUX86State *env, MMXReg *d, MMXReg *s) 1303 { 1304 d->MMX_S(0) = int32_to_float32((int16_t)s->MMX_W(0), &env->mmx_status); 1305 d->MMX_S(1) = int32_to_float32((int16_t)s->MMX_W(2), &env->mmx_status); 1306 } 1307 1308 void helper_pf2id(CPUX86State *env, MMXReg *d, MMXReg *s) 1309 { 1310 d->MMX_L(0) = float32_to_int32_round_to_zero(s->MMX_S(0), &env->mmx_status); 1311 d->MMX_L(1) = float32_to_int32_round_to_zero(s->MMX_S(1), &env->mmx_status); 1312 } 1313 1314 void helper_pf2iw(CPUX86State *env, MMXReg *d, MMXReg *s) 1315 { 1316 d->MMX_L(0) = satsw(float32_to_int32_round_to_zero(s->MMX_S(0), 1317 &env->mmx_status)); 1318 d->MMX_L(1) = satsw(float32_to_int32_round_to_zero(s->MMX_S(1), 1319 &env->mmx_status)); 1320 } 1321 1322 void helper_pfacc(CPUX86State *env, MMXReg *d, MMXReg *s) 1323 { 1324 float32 r; 1325 1326 r = float32_add(d->MMX_S(0), d->MMX_S(1), &env->mmx_status); 1327 d->MMX_S(1) = float32_add(s->MMX_S(0), s->MMX_S(1), &env->mmx_status); 1328 d->MMX_S(0) = r; 1329 } 1330 1331 void helper_pfadd(CPUX86State *env, MMXReg *d, MMXReg *s) 1332 { 1333 d->MMX_S(0) = float32_add(d->MMX_S(0), s->MMX_S(0), &env->mmx_status); 1334 d->MMX_S(1) = float32_add(d->MMX_S(1), s->MMX_S(1), &env->mmx_status); 1335 } 1336 1337 void helper_pfcmpeq(CPUX86State *env, MMXReg *d, MMXReg *s) 1338 { 1339 d->MMX_L(0) = float32_eq_quiet(d->MMX_S(0), s->MMX_S(0), 1340 &env->mmx_status) ? -1 : 0; 1341 d->MMX_L(1) = float32_eq_quiet(d->MMX_S(1), s->MMX_S(1), 1342 &env->mmx_status) ? -1 : 0; 1343 } 1344 1345 void helper_pfcmpge(CPUX86State *env, MMXReg *d, MMXReg *s) 1346 { 1347 d->MMX_L(0) = float32_le(s->MMX_S(0), d->MMX_S(0), 1348 &env->mmx_status) ? -1 : 0; 1349 d->MMX_L(1) = float32_le(s->MMX_S(1), d->MMX_S(1), 1350 &env->mmx_status) ? -1 : 0; 1351 } 1352 1353 void helper_pfcmpgt(CPUX86State *env, MMXReg *d, MMXReg *s) 1354 { 1355 d->MMX_L(0) = float32_lt(s->MMX_S(0), d->MMX_S(0), 1356 &env->mmx_status) ? -1 : 0; 1357 d->MMX_L(1) = float32_lt(s->MMX_S(1), d->MMX_S(1), 1358 &env->mmx_status) ? -1 : 0; 1359 } 1360 1361 void helper_pfmax(CPUX86State *env, MMXReg *d, MMXReg *s) 1362 { 1363 if (float32_lt(d->MMX_S(0), s->MMX_S(0), &env->mmx_status)) { 1364 d->MMX_S(0) = s->MMX_S(0); 1365 } 1366 if (float32_lt(d->MMX_S(1), s->MMX_S(1), &env->mmx_status)) { 1367 d->MMX_S(1) = s->MMX_S(1); 1368 } 1369 } 1370 1371 void helper_pfmin(CPUX86State *env, MMXReg *d, MMXReg *s) 1372 { 1373 if (float32_lt(s->MMX_S(0), d->MMX_S(0), &env->mmx_status)) { 1374 d->MMX_S(0) = s->MMX_S(0); 1375 } 1376 if (float32_lt(s->MMX_S(1), d->MMX_S(1), &env->mmx_status)) { 1377 d->MMX_S(1) = s->MMX_S(1); 1378 } 1379 } 1380 1381 void helper_pfmul(CPUX86State *env, MMXReg *d, MMXReg *s) 1382 { 1383 d->MMX_S(0) = float32_mul(d->MMX_S(0), s->MMX_S(0), &env->mmx_status); 1384 d->MMX_S(1) = float32_mul(d->MMX_S(1), s->MMX_S(1), &env->mmx_status); 1385 } 1386 1387 void helper_pfnacc(CPUX86State *env, MMXReg *d, MMXReg *s) 1388 { 1389 float32 r; 1390 1391 r = float32_sub(d->MMX_S(0), d->MMX_S(1), &env->mmx_status); 1392 d->MMX_S(1) = float32_sub(s->MMX_S(0), s->MMX_S(1), &env->mmx_status); 1393 d->MMX_S(0) = r; 1394 } 1395 1396 void helper_pfpnacc(CPUX86State *env, MMXReg *d, MMXReg *s) 1397 { 1398 float32 r; 1399 1400 r = float32_sub(d->MMX_S(0), d->MMX_S(1), &env->mmx_status); 1401 d->MMX_S(1) = float32_add(s->MMX_S(0), s->MMX_S(1), &env->mmx_status); 1402 d->MMX_S(0) = r; 1403 } 1404 1405 void helper_pfrcp(CPUX86State *env, MMXReg *d, MMXReg *s) 1406 { 1407 d->MMX_S(0) = float32_div(float32_one, s->MMX_S(0), &env->mmx_status); 1408 d->MMX_S(1) = d->MMX_S(0); 1409 } 1410 1411 void helper_pfrsqrt(CPUX86State *env, MMXReg *d, MMXReg *s) 1412 { 1413 d->MMX_L(1) = s->MMX_L(0) & 0x7fffffff; 1414 d->MMX_S(1) = float32_div(float32_one, 1415 float32_sqrt(d->MMX_S(1), &env->mmx_status), 1416 &env->mmx_status); 1417 d->MMX_L(1) |= s->MMX_L(0) & 0x80000000; 1418 d->MMX_L(0) = d->MMX_L(1); 1419 } 1420 1421 void helper_pfsub(CPUX86State *env, MMXReg *d, MMXReg *s) 1422 { 1423 d->MMX_S(0) = float32_sub(d->MMX_S(0), s->MMX_S(0), &env->mmx_status); 1424 d->MMX_S(1) = float32_sub(d->MMX_S(1), s->MMX_S(1), &env->mmx_status); 1425 } 1426 1427 void helper_pfsubr(CPUX86State *env, MMXReg *d, MMXReg *s) 1428 { 1429 d->MMX_S(0) = float32_sub(s->MMX_S(0), d->MMX_S(0), &env->mmx_status); 1430 d->MMX_S(1) = float32_sub(s->MMX_S(1), d->MMX_S(1), &env->mmx_status); 1431 } 1432 1433 void helper_pswapd(CPUX86State *env, MMXReg *d, MMXReg *s) 1434 { 1435 uint32_t r; 1436 1437 r = s->MMX_L(0); 1438 d->MMX_L(0) = s->MMX_L(1); 1439 d->MMX_L(1) = r; 1440 } 1441 #endif 1442 1443 /* SSSE3 op helpers */ 1444 void glue(helper_pshufb, SUFFIX)(CPUX86State *env, Reg *d, Reg *v, Reg *s) 1445 { 1446 int i; 1447 #if SHIFT == 0 1448 uint8_t r[8]; 1449 1450 for (i = 0; i < 8; i++) { 1451 r[i] = (s->B(i) & 0x80) ? 0 : (v->B(s->B(i) & 7)); 1452 } 1453 for (i = 0; i < 8; i++) { 1454 d->B(i) = r[i]; 1455 } 1456 #else 1457 uint8_t r[8 << SHIFT]; 1458 1459 for (i = 0; i < 8 << SHIFT; i++) { 1460 int j = i & ~0xf; 1461 r[i] = (s->B(i) & 0x80) ? 0 : v->B(j | (s->B(i) & 0xf)); 1462 } 1463 for (i = 0; i < 8 << SHIFT; i++) { 1464 d->B(i) = r[i]; 1465 } 1466 #endif 1467 } 1468 1469 #define SSE_HELPER_HW(name, F) \ 1470 void glue(helper_ ## name, SUFFIX)(CPUX86State *env, Reg *d, Reg *v, Reg *s) \ 1471 { \ 1472 uint16_t r[4 << SHIFT]; \ 1473 int i, j, k; \ 1474 for (k = 0; k < 4 << SHIFT; k += LANE_WIDTH / 2) { \ 1475 for (i = j = 0; j < LANE_WIDTH / 2; i++, j += 2) { \ 1476 r[i + k] = F(v->W(j + k), v->W(j + k + 1)); \ 1477 } \ 1478 for (j = 0; j < LANE_WIDTH / 2; i++, j += 2) { \ 1479 r[i + k] = F(s->W(j + k), s->W(j + k + 1)); \ 1480 } \ 1481 } \ 1482 for (i = 0; i < 4 << SHIFT; i++) { \ 1483 d->W(i) = r[i]; \ 1484 } \ 1485 } 1486 1487 #define SSE_HELPER_HL(name, F) \ 1488 void glue(helper_ ## name, SUFFIX)(CPUX86State *env, Reg *d, Reg *v, Reg *s) \ 1489 { \ 1490 uint32_t r[2 << SHIFT]; \ 1491 int i, j, k; \ 1492 for (k = 0; k < 2 << SHIFT; k += LANE_WIDTH / 4) { \ 1493 for (i = j = 0; j < LANE_WIDTH / 4; i++, j += 2) { \ 1494 r[i + k] = F(v->L(j + k), v->L(j + k + 1)); \ 1495 } \ 1496 for (j = 0; j < LANE_WIDTH / 4; i++, j += 2) { \ 1497 r[i + k] = F(s->L(j + k), s->L(j + k + 1)); \ 1498 } \ 1499 } \ 1500 for (i = 0; i < 2 << SHIFT; i++) { \ 1501 d->L(i) = r[i]; \ 1502 } \ 1503 } 1504 1505 SSE_HELPER_HW(phaddw, FADD) 1506 SSE_HELPER_HW(phsubw, FSUB) 1507 SSE_HELPER_HW(phaddsw, FADDSW) 1508 SSE_HELPER_HW(phsubsw, FSUBSW) 1509 SSE_HELPER_HL(phaddd, FADD) 1510 SSE_HELPER_HL(phsubd, FSUB) 1511 1512 #undef SSE_HELPER_HW 1513 #undef SSE_HELPER_HL 1514 1515 void glue(helper_pmaddubsw, SUFFIX)(CPUX86State *env, Reg *d, Reg *v, Reg *s) 1516 { 1517 int i; 1518 for (i = 0; i < 4 << SHIFT; i++) { 1519 d->W(i) = satsw((int8_t)s->B(i * 2) * (uint8_t)v->B(i * 2) + 1520 (int8_t)s->B(i * 2 + 1) * (uint8_t)v->B(i * 2 + 1)); 1521 } 1522 } 1523 1524 #define FMULHRSW(d, s) (((int16_t) d * (int16_t)s + 0x4000) >> 15) 1525 SSE_HELPER_W(helper_pmulhrsw, FMULHRSW) 1526 1527 #define FSIGNB(d, s) (s <= INT8_MAX ? s ? d : 0 : -(int8_t)d) 1528 #define FSIGNW(d, s) (s <= INT16_MAX ? s ? d : 0 : -(int16_t)d) 1529 #define FSIGNL(d, s) (s <= INT32_MAX ? s ? d : 0 : -(int32_t)d) 1530 SSE_HELPER_B(helper_psignb, FSIGNB) 1531 SSE_HELPER_W(helper_psignw, FSIGNW) 1532 SSE_HELPER_L(helper_psignd, FSIGNL) 1533 1534 void glue(helper_palignr, SUFFIX)(CPUX86State *env, Reg *d, Reg *v, Reg *s, 1535 uint32_t imm) 1536 { 1537 int i; 1538 1539 /* XXX could be checked during translation */ 1540 if (imm >= (SHIFT ? 32 : 16)) { 1541 for (i = 0; i < (1 << SHIFT); i++) { 1542 d->Q(i) = 0; 1543 } 1544 } else { 1545 int shift = imm * 8; 1546 #define SHR(v, i) (i < 64 && i > -64 ? i > 0 ? v >> (i) : (v << -(i)) : 0) 1547 #if SHIFT == 0 1548 d->Q(0) = SHR(s->Q(0), shift - 0) | 1549 SHR(v->Q(0), shift - 64); 1550 #else 1551 for (i = 0; i < (1 << SHIFT); i += 2) { 1552 uint64_t r0, r1; 1553 1554 r0 = SHR(s->Q(i), shift - 0) | 1555 SHR(s->Q(i + 1), shift - 64) | 1556 SHR(v->Q(i), shift - 128) | 1557 SHR(v->Q(i + 1), shift - 192); 1558 r1 = SHR(s->Q(i), shift + 64) | 1559 SHR(s->Q(i + 1), shift - 0) | 1560 SHR(v->Q(i), shift - 64) | 1561 SHR(v->Q(i + 1), shift - 128); 1562 d->Q(i) = r0; 1563 d->Q(i + 1) = r1; 1564 } 1565 #endif 1566 #undef SHR 1567 } 1568 } 1569 1570 #if SHIFT >= 1 1571 1572 #define SSE_HELPER_V(name, elem, num, F) \ 1573 void glue(name, SUFFIX)(CPUX86State *env, Reg *d, Reg *v, Reg *s, \ 1574 Reg *m) \ 1575 { \ 1576 int i; \ 1577 for (i = 0; i < num; i++) { \ 1578 d->elem(i) = F(v->elem(i), s->elem(i), m->elem(i)); \ 1579 } \ 1580 } 1581 1582 #define SSE_HELPER_I(name, elem, num, F) \ 1583 void glue(name, SUFFIX)(CPUX86State *env, Reg *d, Reg *v, Reg *s, \ 1584 uint32_t imm) \ 1585 { \ 1586 int i; \ 1587 for (i = 0; i < num; i++) { \ 1588 int j = i & 7; \ 1589 d->elem(i) = F(v->elem(i), s->elem(i), (imm >> j) & 1); \ 1590 } \ 1591 } 1592 1593 /* SSE4.1 op helpers */ 1594 #define FBLENDVB(v, s, m) ((m & 0x80) ? s : v) 1595 #define FBLENDVPS(v, s, m) ((m & 0x80000000) ? s : v) 1596 #define FBLENDVPD(v, s, m) ((m & 0x8000000000000000LL) ? s : v) 1597 SSE_HELPER_V(helper_pblendvb, B, 8 << SHIFT, FBLENDVB) 1598 SSE_HELPER_V(helper_blendvps, L, 2 << SHIFT, FBLENDVPS) 1599 SSE_HELPER_V(helper_blendvpd, Q, 1 << SHIFT, FBLENDVPD) 1600 1601 void glue(helper_ptest, SUFFIX)(CPUX86State *env, Reg *d, Reg *s) 1602 { 1603 uint64_t zf = 0, cf = 0; 1604 int i; 1605 1606 for (i = 0; i < 1 << SHIFT; i++) { 1607 zf |= (s->Q(i) & d->Q(i)); 1608 cf |= (s->Q(i) & ~d->Q(i)); 1609 } 1610 CC_SRC = (zf ? 0 : CC_Z) | (cf ? 0 : CC_C); 1611 } 1612 1613 #define FMOVSLDUP(i) s->L((i) & ~1) 1614 #define FMOVSHDUP(i) s->L((i) | 1) 1615 #define FMOVDLDUP(i) s->Q((i) & ~1) 1616 1617 #define SSE_HELPER_F(name, elem, num, F) \ 1618 void glue(name, SUFFIX)(CPUX86State *env, Reg *d, Reg *s) \ 1619 { \ 1620 int n = num; \ 1621 for (int i = n; --i >= 0; ) { \ 1622 d->elem(i) = F(i); \ 1623 } \ 1624 } 1625 1626 #if SHIFT > 0 1627 SSE_HELPER_F(helper_pmovsxbw, W, 4 << SHIFT, (int8_t) s->B) 1628 SSE_HELPER_F(helper_pmovsxbd, L, 2 << SHIFT, (int8_t) s->B) 1629 SSE_HELPER_F(helper_pmovsxbq, Q, 1 << SHIFT, (int8_t) s->B) 1630 SSE_HELPER_F(helper_pmovsxwd, L, 2 << SHIFT, (int16_t) s->W) 1631 SSE_HELPER_F(helper_pmovsxwq, Q, 1 << SHIFT, (int16_t) s->W) 1632 SSE_HELPER_F(helper_pmovsxdq, Q, 1 << SHIFT, (int32_t) s->L) 1633 SSE_HELPER_F(helper_pmovzxbw, W, 4 << SHIFT, s->B) 1634 SSE_HELPER_F(helper_pmovzxbd, L, 2 << SHIFT, s->B) 1635 SSE_HELPER_F(helper_pmovzxbq, Q, 1 << SHIFT, s->B) 1636 SSE_HELPER_F(helper_pmovzxwd, L, 2 << SHIFT, s->W) 1637 SSE_HELPER_F(helper_pmovzxwq, Q, 1 << SHIFT, s->W) 1638 SSE_HELPER_F(helper_pmovzxdq, Q, 1 << SHIFT, s->L) 1639 SSE_HELPER_F(helper_pmovsldup, L, 2 << SHIFT, FMOVSLDUP) 1640 SSE_HELPER_F(helper_pmovshdup, L, 2 << SHIFT, FMOVSHDUP) 1641 SSE_HELPER_F(helper_pmovdldup, Q, 1 << SHIFT, FMOVDLDUP) 1642 #endif 1643 1644 void glue(helper_pmuldq, SUFFIX)(CPUX86State *env, Reg *d, Reg *v, Reg *s) 1645 { 1646 int i; 1647 1648 for (i = 0; i < 1 << SHIFT; i++) { 1649 d->Q(i) = (int64_t)(int32_t) v->L(2 * i) * (int32_t) s->L(2 * i); 1650 } 1651 } 1652 1653 void glue(helper_packusdw, SUFFIX)(CPUX86State *env, Reg *d, Reg *v, Reg *s) 1654 { 1655 uint16_t r[8]; 1656 int i, j, k; 1657 1658 for (i = 0, j = 0; i <= 2 << SHIFT; i += 8, j += 4) { 1659 r[0] = satuw(v->L(j)); 1660 r[1] = satuw(v->L(j + 1)); 1661 r[2] = satuw(v->L(j + 2)); 1662 r[3] = satuw(v->L(j + 3)); 1663 r[4] = satuw(s->L(j)); 1664 r[5] = satuw(s->L(j + 1)); 1665 r[6] = satuw(s->L(j + 2)); 1666 r[7] = satuw(s->L(j + 3)); 1667 for (k = 0; k < 8; k++) { 1668 d->W(i + k) = r[k]; 1669 } 1670 } 1671 } 1672 1673 #if SHIFT == 1 1674 void glue(helper_phminposuw, SUFFIX)(CPUX86State *env, Reg *d, Reg *s) 1675 { 1676 int idx = 0; 1677 1678 if (s->W(1) < s->W(idx)) { 1679 idx = 1; 1680 } 1681 if (s->W(2) < s->W(idx)) { 1682 idx = 2; 1683 } 1684 if (s->W(3) < s->W(idx)) { 1685 idx = 3; 1686 } 1687 if (s->W(4) < s->W(idx)) { 1688 idx = 4; 1689 } 1690 if (s->W(5) < s->W(idx)) { 1691 idx = 5; 1692 } 1693 if (s->W(6) < s->W(idx)) { 1694 idx = 6; 1695 } 1696 if (s->W(7) < s->W(idx)) { 1697 idx = 7; 1698 } 1699 1700 d->W(0) = s->W(idx); 1701 d->W(1) = idx; 1702 d->L(1) = 0; 1703 d->Q(1) = 0; 1704 } 1705 #endif 1706 1707 void glue(helper_roundps, SUFFIX)(CPUX86State *env, Reg *d, Reg *s, 1708 uint32_t mode) 1709 { 1710 uint8_t old_flags = get_float_exception_flags(&env->sse_status); 1711 signed char prev_rounding_mode; 1712 int i; 1713 1714 prev_rounding_mode = env->sse_status.float_rounding_mode; 1715 if (!(mode & (1 << 2))) { 1716 set_x86_rounding_mode(mode & 3, &env->sse_status); 1717 } 1718 1719 for (i = 0; i < 2 << SHIFT; i++) { 1720 d->ZMM_S(i) = float32_round_to_int(s->ZMM_S(i), &env->sse_status); 1721 } 1722 1723 if (mode & (1 << 3) && !(old_flags & float_flag_inexact)) { 1724 set_float_exception_flags(get_float_exception_flags(&env->sse_status) & 1725 ~float_flag_inexact, 1726 &env->sse_status); 1727 } 1728 env->sse_status.float_rounding_mode = prev_rounding_mode; 1729 } 1730 1731 void glue(helper_roundpd, SUFFIX)(CPUX86State *env, Reg *d, Reg *s, 1732 uint32_t mode) 1733 { 1734 uint8_t old_flags = get_float_exception_flags(&env->sse_status); 1735 signed char prev_rounding_mode; 1736 int i; 1737 1738 prev_rounding_mode = env->sse_status.float_rounding_mode; 1739 if (!(mode & (1 << 2))) { 1740 set_x86_rounding_mode(mode & 3, &env->sse_status); 1741 } 1742 1743 for (i = 0; i < 1 << SHIFT; i++) { 1744 d->ZMM_D(i) = float64_round_to_int(s->ZMM_D(i), &env->sse_status); 1745 } 1746 1747 if (mode & (1 << 3) && !(old_flags & float_flag_inexact)) { 1748 set_float_exception_flags(get_float_exception_flags(&env->sse_status) & 1749 ~float_flag_inexact, 1750 &env->sse_status); 1751 } 1752 env->sse_status.float_rounding_mode = prev_rounding_mode; 1753 } 1754 1755 #if SHIFT == 1 1756 void glue(helper_roundss, SUFFIX)(CPUX86State *env, Reg *d, Reg *v, Reg *s, 1757 uint32_t mode) 1758 { 1759 uint8_t old_flags = get_float_exception_flags(&env->sse_status); 1760 signed char prev_rounding_mode; 1761 int i; 1762 1763 prev_rounding_mode = env->sse_status.float_rounding_mode; 1764 if (!(mode & (1 << 2))) { 1765 set_x86_rounding_mode(mode & 3, &env->sse_status); 1766 } 1767 1768 d->ZMM_S(0) = float32_round_to_int(s->ZMM_S(0), &env->sse_status); 1769 for (i = 1; i < 2 << SHIFT; i++) { 1770 d->ZMM_L(i) = v->ZMM_L(i); 1771 } 1772 1773 if (mode & (1 << 3) && !(old_flags & float_flag_inexact)) { 1774 set_float_exception_flags(get_float_exception_flags(&env->sse_status) & 1775 ~float_flag_inexact, 1776 &env->sse_status); 1777 } 1778 env->sse_status.float_rounding_mode = prev_rounding_mode; 1779 } 1780 1781 void glue(helper_roundsd, SUFFIX)(CPUX86State *env, Reg *d, Reg *v, Reg *s, 1782 uint32_t mode) 1783 { 1784 uint8_t old_flags = get_float_exception_flags(&env->sse_status); 1785 signed char prev_rounding_mode; 1786 int i; 1787 1788 prev_rounding_mode = env->sse_status.float_rounding_mode; 1789 if (!(mode & (1 << 2))) { 1790 set_x86_rounding_mode(mode & 3, &env->sse_status); 1791 } 1792 1793 d->ZMM_D(0) = float64_round_to_int(s->ZMM_D(0), &env->sse_status); 1794 for (i = 1; i < 1 << SHIFT; i++) { 1795 d->ZMM_Q(i) = v->ZMM_Q(i); 1796 } 1797 1798 if (mode & (1 << 3) && !(old_flags & float_flag_inexact)) { 1799 set_float_exception_flags(get_float_exception_flags(&env->sse_status) & 1800 ~float_flag_inexact, 1801 &env->sse_status); 1802 } 1803 env->sse_status.float_rounding_mode = prev_rounding_mode; 1804 } 1805 #endif 1806 1807 #define FBLENDP(v, s, m) (m ? s : v) 1808 SSE_HELPER_I(helper_blendps, L, 2 << SHIFT, FBLENDP) 1809 SSE_HELPER_I(helper_blendpd, Q, 1 << SHIFT, FBLENDP) 1810 SSE_HELPER_I(helper_pblendw, W, 4 << SHIFT, FBLENDP) 1811 1812 void glue(helper_dpps, SUFFIX)(CPUX86State *env, Reg *d, Reg *v, Reg *s, 1813 uint32_t mask) 1814 { 1815 float32 prod1, prod2, temp2, temp3, temp4; 1816 int i; 1817 1818 for (i = 0; i < 2 << SHIFT; i += 4) { 1819 /* 1820 * We must evaluate (A+B)+(C+D), not ((A+B)+C)+D 1821 * to correctly round the intermediate results 1822 */ 1823 if (mask & (1 << 4)) { 1824 prod1 = float32_mul(v->ZMM_S(i), s->ZMM_S(i), &env->sse_status); 1825 } else { 1826 prod1 = float32_zero; 1827 } 1828 if (mask & (1 << 5)) { 1829 prod2 = float32_mul(v->ZMM_S(i+1), s->ZMM_S(i+1), &env->sse_status); 1830 } else { 1831 prod2 = float32_zero; 1832 } 1833 temp2 = float32_add(prod1, prod2, &env->sse_status); 1834 if (mask & (1 << 6)) { 1835 prod1 = float32_mul(v->ZMM_S(i+2), s->ZMM_S(i+2), &env->sse_status); 1836 } else { 1837 prod1 = float32_zero; 1838 } 1839 if (mask & (1 << 7)) { 1840 prod2 = float32_mul(v->ZMM_S(i+3), s->ZMM_S(i+3), &env->sse_status); 1841 } else { 1842 prod2 = float32_zero; 1843 } 1844 temp3 = float32_add(prod1, prod2, &env->sse_status); 1845 temp4 = float32_add(temp2, temp3, &env->sse_status); 1846 1847 d->ZMM_S(i) = (mask & (1 << 0)) ? temp4 : float32_zero; 1848 d->ZMM_S(i+1) = (mask & (1 << 1)) ? temp4 : float32_zero; 1849 d->ZMM_S(i+2) = (mask & (1 << 2)) ? temp4 : float32_zero; 1850 d->ZMM_S(i+3) = (mask & (1 << 3)) ? temp4 : float32_zero; 1851 } 1852 } 1853 1854 #if SHIFT == 1 1855 /* Oddly, there is no ymm version of dppd */ 1856 void glue(helper_dppd, SUFFIX)(CPUX86State *env, 1857 Reg *d, Reg *v, Reg *s, uint32_t mask) 1858 { 1859 float64 prod1, prod2, temp2; 1860 1861 if (mask & (1 << 4)) { 1862 prod1 = float64_mul(v->ZMM_D(0), s->ZMM_D(0), &env->sse_status); 1863 } else { 1864 prod1 = float64_zero; 1865 } 1866 if (mask & (1 << 5)) { 1867 prod2 = float64_mul(v->ZMM_D(1), s->ZMM_D(1), &env->sse_status); 1868 } else { 1869 prod2 = float64_zero; 1870 } 1871 temp2 = float64_add(prod1, prod2, &env->sse_status); 1872 d->ZMM_D(0) = (mask & (1 << 0)) ? temp2 : float64_zero; 1873 d->ZMM_D(1) = (mask & (1 << 1)) ? temp2 : float64_zero; 1874 } 1875 #endif 1876 1877 void glue(helper_mpsadbw, SUFFIX)(CPUX86State *env, Reg *d, Reg *v, Reg *s, 1878 uint32_t offset) 1879 { 1880 int i, j; 1881 uint16_t r[8]; 1882 1883 for (j = 0; j < 4 << SHIFT; ) { 1884 int s0 = (j * 2) + ((offset & 3) << 2); 1885 int d0 = (j * 2) + ((offset & 4) << 0); 1886 for (i = 0; i < LANE_WIDTH / 2; i++, d0++) { 1887 r[i] = 0; 1888 r[i] += abs1(v->B(d0 + 0) - s->B(s0 + 0)); 1889 r[i] += abs1(v->B(d0 + 1) - s->B(s0 + 1)); 1890 r[i] += abs1(v->B(d0 + 2) - s->B(s0 + 2)); 1891 r[i] += abs1(v->B(d0 + 3) - s->B(s0 + 3)); 1892 } 1893 for (i = 0; i < LANE_WIDTH / 2; i++, j++) { 1894 d->W(j) = r[i]; 1895 } 1896 offset >>= 3; 1897 } 1898 } 1899 1900 /* SSE4.2 op helpers */ 1901 #if SHIFT == 1 1902 static inline int pcmp_elen(CPUX86State *env, int reg, uint32_t ctrl) 1903 { 1904 target_long val, limit; 1905 1906 /* Presence of REX.W is indicated by a bit higher than 7 set */ 1907 if (ctrl >> 8) { 1908 val = (target_long)env->regs[reg]; 1909 } else { 1910 val = (int32_t)env->regs[reg]; 1911 } 1912 if (ctrl & 1) { 1913 limit = 8; 1914 } else { 1915 limit = 16; 1916 } 1917 if ((val > limit) || (val < -limit)) { 1918 return limit; 1919 } 1920 return abs1(val); 1921 } 1922 1923 static inline int pcmp_ilen(Reg *r, uint8_t ctrl) 1924 { 1925 int val = 0; 1926 1927 if (ctrl & 1) { 1928 while (val < 8 && r->W(val)) { 1929 val++; 1930 } 1931 } else { 1932 while (val < 16 && r->B(val)) { 1933 val++; 1934 } 1935 } 1936 1937 return val; 1938 } 1939 1940 static inline int pcmp_val(Reg *r, uint8_t ctrl, int i) 1941 { 1942 switch ((ctrl >> 0) & 3) { 1943 case 0: 1944 return r->B(i); 1945 case 1: 1946 return r->W(i); 1947 case 2: 1948 return (int8_t)r->B(i); 1949 case 3: 1950 default: 1951 return (int16_t)r->W(i); 1952 } 1953 } 1954 1955 static inline unsigned pcmpxstrx(CPUX86State *env, Reg *d, Reg *s, 1956 uint8_t ctrl, int valids, int validd) 1957 { 1958 unsigned int res = 0; 1959 int v; 1960 int j, i; 1961 int upper = (ctrl & 1) ? 7 : 15; 1962 1963 valids--; 1964 validd--; 1965 1966 CC_SRC = (valids < upper ? CC_Z : 0) | (validd < upper ? CC_S : 0); 1967 1968 switch ((ctrl >> 2) & 3) { 1969 case 0: 1970 for (j = valids; j >= 0; j--) { 1971 res <<= 1; 1972 v = pcmp_val(s, ctrl, j); 1973 for (i = validd; i >= 0; i--) { 1974 res |= (v == pcmp_val(d, ctrl, i)); 1975 } 1976 } 1977 break; 1978 case 1: 1979 for (j = valids; j >= 0; j--) { 1980 res <<= 1; 1981 v = pcmp_val(s, ctrl, j); 1982 for (i = ((validd - 1) | 1); i >= 0; i -= 2) { 1983 res |= (pcmp_val(d, ctrl, i - 0) >= v && 1984 pcmp_val(d, ctrl, i - 1) <= v); 1985 } 1986 } 1987 break; 1988 case 2: 1989 res = (1 << (upper - MAX(valids, validd))) - 1; 1990 res <<= MAX(valids, validd) - MIN(valids, validd); 1991 for (i = MIN(valids, validd); i >= 0; i--) { 1992 res <<= 1; 1993 v = pcmp_val(s, ctrl, i); 1994 res |= (v == pcmp_val(d, ctrl, i)); 1995 } 1996 break; 1997 case 3: 1998 if (validd == -1) { 1999 res = (2 << upper) - 1; 2000 break; 2001 } 2002 for (j = valids == upper ? valids : valids - validd; j >= 0; j--) { 2003 res <<= 1; 2004 v = 1; 2005 for (i = MIN(valids - j, validd); i >= 0; i--) { 2006 v &= (pcmp_val(s, ctrl, i + j) == pcmp_val(d, ctrl, i)); 2007 } 2008 res |= v; 2009 } 2010 break; 2011 } 2012 2013 switch ((ctrl >> 4) & 3) { 2014 case 1: 2015 res ^= (2 << upper) - 1; 2016 break; 2017 case 3: 2018 res ^= (1 << (valids + 1)) - 1; 2019 break; 2020 } 2021 2022 if (res) { 2023 CC_SRC |= CC_C; 2024 } 2025 if (res & 1) { 2026 CC_SRC |= CC_O; 2027 } 2028 2029 return res; 2030 } 2031 2032 void glue(helper_pcmpestri, SUFFIX)(CPUX86State *env, Reg *d, Reg *s, 2033 uint32_t ctrl) 2034 { 2035 unsigned int res = pcmpxstrx(env, d, s, ctrl, 2036 pcmp_elen(env, R_EDX, ctrl), 2037 pcmp_elen(env, R_EAX, ctrl)); 2038 2039 if (res) { 2040 env->regs[R_ECX] = (ctrl & (1 << 6)) ? 31 - clz32(res) : ctz32(res); 2041 } else { 2042 env->regs[R_ECX] = 16 >> (ctrl & (1 << 0)); 2043 } 2044 } 2045 2046 void glue(helper_pcmpestrm, SUFFIX)(CPUX86State *env, Reg *d, Reg *s, 2047 uint32_t ctrl) 2048 { 2049 int i; 2050 unsigned int res = pcmpxstrx(env, d, s, ctrl, 2051 pcmp_elen(env, R_EDX, ctrl), 2052 pcmp_elen(env, R_EAX, ctrl)); 2053 2054 if ((ctrl >> 6) & 1) { 2055 if (ctrl & 1) { 2056 for (i = 0; i < 8; i++, res >>= 1) { 2057 env->xmm_regs[0].W(i) = (res & 1) ? ~0 : 0; 2058 } 2059 } else { 2060 for (i = 0; i < 16; i++, res >>= 1) { 2061 env->xmm_regs[0].B(i) = (res & 1) ? ~0 : 0; 2062 } 2063 } 2064 } else { 2065 env->xmm_regs[0].Q(1) = 0; 2066 env->xmm_regs[0].Q(0) = res; 2067 } 2068 } 2069 2070 void glue(helper_pcmpistri, SUFFIX)(CPUX86State *env, Reg *d, Reg *s, 2071 uint32_t ctrl) 2072 { 2073 unsigned int res = pcmpxstrx(env, d, s, ctrl, 2074 pcmp_ilen(s, ctrl), 2075 pcmp_ilen(d, ctrl)); 2076 2077 if (res) { 2078 env->regs[R_ECX] = (ctrl & (1 << 6)) ? 31 - clz32(res) : ctz32(res); 2079 } else { 2080 env->regs[R_ECX] = 16 >> (ctrl & (1 << 0)); 2081 } 2082 } 2083 2084 void glue(helper_pcmpistrm, SUFFIX)(CPUX86State *env, Reg *d, Reg *s, 2085 uint32_t ctrl) 2086 { 2087 int i; 2088 unsigned int res = pcmpxstrx(env, d, s, ctrl, 2089 pcmp_ilen(s, ctrl), 2090 pcmp_ilen(d, ctrl)); 2091 2092 if ((ctrl >> 6) & 1) { 2093 if (ctrl & 1) { 2094 for (i = 0; i < 8; i++, res >>= 1) { 2095 env->xmm_regs[0].W(i) = (res & 1) ? ~0 : 0; 2096 } 2097 } else { 2098 for (i = 0; i < 16; i++, res >>= 1) { 2099 env->xmm_regs[0].B(i) = (res & 1) ? ~0 : 0; 2100 } 2101 } 2102 } else { 2103 env->xmm_regs[0].Q(1) = 0; 2104 env->xmm_regs[0].Q(0) = res; 2105 } 2106 } 2107 2108 #define CRCPOLY 0x1edc6f41 2109 #define CRCPOLY_BITREV 0x82f63b78 2110 target_ulong helper_crc32(uint32_t crc1, target_ulong msg, uint32_t len) 2111 { 2112 target_ulong crc = (msg & ((target_ulong) -1 >> 2113 (TARGET_LONG_BITS - len))) ^ crc1; 2114 2115 while (len--) { 2116 crc = (crc >> 1) ^ ((crc & 1) ? CRCPOLY_BITREV : 0); 2117 } 2118 2119 return crc; 2120 } 2121 2122 #endif 2123 2124 #if SHIFT == 1 2125 static void clmulq(uint64_t *dest_l, uint64_t *dest_h, 2126 uint64_t a, uint64_t b) 2127 { 2128 uint64_t al, ah, resh, resl; 2129 2130 ah = 0; 2131 al = a; 2132 resh = resl = 0; 2133 2134 while (b) { 2135 if (b & 1) { 2136 resl ^= al; 2137 resh ^= ah; 2138 } 2139 ah = (ah << 1) | (al >> 63); 2140 al <<= 1; 2141 b >>= 1; 2142 } 2143 2144 *dest_l = resl; 2145 *dest_h = resh; 2146 } 2147 #endif 2148 2149 void glue(helper_pclmulqdq, SUFFIX)(CPUX86State *env, Reg *d, Reg *v, Reg *s, 2150 uint32_t ctrl) 2151 { 2152 uint64_t a, b; 2153 int i; 2154 2155 for (i = 0; i < 1 << SHIFT; i += 2) { 2156 a = v->Q(((ctrl & 1) != 0) + i); 2157 b = s->Q(((ctrl & 16) != 0) + i); 2158 clmulq(&d->Q(i), &d->Q(i + 1), a, b); 2159 } 2160 } 2161 2162 void glue(helper_aesdec, SUFFIX)(CPUX86State *env, Reg *d, Reg *v, Reg *s) 2163 { 2164 int i; 2165 Reg st = *v; 2166 Reg rk = *s; 2167 2168 for (i = 0 ; i < 2 << SHIFT ; i++) { 2169 int j = i & 3; 2170 d->L(i) = rk.L(i) ^ bswap32(AES_Td0[st.B(AES_ishifts[4 * j + 0])] ^ 2171 AES_Td1[st.B(AES_ishifts[4 * j + 1])] ^ 2172 AES_Td2[st.B(AES_ishifts[4 * j + 2])] ^ 2173 AES_Td3[st.B(AES_ishifts[4 * j + 3])]); 2174 } 2175 } 2176 2177 void glue(helper_aesdeclast, SUFFIX)(CPUX86State *env, Reg *d, Reg *v, Reg *s) 2178 { 2179 int i; 2180 Reg st = *v; 2181 Reg rk = *s; 2182 2183 for (i = 0; i < 8 << SHIFT; i++) { 2184 d->B(i) = rk.B(i) ^ (AES_isbox[st.B(AES_ishifts[i & 15] + (i & ~15))]); 2185 } 2186 } 2187 2188 void glue(helper_aesenc, SUFFIX)(CPUX86State *env, Reg *d, Reg *v, Reg *s) 2189 { 2190 int i; 2191 Reg st = *v; 2192 Reg rk = *s; 2193 2194 for (i = 0 ; i < 2 << SHIFT ; i++) { 2195 int j = i & 3; 2196 d->L(i) = rk.L(i) ^ bswap32(AES_Te0[st.B(AES_shifts[4 * j + 0])] ^ 2197 AES_Te1[st.B(AES_shifts[4 * j + 1])] ^ 2198 AES_Te2[st.B(AES_shifts[4 * j + 2])] ^ 2199 AES_Te3[st.B(AES_shifts[4 * j + 3])]); 2200 } 2201 } 2202 2203 void glue(helper_aesenclast, SUFFIX)(CPUX86State *env, Reg *d, Reg *v, Reg *s) 2204 { 2205 int i; 2206 Reg st = *v; 2207 Reg rk = *s; 2208 2209 for (i = 0; i < 8 << SHIFT; i++) { 2210 d->B(i) = rk.B(i) ^ (AES_sbox[st.B(AES_shifts[i & 15] + (i & ~15))]); 2211 } 2212 } 2213 2214 #if SHIFT == 1 2215 void glue(helper_aesimc, SUFFIX)(CPUX86State *env, Reg *d, Reg *s) 2216 { 2217 int i; 2218 Reg tmp = *s; 2219 2220 for (i = 0 ; i < 4 ; i++) { 2221 d->L(i) = bswap32(AES_imc[tmp.B(4 * i + 0)][0] ^ 2222 AES_imc[tmp.B(4 * i + 1)][1] ^ 2223 AES_imc[tmp.B(4 * i + 2)][2] ^ 2224 AES_imc[tmp.B(4 * i + 3)][3]); 2225 } 2226 } 2227 2228 void glue(helper_aeskeygenassist, SUFFIX)(CPUX86State *env, Reg *d, Reg *s, 2229 uint32_t ctrl) 2230 { 2231 int i; 2232 Reg tmp = *s; 2233 2234 for (i = 0 ; i < 4 ; i++) { 2235 d->B(i) = AES_sbox[tmp.B(i + 4)]; 2236 d->B(i + 8) = AES_sbox[tmp.B(i + 12)]; 2237 } 2238 d->L(1) = (d->L(0) << 24 | d->L(0) >> 8) ^ ctrl; 2239 d->L(3) = (d->L(2) << 24 | d->L(2) >> 8) ^ ctrl; 2240 } 2241 #endif 2242 #endif 2243 2244 #if SHIFT >= 1 2245 void glue(helper_vpermilpd, SUFFIX)(CPUX86State *env, Reg *d, Reg *v, Reg *s) 2246 { 2247 uint64_t r0, r1; 2248 int i; 2249 2250 for (i = 0; i < 1 << SHIFT; i += 2) { 2251 r0 = v->Q(i + ((s->Q(i) >> 1) & 1)); 2252 r1 = v->Q(i + ((s->Q(i+1) >> 1) & 1)); 2253 d->Q(i) = r0; 2254 d->Q(i+1) = r1; 2255 } 2256 } 2257 2258 void glue(helper_vpermilps, SUFFIX)(CPUX86State *env, Reg *d, Reg *v, Reg *s) 2259 { 2260 uint32_t r0, r1, r2, r3; 2261 int i; 2262 2263 for (i = 0; i < 2 << SHIFT; i += 4) { 2264 r0 = v->L(i + (s->L(i) & 3)); 2265 r1 = v->L(i + (s->L(i+1) & 3)); 2266 r2 = v->L(i + (s->L(i+2) & 3)); 2267 r3 = v->L(i + (s->L(i+3) & 3)); 2268 d->L(i) = r0; 2269 d->L(i+1) = r1; 2270 d->L(i+2) = r2; 2271 d->L(i+3) = r3; 2272 } 2273 } 2274 2275 void glue(helper_vpermilpd_imm, SUFFIX)(Reg *d, Reg *s, uint32_t order) 2276 { 2277 uint64_t r0, r1; 2278 int i; 2279 2280 for (i = 0; i < 1 << SHIFT; i += 2) { 2281 r0 = s->Q(i + ((order >> 0) & 1)); 2282 r1 = s->Q(i + ((order >> 1) & 1)); 2283 d->Q(i) = r0; 2284 d->Q(i+1) = r1; 2285 2286 order >>= 2; 2287 } 2288 } 2289 2290 void glue(helper_vpermilps_imm, SUFFIX)(Reg *d, Reg *s, uint32_t order) 2291 { 2292 uint32_t r0, r1, r2, r3; 2293 int i; 2294 2295 for (i = 0; i < 2 << SHIFT; i += 4) { 2296 r0 = s->L(i + ((order >> 0) & 3)); 2297 r1 = s->L(i + ((order >> 2) & 3)); 2298 r2 = s->L(i + ((order >> 4) & 3)); 2299 r3 = s->L(i + ((order >> 6) & 3)); 2300 d->L(i) = r0; 2301 d->L(i+1) = r1; 2302 d->L(i+2) = r2; 2303 d->L(i+3) = r3; 2304 } 2305 } 2306 2307 #if SHIFT == 1 2308 #define FPSRLVD(x, c) (c < 32 ? ((x) >> c) : 0) 2309 #define FPSRLVQ(x, c) (c < 64 ? ((x) >> c) : 0) 2310 #define FPSRAVD(x, c) ((int32_t)(x) >> (c < 32 ? c : 31)) 2311 #define FPSRAVQ(x, c) ((int64_t)(x) >> (c < 64 ? c : 63)) 2312 #define FPSLLVD(x, c) (c < 32 ? ((x) << c) : 0) 2313 #define FPSLLVQ(x, c) (c < 64 ? ((x) << c) : 0) 2314 #endif 2315 2316 SSE_HELPER_L(helper_vpsrlvd, FPSRLVD) 2317 SSE_HELPER_L(helper_vpsravd, FPSRAVD) 2318 SSE_HELPER_L(helper_vpsllvd, FPSLLVD) 2319 2320 SSE_HELPER_Q(helper_vpsrlvq, FPSRLVQ) 2321 SSE_HELPER_Q(helper_vpsravq, FPSRAVQ) 2322 SSE_HELPER_Q(helper_vpsllvq, FPSLLVQ) 2323 2324 void glue(helper_vtestps, SUFFIX)(CPUX86State *env, Reg *d, Reg *s) 2325 { 2326 uint32_t zf = 0, cf = 0; 2327 int i; 2328 2329 for (i = 0; i < 2 << SHIFT; i++) { 2330 zf |= (s->L(i) & d->L(i)); 2331 cf |= (s->L(i) & ~d->L(i)); 2332 } 2333 CC_SRC = ((zf >> 31) ? 0 : CC_Z) | ((cf >> 31) ? 0 : CC_C); 2334 } 2335 2336 void glue(helper_vtestpd, SUFFIX)(CPUX86State *env, Reg *d, Reg *s) 2337 { 2338 uint64_t zf = 0, cf = 0; 2339 int i; 2340 2341 for (i = 0; i < 1 << SHIFT; i++) { 2342 zf |= (s->Q(i) & d->Q(i)); 2343 cf |= (s->Q(i) & ~d->Q(i)); 2344 } 2345 CC_SRC = ((zf >> 63) ? 0 : CC_Z) | ((cf >> 63) ? 0 : CC_C); 2346 } 2347 2348 void glue(helper_vpmaskmovd_st, SUFFIX)(CPUX86State *env, 2349 Reg *v, Reg *s, target_ulong a0) 2350 { 2351 int i; 2352 2353 for (i = 0; i < (2 << SHIFT); i++) { 2354 if (v->L(i) >> 31) { 2355 cpu_stl_data_ra(env, a0 + i * 4, s->L(i), GETPC()); 2356 } 2357 } 2358 } 2359 2360 void glue(helper_vpmaskmovq_st, SUFFIX)(CPUX86State *env, 2361 Reg *v, Reg *s, target_ulong a0) 2362 { 2363 int i; 2364 2365 for (i = 0; i < (1 << SHIFT); i++) { 2366 if (v->Q(i) >> 63) { 2367 cpu_stq_data_ra(env, a0 + i * 8, s->Q(i), GETPC()); 2368 } 2369 } 2370 } 2371 2372 void glue(helper_vpmaskmovd, SUFFIX)(CPUX86State *env, Reg *d, Reg *v, Reg *s) 2373 { 2374 int i; 2375 2376 for (i = 0; i < (2 << SHIFT); i++) { 2377 d->L(i) = (v->L(i) >> 31) ? s->L(i) : 0; 2378 } 2379 } 2380 2381 void glue(helper_vpmaskmovq, SUFFIX)(CPUX86State *env, Reg *d, Reg *v, Reg *s) 2382 { 2383 int i; 2384 2385 for (i = 0; i < (1 << SHIFT); i++) { 2386 d->Q(i) = (v->Q(i) >> 63) ? s->Q(i) : 0; 2387 } 2388 } 2389 2390 void glue(helper_vpgatherdd, SUFFIX)(CPUX86State *env, 2391 Reg *d, Reg *v, Reg *s, target_ulong a0, unsigned scale) 2392 { 2393 int i; 2394 for (i = 0; i < (2 << SHIFT); i++) { 2395 if (v->L(i) >> 31) { 2396 target_ulong addr = a0 2397 + ((target_ulong)(int32_t)s->L(i) << scale); 2398 d->L(i) = cpu_ldl_data_ra(env, addr, GETPC()); 2399 } 2400 v->L(i) = 0; 2401 } 2402 } 2403 2404 void glue(helper_vpgatherdq, SUFFIX)(CPUX86State *env, 2405 Reg *d, Reg *v, Reg *s, target_ulong a0, unsigned scale) 2406 { 2407 int i; 2408 for (i = 0; i < (1 << SHIFT); i++) { 2409 if (v->Q(i) >> 63) { 2410 target_ulong addr = a0 2411 + ((target_ulong)(int32_t)s->L(i) << scale); 2412 d->Q(i) = cpu_ldq_data_ra(env, addr, GETPC()); 2413 } 2414 v->Q(i) = 0; 2415 } 2416 } 2417 2418 void glue(helper_vpgatherqd, SUFFIX)(CPUX86State *env, 2419 Reg *d, Reg *v, Reg *s, target_ulong a0, unsigned scale) 2420 { 2421 int i; 2422 for (i = 0; i < (1 << SHIFT); i++) { 2423 if (v->L(i) >> 31) { 2424 target_ulong addr = a0 2425 + ((target_ulong)(int64_t)s->Q(i) << scale); 2426 d->L(i) = cpu_ldl_data_ra(env, addr, GETPC()); 2427 } 2428 v->L(i) = 0; 2429 } 2430 for (i /= 2; i < 1 << SHIFT; i++) { 2431 d->Q(i) = 0; 2432 v->Q(i) = 0; 2433 } 2434 } 2435 2436 void glue(helper_vpgatherqq, SUFFIX)(CPUX86State *env, 2437 Reg *d, Reg *v, Reg *s, target_ulong a0, unsigned scale) 2438 { 2439 int i; 2440 for (i = 0; i < (1 << SHIFT); i++) { 2441 if (v->Q(i) >> 63) { 2442 target_ulong addr = a0 2443 + ((target_ulong)(int64_t)s->Q(i) << scale); 2444 d->Q(i) = cpu_ldq_data_ra(env, addr, GETPC()); 2445 } 2446 v->Q(i) = 0; 2447 } 2448 } 2449 #endif 2450 2451 #if SHIFT >= 2 2452 void helper_vpermdq_ymm(Reg *d, Reg *v, Reg *s, uint32_t order) 2453 { 2454 uint64_t r0, r1, r2, r3; 2455 2456 switch (order & 3) { 2457 case 0: 2458 r0 = v->Q(0); 2459 r1 = v->Q(1); 2460 break; 2461 case 1: 2462 r0 = v->Q(2); 2463 r1 = v->Q(3); 2464 break; 2465 case 2: 2466 r0 = s->Q(0); 2467 r1 = s->Q(1); 2468 break; 2469 case 3: 2470 r0 = s->Q(2); 2471 r1 = s->Q(3); 2472 break; 2473 default: /* default case added to help the compiler to avoid warnings */ 2474 g_assert_not_reached(); 2475 } 2476 switch ((order >> 4) & 3) { 2477 case 0: 2478 r2 = v->Q(0); 2479 r3 = v->Q(1); 2480 break; 2481 case 1: 2482 r2 = v->Q(2); 2483 r3 = v->Q(3); 2484 break; 2485 case 2: 2486 r2 = s->Q(0); 2487 r3 = s->Q(1); 2488 break; 2489 case 3: 2490 r2 = s->Q(2); 2491 r3 = s->Q(3); 2492 break; 2493 default: /* default case added to help the compiler to avoid warnings */ 2494 g_assert_not_reached(); 2495 } 2496 d->Q(0) = r0; 2497 d->Q(1) = r1; 2498 d->Q(2) = r2; 2499 d->Q(3) = r3; 2500 } 2501 2502 void helper_vpermq_ymm(Reg *d, Reg *s, uint32_t order) 2503 { 2504 uint64_t r0, r1, r2, r3; 2505 r0 = s->Q(order & 3); 2506 r1 = s->Q((order >> 2) & 3); 2507 r2 = s->Q((order >> 4) & 3); 2508 r3 = s->Q((order >> 6) & 3); 2509 d->Q(0) = r0; 2510 d->Q(1) = r1; 2511 d->Q(2) = r2; 2512 d->Q(3) = r3; 2513 } 2514 2515 void helper_vpermd_ymm(Reg *d, Reg *v, Reg *s) 2516 { 2517 uint32_t r[8]; 2518 int i; 2519 2520 for (i = 0; i < 8; i++) { 2521 r[i] = s->L(v->L(i) & 7); 2522 } 2523 for (i = 0; i < 8; i++) { 2524 d->L(i) = r[i]; 2525 } 2526 } 2527 #endif 2528 2529 /* FMA3 op helpers */ 2530 #if SHIFT == 1 2531 #define SSE_HELPER_FMAS(name, elem, F) \ 2532 void name(CPUX86State *env, Reg *d, Reg *a, Reg *b, Reg *c, int flags) \ 2533 { \ 2534 d->elem(0) = F(a->elem(0), b->elem(0), c->elem(0), flags, &env->sse_status); \ 2535 } 2536 #define SSE_HELPER_FMAP(name, elem, num, F) \ 2537 void glue(name, SUFFIX)(CPUX86State *env, Reg *d, Reg *a, Reg *b, Reg *c, \ 2538 int flags, int flip) \ 2539 { \ 2540 int i; \ 2541 for (i = 0; i < num; i++) { \ 2542 d->elem(i) = F(a->elem(i), b->elem(i), c->elem(i), flags, &env->sse_status); \ 2543 flags ^= flip; \ 2544 } \ 2545 } 2546 2547 SSE_HELPER_FMAS(helper_fma4ss, ZMM_S, float32_muladd) 2548 SSE_HELPER_FMAS(helper_fma4sd, ZMM_D, float64_muladd) 2549 #endif 2550 2551 #if SHIFT >= 1 2552 SSE_HELPER_FMAP(helper_fma4ps, ZMM_S, 2 << SHIFT, float32_muladd) 2553 SSE_HELPER_FMAP(helper_fma4pd, ZMM_D, 1 << SHIFT, float64_muladd) 2554 #endif 2555 2556 #undef SSE_HELPER_S 2557 2558 #undef LANE_WIDTH 2559 #undef SHIFT 2560 #undef XMM_ONLY 2561 #undef Reg 2562 #undef B 2563 #undef W 2564 #undef L 2565 #undef Q 2566 #undef SUFFIX 2567