1/* 2 * New-style TCG opcode generator for i386 instructions 3 * 4 * Copyright (c) 2022 Red Hat, Inc. 5 * 6 * Author: Paolo Bonzini <pbonzini@redhat.com> 7 * 8 * This library is free software; you can redistribute it and/or 9 * modify it under the terms of the GNU Lesser General Public 10 * License as published by the Free Software Foundation; either 11 * version 2.1 of the License, or (at your option) any later version. 12 * 13 * This library is distributed in the hope that it will be useful, 14 * but WITHOUT ANY WARRANTY; without even the implied warranty of 15 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU 16 * Lesser General Public License for more details. 17 * 18 * You should have received a copy of the GNU Lesser General Public 19 * License along with this library; if not, see <http://www.gnu.org/licenses/>. 20 */ 21 22/* 23 * Sometimes, knowing what the backend has can produce better code. 24 * The exact opcode to check depends on 32- vs. 64-bit. 25 */ 26#ifdef TARGET_X86_64 27#define TCG_TARGET_HAS_extract2_tl TCG_TARGET_HAS_extract2_i64 28#define TCG_TARGET_deposit_tl_valid TCG_TARGET_deposit_i64_valid 29#define TCG_TARGET_extract_tl_valid TCG_TARGET_extract_i64_valid 30#else 31#define TCG_TARGET_HAS_extract2_tl TCG_TARGET_HAS_extract2_i32 32#define TCG_TARGET_deposit_tl_valid TCG_TARGET_deposit_i32_valid 33#define TCG_TARGET_extract_tl_valid TCG_TARGET_extract_i32_valid 34#endif 35 36 37#define ZMM_OFFSET(reg) offsetof(CPUX86State, xmm_regs[reg]) 38 39typedef void (*SSEFunc_i_ep)(TCGv_i32 val, TCGv_ptr env, TCGv_ptr reg); 40typedef void (*SSEFunc_l_ep)(TCGv_i64 val, TCGv_ptr env, TCGv_ptr reg); 41typedef void (*SSEFunc_0_epp)(TCGv_ptr env, TCGv_ptr reg_a, TCGv_ptr reg_b); 42typedef void (*SSEFunc_0_eppp)(TCGv_ptr env, TCGv_ptr reg_a, TCGv_ptr reg_b, 43 TCGv_ptr reg_c); 44typedef void (*SSEFunc_0_epppp)(TCGv_ptr env, TCGv_ptr reg_a, TCGv_ptr reg_b, 45 TCGv_ptr reg_c, TCGv_ptr reg_d); 46typedef void (*SSEFunc_0_eppi)(TCGv_ptr env, TCGv_ptr reg_a, TCGv_ptr reg_b, 47 TCGv_i32 val); 48typedef void (*SSEFunc_0_epppi)(TCGv_ptr env, TCGv_ptr reg_a, TCGv_ptr reg_b, 49 TCGv_ptr reg_c, TCGv_i32 val); 50typedef void (*SSEFunc_0_ppi)(TCGv_ptr reg_a, TCGv_ptr reg_b, TCGv_i32 val); 51typedef void (*SSEFunc_0_pppi)(TCGv_ptr reg_a, TCGv_ptr reg_b, TCGv_ptr reg_c, 52 TCGv_i32 val); 53typedef void (*SSEFunc_0_eppt)(TCGv_ptr env, TCGv_ptr reg_a, TCGv_ptr reg_b, 54 TCGv val); 55typedef void (*SSEFunc_0_epppti)(TCGv_ptr env, TCGv_ptr reg_a, TCGv_ptr reg_b, 56 TCGv_ptr reg_c, TCGv a0, TCGv_i32 scale); 57typedef void (*SSEFunc_0_eppppi)(TCGv_ptr env, TCGv_ptr reg_a, TCGv_ptr reg_b, 58 TCGv_ptr reg_c, TCGv_ptr reg_d, TCGv_i32 flags); 59typedef void (*SSEFunc_0_eppppii)(TCGv_ptr env, TCGv_ptr reg_a, TCGv_ptr reg_b, 60 TCGv_ptr reg_c, TCGv_ptr reg_d, TCGv_i32 even, 61 TCGv_i32 odd); 62 63static void gen_JMP_m(DisasContext *s, CPUX86State *env, X86DecodedInsn *decode); 64static void gen_JMP(DisasContext *s, CPUX86State *env, X86DecodedInsn *decode); 65 66static inline TCGv_i32 tcg_constant8u_i32(uint8_t val) 67{ 68 return tcg_constant_i32(val); 69} 70 71static void gen_NM_exception(DisasContext *s) 72{ 73 gen_exception(s, EXCP07_PREX); 74} 75 76static void gen_load_ea(DisasContext *s, AddressParts *mem, bool is_vsib) 77{ 78 TCGv ea = gen_lea_modrm_1(s, *mem, is_vsib); 79 gen_lea_v_seg(s, ea, mem->def_seg, s->override); 80} 81 82static inline int mmx_offset(MemOp ot) 83{ 84 switch (ot) { 85 case MO_8: 86 return offsetof(MMXReg, MMX_B(0)); 87 case MO_16: 88 return offsetof(MMXReg, MMX_W(0)); 89 case MO_32: 90 return offsetof(MMXReg, MMX_L(0)); 91 case MO_64: 92 return offsetof(MMXReg, MMX_Q(0)); 93 default: 94 g_assert_not_reached(); 95 } 96} 97 98static inline int xmm_offset(MemOp ot) 99{ 100 switch (ot) { 101 case MO_8: 102 return offsetof(ZMMReg, ZMM_B(0)); 103 case MO_16: 104 return offsetof(ZMMReg, ZMM_W(0)); 105 case MO_32: 106 return offsetof(ZMMReg, ZMM_L(0)); 107 case MO_64: 108 return offsetof(ZMMReg, ZMM_Q(0)); 109 case MO_128: 110 return offsetof(ZMMReg, ZMM_X(0)); 111 case MO_256: 112 return offsetof(ZMMReg, ZMM_Y(0)); 113 default: 114 g_assert_not_reached(); 115 } 116} 117 118static int vector_reg_offset(X86DecodedOp *op) 119{ 120 assert(op->unit == X86_OP_MMX || op->unit == X86_OP_SSE); 121 122 if (op->unit == X86_OP_MMX) { 123 return op->offset - mmx_offset(op->ot); 124 } else { 125 return op->offset - xmm_offset(op->ot); 126 } 127} 128 129static int vector_elem_offset(X86DecodedOp *op, MemOp ot, int n) 130{ 131 int base_ofs = vector_reg_offset(op); 132 switch(ot) { 133 case MO_8: 134 if (op->unit == X86_OP_MMX) { 135 return base_ofs + offsetof(MMXReg, MMX_B(n)); 136 } else { 137 return base_ofs + offsetof(ZMMReg, ZMM_B(n)); 138 } 139 case MO_16: 140 if (op->unit == X86_OP_MMX) { 141 return base_ofs + offsetof(MMXReg, MMX_W(n)); 142 } else { 143 return base_ofs + offsetof(ZMMReg, ZMM_W(n)); 144 } 145 case MO_32: 146 if (op->unit == X86_OP_MMX) { 147 return base_ofs + offsetof(MMXReg, MMX_L(n)); 148 } else { 149 return base_ofs + offsetof(ZMMReg, ZMM_L(n)); 150 } 151 case MO_64: 152 if (op->unit == X86_OP_MMX) { 153 return base_ofs; 154 } else { 155 return base_ofs + offsetof(ZMMReg, ZMM_Q(n)); 156 } 157 case MO_128: 158 assert(op->unit == X86_OP_SSE); 159 return base_ofs + offsetof(ZMMReg, ZMM_X(n)); 160 case MO_256: 161 assert(op->unit == X86_OP_SSE); 162 return base_ofs + offsetof(ZMMReg, ZMM_Y(n)); 163 default: 164 g_assert_not_reached(); 165 } 166} 167 168static void compute_mmx_offset(X86DecodedOp *op) 169{ 170 if (!op->has_ea) { 171 op->offset = offsetof(CPUX86State, fpregs[op->n].mmx) + mmx_offset(op->ot); 172 } else { 173 op->offset = offsetof(CPUX86State, mmx_t0) + mmx_offset(op->ot); 174 } 175} 176 177static void compute_xmm_offset(X86DecodedOp *op) 178{ 179 if (!op->has_ea) { 180 op->offset = ZMM_OFFSET(op->n) + xmm_offset(op->ot); 181 } else { 182 op->offset = offsetof(CPUX86State, xmm_t0) + xmm_offset(op->ot); 183 } 184} 185 186static void gen_load_sse(DisasContext *s, TCGv temp, MemOp ot, int dest_ofs, bool aligned) 187{ 188 switch(ot) { 189 case MO_8: 190 gen_op_ld_v(s, MO_8, temp, s->A0); 191 tcg_gen_st8_tl(temp, tcg_env, dest_ofs); 192 break; 193 case MO_16: 194 gen_op_ld_v(s, MO_16, temp, s->A0); 195 tcg_gen_st16_tl(temp, tcg_env, dest_ofs); 196 break; 197 case MO_32: 198 gen_op_ld_v(s, MO_32, temp, s->A0); 199 tcg_gen_st32_tl(temp, tcg_env, dest_ofs); 200 break; 201 case MO_64: 202 gen_ldq_env_A0(s, dest_ofs); 203 break; 204 case MO_128: 205 gen_ldo_env_A0(s, dest_ofs, aligned); 206 break; 207 case MO_256: 208 gen_ldy_env_A0(s, dest_ofs, aligned); 209 break; 210 default: 211 g_assert_not_reached(); 212 } 213} 214 215static bool sse_needs_alignment(DisasContext *s, X86DecodedInsn *decode, MemOp ot) 216{ 217 switch (decode->e.vex_class) { 218 case 2: 219 case 4: 220 if ((s->prefix & PREFIX_VEX) || 221 decode->e.vex_special == X86_VEX_SSEUnaligned) { 222 /* MOST legacy SSE instructions require aligned memory operands, but not all. */ 223 return false; 224 } 225 /* fall through */ 226 case 1: 227 return ot >= MO_128; 228 229 default: 230 return false; 231 } 232} 233 234static void gen_load(DisasContext *s, X86DecodedInsn *decode, int opn, TCGv v) 235{ 236 X86DecodedOp *op = &decode->op[opn]; 237 238 switch (op->unit) { 239 case X86_OP_SKIP: 240 return; 241 case X86_OP_SEG: 242 tcg_gen_ld32u_tl(v, tcg_env, 243 offsetof(CPUX86State,segs[op->n].selector)); 244 break; 245 case X86_OP_CR: 246 tcg_gen_ld_tl(v, tcg_env, offsetof(CPUX86State, cr[op->n])); 247 break; 248 case X86_OP_DR: 249 tcg_gen_ld_tl(v, tcg_env, offsetof(CPUX86State, dr[op->n])); 250 break; 251 case X86_OP_INT: 252 if (op->has_ea) { 253 if (v == s->T0 && decode->e.special == X86_SPECIAL_SExtT0) { 254 gen_op_ld_v(s, op->ot | MO_SIGN, v, s->A0); 255 } else { 256 gen_op_ld_v(s, op->ot, v, s->A0); 257 } 258 259 } else if (op->ot == MO_8 && byte_reg_is_xH(s, op->n)) { 260 if (v == s->T0 && decode->e.special == X86_SPECIAL_SExtT0) { 261 tcg_gen_sextract_tl(v, cpu_regs[op->n - 4], 8, 8); 262 } else { 263 tcg_gen_extract_tl(v, cpu_regs[op->n - 4], 8, 8); 264 } 265 266 } else if (op->ot < MO_TL && v == s->T0 && 267 (decode->e.special == X86_SPECIAL_SExtT0 || 268 decode->e.special == X86_SPECIAL_ZExtT0)) { 269 if (decode->e.special == X86_SPECIAL_SExtT0) { 270 tcg_gen_ext_tl(v, cpu_regs[op->n], op->ot | MO_SIGN); 271 } else { 272 tcg_gen_ext_tl(v, cpu_regs[op->n], op->ot); 273 } 274 275 } else { 276 tcg_gen_mov_tl(v, cpu_regs[op->n]); 277 } 278 break; 279 case X86_OP_IMM: 280 tcg_gen_movi_tl(v, op->imm); 281 break; 282 283 case X86_OP_MMX: 284 compute_mmx_offset(op); 285 goto load_vector; 286 287 case X86_OP_SSE: 288 compute_xmm_offset(op); 289 load_vector: 290 if (op->has_ea) { 291 bool aligned = sse_needs_alignment(s, decode, op->ot); 292 gen_load_sse(s, v, op->ot, op->offset, aligned); 293 } 294 break; 295 296 default: 297 g_assert_not_reached(); 298 } 299} 300 301static TCGv_ptr op_ptr(X86DecodedInsn *decode, int opn) 302{ 303 X86DecodedOp *op = &decode->op[opn]; 304 305 assert(op->unit == X86_OP_MMX || op->unit == X86_OP_SSE); 306 if (op->v_ptr) { 307 return op->v_ptr; 308 } 309 op->v_ptr = tcg_temp_new_ptr(); 310 311 /* The temporary points to the MMXReg or ZMMReg. */ 312 tcg_gen_addi_ptr(op->v_ptr, tcg_env, vector_reg_offset(op)); 313 return op->v_ptr; 314} 315 316#define OP_PTR0 op_ptr(decode, 0) 317#define OP_PTR1 op_ptr(decode, 1) 318#define OP_PTR2 op_ptr(decode, 2) 319 320static void gen_writeback(DisasContext *s, X86DecodedInsn *decode, int opn, TCGv v) 321{ 322 X86DecodedOp *op = &decode->op[opn]; 323 switch (op->unit) { 324 case X86_OP_SKIP: 325 break; 326 case X86_OP_SEG: 327 /* Note that gen_movl_seg takes care of interrupt shadow and TF. */ 328 gen_movl_seg(s, op->n, s->T0); 329 break; 330 case X86_OP_INT: 331 if (op->has_ea) { 332 gen_op_st_v(s, op->ot, v, s->A0); 333 } else { 334 gen_op_mov_reg_v(s, op->ot, op->n, v); 335 } 336 break; 337 case X86_OP_MMX: 338 break; 339 case X86_OP_SSE: 340 if (!op->has_ea && (s->prefix & PREFIX_VEX) && op->ot <= MO_128) { 341 tcg_gen_gvec_dup_imm(MO_64, 342 offsetof(CPUX86State, xmm_regs[op->n].ZMM_X(1)), 343 16, 16, 0); 344 } 345 break; 346 case X86_OP_CR: 347 case X86_OP_DR: 348 default: 349 g_assert_not_reached(); 350 } 351 op->unit = X86_OP_SKIP; 352} 353 354static inline int vector_len(DisasContext *s, X86DecodedInsn *decode) 355{ 356 if (decode->e.special == X86_SPECIAL_MMX && 357 !(s->prefix & (PREFIX_DATA | PREFIX_REPZ | PREFIX_REPNZ))) { 358 return 8; 359 } 360 return s->vex_l ? 32 : 16; 361} 362 363static void prepare_update1_cc(X86DecodedInsn *decode, DisasContext *s, CCOp op) 364{ 365 decode->cc_dst = s->T0; 366 decode->cc_op = op; 367} 368 369static void prepare_update2_cc(X86DecodedInsn *decode, DisasContext *s, CCOp op) 370{ 371 decode->cc_src = s->T1; 372 decode->cc_dst = s->T0; 373 decode->cc_op = op; 374} 375 376static void prepare_update_cc_incdec(X86DecodedInsn *decode, DisasContext *s, CCOp op) 377{ 378 gen_compute_eflags_c(s, s->T1); 379 prepare_update2_cc(decode, s, op); 380} 381 382static void prepare_update3_cc(X86DecodedInsn *decode, DisasContext *s, CCOp op, TCGv reg) 383{ 384 decode->cc_src2 = reg; 385 decode->cc_src = s->T1; 386 decode->cc_dst = s->T0; 387 decode->cc_op = op; 388} 389 390static void gen_store_sse(DisasContext *s, X86DecodedInsn *decode, int src_ofs) 391{ 392 MemOp ot = decode->op[0].ot; 393 int vec_len = vector_len(s, decode); 394 bool aligned = sse_needs_alignment(s, decode, ot); 395 396 if (!decode->op[0].has_ea) { 397 tcg_gen_gvec_mov(MO_64, decode->op[0].offset, src_ofs, vec_len, vec_len); 398 return; 399 } 400 401 switch (ot) { 402 case MO_64: 403 gen_stq_env_A0(s, src_ofs); 404 break; 405 case MO_128: 406 gen_sto_env_A0(s, src_ofs, aligned); 407 break; 408 case MO_256: 409 gen_sty_env_A0(s, src_ofs, aligned); 410 break; 411 default: 412 g_assert_not_reached(); 413 } 414} 415 416static void gen_helper_pavgusb(TCGv_ptr env, TCGv_ptr reg_a, TCGv_ptr reg_b) 417{ 418 gen_helper_pavgb_mmx(env, reg_a, reg_a, reg_b); 419} 420 421#define FN_3DNOW_MOVE ((SSEFunc_0_epp) (uintptr_t) 1) 422static const SSEFunc_0_epp fns_3dnow[] = { 423 [0x0c] = gen_helper_pi2fw, 424 [0x0d] = gen_helper_pi2fd, 425 [0x1c] = gen_helper_pf2iw, 426 [0x1d] = gen_helper_pf2id, 427 [0x8a] = gen_helper_pfnacc, 428 [0x8e] = gen_helper_pfpnacc, 429 [0x90] = gen_helper_pfcmpge, 430 [0x94] = gen_helper_pfmin, 431 [0x96] = gen_helper_pfrcp, 432 [0x97] = gen_helper_pfrsqrt, 433 [0x9a] = gen_helper_pfsub, 434 [0x9e] = gen_helper_pfadd, 435 [0xa0] = gen_helper_pfcmpgt, 436 [0xa4] = gen_helper_pfmax, 437 [0xa6] = FN_3DNOW_MOVE, /* PFRCPIT1; no need to actually increase precision */ 438 [0xa7] = FN_3DNOW_MOVE, /* PFRSQIT1 */ 439 [0xb6] = FN_3DNOW_MOVE, /* PFRCPIT2 */ 440 [0xaa] = gen_helper_pfsubr, 441 [0xae] = gen_helper_pfacc, 442 [0xb0] = gen_helper_pfcmpeq, 443 [0xb4] = gen_helper_pfmul, 444 [0xb7] = gen_helper_pmulhrw_mmx, 445 [0xbb] = gen_helper_pswapd, 446 [0xbf] = gen_helper_pavgusb, 447}; 448 449static void gen_3dnow(DisasContext *s, CPUX86State *env, X86DecodedInsn *decode) 450{ 451 uint8_t b = decode->immediate; 452 SSEFunc_0_epp fn = b < ARRAY_SIZE(fns_3dnow) ? fns_3dnow[b] : NULL; 453 454 if (!fn) { 455 gen_illegal_opcode(s); 456 return; 457 } 458 if (s->flags & HF_TS_MASK) { 459 gen_NM_exception(s); 460 return; 461 } 462 if (s->flags & HF_EM_MASK) { 463 gen_illegal_opcode(s); 464 return; 465 } 466 467 gen_helper_enter_mmx(tcg_env); 468 if (fn == FN_3DNOW_MOVE) { 469 tcg_gen_ld_i64(s->tmp1_i64, tcg_env, decode->op[1].offset); 470 tcg_gen_st_i64(s->tmp1_i64, tcg_env, decode->op[0].offset); 471 } else { 472 fn(tcg_env, OP_PTR0, OP_PTR1); 473 } 474} 475 476/* 477 * 00 = v*ps Vps, Hps, Wpd 478 * 66 = v*pd Vpd, Hpd, Wps 479 * f3 = v*ss Vss, Hss, Wps 480 * f2 = v*sd Vsd, Hsd, Wps 481 */ 482static inline void gen_unary_fp_sse(DisasContext *s, CPUX86State *env, X86DecodedInsn *decode, 483 SSEFunc_0_epp pd_xmm, SSEFunc_0_epp ps_xmm, 484 SSEFunc_0_epp pd_ymm, SSEFunc_0_epp ps_ymm, 485 SSEFunc_0_eppp sd, SSEFunc_0_eppp ss) 486{ 487 if ((s->prefix & (PREFIX_REPZ | PREFIX_REPNZ)) != 0) { 488 SSEFunc_0_eppp fn = s->prefix & PREFIX_REPZ ? ss : sd; 489 if (!fn) { 490 gen_illegal_opcode(s); 491 return; 492 } 493 fn(tcg_env, OP_PTR0, OP_PTR1, OP_PTR2); 494 } else { 495 SSEFunc_0_epp ps, pd, fn; 496 ps = s->vex_l ? ps_ymm : ps_xmm; 497 pd = s->vex_l ? pd_ymm : pd_xmm; 498 fn = s->prefix & PREFIX_DATA ? pd : ps; 499 if (!fn) { 500 gen_illegal_opcode(s); 501 return; 502 } 503 fn(tcg_env, OP_PTR0, OP_PTR2); 504 } 505} 506#define UNARY_FP_SSE(uname, lname) \ 507static void gen_##uname(DisasContext *s, CPUX86State *env, X86DecodedInsn *decode) \ 508{ \ 509 gen_unary_fp_sse(s, env, decode, \ 510 gen_helper_##lname##pd_xmm, \ 511 gen_helper_##lname##ps_xmm, \ 512 gen_helper_##lname##pd_ymm, \ 513 gen_helper_##lname##ps_ymm, \ 514 gen_helper_##lname##sd, \ 515 gen_helper_##lname##ss); \ 516} 517UNARY_FP_SSE(VSQRT, sqrt) 518 519/* 520 * 00 = v*ps Vps, Hps, Wpd 521 * 66 = v*pd Vpd, Hpd, Wps 522 * f3 = v*ss Vss, Hss, Wps 523 * f2 = v*sd Vsd, Hsd, Wps 524 */ 525static inline void gen_fp_sse(DisasContext *s, CPUX86State *env, X86DecodedInsn *decode, 526 SSEFunc_0_eppp pd_xmm, SSEFunc_0_eppp ps_xmm, 527 SSEFunc_0_eppp pd_ymm, SSEFunc_0_eppp ps_ymm, 528 SSEFunc_0_eppp sd, SSEFunc_0_eppp ss) 529{ 530 SSEFunc_0_eppp ps, pd, fn; 531 if ((s->prefix & (PREFIX_REPZ | PREFIX_REPNZ)) != 0) { 532 fn = s->prefix & PREFIX_REPZ ? ss : sd; 533 } else { 534 ps = s->vex_l ? ps_ymm : ps_xmm; 535 pd = s->vex_l ? pd_ymm : pd_xmm; 536 fn = s->prefix & PREFIX_DATA ? pd : ps; 537 } 538 if (fn) { 539 fn(tcg_env, OP_PTR0, OP_PTR1, OP_PTR2); 540 } else { 541 gen_illegal_opcode(s); 542 } 543} 544 545#define FP_SSE(uname, lname) \ 546static void gen_##uname(DisasContext *s, CPUX86State *env, X86DecodedInsn *decode) \ 547{ \ 548 gen_fp_sse(s, env, decode, \ 549 gen_helper_##lname##pd_xmm, \ 550 gen_helper_##lname##ps_xmm, \ 551 gen_helper_##lname##pd_ymm, \ 552 gen_helper_##lname##ps_ymm, \ 553 gen_helper_##lname##sd, \ 554 gen_helper_##lname##ss); \ 555} 556FP_SSE(VADD, add) 557FP_SSE(VMUL, mul) 558FP_SSE(VSUB, sub) 559FP_SSE(VMIN, min) 560FP_SSE(VDIV, div) 561FP_SSE(VMAX, max) 562 563#define FMA_SSE_PACKED(uname, ptr0, ptr1, ptr2, even, odd) \ 564static void gen_##uname##Px(DisasContext *s, CPUX86State *env, X86DecodedInsn *decode) \ 565{ \ 566 SSEFunc_0_eppppii xmm = s->vex_w ? gen_helper_fma4pd_xmm : gen_helper_fma4ps_xmm; \ 567 SSEFunc_0_eppppii ymm = s->vex_w ? gen_helper_fma4pd_ymm : gen_helper_fma4ps_ymm; \ 568 SSEFunc_0_eppppii fn = s->vex_l ? ymm : xmm; \ 569 \ 570 fn(tcg_env, OP_PTR0, ptr0, ptr1, ptr2, \ 571 tcg_constant_i32(even), \ 572 tcg_constant_i32((even) ^ (odd))); \ 573} 574 575#define FMA_SSE(uname, ptr0, ptr1, ptr2, flags) \ 576FMA_SSE_PACKED(uname, ptr0, ptr1, ptr2, flags, flags) \ 577static void gen_##uname##Sx(DisasContext *s, CPUX86State *env, X86DecodedInsn *decode) \ 578{ \ 579 SSEFunc_0_eppppi fn = s->vex_w ? gen_helper_fma4sd : gen_helper_fma4ss; \ 580 \ 581 fn(tcg_env, OP_PTR0, ptr0, ptr1, ptr2, \ 582 tcg_constant_i32(flags)); \ 583} \ 584 585FMA_SSE(VFMADD231, OP_PTR1, OP_PTR2, OP_PTR0, 0) 586FMA_SSE(VFMADD213, OP_PTR1, OP_PTR0, OP_PTR2, 0) 587FMA_SSE(VFMADD132, OP_PTR0, OP_PTR2, OP_PTR1, 0) 588 589FMA_SSE(VFNMADD231, OP_PTR1, OP_PTR2, OP_PTR0, float_muladd_negate_product) 590FMA_SSE(VFNMADD213, OP_PTR1, OP_PTR0, OP_PTR2, float_muladd_negate_product) 591FMA_SSE(VFNMADD132, OP_PTR0, OP_PTR2, OP_PTR1, float_muladd_negate_product) 592 593FMA_SSE(VFMSUB231, OP_PTR1, OP_PTR2, OP_PTR0, float_muladd_negate_c) 594FMA_SSE(VFMSUB213, OP_PTR1, OP_PTR0, OP_PTR2, float_muladd_negate_c) 595FMA_SSE(VFMSUB132, OP_PTR0, OP_PTR2, OP_PTR1, float_muladd_negate_c) 596 597FMA_SSE(VFNMSUB231, OP_PTR1, OP_PTR2, OP_PTR0, float_muladd_negate_c|float_muladd_negate_product) 598FMA_SSE(VFNMSUB213, OP_PTR1, OP_PTR0, OP_PTR2, float_muladd_negate_c|float_muladd_negate_product) 599FMA_SSE(VFNMSUB132, OP_PTR0, OP_PTR2, OP_PTR1, float_muladd_negate_c|float_muladd_negate_product) 600 601FMA_SSE_PACKED(VFMADDSUB231, OP_PTR1, OP_PTR2, OP_PTR0, float_muladd_negate_c, 0) 602FMA_SSE_PACKED(VFMADDSUB213, OP_PTR1, OP_PTR0, OP_PTR2, float_muladd_negate_c, 0) 603FMA_SSE_PACKED(VFMADDSUB132, OP_PTR0, OP_PTR2, OP_PTR1, float_muladd_negate_c, 0) 604 605FMA_SSE_PACKED(VFMSUBADD231, OP_PTR1, OP_PTR2, OP_PTR0, 0, float_muladd_negate_c) 606FMA_SSE_PACKED(VFMSUBADD213, OP_PTR1, OP_PTR0, OP_PTR2, 0, float_muladd_negate_c) 607FMA_SSE_PACKED(VFMSUBADD132, OP_PTR0, OP_PTR2, OP_PTR1, 0, float_muladd_negate_c) 608 609#define FP_UNPACK_SSE(uname, lname) \ 610static void gen_##uname(DisasContext *s, CPUX86State *env, X86DecodedInsn *decode) \ 611{ \ 612 /* PS maps to the DQ integer instruction, PD maps to QDQ. */ \ 613 gen_fp_sse(s, env, decode, \ 614 gen_helper_##lname##qdq_xmm, \ 615 gen_helper_##lname##dq_xmm, \ 616 gen_helper_##lname##qdq_ymm, \ 617 gen_helper_##lname##dq_ymm, \ 618 NULL, NULL); \ 619} 620FP_UNPACK_SSE(VUNPCKLPx, punpckl) 621FP_UNPACK_SSE(VUNPCKHPx, punpckh) 622 623/* 624 * 00 = v*ps Vps, Wpd 625 * f3 = v*ss Vss, Wps 626 */ 627static inline void gen_unary_fp32_sse(DisasContext *s, CPUX86State *env, X86DecodedInsn *decode, 628 SSEFunc_0_epp ps_xmm, 629 SSEFunc_0_epp ps_ymm, 630 SSEFunc_0_eppp ss) 631{ 632 if ((s->prefix & (PREFIX_DATA | PREFIX_REPNZ)) != 0) { 633 goto illegal_op; 634 } else if (s->prefix & PREFIX_REPZ) { 635 if (!ss) { 636 goto illegal_op; 637 } 638 ss(tcg_env, OP_PTR0, OP_PTR1, OP_PTR2); 639 } else { 640 SSEFunc_0_epp fn = s->vex_l ? ps_ymm : ps_xmm; 641 if (!fn) { 642 goto illegal_op; 643 } 644 fn(tcg_env, OP_PTR0, OP_PTR2); 645 } 646 return; 647 648illegal_op: 649 gen_illegal_opcode(s); 650} 651#define UNARY_FP32_SSE(uname, lname) \ 652static void gen_##uname(DisasContext *s, CPUX86State *env, X86DecodedInsn *decode) \ 653{ \ 654 gen_unary_fp32_sse(s, env, decode, \ 655 gen_helper_##lname##ps_xmm, \ 656 gen_helper_##lname##ps_ymm, \ 657 gen_helper_##lname##ss); \ 658} 659UNARY_FP32_SSE(VRSQRT, rsqrt) 660UNARY_FP32_SSE(VRCP, rcp) 661 662/* 663 * 66 = v*pd Vpd, Hpd, Wpd 664 * f2 = v*ps Vps, Hps, Wps 665 */ 666static inline void gen_horizontal_fp_sse(DisasContext *s, CPUX86State *env, X86DecodedInsn *decode, 667 SSEFunc_0_eppp pd_xmm, SSEFunc_0_eppp ps_xmm, 668 SSEFunc_0_eppp pd_ymm, SSEFunc_0_eppp ps_ymm) 669{ 670 SSEFunc_0_eppp ps, pd, fn; 671 ps = s->vex_l ? ps_ymm : ps_xmm; 672 pd = s->vex_l ? pd_ymm : pd_xmm; 673 fn = s->prefix & PREFIX_DATA ? pd : ps; 674 fn(tcg_env, OP_PTR0, OP_PTR1, OP_PTR2); 675} 676#define HORIZONTAL_FP_SSE(uname, lname) \ 677static void gen_##uname(DisasContext *s, CPUX86State *env, X86DecodedInsn *decode) \ 678{ \ 679 gen_horizontal_fp_sse(s, env, decode, \ 680 gen_helper_##lname##pd_xmm, gen_helper_##lname##ps_xmm, \ 681 gen_helper_##lname##pd_ymm, gen_helper_##lname##ps_ymm); \ 682} 683HORIZONTAL_FP_SSE(VHADD, hadd) 684HORIZONTAL_FP_SSE(VHSUB, hsub) 685HORIZONTAL_FP_SSE(VADDSUB, addsub) 686 687static inline void gen_ternary_sse(DisasContext *s, CPUX86State *env, X86DecodedInsn *decode, 688 int op3, SSEFunc_0_epppp xmm, SSEFunc_0_epppp ymm) 689{ 690 SSEFunc_0_epppp fn = s->vex_l ? ymm : xmm; 691 TCGv_ptr ptr3 = tcg_temp_new_ptr(); 692 693 /* The format of the fourth input is Lx */ 694 tcg_gen_addi_ptr(ptr3, tcg_env, ZMM_OFFSET(op3)); 695 fn(tcg_env, OP_PTR0, OP_PTR1, OP_PTR2, ptr3); 696} 697#define TERNARY_SSE(uname, uvname, lname) \ 698static void gen_##uvname(DisasContext *s, CPUX86State *env, X86DecodedInsn *decode) \ 699{ \ 700 gen_ternary_sse(s, env, decode, (uint8_t)decode->immediate >> 4, \ 701 gen_helper_##lname##_xmm, gen_helper_##lname##_ymm); \ 702} \ 703static void gen_##uname(DisasContext *s, CPUX86State *env, X86DecodedInsn *decode) \ 704{ \ 705 gen_ternary_sse(s, env, decode, 0, \ 706 gen_helper_##lname##_xmm, gen_helper_##lname##_ymm); \ 707} 708TERNARY_SSE(BLENDVPS, VBLENDVPS, blendvps) 709TERNARY_SSE(BLENDVPD, VBLENDVPD, blendvpd) 710TERNARY_SSE(PBLENDVB, VPBLENDVB, pblendvb) 711 712static inline void gen_binary_imm_sse(DisasContext *s, CPUX86State *env, X86DecodedInsn *decode, 713 SSEFunc_0_epppi xmm, SSEFunc_0_epppi ymm) 714{ 715 TCGv_i32 imm = tcg_constant8u_i32(decode->immediate); 716 if (!s->vex_l) { 717 xmm(tcg_env, OP_PTR0, OP_PTR1, OP_PTR2, imm); 718 } else { 719 ymm(tcg_env, OP_PTR0, OP_PTR1, OP_PTR2, imm); 720 } 721} 722 723#define BINARY_IMM_SSE(uname, lname) \ 724static void gen_##uname(DisasContext *s, CPUX86State *env, X86DecodedInsn *decode) \ 725{ \ 726 gen_binary_imm_sse(s, env, decode, \ 727 gen_helper_##lname##_xmm, \ 728 gen_helper_##lname##_ymm); \ 729} 730 731BINARY_IMM_SSE(VBLENDPD, blendpd) 732BINARY_IMM_SSE(VBLENDPS, blendps) 733BINARY_IMM_SSE(VPBLENDW, pblendw) 734BINARY_IMM_SSE(VDDPS, dpps) 735#define gen_helper_dppd_ymm NULL 736BINARY_IMM_SSE(VDDPD, dppd) 737BINARY_IMM_SSE(VMPSADBW, mpsadbw) 738BINARY_IMM_SSE(PCLMULQDQ, pclmulqdq) 739 740 741#define UNARY_INT_GVEC(uname, func, ...) \ 742static void gen_##uname(DisasContext *s, CPUX86State *env, X86DecodedInsn *decode) \ 743{ \ 744 int vec_len = vector_len(s, decode); \ 745 \ 746 func(__VA_ARGS__, decode->op[0].offset, \ 747 decode->op[2].offset, vec_len, vec_len); \ 748} 749UNARY_INT_GVEC(PABSB, tcg_gen_gvec_abs, MO_8) 750UNARY_INT_GVEC(PABSW, tcg_gen_gvec_abs, MO_16) 751UNARY_INT_GVEC(PABSD, tcg_gen_gvec_abs, MO_32) 752UNARY_INT_GVEC(VBROADCASTx128, tcg_gen_gvec_dup_mem, MO_128) 753UNARY_INT_GVEC(VPBROADCASTB, tcg_gen_gvec_dup_mem, MO_8) 754UNARY_INT_GVEC(VPBROADCASTW, tcg_gen_gvec_dup_mem, MO_16) 755UNARY_INT_GVEC(VPBROADCASTD, tcg_gen_gvec_dup_mem, MO_32) 756UNARY_INT_GVEC(VPBROADCASTQ, tcg_gen_gvec_dup_mem, MO_64) 757 758 759#define BINARY_INT_GVEC(uname, func, ...) \ 760static void gen_##uname(DisasContext *s, CPUX86State *env, X86DecodedInsn *decode) \ 761{ \ 762 int vec_len = vector_len(s, decode); \ 763 \ 764 func(__VA_ARGS__, \ 765 decode->op[0].offset, decode->op[1].offset, \ 766 decode->op[2].offset, vec_len, vec_len); \ 767} 768 769BINARY_INT_GVEC(PADDB, tcg_gen_gvec_add, MO_8) 770BINARY_INT_GVEC(PADDW, tcg_gen_gvec_add, MO_16) 771BINARY_INT_GVEC(PADDD, tcg_gen_gvec_add, MO_32) 772BINARY_INT_GVEC(PADDQ, tcg_gen_gvec_add, MO_64) 773BINARY_INT_GVEC(PADDSB, tcg_gen_gvec_ssadd, MO_8) 774BINARY_INT_GVEC(PADDSW, tcg_gen_gvec_ssadd, MO_16) 775BINARY_INT_GVEC(PADDUSB, tcg_gen_gvec_usadd, MO_8) 776BINARY_INT_GVEC(PADDUSW, tcg_gen_gvec_usadd, MO_16) 777BINARY_INT_GVEC(PAND, tcg_gen_gvec_and, MO_64) 778BINARY_INT_GVEC(PCMPEQB, tcg_gen_gvec_cmp, TCG_COND_EQ, MO_8) 779BINARY_INT_GVEC(PCMPEQD, tcg_gen_gvec_cmp, TCG_COND_EQ, MO_32) 780BINARY_INT_GVEC(PCMPEQW, tcg_gen_gvec_cmp, TCG_COND_EQ, MO_16) 781BINARY_INT_GVEC(PCMPEQQ, tcg_gen_gvec_cmp, TCG_COND_EQ, MO_64) 782BINARY_INT_GVEC(PCMPGTB, tcg_gen_gvec_cmp, TCG_COND_GT, MO_8) 783BINARY_INT_GVEC(PCMPGTW, tcg_gen_gvec_cmp, TCG_COND_GT, MO_16) 784BINARY_INT_GVEC(PCMPGTD, tcg_gen_gvec_cmp, TCG_COND_GT, MO_32) 785BINARY_INT_GVEC(PCMPGTQ, tcg_gen_gvec_cmp, TCG_COND_GT, MO_64) 786BINARY_INT_GVEC(PMAXSB, tcg_gen_gvec_smax, MO_8) 787BINARY_INT_GVEC(PMAXSW, tcg_gen_gvec_smax, MO_16) 788BINARY_INT_GVEC(PMAXSD, tcg_gen_gvec_smax, MO_32) 789BINARY_INT_GVEC(PMAXUB, tcg_gen_gvec_umax, MO_8) 790BINARY_INT_GVEC(PMAXUW, tcg_gen_gvec_umax, MO_16) 791BINARY_INT_GVEC(PMAXUD, tcg_gen_gvec_umax, MO_32) 792BINARY_INT_GVEC(PMINSB, tcg_gen_gvec_smin, MO_8) 793BINARY_INT_GVEC(PMINSW, tcg_gen_gvec_smin, MO_16) 794BINARY_INT_GVEC(PMINSD, tcg_gen_gvec_smin, MO_32) 795BINARY_INT_GVEC(PMINUB, tcg_gen_gvec_umin, MO_8) 796BINARY_INT_GVEC(PMINUW, tcg_gen_gvec_umin, MO_16) 797BINARY_INT_GVEC(PMINUD, tcg_gen_gvec_umin, MO_32) 798BINARY_INT_GVEC(PMULLW, tcg_gen_gvec_mul, MO_16) 799BINARY_INT_GVEC(PMULLD, tcg_gen_gvec_mul, MO_32) 800BINARY_INT_GVEC(POR, tcg_gen_gvec_or, MO_64) 801BINARY_INT_GVEC(PSUBB, tcg_gen_gvec_sub, MO_8) 802BINARY_INT_GVEC(PSUBW, tcg_gen_gvec_sub, MO_16) 803BINARY_INT_GVEC(PSUBD, tcg_gen_gvec_sub, MO_32) 804BINARY_INT_GVEC(PSUBQ, tcg_gen_gvec_sub, MO_64) 805BINARY_INT_GVEC(PSUBSB, tcg_gen_gvec_sssub, MO_8) 806BINARY_INT_GVEC(PSUBSW, tcg_gen_gvec_sssub, MO_16) 807BINARY_INT_GVEC(PSUBUSB, tcg_gen_gvec_ussub, MO_8) 808BINARY_INT_GVEC(PSUBUSW, tcg_gen_gvec_ussub, MO_16) 809BINARY_INT_GVEC(PXOR, tcg_gen_gvec_xor, MO_64) 810 811 812/* 813 * 00 = p* Pq, Qq (if mmx not NULL; no VEX) 814 * 66 = vp* Vx, Hx, Wx 815 * 816 * These are really the same encoding, because 1) V is the same as P when VEX.V 817 * is not present 2) P and Q are the same as H and W apart from MM/XMM 818 */ 819static inline void gen_binary_int_sse(DisasContext *s, CPUX86State *env, X86DecodedInsn *decode, 820 SSEFunc_0_eppp mmx, SSEFunc_0_eppp xmm, SSEFunc_0_eppp ymm) 821{ 822 assert(!!mmx == !!(decode->e.special == X86_SPECIAL_MMX)); 823 824 if (mmx && (s->prefix & PREFIX_VEX) && !(s->prefix & PREFIX_DATA)) { 825 /* VEX encoding is not applicable to MMX instructions. */ 826 gen_illegal_opcode(s); 827 return; 828 } 829 if (!(s->prefix & PREFIX_DATA)) { 830 mmx(tcg_env, OP_PTR0, OP_PTR1, OP_PTR2); 831 } else if (!s->vex_l) { 832 xmm(tcg_env, OP_PTR0, OP_PTR1, OP_PTR2); 833 } else { 834 ymm(tcg_env, OP_PTR0, OP_PTR1, OP_PTR2); 835 } 836} 837 838 839#define BINARY_INT_MMX(uname, lname) \ 840static void gen_##uname(DisasContext *s, CPUX86State *env, X86DecodedInsn *decode) \ 841{ \ 842 gen_binary_int_sse(s, env, decode, \ 843 gen_helper_##lname##_mmx, \ 844 gen_helper_##lname##_xmm, \ 845 gen_helper_##lname##_ymm); \ 846} 847BINARY_INT_MMX(PUNPCKLBW, punpcklbw) 848BINARY_INT_MMX(PUNPCKLWD, punpcklwd) 849BINARY_INT_MMX(PUNPCKLDQ, punpckldq) 850BINARY_INT_MMX(PACKSSWB, packsswb) 851BINARY_INT_MMX(PACKUSWB, packuswb) 852BINARY_INT_MMX(PUNPCKHBW, punpckhbw) 853BINARY_INT_MMX(PUNPCKHWD, punpckhwd) 854BINARY_INT_MMX(PUNPCKHDQ, punpckhdq) 855BINARY_INT_MMX(PACKSSDW, packssdw) 856 857BINARY_INT_MMX(PAVGB, pavgb) 858BINARY_INT_MMX(PAVGW, pavgw) 859BINARY_INT_MMX(PMADDWD, pmaddwd) 860BINARY_INT_MMX(PMULHUW, pmulhuw) 861BINARY_INT_MMX(PMULHW, pmulhw) 862BINARY_INT_MMX(PMULUDQ, pmuludq) 863BINARY_INT_MMX(PSADBW, psadbw) 864 865BINARY_INT_MMX(PSLLW_r, psllw) 866BINARY_INT_MMX(PSLLD_r, pslld) 867BINARY_INT_MMX(PSLLQ_r, psllq) 868BINARY_INT_MMX(PSRLW_r, psrlw) 869BINARY_INT_MMX(PSRLD_r, psrld) 870BINARY_INT_MMX(PSRLQ_r, psrlq) 871BINARY_INT_MMX(PSRAW_r, psraw) 872BINARY_INT_MMX(PSRAD_r, psrad) 873 874BINARY_INT_MMX(PHADDW, phaddw) 875BINARY_INT_MMX(PHADDSW, phaddsw) 876BINARY_INT_MMX(PHADDD, phaddd) 877BINARY_INT_MMX(PHSUBW, phsubw) 878BINARY_INT_MMX(PHSUBSW, phsubsw) 879BINARY_INT_MMX(PHSUBD, phsubd) 880BINARY_INT_MMX(PMADDUBSW, pmaddubsw) 881BINARY_INT_MMX(PSHUFB, pshufb) 882BINARY_INT_MMX(PSIGNB, psignb) 883BINARY_INT_MMX(PSIGNW, psignw) 884BINARY_INT_MMX(PSIGND, psignd) 885BINARY_INT_MMX(PMULHRSW, pmulhrsw) 886 887/* Instructions with no MMX equivalent. */ 888#define BINARY_INT_SSE(uname, lname) \ 889static void gen_##uname(DisasContext *s, CPUX86State *env, X86DecodedInsn *decode) \ 890{ \ 891 gen_binary_int_sse(s, env, decode, \ 892 NULL, \ 893 gen_helper_##lname##_xmm, \ 894 gen_helper_##lname##_ymm); \ 895} 896 897/* Instructions with no MMX equivalent. */ 898BINARY_INT_SSE(PUNPCKLQDQ, punpcklqdq) 899BINARY_INT_SSE(PUNPCKHQDQ, punpckhqdq) 900BINARY_INT_SSE(VPACKUSDW, packusdw) 901BINARY_INT_SSE(VPERMILPS, vpermilps) 902BINARY_INT_SSE(VPERMILPD, vpermilpd) 903BINARY_INT_SSE(VMASKMOVPS, vpmaskmovd) 904BINARY_INT_SSE(VMASKMOVPD, vpmaskmovq) 905 906BINARY_INT_SSE(PMULDQ, pmuldq) 907 908BINARY_INT_SSE(VAESDEC, aesdec) 909BINARY_INT_SSE(VAESDECLAST, aesdeclast) 910BINARY_INT_SSE(VAESENC, aesenc) 911BINARY_INT_SSE(VAESENCLAST, aesenclast) 912 913#define UNARY_CMP_SSE(uname, lname) \ 914static void gen_##uname(DisasContext *s, CPUX86State *env, X86DecodedInsn *decode) \ 915{ \ 916 if (!s->vex_l) { \ 917 gen_helper_##lname##_xmm(tcg_env, OP_PTR1, OP_PTR2); \ 918 } else { \ 919 gen_helper_##lname##_ymm(tcg_env, OP_PTR1, OP_PTR2); \ 920 } \ 921 assume_cc_op(s, CC_OP_EFLAGS); \ 922} 923UNARY_CMP_SSE(VPTEST, ptest) 924UNARY_CMP_SSE(VTESTPS, vtestps) 925UNARY_CMP_SSE(VTESTPD, vtestpd) 926 927static inline void gen_unary_int_sse(DisasContext *s, CPUX86State *env, X86DecodedInsn *decode, 928 SSEFunc_0_epp xmm, SSEFunc_0_epp ymm) 929{ 930 if (!s->vex_l) { 931 xmm(tcg_env, OP_PTR0, OP_PTR2); 932 } else { 933 ymm(tcg_env, OP_PTR0, OP_PTR2); 934 } 935} 936 937#define UNARY_INT_SSE(uname, lname) \ 938static void gen_##uname(DisasContext *s, CPUX86State *env, X86DecodedInsn *decode) \ 939{ \ 940 gen_unary_int_sse(s, env, decode, \ 941 gen_helper_##lname##_xmm, \ 942 gen_helper_##lname##_ymm); \ 943} 944 945UNARY_INT_SSE(VPMOVSXBW, pmovsxbw) 946UNARY_INT_SSE(VPMOVSXBD, pmovsxbd) 947UNARY_INT_SSE(VPMOVSXBQ, pmovsxbq) 948UNARY_INT_SSE(VPMOVSXWD, pmovsxwd) 949UNARY_INT_SSE(VPMOVSXWQ, pmovsxwq) 950UNARY_INT_SSE(VPMOVSXDQ, pmovsxdq) 951 952UNARY_INT_SSE(VPMOVZXBW, pmovzxbw) 953UNARY_INT_SSE(VPMOVZXBD, pmovzxbd) 954UNARY_INT_SSE(VPMOVZXBQ, pmovzxbq) 955UNARY_INT_SSE(VPMOVZXWD, pmovzxwd) 956UNARY_INT_SSE(VPMOVZXWQ, pmovzxwq) 957UNARY_INT_SSE(VPMOVZXDQ, pmovzxdq) 958 959UNARY_INT_SSE(VMOVSLDUP, pmovsldup) 960UNARY_INT_SSE(VMOVSHDUP, pmovshdup) 961UNARY_INT_SSE(VMOVDDUP, pmovdldup) 962 963UNARY_INT_SSE(VCVTDQ2PD, cvtdq2pd) 964UNARY_INT_SSE(VCVTPD2DQ, cvtpd2dq) 965UNARY_INT_SSE(VCVTTPD2DQ, cvttpd2dq) 966UNARY_INT_SSE(VCVTDQ2PS, cvtdq2ps) 967UNARY_INT_SSE(VCVTPS2DQ, cvtps2dq) 968UNARY_INT_SSE(VCVTTPS2DQ, cvttps2dq) 969UNARY_INT_SSE(VCVTPH2PS, cvtph2ps) 970 971 972static inline void gen_unary_imm_sse(DisasContext *s, CPUX86State *env, X86DecodedInsn *decode, 973 SSEFunc_0_ppi xmm, SSEFunc_0_ppi ymm) 974{ 975 TCGv_i32 imm = tcg_constant8u_i32(decode->immediate); 976 if (!s->vex_l) { 977 xmm(OP_PTR0, OP_PTR1, imm); 978 } else { 979 ymm(OP_PTR0, OP_PTR1, imm); 980 } 981} 982 983#define UNARY_IMM_SSE(uname, lname) \ 984static void gen_##uname(DisasContext *s, CPUX86State *env, X86DecodedInsn *decode) \ 985{ \ 986 gen_unary_imm_sse(s, env, decode, \ 987 gen_helper_##lname##_xmm, \ 988 gen_helper_##lname##_ymm); \ 989} 990 991UNARY_IMM_SSE(PSHUFD, pshufd) 992UNARY_IMM_SSE(PSHUFHW, pshufhw) 993UNARY_IMM_SSE(PSHUFLW, pshuflw) 994#define gen_helper_vpermq_xmm NULL 995UNARY_IMM_SSE(VPERMQ, vpermq) 996UNARY_IMM_SSE(VPERMILPS_i, vpermilps_imm) 997UNARY_IMM_SSE(VPERMILPD_i, vpermilpd_imm) 998 999static inline void gen_unary_imm_fp_sse(DisasContext *s, CPUX86State *env, X86DecodedInsn *decode, 1000 SSEFunc_0_eppi xmm, SSEFunc_0_eppi ymm) 1001{ 1002 TCGv_i32 imm = tcg_constant8u_i32(decode->immediate); 1003 if (!s->vex_l) { 1004 xmm(tcg_env, OP_PTR0, OP_PTR1, imm); 1005 } else { 1006 ymm(tcg_env, OP_PTR0, OP_PTR1, imm); 1007 } 1008} 1009 1010#define UNARY_IMM_FP_SSE(uname, lname) \ 1011static void gen_##uname(DisasContext *s, CPUX86State *env, X86DecodedInsn *decode) \ 1012{ \ 1013 gen_unary_imm_fp_sse(s, env, decode, \ 1014 gen_helper_##lname##_xmm, \ 1015 gen_helper_##lname##_ymm); \ 1016} 1017 1018UNARY_IMM_FP_SSE(VROUNDPS, roundps) 1019UNARY_IMM_FP_SSE(VROUNDPD, roundpd) 1020 1021static inline void gen_vexw_avx(DisasContext *s, CPUX86State *env, X86DecodedInsn *decode, 1022 SSEFunc_0_eppp d_xmm, SSEFunc_0_eppp q_xmm, 1023 SSEFunc_0_eppp d_ymm, SSEFunc_0_eppp q_ymm) 1024{ 1025 SSEFunc_0_eppp d = s->vex_l ? d_ymm : d_xmm; 1026 SSEFunc_0_eppp q = s->vex_l ? q_ymm : q_xmm; 1027 SSEFunc_0_eppp fn = s->vex_w ? q : d; 1028 fn(tcg_env, OP_PTR0, OP_PTR1, OP_PTR2); 1029} 1030 1031/* VEX.W affects whether to operate on 32- or 64-bit elements. */ 1032#define VEXW_AVX(uname, lname) \ 1033static void gen_##uname(DisasContext *s, CPUX86State *env, X86DecodedInsn *decode) \ 1034{ \ 1035 gen_vexw_avx(s, env, decode, \ 1036 gen_helper_##lname##d_xmm, gen_helper_##lname##q_xmm, \ 1037 gen_helper_##lname##d_ymm, gen_helper_##lname##q_ymm); \ 1038} 1039VEXW_AVX(VPSLLV, vpsllv) 1040VEXW_AVX(VPSRLV, vpsrlv) 1041VEXW_AVX(VPSRAV, vpsrav) 1042VEXW_AVX(VPMASKMOV, vpmaskmov) 1043 1044/* Same as above, but with extra arguments to the helper. */ 1045static inline void gen_vsib_avx(DisasContext *s, CPUX86State *env, X86DecodedInsn *decode, 1046 SSEFunc_0_epppti d_xmm, SSEFunc_0_epppti q_xmm, 1047 SSEFunc_0_epppti d_ymm, SSEFunc_0_epppti q_ymm) 1048{ 1049 SSEFunc_0_epppti d = s->vex_l ? d_ymm : d_xmm; 1050 SSEFunc_0_epppti q = s->vex_l ? q_ymm : q_xmm; 1051 SSEFunc_0_epppti fn = s->vex_w ? q : d; 1052 TCGv_i32 scale = tcg_constant_i32(decode->mem.scale); 1053 TCGv_ptr index = tcg_temp_new_ptr(); 1054 1055 /* Pass third input as (index, base, scale) */ 1056 tcg_gen_addi_ptr(index, tcg_env, ZMM_OFFSET(decode->mem.index)); 1057 fn(tcg_env, OP_PTR0, OP_PTR1, index, s->A0, scale); 1058 1059 /* 1060 * There are two output operands, so zero OP1's high 128 bits 1061 * in the VEX.128 case. 1062 */ 1063 if (!s->vex_l) { 1064 int ymmh_ofs = vector_elem_offset(&decode->op[1], MO_128, 1); 1065 tcg_gen_gvec_dup_imm(MO_64, ymmh_ofs, 16, 16, 0); 1066 } 1067} 1068#define VSIB_AVX(uname, lname) \ 1069static void gen_##uname(DisasContext *s, CPUX86State *env, X86DecodedInsn *decode) \ 1070{ \ 1071 gen_vsib_avx(s, env, decode, \ 1072 gen_helper_##lname##d_xmm, gen_helper_##lname##q_xmm, \ 1073 gen_helper_##lname##d_ymm, gen_helper_##lname##q_ymm); \ 1074} 1075VSIB_AVX(VPGATHERD, vpgatherd) 1076VSIB_AVX(VPGATHERQ, vpgatherq) 1077 1078static void gen_AAA(DisasContext *s, CPUX86State *env, X86DecodedInsn *decode) 1079{ 1080 gen_update_cc_op(s); 1081 gen_helper_aaa(tcg_env); 1082 assume_cc_op(s, CC_OP_EFLAGS); 1083} 1084 1085static void gen_AAD(DisasContext *s, CPUX86State *env, X86DecodedInsn *decode) 1086{ 1087 gen_helper_aad(s->T0, s->T0, s->T1); 1088 prepare_update1_cc(decode, s, CC_OP_LOGICB); 1089} 1090 1091static void gen_AAM(DisasContext *s, CPUX86State *env, X86DecodedInsn *decode) 1092{ 1093 if (decode->immediate == 0) { 1094 gen_exception(s, EXCP00_DIVZ); 1095 } else { 1096 gen_helper_aam(s->T0, s->T0, s->T1); 1097 prepare_update1_cc(decode, s, CC_OP_LOGICB); 1098 } 1099} 1100 1101static void gen_AAS(DisasContext *s, CPUX86State *env, X86DecodedInsn *decode) 1102{ 1103 gen_update_cc_op(s); 1104 gen_helper_aas(tcg_env); 1105 assume_cc_op(s, CC_OP_EFLAGS); 1106} 1107 1108static void gen_ADC(DisasContext *s, CPUX86State *env, X86DecodedInsn *decode) 1109{ 1110 MemOp ot = decode->op[1].ot; 1111 TCGv c_in = tcg_temp_new(); 1112 1113 gen_compute_eflags_c(s, c_in); 1114 if (s->prefix & PREFIX_LOCK) { 1115 tcg_gen_add_tl(s->T0, c_in, s->T1); 1116 tcg_gen_atomic_add_fetch_tl(s->T0, s->A0, s->T0, 1117 s->mem_index, ot | MO_LE); 1118 } else { 1119 tcg_gen_add_tl(s->T0, s->T0, s->T1); 1120 tcg_gen_add_tl(s->T0, s->T0, c_in); 1121 } 1122 prepare_update3_cc(decode, s, CC_OP_ADCB + ot, c_in); 1123} 1124 1125/* ADCX/ADOX do not have memory operands and can use set_cc_op. */ 1126static void gen_ADCOX(DisasContext *s, CPUX86State *env, MemOp ot, int cc_op) 1127{ 1128 int opposite_cc_op; 1129 TCGv carry_in = NULL; 1130 TCGv carry_out = (cc_op == CC_OP_ADCX ? cpu_cc_dst : cpu_cc_src2); 1131 TCGv zero; 1132 1133 if (cc_op == s->cc_op || s->cc_op == CC_OP_ADCOX) { 1134 /* Re-use the carry-out from a previous round. */ 1135 carry_in = carry_out; 1136 } else { 1137 /* We don't have a carry-in, get it out of EFLAGS. */ 1138 if (s->cc_op != CC_OP_ADCX && s->cc_op != CC_OP_ADOX) { 1139 gen_compute_eflags(s); 1140 } 1141 carry_in = s->tmp0; 1142 tcg_gen_extract_tl(carry_in, cpu_cc_src, 1143 ctz32(cc_op == CC_OP_ADCX ? CC_C : CC_O), 1); 1144 } 1145 1146 switch (ot) { 1147#ifdef TARGET_X86_64 1148 case MO_32: 1149 /* If TL is 64-bit just do everything in 64-bit arithmetic. */ 1150 tcg_gen_ext32u_tl(s->T0, s->T0); 1151 tcg_gen_ext32u_tl(s->T1, s->T1); 1152 tcg_gen_add_i64(s->T0, s->T0, s->T1); 1153 tcg_gen_add_i64(s->T0, s->T0, carry_in); 1154 tcg_gen_shri_i64(carry_out, s->T0, 32); 1155 break; 1156#endif 1157 default: 1158 zero = tcg_constant_tl(0); 1159 tcg_gen_add2_tl(s->T0, carry_out, s->T0, zero, carry_in, zero); 1160 tcg_gen_add2_tl(s->T0, carry_out, s->T0, carry_out, s->T1, zero); 1161 break; 1162 } 1163 1164 opposite_cc_op = cc_op == CC_OP_ADCX ? CC_OP_ADOX : CC_OP_ADCX; 1165 if (s->cc_op == CC_OP_ADCOX || s->cc_op == opposite_cc_op) { 1166 /* Merge with the carry-out from the opposite instruction. */ 1167 set_cc_op(s, CC_OP_ADCOX); 1168 } else { 1169 set_cc_op(s, cc_op); 1170 } 1171} 1172 1173static void gen_ADCX(DisasContext *s, CPUX86State *env, X86DecodedInsn *decode) 1174{ 1175 gen_ADCOX(s, env, decode->op[0].ot, CC_OP_ADCX); 1176} 1177 1178static void gen_ADD(DisasContext *s, CPUX86State *env, X86DecodedInsn *decode) 1179{ 1180 MemOp ot = decode->op[1].ot; 1181 1182 if (s->prefix & PREFIX_LOCK) { 1183 tcg_gen_atomic_add_fetch_tl(s->T0, s->A0, s->T1, 1184 s->mem_index, ot | MO_LE); 1185 } else { 1186 tcg_gen_add_tl(s->T0, s->T0, s->T1); 1187 } 1188 prepare_update2_cc(decode, s, CC_OP_ADDB + ot); 1189} 1190 1191static void gen_ADOX(DisasContext *s, CPUX86State *env, X86DecodedInsn *decode) 1192{ 1193 gen_ADCOX(s, env, decode->op[0].ot, CC_OP_ADOX); 1194} 1195 1196static void gen_AND(DisasContext *s, CPUX86State *env, X86DecodedInsn *decode) 1197{ 1198 MemOp ot = decode->op[1].ot; 1199 1200 if (s->prefix & PREFIX_LOCK) { 1201 tcg_gen_atomic_and_fetch_tl(s->T0, s->A0, s->T1, 1202 s->mem_index, ot | MO_LE); 1203 } else { 1204 tcg_gen_and_tl(s->T0, s->T0, s->T1); 1205 } 1206 prepare_update1_cc(decode, s, CC_OP_LOGICB + ot); 1207} 1208 1209static void gen_ANDN(DisasContext *s, CPUX86State *env, X86DecodedInsn *decode) 1210{ 1211 MemOp ot = decode->op[0].ot; 1212 1213 tcg_gen_andc_tl(s->T0, s->T1, s->T0); 1214 prepare_update1_cc(decode, s, CC_OP_LOGICB + ot); 1215} 1216 1217static void gen_ARPL(DisasContext *s, CPUX86State *env, X86DecodedInsn *decode) 1218{ 1219 TCGv zf = tcg_temp_new(); 1220 TCGv flags = tcg_temp_new(); 1221 1222 gen_mov_eflags(s, flags); 1223 1224 /* Compute adjusted DST in T1, merging in SRC[RPL]. */ 1225 tcg_gen_deposit_tl(s->T1, s->T0, s->T1, 0, 2); 1226 1227 /* Z flag set if DST[RPL] < SRC[RPL] */ 1228 tcg_gen_setcond_tl(TCG_COND_LTU, zf, s->T0, s->T1); 1229 tcg_gen_deposit_tl(flags, flags, zf, ctz32(CC_Z), 1); 1230 1231 /* Place maximum RPL in DST */ 1232 tcg_gen_umax_tl(s->T0, s->T0, s->T1); 1233 1234 decode->cc_src = flags; 1235 decode->cc_op = CC_OP_EFLAGS; 1236} 1237 1238static void gen_BEXTR(DisasContext *s, CPUX86State *env, X86DecodedInsn *decode) 1239{ 1240 MemOp ot = decode->op[0].ot; 1241 TCGv bound = tcg_constant_tl(ot == MO_64 ? 63 : 31); 1242 TCGv zero = tcg_constant_tl(0); 1243 TCGv mone = tcg_constant_tl(-1); 1244 1245 /* 1246 * Extract START, and shift the operand. 1247 * Shifts larger than operand size get zeros. 1248 */ 1249 tcg_gen_ext8u_tl(s->A0, s->T1); 1250 tcg_gen_shr_tl(s->T0, s->T0, s->A0); 1251 1252 tcg_gen_movcond_tl(TCG_COND_LEU, s->T0, s->A0, bound, s->T0, zero); 1253 1254 /* 1255 * Extract the LEN into an inverse mask. Lengths larger than 1256 * operand size get all zeros, length 0 gets all ones. 1257 */ 1258 tcg_gen_extract_tl(s->A0, s->T1, 8, 8); 1259 tcg_gen_shl_tl(s->T1, mone, s->A0); 1260 tcg_gen_movcond_tl(TCG_COND_LEU, s->T1, s->A0, bound, s->T1, zero); 1261 tcg_gen_andc_tl(s->T0, s->T0, s->T1); 1262 1263 prepare_update1_cc(decode, s, CC_OP_LOGICB + ot); 1264} 1265 1266/* BLSI do not have memory operands and can use set_cc_op. */ 1267static void gen_BLSI(DisasContext *s, CPUX86State *env, X86DecodedInsn *decode) 1268{ 1269 MemOp ot = decode->op[0].ot; 1270 1271 tcg_gen_mov_tl(cpu_cc_src, s->T0); 1272 tcg_gen_neg_tl(s->T1, s->T0); 1273 tcg_gen_and_tl(s->T0, s->T0, s->T1); 1274 tcg_gen_mov_tl(cpu_cc_dst, s->T0); 1275 set_cc_op(s, CC_OP_BMILGB + ot); 1276} 1277 1278/* BLSMSK do not have memory operands and can use set_cc_op. */ 1279static void gen_BLSMSK(DisasContext *s, CPUX86State *env, X86DecodedInsn *decode) 1280{ 1281 MemOp ot = decode->op[0].ot; 1282 1283 tcg_gen_mov_tl(cpu_cc_src, s->T0); 1284 tcg_gen_subi_tl(s->T1, s->T0, 1); 1285 tcg_gen_xor_tl(s->T0, s->T0, s->T1); 1286 tcg_gen_mov_tl(cpu_cc_dst, s->T0); 1287 set_cc_op(s, CC_OP_BMILGB + ot); 1288} 1289 1290/* BLSR do not have memory operands and can use set_cc_op. */ 1291static void gen_BLSR(DisasContext *s, CPUX86State *env, X86DecodedInsn *decode) 1292{ 1293 MemOp ot = decode->op[0].ot; 1294 1295 tcg_gen_mov_tl(cpu_cc_src, s->T0); 1296 tcg_gen_subi_tl(s->T1, s->T0, 1); 1297 tcg_gen_and_tl(s->T0, s->T0, s->T1); 1298 tcg_gen_mov_tl(cpu_cc_dst, s->T0); 1299 set_cc_op(s, CC_OP_BMILGB + ot); 1300} 1301 1302static void gen_BOUND(DisasContext *s, CPUX86State *env, X86DecodedInsn *decode) 1303{ 1304 TCGv_i32 op = tcg_temp_new_i32(); 1305 tcg_gen_trunc_tl_i32(op, s->T0); 1306 if (decode->op[1].ot == MO_16) { 1307 gen_helper_boundw(tcg_env, s->A0, op); 1308 } else { 1309 gen_helper_boundl(tcg_env, s->A0, op); 1310 } 1311} 1312 1313static void gen_BSWAP(DisasContext *s, CPUX86State *env, X86DecodedInsn *decode) 1314{ 1315#ifdef TARGET_X86_64 1316 if (s->dflag == MO_64) { 1317 tcg_gen_bswap64_i64(s->T0, s->T0); 1318 return; 1319 } 1320#endif 1321 tcg_gen_bswap32_tl(s->T0, s->T0, TCG_BSWAP_OZ); 1322} 1323 1324static void gen_BZHI(DisasContext *s, CPUX86State *env, X86DecodedInsn *decode) 1325{ 1326 MemOp ot = decode->op[0].ot; 1327 TCGv bound = tcg_constant_tl(ot == MO_64 ? 63 : 31); 1328 TCGv zero = tcg_constant_tl(0); 1329 TCGv mone = tcg_constant_tl(-1); 1330 1331 tcg_gen_ext8u_tl(s->T1, s->T1); 1332 1333 tcg_gen_shl_tl(s->A0, mone, s->T1); 1334 tcg_gen_movcond_tl(TCG_COND_LEU, s->A0, s->T1, bound, s->A0, zero); 1335 tcg_gen_andc_tl(s->T0, s->T0, s->A0); 1336 /* 1337 * Note that since we're using BMILG (in order to get O 1338 * cleared) we need to store the inverse into C. 1339 */ 1340 tcg_gen_setcond_tl(TCG_COND_LEU, s->T1, s->T1, bound); 1341 prepare_update2_cc(decode, s, CC_OP_BMILGB + ot); 1342} 1343 1344static void gen_CALL(DisasContext *s, CPUX86State *env, X86DecodedInsn *decode) 1345{ 1346 gen_push_v(s, eip_next_tl(s)); 1347 gen_JMP(s, env, decode); 1348} 1349 1350static void gen_CALL_m(DisasContext *s, CPUX86State *env, X86DecodedInsn *decode) 1351{ 1352 gen_push_v(s, eip_next_tl(s)); 1353 gen_JMP_m(s, env, decode); 1354} 1355 1356static void gen_CALLF(DisasContext *s, CPUX86State *env, X86DecodedInsn *decode) 1357{ 1358 gen_far_call(s); 1359} 1360 1361static void gen_CALLF_m(DisasContext *s, CPUX86State *env, X86DecodedInsn *decode) 1362{ 1363 MemOp ot = decode->op[2].ot; 1364 1365 gen_op_ld_v(s, ot, s->T0, s->A0); 1366 gen_add_A0_im(s, 1 << ot); 1367 gen_op_ld_v(s, MO_16, s->T1, s->A0); 1368 gen_far_call(s); 1369} 1370 1371static void gen_CBW(DisasContext *s, CPUX86State *env, X86DecodedInsn *decode) 1372{ 1373 MemOp src_ot = decode->op[0].ot - 1; 1374 1375 tcg_gen_ext_tl(s->T0, s->T0, src_ot | MO_SIGN); 1376} 1377 1378static void gen_CLC(DisasContext *s, CPUX86State *env, X86DecodedInsn *decode) 1379{ 1380 gen_compute_eflags(s); 1381 tcg_gen_andi_tl(cpu_cc_src, cpu_cc_src, ~CC_C); 1382} 1383 1384static void gen_CLD(DisasContext *s, CPUX86State *env, X86DecodedInsn *decode) 1385{ 1386 tcg_gen_st_i32(tcg_constant_i32(1), tcg_env, offsetof(CPUX86State, df)); 1387} 1388 1389static void gen_CLI(DisasContext *s, CPUX86State *env, X86DecodedInsn *decode) 1390{ 1391 gen_reset_eflags(s, IF_MASK); 1392} 1393 1394static void gen_CMC(DisasContext *s, CPUX86State *env, X86DecodedInsn *decode) 1395{ 1396 gen_compute_eflags(s); 1397 tcg_gen_xori_tl(cpu_cc_src, cpu_cc_src, CC_C); 1398} 1399 1400static void gen_CMOVcc(DisasContext *s, CPUX86State *env, X86DecodedInsn *decode) 1401{ 1402 gen_cmovcc1(s, decode->b & 0xf, s->T0, s->T1); 1403} 1404 1405static void gen_CMPccXADD(DisasContext *s, CPUX86State *env, X86DecodedInsn *decode) 1406{ 1407 TCGLabel *label_top = gen_new_label(); 1408 TCGLabel *label_bottom = gen_new_label(); 1409 TCGv oldv = tcg_temp_new(); 1410 TCGv newv = tcg_temp_new(); 1411 TCGv cmpv = tcg_temp_new(); 1412 TCGCond cond; 1413 1414 TCGv cmp_lhs, cmp_rhs; 1415 MemOp ot, ot_full; 1416 1417 int jcc_op = (decode->b >> 1) & 7; 1418 static const TCGCond cond_table[8] = { 1419 [JCC_O] = TCG_COND_LT, /* test sign bit by comparing against 0 */ 1420 [JCC_B] = TCG_COND_LTU, 1421 [JCC_Z] = TCG_COND_EQ, 1422 [JCC_BE] = TCG_COND_LEU, 1423 [JCC_S] = TCG_COND_LT, /* test sign bit by comparing against 0 */ 1424 [JCC_P] = TCG_COND_TSTEQ, /* even parity - tests low bit of popcount */ 1425 [JCC_L] = TCG_COND_LT, 1426 [JCC_LE] = TCG_COND_LE, 1427 }; 1428 1429 cond = cond_table[jcc_op]; 1430 if (decode->b & 1) { 1431 cond = tcg_invert_cond(cond); 1432 } 1433 1434 ot = decode->op[0].ot; 1435 ot_full = ot | MO_LE; 1436 if (jcc_op >= JCC_S) { 1437 /* 1438 * Sign-extend values before subtracting for S, P (zero/sign extension 1439 * does not matter there) L, LE and their inverses. 1440 */ 1441 ot_full |= MO_SIGN; 1442 } 1443 1444 /* 1445 * cmpv will be moved to cc_src *after* cpu_regs[] is written back, so use 1446 * tcg_gen_ext_tl instead of gen_ext_tl. 1447 */ 1448 tcg_gen_ext_tl(cmpv, cpu_regs[decode->op[1].n], ot_full); 1449 1450 /* 1451 * Cmpxchg loop starts here. 1452 * - s->T1: addition operand (from decoder) 1453 * - s->A0: dest address (from decoder) 1454 * - s->cc_srcT: memory operand (lhs for comparison) 1455 * - cmpv: rhs for comparison 1456 */ 1457 gen_set_label(label_top); 1458 gen_op_ld_v(s, ot_full, s->cc_srcT, s->A0); 1459 tcg_gen_sub_tl(s->T0, s->cc_srcT, cmpv); 1460 1461 /* Compute the comparison result by hand, to avoid clobbering cc_*. */ 1462 switch (jcc_op) { 1463 case JCC_O: 1464 /* (src1 ^ src2) & (src1 ^ dst). newv is only used here for a moment */ 1465 tcg_gen_xor_tl(newv, s->cc_srcT, s->T0); 1466 tcg_gen_xor_tl(s->tmp0, s->cc_srcT, cmpv); 1467 tcg_gen_and_tl(s->tmp0, s->tmp0, newv); 1468 tcg_gen_sextract_tl(s->tmp0, s->tmp0, 0, 8 << ot); 1469 cmp_lhs = s->tmp0, cmp_rhs = tcg_constant_tl(0); 1470 break; 1471 1472 case JCC_P: 1473 tcg_gen_ext8u_tl(s->tmp0, s->T0); 1474 tcg_gen_ctpop_tl(s->tmp0, s->tmp0); 1475 cmp_lhs = s->tmp0, cmp_rhs = tcg_constant_tl(1); 1476 break; 1477 1478 case JCC_S: 1479 tcg_gen_sextract_tl(s->tmp0, s->T0, 0, 8 << ot); 1480 cmp_lhs = s->tmp0, cmp_rhs = tcg_constant_tl(0); 1481 break; 1482 1483 default: 1484 cmp_lhs = s->cc_srcT, cmp_rhs = cmpv; 1485 break; 1486 } 1487 1488 /* Compute new value: if condition does not hold, just store back s->cc_srcT */ 1489 tcg_gen_add_tl(newv, s->cc_srcT, s->T1); 1490 tcg_gen_movcond_tl(cond, newv, cmp_lhs, cmp_rhs, newv, s->cc_srcT); 1491 tcg_gen_atomic_cmpxchg_tl(oldv, s->A0, s->cc_srcT, newv, s->mem_index, ot_full); 1492 1493 /* Exit unconditionally if cmpxchg succeeded. */ 1494 tcg_gen_brcond_tl(TCG_COND_EQ, oldv, s->cc_srcT, label_bottom); 1495 1496 /* Try again if there was actually a store to make. */ 1497 tcg_gen_brcond_tl(cond, cmp_lhs, cmp_rhs, label_top); 1498 gen_set_label(label_bottom); 1499 1500 /* Store old value to registers only after a successful store. */ 1501 gen_writeback(s, decode, 1, s->cc_srcT); 1502 1503 decode->cc_dst = s->T0; 1504 decode->cc_src = cmpv; 1505 decode->cc_op = CC_OP_SUBB + ot; 1506} 1507 1508static void gen_CMPS(DisasContext *s, CPUX86State *env, X86DecodedInsn *decode) 1509{ 1510 MemOp ot = decode->op[2].ot; 1511 if (s->prefix & (PREFIX_REPZ | PREFIX_REPNZ)) { 1512 gen_repz_nz(s, ot, gen_cmps); 1513 } else { 1514 gen_cmps(s, ot); 1515 } 1516} 1517 1518static void gen_CRC32(DisasContext *s, CPUX86State *env, X86DecodedInsn *decode) 1519{ 1520 MemOp ot = decode->op[2].ot; 1521 1522 tcg_gen_trunc_tl_i32(s->tmp2_i32, s->T0); 1523 gen_helper_crc32(s->T0, s->tmp2_i32, s->T1, tcg_constant_i32(8 << ot)); 1524} 1525 1526static void gen_CVTPI2Px(DisasContext *s, CPUX86State *env, X86DecodedInsn *decode) 1527{ 1528 gen_helper_enter_mmx(tcg_env); 1529 if (s->prefix & PREFIX_DATA) { 1530 gen_helper_cvtpi2pd(tcg_env, OP_PTR0, OP_PTR2); 1531 } else { 1532 gen_helper_cvtpi2ps(tcg_env, OP_PTR0, OP_PTR2); 1533 } 1534} 1535 1536static void gen_CVTPx2PI(DisasContext *s, CPUX86State *env, X86DecodedInsn *decode) 1537{ 1538 gen_helper_enter_mmx(tcg_env); 1539 if (s->prefix & PREFIX_DATA) { 1540 gen_helper_cvtpd2pi(tcg_env, OP_PTR0, OP_PTR2); 1541 } else { 1542 gen_helper_cvtps2pi(tcg_env, OP_PTR0, OP_PTR2); 1543 } 1544} 1545 1546static void gen_CVTTPx2PI(DisasContext *s, CPUX86State *env, X86DecodedInsn *decode) 1547{ 1548 gen_helper_enter_mmx(tcg_env); 1549 if (s->prefix & PREFIX_DATA) { 1550 gen_helper_cvttpd2pi(tcg_env, OP_PTR0, OP_PTR2); 1551 } else { 1552 gen_helper_cvttps2pi(tcg_env, OP_PTR0, OP_PTR2); 1553 } 1554} 1555 1556static void gen_CWD(DisasContext *s, CPUX86State *env, X86DecodedInsn *decode) 1557{ 1558 int shift = 8 << decode->op[0].ot; 1559 1560 tcg_gen_sextract_tl(s->T0, s->T0, shift - 1, 1); 1561} 1562 1563static void gen_DAA(DisasContext *s, CPUX86State *env, X86DecodedInsn *decode) 1564{ 1565 gen_update_cc_op(s); 1566 gen_helper_daa(tcg_env); 1567 assume_cc_op(s, CC_OP_EFLAGS); 1568} 1569 1570static void gen_DAS(DisasContext *s, CPUX86State *env, X86DecodedInsn *decode) 1571{ 1572 gen_update_cc_op(s); 1573 gen_helper_das(tcg_env); 1574 assume_cc_op(s, CC_OP_EFLAGS); 1575} 1576 1577static void gen_DEC(DisasContext *s, CPUX86State *env, X86DecodedInsn *decode) 1578{ 1579 MemOp ot = decode->op[1].ot; 1580 1581 tcg_gen_movi_tl(s->T1, -1); 1582 if (s->prefix & PREFIX_LOCK) { 1583 tcg_gen_atomic_add_fetch_tl(s->T0, s->A0, s->T1, 1584 s->mem_index, ot | MO_LE); 1585 } else { 1586 tcg_gen_add_tl(s->T0, s->T0, s->T1); 1587 } 1588 prepare_update_cc_incdec(decode, s, CC_OP_DECB + ot); 1589} 1590 1591static void gen_DIV(DisasContext *s, CPUX86State *env, X86DecodedInsn *decode) 1592{ 1593 MemOp ot = decode->op[2].ot; 1594 1595 switch(ot) { 1596 case MO_8: 1597 gen_helper_divb_AL(tcg_env, s->T1); 1598 break; 1599 case MO_16: 1600 gen_helper_divw_AX(tcg_env, s->T1); 1601 break; 1602 default: 1603 case MO_32: 1604 gen_helper_divl_EAX(tcg_env, s->T1); 1605 break; 1606#ifdef TARGET_X86_64 1607 case MO_64: 1608 gen_helper_divq_EAX(tcg_env, s->T1); 1609 break; 1610#endif 1611 } 1612} 1613 1614static void gen_EMMS(DisasContext *s, CPUX86State *env, X86DecodedInsn *decode) 1615{ 1616 gen_helper_emms(tcg_env); 1617} 1618 1619static void gen_ENTER(DisasContext *s, CPUX86State *env, X86DecodedInsn *decode) 1620{ 1621 gen_enter(s, decode->op[1].imm, decode->op[2].imm); 1622} 1623 1624static void gen_EXTRQ_i(DisasContext *s, CPUX86State *env, X86DecodedInsn *decode) 1625{ 1626 TCGv_i32 length = tcg_constant_i32(decode->immediate & 63); 1627 TCGv_i32 index = tcg_constant_i32((decode->immediate >> 8) & 63); 1628 1629 gen_helper_extrq_i(tcg_env, OP_PTR0, index, length); 1630} 1631 1632static void gen_EXTRQ_r(DisasContext *s, CPUX86State *env, X86DecodedInsn *decode) 1633{ 1634 gen_helper_extrq_r(tcg_env, OP_PTR0, OP_PTR2); 1635} 1636 1637static void gen_HLT(DisasContext *s, CPUX86State *env, X86DecodedInsn *decode) 1638{ 1639#ifdef CONFIG_SYSTEM_ONLY 1640 gen_update_cc_op(s); 1641 gen_update_eip_cur(s); 1642 gen_helper_hlt(tcg_env, cur_insn_len_i32(s)); 1643 s->base.is_jmp = DISAS_NORETURN; 1644#endif 1645} 1646 1647static void gen_IDIV(DisasContext *s, CPUX86State *env, X86DecodedInsn *decode) 1648{ 1649 MemOp ot = decode->op[2].ot; 1650 1651 switch(ot) { 1652 case MO_8: 1653 gen_helper_idivb_AL(tcg_env, s->T1); 1654 break; 1655 case MO_16: 1656 gen_helper_idivw_AX(tcg_env, s->T1); 1657 break; 1658 default: 1659 case MO_32: 1660 gen_helper_idivl_EAX(tcg_env, s->T1); 1661 break; 1662#ifdef TARGET_X86_64 1663 case MO_64: 1664 gen_helper_idivq_EAX(tcg_env, s->T1); 1665 break; 1666#endif 1667 } 1668} 1669 1670static void gen_IMUL3(DisasContext *s, CPUX86State *env, X86DecodedInsn *decode) 1671{ 1672 MemOp ot = decode->op[0].ot; 1673 TCGv cc_src_rhs; 1674 1675 switch (ot) { 1676 case MO_16: 1677 /* s->T0 already sign-extended */ 1678 tcg_gen_ext16s_tl(s->T1, s->T1); 1679 tcg_gen_mul_tl(s->T0, s->T0, s->T1); 1680 /* Compare the full result to the extension of the truncated result. */ 1681 tcg_gen_ext16s_tl(s->T1, s->T0); 1682 cc_src_rhs = s->T0; 1683 break; 1684 1685 case MO_32: 1686#ifdef TARGET_X86_64 1687 if (TCG_TARGET_REG_BITS == 64) { 1688 /* 1689 * This produces fewer TCG ops, and better code if flags are needed, 1690 * but it requires a 64-bit multiply even if they are not. Use it 1691 * only if the target has 64-bits registers. 1692 * 1693 * s->T0 is already sign-extended. 1694 */ 1695 tcg_gen_ext32s_tl(s->T1, s->T1); 1696 tcg_gen_mul_tl(s->T0, s->T0, s->T1); 1697 /* Compare the full result to the extension of the truncated result. */ 1698 tcg_gen_ext32s_tl(s->T1, s->T0); 1699 cc_src_rhs = s->T0; 1700 } else { 1701 /* Variant that only needs a 32-bit widening multiply. */ 1702 TCGv_i32 hi = tcg_temp_new_i32(); 1703 TCGv_i32 lo = tcg_temp_new_i32(); 1704 tcg_gen_trunc_tl_i32(lo, s->T0); 1705 tcg_gen_trunc_tl_i32(hi, s->T1); 1706 tcg_gen_muls2_i32(lo, hi, lo, hi); 1707 tcg_gen_extu_i32_tl(s->T0, lo); 1708 1709 cc_src_rhs = tcg_temp_new(); 1710 tcg_gen_extu_i32_tl(cc_src_rhs, hi); 1711 /* Compare the high part to the sign bit of the truncated result */ 1712 tcg_gen_sari_i32(lo, lo, 31); 1713 tcg_gen_extu_i32_tl(s->T1, lo); 1714 } 1715 break; 1716 1717 case MO_64: 1718#endif 1719 cc_src_rhs = tcg_temp_new(); 1720 tcg_gen_muls2_tl(s->T0, cc_src_rhs, s->T0, s->T1); 1721 /* Compare the high part to the sign bit of the truncated result */ 1722 tcg_gen_sari_tl(s->T1, s->T0, TARGET_LONG_BITS - 1); 1723 break; 1724 1725 default: 1726 g_assert_not_reached(); 1727 } 1728 1729 tcg_gen_sub_tl(s->T1, s->T1, cc_src_rhs); 1730 prepare_update2_cc(decode, s, CC_OP_MULB + ot); 1731} 1732 1733static void gen_IMUL(DisasContext *s, CPUX86State *env, X86DecodedInsn *decode) 1734{ 1735 MemOp ot = decode->op[1].ot; 1736 TCGv cc_src_rhs; 1737 1738 switch (ot) { 1739 case MO_8: 1740 /* s->T0 already sign-extended */ 1741 tcg_gen_ext8s_tl(s->T1, s->T1); 1742 tcg_gen_mul_tl(s->T0, s->T0, s->T1); 1743 gen_op_mov_reg_v(s, MO_16, R_EAX, s->T0); 1744 /* Compare the full result to the extension of the truncated result. */ 1745 tcg_gen_ext8s_tl(s->T1, s->T0); 1746 cc_src_rhs = s->T0; 1747 break; 1748 1749 case MO_16: 1750 /* s->T0 already sign-extended */ 1751 tcg_gen_ext16s_tl(s->T1, s->T1); 1752 tcg_gen_mul_tl(s->T0, s->T0, s->T1); 1753 gen_op_mov_reg_v(s, MO_16, R_EAX, s->T0); 1754 tcg_gen_shri_tl(s->T1, s->T0, 16); 1755 gen_op_mov_reg_v(s, MO_16, R_EDX, s->T1); 1756 /* Compare the full result to the extension of the truncated result. */ 1757 tcg_gen_ext16s_tl(s->T1, s->T0); 1758 cc_src_rhs = s->T0; 1759 break; 1760 1761 case MO_32: 1762#ifdef TARGET_X86_64 1763 /* s->T0 already sign-extended */ 1764 tcg_gen_ext32s_tl(s->T1, s->T1); 1765 tcg_gen_mul_tl(s->T0, s->T0, s->T1); 1766 tcg_gen_ext32u_tl(cpu_regs[R_EAX], s->T0); 1767 tcg_gen_shri_tl(cpu_regs[R_EDX], s->T0, 32); 1768 /* Compare the full result to the extension of the truncated result. */ 1769 tcg_gen_ext32s_tl(s->T1, s->T0); 1770 cc_src_rhs = s->T0; 1771 break; 1772 1773 case MO_64: 1774#endif 1775 tcg_gen_muls2_tl(s->T0, cpu_regs[R_EDX], s->T0, s->T1); 1776 tcg_gen_mov_tl(cpu_regs[R_EAX], s->T0); 1777 1778 /* Compare the high part to the sign bit of the truncated result */ 1779 tcg_gen_negsetcondi_tl(TCG_COND_LT, s->T1, s->T0, 0); 1780 cc_src_rhs = cpu_regs[R_EDX]; 1781 break; 1782 1783 default: 1784 g_assert_not_reached(); 1785 } 1786 1787 tcg_gen_sub_tl(s->T1, s->T1, cc_src_rhs); 1788 prepare_update2_cc(decode, s, CC_OP_MULB + ot); 1789} 1790 1791static void gen_IN(DisasContext *s, CPUX86State *env, X86DecodedInsn *decode) 1792{ 1793 MemOp ot = decode->op[0].ot; 1794 TCGv_i32 port = tcg_temp_new_i32(); 1795 1796 tcg_gen_trunc_tl_i32(port, s->T1); 1797 tcg_gen_ext16u_i32(port, port); 1798 if (!gen_check_io(s, ot, port, SVM_IOIO_TYPE_MASK)) { 1799 return; 1800 } 1801 translator_io_start(&s->base); 1802 gen_helper_in_func(ot, s->T0, port); 1803 gen_writeback(s, decode, 0, s->T0); 1804 gen_bpt_io(s, port, ot); 1805} 1806 1807static void gen_INC(DisasContext *s, CPUX86State *env, X86DecodedInsn *decode) 1808{ 1809 MemOp ot = decode->op[1].ot; 1810 1811 tcg_gen_movi_tl(s->T1, 1); 1812 if (s->prefix & PREFIX_LOCK) { 1813 tcg_gen_atomic_add_fetch_tl(s->T0, s->A0, s->T1, 1814 s->mem_index, ot | MO_LE); 1815 } else { 1816 tcg_gen_add_tl(s->T0, s->T0, s->T1); 1817 } 1818 prepare_update_cc_incdec(decode, s, CC_OP_INCB + ot); 1819} 1820 1821static void gen_INS(DisasContext *s, CPUX86State *env, X86DecodedInsn *decode) 1822{ 1823 MemOp ot = decode->op[1].ot; 1824 TCGv_i32 port = tcg_temp_new_i32(); 1825 1826 tcg_gen_trunc_tl_i32(port, s->T1); 1827 tcg_gen_ext16u_i32(port, port); 1828 if (!gen_check_io(s, ot, port, 1829 SVM_IOIO_TYPE_MASK | SVM_IOIO_STR_MASK)) { 1830 return; 1831 } 1832 1833 translator_io_start(&s->base); 1834 if (s->prefix & (PREFIX_REPZ | PREFIX_REPNZ)) { 1835 gen_repz(s, ot, gen_ins); 1836 } else { 1837 gen_ins(s, ot); 1838 } 1839} 1840 1841static void gen_INSERTQ_i(DisasContext *s, CPUX86State *env, X86DecodedInsn *decode) 1842{ 1843 TCGv_i32 length = tcg_constant_i32(decode->immediate & 63); 1844 TCGv_i32 index = tcg_constant_i32((decode->immediate >> 8) & 63); 1845 1846 gen_helper_insertq_i(tcg_env, OP_PTR0, OP_PTR1, index, length); 1847} 1848 1849static void gen_INSERTQ_r(DisasContext *s, CPUX86State *env, X86DecodedInsn *decode) 1850{ 1851 gen_helper_insertq_r(tcg_env, OP_PTR0, OP_PTR2); 1852} 1853 1854static void gen_INT(DisasContext *s, CPUX86State *env, X86DecodedInsn *decode) 1855{ 1856 gen_interrupt(s, decode->immediate); 1857} 1858 1859static void gen_INT1(DisasContext *s, CPUX86State *env, X86DecodedInsn *decode) 1860{ 1861 gen_exception(s, EXCP01_DB); 1862} 1863 1864static void gen_INT3(DisasContext *s, CPUX86State *env, X86DecodedInsn *decode) 1865{ 1866 gen_interrupt(s, EXCP03_INT3); 1867} 1868 1869static void gen_INTO(DisasContext *s, CPUX86State *env, X86DecodedInsn *decode) 1870{ 1871 gen_update_cc_op(s); 1872 gen_update_eip_cur(s); 1873 gen_helper_into(tcg_env, cur_insn_len_i32(s)); 1874} 1875 1876static void gen_IRET(DisasContext *s, CPUX86State *env, X86DecodedInsn *decode) 1877{ 1878 if (!PE(s) || VM86(s)) { 1879 gen_helper_iret_real(tcg_env, tcg_constant_i32(s->dflag - 1)); 1880 } else { 1881 gen_helper_iret_protected(tcg_env, tcg_constant_i32(s->dflag - 1), 1882 eip_next_i32(s)); 1883 } 1884 assume_cc_op(s, CC_OP_EFLAGS); 1885 s->base.is_jmp = DISAS_EOB_ONLY; 1886} 1887 1888static void gen_Jcc(DisasContext *s, CPUX86State *env, X86DecodedInsn *decode) 1889{ 1890 gen_bnd_jmp(s); 1891 gen_jcc(s, decode->b & 0xf, decode->immediate); 1892} 1893 1894static void gen_JCXZ(DisasContext *s, CPUX86State *env, X86DecodedInsn *decode) 1895{ 1896 TCGLabel *taken = gen_new_label(); 1897 1898 gen_update_cc_op(s); 1899 gen_op_jz_ecx(s, taken); 1900 gen_conditional_jump_labels(s, decode->immediate, NULL, taken); 1901} 1902 1903static void gen_JMP(DisasContext *s, CPUX86State *env, X86DecodedInsn *decode) 1904{ 1905 gen_update_cc_op(s); 1906 gen_jmp_rel(s, s->dflag, decode->immediate, 0); 1907} 1908 1909static void gen_JMP_m(DisasContext *s, CPUX86State *env, X86DecodedInsn *decode) 1910{ 1911 gen_op_jmp_v(s, s->T0); 1912 gen_bnd_jmp(s); 1913 s->base.is_jmp = DISAS_JUMP; 1914} 1915 1916static void gen_JMPF(DisasContext *s, CPUX86State *env, X86DecodedInsn *decode) 1917{ 1918 gen_far_jmp(s); 1919} 1920 1921static void gen_JMPF_m(DisasContext *s, CPUX86State *env, X86DecodedInsn *decode) 1922{ 1923 MemOp ot = decode->op[2].ot; 1924 1925 gen_op_ld_v(s, ot, s->T0, s->A0); 1926 gen_add_A0_im(s, 1 << ot); 1927 gen_op_ld_v(s, MO_16, s->T1, s->A0); 1928 gen_far_jmp(s); 1929} 1930 1931static void gen_LAHF(DisasContext *s, CPUX86State *env, X86DecodedInsn *decode) 1932{ 1933 if (CODE64(s) && !(s->cpuid_ext3_features & CPUID_EXT3_LAHF_LM)) { 1934 return gen_illegal_opcode(s); 1935 } 1936 gen_compute_eflags(s); 1937 /* Note: gen_compute_eflags() only gives the condition codes */ 1938 tcg_gen_ori_tl(s->T0, cpu_cc_src, 0x02); 1939 tcg_gen_deposit_tl(cpu_regs[R_EAX], cpu_regs[R_EAX], s->T0, 8, 8); 1940} 1941 1942static void gen_LDMXCSR(DisasContext *s, CPUX86State *env, X86DecodedInsn *decode) 1943{ 1944 tcg_gen_trunc_tl_i32(s->tmp2_i32, s->T1); 1945 gen_helper_ldmxcsr(tcg_env, s->tmp2_i32); 1946} 1947 1948static void gen_lxx_seg(DisasContext *s, CPUX86State *env, X86DecodedInsn *decode, int seg) 1949{ 1950 MemOp ot = decode->op[0].ot; 1951 1952 /* Offset already in s->T0. */ 1953 gen_add_A0_im(s, 1 << ot); 1954 gen_op_ld_v(s, MO_16, s->T1, s->A0); 1955 1956 /* load the segment here to handle exceptions properly */ 1957 gen_movl_seg(s, seg, s->T1); 1958} 1959 1960static void gen_LDS(DisasContext *s, CPUX86State *env, X86DecodedInsn *decode) 1961{ 1962 gen_lxx_seg(s, env, decode, R_DS); 1963} 1964 1965static void gen_LEA(DisasContext *s, CPUX86State *env, X86DecodedInsn *decode) 1966{ 1967 tcg_gen_mov_tl(s->T0, s->A0); 1968} 1969 1970static void gen_LEAVE(DisasContext *s, CPUX86State *env, X86DecodedInsn *decode) 1971{ 1972 gen_leave(s); 1973} 1974 1975static void gen_LES(DisasContext *s, CPUX86State *env, X86DecodedInsn *decode) 1976{ 1977 gen_lxx_seg(s, env, decode, R_ES); 1978} 1979 1980static void gen_LFS(DisasContext *s, CPUX86State *env, X86DecodedInsn *decode) 1981{ 1982 gen_lxx_seg(s, env, decode, R_FS); 1983} 1984 1985static void gen_LGS(DisasContext *s, CPUX86State *env, X86DecodedInsn *decode) 1986{ 1987 gen_lxx_seg(s, env, decode, R_GS); 1988} 1989 1990static void gen_LODS(DisasContext *s, CPUX86State *env, X86DecodedInsn *decode) 1991{ 1992 MemOp ot = decode->op[2].ot; 1993 if (s->prefix & (PREFIX_REPZ | PREFIX_REPNZ)) { 1994 gen_repz(s, ot, gen_lods); 1995 } else { 1996 gen_lods(s, ot); 1997 } 1998} 1999 2000static void gen_LOOP(DisasContext *s, CPUX86State *env, X86DecodedInsn *decode) 2001{ 2002 TCGLabel *taken = gen_new_label(); 2003 2004 gen_update_cc_op(s); 2005 gen_op_add_reg_im(s, s->aflag, R_ECX, -1); 2006 gen_op_jnz_ecx(s, taken); 2007 gen_conditional_jump_labels(s, decode->immediate, NULL, taken); 2008} 2009 2010static void gen_LOOPE(DisasContext *s, CPUX86State *env, X86DecodedInsn *decode) 2011{ 2012 TCGLabel *taken = gen_new_label(); 2013 TCGLabel *not_taken = gen_new_label(); 2014 2015 gen_update_cc_op(s); 2016 gen_op_add_reg_im(s, s->aflag, R_ECX, -1); 2017 gen_op_jz_ecx(s, not_taken); 2018 gen_jcc1(s, (JCC_Z << 1), taken); /* jz taken */ 2019 gen_conditional_jump_labels(s, decode->immediate, not_taken, taken); 2020} 2021 2022static void gen_LOOPNE(DisasContext *s, CPUX86State *env, X86DecodedInsn *decode) 2023{ 2024 TCGLabel *taken = gen_new_label(); 2025 TCGLabel *not_taken = gen_new_label(); 2026 2027 gen_update_cc_op(s); 2028 gen_op_add_reg_im(s, s->aflag, R_ECX, -1); 2029 gen_op_jz_ecx(s, not_taken); 2030 gen_jcc1(s, (JCC_Z << 1) | 1, taken); /* jnz taken */ 2031 gen_conditional_jump_labels(s, decode->immediate, not_taken, taken); 2032} 2033 2034static void gen_LSS(DisasContext *s, CPUX86State *env, X86DecodedInsn *decode) 2035{ 2036 gen_lxx_seg(s, env, decode, R_SS); 2037} 2038 2039static void gen_MOV(DisasContext *s, CPUX86State *env, X86DecodedInsn *decode) 2040{ 2041 /* nothing to do! */ 2042} 2043#define gen_NOP gen_MOV 2044 2045static void gen_MASKMOV(DisasContext *s, CPUX86State *env, X86DecodedInsn *decode) 2046{ 2047 gen_lea_v_seg(s, cpu_regs[R_EDI], R_DS, s->override); 2048 2049 if (s->prefix & PREFIX_DATA) { 2050 gen_helper_maskmov_xmm(tcg_env, OP_PTR1, OP_PTR2, s->A0); 2051 } else { 2052 gen_helper_maskmov_mmx(tcg_env, OP_PTR1, OP_PTR2, s->A0); 2053 } 2054} 2055 2056static void gen_MOVBE(DisasContext *s, CPUX86State *env, X86DecodedInsn *decode) 2057{ 2058 MemOp ot = decode->op[0].ot; 2059 2060 /* M operand type does not load/store */ 2061 if (decode->e.op0 == X86_TYPE_M) { 2062 tcg_gen_qemu_st_tl(s->T0, s->A0, s->mem_index, ot | MO_BE); 2063 } else { 2064 tcg_gen_qemu_ld_tl(s->T0, s->A0, s->mem_index, ot | MO_BE); 2065 } 2066} 2067 2068static void gen_MOVD_from(DisasContext *s, CPUX86State *env, X86DecodedInsn *decode) 2069{ 2070 MemOp ot = decode->op[2].ot; 2071 2072 switch (ot) { 2073 case MO_32: 2074#ifdef TARGET_X86_64 2075 tcg_gen_ld32u_tl(s->T0, tcg_env, decode->op[2].offset); 2076 break; 2077 case MO_64: 2078#endif 2079 tcg_gen_ld_tl(s->T0, tcg_env, decode->op[2].offset); 2080 break; 2081 default: 2082 abort(); 2083 } 2084} 2085 2086static void gen_MOVD_to(DisasContext *s, CPUX86State *env, X86DecodedInsn *decode) 2087{ 2088 MemOp ot = decode->op[2].ot; 2089 int vec_len = vector_len(s, decode); 2090 int lo_ofs = vector_elem_offset(&decode->op[0], ot, 0); 2091 2092 tcg_gen_gvec_dup_imm(MO_64, decode->op[0].offset, vec_len, vec_len, 0); 2093 2094 switch (ot) { 2095 case MO_32: 2096#ifdef TARGET_X86_64 2097 tcg_gen_st32_tl(s->T1, tcg_env, lo_ofs); 2098 break; 2099 case MO_64: 2100#endif 2101 tcg_gen_st_tl(s->T1, tcg_env, lo_ofs); 2102 break; 2103 default: 2104 g_assert_not_reached(); 2105 } 2106} 2107 2108static void gen_MOVDQ(DisasContext *s, CPUX86State *env, X86DecodedInsn *decode) 2109{ 2110 gen_store_sse(s, decode, decode->op[2].offset); 2111} 2112 2113static void gen_MOVMSK(DisasContext *s, CPUX86State *env, X86DecodedInsn *decode) 2114{ 2115 typeof(gen_helper_movmskps_ymm) *ps, *pd, *fn; 2116 ps = s->vex_l ? gen_helper_movmskps_ymm : gen_helper_movmskps_xmm; 2117 pd = s->vex_l ? gen_helper_movmskpd_ymm : gen_helper_movmskpd_xmm; 2118 fn = s->prefix & PREFIX_DATA ? pd : ps; 2119 fn(s->tmp2_i32, tcg_env, OP_PTR2); 2120 tcg_gen_extu_i32_tl(s->T0, s->tmp2_i32); 2121} 2122 2123static void gen_MOVQ(DisasContext *s, CPUX86State *env, X86DecodedInsn *decode) 2124{ 2125 int vec_len = vector_len(s, decode); 2126 int lo_ofs = vector_elem_offset(&decode->op[0], MO_64, 0); 2127 2128 tcg_gen_ld_i64(s->tmp1_i64, tcg_env, decode->op[2].offset); 2129 if (decode->op[0].has_ea) { 2130 tcg_gen_qemu_st_i64(s->tmp1_i64, s->A0, s->mem_index, MO_LEUQ); 2131 } else { 2132 /* 2133 * tcg_gen_gvec_dup_i64(MO_64, op0.offset, 8, vec_len, s->tmp1_64) would 2134 * seem to work, but it does not on big-endian platforms; the cleared parts 2135 * are always at higher addresses, but cross-endian emulation inverts the 2136 * byte order so that the cleared parts need to be at *lower* addresses. 2137 * Because oprsz is 8, we see this here even for SSE; but more in general, 2138 * it disqualifies using oprsz < maxsz to emulate VEX128. 2139 */ 2140 tcg_gen_gvec_dup_imm(MO_64, decode->op[0].offset, vec_len, vec_len, 0); 2141 tcg_gen_st_i64(s->tmp1_i64, tcg_env, lo_ofs); 2142 } 2143} 2144 2145static void gen_MOVq_dq(DisasContext *s, CPUX86State *env, X86DecodedInsn *decode) 2146{ 2147 gen_helper_enter_mmx(tcg_env); 2148 /* Otherwise the same as any other movq. */ 2149 return gen_MOVQ(s, env, decode); 2150} 2151 2152static void gen_MOVS(DisasContext *s, CPUX86State *env, X86DecodedInsn *decode) 2153{ 2154 MemOp ot = decode->op[2].ot; 2155 if (s->prefix & (PREFIX_REPZ | PREFIX_REPNZ)) { 2156 gen_repz(s, ot, gen_movs); 2157 } else { 2158 gen_movs(s, ot); 2159 } 2160} 2161 2162static void gen_MUL(DisasContext *s, CPUX86State *env, X86DecodedInsn *decode) 2163{ 2164 MemOp ot = decode->op[1].ot; 2165 2166 switch (ot) { 2167 case MO_8: 2168 /* s->T0 already zero-extended */ 2169 tcg_gen_ext8u_tl(s->T1, s->T1); 2170 tcg_gen_mul_tl(s->T0, s->T0, s->T1); 2171 gen_op_mov_reg_v(s, MO_16, R_EAX, s->T0); 2172 tcg_gen_andi_tl(s->T1, s->T0, 0xff00); 2173 decode->cc_dst = s->T0; 2174 decode->cc_src = s->T1; 2175 break; 2176 2177 case MO_16: 2178 /* s->T0 already zero-extended */ 2179 tcg_gen_ext16u_tl(s->T1, s->T1); 2180 tcg_gen_mul_tl(s->T0, s->T0, s->T1); 2181 gen_op_mov_reg_v(s, MO_16, R_EAX, s->T0); 2182 tcg_gen_shri_tl(s->T1, s->T0, 16); 2183 gen_op_mov_reg_v(s, MO_16, R_EDX, s->T1); 2184 decode->cc_dst = s->T0; 2185 decode->cc_src = s->T1; 2186 break; 2187 2188 case MO_32: 2189#ifdef TARGET_X86_64 2190 /* s->T0 already zero-extended */ 2191 tcg_gen_ext32u_tl(s->T1, s->T1); 2192 tcg_gen_mul_tl(s->T0, s->T0, s->T1); 2193 tcg_gen_ext32u_tl(cpu_regs[R_EAX], s->T0); 2194 tcg_gen_shri_tl(cpu_regs[R_EDX], s->T0, 32); 2195 decode->cc_dst = cpu_regs[R_EAX]; 2196 decode->cc_src = cpu_regs[R_EDX]; 2197 break; 2198 2199 case MO_64: 2200#endif 2201 tcg_gen_mulu2_tl(cpu_regs[R_EAX], cpu_regs[R_EDX], s->T0, s->T1); 2202 decode->cc_dst = cpu_regs[R_EAX]; 2203 decode->cc_src = cpu_regs[R_EDX]; 2204 break; 2205 2206 default: 2207 g_assert_not_reached(); 2208 } 2209 2210 decode->cc_op = CC_OP_MULB + ot; 2211} 2212 2213static void gen_MULX(DisasContext *s, CPUX86State *env, X86DecodedInsn *decode) 2214{ 2215 MemOp ot = decode->op[0].ot; 2216 2217 /* low part of result in VEX.vvvv, high in MODRM */ 2218 switch (ot) { 2219 case MO_32: 2220#ifdef TARGET_X86_64 2221 tcg_gen_trunc_tl_i32(s->tmp2_i32, s->T0); 2222 tcg_gen_trunc_tl_i32(s->tmp3_i32, s->T1); 2223 tcg_gen_mulu2_i32(s->tmp2_i32, s->tmp3_i32, 2224 s->tmp2_i32, s->tmp3_i32); 2225 tcg_gen_extu_i32_tl(cpu_regs[s->vex_v], s->tmp2_i32); 2226 tcg_gen_extu_i32_tl(s->T0, s->tmp3_i32); 2227 break; 2228 2229 case MO_64: 2230#endif 2231 tcg_gen_mulu2_tl(cpu_regs[s->vex_v], s->T0, s->T0, s->T1); 2232 break; 2233 2234 default: 2235 g_assert_not_reached(); 2236 } 2237} 2238 2239static void gen_NEG(DisasContext *s, CPUX86State *env, X86DecodedInsn *decode) 2240{ 2241 MemOp ot = decode->op[0].ot; 2242 TCGv oldv = tcg_temp_new(); 2243 2244 if (s->prefix & PREFIX_LOCK) { 2245 TCGv newv = tcg_temp_new(); 2246 TCGv cmpv = tcg_temp_new(); 2247 TCGLabel *label1 = gen_new_label(); 2248 2249 gen_set_label(label1); 2250 gen_op_ld_v(s, ot, oldv, s->A0); 2251 tcg_gen_neg_tl(newv, oldv); 2252 tcg_gen_atomic_cmpxchg_tl(cmpv, s->A0, oldv, newv, 2253 s->mem_index, ot | MO_LE); 2254 tcg_gen_brcond_tl(TCG_COND_NE, oldv, cmpv, label1); 2255 } else { 2256 tcg_gen_mov_tl(oldv, s->T0); 2257 } 2258 tcg_gen_neg_tl(s->T0, oldv); 2259 2260 decode->cc_dst = s->T0; 2261 decode->cc_src = oldv; 2262 tcg_gen_movi_tl(s->cc_srcT, 0); 2263 decode->cc_op = CC_OP_SUBB + ot; 2264} 2265 2266static void gen_NOT(DisasContext *s, CPUX86State *env, X86DecodedInsn *decode) 2267{ 2268 MemOp ot = decode->op[0].ot; 2269 2270 if (s->prefix & PREFIX_LOCK) { 2271 tcg_gen_movi_tl(s->T0, ~0); 2272 tcg_gen_atomic_xor_fetch_tl(s->T0, s->A0, s->T0, 2273 s->mem_index, ot | MO_LE); 2274 } else { 2275 tcg_gen_not_tl(s->T0, s->T0); 2276 } 2277} 2278 2279static void gen_OR(DisasContext *s, CPUX86State *env, X86DecodedInsn *decode) 2280{ 2281 MemOp ot = decode->op[1].ot; 2282 2283 if (s->prefix & PREFIX_LOCK) { 2284 tcg_gen_atomic_or_fetch_tl(s->T0, s->A0, s->T1, 2285 s->mem_index, ot | MO_LE); 2286 } else { 2287 tcg_gen_or_tl(s->T0, s->T0, s->T1); 2288 } 2289 prepare_update1_cc(decode, s, CC_OP_LOGICB + ot); 2290} 2291 2292static void gen_OUT(DisasContext *s, CPUX86State *env, X86DecodedInsn *decode) 2293{ 2294 MemOp ot = decode->op[1].ot; 2295 TCGv_i32 port = tcg_temp_new_i32(); 2296 TCGv_i32 value = tcg_temp_new_i32(); 2297 2298 tcg_gen_trunc_tl_i32(port, s->T1); 2299 tcg_gen_ext16u_i32(port, port); 2300 if (!gen_check_io(s, ot, port, 0)) { 2301 return; 2302 } 2303 tcg_gen_trunc_tl_i32(value, s->T0); 2304 translator_io_start(&s->base); 2305 gen_helper_out_func(ot, port, value); 2306 gen_bpt_io(s, port, ot); 2307} 2308 2309static void gen_OUTS(DisasContext *s, CPUX86State *env, X86DecodedInsn *decode) 2310{ 2311 MemOp ot = decode->op[1].ot; 2312 TCGv_i32 port = tcg_temp_new_i32(); 2313 2314 tcg_gen_trunc_tl_i32(port, s->T1); 2315 tcg_gen_ext16u_i32(port, port); 2316 if (!gen_check_io(s, ot, port, SVM_IOIO_STR_MASK)) { 2317 return; 2318 } 2319 2320 translator_io_start(&s->base); 2321 if (s->prefix & (PREFIX_REPZ | PREFIX_REPNZ)) { 2322 gen_repz(s, ot, gen_outs); 2323 } else { 2324 gen_outs(s, ot); 2325 } 2326} 2327 2328static void gen_PALIGNR(DisasContext *s, CPUX86State *env, X86DecodedInsn *decode) 2329{ 2330 TCGv_i32 imm = tcg_constant8u_i32(decode->immediate); 2331 if (!(s->prefix & PREFIX_DATA)) { 2332 gen_helper_palignr_mmx(tcg_env, OP_PTR0, OP_PTR1, OP_PTR2, imm); 2333 } else if (!s->vex_l) { 2334 gen_helper_palignr_xmm(tcg_env, OP_PTR0, OP_PTR1, OP_PTR2, imm); 2335 } else { 2336 gen_helper_palignr_ymm(tcg_env, OP_PTR0, OP_PTR1, OP_PTR2, imm); 2337 } 2338} 2339 2340static void gen_PANDN(DisasContext *s, CPUX86State *env, X86DecodedInsn *decode) 2341{ 2342 int vec_len = vector_len(s, decode); 2343 2344 /* Careful, operand order is reversed! */ 2345 tcg_gen_gvec_andc(MO_64, 2346 decode->op[0].offset, decode->op[2].offset, 2347 decode->op[1].offset, vec_len, vec_len); 2348} 2349 2350static void gen_PCMPESTRI(DisasContext *s, CPUX86State *env, X86DecodedInsn *decode) 2351{ 2352 TCGv_i32 imm = tcg_constant8u_i32(decode->immediate); 2353 gen_helper_pcmpestri_xmm(tcg_env, OP_PTR1, OP_PTR2, imm); 2354 assume_cc_op(s, CC_OP_EFLAGS); 2355} 2356 2357static void gen_PCMPESTRM(DisasContext *s, CPUX86State *env, X86DecodedInsn *decode) 2358{ 2359 TCGv_i32 imm = tcg_constant8u_i32(decode->immediate); 2360 gen_helper_pcmpestrm_xmm(tcg_env, OP_PTR1, OP_PTR2, imm); 2361 assume_cc_op(s, CC_OP_EFLAGS); 2362 if ((s->prefix & PREFIX_VEX) && !s->vex_l) { 2363 tcg_gen_gvec_dup_imm(MO_64, offsetof(CPUX86State, xmm_regs[0].ZMM_X(1)), 2364 16, 16, 0); 2365 } 2366} 2367 2368static void gen_PCMPISTRI(DisasContext *s, CPUX86State *env, X86DecodedInsn *decode) 2369{ 2370 TCGv_i32 imm = tcg_constant8u_i32(decode->immediate); 2371 gen_helper_pcmpistri_xmm(tcg_env, OP_PTR1, OP_PTR2, imm); 2372 assume_cc_op(s, CC_OP_EFLAGS); 2373} 2374 2375static void gen_PCMPISTRM(DisasContext *s, CPUX86State *env, X86DecodedInsn *decode) 2376{ 2377 TCGv_i32 imm = tcg_constant8u_i32(decode->immediate); 2378 gen_helper_pcmpistrm_xmm(tcg_env, OP_PTR1, OP_PTR2, imm); 2379 assume_cc_op(s, CC_OP_EFLAGS); 2380 if ((s->prefix & PREFIX_VEX) && !s->vex_l) { 2381 tcg_gen_gvec_dup_imm(MO_64, offsetof(CPUX86State, xmm_regs[0].ZMM_X(1)), 2382 16, 16, 0); 2383 } 2384} 2385 2386static void gen_PDEP(DisasContext *s, CPUX86State *env, X86DecodedInsn *decode) 2387{ 2388 gen_helper_pdep(s->T0, s->T0, s->T1); 2389} 2390 2391static void gen_PEXT(DisasContext *s, CPUX86State *env, X86DecodedInsn *decode) 2392{ 2393 gen_helper_pext(s->T0, s->T0, s->T1); 2394} 2395 2396static inline void gen_pextr(DisasContext *s, CPUX86State *env, X86DecodedInsn *decode, MemOp ot) 2397{ 2398 int vec_len = vector_len(s, decode); 2399 int mask = (vec_len >> ot) - 1; 2400 int val = decode->immediate & mask; 2401 2402 switch (ot) { 2403 case MO_8: 2404 tcg_gen_ld8u_tl(s->T0, tcg_env, vector_elem_offset(&decode->op[1], ot, val)); 2405 break; 2406 case MO_16: 2407 tcg_gen_ld16u_tl(s->T0, tcg_env, vector_elem_offset(&decode->op[1], ot, val)); 2408 break; 2409 case MO_32: 2410#ifdef TARGET_X86_64 2411 tcg_gen_ld32u_tl(s->T0, tcg_env, vector_elem_offset(&decode->op[1], ot, val)); 2412 break; 2413 case MO_64: 2414#endif 2415 tcg_gen_ld_tl(s->T0, tcg_env, vector_elem_offset(&decode->op[1], ot, val)); 2416 break; 2417 default: 2418 abort(); 2419 } 2420} 2421 2422static void gen_PEXTRB(DisasContext *s, CPUX86State *env, X86DecodedInsn *decode) 2423{ 2424 gen_pextr(s, env, decode, MO_8); 2425} 2426 2427static void gen_PEXTRW(DisasContext *s, CPUX86State *env, X86DecodedInsn *decode) 2428{ 2429 gen_pextr(s, env, decode, MO_16); 2430} 2431 2432static void gen_PEXTR(DisasContext *s, CPUX86State *env, X86DecodedInsn *decode) 2433{ 2434 MemOp ot = decode->op[0].ot; 2435 gen_pextr(s, env, decode, ot); 2436} 2437 2438static inline void gen_pinsr(DisasContext *s, CPUX86State *env, X86DecodedInsn *decode, MemOp ot) 2439{ 2440 int vec_len = vector_len(s, decode); 2441 int mask = (vec_len >> ot) - 1; 2442 int val = decode->immediate & mask; 2443 2444 if (decode->op[1].offset != decode->op[0].offset) { 2445 assert(vec_len == 16); 2446 gen_store_sse(s, decode, decode->op[1].offset); 2447 } 2448 2449 switch (ot) { 2450 case MO_8: 2451 tcg_gen_st8_tl(s->T1, tcg_env, vector_elem_offset(&decode->op[0], ot, val)); 2452 break; 2453 case MO_16: 2454 tcg_gen_st16_tl(s->T1, tcg_env, vector_elem_offset(&decode->op[0], ot, val)); 2455 break; 2456 case MO_32: 2457#ifdef TARGET_X86_64 2458 tcg_gen_st32_tl(s->T1, tcg_env, vector_elem_offset(&decode->op[0], ot, val)); 2459 break; 2460 case MO_64: 2461#endif 2462 tcg_gen_st_tl(s->T1, tcg_env, vector_elem_offset(&decode->op[0], ot, val)); 2463 break; 2464 default: 2465 abort(); 2466 } 2467} 2468 2469static void gen_PINSRB(DisasContext *s, CPUX86State *env, X86DecodedInsn *decode) 2470{ 2471 gen_pinsr(s, env, decode, MO_8); 2472} 2473 2474static void gen_PINSRW(DisasContext *s, CPUX86State *env, X86DecodedInsn *decode) 2475{ 2476 gen_pinsr(s, env, decode, MO_16); 2477} 2478 2479static void gen_PINSR(DisasContext *s, CPUX86State *env, X86DecodedInsn *decode) 2480{ 2481 gen_pinsr(s, env, decode, decode->op[2].ot); 2482} 2483 2484static void gen_pmovmskb_i64(TCGv_i64 d, TCGv_i64 s) 2485{ 2486 TCGv_i64 t = tcg_temp_new_i64(); 2487 2488 tcg_gen_andi_i64(d, s, 0x8080808080808080ull); 2489 2490 /* 2491 * After each shift+or pair: 2492 * 0: a.......b.......c.......d.......e.......f.......g.......h....... 2493 * 7: ab......bc......cd......de......ef......fg......gh......h....... 2494 * 14: abcd....bcde....cdef....defg....efgh....fgh.....gh......h....... 2495 * 28: abcdefghbcdefgh.cdefgh..defgh...efgh....fgh.....gh......h....... 2496 * The result is left in the high bits of the word. 2497 */ 2498 tcg_gen_shli_i64(t, d, 7); 2499 tcg_gen_or_i64(d, d, t); 2500 tcg_gen_shli_i64(t, d, 14); 2501 tcg_gen_or_i64(d, d, t); 2502 tcg_gen_shli_i64(t, d, 28); 2503 tcg_gen_or_i64(d, d, t); 2504} 2505 2506static void gen_pmovmskb_vec(unsigned vece, TCGv_vec d, TCGv_vec s) 2507{ 2508 TCGv_vec t = tcg_temp_new_vec_matching(d); 2509 TCGv_vec m = tcg_constant_vec_matching(d, MO_8, 0x80); 2510 2511 /* See above */ 2512 tcg_gen_and_vec(vece, d, s, m); 2513 tcg_gen_shli_vec(vece, t, d, 7); 2514 tcg_gen_or_vec(vece, d, d, t); 2515 tcg_gen_shli_vec(vece, t, d, 14); 2516 tcg_gen_or_vec(vece, d, d, t); 2517 tcg_gen_shli_vec(vece, t, d, 28); 2518 tcg_gen_or_vec(vece, d, d, t); 2519} 2520 2521static void gen_PMOVMSKB(DisasContext *s, CPUX86State *env, X86DecodedInsn *decode) 2522{ 2523 static const TCGOpcode vecop_list[] = { INDEX_op_shli_vec, 0 }; 2524 static const GVecGen2 g = { 2525 .fni8 = gen_pmovmskb_i64, 2526 .fniv = gen_pmovmskb_vec, 2527 .opt_opc = vecop_list, 2528 .vece = MO_64, 2529 .prefer_i64 = TCG_TARGET_REG_BITS == 64 2530 }; 2531 MemOp ot = decode->op[2].ot; 2532 int vec_len = vector_len(s, decode); 2533 TCGv t = tcg_temp_new(); 2534 2535 tcg_gen_gvec_2(offsetof(CPUX86State, xmm_t0) + xmm_offset(ot), decode->op[2].offset, 2536 vec_len, vec_len, &g); 2537 tcg_gen_ld8u_tl(s->T0, tcg_env, offsetof(CPUX86State, xmm_t0.ZMM_B(vec_len - 1))); 2538 while (vec_len > 8) { 2539 vec_len -= 8; 2540 if (TCG_TARGET_HAS_extract2_tl) { 2541 /* 2542 * Load the next byte of the result into the high byte of T. 2543 * TCG does a similar expansion of deposit to shl+extract2; by 2544 * loading the whole word, the shift left is avoided. 2545 */ 2546#ifdef TARGET_X86_64 2547 tcg_gen_ld_tl(t, tcg_env, offsetof(CPUX86State, xmm_t0.ZMM_Q((vec_len - 1) / 8))); 2548#else 2549 tcg_gen_ld_tl(t, tcg_env, offsetof(CPUX86State, xmm_t0.ZMM_L((vec_len - 1) / 4))); 2550#endif 2551 2552 tcg_gen_extract2_tl(s->T0, t, s->T0, TARGET_LONG_BITS - 8); 2553 } else { 2554 /* 2555 * The _previous_ value is deposited into bits 8 and higher of t. Because 2556 * those bits are known to be zero after ld8u, this becomes a shift+or 2557 * if deposit is not available. 2558 */ 2559 tcg_gen_ld8u_tl(t, tcg_env, offsetof(CPUX86State, xmm_t0.ZMM_B(vec_len - 1))); 2560 tcg_gen_deposit_tl(s->T0, t, s->T0, 8, TARGET_LONG_BITS - 8); 2561 } 2562 } 2563} 2564 2565static void gen_POP(DisasContext *s, CPUX86State *env, X86DecodedInsn *decode) 2566{ 2567 MemOp ot = gen_pop_T0(s); 2568 if (decode->op[0].has_ea) { 2569 /* NOTE: order is important for MMU exceptions */ 2570 gen_op_st_v(s, ot, s->T0, s->A0); 2571 decode->op[0].unit = X86_OP_SKIP; 2572 } 2573 /* NOTE: writing back registers after update is important for pop %sp */ 2574 gen_pop_update(s, ot); 2575} 2576 2577static void gen_POPA(DisasContext *s, CPUX86State *env, X86DecodedInsn *decode) 2578{ 2579 gen_popa(s); 2580} 2581 2582static void gen_POPF(DisasContext *s, CPUX86State *env, X86DecodedInsn *decode) 2583{ 2584 MemOp ot; 2585 int mask = TF_MASK | AC_MASK | ID_MASK | NT_MASK; 2586 2587 if (CPL(s) == 0) { 2588 mask |= IF_MASK | IOPL_MASK; 2589 } else if (CPL(s) <= IOPL(s)) { 2590 mask |= IF_MASK; 2591 } 2592 if (s->dflag == MO_16) { 2593 mask &= 0xffff; 2594 } 2595 2596 ot = gen_pop_T0(s); 2597 gen_helper_write_eflags(tcg_env, s->T0, tcg_constant_i32(mask)); 2598 gen_pop_update(s, ot); 2599 set_cc_op(s, CC_OP_EFLAGS); 2600 /* abort translation because TF/AC flag may change */ 2601 s->base.is_jmp = DISAS_EOB_NEXT; 2602} 2603 2604static void gen_PSHUFW(DisasContext *s, CPUX86State *env, X86DecodedInsn *decode) 2605{ 2606 TCGv_i32 imm = tcg_constant8u_i32(decode->immediate); 2607 gen_helper_pshufw_mmx(OP_PTR0, OP_PTR1, imm); 2608} 2609 2610static void gen_PSRLW_i(DisasContext *s, CPUX86State *env, X86DecodedInsn *decode) 2611{ 2612 int vec_len = vector_len(s, decode); 2613 2614 if (decode->immediate >= 16) { 2615 tcg_gen_gvec_dup_imm(MO_64, decode->op[0].offset, vec_len, vec_len, 0); 2616 } else { 2617 tcg_gen_gvec_shri(MO_16, 2618 decode->op[0].offset, decode->op[1].offset, 2619 decode->immediate, vec_len, vec_len); 2620 } 2621} 2622 2623static void gen_PSLLW_i(DisasContext *s, CPUX86State *env, X86DecodedInsn *decode) 2624{ 2625 int vec_len = vector_len(s, decode); 2626 2627 if (decode->immediate >= 16) { 2628 tcg_gen_gvec_dup_imm(MO_64, decode->op[0].offset, vec_len, vec_len, 0); 2629 } else { 2630 tcg_gen_gvec_shli(MO_16, 2631 decode->op[0].offset, decode->op[1].offset, 2632 decode->immediate, vec_len, vec_len); 2633 } 2634} 2635 2636static void gen_PSRAW_i(DisasContext *s, CPUX86State *env, X86DecodedInsn *decode) 2637{ 2638 int vec_len = vector_len(s, decode); 2639 2640 if (decode->immediate >= 16) { 2641 decode->immediate = 15; 2642 } 2643 tcg_gen_gvec_sari(MO_16, 2644 decode->op[0].offset, decode->op[1].offset, 2645 decode->immediate, vec_len, vec_len); 2646} 2647 2648static void gen_PSRLD_i(DisasContext *s, CPUX86State *env, X86DecodedInsn *decode) 2649{ 2650 int vec_len = vector_len(s, decode); 2651 2652 if (decode->immediate >= 32) { 2653 tcg_gen_gvec_dup_imm(MO_64, decode->op[0].offset, vec_len, vec_len, 0); 2654 } else { 2655 tcg_gen_gvec_shri(MO_32, 2656 decode->op[0].offset, decode->op[1].offset, 2657 decode->immediate, vec_len, vec_len); 2658 } 2659} 2660 2661static void gen_PSLLD_i(DisasContext *s, CPUX86State *env, X86DecodedInsn *decode) 2662{ 2663 int vec_len = vector_len(s, decode); 2664 2665 if (decode->immediate >= 32) { 2666 tcg_gen_gvec_dup_imm(MO_64, decode->op[0].offset, vec_len, vec_len, 0); 2667 } else { 2668 tcg_gen_gvec_shli(MO_32, 2669 decode->op[0].offset, decode->op[1].offset, 2670 decode->immediate, vec_len, vec_len); 2671 } 2672} 2673 2674static void gen_PSRAD_i(DisasContext *s, CPUX86State *env, X86DecodedInsn *decode) 2675{ 2676 int vec_len = vector_len(s, decode); 2677 2678 if (decode->immediate >= 32) { 2679 decode->immediate = 31; 2680 } 2681 tcg_gen_gvec_sari(MO_32, 2682 decode->op[0].offset, decode->op[1].offset, 2683 decode->immediate, vec_len, vec_len); 2684} 2685 2686static void gen_PSRLQ_i(DisasContext *s, CPUX86State *env, X86DecodedInsn *decode) 2687{ 2688 int vec_len = vector_len(s, decode); 2689 2690 if (decode->immediate >= 64) { 2691 tcg_gen_gvec_dup_imm(MO_64, decode->op[0].offset, vec_len, vec_len, 0); 2692 } else { 2693 tcg_gen_gvec_shri(MO_64, 2694 decode->op[0].offset, decode->op[1].offset, 2695 decode->immediate, vec_len, vec_len); 2696 } 2697} 2698 2699static void gen_PSLLQ_i(DisasContext *s, CPUX86State *env, X86DecodedInsn *decode) 2700{ 2701 int vec_len = vector_len(s, decode); 2702 2703 if (decode->immediate >= 64) { 2704 tcg_gen_gvec_dup_imm(MO_64, decode->op[0].offset, vec_len, vec_len, 0); 2705 } else { 2706 tcg_gen_gvec_shli(MO_64, 2707 decode->op[0].offset, decode->op[1].offset, 2708 decode->immediate, vec_len, vec_len); 2709 } 2710} 2711 2712static TCGv_ptr make_imm8u_xmm_vec(uint8_t imm, int vec_len) 2713{ 2714 MemOp ot = vec_len == 16 ? MO_128 : MO_256; 2715 TCGv_i32 imm_v = tcg_constant8u_i32(imm); 2716 TCGv_ptr ptr = tcg_temp_new_ptr(); 2717 2718 tcg_gen_gvec_dup_imm(MO_64, offsetof(CPUX86State, xmm_t0) + xmm_offset(ot), 2719 vec_len, vec_len, 0); 2720 2721 tcg_gen_addi_ptr(ptr, tcg_env, offsetof(CPUX86State, xmm_t0)); 2722 tcg_gen_st_i32(imm_v, tcg_env, offsetof(CPUX86State, xmm_t0.ZMM_L(0))); 2723 return ptr; 2724} 2725 2726static void gen_PSRLDQ_i(DisasContext *s, CPUX86State *env, X86DecodedInsn *decode) 2727{ 2728 int vec_len = vector_len(s, decode); 2729 TCGv_ptr imm_vec = make_imm8u_xmm_vec(decode->immediate, vec_len); 2730 2731 if (s->vex_l) { 2732 gen_helper_psrldq_ymm(tcg_env, OP_PTR0, OP_PTR1, imm_vec); 2733 } else { 2734 gen_helper_psrldq_xmm(tcg_env, OP_PTR0, OP_PTR1, imm_vec); 2735 } 2736} 2737 2738static void gen_PSLLDQ_i(DisasContext *s, CPUX86State *env, X86DecodedInsn *decode) 2739{ 2740 int vec_len = vector_len(s, decode); 2741 TCGv_ptr imm_vec = make_imm8u_xmm_vec(decode->immediate, vec_len); 2742 2743 if (s->vex_l) { 2744 gen_helper_pslldq_ymm(tcg_env, OP_PTR0, OP_PTR1, imm_vec); 2745 } else { 2746 gen_helper_pslldq_xmm(tcg_env, OP_PTR0, OP_PTR1, imm_vec); 2747 } 2748} 2749 2750static void gen_PUSH(DisasContext *s, CPUX86State *env, X86DecodedInsn *decode) 2751{ 2752 gen_push_v(s, s->T1); 2753} 2754 2755static void gen_PUSHA(DisasContext *s, CPUX86State *env, X86DecodedInsn *decode) 2756{ 2757 gen_pusha(s); 2758} 2759 2760static void gen_PUSHF(DisasContext *s, CPUX86State *env, X86DecodedInsn *decode) 2761{ 2762 gen_update_cc_op(s); 2763 gen_helper_read_eflags(s->T0, tcg_env); 2764 gen_push_v(s, s->T0); 2765} 2766 2767static MemOp gen_shift_count(DisasContext *s, X86DecodedInsn *decode, 2768 bool *can_be_zero, TCGv *count) 2769{ 2770 MemOp ot = decode->op[0].ot; 2771 int mask = (ot <= MO_32 ? 0x1f : 0x3f); 2772 2773 *can_be_zero = false; 2774 switch (decode->op[2].unit) { 2775 case X86_OP_INT: 2776 *count = tcg_temp_new(); 2777 tcg_gen_andi_tl(*count, s->T1, mask); 2778 *can_be_zero = true; 2779 break; 2780 2781 case X86_OP_IMM: 2782 if ((decode->immediate & mask) == 0) { 2783 *count = NULL; 2784 break; 2785 } 2786 *count = tcg_temp_new(); 2787 tcg_gen_movi_tl(*count, decode->immediate & mask); 2788 break; 2789 2790 case X86_OP_SKIP: 2791 *count = tcg_temp_new(); 2792 tcg_gen_movi_tl(*count, 1); 2793 break; 2794 2795 default: 2796 g_assert_not_reached(); 2797 } 2798 2799 return ot; 2800} 2801 2802/* 2803 * Compute existing flags in decode->cc_src, for gen_* functions that wants 2804 * to set the cc_op set to CC_OP_ADCOX. In particular, this allows rotate 2805 * operations to compute the carry in decode->cc_dst and the overflow in 2806 * decode->cc_src2. 2807 * 2808 * If need_flags is true, decode->cc_dst and decode->cc_src2 are preloaded 2809 * with the value of CF and OF before the instruction, so that it is possible 2810 * to keep the flags unmodified. 2811 * 2812 * Return true if carry could be made available cheaply as a 1-bit value in 2813 * decode->cc_dst (trying a bit harder if want_carry is true). If false is 2814 * returned, decode->cc_dst is uninitialized and the carry is only available 2815 * as bit 0 of decode->cc_src. 2816 */ 2817static bool gen_eflags_adcox(DisasContext *s, X86DecodedInsn *decode, bool want_carry, bool need_flags) 2818{ 2819 bool got_cf = false; 2820 bool got_of = false; 2821 2822 decode->cc_dst = tcg_temp_new(); 2823 decode->cc_src = tcg_temp_new(); 2824 decode->cc_src2 = tcg_temp_new(); 2825 decode->cc_op = CC_OP_ADCOX; 2826 2827 /* A lot more cc_ops could be "optimized" to avoid the extracts at 2828 * the end (INC/DEC, BMILG, MUL), but they are all really unlikely 2829 * to be followed by rotations within the same basic block. 2830 */ 2831 switch (s->cc_op) { 2832 case CC_OP_ADCOX: 2833 /* No need to compute the full EFLAGS, CF/OF are already isolated. */ 2834 tcg_gen_mov_tl(decode->cc_src, cpu_cc_src); 2835 if (need_flags) { 2836 tcg_gen_mov_tl(decode->cc_src2, cpu_cc_src2); 2837 got_of = true; 2838 } 2839 if (want_carry || need_flags) { 2840 tcg_gen_mov_tl(decode->cc_dst, cpu_cc_dst); 2841 got_cf = true; 2842 } 2843 break; 2844 2845 case CC_OP_LOGICB ... CC_OP_LOGICQ: 2846 /* CF and OF are zero, do it just because it's easy. */ 2847 gen_mov_eflags(s, decode->cc_src); 2848 if (need_flags) { 2849 tcg_gen_movi_tl(decode->cc_src2, 0); 2850 got_of = true; 2851 } 2852 if (want_carry || need_flags) { 2853 tcg_gen_movi_tl(decode->cc_dst, 0); 2854 got_cf = true; 2855 } 2856 break; 2857 2858 case CC_OP_SARB ... CC_OP_SARQ: 2859 /* 2860 * SHR/RCR/SHR/RCR/... is a relatively common occurrence of RCR. 2861 * By computing CF without using eflags, the calls to cc_compute_all 2862 * can be eliminated as dead code (except for the last RCR). 2863 */ 2864 if (want_carry || need_flags) { 2865 tcg_gen_andi_tl(decode->cc_dst, cpu_cc_src, 1); 2866 got_cf = true; 2867 } 2868 gen_mov_eflags(s, decode->cc_src); 2869 break; 2870 2871 case CC_OP_SHLB ... CC_OP_SHLQ: 2872 /* 2873 * Likewise for SHL/RCL/SHL/RCL/... but, if CF is not in the sign 2874 * bit, we might as well fish CF out of EFLAGS and save a shift. 2875 */ 2876 if (want_carry && (!need_flags || s->cc_op == CC_OP_SHLB + MO_TL)) { 2877 tcg_gen_shri_tl(decode->cc_dst, cpu_cc_src, (8 << (s->cc_op - CC_OP_SHLB)) - 1); 2878 got_cf = true; 2879 } 2880 gen_mov_eflags(s, decode->cc_src); 2881 break; 2882 2883 default: 2884 gen_mov_eflags(s, decode->cc_src); 2885 break; 2886 } 2887 2888 if (need_flags) { 2889 /* If the flags could be left unmodified, always load them. */ 2890 if (!got_of) { 2891 tcg_gen_extract_tl(decode->cc_src2, decode->cc_src, ctz32(CC_O), 1); 2892 got_of = true; 2893 } 2894 if (!got_cf) { 2895 tcg_gen_extract_tl(decode->cc_dst, decode->cc_src, ctz32(CC_C), 1); 2896 got_cf = true; 2897 } 2898 } 2899 return got_cf; 2900} 2901 2902static void gen_rot_overflow(X86DecodedInsn *decode, TCGv result, TCGv old, 2903 bool can_be_zero, TCGv count) 2904{ 2905 MemOp ot = decode->op[0].ot; 2906 TCGv temp = can_be_zero ? tcg_temp_new() : decode->cc_src2; 2907 2908 tcg_gen_xor_tl(temp, old, result); 2909 tcg_gen_extract_tl(temp, temp, (8 << ot) - 1, 1); 2910 if (can_be_zero) { 2911 tcg_gen_movcond_tl(TCG_COND_EQ, decode->cc_src2, count, tcg_constant_tl(0), 2912 decode->cc_src2, temp); 2913 } 2914} 2915 2916/* 2917 * RCx operations are invariant modulo 8*operand_size+1. For 8 and 16-bit operands, 2918 * this is less than 0x1f (the mask applied by gen_shift_count) so reduce further. 2919 */ 2920static void gen_rotc_mod(MemOp ot, TCGv count) 2921{ 2922 TCGv temp; 2923 2924 switch (ot) { 2925 case MO_8: 2926 temp = tcg_temp_new(); 2927 tcg_gen_subi_tl(temp, count, 18); 2928 tcg_gen_movcond_tl(TCG_COND_GE, count, temp, tcg_constant_tl(0), temp, count); 2929 tcg_gen_subi_tl(temp, count, 9); 2930 tcg_gen_movcond_tl(TCG_COND_GE, count, temp, tcg_constant_tl(0), temp, count); 2931 break; 2932 2933 case MO_16: 2934 temp = tcg_temp_new(); 2935 tcg_gen_subi_tl(temp, count, 17); 2936 tcg_gen_movcond_tl(TCG_COND_GE, count, temp, tcg_constant_tl(0), temp, count); 2937 break; 2938 2939 default: 2940 break; 2941 } 2942} 2943 2944/* 2945 * The idea here is that the bit to the right of the new bit 0 is the 2946 * new carry, and the bit to the right of the old bit 0 is the old carry. 2947 * Just like a regular rotation, the result of the rotation is composed 2948 * from a right shifted part and a left shifted part of s->T0. The new carry 2949 * is extracted from the right-shifted portion, and the old carry is 2950 * inserted at the end of the left-shifted portion. 2951 * 2952 * Because of the separate shifts involving the carry, gen_RCL and gen_RCR 2953 * mostly operate on count-1. This also comes in handy when computing 2954 * length - count, because (length-1) - (count-1) can be computed with 2955 * a XOR, and that is commutative unlike subtraction. 2956 */ 2957static void gen_RCL(DisasContext *s, CPUX86State *env, X86DecodedInsn *decode) 2958{ 2959 bool have_1bit_cin, can_be_zero; 2960 TCGv count; 2961 TCGLabel *zero_label = NULL; 2962 MemOp ot = gen_shift_count(s, decode, &can_be_zero, &count); 2963 TCGv low, high, low_count; 2964 2965 if (!count) { 2966 return; 2967 } 2968 2969 low = tcg_temp_new(); 2970 high = tcg_temp_new(); 2971 low_count = tcg_temp_new(); 2972 2973 gen_rotc_mod(ot, count); 2974 have_1bit_cin = gen_eflags_adcox(s, decode, true, can_be_zero); 2975 if (can_be_zero) { 2976 zero_label = gen_new_label(); 2977 tcg_gen_brcondi_tl(TCG_COND_EQ, count, 0, zero_label); 2978 } 2979 2980 /* Compute high part, including incoming carry. */ 2981 if (!have_1bit_cin || TCG_TARGET_deposit_tl_valid(1, TARGET_LONG_BITS - 1)) { 2982 /* high = (T0 << 1) | cin */ 2983 TCGv cin = have_1bit_cin ? decode->cc_dst : decode->cc_src; 2984 tcg_gen_deposit_tl(high, cin, s->T0, 1, TARGET_LONG_BITS - 1); 2985 } else { 2986 /* Same as above but without deposit; cin in cc_dst. */ 2987 tcg_gen_add_tl(high, s->T0, decode->cc_dst); 2988 tcg_gen_add_tl(high, high, s->T0); 2989 } 2990 tcg_gen_subi_tl(count, count, 1); 2991 tcg_gen_shl_tl(high, high, count); 2992 2993 /* Compute low part and outgoing carry, incoming s->T0 is zero extended */ 2994 tcg_gen_xori_tl(low_count, count, (8 << ot) - 1); /* LENGTH - 1 - (count - 1) */ 2995 tcg_gen_shr_tl(low, s->T0, low_count); 2996 tcg_gen_andi_tl(decode->cc_dst, low, 1); 2997 tcg_gen_shri_tl(low, low, 1); 2998 2999 /* Compute result and outgoing overflow */ 3000 tcg_gen_mov_tl(decode->cc_src2, s->T0); 3001 tcg_gen_or_tl(s->T0, low, high); 3002 gen_rot_overflow(decode, s->T0, decode->cc_src2, false, NULL); 3003 3004 if (zero_label) { 3005 gen_set_label(zero_label); 3006 } 3007} 3008 3009static void gen_RCR(DisasContext *s, CPUX86State *env, X86DecodedInsn *decode) 3010{ 3011 bool have_1bit_cin, can_be_zero; 3012 TCGv count; 3013 TCGLabel *zero_label = NULL; 3014 MemOp ot = gen_shift_count(s, decode, &can_be_zero, &count); 3015 TCGv low, high, high_count; 3016 3017 if (!count) { 3018 return; 3019 } 3020 3021 low = tcg_temp_new(); 3022 high = tcg_temp_new(); 3023 high_count = tcg_temp_new(); 3024 3025 gen_rotc_mod(ot, count); 3026 have_1bit_cin = gen_eflags_adcox(s, decode, true, can_be_zero); 3027 if (can_be_zero) { 3028 zero_label = gen_new_label(); 3029 tcg_gen_brcondi_tl(TCG_COND_EQ, count, 0, zero_label); 3030 } 3031 3032 /* Save incoming carry into high, it will be shifted later. */ 3033 if (!have_1bit_cin || TCG_TARGET_deposit_tl_valid(1, TARGET_LONG_BITS - 1)) { 3034 TCGv cin = have_1bit_cin ? decode->cc_dst : decode->cc_src; 3035 tcg_gen_deposit_tl(high, cin, s->T0, 1, TARGET_LONG_BITS - 1); 3036 } else { 3037 /* Same as above but without deposit; cin in cc_dst. */ 3038 tcg_gen_add_tl(high, s->T0, decode->cc_dst); 3039 tcg_gen_add_tl(high, high, s->T0); 3040 } 3041 3042 /* Compute low part and outgoing carry, incoming s->T0 is zero extended */ 3043 tcg_gen_subi_tl(count, count, 1); 3044 tcg_gen_shr_tl(low, s->T0, count); 3045 tcg_gen_andi_tl(decode->cc_dst, low, 1); 3046 tcg_gen_shri_tl(low, low, 1); 3047 3048 /* Move high part to the right position */ 3049 tcg_gen_xori_tl(high_count, count, (8 << ot) - 1); /* LENGTH - 1 - (count - 1) */ 3050 tcg_gen_shl_tl(high, high, high_count); 3051 3052 /* Compute result and outgoing overflow */ 3053 tcg_gen_mov_tl(decode->cc_src2, s->T0); 3054 tcg_gen_or_tl(s->T0, low, high); 3055 gen_rot_overflow(decode, s->T0, decode->cc_src2, false, NULL); 3056 3057 if (zero_label) { 3058 gen_set_label(zero_label); 3059 } 3060} 3061 3062static void gen_RET(DisasContext *s, CPUX86State *env, X86DecodedInsn *decode) 3063{ 3064 int16_t adjust = decode->e.op2 == X86_TYPE_I ? decode->immediate : 0; 3065 3066 MemOp ot = gen_pop_T0(s); 3067 gen_stack_update(s, adjust + (1 << ot)); 3068 gen_op_jmp_v(s, s->T0); 3069 gen_bnd_jmp(s); 3070 s->base.is_jmp = DISAS_JUMP; 3071} 3072 3073static void gen_RETF(DisasContext *s, CPUX86State *env, X86DecodedInsn *decode) 3074{ 3075 int16_t adjust = decode->e.op2 == X86_TYPE_I ? decode->immediate : 0; 3076 3077 if (!PE(s) || VM86(s)) { 3078 gen_lea_ss_ofs(s, s->A0, cpu_regs[R_ESP], 0); 3079 /* pop offset */ 3080 gen_op_ld_v(s, s->dflag, s->T0, s->A0); 3081 /* NOTE: keeping EIP updated is not a problem in case of 3082 exception */ 3083 gen_op_jmp_v(s, s->T0); 3084 /* pop selector */ 3085 gen_add_A0_im(s, 1 << s->dflag); 3086 gen_op_ld_v(s, s->dflag, s->T0, s->A0); 3087 gen_op_movl_seg_real(s, R_CS, s->T0); 3088 /* add stack offset */ 3089 gen_stack_update(s, adjust + (2 << s->dflag)); 3090 } else { 3091 gen_update_cc_op(s); 3092 gen_update_eip_cur(s); 3093 gen_helper_lret_protected(tcg_env, tcg_constant_i32(s->dflag - 1), 3094 tcg_constant_i32(adjust)); 3095 } 3096 s->base.is_jmp = DISAS_EOB_ONLY; 3097} 3098 3099/* 3100 * Return non-NULL if a 32-bit rotate works, after possibly replicating the input. 3101 * The input has already been zero-extended upon operand decode. 3102 */ 3103static TCGv_i32 gen_rot_replicate(MemOp ot, TCGv in) 3104{ 3105 TCGv_i32 temp; 3106 switch (ot) { 3107 case MO_8: 3108 temp = tcg_temp_new_i32(); 3109 tcg_gen_trunc_tl_i32(temp, in); 3110 tcg_gen_muli_i32(temp, temp, 0x01010101); 3111 return temp; 3112 3113 case MO_16: 3114 temp = tcg_temp_new_i32(); 3115 tcg_gen_trunc_tl_i32(temp, in); 3116 tcg_gen_deposit_i32(temp, temp, temp, 16, 16); 3117 return temp; 3118 3119#ifdef TARGET_X86_64 3120 case MO_32: 3121 temp = tcg_temp_new_i32(); 3122 tcg_gen_trunc_tl_i32(temp, in); 3123 return temp; 3124#endif 3125 3126 default: 3127 return NULL; 3128 } 3129} 3130 3131static void gen_rot_carry(X86DecodedInsn *decode, TCGv result, 3132 bool can_be_zero, TCGv count, int bit) 3133{ 3134 if (!can_be_zero) { 3135 tcg_gen_extract_tl(decode->cc_dst, result, bit, 1); 3136 } else { 3137 TCGv temp = tcg_temp_new(); 3138 tcg_gen_extract_tl(temp, result, bit, 1); 3139 tcg_gen_movcond_tl(TCG_COND_EQ, decode->cc_dst, count, tcg_constant_tl(0), 3140 decode->cc_dst, temp); 3141 } 3142} 3143 3144static void gen_ROL(DisasContext *s, CPUX86State *env, X86DecodedInsn *decode) 3145{ 3146 bool can_be_zero; 3147 TCGv count; 3148 MemOp ot = gen_shift_count(s, decode, &can_be_zero, &count); 3149 TCGv_i32 temp32, count32; 3150 TCGv old = tcg_temp_new(); 3151 3152 if (!count) { 3153 return; 3154 } 3155 3156 gen_eflags_adcox(s, decode, false, can_be_zero); 3157 tcg_gen_mov_tl(old, s->T0); 3158 temp32 = gen_rot_replicate(ot, s->T0); 3159 if (temp32) { 3160 count32 = tcg_temp_new_i32(); 3161 tcg_gen_trunc_tl_i32(count32, count); 3162 tcg_gen_rotl_i32(temp32, temp32, count32); 3163 /* Zero extend to facilitate later optimization. */ 3164 tcg_gen_extu_i32_tl(s->T0, temp32); 3165 } else { 3166 tcg_gen_rotl_tl(s->T0, s->T0, count); 3167 } 3168 gen_rot_carry(decode, s->T0, can_be_zero, count, 0); 3169 gen_rot_overflow(decode, s->T0, old, can_be_zero, count); 3170} 3171 3172static void gen_ROR(DisasContext *s, CPUX86State *env, X86DecodedInsn *decode) 3173{ 3174 bool can_be_zero; 3175 TCGv count; 3176 MemOp ot = gen_shift_count(s, decode, &can_be_zero, &count); 3177 TCGv_i32 temp32, count32; 3178 TCGv old = tcg_temp_new(); 3179 3180 if (!count) { 3181 return; 3182 } 3183 3184 gen_eflags_adcox(s, decode, false, can_be_zero); 3185 tcg_gen_mov_tl(old, s->T0); 3186 temp32 = gen_rot_replicate(ot, s->T0); 3187 if (temp32) { 3188 count32 = tcg_temp_new_i32(); 3189 tcg_gen_trunc_tl_i32(count32, count); 3190 tcg_gen_rotr_i32(temp32, temp32, count32); 3191 /* Zero extend to facilitate later optimization. */ 3192 tcg_gen_extu_i32_tl(s->T0, temp32); 3193 gen_rot_carry(decode, s->T0, can_be_zero, count, 31); 3194 } else { 3195 tcg_gen_rotr_tl(s->T0, s->T0, count); 3196 gen_rot_carry(decode, s->T0, can_be_zero, count, TARGET_LONG_BITS - 1); 3197 } 3198 gen_rot_overflow(decode, s->T0, old, can_be_zero, count); 3199} 3200 3201static void gen_RORX(DisasContext *s, CPUX86State *env, X86DecodedInsn *decode) 3202{ 3203 MemOp ot = decode->op[0].ot; 3204 int mask = ot == MO_64 ? 63 : 31; 3205 int b = decode->immediate & mask; 3206 3207 switch (ot) { 3208 case MO_32: 3209#ifdef TARGET_X86_64 3210 tcg_gen_trunc_tl_i32(s->tmp2_i32, s->T0); 3211 tcg_gen_rotri_i32(s->tmp2_i32, s->tmp2_i32, b); 3212 tcg_gen_extu_i32_tl(s->T0, s->tmp2_i32); 3213 break; 3214 3215 case MO_64: 3216#endif 3217 tcg_gen_rotri_tl(s->T0, s->T0, b); 3218 break; 3219 3220 default: 3221 g_assert_not_reached(); 3222 } 3223} 3224 3225static void gen_SAHF(DisasContext *s, CPUX86State *env, X86DecodedInsn *decode) 3226{ 3227 if (CODE64(s) && !(s->cpuid_ext3_features & CPUID_EXT3_LAHF_LM)) { 3228 return gen_illegal_opcode(s); 3229 } 3230 tcg_gen_shri_tl(s->T0, cpu_regs[R_EAX], 8); 3231 gen_compute_eflags(s); 3232 tcg_gen_andi_tl(cpu_cc_src, cpu_cc_src, CC_O); 3233 tcg_gen_andi_tl(s->T0, s->T0, CC_S | CC_Z | CC_A | CC_P | CC_C); 3234 tcg_gen_or_tl(cpu_cc_src, cpu_cc_src, s->T0); 3235} 3236 3237static void gen_SALC(DisasContext *s, CPUX86State *env, X86DecodedInsn *decode) 3238{ 3239 gen_compute_eflags_c(s, s->T0); 3240 tcg_gen_neg_tl(s->T0, s->T0); 3241} 3242 3243static void gen_shift_dynamic_flags(DisasContext *s, X86DecodedInsn *decode, TCGv count, CCOp cc_op) 3244{ 3245 TCGv_i32 count32 = tcg_temp_new_i32(); 3246 TCGv_i32 old_cc_op; 3247 3248 decode->cc_op = CC_OP_DYNAMIC; 3249 decode->cc_op_dynamic = tcg_temp_new_i32(); 3250 3251 assert(decode->cc_dst == s->T0); 3252 if (cc_op_live[s->cc_op] & USES_CC_DST) { 3253 decode->cc_dst = tcg_temp_new(); 3254 tcg_gen_movcond_tl(TCG_COND_EQ, decode->cc_dst, count, tcg_constant_tl(0), 3255 cpu_cc_dst, s->T0); 3256 } 3257 3258 if (cc_op_live[s->cc_op] & USES_CC_SRC) { 3259 tcg_gen_movcond_tl(TCG_COND_EQ, decode->cc_src, count, tcg_constant_tl(0), 3260 cpu_cc_src, decode->cc_src); 3261 } 3262 3263 tcg_gen_trunc_tl_i32(count32, count); 3264 if (s->cc_op == CC_OP_DYNAMIC) { 3265 old_cc_op = cpu_cc_op; 3266 } else { 3267 old_cc_op = tcg_constant_i32(s->cc_op); 3268 } 3269 tcg_gen_movcond_i32(TCG_COND_EQ, decode->cc_op_dynamic, count32, tcg_constant_i32(0), 3270 old_cc_op, tcg_constant_i32(cc_op)); 3271} 3272 3273static void gen_SAR(DisasContext *s, CPUX86State *env, X86DecodedInsn *decode) 3274{ 3275 bool can_be_zero; 3276 TCGv count; 3277 MemOp ot = gen_shift_count(s, decode, &can_be_zero, &count); 3278 3279 if (!count) { 3280 return; 3281 } 3282 3283 decode->cc_dst = s->T0; 3284 decode->cc_src = tcg_temp_new(); 3285 tcg_gen_subi_tl(decode->cc_src, count, 1); 3286 tcg_gen_sar_tl(decode->cc_src, s->T0, decode->cc_src); 3287 tcg_gen_sar_tl(s->T0, s->T0, count); 3288 if (can_be_zero) { 3289 gen_shift_dynamic_flags(s, decode, count, CC_OP_SARB + ot); 3290 } else { 3291 decode->cc_op = CC_OP_SARB + ot; 3292 } 3293} 3294 3295static void gen_SARX(DisasContext *s, CPUX86State *env, X86DecodedInsn *decode) 3296{ 3297 MemOp ot = decode->op[0].ot; 3298 int mask; 3299 3300 mask = ot == MO_64 ? 63 : 31; 3301 tcg_gen_andi_tl(s->T1, s->T1, mask); 3302 tcg_gen_sar_tl(s->T0, s->T0, s->T1); 3303} 3304 3305static void gen_SBB(DisasContext *s, CPUX86State *env, X86DecodedInsn *decode) 3306{ 3307 MemOp ot = decode->op[0].ot; 3308 TCGv c_in = tcg_temp_new(); 3309 3310 gen_compute_eflags_c(s, c_in); 3311 if (s->prefix & PREFIX_LOCK) { 3312 tcg_gen_add_tl(s->T0, s->T1, c_in); 3313 tcg_gen_neg_tl(s->T0, s->T0); 3314 tcg_gen_atomic_add_fetch_tl(s->T0, s->A0, s->T0, 3315 s->mem_index, ot | MO_LE); 3316 } else { 3317 /* 3318 * TODO: SBB reg, reg could use gen_prepare_eflags_c followed by 3319 * negsetcond, and CC_OP_SUBB as the cc_op. 3320 */ 3321 tcg_gen_sub_tl(s->T0, s->T0, s->T1); 3322 tcg_gen_sub_tl(s->T0, s->T0, c_in); 3323 } 3324 prepare_update3_cc(decode, s, CC_OP_SBBB + ot, c_in); 3325} 3326 3327static void gen_SCAS(DisasContext *s, CPUX86State *env, X86DecodedInsn *decode) 3328{ 3329 MemOp ot = decode->op[2].ot; 3330 if (s->prefix & (PREFIX_REPZ | PREFIX_REPNZ)) { 3331 gen_repz_nz(s, ot, gen_scas); 3332 } else { 3333 gen_scas(s, ot); 3334 } 3335} 3336 3337static void gen_SETcc(DisasContext *s, CPUX86State *env, X86DecodedInsn *decode) 3338{ 3339 gen_setcc1(s, decode->b & 0xf, s->T0); 3340} 3341 3342static void gen_SHA1NEXTE(DisasContext *s, CPUX86State *env, X86DecodedInsn *decode) 3343{ 3344 gen_helper_sha1nexte(OP_PTR0, OP_PTR1, OP_PTR2); 3345} 3346 3347static void gen_SHA1MSG1(DisasContext *s, CPUX86State *env, X86DecodedInsn *decode) 3348{ 3349 gen_helper_sha1msg1(OP_PTR0, OP_PTR1, OP_PTR2); 3350} 3351 3352static void gen_SHA1MSG2(DisasContext *s, CPUX86State *env, X86DecodedInsn *decode) 3353{ 3354 gen_helper_sha1msg2(OP_PTR0, OP_PTR1, OP_PTR2); 3355} 3356 3357static void gen_SHA1RNDS4(DisasContext *s, CPUX86State *env, X86DecodedInsn *decode) 3358{ 3359 switch(decode->immediate & 3) { 3360 case 0: 3361 gen_helper_sha1rnds4_f0(OP_PTR0, OP_PTR0, OP_PTR1); 3362 break; 3363 case 1: 3364 gen_helper_sha1rnds4_f1(OP_PTR0, OP_PTR0, OP_PTR1); 3365 break; 3366 case 2: 3367 gen_helper_sha1rnds4_f2(OP_PTR0, OP_PTR0, OP_PTR1); 3368 break; 3369 case 3: 3370 gen_helper_sha1rnds4_f3(OP_PTR0, OP_PTR0, OP_PTR1); 3371 break; 3372 } 3373} 3374 3375static void gen_SHA256MSG1(DisasContext *s, CPUX86State *env, X86DecodedInsn *decode) 3376{ 3377 gen_helper_sha256msg1(OP_PTR0, OP_PTR1, OP_PTR2); 3378} 3379 3380static void gen_SHA256MSG2(DisasContext *s, CPUX86State *env, X86DecodedInsn *decode) 3381{ 3382 gen_helper_sha256msg2(OP_PTR0, OP_PTR1, OP_PTR2); 3383} 3384 3385static void gen_SHA256RNDS2(DisasContext *s, CPUX86State *env, X86DecodedInsn *decode) 3386{ 3387 TCGv_i32 wk0 = tcg_temp_new_i32(); 3388 TCGv_i32 wk1 = tcg_temp_new_i32(); 3389 3390 tcg_gen_ld_i32(wk0, tcg_env, ZMM_OFFSET(0) + offsetof(ZMMReg, ZMM_L(0))); 3391 tcg_gen_ld_i32(wk1, tcg_env, ZMM_OFFSET(0) + offsetof(ZMMReg, ZMM_L(1))); 3392 3393 gen_helper_sha256rnds2(OP_PTR0, OP_PTR1, OP_PTR2, wk0, wk1); 3394} 3395 3396static void gen_SHL(DisasContext *s, CPUX86State *env, X86DecodedInsn *decode) 3397{ 3398 bool can_be_zero; 3399 TCGv count; 3400 MemOp ot = gen_shift_count(s, decode, &can_be_zero, &count); 3401 3402 if (!count) { 3403 return; 3404 } 3405 3406 decode->cc_dst = s->T0; 3407 decode->cc_src = tcg_temp_new(); 3408 tcg_gen_subi_tl(decode->cc_src, count, 1); 3409 tcg_gen_shl_tl(decode->cc_src, s->T0, decode->cc_src); 3410 tcg_gen_shl_tl(s->T0, s->T0, count); 3411 if (can_be_zero) { 3412 gen_shift_dynamic_flags(s, decode, count, CC_OP_SHLB + ot); 3413 } else { 3414 decode->cc_op = CC_OP_SHLB + ot; 3415 } 3416} 3417 3418static void gen_SHLX(DisasContext *s, CPUX86State *env, X86DecodedInsn *decode) 3419{ 3420 MemOp ot = decode->op[0].ot; 3421 int mask; 3422 3423 mask = ot == MO_64 ? 63 : 31; 3424 tcg_gen_andi_tl(s->T1, s->T1, mask); 3425 tcg_gen_shl_tl(s->T0, s->T0, s->T1); 3426} 3427 3428static void gen_SHR(DisasContext *s, CPUX86State *env, X86DecodedInsn *decode) 3429{ 3430 bool can_be_zero; 3431 TCGv count; 3432 MemOp ot = gen_shift_count(s, decode, &can_be_zero, &count); 3433 3434 if (!count) { 3435 return; 3436 } 3437 3438 decode->cc_dst = s->T0; 3439 decode->cc_src = tcg_temp_new(); 3440 tcg_gen_subi_tl(decode->cc_src, count, 1); 3441 tcg_gen_shr_tl(decode->cc_src, s->T0, decode->cc_src); 3442 tcg_gen_shr_tl(s->T0, s->T0, count); 3443 if (can_be_zero) { 3444 gen_shift_dynamic_flags(s, decode, count, CC_OP_SARB + ot); 3445 } else { 3446 decode->cc_op = CC_OP_SARB + ot; 3447 } 3448} 3449 3450static void gen_SHRX(DisasContext *s, CPUX86State *env, X86DecodedInsn *decode) 3451{ 3452 MemOp ot = decode->op[0].ot; 3453 int mask; 3454 3455 mask = ot == MO_64 ? 63 : 31; 3456 tcg_gen_andi_tl(s->T1, s->T1, mask); 3457 tcg_gen_shr_tl(s->T0, s->T0, s->T1); 3458} 3459 3460static void gen_STC(DisasContext *s, CPUX86State *env, X86DecodedInsn *decode) 3461{ 3462 gen_compute_eflags(s); 3463 tcg_gen_ori_tl(cpu_cc_src, cpu_cc_src, CC_C); 3464} 3465 3466static void gen_STD(DisasContext *s, CPUX86State *env, X86DecodedInsn *decode) 3467{ 3468 tcg_gen_st_i32(tcg_constant_i32(-1), tcg_env, offsetof(CPUX86State, df)); 3469} 3470 3471static void gen_STI(DisasContext *s, CPUX86State *env, X86DecodedInsn *decode) 3472{ 3473 gen_set_eflags(s, IF_MASK); 3474 s->base.is_jmp = DISAS_EOB_INHIBIT_IRQ; 3475} 3476 3477static void gen_VAESKEYGEN(DisasContext *s, CPUX86State *env, X86DecodedInsn *decode) 3478{ 3479 TCGv_i32 imm = tcg_constant8u_i32(decode->immediate); 3480 assert(!s->vex_l); 3481 gen_helper_aeskeygenassist_xmm(tcg_env, OP_PTR0, OP_PTR1, imm); 3482} 3483 3484static void gen_STMXCSR(DisasContext *s, CPUX86State *env, X86DecodedInsn *decode) 3485{ 3486 gen_helper_update_mxcsr(tcg_env); 3487 tcg_gen_ld32u_tl(s->T0, tcg_env, offsetof(CPUX86State, mxcsr)); 3488} 3489 3490static void gen_STOS(DisasContext *s, CPUX86State *env, X86DecodedInsn *decode) 3491{ 3492 MemOp ot = decode->op[1].ot; 3493 if (s->prefix & (PREFIX_REPZ | PREFIX_REPNZ)) { 3494 gen_repz(s, ot, gen_stos); 3495 } else { 3496 gen_stos(s, ot); 3497 } 3498} 3499 3500static void gen_SUB(DisasContext *s, CPUX86State *env, X86DecodedInsn *decode) 3501{ 3502 MemOp ot = decode->op[1].ot; 3503 3504 if (s->prefix & PREFIX_LOCK) { 3505 tcg_gen_neg_tl(s->T0, s->T1); 3506 tcg_gen_atomic_fetch_add_tl(s->cc_srcT, s->A0, s->T0, 3507 s->mem_index, ot | MO_LE); 3508 tcg_gen_sub_tl(s->T0, s->cc_srcT, s->T1); 3509 } else { 3510 tcg_gen_mov_tl(s->cc_srcT, s->T0); 3511 tcg_gen_sub_tl(s->T0, s->T0, s->T1); 3512 } 3513 prepare_update2_cc(decode, s, CC_OP_SUBB + ot); 3514} 3515 3516static void gen_UD(DisasContext *s, CPUX86State *env, X86DecodedInsn *decode) 3517{ 3518 gen_illegal_opcode(s); 3519} 3520 3521static void gen_VAESIMC(DisasContext *s, CPUX86State *env, X86DecodedInsn *decode) 3522{ 3523 assert(!s->vex_l); 3524 gen_helper_aesimc_xmm(tcg_env, OP_PTR0, OP_PTR2); 3525} 3526 3527/* 3528 * 00 = v*ps Vps, Hps, Wpd 3529 * 66 = v*pd Vpd, Hpd, Wps 3530 * f3 = v*ss Vss, Hss, Wps 3531 * f2 = v*sd Vsd, Hsd, Wps 3532 */ 3533#define SSE_CMP(x) { \ 3534 gen_helper_ ## x ## ps ## _xmm, gen_helper_ ## x ## pd ## _xmm, \ 3535 gen_helper_ ## x ## ss, gen_helper_ ## x ## sd, \ 3536 gen_helper_ ## x ## ps ## _ymm, gen_helper_ ## x ## pd ## _ymm} 3537static const SSEFunc_0_eppp gen_helper_cmp_funcs[32][6] = { 3538 SSE_CMP(cmpeq), 3539 SSE_CMP(cmplt), 3540 SSE_CMP(cmple), 3541 SSE_CMP(cmpunord), 3542 SSE_CMP(cmpneq), 3543 SSE_CMP(cmpnlt), 3544 SSE_CMP(cmpnle), 3545 SSE_CMP(cmpord), 3546 3547 SSE_CMP(cmpequ), 3548 SSE_CMP(cmpnge), 3549 SSE_CMP(cmpngt), 3550 SSE_CMP(cmpfalse), 3551 SSE_CMP(cmpnequ), 3552 SSE_CMP(cmpge), 3553 SSE_CMP(cmpgt), 3554 SSE_CMP(cmptrue), 3555 3556 SSE_CMP(cmpeqs), 3557 SSE_CMP(cmpltq), 3558 SSE_CMP(cmpleq), 3559 SSE_CMP(cmpunords), 3560 SSE_CMP(cmpneqq), 3561 SSE_CMP(cmpnltq), 3562 SSE_CMP(cmpnleq), 3563 SSE_CMP(cmpords), 3564 3565 SSE_CMP(cmpequs), 3566 SSE_CMP(cmpngeq), 3567 SSE_CMP(cmpngtq), 3568 SSE_CMP(cmpfalses), 3569 SSE_CMP(cmpnequs), 3570 SSE_CMP(cmpgeq), 3571 SSE_CMP(cmpgtq), 3572 SSE_CMP(cmptrues), 3573}; 3574#undef SSE_CMP 3575 3576static void gen_VCMP(DisasContext *s, CPUX86State *env, X86DecodedInsn *decode) 3577{ 3578 int index = decode->immediate & (s->prefix & PREFIX_VEX ? 31 : 7); 3579 int b = 3580 s->prefix & PREFIX_REPZ ? 2 /* ss */ : 3581 s->prefix & PREFIX_REPNZ ? 3 /* sd */ : 3582 !!(s->prefix & PREFIX_DATA) /* pd */ + (s->vex_l << 2); 3583 3584 gen_helper_cmp_funcs[index][b](tcg_env, OP_PTR0, OP_PTR1, OP_PTR2); 3585} 3586 3587static void gen_VCOMI(DisasContext *s, CPUX86State *env, X86DecodedInsn *decode) 3588{ 3589 SSEFunc_0_epp fn; 3590 fn = s->prefix & PREFIX_DATA ? gen_helper_comisd : gen_helper_comiss; 3591 fn(tcg_env, OP_PTR1, OP_PTR2); 3592 assume_cc_op(s, CC_OP_EFLAGS); 3593} 3594 3595static void gen_VCVTPD2PS(DisasContext *s, CPUX86State *env, X86DecodedInsn *decode) 3596{ 3597 if (s->vex_l) { 3598 gen_helper_cvtpd2ps_ymm(tcg_env, OP_PTR0, OP_PTR2); 3599 } else { 3600 gen_helper_cvtpd2ps_xmm(tcg_env, OP_PTR0, OP_PTR2); 3601 } 3602} 3603 3604static void gen_VCVTPS2PD(DisasContext *s, CPUX86State *env, X86DecodedInsn *decode) 3605{ 3606 if (s->vex_l) { 3607 gen_helper_cvtps2pd_ymm(tcg_env, OP_PTR0, OP_PTR2); 3608 } else { 3609 gen_helper_cvtps2pd_xmm(tcg_env, OP_PTR0, OP_PTR2); 3610 } 3611} 3612 3613static void gen_VCVTPS2PH(DisasContext *s, CPUX86State *env, X86DecodedInsn *decode) 3614{ 3615 gen_unary_imm_fp_sse(s, env, decode, 3616 gen_helper_cvtps2ph_xmm, 3617 gen_helper_cvtps2ph_ymm); 3618 /* 3619 * VCVTPS2PH is the only instruction that performs an operation on a 3620 * register source and then *stores* into memory. 3621 */ 3622 if (decode->op[0].has_ea) { 3623 gen_store_sse(s, decode, decode->op[0].offset); 3624 } 3625} 3626 3627static void gen_VCVTSD2SS(DisasContext *s, CPUX86State *env, X86DecodedInsn *decode) 3628{ 3629 gen_helper_cvtsd2ss(tcg_env, OP_PTR0, OP_PTR1, OP_PTR2); 3630} 3631 3632static void gen_VCVTSS2SD(DisasContext *s, CPUX86State *env, X86DecodedInsn *decode) 3633{ 3634 gen_helper_cvtss2sd(tcg_env, OP_PTR0, OP_PTR1, OP_PTR2); 3635} 3636 3637static void gen_VCVTSI2Sx(DisasContext *s, CPUX86State *env, X86DecodedInsn *decode) 3638{ 3639 int vec_len = vector_len(s, decode); 3640 TCGv_i32 in; 3641 3642 tcg_gen_gvec_mov(MO_64, decode->op[0].offset, decode->op[1].offset, vec_len, vec_len); 3643 3644#ifdef TARGET_X86_64 3645 MemOp ot = decode->op[2].ot; 3646 if (ot == MO_64) { 3647 if (s->prefix & PREFIX_REPNZ) { 3648 gen_helper_cvtsq2sd(tcg_env, OP_PTR0, s->T1); 3649 } else { 3650 gen_helper_cvtsq2ss(tcg_env, OP_PTR0, s->T1); 3651 } 3652 return; 3653 } 3654 in = s->tmp2_i32; 3655 tcg_gen_trunc_tl_i32(in, s->T1); 3656#else 3657 in = s->T1; 3658#endif 3659 3660 if (s->prefix & PREFIX_REPNZ) { 3661 gen_helper_cvtsi2sd(tcg_env, OP_PTR0, in); 3662 } else { 3663 gen_helper_cvtsi2ss(tcg_env, OP_PTR0, in); 3664 } 3665} 3666 3667static inline void gen_VCVTtSx2SI(DisasContext *s, CPUX86State *env, X86DecodedInsn *decode, 3668 SSEFunc_i_ep ss2si, SSEFunc_l_ep ss2sq, 3669 SSEFunc_i_ep sd2si, SSEFunc_l_ep sd2sq) 3670{ 3671 TCGv_i32 out; 3672 3673#ifdef TARGET_X86_64 3674 MemOp ot = decode->op[0].ot; 3675 if (ot == MO_64) { 3676 if (s->prefix & PREFIX_REPNZ) { 3677 sd2sq(s->T0, tcg_env, OP_PTR2); 3678 } else { 3679 ss2sq(s->T0, tcg_env, OP_PTR2); 3680 } 3681 return; 3682 } 3683 3684 out = s->tmp2_i32; 3685#else 3686 out = s->T0; 3687#endif 3688 if (s->prefix & PREFIX_REPNZ) { 3689 sd2si(out, tcg_env, OP_PTR2); 3690 } else { 3691 ss2si(out, tcg_env, OP_PTR2); 3692 } 3693#ifdef TARGET_X86_64 3694 tcg_gen_extu_i32_tl(s->T0, out); 3695#endif 3696} 3697 3698#ifndef TARGET_X86_64 3699#define gen_helper_cvtss2sq NULL 3700#define gen_helper_cvtsd2sq NULL 3701#define gen_helper_cvttss2sq NULL 3702#define gen_helper_cvttsd2sq NULL 3703#endif 3704 3705static void gen_VCVTSx2SI(DisasContext *s, CPUX86State *env, X86DecodedInsn *decode) 3706{ 3707 gen_VCVTtSx2SI(s, env, decode, 3708 gen_helper_cvtss2si, gen_helper_cvtss2sq, 3709 gen_helper_cvtsd2si, gen_helper_cvtsd2sq); 3710} 3711 3712static void gen_VCVTTSx2SI(DisasContext *s, CPUX86State *env, X86DecodedInsn *decode) 3713{ 3714 gen_VCVTtSx2SI(s, env, decode, 3715 gen_helper_cvttss2si, gen_helper_cvttss2sq, 3716 gen_helper_cvttsd2si, gen_helper_cvttsd2sq); 3717} 3718 3719static void gen_VEXTRACTx128(DisasContext *s, CPUX86State *env, X86DecodedInsn *decode) 3720{ 3721 int mask = decode->immediate & 1; 3722 int src_ofs = vector_elem_offset(&decode->op[1], MO_128, mask); 3723 if (decode->op[0].has_ea) { 3724 /* VEX-only instruction, no alignment requirements. */ 3725 gen_sto_env_A0(s, src_ofs, false); 3726 } else { 3727 tcg_gen_gvec_mov(MO_64, decode->op[0].offset, src_ofs, 16, 16); 3728 } 3729} 3730 3731static void gen_VEXTRACTPS(DisasContext *s, CPUX86State *env, X86DecodedInsn *decode) 3732{ 3733 gen_pextr(s, env, decode, MO_32); 3734} 3735 3736static void gen_vinsertps(DisasContext *s, CPUX86State *env, X86DecodedInsn *decode) 3737{ 3738 int val = decode->immediate; 3739 int dest_word = (val >> 4) & 3; 3740 int new_mask = (val & 15) | (1 << dest_word); 3741 int vec_len = 16; 3742 3743 assert(!s->vex_l); 3744 3745 if (new_mask == 15) { 3746 /* All zeroes except possibly for the inserted element */ 3747 tcg_gen_gvec_dup_imm(MO_64, decode->op[0].offset, vec_len, vec_len, 0); 3748 } else if (decode->op[1].offset != decode->op[0].offset) { 3749 gen_store_sse(s, decode, decode->op[1].offset); 3750 } 3751 3752 if (new_mask != (val & 15)) { 3753 tcg_gen_st_i32(s->tmp2_i32, tcg_env, 3754 vector_elem_offset(&decode->op[0], MO_32, dest_word)); 3755 } 3756 3757 if (new_mask != 15) { 3758 TCGv_i32 zero = tcg_constant_i32(0); /* float32_zero */ 3759 int i; 3760 for (i = 0; i < 4; i++) { 3761 if ((val >> i) & 1) { 3762 tcg_gen_st_i32(zero, tcg_env, 3763 vector_elem_offset(&decode->op[0], MO_32, i)); 3764 } 3765 } 3766 } 3767} 3768 3769static void gen_VINSERTPS_r(DisasContext *s, CPUX86State *env, X86DecodedInsn *decode) 3770{ 3771 int val = decode->immediate; 3772 tcg_gen_ld_i32(s->tmp2_i32, tcg_env, 3773 vector_elem_offset(&decode->op[2], MO_32, (val >> 6) & 3)); 3774 gen_vinsertps(s, env, decode); 3775} 3776 3777static void gen_VINSERTPS_m(DisasContext *s, CPUX86State *env, X86DecodedInsn *decode) 3778{ 3779 tcg_gen_qemu_ld_i32(s->tmp2_i32, s->A0, s->mem_index, MO_LEUL); 3780 gen_vinsertps(s, env, decode); 3781} 3782 3783static void gen_VINSERTx128(DisasContext *s, CPUX86State *env, X86DecodedInsn *decode) 3784{ 3785 int mask = decode->immediate & 1; 3786 tcg_gen_gvec_mov(MO_64, 3787 decode->op[0].offset + offsetof(YMMReg, YMM_X(mask)), 3788 decode->op[2].offset + offsetof(YMMReg, YMM_X(0)), 16, 16); 3789 tcg_gen_gvec_mov(MO_64, 3790 decode->op[0].offset + offsetof(YMMReg, YMM_X(!mask)), 3791 decode->op[1].offset + offsetof(YMMReg, YMM_X(!mask)), 16, 16); 3792} 3793 3794static inline void gen_maskmov(DisasContext *s, CPUX86State *env, X86DecodedInsn *decode, 3795 SSEFunc_0_eppt xmm, SSEFunc_0_eppt ymm) 3796{ 3797 if (!s->vex_l) { 3798 xmm(tcg_env, OP_PTR2, OP_PTR1, s->A0); 3799 } else { 3800 ymm(tcg_env, OP_PTR2, OP_PTR1, s->A0); 3801 } 3802} 3803 3804static void gen_VMASKMOVPD_st(DisasContext *s, CPUX86State *env, X86DecodedInsn *decode) 3805{ 3806 gen_maskmov(s, env, decode, gen_helper_vpmaskmovq_st_xmm, gen_helper_vpmaskmovq_st_ymm); 3807} 3808 3809static void gen_VMASKMOVPS_st(DisasContext *s, CPUX86State *env, X86DecodedInsn *decode) 3810{ 3811 gen_maskmov(s, env, decode, gen_helper_vpmaskmovd_st_xmm, gen_helper_vpmaskmovd_st_ymm); 3812} 3813 3814static void gen_VMOVHPx_ld(DisasContext *s, CPUX86State *env, X86DecodedInsn *decode) 3815{ 3816 gen_ldq_env_A0(s, decode->op[0].offset + offsetof(XMMReg, XMM_Q(1))); 3817 if (decode->op[0].offset != decode->op[1].offset) { 3818 tcg_gen_ld_i64(s->tmp1_i64, tcg_env, decode->op[1].offset + offsetof(XMMReg, XMM_Q(0))); 3819 tcg_gen_st_i64(s->tmp1_i64, tcg_env, decode->op[0].offset + offsetof(XMMReg, XMM_Q(0))); 3820 } 3821} 3822 3823static void gen_VMOVHPx_st(DisasContext *s, CPUX86State *env, X86DecodedInsn *decode) 3824{ 3825 gen_stq_env_A0(s, decode->op[2].offset + offsetof(XMMReg, XMM_Q(1))); 3826} 3827 3828static void gen_VMOVHPx(DisasContext *s, CPUX86State *env, X86DecodedInsn *decode) 3829{ 3830 if (decode->op[0].offset != decode->op[2].offset) { 3831 tcg_gen_ld_i64(s->tmp1_i64, tcg_env, decode->op[2].offset + offsetof(XMMReg, XMM_Q(1))); 3832 tcg_gen_st_i64(s->tmp1_i64, tcg_env, decode->op[0].offset + offsetof(XMMReg, XMM_Q(1))); 3833 } 3834 if (decode->op[0].offset != decode->op[1].offset) { 3835 tcg_gen_ld_i64(s->tmp1_i64, tcg_env, decode->op[1].offset + offsetof(XMMReg, XMM_Q(0))); 3836 tcg_gen_st_i64(s->tmp1_i64, tcg_env, decode->op[0].offset + offsetof(XMMReg, XMM_Q(0))); 3837 } 3838} 3839 3840static void gen_VMOVHLPS(DisasContext *s, CPUX86State *env, X86DecodedInsn *decode) 3841{ 3842 tcg_gen_ld_i64(s->tmp1_i64, tcg_env, decode->op[2].offset + offsetof(XMMReg, XMM_Q(1))); 3843 tcg_gen_st_i64(s->tmp1_i64, tcg_env, decode->op[0].offset + offsetof(XMMReg, XMM_Q(0))); 3844 if (decode->op[0].offset != decode->op[1].offset) { 3845 tcg_gen_ld_i64(s->tmp1_i64, tcg_env, decode->op[1].offset + offsetof(XMMReg, XMM_Q(1))); 3846 tcg_gen_st_i64(s->tmp1_i64, tcg_env, decode->op[0].offset + offsetof(XMMReg, XMM_Q(1))); 3847 } 3848} 3849 3850static void gen_VMOVLHPS(DisasContext *s, CPUX86State *env, X86DecodedInsn *decode) 3851{ 3852 tcg_gen_ld_i64(s->tmp1_i64, tcg_env, decode->op[2].offset); 3853 tcg_gen_st_i64(s->tmp1_i64, tcg_env, decode->op[0].offset + offsetof(XMMReg, XMM_Q(1))); 3854 if (decode->op[0].offset != decode->op[1].offset) { 3855 tcg_gen_ld_i64(s->tmp1_i64, tcg_env, decode->op[1].offset + offsetof(XMMReg, XMM_Q(0))); 3856 tcg_gen_st_i64(s->tmp1_i64, tcg_env, decode->op[0].offset + offsetof(XMMReg, XMM_Q(0))); 3857 } 3858} 3859 3860/* 3861 * Note that MOVLPx supports 256-bit operation unlike MOVHLPx, MOVLHPx, MOXHPx. 3862 * Use a gvec move to move everything above the bottom 64 bits. 3863 */ 3864 3865static void gen_VMOVLPx(DisasContext *s, CPUX86State *env, X86DecodedInsn *decode) 3866{ 3867 int vec_len = vector_len(s, decode); 3868 3869 tcg_gen_ld_i64(s->tmp1_i64, tcg_env, decode->op[2].offset + offsetof(XMMReg, XMM_Q(0))); 3870 tcg_gen_gvec_mov(MO_64, decode->op[0].offset, decode->op[1].offset, vec_len, vec_len); 3871 tcg_gen_st_i64(s->tmp1_i64, tcg_env, decode->op[0].offset + offsetof(XMMReg, XMM_Q(0))); 3872} 3873 3874static void gen_VMOVLPx_ld(DisasContext *s, CPUX86State *env, X86DecodedInsn *decode) 3875{ 3876 int vec_len = vector_len(s, decode); 3877 3878 tcg_gen_qemu_ld_i64(s->tmp1_i64, s->A0, s->mem_index, MO_LEUQ); 3879 tcg_gen_gvec_mov(MO_64, decode->op[0].offset, decode->op[1].offset, vec_len, vec_len); 3880 tcg_gen_st_i64(s->tmp1_i64, OP_PTR0, offsetof(ZMMReg, ZMM_Q(0))); 3881} 3882 3883static void gen_VMOVLPx_st(DisasContext *s, CPUX86State *env, X86DecodedInsn *decode) 3884{ 3885 tcg_gen_ld_i64(s->tmp1_i64, OP_PTR2, offsetof(ZMMReg, ZMM_Q(0))); 3886 tcg_gen_qemu_st_i64(s->tmp1_i64, s->A0, s->mem_index, MO_LEUQ); 3887} 3888 3889static void gen_VMOVSD_ld(DisasContext *s, CPUX86State *env, X86DecodedInsn *decode) 3890{ 3891 TCGv_i64 zero = tcg_constant_i64(0); 3892 3893 tcg_gen_qemu_ld_i64(s->tmp1_i64, s->A0, s->mem_index, MO_LEUQ); 3894 tcg_gen_st_i64(zero, OP_PTR0, offsetof(ZMMReg, ZMM_Q(1))); 3895 tcg_gen_st_i64(s->tmp1_i64, OP_PTR0, offsetof(ZMMReg, ZMM_Q(0))); 3896} 3897 3898static void gen_VMOVSS(DisasContext *s, CPUX86State *env, X86DecodedInsn *decode) 3899{ 3900 int vec_len = vector_len(s, decode); 3901 3902 tcg_gen_ld_i32(s->tmp2_i32, OP_PTR2, offsetof(ZMMReg, ZMM_L(0))); 3903 tcg_gen_gvec_mov(MO_64, decode->op[0].offset, decode->op[1].offset, vec_len, vec_len); 3904 tcg_gen_st_i32(s->tmp2_i32, OP_PTR0, offsetof(ZMMReg, ZMM_L(0))); 3905} 3906 3907static void gen_VMOVSS_ld(DisasContext *s, CPUX86State *env, X86DecodedInsn *decode) 3908{ 3909 int vec_len = vector_len(s, decode); 3910 3911 tcg_gen_qemu_ld_i32(s->tmp2_i32, s->A0, s->mem_index, MO_LEUL); 3912 tcg_gen_gvec_dup_imm(MO_64, decode->op[0].offset, vec_len, vec_len, 0); 3913 tcg_gen_st_i32(s->tmp2_i32, OP_PTR0, offsetof(ZMMReg, ZMM_L(0))); 3914} 3915 3916static void gen_VMOVSS_st(DisasContext *s, CPUX86State *env, X86DecodedInsn *decode) 3917{ 3918 tcg_gen_ld_i32(s->tmp2_i32, OP_PTR2, offsetof(ZMMReg, ZMM_L(0))); 3919 tcg_gen_qemu_st_i32(s->tmp2_i32, s->A0, s->mem_index, MO_LEUL); 3920} 3921 3922static void gen_VPMASKMOV_st(DisasContext *s, CPUX86State *env, X86DecodedInsn *decode) 3923{ 3924 if (s->vex_w) { 3925 gen_VMASKMOVPD_st(s, env, decode); 3926 } else { 3927 gen_VMASKMOVPS_st(s, env, decode); 3928 } 3929} 3930 3931static void gen_VPERMD(DisasContext *s, CPUX86State *env, X86DecodedInsn *decode) 3932{ 3933 assert(s->vex_l); 3934 gen_helper_vpermd_ymm(OP_PTR0, OP_PTR1, OP_PTR2); 3935} 3936 3937static void gen_VPERM2x128(DisasContext *s, CPUX86State *env, X86DecodedInsn *decode) 3938{ 3939 TCGv_i32 imm = tcg_constant8u_i32(decode->immediate); 3940 assert(s->vex_l); 3941 gen_helper_vpermdq_ymm(OP_PTR0, OP_PTR1, OP_PTR2, imm); 3942} 3943 3944static void gen_VPHMINPOSUW(DisasContext *s, CPUX86State *env, X86DecodedInsn *decode) 3945{ 3946 assert(!s->vex_l); 3947 gen_helper_phminposuw_xmm(tcg_env, OP_PTR0, OP_PTR2); 3948} 3949 3950static void gen_VROUNDSD(DisasContext *s, CPUX86State *env, X86DecodedInsn *decode) 3951{ 3952 TCGv_i32 imm = tcg_constant8u_i32(decode->immediate); 3953 assert(!s->vex_l); 3954 gen_helper_roundsd_xmm(tcg_env, OP_PTR0, OP_PTR1, OP_PTR2, imm); 3955} 3956 3957static void gen_VROUNDSS(DisasContext *s, CPUX86State *env, X86DecodedInsn *decode) 3958{ 3959 TCGv_i32 imm = tcg_constant8u_i32(decode->immediate); 3960 assert(!s->vex_l); 3961 gen_helper_roundss_xmm(tcg_env, OP_PTR0, OP_PTR1, OP_PTR2, imm); 3962} 3963 3964static void gen_VSHUF(DisasContext *s, CPUX86State *env, X86DecodedInsn *decode) 3965{ 3966 TCGv_i32 imm = tcg_constant_i32(decode->immediate); 3967 SSEFunc_0_pppi ps, pd, fn; 3968 ps = s->vex_l ? gen_helper_shufps_ymm : gen_helper_shufps_xmm; 3969 pd = s->vex_l ? gen_helper_shufpd_ymm : gen_helper_shufpd_xmm; 3970 fn = s->prefix & PREFIX_DATA ? pd : ps; 3971 fn(OP_PTR0, OP_PTR1, OP_PTR2, imm); 3972} 3973 3974static void gen_VUCOMI(DisasContext *s, CPUX86State *env, X86DecodedInsn *decode) 3975{ 3976 SSEFunc_0_epp fn; 3977 fn = s->prefix & PREFIX_DATA ? gen_helper_ucomisd : gen_helper_ucomiss; 3978 fn(tcg_env, OP_PTR1, OP_PTR2); 3979 assume_cc_op(s, CC_OP_EFLAGS); 3980} 3981 3982static void gen_VZEROALL(DisasContext *s, CPUX86State *env, X86DecodedInsn *decode) 3983{ 3984 TCGv_ptr ptr = tcg_temp_new_ptr(); 3985 3986 tcg_gen_addi_ptr(ptr, tcg_env, offsetof(CPUX86State, xmm_regs)); 3987 gen_helper_memset(ptr, ptr, tcg_constant_i32(0), 3988 tcg_constant_ptr(CPU_NB_REGS * sizeof(ZMMReg))); 3989} 3990 3991static void gen_VZEROUPPER(DisasContext *s, CPUX86State *env, X86DecodedInsn *decode) 3992{ 3993 int i; 3994 3995 for (i = 0; i < CPU_NB_REGS; i++) { 3996 int offset = offsetof(CPUX86State, xmm_regs[i].ZMM_X(1)); 3997 tcg_gen_gvec_dup_imm(MO_64, offset, 16, 16, 0); 3998 } 3999} 4000 4001static void gen_WAIT(DisasContext *s, CPUX86State *env, X86DecodedInsn *decode) 4002{ 4003 if ((s->flags & (HF_MP_MASK | HF_TS_MASK)) == (HF_MP_MASK | HF_TS_MASK)) { 4004 gen_NM_exception(s); 4005 } else { 4006 /* needs to be treated as I/O because of ferr_irq */ 4007 translator_io_start(&s->base); 4008 gen_helper_fwait(tcg_env); 4009 } 4010} 4011 4012static void gen_XCHG(DisasContext *s, CPUX86State *env, X86DecodedInsn *decode) 4013{ 4014 if (decode->b == 0x90 && !REX_B(s)) { 4015 if (s->prefix & PREFIX_REPZ) { 4016 gen_update_cc_op(s); 4017 gen_update_eip_cur(s); 4018 gen_helper_pause(tcg_env, cur_insn_len_i32(s)); 4019 s->base.is_jmp = DISAS_NORETURN; 4020 } 4021 /* No writeback. */ 4022 decode->op[0].unit = X86_OP_SKIP; 4023 return; 4024 } 4025 4026 if (s->prefix & PREFIX_LOCK) { 4027 tcg_gen_atomic_xchg_tl(s->T0, s->A0, s->T1, 4028 s->mem_index, decode->op[0].ot | MO_LE); 4029 /* now store old value into register operand */ 4030 gen_op_mov_reg_v(s, decode->op[2].ot, decode->op[2].n, s->T0); 4031 } else { 4032 /* move destination value into source operand, source preserved in T1 */ 4033 gen_op_mov_reg_v(s, decode->op[2].ot, decode->op[2].n, s->T0); 4034 tcg_gen_mov_tl(s->T0, s->T1); 4035 } 4036} 4037 4038static void gen_XLAT(DisasContext *s, CPUX86State *env, X86DecodedInsn *decode) 4039{ 4040 /* AL is already zero-extended into s->T0. */ 4041 tcg_gen_add_tl(s->A0, cpu_regs[R_EBX], s->T0); 4042 gen_lea_v_seg(s, s->A0, R_DS, s->override); 4043 gen_op_ld_v(s, MO_8, s->T0, s->A0); 4044} 4045 4046static void gen_XOR(DisasContext *s, CPUX86State *env, X86DecodedInsn *decode) 4047{ 4048 /* special case XOR reg, reg */ 4049 if (decode->op[1].unit == X86_OP_INT && 4050 decode->op[2].unit == X86_OP_INT && 4051 decode->op[1].n == decode->op[2].n) { 4052 tcg_gen_movi_tl(s->T0, 0); 4053 decode->cc_op = CC_OP_CLR; 4054 } else { 4055 MemOp ot = decode->op[1].ot; 4056 4057 if (s->prefix & PREFIX_LOCK) { 4058 tcg_gen_atomic_xor_fetch_tl(s->T0, s->A0, s->T1, 4059 s->mem_index, ot | MO_LE); 4060 } else { 4061 tcg_gen_xor_tl(s->T0, s->T0, s->T1); 4062 } 4063 prepare_update1_cc(decode, s, CC_OP_LOGICB + ot); 4064 } 4065} 4066