1/* 2 * New-style TCG opcode generator for i386 instructions 3 * 4 * Copyright (c) 2022 Red Hat, Inc. 5 * 6 * Author: Paolo Bonzini <pbonzini@redhat.com> 7 * 8 * This library is free software; you can redistribute it and/or 9 * modify it under the terms of the GNU Lesser General Public 10 * License as published by the Free Software Foundation; either 11 * version 2.1 of the License, or (at your option) any later version. 12 * 13 * This library is distributed in the hope that it will be useful, 14 * but WITHOUT ANY WARRANTY; without even the implied warranty of 15 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU 16 * Lesser General Public License for more details. 17 * 18 * You should have received a copy of the GNU Lesser General Public 19 * License along with this library; if not, see <http://www.gnu.org/licenses/>. 20 */ 21 22/* 23 * Sometimes, knowing what the backend has can produce better code. 24 * The exact opcode to check depends on 32- vs. 64-bit. 25 */ 26#ifdef TARGET_X86_64 27#define TCG_TARGET_HAS_extract2_tl TCG_TARGET_HAS_extract2_i64 28#define TCG_TARGET_deposit_tl_valid TCG_TARGET_deposit_i64_valid 29#define TCG_TARGET_extract_tl_valid TCG_TARGET_extract_i64_valid 30#else 31#define TCG_TARGET_HAS_extract2_tl TCG_TARGET_HAS_extract2_i32 32#define TCG_TARGET_deposit_tl_valid TCG_TARGET_deposit_i32_valid 33#define TCG_TARGET_extract_tl_valid TCG_TARGET_extract_i32_valid 34#endif 35 36#define MMX_OFFSET(reg) \ 37 ({ assert((reg) >= 0 && (reg) <= 7); \ 38 offsetof(CPUX86State, fpregs[reg].mmx); }) 39 40#define ZMM_OFFSET(reg) \ 41 ({ assert((reg) >= 0 && (reg) <= 15); \ 42 offsetof(CPUX86State, xmm_regs[reg]); }) 43 44typedef void (*SSEFunc_i_ep)(TCGv_i32 val, TCGv_ptr env, TCGv_ptr reg); 45typedef void (*SSEFunc_l_ep)(TCGv_i64 val, TCGv_ptr env, TCGv_ptr reg); 46typedef void (*SSEFunc_0_epp)(TCGv_ptr env, TCGv_ptr reg_a, TCGv_ptr reg_b); 47typedef void (*SSEFunc_0_eppp)(TCGv_ptr env, TCGv_ptr reg_a, TCGv_ptr reg_b, 48 TCGv_ptr reg_c); 49typedef void (*SSEFunc_0_epppp)(TCGv_ptr env, TCGv_ptr reg_a, TCGv_ptr reg_b, 50 TCGv_ptr reg_c, TCGv_ptr reg_d); 51typedef void (*SSEFunc_0_eppi)(TCGv_ptr env, TCGv_ptr reg_a, TCGv_ptr reg_b, 52 TCGv_i32 val); 53typedef void (*SSEFunc_0_epppi)(TCGv_ptr env, TCGv_ptr reg_a, TCGv_ptr reg_b, 54 TCGv_ptr reg_c, TCGv_i32 val); 55typedef void (*SSEFunc_0_ppi)(TCGv_ptr reg_a, TCGv_ptr reg_b, TCGv_i32 val); 56typedef void (*SSEFunc_0_pppi)(TCGv_ptr reg_a, TCGv_ptr reg_b, TCGv_ptr reg_c, 57 TCGv_i32 val); 58typedef void (*SSEFunc_0_eppt)(TCGv_ptr env, TCGv_ptr reg_a, TCGv_ptr reg_b, 59 TCGv val); 60typedef void (*SSEFunc_0_epppti)(TCGv_ptr env, TCGv_ptr reg_a, TCGv_ptr reg_b, 61 TCGv_ptr reg_c, TCGv a0, TCGv_i32 scale); 62typedef void (*SSEFunc_0_eppppi)(TCGv_ptr env, TCGv_ptr reg_a, TCGv_ptr reg_b, 63 TCGv_ptr reg_c, TCGv_ptr reg_d, TCGv_i32 flags); 64typedef void (*SSEFunc_0_eppppii)(TCGv_ptr env, TCGv_ptr reg_a, TCGv_ptr reg_b, 65 TCGv_ptr reg_c, TCGv_ptr reg_d, TCGv_i32 even, 66 TCGv_i32 odd); 67 68static void gen_JMP_m(DisasContext *s, X86DecodedInsn *decode); 69static void gen_JMP(DisasContext *s, X86DecodedInsn *decode); 70 71static inline TCGv_i32 tcg_constant8u_i32(uint8_t val) 72{ 73 return tcg_constant_i32(val); 74} 75 76static void gen_NM_exception(DisasContext *s) 77{ 78 gen_exception(s, EXCP07_PREX); 79} 80 81static void gen_lea_modrm(DisasContext *s, X86DecodedInsn *decode) 82{ 83 AddressParts *mem = &decode->mem; 84 TCGv ea; 85 86 ea = gen_lea_modrm_1(s, *mem, decode->e.vex_class == 12); 87 if (decode->e.special == X86_SPECIAL_BitTest) { 88 MemOp ot = decode->op[1].ot; 89 int poslen = 8 << ot; 90 int opn = decode->op[2].n; 91 TCGv ofs = tcg_temp_new(); 92 93 /* Extract memory displacement from the second operand. */ 94 assert(decode->op[2].unit == X86_OP_INT && decode->op[2].ot != MO_8); 95 tcg_gen_sextract_tl(ofs, cpu_regs[opn], 3, poslen - 3); 96 tcg_gen_andi_tl(ofs, ofs, -1 << ot); 97 tcg_gen_add_tl(s->A0, ea, ofs); 98 ea = s->A0; 99 } 100 101 gen_lea_v_seg(s, ea, mem->def_seg, s->override); 102} 103 104static inline int mmx_offset(MemOp ot) 105{ 106 switch (ot) { 107 case MO_8: 108 return offsetof(MMXReg, MMX_B(0)); 109 case MO_16: 110 return offsetof(MMXReg, MMX_W(0)); 111 case MO_32: 112 return offsetof(MMXReg, MMX_L(0)); 113 case MO_64: 114 return offsetof(MMXReg, MMX_Q(0)); 115 default: 116 g_assert_not_reached(); 117 } 118} 119 120static inline int xmm_offset(MemOp ot) 121{ 122 switch (ot) { 123 case MO_8: 124 return offsetof(ZMMReg, ZMM_B(0)); 125 case MO_16: 126 return offsetof(ZMMReg, ZMM_W(0)); 127 case MO_32: 128 return offsetof(ZMMReg, ZMM_L(0)); 129 case MO_64: 130 return offsetof(ZMMReg, ZMM_Q(0)); 131 case MO_128: 132 return offsetof(ZMMReg, ZMM_X(0)); 133 case MO_256: 134 return offsetof(ZMMReg, ZMM_Y(0)); 135 default: 136 g_assert_not_reached(); 137 } 138} 139 140static int vector_reg_offset(X86DecodedOp *op) 141{ 142 assert(op->unit == X86_OP_MMX || op->unit == X86_OP_SSE); 143 144 if (op->unit == X86_OP_MMX) { 145 return op->offset - mmx_offset(op->ot); 146 } else { 147 return op->offset - xmm_offset(op->ot); 148 } 149} 150 151static int vector_elem_offset(X86DecodedOp *op, MemOp ot, int n) 152{ 153 int base_ofs = vector_reg_offset(op); 154 switch(ot) { 155 case MO_8: 156 if (op->unit == X86_OP_MMX) { 157 return base_ofs + offsetof(MMXReg, MMX_B(n)); 158 } else { 159 return base_ofs + offsetof(ZMMReg, ZMM_B(n)); 160 } 161 case MO_16: 162 if (op->unit == X86_OP_MMX) { 163 return base_ofs + offsetof(MMXReg, MMX_W(n)); 164 } else { 165 return base_ofs + offsetof(ZMMReg, ZMM_W(n)); 166 } 167 case MO_32: 168 if (op->unit == X86_OP_MMX) { 169 return base_ofs + offsetof(MMXReg, MMX_L(n)); 170 } else { 171 return base_ofs + offsetof(ZMMReg, ZMM_L(n)); 172 } 173 case MO_64: 174 if (op->unit == X86_OP_MMX) { 175 return base_ofs; 176 } else { 177 return base_ofs + offsetof(ZMMReg, ZMM_Q(n)); 178 } 179 case MO_128: 180 assert(op->unit == X86_OP_SSE); 181 return base_ofs + offsetof(ZMMReg, ZMM_X(n)); 182 case MO_256: 183 assert(op->unit == X86_OP_SSE); 184 return base_ofs + offsetof(ZMMReg, ZMM_Y(n)); 185 default: 186 g_assert_not_reached(); 187 } 188} 189 190static void compute_mmx_offset(X86DecodedOp *op) 191{ 192 if (!op->has_ea) { 193 op->offset = MMX_OFFSET(op->n) + mmx_offset(op->ot); 194 } else { 195 op->offset = offsetof(CPUX86State, mmx_t0) + mmx_offset(op->ot); 196 } 197} 198 199static void compute_xmm_offset(X86DecodedOp *op) 200{ 201 if (!op->has_ea) { 202 op->offset = ZMM_OFFSET(op->n) + xmm_offset(op->ot); 203 } else { 204 op->offset = offsetof(CPUX86State, xmm_t0) + xmm_offset(op->ot); 205 } 206} 207 208static void gen_load_sse(DisasContext *s, TCGv temp, MemOp ot, int dest_ofs, bool aligned) 209{ 210 switch(ot) { 211 case MO_8: 212 gen_op_ld_v(s, MO_8, temp, s->A0); 213 tcg_gen_st8_tl(temp, tcg_env, dest_ofs); 214 break; 215 case MO_16: 216 gen_op_ld_v(s, MO_16, temp, s->A0); 217 tcg_gen_st16_tl(temp, tcg_env, dest_ofs); 218 break; 219 case MO_32: 220 gen_op_ld_v(s, MO_32, temp, s->A0); 221 tcg_gen_st32_tl(temp, tcg_env, dest_ofs); 222 break; 223 case MO_64: 224 gen_ldq_env_A0(s, dest_ofs); 225 break; 226 case MO_128: 227 gen_ldo_env_A0(s, dest_ofs, aligned); 228 break; 229 case MO_256: 230 gen_ldy_env_A0(s, dest_ofs, aligned); 231 break; 232 default: 233 g_assert_not_reached(); 234 } 235} 236 237static bool sse_needs_alignment(DisasContext *s, X86DecodedInsn *decode, MemOp ot) 238{ 239 switch (decode->e.vex_class) { 240 case 2: 241 case 4: 242 if ((s->prefix & PREFIX_VEX) || 243 decode->e.vex_special == X86_VEX_SSEUnaligned) { 244 /* MOST legacy SSE instructions require aligned memory operands, but not all. */ 245 return false; 246 } 247 /* fall through */ 248 case 1: 249 return ot >= MO_128; 250 251 default: 252 return false; 253 } 254} 255 256static void gen_load(DisasContext *s, X86DecodedInsn *decode, int opn, TCGv v) 257{ 258 X86DecodedOp *op = &decode->op[opn]; 259 260 switch (op->unit) { 261 case X86_OP_SKIP: 262 return; 263 case X86_OP_SEG: 264 tcg_gen_ld32u_tl(v, tcg_env, 265 offsetof(CPUX86State,segs[op->n].selector)); 266 break; 267#ifndef CONFIG_USER_ONLY 268 case X86_OP_CR: 269 if (op->n == 8) { 270 translator_io_start(&s->base); 271 gen_helper_read_cr8(v, tcg_env); 272 } else { 273 tcg_gen_ld_tl(v, tcg_env, offsetof(CPUX86State, cr[op->n])); 274 } 275 break; 276 case X86_OP_DR: 277 /* CR4.DE tested in the helper. */ 278 gen_helper_get_dr(v, tcg_env, tcg_constant_i32(op->n)); 279 break; 280#endif 281 case X86_OP_INT: 282 if (op->has_ea) { 283 if (v == s->T0 && decode->e.special == X86_SPECIAL_SExtT0) { 284 gen_op_ld_v(s, op->ot | MO_SIGN, v, s->A0); 285 } else { 286 gen_op_ld_v(s, op->ot, v, s->A0); 287 } 288 289 } else if (op->ot < MO_TL && v == s->T0 && 290 (decode->e.special == X86_SPECIAL_SExtT0 || 291 decode->e.special == X86_SPECIAL_ZExtT0)) { 292 if (op->ot == MO_8 && byte_reg_is_xH(s, op->n)) { 293 if (decode->e.special == X86_SPECIAL_SExtT0) { 294 tcg_gen_sextract_tl(v, cpu_regs[op->n - 4], 8, 8); 295 } else { 296 tcg_gen_extract_tl(v, cpu_regs[op->n - 4], 8, 8); 297 } 298 } else { 299 if (decode->e.special == X86_SPECIAL_SExtT0) { 300 tcg_gen_ext_tl(v, cpu_regs[op->n], op->ot | MO_SIGN); 301 } else { 302 tcg_gen_ext_tl(v, cpu_regs[op->n], op->ot); 303 } 304 } 305 306 } else { 307 gen_op_mov_v_reg(s, op->ot, v, op->n); 308 } 309 break; 310 case X86_OP_IMM: 311 tcg_gen_movi_tl(v, op->imm); 312 break; 313 314 case X86_OP_MMX: 315 compute_mmx_offset(op); 316 goto load_vector; 317 318 case X86_OP_SSE: 319 compute_xmm_offset(op); 320 load_vector: 321 if (op->has_ea) { 322 bool aligned = sse_needs_alignment(s, decode, op->ot); 323 gen_load_sse(s, v, op->ot, op->offset, aligned); 324 } 325 break; 326 327 default: 328 g_assert_not_reached(); 329 } 330} 331 332static TCGv_ptr op_ptr(X86DecodedInsn *decode, int opn) 333{ 334 X86DecodedOp *op = &decode->op[opn]; 335 336 assert(op->unit == X86_OP_MMX || op->unit == X86_OP_SSE); 337 if (op->v_ptr) { 338 return op->v_ptr; 339 } 340 op->v_ptr = tcg_temp_new_ptr(); 341 342 /* The temporary points to the MMXReg or ZMMReg. */ 343 tcg_gen_addi_ptr(op->v_ptr, tcg_env, vector_reg_offset(op)); 344 return op->v_ptr; 345} 346 347#define OP_PTR0 op_ptr(decode, 0) 348#define OP_PTR1 op_ptr(decode, 1) 349#define OP_PTR2 op_ptr(decode, 2) 350 351static void gen_writeback(DisasContext *s, X86DecodedInsn *decode, int opn, TCGv v) 352{ 353 X86DecodedOp *op = &decode->op[opn]; 354 switch (op->unit) { 355 case X86_OP_SKIP: 356 break; 357 case X86_OP_SEG: 358 /* Note that gen_movl_seg takes care of interrupt shadow and TF. */ 359 gen_movl_seg(s, op->n, s->T0); 360 break; 361 case X86_OP_INT: 362 if (op->has_ea) { 363 gen_op_st_v(s, op->ot, v, s->A0); 364 } else { 365 gen_op_mov_reg_v(s, op->ot, op->n, v); 366 } 367 break; 368 case X86_OP_MMX: 369 break; 370 case X86_OP_SSE: 371 if (!op->has_ea && (s->prefix & PREFIX_VEX) && op->ot <= MO_128) { 372 tcg_gen_gvec_dup_imm(MO_64, 373 offsetof(CPUX86State, xmm_regs[op->n].ZMM_X(1)), 374 16, 16, 0); 375 } 376 break; 377#ifndef CONFIG_USER_ONLY 378 case X86_OP_CR: 379 if (op->n == 8) { 380 translator_io_start(&s->base); 381 } 382 gen_helper_write_crN(tcg_env, tcg_constant_i32(op->n), v); 383 s->base.is_jmp = DISAS_EOB_NEXT; 384 break; 385 case X86_OP_DR: 386 /* CR4.DE tested in the helper. */ 387 gen_helper_set_dr(tcg_env, tcg_constant_i32(op->n), v); 388 s->base.is_jmp = DISAS_EOB_NEXT; 389 break; 390#endif 391 default: 392 g_assert_not_reached(); 393 } 394 op->unit = X86_OP_SKIP; 395} 396 397static inline int vector_len(DisasContext *s, X86DecodedInsn *decode) 398{ 399 if (decode->e.special == X86_SPECIAL_MMX && 400 !(s->prefix & (PREFIX_DATA | PREFIX_REPZ | PREFIX_REPNZ))) { 401 return 8; 402 } 403 return s->vex_l ? 32 : 16; 404} 405 406static void prepare_update1_cc(X86DecodedInsn *decode, DisasContext *s, CCOp op) 407{ 408 decode->cc_dst = s->T0; 409 decode->cc_op = op; 410} 411 412static void prepare_update2_cc(X86DecodedInsn *decode, DisasContext *s, CCOp op) 413{ 414 decode->cc_src = s->T1; 415 decode->cc_dst = s->T0; 416 decode->cc_op = op; 417} 418 419static void prepare_update_cc_incdec(X86DecodedInsn *decode, DisasContext *s, CCOp op) 420{ 421 gen_compute_eflags_c(s, s->T1); 422 prepare_update2_cc(decode, s, op); 423} 424 425static void prepare_update3_cc(X86DecodedInsn *decode, DisasContext *s, CCOp op, TCGv reg) 426{ 427 decode->cc_src2 = reg; 428 decode->cc_src = s->T1; 429 decode->cc_dst = s->T0; 430 decode->cc_op = op; 431} 432 433/* Set up decode->cc_* to modify CF while keeping other flags unchanged. */ 434static void prepare_update_cf(X86DecodedInsn *decode, DisasContext *s, TCGv cf) 435{ 436 switch (s->cc_op) { 437 case CC_OP_ADOX: 438 case CC_OP_ADCOX: 439 decode->cc_src2 = cpu_cc_src2; 440 decode->cc_src = cpu_cc_src; 441 decode->cc_op = CC_OP_ADCOX; 442 break; 443 444 case CC_OP_EFLAGS: 445 case CC_OP_ADCX: 446 decode->cc_src = cpu_cc_src; 447 decode->cc_op = CC_OP_ADCX; 448 break; 449 450 default: 451 decode->cc_src = tcg_temp_new(); 452 gen_mov_eflags(s, decode->cc_src); 453 decode->cc_op = CC_OP_ADCX; 454 break; 455 } 456 decode->cc_dst = cf; 457} 458 459static void gen_store_sse(DisasContext *s, X86DecodedInsn *decode, int src_ofs) 460{ 461 MemOp ot = decode->op[0].ot; 462 int vec_len = vector_len(s, decode); 463 bool aligned = sse_needs_alignment(s, decode, ot); 464 465 if (!decode->op[0].has_ea) { 466 tcg_gen_gvec_mov(MO_64, decode->op[0].offset, src_ofs, vec_len, vec_len); 467 return; 468 } 469 470 switch (ot) { 471 case MO_64: 472 gen_stq_env_A0(s, src_ofs); 473 break; 474 case MO_128: 475 gen_sto_env_A0(s, src_ofs, aligned); 476 break; 477 case MO_256: 478 gen_sty_env_A0(s, src_ofs, aligned); 479 break; 480 default: 481 g_assert_not_reached(); 482 } 483} 484 485static void gen_helper_pavgusb(TCGv_ptr env, TCGv_ptr reg_a, TCGv_ptr reg_b) 486{ 487 gen_helper_pavgb_mmx(env, reg_a, reg_a, reg_b); 488} 489 490#define FN_3DNOW_MOVE ((SSEFunc_0_epp) (uintptr_t) 1) 491static const SSEFunc_0_epp fns_3dnow[] = { 492 [0x0c] = gen_helper_pi2fw, 493 [0x0d] = gen_helper_pi2fd, 494 [0x1c] = gen_helper_pf2iw, 495 [0x1d] = gen_helper_pf2id, 496 [0x8a] = gen_helper_pfnacc, 497 [0x8e] = gen_helper_pfpnacc, 498 [0x90] = gen_helper_pfcmpge, 499 [0x94] = gen_helper_pfmin, 500 [0x96] = gen_helper_pfrcp, 501 [0x97] = gen_helper_pfrsqrt, 502 [0x9a] = gen_helper_pfsub, 503 [0x9e] = gen_helper_pfadd, 504 [0xa0] = gen_helper_pfcmpgt, 505 [0xa4] = gen_helper_pfmax, 506 [0xa6] = FN_3DNOW_MOVE, /* PFRCPIT1; no need to actually increase precision */ 507 [0xa7] = FN_3DNOW_MOVE, /* PFRSQIT1 */ 508 [0xb6] = FN_3DNOW_MOVE, /* PFRCPIT2 */ 509 [0xaa] = gen_helper_pfsubr, 510 [0xae] = gen_helper_pfacc, 511 [0xb0] = gen_helper_pfcmpeq, 512 [0xb4] = gen_helper_pfmul, 513 [0xb7] = gen_helper_pmulhrw_mmx, 514 [0xbb] = gen_helper_pswapd, 515 [0xbf] = gen_helper_pavgusb, 516}; 517 518static void gen_3dnow(DisasContext *s, X86DecodedInsn *decode) 519{ 520 uint8_t b = decode->immediate; 521 SSEFunc_0_epp fn = b < ARRAY_SIZE(fns_3dnow) ? fns_3dnow[b] : NULL; 522 523 if (!fn) { 524 gen_illegal_opcode(s); 525 return; 526 } 527 if (s->flags & HF_TS_MASK) { 528 gen_NM_exception(s); 529 return; 530 } 531 if (s->flags & HF_EM_MASK) { 532 gen_illegal_opcode(s); 533 return; 534 } 535 536 gen_helper_enter_mmx(tcg_env); 537 if (fn == FN_3DNOW_MOVE) { 538 tcg_gen_ld_i64(s->tmp1_i64, tcg_env, decode->op[1].offset); 539 tcg_gen_st_i64(s->tmp1_i64, tcg_env, decode->op[0].offset); 540 } else { 541 fn(tcg_env, OP_PTR0, OP_PTR1); 542 } 543} 544 545/* 546 * 00 = v*ps Vps, Hps, Wpd 547 * 66 = v*pd Vpd, Hpd, Wps 548 * f3 = v*ss Vss, Hss, Wps 549 * f2 = v*sd Vsd, Hsd, Wps 550 */ 551static inline void gen_unary_fp_sse(DisasContext *s, X86DecodedInsn *decode, 552 SSEFunc_0_epp pd_xmm, SSEFunc_0_epp ps_xmm, 553 SSEFunc_0_epp pd_ymm, SSEFunc_0_epp ps_ymm, 554 SSEFunc_0_eppp sd, SSEFunc_0_eppp ss) 555{ 556 if ((s->prefix & (PREFIX_REPZ | PREFIX_REPNZ)) != 0) { 557 SSEFunc_0_eppp fn = s->prefix & PREFIX_REPZ ? ss : sd; 558 if (!fn) { 559 gen_illegal_opcode(s); 560 return; 561 } 562 fn(tcg_env, OP_PTR0, OP_PTR1, OP_PTR2); 563 } else { 564 SSEFunc_0_epp ps, pd, fn; 565 ps = s->vex_l ? ps_ymm : ps_xmm; 566 pd = s->vex_l ? pd_ymm : pd_xmm; 567 fn = s->prefix & PREFIX_DATA ? pd : ps; 568 if (!fn) { 569 gen_illegal_opcode(s); 570 return; 571 } 572 fn(tcg_env, OP_PTR0, OP_PTR2); 573 } 574} 575#define UNARY_FP_SSE(uname, lname) \ 576static void gen_##uname(DisasContext *s, X86DecodedInsn *decode) \ 577{ \ 578 gen_unary_fp_sse(s, decode, \ 579 gen_helper_##lname##pd_xmm, \ 580 gen_helper_##lname##ps_xmm, \ 581 gen_helper_##lname##pd_ymm, \ 582 gen_helper_##lname##ps_ymm, \ 583 gen_helper_##lname##sd, \ 584 gen_helper_##lname##ss); \ 585} 586UNARY_FP_SSE(VSQRT, sqrt) 587 588/* 589 * 00 = v*ps Vps, Hps, Wpd 590 * 66 = v*pd Vpd, Hpd, Wps 591 * f3 = v*ss Vss, Hss, Wps 592 * f2 = v*sd Vsd, Hsd, Wps 593 */ 594static inline void gen_fp_sse(DisasContext *s, X86DecodedInsn *decode, 595 SSEFunc_0_eppp pd_xmm, SSEFunc_0_eppp ps_xmm, 596 SSEFunc_0_eppp pd_ymm, SSEFunc_0_eppp ps_ymm, 597 SSEFunc_0_eppp sd, SSEFunc_0_eppp ss) 598{ 599 SSEFunc_0_eppp ps, pd, fn; 600 if ((s->prefix & (PREFIX_REPZ | PREFIX_REPNZ)) != 0) { 601 fn = s->prefix & PREFIX_REPZ ? ss : sd; 602 } else { 603 ps = s->vex_l ? ps_ymm : ps_xmm; 604 pd = s->vex_l ? pd_ymm : pd_xmm; 605 fn = s->prefix & PREFIX_DATA ? pd : ps; 606 } 607 if (fn) { 608 fn(tcg_env, OP_PTR0, OP_PTR1, OP_PTR2); 609 } else { 610 gen_illegal_opcode(s); 611 } 612} 613 614#define FP_SSE(uname, lname) \ 615static void gen_##uname(DisasContext *s, X86DecodedInsn *decode) \ 616{ \ 617 gen_fp_sse(s, decode, \ 618 gen_helper_##lname##pd_xmm, \ 619 gen_helper_##lname##ps_xmm, \ 620 gen_helper_##lname##pd_ymm, \ 621 gen_helper_##lname##ps_ymm, \ 622 gen_helper_##lname##sd, \ 623 gen_helper_##lname##ss); \ 624} 625FP_SSE(VADD, add) 626FP_SSE(VMUL, mul) 627FP_SSE(VSUB, sub) 628FP_SSE(VMIN, min) 629FP_SSE(VDIV, div) 630FP_SSE(VMAX, max) 631 632#define FMA_SSE_PACKED(uname, ptr0, ptr1, ptr2, even, odd) \ 633static void gen_##uname##Px(DisasContext *s, X86DecodedInsn *decode) \ 634{ \ 635 SSEFunc_0_eppppii xmm = s->vex_w ? gen_helper_fma4pd_xmm : gen_helper_fma4ps_xmm; \ 636 SSEFunc_0_eppppii ymm = s->vex_w ? gen_helper_fma4pd_ymm : gen_helper_fma4ps_ymm; \ 637 SSEFunc_0_eppppii fn = s->vex_l ? ymm : xmm; \ 638 \ 639 fn(tcg_env, OP_PTR0, ptr0, ptr1, ptr2, \ 640 tcg_constant_i32(even), \ 641 tcg_constant_i32((even) ^ (odd))); \ 642} 643 644#define FMA_SSE(uname, ptr0, ptr1, ptr2, flags) \ 645FMA_SSE_PACKED(uname, ptr0, ptr1, ptr2, flags, flags) \ 646static void gen_##uname##Sx(DisasContext *s, X86DecodedInsn *decode) \ 647{ \ 648 SSEFunc_0_eppppi fn = s->vex_w ? gen_helper_fma4sd : gen_helper_fma4ss; \ 649 \ 650 fn(tcg_env, OP_PTR0, ptr0, ptr1, ptr2, \ 651 tcg_constant_i32(flags)); \ 652} \ 653 654FMA_SSE(VFMADD231, OP_PTR1, OP_PTR2, OP_PTR0, 0) 655FMA_SSE(VFMADD213, OP_PTR1, OP_PTR0, OP_PTR2, 0) 656FMA_SSE(VFMADD132, OP_PTR0, OP_PTR2, OP_PTR1, 0) 657 658FMA_SSE(VFNMADD231, OP_PTR1, OP_PTR2, OP_PTR0, float_muladd_negate_product) 659FMA_SSE(VFNMADD213, OP_PTR1, OP_PTR0, OP_PTR2, float_muladd_negate_product) 660FMA_SSE(VFNMADD132, OP_PTR0, OP_PTR2, OP_PTR1, float_muladd_negate_product) 661 662FMA_SSE(VFMSUB231, OP_PTR1, OP_PTR2, OP_PTR0, float_muladd_negate_c) 663FMA_SSE(VFMSUB213, OP_PTR1, OP_PTR0, OP_PTR2, float_muladd_negate_c) 664FMA_SSE(VFMSUB132, OP_PTR0, OP_PTR2, OP_PTR1, float_muladd_negate_c) 665 666FMA_SSE(VFNMSUB231, OP_PTR1, OP_PTR2, OP_PTR0, float_muladd_negate_c|float_muladd_negate_product) 667FMA_SSE(VFNMSUB213, OP_PTR1, OP_PTR0, OP_PTR2, float_muladd_negate_c|float_muladd_negate_product) 668FMA_SSE(VFNMSUB132, OP_PTR0, OP_PTR2, OP_PTR1, float_muladd_negate_c|float_muladd_negate_product) 669 670FMA_SSE_PACKED(VFMADDSUB231, OP_PTR1, OP_PTR2, OP_PTR0, float_muladd_negate_c, 0) 671FMA_SSE_PACKED(VFMADDSUB213, OP_PTR1, OP_PTR0, OP_PTR2, float_muladd_negate_c, 0) 672FMA_SSE_PACKED(VFMADDSUB132, OP_PTR0, OP_PTR2, OP_PTR1, float_muladd_negate_c, 0) 673 674FMA_SSE_PACKED(VFMSUBADD231, OP_PTR1, OP_PTR2, OP_PTR0, 0, float_muladd_negate_c) 675FMA_SSE_PACKED(VFMSUBADD213, OP_PTR1, OP_PTR0, OP_PTR2, 0, float_muladd_negate_c) 676FMA_SSE_PACKED(VFMSUBADD132, OP_PTR0, OP_PTR2, OP_PTR1, 0, float_muladd_negate_c) 677 678#define FP_UNPACK_SSE(uname, lname) \ 679static void gen_##uname(DisasContext *s, X86DecodedInsn *decode) \ 680{ \ 681 /* PS maps to the DQ integer instruction, PD maps to QDQ. */ \ 682 gen_fp_sse(s, decode, \ 683 gen_helper_##lname##qdq_xmm, \ 684 gen_helper_##lname##dq_xmm, \ 685 gen_helper_##lname##qdq_ymm, \ 686 gen_helper_##lname##dq_ymm, \ 687 NULL, NULL); \ 688} 689FP_UNPACK_SSE(VUNPCKLPx, punpckl) 690FP_UNPACK_SSE(VUNPCKHPx, punpckh) 691 692/* 693 * 00 = v*ps Vps, Wpd 694 * f3 = v*ss Vss, Wps 695 */ 696static inline void gen_unary_fp32_sse(DisasContext *s, X86DecodedInsn *decode, 697 SSEFunc_0_epp ps_xmm, 698 SSEFunc_0_epp ps_ymm, 699 SSEFunc_0_eppp ss) 700{ 701 if ((s->prefix & (PREFIX_DATA | PREFIX_REPNZ)) != 0) { 702 goto illegal_op; 703 } else if (s->prefix & PREFIX_REPZ) { 704 if (!ss) { 705 goto illegal_op; 706 } 707 ss(tcg_env, OP_PTR0, OP_PTR1, OP_PTR2); 708 } else { 709 SSEFunc_0_epp fn = s->vex_l ? ps_ymm : ps_xmm; 710 if (!fn) { 711 goto illegal_op; 712 } 713 fn(tcg_env, OP_PTR0, OP_PTR2); 714 } 715 return; 716 717illegal_op: 718 gen_illegal_opcode(s); 719} 720#define UNARY_FP32_SSE(uname, lname) \ 721static void gen_##uname(DisasContext *s, X86DecodedInsn *decode) \ 722{ \ 723 gen_unary_fp32_sse(s, decode, \ 724 gen_helper_##lname##ps_xmm, \ 725 gen_helper_##lname##ps_ymm, \ 726 gen_helper_##lname##ss); \ 727} 728UNARY_FP32_SSE(VRSQRT, rsqrt) 729UNARY_FP32_SSE(VRCP, rcp) 730 731/* 732 * 66 = v*pd Vpd, Hpd, Wpd 733 * f2 = v*ps Vps, Hps, Wps 734 */ 735static inline void gen_horizontal_fp_sse(DisasContext *s, X86DecodedInsn *decode, 736 SSEFunc_0_eppp pd_xmm, SSEFunc_0_eppp ps_xmm, 737 SSEFunc_0_eppp pd_ymm, SSEFunc_0_eppp ps_ymm) 738{ 739 SSEFunc_0_eppp ps, pd, fn; 740 ps = s->vex_l ? ps_ymm : ps_xmm; 741 pd = s->vex_l ? pd_ymm : pd_xmm; 742 fn = s->prefix & PREFIX_DATA ? pd : ps; 743 fn(tcg_env, OP_PTR0, OP_PTR1, OP_PTR2); 744} 745#define HORIZONTAL_FP_SSE(uname, lname) \ 746static void gen_##uname(DisasContext *s, X86DecodedInsn *decode) \ 747{ \ 748 gen_horizontal_fp_sse(s, decode, \ 749 gen_helper_##lname##pd_xmm, gen_helper_##lname##ps_xmm, \ 750 gen_helper_##lname##pd_ymm, gen_helper_##lname##ps_ymm); \ 751} 752HORIZONTAL_FP_SSE(VHADD, hadd) 753HORIZONTAL_FP_SSE(VHSUB, hsub) 754HORIZONTAL_FP_SSE(VADDSUB, addsub) 755 756static inline void gen_ternary_sse(DisasContext *s, X86DecodedInsn *decode, 757 int op3, SSEFunc_0_epppp xmm, SSEFunc_0_epppp ymm) 758{ 759 SSEFunc_0_epppp fn = s->vex_l ? ymm : xmm; 760 TCGv_ptr ptr3 = tcg_temp_new_ptr(); 761 762 /* The format of the fourth input is Lx */ 763 tcg_gen_addi_ptr(ptr3, tcg_env, ZMM_OFFSET(op3)); 764 fn(tcg_env, OP_PTR0, OP_PTR1, OP_PTR2, ptr3); 765} 766#define TERNARY_SSE(uname, uvname, lname) \ 767static void gen_##uvname(DisasContext *s, X86DecodedInsn *decode) \ 768{ \ 769 gen_ternary_sse(s, decode, (uint8_t)decode->immediate >> 4, \ 770 gen_helper_##lname##_xmm, gen_helper_##lname##_ymm); \ 771} \ 772static void gen_##uname(DisasContext *s, X86DecodedInsn *decode) \ 773{ \ 774 gen_ternary_sse(s, decode, 0, \ 775 gen_helper_##lname##_xmm, gen_helper_##lname##_ymm); \ 776} 777TERNARY_SSE(BLENDVPS, VBLENDVPS, blendvps) 778TERNARY_SSE(BLENDVPD, VBLENDVPD, blendvpd) 779TERNARY_SSE(PBLENDVB, VPBLENDVB, pblendvb) 780 781static inline void gen_binary_imm_sse(DisasContext *s, X86DecodedInsn *decode, 782 SSEFunc_0_epppi xmm, SSEFunc_0_epppi ymm) 783{ 784 TCGv_i32 imm = tcg_constant8u_i32(decode->immediate); 785 if (!s->vex_l) { 786 xmm(tcg_env, OP_PTR0, OP_PTR1, OP_PTR2, imm); 787 } else { 788 ymm(tcg_env, OP_PTR0, OP_PTR1, OP_PTR2, imm); 789 } 790} 791 792#define BINARY_IMM_SSE(uname, lname) \ 793static void gen_##uname(DisasContext *s, X86DecodedInsn *decode) \ 794{ \ 795 gen_binary_imm_sse(s, decode, \ 796 gen_helper_##lname##_xmm, \ 797 gen_helper_##lname##_ymm); \ 798} 799 800BINARY_IMM_SSE(VBLENDPD, blendpd) 801BINARY_IMM_SSE(VBLENDPS, blendps) 802BINARY_IMM_SSE(VPBLENDW, pblendw) 803BINARY_IMM_SSE(VDDPS, dpps) 804#define gen_helper_dppd_ymm NULL 805BINARY_IMM_SSE(VDDPD, dppd) 806BINARY_IMM_SSE(VMPSADBW, mpsadbw) 807BINARY_IMM_SSE(PCLMULQDQ, pclmulqdq) 808 809 810#define UNARY_INT_GVEC(uname, func, ...) \ 811static void gen_##uname(DisasContext *s, X86DecodedInsn *decode) \ 812{ \ 813 int vec_len = vector_len(s, decode); \ 814 \ 815 func(__VA_ARGS__, decode->op[0].offset, \ 816 decode->op[2].offset, vec_len, vec_len); \ 817} 818UNARY_INT_GVEC(PABSB, tcg_gen_gvec_abs, MO_8) 819UNARY_INT_GVEC(PABSW, tcg_gen_gvec_abs, MO_16) 820UNARY_INT_GVEC(PABSD, tcg_gen_gvec_abs, MO_32) 821UNARY_INT_GVEC(VBROADCASTx128, tcg_gen_gvec_dup_mem, MO_128) 822UNARY_INT_GVEC(VPBROADCASTB, tcg_gen_gvec_dup_mem, MO_8) 823UNARY_INT_GVEC(VPBROADCASTW, tcg_gen_gvec_dup_mem, MO_16) 824UNARY_INT_GVEC(VPBROADCASTD, tcg_gen_gvec_dup_mem, MO_32) 825UNARY_INT_GVEC(VPBROADCASTQ, tcg_gen_gvec_dup_mem, MO_64) 826 827 828#define BINARY_INT_GVEC(uname, func, ...) \ 829static void gen_##uname(DisasContext *s, X86DecodedInsn *decode) \ 830{ \ 831 int vec_len = vector_len(s, decode); \ 832 \ 833 func(__VA_ARGS__, \ 834 decode->op[0].offset, decode->op[1].offset, \ 835 decode->op[2].offset, vec_len, vec_len); \ 836} 837 838BINARY_INT_GVEC(PADDB, tcg_gen_gvec_add, MO_8) 839BINARY_INT_GVEC(PADDW, tcg_gen_gvec_add, MO_16) 840BINARY_INT_GVEC(PADDD, tcg_gen_gvec_add, MO_32) 841BINARY_INT_GVEC(PADDQ, tcg_gen_gvec_add, MO_64) 842BINARY_INT_GVEC(PADDSB, tcg_gen_gvec_ssadd, MO_8) 843BINARY_INT_GVEC(PADDSW, tcg_gen_gvec_ssadd, MO_16) 844BINARY_INT_GVEC(PADDUSB, tcg_gen_gvec_usadd, MO_8) 845BINARY_INT_GVEC(PADDUSW, tcg_gen_gvec_usadd, MO_16) 846BINARY_INT_GVEC(PAND, tcg_gen_gvec_and, MO_64) 847BINARY_INT_GVEC(PCMPEQB, tcg_gen_gvec_cmp, TCG_COND_EQ, MO_8) 848BINARY_INT_GVEC(PCMPEQD, tcg_gen_gvec_cmp, TCG_COND_EQ, MO_32) 849BINARY_INT_GVEC(PCMPEQW, tcg_gen_gvec_cmp, TCG_COND_EQ, MO_16) 850BINARY_INT_GVEC(PCMPEQQ, tcg_gen_gvec_cmp, TCG_COND_EQ, MO_64) 851BINARY_INT_GVEC(PCMPGTB, tcg_gen_gvec_cmp, TCG_COND_GT, MO_8) 852BINARY_INT_GVEC(PCMPGTW, tcg_gen_gvec_cmp, TCG_COND_GT, MO_16) 853BINARY_INT_GVEC(PCMPGTD, tcg_gen_gvec_cmp, TCG_COND_GT, MO_32) 854BINARY_INT_GVEC(PCMPGTQ, tcg_gen_gvec_cmp, TCG_COND_GT, MO_64) 855BINARY_INT_GVEC(PMAXSB, tcg_gen_gvec_smax, MO_8) 856BINARY_INT_GVEC(PMAXSW, tcg_gen_gvec_smax, MO_16) 857BINARY_INT_GVEC(PMAXSD, tcg_gen_gvec_smax, MO_32) 858BINARY_INT_GVEC(PMAXUB, tcg_gen_gvec_umax, MO_8) 859BINARY_INT_GVEC(PMAXUW, tcg_gen_gvec_umax, MO_16) 860BINARY_INT_GVEC(PMAXUD, tcg_gen_gvec_umax, MO_32) 861BINARY_INT_GVEC(PMINSB, tcg_gen_gvec_smin, MO_8) 862BINARY_INT_GVEC(PMINSW, tcg_gen_gvec_smin, MO_16) 863BINARY_INT_GVEC(PMINSD, tcg_gen_gvec_smin, MO_32) 864BINARY_INT_GVEC(PMINUB, tcg_gen_gvec_umin, MO_8) 865BINARY_INT_GVEC(PMINUW, tcg_gen_gvec_umin, MO_16) 866BINARY_INT_GVEC(PMINUD, tcg_gen_gvec_umin, MO_32) 867BINARY_INT_GVEC(PMULLW, tcg_gen_gvec_mul, MO_16) 868BINARY_INT_GVEC(PMULLD, tcg_gen_gvec_mul, MO_32) 869BINARY_INT_GVEC(POR, tcg_gen_gvec_or, MO_64) 870BINARY_INT_GVEC(PSUBB, tcg_gen_gvec_sub, MO_8) 871BINARY_INT_GVEC(PSUBW, tcg_gen_gvec_sub, MO_16) 872BINARY_INT_GVEC(PSUBD, tcg_gen_gvec_sub, MO_32) 873BINARY_INT_GVEC(PSUBQ, tcg_gen_gvec_sub, MO_64) 874BINARY_INT_GVEC(PSUBSB, tcg_gen_gvec_sssub, MO_8) 875BINARY_INT_GVEC(PSUBSW, tcg_gen_gvec_sssub, MO_16) 876BINARY_INT_GVEC(PSUBUSB, tcg_gen_gvec_ussub, MO_8) 877BINARY_INT_GVEC(PSUBUSW, tcg_gen_gvec_ussub, MO_16) 878BINARY_INT_GVEC(PXOR, tcg_gen_gvec_xor, MO_64) 879 880 881/* 882 * 00 = p* Pq, Qq (if mmx not NULL; no VEX) 883 * 66 = vp* Vx, Hx, Wx 884 * 885 * These are really the same encoding, because 1) V is the same as P when VEX.V 886 * is not present 2) P and Q are the same as H and W apart from MM/XMM 887 */ 888static inline void gen_binary_int_sse(DisasContext *s, X86DecodedInsn *decode, 889 SSEFunc_0_eppp mmx, SSEFunc_0_eppp xmm, SSEFunc_0_eppp ymm) 890{ 891 assert(!!mmx == !!(decode->e.special == X86_SPECIAL_MMX)); 892 893 if (mmx && (s->prefix & PREFIX_VEX) && !(s->prefix & PREFIX_DATA)) { 894 /* VEX encoding is not applicable to MMX instructions. */ 895 gen_illegal_opcode(s); 896 return; 897 } 898 if (!(s->prefix & PREFIX_DATA)) { 899 mmx(tcg_env, OP_PTR0, OP_PTR1, OP_PTR2); 900 } else if (!s->vex_l) { 901 xmm(tcg_env, OP_PTR0, OP_PTR1, OP_PTR2); 902 } else { 903 ymm(tcg_env, OP_PTR0, OP_PTR1, OP_PTR2); 904 } 905} 906 907 908#define BINARY_INT_MMX(uname, lname) \ 909static void gen_##uname(DisasContext *s, X86DecodedInsn *decode) \ 910{ \ 911 gen_binary_int_sse(s, decode, \ 912 gen_helper_##lname##_mmx, \ 913 gen_helper_##lname##_xmm, \ 914 gen_helper_##lname##_ymm); \ 915} 916BINARY_INT_MMX(PUNPCKLBW, punpcklbw) 917BINARY_INT_MMX(PUNPCKLWD, punpcklwd) 918BINARY_INT_MMX(PUNPCKLDQ, punpckldq) 919BINARY_INT_MMX(PACKSSWB, packsswb) 920BINARY_INT_MMX(PACKUSWB, packuswb) 921BINARY_INT_MMX(PUNPCKHBW, punpckhbw) 922BINARY_INT_MMX(PUNPCKHWD, punpckhwd) 923BINARY_INT_MMX(PUNPCKHDQ, punpckhdq) 924BINARY_INT_MMX(PACKSSDW, packssdw) 925 926BINARY_INT_MMX(PAVGB, pavgb) 927BINARY_INT_MMX(PAVGW, pavgw) 928BINARY_INT_MMX(PMADDWD, pmaddwd) 929BINARY_INT_MMX(PMULHUW, pmulhuw) 930BINARY_INT_MMX(PMULHW, pmulhw) 931BINARY_INT_MMX(PMULUDQ, pmuludq) 932BINARY_INT_MMX(PSADBW, psadbw) 933 934BINARY_INT_MMX(PSLLW_r, psllw) 935BINARY_INT_MMX(PSLLD_r, pslld) 936BINARY_INT_MMX(PSLLQ_r, psllq) 937BINARY_INT_MMX(PSRLW_r, psrlw) 938BINARY_INT_MMX(PSRLD_r, psrld) 939BINARY_INT_MMX(PSRLQ_r, psrlq) 940BINARY_INT_MMX(PSRAW_r, psraw) 941BINARY_INT_MMX(PSRAD_r, psrad) 942 943BINARY_INT_MMX(PHADDW, phaddw) 944BINARY_INT_MMX(PHADDSW, phaddsw) 945BINARY_INT_MMX(PHADDD, phaddd) 946BINARY_INT_MMX(PHSUBW, phsubw) 947BINARY_INT_MMX(PHSUBSW, phsubsw) 948BINARY_INT_MMX(PHSUBD, phsubd) 949BINARY_INT_MMX(PMADDUBSW, pmaddubsw) 950BINARY_INT_MMX(PSHUFB, pshufb) 951BINARY_INT_MMX(PSIGNB, psignb) 952BINARY_INT_MMX(PSIGNW, psignw) 953BINARY_INT_MMX(PSIGND, psignd) 954BINARY_INT_MMX(PMULHRSW, pmulhrsw) 955 956/* Instructions with no MMX equivalent. */ 957#define BINARY_INT_SSE(uname, lname) \ 958static void gen_##uname(DisasContext *s, X86DecodedInsn *decode) \ 959{ \ 960 gen_binary_int_sse(s, decode, \ 961 NULL, \ 962 gen_helper_##lname##_xmm, \ 963 gen_helper_##lname##_ymm); \ 964} 965 966/* Instructions with no MMX equivalent. */ 967BINARY_INT_SSE(PUNPCKLQDQ, punpcklqdq) 968BINARY_INT_SSE(PUNPCKHQDQ, punpckhqdq) 969BINARY_INT_SSE(VPACKUSDW, packusdw) 970BINARY_INT_SSE(VPERMILPS, vpermilps) 971BINARY_INT_SSE(VPERMILPD, vpermilpd) 972BINARY_INT_SSE(VMASKMOVPS, vpmaskmovd) 973BINARY_INT_SSE(VMASKMOVPD, vpmaskmovq) 974 975BINARY_INT_SSE(PMULDQ, pmuldq) 976 977BINARY_INT_SSE(VAESDEC, aesdec) 978BINARY_INT_SSE(VAESDECLAST, aesdeclast) 979BINARY_INT_SSE(VAESENC, aesenc) 980BINARY_INT_SSE(VAESENCLAST, aesenclast) 981 982#define UNARY_CMP_SSE(uname, lname) \ 983static void gen_##uname(DisasContext *s, X86DecodedInsn *decode) \ 984{ \ 985 if (!s->vex_l) { \ 986 gen_helper_##lname##_xmm(tcg_env, OP_PTR1, OP_PTR2); \ 987 } else { \ 988 gen_helper_##lname##_ymm(tcg_env, OP_PTR1, OP_PTR2); \ 989 } \ 990 assume_cc_op(s, CC_OP_EFLAGS); \ 991} 992UNARY_CMP_SSE(VPTEST, ptest) 993UNARY_CMP_SSE(VTESTPS, vtestps) 994UNARY_CMP_SSE(VTESTPD, vtestpd) 995 996static inline void gen_unary_int_sse(DisasContext *s, X86DecodedInsn *decode, 997 SSEFunc_0_epp xmm, SSEFunc_0_epp ymm) 998{ 999 if (!s->vex_l) { 1000 xmm(tcg_env, OP_PTR0, OP_PTR2); 1001 } else { 1002 ymm(tcg_env, OP_PTR0, OP_PTR2); 1003 } 1004} 1005 1006#define UNARY_INT_SSE(uname, lname) \ 1007static void gen_##uname(DisasContext *s, X86DecodedInsn *decode) \ 1008{ \ 1009 gen_unary_int_sse(s, decode, \ 1010 gen_helper_##lname##_xmm, \ 1011 gen_helper_##lname##_ymm); \ 1012} 1013 1014UNARY_INT_SSE(VPMOVSXBW, pmovsxbw) 1015UNARY_INT_SSE(VPMOVSXBD, pmovsxbd) 1016UNARY_INT_SSE(VPMOVSXBQ, pmovsxbq) 1017UNARY_INT_SSE(VPMOVSXWD, pmovsxwd) 1018UNARY_INT_SSE(VPMOVSXWQ, pmovsxwq) 1019UNARY_INT_SSE(VPMOVSXDQ, pmovsxdq) 1020 1021UNARY_INT_SSE(VPMOVZXBW, pmovzxbw) 1022UNARY_INT_SSE(VPMOVZXBD, pmovzxbd) 1023UNARY_INT_SSE(VPMOVZXBQ, pmovzxbq) 1024UNARY_INT_SSE(VPMOVZXWD, pmovzxwd) 1025UNARY_INT_SSE(VPMOVZXWQ, pmovzxwq) 1026UNARY_INT_SSE(VPMOVZXDQ, pmovzxdq) 1027 1028UNARY_INT_SSE(VMOVSLDUP, pmovsldup) 1029UNARY_INT_SSE(VMOVSHDUP, pmovshdup) 1030UNARY_INT_SSE(VMOVDDUP, pmovdldup) 1031 1032UNARY_INT_SSE(VCVTDQ2PD, cvtdq2pd) 1033UNARY_INT_SSE(VCVTPD2DQ, cvtpd2dq) 1034UNARY_INT_SSE(VCVTTPD2DQ, cvttpd2dq) 1035UNARY_INT_SSE(VCVTDQ2PS, cvtdq2ps) 1036UNARY_INT_SSE(VCVTPS2DQ, cvtps2dq) 1037UNARY_INT_SSE(VCVTTPS2DQ, cvttps2dq) 1038UNARY_INT_SSE(VCVTPH2PS, cvtph2ps) 1039 1040 1041static inline void gen_unary_imm_sse(DisasContext *s, X86DecodedInsn *decode, 1042 SSEFunc_0_ppi xmm, SSEFunc_0_ppi ymm) 1043{ 1044 TCGv_i32 imm = tcg_constant8u_i32(decode->immediate); 1045 if (!s->vex_l) { 1046 xmm(OP_PTR0, OP_PTR1, imm); 1047 } else { 1048 ymm(OP_PTR0, OP_PTR1, imm); 1049 } 1050} 1051 1052#define UNARY_IMM_SSE(uname, lname) \ 1053static void gen_##uname(DisasContext *s, X86DecodedInsn *decode) \ 1054{ \ 1055 gen_unary_imm_sse(s, decode, \ 1056 gen_helper_##lname##_xmm, \ 1057 gen_helper_##lname##_ymm); \ 1058} 1059 1060UNARY_IMM_SSE(PSHUFD, pshufd) 1061UNARY_IMM_SSE(PSHUFHW, pshufhw) 1062UNARY_IMM_SSE(PSHUFLW, pshuflw) 1063#define gen_helper_vpermq_xmm NULL 1064UNARY_IMM_SSE(VPERMQ, vpermq) 1065UNARY_IMM_SSE(VPERMILPS_i, vpermilps_imm) 1066UNARY_IMM_SSE(VPERMILPD_i, vpermilpd_imm) 1067 1068static inline void gen_unary_imm_fp_sse(DisasContext *s, X86DecodedInsn *decode, 1069 SSEFunc_0_eppi xmm, SSEFunc_0_eppi ymm) 1070{ 1071 TCGv_i32 imm = tcg_constant8u_i32(decode->immediate); 1072 if (!s->vex_l) { 1073 xmm(tcg_env, OP_PTR0, OP_PTR1, imm); 1074 } else { 1075 ymm(tcg_env, OP_PTR0, OP_PTR1, imm); 1076 } 1077} 1078 1079#define UNARY_IMM_FP_SSE(uname, lname) \ 1080static void gen_##uname(DisasContext *s, X86DecodedInsn *decode) \ 1081{ \ 1082 gen_unary_imm_fp_sse(s, decode, \ 1083 gen_helper_##lname##_xmm, \ 1084 gen_helper_##lname##_ymm); \ 1085} 1086 1087UNARY_IMM_FP_SSE(VROUNDPS, roundps) 1088UNARY_IMM_FP_SSE(VROUNDPD, roundpd) 1089 1090static inline void gen_vexw_avx(DisasContext *s, X86DecodedInsn *decode, 1091 SSEFunc_0_eppp d_xmm, SSEFunc_0_eppp q_xmm, 1092 SSEFunc_0_eppp d_ymm, SSEFunc_0_eppp q_ymm) 1093{ 1094 SSEFunc_0_eppp d = s->vex_l ? d_ymm : d_xmm; 1095 SSEFunc_0_eppp q = s->vex_l ? q_ymm : q_xmm; 1096 SSEFunc_0_eppp fn = s->vex_w ? q : d; 1097 fn(tcg_env, OP_PTR0, OP_PTR1, OP_PTR2); 1098} 1099 1100/* VEX.W affects whether to operate on 32- or 64-bit elements. */ 1101#define VEXW_AVX(uname, lname) \ 1102static void gen_##uname(DisasContext *s, X86DecodedInsn *decode) \ 1103{ \ 1104 gen_vexw_avx(s, decode, \ 1105 gen_helper_##lname##d_xmm, gen_helper_##lname##q_xmm, \ 1106 gen_helper_##lname##d_ymm, gen_helper_##lname##q_ymm); \ 1107} 1108VEXW_AVX(VPSLLV, vpsllv) 1109VEXW_AVX(VPSRLV, vpsrlv) 1110VEXW_AVX(VPSRAV, vpsrav) 1111VEXW_AVX(VPMASKMOV, vpmaskmov) 1112 1113/* Same as above, but with extra arguments to the helper. */ 1114static inline void gen_vsib_avx(DisasContext *s, X86DecodedInsn *decode, 1115 SSEFunc_0_epppti d_xmm, SSEFunc_0_epppti q_xmm, 1116 SSEFunc_0_epppti d_ymm, SSEFunc_0_epppti q_ymm) 1117{ 1118 SSEFunc_0_epppti d = s->vex_l ? d_ymm : d_xmm; 1119 SSEFunc_0_epppti q = s->vex_l ? q_ymm : q_xmm; 1120 SSEFunc_0_epppti fn = s->vex_w ? q : d; 1121 TCGv_i32 scale = tcg_constant_i32(decode->mem.scale); 1122 TCGv_ptr index = tcg_temp_new_ptr(); 1123 1124 /* Pass third input as (index, base, scale) */ 1125 tcg_gen_addi_ptr(index, tcg_env, ZMM_OFFSET(decode->mem.index)); 1126 fn(tcg_env, OP_PTR0, OP_PTR1, index, s->A0, scale); 1127 1128 /* 1129 * There are two output operands, so zero OP1's high 128 bits 1130 * in the VEX.128 case. 1131 */ 1132 if (!s->vex_l) { 1133 int ymmh_ofs = vector_elem_offset(&decode->op[1], MO_128, 1); 1134 tcg_gen_gvec_dup_imm(MO_64, ymmh_ofs, 16, 16, 0); 1135 } 1136} 1137#define VSIB_AVX(uname, lname) \ 1138static void gen_##uname(DisasContext *s, X86DecodedInsn *decode) \ 1139{ \ 1140 gen_vsib_avx(s, decode, \ 1141 gen_helper_##lname##d_xmm, gen_helper_##lname##q_xmm, \ 1142 gen_helper_##lname##d_ymm, gen_helper_##lname##q_ymm); \ 1143} 1144VSIB_AVX(VPGATHERD, vpgatherd) 1145VSIB_AVX(VPGATHERQ, vpgatherq) 1146 1147static void gen_AAA(DisasContext *s, X86DecodedInsn *decode) 1148{ 1149 gen_update_cc_op(s); 1150 gen_helper_aaa(tcg_env); 1151 assume_cc_op(s, CC_OP_EFLAGS); 1152} 1153 1154static void gen_AAD(DisasContext *s, X86DecodedInsn *decode) 1155{ 1156 gen_helper_aad(s->T0, s->T0, s->T1); 1157 prepare_update1_cc(decode, s, CC_OP_LOGICB); 1158} 1159 1160static void gen_AAM(DisasContext *s, X86DecodedInsn *decode) 1161{ 1162 if (decode->immediate == 0) { 1163 gen_exception(s, EXCP00_DIVZ); 1164 } else { 1165 gen_helper_aam(s->T0, s->T0, s->T1); 1166 prepare_update1_cc(decode, s, CC_OP_LOGICB); 1167 } 1168} 1169 1170static void gen_AAS(DisasContext *s, X86DecodedInsn *decode) 1171{ 1172 gen_update_cc_op(s); 1173 gen_helper_aas(tcg_env); 1174 assume_cc_op(s, CC_OP_EFLAGS); 1175} 1176 1177static void gen_ADC(DisasContext *s, X86DecodedInsn *decode) 1178{ 1179 MemOp ot = decode->op[1].ot; 1180 TCGv c_in = tcg_temp_new(); 1181 1182 gen_compute_eflags_c(s, c_in); 1183 if (s->prefix & PREFIX_LOCK) { 1184 tcg_gen_add_tl(s->T0, c_in, s->T1); 1185 tcg_gen_atomic_add_fetch_tl(s->T0, s->A0, s->T0, 1186 s->mem_index, ot | MO_LE); 1187 } else { 1188 tcg_gen_add_tl(s->T0, s->T0, s->T1); 1189 tcg_gen_add_tl(s->T0, s->T0, c_in); 1190 } 1191 prepare_update3_cc(decode, s, CC_OP_ADCB + ot, c_in); 1192} 1193 1194static void gen_ADCOX(DisasContext *s, X86DecodedInsn *decode, int cc_op) 1195{ 1196 MemOp ot = decode->op[0].ot; 1197 TCGv carry_in = NULL; 1198 TCGv *carry_out = (cc_op == CC_OP_ADCX ? &decode->cc_dst : &decode->cc_src2); 1199 TCGv zero; 1200 1201 decode->cc_op = cc_op; 1202 *carry_out = tcg_temp_new(); 1203 if (CC_OP_HAS_EFLAGS(s->cc_op)) { 1204 decode->cc_src = cpu_cc_src; 1205 1206 /* Re-use the carry-out from a previous round? */ 1207 if (s->cc_op == cc_op || s->cc_op == CC_OP_ADCOX) { 1208 carry_in = (cc_op == CC_OP_ADCX ? cpu_cc_dst : cpu_cc_src2); 1209 } 1210 1211 /* Preserve the opposite carry from previous rounds? */ 1212 if (s->cc_op != cc_op && s->cc_op != CC_OP_EFLAGS) { 1213 decode->cc_op = CC_OP_ADCOX; 1214 if (carry_out == &decode->cc_dst) { 1215 decode->cc_src2 = cpu_cc_src2; 1216 } else { 1217 decode->cc_dst = cpu_cc_dst; 1218 } 1219 } 1220 } else { 1221 decode->cc_src = tcg_temp_new(); 1222 gen_mov_eflags(s, decode->cc_src); 1223 } 1224 1225 if (!carry_in) { 1226 /* Get carry_in out of EFLAGS. */ 1227 carry_in = tcg_temp_new(); 1228 tcg_gen_extract_tl(carry_in, decode->cc_src, 1229 ctz32(cc_op == CC_OP_ADCX ? CC_C : CC_O), 1); 1230 } 1231 1232 switch (ot) { 1233#ifdef TARGET_X86_64 1234 case MO_32: 1235 /* If TL is 64-bit just do everything in 64-bit arithmetic. */ 1236 tcg_gen_ext32u_tl(s->T0, s->T0); 1237 tcg_gen_ext32u_tl(s->T1, s->T1); 1238 tcg_gen_add_i64(s->T0, s->T0, s->T1); 1239 tcg_gen_add_i64(s->T0, s->T0, carry_in); 1240 tcg_gen_shri_i64(*carry_out, s->T0, 32); 1241 break; 1242#endif 1243 default: 1244 zero = tcg_constant_tl(0); 1245 tcg_gen_add2_tl(s->T0, *carry_out, s->T0, zero, carry_in, zero); 1246 tcg_gen_add2_tl(s->T0, *carry_out, s->T0, *carry_out, s->T1, zero); 1247 break; 1248 } 1249} 1250 1251static void gen_ADCX(DisasContext *s, X86DecodedInsn *decode) 1252{ 1253 gen_ADCOX(s, decode, CC_OP_ADCX); 1254} 1255 1256static void gen_ADD(DisasContext *s, X86DecodedInsn *decode) 1257{ 1258 MemOp ot = decode->op[1].ot; 1259 1260 if (s->prefix & PREFIX_LOCK) { 1261 tcg_gen_atomic_add_fetch_tl(s->T0, s->A0, s->T1, 1262 s->mem_index, ot | MO_LE); 1263 } else { 1264 tcg_gen_add_tl(s->T0, s->T0, s->T1); 1265 } 1266 prepare_update2_cc(decode, s, CC_OP_ADDB + ot); 1267} 1268 1269static void gen_ADOX(DisasContext *s, X86DecodedInsn *decode) 1270{ 1271 gen_ADCOX(s, decode, CC_OP_ADOX); 1272} 1273 1274static void gen_AND(DisasContext *s, X86DecodedInsn *decode) 1275{ 1276 MemOp ot = decode->op[1].ot; 1277 1278 if (s->prefix & PREFIX_LOCK) { 1279 tcg_gen_atomic_and_fetch_tl(s->T0, s->A0, s->T1, 1280 s->mem_index, ot | MO_LE); 1281 } else { 1282 tcg_gen_and_tl(s->T0, s->T0, s->T1); 1283 } 1284 prepare_update1_cc(decode, s, CC_OP_LOGICB + ot); 1285} 1286 1287static void gen_ANDN(DisasContext *s, X86DecodedInsn *decode) 1288{ 1289 MemOp ot = decode->op[0].ot; 1290 1291 tcg_gen_andc_tl(s->T0, s->T1, s->T0); 1292 prepare_update1_cc(decode, s, CC_OP_LOGICB + ot); 1293} 1294 1295static void gen_ARPL(DisasContext *s, X86DecodedInsn *decode) 1296{ 1297 TCGv zf = tcg_temp_new(); 1298 TCGv flags = tcg_temp_new(); 1299 1300 gen_mov_eflags(s, flags); 1301 1302 /* Compute adjusted DST in T1, merging in SRC[RPL]. */ 1303 tcg_gen_deposit_tl(s->T1, s->T0, s->T1, 0, 2); 1304 1305 /* Z flag set if DST[RPL] < SRC[RPL] */ 1306 tcg_gen_setcond_tl(TCG_COND_LTU, zf, s->T0, s->T1); 1307 tcg_gen_deposit_tl(flags, flags, zf, ctz32(CC_Z), 1); 1308 1309 /* Place maximum RPL in DST */ 1310 tcg_gen_umax_tl(s->T0, s->T0, s->T1); 1311 1312 decode->cc_src = flags; 1313 decode->cc_op = CC_OP_EFLAGS; 1314} 1315 1316static void gen_BEXTR(DisasContext *s, X86DecodedInsn *decode) 1317{ 1318 MemOp ot = decode->op[0].ot; 1319 TCGv bound = tcg_constant_tl(ot == MO_64 ? 63 : 31); 1320 TCGv zero = tcg_constant_tl(0); 1321 TCGv mone = tcg_constant_tl(-1); 1322 1323 /* 1324 * Extract START, and shift the operand. 1325 * Shifts larger than operand size get zeros. 1326 */ 1327 tcg_gen_ext8u_tl(s->A0, s->T1); 1328 tcg_gen_shr_tl(s->T0, s->T0, s->A0); 1329 1330 tcg_gen_movcond_tl(TCG_COND_LEU, s->T0, s->A0, bound, s->T0, zero); 1331 1332 /* 1333 * Extract the LEN into an inverse mask. Lengths larger than 1334 * operand size get all zeros, length 0 gets all ones. 1335 */ 1336 tcg_gen_extract_tl(s->A0, s->T1, 8, 8); 1337 tcg_gen_shl_tl(s->T1, mone, s->A0); 1338 tcg_gen_movcond_tl(TCG_COND_LEU, s->T1, s->A0, bound, s->T1, zero); 1339 tcg_gen_andc_tl(s->T0, s->T0, s->T1); 1340 1341 prepare_update1_cc(decode, s, CC_OP_LOGICB + ot); 1342} 1343 1344static void gen_BLSI(DisasContext *s, X86DecodedInsn *decode) 1345{ 1346 MemOp ot = decode->op[0].ot; 1347 1348 /* input in T1, which is ready for prepare_update2_cc */ 1349 tcg_gen_neg_tl(s->T0, s->T1); 1350 tcg_gen_and_tl(s->T0, s->T0, s->T1); 1351 prepare_update2_cc(decode, s, CC_OP_BLSIB + ot); 1352} 1353 1354static void gen_BLSMSK(DisasContext *s, X86DecodedInsn *decode) 1355{ 1356 MemOp ot = decode->op[0].ot; 1357 1358 /* input in T1, which is ready for prepare_update2_cc */ 1359 tcg_gen_subi_tl(s->T0, s->T1, 1); 1360 tcg_gen_xor_tl(s->T0, s->T0, s->T1); 1361 prepare_update2_cc(decode, s, CC_OP_BMILGB + ot); 1362} 1363 1364static void gen_BLSR(DisasContext *s, X86DecodedInsn *decode) 1365{ 1366 MemOp ot = decode->op[0].ot; 1367 1368 /* input in T1, which is ready for prepare_update2_cc */ 1369 tcg_gen_subi_tl(s->T0, s->T1, 1); 1370 tcg_gen_and_tl(s->T0, s->T0, s->T1); 1371 prepare_update2_cc(decode, s, CC_OP_BMILGB + ot); 1372} 1373 1374static void gen_BOUND(DisasContext *s, X86DecodedInsn *decode) 1375{ 1376 TCGv_i32 op = tcg_temp_new_i32(); 1377 tcg_gen_trunc_tl_i32(op, s->T0); 1378 if (decode->op[1].ot == MO_16) { 1379 gen_helper_boundw(tcg_env, s->A0, op); 1380 } else { 1381 gen_helper_boundl(tcg_env, s->A0, op); 1382 } 1383} 1384 1385/* Non-standard convention - on entry T0 is zero-extended input, T1 is the output. */ 1386static void gen_BSF(DisasContext *s, X86DecodedInsn *decode) 1387{ 1388 MemOp ot = decode->op[0].ot; 1389 1390 /* Only the Z bit is defined and it is related to the input. */ 1391 decode->cc_dst = tcg_temp_new(); 1392 decode->cc_op = CC_OP_LOGICB + ot; 1393 tcg_gen_mov_tl(decode->cc_dst, s->T0); 1394 1395 /* 1396 * The manual says that the output is undefined when the 1397 * input is zero, but real hardware leaves it unchanged, and 1398 * real programs appear to depend on that. Accomplish this 1399 * by passing the output as the value to return upon zero. 1400 */ 1401 tcg_gen_ctz_tl(s->T0, s->T0, s->T1); 1402} 1403 1404/* Non-standard convention - on entry T0 is zero-extended input, T1 is the output. */ 1405static void gen_BSR(DisasContext *s, X86DecodedInsn *decode) 1406{ 1407 MemOp ot = decode->op[0].ot; 1408 1409 /* Only the Z bit is defined and it is related to the input. */ 1410 decode->cc_dst = tcg_temp_new(); 1411 decode->cc_op = CC_OP_LOGICB + ot; 1412 tcg_gen_mov_tl(decode->cc_dst, s->T0); 1413 1414 /* 1415 * The manual says that the output is undefined when the 1416 * input is zero, but real hardware leaves it unchanged, and 1417 * real programs appear to depend on that. Accomplish this 1418 * by passing the output as the value to return upon zero. 1419 * Plus, return the bit index of the first 1 bit. 1420 */ 1421 tcg_gen_xori_tl(s->T1, s->T1, TARGET_LONG_BITS - 1); 1422 tcg_gen_clz_tl(s->T0, s->T0, s->T1); 1423 tcg_gen_xori_tl(s->T0, s->T0, TARGET_LONG_BITS - 1); 1424} 1425 1426static void gen_BSWAP(DisasContext *s, X86DecodedInsn *decode) 1427{ 1428#ifdef TARGET_X86_64 1429 if (s->dflag == MO_64) { 1430 tcg_gen_bswap64_i64(s->T0, s->T0); 1431 return; 1432 } 1433#endif 1434 tcg_gen_bswap32_tl(s->T0, s->T0, TCG_BSWAP_OZ); 1435} 1436 1437static TCGv gen_bt_mask(DisasContext *s, X86DecodedInsn *decode) 1438{ 1439 MemOp ot = decode->op[1].ot; 1440 TCGv mask = tcg_temp_new(); 1441 1442 tcg_gen_andi_tl(s->T1, s->T1, (8 << ot) - 1); 1443 tcg_gen_shl_tl(mask, tcg_constant_tl(1), s->T1); 1444 return mask; 1445} 1446 1447/* Expects truncated bit index in COUNT, 1 << COUNT in MASK. */ 1448static void gen_bt_flags(DisasContext *s, X86DecodedInsn *decode, TCGv src, 1449 TCGv count, TCGv mask) 1450{ 1451 TCGv cf; 1452 1453 /* 1454 * C is the result of the test, Z is unchanged, and the others 1455 * are all undefined. 1456 */ 1457 if (s->cc_op == CC_OP_DYNAMIC || CC_OP_HAS_EFLAGS(s->cc_op)) { 1458 /* Generate EFLAGS and replace the C bit. */ 1459 cf = tcg_temp_new(); 1460 tcg_gen_setcond_tl(TCG_COND_TSTNE, cf, src, mask); 1461 prepare_update_cf(decode, s, cf); 1462 } else { 1463 /* 1464 * Z was going to be computed from the non-zero status of CC_DST. 1465 * We can get that same Z value (and the new C value) by leaving 1466 * CC_DST alone, setting CC_SRC, and using a CC_OP_SAR of the 1467 * same width. 1468 */ 1469 decode->cc_src = tcg_temp_new(); 1470 decode->cc_dst = cpu_cc_dst; 1471 decode->cc_op = CC_OP_SARB + cc_op_size(s->cc_op); 1472 tcg_gen_shr_tl(decode->cc_src, src, count); 1473 } 1474} 1475 1476static void gen_BT(DisasContext *s, X86DecodedInsn *decode) 1477{ 1478 TCGv count = s->T1; 1479 TCGv mask; 1480 1481 /* 1482 * Try to ensure that the rhs of the TSTNE condition is a constant (and a 1483 * power of two), as that is more readily available on most TCG backends. 1484 * 1485 * For immediate bit number gen_bt_mask()'s output is already a constant; 1486 * for register bit number, shift the source right and check bit 0. 1487 */ 1488 if (decode->e.op2 == X86_TYPE_I) { 1489 mask = gen_bt_mask(s, decode); 1490 } else { 1491 MemOp ot = decode->op[1].ot; 1492 1493 tcg_gen_andi_tl(s->T1, s->T1, (8 << ot) - 1); 1494 tcg_gen_shr_tl(s->T0, s->T0, s->T1); 1495 1496 count = tcg_constant_tl(0); 1497 mask = tcg_constant_tl(1); 1498 } 1499 gen_bt_flags(s, decode, s->T0, count, mask); 1500} 1501 1502static void gen_BTC(DisasContext *s, X86DecodedInsn *decode) 1503{ 1504 MemOp ot = decode->op[0].ot; 1505 TCGv old = tcg_temp_new(); 1506 TCGv mask = gen_bt_mask(s, decode); 1507 1508 if (s->prefix & PREFIX_LOCK) { 1509 tcg_gen_atomic_fetch_xor_tl(old, s->A0, mask, s->mem_index, ot | MO_LE); 1510 } else { 1511 tcg_gen_mov_tl(old, s->T0); 1512 tcg_gen_xor_tl(s->T0, s->T0, mask); 1513 } 1514 1515 gen_bt_flags(s, decode, old, s->T1, mask); 1516} 1517 1518static void gen_BTR(DisasContext *s, X86DecodedInsn *decode) 1519{ 1520 MemOp ot = decode->op[0].ot; 1521 TCGv old = tcg_temp_new(); 1522 TCGv mask = gen_bt_mask(s, decode); 1523 1524 if (s->prefix & PREFIX_LOCK) { 1525 TCGv maskc = tcg_temp_new(); 1526 tcg_gen_not_tl(maskc, mask); 1527 tcg_gen_atomic_fetch_and_tl(old, s->A0, maskc, s->mem_index, ot | MO_LE); 1528 } else { 1529 tcg_gen_mov_tl(old, s->T0); 1530 tcg_gen_andc_tl(s->T0, s->T0, mask); 1531 } 1532 1533 gen_bt_flags(s, decode, old, s->T1, mask); 1534} 1535 1536static void gen_BTS(DisasContext *s, X86DecodedInsn *decode) 1537{ 1538 MemOp ot = decode->op[0].ot; 1539 TCGv old = tcg_temp_new(); 1540 TCGv mask = gen_bt_mask(s, decode); 1541 1542 if (s->prefix & PREFIX_LOCK) { 1543 tcg_gen_atomic_fetch_or_tl(old, s->A0, mask, s->mem_index, ot | MO_LE); 1544 } else { 1545 tcg_gen_mov_tl(old, s->T0); 1546 tcg_gen_or_tl(s->T0, s->T0, mask); 1547 } 1548 1549 gen_bt_flags(s, decode, old, s->T1, mask); 1550} 1551 1552static void gen_BZHI(DisasContext *s, X86DecodedInsn *decode) 1553{ 1554 MemOp ot = decode->op[0].ot; 1555 TCGv bound = tcg_constant_tl(ot == MO_64 ? 63 : 31); 1556 TCGv zero = tcg_constant_tl(0); 1557 TCGv mone = tcg_constant_tl(-1); 1558 1559 tcg_gen_ext8u_tl(s->T1, s->T1); 1560 1561 tcg_gen_shl_tl(s->A0, mone, s->T1); 1562 tcg_gen_movcond_tl(TCG_COND_LEU, s->A0, s->T1, bound, s->A0, zero); 1563 tcg_gen_andc_tl(s->T0, s->T0, s->A0); 1564 /* 1565 * Note that since we're using BMILG (in order to get O 1566 * cleared) we need to store the inverse into C. 1567 */ 1568 tcg_gen_setcond_tl(TCG_COND_LEU, s->T1, s->T1, bound); 1569 prepare_update2_cc(decode, s, CC_OP_BMILGB + ot); 1570} 1571 1572static void gen_CALL(DisasContext *s, X86DecodedInsn *decode) 1573{ 1574 gen_push_v(s, eip_next_tl(s)); 1575 gen_JMP(s, decode); 1576} 1577 1578static void gen_CALL_m(DisasContext *s, X86DecodedInsn *decode) 1579{ 1580 gen_push_v(s, eip_next_tl(s)); 1581 gen_JMP_m(s, decode); 1582} 1583 1584static void gen_CALLF(DisasContext *s, X86DecodedInsn *decode) 1585{ 1586 gen_far_call(s); 1587} 1588 1589static void gen_CALLF_m(DisasContext *s, X86DecodedInsn *decode) 1590{ 1591 MemOp ot = decode->op[1].ot; 1592 1593 gen_op_ld_v(s, ot, s->T0, s->A0); 1594 gen_add_A0_im(s, 1 << ot); 1595 gen_op_ld_v(s, MO_16, s->T1, s->A0); 1596 gen_far_call(s); 1597} 1598 1599static void gen_CBW(DisasContext *s, X86DecodedInsn *decode) 1600{ 1601 MemOp src_ot = decode->op[0].ot - 1; 1602 1603 tcg_gen_ext_tl(s->T0, s->T0, src_ot | MO_SIGN); 1604} 1605 1606static void gen_CLC(DisasContext *s, X86DecodedInsn *decode) 1607{ 1608 gen_compute_eflags(s); 1609 tcg_gen_andi_tl(cpu_cc_src, cpu_cc_src, ~CC_C); 1610} 1611 1612static void gen_CLD(DisasContext *s, X86DecodedInsn *decode) 1613{ 1614 tcg_gen_st_i32(tcg_constant_i32(1), tcg_env, offsetof(CPUX86State, df)); 1615} 1616 1617static void gen_CLI(DisasContext *s, X86DecodedInsn *decode) 1618{ 1619 gen_reset_eflags(s, IF_MASK); 1620} 1621 1622static void gen_CLTS(DisasContext *s, X86DecodedInsn *decode) 1623{ 1624 gen_helper_clts(tcg_env); 1625 /* abort block because static cpu state changed */ 1626 s->base.is_jmp = DISAS_EOB_NEXT; 1627} 1628 1629static void gen_CMC(DisasContext *s, X86DecodedInsn *decode) 1630{ 1631 gen_compute_eflags(s); 1632 tcg_gen_xori_tl(cpu_cc_src, cpu_cc_src, CC_C); 1633} 1634 1635static void gen_CMOVcc(DisasContext *s, X86DecodedInsn *decode) 1636{ 1637 gen_cmovcc1(s, decode->b & 0xf, s->T0, s->T1); 1638} 1639 1640static void gen_CMPccXADD(DisasContext *s, X86DecodedInsn *decode) 1641{ 1642 TCGLabel *label_top = gen_new_label(); 1643 TCGLabel *label_bottom = gen_new_label(); 1644 TCGv oldv = tcg_temp_new(); 1645 TCGv newv = tcg_temp_new(); 1646 TCGv cmpv = tcg_temp_new(); 1647 TCGCond cond; 1648 1649 TCGv cmp_lhs, cmp_rhs; 1650 MemOp ot, ot_full; 1651 1652 int jcc_op = (decode->b >> 1) & 7; 1653 static const TCGCond cond_table[8] = { 1654 [JCC_O] = TCG_COND_LT, /* test sign bit by comparing against 0 */ 1655 [JCC_B] = TCG_COND_LTU, 1656 [JCC_Z] = TCG_COND_EQ, 1657 [JCC_BE] = TCG_COND_LEU, 1658 [JCC_S] = TCG_COND_LT, /* test sign bit by comparing against 0 */ 1659 [JCC_P] = TCG_COND_TSTEQ, /* even parity - tests low bit of popcount */ 1660 [JCC_L] = TCG_COND_LT, 1661 [JCC_LE] = TCG_COND_LE, 1662 }; 1663 1664 cond = cond_table[jcc_op]; 1665 if (decode->b & 1) { 1666 cond = tcg_invert_cond(cond); 1667 } 1668 1669 ot = decode->op[0].ot; 1670 ot_full = ot | MO_LE; 1671 if (jcc_op >= JCC_S) { 1672 /* 1673 * Sign-extend values before subtracting for S, P (zero/sign extension 1674 * does not matter there) L, LE and their inverses. 1675 */ 1676 ot_full |= MO_SIGN; 1677 } 1678 1679 /* 1680 * cmpv will be moved to cc_src *after* cpu_regs[] is written back, so use 1681 * tcg_gen_ext_tl instead of gen_ext_tl. 1682 */ 1683 tcg_gen_ext_tl(cmpv, cpu_regs[decode->op[1].n], ot_full); 1684 1685 /* 1686 * Cmpxchg loop starts here. 1687 * - s->T1: addition operand (from decoder) 1688 * - s->A0: dest address (from decoder) 1689 * - s->cc_srcT: memory operand (lhs for comparison) 1690 * - cmpv: rhs for comparison 1691 */ 1692 gen_set_label(label_top); 1693 gen_op_ld_v(s, ot_full, s->cc_srcT, s->A0); 1694 tcg_gen_sub_tl(s->T0, s->cc_srcT, cmpv); 1695 1696 /* Compute the comparison result by hand, to avoid clobbering cc_*. */ 1697 switch (jcc_op) { 1698 case JCC_O: 1699 /* (src1 ^ src2) & (src1 ^ dst). newv is only used here for a moment */ 1700 tcg_gen_xor_tl(newv, s->cc_srcT, s->T0); 1701 tcg_gen_xor_tl(s->tmp0, s->cc_srcT, cmpv); 1702 tcg_gen_and_tl(s->tmp0, s->tmp0, newv); 1703 tcg_gen_sextract_tl(s->tmp0, s->tmp0, 0, 8 << ot); 1704 cmp_lhs = s->tmp0, cmp_rhs = tcg_constant_tl(0); 1705 break; 1706 1707 case JCC_P: 1708 tcg_gen_ext8u_tl(s->tmp0, s->T0); 1709 tcg_gen_ctpop_tl(s->tmp0, s->tmp0); 1710 cmp_lhs = s->tmp0, cmp_rhs = tcg_constant_tl(1); 1711 break; 1712 1713 case JCC_S: 1714 tcg_gen_sextract_tl(s->tmp0, s->T0, 0, 8 << ot); 1715 cmp_lhs = s->tmp0, cmp_rhs = tcg_constant_tl(0); 1716 break; 1717 1718 default: 1719 cmp_lhs = s->cc_srcT, cmp_rhs = cmpv; 1720 break; 1721 } 1722 1723 /* Compute new value: if condition does not hold, just store back s->cc_srcT */ 1724 tcg_gen_add_tl(newv, s->cc_srcT, s->T1); 1725 tcg_gen_movcond_tl(cond, newv, cmp_lhs, cmp_rhs, newv, s->cc_srcT); 1726 tcg_gen_atomic_cmpxchg_tl(oldv, s->A0, s->cc_srcT, newv, s->mem_index, ot_full); 1727 1728 /* Exit unconditionally if cmpxchg succeeded. */ 1729 tcg_gen_brcond_tl(TCG_COND_EQ, oldv, s->cc_srcT, label_bottom); 1730 1731 /* Try again if there was actually a store to make. */ 1732 tcg_gen_brcond_tl(cond, cmp_lhs, cmp_rhs, label_top); 1733 gen_set_label(label_bottom); 1734 1735 /* Store old value to registers only after a successful store. */ 1736 gen_writeback(s, decode, 1, s->cc_srcT); 1737 1738 decode->cc_dst = s->T0; 1739 decode->cc_src = cmpv; 1740 decode->cc_op = CC_OP_SUBB + ot; 1741} 1742 1743static void gen_CMPS(DisasContext *s, X86DecodedInsn *decode) 1744{ 1745 MemOp ot = decode->op[2].ot; 1746 if (s->prefix & (PREFIX_REPZ | PREFIX_REPNZ)) { 1747 gen_repz_nz(s, ot, gen_cmps); 1748 } else { 1749 gen_cmps(s, ot); 1750 } 1751} 1752 1753static void gen_CMPXCHG(DisasContext *s, X86DecodedInsn *decode) 1754{ 1755 MemOp ot = decode->op[2].ot; 1756 TCGv cmpv = tcg_temp_new(); 1757 TCGv oldv = tcg_temp_new(); 1758 TCGv newv = tcg_temp_new(); 1759 TCGv dest; 1760 1761 tcg_gen_ext_tl(cmpv, cpu_regs[R_EAX], ot); 1762 tcg_gen_ext_tl(newv, s->T1, ot); 1763 if (s->prefix & PREFIX_LOCK) { 1764 tcg_gen_atomic_cmpxchg_tl(oldv, s->A0, cmpv, newv, 1765 s->mem_index, ot | MO_LE); 1766 } else { 1767 tcg_gen_ext_tl(oldv, s->T0, ot); 1768 if (decode->op[0].has_ea) { 1769 /* 1770 * Perform an unconditional store cycle like physical cpu; 1771 * must be before changing accumulator to ensure 1772 * idempotency if the store faults and the instruction 1773 * is restarted 1774 */ 1775 tcg_gen_movcond_tl(TCG_COND_EQ, newv, oldv, cmpv, newv, oldv); 1776 gen_op_st_v(s, ot, newv, s->A0); 1777 } else { 1778 /* 1779 * Unlike the memory case, where "the destination operand receives 1780 * a write cycle without regard to the result of the comparison", 1781 * rm must not be touched altogether if the write fails, including 1782 * not zero-extending it on 64-bit processors. So, precompute 1783 * the result of a successful writeback and perform the movcond 1784 * directly on cpu_regs. In case rm is part of RAX, note that this 1785 * movcond and the one below are mutually exclusive is executed. 1786 */ 1787 dest = gen_op_deposit_reg_v(s, ot, decode->op[0].n, newv, newv); 1788 tcg_gen_movcond_tl(TCG_COND_EQ, dest, oldv, cmpv, newv, dest); 1789 } 1790 decode->op[0].unit = X86_OP_SKIP; 1791 } 1792 1793 /* Write RAX only if the cmpxchg fails. */ 1794 dest = gen_op_deposit_reg_v(s, ot, R_EAX, s->T0, oldv); 1795 tcg_gen_movcond_tl(TCG_COND_NE, dest, oldv, cmpv, s->T0, dest); 1796 1797 tcg_gen_mov_tl(s->cc_srcT, cmpv); 1798 tcg_gen_sub_tl(cmpv, cmpv, oldv); 1799 decode->cc_dst = cmpv; 1800 decode->cc_src = oldv; 1801 decode->cc_op = CC_OP_SUBB + ot; 1802} 1803 1804static void gen_CMPXCHG16B(DisasContext *s, X86DecodedInsn *decode) 1805{ 1806#ifdef TARGET_X86_64 1807 MemOp mop = MO_TE | MO_128 | MO_ALIGN; 1808 TCGv_i64 t0, t1; 1809 TCGv_i128 cmp, val; 1810 1811 cmp = tcg_temp_new_i128(); 1812 val = tcg_temp_new_i128(); 1813 tcg_gen_concat_i64_i128(cmp, cpu_regs[R_EAX], cpu_regs[R_EDX]); 1814 tcg_gen_concat_i64_i128(val, cpu_regs[R_EBX], cpu_regs[R_ECX]); 1815 1816 /* Only require atomic with LOCK; non-parallel handled in generator. */ 1817 if (s->prefix & PREFIX_LOCK) { 1818 tcg_gen_atomic_cmpxchg_i128(val, s->A0, cmp, val, s->mem_index, mop); 1819 } else { 1820 tcg_gen_nonatomic_cmpxchg_i128(val, s->A0, cmp, val, s->mem_index, mop); 1821 } 1822 1823 tcg_gen_extr_i128_i64(s->T0, s->T1, val); 1824 1825 /* Determine success after the fact. */ 1826 t0 = tcg_temp_new_i64(); 1827 t1 = tcg_temp_new_i64(); 1828 tcg_gen_xor_i64(t0, s->T0, cpu_regs[R_EAX]); 1829 tcg_gen_xor_i64(t1, s->T1, cpu_regs[R_EDX]); 1830 tcg_gen_or_i64(t0, t0, t1); 1831 1832 /* Update Z. */ 1833 gen_compute_eflags(s); 1834 tcg_gen_setcondi_i64(TCG_COND_EQ, t0, t0, 0); 1835 tcg_gen_deposit_tl(cpu_cc_src, cpu_cc_src, t0, ctz32(CC_Z), 1); 1836 1837 /* 1838 * Extract the result values for the register pair. We may do this 1839 * unconditionally, because on success (Z=1), the old value matches 1840 * the previous value in RDX:RAX. 1841 */ 1842 tcg_gen_mov_i64(cpu_regs[R_EAX], s->T0); 1843 tcg_gen_mov_i64(cpu_regs[R_EDX], s->T1); 1844#else 1845 abort(); 1846#endif 1847} 1848 1849static void gen_CMPXCHG8B(DisasContext *s, X86DecodedInsn *decode) 1850{ 1851 TCGv_i64 cmp, val, old; 1852 TCGv Z; 1853 1854 cmp = tcg_temp_new_i64(); 1855 val = tcg_temp_new_i64(); 1856 old = tcg_temp_new_i64(); 1857 1858 /* Construct the comparison values from the register pair. */ 1859 tcg_gen_concat_tl_i64(cmp, cpu_regs[R_EAX], cpu_regs[R_EDX]); 1860 tcg_gen_concat_tl_i64(val, cpu_regs[R_EBX], cpu_regs[R_ECX]); 1861 1862 /* Only require atomic with LOCK; non-parallel handled in generator. */ 1863 if (s->prefix & PREFIX_LOCK) { 1864 tcg_gen_atomic_cmpxchg_i64(old, s->A0, cmp, val, s->mem_index, MO_TEUQ); 1865 } else { 1866 tcg_gen_nonatomic_cmpxchg_i64(old, s->A0, cmp, val, 1867 s->mem_index, MO_TEUQ); 1868 } 1869 1870 /* Set tmp0 to match the required value of Z. */ 1871 tcg_gen_setcond_i64(TCG_COND_EQ, cmp, old, cmp); 1872 Z = tcg_temp_new(); 1873 tcg_gen_trunc_i64_tl(Z, cmp); 1874 1875 /* 1876 * Extract the result values for the register pair. 1877 * For 32-bit, we may do this unconditionally, because on success (Z=1), 1878 * the old value matches the previous value in EDX:EAX. For x86_64, 1879 * the store must be conditional, because we must leave the source 1880 * registers unchanged on success, and zero-extend the writeback 1881 * on failure (Z=0). 1882 */ 1883 if (TARGET_LONG_BITS == 32) { 1884 tcg_gen_extr_i64_tl(cpu_regs[R_EAX], cpu_regs[R_EDX], old); 1885 } else { 1886 TCGv zero = tcg_constant_tl(0); 1887 1888 tcg_gen_extr_i64_tl(s->T0, s->T1, old); 1889 tcg_gen_movcond_tl(TCG_COND_EQ, cpu_regs[R_EAX], Z, zero, 1890 s->T0, cpu_regs[R_EAX]); 1891 tcg_gen_movcond_tl(TCG_COND_EQ, cpu_regs[R_EDX], Z, zero, 1892 s->T1, cpu_regs[R_EDX]); 1893 } 1894 1895 /* Update Z. */ 1896 gen_compute_eflags(s); 1897 tcg_gen_deposit_tl(cpu_cc_src, cpu_cc_src, Z, ctz32(CC_Z), 1); 1898} 1899 1900static void gen_CPUID(DisasContext *s, X86DecodedInsn *decode) 1901{ 1902 gen_update_cc_op(s); 1903 gen_update_eip_cur(s); 1904 gen_helper_cpuid(tcg_env); 1905} 1906 1907static void gen_CRC32(DisasContext *s, X86DecodedInsn *decode) 1908{ 1909 MemOp ot = decode->op[2].ot; 1910 1911 tcg_gen_trunc_tl_i32(s->tmp2_i32, s->T0); 1912 gen_helper_crc32(s->T0, s->tmp2_i32, s->T1, tcg_constant_i32(8 << ot)); 1913} 1914 1915static void gen_CVTPI2Px(DisasContext *s, X86DecodedInsn *decode) 1916{ 1917 gen_helper_enter_mmx(tcg_env); 1918 if (s->prefix & PREFIX_DATA) { 1919 gen_helper_cvtpi2pd(tcg_env, OP_PTR0, OP_PTR2); 1920 } else { 1921 gen_helper_cvtpi2ps(tcg_env, OP_PTR0, OP_PTR2); 1922 } 1923} 1924 1925static void gen_CVTPx2PI(DisasContext *s, X86DecodedInsn *decode) 1926{ 1927 gen_helper_enter_mmx(tcg_env); 1928 if (s->prefix & PREFIX_DATA) { 1929 gen_helper_cvtpd2pi(tcg_env, OP_PTR0, OP_PTR2); 1930 } else { 1931 gen_helper_cvtps2pi(tcg_env, OP_PTR0, OP_PTR2); 1932 } 1933} 1934 1935static void gen_CVTTPx2PI(DisasContext *s, X86DecodedInsn *decode) 1936{ 1937 gen_helper_enter_mmx(tcg_env); 1938 if (s->prefix & PREFIX_DATA) { 1939 gen_helper_cvttpd2pi(tcg_env, OP_PTR0, OP_PTR2); 1940 } else { 1941 gen_helper_cvttps2pi(tcg_env, OP_PTR0, OP_PTR2); 1942 } 1943} 1944 1945static void gen_CWD(DisasContext *s, X86DecodedInsn *decode) 1946{ 1947 int shift = 8 << decode->op[0].ot; 1948 1949 tcg_gen_sextract_tl(s->T0, s->T0, shift - 1, 1); 1950} 1951 1952static void gen_DAA(DisasContext *s, X86DecodedInsn *decode) 1953{ 1954 gen_update_cc_op(s); 1955 gen_helper_daa(tcg_env); 1956 assume_cc_op(s, CC_OP_EFLAGS); 1957} 1958 1959static void gen_DAS(DisasContext *s, X86DecodedInsn *decode) 1960{ 1961 gen_update_cc_op(s); 1962 gen_helper_das(tcg_env); 1963 assume_cc_op(s, CC_OP_EFLAGS); 1964} 1965 1966static void gen_DEC(DisasContext *s, X86DecodedInsn *decode) 1967{ 1968 MemOp ot = decode->op[1].ot; 1969 1970 tcg_gen_movi_tl(s->T1, -1); 1971 if (s->prefix & PREFIX_LOCK) { 1972 tcg_gen_atomic_add_fetch_tl(s->T0, s->A0, s->T1, 1973 s->mem_index, ot | MO_LE); 1974 } else { 1975 tcg_gen_add_tl(s->T0, s->T0, s->T1); 1976 } 1977 prepare_update_cc_incdec(decode, s, CC_OP_DECB + ot); 1978} 1979 1980static void gen_DIV(DisasContext *s, X86DecodedInsn *decode) 1981{ 1982 MemOp ot = decode->op[1].ot; 1983 1984 switch(ot) { 1985 case MO_8: 1986 gen_helper_divb_AL(tcg_env, s->T0); 1987 break; 1988 case MO_16: 1989 gen_helper_divw_AX(tcg_env, s->T0); 1990 break; 1991 default: 1992 case MO_32: 1993 gen_helper_divl_EAX(tcg_env, s->T0); 1994 break; 1995#ifdef TARGET_X86_64 1996 case MO_64: 1997 gen_helper_divq_EAX(tcg_env, s->T0); 1998 break; 1999#endif 2000 } 2001} 2002 2003static void gen_EMMS(DisasContext *s, X86DecodedInsn *decode) 2004{ 2005 gen_helper_emms(tcg_env); 2006} 2007 2008static void gen_ENTER(DisasContext *s, X86DecodedInsn *decode) 2009{ 2010 gen_enter(s, decode->op[1].imm, decode->op[2].imm); 2011} 2012 2013static void gen_EXTRQ_i(DisasContext *s, X86DecodedInsn *decode) 2014{ 2015 TCGv_i32 length = tcg_constant_i32(decode->immediate & 63); 2016 TCGv_i32 index = tcg_constant_i32((decode->immediate >> 8) & 63); 2017 2018 gen_helper_extrq_i(tcg_env, OP_PTR0, index, length); 2019} 2020 2021static void gen_EXTRQ_r(DisasContext *s, X86DecodedInsn *decode) 2022{ 2023 gen_helper_extrq_r(tcg_env, OP_PTR0, OP_PTR2); 2024} 2025 2026static void gen_FXRSTOR(DisasContext *s, X86DecodedInsn *decode) 2027{ 2028 if ((s->flags & HF_EM_MASK) || (s->flags & HF_TS_MASK)) { 2029 gen_NM_exception(s); 2030 } else { 2031 gen_helper_fxrstor(tcg_env, s->A0); 2032 } 2033} 2034 2035static void gen_FXSAVE(DisasContext *s, X86DecodedInsn *decode) 2036{ 2037 if ((s->flags & HF_EM_MASK) || (s->flags & HF_TS_MASK)) { 2038 gen_NM_exception(s); 2039 } else { 2040 gen_helper_fxsave(tcg_env, s->A0); 2041 } 2042} 2043 2044static void gen_HLT(DisasContext *s, X86DecodedInsn *decode) 2045{ 2046#ifdef CONFIG_SYSTEM_ONLY 2047 gen_update_cc_op(s); 2048 gen_update_eip_next(s); 2049 gen_helper_hlt(tcg_env); 2050 s->base.is_jmp = DISAS_NORETURN; 2051#endif 2052} 2053 2054static void gen_IDIV(DisasContext *s, X86DecodedInsn *decode) 2055{ 2056 MemOp ot = decode->op[1].ot; 2057 2058 switch(ot) { 2059 case MO_8: 2060 gen_helper_idivb_AL(tcg_env, s->T0); 2061 break; 2062 case MO_16: 2063 gen_helper_idivw_AX(tcg_env, s->T0); 2064 break; 2065 default: 2066 case MO_32: 2067 gen_helper_idivl_EAX(tcg_env, s->T0); 2068 break; 2069#ifdef TARGET_X86_64 2070 case MO_64: 2071 gen_helper_idivq_EAX(tcg_env, s->T0); 2072 break; 2073#endif 2074 } 2075} 2076 2077static void gen_IMUL3(DisasContext *s, X86DecodedInsn *decode) 2078{ 2079 MemOp ot = decode->op[0].ot; 2080 TCGv cc_src_rhs; 2081 2082 switch (ot) { 2083 case MO_16: 2084 /* s->T0 already sign-extended */ 2085 tcg_gen_ext16s_tl(s->T1, s->T1); 2086 tcg_gen_mul_tl(s->T0, s->T0, s->T1); 2087 /* Compare the full result to the extension of the truncated result. */ 2088 tcg_gen_ext16s_tl(s->T1, s->T0); 2089 cc_src_rhs = s->T0; 2090 break; 2091 2092 case MO_32: 2093#ifdef TARGET_X86_64 2094 if (TCG_TARGET_REG_BITS == 64) { 2095 /* 2096 * This produces fewer TCG ops, and better code if flags are needed, 2097 * but it requires a 64-bit multiply even if they are not. Use it 2098 * only if the target has 64-bits registers. 2099 * 2100 * s->T0 is already sign-extended. 2101 */ 2102 tcg_gen_ext32s_tl(s->T1, s->T1); 2103 tcg_gen_mul_tl(s->T0, s->T0, s->T1); 2104 /* Compare the full result to the extension of the truncated result. */ 2105 tcg_gen_ext32s_tl(s->T1, s->T0); 2106 cc_src_rhs = s->T0; 2107 } else { 2108 /* Variant that only needs a 32-bit widening multiply. */ 2109 TCGv_i32 hi = tcg_temp_new_i32(); 2110 TCGv_i32 lo = tcg_temp_new_i32(); 2111 tcg_gen_trunc_tl_i32(lo, s->T0); 2112 tcg_gen_trunc_tl_i32(hi, s->T1); 2113 tcg_gen_muls2_i32(lo, hi, lo, hi); 2114 tcg_gen_extu_i32_tl(s->T0, lo); 2115 2116 cc_src_rhs = tcg_temp_new(); 2117 tcg_gen_extu_i32_tl(cc_src_rhs, hi); 2118 /* Compare the high part to the sign bit of the truncated result */ 2119 tcg_gen_sari_i32(lo, lo, 31); 2120 tcg_gen_extu_i32_tl(s->T1, lo); 2121 } 2122 break; 2123 2124 case MO_64: 2125#endif 2126 cc_src_rhs = tcg_temp_new(); 2127 tcg_gen_muls2_tl(s->T0, cc_src_rhs, s->T0, s->T1); 2128 /* Compare the high part to the sign bit of the truncated result */ 2129 tcg_gen_sari_tl(s->T1, s->T0, TARGET_LONG_BITS - 1); 2130 break; 2131 2132 default: 2133 g_assert_not_reached(); 2134 } 2135 2136 tcg_gen_sub_tl(s->T1, s->T1, cc_src_rhs); 2137 prepare_update2_cc(decode, s, CC_OP_MULB + ot); 2138} 2139 2140static void gen_IMUL(DisasContext *s, X86DecodedInsn *decode) 2141{ 2142 MemOp ot = decode->op[1].ot; 2143 TCGv cc_src_rhs; 2144 2145 switch (ot) { 2146 case MO_8: 2147 /* s->T0 already sign-extended */ 2148 tcg_gen_ext8s_tl(s->T1, s->T1); 2149 tcg_gen_mul_tl(s->T0, s->T0, s->T1); 2150 gen_op_mov_reg_v(s, MO_16, R_EAX, s->T0); 2151 /* Compare the full result to the extension of the truncated result. */ 2152 tcg_gen_ext8s_tl(s->T1, s->T0); 2153 cc_src_rhs = s->T0; 2154 break; 2155 2156 case MO_16: 2157 /* s->T0 already sign-extended */ 2158 tcg_gen_ext16s_tl(s->T1, s->T1); 2159 tcg_gen_mul_tl(s->T0, s->T0, s->T1); 2160 gen_op_mov_reg_v(s, MO_16, R_EAX, s->T0); 2161 tcg_gen_shri_tl(s->T1, s->T0, 16); 2162 gen_op_mov_reg_v(s, MO_16, R_EDX, s->T1); 2163 /* Compare the full result to the extension of the truncated result. */ 2164 tcg_gen_ext16s_tl(s->T1, s->T0); 2165 cc_src_rhs = s->T0; 2166 break; 2167 2168 case MO_32: 2169#ifdef TARGET_X86_64 2170 /* s->T0 already sign-extended */ 2171 tcg_gen_ext32s_tl(s->T1, s->T1); 2172 tcg_gen_mul_tl(s->T0, s->T0, s->T1); 2173 tcg_gen_ext32u_tl(cpu_regs[R_EAX], s->T0); 2174 tcg_gen_shri_tl(cpu_regs[R_EDX], s->T0, 32); 2175 /* Compare the full result to the extension of the truncated result. */ 2176 tcg_gen_ext32s_tl(s->T1, s->T0); 2177 cc_src_rhs = s->T0; 2178 break; 2179 2180 case MO_64: 2181#endif 2182 tcg_gen_muls2_tl(s->T0, cpu_regs[R_EDX], s->T0, s->T1); 2183 tcg_gen_mov_tl(cpu_regs[R_EAX], s->T0); 2184 2185 /* Compare the high part to the sign bit of the truncated result */ 2186 tcg_gen_negsetcondi_tl(TCG_COND_LT, s->T1, s->T0, 0); 2187 cc_src_rhs = cpu_regs[R_EDX]; 2188 break; 2189 2190 default: 2191 g_assert_not_reached(); 2192 } 2193 2194 tcg_gen_sub_tl(s->T1, s->T1, cc_src_rhs); 2195 prepare_update2_cc(decode, s, CC_OP_MULB + ot); 2196} 2197 2198static void gen_IN(DisasContext *s, X86DecodedInsn *decode) 2199{ 2200 MemOp ot = decode->op[0].ot; 2201 TCGv_i32 port = tcg_temp_new_i32(); 2202 2203 tcg_gen_trunc_tl_i32(port, s->T0); 2204 tcg_gen_ext16u_i32(port, port); 2205 if (!gen_check_io(s, ot, port, SVM_IOIO_TYPE_MASK)) { 2206 return; 2207 } 2208 translator_io_start(&s->base); 2209 gen_helper_in_func(ot, s->T0, port); 2210 gen_writeback(s, decode, 0, s->T0); 2211 gen_bpt_io(s, port, ot); 2212} 2213 2214static void gen_INC(DisasContext *s, X86DecodedInsn *decode) 2215{ 2216 MemOp ot = decode->op[1].ot; 2217 2218 tcg_gen_movi_tl(s->T1, 1); 2219 if (s->prefix & PREFIX_LOCK) { 2220 tcg_gen_atomic_add_fetch_tl(s->T0, s->A0, s->T1, 2221 s->mem_index, ot | MO_LE); 2222 } else { 2223 tcg_gen_add_tl(s->T0, s->T0, s->T1); 2224 } 2225 prepare_update_cc_incdec(decode, s, CC_OP_INCB + ot); 2226} 2227 2228static void gen_INS(DisasContext *s, X86DecodedInsn *decode) 2229{ 2230 MemOp ot = decode->op[1].ot; 2231 TCGv_i32 port = tcg_temp_new_i32(); 2232 2233 tcg_gen_trunc_tl_i32(port, s->T1); 2234 tcg_gen_ext16u_i32(port, port); 2235 if (!gen_check_io(s, ot, port, 2236 SVM_IOIO_TYPE_MASK | SVM_IOIO_STR_MASK)) { 2237 return; 2238 } 2239 2240 translator_io_start(&s->base); 2241 if (s->prefix & (PREFIX_REPZ | PREFIX_REPNZ)) { 2242 gen_repz(s, ot, gen_ins); 2243 } else { 2244 gen_ins(s, ot); 2245 } 2246} 2247 2248static void gen_INSERTQ_i(DisasContext *s, X86DecodedInsn *decode) 2249{ 2250 TCGv_i32 length = tcg_constant_i32(decode->immediate & 63); 2251 TCGv_i32 index = tcg_constant_i32((decode->immediate >> 8) & 63); 2252 2253 gen_helper_insertq_i(tcg_env, OP_PTR0, OP_PTR1, index, length); 2254} 2255 2256static void gen_INSERTQ_r(DisasContext *s, X86DecodedInsn *decode) 2257{ 2258 gen_helper_insertq_r(tcg_env, OP_PTR0, OP_PTR2); 2259} 2260 2261static void gen_INT(DisasContext *s, X86DecodedInsn *decode) 2262{ 2263 gen_interrupt(s, decode->immediate); 2264} 2265 2266static void gen_INT1(DisasContext *s, X86DecodedInsn *decode) 2267{ 2268 gen_update_cc_op(s); 2269 gen_update_eip_next(s); 2270 gen_helper_icebp(tcg_env); 2271 s->base.is_jmp = DISAS_NORETURN; 2272} 2273 2274static void gen_INT3(DisasContext *s, X86DecodedInsn *decode) 2275{ 2276 gen_interrupt(s, EXCP03_INT3); 2277} 2278 2279static void gen_INTO(DisasContext *s, X86DecodedInsn *decode) 2280{ 2281 gen_update_cc_op(s); 2282 gen_update_eip_cur(s); 2283 gen_helper_into(tcg_env, cur_insn_len_i32(s)); 2284} 2285 2286static void gen_IRET(DisasContext *s, X86DecodedInsn *decode) 2287{ 2288 if (!PE(s) || VM86(s)) { 2289 gen_helper_iret_real(tcg_env, tcg_constant_i32(s->dflag - 1)); 2290 } else { 2291 gen_helper_iret_protected(tcg_env, tcg_constant_i32(s->dflag - 1), 2292 eip_next_i32(s)); 2293 } 2294 assume_cc_op(s, CC_OP_EFLAGS); 2295 s->base.is_jmp = DISAS_EOB_ONLY; 2296} 2297 2298static void gen_Jcc(DisasContext *s, X86DecodedInsn *decode) 2299{ 2300 gen_bnd_jmp(s); 2301 gen_jcc(s, decode->b & 0xf, decode->immediate); 2302} 2303 2304static void gen_JCXZ(DisasContext *s, X86DecodedInsn *decode) 2305{ 2306 TCGLabel *taken = gen_new_label(); 2307 2308 gen_update_cc_op(s); 2309 gen_op_jz_ecx(s, taken); 2310 gen_conditional_jump_labels(s, decode->immediate, NULL, taken); 2311} 2312 2313static void gen_JMP(DisasContext *s, X86DecodedInsn *decode) 2314{ 2315 gen_update_cc_op(s); 2316 gen_jmp_rel(s, s->dflag, decode->immediate, 0); 2317} 2318 2319static void gen_JMP_m(DisasContext *s, X86DecodedInsn *decode) 2320{ 2321 gen_op_jmp_v(s, s->T0); 2322 gen_bnd_jmp(s); 2323 s->base.is_jmp = DISAS_JUMP; 2324} 2325 2326static void gen_JMPF(DisasContext *s, X86DecodedInsn *decode) 2327{ 2328 gen_far_jmp(s); 2329} 2330 2331static void gen_JMPF_m(DisasContext *s, X86DecodedInsn *decode) 2332{ 2333 MemOp ot = decode->op[1].ot; 2334 2335 gen_op_ld_v(s, ot, s->T0, s->A0); 2336 gen_add_A0_im(s, 1 << ot); 2337 gen_op_ld_v(s, MO_16, s->T1, s->A0); 2338 gen_far_jmp(s); 2339} 2340 2341static void gen_LAHF(DisasContext *s, X86DecodedInsn *decode) 2342{ 2343 if (CODE64(s) && !(s->cpuid_ext3_features & CPUID_EXT3_LAHF_LM)) { 2344 return gen_illegal_opcode(s); 2345 } 2346 gen_compute_eflags(s); 2347 /* Note: gen_compute_eflags() only gives the condition codes */ 2348 tcg_gen_ori_tl(s->T0, cpu_cc_src, 0x02); 2349 tcg_gen_deposit_tl(cpu_regs[R_EAX], cpu_regs[R_EAX], s->T0, 8, 8); 2350} 2351 2352static void gen_LAR(DisasContext *s, X86DecodedInsn *decode) 2353{ 2354 MemOp ot = decode->op[0].ot; 2355 TCGv result = tcg_temp_new(); 2356 TCGv dest; 2357 2358 gen_compute_eflags(s); 2359 gen_update_cc_op(s); 2360 gen_helper_lar(result, tcg_env, s->T0); 2361 2362 /* Perform writeback here to skip it if ZF=0. */ 2363 decode->op[0].unit = X86_OP_SKIP; 2364 dest = gen_op_deposit_reg_v(s, ot, decode->op[0].n, result, result); 2365 tcg_gen_movcond_tl(TCG_COND_TSTNE, dest, cpu_cc_src, tcg_constant_tl(CC_Z), 2366 result, dest); 2367} 2368 2369static void gen_LDMXCSR(DisasContext *s, X86DecodedInsn *decode) 2370{ 2371 tcg_gen_trunc_tl_i32(s->tmp2_i32, s->T0); 2372 gen_helper_ldmxcsr(tcg_env, s->tmp2_i32); 2373} 2374 2375static void gen_lxx_seg(DisasContext *s, X86DecodedInsn *decode, int seg) 2376{ 2377 MemOp ot = decode->op[0].ot; 2378 2379 /* Offset already in s->T0. */ 2380 gen_add_A0_im(s, 1 << ot); 2381 gen_op_ld_v(s, MO_16, s->T1, s->A0); 2382 2383 /* load the segment here to handle exceptions properly */ 2384 gen_movl_seg(s, seg, s->T1); 2385} 2386 2387static void gen_LDS(DisasContext *s, X86DecodedInsn *decode) 2388{ 2389 gen_lxx_seg(s, decode, R_DS); 2390} 2391 2392static void gen_LEA(DisasContext *s, X86DecodedInsn *decode) 2393{ 2394 TCGv ea = gen_lea_modrm_1(s, decode->mem, false); 2395 gen_lea_v_seg_dest(s, s->aflag, s->T0, ea, -1, -1); 2396} 2397 2398static void gen_LEAVE(DisasContext *s, X86DecodedInsn *decode) 2399{ 2400 gen_leave(s); 2401} 2402 2403static void gen_LES(DisasContext *s, X86DecodedInsn *decode) 2404{ 2405 gen_lxx_seg(s, decode, R_ES); 2406} 2407 2408static void gen_LFENCE(DisasContext *s, X86DecodedInsn *decode) 2409{ 2410 tcg_gen_mb(TCG_MO_LD_LD | TCG_BAR_SC); 2411} 2412 2413static void gen_LFS(DisasContext *s, X86DecodedInsn *decode) 2414{ 2415 gen_lxx_seg(s, decode, R_FS); 2416} 2417 2418static void gen_LGS(DisasContext *s, X86DecodedInsn *decode) 2419{ 2420 gen_lxx_seg(s, decode, R_GS); 2421} 2422 2423static void gen_LODS(DisasContext *s, X86DecodedInsn *decode) 2424{ 2425 MemOp ot = decode->op[1].ot; 2426 if (s->prefix & (PREFIX_REPZ | PREFIX_REPNZ)) { 2427 gen_repz(s, ot, gen_lods); 2428 } else { 2429 gen_lods(s, ot); 2430 } 2431} 2432 2433static void gen_LOOP(DisasContext *s, X86DecodedInsn *decode) 2434{ 2435 TCGLabel *taken = gen_new_label(); 2436 2437 gen_update_cc_op(s); 2438 gen_op_add_reg_im(s, s->aflag, R_ECX, -1); 2439 gen_op_jnz_ecx(s, taken); 2440 gen_conditional_jump_labels(s, decode->immediate, NULL, taken); 2441} 2442 2443static void gen_LOOPE(DisasContext *s, X86DecodedInsn *decode) 2444{ 2445 TCGLabel *taken = gen_new_label(); 2446 TCGLabel *not_taken = gen_new_label(); 2447 2448 gen_update_cc_op(s); 2449 gen_op_add_reg_im(s, s->aflag, R_ECX, -1); 2450 gen_op_jz_ecx(s, not_taken); 2451 gen_jcc1(s, (JCC_Z << 1), taken); /* jz taken */ 2452 gen_conditional_jump_labels(s, decode->immediate, not_taken, taken); 2453} 2454 2455static void gen_LOOPNE(DisasContext *s, X86DecodedInsn *decode) 2456{ 2457 TCGLabel *taken = gen_new_label(); 2458 TCGLabel *not_taken = gen_new_label(); 2459 2460 gen_update_cc_op(s); 2461 gen_op_add_reg_im(s, s->aflag, R_ECX, -1); 2462 gen_op_jz_ecx(s, not_taken); 2463 gen_jcc1(s, (JCC_Z << 1) | 1, taken); /* jnz taken */ 2464 gen_conditional_jump_labels(s, decode->immediate, not_taken, taken); 2465} 2466 2467static void gen_LSL(DisasContext *s, X86DecodedInsn *decode) 2468{ 2469 MemOp ot = decode->op[0].ot; 2470 TCGv result = tcg_temp_new(); 2471 TCGv dest; 2472 2473 gen_compute_eflags(s); 2474 gen_update_cc_op(s); 2475 gen_helper_lsl(result, tcg_env, s->T0); 2476 2477 /* Perform writeback here to skip it if ZF=0. */ 2478 decode->op[0].unit = X86_OP_SKIP; 2479 dest = gen_op_deposit_reg_v(s, ot, decode->op[0].n, result, result); 2480 tcg_gen_movcond_tl(TCG_COND_TSTNE, dest, cpu_cc_src, tcg_constant_tl(CC_Z), 2481 result, dest); 2482} 2483 2484static void gen_LSS(DisasContext *s, X86DecodedInsn *decode) 2485{ 2486 gen_lxx_seg(s, decode, R_SS); 2487} 2488 2489static void gen_LZCNT(DisasContext *s, X86DecodedInsn *decode) 2490{ 2491 MemOp ot = decode->op[0].ot; 2492 2493 /* C bit (cc_src) is defined related to the input. */ 2494 decode->cc_src = tcg_temp_new(); 2495 decode->cc_dst = s->T0; 2496 decode->cc_op = CC_OP_BMILGB + ot; 2497 tcg_gen_mov_tl(decode->cc_src, s->T0); 2498 2499 /* 2500 * Reduce the target_ulong result by the number of zeros that 2501 * we expect to find at the top. 2502 */ 2503 tcg_gen_clzi_tl(s->T0, s->T0, TARGET_LONG_BITS); 2504 tcg_gen_subi_tl(s->T0, s->T0, TARGET_LONG_BITS - (8 << ot)); 2505} 2506 2507static void gen_MFENCE(DisasContext *s, X86DecodedInsn *decode) 2508{ 2509 tcg_gen_mb(TCG_MO_ALL | TCG_BAR_SC); 2510} 2511 2512static void gen_MOV(DisasContext *s, X86DecodedInsn *decode) 2513{ 2514 /* nothing to do! */ 2515} 2516#define gen_NOP gen_MOV 2517 2518static void gen_MASKMOV(DisasContext *s, X86DecodedInsn *decode) 2519{ 2520 gen_lea_v_seg(s, cpu_regs[R_EDI], R_DS, s->override); 2521 2522 if (s->prefix & PREFIX_DATA) { 2523 gen_helper_maskmov_xmm(tcg_env, OP_PTR1, OP_PTR2, s->A0); 2524 } else { 2525 gen_helper_maskmov_mmx(tcg_env, OP_PTR1, OP_PTR2, s->A0); 2526 } 2527} 2528 2529static void gen_MOVBE(DisasContext *s, X86DecodedInsn *decode) 2530{ 2531 MemOp ot = decode->op[0].ot; 2532 2533 /* M operand type does not load/store */ 2534 if (decode->e.op0 == X86_TYPE_M) { 2535 tcg_gen_qemu_st_tl(s->T0, s->A0, s->mem_index, ot | MO_BE); 2536 } else { 2537 tcg_gen_qemu_ld_tl(s->T0, s->A0, s->mem_index, ot | MO_BE); 2538 } 2539} 2540 2541static void gen_MOVD_from(DisasContext *s, X86DecodedInsn *decode) 2542{ 2543 MemOp ot = decode->op[2].ot; 2544 2545 switch (ot) { 2546 case MO_32: 2547#ifdef TARGET_X86_64 2548 tcg_gen_ld32u_tl(s->T0, tcg_env, decode->op[2].offset); 2549 break; 2550 case MO_64: 2551#endif 2552 tcg_gen_ld_tl(s->T0, tcg_env, decode->op[2].offset); 2553 break; 2554 default: 2555 abort(); 2556 } 2557} 2558 2559static void gen_MOVD_to(DisasContext *s, X86DecodedInsn *decode) 2560{ 2561 MemOp ot = decode->op[2].ot; 2562 int vec_len = vector_len(s, decode); 2563 int lo_ofs = vector_elem_offset(&decode->op[0], ot, 0); 2564 2565 tcg_gen_gvec_dup_imm(MO_64, decode->op[0].offset, vec_len, vec_len, 0); 2566 2567 switch (ot) { 2568 case MO_32: 2569#ifdef TARGET_X86_64 2570 tcg_gen_st32_tl(s->T1, tcg_env, lo_ofs); 2571 break; 2572 case MO_64: 2573#endif 2574 tcg_gen_st_tl(s->T1, tcg_env, lo_ofs); 2575 break; 2576 default: 2577 g_assert_not_reached(); 2578 } 2579} 2580 2581static void gen_MOVDQ(DisasContext *s, X86DecodedInsn *decode) 2582{ 2583 gen_store_sse(s, decode, decode->op[2].offset); 2584} 2585 2586static void gen_MOVMSK(DisasContext *s, X86DecodedInsn *decode) 2587{ 2588 typeof(gen_helper_movmskps_ymm) *ps, *pd, *fn; 2589 ps = s->vex_l ? gen_helper_movmskps_ymm : gen_helper_movmskps_xmm; 2590 pd = s->vex_l ? gen_helper_movmskpd_ymm : gen_helper_movmskpd_xmm; 2591 fn = s->prefix & PREFIX_DATA ? pd : ps; 2592 fn(s->tmp2_i32, tcg_env, OP_PTR2); 2593 tcg_gen_extu_i32_tl(s->T0, s->tmp2_i32); 2594} 2595 2596static void gen_MOVQ(DisasContext *s, X86DecodedInsn *decode) 2597{ 2598 int vec_len = vector_len(s, decode); 2599 int lo_ofs = vector_elem_offset(&decode->op[0], MO_64, 0); 2600 2601 tcg_gen_ld_i64(s->tmp1_i64, tcg_env, decode->op[2].offset); 2602 if (decode->op[0].has_ea) { 2603 tcg_gen_qemu_st_i64(s->tmp1_i64, s->A0, s->mem_index, MO_LEUQ); 2604 } else { 2605 /* 2606 * tcg_gen_gvec_dup_i64(MO_64, op0.offset, 8, vec_len, s->tmp1_64) would 2607 * seem to work, but it does not on big-endian platforms; the cleared parts 2608 * are always at higher addresses, but cross-endian emulation inverts the 2609 * byte order so that the cleared parts need to be at *lower* addresses. 2610 * Because oprsz is 8, we see this here even for SSE; but more in general, 2611 * it disqualifies using oprsz < maxsz to emulate VEX128. 2612 */ 2613 tcg_gen_gvec_dup_imm(MO_64, decode->op[0].offset, vec_len, vec_len, 0); 2614 tcg_gen_st_i64(s->tmp1_i64, tcg_env, lo_ofs); 2615 } 2616} 2617 2618static void gen_MOVq_dq(DisasContext *s, X86DecodedInsn *decode) 2619{ 2620 gen_helper_enter_mmx(tcg_env); 2621 /* Otherwise the same as any other movq. */ 2622 return gen_MOVQ(s, decode); 2623} 2624 2625static void gen_MOVS(DisasContext *s, X86DecodedInsn *decode) 2626{ 2627 MemOp ot = decode->op[2].ot; 2628 if (s->prefix & (PREFIX_REPZ | PREFIX_REPNZ)) { 2629 gen_repz(s, ot, gen_movs); 2630 } else { 2631 gen_movs(s, ot); 2632 } 2633} 2634 2635static void gen_MUL(DisasContext *s, X86DecodedInsn *decode) 2636{ 2637 MemOp ot = decode->op[1].ot; 2638 2639 switch (ot) { 2640 case MO_8: 2641 /* s->T0 already zero-extended */ 2642 tcg_gen_ext8u_tl(s->T1, s->T1); 2643 tcg_gen_mul_tl(s->T0, s->T0, s->T1); 2644 gen_op_mov_reg_v(s, MO_16, R_EAX, s->T0); 2645 tcg_gen_andi_tl(s->T1, s->T0, 0xff00); 2646 decode->cc_dst = s->T0; 2647 decode->cc_src = s->T1; 2648 break; 2649 2650 case MO_16: 2651 /* s->T0 already zero-extended */ 2652 tcg_gen_ext16u_tl(s->T1, s->T1); 2653 tcg_gen_mul_tl(s->T0, s->T0, s->T1); 2654 gen_op_mov_reg_v(s, MO_16, R_EAX, s->T0); 2655 tcg_gen_shri_tl(s->T1, s->T0, 16); 2656 gen_op_mov_reg_v(s, MO_16, R_EDX, s->T1); 2657 decode->cc_dst = s->T0; 2658 decode->cc_src = s->T1; 2659 break; 2660 2661 case MO_32: 2662#ifdef TARGET_X86_64 2663 /* s->T0 already zero-extended */ 2664 tcg_gen_ext32u_tl(s->T1, s->T1); 2665 tcg_gen_mul_tl(s->T0, s->T0, s->T1); 2666 tcg_gen_ext32u_tl(cpu_regs[R_EAX], s->T0); 2667 tcg_gen_shri_tl(cpu_regs[R_EDX], s->T0, 32); 2668 decode->cc_dst = cpu_regs[R_EAX]; 2669 decode->cc_src = cpu_regs[R_EDX]; 2670 break; 2671 2672 case MO_64: 2673#endif 2674 tcg_gen_mulu2_tl(cpu_regs[R_EAX], cpu_regs[R_EDX], s->T0, s->T1); 2675 decode->cc_dst = cpu_regs[R_EAX]; 2676 decode->cc_src = cpu_regs[R_EDX]; 2677 break; 2678 2679 default: 2680 g_assert_not_reached(); 2681 } 2682 2683 decode->cc_op = CC_OP_MULB + ot; 2684} 2685 2686static void gen_MULX(DisasContext *s, X86DecodedInsn *decode) 2687{ 2688 MemOp ot = decode->op[0].ot; 2689 2690 /* low part of result in VEX.vvvv, high in MODRM */ 2691 switch (ot) { 2692 case MO_32: 2693#ifdef TARGET_X86_64 2694 tcg_gen_trunc_tl_i32(s->tmp2_i32, s->T0); 2695 tcg_gen_trunc_tl_i32(s->tmp3_i32, s->T1); 2696 tcg_gen_mulu2_i32(s->tmp2_i32, s->tmp3_i32, 2697 s->tmp2_i32, s->tmp3_i32); 2698 tcg_gen_extu_i32_tl(cpu_regs[s->vex_v], s->tmp2_i32); 2699 tcg_gen_extu_i32_tl(s->T0, s->tmp3_i32); 2700 break; 2701 2702 case MO_64: 2703#endif 2704 tcg_gen_mulu2_tl(cpu_regs[s->vex_v], s->T0, s->T0, s->T1); 2705 break; 2706 2707 default: 2708 g_assert_not_reached(); 2709 } 2710} 2711 2712static void gen_NEG(DisasContext *s, X86DecodedInsn *decode) 2713{ 2714 MemOp ot = decode->op[0].ot; 2715 TCGv oldv = tcg_temp_new(); 2716 2717 if (s->prefix & PREFIX_LOCK) { 2718 TCGv newv = tcg_temp_new(); 2719 TCGv cmpv = tcg_temp_new(); 2720 TCGLabel *label1 = gen_new_label(); 2721 2722 gen_set_label(label1); 2723 gen_op_ld_v(s, ot, oldv, s->A0); 2724 tcg_gen_neg_tl(newv, oldv); 2725 tcg_gen_atomic_cmpxchg_tl(cmpv, s->A0, oldv, newv, 2726 s->mem_index, ot | MO_LE); 2727 tcg_gen_brcond_tl(TCG_COND_NE, oldv, cmpv, label1); 2728 } else { 2729 tcg_gen_mov_tl(oldv, s->T0); 2730 } 2731 tcg_gen_neg_tl(s->T0, oldv); 2732 2733 decode->cc_dst = s->T0; 2734 decode->cc_src = oldv; 2735 tcg_gen_movi_tl(s->cc_srcT, 0); 2736 decode->cc_op = CC_OP_SUBB + ot; 2737} 2738 2739static void gen_NOT(DisasContext *s, X86DecodedInsn *decode) 2740{ 2741 MemOp ot = decode->op[0].ot; 2742 2743 if (s->prefix & PREFIX_LOCK) { 2744 tcg_gen_movi_tl(s->T0, ~0); 2745 tcg_gen_atomic_xor_fetch_tl(s->T0, s->A0, s->T0, 2746 s->mem_index, ot | MO_LE); 2747 } else { 2748 tcg_gen_not_tl(s->T0, s->T0); 2749 } 2750} 2751 2752static void gen_OR(DisasContext *s, X86DecodedInsn *decode) 2753{ 2754 MemOp ot = decode->op[1].ot; 2755 2756 if (s->prefix & PREFIX_LOCK) { 2757 tcg_gen_atomic_or_fetch_tl(s->T0, s->A0, s->T1, 2758 s->mem_index, ot | MO_LE); 2759 } else { 2760 tcg_gen_or_tl(s->T0, s->T0, s->T1); 2761 } 2762 prepare_update1_cc(decode, s, CC_OP_LOGICB + ot); 2763} 2764 2765static void gen_OUT(DisasContext *s, X86DecodedInsn *decode) 2766{ 2767 MemOp ot = decode->op[1].ot; 2768 TCGv_i32 port = tcg_temp_new_i32(); 2769 TCGv_i32 value = tcg_temp_new_i32(); 2770 2771 tcg_gen_trunc_tl_i32(port, s->T1); 2772 tcg_gen_ext16u_i32(port, port); 2773 if (!gen_check_io(s, ot, port, 0)) { 2774 return; 2775 } 2776 tcg_gen_trunc_tl_i32(value, s->T0); 2777 translator_io_start(&s->base); 2778 gen_helper_out_func(ot, port, value); 2779 gen_bpt_io(s, port, ot); 2780} 2781 2782static void gen_OUTS(DisasContext *s, X86DecodedInsn *decode) 2783{ 2784 MemOp ot = decode->op[1].ot; 2785 TCGv_i32 port = tcg_temp_new_i32(); 2786 2787 tcg_gen_trunc_tl_i32(port, s->T1); 2788 tcg_gen_ext16u_i32(port, port); 2789 if (!gen_check_io(s, ot, port, SVM_IOIO_STR_MASK)) { 2790 return; 2791 } 2792 2793 translator_io_start(&s->base); 2794 if (s->prefix & (PREFIX_REPZ | PREFIX_REPNZ)) { 2795 gen_repz(s, ot, gen_outs); 2796 } else { 2797 gen_outs(s, ot); 2798 } 2799} 2800 2801static void gen_PALIGNR(DisasContext *s, X86DecodedInsn *decode) 2802{ 2803 TCGv_i32 imm = tcg_constant8u_i32(decode->immediate); 2804 if (!(s->prefix & PREFIX_DATA)) { 2805 gen_helper_palignr_mmx(tcg_env, OP_PTR0, OP_PTR1, OP_PTR2, imm); 2806 } else if (!s->vex_l) { 2807 gen_helper_palignr_xmm(tcg_env, OP_PTR0, OP_PTR1, OP_PTR2, imm); 2808 } else { 2809 gen_helper_palignr_ymm(tcg_env, OP_PTR0, OP_PTR1, OP_PTR2, imm); 2810 } 2811} 2812 2813static void gen_PANDN(DisasContext *s, X86DecodedInsn *decode) 2814{ 2815 int vec_len = vector_len(s, decode); 2816 2817 /* Careful, operand order is reversed! */ 2818 tcg_gen_gvec_andc(MO_64, 2819 decode->op[0].offset, decode->op[2].offset, 2820 decode->op[1].offset, vec_len, vec_len); 2821} 2822 2823static void gen_PAUSE(DisasContext *s, X86DecodedInsn *decode) 2824{ 2825 gen_update_cc_op(s); 2826 gen_update_eip_next(s); 2827 gen_helper_pause(tcg_env); 2828 s->base.is_jmp = DISAS_NORETURN; 2829} 2830 2831static void gen_PCMPESTRI(DisasContext *s, X86DecodedInsn *decode) 2832{ 2833 TCGv_i32 imm = tcg_constant8u_i32(decode->immediate); 2834 gen_helper_pcmpestri_xmm(tcg_env, OP_PTR1, OP_PTR2, imm); 2835 assume_cc_op(s, CC_OP_EFLAGS); 2836} 2837 2838static void gen_PCMPESTRM(DisasContext *s, X86DecodedInsn *decode) 2839{ 2840 TCGv_i32 imm = tcg_constant8u_i32(decode->immediate); 2841 gen_helper_pcmpestrm_xmm(tcg_env, OP_PTR1, OP_PTR2, imm); 2842 assume_cc_op(s, CC_OP_EFLAGS); 2843 if ((s->prefix & PREFIX_VEX) && !s->vex_l) { 2844 tcg_gen_gvec_dup_imm(MO_64, offsetof(CPUX86State, xmm_regs[0].ZMM_X(1)), 2845 16, 16, 0); 2846 } 2847} 2848 2849static void gen_PCMPISTRI(DisasContext *s, X86DecodedInsn *decode) 2850{ 2851 TCGv_i32 imm = tcg_constant8u_i32(decode->immediate); 2852 gen_helper_pcmpistri_xmm(tcg_env, OP_PTR1, OP_PTR2, imm); 2853 assume_cc_op(s, CC_OP_EFLAGS); 2854} 2855 2856static void gen_PCMPISTRM(DisasContext *s, X86DecodedInsn *decode) 2857{ 2858 TCGv_i32 imm = tcg_constant8u_i32(decode->immediate); 2859 gen_helper_pcmpistrm_xmm(tcg_env, OP_PTR1, OP_PTR2, imm); 2860 assume_cc_op(s, CC_OP_EFLAGS); 2861 if ((s->prefix & PREFIX_VEX) && !s->vex_l) { 2862 tcg_gen_gvec_dup_imm(MO_64, offsetof(CPUX86State, xmm_regs[0].ZMM_X(1)), 2863 16, 16, 0); 2864 } 2865} 2866 2867static void gen_PDEP(DisasContext *s, X86DecodedInsn *decode) 2868{ 2869 gen_helper_pdep(s->T0, s->T0, s->T1); 2870} 2871 2872static void gen_PEXT(DisasContext *s, X86DecodedInsn *decode) 2873{ 2874 gen_helper_pext(s->T0, s->T0, s->T1); 2875} 2876 2877static inline void gen_pextr(DisasContext *s, X86DecodedInsn *decode, MemOp ot) 2878{ 2879 int vec_len = vector_len(s, decode); 2880 int mask = (vec_len >> ot) - 1; 2881 int val = decode->immediate & mask; 2882 2883 switch (ot) { 2884 case MO_8: 2885 tcg_gen_ld8u_tl(s->T0, tcg_env, vector_elem_offset(&decode->op[1], ot, val)); 2886 break; 2887 case MO_16: 2888 tcg_gen_ld16u_tl(s->T0, tcg_env, vector_elem_offset(&decode->op[1], ot, val)); 2889 break; 2890 case MO_32: 2891#ifdef TARGET_X86_64 2892 tcg_gen_ld32u_tl(s->T0, tcg_env, vector_elem_offset(&decode->op[1], ot, val)); 2893 break; 2894 case MO_64: 2895#endif 2896 tcg_gen_ld_tl(s->T0, tcg_env, vector_elem_offset(&decode->op[1], ot, val)); 2897 break; 2898 default: 2899 abort(); 2900 } 2901} 2902 2903static void gen_PEXTRB(DisasContext *s, X86DecodedInsn *decode) 2904{ 2905 gen_pextr(s, decode, MO_8); 2906} 2907 2908static void gen_PEXTRW(DisasContext *s, X86DecodedInsn *decode) 2909{ 2910 gen_pextr(s, decode, MO_16); 2911} 2912 2913static void gen_PEXTR(DisasContext *s, X86DecodedInsn *decode) 2914{ 2915 MemOp ot = decode->op[0].ot; 2916 gen_pextr(s, decode, ot); 2917} 2918 2919static inline void gen_pinsr(DisasContext *s, X86DecodedInsn *decode, MemOp ot) 2920{ 2921 int vec_len = vector_len(s, decode); 2922 int mask = (vec_len >> ot) - 1; 2923 int val = decode->immediate & mask; 2924 2925 if (decode->op[1].offset != decode->op[0].offset) { 2926 assert(vec_len == 16); 2927 gen_store_sse(s, decode, decode->op[1].offset); 2928 } 2929 2930 switch (ot) { 2931 case MO_8: 2932 tcg_gen_st8_tl(s->T1, tcg_env, vector_elem_offset(&decode->op[0], ot, val)); 2933 break; 2934 case MO_16: 2935 tcg_gen_st16_tl(s->T1, tcg_env, vector_elem_offset(&decode->op[0], ot, val)); 2936 break; 2937 case MO_32: 2938#ifdef TARGET_X86_64 2939 tcg_gen_st32_tl(s->T1, tcg_env, vector_elem_offset(&decode->op[0], ot, val)); 2940 break; 2941 case MO_64: 2942#endif 2943 tcg_gen_st_tl(s->T1, tcg_env, vector_elem_offset(&decode->op[0], ot, val)); 2944 break; 2945 default: 2946 abort(); 2947 } 2948} 2949 2950static void gen_PINSRB(DisasContext *s, X86DecodedInsn *decode) 2951{ 2952 gen_pinsr(s, decode, MO_8); 2953} 2954 2955static void gen_PINSRW(DisasContext *s, X86DecodedInsn *decode) 2956{ 2957 gen_pinsr(s, decode, MO_16); 2958} 2959 2960static void gen_PINSR(DisasContext *s, X86DecodedInsn *decode) 2961{ 2962 gen_pinsr(s, decode, decode->op[2].ot); 2963} 2964 2965static void gen_pmovmskb_i64(TCGv_i64 d, TCGv_i64 s) 2966{ 2967 TCGv_i64 t = tcg_temp_new_i64(); 2968 2969 tcg_gen_andi_i64(d, s, 0x8080808080808080ull); 2970 2971 /* 2972 * After each shift+or pair: 2973 * 0: a.......b.......c.......d.......e.......f.......g.......h....... 2974 * 7: ab......bc......cd......de......ef......fg......gh......h....... 2975 * 14: abcd....bcde....cdef....defg....efgh....fgh.....gh......h....... 2976 * 28: abcdefghbcdefgh.cdefgh..defgh...efgh....fgh.....gh......h....... 2977 * The result is left in the high bits of the word. 2978 */ 2979 tcg_gen_shli_i64(t, d, 7); 2980 tcg_gen_or_i64(d, d, t); 2981 tcg_gen_shli_i64(t, d, 14); 2982 tcg_gen_or_i64(d, d, t); 2983 tcg_gen_shli_i64(t, d, 28); 2984 tcg_gen_or_i64(d, d, t); 2985} 2986 2987static void gen_pmovmskb_vec(unsigned vece, TCGv_vec d, TCGv_vec s) 2988{ 2989 TCGv_vec t = tcg_temp_new_vec_matching(d); 2990 TCGv_vec m = tcg_constant_vec_matching(d, MO_8, 0x80); 2991 2992 /* See above */ 2993 tcg_gen_and_vec(vece, d, s, m); 2994 tcg_gen_shli_vec(vece, t, d, 7); 2995 tcg_gen_or_vec(vece, d, d, t); 2996 tcg_gen_shli_vec(vece, t, d, 14); 2997 tcg_gen_or_vec(vece, d, d, t); 2998 tcg_gen_shli_vec(vece, t, d, 28); 2999 tcg_gen_or_vec(vece, d, d, t); 3000} 3001 3002static void gen_PMOVMSKB(DisasContext *s, X86DecodedInsn *decode) 3003{ 3004 static const TCGOpcode vecop_list[] = { INDEX_op_shli_vec, 0 }; 3005 static const GVecGen2 g = { 3006 .fni8 = gen_pmovmskb_i64, 3007 .fniv = gen_pmovmskb_vec, 3008 .opt_opc = vecop_list, 3009 .vece = MO_64, 3010 .prefer_i64 = TCG_TARGET_REG_BITS == 64 3011 }; 3012 MemOp ot = decode->op[2].ot; 3013 int vec_len = vector_len(s, decode); 3014 TCGv t = tcg_temp_new(); 3015 3016 tcg_gen_gvec_2(offsetof(CPUX86State, xmm_t0) + xmm_offset(ot), decode->op[2].offset, 3017 vec_len, vec_len, &g); 3018 tcg_gen_ld8u_tl(s->T0, tcg_env, offsetof(CPUX86State, xmm_t0.ZMM_B(vec_len - 1))); 3019 while (vec_len > 8) { 3020 vec_len -= 8; 3021 if (TCG_TARGET_HAS_extract2_tl) { 3022 /* 3023 * Load the next byte of the result into the high byte of T. 3024 * TCG does a similar expansion of deposit to shl+extract2; by 3025 * loading the whole word, the shift left is avoided. 3026 */ 3027#ifdef TARGET_X86_64 3028 tcg_gen_ld_tl(t, tcg_env, offsetof(CPUX86State, xmm_t0.ZMM_Q((vec_len - 1) / 8))); 3029#else 3030 tcg_gen_ld_tl(t, tcg_env, offsetof(CPUX86State, xmm_t0.ZMM_L((vec_len - 1) / 4))); 3031#endif 3032 3033 tcg_gen_extract2_tl(s->T0, t, s->T0, TARGET_LONG_BITS - 8); 3034 } else { 3035 /* 3036 * The _previous_ value is deposited into bits 8 and higher of t. Because 3037 * those bits are known to be zero after ld8u, this becomes a shift+or 3038 * if deposit is not available. 3039 */ 3040 tcg_gen_ld8u_tl(t, tcg_env, offsetof(CPUX86State, xmm_t0.ZMM_B(vec_len - 1))); 3041 tcg_gen_deposit_tl(s->T0, t, s->T0, 8, TARGET_LONG_BITS - 8); 3042 } 3043 } 3044} 3045 3046static void gen_POP(DisasContext *s, X86DecodedInsn *decode) 3047{ 3048 X86DecodedOp *op = &decode->op[0]; 3049 MemOp ot = gen_pop_T0(s); 3050 3051 assert(ot >= op->ot); 3052 if (op->has_ea || op->unit == X86_OP_SEG) { 3053 /* NOTE: order is important for MMU exceptions */ 3054 gen_writeback(s, decode, 0, s->T0); 3055 } 3056 3057 /* NOTE: writing back registers after update is important for pop %sp */ 3058 gen_pop_update(s, ot); 3059} 3060 3061static void gen_POPA(DisasContext *s, X86DecodedInsn *decode) 3062{ 3063 gen_popa(s); 3064} 3065 3066static void gen_POPCNT(DisasContext *s, X86DecodedInsn *decode) 3067{ 3068 decode->cc_dst = tcg_temp_new(); 3069 decode->cc_op = CC_OP_POPCNT; 3070 3071 tcg_gen_mov_tl(decode->cc_dst, s->T0); 3072 tcg_gen_ctpop_tl(s->T0, s->T0); 3073} 3074 3075static void gen_POPF(DisasContext *s, X86DecodedInsn *decode) 3076{ 3077 MemOp ot; 3078 int mask = TF_MASK | AC_MASK | ID_MASK | NT_MASK; 3079 3080 if (CPL(s) == 0) { 3081 mask |= IF_MASK | IOPL_MASK; 3082 } else if (CPL(s) <= IOPL(s)) { 3083 mask |= IF_MASK; 3084 } 3085 if (s->dflag == MO_16) { 3086 mask &= 0xffff; 3087 } 3088 3089 ot = gen_pop_T0(s); 3090 gen_helper_write_eflags(tcg_env, s->T0, tcg_constant_i32(mask)); 3091 gen_pop_update(s, ot); 3092 set_cc_op(s, CC_OP_EFLAGS); 3093 /* abort translation because TF/AC flag may change */ 3094 s->base.is_jmp = DISAS_EOB_NEXT; 3095} 3096 3097static void gen_PSHUFW(DisasContext *s, X86DecodedInsn *decode) 3098{ 3099 TCGv_i32 imm = tcg_constant8u_i32(decode->immediate); 3100 gen_helper_pshufw_mmx(OP_PTR0, OP_PTR1, imm); 3101} 3102 3103static void gen_PSRLW_i(DisasContext *s, X86DecodedInsn *decode) 3104{ 3105 int vec_len = vector_len(s, decode); 3106 3107 if (decode->immediate >= 16) { 3108 tcg_gen_gvec_dup_imm(MO_64, decode->op[0].offset, vec_len, vec_len, 0); 3109 } else { 3110 tcg_gen_gvec_shri(MO_16, 3111 decode->op[0].offset, decode->op[1].offset, 3112 decode->immediate, vec_len, vec_len); 3113 } 3114} 3115 3116static void gen_PSLLW_i(DisasContext *s, X86DecodedInsn *decode) 3117{ 3118 int vec_len = vector_len(s, decode); 3119 3120 if (decode->immediate >= 16) { 3121 tcg_gen_gvec_dup_imm(MO_64, decode->op[0].offset, vec_len, vec_len, 0); 3122 } else { 3123 tcg_gen_gvec_shli(MO_16, 3124 decode->op[0].offset, decode->op[1].offset, 3125 decode->immediate, vec_len, vec_len); 3126 } 3127} 3128 3129static void gen_PSRAW_i(DisasContext *s, X86DecodedInsn *decode) 3130{ 3131 int vec_len = vector_len(s, decode); 3132 3133 if (decode->immediate >= 16) { 3134 decode->immediate = 15; 3135 } 3136 tcg_gen_gvec_sari(MO_16, 3137 decode->op[0].offset, decode->op[1].offset, 3138 decode->immediate, vec_len, vec_len); 3139} 3140 3141static void gen_PSRLD_i(DisasContext *s, X86DecodedInsn *decode) 3142{ 3143 int vec_len = vector_len(s, decode); 3144 3145 if (decode->immediate >= 32) { 3146 tcg_gen_gvec_dup_imm(MO_64, decode->op[0].offset, vec_len, vec_len, 0); 3147 } else { 3148 tcg_gen_gvec_shri(MO_32, 3149 decode->op[0].offset, decode->op[1].offset, 3150 decode->immediate, vec_len, vec_len); 3151 } 3152} 3153 3154static void gen_PSLLD_i(DisasContext *s, X86DecodedInsn *decode) 3155{ 3156 int vec_len = vector_len(s, decode); 3157 3158 if (decode->immediate >= 32) { 3159 tcg_gen_gvec_dup_imm(MO_64, decode->op[0].offset, vec_len, vec_len, 0); 3160 } else { 3161 tcg_gen_gvec_shli(MO_32, 3162 decode->op[0].offset, decode->op[1].offset, 3163 decode->immediate, vec_len, vec_len); 3164 } 3165} 3166 3167static void gen_PSRAD_i(DisasContext *s, X86DecodedInsn *decode) 3168{ 3169 int vec_len = vector_len(s, decode); 3170 3171 if (decode->immediate >= 32) { 3172 decode->immediate = 31; 3173 } 3174 tcg_gen_gvec_sari(MO_32, 3175 decode->op[0].offset, decode->op[1].offset, 3176 decode->immediate, vec_len, vec_len); 3177} 3178 3179static void gen_PSRLQ_i(DisasContext *s, X86DecodedInsn *decode) 3180{ 3181 int vec_len = vector_len(s, decode); 3182 3183 if (decode->immediate >= 64) { 3184 tcg_gen_gvec_dup_imm(MO_64, decode->op[0].offset, vec_len, vec_len, 0); 3185 } else { 3186 tcg_gen_gvec_shri(MO_64, 3187 decode->op[0].offset, decode->op[1].offset, 3188 decode->immediate, vec_len, vec_len); 3189 } 3190} 3191 3192static void gen_PSLLQ_i(DisasContext *s, X86DecodedInsn *decode) 3193{ 3194 int vec_len = vector_len(s, decode); 3195 3196 if (decode->immediate >= 64) { 3197 tcg_gen_gvec_dup_imm(MO_64, decode->op[0].offset, vec_len, vec_len, 0); 3198 } else { 3199 tcg_gen_gvec_shli(MO_64, 3200 decode->op[0].offset, decode->op[1].offset, 3201 decode->immediate, vec_len, vec_len); 3202 } 3203} 3204 3205static TCGv_ptr make_imm8u_xmm_vec(uint8_t imm, int vec_len) 3206{ 3207 MemOp ot = vec_len == 16 ? MO_128 : MO_256; 3208 TCGv_i32 imm_v = tcg_constant8u_i32(imm); 3209 TCGv_ptr ptr = tcg_temp_new_ptr(); 3210 3211 tcg_gen_gvec_dup_imm(MO_64, offsetof(CPUX86State, xmm_t0) + xmm_offset(ot), 3212 vec_len, vec_len, 0); 3213 3214 tcg_gen_addi_ptr(ptr, tcg_env, offsetof(CPUX86State, xmm_t0)); 3215 tcg_gen_st_i32(imm_v, tcg_env, offsetof(CPUX86State, xmm_t0.ZMM_L(0))); 3216 return ptr; 3217} 3218 3219static void gen_PSRLDQ_i(DisasContext *s, X86DecodedInsn *decode) 3220{ 3221 int vec_len = vector_len(s, decode); 3222 TCGv_ptr imm_vec = make_imm8u_xmm_vec(decode->immediate, vec_len); 3223 3224 if (s->vex_l) { 3225 gen_helper_psrldq_ymm(tcg_env, OP_PTR0, OP_PTR1, imm_vec); 3226 } else { 3227 gen_helper_psrldq_xmm(tcg_env, OP_PTR0, OP_PTR1, imm_vec); 3228 } 3229} 3230 3231static void gen_PSLLDQ_i(DisasContext *s, X86DecodedInsn *decode) 3232{ 3233 int vec_len = vector_len(s, decode); 3234 TCGv_ptr imm_vec = make_imm8u_xmm_vec(decode->immediate, vec_len); 3235 3236 if (s->vex_l) { 3237 gen_helper_pslldq_ymm(tcg_env, OP_PTR0, OP_PTR1, imm_vec); 3238 } else { 3239 gen_helper_pslldq_xmm(tcg_env, OP_PTR0, OP_PTR1, imm_vec); 3240 } 3241} 3242 3243static void gen_PUSH(DisasContext *s, X86DecodedInsn *decode) 3244{ 3245 gen_push_v(s, s->T0); 3246} 3247 3248static void gen_PUSHA(DisasContext *s, X86DecodedInsn *decode) 3249{ 3250 gen_pusha(s); 3251} 3252 3253static void gen_PUSHF(DisasContext *s, X86DecodedInsn *decode) 3254{ 3255 gen_update_cc_op(s); 3256 gen_helper_read_eflags(s->T0, tcg_env); 3257 gen_push_v(s, s->T0); 3258} 3259 3260static MemOp gen_shift_count(DisasContext *s, X86DecodedInsn *decode, 3261 bool *can_be_zero, TCGv *count, int unit) 3262{ 3263 MemOp ot = decode->op[0].ot; 3264 int mask = (ot <= MO_32 ? 0x1f : 0x3f); 3265 3266 *can_be_zero = false; 3267 switch (unit) { 3268 case X86_OP_INT: 3269 *count = tcg_temp_new(); 3270 tcg_gen_andi_tl(*count, cpu_regs[R_ECX], mask); 3271 *can_be_zero = true; 3272 break; 3273 3274 case X86_OP_IMM: 3275 if ((decode->immediate & mask) == 0) { 3276 *count = NULL; 3277 break; 3278 } 3279 *count = tcg_temp_new(); 3280 tcg_gen_movi_tl(*count, decode->immediate & mask); 3281 break; 3282 3283 case X86_OP_SKIP: 3284 *count = tcg_temp_new(); 3285 tcg_gen_movi_tl(*count, 1); 3286 break; 3287 3288 default: 3289 g_assert_not_reached(); 3290 } 3291 3292 return ot; 3293} 3294 3295/* 3296 * Compute existing flags in decode->cc_src, for gen_* functions that wants 3297 * to set the cc_op set to CC_OP_ADCOX. In particular, this allows rotate 3298 * operations to compute the carry in decode->cc_dst and the overflow in 3299 * decode->cc_src2. 3300 * 3301 * If need_flags is true, decode->cc_dst and decode->cc_src2 are preloaded 3302 * with the value of CF and OF before the instruction, so that it is possible 3303 * to keep the flags unmodified. 3304 * 3305 * Return true if carry could be made available cheaply as a 1-bit value in 3306 * decode->cc_dst (trying a bit harder if want_carry is true). If false is 3307 * returned, decode->cc_dst is uninitialized and the carry is only available 3308 * as bit 0 of decode->cc_src. 3309 */ 3310static bool gen_eflags_adcox(DisasContext *s, X86DecodedInsn *decode, bool want_carry, bool need_flags) 3311{ 3312 bool got_cf = false; 3313 bool got_of = false; 3314 3315 decode->cc_dst = tcg_temp_new(); 3316 decode->cc_src = tcg_temp_new(); 3317 decode->cc_src2 = tcg_temp_new(); 3318 decode->cc_op = CC_OP_ADCOX; 3319 3320 /* A lot more cc_ops could be "optimized" to avoid the extracts at 3321 * the end (INC/DEC, BMILG, MUL), but they are all really unlikely 3322 * to be followed by rotations within the same basic block. 3323 */ 3324 switch (s->cc_op) { 3325 case CC_OP_ADCOX: 3326 /* No need to compute the full EFLAGS, CF/OF are already isolated. */ 3327 tcg_gen_mov_tl(decode->cc_src, cpu_cc_src); 3328 if (need_flags) { 3329 tcg_gen_mov_tl(decode->cc_src2, cpu_cc_src2); 3330 got_of = true; 3331 } 3332 if (want_carry || need_flags) { 3333 tcg_gen_mov_tl(decode->cc_dst, cpu_cc_dst); 3334 got_cf = true; 3335 } 3336 break; 3337 3338 case CC_OP_LOGICB ... CC_OP_LOGICQ: 3339 /* CF and OF are zero, do it just because it's easy. */ 3340 gen_mov_eflags(s, decode->cc_src); 3341 if (need_flags) { 3342 tcg_gen_movi_tl(decode->cc_src2, 0); 3343 got_of = true; 3344 } 3345 if (want_carry || need_flags) { 3346 tcg_gen_movi_tl(decode->cc_dst, 0); 3347 got_cf = true; 3348 } 3349 break; 3350 3351 case CC_OP_SARB ... CC_OP_SARQ: 3352 /* 3353 * SHR/RCR/SHR/RCR/... is a relatively common occurrence of RCR. 3354 * By computing CF without using eflags, the calls to cc_compute_all 3355 * can be eliminated as dead code (except for the last RCR). 3356 */ 3357 if (want_carry || need_flags) { 3358 tcg_gen_andi_tl(decode->cc_dst, cpu_cc_src, 1); 3359 got_cf = true; 3360 } 3361 gen_mov_eflags(s, decode->cc_src); 3362 break; 3363 3364 case CC_OP_SHLB ... CC_OP_SHLQ: 3365 /* 3366 * Likewise for SHL/RCL/SHL/RCL/... but, if CF is not in the sign 3367 * bit, we might as well fish CF out of EFLAGS and save a shift. 3368 */ 3369 if (want_carry && (!need_flags || s->cc_op == CC_OP_SHLB + MO_TL)) { 3370 MemOp size = cc_op_size(s->cc_op); 3371 tcg_gen_shri_tl(decode->cc_dst, cpu_cc_src, (8 << size) - 1); 3372 got_cf = true; 3373 } 3374 gen_mov_eflags(s, decode->cc_src); 3375 break; 3376 3377 default: 3378 gen_mov_eflags(s, decode->cc_src); 3379 break; 3380 } 3381 3382 if (need_flags) { 3383 /* If the flags could be left unmodified, always load them. */ 3384 if (!got_of) { 3385 tcg_gen_extract_tl(decode->cc_src2, decode->cc_src, ctz32(CC_O), 1); 3386 got_of = true; 3387 } 3388 if (!got_cf) { 3389 tcg_gen_extract_tl(decode->cc_dst, decode->cc_src, ctz32(CC_C), 1); 3390 got_cf = true; 3391 } 3392 } 3393 return got_cf; 3394} 3395 3396static void gen_rot_overflow(X86DecodedInsn *decode, TCGv result, TCGv old, 3397 bool can_be_zero, TCGv count) 3398{ 3399 MemOp ot = decode->op[0].ot; 3400 TCGv temp = can_be_zero ? tcg_temp_new() : decode->cc_src2; 3401 3402 tcg_gen_xor_tl(temp, old, result); 3403 tcg_gen_extract_tl(temp, temp, (8 << ot) - 1, 1); 3404 if (can_be_zero) { 3405 tcg_gen_movcond_tl(TCG_COND_EQ, decode->cc_src2, count, tcg_constant_tl(0), 3406 decode->cc_src2, temp); 3407 } 3408} 3409 3410/* 3411 * RCx operations are invariant modulo 8*operand_size+1. For 8 and 16-bit operands, 3412 * this is less than 0x1f (the mask applied by gen_shift_count) so reduce further. 3413 */ 3414static void gen_rotc_mod(MemOp ot, TCGv count) 3415{ 3416 TCGv temp; 3417 3418 switch (ot) { 3419 case MO_8: 3420 temp = tcg_temp_new(); 3421 tcg_gen_subi_tl(temp, count, 18); 3422 tcg_gen_movcond_tl(TCG_COND_GE, count, temp, tcg_constant_tl(0), temp, count); 3423 tcg_gen_subi_tl(temp, count, 9); 3424 tcg_gen_movcond_tl(TCG_COND_GE, count, temp, tcg_constant_tl(0), temp, count); 3425 break; 3426 3427 case MO_16: 3428 temp = tcg_temp_new(); 3429 tcg_gen_subi_tl(temp, count, 17); 3430 tcg_gen_movcond_tl(TCG_COND_GE, count, temp, tcg_constant_tl(0), temp, count); 3431 break; 3432 3433 default: 3434 break; 3435 } 3436} 3437 3438/* 3439 * The idea here is that the bit to the right of the new bit 0 is the 3440 * new carry, and the bit to the right of the old bit 0 is the old carry. 3441 * Just like a regular rotation, the result of the rotation is composed 3442 * from a right shifted part and a left shifted part of s->T0. The new carry 3443 * is extracted from the right-shifted portion, and the old carry is 3444 * inserted at the end of the left-shifted portion. 3445 * 3446 * Because of the separate shifts involving the carry, gen_RCL and gen_RCR 3447 * mostly operate on count-1. This also comes in handy when computing 3448 * length - count, because (length-1) - (count-1) can be computed with 3449 * a XOR, and that is commutative unlike subtraction. 3450 */ 3451static void gen_RCL(DisasContext *s, X86DecodedInsn *decode) 3452{ 3453 bool have_1bit_cin, can_be_zero; 3454 TCGv count; 3455 TCGLabel *zero_label = NULL; 3456 MemOp ot = gen_shift_count(s, decode, &can_be_zero, &count, decode->op[2].unit); 3457 TCGv low, high, low_count; 3458 3459 if (!count) { 3460 return; 3461 } 3462 3463 low = tcg_temp_new(); 3464 high = tcg_temp_new(); 3465 low_count = tcg_temp_new(); 3466 3467 gen_rotc_mod(ot, count); 3468 have_1bit_cin = gen_eflags_adcox(s, decode, true, can_be_zero); 3469 if (can_be_zero) { 3470 zero_label = gen_new_label(); 3471 tcg_gen_brcondi_tl(TCG_COND_EQ, count, 0, zero_label); 3472 } 3473 3474 /* Compute high part, including incoming carry. */ 3475 if (!have_1bit_cin || TCG_TARGET_deposit_tl_valid(1, TARGET_LONG_BITS - 1)) { 3476 /* high = (T0 << 1) | cin */ 3477 TCGv cin = have_1bit_cin ? decode->cc_dst : decode->cc_src; 3478 tcg_gen_deposit_tl(high, cin, s->T0, 1, TARGET_LONG_BITS - 1); 3479 } else { 3480 /* Same as above but without deposit; cin in cc_dst. */ 3481 tcg_gen_add_tl(high, s->T0, decode->cc_dst); 3482 tcg_gen_add_tl(high, high, s->T0); 3483 } 3484 tcg_gen_subi_tl(count, count, 1); 3485 tcg_gen_shl_tl(high, high, count); 3486 3487 /* Compute low part and outgoing carry, incoming s->T0 is zero extended */ 3488 tcg_gen_xori_tl(low_count, count, (8 << ot) - 1); /* LENGTH - 1 - (count - 1) */ 3489 tcg_gen_shr_tl(low, s->T0, low_count); 3490 tcg_gen_andi_tl(decode->cc_dst, low, 1); 3491 tcg_gen_shri_tl(low, low, 1); 3492 3493 /* Compute result and outgoing overflow */ 3494 tcg_gen_mov_tl(decode->cc_src2, s->T0); 3495 tcg_gen_or_tl(s->T0, low, high); 3496 gen_rot_overflow(decode, s->T0, decode->cc_src2, false, NULL); 3497 3498 if (zero_label) { 3499 gen_set_label(zero_label); 3500 } 3501} 3502 3503static void gen_RCR(DisasContext *s, X86DecodedInsn *decode) 3504{ 3505 bool have_1bit_cin, can_be_zero; 3506 TCGv count; 3507 TCGLabel *zero_label = NULL; 3508 MemOp ot = gen_shift_count(s, decode, &can_be_zero, &count, decode->op[2].unit); 3509 TCGv low, high, high_count; 3510 3511 if (!count) { 3512 return; 3513 } 3514 3515 low = tcg_temp_new(); 3516 high = tcg_temp_new(); 3517 high_count = tcg_temp_new(); 3518 3519 gen_rotc_mod(ot, count); 3520 have_1bit_cin = gen_eflags_adcox(s, decode, true, can_be_zero); 3521 if (can_be_zero) { 3522 zero_label = gen_new_label(); 3523 tcg_gen_brcondi_tl(TCG_COND_EQ, count, 0, zero_label); 3524 } 3525 3526 /* Save incoming carry into high, it will be shifted later. */ 3527 if (!have_1bit_cin || TCG_TARGET_deposit_tl_valid(1, TARGET_LONG_BITS - 1)) { 3528 TCGv cin = have_1bit_cin ? decode->cc_dst : decode->cc_src; 3529 tcg_gen_deposit_tl(high, cin, s->T0, 1, TARGET_LONG_BITS - 1); 3530 } else { 3531 /* Same as above but without deposit; cin in cc_dst. */ 3532 tcg_gen_add_tl(high, s->T0, decode->cc_dst); 3533 tcg_gen_add_tl(high, high, s->T0); 3534 } 3535 3536 /* Compute low part and outgoing carry, incoming s->T0 is zero extended */ 3537 tcg_gen_subi_tl(count, count, 1); 3538 tcg_gen_shr_tl(low, s->T0, count); 3539 tcg_gen_andi_tl(decode->cc_dst, low, 1); 3540 tcg_gen_shri_tl(low, low, 1); 3541 3542 /* Move high part to the right position */ 3543 tcg_gen_xori_tl(high_count, count, (8 << ot) - 1); /* LENGTH - 1 - (count - 1) */ 3544 tcg_gen_shl_tl(high, high, high_count); 3545 3546 /* Compute result and outgoing overflow */ 3547 tcg_gen_mov_tl(decode->cc_src2, s->T0); 3548 tcg_gen_or_tl(s->T0, low, high); 3549 gen_rot_overflow(decode, s->T0, decode->cc_src2, false, NULL); 3550 3551 if (zero_label) { 3552 gen_set_label(zero_label); 3553 } 3554} 3555 3556#ifdef CONFIG_USER_ONLY 3557static void gen_unreachable(DisasContext *s, X86DecodedInsn *decode) 3558{ 3559 g_assert_not_reached(); 3560} 3561#endif 3562 3563#ifndef CONFIG_USER_ONLY 3564static void gen_RDMSR(DisasContext *s, X86DecodedInsn *decode) 3565{ 3566 gen_update_cc_op(s); 3567 gen_update_eip_cur(s); 3568 gen_helper_rdmsr(tcg_env); 3569} 3570#else 3571#define gen_RDMSR gen_unreachable 3572#endif 3573 3574static void gen_RDPMC(DisasContext *s, X86DecodedInsn *decode) 3575{ 3576 gen_update_cc_op(s); 3577 gen_update_eip_cur(s); 3578 translator_io_start(&s->base); 3579 gen_helper_rdpmc(tcg_env); 3580 s->base.is_jmp = DISAS_NORETURN; 3581} 3582 3583static void gen_RDTSC(DisasContext *s, X86DecodedInsn *decode) 3584{ 3585 gen_update_cc_op(s); 3586 gen_update_eip_cur(s); 3587 translator_io_start(&s->base); 3588 gen_helper_rdtsc(tcg_env); 3589} 3590 3591static void gen_RDxxBASE(DisasContext *s, X86DecodedInsn *decode) 3592{ 3593 TCGv base = cpu_seg_base[s->modrm & 8 ? R_GS : R_FS]; 3594 3595 /* Preserve hflags bits by testing CR4 at runtime. */ 3596 gen_helper_cr4_testbit(tcg_env, tcg_constant_i32(CR4_FSGSBASE_MASK)); 3597 tcg_gen_mov_tl(s->T0, base); 3598} 3599 3600static void gen_RET(DisasContext *s, X86DecodedInsn *decode) 3601{ 3602 int16_t adjust = decode->e.op1 == X86_TYPE_I ? decode->immediate : 0; 3603 3604 MemOp ot = gen_pop_T0(s); 3605 gen_stack_update(s, adjust + (1 << ot)); 3606 gen_op_jmp_v(s, s->T0); 3607 gen_bnd_jmp(s); 3608 s->base.is_jmp = DISAS_JUMP; 3609} 3610 3611static void gen_RETF(DisasContext *s, X86DecodedInsn *decode) 3612{ 3613 int16_t adjust = decode->e.op1 == X86_TYPE_I ? decode->immediate : 0; 3614 3615 if (!PE(s) || VM86(s)) { 3616 gen_lea_ss_ofs(s, s->A0, cpu_regs[R_ESP], 0); 3617 /* pop offset */ 3618 gen_op_ld_v(s, s->dflag, s->T0, s->A0); 3619 /* NOTE: keeping EIP updated is not a problem in case of 3620 exception */ 3621 gen_op_jmp_v(s, s->T0); 3622 /* pop selector */ 3623 gen_add_A0_im(s, 1 << s->dflag); 3624 gen_op_ld_v(s, s->dflag, s->T0, s->A0); 3625 gen_op_movl_seg_real(s, R_CS, s->T0); 3626 /* add stack offset */ 3627 gen_stack_update(s, adjust + (2 << s->dflag)); 3628 } else { 3629 gen_update_cc_op(s); 3630 gen_update_eip_cur(s); 3631 gen_helper_lret_protected(tcg_env, tcg_constant_i32(s->dflag - 1), 3632 tcg_constant_i32(adjust)); 3633 } 3634 s->base.is_jmp = DISAS_EOB_ONLY; 3635} 3636 3637/* 3638 * Return non-NULL if a 32-bit rotate works, after possibly replicating the input. 3639 * The input has already been zero-extended upon operand decode. 3640 */ 3641static TCGv_i32 gen_rot_replicate(MemOp ot, TCGv in) 3642{ 3643 TCGv_i32 temp; 3644 switch (ot) { 3645 case MO_8: 3646 temp = tcg_temp_new_i32(); 3647 tcg_gen_trunc_tl_i32(temp, in); 3648 tcg_gen_muli_i32(temp, temp, 0x01010101); 3649 return temp; 3650 3651 case MO_16: 3652 temp = tcg_temp_new_i32(); 3653 tcg_gen_trunc_tl_i32(temp, in); 3654 tcg_gen_deposit_i32(temp, temp, temp, 16, 16); 3655 return temp; 3656 3657#ifdef TARGET_X86_64 3658 case MO_32: 3659 temp = tcg_temp_new_i32(); 3660 tcg_gen_trunc_tl_i32(temp, in); 3661 return temp; 3662#endif 3663 3664 default: 3665 return NULL; 3666 } 3667} 3668 3669static void gen_rot_carry(X86DecodedInsn *decode, TCGv result, 3670 bool can_be_zero, TCGv count, int bit) 3671{ 3672 if (!can_be_zero) { 3673 tcg_gen_extract_tl(decode->cc_dst, result, bit, 1); 3674 } else { 3675 TCGv temp = tcg_temp_new(); 3676 tcg_gen_extract_tl(temp, result, bit, 1); 3677 tcg_gen_movcond_tl(TCG_COND_EQ, decode->cc_dst, count, tcg_constant_tl(0), 3678 decode->cc_dst, temp); 3679 } 3680} 3681 3682static void gen_ROL(DisasContext *s, X86DecodedInsn *decode) 3683{ 3684 bool can_be_zero; 3685 TCGv count; 3686 MemOp ot = gen_shift_count(s, decode, &can_be_zero, &count, decode->op[2].unit); 3687 TCGv_i32 temp32, count32; 3688 TCGv old = tcg_temp_new(); 3689 3690 if (!count) { 3691 return; 3692 } 3693 3694 gen_eflags_adcox(s, decode, false, can_be_zero); 3695 tcg_gen_mov_tl(old, s->T0); 3696 temp32 = gen_rot_replicate(ot, s->T0); 3697 if (temp32) { 3698 count32 = tcg_temp_new_i32(); 3699 tcg_gen_trunc_tl_i32(count32, count); 3700 tcg_gen_rotl_i32(temp32, temp32, count32); 3701 /* Zero extend to facilitate later optimization. */ 3702 tcg_gen_extu_i32_tl(s->T0, temp32); 3703 } else { 3704 tcg_gen_rotl_tl(s->T0, s->T0, count); 3705 } 3706 gen_rot_carry(decode, s->T0, can_be_zero, count, 0); 3707 gen_rot_overflow(decode, s->T0, old, can_be_zero, count); 3708} 3709 3710static void gen_ROR(DisasContext *s, X86DecodedInsn *decode) 3711{ 3712 bool can_be_zero; 3713 TCGv count; 3714 MemOp ot = gen_shift_count(s, decode, &can_be_zero, &count, decode->op[2].unit); 3715 TCGv_i32 temp32, count32; 3716 TCGv old = tcg_temp_new(); 3717 3718 if (!count) { 3719 return; 3720 } 3721 3722 gen_eflags_adcox(s, decode, false, can_be_zero); 3723 tcg_gen_mov_tl(old, s->T0); 3724 temp32 = gen_rot_replicate(ot, s->T0); 3725 if (temp32) { 3726 count32 = tcg_temp_new_i32(); 3727 tcg_gen_trunc_tl_i32(count32, count); 3728 tcg_gen_rotr_i32(temp32, temp32, count32); 3729 /* Zero extend to facilitate later optimization. */ 3730 tcg_gen_extu_i32_tl(s->T0, temp32); 3731 gen_rot_carry(decode, s->T0, can_be_zero, count, 31); 3732 } else { 3733 tcg_gen_rotr_tl(s->T0, s->T0, count); 3734 gen_rot_carry(decode, s->T0, can_be_zero, count, TARGET_LONG_BITS - 1); 3735 } 3736 gen_rot_overflow(decode, s->T0, old, can_be_zero, count); 3737} 3738 3739static void gen_RORX(DisasContext *s, X86DecodedInsn *decode) 3740{ 3741 MemOp ot = decode->op[0].ot; 3742 int mask = ot == MO_64 ? 63 : 31; 3743 int b = decode->immediate & mask; 3744 3745 switch (ot) { 3746 case MO_32: 3747#ifdef TARGET_X86_64 3748 tcg_gen_trunc_tl_i32(s->tmp2_i32, s->T0); 3749 tcg_gen_rotri_i32(s->tmp2_i32, s->tmp2_i32, b); 3750 tcg_gen_extu_i32_tl(s->T0, s->tmp2_i32); 3751 break; 3752 3753 case MO_64: 3754#endif 3755 tcg_gen_rotri_tl(s->T0, s->T0, b); 3756 break; 3757 3758 default: 3759 g_assert_not_reached(); 3760 } 3761} 3762 3763#ifndef CONFIG_USER_ONLY 3764static void gen_RSM(DisasContext *s, X86DecodedInsn *decode) 3765{ 3766 gen_helper_rsm(tcg_env); 3767 assume_cc_op(s, CC_OP_EFLAGS); 3768 s->base.is_jmp = DISAS_EOB_ONLY; 3769} 3770#else 3771#define gen_RSM gen_UD 3772#endif 3773 3774static void gen_SAHF(DisasContext *s, X86DecodedInsn *decode) 3775{ 3776 if (CODE64(s) && !(s->cpuid_ext3_features & CPUID_EXT3_LAHF_LM)) { 3777 return gen_illegal_opcode(s); 3778 } 3779 tcg_gen_shri_tl(s->T0, cpu_regs[R_EAX], 8); 3780 gen_compute_eflags(s); 3781 tcg_gen_andi_tl(cpu_cc_src, cpu_cc_src, CC_O); 3782 tcg_gen_andi_tl(s->T0, s->T0, CC_S | CC_Z | CC_A | CC_P | CC_C); 3783 tcg_gen_or_tl(cpu_cc_src, cpu_cc_src, s->T0); 3784} 3785 3786static void gen_SALC(DisasContext *s, X86DecodedInsn *decode) 3787{ 3788 gen_compute_eflags_c(s, s->T0); 3789 tcg_gen_neg_tl(s->T0, s->T0); 3790} 3791 3792static void gen_shift_dynamic_flags(DisasContext *s, X86DecodedInsn *decode, TCGv count, CCOp cc_op) 3793{ 3794 TCGv_i32 count32 = tcg_temp_new_i32(); 3795 TCGv_i32 old_cc_op; 3796 3797 decode->cc_op = CC_OP_DYNAMIC; 3798 decode->cc_op_dynamic = tcg_temp_new_i32(); 3799 3800 assert(decode->cc_dst == s->T0); 3801 if (cc_op_live(s->cc_op) & USES_CC_DST) { 3802 decode->cc_dst = tcg_temp_new(); 3803 tcg_gen_movcond_tl(TCG_COND_EQ, decode->cc_dst, count, tcg_constant_tl(0), 3804 cpu_cc_dst, s->T0); 3805 } 3806 3807 if (cc_op_live(s->cc_op) & USES_CC_SRC) { 3808 tcg_gen_movcond_tl(TCG_COND_EQ, decode->cc_src, count, tcg_constant_tl(0), 3809 cpu_cc_src, decode->cc_src); 3810 } 3811 3812 tcg_gen_trunc_tl_i32(count32, count); 3813 if (s->cc_op == CC_OP_DYNAMIC) { 3814 old_cc_op = cpu_cc_op; 3815 } else { 3816 old_cc_op = tcg_constant_i32(s->cc_op); 3817 } 3818 tcg_gen_movcond_i32(TCG_COND_EQ, decode->cc_op_dynamic, count32, tcg_constant_i32(0), 3819 old_cc_op, tcg_constant_i32(cc_op)); 3820} 3821 3822static void gen_SAR(DisasContext *s, X86DecodedInsn *decode) 3823{ 3824 bool can_be_zero; 3825 TCGv count; 3826 MemOp ot = gen_shift_count(s, decode, &can_be_zero, &count, decode->op[2].unit); 3827 3828 if (!count) { 3829 return; 3830 } 3831 3832 decode->cc_dst = s->T0; 3833 decode->cc_src = tcg_temp_new(); 3834 tcg_gen_subi_tl(decode->cc_src, count, 1); 3835 tcg_gen_sar_tl(decode->cc_src, s->T0, decode->cc_src); 3836 tcg_gen_sar_tl(s->T0, s->T0, count); 3837 if (can_be_zero) { 3838 gen_shift_dynamic_flags(s, decode, count, CC_OP_SARB + ot); 3839 } else { 3840 decode->cc_op = CC_OP_SARB + ot; 3841 } 3842} 3843 3844static void gen_SARX(DisasContext *s, X86DecodedInsn *decode) 3845{ 3846 MemOp ot = decode->op[0].ot; 3847 int mask; 3848 3849 mask = ot == MO_64 ? 63 : 31; 3850 tcg_gen_andi_tl(s->T1, s->T1, mask); 3851 tcg_gen_sar_tl(s->T0, s->T0, s->T1); 3852} 3853 3854static void gen_SBB(DisasContext *s, X86DecodedInsn *decode) 3855{ 3856 MemOp ot = decode->op[0].ot; 3857 TCGv c_in = tcg_temp_new(); 3858 3859 gen_compute_eflags_c(s, c_in); 3860 if (s->prefix & PREFIX_LOCK) { 3861 tcg_gen_add_tl(s->T0, s->T1, c_in); 3862 tcg_gen_neg_tl(s->T0, s->T0); 3863 tcg_gen_atomic_add_fetch_tl(s->T0, s->A0, s->T0, 3864 s->mem_index, ot | MO_LE); 3865 } else { 3866 /* 3867 * TODO: SBB reg, reg could use gen_prepare_eflags_c followed by 3868 * negsetcond, and CC_OP_SUBB as the cc_op. 3869 */ 3870 tcg_gen_sub_tl(s->T0, s->T0, s->T1); 3871 tcg_gen_sub_tl(s->T0, s->T0, c_in); 3872 } 3873 prepare_update3_cc(decode, s, CC_OP_SBBB + ot, c_in); 3874} 3875 3876static void gen_SCAS(DisasContext *s, X86DecodedInsn *decode) 3877{ 3878 MemOp ot = decode->op[2].ot; 3879 if (s->prefix & (PREFIX_REPZ | PREFIX_REPNZ)) { 3880 gen_repz_nz(s, ot, gen_scas); 3881 } else { 3882 gen_scas(s, ot); 3883 } 3884} 3885 3886static void gen_SETcc(DisasContext *s, X86DecodedInsn *decode) 3887{ 3888 gen_setcc1(s, decode->b & 0xf, s->T0); 3889} 3890 3891static void gen_SFENCE(DisasContext *s, X86DecodedInsn *decode) 3892{ 3893 tcg_gen_mb(TCG_MO_ST_ST | TCG_BAR_SC); 3894} 3895 3896static void gen_SHA1NEXTE(DisasContext *s, X86DecodedInsn *decode) 3897{ 3898 gen_helper_sha1nexte(OP_PTR0, OP_PTR1, OP_PTR2); 3899} 3900 3901static void gen_SHA1MSG1(DisasContext *s, X86DecodedInsn *decode) 3902{ 3903 gen_helper_sha1msg1(OP_PTR0, OP_PTR1, OP_PTR2); 3904} 3905 3906static void gen_SHA1MSG2(DisasContext *s, X86DecodedInsn *decode) 3907{ 3908 gen_helper_sha1msg2(OP_PTR0, OP_PTR1, OP_PTR2); 3909} 3910 3911static void gen_SHA1RNDS4(DisasContext *s, X86DecodedInsn *decode) 3912{ 3913 switch(decode->immediate & 3) { 3914 case 0: 3915 gen_helper_sha1rnds4_f0(OP_PTR0, OP_PTR0, OP_PTR1); 3916 break; 3917 case 1: 3918 gen_helper_sha1rnds4_f1(OP_PTR0, OP_PTR0, OP_PTR1); 3919 break; 3920 case 2: 3921 gen_helper_sha1rnds4_f2(OP_PTR0, OP_PTR0, OP_PTR1); 3922 break; 3923 case 3: 3924 gen_helper_sha1rnds4_f3(OP_PTR0, OP_PTR0, OP_PTR1); 3925 break; 3926 } 3927} 3928 3929static void gen_SHA256MSG1(DisasContext *s, X86DecodedInsn *decode) 3930{ 3931 gen_helper_sha256msg1(OP_PTR0, OP_PTR1, OP_PTR2); 3932} 3933 3934static void gen_SHA256MSG2(DisasContext *s, X86DecodedInsn *decode) 3935{ 3936 gen_helper_sha256msg2(OP_PTR0, OP_PTR1, OP_PTR2); 3937} 3938 3939static void gen_SHA256RNDS2(DisasContext *s, X86DecodedInsn *decode) 3940{ 3941 TCGv_i32 wk0 = tcg_temp_new_i32(); 3942 TCGv_i32 wk1 = tcg_temp_new_i32(); 3943 3944 tcg_gen_ld_i32(wk0, tcg_env, ZMM_OFFSET(0) + offsetof(ZMMReg, ZMM_L(0))); 3945 tcg_gen_ld_i32(wk1, tcg_env, ZMM_OFFSET(0) + offsetof(ZMMReg, ZMM_L(1))); 3946 3947 gen_helper_sha256rnds2(OP_PTR0, OP_PTR1, OP_PTR2, wk0, wk1); 3948} 3949 3950static void gen_SHL(DisasContext *s, X86DecodedInsn *decode) 3951{ 3952 bool can_be_zero; 3953 TCGv count; 3954 MemOp ot = gen_shift_count(s, decode, &can_be_zero, &count, decode->op[2].unit); 3955 3956 if (!count) { 3957 return; 3958 } 3959 3960 decode->cc_dst = s->T0; 3961 decode->cc_src = tcg_temp_new(); 3962 tcg_gen_subi_tl(decode->cc_src, count, 1); 3963 tcg_gen_shl_tl(decode->cc_src, s->T0, decode->cc_src); 3964 tcg_gen_shl_tl(s->T0, s->T0, count); 3965 if (can_be_zero) { 3966 gen_shift_dynamic_flags(s, decode, count, CC_OP_SHLB + ot); 3967 } else { 3968 decode->cc_op = CC_OP_SHLB + ot; 3969 } 3970} 3971 3972static void gen_SHLD(DisasContext *s, X86DecodedInsn *decode) 3973{ 3974 bool can_be_zero; 3975 TCGv count; 3976 int unit = decode->e.op3 == X86_TYPE_I ? X86_OP_IMM : X86_OP_INT; 3977 MemOp ot = gen_shift_count(s, decode, &can_be_zero, &count, unit); 3978 3979 if (!count) { 3980 return; 3981 } 3982 3983 decode->cc_dst = s->T0; 3984 decode->cc_src = s->tmp0; 3985 gen_shiftd_rm_T1(s, ot, false, count); 3986 if (can_be_zero) { 3987 gen_shift_dynamic_flags(s, decode, count, CC_OP_SHLB + ot); 3988 } else { 3989 decode->cc_op = CC_OP_SHLB + ot; 3990 } 3991} 3992 3993static void gen_SHLX(DisasContext *s, X86DecodedInsn *decode) 3994{ 3995 MemOp ot = decode->op[0].ot; 3996 int mask; 3997 3998 mask = ot == MO_64 ? 63 : 31; 3999 tcg_gen_andi_tl(s->T1, s->T1, mask); 4000 tcg_gen_shl_tl(s->T0, s->T0, s->T1); 4001} 4002 4003static void gen_SHR(DisasContext *s, X86DecodedInsn *decode) 4004{ 4005 bool can_be_zero; 4006 TCGv count; 4007 MemOp ot = gen_shift_count(s, decode, &can_be_zero, &count, decode->op[2].unit); 4008 4009 if (!count) { 4010 return; 4011 } 4012 4013 decode->cc_dst = s->T0; 4014 decode->cc_src = tcg_temp_new(); 4015 tcg_gen_subi_tl(decode->cc_src, count, 1); 4016 tcg_gen_shr_tl(decode->cc_src, s->T0, decode->cc_src); 4017 tcg_gen_shr_tl(s->T0, s->T0, count); 4018 if (can_be_zero) { 4019 gen_shift_dynamic_flags(s, decode, count, CC_OP_SARB + ot); 4020 } else { 4021 decode->cc_op = CC_OP_SARB + ot; 4022 } 4023} 4024 4025static void gen_SHRD(DisasContext *s, X86DecodedInsn *decode) 4026{ 4027 bool can_be_zero; 4028 TCGv count; 4029 int unit = decode->e.op3 == X86_TYPE_I ? X86_OP_IMM : X86_OP_INT; 4030 MemOp ot = gen_shift_count(s, decode, &can_be_zero, &count, unit); 4031 4032 if (!count) { 4033 return; 4034 } 4035 4036 decode->cc_dst = s->T0; 4037 decode->cc_src = s->tmp0; 4038 gen_shiftd_rm_T1(s, ot, true, count); 4039 if (can_be_zero) { 4040 gen_shift_dynamic_flags(s, decode, count, CC_OP_SARB + ot); 4041 } else { 4042 decode->cc_op = CC_OP_SARB + ot; 4043 } 4044} 4045 4046static void gen_SHRX(DisasContext *s, X86DecodedInsn *decode) 4047{ 4048 MemOp ot = decode->op[0].ot; 4049 int mask; 4050 4051 mask = ot == MO_64 ? 63 : 31; 4052 tcg_gen_andi_tl(s->T1, s->T1, mask); 4053 tcg_gen_shr_tl(s->T0, s->T0, s->T1); 4054} 4055 4056static void gen_STC(DisasContext *s, X86DecodedInsn *decode) 4057{ 4058 gen_compute_eflags(s); 4059 tcg_gen_ori_tl(cpu_cc_src, cpu_cc_src, CC_C); 4060} 4061 4062static void gen_STD(DisasContext *s, X86DecodedInsn *decode) 4063{ 4064 tcg_gen_st_i32(tcg_constant_i32(-1), tcg_env, offsetof(CPUX86State, df)); 4065} 4066 4067static void gen_STI(DisasContext *s, X86DecodedInsn *decode) 4068{ 4069 gen_set_eflags(s, IF_MASK); 4070 s->base.is_jmp = DISAS_EOB_INHIBIT_IRQ; 4071} 4072 4073static void gen_VAESKEYGEN(DisasContext *s, X86DecodedInsn *decode) 4074{ 4075 TCGv_i32 imm = tcg_constant8u_i32(decode->immediate); 4076 assert(!s->vex_l); 4077 gen_helper_aeskeygenassist_xmm(tcg_env, OP_PTR0, OP_PTR1, imm); 4078} 4079 4080static void gen_STMXCSR(DisasContext *s, X86DecodedInsn *decode) 4081{ 4082 gen_helper_update_mxcsr(tcg_env); 4083 tcg_gen_ld32u_tl(s->T0, tcg_env, offsetof(CPUX86State, mxcsr)); 4084} 4085 4086static void gen_STOS(DisasContext *s, X86DecodedInsn *decode) 4087{ 4088 MemOp ot = decode->op[1].ot; 4089 if (s->prefix & (PREFIX_REPZ | PREFIX_REPNZ)) { 4090 gen_repz(s, ot, gen_stos); 4091 } else { 4092 gen_stos(s, ot); 4093 } 4094} 4095 4096static void gen_SUB(DisasContext *s, X86DecodedInsn *decode) 4097{ 4098 MemOp ot = decode->op[1].ot; 4099 4100 if (s->prefix & PREFIX_LOCK) { 4101 tcg_gen_neg_tl(s->T0, s->T1); 4102 tcg_gen_atomic_fetch_add_tl(s->cc_srcT, s->A0, s->T0, 4103 s->mem_index, ot | MO_LE); 4104 tcg_gen_sub_tl(s->T0, s->cc_srcT, s->T1); 4105 } else { 4106 tcg_gen_mov_tl(s->cc_srcT, s->T0); 4107 tcg_gen_sub_tl(s->T0, s->T0, s->T1); 4108 } 4109 prepare_update2_cc(decode, s, CC_OP_SUBB + ot); 4110} 4111 4112static void gen_SYSCALL(DisasContext *s, X86DecodedInsn *decode) 4113{ 4114 gen_update_cc_op(s); 4115 gen_update_eip_cur(s); 4116 gen_helper_syscall(tcg_env, cur_insn_len_i32(s)); 4117 if (LMA(s)) { 4118 assume_cc_op(s, CC_OP_EFLAGS); 4119 } 4120 4121 /* 4122 * TF handling for the syscall insn is different. The TF bit is checked 4123 * after the syscall insn completes. This allows #DB to not be 4124 * generated after one has entered CPL0 if TF is set in FMASK. 4125 */ 4126 s->base.is_jmp = DISAS_EOB_RECHECK_TF; 4127} 4128 4129static void gen_SYSENTER(DisasContext *s, X86DecodedInsn *decode) 4130{ 4131 gen_helper_sysenter(tcg_env); 4132 s->base.is_jmp = DISAS_EOB_ONLY; 4133} 4134 4135static void gen_SYSEXIT(DisasContext *s, X86DecodedInsn *decode) 4136{ 4137 gen_helper_sysexit(tcg_env, tcg_constant_i32(s->dflag - 1)); 4138 s->base.is_jmp = DISAS_EOB_ONLY; 4139} 4140 4141static void gen_SYSRET(DisasContext *s, X86DecodedInsn *decode) 4142{ 4143 gen_helper_sysret(tcg_env, tcg_constant_i32(s->dflag - 1)); 4144 if (LMA(s)) { 4145 assume_cc_op(s, CC_OP_EFLAGS); 4146 } 4147 4148 /* 4149 * TF handling for the sysret insn is different. The TF bit is checked 4150 * after the sysret insn completes. This allows #DB to be 4151 * generated "as if" the syscall insn in userspace has just 4152 * completed. 4153 */ 4154 s->base.is_jmp = DISAS_EOB_RECHECK_TF; 4155} 4156 4157static void gen_TZCNT(DisasContext *s, X86DecodedInsn *decode) 4158{ 4159 MemOp ot = decode->op[0].ot; 4160 4161 /* C bit (cc_src) is defined related to the input. */ 4162 decode->cc_src = tcg_temp_new(); 4163 decode->cc_dst = s->T0; 4164 decode->cc_op = CC_OP_BMILGB + ot; 4165 tcg_gen_mov_tl(decode->cc_src, s->T0); 4166 4167 /* A zero input returns the operand size. */ 4168 tcg_gen_ctzi_tl(s->T0, s->T0, 8 << ot); 4169} 4170 4171static void gen_UD(DisasContext *s, X86DecodedInsn *decode) 4172{ 4173 gen_illegal_opcode(s); 4174} 4175 4176static void gen_VAESIMC(DisasContext *s, X86DecodedInsn *decode) 4177{ 4178 assert(!s->vex_l); 4179 gen_helper_aesimc_xmm(tcg_env, OP_PTR0, OP_PTR2); 4180} 4181 4182/* 4183 * 00 = v*ps Vps, Hps, Wpd 4184 * 66 = v*pd Vpd, Hpd, Wps 4185 * f3 = v*ss Vss, Hss, Wps 4186 * f2 = v*sd Vsd, Hsd, Wps 4187 */ 4188#define SSE_CMP(x) { \ 4189 gen_helper_ ## x ## ps ## _xmm, gen_helper_ ## x ## pd ## _xmm, \ 4190 gen_helper_ ## x ## ss, gen_helper_ ## x ## sd, \ 4191 gen_helper_ ## x ## ps ## _ymm, gen_helper_ ## x ## pd ## _ymm} 4192static const SSEFunc_0_eppp gen_helper_cmp_funcs[32][6] = { 4193 SSE_CMP(cmpeq), 4194 SSE_CMP(cmplt), 4195 SSE_CMP(cmple), 4196 SSE_CMP(cmpunord), 4197 SSE_CMP(cmpneq), 4198 SSE_CMP(cmpnlt), 4199 SSE_CMP(cmpnle), 4200 SSE_CMP(cmpord), 4201 4202 SSE_CMP(cmpequ), 4203 SSE_CMP(cmpnge), 4204 SSE_CMP(cmpngt), 4205 SSE_CMP(cmpfalse), 4206 SSE_CMP(cmpnequ), 4207 SSE_CMP(cmpge), 4208 SSE_CMP(cmpgt), 4209 SSE_CMP(cmptrue), 4210 4211 SSE_CMP(cmpeqs), 4212 SSE_CMP(cmpltq), 4213 SSE_CMP(cmpleq), 4214 SSE_CMP(cmpunords), 4215 SSE_CMP(cmpneqq), 4216 SSE_CMP(cmpnltq), 4217 SSE_CMP(cmpnleq), 4218 SSE_CMP(cmpords), 4219 4220 SSE_CMP(cmpequs), 4221 SSE_CMP(cmpngeq), 4222 SSE_CMP(cmpngtq), 4223 SSE_CMP(cmpfalses), 4224 SSE_CMP(cmpnequs), 4225 SSE_CMP(cmpgeq), 4226 SSE_CMP(cmpgtq), 4227 SSE_CMP(cmptrues), 4228}; 4229#undef SSE_CMP 4230 4231static void gen_VCMP(DisasContext *s, X86DecodedInsn *decode) 4232{ 4233 int index = decode->immediate & (s->prefix & PREFIX_VEX ? 31 : 7); 4234 int b = 4235 s->prefix & PREFIX_REPZ ? 2 /* ss */ : 4236 s->prefix & PREFIX_REPNZ ? 3 /* sd */ : 4237 !!(s->prefix & PREFIX_DATA) /* pd */ + (s->vex_l << 2); 4238 4239 gen_helper_cmp_funcs[index][b](tcg_env, OP_PTR0, OP_PTR1, OP_PTR2); 4240} 4241 4242static void gen_VCOMI(DisasContext *s, X86DecodedInsn *decode) 4243{ 4244 SSEFunc_0_epp fn; 4245 fn = s->prefix & PREFIX_DATA ? gen_helper_comisd : gen_helper_comiss; 4246 fn(tcg_env, OP_PTR1, OP_PTR2); 4247 assume_cc_op(s, CC_OP_EFLAGS); 4248} 4249 4250static void gen_VCVTPD2PS(DisasContext *s, X86DecodedInsn *decode) 4251{ 4252 if (s->vex_l) { 4253 gen_helper_cvtpd2ps_ymm(tcg_env, OP_PTR0, OP_PTR2); 4254 } else { 4255 gen_helper_cvtpd2ps_xmm(tcg_env, OP_PTR0, OP_PTR2); 4256 } 4257} 4258 4259static void gen_VCVTPS2PD(DisasContext *s, X86DecodedInsn *decode) 4260{ 4261 if (s->vex_l) { 4262 gen_helper_cvtps2pd_ymm(tcg_env, OP_PTR0, OP_PTR2); 4263 } else { 4264 gen_helper_cvtps2pd_xmm(tcg_env, OP_PTR0, OP_PTR2); 4265 } 4266} 4267 4268static void gen_VCVTPS2PH(DisasContext *s, X86DecodedInsn *decode) 4269{ 4270 gen_unary_imm_fp_sse(s, decode, 4271 gen_helper_cvtps2ph_xmm, 4272 gen_helper_cvtps2ph_ymm); 4273 /* 4274 * VCVTPS2PH is the only instruction that performs an operation on a 4275 * register source and then *stores* into memory. 4276 */ 4277 if (decode->op[0].has_ea) { 4278 gen_store_sse(s, decode, decode->op[0].offset); 4279 } 4280} 4281 4282static void gen_VCVTSD2SS(DisasContext *s, X86DecodedInsn *decode) 4283{ 4284 gen_helper_cvtsd2ss(tcg_env, OP_PTR0, OP_PTR1, OP_PTR2); 4285} 4286 4287static void gen_VCVTSS2SD(DisasContext *s, X86DecodedInsn *decode) 4288{ 4289 gen_helper_cvtss2sd(tcg_env, OP_PTR0, OP_PTR1, OP_PTR2); 4290} 4291 4292static void gen_VCVTSI2Sx(DisasContext *s, X86DecodedInsn *decode) 4293{ 4294 int vec_len = vector_len(s, decode); 4295 TCGv_i32 in; 4296 4297 tcg_gen_gvec_mov(MO_64, decode->op[0].offset, decode->op[1].offset, vec_len, vec_len); 4298 4299#ifdef TARGET_X86_64 4300 MemOp ot = decode->op[2].ot; 4301 if (ot == MO_64) { 4302 if (s->prefix & PREFIX_REPNZ) { 4303 gen_helper_cvtsq2sd(tcg_env, OP_PTR0, s->T1); 4304 } else { 4305 gen_helper_cvtsq2ss(tcg_env, OP_PTR0, s->T1); 4306 } 4307 return; 4308 } 4309 in = s->tmp2_i32; 4310 tcg_gen_trunc_tl_i32(in, s->T1); 4311#else 4312 in = s->T1; 4313#endif 4314 4315 if (s->prefix & PREFIX_REPNZ) { 4316 gen_helper_cvtsi2sd(tcg_env, OP_PTR0, in); 4317 } else { 4318 gen_helper_cvtsi2ss(tcg_env, OP_PTR0, in); 4319 } 4320} 4321 4322static inline void gen_VCVTtSx2SI(DisasContext *s, X86DecodedInsn *decode, 4323 SSEFunc_i_ep ss2si, SSEFunc_l_ep ss2sq, 4324 SSEFunc_i_ep sd2si, SSEFunc_l_ep sd2sq) 4325{ 4326 TCGv_i32 out; 4327 4328#ifdef TARGET_X86_64 4329 MemOp ot = decode->op[0].ot; 4330 if (ot == MO_64) { 4331 if (s->prefix & PREFIX_REPNZ) { 4332 sd2sq(s->T0, tcg_env, OP_PTR2); 4333 } else { 4334 ss2sq(s->T0, tcg_env, OP_PTR2); 4335 } 4336 return; 4337 } 4338 4339 out = s->tmp2_i32; 4340#else 4341 out = s->T0; 4342#endif 4343 if (s->prefix & PREFIX_REPNZ) { 4344 sd2si(out, tcg_env, OP_PTR2); 4345 } else { 4346 ss2si(out, tcg_env, OP_PTR2); 4347 } 4348#ifdef TARGET_X86_64 4349 tcg_gen_extu_i32_tl(s->T0, out); 4350#endif 4351} 4352 4353#ifndef TARGET_X86_64 4354#define gen_helper_cvtss2sq NULL 4355#define gen_helper_cvtsd2sq NULL 4356#define gen_helper_cvttss2sq NULL 4357#define gen_helper_cvttsd2sq NULL 4358#endif 4359 4360static void gen_VCVTSx2SI(DisasContext *s, X86DecodedInsn *decode) 4361{ 4362 gen_VCVTtSx2SI(s, decode, 4363 gen_helper_cvtss2si, gen_helper_cvtss2sq, 4364 gen_helper_cvtsd2si, gen_helper_cvtsd2sq); 4365} 4366 4367static void gen_VCVTTSx2SI(DisasContext *s, X86DecodedInsn *decode) 4368{ 4369 gen_VCVTtSx2SI(s, decode, 4370 gen_helper_cvttss2si, gen_helper_cvttss2sq, 4371 gen_helper_cvttsd2si, gen_helper_cvttsd2sq); 4372} 4373 4374static void gen_VEXTRACTx128(DisasContext *s, X86DecodedInsn *decode) 4375{ 4376 int mask = decode->immediate & 1; 4377 int src_ofs = vector_elem_offset(&decode->op[1], MO_128, mask); 4378 if (decode->op[0].has_ea) { 4379 /* VEX-only instruction, no alignment requirements. */ 4380 gen_sto_env_A0(s, src_ofs, false); 4381 } else { 4382 tcg_gen_gvec_mov(MO_64, decode->op[0].offset, src_ofs, 16, 16); 4383 } 4384} 4385 4386static void gen_VEXTRACTPS(DisasContext *s, X86DecodedInsn *decode) 4387{ 4388 gen_pextr(s, decode, MO_32); 4389} 4390 4391static void gen_vinsertps(DisasContext *s, X86DecodedInsn *decode) 4392{ 4393 int val = decode->immediate; 4394 int dest_word = (val >> 4) & 3; 4395 int new_mask = (val & 15) | (1 << dest_word); 4396 int vec_len = 16; 4397 4398 assert(!s->vex_l); 4399 4400 if (new_mask == 15) { 4401 /* All zeroes except possibly for the inserted element */ 4402 tcg_gen_gvec_dup_imm(MO_64, decode->op[0].offset, vec_len, vec_len, 0); 4403 } else if (decode->op[1].offset != decode->op[0].offset) { 4404 gen_store_sse(s, decode, decode->op[1].offset); 4405 } 4406 4407 if (new_mask != (val & 15)) { 4408 tcg_gen_st_i32(s->tmp2_i32, tcg_env, 4409 vector_elem_offset(&decode->op[0], MO_32, dest_word)); 4410 } 4411 4412 if (new_mask != 15) { 4413 TCGv_i32 zero = tcg_constant_i32(0); /* float32_zero */ 4414 int i; 4415 for (i = 0; i < 4; i++) { 4416 if ((val >> i) & 1) { 4417 tcg_gen_st_i32(zero, tcg_env, 4418 vector_elem_offset(&decode->op[0], MO_32, i)); 4419 } 4420 } 4421 } 4422} 4423 4424static void gen_VINSERTPS_r(DisasContext *s, X86DecodedInsn *decode) 4425{ 4426 int val = decode->immediate; 4427 tcg_gen_ld_i32(s->tmp2_i32, tcg_env, 4428 vector_elem_offset(&decode->op[2], MO_32, (val >> 6) & 3)); 4429 gen_vinsertps(s, decode); 4430} 4431 4432static void gen_VINSERTPS_m(DisasContext *s, X86DecodedInsn *decode) 4433{ 4434 tcg_gen_qemu_ld_i32(s->tmp2_i32, s->A0, s->mem_index, MO_LEUL); 4435 gen_vinsertps(s, decode); 4436} 4437 4438static void gen_VINSERTx128(DisasContext *s, X86DecodedInsn *decode) 4439{ 4440 int mask = decode->immediate & 1; 4441 tcg_gen_gvec_mov(MO_64, 4442 decode->op[0].offset + offsetof(YMMReg, YMM_X(mask)), 4443 decode->op[2].offset + offsetof(YMMReg, YMM_X(0)), 16, 16); 4444 tcg_gen_gvec_mov(MO_64, 4445 decode->op[0].offset + offsetof(YMMReg, YMM_X(!mask)), 4446 decode->op[1].offset + offsetof(YMMReg, YMM_X(!mask)), 16, 16); 4447} 4448 4449static inline void gen_maskmov(DisasContext *s, X86DecodedInsn *decode, 4450 SSEFunc_0_eppt xmm, SSEFunc_0_eppt ymm) 4451{ 4452 if (!s->vex_l) { 4453 xmm(tcg_env, OP_PTR2, OP_PTR1, s->A0); 4454 } else { 4455 ymm(tcg_env, OP_PTR2, OP_PTR1, s->A0); 4456 } 4457} 4458 4459static void gen_VMASKMOVPD_st(DisasContext *s, X86DecodedInsn *decode) 4460{ 4461 gen_maskmov(s, decode, gen_helper_vpmaskmovq_st_xmm, gen_helper_vpmaskmovq_st_ymm); 4462} 4463 4464static void gen_VMASKMOVPS_st(DisasContext *s, X86DecodedInsn *decode) 4465{ 4466 gen_maskmov(s, decode, gen_helper_vpmaskmovd_st_xmm, gen_helper_vpmaskmovd_st_ymm); 4467} 4468 4469static void gen_VMOVHPx_ld(DisasContext *s, X86DecodedInsn *decode) 4470{ 4471 gen_ldq_env_A0(s, decode->op[0].offset + offsetof(XMMReg, XMM_Q(1))); 4472 if (decode->op[0].offset != decode->op[1].offset) { 4473 tcg_gen_ld_i64(s->tmp1_i64, tcg_env, decode->op[1].offset + offsetof(XMMReg, XMM_Q(0))); 4474 tcg_gen_st_i64(s->tmp1_i64, tcg_env, decode->op[0].offset + offsetof(XMMReg, XMM_Q(0))); 4475 } 4476} 4477 4478static void gen_VMOVHPx_st(DisasContext *s, X86DecodedInsn *decode) 4479{ 4480 gen_stq_env_A0(s, decode->op[2].offset + offsetof(XMMReg, XMM_Q(1))); 4481} 4482 4483static void gen_VMOVHPx(DisasContext *s, X86DecodedInsn *decode) 4484{ 4485 if (decode->op[0].offset != decode->op[2].offset) { 4486 tcg_gen_ld_i64(s->tmp1_i64, tcg_env, decode->op[2].offset + offsetof(XMMReg, XMM_Q(1))); 4487 tcg_gen_st_i64(s->tmp1_i64, tcg_env, decode->op[0].offset + offsetof(XMMReg, XMM_Q(1))); 4488 } 4489 if (decode->op[0].offset != decode->op[1].offset) { 4490 tcg_gen_ld_i64(s->tmp1_i64, tcg_env, decode->op[1].offset + offsetof(XMMReg, XMM_Q(0))); 4491 tcg_gen_st_i64(s->tmp1_i64, tcg_env, decode->op[0].offset + offsetof(XMMReg, XMM_Q(0))); 4492 } 4493} 4494 4495static void gen_VMOVHLPS(DisasContext *s, X86DecodedInsn *decode) 4496{ 4497 tcg_gen_ld_i64(s->tmp1_i64, tcg_env, decode->op[2].offset + offsetof(XMMReg, XMM_Q(1))); 4498 tcg_gen_st_i64(s->tmp1_i64, tcg_env, decode->op[0].offset + offsetof(XMMReg, XMM_Q(0))); 4499 if (decode->op[0].offset != decode->op[1].offset) { 4500 tcg_gen_ld_i64(s->tmp1_i64, tcg_env, decode->op[1].offset + offsetof(XMMReg, XMM_Q(1))); 4501 tcg_gen_st_i64(s->tmp1_i64, tcg_env, decode->op[0].offset + offsetof(XMMReg, XMM_Q(1))); 4502 } 4503} 4504 4505static void gen_VMOVLHPS(DisasContext *s, X86DecodedInsn *decode) 4506{ 4507 tcg_gen_ld_i64(s->tmp1_i64, tcg_env, decode->op[2].offset); 4508 tcg_gen_st_i64(s->tmp1_i64, tcg_env, decode->op[0].offset + offsetof(XMMReg, XMM_Q(1))); 4509 if (decode->op[0].offset != decode->op[1].offset) { 4510 tcg_gen_ld_i64(s->tmp1_i64, tcg_env, decode->op[1].offset + offsetof(XMMReg, XMM_Q(0))); 4511 tcg_gen_st_i64(s->tmp1_i64, tcg_env, decode->op[0].offset + offsetof(XMMReg, XMM_Q(0))); 4512 } 4513} 4514 4515/* 4516 * Note that MOVLPx supports 256-bit operation unlike MOVHLPx, MOVLHPx, MOXHPx. 4517 * Use a gvec move to move everything above the bottom 64 bits. 4518 */ 4519 4520static void gen_VMOVLPx(DisasContext *s, X86DecodedInsn *decode) 4521{ 4522 int vec_len = vector_len(s, decode); 4523 4524 tcg_gen_ld_i64(s->tmp1_i64, tcg_env, decode->op[2].offset + offsetof(XMMReg, XMM_Q(0))); 4525 tcg_gen_gvec_mov(MO_64, decode->op[0].offset, decode->op[1].offset, vec_len, vec_len); 4526 tcg_gen_st_i64(s->tmp1_i64, tcg_env, decode->op[0].offset + offsetof(XMMReg, XMM_Q(0))); 4527} 4528 4529static void gen_VMOVLPx_ld(DisasContext *s, X86DecodedInsn *decode) 4530{ 4531 int vec_len = vector_len(s, decode); 4532 4533 tcg_gen_qemu_ld_i64(s->tmp1_i64, s->A0, s->mem_index, MO_LEUQ); 4534 tcg_gen_gvec_mov(MO_64, decode->op[0].offset, decode->op[1].offset, vec_len, vec_len); 4535 tcg_gen_st_i64(s->tmp1_i64, OP_PTR0, offsetof(ZMMReg, ZMM_Q(0))); 4536} 4537 4538static void gen_VMOVLPx_st(DisasContext *s, X86DecodedInsn *decode) 4539{ 4540 tcg_gen_ld_i64(s->tmp1_i64, OP_PTR2, offsetof(ZMMReg, ZMM_Q(0))); 4541 tcg_gen_qemu_st_i64(s->tmp1_i64, s->A0, s->mem_index, MO_LEUQ); 4542} 4543 4544static void gen_VMOVSD_ld(DisasContext *s, X86DecodedInsn *decode) 4545{ 4546 TCGv_i64 zero = tcg_constant_i64(0); 4547 4548 tcg_gen_qemu_ld_i64(s->tmp1_i64, s->A0, s->mem_index, MO_LEUQ); 4549 tcg_gen_st_i64(zero, OP_PTR0, offsetof(ZMMReg, ZMM_Q(1))); 4550 tcg_gen_st_i64(s->tmp1_i64, OP_PTR0, offsetof(ZMMReg, ZMM_Q(0))); 4551} 4552 4553static void gen_VMOVSS(DisasContext *s, X86DecodedInsn *decode) 4554{ 4555 int vec_len = vector_len(s, decode); 4556 4557 tcg_gen_ld_i32(s->tmp2_i32, OP_PTR2, offsetof(ZMMReg, ZMM_L(0))); 4558 tcg_gen_gvec_mov(MO_64, decode->op[0].offset, decode->op[1].offset, vec_len, vec_len); 4559 tcg_gen_st_i32(s->tmp2_i32, OP_PTR0, offsetof(ZMMReg, ZMM_L(0))); 4560} 4561 4562static void gen_VMOVSS_ld(DisasContext *s, X86DecodedInsn *decode) 4563{ 4564 int vec_len = vector_len(s, decode); 4565 4566 tcg_gen_qemu_ld_i32(s->tmp2_i32, s->A0, s->mem_index, MO_LEUL); 4567 tcg_gen_gvec_dup_imm(MO_64, decode->op[0].offset, vec_len, vec_len, 0); 4568 tcg_gen_st_i32(s->tmp2_i32, OP_PTR0, offsetof(ZMMReg, ZMM_L(0))); 4569} 4570 4571static void gen_VMOVSS_st(DisasContext *s, X86DecodedInsn *decode) 4572{ 4573 tcg_gen_ld_i32(s->tmp2_i32, OP_PTR2, offsetof(ZMMReg, ZMM_L(0))); 4574 tcg_gen_qemu_st_i32(s->tmp2_i32, s->A0, s->mem_index, MO_LEUL); 4575} 4576 4577static void gen_VPMASKMOV_st(DisasContext *s, X86DecodedInsn *decode) 4578{ 4579 if (s->vex_w) { 4580 gen_VMASKMOVPD_st(s, decode); 4581 } else { 4582 gen_VMASKMOVPS_st(s, decode); 4583 } 4584} 4585 4586static void gen_VPERMD(DisasContext *s, X86DecodedInsn *decode) 4587{ 4588 assert(s->vex_l); 4589 gen_helper_vpermd_ymm(OP_PTR0, OP_PTR1, OP_PTR2); 4590} 4591 4592static void gen_VPERM2x128(DisasContext *s, X86DecodedInsn *decode) 4593{ 4594 TCGv_i32 imm = tcg_constant8u_i32(decode->immediate); 4595 assert(s->vex_l); 4596 gen_helper_vpermdq_ymm(OP_PTR0, OP_PTR1, OP_PTR2, imm); 4597} 4598 4599static void gen_VPHMINPOSUW(DisasContext *s, X86DecodedInsn *decode) 4600{ 4601 assert(!s->vex_l); 4602 gen_helper_phminposuw_xmm(tcg_env, OP_PTR0, OP_PTR2); 4603} 4604 4605static void gen_VROUNDSD(DisasContext *s, X86DecodedInsn *decode) 4606{ 4607 TCGv_i32 imm = tcg_constant8u_i32(decode->immediate); 4608 assert(!s->vex_l); 4609 gen_helper_roundsd_xmm(tcg_env, OP_PTR0, OP_PTR1, OP_PTR2, imm); 4610} 4611 4612static void gen_VROUNDSS(DisasContext *s, X86DecodedInsn *decode) 4613{ 4614 TCGv_i32 imm = tcg_constant8u_i32(decode->immediate); 4615 assert(!s->vex_l); 4616 gen_helper_roundss_xmm(tcg_env, OP_PTR0, OP_PTR1, OP_PTR2, imm); 4617} 4618 4619static void gen_VSHUF(DisasContext *s, X86DecodedInsn *decode) 4620{ 4621 TCGv_i32 imm = tcg_constant_i32(decode->immediate); 4622 SSEFunc_0_pppi ps, pd, fn; 4623 ps = s->vex_l ? gen_helper_shufps_ymm : gen_helper_shufps_xmm; 4624 pd = s->vex_l ? gen_helper_shufpd_ymm : gen_helper_shufpd_xmm; 4625 fn = s->prefix & PREFIX_DATA ? pd : ps; 4626 fn(OP_PTR0, OP_PTR1, OP_PTR2, imm); 4627} 4628 4629static void gen_VUCOMI(DisasContext *s, X86DecodedInsn *decode) 4630{ 4631 SSEFunc_0_epp fn; 4632 fn = s->prefix & PREFIX_DATA ? gen_helper_ucomisd : gen_helper_ucomiss; 4633 fn(tcg_env, OP_PTR1, OP_PTR2); 4634 assume_cc_op(s, CC_OP_EFLAGS); 4635} 4636 4637static void gen_VZEROALL(DisasContext *s, X86DecodedInsn *decode) 4638{ 4639 TCGv_ptr ptr = tcg_temp_new_ptr(); 4640 4641 tcg_gen_addi_ptr(ptr, tcg_env, offsetof(CPUX86State, xmm_regs)); 4642 gen_helper_memset(ptr, ptr, tcg_constant_i32(0), 4643 tcg_constant_ptr(CPU_NB_REGS * sizeof(ZMMReg))); 4644} 4645 4646static void gen_VZEROUPPER(DisasContext *s, X86DecodedInsn *decode) 4647{ 4648 int i; 4649 4650 for (i = 0; i < CPU_NB_REGS; i++) { 4651 int offset = offsetof(CPUX86State, xmm_regs[i].ZMM_X(1)); 4652 tcg_gen_gvec_dup_imm(MO_64, offset, 16, 16, 0); 4653 } 4654} 4655 4656static void gen_WAIT(DisasContext *s, X86DecodedInsn *decode) 4657{ 4658 if ((s->flags & (HF_MP_MASK | HF_TS_MASK)) == (HF_MP_MASK | HF_TS_MASK)) { 4659 gen_NM_exception(s); 4660 } else { 4661 /* needs to be treated as I/O because of ferr_irq */ 4662 translator_io_start(&s->base); 4663 gen_helper_fwait(tcg_env); 4664 } 4665} 4666 4667#ifndef CONFIG_USER_ONLY 4668static void gen_WRMSR(DisasContext *s, X86DecodedInsn *decode) 4669{ 4670 gen_update_cc_op(s); 4671 gen_update_eip_cur(s); 4672 gen_helper_wrmsr(tcg_env); 4673 s->base.is_jmp = DISAS_EOB_NEXT; 4674} 4675#else 4676#define gen_WRMSR gen_unreachable 4677#endif 4678 4679static void gen_WRxxBASE(DisasContext *s, X86DecodedInsn *decode) 4680{ 4681 TCGv base = cpu_seg_base[s->modrm & 8 ? R_GS : R_FS]; 4682 4683 /* Preserve hflags bits by testing CR4 at runtime. */ 4684 gen_helper_cr4_testbit(tcg_env, tcg_constant_i32(CR4_FSGSBASE_MASK)); 4685 tcg_gen_mov_tl(base, s->T0); 4686} 4687 4688static void gen_XADD(DisasContext *s, X86DecodedInsn *decode) 4689{ 4690 MemOp ot = decode->op[1].ot; 4691 4692 decode->cc_dst = tcg_temp_new(); 4693 decode->cc_src = s->T1; 4694 decode->cc_op = CC_OP_ADDB + ot; 4695 4696 if (s->prefix & PREFIX_LOCK) { 4697 tcg_gen_atomic_fetch_add_tl(s->T0, s->A0, s->T1, s->mem_index, ot | MO_LE); 4698 tcg_gen_add_tl(decode->cc_dst, s->T0, s->T1); 4699 } else { 4700 tcg_gen_add_tl(decode->cc_dst, s->T0, s->T1); 4701 /* 4702 * NOTE: writing memory first is important for MMU exceptions, 4703 * but "new result" wins for XADD AX, AX. 4704 */ 4705 gen_writeback(s, decode, 0, decode->cc_dst); 4706 } 4707 if (decode->op[0].has_ea || decode->op[2].n != decode->op[0].n) { 4708 gen_writeback(s, decode, 2, s->T0); 4709 } 4710} 4711 4712static void gen_XCHG(DisasContext *s, X86DecodedInsn *decode) 4713{ 4714 if (s->prefix & PREFIX_LOCK) { 4715 tcg_gen_atomic_xchg_tl(s->T0, s->A0, s->T1, 4716 s->mem_index, decode->op[0].ot | MO_LE); 4717 /* now store old value into register operand */ 4718 gen_op_mov_reg_v(s, decode->op[2].ot, decode->op[2].n, s->T0); 4719 } else { 4720 /* move destination value into source operand, source preserved in T1 */ 4721 gen_op_mov_reg_v(s, decode->op[2].ot, decode->op[2].n, s->T0); 4722 tcg_gen_mov_tl(s->T0, s->T1); 4723 } 4724} 4725 4726static void gen_XLAT(DisasContext *s, X86DecodedInsn *decode) 4727{ 4728 /* AL is already zero-extended into s->T0. */ 4729 tcg_gen_add_tl(s->A0, cpu_regs[R_EBX], s->T0); 4730 gen_lea_v_seg(s, s->A0, R_DS, s->override); 4731 gen_op_ld_v(s, MO_8, s->T0, s->A0); 4732} 4733 4734static void gen_XOR(DisasContext *s, X86DecodedInsn *decode) 4735{ 4736 /* special case XOR reg, reg */ 4737 if (decode->op[1].unit == X86_OP_INT && 4738 decode->op[2].unit == X86_OP_INT && 4739 decode->op[1].n == decode->op[2].n) { 4740 tcg_gen_movi_tl(s->T0, 0); 4741 decode->cc_op = CC_OP_EFLAGS; 4742 decode->cc_src = tcg_constant_tl(CC_Z | CC_P); 4743 } else { 4744 MemOp ot = decode->op[1].ot; 4745 4746 if (s->prefix & PREFIX_LOCK) { 4747 tcg_gen_atomic_xor_fetch_tl(s->T0, s->A0, s->T1, 4748 s->mem_index, ot | MO_LE); 4749 } else { 4750 tcg_gen_xor_tl(s->T0, s->T0, s->T1); 4751 } 4752 prepare_update1_cc(decode, s, CC_OP_LOGICB + ot); 4753 } 4754} 4755 4756static void gen_XRSTOR(DisasContext *s, X86DecodedInsn *decode) 4757{ 4758 TCGv_i64 features = tcg_temp_new_i64(); 4759 4760 tcg_gen_concat_tl_i64(features, cpu_regs[R_EAX], cpu_regs[R_EDX]); 4761 gen_helper_xrstor(tcg_env, s->A0, features); 4762 if (s->cpuid_7_0_ebx_features & CPUID_7_0_EBX_MPX) { 4763 /* 4764 * XRSTOR is how MPX is enabled, which changes how 4765 * we translate. Thus we need to end the TB. 4766 */ 4767 s->base.is_jmp = DISAS_EOB_NEXT; 4768 } 4769} 4770 4771static void gen_XSAVE(DisasContext *s, X86DecodedInsn *decode) 4772{ 4773 TCGv_i64 features = tcg_temp_new_i64(); 4774 4775 tcg_gen_concat_tl_i64(features, cpu_regs[R_EAX], cpu_regs[R_EDX]); 4776 gen_helper_xsave(tcg_env, s->A0, features); 4777} 4778 4779static void gen_XSAVEOPT(DisasContext *s, X86DecodedInsn *decode) 4780{ 4781 TCGv_i64 features = tcg_temp_new_i64(); 4782 4783 tcg_gen_concat_tl_i64(features, cpu_regs[R_EAX], cpu_regs[R_EDX]); 4784 gen_helper_xsave(tcg_env, s->A0, features); 4785} 4786