1/* 2 * New-style TCG opcode generator for i386 instructions 3 * 4 * Copyright (c) 2022 Red Hat, Inc. 5 * 6 * Author: Paolo Bonzini <pbonzini@redhat.com> 7 * 8 * This library is free software; you can redistribute it and/or 9 * modify it under the terms of the GNU Lesser General Public 10 * License as published by the Free Software Foundation; either 11 * version 2.1 of the License, or (at your option) any later version. 12 * 13 * This library is distributed in the hope that it will be useful, 14 * but WITHOUT ANY WARRANTY; without even the implied warranty of 15 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU 16 * Lesser General Public License for more details. 17 * 18 * You should have received a copy of the GNU Lesser General Public 19 * License along with this library; if not, see <http://www.gnu.org/licenses/>. 20 */ 21 22#define MMX_OFFSET(reg) \ 23 ({ assert((reg) >= 0 && (reg) <= 7); \ 24 offsetof(CPUX86State, fpregs[reg].mmx); }) 25 26#define ZMM_OFFSET(reg) \ 27 ({ assert((reg) >= 0 && (reg) <= 15); \ 28 offsetof(CPUX86State, xmm_regs[reg]); }) 29 30typedef void (*SSEFunc_i_ep)(TCGv_i32 val, TCGv_ptr env, TCGv_ptr reg); 31typedef void (*SSEFunc_l_ep)(TCGv_i64 val, TCGv_ptr env, TCGv_ptr reg); 32typedef void (*SSEFunc_0_epp)(TCGv_ptr env, TCGv_ptr reg_a, TCGv_ptr reg_b); 33typedef void (*SSEFunc_0_eppp)(TCGv_ptr env, TCGv_ptr reg_a, TCGv_ptr reg_b, 34 TCGv_ptr reg_c); 35typedef void (*SSEFunc_0_epppp)(TCGv_ptr env, TCGv_ptr reg_a, TCGv_ptr reg_b, 36 TCGv_ptr reg_c, TCGv_ptr reg_d); 37typedef void (*SSEFunc_0_eppi)(TCGv_ptr env, TCGv_ptr reg_a, TCGv_ptr reg_b, 38 TCGv_i32 val); 39typedef void (*SSEFunc_0_epppi)(TCGv_ptr env, TCGv_ptr reg_a, TCGv_ptr reg_b, 40 TCGv_ptr reg_c, TCGv_i32 val); 41typedef void (*SSEFunc_0_ppi)(TCGv_ptr reg_a, TCGv_ptr reg_b, TCGv_i32 val); 42typedef void (*SSEFunc_0_pppi)(TCGv_ptr reg_a, TCGv_ptr reg_b, TCGv_ptr reg_c, 43 TCGv_i32 val); 44typedef void (*SSEFunc_0_eppt)(TCGv_ptr env, TCGv_ptr reg_a, TCGv_ptr reg_b, 45 TCGv val); 46typedef void (*SSEFunc_0_epppti)(TCGv_ptr env, TCGv_ptr reg_a, TCGv_ptr reg_b, 47 TCGv_ptr reg_c, TCGv a0, TCGv_i32 scale); 48typedef void (*SSEFunc_0_eppppi)(TCGv_ptr env, TCGv_ptr reg_a, TCGv_ptr reg_b, 49 TCGv_ptr reg_c, TCGv_ptr reg_d, TCGv_i32 flags); 50typedef void (*SSEFunc_0_eppppii)(TCGv_ptr env, TCGv_ptr reg_a, TCGv_ptr reg_b, 51 TCGv_ptr reg_c, TCGv_ptr reg_d, TCGv_i32 even, 52 TCGv_i32 odd); 53 54static void gen_JMP_m(DisasContext *s, X86DecodedInsn *decode); 55static void gen_JMP(DisasContext *s, X86DecodedInsn *decode); 56 57static inline TCGv_i32 tcg_constant8u_i32(uint8_t val) 58{ 59 return tcg_constant_i32(val); 60} 61 62static void gen_NM_exception(DisasContext *s) 63{ 64 gen_exception(s, EXCP07_PREX); 65} 66 67static void gen_lea_modrm(DisasContext *s, X86DecodedInsn *decode) 68{ 69 AddressParts *mem = &decode->mem; 70 TCGv ea; 71 72 ea = gen_lea_modrm_1(s, *mem, decode->e.vex_class == 12); 73 if (decode->e.special == X86_SPECIAL_BitTest) { 74 MemOp ot = decode->op[1].ot; 75 int poslen = 8 << ot; 76 int opn = decode->op[2].n; 77 TCGv ofs = tcg_temp_new(); 78 79 /* Extract memory displacement from the second operand. */ 80 assert(decode->op[2].unit == X86_OP_INT && decode->op[2].ot != MO_8); 81 tcg_gen_sextract_tl(ofs, cpu_regs[opn], 3, poslen - 3); 82 tcg_gen_andi_tl(ofs, ofs, -1 << ot); 83 tcg_gen_add_tl(s->A0, ea, ofs); 84 ea = s->A0; 85 } 86 87 gen_lea_v_seg(s, ea, mem->def_seg, s->override); 88} 89 90static inline int mmx_offset(MemOp ot) 91{ 92 switch (ot) { 93 case MO_8: 94 return offsetof(MMXReg, MMX_B(0)); 95 case MO_16: 96 return offsetof(MMXReg, MMX_W(0)); 97 case MO_32: 98 return offsetof(MMXReg, MMX_L(0)); 99 case MO_64: 100 return offsetof(MMXReg, MMX_Q(0)); 101 default: 102 g_assert_not_reached(); 103 } 104} 105 106static inline int xmm_offset(MemOp ot) 107{ 108 switch (ot) { 109 case MO_8: 110 return offsetof(ZMMReg, ZMM_B(0)); 111 case MO_16: 112 return offsetof(ZMMReg, ZMM_W(0)); 113 case MO_32: 114 return offsetof(ZMMReg, ZMM_L(0)); 115 case MO_64: 116 return offsetof(ZMMReg, ZMM_Q(0)); 117 case MO_128: 118 return offsetof(ZMMReg, ZMM_X(0)); 119 case MO_256: 120 return offsetof(ZMMReg, ZMM_Y(0)); 121 default: 122 g_assert_not_reached(); 123 } 124} 125 126static int vector_reg_offset(X86DecodedOp *op) 127{ 128 assert(op->unit == X86_OP_MMX || op->unit == X86_OP_SSE); 129 130 if (op->unit == X86_OP_MMX) { 131 return op->offset - mmx_offset(op->ot); 132 } else { 133 return op->offset - xmm_offset(op->ot); 134 } 135} 136 137static int vector_elem_offset(X86DecodedOp *op, MemOp ot, int n) 138{ 139 int base_ofs = vector_reg_offset(op); 140 switch(ot) { 141 case MO_8: 142 if (op->unit == X86_OP_MMX) { 143 return base_ofs + offsetof(MMXReg, MMX_B(n)); 144 } else { 145 return base_ofs + offsetof(ZMMReg, ZMM_B(n)); 146 } 147 case MO_16: 148 if (op->unit == X86_OP_MMX) { 149 return base_ofs + offsetof(MMXReg, MMX_W(n)); 150 } else { 151 return base_ofs + offsetof(ZMMReg, ZMM_W(n)); 152 } 153 case MO_32: 154 if (op->unit == X86_OP_MMX) { 155 return base_ofs + offsetof(MMXReg, MMX_L(n)); 156 } else { 157 return base_ofs + offsetof(ZMMReg, ZMM_L(n)); 158 } 159 case MO_64: 160 if (op->unit == X86_OP_MMX) { 161 return base_ofs; 162 } else { 163 return base_ofs + offsetof(ZMMReg, ZMM_Q(n)); 164 } 165 case MO_128: 166 assert(op->unit == X86_OP_SSE); 167 return base_ofs + offsetof(ZMMReg, ZMM_X(n)); 168 case MO_256: 169 assert(op->unit == X86_OP_SSE); 170 return base_ofs + offsetof(ZMMReg, ZMM_Y(n)); 171 default: 172 g_assert_not_reached(); 173 } 174} 175 176static void compute_mmx_offset(X86DecodedOp *op) 177{ 178 if (!op->has_ea) { 179 op->offset = MMX_OFFSET(op->n) + mmx_offset(op->ot); 180 } else { 181 op->offset = offsetof(CPUX86State, mmx_t0) + mmx_offset(op->ot); 182 } 183} 184 185static void compute_xmm_offset(X86DecodedOp *op) 186{ 187 if (!op->has_ea) { 188 op->offset = ZMM_OFFSET(op->n) + xmm_offset(op->ot); 189 } else { 190 op->offset = offsetof(CPUX86State, xmm_t0) + xmm_offset(op->ot); 191 } 192} 193 194static void gen_load_sse(DisasContext *s, TCGv temp, MemOp ot, int dest_ofs, bool aligned) 195{ 196 switch(ot) { 197 case MO_8: 198 gen_op_ld_v(s, MO_8, temp, s->A0); 199 tcg_gen_st8_tl(temp, tcg_env, dest_ofs); 200 break; 201 case MO_16: 202 gen_op_ld_v(s, MO_16, temp, s->A0); 203 tcg_gen_st16_tl(temp, tcg_env, dest_ofs); 204 break; 205 case MO_32: 206 gen_op_ld_v(s, MO_32, temp, s->A0); 207 tcg_gen_st32_tl(temp, tcg_env, dest_ofs); 208 break; 209 case MO_64: 210 gen_ldq_env_A0(s, dest_ofs); 211 break; 212 case MO_128: 213 gen_ldo_env_A0(s, dest_ofs, aligned); 214 break; 215 case MO_256: 216 gen_ldy_env_A0(s, dest_ofs, aligned); 217 break; 218 default: 219 g_assert_not_reached(); 220 } 221} 222 223static bool sse_needs_alignment(DisasContext *s, X86DecodedInsn *decode, MemOp ot) 224{ 225 switch (decode->e.vex_class) { 226 case 2: 227 case 4: 228 if ((s->prefix & PREFIX_VEX) || 229 decode->e.vex_special == X86_VEX_SSEUnaligned) { 230 /* MOST legacy SSE instructions require aligned memory operands, but not all. */ 231 return false; 232 } 233 /* fall through */ 234 case 1: 235 return ot >= MO_128; 236 237 default: 238 return false; 239 } 240} 241 242static void gen_load(DisasContext *s, X86DecodedInsn *decode, int opn, TCGv v) 243{ 244 X86DecodedOp *op = &decode->op[opn]; 245 246 switch (op->unit) { 247 case X86_OP_SKIP: 248 return; 249 case X86_OP_SEG: 250 tcg_gen_ld32u_tl(v, tcg_env, 251 offsetof(CPUX86State,segs[op->n].selector)); 252 break; 253#ifndef CONFIG_USER_ONLY 254 case X86_OP_CR: 255 if (op->n == 8) { 256 translator_io_start(&s->base); 257 gen_helper_read_cr8(v, tcg_env); 258 } else { 259 tcg_gen_ld_tl(v, tcg_env, offsetof(CPUX86State, cr[op->n])); 260 } 261 break; 262 case X86_OP_DR: 263 /* CR4.DE tested in the helper. */ 264 gen_helper_get_dr(v, tcg_env, tcg_constant_i32(op->n)); 265 break; 266#endif 267 case X86_OP_INT: 268 if (op->has_ea) { 269 if (v == s->T0 && decode->e.special == X86_SPECIAL_SExtT0) { 270 gen_op_ld_v(s, op->ot | MO_SIGN, v, s->A0); 271 } else { 272 gen_op_ld_v(s, op->ot, v, s->A0); 273 } 274 275 } else if (op->ot < MO_TL && v == s->T0 && 276 (decode->e.special == X86_SPECIAL_SExtT0 || 277 decode->e.special == X86_SPECIAL_ZExtT0)) { 278 if (op->ot == MO_8 && byte_reg_is_xH(s, op->n)) { 279 if (decode->e.special == X86_SPECIAL_SExtT0) { 280 tcg_gen_sextract_tl(v, cpu_regs[op->n - 4], 8, 8); 281 } else { 282 tcg_gen_extract_tl(v, cpu_regs[op->n - 4], 8, 8); 283 } 284 } else { 285 if (decode->e.special == X86_SPECIAL_SExtT0) { 286 tcg_gen_ext_tl(v, cpu_regs[op->n], op->ot | MO_SIGN); 287 } else { 288 tcg_gen_ext_tl(v, cpu_regs[op->n], op->ot); 289 } 290 } 291 292 } else { 293 gen_op_mov_v_reg(s, op->ot, v, op->n); 294 } 295 break; 296 case X86_OP_IMM: 297 tcg_gen_movi_tl(v, op->imm); 298 break; 299 300 case X86_OP_MMX: 301 compute_mmx_offset(op); 302 goto load_vector; 303 304 case X86_OP_SSE: 305 compute_xmm_offset(op); 306 load_vector: 307 if (op->has_ea) { 308 bool aligned = sse_needs_alignment(s, decode, op->ot); 309 gen_load_sse(s, v, op->ot, op->offset, aligned); 310 } 311 break; 312 313 default: 314 g_assert_not_reached(); 315 } 316} 317 318static TCGv_ptr op_ptr(X86DecodedInsn *decode, int opn) 319{ 320 X86DecodedOp *op = &decode->op[opn]; 321 322 assert(op->unit == X86_OP_MMX || op->unit == X86_OP_SSE); 323 if (op->v_ptr) { 324 return op->v_ptr; 325 } 326 op->v_ptr = tcg_temp_new_ptr(); 327 328 /* The temporary points to the MMXReg or ZMMReg. */ 329 tcg_gen_addi_ptr(op->v_ptr, tcg_env, vector_reg_offset(op)); 330 return op->v_ptr; 331} 332 333#define OP_PTR0 op_ptr(decode, 0) 334#define OP_PTR1 op_ptr(decode, 1) 335#define OP_PTR2 op_ptr(decode, 2) 336 337static void gen_writeback(DisasContext *s, X86DecodedInsn *decode, int opn, TCGv v) 338{ 339 X86DecodedOp *op = &decode->op[opn]; 340 switch (op->unit) { 341 case X86_OP_SKIP: 342 break; 343 case X86_OP_SEG: 344 /* Note that gen_movl_seg takes care of interrupt shadow and TF. */ 345 gen_movl_seg(s, op->n, v, op->n == R_SS); 346 break; 347 case X86_OP_INT: 348 if (op->has_ea) { 349 gen_op_st_v(s, op->ot, v, s->A0); 350 } else { 351 gen_op_mov_reg_v(s, op->ot, op->n, v); 352 } 353 break; 354 case X86_OP_MMX: 355 break; 356 case X86_OP_SSE: 357 if (!op->has_ea && (s->prefix & PREFIX_VEX) && op->ot <= MO_128) { 358 tcg_gen_gvec_dup_imm(MO_64, 359 offsetof(CPUX86State, xmm_regs[op->n].ZMM_X(1)), 360 16, 16, 0); 361 } 362 break; 363#ifndef CONFIG_USER_ONLY 364 case X86_OP_CR: 365 if (op->n == 8) { 366 translator_io_start(&s->base); 367 } 368 gen_helper_write_crN(tcg_env, tcg_constant_i32(op->n), v); 369 s->base.is_jmp = DISAS_EOB_NEXT; 370 break; 371 case X86_OP_DR: 372 /* CR4.DE tested in the helper. */ 373 gen_helper_set_dr(tcg_env, tcg_constant_i32(op->n), v); 374 s->base.is_jmp = DISAS_EOB_NEXT; 375 break; 376#endif 377 default: 378 g_assert_not_reached(); 379 } 380 op->unit = X86_OP_SKIP; 381} 382 383static inline int vector_len(DisasContext *s, X86DecodedInsn *decode) 384{ 385 if (decode->e.special == X86_SPECIAL_MMX && 386 !(s->prefix & (PREFIX_DATA | PREFIX_REPZ | PREFIX_REPNZ))) { 387 return 8; 388 } 389 return s->vex_l ? 32 : 16; 390} 391 392static void prepare_update1_cc(X86DecodedInsn *decode, DisasContext *s, CCOp op) 393{ 394 decode->cc_dst = s->T0; 395 decode->cc_op = op; 396} 397 398static void prepare_update2_cc(X86DecodedInsn *decode, DisasContext *s, CCOp op) 399{ 400 decode->cc_src = s->T1; 401 decode->cc_dst = s->T0; 402 decode->cc_op = op; 403} 404 405static void prepare_update_cc_incdec(X86DecodedInsn *decode, DisasContext *s, CCOp op) 406{ 407 gen_compute_eflags_c(s, s->T1); 408 prepare_update2_cc(decode, s, op); 409} 410 411static void prepare_update3_cc(X86DecodedInsn *decode, DisasContext *s, CCOp op, TCGv reg) 412{ 413 decode->cc_src2 = reg; 414 decode->cc_src = s->T1; 415 decode->cc_dst = s->T0; 416 decode->cc_op = op; 417} 418 419/* Set up decode->cc_* to modify CF while keeping other flags unchanged. */ 420static void prepare_update_cf(X86DecodedInsn *decode, DisasContext *s, TCGv cf) 421{ 422 switch (s->cc_op) { 423 case CC_OP_ADOX: 424 case CC_OP_ADCOX: 425 decode->cc_src2 = cpu_cc_src2; 426 decode->cc_src = cpu_cc_src; 427 decode->cc_op = CC_OP_ADCOX; 428 break; 429 430 case CC_OP_EFLAGS: 431 case CC_OP_ADCX: 432 decode->cc_src = cpu_cc_src; 433 decode->cc_op = CC_OP_ADCX; 434 break; 435 436 default: 437 decode->cc_src = tcg_temp_new(); 438 gen_mov_eflags(s, decode->cc_src); 439 decode->cc_op = CC_OP_ADCX; 440 break; 441 } 442 decode->cc_dst = cf; 443} 444 445static void gen_store_sse(DisasContext *s, X86DecodedInsn *decode, int src_ofs) 446{ 447 MemOp ot = decode->op[0].ot; 448 int vec_len = vector_len(s, decode); 449 bool aligned = sse_needs_alignment(s, decode, ot); 450 451 if (!decode->op[0].has_ea) { 452 tcg_gen_gvec_mov(MO_64, decode->op[0].offset, src_ofs, vec_len, vec_len); 453 return; 454 } 455 456 switch (ot) { 457 case MO_64: 458 gen_stq_env_A0(s, src_ofs); 459 break; 460 case MO_128: 461 gen_sto_env_A0(s, src_ofs, aligned); 462 break; 463 case MO_256: 464 gen_sty_env_A0(s, src_ofs, aligned); 465 break; 466 default: 467 g_assert_not_reached(); 468 } 469} 470 471static void gen_helper_pavgusb(TCGv_ptr env, TCGv_ptr reg_a, TCGv_ptr reg_b) 472{ 473 gen_helper_pavgb_mmx(env, reg_a, reg_a, reg_b); 474} 475 476#define FN_3DNOW_MOVE ((SSEFunc_0_epp) (uintptr_t) 1) 477static const SSEFunc_0_epp fns_3dnow[] = { 478 [0x0c] = gen_helper_pi2fw, 479 [0x0d] = gen_helper_pi2fd, 480 [0x1c] = gen_helper_pf2iw, 481 [0x1d] = gen_helper_pf2id, 482 [0x8a] = gen_helper_pfnacc, 483 [0x8e] = gen_helper_pfpnacc, 484 [0x90] = gen_helper_pfcmpge, 485 [0x94] = gen_helper_pfmin, 486 [0x96] = gen_helper_pfrcp, 487 [0x97] = gen_helper_pfrsqrt, 488 [0x9a] = gen_helper_pfsub, 489 [0x9e] = gen_helper_pfadd, 490 [0xa0] = gen_helper_pfcmpgt, 491 [0xa4] = gen_helper_pfmax, 492 [0xa6] = FN_3DNOW_MOVE, /* PFRCPIT1; no need to actually increase precision */ 493 [0xa7] = FN_3DNOW_MOVE, /* PFRSQIT1 */ 494 [0xb6] = FN_3DNOW_MOVE, /* PFRCPIT2 */ 495 [0xaa] = gen_helper_pfsubr, 496 [0xae] = gen_helper_pfacc, 497 [0xb0] = gen_helper_pfcmpeq, 498 [0xb4] = gen_helper_pfmul, 499 [0xb7] = gen_helper_pmulhrw_mmx, 500 [0xbb] = gen_helper_pswapd, 501 [0xbf] = gen_helper_pavgusb, 502}; 503 504static void gen_3dnow(DisasContext *s, X86DecodedInsn *decode) 505{ 506 uint8_t b = decode->immediate; 507 SSEFunc_0_epp fn = b < ARRAY_SIZE(fns_3dnow) ? fns_3dnow[b] : NULL; 508 509 if (!fn) { 510 gen_illegal_opcode(s); 511 return; 512 } 513 if (s->flags & HF_TS_MASK) { 514 gen_NM_exception(s); 515 return; 516 } 517 if (s->flags & HF_EM_MASK) { 518 gen_illegal_opcode(s); 519 return; 520 } 521 522 gen_helper_enter_mmx(tcg_env); 523 if (fn == FN_3DNOW_MOVE) { 524 tcg_gen_ld_i64(s->tmp1_i64, tcg_env, decode->op[1].offset); 525 tcg_gen_st_i64(s->tmp1_i64, tcg_env, decode->op[0].offset); 526 } else { 527 fn(tcg_env, OP_PTR0, OP_PTR1); 528 } 529} 530 531/* 532 * 00 = v*ps Vps, Hps, Wpd 533 * 66 = v*pd Vpd, Hpd, Wps 534 * f3 = v*ss Vss, Hss, Wps 535 * f2 = v*sd Vsd, Hsd, Wps 536 */ 537static inline void gen_unary_fp_sse(DisasContext *s, X86DecodedInsn *decode, 538 SSEFunc_0_epp pd_xmm, SSEFunc_0_epp ps_xmm, 539 SSEFunc_0_epp pd_ymm, SSEFunc_0_epp ps_ymm, 540 SSEFunc_0_eppp sd, SSEFunc_0_eppp ss) 541{ 542 if ((s->prefix & (PREFIX_REPZ | PREFIX_REPNZ)) != 0) { 543 SSEFunc_0_eppp fn = s->prefix & PREFIX_REPZ ? ss : sd; 544 if (!fn) { 545 gen_illegal_opcode(s); 546 return; 547 } 548 fn(tcg_env, OP_PTR0, OP_PTR1, OP_PTR2); 549 } else { 550 SSEFunc_0_epp ps, pd, fn; 551 ps = s->vex_l ? ps_ymm : ps_xmm; 552 pd = s->vex_l ? pd_ymm : pd_xmm; 553 fn = s->prefix & PREFIX_DATA ? pd : ps; 554 if (!fn) { 555 gen_illegal_opcode(s); 556 return; 557 } 558 fn(tcg_env, OP_PTR0, OP_PTR2); 559 } 560} 561#define UNARY_FP_SSE(uname, lname) \ 562static void gen_##uname(DisasContext *s, X86DecodedInsn *decode) \ 563{ \ 564 gen_unary_fp_sse(s, decode, \ 565 gen_helper_##lname##pd_xmm, \ 566 gen_helper_##lname##ps_xmm, \ 567 gen_helper_##lname##pd_ymm, \ 568 gen_helper_##lname##ps_ymm, \ 569 gen_helper_##lname##sd, \ 570 gen_helper_##lname##ss); \ 571} 572UNARY_FP_SSE(VSQRT, sqrt) 573 574/* 575 * 00 = v*ps Vps, Hps, Wpd 576 * 66 = v*pd Vpd, Hpd, Wps 577 * f3 = v*ss Vss, Hss, Wps 578 * f2 = v*sd Vsd, Hsd, Wps 579 */ 580static inline void gen_fp_sse(DisasContext *s, X86DecodedInsn *decode, 581 SSEFunc_0_eppp pd_xmm, SSEFunc_0_eppp ps_xmm, 582 SSEFunc_0_eppp pd_ymm, SSEFunc_0_eppp ps_ymm, 583 SSEFunc_0_eppp sd, SSEFunc_0_eppp ss) 584{ 585 SSEFunc_0_eppp ps, pd, fn; 586 if ((s->prefix & (PREFIX_REPZ | PREFIX_REPNZ)) != 0) { 587 fn = s->prefix & PREFIX_REPZ ? ss : sd; 588 } else { 589 ps = s->vex_l ? ps_ymm : ps_xmm; 590 pd = s->vex_l ? pd_ymm : pd_xmm; 591 fn = s->prefix & PREFIX_DATA ? pd : ps; 592 } 593 if (fn) { 594 fn(tcg_env, OP_PTR0, OP_PTR1, OP_PTR2); 595 } else { 596 gen_illegal_opcode(s); 597 } 598} 599 600#define FP_SSE(uname, lname) \ 601static void gen_##uname(DisasContext *s, X86DecodedInsn *decode) \ 602{ \ 603 gen_fp_sse(s, decode, \ 604 gen_helper_##lname##pd_xmm, \ 605 gen_helper_##lname##ps_xmm, \ 606 gen_helper_##lname##pd_ymm, \ 607 gen_helper_##lname##ps_ymm, \ 608 gen_helper_##lname##sd, \ 609 gen_helper_##lname##ss); \ 610} 611FP_SSE(VADD, add) 612FP_SSE(VMUL, mul) 613FP_SSE(VSUB, sub) 614FP_SSE(VMIN, min) 615FP_SSE(VDIV, div) 616FP_SSE(VMAX, max) 617 618#define FMA_SSE_PACKED(uname, ptr0, ptr1, ptr2, even, odd) \ 619static void gen_##uname##Px(DisasContext *s, X86DecodedInsn *decode) \ 620{ \ 621 SSEFunc_0_eppppii xmm = s->vex_w ? gen_helper_fma4pd_xmm : gen_helper_fma4ps_xmm; \ 622 SSEFunc_0_eppppii ymm = s->vex_w ? gen_helper_fma4pd_ymm : gen_helper_fma4ps_ymm; \ 623 SSEFunc_0_eppppii fn = s->vex_l ? ymm : xmm; \ 624 \ 625 fn(tcg_env, OP_PTR0, ptr0, ptr1, ptr2, \ 626 tcg_constant_i32(even), \ 627 tcg_constant_i32((even) ^ (odd))); \ 628} 629 630#define FMA_SSE(uname, ptr0, ptr1, ptr2, flags) \ 631FMA_SSE_PACKED(uname, ptr0, ptr1, ptr2, flags, flags) \ 632static void gen_##uname##Sx(DisasContext *s, X86DecodedInsn *decode) \ 633{ \ 634 SSEFunc_0_eppppi fn = s->vex_w ? gen_helper_fma4sd : gen_helper_fma4ss; \ 635 \ 636 fn(tcg_env, OP_PTR0, ptr0, ptr1, ptr2, \ 637 tcg_constant_i32(flags)); \ 638} \ 639 640FMA_SSE(VFMADD231, OP_PTR1, OP_PTR2, OP_PTR0, 0) 641FMA_SSE(VFMADD213, OP_PTR1, OP_PTR0, OP_PTR2, 0) 642FMA_SSE(VFMADD132, OP_PTR0, OP_PTR2, OP_PTR1, 0) 643 644FMA_SSE(VFNMADD231, OP_PTR1, OP_PTR2, OP_PTR0, float_muladd_negate_product) 645FMA_SSE(VFNMADD213, OP_PTR1, OP_PTR0, OP_PTR2, float_muladd_negate_product) 646FMA_SSE(VFNMADD132, OP_PTR0, OP_PTR2, OP_PTR1, float_muladd_negate_product) 647 648FMA_SSE(VFMSUB231, OP_PTR1, OP_PTR2, OP_PTR0, float_muladd_negate_c) 649FMA_SSE(VFMSUB213, OP_PTR1, OP_PTR0, OP_PTR2, float_muladd_negate_c) 650FMA_SSE(VFMSUB132, OP_PTR0, OP_PTR2, OP_PTR1, float_muladd_negate_c) 651 652FMA_SSE(VFNMSUB231, OP_PTR1, OP_PTR2, OP_PTR0, float_muladd_negate_c|float_muladd_negate_product) 653FMA_SSE(VFNMSUB213, OP_PTR1, OP_PTR0, OP_PTR2, float_muladd_negate_c|float_muladd_negate_product) 654FMA_SSE(VFNMSUB132, OP_PTR0, OP_PTR2, OP_PTR1, float_muladd_negate_c|float_muladd_negate_product) 655 656FMA_SSE_PACKED(VFMADDSUB231, OP_PTR1, OP_PTR2, OP_PTR0, float_muladd_negate_c, 0) 657FMA_SSE_PACKED(VFMADDSUB213, OP_PTR1, OP_PTR0, OP_PTR2, float_muladd_negate_c, 0) 658FMA_SSE_PACKED(VFMADDSUB132, OP_PTR0, OP_PTR2, OP_PTR1, float_muladd_negate_c, 0) 659 660FMA_SSE_PACKED(VFMSUBADD231, OP_PTR1, OP_PTR2, OP_PTR0, 0, float_muladd_negate_c) 661FMA_SSE_PACKED(VFMSUBADD213, OP_PTR1, OP_PTR0, OP_PTR2, 0, float_muladd_negate_c) 662FMA_SSE_PACKED(VFMSUBADD132, OP_PTR0, OP_PTR2, OP_PTR1, 0, float_muladd_negate_c) 663 664#define FP_UNPACK_SSE(uname, lname) \ 665static void gen_##uname(DisasContext *s, X86DecodedInsn *decode) \ 666{ \ 667 /* PS maps to the DQ integer instruction, PD maps to QDQ. */ \ 668 gen_fp_sse(s, decode, \ 669 gen_helper_##lname##qdq_xmm, \ 670 gen_helper_##lname##dq_xmm, \ 671 gen_helper_##lname##qdq_ymm, \ 672 gen_helper_##lname##dq_ymm, \ 673 NULL, NULL); \ 674} 675FP_UNPACK_SSE(VUNPCKLPx, punpckl) 676FP_UNPACK_SSE(VUNPCKHPx, punpckh) 677 678/* 679 * 00 = v*ps Vps, Wpd 680 * f3 = v*ss Vss, Wps 681 */ 682static inline void gen_unary_fp32_sse(DisasContext *s, X86DecodedInsn *decode, 683 SSEFunc_0_epp ps_xmm, 684 SSEFunc_0_epp ps_ymm, 685 SSEFunc_0_eppp ss) 686{ 687 if ((s->prefix & (PREFIX_DATA | PREFIX_REPNZ)) != 0) { 688 goto illegal_op; 689 } else if (s->prefix & PREFIX_REPZ) { 690 if (!ss) { 691 goto illegal_op; 692 } 693 ss(tcg_env, OP_PTR0, OP_PTR1, OP_PTR2); 694 } else { 695 SSEFunc_0_epp fn = s->vex_l ? ps_ymm : ps_xmm; 696 if (!fn) { 697 goto illegal_op; 698 } 699 fn(tcg_env, OP_PTR0, OP_PTR2); 700 } 701 return; 702 703illegal_op: 704 gen_illegal_opcode(s); 705} 706#define UNARY_FP32_SSE(uname, lname) \ 707static void gen_##uname(DisasContext *s, X86DecodedInsn *decode) \ 708{ \ 709 gen_unary_fp32_sse(s, decode, \ 710 gen_helper_##lname##ps_xmm, \ 711 gen_helper_##lname##ps_ymm, \ 712 gen_helper_##lname##ss); \ 713} 714UNARY_FP32_SSE(VRSQRT, rsqrt) 715UNARY_FP32_SSE(VRCP, rcp) 716 717/* 718 * 66 = v*pd Vpd, Hpd, Wpd 719 * f2 = v*ps Vps, Hps, Wps 720 */ 721static inline void gen_horizontal_fp_sse(DisasContext *s, X86DecodedInsn *decode, 722 SSEFunc_0_eppp pd_xmm, SSEFunc_0_eppp ps_xmm, 723 SSEFunc_0_eppp pd_ymm, SSEFunc_0_eppp ps_ymm) 724{ 725 SSEFunc_0_eppp ps, pd, fn; 726 ps = s->vex_l ? ps_ymm : ps_xmm; 727 pd = s->vex_l ? pd_ymm : pd_xmm; 728 fn = s->prefix & PREFIX_DATA ? pd : ps; 729 fn(tcg_env, OP_PTR0, OP_PTR1, OP_PTR2); 730} 731#define HORIZONTAL_FP_SSE(uname, lname) \ 732static void gen_##uname(DisasContext *s, X86DecodedInsn *decode) \ 733{ \ 734 gen_horizontal_fp_sse(s, decode, \ 735 gen_helper_##lname##pd_xmm, gen_helper_##lname##ps_xmm, \ 736 gen_helper_##lname##pd_ymm, gen_helper_##lname##ps_ymm); \ 737} 738HORIZONTAL_FP_SSE(VHADD, hadd) 739HORIZONTAL_FP_SSE(VHSUB, hsub) 740HORIZONTAL_FP_SSE(VADDSUB, addsub) 741 742static inline void gen_ternary_sse(DisasContext *s, X86DecodedInsn *decode, 743 int op3, SSEFunc_0_epppp xmm, SSEFunc_0_epppp ymm) 744{ 745 SSEFunc_0_epppp fn = s->vex_l ? ymm : xmm; 746 TCGv_ptr ptr3 = tcg_temp_new_ptr(); 747 748 /* The format of the fourth input is Lx */ 749 tcg_gen_addi_ptr(ptr3, tcg_env, ZMM_OFFSET(op3)); 750 fn(tcg_env, OP_PTR0, OP_PTR1, OP_PTR2, ptr3); 751} 752#define TERNARY_SSE(uname, uvname, lname) \ 753static void gen_##uvname(DisasContext *s, X86DecodedInsn *decode) \ 754{ \ 755 gen_ternary_sse(s, decode, (uint8_t)decode->immediate >> 4, \ 756 gen_helper_##lname##_xmm, gen_helper_##lname##_ymm); \ 757} \ 758static void gen_##uname(DisasContext *s, X86DecodedInsn *decode) \ 759{ \ 760 gen_ternary_sse(s, decode, 0, \ 761 gen_helper_##lname##_xmm, gen_helper_##lname##_ymm); \ 762} 763TERNARY_SSE(BLENDVPS, VBLENDVPS, blendvps) 764TERNARY_SSE(BLENDVPD, VBLENDVPD, blendvpd) 765TERNARY_SSE(PBLENDVB, VPBLENDVB, pblendvb) 766 767static inline void gen_binary_imm_sse(DisasContext *s, X86DecodedInsn *decode, 768 SSEFunc_0_epppi xmm, SSEFunc_0_epppi ymm) 769{ 770 TCGv_i32 imm = tcg_constant8u_i32(decode->immediate); 771 if (!s->vex_l) { 772 xmm(tcg_env, OP_PTR0, OP_PTR1, OP_PTR2, imm); 773 } else { 774 ymm(tcg_env, OP_PTR0, OP_PTR1, OP_PTR2, imm); 775 } 776} 777 778#define BINARY_IMM_SSE(uname, lname) \ 779static void gen_##uname(DisasContext *s, X86DecodedInsn *decode) \ 780{ \ 781 gen_binary_imm_sse(s, decode, \ 782 gen_helper_##lname##_xmm, \ 783 gen_helper_##lname##_ymm); \ 784} 785 786BINARY_IMM_SSE(VBLENDPD, blendpd) 787BINARY_IMM_SSE(VBLENDPS, blendps) 788BINARY_IMM_SSE(VPBLENDW, pblendw) 789BINARY_IMM_SSE(VDDPS, dpps) 790#define gen_helper_dppd_ymm NULL 791BINARY_IMM_SSE(VDDPD, dppd) 792BINARY_IMM_SSE(VMPSADBW, mpsadbw) 793BINARY_IMM_SSE(PCLMULQDQ, pclmulqdq) 794 795 796#define UNARY_INT_GVEC(uname, func, ...) \ 797static void gen_##uname(DisasContext *s, X86DecodedInsn *decode) \ 798{ \ 799 int vec_len = vector_len(s, decode); \ 800 \ 801 func(__VA_ARGS__, decode->op[0].offset, \ 802 decode->op[2].offset, vec_len, vec_len); \ 803} 804UNARY_INT_GVEC(PABSB, tcg_gen_gvec_abs, MO_8) 805UNARY_INT_GVEC(PABSW, tcg_gen_gvec_abs, MO_16) 806UNARY_INT_GVEC(PABSD, tcg_gen_gvec_abs, MO_32) 807UNARY_INT_GVEC(VBROADCASTx128, tcg_gen_gvec_dup_mem, MO_128) 808UNARY_INT_GVEC(VPBROADCASTB, tcg_gen_gvec_dup_mem, MO_8) 809UNARY_INT_GVEC(VPBROADCASTW, tcg_gen_gvec_dup_mem, MO_16) 810UNARY_INT_GVEC(VPBROADCASTD, tcg_gen_gvec_dup_mem, MO_32) 811UNARY_INT_GVEC(VPBROADCASTQ, tcg_gen_gvec_dup_mem, MO_64) 812 813 814#define BINARY_INT_GVEC(uname, func, ...) \ 815static void gen_##uname(DisasContext *s, X86DecodedInsn *decode) \ 816{ \ 817 int vec_len = vector_len(s, decode); \ 818 \ 819 func(__VA_ARGS__, \ 820 decode->op[0].offset, decode->op[1].offset, \ 821 decode->op[2].offset, vec_len, vec_len); \ 822} 823 824BINARY_INT_GVEC(PADDB, tcg_gen_gvec_add, MO_8) 825BINARY_INT_GVEC(PADDW, tcg_gen_gvec_add, MO_16) 826BINARY_INT_GVEC(PADDD, tcg_gen_gvec_add, MO_32) 827BINARY_INT_GVEC(PADDQ, tcg_gen_gvec_add, MO_64) 828BINARY_INT_GVEC(PADDSB, tcg_gen_gvec_ssadd, MO_8) 829BINARY_INT_GVEC(PADDSW, tcg_gen_gvec_ssadd, MO_16) 830BINARY_INT_GVEC(PADDUSB, tcg_gen_gvec_usadd, MO_8) 831BINARY_INT_GVEC(PADDUSW, tcg_gen_gvec_usadd, MO_16) 832BINARY_INT_GVEC(PAND, tcg_gen_gvec_and, MO_64) 833BINARY_INT_GVEC(PCMPEQB, tcg_gen_gvec_cmp, TCG_COND_EQ, MO_8) 834BINARY_INT_GVEC(PCMPEQD, tcg_gen_gvec_cmp, TCG_COND_EQ, MO_32) 835BINARY_INT_GVEC(PCMPEQW, tcg_gen_gvec_cmp, TCG_COND_EQ, MO_16) 836BINARY_INT_GVEC(PCMPEQQ, tcg_gen_gvec_cmp, TCG_COND_EQ, MO_64) 837BINARY_INT_GVEC(PCMPGTB, tcg_gen_gvec_cmp, TCG_COND_GT, MO_8) 838BINARY_INT_GVEC(PCMPGTW, tcg_gen_gvec_cmp, TCG_COND_GT, MO_16) 839BINARY_INT_GVEC(PCMPGTD, tcg_gen_gvec_cmp, TCG_COND_GT, MO_32) 840BINARY_INT_GVEC(PCMPGTQ, tcg_gen_gvec_cmp, TCG_COND_GT, MO_64) 841BINARY_INT_GVEC(PMAXSB, tcg_gen_gvec_smax, MO_8) 842BINARY_INT_GVEC(PMAXSW, tcg_gen_gvec_smax, MO_16) 843BINARY_INT_GVEC(PMAXSD, tcg_gen_gvec_smax, MO_32) 844BINARY_INT_GVEC(PMAXUB, tcg_gen_gvec_umax, MO_8) 845BINARY_INT_GVEC(PMAXUW, tcg_gen_gvec_umax, MO_16) 846BINARY_INT_GVEC(PMAXUD, tcg_gen_gvec_umax, MO_32) 847BINARY_INT_GVEC(PMINSB, tcg_gen_gvec_smin, MO_8) 848BINARY_INT_GVEC(PMINSW, tcg_gen_gvec_smin, MO_16) 849BINARY_INT_GVEC(PMINSD, tcg_gen_gvec_smin, MO_32) 850BINARY_INT_GVEC(PMINUB, tcg_gen_gvec_umin, MO_8) 851BINARY_INT_GVEC(PMINUW, tcg_gen_gvec_umin, MO_16) 852BINARY_INT_GVEC(PMINUD, tcg_gen_gvec_umin, MO_32) 853BINARY_INT_GVEC(PMULLW, tcg_gen_gvec_mul, MO_16) 854BINARY_INT_GVEC(PMULLD, tcg_gen_gvec_mul, MO_32) 855BINARY_INT_GVEC(POR, tcg_gen_gvec_or, MO_64) 856BINARY_INT_GVEC(PSUBB, tcg_gen_gvec_sub, MO_8) 857BINARY_INT_GVEC(PSUBW, tcg_gen_gvec_sub, MO_16) 858BINARY_INT_GVEC(PSUBD, tcg_gen_gvec_sub, MO_32) 859BINARY_INT_GVEC(PSUBQ, tcg_gen_gvec_sub, MO_64) 860BINARY_INT_GVEC(PSUBSB, tcg_gen_gvec_sssub, MO_8) 861BINARY_INT_GVEC(PSUBSW, tcg_gen_gvec_sssub, MO_16) 862BINARY_INT_GVEC(PSUBUSB, tcg_gen_gvec_ussub, MO_8) 863BINARY_INT_GVEC(PSUBUSW, tcg_gen_gvec_ussub, MO_16) 864BINARY_INT_GVEC(PXOR, tcg_gen_gvec_xor, MO_64) 865 866 867/* 868 * 00 = p* Pq, Qq (if mmx not NULL; no VEX) 869 * 66 = vp* Vx, Hx, Wx 870 * 871 * These are really the same encoding, because 1) V is the same as P when VEX.V 872 * is not present 2) P and Q are the same as H and W apart from MM/XMM 873 */ 874static inline void gen_binary_int_sse(DisasContext *s, X86DecodedInsn *decode, 875 SSEFunc_0_eppp mmx, SSEFunc_0_eppp xmm, SSEFunc_0_eppp ymm) 876{ 877 assert(!!mmx == !!(decode->e.special == X86_SPECIAL_MMX)); 878 879 if (mmx && (s->prefix & PREFIX_VEX) && !(s->prefix & PREFIX_DATA)) { 880 /* VEX encoding is not applicable to MMX instructions. */ 881 gen_illegal_opcode(s); 882 return; 883 } 884 if (!(s->prefix & PREFIX_DATA)) { 885 mmx(tcg_env, OP_PTR0, OP_PTR1, OP_PTR2); 886 } else if (!s->vex_l) { 887 xmm(tcg_env, OP_PTR0, OP_PTR1, OP_PTR2); 888 } else { 889 ymm(tcg_env, OP_PTR0, OP_PTR1, OP_PTR2); 890 } 891} 892 893 894#define BINARY_INT_MMX(uname, lname) \ 895static void gen_##uname(DisasContext *s, X86DecodedInsn *decode) \ 896{ \ 897 gen_binary_int_sse(s, decode, \ 898 gen_helper_##lname##_mmx, \ 899 gen_helper_##lname##_xmm, \ 900 gen_helper_##lname##_ymm); \ 901} 902BINARY_INT_MMX(PUNPCKLBW, punpcklbw) 903BINARY_INT_MMX(PUNPCKLWD, punpcklwd) 904BINARY_INT_MMX(PUNPCKLDQ, punpckldq) 905BINARY_INT_MMX(PACKSSWB, packsswb) 906BINARY_INT_MMX(PACKUSWB, packuswb) 907BINARY_INT_MMX(PUNPCKHBW, punpckhbw) 908BINARY_INT_MMX(PUNPCKHWD, punpckhwd) 909BINARY_INT_MMX(PUNPCKHDQ, punpckhdq) 910BINARY_INT_MMX(PACKSSDW, packssdw) 911 912BINARY_INT_MMX(PAVGB, pavgb) 913BINARY_INT_MMX(PAVGW, pavgw) 914BINARY_INT_MMX(PMADDWD, pmaddwd) 915BINARY_INT_MMX(PMULHUW, pmulhuw) 916BINARY_INT_MMX(PMULHW, pmulhw) 917BINARY_INT_MMX(PMULUDQ, pmuludq) 918BINARY_INT_MMX(PSADBW, psadbw) 919 920BINARY_INT_MMX(PSLLW_r, psllw) 921BINARY_INT_MMX(PSLLD_r, pslld) 922BINARY_INT_MMX(PSLLQ_r, psllq) 923BINARY_INT_MMX(PSRLW_r, psrlw) 924BINARY_INT_MMX(PSRLD_r, psrld) 925BINARY_INT_MMX(PSRLQ_r, psrlq) 926BINARY_INT_MMX(PSRAW_r, psraw) 927BINARY_INT_MMX(PSRAD_r, psrad) 928 929BINARY_INT_MMX(PHADDW, phaddw) 930BINARY_INT_MMX(PHADDSW, phaddsw) 931BINARY_INT_MMX(PHADDD, phaddd) 932BINARY_INT_MMX(PHSUBW, phsubw) 933BINARY_INT_MMX(PHSUBSW, phsubsw) 934BINARY_INT_MMX(PHSUBD, phsubd) 935BINARY_INT_MMX(PMADDUBSW, pmaddubsw) 936BINARY_INT_MMX(PSHUFB, pshufb) 937BINARY_INT_MMX(PSIGNB, psignb) 938BINARY_INT_MMX(PSIGNW, psignw) 939BINARY_INT_MMX(PSIGND, psignd) 940BINARY_INT_MMX(PMULHRSW, pmulhrsw) 941 942/* Instructions with no MMX equivalent. */ 943#define BINARY_INT_SSE(uname, lname) \ 944static void gen_##uname(DisasContext *s, X86DecodedInsn *decode) \ 945{ \ 946 gen_binary_int_sse(s, decode, \ 947 NULL, \ 948 gen_helper_##lname##_xmm, \ 949 gen_helper_##lname##_ymm); \ 950} 951 952/* Instructions with no MMX equivalent. */ 953BINARY_INT_SSE(PUNPCKLQDQ, punpcklqdq) 954BINARY_INT_SSE(PUNPCKHQDQ, punpckhqdq) 955BINARY_INT_SSE(VPACKUSDW, packusdw) 956BINARY_INT_SSE(VPERMILPS, vpermilps) 957BINARY_INT_SSE(VPERMILPD, vpermilpd) 958BINARY_INT_SSE(VMASKMOVPS, vpmaskmovd) 959BINARY_INT_SSE(VMASKMOVPD, vpmaskmovq) 960 961BINARY_INT_SSE(PMULDQ, pmuldq) 962 963BINARY_INT_SSE(VAESDEC, aesdec) 964BINARY_INT_SSE(VAESDECLAST, aesdeclast) 965BINARY_INT_SSE(VAESENC, aesenc) 966BINARY_INT_SSE(VAESENCLAST, aesenclast) 967 968#define UNARY_CMP_SSE(uname, lname) \ 969static void gen_##uname(DisasContext *s, X86DecodedInsn *decode) \ 970{ \ 971 if (!s->vex_l) { \ 972 gen_helper_##lname##_xmm(tcg_env, OP_PTR1, OP_PTR2); \ 973 } else { \ 974 gen_helper_##lname##_ymm(tcg_env, OP_PTR1, OP_PTR2); \ 975 } \ 976 assume_cc_op(s, CC_OP_EFLAGS); \ 977} 978UNARY_CMP_SSE(VPTEST, ptest) 979UNARY_CMP_SSE(VTESTPS, vtestps) 980UNARY_CMP_SSE(VTESTPD, vtestpd) 981 982static inline void gen_unary_int_sse(DisasContext *s, X86DecodedInsn *decode, 983 SSEFunc_0_epp xmm, SSEFunc_0_epp ymm) 984{ 985 if (!s->vex_l) { 986 xmm(tcg_env, OP_PTR0, OP_PTR2); 987 } else { 988 ymm(tcg_env, OP_PTR0, OP_PTR2); 989 } 990} 991 992#define UNARY_INT_SSE(uname, lname) \ 993static void gen_##uname(DisasContext *s, X86DecodedInsn *decode) \ 994{ \ 995 gen_unary_int_sse(s, decode, \ 996 gen_helper_##lname##_xmm, \ 997 gen_helper_##lname##_ymm); \ 998} 999 1000UNARY_INT_SSE(VPMOVSXBW, pmovsxbw) 1001UNARY_INT_SSE(VPMOVSXBD, pmovsxbd) 1002UNARY_INT_SSE(VPMOVSXBQ, pmovsxbq) 1003UNARY_INT_SSE(VPMOVSXWD, pmovsxwd) 1004UNARY_INT_SSE(VPMOVSXWQ, pmovsxwq) 1005UNARY_INT_SSE(VPMOVSXDQ, pmovsxdq) 1006 1007UNARY_INT_SSE(VPMOVZXBW, pmovzxbw) 1008UNARY_INT_SSE(VPMOVZXBD, pmovzxbd) 1009UNARY_INT_SSE(VPMOVZXBQ, pmovzxbq) 1010UNARY_INT_SSE(VPMOVZXWD, pmovzxwd) 1011UNARY_INT_SSE(VPMOVZXWQ, pmovzxwq) 1012UNARY_INT_SSE(VPMOVZXDQ, pmovzxdq) 1013 1014UNARY_INT_SSE(VMOVSLDUP, pmovsldup) 1015UNARY_INT_SSE(VMOVSHDUP, pmovshdup) 1016UNARY_INT_SSE(VMOVDDUP, pmovdldup) 1017 1018UNARY_INT_SSE(VCVTDQ2PD, cvtdq2pd) 1019UNARY_INT_SSE(VCVTPD2DQ, cvtpd2dq) 1020UNARY_INT_SSE(VCVTTPD2DQ, cvttpd2dq) 1021UNARY_INT_SSE(VCVTDQ2PS, cvtdq2ps) 1022UNARY_INT_SSE(VCVTPS2DQ, cvtps2dq) 1023UNARY_INT_SSE(VCVTTPS2DQ, cvttps2dq) 1024UNARY_INT_SSE(VCVTPH2PS, cvtph2ps) 1025 1026 1027static inline void gen_unary_imm_sse(DisasContext *s, X86DecodedInsn *decode, 1028 SSEFunc_0_ppi xmm, SSEFunc_0_ppi ymm) 1029{ 1030 TCGv_i32 imm = tcg_constant8u_i32(decode->immediate); 1031 if (!s->vex_l) { 1032 xmm(OP_PTR0, OP_PTR1, imm); 1033 } else { 1034 ymm(OP_PTR0, OP_PTR1, imm); 1035 } 1036} 1037 1038#define UNARY_IMM_SSE(uname, lname) \ 1039static void gen_##uname(DisasContext *s, X86DecodedInsn *decode) \ 1040{ \ 1041 gen_unary_imm_sse(s, decode, \ 1042 gen_helper_##lname##_xmm, \ 1043 gen_helper_##lname##_ymm); \ 1044} 1045 1046UNARY_IMM_SSE(PSHUFD, pshufd) 1047UNARY_IMM_SSE(PSHUFHW, pshufhw) 1048UNARY_IMM_SSE(PSHUFLW, pshuflw) 1049#define gen_helper_vpermq_xmm NULL 1050UNARY_IMM_SSE(VPERMQ, vpermq) 1051UNARY_IMM_SSE(VPERMILPS_i, vpermilps_imm) 1052UNARY_IMM_SSE(VPERMILPD_i, vpermilpd_imm) 1053 1054static inline void gen_unary_imm_fp_sse(DisasContext *s, X86DecodedInsn *decode, 1055 SSEFunc_0_eppi xmm, SSEFunc_0_eppi ymm) 1056{ 1057 TCGv_i32 imm = tcg_constant8u_i32(decode->immediate); 1058 if (!s->vex_l) { 1059 xmm(tcg_env, OP_PTR0, OP_PTR1, imm); 1060 } else { 1061 ymm(tcg_env, OP_PTR0, OP_PTR1, imm); 1062 } 1063} 1064 1065#define UNARY_IMM_FP_SSE(uname, lname) \ 1066static void gen_##uname(DisasContext *s, X86DecodedInsn *decode) \ 1067{ \ 1068 gen_unary_imm_fp_sse(s, decode, \ 1069 gen_helper_##lname##_xmm, \ 1070 gen_helper_##lname##_ymm); \ 1071} 1072 1073UNARY_IMM_FP_SSE(VROUNDPS, roundps) 1074UNARY_IMM_FP_SSE(VROUNDPD, roundpd) 1075 1076static inline void gen_vexw_avx(DisasContext *s, X86DecodedInsn *decode, 1077 SSEFunc_0_eppp d_xmm, SSEFunc_0_eppp q_xmm, 1078 SSEFunc_0_eppp d_ymm, SSEFunc_0_eppp q_ymm) 1079{ 1080 SSEFunc_0_eppp d = s->vex_l ? d_ymm : d_xmm; 1081 SSEFunc_0_eppp q = s->vex_l ? q_ymm : q_xmm; 1082 SSEFunc_0_eppp fn = s->vex_w ? q : d; 1083 fn(tcg_env, OP_PTR0, OP_PTR1, OP_PTR2); 1084} 1085 1086/* VEX.W affects whether to operate on 32- or 64-bit elements. */ 1087#define VEXW_AVX(uname, lname) \ 1088static void gen_##uname(DisasContext *s, X86DecodedInsn *decode) \ 1089{ \ 1090 gen_vexw_avx(s, decode, \ 1091 gen_helper_##lname##d_xmm, gen_helper_##lname##q_xmm, \ 1092 gen_helper_##lname##d_ymm, gen_helper_##lname##q_ymm); \ 1093} 1094VEXW_AVX(VPSLLV, vpsllv) 1095VEXW_AVX(VPSRLV, vpsrlv) 1096VEXW_AVX(VPSRAV, vpsrav) 1097VEXW_AVX(VPMASKMOV, vpmaskmov) 1098 1099/* Same as above, but with extra arguments to the helper. */ 1100static inline void gen_vsib_avx(DisasContext *s, X86DecodedInsn *decode, 1101 SSEFunc_0_epppti d_xmm, SSEFunc_0_epppti q_xmm, 1102 SSEFunc_0_epppti d_ymm, SSEFunc_0_epppti q_ymm) 1103{ 1104 SSEFunc_0_epppti d = s->vex_l ? d_ymm : d_xmm; 1105 SSEFunc_0_epppti q = s->vex_l ? q_ymm : q_xmm; 1106 SSEFunc_0_epppti fn = s->vex_w ? q : d; 1107 TCGv_i32 scale = tcg_constant_i32(decode->mem.scale); 1108 TCGv_ptr index = tcg_temp_new_ptr(); 1109 1110 /* Pass third input as (index, base, scale) */ 1111 tcg_gen_addi_ptr(index, tcg_env, ZMM_OFFSET(decode->mem.index)); 1112 fn(tcg_env, OP_PTR0, OP_PTR1, index, s->A0, scale); 1113 1114 /* 1115 * There are two output operands, so zero OP1's high 128 bits 1116 * in the VEX.128 case. 1117 */ 1118 if (!s->vex_l) { 1119 int ymmh_ofs = vector_elem_offset(&decode->op[1], MO_128, 1); 1120 tcg_gen_gvec_dup_imm(MO_64, ymmh_ofs, 16, 16, 0); 1121 } 1122} 1123#define VSIB_AVX(uname, lname) \ 1124static void gen_##uname(DisasContext *s, X86DecodedInsn *decode) \ 1125{ \ 1126 gen_vsib_avx(s, decode, \ 1127 gen_helper_##lname##d_xmm, gen_helper_##lname##q_xmm, \ 1128 gen_helper_##lname##d_ymm, gen_helper_##lname##q_ymm); \ 1129} 1130VSIB_AVX(VPGATHERD, vpgatherd) 1131VSIB_AVX(VPGATHERQ, vpgatherq) 1132 1133static void gen_AAA(DisasContext *s, X86DecodedInsn *decode) 1134{ 1135 gen_update_cc_op(s); 1136 gen_helper_aaa(tcg_env); 1137 assume_cc_op(s, CC_OP_EFLAGS); 1138} 1139 1140static void gen_AAD(DisasContext *s, X86DecodedInsn *decode) 1141{ 1142 gen_helper_aad(s->T0, s->T0, s->T1); 1143 prepare_update1_cc(decode, s, CC_OP_LOGICB); 1144} 1145 1146static void gen_AAM(DisasContext *s, X86DecodedInsn *decode) 1147{ 1148 if (decode->immediate == 0) { 1149 gen_exception(s, EXCP00_DIVZ); 1150 } else { 1151 gen_helper_aam(s->T0, s->T0, s->T1); 1152 prepare_update1_cc(decode, s, CC_OP_LOGICB); 1153 } 1154} 1155 1156static void gen_AAS(DisasContext *s, X86DecodedInsn *decode) 1157{ 1158 gen_update_cc_op(s); 1159 gen_helper_aas(tcg_env); 1160 assume_cc_op(s, CC_OP_EFLAGS); 1161} 1162 1163static void gen_ADD(DisasContext *s, X86DecodedInsn *decode); 1164static void gen_ADC(DisasContext *s, X86DecodedInsn *decode) 1165{ 1166 MemOp ot = decode->op[1].ot; 1167 TCGv c_in; 1168 1169 /* 1170 * Try to avoid CC_OP_ADC by transforming as follows: 1171 * CC_ADC: src1 = dst + c_in, src2 = 0, src3 = c_in 1172 * CC_ADD: src1 = dst + c_in, src2 = c_in (no src3) 1173 * 1174 * In general src2 vs. src3 matters when computing AF and OF, but not here: 1175 * - AF is bit 4 of dst^src1^src2, which is bit 4 of dst^src1 in both cases 1176 * - OF is a function of the two MSBs, and in both cases they are zero for src2 1177 */ 1178 if (decode->e.op2 == X86_TYPE_I && decode->immediate == 0) { 1179 gen_compute_eflags_c(s, s->T1); 1180 gen_ADD(s, decode); 1181 return; 1182 } 1183 1184 c_in = tcg_temp_new(); 1185 gen_compute_eflags_c(s, c_in); 1186 if (s->prefix & PREFIX_LOCK) { 1187 tcg_gen_add_tl(s->T0, c_in, s->T1); 1188 tcg_gen_atomic_add_fetch_tl(s->T0, s->A0, s->T0, 1189 s->mem_index, ot | MO_LE); 1190 } else { 1191 tcg_gen_add_tl(s->T0, s->T0, s->T1); 1192 tcg_gen_add_tl(s->T0, s->T0, c_in); 1193 } 1194 prepare_update3_cc(decode, s, CC_OP_ADCB + ot, c_in); 1195} 1196 1197static void gen_ADCOX(DisasContext *s, X86DecodedInsn *decode, int cc_op) 1198{ 1199 MemOp ot = decode->op[0].ot; 1200 TCGv carry_in = NULL; 1201 TCGv *carry_out = (cc_op == CC_OP_ADCX ? &decode->cc_dst : &decode->cc_src2); 1202 TCGv zero; 1203 1204 decode->cc_op = cc_op; 1205 *carry_out = tcg_temp_new(); 1206 if (CC_OP_HAS_EFLAGS(s->cc_op)) { 1207 decode->cc_src = cpu_cc_src; 1208 1209 /* Re-use the carry-out from a previous round? */ 1210 if (s->cc_op == cc_op || s->cc_op == CC_OP_ADCOX) { 1211 carry_in = (cc_op == CC_OP_ADCX ? cpu_cc_dst : cpu_cc_src2); 1212 } 1213 1214 /* Preserve the opposite carry from previous rounds? */ 1215 if (s->cc_op != cc_op && s->cc_op != CC_OP_EFLAGS) { 1216 decode->cc_op = CC_OP_ADCOX; 1217 if (carry_out == &decode->cc_dst) { 1218 decode->cc_src2 = cpu_cc_src2; 1219 } else { 1220 decode->cc_dst = cpu_cc_dst; 1221 } 1222 } 1223 } else { 1224 decode->cc_src = tcg_temp_new(); 1225 gen_mov_eflags(s, decode->cc_src); 1226 } 1227 1228 if (!carry_in) { 1229 /* Get carry_in out of EFLAGS. */ 1230 carry_in = tcg_temp_new(); 1231 tcg_gen_extract_tl(carry_in, decode->cc_src, 1232 ctz32(cc_op == CC_OP_ADCX ? CC_C : CC_O), 1); 1233 } 1234 1235 switch (ot) { 1236#ifdef TARGET_X86_64 1237 case MO_32: 1238 /* If TL is 64-bit just do everything in 64-bit arithmetic. */ 1239 tcg_gen_ext32u_tl(s->T0, s->T0); 1240 tcg_gen_ext32u_tl(s->T1, s->T1); 1241 tcg_gen_add_i64(s->T0, s->T0, s->T1); 1242 tcg_gen_add_i64(s->T0, s->T0, carry_in); 1243 tcg_gen_shri_i64(*carry_out, s->T0, 32); 1244 break; 1245#endif 1246 default: 1247 zero = tcg_constant_tl(0); 1248 tcg_gen_add2_tl(s->T0, *carry_out, s->T0, zero, carry_in, zero); 1249 tcg_gen_add2_tl(s->T0, *carry_out, s->T0, *carry_out, s->T1, zero); 1250 break; 1251 } 1252} 1253 1254static void gen_ADCX(DisasContext *s, X86DecodedInsn *decode) 1255{ 1256 gen_ADCOX(s, decode, CC_OP_ADCX); 1257} 1258 1259static void gen_ADD(DisasContext *s, X86DecodedInsn *decode) 1260{ 1261 MemOp ot = decode->op[1].ot; 1262 1263 if (s->prefix & PREFIX_LOCK) { 1264 tcg_gen_atomic_add_fetch_tl(s->T0, s->A0, s->T1, 1265 s->mem_index, ot | MO_LE); 1266 } else { 1267 tcg_gen_add_tl(s->T0, s->T0, s->T1); 1268 } 1269 prepare_update2_cc(decode, s, CC_OP_ADDB + ot); 1270} 1271 1272static void gen_ADOX(DisasContext *s, X86DecodedInsn *decode) 1273{ 1274 gen_ADCOX(s, decode, CC_OP_ADOX); 1275} 1276 1277static void gen_AND(DisasContext *s, X86DecodedInsn *decode) 1278{ 1279 MemOp ot = decode->op[1].ot; 1280 1281 if (s->prefix & PREFIX_LOCK) { 1282 tcg_gen_atomic_and_fetch_tl(s->T0, s->A0, s->T1, 1283 s->mem_index, ot | MO_LE); 1284 } else { 1285 tcg_gen_and_tl(s->T0, s->T0, s->T1); 1286 } 1287 prepare_update1_cc(decode, s, CC_OP_LOGICB + ot); 1288} 1289 1290static void gen_ANDN(DisasContext *s, X86DecodedInsn *decode) 1291{ 1292 MemOp ot = decode->op[0].ot; 1293 1294 tcg_gen_andc_tl(s->T0, s->T1, s->T0); 1295 prepare_update1_cc(decode, s, CC_OP_LOGICB + ot); 1296} 1297 1298static void gen_ARPL(DisasContext *s, X86DecodedInsn *decode) 1299{ 1300 TCGv zf = tcg_temp_new(); 1301 TCGv flags = tcg_temp_new(); 1302 1303 gen_mov_eflags(s, flags); 1304 1305 /* Compute adjusted DST in T1, merging in SRC[RPL]. */ 1306 tcg_gen_deposit_tl(s->T1, s->T0, s->T1, 0, 2); 1307 1308 /* Z flag set if DST[RPL] < SRC[RPL] */ 1309 tcg_gen_setcond_tl(TCG_COND_LTU, zf, s->T0, s->T1); 1310 tcg_gen_deposit_tl(flags, flags, zf, ctz32(CC_Z), 1); 1311 1312 /* Place maximum RPL in DST */ 1313 tcg_gen_umax_tl(s->T0, s->T0, s->T1); 1314 1315 decode->cc_src = flags; 1316 decode->cc_op = CC_OP_EFLAGS; 1317} 1318 1319static void gen_BEXTR(DisasContext *s, X86DecodedInsn *decode) 1320{ 1321 MemOp ot = decode->op[0].ot; 1322 TCGv bound = tcg_constant_tl(ot == MO_64 ? 63 : 31); 1323 TCGv zero = tcg_constant_tl(0); 1324 TCGv mone = tcg_constant_tl(-1); 1325 1326 /* 1327 * Extract START, and shift the operand. 1328 * Shifts larger than operand size get zeros. 1329 */ 1330 tcg_gen_ext8u_tl(s->A0, s->T1); 1331 tcg_gen_shr_tl(s->T0, s->T0, s->A0); 1332 1333 tcg_gen_movcond_tl(TCG_COND_LEU, s->T0, s->A0, bound, s->T0, zero); 1334 1335 /* 1336 * Extract the LEN into an inverse mask. Lengths larger than 1337 * operand size get all zeros, length 0 gets all ones. 1338 */ 1339 tcg_gen_extract_tl(s->A0, s->T1, 8, 8); 1340 tcg_gen_shl_tl(s->T1, mone, s->A0); 1341 tcg_gen_movcond_tl(TCG_COND_LEU, s->T1, s->A0, bound, s->T1, zero); 1342 tcg_gen_andc_tl(s->T0, s->T0, s->T1); 1343 1344 prepare_update1_cc(decode, s, CC_OP_LOGICB + ot); 1345} 1346 1347static void gen_BLSI(DisasContext *s, X86DecodedInsn *decode) 1348{ 1349 MemOp ot = decode->op[0].ot; 1350 1351 /* input in T1, which is ready for prepare_update2_cc */ 1352 tcg_gen_neg_tl(s->T0, s->T1); 1353 tcg_gen_and_tl(s->T0, s->T0, s->T1); 1354 prepare_update2_cc(decode, s, CC_OP_BLSIB + ot); 1355} 1356 1357static void gen_BLSMSK(DisasContext *s, X86DecodedInsn *decode) 1358{ 1359 MemOp ot = decode->op[0].ot; 1360 1361 /* input in T1, which is ready for prepare_update2_cc */ 1362 tcg_gen_subi_tl(s->T0, s->T1, 1); 1363 tcg_gen_xor_tl(s->T0, s->T0, s->T1); 1364 prepare_update2_cc(decode, s, CC_OP_BMILGB + ot); 1365} 1366 1367static void gen_BLSR(DisasContext *s, X86DecodedInsn *decode) 1368{ 1369 MemOp ot = decode->op[0].ot; 1370 1371 /* input in T1, which is ready for prepare_update2_cc */ 1372 tcg_gen_subi_tl(s->T0, s->T1, 1); 1373 tcg_gen_and_tl(s->T0, s->T0, s->T1); 1374 prepare_update2_cc(decode, s, CC_OP_BMILGB + ot); 1375} 1376 1377static void gen_BOUND(DisasContext *s, X86DecodedInsn *decode) 1378{ 1379 TCGv_i32 op = tcg_temp_new_i32(); 1380 tcg_gen_trunc_tl_i32(op, s->T0); 1381 if (decode->op[1].ot == MO_16) { 1382 gen_helper_boundw(tcg_env, s->A0, op); 1383 } else { 1384 gen_helper_boundl(tcg_env, s->A0, op); 1385 } 1386} 1387 1388/* Non-standard convention - on entry T0 is zero-extended input, T1 is the output. */ 1389static void gen_BSF(DisasContext *s, X86DecodedInsn *decode) 1390{ 1391 MemOp ot = decode->op[0].ot; 1392 1393 /* Only the Z bit is defined and it is related to the input. */ 1394 decode->cc_dst = tcg_temp_new(); 1395 decode->cc_op = CC_OP_LOGICB + ot; 1396 tcg_gen_mov_tl(decode->cc_dst, s->T0); 1397 1398 /* 1399 * The manual says that the output is undefined when the 1400 * input is zero, but real hardware leaves it unchanged, and 1401 * real programs appear to depend on that. Accomplish this 1402 * by passing the output as the value to return upon zero. 1403 */ 1404 tcg_gen_ctz_tl(s->T0, s->T0, s->T1); 1405} 1406 1407/* Non-standard convention - on entry T0 is zero-extended input, T1 is the output. */ 1408static void gen_BSR(DisasContext *s, X86DecodedInsn *decode) 1409{ 1410 MemOp ot = decode->op[0].ot; 1411 1412 /* Only the Z bit is defined and it is related to the input. */ 1413 decode->cc_dst = tcg_temp_new(); 1414 decode->cc_op = CC_OP_LOGICB + ot; 1415 tcg_gen_mov_tl(decode->cc_dst, s->T0); 1416 1417 /* 1418 * The manual says that the output is undefined when the 1419 * input is zero, but real hardware leaves it unchanged, and 1420 * real programs appear to depend on that. Accomplish this 1421 * by passing the output as the value to return upon zero. 1422 * Plus, return the bit index of the first 1 bit. 1423 */ 1424 tcg_gen_xori_tl(s->T1, s->T1, TARGET_LONG_BITS - 1); 1425 tcg_gen_clz_tl(s->T0, s->T0, s->T1); 1426 tcg_gen_xori_tl(s->T0, s->T0, TARGET_LONG_BITS - 1); 1427} 1428 1429static void gen_BSWAP(DisasContext *s, X86DecodedInsn *decode) 1430{ 1431#ifdef TARGET_X86_64 1432 if (s->dflag == MO_64) { 1433 tcg_gen_bswap64_i64(s->T0, s->T0); 1434 return; 1435 } 1436#endif 1437 tcg_gen_bswap32_tl(s->T0, s->T0, TCG_BSWAP_OZ); 1438} 1439 1440static TCGv gen_bt_mask(DisasContext *s, X86DecodedInsn *decode) 1441{ 1442 MemOp ot = decode->op[1].ot; 1443 TCGv mask = tcg_temp_new(); 1444 1445 tcg_gen_andi_tl(s->T1, s->T1, (8 << ot) - 1); 1446 tcg_gen_shl_tl(mask, tcg_constant_tl(1), s->T1); 1447 return mask; 1448} 1449 1450/* Expects truncated bit index in COUNT, 1 << COUNT in MASK. */ 1451static void gen_bt_flags(DisasContext *s, X86DecodedInsn *decode, TCGv src, 1452 TCGv count, TCGv mask) 1453{ 1454 TCGv cf; 1455 1456 /* 1457 * C is the result of the test, Z is unchanged, and the others 1458 * are all undefined. 1459 */ 1460 if (s->cc_op == CC_OP_DYNAMIC || CC_OP_HAS_EFLAGS(s->cc_op)) { 1461 /* Generate EFLAGS and replace the C bit. */ 1462 cf = tcg_temp_new(); 1463 tcg_gen_setcond_tl(TCG_COND_TSTNE, cf, src, mask); 1464 prepare_update_cf(decode, s, cf); 1465 } else { 1466 /* 1467 * Z was going to be computed from the non-zero status of CC_DST. 1468 * We can get that same Z value (and the new C value) by leaving 1469 * CC_DST alone, setting CC_SRC, and using a CC_OP_SAR of the 1470 * same width. 1471 */ 1472 decode->cc_src = tcg_temp_new(); 1473 decode->cc_dst = cpu_cc_dst; 1474 decode->cc_op = CC_OP_SARB + cc_op_size(s->cc_op); 1475 tcg_gen_shr_tl(decode->cc_src, src, count); 1476 } 1477} 1478 1479static void gen_BT(DisasContext *s, X86DecodedInsn *decode) 1480{ 1481 TCGv count = s->T1; 1482 TCGv mask; 1483 1484 /* 1485 * Try to ensure that the rhs of the TSTNE condition is a constant (and a 1486 * power of two), as that is more readily available on most TCG backends. 1487 * 1488 * For immediate bit number gen_bt_mask()'s output is already a constant; 1489 * for register bit number, shift the source right and check bit 0. 1490 */ 1491 if (decode->e.op2 == X86_TYPE_I) { 1492 mask = gen_bt_mask(s, decode); 1493 } else { 1494 MemOp ot = decode->op[1].ot; 1495 1496 tcg_gen_andi_tl(s->T1, s->T1, (8 << ot) - 1); 1497 tcg_gen_shr_tl(s->T0, s->T0, s->T1); 1498 1499 count = tcg_constant_tl(0); 1500 mask = tcg_constant_tl(1); 1501 } 1502 gen_bt_flags(s, decode, s->T0, count, mask); 1503} 1504 1505static void gen_BTC(DisasContext *s, X86DecodedInsn *decode) 1506{ 1507 MemOp ot = decode->op[0].ot; 1508 TCGv old = tcg_temp_new(); 1509 TCGv mask = gen_bt_mask(s, decode); 1510 1511 if (s->prefix & PREFIX_LOCK) { 1512 tcg_gen_atomic_fetch_xor_tl(old, s->A0, mask, s->mem_index, ot | MO_LE); 1513 } else { 1514 tcg_gen_mov_tl(old, s->T0); 1515 tcg_gen_xor_tl(s->T0, s->T0, mask); 1516 } 1517 1518 gen_bt_flags(s, decode, old, s->T1, mask); 1519} 1520 1521static void gen_BTR(DisasContext *s, X86DecodedInsn *decode) 1522{ 1523 MemOp ot = decode->op[0].ot; 1524 TCGv old = tcg_temp_new(); 1525 TCGv mask = gen_bt_mask(s, decode); 1526 1527 if (s->prefix & PREFIX_LOCK) { 1528 TCGv maskc = tcg_temp_new(); 1529 tcg_gen_not_tl(maskc, mask); 1530 tcg_gen_atomic_fetch_and_tl(old, s->A0, maskc, s->mem_index, ot | MO_LE); 1531 } else { 1532 tcg_gen_mov_tl(old, s->T0); 1533 tcg_gen_andc_tl(s->T0, s->T0, mask); 1534 } 1535 1536 gen_bt_flags(s, decode, old, s->T1, mask); 1537} 1538 1539static void gen_BTS(DisasContext *s, X86DecodedInsn *decode) 1540{ 1541 MemOp ot = decode->op[0].ot; 1542 TCGv old = tcg_temp_new(); 1543 TCGv mask = gen_bt_mask(s, decode); 1544 1545 if (s->prefix & PREFIX_LOCK) { 1546 tcg_gen_atomic_fetch_or_tl(old, s->A0, mask, s->mem_index, ot | MO_LE); 1547 } else { 1548 tcg_gen_mov_tl(old, s->T0); 1549 tcg_gen_or_tl(s->T0, s->T0, mask); 1550 } 1551 1552 gen_bt_flags(s, decode, old, s->T1, mask); 1553} 1554 1555static void gen_BZHI(DisasContext *s, X86DecodedInsn *decode) 1556{ 1557 MemOp ot = decode->op[0].ot; 1558 TCGv bound = tcg_constant_tl(ot == MO_64 ? 63 : 31); 1559 TCGv zero = tcg_constant_tl(0); 1560 TCGv mone = tcg_constant_tl(-1); 1561 1562 tcg_gen_ext8u_tl(s->T1, s->T1); 1563 1564 tcg_gen_shl_tl(s->A0, mone, s->T1); 1565 tcg_gen_movcond_tl(TCG_COND_LEU, s->A0, s->T1, bound, s->A0, zero); 1566 tcg_gen_andc_tl(s->T0, s->T0, s->A0); 1567 /* 1568 * Note that since we're using BMILG (in order to get O 1569 * cleared) we need to store the inverse into C. 1570 */ 1571 tcg_gen_setcond_tl(TCG_COND_LEU, s->T1, s->T1, bound); 1572 prepare_update2_cc(decode, s, CC_OP_BMILGB + ot); 1573} 1574 1575static void gen_CALL(DisasContext *s, X86DecodedInsn *decode) 1576{ 1577 gen_push_v(s, eip_next_tl(s)); 1578 gen_JMP(s, decode); 1579} 1580 1581static void gen_CALL_m(DisasContext *s, X86DecodedInsn *decode) 1582{ 1583 gen_push_v(s, eip_next_tl(s)); 1584 gen_JMP_m(s, decode); 1585} 1586 1587static void gen_CALLF(DisasContext *s, X86DecodedInsn *decode) 1588{ 1589 gen_far_call(s); 1590} 1591 1592static void gen_CALLF_m(DisasContext *s, X86DecodedInsn *decode) 1593{ 1594 MemOp ot = decode->op[1].ot; 1595 1596 gen_op_ld_v(s, ot, s->T0, s->A0); 1597 gen_add_A0_im(s, 1 << ot); 1598 gen_op_ld_v(s, MO_16, s->T1, s->A0); 1599 gen_far_call(s); 1600} 1601 1602static void gen_CBW(DisasContext *s, X86DecodedInsn *decode) 1603{ 1604 MemOp src_ot = decode->op[0].ot - 1; 1605 1606 tcg_gen_ext_tl(s->T0, s->T0, src_ot | MO_SIGN); 1607} 1608 1609static void gen_CLC(DisasContext *s, X86DecodedInsn *decode) 1610{ 1611 gen_compute_eflags(s); 1612 tcg_gen_andi_tl(cpu_cc_src, cpu_cc_src, ~CC_C); 1613} 1614 1615static void gen_CLD(DisasContext *s, X86DecodedInsn *decode) 1616{ 1617 tcg_gen_st_i32(tcg_constant_i32(1), tcg_env, offsetof(CPUX86State, df)); 1618} 1619 1620static void gen_CLI(DisasContext *s, X86DecodedInsn *decode) 1621{ 1622 gen_reset_eflags(s, IF_MASK); 1623} 1624 1625static void gen_CLTS(DisasContext *s, X86DecodedInsn *decode) 1626{ 1627 gen_helper_clts(tcg_env); 1628 /* abort block because static cpu state changed */ 1629 s->base.is_jmp = DISAS_EOB_NEXT; 1630} 1631 1632static void gen_CMC(DisasContext *s, X86DecodedInsn *decode) 1633{ 1634 gen_compute_eflags(s); 1635 tcg_gen_xori_tl(cpu_cc_src, cpu_cc_src, CC_C); 1636} 1637 1638static void gen_CMOVcc(DisasContext *s, X86DecodedInsn *decode) 1639{ 1640 gen_cmovcc(s, decode->b & 0xf, s->T0, s->T1); 1641} 1642 1643static void gen_CMPccXADD(DisasContext *s, X86DecodedInsn *decode) 1644{ 1645 TCGLabel *label_top = gen_new_label(); 1646 TCGLabel *label_bottom = gen_new_label(); 1647 TCGv oldv = tcg_temp_new(); 1648 TCGv newv = tcg_temp_new(); 1649 TCGv cmpv = tcg_temp_new(); 1650 TCGCond cond; 1651 1652 TCGv cmp_lhs, cmp_rhs; 1653 MemOp ot, ot_full; 1654 1655 int jcc_op = (decode->b >> 1) & 7; 1656 static const TCGCond cond_table[8] = { 1657 [JCC_O] = TCG_COND_LT, /* test sign bit by comparing against 0 */ 1658 [JCC_B] = TCG_COND_LTU, 1659 [JCC_Z] = TCG_COND_EQ, 1660 [JCC_BE] = TCG_COND_LEU, 1661 [JCC_S] = TCG_COND_LT, /* test sign bit by comparing against 0 */ 1662 [JCC_P] = TCG_COND_TSTEQ, /* even parity - tests low bit of popcount */ 1663 [JCC_L] = TCG_COND_LT, 1664 [JCC_LE] = TCG_COND_LE, 1665 }; 1666 1667 cond = cond_table[jcc_op]; 1668 if (decode->b & 1) { 1669 cond = tcg_invert_cond(cond); 1670 } 1671 1672 ot = decode->op[0].ot; 1673 ot_full = ot | MO_LE; 1674 if (jcc_op >= JCC_S) { 1675 /* 1676 * Sign-extend values before subtracting for S, P (zero/sign extension 1677 * does not matter there) L, LE and their inverses. 1678 */ 1679 ot_full |= MO_SIGN; 1680 } 1681 1682 /* 1683 * cmpv will be moved to cc_src *after* cpu_regs[] is written back, so use 1684 * tcg_gen_ext_tl instead of gen_ext_tl. 1685 */ 1686 tcg_gen_ext_tl(cmpv, cpu_regs[decode->op[1].n], ot_full); 1687 1688 /* 1689 * Cmpxchg loop starts here. 1690 * - s->T1: addition operand (from decoder) 1691 * - s->A0: dest address (from decoder) 1692 * - s->cc_srcT: memory operand (lhs for comparison) 1693 * - cmpv: rhs for comparison 1694 */ 1695 gen_set_label(label_top); 1696 gen_op_ld_v(s, ot_full, s->cc_srcT, s->A0); 1697 tcg_gen_sub_tl(s->T0, s->cc_srcT, cmpv); 1698 1699 /* Compute the comparison result by hand, to avoid clobbering cc_*. */ 1700 switch (jcc_op) { 1701 case JCC_O: 1702 /* (src1 ^ src2) & (src1 ^ dst). newv is only used here for a moment */ 1703 cmp_lhs = tcg_temp_new(), cmp_rhs = tcg_constant_tl(0); 1704 tcg_gen_xor_tl(newv, s->cc_srcT, s->T0); 1705 tcg_gen_xor_tl(cmp_lhs, s->cc_srcT, cmpv); 1706 tcg_gen_and_tl(cmp_lhs, cmp_lhs, newv); 1707 tcg_gen_sextract_tl(cmp_lhs, cmp_lhs, 0, 8 << ot); 1708 break; 1709 1710 case JCC_P: 1711 cmp_lhs = tcg_temp_new(), cmp_rhs = tcg_constant_tl(1); 1712 tcg_gen_ext8u_tl(cmp_lhs, s->T0); 1713 tcg_gen_ctpop_tl(cmp_lhs, cmp_lhs); 1714 break; 1715 1716 case JCC_S: 1717 cmp_lhs = tcg_temp_new(), cmp_rhs = tcg_constant_tl(0); 1718 tcg_gen_sextract_tl(cmp_lhs, s->T0, 0, 8 << ot); 1719 break; 1720 1721 default: 1722 cmp_lhs = s->cc_srcT, cmp_rhs = cmpv; 1723 break; 1724 } 1725 1726 /* Compute new value: if condition does not hold, just store back s->cc_srcT */ 1727 tcg_gen_add_tl(newv, s->cc_srcT, s->T1); 1728 tcg_gen_movcond_tl(cond, newv, cmp_lhs, cmp_rhs, newv, s->cc_srcT); 1729 tcg_gen_atomic_cmpxchg_tl(oldv, s->A0, s->cc_srcT, newv, s->mem_index, ot_full); 1730 1731 /* Exit unconditionally if cmpxchg succeeded. */ 1732 tcg_gen_brcond_tl(TCG_COND_EQ, oldv, s->cc_srcT, label_bottom); 1733 1734 /* Try again if there was actually a store to make. */ 1735 tcg_gen_brcond_tl(cond, cmp_lhs, cmp_rhs, label_top); 1736 gen_set_label(label_bottom); 1737 1738 /* Store old value to registers only after a successful store. */ 1739 gen_writeback(s, decode, 1, s->cc_srcT); 1740 1741 decode->cc_dst = s->T0; 1742 decode->cc_src = cmpv; 1743 decode->cc_op = CC_OP_SUBB + ot; 1744} 1745 1746static void gen_CMPS(DisasContext *s, X86DecodedInsn *decode) 1747{ 1748 MemOp ot = decode->op[2].ot; 1749 gen_repz_nz(s, ot, gen_cmps); 1750} 1751 1752static void gen_CMPXCHG(DisasContext *s, X86DecodedInsn *decode) 1753{ 1754 MemOp ot = decode->op[2].ot; 1755 TCGv cmpv = tcg_temp_new(); 1756 TCGv oldv = tcg_temp_new(); 1757 TCGv newv = tcg_temp_new(); 1758 TCGv dest; 1759 1760 tcg_gen_ext_tl(cmpv, cpu_regs[R_EAX], ot); 1761 tcg_gen_ext_tl(newv, s->T1, ot); 1762 if (s->prefix & PREFIX_LOCK) { 1763 tcg_gen_atomic_cmpxchg_tl(oldv, s->A0, cmpv, newv, 1764 s->mem_index, ot | MO_LE); 1765 } else { 1766 tcg_gen_ext_tl(oldv, s->T0, ot); 1767 if (decode->op[0].has_ea) { 1768 /* 1769 * Perform an unconditional store cycle like physical cpu; 1770 * must be before changing accumulator to ensure 1771 * idempotency if the store faults and the instruction 1772 * is restarted 1773 */ 1774 tcg_gen_movcond_tl(TCG_COND_EQ, newv, oldv, cmpv, newv, oldv); 1775 gen_op_st_v(s, ot, newv, s->A0); 1776 } else { 1777 /* 1778 * Unlike the memory case, where "the destination operand receives 1779 * a write cycle without regard to the result of the comparison", 1780 * rm must not be touched altogether if the write fails, including 1781 * not zero-extending it on 64-bit processors. So, precompute 1782 * the result of a successful writeback and perform the movcond 1783 * directly on cpu_regs. In case rm is part of RAX, note that this 1784 * movcond and the one below are mutually exclusive is executed. 1785 */ 1786 dest = gen_op_deposit_reg_v(s, ot, decode->op[0].n, newv, newv); 1787 tcg_gen_movcond_tl(TCG_COND_EQ, dest, oldv, cmpv, newv, dest); 1788 } 1789 decode->op[0].unit = X86_OP_SKIP; 1790 } 1791 1792 /* Write RAX only if the cmpxchg fails. */ 1793 dest = gen_op_deposit_reg_v(s, ot, R_EAX, s->T0, oldv); 1794 tcg_gen_movcond_tl(TCG_COND_NE, dest, oldv, cmpv, s->T0, dest); 1795 1796 tcg_gen_mov_tl(s->cc_srcT, cmpv); 1797 tcg_gen_sub_tl(cmpv, cmpv, oldv); 1798 decode->cc_dst = cmpv; 1799 decode->cc_src = oldv; 1800 decode->cc_op = CC_OP_SUBB + ot; 1801} 1802 1803static void gen_CMPXCHG16B(DisasContext *s, X86DecodedInsn *decode) 1804{ 1805#ifdef TARGET_X86_64 1806 MemOp mop = MO_LE | MO_128 | MO_ALIGN; 1807 TCGv_i64 t0, t1; 1808 TCGv_i128 cmp, val; 1809 1810 cmp = tcg_temp_new_i128(); 1811 val = tcg_temp_new_i128(); 1812 tcg_gen_concat_i64_i128(cmp, cpu_regs[R_EAX], cpu_regs[R_EDX]); 1813 tcg_gen_concat_i64_i128(val, cpu_regs[R_EBX], cpu_regs[R_ECX]); 1814 1815 /* Only require atomic with LOCK; non-parallel handled in generator. */ 1816 if (s->prefix & PREFIX_LOCK) { 1817 tcg_gen_atomic_cmpxchg_i128(val, s->A0, cmp, val, s->mem_index, mop); 1818 } else { 1819 tcg_gen_nonatomic_cmpxchg_i128(val, s->A0, cmp, val, s->mem_index, mop); 1820 } 1821 1822 tcg_gen_extr_i128_i64(s->T0, s->T1, val); 1823 1824 /* Determine success after the fact. */ 1825 t0 = tcg_temp_new_i64(); 1826 t1 = tcg_temp_new_i64(); 1827 tcg_gen_xor_i64(t0, s->T0, cpu_regs[R_EAX]); 1828 tcg_gen_xor_i64(t1, s->T1, cpu_regs[R_EDX]); 1829 tcg_gen_or_i64(t0, t0, t1); 1830 1831 /* Update Z. */ 1832 gen_compute_eflags(s); 1833 tcg_gen_setcondi_i64(TCG_COND_EQ, t0, t0, 0); 1834 tcg_gen_deposit_tl(cpu_cc_src, cpu_cc_src, t0, ctz32(CC_Z), 1); 1835 1836 /* 1837 * Extract the result values for the register pair. We may do this 1838 * unconditionally, because on success (Z=1), the old value matches 1839 * the previous value in RDX:RAX. 1840 */ 1841 tcg_gen_mov_i64(cpu_regs[R_EAX], s->T0); 1842 tcg_gen_mov_i64(cpu_regs[R_EDX], s->T1); 1843#else 1844 abort(); 1845#endif 1846} 1847 1848static void gen_CMPXCHG8B(DisasContext *s, X86DecodedInsn *decode) 1849{ 1850 TCGv_i64 cmp, val, old; 1851 TCGv Z; 1852 1853 cmp = tcg_temp_new_i64(); 1854 val = tcg_temp_new_i64(); 1855 old = tcg_temp_new_i64(); 1856 1857 /* Construct the comparison values from the register pair. */ 1858 tcg_gen_concat_tl_i64(cmp, cpu_regs[R_EAX], cpu_regs[R_EDX]); 1859 tcg_gen_concat_tl_i64(val, cpu_regs[R_EBX], cpu_regs[R_ECX]); 1860 1861 /* Only require atomic with LOCK; non-parallel handled in generator. */ 1862 if (s->prefix & PREFIX_LOCK) { 1863 tcg_gen_atomic_cmpxchg_i64(old, s->A0, cmp, val, s->mem_index, MO_LEUQ); 1864 } else { 1865 tcg_gen_nonatomic_cmpxchg_i64(old, s->A0, cmp, val, 1866 s->mem_index, MO_LEUQ); 1867 } 1868 1869 /* Compute the required value of Z. */ 1870 tcg_gen_setcond_i64(TCG_COND_EQ, cmp, old, cmp); 1871 Z = tcg_temp_new(); 1872 tcg_gen_trunc_i64_tl(Z, cmp); 1873 1874 /* 1875 * Extract the result values for the register pair. 1876 * For 32-bit, we may do this unconditionally, because on success (Z=1), 1877 * the old value matches the previous value in EDX:EAX. For x86_64, 1878 * the store must be conditional, because we must leave the source 1879 * registers unchanged on success, and zero-extend the writeback 1880 * on failure (Z=0). 1881 */ 1882 if (TARGET_LONG_BITS == 32) { 1883 tcg_gen_extr_i64_tl(cpu_regs[R_EAX], cpu_regs[R_EDX], old); 1884 } else { 1885 TCGv zero = tcg_constant_tl(0); 1886 1887 tcg_gen_extr_i64_tl(s->T0, s->T1, old); 1888 tcg_gen_movcond_tl(TCG_COND_EQ, cpu_regs[R_EAX], Z, zero, 1889 s->T0, cpu_regs[R_EAX]); 1890 tcg_gen_movcond_tl(TCG_COND_EQ, cpu_regs[R_EDX], Z, zero, 1891 s->T1, cpu_regs[R_EDX]); 1892 } 1893 1894 /* Update Z. */ 1895 gen_compute_eflags(s); 1896 tcg_gen_deposit_tl(cpu_cc_src, cpu_cc_src, Z, ctz32(CC_Z), 1); 1897} 1898 1899static void gen_CPUID(DisasContext *s, X86DecodedInsn *decode) 1900{ 1901 gen_update_cc_op(s); 1902 gen_update_eip_cur(s); 1903 gen_helper_cpuid(tcg_env); 1904} 1905 1906static void gen_CRC32(DisasContext *s, X86DecodedInsn *decode) 1907{ 1908 MemOp ot = decode->op[2].ot; 1909 TCGv_i32 tmp = tcg_temp_new_i32(); 1910 1911 tcg_gen_trunc_tl_i32(tmp, s->T0); 1912 gen_helper_crc32(s->T0, tmp, s->T1, tcg_constant_i32(8 << ot)); 1913} 1914 1915static void gen_CVTPI2Px(DisasContext *s, X86DecodedInsn *decode) 1916{ 1917 gen_helper_enter_mmx(tcg_env); 1918 if (s->prefix & PREFIX_DATA) { 1919 gen_helper_cvtpi2pd(tcg_env, OP_PTR0, OP_PTR2); 1920 } else { 1921 gen_helper_cvtpi2ps(tcg_env, OP_PTR0, OP_PTR2); 1922 } 1923} 1924 1925static void gen_CVTPx2PI(DisasContext *s, X86DecodedInsn *decode) 1926{ 1927 gen_helper_enter_mmx(tcg_env); 1928 if (s->prefix & PREFIX_DATA) { 1929 gen_helper_cvtpd2pi(tcg_env, OP_PTR0, OP_PTR2); 1930 } else { 1931 gen_helper_cvtps2pi(tcg_env, OP_PTR0, OP_PTR2); 1932 } 1933} 1934 1935static void gen_CVTTPx2PI(DisasContext *s, X86DecodedInsn *decode) 1936{ 1937 gen_helper_enter_mmx(tcg_env); 1938 if (s->prefix & PREFIX_DATA) { 1939 gen_helper_cvttpd2pi(tcg_env, OP_PTR0, OP_PTR2); 1940 } else { 1941 gen_helper_cvttps2pi(tcg_env, OP_PTR0, OP_PTR2); 1942 } 1943} 1944 1945static void gen_CWD(DisasContext *s, X86DecodedInsn *decode) 1946{ 1947 int shift = 8 << decode->op[0].ot; 1948 1949 tcg_gen_sextract_tl(s->T0, s->T0, shift - 1, 1); 1950} 1951 1952static void gen_DAA(DisasContext *s, X86DecodedInsn *decode) 1953{ 1954 gen_update_cc_op(s); 1955 gen_helper_daa(tcg_env); 1956 assume_cc_op(s, CC_OP_EFLAGS); 1957} 1958 1959static void gen_DAS(DisasContext *s, X86DecodedInsn *decode) 1960{ 1961 gen_update_cc_op(s); 1962 gen_helper_das(tcg_env); 1963 assume_cc_op(s, CC_OP_EFLAGS); 1964} 1965 1966static void gen_DEC(DisasContext *s, X86DecodedInsn *decode) 1967{ 1968 MemOp ot = decode->op[1].ot; 1969 1970 tcg_gen_movi_tl(s->T1, -1); 1971 if (s->prefix & PREFIX_LOCK) { 1972 tcg_gen_atomic_add_fetch_tl(s->T0, s->A0, s->T1, 1973 s->mem_index, ot | MO_LE); 1974 } else { 1975 tcg_gen_add_tl(s->T0, s->T0, s->T1); 1976 } 1977 prepare_update_cc_incdec(decode, s, CC_OP_DECB + ot); 1978} 1979 1980static void gen_DIV(DisasContext *s, X86DecodedInsn *decode) 1981{ 1982 MemOp ot = decode->op[1].ot; 1983 1984 switch(ot) { 1985 case MO_8: 1986 gen_helper_divb_AL(tcg_env, s->T0); 1987 break; 1988 case MO_16: 1989 gen_helper_divw_AX(tcg_env, s->T0); 1990 break; 1991 default: 1992 case MO_32: 1993 gen_helper_divl_EAX(tcg_env, s->T0); 1994 break; 1995#ifdef TARGET_X86_64 1996 case MO_64: 1997 gen_helper_divq_EAX(tcg_env, s->T0); 1998 break; 1999#endif 2000 } 2001} 2002 2003static void gen_EMMS(DisasContext *s, X86DecodedInsn *decode) 2004{ 2005 gen_helper_emms(tcg_env); 2006} 2007 2008static void gen_ENTER(DisasContext *s, X86DecodedInsn *decode) 2009{ 2010 gen_enter(s, decode->op[1].imm, decode->op[2].imm); 2011} 2012 2013static void gen_EXTRQ_i(DisasContext *s, X86DecodedInsn *decode) 2014{ 2015 TCGv_i32 length = tcg_constant_i32(decode->immediate & 63); 2016 TCGv_i32 index = tcg_constant_i32((decode->immediate >> 8) & 63); 2017 2018 gen_helper_extrq_i(tcg_env, OP_PTR0, index, length); 2019} 2020 2021static void gen_EXTRQ_r(DisasContext *s, X86DecodedInsn *decode) 2022{ 2023 gen_helper_extrq_r(tcg_env, OP_PTR0, OP_PTR2); 2024} 2025 2026static void gen_FXRSTOR(DisasContext *s, X86DecodedInsn *decode) 2027{ 2028 if ((s->flags & HF_EM_MASK) || (s->flags & HF_TS_MASK)) { 2029 gen_NM_exception(s); 2030 } else { 2031 gen_helper_fxrstor(tcg_env, s->A0); 2032 } 2033} 2034 2035static void gen_FXSAVE(DisasContext *s, X86DecodedInsn *decode) 2036{ 2037 if ((s->flags & HF_EM_MASK) || (s->flags & HF_TS_MASK)) { 2038 gen_NM_exception(s); 2039 } else { 2040 gen_helper_fxsave(tcg_env, s->A0); 2041 } 2042} 2043 2044static void gen_HLT(DisasContext *s, X86DecodedInsn *decode) 2045{ 2046#ifdef CONFIG_SYSTEM_ONLY 2047 gen_update_cc_op(s); 2048 gen_update_eip_next(s); 2049 gen_helper_hlt(tcg_env); 2050 s->base.is_jmp = DISAS_NORETURN; 2051#endif 2052} 2053 2054static void gen_IDIV(DisasContext *s, X86DecodedInsn *decode) 2055{ 2056 MemOp ot = decode->op[1].ot; 2057 2058 switch(ot) { 2059 case MO_8: 2060 gen_helper_idivb_AL(tcg_env, s->T0); 2061 break; 2062 case MO_16: 2063 gen_helper_idivw_AX(tcg_env, s->T0); 2064 break; 2065 default: 2066 case MO_32: 2067 gen_helper_idivl_EAX(tcg_env, s->T0); 2068 break; 2069#ifdef TARGET_X86_64 2070 case MO_64: 2071 gen_helper_idivq_EAX(tcg_env, s->T0); 2072 break; 2073#endif 2074 } 2075} 2076 2077static void gen_IMUL3(DisasContext *s, X86DecodedInsn *decode) 2078{ 2079 MemOp ot = decode->op[0].ot; 2080 TCGv cc_src_rhs; 2081 2082 switch (ot) { 2083 case MO_16: 2084 /* s->T0 already sign-extended */ 2085 tcg_gen_ext16s_tl(s->T1, s->T1); 2086 tcg_gen_mul_tl(s->T0, s->T0, s->T1); 2087 /* Compare the full result to the extension of the truncated result. */ 2088 tcg_gen_ext16s_tl(s->T1, s->T0); 2089 cc_src_rhs = s->T0; 2090 break; 2091 2092 case MO_32: 2093#ifdef TARGET_X86_64 2094 if (TCG_TARGET_REG_BITS == 64) { 2095 /* 2096 * This produces fewer TCG ops, and better code if flags are needed, 2097 * but it requires a 64-bit multiply even if they are not. Use it 2098 * only if the target has 64-bits registers. 2099 * 2100 * s->T0 is already sign-extended. 2101 */ 2102 tcg_gen_ext32s_tl(s->T1, s->T1); 2103 tcg_gen_mul_tl(s->T0, s->T0, s->T1); 2104 /* Compare the full result to the extension of the truncated result. */ 2105 tcg_gen_ext32s_tl(s->T1, s->T0); 2106 cc_src_rhs = s->T0; 2107 } else { 2108 /* Variant that only needs a 32-bit widening multiply. */ 2109 TCGv_i32 hi = tcg_temp_new_i32(); 2110 TCGv_i32 lo = tcg_temp_new_i32(); 2111 tcg_gen_trunc_tl_i32(lo, s->T0); 2112 tcg_gen_trunc_tl_i32(hi, s->T1); 2113 tcg_gen_muls2_i32(lo, hi, lo, hi); 2114 tcg_gen_extu_i32_tl(s->T0, lo); 2115 2116 cc_src_rhs = tcg_temp_new(); 2117 tcg_gen_extu_i32_tl(cc_src_rhs, hi); 2118 /* Compare the high part to the sign bit of the truncated result */ 2119 tcg_gen_sari_i32(lo, lo, 31); 2120 tcg_gen_extu_i32_tl(s->T1, lo); 2121 } 2122 break; 2123 2124 case MO_64: 2125#endif 2126 cc_src_rhs = tcg_temp_new(); 2127 tcg_gen_muls2_tl(s->T0, cc_src_rhs, s->T0, s->T1); 2128 /* Compare the high part to the sign bit of the truncated result */ 2129 tcg_gen_sari_tl(s->T1, s->T0, TARGET_LONG_BITS - 1); 2130 break; 2131 2132 default: 2133 g_assert_not_reached(); 2134 } 2135 2136 tcg_gen_sub_tl(s->T1, s->T1, cc_src_rhs); 2137 prepare_update2_cc(decode, s, CC_OP_MULB + ot); 2138} 2139 2140static void gen_IMUL(DisasContext *s, X86DecodedInsn *decode) 2141{ 2142 MemOp ot = decode->op[1].ot; 2143 TCGv cc_src_rhs; 2144 2145 switch (ot) { 2146 case MO_8: 2147 /* s->T0 already sign-extended */ 2148 tcg_gen_ext8s_tl(s->T1, s->T1); 2149 tcg_gen_mul_tl(s->T0, s->T0, s->T1); 2150 gen_op_mov_reg_v(s, MO_16, R_EAX, s->T0); 2151 /* Compare the full result to the extension of the truncated result. */ 2152 tcg_gen_ext8s_tl(s->T1, s->T0); 2153 cc_src_rhs = s->T0; 2154 break; 2155 2156 case MO_16: 2157 /* s->T0 already sign-extended */ 2158 tcg_gen_ext16s_tl(s->T1, s->T1); 2159 tcg_gen_mul_tl(s->T0, s->T0, s->T1); 2160 gen_op_mov_reg_v(s, MO_16, R_EAX, s->T0); 2161 tcg_gen_shri_tl(s->T1, s->T0, 16); 2162 gen_op_mov_reg_v(s, MO_16, R_EDX, s->T1); 2163 /* Compare the full result to the extension of the truncated result. */ 2164 tcg_gen_ext16s_tl(s->T1, s->T0); 2165 cc_src_rhs = s->T0; 2166 break; 2167 2168 case MO_32: 2169#ifdef TARGET_X86_64 2170 /* s->T0 already sign-extended */ 2171 tcg_gen_ext32s_tl(s->T1, s->T1); 2172 tcg_gen_mul_tl(s->T0, s->T0, s->T1); 2173 tcg_gen_ext32u_tl(cpu_regs[R_EAX], s->T0); 2174 tcg_gen_shri_tl(cpu_regs[R_EDX], s->T0, 32); 2175 /* Compare the full result to the extension of the truncated result. */ 2176 tcg_gen_ext32s_tl(s->T1, s->T0); 2177 cc_src_rhs = s->T0; 2178 break; 2179 2180 case MO_64: 2181#endif 2182 tcg_gen_muls2_tl(s->T0, cpu_regs[R_EDX], s->T0, s->T1); 2183 tcg_gen_mov_tl(cpu_regs[R_EAX], s->T0); 2184 2185 /* Compare the high part to the sign bit of the truncated result */ 2186 tcg_gen_negsetcondi_tl(TCG_COND_LT, s->T1, s->T0, 0); 2187 cc_src_rhs = cpu_regs[R_EDX]; 2188 break; 2189 2190 default: 2191 g_assert_not_reached(); 2192 } 2193 2194 tcg_gen_sub_tl(s->T1, s->T1, cc_src_rhs); 2195 prepare_update2_cc(decode, s, CC_OP_MULB + ot); 2196} 2197 2198static void gen_IN(DisasContext *s, X86DecodedInsn *decode) 2199{ 2200 MemOp ot = decode->op[0].ot; 2201 TCGv_i32 port = tcg_temp_new_i32(); 2202 2203 tcg_gen_trunc_tl_i32(port, s->T0); 2204 tcg_gen_ext16u_i32(port, port); 2205 if (!gen_check_io(s, ot, port, SVM_IOIO_TYPE_MASK)) { 2206 return; 2207 } 2208 translator_io_start(&s->base); 2209 gen_helper_in_func(ot, s->T0, port); 2210 gen_writeback(s, decode, 0, s->T0); 2211 gen_bpt_io(s, port, ot); 2212} 2213 2214static void gen_INC(DisasContext *s, X86DecodedInsn *decode) 2215{ 2216 MemOp ot = decode->op[1].ot; 2217 2218 tcg_gen_movi_tl(s->T1, 1); 2219 if (s->prefix & PREFIX_LOCK) { 2220 tcg_gen_atomic_add_fetch_tl(s->T0, s->A0, s->T1, 2221 s->mem_index, ot | MO_LE); 2222 } else { 2223 tcg_gen_add_tl(s->T0, s->T0, s->T1); 2224 } 2225 prepare_update_cc_incdec(decode, s, CC_OP_INCB + ot); 2226} 2227 2228static void gen_INS(DisasContext *s, X86DecodedInsn *decode) 2229{ 2230 MemOp ot = decode->op[1].ot; 2231 TCGv_i32 port = tcg_temp_new_i32(); 2232 2233 tcg_gen_trunc_tl_i32(port, s->T1); 2234 tcg_gen_ext16u_i32(port, port); 2235 if (!gen_check_io(s, ot, port, 2236 SVM_IOIO_TYPE_MASK | SVM_IOIO_STR_MASK)) { 2237 return; 2238 } 2239 2240 translator_io_start(&s->base); 2241 gen_repz(s, ot, gen_ins); 2242} 2243 2244static void gen_INSERTQ_i(DisasContext *s, X86DecodedInsn *decode) 2245{ 2246 TCGv_i32 length = tcg_constant_i32(decode->immediate & 63); 2247 TCGv_i32 index = tcg_constant_i32((decode->immediate >> 8) & 63); 2248 2249 gen_helper_insertq_i(tcg_env, OP_PTR0, OP_PTR1, index, length); 2250} 2251 2252static void gen_INSERTQ_r(DisasContext *s, X86DecodedInsn *decode) 2253{ 2254 gen_helper_insertq_r(tcg_env, OP_PTR0, OP_PTR2); 2255} 2256 2257static void gen_INT(DisasContext *s, X86DecodedInsn *decode) 2258{ 2259 gen_interrupt(s, decode->immediate); 2260} 2261 2262static void gen_INT1(DisasContext *s, X86DecodedInsn *decode) 2263{ 2264 gen_update_cc_op(s); 2265 gen_update_eip_next(s); 2266 gen_helper_icebp(tcg_env); 2267 s->base.is_jmp = DISAS_NORETURN; 2268} 2269 2270static void gen_INT3(DisasContext *s, X86DecodedInsn *decode) 2271{ 2272 gen_interrupt(s, EXCP03_INT3); 2273} 2274 2275static void gen_INTO(DisasContext *s, X86DecodedInsn *decode) 2276{ 2277 gen_update_cc_op(s); 2278 gen_update_eip_cur(s); 2279 gen_helper_into(tcg_env, cur_insn_len_i32(s)); 2280} 2281 2282static void gen_IRET(DisasContext *s, X86DecodedInsn *decode) 2283{ 2284 if (!PE(s) || VM86(s)) { 2285 gen_helper_iret_real(tcg_env, tcg_constant_i32(s->dflag - 1)); 2286 } else { 2287 gen_helper_iret_protected(tcg_env, tcg_constant_i32(s->dflag - 1), 2288 eip_next_i32(s)); 2289 } 2290 assume_cc_op(s, CC_OP_EFLAGS); 2291 s->base.is_jmp = DISAS_EOB_ONLY; 2292} 2293 2294static void gen_Jcc(DisasContext *s, X86DecodedInsn *decode) 2295{ 2296 TCGLabel *taken = gen_new_label(); 2297 2298 gen_bnd_jmp(s); 2299 gen_jcc(s, decode->b & 0xf, taken); 2300 gen_conditional_jump_labels(s, decode->immediate, NULL, taken); 2301} 2302 2303static void gen_JCXZ(DisasContext *s, X86DecodedInsn *decode) 2304{ 2305 TCGLabel *taken = gen_new_label(); 2306 2307 gen_update_cc_op(s); 2308 gen_op_jz_ecx(s, taken); 2309 gen_conditional_jump_labels(s, decode->immediate, NULL, taken); 2310} 2311 2312static void gen_JMP(DisasContext *s, X86DecodedInsn *decode) 2313{ 2314 gen_update_cc_op(s); 2315 gen_jmp_rel(s, s->dflag, decode->immediate, 0); 2316} 2317 2318static void gen_JMP_m(DisasContext *s, X86DecodedInsn *decode) 2319{ 2320 gen_op_jmp_v(s, s->T0); 2321 gen_bnd_jmp(s); 2322 s->base.is_jmp = DISAS_JUMP; 2323} 2324 2325static void gen_JMPF(DisasContext *s, X86DecodedInsn *decode) 2326{ 2327 gen_far_jmp(s); 2328} 2329 2330static void gen_JMPF_m(DisasContext *s, X86DecodedInsn *decode) 2331{ 2332 MemOp ot = decode->op[1].ot; 2333 2334 gen_op_ld_v(s, ot, s->T0, s->A0); 2335 gen_add_A0_im(s, 1 << ot); 2336 gen_op_ld_v(s, MO_16, s->T1, s->A0); 2337 gen_far_jmp(s); 2338} 2339 2340static void gen_LAHF(DisasContext *s, X86DecodedInsn *decode) 2341{ 2342 if (CODE64(s) && !(s->cpuid_ext3_features & CPUID_EXT3_LAHF_LM)) { 2343 return gen_illegal_opcode(s); 2344 } 2345 gen_compute_eflags(s); 2346 /* Note: gen_compute_eflags() only gives the condition codes */ 2347 tcg_gen_ori_tl(s->T0, cpu_cc_src, 0x02); 2348 tcg_gen_deposit_tl(cpu_regs[R_EAX], cpu_regs[R_EAX], s->T0, 8, 8); 2349} 2350 2351static void gen_LAR(DisasContext *s, X86DecodedInsn *decode) 2352{ 2353 MemOp ot = decode->op[0].ot; 2354 TCGv result = tcg_temp_new(); 2355 TCGv dest; 2356 2357 gen_compute_eflags(s); 2358 gen_update_cc_op(s); 2359 gen_helper_lar(result, tcg_env, s->T0); 2360 2361 /* Perform writeback here to skip it if ZF=0. */ 2362 decode->op[0].unit = X86_OP_SKIP; 2363 dest = gen_op_deposit_reg_v(s, ot, decode->op[0].n, result, result); 2364 tcg_gen_movcond_tl(TCG_COND_TSTNE, dest, cpu_cc_src, tcg_constant_tl(CC_Z), 2365 result, dest); 2366} 2367 2368static void gen_LDMXCSR(DisasContext *s, X86DecodedInsn *decode) 2369{ 2370 TCGv_i32 tmp = tcg_temp_new_i32(); 2371 2372 tcg_gen_trunc_tl_i32(tmp, s->T0); 2373 gen_helper_ldmxcsr(tcg_env, tmp); 2374} 2375 2376static void gen_lxx_seg(DisasContext *s, X86DecodedInsn *decode, int seg) 2377{ 2378 MemOp ot = decode->op[0].ot; 2379 2380 /* Offset already in s->T0. */ 2381 gen_add_A0_im(s, 1 << ot); 2382 gen_op_ld_v(s, MO_16, s->T1, s->A0); 2383 2384 /* load the segment here to handle exceptions properly */ 2385 gen_movl_seg(s, seg, s->T1, false); 2386} 2387 2388static void gen_LDS(DisasContext *s, X86DecodedInsn *decode) 2389{ 2390 gen_lxx_seg(s, decode, R_DS); 2391} 2392 2393static void gen_LEA(DisasContext *s, X86DecodedInsn *decode) 2394{ 2395 TCGv ea = gen_lea_modrm_1(s, decode->mem, false); 2396 gen_lea_v_seg_dest(s, s->aflag, s->T0, ea, -1, -1); 2397} 2398 2399static void gen_LEAVE(DisasContext *s, X86DecodedInsn *decode) 2400{ 2401 gen_leave(s); 2402} 2403 2404static void gen_LES(DisasContext *s, X86DecodedInsn *decode) 2405{ 2406 gen_lxx_seg(s, decode, R_ES); 2407} 2408 2409static void gen_LFENCE(DisasContext *s, X86DecodedInsn *decode) 2410{ 2411 tcg_gen_mb(TCG_MO_LD_LD | TCG_BAR_SC); 2412} 2413 2414static void gen_LFS(DisasContext *s, X86DecodedInsn *decode) 2415{ 2416 gen_lxx_seg(s, decode, R_FS); 2417} 2418 2419static void gen_LGS(DisasContext *s, X86DecodedInsn *decode) 2420{ 2421 gen_lxx_seg(s, decode, R_GS); 2422} 2423 2424static void gen_LODS(DisasContext *s, X86DecodedInsn *decode) 2425{ 2426 MemOp ot = decode->op[1].ot; 2427 gen_repz(s, ot, gen_lods); 2428} 2429 2430static void gen_LOOP(DisasContext *s, X86DecodedInsn *decode) 2431{ 2432 TCGLabel *taken = gen_new_label(); 2433 2434 gen_update_cc_op(s); 2435 gen_op_add_reg_im(s, s->aflag, R_ECX, -1); 2436 gen_op_jnz_ecx(s, taken); 2437 gen_conditional_jump_labels(s, decode->immediate, NULL, taken); 2438} 2439 2440static void gen_LOOPE(DisasContext *s, X86DecodedInsn *decode) 2441{ 2442 TCGLabel *taken = gen_new_label(); 2443 TCGLabel *not_taken = gen_new_label(); 2444 2445 gen_update_cc_op(s); 2446 gen_op_add_reg_im(s, s->aflag, R_ECX, -1); 2447 gen_op_jz_ecx(s, not_taken); 2448 gen_jcc(s, (JCC_Z << 1), taken); /* jz taken */ 2449 gen_conditional_jump_labels(s, decode->immediate, not_taken, taken); 2450} 2451 2452static void gen_LOOPNE(DisasContext *s, X86DecodedInsn *decode) 2453{ 2454 TCGLabel *taken = gen_new_label(); 2455 TCGLabel *not_taken = gen_new_label(); 2456 2457 gen_update_cc_op(s); 2458 gen_op_add_reg_im(s, s->aflag, R_ECX, -1); 2459 gen_op_jz_ecx(s, not_taken); 2460 gen_jcc(s, (JCC_Z << 1) | 1, taken); /* jnz taken */ 2461 gen_conditional_jump_labels(s, decode->immediate, not_taken, taken); 2462} 2463 2464static void gen_LSL(DisasContext *s, X86DecodedInsn *decode) 2465{ 2466 MemOp ot = decode->op[0].ot; 2467 TCGv result = tcg_temp_new(); 2468 TCGv dest; 2469 2470 gen_compute_eflags(s); 2471 gen_update_cc_op(s); 2472 gen_helper_lsl(result, tcg_env, s->T0); 2473 2474 /* Perform writeback here to skip it if ZF=0. */ 2475 decode->op[0].unit = X86_OP_SKIP; 2476 dest = gen_op_deposit_reg_v(s, ot, decode->op[0].n, result, result); 2477 tcg_gen_movcond_tl(TCG_COND_TSTNE, dest, cpu_cc_src, tcg_constant_tl(CC_Z), 2478 result, dest); 2479} 2480 2481static void gen_LSS(DisasContext *s, X86DecodedInsn *decode) 2482{ 2483 gen_lxx_seg(s, decode, R_SS); 2484} 2485 2486static void gen_LZCNT(DisasContext *s, X86DecodedInsn *decode) 2487{ 2488 MemOp ot = decode->op[0].ot; 2489 2490 /* C bit (cc_src) is defined related to the input. */ 2491 decode->cc_src = tcg_temp_new(); 2492 decode->cc_dst = s->T0; 2493 decode->cc_op = CC_OP_BMILGB + ot; 2494 tcg_gen_mov_tl(decode->cc_src, s->T0); 2495 2496 /* 2497 * Reduce the target_ulong result by the number of zeros that 2498 * we expect to find at the top. 2499 */ 2500 tcg_gen_clzi_tl(s->T0, s->T0, TARGET_LONG_BITS); 2501 tcg_gen_subi_tl(s->T0, s->T0, TARGET_LONG_BITS - (8 << ot)); 2502} 2503 2504static void gen_MFENCE(DisasContext *s, X86DecodedInsn *decode) 2505{ 2506 tcg_gen_mb(TCG_MO_ALL | TCG_BAR_SC); 2507} 2508 2509static void gen_MOV(DisasContext *s, X86DecodedInsn *decode) 2510{ 2511 /* nothing to do! */ 2512} 2513#define gen_NOP gen_MOV 2514 2515static void gen_MASKMOV(DisasContext *s, X86DecodedInsn *decode) 2516{ 2517 gen_lea_v_seg(s, cpu_regs[R_EDI], R_DS, s->override); 2518 2519 if (s->prefix & PREFIX_DATA) { 2520 gen_helper_maskmov_xmm(tcg_env, OP_PTR1, OP_PTR2, s->A0); 2521 } else { 2522 gen_helper_maskmov_mmx(tcg_env, OP_PTR1, OP_PTR2, s->A0); 2523 } 2524} 2525 2526static void gen_MOVBE(DisasContext *s, X86DecodedInsn *decode) 2527{ 2528 MemOp ot = decode->op[0].ot; 2529 2530 /* M operand type does not load/store */ 2531 if (decode->e.op0 == X86_TYPE_M) { 2532 tcg_gen_qemu_st_tl(s->T0, s->A0, s->mem_index, ot | MO_BE); 2533 } else { 2534 tcg_gen_qemu_ld_tl(s->T0, s->A0, s->mem_index, ot | MO_BE); 2535 } 2536} 2537 2538static void gen_MOVD_from(DisasContext *s, X86DecodedInsn *decode) 2539{ 2540 MemOp ot = decode->op[2].ot; 2541 2542 switch (ot) { 2543 case MO_32: 2544#ifdef TARGET_X86_64 2545 tcg_gen_ld32u_tl(s->T0, tcg_env, decode->op[2].offset); 2546 break; 2547 case MO_64: 2548#endif 2549 tcg_gen_ld_tl(s->T0, tcg_env, decode->op[2].offset); 2550 break; 2551 default: 2552 abort(); 2553 } 2554} 2555 2556static void gen_MOVD_to(DisasContext *s, X86DecodedInsn *decode) 2557{ 2558 MemOp ot = decode->op[2].ot; 2559 int vec_len = vector_len(s, decode); 2560 int lo_ofs = vector_elem_offset(&decode->op[0], ot, 0); 2561 2562 tcg_gen_gvec_dup_imm(MO_64, decode->op[0].offset, vec_len, vec_len, 0); 2563 2564 switch (ot) { 2565 case MO_32: 2566#ifdef TARGET_X86_64 2567 tcg_gen_st32_tl(s->T1, tcg_env, lo_ofs); 2568 break; 2569 case MO_64: 2570#endif 2571 tcg_gen_st_tl(s->T1, tcg_env, lo_ofs); 2572 break; 2573 default: 2574 g_assert_not_reached(); 2575 } 2576} 2577 2578static void gen_MOVDQ(DisasContext *s, X86DecodedInsn *decode) 2579{ 2580 gen_store_sse(s, decode, decode->op[2].offset); 2581} 2582 2583static void gen_MOVMSK(DisasContext *s, X86DecodedInsn *decode) 2584{ 2585 typeof(gen_helper_movmskps_ymm) *ps, *pd, *fn; 2586 TCGv_i32 tmp = tcg_temp_new_i32(); 2587 2588 ps = s->vex_l ? gen_helper_movmskps_ymm : gen_helper_movmskps_xmm; 2589 pd = s->vex_l ? gen_helper_movmskpd_ymm : gen_helper_movmskpd_xmm; 2590 fn = s->prefix & PREFIX_DATA ? pd : ps; 2591 fn(tmp, tcg_env, OP_PTR2); 2592 tcg_gen_extu_i32_tl(s->T0, tmp); 2593} 2594 2595static void gen_MOVQ(DisasContext *s, X86DecodedInsn *decode) 2596{ 2597 int vec_len = vector_len(s, decode); 2598 int lo_ofs = vector_elem_offset(&decode->op[0], MO_64, 0); 2599 2600 tcg_gen_ld_i64(s->tmp1_i64, tcg_env, decode->op[2].offset); 2601 if (decode->op[0].has_ea) { 2602 tcg_gen_qemu_st_i64(s->tmp1_i64, s->A0, s->mem_index, MO_LEUQ); 2603 } else { 2604 /* 2605 * tcg_gen_gvec_dup_i64(MO_64, op0.offset, 8, vec_len, s->tmp1_64) would 2606 * seem to work, but it does not on big-endian platforms; the cleared parts 2607 * are always at higher addresses, but cross-endian emulation inverts the 2608 * byte order so that the cleared parts need to be at *lower* addresses. 2609 * Because oprsz is 8, we see this here even for SSE; but more in general, 2610 * it disqualifies using oprsz < maxsz to emulate VEX128. 2611 */ 2612 tcg_gen_gvec_dup_imm(MO_64, decode->op[0].offset, vec_len, vec_len, 0); 2613 tcg_gen_st_i64(s->tmp1_i64, tcg_env, lo_ofs); 2614 } 2615} 2616 2617static void gen_MOVq_dq(DisasContext *s, X86DecodedInsn *decode) 2618{ 2619 gen_helper_enter_mmx(tcg_env); 2620 /* Otherwise the same as any other movq. */ 2621 return gen_MOVQ(s, decode); 2622} 2623 2624static void gen_MOVS(DisasContext *s, X86DecodedInsn *decode) 2625{ 2626 MemOp ot = decode->op[2].ot; 2627 gen_repz(s, ot, gen_movs); 2628} 2629 2630static void gen_MUL(DisasContext *s, X86DecodedInsn *decode) 2631{ 2632 MemOp ot = decode->op[1].ot; 2633 2634 switch (ot) { 2635 case MO_8: 2636 /* s->T0 already zero-extended */ 2637 tcg_gen_ext8u_tl(s->T1, s->T1); 2638 tcg_gen_mul_tl(s->T0, s->T0, s->T1); 2639 gen_op_mov_reg_v(s, MO_16, R_EAX, s->T0); 2640 tcg_gen_andi_tl(s->T1, s->T0, 0xff00); 2641 decode->cc_dst = s->T0; 2642 decode->cc_src = s->T1; 2643 break; 2644 2645 case MO_16: 2646 /* s->T0 already zero-extended */ 2647 tcg_gen_ext16u_tl(s->T1, s->T1); 2648 tcg_gen_mul_tl(s->T0, s->T0, s->T1); 2649 gen_op_mov_reg_v(s, MO_16, R_EAX, s->T0); 2650 tcg_gen_shri_tl(s->T1, s->T0, 16); 2651 gen_op_mov_reg_v(s, MO_16, R_EDX, s->T1); 2652 decode->cc_dst = s->T0; 2653 decode->cc_src = s->T1; 2654 break; 2655 2656 case MO_32: 2657#ifdef TARGET_X86_64 2658 /* s->T0 already zero-extended */ 2659 tcg_gen_ext32u_tl(s->T1, s->T1); 2660 tcg_gen_mul_tl(s->T0, s->T0, s->T1); 2661 tcg_gen_ext32u_tl(cpu_regs[R_EAX], s->T0); 2662 tcg_gen_shri_tl(cpu_regs[R_EDX], s->T0, 32); 2663 decode->cc_dst = cpu_regs[R_EAX]; 2664 decode->cc_src = cpu_regs[R_EDX]; 2665 break; 2666 2667 case MO_64: 2668#endif 2669 tcg_gen_mulu2_tl(cpu_regs[R_EAX], cpu_regs[R_EDX], s->T0, s->T1); 2670 decode->cc_dst = cpu_regs[R_EAX]; 2671 decode->cc_src = cpu_regs[R_EDX]; 2672 break; 2673 2674 default: 2675 g_assert_not_reached(); 2676 } 2677 2678 decode->cc_op = CC_OP_MULB + ot; 2679} 2680 2681static void gen_MULX(DisasContext *s, X86DecodedInsn *decode) 2682{ 2683 MemOp ot = decode->op[0].ot; 2684 2685 /* low part of result in VEX.vvvv, high in MODRM */ 2686 switch (ot) { 2687 case MO_32: 2688#ifdef TARGET_X86_64 2689 { 2690 TCGv_i32 t0 = tcg_temp_new_i32(); 2691 TCGv_i32 t1 = tcg_temp_new_i32(); 2692 2693 tcg_gen_trunc_tl_i32(t0, s->T0); 2694 tcg_gen_trunc_tl_i32(t1, s->T1); 2695 tcg_gen_mulu2_i32(t0, t1, t0, t1); 2696 tcg_gen_extu_i32_tl(cpu_regs[s->vex_v], t0); 2697 tcg_gen_extu_i32_tl(s->T0, t1); 2698 break; 2699 } 2700 2701 case MO_64: 2702#endif 2703 tcg_gen_mulu2_tl(cpu_regs[s->vex_v], s->T0, s->T0, s->T1); 2704 break; 2705 2706 default: 2707 g_assert_not_reached(); 2708 } 2709} 2710 2711static void gen_NEG(DisasContext *s, X86DecodedInsn *decode) 2712{ 2713 MemOp ot = decode->op[0].ot; 2714 TCGv oldv = tcg_temp_new(); 2715 2716 if (s->prefix & PREFIX_LOCK) { 2717 TCGv newv = tcg_temp_new(); 2718 TCGv cmpv = tcg_temp_new(); 2719 TCGLabel *label1 = gen_new_label(); 2720 2721 gen_set_label(label1); 2722 gen_op_ld_v(s, ot, oldv, s->A0); 2723 tcg_gen_neg_tl(newv, oldv); 2724 tcg_gen_atomic_cmpxchg_tl(cmpv, s->A0, oldv, newv, 2725 s->mem_index, ot | MO_LE); 2726 tcg_gen_brcond_tl(TCG_COND_NE, oldv, cmpv, label1); 2727 } else { 2728 tcg_gen_mov_tl(oldv, s->T0); 2729 } 2730 tcg_gen_neg_tl(s->T0, oldv); 2731 2732 decode->cc_dst = s->T0; 2733 decode->cc_src = oldv; 2734 tcg_gen_movi_tl(s->cc_srcT, 0); 2735 decode->cc_op = CC_OP_SUBB + ot; 2736} 2737 2738static void gen_NOT(DisasContext *s, X86DecodedInsn *decode) 2739{ 2740 MemOp ot = decode->op[0].ot; 2741 2742 if (s->prefix & PREFIX_LOCK) { 2743 tcg_gen_movi_tl(s->T0, ~0); 2744 tcg_gen_atomic_xor_fetch_tl(s->T0, s->A0, s->T0, 2745 s->mem_index, ot | MO_LE); 2746 } else { 2747 tcg_gen_not_tl(s->T0, s->T0); 2748 } 2749} 2750 2751static void gen_OR(DisasContext *s, X86DecodedInsn *decode) 2752{ 2753 MemOp ot = decode->op[1].ot; 2754 2755 if (s->prefix & PREFIX_LOCK) { 2756 tcg_gen_atomic_or_fetch_tl(s->T0, s->A0, s->T1, 2757 s->mem_index, ot | MO_LE); 2758 } else { 2759 tcg_gen_or_tl(s->T0, s->T0, s->T1); 2760 } 2761 prepare_update1_cc(decode, s, CC_OP_LOGICB + ot); 2762} 2763 2764static void gen_OUT(DisasContext *s, X86DecodedInsn *decode) 2765{ 2766 MemOp ot = decode->op[1].ot; 2767 TCGv_i32 port = tcg_temp_new_i32(); 2768 TCGv_i32 value = tcg_temp_new_i32(); 2769 2770 tcg_gen_trunc_tl_i32(port, s->T1); 2771 tcg_gen_ext16u_i32(port, port); 2772 if (!gen_check_io(s, ot, port, 0)) { 2773 return; 2774 } 2775 tcg_gen_trunc_tl_i32(value, s->T0); 2776 translator_io_start(&s->base); 2777 gen_helper_out_func(ot, port, value); 2778 gen_bpt_io(s, port, ot); 2779} 2780 2781static void gen_OUTS(DisasContext *s, X86DecodedInsn *decode) 2782{ 2783 MemOp ot = decode->op[1].ot; 2784 TCGv_i32 port = tcg_temp_new_i32(); 2785 2786 tcg_gen_trunc_tl_i32(port, s->T1); 2787 tcg_gen_ext16u_i32(port, port); 2788 if (!gen_check_io(s, ot, port, SVM_IOIO_STR_MASK)) { 2789 return; 2790 } 2791 2792 translator_io_start(&s->base); 2793 gen_repz(s, ot, gen_outs); 2794} 2795 2796static void gen_PALIGNR(DisasContext *s, X86DecodedInsn *decode) 2797{ 2798 TCGv_i32 imm = tcg_constant8u_i32(decode->immediate); 2799 if (!(s->prefix & PREFIX_DATA)) { 2800 gen_helper_palignr_mmx(tcg_env, OP_PTR0, OP_PTR1, OP_PTR2, imm); 2801 } else if (!s->vex_l) { 2802 gen_helper_palignr_xmm(tcg_env, OP_PTR0, OP_PTR1, OP_PTR2, imm); 2803 } else { 2804 gen_helper_palignr_ymm(tcg_env, OP_PTR0, OP_PTR1, OP_PTR2, imm); 2805 } 2806} 2807 2808static void gen_PANDN(DisasContext *s, X86DecodedInsn *decode) 2809{ 2810 int vec_len = vector_len(s, decode); 2811 2812 /* Careful, operand order is reversed! */ 2813 tcg_gen_gvec_andc(MO_64, 2814 decode->op[0].offset, decode->op[2].offset, 2815 decode->op[1].offset, vec_len, vec_len); 2816} 2817 2818static void gen_PAUSE(DisasContext *s, X86DecodedInsn *decode) 2819{ 2820 gen_update_cc_op(s); 2821 gen_update_eip_next(s); 2822 gen_helper_pause(tcg_env); 2823 s->base.is_jmp = DISAS_NORETURN; 2824} 2825 2826static void gen_PCMPESTRI(DisasContext *s, X86DecodedInsn *decode) 2827{ 2828 TCGv_i32 imm = tcg_constant8u_i32(decode->immediate); 2829 gen_helper_pcmpestri_xmm(tcg_env, OP_PTR1, OP_PTR2, imm); 2830 assume_cc_op(s, CC_OP_EFLAGS); 2831} 2832 2833static void gen_PCMPESTRM(DisasContext *s, X86DecodedInsn *decode) 2834{ 2835 TCGv_i32 imm = tcg_constant8u_i32(decode->immediate); 2836 gen_helper_pcmpestrm_xmm(tcg_env, OP_PTR1, OP_PTR2, imm); 2837 assume_cc_op(s, CC_OP_EFLAGS); 2838 if ((s->prefix & PREFIX_VEX) && !s->vex_l) { 2839 tcg_gen_gvec_dup_imm(MO_64, offsetof(CPUX86State, xmm_regs[0].ZMM_X(1)), 2840 16, 16, 0); 2841 } 2842} 2843 2844static void gen_PCMPISTRI(DisasContext *s, X86DecodedInsn *decode) 2845{ 2846 TCGv_i32 imm = tcg_constant8u_i32(decode->immediate); 2847 gen_helper_pcmpistri_xmm(tcg_env, OP_PTR1, OP_PTR2, imm); 2848 assume_cc_op(s, CC_OP_EFLAGS); 2849} 2850 2851static void gen_PCMPISTRM(DisasContext *s, X86DecodedInsn *decode) 2852{ 2853 TCGv_i32 imm = tcg_constant8u_i32(decode->immediate); 2854 gen_helper_pcmpistrm_xmm(tcg_env, OP_PTR1, OP_PTR2, imm); 2855 assume_cc_op(s, CC_OP_EFLAGS); 2856 if ((s->prefix & PREFIX_VEX) && !s->vex_l) { 2857 tcg_gen_gvec_dup_imm(MO_64, offsetof(CPUX86State, xmm_regs[0].ZMM_X(1)), 2858 16, 16, 0); 2859 } 2860} 2861 2862static void gen_PDEP(DisasContext *s, X86DecodedInsn *decode) 2863{ 2864 gen_helper_pdep(s->T0, s->T0, s->T1); 2865} 2866 2867static void gen_PEXT(DisasContext *s, X86DecodedInsn *decode) 2868{ 2869 gen_helper_pext(s->T0, s->T0, s->T1); 2870} 2871 2872static inline void gen_pextr(DisasContext *s, X86DecodedInsn *decode, MemOp ot) 2873{ 2874 int vec_len = vector_len(s, decode); 2875 int mask = (vec_len >> ot) - 1; 2876 int val = decode->immediate & mask; 2877 2878 switch (ot) { 2879 case MO_8: 2880 tcg_gen_ld8u_tl(s->T0, tcg_env, vector_elem_offset(&decode->op[1], ot, val)); 2881 break; 2882 case MO_16: 2883 tcg_gen_ld16u_tl(s->T0, tcg_env, vector_elem_offset(&decode->op[1], ot, val)); 2884 break; 2885 case MO_32: 2886#ifdef TARGET_X86_64 2887 tcg_gen_ld32u_tl(s->T0, tcg_env, vector_elem_offset(&decode->op[1], ot, val)); 2888 break; 2889 case MO_64: 2890#endif 2891 tcg_gen_ld_tl(s->T0, tcg_env, vector_elem_offset(&decode->op[1], ot, val)); 2892 break; 2893 default: 2894 abort(); 2895 } 2896} 2897 2898static void gen_PEXTRB(DisasContext *s, X86DecodedInsn *decode) 2899{ 2900 gen_pextr(s, decode, MO_8); 2901} 2902 2903static void gen_PEXTRW(DisasContext *s, X86DecodedInsn *decode) 2904{ 2905 gen_pextr(s, decode, MO_16); 2906} 2907 2908static void gen_PEXTR(DisasContext *s, X86DecodedInsn *decode) 2909{ 2910 MemOp ot = decode->op[0].ot; 2911 gen_pextr(s, decode, ot); 2912} 2913 2914static inline void gen_pinsr(DisasContext *s, X86DecodedInsn *decode, MemOp ot) 2915{ 2916 int vec_len = vector_len(s, decode); 2917 int mask = (vec_len >> ot) - 1; 2918 int val = decode->immediate & mask; 2919 2920 if (decode->op[1].offset != decode->op[0].offset) { 2921 assert(vec_len == 16); 2922 gen_store_sse(s, decode, decode->op[1].offset); 2923 } 2924 2925 switch (ot) { 2926 case MO_8: 2927 tcg_gen_st8_tl(s->T1, tcg_env, vector_elem_offset(&decode->op[0], ot, val)); 2928 break; 2929 case MO_16: 2930 tcg_gen_st16_tl(s->T1, tcg_env, vector_elem_offset(&decode->op[0], ot, val)); 2931 break; 2932 case MO_32: 2933#ifdef TARGET_X86_64 2934 tcg_gen_st32_tl(s->T1, tcg_env, vector_elem_offset(&decode->op[0], ot, val)); 2935 break; 2936 case MO_64: 2937#endif 2938 tcg_gen_st_tl(s->T1, tcg_env, vector_elem_offset(&decode->op[0], ot, val)); 2939 break; 2940 default: 2941 abort(); 2942 } 2943} 2944 2945static void gen_PINSRB(DisasContext *s, X86DecodedInsn *decode) 2946{ 2947 gen_pinsr(s, decode, MO_8); 2948} 2949 2950static void gen_PINSRW(DisasContext *s, X86DecodedInsn *decode) 2951{ 2952 gen_pinsr(s, decode, MO_16); 2953} 2954 2955static void gen_PINSR(DisasContext *s, X86DecodedInsn *decode) 2956{ 2957 gen_pinsr(s, decode, decode->op[2].ot); 2958} 2959 2960static void gen_pmovmskb_i64(TCGv_i64 d, TCGv_i64 s) 2961{ 2962 TCGv_i64 t = tcg_temp_new_i64(); 2963 2964 tcg_gen_andi_i64(d, s, 0x8080808080808080ull); 2965 2966 /* 2967 * After each shift+or pair: 2968 * 0: a.......b.......c.......d.......e.......f.......g.......h....... 2969 * 7: ab......bc......cd......de......ef......fg......gh......h....... 2970 * 14: abcd....bcde....cdef....defg....efgh....fgh.....gh......h....... 2971 * 28: abcdefghbcdefgh.cdefgh..defgh...efgh....fgh.....gh......h....... 2972 * The result is left in the high bits of the word. 2973 */ 2974 tcg_gen_shli_i64(t, d, 7); 2975 tcg_gen_or_i64(d, d, t); 2976 tcg_gen_shli_i64(t, d, 14); 2977 tcg_gen_or_i64(d, d, t); 2978 tcg_gen_shli_i64(t, d, 28); 2979 tcg_gen_or_i64(d, d, t); 2980} 2981 2982static void gen_pmovmskb_vec(unsigned vece, TCGv_vec d, TCGv_vec s) 2983{ 2984 TCGv_vec t = tcg_temp_new_vec_matching(d); 2985 TCGv_vec m = tcg_constant_vec_matching(d, MO_8, 0x80); 2986 2987 /* See above */ 2988 tcg_gen_and_vec(vece, d, s, m); 2989 tcg_gen_shli_vec(vece, t, d, 7); 2990 tcg_gen_or_vec(vece, d, d, t); 2991 tcg_gen_shli_vec(vece, t, d, 14); 2992 tcg_gen_or_vec(vece, d, d, t); 2993 tcg_gen_shli_vec(vece, t, d, 28); 2994 tcg_gen_or_vec(vece, d, d, t); 2995} 2996 2997static void gen_PMOVMSKB(DisasContext *s, X86DecodedInsn *decode) 2998{ 2999 static const TCGOpcode vecop_list[] = { INDEX_op_shli_vec, 0 }; 3000 static const GVecGen2 g = { 3001 .fni8 = gen_pmovmskb_i64, 3002 .fniv = gen_pmovmskb_vec, 3003 .opt_opc = vecop_list, 3004 .vece = MO_64, 3005 .prefer_i64 = TCG_TARGET_REG_BITS == 64 3006 }; 3007 MemOp ot = decode->op[2].ot; 3008 int vec_len = vector_len(s, decode); 3009 TCGv t = tcg_temp_new(); 3010 3011 tcg_gen_gvec_2(offsetof(CPUX86State, xmm_t0) + xmm_offset(ot), decode->op[2].offset, 3012 vec_len, vec_len, &g); 3013 tcg_gen_ld8u_tl(s->T0, tcg_env, offsetof(CPUX86State, xmm_t0.ZMM_B(vec_len - 1))); 3014 while (vec_len > 8) { 3015 vec_len -= 8; 3016 if (tcg_op_supported(INDEX_op_extract2, TCG_TYPE_TL, 0)) { 3017 /* 3018 * Load the next byte of the result into the high byte of T. 3019 * TCG does a similar expansion of deposit to shl+extract2; by 3020 * loading the whole word, the shift left is avoided. 3021 */ 3022#ifdef TARGET_X86_64 3023 tcg_gen_ld_tl(t, tcg_env, offsetof(CPUX86State, xmm_t0.ZMM_Q((vec_len - 1) / 8))); 3024#else 3025 tcg_gen_ld_tl(t, tcg_env, offsetof(CPUX86State, xmm_t0.ZMM_L((vec_len - 1) / 4))); 3026#endif 3027 3028 tcg_gen_extract2_tl(s->T0, t, s->T0, TARGET_LONG_BITS - 8); 3029 } else { 3030 /* 3031 * The _previous_ value is deposited into bits 8 and higher of t. Because 3032 * those bits are known to be zero after ld8u, this becomes a shift+or 3033 * if deposit is not available. 3034 */ 3035 tcg_gen_ld8u_tl(t, tcg_env, offsetof(CPUX86State, xmm_t0.ZMM_B(vec_len - 1))); 3036 tcg_gen_deposit_tl(s->T0, t, s->T0, 8, TARGET_LONG_BITS - 8); 3037 } 3038 } 3039} 3040 3041static void gen_POP(DisasContext *s, X86DecodedInsn *decode) 3042{ 3043 X86DecodedOp *op = &decode->op[0]; 3044 MemOp ot = gen_pop_T0(s); 3045 3046 assert(ot >= op->ot); 3047 if (op->has_ea || op->unit == X86_OP_SEG) { 3048 /* NOTE: order is important for MMU exceptions */ 3049 gen_writeback(s, decode, 0, s->T0); 3050 } 3051 3052 /* NOTE: writing back registers after update is important for pop %sp */ 3053 gen_pop_update(s, ot); 3054} 3055 3056static void gen_POPA(DisasContext *s, X86DecodedInsn *decode) 3057{ 3058 gen_popa(s); 3059} 3060 3061static void gen_POPCNT(DisasContext *s, X86DecodedInsn *decode) 3062{ 3063 decode->cc_dst = tcg_temp_new(); 3064 decode->cc_op = CC_OP_POPCNT; 3065 3066 tcg_gen_mov_tl(decode->cc_dst, s->T0); 3067 tcg_gen_ctpop_tl(s->T0, s->T0); 3068} 3069 3070static void gen_POPF(DisasContext *s, X86DecodedInsn *decode) 3071{ 3072 MemOp ot; 3073 int mask = TF_MASK | AC_MASK | ID_MASK | NT_MASK; 3074 3075 if (CPL(s) == 0) { 3076 mask |= IF_MASK | IOPL_MASK; 3077 } else if (CPL(s) <= IOPL(s)) { 3078 mask |= IF_MASK; 3079 } 3080 if (s->dflag == MO_16) { 3081 mask &= 0xffff; 3082 } 3083 3084 ot = gen_pop_T0(s); 3085 gen_helper_write_eflags(tcg_env, s->T0, tcg_constant_i32(mask)); 3086 gen_pop_update(s, ot); 3087 set_cc_op(s, CC_OP_EFLAGS); 3088 /* abort translation because TF/AC flag may change */ 3089 s->base.is_jmp = DISAS_EOB_NEXT; 3090} 3091 3092static void gen_PSHUFW(DisasContext *s, X86DecodedInsn *decode) 3093{ 3094 TCGv_i32 imm = tcg_constant8u_i32(decode->immediate); 3095 gen_helper_pshufw_mmx(OP_PTR0, OP_PTR1, imm); 3096} 3097 3098static void gen_PSRLW_i(DisasContext *s, X86DecodedInsn *decode) 3099{ 3100 int vec_len = vector_len(s, decode); 3101 3102 if (decode->immediate >= 16) { 3103 tcg_gen_gvec_dup_imm(MO_64, decode->op[0].offset, vec_len, vec_len, 0); 3104 } else { 3105 tcg_gen_gvec_shri(MO_16, 3106 decode->op[0].offset, decode->op[1].offset, 3107 decode->immediate, vec_len, vec_len); 3108 } 3109} 3110 3111static void gen_PSLLW_i(DisasContext *s, X86DecodedInsn *decode) 3112{ 3113 int vec_len = vector_len(s, decode); 3114 3115 if (decode->immediate >= 16) { 3116 tcg_gen_gvec_dup_imm(MO_64, decode->op[0].offset, vec_len, vec_len, 0); 3117 } else { 3118 tcg_gen_gvec_shli(MO_16, 3119 decode->op[0].offset, decode->op[1].offset, 3120 decode->immediate, vec_len, vec_len); 3121 } 3122} 3123 3124static void gen_PSRAW_i(DisasContext *s, X86DecodedInsn *decode) 3125{ 3126 int vec_len = vector_len(s, decode); 3127 3128 if (decode->immediate >= 16) { 3129 decode->immediate = 15; 3130 } 3131 tcg_gen_gvec_sari(MO_16, 3132 decode->op[0].offset, decode->op[1].offset, 3133 decode->immediate, vec_len, vec_len); 3134} 3135 3136static void gen_PSRLD_i(DisasContext *s, X86DecodedInsn *decode) 3137{ 3138 int vec_len = vector_len(s, decode); 3139 3140 if (decode->immediate >= 32) { 3141 tcg_gen_gvec_dup_imm(MO_64, decode->op[0].offset, vec_len, vec_len, 0); 3142 } else { 3143 tcg_gen_gvec_shri(MO_32, 3144 decode->op[0].offset, decode->op[1].offset, 3145 decode->immediate, vec_len, vec_len); 3146 } 3147} 3148 3149static void gen_PSLLD_i(DisasContext *s, X86DecodedInsn *decode) 3150{ 3151 int vec_len = vector_len(s, decode); 3152 3153 if (decode->immediate >= 32) { 3154 tcg_gen_gvec_dup_imm(MO_64, decode->op[0].offset, vec_len, vec_len, 0); 3155 } else { 3156 tcg_gen_gvec_shli(MO_32, 3157 decode->op[0].offset, decode->op[1].offset, 3158 decode->immediate, vec_len, vec_len); 3159 } 3160} 3161 3162static void gen_PSRAD_i(DisasContext *s, X86DecodedInsn *decode) 3163{ 3164 int vec_len = vector_len(s, decode); 3165 3166 if (decode->immediate >= 32) { 3167 decode->immediate = 31; 3168 } 3169 tcg_gen_gvec_sari(MO_32, 3170 decode->op[0].offset, decode->op[1].offset, 3171 decode->immediate, vec_len, vec_len); 3172} 3173 3174static void gen_PSRLQ_i(DisasContext *s, X86DecodedInsn *decode) 3175{ 3176 int vec_len = vector_len(s, decode); 3177 3178 if (decode->immediate >= 64) { 3179 tcg_gen_gvec_dup_imm(MO_64, decode->op[0].offset, vec_len, vec_len, 0); 3180 } else { 3181 tcg_gen_gvec_shri(MO_64, 3182 decode->op[0].offset, decode->op[1].offset, 3183 decode->immediate, vec_len, vec_len); 3184 } 3185} 3186 3187static void gen_PSLLQ_i(DisasContext *s, X86DecodedInsn *decode) 3188{ 3189 int vec_len = vector_len(s, decode); 3190 3191 if (decode->immediate >= 64) { 3192 tcg_gen_gvec_dup_imm(MO_64, decode->op[0].offset, vec_len, vec_len, 0); 3193 } else { 3194 tcg_gen_gvec_shli(MO_64, 3195 decode->op[0].offset, decode->op[1].offset, 3196 decode->immediate, vec_len, vec_len); 3197 } 3198} 3199 3200static TCGv_ptr make_imm8u_xmm_vec(uint8_t imm, int vec_len) 3201{ 3202 MemOp ot = vec_len == 16 ? MO_128 : MO_256; 3203 TCGv_i32 imm_v = tcg_constant8u_i32(imm); 3204 TCGv_ptr ptr = tcg_temp_new_ptr(); 3205 3206 tcg_gen_gvec_dup_imm(MO_64, offsetof(CPUX86State, xmm_t0) + xmm_offset(ot), 3207 vec_len, vec_len, 0); 3208 3209 tcg_gen_addi_ptr(ptr, tcg_env, offsetof(CPUX86State, xmm_t0)); 3210 tcg_gen_st_i32(imm_v, tcg_env, offsetof(CPUX86State, xmm_t0.ZMM_L(0))); 3211 return ptr; 3212} 3213 3214static void gen_PSRLDQ_i(DisasContext *s, X86DecodedInsn *decode) 3215{ 3216 int vec_len = vector_len(s, decode); 3217 TCGv_ptr imm_vec = make_imm8u_xmm_vec(decode->immediate, vec_len); 3218 3219 if (s->vex_l) { 3220 gen_helper_psrldq_ymm(tcg_env, OP_PTR0, OP_PTR1, imm_vec); 3221 } else { 3222 gen_helper_psrldq_xmm(tcg_env, OP_PTR0, OP_PTR1, imm_vec); 3223 } 3224} 3225 3226static void gen_PSLLDQ_i(DisasContext *s, X86DecodedInsn *decode) 3227{ 3228 int vec_len = vector_len(s, decode); 3229 TCGv_ptr imm_vec = make_imm8u_xmm_vec(decode->immediate, vec_len); 3230 3231 if (s->vex_l) { 3232 gen_helper_pslldq_ymm(tcg_env, OP_PTR0, OP_PTR1, imm_vec); 3233 } else { 3234 gen_helper_pslldq_xmm(tcg_env, OP_PTR0, OP_PTR1, imm_vec); 3235 } 3236} 3237 3238static void gen_PUSH(DisasContext *s, X86DecodedInsn *decode) 3239{ 3240 gen_push_v(s, s->T0); 3241} 3242 3243static void gen_PUSHA(DisasContext *s, X86DecodedInsn *decode) 3244{ 3245 gen_pusha(s); 3246} 3247 3248static void gen_PUSHF(DisasContext *s, X86DecodedInsn *decode) 3249{ 3250 gen_update_cc_op(s); 3251 gen_helper_read_eflags(s->T0, tcg_env); 3252 gen_push_v(s, s->T0); 3253} 3254 3255static MemOp gen_shift_count(DisasContext *s, X86DecodedInsn *decode, 3256 bool *can_be_zero, TCGv *count, int unit) 3257{ 3258 MemOp ot = decode->op[0].ot; 3259 int mask = (ot <= MO_32 ? 0x1f : 0x3f); 3260 3261 *can_be_zero = false; 3262 switch (unit) { 3263 case X86_OP_INT: 3264 *count = tcg_temp_new(); 3265 tcg_gen_andi_tl(*count, cpu_regs[R_ECX], mask); 3266 *can_be_zero = true; 3267 break; 3268 3269 case X86_OP_IMM: 3270 if ((decode->immediate & mask) == 0) { 3271 *count = NULL; 3272 break; 3273 } 3274 *count = tcg_temp_new(); 3275 tcg_gen_movi_tl(*count, decode->immediate & mask); 3276 break; 3277 3278 case X86_OP_SKIP: 3279 *count = tcg_temp_new(); 3280 tcg_gen_movi_tl(*count, 1); 3281 break; 3282 3283 default: 3284 g_assert_not_reached(); 3285 } 3286 3287 return ot; 3288} 3289 3290/* 3291 * Compute existing flags in decode->cc_src, for gen_* functions that wants 3292 * to set the cc_op set to CC_OP_ADCOX. In particular, this allows rotate 3293 * operations to compute the carry in decode->cc_dst and the overflow in 3294 * decode->cc_src2. 3295 * 3296 * If need_flags is true, decode->cc_dst and decode->cc_src2 are preloaded 3297 * with the value of CF and OF before the instruction, so that it is possible 3298 * to keep the flags unmodified. 3299 * 3300 * Return true if carry could be made available cheaply as a 1-bit value in 3301 * decode->cc_dst (trying a bit harder if want_carry is true). If false is 3302 * returned, decode->cc_dst is uninitialized and the carry is only available 3303 * as bit 0 of decode->cc_src. 3304 */ 3305static bool gen_eflags_adcox(DisasContext *s, X86DecodedInsn *decode, bool want_carry, bool need_flags) 3306{ 3307 bool got_cf = false; 3308 bool got_of = false; 3309 3310 decode->cc_dst = tcg_temp_new(); 3311 decode->cc_src = tcg_temp_new(); 3312 decode->cc_src2 = tcg_temp_new(); 3313 decode->cc_op = CC_OP_ADCOX; 3314 3315 /* A lot more cc_ops could be "optimized" to avoid the extracts at 3316 * the end (INC/DEC, BMILG, MUL), but they are all really unlikely 3317 * to be followed by rotations within the same basic block. 3318 */ 3319 switch (s->cc_op) { 3320 case CC_OP_ADCOX: 3321 /* No need to compute the full EFLAGS, CF/OF are already isolated. */ 3322 tcg_gen_mov_tl(decode->cc_src, cpu_cc_src); 3323 if (need_flags) { 3324 tcg_gen_mov_tl(decode->cc_src2, cpu_cc_src2); 3325 got_of = true; 3326 } 3327 if (want_carry || need_flags) { 3328 tcg_gen_mov_tl(decode->cc_dst, cpu_cc_dst); 3329 got_cf = true; 3330 } 3331 break; 3332 3333 case CC_OP_LOGICB ... CC_OP_LOGICQ: 3334 /* CF and OF are zero, do it just because it's easy. */ 3335 gen_mov_eflags(s, decode->cc_src); 3336 if (need_flags) { 3337 tcg_gen_movi_tl(decode->cc_src2, 0); 3338 got_of = true; 3339 } 3340 if (want_carry || need_flags) { 3341 tcg_gen_movi_tl(decode->cc_dst, 0); 3342 got_cf = true; 3343 } 3344 break; 3345 3346 case CC_OP_SARB ... CC_OP_SARQ: 3347 /* 3348 * SHR/RCR/SHR/RCR/... is a relatively common occurrence of RCR. 3349 * By computing CF without using eflags, the calls to cc_compute_all 3350 * can be eliminated as dead code (except for the last RCR). 3351 */ 3352 if (want_carry || need_flags) { 3353 tcg_gen_andi_tl(decode->cc_dst, cpu_cc_src, 1); 3354 got_cf = true; 3355 } 3356 gen_mov_eflags(s, decode->cc_src); 3357 break; 3358 3359 case CC_OP_SHLB ... CC_OP_SHLQ: 3360 /* 3361 * Likewise for SHL/RCL/SHL/RCL/... but, if CF is not in the sign 3362 * bit, we might as well fish CF out of EFLAGS and save a shift. 3363 */ 3364 if (want_carry && (!need_flags || s->cc_op == CC_OP_SHLB + MO_TL)) { 3365 MemOp size = cc_op_size(s->cc_op); 3366 tcg_gen_shri_tl(decode->cc_dst, cpu_cc_src, (8 << size) - 1); 3367 got_cf = true; 3368 } 3369 gen_mov_eflags(s, decode->cc_src); 3370 break; 3371 3372 default: 3373 gen_mov_eflags(s, decode->cc_src); 3374 break; 3375 } 3376 3377 if (need_flags) { 3378 /* If the flags could be left unmodified, always load them. */ 3379 if (!got_of) { 3380 tcg_gen_extract_tl(decode->cc_src2, decode->cc_src, ctz32(CC_O), 1); 3381 got_of = true; 3382 } 3383 if (!got_cf) { 3384 tcg_gen_extract_tl(decode->cc_dst, decode->cc_src, ctz32(CC_C), 1); 3385 got_cf = true; 3386 } 3387 } 3388 return got_cf; 3389} 3390 3391static void gen_rot_overflow(X86DecodedInsn *decode, TCGv result, TCGv old, 3392 bool can_be_zero, TCGv count) 3393{ 3394 MemOp ot = decode->op[0].ot; 3395 TCGv temp = can_be_zero ? tcg_temp_new() : decode->cc_src2; 3396 3397 tcg_gen_xor_tl(temp, old, result); 3398 tcg_gen_extract_tl(temp, temp, (8 << ot) - 1, 1); 3399 if (can_be_zero) { 3400 tcg_gen_movcond_tl(TCG_COND_EQ, decode->cc_src2, count, tcg_constant_tl(0), 3401 decode->cc_src2, temp); 3402 } 3403} 3404 3405/* 3406 * RCx operations are invariant modulo 8*operand_size+1. For 8 and 16-bit operands, 3407 * this is less than 0x1f (the mask applied by gen_shift_count) so reduce further. 3408 */ 3409static void gen_rotc_mod(MemOp ot, TCGv count) 3410{ 3411 TCGv temp; 3412 3413 switch (ot) { 3414 case MO_8: 3415 temp = tcg_temp_new(); 3416 tcg_gen_subi_tl(temp, count, 18); 3417 tcg_gen_movcond_tl(TCG_COND_GE, count, temp, tcg_constant_tl(0), temp, count); 3418 tcg_gen_subi_tl(temp, count, 9); 3419 tcg_gen_movcond_tl(TCG_COND_GE, count, temp, tcg_constant_tl(0), temp, count); 3420 break; 3421 3422 case MO_16: 3423 temp = tcg_temp_new(); 3424 tcg_gen_subi_tl(temp, count, 17); 3425 tcg_gen_movcond_tl(TCG_COND_GE, count, temp, tcg_constant_tl(0), temp, count); 3426 break; 3427 3428 default: 3429 break; 3430 } 3431} 3432 3433/* 3434 * The idea here is that the bit to the right of the new bit 0 is the 3435 * new carry, and the bit to the right of the old bit 0 is the old carry. 3436 * Just like a regular rotation, the result of the rotation is composed 3437 * from a right shifted part and a left shifted part of s->T0. The new carry 3438 * is extracted from the right-shifted portion, and the old carry is 3439 * inserted at the end of the left-shifted portion. 3440 * 3441 * Because of the separate shifts involving the carry, gen_RCL and gen_RCR 3442 * mostly operate on count-1. This also comes in handy when computing 3443 * length - count, because (length-1) - (count-1) can be computed with 3444 * a XOR, and that is commutative unlike subtraction. 3445 */ 3446static void gen_RCL(DisasContext *s, X86DecodedInsn *decode) 3447{ 3448 bool have_1bit_cin, can_be_zero; 3449 TCGv count; 3450 TCGLabel *zero_label = NULL; 3451 MemOp ot = gen_shift_count(s, decode, &can_be_zero, &count, decode->op[2].unit); 3452 TCGv low, high, low_count; 3453 3454 if (!count) { 3455 return; 3456 } 3457 3458 low = tcg_temp_new(); 3459 high = tcg_temp_new(); 3460 low_count = tcg_temp_new(); 3461 3462 gen_rotc_mod(ot, count); 3463 have_1bit_cin = gen_eflags_adcox(s, decode, true, can_be_zero); 3464 if (can_be_zero) { 3465 zero_label = gen_new_label(); 3466 tcg_gen_brcondi_tl(TCG_COND_EQ, count, 0, zero_label); 3467 } 3468 3469 /* Compute high part, including incoming carry. */ 3470 if (!have_1bit_cin || tcg_op_deposit_valid(TCG_TYPE_TL, 1, TARGET_LONG_BITS - 1)) { 3471 /* high = (T0 << 1) | cin */ 3472 TCGv cin = have_1bit_cin ? decode->cc_dst : decode->cc_src; 3473 tcg_gen_deposit_tl(high, cin, s->T0, 1, TARGET_LONG_BITS - 1); 3474 } else { 3475 /* Same as above but without deposit; cin in cc_dst. */ 3476 tcg_gen_add_tl(high, s->T0, decode->cc_dst); 3477 tcg_gen_add_tl(high, high, s->T0); 3478 } 3479 tcg_gen_subi_tl(count, count, 1); 3480 tcg_gen_shl_tl(high, high, count); 3481 3482 /* Compute low part and outgoing carry, incoming s->T0 is zero extended */ 3483 tcg_gen_xori_tl(low_count, count, (8 << ot) - 1); /* LENGTH - 1 - (count - 1) */ 3484 tcg_gen_shr_tl(low, s->T0, low_count); 3485 tcg_gen_andi_tl(decode->cc_dst, low, 1); 3486 tcg_gen_shri_tl(low, low, 1); 3487 3488 /* Compute result and outgoing overflow */ 3489 tcg_gen_mov_tl(decode->cc_src2, s->T0); 3490 tcg_gen_or_tl(s->T0, low, high); 3491 gen_rot_overflow(decode, s->T0, decode->cc_src2, false, NULL); 3492 3493 if (zero_label) { 3494 gen_set_label(zero_label); 3495 } 3496} 3497 3498static void gen_RCR(DisasContext *s, X86DecodedInsn *decode) 3499{ 3500 bool have_1bit_cin, can_be_zero; 3501 TCGv count; 3502 TCGLabel *zero_label = NULL; 3503 MemOp ot = gen_shift_count(s, decode, &can_be_zero, &count, decode->op[2].unit); 3504 TCGv low, high, high_count; 3505 3506 if (!count) { 3507 return; 3508 } 3509 3510 low = tcg_temp_new(); 3511 high = tcg_temp_new(); 3512 high_count = tcg_temp_new(); 3513 3514 gen_rotc_mod(ot, count); 3515 have_1bit_cin = gen_eflags_adcox(s, decode, true, can_be_zero); 3516 if (can_be_zero) { 3517 zero_label = gen_new_label(); 3518 tcg_gen_brcondi_tl(TCG_COND_EQ, count, 0, zero_label); 3519 } 3520 3521 /* Save incoming carry into high, it will be shifted later. */ 3522 if (!have_1bit_cin || tcg_op_deposit_valid(TCG_TYPE_TL, 1, TARGET_LONG_BITS - 1)) { 3523 TCGv cin = have_1bit_cin ? decode->cc_dst : decode->cc_src; 3524 tcg_gen_deposit_tl(high, cin, s->T0, 1, TARGET_LONG_BITS - 1); 3525 } else { 3526 /* Same as above but without deposit; cin in cc_dst. */ 3527 tcg_gen_add_tl(high, s->T0, decode->cc_dst); 3528 tcg_gen_add_tl(high, high, s->T0); 3529 } 3530 3531 /* Compute low part and outgoing carry, incoming s->T0 is zero extended */ 3532 tcg_gen_subi_tl(count, count, 1); 3533 tcg_gen_shr_tl(low, s->T0, count); 3534 tcg_gen_andi_tl(decode->cc_dst, low, 1); 3535 tcg_gen_shri_tl(low, low, 1); 3536 3537 /* Move high part to the right position */ 3538 tcg_gen_xori_tl(high_count, count, (8 << ot) - 1); /* LENGTH - 1 - (count - 1) */ 3539 tcg_gen_shl_tl(high, high, high_count); 3540 3541 /* Compute result and outgoing overflow */ 3542 tcg_gen_mov_tl(decode->cc_src2, s->T0); 3543 tcg_gen_or_tl(s->T0, low, high); 3544 gen_rot_overflow(decode, s->T0, decode->cc_src2, false, NULL); 3545 3546 if (zero_label) { 3547 gen_set_label(zero_label); 3548 } 3549} 3550 3551#ifdef CONFIG_USER_ONLY 3552static void gen_unreachable(DisasContext *s, X86DecodedInsn *decode) 3553{ 3554 g_assert_not_reached(); 3555} 3556#endif 3557 3558#ifndef CONFIG_USER_ONLY 3559static void gen_RDMSR(DisasContext *s, X86DecodedInsn *decode) 3560{ 3561 gen_update_cc_op(s); 3562 gen_update_eip_cur(s); 3563 gen_helper_rdmsr(tcg_env); 3564} 3565#else 3566#define gen_RDMSR gen_unreachable 3567#endif 3568 3569static void gen_RDPMC(DisasContext *s, X86DecodedInsn *decode) 3570{ 3571 gen_update_cc_op(s); 3572 gen_update_eip_cur(s); 3573 translator_io_start(&s->base); 3574 gen_helper_rdpmc(tcg_env); 3575 s->base.is_jmp = DISAS_NORETURN; 3576} 3577 3578static void gen_RDTSC(DisasContext *s, X86DecodedInsn *decode) 3579{ 3580 gen_update_cc_op(s); 3581 gen_update_eip_cur(s); 3582 translator_io_start(&s->base); 3583 gen_helper_rdtsc(tcg_env); 3584} 3585 3586static void gen_RDxxBASE(DisasContext *s, X86DecodedInsn *decode) 3587{ 3588 TCGv base = cpu_seg_base[s->modrm & 8 ? R_GS : R_FS]; 3589 3590 /* Preserve hflags bits by testing CR4 at runtime. */ 3591 gen_helper_cr4_testbit(tcg_env, tcg_constant_i32(CR4_FSGSBASE_MASK)); 3592 tcg_gen_mov_tl(s->T0, base); 3593} 3594 3595static void gen_RET(DisasContext *s, X86DecodedInsn *decode) 3596{ 3597 int16_t adjust = decode->e.op1 == X86_TYPE_I ? decode->immediate : 0; 3598 3599 MemOp ot = gen_pop_T0(s); 3600 gen_stack_update(s, adjust + (1 << ot)); 3601 gen_op_jmp_v(s, s->T0); 3602 gen_bnd_jmp(s); 3603 s->base.is_jmp = DISAS_JUMP; 3604} 3605 3606static void gen_RETF(DisasContext *s, X86DecodedInsn *decode) 3607{ 3608 int16_t adjust = decode->e.op1 == X86_TYPE_I ? decode->immediate : 0; 3609 3610 if (!PE(s) || VM86(s)) { 3611 gen_lea_ss_ofs(s, s->A0, cpu_regs[R_ESP], 0); 3612 /* pop offset */ 3613 gen_op_ld_v(s, s->dflag, s->T0, s->A0); 3614 /* NOTE: keeping EIP updated is not a problem in case of 3615 exception */ 3616 gen_op_jmp_v(s, s->T0); 3617 /* pop selector */ 3618 gen_add_A0_im(s, 1 << s->dflag); 3619 gen_op_ld_v(s, s->dflag, s->T0, s->A0); 3620 gen_op_movl_seg_real(s, R_CS, s->T0); 3621 /* add stack offset */ 3622 gen_stack_update(s, adjust + (2 << s->dflag)); 3623 } else { 3624 gen_update_cc_op(s); 3625 gen_update_eip_cur(s); 3626 gen_helper_lret_protected(tcg_env, tcg_constant_i32(s->dflag - 1), 3627 tcg_constant_i32(adjust)); 3628 } 3629 s->base.is_jmp = DISAS_EOB_ONLY; 3630} 3631 3632/* 3633 * Return non-NULL if a 32-bit rotate works, after possibly replicating the input. 3634 * The input has already been zero-extended upon operand decode. 3635 */ 3636static TCGv_i32 gen_rot_replicate(MemOp ot, TCGv in) 3637{ 3638 TCGv_i32 temp; 3639 switch (ot) { 3640 case MO_8: 3641 temp = tcg_temp_new_i32(); 3642 tcg_gen_trunc_tl_i32(temp, in); 3643 tcg_gen_muli_i32(temp, temp, 0x01010101); 3644 return temp; 3645 3646 case MO_16: 3647 temp = tcg_temp_new_i32(); 3648 tcg_gen_trunc_tl_i32(temp, in); 3649 tcg_gen_deposit_i32(temp, temp, temp, 16, 16); 3650 return temp; 3651 3652#ifdef TARGET_X86_64 3653 case MO_32: 3654 temp = tcg_temp_new_i32(); 3655 tcg_gen_trunc_tl_i32(temp, in); 3656 return temp; 3657#endif 3658 3659 default: 3660 return NULL; 3661 } 3662} 3663 3664static void gen_rot_carry(X86DecodedInsn *decode, TCGv result, 3665 bool can_be_zero, TCGv count, int bit) 3666{ 3667 if (!can_be_zero) { 3668 tcg_gen_extract_tl(decode->cc_dst, result, bit, 1); 3669 } else { 3670 TCGv temp = tcg_temp_new(); 3671 tcg_gen_extract_tl(temp, result, bit, 1); 3672 tcg_gen_movcond_tl(TCG_COND_EQ, decode->cc_dst, count, tcg_constant_tl(0), 3673 decode->cc_dst, temp); 3674 } 3675} 3676 3677static void gen_ROL(DisasContext *s, X86DecodedInsn *decode) 3678{ 3679 bool can_be_zero; 3680 TCGv count; 3681 MemOp ot = gen_shift_count(s, decode, &can_be_zero, &count, decode->op[2].unit); 3682 TCGv_i32 temp32, count32; 3683 TCGv old = tcg_temp_new(); 3684 3685 if (!count) { 3686 return; 3687 } 3688 3689 gen_eflags_adcox(s, decode, false, can_be_zero); 3690 tcg_gen_mov_tl(old, s->T0); 3691 temp32 = gen_rot_replicate(ot, s->T0); 3692 if (temp32) { 3693 count32 = tcg_temp_new_i32(); 3694 tcg_gen_trunc_tl_i32(count32, count); 3695 tcg_gen_rotl_i32(temp32, temp32, count32); 3696 /* Zero extend to facilitate later optimization. */ 3697 tcg_gen_extu_i32_tl(s->T0, temp32); 3698 } else { 3699 tcg_gen_rotl_tl(s->T0, s->T0, count); 3700 } 3701 gen_rot_carry(decode, s->T0, can_be_zero, count, 0); 3702 gen_rot_overflow(decode, s->T0, old, can_be_zero, count); 3703} 3704 3705static void gen_ROR(DisasContext *s, X86DecodedInsn *decode) 3706{ 3707 bool can_be_zero; 3708 TCGv count; 3709 MemOp ot = gen_shift_count(s, decode, &can_be_zero, &count, decode->op[2].unit); 3710 TCGv_i32 temp32, count32; 3711 TCGv old = tcg_temp_new(); 3712 3713 if (!count) { 3714 return; 3715 } 3716 3717 gen_eflags_adcox(s, decode, false, can_be_zero); 3718 tcg_gen_mov_tl(old, s->T0); 3719 temp32 = gen_rot_replicate(ot, s->T0); 3720 if (temp32) { 3721 count32 = tcg_temp_new_i32(); 3722 tcg_gen_trunc_tl_i32(count32, count); 3723 tcg_gen_rotr_i32(temp32, temp32, count32); 3724 /* Zero extend to facilitate later optimization. */ 3725 tcg_gen_extu_i32_tl(s->T0, temp32); 3726 gen_rot_carry(decode, s->T0, can_be_zero, count, 31); 3727 } else { 3728 tcg_gen_rotr_tl(s->T0, s->T0, count); 3729 gen_rot_carry(decode, s->T0, can_be_zero, count, TARGET_LONG_BITS - 1); 3730 } 3731 gen_rot_overflow(decode, s->T0, old, can_be_zero, count); 3732} 3733 3734static void gen_RORX(DisasContext *s, X86DecodedInsn *decode) 3735{ 3736 MemOp ot = decode->op[0].ot; 3737 int mask = ot == MO_64 ? 63 : 31; 3738 int b = decode->immediate & mask; 3739 3740 switch (ot) { 3741 case MO_32: 3742#ifdef TARGET_X86_64 3743 { 3744 TCGv_i32 tmp = tcg_temp_new_i32(); 3745 3746 tcg_gen_trunc_tl_i32(tmp, s->T0); 3747 tcg_gen_rotri_i32(tmp, tmp, b); 3748 tcg_gen_extu_i32_tl(s->T0, tmp); 3749 break; 3750 } 3751 3752 case MO_64: 3753#endif 3754 tcg_gen_rotri_tl(s->T0, s->T0, b); 3755 break; 3756 3757 default: 3758 g_assert_not_reached(); 3759 } 3760} 3761 3762#ifndef CONFIG_USER_ONLY 3763static void gen_RSM(DisasContext *s, X86DecodedInsn *decode) 3764{ 3765 gen_helper_rsm(tcg_env); 3766 assume_cc_op(s, CC_OP_EFLAGS); 3767 s->base.is_jmp = DISAS_EOB_ONLY; 3768} 3769#else 3770#define gen_RSM gen_UD 3771#endif 3772 3773static void gen_SAHF(DisasContext *s, X86DecodedInsn *decode) 3774{ 3775 if (CODE64(s) && !(s->cpuid_ext3_features & CPUID_EXT3_LAHF_LM)) { 3776 return gen_illegal_opcode(s); 3777 } 3778 tcg_gen_shri_tl(s->T0, cpu_regs[R_EAX], 8); 3779 gen_compute_eflags(s); 3780 tcg_gen_andi_tl(cpu_cc_src, cpu_cc_src, CC_O); 3781 tcg_gen_andi_tl(s->T0, s->T0, CC_S | CC_Z | CC_A | CC_P | CC_C); 3782 tcg_gen_or_tl(cpu_cc_src, cpu_cc_src, s->T0); 3783} 3784 3785static void gen_SALC(DisasContext *s, X86DecodedInsn *decode) 3786{ 3787 gen_compute_eflags_c(s, s->T0); 3788 tcg_gen_neg_tl(s->T0, s->T0); 3789} 3790 3791static void gen_shift_dynamic_flags(DisasContext *s, X86DecodedInsn *decode, TCGv count, CCOp cc_op) 3792{ 3793 TCGv_i32 count32 = tcg_temp_new_i32(); 3794 TCGv_i32 old_cc_op; 3795 3796 decode->cc_op = CC_OP_DYNAMIC; 3797 decode->cc_op_dynamic = tcg_temp_new_i32(); 3798 3799 assert(decode->cc_dst == s->T0); 3800 if (cc_op_live(s->cc_op) & USES_CC_DST) { 3801 decode->cc_dst = tcg_temp_new(); 3802 tcg_gen_movcond_tl(TCG_COND_EQ, decode->cc_dst, count, tcg_constant_tl(0), 3803 cpu_cc_dst, s->T0); 3804 } 3805 3806 if (cc_op_live(s->cc_op) & USES_CC_SRC) { 3807 tcg_gen_movcond_tl(TCG_COND_EQ, decode->cc_src, count, tcg_constant_tl(0), 3808 cpu_cc_src, decode->cc_src); 3809 } 3810 3811 tcg_gen_trunc_tl_i32(count32, count); 3812 if (s->cc_op == CC_OP_DYNAMIC) { 3813 old_cc_op = cpu_cc_op; 3814 } else { 3815 old_cc_op = tcg_constant_i32(s->cc_op); 3816 } 3817 tcg_gen_movcond_i32(TCG_COND_EQ, decode->cc_op_dynamic, count32, tcg_constant_i32(0), 3818 old_cc_op, tcg_constant_i32(cc_op)); 3819} 3820 3821static void gen_SAR(DisasContext *s, X86DecodedInsn *decode) 3822{ 3823 bool can_be_zero; 3824 TCGv count; 3825 MemOp ot = gen_shift_count(s, decode, &can_be_zero, &count, decode->op[2].unit); 3826 3827 if (!count) { 3828 return; 3829 } 3830 3831 decode->cc_dst = s->T0; 3832 decode->cc_src = tcg_temp_new(); 3833 tcg_gen_subi_tl(decode->cc_src, count, 1); 3834 tcg_gen_sar_tl(decode->cc_src, s->T0, decode->cc_src); 3835 tcg_gen_sar_tl(s->T0, s->T0, count); 3836 if (can_be_zero) { 3837 gen_shift_dynamic_flags(s, decode, count, CC_OP_SARB + ot); 3838 } else { 3839 decode->cc_op = CC_OP_SARB + ot; 3840 } 3841} 3842 3843static void gen_SARX(DisasContext *s, X86DecodedInsn *decode) 3844{ 3845 MemOp ot = decode->op[0].ot; 3846 int mask; 3847 3848 mask = ot == MO_64 ? 63 : 31; 3849 tcg_gen_andi_tl(s->T1, s->T1, mask); 3850 tcg_gen_sar_tl(s->T0, s->T0, s->T1); 3851} 3852 3853static void gen_SUB(DisasContext *s, X86DecodedInsn *decode); 3854static void gen_SBB(DisasContext *s, X86DecodedInsn *decode) 3855{ 3856 MemOp ot = decode->op[0].ot; 3857 TCGv c_in; 3858 3859 /* 3860 * Try to avoid CC_OP_SBB by transforming as follows: 3861 * CC_SBB: src1 = dst + c_in, src2 = 0, src3 = c_in 3862 * CC_SUB: src1 = dst + c_in, src2 = c_in (no src3) 3863 * 3864 * In general src2 vs. src3 matters when computing AF and OF, but not here: 3865 * - AF is bit 4 of dst^src1^src2, which is bit 4 of dst^src1 in both cases 3866 * - OF is a function of the two MSBs, and in both cases they are zero for src2 3867 */ 3868 if (decode->e.op2 == X86_TYPE_I && decode->immediate == 0) { 3869 gen_compute_eflags_c(s, s->T1); 3870 gen_SUB(s, decode); 3871 return; 3872 } 3873 3874 c_in = tcg_temp_new(); 3875 gen_compute_eflags_c(s, c_in); 3876 3877 /* 3878 * Here the change is as follows: 3879 * CC_SBB: src1 = T0, src2 = T0, src3 = c_in 3880 * CC_SUB: src1 = 0, src2 = c_in (no src3) 3881 * 3882 * The difference also does not matter: 3883 * - AF is bit 4 of dst^src1^src2, but bit 4 of src1^src2 is zero in both cases 3884 * therefore AF comes straight from dst (in fact it is c_in) 3885 * - for OF, src1 and src2 have the same sign in both cases, meaning there 3886 * can be no overflow 3887 */ 3888 if (decode->e.op2 != X86_TYPE_I && !decode->op[0].has_ea && decode->op[0].n == decode->op[2].n) { 3889 if (s->cc_op == CC_OP_DYNAMIC) { 3890 tcg_gen_neg_tl(s->T0, c_in); 3891 } else { 3892 /* 3893 * Do not negate c_in because it will often be dead and only the 3894 * instruction generated by negsetcond will survive. 3895 */ 3896 gen_neg_setcc(s, JCC_B << 1, s->T0); 3897 } 3898 tcg_gen_movi_tl(s->cc_srcT, 0); 3899 decode->cc_src = c_in; 3900 decode->cc_dst = s->T0; 3901 decode->cc_op = CC_OP_SUBB + ot; 3902 return; 3903 } 3904 3905 if (s->prefix & PREFIX_LOCK) { 3906 tcg_gen_add_tl(s->T0, s->T1, c_in); 3907 tcg_gen_neg_tl(s->T0, s->T0); 3908 tcg_gen_atomic_add_fetch_tl(s->T0, s->A0, s->T0, 3909 s->mem_index, ot | MO_LE); 3910 } else { 3911 tcg_gen_sub_tl(s->T0, s->T0, s->T1); 3912 tcg_gen_sub_tl(s->T0, s->T0, c_in); 3913 } 3914 prepare_update3_cc(decode, s, CC_OP_SBBB + ot, c_in); 3915} 3916 3917static void gen_SCAS(DisasContext *s, X86DecodedInsn *decode) 3918{ 3919 MemOp ot = decode->op[2].ot; 3920 gen_repz_nz(s, ot, gen_scas); 3921} 3922 3923static void gen_SETcc(DisasContext *s, X86DecodedInsn *decode) 3924{ 3925 gen_setcc(s, decode->b & 0xf, s->T0); 3926} 3927 3928static void gen_SFENCE(DisasContext *s, X86DecodedInsn *decode) 3929{ 3930 tcg_gen_mb(TCG_MO_ST_ST | TCG_BAR_SC); 3931} 3932 3933static void gen_SHA1NEXTE(DisasContext *s, X86DecodedInsn *decode) 3934{ 3935 gen_helper_sha1nexte(OP_PTR0, OP_PTR1, OP_PTR2); 3936} 3937 3938static void gen_SHA1MSG1(DisasContext *s, X86DecodedInsn *decode) 3939{ 3940 gen_helper_sha1msg1(OP_PTR0, OP_PTR1, OP_PTR2); 3941} 3942 3943static void gen_SHA1MSG2(DisasContext *s, X86DecodedInsn *decode) 3944{ 3945 gen_helper_sha1msg2(OP_PTR0, OP_PTR1, OP_PTR2); 3946} 3947 3948static void gen_SHA1RNDS4(DisasContext *s, X86DecodedInsn *decode) 3949{ 3950 switch(decode->immediate & 3) { 3951 case 0: 3952 gen_helper_sha1rnds4_f0(OP_PTR0, OP_PTR0, OP_PTR1); 3953 break; 3954 case 1: 3955 gen_helper_sha1rnds4_f1(OP_PTR0, OP_PTR0, OP_PTR1); 3956 break; 3957 case 2: 3958 gen_helper_sha1rnds4_f2(OP_PTR0, OP_PTR0, OP_PTR1); 3959 break; 3960 case 3: 3961 gen_helper_sha1rnds4_f3(OP_PTR0, OP_PTR0, OP_PTR1); 3962 break; 3963 } 3964} 3965 3966static void gen_SHA256MSG1(DisasContext *s, X86DecodedInsn *decode) 3967{ 3968 gen_helper_sha256msg1(OP_PTR0, OP_PTR1, OP_PTR2); 3969} 3970 3971static void gen_SHA256MSG2(DisasContext *s, X86DecodedInsn *decode) 3972{ 3973 gen_helper_sha256msg2(OP_PTR0, OP_PTR1, OP_PTR2); 3974} 3975 3976static void gen_SHA256RNDS2(DisasContext *s, X86DecodedInsn *decode) 3977{ 3978 TCGv_i32 wk0 = tcg_temp_new_i32(); 3979 TCGv_i32 wk1 = tcg_temp_new_i32(); 3980 3981 tcg_gen_ld_i32(wk0, tcg_env, ZMM_OFFSET(0) + offsetof(ZMMReg, ZMM_L(0))); 3982 tcg_gen_ld_i32(wk1, tcg_env, ZMM_OFFSET(0) + offsetof(ZMMReg, ZMM_L(1))); 3983 3984 gen_helper_sha256rnds2(OP_PTR0, OP_PTR1, OP_PTR2, wk0, wk1); 3985} 3986 3987static void gen_SHL(DisasContext *s, X86DecodedInsn *decode) 3988{ 3989 bool can_be_zero; 3990 TCGv count; 3991 MemOp ot = gen_shift_count(s, decode, &can_be_zero, &count, decode->op[2].unit); 3992 3993 if (!count) { 3994 return; 3995 } 3996 3997 decode->cc_dst = s->T0; 3998 decode->cc_src = tcg_temp_new(); 3999 tcg_gen_subi_tl(decode->cc_src, count, 1); 4000 tcg_gen_shl_tl(decode->cc_src, s->T0, decode->cc_src); 4001 tcg_gen_shl_tl(s->T0, s->T0, count); 4002 if (can_be_zero) { 4003 gen_shift_dynamic_flags(s, decode, count, CC_OP_SHLB + ot); 4004 } else { 4005 decode->cc_op = CC_OP_SHLB + ot; 4006 } 4007} 4008 4009static void gen_SHLD(DisasContext *s, X86DecodedInsn *decode) 4010{ 4011 bool can_be_zero; 4012 TCGv count; 4013 int unit = decode->e.op3 == X86_TYPE_I ? X86_OP_IMM : X86_OP_INT; 4014 MemOp ot = gen_shift_count(s, decode, &can_be_zero, &count, unit); 4015 4016 if (!count) { 4017 return; 4018 } 4019 4020 decode->cc_dst = s->T0; 4021 decode->cc_src = gen_shiftd_rm_T1(s, ot, false, count); 4022 if (can_be_zero) { 4023 gen_shift_dynamic_flags(s, decode, count, CC_OP_SHLB + ot); 4024 } else { 4025 decode->cc_op = CC_OP_SHLB + ot; 4026 } 4027} 4028 4029static void gen_SHLX(DisasContext *s, X86DecodedInsn *decode) 4030{ 4031 MemOp ot = decode->op[0].ot; 4032 int mask; 4033 4034 mask = ot == MO_64 ? 63 : 31; 4035 tcg_gen_andi_tl(s->T1, s->T1, mask); 4036 tcg_gen_shl_tl(s->T0, s->T0, s->T1); 4037} 4038 4039static void gen_SHR(DisasContext *s, X86DecodedInsn *decode) 4040{ 4041 bool can_be_zero; 4042 TCGv count; 4043 MemOp ot = gen_shift_count(s, decode, &can_be_zero, &count, decode->op[2].unit); 4044 4045 if (!count) { 4046 return; 4047 } 4048 4049 decode->cc_dst = s->T0; 4050 decode->cc_src = tcg_temp_new(); 4051 tcg_gen_subi_tl(decode->cc_src, count, 1); 4052 tcg_gen_shr_tl(decode->cc_src, s->T0, decode->cc_src); 4053 tcg_gen_shr_tl(s->T0, s->T0, count); 4054 if (can_be_zero) { 4055 gen_shift_dynamic_flags(s, decode, count, CC_OP_SARB + ot); 4056 } else { 4057 decode->cc_op = CC_OP_SARB + ot; 4058 } 4059} 4060 4061static void gen_SHRD(DisasContext *s, X86DecodedInsn *decode) 4062{ 4063 bool can_be_zero; 4064 TCGv count; 4065 int unit = decode->e.op3 == X86_TYPE_I ? X86_OP_IMM : X86_OP_INT; 4066 MemOp ot = gen_shift_count(s, decode, &can_be_zero, &count, unit); 4067 4068 if (!count) { 4069 return; 4070 } 4071 4072 decode->cc_dst = s->T0; 4073 decode->cc_src = gen_shiftd_rm_T1(s, ot, true, count); 4074 if (can_be_zero) { 4075 gen_shift_dynamic_flags(s, decode, count, CC_OP_SARB + ot); 4076 } else { 4077 decode->cc_op = CC_OP_SARB + ot; 4078 } 4079} 4080 4081static void gen_SHRX(DisasContext *s, X86DecodedInsn *decode) 4082{ 4083 MemOp ot = decode->op[0].ot; 4084 int mask; 4085 4086 mask = ot == MO_64 ? 63 : 31; 4087 tcg_gen_andi_tl(s->T1, s->T1, mask); 4088 tcg_gen_shr_tl(s->T0, s->T0, s->T1); 4089} 4090 4091static void gen_STC(DisasContext *s, X86DecodedInsn *decode) 4092{ 4093 gen_compute_eflags(s); 4094 tcg_gen_ori_tl(cpu_cc_src, cpu_cc_src, CC_C); 4095} 4096 4097static void gen_STD(DisasContext *s, X86DecodedInsn *decode) 4098{ 4099 tcg_gen_st_i32(tcg_constant_i32(-1), tcg_env, offsetof(CPUX86State, df)); 4100} 4101 4102static void gen_STI(DisasContext *s, X86DecodedInsn *decode) 4103{ 4104 gen_set_eflags(s, IF_MASK); 4105 s->base.is_jmp = DISAS_EOB_INHIBIT_IRQ; 4106} 4107 4108static void gen_VAESKEYGEN(DisasContext *s, X86DecodedInsn *decode) 4109{ 4110 TCGv_i32 imm = tcg_constant8u_i32(decode->immediate); 4111 assert(!s->vex_l); 4112 gen_helper_aeskeygenassist_xmm(tcg_env, OP_PTR0, OP_PTR1, imm); 4113} 4114 4115static void gen_STMXCSR(DisasContext *s, X86DecodedInsn *decode) 4116{ 4117 gen_helper_update_mxcsr(tcg_env); 4118 tcg_gen_ld32u_tl(s->T0, tcg_env, offsetof(CPUX86State, mxcsr)); 4119} 4120 4121static void gen_STOS(DisasContext *s, X86DecodedInsn *decode) 4122{ 4123 MemOp ot = decode->op[1].ot; 4124 gen_repz(s, ot, gen_stos); 4125} 4126 4127static void gen_SUB(DisasContext *s, X86DecodedInsn *decode) 4128{ 4129 MemOp ot = decode->op[1].ot; 4130 4131 if (s->prefix & PREFIX_LOCK) { 4132 tcg_gen_neg_tl(s->T0, s->T1); 4133 tcg_gen_atomic_fetch_add_tl(s->cc_srcT, s->A0, s->T0, 4134 s->mem_index, ot | MO_LE); 4135 tcg_gen_sub_tl(s->T0, s->cc_srcT, s->T1); 4136 } else { 4137 tcg_gen_mov_tl(s->cc_srcT, s->T0); 4138 tcg_gen_sub_tl(s->T0, s->T0, s->T1); 4139 } 4140 prepare_update2_cc(decode, s, CC_OP_SUBB + ot); 4141} 4142 4143static void gen_SYSCALL(DisasContext *s, X86DecodedInsn *decode) 4144{ 4145 gen_update_cc_op(s); 4146 gen_update_eip_cur(s); 4147 gen_helper_syscall(tcg_env, cur_insn_len_i32(s)); 4148 if (LMA(s)) { 4149 assume_cc_op(s, CC_OP_EFLAGS); 4150 } 4151 4152 /* 4153 * TF handling for the syscall insn is different. The TF bit is checked 4154 * after the syscall insn completes. This allows #DB to not be 4155 * generated after one has entered CPL0 if TF is set in FMASK. 4156 */ 4157 s->base.is_jmp = DISAS_EOB_RECHECK_TF; 4158} 4159 4160static void gen_SYSENTER(DisasContext *s, X86DecodedInsn *decode) 4161{ 4162 gen_helper_sysenter(tcg_env); 4163 s->base.is_jmp = DISAS_EOB_ONLY; 4164} 4165 4166static void gen_SYSEXIT(DisasContext *s, X86DecodedInsn *decode) 4167{ 4168 gen_helper_sysexit(tcg_env, tcg_constant_i32(s->dflag - 1)); 4169 s->base.is_jmp = DISAS_EOB_ONLY; 4170} 4171 4172static void gen_SYSRET(DisasContext *s, X86DecodedInsn *decode) 4173{ 4174 gen_helper_sysret(tcg_env, tcg_constant_i32(s->dflag - 1)); 4175 if (LMA(s)) { 4176 assume_cc_op(s, CC_OP_EFLAGS); 4177 } 4178 4179 /* 4180 * TF handling for the sysret insn is different. The TF bit is checked 4181 * after the sysret insn completes. This allows #DB to be 4182 * generated "as if" the syscall insn in userspace has just 4183 * completed. 4184 */ 4185 s->base.is_jmp = DISAS_EOB_RECHECK_TF; 4186} 4187 4188static void gen_TZCNT(DisasContext *s, X86DecodedInsn *decode) 4189{ 4190 MemOp ot = decode->op[0].ot; 4191 4192 /* C bit (cc_src) is defined related to the input. */ 4193 decode->cc_src = tcg_temp_new(); 4194 decode->cc_dst = s->T0; 4195 decode->cc_op = CC_OP_BMILGB + ot; 4196 tcg_gen_mov_tl(decode->cc_src, s->T0); 4197 4198 /* A zero input returns the operand size. */ 4199 tcg_gen_ctzi_tl(s->T0, s->T0, 8 << ot); 4200} 4201 4202static void gen_UD(DisasContext *s, X86DecodedInsn *decode) 4203{ 4204 gen_illegal_opcode(s); 4205} 4206 4207static void gen_VAESIMC(DisasContext *s, X86DecodedInsn *decode) 4208{ 4209 assert(!s->vex_l); 4210 gen_helper_aesimc_xmm(tcg_env, OP_PTR0, OP_PTR2); 4211} 4212 4213/* 4214 * 00 = v*ps Vps, Hps, Wpd 4215 * 66 = v*pd Vpd, Hpd, Wps 4216 * f3 = v*ss Vss, Hss, Wps 4217 * f2 = v*sd Vsd, Hsd, Wps 4218 */ 4219#define SSE_CMP(x) { \ 4220 gen_helper_ ## x ## ps ## _xmm, gen_helper_ ## x ## pd ## _xmm, \ 4221 gen_helper_ ## x ## ss, gen_helper_ ## x ## sd, \ 4222 gen_helper_ ## x ## ps ## _ymm, gen_helper_ ## x ## pd ## _ymm} 4223static const SSEFunc_0_eppp gen_helper_cmp_funcs[32][6] = { 4224 SSE_CMP(cmpeq), 4225 SSE_CMP(cmplt), 4226 SSE_CMP(cmple), 4227 SSE_CMP(cmpunord), 4228 SSE_CMP(cmpneq), 4229 SSE_CMP(cmpnlt), 4230 SSE_CMP(cmpnle), 4231 SSE_CMP(cmpord), 4232 4233 SSE_CMP(cmpequ), 4234 SSE_CMP(cmpnge), 4235 SSE_CMP(cmpngt), 4236 SSE_CMP(cmpfalse), 4237 SSE_CMP(cmpnequ), 4238 SSE_CMP(cmpge), 4239 SSE_CMP(cmpgt), 4240 SSE_CMP(cmptrue), 4241 4242 SSE_CMP(cmpeqs), 4243 SSE_CMP(cmpltq), 4244 SSE_CMP(cmpleq), 4245 SSE_CMP(cmpunords), 4246 SSE_CMP(cmpneqq), 4247 SSE_CMP(cmpnltq), 4248 SSE_CMP(cmpnleq), 4249 SSE_CMP(cmpords), 4250 4251 SSE_CMP(cmpequs), 4252 SSE_CMP(cmpngeq), 4253 SSE_CMP(cmpngtq), 4254 SSE_CMP(cmpfalses), 4255 SSE_CMP(cmpnequs), 4256 SSE_CMP(cmpgeq), 4257 SSE_CMP(cmpgtq), 4258 SSE_CMP(cmptrues), 4259}; 4260#undef SSE_CMP 4261 4262static void gen_VCMP(DisasContext *s, X86DecodedInsn *decode) 4263{ 4264 int index = decode->immediate & (s->prefix & PREFIX_VEX ? 31 : 7); 4265 int b = 4266 s->prefix & PREFIX_REPZ ? 2 /* ss */ : 4267 s->prefix & PREFIX_REPNZ ? 3 /* sd */ : 4268 !!(s->prefix & PREFIX_DATA) /* pd */ + (s->vex_l << 2); 4269 4270 gen_helper_cmp_funcs[index][b](tcg_env, OP_PTR0, OP_PTR1, OP_PTR2); 4271} 4272 4273static void gen_VCOMI(DisasContext *s, X86DecodedInsn *decode) 4274{ 4275 SSEFunc_0_epp fn; 4276 fn = s->prefix & PREFIX_DATA ? gen_helper_comisd : gen_helper_comiss; 4277 fn(tcg_env, OP_PTR1, OP_PTR2); 4278 assume_cc_op(s, CC_OP_EFLAGS); 4279} 4280 4281static void gen_VCVTPD2PS(DisasContext *s, X86DecodedInsn *decode) 4282{ 4283 if (s->vex_l) { 4284 gen_helper_cvtpd2ps_ymm(tcg_env, OP_PTR0, OP_PTR2); 4285 } else { 4286 gen_helper_cvtpd2ps_xmm(tcg_env, OP_PTR0, OP_PTR2); 4287 } 4288} 4289 4290static void gen_VCVTPS2PD(DisasContext *s, X86DecodedInsn *decode) 4291{ 4292 if (s->vex_l) { 4293 gen_helper_cvtps2pd_ymm(tcg_env, OP_PTR0, OP_PTR2); 4294 } else { 4295 gen_helper_cvtps2pd_xmm(tcg_env, OP_PTR0, OP_PTR2); 4296 } 4297} 4298 4299static void gen_VCVTPS2PH(DisasContext *s, X86DecodedInsn *decode) 4300{ 4301 gen_unary_imm_fp_sse(s, decode, 4302 gen_helper_cvtps2ph_xmm, 4303 gen_helper_cvtps2ph_ymm); 4304 /* 4305 * VCVTPS2PH is the only instruction that performs an operation on a 4306 * register source and then *stores* into memory. 4307 */ 4308 if (decode->op[0].has_ea) { 4309 gen_store_sse(s, decode, decode->op[0].offset); 4310 } 4311} 4312 4313static void gen_VCVTSD2SS(DisasContext *s, X86DecodedInsn *decode) 4314{ 4315 gen_helper_cvtsd2ss(tcg_env, OP_PTR0, OP_PTR1, OP_PTR2); 4316} 4317 4318static void gen_VCVTSS2SD(DisasContext *s, X86DecodedInsn *decode) 4319{ 4320 gen_helper_cvtss2sd(tcg_env, OP_PTR0, OP_PTR1, OP_PTR2); 4321} 4322 4323static void gen_VCVTSI2Sx(DisasContext *s, X86DecodedInsn *decode) 4324{ 4325 int vec_len = vector_len(s, decode); 4326 TCGv_i32 in; 4327 4328 tcg_gen_gvec_mov(MO_64, decode->op[0].offset, decode->op[1].offset, vec_len, vec_len); 4329 4330#ifdef TARGET_X86_64 4331 MemOp ot = decode->op[2].ot; 4332 if (ot == MO_64) { 4333 if (s->prefix & PREFIX_REPNZ) { 4334 gen_helper_cvtsq2sd(tcg_env, OP_PTR0, s->T1); 4335 } else { 4336 gen_helper_cvtsq2ss(tcg_env, OP_PTR0, s->T1); 4337 } 4338 return; 4339 } 4340 in = tcg_temp_new_i32(); 4341 tcg_gen_trunc_tl_i32(in, s->T1); 4342#else 4343 in = s->T1; 4344#endif 4345 4346 if (s->prefix & PREFIX_REPNZ) { 4347 gen_helper_cvtsi2sd(tcg_env, OP_PTR0, in); 4348 } else { 4349 gen_helper_cvtsi2ss(tcg_env, OP_PTR0, in); 4350 } 4351} 4352 4353static inline void gen_VCVTtSx2SI(DisasContext *s, X86DecodedInsn *decode, 4354 SSEFunc_i_ep ss2si, SSEFunc_l_ep ss2sq, 4355 SSEFunc_i_ep sd2si, SSEFunc_l_ep sd2sq) 4356{ 4357 TCGv_i32 out; 4358 4359#ifdef TARGET_X86_64 4360 MemOp ot = decode->op[0].ot; 4361 if (ot == MO_64) { 4362 if (s->prefix & PREFIX_REPNZ) { 4363 sd2sq(s->T0, tcg_env, OP_PTR2); 4364 } else { 4365 ss2sq(s->T0, tcg_env, OP_PTR2); 4366 } 4367 return; 4368 } 4369 4370 out = tcg_temp_new_i32(); 4371#else 4372 out = s->T0; 4373#endif 4374 if (s->prefix & PREFIX_REPNZ) { 4375 sd2si(out, tcg_env, OP_PTR2); 4376 } else { 4377 ss2si(out, tcg_env, OP_PTR2); 4378 } 4379#ifdef TARGET_X86_64 4380 tcg_gen_extu_i32_tl(s->T0, out); 4381#endif 4382} 4383 4384#ifndef TARGET_X86_64 4385#define gen_helper_cvtss2sq NULL 4386#define gen_helper_cvtsd2sq NULL 4387#define gen_helper_cvttss2sq NULL 4388#define gen_helper_cvttsd2sq NULL 4389#endif 4390 4391static void gen_VCVTSx2SI(DisasContext *s, X86DecodedInsn *decode) 4392{ 4393 gen_VCVTtSx2SI(s, decode, 4394 gen_helper_cvtss2si, gen_helper_cvtss2sq, 4395 gen_helper_cvtsd2si, gen_helper_cvtsd2sq); 4396} 4397 4398static void gen_VCVTTSx2SI(DisasContext *s, X86DecodedInsn *decode) 4399{ 4400 gen_VCVTtSx2SI(s, decode, 4401 gen_helper_cvttss2si, gen_helper_cvttss2sq, 4402 gen_helper_cvttsd2si, gen_helper_cvttsd2sq); 4403} 4404 4405static void gen_VEXTRACTx128(DisasContext *s, X86DecodedInsn *decode) 4406{ 4407 int mask = decode->immediate & 1; 4408 int src_ofs = vector_elem_offset(&decode->op[1], MO_128, mask); 4409 if (decode->op[0].has_ea) { 4410 /* VEX-only instruction, no alignment requirements. */ 4411 gen_sto_env_A0(s, src_ofs, false); 4412 } else { 4413 tcg_gen_gvec_mov(MO_64, decode->op[0].offset, src_ofs, 16, 16); 4414 } 4415} 4416 4417static void gen_VEXTRACTPS(DisasContext *s, X86DecodedInsn *decode) 4418{ 4419 gen_pextr(s, decode, MO_32); 4420} 4421 4422static void gen_vinsertps(DisasContext *s, X86DecodedInsn *decode, TCGv_i32 tmp) 4423{ 4424 int val = decode->immediate; 4425 int dest_word = (val >> 4) & 3; 4426 int new_mask = (val & 15) | (1 << dest_word); 4427 int vec_len = 16; 4428 4429 assert(!s->vex_l); 4430 4431 if (new_mask == 15) { 4432 /* All zeroes except possibly for the inserted element */ 4433 tcg_gen_gvec_dup_imm(MO_64, decode->op[0].offset, vec_len, vec_len, 0); 4434 } else if (decode->op[1].offset != decode->op[0].offset) { 4435 gen_store_sse(s, decode, decode->op[1].offset); 4436 } 4437 4438 if (new_mask != (val & 15)) { 4439 tcg_gen_st_i32(tmp, tcg_env, 4440 vector_elem_offset(&decode->op[0], MO_32, dest_word)); 4441 } 4442 4443 if (new_mask != 15) { 4444 TCGv_i32 zero = tcg_constant_i32(0); /* float32_zero */ 4445 int i; 4446 for (i = 0; i < 4; i++) { 4447 if ((val >> i) & 1) { 4448 tcg_gen_st_i32(zero, tcg_env, 4449 vector_elem_offset(&decode->op[0], MO_32, i)); 4450 } 4451 } 4452 } 4453} 4454 4455static void gen_VINSERTPS_r(DisasContext *s, X86DecodedInsn *decode) 4456{ 4457 int val = decode->immediate; 4458 TCGv_i32 tmp = tcg_temp_new_i32(); 4459 4460 tcg_gen_ld_i32(tmp, tcg_env, 4461 vector_elem_offset(&decode->op[2], MO_32, (val >> 6) & 3)); 4462 gen_vinsertps(s, decode, tmp); 4463} 4464 4465static void gen_VINSERTPS_m(DisasContext *s, X86DecodedInsn *decode) 4466{ 4467 TCGv_i32 tmp = tcg_temp_new_i32(); 4468 4469 tcg_gen_qemu_ld_i32(tmp, s->A0, s->mem_index, MO_LEUL); 4470 gen_vinsertps(s, decode, tmp); 4471} 4472 4473static void gen_VINSERTx128(DisasContext *s, X86DecodedInsn *decode) 4474{ 4475 int mask = decode->immediate & 1; 4476 tcg_gen_gvec_mov(MO_64, 4477 decode->op[0].offset + offsetof(YMMReg, YMM_X(mask)), 4478 decode->op[2].offset + offsetof(YMMReg, YMM_X(0)), 16, 16); 4479 tcg_gen_gvec_mov(MO_64, 4480 decode->op[0].offset + offsetof(YMMReg, YMM_X(!mask)), 4481 decode->op[1].offset + offsetof(YMMReg, YMM_X(!mask)), 16, 16); 4482} 4483 4484static inline void gen_maskmov(DisasContext *s, X86DecodedInsn *decode, 4485 SSEFunc_0_eppt xmm, SSEFunc_0_eppt ymm) 4486{ 4487 if (!s->vex_l) { 4488 xmm(tcg_env, OP_PTR2, OP_PTR1, s->A0); 4489 } else { 4490 ymm(tcg_env, OP_PTR2, OP_PTR1, s->A0); 4491 } 4492} 4493 4494static void gen_VMASKMOVPD_st(DisasContext *s, X86DecodedInsn *decode) 4495{ 4496 gen_maskmov(s, decode, gen_helper_vpmaskmovq_st_xmm, gen_helper_vpmaskmovq_st_ymm); 4497} 4498 4499static void gen_VMASKMOVPS_st(DisasContext *s, X86DecodedInsn *decode) 4500{ 4501 gen_maskmov(s, decode, gen_helper_vpmaskmovd_st_xmm, gen_helper_vpmaskmovd_st_ymm); 4502} 4503 4504static void gen_VMOVHPx_ld(DisasContext *s, X86DecodedInsn *decode) 4505{ 4506 gen_ldq_env_A0(s, decode->op[0].offset + offsetof(XMMReg, XMM_Q(1))); 4507 if (decode->op[0].offset != decode->op[1].offset) { 4508 tcg_gen_ld_i64(s->tmp1_i64, tcg_env, decode->op[1].offset + offsetof(XMMReg, XMM_Q(0))); 4509 tcg_gen_st_i64(s->tmp1_i64, tcg_env, decode->op[0].offset + offsetof(XMMReg, XMM_Q(0))); 4510 } 4511} 4512 4513static void gen_VMOVHPx_st(DisasContext *s, X86DecodedInsn *decode) 4514{ 4515 gen_stq_env_A0(s, decode->op[2].offset + offsetof(XMMReg, XMM_Q(1))); 4516} 4517 4518static void gen_VMOVHPx(DisasContext *s, X86DecodedInsn *decode) 4519{ 4520 if (decode->op[0].offset != decode->op[2].offset) { 4521 tcg_gen_ld_i64(s->tmp1_i64, tcg_env, decode->op[2].offset + offsetof(XMMReg, XMM_Q(1))); 4522 tcg_gen_st_i64(s->tmp1_i64, tcg_env, decode->op[0].offset + offsetof(XMMReg, XMM_Q(1))); 4523 } 4524 if (decode->op[0].offset != decode->op[1].offset) { 4525 tcg_gen_ld_i64(s->tmp1_i64, tcg_env, decode->op[1].offset + offsetof(XMMReg, XMM_Q(0))); 4526 tcg_gen_st_i64(s->tmp1_i64, tcg_env, decode->op[0].offset + offsetof(XMMReg, XMM_Q(0))); 4527 } 4528} 4529 4530static void gen_VMOVHLPS(DisasContext *s, X86DecodedInsn *decode) 4531{ 4532 tcg_gen_ld_i64(s->tmp1_i64, tcg_env, decode->op[2].offset + offsetof(XMMReg, XMM_Q(1))); 4533 tcg_gen_st_i64(s->tmp1_i64, tcg_env, decode->op[0].offset + offsetof(XMMReg, XMM_Q(0))); 4534 if (decode->op[0].offset != decode->op[1].offset) { 4535 tcg_gen_ld_i64(s->tmp1_i64, tcg_env, decode->op[1].offset + offsetof(XMMReg, XMM_Q(1))); 4536 tcg_gen_st_i64(s->tmp1_i64, tcg_env, decode->op[0].offset + offsetof(XMMReg, XMM_Q(1))); 4537 } 4538} 4539 4540static void gen_VMOVLHPS(DisasContext *s, X86DecodedInsn *decode) 4541{ 4542 tcg_gen_ld_i64(s->tmp1_i64, tcg_env, decode->op[2].offset); 4543 tcg_gen_st_i64(s->tmp1_i64, tcg_env, decode->op[0].offset + offsetof(XMMReg, XMM_Q(1))); 4544 if (decode->op[0].offset != decode->op[1].offset) { 4545 tcg_gen_ld_i64(s->tmp1_i64, tcg_env, decode->op[1].offset + offsetof(XMMReg, XMM_Q(0))); 4546 tcg_gen_st_i64(s->tmp1_i64, tcg_env, decode->op[0].offset + offsetof(XMMReg, XMM_Q(0))); 4547 } 4548} 4549 4550/* 4551 * Note that MOVLPx supports 256-bit operation unlike MOVHLPx, MOVLHPx, MOXHPx. 4552 * Use a gvec move to move everything above the bottom 64 bits. 4553 */ 4554 4555static void gen_VMOVLPx(DisasContext *s, X86DecodedInsn *decode) 4556{ 4557 int vec_len = vector_len(s, decode); 4558 4559 tcg_gen_ld_i64(s->tmp1_i64, tcg_env, decode->op[2].offset + offsetof(XMMReg, XMM_Q(0))); 4560 tcg_gen_gvec_mov(MO_64, decode->op[0].offset, decode->op[1].offset, vec_len, vec_len); 4561 tcg_gen_st_i64(s->tmp1_i64, tcg_env, decode->op[0].offset + offsetof(XMMReg, XMM_Q(0))); 4562} 4563 4564static void gen_VMOVLPx_ld(DisasContext *s, X86DecodedInsn *decode) 4565{ 4566 int vec_len = vector_len(s, decode); 4567 4568 tcg_gen_qemu_ld_i64(s->tmp1_i64, s->A0, s->mem_index, MO_LEUQ); 4569 tcg_gen_gvec_mov(MO_64, decode->op[0].offset, decode->op[1].offset, vec_len, vec_len); 4570 tcg_gen_st_i64(s->tmp1_i64, OP_PTR0, offsetof(ZMMReg, ZMM_Q(0))); 4571} 4572 4573static void gen_VMOVLPx_st(DisasContext *s, X86DecodedInsn *decode) 4574{ 4575 tcg_gen_ld_i64(s->tmp1_i64, OP_PTR2, offsetof(ZMMReg, ZMM_Q(0))); 4576 tcg_gen_qemu_st_i64(s->tmp1_i64, s->A0, s->mem_index, MO_LEUQ); 4577} 4578 4579static void gen_VMOVSD_ld(DisasContext *s, X86DecodedInsn *decode) 4580{ 4581 TCGv_i64 zero = tcg_constant_i64(0); 4582 4583 tcg_gen_qemu_ld_i64(s->tmp1_i64, s->A0, s->mem_index, MO_LEUQ); 4584 tcg_gen_st_i64(zero, OP_PTR0, offsetof(ZMMReg, ZMM_Q(1))); 4585 tcg_gen_st_i64(s->tmp1_i64, OP_PTR0, offsetof(ZMMReg, ZMM_Q(0))); 4586} 4587 4588static void gen_VMOVSS(DisasContext *s, X86DecodedInsn *decode) 4589{ 4590 int vec_len = vector_len(s, decode); 4591 TCGv_i32 tmp = tcg_temp_new_i32(); 4592 4593 tcg_gen_ld_i32(tmp, OP_PTR2, offsetof(ZMMReg, ZMM_L(0))); 4594 tcg_gen_gvec_mov(MO_64, decode->op[0].offset, decode->op[1].offset, vec_len, vec_len); 4595 tcg_gen_st_i32(tmp, OP_PTR0, offsetof(ZMMReg, ZMM_L(0))); 4596} 4597 4598static void gen_VMOVSS_ld(DisasContext *s, X86DecodedInsn *decode) 4599{ 4600 int vec_len = vector_len(s, decode); 4601 TCGv_i32 tmp = tcg_temp_new_i32(); 4602 4603 tcg_gen_qemu_ld_i32(tmp, s->A0, s->mem_index, MO_LEUL); 4604 tcg_gen_gvec_dup_imm(MO_64, decode->op[0].offset, vec_len, vec_len, 0); 4605 tcg_gen_st_i32(tmp, OP_PTR0, offsetof(ZMMReg, ZMM_L(0))); 4606} 4607 4608static void gen_VMOVSS_st(DisasContext *s, X86DecodedInsn *decode) 4609{ 4610 TCGv_i32 tmp = tcg_temp_new_i32(); 4611 4612 tcg_gen_ld_i32(tmp, OP_PTR2, offsetof(ZMMReg, ZMM_L(0))); 4613 tcg_gen_qemu_st_i32(tmp, s->A0, s->mem_index, MO_LEUL); 4614} 4615 4616static void gen_VPMASKMOV_st(DisasContext *s, X86DecodedInsn *decode) 4617{ 4618 if (s->vex_w) { 4619 gen_VMASKMOVPD_st(s, decode); 4620 } else { 4621 gen_VMASKMOVPS_st(s, decode); 4622 } 4623} 4624 4625static void gen_VPERMD(DisasContext *s, X86DecodedInsn *decode) 4626{ 4627 assert(s->vex_l); 4628 gen_helper_vpermd_ymm(OP_PTR0, OP_PTR1, OP_PTR2); 4629} 4630 4631static void gen_VPERM2x128(DisasContext *s, X86DecodedInsn *decode) 4632{ 4633 TCGv_i32 imm = tcg_constant8u_i32(decode->immediate); 4634 assert(s->vex_l); 4635 gen_helper_vpermdq_ymm(OP_PTR0, OP_PTR1, OP_PTR2, imm); 4636} 4637 4638static void gen_VPHMINPOSUW(DisasContext *s, X86DecodedInsn *decode) 4639{ 4640 assert(!s->vex_l); 4641 gen_helper_phminposuw_xmm(tcg_env, OP_PTR0, OP_PTR2); 4642} 4643 4644static void gen_VROUNDSD(DisasContext *s, X86DecodedInsn *decode) 4645{ 4646 TCGv_i32 imm = tcg_constant8u_i32(decode->immediate); 4647 assert(!s->vex_l); 4648 gen_helper_roundsd_xmm(tcg_env, OP_PTR0, OP_PTR1, OP_PTR2, imm); 4649} 4650 4651static void gen_VROUNDSS(DisasContext *s, X86DecodedInsn *decode) 4652{ 4653 TCGv_i32 imm = tcg_constant8u_i32(decode->immediate); 4654 assert(!s->vex_l); 4655 gen_helper_roundss_xmm(tcg_env, OP_PTR0, OP_PTR1, OP_PTR2, imm); 4656} 4657 4658static void gen_VSHUF(DisasContext *s, X86DecodedInsn *decode) 4659{ 4660 TCGv_i32 imm = tcg_constant_i32(decode->immediate); 4661 SSEFunc_0_pppi ps, pd, fn; 4662 ps = s->vex_l ? gen_helper_shufps_ymm : gen_helper_shufps_xmm; 4663 pd = s->vex_l ? gen_helper_shufpd_ymm : gen_helper_shufpd_xmm; 4664 fn = s->prefix & PREFIX_DATA ? pd : ps; 4665 fn(OP_PTR0, OP_PTR1, OP_PTR2, imm); 4666} 4667 4668static void gen_VUCOMI(DisasContext *s, X86DecodedInsn *decode) 4669{ 4670 SSEFunc_0_epp fn; 4671 fn = s->prefix & PREFIX_DATA ? gen_helper_ucomisd : gen_helper_ucomiss; 4672 fn(tcg_env, OP_PTR1, OP_PTR2); 4673 assume_cc_op(s, CC_OP_EFLAGS); 4674} 4675 4676static void gen_VZEROALL(DisasContext *s, X86DecodedInsn *decode) 4677{ 4678 TCGv_ptr ptr = tcg_temp_new_ptr(); 4679 4680 tcg_gen_addi_ptr(ptr, tcg_env, offsetof(CPUX86State, xmm_regs)); 4681 gen_helper_memset(ptr, ptr, tcg_constant_i32(0), 4682 tcg_constant_ptr(CPU_NB_REGS * sizeof(ZMMReg))); 4683} 4684 4685static void gen_VZEROUPPER(DisasContext *s, X86DecodedInsn *decode) 4686{ 4687 int i; 4688 4689 for (i = 0; i < CPU_NB_REGS; i++) { 4690 int offset = offsetof(CPUX86State, xmm_regs[i].ZMM_X(1)); 4691 tcg_gen_gvec_dup_imm(MO_64, offset, 16, 16, 0); 4692 } 4693} 4694 4695static void gen_WAIT(DisasContext *s, X86DecodedInsn *decode) 4696{ 4697 if ((s->flags & (HF_MP_MASK | HF_TS_MASK)) == (HF_MP_MASK | HF_TS_MASK)) { 4698 gen_NM_exception(s); 4699 } else { 4700 /* needs to be treated as I/O because of ferr_irq */ 4701 translator_io_start(&s->base); 4702 gen_helper_fwait(tcg_env); 4703 } 4704} 4705 4706#ifndef CONFIG_USER_ONLY 4707static void gen_WRMSR(DisasContext *s, X86DecodedInsn *decode) 4708{ 4709 gen_update_cc_op(s); 4710 gen_update_eip_cur(s); 4711 gen_helper_wrmsr(tcg_env); 4712 s->base.is_jmp = DISAS_EOB_NEXT; 4713} 4714#else 4715#define gen_WRMSR gen_unreachable 4716#endif 4717 4718static void gen_WRxxBASE(DisasContext *s, X86DecodedInsn *decode) 4719{ 4720 TCGv base = cpu_seg_base[s->modrm & 8 ? R_GS : R_FS]; 4721 4722 /* Preserve hflags bits by testing CR4 at runtime. */ 4723 gen_helper_cr4_testbit(tcg_env, tcg_constant_i32(CR4_FSGSBASE_MASK)); 4724 tcg_gen_mov_tl(base, s->T0); 4725} 4726 4727static void gen_XADD(DisasContext *s, X86DecodedInsn *decode) 4728{ 4729 MemOp ot = decode->op[1].ot; 4730 4731 decode->cc_dst = tcg_temp_new(); 4732 decode->cc_src = s->T1; 4733 decode->cc_op = CC_OP_ADDB + ot; 4734 4735 if (s->prefix & PREFIX_LOCK) { 4736 tcg_gen_atomic_fetch_add_tl(s->T0, s->A0, s->T1, s->mem_index, ot | MO_LE); 4737 tcg_gen_add_tl(decode->cc_dst, s->T0, s->T1); 4738 } else { 4739 tcg_gen_add_tl(decode->cc_dst, s->T0, s->T1); 4740 /* 4741 * NOTE: writing memory first is important for MMU exceptions, 4742 * but "new result" wins for XADD AX, AX. 4743 */ 4744 gen_writeback(s, decode, 0, decode->cc_dst); 4745 } 4746 if (decode->op[0].has_ea || decode->op[2].n != decode->op[0].n) { 4747 gen_writeback(s, decode, 2, s->T0); 4748 } 4749} 4750 4751static void gen_XCHG(DisasContext *s, X86DecodedInsn *decode) 4752{ 4753 if (s->prefix & PREFIX_LOCK) { 4754 tcg_gen_atomic_xchg_tl(s->T0, s->A0, s->T1, 4755 s->mem_index, decode->op[0].ot | MO_LE); 4756 /* now store old value into register operand */ 4757 gen_op_mov_reg_v(s, decode->op[2].ot, decode->op[2].n, s->T0); 4758 } else { 4759 /* move destination value into source operand, source preserved in T1 */ 4760 gen_op_mov_reg_v(s, decode->op[2].ot, decode->op[2].n, s->T0); 4761 tcg_gen_mov_tl(s->T0, s->T1); 4762 } 4763} 4764 4765static void gen_XLAT(DisasContext *s, X86DecodedInsn *decode) 4766{ 4767 /* AL is already zero-extended into s->T0. */ 4768 tcg_gen_add_tl(s->A0, cpu_regs[R_EBX], s->T0); 4769 gen_lea_v_seg(s, s->A0, R_DS, s->override); 4770 gen_op_ld_v(s, MO_8, s->T0, s->A0); 4771} 4772 4773static void gen_XOR(DisasContext *s, X86DecodedInsn *decode) 4774{ 4775 /* special case XOR reg, reg */ 4776 if (decode->op[1].unit == X86_OP_INT && 4777 decode->op[2].unit == X86_OP_INT && 4778 decode->op[1].n == decode->op[2].n) { 4779 tcg_gen_movi_tl(s->T0, 0); 4780 decode->cc_op = CC_OP_EFLAGS; 4781 decode->cc_src = tcg_constant_tl(CC_Z | CC_P); 4782 } else { 4783 MemOp ot = decode->op[1].ot; 4784 4785 if (s->prefix & PREFIX_LOCK) { 4786 tcg_gen_atomic_xor_fetch_tl(s->T0, s->A0, s->T1, 4787 s->mem_index, ot | MO_LE); 4788 } else { 4789 tcg_gen_xor_tl(s->T0, s->T0, s->T1); 4790 } 4791 prepare_update1_cc(decode, s, CC_OP_LOGICB + ot); 4792 } 4793} 4794 4795static void gen_XRSTOR(DisasContext *s, X86DecodedInsn *decode) 4796{ 4797 TCGv_i64 features = tcg_temp_new_i64(); 4798 4799 tcg_gen_concat_tl_i64(features, cpu_regs[R_EAX], cpu_regs[R_EDX]); 4800 gen_helper_xrstor(tcg_env, s->A0, features); 4801 if (s->cpuid_7_0_ebx_features & CPUID_7_0_EBX_MPX) { 4802 /* 4803 * XRSTOR is how MPX is enabled, which changes how 4804 * we translate. Thus we need to end the TB. 4805 */ 4806 s->base.is_jmp = DISAS_EOB_NEXT; 4807 } 4808} 4809 4810static void gen_XSAVE(DisasContext *s, X86DecodedInsn *decode) 4811{ 4812 TCGv_i64 features = tcg_temp_new_i64(); 4813 4814 tcg_gen_concat_tl_i64(features, cpu_regs[R_EAX], cpu_regs[R_EDX]); 4815 gen_helper_xsave(tcg_env, s->A0, features); 4816} 4817 4818static void gen_XSAVEOPT(DisasContext *s, X86DecodedInsn *decode) 4819{ 4820 TCGv_i64 features = tcg_temp_new_i64(); 4821 4822 tcg_gen_concat_tl_i64(features, cpu_regs[R_EAX], cpu_regs[R_EDX]); 4823 gen_helper_xsave(tcg_env, s->A0, features); 4824} 4825