1/* 2 * New-style TCG opcode generator for i386 instructions 3 * 4 * Copyright (c) 2022 Red Hat, Inc. 5 * 6 * Author: Paolo Bonzini <pbonzini@redhat.com> 7 * 8 * This library is free software; you can redistribute it and/or 9 * modify it under the terms of the GNU Lesser General Public 10 * License as published by the Free Software Foundation; either 11 * version 2.1 of the License, or (at your option) any later version. 12 * 13 * This library is distributed in the hope that it will be useful, 14 * but WITHOUT ANY WARRANTY; without even the implied warranty of 15 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU 16 * Lesser General Public License for more details. 17 * 18 * You should have received a copy of the GNU Lesser General Public 19 * License along with this library; if not, see <http://www.gnu.org/licenses/>. 20 */ 21 22/* 23 * Sometimes, knowing what the backend has can produce better code. 24 * The exact opcode to check depends on 32- vs. 64-bit. 25 */ 26#ifdef TARGET_X86_64 27#define TCG_TARGET_HAS_extract2_tl TCG_TARGET_HAS_extract2_i64 28#define TCG_TARGET_deposit_tl_valid TCG_TARGET_deposit_i64_valid 29#define TCG_TARGET_extract_tl_valid TCG_TARGET_extract_i64_valid 30#else 31#define TCG_TARGET_HAS_extract2_tl TCG_TARGET_HAS_extract2_i32 32#define TCG_TARGET_deposit_tl_valid TCG_TARGET_deposit_i32_valid 33#define TCG_TARGET_extract_tl_valid TCG_TARGET_extract_i32_valid 34#endif 35 36#define MMX_OFFSET(reg) \ 37 ({ assert((reg) >= 0 && (reg) <= 7); \ 38 offsetof(CPUX86State, fpregs[reg].mmx); }) 39 40#define ZMM_OFFSET(reg) \ 41 ({ assert((reg) >= 0 && (reg) <= 15); \ 42 offsetof(CPUX86State, xmm_regs[reg]); }) 43 44typedef void (*SSEFunc_i_ep)(TCGv_i32 val, TCGv_ptr env, TCGv_ptr reg); 45typedef void (*SSEFunc_l_ep)(TCGv_i64 val, TCGv_ptr env, TCGv_ptr reg); 46typedef void (*SSEFunc_0_epp)(TCGv_ptr env, TCGv_ptr reg_a, TCGv_ptr reg_b); 47typedef void (*SSEFunc_0_eppp)(TCGv_ptr env, TCGv_ptr reg_a, TCGv_ptr reg_b, 48 TCGv_ptr reg_c); 49typedef void (*SSEFunc_0_epppp)(TCGv_ptr env, TCGv_ptr reg_a, TCGv_ptr reg_b, 50 TCGv_ptr reg_c, TCGv_ptr reg_d); 51typedef void (*SSEFunc_0_eppi)(TCGv_ptr env, TCGv_ptr reg_a, TCGv_ptr reg_b, 52 TCGv_i32 val); 53typedef void (*SSEFunc_0_epppi)(TCGv_ptr env, TCGv_ptr reg_a, TCGv_ptr reg_b, 54 TCGv_ptr reg_c, TCGv_i32 val); 55typedef void (*SSEFunc_0_ppi)(TCGv_ptr reg_a, TCGv_ptr reg_b, TCGv_i32 val); 56typedef void (*SSEFunc_0_pppi)(TCGv_ptr reg_a, TCGv_ptr reg_b, TCGv_ptr reg_c, 57 TCGv_i32 val); 58typedef void (*SSEFunc_0_eppt)(TCGv_ptr env, TCGv_ptr reg_a, TCGv_ptr reg_b, 59 TCGv val); 60typedef void (*SSEFunc_0_epppti)(TCGv_ptr env, TCGv_ptr reg_a, TCGv_ptr reg_b, 61 TCGv_ptr reg_c, TCGv a0, TCGv_i32 scale); 62typedef void (*SSEFunc_0_eppppi)(TCGv_ptr env, TCGv_ptr reg_a, TCGv_ptr reg_b, 63 TCGv_ptr reg_c, TCGv_ptr reg_d, TCGv_i32 flags); 64typedef void (*SSEFunc_0_eppppii)(TCGv_ptr env, TCGv_ptr reg_a, TCGv_ptr reg_b, 65 TCGv_ptr reg_c, TCGv_ptr reg_d, TCGv_i32 even, 66 TCGv_i32 odd); 67 68static void gen_JMP_m(DisasContext *s, X86DecodedInsn *decode); 69static void gen_JMP(DisasContext *s, X86DecodedInsn *decode); 70 71static inline TCGv_i32 tcg_constant8u_i32(uint8_t val) 72{ 73 return tcg_constant_i32(val); 74} 75 76static void gen_NM_exception(DisasContext *s) 77{ 78 gen_exception(s, EXCP07_PREX); 79} 80 81static void gen_lea_modrm(DisasContext *s, X86DecodedInsn *decode) 82{ 83 AddressParts *mem = &decode->mem; 84 TCGv ea; 85 86 ea = gen_lea_modrm_1(s, *mem, decode->e.vex_class == 12); 87 if (decode->e.special == X86_SPECIAL_BitTest) { 88 MemOp ot = decode->op[1].ot; 89 int poslen = 8 << ot; 90 int opn = decode->op[2].n; 91 TCGv ofs = tcg_temp_new(); 92 93 /* Extract memory displacement from the second operand. */ 94 assert(decode->op[2].unit == X86_OP_INT && decode->op[2].ot != MO_8); 95 tcg_gen_sextract_tl(ofs, cpu_regs[opn], 3, poslen - 3); 96 tcg_gen_andi_tl(ofs, ofs, -1 << ot); 97 tcg_gen_add_tl(s->A0, ea, ofs); 98 ea = s->A0; 99 } 100 101 gen_lea_v_seg(s, ea, mem->def_seg, s->override); 102} 103 104static inline int mmx_offset(MemOp ot) 105{ 106 switch (ot) { 107 case MO_8: 108 return offsetof(MMXReg, MMX_B(0)); 109 case MO_16: 110 return offsetof(MMXReg, MMX_W(0)); 111 case MO_32: 112 return offsetof(MMXReg, MMX_L(0)); 113 case MO_64: 114 return offsetof(MMXReg, MMX_Q(0)); 115 default: 116 g_assert_not_reached(); 117 } 118} 119 120static inline int xmm_offset(MemOp ot) 121{ 122 switch (ot) { 123 case MO_8: 124 return offsetof(ZMMReg, ZMM_B(0)); 125 case MO_16: 126 return offsetof(ZMMReg, ZMM_W(0)); 127 case MO_32: 128 return offsetof(ZMMReg, ZMM_L(0)); 129 case MO_64: 130 return offsetof(ZMMReg, ZMM_Q(0)); 131 case MO_128: 132 return offsetof(ZMMReg, ZMM_X(0)); 133 case MO_256: 134 return offsetof(ZMMReg, ZMM_Y(0)); 135 default: 136 g_assert_not_reached(); 137 } 138} 139 140static int vector_reg_offset(X86DecodedOp *op) 141{ 142 assert(op->unit == X86_OP_MMX || op->unit == X86_OP_SSE); 143 144 if (op->unit == X86_OP_MMX) { 145 return op->offset - mmx_offset(op->ot); 146 } else { 147 return op->offset - xmm_offset(op->ot); 148 } 149} 150 151static int vector_elem_offset(X86DecodedOp *op, MemOp ot, int n) 152{ 153 int base_ofs = vector_reg_offset(op); 154 switch(ot) { 155 case MO_8: 156 if (op->unit == X86_OP_MMX) { 157 return base_ofs + offsetof(MMXReg, MMX_B(n)); 158 } else { 159 return base_ofs + offsetof(ZMMReg, ZMM_B(n)); 160 } 161 case MO_16: 162 if (op->unit == X86_OP_MMX) { 163 return base_ofs + offsetof(MMXReg, MMX_W(n)); 164 } else { 165 return base_ofs + offsetof(ZMMReg, ZMM_W(n)); 166 } 167 case MO_32: 168 if (op->unit == X86_OP_MMX) { 169 return base_ofs + offsetof(MMXReg, MMX_L(n)); 170 } else { 171 return base_ofs + offsetof(ZMMReg, ZMM_L(n)); 172 } 173 case MO_64: 174 if (op->unit == X86_OP_MMX) { 175 return base_ofs; 176 } else { 177 return base_ofs + offsetof(ZMMReg, ZMM_Q(n)); 178 } 179 case MO_128: 180 assert(op->unit == X86_OP_SSE); 181 return base_ofs + offsetof(ZMMReg, ZMM_X(n)); 182 case MO_256: 183 assert(op->unit == X86_OP_SSE); 184 return base_ofs + offsetof(ZMMReg, ZMM_Y(n)); 185 default: 186 g_assert_not_reached(); 187 } 188} 189 190static void compute_mmx_offset(X86DecodedOp *op) 191{ 192 if (!op->has_ea) { 193 op->offset = MMX_OFFSET(op->n) + mmx_offset(op->ot); 194 } else { 195 op->offset = offsetof(CPUX86State, mmx_t0) + mmx_offset(op->ot); 196 } 197} 198 199static void compute_xmm_offset(X86DecodedOp *op) 200{ 201 if (!op->has_ea) { 202 op->offset = ZMM_OFFSET(op->n) + xmm_offset(op->ot); 203 } else { 204 op->offset = offsetof(CPUX86State, xmm_t0) + xmm_offset(op->ot); 205 } 206} 207 208static void gen_load_sse(DisasContext *s, TCGv temp, MemOp ot, int dest_ofs, bool aligned) 209{ 210 switch(ot) { 211 case MO_8: 212 gen_op_ld_v(s, MO_8, temp, s->A0); 213 tcg_gen_st8_tl(temp, tcg_env, dest_ofs); 214 break; 215 case MO_16: 216 gen_op_ld_v(s, MO_16, temp, s->A0); 217 tcg_gen_st16_tl(temp, tcg_env, dest_ofs); 218 break; 219 case MO_32: 220 gen_op_ld_v(s, MO_32, temp, s->A0); 221 tcg_gen_st32_tl(temp, tcg_env, dest_ofs); 222 break; 223 case MO_64: 224 gen_ldq_env_A0(s, dest_ofs); 225 break; 226 case MO_128: 227 gen_ldo_env_A0(s, dest_ofs, aligned); 228 break; 229 case MO_256: 230 gen_ldy_env_A0(s, dest_ofs, aligned); 231 break; 232 default: 233 g_assert_not_reached(); 234 } 235} 236 237static bool sse_needs_alignment(DisasContext *s, X86DecodedInsn *decode, MemOp ot) 238{ 239 switch (decode->e.vex_class) { 240 case 2: 241 case 4: 242 if ((s->prefix & PREFIX_VEX) || 243 decode->e.vex_special == X86_VEX_SSEUnaligned) { 244 /* MOST legacy SSE instructions require aligned memory operands, but not all. */ 245 return false; 246 } 247 /* fall through */ 248 case 1: 249 return ot >= MO_128; 250 251 default: 252 return false; 253 } 254} 255 256static void gen_load(DisasContext *s, X86DecodedInsn *decode, int opn, TCGv v) 257{ 258 X86DecodedOp *op = &decode->op[opn]; 259 260 switch (op->unit) { 261 case X86_OP_SKIP: 262 return; 263 case X86_OP_SEG: 264 tcg_gen_ld32u_tl(v, tcg_env, 265 offsetof(CPUX86State,segs[op->n].selector)); 266 break; 267#ifndef CONFIG_USER_ONLY 268 case X86_OP_CR: 269 if (op->n == 8) { 270 translator_io_start(&s->base); 271 gen_helper_read_cr8(v, tcg_env); 272 } else { 273 tcg_gen_ld_tl(v, tcg_env, offsetof(CPUX86State, cr[op->n])); 274 } 275 break; 276 case X86_OP_DR: 277 /* CR4.DE tested in the helper. */ 278 gen_helper_get_dr(v, tcg_env, tcg_constant_i32(op->n)); 279 break; 280#endif 281 case X86_OP_INT: 282 if (op->has_ea) { 283 if (v == s->T0 && decode->e.special == X86_SPECIAL_SExtT0) { 284 gen_op_ld_v(s, op->ot | MO_SIGN, v, s->A0); 285 } else { 286 gen_op_ld_v(s, op->ot, v, s->A0); 287 } 288 289 } else if (op->ot == MO_8 && byte_reg_is_xH(s, op->n)) { 290 if (v == s->T0 && decode->e.special == X86_SPECIAL_SExtT0) { 291 tcg_gen_sextract_tl(v, cpu_regs[op->n - 4], 8, 8); 292 } else { 293 tcg_gen_extract_tl(v, cpu_regs[op->n - 4], 8, 8); 294 } 295 296 } else if (op->ot < MO_TL && v == s->T0 && 297 (decode->e.special == X86_SPECIAL_SExtT0 || 298 decode->e.special == X86_SPECIAL_ZExtT0)) { 299 if (decode->e.special == X86_SPECIAL_SExtT0) { 300 tcg_gen_ext_tl(v, cpu_regs[op->n], op->ot | MO_SIGN); 301 } else { 302 tcg_gen_ext_tl(v, cpu_regs[op->n], op->ot); 303 } 304 305 } else { 306 tcg_gen_mov_tl(v, cpu_regs[op->n]); 307 } 308 break; 309 case X86_OP_IMM: 310 tcg_gen_movi_tl(v, op->imm); 311 break; 312 313 case X86_OP_MMX: 314 compute_mmx_offset(op); 315 goto load_vector; 316 317 case X86_OP_SSE: 318 compute_xmm_offset(op); 319 load_vector: 320 if (op->has_ea) { 321 bool aligned = sse_needs_alignment(s, decode, op->ot); 322 gen_load_sse(s, v, op->ot, op->offset, aligned); 323 } 324 break; 325 326 default: 327 g_assert_not_reached(); 328 } 329} 330 331static TCGv_ptr op_ptr(X86DecodedInsn *decode, int opn) 332{ 333 X86DecodedOp *op = &decode->op[opn]; 334 335 assert(op->unit == X86_OP_MMX || op->unit == X86_OP_SSE); 336 if (op->v_ptr) { 337 return op->v_ptr; 338 } 339 op->v_ptr = tcg_temp_new_ptr(); 340 341 /* The temporary points to the MMXReg or ZMMReg. */ 342 tcg_gen_addi_ptr(op->v_ptr, tcg_env, vector_reg_offset(op)); 343 return op->v_ptr; 344} 345 346#define OP_PTR0 op_ptr(decode, 0) 347#define OP_PTR1 op_ptr(decode, 1) 348#define OP_PTR2 op_ptr(decode, 2) 349 350static void gen_writeback(DisasContext *s, X86DecodedInsn *decode, int opn, TCGv v) 351{ 352 X86DecodedOp *op = &decode->op[opn]; 353 switch (op->unit) { 354 case X86_OP_SKIP: 355 break; 356 case X86_OP_SEG: 357 /* Note that gen_movl_seg takes care of interrupt shadow and TF. */ 358 gen_movl_seg(s, op->n, s->T0); 359 break; 360 case X86_OP_INT: 361 if (op->has_ea) { 362 gen_op_st_v(s, op->ot, v, s->A0); 363 } else { 364 gen_op_mov_reg_v(s, op->ot, op->n, v); 365 } 366 break; 367 case X86_OP_MMX: 368 break; 369 case X86_OP_SSE: 370 if (!op->has_ea && (s->prefix & PREFIX_VEX) && op->ot <= MO_128) { 371 tcg_gen_gvec_dup_imm(MO_64, 372 offsetof(CPUX86State, xmm_regs[op->n].ZMM_X(1)), 373 16, 16, 0); 374 } 375 break; 376#ifndef CONFIG_USER_ONLY 377 case X86_OP_CR: 378 if (op->n == 8) { 379 translator_io_start(&s->base); 380 } 381 gen_helper_write_crN(tcg_env, tcg_constant_i32(op->n), v); 382 s->base.is_jmp = DISAS_EOB_NEXT; 383 break; 384 case X86_OP_DR: 385 /* CR4.DE tested in the helper. */ 386 gen_helper_set_dr(tcg_env, tcg_constant_i32(op->n), v); 387 s->base.is_jmp = DISAS_EOB_NEXT; 388 break; 389#endif 390 default: 391 g_assert_not_reached(); 392 } 393 op->unit = X86_OP_SKIP; 394} 395 396static inline int vector_len(DisasContext *s, X86DecodedInsn *decode) 397{ 398 if (decode->e.special == X86_SPECIAL_MMX && 399 !(s->prefix & (PREFIX_DATA | PREFIX_REPZ | PREFIX_REPNZ))) { 400 return 8; 401 } 402 return s->vex_l ? 32 : 16; 403} 404 405static void prepare_update1_cc(X86DecodedInsn *decode, DisasContext *s, CCOp op) 406{ 407 decode->cc_dst = s->T0; 408 decode->cc_op = op; 409} 410 411static void prepare_update2_cc(X86DecodedInsn *decode, DisasContext *s, CCOp op) 412{ 413 decode->cc_src = s->T1; 414 decode->cc_dst = s->T0; 415 decode->cc_op = op; 416} 417 418static void prepare_update_cc_incdec(X86DecodedInsn *decode, DisasContext *s, CCOp op) 419{ 420 gen_compute_eflags_c(s, s->T1); 421 prepare_update2_cc(decode, s, op); 422} 423 424static void prepare_update3_cc(X86DecodedInsn *decode, DisasContext *s, CCOp op, TCGv reg) 425{ 426 decode->cc_src2 = reg; 427 decode->cc_src = s->T1; 428 decode->cc_dst = s->T0; 429 decode->cc_op = op; 430} 431 432/* Set up decode->cc_* to modify CF while keeping other flags unchanged. */ 433static void prepare_update_cf(X86DecodedInsn *decode, DisasContext *s, TCGv cf) 434{ 435 switch (s->cc_op) { 436 case CC_OP_ADOX: 437 case CC_OP_ADCOX: 438 decode->cc_src2 = cpu_cc_src2; 439 decode->cc_src = cpu_cc_src; 440 decode->cc_op = CC_OP_ADCOX; 441 break; 442 443 case CC_OP_EFLAGS: 444 case CC_OP_ADCX: 445 decode->cc_src = cpu_cc_src; 446 decode->cc_op = CC_OP_ADCX; 447 break; 448 449 default: 450 decode->cc_src = tcg_temp_new(); 451 gen_mov_eflags(s, decode->cc_src); 452 decode->cc_op = CC_OP_ADCX; 453 break; 454 } 455 decode->cc_dst = cf; 456} 457 458static void gen_store_sse(DisasContext *s, X86DecodedInsn *decode, int src_ofs) 459{ 460 MemOp ot = decode->op[0].ot; 461 int vec_len = vector_len(s, decode); 462 bool aligned = sse_needs_alignment(s, decode, ot); 463 464 if (!decode->op[0].has_ea) { 465 tcg_gen_gvec_mov(MO_64, decode->op[0].offset, src_ofs, vec_len, vec_len); 466 return; 467 } 468 469 switch (ot) { 470 case MO_64: 471 gen_stq_env_A0(s, src_ofs); 472 break; 473 case MO_128: 474 gen_sto_env_A0(s, src_ofs, aligned); 475 break; 476 case MO_256: 477 gen_sty_env_A0(s, src_ofs, aligned); 478 break; 479 default: 480 g_assert_not_reached(); 481 } 482} 483 484static void gen_helper_pavgusb(TCGv_ptr env, TCGv_ptr reg_a, TCGv_ptr reg_b) 485{ 486 gen_helper_pavgb_mmx(env, reg_a, reg_a, reg_b); 487} 488 489#define FN_3DNOW_MOVE ((SSEFunc_0_epp) (uintptr_t) 1) 490static const SSEFunc_0_epp fns_3dnow[] = { 491 [0x0c] = gen_helper_pi2fw, 492 [0x0d] = gen_helper_pi2fd, 493 [0x1c] = gen_helper_pf2iw, 494 [0x1d] = gen_helper_pf2id, 495 [0x8a] = gen_helper_pfnacc, 496 [0x8e] = gen_helper_pfpnacc, 497 [0x90] = gen_helper_pfcmpge, 498 [0x94] = gen_helper_pfmin, 499 [0x96] = gen_helper_pfrcp, 500 [0x97] = gen_helper_pfrsqrt, 501 [0x9a] = gen_helper_pfsub, 502 [0x9e] = gen_helper_pfadd, 503 [0xa0] = gen_helper_pfcmpgt, 504 [0xa4] = gen_helper_pfmax, 505 [0xa6] = FN_3DNOW_MOVE, /* PFRCPIT1; no need to actually increase precision */ 506 [0xa7] = FN_3DNOW_MOVE, /* PFRSQIT1 */ 507 [0xb6] = FN_3DNOW_MOVE, /* PFRCPIT2 */ 508 [0xaa] = gen_helper_pfsubr, 509 [0xae] = gen_helper_pfacc, 510 [0xb0] = gen_helper_pfcmpeq, 511 [0xb4] = gen_helper_pfmul, 512 [0xb7] = gen_helper_pmulhrw_mmx, 513 [0xbb] = gen_helper_pswapd, 514 [0xbf] = gen_helper_pavgusb, 515}; 516 517static void gen_3dnow(DisasContext *s, X86DecodedInsn *decode) 518{ 519 uint8_t b = decode->immediate; 520 SSEFunc_0_epp fn = b < ARRAY_SIZE(fns_3dnow) ? fns_3dnow[b] : NULL; 521 522 if (!fn) { 523 gen_illegal_opcode(s); 524 return; 525 } 526 if (s->flags & HF_TS_MASK) { 527 gen_NM_exception(s); 528 return; 529 } 530 if (s->flags & HF_EM_MASK) { 531 gen_illegal_opcode(s); 532 return; 533 } 534 535 gen_helper_enter_mmx(tcg_env); 536 if (fn == FN_3DNOW_MOVE) { 537 tcg_gen_ld_i64(s->tmp1_i64, tcg_env, decode->op[1].offset); 538 tcg_gen_st_i64(s->tmp1_i64, tcg_env, decode->op[0].offset); 539 } else { 540 fn(tcg_env, OP_PTR0, OP_PTR1); 541 } 542} 543 544/* 545 * 00 = v*ps Vps, Hps, Wpd 546 * 66 = v*pd Vpd, Hpd, Wps 547 * f3 = v*ss Vss, Hss, Wps 548 * f2 = v*sd Vsd, Hsd, Wps 549 */ 550static inline void gen_unary_fp_sse(DisasContext *s, X86DecodedInsn *decode, 551 SSEFunc_0_epp pd_xmm, SSEFunc_0_epp ps_xmm, 552 SSEFunc_0_epp pd_ymm, SSEFunc_0_epp ps_ymm, 553 SSEFunc_0_eppp sd, SSEFunc_0_eppp ss) 554{ 555 if ((s->prefix & (PREFIX_REPZ | PREFIX_REPNZ)) != 0) { 556 SSEFunc_0_eppp fn = s->prefix & PREFIX_REPZ ? ss : sd; 557 if (!fn) { 558 gen_illegal_opcode(s); 559 return; 560 } 561 fn(tcg_env, OP_PTR0, OP_PTR1, OP_PTR2); 562 } else { 563 SSEFunc_0_epp ps, pd, fn; 564 ps = s->vex_l ? ps_ymm : ps_xmm; 565 pd = s->vex_l ? pd_ymm : pd_xmm; 566 fn = s->prefix & PREFIX_DATA ? pd : ps; 567 if (!fn) { 568 gen_illegal_opcode(s); 569 return; 570 } 571 fn(tcg_env, OP_PTR0, OP_PTR2); 572 } 573} 574#define UNARY_FP_SSE(uname, lname) \ 575static void gen_##uname(DisasContext *s, X86DecodedInsn *decode) \ 576{ \ 577 gen_unary_fp_sse(s, decode, \ 578 gen_helper_##lname##pd_xmm, \ 579 gen_helper_##lname##ps_xmm, \ 580 gen_helper_##lname##pd_ymm, \ 581 gen_helper_##lname##ps_ymm, \ 582 gen_helper_##lname##sd, \ 583 gen_helper_##lname##ss); \ 584} 585UNARY_FP_SSE(VSQRT, sqrt) 586 587/* 588 * 00 = v*ps Vps, Hps, Wpd 589 * 66 = v*pd Vpd, Hpd, Wps 590 * f3 = v*ss Vss, Hss, Wps 591 * f2 = v*sd Vsd, Hsd, Wps 592 */ 593static inline void gen_fp_sse(DisasContext *s, X86DecodedInsn *decode, 594 SSEFunc_0_eppp pd_xmm, SSEFunc_0_eppp ps_xmm, 595 SSEFunc_0_eppp pd_ymm, SSEFunc_0_eppp ps_ymm, 596 SSEFunc_0_eppp sd, SSEFunc_0_eppp ss) 597{ 598 SSEFunc_0_eppp ps, pd, fn; 599 if ((s->prefix & (PREFIX_REPZ | PREFIX_REPNZ)) != 0) { 600 fn = s->prefix & PREFIX_REPZ ? ss : sd; 601 } else { 602 ps = s->vex_l ? ps_ymm : ps_xmm; 603 pd = s->vex_l ? pd_ymm : pd_xmm; 604 fn = s->prefix & PREFIX_DATA ? pd : ps; 605 } 606 if (fn) { 607 fn(tcg_env, OP_PTR0, OP_PTR1, OP_PTR2); 608 } else { 609 gen_illegal_opcode(s); 610 } 611} 612 613#define FP_SSE(uname, lname) \ 614static void gen_##uname(DisasContext *s, X86DecodedInsn *decode) \ 615{ \ 616 gen_fp_sse(s, decode, \ 617 gen_helper_##lname##pd_xmm, \ 618 gen_helper_##lname##ps_xmm, \ 619 gen_helper_##lname##pd_ymm, \ 620 gen_helper_##lname##ps_ymm, \ 621 gen_helper_##lname##sd, \ 622 gen_helper_##lname##ss); \ 623} 624FP_SSE(VADD, add) 625FP_SSE(VMUL, mul) 626FP_SSE(VSUB, sub) 627FP_SSE(VMIN, min) 628FP_SSE(VDIV, div) 629FP_SSE(VMAX, max) 630 631#define FMA_SSE_PACKED(uname, ptr0, ptr1, ptr2, even, odd) \ 632static void gen_##uname##Px(DisasContext *s, X86DecodedInsn *decode) \ 633{ \ 634 SSEFunc_0_eppppii xmm = s->vex_w ? gen_helper_fma4pd_xmm : gen_helper_fma4ps_xmm; \ 635 SSEFunc_0_eppppii ymm = s->vex_w ? gen_helper_fma4pd_ymm : gen_helper_fma4ps_ymm; \ 636 SSEFunc_0_eppppii fn = s->vex_l ? ymm : xmm; \ 637 \ 638 fn(tcg_env, OP_PTR0, ptr0, ptr1, ptr2, \ 639 tcg_constant_i32(even), \ 640 tcg_constant_i32((even) ^ (odd))); \ 641} 642 643#define FMA_SSE(uname, ptr0, ptr1, ptr2, flags) \ 644FMA_SSE_PACKED(uname, ptr0, ptr1, ptr2, flags, flags) \ 645static void gen_##uname##Sx(DisasContext *s, X86DecodedInsn *decode) \ 646{ \ 647 SSEFunc_0_eppppi fn = s->vex_w ? gen_helper_fma4sd : gen_helper_fma4ss; \ 648 \ 649 fn(tcg_env, OP_PTR0, ptr0, ptr1, ptr2, \ 650 tcg_constant_i32(flags)); \ 651} \ 652 653FMA_SSE(VFMADD231, OP_PTR1, OP_PTR2, OP_PTR0, 0) 654FMA_SSE(VFMADD213, OP_PTR1, OP_PTR0, OP_PTR2, 0) 655FMA_SSE(VFMADD132, OP_PTR0, OP_PTR2, OP_PTR1, 0) 656 657FMA_SSE(VFNMADD231, OP_PTR1, OP_PTR2, OP_PTR0, float_muladd_negate_product) 658FMA_SSE(VFNMADD213, OP_PTR1, OP_PTR0, OP_PTR2, float_muladd_negate_product) 659FMA_SSE(VFNMADD132, OP_PTR0, OP_PTR2, OP_PTR1, float_muladd_negate_product) 660 661FMA_SSE(VFMSUB231, OP_PTR1, OP_PTR2, OP_PTR0, float_muladd_negate_c) 662FMA_SSE(VFMSUB213, OP_PTR1, OP_PTR0, OP_PTR2, float_muladd_negate_c) 663FMA_SSE(VFMSUB132, OP_PTR0, OP_PTR2, OP_PTR1, float_muladd_negate_c) 664 665FMA_SSE(VFNMSUB231, OP_PTR1, OP_PTR2, OP_PTR0, float_muladd_negate_c|float_muladd_negate_product) 666FMA_SSE(VFNMSUB213, OP_PTR1, OP_PTR0, OP_PTR2, float_muladd_negate_c|float_muladd_negate_product) 667FMA_SSE(VFNMSUB132, OP_PTR0, OP_PTR2, OP_PTR1, float_muladd_negate_c|float_muladd_negate_product) 668 669FMA_SSE_PACKED(VFMADDSUB231, OP_PTR1, OP_PTR2, OP_PTR0, float_muladd_negate_c, 0) 670FMA_SSE_PACKED(VFMADDSUB213, OP_PTR1, OP_PTR0, OP_PTR2, float_muladd_negate_c, 0) 671FMA_SSE_PACKED(VFMADDSUB132, OP_PTR0, OP_PTR2, OP_PTR1, float_muladd_negate_c, 0) 672 673FMA_SSE_PACKED(VFMSUBADD231, OP_PTR1, OP_PTR2, OP_PTR0, 0, float_muladd_negate_c) 674FMA_SSE_PACKED(VFMSUBADD213, OP_PTR1, OP_PTR0, OP_PTR2, 0, float_muladd_negate_c) 675FMA_SSE_PACKED(VFMSUBADD132, OP_PTR0, OP_PTR2, OP_PTR1, 0, float_muladd_negate_c) 676 677#define FP_UNPACK_SSE(uname, lname) \ 678static void gen_##uname(DisasContext *s, X86DecodedInsn *decode) \ 679{ \ 680 /* PS maps to the DQ integer instruction, PD maps to QDQ. */ \ 681 gen_fp_sse(s, decode, \ 682 gen_helper_##lname##qdq_xmm, \ 683 gen_helper_##lname##dq_xmm, \ 684 gen_helper_##lname##qdq_ymm, \ 685 gen_helper_##lname##dq_ymm, \ 686 NULL, NULL); \ 687} 688FP_UNPACK_SSE(VUNPCKLPx, punpckl) 689FP_UNPACK_SSE(VUNPCKHPx, punpckh) 690 691/* 692 * 00 = v*ps Vps, Wpd 693 * f3 = v*ss Vss, Wps 694 */ 695static inline void gen_unary_fp32_sse(DisasContext *s, X86DecodedInsn *decode, 696 SSEFunc_0_epp ps_xmm, 697 SSEFunc_0_epp ps_ymm, 698 SSEFunc_0_eppp ss) 699{ 700 if ((s->prefix & (PREFIX_DATA | PREFIX_REPNZ)) != 0) { 701 goto illegal_op; 702 } else if (s->prefix & PREFIX_REPZ) { 703 if (!ss) { 704 goto illegal_op; 705 } 706 ss(tcg_env, OP_PTR0, OP_PTR1, OP_PTR2); 707 } else { 708 SSEFunc_0_epp fn = s->vex_l ? ps_ymm : ps_xmm; 709 if (!fn) { 710 goto illegal_op; 711 } 712 fn(tcg_env, OP_PTR0, OP_PTR2); 713 } 714 return; 715 716illegal_op: 717 gen_illegal_opcode(s); 718} 719#define UNARY_FP32_SSE(uname, lname) \ 720static void gen_##uname(DisasContext *s, X86DecodedInsn *decode) \ 721{ \ 722 gen_unary_fp32_sse(s, decode, \ 723 gen_helper_##lname##ps_xmm, \ 724 gen_helper_##lname##ps_ymm, \ 725 gen_helper_##lname##ss); \ 726} 727UNARY_FP32_SSE(VRSQRT, rsqrt) 728UNARY_FP32_SSE(VRCP, rcp) 729 730/* 731 * 66 = v*pd Vpd, Hpd, Wpd 732 * f2 = v*ps Vps, Hps, Wps 733 */ 734static inline void gen_horizontal_fp_sse(DisasContext *s, X86DecodedInsn *decode, 735 SSEFunc_0_eppp pd_xmm, SSEFunc_0_eppp ps_xmm, 736 SSEFunc_0_eppp pd_ymm, SSEFunc_0_eppp ps_ymm) 737{ 738 SSEFunc_0_eppp ps, pd, fn; 739 ps = s->vex_l ? ps_ymm : ps_xmm; 740 pd = s->vex_l ? pd_ymm : pd_xmm; 741 fn = s->prefix & PREFIX_DATA ? pd : ps; 742 fn(tcg_env, OP_PTR0, OP_PTR1, OP_PTR2); 743} 744#define HORIZONTAL_FP_SSE(uname, lname) \ 745static void gen_##uname(DisasContext *s, X86DecodedInsn *decode) \ 746{ \ 747 gen_horizontal_fp_sse(s, decode, \ 748 gen_helper_##lname##pd_xmm, gen_helper_##lname##ps_xmm, \ 749 gen_helper_##lname##pd_ymm, gen_helper_##lname##ps_ymm); \ 750} 751HORIZONTAL_FP_SSE(VHADD, hadd) 752HORIZONTAL_FP_SSE(VHSUB, hsub) 753HORIZONTAL_FP_SSE(VADDSUB, addsub) 754 755static inline void gen_ternary_sse(DisasContext *s, X86DecodedInsn *decode, 756 int op3, SSEFunc_0_epppp xmm, SSEFunc_0_epppp ymm) 757{ 758 SSEFunc_0_epppp fn = s->vex_l ? ymm : xmm; 759 TCGv_ptr ptr3 = tcg_temp_new_ptr(); 760 761 /* The format of the fourth input is Lx */ 762 tcg_gen_addi_ptr(ptr3, tcg_env, ZMM_OFFSET(op3)); 763 fn(tcg_env, OP_PTR0, OP_PTR1, OP_PTR2, ptr3); 764} 765#define TERNARY_SSE(uname, uvname, lname) \ 766static void gen_##uvname(DisasContext *s, X86DecodedInsn *decode) \ 767{ \ 768 gen_ternary_sse(s, decode, (uint8_t)decode->immediate >> 4, \ 769 gen_helper_##lname##_xmm, gen_helper_##lname##_ymm); \ 770} \ 771static void gen_##uname(DisasContext *s, X86DecodedInsn *decode) \ 772{ \ 773 gen_ternary_sse(s, decode, 0, \ 774 gen_helper_##lname##_xmm, gen_helper_##lname##_ymm); \ 775} 776TERNARY_SSE(BLENDVPS, VBLENDVPS, blendvps) 777TERNARY_SSE(BLENDVPD, VBLENDVPD, blendvpd) 778TERNARY_SSE(PBLENDVB, VPBLENDVB, pblendvb) 779 780static inline void gen_binary_imm_sse(DisasContext *s, X86DecodedInsn *decode, 781 SSEFunc_0_epppi xmm, SSEFunc_0_epppi ymm) 782{ 783 TCGv_i32 imm = tcg_constant8u_i32(decode->immediate); 784 if (!s->vex_l) { 785 xmm(tcg_env, OP_PTR0, OP_PTR1, OP_PTR2, imm); 786 } else { 787 ymm(tcg_env, OP_PTR0, OP_PTR1, OP_PTR2, imm); 788 } 789} 790 791#define BINARY_IMM_SSE(uname, lname) \ 792static void gen_##uname(DisasContext *s, X86DecodedInsn *decode) \ 793{ \ 794 gen_binary_imm_sse(s, decode, \ 795 gen_helper_##lname##_xmm, \ 796 gen_helper_##lname##_ymm); \ 797} 798 799BINARY_IMM_SSE(VBLENDPD, blendpd) 800BINARY_IMM_SSE(VBLENDPS, blendps) 801BINARY_IMM_SSE(VPBLENDW, pblendw) 802BINARY_IMM_SSE(VDDPS, dpps) 803#define gen_helper_dppd_ymm NULL 804BINARY_IMM_SSE(VDDPD, dppd) 805BINARY_IMM_SSE(VMPSADBW, mpsadbw) 806BINARY_IMM_SSE(PCLMULQDQ, pclmulqdq) 807 808 809#define UNARY_INT_GVEC(uname, func, ...) \ 810static void gen_##uname(DisasContext *s, X86DecodedInsn *decode) \ 811{ \ 812 int vec_len = vector_len(s, decode); \ 813 \ 814 func(__VA_ARGS__, decode->op[0].offset, \ 815 decode->op[2].offset, vec_len, vec_len); \ 816} 817UNARY_INT_GVEC(PABSB, tcg_gen_gvec_abs, MO_8) 818UNARY_INT_GVEC(PABSW, tcg_gen_gvec_abs, MO_16) 819UNARY_INT_GVEC(PABSD, tcg_gen_gvec_abs, MO_32) 820UNARY_INT_GVEC(VBROADCASTx128, tcg_gen_gvec_dup_mem, MO_128) 821UNARY_INT_GVEC(VPBROADCASTB, tcg_gen_gvec_dup_mem, MO_8) 822UNARY_INT_GVEC(VPBROADCASTW, tcg_gen_gvec_dup_mem, MO_16) 823UNARY_INT_GVEC(VPBROADCASTD, tcg_gen_gvec_dup_mem, MO_32) 824UNARY_INT_GVEC(VPBROADCASTQ, tcg_gen_gvec_dup_mem, MO_64) 825 826 827#define BINARY_INT_GVEC(uname, func, ...) \ 828static void gen_##uname(DisasContext *s, X86DecodedInsn *decode) \ 829{ \ 830 int vec_len = vector_len(s, decode); \ 831 \ 832 func(__VA_ARGS__, \ 833 decode->op[0].offset, decode->op[1].offset, \ 834 decode->op[2].offset, vec_len, vec_len); \ 835} 836 837BINARY_INT_GVEC(PADDB, tcg_gen_gvec_add, MO_8) 838BINARY_INT_GVEC(PADDW, tcg_gen_gvec_add, MO_16) 839BINARY_INT_GVEC(PADDD, tcg_gen_gvec_add, MO_32) 840BINARY_INT_GVEC(PADDQ, tcg_gen_gvec_add, MO_64) 841BINARY_INT_GVEC(PADDSB, tcg_gen_gvec_ssadd, MO_8) 842BINARY_INT_GVEC(PADDSW, tcg_gen_gvec_ssadd, MO_16) 843BINARY_INT_GVEC(PADDUSB, tcg_gen_gvec_usadd, MO_8) 844BINARY_INT_GVEC(PADDUSW, tcg_gen_gvec_usadd, MO_16) 845BINARY_INT_GVEC(PAND, tcg_gen_gvec_and, MO_64) 846BINARY_INT_GVEC(PCMPEQB, tcg_gen_gvec_cmp, TCG_COND_EQ, MO_8) 847BINARY_INT_GVEC(PCMPEQD, tcg_gen_gvec_cmp, TCG_COND_EQ, MO_32) 848BINARY_INT_GVEC(PCMPEQW, tcg_gen_gvec_cmp, TCG_COND_EQ, MO_16) 849BINARY_INT_GVEC(PCMPEQQ, tcg_gen_gvec_cmp, TCG_COND_EQ, MO_64) 850BINARY_INT_GVEC(PCMPGTB, tcg_gen_gvec_cmp, TCG_COND_GT, MO_8) 851BINARY_INT_GVEC(PCMPGTW, tcg_gen_gvec_cmp, TCG_COND_GT, MO_16) 852BINARY_INT_GVEC(PCMPGTD, tcg_gen_gvec_cmp, TCG_COND_GT, MO_32) 853BINARY_INT_GVEC(PCMPGTQ, tcg_gen_gvec_cmp, TCG_COND_GT, MO_64) 854BINARY_INT_GVEC(PMAXSB, tcg_gen_gvec_smax, MO_8) 855BINARY_INT_GVEC(PMAXSW, tcg_gen_gvec_smax, MO_16) 856BINARY_INT_GVEC(PMAXSD, tcg_gen_gvec_smax, MO_32) 857BINARY_INT_GVEC(PMAXUB, tcg_gen_gvec_umax, MO_8) 858BINARY_INT_GVEC(PMAXUW, tcg_gen_gvec_umax, MO_16) 859BINARY_INT_GVEC(PMAXUD, tcg_gen_gvec_umax, MO_32) 860BINARY_INT_GVEC(PMINSB, tcg_gen_gvec_smin, MO_8) 861BINARY_INT_GVEC(PMINSW, tcg_gen_gvec_smin, MO_16) 862BINARY_INT_GVEC(PMINSD, tcg_gen_gvec_smin, MO_32) 863BINARY_INT_GVEC(PMINUB, tcg_gen_gvec_umin, MO_8) 864BINARY_INT_GVEC(PMINUW, tcg_gen_gvec_umin, MO_16) 865BINARY_INT_GVEC(PMINUD, tcg_gen_gvec_umin, MO_32) 866BINARY_INT_GVEC(PMULLW, tcg_gen_gvec_mul, MO_16) 867BINARY_INT_GVEC(PMULLD, tcg_gen_gvec_mul, MO_32) 868BINARY_INT_GVEC(POR, tcg_gen_gvec_or, MO_64) 869BINARY_INT_GVEC(PSUBB, tcg_gen_gvec_sub, MO_8) 870BINARY_INT_GVEC(PSUBW, tcg_gen_gvec_sub, MO_16) 871BINARY_INT_GVEC(PSUBD, tcg_gen_gvec_sub, MO_32) 872BINARY_INT_GVEC(PSUBQ, tcg_gen_gvec_sub, MO_64) 873BINARY_INT_GVEC(PSUBSB, tcg_gen_gvec_sssub, MO_8) 874BINARY_INT_GVEC(PSUBSW, tcg_gen_gvec_sssub, MO_16) 875BINARY_INT_GVEC(PSUBUSB, tcg_gen_gvec_ussub, MO_8) 876BINARY_INT_GVEC(PSUBUSW, tcg_gen_gvec_ussub, MO_16) 877BINARY_INT_GVEC(PXOR, tcg_gen_gvec_xor, MO_64) 878 879 880/* 881 * 00 = p* Pq, Qq (if mmx not NULL; no VEX) 882 * 66 = vp* Vx, Hx, Wx 883 * 884 * These are really the same encoding, because 1) V is the same as P when VEX.V 885 * is not present 2) P and Q are the same as H and W apart from MM/XMM 886 */ 887static inline void gen_binary_int_sse(DisasContext *s, X86DecodedInsn *decode, 888 SSEFunc_0_eppp mmx, SSEFunc_0_eppp xmm, SSEFunc_0_eppp ymm) 889{ 890 assert(!!mmx == !!(decode->e.special == X86_SPECIAL_MMX)); 891 892 if (mmx && (s->prefix & PREFIX_VEX) && !(s->prefix & PREFIX_DATA)) { 893 /* VEX encoding is not applicable to MMX instructions. */ 894 gen_illegal_opcode(s); 895 return; 896 } 897 if (!(s->prefix & PREFIX_DATA)) { 898 mmx(tcg_env, OP_PTR0, OP_PTR1, OP_PTR2); 899 } else if (!s->vex_l) { 900 xmm(tcg_env, OP_PTR0, OP_PTR1, OP_PTR2); 901 } else { 902 ymm(tcg_env, OP_PTR0, OP_PTR1, OP_PTR2); 903 } 904} 905 906 907#define BINARY_INT_MMX(uname, lname) \ 908static void gen_##uname(DisasContext *s, X86DecodedInsn *decode) \ 909{ \ 910 gen_binary_int_sse(s, decode, \ 911 gen_helper_##lname##_mmx, \ 912 gen_helper_##lname##_xmm, \ 913 gen_helper_##lname##_ymm); \ 914} 915BINARY_INT_MMX(PUNPCKLBW, punpcklbw) 916BINARY_INT_MMX(PUNPCKLWD, punpcklwd) 917BINARY_INT_MMX(PUNPCKLDQ, punpckldq) 918BINARY_INT_MMX(PACKSSWB, packsswb) 919BINARY_INT_MMX(PACKUSWB, packuswb) 920BINARY_INT_MMX(PUNPCKHBW, punpckhbw) 921BINARY_INT_MMX(PUNPCKHWD, punpckhwd) 922BINARY_INT_MMX(PUNPCKHDQ, punpckhdq) 923BINARY_INT_MMX(PACKSSDW, packssdw) 924 925BINARY_INT_MMX(PAVGB, pavgb) 926BINARY_INT_MMX(PAVGW, pavgw) 927BINARY_INT_MMX(PMADDWD, pmaddwd) 928BINARY_INT_MMX(PMULHUW, pmulhuw) 929BINARY_INT_MMX(PMULHW, pmulhw) 930BINARY_INT_MMX(PMULUDQ, pmuludq) 931BINARY_INT_MMX(PSADBW, psadbw) 932 933BINARY_INT_MMX(PSLLW_r, psllw) 934BINARY_INT_MMX(PSLLD_r, pslld) 935BINARY_INT_MMX(PSLLQ_r, psllq) 936BINARY_INT_MMX(PSRLW_r, psrlw) 937BINARY_INT_MMX(PSRLD_r, psrld) 938BINARY_INT_MMX(PSRLQ_r, psrlq) 939BINARY_INT_MMX(PSRAW_r, psraw) 940BINARY_INT_MMX(PSRAD_r, psrad) 941 942BINARY_INT_MMX(PHADDW, phaddw) 943BINARY_INT_MMX(PHADDSW, phaddsw) 944BINARY_INT_MMX(PHADDD, phaddd) 945BINARY_INT_MMX(PHSUBW, phsubw) 946BINARY_INT_MMX(PHSUBSW, phsubsw) 947BINARY_INT_MMX(PHSUBD, phsubd) 948BINARY_INT_MMX(PMADDUBSW, pmaddubsw) 949BINARY_INT_MMX(PSHUFB, pshufb) 950BINARY_INT_MMX(PSIGNB, psignb) 951BINARY_INT_MMX(PSIGNW, psignw) 952BINARY_INT_MMX(PSIGND, psignd) 953BINARY_INT_MMX(PMULHRSW, pmulhrsw) 954 955/* Instructions with no MMX equivalent. */ 956#define BINARY_INT_SSE(uname, lname) \ 957static void gen_##uname(DisasContext *s, X86DecodedInsn *decode) \ 958{ \ 959 gen_binary_int_sse(s, decode, \ 960 NULL, \ 961 gen_helper_##lname##_xmm, \ 962 gen_helper_##lname##_ymm); \ 963} 964 965/* Instructions with no MMX equivalent. */ 966BINARY_INT_SSE(PUNPCKLQDQ, punpcklqdq) 967BINARY_INT_SSE(PUNPCKHQDQ, punpckhqdq) 968BINARY_INT_SSE(VPACKUSDW, packusdw) 969BINARY_INT_SSE(VPERMILPS, vpermilps) 970BINARY_INT_SSE(VPERMILPD, vpermilpd) 971BINARY_INT_SSE(VMASKMOVPS, vpmaskmovd) 972BINARY_INT_SSE(VMASKMOVPD, vpmaskmovq) 973 974BINARY_INT_SSE(PMULDQ, pmuldq) 975 976BINARY_INT_SSE(VAESDEC, aesdec) 977BINARY_INT_SSE(VAESDECLAST, aesdeclast) 978BINARY_INT_SSE(VAESENC, aesenc) 979BINARY_INT_SSE(VAESENCLAST, aesenclast) 980 981#define UNARY_CMP_SSE(uname, lname) \ 982static void gen_##uname(DisasContext *s, X86DecodedInsn *decode) \ 983{ \ 984 if (!s->vex_l) { \ 985 gen_helper_##lname##_xmm(tcg_env, OP_PTR1, OP_PTR2); \ 986 } else { \ 987 gen_helper_##lname##_ymm(tcg_env, OP_PTR1, OP_PTR2); \ 988 } \ 989 assume_cc_op(s, CC_OP_EFLAGS); \ 990} 991UNARY_CMP_SSE(VPTEST, ptest) 992UNARY_CMP_SSE(VTESTPS, vtestps) 993UNARY_CMP_SSE(VTESTPD, vtestpd) 994 995static inline void gen_unary_int_sse(DisasContext *s, X86DecodedInsn *decode, 996 SSEFunc_0_epp xmm, SSEFunc_0_epp ymm) 997{ 998 if (!s->vex_l) { 999 xmm(tcg_env, OP_PTR0, OP_PTR2); 1000 } else { 1001 ymm(tcg_env, OP_PTR0, OP_PTR2); 1002 } 1003} 1004 1005#define UNARY_INT_SSE(uname, lname) \ 1006static void gen_##uname(DisasContext *s, X86DecodedInsn *decode) \ 1007{ \ 1008 gen_unary_int_sse(s, decode, \ 1009 gen_helper_##lname##_xmm, \ 1010 gen_helper_##lname##_ymm); \ 1011} 1012 1013UNARY_INT_SSE(VPMOVSXBW, pmovsxbw) 1014UNARY_INT_SSE(VPMOVSXBD, pmovsxbd) 1015UNARY_INT_SSE(VPMOVSXBQ, pmovsxbq) 1016UNARY_INT_SSE(VPMOVSXWD, pmovsxwd) 1017UNARY_INT_SSE(VPMOVSXWQ, pmovsxwq) 1018UNARY_INT_SSE(VPMOVSXDQ, pmovsxdq) 1019 1020UNARY_INT_SSE(VPMOVZXBW, pmovzxbw) 1021UNARY_INT_SSE(VPMOVZXBD, pmovzxbd) 1022UNARY_INT_SSE(VPMOVZXBQ, pmovzxbq) 1023UNARY_INT_SSE(VPMOVZXWD, pmovzxwd) 1024UNARY_INT_SSE(VPMOVZXWQ, pmovzxwq) 1025UNARY_INT_SSE(VPMOVZXDQ, pmovzxdq) 1026 1027UNARY_INT_SSE(VMOVSLDUP, pmovsldup) 1028UNARY_INT_SSE(VMOVSHDUP, pmovshdup) 1029UNARY_INT_SSE(VMOVDDUP, pmovdldup) 1030 1031UNARY_INT_SSE(VCVTDQ2PD, cvtdq2pd) 1032UNARY_INT_SSE(VCVTPD2DQ, cvtpd2dq) 1033UNARY_INT_SSE(VCVTTPD2DQ, cvttpd2dq) 1034UNARY_INT_SSE(VCVTDQ2PS, cvtdq2ps) 1035UNARY_INT_SSE(VCVTPS2DQ, cvtps2dq) 1036UNARY_INT_SSE(VCVTTPS2DQ, cvttps2dq) 1037UNARY_INT_SSE(VCVTPH2PS, cvtph2ps) 1038 1039 1040static inline void gen_unary_imm_sse(DisasContext *s, X86DecodedInsn *decode, 1041 SSEFunc_0_ppi xmm, SSEFunc_0_ppi ymm) 1042{ 1043 TCGv_i32 imm = tcg_constant8u_i32(decode->immediate); 1044 if (!s->vex_l) { 1045 xmm(OP_PTR0, OP_PTR1, imm); 1046 } else { 1047 ymm(OP_PTR0, OP_PTR1, imm); 1048 } 1049} 1050 1051#define UNARY_IMM_SSE(uname, lname) \ 1052static void gen_##uname(DisasContext *s, X86DecodedInsn *decode) \ 1053{ \ 1054 gen_unary_imm_sse(s, decode, \ 1055 gen_helper_##lname##_xmm, \ 1056 gen_helper_##lname##_ymm); \ 1057} 1058 1059UNARY_IMM_SSE(PSHUFD, pshufd) 1060UNARY_IMM_SSE(PSHUFHW, pshufhw) 1061UNARY_IMM_SSE(PSHUFLW, pshuflw) 1062#define gen_helper_vpermq_xmm NULL 1063UNARY_IMM_SSE(VPERMQ, vpermq) 1064UNARY_IMM_SSE(VPERMILPS_i, vpermilps_imm) 1065UNARY_IMM_SSE(VPERMILPD_i, vpermilpd_imm) 1066 1067static inline void gen_unary_imm_fp_sse(DisasContext *s, X86DecodedInsn *decode, 1068 SSEFunc_0_eppi xmm, SSEFunc_0_eppi ymm) 1069{ 1070 TCGv_i32 imm = tcg_constant8u_i32(decode->immediate); 1071 if (!s->vex_l) { 1072 xmm(tcg_env, OP_PTR0, OP_PTR1, imm); 1073 } else { 1074 ymm(tcg_env, OP_PTR0, OP_PTR1, imm); 1075 } 1076} 1077 1078#define UNARY_IMM_FP_SSE(uname, lname) \ 1079static void gen_##uname(DisasContext *s, X86DecodedInsn *decode) \ 1080{ \ 1081 gen_unary_imm_fp_sse(s, decode, \ 1082 gen_helper_##lname##_xmm, \ 1083 gen_helper_##lname##_ymm); \ 1084} 1085 1086UNARY_IMM_FP_SSE(VROUNDPS, roundps) 1087UNARY_IMM_FP_SSE(VROUNDPD, roundpd) 1088 1089static inline void gen_vexw_avx(DisasContext *s, X86DecodedInsn *decode, 1090 SSEFunc_0_eppp d_xmm, SSEFunc_0_eppp q_xmm, 1091 SSEFunc_0_eppp d_ymm, SSEFunc_0_eppp q_ymm) 1092{ 1093 SSEFunc_0_eppp d = s->vex_l ? d_ymm : d_xmm; 1094 SSEFunc_0_eppp q = s->vex_l ? q_ymm : q_xmm; 1095 SSEFunc_0_eppp fn = s->vex_w ? q : d; 1096 fn(tcg_env, OP_PTR0, OP_PTR1, OP_PTR2); 1097} 1098 1099/* VEX.W affects whether to operate on 32- or 64-bit elements. */ 1100#define VEXW_AVX(uname, lname) \ 1101static void gen_##uname(DisasContext *s, X86DecodedInsn *decode) \ 1102{ \ 1103 gen_vexw_avx(s, decode, \ 1104 gen_helper_##lname##d_xmm, gen_helper_##lname##q_xmm, \ 1105 gen_helper_##lname##d_ymm, gen_helper_##lname##q_ymm); \ 1106} 1107VEXW_AVX(VPSLLV, vpsllv) 1108VEXW_AVX(VPSRLV, vpsrlv) 1109VEXW_AVX(VPSRAV, vpsrav) 1110VEXW_AVX(VPMASKMOV, vpmaskmov) 1111 1112/* Same as above, but with extra arguments to the helper. */ 1113static inline void gen_vsib_avx(DisasContext *s, X86DecodedInsn *decode, 1114 SSEFunc_0_epppti d_xmm, SSEFunc_0_epppti q_xmm, 1115 SSEFunc_0_epppti d_ymm, SSEFunc_0_epppti q_ymm) 1116{ 1117 SSEFunc_0_epppti d = s->vex_l ? d_ymm : d_xmm; 1118 SSEFunc_0_epppti q = s->vex_l ? q_ymm : q_xmm; 1119 SSEFunc_0_epppti fn = s->vex_w ? q : d; 1120 TCGv_i32 scale = tcg_constant_i32(decode->mem.scale); 1121 TCGv_ptr index = tcg_temp_new_ptr(); 1122 1123 /* Pass third input as (index, base, scale) */ 1124 tcg_gen_addi_ptr(index, tcg_env, ZMM_OFFSET(decode->mem.index)); 1125 fn(tcg_env, OP_PTR0, OP_PTR1, index, s->A0, scale); 1126 1127 /* 1128 * There are two output operands, so zero OP1's high 128 bits 1129 * in the VEX.128 case. 1130 */ 1131 if (!s->vex_l) { 1132 int ymmh_ofs = vector_elem_offset(&decode->op[1], MO_128, 1); 1133 tcg_gen_gvec_dup_imm(MO_64, ymmh_ofs, 16, 16, 0); 1134 } 1135} 1136#define VSIB_AVX(uname, lname) \ 1137static void gen_##uname(DisasContext *s, X86DecodedInsn *decode) \ 1138{ \ 1139 gen_vsib_avx(s, decode, \ 1140 gen_helper_##lname##d_xmm, gen_helper_##lname##q_xmm, \ 1141 gen_helper_##lname##d_ymm, gen_helper_##lname##q_ymm); \ 1142} 1143VSIB_AVX(VPGATHERD, vpgatherd) 1144VSIB_AVX(VPGATHERQ, vpgatherq) 1145 1146static void gen_AAA(DisasContext *s, X86DecodedInsn *decode) 1147{ 1148 gen_update_cc_op(s); 1149 gen_helper_aaa(tcg_env); 1150 assume_cc_op(s, CC_OP_EFLAGS); 1151} 1152 1153static void gen_AAD(DisasContext *s, X86DecodedInsn *decode) 1154{ 1155 gen_helper_aad(s->T0, s->T0, s->T1); 1156 prepare_update1_cc(decode, s, CC_OP_LOGICB); 1157} 1158 1159static void gen_AAM(DisasContext *s, X86DecodedInsn *decode) 1160{ 1161 if (decode->immediate == 0) { 1162 gen_exception(s, EXCP00_DIVZ); 1163 } else { 1164 gen_helper_aam(s->T0, s->T0, s->T1); 1165 prepare_update1_cc(decode, s, CC_OP_LOGICB); 1166 } 1167} 1168 1169static void gen_AAS(DisasContext *s, X86DecodedInsn *decode) 1170{ 1171 gen_update_cc_op(s); 1172 gen_helper_aas(tcg_env); 1173 assume_cc_op(s, CC_OP_EFLAGS); 1174} 1175 1176static void gen_ADC(DisasContext *s, X86DecodedInsn *decode) 1177{ 1178 MemOp ot = decode->op[1].ot; 1179 TCGv c_in = tcg_temp_new(); 1180 1181 gen_compute_eflags_c(s, c_in); 1182 if (s->prefix & PREFIX_LOCK) { 1183 tcg_gen_add_tl(s->T0, c_in, s->T1); 1184 tcg_gen_atomic_add_fetch_tl(s->T0, s->A0, s->T0, 1185 s->mem_index, ot | MO_LE); 1186 } else { 1187 tcg_gen_add_tl(s->T0, s->T0, s->T1); 1188 tcg_gen_add_tl(s->T0, s->T0, c_in); 1189 } 1190 prepare_update3_cc(decode, s, CC_OP_ADCB + ot, c_in); 1191} 1192 1193static void gen_ADCOX(DisasContext *s, X86DecodedInsn *decode, int cc_op) 1194{ 1195 MemOp ot = decode->op[0].ot; 1196 TCGv carry_in = NULL; 1197 TCGv *carry_out = (cc_op == CC_OP_ADCX ? &decode->cc_dst : &decode->cc_src2); 1198 TCGv zero; 1199 1200 decode->cc_op = cc_op; 1201 *carry_out = tcg_temp_new(); 1202 if (CC_OP_HAS_EFLAGS(s->cc_op)) { 1203 decode->cc_src = cpu_cc_src; 1204 1205 /* Re-use the carry-out from a previous round? */ 1206 if (s->cc_op == cc_op || s->cc_op == CC_OP_ADCOX) { 1207 carry_in = (cc_op == CC_OP_ADCX ? cpu_cc_dst : cpu_cc_src2); 1208 } 1209 1210 /* Preserve the opposite carry from previous rounds? */ 1211 if (s->cc_op != cc_op && s->cc_op != CC_OP_EFLAGS) { 1212 decode->cc_op = CC_OP_ADCOX; 1213 if (carry_out == &decode->cc_dst) { 1214 decode->cc_src2 = cpu_cc_src2; 1215 } else { 1216 decode->cc_dst = cpu_cc_dst; 1217 } 1218 } 1219 } else { 1220 decode->cc_src = tcg_temp_new(); 1221 gen_mov_eflags(s, decode->cc_src); 1222 } 1223 1224 if (!carry_in) { 1225 /* Get carry_in out of EFLAGS. */ 1226 carry_in = tcg_temp_new(); 1227 tcg_gen_extract_tl(carry_in, decode->cc_src, 1228 ctz32(cc_op == CC_OP_ADCX ? CC_C : CC_O), 1); 1229 } 1230 1231 switch (ot) { 1232#ifdef TARGET_X86_64 1233 case MO_32: 1234 /* If TL is 64-bit just do everything in 64-bit arithmetic. */ 1235 tcg_gen_ext32u_tl(s->T0, s->T0); 1236 tcg_gen_ext32u_tl(s->T1, s->T1); 1237 tcg_gen_add_i64(s->T0, s->T0, s->T1); 1238 tcg_gen_add_i64(s->T0, s->T0, carry_in); 1239 tcg_gen_shri_i64(*carry_out, s->T0, 32); 1240 break; 1241#endif 1242 default: 1243 zero = tcg_constant_tl(0); 1244 tcg_gen_add2_tl(s->T0, *carry_out, s->T0, zero, carry_in, zero); 1245 tcg_gen_add2_tl(s->T0, *carry_out, s->T0, *carry_out, s->T1, zero); 1246 break; 1247 } 1248} 1249 1250static void gen_ADCX(DisasContext *s, X86DecodedInsn *decode) 1251{ 1252 gen_ADCOX(s, decode, CC_OP_ADCX); 1253} 1254 1255static void gen_ADD(DisasContext *s, X86DecodedInsn *decode) 1256{ 1257 MemOp ot = decode->op[1].ot; 1258 1259 if (s->prefix & PREFIX_LOCK) { 1260 tcg_gen_atomic_add_fetch_tl(s->T0, s->A0, s->T1, 1261 s->mem_index, ot | MO_LE); 1262 } else { 1263 tcg_gen_add_tl(s->T0, s->T0, s->T1); 1264 } 1265 prepare_update2_cc(decode, s, CC_OP_ADDB + ot); 1266} 1267 1268static void gen_ADOX(DisasContext *s, X86DecodedInsn *decode) 1269{ 1270 gen_ADCOX(s, decode, CC_OP_ADOX); 1271} 1272 1273static void gen_AND(DisasContext *s, X86DecodedInsn *decode) 1274{ 1275 MemOp ot = decode->op[1].ot; 1276 1277 if (s->prefix & PREFIX_LOCK) { 1278 tcg_gen_atomic_and_fetch_tl(s->T0, s->A0, s->T1, 1279 s->mem_index, ot | MO_LE); 1280 } else { 1281 tcg_gen_and_tl(s->T0, s->T0, s->T1); 1282 } 1283 prepare_update1_cc(decode, s, CC_OP_LOGICB + ot); 1284} 1285 1286static void gen_ANDN(DisasContext *s, X86DecodedInsn *decode) 1287{ 1288 MemOp ot = decode->op[0].ot; 1289 1290 tcg_gen_andc_tl(s->T0, s->T1, s->T0); 1291 prepare_update1_cc(decode, s, CC_OP_LOGICB + ot); 1292} 1293 1294static void gen_ARPL(DisasContext *s, X86DecodedInsn *decode) 1295{ 1296 TCGv zf = tcg_temp_new(); 1297 TCGv flags = tcg_temp_new(); 1298 1299 gen_mov_eflags(s, flags); 1300 1301 /* Compute adjusted DST in T1, merging in SRC[RPL]. */ 1302 tcg_gen_deposit_tl(s->T1, s->T0, s->T1, 0, 2); 1303 1304 /* Z flag set if DST[RPL] < SRC[RPL] */ 1305 tcg_gen_setcond_tl(TCG_COND_LTU, zf, s->T0, s->T1); 1306 tcg_gen_deposit_tl(flags, flags, zf, ctz32(CC_Z), 1); 1307 1308 /* Place maximum RPL in DST */ 1309 tcg_gen_umax_tl(s->T0, s->T0, s->T1); 1310 1311 decode->cc_src = flags; 1312 decode->cc_op = CC_OP_EFLAGS; 1313} 1314 1315static void gen_BEXTR(DisasContext *s, X86DecodedInsn *decode) 1316{ 1317 MemOp ot = decode->op[0].ot; 1318 TCGv bound = tcg_constant_tl(ot == MO_64 ? 63 : 31); 1319 TCGv zero = tcg_constant_tl(0); 1320 TCGv mone = tcg_constant_tl(-1); 1321 1322 /* 1323 * Extract START, and shift the operand. 1324 * Shifts larger than operand size get zeros. 1325 */ 1326 tcg_gen_ext8u_tl(s->A0, s->T1); 1327 tcg_gen_shr_tl(s->T0, s->T0, s->A0); 1328 1329 tcg_gen_movcond_tl(TCG_COND_LEU, s->T0, s->A0, bound, s->T0, zero); 1330 1331 /* 1332 * Extract the LEN into an inverse mask. Lengths larger than 1333 * operand size get all zeros, length 0 gets all ones. 1334 */ 1335 tcg_gen_extract_tl(s->A0, s->T1, 8, 8); 1336 tcg_gen_shl_tl(s->T1, mone, s->A0); 1337 tcg_gen_movcond_tl(TCG_COND_LEU, s->T1, s->A0, bound, s->T1, zero); 1338 tcg_gen_andc_tl(s->T0, s->T0, s->T1); 1339 1340 prepare_update1_cc(decode, s, CC_OP_LOGICB + ot); 1341} 1342 1343static void gen_BLSI(DisasContext *s, X86DecodedInsn *decode) 1344{ 1345 MemOp ot = decode->op[0].ot; 1346 1347 /* input in T1, which is ready for prepare_update2_cc */ 1348 tcg_gen_neg_tl(s->T0, s->T1); 1349 tcg_gen_and_tl(s->T0, s->T0, s->T1); 1350 prepare_update2_cc(decode, s, CC_OP_BLSIB + ot); 1351} 1352 1353static void gen_BLSMSK(DisasContext *s, X86DecodedInsn *decode) 1354{ 1355 MemOp ot = decode->op[0].ot; 1356 1357 /* input in T1, which is ready for prepare_update2_cc */ 1358 tcg_gen_subi_tl(s->T0, s->T1, 1); 1359 tcg_gen_xor_tl(s->T0, s->T0, s->T1); 1360 prepare_update2_cc(decode, s, CC_OP_BMILGB + ot); 1361} 1362 1363static void gen_BLSR(DisasContext *s, X86DecodedInsn *decode) 1364{ 1365 MemOp ot = decode->op[0].ot; 1366 1367 /* input in T1, which is ready for prepare_update2_cc */ 1368 tcg_gen_subi_tl(s->T0, s->T1, 1); 1369 tcg_gen_and_tl(s->T0, s->T0, s->T1); 1370 prepare_update2_cc(decode, s, CC_OP_BMILGB + ot); 1371} 1372 1373static void gen_BOUND(DisasContext *s, X86DecodedInsn *decode) 1374{ 1375 TCGv_i32 op = tcg_temp_new_i32(); 1376 tcg_gen_trunc_tl_i32(op, s->T0); 1377 if (decode->op[1].ot == MO_16) { 1378 gen_helper_boundw(tcg_env, s->A0, op); 1379 } else { 1380 gen_helper_boundl(tcg_env, s->A0, op); 1381 } 1382} 1383 1384/* Non-standard convention - on entry T0 is zero-extended input, T1 is the output. */ 1385static void gen_BSF(DisasContext *s, X86DecodedInsn *decode) 1386{ 1387 MemOp ot = decode->op[0].ot; 1388 1389 /* Only the Z bit is defined and it is related to the input. */ 1390 decode->cc_dst = tcg_temp_new(); 1391 decode->cc_op = CC_OP_LOGICB + ot; 1392 tcg_gen_mov_tl(decode->cc_dst, s->T0); 1393 1394 /* 1395 * The manual says that the output is undefined when the 1396 * input is zero, but real hardware leaves it unchanged, and 1397 * real programs appear to depend on that. Accomplish this 1398 * by passing the output as the value to return upon zero. 1399 */ 1400 tcg_gen_ctz_tl(s->T0, s->T0, s->T1); 1401} 1402 1403/* Non-standard convention - on entry T0 is zero-extended input, T1 is the output. */ 1404static void gen_BSR(DisasContext *s, X86DecodedInsn *decode) 1405{ 1406 MemOp ot = decode->op[0].ot; 1407 1408 /* Only the Z bit is defined and it is related to the input. */ 1409 decode->cc_dst = tcg_temp_new(); 1410 decode->cc_op = CC_OP_LOGICB + ot; 1411 tcg_gen_mov_tl(decode->cc_dst, s->T0); 1412 1413 /* 1414 * The manual says that the output is undefined when the 1415 * input is zero, but real hardware leaves it unchanged, and 1416 * real programs appear to depend on that. Accomplish this 1417 * by passing the output as the value to return upon zero. 1418 * Plus, return the bit index of the first 1 bit. 1419 */ 1420 tcg_gen_xori_tl(s->T1, s->T1, TARGET_LONG_BITS - 1); 1421 tcg_gen_clz_tl(s->T0, s->T0, s->T1); 1422 tcg_gen_xori_tl(s->T0, s->T0, TARGET_LONG_BITS - 1); 1423} 1424 1425static void gen_BSWAP(DisasContext *s, X86DecodedInsn *decode) 1426{ 1427#ifdef TARGET_X86_64 1428 if (s->dflag == MO_64) { 1429 tcg_gen_bswap64_i64(s->T0, s->T0); 1430 return; 1431 } 1432#endif 1433 tcg_gen_bswap32_tl(s->T0, s->T0, TCG_BSWAP_OZ); 1434} 1435 1436static TCGv gen_bt_mask(DisasContext *s, X86DecodedInsn *decode) 1437{ 1438 MemOp ot = decode->op[1].ot; 1439 TCGv mask = tcg_temp_new(); 1440 1441 tcg_gen_andi_tl(s->T1, s->T1, (8 << ot) - 1); 1442 tcg_gen_shl_tl(mask, tcg_constant_tl(1), s->T1); 1443 return mask; 1444} 1445 1446/* Expects truncated bit index in s->T1, 1 << s->T1 in MASK. */ 1447static void gen_bt_flags(DisasContext *s, X86DecodedInsn *decode, TCGv src, TCGv mask) 1448{ 1449 TCGv cf; 1450 1451 /* 1452 * C is the result of the test, Z is unchanged, and the others 1453 * are all undefined. 1454 */ 1455 switch (s->cc_op) { 1456 case CC_OP_DYNAMIC: 1457 case CC_OP_CLR: 1458 case CC_OP_EFLAGS: 1459 case CC_OP_ADCX: 1460 case CC_OP_ADOX: 1461 case CC_OP_ADCOX: 1462 /* Generate EFLAGS and replace the C bit. */ 1463 cf = tcg_temp_new(); 1464 tcg_gen_setcond_tl(TCG_COND_TSTNE, cf, src, mask); 1465 prepare_update_cf(decode, s, cf); 1466 break; 1467 default: 1468 /* 1469 * Z was going to be computed from the non-zero status of CC_DST. 1470 * We can get that same Z value (and the new C value) by leaving 1471 * CC_DST alone, setting CC_SRC, and using a CC_OP_SAR of the 1472 * same width. 1473 */ 1474 decode->cc_src = tcg_temp_new(); 1475 decode->cc_dst = cpu_cc_dst; 1476 decode->cc_op = ((s->cc_op - CC_OP_MULB) & 3) + CC_OP_SARB; 1477 tcg_gen_shr_tl(decode->cc_src, src, s->T1); 1478 break; 1479 } 1480} 1481 1482static void gen_BT(DisasContext *s, X86DecodedInsn *decode) 1483{ 1484 TCGv mask = gen_bt_mask(s, decode); 1485 1486 gen_bt_flags(s, decode, s->T0, mask); 1487} 1488 1489static void gen_BTC(DisasContext *s, X86DecodedInsn *decode) 1490{ 1491 MemOp ot = decode->op[0].ot; 1492 TCGv old = tcg_temp_new(); 1493 TCGv mask = gen_bt_mask(s, decode); 1494 1495 if (s->prefix & PREFIX_LOCK) { 1496 tcg_gen_atomic_fetch_xor_tl(old, s->A0, mask, s->mem_index, ot | MO_LE); 1497 } else { 1498 tcg_gen_mov_tl(old, s->T0); 1499 tcg_gen_xor_tl(s->T0, s->T0, mask); 1500 } 1501 1502 gen_bt_flags(s, decode, old, mask); 1503} 1504 1505static void gen_BTR(DisasContext *s, X86DecodedInsn *decode) 1506{ 1507 MemOp ot = decode->op[0].ot; 1508 TCGv old = tcg_temp_new(); 1509 TCGv mask = gen_bt_mask(s, decode); 1510 1511 if (s->prefix & PREFIX_LOCK) { 1512 TCGv maskc = tcg_temp_new(); 1513 tcg_gen_not_tl(maskc, mask); 1514 tcg_gen_atomic_fetch_and_tl(old, s->A0, maskc, s->mem_index, ot | MO_LE); 1515 } else { 1516 tcg_gen_mov_tl(old, s->T0); 1517 tcg_gen_andc_tl(s->T0, s->T0, mask); 1518 } 1519 1520 gen_bt_flags(s, decode, old, mask); 1521} 1522 1523static void gen_BTS(DisasContext *s, X86DecodedInsn *decode) 1524{ 1525 MemOp ot = decode->op[0].ot; 1526 TCGv old = tcg_temp_new(); 1527 TCGv mask = gen_bt_mask(s, decode); 1528 1529 if (s->prefix & PREFIX_LOCK) { 1530 tcg_gen_atomic_fetch_or_tl(old, s->A0, mask, s->mem_index, ot | MO_LE); 1531 } else { 1532 tcg_gen_mov_tl(old, s->T0); 1533 tcg_gen_or_tl(s->T0, s->T0, mask); 1534 } 1535 1536 gen_bt_flags(s, decode, old, mask); 1537} 1538 1539static void gen_BZHI(DisasContext *s, X86DecodedInsn *decode) 1540{ 1541 MemOp ot = decode->op[0].ot; 1542 TCGv bound = tcg_constant_tl(ot == MO_64 ? 63 : 31); 1543 TCGv zero = tcg_constant_tl(0); 1544 TCGv mone = tcg_constant_tl(-1); 1545 1546 tcg_gen_ext8u_tl(s->T1, s->T1); 1547 1548 tcg_gen_shl_tl(s->A0, mone, s->T1); 1549 tcg_gen_movcond_tl(TCG_COND_LEU, s->A0, s->T1, bound, s->A0, zero); 1550 tcg_gen_andc_tl(s->T0, s->T0, s->A0); 1551 /* 1552 * Note that since we're using BMILG (in order to get O 1553 * cleared) we need to store the inverse into C. 1554 */ 1555 tcg_gen_setcond_tl(TCG_COND_LEU, s->T1, s->T1, bound); 1556 prepare_update2_cc(decode, s, CC_OP_BMILGB + ot); 1557} 1558 1559static void gen_CALL(DisasContext *s, X86DecodedInsn *decode) 1560{ 1561 gen_push_v(s, eip_next_tl(s)); 1562 gen_JMP(s, decode); 1563} 1564 1565static void gen_CALL_m(DisasContext *s, X86DecodedInsn *decode) 1566{ 1567 gen_push_v(s, eip_next_tl(s)); 1568 gen_JMP_m(s, decode); 1569} 1570 1571static void gen_CALLF(DisasContext *s, X86DecodedInsn *decode) 1572{ 1573 gen_far_call(s); 1574} 1575 1576static void gen_CALLF_m(DisasContext *s, X86DecodedInsn *decode) 1577{ 1578 MemOp ot = decode->op[1].ot; 1579 1580 gen_op_ld_v(s, ot, s->T0, s->A0); 1581 gen_add_A0_im(s, 1 << ot); 1582 gen_op_ld_v(s, MO_16, s->T1, s->A0); 1583 gen_far_call(s); 1584} 1585 1586static void gen_CBW(DisasContext *s, X86DecodedInsn *decode) 1587{ 1588 MemOp src_ot = decode->op[0].ot - 1; 1589 1590 tcg_gen_ext_tl(s->T0, s->T0, src_ot | MO_SIGN); 1591} 1592 1593static void gen_CLC(DisasContext *s, X86DecodedInsn *decode) 1594{ 1595 gen_compute_eflags(s); 1596 tcg_gen_andi_tl(cpu_cc_src, cpu_cc_src, ~CC_C); 1597} 1598 1599static void gen_CLD(DisasContext *s, X86DecodedInsn *decode) 1600{ 1601 tcg_gen_st_i32(tcg_constant_i32(1), tcg_env, offsetof(CPUX86State, df)); 1602} 1603 1604static void gen_CLI(DisasContext *s, X86DecodedInsn *decode) 1605{ 1606 gen_reset_eflags(s, IF_MASK); 1607} 1608 1609static void gen_CLTS(DisasContext *s, X86DecodedInsn *decode) 1610{ 1611 gen_helper_clts(tcg_env); 1612 /* abort block because static cpu state changed */ 1613 s->base.is_jmp = DISAS_EOB_NEXT; 1614} 1615 1616static void gen_CMC(DisasContext *s, X86DecodedInsn *decode) 1617{ 1618 gen_compute_eflags(s); 1619 tcg_gen_xori_tl(cpu_cc_src, cpu_cc_src, CC_C); 1620} 1621 1622static void gen_CMOVcc(DisasContext *s, X86DecodedInsn *decode) 1623{ 1624 gen_cmovcc1(s, decode->b & 0xf, s->T0, s->T1); 1625} 1626 1627static void gen_CMPccXADD(DisasContext *s, X86DecodedInsn *decode) 1628{ 1629 TCGLabel *label_top = gen_new_label(); 1630 TCGLabel *label_bottom = gen_new_label(); 1631 TCGv oldv = tcg_temp_new(); 1632 TCGv newv = tcg_temp_new(); 1633 TCGv cmpv = tcg_temp_new(); 1634 TCGCond cond; 1635 1636 TCGv cmp_lhs, cmp_rhs; 1637 MemOp ot, ot_full; 1638 1639 int jcc_op = (decode->b >> 1) & 7; 1640 static const TCGCond cond_table[8] = { 1641 [JCC_O] = TCG_COND_LT, /* test sign bit by comparing against 0 */ 1642 [JCC_B] = TCG_COND_LTU, 1643 [JCC_Z] = TCG_COND_EQ, 1644 [JCC_BE] = TCG_COND_LEU, 1645 [JCC_S] = TCG_COND_LT, /* test sign bit by comparing against 0 */ 1646 [JCC_P] = TCG_COND_TSTEQ, /* even parity - tests low bit of popcount */ 1647 [JCC_L] = TCG_COND_LT, 1648 [JCC_LE] = TCG_COND_LE, 1649 }; 1650 1651 cond = cond_table[jcc_op]; 1652 if (decode->b & 1) { 1653 cond = tcg_invert_cond(cond); 1654 } 1655 1656 ot = decode->op[0].ot; 1657 ot_full = ot | MO_LE; 1658 if (jcc_op >= JCC_S) { 1659 /* 1660 * Sign-extend values before subtracting for S, P (zero/sign extension 1661 * does not matter there) L, LE and their inverses. 1662 */ 1663 ot_full |= MO_SIGN; 1664 } 1665 1666 /* 1667 * cmpv will be moved to cc_src *after* cpu_regs[] is written back, so use 1668 * tcg_gen_ext_tl instead of gen_ext_tl. 1669 */ 1670 tcg_gen_ext_tl(cmpv, cpu_regs[decode->op[1].n], ot_full); 1671 1672 /* 1673 * Cmpxchg loop starts here. 1674 * - s->T1: addition operand (from decoder) 1675 * - s->A0: dest address (from decoder) 1676 * - s->cc_srcT: memory operand (lhs for comparison) 1677 * - cmpv: rhs for comparison 1678 */ 1679 gen_set_label(label_top); 1680 gen_op_ld_v(s, ot_full, s->cc_srcT, s->A0); 1681 tcg_gen_sub_tl(s->T0, s->cc_srcT, cmpv); 1682 1683 /* Compute the comparison result by hand, to avoid clobbering cc_*. */ 1684 switch (jcc_op) { 1685 case JCC_O: 1686 /* (src1 ^ src2) & (src1 ^ dst). newv is only used here for a moment */ 1687 tcg_gen_xor_tl(newv, s->cc_srcT, s->T0); 1688 tcg_gen_xor_tl(s->tmp0, s->cc_srcT, cmpv); 1689 tcg_gen_and_tl(s->tmp0, s->tmp0, newv); 1690 tcg_gen_sextract_tl(s->tmp0, s->tmp0, 0, 8 << ot); 1691 cmp_lhs = s->tmp0, cmp_rhs = tcg_constant_tl(0); 1692 break; 1693 1694 case JCC_P: 1695 tcg_gen_ext8u_tl(s->tmp0, s->T0); 1696 tcg_gen_ctpop_tl(s->tmp0, s->tmp0); 1697 cmp_lhs = s->tmp0, cmp_rhs = tcg_constant_tl(1); 1698 break; 1699 1700 case JCC_S: 1701 tcg_gen_sextract_tl(s->tmp0, s->T0, 0, 8 << ot); 1702 cmp_lhs = s->tmp0, cmp_rhs = tcg_constant_tl(0); 1703 break; 1704 1705 default: 1706 cmp_lhs = s->cc_srcT, cmp_rhs = cmpv; 1707 break; 1708 } 1709 1710 /* Compute new value: if condition does not hold, just store back s->cc_srcT */ 1711 tcg_gen_add_tl(newv, s->cc_srcT, s->T1); 1712 tcg_gen_movcond_tl(cond, newv, cmp_lhs, cmp_rhs, newv, s->cc_srcT); 1713 tcg_gen_atomic_cmpxchg_tl(oldv, s->A0, s->cc_srcT, newv, s->mem_index, ot_full); 1714 1715 /* Exit unconditionally if cmpxchg succeeded. */ 1716 tcg_gen_brcond_tl(TCG_COND_EQ, oldv, s->cc_srcT, label_bottom); 1717 1718 /* Try again if there was actually a store to make. */ 1719 tcg_gen_brcond_tl(cond, cmp_lhs, cmp_rhs, label_top); 1720 gen_set_label(label_bottom); 1721 1722 /* Store old value to registers only after a successful store. */ 1723 gen_writeback(s, decode, 1, s->cc_srcT); 1724 1725 decode->cc_dst = s->T0; 1726 decode->cc_src = cmpv; 1727 decode->cc_op = CC_OP_SUBB + ot; 1728} 1729 1730static void gen_CMPS(DisasContext *s, X86DecodedInsn *decode) 1731{ 1732 MemOp ot = decode->op[2].ot; 1733 if (s->prefix & (PREFIX_REPZ | PREFIX_REPNZ)) { 1734 gen_repz_nz(s, ot, gen_cmps); 1735 } else { 1736 gen_cmps(s, ot); 1737 } 1738} 1739 1740static void gen_CMPXCHG(DisasContext *s, X86DecodedInsn *decode) 1741{ 1742 MemOp ot = decode->op[2].ot; 1743 TCGv cmpv = tcg_temp_new(); 1744 TCGv oldv = tcg_temp_new(); 1745 TCGv newv = tcg_temp_new(); 1746 TCGv dest; 1747 1748 tcg_gen_ext_tl(cmpv, cpu_regs[R_EAX], ot); 1749 tcg_gen_ext_tl(newv, s->T1, ot); 1750 if (s->prefix & PREFIX_LOCK) { 1751 tcg_gen_atomic_cmpxchg_tl(oldv, s->A0, cmpv, newv, 1752 s->mem_index, ot | MO_LE); 1753 } else { 1754 tcg_gen_ext_tl(oldv, s->T0, ot); 1755 if (decode->op[0].has_ea) { 1756 /* 1757 * Perform an unconditional store cycle like physical cpu; 1758 * must be before changing accumulator to ensure 1759 * idempotency if the store faults and the instruction 1760 * is restarted 1761 */ 1762 tcg_gen_movcond_tl(TCG_COND_EQ, newv, oldv, cmpv, newv, oldv); 1763 gen_op_st_v(s, ot, newv, s->A0); 1764 } else { 1765 /* 1766 * Unlike the memory case, where "the destination operand receives 1767 * a write cycle without regard to the result of the comparison", 1768 * rm must not be touched altogether if the write fails, including 1769 * not zero-extending it on 64-bit processors. So, precompute 1770 * the result of a successful writeback and perform the movcond 1771 * directly on cpu_regs. In case rm is part of RAX, note that this 1772 * movcond and the one below are mutually exclusive is executed. 1773 */ 1774 dest = gen_op_deposit_reg_v(s, ot, decode->op[0].n, newv, newv); 1775 tcg_gen_movcond_tl(TCG_COND_EQ, dest, oldv, cmpv, newv, dest); 1776 } 1777 decode->op[0].unit = X86_OP_SKIP; 1778 } 1779 1780 /* Write RAX only if the cmpxchg fails. */ 1781 dest = gen_op_deposit_reg_v(s, ot, R_EAX, s->T0, oldv); 1782 tcg_gen_movcond_tl(TCG_COND_NE, dest, oldv, cmpv, s->T0, dest); 1783 1784 tcg_gen_mov_tl(s->cc_srcT, cmpv); 1785 tcg_gen_sub_tl(cmpv, cmpv, oldv); 1786 decode->cc_dst = cmpv; 1787 decode->cc_src = oldv; 1788 decode->cc_op = CC_OP_SUBB + ot; 1789} 1790 1791static void gen_CMPXCHG16B(DisasContext *s, X86DecodedInsn *decode) 1792{ 1793#ifdef TARGET_X86_64 1794 MemOp mop = MO_TE | MO_128 | MO_ALIGN; 1795 TCGv_i64 t0, t1; 1796 TCGv_i128 cmp, val; 1797 1798 cmp = tcg_temp_new_i128(); 1799 val = tcg_temp_new_i128(); 1800 tcg_gen_concat_i64_i128(cmp, cpu_regs[R_EAX], cpu_regs[R_EDX]); 1801 tcg_gen_concat_i64_i128(val, cpu_regs[R_EBX], cpu_regs[R_ECX]); 1802 1803 /* Only require atomic with LOCK; non-parallel handled in generator. */ 1804 if (s->prefix & PREFIX_LOCK) { 1805 tcg_gen_atomic_cmpxchg_i128(val, s->A0, cmp, val, s->mem_index, mop); 1806 } else { 1807 tcg_gen_nonatomic_cmpxchg_i128(val, s->A0, cmp, val, s->mem_index, mop); 1808 } 1809 1810 tcg_gen_extr_i128_i64(s->T0, s->T1, val); 1811 1812 /* Determine success after the fact. */ 1813 t0 = tcg_temp_new_i64(); 1814 t1 = tcg_temp_new_i64(); 1815 tcg_gen_xor_i64(t0, s->T0, cpu_regs[R_EAX]); 1816 tcg_gen_xor_i64(t1, s->T1, cpu_regs[R_EDX]); 1817 tcg_gen_or_i64(t0, t0, t1); 1818 1819 /* Update Z. */ 1820 gen_compute_eflags(s); 1821 tcg_gen_setcondi_i64(TCG_COND_EQ, t0, t0, 0); 1822 tcg_gen_deposit_tl(cpu_cc_src, cpu_cc_src, t0, ctz32(CC_Z), 1); 1823 1824 /* 1825 * Extract the result values for the register pair. We may do this 1826 * unconditionally, because on success (Z=1), the old value matches 1827 * the previous value in RDX:RAX. 1828 */ 1829 tcg_gen_mov_i64(cpu_regs[R_EAX], s->T0); 1830 tcg_gen_mov_i64(cpu_regs[R_EDX], s->T1); 1831#else 1832 abort(); 1833#endif 1834} 1835 1836static void gen_CMPXCHG8B(DisasContext *s, X86DecodedInsn *decode) 1837{ 1838 TCGv_i64 cmp, val, old; 1839 TCGv Z; 1840 1841 cmp = tcg_temp_new_i64(); 1842 val = tcg_temp_new_i64(); 1843 old = tcg_temp_new_i64(); 1844 1845 /* Construct the comparison values from the register pair. */ 1846 tcg_gen_concat_tl_i64(cmp, cpu_regs[R_EAX], cpu_regs[R_EDX]); 1847 tcg_gen_concat_tl_i64(val, cpu_regs[R_EBX], cpu_regs[R_ECX]); 1848 1849 /* Only require atomic with LOCK; non-parallel handled in generator. */ 1850 if (s->prefix & PREFIX_LOCK) { 1851 tcg_gen_atomic_cmpxchg_i64(old, s->A0, cmp, val, s->mem_index, MO_TEUQ); 1852 } else { 1853 tcg_gen_nonatomic_cmpxchg_i64(old, s->A0, cmp, val, 1854 s->mem_index, MO_TEUQ); 1855 } 1856 1857 /* Set tmp0 to match the required value of Z. */ 1858 tcg_gen_setcond_i64(TCG_COND_EQ, cmp, old, cmp); 1859 Z = tcg_temp_new(); 1860 tcg_gen_trunc_i64_tl(Z, cmp); 1861 1862 /* 1863 * Extract the result values for the register pair. 1864 * For 32-bit, we may do this unconditionally, because on success (Z=1), 1865 * the old value matches the previous value in EDX:EAX. For x86_64, 1866 * the store must be conditional, because we must leave the source 1867 * registers unchanged on success, and zero-extend the writeback 1868 * on failure (Z=0). 1869 */ 1870 if (TARGET_LONG_BITS == 32) { 1871 tcg_gen_extr_i64_tl(cpu_regs[R_EAX], cpu_regs[R_EDX], old); 1872 } else { 1873 TCGv zero = tcg_constant_tl(0); 1874 1875 tcg_gen_extr_i64_tl(s->T0, s->T1, old); 1876 tcg_gen_movcond_tl(TCG_COND_EQ, cpu_regs[R_EAX], Z, zero, 1877 s->T0, cpu_regs[R_EAX]); 1878 tcg_gen_movcond_tl(TCG_COND_EQ, cpu_regs[R_EDX], Z, zero, 1879 s->T1, cpu_regs[R_EDX]); 1880 } 1881 1882 /* Update Z. */ 1883 gen_compute_eflags(s); 1884 tcg_gen_deposit_tl(cpu_cc_src, cpu_cc_src, Z, ctz32(CC_Z), 1); 1885} 1886 1887static void gen_CPUID(DisasContext *s, X86DecodedInsn *decode) 1888{ 1889 gen_update_cc_op(s); 1890 gen_update_eip_cur(s); 1891 gen_helper_cpuid(tcg_env); 1892} 1893 1894static void gen_CRC32(DisasContext *s, X86DecodedInsn *decode) 1895{ 1896 MemOp ot = decode->op[2].ot; 1897 1898 tcg_gen_trunc_tl_i32(s->tmp2_i32, s->T0); 1899 gen_helper_crc32(s->T0, s->tmp2_i32, s->T1, tcg_constant_i32(8 << ot)); 1900} 1901 1902static void gen_CVTPI2Px(DisasContext *s, X86DecodedInsn *decode) 1903{ 1904 gen_helper_enter_mmx(tcg_env); 1905 if (s->prefix & PREFIX_DATA) { 1906 gen_helper_cvtpi2pd(tcg_env, OP_PTR0, OP_PTR2); 1907 } else { 1908 gen_helper_cvtpi2ps(tcg_env, OP_PTR0, OP_PTR2); 1909 } 1910} 1911 1912static void gen_CVTPx2PI(DisasContext *s, X86DecodedInsn *decode) 1913{ 1914 gen_helper_enter_mmx(tcg_env); 1915 if (s->prefix & PREFIX_DATA) { 1916 gen_helper_cvtpd2pi(tcg_env, OP_PTR0, OP_PTR2); 1917 } else { 1918 gen_helper_cvtps2pi(tcg_env, OP_PTR0, OP_PTR2); 1919 } 1920} 1921 1922static void gen_CVTTPx2PI(DisasContext *s, X86DecodedInsn *decode) 1923{ 1924 gen_helper_enter_mmx(tcg_env); 1925 if (s->prefix & PREFIX_DATA) { 1926 gen_helper_cvttpd2pi(tcg_env, OP_PTR0, OP_PTR2); 1927 } else { 1928 gen_helper_cvttps2pi(tcg_env, OP_PTR0, OP_PTR2); 1929 } 1930} 1931 1932static void gen_CWD(DisasContext *s, X86DecodedInsn *decode) 1933{ 1934 int shift = 8 << decode->op[0].ot; 1935 1936 tcg_gen_sextract_tl(s->T0, s->T0, shift - 1, 1); 1937} 1938 1939static void gen_DAA(DisasContext *s, X86DecodedInsn *decode) 1940{ 1941 gen_update_cc_op(s); 1942 gen_helper_daa(tcg_env); 1943 assume_cc_op(s, CC_OP_EFLAGS); 1944} 1945 1946static void gen_DAS(DisasContext *s, X86DecodedInsn *decode) 1947{ 1948 gen_update_cc_op(s); 1949 gen_helper_das(tcg_env); 1950 assume_cc_op(s, CC_OP_EFLAGS); 1951} 1952 1953static void gen_DEC(DisasContext *s, X86DecodedInsn *decode) 1954{ 1955 MemOp ot = decode->op[1].ot; 1956 1957 tcg_gen_movi_tl(s->T1, -1); 1958 if (s->prefix & PREFIX_LOCK) { 1959 tcg_gen_atomic_add_fetch_tl(s->T0, s->A0, s->T1, 1960 s->mem_index, ot | MO_LE); 1961 } else { 1962 tcg_gen_add_tl(s->T0, s->T0, s->T1); 1963 } 1964 prepare_update_cc_incdec(decode, s, CC_OP_DECB + ot); 1965} 1966 1967static void gen_DIV(DisasContext *s, X86DecodedInsn *decode) 1968{ 1969 MemOp ot = decode->op[1].ot; 1970 1971 switch(ot) { 1972 case MO_8: 1973 gen_helper_divb_AL(tcg_env, s->T0); 1974 break; 1975 case MO_16: 1976 gen_helper_divw_AX(tcg_env, s->T0); 1977 break; 1978 default: 1979 case MO_32: 1980 gen_helper_divl_EAX(tcg_env, s->T0); 1981 break; 1982#ifdef TARGET_X86_64 1983 case MO_64: 1984 gen_helper_divq_EAX(tcg_env, s->T0); 1985 break; 1986#endif 1987 } 1988} 1989 1990static void gen_EMMS(DisasContext *s, X86DecodedInsn *decode) 1991{ 1992 gen_helper_emms(tcg_env); 1993} 1994 1995static void gen_ENTER(DisasContext *s, X86DecodedInsn *decode) 1996{ 1997 gen_enter(s, decode->op[1].imm, decode->op[2].imm); 1998} 1999 2000static void gen_EXTRQ_i(DisasContext *s, X86DecodedInsn *decode) 2001{ 2002 TCGv_i32 length = tcg_constant_i32(decode->immediate & 63); 2003 TCGv_i32 index = tcg_constant_i32((decode->immediate >> 8) & 63); 2004 2005 gen_helper_extrq_i(tcg_env, OP_PTR0, index, length); 2006} 2007 2008static void gen_EXTRQ_r(DisasContext *s, X86DecodedInsn *decode) 2009{ 2010 gen_helper_extrq_r(tcg_env, OP_PTR0, OP_PTR2); 2011} 2012 2013static void gen_FXRSTOR(DisasContext *s, X86DecodedInsn *decode) 2014{ 2015 if ((s->flags & HF_EM_MASK) || (s->flags & HF_TS_MASK)) { 2016 gen_NM_exception(s); 2017 } else { 2018 gen_helper_fxrstor(tcg_env, s->A0); 2019 } 2020} 2021 2022static void gen_FXSAVE(DisasContext *s, X86DecodedInsn *decode) 2023{ 2024 if ((s->flags & HF_EM_MASK) || (s->flags & HF_TS_MASK)) { 2025 gen_NM_exception(s); 2026 } else { 2027 gen_helper_fxsave(tcg_env, s->A0); 2028 } 2029} 2030 2031static void gen_HLT(DisasContext *s, X86DecodedInsn *decode) 2032{ 2033#ifdef CONFIG_SYSTEM_ONLY 2034 gen_update_cc_op(s); 2035 gen_update_eip_next(s); 2036 gen_helper_hlt(tcg_env); 2037 s->base.is_jmp = DISAS_NORETURN; 2038#endif 2039} 2040 2041static void gen_IDIV(DisasContext *s, X86DecodedInsn *decode) 2042{ 2043 MemOp ot = decode->op[1].ot; 2044 2045 switch(ot) { 2046 case MO_8: 2047 gen_helper_idivb_AL(tcg_env, s->T0); 2048 break; 2049 case MO_16: 2050 gen_helper_idivw_AX(tcg_env, s->T0); 2051 break; 2052 default: 2053 case MO_32: 2054 gen_helper_idivl_EAX(tcg_env, s->T0); 2055 break; 2056#ifdef TARGET_X86_64 2057 case MO_64: 2058 gen_helper_idivq_EAX(tcg_env, s->T0); 2059 break; 2060#endif 2061 } 2062} 2063 2064static void gen_IMUL3(DisasContext *s, X86DecodedInsn *decode) 2065{ 2066 MemOp ot = decode->op[0].ot; 2067 TCGv cc_src_rhs; 2068 2069 switch (ot) { 2070 case MO_16: 2071 /* s->T0 already sign-extended */ 2072 tcg_gen_ext16s_tl(s->T1, s->T1); 2073 tcg_gen_mul_tl(s->T0, s->T0, s->T1); 2074 /* Compare the full result to the extension of the truncated result. */ 2075 tcg_gen_ext16s_tl(s->T1, s->T0); 2076 cc_src_rhs = s->T0; 2077 break; 2078 2079 case MO_32: 2080#ifdef TARGET_X86_64 2081 if (TCG_TARGET_REG_BITS == 64) { 2082 /* 2083 * This produces fewer TCG ops, and better code if flags are needed, 2084 * but it requires a 64-bit multiply even if they are not. Use it 2085 * only if the target has 64-bits registers. 2086 * 2087 * s->T0 is already sign-extended. 2088 */ 2089 tcg_gen_ext32s_tl(s->T1, s->T1); 2090 tcg_gen_mul_tl(s->T0, s->T0, s->T1); 2091 /* Compare the full result to the extension of the truncated result. */ 2092 tcg_gen_ext32s_tl(s->T1, s->T0); 2093 cc_src_rhs = s->T0; 2094 } else { 2095 /* Variant that only needs a 32-bit widening multiply. */ 2096 TCGv_i32 hi = tcg_temp_new_i32(); 2097 TCGv_i32 lo = tcg_temp_new_i32(); 2098 tcg_gen_trunc_tl_i32(lo, s->T0); 2099 tcg_gen_trunc_tl_i32(hi, s->T1); 2100 tcg_gen_muls2_i32(lo, hi, lo, hi); 2101 tcg_gen_extu_i32_tl(s->T0, lo); 2102 2103 cc_src_rhs = tcg_temp_new(); 2104 tcg_gen_extu_i32_tl(cc_src_rhs, hi); 2105 /* Compare the high part to the sign bit of the truncated result */ 2106 tcg_gen_sari_i32(lo, lo, 31); 2107 tcg_gen_extu_i32_tl(s->T1, lo); 2108 } 2109 break; 2110 2111 case MO_64: 2112#endif 2113 cc_src_rhs = tcg_temp_new(); 2114 tcg_gen_muls2_tl(s->T0, cc_src_rhs, s->T0, s->T1); 2115 /* Compare the high part to the sign bit of the truncated result */ 2116 tcg_gen_sari_tl(s->T1, s->T0, TARGET_LONG_BITS - 1); 2117 break; 2118 2119 default: 2120 g_assert_not_reached(); 2121 } 2122 2123 tcg_gen_sub_tl(s->T1, s->T1, cc_src_rhs); 2124 prepare_update2_cc(decode, s, CC_OP_MULB + ot); 2125} 2126 2127static void gen_IMUL(DisasContext *s, X86DecodedInsn *decode) 2128{ 2129 MemOp ot = decode->op[1].ot; 2130 TCGv cc_src_rhs; 2131 2132 switch (ot) { 2133 case MO_8: 2134 /* s->T0 already sign-extended */ 2135 tcg_gen_ext8s_tl(s->T1, s->T1); 2136 tcg_gen_mul_tl(s->T0, s->T0, s->T1); 2137 gen_op_mov_reg_v(s, MO_16, R_EAX, s->T0); 2138 /* Compare the full result to the extension of the truncated result. */ 2139 tcg_gen_ext8s_tl(s->T1, s->T0); 2140 cc_src_rhs = s->T0; 2141 break; 2142 2143 case MO_16: 2144 /* s->T0 already sign-extended */ 2145 tcg_gen_ext16s_tl(s->T1, s->T1); 2146 tcg_gen_mul_tl(s->T0, s->T0, s->T1); 2147 gen_op_mov_reg_v(s, MO_16, R_EAX, s->T0); 2148 tcg_gen_shri_tl(s->T1, s->T0, 16); 2149 gen_op_mov_reg_v(s, MO_16, R_EDX, s->T1); 2150 /* Compare the full result to the extension of the truncated result. */ 2151 tcg_gen_ext16s_tl(s->T1, s->T0); 2152 cc_src_rhs = s->T0; 2153 break; 2154 2155 case MO_32: 2156#ifdef TARGET_X86_64 2157 /* s->T0 already sign-extended */ 2158 tcg_gen_ext32s_tl(s->T1, s->T1); 2159 tcg_gen_mul_tl(s->T0, s->T0, s->T1); 2160 tcg_gen_ext32u_tl(cpu_regs[R_EAX], s->T0); 2161 tcg_gen_shri_tl(cpu_regs[R_EDX], s->T0, 32); 2162 /* Compare the full result to the extension of the truncated result. */ 2163 tcg_gen_ext32s_tl(s->T1, s->T0); 2164 cc_src_rhs = s->T0; 2165 break; 2166 2167 case MO_64: 2168#endif 2169 tcg_gen_muls2_tl(s->T0, cpu_regs[R_EDX], s->T0, s->T1); 2170 tcg_gen_mov_tl(cpu_regs[R_EAX], s->T0); 2171 2172 /* Compare the high part to the sign bit of the truncated result */ 2173 tcg_gen_negsetcondi_tl(TCG_COND_LT, s->T1, s->T0, 0); 2174 cc_src_rhs = cpu_regs[R_EDX]; 2175 break; 2176 2177 default: 2178 g_assert_not_reached(); 2179 } 2180 2181 tcg_gen_sub_tl(s->T1, s->T1, cc_src_rhs); 2182 prepare_update2_cc(decode, s, CC_OP_MULB + ot); 2183} 2184 2185static void gen_IN(DisasContext *s, X86DecodedInsn *decode) 2186{ 2187 MemOp ot = decode->op[0].ot; 2188 TCGv_i32 port = tcg_temp_new_i32(); 2189 2190 tcg_gen_trunc_tl_i32(port, s->T0); 2191 tcg_gen_ext16u_i32(port, port); 2192 if (!gen_check_io(s, ot, port, SVM_IOIO_TYPE_MASK)) { 2193 return; 2194 } 2195 translator_io_start(&s->base); 2196 gen_helper_in_func(ot, s->T0, port); 2197 gen_writeback(s, decode, 0, s->T0); 2198 gen_bpt_io(s, port, ot); 2199} 2200 2201static void gen_INC(DisasContext *s, X86DecodedInsn *decode) 2202{ 2203 MemOp ot = decode->op[1].ot; 2204 2205 tcg_gen_movi_tl(s->T1, 1); 2206 if (s->prefix & PREFIX_LOCK) { 2207 tcg_gen_atomic_add_fetch_tl(s->T0, s->A0, s->T1, 2208 s->mem_index, ot | MO_LE); 2209 } else { 2210 tcg_gen_add_tl(s->T0, s->T0, s->T1); 2211 } 2212 prepare_update_cc_incdec(decode, s, CC_OP_INCB + ot); 2213} 2214 2215static void gen_INS(DisasContext *s, X86DecodedInsn *decode) 2216{ 2217 MemOp ot = decode->op[1].ot; 2218 TCGv_i32 port = tcg_temp_new_i32(); 2219 2220 tcg_gen_trunc_tl_i32(port, s->T1); 2221 tcg_gen_ext16u_i32(port, port); 2222 if (!gen_check_io(s, ot, port, 2223 SVM_IOIO_TYPE_MASK | SVM_IOIO_STR_MASK)) { 2224 return; 2225 } 2226 2227 translator_io_start(&s->base); 2228 if (s->prefix & (PREFIX_REPZ | PREFIX_REPNZ)) { 2229 gen_repz(s, ot, gen_ins); 2230 } else { 2231 gen_ins(s, ot); 2232 } 2233} 2234 2235static void gen_INSERTQ_i(DisasContext *s, X86DecodedInsn *decode) 2236{ 2237 TCGv_i32 length = tcg_constant_i32(decode->immediate & 63); 2238 TCGv_i32 index = tcg_constant_i32((decode->immediate >> 8) & 63); 2239 2240 gen_helper_insertq_i(tcg_env, OP_PTR0, OP_PTR1, index, length); 2241} 2242 2243static void gen_INSERTQ_r(DisasContext *s, X86DecodedInsn *decode) 2244{ 2245 gen_helper_insertq_r(tcg_env, OP_PTR0, OP_PTR2); 2246} 2247 2248static void gen_INT(DisasContext *s, X86DecodedInsn *decode) 2249{ 2250 gen_interrupt(s, decode->immediate); 2251} 2252 2253static void gen_INT1(DisasContext *s, X86DecodedInsn *decode) 2254{ 2255 gen_update_cc_op(s); 2256 gen_update_eip_next(s); 2257 gen_helper_icebp(tcg_env); 2258 s->base.is_jmp = DISAS_NORETURN; 2259} 2260 2261static void gen_INT3(DisasContext *s, X86DecodedInsn *decode) 2262{ 2263 gen_interrupt(s, EXCP03_INT3); 2264} 2265 2266static void gen_INTO(DisasContext *s, X86DecodedInsn *decode) 2267{ 2268 gen_update_cc_op(s); 2269 gen_update_eip_cur(s); 2270 gen_helper_into(tcg_env, cur_insn_len_i32(s)); 2271} 2272 2273static void gen_IRET(DisasContext *s, X86DecodedInsn *decode) 2274{ 2275 if (!PE(s) || VM86(s)) { 2276 gen_helper_iret_real(tcg_env, tcg_constant_i32(s->dflag - 1)); 2277 } else { 2278 gen_helper_iret_protected(tcg_env, tcg_constant_i32(s->dflag - 1), 2279 eip_next_i32(s)); 2280 } 2281 assume_cc_op(s, CC_OP_EFLAGS); 2282 s->base.is_jmp = DISAS_EOB_ONLY; 2283} 2284 2285static void gen_Jcc(DisasContext *s, X86DecodedInsn *decode) 2286{ 2287 gen_bnd_jmp(s); 2288 gen_jcc(s, decode->b & 0xf, decode->immediate); 2289} 2290 2291static void gen_JCXZ(DisasContext *s, X86DecodedInsn *decode) 2292{ 2293 TCGLabel *taken = gen_new_label(); 2294 2295 gen_update_cc_op(s); 2296 gen_op_jz_ecx(s, taken); 2297 gen_conditional_jump_labels(s, decode->immediate, NULL, taken); 2298} 2299 2300static void gen_JMP(DisasContext *s, X86DecodedInsn *decode) 2301{ 2302 gen_update_cc_op(s); 2303 gen_jmp_rel(s, s->dflag, decode->immediate, 0); 2304} 2305 2306static void gen_JMP_m(DisasContext *s, X86DecodedInsn *decode) 2307{ 2308 gen_op_jmp_v(s, s->T0); 2309 gen_bnd_jmp(s); 2310 s->base.is_jmp = DISAS_JUMP; 2311} 2312 2313static void gen_JMPF(DisasContext *s, X86DecodedInsn *decode) 2314{ 2315 gen_far_jmp(s); 2316} 2317 2318static void gen_JMPF_m(DisasContext *s, X86DecodedInsn *decode) 2319{ 2320 MemOp ot = decode->op[1].ot; 2321 2322 gen_op_ld_v(s, ot, s->T0, s->A0); 2323 gen_add_A0_im(s, 1 << ot); 2324 gen_op_ld_v(s, MO_16, s->T1, s->A0); 2325 gen_far_jmp(s); 2326} 2327 2328static void gen_LAHF(DisasContext *s, X86DecodedInsn *decode) 2329{ 2330 if (CODE64(s) && !(s->cpuid_ext3_features & CPUID_EXT3_LAHF_LM)) { 2331 return gen_illegal_opcode(s); 2332 } 2333 gen_compute_eflags(s); 2334 /* Note: gen_compute_eflags() only gives the condition codes */ 2335 tcg_gen_ori_tl(s->T0, cpu_cc_src, 0x02); 2336 tcg_gen_deposit_tl(cpu_regs[R_EAX], cpu_regs[R_EAX], s->T0, 8, 8); 2337} 2338 2339static void gen_LAR(DisasContext *s, X86DecodedInsn *decode) 2340{ 2341 MemOp ot = decode->op[0].ot; 2342 TCGv result = tcg_temp_new(); 2343 TCGv dest; 2344 2345 gen_compute_eflags(s); 2346 gen_update_cc_op(s); 2347 gen_helper_lar(result, tcg_env, s->T0); 2348 2349 /* Perform writeback here to skip it if ZF=0. */ 2350 decode->op[0].unit = X86_OP_SKIP; 2351 dest = gen_op_deposit_reg_v(s, ot, decode->op[0].n, result, result); 2352 tcg_gen_movcond_tl(TCG_COND_TSTNE, dest, cpu_cc_src, tcg_constant_tl(CC_Z), 2353 result, dest); 2354} 2355 2356static void gen_LDMXCSR(DisasContext *s, X86DecodedInsn *decode) 2357{ 2358 tcg_gen_trunc_tl_i32(s->tmp2_i32, s->T0); 2359 gen_helper_ldmxcsr(tcg_env, s->tmp2_i32); 2360} 2361 2362static void gen_lxx_seg(DisasContext *s, X86DecodedInsn *decode, int seg) 2363{ 2364 MemOp ot = decode->op[0].ot; 2365 2366 /* Offset already in s->T0. */ 2367 gen_add_A0_im(s, 1 << ot); 2368 gen_op_ld_v(s, MO_16, s->T1, s->A0); 2369 2370 /* load the segment here to handle exceptions properly */ 2371 gen_movl_seg(s, seg, s->T1); 2372} 2373 2374static void gen_LDS(DisasContext *s, X86DecodedInsn *decode) 2375{ 2376 gen_lxx_seg(s, decode, R_DS); 2377} 2378 2379static void gen_LEA(DisasContext *s, X86DecodedInsn *decode) 2380{ 2381 TCGv ea = gen_lea_modrm_1(s, decode->mem, false); 2382 gen_lea_v_seg_dest(s, s->aflag, s->T0, ea, -1, -1); 2383} 2384 2385static void gen_LEAVE(DisasContext *s, X86DecodedInsn *decode) 2386{ 2387 gen_leave(s); 2388} 2389 2390static void gen_LES(DisasContext *s, X86DecodedInsn *decode) 2391{ 2392 gen_lxx_seg(s, decode, R_ES); 2393} 2394 2395static void gen_LFENCE(DisasContext *s, X86DecodedInsn *decode) 2396{ 2397 tcg_gen_mb(TCG_MO_LD_LD | TCG_BAR_SC); 2398} 2399 2400static void gen_LFS(DisasContext *s, X86DecodedInsn *decode) 2401{ 2402 gen_lxx_seg(s, decode, R_FS); 2403} 2404 2405static void gen_LGS(DisasContext *s, X86DecodedInsn *decode) 2406{ 2407 gen_lxx_seg(s, decode, R_GS); 2408} 2409 2410static void gen_LODS(DisasContext *s, X86DecodedInsn *decode) 2411{ 2412 MemOp ot = decode->op[1].ot; 2413 if (s->prefix & (PREFIX_REPZ | PREFIX_REPNZ)) { 2414 gen_repz(s, ot, gen_lods); 2415 } else { 2416 gen_lods(s, ot); 2417 } 2418} 2419 2420static void gen_LOOP(DisasContext *s, X86DecodedInsn *decode) 2421{ 2422 TCGLabel *taken = gen_new_label(); 2423 2424 gen_update_cc_op(s); 2425 gen_op_add_reg_im(s, s->aflag, R_ECX, -1); 2426 gen_op_jnz_ecx(s, taken); 2427 gen_conditional_jump_labels(s, decode->immediate, NULL, taken); 2428} 2429 2430static void gen_LOOPE(DisasContext *s, X86DecodedInsn *decode) 2431{ 2432 TCGLabel *taken = gen_new_label(); 2433 TCGLabel *not_taken = gen_new_label(); 2434 2435 gen_update_cc_op(s); 2436 gen_op_add_reg_im(s, s->aflag, R_ECX, -1); 2437 gen_op_jz_ecx(s, not_taken); 2438 gen_jcc1(s, (JCC_Z << 1), taken); /* jz taken */ 2439 gen_conditional_jump_labels(s, decode->immediate, not_taken, taken); 2440} 2441 2442static void gen_LOOPNE(DisasContext *s, X86DecodedInsn *decode) 2443{ 2444 TCGLabel *taken = gen_new_label(); 2445 TCGLabel *not_taken = gen_new_label(); 2446 2447 gen_update_cc_op(s); 2448 gen_op_add_reg_im(s, s->aflag, R_ECX, -1); 2449 gen_op_jz_ecx(s, not_taken); 2450 gen_jcc1(s, (JCC_Z << 1) | 1, taken); /* jnz taken */ 2451 gen_conditional_jump_labels(s, decode->immediate, not_taken, taken); 2452} 2453 2454static void gen_LSL(DisasContext *s, X86DecodedInsn *decode) 2455{ 2456 MemOp ot = decode->op[0].ot; 2457 TCGv result = tcg_temp_new(); 2458 TCGv dest; 2459 2460 gen_compute_eflags(s); 2461 gen_update_cc_op(s); 2462 gen_helper_lsl(result, tcg_env, s->T0); 2463 2464 /* Perform writeback here to skip it if ZF=0. */ 2465 decode->op[0].unit = X86_OP_SKIP; 2466 dest = gen_op_deposit_reg_v(s, ot, decode->op[0].n, result, result); 2467 tcg_gen_movcond_tl(TCG_COND_TSTNE, dest, cpu_cc_src, tcg_constant_tl(CC_Z), 2468 result, dest); 2469} 2470 2471static void gen_LSS(DisasContext *s, X86DecodedInsn *decode) 2472{ 2473 gen_lxx_seg(s, decode, R_SS); 2474} 2475 2476static void gen_LZCNT(DisasContext *s, X86DecodedInsn *decode) 2477{ 2478 MemOp ot = decode->op[0].ot; 2479 2480 /* C bit (cc_src) is defined related to the input. */ 2481 decode->cc_src = tcg_temp_new(); 2482 decode->cc_dst = s->T0; 2483 decode->cc_op = CC_OP_BMILGB + ot; 2484 tcg_gen_mov_tl(decode->cc_src, s->T0); 2485 2486 /* 2487 * Reduce the target_ulong result by the number of zeros that 2488 * we expect to find at the top. 2489 */ 2490 tcg_gen_clzi_tl(s->T0, s->T0, TARGET_LONG_BITS); 2491 tcg_gen_subi_tl(s->T0, s->T0, TARGET_LONG_BITS - (8 << ot)); 2492} 2493 2494static void gen_MFENCE(DisasContext *s, X86DecodedInsn *decode) 2495{ 2496 tcg_gen_mb(TCG_MO_ALL | TCG_BAR_SC); 2497} 2498 2499static void gen_MOV(DisasContext *s, X86DecodedInsn *decode) 2500{ 2501 /* nothing to do! */ 2502} 2503#define gen_NOP gen_MOV 2504 2505static void gen_MASKMOV(DisasContext *s, X86DecodedInsn *decode) 2506{ 2507 gen_lea_v_seg(s, cpu_regs[R_EDI], R_DS, s->override); 2508 2509 if (s->prefix & PREFIX_DATA) { 2510 gen_helper_maskmov_xmm(tcg_env, OP_PTR1, OP_PTR2, s->A0); 2511 } else { 2512 gen_helper_maskmov_mmx(tcg_env, OP_PTR1, OP_PTR2, s->A0); 2513 } 2514} 2515 2516static void gen_MOVBE(DisasContext *s, X86DecodedInsn *decode) 2517{ 2518 MemOp ot = decode->op[0].ot; 2519 2520 /* M operand type does not load/store */ 2521 if (decode->e.op0 == X86_TYPE_M) { 2522 tcg_gen_qemu_st_tl(s->T0, s->A0, s->mem_index, ot | MO_BE); 2523 } else { 2524 tcg_gen_qemu_ld_tl(s->T0, s->A0, s->mem_index, ot | MO_BE); 2525 } 2526} 2527 2528static void gen_MOVD_from(DisasContext *s, X86DecodedInsn *decode) 2529{ 2530 MemOp ot = decode->op[2].ot; 2531 2532 switch (ot) { 2533 case MO_32: 2534#ifdef TARGET_X86_64 2535 tcg_gen_ld32u_tl(s->T0, tcg_env, decode->op[2].offset); 2536 break; 2537 case MO_64: 2538#endif 2539 tcg_gen_ld_tl(s->T0, tcg_env, decode->op[2].offset); 2540 break; 2541 default: 2542 abort(); 2543 } 2544} 2545 2546static void gen_MOVD_to(DisasContext *s, X86DecodedInsn *decode) 2547{ 2548 MemOp ot = decode->op[2].ot; 2549 int vec_len = vector_len(s, decode); 2550 int lo_ofs = vector_elem_offset(&decode->op[0], ot, 0); 2551 2552 tcg_gen_gvec_dup_imm(MO_64, decode->op[0].offset, vec_len, vec_len, 0); 2553 2554 switch (ot) { 2555 case MO_32: 2556#ifdef TARGET_X86_64 2557 tcg_gen_st32_tl(s->T1, tcg_env, lo_ofs); 2558 break; 2559 case MO_64: 2560#endif 2561 tcg_gen_st_tl(s->T1, tcg_env, lo_ofs); 2562 break; 2563 default: 2564 g_assert_not_reached(); 2565 } 2566} 2567 2568static void gen_MOVDQ(DisasContext *s, X86DecodedInsn *decode) 2569{ 2570 gen_store_sse(s, decode, decode->op[2].offset); 2571} 2572 2573static void gen_MOVMSK(DisasContext *s, X86DecodedInsn *decode) 2574{ 2575 typeof(gen_helper_movmskps_ymm) *ps, *pd, *fn; 2576 ps = s->vex_l ? gen_helper_movmskps_ymm : gen_helper_movmskps_xmm; 2577 pd = s->vex_l ? gen_helper_movmskpd_ymm : gen_helper_movmskpd_xmm; 2578 fn = s->prefix & PREFIX_DATA ? pd : ps; 2579 fn(s->tmp2_i32, tcg_env, OP_PTR2); 2580 tcg_gen_extu_i32_tl(s->T0, s->tmp2_i32); 2581} 2582 2583static void gen_MOVQ(DisasContext *s, X86DecodedInsn *decode) 2584{ 2585 int vec_len = vector_len(s, decode); 2586 int lo_ofs = vector_elem_offset(&decode->op[0], MO_64, 0); 2587 2588 tcg_gen_ld_i64(s->tmp1_i64, tcg_env, decode->op[2].offset); 2589 if (decode->op[0].has_ea) { 2590 tcg_gen_qemu_st_i64(s->tmp1_i64, s->A0, s->mem_index, MO_LEUQ); 2591 } else { 2592 /* 2593 * tcg_gen_gvec_dup_i64(MO_64, op0.offset, 8, vec_len, s->tmp1_64) would 2594 * seem to work, but it does not on big-endian platforms; the cleared parts 2595 * are always at higher addresses, but cross-endian emulation inverts the 2596 * byte order so that the cleared parts need to be at *lower* addresses. 2597 * Because oprsz is 8, we see this here even for SSE; but more in general, 2598 * it disqualifies using oprsz < maxsz to emulate VEX128. 2599 */ 2600 tcg_gen_gvec_dup_imm(MO_64, decode->op[0].offset, vec_len, vec_len, 0); 2601 tcg_gen_st_i64(s->tmp1_i64, tcg_env, lo_ofs); 2602 } 2603} 2604 2605static void gen_MOVq_dq(DisasContext *s, X86DecodedInsn *decode) 2606{ 2607 gen_helper_enter_mmx(tcg_env); 2608 /* Otherwise the same as any other movq. */ 2609 return gen_MOVQ(s, decode); 2610} 2611 2612static void gen_MOVS(DisasContext *s, X86DecodedInsn *decode) 2613{ 2614 MemOp ot = decode->op[2].ot; 2615 if (s->prefix & (PREFIX_REPZ | PREFIX_REPNZ)) { 2616 gen_repz(s, ot, gen_movs); 2617 } else { 2618 gen_movs(s, ot); 2619 } 2620} 2621 2622static void gen_MUL(DisasContext *s, X86DecodedInsn *decode) 2623{ 2624 MemOp ot = decode->op[1].ot; 2625 2626 switch (ot) { 2627 case MO_8: 2628 /* s->T0 already zero-extended */ 2629 tcg_gen_ext8u_tl(s->T1, s->T1); 2630 tcg_gen_mul_tl(s->T0, s->T0, s->T1); 2631 gen_op_mov_reg_v(s, MO_16, R_EAX, s->T0); 2632 tcg_gen_andi_tl(s->T1, s->T0, 0xff00); 2633 decode->cc_dst = s->T0; 2634 decode->cc_src = s->T1; 2635 break; 2636 2637 case MO_16: 2638 /* s->T0 already zero-extended */ 2639 tcg_gen_ext16u_tl(s->T1, s->T1); 2640 tcg_gen_mul_tl(s->T0, s->T0, s->T1); 2641 gen_op_mov_reg_v(s, MO_16, R_EAX, s->T0); 2642 tcg_gen_shri_tl(s->T1, s->T0, 16); 2643 gen_op_mov_reg_v(s, MO_16, R_EDX, s->T1); 2644 decode->cc_dst = s->T0; 2645 decode->cc_src = s->T1; 2646 break; 2647 2648 case MO_32: 2649#ifdef TARGET_X86_64 2650 /* s->T0 already zero-extended */ 2651 tcg_gen_ext32u_tl(s->T1, s->T1); 2652 tcg_gen_mul_tl(s->T0, s->T0, s->T1); 2653 tcg_gen_ext32u_tl(cpu_regs[R_EAX], s->T0); 2654 tcg_gen_shri_tl(cpu_regs[R_EDX], s->T0, 32); 2655 decode->cc_dst = cpu_regs[R_EAX]; 2656 decode->cc_src = cpu_regs[R_EDX]; 2657 break; 2658 2659 case MO_64: 2660#endif 2661 tcg_gen_mulu2_tl(cpu_regs[R_EAX], cpu_regs[R_EDX], s->T0, s->T1); 2662 decode->cc_dst = cpu_regs[R_EAX]; 2663 decode->cc_src = cpu_regs[R_EDX]; 2664 break; 2665 2666 default: 2667 g_assert_not_reached(); 2668 } 2669 2670 decode->cc_op = CC_OP_MULB + ot; 2671} 2672 2673static void gen_MULX(DisasContext *s, X86DecodedInsn *decode) 2674{ 2675 MemOp ot = decode->op[0].ot; 2676 2677 /* low part of result in VEX.vvvv, high in MODRM */ 2678 switch (ot) { 2679 case MO_32: 2680#ifdef TARGET_X86_64 2681 tcg_gen_trunc_tl_i32(s->tmp2_i32, s->T0); 2682 tcg_gen_trunc_tl_i32(s->tmp3_i32, s->T1); 2683 tcg_gen_mulu2_i32(s->tmp2_i32, s->tmp3_i32, 2684 s->tmp2_i32, s->tmp3_i32); 2685 tcg_gen_extu_i32_tl(cpu_regs[s->vex_v], s->tmp2_i32); 2686 tcg_gen_extu_i32_tl(s->T0, s->tmp3_i32); 2687 break; 2688 2689 case MO_64: 2690#endif 2691 tcg_gen_mulu2_tl(cpu_regs[s->vex_v], s->T0, s->T0, s->T1); 2692 break; 2693 2694 default: 2695 g_assert_not_reached(); 2696 } 2697} 2698 2699static void gen_NEG(DisasContext *s, X86DecodedInsn *decode) 2700{ 2701 MemOp ot = decode->op[0].ot; 2702 TCGv oldv = tcg_temp_new(); 2703 2704 if (s->prefix & PREFIX_LOCK) { 2705 TCGv newv = tcg_temp_new(); 2706 TCGv cmpv = tcg_temp_new(); 2707 TCGLabel *label1 = gen_new_label(); 2708 2709 gen_set_label(label1); 2710 gen_op_ld_v(s, ot, oldv, s->A0); 2711 tcg_gen_neg_tl(newv, oldv); 2712 tcg_gen_atomic_cmpxchg_tl(cmpv, s->A0, oldv, newv, 2713 s->mem_index, ot | MO_LE); 2714 tcg_gen_brcond_tl(TCG_COND_NE, oldv, cmpv, label1); 2715 } else { 2716 tcg_gen_mov_tl(oldv, s->T0); 2717 } 2718 tcg_gen_neg_tl(s->T0, oldv); 2719 2720 decode->cc_dst = s->T0; 2721 decode->cc_src = oldv; 2722 tcg_gen_movi_tl(s->cc_srcT, 0); 2723 decode->cc_op = CC_OP_SUBB + ot; 2724} 2725 2726static void gen_NOT(DisasContext *s, X86DecodedInsn *decode) 2727{ 2728 MemOp ot = decode->op[0].ot; 2729 2730 if (s->prefix & PREFIX_LOCK) { 2731 tcg_gen_movi_tl(s->T0, ~0); 2732 tcg_gen_atomic_xor_fetch_tl(s->T0, s->A0, s->T0, 2733 s->mem_index, ot | MO_LE); 2734 } else { 2735 tcg_gen_not_tl(s->T0, s->T0); 2736 } 2737} 2738 2739static void gen_OR(DisasContext *s, X86DecodedInsn *decode) 2740{ 2741 MemOp ot = decode->op[1].ot; 2742 2743 if (s->prefix & PREFIX_LOCK) { 2744 tcg_gen_atomic_or_fetch_tl(s->T0, s->A0, s->T1, 2745 s->mem_index, ot | MO_LE); 2746 } else { 2747 tcg_gen_or_tl(s->T0, s->T0, s->T1); 2748 } 2749 prepare_update1_cc(decode, s, CC_OP_LOGICB + ot); 2750} 2751 2752static void gen_OUT(DisasContext *s, X86DecodedInsn *decode) 2753{ 2754 MemOp ot = decode->op[1].ot; 2755 TCGv_i32 port = tcg_temp_new_i32(); 2756 TCGv_i32 value = tcg_temp_new_i32(); 2757 2758 tcg_gen_trunc_tl_i32(port, s->T1); 2759 tcg_gen_ext16u_i32(port, port); 2760 if (!gen_check_io(s, ot, port, 0)) { 2761 return; 2762 } 2763 tcg_gen_trunc_tl_i32(value, s->T0); 2764 translator_io_start(&s->base); 2765 gen_helper_out_func(ot, port, value); 2766 gen_bpt_io(s, port, ot); 2767} 2768 2769static void gen_OUTS(DisasContext *s, X86DecodedInsn *decode) 2770{ 2771 MemOp ot = decode->op[1].ot; 2772 TCGv_i32 port = tcg_temp_new_i32(); 2773 2774 tcg_gen_trunc_tl_i32(port, s->T1); 2775 tcg_gen_ext16u_i32(port, port); 2776 if (!gen_check_io(s, ot, port, SVM_IOIO_STR_MASK)) { 2777 return; 2778 } 2779 2780 translator_io_start(&s->base); 2781 if (s->prefix & (PREFIX_REPZ | PREFIX_REPNZ)) { 2782 gen_repz(s, ot, gen_outs); 2783 } else { 2784 gen_outs(s, ot); 2785 } 2786} 2787 2788static void gen_PALIGNR(DisasContext *s, X86DecodedInsn *decode) 2789{ 2790 TCGv_i32 imm = tcg_constant8u_i32(decode->immediate); 2791 if (!(s->prefix & PREFIX_DATA)) { 2792 gen_helper_palignr_mmx(tcg_env, OP_PTR0, OP_PTR1, OP_PTR2, imm); 2793 } else if (!s->vex_l) { 2794 gen_helper_palignr_xmm(tcg_env, OP_PTR0, OP_PTR1, OP_PTR2, imm); 2795 } else { 2796 gen_helper_palignr_ymm(tcg_env, OP_PTR0, OP_PTR1, OP_PTR2, imm); 2797 } 2798} 2799 2800static void gen_PANDN(DisasContext *s, X86DecodedInsn *decode) 2801{ 2802 int vec_len = vector_len(s, decode); 2803 2804 /* Careful, operand order is reversed! */ 2805 tcg_gen_gvec_andc(MO_64, 2806 decode->op[0].offset, decode->op[2].offset, 2807 decode->op[1].offset, vec_len, vec_len); 2808} 2809 2810static void gen_PAUSE(DisasContext *s, X86DecodedInsn *decode) 2811{ 2812 gen_update_cc_op(s); 2813 gen_update_eip_next(s); 2814 gen_helper_pause(tcg_env); 2815 s->base.is_jmp = DISAS_NORETURN; 2816} 2817 2818static void gen_PCMPESTRI(DisasContext *s, X86DecodedInsn *decode) 2819{ 2820 TCGv_i32 imm = tcg_constant8u_i32(decode->immediate); 2821 gen_helper_pcmpestri_xmm(tcg_env, OP_PTR1, OP_PTR2, imm); 2822 assume_cc_op(s, CC_OP_EFLAGS); 2823} 2824 2825static void gen_PCMPESTRM(DisasContext *s, X86DecodedInsn *decode) 2826{ 2827 TCGv_i32 imm = tcg_constant8u_i32(decode->immediate); 2828 gen_helper_pcmpestrm_xmm(tcg_env, OP_PTR1, OP_PTR2, imm); 2829 assume_cc_op(s, CC_OP_EFLAGS); 2830 if ((s->prefix & PREFIX_VEX) && !s->vex_l) { 2831 tcg_gen_gvec_dup_imm(MO_64, offsetof(CPUX86State, xmm_regs[0].ZMM_X(1)), 2832 16, 16, 0); 2833 } 2834} 2835 2836static void gen_PCMPISTRI(DisasContext *s, X86DecodedInsn *decode) 2837{ 2838 TCGv_i32 imm = tcg_constant8u_i32(decode->immediate); 2839 gen_helper_pcmpistri_xmm(tcg_env, OP_PTR1, OP_PTR2, imm); 2840 assume_cc_op(s, CC_OP_EFLAGS); 2841} 2842 2843static void gen_PCMPISTRM(DisasContext *s, X86DecodedInsn *decode) 2844{ 2845 TCGv_i32 imm = tcg_constant8u_i32(decode->immediate); 2846 gen_helper_pcmpistrm_xmm(tcg_env, OP_PTR1, OP_PTR2, imm); 2847 assume_cc_op(s, CC_OP_EFLAGS); 2848 if ((s->prefix & PREFIX_VEX) && !s->vex_l) { 2849 tcg_gen_gvec_dup_imm(MO_64, offsetof(CPUX86State, xmm_regs[0].ZMM_X(1)), 2850 16, 16, 0); 2851 } 2852} 2853 2854static void gen_PDEP(DisasContext *s, X86DecodedInsn *decode) 2855{ 2856 gen_helper_pdep(s->T0, s->T0, s->T1); 2857} 2858 2859static void gen_PEXT(DisasContext *s, X86DecodedInsn *decode) 2860{ 2861 gen_helper_pext(s->T0, s->T0, s->T1); 2862} 2863 2864static inline void gen_pextr(DisasContext *s, X86DecodedInsn *decode, MemOp ot) 2865{ 2866 int vec_len = vector_len(s, decode); 2867 int mask = (vec_len >> ot) - 1; 2868 int val = decode->immediate & mask; 2869 2870 switch (ot) { 2871 case MO_8: 2872 tcg_gen_ld8u_tl(s->T0, tcg_env, vector_elem_offset(&decode->op[1], ot, val)); 2873 break; 2874 case MO_16: 2875 tcg_gen_ld16u_tl(s->T0, tcg_env, vector_elem_offset(&decode->op[1], ot, val)); 2876 break; 2877 case MO_32: 2878#ifdef TARGET_X86_64 2879 tcg_gen_ld32u_tl(s->T0, tcg_env, vector_elem_offset(&decode->op[1], ot, val)); 2880 break; 2881 case MO_64: 2882#endif 2883 tcg_gen_ld_tl(s->T0, tcg_env, vector_elem_offset(&decode->op[1], ot, val)); 2884 break; 2885 default: 2886 abort(); 2887 } 2888} 2889 2890static void gen_PEXTRB(DisasContext *s, X86DecodedInsn *decode) 2891{ 2892 gen_pextr(s, decode, MO_8); 2893} 2894 2895static void gen_PEXTRW(DisasContext *s, X86DecodedInsn *decode) 2896{ 2897 gen_pextr(s, decode, MO_16); 2898} 2899 2900static void gen_PEXTR(DisasContext *s, X86DecodedInsn *decode) 2901{ 2902 MemOp ot = decode->op[0].ot; 2903 gen_pextr(s, decode, ot); 2904} 2905 2906static inline void gen_pinsr(DisasContext *s, X86DecodedInsn *decode, MemOp ot) 2907{ 2908 int vec_len = vector_len(s, decode); 2909 int mask = (vec_len >> ot) - 1; 2910 int val = decode->immediate & mask; 2911 2912 if (decode->op[1].offset != decode->op[0].offset) { 2913 assert(vec_len == 16); 2914 gen_store_sse(s, decode, decode->op[1].offset); 2915 } 2916 2917 switch (ot) { 2918 case MO_8: 2919 tcg_gen_st8_tl(s->T1, tcg_env, vector_elem_offset(&decode->op[0], ot, val)); 2920 break; 2921 case MO_16: 2922 tcg_gen_st16_tl(s->T1, tcg_env, vector_elem_offset(&decode->op[0], ot, val)); 2923 break; 2924 case MO_32: 2925#ifdef TARGET_X86_64 2926 tcg_gen_st32_tl(s->T1, tcg_env, vector_elem_offset(&decode->op[0], ot, val)); 2927 break; 2928 case MO_64: 2929#endif 2930 tcg_gen_st_tl(s->T1, tcg_env, vector_elem_offset(&decode->op[0], ot, val)); 2931 break; 2932 default: 2933 abort(); 2934 } 2935} 2936 2937static void gen_PINSRB(DisasContext *s, X86DecodedInsn *decode) 2938{ 2939 gen_pinsr(s, decode, MO_8); 2940} 2941 2942static void gen_PINSRW(DisasContext *s, X86DecodedInsn *decode) 2943{ 2944 gen_pinsr(s, decode, MO_16); 2945} 2946 2947static void gen_PINSR(DisasContext *s, X86DecodedInsn *decode) 2948{ 2949 gen_pinsr(s, decode, decode->op[2].ot); 2950} 2951 2952static void gen_pmovmskb_i64(TCGv_i64 d, TCGv_i64 s) 2953{ 2954 TCGv_i64 t = tcg_temp_new_i64(); 2955 2956 tcg_gen_andi_i64(d, s, 0x8080808080808080ull); 2957 2958 /* 2959 * After each shift+or pair: 2960 * 0: a.......b.......c.......d.......e.......f.......g.......h....... 2961 * 7: ab......bc......cd......de......ef......fg......gh......h....... 2962 * 14: abcd....bcde....cdef....defg....efgh....fgh.....gh......h....... 2963 * 28: abcdefghbcdefgh.cdefgh..defgh...efgh....fgh.....gh......h....... 2964 * The result is left in the high bits of the word. 2965 */ 2966 tcg_gen_shli_i64(t, d, 7); 2967 tcg_gen_or_i64(d, d, t); 2968 tcg_gen_shli_i64(t, d, 14); 2969 tcg_gen_or_i64(d, d, t); 2970 tcg_gen_shli_i64(t, d, 28); 2971 tcg_gen_or_i64(d, d, t); 2972} 2973 2974static void gen_pmovmskb_vec(unsigned vece, TCGv_vec d, TCGv_vec s) 2975{ 2976 TCGv_vec t = tcg_temp_new_vec_matching(d); 2977 TCGv_vec m = tcg_constant_vec_matching(d, MO_8, 0x80); 2978 2979 /* See above */ 2980 tcg_gen_and_vec(vece, d, s, m); 2981 tcg_gen_shli_vec(vece, t, d, 7); 2982 tcg_gen_or_vec(vece, d, d, t); 2983 tcg_gen_shli_vec(vece, t, d, 14); 2984 tcg_gen_or_vec(vece, d, d, t); 2985 tcg_gen_shli_vec(vece, t, d, 28); 2986 tcg_gen_or_vec(vece, d, d, t); 2987} 2988 2989static void gen_PMOVMSKB(DisasContext *s, X86DecodedInsn *decode) 2990{ 2991 static const TCGOpcode vecop_list[] = { INDEX_op_shli_vec, 0 }; 2992 static const GVecGen2 g = { 2993 .fni8 = gen_pmovmskb_i64, 2994 .fniv = gen_pmovmskb_vec, 2995 .opt_opc = vecop_list, 2996 .vece = MO_64, 2997 .prefer_i64 = TCG_TARGET_REG_BITS == 64 2998 }; 2999 MemOp ot = decode->op[2].ot; 3000 int vec_len = vector_len(s, decode); 3001 TCGv t = tcg_temp_new(); 3002 3003 tcg_gen_gvec_2(offsetof(CPUX86State, xmm_t0) + xmm_offset(ot), decode->op[2].offset, 3004 vec_len, vec_len, &g); 3005 tcg_gen_ld8u_tl(s->T0, tcg_env, offsetof(CPUX86State, xmm_t0.ZMM_B(vec_len - 1))); 3006 while (vec_len > 8) { 3007 vec_len -= 8; 3008 if (TCG_TARGET_HAS_extract2_tl) { 3009 /* 3010 * Load the next byte of the result into the high byte of T. 3011 * TCG does a similar expansion of deposit to shl+extract2; by 3012 * loading the whole word, the shift left is avoided. 3013 */ 3014#ifdef TARGET_X86_64 3015 tcg_gen_ld_tl(t, tcg_env, offsetof(CPUX86State, xmm_t0.ZMM_Q((vec_len - 1) / 8))); 3016#else 3017 tcg_gen_ld_tl(t, tcg_env, offsetof(CPUX86State, xmm_t0.ZMM_L((vec_len - 1) / 4))); 3018#endif 3019 3020 tcg_gen_extract2_tl(s->T0, t, s->T0, TARGET_LONG_BITS - 8); 3021 } else { 3022 /* 3023 * The _previous_ value is deposited into bits 8 and higher of t. Because 3024 * those bits are known to be zero after ld8u, this becomes a shift+or 3025 * if deposit is not available. 3026 */ 3027 tcg_gen_ld8u_tl(t, tcg_env, offsetof(CPUX86State, xmm_t0.ZMM_B(vec_len - 1))); 3028 tcg_gen_deposit_tl(s->T0, t, s->T0, 8, TARGET_LONG_BITS - 8); 3029 } 3030 } 3031} 3032 3033static void gen_POP(DisasContext *s, X86DecodedInsn *decode) 3034{ 3035 X86DecodedOp *op = &decode->op[0]; 3036 MemOp ot = gen_pop_T0(s); 3037 3038 assert(ot >= op->ot); 3039 if (op->has_ea || op->unit == X86_OP_SEG) { 3040 /* NOTE: order is important for MMU exceptions */ 3041 gen_writeback(s, decode, 0, s->T0); 3042 } 3043 3044 /* NOTE: writing back registers after update is important for pop %sp */ 3045 gen_pop_update(s, ot); 3046} 3047 3048static void gen_POPA(DisasContext *s, X86DecodedInsn *decode) 3049{ 3050 gen_popa(s); 3051} 3052 3053static void gen_POPCNT(DisasContext *s, X86DecodedInsn *decode) 3054{ 3055 decode->cc_dst = tcg_temp_new(); 3056 decode->cc_op = CC_OP_POPCNT; 3057 3058 tcg_gen_mov_tl(decode->cc_dst, s->T0); 3059 tcg_gen_ctpop_tl(s->T0, s->T0); 3060} 3061 3062static void gen_POPF(DisasContext *s, X86DecodedInsn *decode) 3063{ 3064 MemOp ot; 3065 int mask = TF_MASK | AC_MASK | ID_MASK | NT_MASK; 3066 3067 if (CPL(s) == 0) { 3068 mask |= IF_MASK | IOPL_MASK; 3069 } else if (CPL(s) <= IOPL(s)) { 3070 mask |= IF_MASK; 3071 } 3072 if (s->dflag == MO_16) { 3073 mask &= 0xffff; 3074 } 3075 3076 ot = gen_pop_T0(s); 3077 gen_helper_write_eflags(tcg_env, s->T0, tcg_constant_i32(mask)); 3078 gen_pop_update(s, ot); 3079 set_cc_op(s, CC_OP_EFLAGS); 3080 /* abort translation because TF/AC flag may change */ 3081 s->base.is_jmp = DISAS_EOB_NEXT; 3082} 3083 3084static void gen_PSHUFW(DisasContext *s, X86DecodedInsn *decode) 3085{ 3086 TCGv_i32 imm = tcg_constant8u_i32(decode->immediate); 3087 gen_helper_pshufw_mmx(OP_PTR0, OP_PTR1, imm); 3088} 3089 3090static void gen_PSRLW_i(DisasContext *s, X86DecodedInsn *decode) 3091{ 3092 int vec_len = vector_len(s, decode); 3093 3094 if (decode->immediate >= 16) { 3095 tcg_gen_gvec_dup_imm(MO_64, decode->op[0].offset, vec_len, vec_len, 0); 3096 } else { 3097 tcg_gen_gvec_shri(MO_16, 3098 decode->op[0].offset, decode->op[1].offset, 3099 decode->immediate, vec_len, vec_len); 3100 } 3101} 3102 3103static void gen_PSLLW_i(DisasContext *s, X86DecodedInsn *decode) 3104{ 3105 int vec_len = vector_len(s, decode); 3106 3107 if (decode->immediate >= 16) { 3108 tcg_gen_gvec_dup_imm(MO_64, decode->op[0].offset, vec_len, vec_len, 0); 3109 } else { 3110 tcg_gen_gvec_shli(MO_16, 3111 decode->op[0].offset, decode->op[1].offset, 3112 decode->immediate, vec_len, vec_len); 3113 } 3114} 3115 3116static void gen_PSRAW_i(DisasContext *s, X86DecodedInsn *decode) 3117{ 3118 int vec_len = vector_len(s, decode); 3119 3120 if (decode->immediate >= 16) { 3121 decode->immediate = 15; 3122 } 3123 tcg_gen_gvec_sari(MO_16, 3124 decode->op[0].offset, decode->op[1].offset, 3125 decode->immediate, vec_len, vec_len); 3126} 3127 3128static void gen_PSRLD_i(DisasContext *s, X86DecodedInsn *decode) 3129{ 3130 int vec_len = vector_len(s, decode); 3131 3132 if (decode->immediate >= 32) { 3133 tcg_gen_gvec_dup_imm(MO_64, decode->op[0].offset, vec_len, vec_len, 0); 3134 } else { 3135 tcg_gen_gvec_shri(MO_32, 3136 decode->op[0].offset, decode->op[1].offset, 3137 decode->immediate, vec_len, vec_len); 3138 } 3139} 3140 3141static void gen_PSLLD_i(DisasContext *s, X86DecodedInsn *decode) 3142{ 3143 int vec_len = vector_len(s, decode); 3144 3145 if (decode->immediate >= 32) { 3146 tcg_gen_gvec_dup_imm(MO_64, decode->op[0].offset, vec_len, vec_len, 0); 3147 } else { 3148 tcg_gen_gvec_shli(MO_32, 3149 decode->op[0].offset, decode->op[1].offset, 3150 decode->immediate, vec_len, vec_len); 3151 } 3152} 3153 3154static void gen_PSRAD_i(DisasContext *s, X86DecodedInsn *decode) 3155{ 3156 int vec_len = vector_len(s, decode); 3157 3158 if (decode->immediate >= 32) { 3159 decode->immediate = 31; 3160 } 3161 tcg_gen_gvec_sari(MO_32, 3162 decode->op[0].offset, decode->op[1].offset, 3163 decode->immediate, vec_len, vec_len); 3164} 3165 3166static void gen_PSRLQ_i(DisasContext *s, X86DecodedInsn *decode) 3167{ 3168 int vec_len = vector_len(s, decode); 3169 3170 if (decode->immediate >= 64) { 3171 tcg_gen_gvec_dup_imm(MO_64, decode->op[0].offset, vec_len, vec_len, 0); 3172 } else { 3173 tcg_gen_gvec_shri(MO_64, 3174 decode->op[0].offset, decode->op[1].offset, 3175 decode->immediate, vec_len, vec_len); 3176 } 3177} 3178 3179static void gen_PSLLQ_i(DisasContext *s, X86DecodedInsn *decode) 3180{ 3181 int vec_len = vector_len(s, decode); 3182 3183 if (decode->immediate >= 64) { 3184 tcg_gen_gvec_dup_imm(MO_64, decode->op[0].offset, vec_len, vec_len, 0); 3185 } else { 3186 tcg_gen_gvec_shli(MO_64, 3187 decode->op[0].offset, decode->op[1].offset, 3188 decode->immediate, vec_len, vec_len); 3189 } 3190} 3191 3192static TCGv_ptr make_imm8u_xmm_vec(uint8_t imm, int vec_len) 3193{ 3194 MemOp ot = vec_len == 16 ? MO_128 : MO_256; 3195 TCGv_i32 imm_v = tcg_constant8u_i32(imm); 3196 TCGv_ptr ptr = tcg_temp_new_ptr(); 3197 3198 tcg_gen_gvec_dup_imm(MO_64, offsetof(CPUX86State, xmm_t0) + xmm_offset(ot), 3199 vec_len, vec_len, 0); 3200 3201 tcg_gen_addi_ptr(ptr, tcg_env, offsetof(CPUX86State, xmm_t0)); 3202 tcg_gen_st_i32(imm_v, tcg_env, offsetof(CPUX86State, xmm_t0.ZMM_L(0))); 3203 return ptr; 3204} 3205 3206static void gen_PSRLDQ_i(DisasContext *s, X86DecodedInsn *decode) 3207{ 3208 int vec_len = vector_len(s, decode); 3209 TCGv_ptr imm_vec = make_imm8u_xmm_vec(decode->immediate, vec_len); 3210 3211 if (s->vex_l) { 3212 gen_helper_psrldq_ymm(tcg_env, OP_PTR0, OP_PTR1, imm_vec); 3213 } else { 3214 gen_helper_psrldq_xmm(tcg_env, OP_PTR0, OP_PTR1, imm_vec); 3215 } 3216} 3217 3218static void gen_PSLLDQ_i(DisasContext *s, X86DecodedInsn *decode) 3219{ 3220 int vec_len = vector_len(s, decode); 3221 TCGv_ptr imm_vec = make_imm8u_xmm_vec(decode->immediate, vec_len); 3222 3223 if (s->vex_l) { 3224 gen_helper_pslldq_ymm(tcg_env, OP_PTR0, OP_PTR1, imm_vec); 3225 } else { 3226 gen_helper_pslldq_xmm(tcg_env, OP_PTR0, OP_PTR1, imm_vec); 3227 } 3228} 3229 3230static void gen_PUSH(DisasContext *s, X86DecodedInsn *decode) 3231{ 3232 gen_push_v(s, s->T0); 3233} 3234 3235static void gen_PUSHA(DisasContext *s, X86DecodedInsn *decode) 3236{ 3237 gen_pusha(s); 3238} 3239 3240static void gen_PUSHF(DisasContext *s, X86DecodedInsn *decode) 3241{ 3242 gen_update_cc_op(s); 3243 gen_helper_read_eflags(s->T0, tcg_env); 3244 gen_push_v(s, s->T0); 3245} 3246 3247static MemOp gen_shift_count(DisasContext *s, X86DecodedInsn *decode, 3248 bool *can_be_zero, TCGv *count, int unit) 3249{ 3250 MemOp ot = decode->op[0].ot; 3251 int mask = (ot <= MO_32 ? 0x1f : 0x3f); 3252 3253 *can_be_zero = false; 3254 switch (unit) { 3255 case X86_OP_INT: 3256 *count = tcg_temp_new(); 3257 tcg_gen_andi_tl(*count, cpu_regs[R_ECX], mask); 3258 *can_be_zero = true; 3259 break; 3260 3261 case X86_OP_IMM: 3262 if ((decode->immediate & mask) == 0) { 3263 *count = NULL; 3264 break; 3265 } 3266 *count = tcg_temp_new(); 3267 tcg_gen_movi_tl(*count, decode->immediate & mask); 3268 break; 3269 3270 case X86_OP_SKIP: 3271 *count = tcg_temp_new(); 3272 tcg_gen_movi_tl(*count, 1); 3273 break; 3274 3275 default: 3276 g_assert_not_reached(); 3277 } 3278 3279 return ot; 3280} 3281 3282/* 3283 * Compute existing flags in decode->cc_src, for gen_* functions that wants 3284 * to set the cc_op set to CC_OP_ADCOX. In particular, this allows rotate 3285 * operations to compute the carry in decode->cc_dst and the overflow in 3286 * decode->cc_src2. 3287 * 3288 * If need_flags is true, decode->cc_dst and decode->cc_src2 are preloaded 3289 * with the value of CF and OF before the instruction, so that it is possible 3290 * to keep the flags unmodified. 3291 * 3292 * Return true if carry could be made available cheaply as a 1-bit value in 3293 * decode->cc_dst (trying a bit harder if want_carry is true). If false is 3294 * returned, decode->cc_dst is uninitialized and the carry is only available 3295 * as bit 0 of decode->cc_src. 3296 */ 3297static bool gen_eflags_adcox(DisasContext *s, X86DecodedInsn *decode, bool want_carry, bool need_flags) 3298{ 3299 bool got_cf = false; 3300 bool got_of = false; 3301 3302 decode->cc_dst = tcg_temp_new(); 3303 decode->cc_src = tcg_temp_new(); 3304 decode->cc_src2 = tcg_temp_new(); 3305 decode->cc_op = CC_OP_ADCOX; 3306 3307 /* A lot more cc_ops could be "optimized" to avoid the extracts at 3308 * the end (INC/DEC, BMILG, MUL), but they are all really unlikely 3309 * to be followed by rotations within the same basic block. 3310 */ 3311 switch (s->cc_op) { 3312 case CC_OP_ADCOX: 3313 /* No need to compute the full EFLAGS, CF/OF are already isolated. */ 3314 tcg_gen_mov_tl(decode->cc_src, cpu_cc_src); 3315 if (need_flags) { 3316 tcg_gen_mov_tl(decode->cc_src2, cpu_cc_src2); 3317 got_of = true; 3318 } 3319 if (want_carry || need_flags) { 3320 tcg_gen_mov_tl(decode->cc_dst, cpu_cc_dst); 3321 got_cf = true; 3322 } 3323 break; 3324 3325 case CC_OP_LOGICB ... CC_OP_LOGICQ: 3326 /* CF and OF are zero, do it just because it's easy. */ 3327 gen_mov_eflags(s, decode->cc_src); 3328 if (need_flags) { 3329 tcg_gen_movi_tl(decode->cc_src2, 0); 3330 got_of = true; 3331 } 3332 if (want_carry || need_flags) { 3333 tcg_gen_movi_tl(decode->cc_dst, 0); 3334 got_cf = true; 3335 } 3336 break; 3337 3338 case CC_OP_SARB ... CC_OP_SARQ: 3339 /* 3340 * SHR/RCR/SHR/RCR/... is a relatively common occurrence of RCR. 3341 * By computing CF without using eflags, the calls to cc_compute_all 3342 * can be eliminated as dead code (except for the last RCR). 3343 */ 3344 if (want_carry || need_flags) { 3345 tcg_gen_andi_tl(decode->cc_dst, cpu_cc_src, 1); 3346 got_cf = true; 3347 } 3348 gen_mov_eflags(s, decode->cc_src); 3349 break; 3350 3351 case CC_OP_SHLB ... CC_OP_SHLQ: 3352 /* 3353 * Likewise for SHL/RCL/SHL/RCL/... but, if CF is not in the sign 3354 * bit, we might as well fish CF out of EFLAGS and save a shift. 3355 */ 3356 if (want_carry && (!need_flags || s->cc_op == CC_OP_SHLB + MO_TL)) { 3357 tcg_gen_shri_tl(decode->cc_dst, cpu_cc_src, (8 << (s->cc_op - CC_OP_SHLB)) - 1); 3358 got_cf = true; 3359 } 3360 gen_mov_eflags(s, decode->cc_src); 3361 break; 3362 3363 default: 3364 gen_mov_eflags(s, decode->cc_src); 3365 break; 3366 } 3367 3368 if (need_flags) { 3369 /* If the flags could be left unmodified, always load them. */ 3370 if (!got_of) { 3371 tcg_gen_extract_tl(decode->cc_src2, decode->cc_src, ctz32(CC_O), 1); 3372 got_of = true; 3373 } 3374 if (!got_cf) { 3375 tcg_gen_extract_tl(decode->cc_dst, decode->cc_src, ctz32(CC_C), 1); 3376 got_cf = true; 3377 } 3378 } 3379 return got_cf; 3380} 3381 3382static void gen_rot_overflow(X86DecodedInsn *decode, TCGv result, TCGv old, 3383 bool can_be_zero, TCGv count) 3384{ 3385 MemOp ot = decode->op[0].ot; 3386 TCGv temp = can_be_zero ? tcg_temp_new() : decode->cc_src2; 3387 3388 tcg_gen_xor_tl(temp, old, result); 3389 tcg_gen_extract_tl(temp, temp, (8 << ot) - 1, 1); 3390 if (can_be_zero) { 3391 tcg_gen_movcond_tl(TCG_COND_EQ, decode->cc_src2, count, tcg_constant_tl(0), 3392 decode->cc_src2, temp); 3393 } 3394} 3395 3396/* 3397 * RCx operations are invariant modulo 8*operand_size+1. For 8 and 16-bit operands, 3398 * this is less than 0x1f (the mask applied by gen_shift_count) so reduce further. 3399 */ 3400static void gen_rotc_mod(MemOp ot, TCGv count) 3401{ 3402 TCGv temp; 3403 3404 switch (ot) { 3405 case MO_8: 3406 temp = tcg_temp_new(); 3407 tcg_gen_subi_tl(temp, count, 18); 3408 tcg_gen_movcond_tl(TCG_COND_GE, count, temp, tcg_constant_tl(0), temp, count); 3409 tcg_gen_subi_tl(temp, count, 9); 3410 tcg_gen_movcond_tl(TCG_COND_GE, count, temp, tcg_constant_tl(0), temp, count); 3411 break; 3412 3413 case MO_16: 3414 temp = tcg_temp_new(); 3415 tcg_gen_subi_tl(temp, count, 17); 3416 tcg_gen_movcond_tl(TCG_COND_GE, count, temp, tcg_constant_tl(0), temp, count); 3417 break; 3418 3419 default: 3420 break; 3421 } 3422} 3423 3424/* 3425 * The idea here is that the bit to the right of the new bit 0 is the 3426 * new carry, and the bit to the right of the old bit 0 is the old carry. 3427 * Just like a regular rotation, the result of the rotation is composed 3428 * from a right shifted part and a left shifted part of s->T0. The new carry 3429 * is extracted from the right-shifted portion, and the old carry is 3430 * inserted at the end of the left-shifted portion. 3431 * 3432 * Because of the separate shifts involving the carry, gen_RCL and gen_RCR 3433 * mostly operate on count-1. This also comes in handy when computing 3434 * length - count, because (length-1) - (count-1) can be computed with 3435 * a XOR, and that is commutative unlike subtraction. 3436 */ 3437static void gen_RCL(DisasContext *s, X86DecodedInsn *decode) 3438{ 3439 bool have_1bit_cin, can_be_zero; 3440 TCGv count; 3441 TCGLabel *zero_label = NULL; 3442 MemOp ot = gen_shift_count(s, decode, &can_be_zero, &count, decode->op[2].unit); 3443 TCGv low, high, low_count; 3444 3445 if (!count) { 3446 return; 3447 } 3448 3449 low = tcg_temp_new(); 3450 high = tcg_temp_new(); 3451 low_count = tcg_temp_new(); 3452 3453 gen_rotc_mod(ot, count); 3454 have_1bit_cin = gen_eflags_adcox(s, decode, true, can_be_zero); 3455 if (can_be_zero) { 3456 zero_label = gen_new_label(); 3457 tcg_gen_brcondi_tl(TCG_COND_EQ, count, 0, zero_label); 3458 } 3459 3460 /* Compute high part, including incoming carry. */ 3461 if (!have_1bit_cin || TCG_TARGET_deposit_tl_valid(1, TARGET_LONG_BITS - 1)) { 3462 /* high = (T0 << 1) | cin */ 3463 TCGv cin = have_1bit_cin ? decode->cc_dst : decode->cc_src; 3464 tcg_gen_deposit_tl(high, cin, s->T0, 1, TARGET_LONG_BITS - 1); 3465 } else { 3466 /* Same as above but without deposit; cin in cc_dst. */ 3467 tcg_gen_add_tl(high, s->T0, decode->cc_dst); 3468 tcg_gen_add_tl(high, high, s->T0); 3469 } 3470 tcg_gen_subi_tl(count, count, 1); 3471 tcg_gen_shl_tl(high, high, count); 3472 3473 /* Compute low part and outgoing carry, incoming s->T0 is zero extended */ 3474 tcg_gen_xori_tl(low_count, count, (8 << ot) - 1); /* LENGTH - 1 - (count - 1) */ 3475 tcg_gen_shr_tl(low, s->T0, low_count); 3476 tcg_gen_andi_tl(decode->cc_dst, low, 1); 3477 tcg_gen_shri_tl(low, low, 1); 3478 3479 /* Compute result and outgoing overflow */ 3480 tcg_gen_mov_tl(decode->cc_src2, s->T0); 3481 tcg_gen_or_tl(s->T0, low, high); 3482 gen_rot_overflow(decode, s->T0, decode->cc_src2, false, NULL); 3483 3484 if (zero_label) { 3485 gen_set_label(zero_label); 3486 } 3487} 3488 3489static void gen_RCR(DisasContext *s, X86DecodedInsn *decode) 3490{ 3491 bool have_1bit_cin, can_be_zero; 3492 TCGv count; 3493 TCGLabel *zero_label = NULL; 3494 MemOp ot = gen_shift_count(s, decode, &can_be_zero, &count, decode->op[2].unit); 3495 TCGv low, high, high_count; 3496 3497 if (!count) { 3498 return; 3499 } 3500 3501 low = tcg_temp_new(); 3502 high = tcg_temp_new(); 3503 high_count = tcg_temp_new(); 3504 3505 gen_rotc_mod(ot, count); 3506 have_1bit_cin = gen_eflags_adcox(s, decode, true, can_be_zero); 3507 if (can_be_zero) { 3508 zero_label = gen_new_label(); 3509 tcg_gen_brcondi_tl(TCG_COND_EQ, count, 0, zero_label); 3510 } 3511 3512 /* Save incoming carry into high, it will be shifted later. */ 3513 if (!have_1bit_cin || TCG_TARGET_deposit_tl_valid(1, TARGET_LONG_BITS - 1)) { 3514 TCGv cin = have_1bit_cin ? decode->cc_dst : decode->cc_src; 3515 tcg_gen_deposit_tl(high, cin, s->T0, 1, TARGET_LONG_BITS - 1); 3516 } else { 3517 /* Same as above but without deposit; cin in cc_dst. */ 3518 tcg_gen_add_tl(high, s->T0, decode->cc_dst); 3519 tcg_gen_add_tl(high, high, s->T0); 3520 } 3521 3522 /* Compute low part and outgoing carry, incoming s->T0 is zero extended */ 3523 tcg_gen_subi_tl(count, count, 1); 3524 tcg_gen_shr_tl(low, s->T0, count); 3525 tcg_gen_andi_tl(decode->cc_dst, low, 1); 3526 tcg_gen_shri_tl(low, low, 1); 3527 3528 /* Move high part to the right position */ 3529 tcg_gen_xori_tl(high_count, count, (8 << ot) - 1); /* LENGTH - 1 - (count - 1) */ 3530 tcg_gen_shl_tl(high, high, high_count); 3531 3532 /* Compute result and outgoing overflow */ 3533 tcg_gen_mov_tl(decode->cc_src2, s->T0); 3534 tcg_gen_or_tl(s->T0, low, high); 3535 gen_rot_overflow(decode, s->T0, decode->cc_src2, false, NULL); 3536 3537 if (zero_label) { 3538 gen_set_label(zero_label); 3539 } 3540} 3541 3542#ifdef CONFIG_USER_ONLY 3543static void gen_unreachable(DisasContext *s, X86DecodedInsn *decode) 3544{ 3545 g_assert_not_reached(); 3546} 3547#endif 3548 3549#ifndef CONFIG_USER_ONLY 3550static void gen_RDMSR(DisasContext *s, X86DecodedInsn *decode) 3551{ 3552 gen_update_cc_op(s); 3553 gen_update_eip_cur(s); 3554 gen_helper_rdmsr(tcg_env); 3555} 3556#else 3557#define gen_RDMSR gen_unreachable 3558#endif 3559 3560static void gen_RDPMC(DisasContext *s, X86DecodedInsn *decode) 3561{ 3562 gen_update_cc_op(s); 3563 gen_update_eip_cur(s); 3564 translator_io_start(&s->base); 3565 gen_helper_rdpmc(tcg_env); 3566 s->base.is_jmp = DISAS_NORETURN; 3567} 3568 3569static void gen_RDTSC(DisasContext *s, X86DecodedInsn *decode) 3570{ 3571 gen_update_cc_op(s); 3572 gen_update_eip_cur(s); 3573 translator_io_start(&s->base); 3574 gen_helper_rdtsc(tcg_env); 3575} 3576 3577static void gen_RDxxBASE(DisasContext *s, X86DecodedInsn *decode) 3578{ 3579 TCGv base = cpu_seg_base[s->modrm & 8 ? R_GS : R_FS]; 3580 3581 /* Preserve hflags bits by testing CR4 at runtime. */ 3582 gen_helper_cr4_testbit(tcg_env, tcg_constant_i32(CR4_FSGSBASE_MASK)); 3583 tcg_gen_mov_tl(s->T0, base); 3584} 3585 3586static void gen_RET(DisasContext *s, X86DecodedInsn *decode) 3587{ 3588 int16_t adjust = decode->e.op1 == X86_TYPE_I ? decode->immediate : 0; 3589 3590 MemOp ot = gen_pop_T0(s); 3591 gen_stack_update(s, adjust + (1 << ot)); 3592 gen_op_jmp_v(s, s->T0); 3593 gen_bnd_jmp(s); 3594 s->base.is_jmp = DISAS_JUMP; 3595} 3596 3597static void gen_RETF(DisasContext *s, X86DecodedInsn *decode) 3598{ 3599 int16_t adjust = decode->e.op1 == X86_TYPE_I ? decode->immediate : 0; 3600 3601 if (!PE(s) || VM86(s)) { 3602 gen_lea_ss_ofs(s, s->A0, cpu_regs[R_ESP], 0); 3603 /* pop offset */ 3604 gen_op_ld_v(s, s->dflag, s->T0, s->A0); 3605 /* NOTE: keeping EIP updated is not a problem in case of 3606 exception */ 3607 gen_op_jmp_v(s, s->T0); 3608 /* pop selector */ 3609 gen_add_A0_im(s, 1 << s->dflag); 3610 gen_op_ld_v(s, s->dflag, s->T0, s->A0); 3611 gen_op_movl_seg_real(s, R_CS, s->T0); 3612 /* add stack offset */ 3613 gen_stack_update(s, adjust + (2 << s->dflag)); 3614 } else { 3615 gen_update_cc_op(s); 3616 gen_update_eip_cur(s); 3617 gen_helper_lret_protected(tcg_env, tcg_constant_i32(s->dflag - 1), 3618 tcg_constant_i32(adjust)); 3619 } 3620 s->base.is_jmp = DISAS_EOB_ONLY; 3621} 3622 3623/* 3624 * Return non-NULL if a 32-bit rotate works, after possibly replicating the input. 3625 * The input has already been zero-extended upon operand decode. 3626 */ 3627static TCGv_i32 gen_rot_replicate(MemOp ot, TCGv in) 3628{ 3629 TCGv_i32 temp; 3630 switch (ot) { 3631 case MO_8: 3632 temp = tcg_temp_new_i32(); 3633 tcg_gen_trunc_tl_i32(temp, in); 3634 tcg_gen_muli_i32(temp, temp, 0x01010101); 3635 return temp; 3636 3637 case MO_16: 3638 temp = tcg_temp_new_i32(); 3639 tcg_gen_trunc_tl_i32(temp, in); 3640 tcg_gen_deposit_i32(temp, temp, temp, 16, 16); 3641 return temp; 3642 3643#ifdef TARGET_X86_64 3644 case MO_32: 3645 temp = tcg_temp_new_i32(); 3646 tcg_gen_trunc_tl_i32(temp, in); 3647 return temp; 3648#endif 3649 3650 default: 3651 return NULL; 3652 } 3653} 3654 3655static void gen_rot_carry(X86DecodedInsn *decode, TCGv result, 3656 bool can_be_zero, TCGv count, int bit) 3657{ 3658 if (!can_be_zero) { 3659 tcg_gen_extract_tl(decode->cc_dst, result, bit, 1); 3660 } else { 3661 TCGv temp = tcg_temp_new(); 3662 tcg_gen_extract_tl(temp, result, bit, 1); 3663 tcg_gen_movcond_tl(TCG_COND_EQ, decode->cc_dst, count, tcg_constant_tl(0), 3664 decode->cc_dst, temp); 3665 } 3666} 3667 3668static void gen_ROL(DisasContext *s, X86DecodedInsn *decode) 3669{ 3670 bool can_be_zero; 3671 TCGv count; 3672 MemOp ot = gen_shift_count(s, decode, &can_be_zero, &count, decode->op[2].unit); 3673 TCGv_i32 temp32, count32; 3674 TCGv old = tcg_temp_new(); 3675 3676 if (!count) { 3677 return; 3678 } 3679 3680 gen_eflags_adcox(s, decode, false, can_be_zero); 3681 tcg_gen_mov_tl(old, s->T0); 3682 temp32 = gen_rot_replicate(ot, s->T0); 3683 if (temp32) { 3684 count32 = tcg_temp_new_i32(); 3685 tcg_gen_trunc_tl_i32(count32, count); 3686 tcg_gen_rotl_i32(temp32, temp32, count32); 3687 /* Zero extend to facilitate later optimization. */ 3688 tcg_gen_extu_i32_tl(s->T0, temp32); 3689 } else { 3690 tcg_gen_rotl_tl(s->T0, s->T0, count); 3691 } 3692 gen_rot_carry(decode, s->T0, can_be_zero, count, 0); 3693 gen_rot_overflow(decode, s->T0, old, can_be_zero, count); 3694} 3695 3696static void gen_ROR(DisasContext *s, X86DecodedInsn *decode) 3697{ 3698 bool can_be_zero; 3699 TCGv count; 3700 MemOp ot = gen_shift_count(s, decode, &can_be_zero, &count, decode->op[2].unit); 3701 TCGv_i32 temp32, count32; 3702 TCGv old = tcg_temp_new(); 3703 3704 if (!count) { 3705 return; 3706 } 3707 3708 gen_eflags_adcox(s, decode, false, can_be_zero); 3709 tcg_gen_mov_tl(old, s->T0); 3710 temp32 = gen_rot_replicate(ot, s->T0); 3711 if (temp32) { 3712 count32 = tcg_temp_new_i32(); 3713 tcg_gen_trunc_tl_i32(count32, count); 3714 tcg_gen_rotr_i32(temp32, temp32, count32); 3715 /* Zero extend to facilitate later optimization. */ 3716 tcg_gen_extu_i32_tl(s->T0, temp32); 3717 gen_rot_carry(decode, s->T0, can_be_zero, count, 31); 3718 } else { 3719 tcg_gen_rotr_tl(s->T0, s->T0, count); 3720 gen_rot_carry(decode, s->T0, can_be_zero, count, TARGET_LONG_BITS - 1); 3721 } 3722 gen_rot_overflow(decode, s->T0, old, can_be_zero, count); 3723} 3724 3725static void gen_RORX(DisasContext *s, X86DecodedInsn *decode) 3726{ 3727 MemOp ot = decode->op[0].ot; 3728 int mask = ot == MO_64 ? 63 : 31; 3729 int b = decode->immediate & mask; 3730 3731 switch (ot) { 3732 case MO_32: 3733#ifdef TARGET_X86_64 3734 tcg_gen_trunc_tl_i32(s->tmp2_i32, s->T0); 3735 tcg_gen_rotri_i32(s->tmp2_i32, s->tmp2_i32, b); 3736 tcg_gen_extu_i32_tl(s->T0, s->tmp2_i32); 3737 break; 3738 3739 case MO_64: 3740#endif 3741 tcg_gen_rotri_tl(s->T0, s->T0, b); 3742 break; 3743 3744 default: 3745 g_assert_not_reached(); 3746 } 3747} 3748 3749#ifndef CONFIG_USER_ONLY 3750static void gen_RSM(DisasContext *s, X86DecodedInsn *decode) 3751{ 3752 gen_helper_rsm(tcg_env); 3753 assume_cc_op(s, CC_OP_EFLAGS); 3754 s->base.is_jmp = DISAS_EOB_ONLY; 3755} 3756#else 3757#define gen_RSM gen_UD 3758#endif 3759 3760static void gen_SAHF(DisasContext *s, X86DecodedInsn *decode) 3761{ 3762 if (CODE64(s) && !(s->cpuid_ext3_features & CPUID_EXT3_LAHF_LM)) { 3763 return gen_illegal_opcode(s); 3764 } 3765 tcg_gen_shri_tl(s->T0, cpu_regs[R_EAX], 8); 3766 gen_compute_eflags(s); 3767 tcg_gen_andi_tl(cpu_cc_src, cpu_cc_src, CC_O); 3768 tcg_gen_andi_tl(s->T0, s->T0, CC_S | CC_Z | CC_A | CC_P | CC_C); 3769 tcg_gen_or_tl(cpu_cc_src, cpu_cc_src, s->T0); 3770} 3771 3772static void gen_SALC(DisasContext *s, X86DecodedInsn *decode) 3773{ 3774 gen_compute_eflags_c(s, s->T0); 3775 tcg_gen_neg_tl(s->T0, s->T0); 3776} 3777 3778static void gen_shift_dynamic_flags(DisasContext *s, X86DecodedInsn *decode, TCGv count, CCOp cc_op) 3779{ 3780 TCGv_i32 count32 = tcg_temp_new_i32(); 3781 TCGv_i32 old_cc_op; 3782 3783 decode->cc_op = CC_OP_DYNAMIC; 3784 decode->cc_op_dynamic = tcg_temp_new_i32(); 3785 3786 assert(decode->cc_dst == s->T0); 3787 if (cc_op_live[s->cc_op] & USES_CC_DST) { 3788 decode->cc_dst = tcg_temp_new(); 3789 tcg_gen_movcond_tl(TCG_COND_EQ, decode->cc_dst, count, tcg_constant_tl(0), 3790 cpu_cc_dst, s->T0); 3791 } 3792 3793 if (cc_op_live[s->cc_op] & USES_CC_SRC) { 3794 tcg_gen_movcond_tl(TCG_COND_EQ, decode->cc_src, count, tcg_constant_tl(0), 3795 cpu_cc_src, decode->cc_src); 3796 } 3797 3798 tcg_gen_trunc_tl_i32(count32, count); 3799 if (s->cc_op == CC_OP_DYNAMIC) { 3800 old_cc_op = cpu_cc_op; 3801 } else { 3802 old_cc_op = tcg_constant_i32(s->cc_op); 3803 } 3804 tcg_gen_movcond_i32(TCG_COND_EQ, decode->cc_op_dynamic, count32, tcg_constant_i32(0), 3805 old_cc_op, tcg_constant_i32(cc_op)); 3806} 3807 3808static void gen_SAR(DisasContext *s, X86DecodedInsn *decode) 3809{ 3810 bool can_be_zero; 3811 TCGv count; 3812 MemOp ot = gen_shift_count(s, decode, &can_be_zero, &count, decode->op[2].unit); 3813 3814 if (!count) { 3815 return; 3816 } 3817 3818 decode->cc_dst = s->T0; 3819 decode->cc_src = tcg_temp_new(); 3820 tcg_gen_subi_tl(decode->cc_src, count, 1); 3821 tcg_gen_sar_tl(decode->cc_src, s->T0, decode->cc_src); 3822 tcg_gen_sar_tl(s->T0, s->T0, count); 3823 if (can_be_zero) { 3824 gen_shift_dynamic_flags(s, decode, count, CC_OP_SARB + ot); 3825 } else { 3826 decode->cc_op = CC_OP_SARB + ot; 3827 } 3828} 3829 3830static void gen_SARX(DisasContext *s, X86DecodedInsn *decode) 3831{ 3832 MemOp ot = decode->op[0].ot; 3833 int mask; 3834 3835 mask = ot == MO_64 ? 63 : 31; 3836 tcg_gen_andi_tl(s->T1, s->T1, mask); 3837 tcg_gen_sar_tl(s->T0, s->T0, s->T1); 3838} 3839 3840static void gen_SBB(DisasContext *s, X86DecodedInsn *decode) 3841{ 3842 MemOp ot = decode->op[0].ot; 3843 TCGv c_in = tcg_temp_new(); 3844 3845 gen_compute_eflags_c(s, c_in); 3846 if (s->prefix & PREFIX_LOCK) { 3847 tcg_gen_add_tl(s->T0, s->T1, c_in); 3848 tcg_gen_neg_tl(s->T0, s->T0); 3849 tcg_gen_atomic_add_fetch_tl(s->T0, s->A0, s->T0, 3850 s->mem_index, ot | MO_LE); 3851 } else { 3852 /* 3853 * TODO: SBB reg, reg could use gen_prepare_eflags_c followed by 3854 * negsetcond, and CC_OP_SUBB as the cc_op. 3855 */ 3856 tcg_gen_sub_tl(s->T0, s->T0, s->T1); 3857 tcg_gen_sub_tl(s->T0, s->T0, c_in); 3858 } 3859 prepare_update3_cc(decode, s, CC_OP_SBBB + ot, c_in); 3860} 3861 3862static void gen_SCAS(DisasContext *s, X86DecodedInsn *decode) 3863{ 3864 MemOp ot = decode->op[2].ot; 3865 if (s->prefix & (PREFIX_REPZ | PREFIX_REPNZ)) { 3866 gen_repz_nz(s, ot, gen_scas); 3867 } else { 3868 gen_scas(s, ot); 3869 } 3870} 3871 3872static void gen_SETcc(DisasContext *s, X86DecodedInsn *decode) 3873{ 3874 gen_setcc1(s, decode->b & 0xf, s->T0); 3875} 3876 3877static void gen_SFENCE(DisasContext *s, X86DecodedInsn *decode) 3878{ 3879 tcg_gen_mb(TCG_MO_ST_ST | TCG_BAR_SC); 3880} 3881 3882static void gen_SHA1NEXTE(DisasContext *s, X86DecodedInsn *decode) 3883{ 3884 gen_helper_sha1nexte(OP_PTR0, OP_PTR1, OP_PTR2); 3885} 3886 3887static void gen_SHA1MSG1(DisasContext *s, X86DecodedInsn *decode) 3888{ 3889 gen_helper_sha1msg1(OP_PTR0, OP_PTR1, OP_PTR2); 3890} 3891 3892static void gen_SHA1MSG2(DisasContext *s, X86DecodedInsn *decode) 3893{ 3894 gen_helper_sha1msg2(OP_PTR0, OP_PTR1, OP_PTR2); 3895} 3896 3897static void gen_SHA1RNDS4(DisasContext *s, X86DecodedInsn *decode) 3898{ 3899 switch(decode->immediate & 3) { 3900 case 0: 3901 gen_helper_sha1rnds4_f0(OP_PTR0, OP_PTR0, OP_PTR1); 3902 break; 3903 case 1: 3904 gen_helper_sha1rnds4_f1(OP_PTR0, OP_PTR0, OP_PTR1); 3905 break; 3906 case 2: 3907 gen_helper_sha1rnds4_f2(OP_PTR0, OP_PTR0, OP_PTR1); 3908 break; 3909 case 3: 3910 gen_helper_sha1rnds4_f3(OP_PTR0, OP_PTR0, OP_PTR1); 3911 break; 3912 } 3913} 3914 3915static void gen_SHA256MSG1(DisasContext *s, X86DecodedInsn *decode) 3916{ 3917 gen_helper_sha256msg1(OP_PTR0, OP_PTR1, OP_PTR2); 3918} 3919 3920static void gen_SHA256MSG2(DisasContext *s, X86DecodedInsn *decode) 3921{ 3922 gen_helper_sha256msg2(OP_PTR0, OP_PTR1, OP_PTR2); 3923} 3924 3925static void gen_SHA256RNDS2(DisasContext *s, X86DecodedInsn *decode) 3926{ 3927 TCGv_i32 wk0 = tcg_temp_new_i32(); 3928 TCGv_i32 wk1 = tcg_temp_new_i32(); 3929 3930 tcg_gen_ld_i32(wk0, tcg_env, ZMM_OFFSET(0) + offsetof(ZMMReg, ZMM_L(0))); 3931 tcg_gen_ld_i32(wk1, tcg_env, ZMM_OFFSET(0) + offsetof(ZMMReg, ZMM_L(1))); 3932 3933 gen_helper_sha256rnds2(OP_PTR0, OP_PTR1, OP_PTR2, wk0, wk1); 3934} 3935 3936static void gen_SHL(DisasContext *s, X86DecodedInsn *decode) 3937{ 3938 bool can_be_zero; 3939 TCGv count; 3940 MemOp ot = gen_shift_count(s, decode, &can_be_zero, &count, decode->op[2].unit); 3941 3942 if (!count) { 3943 return; 3944 } 3945 3946 decode->cc_dst = s->T0; 3947 decode->cc_src = tcg_temp_new(); 3948 tcg_gen_subi_tl(decode->cc_src, count, 1); 3949 tcg_gen_shl_tl(decode->cc_src, s->T0, decode->cc_src); 3950 tcg_gen_shl_tl(s->T0, s->T0, count); 3951 if (can_be_zero) { 3952 gen_shift_dynamic_flags(s, decode, count, CC_OP_SHLB + ot); 3953 } else { 3954 decode->cc_op = CC_OP_SHLB + ot; 3955 } 3956} 3957 3958static void gen_SHLD(DisasContext *s, X86DecodedInsn *decode) 3959{ 3960 bool can_be_zero; 3961 TCGv count; 3962 int unit = decode->e.op3 == X86_TYPE_I ? X86_OP_IMM : X86_OP_INT; 3963 MemOp ot = gen_shift_count(s, decode, &can_be_zero, &count, unit); 3964 3965 if (!count) { 3966 return; 3967 } 3968 3969 decode->cc_dst = s->T0; 3970 decode->cc_src = s->tmp0; 3971 gen_shiftd_rm_T1(s, ot, false, count); 3972 if (can_be_zero) { 3973 gen_shift_dynamic_flags(s, decode, count, CC_OP_SHLB + ot); 3974 } else { 3975 decode->cc_op = CC_OP_SHLB + ot; 3976 } 3977} 3978 3979static void gen_SHLX(DisasContext *s, X86DecodedInsn *decode) 3980{ 3981 MemOp ot = decode->op[0].ot; 3982 int mask; 3983 3984 mask = ot == MO_64 ? 63 : 31; 3985 tcg_gen_andi_tl(s->T1, s->T1, mask); 3986 tcg_gen_shl_tl(s->T0, s->T0, s->T1); 3987} 3988 3989static void gen_SHR(DisasContext *s, X86DecodedInsn *decode) 3990{ 3991 bool can_be_zero; 3992 TCGv count; 3993 MemOp ot = gen_shift_count(s, decode, &can_be_zero, &count, decode->op[2].unit); 3994 3995 if (!count) { 3996 return; 3997 } 3998 3999 decode->cc_dst = s->T0; 4000 decode->cc_src = tcg_temp_new(); 4001 tcg_gen_subi_tl(decode->cc_src, count, 1); 4002 tcg_gen_shr_tl(decode->cc_src, s->T0, decode->cc_src); 4003 tcg_gen_shr_tl(s->T0, s->T0, count); 4004 if (can_be_zero) { 4005 gen_shift_dynamic_flags(s, decode, count, CC_OP_SARB + ot); 4006 } else { 4007 decode->cc_op = CC_OP_SARB + ot; 4008 } 4009} 4010 4011static void gen_SHRD(DisasContext *s, X86DecodedInsn *decode) 4012{ 4013 bool can_be_zero; 4014 TCGv count; 4015 int unit = decode->e.op3 == X86_TYPE_I ? X86_OP_IMM : X86_OP_INT; 4016 MemOp ot = gen_shift_count(s, decode, &can_be_zero, &count, unit); 4017 4018 if (!count) { 4019 return; 4020 } 4021 4022 decode->cc_dst = s->T0; 4023 decode->cc_src = s->tmp0; 4024 gen_shiftd_rm_T1(s, ot, true, count); 4025 if (can_be_zero) { 4026 gen_shift_dynamic_flags(s, decode, count, CC_OP_SARB + ot); 4027 } else { 4028 decode->cc_op = CC_OP_SARB + ot; 4029 } 4030} 4031 4032static void gen_SHRX(DisasContext *s, X86DecodedInsn *decode) 4033{ 4034 MemOp ot = decode->op[0].ot; 4035 int mask; 4036 4037 mask = ot == MO_64 ? 63 : 31; 4038 tcg_gen_andi_tl(s->T1, s->T1, mask); 4039 tcg_gen_shr_tl(s->T0, s->T0, s->T1); 4040} 4041 4042static void gen_STC(DisasContext *s, X86DecodedInsn *decode) 4043{ 4044 gen_compute_eflags(s); 4045 tcg_gen_ori_tl(cpu_cc_src, cpu_cc_src, CC_C); 4046} 4047 4048static void gen_STD(DisasContext *s, X86DecodedInsn *decode) 4049{ 4050 tcg_gen_st_i32(tcg_constant_i32(-1), tcg_env, offsetof(CPUX86State, df)); 4051} 4052 4053static void gen_STI(DisasContext *s, X86DecodedInsn *decode) 4054{ 4055 gen_set_eflags(s, IF_MASK); 4056 s->base.is_jmp = DISAS_EOB_INHIBIT_IRQ; 4057} 4058 4059static void gen_VAESKEYGEN(DisasContext *s, X86DecodedInsn *decode) 4060{ 4061 TCGv_i32 imm = tcg_constant8u_i32(decode->immediate); 4062 assert(!s->vex_l); 4063 gen_helper_aeskeygenassist_xmm(tcg_env, OP_PTR0, OP_PTR1, imm); 4064} 4065 4066static void gen_STMXCSR(DisasContext *s, X86DecodedInsn *decode) 4067{ 4068 gen_helper_update_mxcsr(tcg_env); 4069 tcg_gen_ld32u_tl(s->T0, tcg_env, offsetof(CPUX86State, mxcsr)); 4070} 4071 4072static void gen_STOS(DisasContext *s, X86DecodedInsn *decode) 4073{ 4074 MemOp ot = decode->op[1].ot; 4075 if (s->prefix & (PREFIX_REPZ | PREFIX_REPNZ)) { 4076 gen_repz(s, ot, gen_stos); 4077 } else { 4078 gen_stos(s, ot); 4079 } 4080} 4081 4082static void gen_SUB(DisasContext *s, X86DecodedInsn *decode) 4083{ 4084 MemOp ot = decode->op[1].ot; 4085 4086 if (s->prefix & PREFIX_LOCK) { 4087 tcg_gen_neg_tl(s->T0, s->T1); 4088 tcg_gen_atomic_fetch_add_tl(s->cc_srcT, s->A0, s->T0, 4089 s->mem_index, ot | MO_LE); 4090 tcg_gen_sub_tl(s->T0, s->cc_srcT, s->T1); 4091 } else { 4092 tcg_gen_mov_tl(s->cc_srcT, s->T0); 4093 tcg_gen_sub_tl(s->T0, s->T0, s->T1); 4094 } 4095 prepare_update2_cc(decode, s, CC_OP_SUBB + ot); 4096} 4097 4098static void gen_SYSCALL(DisasContext *s, X86DecodedInsn *decode) 4099{ 4100 gen_update_cc_op(s); 4101 gen_update_eip_cur(s); 4102 gen_helper_syscall(tcg_env, cur_insn_len_i32(s)); 4103 if (LMA(s)) { 4104 assume_cc_op(s, CC_OP_EFLAGS); 4105 } 4106 4107 /* 4108 * TF handling for the syscall insn is different. The TF bit is checked 4109 * after the syscall insn completes. This allows #DB to not be 4110 * generated after one has entered CPL0 if TF is set in FMASK. 4111 */ 4112 s->base.is_jmp = DISAS_EOB_RECHECK_TF; 4113} 4114 4115static void gen_SYSENTER(DisasContext *s, X86DecodedInsn *decode) 4116{ 4117 gen_helper_sysenter(tcg_env); 4118 s->base.is_jmp = DISAS_EOB_ONLY; 4119} 4120 4121static void gen_SYSEXIT(DisasContext *s, X86DecodedInsn *decode) 4122{ 4123 gen_helper_sysexit(tcg_env, tcg_constant_i32(s->dflag - 1)); 4124 s->base.is_jmp = DISAS_EOB_ONLY; 4125} 4126 4127static void gen_SYSRET(DisasContext *s, X86DecodedInsn *decode) 4128{ 4129 gen_helper_sysret(tcg_env, tcg_constant_i32(s->dflag - 1)); 4130 if (LMA(s)) { 4131 assume_cc_op(s, CC_OP_EFLAGS); 4132 } 4133 4134 /* 4135 * TF handling for the sysret insn is different. The TF bit is checked 4136 * after the sysret insn completes. This allows #DB to be 4137 * generated "as if" the syscall insn in userspace has just 4138 * completed. 4139 */ 4140 s->base.is_jmp = DISAS_EOB_RECHECK_TF; 4141} 4142 4143static void gen_TZCNT(DisasContext *s, X86DecodedInsn *decode) 4144{ 4145 MemOp ot = decode->op[0].ot; 4146 4147 /* C bit (cc_src) is defined related to the input. */ 4148 decode->cc_src = tcg_temp_new(); 4149 decode->cc_dst = s->T0; 4150 decode->cc_op = CC_OP_BMILGB + ot; 4151 tcg_gen_mov_tl(decode->cc_src, s->T0); 4152 4153 /* A zero input returns the operand size. */ 4154 tcg_gen_ctzi_tl(s->T0, s->T0, 8 << ot); 4155} 4156 4157static void gen_UD(DisasContext *s, X86DecodedInsn *decode) 4158{ 4159 gen_illegal_opcode(s); 4160} 4161 4162static void gen_VAESIMC(DisasContext *s, X86DecodedInsn *decode) 4163{ 4164 assert(!s->vex_l); 4165 gen_helper_aesimc_xmm(tcg_env, OP_PTR0, OP_PTR2); 4166} 4167 4168/* 4169 * 00 = v*ps Vps, Hps, Wpd 4170 * 66 = v*pd Vpd, Hpd, Wps 4171 * f3 = v*ss Vss, Hss, Wps 4172 * f2 = v*sd Vsd, Hsd, Wps 4173 */ 4174#define SSE_CMP(x) { \ 4175 gen_helper_ ## x ## ps ## _xmm, gen_helper_ ## x ## pd ## _xmm, \ 4176 gen_helper_ ## x ## ss, gen_helper_ ## x ## sd, \ 4177 gen_helper_ ## x ## ps ## _ymm, gen_helper_ ## x ## pd ## _ymm} 4178static const SSEFunc_0_eppp gen_helper_cmp_funcs[32][6] = { 4179 SSE_CMP(cmpeq), 4180 SSE_CMP(cmplt), 4181 SSE_CMP(cmple), 4182 SSE_CMP(cmpunord), 4183 SSE_CMP(cmpneq), 4184 SSE_CMP(cmpnlt), 4185 SSE_CMP(cmpnle), 4186 SSE_CMP(cmpord), 4187 4188 SSE_CMP(cmpequ), 4189 SSE_CMP(cmpnge), 4190 SSE_CMP(cmpngt), 4191 SSE_CMP(cmpfalse), 4192 SSE_CMP(cmpnequ), 4193 SSE_CMP(cmpge), 4194 SSE_CMP(cmpgt), 4195 SSE_CMP(cmptrue), 4196 4197 SSE_CMP(cmpeqs), 4198 SSE_CMP(cmpltq), 4199 SSE_CMP(cmpleq), 4200 SSE_CMP(cmpunords), 4201 SSE_CMP(cmpneqq), 4202 SSE_CMP(cmpnltq), 4203 SSE_CMP(cmpnleq), 4204 SSE_CMP(cmpords), 4205 4206 SSE_CMP(cmpequs), 4207 SSE_CMP(cmpngeq), 4208 SSE_CMP(cmpngtq), 4209 SSE_CMP(cmpfalses), 4210 SSE_CMP(cmpnequs), 4211 SSE_CMP(cmpgeq), 4212 SSE_CMP(cmpgtq), 4213 SSE_CMP(cmptrues), 4214}; 4215#undef SSE_CMP 4216 4217static void gen_VCMP(DisasContext *s, X86DecodedInsn *decode) 4218{ 4219 int index = decode->immediate & (s->prefix & PREFIX_VEX ? 31 : 7); 4220 int b = 4221 s->prefix & PREFIX_REPZ ? 2 /* ss */ : 4222 s->prefix & PREFIX_REPNZ ? 3 /* sd */ : 4223 !!(s->prefix & PREFIX_DATA) /* pd */ + (s->vex_l << 2); 4224 4225 gen_helper_cmp_funcs[index][b](tcg_env, OP_PTR0, OP_PTR1, OP_PTR2); 4226} 4227 4228static void gen_VCOMI(DisasContext *s, X86DecodedInsn *decode) 4229{ 4230 SSEFunc_0_epp fn; 4231 fn = s->prefix & PREFIX_DATA ? gen_helper_comisd : gen_helper_comiss; 4232 fn(tcg_env, OP_PTR1, OP_PTR2); 4233 assume_cc_op(s, CC_OP_EFLAGS); 4234} 4235 4236static void gen_VCVTPD2PS(DisasContext *s, X86DecodedInsn *decode) 4237{ 4238 if (s->vex_l) { 4239 gen_helper_cvtpd2ps_ymm(tcg_env, OP_PTR0, OP_PTR2); 4240 } else { 4241 gen_helper_cvtpd2ps_xmm(tcg_env, OP_PTR0, OP_PTR2); 4242 } 4243} 4244 4245static void gen_VCVTPS2PD(DisasContext *s, X86DecodedInsn *decode) 4246{ 4247 if (s->vex_l) { 4248 gen_helper_cvtps2pd_ymm(tcg_env, OP_PTR0, OP_PTR2); 4249 } else { 4250 gen_helper_cvtps2pd_xmm(tcg_env, OP_PTR0, OP_PTR2); 4251 } 4252} 4253 4254static void gen_VCVTPS2PH(DisasContext *s, X86DecodedInsn *decode) 4255{ 4256 gen_unary_imm_fp_sse(s, decode, 4257 gen_helper_cvtps2ph_xmm, 4258 gen_helper_cvtps2ph_ymm); 4259 /* 4260 * VCVTPS2PH is the only instruction that performs an operation on a 4261 * register source and then *stores* into memory. 4262 */ 4263 if (decode->op[0].has_ea) { 4264 gen_store_sse(s, decode, decode->op[0].offset); 4265 } 4266} 4267 4268static void gen_VCVTSD2SS(DisasContext *s, X86DecodedInsn *decode) 4269{ 4270 gen_helper_cvtsd2ss(tcg_env, OP_PTR0, OP_PTR1, OP_PTR2); 4271} 4272 4273static void gen_VCVTSS2SD(DisasContext *s, X86DecodedInsn *decode) 4274{ 4275 gen_helper_cvtss2sd(tcg_env, OP_PTR0, OP_PTR1, OP_PTR2); 4276} 4277 4278static void gen_VCVTSI2Sx(DisasContext *s, X86DecodedInsn *decode) 4279{ 4280 int vec_len = vector_len(s, decode); 4281 TCGv_i32 in; 4282 4283 tcg_gen_gvec_mov(MO_64, decode->op[0].offset, decode->op[1].offset, vec_len, vec_len); 4284 4285#ifdef TARGET_X86_64 4286 MemOp ot = decode->op[2].ot; 4287 if (ot == MO_64) { 4288 if (s->prefix & PREFIX_REPNZ) { 4289 gen_helper_cvtsq2sd(tcg_env, OP_PTR0, s->T1); 4290 } else { 4291 gen_helper_cvtsq2ss(tcg_env, OP_PTR0, s->T1); 4292 } 4293 return; 4294 } 4295 in = s->tmp2_i32; 4296 tcg_gen_trunc_tl_i32(in, s->T1); 4297#else 4298 in = s->T1; 4299#endif 4300 4301 if (s->prefix & PREFIX_REPNZ) { 4302 gen_helper_cvtsi2sd(tcg_env, OP_PTR0, in); 4303 } else { 4304 gen_helper_cvtsi2ss(tcg_env, OP_PTR0, in); 4305 } 4306} 4307 4308static inline void gen_VCVTtSx2SI(DisasContext *s, X86DecodedInsn *decode, 4309 SSEFunc_i_ep ss2si, SSEFunc_l_ep ss2sq, 4310 SSEFunc_i_ep sd2si, SSEFunc_l_ep sd2sq) 4311{ 4312 TCGv_i32 out; 4313 4314#ifdef TARGET_X86_64 4315 MemOp ot = decode->op[0].ot; 4316 if (ot == MO_64) { 4317 if (s->prefix & PREFIX_REPNZ) { 4318 sd2sq(s->T0, tcg_env, OP_PTR2); 4319 } else { 4320 ss2sq(s->T0, tcg_env, OP_PTR2); 4321 } 4322 return; 4323 } 4324 4325 out = s->tmp2_i32; 4326#else 4327 out = s->T0; 4328#endif 4329 if (s->prefix & PREFIX_REPNZ) { 4330 sd2si(out, tcg_env, OP_PTR2); 4331 } else { 4332 ss2si(out, tcg_env, OP_PTR2); 4333 } 4334#ifdef TARGET_X86_64 4335 tcg_gen_extu_i32_tl(s->T0, out); 4336#endif 4337} 4338 4339#ifndef TARGET_X86_64 4340#define gen_helper_cvtss2sq NULL 4341#define gen_helper_cvtsd2sq NULL 4342#define gen_helper_cvttss2sq NULL 4343#define gen_helper_cvttsd2sq NULL 4344#endif 4345 4346static void gen_VCVTSx2SI(DisasContext *s, X86DecodedInsn *decode) 4347{ 4348 gen_VCVTtSx2SI(s, decode, 4349 gen_helper_cvtss2si, gen_helper_cvtss2sq, 4350 gen_helper_cvtsd2si, gen_helper_cvtsd2sq); 4351} 4352 4353static void gen_VCVTTSx2SI(DisasContext *s, X86DecodedInsn *decode) 4354{ 4355 gen_VCVTtSx2SI(s, decode, 4356 gen_helper_cvttss2si, gen_helper_cvttss2sq, 4357 gen_helper_cvttsd2si, gen_helper_cvttsd2sq); 4358} 4359 4360static void gen_VEXTRACTx128(DisasContext *s, X86DecodedInsn *decode) 4361{ 4362 int mask = decode->immediate & 1; 4363 int src_ofs = vector_elem_offset(&decode->op[1], MO_128, mask); 4364 if (decode->op[0].has_ea) { 4365 /* VEX-only instruction, no alignment requirements. */ 4366 gen_sto_env_A0(s, src_ofs, false); 4367 } else { 4368 tcg_gen_gvec_mov(MO_64, decode->op[0].offset, src_ofs, 16, 16); 4369 } 4370} 4371 4372static void gen_VEXTRACTPS(DisasContext *s, X86DecodedInsn *decode) 4373{ 4374 gen_pextr(s, decode, MO_32); 4375} 4376 4377static void gen_vinsertps(DisasContext *s, X86DecodedInsn *decode) 4378{ 4379 int val = decode->immediate; 4380 int dest_word = (val >> 4) & 3; 4381 int new_mask = (val & 15) | (1 << dest_word); 4382 int vec_len = 16; 4383 4384 assert(!s->vex_l); 4385 4386 if (new_mask == 15) { 4387 /* All zeroes except possibly for the inserted element */ 4388 tcg_gen_gvec_dup_imm(MO_64, decode->op[0].offset, vec_len, vec_len, 0); 4389 } else if (decode->op[1].offset != decode->op[0].offset) { 4390 gen_store_sse(s, decode, decode->op[1].offset); 4391 } 4392 4393 if (new_mask != (val & 15)) { 4394 tcg_gen_st_i32(s->tmp2_i32, tcg_env, 4395 vector_elem_offset(&decode->op[0], MO_32, dest_word)); 4396 } 4397 4398 if (new_mask != 15) { 4399 TCGv_i32 zero = tcg_constant_i32(0); /* float32_zero */ 4400 int i; 4401 for (i = 0; i < 4; i++) { 4402 if ((val >> i) & 1) { 4403 tcg_gen_st_i32(zero, tcg_env, 4404 vector_elem_offset(&decode->op[0], MO_32, i)); 4405 } 4406 } 4407 } 4408} 4409 4410static void gen_VINSERTPS_r(DisasContext *s, X86DecodedInsn *decode) 4411{ 4412 int val = decode->immediate; 4413 tcg_gen_ld_i32(s->tmp2_i32, tcg_env, 4414 vector_elem_offset(&decode->op[2], MO_32, (val >> 6) & 3)); 4415 gen_vinsertps(s, decode); 4416} 4417 4418static void gen_VINSERTPS_m(DisasContext *s, X86DecodedInsn *decode) 4419{ 4420 tcg_gen_qemu_ld_i32(s->tmp2_i32, s->A0, s->mem_index, MO_LEUL); 4421 gen_vinsertps(s, decode); 4422} 4423 4424static void gen_VINSERTx128(DisasContext *s, X86DecodedInsn *decode) 4425{ 4426 int mask = decode->immediate & 1; 4427 tcg_gen_gvec_mov(MO_64, 4428 decode->op[0].offset + offsetof(YMMReg, YMM_X(mask)), 4429 decode->op[2].offset + offsetof(YMMReg, YMM_X(0)), 16, 16); 4430 tcg_gen_gvec_mov(MO_64, 4431 decode->op[0].offset + offsetof(YMMReg, YMM_X(!mask)), 4432 decode->op[1].offset + offsetof(YMMReg, YMM_X(!mask)), 16, 16); 4433} 4434 4435static inline void gen_maskmov(DisasContext *s, X86DecodedInsn *decode, 4436 SSEFunc_0_eppt xmm, SSEFunc_0_eppt ymm) 4437{ 4438 if (!s->vex_l) { 4439 xmm(tcg_env, OP_PTR2, OP_PTR1, s->A0); 4440 } else { 4441 ymm(tcg_env, OP_PTR2, OP_PTR1, s->A0); 4442 } 4443} 4444 4445static void gen_VMASKMOVPD_st(DisasContext *s, X86DecodedInsn *decode) 4446{ 4447 gen_maskmov(s, decode, gen_helper_vpmaskmovq_st_xmm, gen_helper_vpmaskmovq_st_ymm); 4448} 4449 4450static void gen_VMASKMOVPS_st(DisasContext *s, X86DecodedInsn *decode) 4451{ 4452 gen_maskmov(s, decode, gen_helper_vpmaskmovd_st_xmm, gen_helper_vpmaskmovd_st_ymm); 4453} 4454 4455static void gen_VMOVHPx_ld(DisasContext *s, X86DecodedInsn *decode) 4456{ 4457 gen_ldq_env_A0(s, decode->op[0].offset + offsetof(XMMReg, XMM_Q(1))); 4458 if (decode->op[0].offset != decode->op[1].offset) { 4459 tcg_gen_ld_i64(s->tmp1_i64, tcg_env, decode->op[1].offset + offsetof(XMMReg, XMM_Q(0))); 4460 tcg_gen_st_i64(s->tmp1_i64, tcg_env, decode->op[0].offset + offsetof(XMMReg, XMM_Q(0))); 4461 } 4462} 4463 4464static void gen_VMOVHPx_st(DisasContext *s, X86DecodedInsn *decode) 4465{ 4466 gen_stq_env_A0(s, decode->op[2].offset + offsetof(XMMReg, XMM_Q(1))); 4467} 4468 4469static void gen_VMOVHPx(DisasContext *s, X86DecodedInsn *decode) 4470{ 4471 if (decode->op[0].offset != decode->op[2].offset) { 4472 tcg_gen_ld_i64(s->tmp1_i64, tcg_env, decode->op[2].offset + offsetof(XMMReg, XMM_Q(1))); 4473 tcg_gen_st_i64(s->tmp1_i64, tcg_env, decode->op[0].offset + offsetof(XMMReg, XMM_Q(1))); 4474 } 4475 if (decode->op[0].offset != decode->op[1].offset) { 4476 tcg_gen_ld_i64(s->tmp1_i64, tcg_env, decode->op[1].offset + offsetof(XMMReg, XMM_Q(0))); 4477 tcg_gen_st_i64(s->tmp1_i64, tcg_env, decode->op[0].offset + offsetof(XMMReg, XMM_Q(0))); 4478 } 4479} 4480 4481static void gen_VMOVHLPS(DisasContext *s, X86DecodedInsn *decode) 4482{ 4483 tcg_gen_ld_i64(s->tmp1_i64, tcg_env, decode->op[2].offset + offsetof(XMMReg, XMM_Q(1))); 4484 tcg_gen_st_i64(s->tmp1_i64, tcg_env, decode->op[0].offset + offsetof(XMMReg, XMM_Q(0))); 4485 if (decode->op[0].offset != decode->op[1].offset) { 4486 tcg_gen_ld_i64(s->tmp1_i64, tcg_env, decode->op[1].offset + offsetof(XMMReg, XMM_Q(1))); 4487 tcg_gen_st_i64(s->tmp1_i64, tcg_env, decode->op[0].offset + offsetof(XMMReg, XMM_Q(1))); 4488 } 4489} 4490 4491static void gen_VMOVLHPS(DisasContext *s, X86DecodedInsn *decode) 4492{ 4493 tcg_gen_ld_i64(s->tmp1_i64, tcg_env, decode->op[2].offset); 4494 tcg_gen_st_i64(s->tmp1_i64, tcg_env, decode->op[0].offset + offsetof(XMMReg, XMM_Q(1))); 4495 if (decode->op[0].offset != decode->op[1].offset) { 4496 tcg_gen_ld_i64(s->tmp1_i64, tcg_env, decode->op[1].offset + offsetof(XMMReg, XMM_Q(0))); 4497 tcg_gen_st_i64(s->tmp1_i64, tcg_env, decode->op[0].offset + offsetof(XMMReg, XMM_Q(0))); 4498 } 4499} 4500 4501/* 4502 * Note that MOVLPx supports 256-bit operation unlike MOVHLPx, MOVLHPx, MOXHPx. 4503 * Use a gvec move to move everything above the bottom 64 bits. 4504 */ 4505 4506static void gen_VMOVLPx(DisasContext *s, X86DecodedInsn *decode) 4507{ 4508 int vec_len = vector_len(s, decode); 4509 4510 tcg_gen_ld_i64(s->tmp1_i64, tcg_env, decode->op[2].offset + offsetof(XMMReg, XMM_Q(0))); 4511 tcg_gen_gvec_mov(MO_64, decode->op[0].offset, decode->op[1].offset, vec_len, vec_len); 4512 tcg_gen_st_i64(s->tmp1_i64, tcg_env, decode->op[0].offset + offsetof(XMMReg, XMM_Q(0))); 4513} 4514 4515static void gen_VMOVLPx_ld(DisasContext *s, X86DecodedInsn *decode) 4516{ 4517 int vec_len = vector_len(s, decode); 4518 4519 tcg_gen_qemu_ld_i64(s->tmp1_i64, s->A0, s->mem_index, MO_LEUQ); 4520 tcg_gen_gvec_mov(MO_64, decode->op[0].offset, decode->op[1].offset, vec_len, vec_len); 4521 tcg_gen_st_i64(s->tmp1_i64, OP_PTR0, offsetof(ZMMReg, ZMM_Q(0))); 4522} 4523 4524static void gen_VMOVLPx_st(DisasContext *s, X86DecodedInsn *decode) 4525{ 4526 tcg_gen_ld_i64(s->tmp1_i64, OP_PTR2, offsetof(ZMMReg, ZMM_Q(0))); 4527 tcg_gen_qemu_st_i64(s->tmp1_i64, s->A0, s->mem_index, MO_LEUQ); 4528} 4529 4530static void gen_VMOVSD_ld(DisasContext *s, X86DecodedInsn *decode) 4531{ 4532 TCGv_i64 zero = tcg_constant_i64(0); 4533 4534 tcg_gen_qemu_ld_i64(s->tmp1_i64, s->A0, s->mem_index, MO_LEUQ); 4535 tcg_gen_st_i64(zero, OP_PTR0, offsetof(ZMMReg, ZMM_Q(1))); 4536 tcg_gen_st_i64(s->tmp1_i64, OP_PTR0, offsetof(ZMMReg, ZMM_Q(0))); 4537} 4538 4539static void gen_VMOVSS(DisasContext *s, X86DecodedInsn *decode) 4540{ 4541 int vec_len = vector_len(s, decode); 4542 4543 tcg_gen_ld_i32(s->tmp2_i32, OP_PTR2, offsetof(ZMMReg, ZMM_L(0))); 4544 tcg_gen_gvec_mov(MO_64, decode->op[0].offset, decode->op[1].offset, vec_len, vec_len); 4545 tcg_gen_st_i32(s->tmp2_i32, OP_PTR0, offsetof(ZMMReg, ZMM_L(0))); 4546} 4547 4548static void gen_VMOVSS_ld(DisasContext *s, X86DecodedInsn *decode) 4549{ 4550 int vec_len = vector_len(s, decode); 4551 4552 tcg_gen_qemu_ld_i32(s->tmp2_i32, s->A0, s->mem_index, MO_LEUL); 4553 tcg_gen_gvec_dup_imm(MO_64, decode->op[0].offset, vec_len, vec_len, 0); 4554 tcg_gen_st_i32(s->tmp2_i32, OP_PTR0, offsetof(ZMMReg, ZMM_L(0))); 4555} 4556 4557static void gen_VMOVSS_st(DisasContext *s, X86DecodedInsn *decode) 4558{ 4559 tcg_gen_ld_i32(s->tmp2_i32, OP_PTR2, offsetof(ZMMReg, ZMM_L(0))); 4560 tcg_gen_qemu_st_i32(s->tmp2_i32, s->A0, s->mem_index, MO_LEUL); 4561} 4562 4563static void gen_VPMASKMOV_st(DisasContext *s, X86DecodedInsn *decode) 4564{ 4565 if (s->vex_w) { 4566 gen_VMASKMOVPD_st(s, decode); 4567 } else { 4568 gen_VMASKMOVPS_st(s, decode); 4569 } 4570} 4571 4572static void gen_VPERMD(DisasContext *s, X86DecodedInsn *decode) 4573{ 4574 assert(s->vex_l); 4575 gen_helper_vpermd_ymm(OP_PTR0, OP_PTR1, OP_PTR2); 4576} 4577 4578static void gen_VPERM2x128(DisasContext *s, X86DecodedInsn *decode) 4579{ 4580 TCGv_i32 imm = tcg_constant8u_i32(decode->immediate); 4581 assert(s->vex_l); 4582 gen_helper_vpermdq_ymm(OP_PTR0, OP_PTR1, OP_PTR2, imm); 4583} 4584 4585static void gen_VPHMINPOSUW(DisasContext *s, X86DecodedInsn *decode) 4586{ 4587 assert(!s->vex_l); 4588 gen_helper_phminposuw_xmm(tcg_env, OP_PTR0, OP_PTR2); 4589} 4590 4591static void gen_VROUNDSD(DisasContext *s, X86DecodedInsn *decode) 4592{ 4593 TCGv_i32 imm = tcg_constant8u_i32(decode->immediate); 4594 assert(!s->vex_l); 4595 gen_helper_roundsd_xmm(tcg_env, OP_PTR0, OP_PTR1, OP_PTR2, imm); 4596} 4597 4598static void gen_VROUNDSS(DisasContext *s, X86DecodedInsn *decode) 4599{ 4600 TCGv_i32 imm = tcg_constant8u_i32(decode->immediate); 4601 assert(!s->vex_l); 4602 gen_helper_roundss_xmm(tcg_env, OP_PTR0, OP_PTR1, OP_PTR2, imm); 4603} 4604 4605static void gen_VSHUF(DisasContext *s, X86DecodedInsn *decode) 4606{ 4607 TCGv_i32 imm = tcg_constant_i32(decode->immediate); 4608 SSEFunc_0_pppi ps, pd, fn; 4609 ps = s->vex_l ? gen_helper_shufps_ymm : gen_helper_shufps_xmm; 4610 pd = s->vex_l ? gen_helper_shufpd_ymm : gen_helper_shufpd_xmm; 4611 fn = s->prefix & PREFIX_DATA ? pd : ps; 4612 fn(OP_PTR0, OP_PTR1, OP_PTR2, imm); 4613} 4614 4615static void gen_VUCOMI(DisasContext *s, X86DecodedInsn *decode) 4616{ 4617 SSEFunc_0_epp fn; 4618 fn = s->prefix & PREFIX_DATA ? gen_helper_ucomisd : gen_helper_ucomiss; 4619 fn(tcg_env, OP_PTR1, OP_PTR2); 4620 assume_cc_op(s, CC_OP_EFLAGS); 4621} 4622 4623static void gen_VZEROALL(DisasContext *s, X86DecodedInsn *decode) 4624{ 4625 TCGv_ptr ptr = tcg_temp_new_ptr(); 4626 4627 tcg_gen_addi_ptr(ptr, tcg_env, offsetof(CPUX86State, xmm_regs)); 4628 gen_helper_memset(ptr, ptr, tcg_constant_i32(0), 4629 tcg_constant_ptr(CPU_NB_REGS * sizeof(ZMMReg))); 4630} 4631 4632static void gen_VZEROUPPER(DisasContext *s, X86DecodedInsn *decode) 4633{ 4634 int i; 4635 4636 for (i = 0; i < CPU_NB_REGS; i++) { 4637 int offset = offsetof(CPUX86State, xmm_regs[i].ZMM_X(1)); 4638 tcg_gen_gvec_dup_imm(MO_64, offset, 16, 16, 0); 4639 } 4640} 4641 4642static void gen_WAIT(DisasContext *s, X86DecodedInsn *decode) 4643{ 4644 if ((s->flags & (HF_MP_MASK | HF_TS_MASK)) == (HF_MP_MASK | HF_TS_MASK)) { 4645 gen_NM_exception(s); 4646 } else { 4647 /* needs to be treated as I/O because of ferr_irq */ 4648 translator_io_start(&s->base); 4649 gen_helper_fwait(tcg_env); 4650 } 4651} 4652 4653#ifndef CONFIG_USER_ONLY 4654static void gen_WRMSR(DisasContext *s, X86DecodedInsn *decode) 4655{ 4656 gen_update_cc_op(s); 4657 gen_update_eip_cur(s); 4658 gen_helper_wrmsr(tcg_env); 4659 s->base.is_jmp = DISAS_EOB_NEXT; 4660} 4661#else 4662#define gen_WRMSR gen_unreachable 4663#endif 4664 4665static void gen_WRxxBASE(DisasContext *s, X86DecodedInsn *decode) 4666{ 4667 TCGv base = cpu_seg_base[s->modrm & 8 ? R_GS : R_FS]; 4668 4669 /* Preserve hflags bits by testing CR4 at runtime. */ 4670 gen_helper_cr4_testbit(tcg_env, tcg_constant_i32(CR4_FSGSBASE_MASK)); 4671 tcg_gen_mov_tl(base, s->T0); 4672} 4673 4674static void gen_XADD(DisasContext *s, X86DecodedInsn *decode) 4675{ 4676 MemOp ot = decode->op[1].ot; 4677 4678 decode->cc_dst = tcg_temp_new(); 4679 decode->cc_src = s->T1; 4680 decode->cc_op = CC_OP_ADDB + ot; 4681 4682 if (s->prefix & PREFIX_LOCK) { 4683 tcg_gen_atomic_fetch_add_tl(s->T0, s->A0, s->T1, s->mem_index, ot | MO_LE); 4684 tcg_gen_add_tl(decode->cc_dst, s->T0, s->T1); 4685 } else { 4686 tcg_gen_add_tl(decode->cc_dst, s->T0, s->T1); 4687 /* 4688 * NOTE: writing memory first is important for MMU exceptions, 4689 * but "new result" wins for XADD AX, AX. 4690 */ 4691 gen_writeback(s, decode, 0, decode->cc_dst); 4692 } 4693 if (decode->op[0].has_ea || decode->op[2].n != decode->op[0].n) { 4694 gen_writeback(s, decode, 2, s->T0); 4695 } 4696} 4697 4698static void gen_XCHG(DisasContext *s, X86DecodedInsn *decode) 4699{ 4700 if (s->prefix & PREFIX_LOCK) { 4701 tcg_gen_atomic_xchg_tl(s->T0, s->A0, s->T1, 4702 s->mem_index, decode->op[0].ot | MO_LE); 4703 /* now store old value into register operand */ 4704 gen_op_mov_reg_v(s, decode->op[2].ot, decode->op[2].n, s->T0); 4705 } else { 4706 /* move destination value into source operand, source preserved in T1 */ 4707 gen_op_mov_reg_v(s, decode->op[2].ot, decode->op[2].n, s->T0); 4708 tcg_gen_mov_tl(s->T0, s->T1); 4709 } 4710} 4711 4712static void gen_XLAT(DisasContext *s, X86DecodedInsn *decode) 4713{ 4714 /* AL is already zero-extended into s->T0. */ 4715 tcg_gen_add_tl(s->A0, cpu_regs[R_EBX], s->T0); 4716 gen_lea_v_seg(s, s->A0, R_DS, s->override); 4717 gen_op_ld_v(s, MO_8, s->T0, s->A0); 4718} 4719 4720static void gen_XOR(DisasContext *s, X86DecodedInsn *decode) 4721{ 4722 /* special case XOR reg, reg */ 4723 if (decode->op[1].unit == X86_OP_INT && 4724 decode->op[2].unit == X86_OP_INT && 4725 decode->op[1].n == decode->op[2].n) { 4726 tcg_gen_movi_tl(s->T0, 0); 4727 decode->cc_op = CC_OP_CLR; 4728 } else { 4729 MemOp ot = decode->op[1].ot; 4730 4731 if (s->prefix & PREFIX_LOCK) { 4732 tcg_gen_atomic_xor_fetch_tl(s->T0, s->A0, s->T1, 4733 s->mem_index, ot | MO_LE); 4734 } else { 4735 tcg_gen_xor_tl(s->T0, s->T0, s->T1); 4736 } 4737 prepare_update1_cc(decode, s, CC_OP_LOGICB + ot); 4738 } 4739} 4740 4741static void gen_XRSTOR(DisasContext *s, X86DecodedInsn *decode) 4742{ 4743 TCGv_i64 features = tcg_temp_new_i64(); 4744 4745 tcg_gen_concat_tl_i64(features, cpu_regs[R_EAX], cpu_regs[R_EDX]); 4746 gen_helper_xrstor(tcg_env, s->A0, features); 4747 if (s->cpuid_7_0_ebx_features & CPUID_7_0_EBX_MPX) { 4748 /* 4749 * XRSTOR is how MPX is enabled, which changes how 4750 * we translate. Thus we need to end the TB. 4751 */ 4752 s->base.is_jmp = DISAS_EOB_NEXT; 4753 } 4754} 4755 4756static void gen_XSAVE(DisasContext *s, X86DecodedInsn *decode) 4757{ 4758 TCGv_i64 features = tcg_temp_new_i64(); 4759 4760 tcg_gen_concat_tl_i64(features, cpu_regs[R_EAX], cpu_regs[R_EDX]); 4761 gen_helper_xsave(tcg_env, s->A0, features); 4762} 4763 4764static void gen_XSAVEOPT(DisasContext *s, X86DecodedInsn *decode) 4765{ 4766 TCGv_i64 features = tcg_temp_new_i64(); 4767 4768 tcg_gen_concat_tl_i64(features, cpu_regs[R_EAX], cpu_regs[R_EDX]); 4769 gen_helper_xsave(tcg_env, s->A0, features); 4770} 4771