1/* 2 * New-style TCG opcode generator for i386 instructions 3 * 4 * Copyright (c) 2022 Red Hat, Inc. 5 * 6 * Author: Paolo Bonzini <pbonzini@redhat.com> 7 * 8 * This library is free software; you can redistribute it and/or 9 * modify it under the terms of the GNU Lesser General Public 10 * License as published by the Free Software Foundation; either 11 * version 2.1 of the License, or (at your option) any later version. 12 * 13 * This library is distributed in the hope that it will be useful, 14 * but WITHOUT ANY WARRANTY; without even the implied warranty of 15 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU 16 * Lesser General Public License for more details. 17 * 18 * You should have received a copy of the GNU Lesser General Public 19 * License along with this library; if not, see <http://www.gnu.org/licenses/>. 20 */ 21 22/* 23 * Sometimes, knowing what the backend has can produce better code. 24 * The exact opcode to check depends on 32- vs. 64-bit. 25 */ 26#ifdef TARGET_X86_64 27#define TCG_TARGET_HAS_extract2_tl TCG_TARGET_HAS_extract2_i64 28#define TCG_TARGET_deposit_tl_valid TCG_TARGET_deposit_i64_valid 29#define TCG_TARGET_extract_tl_valid TCG_TARGET_extract_i64_valid 30#else 31#define TCG_TARGET_HAS_extract2_tl TCG_TARGET_HAS_extract2_i32 32#define TCG_TARGET_deposit_tl_valid TCG_TARGET_deposit_i32_valid 33#define TCG_TARGET_extract_tl_valid TCG_TARGET_extract_i32_valid 34#endif 35 36 37#define ZMM_OFFSET(reg) offsetof(CPUX86State, xmm_regs[reg]) 38 39typedef void (*SSEFunc_i_ep)(TCGv_i32 val, TCGv_ptr env, TCGv_ptr reg); 40typedef void (*SSEFunc_l_ep)(TCGv_i64 val, TCGv_ptr env, TCGv_ptr reg); 41typedef void (*SSEFunc_0_epp)(TCGv_ptr env, TCGv_ptr reg_a, TCGv_ptr reg_b); 42typedef void (*SSEFunc_0_eppp)(TCGv_ptr env, TCGv_ptr reg_a, TCGv_ptr reg_b, 43 TCGv_ptr reg_c); 44typedef void (*SSEFunc_0_epppp)(TCGv_ptr env, TCGv_ptr reg_a, TCGv_ptr reg_b, 45 TCGv_ptr reg_c, TCGv_ptr reg_d); 46typedef void (*SSEFunc_0_eppi)(TCGv_ptr env, TCGv_ptr reg_a, TCGv_ptr reg_b, 47 TCGv_i32 val); 48typedef void (*SSEFunc_0_epppi)(TCGv_ptr env, TCGv_ptr reg_a, TCGv_ptr reg_b, 49 TCGv_ptr reg_c, TCGv_i32 val); 50typedef void (*SSEFunc_0_ppi)(TCGv_ptr reg_a, TCGv_ptr reg_b, TCGv_i32 val); 51typedef void (*SSEFunc_0_pppi)(TCGv_ptr reg_a, TCGv_ptr reg_b, TCGv_ptr reg_c, 52 TCGv_i32 val); 53typedef void (*SSEFunc_0_eppt)(TCGv_ptr env, TCGv_ptr reg_a, TCGv_ptr reg_b, 54 TCGv val); 55typedef void (*SSEFunc_0_epppti)(TCGv_ptr env, TCGv_ptr reg_a, TCGv_ptr reg_b, 56 TCGv_ptr reg_c, TCGv a0, TCGv_i32 scale); 57typedef void (*SSEFunc_0_eppppi)(TCGv_ptr env, TCGv_ptr reg_a, TCGv_ptr reg_b, 58 TCGv_ptr reg_c, TCGv_ptr reg_d, TCGv_i32 flags); 59typedef void (*SSEFunc_0_eppppii)(TCGv_ptr env, TCGv_ptr reg_a, TCGv_ptr reg_b, 60 TCGv_ptr reg_c, TCGv_ptr reg_d, TCGv_i32 even, 61 TCGv_i32 odd); 62 63static void gen_JMP_m(DisasContext *s, X86DecodedInsn *decode); 64static void gen_JMP(DisasContext *s, X86DecodedInsn *decode); 65 66static inline TCGv_i32 tcg_constant8u_i32(uint8_t val) 67{ 68 return tcg_constant_i32(val); 69} 70 71static void gen_NM_exception(DisasContext *s) 72{ 73 gen_exception(s, EXCP07_PREX); 74} 75 76static void gen_load_ea(DisasContext *s, AddressParts *mem, bool is_vsib) 77{ 78 TCGv ea = gen_lea_modrm_1(s, *mem, is_vsib); 79 gen_lea_v_seg(s, ea, mem->def_seg, s->override); 80} 81 82static inline int mmx_offset(MemOp ot) 83{ 84 switch (ot) { 85 case MO_8: 86 return offsetof(MMXReg, MMX_B(0)); 87 case MO_16: 88 return offsetof(MMXReg, MMX_W(0)); 89 case MO_32: 90 return offsetof(MMXReg, MMX_L(0)); 91 case MO_64: 92 return offsetof(MMXReg, MMX_Q(0)); 93 default: 94 g_assert_not_reached(); 95 } 96} 97 98static inline int xmm_offset(MemOp ot) 99{ 100 switch (ot) { 101 case MO_8: 102 return offsetof(ZMMReg, ZMM_B(0)); 103 case MO_16: 104 return offsetof(ZMMReg, ZMM_W(0)); 105 case MO_32: 106 return offsetof(ZMMReg, ZMM_L(0)); 107 case MO_64: 108 return offsetof(ZMMReg, ZMM_Q(0)); 109 case MO_128: 110 return offsetof(ZMMReg, ZMM_X(0)); 111 case MO_256: 112 return offsetof(ZMMReg, ZMM_Y(0)); 113 default: 114 g_assert_not_reached(); 115 } 116} 117 118static int vector_reg_offset(X86DecodedOp *op) 119{ 120 assert(op->unit == X86_OP_MMX || op->unit == X86_OP_SSE); 121 122 if (op->unit == X86_OP_MMX) { 123 return op->offset - mmx_offset(op->ot); 124 } else { 125 return op->offset - xmm_offset(op->ot); 126 } 127} 128 129static int vector_elem_offset(X86DecodedOp *op, MemOp ot, int n) 130{ 131 int base_ofs = vector_reg_offset(op); 132 switch(ot) { 133 case MO_8: 134 if (op->unit == X86_OP_MMX) { 135 return base_ofs + offsetof(MMXReg, MMX_B(n)); 136 } else { 137 return base_ofs + offsetof(ZMMReg, ZMM_B(n)); 138 } 139 case MO_16: 140 if (op->unit == X86_OP_MMX) { 141 return base_ofs + offsetof(MMXReg, MMX_W(n)); 142 } else { 143 return base_ofs + offsetof(ZMMReg, ZMM_W(n)); 144 } 145 case MO_32: 146 if (op->unit == X86_OP_MMX) { 147 return base_ofs + offsetof(MMXReg, MMX_L(n)); 148 } else { 149 return base_ofs + offsetof(ZMMReg, ZMM_L(n)); 150 } 151 case MO_64: 152 if (op->unit == X86_OP_MMX) { 153 return base_ofs; 154 } else { 155 return base_ofs + offsetof(ZMMReg, ZMM_Q(n)); 156 } 157 case MO_128: 158 assert(op->unit == X86_OP_SSE); 159 return base_ofs + offsetof(ZMMReg, ZMM_X(n)); 160 case MO_256: 161 assert(op->unit == X86_OP_SSE); 162 return base_ofs + offsetof(ZMMReg, ZMM_Y(n)); 163 default: 164 g_assert_not_reached(); 165 } 166} 167 168static void compute_mmx_offset(X86DecodedOp *op) 169{ 170 if (!op->has_ea) { 171 op->offset = offsetof(CPUX86State, fpregs[op->n].mmx) + mmx_offset(op->ot); 172 } else { 173 op->offset = offsetof(CPUX86State, mmx_t0) + mmx_offset(op->ot); 174 } 175} 176 177static void compute_xmm_offset(X86DecodedOp *op) 178{ 179 if (!op->has_ea) { 180 op->offset = ZMM_OFFSET(op->n) + xmm_offset(op->ot); 181 } else { 182 op->offset = offsetof(CPUX86State, xmm_t0) + xmm_offset(op->ot); 183 } 184} 185 186static void gen_load_sse(DisasContext *s, TCGv temp, MemOp ot, int dest_ofs, bool aligned) 187{ 188 switch(ot) { 189 case MO_8: 190 gen_op_ld_v(s, MO_8, temp, s->A0); 191 tcg_gen_st8_tl(temp, tcg_env, dest_ofs); 192 break; 193 case MO_16: 194 gen_op_ld_v(s, MO_16, temp, s->A0); 195 tcg_gen_st16_tl(temp, tcg_env, dest_ofs); 196 break; 197 case MO_32: 198 gen_op_ld_v(s, MO_32, temp, s->A0); 199 tcg_gen_st32_tl(temp, tcg_env, dest_ofs); 200 break; 201 case MO_64: 202 gen_ldq_env_A0(s, dest_ofs); 203 break; 204 case MO_128: 205 gen_ldo_env_A0(s, dest_ofs, aligned); 206 break; 207 case MO_256: 208 gen_ldy_env_A0(s, dest_ofs, aligned); 209 break; 210 default: 211 g_assert_not_reached(); 212 } 213} 214 215static bool sse_needs_alignment(DisasContext *s, X86DecodedInsn *decode, MemOp ot) 216{ 217 switch (decode->e.vex_class) { 218 case 2: 219 case 4: 220 if ((s->prefix & PREFIX_VEX) || 221 decode->e.vex_special == X86_VEX_SSEUnaligned) { 222 /* MOST legacy SSE instructions require aligned memory operands, but not all. */ 223 return false; 224 } 225 /* fall through */ 226 case 1: 227 return ot >= MO_128; 228 229 default: 230 return false; 231 } 232} 233 234static void gen_load(DisasContext *s, X86DecodedInsn *decode, int opn, TCGv v) 235{ 236 X86DecodedOp *op = &decode->op[opn]; 237 238 switch (op->unit) { 239 case X86_OP_SKIP: 240 return; 241 case X86_OP_SEG: 242 tcg_gen_ld32u_tl(v, tcg_env, 243 offsetof(CPUX86State,segs[op->n].selector)); 244 break; 245#ifndef CONFIG_USER_ONLY 246 case X86_OP_CR: 247 if (op->n == 8) { 248 translator_io_start(&s->base); 249 gen_helper_read_cr8(v, tcg_env); 250 } else { 251 tcg_gen_ld_tl(v, tcg_env, offsetof(CPUX86State, cr[op->n])); 252 } 253 break; 254 case X86_OP_DR: 255 /* CR4.DE tested in the helper. */ 256 gen_helper_get_dr(v, tcg_env, tcg_constant_i32(op->n)); 257 break; 258#endif 259 case X86_OP_INT: 260 if (op->has_ea) { 261 if (v == s->T0 && decode->e.special == X86_SPECIAL_SExtT0) { 262 gen_op_ld_v(s, op->ot | MO_SIGN, v, s->A0); 263 } else { 264 gen_op_ld_v(s, op->ot, v, s->A0); 265 } 266 267 } else if (op->ot == MO_8 && byte_reg_is_xH(s, op->n)) { 268 if (v == s->T0 && decode->e.special == X86_SPECIAL_SExtT0) { 269 tcg_gen_sextract_tl(v, cpu_regs[op->n - 4], 8, 8); 270 } else { 271 tcg_gen_extract_tl(v, cpu_regs[op->n - 4], 8, 8); 272 } 273 274 } else if (op->ot < MO_TL && v == s->T0 && 275 (decode->e.special == X86_SPECIAL_SExtT0 || 276 decode->e.special == X86_SPECIAL_ZExtT0)) { 277 if (decode->e.special == X86_SPECIAL_SExtT0) { 278 tcg_gen_ext_tl(v, cpu_regs[op->n], op->ot | MO_SIGN); 279 } else { 280 tcg_gen_ext_tl(v, cpu_regs[op->n], op->ot); 281 } 282 283 } else { 284 tcg_gen_mov_tl(v, cpu_regs[op->n]); 285 } 286 break; 287 case X86_OP_IMM: 288 tcg_gen_movi_tl(v, op->imm); 289 break; 290 291 case X86_OP_MMX: 292 compute_mmx_offset(op); 293 goto load_vector; 294 295 case X86_OP_SSE: 296 compute_xmm_offset(op); 297 load_vector: 298 if (op->has_ea) { 299 bool aligned = sse_needs_alignment(s, decode, op->ot); 300 gen_load_sse(s, v, op->ot, op->offset, aligned); 301 } 302 break; 303 304 default: 305 g_assert_not_reached(); 306 } 307} 308 309static TCGv_ptr op_ptr(X86DecodedInsn *decode, int opn) 310{ 311 X86DecodedOp *op = &decode->op[opn]; 312 313 assert(op->unit == X86_OP_MMX || op->unit == X86_OP_SSE); 314 if (op->v_ptr) { 315 return op->v_ptr; 316 } 317 op->v_ptr = tcg_temp_new_ptr(); 318 319 /* The temporary points to the MMXReg or ZMMReg. */ 320 tcg_gen_addi_ptr(op->v_ptr, tcg_env, vector_reg_offset(op)); 321 return op->v_ptr; 322} 323 324#define OP_PTR0 op_ptr(decode, 0) 325#define OP_PTR1 op_ptr(decode, 1) 326#define OP_PTR2 op_ptr(decode, 2) 327 328static void gen_writeback(DisasContext *s, X86DecodedInsn *decode, int opn, TCGv v) 329{ 330 X86DecodedOp *op = &decode->op[opn]; 331 switch (op->unit) { 332 case X86_OP_SKIP: 333 break; 334 case X86_OP_SEG: 335 /* Note that gen_movl_seg takes care of interrupt shadow and TF. */ 336 gen_movl_seg(s, op->n, s->T0); 337 break; 338 case X86_OP_INT: 339 if (op->has_ea) { 340 gen_op_st_v(s, op->ot, v, s->A0); 341 } else { 342 gen_op_mov_reg_v(s, op->ot, op->n, v); 343 } 344 break; 345 case X86_OP_MMX: 346 break; 347 case X86_OP_SSE: 348 if (!op->has_ea && (s->prefix & PREFIX_VEX) && op->ot <= MO_128) { 349 tcg_gen_gvec_dup_imm(MO_64, 350 offsetof(CPUX86State, xmm_regs[op->n].ZMM_X(1)), 351 16, 16, 0); 352 } 353 break; 354#ifndef CONFIG_USER_ONLY 355 case X86_OP_CR: 356 if (op->n == 8) { 357 translator_io_start(&s->base); 358 } 359 gen_helper_write_crN(tcg_env, tcg_constant_i32(op->n), v); 360 s->base.is_jmp = DISAS_EOB_NEXT; 361 break; 362 case X86_OP_DR: 363 /* CR4.DE tested in the helper. */ 364 gen_helper_set_dr(tcg_env, tcg_constant_i32(op->n), v); 365 s->base.is_jmp = DISAS_EOB_NEXT; 366 break; 367#endif 368 default: 369 g_assert_not_reached(); 370 } 371 op->unit = X86_OP_SKIP; 372} 373 374static inline int vector_len(DisasContext *s, X86DecodedInsn *decode) 375{ 376 if (decode->e.special == X86_SPECIAL_MMX && 377 !(s->prefix & (PREFIX_DATA | PREFIX_REPZ | PREFIX_REPNZ))) { 378 return 8; 379 } 380 return s->vex_l ? 32 : 16; 381} 382 383static void prepare_update1_cc(X86DecodedInsn *decode, DisasContext *s, CCOp op) 384{ 385 decode->cc_dst = s->T0; 386 decode->cc_op = op; 387} 388 389static void prepare_update2_cc(X86DecodedInsn *decode, DisasContext *s, CCOp op) 390{ 391 decode->cc_src = s->T1; 392 decode->cc_dst = s->T0; 393 decode->cc_op = op; 394} 395 396static void prepare_update_cc_incdec(X86DecodedInsn *decode, DisasContext *s, CCOp op) 397{ 398 gen_compute_eflags_c(s, s->T1); 399 prepare_update2_cc(decode, s, op); 400} 401 402static void prepare_update3_cc(X86DecodedInsn *decode, DisasContext *s, CCOp op, TCGv reg) 403{ 404 decode->cc_src2 = reg; 405 decode->cc_src = s->T1; 406 decode->cc_dst = s->T0; 407 decode->cc_op = op; 408} 409 410static void gen_store_sse(DisasContext *s, X86DecodedInsn *decode, int src_ofs) 411{ 412 MemOp ot = decode->op[0].ot; 413 int vec_len = vector_len(s, decode); 414 bool aligned = sse_needs_alignment(s, decode, ot); 415 416 if (!decode->op[0].has_ea) { 417 tcg_gen_gvec_mov(MO_64, decode->op[0].offset, src_ofs, vec_len, vec_len); 418 return; 419 } 420 421 switch (ot) { 422 case MO_64: 423 gen_stq_env_A0(s, src_ofs); 424 break; 425 case MO_128: 426 gen_sto_env_A0(s, src_ofs, aligned); 427 break; 428 case MO_256: 429 gen_sty_env_A0(s, src_ofs, aligned); 430 break; 431 default: 432 g_assert_not_reached(); 433 } 434} 435 436static void gen_helper_pavgusb(TCGv_ptr env, TCGv_ptr reg_a, TCGv_ptr reg_b) 437{ 438 gen_helper_pavgb_mmx(env, reg_a, reg_a, reg_b); 439} 440 441#define FN_3DNOW_MOVE ((SSEFunc_0_epp) (uintptr_t) 1) 442static const SSEFunc_0_epp fns_3dnow[] = { 443 [0x0c] = gen_helper_pi2fw, 444 [0x0d] = gen_helper_pi2fd, 445 [0x1c] = gen_helper_pf2iw, 446 [0x1d] = gen_helper_pf2id, 447 [0x8a] = gen_helper_pfnacc, 448 [0x8e] = gen_helper_pfpnacc, 449 [0x90] = gen_helper_pfcmpge, 450 [0x94] = gen_helper_pfmin, 451 [0x96] = gen_helper_pfrcp, 452 [0x97] = gen_helper_pfrsqrt, 453 [0x9a] = gen_helper_pfsub, 454 [0x9e] = gen_helper_pfadd, 455 [0xa0] = gen_helper_pfcmpgt, 456 [0xa4] = gen_helper_pfmax, 457 [0xa6] = FN_3DNOW_MOVE, /* PFRCPIT1; no need to actually increase precision */ 458 [0xa7] = FN_3DNOW_MOVE, /* PFRSQIT1 */ 459 [0xb6] = FN_3DNOW_MOVE, /* PFRCPIT2 */ 460 [0xaa] = gen_helper_pfsubr, 461 [0xae] = gen_helper_pfacc, 462 [0xb0] = gen_helper_pfcmpeq, 463 [0xb4] = gen_helper_pfmul, 464 [0xb7] = gen_helper_pmulhrw_mmx, 465 [0xbb] = gen_helper_pswapd, 466 [0xbf] = gen_helper_pavgusb, 467}; 468 469static void gen_3dnow(DisasContext *s, X86DecodedInsn *decode) 470{ 471 uint8_t b = decode->immediate; 472 SSEFunc_0_epp fn = b < ARRAY_SIZE(fns_3dnow) ? fns_3dnow[b] : NULL; 473 474 if (!fn) { 475 gen_illegal_opcode(s); 476 return; 477 } 478 if (s->flags & HF_TS_MASK) { 479 gen_NM_exception(s); 480 return; 481 } 482 if (s->flags & HF_EM_MASK) { 483 gen_illegal_opcode(s); 484 return; 485 } 486 487 gen_helper_enter_mmx(tcg_env); 488 if (fn == FN_3DNOW_MOVE) { 489 tcg_gen_ld_i64(s->tmp1_i64, tcg_env, decode->op[1].offset); 490 tcg_gen_st_i64(s->tmp1_i64, tcg_env, decode->op[0].offset); 491 } else { 492 fn(tcg_env, OP_PTR0, OP_PTR1); 493 } 494} 495 496/* 497 * 00 = v*ps Vps, Hps, Wpd 498 * 66 = v*pd Vpd, Hpd, Wps 499 * f3 = v*ss Vss, Hss, Wps 500 * f2 = v*sd Vsd, Hsd, Wps 501 */ 502static inline void gen_unary_fp_sse(DisasContext *s, X86DecodedInsn *decode, 503 SSEFunc_0_epp pd_xmm, SSEFunc_0_epp ps_xmm, 504 SSEFunc_0_epp pd_ymm, SSEFunc_0_epp ps_ymm, 505 SSEFunc_0_eppp sd, SSEFunc_0_eppp ss) 506{ 507 if ((s->prefix & (PREFIX_REPZ | PREFIX_REPNZ)) != 0) { 508 SSEFunc_0_eppp fn = s->prefix & PREFIX_REPZ ? ss : sd; 509 if (!fn) { 510 gen_illegal_opcode(s); 511 return; 512 } 513 fn(tcg_env, OP_PTR0, OP_PTR1, OP_PTR2); 514 } else { 515 SSEFunc_0_epp ps, pd, fn; 516 ps = s->vex_l ? ps_ymm : ps_xmm; 517 pd = s->vex_l ? pd_ymm : pd_xmm; 518 fn = s->prefix & PREFIX_DATA ? pd : ps; 519 if (!fn) { 520 gen_illegal_opcode(s); 521 return; 522 } 523 fn(tcg_env, OP_PTR0, OP_PTR2); 524 } 525} 526#define UNARY_FP_SSE(uname, lname) \ 527static void gen_##uname(DisasContext *s, X86DecodedInsn *decode) \ 528{ \ 529 gen_unary_fp_sse(s, decode, \ 530 gen_helper_##lname##pd_xmm, \ 531 gen_helper_##lname##ps_xmm, \ 532 gen_helper_##lname##pd_ymm, \ 533 gen_helper_##lname##ps_ymm, \ 534 gen_helper_##lname##sd, \ 535 gen_helper_##lname##ss); \ 536} 537UNARY_FP_SSE(VSQRT, sqrt) 538 539/* 540 * 00 = v*ps Vps, Hps, Wpd 541 * 66 = v*pd Vpd, Hpd, Wps 542 * f3 = v*ss Vss, Hss, Wps 543 * f2 = v*sd Vsd, Hsd, Wps 544 */ 545static inline void gen_fp_sse(DisasContext *s, X86DecodedInsn *decode, 546 SSEFunc_0_eppp pd_xmm, SSEFunc_0_eppp ps_xmm, 547 SSEFunc_0_eppp pd_ymm, SSEFunc_0_eppp ps_ymm, 548 SSEFunc_0_eppp sd, SSEFunc_0_eppp ss) 549{ 550 SSEFunc_0_eppp ps, pd, fn; 551 if ((s->prefix & (PREFIX_REPZ | PREFIX_REPNZ)) != 0) { 552 fn = s->prefix & PREFIX_REPZ ? ss : sd; 553 } else { 554 ps = s->vex_l ? ps_ymm : ps_xmm; 555 pd = s->vex_l ? pd_ymm : pd_xmm; 556 fn = s->prefix & PREFIX_DATA ? pd : ps; 557 } 558 if (fn) { 559 fn(tcg_env, OP_PTR0, OP_PTR1, OP_PTR2); 560 } else { 561 gen_illegal_opcode(s); 562 } 563} 564 565#define FP_SSE(uname, lname) \ 566static void gen_##uname(DisasContext *s, X86DecodedInsn *decode) \ 567{ \ 568 gen_fp_sse(s, decode, \ 569 gen_helper_##lname##pd_xmm, \ 570 gen_helper_##lname##ps_xmm, \ 571 gen_helper_##lname##pd_ymm, \ 572 gen_helper_##lname##ps_ymm, \ 573 gen_helper_##lname##sd, \ 574 gen_helper_##lname##ss); \ 575} 576FP_SSE(VADD, add) 577FP_SSE(VMUL, mul) 578FP_SSE(VSUB, sub) 579FP_SSE(VMIN, min) 580FP_SSE(VDIV, div) 581FP_SSE(VMAX, max) 582 583#define FMA_SSE_PACKED(uname, ptr0, ptr1, ptr2, even, odd) \ 584static void gen_##uname##Px(DisasContext *s, X86DecodedInsn *decode) \ 585{ \ 586 SSEFunc_0_eppppii xmm = s->vex_w ? gen_helper_fma4pd_xmm : gen_helper_fma4ps_xmm; \ 587 SSEFunc_0_eppppii ymm = s->vex_w ? gen_helper_fma4pd_ymm : gen_helper_fma4ps_ymm; \ 588 SSEFunc_0_eppppii fn = s->vex_l ? ymm : xmm; \ 589 \ 590 fn(tcg_env, OP_PTR0, ptr0, ptr1, ptr2, \ 591 tcg_constant_i32(even), \ 592 tcg_constant_i32((even) ^ (odd))); \ 593} 594 595#define FMA_SSE(uname, ptr0, ptr1, ptr2, flags) \ 596FMA_SSE_PACKED(uname, ptr0, ptr1, ptr2, flags, flags) \ 597static void gen_##uname##Sx(DisasContext *s, X86DecodedInsn *decode) \ 598{ \ 599 SSEFunc_0_eppppi fn = s->vex_w ? gen_helper_fma4sd : gen_helper_fma4ss; \ 600 \ 601 fn(tcg_env, OP_PTR0, ptr0, ptr1, ptr2, \ 602 tcg_constant_i32(flags)); \ 603} \ 604 605FMA_SSE(VFMADD231, OP_PTR1, OP_PTR2, OP_PTR0, 0) 606FMA_SSE(VFMADD213, OP_PTR1, OP_PTR0, OP_PTR2, 0) 607FMA_SSE(VFMADD132, OP_PTR0, OP_PTR2, OP_PTR1, 0) 608 609FMA_SSE(VFNMADD231, OP_PTR1, OP_PTR2, OP_PTR0, float_muladd_negate_product) 610FMA_SSE(VFNMADD213, OP_PTR1, OP_PTR0, OP_PTR2, float_muladd_negate_product) 611FMA_SSE(VFNMADD132, OP_PTR0, OP_PTR2, OP_PTR1, float_muladd_negate_product) 612 613FMA_SSE(VFMSUB231, OP_PTR1, OP_PTR2, OP_PTR0, float_muladd_negate_c) 614FMA_SSE(VFMSUB213, OP_PTR1, OP_PTR0, OP_PTR2, float_muladd_negate_c) 615FMA_SSE(VFMSUB132, OP_PTR0, OP_PTR2, OP_PTR1, float_muladd_negate_c) 616 617FMA_SSE(VFNMSUB231, OP_PTR1, OP_PTR2, OP_PTR0, float_muladd_negate_c|float_muladd_negate_product) 618FMA_SSE(VFNMSUB213, OP_PTR1, OP_PTR0, OP_PTR2, float_muladd_negate_c|float_muladd_negate_product) 619FMA_SSE(VFNMSUB132, OP_PTR0, OP_PTR2, OP_PTR1, float_muladd_negate_c|float_muladd_negate_product) 620 621FMA_SSE_PACKED(VFMADDSUB231, OP_PTR1, OP_PTR2, OP_PTR0, float_muladd_negate_c, 0) 622FMA_SSE_PACKED(VFMADDSUB213, OP_PTR1, OP_PTR0, OP_PTR2, float_muladd_negate_c, 0) 623FMA_SSE_PACKED(VFMADDSUB132, OP_PTR0, OP_PTR2, OP_PTR1, float_muladd_negate_c, 0) 624 625FMA_SSE_PACKED(VFMSUBADD231, OP_PTR1, OP_PTR2, OP_PTR0, 0, float_muladd_negate_c) 626FMA_SSE_PACKED(VFMSUBADD213, OP_PTR1, OP_PTR0, OP_PTR2, 0, float_muladd_negate_c) 627FMA_SSE_PACKED(VFMSUBADD132, OP_PTR0, OP_PTR2, OP_PTR1, 0, float_muladd_negate_c) 628 629#define FP_UNPACK_SSE(uname, lname) \ 630static void gen_##uname(DisasContext *s, X86DecodedInsn *decode) \ 631{ \ 632 /* PS maps to the DQ integer instruction, PD maps to QDQ. */ \ 633 gen_fp_sse(s, decode, \ 634 gen_helper_##lname##qdq_xmm, \ 635 gen_helper_##lname##dq_xmm, \ 636 gen_helper_##lname##qdq_ymm, \ 637 gen_helper_##lname##dq_ymm, \ 638 NULL, NULL); \ 639} 640FP_UNPACK_SSE(VUNPCKLPx, punpckl) 641FP_UNPACK_SSE(VUNPCKHPx, punpckh) 642 643/* 644 * 00 = v*ps Vps, Wpd 645 * f3 = v*ss Vss, Wps 646 */ 647static inline void gen_unary_fp32_sse(DisasContext *s, X86DecodedInsn *decode, 648 SSEFunc_0_epp ps_xmm, 649 SSEFunc_0_epp ps_ymm, 650 SSEFunc_0_eppp ss) 651{ 652 if ((s->prefix & (PREFIX_DATA | PREFIX_REPNZ)) != 0) { 653 goto illegal_op; 654 } else if (s->prefix & PREFIX_REPZ) { 655 if (!ss) { 656 goto illegal_op; 657 } 658 ss(tcg_env, OP_PTR0, OP_PTR1, OP_PTR2); 659 } else { 660 SSEFunc_0_epp fn = s->vex_l ? ps_ymm : ps_xmm; 661 if (!fn) { 662 goto illegal_op; 663 } 664 fn(tcg_env, OP_PTR0, OP_PTR2); 665 } 666 return; 667 668illegal_op: 669 gen_illegal_opcode(s); 670} 671#define UNARY_FP32_SSE(uname, lname) \ 672static void gen_##uname(DisasContext *s, X86DecodedInsn *decode) \ 673{ \ 674 gen_unary_fp32_sse(s, decode, \ 675 gen_helper_##lname##ps_xmm, \ 676 gen_helper_##lname##ps_ymm, \ 677 gen_helper_##lname##ss); \ 678} 679UNARY_FP32_SSE(VRSQRT, rsqrt) 680UNARY_FP32_SSE(VRCP, rcp) 681 682/* 683 * 66 = v*pd Vpd, Hpd, Wpd 684 * f2 = v*ps Vps, Hps, Wps 685 */ 686static inline void gen_horizontal_fp_sse(DisasContext *s, X86DecodedInsn *decode, 687 SSEFunc_0_eppp pd_xmm, SSEFunc_0_eppp ps_xmm, 688 SSEFunc_0_eppp pd_ymm, SSEFunc_0_eppp ps_ymm) 689{ 690 SSEFunc_0_eppp ps, pd, fn; 691 ps = s->vex_l ? ps_ymm : ps_xmm; 692 pd = s->vex_l ? pd_ymm : pd_xmm; 693 fn = s->prefix & PREFIX_DATA ? pd : ps; 694 fn(tcg_env, OP_PTR0, OP_PTR1, OP_PTR2); 695} 696#define HORIZONTAL_FP_SSE(uname, lname) \ 697static void gen_##uname(DisasContext *s, X86DecodedInsn *decode) \ 698{ \ 699 gen_horizontal_fp_sse(s, decode, \ 700 gen_helper_##lname##pd_xmm, gen_helper_##lname##ps_xmm, \ 701 gen_helper_##lname##pd_ymm, gen_helper_##lname##ps_ymm); \ 702} 703HORIZONTAL_FP_SSE(VHADD, hadd) 704HORIZONTAL_FP_SSE(VHSUB, hsub) 705HORIZONTAL_FP_SSE(VADDSUB, addsub) 706 707static inline void gen_ternary_sse(DisasContext *s, X86DecodedInsn *decode, 708 int op3, SSEFunc_0_epppp xmm, SSEFunc_0_epppp ymm) 709{ 710 SSEFunc_0_epppp fn = s->vex_l ? ymm : xmm; 711 TCGv_ptr ptr3 = tcg_temp_new_ptr(); 712 713 /* The format of the fourth input is Lx */ 714 tcg_gen_addi_ptr(ptr3, tcg_env, ZMM_OFFSET(op3)); 715 fn(tcg_env, OP_PTR0, OP_PTR1, OP_PTR2, ptr3); 716} 717#define TERNARY_SSE(uname, uvname, lname) \ 718static void gen_##uvname(DisasContext *s, X86DecodedInsn *decode) \ 719{ \ 720 gen_ternary_sse(s, decode, (uint8_t)decode->immediate >> 4, \ 721 gen_helper_##lname##_xmm, gen_helper_##lname##_ymm); \ 722} \ 723static void gen_##uname(DisasContext *s, X86DecodedInsn *decode) \ 724{ \ 725 gen_ternary_sse(s, decode, 0, \ 726 gen_helper_##lname##_xmm, gen_helper_##lname##_ymm); \ 727} 728TERNARY_SSE(BLENDVPS, VBLENDVPS, blendvps) 729TERNARY_SSE(BLENDVPD, VBLENDVPD, blendvpd) 730TERNARY_SSE(PBLENDVB, VPBLENDVB, pblendvb) 731 732static inline void gen_binary_imm_sse(DisasContext *s, X86DecodedInsn *decode, 733 SSEFunc_0_epppi xmm, SSEFunc_0_epppi ymm) 734{ 735 TCGv_i32 imm = tcg_constant8u_i32(decode->immediate); 736 if (!s->vex_l) { 737 xmm(tcg_env, OP_PTR0, OP_PTR1, OP_PTR2, imm); 738 } else { 739 ymm(tcg_env, OP_PTR0, OP_PTR1, OP_PTR2, imm); 740 } 741} 742 743#define BINARY_IMM_SSE(uname, lname) \ 744static void gen_##uname(DisasContext *s, X86DecodedInsn *decode) \ 745{ \ 746 gen_binary_imm_sse(s, decode, \ 747 gen_helper_##lname##_xmm, \ 748 gen_helper_##lname##_ymm); \ 749} 750 751BINARY_IMM_SSE(VBLENDPD, blendpd) 752BINARY_IMM_SSE(VBLENDPS, blendps) 753BINARY_IMM_SSE(VPBLENDW, pblendw) 754BINARY_IMM_SSE(VDDPS, dpps) 755#define gen_helper_dppd_ymm NULL 756BINARY_IMM_SSE(VDDPD, dppd) 757BINARY_IMM_SSE(VMPSADBW, mpsadbw) 758BINARY_IMM_SSE(PCLMULQDQ, pclmulqdq) 759 760 761#define UNARY_INT_GVEC(uname, func, ...) \ 762static void gen_##uname(DisasContext *s, X86DecodedInsn *decode) \ 763{ \ 764 int vec_len = vector_len(s, decode); \ 765 \ 766 func(__VA_ARGS__, decode->op[0].offset, \ 767 decode->op[2].offset, vec_len, vec_len); \ 768} 769UNARY_INT_GVEC(PABSB, tcg_gen_gvec_abs, MO_8) 770UNARY_INT_GVEC(PABSW, tcg_gen_gvec_abs, MO_16) 771UNARY_INT_GVEC(PABSD, tcg_gen_gvec_abs, MO_32) 772UNARY_INT_GVEC(VBROADCASTx128, tcg_gen_gvec_dup_mem, MO_128) 773UNARY_INT_GVEC(VPBROADCASTB, tcg_gen_gvec_dup_mem, MO_8) 774UNARY_INT_GVEC(VPBROADCASTW, tcg_gen_gvec_dup_mem, MO_16) 775UNARY_INT_GVEC(VPBROADCASTD, tcg_gen_gvec_dup_mem, MO_32) 776UNARY_INT_GVEC(VPBROADCASTQ, tcg_gen_gvec_dup_mem, MO_64) 777 778 779#define BINARY_INT_GVEC(uname, func, ...) \ 780static void gen_##uname(DisasContext *s, X86DecodedInsn *decode) \ 781{ \ 782 int vec_len = vector_len(s, decode); \ 783 \ 784 func(__VA_ARGS__, \ 785 decode->op[0].offset, decode->op[1].offset, \ 786 decode->op[2].offset, vec_len, vec_len); \ 787} 788 789BINARY_INT_GVEC(PADDB, tcg_gen_gvec_add, MO_8) 790BINARY_INT_GVEC(PADDW, tcg_gen_gvec_add, MO_16) 791BINARY_INT_GVEC(PADDD, tcg_gen_gvec_add, MO_32) 792BINARY_INT_GVEC(PADDQ, tcg_gen_gvec_add, MO_64) 793BINARY_INT_GVEC(PADDSB, tcg_gen_gvec_ssadd, MO_8) 794BINARY_INT_GVEC(PADDSW, tcg_gen_gvec_ssadd, MO_16) 795BINARY_INT_GVEC(PADDUSB, tcg_gen_gvec_usadd, MO_8) 796BINARY_INT_GVEC(PADDUSW, tcg_gen_gvec_usadd, MO_16) 797BINARY_INT_GVEC(PAND, tcg_gen_gvec_and, MO_64) 798BINARY_INT_GVEC(PCMPEQB, tcg_gen_gvec_cmp, TCG_COND_EQ, MO_8) 799BINARY_INT_GVEC(PCMPEQD, tcg_gen_gvec_cmp, TCG_COND_EQ, MO_32) 800BINARY_INT_GVEC(PCMPEQW, tcg_gen_gvec_cmp, TCG_COND_EQ, MO_16) 801BINARY_INT_GVEC(PCMPEQQ, tcg_gen_gvec_cmp, TCG_COND_EQ, MO_64) 802BINARY_INT_GVEC(PCMPGTB, tcg_gen_gvec_cmp, TCG_COND_GT, MO_8) 803BINARY_INT_GVEC(PCMPGTW, tcg_gen_gvec_cmp, TCG_COND_GT, MO_16) 804BINARY_INT_GVEC(PCMPGTD, tcg_gen_gvec_cmp, TCG_COND_GT, MO_32) 805BINARY_INT_GVEC(PCMPGTQ, tcg_gen_gvec_cmp, TCG_COND_GT, MO_64) 806BINARY_INT_GVEC(PMAXSB, tcg_gen_gvec_smax, MO_8) 807BINARY_INT_GVEC(PMAXSW, tcg_gen_gvec_smax, MO_16) 808BINARY_INT_GVEC(PMAXSD, tcg_gen_gvec_smax, MO_32) 809BINARY_INT_GVEC(PMAXUB, tcg_gen_gvec_umax, MO_8) 810BINARY_INT_GVEC(PMAXUW, tcg_gen_gvec_umax, MO_16) 811BINARY_INT_GVEC(PMAXUD, tcg_gen_gvec_umax, MO_32) 812BINARY_INT_GVEC(PMINSB, tcg_gen_gvec_smin, MO_8) 813BINARY_INT_GVEC(PMINSW, tcg_gen_gvec_smin, MO_16) 814BINARY_INT_GVEC(PMINSD, tcg_gen_gvec_smin, MO_32) 815BINARY_INT_GVEC(PMINUB, tcg_gen_gvec_umin, MO_8) 816BINARY_INT_GVEC(PMINUW, tcg_gen_gvec_umin, MO_16) 817BINARY_INT_GVEC(PMINUD, tcg_gen_gvec_umin, MO_32) 818BINARY_INT_GVEC(PMULLW, tcg_gen_gvec_mul, MO_16) 819BINARY_INT_GVEC(PMULLD, tcg_gen_gvec_mul, MO_32) 820BINARY_INT_GVEC(POR, tcg_gen_gvec_or, MO_64) 821BINARY_INT_GVEC(PSUBB, tcg_gen_gvec_sub, MO_8) 822BINARY_INT_GVEC(PSUBW, tcg_gen_gvec_sub, MO_16) 823BINARY_INT_GVEC(PSUBD, tcg_gen_gvec_sub, MO_32) 824BINARY_INT_GVEC(PSUBQ, tcg_gen_gvec_sub, MO_64) 825BINARY_INT_GVEC(PSUBSB, tcg_gen_gvec_sssub, MO_8) 826BINARY_INT_GVEC(PSUBSW, tcg_gen_gvec_sssub, MO_16) 827BINARY_INT_GVEC(PSUBUSB, tcg_gen_gvec_ussub, MO_8) 828BINARY_INT_GVEC(PSUBUSW, tcg_gen_gvec_ussub, MO_16) 829BINARY_INT_GVEC(PXOR, tcg_gen_gvec_xor, MO_64) 830 831 832/* 833 * 00 = p* Pq, Qq (if mmx not NULL; no VEX) 834 * 66 = vp* Vx, Hx, Wx 835 * 836 * These are really the same encoding, because 1) V is the same as P when VEX.V 837 * is not present 2) P and Q are the same as H and W apart from MM/XMM 838 */ 839static inline void gen_binary_int_sse(DisasContext *s, X86DecodedInsn *decode, 840 SSEFunc_0_eppp mmx, SSEFunc_0_eppp xmm, SSEFunc_0_eppp ymm) 841{ 842 assert(!!mmx == !!(decode->e.special == X86_SPECIAL_MMX)); 843 844 if (mmx && (s->prefix & PREFIX_VEX) && !(s->prefix & PREFIX_DATA)) { 845 /* VEX encoding is not applicable to MMX instructions. */ 846 gen_illegal_opcode(s); 847 return; 848 } 849 if (!(s->prefix & PREFIX_DATA)) { 850 mmx(tcg_env, OP_PTR0, OP_PTR1, OP_PTR2); 851 } else if (!s->vex_l) { 852 xmm(tcg_env, OP_PTR0, OP_PTR1, OP_PTR2); 853 } else { 854 ymm(tcg_env, OP_PTR0, OP_PTR1, OP_PTR2); 855 } 856} 857 858 859#define BINARY_INT_MMX(uname, lname) \ 860static void gen_##uname(DisasContext *s, X86DecodedInsn *decode) \ 861{ \ 862 gen_binary_int_sse(s, decode, \ 863 gen_helper_##lname##_mmx, \ 864 gen_helper_##lname##_xmm, \ 865 gen_helper_##lname##_ymm); \ 866} 867BINARY_INT_MMX(PUNPCKLBW, punpcklbw) 868BINARY_INT_MMX(PUNPCKLWD, punpcklwd) 869BINARY_INT_MMX(PUNPCKLDQ, punpckldq) 870BINARY_INT_MMX(PACKSSWB, packsswb) 871BINARY_INT_MMX(PACKUSWB, packuswb) 872BINARY_INT_MMX(PUNPCKHBW, punpckhbw) 873BINARY_INT_MMX(PUNPCKHWD, punpckhwd) 874BINARY_INT_MMX(PUNPCKHDQ, punpckhdq) 875BINARY_INT_MMX(PACKSSDW, packssdw) 876 877BINARY_INT_MMX(PAVGB, pavgb) 878BINARY_INT_MMX(PAVGW, pavgw) 879BINARY_INT_MMX(PMADDWD, pmaddwd) 880BINARY_INT_MMX(PMULHUW, pmulhuw) 881BINARY_INT_MMX(PMULHW, pmulhw) 882BINARY_INT_MMX(PMULUDQ, pmuludq) 883BINARY_INT_MMX(PSADBW, psadbw) 884 885BINARY_INT_MMX(PSLLW_r, psllw) 886BINARY_INT_MMX(PSLLD_r, pslld) 887BINARY_INT_MMX(PSLLQ_r, psllq) 888BINARY_INT_MMX(PSRLW_r, psrlw) 889BINARY_INT_MMX(PSRLD_r, psrld) 890BINARY_INT_MMX(PSRLQ_r, psrlq) 891BINARY_INT_MMX(PSRAW_r, psraw) 892BINARY_INT_MMX(PSRAD_r, psrad) 893 894BINARY_INT_MMX(PHADDW, phaddw) 895BINARY_INT_MMX(PHADDSW, phaddsw) 896BINARY_INT_MMX(PHADDD, phaddd) 897BINARY_INT_MMX(PHSUBW, phsubw) 898BINARY_INT_MMX(PHSUBSW, phsubsw) 899BINARY_INT_MMX(PHSUBD, phsubd) 900BINARY_INT_MMX(PMADDUBSW, pmaddubsw) 901BINARY_INT_MMX(PSHUFB, pshufb) 902BINARY_INT_MMX(PSIGNB, psignb) 903BINARY_INT_MMX(PSIGNW, psignw) 904BINARY_INT_MMX(PSIGND, psignd) 905BINARY_INT_MMX(PMULHRSW, pmulhrsw) 906 907/* Instructions with no MMX equivalent. */ 908#define BINARY_INT_SSE(uname, lname) \ 909static void gen_##uname(DisasContext *s, X86DecodedInsn *decode) \ 910{ \ 911 gen_binary_int_sse(s, decode, \ 912 NULL, \ 913 gen_helper_##lname##_xmm, \ 914 gen_helper_##lname##_ymm); \ 915} 916 917/* Instructions with no MMX equivalent. */ 918BINARY_INT_SSE(PUNPCKLQDQ, punpcklqdq) 919BINARY_INT_SSE(PUNPCKHQDQ, punpckhqdq) 920BINARY_INT_SSE(VPACKUSDW, packusdw) 921BINARY_INT_SSE(VPERMILPS, vpermilps) 922BINARY_INT_SSE(VPERMILPD, vpermilpd) 923BINARY_INT_SSE(VMASKMOVPS, vpmaskmovd) 924BINARY_INT_SSE(VMASKMOVPD, vpmaskmovq) 925 926BINARY_INT_SSE(PMULDQ, pmuldq) 927 928BINARY_INT_SSE(VAESDEC, aesdec) 929BINARY_INT_SSE(VAESDECLAST, aesdeclast) 930BINARY_INT_SSE(VAESENC, aesenc) 931BINARY_INT_SSE(VAESENCLAST, aesenclast) 932 933#define UNARY_CMP_SSE(uname, lname) \ 934static void gen_##uname(DisasContext *s, X86DecodedInsn *decode) \ 935{ \ 936 if (!s->vex_l) { \ 937 gen_helper_##lname##_xmm(tcg_env, OP_PTR1, OP_PTR2); \ 938 } else { \ 939 gen_helper_##lname##_ymm(tcg_env, OP_PTR1, OP_PTR2); \ 940 } \ 941 assume_cc_op(s, CC_OP_EFLAGS); \ 942} 943UNARY_CMP_SSE(VPTEST, ptest) 944UNARY_CMP_SSE(VTESTPS, vtestps) 945UNARY_CMP_SSE(VTESTPD, vtestpd) 946 947static inline void gen_unary_int_sse(DisasContext *s, X86DecodedInsn *decode, 948 SSEFunc_0_epp xmm, SSEFunc_0_epp ymm) 949{ 950 if (!s->vex_l) { 951 xmm(tcg_env, OP_PTR0, OP_PTR2); 952 } else { 953 ymm(tcg_env, OP_PTR0, OP_PTR2); 954 } 955} 956 957#define UNARY_INT_SSE(uname, lname) \ 958static void gen_##uname(DisasContext *s, X86DecodedInsn *decode) \ 959{ \ 960 gen_unary_int_sse(s, decode, \ 961 gen_helper_##lname##_xmm, \ 962 gen_helper_##lname##_ymm); \ 963} 964 965UNARY_INT_SSE(VPMOVSXBW, pmovsxbw) 966UNARY_INT_SSE(VPMOVSXBD, pmovsxbd) 967UNARY_INT_SSE(VPMOVSXBQ, pmovsxbq) 968UNARY_INT_SSE(VPMOVSXWD, pmovsxwd) 969UNARY_INT_SSE(VPMOVSXWQ, pmovsxwq) 970UNARY_INT_SSE(VPMOVSXDQ, pmovsxdq) 971 972UNARY_INT_SSE(VPMOVZXBW, pmovzxbw) 973UNARY_INT_SSE(VPMOVZXBD, pmovzxbd) 974UNARY_INT_SSE(VPMOVZXBQ, pmovzxbq) 975UNARY_INT_SSE(VPMOVZXWD, pmovzxwd) 976UNARY_INT_SSE(VPMOVZXWQ, pmovzxwq) 977UNARY_INT_SSE(VPMOVZXDQ, pmovzxdq) 978 979UNARY_INT_SSE(VMOVSLDUP, pmovsldup) 980UNARY_INT_SSE(VMOVSHDUP, pmovshdup) 981UNARY_INT_SSE(VMOVDDUP, pmovdldup) 982 983UNARY_INT_SSE(VCVTDQ2PD, cvtdq2pd) 984UNARY_INT_SSE(VCVTPD2DQ, cvtpd2dq) 985UNARY_INT_SSE(VCVTTPD2DQ, cvttpd2dq) 986UNARY_INT_SSE(VCVTDQ2PS, cvtdq2ps) 987UNARY_INT_SSE(VCVTPS2DQ, cvtps2dq) 988UNARY_INT_SSE(VCVTTPS2DQ, cvttps2dq) 989UNARY_INT_SSE(VCVTPH2PS, cvtph2ps) 990 991 992static inline void gen_unary_imm_sse(DisasContext *s, X86DecodedInsn *decode, 993 SSEFunc_0_ppi xmm, SSEFunc_0_ppi ymm) 994{ 995 TCGv_i32 imm = tcg_constant8u_i32(decode->immediate); 996 if (!s->vex_l) { 997 xmm(OP_PTR0, OP_PTR1, imm); 998 } else { 999 ymm(OP_PTR0, OP_PTR1, imm); 1000 } 1001} 1002 1003#define UNARY_IMM_SSE(uname, lname) \ 1004static void gen_##uname(DisasContext *s, X86DecodedInsn *decode) \ 1005{ \ 1006 gen_unary_imm_sse(s, decode, \ 1007 gen_helper_##lname##_xmm, \ 1008 gen_helper_##lname##_ymm); \ 1009} 1010 1011UNARY_IMM_SSE(PSHUFD, pshufd) 1012UNARY_IMM_SSE(PSHUFHW, pshufhw) 1013UNARY_IMM_SSE(PSHUFLW, pshuflw) 1014#define gen_helper_vpermq_xmm NULL 1015UNARY_IMM_SSE(VPERMQ, vpermq) 1016UNARY_IMM_SSE(VPERMILPS_i, vpermilps_imm) 1017UNARY_IMM_SSE(VPERMILPD_i, vpermilpd_imm) 1018 1019static inline void gen_unary_imm_fp_sse(DisasContext *s, X86DecodedInsn *decode, 1020 SSEFunc_0_eppi xmm, SSEFunc_0_eppi ymm) 1021{ 1022 TCGv_i32 imm = tcg_constant8u_i32(decode->immediate); 1023 if (!s->vex_l) { 1024 xmm(tcg_env, OP_PTR0, OP_PTR1, imm); 1025 } else { 1026 ymm(tcg_env, OP_PTR0, OP_PTR1, imm); 1027 } 1028} 1029 1030#define UNARY_IMM_FP_SSE(uname, lname) \ 1031static void gen_##uname(DisasContext *s, X86DecodedInsn *decode) \ 1032{ \ 1033 gen_unary_imm_fp_sse(s, decode, \ 1034 gen_helper_##lname##_xmm, \ 1035 gen_helper_##lname##_ymm); \ 1036} 1037 1038UNARY_IMM_FP_SSE(VROUNDPS, roundps) 1039UNARY_IMM_FP_SSE(VROUNDPD, roundpd) 1040 1041static inline void gen_vexw_avx(DisasContext *s, X86DecodedInsn *decode, 1042 SSEFunc_0_eppp d_xmm, SSEFunc_0_eppp q_xmm, 1043 SSEFunc_0_eppp d_ymm, SSEFunc_0_eppp q_ymm) 1044{ 1045 SSEFunc_0_eppp d = s->vex_l ? d_ymm : d_xmm; 1046 SSEFunc_0_eppp q = s->vex_l ? q_ymm : q_xmm; 1047 SSEFunc_0_eppp fn = s->vex_w ? q : d; 1048 fn(tcg_env, OP_PTR0, OP_PTR1, OP_PTR2); 1049} 1050 1051/* VEX.W affects whether to operate on 32- or 64-bit elements. */ 1052#define VEXW_AVX(uname, lname) \ 1053static void gen_##uname(DisasContext *s, X86DecodedInsn *decode) \ 1054{ \ 1055 gen_vexw_avx(s, decode, \ 1056 gen_helper_##lname##d_xmm, gen_helper_##lname##q_xmm, \ 1057 gen_helper_##lname##d_ymm, gen_helper_##lname##q_ymm); \ 1058} 1059VEXW_AVX(VPSLLV, vpsllv) 1060VEXW_AVX(VPSRLV, vpsrlv) 1061VEXW_AVX(VPSRAV, vpsrav) 1062VEXW_AVX(VPMASKMOV, vpmaskmov) 1063 1064/* Same as above, but with extra arguments to the helper. */ 1065static inline void gen_vsib_avx(DisasContext *s, X86DecodedInsn *decode, 1066 SSEFunc_0_epppti d_xmm, SSEFunc_0_epppti q_xmm, 1067 SSEFunc_0_epppti d_ymm, SSEFunc_0_epppti q_ymm) 1068{ 1069 SSEFunc_0_epppti d = s->vex_l ? d_ymm : d_xmm; 1070 SSEFunc_0_epppti q = s->vex_l ? q_ymm : q_xmm; 1071 SSEFunc_0_epppti fn = s->vex_w ? q : d; 1072 TCGv_i32 scale = tcg_constant_i32(decode->mem.scale); 1073 TCGv_ptr index = tcg_temp_new_ptr(); 1074 1075 /* Pass third input as (index, base, scale) */ 1076 tcg_gen_addi_ptr(index, tcg_env, ZMM_OFFSET(decode->mem.index)); 1077 fn(tcg_env, OP_PTR0, OP_PTR1, index, s->A0, scale); 1078 1079 /* 1080 * There are two output operands, so zero OP1's high 128 bits 1081 * in the VEX.128 case. 1082 */ 1083 if (!s->vex_l) { 1084 int ymmh_ofs = vector_elem_offset(&decode->op[1], MO_128, 1); 1085 tcg_gen_gvec_dup_imm(MO_64, ymmh_ofs, 16, 16, 0); 1086 } 1087} 1088#define VSIB_AVX(uname, lname) \ 1089static void gen_##uname(DisasContext *s, X86DecodedInsn *decode) \ 1090{ \ 1091 gen_vsib_avx(s, decode, \ 1092 gen_helper_##lname##d_xmm, gen_helper_##lname##q_xmm, \ 1093 gen_helper_##lname##d_ymm, gen_helper_##lname##q_ymm); \ 1094} 1095VSIB_AVX(VPGATHERD, vpgatherd) 1096VSIB_AVX(VPGATHERQ, vpgatherq) 1097 1098static void gen_AAA(DisasContext *s, X86DecodedInsn *decode) 1099{ 1100 gen_update_cc_op(s); 1101 gen_helper_aaa(tcg_env); 1102 assume_cc_op(s, CC_OP_EFLAGS); 1103} 1104 1105static void gen_AAD(DisasContext *s, X86DecodedInsn *decode) 1106{ 1107 gen_helper_aad(s->T0, s->T0, s->T1); 1108 prepare_update1_cc(decode, s, CC_OP_LOGICB); 1109} 1110 1111static void gen_AAM(DisasContext *s, X86DecodedInsn *decode) 1112{ 1113 if (decode->immediate == 0) { 1114 gen_exception(s, EXCP00_DIVZ); 1115 } else { 1116 gen_helper_aam(s->T0, s->T0, s->T1); 1117 prepare_update1_cc(decode, s, CC_OP_LOGICB); 1118 } 1119} 1120 1121static void gen_AAS(DisasContext *s, X86DecodedInsn *decode) 1122{ 1123 gen_update_cc_op(s); 1124 gen_helper_aas(tcg_env); 1125 assume_cc_op(s, CC_OP_EFLAGS); 1126} 1127 1128static void gen_ADC(DisasContext *s, X86DecodedInsn *decode) 1129{ 1130 MemOp ot = decode->op[1].ot; 1131 TCGv c_in = tcg_temp_new(); 1132 1133 gen_compute_eflags_c(s, c_in); 1134 if (s->prefix & PREFIX_LOCK) { 1135 tcg_gen_add_tl(s->T0, c_in, s->T1); 1136 tcg_gen_atomic_add_fetch_tl(s->T0, s->A0, s->T0, 1137 s->mem_index, ot | MO_LE); 1138 } else { 1139 tcg_gen_add_tl(s->T0, s->T0, s->T1); 1140 tcg_gen_add_tl(s->T0, s->T0, c_in); 1141 } 1142 prepare_update3_cc(decode, s, CC_OP_ADCB + ot, c_in); 1143} 1144 1145static void gen_ADCOX(DisasContext *s, X86DecodedInsn *decode, int cc_op) 1146{ 1147 MemOp ot = decode->op[0].ot; 1148 TCGv carry_in = NULL; 1149 TCGv *carry_out = (cc_op == CC_OP_ADCX ? &decode->cc_dst : &decode->cc_src2); 1150 TCGv zero; 1151 1152 decode->cc_op = cc_op; 1153 *carry_out = tcg_temp_new(); 1154 if (CC_OP_HAS_EFLAGS(s->cc_op)) { 1155 decode->cc_src = cpu_cc_src; 1156 1157 /* Re-use the carry-out from a previous round? */ 1158 if (s->cc_op == cc_op || s->cc_op == CC_OP_ADCOX) { 1159 carry_in = (cc_op == CC_OP_ADCX ? cpu_cc_dst : cpu_cc_src2); 1160 } 1161 1162 /* Preserve the opposite carry from previous rounds? */ 1163 if (s->cc_op != cc_op && s->cc_op != CC_OP_EFLAGS) { 1164 decode->cc_op = CC_OP_ADCOX; 1165 if (carry_out == &decode->cc_dst) { 1166 decode->cc_src2 = cpu_cc_src2; 1167 } else { 1168 decode->cc_dst = cpu_cc_dst; 1169 } 1170 } 1171 } else { 1172 decode->cc_src = tcg_temp_new(); 1173 gen_mov_eflags(s, decode->cc_src); 1174 } 1175 1176 if (!carry_in) { 1177 /* Get carry_in out of EFLAGS. */ 1178 carry_in = tcg_temp_new(); 1179 tcg_gen_extract_tl(carry_in, decode->cc_src, 1180 ctz32(cc_op == CC_OP_ADCX ? CC_C : CC_O), 1); 1181 } 1182 1183 switch (ot) { 1184#ifdef TARGET_X86_64 1185 case MO_32: 1186 /* If TL is 64-bit just do everything in 64-bit arithmetic. */ 1187 tcg_gen_ext32u_tl(s->T0, s->T0); 1188 tcg_gen_ext32u_tl(s->T1, s->T1); 1189 tcg_gen_add_i64(s->T0, s->T0, s->T1); 1190 tcg_gen_add_i64(s->T0, s->T0, carry_in); 1191 tcg_gen_shri_i64(*carry_out, s->T0, 32); 1192 break; 1193#endif 1194 default: 1195 zero = tcg_constant_tl(0); 1196 tcg_gen_add2_tl(s->T0, *carry_out, s->T0, zero, carry_in, zero); 1197 tcg_gen_add2_tl(s->T0, *carry_out, s->T0, *carry_out, s->T1, zero); 1198 break; 1199 } 1200} 1201 1202static void gen_ADCX(DisasContext *s, X86DecodedInsn *decode) 1203{ 1204 gen_ADCOX(s, decode, CC_OP_ADCX); 1205} 1206 1207static void gen_ADD(DisasContext *s, X86DecodedInsn *decode) 1208{ 1209 MemOp ot = decode->op[1].ot; 1210 1211 if (s->prefix & PREFIX_LOCK) { 1212 tcg_gen_atomic_add_fetch_tl(s->T0, s->A0, s->T1, 1213 s->mem_index, ot | MO_LE); 1214 } else { 1215 tcg_gen_add_tl(s->T0, s->T0, s->T1); 1216 } 1217 prepare_update2_cc(decode, s, CC_OP_ADDB + ot); 1218} 1219 1220static void gen_ADOX(DisasContext *s, X86DecodedInsn *decode) 1221{ 1222 gen_ADCOX(s, decode, CC_OP_ADOX); 1223} 1224 1225static void gen_AND(DisasContext *s, X86DecodedInsn *decode) 1226{ 1227 MemOp ot = decode->op[1].ot; 1228 1229 if (s->prefix & PREFIX_LOCK) { 1230 tcg_gen_atomic_and_fetch_tl(s->T0, s->A0, s->T1, 1231 s->mem_index, ot | MO_LE); 1232 } else { 1233 tcg_gen_and_tl(s->T0, s->T0, s->T1); 1234 } 1235 prepare_update1_cc(decode, s, CC_OP_LOGICB + ot); 1236} 1237 1238static void gen_ANDN(DisasContext *s, X86DecodedInsn *decode) 1239{ 1240 MemOp ot = decode->op[0].ot; 1241 1242 tcg_gen_andc_tl(s->T0, s->T1, s->T0); 1243 prepare_update1_cc(decode, s, CC_OP_LOGICB + ot); 1244} 1245 1246static void gen_ARPL(DisasContext *s, X86DecodedInsn *decode) 1247{ 1248 TCGv zf = tcg_temp_new(); 1249 TCGv flags = tcg_temp_new(); 1250 1251 gen_mov_eflags(s, flags); 1252 1253 /* Compute adjusted DST in T1, merging in SRC[RPL]. */ 1254 tcg_gen_deposit_tl(s->T1, s->T0, s->T1, 0, 2); 1255 1256 /* Z flag set if DST[RPL] < SRC[RPL] */ 1257 tcg_gen_setcond_tl(TCG_COND_LTU, zf, s->T0, s->T1); 1258 tcg_gen_deposit_tl(flags, flags, zf, ctz32(CC_Z), 1); 1259 1260 /* Place maximum RPL in DST */ 1261 tcg_gen_umax_tl(s->T0, s->T0, s->T1); 1262 1263 decode->cc_src = flags; 1264 decode->cc_op = CC_OP_EFLAGS; 1265} 1266 1267static void gen_BEXTR(DisasContext *s, X86DecodedInsn *decode) 1268{ 1269 MemOp ot = decode->op[0].ot; 1270 TCGv bound = tcg_constant_tl(ot == MO_64 ? 63 : 31); 1271 TCGv zero = tcg_constant_tl(0); 1272 TCGv mone = tcg_constant_tl(-1); 1273 1274 /* 1275 * Extract START, and shift the operand. 1276 * Shifts larger than operand size get zeros. 1277 */ 1278 tcg_gen_ext8u_tl(s->A0, s->T1); 1279 tcg_gen_shr_tl(s->T0, s->T0, s->A0); 1280 1281 tcg_gen_movcond_tl(TCG_COND_LEU, s->T0, s->A0, bound, s->T0, zero); 1282 1283 /* 1284 * Extract the LEN into an inverse mask. Lengths larger than 1285 * operand size get all zeros, length 0 gets all ones. 1286 */ 1287 tcg_gen_extract_tl(s->A0, s->T1, 8, 8); 1288 tcg_gen_shl_tl(s->T1, mone, s->A0); 1289 tcg_gen_movcond_tl(TCG_COND_LEU, s->T1, s->A0, bound, s->T1, zero); 1290 tcg_gen_andc_tl(s->T0, s->T0, s->T1); 1291 1292 prepare_update1_cc(decode, s, CC_OP_LOGICB + ot); 1293} 1294 1295static void gen_BLSI(DisasContext *s, X86DecodedInsn *decode) 1296{ 1297 MemOp ot = decode->op[0].ot; 1298 1299 /* input in T1, which is ready for prepare_update2_cc */ 1300 tcg_gen_neg_tl(s->T0, s->T1); 1301 tcg_gen_and_tl(s->T0, s->T0, s->T1); 1302 prepare_update2_cc(decode, s, CC_OP_BMILGB + ot); 1303} 1304 1305static void gen_BLSMSK(DisasContext *s, X86DecodedInsn *decode) 1306{ 1307 MemOp ot = decode->op[0].ot; 1308 1309 /* input in T1, which is ready for prepare_update2_cc */ 1310 tcg_gen_subi_tl(s->T0, s->T1, 1); 1311 tcg_gen_xor_tl(s->T0, s->T0, s->T1); 1312 prepare_update2_cc(decode, s, CC_OP_BMILGB + ot); 1313} 1314 1315static void gen_BLSR(DisasContext *s, X86DecodedInsn *decode) 1316{ 1317 MemOp ot = decode->op[0].ot; 1318 1319 /* input in T1, which is ready for prepare_update2_cc */ 1320 tcg_gen_subi_tl(s->T0, s->T1, 1); 1321 tcg_gen_and_tl(s->T0, s->T0, s->T1); 1322 prepare_update2_cc(decode, s, CC_OP_BMILGB + ot); 1323} 1324 1325static void gen_BOUND(DisasContext *s, X86DecodedInsn *decode) 1326{ 1327 TCGv_i32 op = tcg_temp_new_i32(); 1328 tcg_gen_trunc_tl_i32(op, s->T0); 1329 if (decode->op[1].ot == MO_16) { 1330 gen_helper_boundw(tcg_env, s->A0, op); 1331 } else { 1332 gen_helper_boundl(tcg_env, s->A0, op); 1333 } 1334} 1335 1336/* Non-standard convention - on entry T0 is zero-extended input, T1 is the output. */ 1337static void gen_BSF(DisasContext *s, X86DecodedInsn *decode) 1338{ 1339 MemOp ot = decode->op[0].ot; 1340 1341 /* Only the Z bit is defined and it is related to the input. */ 1342 decode->cc_dst = tcg_temp_new(); 1343 decode->cc_op = CC_OP_LOGICB + ot; 1344 tcg_gen_mov_tl(decode->cc_dst, s->T0); 1345 1346 /* 1347 * The manual says that the output is undefined when the 1348 * input is zero, but real hardware leaves it unchanged, and 1349 * real programs appear to depend on that. Accomplish this 1350 * by passing the output as the value to return upon zero. 1351 */ 1352 tcg_gen_ctz_tl(s->T0, s->T0, s->T1); 1353} 1354 1355/* Non-standard convention - on entry T0 is zero-extended input, T1 is the output. */ 1356static void gen_BSR(DisasContext *s, X86DecodedInsn *decode) 1357{ 1358 MemOp ot = decode->op[0].ot; 1359 1360 /* Only the Z bit is defined and it is related to the input. */ 1361 decode->cc_dst = tcg_temp_new(); 1362 decode->cc_op = CC_OP_LOGICB + ot; 1363 tcg_gen_mov_tl(decode->cc_dst, s->T0); 1364 1365 /* 1366 * The manual says that the output is undefined when the 1367 * input is zero, but real hardware leaves it unchanged, and 1368 * real programs appear to depend on that. Accomplish this 1369 * by passing the output as the value to return upon zero. 1370 * Plus, return the bit index of the first 1 bit. 1371 */ 1372 tcg_gen_xori_tl(s->T1, s->T1, TARGET_LONG_BITS - 1); 1373 tcg_gen_clz_tl(s->T0, s->T0, s->T1); 1374 tcg_gen_xori_tl(s->T0, s->T0, TARGET_LONG_BITS - 1); 1375} 1376 1377static void gen_BSWAP(DisasContext *s, X86DecodedInsn *decode) 1378{ 1379#ifdef TARGET_X86_64 1380 if (s->dflag == MO_64) { 1381 tcg_gen_bswap64_i64(s->T0, s->T0); 1382 return; 1383 } 1384#endif 1385 tcg_gen_bswap32_tl(s->T0, s->T0, TCG_BSWAP_OZ); 1386} 1387 1388static void gen_BZHI(DisasContext *s, X86DecodedInsn *decode) 1389{ 1390 MemOp ot = decode->op[0].ot; 1391 TCGv bound = tcg_constant_tl(ot == MO_64 ? 63 : 31); 1392 TCGv zero = tcg_constant_tl(0); 1393 TCGv mone = tcg_constant_tl(-1); 1394 1395 tcg_gen_ext8u_tl(s->T1, s->T1); 1396 1397 tcg_gen_shl_tl(s->A0, mone, s->T1); 1398 tcg_gen_movcond_tl(TCG_COND_LEU, s->A0, s->T1, bound, s->A0, zero); 1399 tcg_gen_andc_tl(s->T0, s->T0, s->A0); 1400 /* 1401 * Note that since we're using BMILG (in order to get O 1402 * cleared) we need to store the inverse into C. 1403 */ 1404 tcg_gen_setcond_tl(TCG_COND_LEU, s->T1, s->T1, bound); 1405 prepare_update2_cc(decode, s, CC_OP_BMILGB + ot); 1406} 1407 1408static void gen_CALL(DisasContext *s, X86DecodedInsn *decode) 1409{ 1410 gen_push_v(s, eip_next_tl(s)); 1411 gen_JMP(s, decode); 1412} 1413 1414static void gen_CALL_m(DisasContext *s, X86DecodedInsn *decode) 1415{ 1416 gen_push_v(s, eip_next_tl(s)); 1417 gen_JMP_m(s, decode); 1418} 1419 1420static void gen_CALLF(DisasContext *s, X86DecodedInsn *decode) 1421{ 1422 gen_far_call(s); 1423} 1424 1425static void gen_CALLF_m(DisasContext *s, X86DecodedInsn *decode) 1426{ 1427 MemOp ot = decode->op[1].ot; 1428 1429 gen_op_ld_v(s, ot, s->T0, s->A0); 1430 gen_add_A0_im(s, 1 << ot); 1431 gen_op_ld_v(s, MO_16, s->T1, s->A0); 1432 gen_far_call(s); 1433} 1434 1435static void gen_CBW(DisasContext *s, X86DecodedInsn *decode) 1436{ 1437 MemOp src_ot = decode->op[0].ot - 1; 1438 1439 tcg_gen_ext_tl(s->T0, s->T0, src_ot | MO_SIGN); 1440} 1441 1442static void gen_CLC(DisasContext *s, X86DecodedInsn *decode) 1443{ 1444 gen_compute_eflags(s); 1445 tcg_gen_andi_tl(cpu_cc_src, cpu_cc_src, ~CC_C); 1446} 1447 1448static void gen_CLD(DisasContext *s, X86DecodedInsn *decode) 1449{ 1450 tcg_gen_st_i32(tcg_constant_i32(1), tcg_env, offsetof(CPUX86State, df)); 1451} 1452 1453static void gen_CLI(DisasContext *s, X86DecodedInsn *decode) 1454{ 1455 gen_reset_eflags(s, IF_MASK); 1456} 1457 1458static void gen_CLTS(DisasContext *s, X86DecodedInsn *decode) 1459{ 1460 gen_helper_clts(tcg_env); 1461 /* abort block because static cpu state changed */ 1462 s->base.is_jmp = DISAS_EOB_NEXT; 1463} 1464 1465static void gen_CMC(DisasContext *s, X86DecodedInsn *decode) 1466{ 1467 gen_compute_eflags(s); 1468 tcg_gen_xori_tl(cpu_cc_src, cpu_cc_src, CC_C); 1469} 1470 1471static void gen_CMOVcc(DisasContext *s, X86DecodedInsn *decode) 1472{ 1473 gen_cmovcc1(s, decode->b & 0xf, s->T0, s->T1); 1474} 1475 1476static void gen_CMPccXADD(DisasContext *s, X86DecodedInsn *decode) 1477{ 1478 TCGLabel *label_top = gen_new_label(); 1479 TCGLabel *label_bottom = gen_new_label(); 1480 TCGv oldv = tcg_temp_new(); 1481 TCGv newv = tcg_temp_new(); 1482 TCGv cmpv = tcg_temp_new(); 1483 TCGCond cond; 1484 1485 TCGv cmp_lhs, cmp_rhs; 1486 MemOp ot, ot_full; 1487 1488 int jcc_op = (decode->b >> 1) & 7; 1489 static const TCGCond cond_table[8] = { 1490 [JCC_O] = TCG_COND_LT, /* test sign bit by comparing against 0 */ 1491 [JCC_B] = TCG_COND_LTU, 1492 [JCC_Z] = TCG_COND_EQ, 1493 [JCC_BE] = TCG_COND_LEU, 1494 [JCC_S] = TCG_COND_LT, /* test sign bit by comparing against 0 */ 1495 [JCC_P] = TCG_COND_TSTEQ, /* even parity - tests low bit of popcount */ 1496 [JCC_L] = TCG_COND_LT, 1497 [JCC_LE] = TCG_COND_LE, 1498 }; 1499 1500 cond = cond_table[jcc_op]; 1501 if (decode->b & 1) { 1502 cond = tcg_invert_cond(cond); 1503 } 1504 1505 ot = decode->op[0].ot; 1506 ot_full = ot | MO_LE; 1507 if (jcc_op >= JCC_S) { 1508 /* 1509 * Sign-extend values before subtracting for S, P (zero/sign extension 1510 * does not matter there) L, LE and their inverses. 1511 */ 1512 ot_full |= MO_SIGN; 1513 } 1514 1515 /* 1516 * cmpv will be moved to cc_src *after* cpu_regs[] is written back, so use 1517 * tcg_gen_ext_tl instead of gen_ext_tl. 1518 */ 1519 tcg_gen_ext_tl(cmpv, cpu_regs[decode->op[1].n], ot_full); 1520 1521 /* 1522 * Cmpxchg loop starts here. 1523 * - s->T1: addition operand (from decoder) 1524 * - s->A0: dest address (from decoder) 1525 * - s->cc_srcT: memory operand (lhs for comparison) 1526 * - cmpv: rhs for comparison 1527 */ 1528 gen_set_label(label_top); 1529 gen_op_ld_v(s, ot_full, s->cc_srcT, s->A0); 1530 tcg_gen_sub_tl(s->T0, s->cc_srcT, cmpv); 1531 1532 /* Compute the comparison result by hand, to avoid clobbering cc_*. */ 1533 switch (jcc_op) { 1534 case JCC_O: 1535 /* (src1 ^ src2) & (src1 ^ dst). newv is only used here for a moment */ 1536 tcg_gen_xor_tl(newv, s->cc_srcT, s->T0); 1537 tcg_gen_xor_tl(s->tmp0, s->cc_srcT, cmpv); 1538 tcg_gen_and_tl(s->tmp0, s->tmp0, newv); 1539 tcg_gen_sextract_tl(s->tmp0, s->tmp0, 0, 8 << ot); 1540 cmp_lhs = s->tmp0, cmp_rhs = tcg_constant_tl(0); 1541 break; 1542 1543 case JCC_P: 1544 tcg_gen_ext8u_tl(s->tmp0, s->T0); 1545 tcg_gen_ctpop_tl(s->tmp0, s->tmp0); 1546 cmp_lhs = s->tmp0, cmp_rhs = tcg_constant_tl(1); 1547 break; 1548 1549 case JCC_S: 1550 tcg_gen_sextract_tl(s->tmp0, s->T0, 0, 8 << ot); 1551 cmp_lhs = s->tmp0, cmp_rhs = tcg_constant_tl(0); 1552 break; 1553 1554 default: 1555 cmp_lhs = s->cc_srcT, cmp_rhs = cmpv; 1556 break; 1557 } 1558 1559 /* Compute new value: if condition does not hold, just store back s->cc_srcT */ 1560 tcg_gen_add_tl(newv, s->cc_srcT, s->T1); 1561 tcg_gen_movcond_tl(cond, newv, cmp_lhs, cmp_rhs, newv, s->cc_srcT); 1562 tcg_gen_atomic_cmpxchg_tl(oldv, s->A0, s->cc_srcT, newv, s->mem_index, ot_full); 1563 1564 /* Exit unconditionally if cmpxchg succeeded. */ 1565 tcg_gen_brcond_tl(TCG_COND_EQ, oldv, s->cc_srcT, label_bottom); 1566 1567 /* Try again if there was actually a store to make. */ 1568 tcg_gen_brcond_tl(cond, cmp_lhs, cmp_rhs, label_top); 1569 gen_set_label(label_bottom); 1570 1571 /* Store old value to registers only after a successful store. */ 1572 gen_writeback(s, decode, 1, s->cc_srcT); 1573 1574 decode->cc_dst = s->T0; 1575 decode->cc_src = cmpv; 1576 decode->cc_op = CC_OP_SUBB + ot; 1577} 1578 1579static void gen_CMPS(DisasContext *s, X86DecodedInsn *decode) 1580{ 1581 MemOp ot = decode->op[2].ot; 1582 if (s->prefix & (PREFIX_REPZ | PREFIX_REPNZ)) { 1583 gen_repz_nz(s, ot, gen_cmps); 1584 } else { 1585 gen_cmps(s, ot); 1586 } 1587} 1588 1589static void gen_CMPXCHG(DisasContext *s, X86DecodedInsn *decode) 1590{ 1591 MemOp ot = decode->op[2].ot; 1592 TCGv cmpv = tcg_temp_new(); 1593 TCGv oldv = tcg_temp_new(); 1594 TCGv newv = tcg_temp_new(); 1595 TCGv dest; 1596 1597 tcg_gen_ext_tl(cmpv, cpu_regs[R_EAX], ot); 1598 tcg_gen_ext_tl(newv, s->T1, ot); 1599 if (s->prefix & PREFIX_LOCK) { 1600 tcg_gen_atomic_cmpxchg_tl(oldv, s->A0, cmpv, newv, 1601 s->mem_index, ot | MO_LE); 1602 } else { 1603 tcg_gen_ext_tl(oldv, s->T0, ot); 1604 if (decode->op[0].has_ea) { 1605 /* 1606 * Perform an unconditional store cycle like physical cpu; 1607 * must be before changing accumulator to ensure 1608 * idempotency if the store faults and the instruction 1609 * is restarted 1610 */ 1611 tcg_gen_movcond_tl(TCG_COND_EQ, newv, oldv, cmpv, newv, oldv); 1612 gen_op_st_v(s, ot, newv, s->A0); 1613 } else { 1614 /* 1615 * Unlike the memory case, where "the destination operand receives 1616 * a write cycle without regard to the result of the comparison", 1617 * rm must not be touched altogether if the write fails, including 1618 * not zero-extending it on 64-bit processors. So, precompute 1619 * the result of a successful writeback and perform the movcond 1620 * directly on cpu_regs. In case rm is part of RAX, note that this 1621 * movcond and the one below are mutually exclusive is executed. 1622 */ 1623 dest = gen_op_deposit_reg_v(s, ot, decode->op[0].n, newv, newv); 1624 tcg_gen_movcond_tl(TCG_COND_EQ, dest, oldv, cmpv, newv, dest); 1625 } 1626 decode->op[0].unit = X86_OP_SKIP; 1627 } 1628 1629 /* Write RAX only if the cmpxchg fails. */ 1630 dest = gen_op_deposit_reg_v(s, ot, R_EAX, s->T0, oldv); 1631 tcg_gen_movcond_tl(TCG_COND_NE, dest, oldv, cmpv, s->T0, dest); 1632 1633 tcg_gen_mov_tl(s->cc_srcT, cmpv); 1634 tcg_gen_sub_tl(cmpv, cmpv, oldv); 1635 decode->cc_dst = cmpv; 1636 decode->cc_src = oldv; 1637 decode->cc_op = CC_OP_SUBB + ot; 1638} 1639 1640static void gen_CPUID(DisasContext *s, X86DecodedInsn *decode) 1641{ 1642 gen_update_cc_op(s); 1643 gen_update_eip_cur(s); 1644 gen_helper_cpuid(tcg_env); 1645} 1646 1647static void gen_CRC32(DisasContext *s, X86DecodedInsn *decode) 1648{ 1649 MemOp ot = decode->op[2].ot; 1650 1651 tcg_gen_trunc_tl_i32(s->tmp2_i32, s->T0); 1652 gen_helper_crc32(s->T0, s->tmp2_i32, s->T1, tcg_constant_i32(8 << ot)); 1653} 1654 1655static void gen_CVTPI2Px(DisasContext *s, X86DecodedInsn *decode) 1656{ 1657 gen_helper_enter_mmx(tcg_env); 1658 if (s->prefix & PREFIX_DATA) { 1659 gen_helper_cvtpi2pd(tcg_env, OP_PTR0, OP_PTR2); 1660 } else { 1661 gen_helper_cvtpi2ps(tcg_env, OP_PTR0, OP_PTR2); 1662 } 1663} 1664 1665static void gen_CVTPx2PI(DisasContext *s, X86DecodedInsn *decode) 1666{ 1667 gen_helper_enter_mmx(tcg_env); 1668 if (s->prefix & PREFIX_DATA) { 1669 gen_helper_cvtpd2pi(tcg_env, OP_PTR0, OP_PTR2); 1670 } else { 1671 gen_helper_cvtps2pi(tcg_env, OP_PTR0, OP_PTR2); 1672 } 1673} 1674 1675static void gen_CVTTPx2PI(DisasContext *s, X86DecodedInsn *decode) 1676{ 1677 gen_helper_enter_mmx(tcg_env); 1678 if (s->prefix & PREFIX_DATA) { 1679 gen_helper_cvttpd2pi(tcg_env, OP_PTR0, OP_PTR2); 1680 } else { 1681 gen_helper_cvttps2pi(tcg_env, OP_PTR0, OP_PTR2); 1682 } 1683} 1684 1685static void gen_CWD(DisasContext *s, X86DecodedInsn *decode) 1686{ 1687 int shift = 8 << decode->op[0].ot; 1688 1689 tcg_gen_sextract_tl(s->T0, s->T0, shift - 1, 1); 1690} 1691 1692static void gen_DAA(DisasContext *s, X86DecodedInsn *decode) 1693{ 1694 gen_update_cc_op(s); 1695 gen_helper_daa(tcg_env); 1696 assume_cc_op(s, CC_OP_EFLAGS); 1697} 1698 1699static void gen_DAS(DisasContext *s, X86DecodedInsn *decode) 1700{ 1701 gen_update_cc_op(s); 1702 gen_helper_das(tcg_env); 1703 assume_cc_op(s, CC_OP_EFLAGS); 1704} 1705 1706static void gen_DEC(DisasContext *s, X86DecodedInsn *decode) 1707{ 1708 MemOp ot = decode->op[1].ot; 1709 1710 tcg_gen_movi_tl(s->T1, -1); 1711 if (s->prefix & PREFIX_LOCK) { 1712 tcg_gen_atomic_add_fetch_tl(s->T0, s->A0, s->T1, 1713 s->mem_index, ot | MO_LE); 1714 } else { 1715 tcg_gen_add_tl(s->T0, s->T0, s->T1); 1716 } 1717 prepare_update_cc_incdec(decode, s, CC_OP_DECB + ot); 1718} 1719 1720static void gen_DIV(DisasContext *s, X86DecodedInsn *decode) 1721{ 1722 MemOp ot = decode->op[1].ot; 1723 1724 switch(ot) { 1725 case MO_8: 1726 gen_helper_divb_AL(tcg_env, s->T0); 1727 break; 1728 case MO_16: 1729 gen_helper_divw_AX(tcg_env, s->T0); 1730 break; 1731 default: 1732 case MO_32: 1733 gen_helper_divl_EAX(tcg_env, s->T0); 1734 break; 1735#ifdef TARGET_X86_64 1736 case MO_64: 1737 gen_helper_divq_EAX(tcg_env, s->T0); 1738 break; 1739#endif 1740 } 1741} 1742 1743static void gen_EMMS(DisasContext *s, X86DecodedInsn *decode) 1744{ 1745 gen_helper_emms(tcg_env); 1746} 1747 1748static void gen_ENTER(DisasContext *s, X86DecodedInsn *decode) 1749{ 1750 gen_enter(s, decode->op[1].imm, decode->op[2].imm); 1751} 1752 1753static void gen_EXTRQ_i(DisasContext *s, X86DecodedInsn *decode) 1754{ 1755 TCGv_i32 length = tcg_constant_i32(decode->immediate & 63); 1756 TCGv_i32 index = tcg_constant_i32((decode->immediate >> 8) & 63); 1757 1758 gen_helper_extrq_i(tcg_env, OP_PTR0, index, length); 1759} 1760 1761static void gen_EXTRQ_r(DisasContext *s, X86DecodedInsn *decode) 1762{ 1763 gen_helper_extrq_r(tcg_env, OP_PTR0, OP_PTR2); 1764} 1765 1766static void gen_FXRSTOR(DisasContext *s, X86DecodedInsn *decode) 1767{ 1768 if ((s->flags & HF_EM_MASK) || (s->flags & HF_TS_MASK)) { 1769 gen_NM_exception(s); 1770 } else { 1771 gen_helper_fxrstor(tcg_env, s->A0); 1772 } 1773} 1774 1775static void gen_FXSAVE(DisasContext *s, X86DecodedInsn *decode) 1776{ 1777 if ((s->flags & HF_EM_MASK) || (s->flags & HF_TS_MASK)) { 1778 gen_NM_exception(s); 1779 } else { 1780 gen_helper_fxsave(tcg_env, s->A0); 1781 } 1782} 1783 1784static void gen_HLT(DisasContext *s, X86DecodedInsn *decode) 1785{ 1786#ifdef CONFIG_SYSTEM_ONLY 1787 gen_update_cc_op(s); 1788 gen_update_eip_next(s); 1789 gen_helper_hlt(tcg_env); 1790 s->base.is_jmp = DISAS_NORETURN; 1791#endif 1792} 1793 1794static void gen_IDIV(DisasContext *s, X86DecodedInsn *decode) 1795{ 1796 MemOp ot = decode->op[1].ot; 1797 1798 switch(ot) { 1799 case MO_8: 1800 gen_helper_idivb_AL(tcg_env, s->T0); 1801 break; 1802 case MO_16: 1803 gen_helper_idivw_AX(tcg_env, s->T0); 1804 break; 1805 default: 1806 case MO_32: 1807 gen_helper_idivl_EAX(tcg_env, s->T0); 1808 break; 1809#ifdef TARGET_X86_64 1810 case MO_64: 1811 gen_helper_idivq_EAX(tcg_env, s->T0); 1812 break; 1813#endif 1814 } 1815} 1816 1817static void gen_IMUL3(DisasContext *s, X86DecodedInsn *decode) 1818{ 1819 MemOp ot = decode->op[0].ot; 1820 TCGv cc_src_rhs; 1821 1822 switch (ot) { 1823 case MO_16: 1824 /* s->T0 already sign-extended */ 1825 tcg_gen_ext16s_tl(s->T1, s->T1); 1826 tcg_gen_mul_tl(s->T0, s->T0, s->T1); 1827 /* Compare the full result to the extension of the truncated result. */ 1828 tcg_gen_ext16s_tl(s->T1, s->T0); 1829 cc_src_rhs = s->T0; 1830 break; 1831 1832 case MO_32: 1833#ifdef TARGET_X86_64 1834 if (TCG_TARGET_REG_BITS == 64) { 1835 /* 1836 * This produces fewer TCG ops, and better code if flags are needed, 1837 * but it requires a 64-bit multiply even if they are not. Use it 1838 * only if the target has 64-bits registers. 1839 * 1840 * s->T0 is already sign-extended. 1841 */ 1842 tcg_gen_ext32s_tl(s->T1, s->T1); 1843 tcg_gen_mul_tl(s->T0, s->T0, s->T1); 1844 /* Compare the full result to the extension of the truncated result. */ 1845 tcg_gen_ext32s_tl(s->T1, s->T0); 1846 cc_src_rhs = s->T0; 1847 } else { 1848 /* Variant that only needs a 32-bit widening multiply. */ 1849 TCGv_i32 hi = tcg_temp_new_i32(); 1850 TCGv_i32 lo = tcg_temp_new_i32(); 1851 tcg_gen_trunc_tl_i32(lo, s->T0); 1852 tcg_gen_trunc_tl_i32(hi, s->T1); 1853 tcg_gen_muls2_i32(lo, hi, lo, hi); 1854 tcg_gen_extu_i32_tl(s->T0, lo); 1855 1856 cc_src_rhs = tcg_temp_new(); 1857 tcg_gen_extu_i32_tl(cc_src_rhs, hi); 1858 /* Compare the high part to the sign bit of the truncated result */ 1859 tcg_gen_sari_i32(lo, lo, 31); 1860 tcg_gen_extu_i32_tl(s->T1, lo); 1861 } 1862 break; 1863 1864 case MO_64: 1865#endif 1866 cc_src_rhs = tcg_temp_new(); 1867 tcg_gen_muls2_tl(s->T0, cc_src_rhs, s->T0, s->T1); 1868 /* Compare the high part to the sign bit of the truncated result */ 1869 tcg_gen_sari_tl(s->T1, s->T0, TARGET_LONG_BITS - 1); 1870 break; 1871 1872 default: 1873 g_assert_not_reached(); 1874 } 1875 1876 tcg_gen_sub_tl(s->T1, s->T1, cc_src_rhs); 1877 prepare_update2_cc(decode, s, CC_OP_MULB + ot); 1878} 1879 1880static void gen_IMUL(DisasContext *s, X86DecodedInsn *decode) 1881{ 1882 MemOp ot = decode->op[1].ot; 1883 TCGv cc_src_rhs; 1884 1885 switch (ot) { 1886 case MO_8: 1887 /* s->T0 already sign-extended */ 1888 tcg_gen_ext8s_tl(s->T1, s->T1); 1889 tcg_gen_mul_tl(s->T0, s->T0, s->T1); 1890 gen_op_mov_reg_v(s, MO_16, R_EAX, s->T0); 1891 /* Compare the full result to the extension of the truncated result. */ 1892 tcg_gen_ext8s_tl(s->T1, s->T0); 1893 cc_src_rhs = s->T0; 1894 break; 1895 1896 case MO_16: 1897 /* s->T0 already sign-extended */ 1898 tcg_gen_ext16s_tl(s->T1, s->T1); 1899 tcg_gen_mul_tl(s->T0, s->T0, s->T1); 1900 gen_op_mov_reg_v(s, MO_16, R_EAX, s->T0); 1901 tcg_gen_shri_tl(s->T1, s->T0, 16); 1902 gen_op_mov_reg_v(s, MO_16, R_EDX, s->T1); 1903 /* Compare the full result to the extension of the truncated result. */ 1904 tcg_gen_ext16s_tl(s->T1, s->T0); 1905 cc_src_rhs = s->T0; 1906 break; 1907 1908 case MO_32: 1909#ifdef TARGET_X86_64 1910 /* s->T0 already sign-extended */ 1911 tcg_gen_ext32s_tl(s->T1, s->T1); 1912 tcg_gen_mul_tl(s->T0, s->T0, s->T1); 1913 tcg_gen_ext32u_tl(cpu_regs[R_EAX], s->T0); 1914 tcg_gen_shri_tl(cpu_regs[R_EDX], s->T0, 32); 1915 /* Compare the full result to the extension of the truncated result. */ 1916 tcg_gen_ext32s_tl(s->T1, s->T0); 1917 cc_src_rhs = s->T0; 1918 break; 1919 1920 case MO_64: 1921#endif 1922 tcg_gen_muls2_tl(s->T0, cpu_regs[R_EDX], s->T0, s->T1); 1923 tcg_gen_mov_tl(cpu_regs[R_EAX], s->T0); 1924 1925 /* Compare the high part to the sign bit of the truncated result */ 1926 tcg_gen_negsetcondi_tl(TCG_COND_LT, s->T1, s->T0, 0); 1927 cc_src_rhs = cpu_regs[R_EDX]; 1928 break; 1929 1930 default: 1931 g_assert_not_reached(); 1932 } 1933 1934 tcg_gen_sub_tl(s->T1, s->T1, cc_src_rhs); 1935 prepare_update2_cc(decode, s, CC_OP_MULB + ot); 1936} 1937 1938static void gen_IN(DisasContext *s, X86DecodedInsn *decode) 1939{ 1940 MemOp ot = decode->op[0].ot; 1941 TCGv_i32 port = tcg_temp_new_i32(); 1942 1943 tcg_gen_trunc_tl_i32(port, s->T0); 1944 tcg_gen_ext16u_i32(port, port); 1945 if (!gen_check_io(s, ot, port, SVM_IOIO_TYPE_MASK)) { 1946 return; 1947 } 1948 translator_io_start(&s->base); 1949 gen_helper_in_func(ot, s->T0, port); 1950 gen_writeback(s, decode, 0, s->T0); 1951 gen_bpt_io(s, port, ot); 1952} 1953 1954static void gen_INC(DisasContext *s, X86DecodedInsn *decode) 1955{ 1956 MemOp ot = decode->op[1].ot; 1957 1958 tcg_gen_movi_tl(s->T1, 1); 1959 if (s->prefix & PREFIX_LOCK) { 1960 tcg_gen_atomic_add_fetch_tl(s->T0, s->A0, s->T1, 1961 s->mem_index, ot | MO_LE); 1962 } else { 1963 tcg_gen_add_tl(s->T0, s->T0, s->T1); 1964 } 1965 prepare_update_cc_incdec(decode, s, CC_OP_INCB + ot); 1966} 1967 1968static void gen_INS(DisasContext *s, X86DecodedInsn *decode) 1969{ 1970 MemOp ot = decode->op[1].ot; 1971 TCGv_i32 port = tcg_temp_new_i32(); 1972 1973 tcg_gen_trunc_tl_i32(port, s->T1); 1974 tcg_gen_ext16u_i32(port, port); 1975 if (!gen_check_io(s, ot, port, 1976 SVM_IOIO_TYPE_MASK | SVM_IOIO_STR_MASK)) { 1977 return; 1978 } 1979 1980 translator_io_start(&s->base); 1981 if (s->prefix & (PREFIX_REPZ | PREFIX_REPNZ)) { 1982 gen_repz(s, ot, gen_ins); 1983 } else { 1984 gen_ins(s, ot); 1985 } 1986} 1987 1988static void gen_INSERTQ_i(DisasContext *s, X86DecodedInsn *decode) 1989{ 1990 TCGv_i32 length = tcg_constant_i32(decode->immediate & 63); 1991 TCGv_i32 index = tcg_constant_i32((decode->immediate >> 8) & 63); 1992 1993 gen_helper_insertq_i(tcg_env, OP_PTR0, OP_PTR1, index, length); 1994} 1995 1996static void gen_INSERTQ_r(DisasContext *s, X86DecodedInsn *decode) 1997{ 1998 gen_helper_insertq_r(tcg_env, OP_PTR0, OP_PTR2); 1999} 2000 2001static void gen_INT(DisasContext *s, X86DecodedInsn *decode) 2002{ 2003 gen_interrupt(s, decode->immediate); 2004} 2005 2006static void gen_INT1(DisasContext *s, X86DecodedInsn *decode) 2007{ 2008 gen_update_cc_op(s); 2009 gen_update_eip_next(s); 2010 gen_helper_icebp(tcg_env); 2011 s->base.is_jmp = DISAS_NORETURN; 2012} 2013 2014static void gen_INT3(DisasContext *s, X86DecodedInsn *decode) 2015{ 2016 gen_interrupt(s, EXCP03_INT3); 2017} 2018 2019static void gen_INTO(DisasContext *s, X86DecodedInsn *decode) 2020{ 2021 gen_update_cc_op(s); 2022 gen_update_eip_cur(s); 2023 gen_helper_into(tcg_env, cur_insn_len_i32(s)); 2024} 2025 2026static void gen_IRET(DisasContext *s, X86DecodedInsn *decode) 2027{ 2028 if (!PE(s) || VM86(s)) { 2029 gen_helper_iret_real(tcg_env, tcg_constant_i32(s->dflag - 1)); 2030 } else { 2031 gen_helper_iret_protected(tcg_env, tcg_constant_i32(s->dflag - 1), 2032 eip_next_i32(s)); 2033 } 2034 assume_cc_op(s, CC_OP_EFLAGS); 2035 s->base.is_jmp = DISAS_EOB_ONLY; 2036} 2037 2038static void gen_Jcc(DisasContext *s, X86DecodedInsn *decode) 2039{ 2040 gen_bnd_jmp(s); 2041 gen_jcc(s, decode->b & 0xf, decode->immediate); 2042} 2043 2044static void gen_JCXZ(DisasContext *s, X86DecodedInsn *decode) 2045{ 2046 TCGLabel *taken = gen_new_label(); 2047 2048 gen_update_cc_op(s); 2049 gen_op_jz_ecx(s, taken); 2050 gen_conditional_jump_labels(s, decode->immediate, NULL, taken); 2051} 2052 2053static void gen_JMP(DisasContext *s, X86DecodedInsn *decode) 2054{ 2055 gen_update_cc_op(s); 2056 gen_jmp_rel(s, s->dflag, decode->immediate, 0); 2057} 2058 2059static void gen_JMP_m(DisasContext *s, X86DecodedInsn *decode) 2060{ 2061 gen_op_jmp_v(s, s->T0); 2062 gen_bnd_jmp(s); 2063 s->base.is_jmp = DISAS_JUMP; 2064} 2065 2066static void gen_JMPF(DisasContext *s, X86DecodedInsn *decode) 2067{ 2068 gen_far_jmp(s); 2069} 2070 2071static void gen_JMPF_m(DisasContext *s, X86DecodedInsn *decode) 2072{ 2073 MemOp ot = decode->op[1].ot; 2074 2075 gen_op_ld_v(s, ot, s->T0, s->A0); 2076 gen_add_A0_im(s, 1 << ot); 2077 gen_op_ld_v(s, MO_16, s->T1, s->A0); 2078 gen_far_jmp(s); 2079} 2080 2081static void gen_LAHF(DisasContext *s, X86DecodedInsn *decode) 2082{ 2083 if (CODE64(s) && !(s->cpuid_ext3_features & CPUID_EXT3_LAHF_LM)) { 2084 return gen_illegal_opcode(s); 2085 } 2086 gen_compute_eflags(s); 2087 /* Note: gen_compute_eflags() only gives the condition codes */ 2088 tcg_gen_ori_tl(s->T0, cpu_cc_src, 0x02); 2089 tcg_gen_deposit_tl(cpu_regs[R_EAX], cpu_regs[R_EAX], s->T0, 8, 8); 2090} 2091 2092static void gen_LAR(DisasContext *s, X86DecodedInsn *decode) 2093{ 2094 MemOp ot = decode->op[0].ot; 2095 TCGv result = tcg_temp_new(); 2096 TCGv dest; 2097 2098 gen_compute_eflags(s); 2099 gen_update_cc_op(s); 2100 gen_helper_lar(result, tcg_env, s->T0); 2101 2102 /* Perform writeback here to skip it if ZF=0. */ 2103 decode->op[0].unit = X86_OP_SKIP; 2104 dest = gen_op_deposit_reg_v(s, ot, decode->op[0].n, result, result); 2105 tcg_gen_movcond_tl(TCG_COND_TSTNE, dest, cpu_cc_src, tcg_constant_tl(CC_Z), 2106 result, dest); 2107} 2108 2109static void gen_LDMXCSR(DisasContext *s, X86DecodedInsn *decode) 2110{ 2111 tcg_gen_trunc_tl_i32(s->tmp2_i32, s->T0); 2112 gen_helper_ldmxcsr(tcg_env, s->tmp2_i32); 2113} 2114 2115static void gen_lxx_seg(DisasContext *s, X86DecodedInsn *decode, int seg) 2116{ 2117 MemOp ot = decode->op[0].ot; 2118 2119 /* Offset already in s->T0. */ 2120 gen_add_A0_im(s, 1 << ot); 2121 gen_op_ld_v(s, MO_16, s->T1, s->A0); 2122 2123 /* load the segment here to handle exceptions properly */ 2124 gen_movl_seg(s, seg, s->T1); 2125} 2126 2127static void gen_LDS(DisasContext *s, X86DecodedInsn *decode) 2128{ 2129 gen_lxx_seg(s, decode, R_DS); 2130} 2131 2132static void gen_LEA(DisasContext *s, X86DecodedInsn *decode) 2133{ 2134 TCGv ea = gen_lea_modrm_1(s, decode->mem, false); 2135 gen_lea_v_seg_dest(s, s->aflag, s->T0, ea, -1, -1); 2136} 2137 2138static void gen_LEAVE(DisasContext *s, X86DecodedInsn *decode) 2139{ 2140 gen_leave(s); 2141} 2142 2143static void gen_LES(DisasContext *s, X86DecodedInsn *decode) 2144{ 2145 gen_lxx_seg(s, decode, R_ES); 2146} 2147 2148static void gen_LFENCE(DisasContext *s, X86DecodedInsn *decode) 2149{ 2150 tcg_gen_mb(TCG_MO_LD_LD | TCG_BAR_SC); 2151} 2152 2153static void gen_LFS(DisasContext *s, X86DecodedInsn *decode) 2154{ 2155 gen_lxx_seg(s, decode, R_FS); 2156} 2157 2158static void gen_LGS(DisasContext *s, X86DecodedInsn *decode) 2159{ 2160 gen_lxx_seg(s, decode, R_GS); 2161} 2162 2163static void gen_LODS(DisasContext *s, X86DecodedInsn *decode) 2164{ 2165 MemOp ot = decode->op[1].ot; 2166 if (s->prefix & (PREFIX_REPZ | PREFIX_REPNZ)) { 2167 gen_repz(s, ot, gen_lods); 2168 } else { 2169 gen_lods(s, ot); 2170 } 2171} 2172 2173static void gen_LOOP(DisasContext *s, X86DecodedInsn *decode) 2174{ 2175 TCGLabel *taken = gen_new_label(); 2176 2177 gen_update_cc_op(s); 2178 gen_op_add_reg_im(s, s->aflag, R_ECX, -1); 2179 gen_op_jnz_ecx(s, taken); 2180 gen_conditional_jump_labels(s, decode->immediate, NULL, taken); 2181} 2182 2183static void gen_LOOPE(DisasContext *s, X86DecodedInsn *decode) 2184{ 2185 TCGLabel *taken = gen_new_label(); 2186 TCGLabel *not_taken = gen_new_label(); 2187 2188 gen_update_cc_op(s); 2189 gen_op_add_reg_im(s, s->aflag, R_ECX, -1); 2190 gen_op_jz_ecx(s, not_taken); 2191 gen_jcc1(s, (JCC_Z << 1), taken); /* jz taken */ 2192 gen_conditional_jump_labels(s, decode->immediate, not_taken, taken); 2193} 2194 2195static void gen_LOOPNE(DisasContext *s, X86DecodedInsn *decode) 2196{ 2197 TCGLabel *taken = gen_new_label(); 2198 TCGLabel *not_taken = gen_new_label(); 2199 2200 gen_update_cc_op(s); 2201 gen_op_add_reg_im(s, s->aflag, R_ECX, -1); 2202 gen_op_jz_ecx(s, not_taken); 2203 gen_jcc1(s, (JCC_Z << 1) | 1, taken); /* jnz taken */ 2204 gen_conditional_jump_labels(s, decode->immediate, not_taken, taken); 2205} 2206 2207static void gen_LSL(DisasContext *s, X86DecodedInsn *decode) 2208{ 2209 MemOp ot = decode->op[0].ot; 2210 TCGv result = tcg_temp_new(); 2211 TCGv dest; 2212 2213 gen_compute_eflags(s); 2214 gen_update_cc_op(s); 2215 gen_helper_lsl(result, tcg_env, s->T0); 2216 2217 /* Perform writeback here to skip it if ZF=0. */ 2218 decode->op[0].unit = X86_OP_SKIP; 2219 dest = gen_op_deposit_reg_v(s, ot, decode->op[0].n, result, result); 2220 tcg_gen_movcond_tl(TCG_COND_TSTNE, dest, cpu_cc_src, tcg_constant_tl(CC_Z), 2221 result, dest); 2222} 2223 2224static void gen_LSS(DisasContext *s, X86DecodedInsn *decode) 2225{ 2226 gen_lxx_seg(s, decode, R_SS); 2227} 2228 2229static void gen_LZCNT(DisasContext *s, X86DecodedInsn *decode) 2230{ 2231 MemOp ot = decode->op[0].ot; 2232 2233 /* C bit (cc_src) is defined related to the input. */ 2234 decode->cc_src = tcg_temp_new(); 2235 decode->cc_dst = s->T0; 2236 decode->cc_op = CC_OP_BMILGB + ot; 2237 tcg_gen_mov_tl(decode->cc_src, s->T0); 2238 2239 /* 2240 * Reduce the target_ulong result by the number of zeros that 2241 * we expect to find at the top. 2242 */ 2243 tcg_gen_clzi_tl(s->T0, s->T0, TARGET_LONG_BITS); 2244 tcg_gen_subi_tl(s->T0, s->T0, TARGET_LONG_BITS - (8 << ot)); 2245} 2246 2247static void gen_MFENCE(DisasContext *s, X86DecodedInsn *decode) 2248{ 2249 tcg_gen_mb(TCG_MO_ALL | TCG_BAR_SC); 2250} 2251 2252static void gen_MOV(DisasContext *s, X86DecodedInsn *decode) 2253{ 2254 /* nothing to do! */ 2255} 2256#define gen_NOP gen_MOV 2257 2258static void gen_MASKMOV(DisasContext *s, X86DecodedInsn *decode) 2259{ 2260 gen_lea_v_seg(s, cpu_regs[R_EDI], R_DS, s->override); 2261 2262 if (s->prefix & PREFIX_DATA) { 2263 gen_helper_maskmov_xmm(tcg_env, OP_PTR1, OP_PTR2, s->A0); 2264 } else { 2265 gen_helper_maskmov_mmx(tcg_env, OP_PTR1, OP_PTR2, s->A0); 2266 } 2267} 2268 2269static void gen_MOVBE(DisasContext *s, X86DecodedInsn *decode) 2270{ 2271 MemOp ot = decode->op[0].ot; 2272 2273 /* M operand type does not load/store */ 2274 if (decode->e.op0 == X86_TYPE_M) { 2275 tcg_gen_qemu_st_tl(s->T0, s->A0, s->mem_index, ot | MO_BE); 2276 } else { 2277 tcg_gen_qemu_ld_tl(s->T0, s->A0, s->mem_index, ot | MO_BE); 2278 } 2279} 2280 2281static void gen_MOVD_from(DisasContext *s, X86DecodedInsn *decode) 2282{ 2283 MemOp ot = decode->op[2].ot; 2284 2285 switch (ot) { 2286 case MO_32: 2287#ifdef TARGET_X86_64 2288 tcg_gen_ld32u_tl(s->T0, tcg_env, decode->op[2].offset); 2289 break; 2290 case MO_64: 2291#endif 2292 tcg_gen_ld_tl(s->T0, tcg_env, decode->op[2].offset); 2293 break; 2294 default: 2295 abort(); 2296 } 2297} 2298 2299static void gen_MOVD_to(DisasContext *s, X86DecodedInsn *decode) 2300{ 2301 MemOp ot = decode->op[2].ot; 2302 int vec_len = vector_len(s, decode); 2303 int lo_ofs = vector_elem_offset(&decode->op[0], ot, 0); 2304 2305 tcg_gen_gvec_dup_imm(MO_64, decode->op[0].offset, vec_len, vec_len, 0); 2306 2307 switch (ot) { 2308 case MO_32: 2309#ifdef TARGET_X86_64 2310 tcg_gen_st32_tl(s->T1, tcg_env, lo_ofs); 2311 break; 2312 case MO_64: 2313#endif 2314 tcg_gen_st_tl(s->T1, tcg_env, lo_ofs); 2315 break; 2316 default: 2317 g_assert_not_reached(); 2318 } 2319} 2320 2321static void gen_MOVDQ(DisasContext *s, X86DecodedInsn *decode) 2322{ 2323 gen_store_sse(s, decode, decode->op[2].offset); 2324} 2325 2326static void gen_MOVMSK(DisasContext *s, X86DecodedInsn *decode) 2327{ 2328 typeof(gen_helper_movmskps_ymm) *ps, *pd, *fn; 2329 ps = s->vex_l ? gen_helper_movmskps_ymm : gen_helper_movmskps_xmm; 2330 pd = s->vex_l ? gen_helper_movmskpd_ymm : gen_helper_movmskpd_xmm; 2331 fn = s->prefix & PREFIX_DATA ? pd : ps; 2332 fn(s->tmp2_i32, tcg_env, OP_PTR2); 2333 tcg_gen_extu_i32_tl(s->T0, s->tmp2_i32); 2334} 2335 2336static void gen_MOVQ(DisasContext *s, X86DecodedInsn *decode) 2337{ 2338 int vec_len = vector_len(s, decode); 2339 int lo_ofs = vector_elem_offset(&decode->op[0], MO_64, 0); 2340 2341 tcg_gen_ld_i64(s->tmp1_i64, tcg_env, decode->op[2].offset); 2342 if (decode->op[0].has_ea) { 2343 tcg_gen_qemu_st_i64(s->tmp1_i64, s->A0, s->mem_index, MO_LEUQ); 2344 } else { 2345 /* 2346 * tcg_gen_gvec_dup_i64(MO_64, op0.offset, 8, vec_len, s->tmp1_64) would 2347 * seem to work, but it does not on big-endian platforms; the cleared parts 2348 * are always at higher addresses, but cross-endian emulation inverts the 2349 * byte order so that the cleared parts need to be at *lower* addresses. 2350 * Because oprsz is 8, we see this here even for SSE; but more in general, 2351 * it disqualifies using oprsz < maxsz to emulate VEX128. 2352 */ 2353 tcg_gen_gvec_dup_imm(MO_64, decode->op[0].offset, vec_len, vec_len, 0); 2354 tcg_gen_st_i64(s->tmp1_i64, tcg_env, lo_ofs); 2355 } 2356} 2357 2358static void gen_MOVq_dq(DisasContext *s, X86DecodedInsn *decode) 2359{ 2360 gen_helper_enter_mmx(tcg_env); 2361 /* Otherwise the same as any other movq. */ 2362 return gen_MOVQ(s, decode); 2363} 2364 2365static void gen_MOVS(DisasContext *s, X86DecodedInsn *decode) 2366{ 2367 MemOp ot = decode->op[2].ot; 2368 if (s->prefix & (PREFIX_REPZ | PREFIX_REPNZ)) { 2369 gen_repz(s, ot, gen_movs); 2370 } else { 2371 gen_movs(s, ot); 2372 } 2373} 2374 2375static void gen_MUL(DisasContext *s, X86DecodedInsn *decode) 2376{ 2377 MemOp ot = decode->op[1].ot; 2378 2379 switch (ot) { 2380 case MO_8: 2381 /* s->T0 already zero-extended */ 2382 tcg_gen_ext8u_tl(s->T1, s->T1); 2383 tcg_gen_mul_tl(s->T0, s->T0, s->T1); 2384 gen_op_mov_reg_v(s, MO_16, R_EAX, s->T0); 2385 tcg_gen_andi_tl(s->T1, s->T0, 0xff00); 2386 decode->cc_dst = s->T0; 2387 decode->cc_src = s->T1; 2388 break; 2389 2390 case MO_16: 2391 /* s->T0 already zero-extended */ 2392 tcg_gen_ext16u_tl(s->T1, s->T1); 2393 tcg_gen_mul_tl(s->T0, s->T0, s->T1); 2394 gen_op_mov_reg_v(s, MO_16, R_EAX, s->T0); 2395 tcg_gen_shri_tl(s->T1, s->T0, 16); 2396 gen_op_mov_reg_v(s, MO_16, R_EDX, s->T1); 2397 decode->cc_dst = s->T0; 2398 decode->cc_src = s->T1; 2399 break; 2400 2401 case MO_32: 2402#ifdef TARGET_X86_64 2403 /* s->T0 already zero-extended */ 2404 tcg_gen_ext32u_tl(s->T1, s->T1); 2405 tcg_gen_mul_tl(s->T0, s->T0, s->T1); 2406 tcg_gen_ext32u_tl(cpu_regs[R_EAX], s->T0); 2407 tcg_gen_shri_tl(cpu_regs[R_EDX], s->T0, 32); 2408 decode->cc_dst = cpu_regs[R_EAX]; 2409 decode->cc_src = cpu_regs[R_EDX]; 2410 break; 2411 2412 case MO_64: 2413#endif 2414 tcg_gen_mulu2_tl(cpu_regs[R_EAX], cpu_regs[R_EDX], s->T0, s->T1); 2415 decode->cc_dst = cpu_regs[R_EAX]; 2416 decode->cc_src = cpu_regs[R_EDX]; 2417 break; 2418 2419 default: 2420 g_assert_not_reached(); 2421 } 2422 2423 decode->cc_op = CC_OP_MULB + ot; 2424} 2425 2426static void gen_MULX(DisasContext *s, X86DecodedInsn *decode) 2427{ 2428 MemOp ot = decode->op[0].ot; 2429 2430 /* low part of result in VEX.vvvv, high in MODRM */ 2431 switch (ot) { 2432 case MO_32: 2433#ifdef TARGET_X86_64 2434 tcg_gen_trunc_tl_i32(s->tmp2_i32, s->T0); 2435 tcg_gen_trunc_tl_i32(s->tmp3_i32, s->T1); 2436 tcg_gen_mulu2_i32(s->tmp2_i32, s->tmp3_i32, 2437 s->tmp2_i32, s->tmp3_i32); 2438 tcg_gen_extu_i32_tl(cpu_regs[s->vex_v], s->tmp2_i32); 2439 tcg_gen_extu_i32_tl(s->T0, s->tmp3_i32); 2440 break; 2441 2442 case MO_64: 2443#endif 2444 tcg_gen_mulu2_tl(cpu_regs[s->vex_v], s->T0, s->T0, s->T1); 2445 break; 2446 2447 default: 2448 g_assert_not_reached(); 2449 } 2450} 2451 2452static void gen_NEG(DisasContext *s, X86DecodedInsn *decode) 2453{ 2454 MemOp ot = decode->op[0].ot; 2455 TCGv oldv = tcg_temp_new(); 2456 2457 if (s->prefix & PREFIX_LOCK) { 2458 TCGv newv = tcg_temp_new(); 2459 TCGv cmpv = tcg_temp_new(); 2460 TCGLabel *label1 = gen_new_label(); 2461 2462 gen_set_label(label1); 2463 gen_op_ld_v(s, ot, oldv, s->A0); 2464 tcg_gen_neg_tl(newv, oldv); 2465 tcg_gen_atomic_cmpxchg_tl(cmpv, s->A0, oldv, newv, 2466 s->mem_index, ot | MO_LE); 2467 tcg_gen_brcond_tl(TCG_COND_NE, oldv, cmpv, label1); 2468 } else { 2469 tcg_gen_mov_tl(oldv, s->T0); 2470 } 2471 tcg_gen_neg_tl(s->T0, oldv); 2472 2473 decode->cc_dst = s->T0; 2474 decode->cc_src = oldv; 2475 tcg_gen_movi_tl(s->cc_srcT, 0); 2476 decode->cc_op = CC_OP_SUBB + ot; 2477} 2478 2479static void gen_NOT(DisasContext *s, X86DecodedInsn *decode) 2480{ 2481 MemOp ot = decode->op[0].ot; 2482 2483 if (s->prefix & PREFIX_LOCK) { 2484 tcg_gen_movi_tl(s->T0, ~0); 2485 tcg_gen_atomic_xor_fetch_tl(s->T0, s->A0, s->T0, 2486 s->mem_index, ot | MO_LE); 2487 } else { 2488 tcg_gen_not_tl(s->T0, s->T0); 2489 } 2490} 2491 2492static void gen_OR(DisasContext *s, X86DecodedInsn *decode) 2493{ 2494 MemOp ot = decode->op[1].ot; 2495 2496 if (s->prefix & PREFIX_LOCK) { 2497 tcg_gen_atomic_or_fetch_tl(s->T0, s->A0, s->T1, 2498 s->mem_index, ot | MO_LE); 2499 } else { 2500 tcg_gen_or_tl(s->T0, s->T0, s->T1); 2501 } 2502 prepare_update1_cc(decode, s, CC_OP_LOGICB + ot); 2503} 2504 2505static void gen_OUT(DisasContext *s, X86DecodedInsn *decode) 2506{ 2507 MemOp ot = decode->op[1].ot; 2508 TCGv_i32 port = tcg_temp_new_i32(); 2509 TCGv_i32 value = tcg_temp_new_i32(); 2510 2511 tcg_gen_trunc_tl_i32(port, s->T1); 2512 tcg_gen_ext16u_i32(port, port); 2513 if (!gen_check_io(s, ot, port, 0)) { 2514 return; 2515 } 2516 tcg_gen_trunc_tl_i32(value, s->T0); 2517 translator_io_start(&s->base); 2518 gen_helper_out_func(ot, port, value); 2519 gen_bpt_io(s, port, ot); 2520} 2521 2522static void gen_OUTS(DisasContext *s, X86DecodedInsn *decode) 2523{ 2524 MemOp ot = decode->op[1].ot; 2525 TCGv_i32 port = tcg_temp_new_i32(); 2526 2527 tcg_gen_trunc_tl_i32(port, s->T1); 2528 tcg_gen_ext16u_i32(port, port); 2529 if (!gen_check_io(s, ot, port, SVM_IOIO_STR_MASK)) { 2530 return; 2531 } 2532 2533 translator_io_start(&s->base); 2534 if (s->prefix & (PREFIX_REPZ | PREFIX_REPNZ)) { 2535 gen_repz(s, ot, gen_outs); 2536 } else { 2537 gen_outs(s, ot); 2538 } 2539} 2540 2541static void gen_PALIGNR(DisasContext *s, X86DecodedInsn *decode) 2542{ 2543 TCGv_i32 imm = tcg_constant8u_i32(decode->immediate); 2544 if (!(s->prefix & PREFIX_DATA)) { 2545 gen_helper_palignr_mmx(tcg_env, OP_PTR0, OP_PTR1, OP_PTR2, imm); 2546 } else if (!s->vex_l) { 2547 gen_helper_palignr_xmm(tcg_env, OP_PTR0, OP_PTR1, OP_PTR2, imm); 2548 } else { 2549 gen_helper_palignr_ymm(tcg_env, OP_PTR0, OP_PTR1, OP_PTR2, imm); 2550 } 2551} 2552 2553static void gen_PANDN(DisasContext *s, X86DecodedInsn *decode) 2554{ 2555 int vec_len = vector_len(s, decode); 2556 2557 /* Careful, operand order is reversed! */ 2558 tcg_gen_gvec_andc(MO_64, 2559 decode->op[0].offset, decode->op[2].offset, 2560 decode->op[1].offset, vec_len, vec_len); 2561} 2562 2563static void gen_PAUSE(DisasContext *s, X86DecodedInsn *decode) 2564{ 2565 gen_update_cc_op(s); 2566 gen_update_eip_next(s); 2567 gen_helper_pause(tcg_env); 2568 s->base.is_jmp = DISAS_NORETURN; 2569} 2570 2571static void gen_PCMPESTRI(DisasContext *s, X86DecodedInsn *decode) 2572{ 2573 TCGv_i32 imm = tcg_constant8u_i32(decode->immediate); 2574 gen_helper_pcmpestri_xmm(tcg_env, OP_PTR1, OP_PTR2, imm); 2575 assume_cc_op(s, CC_OP_EFLAGS); 2576} 2577 2578static void gen_PCMPESTRM(DisasContext *s, X86DecodedInsn *decode) 2579{ 2580 TCGv_i32 imm = tcg_constant8u_i32(decode->immediate); 2581 gen_helper_pcmpestrm_xmm(tcg_env, OP_PTR1, OP_PTR2, imm); 2582 assume_cc_op(s, CC_OP_EFLAGS); 2583 if ((s->prefix & PREFIX_VEX) && !s->vex_l) { 2584 tcg_gen_gvec_dup_imm(MO_64, offsetof(CPUX86State, xmm_regs[0].ZMM_X(1)), 2585 16, 16, 0); 2586 } 2587} 2588 2589static void gen_PCMPISTRI(DisasContext *s, X86DecodedInsn *decode) 2590{ 2591 TCGv_i32 imm = tcg_constant8u_i32(decode->immediate); 2592 gen_helper_pcmpistri_xmm(tcg_env, OP_PTR1, OP_PTR2, imm); 2593 assume_cc_op(s, CC_OP_EFLAGS); 2594} 2595 2596static void gen_PCMPISTRM(DisasContext *s, X86DecodedInsn *decode) 2597{ 2598 TCGv_i32 imm = tcg_constant8u_i32(decode->immediate); 2599 gen_helper_pcmpistrm_xmm(tcg_env, OP_PTR1, OP_PTR2, imm); 2600 assume_cc_op(s, CC_OP_EFLAGS); 2601 if ((s->prefix & PREFIX_VEX) && !s->vex_l) { 2602 tcg_gen_gvec_dup_imm(MO_64, offsetof(CPUX86State, xmm_regs[0].ZMM_X(1)), 2603 16, 16, 0); 2604 } 2605} 2606 2607static void gen_PDEP(DisasContext *s, X86DecodedInsn *decode) 2608{ 2609 gen_helper_pdep(s->T0, s->T0, s->T1); 2610} 2611 2612static void gen_PEXT(DisasContext *s, X86DecodedInsn *decode) 2613{ 2614 gen_helper_pext(s->T0, s->T0, s->T1); 2615} 2616 2617static inline void gen_pextr(DisasContext *s, X86DecodedInsn *decode, MemOp ot) 2618{ 2619 int vec_len = vector_len(s, decode); 2620 int mask = (vec_len >> ot) - 1; 2621 int val = decode->immediate & mask; 2622 2623 switch (ot) { 2624 case MO_8: 2625 tcg_gen_ld8u_tl(s->T0, tcg_env, vector_elem_offset(&decode->op[1], ot, val)); 2626 break; 2627 case MO_16: 2628 tcg_gen_ld16u_tl(s->T0, tcg_env, vector_elem_offset(&decode->op[1], ot, val)); 2629 break; 2630 case MO_32: 2631#ifdef TARGET_X86_64 2632 tcg_gen_ld32u_tl(s->T0, tcg_env, vector_elem_offset(&decode->op[1], ot, val)); 2633 break; 2634 case MO_64: 2635#endif 2636 tcg_gen_ld_tl(s->T0, tcg_env, vector_elem_offset(&decode->op[1], ot, val)); 2637 break; 2638 default: 2639 abort(); 2640 } 2641} 2642 2643static void gen_PEXTRB(DisasContext *s, X86DecodedInsn *decode) 2644{ 2645 gen_pextr(s, decode, MO_8); 2646} 2647 2648static void gen_PEXTRW(DisasContext *s, X86DecodedInsn *decode) 2649{ 2650 gen_pextr(s, decode, MO_16); 2651} 2652 2653static void gen_PEXTR(DisasContext *s, X86DecodedInsn *decode) 2654{ 2655 MemOp ot = decode->op[0].ot; 2656 gen_pextr(s, decode, ot); 2657} 2658 2659static inline void gen_pinsr(DisasContext *s, X86DecodedInsn *decode, MemOp ot) 2660{ 2661 int vec_len = vector_len(s, decode); 2662 int mask = (vec_len >> ot) - 1; 2663 int val = decode->immediate & mask; 2664 2665 if (decode->op[1].offset != decode->op[0].offset) { 2666 assert(vec_len == 16); 2667 gen_store_sse(s, decode, decode->op[1].offset); 2668 } 2669 2670 switch (ot) { 2671 case MO_8: 2672 tcg_gen_st8_tl(s->T1, tcg_env, vector_elem_offset(&decode->op[0], ot, val)); 2673 break; 2674 case MO_16: 2675 tcg_gen_st16_tl(s->T1, tcg_env, vector_elem_offset(&decode->op[0], ot, val)); 2676 break; 2677 case MO_32: 2678#ifdef TARGET_X86_64 2679 tcg_gen_st32_tl(s->T1, tcg_env, vector_elem_offset(&decode->op[0], ot, val)); 2680 break; 2681 case MO_64: 2682#endif 2683 tcg_gen_st_tl(s->T1, tcg_env, vector_elem_offset(&decode->op[0], ot, val)); 2684 break; 2685 default: 2686 abort(); 2687 } 2688} 2689 2690static void gen_PINSRB(DisasContext *s, X86DecodedInsn *decode) 2691{ 2692 gen_pinsr(s, decode, MO_8); 2693} 2694 2695static void gen_PINSRW(DisasContext *s, X86DecodedInsn *decode) 2696{ 2697 gen_pinsr(s, decode, MO_16); 2698} 2699 2700static void gen_PINSR(DisasContext *s, X86DecodedInsn *decode) 2701{ 2702 gen_pinsr(s, decode, decode->op[2].ot); 2703} 2704 2705static void gen_pmovmskb_i64(TCGv_i64 d, TCGv_i64 s) 2706{ 2707 TCGv_i64 t = tcg_temp_new_i64(); 2708 2709 tcg_gen_andi_i64(d, s, 0x8080808080808080ull); 2710 2711 /* 2712 * After each shift+or pair: 2713 * 0: a.......b.......c.......d.......e.......f.......g.......h....... 2714 * 7: ab......bc......cd......de......ef......fg......gh......h....... 2715 * 14: abcd....bcde....cdef....defg....efgh....fgh.....gh......h....... 2716 * 28: abcdefghbcdefgh.cdefgh..defgh...efgh....fgh.....gh......h....... 2717 * The result is left in the high bits of the word. 2718 */ 2719 tcg_gen_shli_i64(t, d, 7); 2720 tcg_gen_or_i64(d, d, t); 2721 tcg_gen_shli_i64(t, d, 14); 2722 tcg_gen_or_i64(d, d, t); 2723 tcg_gen_shli_i64(t, d, 28); 2724 tcg_gen_or_i64(d, d, t); 2725} 2726 2727static void gen_pmovmskb_vec(unsigned vece, TCGv_vec d, TCGv_vec s) 2728{ 2729 TCGv_vec t = tcg_temp_new_vec_matching(d); 2730 TCGv_vec m = tcg_constant_vec_matching(d, MO_8, 0x80); 2731 2732 /* See above */ 2733 tcg_gen_and_vec(vece, d, s, m); 2734 tcg_gen_shli_vec(vece, t, d, 7); 2735 tcg_gen_or_vec(vece, d, d, t); 2736 tcg_gen_shli_vec(vece, t, d, 14); 2737 tcg_gen_or_vec(vece, d, d, t); 2738 tcg_gen_shli_vec(vece, t, d, 28); 2739 tcg_gen_or_vec(vece, d, d, t); 2740} 2741 2742static void gen_PMOVMSKB(DisasContext *s, X86DecodedInsn *decode) 2743{ 2744 static const TCGOpcode vecop_list[] = { INDEX_op_shli_vec, 0 }; 2745 static const GVecGen2 g = { 2746 .fni8 = gen_pmovmskb_i64, 2747 .fniv = gen_pmovmskb_vec, 2748 .opt_opc = vecop_list, 2749 .vece = MO_64, 2750 .prefer_i64 = TCG_TARGET_REG_BITS == 64 2751 }; 2752 MemOp ot = decode->op[2].ot; 2753 int vec_len = vector_len(s, decode); 2754 TCGv t = tcg_temp_new(); 2755 2756 tcg_gen_gvec_2(offsetof(CPUX86State, xmm_t0) + xmm_offset(ot), decode->op[2].offset, 2757 vec_len, vec_len, &g); 2758 tcg_gen_ld8u_tl(s->T0, tcg_env, offsetof(CPUX86State, xmm_t0.ZMM_B(vec_len - 1))); 2759 while (vec_len > 8) { 2760 vec_len -= 8; 2761 if (TCG_TARGET_HAS_extract2_tl) { 2762 /* 2763 * Load the next byte of the result into the high byte of T. 2764 * TCG does a similar expansion of deposit to shl+extract2; by 2765 * loading the whole word, the shift left is avoided. 2766 */ 2767#ifdef TARGET_X86_64 2768 tcg_gen_ld_tl(t, tcg_env, offsetof(CPUX86State, xmm_t0.ZMM_Q((vec_len - 1) / 8))); 2769#else 2770 tcg_gen_ld_tl(t, tcg_env, offsetof(CPUX86State, xmm_t0.ZMM_L((vec_len - 1) / 4))); 2771#endif 2772 2773 tcg_gen_extract2_tl(s->T0, t, s->T0, TARGET_LONG_BITS - 8); 2774 } else { 2775 /* 2776 * The _previous_ value is deposited into bits 8 and higher of t. Because 2777 * those bits are known to be zero after ld8u, this becomes a shift+or 2778 * if deposit is not available. 2779 */ 2780 tcg_gen_ld8u_tl(t, tcg_env, offsetof(CPUX86State, xmm_t0.ZMM_B(vec_len - 1))); 2781 tcg_gen_deposit_tl(s->T0, t, s->T0, 8, TARGET_LONG_BITS - 8); 2782 } 2783 } 2784} 2785 2786static void gen_POP(DisasContext *s, X86DecodedInsn *decode) 2787{ 2788 X86DecodedOp *op = &decode->op[0]; 2789 MemOp ot = gen_pop_T0(s); 2790 2791 assert(ot >= op->ot); 2792 if (op->has_ea || op->unit == X86_OP_SEG) { 2793 /* NOTE: order is important for MMU exceptions */ 2794 gen_writeback(s, decode, 0, s->T0); 2795 } 2796 2797 /* NOTE: writing back registers after update is important for pop %sp */ 2798 gen_pop_update(s, ot); 2799} 2800 2801static void gen_POPA(DisasContext *s, X86DecodedInsn *decode) 2802{ 2803 gen_popa(s); 2804} 2805 2806static void gen_POPCNT(DisasContext *s, X86DecodedInsn *decode) 2807{ 2808 decode->cc_dst = tcg_temp_new(); 2809 decode->cc_op = CC_OP_POPCNT; 2810 2811 tcg_gen_mov_tl(decode->cc_dst, s->T0); 2812 tcg_gen_ctpop_tl(s->T0, s->T0); 2813} 2814 2815static void gen_POPF(DisasContext *s, X86DecodedInsn *decode) 2816{ 2817 MemOp ot; 2818 int mask = TF_MASK | AC_MASK | ID_MASK | NT_MASK; 2819 2820 if (CPL(s) == 0) { 2821 mask |= IF_MASK | IOPL_MASK; 2822 } else if (CPL(s) <= IOPL(s)) { 2823 mask |= IF_MASK; 2824 } 2825 if (s->dflag == MO_16) { 2826 mask &= 0xffff; 2827 } 2828 2829 ot = gen_pop_T0(s); 2830 gen_helper_write_eflags(tcg_env, s->T0, tcg_constant_i32(mask)); 2831 gen_pop_update(s, ot); 2832 set_cc_op(s, CC_OP_EFLAGS); 2833 /* abort translation because TF/AC flag may change */ 2834 s->base.is_jmp = DISAS_EOB_NEXT; 2835} 2836 2837static void gen_PSHUFW(DisasContext *s, X86DecodedInsn *decode) 2838{ 2839 TCGv_i32 imm = tcg_constant8u_i32(decode->immediate); 2840 gen_helper_pshufw_mmx(OP_PTR0, OP_PTR1, imm); 2841} 2842 2843static void gen_PSRLW_i(DisasContext *s, X86DecodedInsn *decode) 2844{ 2845 int vec_len = vector_len(s, decode); 2846 2847 if (decode->immediate >= 16) { 2848 tcg_gen_gvec_dup_imm(MO_64, decode->op[0].offset, vec_len, vec_len, 0); 2849 } else { 2850 tcg_gen_gvec_shri(MO_16, 2851 decode->op[0].offset, decode->op[1].offset, 2852 decode->immediate, vec_len, vec_len); 2853 } 2854} 2855 2856static void gen_PSLLW_i(DisasContext *s, X86DecodedInsn *decode) 2857{ 2858 int vec_len = vector_len(s, decode); 2859 2860 if (decode->immediate >= 16) { 2861 tcg_gen_gvec_dup_imm(MO_64, decode->op[0].offset, vec_len, vec_len, 0); 2862 } else { 2863 tcg_gen_gvec_shli(MO_16, 2864 decode->op[0].offset, decode->op[1].offset, 2865 decode->immediate, vec_len, vec_len); 2866 } 2867} 2868 2869static void gen_PSRAW_i(DisasContext *s, X86DecodedInsn *decode) 2870{ 2871 int vec_len = vector_len(s, decode); 2872 2873 if (decode->immediate >= 16) { 2874 decode->immediate = 15; 2875 } 2876 tcg_gen_gvec_sari(MO_16, 2877 decode->op[0].offset, decode->op[1].offset, 2878 decode->immediate, vec_len, vec_len); 2879} 2880 2881static void gen_PSRLD_i(DisasContext *s, X86DecodedInsn *decode) 2882{ 2883 int vec_len = vector_len(s, decode); 2884 2885 if (decode->immediate >= 32) { 2886 tcg_gen_gvec_dup_imm(MO_64, decode->op[0].offset, vec_len, vec_len, 0); 2887 } else { 2888 tcg_gen_gvec_shri(MO_32, 2889 decode->op[0].offset, decode->op[1].offset, 2890 decode->immediate, vec_len, vec_len); 2891 } 2892} 2893 2894static void gen_PSLLD_i(DisasContext *s, X86DecodedInsn *decode) 2895{ 2896 int vec_len = vector_len(s, decode); 2897 2898 if (decode->immediate >= 32) { 2899 tcg_gen_gvec_dup_imm(MO_64, decode->op[0].offset, vec_len, vec_len, 0); 2900 } else { 2901 tcg_gen_gvec_shli(MO_32, 2902 decode->op[0].offset, decode->op[1].offset, 2903 decode->immediate, vec_len, vec_len); 2904 } 2905} 2906 2907static void gen_PSRAD_i(DisasContext *s, X86DecodedInsn *decode) 2908{ 2909 int vec_len = vector_len(s, decode); 2910 2911 if (decode->immediate >= 32) { 2912 decode->immediate = 31; 2913 } 2914 tcg_gen_gvec_sari(MO_32, 2915 decode->op[0].offset, decode->op[1].offset, 2916 decode->immediate, vec_len, vec_len); 2917} 2918 2919static void gen_PSRLQ_i(DisasContext *s, X86DecodedInsn *decode) 2920{ 2921 int vec_len = vector_len(s, decode); 2922 2923 if (decode->immediate >= 64) { 2924 tcg_gen_gvec_dup_imm(MO_64, decode->op[0].offset, vec_len, vec_len, 0); 2925 } else { 2926 tcg_gen_gvec_shri(MO_64, 2927 decode->op[0].offset, decode->op[1].offset, 2928 decode->immediate, vec_len, vec_len); 2929 } 2930} 2931 2932static void gen_PSLLQ_i(DisasContext *s, X86DecodedInsn *decode) 2933{ 2934 int vec_len = vector_len(s, decode); 2935 2936 if (decode->immediate >= 64) { 2937 tcg_gen_gvec_dup_imm(MO_64, decode->op[0].offset, vec_len, vec_len, 0); 2938 } else { 2939 tcg_gen_gvec_shli(MO_64, 2940 decode->op[0].offset, decode->op[1].offset, 2941 decode->immediate, vec_len, vec_len); 2942 } 2943} 2944 2945static TCGv_ptr make_imm8u_xmm_vec(uint8_t imm, int vec_len) 2946{ 2947 MemOp ot = vec_len == 16 ? MO_128 : MO_256; 2948 TCGv_i32 imm_v = tcg_constant8u_i32(imm); 2949 TCGv_ptr ptr = tcg_temp_new_ptr(); 2950 2951 tcg_gen_gvec_dup_imm(MO_64, offsetof(CPUX86State, xmm_t0) + xmm_offset(ot), 2952 vec_len, vec_len, 0); 2953 2954 tcg_gen_addi_ptr(ptr, tcg_env, offsetof(CPUX86State, xmm_t0)); 2955 tcg_gen_st_i32(imm_v, tcg_env, offsetof(CPUX86State, xmm_t0.ZMM_L(0))); 2956 return ptr; 2957} 2958 2959static void gen_PSRLDQ_i(DisasContext *s, X86DecodedInsn *decode) 2960{ 2961 int vec_len = vector_len(s, decode); 2962 TCGv_ptr imm_vec = make_imm8u_xmm_vec(decode->immediate, vec_len); 2963 2964 if (s->vex_l) { 2965 gen_helper_psrldq_ymm(tcg_env, OP_PTR0, OP_PTR1, imm_vec); 2966 } else { 2967 gen_helper_psrldq_xmm(tcg_env, OP_PTR0, OP_PTR1, imm_vec); 2968 } 2969} 2970 2971static void gen_PSLLDQ_i(DisasContext *s, X86DecodedInsn *decode) 2972{ 2973 int vec_len = vector_len(s, decode); 2974 TCGv_ptr imm_vec = make_imm8u_xmm_vec(decode->immediate, vec_len); 2975 2976 if (s->vex_l) { 2977 gen_helper_pslldq_ymm(tcg_env, OP_PTR0, OP_PTR1, imm_vec); 2978 } else { 2979 gen_helper_pslldq_xmm(tcg_env, OP_PTR0, OP_PTR1, imm_vec); 2980 } 2981} 2982 2983static void gen_PUSH(DisasContext *s, X86DecodedInsn *decode) 2984{ 2985 gen_push_v(s, s->T0); 2986} 2987 2988static void gen_PUSHA(DisasContext *s, X86DecodedInsn *decode) 2989{ 2990 gen_pusha(s); 2991} 2992 2993static void gen_PUSHF(DisasContext *s, X86DecodedInsn *decode) 2994{ 2995 gen_update_cc_op(s); 2996 gen_helper_read_eflags(s->T0, tcg_env); 2997 gen_push_v(s, s->T0); 2998} 2999 3000static MemOp gen_shift_count(DisasContext *s, X86DecodedInsn *decode, 3001 bool *can_be_zero, TCGv *count, int unit) 3002{ 3003 MemOp ot = decode->op[0].ot; 3004 int mask = (ot <= MO_32 ? 0x1f : 0x3f); 3005 3006 *can_be_zero = false; 3007 switch (unit) { 3008 case X86_OP_INT: 3009 *count = tcg_temp_new(); 3010 tcg_gen_andi_tl(*count, cpu_regs[R_ECX], mask); 3011 *can_be_zero = true; 3012 break; 3013 3014 case X86_OP_IMM: 3015 if ((decode->immediate & mask) == 0) { 3016 *count = NULL; 3017 break; 3018 } 3019 *count = tcg_temp_new(); 3020 tcg_gen_movi_tl(*count, decode->immediate & mask); 3021 break; 3022 3023 case X86_OP_SKIP: 3024 *count = tcg_temp_new(); 3025 tcg_gen_movi_tl(*count, 1); 3026 break; 3027 3028 default: 3029 g_assert_not_reached(); 3030 } 3031 3032 return ot; 3033} 3034 3035/* 3036 * Compute existing flags in decode->cc_src, for gen_* functions that wants 3037 * to set the cc_op set to CC_OP_ADCOX. In particular, this allows rotate 3038 * operations to compute the carry in decode->cc_dst and the overflow in 3039 * decode->cc_src2. 3040 * 3041 * If need_flags is true, decode->cc_dst and decode->cc_src2 are preloaded 3042 * with the value of CF and OF before the instruction, so that it is possible 3043 * to keep the flags unmodified. 3044 * 3045 * Return true if carry could be made available cheaply as a 1-bit value in 3046 * decode->cc_dst (trying a bit harder if want_carry is true). If false is 3047 * returned, decode->cc_dst is uninitialized and the carry is only available 3048 * as bit 0 of decode->cc_src. 3049 */ 3050static bool gen_eflags_adcox(DisasContext *s, X86DecodedInsn *decode, bool want_carry, bool need_flags) 3051{ 3052 bool got_cf = false; 3053 bool got_of = false; 3054 3055 decode->cc_dst = tcg_temp_new(); 3056 decode->cc_src = tcg_temp_new(); 3057 decode->cc_src2 = tcg_temp_new(); 3058 decode->cc_op = CC_OP_ADCOX; 3059 3060 /* A lot more cc_ops could be "optimized" to avoid the extracts at 3061 * the end (INC/DEC, BMILG, MUL), but they are all really unlikely 3062 * to be followed by rotations within the same basic block. 3063 */ 3064 switch (s->cc_op) { 3065 case CC_OP_ADCOX: 3066 /* No need to compute the full EFLAGS, CF/OF are already isolated. */ 3067 tcg_gen_mov_tl(decode->cc_src, cpu_cc_src); 3068 if (need_flags) { 3069 tcg_gen_mov_tl(decode->cc_src2, cpu_cc_src2); 3070 got_of = true; 3071 } 3072 if (want_carry || need_flags) { 3073 tcg_gen_mov_tl(decode->cc_dst, cpu_cc_dst); 3074 got_cf = true; 3075 } 3076 break; 3077 3078 case CC_OP_LOGICB ... CC_OP_LOGICQ: 3079 /* CF and OF are zero, do it just because it's easy. */ 3080 gen_mov_eflags(s, decode->cc_src); 3081 if (need_flags) { 3082 tcg_gen_movi_tl(decode->cc_src2, 0); 3083 got_of = true; 3084 } 3085 if (want_carry || need_flags) { 3086 tcg_gen_movi_tl(decode->cc_dst, 0); 3087 got_cf = true; 3088 } 3089 break; 3090 3091 case CC_OP_SARB ... CC_OP_SARQ: 3092 /* 3093 * SHR/RCR/SHR/RCR/... is a relatively common occurrence of RCR. 3094 * By computing CF without using eflags, the calls to cc_compute_all 3095 * can be eliminated as dead code (except for the last RCR). 3096 */ 3097 if (want_carry || need_flags) { 3098 tcg_gen_andi_tl(decode->cc_dst, cpu_cc_src, 1); 3099 got_cf = true; 3100 } 3101 gen_mov_eflags(s, decode->cc_src); 3102 break; 3103 3104 case CC_OP_SHLB ... CC_OP_SHLQ: 3105 /* 3106 * Likewise for SHL/RCL/SHL/RCL/... but, if CF is not in the sign 3107 * bit, we might as well fish CF out of EFLAGS and save a shift. 3108 */ 3109 if (want_carry && (!need_flags || s->cc_op == CC_OP_SHLB + MO_TL)) { 3110 tcg_gen_shri_tl(decode->cc_dst, cpu_cc_src, (8 << (s->cc_op - CC_OP_SHLB)) - 1); 3111 got_cf = true; 3112 } 3113 gen_mov_eflags(s, decode->cc_src); 3114 break; 3115 3116 default: 3117 gen_mov_eflags(s, decode->cc_src); 3118 break; 3119 } 3120 3121 if (need_flags) { 3122 /* If the flags could be left unmodified, always load them. */ 3123 if (!got_of) { 3124 tcg_gen_extract_tl(decode->cc_src2, decode->cc_src, ctz32(CC_O), 1); 3125 got_of = true; 3126 } 3127 if (!got_cf) { 3128 tcg_gen_extract_tl(decode->cc_dst, decode->cc_src, ctz32(CC_C), 1); 3129 got_cf = true; 3130 } 3131 } 3132 return got_cf; 3133} 3134 3135static void gen_rot_overflow(X86DecodedInsn *decode, TCGv result, TCGv old, 3136 bool can_be_zero, TCGv count) 3137{ 3138 MemOp ot = decode->op[0].ot; 3139 TCGv temp = can_be_zero ? tcg_temp_new() : decode->cc_src2; 3140 3141 tcg_gen_xor_tl(temp, old, result); 3142 tcg_gen_extract_tl(temp, temp, (8 << ot) - 1, 1); 3143 if (can_be_zero) { 3144 tcg_gen_movcond_tl(TCG_COND_EQ, decode->cc_src2, count, tcg_constant_tl(0), 3145 decode->cc_src2, temp); 3146 } 3147} 3148 3149/* 3150 * RCx operations are invariant modulo 8*operand_size+1. For 8 and 16-bit operands, 3151 * this is less than 0x1f (the mask applied by gen_shift_count) so reduce further. 3152 */ 3153static void gen_rotc_mod(MemOp ot, TCGv count) 3154{ 3155 TCGv temp; 3156 3157 switch (ot) { 3158 case MO_8: 3159 temp = tcg_temp_new(); 3160 tcg_gen_subi_tl(temp, count, 18); 3161 tcg_gen_movcond_tl(TCG_COND_GE, count, temp, tcg_constant_tl(0), temp, count); 3162 tcg_gen_subi_tl(temp, count, 9); 3163 tcg_gen_movcond_tl(TCG_COND_GE, count, temp, tcg_constant_tl(0), temp, count); 3164 break; 3165 3166 case MO_16: 3167 temp = tcg_temp_new(); 3168 tcg_gen_subi_tl(temp, count, 17); 3169 tcg_gen_movcond_tl(TCG_COND_GE, count, temp, tcg_constant_tl(0), temp, count); 3170 break; 3171 3172 default: 3173 break; 3174 } 3175} 3176 3177/* 3178 * The idea here is that the bit to the right of the new bit 0 is the 3179 * new carry, and the bit to the right of the old bit 0 is the old carry. 3180 * Just like a regular rotation, the result of the rotation is composed 3181 * from a right shifted part and a left shifted part of s->T0. The new carry 3182 * is extracted from the right-shifted portion, and the old carry is 3183 * inserted at the end of the left-shifted portion. 3184 * 3185 * Because of the separate shifts involving the carry, gen_RCL and gen_RCR 3186 * mostly operate on count-1. This also comes in handy when computing 3187 * length - count, because (length-1) - (count-1) can be computed with 3188 * a XOR, and that is commutative unlike subtraction. 3189 */ 3190static void gen_RCL(DisasContext *s, X86DecodedInsn *decode) 3191{ 3192 bool have_1bit_cin, can_be_zero; 3193 TCGv count; 3194 TCGLabel *zero_label = NULL; 3195 MemOp ot = gen_shift_count(s, decode, &can_be_zero, &count, decode->op[2].unit); 3196 TCGv low, high, low_count; 3197 3198 if (!count) { 3199 return; 3200 } 3201 3202 low = tcg_temp_new(); 3203 high = tcg_temp_new(); 3204 low_count = tcg_temp_new(); 3205 3206 gen_rotc_mod(ot, count); 3207 have_1bit_cin = gen_eflags_adcox(s, decode, true, can_be_zero); 3208 if (can_be_zero) { 3209 zero_label = gen_new_label(); 3210 tcg_gen_brcondi_tl(TCG_COND_EQ, count, 0, zero_label); 3211 } 3212 3213 /* Compute high part, including incoming carry. */ 3214 if (!have_1bit_cin || TCG_TARGET_deposit_tl_valid(1, TARGET_LONG_BITS - 1)) { 3215 /* high = (T0 << 1) | cin */ 3216 TCGv cin = have_1bit_cin ? decode->cc_dst : decode->cc_src; 3217 tcg_gen_deposit_tl(high, cin, s->T0, 1, TARGET_LONG_BITS - 1); 3218 } else { 3219 /* Same as above but without deposit; cin in cc_dst. */ 3220 tcg_gen_add_tl(high, s->T0, decode->cc_dst); 3221 tcg_gen_add_tl(high, high, s->T0); 3222 } 3223 tcg_gen_subi_tl(count, count, 1); 3224 tcg_gen_shl_tl(high, high, count); 3225 3226 /* Compute low part and outgoing carry, incoming s->T0 is zero extended */ 3227 tcg_gen_xori_tl(low_count, count, (8 << ot) - 1); /* LENGTH - 1 - (count - 1) */ 3228 tcg_gen_shr_tl(low, s->T0, low_count); 3229 tcg_gen_andi_tl(decode->cc_dst, low, 1); 3230 tcg_gen_shri_tl(low, low, 1); 3231 3232 /* Compute result and outgoing overflow */ 3233 tcg_gen_mov_tl(decode->cc_src2, s->T0); 3234 tcg_gen_or_tl(s->T0, low, high); 3235 gen_rot_overflow(decode, s->T0, decode->cc_src2, false, NULL); 3236 3237 if (zero_label) { 3238 gen_set_label(zero_label); 3239 } 3240} 3241 3242static void gen_RCR(DisasContext *s, X86DecodedInsn *decode) 3243{ 3244 bool have_1bit_cin, can_be_zero; 3245 TCGv count; 3246 TCGLabel *zero_label = NULL; 3247 MemOp ot = gen_shift_count(s, decode, &can_be_zero, &count, decode->op[2].unit); 3248 TCGv low, high, high_count; 3249 3250 if (!count) { 3251 return; 3252 } 3253 3254 low = tcg_temp_new(); 3255 high = tcg_temp_new(); 3256 high_count = tcg_temp_new(); 3257 3258 gen_rotc_mod(ot, count); 3259 have_1bit_cin = gen_eflags_adcox(s, decode, true, can_be_zero); 3260 if (can_be_zero) { 3261 zero_label = gen_new_label(); 3262 tcg_gen_brcondi_tl(TCG_COND_EQ, count, 0, zero_label); 3263 } 3264 3265 /* Save incoming carry into high, it will be shifted later. */ 3266 if (!have_1bit_cin || TCG_TARGET_deposit_tl_valid(1, TARGET_LONG_BITS - 1)) { 3267 TCGv cin = have_1bit_cin ? decode->cc_dst : decode->cc_src; 3268 tcg_gen_deposit_tl(high, cin, s->T0, 1, TARGET_LONG_BITS - 1); 3269 } else { 3270 /* Same as above but without deposit; cin in cc_dst. */ 3271 tcg_gen_add_tl(high, s->T0, decode->cc_dst); 3272 tcg_gen_add_tl(high, high, s->T0); 3273 } 3274 3275 /* Compute low part and outgoing carry, incoming s->T0 is zero extended */ 3276 tcg_gen_subi_tl(count, count, 1); 3277 tcg_gen_shr_tl(low, s->T0, count); 3278 tcg_gen_andi_tl(decode->cc_dst, low, 1); 3279 tcg_gen_shri_tl(low, low, 1); 3280 3281 /* Move high part to the right position */ 3282 tcg_gen_xori_tl(high_count, count, (8 << ot) - 1); /* LENGTH - 1 - (count - 1) */ 3283 tcg_gen_shl_tl(high, high, high_count); 3284 3285 /* Compute result and outgoing overflow */ 3286 tcg_gen_mov_tl(decode->cc_src2, s->T0); 3287 tcg_gen_or_tl(s->T0, low, high); 3288 gen_rot_overflow(decode, s->T0, decode->cc_src2, false, NULL); 3289 3290 if (zero_label) { 3291 gen_set_label(zero_label); 3292 } 3293} 3294 3295#ifdef CONFIG_USER_ONLY 3296static void gen_unreachable(DisasContext *s, X86DecodedInsn *decode) 3297{ 3298 g_assert_not_reached(); 3299} 3300#endif 3301 3302#ifndef CONFIG_USER_ONLY 3303static void gen_RDMSR(DisasContext *s, X86DecodedInsn *decode) 3304{ 3305 gen_update_cc_op(s); 3306 gen_update_eip_cur(s); 3307 gen_helper_rdmsr(tcg_env); 3308} 3309#else 3310#define gen_RDMSR gen_unreachable 3311#endif 3312 3313static void gen_RDPMC(DisasContext *s, X86DecodedInsn *decode) 3314{ 3315 gen_update_cc_op(s); 3316 gen_update_eip_cur(s); 3317 translator_io_start(&s->base); 3318 gen_helper_rdpmc(tcg_env); 3319 s->base.is_jmp = DISAS_NORETURN; 3320} 3321 3322static void gen_RDTSC(DisasContext *s, X86DecodedInsn *decode) 3323{ 3324 gen_update_cc_op(s); 3325 gen_update_eip_cur(s); 3326 translator_io_start(&s->base); 3327 gen_helper_rdtsc(tcg_env); 3328} 3329 3330static void gen_RDxxBASE(DisasContext *s, X86DecodedInsn *decode) 3331{ 3332 TCGv base = cpu_seg_base[s->modrm & 8 ? R_GS : R_FS]; 3333 3334 /* Preserve hflags bits by testing CR4 at runtime. */ 3335 gen_helper_cr4_testbit(tcg_env, tcg_constant_i32(CR4_FSGSBASE_MASK)); 3336 tcg_gen_mov_tl(s->T0, base); 3337} 3338 3339static void gen_RET(DisasContext *s, X86DecodedInsn *decode) 3340{ 3341 int16_t adjust = decode->e.op1 == X86_TYPE_I ? decode->immediate : 0; 3342 3343 MemOp ot = gen_pop_T0(s); 3344 gen_stack_update(s, adjust + (1 << ot)); 3345 gen_op_jmp_v(s, s->T0); 3346 gen_bnd_jmp(s); 3347 s->base.is_jmp = DISAS_JUMP; 3348} 3349 3350static void gen_RETF(DisasContext *s, X86DecodedInsn *decode) 3351{ 3352 int16_t adjust = decode->e.op1 == X86_TYPE_I ? decode->immediate : 0; 3353 3354 if (!PE(s) || VM86(s)) { 3355 gen_lea_ss_ofs(s, s->A0, cpu_regs[R_ESP], 0); 3356 /* pop offset */ 3357 gen_op_ld_v(s, s->dflag, s->T0, s->A0); 3358 /* NOTE: keeping EIP updated is not a problem in case of 3359 exception */ 3360 gen_op_jmp_v(s, s->T0); 3361 /* pop selector */ 3362 gen_add_A0_im(s, 1 << s->dflag); 3363 gen_op_ld_v(s, s->dflag, s->T0, s->A0); 3364 gen_op_movl_seg_real(s, R_CS, s->T0); 3365 /* add stack offset */ 3366 gen_stack_update(s, adjust + (2 << s->dflag)); 3367 } else { 3368 gen_update_cc_op(s); 3369 gen_update_eip_cur(s); 3370 gen_helper_lret_protected(tcg_env, tcg_constant_i32(s->dflag - 1), 3371 tcg_constant_i32(adjust)); 3372 } 3373 s->base.is_jmp = DISAS_EOB_ONLY; 3374} 3375 3376/* 3377 * Return non-NULL if a 32-bit rotate works, after possibly replicating the input. 3378 * The input has already been zero-extended upon operand decode. 3379 */ 3380static TCGv_i32 gen_rot_replicate(MemOp ot, TCGv in) 3381{ 3382 TCGv_i32 temp; 3383 switch (ot) { 3384 case MO_8: 3385 temp = tcg_temp_new_i32(); 3386 tcg_gen_trunc_tl_i32(temp, in); 3387 tcg_gen_muli_i32(temp, temp, 0x01010101); 3388 return temp; 3389 3390 case MO_16: 3391 temp = tcg_temp_new_i32(); 3392 tcg_gen_trunc_tl_i32(temp, in); 3393 tcg_gen_deposit_i32(temp, temp, temp, 16, 16); 3394 return temp; 3395 3396#ifdef TARGET_X86_64 3397 case MO_32: 3398 temp = tcg_temp_new_i32(); 3399 tcg_gen_trunc_tl_i32(temp, in); 3400 return temp; 3401#endif 3402 3403 default: 3404 return NULL; 3405 } 3406} 3407 3408static void gen_rot_carry(X86DecodedInsn *decode, TCGv result, 3409 bool can_be_zero, TCGv count, int bit) 3410{ 3411 if (!can_be_zero) { 3412 tcg_gen_extract_tl(decode->cc_dst, result, bit, 1); 3413 } else { 3414 TCGv temp = tcg_temp_new(); 3415 tcg_gen_extract_tl(temp, result, bit, 1); 3416 tcg_gen_movcond_tl(TCG_COND_EQ, decode->cc_dst, count, tcg_constant_tl(0), 3417 decode->cc_dst, temp); 3418 } 3419} 3420 3421static void gen_ROL(DisasContext *s, X86DecodedInsn *decode) 3422{ 3423 bool can_be_zero; 3424 TCGv count; 3425 MemOp ot = gen_shift_count(s, decode, &can_be_zero, &count, decode->op[2].unit); 3426 TCGv_i32 temp32, count32; 3427 TCGv old = tcg_temp_new(); 3428 3429 if (!count) { 3430 return; 3431 } 3432 3433 gen_eflags_adcox(s, decode, false, can_be_zero); 3434 tcg_gen_mov_tl(old, s->T0); 3435 temp32 = gen_rot_replicate(ot, s->T0); 3436 if (temp32) { 3437 count32 = tcg_temp_new_i32(); 3438 tcg_gen_trunc_tl_i32(count32, count); 3439 tcg_gen_rotl_i32(temp32, temp32, count32); 3440 /* Zero extend to facilitate later optimization. */ 3441 tcg_gen_extu_i32_tl(s->T0, temp32); 3442 } else { 3443 tcg_gen_rotl_tl(s->T0, s->T0, count); 3444 } 3445 gen_rot_carry(decode, s->T0, can_be_zero, count, 0); 3446 gen_rot_overflow(decode, s->T0, old, can_be_zero, count); 3447} 3448 3449static void gen_ROR(DisasContext *s, X86DecodedInsn *decode) 3450{ 3451 bool can_be_zero; 3452 TCGv count; 3453 MemOp ot = gen_shift_count(s, decode, &can_be_zero, &count, decode->op[2].unit); 3454 TCGv_i32 temp32, count32; 3455 TCGv old = tcg_temp_new(); 3456 3457 if (!count) { 3458 return; 3459 } 3460 3461 gen_eflags_adcox(s, decode, false, can_be_zero); 3462 tcg_gen_mov_tl(old, s->T0); 3463 temp32 = gen_rot_replicate(ot, s->T0); 3464 if (temp32) { 3465 count32 = tcg_temp_new_i32(); 3466 tcg_gen_trunc_tl_i32(count32, count); 3467 tcg_gen_rotr_i32(temp32, temp32, count32); 3468 /* Zero extend to facilitate later optimization. */ 3469 tcg_gen_extu_i32_tl(s->T0, temp32); 3470 gen_rot_carry(decode, s->T0, can_be_zero, count, 31); 3471 } else { 3472 tcg_gen_rotr_tl(s->T0, s->T0, count); 3473 gen_rot_carry(decode, s->T0, can_be_zero, count, TARGET_LONG_BITS - 1); 3474 } 3475 gen_rot_overflow(decode, s->T0, old, can_be_zero, count); 3476} 3477 3478static void gen_RORX(DisasContext *s, X86DecodedInsn *decode) 3479{ 3480 MemOp ot = decode->op[0].ot; 3481 int mask = ot == MO_64 ? 63 : 31; 3482 int b = decode->immediate & mask; 3483 3484 switch (ot) { 3485 case MO_32: 3486#ifdef TARGET_X86_64 3487 tcg_gen_trunc_tl_i32(s->tmp2_i32, s->T0); 3488 tcg_gen_rotri_i32(s->tmp2_i32, s->tmp2_i32, b); 3489 tcg_gen_extu_i32_tl(s->T0, s->tmp2_i32); 3490 break; 3491 3492 case MO_64: 3493#endif 3494 tcg_gen_rotri_tl(s->T0, s->T0, b); 3495 break; 3496 3497 default: 3498 g_assert_not_reached(); 3499 } 3500} 3501 3502#ifndef CONFIG_USER_ONLY 3503static void gen_RSM(DisasContext *s, X86DecodedInsn *decode) 3504{ 3505 gen_helper_rsm(tcg_env); 3506 assume_cc_op(s, CC_OP_EFLAGS); 3507 s->base.is_jmp = DISAS_EOB_ONLY; 3508} 3509#else 3510#define gen_RSM gen_UD 3511#endif 3512 3513static void gen_SAHF(DisasContext *s, X86DecodedInsn *decode) 3514{ 3515 if (CODE64(s) && !(s->cpuid_ext3_features & CPUID_EXT3_LAHF_LM)) { 3516 return gen_illegal_opcode(s); 3517 } 3518 tcg_gen_shri_tl(s->T0, cpu_regs[R_EAX], 8); 3519 gen_compute_eflags(s); 3520 tcg_gen_andi_tl(cpu_cc_src, cpu_cc_src, CC_O); 3521 tcg_gen_andi_tl(s->T0, s->T0, CC_S | CC_Z | CC_A | CC_P | CC_C); 3522 tcg_gen_or_tl(cpu_cc_src, cpu_cc_src, s->T0); 3523} 3524 3525static void gen_SALC(DisasContext *s, X86DecodedInsn *decode) 3526{ 3527 gen_compute_eflags_c(s, s->T0); 3528 tcg_gen_neg_tl(s->T0, s->T0); 3529} 3530 3531static void gen_shift_dynamic_flags(DisasContext *s, X86DecodedInsn *decode, TCGv count, CCOp cc_op) 3532{ 3533 TCGv_i32 count32 = tcg_temp_new_i32(); 3534 TCGv_i32 old_cc_op; 3535 3536 decode->cc_op = CC_OP_DYNAMIC; 3537 decode->cc_op_dynamic = tcg_temp_new_i32(); 3538 3539 assert(decode->cc_dst == s->T0); 3540 if (cc_op_live[s->cc_op] & USES_CC_DST) { 3541 decode->cc_dst = tcg_temp_new(); 3542 tcg_gen_movcond_tl(TCG_COND_EQ, decode->cc_dst, count, tcg_constant_tl(0), 3543 cpu_cc_dst, s->T0); 3544 } 3545 3546 if (cc_op_live[s->cc_op] & USES_CC_SRC) { 3547 tcg_gen_movcond_tl(TCG_COND_EQ, decode->cc_src, count, tcg_constant_tl(0), 3548 cpu_cc_src, decode->cc_src); 3549 } 3550 3551 tcg_gen_trunc_tl_i32(count32, count); 3552 if (s->cc_op == CC_OP_DYNAMIC) { 3553 old_cc_op = cpu_cc_op; 3554 } else { 3555 old_cc_op = tcg_constant_i32(s->cc_op); 3556 } 3557 tcg_gen_movcond_i32(TCG_COND_EQ, decode->cc_op_dynamic, count32, tcg_constant_i32(0), 3558 old_cc_op, tcg_constant_i32(cc_op)); 3559} 3560 3561static void gen_SAR(DisasContext *s, X86DecodedInsn *decode) 3562{ 3563 bool can_be_zero; 3564 TCGv count; 3565 MemOp ot = gen_shift_count(s, decode, &can_be_zero, &count, decode->op[2].unit); 3566 3567 if (!count) { 3568 return; 3569 } 3570 3571 decode->cc_dst = s->T0; 3572 decode->cc_src = tcg_temp_new(); 3573 tcg_gen_subi_tl(decode->cc_src, count, 1); 3574 tcg_gen_sar_tl(decode->cc_src, s->T0, decode->cc_src); 3575 tcg_gen_sar_tl(s->T0, s->T0, count); 3576 if (can_be_zero) { 3577 gen_shift_dynamic_flags(s, decode, count, CC_OP_SARB + ot); 3578 } else { 3579 decode->cc_op = CC_OP_SARB + ot; 3580 } 3581} 3582 3583static void gen_SARX(DisasContext *s, X86DecodedInsn *decode) 3584{ 3585 MemOp ot = decode->op[0].ot; 3586 int mask; 3587 3588 mask = ot == MO_64 ? 63 : 31; 3589 tcg_gen_andi_tl(s->T1, s->T1, mask); 3590 tcg_gen_sar_tl(s->T0, s->T0, s->T1); 3591} 3592 3593static void gen_SBB(DisasContext *s, X86DecodedInsn *decode) 3594{ 3595 MemOp ot = decode->op[0].ot; 3596 TCGv c_in = tcg_temp_new(); 3597 3598 gen_compute_eflags_c(s, c_in); 3599 if (s->prefix & PREFIX_LOCK) { 3600 tcg_gen_add_tl(s->T0, s->T1, c_in); 3601 tcg_gen_neg_tl(s->T0, s->T0); 3602 tcg_gen_atomic_add_fetch_tl(s->T0, s->A0, s->T0, 3603 s->mem_index, ot | MO_LE); 3604 } else { 3605 /* 3606 * TODO: SBB reg, reg could use gen_prepare_eflags_c followed by 3607 * negsetcond, and CC_OP_SUBB as the cc_op. 3608 */ 3609 tcg_gen_sub_tl(s->T0, s->T0, s->T1); 3610 tcg_gen_sub_tl(s->T0, s->T0, c_in); 3611 } 3612 prepare_update3_cc(decode, s, CC_OP_SBBB + ot, c_in); 3613} 3614 3615static void gen_SCAS(DisasContext *s, X86DecodedInsn *decode) 3616{ 3617 MemOp ot = decode->op[2].ot; 3618 if (s->prefix & (PREFIX_REPZ | PREFIX_REPNZ)) { 3619 gen_repz_nz(s, ot, gen_scas); 3620 } else { 3621 gen_scas(s, ot); 3622 } 3623} 3624 3625static void gen_SETcc(DisasContext *s, X86DecodedInsn *decode) 3626{ 3627 gen_setcc1(s, decode->b & 0xf, s->T0); 3628} 3629 3630static void gen_SFENCE(DisasContext *s, X86DecodedInsn *decode) 3631{ 3632 tcg_gen_mb(TCG_MO_ST_ST | TCG_BAR_SC); 3633} 3634 3635static void gen_SHA1NEXTE(DisasContext *s, X86DecodedInsn *decode) 3636{ 3637 gen_helper_sha1nexte(OP_PTR0, OP_PTR1, OP_PTR2); 3638} 3639 3640static void gen_SHA1MSG1(DisasContext *s, X86DecodedInsn *decode) 3641{ 3642 gen_helper_sha1msg1(OP_PTR0, OP_PTR1, OP_PTR2); 3643} 3644 3645static void gen_SHA1MSG2(DisasContext *s, X86DecodedInsn *decode) 3646{ 3647 gen_helper_sha1msg2(OP_PTR0, OP_PTR1, OP_PTR2); 3648} 3649 3650static void gen_SHA1RNDS4(DisasContext *s, X86DecodedInsn *decode) 3651{ 3652 switch(decode->immediate & 3) { 3653 case 0: 3654 gen_helper_sha1rnds4_f0(OP_PTR0, OP_PTR0, OP_PTR1); 3655 break; 3656 case 1: 3657 gen_helper_sha1rnds4_f1(OP_PTR0, OP_PTR0, OP_PTR1); 3658 break; 3659 case 2: 3660 gen_helper_sha1rnds4_f2(OP_PTR0, OP_PTR0, OP_PTR1); 3661 break; 3662 case 3: 3663 gen_helper_sha1rnds4_f3(OP_PTR0, OP_PTR0, OP_PTR1); 3664 break; 3665 } 3666} 3667 3668static void gen_SHA256MSG1(DisasContext *s, X86DecodedInsn *decode) 3669{ 3670 gen_helper_sha256msg1(OP_PTR0, OP_PTR1, OP_PTR2); 3671} 3672 3673static void gen_SHA256MSG2(DisasContext *s, X86DecodedInsn *decode) 3674{ 3675 gen_helper_sha256msg2(OP_PTR0, OP_PTR1, OP_PTR2); 3676} 3677 3678static void gen_SHA256RNDS2(DisasContext *s, X86DecodedInsn *decode) 3679{ 3680 TCGv_i32 wk0 = tcg_temp_new_i32(); 3681 TCGv_i32 wk1 = tcg_temp_new_i32(); 3682 3683 tcg_gen_ld_i32(wk0, tcg_env, ZMM_OFFSET(0) + offsetof(ZMMReg, ZMM_L(0))); 3684 tcg_gen_ld_i32(wk1, tcg_env, ZMM_OFFSET(0) + offsetof(ZMMReg, ZMM_L(1))); 3685 3686 gen_helper_sha256rnds2(OP_PTR0, OP_PTR1, OP_PTR2, wk0, wk1); 3687} 3688 3689static void gen_SHL(DisasContext *s, X86DecodedInsn *decode) 3690{ 3691 bool can_be_zero; 3692 TCGv count; 3693 MemOp ot = gen_shift_count(s, decode, &can_be_zero, &count, decode->op[2].unit); 3694 3695 if (!count) { 3696 return; 3697 } 3698 3699 decode->cc_dst = s->T0; 3700 decode->cc_src = tcg_temp_new(); 3701 tcg_gen_subi_tl(decode->cc_src, count, 1); 3702 tcg_gen_shl_tl(decode->cc_src, s->T0, decode->cc_src); 3703 tcg_gen_shl_tl(s->T0, s->T0, count); 3704 if (can_be_zero) { 3705 gen_shift_dynamic_flags(s, decode, count, CC_OP_SHLB + ot); 3706 } else { 3707 decode->cc_op = CC_OP_SHLB + ot; 3708 } 3709} 3710 3711static void gen_SHLD(DisasContext *s, X86DecodedInsn *decode) 3712{ 3713 bool can_be_zero; 3714 TCGv count; 3715 int unit = decode->e.op3 == X86_TYPE_I ? X86_OP_IMM : X86_OP_INT; 3716 MemOp ot = gen_shift_count(s, decode, &can_be_zero, &count, unit); 3717 3718 if (!count) { 3719 return; 3720 } 3721 3722 decode->cc_dst = s->T0; 3723 decode->cc_src = s->tmp0; 3724 gen_shiftd_rm_T1(s, ot, false, count); 3725 if (can_be_zero) { 3726 gen_shift_dynamic_flags(s, decode, count, CC_OP_SHLB + ot); 3727 } else { 3728 decode->cc_op = CC_OP_SHLB + ot; 3729 } 3730} 3731 3732static void gen_SHLX(DisasContext *s, X86DecodedInsn *decode) 3733{ 3734 MemOp ot = decode->op[0].ot; 3735 int mask; 3736 3737 mask = ot == MO_64 ? 63 : 31; 3738 tcg_gen_andi_tl(s->T1, s->T1, mask); 3739 tcg_gen_shl_tl(s->T0, s->T0, s->T1); 3740} 3741 3742static void gen_SHR(DisasContext *s, X86DecodedInsn *decode) 3743{ 3744 bool can_be_zero; 3745 TCGv count; 3746 MemOp ot = gen_shift_count(s, decode, &can_be_zero, &count, decode->op[2].unit); 3747 3748 if (!count) { 3749 return; 3750 } 3751 3752 decode->cc_dst = s->T0; 3753 decode->cc_src = tcg_temp_new(); 3754 tcg_gen_subi_tl(decode->cc_src, count, 1); 3755 tcg_gen_shr_tl(decode->cc_src, s->T0, decode->cc_src); 3756 tcg_gen_shr_tl(s->T0, s->T0, count); 3757 if (can_be_zero) { 3758 gen_shift_dynamic_flags(s, decode, count, CC_OP_SARB + ot); 3759 } else { 3760 decode->cc_op = CC_OP_SARB + ot; 3761 } 3762} 3763 3764static void gen_SHRD(DisasContext *s, X86DecodedInsn *decode) 3765{ 3766 bool can_be_zero; 3767 TCGv count; 3768 int unit = decode->e.op3 == X86_TYPE_I ? X86_OP_IMM : X86_OP_INT; 3769 MemOp ot = gen_shift_count(s, decode, &can_be_zero, &count, unit); 3770 3771 if (!count) { 3772 return; 3773 } 3774 3775 decode->cc_dst = s->T0; 3776 decode->cc_src = s->tmp0; 3777 gen_shiftd_rm_T1(s, ot, true, count); 3778 if (can_be_zero) { 3779 gen_shift_dynamic_flags(s, decode, count, CC_OP_SARB + ot); 3780 } else { 3781 decode->cc_op = CC_OP_SARB + ot; 3782 } 3783} 3784 3785static void gen_SHRX(DisasContext *s, X86DecodedInsn *decode) 3786{ 3787 MemOp ot = decode->op[0].ot; 3788 int mask; 3789 3790 mask = ot == MO_64 ? 63 : 31; 3791 tcg_gen_andi_tl(s->T1, s->T1, mask); 3792 tcg_gen_shr_tl(s->T0, s->T0, s->T1); 3793} 3794 3795static void gen_STC(DisasContext *s, X86DecodedInsn *decode) 3796{ 3797 gen_compute_eflags(s); 3798 tcg_gen_ori_tl(cpu_cc_src, cpu_cc_src, CC_C); 3799} 3800 3801static void gen_STD(DisasContext *s, X86DecodedInsn *decode) 3802{ 3803 tcg_gen_st_i32(tcg_constant_i32(-1), tcg_env, offsetof(CPUX86State, df)); 3804} 3805 3806static void gen_STI(DisasContext *s, X86DecodedInsn *decode) 3807{ 3808 gen_set_eflags(s, IF_MASK); 3809 s->base.is_jmp = DISAS_EOB_INHIBIT_IRQ; 3810} 3811 3812static void gen_VAESKEYGEN(DisasContext *s, X86DecodedInsn *decode) 3813{ 3814 TCGv_i32 imm = tcg_constant8u_i32(decode->immediate); 3815 assert(!s->vex_l); 3816 gen_helper_aeskeygenassist_xmm(tcg_env, OP_PTR0, OP_PTR1, imm); 3817} 3818 3819static void gen_STMXCSR(DisasContext *s, X86DecodedInsn *decode) 3820{ 3821 gen_helper_update_mxcsr(tcg_env); 3822 tcg_gen_ld32u_tl(s->T0, tcg_env, offsetof(CPUX86State, mxcsr)); 3823} 3824 3825static void gen_STOS(DisasContext *s, X86DecodedInsn *decode) 3826{ 3827 MemOp ot = decode->op[1].ot; 3828 if (s->prefix & (PREFIX_REPZ | PREFIX_REPNZ)) { 3829 gen_repz(s, ot, gen_stos); 3830 } else { 3831 gen_stos(s, ot); 3832 } 3833} 3834 3835static void gen_SUB(DisasContext *s, X86DecodedInsn *decode) 3836{ 3837 MemOp ot = decode->op[1].ot; 3838 3839 if (s->prefix & PREFIX_LOCK) { 3840 tcg_gen_neg_tl(s->T0, s->T1); 3841 tcg_gen_atomic_fetch_add_tl(s->cc_srcT, s->A0, s->T0, 3842 s->mem_index, ot | MO_LE); 3843 tcg_gen_sub_tl(s->T0, s->cc_srcT, s->T1); 3844 } else { 3845 tcg_gen_mov_tl(s->cc_srcT, s->T0); 3846 tcg_gen_sub_tl(s->T0, s->T0, s->T1); 3847 } 3848 prepare_update2_cc(decode, s, CC_OP_SUBB + ot); 3849} 3850 3851static void gen_SYSCALL(DisasContext *s, X86DecodedInsn *decode) 3852{ 3853 gen_update_cc_op(s); 3854 gen_update_eip_cur(s); 3855 gen_helper_syscall(tcg_env, cur_insn_len_i32(s)); 3856 if (LMA(s)) { 3857 assume_cc_op(s, CC_OP_EFLAGS); 3858 } 3859 3860 /* 3861 * TF handling for the syscall insn is different. The TF bit is checked 3862 * after the syscall insn completes. This allows #DB to not be 3863 * generated after one has entered CPL0 if TF is set in FMASK. 3864 */ 3865 s->base.is_jmp = DISAS_EOB_RECHECK_TF; 3866} 3867 3868static void gen_SYSENTER(DisasContext *s, X86DecodedInsn *decode) 3869{ 3870 gen_helper_sysenter(tcg_env); 3871 s->base.is_jmp = DISAS_EOB_ONLY; 3872} 3873 3874static void gen_SYSEXIT(DisasContext *s, X86DecodedInsn *decode) 3875{ 3876 gen_helper_sysexit(tcg_env, tcg_constant_i32(s->dflag - 1)); 3877 s->base.is_jmp = DISAS_EOB_ONLY; 3878} 3879 3880static void gen_SYSRET(DisasContext *s, X86DecodedInsn *decode) 3881{ 3882 gen_helper_sysret(tcg_env, tcg_constant_i32(s->dflag - 1)); 3883 if (LMA(s)) { 3884 assume_cc_op(s, CC_OP_EFLAGS); 3885 } 3886 3887 /* 3888 * TF handling for the sysret insn is different. The TF bit is checked 3889 * after the sysret insn completes. This allows #DB to be 3890 * generated "as if" the syscall insn in userspace has just 3891 * completed. 3892 */ 3893 s->base.is_jmp = DISAS_EOB_RECHECK_TF; 3894} 3895 3896static void gen_TZCNT(DisasContext *s, X86DecodedInsn *decode) 3897{ 3898 MemOp ot = decode->op[0].ot; 3899 3900 /* C bit (cc_src) is defined related to the input. */ 3901 decode->cc_src = tcg_temp_new(); 3902 decode->cc_dst = s->T0; 3903 decode->cc_op = CC_OP_BMILGB + ot; 3904 tcg_gen_mov_tl(decode->cc_src, s->T0); 3905 3906 /* A zero input returns the operand size. */ 3907 tcg_gen_ctzi_tl(s->T0, s->T0, 8 << ot); 3908} 3909 3910static void gen_UD(DisasContext *s, X86DecodedInsn *decode) 3911{ 3912 gen_illegal_opcode(s); 3913} 3914 3915static void gen_VAESIMC(DisasContext *s, X86DecodedInsn *decode) 3916{ 3917 assert(!s->vex_l); 3918 gen_helper_aesimc_xmm(tcg_env, OP_PTR0, OP_PTR2); 3919} 3920 3921/* 3922 * 00 = v*ps Vps, Hps, Wpd 3923 * 66 = v*pd Vpd, Hpd, Wps 3924 * f3 = v*ss Vss, Hss, Wps 3925 * f2 = v*sd Vsd, Hsd, Wps 3926 */ 3927#define SSE_CMP(x) { \ 3928 gen_helper_ ## x ## ps ## _xmm, gen_helper_ ## x ## pd ## _xmm, \ 3929 gen_helper_ ## x ## ss, gen_helper_ ## x ## sd, \ 3930 gen_helper_ ## x ## ps ## _ymm, gen_helper_ ## x ## pd ## _ymm} 3931static const SSEFunc_0_eppp gen_helper_cmp_funcs[32][6] = { 3932 SSE_CMP(cmpeq), 3933 SSE_CMP(cmplt), 3934 SSE_CMP(cmple), 3935 SSE_CMP(cmpunord), 3936 SSE_CMP(cmpneq), 3937 SSE_CMP(cmpnlt), 3938 SSE_CMP(cmpnle), 3939 SSE_CMP(cmpord), 3940 3941 SSE_CMP(cmpequ), 3942 SSE_CMP(cmpnge), 3943 SSE_CMP(cmpngt), 3944 SSE_CMP(cmpfalse), 3945 SSE_CMP(cmpnequ), 3946 SSE_CMP(cmpge), 3947 SSE_CMP(cmpgt), 3948 SSE_CMP(cmptrue), 3949 3950 SSE_CMP(cmpeqs), 3951 SSE_CMP(cmpltq), 3952 SSE_CMP(cmpleq), 3953 SSE_CMP(cmpunords), 3954 SSE_CMP(cmpneqq), 3955 SSE_CMP(cmpnltq), 3956 SSE_CMP(cmpnleq), 3957 SSE_CMP(cmpords), 3958 3959 SSE_CMP(cmpequs), 3960 SSE_CMP(cmpngeq), 3961 SSE_CMP(cmpngtq), 3962 SSE_CMP(cmpfalses), 3963 SSE_CMP(cmpnequs), 3964 SSE_CMP(cmpgeq), 3965 SSE_CMP(cmpgtq), 3966 SSE_CMP(cmptrues), 3967}; 3968#undef SSE_CMP 3969 3970static void gen_VCMP(DisasContext *s, X86DecodedInsn *decode) 3971{ 3972 int index = decode->immediate & (s->prefix & PREFIX_VEX ? 31 : 7); 3973 int b = 3974 s->prefix & PREFIX_REPZ ? 2 /* ss */ : 3975 s->prefix & PREFIX_REPNZ ? 3 /* sd */ : 3976 !!(s->prefix & PREFIX_DATA) /* pd */ + (s->vex_l << 2); 3977 3978 gen_helper_cmp_funcs[index][b](tcg_env, OP_PTR0, OP_PTR1, OP_PTR2); 3979} 3980 3981static void gen_VCOMI(DisasContext *s, X86DecodedInsn *decode) 3982{ 3983 SSEFunc_0_epp fn; 3984 fn = s->prefix & PREFIX_DATA ? gen_helper_comisd : gen_helper_comiss; 3985 fn(tcg_env, OP_PTR1, OP_PTR2); 3986 assume_cc_op(s, CC_OP_EFLAGS); 3987} 3988 3989static void gen_VCVTPD2PS(DisasContext *s, X86DecodedInsn *decode) 3990{ 3991 if (s->vex_l) { 3992 gen_helper_cvtpd2ps_ymm(tcg_env, OP_PTR0, OP_PTR2); 3993 } else { 3994 gen_helper_cvtpd2ps_xmm(tcg_env, OP_PTR0, OP_PTR2); 3995 } 3996} 3997 3998static void gen_VCVTPS2PD(DisasContext *s, X86DecodedInsn *decode) 3999{ 4000 if (s->vex_l) { 4001 gen_helper_cvtps2pd_ymm(tcg_env, OP_PTR0, OP_PTR2); 4002 } else { 4003 gen_helper_cvtps2pd_xmm(tcg_env, OP_PTR0, OP_PTR2); 4004 } 4005} 4006 4007static void gen_VCVTPS2PH(DisasContext *s, X86DecodedInsn *decode) 4008{ 4009 gen_unary_imm_fp_sse(s, decode, 4010 gen_helper_cvtps2ph_xmm, 4011 gen_helper_cvtps2ph_ymm); 4012 /* 4013 * VCVTPS2PH is the only instruction that performs an operation on a 4014 * register source and then *stores* into memory. 4015 */ 4016 if (decode->op[0].has_ea) { 4017 gen_store_sse(s, decode, decode->op[0].offset); 4018 } 4019} 4020 4021static void gen_VCVTSD2SS(DisasContext *s, X86DecodedInsn *decode) 4022{ 4023 gen_helper_cvtsd2ss(tcg_env, OP_PTR0, OP_PTR1, OP_PTR2); 4024} 4025 4026static void gen_VCVTSS2SD(DisasContext *s, X86DecodedInsn *decode) 4027{ 4028 gen_helper_cvtss2sd(tcg_env, OP_PTR0, OP_PTR1, OP_PTR2); 4029} 4030 4031static void gen_VCVTSI2Sx(DisasContext *s, X86DecodedInsn *decode) 4032{ 4033 int vec_len = vector_len(s, decode); 4034 TCGv_i32 in; 4035 4036 tcg_gen_gvec_mov(MO_64, decode->op[0].offset, decode->op[1].offset, vec_len, vec_len); 4037 4038#ifdef TARGET_X86_64 4039 MemOp ot = decode->op[2].ot; 4040 if (ot == MO_64) { 4041 if (s->prefix & PREFIX_REPNZ) { 4042 gen_helper_cvtsq2sd(tcg_env, OP_PTR0, s->T1); 4043 } else { 4044 gen_helper_cvtsq2ss(tcg_env, OP_PTR0, s->T1); 4045 } 4046 return; 4047 } 4048 in = s->tmp2_i32; 4049 tcg_gen_trunc_tl_i32(in, s->T1); 4050#else 4051 in = s->T1; 4052#endif 4053 4054 if (s->prefix & PREFIX_REPNZ) { 4055 gen_helper_cvtsi2sd(tcg_env, OP_PTR0, in); 4056 } else { 4057 gen_helper_cvtsi2ss(tcg_env, OP_PTR0, in); 4058 } 4059} 4060 4061static inline void gen_VCVTtSx2SI(DisasContext *s, X86DecodedInsn *decode, 4062 SSEFunc_i_ep ss2si, SSEFunc_l_ep ss2sq, 4063 SSEFunc_i_ep sd2si, SSEFunc_l_ep sd2sq) 4064{ 4065 TCGv_i32 out; 4066 4067#ifdef TARGET_X86_64 4068 MemOp ot = decode->op[0].ot; 4069 if (ot == MO_64) { 4070 if (s->prefix & PREFIX_REPNZ) { 4071 sd2sq(s->T0, tcg_env, OP_PTR2); 4072 } else { 4073 ss2sq(s->T0, tcg_env, OP_PTR2); 4074 } 4075 return; 4076 } 4077 4078 out = s->tmp2_i32; 4079#else 4080 out = s->T0; 4081#endif 4082 if (s->prefix & PREFIX_REPNZ) { 4083 sd2si(out, tcg_env, OP_PTR2); 4084 } else { 4085 ss2si(out, tcg_env, OP_PTR2); 4086 } 4087#ifdef TARGET_X86_64 4088 tcg_gen_extu_i32_tl(s->T0, out); 4089#endif 4090} 4091 4092#ifndef TARGET_X86_64 4093#define gen_helper_cvtss2sq NULL 4094#define gen_helper_cvtsd2sq NULL 4095#define gen_helper_cvttss2sq NULL 4096#define gen_helper_cvttsd2sq NULL 4097#endif 4098 4099static void gen_VCVTSx2SI(DisasContext *s, X86DecodedInsn *decode) 4100{ 4101 gen_VCVTtSx2SI(s, decode, 4102 gen_helper_cvtss2si, gen_helper_cvtss2sq, 4103 gen_helper_cvtsd2si, gen_helper_cvtsd2sq); 4104} 4105 4106static void gen_VCVTTSx2SI(DisasContext *s, X86DecodedInsn *decode) 4107{ 4108 gen_VCVTtSx2SI(s, decode, 4109 gen_helper_cvttss2si, gen_helper_cvttss2sq, 4110 gen_helper_cvttsd2si, gen_helper_cvttsd2sq); 4111} 4112 4113static void gen_VEXTRACTx128(DisasContext *s, X86DecodedInsn *decode) 4114{ 4115 int mask = decode->immediate & 1; 4116 int src_ofs = vector_elem_offset(&decode->op[1], MO_128, mask); 4117 if (decode->op[0].has_ea) { 4118 /* VEX-only instruction, no alignment requirements. */ 4119 gen_sto_env_A0(s, src_ofs, false); 4120 } else { 4121 tcg_gen_gvec_mov(MO_64, decode->op[0].offset, src_ofs, 16, 16); 4122 } 4123} 4124 4125static void gen_VEXTRACTPS(DisasContext *s, X86DecodedInsn *decode) 4126{ 4127 gen_pextr(s, decode, MO_32); 4128} 4129 4130static void gen_vinsertps(DisasContext *s, X86DecodedInsn *decode) 4131{ 4132 int val = decode->immediate; 4133 int dest_word = (val >> 4) & 3; 4134 int new_mask = (val & 15) | (1 << dest_word); 4135 int vec_len = 16; 4136 4137 assert(!s->vex_l); 4138 4139 if (new_mask == 15) { 4140 /* All zeroes except possibly for the inserted element */ 4141 tcg_gen_gvec_dup_imm(MO_64, decode->op[0].offset, vec_len, vec_len, 0); 4142 } else if (decode->op[1].offset != decode->op[0].offset) { 4143 gen_store_sse(s, decode, decode->op[1].offset); 4144 } 4145 4146 if (new_mask != (val & 15)) { 4147 tcg_gen_st_i32(s->tmp2_i32, tcg_env, 4148 vector_elem_offset(&decode->op[0], MO_32, dest_word)); 4149 } 4150 4151 if (new_mask != 15) { 4152 TCGv_i32 zero = tcg_constant_i32(0); /* float32_zero */ 4153 int i; 4154 for (i = 0; i < 4; i++) { 4155 if ((val >> i) & 1) { 4156 tcg_gen_st_i32(zero, tcg_env, 4157 vector_elem_offset(&decode->op[0], MO_32, i)); 4158 } 4159 } 4160 } 4161} 4162 4163static void gen_VINSERTPS_r(DisasContext *s, X86DecodedInsn *decode) 4164{ 4165 int val = decode->immediate; 4166 tcg_gen_ld_i32(s->tmp2_i32, tcg_env, 4167 vector_elem_offset(&decode->op[2], MO_32, (val >> 6) & 3)); 4168 gen_vinsertps(s, decode); 4169} 4170 4171static void gen_VINSERTPS_m(DisasContext *s, X86DecodedInsn *decode) 4172{ 4173 tcg_gen_qemu_ld_i32(s->tmp2_i32, s->A0, s->mem_index, MO_LEUL); 4174 gen_vinsertps(s, decode); 4175} 4176 4177static void gen_VINSERTx128(DisasContext *s, X86DecodedInsn *decode) 4178{ 4179 int mask = decode->immediate & 1; 4180 tcg_gen_gvec_mov(MO_64, 4181 decode->op[0].offset + offsetof(YMMReg, YMM_X(mask)), 4182 decode->op[2].offset + offsetof(YMMReg, YMM_X(0)), 16, 16); 4183 tcg_gen_gvec_mov(MO_64, 4184 decode->op[0].offset + offsetof(YMMReg, YMM_X(!mask)), 4185 decode->op[1].offset + offsetof(YMMReg, YMM_X(!mask)), 16, 16); 4186} 4187 4188static inline void gen_maskmov(DisasContext *s, X86DecodedInsn *decode, 4189 SSEFunc_0_eppt xmm, SSEFunc_0_eppt ymm) 4190{ 4191 if (!s->vex_l) { 4192 xmm(tcg_env, OP_PTR2, OP_PTR1, s->A0); 4193 } else { 4194 ymm(tcg_env, OP_PTR2, OP_PTR1, s->A0); 4195 } 4196} 4197 4198static void gen_VMASKMOVPD_st(DisasContext *s, X86DecodedInsn *decode) 4199{ 4200 gen_maskmov(s, decode, gen_helper_vpmaskmovq_st_xmm, gen_helper_vpmaskmovq_st_ymm); 4201} 4202 4203static void gen_VMASKMOVPS_st(DisasContext *s, X86DecodedInsn *decode) 4204{ 4205 gen_maskmov(s, decode, gen_helper_vpmaskmovd_st_xmm, gen_helper_vpmaskmovd_st_ymm); 4206} 4207 4208static void gen_VMOVHPx_ld(DisasContext *s, X86DecodedInsn *decode) 4209{ 4210 gen_ldq_env_A0(s, decode->op[0].offset + offsetof(XMMReg, XMM_Q(1))); 4211 if (decode->op[0].offset != decode->op[1].offset) { 4212 tcg_gen_ld_i64(s->tmp1_i64, tcg_env, decode->op[1].offset + offsetof(XMMReg, XMM_Q(0))); 4213 tcg_gen_st_i64(s->tmp1_i64, tcg_env, decode->op[0].offset + offsetof(XMMReg, XMM_Q(0))); 4214 } 4215} 4216 4217static void gen_VMOVHPx_st(DisasContext *s, X86DecodedInsn *decode) 4218{ 4219 gen_stq_env_A0(s, decode->op[2].offset + offsetof(XMMReg, XMM_Q(1))); 4220} 4221 4222static void gen_VMOVHPx(DisasContext *s, X86DecodedInsn *decode) 4223{ 4224 if (decode->op[0].offset != decode->op[2].offset) { 4225 tcg_gen_ld_i64(s->tmp1_i64, tcg_env, decode->op[2].offset + offsetof(XMMReg, XMM_Q(1))); 4226 tcg_gen_st_i64(s->tmp1_i64, tcg_env, decode->op[0].offset + offsetof(XMMReg, XMM_Q(1))); 4227 } 4228 if (decode->op[0].offset != decode->op[1].offset) { 4229 tcg_gen_ld_i64(s->tmp1_i64, tcg_env, decode->op[1].offset + offsetof(XMMReg, XMM_Q(0))); 4230 tcg_gen_st_i64(s->tmp1_i64, tcg_env, decode->op[0].offset + offsetof(XMMReg, XMM_Q(0))); 4231 } 4232} 4233 4234static void gen_VMOVHLPS(DisasContext *s, X86DecodedInsn *decode) 4235{ 4236 tcg_gen_ld_i64(s->tmp1_i64, tcg_env, decode->op[2].offset + offsetof(XMMReg, XMM_Q(1))); 4237 tcg_gen_st_i64(s->tmp1_i64, tcg_env, decode->op[0].offset + offsetof(XMMReg, XMM_Q(0))); 4238 if (decode->op[0].offset != decode->op[1].offset) { 4239 tcg_gen_ld_i64(s->tmp1_i64, tcg_env, decode->op[1].offset + offsetof(XMMReg, XMM_Q(1))); 4240 tcg_gen_st_i64(s->tmp1_i64, tcg_env, decode->op[0].offset + offsetof(XMMReg, XMM_Q(1))); 4241 } 4242} 4243 4244static void gen_VMOVLHPS(DisasContext *s, X86DecodedInsn *decode) 4245{ 4246 tcg_gen_ld_i64(s->tmp1_i64, tcg_env, decode->op[2].offset); 4247 tcg_gen_st_i64(s->tmp1_i64, tcg_env, decode->op[0].offset + offsetof(XMMReg, XMM_Q(1))); 4248 if (decode->op[0].offset != decode->op[1].offset) { 4249 tcg_gen_ld_i64(s->tmp1_i64, tcg_env, decode->op[1].offset + offsetof(XMMReg, XMM_Q(0))); 4250 tcg_gen_st_i64(s->tmp1_i64, tcg_env, decode->op[0].offset + offsetof(XMMReg, XMM_Q(0))); 4251 } 4252} 4253 4254/* 4255 * Note that MOVLPx supports 256-bit operation unlike MOVHLPx, MOVLHPx, MOXHPx. 4256 * Use a gvec move to move everything above the bottom 64 bits. 4257 */ 4258 4259static void gen_VMOVLPx(DisasContext *s, X86DecodedInsn *decode) 4260{ 4261 int vec_len = vector_len(s, decode); 4262 4263 tcg_gen_ld_i64(s->tmp1_i64, tcg_env, decode->op[2].offset + offsetof(XMMReg, XMM_Q(0))); 4264 tcg_gen_gvec_mov(MO_64, decode->op[0].offset, decode->op[1].offset, vec_len, vec_len); 4265 tcg_gen_st_i64(s->tmp1_i64, tcg_env, decode->op[0].offset + offsetof(XMMReg, XMM_Q(0))); 4266} 4267 4268static void gen_VMOVLPx_ld(DisasContext *s, X86DecodedInsn *decode) 4269{ 4270 int vec_len = vector_len(s, decode); 4271 4272 tcg_gen_qemu_ld_i64(s->tmp1_i64, s->A0, s->mem_index, MO_LEUQ); 4273 tcg_gen_gvec_mov(MO_64, decode->op[0].offset, decode->op[1].offset, vec_len, vec_len); 4274 tcg_gen_st_i64(s->tmp1_i64, OP_PTR0, offsetof(ZMMReg, ZMM_Q(0))); 4275} 4276 4277static void gen_VMOVLPx_st(DisasContext *s, X86DecodedInsn *decode) 4278{ 4279 tcg_gen_ld_i64(s->tmp1_i64, OP_PTR2, offsetof(ZMMReg, ZMM_Q(0))); 4280 tcg_gen_qemu_st_i64(s->tmp1_i64, s->A0, s->mem_index, MO_LEUQ); 4281} 4282 4283static void gen_VMOVSD_ld(DisasContext *s, X86DecodedInsn *decode) 4284{ 4285 TCGv_i64 zero = tcg_constant_i64(0); 4286 4287 tcg_gen_qemu_ld_i64(s->tmp1_i64, s->A0, s->mem_index, MO_LEUQ); 4288 tcg_gen_st_i64(zero, OP_PTR0, offsetof(ZMMReg, ZMM_Q(1))); 4289 tcg_gen_st_i64(s->tmp1_i64, OP_PTR0, offsetof(ZMMReg, ZMM_Q(0))); 4290} 4291 4292static void gen_VMOVSS(DisasContext *s, X86DecodedInsn *decode) 4293{ 4294 int vec_len = vector_len(s, decode); 4295 4296 tcg_gen_ld_i32(s->tmp2_i32, OP_PTR2, offsetof(ZMMReg, ZMM_L(0))); 4297 tcg_gen_gvec_mov(MO_64, decode->op[0].offset, decode->op[1].offset, vec_len, vec_len); 4298 tcg_gen_st_i32(s->tmp2_i32, OP_PTR0, offsetof(ZMMReg, ZMM_L(0))); 4299} 4300 4301static void gen_VMOVSS_ld(DisasContext *s, X86DecodedInsn *decode) 4302{ 4303 int vec_len = vector_len(s, decode); 4304 4305 tcg_gen_qemu_ld_i32(s->tmp2_i32, s->A0, s->mem_index, MO_LEUL); 4306 tcg_gen_gvec_dup_imm(MO_64, decode->op[0].offset, vec_len, vec_len, 0); 4307 tcg_gen_st_i32(s->tmp2_i32, OP_PTR0, offsetof(ZMMReg, ZMM_L(0))); 4308} 4309 4310static void gen_VMOVSS_st(DisasContext *s, X86DecodedInsn *decode) 4311{ 4312 tcg_gen_ld_i32(s->tmp2_i32, OP_PTR2, offsetof(ZMMReg, ZMM_L(0))); 4313 tcg_gen_qemu_st_i32(s->tmp2_i32, s->A0, s->mem_index, MO_LEUL); 4314} 4315 4316static void gen_VPMASKMOV_st(DisasContext *s, X86DecodedInsn *decode) 4317{ 4318 if (s->vex_w) { 4319 gen_VMASKMOVPD_st(s, decode); 4320 } else { 4321 gen_VMASKMOVPS_st(s, decode); 4322 } 4323} 4324 4325static void gen_VPERMD(DisasContext *s, X86DecodedInsn *decode) 4326{ 4327 assert(s->vex_l); 4328 gen_helper_vpermd_ymm(OP_PTR0, OP_PTR1, OP_PTR2); 4329} 4330 4331static void gen_VPERM2x128(DisasContext *s, X86DecodedInsn *decode) 4332{ 4333 TCGv_i32 imm = tcg_constant8u_i32(decode->immediate); 4334 assert(s->vex_l); 4335 gen_helper_vpermdq_ymm(OP_PTR0, OP_PTR1, OP_PTR2, imm); 4336} 4337 4338static void gen_VPHMINPOSUW(DisasContext *s, X86DecodedInsn *decode) 4339{ 4340 assert(!s->vex_l); 4341 gen_helper_phminposuw_xmm(tcg_env, OP_PTR0, OP_PTR2); 4342} 4343 4344static void gen_VROUNDSD(DisasContext *s, X86DecodedInsn *decode) 4345{ 4346 TCGv_i32 imm = tcg_constant8u_i32(decode->immediate); 4347 assert(!s->vex_l); 4348 gen_helper_roundsd_xmm(tcg_env, OP_PTR0, OP_PTR1, OP_PTR2, imm); 4349} 4350 4351static void gen_VROUNDSS(DisasContext *s, X86DecodedInsn *decode) 4352{ 4353 TCGv_i32 imm = tcg_constant8u_i32(decode->immediate); 4354 assert(!s->vex_l); 4355 gen_helper_roundss_xmm(tcg_env, OP_PTR0, OP_PTR1, OP_PTR2, imm); 4356} 4357 4358static void gen_VSHUF(DisasContext *s, X86DecodedInsn *decode) 4359{ 4360 TCGv_i32 imm = tcg_constant_i32(decode->immediate); 4361 SSEFunc_0_pppi ps, pd, fn; 4362 ps = s->vex_l ? gen_helper_shufps_ymm : gen_helper_shufps_xmm; 4363 pd = s->vex_l ? gen_helper_shufpd_ymm : gen_helper_shufpd_xmm; 4364 fn = s->prefix & PREFIX_DATA ? pd : ps; 4365 fn(OP_PTR0, OP_PTR1, OP_PTR2, imm); 4366} 4367 4368static void gen_VUCOMI(DisasContext *s, X86DecodedInsn *decode) 4369{ 4370 SSEFunc_0_epp fn; 4371 fn = s->prefix & PREFIX_DATA ? gen_helper_ucomisd : gen_helper_ucomiss; 4372 fn(tcg_env, OP_PTR1, OP_PTR2); 4373 assume_cc_op(s, CC_OP_EFLAGS); 4374} 4375 4376static void gen_VZEROALL(DisasContext *s, X86DecodedInsn *decode) 4377{ 4378 TCGv_ptr ptr = tcg_temp_new_ptr(); 4379 4380 tcg_gen_addi_ptr(ptr, tcg_env, offsetof(CPUX86State, xmm_regs)); 4381 gen_helper_memset(ptr, ptr, tcg_constant_i32(0), 4382 tcg_constant_ptr(CPU_NB_REGS * sizeof(ZMMReg))); 4383} 4384 4385static void gen_VZEROUPPER(DisasContext *s, X86DecodedInsn *decode) 4386{ 4387 int i; 4388 4389 for (i = 0; i < CPU_NB_REGS; i++) { 4390 int offset = offsetof(CPUX86State, xmm_regs[i].ZMM_X(1)); 4391 tcg_gen_gvec_dup_imm(MO_64, offset, 16, 16, 0); 4392 } 4393} 4394 4395static void gen_WAIT(DisasContext *s, X86DecodedInsn *decode) 4396{ 4397 if ((s->flags & (HF_MP_MASK | HF_TS_MASK)) == (HF_MP_MASK | HF_TS_MASK)) { 4398 gen_NM_exception(s); 4399 } else { 4400 /* needs to be treated as I/O because of ferr_irq */ 4401 translator_io_start(&s->base); 4402 gen_helper_fwait(tcg_env); 4403 } 4404} 4405 4406#ifndef CONFIG_USER_ONLY 4407static void gen_WRMSR(DisasContext *s, X86DecodedInsn *decode) 4408{ 4409 gen_update_cc_op(s); 4410 gen_update_eip_cur(s); 4411 gen_helper_wrmsr(tcg_env); 4412 s->base.is_jmp = DISAS_EOB_NEXT; 4413} 4414#else 4415#define gen_WRMSR gen_unreachable 4416#endif 4417 4418static void gen_WRxxBASE(DisasContext *s, X86DecodedInsn *decode) 4419{ 4420 TCGv base = cpu_seg_base[s->modrm & 8 ? R_GS : R_FS]; 4421 4422 /* Preserve hflags bits by testing CR4 at runtime. */ 4423 gen_helper_cr4_testbit(tcg_env, tcg_constant_i32(CR4_FSGSBASE_MASK)); 4424 tcg_gen_mov_tl(base, s->T0); 4425} 4426 4427static void gen_XADD(DisasContext *s, X86DecodedInsn *decode) 4428{ 4429 MemOp ot = decode->op[1].ot; 4430 4431 decode->cc_dst = tcg_temp_new(); 4432 decode->cc_src = s->T1; 4433 decode->cc_op = CC_OP_ADDB + ot; 4434 4435 if (s->prefix & PREFIX_LOCK) { 4436 tcg_gen_atomic_fetch_add_tl(s->T0, s->A0, s->T1, s->mem_index, ot | MO_LE); 4437 tcg_gen_add_tl(decode->cc_dst, s->T0, s->T1); 4438 } else { 4439 tcg_gen_add_tl(decode->cc_dst, s->T0, s->T1); 4440 /* 4441 * NOTE: writing memory first is important for MMU exceptions, 4442 * but "new result" wins for XADD AX, AX. 4443 */ 4444 gen_writeback(s, decode, 0, decode->cc_dst); 4445 } 4446 if (decode->op[0].has_ea || decode->op[2].n != decode->op[0].n) { 4447 gen_writeback(s, decode, 2, s->T0); 4448 } 4449} 4450 4451static void gen_XCHG(DisasContext *s, X86DecodedInsn *decode) 4452{ 4453 if (s->prefix & PREFIX_LOCK) { 4454 tcg_gen_atomic_xchg_tl(s->T0, s->A0, s->T1, 4455 s->mem_index, decode->op[0].ot | MO_LE); 4456 /* now store old value into register operand */ 4457 gen_op_mov_reg_v(s, decode->op[2].ot, decode->op[2].n, s->T0); 4458 } else { 4459 /* move destination value into source operand, source preserved in T1 */ 4460 gen_op_mov_reg_v(s, decode->op[2].ot, decode->op[2].n, s->T0); 4461 tcg_gen_mov_tl(s->T0, s->T1); 4462 } 4463} 4464 4465static void gen_XLAT(DisasContext *s, X86DecodedInsn *decode) 4466{ 4467 /* AL is already zero-extended into s->T0. */ 4468 tcg_gen_add_tl(s->A0, cpu_regs[R_EBX], s->T0); 4469 gen_lea_v_seg(s, s->A0, R_DS, s->override); 4470 gen_op_ld_v(s, MO_8, s->T0, s->A0); 4471} 4472 4473static void gen_XOR(DisasContext *s, X86DecodedInsn *decode) 4474{ 4475 /* special case XOR reg, reg */ 4476 if (decode->op[1].unit == X86_OP_INT && 4477 decode->op[2].unit == X86_OP_INT && 4478 decode->op[1].n == decode->op[2].n) { 4479 tcg_gen_movi_tl(s->T0, 0); 4480 decode->cc_op = CC_OP_CLR; 4481 } else { 4482 MemOp ot = decode->op[1].ot; 4483 4484 if (s->prefix & PREFIX_LOCK) { 4485 tcg_gen_atomic_xor_fetch_tl(s->T0, s->A0, s->T1, 4486 s->mem_index, ot | MO_LE); 4487 } else { 4488 tcg_gen_xor_tl(s->T0, s->T0, s->T1); 4489 } 4490 prepare_update1_cc(decode, s, CC_OP_LOGICB + ot); 4491 } 4492} 4493 4494static void gen_XRSTOR(DisasContext *s, X86DecodedInsn *decode) 4495{ 4496 TCGv_i64 features = tcg_temp_new_i64(); 4497 4498 tcg_gen_concat_tl_i64(features, cpu_regs[R_EAX], cpu_regs[R_EDX]); 4499 gen_helper_xrstor(tcg_env, s->A0, features); 4500 if (s->cpuid_7_0_ebx_features & CPUID_7_0_EBX_MPX) { 4501 /* 4502 * XRSTOR is how MPX is enabled, which changes how 4503 * we translate. Thus we need to end the TB. 4504 */ 4505 s->base.is_jmp = DISAS_EOB_NEXT; 4506 } 4507} 4508 4509static void gen_XSAVE(DisasContext *s, X86DecodedInsn *decode) 4510{ 4511 TCGv_i64 features = tcg_temp_new_i64(); 4512 4513 tcg_gen_concat_tl_i64(features, cpu_regs[R_EAX], cpu_regs[R_EDX]); 4514 gen_helper_xsave(tcg_env, s->A0, features); 4515} 4516 4517static void gen_XSAVEOPT(DisasContext *s, X86DecodedInsn *decode) 4518{ 4519 TCGv_i64 features = tcg_temp_new_i64(); 4520 4521 tcg_gen_concat_tl_i64(features, cpu_regs[R_EAX], cpu_regs[R_EDX]); 4522 gen_helper_xsave(tcg_env, s->A0, features); 4523} 4524