1/* 2 * New-style TCG opcode generator for i386 instructions 3 * 4 * Copyright (c) 2022 Red Hat, Inc. 5 * 6 * Author: Paolo Bonzini <pbonzini@redhat.com> 7 * 8 * This library is free software; you can redistribute it and/or 9 * modify it under the terms of the GNU Lesser General Public 10 * License as published by the Free Software Foundation; either 11 * version 2.1 of the License, or (at your option) any later version. 12 * 13 * This library is distributed in the hope that it will be useful, 14 * but WITHOUT ANY WARRANTY; without even the implied warranty of 15 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU 16 * Lesser General Public License for more details. 17 * 18 * You should have received a copy of the GNU Lesser General Public 19 * License along with this library; if not, see <http://www.gnu.org/licenses/>. 20 */ 21 22/* 23 * Sometimes, knowing what the backend has can produce better code. 24 * The exact opcode to check depends on 32- vs. 64-bit. 25 */ 26#ifdef TARGET_X86_64 27#define INDEX_op_extract2_tl INDEX_op_extract2_i64 28#else 29#define INDEX_op_extract2_tl INDEX_op_extract2_i32 30#endif 31 32#define MMX_OFFSET(reg) \ 33 ({ assert((reg) >= 0 && (reg) <= 7); \ 34 offsetof(CPUX86State, fpregs[reg].mmx); }) 35 36#define ZMM_OFFSET(reg) \ 37 ({ assert((reg) >= 0 && (reg) <= 15); \ 38 offsetof(CPUX86State, xmm_regs[reg]); }) 39 40typedef void (*SSEFunc_i_ep)(TCGv_i32 val, TCGv_ptr env, TCGv_ptr reg); 41typedef void (*SSEFunc_l_ep)(TCGv_i64 val, TCGv_ptr env, TCGv_ptr reg); 42typedef void (*SSEFunc_0_epp)(TCGv_ptr env, TCGv_ptr reg_a, TCGv_ptr reg_b); 43typedef void (*SSEFunc_0_eppp)(TCGv_ptr env, TCGv_ptr reg_a, TCGv_ptr reg_b, 44 TCGv_ptr reg_c); 45typedef void (*SSEFunc_0_epppp)(TCGv_ptr env, TCGv_ptr reg_a, TCGv_ptr reg_b, 46 TCGv_ptr reg_c, TCGv_ptr reg_d); 47typedef void (*SSEFunc_0_eppi)(TCGv_ptr env, TCGv_ptr reg_a, TCGv_ptr reg_b, 48 TCGv_i32 val); 49typedef void (*SSEFunc_0_epppi)(TCGv_ptr env, TCGv_ptr reg_a, TCGv_ptr reg_b, 50 TCGv_ptr reg_c, TCGv_i32 val); 51typedef void (*SSEFunc_0_ppi)(TCGv_ptr reg_a, TCGv_ptr reg_b, TCGv_i32 val); 52typedef void (*SSEFunc_0_pppi)(TCGv_ptr reg_a, TCGv_ptr reg_b, TCGv_ptr reg_c, 53 TCGv_i32 val); 54typedef void (*SSEFunc_0_eppt)(TCGv_ptr env, TCGv_ptr reg_a, TCGv_ptr reg_b, 55 TCGv val); 56typedef void (*SSEFunc_0_epppti)(TCGv_ptr env, TCGv_ptr reg_a, TCGv_ptr reg_b, 57 TCGv_ptr reg_c, TCGv a0, TCGv_i32 scale); 58typedef void (*SSEFunc_0_eppppi)(TCGv_ptr env, TCGv_ptr reg_a, TCGv_ptr reg_b, 59 TCGv_ptr reg_c, TCGv_ptr reg_d, TCGv_i32 flags); 60typedef void (*SSEFunc_0_eppppii)(TCGv_ptr env, TCGv_ptr reg_a, TCGv_ptr reg_b, 61 TCGv_ptr reg_c, TCGv_ptr reg_d, TCGv_i32 even, 62 TCGv_i32 odd); 63 64static void gen_JMP_m(DisasContext *s, X86DecodedInsn *decode); 65static void gen_JMP(DisasContext *s, X86DecodedInsn *decode); 66 67static inline TCGv_i32 tcg_constant8u_i32(uint8_t val) 68{ 69 return tcg_constant_i32(val); 70} 71 72static void gen_NM_exception(DisasContext *s) 73{ 74 gen_exception(s, EXCP07_PREX); 75} 76 77static void gen_lea_modrm(DisasContext *s, X86DecodedInsn *decode) 78{ 79 AddressParts *mem = &decode->mem; 80 TCGv ea; 81 82 ea = gen_lea_modrm_1(s, *mem, decode->e.vex_class == 12); 83 if (decode->e.special == X86_SPECIAL_BitTest) { 84 MemOp ot = decode->op[1].ot; 85 int poslen = 8 << ot; 86 int opn = decode->op[2].n; 87 TCGv ofs = tcg_temp_new(); 88 89 /* Extract memory displacement from the second operand. */ 90 assert(decode->op[2].unit == X86_OP_INT && decode->op[2].ot != MO_8); 91 tcg_gen_sextract_tl(ofs, cpu_regs[opn], 3, poslen - 3); 92 tcg_gen_andi_tl(ofs, ofs, -1 << ot); 93 tcg_gen_add_tl(s->A0, ea, ofs); 94 ea = s->A0; 95 } 96 97 gen_lea_v_seg(s, ea, mem->def_seg, s->override); 98} 99 100static inline int mmx_offset(MemOp ot) 101{ 102 switch (ot) { 103 case MO_8: 104 return offsetof(MMXReg, MMX_B(0)); 105 case MO_16: 106 return offsetof(MMXReg, MMX_W(0)); 107 case MO_32: 108 return offsetof(MMXReg, MMX_L(0)); 109 case MO_64: 110 return offsetof(MMXReg, MMX_Q(0)); 111 default: 112 g_assert_not_reached(); 113 } 114} 115 116static inline int xmm_offset(MemOp ot) 117{ 118 switch (ot) { 119 case MO_8: 120 return offsetof(ZMMReg, ZMM_B(0)); 121 case MO_16: 122 return offsetof(ZMMReg, ZMM_W(0)); 123 case MO_32: 124 return offsetof(ZMMReg, ZMM_L(0)); 125 case MO_64: 126 return offsetof(ZMMReg, ZMM_Q(0)); 127 case MO_128: 128 return offsetof(ZMMReg, ZMM_X(0)); 129 case MO_256: 130 return offsetof(ZMMReg, ZMM_Y(0)); 131 default: 132 g_assert_not_reached(); 133 } 134} 135 136static int vector_reg_offset(X86DecodedOp *op) 137{ 138 assert(op->unit == X86_OP_MMX || op->unit == X86_OP_SSE); 139 140 if (op->unit == X86_OP_MMX) { 141 return op->offset - mmx_offset(op->ot); 142 } else { 143 return op->offset - xmm_offset(op->ot); 144 } 145} 146 147static int vector_elem_offset(X86DecodedOp *op, MemOp ot, int n) 148{ 149 int base_ofs = vector_reg_offset(op); 150 switch(ot) { 151 case MO_8: 152 if (op->unit == X86_OP_MMX) { 153 return base_ofs + offsetof(MMXReg, MMX_B(n)); 154 } else { 155 return base_ofs + offsetof(ZMMReg, ZMM_B(n)); 156 } 157 case MO_16: 158 if (op->unit == X86_OP_MMX) { 159 return base_ofs + offsetof(MMXReg, MMX_W(n)); 160 } else { 161 return base_ofs + offsetof(ZMMReg, ZMM_W(n)); 162 } 163 case MO_32: 164 if (op->unit == X86_OP_MMX) { 165 return base_ofs + offsetof(MMXReg, MMX_L(n)); 166 } else { 167 return base_ofs + offsetof(ZMMReg, ZMM_L(n)); 168 } 169 case MO_64: 170 if (op->unit == X86_OP_MMX) { 171 return base_ofs; 172 } else { 173 return base_ofs + offsetof(ZMMReg, ZMM_Q(n)); 174 } 175 case MO_128: 176 assert(op->unit == X86_OP_SSE); 177 return base_ofs + offsetof(ZMMReg, ZMM_X(n)); 178 case MO_256: 179 assert(op->unit == X86_OP_SSE); 180 return base_ofs + offsetof(ZMMReg, ZMM_Y(n)); 181 default: 182 g_assert_not_reached(); 183 } 184} 185 186static void compute_mmx_offset(X86DecodedOp *op) 187{ 188 if (!op->has_ea) { 189 op->offset = MMX_OFFSET(op->n) + mmx_offset(op->ot); 190 } else { 191 op->offset = offsetof(CPUX86State, mmx_t0) + mmx_offset(op->ot); 192 } 193} 194 195static void compute_xmm_offset(X86DecodedOp *op) 196{ 197 if (!op->has_ea) { 198 op->offset = ZMM_OFFSET(op->n) + xmm_offset(op->ot); 199 } else { 200 op->offset = offsetof(CPUX86State, xmm_t0) + xmm_offset(op->ot); 201 } 202} 203 204static void gen_load_sse(DisasContext *s, TCGv temp, MemOp ot, int dest_ofs, bool aligned) 205{ 206 switch(ot) { 207 case MO_8: 208 gen_op_ld_v(s, MO_8, temp, s->A0); 209 tcg_gen_st8_tl(temp, tcg_env, dest_ofs); 210 break; 211 case MO_16: 212 gen_op_ld_v(s, MO_16, temp, s->A0); 213 tcg_gen_st16_tl(temp, tcg_env, dest_ofs); 214 break; 215 case MO_32: 216 gen_op_ld_v(s, MO_32, temp, s->A0); 217 tcg_gen_st32_tl(temp, tcg_env, dest_ofs); 218 break; 219 case MO_64: 220 gen_ldq_env_A0(s, dest_ofs); 221 break; 222 case MO_128: 223 gen_ldo_env_A0(s, dest_ofs, aligned); 224 break; 225 case MO_256: 226 gen_ldy_env_A0(s, dest_ofs, aligned); 227 break; 228 default: 229 g_assert_not_reached(); 230 } 231} 232 233static bool sse_needs_alignment(DisasContext *s, X86DecodedInsn *decode, MemOp ot) 234{ 235 switch (decode->e.vex_class) { 236 case 2: 237 case 4: 238 if ((s->prefix & PREFIX_VEX) || 239 decode->e.vex_special == X86_VEX_SSEUnaligned) { 240 /* MOST legacy SSE instructions require aligned memory operands, but not all. */ 241 return false; 242 } 243 /* fall through */ 244 case 1: 245 return ot >= MO_128; 246 247 default: 248 return false; 249 } 250} 251 252static void gen_load(DisasContext *s, X86DecodedInsn *decode, int opn, TCGv v) 253{ 254 X86DecodedOp *op = &decode->op[opn]; 255 256 switch (op->unit) { 257 case X86_OP_SKIP: 258 return; 259 case X86_OP_SEG: 260 tcg_gen_ld32u_tl(v, tcg_env, 261 offsetof(CPUX86State,segs[op->n].selector)); 262 break; 263#ifndef CONFIG_USER_ONLY 264 case X86_OP_CR: 265 if (op->n == 8) { 266 translator_io_start(&s->base); 267 gen_helper_read_cr8(v, tcg_env); 268 } else { 269 tcg_gen_ld_tl(v, tcg_env, offsetof(CPUX86State, cr[op->n])); 270 } 271 break; 272 case X86_OP_DR: 273 /* CR4.DE tested in the helper. */ 274 gen_helper_get_dr(v, tcg_env, tcg_constant_i32(op->n)); 275 break; 276#endif 277 case X86_OP_INT: 278 if (op->has_ea) { 279 if (v == s->T0 && decode->e.special == X86_SPECIAL_SExtT0) { 280 gen_op_ld_v(s, op->ot | MO_SIGN, v, s->A0); 281 } else { 282 gen_op_ld_v(s, op->ot, v, s->A0); 283 } 284 285 } else if (op->ot == MO_8 && byte_reg_is_xH(s, op->n)) { 286 if (v == s->T0 && decode->e.special == X86_SPECIAL_SExtT0) { 287 tcg_gen_sextract_tl(v, cpu_regs[op->n - 4], 8, 8); 288 } else { 289 tcg_gen_extract_tl(v, cpu_regs[op->n - 4], 8, 8); 290 } 291 292 } else if (op->ot < MO_TL && v == s->T0 && 293 (decode->e.special == X86_SPECIAL_SExtT0 || 294 decode->e.special == X86_SPECIAL_ZExtT0)) { 295 if (decode->e.special == X86_SPECIAL_SExtT0) { 296 tcg_gen_ext_tl(v, cpu_regs[op->n], op->ot | MO_SIGN); 297 } else { 298 tcg_gen_ext_tl(v, cpu_regs[op->n], op->ot); 299 } 300 301 } else { 302 tcg_gen_mov_tl(v, cpu_regs[op->n]); 303 } 304 break; 305 case X86_OP_IMM: 306 tcg_gen_movi_tl(v, op->imm); 307 break; 308 309 case X86_OP_MMX: 310 compute_mmx_offset(op); 311 goto load_vector; 312 313 case X86_OP_SSE: 314 compute_xmm_offset(op); 315 load_vector: 316 if (op->has_ea) { 317 bool aligned = sse_needs_alignment(s, decode, op->ot); 318 gen_load_sse(s, v, op->ot, op->offset, aligned); 319 } 320 break; 321 322 default: 323 g_assert_not_reached(); 324 } 325} 326 327static TCGv_ptr op_ptr(X86DecodedInsn *decode, int opn) 328{ 329 X86DecodedOp *op = &decode->op[opn]; 330 331 assert(op->unit == X86_OP_MMX || op->unit == X86_OP_SSE); 332 if (op->v_ptr) { 333 return op->v_ptr; 334 } 335 op->v_ptr = tcg_temp_new_ptr(); 336 337 /* The temporary points to the MMXReg or ZMMReg. */ 338 tcg_gen_addi_ptr(op->v_ptr, tcg_env, vector_reg_offset(op)); 339 return op->v_ptr; 340} 341 342#define OP_PTR0 op_ptr(decode, 0) 343#define OP_PTR1 op_ptr(decode, 1) 344#define OP_PTR2 op_ptr(decode, 2) 345 346static void gen_writeback(DisasContext *s, X86DecodedInsn *decode, int opn, TCGv v) 347{ 348 X86DecodedOp *op = &decode->op[opn]; 349 switch (op->unit) { 350 case X86_OP_SKIP: 351 break; 352 case X86_OP_SEG: 353 /* Note that gen_movl_seg takes care of interrupt shadow and TF. */ 354 gen_movl_seg(s, op->n, s->T0); 355 break; 356 case X86_OP_INT: 357 if (op->has_ea) { 358 gen_op_st_v(s, op->ot, v, s->A0); 359 } else { 360 gen_op_mov_reg_v(s, op->ot, op->n, v); 361 } 362 break; 363 case X86_OP_MMX: 364 break; 365 case X86_OP_SSE: 366 if (!op->has_ea && (s->prefix & PREFIX_VEX) && op->ot <= MO_128) { 367 tcg_gen_gvec_dup_imm(MO_64, 368 offsetof(CPUX86State, xmm_regs[op->n].ZMM_X(1)), 369 16, 16, 0); 370 } 371 break; 372#ifndef CONFIG_USER_ONLY 373 case X86_OP_CR: 374 if (op->n == 8) { 375 translator_io_start(&s->base); 376 } 377 gen_helper_write_crN(tcg_env, tcg_constant_i32(op->n), v); 378 s->base.is_jmp = DISAS_EOB_NEXT; 379 break; 380 case X86_OP_DR: 381 /* CR4.DE tested in the helper. */ 382 gen_helper_set_dr(tcg_env, tcg_constant_i32(op->n), v); 383 s->base.is_jmp = DISAS_EOB_NEXT; 384 break; 385#endif 386 default: 387 g_assert_not_reached(); 388 } 389 op->unit = X86_OP_SKIP; 390} 391 392static inline int vector_len(DisasContext *s, X86DecodedInsn *decode) 393{ 394 if (decode->e.special == X86_SPECIAL_MMX && 395 !(s->prefix & (PREFIX_DATA | PREFIX_REPZ | PREFIX_REPNZ))) { 396 return 8; 397 } 398 return s->vex_l ? 32 : 16; 399} 400 401static void prepare_update1_cc(X86DecodedInsn *decode, DisasContext *s, CCOp op) 402{ 403 decode->cc_dst = s->T0; 404 decode->cc_op = op; 405} 406 407static void prepare_update2_cc(X86DecodedInsn *decode, DisasContext *s, CCOp op) 408{ 409 decode->cc_src = s->T1; 410 decode->cc_dst = s->T0; 411 decode->cc_op = op; 412} 413 414static void prepare_update_cc_incdec(X86DecodedInsn *decode, DisasContext *s, CCOp op) 415{ 416 gen_compute_eflags_c(s, s->T1); 417 prepare_update2_cc(decode, s, op); 418} 419 420static void prepare_update3_cc(X86DecodedInsn *decode, DisasContext *s, CCOp op, TCGv reg) 421{ 422 decode->cc_src2 = reg; 423 decode->cc_src = s->T1; 424 decode->cc_dst = s->T0; 425 decode->cc_op = op; 426} 427 428/* Set up decode->cc_* to modify CF while keeping other flags unchanged. */ 429static void prepare_update_cf(X86DecodedInsn *decode, DisasContext *s, TCGv cf) 430{ 431 switch (s->cc_op) { 432 case CC_OP_ADOX: 433 case CC_OP_ADCOX: 434 decode->cc_src2 = cpu_cc_src2; 435 decode->cc_src = cpu_cc_src; 436 decode->cc_op = CC_OP_ADCOX; 437 break; 438 439 case CC_OP_EFLAGS: 440 case CC_OP_ADCX: 441 decode->cc_src = cpu_cc_src; 442 decode->cc_op = CC_OP_ADCX; 443 break; 444 445 default: 446 decode->cc_src = tcg_temp_new(); 447 gen_mov_eflags(s, decode->cc_src); 448 decode->cc_op = CC_OP_ADCX; 449 break; 450 } 451 decode->cc_dst = cf; 452} 453 454static void gen_store_sse(DisasContext *s, X86DecodedInsn *decode, int src_ofs) 455{ 456 MemOp ot = decode->op[0].ot; 457 int vec_len = vector_len(s, decode); 458 bool aligned = sse_needs_alignment(s, decode, ot); 459 460 if (!decode->op[0].has_ea) { 461 tcg_gen_gvec_mov(MO_64, decode->op[0].offset, src_ofs, vec_len, vec_len); 462 return; 463 } 464 465 switch (ot) { 466 case MO_64: 467 gen_stq_env_A0(s, src_ofs); 468 break; 469 case MO_128: 470 gen_sto_env_A0(s, src_ofs, aligned); 471 break; 472 case MO_256: 473 gen_sty_env_A0(s, src_ofs, aligned); 474 break; 475 default: 476 g_assert_not_reached(); 477 } 478} 479 480static void gen_helper_pavgusb(TCGv_ptr env, TCGv_ptr reg_a, TCGv_ptr reg_b) 481{ 482 gen_helper_pavgb_mmx(env, reg_a, reg_a, reg_b); 483} 484 485#define FN_3DNOW_MOVE ((SSEFunc_0_epp) (uintptr_t) 1) 486static const SSEFunc_0_epp fns_3dnow[] = { 487 [0x0c] = gen_helper_pi2fw, 488 [0x0d] = gen_helper_pi2fd, 489 [0x1c] = gen_helper_pf2iw, 490 [0x1d] = gen_helper_pf2id, 491 [0x8a] = gen_helper_pfnacc, 492 [0x8e] = gen_helper_pfpnacc, 493 [0x90] = gen_helper_pfcmpge, 494 [0x94] = gen_helper_pfmin, 495 [0x96] = gen_helper_pfrcp, 496 [0x97] = gen_helper_pfrsqrt, 497 [0x9a] = gen_helper_pfsub, 498 [0x9e] = gen_helper_pfadd, 499 [0xa0] = gen_helper_pfcmpgt, 500 [0xa4] = gen_helper_pfmax, 501 [0xa6] = FN_3DNOW_MOVE, /* PFRCPIT1; no need to actually increase precision */ 502 [0xa7] = FN_3DNOW_MOVE, /* PFRSQIT1 */ 503 [0xb6] = FN_3DNOW_MOVE, /* PFRCPIT2 */ 504 [0xaa] = gen_helper_pfsubr, 505 [0xae] = gen_helper_pfacc, 506 [0xb0] = gen_helper_pfcmpeq, 507 [0xb4] = gen_helper_pfmul, 508 [0xb7] = gen_helper_pmulhrw_mmx, 509 [0xbb] = gen_helper_pswapd, 510 [0xbf] = gen_helper_pavgusb, 511}; 512 513static void gen_3dnow(DisasContext *s, X86DecodedInsn *decode) 514{ 515 uint8_t b = decode->immediate; 516 SSEFunc_0_epp fn = b < ARRAY_SIZE(fns_3dnow) ? fns_3dnow[b] : NULL; 517 518 if (!fn) { 519 gen_illegal_opcode(s); 520 return; 521 } 522 if (s->flags & HF_TS_MASK) { 523 gen_NM_exception(s); 524 return; 525 } 526 if (s->flags & HF_EM_MASK) { 527 gen_illegal_opcode(s); 528 return; 529 } 530 531 gen_helper_enter_mmx(tcg_env); 532 if (fn == FN_3DNOW_MOVE) { 533 tcg_gen_ld_i64(s->tmp1_i64, tcg_env, decode->op[1].offset); 534 tcg_gen_st_i64(s->tmp1_i64, tcg_env, decode->op[0].offset); 535 } else { 536 fn(tcg_env, OP_PTR0, OP_PTR1); 537 } 538} 539 540/* 541 * 00 = v*ps Vps, Hps, Wpd 542 * 66 = v*pd Vpd, Hpd, Wps 543 * f3 = v*ss Vss, Hss, Wps 544 * f2 = v*sd Vsd, Hsd, Wps 545 */ 546static inline void gen_unary_fp_sse(DisasContext *s, X86DecodedInsn *decode, 547 SSEFunc_0_epp pd_xmm, SSEFunc_0_epp ps_xmm, 548 SSEFunc_0_epp pd_ymm, SSEFunc_0_epp ps_ymm, 549 SSEFunc_0_eppp sd, SSEFunc_0_eppp ss) 550{ 551 if ((s->prefix & (PREFIX_REPZ | PREFIX_REPNZ)) != 0) { 552 SSEFunc_0_eppp fn = s->prefix & PREFIX_REPZ ? ss : sd; 553 if (!fn) { 554 gen_illegal_opcode(s); 555 return; 556 } 557 fn(tcg_env, OP_PTR0, OP_PTR1, OP_PTR2); 558 } else { 559 SSEFunc_0_epp ps, pd, fn; 560 ps = s->vex_l ? ps_ymm : ps_xmm; 561 pd = s->vex_l ? pd_ymm : pd_xmm; 562 fn = s->prefix & PREFIX_DATA ? pd : ps; 563 if (!fn) { 564 gen_illegal_opcode(s); 565 return; 566 } 567 fn(tcg_env, OP_PTR0, OP_PTR2); 568 } 569} 570#define UNARY_FP_SSE(uname, lname) \ 571static void gen_##uname(DisasContext *s, X86DecodedInsn *decode) \ 572{ \ 573 gen_unary_fp_sse(s, decode, \ 574 gen_helper_##lname##pd_xmm, \ 575 gen_helper_##lname##ps_xmm, \ 576 gen_helper_##lname##pd_ymm, \ 577 gen_helper_##lname##ps_ymm, \ 578 gen_helper_##lname##sd, \ 579 gen_helper_##lname##ss); \ 580} 581UNARY_FP_SSE(VSQRT, sqrt) 582 583/* 584 * 00 = v*ps Vps, Hps, Wpd 585 * 66 = v*pd Vpd, Hpd, Wps 586 * f3 = v*ss Vss, Hss, Wps 587 * f2 = v*sd Vsd, Hsd, Wps 588 */ 589static inline void gen_fp_sse(DisasContext *s, X86DecodedInsn *decode, 590 SSEFunc_0_eppp pd_xmm, SSEFunc_0_eppp ps_xmm, 591 SSEFunc_0_eppp pd_ymm, SSEFunc_0_eppp ps_ymm, 592 SSEFunc_0_eppp sd, SSEFunc_0_eppp ss) 593{ 594 SSEFunc_0_eppp ps, pd, fn; 595 if ((s->prefix & (PREFIX_REPZ | PREFIX_REPNZ)) != 0) { 596 fn = s->prefix & PREFIX_REPZ ? ss : sd; 597 } else { 598 ps = s->vex_l ? ps_ymm : ps_xmm; 599 pd = s->vex_l ? pd_ymm : pd_xmm; 600 fn = s->prefix & PREFIX_DATA ? pd : ps; 601 } 602 if (fn) { 603 fn(tcg_env, OP_PTR0, OP_PTR1, OP_PTR2); 604 } else { 605 gen_illegal_opcode(s); 606 } 607} 608 609#define FP_SSE(uname, lname) \ 610static void gen_##uname(DisasContext *s, X86DecodedInsn *decode) \ 611{ \ 612 gen_fp_sse(s, decode, \ 613 gen_helper_##lname##pd_xmm, \ 614 gen_helper_##lname##ps_xmm, \ 615 gen_helper_##lname##pd_ymm, \ 616 gen_helper_##lname##ps_ymm, \ 617 gen_helper_##lname##sd, \ 618 gen_helper_##lname##ss); \ 619} 620FP_SSE(VADD, add) 621FP_SSE(VMUL, mul) 622FP_SSE(VSUB, sub) 623FP_SSE(VMIN, min) 624FP_SSE(VDIV, div) 625FP_SSE(VMAX, max) 626 627#define FMA_SSE_PACKED(uname, ptr0, ptr1, ptr2, even, odd) \ 628static void gen_##uname##Px(DisasContext *s, X86DecodedInsn *decode) \ 629{ \ 630 SSEFunc_0_eppppii xmm = s->vex_w ? gen_helper_fma4pd_xmm : gen_helper_fma4ps_xmm; \ 631 SSEFunc_0_eppppii ymm = s->vex_w ? gen_helper_fma4pd_ymm : gen_helper_fma4ps_ymm; \ 632 SSEFunc_0_eppppii fn = s->vex_l ? ymm : xmm; \ 633 \ 634 fn(tcg_env, OP_PTR0, ptr0, ptr1, ptr2, \ 635 tcg_constant_i32(even), \ 636 tcg_constant_i32((even) ^ (odd))); \ 637} 638 639#define FMA_SSE(uname, ptr0, ptr1, ptr2, flags) \ 640FMA_SSE_PACKED(uname, ptr0, ptr1, ptr2, flags, flags) \ 641static void gen_##uname##Sx(DisasContext *s, X86DecodedInsn *decode) \ 642{ \ 643 SSEFunc_0_eppppi fn = s->vex_w ? gen_helper_fma4sd : gen_helper_fma4ss; \ 644 \ 645 fn(tcg_env, OP_PTR0, ptr0, ptr1, ptr2, \ 646 tcg_constant_i32(flags)); \ 647} \ 648 649FMA_SSE(VFMADD231, OP_PTR1, OP_PTR2, OP_PTR0, 0) 650FMA_SSE(VFMADD213, OP_PTR1, OP_PTR0, OP_PTR2, 0) 651FMA_SSE(VFMADD132, OP_PTR0, OP_PTR2, OP_PTR1, 0) 652 653FMA_SSE(VFNMADD231, OP_PTR1, OP_PTR2, OP_PTR0, float_muladd_negate_product) 654FMA_SSE(VFNMADD213, OP_PTR1, OP_PTR0, OP_PTR2, float_muladd_negate_product) 655FMA_SSE(VFNMADD132, OP_PTR0, OP_PTR2, OP_PTR1, float_muladd_negate_product) 656 657FMA_SSE(VFMSUB231, OP_PTR1, OP_PTR2, OP_PTR0, float_muladd_negate_c) 658FMA_SSE(VFMSUB213, OP_PTR1, OP_PTR0, OP_PTR2, float_muladd_negate_c) 659FMA_SSE(VFMSUB132, OP_PTR0, OP_PTR2, OP_PTR1, float_muladd_negate_c) 660 661FMA_SSE(VFNMSUB231, OP_PTR1, OP_PTR2, OP_PTR0, float_muladd_negate_c|float_muladd_negate_product) 662FMA_SSE(VFNMSUB213, OP_PTR1, OP_PTR0, OP_PTR2, float_muladd_negate_c|float_muladd_negate_product) 663FMA_SSE(VFNMSUB132, OP_PTR0, OP_PTR2, OP_PTR1, float_muladd_negate_c|float_muladd_negate_product) 664 665FMA_SSE_PACKED(VFMADDSUB231, OP_PTR1, OP_PTR2, OP_PTR0, float_muladd_negate_c, 0) 666FMA_SSE_PACKED(VFMADDSUB213, OP_PTR1, OP_PTR0, OP_PTR2, float_muladd_negate_c, 0) 667FMA_SSE_PACKED(VFMADDSUB132, OP_PTR0, OP_PTR2, OP_PTR1, float_muladd_negate_c, 0) 668 669FMA_SSE_PACKED(VFMSUBADD231, OP_PTR1, OP_PTR2, OP_PTR0, 0, float_muladd_negate_c) 670FMA_SSE_PACKED(VFMSUBADD213, OP_PTR1, OP_PTR0, OP_PTR2, 0, float_muladd_negate_c) 671FMA_SSE_PACKED(VFMSUBADD132, OP_PTR0, OP_PTR2, OP_PTR1, 0, float_muladd_negate_c) 672 673#define FP_UNPACK_SSE(uname, lname) \ 674static void gen_##uname(DisasContext *s, X86DecodedInsn *decode) \ 675{ \ 676 /* PS maps to the DQ integer instruction, PD maps to QDQ. */ \ 677 gen_fp_sse(s, decode, \ 678 gen_helper_##lname##qdq_xmm, \ 679 gen_helper_##lname##dq_xmm, \ 680 gen_helper_##lname##qdq_ymm, \ 681 gen_helper_##lname##dq_ymm, \ 682 NULL, NULL); \ 683} 684FP_UNPACK_SSE(VUNPCKLPx, punpckl) 685FP_UNPACK_SSE(VUNPCKHPx, punpckh) 686 687/* 688 * 00 = v*ps Vps, Wpd 689 * f3 = v*ss Vss, Wps 690 */ 691static inline void gen_unary_fp32_sse(DisasContext *s, X86DecodedInsn *decode, 692 SSEFunc_0_epp ps_xmm, 693 SSEFunc_0_epp ps_ymm, 694 SSEFunc_0_eppp ss) 695{ 696 if ((s->prefix & (PREFIX_DATA | PREFIX_REPNZ)) != 0) { 697 goto illegal_op; 698 } else if (s->prefix & PREFIX_REPZ) { 699 if (!ss) { 700 goto illegal_op; 701 } 702 ss(tcg_env, OP_PTR0, OP_PTR1, OP_PTR2); 703 } else { 704 SSEFunc_0_epp fn = s->vex_l ? ps_ymm : ps_xmm; 705 if (!fn) { 706 goto illegal_op; 707 } 708 fn(tcg_env, OP_PTR0, OP_PTR2); 709 } 710 return; 711 712illegal_op: 713 gen_illegal_opcode(s); 714} 715#define UNARY_FP32_SSE(uname, lname) \ 716static void gen_##uname(DisasContext *s, X86DecodedInsn *decode) \ 717{ \ 718 gen_unary_fp32_sse(s, decode, \ 719 gen_helper_##lname##ps_xmm, \ 720 gen_helper_##lname##ps_ymm, \ 721 gen_helper_##lname##ss); \ 722} 723UNARY_FP32_SSE(VRSQRT, rsqrt) 724UNARY_FP32_SSE(VRCP, rcp) 725 726/* 727 * 66 = v*pd Vpd, Hpd, Wpd 728 * f2 = v*ps Vps, Hps, Wps 729 */ 730static inline void gen_horizontal_fp_sse(DisasContext *s, X86DecodedInsn *decode, 731 SSEFunc_0_eppp pd_xmm, SSEFunc_0_eppp ps_xmm, 732 SSEFunc_0_eppp pd_ymm, SSEFunc_0_eppp ps_ymm) 733{ 734 SSEFunc_0_eppp ps, pd, fn; 735 ps = s->vex_l ? ps_ymm : ps_xmm; 736 pd = s->vex_l ? pd_ymm : pd_xmm; 737 fn = s->prefix & PREFIX_DATA ? pd : ps; 738 fn(tcg_env, OP_PTR0, OP_PTR1, OP_PTR2); 739} 740#define HORIZONTAL_FP_SSE(uname, lname) \ 741static void gen_##uname(DisasContext *s, X86DecodedInsn *decode) \ 742{ \ 743 gen_horizontal_fp_sse(s, decode, \ 744 gen_helper_##lname##pd_xmm, gen_helper_##lname##ps_xmm, \ 745 gen_helper_##lname##pd_ymm, gen_helper_##lname##ps_ymm); \ 746} 747HORIZONTAL_FP_SSE(VHADD, hadd) 748HORIZONTAL_FP_SSE(VHSUB, hsub) 749HORIZONTAL_FP_SSE(VADDSUB, addsub) 750 751static inline void gen_ternary_sse(DisasContext *s, X86DecodedInsn *decode, 752 int op3, SSEFunc_0_epppp xmm, SSEFunc_0_epppp ymm) 753{ 754 SSEFunc_0_epppp fn = s->vex_l ? ymm : xmm; 755 TCGv_ptr ptr3 = tcg_temp_new_ptr(); 756 757 /* The format of the fourth input is Lx */ 758 tcg_gen_addi_ptr(ptr3, tcg_env, ZMM_OFFSET(op3)); 759 fn(tcg_env, OP_PTR0, OP_PTR1, OP_PTR2, ptr3); 760} 761#define TERNARY_SSE(uname, uvname, lname) \ 762static void gen_##uvname(DisasContext *s, X86DecodedInsn *decode) \ 763{ \ 764 gen_ternary_sse(s, decode, (uint8_t)decode->immediate >> 4, \ 765 gen_helper_##lname##_xmm, gen_helper_##lname##_ymm); \ 766} \ 767static void gen_##uname(DisasContext *s, X86DecodedInsn *decode) \ 768{ \ 769 gen_ternary_sse(s, decode, 0, \ 770 gen_helper_##lname##_xmm, gen_helper_##lname##_ymm); \ 771} 772TERNARY_SSE(BLENDVPS, VBLENDVPS, blendvps) 773TERNARY_SSE(BLENDVPD, VBLENDVPD, blendvpd) 774TERNARY_SSE(PBLENDVB, VPBLENDVB, pblendvb) 775 776static inline void gen_binary_imm_sse(DisasContext *s, X86DecodedInsn *decode, 777 SSEFunc_0_epppi xmm, SSEFunc_0_epppi ymm) 778{ 779 TCGv_i32 imm = tcg_constant8u_i32(decode->immediate); 780 if (!s->vex_l) { 781 xmm(tcg_env, OP_PTR0, OP_PTR1, OP_PTR2, imm); 782 } else { 783 ymm(tcg_env, OP_PTR0, OP_PTR1, OP_PTR2, imm); 784 } 785} 786 787#define BINARY_IMM_SSE(uname, lname) \ 788static void gen_##uname(DisasContext *s, X86DecodedInsn *decode) \ 789{ \ 790 gen_binary_imm_sse(s, decode, \ 791 gen_helper_##lname##_xmm, \ 792 gen_helper_##lname##_ymm); \ 793} 794 795BINARY_IMM_SSE(VBLENDPD, blendpd) 796BINARY_IMM_SSE(VBLENDPS, blendps) 797BINARY_IMM_SSE(VPBLENDW, pblendw) 798BINARY_IMM_SSE(VDDPS, dpps) 799#define gen_helper_dppd_ymm NULL 800BINARY_IMM_SSE(VDDPD, dppd) 801BINARY_IMM_SSE(VMPSADBW, mpsadbw) 802BINARY_IMM_SSE(PCLMULQDQ, pclmulqdq) 803 804 805#define UNARY_INT_GVEC(uname, func, ...) \ 806static void gen_##uname(DisasContext *s, X86DecodedInsn *decode) \ 807{ \ 808 int vec_len = vector_len(s, decode); \ 809 \ 810 func(__VA_ARGS__, decode->op[0].offset, \ 811 decode->op[2].offset, vec_len, vec_len); \ 812} 813UNARY_INT_GVEC(PABSB, tcg_gen_gvec_abs, MO_8) 814UNARY_INT_GVEC(PABSW, tcg_gen_gvec_abs, MO_16) 815UNARY_INT_GVEC(PABSD, tcg_gen_gvec_abs, MO_32) 816UNARY_INT_GVEC(VBROADCASTx128, tcg_gen_gvec_dup_mem, MO_128) 817UNARY_INT_GVEC(VPBROADCASTB, tcg_gen_gvec_dup_mem, MO_8) 818UNARY_INT_GVEC(VPBROADCASTW, tcg_gen_gvec_dup_mem, MO_16) 819UNARY_INT_GVEC(VPBROADCASTD, tcg_gen_gvec_dup_mem, MO_32) 820UNARY_INT_GVEC(VPBROADCASTQ, tcg_gen_gvec_dup_mem, MO_64) 821 822 823#define BINARY_INT_GVEC(uname, func, ...) \ 824static void gen_##uname(DisasContext *s, X86DecodedInsn *decode) \ 825{ \ 826 int vec_len = vector_len(s, decode); \ 827 \ 828 func(__VA_ARGS__, \ 829 decode->op[0].offset, decode->op[1].offset, \ 830 decode->op[2].offset, vec_len, vec_len); \ 831} 832 833BINARY_INT_GVEC(PADDB, tcg_gen_gvec_add, MO_8) 834BINARY_INT_GVEC(PADDW, tcg_gen_gvec_add, MO_16) 835BINARY_INT_GVEC(PADDD, tcg_gen_gvec_add, MO_32) 836BINARY_INT_GVEC(PADDQ, tcg_gen_gvec_add, MO_64) 837BINARY_INT_GVEC(PADDSB, tcg_gen_gvec_ssadd, MO_8) 838BINARY_INT_GVEC(PADDSW, tcg_gen_gvec_ssadd, MO_16) 839BINARY_INT_GVEC(PADDUSB, tcg_gen_gvec_usadd, MO_8) 840BINARY_INT_GVEC(PADDUSW, tcg_gen_gvec_usadd, MO_16) 841BINARY_INT_GVEC(PAND, tcg_gen_gvec_and, MO_64) 842BINARY_INT_GVEC(PCMPEQB, tcg_gen_gvec_cmp, TCG_COND_EQ, MO_8) 843BINARY_INT_GVEC(PCMPEQD, tcg_gen_gvec_cmp, TCG_COND_EQ, MO_32) 844BINARY_INT_GVEC(PCMPEQW, tcg_gen_gvec_cmp, TCG_COND_EQ, MO_16) 845BINARY_INT_GVEC(PCMPEQQ, tcg_gen_gvec_cmp, TCG_COND_EQ, MO_64) 846BINARY_INT_GVEC(PCMPGTB, tcg_gen_gvec_cmp, TCG_COND_GT, MO_8) 847BINARY_INT_GVEC(PCMPGTW, tcg_gen_gvec_cmp, TCG_COND_GT, MO_16) 848BINARY_INT_GVEC(PCMPGTD, tcg_gen_gvec_cmp, TCG_COND_GT, MO_32) 849BINARY_INT_GVEC(PCMPGTQ, tcg_gen_gvec_cmp, TCG_COND_GT, MO_64) 850BINARY_INT_GVEC(PMAXSB, tcg_gen_gvec_smax, MO_8) 851BINARY_INT_GVEC(PMAXSW, tcg_gen_gvec_smax, MO_16) 852BINARY_INT_GVEC(PMAXSD, tcg_gen_gvec_smax, MO_32) 853BINARY_INT_GVEC(PMAXUB, tcg_gen_gvec_umax, MO_8) 854BINARY_INT_GVEC(PMAXUW, tcg_gen_gvec_umax, MO_16) 855BINARY_INT_GVEC(PMAXUD, tcg_gen_gvec_umax, MO_32) 856BINARY_INT_GVEC(PMINSB, tcg_gen_gvec_smin, MO_8) 857BINARY_INT_GVEC(PMINSW, tcg_gen_gvec_smin, MO_16) 858BINARY_INT_GVEC(PMINSD, tcg_gen_gvec_smin, MO_32) 859BINARY_INT_GVEC(PMINUB, tcg_gen_gvec_umin, MO_8) 860BINARY_INT_GVEC(PMINUW, tcg_gen_gvec_umin, MO_16) 861BINARY_INT_GVEC(PMINUD, tcg_gen_gvec_umin, MO_32) 862BINARY_INT_GVEC(PMULLW, tcg_gen_gvec_mul, MO_16) 863BINARY_INT_GVEC(PMULLD, tcg_gen_gvec_mul, MO_32) 864BINARY_INT_GVEC(POR, tcg_gen_gvec_or, MO_64) 865BINARY_INT_GVEC(PSUBB, tcg_gen_gvec_sub, MO_8) 866BINARY_INT_GVEC(PSUBW, tcg_gen_gvec_sub, MO_16) 867BINARY_INT_GVEC(PSUBD, tcg_gen_gvec_sub, MO_32) 868BINARY_INT_GVEC(PSUBQ, tcg_gen_gvec_sub, MO_64) 869BINARY_INT_GVEC(PSUBSB, tcg_gen_gvec_sssub, MO_8) 870BINARY_INT_GVEC(PSUBSW, tcg_gen_gvec_sssub, MO_16) 871BINARY_INT_GVEC(PSUBUSB, tcg_gen_gvec_ussub, MO_8) 872BINARY_INT_GVEC(PSUBUSW, tcg_gen_gvec_ussub, MO_16) 873BINARY_INT_GVEC(PXOR, tcg_gen_gvec_xor, MO_64) 874 875 876/* 877 * 00 = p* Pq, Qq (if mmx not NULL; no VEX) 878 * 66 = vp* Vx, Hx, Wx 879 * 880 * These are really the same encoding, because 1) V is the same as P when VEX.V 881 * is not present 2) P and Q are the same as H and W apart from MM/XMM 882 */ 883static inline void gen_binary_int_sse(DisasContext *s, X86DecodedInsn *decode, 884 SSEFunc_0_eppp mmx, SSEFunc_0_eppp xmm, SSEFunc_0_eppp ymm) 885{ 886 assert(!!mmx == !!(decode->e.special == X86_SPECIAL_MMX)); 887 888 if (mmx && (s->prefix & PREFIX_VEX) && !(s->prefix & PREFIX_DATA)) { 889 /* VEX encoding is not applicable to MMX instructions. */ 890 gen_illegal_opcode(s); 891 return; 892 } 893 if (!(s->prefix & PREFIX_DATA)) { 894 mmx(tcg_env, OP_PTR0, OP_PTR1, OP_PTR2); 895 } else if (!s->vex_l) { 896 xmm(tcg_env, OP_PTR0, OP_PTR1, OP_PTR2); 897 } else { 898 ymm(tcg_env, OP_PTR0, OP_PTR1, OP_PTR2); 899 } 900} 901 902 903#define BINARY_INT_MMX(uname, lname) \ 904static void gen_##uname(DisasContext *s, X86DecodedInsn *decode) \ 905{ \ 906 gen_binary_int_sse(s, decode, \ 907 gen_helper_##lname##_mmx, \ 908 gen_helper_##lname##_xmm, \ 909 gen_helper_##lname##_ymm); \ 910} 911BINARY_INT_MMX(PUNPCKLBW, punpcklbw) 912BINARY_INT_MMX(PUNPCKLWD, punpcklwd) 913BINARY_INT_MMX(PUNPCKLDQ, punpckldq) 914BINARY_INT_MMX(PACKSSWB, packsswb) 915BINARY_INT_MMX(PACKUSWB, packuswb) 916BINARY_INT_MMX(PUNPCKHBW, punpckhbw) 917BINARY_INT_MMX(PUNPCKHWD, punpckhwd) 918BINARY_INT_MMX(PUNPCKHDQ, punpckhdq) 919BINARY_INT_MMX(PACKSSDW, packssdw) 920 921BINARY_INT_MMX(PAVGB, pavgb) 922BINARY_INT_MMX(PAVGW, pavgw) 923BINARY_INT_MMX(PMADDWD, pmaddwd) 924BINARY_INT_MMX(PMULHUW, pmulhuw) 925BINARY_INT_MMX(PMULHW, pmulhw) 926BINARY_INT_MMX(PMULUDQ, pmuludq) 927BINARY_INT_MMX(PSADBW, psadbw) 928 929BINARY_INT_MMX(PSLLW_r, psllw) 930BINARY_INT_MMX(PSLLD_r, pslld) 931BINARY_INT_MMX(PSLLQ_r, psllq) 932BINARY_INT_MMX(PSRLW_r, psrlw) 933BINARY_INT_MMX(PSRLD_r, psrld) 934BINARY_INT_MMX(PSRLQ_r, psrlq) 935BINARY_INT_MMX(PSRAW_r, psraw) 936BINARY_INT_MMX(PSRAD_r, psrad) 937 938BINARY_INT_MMX(PHADDW, phaddw) 939BINARY_INT_MMX(PHADDSW, phaddsw) 940BINARY_INT_MMX(PHADDD, phaddd) 941BINARY_INT_MMX(PHSUBW, phsubw) 942BINARY_INT_MMX(PHSUBSW, phsubsw) 943BINARY_INT_MMX(PHSUBD, phsubd) 944BINARY_INT_MMX(PMADDUBSW, pmaddubsw) 945BINARY_INT_MMX(PSHUFB, pshufb) 946BINARY_INT_MMX(PSIGNB, psignb) 947BINARY_INT_MMX(PSIGNW, psignw) 948BINARY_INT_MMX(PSIGND, psignd) 949BINARY_INT_MMX(PMULHRSW, pmulhrsw) 950 951/* Instructions with no MMX equivalent. */ 952#define BINARY_INT_SSE(uname, lname) \ 953static void gen_##uname(DisasContext *s, X86DecodedInsn *decode) \ 954{ \ 955 gen_binary_int_sse(s, decode, \ 956 NULL, \ 957 gen_helper_##lname##_xmm, \ 958 gen_helper_##lname##_ymm); \ 959} 960 961/* Instructions with no MMX equivalent. */ 962BINARY_INT_SSE(PUNPCKLQDQ, punpcklqdq) 963BINARY_INT_SSE(PUNPCKHQDQ, punpckhqdq) 964BINARY_INT_SSE(VPACKUSDW, packusdw) 965BINARY_INT_SSE(VPERMILPS, vpermilps) 966BINARY_INT_SSE(VPERMILPD, vpermilpd) 967BINARY_INT_SSE(VMASKMOVPS, vpmaskmovd) 968BINARY_INT_SSE(VMASKMOVPD, vpmaskmovq) 969 970BINARY_INT_SSE(PMULDQ, pmuldq) 971 972BINARY_INT_SSE(VAESDEC, aesdec) 973BINARY_INT_SSE(VAESDECLAST, aesdeclast) 974BINARY_INT_SSE(VAESENC, aesenc) 975BINARY_INT_SSE(VAESENCLAST, aesenclast) 976 977#define UNARY_CMP_SSE(uname, lname) \ 978static void gen_##uname(DisasContext *s, X86DecodedInsn *decode) \ 979{ \ 980 if (!s->vex_l) { \ 981 gen_helper_##lname##_xmm(tcg_env, OP_PTR1, OP_PTR2); \ 982 } else { \ 983 gen_helper_##lname##_ymm(tcg_env, OP_PTR1, OP_PTR2); \ 984 } \ 985 assume_cc_op(s, CC_OP_EFLAGS); \ 986} 987UNARY_CMP_SSE(VPTEST, ptest) 988UNARY_CMP_SSE(VTESTPS, vtestps) 989UNARY_CMP_SSE(VTESTPD, vtestpd) 990 991static inline void gen_unary_int_sse(DisasContext *s, X86DecodedInsn *decode, 992 SSEFunc_0_epp xmm, SSEFunc_0_epp ymm) 993{ 994 if (!s->vex_l) { 995 xmm(tcg_env, OP_PTR0, OP_PTR2); 996 } else { 997 ymm(tcg_env, OP_PTR0, OP_PTR2); 998 } 999} 1000 1001#define UNARY_INT_SSE(uname, lname) \ 1002static void gen_##uname(DisasContext *s, X86DecodedInsn *decode) \ 1003{ \ 1004 gen_unary_int_sse(s, decode, \ 1005 gen_helper_##lname##_xmm, \ 1006 gen_helper_##lname##_ymm); \ 1007} 1008 1009UNARY_INT_SSE(VPMOVSXBW, pmovsxbw) 1010UNARY_INT_SSE(VPMOVSXBD, pmovsxbd) 1011UNARY_INT_SSE(VPMOVSXBQ, pmovsxbq) 1012UNARY_INT_SSE(VPMOVSXWD, pmovsxwd) 1013UNARY_INT_SSE(VPMOVSXWQ, pmovsxwq) 1014UNARY_INT_SSE(VPMOVSXDQ, pmovsxdq) 1015 1016UNARY_INT_SSE(VPMOVZXBW, pmovzxbw) 1017UNARY_INT_SSE(VPMOVZXBD, pmovzxbd) 1018UNARY_INT_SSE(VPMOVZXBQ, pmovzxbq) 1019UNARY_INT_SSE(VPMOVZXWD, pmovzxwd) 1020UNARY_INT_SSE(VPMOVZXWQ, pmovzxwq) 1021UNARY_INT_SSE(VPMOVZXDQ, pmovzxdq) 1022 1023UNARY_INT_SSE(VMOVSLDUP, pmovsldup) 1024UNARY_INT_SSE(VMOVSHDUP, pmovshdup) 1025UNARY_INT_SSE(VMOVDDUP, pmovdldup) 1026 1027UNARY_INT_SSE(VCVTDQ2PD, cvtdq2pd) 1028UNARY_INT_SSE(VCVTPD2DQ, cvtpd2dq) 1029UNARY_INT_SSE(VCVTTPD2DQ, cvttpd2dq) 1030UNARY_INT_SSE(VCVTDQ2PS, cvtdq2ps) 1031UNARY_INT_SSE(VCVTPS2DQ, cvtps2dq) 1032UNARY_INT_SSE(VCVTTPS2DQ, cvttps2dq) 1033UNARY_INT_SSE(VCVTPH2PS, cvtph2ps) 1034 1035 1036static inline void gen_unary_imm_sse(DisasContext *s, X86DecodedInsn *decode, 1037 SSEFunc_0_ppi xmm, SSEFunc_0_ppi ymm) 1038{ 1039 TCGv_i32 imm = tcg_constant8u_i32(decode->immediate); 1040 if (!s->vex_l) { 1041 xmm(OP_PTR0, OP_PTR1, imm); 1042 } else { 1043 ymm(OP_PTR0, OP_PTR1, imm); 1044 } 1045} 1046 1047#define UNARY_IMM_SSE(uname, lname) \ 1048static void gen_##uname(DisasContext *s, X86DecodedInsn *decode) \ 1049{ \ 1050 gen_unary_imm_sse(s, decode, \ 1051 gen_helper_##lname##_xmm, \ 1052 gen_helper_##lname##_ymm); \ 1053} 1054 1055UNARY_IMM_SSE(PSHUFD, pshufd) 1056UNARY_IMM_SSE(PSHUFHW, pshufhw) 1057UNARY_IMM_SSE(PSHUFLW, pshuflw) 1058#define gen_helper_vpermq_xmm NULL 1059UNARY_IMM_SSE(VPERMQ, vpermq) 1060UNARY_IMM_SSE(VPERMILPS_i, vpermilps_imm) 1061UNARY_IMM_SSE(VPERMILPD_i, vpermilpd_imm) 1062 1063static inline void gen_unary_imm_fp_sse(DisasContext *s, X86DecodedInsn *decode, 1064 SSEFunc_0_eppi xmm, SSEFunc_0_eppi ymm) 1065{ 1066 TCGv_i32 imm = tcg_constant8u_i32(decode->immediate); 1067 if (!s->vex_l) { 1068 xmm(tcg_env, OP_PTR0, OP_PTR1, imm); 1069 } else { 1070 ymm(tcg_env, OP_PTR0, OP_PTR1, imm); 1071 } 1072} 1073 1074#define UNARY_IMM_FP_SSE(uname, lname) \ 1075static void gen_##uname(DisasContext *s, X86DecodedInsn *decode) \ 1076{ \ 1077 gen_unary_imm_fp_sse(s, decode, \ 1078 gen_helper_##lname##_xmm, \ 1079 gen_helper_##lname##_ymm); \ 1080} 1081 1082UNARY_IMM_FP_SSE(VROUNDPS, roundps) 1083UNARY_IMM_FP_SSE(VROUNDPD, roundpd) 1084 1085static inline void gen_vexw_avx(DisasContext *s, X86DecodedInsn *decode, 1086 SSEFunc_0_eppp d_xmm, SSEFunc_0_eppp q_xmm, 1087 SSEFunc_0_eppp d_ymm, SSEFunc_0_eppp q_ymm) 1088{ 1089 SSEFunc_0_eppp d = s->vex_l ? d_ymm : d_xmm; 1090 SSEFunc_0_eppp q = s->vex_l ? q_ymm : q_xmm; 1091 SSEFunc_0_eppp fn = s->vex_w ? q : d; 1092 fn(tcg_env, OP_PTR0, OP_PTR1, OP_PTR2); 1093} 1094 1095/* VEX.W affects whether to operate on 32- or 64-bit elements. */ 1096#define VEXW_AVX(uname, lname) \ 1097static void gen_##uname(DisasContext *s, X86DecodedInsn *decode) \ 1098{ \ 1099 gen_vexw_avx(s, decode, \ 1100 gen_helper_##lname##d_xmm, gen_helper_##lname##q_xmm, \ 1101 gen_helper_##lname##d_ymm, gen_helper_##lname##q_ymm); \ 1102} 1103VEXW_AVX(VPSLLV, vpsllv) 1104VEXW_AVX(VPSRLV, vpsrlv) 1105VEXW_AVX(VPSRAV, vpsrav) 1106VEXW_AVX(VPMASKMOV, vpmaskmov) 1107 1108/* Same as above, but with extra arguments to the helper. */ 1109static inline void gen_vsib_avx(DisasContext *s, X86DecodedInsn *decode, 1110 SSEFunc_0_epppti d_xmm, SSEFunc_0_epppti q_xmm, 1111 SSEFunc_0_epppti d_ymm, SSEFunc_0_epppti q_ymm) 1112{ 1113 SSEFunc_0_epppti d = s->vex_l ? d_ymm : d_xmm; 1114 SSEFunc_0_epppti q = s->vex_l ? q_ymm : q_xmm; 1115 SSEFunc_0_epppti fn = s->vex_w ? q : d; 1116 TCGv_i32 scale = tcg_constant_i32(decode->mem.scale); 1117 TCGv_ptr index = tcg_temp_new_ptr(); 1118 1119 /* Pass third input as (index, base, scale) */ 1120 tcg_gen_addi_ptr(index, tcg_env, ZMM_OFFSET(decode->mem.index)); 1121 fn(tcg_env, OP_PTR0, OP_PTR1, index, s->A0, scale); 1122 1123 /* 1124 * There are two output operands, so zero OP1's high 128 bits 1125 * in the VEX.128 case. 1126 */ 1127 if (!s->vex_l) { 1128 int ymmh_ofs = vector_elem_offset(&decode->op[1], MO_128, 1); 1129 tcg_gen_gvec_dup_imm(MO_64, ymmh_ofs, 16, 16, 0); 1130 } 1131} 1132#define VSIB_AVX(uname, lname) \ 1133static void gen_##uname(DisasContext *s, X86DecodedInsn *decode) \ 1134{ \ 1135 gen_vsib_avx(s, decode, \ 1136 gen_helper_##lname##d_xmm, gen_helper_##lname##q_xmm, \ 1137 gen_helper_##lname##d_ymm, gen_helper_##lname##q_ymm); \ 1138} 1139VSIB_AVX(VPGATHERD, vpgatherd) 1140VSIB_AVX(VPGATHERQ, vpgatherq) 1141 1142static void gen_AAA(DisasContext *s, X86DecodedInsn *decode) 1143{ 1144 gen_update_cc_op(s); 1145 gen_helper_aaa(tcg_env); 1146 assume_cc_op(s, CC_OP_EFLAGS); 1147} 1148 1149static void gen_AAD(DisasContext *s, X86DecodedInsn *decode) 1150{ 1151 gen_helper_aad(s->T0, s->T0, s->T1); 1152 prepare_update1_cc(decode, s, CC_OP_LOGICB); 1153} 1154 1155static void gen_AAM(DisasContext *s, X86DecodedInsn *decode) 1156{ 1157 if (decode->immediate == 0) { 1158 gen_exception(s, EXCP00_DIVZ); 1159 } else { 1160 gen_helper_aam(s->T0, s->T0, s->T1); 1161 prepare_update1_cc(decode, s, CC_OP_LOGICB); 1162 } 1163} 1164 1165static void gen_AAS(DisasContext *s, X86DecodedInsn *decode) 1166{ 1167 gen_update_cc_op(s); 1168 gen_helper_aas(tcg_env); 1169 assume_cc_op(s, CC_OP_EFLAGS); 1170} 1171 1172static void gen_ADC(DisasContext *s, X86DecodedInsn *decode) 1173{ 1174 MemOp ot = decode->op[1].ot; 1175 TCGv c_in = tcg_temp_new(); 1176 1177 gen_compute_eflags_c(s, c_in); 1178 if (s->prefix & PREFIX_LOCK) { 1179 tcg_gen_add_tl(s->T0, c_in, s->T1); 1180 tcg_gen_atomic_add_fetch_tl(s->T0, s->A0, s->T0, 1181 s->mem_index, ot | MO_LE); 1182 } else { 1183 tcg_gen_add_tl(s->T0, s->T0, s->T1); 1184 tcg_gen_add_tl(s->T0, s->T0, c_in); 1185 } 1186 prepare_update3_cc(decode, s, CC_OP_ADCB + ot, c_in); 1187} 1188 1189static void gen_ADCOX(DisasContext *s, X86DecodedInsn *decode, int cc_op) 1190{ 1191 MemOp ot = decode->op[0].ot; 1192 TCGv carry_in = NULL; 1193 TCGv *carry_out = (cc_op == CC_OP_ADCX ? &decode->cc_dst : &decode->cc_src2); 1194 TCGv zero; 1195 1196 decode->cc_op = cc_op; 1197 *carry_out = tcg_temp_new(); 1198 if (CC_OP_HAS_EFLAGS(s->cc_op)) { 1199 decode->cc_src = cpu_cc_src; 1200 1201 /* Re-use the carry-out from a previous round? */ 1202 if (s->cc_op == cc_op || s->cc_op == CC_OP_ADCOX) { 1203 carry_in = (cc_op == CC_OP_ADCX ? cpu_cc_dst : cpu_cc_src2); 1204 } 1205 1206 /* Preserve the opposite carry from previous rounds? */ 1207 if (s->cc_op != cc_op && s->cc_op != CC_OP_EFLAGS) { 1208 decode->cc_op = CC_OP_ADCOX; 1209 if (carry_out == &decode->cc_dst) { 1210 decode->cc_src2 = cpu_cc_src2; 1211 } else { 1212 decode->cc_dst = cpu_cc_dst; 1213 } 1214 } 1215 } else { 1216 decode->cc_src = tcg_temp_new(); 1217 gen_mov_eflags(s, decode->cc_src); 1218 } 1219 1220 if (!carry_in) { 1221 /* Get carry_in out of EFLAGS. */ 1222 carry_in = tcg_temp_new(); 1223 tcg_gen_extract_tl(carry_in, decode->cc_src, 1224 ctz32(cc_op == CC_OP_ADCX ? CC_C : CC_O), 1); 1225 } 1226 1227 switch (ot) { 1228#ifdef TARGET_X86_64 1229 case MO_32: 1230 /* If TL is 64-bit just do everything in 64-bit arithmetic. */ 1231 tcg_gen_ext32u_tl(s->T0, s->T0); 1232 tcg_gen_ext32u_tl(s->T1, s->T1); 1233 tcg_gen_add_i64(s->T0, s->T0, s->T1); 1234 tcg_gen_add_i64(s->T0, s->T0, carry_in); 1235 tcg_gen_shri_i64(*carry_out, s->T0, 32); 1236 break; 1237#endif 1238 default: 1239 zero = tcg_constant_tl(0); 1240 tcg_gen_add2_tl(s->T0, *carry_out, s->T0, zero, carry_in, zero); 1241 tcg_gen_add2_tl(s->T0, *carry_out, s->T0, *carry_out, s->T1, zero); 1242 break; 1243 } 1244} 1245 1246static void gen_ADCX(DisasContext *s, X86DecodedInsn *decode) 1247{ 1248 gen_ADCOX(s, decode, CC_OP_ADCX); 1249} 1250 1251static void gen_ADD(DisasContext *s, X86DecodedInsn *decode) 1252{ 1253 MemOp ot = decode->op[1].ot; 1254 1255 if (s->prefix & PREFIX_LOCK) { 1256 tcg_gen_atomic_add_fetch_tl(s->T0, s->A0, s->T1, 1257 s->mem_index, ot | MO_LE); 1258 } else { 1259 tcg_gen_add_tl(s->T0, s->T0, s->T1); 1260 } 1261 prepare_update2_cc(decode, s, CC_OP_ADDB + ot); 1262} 1263 1264static void gen_ADOX(DisasContext *s, X86DecodedInsn *decode) 1265{ 1266 gen_ADCOX(s, decode, CC_OP_ADOX); 1267} 1268 1269static void gen_AND(DisasContext *s, X86DecodedInsn *decode) 1270{ 1271 MemOp ot = decode->op[1].ot; 1272 1273 if (s->prefix & PREFIX_LOCK) { 1274 tcg_gen_atomic_and_fetch_tl(s->T0, s->A0, s->T1, 1275 s->mem_index, ot | MO_LE); 1276 } else { 1277 tcg_gen_and_tl(s->T0, s->T0, s->T1); 1278 } 1279 prepare_update1_cc(decode, s, CC_OP_LOGICB + ot); 1280} 1281 1282static void gen_ANDN(DisasContext *s, X86DecodedInsn *decode) 1283{ 1284 MemOp ot = decode->op[0].ot; 1285 1286 tcg_gen_andc_tl(s->T0, s->T1, s->T0); 1287 prepare_update1_cc(decode, s, CC_OP_LOGICB + ot); 1288} 1289 1290static void gen_ARPL(DisasContext *s, X86DecodedInsn *decode) 1291{ 1292 TCGv zf = tcg_temp_new(); 1293 TCGv flags = tcg_temp_new(); 1294 1295 gen_mov_eflags(s, flags); 1296 1297 /* Compute adjusted DST in T1, merging in SRC[RPL]. */ 1298 tcg_gen_deposit_tl(s->T1, s->T0, s->T1, 0, 2); 1299 1300 /* Z flag set if DST[RPL] < SRC[RPL] */ 1301 tcg_gen_setcond_tl(TCG_COND_LTU, zf, s->T0, s->T1); 1302 tcg_gen_deposit_tl(flags, flags, zf, ctz32(CC_Z), 1); 1303 1304 /* Place maximum RPL in DST */ 1305 tcg_gen_umax_tl(s->T0, s->T0, s->T1); 1306 1307 decode->cc_src = flags; 1308 decode->cc_op = CC_OP_EFLAGS; 1309} 1310 1311static void gen_BEXTR(DisasContext *s, X86DecodedInsn *decode) 1312{ 1313 MemOp ot = decode->op[0].ot; 1314 TCGv bound = tcg_constant_tl(ot == MO_64 ? 63 : 31); 1315 TCGv zero = tcg_constant_tl(0); 1316 TCGv mone = tcg_constant_tl(-1); 1317 1318 /* 1319 * Extract START, and shift the operand. 1320 * Shifts larger than operand size get zeros. 1321 */ 1322 tcg_gen_ext8u_tl(s->A0, s->T1); 1323 tcg_gen_shr_tl(s->T0, s->T0, s->A0); 1324 1325 tcg_gen_movcond_tl(TCG_COND_LEU, s->T0, s->A0, bound, s->T0, zero); 1326 1327 /* 1328 * Extract the LEN into an inverse mask. Lengths larger than 1329 * operand size get all zeros, length 0 gets all ones. 1330 */ 1331 tcg_gen_extract_tl(s->A0, s->T1, 8, 8); 1332 tcg_gen_shl_tl(s->T1, mone, s->A0); 1333 tcg_gen_movcond_tl(TCG_COND_LEU, s->T1, s->A0, bound, s->T1, zero); 1334 tcg_gen_andc_tl(s->T0, s->T0, s->T1); 1335 1336 prepare_update1_cc(decode, s, CC_OP_LOGICB + ot); 1337} 1338 1339static void gen_BLSI(DisasContext *s, X86DecodedInsn *decode) 1340{ 1341 MemOp ot = decode->op[0].ot; 1342 1343 /* input in T1, which is ready for prepare_update2_cc */ 1344 tcg_gen_neg_tl(s->T0, s->T1); 1345 tcg_gen_and_tl(s->T0, s->T0, s->T1); 1346 prepare_update2_cc(decode, s, CC_OP_BLSIB + ot); 1347} 1348 1349static void gen_BLSMSK(DisasContext *s, X86DecodedInsn *decode) 1350{ 1351 MemOp ot = decode->op[0].ot; 1352 1353 /* input in T1, which is ready for prepare_update2_cc */ 1354 tcg_gen_subi_tl(s->T0, s->T1, 1); 1355 tcg_gen_xor_tl(s->T0, s->T0, s->T1); 1356 prepare_update2_cc(decode, s, CC_OP_BMILGB + ot); 1357} 1358 1359static void gen_BLSR(DisasContext *s, X86DecodedInsn *decode) 1360{ 1361 MemOp ot = decode->op[0].ot; 1362 1363 /* input in T1, which is ready for prepare_update2_cc */ 1364 tcg_gen_subi_tl(s->T0, s->T1, 1); 1365 tcg_gen_and_tl(s->T0, s->T0, s->T1); 1366 prepare_update2_cc(decode, s, CC_OP_BMILGB + ot); 1367} 1368 1369static void gen_BOUND(DisasContext *s, X86DecodedInsn *decode) 1370{ 1371 TCGv_i32 op = tcg_temp_new_i32(); 1372 tcg_gen_trunc_tl_i32(op, s->T0); 1373 if (decode->op[1].ot == MO_16) { 1374 gen_helper_boundw(tcg_env, s->A0, op); 1375 } else { 1376 gen_helper_boundl(tcg_env, s->A0, op); 1377 } 1378} 1379 1380/* Non-standard convention - on entry T0 is zero-extended input, T1 is the output. */ 1381static void gen_BSF(DisasContext *s, X86DecodedInsn *decode) 1382{ 1383 MemOp ot = decode->op[0].ot; 1384 1385 /* Only the Z bit is defined and it is related to the input. */ 1386 decode->cc_dst = tcg_temp_new(); 1387 decode->cc_op = CC_OP_LOGICB + ot; 1388 tcg_gen_mov_tl(decode->cc_dst, s->T0); 1389 1390 /* 1391 * The manual says that the output is undefined when the 1392 * input is zero, but real hardware leaves it unchanged, and 1393 * real programs appear to depend on that. Accomplish this 1394 * by passing the output as the value to return upon zero. 1395 */ 1396 tcg_gen_ctz_tl(s->T0, s->T0, s->T1); 1397} 1398 1399/* Non-standard convention - on entry T0 is zero-extended input, T1 is the output. */ 1400static void gen_BSR(DisasContext *s, X86DecodedInsn *decode) 1401{ 1402 MemOp ot = decode->op[0].ot; 1403 1404 /* Only the Z bit is defined and it is related to the input. */ 1405 decode->cc_dst = tcg_temp_new(); 1406 decode->cc_op = CC_OP_LOGICB + ot; 1407 tcg_gen_mov_tl(decode->cc_dst, s->T0); 1408 1409 /* 1410 * The manual says that the output is undefined when the 1411 * input is zero, but real hardware leaves it unchanged, and 1412 * real programs appear to depend on that. Accomplish this 1413 * by passing the output as the value to return upon zero. 1414 * Plus, return the bit index of the first 1 bit. 1415 */ 1416 tcg_gen_xori_tl(s->T1, s->T1, TARGET_LONG_BITS - 1); 1417 tcg_gen_clz_tl(s->T0, s->T0, s->T1); 1418 tcg_gen_xori_tl(s->T0, s->T0, TARGET_LONG_BITS - 1); 1419} 1420 1421static void gen_BSWAP(DisasContext *s, X86DecodedInsn *decode) 1422{ 1423#ifdef TARGET_X86_64 1424 if (s->dflag == MO_64) { 1425 tcg_gen_bswap64_i64(s->T0, s->T0); 1426 return; 1427 } 1428#endif 1429 tcg_gen_bswap32_tl(s->T0, s->T0, TCG_BSWAP_OZ); 1430} 1431 1432static TCGv gen_bt_mask(DisasContext *s, X86DecodedInsn *decode) 1433{ 1434 MemOp ot = decode->op[1].ot; 1435 TCGv mask = tcg_temp_new(); 1436 1437 tcg_gen_andi_tl(s->T1, s->T1, (8 << ot) - 1); 1438 tcg_gen_shl_tl(mask, tcg_constant_tl(1), s->T1); 1439 return mask; 1440} 1441 1442/* Expects truncated bit index in s->T1, 1 << s->T1 in MASK. */ 1443static void gen_bt_flags(DisasContext *s, X86DecodedInsn *decode, TCGv src, TCGv mask) 1444{ 1445 TCGv cf; 1446 1447 /* 1448 * C is the result of the test, Z is unchanged, and the others 1449 * are all undefined. 1450 */ 1451 if (s->cc_op == CC_OP_DYNAMIC || CC_OP_HAS_EFLAGS(s->cc_op)) { 1452 /* Generate EFLAGS and replace the C bit. */ 1453 cf = tcg_temp_new(); 1454 tcg_gen_setcond_tl(TCG_COND_TSTNE, cf, src, mask); 1455 prepare_update_cf(decode, s, cf); 1456 } else { 1457 /* 1458 * Z was going to be computed from the non-zero status of CC_DST. 1459 * We can get that same Z value (and the new C value) by leaving 1460 * CC_DST alone, setting CC_SRC, and using a CC_OP_SAR of the 1461 * same width. 1462 */ 1463 decode->cc_src = tcg_temp_new(); 1464 decode->cc_dst = cpu_cc_dst; 1465 decode->cc_op = CC_OP_SARB + cc_op_size(s->cc_op); 1466 tcg_gen_shr_tl(decode->cc_src, src, s->T1); 1467 } 1468} 1469 1470static void gen_BT(DisasContext *s, X86DecodedInsn *decode) 1471{ 1472 TCGv mask = gen_bt_mask(s, decode); 1473 1474 gen_bt_flags(s, decode, s->T0, mask); 1475} 1476 1477static void gen_BTC(DisasContext *s, X86DecodedInsn *decode) 1478{ 1479 MemOp ot = decode->op[0].ot; 1480 TCGv old = tcg_temp_new(); 1481 TCGv mask = gen_bt_mask(s, decode); 1482 1483 if (s->prefix & PREFIX_LOCK) { 1484 tcg_gen_atomic_fetch_xor_tl(old, s->A0, mask, s->mem_index, ot | MO_LE); 1485 } else { 1486 tcg_gen_mov_tl(old, s->T0); 1487 tcg_gen_xor_tl(s->T0, s->T0, mask); 1488 } 1489 1490 gen_bt_flags(s, decode, old, mask); 1491} 1492 1493static void gen_BTR(DisasContext *s, X86DecodedInsn *decode) 1494{ 1495 MemOp ot = decode->op[0].ot; 1496 TCGv old = tcg_temp_new(); 1497 TCGv mask = gen_bt_mask(s, decode); 1498 1499 if (s->prefix & PREFIX_LOCK) { 1500 TCGv maskc = tcg_temp_new(); 1501 tcg_gen_not_tl(maskc, mask); 1502 tcg_gen_atomic_fetch_and_tl(old, s->A0, maskc, s->mem_index, ot | MO_LE); 1503 } else { 1504 tcg_gen_mov_tl(old, s->T0); 1505 tcg_gen_andc_tl(s->T0, s->T0, mask); 1506 } 1507 1508 gen_bt_flags(s, decode, old, mask); 1509} 1510 1511static void gen_BTS(DisasContext *s, X86DecodedInsn *decode) 1512{ 1513 MemOp ot = decode->op[0].ot; 1514 TCGv old = tcg_temp_new(); 1515 TCGv mask = gen_bt_mask(s, decode); 1516 1517 if (s->prefix & PREFIX_LOCK) { 1518 tcg_gen_atomic_fetch_or_tl(old, s->A0, mask, s->mem_index, ot | MO_LE); 1519 } else { 1520 tcg_gen_mov_tl(old, s->T0); 1521 tcg_gen_or_tl(s->T0, s->T0, mask); 1522 } 1523 1524 gen_bt_flags(s, decode, old, mask); 1525} 1526 1527static void gen_BZHI(DisasContext *s, X86DecodedInsn *decode) 1528{ 1529 MemOp ot = decode->op[0].ot; 1530 TCGv bound = tcg_constant_tl(ot == MO_64 ? 63 : 31); 1531 TCGv zero = tcg_constant_tl(0); 1532 TCGv mone = tcg_constant_tl(-1); 1533 1534 tcg_gen_ext8u_tl(s->T1, s->T1); 1535 1536 tcg_gen_shl_tl(s->A0, mone, s->T1); 1537 tcg_gen_movcond_tl(TCG_COND_LEU, s->A0, s->T1, bound, s->A0, zero); 1538 tcg_gen_andc_tl(s->T0, s->T0, s->A0); 1539 /* 1540 * Note that since we're using BMILG (in order to get O 1541 * cleared) we need to store the inverse into C. 1542 */ 1543 tcg_gen_setcond_tl(TCG_COND_LEU, s->T1, s->T1, bound); 1544 prepare_update2_cc(decode, s, CC_OP_BMILGB + ot); 1545} 1546 1547static void gen_CALL(DisasContext *s, X86DecodedInsn *decode) 1548{ 1549 gen_push_v(s, eip_next_tl(s)); 1550 gen_JMP(s, decode); 1551} 1552 1553static void gen_CALL_m(DisasContext *s, X86DecodedInsn *decode) 1554{ 1555 gen_push_v(s, eip_next_tl(s)); 1556 gen_JMP_m(s, decode); 1557} 1558 1559static void gen_CALLF(DisasContext *s, X86DecodedInsn *decode) 1560{ 1561 gen_far_call(s); 1562} 1563 1564static void gen_CALLF_m(DisasContext *s, X86DecodedInsn *decode) 1565{ 1566 MemOp ot = decode->op[1].ot; 1567 1568 gen_op_ld_v(s, ot, s->T0, s->A0); 1569 gen_add_A0_im(s, 1 << ot); 1570 gen_op_ld_v(s, MO_16, s->T1, s->A0); 1571 gen_far_call(s); 1572} 1573 1574static void gen_CBW(DisasContext *s, X86DecodedInsn *decode) 1575{ 1576 MemOp src_ot = decode->op[0].ot - 1; 1577 1578 tcg_gen_ext_tl(s->T0, s->T0, src_ot | MO_SIGN); 1579} 1580 1581static void gen_CLC(DisasContext *s, X86DecodedInsn *decode) 1582{ 1583 gen_compute_eflags(s); 1584 tcg_gen_andi_tl(cpu_cc_src, cpu_cc_src, ~CC_C); 1585} 1586 1587static void gen_CLD(DisasContext *s, X86DecodedInsn *decode) 1588{ 1589 tcg_gen_st_i32(tcg_constant_i32(1), tcg_env, offsetof(CPUX86State, df)); 1590} 1591 1592static void gen_CLI(DisasContext *s, X86DecodedInsn *decode) 1593{ 1594 gen_reset_eflags(s, IF_MASK); 1595} 1596 1597static void gen_CLTS(DisasContext *s, X86DecodedInsn *decode) 1598{ 1599 gen_helper_clts(tcg_env); 1600 /* abort block because static cpu state changed */ 1601 s->base.is_jmp = DISAS_EOB_NEXT; 1602} 1603 1604static void gen_CMC(DisasContext *s, X86DecodedInsn *decode) 1605{ 1606 gen_compute_eflags(s); 1607 tcg_gen_xori_tl(cpu_cc_src, cpu_cc_src, CC_C); 1608} 1609 1610static void gen_CMOVcc(DisasContext *s, X86DecodedInsn *decode) 1611{ 1612 gen_cmovcc1(s, decode->b & 0xf, s->T0, s->T1); 1613} 1614 1615static void gen_CMPccXADD(DisasContext *s, X86DecodedInsn *decode) 1616{ 1617 TCGLabel *label_top = gen_new_label(); 1618 TCGLabel *label_bottom = gen_new_label(); 1619 TCGv oldv = tcg_temp_new(); 1620 TCGv newv = tcg_temp_new(); 1621 TCGv cmpv = tcg_temp_new(); 1622 TCGCond cond; 1623 1624 TCGv cmp_lhs, cmp_rhs; 1625 MemOp ot, ot_full; 1626 1627 int jcc_op = (decode->b >> 1) & 7; 1628 static const TCGCond cond_table[8] = { 1629 [JCC_O] = TCG_COND_LT, /* test sign bit by comparing against 0 */ 1630 [JCC_B] = TCG_COND_LTU, 1631 [JCC_Z] = TCG_COND_EQ, 1632 [JCC_BE] = TCG_COND_LEU, 1633 [JCC_S] = TCG_COND_LT, /* test sign bit by comparing against 0 */ 1634 [JCC_P] = TCG_COND_TSTEQ, /* even parity - tests low bit of popcount */ 1635 [JCC_L] = TCG_COND_LT, 1636 [JCC_LE] = TCG_COND_LE, 1637 }; 1638 1639 cond = cond_table[jcc_op]; 1640 if (decode->b & 1) { 1641 cond = tcg_invert_cond(cond); 1642 } 1643 1644 ot = decode->op[0].ot; 1645 ot_full = ot | MO_LE; 1646 if (jcc_op >= JCC_S) { 1647 /* 1648 * Sign-extend values before subtracting for S, P (zero/sign extension 1649 * does not matter there) L, LE and their inverses. 1650 */ 1651 ot_full |= MO_SIGN; 1652 } 1653 1654 /* 1655 * cmpv will be moved to cc_src *after* cpu_regs[] is written back, so use 1656 * tcg_gen_ext_tl instead of gen_ext_tl. 1657 */ 1658 tcg_gen_ext_tl(cmpv, cpu_regs[decode->op[1].n], ot_full); 1659 1660 /* 1661 * Cmpxchg loop starts here. 1662 * - s->T1: addition operand (from decoder) 1663 * - s->A0: dest address (from decoder) 1664 * - s->cc_srcT: memory operand (lhs for comparison) 1665 * - cmpv: rhs for comparison 1666 */ 1667 gen_set_label(label_top); 1668 gen_op_ld_v(s, ot_full, s->cc_srcT, s->A0); 1669 tcg_gen_sub_tl(s->T0, s->cc_srcT, cmpv); 1670 1671 /* Compute the comparison result by hand, to avoid clobbering cc_*. */ 1672 switch (jcc_op) { 1673 case JCC_O: 1674 /* (src1 ^ src2) & (src1 ^ dst). newv is only used here for a moment */ 1675 tcg_gen_xor_tl(newv, s->cc_srcT, s->T0); 1676 tcg_gen_xor_tl(s->tmp0, s->cc_srcT, cmpv); 1677 tcg_gen_and_tl(s->tmp0, s->tmp0, newv); 1678 tcg_gen_sextract_tl(s->tmp0, s->tmp0, 0, 8 << ot); 1679 cmp_lhs = s->tmp0, cmp_rhs = tcg_constant_tl(0); 1680 break; 1681 1682 case JCC_P: 1683 tcg_gen_ext8u_tl(s->tmp0, s->T0); 1684 tcg_gen_ctpop_tl(s->tmp0, s->tmp0); 1685 cmp_lhs = s->tmp0, cmp_rhs = tcg_constant_tl(1); 1686 break; 1687 1688 case JCC_S: 1689 tcg_gen_sextract_tl(s->tmp0, s->T0, 0, 8 << ot); 1690 cmp_lhs = s->tmp0, cmp_rhs = tcg_constant_tl(0); 1691 break; 1692 1693 default: 1694 cmp_lhs = s->cc_srcT, cmp_rhs = cmpv; 1695 break; 1696 } 1697 1698 /* Compute new value: if condition does not hold, just store back s->cc_srcT */ 1699 tcg_gen_add_tl(newv, s->cc_srcT, s->T1); 1700 tcg_gen_movcond_tl(cond, newv, cmp_lhs, cmp_rhs, newv, s->cc_srcT); 1701 tcg_gen_atomic_cmpxchg_tl(oldv, s->A0, s->cc_srcT, newv, s->mem_index, ot_full); 1702 1703 /* Exit unconditionally if cmpxchg succeeded. */ 1704 tcg_gen_brcond_tl(TCG_COND_EQ, oldv, s->cc_srcT, label_bottom); 1705 1706 /* Try again if there was actually a store to make. */ 1707 tcg_gen_brcond_tl(cond, cmp_lhs, cmp_rhs, label_top); 1708 gen_set_label(label_bottom); 1709 1710 /* Store old value to registers only after a successful store. */ 1711 gen_writeback(s, decode, 1, s->cc_srcT); 1712 1713 decode->cc_dst = s->T0; 1714 decode->cc_src = cmpv; 1715 decode->cc_op = CC_OP_SUBB + ot; 1716} 1717 1718static void gen_CMPS(DisasContext *s, X86DecodedInsn *decode) 1719{ 1720 MemOp ot = decode->op[2].ot; 1721 if (s->prefix & (PREFIX_REPZ | PREFIX_REPNZ)) { 1722 gen_repz_nz(s, ot, gen_cmps); 1723 } else { 1724 gen_cmps(s, ot); 1725 } 1726} 1727 1728static void gen_CMPXCHG(DisasContext *s, X86DecodedInsn *decode) 1729{ 1730 MemOp ot = decode->op[2].ot; 1731 TCGv cmpv = tcg_temp_new(); 1732 TCGv oldv = tcg_temp_new(); 1733 TCGv newv = tcg_temp_new(); 1734 TCGv dest; 1735 1736 tcg_gen_ext_tl(cmpv, cpu_regs[R_EAX], ot); 1737 tcg_gen_ext_tl(newv, s->T1, ot); 1738 if (s->prefix & PREFIX_LOCK) { 1739 tcg_gen_atomic_cmpxchg_tl(oldv, s->A0, cmpv, newv, 1740 s->mem_index, ot | MO_LE); 1741 } else { 1742 tcg_gen_ext_tl(oldv, s->T0, ot); 1743 if (decode->op[0].has_ea) { 1744 /* 1745 * Perform an unconditional store cycle like physical cpu; 1746 * must be before changing accumulator to ensure 1747 * idempotency if the store faults and the instruction 1748 * is restarted 1749 */ 1750 tcg_gen_movcond_tl(TCG_COND_EQ, newv, oldv, cmpv, newv, oldv); 1751 gen_op_st_v(s, ot, newv, s->A0); 1752 } else { 1753 /* 1754 * Unlike the memory case, where "the destination operand receives 1755 * a write cycle without regard to the result of the comparison", 1756 * rm must not be touched altogether if the write fails, including 1757 * not zero-extending it on 64-bit processors. So, precompute 1758 * the result of a successful writeback and perform the movcond 1759 * directly on cpu_regs. In case rm is part of RAX, note that this 1760 * movcond and the one below are mutually exclusive is executed. 1761 */ 1762 dest = gen_op_deposit_reg_v(s, ot, decode->op[0].n, newv, newv); 1763 tcg_gen_movcond_tl(TCG_COND_EQ, dest, oldv, cmpv, newv, dest); 1764 } 1765 decode->op[0].unit = X86_OP_SKIP; 1766 } 1767 1768 /* Write RAX only if the cmpxchg fails. */ 1769 dest = gen_op_deposit_reg_v(s, ot, R_EAX, s->T0, oldv); 1770 tcg_gen_movcond_tl(TCG_COND_NE, dest, oldv, cmpv, s->T0, dest); 1771 1772 tcg_gen_mov_tl(s->cc_srcT, cmpv); 1773 tcg_gen_sub_tl(cmpv, cmpv, oldv); 1774 decode->cc_dst = cmpv; 1775 decode->cc_src = oldv; 1776 decode->cc_op = CC_OP_SUBB + ot; 1777} 1778 1779static void gen_CMPXCHG16B(DisasContext *s, X86DecodedInsn *decode) 1780{ 1781#ifdef TARGET_X86_64 1782 MemOp mop = MO_TE | MO_128 | MO_ALIGN; 1783 TCGv_i64 t0, t1; 1784 TCGv_i128 cmp, val; 1785 1786 cmp = tcg_temp_new_i128(); 1787 val = tcg_temp_new_i128(); 1788 tcg_gen_concat_i64_i128(cmp, cpu_regs[R_EAX], cpu_regs[R_EDX]); 1789 tcg_gen_concat_i64_i128(val, cpu_regs[R_EBX], cpu_regs[R_ECX]); 1790 1791 /* Only require atomic with LOCK; non-parallel handled in generator. */ 1792 if (s->prefix & PREFIX_LOCK) { 1793 tcg_gen_atomic_cmpxchg_i128(val, s->A0, cmp, val, s->mem_index, mop); 1794 } else { 1795 tcg_gen_nonatomic_cmpxchg_i128(val, s->A0, cmp, val, s->mem_index, mop); 1796 } 1797 1798 tcg_gen_extr_i128_i64(s->T0, s->T1, val); 1799 1800 /* Determine success after the fact. */ 1801 t0 = tcg_temp_new_i64(); 1802 t1 = tcg_temp_new_i64(); 1803 tcg_gen_xor_i64(t0, s->T0, cpu_regs[R_EAX]); 1804 tcg_gen_xor_i64(t1, s->T1, cpu_regs[R_EDX]); 1805 tcg_gen_or_i64(t0, t0, t1); 1806 1807 /* Update Z. */ 1808 gen_compute_eflags(s); 1809 tcg_gen_setcondi_i64(TCG_COND_EQ, t0, t0, 0); 1810 tcg_gen_deposit_tl(cpu_cc_src, cpu_cc_src, t0, ctz32(CC_Z), 1); 1811 1812 /* 1813 * Extract the result values for the register pair. We may do this 1814 * unconditionally, because on success (Z=1), the old value matches 1815 * the previous value in RDX:RAX. 1816 */ 1817 tcg_gen_mov_i64(cpu_regs[R_EAX], s->T0); 1818 tcg_gen_mov_i64(cpu_regs[R_EDX], s->T1); 1819#else 1820 abort(); 1821#endif 1822} 1823 1824static void gen_CMPXCHG8B(DisasContext *s, X86DecodedInsn *decode) 1825{ 1826 TCGv_i64 cmp, val, old; 1827 TCGv Z; 1828 1829 cmp = tcg_temp_new_i64(); 1830 val = tcg_temp_new_i64(); 1831 old = tcg_temp_new_i64(); 1832 1833 /* Construct the comparison values from the register pair. */ 1834 tcg_gen_concat_tl_i64(cmp, cpu_regs[R_EAX], cpu_regs[R_EDX]); 1835 tcg_gen_concat_tl_i64(val, cpu_regs[R_EBX], cpu_regs[R_ECX]); 1836 1837 /* Only require atomic with LOCK; non-parallel handled in generator. */ 1838 if (s->prefix & PREFIX_LOCK) { 1839 tcg_gen_atomic_cmpxchg_i64(old, s->A0, cmp, val, s->mem_index, MO_TEUQ); 1840 } else { 1841 tcg_gen_nonatomic_cmpxchg_i64(old, s->A0, cmp, val, 1842 s->mem_index, MO_TEUQ); 1843 } 1844 1845 /* Set tmp0 to match the required value of Z. */ 1846 tcg_gen_setcond_i64(TCG_COND_EQ, cmp, old, cmp); 1847 Z = tcg_temp_new(); 1848 tcg_gen_trunc_i64_tl(Z, cmp); 1849 1850 /* 1851 * Extract the result values for the register pair. 1852 * For 32-bit, we may do this unconditionally, because on success (Z=1), 1853 * the old value matches the previous value in EDX:EAX. For x86_64, 1854 * the store must be conditional, because we must leave the source 1855 * registers unchanged on success, and zero-extend the writeback 1856 * on failure (Z=0). 1857 */ 1858 if (TARGET_LONG_BITS == 32) { 1859 tcg_gen_extr_i64_tl(cpu_regs[R_EAX], cpu_regs[R_EDX], old); 1860 } else { 1861 TCGv zero = tcg_constant_tl(0); 1862 1863 tcg_gen_extr_i64_tl(s->T0, s->T1, old); 1864 tcg_gen_movcond_tl(TCG_COND_EQ, cpu_regs[R_EAX], Z, zero, 1865 s->T0, cpu_regs[R_EAX]); 1866 tcg_gen_movcond_tl(TCG_COND_EQ, cpu_regs[R_EDX], Z, zero, 1867 s->T1, cpu_regs[R_EDX]); 1868 } 1869 1870 /* Update Z. */ 1871 gen_compute_eflags(s); 1872 tcg_gen_deposit_tl(cpu_cc_src, cpu_cc_src, Z, ctz32(CC_Z), 1); 1873} 1874 1875static void gen_CPUID(DisasContext *s, X86DecodedInsn *decode) 1876{ 1877 gen_update_cc_op(s); 1878 gen_update_eip_cur(s); 1879 gen_helper_cpuid(tcg_env); 1880} 1881 1882static void gen_CRC32(DisasContext *s, X86DecodedInsn *decode) 1883{ 1884 MemOp ot = decode->op[2].ot; 1885 1886 tcg_gen_trunc_tl_i32(s->tmp2_i32, s->T0); 1887 gen_helper_crc32(s->T0, s->tmp2_i32, s->T1, tcg_constant_i32(8 << ot)); 1888} 1889 1890static void gen_CVTPI2Px(DisasContext *s, X86DecodedInsn *decode) 1891{ 1892 gen_helper_enter_mmx(tcg_env); 1893 if (s->prefix & PREFIX_DATA) { 1894 gen_helper_cvtpi2pd(tcg_env, OP_PTR0, OP_PTR2); 1895 } else { 1896 gen_helper_cvtpi2ps(tcg_env, OP_PTR0, OP_PTR2); 1897 } 1898} 1899 1900static void gen_CVTPx2PI(DisasContext *s, X86DecodedInsn *decode) 1901{ 1902 gen_helper_enter_mmx(tcg_env); 1903 if (s->prefix & PREFIX_DATA) { 1904 gen_helper_cvtpd2pi(tcg_env, OP_PTR0, OP_PTR2); 1905 } else { 1906 gen_helper_cvtps2pi(tcg_env, OP_PTR0, OP_PTR2); 1907 } 1908} 1909 1910static void gen_CVTTPx2PI(DisasContext *s, X86DecodedInsn *decode) 1911{ 1912 gen_helper_enter_mmx(tcg_env); 1913 if (s->prefix & PREFIX_DATA) { 1914 gen_helper_cvttpd2pi(tcg_env, OP_PTR0, OP_PTR2); 1915 } else { 1916 gen_helper_cvttps2pi(tcg_env, OP_PTR0, OP_PTR2); 1917 } 1918} 1919 1920static void gen_CWD(DisasContext *s, X86DecodedInsn *decode) 1921{ 1922 int shift = 8 << decode->op[0].ot; 1923 1924 tcg_gen_sextract_tl(s->T0, s->T0, shift - 1, 1); 1925} 1926 1927static void gen_DAA(DisasContext *s, X86DecodedInsn *decode) 1928{ 1929 gen_update_cc_op(s); 1930 gen_helper_daa(tcg_env); 1931 assume_cc_op(s, CC_OP_EFLAGS); 1932} 1933 1934static void gen_DAS(DisasContext *s, X86DecodedInsn *decode) 1935{ 1936 gen_update_cc_op(s); 1937 gen_helper_das(tcg_env); 1938 assume_cc_op(s, CC_OP_EFLAGS); 1939} 1940 1941static void gen_DEC(DisasContext *s, X86DecodedInsn *decode) 1942{ 1943 MemOp ot = decode->op[1].ot; 1944 1945 tcg_gen_movi_tl(s->T1, -1); 1946 if (s->prefix & PREFIX_LOCK) { 1947 tcg_gen_atomic_add_fetch_tl(s->T0, s->A0, s->T1, 1948 s->mem_index, ot | MO_LE); 1949 } else { 1950 tcg_gen_add_tl(s->T0, s->T0, s->T1); 1951 } 1952 prepare_update_cc_incdec(decode, s, CC_OP_DECB + ot); 1953} 1954 1955static void gen_DIV(DisasContext *s, X86DecodedInsn *decode) 1956{ 1957 MemOp ot = decode->op[1].ot; 1958 1959 switch(ot) { 1960 case MO_8: 1961 gen_helper_divb_AL(tcg_env, s->T0); 1962 break; 1963 case MO_16: 1964 gen_helper_divw_AX(tcg_env, s->T0); 1965 break; 1966 default: 1967 case MO_32: 1968 gen_helper_divl_EAX(tcg_env, s->T0); 1969 break; 1970#ifdef TARGET_X86_64 1971 case MO_64: 1972 gen_helper_divq_EAX(tcg_env, s->T0); 1973 break; 1974#endif 1975 } 1976} 1977 1978static void gen_EMMS(DisasContext *s, X86DecodedInsn *decode) 1979{ 1980 gen_helper_emms(tcg_env); 1981} 1982 1983static void gen_ENTER(DisasContext *s, X86DecodedInsn *decode) 1984{ 1985 gen_enter(s, decode->op[1].imm, decode->op[2].imm); 1986} 1987 1988static void gen_EXTRQ_i(DisasContext *s, X86DecodedInsn *decode) 1989{ 1990 TCGv_i32 length = tcg_constant_i32(decode->immediate & 63); 1991 TCGv_i32 index = tcg_constant_i32((decode->immediate >> 8) & 63); 1992 1993 gen_helper_extrq_i(tcg_env, OP_PTR0, index, length); 1994} 1995 1996static void gen_EXTRQ_r(DisasContext *s, X86DecodedInsn *decode) 1997{ 1998 gen_helper_extrq_r(tcg_env, OP_PTR0, OP_PTR2); 1999} 2000 2001static void gen_FXRSTOR(DisasContext *s, X86DecodedInsn *decode) 2002{ 2003 if ((s->flags & HF_EM_MASK) || (s->flags & HF_TS_MASK)) { 2004 gen_NM_exception(s); 2005 } else { 2006 gen_helper_fxrstor(tcg_env, s->A0); 2007 } 2008} 2009 2010static void gen_FXSAVE(DisasContext *s, X86DecodedInsn *decode) 2011{ 2012 if ((s->flags & HF_EM_MASK) || (s->flags & HF_TS_MASK)) { 2013 gen_NM_exception(s); 2014 } else { 2015 gen_helper_fxsave(tcg_env, s->A0); 2016 } 2017} 2018 2019static void gen_HLT(DisasContext *s, X86DecodedInsn *decode) 2020{ 2021#ifdef CONFIG_SYSTEM_ONLY 2022 gen_update_cc_op(s); 2023 gen_update_eip_next(s); 2024 gen_helper_hlt(tcg_env); 2025 s->base.is_jmp = DISAS_NORETURN; 2026#endif 2027} 2028 2029static void gen_IDIV(DisasContext *s, X86DecodedInsn *decode) 2030{ 2031 MemOp ot = decode->op[1].ot; 2032 2033 switch(ot) { 2034 case MO_8: 2035 gen_helper_idivb_AL(tcg_env, s->T0); 2036 break; 2037 case MO_16: 2038 gen_helper_idivw_AX(tcg_env, s->T0); 2039 break; 2040 default: 2041 case MO_32: 2042 gen_helper_idivl_EAX(tcg_env, s->T0); 2043 break; 2044#ifdef TARGET_X86_64 2045 case MO_64: 2046 gen_helper_idivq_EAX(tcg_env, s->T0); 2047 break; 2048#endif 2049 } 2050} 2051 2052static void gen_IMUL3(DisasContext *s, X86DecodedInsn *decode) 2053{ 2054 MemOp ot = decode->op[0].ot; 2055 TCGv cc_src_rhs; 2056 2057 switch (ot) { 2058 case MO_16: 2059 /* s->T0 already sign-extended */ 2060 tcg_gen_ext16s_tl(s->T1, s->T1); 2061 tcg_gen_mul_tl(s->T0, s->T0, s->T1); 2062 /* Compare the full result to the extension of the truncated result. */ 2063 tcg_gen_ext16s_tl(s->T1, s->T0); 2064 cc_src_rhs = s->T0; 2065 break; 2066 2067 case MO_32: 2068#ifdef TARGET_X86_64 2069 if (TCG_TARGET_REG_BITS == 64) { 2070 /* 2071 * This produces fewer TCG ops, and better code if flags are needed, 2072 * but it requires a 64-bit multiply even if they are not. Use it 2073 * only if the target has 64-bits registers. 2074 * 2075 * s->T0 is already sign-extended. 2076 */ 2077 tcg_gen_ext32s_tl(s->T1, s->T1); 2078 tcg_gen_mul_tl(s->T0, s->T0, s->T1); 2079 /* Compare the full result to the extension of the truncated result. */ 2080 tcg_gen_ext32s_tl(s->T1, s->T0); 2081 cc_src_rhs = s->T0; 2082 } else { 2083 /* Variant that only needs a 32-bit widening multiply. */ 2084 TCGv_i32 hi = tcg_temp_new_i32(); 2085 TCGv_i32 lo = tcg_temp_new_i32(); 2086 tcg_gen_trunc_tl_i32(lo, s->T0); 2087 tcg_gen_trunc_tl_i32(hi, s->T1); 2088 tcg_gen_muls2_i32(lo, hi, lo, hi); 2089 tcg_gen_extu_i32_tl(s->T0, lo); 2090 2091 cc_src_rhs = tcg_temp_new(); 2092 tcg_gen_extu_i32_tl(cc_src_rhs, hi); 2093 /* Compare the high part to the sign bit of the truncated result */ 2094 tcg_gen_sari_i32(lo, lo, 31); 2095 tcg_gen_extu_i32_tl(s->T1, lo); 2096 } 2097 break; 2098 2099 case MO_64: 2100#endif 2101 cc_src_rhs = tcg_temp_new(); 2102 tcg_gen_muls2_tl(s->T0, cc_src_rhs, s->T0, s->T1); 2103 /* Compare the high part to the sign bit of the truncated result */ 2104 tcg_gen_sari_tl(s->T1, s->T0, TARGET_LONG_BITS - 1); 2105 break; 2106 2107 default: 2108 g_assert_not_reached(); 2109 } 2110 2111 tcg_gen_sub_tl(s->T1, s->T1, cc_src_rhs); 2112 prepare_update2_cc(decode, s, CC_OP_MULB + ot); 2113} 2114 2115static void gen_IMUL(DisasContext *s, X86DecodedInsn *decode) 2116{ 2117 MemOp ot = decode->op[1].ot; 2118 TCGv cc_src_rhs; 2119 2120 switch (ot) { 2121 case MO_8: 2122 /* s->T0 already sign-extended */ 2123 tcg_gen_ext8s_tl(s->T1, s->T1); 2124 tcg_gen_mul_tl(s->T0, s->T0, s->T1); 2125 gen_op_mov_reg_v(s, MO_16, R_EAX, s->T0); 2126 /* Compare the full result to the extension of the truncated result. */ 2127 tcg_gen_ext8s_tl(s->T1, s->T0); 2128 cc_src_rhs = s->T0; 2129 break; 2130 2131 case MO_16: 2132 /* s->T0 already sign-extended */ 2133 tcg_gen_ext16s_tl(s->T1, s->T1); 2134 tcg_gen_mul_tl(s->T0, s->T0, s->T1); 2135 gen_op_mov_reg_v(s, MO_16, R_EAX, s->T0); 2136 tcg_gen_shri_tl(s->T1, s->T0, 16); 2137 gen_op_mov_reg_v(s, MO_16, R_EDX, s->T1); 2138 /* Compare the full result to the extension of the truncated result. */ 2139 tcg_gen_ext16s_tl(s->T1, s->T0); 2140 cc_src_rhs = s->T0; 2141 break; 2142 2143 case MO_32: 2144#ifdef TARGET_X86_64 2145 /* s->T0 already sign-extended */ 2146 tcg_gen_ext32s_tl(s->T1, s->T1); 2147 tcg_gen_mul_tl(s->T0, s->T0, s->T1); 2148 tcg_gen_ext32u_tl(cpu_regs[R_EAX], s->T0); 2149 tcg_gen_shri_tl(cpu_regs[R_EDX], s->T0, 32); 2150 /* Compare the full result to the extension of the truncated result. */ 2151 tcg_gen_ext32s_tl(s->T1, s->T0); 2152 cc_src_rhs = s->T0; 2153 break; 2154 2155 case MO_64: 2156#endif 2157 tcg_gen_muls2_tl(s->T0, cpu_regs[R_EDX], s->T0, s->T1); 2158 tcg_gen_mov_tl(cpu_regs[R_EAX], s->T0); 2159 2160 /* Compare the high part to the sign bit of the truncated result */ 2161 tcg_gen_negsetcondi_tl(TCG_COND_LT, s->T1, s->T0, 0); 2162 cc_src_rhs = cpu_regs[R_EDX]; 2163 break; 2164 2165 default: 2166 g_assert_not_reached(); 2167 } 2168 2169 tcg_gen_sub_tl(s->T1, s->T1, cc_src_rhs); 2170 prepare_update2_cc(decode, s, CC_OP_MULB + ot); 2171} 2172 2173static void gen_IN(DisasContext *s, X86DecodedInsn *decode) 2174{ 2175 MemOp ot = decode->op[0].ot; 2176 TCGv_i32 port = tcg_temp_new_i32(); 2177 2178 tcg_gen_trunc_tl_i32(port, s->T0); 2179 tcg_gen_ext16u_i32(port, port); 2180 if (!gen_check_io(s, ot, port, SVM_IOIO_TYPE_MASK)) { 2181 return; 2182 } 2183 translator_io_start(&s->base); 2184 gen_helper_in_func(ot, s->T0, port); 2185 gen_writeback(s, decode, 0, s->T0); 2186 gen_bpt_io(s, port, ot); 2187} 2188 2189static void gen_INC(DisasContext *s, X86DecodedInsn *decode) 2190{ 2191 MemOp ot = decode->op[1].ot; 2192 2193 tcg_gen_movi_tl(s->T1, 1); 2194 if (s->prefix & PREFIX_LOCK) { 2195 tcg_gen_atomic_add_fetch_tl(s->T0, s->A0, s->T1, 2196 s->mem_index, ot | MO_LE); 2197 } else { 2198 tcg_gen_add_tl(s->T0, s->T0, s->T1); 2199 } 2200 prepare_update_cc_incdec(decode, s, CC_OP_INCB + ot); 2201} 2202 2203static void gen_INS(DisasContext *s, X86DecodedInsn *decode) 2204{ 2205 MemOp ot = decode->op[1].ot; 2206 TCGv_i32 port = tcg_temp_new_i32(); 2207 2208 tcg_gen_trunc_tl_i32(port, s->T1); 2209 tcg_gen_ext16u_i32(port, port); 2210 if (!gen_check_io(s, ot, port, 2211 SVM_IOIO_TYPE_MASK | SVM_IOIO_STR_MASK)) { 2212 return; 2213 } 2214 2215 translator_io_start(&s->base); 2216 if (s->prefix & (PREFIX_REPZ | PREFIX_REPNZ)) { 2217 gen_repz(s, ot, gen_ins); 2218 } else { 2219 gen_ins(s, ot); 2220 } 2221} 2222 2223static void gen_INSERTQ_i(DisasContext *s, X86DecodedInsn *decode) 2224{ 2225 TCGv_i32 length = tcg_constant_i32(decode->immediate & 63); 2226 TCGv_i32 index = tcg_constant_i32((decode->immediate >> 8) & 63); 2227 2228 gen_helper_insertq_i(tcg_env, OP_PTR0, OP_PTR1, index, length); 2229} 2230 2231static void gen_INSERTQ_r(DisasContext *s, X86DecodedInsn *decode) 2232{ 2233 gen_helper_insertq_r(tcg_env, OP_PTR0, OP_PTR2); 2234} 2235 2236static void gen_INT(DisasContext *s, X86DecodedInsn *decode) 2237{ 2238 gen_interrupt(s, decode->immediate); 2239} 2240 2241static void gen_INT1(DisasContext *s, X86DecodedInsn *decode) 2242{ 2243 gen_update_cc_op(s); 2244 gen_update_eip_next(s); 2245 gen_helper_icebp(tcg_env); 2246 s->base.is_jmp = DISAS_NORETURN; 2247} 2248 2249static void gen_INT3(DisasContext *s, X86DecodedInsn *decode) 2250{ 2251 gen_interrupt(s, EXCP03_INT3); 2252} 2253 2254static void gen_INTO(DisasContext *s, X86DecodedInsn *decode) 2255{ 2256 gen_update_cc_op(s); 2257 gen_update_eip_cur(s); 2258 gen_helper_into(tcg_env, cur_insn_len_i32(s)); 2259} 2260 2261static void gen_IRET(DisasContext *s, X86DecodedInsn *decode) 2262{ 2263 if (!PE(s) || VM86(s)) { 2264 gen_helper_iret_real(tcg_env, tcg_constant_i32(s->dflag - 1)); 2265 } else { 2266 gen_helper_iret_protected(tcg_env, tcg_constant_i32(s->dflag - 1), 2267 eip_next_i32(s)); 2268 } 2269 assume_cc_op(s, CC_OP_EFLAGS); 2270 s->base.is_jmp = DISAS_EOB_ONLY; 2271} 2272 2273static void gen_Jcc(DisasContext *s, X86DecodedInsn *decode) 2274{ 2275 gen_bnd_jmp(s); 2276 gen_jcc(s, decode->b & 0xf, decode->immediate); 2277} 2278 2279static void gen_JCXZ(DisasContext *s, X86DecodedInsn *decode) 2280{ 2281 TCGLabel *taken = gen_new_label(); 2282 2283 gen_update_cc_op(s); 2284 gen_op_jz_ecx(s, taken); 2285 gen_conditional_jump_labels(s, decode->immediate, NULL, taken); 2286} 2287 2288static void gen_JMP(DisasContext *s, X86DecodedInsn *decode) 2289{ 2290 gen_update_cc_op(s); 2291 gen_jmp_rel(s, s->dflag, decode->immediate, 0); 2292} 2293 2294static void gen_JMP_m(DisasContext *s, X86DecodedInsn *decode) 2295{ 2296 gen_op_jmp_v(s, s->T0); 2297 gen_bnd_jmp(s); 2298 s->base.is_jmp = DISAS_JUMP; 2299} 2300 2301static void gen_JMPF(DisasContext *s, X86DecodedInsn *decode) 2302{ 2303 gen_far_jmp(s); 2304} 2305 2306static void gen_JMPF_m(DisasContext *s, X86DecodedInsn *decode) 2307{ 2308 MemOp ot = decode->op[1].ot; 2309 2310 gen_op_ld_v(s, ot, s->T0, s->A0); 2311 gen_add_A0_im(s, 1 << ot); 2312 gen_op_ld_v(s, MO_16, s->T1, s->A0); 2313 gen_far_jmp(s); 2314} 2315 2316static void gen_LAHF(DisasContext *s, X86DecodedInsn *decode) 2317{ 2318 if (CODE64(s) && !(s->cpuid_ext3_features & CPUID_EXT3_LAHF_LM)) { 2319 return gen_illegal_opcode(s); 2320 } 2321 gen_compute_eflags(s); 2322 /* Note: gen_compute_eflags() only gives the condition codes */ 2323 tcg_gen_ori_tl(s->T0, cpu_cc_src, 0x02); 2324 tcg_gen_deposit_tl(cpu_regs[R_EAX], cpu_regs[R_EAX], s->T0, 8, 8); 2325} 2326 2327static void gen_LAR(DisasContext *s, X86DecodedInsn *decode) 2328{ 2329 MemOp ot = decode->op[0].ot; 2330 TCGv result = tcg_temp_new(); 2331 TCGv dest; 2332 2333 gen_compute_eflags(s); 2334 gen_update_cc_op(s); 2335 gen_helper_lar(result, tcg_env, s->T0); 2336 2337 /* Perform writeback here to skip it if ZF=0. */ 2338 decode->op[0].unit = X86_OP_SKIP; 2339 dest = gen_op_deposit_reg_v(s, ot, decode->op[0].n, result, result); 2340 tcg_gen_movcond_tl(TCG_COND_TSTNE, dest, cpu_cc_src, tcg_constant_tl(CC_Z), 2341 result, dest); 2342} 2343 2344static void gen_LDMXCSR(DisasContext *s, X86DecodedInsn *decode) 2345{ 2346 tcg_gen_trunc_tl_i32(s->tmp2_i32, s->T0); 2347 gen_helper_ldmxcsr(tcg_env, s->tmp2_i32); 2348} 2349 2350static void gen_lxx_seg(DisasContext *s, X86DecodedInsn *decode, int seg) 2351{ 2352 MemOp ot = decode->op[0].ot; 2353 2354 /* Offset already in s->T0. */ 2355 gen_add_A0_im(s, 1 << ot); 2356 gen_op_ld_v(s, MO_16, s->T1, s->A0); 2357 2358 /* load the segment here to handle exceptions properly */ 2359 gen_movl_seg(s, seg, s->T1); 2360} 2361 2362static void gen_LDS(DisasContext *s, X86DecodedInsn *decode) 2363{ 2364 gen_lxx_seg(s, decode, R_DS); 2365} 2366 2367static void gen_LEA(DisasContext *s, X86DecodedInsn *decode) 2368{ 2369 TCGv ea = gen_lea_modrm_1(s, decode->mem, false); 2370 gen_lea_v_seg_dest(s, s->aflag, s->T0, ea, -1, -1); 2371} 2372 2373static void gen_LEAVE(DisasContext *s, X86DecodedInsn *decode) 2374{ 2375 gen_leave(s); 2376} 2377 2378static void gen_LES(DisasContext *s, X86DecodedInsn *decode) 2379{ 2380 gen_lxx_seg(s, decode, R_ES); 2381} 2382 2383static void gen_LFENCE(DisasContext *s, X86DecodedInsn *decode) 2384{ 2385 tcg_gen_mb(TCG_MO_LD_LD | TCG_BAR_SC); 2386} 2387 2388static void gen_LFS(DisasContext *s, X86DecodedInsn *decode) 2389{ 2390 gen_lxx_seg(s, decode, R_FS); 2391} 2392 2393static void gen_LGS(DisasContext *s, X86DecodedInsn *decode) 2394{ 2395 gen_lxx_seg(s, decode, R_GS); 2396} 2397 2398static void gen_LODS(DisasContext *s, X86DecodedInsn *decode) 2399{ 2400 MemOp ot = decode->op[1].ot; 2401 if (s->prefix & (PREFIX_REPZ | PREFIX_REPNZ)) { 2402 gen_repz(s, ot, gen_lods); 2403 } else { 2404 gen_lods(s, ot); 2405 } 2406} 2407 2408static void gen_LOOP(DisasContext *s, X86DecodedInsn *decode) 2409{ 2410 TCGLabel *taken = gen_new_label(); 2411 2412 gen_update_cc_op(s); 2413 gen_op_add_reg_im(s, s->aflag, R_ECX, -1); 2414 gen_op_jnz_ecx(s, taken); 2415 gen_conditional_jump_labels(s, decode->immediate, NULL, taken); 2416} 2417 2418static void gen_LOOPE(DisasContext *s, X86DecodedInsn *decode) 2419{ 2420 TCGLabel *taken = gen_new_label(); 2421 TCGLabel *not_taken = gen_new_label(); 2422 2423 gen_update_cc_op(s); 2424 gen_op_add_reg_im(s, s->aflag, R_ECX, -1); 2425 gen_op_jz_ecx(s, not_taken); 2426 gen_jcc1(s, (JCC_Z << 1), taken); /* jz taken */ 2427 gen_conditional_jump_labels(s, decode->immediate, not_taken, taken); 2428} 2429 2430static void gen_LOOPNE(DisasContext *s, X86DecodedInsn *decode) 2431{ 2432 TCGLabel *taken = gen_new_label(); 2433 TCGLabel *not_taken = gen_new_label(); 2434 2435 gen_update_cc_op(s); 2436 gen_op_add_reg_im(s, s->aflag, R_ECX, -1); 2437 gen_op_jz_ecx(s, not_taken); 2438 gen_jcc1(s, (JCC_Z << 1) | 1, taken); /* jnz taken */ 2439 gen_conditional_jump_labels(s, decode->immediate, not_taken, taken); 2440} 2441 2442static void gen_LSL(DisasContext *s, X86DecodedInsn *decode) 2443{ 2444 MemOp ot = decode->op[0].ot; 2445 TCGv result = tcg_temp_new(); 2446 TCGv dest; 2447 2448 gen_compute_eflags(s); 2449 gen_update_cc_op(s); 2450 gen_helper_lsl(result, tcg_env, s->T0); 2451 2452 /* Perform writeback here to skip it if ZF=0. */ 2453 decode->op[0].unit = X86_OP_SKIP; 2454 dest = gen_op_deposit_reg_v(s, ot, decode->op[0].n, result, result); 2455 tcg_gen_movcond_tl(TCG_COND_TSTNE, dest, cpu_cc_src, tcg_constant_tl(CC_Z), 2456 result, dest); 2457} 2458 2459static void gen_LSS(DisasContext *s, X86DecodedInsn *decode) 2460{ 2461 gen_lxx_seg(s, decode, R_SS); 2462} 2463 2464static void gen_LZCNT(DisasContext *s, X86DecodedInsn *decode) 2465{ 2466 MemOp ot = decode->op[0].ot; 2467 2468 /* C bit (cc_src) is defined related to the input. */ 2469 decode->cc_src = tcg_temp_new(); 2470 decode->cc_dst = s->T0; 2471 decode->cc_op = CC_OP_BMILGB + ot; 2472 tcg_gen_mov_tl(decode->cc_src, s->T0); 2473 2474 /* 2475 * Reduce the target_ulong result by the number of zeros that 2476 * we expect to find at the top. 2477 */ 2478 tcg_gen_clzi_tl(s->T0, s->T0, TARGET_LONG_BITS); 2479 tcg_gen_subi_tl(s->T0, s->T0, TARGET_LONG_BITS - (8 << ot)); 2480} 2481 2482static void gen_MFENCE(DisasContext *s, X86DecodedInsn *decode) 2483{ 2484 tcg_gen_mb(TCG_MO_ALL | TCG_BAR_SC); 2485} 2486 2487static void gen_MOV(DisasContext *s, X86DecodedInsn *decode) 2488{ 2489 /* nothing to do! */ 2490} 2491#define gen_NOP gen_MOV 2492 2493static void gen_MASKMOV(DisasContext *s, X86DecodedInsn *decode) 2494{ 2495 gen_lea_v_seg(s, cpu_regs[R_EDI], R_DS, s->override); 2496 2497 if (s->prefix & PREFIX_DATA) { 2498 gen_helper_maskmov_xmm(tcg_env, OP_PTR1, OP_PTR2, s->A0); 2499 } else { 2500 gen_helper_maskmov_mmx(tcg_env, OP_PTR1, OP_PTR2, s->A0); 2501 } 2502} 2503 2504static void gen_MOVBE(DisasContext *s, X86DecodedInsn *decode) 2505{ 2506 MemOp ot = decode->op[0].ot; 2507 2508 /* M operand type does not load/store */ 2509 if (decode->e.op0 == X86_TYPE_M) { 2510 tcg_gen_qemu_st_tl(s->T0, s->A0, s->mem_index, ot | MO_BE); 2511 } else { 2512 tcg_gen_qemu_ld_tl(s->T0, s->A0, s->mem_index, ot | MO_BE); 2513 } 2514} 2515 2516static void gen_MOVD_from(DisasContext *s, X86DecodedInsn *decode) 2517{ 2518 MemOp ot = decode->op[2].ot; 2519 2520 switch (ot) { 2521 case MO_32: 2522#ifdef TARGET_X86_64 2523 tcg_gen_ld32u_tl(s->T0, tcg_env, decode->op[2].offset); 2524 break; 2525 case MO_64: 2526#endif 2527 tcg_gen_ld_tl(s->T0, tcg_env, decode->op[2].offset); 2528 break; 2529 default: 2530 abort(); 2531 } 2532} 2533 2534static void gen_MOVD_to(DisasContext *s, X86DecodedInsn *decode) 2535{ 2536 MemOp ot = decode->op[2].ot; 2537 int vec_len = vector_len(s, decode); 2538 int lo_ofs = vector_elem_offset(&decode->op[0], ot, 0); 2539 2540 tcg_gen_gvec_dup_imm(MO_64, decode->op[0].offset, vec_len, vec_len, 0); 2541 2542 switch (ot) { 2543 case MO_32: 2544#ifdef TARGET_X86_64 2545 tcg_gen_st32_tl(s->T1, tcg_env, lo_ofs); 2546 break; 2547 case MO_64: 2548#endif 2549 tcg_gen_st_tl(s->T1, tcg_env, lo_ofs); 2550 break; 2551 default: 2552 g_assert_not_reached(); 2553 } 2554} 2555 2556static void gen_MOVDQ(DisasContext *s, X86DecodedInsn *decode) 2557{ 2558 gen_store_sse(s, decode, decode->op[2].offset); 2559} 2560 2561static void gen_MOVMSK(DisasContext *s, X86DecodedInsn *decode) 2562{ 2563 typeof(gen_helper_movmskps_ymm) *ps, *pd, *fn; 2564 ps = s->vex_l ? gen_helper_movmskps_ymm : gen_helper_movmskps_xmm; 2565 pd = s->vex_l ? gen_helper_movmskpd_ymm : gen_helper_movmskpd_xmm; 2566 fn = s->prefix & PREFIX_DATA ? pd : ps; 2567 fn(s->tmp2_i32, tcg_env, OP_PTR2); 2568 tcg_gen_extu_i32_tl(s->T0, s->tmp2_i32); 2569} 2570 2571static void gen_MOVQ(DisasContext *s, X86DecodedInsn *decode) 2572{ 2573 int vec_len = vector_len(s, decode); 2574 int lo_ofs = vector_elem_offset(&decode->op[0], MO_64, 0); 2575 2576 tcg_gen_ld_i64(s->tmp1_i64, tcg_env, decode->op[2].offset); 2577 if (decode->op[0].has_ea) { 2578 tcg_gen_qemu_st_i64(s->tmp1_i64, s->A0, s->mem_index, MO_LEUQ); 2579 } else { 2580 /* 2581 * tcg_gen_gvec_dup_i64(MO_64, op0.offset, 8, vec_len, s->tmp1_64) would 2582 * seem to work, but it does not on big-endian platforms; the cleared parts 2583 * are always at higher addresses, but cross-endian emulation inverts the 2584 * byte order so that the cleared parts need to be at *lower* addresses. 2585 * Because oprsz is 8, we see this here even for SSE; but more in general, 2586 * it disqualifies using oprsz < maxsz to emulate VEX128. 2587 */ 2588 tcg_gen_gvec_dup_imm(MO_64, decode->op[0].offset, vec_len, vec_len, 0); 2589 tcg_gen_st_i64(s->tmp1_i64, tcg_env, lo_ofs); 2590 } 2591} 2592 2593static void gen_MOVq_dq(DisasContext *s, X86DecodedInsn *decode) 2594{ 2595 gen_helper_enter_mmx(tcg_env); 2596 /* Otherwise the same as any other movq. */ 2597 return gen_MOVQ(s, decode); 2598} 2599 2600static void gen_MOVS(DisasContext *s, X86DecodedInsn *decode) 2601{ 2602 MemOp ot = decode->op[2].ot; 2603 if (s->prefix & (PREFIX_REPZ | PREFIX_REPNZ)) { 2604 gen_repz(s, ot, gen_movs); 2605 } else { 2606 gen_movs(s, ot); 2607 } 2608} 2609 2610static void gen_MUL(DisasContext *s, X86DecodedInsn *decode) 2611{ 2612 MemOp ot = decode->op[1].ot; 2613 2614 switch (ot) { 2615 case MO_8: 2616 /* s->T0 already zero-extended */ 2617 tcg_gen_ext8u_tl(s->T1, s->T1); 2618 tcg_gen_mul_tl(s->T0, s->T0, s->T1); 2619 gen_op_mov_reg_v(s, MO_16, R_EAX, s->T0); 2620 tcg_gen_andi_tl(s->T1, s->T0, 0xff00); 2621 decode->cc_dst = s->T0; 2622 decode->cc_src = s->T1; 2623 break; 2624 2625 case MO_16: 2626 /* s->T0 already zero-extended */ 2627 tcg_gen_ext16u_tl(s->T1, s->T1); 2628 tcg_gen_mul_tl(s->T0, s->T0, s->T1); 2629 gen_op_mov_reg_v(s, MO_16, R_EAX, s->T0); 2630 tcg_gen_shri_tl(s->T1, s->T0, 16); 2631 gen_op_mov_reg_v(s, MO_16, R_EDX, s->T1); 2632 decode->cc_dst = s->T0; 2633 decode->cc_src = s->T1; 2634 break; 2635 2636 case MO_32: 2637#ifdef TARGET_X86_64 2638 /* s->T0 already zero-extended */ 2639 tcg_gen_ext32u_tl(s->T1, s->T1); 2640 tcg_gen_mul_tl(s->T0, s->T0, s->T1); 2641 tcg_gen_ext32u_tl(cpu_regs[R_EAX], s->T0); 2642 tcg_gen_shri_tl(cpu_regs[R_EDX], s->T0, 32); 2643 decode->cc_dst = cpu_regs[R_EAX]; 2644 decode->cc_src = cpu_regs[R_EDX]; 2645 break; 2646 2647 case MO_64: 2648#endif 2649 tcg_gen_mulu2_tl(cpu_regs[R_EAX], cpu_regs[R_EDX], s->T0, s->T1); 2650 decode->cc_dst = cpu_regs[R_EAX]; 2651 decode->cc_src = cpu_regs[R_EDX]; 2652 break; 2653 2654 default: 2655 g_assert_not_reached(); 2656 } 2657 2658 decode->cc_op = CC_OP_MULB + ot; 2659} 2660 2661static void gen_MULX(DisasContext *s, X86DecodedInsn *decode) 2662{ 2663 MemOp ot = decode->op[0].ot; 2664 2665 /* low part of result in VEX.vvvv, high in MODRM */ 2666 switch (ot) { 2667 case MO_32: 2668#ifdef TARGET_X86_64 2669 tcg_gen_trunc_tl_i32(s->tmp2_i32, s->T0); 2670 tcg_gen_trunc_tl_i32(s->tmp3_i32, s->T1); 2671 tcg_gen_mulu2_i32(s->tmp2_i32, s->tmp3_i32, 2672 s->tmp2_i32, s->tmp3_i32); 2673 tcg_gen_extu_i32_tl(cpu_regs[s->vex_v], s->tmp2_i32); 2674 tcg_gen_extu_i32_tl(s->T0, s->tmp3_i32); 2675 break; 2676 2677 case MO_64: 2678#endif 2679 tcg_gen_mulu2_tl(cpu_regs[s->vex_v], s->T0, s->T0, s->T1); 2680 break; 2681 2682 default: 2683 g_assert_not_reached(); 2684 } 2685} 2686 2687static void gen_NEG(DisasContext *s, X86DecodedInsn *decode) 2688{ 2689 MemOp ot = decode->op[0].ot; 2690 TCGv oldv = tcg_temp_new(); 2691 2692 if (s->prefix & PREFIX_LOCK) { 2693 TCGv newv = tcg_temp_new(); 2694 TCGv cmpv = tcg_temp_new(); 2695 TCGLabel *label1 = gen_new_label(); 2696 2697 gen_set_label(label1); 2698 gen_op_ld_v(s, ot, oldv, s->A0); 2699 tcg_gen_neg_tl(newv, oldv); 2700 tcg_gen_atomic_cmpxchg_tl(cmpv, s->A0, oldv, newv, 2701 s->mem_index, ot | MO_LE); 2702 tcg_gen_brcond_tl(TCG_COND_NE, oldv, cmpv, label1); 2703 } else { 2704 tcg_gen_mov_tl(oldv, s->T0); 2705 } 2706 tcg_gen_neg_tl(s->T0, oldv); 2707 2708 decode->cc_dst = s->T0; 2709 decode->cc_src = oldv; 2710 tcg_gen_movi_tl(s->cc_srcT, 0); 2711 decode->cc_op = CC_OP_SUBB + ot; 2712} 2713 2714static void gen_NOT(DisasContext *s, X86DecodedInsn *decode) 2715{ 2716 MemOp ot = decode->op[0].ot; 2717 2718 if (s->prefix & PREFIX_LOCK) { 2719 tcg_gen_movi_tl(s->T0, ~0); 2720 tcg_gen_atomic_xor_fetch_tl(s->T0, s->A0, s->T0, 2721 s->mem_index, ot | MO_LE); 2722 } else { 2723 tcg_gen_not_tl(s->T0, s->T0); 2724 } 2725} 2726 2727static void gen_OR(DisasContext *s, X86DecodedInsn *decode) 2728{ 2729 MemOp ot = decode->op[1].ot; 2730 2731 if (s->prefix & PREFIX_LOCK) { 2732 tcg_gen_atomic_or_fetch_tl(s->T0, s->A0, s->T1, 2733 s->mem_index, ot | MO_LE); 2734 } else { 2735 tcg_gen_or_tl(s->T0, s->T0, s->T1); 2736 } 2737 prepare_update1_cc(decode, s, CC_OP_LOGICB + ot); 2738} 2739 2740static void gen_OUT(DisasContext *s, X86DecodedInsn *decode) 2741{ 2742 MemOp ot = decode->op[1].ot; 2743 TCGv_i32 port = tcg_temp_new_i32(); 2744 TCGv_i32 value = tcg_temp_new_i32(); 2745 2746 tcg_gen_trunc_tl_i32(port, s->T1); 2747 tcg_gen_ext16u_i32(port, port); 2748 if (!gen_check_io(s, ot, port, 0)) { 2749 return; 2750 } 2751 tcg_gen_trunc_tl_i32(value, s->T0); 2752 translator_io_start(&s->base); 2753 gen_helper_out_func(ot, port, value); 2754 gen_bpt_io(s, port, ot); 2755} 2756 2757static void gen_OUTS(DisasContext *s, X86DecodedInsn *decode) 2758{ 2759 MemOp ot = decode->op[1].ot; 2760 TCGv_i32 port = tcg_temp_new_i32(); 2761 2762 tcg_gen_trunc_tl_i32(port, s->T1); 2763 tcg_gen_ext16u_i32(port, port); 2764 if (!gen_check_io(s, ot, port, SVM_IOIO_STR_MASK)) { 2765 return; 2766 } 2767 2768 translator_io_start(&s->base); 2769 if (s->prefix & (PREFIX_REPZ | PREFIX_REPNZ)) { 2770 gen_repz(s, ot, gen_outs); 2771 } else { 2772 gen_outs(s, ot); 2773 } 2774} 2775 2776static void gen_PALIGNR(DisasContext *s, X86DecodedInsn *decode) 2777{ 2778 TCGv_i32 imm = tcg_constant8u_i32(decode->immediate); 2779 if (!(s->prefix & PREFIX_DATA)) { 2780 gen_helper_palignr_mmx(tcg_env, OP_PTR0, OP_PTR1, OP_PTR2, imm); 2781 } else if (!s->vex_l) { 2782 gen_helper_palignr_xmm(tcg_env, OP_PTR0, OP_PTR1, OP_PTR2, imm); 2783 } else { 2784 gen_helper_palignr_ymm(tcg_env, OP_PTR0, OP_PTR1, OP_PTR2, imm); 2785 } 2786} 2787 2788static void gen_PANDN(DisasContext *s, X86DecodedInsn *decode) 2789{ 2790 int vec_len = vector_len(s, decode); 2791 2792 /* Careful, operand order is reversed! */ 2793 tcg_gen_gvec_andc(MO_64, 2794 decode->op[0].offset, decode->op[2].offset, 2795 decode->op[1].offset, vec_len, vec_len); 2796} 2797 2798static void gen_PAUSE(DisasContext *s, X86DecodedInsn *decode) 2799{ 2800 gen_update_cc_op(s); 2801 gen_update_eip_next(s); 2802 gen_helper_pause(tcg_env); 2803 s->base.is_jmp = DISAS_NORETURN; 2804} 2805 2806static void gen_PCMPESTRI(DisasContext *s, X86DecodedInsn *decode) 2807{ 2808 TCGv_i32 imm = tcg_constant8u_i32(decode->immediate); 2809 gen_helper_pcmpestri_xmm(tcg_env, OP_PTR1, OP_PTR2, imm); 2810 assume_cc_op(s, CC_OP_EFLAGS); 2811} 2812 2813static void gen_PCMPESTRM(DisasContext *s, X86DecodedInsn *decode) 2814{ 2815 TCGv_i32 imm = tcg_constant8u_i32(decode->immediate); 2816 gen_helper_pcmpestrm_xmm(tcg_env, OP_PTR1, OP_PTR2, imm); 2817 assume_cc_op(s, CC_OP_EFLAGS); 2818 if ((s->prefix & PREFIX_VEX) && !s->vex_l) { 2819 tcg_gen_gvec_dup_imm(MO_64, offsetof(CPUX86State, xmm_regs[0].ZMM_X(1)), 2820 16, 16, 0); 2821 } 2822} 2823 2824static void gen_PCMPISTRI(DisasContext *s, X86DecodedInsn *decode) 2825{ 2826 TCGv_i32 imm = tcg_constant8u_i32(decode->immediate); 2827 gen_helper_pcmpistri_xmm(tcg_env, OP_PTR1, OP_PTR2, imm); 2828 assume_cc_op(s, CC_OP_EFLAGS); 2829} 2830 2831static void gen_PCMPISTRM(DisasContext *s, X86DecodedInsn *decode) 2832{ 2833 TCGv_i32 imm = tcg_constant8u_i32(decode->immediate); 2834 gen_helper_pcmpistrm_xmm(tcg_env, OP_PTR1, OP_PTR2, imm); 2835 assume_cc_op(s, CC_OP_EFLAGS); 2836 if ((s->prefix & PREFIX_VEX) && !s->vex_l) { 2837 tcg_gen_gvec_dup_imm(MO_64, offsetof(CPUX86State, xmm_regs[0].ZMM_X(1)), 2838 16, 16, 0); 2839 } 2840} 2841 2842static void gen_PDEP(DisasContext *s, X86DecodedInsn *decode) 2843{ 2844 gen_helper_pdep(s->T0, s->T0, s->T1); 2845} 2846 2847static void gen_PEXT(DisasContext *s, X86DecodedInsn *decode) 2848{ 2849 gen_helper_pext(s->T0, s->T0, s->T1); 2850} 2851 2852static inline void gen_pextr(DisasContext *s, X86DecodedInsn *decode, MemOp ot) 2853{ 2854 int vec_len = vector_len(s, decode); 2855 int mask = (vec_len >> ot) - 1; 2856 int val = decode->immediate & mask; 2857 2858 switch (ot) { 2859 case MO_8: 2860 tcg_gen_ld8u_tl(s->T0, tcg_env, vector_elem_offset(&decode->op[1], ot, val)); 2861 break; 2862 case MO_16: 2863 tcg_gen_ld16u_tl(s->T0, tcg_env, vector_elem_offset(&decode->op[1], ot, val)); 2864 break; 2865 case MO_32: 2866#ifdef TARGET_X86_64 2867 tcg_gen_ld32u_tl(s->T0, tcg_env, vector_elem_offset(&decode->op[1], ot, val)); 2868 break; 2869 case MO_64: 2870#endif 2871 tcg_gen_ld_tl(s->T0, tcg_env, vector_elem_offset(&decode->op[1], ot, val)); 2872 break; 2873 default: 2874 abort(); 2875 } 2876} 2877 2878static void gen_PEXTRB(DisasContext *s, X86DecodedInsn *decode) 2879{ 2880 gen_pextr(s, decode, MO_8); 2881} 2882 2883static void gen_PEXTRW(DisasContext *s, X86DecodedInsn *decode) 2884{ 2885 gen_pextr(s, decode, MO_16); 2886} 2887 2888static void gen_PEXTR(DisasContext *s, X86DecodedInsn *decode) 2889{ 2890 MemOp ot = decode->op[0].ot; 2891 gen_pextr(s, decode, ot); 2892} 2893 2894static inline void gen_pinsr(DisasContext *s, X86DecodedInsn *decode, MemOp ot) 2895{ 2896 int vec_len = vector_len(s, decode); 2897 int mask = (vec_len >> ot) - 1; 2898 int val = decode->immediate & mask; 2899 2900 if (decode->op[1].offset != decode->op[0].offset) { 2901 assert(vec_len == 16); 2902 gen_store_sse(s, decode, decode->op[1].offset); 2903 } 2904 2905 switch (ot) { 2906 case MO_8: 2907 tcg_gen_st8_tl(s->T1, tcg_env, vector_elem_offset(&decode->op[0], ot, val)); 2908 break; 2909 case MO_16: 2910 tcg_gen_st16_tl(s->T1, tcg_env, vector_elem_offset(&decode->op[0], ot, val)); 2911 break; 2912 case MO_32: 2913#ifdef TARGET_X86_64 2914 tcg_gen_st32_tl(s->T1, tcg_env, vector_elem_offset(&decode->op[0], ot, val)); 2915 break; 2916 case MO_64: 2917#endif 2918 tcg_gen_st_tl(s->T1, tcg_env, vector_elem_offset(&decode->op[0], ot, val)); 2919 break; 2920 default: 2921 abort(); 2922 } 2923} 2924 2925static void gen_PINSRB(DisasContext *s, X86DecodedInsn *decode) 2926{ 2927 gen_pinsr(s, decode, MO_8); 2928} 2929 2930static void gen_PINSRW(DisasContext *s, X86DecodedInsn *decode) 2931{ 2932 gen_pinsr(s, decode, MO_16); 2933} 2934 2935static void gen_PINSR(DisasContext *s, X86DecodedInsn *decode) 2936{ 2937 gen_pinsr(s, decode, decode->op[2].ot); 2938} 2939 2940static void gen_pmovmskb_i64(TCGv_i64 d, TCGv_i64 s) 2941{ 2942 TCGv_i64 t = tcg_temp_new_i64(); 2943 2944 tcg_gen_andi_i64(d, s, 0x8080808080808080ull); 2945 2946 /* 2947 * After each shift+or pair: 2948 * 0: a.......b.......c.......d.......e.......f.......g.......h....... 2949 * 7: ab......bc......cd......de......ef......fg......gh......h....... 2950 * 14: abcd....bcde....cdef....defg....efgh....fgh.....gh......h....... 2951 * 28: abcdefghbcdefgh.cdefgh..defgh...efgh....fgh.....gh......h....... 2952 * The result is left in the high bits of the word. 2953 */ 2954 tcg_gen_shli_i64(t, d, 7); 2955 tcg_gen_or_i64(d, d, t); 2956 tcg_gen_shli_i64(t, d, 14); 2957 tcg_gen_or_i64(d, d, t); 2958 tcg_gen_shli_i64(t, d, 28); 2959 tcg_gen_or_i64(d, d, t); 2960} 2961 2962static void gen_pmovmskb_vec(unsigned vece, TCGv_vec d, TCGv_vec s) 2963{ 2964 TCGv_vec t = tcg_temp_new_vec_matching(d); 2965 TCGv_vec m = tcg_constant_vec_matching(d, MO_8, 0x80); 2966 2967 /* See above */ 2968 tcg_gen_and_vec(vece, d, s, m); 2969 tcg_gen_shli_vec(vece, t, d, 7); 2970 tcg_gen_or_vec(vece, d, d, t); 2971 tcg_gen_shli_vec(vece, t, d, 14); 2972 tcg_gen_or_vec(vece, d, d, t); 2973 tcg_gen_shli_vec(vece, t, d, 28); 2974 tcg_gen_or_vec(vece, d, d, t); 2975} 2976 2977static void gen_PMOVMSKB(DisasContext *s, X86DecodedInsn *decode) 2978{ 2979 static const TCGOpcode vecop_list[] = { INDEX_op_shli_vec, 0 }; 2980 static const GVecGen2 g = { 2981 .fni8 = gen_pmovmskb_i64, 2982 .fniv = gen_pmovmskb_vec, 2983 .opt_opc = vecop_list, 2984 .vece = MO_64, 2985 .prefer_i64 = TCG_TARGET_REG_BITS == 64 2986 }; 2987 MemOp ot = decode->op[2].ot; 2988 int vec_len = vector_len(s, decode); 2989 TCGv t = tcg_temp_new(); 2990 2991 tcg_gen_gvec_2(offsetof(CPUX86State, xmm_t0) + xmm_offset(ot), decode->op[2].offset, 2992 vec_len, vec_len, &g); 2993 tcg_gen_ld8u_tl(s->T0, tcg_env, offsetof(CPUX86State, xmm_t0.ZMM_B(vec_len - 1))); 2994 while (vec_len > 8) { 2995 vec_len -= 8; 2996 if (tcg_op_supported(INDEX_op_extract2_tl, TCG_TYPE_TL, 0)) { 2997 /* 2998 * Load the next byte of the result into the high byte of T. 2999 * TCG does a similar expansion of deposit to shl+extract2; by 3000 * loading the whole word, the shift left is avoided. 3001 */ 3002#ifdef TARGET_X86_64 3003 tcg_gen_ld_tl(t, tcg_env, offsetof(CPUX86State, xmm_t0.ZMM_Q((vec_len - 1) / 8))); 3004#else 3005 tcg_gen_ld_tl(t, tcg_env, offsetof(CPUX86State, xmm_t0.ZMM_L((vec_len - 1) / 4))); 3006#endif 3007 3008 tcg_gen_extract2_tl(s->T0, t, s->T0, TARGET_LONG_BITS - 8); 3009 } else { 3010 /* 3011 * The _previous_ value is deposited into bits 8 and higher of t. Because 3012 * those bits are known to be zero after ld8u, this becomes a shift+or 3013 * if deposit is not available. 3014 */ 3015 tcg_gen_ld8u_tl(t, tcg_env, offsetof(CPUX86State, xmm_t0.ZMM_B(vec_len - 1))); 3016 tcg_gen_deposit_tl(s->T0, t, s->T0, 8, TARGET_LONG_BITS - 8); 3017 } 3018 } 3019} 3020 3021static void gen_POP(DisasContext *s, X86DecodedInsn *decode) 3022{ 3023 X86DecodedOp *op = &decode->op[0]; 3024 MemOp ot = gen_pop_T0(s); 3025 3026 assert(ot >= op->ot); 3027 if (op->has_ea || op->unit == X86_OP_SEG) { 3028 /* NOTE: order is important for MMU exceptions */ 3029 gen_writeback(s, decode, 0, s->T0); 3030 } 3031 3032 /* NOTE: writing back registers after update is important for pop %sp */ 3033 gen_pop_update(s, ot); 3034} 3035 3036static void gen_POPA(DisasContext *s, X86DecodedInsn *decode) 3037{ 3038 gen_popa(s); 3039} 3040 3041static void gen_POPCNT(DisasContext *s, X86DecodedInsn *decode) 3042{ 3043 decode->cc_dst = tcg_temp_new(); 3044 decode->cc_op = CC_OP_POPCNT; 3045 3046 tcg_gen_mov_tl(decode->cc_dst, s->T0); 3047 tcg_gen_ctpop_tl(s->T0, s->T0); 3048} 3049 3050static void gen_POPF(DisasContext *s, X86DecodedInsn *decode) 3051{ 3052 MemOp ot; 3053 int mask = TF_MASK | AC_MASK | ID_MASK | NT_MASK; 3054 3055 if (CPL(s) == 0) { 3056 mask |= IF_MASK | IOPL_MASK; 3057 } else if (CPL(s) <= IOPL(s)) { 3058 mask |= IF_MASK; 3059 } 3060 if (s->dflag == MO_16) { 3061 mask &= 0xffff; 3062 } 3063 3064 ot = gen_pop_T0(s); 3065 gen_helper_write_eflags(tcg_env, s->T0, tcg_constant_i32(mask)); 3066 gen_pop_update(s, ot); 3067 set_cc_op(s, CC_OP_EFLAGS); 3068 /* abort translation because TF/AC flag may change */ 3069 s->base.is_jmp = DISAS_EOB_NEXT; 3070} 3071 3072static void gen_PSHUFW(DisasContext *s, X86DecodedInsn *decode) 3073{ 3074 TCGv_i32 imm = tcg_constant8u_i32(decode->immediate); 3075 gen_helper_pshufw_mmx(OP_PTR0, OP_PTR1, imm); 3076} 3077 3078static void gen_PSRLW_i(DisasContext *s, X86DecodedInsn *decode) 3079{ 3080 int vec_len = vector_len(s, decode); 3081 3082 if (decode->immediate >= 16) { 3083 tcg_gen_gvec_dup_imm(MO_64, decode->op[0].offset, vec_len, vec_len, 0); 3084 } else { 3085 tcg_gen_gvec_shri(MO_16, 3086 decode->op[0].offset, decode->op[1].offset, 3087 decode->immediate, vec_len, vec_len); 3088 } 3089} 3090 3091static void gen_PSLLW_i(DisasContext *s, X86DecodedInsn *decode) 3092{ 3093 int vec_len = vector_len(s, decode); 3094 3095 if (decode->immediate >= 16) { 3096 tcg_gen_gvec_dup_imm(MO_64, decode->op[0].offset, vec_len, vec_len, 0); 3097 } else { 3098 tcg_gen_gvec_shli(MO_16, 3099 decode->op[0].offset, decode->op[1].offset, 3100 decode->immediate, vec_len, vec_len); 3101 } 3102} 3103 3104static void gen_PSRAW_i(DisasContext *s, X86DecodedInsn *decode) 3105{ 3106 int vec_len = vector_len(s, decode); 3107 3108 if (decode->immediate >= 16) { 3109 decode->immediate = 15; 3110 } 3111 tcg_gen_gvec_sari(MO_16, 3112 decode->op[0].offset, decode->op[1].offset, 3113 decode->immediate, vec_len, vec_len); 3114} 3115 3116static void gen_PSRLD_i(DisasContext *s, X86DecodedInsn *decode) 3117{ 3118 int vec_len = vector_len(s, decode); 3119 3120 if (decode->immediate >= 32) { 3121 tcg_gen_gvec_dup_imm(MO_64, decode->op[0].offset, vec_len, vec_len, 0); 3122 } else { 3123 tcg_gen_gvec_shri(MO_32, 3124 decode->op[0].offset, decode->op[1].offset, 3125 decode->immediate, vec_len, vec_len); 3126 } 3127} 3128 3129static void gen_PSLLD_i(DisasContext *s, X86DecodedInsn *decode) 3130{ 3131 int vec_len = vector_len(s, decode); 3132 3133 if (decode->immediate >= 32) { 3134 tcg_gen_gvec_dup_imm(MO_64, decode->op[0].offset, vec_len, vec_len, 0); 3135 } else { 3136 tcg_gen_gvec_shli(MO_32, 3137 decode->op[0].offset, decode->op[1].offset, 3138 decode->immediate, vec_len, vec_len); 3139 } 3140} 3141 3142static void gen_PSRAD_i(DisasContext *s, X86DecodedInsn *decode) 3143{ 3144 int vec_len = vector_len(s, decode); 3145 3146 if (decode->immediate >= 32) { 3147 decode->immediate = 31; 3148 } 3149 tcg_gen_gvec_sari(MO_32, 3150 decode->op[0].offset, decode->op[1].offset, 3151 decode->immediate, vec_len, vec_len); 3152} 3153 3154static void gen_PSRLQ_i(DisasContext *s, X86DecodedInsn *decode) 3155{ 3156 int vec_len = vector_len(s, decode); 3157 3158 if (decode->immediate >= 64) { 3159 tcg_gen_gvec_dup_imm(MO_64, decode->op[0].offset, vec_len, vec_len, 0); 3160 } else { 3161 tcg_gen_gvec_shri(MO_64, 3162 decode->op[0].offset, decode->op[1].offset, 3163 decode->immediate, vec_len, vec_len); 3164 } 3165} 3166 3167static void gen_PSLLQ_i(DisasContext *s, X86DecodedInsn *decode) 3168{ 3169 int vec_len = vector_len(s, decode); 3170 3171 if (decode->immediate >= 64) { 3172 tcg_gen_gvec_dup_imm(MO_64, decode->op[0].offset, vec_len, vec_len, 0); 3173 } else { 3174 tcg_gen_gvec_shli(MO_64, 3175 decode->op[0].offset, decode->op[1].offset, 3176 decode->immediate, vec_len, vec_len); 3177 } 3178} 3179 3180static TCGv_ptr make_imm8u_xmm_vec(uint8_t imm, int vec_len) 3181{ 3182 MemOp ot = vec_len == 16 ? MO_128 : MO_256; 3183 TCGv_i32 imm_v = tcg_constant8u_i32(imm); 3184 TCGv_ptr ptr = tcg_temp_new_ptr(); 3185 3186 tcg_gen_gvec_dup_imm(MO_64, offsetof(CPUX86State, xmm_t0) + xmm_offset(ot), 3187 vec_len, vec_len, 0); 3188 3189 tcg_gen_addi_ptr(ptr, tcg_env, offsetof(CPUX86State, xmm_t0)); 3190 tcg_gen_st_i32(imm_v, tcg_env, offsetof(CPUX86State, xmm_t0.ZMM_L(0))); 3191 return ptr; 3192} 3193 3194static void gen_PSRLDQ_i(DisasContext *s, X86DecodedInsn *decode) 3195{ 3196 int vec_len = vector_len(s, decode); 3197 TCGv_ptr imm_vec = make_imm8u_xmm_vec(decode->immediate, vec_len); 3198 3199 if (s->vex_l) { 3200 gen_helper_psrldq_ymm(tcg_env, OP_PTR0, OP_PTR1, imm_vec); 3201 } else { 3202 gen_helper_psrldq_xmm(tcg_env, OP_PTR0, OP_PTR1, imm_vec); 3203 } 3204} 3205 3206static void gen_PSLLDQ_i(DisasContext *s, X86DecodedInsn *decode) 3207{ 3208 int vec_len = vector_len(s, decode); 3209 TCGv_ptr imm_vec = make_imm8u_xmm_vec(decode->immediate, vec_len); 3210 3211 if (s->vex_l) { 3212 gen_helper_pslldq_ymm(tcg_env, OP_PTR0, OP_PTR1, imm_vec); 3213 } else { 3214 gen_helper_pslldq_xmm(tcg_env, OP_PTR0, OP_PTR1, imm_vec); 3215 } 3216} 3217 3218static void gen_PUSH(DisasContext *s, X86DecodedInsn *decode) 3219{ 3220 gen_push_v(s, s->T0); 3221} 3222 3223static void gen_PUSHA(DisasContext *s, X86DecodedInsn *decode) 3224{ 3225 gen_pusha(s); 3226} 3227 3228static void gen_PUSHF(DisasContext *s, X86DecodedInsn *decode) 3229{ 3230 gen_update_cc_op(s); 3231 gen_helper_read_eflags(s->T0, tcg_env); 3232 gen_push_v(s, s->T0); 3233} 3234 3235static MemOp gen_shift_count(DisasContext *s, X86DecodedInsn *decode, 3236 bool *can_be_zero, TCGv *count, int unit) 3237{ 3238 MemOp ot = decode->op[0].ot; 3239 int mask = (ot <= MO_32 ? 0x1f : 0x3f); 3240 3241 *can_be_zero = false; 3242 switch (unit) { 3243 case X86_OP_INT: 3244 *count = tcg_temp_new(); 3245 tcg_gen_andi_tl(*count, cpu_regs[R_ECX], mask); 3246 *can_be_zero = true; 3247 break; 3248 3249 case X86_OP_IMM: 3250 if ((decode->immediate & mask) == 0) { 3251 *count = NULL; 3252 break; 3253 } 3254 *count = tcg_temp_new(); 3255 tcg_gen_movi_tl(*count, decode->immediate & mask); 3256 break; 3257 3258 case X86_OP_SKIP: 3259 *count = tcg_temp_new(); 3260 tcg_gen_movi_tl(*count, 1); 3261 break; 3262 3263 default: 3264 g_assert_not_reached(); 3265 } 3266 3267 return ot; 3268} 3269 3270/* 3271 * Compute existing flags in decode->cc_src, for gen_* functions that wants 3272 * to set the cc_op set to CC_OP_ADCOX. In particular, this allows rotate 3273 * operations to compute the carry in decode->cc_dst and the overflow in 3274 * decode->cc_src2. 3275 * 3276 * If need_flags is true, decode->cc_dst and decode->cc_src2 are preloaded 3277 * with the value of CF and OF before the instruction, so that it is possible 3278 * to keep the flags unmodified. 3279 * 3280 * Return true if carry could be made available cheaply as a 1-bit value in 3281 * decode->cc_dst (trying a bit harder if want_carry is true). If false is 3282 * returned, decode->cc_dst is uninitialized and the carry is only available 3283 * as bit 0 of decode->cc_src. 3284 */ 3285static bool gen_eflags_adcox(DisasContext *s, X86DecodedInsn *decode, bool want_carry, bool need_flags) 3286{ 3287 bool got_cf = false; 3288 bool got_of = false; 3289 3290 decode->cc_dst = tcg_temp_new(); 3291 decode->cc_src = tcg_temp_new(); 3292 decode->cc_src2 = tcg_temp_new(); 3293 decode->cc_op = CC_OP_ADCOX; 3294 3295 /* A lot more cc_ops could be "optimized" to avoid the extracts at 3296 * the end (INC/DEC, BMILG, MUL), but they are all really unlikely 3297 * to be followed by rotations within the same basic block. 3298 */ 3299 switch (s->cc_op) { 3300 case CC_OP_ADCOX: 3301 /* No need to compute the full EFLAGS, CF/OF are already isolated. */ 3302 tcg_gen_mov_tl(decode->cc_src, cpu_cc_src); 3303 if (need_flags) { 3304 tcg_gen_mov_tl(decode->cc_src2, cpu_cc_src2); 3305 got_of = true; 3306 } 3307 if (want_carry || need_flags) { 3308 tcg_gen_mov_tl(decode->cc_dst, cpu_cc_dst); 3309 got_cf = true; 3310 } 3311 break; 3312 3313 case CC_OP_LOGICB ... CC_OP_LOGICQ: 3314 /* CF and OF are zero, do it just because it's easy. */ 3315 gen_mov_eflags(s, decode->cc_src); 3316 if (need_flags) { 3317 tcg_gen_movi_tl(decode->cc_src2, 0); 3318 got_of = true; 3319 } 3320 if (want_carry || need_flags) { 3321 tcg_gen_movi_tl(decode->cc_dst, 0); 3322 got_cf = true; 3323 } 3324 break; 3325 3326 case CC_OP_SARB ... CC_OP_SARQ: 3327 /* 3328 * SHR/RCR/SHR/RCR/... is a relatively common occurrence of RCR. 3329 * By computing CF without using eflags, the calls to cc_compute_all 3330 * can be eliminated as dead code (except for the last RCR). 3331 */ 3332 if (want_carry || need_flags) { 3333 tcg_gen_andi_tl(decode->cc_dst, cpu_cc_src, 1); 3334 got_cf = true; 3335 } 3336 gen_mov_eflags(s, decode->cc_src); 3337 break; 3338 3339 case CC_OP_SHLB ... CC_OP_SHLQ: 3340 /* 3341 * Likewise for SHL/RCL/SHL/RCL/... but, if CF is not in the sign 3342 * bit, we might as well fish CF out of EFLAGS and save a shift. 3343 */ 3344 if (want_carry && (!need_flags || s->cc_op == CC_OP_SHLB + MO_TL)) { 3345 MemOp size = cc_op_size(s->cc_op); 3346 tcg_gen_shri_tl(decode->cc_dst, cpu_cc_src, (8 << size) - 1); 3347 got_cf = true; 3348 } 3349 gen_mov_eflags(s, decode->cc_src); 3350 break; 3351 3352 default: 3353 gen_mov_eflags(s, decode->cc_src); 3354 break; 3355 } 3356 3357 if (need_flags) { 3358 /* If the flags could be left unmodified, always load them. */ 3359 if (!got_of) { 3360 tcg_gen_extract_tl(decode->cc_src2, decode->cc_src, ctz32(CC_O), 1); 3361 got_of = true; 3362 } 3363 if (!got_cf) { 3364 tcg_gen_extract_tl(decode->cc_dst, decode->cc_src, ctz32(CC_C), 1); 3365 got_cf = true; 3366 } 3367 } 3368 return got_cf; 3369} 3370 3371static void gen_rot_overflow(X86DecodedInsn *decode, TCGv result, TCGv old, 3372 bool can_be_zero, TCGv count) 3373{ 3374 MemOp ot = decode->op[0].ot; 3375 TCGv temp = can_be_zero ? tcg_temp_new() : decode->cc_src2; 3376 3377 tcg_gen_xor_tl(temp, old, result); 3378 tcg_gen_extract_tl(temp, temp, (8 << ot) - 1, 1); 3379 if (can_be_zero) { 3380 tcg_gen_movcond_tl(TCG_COND_EQ, decode->cc_src2, count, tcg_constant_tl(0), 3381 decode->cc_src2, temp); 3382 } 3383} 3384 3385/* 3386 * RCx operations are invariant modulo 8*operand_size+1. For 8 and 16-bit operands, 3387 * this is less than 0x1f (the mask applied by gen_shift_count) so reduce further. 3388 */ 3389static void gen_rotc_mod(MemOp ot, TCGv count) 3390{ 3391 TCGv temp; 3392 3393 switch (ot) { 3394 case MO_8: 3395 temp = tcg_temp_new(); 3396 tcg_gen_subi_tl(temp, count, 18); 3397 tcg_gen_movcond_tl(TCG_COND_GE, count, temp, tcg_constant_tl(0), temp, count); 3398 tcg_gen_subi_tl(temp, count, 9); 3399 tcg_gen_movcond_tl(TCG_COND_GE, count, temp, tcg_constant_tl(0), temp, count); 3400 break; 3401 3402 case MO_16: 3403 temp = tcg_temp_new(); 3404 tcg_gen_subi_tl(temp, count, 17); 3405 tcg_gen_movcond_tl(TCG_COND_GE, count, temp, tcg_constant_tl(0), temp, count); 3406 break; 3407 3408 default: 3409 break; 3410 } 3411} 3412 3413/* 3414 * The idea here is that the bit to the right of the new bit 0 is the 3415 * new carry, and the bit to the right of the old bit 0 is the old carry. 3416 * Just like a regular rotation, the result of the rotation is composed 3417 * from a right shifted part and a left shifted part of s->T0. The new carry 3418 * is extracted from the right-shifted portion, and the old carry is 3419 * inserted at the end of the left-shifted portion. 3420 * 3421 * Because of the separate shifts involving the carry, gen_RCL and gen_RCR 3422 * mostly operate on count-1. This also comes in handy when computing 3423 * length - count, because (length-1) - (count-1) can be computed with 3424 * a XOR, and that is commutative unlike subtraction. 3425 */ 3426static void gen_RCL(DisasContext *s, X86DecodedInsn *decode) 3427{ 3428 bool have_1bit_cin, can_be_zero; 3429 TCGv count; 3430 TCGLabel *zero_label = NULL; 3431 MemOp ot = gen_shift_count(s, decode, &can_be_zero, &count, decode->op[2].unit); 3432 TCGv low, high, low_count; 3433 3434 if (!count) { 3435 return; 3436 } 3437 3438 low = tcg_temp_new(); 3439 high = tcg_temp_new(); 3440 low_count = tcg_temp_new(); 3441 3442 gen_rotc_mod(ot, count); 3443 have_1bit_cin = gen_eflags_adcox(s, decode, true, can_be_zero); 3444 if (can_be_zero) { 3445 zero_label = gen_new_label(); 3446 tcg_gen_brcondi_tl(TCG_COND_EQ, count, 0, zero_label); 3447 } 3448 3449 /* Compute high part, including incoming carry. */ 3450 if (!have_1bit_cin || tcg_op_deposit_valid(TCG_TYPE_TL, 1, TARGET_LONG_BITS - 1)) { 3451 /* high = (T0 << 1) | cin */ 3452 TCGv cin = have_1bit_cin ? decode->cc_dst : decode->cc_src; 3453 tcg_gen_deposit_tl(high, cin, s->T0, 1, TARGET_LONG_BITS - 1); 3454 } else { 3455 /* Same as above but without deposit; cin in cc_dst. */ 3456 tcg_gen_add_tl(high, s->T0, decode->cc_dst); 3457 tcg_gen_add_tl(high, high, s->T0); 3458 } 3459 tcg_gen_subi_tl(count, count, 1); 3460 tcg_gen_shl_tl(high, high, count); 3461 3462 /* Compute low part and outgoing carry, incoming s->T0 is zero extended */ 3463 tcg_gen_xori_tl(low_count, count, (8 << ot) - 1); /* LENGTH - 1 - (count - 1) */ 3464 tcg_gen_shr_tl(low, s->T0, low_count); 3465 tcg_gen_andi_tl(decode->cc_dst, low, 1); 3466 tcg_gen_shri_tl(low, low, 1); 3467 3468 /* Compute result and outgoing overflow */ 3469 tcg_gen_mov_tl(decode->cc_src2, s->T0); 3470 tcg_gen_or_tl(s->T0, low, high); 3471 gen_rot_overflow(decode, s->T0, decode->cc_src2, false, NULL); 3472 3473 if (zero_label) { 3474 gen_set_label(zero_label); 3475 } 3476} 3477 3478static void gen_RCR(DisasContext *s, X86DecodedInsn *decode) 3479{ 3480 bool have_1bit_cin, can_be_zero; 3481 TCGv count; 3482 TCGLabel *zero_label = NULL; 3483 MemOp ot = gen_shift_count(s, decode, &can_be_zero, &count, decode->op[2].unit); 3484 TCGv low, high, high_count; 3485 3486 if (!count) { 3487 return; 3488 } 3489 3490 low = tcg_temp_new(); 3491 high = tcg_temp_new(); 3492 high_count = tcg_temp_new(); 3493 3494 gen_rotc_mod(ot, count); 3495 have_1bit_cin = gen_eflags_adcox(s, decode, true, can_be_zero); 3496 if (can_be_zero) { 3497 zero_label = gen_new_label(); 3498 tcg_gen_brcondi_tl(TCG_COND_EQ, count, 0, zero_label); 3499 } 3500 3501 /* Save incoming carry into high, it will be shifted later. */ 3502 if (!have_1bit_cin || tcg_op_deposit_valid(TCG_TYPE_TL, 1, TARGET_LONG_BITS - 1)) { 3503 TCGv cin = have_1bit_cin ? decode->cc_dst : decode->cc_src; 3504 tcg_gen_deposit_tl(high, cin, s->T0, 1, TARGET_LONG_BITS - 1); 3505 } else { 3506 /* Same as above but without deposit; cin in cc_dst. */ 3507 tcg_gen_add_tl(high, s->T0, decode->cc_dst); 3508 tcg_gen_add_tl(high, high, s->T0); 3509 } 3510 3511 /* Compute low part and outgoing carry, incoming s->T0 is zero extended */ 3512 tcg_gen_subi_tl(count, count, 1); 3513 tcg_gen_shr_tl(low, s->T0, count); 3514 tcg_gen_andi_tl(decode->cc_dst, low, 1); 3515 tcg_gen_shri_tl(low, low, 1); 3516 3517 /* Move high part to the right position */ 3518 tcg_gen_xori_tl(high_count, count, (8 << ot) - 1); /* LENGTH - 1 - (count - 1) */ 3519 tcg_gen_shl_tl(high, high, high_count); 3520 3521 /* Compute result and outgoing overflow */ 3522 tcg_gen_mov_tl(decode->cc_src2, s->T0); 3523 tcg_gen_or_tl(s->T0, low, high); 3524 gen_rot_overflow(decode, s->T0, decode->cc_src2, false, NULL); 3525 3526 if (zero_label) { 3527 gen_set_label(zero_label); 3528 } 3529} 3530 3531#ifdef CONFIG_USER_ONLY 3532static void gen_unreachable(DisasContext *s, X86DecodedInsn *decode) 3533{ 3534 g_assert_not_reached(); 3535} 3536#endif 3537 3538#ifndef CONFIG_USER_ONLY 3539static void gen_RDMSR(DisasContext *s, X86DecodedInsn *decode) 3540{ 3541 gen_update_cc_op(s); 3542 gen_update_eip_cur(s); 3543 gen_helper_rdmsr(tcg_env); 3544} 3545#else 3546#define gen_RDMSR gen_unreachable 3547#endif 3548 3549static void gen_RDPMC(DisasContext *s, X86DecodedInsn *decode) 3550{ 3551 gen_update_cc_op(s); 3552 gen_update_eip_cur(s); 3553 translator_io_start(&s->base); 3554 gen_helper_rdpmc(tcg_env); 3555 s->base.is_jmp = DISAS_NORETURN; 3556} 3557 3558static void gen_RDTSC(DisasContext *s, X86DecodedInsn *decode) 3559{ 3560 gen_update_cc_op(s); 3561 gen_update_eip_cur(s); 3562 translator_io_start(&s->base); 3563 gen_helper_rdtsc(tcg_env); 3564} 3565 3566static void gen_RDxxBASE(DisasContext *s, X86DecodedInsn *decode) 3567{ 3568 TCGv base = cpu_seg_base[s->modrm & 8 ? R_GS : R_FS]; 3569 3570 /* Preserve hflags bits by testing CR4 at runtime. */ 3571 gen_helper_cr4_testbit(tcg_env, tcg_constant_i32(CR4_FSGSBASE_MASK)); 3572 tcg_gen_mov_tl(s->T0, base); 3573} 3574 3575static void gen_RET(DisasContext *s, X86DecodedInsn *decode) 3576{ 3577 int16_t adjust = decode->e.op1 == X86_TYPE_I ? decode->immediate : 0; 3578 3579 MemOp ot = gen_pop_T0(s); 3580 gen_stack_update(s, adjust + (1 << ot)); 3581 gen_op_jmp_v(s, s->T0); 3582 gen_bnd_jmp(s); 3583 s->base.is_jmp = DISAS_JUMP; 3584} 3585 3586static void gen_RETF(DisasContext *s, X86DecodedInsn *decode) 3587{ 3588 int16_t adjust = decode->e.op1 == X86_TYPE_I ? decode->immediate : 0; 3589 3590 if (!PE(s) || VM86(s)) { 3591 gen_lea_ss_ofs(s, s->A0, cpu_regs[R_ESP], 0); 3592 /* pop offset */ 3593 gen_op_ld_v(s, s->dflag, s->T0, s->A0); 3594 /* NOTE: keeping EIP updated is not a problem in case of 3595 exception */ 3596 gen_op_jmp_v(s, s->T0); 3597 /* pop selector */ 3598 gen_add_A0_im(s, 1 << s->dflag); 3599 gen_op_ld_v(s, s->dflag, s->T0, s->A0); 3600 gen_op_movl_seg_real(s, R_CS, s->T0); 3601 /* add stack offset */ 3602 gen_stack_update(s, adjust + (2 << s->dflag)); 3603 } else { 3604 gen_update_cc_op(s); 3605 gen_update_eip_cur(s); 3606 gen_helper_lret_protected(tcg_env, tcg_constant_i32(s->dflag - 1), 3607 tcg_constant_i32(adjust)); 3608 } 3609 s->base.is_jmp = DISAS_EOB_ONLY; 3610} 3611 3612/* 3613 * Return non-NULL if a 32-bit rotate works, after possibly replicating the input. 3614 * The input has already been zero-extended upon operand decode. 3615 */ 3616static TCGv_i32 gen_rot_replicate(MemOp ot, TCGv in) 3617{ 3618 TCGv_i32 temp; 3619 switch (ot) { 3620 case MO_8: 3621 temp = tcg_temp_new_i32(); 3622 tcg_gen_trunc_tl_i32(temp, in); 3623 tcg_gen_muli_i32(temp, temp, 0x01010101); 3624 return temp; 3625 3626 case MO_16: 3627 temp = tcg_temp_new_i32(); 3628 tcg_gen_trunc_tl_i32(temp, in); 3629 tcg_gen_deposit_i32(temp, temp, temp, 16, 16); 3630 return temp; 3631 3632#ifdef TARGET_X86_64 3633 case MO_32: 3634 temp = tcg_temp_new_i32(); 3635 tcg_gen_trunc_tl_i32(temp, in); 3636 return temp; 3637#endif 3638 3639 default: 3640 return NULL; 3641 } 3642} 3643 3644static void gen_rot_carry(X86DecodedInsn *decode, TCGv result, 3645 bool can_be_zero, TCGv count, int bit) 3646{ 3647 if (!can_be_zero) { 3648 tcg_gen_extract_tl(decode->cc_dst, result, bit, 1); 3649 } else { 3650 TCGv temp = tcg_temp_new(); 3651 tcg_gen_extract_tl(temp, result, bit, 1); 3652 tcg_gen_movcond_tl(TCG_COND_EQ, decode->cc_dst, count, tcg_constant_tl(0), 3653 decode->cc_dst, temp); 3654 } 3655} 3656 3657static void gen_ROL(DisasContext *s, X86DecodedInsn *decode) 3658{ 3659 bool can_be_zero; 3660 TCGv count; 3661 MemOp ot = gen_shift_count(s, decode, &can_be_zero, &count, decode->op[2].unit); 3662 TCGv_i32 temp32, count32; 3663 TCGv old = tcg_temp_new(); 3664 3665 if (!count) { 3666 return; 3667 } 3668 3669 gen_eflags_adcox(s, decode, false, can_be_zero); 3670 tcg_gen_mov_tl(old, s->T0); 3671 temp32 = gen_rot_replicate(ot, s->T0); 3672 if (temp32) { 3673 count32 = tcg_temp_new_i32(); 3674 tcg_gen_trunc_tl_i32(count32, count); 3675 tcg_gen_rotl_i32(temp32, temp32, count32); 3676 /* Zero extend to facilitate later optimization. */ 3677 tcg_gen_extu_i32_tl(s->T0, temp32); 3678 } else { 3679 tcg_gen_rotl_tl(s->T0, s->T0, count); 3680 } 3681 gen_rot_carry(decode, s->T0, can_be_zero, count, 0); 3682 gen_rot_overflow(decode, s->T0, old, can_be_zero, count); 3683} 3684 3685static void gen_ROR(DisasContext *s, X86DecodedInsn *decode) 3686{ 3687 bool can_be_zero; 3688 TCGv count; 3689 MemOp ot = gen_shift_count(s, decode, &can_be_zero, &count, decode->op[2].unit); 3690 TCGv_i32 temp32, count32; 3691 TCGv old = tcg_temp_new(); 3692 3693 if (!count) { 3694 return; 3695 } 3696 3697 gen_eflags_adcox(s, decode, false, can_be_zero); 3698 tcg_gen_mov_tl(old, s->T0); 3699 temp32 = gen_rot_replicate(ot, s->T0); 3700 if (temp32) { 3701 count32 = tcg_temp_new_i32(); 3702 tcg_gen_trunc_tl_i32(count32, count); 3703 tcg_gen_rotr_i32(temp32, temp32, count32); 3704 /* Zero extend to facilitate later optimization. */ 3705 tcg_gen_extu_i32_tl(s->T0, temp32); 3706 gen_rot_carry(decode, s->T0, can_be_zero, count, 31); 3707 } else { 3708 tcg_gen_rotr_tl(s->T0, s->T0, count); 3709 gen_rot_carry(decode, s->T0, can_be_zero, count, TARGET_LONG_BITS - 1); 3710 } 3711 gen_rot_overflow(decode, s->T0, old, can_be_zero, count); 3712} 3713 3714static void gen_RORX(DisasContext *s, X86DecodedInsn *decode) 3715{ 3716 MemOp ot = decode->op[0].ot; 3717 int mask = ot == MO_64 ? 63 : 31; 3718 int b = decode->immediate & mask; 3719 3720 switch (ot) { 3721 case MO_32: 3722#ifdef TARGET_X86_64 3723 tcg_gen_trunc_tl_i32(s->tmp2_i32, s->T0); 3724 tcg_gen_rotri_i32(s->tmp2_i32, s->tmp2_i32, b); 3725 tcg_gen_extu_i32_tl(s->T0, s->tmp2_i32); 3726 break; 3727 3728 case MO_64: 3729#endif 3730 tcg_gen_rotri_tl(s->T0, s->T0, b); 3731 break; 3732 3733 default: 3734 g_assert_not_reached(); 3735 } 3736} 3737 3738#ifndef CONFIG_USER_ONLY 3739static void gen_RSM(DisasContext *s, X86DecodedInsn *decode) 3740{ 3741 gen_helper_rsm(tcg_env); 3742 assume_cc_op(s, CC_OP_EFLAGS); 3743 s->base.is_jmp = DISAS_EOB_ONLY; 3744} 3745#else 3746#define gen_RSM gen_UD 3747#endif 3748 3749static void gen_SAHF(DisasContext *s, X86DecodedInsn *decode) 3750{ 3751 if (CODE64(s) && !(s->cpuid_ext3_features & CPUID_EXT3_LAHF_LM)) { 3752 return gen_illegal_opcode(s); 3753 } 3754 tcg_gen_shri_tl(s->T0, cpu_regs[R_EAX], 8); 3755 gen_compute_eflags(s); 3756 tcg_gen_andi_tl(cpu_cc_src, cpu_cc_src, CC_O); 3757 tcg_gen_andi_tl(s->T0, s->T0, CC_S | CC_Z | CC_A | CC_P | CC_C); 3758 tcg_gen_or_tl(cpu_cc_src, cpu_cc_src, s->T0); 3759} 3760 3761static void gen_SALC(DisasContext *s, X86DecodedInsn *decode) 3762{ 3763 gen_compute_eflags_c(s, s->T0); 3764 tcg_gen_neg_tl(s->T0, s->T0); 3765} 3766 3767static void gen_shift_dynamic_flags(DisasContext *s, X86DecodedInsn *decode, TCGv count, CCOp cc_op) 3768{ 3769 TCGv_i32 count32 = tcg_temp_new_i32(); 3770 TCGv_i32 old_cc_op; 3771 3772 decode->cc_op = CC_OP_DYNAMIC; 3773 decode->cc_op_dynamic = tcg_temp_new_i32(); 3774 3775 assert(decode->cc_dst == s->T0); 3776 if (cc_op_live(s->cc_op) & USES_CC_DST) { 3777 decode->cc_dst = tcg_temp_new(); 3778 tcg_gen_movcond_tl(TCG_COND_EQ, decode->cc_dst, count, tcg_constant_tl(0), 3779 cpu_cc_dst, s->T0); 3780 } 3781 3782 if (cc_op_live(s->cc_op) & USES_CC_SRC) { 3783 tcg_gen_movcond_tl(TCG_COND_EQ, decode->cc_src, count, tcg_constant_tl(0), 3784 cpu_cc_src, decode->cc_src); 3785 } 3786 3787 tcg_gen_trunc_tl_i32(count32, count); 3788 if (s->cc_op == CC_OP_DYNAMIC) { 3789 old_cc_op = cpu_cc_op; 3790 } else { 3791 old_cc_op = tcg_constant_i32(s->cc_op); 3792 } 3793 tcg_gen_movcond_i32(TCG_COND_EQ, decode->cc_op_dynamic, count32, tcg_constant_i32(0), 3794 old_cc_op, tcg_constant_i32(cc_op)); 3795} 3796 3797static void gen_SAR(DisasContext *s, X86DecodedInsn *decode) 3798{ 3799 bool can_be_zero; 3800 TCGv count; 3801 MemOp ot = gen_shift_count(s, decode, &can_be_zero, &count, decode->op[2].unit); 3802 3803 if (!count) { 3804 return; 3805 } 3806 3807 decode->cc_dst = s->T0; 3808 decode->cc_src = tcg_temp_new(); 3809 tcg_gen_subi_tl(decode->cc_src, count, 1); 3810 tcg_gen_sar_tl(decode->cc_src, s->T0, decode->cc_src); 3811 tcg_gen_sar_tl(s->T0, s->T0, count); 3812 if (can_be_zero) { 3813 gen_shift_dynamic_flags(s, decode, count, CC_OP_SARB + ot); 3814 } else { 3815 decode->cc_op = CC_OP_SARB + ot; 3816 } 3817} 3818 3819static void gen_SARX(DisasContext *s, X86DecodedInsn *decode) 3820{ 3821 MemOp ot = decode->op[0].ot; 3822 int mask; 3823 3824 mask = ot == MO_64 ? 63 : 31; 3825 tcg_gen_andi_tl(s->T1, s->T1, mask); 3826 tcg_gen_sar_tl(s->T0, s->T0, s->T1); 3827} 3828 3829static void gen_SBB(DisasContext *s, X86DecodedInsn *decode) 3830{ 3831 MemOp ot = decode->op[0].ot; 3832 TCGv c_in = tcg_temp_new(); 3833 3834 gen_compute_eflags_c(s, c_in); 3835 if (s->prefix & PREFIX_LOCK) { 3836 tcg_gen_add_tl(s->T0, s->T1, c_in); 3837 tcg_gen_neg_tl(s->T0, s->T0); 3838 tcg_gen_atomic_add_fetch_tl(s->T0, s->A0, s->T0, 3839 s->mem_index, ot | MO_LE); 3840 } else { 3841 /* 3842 * TODO: SBB reg, reg could use gen_prepare_eflags_c followed by 3843 * negsetcond, and CC_OP_SUBB as the cc_op. 3844 */ 3845 tcg_gen_sub_tl(s->T0, s->T0, s->T1); 3846 tcg_gen_sub_tl(s->T0, s->T0, c_in); 3847 } 3848 prepare_update3_cc(decode, s, CC_OP_SBBB + ot, c_in); 3849} 3850 3851static void gen_SCAS(DisasContext *s, X86DecodedInsn *decode) 3852{ 3853 MemOp ot = decode->op[2].ot; 3854 if (s->prefix & (PREFIX_REPZ | PREFIX_REPNZ)) { 3855 gen_repz_nz(s, ot, gen_scas); 3856 } else { 3857 gen_scas(s, ot); 3858 } 3859} 3860 3861static void gen_SETcc(DisasContext *s, X86DecodedInsn *decode) 3862{ 3863 gen_setcc1(s, decode->b & 0xf, s->T0); 3864} 3865 3866static void gen_SFENCE(DisasContext *s, X86DecodedInsn *decode) 3867{ 3868 tcg_gen_mb(TCG_MO_ST_ST | TCG_BAR_SC); 3869} 3870 3871static void gen_SHA1NEXTE(DisasContext *s, X86DecodedInsn *decode) 3872{ 3873 gen_helper_sha1nexte(OP_PTR0, OP_PTR1, OP_PTR2); 3874} 3875 3876static void gen_SHA1MSG1(DisasContext *s, X86DecodedInsn *decode) 3877{ 3878 gen_helper_sha1msg1(OP_PTR0, OP_PTR1, OP_PTR2); 3879} 3880 3881static void gen_SHA1MSG2(DisasContext *s, X86DecodedInsn *decode) 3882{ 3883 gen_helper_sha1msg2(OP_PTR0, OP_PTR1, OP_PTR2); 3884} 3885 3886static void gen_SHA1RNDS4(DisasContext *s, X86DecodedInsn *decode) 3887{ 3888 switch(decode->immediate & 3) { 3889 case 0: 3890 gen_helper_sha1rnds4_f0(OP_PTR0, OP_PTR0, OP_PTR1); 3891 break; 3892 case 1: 3893 gen_helper_sha1rnds4_f1(OP_PTR0, OP_PTR0, OP_PTR1); 3894 break; 3895 case 2: 3896 gen_helper_sha1rnds4_f2(OP_PTR0, OP_PTR0, OP_PTR1); 3897 break; 3898 case 3: 3899 gen_helper_sha1rnds4_f3(OP_PTR0, OP_PTR0, OP_PTR1); 3900 break; 3901 } 3902} 3903 3904static void gen_SHA256MSG1(DisasContext *s, X86DecodedInsn *decode) 3905{ 3906 gen_helper_sha256msg1(OP_PTR0, OP_PTR1, OP_PTR2); 3907} 3908 3909static void gen_SHA256MSG2(DisasContext *s, X86DecodedInsn *decode) 3910{ 3911 gen_helper_sha256msg2(OP_PTR0, OP_PTR1, OP_PTR2); 3912} 3913 3914static void gen_SHA256RNDS2(DisasContext *s, X86DecodedInsn *decode) 3915{ 3916 TCGv_i32 wk0 = tcg_temp_new_i32(); 3917 TCGv_i32 wk1 = tcg_temp_new_i32(); 3918 3919 tcg_gen_ld_i32(wk0, tcg_env, ZMM_OFFSET(0) + offsetof(ZMMReg, ZMM_L(0))); 3920 tcg_gen_ld_i32(wk1, tcg_env, ZMM_OFFSET(0) + offsetof(ZMMReg, ZMM_L(1))); 3921 3922 gen_helper_sha256rnds2(OP_PTR0, OP_PTR1, OP_PTR2, wk0, wk1); 3923} 3924 3925static void gen_SHL(DisasContext *s, X86DecodedInsn *decode) 3926{ 3927 bool can_be_zero; 3928 TCGv count; 3929 MemOp ot = gen_shift_count(s, decode, &can_be_zero, &count, decode->op[2].unit); 3930 3931 if (!count) { 3932 return; 3933 } 3934 3935 decode->cc_dst = s->T0; 3936 decode->cc_src = tcg_temp_new(); 3937 tcg_gen_subi_tl(decode->cc_src, count, 1); 3938 tcg_gen_shl_tl(decode->cc_src, s->T0, decode->cc_src); 3939 tcg_gen_shl_tl(s->T0, s->T0, count); 3940 if (can_be_zero) { 3941 gen_shift_dynamic_flags(s, decode, count, CC_OP_SHLB + ot); 3942 } else { 3943 decode->cc_op = CC_OP_SHLB + ot; 3944 } 3945} 3946 3947static void gen_SHLD(DisasContext *s, X86DecodedInsn *decode) 3948{ 3949 bool can_be_zero; 3950 TCGv count; 3951 int unit = decode->e.op3 == X86_TYPE_I ? X86_OP_IMM : X86_OP_INT; 3952 MemOp ot = gen_shift_count(s, decode, &can_be_zero, &count, unit); 3953 3954 if (!count) { 3955 return; 3956 } 3957 3958 decode->cc_dst = s->T0; 3959 decode->cc_src = s->tmp0; 3960 gen_shiftd_rm_T1(s, ot, false, count); 3961 if (can_be_zero) { 3962 gen_shift_dynamic_flags(s, decode, count, CC_OP_SHLB + ot); 3963 } else { 3964 decode->cc_op = CC_OP_SHLB + ot; 3965 } 3966} 3967 3968static void gen_SHLX(DisasContext *s, X86DecodedInsn *decode) 3969{ 3970 MemOp ot = decode->op[0].ot; 3971 int mask; 3972 3973 mask = ot == MO_64 ? 63 : 31; 3974 tcg_gen_andi_tl(s->T1, s->T1, mask); 3975 tcg_gen_shl_tl(s->T0, s->T0, s->T1); 3976} 3977 3978static void gen_SHR(DisasContext *s, X86DecodedInsn *decode) 3979{ 3980 bool can_be_zero; 3981 TCGv count; 3982 MemOp ot = gen_shift_count(s, decode, &can_be_zero, &count, decode->op[2].unit); 3983 3984 if (!count) { 3985 return; 3986 } 3987 3988 decode->cc_dst = s->T0; 3989 decode->cc_src = tcg_temp_new(); 3990 tcg_gen_subi_tl(decode->cc_src, count, 1); 3991 tcg_gen_shr_tl(decode->cc_src, s->T0, decode->cc_src); 3992 tcg_gen_shr_tl(s->T0, s->T0, count); 3993 if (can_be_zero) { 3994 gen_shift_dynamic_flags(s, decode, count, CC_OP_SARB + ot); 3995 } else { 3996 decode->cc_op = CC_OP_SARB + ot; 3997 } 3998} 3999 4000static void gen_SHRD(DisasContext *s, X86DecodedInsn *decode) 4001{ 4002 bool can_be_zero; 4003 TCGv count; 4004 int unit = decode->e.op3 == X86_TYPE_I ? X86_OP_IMM : X86_OP_INT; 4005 MemOp ot = gen_shift_count(s, decode, &can_be_zero, &count, unit); 4006 4007 if (!count) { 4008 return; 4009 } 4010 4011 decode->cc_dst = s->T0; 4012 decode->cc_src = s->tmp0; 4013 gen_shiftd_rm_T1(s, ot, true, count); 4014 if (can_be_zero) { 4015 gen_shift_dynamic_flags(s, decode, count, CC_OP_SARB + ot); 4016 } else { 4017 decode->cc_op = CC_OP_SARB + ot; 4018 } 4019} 4020 4021static void gen_SHRX(DisasContext *s, X86DecodedInsn *decode) 4022{ 4023 MemOp ot = decode->op[0].ot; 4024 int mask; 4025 4026 mask = ot == MO_64 ? 63 : 31; 4027 tcg_gen_andi_tl(s->T1, s->T1, mask); 4028 tcg_gen_shr_tl(s->T0, s->T0, s->T1); 4029} 4030 4031static void gen_STC(DisasContext *s, X86DecodedInsn *decode) 4032{ 4033 gen_compute_eflags(s); 4034 tcg_gen_ori_tl(cpu_cc_src, cpu_cc_src, CC_C); 4035} 4036 4037static void gen_STD(DisasContext *s, X86DecodedInsn *decode) 4038{ 4039 tcg_gen_st_i32(tcg_constant_i32(-1), tcg_env, offsetof(CPUX86State, df)); 4040} 4041 4042static void gen_STI(DisasContext *s, X86DecodedInsn *decode) 4043{ 4044 gen_set_eflags(s, IF_MASK); 4045 s->base.is_jmp = DISAS_EOB_INHIBIT_IRQ; 4046} 4047 4048static void gen_VAESKEYGEN(DisasContext *s, X86DecodedInsn *decode) 4049{ 4050 TCGv_i32 imm = tcg_constant8u_i32(decode->immediate); 4051 assert(!s->vex_l); 4052 gen_helper_aeskeygenassist_xmm(tcg_env, OP_PTR0, OP_PTR1, imm); 4053} 4054 4055static void gen_STMXCSR(DisasContext *s, X86DecodedInsn *decode) 4056{ 4057 gen_helper_update_mxcsr(tcg_env); 4058 tcg_gen_ld32u_tl(s->T0, tcg_env, offsetof(CPUX86State, mxcsr)); 4059} 4060 4061static void gen_STOS(DisasContext *s, X86DecodedInsn *decode) 4062{ 4063 MemOp ot = decode->op[1].ot; 4064 if (s->prefix & (PREFIX_REPZ | PREFIX_REPNZ)) { 4065 gen_repz(s, ot, gen_stos); 4066 } else { 4067 gen_stos(s, ot); 4068 } 4069} 4070 4071static void gen_SUB(DisasContext *s, X86DecodedInsn *decode) 4072{ 4073 MemOp ot = decode->op[1].ot; 4074 4075 if (s->prefix & PREFIX_LOCK) { 4076 tcg_gen_neg_tl(s->T0, s->T1); 4077 tcg_gen_atomic_fetch_add_tl(s->cc_srcT, s->A0, s->T0, 4078 s->mem_index, ot | MO_LE); 4079 tcg_gen_sub_tl(s->T0, s->cc_srcT, s->T1); 4080 } else { 4081 tcg_gen_mov_tl(s->cc_srcT, s->T0); 4082 tcg_gen_sub_tl(s->T0, s->T0, s->T1); 4083 } 4084 prepare_update2_cc(decode, s, CC_OP_SUBB + ot); 4085} 4086 4087static void gen_SYSCALL(DisasContext *s, X86DecodedInsn *decode) 4088{ 4089 gen_update_cc_op(s); 4090 gen_update_eip_cur(s); 4091 gen_helper_syscall(tcg_env, cur_insn_len_i32(s)); 4092 if (LMA(s)) { 4093 assume_cc_op(s, CC_OP_EFLAGS); 4094 } 4095 4096 /* 4097 * TF handling for the syscall insn is different. The TF bit is checked 4098 * after the syscall insn completes. This allows #DB to not be 4099 * generated after one has entered CPL0 if TF is set in FMASK. 4100 */ 4101 s->base.is_jmp = DISAS_EOB_RECHECK_TF; 4102} 4103 4104static void gen_SYSENTER(DisasContext *s, X86DecodedInsn *decode) 4105{ 4106 gen_helper_sysenter(tcg_env); 4107 s->base.is_jmp = DISAS_EOB_ONLY; 4108} 4109 4110static void gen_SYSEXIT(DisasContext *s, X86DecodedInsn *decode) 4111{ 4112 gen_helper_sysexit(tcg_env, tcg_constant_i32(s->dflag - 1)); 4113 s->base.is_jmp = DISAS_EOB_ONLY; 4114} 4115 4116static void gen_SYSRET(DisasContext *s, X86DecodedInsn *decode) 4117{ 4118 gen_helper_sysret(tcg_env, tcg_constant_i32(s->dflag - 1)); 4119 if (LMA(s)) { 4120 assume_cc_op(s, CC_OP_EFLAGS); 4121 } 4122 4123 /* 4124 * TF handling for the sysret insn is different. The TF bit is checked 4125 * after the sysret insn completes. This allows #DB to be 4126 * generated "as if" the syscall insn in userspace has just 4127 * completed. 4128 */ 4129 s->base.is_jmp = DISAS_EOB_RECHECK_TF; 4130} 4131 4132static void gen_TZCNT(DisasContext *s, X86DecodedInsn *decode) 4133{ 4134 MemOp ot = decode->op[0].ot; 4135 4136 /* C bit (cc_src) is defined related to the input. */ 4137 decode->cc_src = tcg_temp_new(); 4138 decode->cc_dst = s->T0; 4139 decode->cc_op = CC_OP_BMILGB + ot; 4140 tcg_gen_mov_tl(decode->cc_src, s->T0); 4141 4142 /* A zero input returns the operand size. */ 4143 tcg_gen_ctzi_tl(s->T0, s->T0, 8 << ot); 4144} 4145 4146static void gen_UD(DisasContext *s, X86DecodedInsn *decode) 4147{ 4148 gen_illegal_opcode(s); 4149} 4150 4151static void gen_VAESIMC(DisasContext *s, X86DecodedInsn *decode) 4152{ 4153 assert(!s->vex_l); 4154 gen_helper_aesimc_xmm(tcg_env, OP_PTR0, OP_PTR2); 4155} 4156 4157/* 4158 * 00 = v*ps Vps, Hps, Wpd 4159 * 66 = v*pd Vpd, Hpd, Wps 4160 * f3 = v*ss Vss, Hss, Wps 4161 * f2 = v*sd Vsd, Hsd, Wps 4162 */ 4163#define SSE_CMP(x) { \ 4164 gen_helper_ ## x ## ps ## _xmm, gen_helper_ ## x ## pd ## _xmm, \ 4165 gen_helper_ ## x ## ss, gen_helper_ ## x ## sd, \ 4166 gen_helper_ ## x ## ps ## _ymm, gen_helper_ ## x ## pd ## _ymm} 4167static const SSEFunc_0_eppp gen_helper_cmp_funcs[32][6] = { 4168 SSE_CMP(cmpeq), 4169 SSE_CMP(cmplt), 4170 SSE_CMP(cmple), 4171 SSE_CMP(cmpunord), 4172 SSE_CMP(cmpneq), 4173 SSE_CMP(cmpnlt), 4174 SSE_CMP(cmpnle), 4175 SSE_CMP(cmpord), 4176 4177 SSE_CMP(cmpequ), 4178 SSE_CMP(cmpnge), 4179 SSE_CMP(cmpngt), 4180 SSE_CMP(cmpfalse), 4181 SSE_CMP(cmpnequ), 4182 SSE_CMP(cmpge), 4183 SSE_CMP(cmpgt), 4184 SSE_CMP(cmptrue), 4185 4186 SSE_CMP(cmpeqs), 4187 SSE_CMP(cmpltq), 4188 SSE_CMP(cmpleq), 4189 SSE_CMP(cmpunords), 4190 SSE_CMP(cmpneqq), 4191 SSE_CMP(cmpnltq), 4192 SSE_CMP(cmpnleq), 4193 SSE_CMP(cmpords), 4194 4195 SSE_CMP(cmpequs), 4196 SSE_CMP(cmpngeq), 4197 SSE_CMP(cmpngtq), 4198 SSE_CMP(cmpfalses), 4199 SSE_CMP(cmpnequs), 4200 SSE_CMP(cmpgeq), 4201 SSE_CMP(cmpgtq), 4202 SSE_CMP(cmptrues), 4203}; 4204#undef SSE_CMP 4205 4206static void gen_VCMP(DisasContext *s, X86DecodedInsn *decode) 4207{ 4208 int index = decode->immediate & (s->prefix & PREFIX_VEX ? 31 : 7); 4209 int b = 4210 s->prefix & PREFIX_REPZ ? 2 /* ss */ : 4211 s->prefix & PREFIX_REPNZ ? 3 /* sd */ : 4212 !!(s->prefix & PREFIX_DATA) /* pd */ + (s->vex_l << 2); 4213 4214 gen_helper_cmp_funcs[index][b](tcg_env, OP_PTR0, OP_PTR1, OP_PTR2); 4215} 4216 4217static void gen_VCOMI(DisasContext *s, X86DecodedInsn *decode) 4218{ 4219 SSEFunc_0_epp fn; 4220 fn = s->prefix & PREFIX_DATA ? gen_helper_comisd : gen_helper_comiss; 4221 fn(tcg_env, OP_PTR1, OP_PTR2); 4222 assume_cc_op(s, CC_OP_EFLAGS); 4223} 4224 4225static void gen_VCVTPD2PS(DisasContext *s, X86DecodedInsn *decode) 4226{ 4227 if (s->vex_l) { 4228 gen_helper_cvtpd2ps_ymm(tcg_env, OP_PTR0, OP_PTR2); 4229 } else { 4230 gen_helper_cvtpd2ps_xmm(tcg_env, OP_PTR0, OP_PTR2); 4231 } 4232} 4233 4234static void gen_VCVTPS2PD(DisasContext *s, X86DecodedInsn *decode) 4235{ 4236 if (s->vex_l) { 4237 gen_helper_cvtps2pd_ymm(tcg_env, OP_PTR0, OP_PTR2); 4238 } else { 4239 gen_helper_cvtps2pd_xmm(tcg_env, OP_PTR0, OP_PTR2); 4240 } 4241} 4242 4243static void gen_VCVTPS2PH(DisasContext *s, X86DecodedInsn *decode) 4244{ 4245 gen_unary_imm_fp_sse(s, decode, 4246 gen_helper_cvtps2ph_xmm, 4247 gen_helper_cvtps2ph_ymm); 4248 /* 4249 * VCVTPS2PH is the only instruction that performs an operation on a 4250 * register source and then *stores* into memory. 4251 */ 4252 if (decode->op[0].has_ea) { 4253 gen_store_sse(s, decode, decode->op[0].offset); 4254 } 4255} 4256 4257static void gen_VCVTSD2SS(DisasContext *s, X86DecodedInsn *decode) 4258{ 4259 gen_helper_cvtsd2ss(tcg_env, OP_PTR0, OP_PTR1, OP_PTR2); 4260} 4261 4262static void gen_VCVTSS2SD(DisasContext *s, X86DecodedInsn *decode) 4263{ 4264 gen_helper_cvtss2sd(tcg_env, OP_PTR0, OP_PTR1, OP_PTR2); 4265} 4266 4267static void gen_VCVTSI2Sx(DisasContext *s, X86DecodedInsn *decode) 4268{ 4269 int vec_len = vector_len(s, decode); 4270 TCGv_i32 in; 4271 4272 tcg_gen_gvec_mov(MO_64, decode->op[0].offset, decode->op[1].offset, vec_len, vec_len); 4273 4274#ifdef TARGET_X86_64 4275 MemOp ot = decode->op[2].ot; 4276 if (ot == MO_64) { 4277 if (s->prefix & PREFIX_REPNZ) { 4278 gen_helper_cvtsq2sd(tcg_env, OP_PTR0, s->T1); 4279 } else { 4280 gen_helper_cvtsq2ss(tcg_env, OP_PTR0, s->T1); 4281 } 4282 return; 4283 } 4284 in = s->tmp2_i32; 4285 tcg_gen_trunc_tl_i32(in, s->T1); 4286#else 4287 in = s->T1; 4288#endif 4289 4290 if (s->prefix & PREFIX_REPNZ) { 4291 gen_helper_cvtsi2sd(tcg_env, OP_PTR0, in); 4292 } else { 4293 gen_helper_cvtsi2ss(tcg_env, OP_PTR0, in); 4294 } 4295} 4296 4297static inline void gen_VCVTtSx2SI(DisasContext *s, X86DecodedInsn *decode, 4298 SSEFunc_i_ep ss2si, SSEFunc_l_ep ss2sq, 4299 SSEFunc_i_ep sd2si, SSEFunc_l_ep sd2sq) 4300{ 4301 TCGv_i32 out; 4302 4303#ifdef TARGET_X86_64 4304 MemOp ot = decode->op[0].ot; 4305 if (ot == MO_64) { 4306 if (s->prefix & PREFIX_REPNZ) { 4307 sd2sq(s->T0, tcg_env, OP_PTR2); 4308 } else { 4309 ss2sq(s->T0, tcg_env, OP_PTR2); 4310 } 4311 return; 4312 } 4313 4314 out = s->tmp2_i32; 4315#else 4316 out = s->T0; 4317#endif 4318 if (s->prefix & PREFIX_REPNZ) { 4319 sd2si(out, tcg_env, OP_PTR2); 4320 } else { 4321 ss2si(out, tcg_env, OP_PTR2); 4322 } 4323#ifdef TARGET_X86_64 4324 tcg_gen_extu_i32_tl(s->T0, out); 4325#endif 4326} 4327 4328#ifndef TARGET_X86_64 4329#define gen_helper_cvtss2sq NULL 4330#define gen_helper_cvtsd2sq NULL 4331#define gen_helper_cvttss2sq NULL 4332#define gen_helper_cvttsd2sq NULL 4333#endif 4334 4335static void gen_VCVTSx2SI(DisasContext *s, X86DecodedInsn *decode) 4336{ 4337 gen_VCVTtSx2SI(s, decode, 4338 gen_helper_cvtss2si, gen_helper_cvtss2sq, 4339 gen_helper_cvtsd2si, gen_helper_cvtsd2sq); 4340} 4341 4342static void gen_VCVTTSx2SI(DisasContext *s, X86DecodedInsn *decode) 4343{ 4344 gen_VCVTtSx2SI(s, decode, 4345 gen_helper_cvttss2si, gen_helper_cvttss2sq, 4346 gen_helper_cvttsd2si, gen_helper_cvttsd2sq); 4347} 4348 4349static void gen_VEXTRACTx128(DisasContext *s, X86DecodedInsn *decode) 4350{ 4351 int mask = decode->immediate & 1; 4352 int src_ofs = vector_elem_offset(&decode->op[1], MO_128, mask); 4353 if (decode->op[0].has_ea) { 4354 /* VEX-only instruction, no alignment requirements. */ 4355 gen_sto_env_A0(s, src_ofs, false); 4356 } else { 4357 tcg_gen_gvec_mov(MO_64, decode->op[0].offset, src_ofs, 16, 16); 4358 } 4359} 4360 4361static void gen_VEXTRACTPS(DisasContext *s, X86DecodedInsn *decode) 4362{ 4363 gen_pextr(s, decode, MO_32); 4364} 4365 4366static void gen_vinsertps(DisasContext *s, X86DecodedInsn *decode) 4367{ 4368 int val = decode->immediate; 4369 int dest_word = (val >> 4) & 3; 4370 int new_mask = (val & 15) | (1 << dest_word); 4371 int vec_len = 16; 4372 4373 assert(!s->vex_l); 4374 4375 if (new_mask == 15) { 4376 /* All zeroes except possibly for the inserted element */ 4377 tcg_gen_gvec_dup_imm(MO_64, decode->op[0].offset, vec_len, vec_len, 0); 4378 } else if (decode->op[1].offset != decode->op[0].offset) { 4379 gen_store_sse(s, decode, decode->op[1].offset); 4380 } 4381 4382 if (new_mask != (val & 15)) { 4383 tcg_gen_st_i32(s->tmp2_i32, tcg_env, 4384 vector_elem_offset(&decode->op[0], MO_32, dest_word)); 4385 } 4386 4387 if (new_mask != 15) { 4388 TCGv_i32 zero = tcg_constant_i32(0); /* float32_zero */ 4389 int i; 4390 for (i = 0; i < 4; i++) { 4391 if ((val >> i) & 1) { 4392 tcg_gen_st_i32(zero, tcg_env, 4393 vector_elem_offset(&decode->op[0], MO_32, i)); 4394 } 4395 } 4396 } 4397} 4398 4399static void gen_VINSERTPS_r(DisasContext *s, X86DecodedInsn *decode) 4400{ 4401 int val = decode->immediate; 4402 tcg_gen_ld_i32(s->tmp2_i32, tcg_env, 4403 vector_elem_offset(&decode->op[2], MO_32, (val >> 6) & 3)); 4404 gen_vinsertps(s, decode); 4405} 4406 4407static void gen_VINSERTPS_m(DisasContext *s, X86DecodedInsn *decode) 4408{ 4409 tcg_gen_qemu_ld_i32(s->tmp2_i32, s->A0, s->mem_index, MO_LEUL); 4410 gen_vinsertps(s, decode); 4411} 4412 4413static void gen_VINSERTx128(DisasContext *s, X86DecodedInsn *decode) 4414{ 4415 int mask = decode->immediate & 1; 4416 tcg_gen_gvec_mov(MO_64, 4417 decode->op[0].offset + offsetof(YMMReg, YMM_X(mask)), 4418 decode->op[2].offset + offsetof(YMMReg, YMM_X(0)), 16, 16); 4419 tcg_gen_gvec_mov(MO_64, 4420 decode->op[0].offset + offsetof(YMMReg, YMM_X(!mask)), 4421 decode->op[1].offset + offsetof(YMMReg, YMM_X(!mask)), 16, 16); 4422} 4423 4424static inline void gen_maskmov(DisasContext *s, X86DecodedInsn *decode, 4425 SSEFunc_0_eppt xmm, SSEFunc_0_eppt ymm) 4426{ 4427 if (!s->vex_l) { 4428 xmm(tcg_env, OP_PTR2, OP_PTR1, s->A0); 4429 } else { 4430 ymm(tcg_env, OP_PTR2, OP_PTR1, s->A0); 4431 } 4432} 4433 4434static void gen_VMASKMOVPD_st(DisasContext *s, X86DecodedInsn *decode) 4435{ 4436 gen_maskmov(s, decode, gen_helper_vpmaskmovq_st_xmm, gen_helper_vpmaskmovq_st_ymm); 4437} 4438 4439static void gen_VMASKMOVPS_st(DisasContext *s, X86DecodedInsn *decode) 4440{ 4441 gen_maskmov(s, decode, gen_helper_vpmaskmovd_st_xmm, gen_helper_vpmaskmovd_st_ymm); 4442} 4443 4444static void gen_VMOVHPx_ld(DisasContext *s, X86DecodedInsn *decode) 4445{ 4446 gen_ldq_env_A0(s, decode->op[0].offset + offsetof(XMMReg, XMM_Q(1))); 4447 if (decode->op[0].offset != decode->op[1].offset) { 4448 tcg_gen_ld_i64(s->tmp1_i64, tcg_env, decode->op[1].offset + offsetof(XMMReg, XMM_Q(0))); 4449 tcg_gen_st_i64(s->tmp1_i64, tcg_env, decode->op[0].offset + offsetof(XMMReg, XMM_Q(0))); 4450 } 4451} 4452 4453static void gen_VMOVHPx_st(DisasContext *s, X86DecodedInsn *decode) 4454{ 4455 gen_stq_env_A0(s, decode->op[2].offset + offsetof(XMMReg, XMM_Q(1))); 4456} 4457 4458static void gen_VMOVHPx(DisasContext *s, X86DecodedInsn *decode) 4459{ 4460 if (decode->op[0].offset != decode->op[2].offset) { 4461 tcg_gen_ld_i64(s->tmp1_i64, tcg_env, decode->op[2].offset + offsetof(XMMReg, XMM_Q(1))); 4462 tcg_gen_st_i64(s->tmp1_i64, tcg_env, decode->op[0].offset + offsetof(XMMReg, XMM_Q(1))); 4463 } 4464 if (decode->op[0].offset != decode->op[1].offset) { 4465 tcg_gen_ld_i64(s->tmp1_i64, tcg_env, decode->op[1].offset + offsetof(XMMReg, XMM_Q(0))); 4466 tcg_gen_st_i64(s->tmp1_i64, tcg_env, decode->op[0].offset + offsetof(XMMReg, XMM_Q(0))); 4467 } 4468} 4469 4470static void gen_VMOVHLPS(DisasContext *s, X86DecodedInsn *decode) 4471{ 4472 tcg_gen_ld_i64(s->tmp1_i64, tcg_env, decode->op[2].offset + offsetof(XMMReg, XMM_Q(1))); 4473 tcg_gen_st_i64(s->tmp1_i64, tcg_env, decode->op[0].offset + offsetof(XMMReg, XMM_Q(0))); 4474 if (decode->op[0].offset != decode->op[1].offset) { 4475 tcg_gen_ld_i64(s->tmp1_i64, tcg_env, decode->op[1].offset + offsetof(XMMReg, XMM_Q(1))); 4476 tcg_gen_st_i64(s->tmp1_i64, tcg_env, decode->op[0].offset + offsetof(XMMReg, XMM_Q(1))); 4477 } 4478} 4479 4480static void gen_VMOVLHPS(DisasContext *s, X86DecodedInsn *decode) 4481{ 4482 tcg_gen_ld_i64(s->tmp1_i64, tcg_env, decode->op[2].offset); 4483 tcg_gen_st_i64(s->tmp1_i64, tcg_env, decode->op[0].offset + offsetof(XMMReg, XMM_Q(1))); 4484 if (decode->op[0].offset != decode->op[1].offset) { 4485 tcg_gen_ld_i64(s->tmp1_i64, tcg_env, decode->op[1].offset + offsetof(XMMReg, XMM_Q(0))); 4486 tcg_gen_st_i64(s->tmp1_i64, tcg_env, decode->op[0].offset + offsetof(XMMReg, XMM_Q(0))); 4487 } 4488} 4489 4490/* 4491 * Note that MOVLPx supports 256-bit operation unlike MOVHLPx, MOVLHPx, MOXHPx. 4492 * Use a gvec move to move everything above the bottom 64 bits. 4493 */ 4494 4495static void gen_VMOVLPx(DisasContext *s, X86DecodedInsn *decode) 4496{ 4497 int vec_len = vector_len(s, decode); 4498 4499 tcg_gen_ld_i64(s->tmp1_i64, tcg_env, decode->op[2].offset + offsetof(XMMReg, XMM_Q(0))); 4500 tcg_gen_gvec_mov(MO_64, decode->op[0].offset, decode->op[1].offset, vec_len, vec_len); 4501 tcg_gen_st_i64(s->tmp1_i64, tcg_env, decode->op[0].offset + offsetof(XMMReg, XMM_Q(0))); 4502} 4503 4504static void gen_VMOVLPx_ld(DisasContext *s, X86DecodedInsn *decode) 4505{ 4506 int vec_len = vector_len(s, decode); 4507 4508 tcg_gen_qemu_ld_i64(s->tmp1_i64, s->A0, s->mem_index, MO_LEUQ); 4509 tcg_gen_gvec_mov(MO_64, decode->op[0].offset, decode->op[1].offset, vec_len, vec_len); 4510 tcg_gen_st_i64(s->tmp1_i64, OP_PTR0, offsetof(ZMMReg, ZMM_Q(0))); 4511} 4512 4513static void gen_VMOVLPx_st(DisasContext *s, X86DecodedInsn *decode) 4514{ 4515 tcg_gen_ld_i64(s->tmp1_i64, OP_PTR2, offsetof(ZMMReg, ZMM_Q(0))); 4516 tcg_gen_qemu_st_i64(s->tmp1_i64, s->A0, s->mem_index, MO_LEUQ); 4517} 4518 4519static void gen_VMOVSD_ld(DisasContext *s, X86DecodedInsn *decode) 4520{ 4521 TCGv_i64 zero = tcg_constant_i64(0); 4522 4523 tcg_gen_qemu_ld_i64(s->tmp1_i64, s->A0, s->mem_index, MO_LEUQ); 4524 tcg_gen_st_i64(zero, OP_PTR0, offsetof(ZMMReg, ZMM_Q(1))); 4525 tcg_gen_st_i64(s->tmp1_i64, OP_PTR0, offsetof(ZMMReg, ZMM_Q(0))); 4526} 4527 4528static void gen_VMOVSS(DisasContext *s, X86DecodedInsn *decode) 4529{ 4530 int vec_len = vector_len(s, decode); 4531 4532 tcg_gen_ld_i32(s->tmp2_i32, OP_PTR2, offsetof(ZMMReg, ZMM_L(0))); 4533 tcg_gen_gvec_mov(MO_64, decode->op[0].offset, decode->op[1].offset, vec_len, vec_len); 4534 tcg_gen_st_i32(s->tmp2_i32, OP_PTR0, offsetof(ZMMReg, ZMM_L(0))); 4535} 4536 4537static void gen_VMOVSS_ld(DisasContext *s, X86DecodedInsn *decode) 4538{ 4539 int vec_len = vector_len(s, decode); 4540 4541 tcg_gen_qemu_ld_i32(s->tmp2_i32, s->A0, s->mem_index, MO_LEUL); 4542 tcg_gen_gvec_dup_imm(MO_64, decode->op[0].offset, vec_len, vec_len, 0); 4543 tcg_gen_st_i32(s->tmp2_i32, OP_PTR0, offsetof(ZMMReg, ZMM_L(0))); 4544} 4545 4546static void gen_VMOVSS_st(DisasContext *s, X86DecodedInsn *decode) 4547{ 4548 tcg_gen_ld_i32(s->tmp2_i32, OP_PTR2, offsetof(ZMMReg, ZMM_L(0))); 4549 tcg_gen_qemu_st_i32(s->tmp2_i32, s->A0, s->mem_index, MO_LEUL); 4550} 4551 4552static void gen_VPMASKMOV_st(DisasContext *s, X86DecodedInsn *decode) 4553{ 4554 if (s->vex_w) { 4555 gen_VMASKMOVPD_st(s, decode); 4556 } else { 4557 gen_VMASKMOVPS_st(s, decode); 4558 } 4559} 4560 4561static void gen_VPERMD(DisasContext *s, X86DecodedInsn *decode) 4562{ 4563 assert(s->vex_l); 4564 gen_helper_vpermd_ymm(OP_PTR0, OP_PTR1, OP_PTR2); 4565} 4566 4567static void gen_VPERM2x128(DisasContext *s, X86DecodedInsn *decode) 4568{ 4569 TCGv_i32 imm = tcg_constant8u_i32(decode->immediate); 4570 assert(s->vex_l); 4571 gen_helper_vpermdq_ymm(OP_PTR0, OP_PTR1, OP_PTR2, imm); 4572} 4573 4574static void gen_VPHMINPOSUW(DisasContext *s, X86DecodedInsn *decode) 4575{ 4576 assert(!s->vex_l); 4577 gen_helper_phminposuw_xmm(tcg_env, OP_PTR0, OP_PTR2); 4578} 4579 4580static void gen_VROUNDSD(DisasContext *s, X86DecodedInsn *decode) 4581{ 4582 TCGv_i32 imm = tcg_constant8u_i32(decode->immediate); 4583 assert(!s->vex_l); 4584 gen_helper_roundsd_xmm(tcg_env, OP_PTR0, OP_PTR1, OP_PTR2, imm); 4585} 4586 4587static void gen_VROUNDSS(DisasContext *s, X86DecodedInsn *decode) 4588{ 4589 TCGv_i32 imm = tcg_constant8u_i32(decode->immediate); 4590 assert(!s->vex_l); 4591 gen_helper_roundss_xmm(tcg_env, OP_PTR0, OP_PTR1, OP_PTR2, imm); 4592} 4593 4594static void gen_VSHUF(DisasContext *s, X86DecodedInsn *decode) 4595{ 4596 TCGv_i32 imm = tcg_constant_i32(decode->immediate); 4597 SSEFunc_0_pppi ps, pd, fn; 4598 ps = s->vex_l ? gen_helper_shufps_ymm : gen_helper_shufps_xmm; 4599 pd = s->vex_l ? gen_helper_shufpd_ymm : gen_helper_shufpd_xmm; 4600 fn = s->prefix & PREFIX_DATA ? pd : ps; 4601 fn(OP_PTR0, OP_PTR1, OP_PTR2, imm); 4602} 4603 4604static void gen_VUCOMI(DisasContext *s, X86DecodedInsn *decode) 4605{ 4606 SSEFunc_0_epp fn; 4607 fn = s->prefix & PREFIX_DATA ? gen_helper_ucomisd : gen_helper_ucomiss; 4608 fn(tcg_env, OP_PTR1, OP_PTR2); 4609 assume_cc_op(s, CC_OP_EFLAGS); 4610} 4611 4612static void gen_VZEROALL(DisasContext *s, X86DecodedInsn *decode) 4613{ 4614 TCGv_ptr ptr = tcg_temp_new_ptr(); 4615 4616 tcg_gen_addi_ptr(ptr, tcg_env, offsetof(CPUX86State, xmm_regs)); 4617 gen_helper_memset(ptr, ptr, tcg_constant_i32(0), 4618 tcg_constant_ptr(CPU_NB_REGS * sizeof(ZMMReg))); 4619} 4620 4621static void gen_VZEROUPPER(DisasContext *s, X86DecodedInsn *decode) 4622{ 4623 int i; 4624 4625 for (i = 0; i < CPU_NB_REGS; i++) { 4626 int offset = offsetof(CPUX86State, xmm_regs[i].ZMM_X(1)); 4627 tcg_gen_gvec_dup_imm(MO_64, offset, 16, 16, 0); 4628 } 4629} 4630 4631static void gen_WAIT(DisasContext *s, X86DecodedInsn *decode) 4632{ 4633 if ((s->flags & (HF_MP_MASK | HF_TS_MASK)) == (HF_MP_MASK | HF_TS_MASK)) { 4634 gen_NM_exception(s); 4635 } else { 4636 /* needs to be treated as I/O because of ferr_irq */ 4637 translator_io_start(&s->base); 4638 gen_helper_fwait(tcg_env); 4639 } 4640} 4641 4642#ifndef CONFIG_USER_ONLY 4643static void gen_WRMSR(DisasContext *s, X86DecodedInsn *decode) 4644{ 4645 gen_update_cc_op(s); 4646 gen_update_eip_cur(s); 4647 gen_helper_wrmsr(tcg_env); 4648 s->base.is_jmp = DISAS_EOB_NEXT; 4649} 4650#else 4651#define gen_WRMSR gen_unreachable 4652#endif 4653 4654static void gen_WRxxBASE(DisasContext *s, X86DecodedInsn *decode) 4655{ 4656 TCGv base = cpu_seg_base[s->modrm & 8 ? R_GS : R_FS]; 4657 4658 /* Preserve hflags bits by testing CR4 at runtime. */ 4659 gen_helper_cr4_testbit(tcg_env, tcg_constant_i32(CR4_FSGSBASE_MASK)); 4660 tcg_gen_mov_tl(base, s->T0); 4661} 4662 4663static void gen_XADD(DisasContext *s, X86DecodedInsn *decode) 4664{ 4665 MemOp ot = decode->op[1].ot; 4666 4667 decode->cc_dst = tcg_temp_new(); 4668 decode->cc_src = s->T1; 4669 decode->cc_op = CC_OP_ADDB + ot; 4670 4671 if (s->prefix & PREFIX_LOCK) { 4672 tcg_gen_atomic_fetch_add_tl(s->T0, s->A0, s->T1, s->mem_index, ot | MO_LE); 4673 tcg_gen_add_tl(decode->cc_dst, s->T0, s->T1); 4674 } else { 4675 tcg_gen_add_tl(decode->cc_dst, s->T0, s->T1); 4676 /* 4677 * NOTE: writing memory first is important for MMU exceptions, 4678 * but "new result" wins for XADD AX, AX. 4679 */ 4680 gen_writeback(s, decode, 0, decode->cc_dst); 4681 } 4682 if (decode->op[0].has_ea || decode->op[2].n != decode->op[0].n) { 4683 gen_writeback(s, decode, 2, s->T0); 4684 } 4685} 4686 4687static void gen_XCHG(DisasContext *s, X86DecodedInsn *decode) 4688{ 4689 if (s->prefix & PREFIX_LOCK) { 4690 tcg_gen_atomic_xchg_tl(s->T0, s->A0, s->T1, 4691 s->mem_index, decode->op[0].ot | MO_LE); 4692 /* now store old value into register operand */ 4693 gen_op_mov_reg_v(s, decode->op[2].ot, decode->op[2].n, s->T0); 4694 } else { 4695 /* move destination value into source operand, source preserved in T1 */ 4696 gen_op_mov_reg_v(s, decode->op[2].ot, decode->op[2].n, s->T0); 4697 tcg_gen_mov_tl(s->T0, s->T1); 4698 } 4699} 4700 4701static void gen_XLAT(DisasContext *s, X86DecodedInsn *decode) 4702{ 4703 /* AL is already zero-extended into s->T0. */ 4704 tcg_gen_add_tl(s->A0, cpu_regs[R_EBX], s->T0); 4705 gen_lea_v_seg(s, s->A0, R_DS, s->override); 4706 gen_op_ld_v(s, MO_8, s->T0, s->A0); 4707} 4708 4709static void gen_XOR(DisasContext *s, X86DecodedInsn *decode) 4710{ 4711 /* special case XOR reg, reg */ 4712 if (decode->op[1].unit == X86_OP_INT && 4713 decode->op[2].unit == X86_OP_INT && 4714 decode->op[1].n == decode->op[2].n) { 4715 tcg_gen_movi_tl(s->T0, 0); 4716 decode->cc_op = CC_OP_EFLAGS; 4717 decode->cc_src = tcg_constant_tl(CC_Z | CC_P); 4718 } else { 4719 MemOp ot = decode->op[1].ot; 4720 4721 if (s->prefix & PREFIX_LOCK) { 4722 tcg_gen_atomic_xor_fetch_tl(s->T0, s->A0, s->T1, 4723 s->mem_index, ot | MO_LE); 4724 } else { 4725 tcg_gen_xor_tl(s->T0, s->T0, s->T1); 4726 } 4727 prepare_update1_cc(decode, s, CC_OP_LOGICB + ot); 4728 } 4729} 4730 4731static void gen_XRSTOR(DisasContext *s, X86DecodedInsn *decode) 4732{ 4733 TCGv_i64 features = tcg_temp_new_i64(); 4734 4735 tcg_gen_concat_tl_i64(features, cpu_regs[R_EAX], cpu_regs[R_EDX]); 4736 gen_helper_xrstor(tcg_env, s->A0, features); 4737 if (s->cpuid_7_0_ebx_features & CPUID_7_0_EBX_MPX) { 4738 /* 4739 * XRSTOR is how MPX is enabled, which changes how 4740 * we translate. Thus we need to end the TB. 4741 */ 4742 s->base.is_jmp = DISAS_EOB_NEXT; 4743 } 4744} 4745 4746static void gen_XSAVE(DisasContext *s, X86DecodedInsn *decode) 4747{ 4748 TCGv_i64 features = tcg_temp_new_i64(); 4749 4750 tcg_gen_concat_tl_i64(features, cpu_regs[R_EAX], cpu_regs[R_EDX]); 4751 gen_helper_xsave(tcg_env, s->A0, features); 4752} 4753 4754static void gen_XSAVEOPT(DisasContext *s, X86DecodedInsn *decode) 4755{ 4756 TCGv_i64 features = tcg_temp_new_i64(); 4757 4758 tcg_gen_concat_tl_i64(features, cpu_regs[R_EAX], cpu_regs[R_EDX]); 4759 gen_helper_xsave(tcg_env, s->A0, features); 4760} 4761