1/* 2 * Tiny Code Generator for QEMU 3 * 4 * Copyright (c) 2008 Fabrice Bellard 5 * 6 * Permission is hereby granted, free of charge, to any person obtaining a copy 7 * of this software and associated documentation files (the "Software"), to deal 8 * in the Software without restriction, including without limitation the rights 9 * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 10 * copies of the Software, and to permit persons to whom the Software is 11 * furnished to do so, subject to the following conditions: 12 * 13 * The above copyright notice and this permission notice shall be included in 14 * all copies or substantial portions of the Software. 15 * 16 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 17 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 18 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL 19 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 20 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 21 * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN 22 * THE SOFTWARE. 23 */ 24 25#include "../tcg-ldst.c.inc" 26#include "../tcg-pool.c.inc" 27 28#ifdef CONFIG_DEBUG_TCG 29static const char * const tcg_target_reg_names[TCG_TARGET_NB_REGS] = { 30#if TCG_TARGET_REG_BITS == 64 31 "%rax", "%rcx", "%rdx", "%rbx", "%rsp", "%rbp", "%rsi", "%rdi", 32#else 33 "%eax", "%ecx", "%edx", "%ebx", "%esp", "%ebp", "%esi", "%edi", 34#endif 35 "%r8", "%r9", "%r10", "%r11", "%r12", "%r13", "%r14", "%r15", 36 "%xmm0", "%xmm1", "%xmm2", "%xmm3", "%xmm4", "%xmm5", "%xmm6", "%xmm7", 37#if TCG_TARGET_REG_BITS == 64 38 "%xmm8", "%xmm9", "%xmm10", "%xmm11", 39 "%xmm12", "%xmm13", "%xmm14", "%xmm15", 40#endif 41}; 42#endif 43 44static const int tcg_target_reg_alloc_order[] = { 45#if TCG_TARGET_REG_BITS == 64 46 TCG_REG_RBP, 47 TCG_REG_RBX, 48 TCG_REG_R12, 49 TCG_REG_R13, 50 TCG_REG_R14, 51 TCG_REG_R15, 52 TCG_REG_R10, 53 TCG_REG_R11, 54 TCG_REG_R9, 55 TCG_REG_R8, 56 TCG_REG_RCX, 57 TCG_REG_RDX, 58 TCG_REG_RSI, 59 TCG_REG_RDI, 60 TCG_REG_RAX, 61#else 62 TCG_REG_EBX, 63 TCG_REG_ESI, 64 TCG_REG_EDI, 65 TCG_REG_EBP, 66 TCG_REG_ECX, 67 TCG_REG_EDX, 68 TCG_REG_EAX, 69#endif 70 TCG_REG_XMM0, 71 TCG_REG_XMM1, 72 TCG_REG_XMM2, 73 TCG_REG_XMM3, 74 TCG_REG_XMM4, 75 TCG_REG_XMM5, 76#ifndef _WIN64 77 /* The Win64 ABI has xmm6-xmm15 as caller-saves, and we do not save 78 any of them. Therefore only allow xmm0-xmm5 to be allocated. */ 79 TCG_REG_XMM6, 80 TCG_REG_XMM7, 81#if TCG_TARGET_REG_BITS == 64 82 TCG_REG_XMM8, 83 TCG_REG_XMM9, 84 TCG_REG_XMM10, 85 TCG_REG_XMM11, 86 TCG_REG_XMM12, 87 TCG_REG_XMM13, 88 TCG_REG_XMM14, 89 TCG_REG_XMM15, 90#endif 91#endif 92}; 93 94static const int tcg_target_call_iarg_regs[] = { 95#if TCG_TARGET_REG_BITS == 64 96#if defined(_WIN64) 97 TCG_REG_RCX, 98 TCG_REG_RDX, 99#else 100 TCG_REG_RDI, 101 TCG_REG_RSI, 102 TCG_REG_RDX, 103 TCG_REG_RCX, 104#endif 105 TCG_REG_R8, 106 TCG_REG_R9, 107#else 108 /* 32 bit mode uses stack based calling convention (GCC default). */ 109#endif 110}; 111 112static const int tcg_target_call_oarg_regs[] = { 113 TCG_REG_EAX, 114#if TCG_TARGET_REG_BITS == 32 115 TCG_REG_EDX 116#endif 117}; 118 119/* Constants we accept. */ 120#define TCG_CT_CONST_S32 0x100 121#define TCG_CT_CONST_U32 0x200 122#define TCG_CT_CONST_I32 0x400 123#define TCG_CT_CONST_WSZ 0x800 124 125/* Registers used with L constraint, which are the first argument 126 registers on x86_64, and two random call clobbered registers on 127 i386. */ 128#if TCG_TARGET_REG_BITS == 64 129# define TCG_REG_L0 tcg_target_call_iarg_regs[0] 130# define TCG_REG_L1 tcg_target_call_iarg_regs[1] 131#else 132# define TCG_REG_L0 TCG_REG_EAX 133# define TCG_REG_L1 TCG_REG_EDX 134#endif 135 136#define ALL_BYTEH_REGS 0x0000000fu 137#if TCG_TARGET_REG_BITS == 64 138# define ALL_GENERAL_REGS 0x0000ffffu 139# define ALL_VECTOR_REGS 0xffff0000u 140# define ALL_BYTEL_REGS ALL_GENERAL_REGS 141#else 142# define ALL_GENERAL_REGS 0x000000ffu 143# define ALL_VECTOR_REGS 0x00ff0000u 144# define ALL_BYTEL_REGS ALL_BYTEH_REGS 145#endif 146#ifdef CONFIG_SOFTMMU 147# define SOFTMMU_RESERVE_REGS ((1 << TCG_REG_L0) | (1 << TCG_REG_L1)) 148#else 149# define SOFTMMU_RESERVE_REGS 0 150#endif 151 152/* The host compiler should supply <cpuid.h> to enable runtime features 153 detection, as we're not going to go so far as our own inline assembly. 154 If not available, default values will be assumed. */ 155#if defined(CONFIG_CPUID_H) 156#include "qemu/cpuid.h" 157#endif 158 159/* For 64-bit, we always know that CMOV is available. */ 160#if TCG_TARGET_REG_BITS == 64 161# define have_cmov 1 162#elif defined(CONFIG_CPUID_H) 163static bool have_cmov; 164#else 165# define have_cmov 0 166#endif 167 168/* We need these symbols in tcg-target.h, and we can't properly conditionalize 169 it there. Therefore we always define the variable. */ 170bool have_bmi1; 171bool have_popcnt; 172bool have_avx1; 173bool have_avx2; 174bool have_avx512bw; 175bool have_avx512dq; 176bool have_avx512vbmi2; 177bool have_avx512vl; 178bool have_movbe; 179 180#ifdef CONFIG_CPUID_H 181static bool have_bmi2; 182static bool have_lzcnt; 183#else 184# define have_bmi2 0 185# define have_lzcnt 0 186#endif 187 188static const tcg_insn_unit *tb_ret_addr; 189 190static bool patch_reloc(tcg_insn_unit *code_ptr, int type, 191 intptr_t value, intptr_t addend) 192{ 193 value += addend; 194 switch(type) { 195 case R_386_PC32: 196 value -= (uintptr_t)tcg_splitwx_to_rx(code_ptr); 197 if (value != (int32_t)value) { 198 return false; 199 } 200 /* FALLTHRU */ 201 case R_386_32: 202 tcg_patch32(code_ptr, value); 203 break; 204 case R_386_PC8: 205 value -= (uintptr_t)tcg_splitwx_to_rx(code_ptr); 206 if (value != (int8_t)value) { 207 return false; 208 } 209 tcg_patch8(code_ptr, value); 210 break; 211 default: 212 tcg_abort(); 213 } 214 return true; 215} 216 217/* test if a constant matches the constraint */ 218static bool tcg_target_const_match(int64_t val, TCGType type, int ct) 219{ 220 if (ct & TCG_CT_CONST) { 221 return 1; 222 } 223 if (type == TCG_TYPE_I32) { 224 if (ct & (TCG_CT_CONST_S32 | TCG_CT_CONST_U32 | TCG_CT_CONST_I32)) { 225 return 1; 226 } 227 } else { 228 if ((ct & TCG_CT_CONST_S32) && val == (int32_t)val) { 229 return 1; 230 } 231 if ((ct & TCG_CT_CONST_U32) && val == (uint32_t)val) { 232 return 1; 233 } 234 if ((ct & TCG_CT_CONST_I32) && ~val == (int32_t)~val) { 235 return 1; 236 } 237 } 238 if ((ct & TCG_CT_CONST_WSZ) && val == (type == TCG_TYPE_I32 ? 32 : 64)) { 239 return 1; 240 } 241 return 0; 242} 243 244# define LOWREGMASK(x) ((x) & 7) 245 246#define P_EXT 0x100 /* 0x0f opcode prefix */ 247#define P_EXT38 0x200 /* 0x0f 0x38 opcode prefix */ 248#define P_DATA16 0x400 /* 0x66 opcode prefix */ 249#define P_VEXW 0x1000 /* Set VEX.W = 1 */ 250#if TCG_TARGET_REG_BITS == 64 251# define P_REXW P_VEXW /* Set REX.W = 1; match VEXW */ 252# define P_REXB_R 0x2000 /* REG field as byte register */ 253# define P_REXB_RM 0x4000 /* R/M field as byte register */ 254# define P_GS 0x8000 /* gs segment override */ 255#else 256# define P_REXW 0 257# define P_REXB_R 0 258# define P_REXB_RM 0 259# define P_GS 0 260#endif 261#define P_EXT3A 0x10000 /* 0x0f 0x3a opcode prefix */ 262#define P_SIMDF3 0x20000 /* 0xf3 opcode prefix */ 263#define P_SIMDF2 0x40000 /* 0xf2 opcode prefix */ 264#define P_VEXL 0x80000 /* Set VEX.L = 1 */ 265#define P_EVEX 0x100000 /* Requires EVEX encoding */ 266 267#define OPC_ARITH_EvIz (0x81) 268#define OPC_ARITH_EvIb (0x83) 269#define OPC_ARITH_GvEv (0x03) /* ... plus (ARITH_FOO << 3) */ 270#define OPC_ANDN (0xf2 | P_EXT38) 271#define OPC_ADD_GvEv (OPC_ARITH_GvEv | (ARITH_ADD << 3)) 272#define OPC_AND_GvEv (OPC_ARITH_GvEv | (ARITH_AND << 3)) 273#define OPC_BLENDPS (0x0c | P_EXT3A | P_DATA16) 274#define OPC_BSF (0xbc | P_EXT) 275#define OPC_BSR (0xbd | P_EXT) 276#define OPC_BSWAP (0xc8 | P_EXT) 277#define OPC_CALL_Jz (0xe8) 278#define OPC_CMOVCC (0x40 | P_EXT) /* ... plus condition code */ 279#define OPC_CMP_GvEv (OPC_ARITH_GvEv | (ARITH_CMP << 3)) 280#define OPC_DEC_r32 (0x48) 281#define OPC_IMUL_GvEv (0xaf | P_EXT) 282#define OPC_IMUL_GvEvIb (0x6b) 283#define OPC_IMUL_GvEvIz (0x69) 284#define OPC_INC_r32 (0x40) 285#define OPC_JCC_long (0x80 | P_EXT) /* ... plus condition code */ 286#define OPC_JCC_short (0x70) /* ... plus condition code */ 287#define OPC_JMP_long (0xe9) 288#define OPC_JMP_short (0xeb) 289#define OPC_LEA (0x8d) 290#define OPC_LZCNT (0xbd | P_EXT | P_SIMDF3) 291#define OPC_MOVB_EvGv (0x88) /* stores, more or less */ 292#define OPC_MOVL_EvGv (0x89) /* stores, more or less */ 293#define OPC_MOVL_GvEv (0x8b) /* loads, more or less */ 294#define OPC_MOVB_EvIz (0xc6) 295#define OPC_MOVL_EvIz (0xc7) 296#define OPC_MOVL_Iv (0xb8) 297#define OPC_MOVBE_GyMy (0xf0 | P_EXT38) 298#define OPC_MOVBE_MyGy (0xf1 | P_EXT38) 299#define OPC_MOVD_VyEy (0x6e | P_EXT | P_DATA16) 300#define OPC_MOVD_EyVy (0x7e | P_EXT | P_DATA16) 301#define OPC_MOVDDUP (0x12 | P_EXT | P_SIMDF2) 302#define OPC_MOVDQA_VxWx (0x6f | P_EXT | P_DATA16) 303#define OPC_MOVDQA_WxVx (0x7f | P_EXT | P_DATA16) 304#define OPC_MOVDQU_VxWx (0x6f | P_EXT | P_SIMDF3) 305#define OPC_MOVDQU_WxVx (0x7f | P_EXT | P_SIMDF3) 306#define OPC_MOVQ_VqWq (0x7e | P_EXT | P_SIMDF3) 307#define OPC_MOVQ_WqVq (0xd6 | P_EXT | P_DATA16) 308#define OPC_MOVSBL (0xbe | P_EXT) 309#define OPC_MOVSWL (0xbf | P_EXT) 310#define OPC_MOVSLQ (0x63 | P_REXW) 311#define OPC_MOVZBL (0xb6 | P_EXT) 312#define OPC_MOVZWL (0xb7 | P_EXT) 313#define OPC_PABSB (0x1c | P_EXT38 | P_DATA16) 314#define OPC_PABSW (0x1d | P_EXT38 | P_DATA16) 315#define OPC_PABSD (0x1e | P_EXT38 | P_DATA16) 316#define OPC_VPABSQ (0x1f | P_EXT38 | P_DATA16 | P_VEXW | P_EVEX) 317#define OPC_PACKSSDW (0x6b | P_EXT | P_DATA16) 318#define OPC_PACKSSWB (0x63 | P_EXT | P_DATA16) 319#define OPC_PACKUSDW (0x2b | P_EXT38 | P_DATA16) 320#define OPC_PACKUSWB (0x67 | P_EXT | P_DATA16) 321#define OPC_PADDB (0xfc | P_EXT | P_DATA16) 322#define OPC_PADDW (0xfd | P_EXT | P_DATA16) 323#define OPC_PADDD (0xfe | P_EXT | P_DATA16) 324#define OPC_PADDQ (0xd4 | P_EXT | P_DATA16) 325#define OPC_PADDSB (0xec | P_EXT | P_DATA16) 326#define OPC_PADDSW (0xed | P_EXT | P_DATA16) 327#define OPC_PADDUB (0xdc | P_EXT | P_DATA16) 328#define OPC_PADDUW (0xdd | P_EXT | P_DATA16) 329#define OPC_PAND (0xdb | P_EXT | P_DATA16) 330#define OPC_PANDN (0xdf | P_EXT | P_DATA16) 331#define OPC_PBLENDW (0x0e | P_EXT3A | P_DATA16) 332#define OPC_PCMPEQB (0x74 | P_EXT | P_DATA16) 333#define OPC_PCMPEQW (0x75 | P_EXT | P_DATA16) 334#define OPC_PCMPEQD (0x76 | P_EXT | P_DATA16) 335#define OPC_PCMPEQQ (0x29 | P_EXT38 | P_DATA16) 336#define OPC_PCMPGTB (0x64 | P_EXT | P_DATA16) 337#define OPC_PCMPGTW (0x65 | P_EXT | P_DATA16) 338#define OPC_PCMPGTD (0x66 | P_EXT | P_DATA16) 339#define OPC_PCMPGTQ (0x37 | P_EXT38 | P_DATA16) 340#define OPC_PMAXSB (0x3c | P_EXT38 | P_DATA16) 341#define OPC_PMAXSW (0xee | P_EXT | P_DATA16) 342#define OPC_PMAXSD (0x3d | P_EXT38 | P_DATA16) 343#define OPC_VPMAXSQ (0x3d | P_EXT38 | P_DATA16 | P_VEXW | P_EVEX) 344#define OPC_PMAXUB (0xde | P_EXT | P_DATA16) 345#define OPC_PMAXUW (0x3e | P_EXT38 | P_DATA16) 346#define OPC_PMAXUD (0x3f | P_EXT38 | P_DATA16) 347#define OPC_VPMAXUQ (0x3f | P_EXT38 | P_DATA16 | P_VEXW | P_EVEX) 348#define OPC_PMINSB (0x38 | P_EXT38 | P_DATA16) 349#define OPC_PMINSW (0xea | P_EXT | P_DATA16) 350#define OPC_PMINSD (0x39 | P_EXT38 | P_DATA16) 351#define OPC_VPMINSQ (0x39 | P_EXT38 | P_DATA16 | P_VEXW | P_EVEX) 352#define OPC_PMINUB (0xda | P_EXT | P_DATA16) 353#define OPC_PMINUW (0x3a | P_EXT38 | P_DATA16) 354#define OPC_PMINUD (0x3b | P_EXT38 | P_DATA16) 355#define OPC_VPMINUQ (0x3b | P_EXT38 | P_DATA16 | P_VEXW | P_EVEX) 356#define OPC_PMOVSXBW (0x20 | P_EXT38 | P_DATA16) 357#define OPC_PMOVSXWD (0x23 | P_EXT38 | P_DATA16) 358#define OPC_PMOVSXDQ (0x25 | P_EXT38 | P_DATA16) 359#define OPC_PMOVZXBW (0x30 | P_EXT38 | P_DATA16) 360#define OPC_PMOVZXWD (0x33 | P_EXT38 | P_DATA16) 361#define OPC_PMOVZXDQ (0x35 | P_EXT38 | P_DATA16) 362#define OPC_PMULLW (0xd5 | P_EXT | P_DATA16) 363#define OPC_PMULLD (0x40 | P_EXT38 | P_DATA16) 364#define OPC_VPMULLQ (0x40 | P_EXT38 | P_DATA16 | P_VEXW | P_EVEX) 365#define OPC_POR (0xeb | P_EXT | P_DATA16) 366#define OPC_PSHUFB (0x00 | P_EXT38 | P_DATA16) 367#define OPC_PSHUFD (0x70 | P_EXT | P_DATA16) 368#define OPC_PSHUFLW (0x70 | P_EXT | P_SIMDF2) 369#define OPC_PSHUFHW (0x70 | P_EXT | P_SIMDF3) 370#define OPC_PSHIFTW_Ib (0x71 | P_EXT | P_DATA16) /* /2 /6 /4 */ 371#define OPC_PSHIFTD_Ib (0x72 | P_EXT | P_DATA16) /* /1 /2 /6 /4 */ 372#define OPC_PSHIFTQ_Ib (0x73 | P_EXT | P_DATA16) /* /2 /6 /4 */ 373#define OPC_PSLLW (0xf1 | P_EXT | P_DATA16) 374#define OPC_PSLLD (0xf2 | P_EXT | P_DATA16) 375#define OPC_PSLLQ (0xf3 | P_EXT | P_DATA16) 376#define OPC_PSRAW (0xe1 | P_EXT | P_DATA16) 377#define OPC_PSRAD (0xe2 | P_EXT | P_DATA16) 378#define OPC_VPSRAQ (0xe2 | P_EXT | P_DATA16 | P_VEXW | P_EVEX) 379#define OPC_PSRLW (0xd1 | P_EXT | P_DATA16) 380#define OPC_PSRLD (0xd2 | P_EXT | P_DATA16) 381#define OPC_PSRLQ (0xd3 | P_EXT | P_DATA16) 382#define OPC_PSUBB (0xf8 | P_EXT | P_DATA16) 383#define OPC_PSUBW (0xf9 | P_EXT | P_DATA16) 384#define OPC_PSUBD (0xfa | P_EXT | P_DATA16) 385#define OPC_PSUBQ (0xfb | P_EXT | P_DATA16) 386#define OPC_PSUBSB (0xe8 | P_EXT | P_DATA16) 387#define OPC_PSUBSW (0xe9 | P_EXT | P_DATA16) 388#define OPC_PSUBUB (0xd8 | P_EXT | P_DATA16) 389#define OPC_PSUBUW (0xd9 | P_EXT | P_DATA16) 390#define OPC_PUNPCKLBW (0x60 | P_EXT | P_DATA16) 391#define OPC_PUNPCKLWD (0x61 | P_EXT | P_DATA16) 392#define OPC_PUNPCKLDQ (0x62 | P_EXT | P_DATA16) 393#define OPC_PUNPCKLQDQ (0x6c | P_EXT | P_DATA16) 394#define OPC_PUNPCKHBW (0x68 | P_EXT | P_DATA16) 395#define OPC_PUNPCKHWD (0x69 | P_EXT | P_DATA16) 396#define OPC_PUNPCKHDQ (0x6a | P_EXT | P_DATA16) 397#define OPC_PUNPCKHQDQ (0x6d | P_EXT | P_DATA16) 398#define OPC_PXOR (0xef | P_EXT | P_DATA16) 399#define OPC_POP_r32 (0x58) 400#define OPC_POPCNT (0xb8 | P_EXT | P_SIMDF3) 401#define OPC_PUSH_r32 (0x50) 402#define OPC_PUSH_Iv (0x68) 403#define OPC_PUSH_Ib (0x6a) 404#define OPC_RET (0xc3) 405#define OPC_SETCC (0x90 | P_EXT | P_REXB_RM) /* ... plus cc */ 406#define OPC_SHIFT_1 (0xd1) 407#define OPC_SHIFT_Ib (0xc1) 408#define OPC_SHIFT_cl (0xd3) 409#define OPC_SARX (0xf7 | P_EXT38 | P_SIMDF3) 410#define OPC_SHUFPS (0xc6 | P_EXT) 411#define OPC_SHLX (0xf7 | P_EXT38 | P_DATA16) 412#define OPC_SHRX (0xf7 | P_EXT38 | P_SIMDF2) 413#define OPC_SHRD_Ib (0xac | P_EXT) 414#define OPC_TESTL (0x85) 415#define OPC_TZCNT (0xbc | P_EXT | P_SIMDF3) 416#define OPC_UD2 (0x0b | P_EXT) 417#define OPC_VPBLENDD (0x02 | P_EXT3A | P_DATA16) 418#define OPC_VPBLENDVB (0x4c | P_EXT3A | P_DATA16) 419#define OPC_VPINSRB (0x20 | P_EXT3A | P_DATA16) 420#define OPC_VPINSRW (0xc4 | P_EXT | P_DATA16) 421#define OPC_VBROADCASTSS (0x18 | P_EXT38 | P_DATA16) 422#define OPC_VBROADCASTSD (0x19 | P_EXT38 | P_DATA16) 423#define OPC_VPBROADCASTB (0x78 | P_EXT38 | P_DATA16) 424#define OPC_VPBROADCASTW (0x79 | P_EXT38 | P_DATA16) 425#define OPC_VPBROADCASTD (0x58 | P_EXT38 | P_DATA16) 426#define OPC_VPBROADCASTQ (0x59 | P_EXT38 | P_DATA16) 427#define OPC_VPERMQ (0x00 | P_EXT3A | P_DATA16 | P_VEXW) 428#define OPC_VPERM2I128 (0x46 | P_EXT3A | P_DATA16 | P_VEXL) 429#define OPC_VPROLVD (0x15 | P_EXT38 | P_DATA16 | P_EVEX) 430#define OPC_VPROLVQ (0x15 | P_EXT38 | P_DATA16 | P_VEXW | P_EVEX) 431#define OPC_VPRORVD (0x14 | P_EXT38 | P_DATA16 | P_EVEX) 432#define OPC_VPRORVQ (0x14 | P_EXT38 | P_DATA16 | P_VEXW | P_EVEX) 433#define OPC_VPSHLDW (0x70 | P_EXT3A | P_DATA16 | P_VEXW | P_EVEX) 434#define OPC_VPSHLDD (0x71 | P_EXT3A | P_DATA16 | P_EVEX) 435#define OPC_VPSHLDQ (0x71 | P_EXT3A | P_DATA16 | P_VEXW | P_EVEX) 436#define OPC_VPSHLDVW (0x70 | P_EXT38 | P_DATA16 | P_VEXW | P_EVEX) 437#define OPC_VPSHLDVD (0x71 | P_EXT38 | P_DATA16 | P_EVEX) 438#define OPC_VPSHLDVQ (0x71 | P_EXT38 | P_DATA16 | P_VEXW | P_EVEX) 439#define OPC_VPSHRDVW (0x72 | P_EXT38 | P_DATA16 | P_VEXW | P_EVEX) 440#define OPC_VPSHRDVD (0x73 | P_EXT38 | P_DATA16 | P_EVEX) 441#define OPC_VPSHRDVQ (0x73 | P_EXT38 | P_DATA16 | P_VEXW | P_EVEX) 442#define OPC_VPSLLVW (0x12 | P_EXT38 | P_DATA16 | P_VEXW | P_EVEX) 443#define OPC_VPSLLVD (0x47 | P_EXT38 | P_DATA16) 444#define OPC_VPSLLVQ (0x47 | P_EXT38 | P_DATA16 | P_VEXW) 445#define OPC_VPSRAVW (0x11 | P_EXT38 | P_DATA16 | P_VEXW | P_EVEX) 446#define OPC_VPSRAVD (0x46 | P_EXT38 | P_DATA16) 447#define OPC_VPSRAVQ (0x46 | P_EXT38 | P_DATA16 | P_VEXW | P_EVEX) 448#define OPC_VPSRLVW (0x10 | P_EXT38 | P_DATA16 | P_VEXW | P_EVEX) 449#define OPC_VPSRLVD (0x45 | P_EXT38 | P_DATA16) 450#define OPC_VPSRLVQ (0x45 | P_EXT38 | P_DATA16 | P_VEXW) 451#define OPC_VPTERNLOGQ (0x25 | P_EXT3A | P_DATA16 | P_VEXW | P_EVEX) 452#define OPC_VZEROUPPER (0x77 | P_EXT) 453#define OPC_XCHG_ax_r32 (0x90) 454 455#define OPC_GRP3_Eb (0xf6) 456#define OPC_GRP3_Ev (0xf7) 457#define OPC_GRP5 (0xff) 458#define OPC_GRP14 (0x73 | P_EXT | P_DATA16) 459 460/* Group 1 opcode extensions for 0x80-0x83. 461 These are also used as modifiers for OPC_ARITH. */ 462#define ARITH_ADD 0 463#define ARITH_OR 1 464#define ARITH_ADC 2 465#define ARITH_SBB 3 466#define ARITH_AND 4 467#define ARITH_SUB 5 468#define ARITH_XOR 6 469#define ARITH_CMP 7 470 471/* Group 2 opcode extensions for 0xc0, 0xc1, 0xd0-0xd3. */ 472#define SHIFT_ROL 0 473#define SHIFT_ROR 1 474#define SHIFT_SHL 4 475#define SHIFT_SHR 5 476#define SHIFT_SAR 7 477 478/* Group 3 opcode extensions for 0xf6, 0xf7. To be used with OPC_GRP3. */ 479#define EXT3_TESTi 0 480#define EXT3_NOT 2 481#define EXT3_NEG 3 482#define EXT3_MUL 4 483#define EXT3_IMUL 5 484#define EXT3_DIV 6 485#define EXT3_IDIV 7 486 487/* Group 5 opcode extensions for 0xff. To be used with OPC_GRP5. */ 488#define EXT5_INC_Ev 0 489#define EXT5_DEC_Ev 1 490#define EXT5_CALLN_Ev 2 491#define EXT5_JMPN_Ev 4 492 493/* Condition codes to be added to OPC_JCC_{long,short}. */ 494#define JCC_JMP (-1) 495#define JCC_JO 0x0 496#define JCC_JNO 0x1 497#define JCC_JB 0x2 498#define JCC_JAE 0x3 499#define JCC_JE 0x4 500#define JCC_JNE 0x5 501#define JCC_JBE 0x6 502#define JCC_JA 0x7 503#define JCC_JS 0x8 504#define JCC_JNS 0x9 505#define JCC_JP 0xa 506#define JCC_JNP 0xb 507#define JCC_JL 0xc 508#define JCC_JGE 0xd 509#define JCC_JLE 0xe 510#define JCC_JG 0xf 511 512static const uint8_t tcg_cond_to_jcc[] = { 513 [TCG_COND_EQ] = JCC_JE, 514 [TCG_COND_NE] = JCC_JNE, 515 [TCG_COND_LT] = JCC_JL, 516 [TCG_COND_GE] = JCC_JGE, 517 [TCG_COND_LE] = JCC_JLE, 518 [TCG_COND_GT] = JCC_JG, 519 [TCG_COND_LTU] = JCC_JB, 520 [TCG_COND_GEU] = JCC_JAE, 521 [TCG_COND_LEU] = JCC_JBE, 522 [TCG_COND_GTU] = JCC_JA, 523}; 524 525#if TCG_TARGET_REG_BITS == 64 526static void tcg_out_opc(TCGContext *s, int opc, int r, int rm, int x) 527{ 528 int rex; 529 530 if (opc & P_GS) { 531 tcg_out8(s, 0x65); 532 } 533 if (opc & P_DATA16) { 534 /* We should never be asking for both 16 and 64-bit operation. */ 535 tcg_debug_assert((opc & P_REXW) == 0); 536 tcg_out8(s, 0x66); 537 } 538 if (opc & P_SIMDF3) { 539 tcg_out8(s, 0xf3); 540 } else if (opc & P_SIMDF2) { 541 tcg_out8(s, 0xf2); 542 } 543 544 rex = 0; 545 rex |= (opc & P_REXW) ? 0x8 : 0x0; /* REX.W */ 546 rex |= (r & 8) >> 1; /* REX.R */ 547 rex |= (x & 8) >> 2; /* REX.X */ 548 rex |= (rm & 8) >> 3; /* REX.B */ 549 550 /* P_REXB_{R,RM} indicates that the given register is the low byte. 551 For %[abcd]l we need no REX prefix, but for %{si,di,bp,sp}l we do, 552 as otherwise the encoding indicates %[abcd]h. Note that the values 553 that are ORed in merely indicate that the REX byte must be present; 554 those bits get discarded in output. */ 555 rex |= opc & (r >= 4 ? P_REXB_R : 0); 556 rex |= opc & (rm >= 4 ? P_REXB_RM : 0); 557 558 if (rex) { 559 tcg_out8(s, (uint8_t)(rex | 0x40)); 560 } 561 562 if (opc & (P_EXT | P_EXT38 | P_EXT3A)) { 563 tcg_out8(s, 0x0f); 564 if (opc & P_EXT38) { 565 tcg_out8(s, 0x38); 566 } else if (opc & P_EXT3A) { 567 tcg_out8(s, 0x3a); 568 } 569 } 570 571 tcg_out8(s, opc); 572} 573#else 574static void tcg_out_opc(TCGContext *s, int opc) 575{ 576 if (opc & P_DATA16) { 577 tcg_out8(s, 0x66); 578 } 579 if (opc & P_SIMDF3) { 580 tcg_out8(s, 0xf3); 581 } else if (opc & P_SIMDF2) { 582 tcg_out8(s, 0xf2); 583 } 584 if (opc & (P_EXT | P_EXT38 | P_EXT3A)) { 585 tcg_out8(s, 0x0f); 586 if (opc & P_EXT38) { 587 tcg_out8(s, 0x38); 588 } else if (opc & P_EXT3A) { 589 tcg_out8(s, 0x3a); 590 } 591 } 592 tcg_out8(s, opc); 593} 594/* Discard the register arguments to tcg_out_opc early, so as not to penalize 595 the 32-bit compilation paths. This method works with all versions of gcc, 596 whereas relying on optimization may not be able to exclude them. */ 597#define tcg_out_opc(s, opc, r, rm, x) (tcg_out_opc)(s, opc) 598#endif 599 600static void tcg_out_modrm(TCGContext *s, int opc, int r, int rm) 601{ 602 tcg_out_opc(s, opc, r, rm, 0); 603 tcg_out8(s, 0xc0 | (LOWREGMASK(r) << 3) | LOWREGMASK(rm)); 604} 605 606static void tcg_out_vex_opc(TCGContext *s, int opc, int r, int v, 607 int rm, int index) 608{ 609 int tmp; 610 611 /* Use the two byte form if possible, which cannot encode 612 VEX.W, VEX.B, VEX.X, or an m-mmmm field other than P_EXT. */ 613 if ((opc & (P_EXT | P_EXT38 | P_EXT3A | P_VEXW)) == P_EXT 614 && ((rm | index) & 8) == 0) { 615 /* Two byte VEX prefix. */ 616 tcg_out8(s, 0xc5); 617 618 tmp = (r & 8 ? 0 : 0x80); /* VEX.R */ 619 } else { 620 /* Three byte VEX prefix. */ 621 tcg_out8(s, 0xc4); 622 623 /* VEX.m-mmmm */ 624 if (opc & P_EXT3A) { 625 tmp = 3; 626 } else if (opc & P_EXT38) { 627 tmp = 2; 628 } else if (opc & P_EXT) { 629 tmp = 1; 630 } else { 631 g_assert_not_reached(); 632 } 633 tmp |= (r & 8 ? 0 : 0x80); /* VEX.R */ 634 tmp |= (index & 8 ? 0 : 0x40); /* VEX.X */ 635 tmp |= (rm & 8 ? 0 : 0x20); /* VEX.B */ 636 tcg_out8(s, tmp); 637 638 tmp = (opc & P_VEXW ? 0x80 : 0); /* VEX.W */ 639 } 640 641 tmp |= (opc & P_VEXL ? 0x04 : 0); /* VEX.L */ 642 /* VEX.pp */ 643 if (opc & P_DATA16) { 644 tmp |= 1; /* 0x66 */ 645 } else if (opc & P_SIMDF3) { 646 tmp |= 2; /* 0xf3 */ 647 } else if (opc & P_SIMDF2) { 648 tmp |= 3; /* 0xf2 */ 649 } 650 tmp |= (~v & 15) << 3; /* VEX.vvvv */ 651 tcg_out8(s, tmp); 652 tcg_out8(s, opc); 653} 654 655static void tcg_out_evex_opc(TCGContext *s, int opc, int r, int v, 656 int rm, int index) 657{ 658 /* The entire 4-byte evex prefix; with R' and V' set. */ 659 uint32_t p = 0x08041062; 660 int mm, pp; 661 662 tcg_debug_assert(have_avx512vl); 663 664 /* EVEX.mm */ 665 if (opc & P_EXT3A) { 666 mm = 3; 667 } else if (opc & P_EXT38) { 668 mm = 2; 669 } else if (opc & P_EXT) { 670 mm = 1; 671 } else { 672 g_assert_not_reached(); 673 } 674 675 /* EVEX.pp */ 676 if (opc & P_DATA16) { 677 pp = 1; /* 0x66 */ 678 } else if (opc & P_SIMDF3) { 679 pp = 2; /* 0xf3 */ 680 } else if (opc & P_SIMDF2) { 681 pp = 3; /* 0xf2 */ 682 } else { 683 pp = 0; 684 } 685 686 p = deposit32(p, 8, 2, mm); 687 p = deposit32(p, 13, 1, (rm & 8) == 0); /* EVEX.RXB.B */ 688 p = deposit32(p, 14, 1, (index & 8) == 0); /* EVEX.RXB.X */ 689 p = deposit32(p, 15, 1, (r & 8) == 0); /* EVEX.RXB.R */ 690 p = deposit32(p, 16, 2, pp); 691 p = deposit32(p, 19, 4, ~v); 692 p = deposit32(p, 23, 1, (opc & P_VEXW) != 0); 693 p = deposit32(p, 29, 2, (opc & P_VEXL) != 0); 694 695 tcg_out32(s, p); 696 tcg_out8(s, opc); 697} 698 699static void tcg_out_vex_modrm(TCGContext *s, int opc, int r, int v, int rm) 700{ 701 if (opc & P_EVEX) { 702 tcg_out_evex_opc(s, opc, r, v, rm, 0); 703 } else { 704 tcg_out_vex_opc(s, opc, r, v, rm, 0); 705 } 706 tcg_out8(s, 0xc0 | (LOWREGMASK(r) << 3) | LOWREGMASK(rm)); 707} 708 709/* Output an opcode with a full "rm + (index<<shift) + offset" address mode. 710 We handle either RM and INDEX missing with a negative value. In 64-bit 711 mode for absolute addresses, ~RM is the size of the immediate operand 712 that will follow the instruction. */ 713 714static void tcg_out_sib_offset(TCGContext *s, int r, int rm, int index, 715 int shift, intptr_t offset) 716{ 717 int mod, len; 718 719 if (index < 0 && rm < 0) { 720 if (TCG_TARGET_REG_BITS == 64) { 721 /* Try for a rip-relative addressing mode. This has replaced 722 the 32-bit-mode absolute addressing encoding. */ 723 intptr_t pc = (intptr_t)s->code_ptr + 5 + ~rm; 724 intptr_t disp = offset - pc; 725 if (disp == (int32_t)disp) { 726 tcg_out8(s, (LOWREGMASK(r) << 3) | 5); 727 tcg_out32(s, disp); 728 return; 729 } 730 731 /* Try for an absolute address encoding. This requires the 732 use of the MODRM+SIB encoding and is therefore larger than 733 rip-relative addressing. */ 734 if (offset == (int32_t)offset) { 735 tcg_out8(s, (LOWREGMASK(r) << 3) | 4); 736 tcg_out8(s, (4 << 3) | 5); 737 tcg_out32(s, offset); 738 return; 739 } 740 741 /* ??? The memory isn't directly addressable. */ 742 g_assert_not_reached(); 743 } else { 744 /* Absolute address. */ 745 tcg_out8(s, (r << 3) | 5); 746 tcg_out32(s, offset); 747 return; 748 } 749 } 750 751 /* Find the length of the immediate addend. Note that the encoding 752 that would be used for (%ebp) indicates absolute addressing. */ 753 if (rm < 0) { 754 mod = 0, len = 4, rm = 5; 755 } else if (offset == 0 && LOWREGMASK(rm) != TCG_REG_EBP) { 756 mod = 0, len = 0; 757 } else if (offset == (int8_t)offset) { 758 mod = 0x40, len = 1; 759 } else { 760 mod = 0x80, len = 4; 761 } 762 763 /* Use a single byte MODRM format if possible. Note that the encoding 764 that would be used for %esp is the escape to the two byte form. */ 765 if (index < 0 && LOWREGMASK(rm) != TCG_REG_ESP) { 766 /* Single byte MODRM format. */ 767 tcg_out8(s, mod | (LOWREGMASK(r) << 3) | LOWREGMASK(rm)); 768 } else { 769 /* Two byte MODRM+SIB format. */ 770 771 /* Note that the encoding that would place %esp into the index 772 field indicates no index register. In 64-bit mode, the REX.X 773 bit counts, so %r12 can be used as the index. */ 774 if (index < 0) { 775 index = 4; 776 } else { 777 tcg_debug_assert(index != TCG_REG_ESP); 778 } 779 780 tcg_out8(s, mod | (LOWREGMASK(r) << 3) | 4); 781 tcg_out8(s, (shift << 6) | (LOWREGMASK(index) << 3) | LOWREGMASK(rm)); 782 } 783 784 if (len == 1) { 785 tcg_out8(s, offset); 786 } else if (len == 4) { 787 tcg_out32(s, offset); 788 } 789} 790 791static void tcg_out_modrm_sib_offset(TCGContext *s, int opc, int r, int rm, 792 int index, int shift, intptr_t offset) 793{ 794 tcg_out_opc(s, opc, r, rm < 0 ? 0 : rm, index < 0 ? 0 : index); 795 tcg_out_sib_offset(s, r, rm, index, shift, offset); 796} 797 798static void tcg_out_vex_modrm_sib_offset(TCGContext *s, int opc, int r, int v, 799 int rm, int index, int shift, 800 intptr_t offset) 801{ 802 tcg_out_vex_opc(s, opc, r, v, rm < 0 ? 0 : rm, index < 0 ? 0 : index); 803 tcg_out_sib_offset(s, r, rm, index, shift, offset); 804} 805 806/* A simplification of the above with no index or shift. */ 807static inline void tcg_out_modrm_offset(TCGContext *s, int opc, int r, 808 int rm, intptr_t offset) 809{ 810 tcg_out_modrm_sib_offset(s, opc, r, rm, -1, 0, offset); 811} 812 813static inline void tcg_out_vex_modrm_offset(TCGContext *s, int opc, int r, 814 int v, int rm, intptr_t offset) 815{ 816 tcg_out_vex_modrm_sib_offset(s, opc, r, v, rm, -1, 0, offset); 817} 818 819/* Output an opcode with an expected reference to the constant pool. */ 820static inline void tcg_out_modrm_pool(TCGContext *s, int opc, int r) 821{ 822 tcg_out_opc(s, opc, r, 0, 0); 823 /* Absolute for 32-bit, pc-relative for 64-bit. */ 824 tcg_out8(s, LOWREGMASK(r) << 3 | 5); 825 tcg_out32(s, 0); 826} 827 828/* Output an opcode with an expected reference to the constant pool. */ 829static inline void tcg_out_vex_modrm_pool(TCGContext *s, int opc, int r) 830{ 831 tcg_out_vex_opc(s, opc, r, 0, 0, 0); 832 /* Absolute for 32-bit, pc-relative for 64-bit. */ 833 tcg_out8(s, LOWREGMASK(r) << 3 | 5); 834 tcg_out32(s, 0); 835} 836 837/* Generate dest op= src. Uses the same ARITH_* codes as tgen_arithi. */ 838static inline void tgen_arithr(TCGContext *s, int subop, int dest, int src) 839{ 840 /* Propagate an opcode prefix, such as P_REXW. */ 841 int ext = subop & ~0x7; 842 subop &= 0x7; 843 844 tcg_out_modrm(s, OPC_ARITH_GvEv + (subop << 3) + ext, dest, src); 845} 846 847static bool tcg_out_mov(TCGContext *s, TCGType type, TCGReg ret, TCGReg arg) 848{ 849 int rexw = 0; 850 851 if (arg == ret) { 852 return true; 853 } 854 switch (type) { 855 case TCG_TYPE_I64: 856 rexw = P_REXW; 857 /* fallthru */ 858 case TCG_TYPE_I32: 859 if (ret < 16) { 860 if (arg < 16) { 861 tcg_out_modrm(s, OPC_MOVL_GvEv + rexw, ret, arg); 862 } else { 863 tcg_out_vex_modrm(s, OPC_MOVD_EyVy + rexw, arg, 0, ret); 864 } 865 } else { 866 if (arg < 16) { 867 tcg_out_vex_modrm(s, OPC_MOVD_VyEy + rexw, ret, 0, arg); 868 } else { 869 tcg_out_vex_modrm(s, OPC_MOVQ_VqWq, ret, 0, arg); 870 } 871 } 872 break; 873 874 case TCG_TYPE_V64: 875 tcg_debug_assert(ret >= 16 && arg >= 16); 876 tcg_out_vex_modrm(s, OPC_MOVQ_VqWq, ret, 0, arg); 877 break; 878 case TCG_TYPE_V128: 879 tcg_debug_assert(ret >= 16 && arg >= 16); 880 tcg_out_vex_modrm(s, OPC_MOVDQA_VxWx, ret, 0, arg); 881 break; 882 case TCG_TYPE_V256: 883 tcg_debug_assert(ret >= 16 && arg >= 16); 884 tcg_out_vex_modrm(s, OPC_MOVDQA_VxWx | P_VEXL, ret, 0, arg); 885 break; 886 887 default: 888 g_assert_not_reached(); 889 } 890 return true; 891} 892 893static const int avx2_dup_insn[4] = { 894 OPC_VPBROADCASTB, OPC_VPBROADCASTW, 895 OPC_VPBROADCASTD, OPC_VPBROADCASTQ, 896}; 897 898static bool tcg_out_dup_vec(TCGContext *s, TCGType type, unsigned vece, 899 TCGReg r, TCGReg a) 900{ 901 if (have_avx2) { 902 int vex_l = (type == TCG_TYPE_V256 ? P_VEXL : 0); 903 tcg_out_vex_modrm(s, avx2_dup_insn[vece] + vex_l, r, 0, a); 904 } else { 905 switch (vece) { 906 case MO_8: 907 /* ??? With zero in a register, use PSHUFB. */ 908 tcg_out_vex_modrm(s, OPC_PUNPCKLBW, r, a, a); 909 a = r; 910 /* FALLTHRU */ 911 case MO_16: 912 tcg_out_vex_modrm(s, OPC_PUNPCKLWD, r, a, a); 913 a = r; 914 /* FALLTHRU */ 915 case MO_32: 916 tcg_out_vex_modrm(s, OPC_PSHUFD, r, 0, a); 917 /* imm8 operand: all output lanes selected from input lane 0. */ 918 tcg_out8(s, 0); 919 break; 920 case MO_64: 921 tcg_out_vex_modrm(s, OPC_PUNPCKLQDQ, r, a, a); 922 break; 923 default: 924 g_assert_not_reached(); 925 } 926 } 927 return true; 928} 929 930static bool tcg_out_dupm_vec(TCGContext *s, TCGType type, unsigned vece, 931 TCGReg r, TCGReg base, intptr_t offset) 932{ 933 if (have_avx2) { 934 int vex_l = (type == TCG_TYPE_V256 ? P_VEXL : 0); 935 tcg_out_vex_modrm_offset(s, avx2_dup_insn[vece] + vex_l, 936 r, 0, base, offset); 937 } else { 938 switch (vece) { 939 case MO_64: 940 tcg_out_vex_modrm_offset(s, OPC_MOVDDUP, r, 0, base, offset); 941 break; 942 case MO_32: 943 tcg_out_vex_modrm_offset(s, OPC_VBROADCASTSS, r, 0, base, offset); 944 break; 945 case MO_16: 946 tcg_out_vex_modrm_offset(s, OPC_VPINSRW, r, r, base, offset); 947 tcg_out8(s, 0); /* imm8 */ 948 tcg_out_dup_vec(s, type, vece, r, r); 949 break; 950 case MO_8: 951 tcg_out_vex_modrm_offset(s, OPC_VPINSRB, r, r, base, offset); 952 tcg_out8(s, 0); /* imm8 */ 953 tcg_out_dup_vec(s, type, vece, r, r); 954 break; 955 default: 956 g_assert_not_reached(); 957 } 958 } 959 return true; 960} 961 962static void tcg_out_dupi_vec(TCGContext *s, TCGType type, unsigned vece, 963 TCGReg ret, int64_t arg) 964{ 965 int vex_l = (type == TCG_TYPE_V256 ? P_VEXL : 0); 966 967 if (arg == 0) { 968 tcg_out_vex_modrm(s, OPC_PXOR, ret, ret, ret); 969 return; 970 } 971 if (arg == -1) { 972 tcg_out_vex_modrm(s, OPC_PCMPEQB + vex_l, ret, ret, ret); 973 return; 974 } 975 976 if (TCG_TARGET_REG_BITS == 32 && vece < MO_64) { 977 if (have_avx2) { 978 tcg_out_vex_modrm_pool(s, OPC_VPBROADCASTD + vex_l, ret); 979 } else { 980 tcg_out_vex_modrm_pool(s, OPC_VBROADCASTSS, ret); 981 } 982 new_pool_label(s, arg, R_386_32, s->code_ptr - 4, 0); 983 } else { 984 if (type == TCG_TYPE_V64) { 985 tcg_out_vex_modrm_pool(s, OPC_MOVQ_VqWq, ret); 986 } else if (have_avx2) { 987 tcg_out_vex_modrm_pool(s, OPC_VPBROADCASTQ + vex_l, ret); 988 } else { 989 tcg_out_vex_modrm_pool(s, OPC_MOVDDUP, ret); 990 } 991 if (TCG_TARGET_REG_BITS == 64) { 992 new_pool_label(s, arg, R_386_PC32, s->code_ptr - 4, -4); 993 } else { 994 new_pool_l2(s, R_386_32, s->code_ptr - 4, 0, arg, arg >> 32); 995 } 996 } 997} 998 999static void tcg_out_movi_vec(TCGContext *s, TCGType type, 1000 TCGReg ret, tcg_target_long arg) 1001{ 1002 if (arg == 0) { 1003 tcg_out_vex_modrm(s, OPC_PXOR, ret, ret, ret); 1004 return; 1005 } 1006 if (arg == -1) { 1007 tcg_out_vex_modrm(s, OPC_PCMPEQB, ret, ret, ret); 1008 return; 1009 } 1010 1011 int rexw = (type == TCG_TYPE_I32 ? 0 : P_REXW); 1012 tcg_out_vex_modrm_pool(s, OPC_MOVD_VyEy + rexw, ret); 1013 if (TCG_TARGET_REG_BITS == 64) { 1014 new_pool_label(s, arg, R_386_PC32, s->code_ptr - 4, -4); 1015 } else { 1016 new_pool_label(s, arg, R_386_32, s->code_ptr - 4, 0); 1017 } 1018} 1019 1020static void tcg_out_movi_int(TCGContext *s, TCGType type, 1021 TCGReg ret, tcg_target_long arg) 1022{ 1023 tcg_target_long diff; 1024 1025 if (arg == 0) { 1026 tgen_arithr(s, ARITH_XOR, ret, ret); 1027 return; 1028 } 1029 if (arg == (uint32_t)arg || type == TCG_TYPE_I32) { 1030 tcg_out_opc(s, OPC_MOVL_Iv + LOWREGMASK(ret), 0, ret, 0); 1031 tcg_out32(s, arg); 1032 return; 1033 } 1034 if (arg == (int32_t)arg) { 1035 tcg_out_modrm(s, OPC_MOVL_EvIz + P_REXW, 0, ret); 1036 tcg_out32(s, arg); 1037 return; 1038 } 1039 1040 /* Try a 7 byte pc-relative lea before the 10 byte movq. */ 1041 diff = tcg_pcrel_diff(s, (const void *)arg) - 7; 1042 if (diff == (int32_t)diff) { 1043 tcg_out_opc(s, OPC_LEA | P_REXW, ret, 0, 0); 1044 tcg_out8(s, (LOWREGMASK(ret) << 3) | 5); 1045 tcg_out32(s, diff); 1046 return; 1047 } 1048 1049 tcg_out_opc(s, OPC_MOVL_Iv + P_REXW + LOWREGMASK(ret), 0, ret, 0); 1050 tcg_out64(s, arg); 1051} 1052 1053static void tcg_out_movi(TCGContext *s, TCGType type, 1054 TCGReg ret, tcg_target_long arg) 1055{ 1056 switch (type) { 1057 case TCG_TYPE_I32: 1058#if TCG_TARGET_REG_BITS == 64 1059 case TCG_TYPE_I64: 1060#endif 1061 if (ret < 16) { 1062 tcg_out_movi_int(s, type, ret, arg); 1063 } else { 1064 tcg_out_movi_vec(s, type, ret, arg); 1065 } 1066 break; 1067 default: 1068 g_assert_not_reached(); 1069 } 1070} 1071 1072static inline void tcg_out_pushi(TCGContext *s, tcg_target_long val) 1073{ 1074 if (val == (int8_t)val) { 1075 tcg_out_opc(s, OPC_PUSH_Ib, 0, 0, 0); 1076 tcg_out8(s, val); 1077 } else if (val == (int32_t)val) { 1078 tcg_out_opc(s, OPC_PUSH_Iv, 0, 0, 0); 1079 tcg_out32(s, val); 1080 } else { 1081 tcg_abort(); 1082 } 1083} 1084 1085static inline void tcg_out_mb(TCGContext *s, TCGArg a0) 1086{ 1087 /* Given the strength of x86 memory ordering, we only need care for 1088 store-load ordering. Experimentally, "lock orl $0,0(%esp)" is 1089 faster than "mfence", so don't bother with the sse insn. */ 1090 if (a0 & TCG_MO_ST_LD) { 1091 tcg_out8(s, 0xf0); 1092 tcg_out_modrm_offset(s, OPC_ARITH_EvIb, ARITH_OR, TCG_REG_ESP, 0); 1093 tcg_out8(s, 0); 1094 } 1095} 1096 1097static inline void tcg_out_push(TCGContext *s, int reg) 1098{ 1099 tcg_out_opc(s, OPC_PUSH_r32 + LOWREGMASK(reg), 0, reg, 0); 1100} 1101 1102static inline void tcg_out_pop(TCGContext *s, int reg) 1103{ 1104 tcg_out_opc(s, OPC_POP_r32 + LOWREGMASK(reg), 0, reg, 0); 1105} 1106 1107static void tcg_out_ld(TCGContext *s, TCGType type, TCGReg ret, 1108 TCGReg arg1, intptr_t arg2) 1109{ 1110 switch (type) { 1111 case TCG_TYPE_I32: 1112 if (ret < 16) { 1113 tcg_out_modrm_offset(s, OPC_MOVL_GvEv, ret, arg1, arg2); 1114 } else { 1115 tcg_out_vex_modrm_offset(s, OPC_MOVD_VyEy, ret, 0, arg1, arg2); 1116 } 1117 break; 1118 case TCG_TYPE_I64: 1119 if (ret < 16) { 1120 tcg_out_modrm_offset(s, OPC_MOVL_GvEv | P_REXW, ret, arg1, arg2); 1121 break; 1122 } 1123 /* FALLTHRU */ 1124 case TCG_TYPE_V64: 1125 /* There is no instruction that can validate 8-byte alignment. */ 1126 tcg_debug_assert(ret >= 16); 1127 tcg_out_vex_modrm_offset(s, OPC_MOVQ_VqWq, ret, 0, arg1, arg2); 1128 break; 1129 case TCG_TYPE_V128: 1130 /* 1131 * The gvec infrastructure is asserts that v128 vector loads 1132 * and stores use a 16-byte aligned offset. Validate that the 1133 * final pointer is aligned by using an insn that will SIGSEGV. 1134 */ 1135 tcg_debug_assert(ret >= 16); 1136 tcg_out_vex_modrm_offset(s, OPC_MOVDQA_VxWx, ret, 0, arg1, arg2); 1137 break; 1138 case TCG_TYPE_V256: 1139 /* 1140 * The gvec infrastructure only requires 16-byte alignment, 1141 * so here we must use an unaligned load. 1142 */ 1143 tcg_debug_assert(ret >= 16); 1144 tcg_out_vex_modrm_offset(s, OPC_MOVDQU_VxWx | P_VEXL, 1145 ret, 0, arg1, arg2); 1146 break; 1147 default: 1148 g_assert_not_reached(); 1149 } 1150} 1151 1152static void tcg_out_st(TCGContext *s, TCGType type, TCGReg arg, 1153 TCGReg arg1, intptr_t arg2) 1154{ 1155 switch (type) { 1156 case TCG_TYPE_I32: 1157 if (arg < 16) { 1158 tcg_out_modrm_offset(s, OPC_MOVL_EvGv, arg, arg1, arg2); 1159 } else { 1160 tcg_out_vex_modrm_offset(s, OPC_MOVD_EyVy, arg, 0, arg1, arg2); 1161 } 1162 break; 1163 case TCG_TYPE_I64: 1164 if (arg < 16) { 1165 tcg_out_modrm_offset(s, OPC_MOVL_EvGv | P_REXW, arg, arg1, arg2); 1166 break; 1167 } 1168 /* FALLTHRU */ 1169 case TCG_TYPE_V64: 1170 /* There is no instruction that can validate 8-byte alignment. */ 1171 tcg_debug_assert(arg >= 16); 1172 tcg_out_vex_modrm_offset(s, OPC_MOVQ_WqVq, arg, 0, arg1, arg2); 1173 break; 1174 case TCG_TYPE_V128: 1175 /* 1176 * The gvec infrastructure is asserts that v128 vector loads 1177 * and stores use a 16-byte aligned offset. Validate that the 1178 * final pointer is aligned by using an insn that will SIGSEGV. 1179 */ 1180 tcg_debug_assert(arg >= 16); 1181 tcg_out_vex_modrm_offset(s, OPC_MOVDQA_WxVx, arg, 0, arg1, arg2); 1182 break; 1183 case TCG_TYPE_V256: 1184 /* 1185 * The gvec infrastructure only requires 16-byte alignment, 1186 * so here we must use an unaligned store. 1187 */ 1188 tcg_debug_assert(arg >= 16); 1189 tcg_out_vex_modrm_offset(s, OPC_MOVDQU_WxVx | P_VEXL, 1190 arg, 0, arg1, arg2); 1191 break; 1192 default: 1193 g_assert_not_reached(); 1194 } 1195} 1196 1197static bool tcg_out_sti(TCGContext *s, TCGType type, TCGArg val, 1198 TCGReg base, intptr_t ofs) 1199{ 1200 int rexw = 0; 1201 if (TCG_TARGET_REG_BITS == 64 && type == TCG_TYPE_I64) { 1202 if (val != (int32_t)val) { 1203 return false; 1204 } 1205 rexw = P_REXW; 1206 } else if (type != TCG_TYPE_I32) { 1207 return false; 1208 } 1209 tcg_out_modrm_offset(s, OPC_MOVL_EvIz | rexw, 0, base, ofs); 1210 tcg_out32(s, val); 1211 return true; 1212} 1213 1214static void tcg_out_shifti(TCGContext *s, int subopc, int reg, int count) 1215{ 1216 /* Propagate an opcode prefix, such as P_DATA16. */ 1217 int ext = subopc & ~0x7; 1218 subopc &= 0x7; 1219 1220 if (count == 1) { 1221 tcg_out_modrm(s, OPC_SHIFT_1 + ext, subopc, reg); 1222 } else { 1223 tcg_out_modrm(s, OPC_SHIFT_Ib + ext, subopc, reg); 1224 tcg_out8(s, count); 1225 } 1226} 1227 1228static inline void tcg_out_bswap32(TCGContext *s, int reg) 1229{ 1230 tcg_out_opc(s, OPC_BSWAP + LOWREGMASK(reg), 0, reg, 0); 1231} 1232 1233static inline void tcg_out_rolw_8(TCGContext *s, int reg) 1234{ 1235 tcg_out_shifti(s, SHIFT_ROL + P_DATA16, reg, 8); 1236} 1237 1238static inline void tcg_out_ext8u(TCGContext *s, int dest, int src) 1239{ 1240 /* movzbl */ 1241 tcg_debug_assert(src < 4 || TCG_TARGET_REG_BITS == 64); 1242 tcg_out_modrm(s, OPC_MOVZBL + P_REXB_RM, dest, src); 1243} 1244 1245static void tcg_out_ext8s(TCGContext *s, int dest, int src, int rexw) 1246{ 1247 /* movsbl */ 1248 tcg_debug_assert(src < 4 || TCG_TARGET_REG_BITS == 64); 1249 tcg_out_modrm(s, OPC_MOVSBL + P_REXB_RM + rexw, dest, src); 1250} 1251 1252static inline void tcg_out_ext16u(TCGContext *s, int dest, int src) 1253{ 1254 /* movzwl */ 1255 tcg_out_modrm(s, OPC_MOVZWL, dest, src); 1256} 1257 1258static inline void tcg_out_ext16s(TCGContext *s, int dest, int src, int rexw) 1259{ 1260 /* movsw[lq] */ 1261 tcg_out_modrm(s, OPC_MOVSWL + rexw, dest, src); 1262} 1263 1264static inline void tcg_out_ext32u(TCGContext *s, int dest, int src) 1265{ 1266 /* 32-bit mov zero extends. */ 1267 tcg_out_modrm(s, OPC_MOVL_GvEv, dest, src); 1268} 1269 1270static inline void tcg_out_ext32s(TCGContext *s, int dest, int src) 1271{ 1272 tcg_out_modrm(s, OPC_MOVSLQ, dest, src); 1273} 1274 1275static inline void tcg_out_bswap64(TCGContext *s, int reg) 1276{ 1277 tcg_out_opc(s, OPC_BSWAP + P_REXW + LOWREGMASK(reg), 0, reg, 0); 1278} 1279 1280static void tgen_arithi(TCGContext *s, int c, int r0, 1281 tcg_target_long val, int cf) 1282{ 1283 int rexw = 0; 1284 1285 if (TCG_TARGET_REG_BITS == 64) { 1286 rexw = c & -8; 1287 c &= 7; 1288 } 1289 1290 /* ??? While INC is 2 bytes shorter than ADDL $1, they also induce 1291 partial flags update stalls on Pentium4 and are not recommended 1292 by current Intel optimization manuals. */ 1293 if (!cf && (c == ARITH_ADD || c == ARITH_SUB) && (val == 1 || val == -1)) { 1294 int is_inc = (c == ARITH_ADD) ^ (val < 0); 1295 if (TCG_TARGET_REG_BITS == 64) { 1296 /* The single-byte increment encodings are re-tasked as the 1297 REX prefixes. Use the MODRM encoding. */ 1298 tcg_out_modrm(s, OPC_GRP5 + rexw, 1299 (is_inc ? EXT5_INC_Ev : EXT5_DEC_Ev), r0); 1300 } else { 1301 tcg_out8(s, (is_inc ? OPC_INC_r32 : OPC_DEC_r32) + r0); 1302 } 1303 return; 1304 } 1305 1306 if (c == ARITH_AND) { 1307 if (TCG_TARGET_REG_BITS == 64) { 1308 if (val == 0xffffffffu) { 1309 tcg_out_ext32u(s, r0, r0); 1310 return; 1311 } 1312 if (val == (uint32_t)val) { 1313 /* AND with no high bits set can use a 32-bit operation. */ 1314 rexw = 0; 1315 } 1316 } 1317 if (val == 0xffu && (r0 < 4 || TCG_TARGET_REG_BITS == 64)) { 1318 tcg_out_ext8u(s, r0, r0); 1319 return; 1320 } 1321 if (val == 0xffffu) { 1322 tcg_out_ext16u(s, r0, r0); 1323 return; 1324 } 1325 } 1326 1327 if (val == (int8_t)val) { 1328 tcg_out_modrm(s, OPC_ARITH_EvIb + rexw, c, r0); 1329 tcg_out8(s, val); 1330 return; 1331 } 1332 if (rexw == 0 || val == (int32_t)val) { 1333 tcg_out_modrm(s, OPC_ARITH_EvIz + rexw, c, r0); 1334 tcg_out32(s, val); 1335 return; 1336 } 1337 1338 tcg_abort(); 1339} 1340 1341static void tcg_out_addi(TCGContext *s, int reg, tcg_target_long val) 1342{ 1343 if (val != 0) { 1344 tgen_arithi(s, ARITH_ADD + P_REXW, reg, val, 0); 1345 } 1346} 1347 1348/* Use SMALL != 0 to force a short forward branch. */ 1349static void tcg_out_jxx(TCGContext *s, int opc, TCGLabel *l, int small) 1350{ 1351 int32_t val, val1; 1352 1353 if (l->has_value) { 1354 val = tcg_pcrel_diff(s, l->u.value_ptr); 1355 val1 = val - 2; 1356 if ((int8_t)val1 == val1) { 1357 if (opc == -1) { 1358 tcg_out8(s, OPC_JMP_short); 1359 } else { 1360 tcg_out8(s, OPC_JCC_short + opc); 1361 } 1362 tcg_out8(s, val1); 1363 } else { 1364 if (small) { 1365 tcg_abort(); 1366 } 1367 if (opc == -1) { 1368 tcg_out8(s, OPC_JMP_long); 1369 tcg_out32(s, val - 5); 1370 } else { 1371 tcg_out_opc(s, OPC_JCC_long + opc, 0, 0, 0); 1372 tcg_out32(s, val - 6); 1373 } 1374 } 1375 } else if (small) { 1376 if (opc == -1) { 1377 tcg_out8(s, OPC_JMP_short); 1378 } else { 1379 tcg_out8(s, OPC_JCC_short + opc); 1380 } 1381 tcg_out_reloc(s, s->code_ptr, R_386_PC8, l, -1); 1382 s->code_ptr += 1; 1383 } else { 1384 if (opc == -1) { 1385 tcg_out8(s, OPC_JMP_long); 1386 } else { 1387 tcg_out_opc(s, OPC_JCC_long + opc, 0, 0, 0); 1388 } 1389 tcg_out_reloc(s, s->code_ptr, R_386_PC32, l, -4); 1390 s->code_ptr += 4; 1391 } 1392} 1393 1394static void tcg_out_cmp(TCGContext *s, TCGArg arg1, TCGArg arg2, 1395 int const_arg2, int rexw) 1396{ 1397 if (const_arg2) { 1398 if (arg2 == 0) { 1399 /* test r, r */ 1400 tcg_out_modrm(s, OPC_TESTL + rexw, arg1, arg1); 1401 } else { 1402 tgen_arithi(s, ARITH_CMP + rexw, arg1, arg2, 0); 1403 } 1404 } else { 1405 tgen_arithr(s, ARITH_CMP + rexw, arg1, arg2); 1406 } 1407} 1408 1409static void tcg_out_brcond32(TCGContext *s, TCGCond cond, 1410 TCGArg arg1, TCGArg arg2, int const_arg2, 1411 TCGLabel *label, int small) 1412{ 1413 tcg_out_cmp(s, arg1, arg2, const_arg2, 0); 1414 tcg_out_jxx(s, tcg_cond_to_jcc[cond], label, small); 1415} 1416 1417#if TCG_TARGET_REG_BITS == 64 1418static void tcg_out_brcond64(TCGContext *s, TCGCond cond, 1419 TCGArg arg1, TCGArg arg2, int const_arg2, 1420 TCGLabel *label, int small) 1421{ 1422 tcg_out_cmp(s, arg1, arg2, const_arg2, P_REXW); 1423 tcg_out_jxx(s, tcg_cond_to_jcc[cond], label, small); 1424} 1425#else 1426/* XXX: we implement it at the target level to avoid having to 1427 handle cross basic blocks temporaries */ 1428static void tcg_out_brcond2(TCGContext *s, const TCGArg *args, 1429 const int *const_args, int small) 1430{ 1431 TCGLabel *label_next = gen_new_label(); 1432 TCGLabel *label_this = arg_label(args[5]); 1433 1434 switch(args[4]) { 1435 case TCG_COND_EQ: 1436 tcg_out_brcond32(s, TCG_COND_NE, args[0], args[2], const_args[2], 1437 label_next, 1); 1438 tcg_out_brcond32(s, TCG_COND_EQ, args[1], args[3], const_args[3], 1439 label_this, small); 1440 break; 1441 case TCG_COND_NE: 1442 tcg_out_brcond32(s, TCG_COND_NE, args[0], args[2], const_args[2], 1443 label_this, small); 1444 tcg_out_brcond32(s, TCG_COND_NE, args[1], args[3], const_args[3], 1445 label_this, small); 1446 break; 1447 case TCG_COND_LT: 1448 tcg_out_brcond32(s, TCG_COND_LT, args[1], args[3], const_args[3], 1449 label_this, small); 1450 tcg_out_jxx(s, JCC_JNE, label_next, 1); 1451 tcg_out_brcond32(s, TCG_COND_LTU, args[0], args[2], const_args[2], 1452 label_this, small); 1453 break; 1454 case TCG_COND_LE: 1455 tcg_out_brcond32(s, TCG_COND_LT, args[1], args[3], const_args[3], 1456 label_this, small); 1457 tcg_out_jxx(s, JCC_JNE, label_next, 1); 1458 tcg_out_brcond32(s, TCG_COND_LEU, args[0], args[2], const_args[2], 1459 label_this, small); 1460 break; 1461 case TCG_COND_GT: 1462 tcg_out_brcond32(s, TCG_COND_GT, args[1], args[3], const_args[3], 1463 label_this, small); 1464 tcg_out_jxx(s, JCC_JNE, label_next, 1); 1465 tcg_out_brcond32(s, TCG_COND_GTU, args[0], args[2], const_args[2], 1466 label_this, small); 1467 break; 1468 case TCG_COND_GE: 1469 tcg_out_brcond32(s, TCG_COND_GT, args[1], args[3], const_args[3], 1470 label_this, small); 1471 tcg_out_jxx(s, JCC_JNE, label_next, 1); 1472 tcg_out_brcond32(s, TCG_COND_GEU, args[0], args[2], const_args[2], 1473 label_this, small); 1474 break; 1475 case TCG_COND_LTU: 1476 tcg_out_brcond32(s, TCG_COND_LTU, args[1], args[3], const_args[3], 1477 label_this, small); 1478 tcg_out_jxx(s, JCC_JNE, label_next, 1); 1479 tcg_out_brcond32(s, TCG_COND_LTU, args[0], args[2], const_args[2], 1480 label_this, small); 1481 break; 1482 case TCG_COND_LEU: 1483 tcg_out_brcond32(s, TCG_COND_LTU, args[1], args[3], const_args[3], 1484 label_this, small); 1485 tcg_out_jxx(s, JCC_JNE, label_next, 1); 1486 tcg_out_brcond32(s, TCG_COND_LEU, args[0], args[2], const_args[2], 1487 label_this, small); 1488 break; 1489 case TCG_COND_GTU: 1490 tcg_out_brcond32(s, TCG_COND_GTU, args[1], args[3], const_args[3], 1491 label_this, small); 1492 tcg_out_jxx(s, JCC_JNE, label_next, 1); 1493 tcg_out_brcond32(s, TCG_COND_GTU, args[0], args[2], const_args[2], 1494 label_this, small); 1495 break; 1496 case TCG_COND_GEU: 1497 tcg_out_brcond32(s, TCG_COND_GTU, args[1], args[3], const_args[3], 1498 label_this, small); 1499 tcg_out_jxx(s, JCC_JNE, label_next, 1); 1500 tcg_out_brcond32(s, TCG_COND_GEU, args[0], args[2], const_args[2], 1501 label_this, small); 1502 break; 1503 default: 1504 tcg_abort(); 1505 } 1506 tcg_out_label(s, label_next); 1507} 1508#endif 1509 1510static void tcg_out_setcond32(TCGContext *s, TCGCond cond, TCGArg dest, 1511 TCGArg arg1, TCGArg arg2, int const_arg2) 1512{ 1513 tcg_out_cmp(s, arg1, arg2, const_arg2, 0); 1514 tcg_out_modrm(s, OPC_SETCC | tcg_cond_to_jcc[cond], 0, dest); 1515 tcg_out_ext8u(s, dest, dest); 1516} 1517 1518#if TCG_TARGET_REG_BITS == 64 1519static void tcg_out_setcond64(TCGContext *s, TCGCond cond, TCGArg dest, 1520 TCGArg arg1, TCGArg arg2, int const_arg2) 1521{ 1522 tcg_out_cmp(s, arg1, arg2, const_arg2, P_REXW); 1523 tcg_out_modrm(s, OPC_SETCC | tcg_cond_to_jcc[cond], 0, dest); 1524 tcg_out_ext8u(s, dest, dest); 1525} 1526#else 1527static void tcg_out_setcond2(TCGContext *s, const TCGArg *args, 1528 const int *const_args) 1529{ 1530 TCGArg new_args[6]; 1531 TCGLabel *label_true, *label_over; 1532 1533 memcpy(new_args, args+1, 5*sizeof(TCGArg)); 1534 1535 if (args[0] == args[1] || args[0] == args[2] 1536 || (!const_args[3] && args[0] == args[3]) 1537 || (!const_args[4] && args[0] == args[4])) { 1538 /* When the destination overlaps with one of the argument 1539 registers, don't do anything tricky. */ 1540 label_true = gen_new_label(); 1541 label_over = gen_new_label(); 1542 1543 new_args[5] = label_arg(label_true); 1544 tcg_out_brcond2(s, new_args, const_args+1, 1); 1545 1546 tcg_out_movi(s, TCG_TYPE_I32, args[0], 0); 1547 tcg_out_jxx(s, JCC_JMP, label_over, 1); 1548 tcg_out_label(s, label_true); 1549 1550 tcg_out_movi(s, TCG_TYPE_I32, args[0], 1); 1551 tcg_out_label(s, label_over); 1552 } else { 1553 /* When the destination does not overlap one of the arguments, 1554 clear the destination first, jump if cond false, and emit an 1555 increment in the true case. This results in smaller code. */ 1556 1557 tcg_out_movi(s, TCG_TYPE_I32, args[0], 0); 1558 1559 label_over = gen_new_label(); 1560 new_args[4] = tcg_invert_cond(new_args[4]); 1561 new_args[5] = label_arg(label_over); 1562 tcg_out_brcond2(s, new_args, const_args+1, 1); 1563 1564 tgen_arithi(s, ARITH_ADD, args[0], 1, 0); 1565 tcg_out_label(s, label_over); 1566 } 1567} 1568#endif 1569 1570static void tcg_out_cmov(TCGContext *s, TCGCond cond, int rexw, 1571 TCGReg dest, TCGReg v1) 1572{ 1573 if (have_cmov) { 1574 tcg_out_modrm(s, OPC_CMOVCC | tcg_cond_to_jcc[cond] | rexw, dest, v1); 1575 } else { 1576 TCGLabel *over = gen_new_label(); 1577 tcg_out_jxx(s, tcg_cond_to_jcc[tcg_invert_cond(cond)], over, 1); 1578 tcg_out_mov(s, TCG_TYPE_I32, dest, v1); 1579 tcg_out_label(s, over); 1580 } 1581} 1582 1583static void tcg_out_movcond32(TCGContext *s, TCGCond cond, TCGReg dest, 1584 TCGReg c1, TCGArg c2, int const_c2, 1585 TCGReg v1) 1586{ 1587 tcg_out_cmp(s, c1, c2, const_c2, 0); 1588 tcg_out_cmov(s, cond, 0, dest, v1); 1589} 1590 1591#if TCG_TARGET_REG_BITS == 64 1592static void tcg_out_movcond64(TCGContext *s, TCGCond cond, TCGReg dest, 1593 TCGReg c1, TCGArg c2, int const_c2, 1594 TCGReg v1) 1595{ 1596 tcg_out_cmp(s, c1, c2, const_c2, P_REXW); 1597 tcg_out_cmov(s, cond, P_REXW, dest, v1); 1598} 1599#endif 1600 1601static void tcg_out_ctz(TCGContext *s, int rexw, TCGReg dest, TCGReg arg1, 1602 TCGArg arg2, bool const_a2) 1603{ 1604 if (have_bmi1) { 1605 tcg_out_modrm(s, OPC_TZCNT + rexw, dest, arg1); 1606 if (const_a2) { 1607 tcg_debug_assert(arg2 == (rexw ? 64 : 32)); 1608 } else { 1609 tcg_debug_assert(dest != arg2); 1610 tcg_out_cmov(s, TCG_COND_LTU, rexw, dest, arg2); 1611 } 1612 } else { 1613 tcg_debug_assert(dest != arg2); 1614 tcg_out_modrm(s, OPC_BSF + rexw, dest, arg1); 1615 tcg_out_cmov(s, TCG_COND_EQ, rexw, dest, arg2); 1616 } 1617} 1618 1619static void tcg_out_clz(TCGContext *s, int rexw, TCGReg dest, TCGReg arg1, 1620 TCGArg arg2, bool const_a2) 1621{ 1622 if (have_lzcnt) { 1623 tcg_out_modrm(s, OPC_LZCNT + rexw, dest, arg1); 1624 if (const_a2) { 1625 tcg_debug_assert(arg2 == (rexw ? 64 : 32)); 1626 } else { 1627 tcg_debug_assert(dest != arg2); 1628 tcg_out_cmov(s, TCG_COND_LTU, rexw, dest, arg2); 1629 } 1630 } else { 1631 tcg_debug_assert(!const_a2); 1632 tcg_debug_assert(dest != arg1); 1633 tcg_debug_assert(dest != arg2); 1634 1635 /* Recall that the output of BSR is the index not the count. */ 1636 tcg_out_modrm(s, OPC_BSR + rexw, dest, arg1); 1637 tgen_arithi(s, ARITH_XOR + rexw, dest, rexw ? 63 : 31, 0); 1638 1639 /* Since we have destroyed the flags from BSR, we have to re-test. */ 1640 tcg_out_cmp(s, arg1, 0, 1, rexw); 1641 tcg_out_cmov(s, TCG_COND_EQ, rexw, dest, arg2); 1642 } 1643} 1644 1645static void tcg_out_branch(TCGContext *s, int call, const tcg_insn_unit *dest) 1646{ 1647 intptr_t disp = tcg_pcrel_diff(s, dest) - 5; 1648 1649 if (disp == (int32_t)disp) { 1650 tcg_out_opc(s, call ? OPC_CALL_Jz : OPC_JMP_long, 0, 0, 0); 1651 tcg_out32(s, disp); 1652 } else { 1653 /* rip-relative addressing into the constant pool. 1654 This is 6 + 8 = 14 bytes, as compared to using an 1655 immediate load 10 + 6 = 16 bytes, plus we may 1656 be able to re-use the pool constant for more calls. */ 1657 tcg_out_opc(s, OPC_GRP5, 0, 0, 0); 1658 tcg_out8(s, (call ? EXT5_CALLN_Ev : EXT5_JMPN_Ev) << 3 | 5); 1659 new_pool_label(s, (uintptr_t)dest, R_386_PC32, s->code_ptr, -4); 1660 tcg_out32(s, 0); 1661 } 1662} 1663 1664static void tcg_out_call(TCGContext *s, const tcg_insn_unit *dest, 1665 const TCGHelperInfo *info) 1666{ 1667 tcg_out_branch(s, 1, dest); 1668} 1669 1670static void tcg_out_jmp(TCGContext *s, const tcg_insn_unit *dest) 1671{ 1672 tcg_out_branch(s, 0, dest); 1673} 1674 1675static void tcg_out_nopn(TCGContext *s, int n) 1676{ 1677 int i; 1678 /* Emit 1 or 2 operand size prefixes for the standard one byte nop, 1679 * "xchg %eax,%eax", forming "xchg %ax,%ax". All cores accept the 1680 * duplicate prefix, and all of the interesting recent cores can 1681 * decode and discard the duplicates in a single cycle. 1682 */ 1683 tcg_debug_assert(n >= 1); 1684 for (i = 1; i < n; ++i) { 1685 tcg_out8(s, 0x66); 1686 } 1687 tcg_out8(s, 0x90); 1688} 1689 1690#if defined(CONFIG_SOFTMMU) 1691/* helper signature: helper_ret_ld_mmu(CPUState *env, target_ulong addr, 1692 * int mmu_idx, uintptr_t ra) 1693 */ 1694static void * const qemu_ld_helpers[(MO_SIZE | MO_BSWAP) + 1] = { 1695 [MO_UB] = helper_ret_ldub_mmu, 1696 [MO_LEUW] = helper_le_lduw_mmu, 1697 [MO_LEUL] = helper_le_ldul_mmu, 1698 [MO_LEUQ] = helper_le_ldq_mmu, 1699 [MO_BEUW] = helper_be_lduw_mmu, 1700 [MO_BEUL] = helper_be_ldul_mmu, 1701 [MO_BEUQ] = helper_be_ldq_mmu, 1702}; 1703 1704/* helper signature: helper_ret_st_mmu(CPUState *env, target_ulong addr, 1705 * uintxx_t val, int mmu_idx, uintptr_t ra) 1706 */ 1707static void * const qemu_st_helpers[(MO_SIZE | MO_BSWAP) + 1] = { 1708 [MO_UB] = helper_ret_stb_mmu, 1709 [MO_LEUW] = helper_le_stw_mmu, 1710 [MO_LEUL] = helper_le_stl_mmu, 1711 [MO_LEUQ] = helper_le_stq_mmu, 1712 [MO_BEUW] = helper_be_stw_mmu, 1713 [MO_BEUL] = helper_be_stl_mmu, 1714 [MO_BEUQ] = helper_be_stq_mmu, 1715}; 1716 1717/* Perform the TLB load and compare. 1718 1719 Inputs: 1720 ADDRLO and ADDRHI contain the low and high part of the address. 1721 1722 MEM_INDEX and S_BITS are the memory context and log2 size of the load. 1723 1724 WHICH is the offset into the CPUTLBEntry structure of the slot to read. 1725 This should be offsetof addr_read or addr_write. 1726 1727 Outputs: 1728 LABEL_PTRS is filled with 1 (32-bit addresses) or 2 (64-bit addresses) 1729 positions of the displacements of forward jumps to the TLB miss case. 1730 1731 Second argument register is loaded with the low part of the address. 1732 In the TLB hit case, it has been adjusted as indicated by the TLB 1733 and so is a host address. In the TLB miss case, it continues to 1734 hold a guest address. 1735 1736 First argument register is clobbered. */ 1737 1738static inline void tcg_out_tlb_load(TCGContext *s, TCGReg addrlo, TCGReg addrhi, 1739 int mem_index, MemOp opc, 1740 tcg_insn_unit **label_ptr, int which) 1741{ 1742 const TCGReg r0 = TCG_REG_L0; 1743 const TCGReg r1 = TCG_REG_L1; 1744 TCGType ttype = TCG_TYPE_I32; 1745 TCGType tlbtype = TCG_TYPE_I32; 1746 int trexw = 0, hrexw = 0, tlbrexw = 0; 1747 unsigned a_bits = get_alignment_bits(opc); 1748 unsigned s_bits = opc & MO_SIZE; 1749 unsigned a_mask = (1 << a_bits) - 1; 1750 unsigned s_mask = (1 << s_bits) - 1; 1751 target_ulong tlb_mask; 1752 1753 if (TCG_TARGET_REG_BITS == 64) { 1754 if (TARGET_LONG_BITS == 64) { 1755 ttype = TCG_TYPE_I64; 1756 trexw = P_REXW; 1757 } 1758 if (TCG_TYPE_PTR == TCG_TYPE_I64) { 1759 hrexw = P_REXW; 1760 if (TARGET_PAGE_BITS + CPU_TLB_DYN_MAX_BITS > 32) { 1761 tlbtype = TCG_TYPE_I64; 1762 tlbrexw = P_REXW; 1763 } 1764 } 1765 } 1766 1767 tcg_out_mov(s, tlbtype, r0, addrlo); 1768 tcg_out_shifti(s, SHIFT_SHR + tlbrexw, r0, 1769 TARGET_PAGE_BITS - CPU_TLB_ENTRY_BITS); 1770 1771 tcg_out_modrm_offset(s, OPC_AND_GvEv + trexw, r0, TCG_AREG0, 1772 TLB_MASK_TABLE_OFS(mem_index) + 1773 offsetof(CPUTLBDescFast, mask)); 1774 1775 tcg_out_modrm_offset(s, OPC_ADD_GvEv + hrexw, r0, TCG_AREG0, 1776 TLB_MASK_TABLE_OFS(mem_index) + 1777 offsetof(CPUTLBDescFast, table)); 1778 1779 /* If the required alignment is at least as large as the access, simply 1780 copy the address and mask. For lesser alignments, check that we don't 1781 cross pages for the complete access. */ 1782 if (a_bits >= s_bits) { 1783 tcg_out_mov(s, ttype, r1, addrlo); 1784 } else { 1785 tcg_out_modrm_offset(s, OPC_LEA + trexw, r1, addrlo, s_mask - a_mask); 1786 } 1787 tlb_mask = (target_ulong)TARGET_PAGE_MASK | a_mask; 1788 tgen_arithi(s, ARITH_AND + trexw, r1, tlb_mask, 0); 1789 1790 /* cmp 0(r0), r1 */ 1791 tcg_out_modrm_offset(s, OPC_CMP_GvEv + trexw, r1, r0, which); 1792 1793 /* Prepare for both the fast path add of the tlb addend, and the slow 1794 path function argument setup. */ 1795 tcg_out_mov(s, ttype, r1, addrlo); 1796 1797 /* jne slow_path */ 1798 tcg_out_opc(s, OPC_JCC_long + JCC_JNE, 0, 0, 0); 1799 label_ptr[0] = s->code_ptr; 1800 s->code_ptr += 4; 1801 1802 if (TARGET_LONG_BITS > TCG_TARGET_REG_BITS) { 1803 /* cmp 4(r0), addrhi */ 1804 tcg_out_modrm_offset(s, OPC_CMP_GvEv, addrhi, r0, which + 4); 1805 1806 /* jne slow_path */ 1807 tcg_out_opc(s, OPC_JCC_long + JCC_JNE, 0, 0, 0); 1808 label_ptr[1] = s->code_ptr; 1809 s->code_ptr += 4; 1810 } 1811 1812 /* TLB Hit. */ 1813 1814 /* add addend(r0), r1 */ 1815 tcg_out_modrm_offset(s, OPC_ADD_GvEv + hrexw, r1, r0, 1816 offsetof(CPUTLBEntry, addend)); 1817} 1818 1819/* 1820 * Record the context of a call to the out of line helper code for the slow path 1821 * for a load or store, so that we can later generate the correct helper code 1822 */ 1823static void add_qemu_ldst_label(TCGContext *s, bool is_ld, bool is_64, 1824 MemOpIdx oi, 1825 TCGReg datalo, TCGReg datahi, 1826 TCGReg addrlo, TCGReg addrhi, 1827 tcg_insn_unit *raddr, 1828 tcg_insn_unit **label_ptr) 1829{ 1830 TCGLabelQemuLdst *label = new_ldst_label(s); 1831 1832 label->is_ld = is_ld; 1833 label->oi = oi; 1834 label->type = is_64 ? TCG_TYPE_I64 : TCG_TYPE_I32; 1835 label->datalo_reg = datalo; 1836 label->datahi_reg = datahi; 1837 label->addrlo_reg = addrlo; 1838 label->addrhi_reg = addrhi; 1839 label->raddr = tcg_splitwx_to_rx(raddr); 1840 label->label_ptr[0] = label_ptr[0]; 1841 if (TARGET_LONG_BITS > TCG_TARGET_REG_BITS) { 1842 label->label_ptr[1] = label_ptr[1]; 1843 } 1844} 1845 1846/* 1847 * Generate code for the slow path for a load at the end of block 1848 */ 1849static bool tcg_out_qemu_ld_slow_path(TCGContext *s, TCGLabelQemuLdst *l) 1850{ 1851 MemOpIdx oi = l->oi; 1852 MemOp opc = get_memop(oi); 1853 TCGReg data_reg; 1854 tcg_insn_unit **label_ptr = &l->label_ptr[0]; 1855 int rexw = (l->type == TCG_TYPE_I64 ? P_REXW : 0); 1856 1857 /* resolve label address */ 1858 tcg_patch32(label_ptr[0], s->code_ptr - label_ptr[0] - 4); 1859 if (TARGET_LONG_BITS > TCG_TARGET_REG_BITS) { 1860 tcg_patch32(label_ptr[1], s->code_ptr - label_ptr[1] - 4); 1861 } 1862 1863 if (TCG_TARGET_REG_BITS == 32) { 1864 int ofs = 0; 1865 1866 tcg_out_st(s, TCG_TYPE_PTR, TCG_AREG0, TCG_REG_ESP, ofs); 1867 ofs += 4; 1868 1869 tcg_out_st(s, TCG_TYPE_I32, l->addrlo_reg, TCG_REG_ESP, ofs); 1870 ofs += 4; 1871 1872 if (TARGET_LONG_BITS == 64) { 1873 tcg_out_st(s, TCG_TYPE_I32, l->addrhi_reg, TCG_REG_ESP, ofs); 1874 ofs += 4; 1875 } 1876 1877 tcg_out_sti(s, TCG_TYPE_I32, oi, TCG_REG_ESP, ofs); 1878 ofs += 4; 1879 1880 tcg_out_sti(s, TCG_TYPE_PTR, (uintptr_t)l->raddr, TCG_REG_ESP, ofs); 1881 } else { 1882 tcg_out_mov(s, TCG_TYPE_PTR, tcg_target_call_iarg_regs[0], TCG_AREG0); 1883 /* The second argument is already loaded with addrlo. */ 1884 tcg_out_movi(s, TCG_TYPE_I32, tcg_target_call_iarg_regs[2], oi); 1885 tcg_out_movi(s, TCG_TYPE_PTR, tcg_target_call_iarg_regs[3], 1886 (uintptr_t)l->raddr); 1887 } 1888 1889 tcg_out_branch(s, 1, qemu_ld_helpers[opc & (MO_BSWAP | MO_SIZE)]); 1890 1891 data_reg = l->datalo_reg; 1892 switch (opc & MO_SSIZE) { 1893 case MO_SB: 1894 tcg_out_ext8s(s, data_reg, TCG_REG_EAX, rexw); 1895 break; 1896 case MO_SW: 1897 tcg_out_ext16s(s, data_reg, TCG_REG_EAX, rexw); 1898 break; 1899#if TCG_TARGET_REG_BITS == 64 1900 case MO_SL: 1901 tcg_out_ext32s(s, data_reg, TCG_REG_EAX); 1902 break; 1903#endif 1904 case MO_UB: 1905 case MO_UW: 1906 /* Note that the helpers have zero-extended to tcg_target_long. */ 1907 case MO_UL: 1908 tcg_out_mov(s, TCG_TYPE_I32, data_reg, TCG_REG_EAX); 1909 break; 1910 case MO_UQ: 1911 if (TCG_TARGET_REG_BITS == 64) { 1912 tcg_out_mov(s, TCG_TYPE_I64, data_reg, TCG_REG_RAX); 1913 } else if (data_reg == TCG_REG_EDX) { 1914 /* xchg %edx, %eax */ 1915 tcg_out_opc(s, OPC_XCHG_ax_r32 + TCG_REG_EDX, 0, 0, 0); 1916 tcg_out_mov(s, TCG_TYPE_I32, l->datahi_reg, TCG_REG_EAX); 1917 } else { 1918 tcg_out_mov(s, TCG_TYPE_I32, data_reg, TCG_REG_EAX); 1919 tcg_out_mov(s, TCG_TYPE_I32, l->datahi_reg, TCG_REG_EDX); 1920 } 1921 break; 1922 default: 1923 tcg_abort(); 1924 } 1925 1926 /* Jump to the code corresponding to next IR of qemu_st */ 1927 tcg_out_jmp(s, l->raddr); 1928 return true; 1929} 1930 1931/* 1932 * Generate code for the slow path for a store at the end of block 1933 */ 1934static bool tcg_out_qemu_st_slow_path(TCGContext *s, TCGLabelQemuLdst *l) 1935{ 1936 MemOpIdx oi = l->oi; 1937 MemOp opc = get_memop(oi); 1938 MemOp s_bits = opc & MO_SIZE; 1939 tcg_insn_unit **label_ptr = &l->label_ptr[0]; 1940 TCGReg retaddr; 1941 1942 /* resolve label address */ 1943 tcg_patch32(label_ptr[0], s->code_ptr - label_ptr[0] - 4); 1944 if (TARGET_LONG_BITS > TCG_TARGET_REG_BITS) { 1945 tcg_patch32(label_ptr[1], s->code_ptr - label_ptr[1] - 4); 1946 } 1947 1948 if (TCG_TARGET_REG_BITS == 32) { 1949 int ofs = 0; 1950 1951 tcg_out_st(s, TCG_TYPE_PTR, TCG_AREG0, TCG_REG_ESP, ofs); 1952 ofs += 4; 1953 1954 tcg_out_st(s, TCG_TYPE_I32, l->addrlo_reg, TCG_REG_ESP, ofs); 1955 ofs += 4; 1956 1957 if (TARGET_LONG_BITS == 64) { 1958 tcg_out_st(s, TCG_TYPE_I32, l->addrhi_reg, TCG_REG_ESP, ofs); 1959 ofs += 4; 1960 } 1961 1962 tcg_out_st(s, TCG_TYPE_I32, l->datalo_reg, TCG_REG_ESP, ofs); 1963 ofs += 4; 1964 1965 if (s_bits == MO_64) { 1966 tcg_out_st(s, TCG_TYPE_I32, l->datahi_reg, TCG_REG_ESP, ofs); 1967 ofs += 4; 1968 } 1969 1970 tcg_out_sti(s, TCG_TYPE_I32, oi, TCG_REG_ESP, ofs); 1971 ofs += 4; 1972 1973 retaddr = TCG_REG_EAX; 1974 tcg_out_movi(s, TCG_TYPE_PTR, retaddr, (uintptr_t)l->raddr); 1975 tcg_out_st(s, TCG_TYPE_PTR, retaddr, TCG_REG_ESP, ofs); 1976 } else { 1977 tcg_out_mov(s, TCG_TYPE_PTR, tcg_target_call_iarg_regs[0], TCG_AREG0); 1978 /* The second argument is already loaded with addrlo. */ 1979 tcg_out_mov(s, (s_bits == MO_64 ? TCG_TYPE_I64 : TCG_TYPE_I32), 1980 tcg_target_call_iarg_regs[2], l->datalo_reg); 1981 tcg_out_movi(s, TCG_TYPE_I32, tcg_target_call_iarg_regs[3], oi); 1982 1983 if (ARRAY_SIZE(tcg_target_call_iarg_regs) > 4) { 1984 retaddr = tcg_target_call_iarg_regs[4]; 1985 tcg_out_movi(s, TCG_TYPE_PTR, retaddr, (uintptr_t)l->raddr); 1986 } else { 1987 retaddr = TCG_REG_RAX; 1988 tcg_out_movi(s, TCG_TYPE_PTR, retaddr, (uintptr_t)l->raddr); 1989 tcg_out_st(s, TCG_TYPE_PTR, retaddr, TCG_REG_ESP, 1990 TCG_TARGET_CALL_STACK_OFFSET); 1991 } 1992 } 1993 1994 /* "Tail call" to the helper, with the return address back inline. */ 1995 tcg_out_push(s, retaddr); 1996 tcg_out_jmp(s, qemu_st_helpers[opc & (MO_BSWAP | MO_SIZE)]); 1997 return true; 1998} 1999#else 2000 2001static void tcg_out_test_alignment(TCGContext *s, bool is_ld, TCGReg addrlo, 2002 TCGReg addrhi, unsigned a_bits) 2003{ 2004 unsigned a_mask = (1 << a_bits) - 1; 2005 TCGLabelQemuLdst *label; 2006 2007 /* 2008 * We are expecting a_bits to max out at 7, so we can usually use testb. 2009 * For i686, we have to use testl for %esi/%edi. 2010 */ 2011 if (a_mask <= 0xff && (TCG_TARGET_REG_BITS == 64 || addrlo < 4)) { 2012 tcg_out_modrm(s, OPC_GRP3_Eb | P_REXB_RM, EXT3_TESTi, addrlo); 2013 tcg_out8(s, a_mask); 2014 } else { 2015 tcg_out_modrm(s, OPC_GRP3_Ev, EXT3_TESTi, addrlo); 2016 tcg_out32(s, a_mask); 2017 } 2018 2019 /* jne slow_path */ 2020 tcg_out_opc(s, OPC_JCC_long + JCC_JNE, 0, 0, 0); 2021 2022 label = new_ldst_label(s); 2023 label->is_ld = is_ld; 2024 label->addrlo_reg = addrlo; 2025 label->addrhi_reg = addrhi; 2026 label->raddr = tcg_splitwx_to_rx(s->code_ptr + 4); 2027 label->label_ptr[0] = s->code_ptr; 2028 2029 s->code_ptr += 4; 2030} 2031 2032static bool tcg_out_fail_alignment(TCGContext *s, TCGLabelQemuLdst *l) 2033{ 2034 /* resolve label address */ 2035 tcg_patch32(l->label_ptr[0], s->code_ptr - l->label_ptr[0] - 4); 2036 2037 if (TCG_TARGET_REG_BITS == 32) { 2038 int ofs = 0; 2039 2040 tcg_out_st(s, TCG_TYPE_PTR, TCG_AREG0, TCG_REG_ESP, ofs); 2041 ofs += 4; 2042 2043 tcg_out_st(s, TCG_TYPE_I32, l->addrlo_reg, TCG_REG_ESP, ofs); 2044 ofs += 4; 2045 if (TARGET_LONG_BITS == 64) { 2046 tcg_out_st(s, TCG_TYPE_I32, l->addrhi_reg, TCG_REG_ESP, ofs); 2047 ofs += 4; 2048 } 2049 2050 tcg_out_pushi(s, (uintptr_t)l->raddr); 2051 } else { 2052 tcg_out_mov(s, TCG_TYPE_TL, tcg_target_call_iarg_regs[1], 2053 l->addrlo_reg); 2054 tcg_out_mov(s, TCG_TYPE_PTR, tcg_target_call_iarg_regs[0], TCG_AREG0); 2055 2056 tcg_out_movi(s, TCG_TYPE_PTR, TCG_REG_RAX, (uintptr_t)l->raddr); 2057 tcg_out_push(s, TCG_REG_RAX); 2058 } 2059 2060 /* "Tail call" to the helper, with the return address back inline. */ 2061 tcg_out_jmp(s, (const void *)(l->is_ld ? helper_unaligned_ld 2062 : helper_unaligned_st)); 2063 return true; 2064} 2065 2066static bool tcg_out_qemu_ld_slow_path(TCGContext *s, TCGLabelQemuLdst *l) 2067{ 2068 return tcg_out_fail_alignment(s, l); 2069} 2070 2071static bool tcg_out_qemu_st_slow_path(TCGContext *s, TCGLabelQemuLdst *l) 2072{ 2073 return tcg_out_fail_alignment(s, l); 2074} 2075 2076#if TCG_TARGET_REG_BITS == 32 2077# define x86_guest_base_seg 0 2078# define x86_guest_base_index -1 2079# define x86_guest_base_offset guest_base 2080#else 2081static int x86_guest_base_seg; 2082static int x86_guest_base_index = -1; 2083static int32_t x86_guest_base_offset; 2084# if defined(__x86_64__) && defined(__linux__) 2085# include <asm/prctl.h> 2086# include <sys/prctl.h> 2087int arch_prctl(int code, unsigned long addr); 2088static inline int setup_guest_base_seg(void) 2089{ 2090 if (arch_prctl(ARCH_SET_GS, guest_base) == 0) { 2091 return P_GS; 2092 } 2093 return 0; 2094} 2095# elif defined (__FreeBSD__) || defined (__FreeBSD_kernel__) 2096# include <machine/sysarch.h> 2097static inline int setup_guest_base_seg(void) 2098{ 2099 if (sysarch(AMD64_SET_GSBASE, &guest_base) == 0) { 2100 return P_GS; 2101 } 2102 return 0; 2103} 2104# else 2105static inline int setup_guest_base_seg(void) 2106{ 2107 return 0; 2108} 2109# endif 2110#endif 2111#endif /* SOFTMMU */ 2112 2113static void tcg_out_qemu_ld_direct(TCGContext *s, TCGReg datalo, TCGReg datahi, 2114 TCGReg base, int index, intptr_t ofs, 2115 int seg, bool is64, MemOp memop) 2116{ 2117 bool use_movbe = false; 2118 int rexw = is64 * P_REXW; 2119 int movop = OPC_MOVL_GvEv; 2120 2121 /* Do big-endian loads with movbe. */ 2122 if (memop & MO_BSWAP) { 2123 tcg_debug_assert(have_movbe); 2124 use_movbe = true; 2125 movop = OPC_MOVBE_GyMy; 2126 } 2127 2128 switch (memop & MO_SSIZE) { 2129 case MO_UB: 2130 tcg_out_modrm_sib_offset(s, OPC_MOVZBL + seg, datalo, 2131 base, index, 0, ofs); 2132 break; 2133 case MO_SB: 2134 tcg_out_modrm_sib_offset(s, OPC_MOVSBL + rexw + seg, datalo, 2135 base, index, 0, ofs); 2136 break; 2137 case MO_UW: 2138 if (use_movbe) { 2139 /* There is no extending movbe; only low 16-bits are modified. */ 2140 if (datalo != base && datalo != index) { 2141 /* XOR breaks dependency chains. */ 2142 tgen_arithr(s, ARITH_XOR, datalo, datalo); 2143 tcg_out_modrm_sib_offset(s, OPC_MOVBE_GyMy + P_DATA16 + seg, 2144 datalo, base, index, 0, ofs); 2145 } else { 2146 tcg_out_modrm_sib_offset(s, OPC_MOVBE_GyMy + P_DATA16 + seg, 2147 datalo, base, index, 0, ofs); 2148 tcg_out_ext16u(s, datalo, datalo); 2149 } 2150 } else { 2151 tcg_out_modrm_sib_offset(s, OPC_MOVZWL + seg, datalo, 2152 base, index, 0, ofs); 2153 } 2154 break; 2155 case MO_SW: 2156 if (use_movbe) { 2157 tcg_out_modrm_sib_offset(s, OPC_MOVBE_GyMy + P_DATA16 + seg, 2158 datalo, base, index, 0, ofs); 2159 tcg_out_ext16s(s, datalo, datalo, rexw); 2160 } else { 2161 tcg_out_modrm_sib_offset(s, OPC_MOVSWL + rexw + seg, 2162 datalo, base, index, 0, ofs); 2163 } 2164 break; 2165 case MO_UL: 2166 tcg_out_modrm_sib_offset(s, movop + seg, datalo, base, index, 0, ofs); 2167 break; 2168#if TCG_TARGET_REG_BITS == 64 2169 case MO_SL: 2170 if (use_movbe) { 2171 tcg_out_modrm_sib_offset(s, OPC_MOVBE_GyMy + seg, datalo, 2172 base, index, 0, ofs); 2173 tcg_out_ext32s(s, datalo, datalo); 2174 } else { 2175 tcg_out_modrm_sib_offset(s, OPC_MOVSLQ + seg, datalo, 2176 base, index, 0, ofs); 2177 } 2178 break; 2179#endif 2180 case MO_UQ: 2181 if (TCG_TARGET_REG_BITS == 64) { 2182 tcg_out_modrm_sib_offset(s, movop + P_REXW + seg, datalo, 2183 base, index, 0, ofs); 2184 } else { 2185 if (use_movbe) { 2186 TCGReg t = datalo; 2187 datalo = datahi; 2188 datahi = t; 2189 } 2190 if (base != datalo) { 2191 tcg_out_modrm_sib_offset(s, movop + seg, datalo, 2192 base, index, 0, ofs); 2193 tcg_out_modrm_sib_offset(s, movop + seg, datahi, 2194 base, index, 0, ofs + 4); 2195 } else { 2196 tcg_out_modrm_sib_offset(s, movop + seg, datahi, 2197 base, index, 0, ofs + 4); 2198 tcg_out_modrm_sib_offset(s, movop + seg, datalo, 2199 base, index, 0, ofs); 2200 } 2201 } 2202 break; 2203 default: 2204 g_assert_not_reached(); 2205 } 2206} 2207 2208/* XXX: qemu_ld and qemu_st could be modified to clobber only EDX and 2209 EAX. It will be useful once fixed registers globals are less 2210 common. */ 2211static void tcg_out_qemu_ld(TCGContext *s, const TCGArg *args, bool is64) 2212{ 2213 TCGReg datalo, datahi, addrlo; 2214 TCGReg addrhi __attribute__((unused)); 2215 MemOpIdx oi; 2216 MemOp opc; 2217#if defined(CONFIG_SOFTMMU) 2218 int mem_index; 2219 tcg_insn_unit *label_ptr[2]; 2220#else 2221 unsigned a_bits; 2222#endif 2223 2224 datalo = *args++; 2225 datahi = (TCG_TARGET_REG_BITS == 32 && is64 ? *args++ : 0); 2226 addrlo = *args++; 2227 addrhi = (TARGET_LONG_BITS > TCG_TARGET_REG_BITS ? *args++ : 0); 2228 oi = *args++; 2229 opc = get_memop(oi); 2230 2231#if defined(CONFIG_SOFTMMU) 2232 mem_index = get_mmuidx(oi); 2233 2234 tcg_out_tlb_load(s, addrlo, addrhi, mem_index, opc, 2235 label_ptr, offsetof(CPUTLBEntry, addr_read)); 2236 2237 /* TLB Hit. */ 2238 tcg_out_qemu_ld_direct(s, datalo, datahi, TCG_REG_L1, -1, 0, 0, is64, opc); 2239 2240 /* Record the current context of a load into ldst label */ 2241 add_qemu_ldst_label(s, true, is64, oi, datalo, datahi, addrlo, addrhi, 2242 s->code_ptr, label_ptr); 2243#else 2244 a_bits = get_alignment_bits(opc); 2245 if (a_bits) { 2246 tcg_out_test_alignment(s, true, addrlo, addrhi, a_bits); 2247 } 2248 2249 tcg_out_qemu_ld_direct(s, datalo, datahi, addrlo, x86_guest_base_index, 2250 x86_guest_base_offset, x86_guest_base_seg, 2251 is64, opc); 2252#endif 2253} 2254 2255static void tcg_out_qemu_st_direct(TCGContext *s, TCGReg datalo, TCGReg datahi, 2256 TCGReg base, int index, intptr_t ofs, 2257 int seg, MemOp memop) 2258{ 2259 bool use_movbe = false; 2260 int movop = OPC_MOVL_EvGv; 2261 2262 /* 2263 * Do big-endian stores with movbe or softmmu. 2264 * User-only without movbe will have its swapping done generically. 2265 */ 2266 if (memop & MO_BSWAP) { 2267 tcg_debug_assert(have_movbe); 2268 use_movbe = true; 2269 movop = OPC_MOVBE_MyGy; 2270 } 2271 2272 switch (memop & MO_SIZE) { 2273 case MO_8: 2274 /* This is handled with constraints on INDEX_op_qemu_st8_i32. */ 2275 tcg_debug_assert(TCG_TARGET_REG_BITS == 64 || datalo < 4); 2276 tcg_out_modrm_sib_offset(s, OPC_MOVB_EvGv + P_REXB_R + seg, 2277 datalo, base, index, 0, ofs); 2278 break; 2279 case MO_16: 2280 tcg_out_modrm_sib_offset(s, movop + P_DATA16 + seg, datalo, 2281 base, index, 0, ofs); 2282 break; 2283 case MO_32: 2284 tcg_out_modrm_sib_offset(s, movop + seg, datalo, base, index, 0, ofs); 2285 break; 2286 case MO_64: 2287 if (TCG_TARGET_REG_BITS == 64) { 2288 tcg_out_modrm_sib_offset(s, movop + P_REXW + seg, datalo, 2289 base, index, 0, ofs); 2290 } else { 2291 if (use_movbe) { 2292 TCGReg t = datalo; 2293 datalo = datahi; 2294 datahi = t; 2295 } 2296 tcg_out_modrm_sib_offset(s, movop + seg, datalo, 2297 base, index, 0, ofs); 2298 tcg_out_modrm_sib_offset(s, movop + seg, datahi, 2299 base, index, 0, ofs + 4); 2300 } 2301 break; 2302 default: 2303 g_assert_not_reached(); 2304 } 2305} 2306 2307static void tcg_out_qemu_st(TCGContext *s, const TCGArg *args, bool is64) 2308{ 2309 TCGReg datalo, datahi, addrlo; 2310 TCGReg addrhi __attribute__((unused)); 2311 MemOpIdx oi; 2312 MemOp opc; 2313#if defined(CONFIG_SOFTMMU) 2314 int mem_index; 2315 tcg_insn_unit *label_ptr[2]; 2316#else 2317 unsigned a_bits; 2318#endif 2319 2320 datalo = *args++; 2321 datahi = (TCG_TARGET_REG_BITS == 32 && is64 ? *args++ : 0); 2322 addrlo = *args++; 2323 addrhi = (TARGET_LONG_BITS > TCG_TARGET_REG_BITS ? *args++ : 0); 2324 oi = *args++; 2325 opc = get_memop(oi); 2326 2327#if defined(CONFIG_SOFTMMU) 2328 mem_index = get_mmuidx(oi); 2329 2330 tcg_out_tlb_load(s, addrlo, addrhi, mem_index, opc, 2331 label_ptr, offsetof(CPUTLBEntry, addr_write)); 2332 2333 /* TLB Hit. */ 2334 tcg_out_qemu_st_direct(s, datalo, datahi, TCG_REG_L1, -1, 0, 0, opc); 2335 2336 /* Record the current context of a store into ldst label */ 2337 add_qemu_ldst_label(s, false, is64, oi, datalo, datahi, addrlo, addrhi, 2338 s->code_ptr, label_ptr); 2339#else 2340 a_bits = get_alignment_bits(opc); 2341 if (a_bits) { 2342 tcg_out_test_alignment(s, false, addrlo, addrhi, a_bits); 2343 } 2344 2345 tcg_out_qemu_st_direct(s, datalo, datahi, addrlo, x86_guest_base_index, 2346 x86_guest_base_offset, x86_guest_base_seg, opc); 2347#endif 2348} 2349 2350static void tcg_out_exit_tb(TCGContext *s, uintptr_t a0) 2351{ 2352 /* Reuse the zeroing that exists for goto_ptr. */ 2353 if (a0 == 0) { 2354 tcg_out_jmp(s, tcg_code_gen_epilogue); 2355 } else { 2356 tcg_out_movi(s, TCG_TYPE_PTR, TCG_REG_EAX, a0); 2357 tcg_out_jmp(s, tb_ret_addr); 2358 } 2359} 2360 2361static void tcg_out_goto_tb(TCGContext *s, int which) 2362{ 2363 /* 2364 * Jump displacement must be aligned for atomic patching; 2365 * see if we need to add extra nops before jump 2366 */ 2367 int gap = QEMU_ALIGN_PTR_UP(s->code_ptr + 1, 4) - s->code_ptr; 2368 if (gap != 1) { 2369 tcg_out_nopn(s, gap - 1); 2370 } 2371 tcg_out8(s, OPC_JMP_long); /* jmp im */ 2372 set_jmp_insn_offset(s, which); 2373 tcg_out32(s, 0); 2374 set_jmp_reset_offset(s, which); 2375} 2376 2377void tb_target_set_jmp_target(const TranslationBlock *tb, int n, 2378 uintptr_t jmp_rx, uintptr_t jmp_rw) 2379{ 2380 /* patch the branch destination */ 2381 uintptr_t addr = tb->jmp_target_addr[n]; 2382 qatomic_set((int32_t *)jmp_rw, addr - (jmp_rx + 4)); 2383 /* no need to flush icache explicitly */ 2384} 2385 2386static inline void tcg_out_op(TCGContext *s, TCGOpcode opc, 2387 const TCGArg args[TCG_MAX_OP_ARGS], 2388 const int const_args[TCG_MAX_OP_ARGS]) 2389{ 2390 TCGArg a0, a1, a2; 2391 int c, const_a2, vexop, rexw = 0; 2392 2393#if TCG_TARGET_REG_BITS == 64 2394# define OP_32_64(x) \ 2395 case glue(glue(INDEX_op_, x), _i64): \ 2396 rexw = P_REXW; /* FALLTHRU */ \ 2397 case glue(glue(INDEX_op_, x), _i32) 2398#else 2399# define OP_32_64(x) \ 2400 case glue(glue(INDEX_op_, x), _i32) 2401#endif 2402 2403 /* Hoist the loads of the most common arguments. */ 2404 a0 = args[0]; 2405 a1 = args[1]; 2406 a2 = args[2]; 2407 const_a2 = const_args[2]; 2408 2409 switch (opc) { 2410 case INDEX_op_goto_ptr: 2411 /* jmp to the given host address (could be epilogue) */ 2412 tcg_out_modrm(s, OPC_GRP5, EXT5_JMPN_Ev, a0); 2413 break; 2414 case INDEX_op_br: 2415 tcg_out_jxx(s, JCC_JMP, arg_label(a0), 0); 2416 break; 2417 OP_32_64(ld8u): 2418 /* Note that we can ignore REXW for the zero-extend to 64-bit. */ 2419 tcg_out_modrm_offset(s, OPC_MOVZBL, a0, a1, a2); 2420 break; 2421 OP_32_64(ld8s): 2422 tcg_out_modrm_offset(s, OPC_MOVSBL + rexw, a0, a1, a2); 2423 break; 2424 OP_32_64(ld16u): 2425 /* Note that we can ignore REXW for the zero-extend to 64-bit. */ 2426 tcg_out_modrm_offset(s, OPC_MOVZWL, a0, a1, a2); 2427 break; 2428 OP_32_64(ld16s): 2429 tcg_out_modrm_offset(s, OPC_MOVSWL + rexw, a0, a1, a2); 2430 break; 2431#if TCG_TARGET_REG_BITS == 64 2432 case INDEX_op_ld32u_i64: 2433#endif 2434 case INDEX_op_ld_i32: 2435 tcg_out_ld(s, TCG_TYPE_I32, a0, a1, a2); 2436 break; 2437 2438 OP_32_64(st8): 2439 if (const_args[0]) { 2440 tcg_out_modrm_offset(s, OPC_MOVB_EvIz, 0, a1, a2); 2441 tcg_out8(s, a0); 2442 } else { 2443 tcg_out_modrm_offset(s, OPC_MOVB_EvGv | P_REXB_R, a0, a1, a2); 2444 } 2445 break; 2446 OP_32_64(st16): 2447 if (const_args[0]) { 2448 tcg_out_modrm_offset(s, OPC_MOVL_EvIz | P_DATA16, 0, a1, a2); 2449 tcg_out16(s, a0); 2450 } else { 2451 tcg_out_modrm_offset(s, OPC_MOVL_EvGv | P_DATA16, a0, a1, a2); 2452 } 2453 break; 2454#if TCG_TARGET_REG_BITS == 64 2455 case INDEX_op_st32_i64: 2456#endif 2457 case INDEX_op_st_i32: 2458 if (const_args[0]) { 2459 tcg_out_modrm_offset(s, OPC_MOVL_EvIz, 0, a1, a2); 2460 tcg_out32(s, a0); 2461 } else { 2462 tcg_out_st(s, TCG_TYPE_I32, a0, a1, a2); 2463 } 2464 break; 2465 2466 OP_32_64(add): 2467 /* For 3-operand addition, use LEA. */ 2468 if (a0 != a1) { 2469 TCGArg c3 = 0; 2470 if (const_a2) { 2471 c3 = a2, a2 = -1; 2472 } else if (a0 == a2) { 2473 /* Watch out for dest = src + dest, since we've removed 2474 the matching constraint on the add. */ 2475 tgen_arithr(s, ARITH_ADD + rexw, a0, a1); 2476 break; 2477 } 2478 2479 tcg_out_modrm_sib_offset(s, OPC_LEA + rexw, a0, a1, a2, 0, c3); 2480 break; 2481 } 2482 c = ARITH_ADD; 2483 goto gen_arith; 2484 OP_32_64(sub): 2485 c = ARITH_SUB; 2486 goto gen_arith; 2487 OP_32_64(and): 2488 c = ARITH_AND; 2489 goto gen_arith; 2490 OP_32_64(or): 2491 c = ARITH_OR; 2492 goto gen_arith; 2493 OP_32_64(xor): 2494 c = ARITH_XOR; 2495 goto gen_arith; 2496 gen_arith: 2497 if (const_a2) { 2498 tgen_arithi(s, c + rexw, a0, a2, 0); 2499 } else { 2500 tgen_arithr(s, c + rexw, a0, a2); 2501 } 2502 break; 2503 2504 OP_32_64(andc): 2505 if (const_a2) { 2506 tcg_out_mov(s, rexw ? TCG_TYPE_I64 : TCG_TYPE_I32, a0, a1); 2507 tgen_arithi(s, ARITH_AND + rexw, a0, ~a2, 0); 2508 } else { 2509 tcg_out_vex_modrm(s, OPC_ANDN + rexw, a0, a2, a1); 2510 } 2511 break; 2512 2513 OP_32_64(mul): 2514 if (const_a2) { 2515 int32_t val; 2516 val = a2; 2517 if (val == (int8_t)val) { 2518 tcg_out_modrm(s, OPC_IMUL_GvEvIb + rexw, a0, a0); 2519 tcg_out8(s, val); 2520 } else { 2521 tcg_out_modrm(s, OPC_IMUL_GvEvIz + rexw, a0, a0); 2522 tcg_out32(s, val); 2523 } 2524 } else { 2525 tcg_out_modrm(s, OPC_IMUL_GvEv + rexw, a0, a2); 2526 } 2527 break; 2528 2529 OP_32_64(div2): 2530 tcg_out_modrm(s, OPC_GRP3_Ev + rexw, EXT3_IDIV, args[4]); 2531 break; 2532 OP_32_64(divu2): 2533 tcg_out_modrm(s, OPC_GRP3_Ev + rexw, EXT3_DIV, args[4]); 2534 break; 2535 2536 OP_32_64(shl): 2537 /* For small constant 3-operand shift, use LEA. */ 2538 if (const_a2 && a0 != a1 && (a2 - 1) < 3) { 2539 if (a2 - 1 == 0) { 2540 /* shl $1,a1,a0 -> lea (a1,a1),a0 */ 2541 tcg_out_modrm_sib_offset(s, OPC_LEA + rexw, a0, a1, a1, 0, 0); 2542 } else { 2543 /* shl $n,a1,a0 -> lea 0(,a1,n),a0 */ 2544 tcg_out_modrm_sib_offset(s, OPC_LEA + rexw, a0, -1, a1, a2, 0); 2545 } 2546 break; 2547 } 2548 c = SHIFT_SHL; 2549 vexop = OPC_SHLX; 2550 goto gen_shift_maybe_vex; 2551 OP_32_64(shr): 2552 c = SHIFT_SHR; 2553 vexop = OPC_SHRX; 2554 goto gen_shift_maybe_vex; 2555 OP_32_64(sar): 2556 c = SHIFT_SAR; 2557 vexop = OPC_SARX; 2558 goto gen_shift_maybe_vex; 2559 OP_32_64(rotl): 2560 c = SHIFT_ROL; 2561 goto gen_shift; 2562 OP_32_64(rotr): 2563 c = SHIFT_ROR; 2564 goto gen_shift; 2565 gen_shift_maybe_vex: 2566 if (have_bmi2) { 2567 if (!const_a2) { 2568 tcg_out_vex_modrm(s, vexop + rexw, a0, a2, a1); 2569 break; 2570 } 2571 tcg_out_mov(s, rexw ? TCG_TYPE_I64 : TCG_TYPE_I32, a0, a1); 2572 } 2573 /* FALLTHRU */ 2574 gen_shift: 2575 if (const_a2) { 2576 tcg_out_shifti(s, c + rexw, a0, a2); 2577 } else { 2578 tcg_out_modrm(s, OPC_SHIFT_cl + rexw, c, a0); 2579 } 2580 break; 2581 2582 OP_32_64(ctz): 2583 tcg_out_ctz(s, rexw, args[0], args[1], args[2], const_args[2]); 2584 break; 2585 OP_32_64(clz): 2586 tcg_out_clz(s, rexw, args[0], args[1], args[2], const_args[2]); 2587 break; 2588 OP_32_64(ctpop): 2589 tcg_out_modrm(s, OPC_POPCNT + rexw, a0, a1); 2590 break; 2591 2592 case INDEX_op_brcond_i32: 2593 tcg_out_brcond32(s, a2, a0, a1, const_args[1], arg_label(args[3]), 0); 2594 break; 2595 case INDEX_op_setcond_i32: 2596 tcg_out_setcond32(s, args[3], a0, a1, a2, const_a2); 2597 break; 2598 case INDEX_op_movcond_i32: 2599 tcg_out_movcond32(s, args[5], a0, a1, a2, const_a2, args[3]); 2600 break; 2601 2602 OP_32_64(bswap16): 2603 if (a2 & TCG_BSWAP_OS) { 2604 /* Output must be sign-extended. */ 2605 if (rexw) { 2606 tcg_out_bswap64(s, a0); 2607 tcg_out_shifti(s, SHIFT_SAR + rexw, a0, 48); 2608 } else { 2609 tcg_out_bswap32(s, a0); 2610 tcg_out_shifti(s, SHIFT_SAR, a0, 16); 2611 } 2612 } else if ((a2 & (TCG_BSWAP_IZ | TCG_BSWAP_OZ)) == TCG_BSWAP_OZ) { 2613 /* Output must be zero-extended, but input isn't. */ 2614 tcg_out_bswap32(s, a0); 2615 tcg_out_shifti(s, SHIFT_SHR, a0, 16); 2616 } else { 2617 tcg_out_rolw_8(s, a0); 2618 } 2619 break; 2620 OP_32_64(bswap32): 2621 tcg_out_bswap32(s, a0); 2622 if (rexw && (a2 & TCG_BSWAP_OS)) { 2623 tcg_out_ext32s(s, a0, a0); 2624 } 2625 break; 2626 2627 OP_32_64(neg): 2628 tcg_out_modrm(s, OPC_GRP3_Ev + rexw, EXT3_NEG, a0); 2629 break; 2630 OP_32_64(not): 2631 tcg_out_modrm(s, OPC_GRP3_Ev + rexw, EXT3_NOT, a0); 2632 break; 2633 2634 OP_32_64(ext8s): 2635 tcg_out_ext8s(s, a0, a1, rexw); 2636 break; 2637 OP_32_64(ext16s): 2638 tcg_out_ext16s(s, a0, a1, rexw); 2639 break; 2640 OP_32_64(ext8u): 2641 tcg_out_ext8u(s, a0, a1); 2642 break; 2643 OP_32_64(ext16u): 2644 tcg_out_ext16u(s, a0, a1); 2645 break; 2646 2647 case INDEX_op_qemu_ld_i32: 2648 tcg_out_qemu_ld(s, args, 0); 2649 break; 2650 case INDEX_op_qemu_ld_i64: 2651 tcg_out_qemu_ld(s, args, 1); 2652 break; 2653 case INDEX_op_qemu_st_i32: 2654 case INDEX_op_qemu_st8_i32: 2655 tcg_out_qemu_st(s, args, 0); 2656 break; 2657 case INDEX_op_qemu_st_i64: 2658 tcg_out_qemu_st(s, args, 1); 2659 break; 2660 2661 OP_32_64(mulu2): 2662 tcg_out_modrm(s, OPC_GRP3_Ev + rexw, EXT3_MUL, args[3]); 2663 break; 2664 OP_32_64(muls2): 2665 tcg_out_modrm(s, OPC_GRP3_Ev + rexw, EXT3_IMUL, args[3]); 2666 break; 2667 OP_32_64(add2): 2668 if (const_args[4]) { 2669 tgen_arithi(s, ARITH_ADD + rexw, a0, args[4], 1); 2670 } else { 2671 tgen_arithr(s, ARITH_ADD + rexw, a0, args[4]); 2672 } 2673 if (const_args[5]) { 2674 tgen_arithi(s, ARITH_ADC + rexw, a1, args[5], 1); 2675 } else { 2676 tgen_arithr(s, ARITH_ADC + rexw, a1, args[5]); 2677 } 2678 break; 2679 OP_32_64(sub2): 2680 if (const_args[4]) { 2681 tgen_arithi(s, ARITH_SUB + rexw, a0, args[4], 1); 2682 } else { 2683 tgen_arithr(s, ARITH_SUB + rexw, a0, args[4]); 2684 } 2685 if (const_args[5]) { 2686 tgen_arithi(s, ARITH_SBB + rexw, a1, args[5], 1); 2687 } else { 2688 tgen_arithr(s, ARITH_SBB + rexw, a1, args[5]); 2689 } 2690 break; 2691 2692#if TCG_TARGET_REG_BITS == 32 2693 case INDEX_op_brcond2_i32: 2694 tcg_out_brcond2(s, args, const_args, 0); 2695 break; 2696 case INDEX_op_setcond2_i32: 2697 tcg_out_setcond2(s, args, const_args); 2698 break; 2699#else /* TCG_TARGET_REG_BITS == 64 */ 2700 case INDEX_op_ld32s_i64: 2701 tcg_out_modrm_offset(s, OPC_MOVSLQ, a0, a1, a2); 2702 break; 2703 case INDEX_op_ld_i64: 2704 tcg_out_ld(s, TCG_TYPE_I64, a0, a1, a2); 2705 break; 2706 case INDEX_op_st_i64: 2707 if (const_args[0]) { 2708 tcg_out_modrm_offset(s, OPC_MOVL_EvIz | P_REXW, 0, a1, a2); 2709 tcg_out32(s, a0); 2710 } else { 2711 tcg_out_st(s, TCG_TYPE_I64, a0, a1, a2); 2712 } 2713 break; 2714 2715 case INDEX_op_brcond_i64: 2716 tcg_out_brcond64(s, a2, a0, a1, const_args[1], arg_label(args[3]), 0); 2717 break; 2718 case INDEX_op_setcond_i64: 2719 tcg_out_setcond64(s, args[3], a0, a1, a2, const_a2); 2720 break; 2721 case INDEX_op_movcond_i64: 2722 tcg_out_movcond64(s, args[5], a0, a1, a2, const_a2, args[3]); 2723 break; 2724 2725 case INDEX_op_bswap64_i64: 2726 tcg_out_bswap64(s, a0); 2727 break; 2728 case INDEX_op_extu_i32_i64: 2729 case INDEX_op_ext32u_i64: 2730 case INDEX_op_extrl_i64_i32: 2731 tcg_out_ext32u(s, a0, a1); 2732 break; 2733 case INDEX_op_ext_i32_i64: 2734 case INDEX_op_ext32s_i64: 2735 tcg_out_ext32s(s, a0, a1); 2736 break; 2737 case INDEX_op_extrh_i64_i32: 2738 tcg_out_shifti(s, SHIFT_SHR + P_REXW, a0, 32); 2739 break; 2740#endif 2741 2742 OP_32_64(deposit): 2743 if (args[3] == 0 && args[4] == 8) { 2744 /* load bits 0..7 */ 2745 tcg_out_modrm(s, OPC_MOVB_EvGv | P_REXB_R | P_REXB_RM, a2, a0); 2746 } else if (args[3] == 8 && args[4] == 8) { 2747 /* load bits 8..15 */ 2748 tcg_out_modrm(s, OPC_MOVB_EvGv, a2, a0 + 4); 2749 } else if (args[3] == 0 && args[4] == 16) { 2750 /* load bits 0..15 */ 2751 tcg_out_modrm(s, OPC_MOVL_EvGv | P_DATA16, a2, a0); 2752 } else { 2753 tcg_abort(); 2754 } 2755 break; 2756 2757 case INDEX_op_extract_i64: 2758 if (a2 + args[3] == 32) { 2759 /* This is a 32-bit zero-extending right shift. */ 2760 tcg_out_mov(s, TCG_TYPE_I32, a0, a1); 2761 tcg_out_shifti(s, SHIFT_SHR, a0, a2); 2762 break; 2763 } 2764 /* FALLTHRU */ 2765 case INDEX_op_extract_i32: 2766 /* On the off-chance that we can use the high-byte registers. 2767 Otherwise we emit the same ext16 + shift pattern that we 2768 would have gotten from the normal tcg-op.c expansion. */ 2769 tcg_debug_assert(a2 == 8 && args[3] == 8); 2770 if (a1 < 4 && a0 < 8) { 2771 tcg_out_modrm(s, OPC_MOVZBL, a0, a1 + 4); 2772 } else { 2773 tcg_out_ext16u(s, a0, a1); 2774 tcg_out_shifti(s, SHIFT_SHR, a0, 8); 2775 } 2776 break; 2777 2778 case INDEX_op_sextract_i32: 2779 /* We don't implement sextract_i64, as we cannot sign-extend to 2780 64-bits without using the REX prefix that explicitly excludes 2781 access to the high-byte registers. */ 2782 tcg_debug_assert(a2 == 8 && args[3] == 8); 2783 if (a1 < 4 && a0 < 8) { 2784 tcg_out_modrm(s, OPC_MOVSBL, a0, a1 + 4); 2785 } else { 2786 tcg_out_ext16s(s, a0, a1, 0); 2787 tcg_out_shifti(s, SHIFT_SAR, a0, 8); 2788 } 2789 break; 2790 2791 OP_32_64(extract2): 2792 /* Note that SHRD outputs to the r/m operand. */ 2793 tcg_out_modrm(s, OPC_SHRD_Ib + rexw, a2, a0); 2794 tcg_out8(s, args[3]); 2795 break; 2796 2797 case INDEX_op_mb: 2798 tcg_out_mb(s, a0); 2799 break; 2800 case INDEX_op_mov_i32: /* Always emitted via tcg_out_mov. */ 2801 case INDEX_op_mov_i64: 2802 case INDEX_op_call: /* Always emitted via tcg_out_call. */ 2803 case INDEX_op_exit_tb: /* Always emitted via tcg_out_exit_tb. */ 2804 case INDEX_op_goto_tb: /* Always emitted via tcg_out_goto_tb. */ 2805 default: 2806 tcg_abort(); 2807 } 2808 2809#undef OP_32_64 2810} 2811 2812static void tcg_out_vec_op(TCGContext *s, TCGOpcode opc, 2813 unsigned vecl, unsigned vece, 2814 const TCGArg args[TCG_MAX_OP_ARGS], 2815 const int const_args[TCG_MAX_OP_ARGS]) 2816{ 2817 static int const add_insn[4] = { 2818 OPC_PADDB, OPC_PADDW, OPC_PADDD, OPC_PADDQ 2819 }; 2820 static int const ssadd_insn[4] = { 2821 OPC_PADDSB, OPC_PADDSW, OPC_UD2, OPC_UD2 2822 }; 2823 static int const usadd_insn[4] = { 2824 OPC_PADDUB, OPC_PADDUW, OPC_UD2, OPC_UD2 2825 }; 2826 static int const sub_insn[4] = { 2827 OPC_PSUBB, OPC_PSUBW, OPC_PSUBD, OPC_PSUBQ 2828 }; 2829 static int const sssub_insn[4] = { 2830 OPC_PSUBSB, OPC_PSUBSW, OPC_UD2, OPC_UD2 2831 }; 2832 static int const ussub_insn[4] = { 2833 OPC_PSUBUB, OPC_PSUBUW, OPC_UD2, OPC_UD2 2834 }; 2835 static int const mul_insn[4] = { 2836 OPC_UD2, OPC_PMULLW, OPC_PMULLD, OPC_VPMULLQ 2837 }; 2838 static int const shift_imm_insn[4] = { 2839 OPC_UD2, OPC_PSHIFTW_Ib, OPC_PSHIFTD_Ib, OPC_PSHIFTQ_Ib 2840 }; 2841 static int const cmpeq_insn[4] = { 2842 OPC_PCMPEQB, OPC_PCMPEQW, OPC_PCMPEQD, OPC_PCMPEQQ 2843 }; 2844 static int const cmpgt_insn[4] = { 2845 OPC_PCMPGTB, OPC_PCMPGTW, OPC_PCMPGTD, OPC_PCMPGTQ 2846 }; 2847 static int const punpckl_insn[4] = { 2848 OPC_PUNPCKLBW, OPC_PUNPCKLWD, OPC_PUNPCKLDQ, OPC_PUNPCKLQDQ 2849 }; 2850 static int const punpckh_insn[4] = { 2851 OPC_PUNPCKHBW, OPC_PUNPCKHWD, OPC_PUNPCKHDQ, OPC_PUNPCKHQDQ 2852 }; 2853 static int const packss_insn[4] = { 2854 OPC_PACKSSWB, OPC_PACKSSDW, OPC_UD2, OPC_UD2 2855 }; 2856 static int const packus_insn[4] = { 2857 OPC_PACKUSWB, OPC_PACKUSDW, OPC_UD2, OPC_UD2 2858 }; 2859 static int const smin_insn[4] = { 2860 OPC_PMINSB, OPC_PMINSW, OPC_PMINSD, OPC_VPMINSQ 2861 }; 2862 static int const smax_insn[4] = { 2863 OPC_PMAXSB, OPC_PMAXSW, OPC_PMAXSD, OPC_VPMAXSQ 2864 }; 2865 static int const umin_insn[4] = { 2866 OPC_PMINUB, OPC_PMINUW, OPC_PMINUD, OPC_VPMINUQ 2867 }; 2868 static int const umax_insn[4] = { 2869 OPC_PMAXUB, OPC_PMAXUW, OPC_PMAXUD, OPC_VPMAXUQ 2870 }; 2871 static int const rotlv_insn[4] = { 2872 OPC_UD2, OPC_UD2, OPC_VPROLVD, OPC_VPROLVQ 2873 }; 2874 static int const rotrv_insn[4] = { 2875 OPC_UD2, OPC_UD2, OPC_VPRORVD, OPC_VPRORVQ 2876 }; 2877 static int const shlv_insn[4] = { 2878 OPC_UD2, OPC_VPSLLVW, OPC_VPSLLVD, OPC_VPSLLVQ 2879 }; 2880 static int const shrv_insn[4] = { 2881 OPC_UD2, OPC_VPSRLVW, OPC_VPSRLVD, OPC_VPSRLVQ 2882 }; 2883 static int const sarv_insn[4] = { 2884 OPC_UD2, OPC_VPSRAVW, OPC_VPSRAVD, OPC_VPSRAVQ 2885 }; 2886 static int const shls_insn[4] = { 2887 OPC_UD2, OPC_PSLLW, OPC_PSLLD, OPC_PSLLQ 2888 }; 2889 static int const shrs_insn[4] = { 2890 OPC_UD2, OPC_PSRLW, OPC_PSRLD, OPC_PSRLQ 2891 }; 2892 static int const sars_insn[4] = { 2893 OPC_UD2, OPC_PSRAW, OPC_PSRAD, OPC_VPSRAQ 2894 }; 2895 static int const vpshldi_insn[4] = { 2896 OPC_UD2, OPC_VPSHLDW, OPC_VPSHLDD, OPC_VPSHLDQ 2897 }; 2898 static int const vpshldv_insn[4] = { 2899 OPC_UD2, OPC_VPSHLDVW, OPC_VPSHLDVD, OPC_VPSHLDVQ 2900 }; 2901 static int const vpshrdv_insn[4] = { 2902 OPC_UD2, OPC_VPSHRDVW, OPC_VPSHRDVD, OPC_VPSHRDVQ 2903 }; 2904 static int const abs_insn[4] = { 2905 OPC_PABSB, OPC_PABSW, OPC_PABSD, OPC_VPABSQ 2906 }; 2907 2908 TCGType type = vecl + TCG_TYPE_V64; 2909 int insn, sub; 2910 TCGArg a0, a1, a2, a3; 2911 2912 a0 = args[0]; 2913 a1 = args[1]; 2914 a2 = args[2]; 2915 2916 switch (opc) { 2917 case INDEX_op_add_vec: 2918 insn = add_insn[vece]; 2919 goto gen_simd; 2920 case INDEX_op_ssadd_vec: 2921 insn = ssadd_insn[vece]; 2922 goto gen_simd; 2923 case INDEX_op_usadd_vec: 2924 insn = usadd_insn[vece]; 2925 goto gen_simd; 2926 case INDEX_op_sub_vec: 2927 insn = sub_insn[vece]; 2928 goto gen_simd; 2929 case INDEX_op_sssub_vec: 2930 insn = sssub_insn[vece]; 2931 goto gen_simd; 2932 case INDEX_op_ussub_vec: 2933 insn = ussub_insn[vece]; 2934 goto gen_simd; 2935 case INDEX_op_mul_vec: 2936 insn = mul_insn[vece]; 2937 goto gen_simd; 2938 case INDEX_op_and_vec: 2939 insn = OPC_PAND; 2940 goto gen_simd; 2941 case INDEX_op_or_vec: 2942 insn = OPC_POR; 2943 goto gen_simd; 2944 case INDEX_op_xor_vec: 2945 insn = OPC_PXOR; 2946 goto gen_simd; 2947 case INDEX_op_smin_vec: 2948 insn = smin_insn[vece]; 2949 goto gen_simd; 2950 case INDEX_op_umin_vec: 2951 insn = umin_insn[vece]; 2952 goto gen_simd; 2953 case INDEX_op_smax_vec: 2954 insn = smax_insn[vece]; 2955 goto gen_simd; 2956 case INDEX_op_umax_vec: 2957 insn = umax_insn[vece]; 2958 goto gen_simd; 2959 case INDEX_op_shlv_vec: 2960 insn = shlv_insn[vece]; 2961 goto gen_simd; 2962 case INDEX_op_shrv_vec: 2963 insn = shrv_insn[vece]; 2964 goto gen_simd; 2965 case INDEX_op_sarv_vec: 2966 insn = sarv_insn[vece]; 2967 goto gen_simd; 2968 case INDEX_op_rotlv_vec: 2969 insn = rotlv_insn[vece]; 2970 goto gen_simd; 2971 case INDEX_op_rotrv_vec: 2972 insn = rotrv_insn[vece]; 2973 goto gen_simd; 2974 case INDEX_op_shls_vec: 2975 insn = shls_insn[vece]; 2976 goto gen_simd; 2977 case INDEX_op_shrs_vec: 2978 insn = shrs_insn[vece]; 2979 goto gen_simd; 2980 case INDEX_op_sars_vec: 2981 insn = sars_insn[vece]; 2982 goto gen_simd; 2983 case INDEX_op_x86_punpckl_vec: 2984 insn = punpckl_insn[vece]; 2985 goto gen_simd; 2986 case INDEX_op_x86_punpckh_vec: 2987 insn = punpckh_insn[vece]; 2988 goto gen_simd; 2989 case INDEX_op_x86_packss_vec: 2990 insn = packss_insn[vece]; 2991 goto gen_simd; 2992 case INDEX_op_x86_packus_vec: 2993 insn = packus_insn[vece]; 2994 goto gen_simd; 2995 case INDEX_op_x86_vpshldv_vec: 2996 insn = vpshldv_insn[vece]; 2997 a1 = a2; 2998 a2 = args[3]; 2999 goto gen_simd; 3000 case INDEX_op_x86_vpshrdv_vec: 3001 insn = vpshrdv_insn[vece]; 3002 a1 = a2; 3003 a2 = args[3]; 3004 goto gen_simd; 3005#if TCG_TARGET_REG_BITS == 32 3006 case INDEX_op_dup2_vec: 3007 /* First merge the two 32-bit inputs to a single 64-bit element. */ 3008 tcg_out_vex_modrm(s, OPC_PUNPCKLDQ, a0, a1, a2); 3009 /* Then replicate the 64-bit elements across the rest of the vector. */ 3010 if (type != TCG_TYPE_V64) { 3011 tcg_out_dup_vec(s, type, MO_64, a0, a0); 3012 } 3013 break; 3014#endif 3015 case INDEX_op_abs_vec: 3016 insn = abs_insn[vece]; 3017 a2 = a1; 3018 a1 = 0; 3019 goto gen_simd; 3020 gen_simd: 3021 tcg_debug_assert(insn != OPC_UD2); 3022 if (type == TCG_TYPE_V256) { 3023 insn |= P_VEXL; 3024 } 3025 tcg_out_vex_modrm(s, insn, a0, a1, a2); 3026 break; 3027 3028 case INDEX_op_cmp_vec: 3029 sub = args[3]; 3030 if (sub == TCG_COND_EQ) { 3031 insn = cmpeq_insn[vece]; 3032 } else if (sub == TCG_COND_GT) { 3033 insn = cmpgt_insn[vece]; 3034 } else { 3035 g_assert_not_reached(); 3036 } 3037 goto gen_simd; 3038 3039 case INDEX_op_andc_vec: 3040 insn = OPC_PANDN; 3041 if (type == TCG_TYPE_V256) { 3042 insn |= P_VEXL; 3043 } 3044 tcg_out_vex_modrm(s, insn, a0, a2, a1); 3045 break; 3046 3047 case INDEX_op_shli_vec: 3048 insn = shift_imm_insn[vece]; 3049 sub = 6; 3050 goto gen_shift; 3051 case INDEX_op_shri_vec: 3052 insn = shift_imm_insn[vece]; 3053 sub = 2; 3054 goto gen_shift; 3055 case INDEX_op_sari_vec: 3056 if (vece == MO_64) { 3057 insn = OPC_PSHIFTD_Ib | P_VEXW | P_EVEX; 3058 } else { 3059 insn = shift_imm_insn[vece]; 3060 } 3061 sub = 4; 3062 goto gen_shift; 3063 case INDEX_op_rotli_vec: 3064 insn = OPC_PSHIFTD_Ib | P_EVEX; /* VPROL[DQ] */ 3065 if (vece == MO_64) { 3066 insn |= P_VEXW; 3067 } 3068 sub = 1; 3069 goto gen_shift; 3070 gen_shift: 3071 tcg_debug_assert(vece != MO_8); 3072 if (type == TCG_TYPE_V256) { 3073 insn |= P_VEXL; 3074 } 3075 tcg_out_vex_modrm(s, insn, sub, a0, a1); 3076 tcg_out8(s, a2); 3077 break; 3078 3079 case INDEX_op_ld_vec: 3080 tcg_out_ld(s, type, a0, a1, a2); 3081 break; 3082 case INDEX_op_st_vec: 3083 tcg_out_st(s, type, a0, a1, a2); 3084 break; 3085 case INDEX_op_dupm_vec: 3086 tcg_out_dupm_vec(s, type, vece, a0, a1, a2); 3087 break; 3088 3089 case INDEX_op_x86_shufps_vec: 3090 insn = OPC_SHUFPS; 3091 sub = args[3]; 3092 goto gen_simd_imm8; 3093 case INDEX_op_x86_blend_vec: 3094 if (vece == MO_16) { 3095 insn = OPC_PBLENDW; 3096 } else if (vece == MO_32) { 3097 insn = (have_avx2 ? OPC_VPBLENDD : OPC_BLENDPS); 3098 } else { 3099 g_assert_not_reached(); 3100 } 3101 sub = args[3]; 3102 goto gen_simd_imm8; 3103 case INDEX_op_x86_vperm2i128_vec: 3104 insn = OPC_VPERM2I128; 3105 sub = args[3]; 3106 goto gen_simd_imm8; 3107 case INDEX_op_x86_vpshldi_vec: 3108 insn = vpshldi_insn[vece]; 3109 sub = args[3]; 3110 goto gen_simd_imm8; 3111 3112 case INDEX_op_not_vec: 3113 insn = OPC_VPTERNLOGQ; 3114 a2 = a1; 3115 sub = 0x33; /* !B */ 3116 goto gen_simd_imm8; 3117 case INDEX_op_nor_vec: 3118 insn = OPC_VPTERNLOGQ; 3119 sub = 0x11; /* norCB */ 3120 goto gen_simd_imm8; 3121 case INDEX_op_nand_vec: 3122 insn = OPC_VPTERNLOGQ; 3123 sub = 0x77; /* nandCB */ 3124 goto gen_simd_imm8; 3125 case INDEX_op_eqv_vec: 3126 insn = OPC_VPTERNLOGQ; 3127 sub = 0x99; /* xnorCB */ 3128 goto gen_simd_imm8; 3129 case INDEX_op_orc_vec: 3130 insn = OPC_VPTERNLOGQ; 3131 sub = 0xdd; /* orB!C */ 3132 goto gen_simd_imm8; 3133 3134 case INDEX_op_bitsel_vec: 3135 insn = OPC_VPTERNLOGQ; 3136 a3 = args[3]; 3137 if (a0 == a1) { 3138 a1 = a2; 3139 a2 = a3; 3140 sub = 0xca; /* A?B:C */ 3141 } else if (a0 == a2) { 3142 a2 = a3; 3143 sub = 0xe2; /* B?A:C */ 3144 } else { 3145 tcg_out_mov(s, type, a0, a3); 3146 sub = 0xb8; /* B?C:A */ 3147 } 3148 goto gen_simd_imm8; 3149 3150 gen_simd_imm8: 3151 tcg_debug_assert(insn != OPC_UD2); 3152 if (type == TCG_TYPE_V256) { 3153 insn |= P_VEXL; 3154 } 3155 tcg_out_vex_modrm(s, insn, a0, a1, a2); 3156 tcg_out8(s, sub); 3157 break; 3158 3159 case INDEX_op_x86_vpblendvb_vec: 3160 insn = OPC_VPBLENDVB; 3161 if (type == TCG_TYPE_V256) { 3162 insn |= P_VEXL; 3163 } 3164 tcg_out_vex_modrm(s, insn, a0, a1, a2); 3165 tcg_out8(s, args[3] << 4); 3166 break; 3167 3168 case INDEX_op_x86_psrldq_vec: 3169 tcg_out_vex_modrm(s, OPC_GRP14, 3, a0, a1); 3170 tcg_out8(s, a2); 3171 break; 3172 3173 case INDEX_op_mov_vec: /* Always emitted via tcg_out_mov. */ 3174 case INDEX_op_dup_vec: /* Always emitted via tcg_out_dup_vec. */ 3175 default: 3176 g_assert_not_reached(); 3177 } 3178} 3179 3180static TCGConstraintSetIndex tcg_target_op_def(TCGOpcode op) 3181{ 3182 switch (op) { 3183 case INDEX_op_goto_ptr: 3184 return C_O0_I1(r); 3185 3186 case INDEX_op_ld8u_i32: 3187 case INDEX_op_ld8u_i64: 3188 case INDEX_op_ld8s_i32: 3189 case INDEX_op_ld8s_i64: 3190 case INDEX_op_ld16u_i32: 3191 case INDEX_op_ld16u_i64: 3192 case INDEX_op_ld16s_i32: 3193 case INDEX_op_ld16s_i64: 3194 case INDEX_op_ld_i32: 3195 case INDEX_op_ld32u_i64: 3196 case INDEX_op_ld32s_i64: 3197 case INDEX_op_ld_i64: 3198 return C_O1_I1(r, r); 3199 3200 case INDEX_op_st8_i32: 3201 case INDEX_op_st8_i64: 3202 return C_O0_I2(qi, r); 3203 3204 case INDEX_op_st16_i32: 3205 case INDEX_op_st16_i64: 3206 case INDEX_op_st_i32: 3207 case INDEX_op_st32_i64: 3208 return C_O0_I2(ri, r); 3209 3210 case INDEX_op_st_i64: 3211 return C_O0_I2(re, r); 3212 3213 case INDEX_op_add_i32: 3214 case INDEX_op_add_i64: 3215 return C_O1_I2(r, r, re); 3216 3217 case INDEX_op_sub_i32: 3218 case INDEX_op_sub_i64: 3219 case INDEX_op_mul_i32: 3220 case INDEX_op_mul_i64: 3221 case INDEX_op_or_i32: 3222 case INDEX_op_or_i64: 3223 case INDEX_op_xor_i32: 3224 case INDEX_op_xor_i64: 3225 return C_O1_I2(r, 0, re); 3226 3227 case INDEX_op_and_i32: 3228 case INDEX_op_and_i64: 3229 return C_O1_I2(r, 0, reZ); 3230 3231 case INDEX_op_andc_i32: 3232 case INDEX_op_andc_i64: 3233 return C_O1_I2(r, r, rI); 3234 3235 case INDEX_op_shl_i32: 3236 case INDEX_op_shl_i64: 3237 case INDEX_op_shr_i32: 3238 case INDEX_op_shr_i64: 3239 case INDEX_op_sar_i32: 3240 case INDEX_op_sar_i64: 3241 return have_bmi2 ? C_O1_I2(r, r, ri) : C_O1_I2(r, 0, ci); 3242 3243 case INDEX_op_rotl_i32: 3244 case INDEX_op_rotl_i64: 3245 case INDEX_op_rotr_i32: 3246 case INDEX_op_rotr_i64: 3247 return C_O1_I2(r, 0, ci); 3248 3249 case INDEX_op_brcond_i32: 3250 case INDEX_op_brcond_i64: 3251 return C_O0_I2(r, re); 3252 3253 case INDEX_op_bswap16_i32: 3254 case INDEX_op_bswap16_i64: 3255 case INDEX_op_bswap32_i32: 3256 case INDEX_op_bswap32_i64: 3257 case INDEX_op_bswap64_i64: 3258 case INDEX_op_neg_i32: 3259 case INDEX_op_neg_i64: 3260 case INDEX_op_not_i32: 3261 case INDEX_op_not_i64: 3262 case INDEX_op_extrh_i64_i32: 3263 return C_O1_I1(r, 0); 3264 3265 case INDEX_op_ext8s_i32: 3266 case INDEX_op_ext8s_i64: 3267 case INDEX_op_ext8u_i32: 3268 case INDEX_op_ext8u_i64: 3269 return C_O1_I1(r, q); 3270 3271 case INDEX_op_ext16s_i32: 3272 case INDEX_op_ext16s_i64: 3273 case INDEX_op_ext16u_i32: 3274 case INDEX_op_ext16u_i64: 3275 case INDEX_op_ext32s_i64: 3276 case INDEX_op_ext32u_i64: 3277 case INDEX_op_ext_i32_i64: 3278 case INDEX_op_extu_i32_i64: 3279 case INDEX_op_extrl_i64_i32: 3280 case INDEX_op_extract_i32: 3281 case INDEX_op_extract_i64: 3282 case INDEX_op_sextract_i32: 3283 case INDEX_op_ctpop_i32: 3284 case INDEX_op_ctpop_i64: 3285 return C_O1_I1(r, r); 3286 3287 case INDEX_op_extract2_i32: 3288 case INDEX_op_extract2_i64: 3289 return C_O1_I2(r, 0, r); 3290 3291 case INDEX_op_deposit_i32: 3292 case INDEX_op_deposit_i64: 3293 return C_O1_I2(Q, 0, Q); 3294 3295 case INDEX_op_setcond_i32: 3296 case INDEX_op_setcond_i64: 3297 return C_O1_I2(q, r, re); 3298 3299 case INDEX_op_movcond_i32: 3300 case INDEX_op_movcond_i64: 3301 return C_O1_I4(r, r, re, r, 0); 3302 3303 case INDEX_op_div2_i32: 3304 case INDEX_op_div2_i64: 3305 case INDEX_op_divu2_i32: 3306 case INDEX_op_divu2_i64: 3307 return C_O2_I3(a, d, 0, 1, r); 3308 3309 case INDEX_op_mulu2_i32: 3310 case INDEX_op_mulu2_i64: 3311 case INDEX_op_muls2_i32: 3312 case INDEX_op_muls2_i64: 3313 return C_O2_I2(a, d, a, r); 3314 3315 case INDEX_op_add2_i32: 3316 case INDEX_op_add2_i64: 3317 case INDEX_op_sub2_i32: 3318 case INDEX_op_sub2_i64: 3319 return C_O2_I4(r, r, 0, 1, re, re); 3320 3321 case INDEX_op_ctz_i32: 3322 case INDEX_op_ctz_i64: 3323 return have_bmi1 ? C_N1_I2(r, r, rW) : C_N1_I2(r, r, r); 3324 3325 case INDEX_op_clz_i32: 3326 case INDEX_op_clz_i64: 3327 return have_lzcnt ? C_N1_I2(r, r, rW) : C_N1_I2(r, r, r); 3328 3329 case INDEX_op_qemu_ld_i32: 3330 return (TARGET_LONG_BITS <= TCG_TARGET_REG_BITS 3331 ? C_O1_I1(r, L) : C_O1_I2(r, L, L)); 3332 3333 case INDEX_op_qemu_st_i32: 3334 return (TARGET_LONG_BITS <= TCG_TARGET_REG_BITS 3335 ? C_O0_I2(L, L) : C_O0_I3(L, L, L)); 3336 case INDEX_op_qemu_st8_i32: 3337 return (TARGET_LONG_BITS <= TCG_TARGET_REG_BITS 3338 ? C_O0_I2(s, L) : C_O0_I3(s, L, L)); 3339 3340 case INDEX_op_qemu_ld_i64: 3341 return (TCG_TARGET_REG_BITS == 64 ? C_O1_I1(r, L) 3342 : TARGET_LONG_BITS <= TCG_TARGET_REG_BITS ? C_O2_I1(r, r, L) 3343 : C_O2_I2(r, r, L, L)); 3344 3345 case INDEX_op_qemu_st_i64: 3346 return (TCG_TARGET_REG_BITS == 64 ? C_O0_I2(L, L) 3347 : TARGET_LONG_BITS <= TCG_TARGET_REG_BITS ? C_O0_I3(L, L, L) 3348 : C_O0_I4(L, L, L, L)); 3349 3350 case INDEX_op_brcond2_i32: 3351 return C_O0_I4(r, r, ri, ri); 3352 3353 case INDEX_op_setcond2_i32: 3354 return C_O1_I4(r, r, r, ri, ri); 3355 3356 case INDEX_op_ld_vec: 3357 case INDEX_op_dupm_vec: 3358 return C_O1_I1(x, r); 3359 3360 case INDEX_op_st_vec: 3361 return C_O0_I2(x, r); 3362 3363 case INDEX_op_add_vec: 3364 case INDEX_op_sub_vec: 3365 case INDEX_op_mul_vec: 3366 case INDEX_op_and_vec: 3367 case INDEX_op_or_vec: 3368 case INDEX_op_xor_vec: 3369 case INDEX_op_andc_vec: 3370 case INDEX_op_orc_vec: 3371 case INDEX_op_nand_vec: 3372 case INDEX_op_nor_vec: 3373 case INDEX_op_eqv_vec: 3374 case INDEX_op_ssadd_vec: 3375 case INDEX_op_usadd_vec: 3376 case INDEX_op_sssub_vec: 3377 case INDEX_op_ussub_vec: 3378 case INDEX_op_smin_vec: 3379 case INDEX_op_umin_vec: 3380 case INDEX_op_smax_vec: 3381 case INDEX_op_umax_vec: 3382 case INDEX_op_shlv_vec: 3383 case INDEX_op_shrv_vec: 3384 case INDEX_op_sarv_vec: 3385 case INDEX_op_rotlv_vec: 3386 case INDEX_op_rotrv_vec: 3387 case INDEX_op_shls_vec: 3388 case INDEX_op_shrs_vec: 3389 case INDEX_op_sars_vec: 3390 case INDEX_op_cmp_vec: 3391 case INDEX_op_x86_shufps_vec: 3392 case INDEX_op_x86_blend_vec: 3393 case INDEX_op_x86_packss_vec: 3394 case INDEX_op_x86_packus_vec: 3395 case INDEX_op_x86_vperm2i128_vec: 3396 case INDEX_op_x86_punpckl_vec: 3397 case INDEX_op_x86_punpckh_vec: 3398 case INDEX_op_x86_vpshldi_vec: 3399#if TCG_TARGET_REG_BITS == 32 3400 case INDEX_op_dup2_vec: 3401#endif 3402 return C_O1_I2(x, x, x); 3403 3404 case INDEX_op_abs_vec: 3405 case INDEX_op_dup_vec: 3406 case INDEX_op_not_vec: 3407 case INDEX_op_shli_vec: 3408 case INDEX_op_shri_vec: 3409 case INDEX_op_sari_vec: 3410 case INDEX_op_rotli_vec: 3411 case INDEX_op_x86_psrldq_vec: 3412 return C_O1_I1(x, x); 3413 3414 case INDEX_op_x86_vpshldv_vec: 3415 case INDEX_op_x86_vpshrdv_vec: 3416 return C_O1_I3(x, 0, x, x); 3417 3418 case INDEX_op_bitsel_vec: 3419 case INDEX_op_x86_vpblendvb_vec: 3420 return C_O1_I3(x, x, x, x); 3421 3422 default: 3423 g_assert_not_reached(); 3424 } 3425} 3426 3427int tcg_can_emit_vec_op(TCGOpcode opc, TCGType type, unsigned vece) 3428{ 3429 switch (opc) { 3430 case INDEX_op_add_vec: 3431 case INDEX_op_sub_vec: 3432 case INDEX_op_and_vec: 3433 case INDEX_op_or_vec: 3434 case INDEX_op_xor_vec: 3435 case INDEX_op_andc_vec: 3436 case INDEX_op_orc_vec: 3437 case INDEX_op_nand_vec: 3438 case INDEX_op_nor_vec: 3439 case INDEX_op_eqv_vec: 3440 case INDEX_op_not_vec: 3441 case INDEX_op_bitsel_vec: 3442 return 1; 3443 case INDEX_op_cmp_vec: 3444 case INDEX_op_cmpsel_vec: 3445 return -1; 3446 3447 case INDEX_op_rotli_vec: 3448 return have_avx512vl && vece >= MO_32 ? 1 : -1; 3449 3450 case INDEX_op_shli_vec: 3451 case INDEX_op_shri_vec: 3452 /* We must expand the operation for MO_8. */ 3453 return vece == MO_8 ? -1 : 1; 3454 3455 case INDEX_op_sari_vec: 3456 switch (vece) { 3457 case MO_8: 3458 return -1; 3459 case MO_16: 3460 case MO_32: 3461 return 1; 3462 case MO_64: 3463 if (have_avx512vl) { 3464 return 1; 3465 } 3466 /* 3467 * We can emulate this for MO_64, but it does not pay off 3468 * unless we're producing at least 4 values. 3469 */ 3470 return type >= TCG_TYPE_V256 ? -1 : 0; 3471 } 3472 return 0; 3473 3474 case INDEX_op_shls_vec: 3475 case INDEX_op_shrs_vec: 3476 return vece >= MO_16; 3477 case INDEX_op_sars_vec: 3478 switch (vece) { 3479 case MO_16: 3480 case MO_32: 3481 return 1; 3482 case MO_64: 3483 return have_avx512vl; 3484 } 3485 return 0; 3486 case INDEX_op_rotls_vec: 3487 return vece >= MO_16 ? -1 : 0; 3488 3489 case INDEX_op_shlv_vec: 3490 case INDEX_op_shrv_vec: 3491 switch (vece) { 3492 case MO_16: 3493 return have_avx512bw; 3494 case MO_32: 3495 case MO_64: 3496 return have_avx2; 3497 } 3498 return 0; 3499 case INDEX_op_sarv_vec: 3500 switch (vece) { 3501 case MO_16: 3502 return have_avx512bw; 3503 case MO_32: 3504 return have_avx2; 3505 case MO_64: 3506 return have_avx512vl; 3507 } 3508 return 0; 3509 case INDEX_op_rotlv_vec: 3510 case INDEX_op_rotrv_vec: 3511 switch (vece) { 3512 case MO_16: 3513 return have_avx512vbmi2 ? -1 : 0; 3514 case MO_32: 3515 case MO_64: 3516 return have_avx512vl ? 1 : have_avx2 ? -1 : 0; 3517 } 3518 return 0; 3519 3520 case INDEX_op_mul_vec: 3521 switch (vece) { 3522 case MO_8: 3523 return -1; 3524 case MO_64: 3525 return have_avx512dq; 3526 } 3527 return 1; 3528 3529 case INDEX_op_ssadd_vec: 3530 case INDEX_op_usadd_vec: 3531 case INDEX_op_sssub_vec: 3532 case INDEX_op_ussub_vec: 3533 return vece <= MO_16; 3534 case INDEX_op_smin_vec: 3535 case INDEX_op_smax_vec: 3536 case INDEX_op_umin_vec: 3537 case INDEX_op_umax_vec: 3538 case INDEX_op_abs_vec: 3539 return vece <= MO_32 || have_avx512vl; 3540 3541 default: 3542 return 0; 3543 } 3544} 3545 3546static void expand_vec_shi(TCGType type, unsigned vece, TCGOpcode opc, 3547 TCGv_vec v0, TCGv_vec v1, TCGArg imm) 3548{ 3549 TCGv_vec t1, t2; 3550 3551 tcg_debug_assert(vece == MO_8); 3552 3553 t1 = tcg_temp_new_vec(type); 3554 t2 = tcg_temp_new_vec(type); 3555 3556 /* 3557 * Unpack to W, shift, and repack. Tricky bits: 3558 * (1) Use punpck*bw x,x to produce DDCCBBAA, 3559 * i.e. duplicate in other half of the 16-bit lane. 3560 * (2) For right-shift, add 8 so that the high half of the lane 3561 * becomes zero. For left-shift, and left-rotate, we must 3562 * shift up and down again. 3563 * (3) Step 2 leaves high half zero such that PACKUSWB 3564 * (pack with unsigned saturation) does not modify 3565 * the quantity. 3566 */ 3567 vec_gen_3(INDEX_op_x86_punpckl_vec, type, MO_8, 3568 tcgv_vec_arg(t1), tcgv_vec_arg(v1), tcgv_vec_arg(v1)); 3569 vec_gen_3(INDEX_op_x86_punpckh_vec, type, MO_8, 3570 tcgv_vec_arg(t2), tcgv_vec_arg(v1), tcgv_vec_arg(v1)); 3571 3572 if (opc != INDEX_op_rotli_vec) { 3573 imm += 8; 3574 } 3575 if (opc == INDEX_op_shri_vec) { 3576 tcg_gen_shri_vec(MO_16, t1, t1, imm); 3577 tcg_gen_shri_vec(MO_16, t2, t2, imm); 3578 } else { 3579 tcg_gen_shli_vec(MO_16, t1, t1, imm); 3580 tcg_gen_shli_vec(MO_16, t2, t2, imm); 3581 tcg_gen_shri_vec(MO_16, t1, t1, 8); 3582 tcg_gen_shri_vec(MO_16, t2, t2, 8); 3583 } 3584 3585 vec_gen_3(INDEX_op_x86_packus_vec, type, MO_8, 3586 tcgv_vec_arg(v0), tcgv_vec_arg(t1), tcgv_vec_arg(t2)); 3587 tcg_temp_free_vec(t1); 3588 tcg_temp_free_vec(t2); 3589} 3590 3591static void expand_vec_sari(TCGType type, unsigned vece, 3592 TCGv_vec v0, TCGv_vec v1, TCGArg imm) 3593{ 3594 TCGv_vec t1, t2; 3595 3596 switch (vece) { 3597 case MO_8: 3598 /* Unpack to W, shift, and repack, as in expand_vec_shi. */ 3599 t1 = tcg_temp_new_vec(type); 3600 t2 = tcg_temp_new_vec(type); 3601 vec_gen_3(INDEX_op_x86_punpckl_vec, type, MO_8, 3602 tcgv_vec_arg(t1), tcgv_vec_arg(v1), tcgv_vec_arg(v1)); 3603 vec_gen_3(INDEX_op_x86_punpckh_vec, type, MO_8, 3604 tcgv_vec_arg(t2), tcgv_vec_arg(v1), tcgv_vec_arg(v1)); 3605 tcg_gen_sari_vec(MO_16, t1, t1, imm + 8); 3606 tcg_gen_sari_vec(MO_16, t2, t2, imm + 8); 3607 vec_gen_3(INDEX_op_x86_packss_vec, type, MO_8, 3608 tcgv_vec_arg(v0), tcgv_vec_arg(t1), tcgv_vec_arg(t2)); 3609 tcg_temp_free_vec(t1); 3610 tcg_temp_free_vec(t2); 3611 break; 3612 3613 case MO_64: 3614 if (imm <= 32) { 3615 /* 3616 * We can emulate a small sign extend by performing an arithmetic 3617 * 32-bit shift and overwriting the high half of a 64-bit logical 3618 * shift. Note that the ISA says shift of 32 is valid, but TCG 3619 * does not, so we have to bound the smaller shift -- we get the 3620 * same result in the high half either way. 3621 */ 3622 t1 = tcg_temp_new_vec(type); 3623 tcg_gen_sari_vec(MO_32, t1, v1, MIN(imm, 31)); 3624 tcg_gen_shri_vec(MO_64, v0, v1, imm); 3625 vec_gen_4(INDEX_op_x86_blend_vec, type, MO_32, 3626 tcgv_vec_arg(v0), tcgv_vec_arg(v0), 3627 tcgv_vec_arg(t1), 0xaa); 3628 tcg_temp_free_vec(t1); 3629 } else { 3630 /* Otherwise we will need to use a compare vs 0 to produce 3631 * the sign-extend, shift and merge. 3632 */ 3633 t1 = tcg_const_zeros_vec(type); 3634 tcg_gen_cmp_vec(TCG_COND_GT, MO_64, t1, t1, v1); 3635 tcg_gen_shri_vec(MO_64, v0, v1, imm); 3636 tcg_gen_shli_vec(MO_64, t1, t1, 64 - imm); 3637 tcg_gen_or_vec(MO_64, v0, v0, t1); 3638 tcg_temp_free_vec(t1); 3639 } 3640 break; 3641 3642 default: 3643 g_assert_not_reached(); 3644 } 3645} 3646 3647static void expand_vec_rotli(TCGType type, unsigned vece, 3648 TCGv_vec v0, TCGv_vec v1, TCGArg imm) 3649{ 3650 TCGv_vec t; 3651 3652 if (vece == MO_8) { 3653 expand_vec_shi(type, vece, INDEX_op_rotli_vec, v0, v1, imm); 3654 return; 3655 } 3656 3657 if (have_avx512vbmi2) { 3658 vec_gen_4(INDEX_op_x86_vpshldi_vec, type, vece, 3659 tcgv_vec_arg(v0), tcgv_vec_arg(v1), tcgv_vec_arg(v1), imm); 3660 return; 3661 } 3662 3663 t = tcg_temp_new_vec(type); 3664 tcg_gen_shli_vec(vece, t, v1, imm); 3665 tcg_gen_shri_vec(vece, v0, v1, (8 << vece) - imm); 3666 tcg_gen_or_vec(vece, v0, v0, t); 3667 tcg_temp_free_vec(t); 3668} 3669 3670static void expand_vec_rotv(TCGType type, unsigned vece, TCGv_vec v0, 3671 TCGv_vec v1, TCGv_vec sh, bool right) 3672{ 3673 TCGv_vec t; 3674 3675 if (have_avx512vbmi2) { 3676 vec_gen_4(right ? INDEX_op_x86_vpshrdv_vec : INDEX_op_x86_vpshldv_vec, 3677 type, vece, tcgv_vec_arg(v0), tcgv_vec_arg(v1), 3678 tcgv_vec_arg(v1), tcgv_vec_arg(sh)); 3679 return; 3680 } 3681 3682 t = tcg_temp_new_vec(type); 3683 tcg_gen_dupi_vec(vece, t, 8 << vece); 3684 tcg_gen_sub_vec(vece, t, t, sh); 3685 if (right) { 3686 tcg_gen_shlv_vec(vece, t, v1, t); 3687 tcg_gen_shrv_vec(vece, v0, v1, sh); 3688 } else { 3689 tcg_gen_shrv_vec(vece, t, v1, t); 3690 tcg_gen_shlv_vec(vece, v0, v1, sh); 3691 } 3692 tcg_gen_or_vec(vece, v0, v0, t); 3693 tcg_temp_free_vec(t); 3694} 3695 3696static void expand_vec_rotls(TCGType type, unsigned vece, 3697 TCGv_vec v0, TCGv_vec v1, TCGv_i32 lsh) 3698{ 3699 TCGv_vec t = tcg_temp_new_vec(type); 3700 3701 tcg_debug_assert(vece != MO_8); 3702 3703 if (vece >= MO_32 ? have_avx512vl : have_avx512vbmi2) { 3704 tcg_gen_dup_i32_vec(vece, t, lsh); 3705 if (vece >= MO_32) { 3706 tcg_gen_rotlv_vec(vece, v0, v1, t); 3707 } else { 3708 expand_vec_rotv(type, vece, v0, v1, t, false); 3709 } 3710 } else { 3711 TCGv_i32 rsh = tcg_temp_new_i32(); 3712 3713 tcg_gen_neg_i32(rsh, lsh); 3714 tcg_gen_andi_i32(rsh, rsh, (8 << vece) - 1); 3715 tcg_gen_shls_vec(vece, t, v1, lsh); 3716 tcg_gen_shrs_vec(vece, v0, v1, rsh); 3717 tcg_gen_or_vec(vece, v0, v0, t); 3718 3719 tcg_temp_free_i32(rsh); 3720 } 3721 3722 tcg_temp_free_vec(t); 3723} 3724 3725static void expand_vec_mul(TCGType type, unsigned vece, 3726 TCGv_vec v0, TCGv_vec v1, TCGv_vec v2) 3727{ 3728 TCGv_vec t1, t2, t3, t4, zero; 3729 3730 tcg_debug_assert(vece == MO_8); 3731 3732 /* 3733 * Unpack v1 bytes to words, 0 | x. 3734 * Unpack v2 bytes to words, y | 0. 3735 * This leaves the 8-bit result, x * y, with 8 bits of right padding. 3736 * Shift logical right by 8 bits to clear the high 8 bytes before 3737 * using an unsigned saturated pack. 3738 * 3739 * The difference between the V64, V128 and V256 cases is merely how 3740 * we distribute the expansion between temporaries. 3741 */ 3742 switch (type) { 3743 case TCG_TYPE_V64: 3744 t1 = tcg_temp_new_vec(TCG_TYPE_V128); 3745 t2 = tcg_temp_new_vec(TCG_TYPE_V128); 3746 zero = tcg_constant_vec(TCG_TYPE_V128, MO_8, 0); 3747 vec_gen_3(INDEX_op_x86_punpckl_vec, TCG_TYPE_V128, MO_8, 3748 tcgv_vec_arg(t1), tcgv_vec_arg(v1), tcgv_vec_arg(zero)); 3749 vec_gen_3(INDEX_op_x86_punpckl_vec, TCG_TYPE_V128, MO_8, 3750 tcgv_vec_arg(t2), tcgv_vec_arg(zero), tcgv_vec_arg(v2)); 3751 tcg_gen_mul_vec(MO_16, t1, t1, t2); 3752 tcg_gen_shri_vec(MO_16, t1, t1, 8); 3753 vec_gen_3(INDEX_op_x86_packus_vec, TCG_TYPE_V128, MO_8, 3754 tcgv_vec_arg(v0), tcgv_vec_arg(t1), tcgv_vec_arg(t1)); 3755 tcg_temp_free_vec(t1); 3756 tcg_temp_free_vec(t2); 3757 break; 3758 3759 case TCG_TYPE_V128: 3760 case TCG_TYPE_V256: 3761 t1 = tcg_temp_new_vec(type); 3762 t2 = tcg_temp_new_vec(type); 3763 t3 = tcg_temp_new_vec(type); 3764 t4 = tcg_temp_new_vec(type); 3765 zero = tcg_constant_vec(TCG_TYPE_V128, MO_8, 0); 3766 vec_gen_3(INDEX_op_x86_punpckl_vec, type, MO_8, 3767 tcgv_vec_arg(t1), tcgv_vec_arg(v1), tcgv_vec_arg(zero)); 3768 vec_gen_3(INDEX_op_x86_punpckl_vec, type, MO_8, 3769 tcgv_vec_arg(t2), tcgv_vec_arg(zero), tcgv_vec_arg(v2)); 3770 vec_gen_3(INDEX_op_x86_punpckh_vec, type, MO_8, 3771 tcgv_vec_arg(t3), tcgv_vec_arg(v1), tcgv_vec_arg(zero)); 3772 vec_gen_3(INDEX_op_x86_punpckh_vec, type, MO_8, 3773 tcgv_vec_arg(t4), tcgv_vec_arg(zero), tcgv_vec_arg(v2)); 3774 tcg_gen_mul_vec(MO_16, t1, t1, t2); 3775 tcg_gen_mul_vec(MO_16, t3, t3, t4); 3776 tcg_gen_shri_vec(MO_16, t1, t1, 8); 3777 tcg_gen_shri_vec(MO_16, t3, t3, 8); 3778 vec_gen_3(INDEX_op_x86_packus_vec, type, MO_8, 3779 tcgv_vec_arg(v0), tcgv_vec_arg(t1), tcgv_vec_arg(t3)); 3780 tcg_temp_free_vec(t1); 3781 tcg_temp_free_vec(t2); 3782 tcg_temp_free_vec(t3); 3783 tcg_temp_free_vec(t4); 3784 break; 3785 3786 default: 3787 g_assert_not_reached(); 3788 } 3789} 3790 3791static bool expand_vec_cmp_noinv(TCGType type, unsigned vece, TCGv_vec v0, 3792 TCGv_vec v1, TCGv_vec v2, TCGCond cond) 3793{ 3794 enum { 3795 NEED_INV = 1, 3796 NEED_SWAP = 2, 3797 NEED_BIAS = 4, 3798 NEED_UMIN = 8, 3799 NEED_UMAX = 16, 3800 }; 3801 TCGv_vec t1, t2, t3; 3802 uint8_t fixup; 3803 3804 switch (cond) { 3805 case TCG_COND_EQ: 3806 case TCG_COND_GT: 3807 fixup = 0; 3808 break; 3809 case TCG_COND_NE: 3810 case TCG_COND_LE: 3811 fixup = NEED_INV; 3812 break; 3813 case TCG_COND_LT: 3814 fixup = NEED_SWAP; 3815 break; 3816 case TCG_COND_GE: 3817 fixup = NEED_SWAP | NEED_INV; 3818 break; 3819 case TCG_COND_LEU: 3820 if (tcg_can_emit_vec_op(INDEX_op_umin_vec, type, vece)) { 3821 fixup = NEED_UMIN; 3822 } else { 3823 fixup = NEED_BIAS | NEED_INV; 3824 } 3825 break; 3826 case TCG_COND_GTU: 3827 if (tcg_can_emit_vec_op(INDEX_op_umin_vec, type, vece)) { 3828 fixup = NEED_UMIN | NEED_INV; 3829 } else { 3830 fixup = NEED_BIAS; 3831 } 3832 break; 3833 case TCG_COND_GEU: 3834 if (tcg_can_emit_vec_op(INDEX_op_umax_vec, type, vece)) { 3835 fixup = NEED_UMAX; 3836 } else { 3837 fixup = NEED_BIAS | NEED_SWAP | NEED_INV; 3838 } 3839 break; 3840 case TCG_COND_LTU: 3841 if (tcg_can_emit_vec_op(INDEX_op_umax_vec, type, vece)) { 3842 fixup = NEED_UMAX | NEED_INV; 3843 } else { 3844 fixup = NEED_BIAS | NEED_SWAP; 3845 } 3846 break; 3847 default: 3848 g_assert_not_reached(); 3849 } 3850 3851 if (fixup & NEED_INV) { 3852 cond = tcg_invert_cond(cond); 3853 } 3854 if (fixup & NEED_SWAP) { 3855 t1 = v1, v1 = v2, v2 = t1; 3856 cond = tcg_swap_cond(cond); 3857 } 3858 3859 t1 = t2 = NULL; 3860 if (fixup & (NEED_UMIN | NEED_UMAX)) { 3861 t1 = tcg_temp_new_vec(type); 3862 if (fixup & NEED_UMIN) { 3863 tcg_gen_umin_vec(vece, t1, v1, v2); 3864 } else { 3865 tcg_gen_umax_vec(vece, t1, v1, v2); 3866 } 3867 v2 = t1; 3868 cond = TCG_COND_EQ; 3869 } else if (fixup & NEED_BIAS) { 3870 t1 = tcg_temp_new_vec(type); 3871 t2 = tcg_temp_new_vec(type); 3872 t3 = tcg_constant_vec(type, vece, 1ull << ((8 << vece) - 1)); 3873 tcg_gen_sub_vec(vece, t1, v1, t3); 3874 tcg_gen_sub_vec(vece, t2, v2, t3); 3875 v1 = t1; 3876 v2 = t2; 3877 cond = tcg_signed_cond(cond); 3878 } 3879 3880 tcg_debug_assert(cond == TCG_COND_EQ || cond == TCG_COND_GT); 3881 /* Expand directly; do not recurse. */ 3882 vec_gen_4(INDEX_op_cmp_vec, type, vece, 3883 tcgv_vec_arg(v0), tcgv_vec_arg(v1), tcgv_vec_arg(v2), cond); 3884 3885 if (t1) { 3886 tcg_temp_free_vec(t1); 3887 if (t2) { 3888 tcg_temp_free_vec(t2); 3889 } 3890 } 3891 return fixup & NEED_INV; 3892} 3893 3894static void expand_vec_cmp(TCGType type, unsigned vece, TCGv_vec v0, 3895 TCGv_vec v1, TCGv_vec v2, TCGCond cond) 3896{ 3897 if (expand_vec_cmp_noinv(type, vece, v0, v1, v2, cond)) { 3898 tcg_gen_not_vec(vece, v0, v0); 3899 } 3900} 3901 3902static void expand_vec_cmpsel(TCGType type, unsigned vece, TCGv_vec v0, 3903 TCGv_vec c1, TCGv_vec c2, 3904 TCGv_vec v3, TCGv_vec v4, TCGCond cond) 3905{ 3906 TCGv_vec t = tcg_temp_new_vec(type); 3907 3908 if (expand_vec_cmp_noinv(type, vece, t, c1, c2, cond)) { 3909 /* Invert the sense of the compare by swapping arguments. */ 3910 TCGv_vec x; 3911 x = v3, v3 = v4, v4 = x; 3912 } 3913 vec_gen_4(INDEX_op_x86_vpblendvb_vec, type, vece, 3914 tcgv_vec_arg(v0), tcgv_vec_arg(v4), 3915 tcgv_vec_arg(v3), tcgv_vec_arg(t)); 3916 tcg_temp_free_vec(t); 3917} 3918 3919void tcg_expand_vec_op(TCGOpcode opc, TCGType type, unsigned vece, 3920 TCGArg a0, ...) 3921{ 3922 va_list va; 3923 TCGArg a2; 3924 TCGv_vec v0, v1, v2, v3, v4; 3925 3926 va_start(va, a0); 3927 v0 = temp_tcgv_vec(arg_temp(a0)); 3928 v1 = temp_tcgv_vec(arg_temp(va_arg(va, TCGArg))); 3929 a2 = va_arg(va, TCGArg); 3930 3931 switch (opc) { 3932 case INDEX_op_shli_vec: 3933 case INDEX_op_shri_vec: 3934 expand_vec_shi(type, vece, opc, v0, v1, a2); 3935 break; 3936 3937 case INDEX_op_sari_vec: 3938 expand_vec_sari(type, vece, v0, v1, a2); 3939 break; 3940 3941 case INDEX_op_rotli_vec: 3942 expand_vec_rotli(type, vece, v0, v1, a2); 3943 break; 3944 3945 case INDEX_op_rotls_vec: 3946 expand_vec_rotls(type, vece, v0, v1, temp_tcgv_i32(arg_temp(a2))); 3947 break; 3948 3949 case INDEX_op_rotlv_vec: 3950 v2 = temp_tcgv_vec(arg_temp(a2)); 3951 expand_vec_rotv(type, vece, v0, v1, v2, false); 3952 break; 3953 case INDEX_op_rotrv_vec: 3954 v2 = temp_tcgv_vec(arg_temp(a2)); 3955 expand_vec_rotv(type, vece, v0, v1, v2, true); 3956 break; 3957 3958 case INDEX_op_mul_vec: 3959 v2 = temp_tcgv_vec(arg_temp(a2)); 3960 expand_vec_mul(type, vece, v0, v1, v2); 3961 break; 3962 3963 case INDEX_op_cmp_vec: 3964 v2 = temp_tcgv_vec(arg_temp(a2)); 3965 expand_vec_cmp(type, vece, v0, v1, v2, va_arg(va, TCGArg)); 3966 break; 3967 3968 case INDEX_op_cmpsel_vec: 3969 v2 = temp_tcgv_vec(arg_temp(a2)); 3970 v3 = temp_tcgv_vec(arg_temp(va_arg(va, TCGArg))); 3971 v4 = temp_tcgv_vec(arg_temp(va_arg(va, TCGArg))); 3972 expand_vec_cmpsel(type, vece, v0, v1, v2, v3, v4, va_arg(va, TCGArg)); 3973 break; 3974 3975 default: 3976 break; 3977 } 3978 3979 va_end(va); 3980} 3981 3982static const int tcg_target_callee_save_regs[] = { 3983#if TCG_TARGET_REG_BITS == 64 3984 TCG_REG_RBP, 3985 TCG_REG_RBX, 3986#if defined(_WIN64) 3987 TCG_REG_RDI, 3988 TCG_REG_RSI, 3989#endif 3990 TCG_REG_R12, 3991 TCG_REG_R13, 3992 TCG_REG_R14, /* Currently used for the global env. */ 3993 TCG_REG_R15, 3994#else 3995 TCG_REG_EBP, /* Currently used for the global env. */ 3996 TCG_REG_EBX, 3997 TCG_REG_ESI, 3998 TCG_REG_EDI, 3999#endif 4000}; 4001 4002/* Compute frame size via macros, to share between tcg_target_qemu_prologue 4003 and tcg_register_jit. */ 4004 4005#define PUSH_SIZE \ 4006 ((1 + ARRAY_SIZE(tcg_target_callee_save_regs)) \ 4007 * (TCG_TARGET_REG_BITS / 8)) 4008 4009#define FRAME_SIZE \ 4010 ((PUSH_SIZE \ 4011 + TCG_STATIC_CALL_ARGS_SIZE \ 4012 + CPU_TEMP_BUF_NLONGS * sizeof(long) \ 4013 + TCG_TARGET_STACK_ALIGN - 1) \ 4014 & ~(TCG_TARGET_STACK_ALIGN - 1)) 4015 4016/* Generate global QEMU prologue and epilogue code */ 4017static void tcg_target_qemu_prologue(TCGContext *s) 4018{ 4019 int i, stack_addend; 4020 4021 /* TB prologue */ 4022 4023 /* Reserve some stack space, also for TCG temps. */ 4024 stack_addend = FRAME_SIZE - PUSH_SIZE; 4025 tcg_set_frame(s, TCG_REG_CALL_STACK, TCG_STATIC_CALL_ARGS_SIZE, 4026 CPU_TEMP_BUF_NLONGS * sizeof(long)); 4027 4028 /* Save all callee saved registers. */ 4029 for (i = 0; i < ARRAY_SIZE(tcg_target_callee_save_regs); i++) { 4030 tcg_out_push(s, tcg_target_callee_save_regs[i]); 4031 } 4032 4033#if TCG_TARGET_REG_BITS == 32 4034 tcg_out_ld(s, TCG_TYPE_PTR, TCG_AREG0, TCG_REG_ESP, 4035 (ARRAY_SIZE(tcg_target_callee_save_regs) + 1) * 4); 4036 tcg_out_addi(s, TCG_REG_ESP, -stack_addend); 4037 /* jmp *tb. */ 4038 tcg_out_modrm_offset(s, OPC_GRP5, EXT5_JMPN_Ev, TCG_REG_ESP, 4039 (ARRAY_SIZE(tcg_target_callee_save_regs) + 2) * 4 4040 + stack_addend); 4041#else 4042# if !defined(CONFIG_SOFTMMU) && TCG_TARGET_REG_BITS == 64 4043 if (guest_base) { 4044 int seg = setup_guest_base_seg(); 4045 if (seg != 0) { 4046 x86_guest_base_seg = seg; 4047 } else if (guest_base == (int32_t)guest_base) { 4048 x86_guest_base_offset = guest_base; 4049 } else { 4050 /* Choose R12 because, as a base, it requires a SIB byte. */ 4051 x86_guest_base_index = TCG_REG_R12; 4052 tcg_out_movi(s, TCG_TYPE_PTR, x86_guest_base_index, guest_base); 4053 tcg_regset_set_reg(s->reserved_regs, x86_guest_base_index); 4054 } 4055 } 4056# endif 4057 tcg_out_mov(s, TCG_TYPE_PTR, TCG_AREG0, tcg_target_call_iarg_regs[0]); 4058 tcg_out_addi(s, TCG_REG_ESP, -stack_addend); 4059 /* jmp *tb. */ 4060 tcg_out_modrm(s, OPC_GRP5, EXT5_JMPN_Ev, tcg_target_call_iarg_regs[1]); 4061#endif 4062 4063 /* 4064 * Return path for goto_ptr. Set return value to 0, a-la exit_tb, 4065 * and fall through to the rest of the epilogue. 4066 */ 4067 tcg_code_gen_epilogue = tcg_splitwx_to_rx(s->code_ptr); 4068 tcg_out_movi(s, TCG_TYPE_REG, TCG_REG_EAX, 0); 4069 4070 /* TB epilogue */ 4071 tb_ret_addr = tcg_splitwx_to_rx(s->code_ptr); 4072 4073 tcg_out_addi(s, TCG_REG_CALL_STACK, stack_addend); 4074 4075 if (have_avx2) { 4076 tcg_out_vex_opc(s, OPC_VZEROUPPER, 0, 0, 0, 0); 4077 } 4078 for (i = ARRAY_SIZE(tcg_target_callee_save_regs) - 1; i >= 0; i--) { 4079 tcg_out_pop(s, tcg_target_callee_save_regs[i]); 4080 } 4081 tcg_out_opc(s, OPC_RET, 0, 0, 0); 4082} 4083 4084static void tcg_out_nop_fill(tcg_insn_unit *p, int count) 4085{ 4086 memset(p, 0x90, count); 4087} 4088 4089static void tcg_target_init(TCGContext *s) 4090{ 4091#ifdef CONFIG_CPUID_H 4092 unsigned a, b, c, d, b7 = 0, c7 = 0; 4093 unsigned max = __get_cpuid_max(0, 0); 4094 4095 if (max >= 7) { 4096 /* BMI1 is available on AMD Piledriver and Intel Haswell CPUs. */ 4097 __cpuid_count(7, 0, a, b7, c7, d); 4098 have_bmi1 = (b7 & bit_BMI) != 0; 4099 have_bmi2 = (b7 & bit_BMI2) != 0; 4100 } 4101 4102 if (max >= 1) { 4103 __cpuid(1, a, b, c, d); 4104#ifndef have_cmov 4105 /* For 32-bit, 99% certainty that we're running on hardware that 4106 supports cmov, but we still need to check. In case cmov is not 4107 available, we'll use a small forward branch. */ 4108 have_cmov = (d & bit_CMOV) != 0; 4109#endif 4110 4111 /* MOVBE is only available on Intel Atom and Haswell CPUs, so we 4112 need to probe for it. */ 4113 have_movbe = (c & bit_MOVBE) != 0; 4114 have_popcnt = (c & bit_POPCNT) != 0; 4115 4116 /* There are a number of things we must check before we can be 4117 sure of not hitting invalid opcode. */ 4118 if (c & bit_OSXSAVE) { 4119 unsigned xcrl, xcrh; 4120 /* The xgetbv instruction is not available to older versions of 4121 * the assembler, so we encode the instruction manually. 4122 */ 4123 asm(".byte 0x0f, 0x01, 0xd0" : "=a" (xcrl), "=d" (xcrh) : "c" (0)); 4124 if ((xcrl & 6) == 6) { 4125 have_avx1 = (c & bit_AVX) != 0; 4126 have_avx2 = (b7 & bit_AVX2) != 0; 4127 4128 /* 4129 * There are interesting instructions in AVX512, so long 4130 * as we have AVX512VL, which indicates support for EVEX 4131 * on sizes smaller than 512 bits. We are required to 4132 * check that OPMASK and all extended ZMM state are enabled 4133 * even if we're not using them -- the insns will fault. 4134 */ 4135 if ((xcrl & 0xe0) == 0xe0 4136 && (b7 & bit_AVX512F) 4137 && (b7 & bit_AVX512VL)) { 4138 have_avx512vl = true; 4139 have_avx512bw = (b7 & bit_AVX512BW) != 0; 4140 have_avx512dq = (b7 & bit_AVX512DQ) != 0; 4141 have_avx512vbmi2 = (c7 & bit_AVX512VBMI2) != 0; 4142 } 4143 } 4144 } 4145 } 4146 4147 max = __get_cpuid_max(0x8000000, 0); 4148 if (max >= 1) { 4149 __cpuid(0x80000001, a, b, c, d); 4150 /* LZCNT was introduced with AMD Barcelona and Intel Haswell CPUs. */ 4151 have_lzcnt = (c & bit_LZCNT) != 0; 4152 } 4153#endif /* CONFIG_CPUID_H */ 4154 4155 tcg_target_available_regs[TCG_TYPE_I32] = ALL_GENERAL_REGS; 4156 if (TCG_TARGET_REG_BITS == 64) { 4157 tcg_target_available_regs[TCG_TYPE_I64] = ALL_GENERAL_REGS; 4158 } 4159 if (have_avx1) { 4160 tcg_target_available_regs[TCG_TYPE_V64] = ALL_VECTOR_REGS; 4161 tcg_target_available_regs[TCG_TYPE_V128] = ALL_VECTOR_REGS; 4162 } 4163 if (have_avx2) { 4164 tcg_target_available_regs[TCG_TYPE_V256] = ALL_VECTOR_REGS; 4165 } 4166 4167 tcg_target_call_clobber_regs = ALL_VECTOR_REGS; 4168 tcg_regset_set_reg(tcg_target_call_clobber_regs, TCG_REG_EAX); 4169 tcg_regset_set_reg(tcg_target_call_clobber_regs, TCG_REG_EDX); 4170 tcg_regset_set_reg(tcg_target_call_clobber_regs, TCG_REG_ECX); 4171 if (TCG_TARGET_REG_BITS == 64) { 4172#if !defined(_WIN64) 4173 tcg_regset_set_reg(tcg_target_call_clobber_regs, TCG_REG_RDI); 4174 tcg_regset_set_reg(tcg_target_call_clobber_regs, TCG_REG_RSI); 4175#endif 4176 tcg_regset_set_reg(tcg_target_call_clobber_regs, TCG_REG_R8); 4177 tcg_regset_set_reg(tcg_target_call_clobber_regs, TCG_REG_R9); 4178 tcg_regset_set_reg(tcg_target_call_clobber_regs, TCG_REG_R10); 4179 tcg_regset_set_reg(tcg_target_call_clobber_regs, TCG_REG_R11); 4180 } 4181 4182 s->reserved_regs = 0; 4183 tcg_regset_set_reg(s->reserved_regs, TCG_REG_CALL_STACK); 4184} 4185 4186typedef struct { 4187 DebugFrameHeader h; 4188 uint8_t fde_def_cfa[4]; 4189 uint8_t fde_reg_ofs[14]; 4190} DebugFrame; 4191 4192/* We're expecting a 2 byte uleb128 encoded value. */ 4193QEMU_BUILD_BUG_ON(FRAME_SIZE >= (1 << 14)); 4194 4195#if !defined(__ELF__) 4196 /* Host machine without ELF. */ 4197#elif TCG_TARGET_REG_BITS == 64 4198#define ELF_HOST_MACHINE EM_X86_64 4199static const DebugFrame debug_frame = { 4200 .h.cie.len = sizeof(DebugFrameCIE)-4, /* length after .len member */ 4201 .h.cie.id = -1, 4202 .h.cie.version = 1, 4203 .h.cie.code_align = 1, 4204 .h.cie.data_align = 0x78, /* sleb128 -8 */ 4205 .h.cie.return_column = 16, 4206 4207 /* Total FDE size does not include the "len" member. */ 4208 .h.fde.len = sizeof(DebugFrame) - offsetof(DebugFrame, h.fde.cie_offset), 4209 4210 .fde_def_cfa = { 4211 12, 7, /* DW_CFA_def_cfa %rsp, ... */ 4212 (FRAME_SIZE & 0x7f) | 0x80, /* ... uleb128 FRAME_SIZE */ 4213 (FRAME_SIZE >> 7) 4214 }, 4215 .fde_reg_ofs = { 4216 0x90, 1, /* DW_CFA_offset, %rip, -8 */ 4217 /* The following ordering must match tcg_target_callee_save_regs. */ 4218 0x86, 2, /* DW_CFA_offset, %rbp, -16 */ 4219 0x83, 3, /* DW_CFA_offset, %rbx, -24 */ 4220 0x8c, 4, /* DW_CFA_offset, %r12, -32 */ 4221 0x8d, 5, /* DW_CFA_offset, %r13, -40 */ 4222 0x8e, 6, /* DW_CFA_offset, %r14, -48 */ 4223 0x8f, 7, /* DW_CFA_offset, %r15, -56 */ 4224 } 4225}; 4226#else 4227#define ELF_HOST_MACHINE EM_386 4228static const DebugFrame debug_frame = { 4229 .h.cie.len = sizeof(DebugFrameCIE)-4, /* length after .len member */ 4230 .h.cie.id = -1, 4231 .h.cie.version = 1, 4232 .h.cie.code_align = 1, 4233 .h.cie.data_align = 0x7c, /* sleb128 -4 */ 4234 .h.cie.return_column = 8, 4235 4236 /* Total FDE size does not include the "len" member. */ 4237 .h.fde.len = sizeof(DebugFrame) - offsetof(DebugFrame, h.fde.cie_offset), 4238 4239 .fde_def_cfa = { 4240 12, 4, /* DW_CFA_def_cfa %esp, ... */ 4241 (FRAME_SIZE & 0x7f) | 0x80, /* ... uleb128 FRAME_SIZE */ 4242 (FRAME_SIZE >> 7) 4243 }, 4244 .fde_reg_ofs = { 4245 0x88, 1, /* DW_CFA_offset, %eip, -4 */ 4246 /* The following ordering must match tcg_target_callee_save_regs. */ 4247 0x85, 2, /* DW_CFA_offset, %ebp, -8 */ 4248 0x83, 3, /* DW_CFA_offset, %ebx, -12 */ 4249 0x86, 4, /* DW_CFA_offset, %esi, -16 */ 4250 0x87, 5, /* DW_CFA_offset, %edi, -20 */ 4251 } 4252}; 4253#endif 4254 4255#if defined(ELF_HOST_MACHINE) 4256void tcg_register_jit(const void *buf, size_t buf_size) 4257{ 4258 tcg_register_jit_int(buf, buf_size, &debug_frame, sizeof(debug_frame)); 4259} 4260#endif 4261