xref: /openbmc/qemu/tcg/i386/tcg-target.c.inc (revision 6c187695)
1/*
2 * Tiny Code Generator for QEMU
3 *
4 * Copyright (c) 2008 Fabrice Bellard
5 *
6 * Permission is hereby granted, free of charge, to any person obtaining a copy
7 * of this software and associated documentation files (the "Software"), to deal
8 * in the Software without restriction, including without limitation the rights
9 * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
10 * copies of the Software, and to permit persons to whom the Software is
11 * furnished to do so, subject to the following conditions:
12 *
13 * The above copyright notice and this permission notice shall be included in
14 * all copies or substantial portions of the Software.
15 *
16 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
17 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
18 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
19 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
20 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
21 * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
22 * THE SOFTWARE.
23 */
24
25#include "../tcg-ldst.c.inc"
26#include "../tcg-pool.c.inc"
27
28#ifdef CONFIG_DEBUG_TCG
29static const char * const tcg_target_reg_names[TCG_TARGET_NB_REGS] = {
30#if TCG_TARGET_REG_BITS == 64
31    "%rax", "%rcx", "%rdx", "%rbx", "%rsp", "%rbp", "%rsi", "%rdi",
32#else
33    "%eax", "%ecx", "%edx", "%ebx", "%esp", "%ebp", "%esi", "%edi",
34#endif
35    "%r8",  "%r9",  "%r10", "%r11", "%r12", "%r13", "%r14", "%r15",
36    "%xmm0", "%xmm1", "%xmm2", "%xmm3", "%xmm4", "%xmm5", "%xmm6", "%xmm7",
37#if TCG_TARGET_REG_BITS == 64
38    "%xmm8", "%xmm9", "%xmm10", "%xmm11",
39    "%xmm12", "%xmm13", "%xmm14", "%xmm15",
40#endif
41};
42#endif
43
44static const int tcg_target_reg_alloc_order[] = {
45#if TCG_TARGET_REG_BITS == 64
46    TCG_REG_RBP,
47    TCG_REG_RBX,
48    TCG_REG_R12,
49    TCG_REG_R13,
50    TCG_REG_R14,
51    TCG_REG_R15,
52    TCG_REG_R10,
53    TCG_REG_R11,
54    TCG_REG_R9,
55    TCG_REG_R8,
56    TCG_REG_RCX,
57    TCG_REG_RDX,
58    TCG_REG_RSI,
59    TCG_REG_RDI,
60    TCG_REG_RAX,
61#else
62    TCG_REG_EBX,
63    TCG_REG_ESI,
64    TCG_REG_EDI,
65    TCG_REG_EBP,
66    TCG_REG_ECX,
67    TCG_REG_EDX,
68    TCG_REG_EAX,
69#endif
70    TCG_REG_XMM0,
71    TCG_REG_XMM1,
72    TCG_REG_XMM2,
73    TCG_REG_XMM3,
74    TCG_REG_XMM4,
75    TCG_REG_XMM5,
76#ifndef _WIN64
77    /* The Win64 ABI has xmm6-xmm15 as caller-saves, and we do not save
78       any of them.  Therefore only allow xmm0-xmm5 to be allocated.  */
79    TCG_REG_XMM6,
80    TCG_REG_XMM7,
81#if TCG_TARGET_REG_BITS == 64
82    TCG_REG_XMM8,
83    TCG_REG_XMM9,
84    TCG_REG_XMM10,
85    TCG_REG_XMM11,
86    TCG_REG_XMM12,
87    TCG_REG_XMM13,
88    TCG_REG_XMM14,
89    TCG_REG_XMM15,
90#endif
91#endif
92};
93
94static const int tcg_target_call_iarg_regs[] = {
95#if TCG_TARGET_REG_BITS == 64
96#if defined(_WIN64)
97    TCG_REG_RCX,
98    TCG_REG_RDX,
99#else
100    TCG_REG_RDI,
101    TCG_REG_RSI,
102    TCG_REG_RDX,
103    TCG_REG_RCX,
104#endif
105    TCG_REG_R8,
106    TCG_REG_R9,
107#else
108    /* 32 bit mode uses stack based calling convention (GCC default). */
109#endif
110};
111
112static const int tcg_target_call_oarg_regs[] = {
113    TCG_REG_EAX,
114#if TCG_TARGET_REG_BITS == 32
115    TCG_REG_EDX
116#endif
117};
118
119/* Constants we accept.  */
120#define TCG_CT_CONST_S32 0x100
121#define TCG_CT_CONST_U32 0x200
122#define TCG_CT_CONST_I32 0x400
123#define TCG_CT_CONST_WSZ 0x800
124
125/* Registers used with L constraint, which are the first argument
126   registers on x86_64, and two random call clobbered registers on
127   i386. */
128#if TCG_TARGET_REG_BITS == 64
129# define TCG_REG_L0 tcg_target_call_iarg_regs[0]
130# define TCG_REG_L1 tcg_target_call_iarg_regs[1]
131#else
132# define TCG_REG_L0 TCG_REG_EAX
133# define TCG_REG_L1 TCG_REG_EDX
134#endif
135
136#define ALL_BYTEH_REGS         0x0000000fu
137#if TCG_TARGET_REG_BITS == 64
138# define ALL_GENERAL_REGS      0x0000ffffu
139# define ALL_VECTOR_REGS       0xffff0000u
140# define ALL_BYTEL_REGS        ALL_GENERAL_REGS
141#else
142# define ALL_GENERAL_REGS      0x000000ffu
143# define ALL_VECTOR_REGS       0x00ff0000u
144# define ALL_BYTEL_REGS        ALL_BYTEH_REGS
145#endif
146#ifdef CONFIG_SOFTMMU
147# define SOFTMMU_RESERVE_REGS  ((1 << TCG_REG_L0) | (1 << TCG_REG_L1))
148#else
149# define SOFTMMU_RESERVE_REGS  0
150#endif
151
152/* The host compiler should supply <cpuid.h> to enable runtime features
153   detection, as we're not going to go so far as our own inline assembly.
154   If not available, default values will be assumed.  */
155#if defined(CONFIG_CPUID_H)
156#include "qemu/cpuid.h"
157#endif
158
159/* For 64-bit, we always know that CMOV is available.  */
160#if TCG_TARGET_REG_BITS == 64
161# define have_cmov 1
162#elif defined(CONFIG_CPUID_H)
163static bool have_cmov;
164#else
165# define have_cmov 0
166#endif
167
168/* We need these symbols in tcg-target.h, and we can't properly conditionalize
169   it there.  Therefore we always define the variable.  */
170bool have_bmi1;
171bool have_popcnt;
172bool have_avx1;
173bool have_avx2;
174bool have_avx512bw;
175bool have_avx512dq;
176bool have_avx512vbmi2;
177bool have_avx512vl;
178bool have_movbe;
179
180#ifdef CONFIG_CPUID_H
181static bool have_bmi2;
182static bool have_lzcnt;
183#else
184# define have_bmi2 0
185# define have_lzcnt 0
186#endif
187
188static const tcg_insn_unit *tb_ret_addr;
189
190static bool patch_reloc(tcg_insn_unit *code_ptr, int type,
191                        intptr_t value, intptr_t addend)
192{
193    value += addend;
194    switch(type) {
195    case R_386_PC32:
196        value -= (uintptr_t)tcg_splitwx_to_rx(code_ptr);
197        if (value != (int32_t)value) {
198            return false;
199        }
200        /* FALLTHRU */
201    case R_386_32:
202        tcg_patch32(code_ptr, value);
203        break;
204    case R_386_PC8:
205        value -= (uintptr_t)tcg_splitwx_to_rx(code_ptr);
206        if (value != (int8_t)value) {
207            return false;
208        }
209        tcg_patch8(code_ptr, value);
210        break;
211    default:
212        tcg_abort();
213    }
214    return true;
215}
216
217/* test if a constant matches the constraint */
218static bool tcg_target_const_match(int64_t val, TCGType type, int ct)
219{
220    if (ct & TCG_CT_CONST) {
221        return 1;
222    }
223    if (type == TCG_TYPE_I32) {
224        if (ct & (TCG_CT_CONST_S32 | TCG_CT_CONST_U32 | TCG_CT_CONST_I32)) {
225            return 1;
226        }
227    } else {
228        if ((ct & TCG_CT_CONST_S32) && val == (int32_t)val) {
229            return 1;
230        }
231        if ((ct & TCG_CT_CONST_U32) && val == (uint32_t)val) {
232            return 1;
233        }
234        if ((ct & TCG_CT_CONST_I32) && ~val == (int32_t)~val) {
235            return 1;
236        }
237    }
238    if ((ct & TCG_CT_CONST_WSZ) && val == (type == TCG_TYPE_I32 ? 32 : 64)) {
239        return 1;
240    }
241    return 0;
242}
243
244# define LOWREGMASK(x)	((x) & 7)
245
246#define P_EXT		0x100		/* 0x0f opcode prefix */
247#define P_EXT38         0x200           /* 0x0f 0x38 opcode prefix */
248#define P_DATA16        0x400           /* 0x66 opcode prefix */
249#define P_VEXW          0x1000          /* Set VEX.W = 1 */
250#if TCG_TARGET_REG_BITS == 64
251# define P_REXW         P_VEXW          /* Set REX.W = 1; match VEXW */
252# define P_REXB_R       0x2000          /* REG field as byte register */
253# define P_REXB_RM      0x4000          /* R/M field as byte register */
254# define P_GS           0x8000          /* gs segment override */
255#else
256# define P_REXW		0
257# define P_REXB_R	0
258# define P_REXB_RM	0
259# define P_GS           0
260#endif
261#define P_EXT3A         0x10000         /* 0x0f 0x3a opcode prefix */
262#define P_SIMDF3        0x20000         /* 0xf3 opcode prefix */
263#define P_SIMDF2        0x40000         /* 0xf2 opcode prefix */
264#define P_VEXL          0x80000         /* Set VEX.L = 1 */
265#define P_EVEX          0x100000        /* Requires EVEX encoding */
266
267#define OPC_ARITH_EvIz	(0x81)
268#define OPC_ARITH_EvIb	(0x83)
269#define OPC_ARITH_GvEv	(0x03)		/* ... plus (ARITH_FOO << 3) */
270#define OPC_ANDN        (0xf2 | P_EXT38)
271#define OPC_ADD_GvEv	(OPC_ARITH_GvEv | (ARITH_ADD << 3))
272#define OPC_AND_GvEv    (OPC_ARITH_GvEv | (ARITH_AND << 3))
273#define OPC_BLENDPS     (0x0c | P_EXT3A | P_DATA16)
274#define OPC_BSF         (0xbc | P_EXT)
275#define OPC_BSR         (0xbd | P_EXT)
276#define OPC_BSWAP	(0xc8 | P_EXT)
277#define OPC_CALL_Jz	(0xe8)
278#define OPC_CMOVCC      (0x40 | P_EXT)  /* ... plus condition code */
279#define OPC_CMP_GvEv	(OPC_ARITH_GvEv | (ARITH_CMP << 3))
280#define OPC_DEC_r32	(0x48)
281#define OPC_IMUL_GvEv	(0xaf | P_EXT)
282#define OPC_IMUL_GvEvIb	(0x6b)
283#define OPC_IMUL_GvEvIz	(0x69)
284#define OPC_INC_r32	(0x40)
285#define OPC_JCC_long	(0x80 | P_EXT)	/* ... plus condition code */
286#define OPC_JCC_short	(0x70)		/* ... plus condition code */
287#define OPC_JMP_long	(0xe9)
288#define OPC_JMP_short	(0xeb)
289#define OPC_LEA         (0x8d)
290#define OPC_LZCNT       (0xbd | P_EXT | P_SIMDF3)
291#define OPC_MOVB_EvGv	(0x88)		/* stores, more or less */
292#define OPC_MOVL_EvGv	(0x89)		/* stores, more or less */
293#define OPC_MOVL_GvEv	(0x8b)		/* loads, more or less */
294#define OPC_MOVB_EvIz   (0xc6)
295#define OPC_MOVL_EvIz	(0xc7)
296#define OPC_MOVL_Iv     (0xb8)
297#define OPC_MOVBE_GyMy  (0xf0 | P_EXT38)
298#define OPC_MOVBE_MyGy  (0xf1 | P_EXT38)
299#define OPC_MOVD_VyEy   (0x6e | P_EXT | P_DATA16)
300#define OPC_MOVD_EyVy   (0x7e | P_EXT | P_DATA16)
301#define OPC_MOVDDUP     (0x12 | P_EXT | P_SIMDF2)
302#define OPC_MOVDQA_VxWx (0x6f | P_EXT | P_DATA16)
303#define OPC_MOVDQA_WxVx (0x7f | P_EXT | P_DATA16)
304#define OPC_MOVDQU_VxWx (0x6f | P_EXT | P_SIMDF3)
305#define OPC_MOVDQU_WxVx (0x7f | P_EXT | P_SIMDF3)
306#define OPC_MOVQ_VqWq   (0x7e | P_EXT | P_SIMDF3)
307#define OPC_MOVQ_WqVq   (0xd6 | P_EXT | P_DATA16)
308#define OPC_MOVSBL	(0xbe | P_EXT)
309#define OPC_MOVSWL	(0xbf | P_EXT)
310#define OPC_MOVSLQ	(0x63 | P_REXW)
311#define OPC_MOVZBL	(0xb6 | P_EXT)
312#define OPC_MOVZWL	(0xb7 | P_EXT)
313#define OPC_PABSB       (0x1c | P_EXT38 | P_DATA16)
314#define OPC_PABSW       (0x1d | P_EXT38 | P_DATA16)
315#define OPC_PABSD       (0x1e | P_EXT38 | P_DATA16)
316#define OPC_VPABSQ      (0x1f | P_EXT38 | P_DATA16 | P_VEXW | P_EVEX)
317#define OPC_PACKSSDW    (0x6b | P_EXT | P_DATA16)
318#define OPC_PACKSSWB    (0x63 | P_EXT | P_DATA16)
319#define OPC_PACKUSDW    (0x2b | P_EXT38 | P_DATA16)
320#define OPC_PACKUSWB    (0x67 | P_EXT | P_DATA16)
321#define OPC_PADDB       (0xfc | P_EXT | P_DATA16)
322#define OPC_PADDW       (0xfd | P_EXT | P_DATA16)
323#define OPC_PADDD       (0xfe | P_EXT | P_DATA16)
324#define OPC_PADDQ       (0xd4 | P_EXT | P_DATA16)
325#define OPC_PADDSB      (0xec | P_EXT | P_DATA16)
326#define OPC_PADDSW      (0xed | P_EXT | P_DATA16)
327#define OPC_PADDUB      (0xdc | P_EXT | P_DATA16)
328#define OPC_PADDUW      (0xdd | P_EXT | P_DATA16)
329#define OPC_PAND        (0xdb | P_EXT | P_DATA16)
330#define OPC_PANDN       (0xdf | P_EXT | P_DATA16)
331#define OPC_PBLENDW     (0x0e | P_EXT3A | P_DATA16)
332#define OPC_PCMPEQB     (0x74 | P_EXT | P_DATA16)
333#define OPC_PCMPEQW     (0x75 | P_EXT | P_DATA16)
334#define OPC_PCMPEQD     (0x76 | P_EXT | P_DATA16)
335#define OPC_PCMPEQQ     (0x29 | P_EXT38 | P_DATA16)
336#define OPC_PCMPGTB     (0x64 | P_EXT | P_DATA16)
337#define OPC_PCMPGTW     (0x65 | P_EXT | P_DATA16)
338#define OPC_PCMPGTD     (0x66 | P_EXT | P_DATA16)
339#define OPC_PCMPGTQ     (0x37 | P_EXT38 | P_DATA16)
340#define OPC_PMAXSB      (0x3c | P_EXT38 | P_DATA16)
341#define OPC_PMAXSW      (0xee | P_EXT | P_DATA16)
342#define OPC_PMAXSD      (0x3d | P_EXT38 | P_DATA16)
343#define OPC_VPMAXSQ     (0x3d | P_EXT38 | P_DATA16 | P_VEXW | P_EVEX)
344#define OPC_PMAXUB      (0xde | P_EXT | P_DATA16)
345#define OPC_PMAXUW      (0x3e | P_EXT38 | P_DATA16)
346#define OPC_PMAXUD      (0x3f | P_EXT38 | P_DATA16)
347#define OPC_VPMAXUQ     (0x3f | P_EXT38 | P_DATA16 | P_VEXW | P_EVEX)
348#define OPC_PMINSB      (0x38 | P_EXT38 | P_DATA16)
349#define OPC_PMINSW      (0xea | P_EXT | P_DATA16)
350#define OPC_PMINSD      (0x39 | P_EXT38 | P_DATA16)
351#define OPC_VPMINSQ     (0x39 | P_EXT38 | P_DATA16 | P_VEXW | P_EVEX)
352#define OPC_PMINUB      (0xda | P_EXT | P_DATA16)
353#define OPC_PMINUW      (0x3a | P_EXT38 | P_DATA16)
354#define OPC_PMINUD      (0x3b | P_EXT38 | P_DATA16)
355#define OPC_VPMINUQ     (0x3b | P_EXT38 | P_DATA16 | P_VEXW | P_EVEX)
356#define OPC_PMOVSXBW    (0x20 | P_EXT38 | P_DATA16)
357#define OPC_PMOVSXWD    (0x23 | P_EXT38 | P_DATA16)
358#define OPC_PMOVSXDQ    (0x25 | P_EXT38 | P_DATA16)
359#define OPC_PMOVZXBW    (0x30 | P_EXT38 | P_DATA16)
360#define OPC_PMOVZXWD    (0x33 | P_EXT38 | P_DATA16)
361#define OPC_PMOVZXDQ    (0x35 | P_EXT38 | P_DATA16)
362#define OPC_PMULLW      (0xd5 | P_EXT | P_DATA16)
363#define OPC_PMULLD      (0x40 | P_EXT38 | P_DATA16)
364#define OPC_VPMULLQ     (0x40 | P_EXT38 | P_DATA16 | P_VEXW | P_EVEX)
365#define OPC_POR         (0xeb | P_EXT | P_DATA16)
366#define OPC_PSHUFB      (0x00 | P_EXT38 | P_DATA16)
367#define OPC_PSHUFD      (0x70 | P_EXT | P_DATA16)
368#define OPC_PSHUFLW     (0x70 | P_EXT | P_SIMDF2)
369#define OPC_PSHUFHW     (0x70 | P_EXT | P_SIMDF3)
370#define OPC_PSHIFTW_Ib  (0x71 | P_EXT | P_DATA16) /* /2 /6 /4 */
371#define OPC_PSHIFTD_Ib  (0x72 | P_EXT | P_DATA16) /* /1 /2 /6 /4 */
372#define OPC_PSHIFTQ_Ib  (0x73 | P_EXT | P_DATA16) /* /2 /6 /4 */
373#define OPC_PSLLW       (0xf1 | P_EXT | P_DATA16)
374#define OPC_PSLLD       (0xf2 | P_EXT | P_DATA16)
375#define OPC_PSLLQ       (0xf3 | P_EXT | P_DATA16)
376#define OPC_PSRAW       (0xe1 | P_EXT | P_DATA16)
377#define OPC_PSRAD       (0xe2 | P_EXT | P_DATA16)
378#define OPC_VPSRAQ      (0xe2 | P_EXT | P_DATA16 | P_VEXW | P_EVEX)
379#define OPC_PSRLW       (0xd1 | P_EXT | P_DATA16)
380#define OPC_PSRLD       (0xd2 | P_EXT | P_DATA16)
381#define OPC_PSRLQ       (0xd3 | P_EXT | P_DATA16)
382#define OPC_PSUBB       (0xf8 | P_EXT | P_DATA16)
383#define OPC_PSUBW       (0xf9 | P_EXT | P_DATA16)
384#define OPC_PSUBD       (0xfa | P_EXT | P_DATA16)
385#define OPC_PSUBQ       (0xfb | P_EXT | P_DATA16)
386#define OPC_PSUBSB      (0xe8 | P_EXT | P_DATA16)
387#define OPC_PSUBSW      (0xe9 | P_EXT | P_DATA16)
388#define OPC_PSUBUB      (0xd8 | P_EXT | P_DATA16)
389#define OPC_PSUBUW      (0xd9 | P_EXT | P_DATA16)
390#define OPC_PUNPCKLBW   (0x60 | P_EXT | P_DATA16)
391#define OPC_PUNPCKLWD   (0x61 | P_EXT | P_DATA16)
392#define OPC_PUNPCKLDQ   (0x62 | P_EXT | P_DATA16)
393#define OPC_PUNPCKLQDQ  (0x6c | P_EXT | P_DATA16)
394#define OPC_PUNPCKHBW   (0x68 | P_EXT | P_DATA16)
395#define OPC_PUNPCKHWD   (0x69 | P_EXT | P_DATA16)
396#define OPC_PUNPCKHDQ   (0x6a | P_EXT | P_DATA16)
397#define OPC_PUNPCKHQDQ  (0x6d | P_EXT | P_DATA16)
398#define OPC_PXOR        (0xef | P_EXT | P_DATA16)
399#define OPC_POP_r32	(0x58)
400#define OPC_POPCNT      (0xb8 | P_EXT | P_SIMDF3)
401#define OPC_PUSH_r32	(0x50)
402#define OPC_PUSH_Iv	(0x68)
403#define OPC_PUSH_Ib	(0x6a)
404#define OPC_RET		(0xc3)
405#define OPC_SETCC	(0x90 | P_EXT | P_REXB_RM) /* ... plus cc */
406#define OPC_SHIFT_1	(0xd1)
407#define OPC_SHIFT_Ib	(0xc1)
408#define OPC_SHIFT_cl	(0xd3)
409#define OPC_SARX        (0xf7 | P_EXT38 | P_SIMDF3)
410#define OPC_SHUFPS      (0xc6 | P_EXT)
411#define OPC_SHLX        (0xf7 | P_EXT38 | P_DATA16)
412#define OPC_SHRX        (0xf7 | P_EXT38 | P_SIMDF2)
413#define OPC_SHRD_Ib     (0xac | P_EXT)
414#define OPC_TESTL	(0x85)
415#define OPC_TZCNT       (0xbc | P_EXT | P_SIMDF3)
416#define OPC_UD2         (0x0b | P_EXT)
417#define OPC_VPBLENDD    (0x02 | P_EXT3A | P_DATA16)
418#define OPC_VPBLENDVB   (0x4c | P_EXT3A | P_DATA16)
419#define OPC_VPINSRB     (0x20 | P_EXT3A | P_DATA16)
420#define OPC_VPINSRW     (0xc4 | P_EXT | P_DATA16)
421#define OPC_VBROADCASTSS (0x18 | P_EXT38 | P_DATA16)
422#define OPC_VBROADCASTSD (0x19 | P_EXT38 | P_DATA16)
423#define OPC_VPBROADCASTB (0x78 | P_EXT38 | P_DATA16)
424#define OPC_VPBROADCASTW (0x79 | P_EXT38 | P_DATA16)
425#define OPC_VPBROADCASTD (0x58 | P_EXT38 | P_DATA16)
426#define OPC_VPBROADCASTQ (0x59 | P_EXT38 | P_DATA16)
427#define OPC_VPERMQ      (0x00 | P_EXT3A | P_DATA16 | P_VEXW)
428#define OPC_VPERM2I128  (0x46 | P_EXT3A | P_DATA16 | P_VEXL)
429#define OPC_VPROLVD     (0x15 | P_EXT38 | P_DATA16 | P_EVEX)
430#define OPC_VPROLVQ     (0x15 | P_EXT38 | P_DATA16 | P_VEXW | P_EVEX)
431#define OPC_VPRORVD     (0x14 | P_EXT38 | P_DATA16 | P_EVEX)
432#define OPC_VPRORVQ     (0x14 | P_EXT38 | P_DATA16 | P_VEXW | P_EVEX)
433#define OPC_VPSHLDW     (0x70 | P_EXT3A | P_DATA16 | P_VEXW | P_EVEX)
434#define OPC_VPSHLDD     (0x71 | P_EXT3A | P_DATA16 | P_EVEX)
435#define OPC_VPSHLDQ     (0x71 | P_EXT3A | P_DATA16 | P_VEXW | P_EVEX)
436#define OPC_VPSHLDVW    (0x70 | P_EXT38 | P_DATA16 | P_VEXW | P_EVEX)
437#define OPC_VPSHLDVD    (0x71 | P_EXT38 | P_DATA16 | P_EVEX)
438#define OPC_VPSHLDVQ    (0x71 | P_EXT38 | P_DATA16 | P_VEXW | P_EVEX)
439#define OPC_VPSHRDVW    (0x72 | P_EXT38 | P_DATA16 | P_VEXW | P_EVEX)
440#define OPC_VPSHRDVD    (0x73 | P_EXT38 | P_DATA16 | P_EVEX)
441#define OPC_VPSHRDVQ    (0x73 | P_EXT38 | P_DATA16 | P_VEXW | P_EVEX)
442#define OPC_VPSLLVW     (0x12 | P_EXT38 | P_DATA16 | P_VEXW | P_EVEX)
443#define OPC_VPSLLVD     (0x47 | P_EXT38 | P_DATA16)
444#define OPC_VPSLLVQ     (0x47 | P_EXT38 | P_DATA16 | P_VEXW)
445#define OPC_VPSRAVW     (0x11 | P_EXT38 | P_DATA16 | P_VEXW | P_EVEX)
446#define OPC_VPSRAVD     (0x46 | P_EXT38 | P_DATA16)
447#define OPC_VPSRAVQ     (0x46 | P_EXT38 | P_DATA16 | P_VEXW | P_EVEX)
448#define OPC_VPSRLVW     (0x10 | P_EXT38 | P_DATA16 | P_VEXW | P_EVEX)
449#define OPC_VPSRLVD     (0x45 | P_EXT38 | P_DATA16)
450#define OPC_VPSRLVQ     (0x45 | P_EXT38 | P_DATA16 | P_VEXW)
451#define OPC_VPTERNLOGQ  (0x25 | P_EXT3A | P_DATA16 | P_VEXW | P_EVEX)
452#define OPC_VZEROUPPER  (0x77 | P_EXT)
453#define OPC_XCHG_ax_r32	(0x90)
454
455#define OPC_GRP3_Eb     (0xf6)
456#define OPC_GRP3_Ev     (0xf7)
457#define OPC_GRP5        (0xff)
458#define OPC_GRP14       (0x73 | P_EXT | P_DATA16)
459
460/* Group 1 opcode extensions for 0x80-0x83.
461   These are also used as modifiers for OPC_ARITH.  */
462#define ARITH_ADD 0
463#define ARITH_OR  1
464#define ARITH_ADC 2
465#define ARITH_SBB 3
466#define ARITH_AND 4
467#define ARITH_SUB 5
468#define ARITH_XOR 6
469#define ARITH_CMP 7
470
471/* Group 2 opcode extensions for 0xc0, 0xc1, 0xd0-0xd3.  */
472#define SHIFT_ROL 0
473#define SHIFT_ROR 1
474#define SHIFT_SHL 4
475#define SHIFT_SHR 5
476#define SHIFT_SAR 7
477
478/* Group 3 opcode extensions for 0xf6, 0xf7.  To be used with OPC_GRP3.  */
479#define EXT3_TESTi 0
480#define EXT3_NOT   2
481#define EXT3_NEG   3
482#define EXT3_MUL   4
483#define EXT3_IMUL  5
484#define EXT3_DIV   6
485#define EXT3_IDIV  7
486
487/* Group 5 opcode extensions for 0xff.  To be used with OPC_GRP5.  */
488#define EXT5_INC_Ev	0
489#define EXT5_DEC_Ev	1
490#define EXT5_CALLN_Ev	2
491#define EXT5_JMPN_Ev	4
492
493/* Condition codes to be added to OPC_JCC_{long,short}.  */
494#define JCC_JMP (-1)
495#define JCC_JO  0x0
496#define JCC_JNO 0x1
497#define JCC_JB  0x2
498#define JCC_JAE 0x3
499#define JCC_JE  0x4
500#define JCC_JNE 0x5
501#define JCC_JBE 0x6
502#define JCC_JA  0x7
503#define JCC_JS  0x8
504#define JCC_JNS 0x9
505#define JCC_JP  0xa
506#define JCC_JNP 0xb
507#define JCC_JL  0xc
508#define JCC_JGE 0xd
509#define JCC_JLE 0xe
510#define JCC_JG  0xf
511
512static const uint8_t tcg_cond_to_jcc[] = {
513    [TCG_COND_EQ] = JCC_JE,
514    [TCG_COND_NE] = JCC_JNE,
515    [TCG_COND_LT] = JCC_JL,
516    [TCG_COND_GE] = JCC_JGE,
517    [TCG_COND_LE] = JCC_JLE,
518    [TCG_COND_GT] = JCC_JG,
519    [TCG_COND_LTU] = JCC_JB,
520    [TCG_COND_GEU] = JCC_JAE,
521    [TCG_COND_LEU] = JCC_JBE,
522    [TCG_COND_GTU] = JCC_JA,
523};
524
525#if TCG_TARGET_REG_BITS == 64
526static void tcg_out_opc(TCGContext *s, int opc, int r, int rm, int x)
527{
528    int rex;
529
530    if (opc & P_GS) {
531        tcg_out8(s, 0x65);
532    }
533    if (opc & P_DATA16) {
534        /* We should never be asking for both 16 and 64-bit operation.  */
535        tcg_debug_assert((opc & P_REXW) == 0);
536        tcg_out8(s, 0x66);
537    }
538    if (opc & P_SIMDF3) {
539        tcg_out8(s, 0xf3);
540    } else if (opc & P_SIMDF2) {
541        tcg_out8(s, 0xf2);
542    }
543
544    rex = 0;
545    rex |= (opc & P_REXW) ? 0x8 : 0x0;  /* REX.W */
546    rex |= (r & 8) >> 1;                /* REX.R */
547    rex |= (x & 8) >> 2;                /* REX.X */
548    rex |= (rm & 8) >> 3;               /* REX.B */
549
550    /* P_REXB_{R,RM} indicates that the given register is the low byte.
551       For %[abcd]l we need no REX prefix, but for %{si,di,bp,sp}l we do,
552       as otherwise the encoding indicates %[abcd]h.  Note that the values
553       that are ORed in merely indicate that the REX byte must be present;
554       those bits get discarded in output.  */
555    rex |= opc & (r >= 4 ? P_REXB_R : 0);
556    rex |= opc & (rm >= 4 ? P_REXB_RM : 0);
557
558    if (rex) {
559        tcg_out8(s, (uint8_t)(rex | 0x40));
560    }
561
562    if (opc & (P_EXT | P_EXT38 | P_EXT3A)) {
563        tcg_out8(s, 0x0f);
564        if (opc & P_EXT38) {
565            tcg_out8(s, 0x38);
566        } else if (opc & P_EXT3A) {
567            tcg_out8(s, 0x3a);
568        }
569    }
570
571    tcg_out8(s, opc);
572}
573#else
574static void tcg_out_opc(TCGContext *s, int opc)
575{
576    if (opc & P_DATA16) {
577        tcg_out8(s, 0x66);
578    }
579    if (opc & P_SIMDF3) {
580        tcg_out8(s, 0xf3);
581    } else if (opc & P_SIMDF2) {
582        tcg_out8(s, 0xf2);
583    }
584    if (opc & (P_EXT | P_EXT38 | P_EXT3A)) {
585        tcg_out8(s, 0x0f);
586        if (opc & P_EXT38) {
587            tcg_out8(s, 0x38);
588        } else if (opc & P_EXT3A) {
589            tcg_out8(s, 0x3a);
590        }
591    }
592    tcg_out8(s, opc);
593}
594/* Discard the register arguments to tcg_out_opc early, so as not to penalize
595   the 32-bit compilation paths.  This method works with all versions of gcc,
596   whereas relying on optimization may not be able to exclude them.  */
597#define tcg_out_opc(s, opc, r, rm, x)  (tcg_out_opc)(s, opc)
598#endif
599
600static void tcg_out_modrm(TCGContext *s, int opc, int r, int rm)
601{
602    tcg_out_opc(s, opc, r, rm, 0);
603    tcg_out8(s, 0xc0 | (LOWREGMASK(r) << 3) | LOWREGMASK(rm));
604}
605
606static void tcg_out_vex_opc(TCGContext *s, int opc, int r, int v,
607                            int rm, int index)
608{
609    int tmp;
610
611    /* Use the two byte form if possible, which cannot encode
612       VEX.W, VEX.B, VEX.X, or an m-mmmm field other than P_EXT.  */
613    if ((opc & (P_EXT | P_EXT38 | P_EXT3A | P_VEXW)) == P_EXT
614        && ((rm | index) & 8) == 0) {
615        /* Two byte VEX prefix.  */
616        tcg_out8(s, 0xc5);
617
618        tmp = (r & 8 ? 0 : 0x80);              /* VEX.R */
619    } else {
620        /* Three byte VEX prefix.  */
621        tcg_out8(s, 0xc4);
622
623        /* VEX.m-mmmm */
624        if (opc & P_EXT3A) {
625            tmp = 3;
626        } else if (opc & P_EXT38) {
627            tmp = 2;
628        } else if (opc & P_EXT) {
629            tmp = 1;
630        } else {
631            g_assert_not_reached();
632        }
633        tmp |= (r & 8 ? 0 : 0x80);             /* VEX.R */
634        tmp |= (index & 8 ? 0 : 0x40);         /* VEX.X */
635        tmp |= (rm & 8 ? 0 : 0x20);            /* VEX.B */
636        tcg_out8(s, tmp);
637
638        tmp = (opc & P_VEXW ? 0x80 : 0);       /* VEX.W */
639    }
640
641    tmp |= (opc & P_VEXL ? 0x04 : 0);      /* VEX.L */
642    /* VEX.pp */
643    if (opc & P_DATA16) {
644        tmp |= 1;                          /* 0x66 */
645    } else if (opc & P_SIMDF3) {
646        tmp |= 2;                          /* 0xf3 */
647    } else if (opc & P_SIMDF2) {
648        tmp |= 3;                          /* 0xf2 */
649    }
650    tmp |= (~v & 15) << 3;                 /* VEX.vvvv */
651    tcg_out8(s, tmp);
652    tcg_out8(s, opc);
653}
654
655static void tcg_out_evex_opc(TCGContext *s, int opc, int r, int v,
656                             int rm, int index)
657{
658    /* The entire 4-byte evex prefix; with R' and V' set. */
659    uint32_t p = 0x08041062;
660    int mm, pp;
661
662    tcg_debug_assert(have_avx512vl);
663
664    /* EVEX.mm */
665    if (opc & P_EXT3A) {
666        mm = 3;
667    } else if (opc & P_EXT38) {
668        mm = 2;
669    } else if (opc & P_EXT) {
670        mm = 1;
671    } else {
672        g_assert_not_reached();
673    }
674
675    /* EVEX.pp */
676    if (opc & P_DATA16) {
677        pp = 1;                          /* 0x66 */
678    } else if (opc & P_SIMDF3) {
679        pp = 2;                          /* 0xf3 */
680    } else if (opc & P_SIMDF2) {
681        pp = 3;                          /* 0xf2 */
682    } else {
683        pp = 0;
684    }
685
686    p = deposit32(p, 8, 2, mm);
687    p = deposit32(p, 13, 1, (rm & 8) == 0);             /* EVEX.RXB.B */
688    p = deposit32(p, 14, 1, (index & 8) == 0);          /* EVEX.RXB.X */
689    p = deposit32(p, 15, 1, (r & 8) == 0);              /* EVEX.RXB.R */
690    p = deposit32(p, 16, 2, pp);
691    p = deposit32(p, 19, 4, ~v);
692    p = deposit32(p, 23, 1, (opc & P_VEXW) != 0);
693    p = deposit32(p, 29, 2, (opc & P_VEXL) != 0);
694
695    tcg_out32(s, p);
696    tcg_out8(s, opc);
697}
698
699static void tcg_out_vex_modrm(TCGContext *s, int opc, int r, int v, int rm)
700{
701    if (opc & P_EVEX) {
702        tcg_out_evex_opc(s, opc, r, v, rm, 0);
703    } else {
704        tcg_out_vex_opc(s, opc, r, v, rm, 0);
705    }
706    tcg_out8(s, 0xc0 | (LOWREGMASK(r) << 3) | LOWREGMASK(rm));
707}
708
709/* Output an opcode with a full "rm + (index<<shift) + offset" address mode.
710   We handle either RM and INDEX missing with a negative value.  In 64-bit
711   mode for absolute addresses, ~RM is the size of the immediate operand
712   that will follow the instruction.  */
713
714static void tcg_out_sib_offset(TCGContext *s, int r, int rm, int index,
715                               int shift, intptr_t offset)
716{
717    int mod, len;
718
719    if (index < 0 && rm < 0) {
720        if (TCG_TARGET_REG_BITS == 64) {
721            /* Try for a rip-relative addressing mode.  This has replaced
722               the 32-bit-mode absolute addressing encoding.  */
723            intptr_t pc = (intptr_t)s->code_ptr + 5 + ~rm;
724            intptr_t disp = offset - pc;
725            if (disp == (int32_t)disp) {
726                tcg_out8(s, (LOWREGMASK(r) << 3) | 5);
727                tcg_out32(s, disp);
728                return;
729            }
730
731            /* Try for an absolute address encoding.  This requires the
732               use of the MODRM+SIB encoding and is therefore larger than
733               rip-relative addressing.  */
734            if (offset == (int32_t)offset) {
735                tcg_out8(s, (LOWREGMASK(r) << 3) | 4);
736                tcg_out8(s, (4 << 3) | 5);
737                tcg_out32(s, offset);
738                return;
739            }
740
741            /* ??? The memory isn't directly addressable.  */
742            g_assert_not_reached();
743        } else {
744            /* Absolute address.  */
745            tcg_out8(s, (r << 3) | 5);
746            tcg_out32(s, offset);
747            return;
748        }
749    }
750
751    /* Find the length of the immediate addend.  Note that the encoding
752       that would be used for (%ebp) indicates absolute addressing.  */
753    if (rm < 0) {
754        mod = 0, len = 4, rm = 5;
755    } else if (offset == 0 && LOWREGMASK(rm) != TCG_REG_EBP) {
756        mod = 0, len = 0;
757    } else if (offset == (int8_t)offset) {
758        mod = 0x40, len = 1;
759    } else {
760        mod = 0x80, len = 4;
761    }
762
763    /* Use a single byte MODRM format if possible.  Note that the encoding
764       that would be used for %esp is the escape to the two byte form.  */
765    if (index < 0 && LOWREGMASK(rm) != TCG_REG_ESP) {
766        /* Single byte MODRM format.  */
767        tcg_out8(s, mod | (LOWREGMASK(r) << 3) | LOWREGMASK(rm));
768    } else {
769        /* Two byte MODRM+SIB format.  */
770
771        /* Note that the encoding that would place %esp into the index
772           field indicates no index register.  In 64-bit mode, the REX.X
773           bit counts, so %r12 can be used as the index.  */
774        if (index < 0) {
775            index = 4;
776        } else {
777            tcg_debug_assert(index != TCG_REG_ESP);
778        }
779
780        tcg_out8(s, mod | (LOWREGMASK(r) << 3) | 4);
781        tcg_out8(s, (shift << 6) | (LOWREGMASK(index) << 3) | LOWREGMASK(rm));
782    }
783
784    if (len == 1) {
785        tcg_out8(s, offset);
786    } else if (len == 4) {
787        tcg_out32(s, offset);
788    }
789}
790
791static void tcg_out_modrm_sib_offset(TCGContext *s, int opc, int r, int rm,
792                                     int index, int shift, intptr_t offset)
793{
794    tcg_out_opc(s, opc, r, rm < 0 ? 0 : rm, index < 0 ? 0 : index);
795    tcg_out_sib_offset(s, r, rm, index, shift, offset);
796}
797
798static void tcg_out_vex_modrm_sib_offset(TCGContext *s, int opc, int r, int v,
799                                         int rm, int index, int shift,
800                                         intptr_t offset)
801{
802    tcg_out_vex_opc(s, opc, r, v, rm < 0 ? 0 : rm, index < 0 ? 0 : index);
803    tcg_out_sib_offset(s, r, rm, index, shift, offset);
804}
805
806/* A simplification of the above with no index or shift.  */
807static inline void tcg_out_modrm_offset(TCGContext *s, int opc, int r,
808                                        int rm, intptr_t offset)
809{
810    tcg_out_modrm_sib_offset(s, opc, r, rm, -1, 0, offset);
811}
812
813static inline void tcg_out_vex_modrm_offset(TCGContext *s, int opc, int r,
814                                            int v, int rm, intptr_t offset)
815{
816    tcg_out_vex_modrm_sib_offset(s, opc, r, v, rm, -1, 0, offset);
817}
818
819/* Output an opcode with an expected reference to the constant pool.  */
820static inline void tcg_out_modrm_pool(TCGContext *s, int opc, int r)
821{
822    tcg_out_opc(s, opc, r, 0, 0);
823    /* Absolute for 32-bit, pc-relative for 64-bit.  */
824    tcg_out8(s, LOWREGMASK(r) << 3 | 5);
825    tcg_out32(s, 0);
826}
827
828/* Output an opcode with an expected reference to the constant pool.  */
829static inline void tcg_out_vex_modrm_pool(TCGContext *s, int opc, int r)
830{
831    tcg_out_vex_opc(s, opc, r, 0, 0, 0);
832    /* Absolute for 32-bit, pc-relative for 64-bit.  */
833    tcg_out8(s, LOWREGMASK(r) << 3 | 5);
834    tcg_out32(s, 0);
835}
836
837/* Generate dest op= src.  Uses the same ARITH_* codes as tgen_arithi.  */
838static inline void tgen_arithr(TCGContext *s, int subop, int dest, int src)
839{
840    /* Propagate an opcode prefix, such as P_REXW.  */
841    int ext = subop & ~0x7;
842    subop &= 0x7;
843
844    tcg_out_modrm(s, OPC_ARITH_GvEv + (subop << 3) + ext, dest, src);
845}
846
847static bool tcg_out_mov(TCGContext *s, TCGType type, TCGReg ret, TCGReg arg)
848{
849    int rexw = 0;
850
851    if (arg == ret) {
852        return true;
853    }
854    switch (type) {
855    case TCG_TYPE_I64:
856        rexw = P_REXW;
857        /* fallthru */
858    case TCG_TYPE_I32:
859        if (ret < 16) {
860            if (arg < 16) {
861                tcg_out_modrm(s, OPC_MOVL_GvEv + rexw, ret, arg);
862            } else {
863                tcg_out_vex_modrm(s, OPC_MOVD_EyVy + rexw, arg, 0, ret);
864            }
865        } else {
866            if (arg < 16) {
867                tcg_out_vex_modrm(s, OPC_MOVD_VyEy + rexw, ret, 0, arg);
868            } else {
869                tcg_out_vex_modrm(s, OPC_MOVQ_VqWq, ret, 0, arg);
870            }
871        }
872        break;
873
874    case TCG_TYPE_V64:
875        tcg_debug_assert(ret >= 16 && arg >= 16);
876        tcg_out_vex_modrm(s, OPC_MOVQ_VqWq, ret, 0, arg);
877        break;
878    case TCG_TYPE_V128:
879        tcg_debug_assert(ret >= 16 && arg >= 16);
880        tcg_out_vex_modrm(s, OPC_MOVDQA_VxWx, ret, 0, arg);
881        break;
882    case TCG_TYPE_V256:
883        tcg_debug_assert(ret >= 16 && arg >= 16);
884        tcg_out_vex_modrm(s, OPC_MOVDQA_VxWx | P_VEXL, ret, 0, arg);
885        break;
886
887    default:
888        g_assert_not_reached();
889    }
890    return true;
891}
892
893static const int avx2_dup_insn[4] = {
894    OPC_VPBROADCASTB, OPC_VPBROADCASTW,
895    OPC_VPBROADCASTD, OPC_VPBROADCASTQ,
896};
897
898static bool tcg_out_dup_vec(TCGContext *s, TCGType type, unsigned vece,
899                            TCGReg r, TCGReg a)
900{
901    if (have_avx2) {
902        int vex_l = (type == TCG_TYPE_V256 ? P_VEXL : 0);
903        tcg_out_vex_modrm(s, avx2_dup_insn[vece] + vex_l, r, 0, a);
904    } else {
905        switch (vece) {
906        case MO_8:
907            /* ??? With zero in a register, use PSHUFB.  */
908            tcg_out_vex_modrm(s, OPC_PUNPCKLBW, r, a, a);
909            a = r;
910            /* FALLTHRU */
911        case MO_16:
912            tcg_out_vex_modrm(s, OPC_PUNPCKLWD, r, a, a);
913            a = r;
914            /* FALLTHRU */
915        case MO_32:
916            tcg_out_vex_modrm(s, OPC_PSHUFD, r, 0, a);
917            /* imm8 operand: all output lanes selected from input lane 0.  */
918            tcg_out8(s, 0);
919            break;
920        case MO_64:
921            tcg_out_vex_modrm(s, OPC_PUNPCKLQDQ, r, a, a);
922            break;
923        default:
924            g_assert_not_reached();
925        }
926    }
927    return true;
928}
929
930static bool tcg_out_dupm_vec(TCGContext *s, TCGType type, unsigned vece,
931                             TCGReg r, TCGReg base, intptr_t offset)
932{
933    if (have_avx2) {
934        int vex_l = (type == TCG_TYPE_V256 ? P_VEXL : 0);
935        tcg_out_vex_modrm_offset(s, avx2_dup_insn[vece] + vex_l,
936                                 r, 0, base, offset);
937    } else {
938        switch (vece) {
939        case MO_64:
940            tcg_out_vex_modrm_offset(s, OPC_MOVDDUP, r, 0, base, offset);
941            break;
942        case MO_32:
943            tcg_out_vex_modrm_offset(s, OPC_VBROADCASTSS, r, 0, base, offset);
944            break;
945        case MO_16:
946            tcg_out_vex_modrm_offset(s, OPC_VPINSRW, r, r, base, offset);
947            tcg_out8(s, 0); /* imm8 */
948            tcg_out_dup_vec(s, type, vece, r, r);
949            break;
950        case MO_8:
951            tcg_out_vex_modrm_offset(s, OPC_VPINSRB, r, r, base, offset);
952            tcg_out8(s, 0); /* imm8 */
953            tcg_out_dup_vec(s, type, vece, r, r);
954            break;
955        default:
956            g_assert_not_reached();
957        }
958    }
959    return true;
960}
961
962static void tcg_out_dupi_vec(TCGContext *s, TCGType type, unsigned vece,
963                             TCGReg ret, int64_t arg)
964{
965    int vex_l = (type == TCG_TYPE_V256 ? P_VEXL : 0);
966
967    if (arg == 0) {
968        tcg_out_vex_modrm(s, OPC_PXOR, ret, ret, ret);
969        return;
970    }
971    if (arg == -1) {
972        tcg_out_vex_modrm(s, OPC_PCMPEQB + vex_l, ret, ret, ret);
973        return;
974    }
975
976    if (TCG_TARGET_REG_BITS == 32 && vece < MO_64) {
977        if (have_avx2) {
978            tcg_out_vex_modrm_pool(s, OPC_VPBROADCASTD + vex_l, ret);
979        } else {
980            tcg_out_vex_modrm_pool(s, OPC_VBROADCASTSS, ret);
981        }
982        new_pool_label(s, arg, R_386_32, s->code_ptr - 4, 0);
983    } else {
984        if (type == TCG_TYPE_V64) {
985            tcg_out_vex_modrm_pool(s, OPC_MOVQ_VqWq, ret);
986        } else if (have_avx2) {
987            tcg_out_vex_modrm_pool(s, OPC_VPBROADCASTQ + vex_l, ret);
988        } else {
989            tcg_out_vex_modrm_pool(s, OPC_MOVDDUP, ret);
990        }
991        if (TCG_TARGET_REG_BITS == 64) {
992            new_pool_label(s, arg, R_386_PC32, s->code_ptr - 4, -4);
993        } else {
994            new_pool_l2(s, R_386_32, s->code_ptr - 4, 0, arg, arg >> 32);
995        }
996    }
997}
998
999static void tcg_out_movi_vec(TCGContext *s, TCGType type,
1000                             TCGReg ret, tcg_target_long arg)
1001{
1002    if (arg == 0) {
1003        tcg_out_vex_modrm(s, OPC_PXOR, ret, ret, ret);
1004        return;
1005    }
1006    if (arg == -1) {
1007        tcg_out_vex_modrm(s, OPC_PCMPEQB, ret, ret, ret);
1008        return;
1009    }
1010
1011    int rexw = (type == TCG_TYPE_I32 ? 0 : P_REXW);
1012    tcg_out_vex_modrm_pool(s, OPC_MOVD_VyEy + rexw, ret);
1013    if (TCG_TARGET_REG_BITS == 64) {
1014        new_pool_label(s, arg, R_386_PC32, s->code_ptr - 4, -4);
1015    } else {
1016        new_pool_label(s, arg, R_386_32, s->code_ptr - 4, 0);
1017    }
1018}
1019
1020static void tcg_out_movi_int(TCGContext *s, TCGType type,
1021                             TCGReg ret, tcg_target_long arg)
1022{
1023    tcg_target_long diff;
1024
1025    if (arg == 0) {
1026        tgen_arithr(s, ARITH_XOR, ret, ret);
1027        return;
1028    }
1029    if (arg == (uint32_t)arg || type == TCG_TYPE_I32) {
1030        tcg_out_opc(s, OPC_MOVL_Iv + LOWREGMASK(ret), 0, ret, 0);
1031        tcg_out32(s, arg);
1032        return;
1033    }
1034    if (arg == (int32_t)arg) {
1035        tcg_out_modrm(s, OPC_MOVL_EvIz + P_REXW, 0, ret);
1036        tcg_out32(s, arg);
1037        return;
1038    }
1039
1040    /* Try a 7 byte pc-relative lea before the 10 byte movq.  */
1041    diff = tcg_pcrel_diff(s, (const void *)arg) - 7;
1042    if (diff == (int32_t)diff) {
1043        tcg_out_opc(s, OPC_LEA | P_REXW, ret, 0, 0);
1044        tcg_out8(s, (LOWREGMASK(ret) << 3) | 5);
1045        tcg_out32(s, diff);
1046        return;
1047    }
1048
1049    tcg_out_opc(s, OPC_MOVL_Iv + P_REXW + LOWREGMASK(ret), 0, ret, 0);
1050    tcg_out64(s, arg);
1051}
1052
1053static void tcg_out_movi(TCGContext *s, TCGType type,
1054                         TCGReg ret, tcg_target_long arg)
1055{
1056    switch (type) {
1057    case TCG_TYPE_I32:
1058#if TCG_TARGET_REG_BITS == 64
1059    case TCG_TYPE_I64:
1060#endif
1061        if (ret < 16) {
1062            tcg_out_movi_int(s, type, ret, arg);
1063        } else {
1064            tcg_out_movi_vec(s, type, ret, arg);
1065        }
1066        break;
1067    default:
1068        g_assert_not_reached();
1069    }
1070}
1071
1072static inline void tcg_out_pushi(TCGContext *s, tcg_target_long val)
1073{
1074    if (val == (int8_t)val) {
1075        tcg_out_opc(s, OPC_PUSH_Ib, 0, 0, 0);
1076        tcg_out8(s, val);
1077    } else if (val == (int32_t)val) {
1078        tcg_out_opc(s, OPC_PUSH_Iv, 0, 0, 0);
1079        tcg_out32(s, val);
1080    } else {
1081        tcg_abort();
1082    }
1083}
1084
1085static inline void tcg_out_mb(TCGContext *s, TCGArg a0)
1086{
1087    /* Given the strength of x86 memory ordering, we only need care for
1088       store-load ordering.  Experimentally, "lock orl $0,0(%esp)" is
1089       faster than "mfence", so don't bother with the sse insn.  */
1090    if (a0 & TCG_MO_ST_LD) {
1091        tcg_out8(s, 0xf0);
1092        tcg_out_modrm_offset(s, OPC_ARITH_EvIb, ARITH_OR, TCG_REG_ESP, 0);
1093        tcg_out8(s, 0);
1094    }
1095}
1096
1097static inline void tcg_out_push(TCGContext *s, int reg)
1098{
1099    tcg_out_opc(s, OPC_PUSH_r32 + LOWREGMASK(reg), 0, reg, 0);
1100}
1101
1102static inline void tcg_out_pop(TCGContext *s, int reg)
1103{
1104    tcg_out_opc(s, OPC_POP_r32 + LOWREGMASK(reg), 0, reg, 0);
1105}
1106
1107static void tcg_out_ld(TCGContext *s, TCGType type, TCGReg ret,
1108                       TCGReg arg1, intptr_t arg2)
1109{
1110    switch (type) {
1111    case TCG_TYPE_I32:
1112        if (ret < 16) {
1113            tcg_out_modrm_offset(s, OPC_MOVL_GvEv, ret, arg1, arg2);
1114        } else {
1115            tcg_out_vex_modrm_offset(s, OPC_MOVD_VyEy, ret, 0, arg1, arg2);
1116        }
1117        break;
1118    case TCG_TYPE_I64:
1119        if (ret < 16) {
1120            tcg_out_modrm_offset(s, OPC_MOVL_GvEv | P_REXW, ret, arg1, arg2);
1121            break;
1122        }
1123        /* FALLTHRU */
1124    case TCG_TYPE_V64:
1125        /* There is no instruction that can validate 8-byte alignment.  */
1126        tcg_debug_assert(ret >= 16);
1127        tcg_out_vex_modrm_offset(s, OPC_MOVQ_VqWq, ret, 0, arg1, arg2);
1128        break;
1129    case TCG_TYPE_V128:
1130        /*
1131         * The gvec infrastructure is asserts that v128 vector loads
1132         * and stores use a 16-byte aligned offset.  Validate that the
1133         * final pointer is aligned by using an insn that will SIGSEGV.
1134         */
1135        tcg_debug_assert(ret >= 16);
1136        tcg_out_vex_modrm_offset(s, OPC_MOVDQA_VxWx, ret, 0, arg1, arg2);
1137        break;
1138    case TCG_TYPE_V256:
1139        /*
1140         * The gvec infrastructure only requires 16-byte alignment,
1141         * so here we must use an unaligned load.
1142         */
1143        tcg_debug_assert(ret >= 16);
1144        tcg_out_vex_modrm_offset(s, OPC_MOVDQU_VxWx | P_VEXL,
1145                                 ret, 0, arg1, arg2);
1146        break;
1147    default:
1148        g_assert_not_reached();
1149    }
1150}
1151
1152static void tcg_out_st(TCGContext *s, TCGType type, TCGReg arg,
1153                       TCGReg arg1, intptr_t arg2)
1154{
1155    switch (type) {
1156    case TCG_TYPE_I32:
1157        if (arg < 16) {
1158            tcg_out_modrm_offset(s, OPC_MOVL_EvGv, arg, arg1, arg2);
1159        } else {
1160            tcg_out_vex_modrm_offset(s, OPC_MOVD_EyVy, arg, 0, arg1, arg2);
1161        }
1162        break;
1163    case TCG_TYPE_I64:
1164        if (arg < 16) {
1165            tcg_out_modrm_offset(s, OPC_MOVL_EvGv | P_REXW, arg, arg1, arg2);
1166            break;
1167        }
1168        /* FALLTHRU */
1169    case TCG_TYPE_V64:
1170        /* There is no instruction that can validate 8-byte alignment.  */
1171        tcg_debug_assert(arg >= 16);
1172        tcg_out_vex_modrm_offset(s, OPC_MOVQ_WqVq, arg, 0, arg1, arg2);
1173        break;
1174    case TCG_TYPE_V128:
1175        /*
1176         * The gvec infrastructure is asserts that v128 vector loads
1177         * and stores use a 16-byte aligned offset.  Validate that the
1178         * final pointer is aligned by using an insn that will SIGSEGV.
1179         */
1180        tcg_debug_assert(arg >= 16);
1181        tcg_out_vex_modrm_offset(s, OPC_MOVDQA_WxVx, arg, 0, arg1, arg2);
1182        break;
1183    case TCG_TYPE_V256:
1184        /*
1185         * The gvec infrastructure only requires 16-byte alignment,
1186         * so here we must use an unaligned store.
1187         */
1188        tcg_debug_assert(arg >= 16);
1189        tcg_out_vex_modrm_offset(s, OPC_MOVDQU_WxVx | P_VEXL,
1190                                 arg, 0, arg1, arg2);
1191        break;
1192    default:
1193        g_assert_not_reached();
1194    }
1195}
1196
1197static bool tcg_out_sti(TCGContext *s, TCGType type, TCGArg val,
1198                        TCGReg base, intptr_t ofs)
1199{
1200    int rexw = 0;
1201    if (TCG_TARGET_REG_BITS == 64 && type == TCG_TYPE_I64) {
1202        if (val != (int32_t)val) {
1203            return false;
1204        }
1205        rexw = P_REXW;
1206    } else if (type != TCG_TYPE_I32) {
1207        return false;
1208    }
1209    tcg_out_modrm_offset(s, OPC_MOVL_EvIz | rexw, 0, base, ofs);
1210    tcg_out32(s, val);
1211    return true;
1212}
1213
1214static void tcg_out_shifti(TCGContext *s, int subopc, int reg, int count)
1215{
1216    /* Propagate an opcode prefix, such as P_DATA16.  */
1217    int ext = subopc & ~0x7;
1218    subopc &= 0x7;
1219
1220    if (count == 1) {
1221        tcg_out_modrm(s, OPC_SHIFT_1 + ext, subopc, reg);
1222    } else {
1223        tcg_out_modrm(s, OPC_SHIFT_Ib + ext, subopc, reg);
1224        tcg_out8(s, count);
1225    }
1226}
1227
1228static inline void tcg_out_bswap32(TCGContext *s, int reg)
1229{
1230    tcg_out_opc(s, OPC_BSWAP + LOWREGMASK(reg), 0, reg, 0);
1231}
1232
1233static inline void tcg_out_rolw_8(TCGContext *s, int reg)
1234{
1235    tcg_out_shifti(s, SHIFT_ROL + P_DATA16, reg, 8);
1236}
1237
1238static inline void tcg_out_ext8u(TCGContext *s, int dest, int src)
1239{
1240    /* movzbl */
1241    tcg_debug_assert(src < 4 || TCG_TARGET_REG_BITS == 64);
1242    tcg_out_modrm(s, OPC_MOVZBL + P_REXB_RM, dest, src);
1243}
1244
1245static void tcg_out_ext8s(TCGContext *s, int dest, int src, int rexw)
1246{
1247    /* movsbl */
1248    tcg_debug_assert(src < 4 || TCG_TARGET_REG_BITS == 64);
1249    tcg_out_modrm(s, OPC_MOVSBL + P_REXB_RM + rexw, dest, src);
1250}
1251
1252static inline void tcg_out_ext16u(TCGContext *s, int dest, int src)
1253{
1254    /* movzwl */
1255    tcg_out_modrm(s, OPC_MOVZWL, dest, src);
1256}
1257
1258static inline void tcg_out_ext16s(TCGContext *s, int dest, int src, int rexw)
1259{
1260    /* movsw[lq] */
1261    tcg_out_modrm(s, OPC_MOVSWL + rexw, dest, src);
1262}
1263
1264static inline void tcg_out_ext32u(TCGContext *s, int dest, int src)
1265{
1266    /* 32-bit mov zero extends.  */
1267    tcg_out_modrm(s, OPC_MOVL_GvEv, dest, src);
1268}
1269
1270static inline void tcg_out_ext32s(TCGContext *s, int dest, int src)
1271{
1272    tcg_out_modrm(s, OPC_MOVSLQ, dest, src);
1273}
1274
1275static inline void tcg_out_bswap64(TCGContext *s, int reg)
1276{
1277    tcg_out_opc(s, OPC_BSWAP + P_REXW + LOWREGMASK(reg), 0, reg, 0);
1278}
1279
1280static void tgen_arithi(TCGContext *s, int c, int r0,
1281                        tcg_target_long val, int cf)
1282{
1283    int rexw = 0;
1284
1285    if (TCG_TARGET_REG_BITS == 64) {
1286        rexw = c & -8;
1287        c &= 7;
1288    }
1289
1290    /* ??? While INC is 2 bytes shorter than ADDL $1, they also induce
1291       partial flags update stalls on Pentium4 and are not recommended
1292       by current Intel optimization manuals.  */
1293    if (!cf && (c == ARITH_ADD || c == ARITH_SUB) && (val == 1 || val == -1)) {
1294        int is_inc = (c == ARITH_ADD) ^ (val < 0);
1295        if (TCG_TARGET_REG_BITS == 64) {
1296            /* The single-byte increment encodings are re-tasked as the
1297               REX prefixes.  Use the MODRM encoding.  */
1298            tcg_out_modrm(s, OPC_GRP5 + rexw,
1299                          (is_inc ? EXT5_INC_Ev : EXT5_DEC_Ev), r0);
1300        } else {
1301            tcg_out8(s, (is_inc ? OPC_INC_r32 : OPC_DEC_r32) + r0);
1302        }
1303        return;
1304    }
1305
1306    if (c == ARITH_AND) {
1307        if (TCG_TARGET_REG_BITS == 64) {
1308            if (val == 0xffffffffu) {
1309                tcg_out_ext32u(s, r0, r0);
1310                return;
1311            }
1312            if (val == (uint32_t)val) {
1313                /* AND with no high bits set can use a 32-bit operation.  */
1314                rexw = 0;
1315            }
1316        }
1317        if (val == 0xffu && (r0 < 4 || TCG_TARGET_REG_BITS == 64)) {
1318            tcg_out_ext8u(s, r0, r0);
1319            return;
1320        }
1321        if (val == 0xffffu) {
1322            tcg_out_ext16u(s, r0, r0);
1323            return;
1324        }
1325    }
1326
1327    if (val == (int8_t)val) {
1328        tcg_out_modrm(s, OPC_ARITH_EvIb + rexw, c, r0);
1329        tcg_out8(s, val);
1330        return;
1331    }
1332    if (rexw == 0 || val == (int32_t)val) {
1333        tcg_out_modrm(s, OPC_ARITH_EvIz + rexw, c, r0);
1334        tcg_out32(s, val);
1335        return;
1336    }
1337
1338    tcg_abort();
1339}
1340
1341static void tcg_out_addi(TCGContext *s, int reg, tcg_target_long val)
1342{
1343    if (val != 0) {
1344        tgen_arithi(s, ARITH_ADD + P_REXW, reg, val, 0);
1345    }
1346}
1347
1348/* Use SMALL != 0 to force a short forward branch.  */
1349static void tcg_out_jxx(TCGContext *s, int opc, TCGLabel *l, int small)
1350{
1351    int32_t val, val1;
1352
1353    if (l->has_value) {
1354        val = tcg_pcrel_diff(s, l->u.value_ptr);
1355        val1 = val - 2;
1356        if ((int8_t)val1 == val1) {
1357            if (opc == -1) {
1358                tcg_out8(s, OPC_JMP_short);
1359            } else {
1360                tcg_out8(s, OPC_JCC_short + opc);
1361            }
1362            tcg_out8(s, val1);
1363        } else {
1364            if (small) {
1365                tcg_abort();
1366            }
1367            if (opc == -1) {
1368                tcg_out8(s, OPC_JMP_long);
1369                tcg_out32(s, val - 5);
1370            } else {
1371                tcg_out_opc(s, OPC_JCC_long + opc, 0, 0, 0);
1372                tcg_out32(s, val - 6);
1373            }
1374        }
1375    } else if (small) {
1376        if (opc == -1) {
1377            tcg_out8(s, OPC_JMP_short);
1378        } else {
1379            tcg_out8(s, OPC_JCC_short + opc);
1380        }
1381        tcg_out_reloc(s, s->code_ptr, R_386_PC8, l, -1);
1382        s->code_ptr += 1;
1383    } else {
1384        if (opc == -1) {
1385            tcg_out8(s, OPC_JMP_long);
1386        } else {
1387            tcg_out_opc(s, OPC_JCC_long + opc, 0, 0, 0);
1388        }
1389        tcg_out_reloc(s, s->code_ptr, R_386_PC32, l, -4);
1390        s->code_ptr += 4;
1391    }
1392}
1393
1394static void tcg_out_cmp(TCGContext *s, TCGArg arg1, TCGArg arg2,
1395                        int const_arg2, int rexw)
1396{
1397    if (const_arg2) {
1398        if (arg2 == 0) {
1399            /* test r, r */
1400            tcg_out_modrm(s, OPC_TESTL + rexw, arg1, arg1);
1401        } else {
1402            tgen_arithi(s, ARITH_CMP + rexw, arg1, arg2, 0);
1403        }
1404    } else {
1405        tgen_arithr(s, ARITH_CMP + rexw, arg1, arg2);
1406    }
1407}
1408
1409static void tcg_out_brcond32(TCGContext *s, TCGCond cond,
1410                             TCGArg arg1, TCGArg arg2, int const_arg2,
1411                             TCGLabel *label, int small)
1412{
1413    tcg_out_cmp(s, arg1, arg2, const_arg2, 0);
1414    tcg_out_jxx(s, tcg_cond_to_jcc[cond], label, small);
1415}
1416
1417#if TCG_TARGET_REG_BITS == 64
1418static void tcg_out_brcond64(TCGContext *s, TCGCond cond,
1419                             TCGArg arg1, TCGArg arg2, int const_arg2,
1420                             TCGLabel *label, int small)
1421{
1422    tcg_out_cmp(s, arg1, arg2, const_arg2, P_REXW);
1423    tcg_out_jxx(s, tcg_cond_to_jcc[cond], label, small);
1424}
1425#else
1426/* XXX: we implement it at the target level to avoid having to
1427   handle cross basic blocks temporaries */
1428static void tcg_out_brcond2(TCGContext *s, const TCGArg *args,
1429                            const int *const_args, int small)
1430{
1431    TCGLabel *label_next = gen_new_label();
1432    TCGLabel *label_this = arg_label(args[5]);
1433
1434    switch(args[4]) {
1435    case TCG_COND_EQ:
1436        tcg_out_brcond32(s, TCG_COND_NE, args[0], args[2], const_args[2],
1437                         label_next, 1);
1438        tcg_out_brcond32(s, TCG_COND_EQ, args[1], args[3], const_args[3],
1439                         label_this, small);
1440        break;
1441    case TCG_COND_NE:
1442        tcg_out_brcond32(s, TCG_COND_NE, args[0], args[2], const_args[2],
1443                         label_this, small);
1444        tcg_out_brcond32(s, TCG_COND_NE, args[1], args[3], const_args[3],
1445                         label_this, small);
1446        break;
1447    case TCG_COND_LT:
1448        tcg_out_brcond32(s, TCG_COND_LT, args[1], args[3], const_args[3],
1449                         label_this, small);
1450        tcg_out_jxx(s, JCC_JNE, label_next, 1);
1451        tcg_out_brcond32(s, TCG_COND_LTU, args[0], args[2], const_args[2],
1452                         label_this, small);
1453        break;
1454    case TCG_COND_LE:
1455        tcg_out_brcond32(s, TCG_COND_LT, args[1], args[3], const_args[3],
1456                         label_this, small);
1457        tcg_out_jxx(s, JCC_JNE, label_next, 1);
1458        tcg_out_brcond32(s, TCG_COND_LEU, args[0], args[2], const_args[2],
1459                         label_this, small);
1460        break;
1461    case TCG_COND_GT:
1462        tcg_out_brcond32(s, TCG_COND_GT, args[1], args[3], const_args[3],
1463                         label_this, small);
1464        tcg_out_jxx(s, JCC_JNE, label_next, 1);
1465        tcg_out_brcond32(s, TCG_COND_GTU, args[0], args[2], const_args[2],
1466                         label_this, small);
1467        break;
1468    case TCG_COND_GE:
1469        tcg_out_brcond32(s, TCG_COND_GT, args[1], args[3], const_args[3],
1470                         label_this, small);
1471        tcg_out_jxx(s, JCC_JNE, label_next, 1);
1472        tcg_out_brcond32(s, TCG_COND_GEU, args[0], args[2], const_args[2],
1473                         label_this, small);
1474        break;
1475    case TCG_COND_LTU:
1476        tcg_out_brcond32(s, TCG_COND_LTU, args[1], args[3], const_args[3],
1477                         label_this, small);
1478        tcg_out_jxx(s, JCC_JNE, label_next, 1);
1479        tcg_out_brcond32(s, TCG_COND_LTU, args[0], args[2], const_args[2],
1480                         label_this, small);
1481        break;
1482    case TCG_COND_LEU:
1483        tcg_out_brcond32(s, TCG_COND_LTU, args[1], args[3], const_args[3],
1484                         label_this, small);
1485        tcg_out_jxx(s, JCC_JNE, label_next, 1);
1486        tcg_out_brcond32(s, TCG_COND_LEU, args[0], args[2], const_args[2],
1487                         label_this, small);
1488        break;
1489    case TCG_COND_GTU:
1490        tcg_out_brcond32(s, TCG_COND_GTU, args[1], args[3], const_args[3],
1491                         label_this, small);
1492        tcg_out_jxx(s, JCC_JNE, label_next, 1);
1493        tcg_out_brcond32(s, TCG_COND_GTU, args[0], args[2], const_args[2],
1494                         label_this, small);
1495        break;
1496    case TCG_COND_GEU:
1497        tcg_out_brcond32(s, TCG_COND_GTU, args[1], args[3], const_args[3],
1498                         label_this, small);
1499        tcg_out_jxx(s, JCC_JNE, label_next, 1);
1500        tcg_out_brcond32(s, TCG_COND_GEU, args[0], args[2], const_args[2],
1501                         label_this, small);
1502        break;
1503    default:
1504        tcg_abort();
1505    }
1506    tcg_out_label(s, label_next);
1507}
1508#endif
1509
1510static void tcg_out_setcond32(TCGContext *s, TCGCond cond, TCGArg dest,
1511                              TCGArg arg1, TCGArg arg2, int const_arg2)
1512{
1513    tcg_out_cmp(s, arg1, arg2, const_arg2, 0);
1514    tcg_out_modrm(s, OPC_SETCC | tcg_cond_to_jcc[cond], 0, dest);
1515    tcg_out_ext8u(s, dest, dest);
1516}
1517
1518#if TCG_TARGET_REG_BITS == 64
1519static void tcg_out_setcond64(TCGContext *s, TCGCond cond, TCGArg dest,
1520                              TCGArg arg1, TCGArg arg2, int const_arg2)
1521{
1522    tcg_out_cmp(s, arg1, arg2, const_arg2, P_REXW);
1523    tcg_out_modrm(s, OPC_SETCC | tcg_cond_to_jcc[cond], 0, dest);
1524    tcg_out_ext8u(s, dest, dest);
1525}
1526#else
1527static void tcg_out_setcond2(TCGContext *s, const TCGArg *args,
1528                             const int *const_args)
1529{
1530    TCGArg new_args[6];
1531    TCGLabel *label_true, *label_over;
1532
1533    memcpy(new_args, args+1, 5*sizeof(TCGArg));
1534
1535    if (args[0] == args[1] || args[0] == args[2]
1536        || (!const_args[3] && args[0] == args[3])
1537        || (!const_args[4] && args[0] == args[4])) {
1538        /* When the destination overlaps with one of the argument
1539           registers, don't do anything tricky.  */
1540        label_true = gen_new_label();
1541        label_over = gen_new_label();
1542
1543        new_args[5] = label_arg(label_true);
1544        tcg_out_brcond2(s, new_args, const_args+1, 1);
1545
1546        tcg_out_movi(s, TCG_TYPE_I32, args[0], 0);
1547        tcg_out_jxx(s, JCC_JMP, label_over, 1);
1548        tcg_out_label(s, label_true);
1549
1550        tcg_out_movi(s, TCG_TYPE_I32, args[0], 1);
1551        tcg_out_label(s, label_over);
1552    } else {
1553        /* When the destination does not overlap one of the arguments,
1554           clear the destination first, jump if cond false, and emit an
1555           increment in the true case.  This results in smaller code.  */
1556
1557        tcg_out_movi(s, TCG_TYPE_I32, args[0], 0);
1558
1559        label_over = gen_new_label();
1560        new_args[4] = tcg_invert_cond(new_args[4]);
1561        new_args[5] = label_arg(label_over);
1562        tcg_out_brcond2(s, new_args, const_args+1, 1);
1563
1564        tgen_arithi(s, ARITH_ADD, args[0], 1, 0);
1565        tcg_out_label(s, label_over);
1566    }
1567}
1568#endif
1569
1570static void tcg_out_cmov(TCGContext *s, TCGCond cond, int rexw,
1571                         TCGReg dest, TCGReg v1)
1572{
1573    if (have_cmov) {
1574        tcg_out_modrm(s, OPC_CMOVCC | tcg_cond_to_jcc[cond] | rexw, dest, v1);
1575    } else {
1576        TCGLabel *over = gen_new_label();
1577        tcg_out_jxx(s, tcg_cond_to_jcc[tcg_invert_cond(cond)], over, 1);
1578        tcg_out_mov(s, TCG_TYPE_I32, dest, v1);
1579        tcg_out_label(s, over);
1580    }
1581}
1582
1583static void tcg_out_movcond32(TCGContext *s, TCGCond cond, TCGReg dest,
1584                              TCGReg c1, TCGArg c2, int const_c2,
1585                              TCGReg v1)
1586{
1587    tcg_out_cmp(s, c1, c2, const_c2, 0);
1588    tcg_out_cmov(s, cond, 0, dest, v1);
1589}
1590
1591#if TCG_TARGET_REG_BITS == 64
1592static void tcg_out_movcond64(TCGContext *s, TCGCond cond, TCGReg dest,
1593                              TCGReg c1, TCGArg c2, int const_c2,
1594                              TCGReg v1)
1595{
1596    tcg_out_cmp(s, c1, c2, const_c2, P_REXW);
1597    tcg_out_cmov(s, cond, P_REXW, dest, v1);
1598}
1599#endif
1600
1601static void tcg_out_ctz(TCGContext *s, int rexw, TCGReg dest, TCGReg arg1,
1602                        TCGArg arg2, bool const_a2)
1603{
1604    if (have_bmi1) {
1605        tcg_out_modrm(s, OPC_TZCNT + rexw, dest, arg1);
1606        if (const_a2) {
1607            tcg_debug_assert(arg2 == (rexw ? 64 : 32));
1608        } else {
1609            tcg_debug_assert(dest != arg2);
1610            tcg_out_cmov(s, TCG_COND_LTU, rexw, dest, arg2);
1611        }
1612    } else {
1613        tcg_debug_assert(dest != arg2);
1614        tcg_out_modrm(s, OPC_BSF + rexw, dest, arg1);
1615        tcg_out_cmov(s, TCG_COND_EQ, rexw, dest, arg2);
1616    }
1617}
1618
1619static void tcg_out_clz(TCGContext *s, int rexw, TCGReg dest, TCGReg arg1,
1620                        TCGArg arg2, bool const_a2)
1621{
1622    if (have_lzcnt) {
1623        tcg_out_modrm(s, OPC_LZCNT + rexw, dest, arg1);
1624        if (const_a2) {
1625            tcg_debug_assert(arg2 == (rexw ? 64 : 32));
1626        } else {
1627            tcg_debug_assert(dest != arg2);
1628            tcg_out_cmov(s, TCG_COND_LTU, rexw, dest, arg2);
1629        }
1630    } else {
1631        tcg_debug_assert(!const_a2);
1632        tcg_debug_assert(dest != arg1);
1633        tcg_debug_assert(dest != arg2);
1634
1635        /* Recall that the output of BSR is the index not the count.  */
1636        tcg_out_modrm(s, OPC_BSR + rexw, dest, arg1);
1637        tgen_arithi(s, ARITH_XOR + rexw, dest, rexw ? 63 : 31, 0);
1638
1639        /* Since we have destroyed the flags from BSR, we have to re-test.  */
1640        tcg_out_cmp(s, arg1, 0, 1, rexw);
1641        tcg_out_cmov(s, TCG_COND_EQ, rexw, dest, arg2);
1642    }
1643}
1644
1645static void tcg_out_branch(TCGContext *s, int call, const tcg_insn_unit *dest)
1646{
1647    intptr_t disp = tcg_pcrel_diff(s, dest) - 5;
1648
1649    if (disp == (int32_t)disp) {
1650        tcg_out_opc(s, call ? OPC_CALL_Jz : OPC_JMP_long, 0, 0, 0);
1651        tcg_out32(s, disp);
1652    } else {
1653        /* rip-relative addressing into the constant pool.
1654           This is 6 + 8 = 14 bytes, as compared to using an
1655           immediate load 10 + 6 = 16 bytes, plus we may
1656           be able to re-use the pool constant for more calls.  */
1657        tcg_out_opc(s, OPC_GRP5, 0, 0, 0);
1658        tcg_out8(s, (call ? EXT5_CALLN_Ev : EXT5_JMPN_Ev) << 3 | 5);
1659        new_pool_label(s, (uintptr_t)dest, R_386_PC32, s->code_ptr, -4);
1660        tcg_out32(s, 0);
1661    }
1662}
1663
1664static inline void tcg_out_call(TCGContext *s, const tcg_insn_unit *dest)
1665{
1666    tcg_out_branch(s, 1, dest);
1667}
1668
1669static void tcg_out_jmp(TCGContext *s, const tcg_insn_unit *dest)
1670{
1671    tcg_out_branch(s, 0, dest);
1672}
1673
1674static void tcg_out_nopn(TCGContext *s, int n)
1675{
1676    int i;
1677    /* Emit 1 or 2 operand size prefixes for the standard one byte nop,
1678     * "xchg %eax,%eax", forming "xchg %ax,%ax". All cores accept the
1679     * duplicate prefix, and all of the interesting recent cores can
1680     * decode and discard the duplicates in a single cycle.
1681     */
1682    tcg_debug_assert(n >= 1);
1683    for (i = 1; i < n; ++i) {
1684        tcg_out8(s, 0x66);
1685    }
1686    tcg_out8(s, 0x90);
1687}
1688
1689#if defined(CONFIG_SOFTMMU)
1690/* helper signature: helper_ret_ld_mmu(CPUState *env, target_ulong addr,
1691 *                                     int mmu_idx, uintptr_t ra)
1692 */
1693static void * const qemu_ld_helpers[(MO_SIZE | MO_BSWAP) + 1] = {
1694    [MO_UB]   = helper_ret_ldub_mmu,
1695    [MO_LEUW] = helper_le_lduw_mmu,
1696    [MO_LEUL] = helper_le_ldul_mmu,
1697    [MO_LEUQ] = helper_le_ldq_mmu,
1698    [MO_BEUW] = helper_be_lduw_mmu,
1699    [MO_BEUL] = helper_be_ldul_mmu,
1700    [MO_BEUQ] = helper_be_ldq_mmu,
1701};
1702
1703/* helper signature: helper_ret_st_mmu(CPUState *env, target_ulong addr,
1704 *                                     uintxx_t val, int mmu_idx, uintptr_t ra)
1705 */
1706static void * const qemu_st_helpers[(MO_SIZE | MO_BSWAP) + 1] = {
1707    [MO_UB]   = helper_ret_stb_mmu,
1708    [MO_LEUW] = helper_le_stw_mmu,
1709    [MO_LEUL] = helper_le_stl_mmu,
1710    [MO_LEUQ] = helper_le_stq_mmu,
1711    [MO_BEUW] = helper_be_stw_mmu,
1712    [MO_BEUL] = helper_be_stl_mmu,
1713    [MO_BEUQ] = helper_be_stq_mmu,
1714};
1715
1716/* Perform the TLB load and compare.
1717
1718   Inputs:
1719   ADDRLO and ADDRHI contain the low and high part of the address.
1720
1721   MEM_INDEX and S_BITS are the memory context and log2 size of the load.
1722
1723   WHICH is the offset into the CPUTLBEntry structure of the slot to read.
1724   This should be offsetof addr_read or addr_write.
1725
1726   Outputs:
1727   LABEL_PTRS is filled with 1 (32-bit addresses) or 2 (64-bit addresses)
1728   positions of the displacements of forward jumps to the TLB miss case.
1729
1730   Second argument register is loaded with the low part of the address.
1731   In the TLB hit case, it has been adjusted as indicated by the TLB
1732   and so is a host address.  In the TLB miss case, it continues to
1733   hold a guest address.
1734
1735   First argument register is clobbered.  */
1736
1737static inline void tcg_out_tlb_load(TCGContext *s, TCGReg addrlo, TCGReg addrhi,
1738                                    int mem_index, MemOp opc,
1739                                    tcg_insn_unit **label_ptr, int which)
1740{
1741    const TCGReg r0 = TCG_REG_L0;
1742    const TCGReg r1 = TCG_REG_L1;
1743    TCGType ttype = TCG_TYPE_I32;
1744    TCGType tlbtype = TCG_TYPE_I32;
1745    int trexw = 0, hrexw = 0, tlbrexw = 0;
1746    unsigned a_bits = get_alignment_bits(opc);
1747    unsigned s_bits = opc & MO_SIZE;
1748    unsigned a_mask = (1 << a_bits) - 1;
1749    unsigned s_mask = (1 << s_bits) - 1;
1750    target_ulong tlb_mask;
1751
1752    if (TCG_TARGET_REG_BITS == 64) {
1753        if (TARGET_LONG_BITS == 64) {
1754            ttype = TCG_TYPE_I64;
1755            trexw = P_REXW;
1756        }
1757        if (TCG_TYPE_PTR == TCG_TYPE_I64) {
1758            hrexw = P_REXW;
1759            if (TARGET_PAGE_BITS + CPU_TLB_DYN_MAX_BITS > 32) {
1760                tlbtype = TCG_TYPE_I64;
1761                tlbrexw = P_REXW;
1762            }
1763        }
1764    }
1765
1766    tcg_out_mov(s, tlbtype, r0, addrlo);
1767    tcg_out_shifti(s, SHIFT_SHR + tlbrexw, r0,
1768                   TARGET_PAGE_BITS - CPU_TLB_ENTRY_BITS);
1769
1770    tcg_out_modrm_offset(s, OPC_AND_GvEv + trexw, r0, TCG_AREG0,
1771                         TLB_MASK_TABLE_OFS(mem_index) +
1772                         offsetof(CPUTLBDescFast, mask));
1773
1774    tcg_out_modrm_offset(s, OPC_ADD_GvEv + hrexw, r0, TCG_AREG0,
1775                         TLB_MASK_TABLE_OFS(mem_index) +
1776                         offsetof(CPUTLBDescFast, table));
1777
1778    /* If the required alignment is at least as large as the access, simply
1779       copy the address and mask.  For lesser alignments, check that we don't
1780       cross pages for the complete access.  */
1781    if (a_bits >= s_bits) {
1782        tcg_out_mov(s, ttype, r1, addrlo);
1783    } else {
1784        tcg_out_modrm_offset(s, OPC_LEA + trexw, r1, addrlo, s_mask - a_mask);
1785    }
1786    tlb_mask = (target_ulong)TARGET_PAGE_MASK | a_mask;
1787    tgen_arithi(s, ARITH_AND + trexw, r1, tlb_mask, 0);
1788
1789    /* cmp 0(r0), r1 */
1790    tcg_out_modrm_offset(s, OPC_CMP_GvEv + trexw, r1, r0, which);
1791
1792    /* Prepare for both the fast path add of the tlb addend, and the slow
1793       path function argument setup.  */
1794    tcg_out_mov(s, ttype, r1, addrlo);
1795
1796    /* jne slow_path */
1797    tcg_out_opc(s, OPC_JCC_long + JCC_JNE, 0, 0, 0);
1798    label_ptr[0] = s->code_ptr;
1799    s->code_ptr += 4;
1800
1801    if (TARGET_LONG_BITS > TCG_TARGET_REG_BITS) {
1802        /* cmp 4(r0), addrhi */
1803        tcg_out_modrm_offset(s, OPC_CMP_GvEv, addrhi, r0, which + 4);
1804
1805        /* jne slow_path */
1806        tcg_out_opc(s, OPC_JCC_long + JCC_JNE, 0, 0, 0);
1807        label_ptr[1] = s->code_ptr;
1808        s->code_ptr += 4;
1809    }
1810
1811    /* TLB Hit.  */
1812
1813    /* add addend(r0), r1 */
1814    tcg_out_modrm_offset(s, OPC_ADD_GvEv + hrexw, r1, r0,
1815                         offsetof(CPUTLBEntry, addend));
1816}
1817
1818/*
1819 * Record the context of a call to the out of line helper code for the slow path
1820 * for a load or store, so that we can later generate the correct helper code
1821 */
1822static void add_qemu_ldst_label(TCGContext *s, bool is_ld, bool is_64,
1823                                MemOpIdx oi,
1824                                TCGReg datalo, TCGReg datahi,
1825                                TCGReg addrlo, TCGReg addrhi,
1826                                tcg_insn_unit *raddr,
1827                                tcg_insn_unit **label_ptr)
1828{
1829    TCGLabelQemuLdst *label = new_ldst_label(s);
1830
1831    label->is_ld = is_ld;
1832    label->oi = oi;
1833    label->type = is_64 ? TCG_TYPE_I64 : TCG_TYPE_I32;
1834    label->datalo_reg = datalo;
1835    label->datahi_reg = datahi;
1836    label->addrlo_reg = addrlo;
1837    label->addrhi_reg = addrhi;
1838    label->raddr = tcg_splitwx_to_rx(raddr);
1839    label->label_ptr[0] = label_ptr[0];
1840    if (TARGET_LONG_BITS > TCG_TARGET_REG_BITS) {
1841        label->label_ptr[1] = label_ptr[1];
1842    }
1843}
1844
1845/*
1846 * Generate code for the slow path for a load at the end of block
1847 */
1848static bool tcg_out_qemu_ld_slow_path(TCGContext *s, TCGLabelQemuLdst *l)
1849{
1850    MemOpIdx oi = l->oi;
1851    MemOp opc = get_memop(oi);
1852    TCGReg data_reg;
1853    tcg_insn_unit **label_ptr = &l->label_ptr[0];
1854    int rexw = (l->type == TCG_TYPE_I64 ? P_REXW : 0);
1855
1856    /* resolve label address */
1857    tcg_patch32(label_ptr[0], s->code_ptr - label_ptr[0] - 4);
1858    if (TARGET_LONG_BITS > TCG_TARGET_REG_BITS) {
1859        tcg_patch32(label_ptr[1], s->code_ptr - label_ptr[1] - 4);
1860    }
1861
1862    if (TCG_TARGET_REG_BITS == 32) {
1863        int ofs = 0;
1864
1865        tcg_out_st(s, TCG_TYPE_PTR, TCG_AREG0, TCG_REG_ESP, ofs);
1866        ofs += 4;
1867
1868        tcg_out_st(s, TCG_TYPE_I32, l->addrlo_reg, TCG_REG_ESP, ofs);
1869        ofs += 4;
1870
1871        if (TARGET_LONG_BITS == 64) {
1872            tcg_out_st(s, TCG_TYPE_I32, l->addrhi_reg, TCG_REG_ESP, ofs);
1873            ofs += 4;
1874        }
1875
1876        tcg_out_sti(s, TCG_TYPE_I32, oi, TCG_REG_ESP, ofs);
1877        ofs += 4;
1878
1879        tcg_out_sti(s, TCG_TYPE_PTR, (uintptr_t)l->raddr, TCG_REG_ESP, ofs);
1880    } else {
1881        tcg_out_mov(s, TCG_TYPE_PTR, tcg_target_call_iarg_regs[0], TCG_AREG0);
1882        /* The second argument is already loaded with addrlo.  */
1883        tcg_out_movi(s, TCG_TYPE_I32, tcg_target_call_iarg_regs[2], oi);
1884        tcg_out_movi(s, TCG_TYPE_PTR, tcg_target_call_iarg_regs[3],
1885                     (uintptr_t)l->raddr);
1886    }
1887
1888    tcg_out_call(s, qemu_ld_helpers[opc & (MO_BSWAP | MO_SIZE)]);
1889
1890    data_reg = l->datalo_reg;
1891    switch (opc & MO_SSIZE) {
1892    case MO_SB:
1893        tcg_out_ext8s(s, data_reg, TCG_REG_EAX, rexw);
1894        break;
1895    case MO_SW:
1896        tcg_out_ext16s(s, data_reg, TCG_REG_EAX, rexw);
1897        break;
1898#if TCG_TARGET_REG_BITS == 64
1899    case MO_SL:
1900        tcg_out_ext32s(s, data_reg, TCG_REG_EAX);
1901        break;
1902#endif
1903    case MO_UB:
1904    case MO_UW:
1905        /* Note that the helpers have zero-extended to tcg_target_long.  */
1906    case MO_UL:
1907        tcg_out_mov(s, TCG_TYPE_I32, data_reg, TCG_REG_EAX);
1908        break;
1909    case MO_UQ:
1910        if (TCG_TARGET_REG_BITS == 64) {
1911            tcg_out_mov(s, TCG_TYPE_I64, data_reg, TCG_REG_RAX);
1912        } else if (data_reg == TCG_REG_EDX) {
1913            /* xchg %edx, %eax */
1914            tcg_out_opc(s, OPC_XCHG_ax_r32 + TCG_REG_EDX, 0, 0, 0);
1915            tcg_out_mov(s, TCG_TYPE_I32, l->datahi_reg, TCG_REG_EAX);
1916        } else {
1917            tcg_out_mov(s, TCG_TYPE_I32, data_reg, TCG_REG_EAX);
1918            tcg_out_mov(s, TCG_TYPE_I32, l->datahi_reg, TCG_REG_EDX);
1919        }
1920        break;
1921    default:
1922        tcg_abort();
1923    }
1924
1925    /* Jump to the code corresponding to next IR of qemu_st */
1926    tcg_out_jmp(s, l->raddr);
1927    return true;
1928}
1929
1930/*
1931 * Generate code for the slow path for a store at the end of block
1932 */
1933static bool tcg_out_qemu_st_slow_path(TCGContext *s, TCGLabelQemuLdst *l)
1934{
1935    MemOpIdx oi = l->oi;
1936    MemOp opc = get_memop(oi);
1937    MemOp s_bits = opc & MO_SIZE;
1938    tcg_insn_unit **label_ptr = &l->label_ptr[0];
1939    TCGReg retaddr;
1940
1941    /* resolve label address */
1942    tcg_patch32(label_ptr[0], s->code_ptr - label_ptr[0] - 4);
1943    if (TARGET_LONG_BITS > TCG_TARGET_REG_BITS) {
1944        tcg_patch32(label_ptr[1], s->code_ptr - label_ptr[1] - 4);
1945    }
1946
1947    if (TCG_TARGET_REG_BITS == 32) {
1948        int ofs = 0;
1949
1950        tcg_out_st(s, TCG_TYPE_PTR, TCG_AREG0, TCG_REG_ESP, ofs);
1951        ofs += 4;
1952
1953        tcg_out_st(s, TCG_TYPE_I32, l->addrlo_reg, TCG_REG_ESP, ofs);
1954        ofs += 4;
1955
1956        if (TARGET_LONG_BITS == 64) {
1957            tcg_out_st(s, TCG_TYPE_I32, l->addrhi_reg, TCG_REG_ESP, ofs);
1958            ofs += 4;
1959        }
1960
1961        tcg_out_st(s, TCG_TYPE_I32, l->datalo_reg, TCG_REG_ESP, ofs);
1962        ofs += 4;
1963
1964        if (s_bits == MO_64) {
1965            tcg_out_st(s, TCG_TYPE_I32, l->datahi_reg, TCG_REG_ESP, ofs);
1966            ofs += 4;
1967        }
1968
1969        tcg_out_sti(s, TCG_TYPE_I32, oi, TCG_REG_ESP, ofs);
1970        ofs += 4;
1971
1972        retaddr = TCG_REG_EAX;
1973        tcg_out_movi(s, TCG_TYPE_PTR, retaddr, (uintptr_t)l->raddr);
1974        tcg_out_st(s, TCG_TYPE_PTR, retaddr, TCG_REG_ESP, ofs);
1975    } else {
1976        tcg_out_mov(s, TCG_TYPE_PTR, tcg_target_call_iarg_regs[0], TCG_AREG0);
1977        /* The second argument is already loaded with addrlo.  */
1978        tcg_out_mov(s, (s_bits == MO_64 ? TCG_TYPE_I64 : TCG_TYPE_I32),
1979                    tcg_target_call_iarg_regs[2], l->datalo_reg);
1980        tcg_out_movi(s, TCG_TYPE_I32, tcg_target_call_iarg_regs[3], oi);
1981
1982        if (ARRAY_SIZE(tcg_target_call_iarg_regs) > 4) {
1983            retaddr = tcg_target_call_iarg_regs[4];
1984            tcg_out_movi(s, TCG_TYPE_PTR, retaddr, (uintptr_t)l->raddr);
1985        } else {
1986            retaddr = TCG_REG_RAX;
1987            tcg_out_movi(s, TCG_TYPE_PTR, retaddr, (uintptr_t)l->raddr);
1988            tcg_out_st(s, TCG_TYPE_PTR, retaddr, TCG_REG_ESP,
1989                       TCG_TARGET_CALL_STACK_OFFSET);
1990        }
1991    }
1992
1993    /* "Tail call" to the helper, with the return address back inline.  */
1994    tcg_out_push(s, retaddr);
1995    tcg_out_jmp(s, qemu_st_helpers[opc & (MO_BSWAP | MO_SIZE)]);
1996    return true;
1997}
1998#else
1999
2000static void tcg_out_test_alignment(TCGContext *s, bool is_ld, TCGReg addrlo,
2001                                   TCGReg addrhi, unsigned a_bits)
2002{
2003    unsigned a_mask = (1 << a_bits) - 1;
2004    TCGLabelQemuLdst *label;
2005
2006    /*
2007     * We are expecting a_bits to max out at 7, so we can usually use testb.
2008     * For i686, we have to use testl for %esi/%edi.
2009     */
2010    if (a_mask <= 0xff && (TCG_TARGET_REG_BITS == 64 || addrlo < 4)) {
2011        tcg_out_modrm(s, OPC_GRP3_Eb | P_REXB_RM, EXT3_TESTi, addrlo);
2012        tcg_out8(s, a_mask);
2013    } else {
2014        tcg_out_modrm(s, OPC_GRP3_Ev, EXT3_TESTi, addrlo);
2015        tcg_out32(s, a_mask);
2016    }
2017
2018    /* jne slow_path */
2019    tcg_out_opc(s, OPC_JCC_long + JCC_JNE, 0, 0, 0);
2020
2021    label = new_ldst_label(s);
2022    label->is_ld = is_ld;
2023    label->addrlo_reg = addrlo;
2024    label->addrhi_reg = addrhi;
2025    label->raddr = tcg_splitwx_to_rx(s->code_ptr + 4);
2026    label->label_ptr[0] = s->code_ptr;
2027
2028    s->code_ptr += 4;
2029}
2030
2031static bool tcg_out_fail_alignment(TCGContext *s, TCGLabelQemuLdst *l)
2032{
2033    /* resolve label address */
2034    tcg_patch32(l->label_ptr[0], s->code_ptr - l->label_ptr[0] - 4);
2035
2036    if (TCG_TARGET_REG_BITS == 32) {
2037        int ofs = 0;
2038
2039        tcg_out_st(s, TCG_TYPE_PTR, TCG_AREG0, TCG_REG_ESP, ofs);
2040        ofs += 4;
2041
2042        tcg_out_st(s, TCG_TYPE_I32, l->addrlo_reg, TCG_REG_ESP, ofs);
2043        ofs += 4;
2044        if (TARGET_LONG_BITS == 64) {
2045            tcg_out_st(s, TCG_TYPE_I32, l->addrhi_reg, TCG_REG_ESP, ofs);
2046            ofs += 4;
2047        }
2048
2049        tcg_out_pushi(s, (uintptr_t)l->raddr);
2050    } else {
2051        tcg_out_mov(s, TCG_TYPE_TL, tcg_target_call_iarg_regs[1],
2052                    l->addrlo_reg);
2053        tcg_out_mov(s, TCG_TYPE_PTR, tcg_target_call_iarg_regs[0], TCG_AREG0);
2054
2055        tcg_out_movi(s, TCG_TYPE_PTR, TCG_REG_RAX, (uintptr_t)l->raddr);
2056        tcg_out_push(s, TCG_REG_RAX);
2057    }
2058
2059    /* "Tail call" to the helper, with the return address back inline. */
2060    tcg_out_jmp(s, (const void *)(l->is_ld ? helper_unaligned_ld
2061                                  : helper_unaligned_st));
2062    return true;
2063}
2064
2065static bool tcg_out_qemu_ld_slow_path(TCGContext *s, TCGLabelQemuLdst *l)
2066{
2067    return tcg_out_fail_alignment(s, l);
2068}
2069
2070static bool tcg_out_qemu_st_slow_path(TCGContext *s, TCGLabelQemuLdst *l)
2071{
2072    return tcg_out_fail_alignment(s, l);
2073}
2074
2075#if TCG_TARGET_REG_BITS == 32
2076# define x86_guest_base_seg     0
2077# define x86_guest_base_index   -1
2078# define x86_guest_base_offset  guest_base
2079#else
2080static int x86_guest_base_seg;
2081static int x86_guest_base_index = -1;
2082static int32_t x86_guest_base_offset;
2083# if defined(__x86_64__) && defined(__linux__)
2084#  include <asm/prctl.h>
2085#  include <sys/prctl.h>
2086int arch_prctl(int code, unsigned long addr);
2087static inline int setup_guest_base_seg(void)
2088{
2089    if (arch_prctl(ARCH_SET_GS, guest_base) == 0) {
2090        return P_GS;
2091    }
2092    return 0;
2093}
2094# elif defined (__FreeBSD__) || defined (__FreeBSD_kernel__)
2095#  include <machine/sysarch.h>
2096static inline int setup_guest_base_seg(void)
2097{
2098    if (sysarch(AMD64_SET_GSBASE, &guest_base) == 0) {
2099        return P_GS;
2100    }
2101    return 0;
2102}
2103# else
2104static inline int setup_guest_base_seg(void)
2105{
2106    return 0;
2107}
2108# endif
2109#endif
2110#endif /* SOFTMMU */
2111
2112static void tcg_out_qemu_ld_direct(TCGContext *s, TCGReg datalo, TCGReg datahi,
2113                                   TCGReg base, int index, intptr_t ofs,
2114                                   int seg, bool is64, MemOp memop)
2115{
2116    bool use_movbe = false;
2117    int rexw = is64 * P_REXW;
2118    int movop = OPC_MOVL_GvEv;
2119
2120    /* Do big-endian loads with movbe.  */
2121    if (memop & MO_BSWAP) {
2122        tcg_debug_assert(have_movbe);
2123        use_movbe = true;
2124        movop = OPC_MOVBE_GyMy;
2125    }
2126
2127    switch (memop & MO_SSIZE) {
2128    case MO_UB:
2129        tcg_out_modrm_sib_offset(s, OPC_MOVZBL + seg, datalo,
2130                                 base, index, 0, ofs);
2131        break;
2132    case MO_SB:
2133        tcg_out_modrm_sib_offset(s, OPC_MOVSBL + rexw + seg, datalo,
2134                                 base, index, 0, ofs);
2135        break;
2136    case MO_UW:
2137        if (use_movbe) {
2138            /* There is no extending movbe; only low 16-bits are modified.  */
2139            if (datalo != base && datalo != index) {
2140                /* XOR breaks dependency chains.  */
2141                tgen_arithr(s, ARITH_XOR, datalo, datalo);
2142                tcg_out_modrm_sib_offset(s, OPC_MOVBE_GyMy + P_DATA16 + seg,
2143                                         datalo, base, index, 0, ofs);
2144            } else {
2145                tcg_out_modrm_sib_offset(s, OPC_MOVBE_GyMy + P_DATA16 + seg,
2146                                         datalo, base, index, 0, ofs);
2147                tcg_out_ext16u(s, datalo, datalo);
2148            }
2149        } else {
2150            tcg_out_modrm_sib_offset(s, OPC_MOVZWL + seg, datalo,
2151                                     base, index, 0, ofs);
2152        }
2153        break;
2154    case MO_SW:
2155        if (use_movbe) {
2156            tcg_out_modrm_sib_offset(s, OPC_MOVBE_GyMy + P_DATA16 + seg,
2157                                     datalo, base, index, 0, ofs);
2158            tcg_out_ext16s(s, datalo, datalo, rexw);
2159        } else {
2160            tcg_out_modrm_sib_offset(s, OPC_MOVSWL + rexw + seg,
2161                                     datalo, base, index, 0, ofs);
2162        }
2163        break;
2164    case MO_UL:
2165        tcg_out_modrm_sib_offset(s, movop + seg, datalo, base, index, 0, ofs);
2166        break;
2167#if TCG_TARGET_REG_BITS == 64
2168    case MO_SL:
2169        if (use_movbe) {
2170            tcg_out_modrm_sib_offset(s, OPC_MOVBE_GyMy + seg, datalo,
2171                                     base, index, 0, ofs);
2172            tcg_out_ext32s(s, datalo, datalo);
2173        } else {
2174            tcg_out_modrm_sib_offset(s, OPC_MOVSLQ + seg, datalo,
2175                                     base, index, 0, ofs);
2176        }
2177        break;
2178#endif
2179    case MO_UQ:
2180        if (TCG_TARGET_REG_BITS == 64) {
2181            tcg_out_modrm_sib_offset(s, movop + P_REXW + seg, datalo,
2182                                     base, index, 0, ofs);
2183        } else {
2184            if (use_movbe) {
2185                TCGReg t = datalo;
2186                datalo = datahi;
2187                datahi = t;
2188            }
2189            if (base != datalo) {
2190                tcg_out_modrm_sib_offset(s, movop + seg, datalo,
2191                                         base, index, 0, ofs);
2192                tcg_out_modrm_sib_offset(s, movop + seg, datahi,
2193                                         base, index, 0, ofs + 4);
2194            } else {
2195                tcg_out_modrm_sib_offset(s, movop + seg, datahi,
2196                                         base, index, 0, ofs + 4);
2197                tcg_out_modrm_sib_offset(s, movop + seg, datalo,
2198                                         base, index, 0, ofs);
2199            }
2200        }
2201        break;
2202    default:
2203        g_assert_not_reached();
2204    }
2205}
2206
2207/* XXX: qemu_ld and qemu_st could be modified to clobber only EDX and
2208   EAX. It will be useful once fixed registers globals are less
2209   common. */
2210static void tcg_out_qemu_ld(TCGContext *s, const TCGArg *args, bool is64)
2211{
2212    TCGReg datalo, datahi, addrlo;
2213    TCGReg addrhi __attribute__((unused));
2214    MemOpIdx oi;
2215    MemOp opc;
2216#if defined(CONFIG_SOFTMMU)
2217    int mem_index;
2218    tcg_insn_unit *label_ptr[2];
2219#else
2220    unsigned a_bits;
2221#endif
2222
2223    datalo = *args++;
2224    datahi = (TCG_TARGET_REG_BITS == 32 && is64 ? *args++ : 0);
2225    addrlo = *args++;
2226    addrhi = (TARGET_LONG_BITS > TCG_TARGET_REG_BITS ? *args++ : 0);
2227    oi = *args++;
2228    opc = get_memop(oi);
2229
2230#if defined(CONFIG_SOFTMMU)
2231    mem_index = get_mmuidx(oi);
2232
2233    tcg_out_tlb_load(s, addrlo, addrhi, mem_index, opc,
2234                     label_ptr, offsetof(CPUTLBEntry, addr_read));
2235
2236    /* TLB Hit.  */
2237    tcg_out_qemu_ld_direct(s, datalo, datahi, TCG_REG_L1, -1, 0, 0, is64, opc);
2238
2239    /* Record the current context of a load into ldst label */
2240    add_qemu_ldst_label(s, true, is64, oi, datalo, datahi, addrlo, addrhi,
2241                        s->code_ptr, label_ptr);
2242#else
2243    a_bits = get_alignment_bits(opc);
2244    if (a_bits) {
2245        tcg_out_test_alignment(s, true, addrlo, addrhi, a_bits);
2246    }
2247
2248    tcg_out_qemu_ld_direct(s, datalo, datahi, addrlo, x86_guest_base_index,
2249                           x86_guest_base_offset, x86_guest_base_seg,
2250                           is64, opc);
2251#endif
2252}
2253
2254static void tcg_out_qemu_st_direct(TCGContext *s, TCGReg datalo, TCGReg datahi,
2255                                   TCGReg base, int index, intptr_t ofs,
2256                                   int seg, MemOp memop)
2257{
2258    bool use_movbe = false;
2259    int movop = OPC_MOVL_EvGv;
2260
2261    /*
2262     * Do big-endian stores with movbe or softmmu.
2263     * User-only without movbe will have its swapping done generically.
2264     */
2265    if (memop & MO_BSWAP) {
2266        tcg_debug_assert(have_movbe);
2267        use_movbe = true;
2268        movop = OPC_MOVBE_MyGy;
2269    }
2270
2271    switch (memop & MO_SIZE) {
2272    case MO_8:
2273        /* This is handled with constraints on INDEX_op_qemu_st8_i32. */
2274        tcg_debug_assert(TCG_TARGET_REG_BITS == 64 || datalo < 4);
2275        tcg_out_modrm_sib_offset(s, OPC_MOVB_EvGv + P_REXB_R + seg,
2276                                 datalo, base, index, 0, ofs);
2277        break;
2278    case MO_16:
2279        tcg_out_modrm_sib_offset(s, movop + P_DATA16 + seg, datalo,
2280                                 base, index, 0, ofs);
2281        break;
2282    case MO_32:
2283        tcg_out_modrm_sib_offset(s, movop + seg, datalo, base, index, 0, ofs);
2284        break;
2285    case MO_64:
2286        if (TCG_TARGET_REG_BITS == 64) {
2287            tcg_out_modrm_sib_offset(s, movop + P_REXW + seg, datalo,
2288                                     base, index, 0, ofs);
2289        } else {
2290            if (use_movbe) {
2291                TCGReg t = datalo;
2292                datalo = datahi;
2293                datahi = t;
2294            }
2295            tcg_out_modrm_sib_offset(s, movop + seg, datalo,
2296                                     base, index, 0, ofs);
2297            tcg_out_modrm_sib_offset(s, movop + seg, datahi,
2298                                     base, index, 0, ofs + 4);
2299        }
2300        break;
2301    default:
2302        g_assert_not_reached();
2303    }
2304}
2305
2306static void tcg_out_qemu_st(TCGContext *s, const TCGArg *args, bool is64)
2307{
2308    TCGReg datalo, datahi, addrlo;
2309    TCGReg addrhi __attribute__((unused));
2310    MemOpIdx oi;
2311    MemOp opc;
2312#if defined(CONFIG_SOFTMMU)
2313    int mem_index;
2314    tcg_insn_unit *label_ptr[2];
2315#else
2316    unsigned a_bits;
2317#endif
2318
2319    datalo = *args++;
2320    datahi = (TCG_TARGET_REG_BITS == 32 && is64 ? *args++ : 0);
2321    addrlo = *args++;
2322    addrhi = (TARGET_LONG_BITS > TCG_TARGET_REG_BITS ? *args++ : 0);
2323    oi = *args++;
2324    opc = get_memop(oi);
2325
2326#if defined(CONFIG_SOFTMMU)
2327    mem_index = get_mmuidx(oi);
2328
2329    tcg_out_tlb_load(s, addrlo, addrhi, mem_index, opc,
2330                     label_ptr, offsetof(CPUTLBEntry, addr_write));
2331
2332    /* TLB Hit.  */
2333    tcg_out_qemu_st_direct(s, datalo, datahi, TCG_REG_L1, -1, 0, 0, opc);
2334
2335    /* Record the current context of a store into ldst label */
2336    add_qemu_ldst_label(s, false, is64, oi, datalo, datahi, addrlo, addrhi,
2337                        s->code_ptr, label_ptr);
2338#else
2339    a_bits = get_alignment_bits(opc);
2340    if (a_bits) {
2341        tcg_out_test_alignment(s, false, addrlo, addrhi, a_bits);
2342    }
2343
2344    tcg_out_qemu_st_direct(s, datalo, datahi, addrlo, x86_guest_base_index,
2345                           x86_guest_base_offset, x86_guest_base_seg, opc);
2346#endif
2347}
2348
2349static inline void tcg_out_op(TCGContext *s, TCGOpcode opc,
2350                              const TCGArg args[TCG_MAX_OP_ARGS],
2351                              const int const_args[TCG_MAX_OP_ARGS])
2352{
2353    TCGArg a0, a1, a2;
2354    int c, const_a2, vexop, rexw = 0;
2355
2356#if TCG_TARGET_REG_BITS == 64
2357# define OP_32_64(x) \
2358        case glue(glue(INDEX_op_, x), _i64): \
2359            rexw = P_REXW; /* FALLTHRU */    \
2360        case glue(glue(INDEX_op_, x), _i32)
2361#else
2362# define OP_32_64(x) \
2363        case glue(glue(INDEX_op_, x), _i32)
2364#endif
2365
2366    /* Hoist the loads of the most common arguments.  */
2367    a0 = args[0];
2368    a1 = args[1];
2369    a2 = args[2];
2370    const_a2 = const_args[2];
2371
2372    switch (opc) {
2373    case INDEX_op_exit_tb:
2374        /* Reuse the zeroing that exists for goto_ptr.  */
2375        if (a0 == 0) {
2376            tcg_out_jmp(s, tcg_code_gen_epilogue);
2377        } else {
2378            tcg_out_movi(s, TCG_TYPE_PTR, TCG_REG_EAX, a0);
2379            tcg_out_jmp(s, tb_ret_addr);
2380        }
2381        break;
2382    case INDEX_op_goto_tb:
2383        if (s->tb_jmp_insn_offset) {
2384            /* direct jump method */
2385            int gap;
2386            /* jump displacement must be aligned for atomic patching;
2387             * see if we need to add extra nops before jump
2388             */
2389            gap = QEMU_ALIGN_PTR_UP(s->code_ptr + 1, 4) - s->code_ptr;
2390            if (gap != 1) {
2391                tcg_out_nopn(s, gap - 1);
2392            }
2393            tcg_out8(s, OPC_JMP_long); /* jmp im */
2394            s->tb_jmp_insn_offset[a0] = tcg_current_code_size(s);
2395            tcg_out32(s, 0);
2396        } else {
2397            /* indirect jump method */
2398            tcg_out_modrm_offset(s, OPC_GRP5, EXT5_JMPN_Ev, -1,
2399                                 (intptr_t)(s->tb_jmp_target_addr + a0));
2400        }
2401        set_jmp_reset_offset(s, a0);
2402        break;
2403    case INDEX_op_goto_ptr:
2404        /* jmp to the given host address (could be epilogue) */
2405        tcg_out_modrm(s, OPC_GRP5, EXT5_JMPN_Ev, a0);
2406        break;
2407    case INDEX_op_br:
2408        tcg_out_jxx(s, JCC_JMP, arg_label(a0), 0);
2409        break;
2410    OP_32_64(ld8u):
2411        /* Note that we can ignore REXW for the zero-extend to 64-bit.  */
2412        tcg_out_modrm_offset(s, OPC_MOVZBL, a0, a1, a2);
2413        break;
2414    OP_32_64(ld8s):
2415        tcg_out_modrm_offset(s, OPC_MOVSBL + rexw, a0, a1, a2);
2416        break;
2417    OP_32_64(ld16u):
2418        /* Note that we can ignore REXW for the zero-extend to 64-bit.  */
2419        tcg_out_modrm_offset(s, OPC_MOVZWL, a0, a1, a2);
2420        break;
2421    OP_32_64(ld16s):
2422        tcg_out_modrm_offset(s, OPC_MOVSWL + rexw, a0, a1, a2);
2423        break;
2424#if TCG_TARGET_REG_BITS == 64
2425    case INDEX_op_ld32u_i64:
2426#endif
2427    case INDEX_op_ld_i32:
2428        tcg_out_ld(s, TCG_TYPE_I32, a0, a1, a2);
2429        break;
2430
2431    OP_32_64(st8):
2432        if (const_args[0]) {
2433            tcg_out_modrm_offset(s, OPC_MOVB_EvIz, 0, a1, a2);
2434            tcg_out8(s, a0);
2435        } else {
2436            tcg_out_modrm_offset(s, OPC_MOVB_EvGv | P_REXB_R, a0, a1, a2);
2437        }
2438        break;
2439    OP_32_64(st16):
2440        if (const_args[0]) {
2441            tcg_out_modrm_offset(s, OPC_MOVL_EvIz | P_DATA16, 0, a1, a2);
2442            tcg_out16(s, a0);
2443        } else {
2444            tcg_out_modrm_offset(s, OPC_MOVL_EvGv | P_DATA16, a0, a1, a2);
2445        }
2446        break;
2447#if TCG_TARGET_REG_BITS == 64
2448    case INDEX_op_st32_i64:
2449#endif
2450    case INDEX_op_st_i32:
2451        if (const_args[0]) {
2452            tcg_out_modrm_offset(s, OPC_MOVL_EvIz, 0, a1, a2);
2453            tcg_out32(s, a0);
2454        } else {
2455            tcg_out_st(s, TCG_TYPE_I32, a0, a1, a2);
2456        }
2457        break;
2458
2459    OP_32_64(add):
2460        /* For 3-operand addition, use LEA.  */
2461        if (a0 != a1) {
2462            TCGArg c3 = 0;
2463            if (const_a2) {
2464                c3 = a2, a2 = -1;
2465            } else if (a0 == a2) {
2466                /* Watch out for dest = src + dest, since we've removed
2467                   the matching constraint on the add.  */
2468                tgen_arithr(s, ARITH_ADD + rexw, a0, a1);
2469                break;
2470            }
2471
2472            tcg_out_modrm_sib_offset(s, OPC_LEA + rexw, a0, a1, a2, 0, c3);
2473            break;
2474        }
2475        c = ARITH_ADD;
2476        goto gen_arith;
2477    OP_32_64(sub):
2478        c = ARITH_SUB;
2479        goto gen_arith;
2480    OP_32_64(and):
2481        c = ARITH_AND;
2482        goto gen_arith;
2483    OP_32_64(or):
2484        c = ARITH_OR;
2485        goto gen_arith;
2486    OP_32_64(xor):
2487        c = ARITH_XOR;
2488        goto gen_arith;
2489    gen_arith:
2490        if (const_a2) {
2491            tgen_arithi(s, c + rexw, a0, a2, 0);
2492        } else {
2493            tgen_arithr(s, c + rexw, a0, a2);
2494        }
2495        break;
2496
2497    OP_32_64(andc):
2498        if (const_a2) {
2499            tcg_out_mov(s, rexw ? TCG_TYPE_I64 : TCG_TYPE_I32, a0, a1);
2500            tgen_arithi(s, ARITH_AND + rexw, a0, ~a2, 0);
2501        } else {
2502            tcg_out_vex_modrm(s, OPC_ANDN + rexw, a0, a2, a1);
2503        }
2504        break;
2505
2506    OP_32_64(mul):
2507        if (const_a2) {
2508            int32_t val;
2509            val = a2;
2510            if (val == (int8_t)val) {
2511                tcg_out_modrm(s, OPC_IMUL_GvEvIb + rexw, a0, a0);
2512                tcg_out8(s, val);
2513            } else {
2514                tcg_out_modrm(s, OPC_IMUL_GvEvIz + rexw, a0, a0);
2515                tcg_out32(s, val);
2516            }
2517        } else {
2518            tcg_out_modrm(s, OPC_IMUL_GvEv + rexw, a0, a2);
2519        }
2520        break;
2521
2522    OP_32_64(div2):
2523        tcg_out_modrm(s, OPC_GRP3_Ev + rexw, EXT3_IDIV, args[4]);
2524        break;
2525    OP_32_64(divu2):
2526        tcg_out_modrm(s, OPC_GRP3_Ev + rexw, EXT3_DIV, args[4]);
2527        break;
2528
2529    OP_32_64(shl):
2530        /* For small constant 3-operand shift, use LEA.  */
2531        if (const_a2 && a0 != a1 && (a2 - 1) < 3) {
2532            if (a2 - 1 == 0) {
2533                /* shl $1,a1,a0 -> lea (a1,a1),a0 */
2534                tcg_out_modrm_sib_offset(s, OPC_LEA + rexw, a0, a1, a1, 0, 0);
2535            } else {
2536                /* shl $n,a1,a0 -> lea 0(,a1,n),a0 */
2537                tcg_out_modrm_sib_offset(s, OPC_LEA + rexw, a0, -1, a1, a2, 0);
2538            }
2539            break;
2540        }
2541        c = SHIFT_SHL;
2542        vexop = OPC_SHLX;
2543        goto gen_shift_maybe_vex;
2544    OP_32_64(shr):
2545        c = SHIFT_SHR;
2546        vexop = OPC_SHRX;
2547        goto gen_shift_maybe_vex;
2548    OP_32_64(sar):
2549        c = SHIFT_SAR;
2550        vexop = OPC_SARX;
2551        goto gen_shift_maybe_vex;
2552    OP_32_64(rotl):
2553        c = SHIFT_ROL;
2554        goto gen_shift;
2555    OP_32_64(rotr):
2556        c = SHIFT_ROR;
2557        goto gen_shift;
2558    gen_shift_maybe_vex:
2559        if (have_bmi2) {
2560            if (!const_a2) {
2561                tcg_out_vex_modrm(s, vexop + rexw, a0, a2, a1);
2562                break;
2563            }
2564            tcg_out_mov(s, rexw ? TCG_TYPE_I64 : TCG_TYPE_I32, a0, a1);
2565        }
2566        /* FALLTHRU */
2567    gen_shift:
2568        if (const_a2) {
2569            tcg_out_shifti(s, c + rexw, a0, a2);
2570        } else {
2571            tcg_out_modrm(s, OPC_SHIFT_cl + rexw, c, a0);
2572        }
2573        break;
2574
2575    OP_32_64(ctz):
2576        tcg_out_ctz(s, rexw, args[0], args[1], args[2], const_args[2]);
2577        break;
2578    OP_32_64(clz):
2579        tcg_out_clz(s, rexw, args[0], args[1], args[2], const_args[2]);
2580        break;
2581    OP_32_64(ctpop):
2582        tcg_out_modrm(s, OPC_POPCNT + rexw, a0, a1);
2583        break;
2584
2585    case INDEX_op_brcond_i32:
2586        tcg_out_brcond32(s, a2, a0, a1, const_args[1], arg_label(args[3]), 0);
2587        break;
2588    case INDEX_op_setcond_i32:
2589        tcg_out_setcond32(s, args[3], a0, a1, a2, const_a2);
2590        break;
2591    case INDEX_op_movcond_i32:
2592        tcg_out_movcond32(s, args[5], a0, a1, a2, const_a2, args[3]);
2593        break;
2594
2595    OP_32_64(bswap16):
2596        if (a2 & TCG_BSWAP_OS) {
2597            /* Output must be sign-extended. */
2598            if (rexw) {
2599                tcg_out_bswap64(s, a0);
2600                tcg_out_shifti(s, SHIFT_SAR + rexw, a0, 48);
2601            } else {
2602                tcg_out_bswap32(s, a0);
2603                tcg_out_shifti(s, SHIFT_SAR, a0, 16);
2604            }
2605        } else if ((a2 & (TCG_BSWAP_IZ | TCG_BSWAP_OZ)) == TCG_BSWAP_OZ) {
2606            /* Output must be zero-extended, but input isn't. */
2607            tcg_out_bswap32(s, a0);
2608            tcg_out_shifti(s, SHIFT_SHR, a0, 16);
2609        } else {
2610            tcg_out_rolw_8(s, a0);
2611        }
2612        break;
2613    OP_32_64(bswap32):
2614        tcg_out_bswap32(s, a0);
2615        if (rexw && (a2 & TCG_BSWAP_OS)) {
2616            tcg_out_ext32s(s, a0, a0);
2617        }
2618        break;
2619
2620    OP_32_64(neg):
2621        tcg_out_modrm(s, OPC_GRP3_Ev + rexw, EXT3_NEG, a0);
2622        break;
2623    OP_32_64(not):
2624        tcg_out_modrm(s, OPC_GRP3_Ev + rexw, EXT3_NOT, a0);
2625        break;
2626
2627    OP_32_64(ext8s):
2628        tcg_out_ext8s(s, a0, a1, rexw);
2629        break;
2630    OP_32_64(ext16s):
2631        tcg_out_ext16s(s, a0, a1, rexw);
2632        break;
2633    OP_32_64(ext8u):
2634        tcg_out_ext8u(s, a0, a1);
2635        break;
2636    OP_32_64(ext16u):
2637        tcg_out_ext16u(s, a0, a1);
2638        break;
2639
2640    case INDEX_op_qemu_ld_i32:
2641        tcg_out_qemu_ld(s, args, 0);
2642        break;
2643    case INDEX_op_qemu_ld_i64:
2644        tcg_out_qemu_ld(s, args, 1);
2645        break;
2646    case INDEX_op_qemu_st_i32:
2647    case INDEX_op_qemu_st8_i32:
2648        tcg_out_qemu_st(s, args, 0);
2649        break;
2650    case INDEX_op_qemu_st_i64:
2651        tcg_out_qemu_st(s, args, 1);
2652        break;
2653
2654    OP_32_64(mulu2):
2655        tcg_out_modrm(s, OPC_GRP3_Ev + rexw, EXT3_MUL, args[3]);
2656        break;
2657    OP_32_64(muls2):
2658        tcg_out_modrm(s, OPC_GRP3_Ev + rexw, EXT3_IMUL, args[3]);
2659        break;
2660    OP_32_64(add2):
2661        if (const_args[4]) {
2662            tgen_arithi(s, ARITH_ADD + rexw, a0, args[4], 1);
2663        } else {
2664            tgen_arithr(s, ARITH_ADD + rexw, a0, args[4]);
2665        }
2666        if (const_args[5]) {
2667            tgen_arithi(s, ARITH_ADC + rexw, a1, args[5], 1);
2668        } else {
2669            tgen_arithr(s, ARITH_ADC + rexw, a1, args[5]);
2670        }
2671        break;
2672    OP_32_64(sub2):
2673        if (const_args[4]) {
2674            tgen_arithi(s, ARITH_SUB + rexw, a0, args[4], 1);
2675        } else {
2676            tgen_arithr(s, ARITH_SUB + rexw, a0, args[4]);
2677        }
2678        if (const_args[5]) {
2679            tgen_arithi(s, ARITH_SBB + rexw, a1, args[5], 1);
2680        } else {
2681            tgen_arithr(s, ARITH_SBB + rexw, a1, args[5]);
2682        }
2683        break;
2684
2685#if TCG_TARGET_REG_BITS == 32
2686    case INDEX_op_brcond2_i32:
2687        tcg_out_brcond2(s, args, const_args, 0);
2688        break;
2689    case INDEX_op_setcond2_i32:
2690        tcg_out_setcond2(s, args, const_args);
2691        break;
2692#else /* TCG_TARGET_REG_BITS == 64 */
2693    case INDEX_op_ld32s_i64:
2694        tcg_out_modrm_offset(s, OPC_MOVSLQ, a0, a1, a2);
2695        break;
2696    case INDEX_op_ld_i64:
2697        tcg_out_ld(s, TCG_TYPE_I64, a0, a1, a2);
2698        break;
2699    case INDEX_op_st_i64:
2700        if (const_args[0]) {
2701            tcg_out_modrm_offset(s, OPC_MOVL_EvIz | P_REXW, 0, a1, a2);
2702            tcg_out32(s, a0);
2703        } else {
2704            tcg_out_st(s, TCG_TYPE_I64, a0, a1, a2);
2705        }
2706        break;
2707
2708    case INDEX_op_brcond_i64:
2709        tcg_out_brcond64(s, a2, a0, a1, const_args[1], arg_label(args[3]), 0);
2710        break;
2711    case INDEX_op_setcond_i64:
2712        tcg_out_setcond64(s, args[3], a0, a1, a2, const_a2);
2713        break;
2714    case INDEX_op_movcond_i64:
2715        tcg_out_movcond64(s, args[5], a0, a1, a2, const_a2, args[3]);
2716        break;
2717
2718    case INDEX_op_bswap64_i64:
2719        tcg_out_bswap64(s, a0);
2720        break;
2721    case INDEX_op_extu_i32_i64:
2722    case INDEX_op_ext32u_i64:
2723    case INDEX_op_extrl_i64_i32:
2724        tcg_out_ext32u(s, a0, a1);
2725        break;
2726    case INDEX_op_ext_i32_i64:
2727    case INDEX_op_ext32s_i64:
2728        tcg_out_ext32s(s, a0, a1);
2729        break;
2730    case INDEX_op_extrh_i64_i32:
2731        tcg_out_shifti(s, SHIFT_SHR + P_REXW, a0, 32);
2732        break;
2733#endif
2734
2735    OP_32_64(deposit):
2736        if (args[3] == 0 && args[4] == 8) {
2737            /* load bits 0..7 */
2738            tcg_out_modrm(s, OPC_MOVB_EvGv | P_REXB_R | P_REXB_RM, a2, a0);
2739        } else if (args[3] == 8 && args[4] == 8) {
2740            /* load bits 8..15 */
2741            tcg_out_modrm(s, OPC_MOVB_EvGv, a2, a0 + 4);
2742        } else if (args[3] == 0 && args[4] == 16) {
2743            /* load bits 0..15 */
2744            tcg_out_modrm(s, OPC_MOVL_EvGv | P_DATA16, a2, a0);
2745        } else {
2746            tcg_abort();
2747        }
2748        break;
2749
2750    case INDEX_op_extract_i64:
2751        if (a2 + args[3] == 32) {
2752            /* This is a 32-bit zero-extending right shift.  */
2753            tcg_out_mov(s, TCG_TYPE_I32, a0, a1);
2754            tcg_out_shifti(s, SHIFT_SHR, a0, a2);
2755            break;
2756        }
2757        /* FALLTHRU */
2758    case INDEX_op_extract_i32:
2759        /* On the off-chance that we can use the high-byte registers.
2760           Otherwise we emit the same ext16 + shift pattern that we
2761           would have gotten from the normal tcg-op.c expansion.  */
2762        tcg_debug_assert(a2 == 8 && args[3] == 8);
2763        if (a1 < 4 && a0 < 8) {
2764            tcg_out_modrm(s, OPC_MOVZBL, a0, a1 + 4);
2765        } else {
2766            tcg_out_ext16u(s, a0, a1);
2767            tcg_out_shifti(s, SHIFT_SHR, a0, 8);
2768        }
2769        break;
2770
2771    case INDEX_op_sextract_i32:
2772        /* We don't implement sextract_i64, as we cannot sign-extend to
2773           64-bits without using the REX prefix that explicitly excludes
2774           access to the high-byte registers.  */
2775        tcg_debug_assert(a2 == 8 && args[3] == 8);
2776        if (a1 < 4 && a0 < 8) {
2777            tcg_out_modrm(s, OPC_MOVSBL, a0, a1 + 4);
2778        } else {
2779            tcg_out_ext16s(s, a0, a1, 0);
2780            tcg_out_shifti(s, SHIFT_SAR, a0, 8);
2781        }
2782        break;
2783
2784    OP_32_64(extract2):
2785        /* Note that SHRD outputs to the r/m operand.  */
2786        tcg_out_modrm(s, OPC_SHRD_Ib + rexw, a2, a0);
2787        tcg_out8(s, args[3]);
2788        break;
2789
2790    case INDEX_op_mb:
2791        tcg_out_mb(s, a0);
2792        break;
2793    case INDEX_op_mov_i32:  /* Always emitted via tcg_out_mov.  */
2794    case INDEX_op_mov_i64:
2795    case INDEX_op_call:     /* Always emitted via tcg_out_call.  */
2796    default:
2797        tcg_abort();
2798    }
2799
2800#undef OP_32_64
2801}
2802
2803static void tcg_out_vec_op(TCGContext *s, TCGOpcode opc,
2804                           unsigned vecl, unsigned vece,
2805                           const TCGArg args[TCG_MAX_OP_ARGS],
2806                           const int const_args[TCG_MAX_OP_ARGS])
2807{
2808    static int const add_insn[4] = {
2809        OPC_PADDB, OPC_PADDW, OPC_PADDD, OPC_PADDQ
2810    };
2811    static int const ssadd_insn[4] = {
2812        OPC_PADDSB, OPC_PADDSW, OPC_UD2, OPC_UD2
2813    };
2814    static int const usadd_insn[4] = {
2815        OPC_PADDUB, OPC_PADDUW, OPC_UD2, OPC_UD2
2816    };
2817    static int const sub_insn[4] = {
2818        OPC_PSUBB, OPC_PSUBW, OPC_PSUBD, OPC_PSUBQ
2819    };
2820    static int const sssub_insn[4] = {
2821        OPC_PSUBSB, OPC_PSUBSW, OPC_UD2, OPC_UD2
2822    };
2823    static int const ussub_insn[4] = {
2824        OPC_PSUBUB, OPC_PSUBUW, OPC_UD2, OPC_UD2
2825    };
2826    static int const mul_insn[4] = {
2827        OPC_UD2, OPC_PMULLW, OPC_PMULLD, OPC_VPMULLQ
2828    };
2829    static int const shift_imm_insn[4] = {
2830        OPC_UD2, OPC_PSHIFTW_Ib, OPC_PSHIFTD_Ib, OPC_PSHIFTQ_Ib
2831    };
2832    static int const cmpeq_insn[4] = {
2833        OPC_PCMPEQB, OPC_PCMPEQW, OPC_PCMPEQD, OPC_PCMPEQQ
2834    };
2835    static int const cmpgt_insn[4] = {
2836        OPC_PCMPGTB, OPC_PCMPGTW, OPC_PCMPGTD, OPC_PCMPGTQ
2837    };
2838    static int const punpckl_insn[4] = {
2839        OPC_PUNPCKLBW, OPC_PUNPCKLWD, OPC_PUNPCKLDQ, OPC_PUNPCKLQDQ
2840    };
2841    static int const punpckh_insn[4] = {
2842        OPC_PUNPCKHBW, OPC_PUNPCKHWD, OPC_PUNPCKHDQ, OPC_PUNPCKHQDQ
2843    };
2844    static int const packss_insn[4] = {
2845        OPC_PACKSSWB, OPC_PACKSSDW, OPC_UD2, OPC_UD2
2846    };
2847    static int const packus_insn[4] = {
2848        OPC_PACKUSWB, OPC_PACKUSDW, OPC_UD2, OPC_UD2
2849    };
2850    static int const smin_insn[4] = {
2851        OPC_PMINSB, OPC_PMINSW, OPC_PMINSD, OPC_VPMINSQ
2852    };
2853    static int const smax_insn[4] = {
2854        OPC_PMAXSB, OPC_PMAXSW, OPC_PMAXSD, OPC_VPMAXSQ
2855    };
2856    static int const umin_insn[4] = {
2857        OPC_PMINUB, OPC_PMINUW, OPC_PMINUD, OPC_VPMINUQ
2858    };
2859    static int const umax_insn[4] = {
2860        OPC_PMAXUB, OPC_PMAXUW, OPC_PMAXUD, OPC_VPMAXUQ
2861    };
2862    static int const rotlv_insn[4] = {
2863        OPC_UD2, OPC_UD2, OPC_VPROLVD, OPC_VPROLVQ
2864    };
2865    static int const rotrv_insn[4] = {
2866        OPC_UD2, OPC_UD2, OPC_VPRORVD, OPC_VPRORVQ
2867    };
2868    static int const shlv_insn[4] = {
2869        OPC_UD2, OPC_VPSLLVW, OPC_VPSLLVD, OPC_VPSLLVQ
2870    };
2871    static int const shrv_insn[4] = {
2872        OPC_UD2, OPC_VPSRLVW, OPC_VPSRLVD, OPC_VPSRLVQ
2873    };
2874    static int const sarv_insn[4] = {
2875        OPC_UD2, OPC_VPSRAVW, OPC_VPSRAVD, OPC_VPSRAVQ
2876    };
2877    static int const shls_insn[4] = {
2878        OPC_UD2, OPC_PSLLW, OPC_PSLLD, OPC_PSLLQ
2879    };
2880    static int const shrs_insn[4] = {
2881        OPC_UD2, OPC_PSRLW, OPC_PSRLD, OPC_PSRLQ
2882    };
2883    static int const sars_insn[4] = {
2884        OPC_UD2, OPC_PSRAW, OPC_PSRAD, OPC_VPSRAQ
2885    };
2886    static int const vpshldi_insn[4] = {
2887        OPC_UD2, OPC_VPSHLDW, OPC_VPSHLDD, OPC_VPSHLDQ
2888    };
2889    static int const vpshldv_insn[4] = {
2890        OPC_UD2, OPC_VPSHLDVW, OPC_VPSHLDVD, OPC_VPSHLDVQ
2891    };
2892    static int const vpshrdv_insn[4] = {
2893        OPC_UD2, OPC_VPSHRDVW, OPC_VPSHRDVD, OPC_VPSHRDVQ
2894    };
2895    static int const abs_insn[4] = {
2896        OPC_PABSB, OPC_PABSW, OPC_PABSD, OPC_VPABSQ
2897    };
2898
2899    TCGType type = vecl + TCG_TYPE_V64;
2900    int insn, sub;
2901    TCGArg a0, a1, a2, a3;
2902
2903    a0 = args[0];
2904    a1 = args[1];
2905    a2 = args[2];
2906
2907    switch (opc) {
2908    case INDEX_op_add_vec:
2909        insn = add_insn[vece];
2910        goto gen_simd;
2911    case INDEX_op_ssadd_vec:
2912        insn = ssadd_insn[vece];
2913        goto gen_simd;
2914    case INDEX_op_usadd_vec:
2915        insn = usadd_insn[vece];
2916        goto gen_simd;
2917    case INDEX_op_sub_vec:
2918        insn = sub_insn[vece];
2919        goto gen_simd;
2920    case INDEX_op_sssub_vec:
2921        insn = sssub_insn[vece];
2922        goto gen_simd;
2923    case INDEX_op_ussub_vec:
2924        insn = ussub_insn[vece];
2925        goto gen_simd;
2926    case INDEX_op_mul_vec:
2927        insn = mul_insn[vece];
2928        goto gen_simd;
2929    case INDEX_op_and_vec:
2930        insn = OPC_PAND;
2931        goto gen_simd;
2932    case INDEX_op_or_vec:
2933        insn = OPC_POR;
2934        goto gen_simd;
2935    case INDEX_op_xor_vec:
2936        insn = OPC_PXOR;
2937        goto gen_simd;
2938    case INDEX_op_smin_vec:
2939        insn = smin_insn[vece];
2940        goto gen_simd;
2941    case INDEX_op_umin_vec:
2942        insn = umin_insn[vece];
2943        goto gen_simd;
2944    case INDEX_op_smax_vec:
2945        insn = smax_insn[vece];
2946        goto gen_simd;
2947    case INDEX_op_umax_vec:
2948        insn = umax_insn[vece];
2949        goto gen_simd;
2950    case INDEX_op_shlv_vec:
2951        insn = shlv_insn[vece];
2952        goto gen_simd;
2953    case INDEX_op_shrv_vec:
2954        insn = shrv_insn[vece];
2955        goto gen_simd;
2956    case INDEX_op_sarv_vec:
2957        insn = sarv_insn[vece];
2958        goto gen_simd;
2959    case INDEX_op_rotlv_vec:
2960        insn = rotlv_insn[vece];
2961        goto gen_simd;
2962    case INDEX_op_rotrv_vec:
2963        insn = rotrv_insn[vece];
2964        goto gen_simd;
2965    case INDEX_op_shls_vec:
2966        insn = shls_insn[vece];
2967        goto gen_simd;
2968    case INDEX_op_shrs_vec:
2969        insn = shrs_insn[vece];
2970        goto gen_simd;
2971    case INDEX_op_sars_vec:
2972        insn = sars_insn[vece];
2973        goto gen_simd;
2974    case INDEX_op_x86_punpckl_vec:
2975        insn = punpckl_insn[vece];
2976        goto gen_simd;
2977    case INDEX_op_x86_punpckh_vec:
2978        insn = punpckh_insn[vece];
2979        goto gen_simd;
2980    case INDEX_op_x86_packss_vec:
2981        insn = packss_insn[vece];
2982        goto gen_simd;
2983    case INDEX_op_x86_packus_vec:
2984        insn = packus_insn[vece];
2985        goto gen_simd;
2986    case INDEX_op_x86_vpshldv_vec:
2987        insn = vpshldv_insn[vece];
2988        a1 = a2;
2989        a2 = args[3];
2990        goto gen_simd;
2991    case INDEX_op_x86_vpshrdv_vec:
2992        insn = vpshrdv_insn[vece];
2993        a1 = a2;
2994        a2 = args[3];
2995        goto gen_simd;
2996#if TCG_TARGET_REG_BITS == 32
2997    case INDEX_op_dup2_vec:
2998        /* First merge the two 32-bit inputs to a single 64-bit element. */
2999        tcg_out_vex_modrm(s, OPC_PUNPCKLDQ, a0, a1, a2);
3000        /* Then replicate the 64-bit elements across the rest of the vector. */
3001        if (type != TCG_TYPE_V64) {
3002            tcg_out_dup_vec(s, type, MO_64, a0, a0);
3003        }
3004        break;
3005#endif
3006    case INDEX_op_abs_vec:
3007        insn = abs_insn[vece];
3008        a2 = a1;
3009        a1 = 0;
3010        goto gen_simd;
3011    gen_simd:
3012        tcg_debug_assert(insn != OPC_UD2);
3013        if (type == TCG_TYPE_V256) {
3014            insn |= P_VEXL;
3015        }
3016        tcg_out_vex_modrm(s, insn, a0, a1, a2);
3017        break;
3018
3019    case INDEX_op_cmp_vec:
3020        sub = args[3];
3021        if (sub == TCG_COND_EQ) {
3022            insn = cmpeq_insn[vece];
3023        } else if (sub == TCG_COND_GT) {
3024            insn = cmpgt_insn[vece];
3025        } else {
3026            g_assert_not_reached();
3027        }
3028        goto gen_simd;
3029
3030    case INDEX_op_andc_vec:
3031        insn = OPC_PANDN;
3032        if (type == TCG_TYPE_V256) {
3033            insn |= P_VEXL;
3034        }
3035        tcg_out_vex_modrm(s, insn, a0, a2, a1);
3036        break;
3037
3038    case INDEX_op_shli_vec:
3039        insn = shift_imm_insn[vece];
3040        sub = 6;
3041        goto gen_shift;
3042    case INDEX_op_shri_vec:
3043        insn = shift_imm_insn[vece];
3044        sub = 2;
3045        goto gen_shift;
3046    case INDEX_op_sari_vec:
3047        if (vece == MO_64) {
3048            insn = OPC_PSHIFTD_Ib | P_VEXW | P_EVEX;
3049        } else {
3050            insn = shift_imm_insn[vece];
3051        }
3052        sub = 4;
3053        goto gen_shift;
3054    case INDEX_op_rotli_vec:
3055        insn = OPC_PSHIFTD_Ib | P_EVEX;  /* VPROL[DQ] */
3056        if (vece == MO_64) {
3057            insn |= P_VEXW;
3058        }
3059        sub = 1;
3060        goto gen_shift;
3061    gen_shift:
3062        tcg_debug_assert(vece != MO_8);
3063        if (type == TCG_TYPE_V256) {
3064            insn |= P_VEXL;
3065        }
3066        tcg_out_vex_modrm(s, insn, sub, a0, a1);
3067        tcg_out8(s, a2);
3068        break;
3069
3070    case INDEX_op_ld_vec:
3071        tcg_out_ld(s, type, a0, a1, a2);
3072        break;
3073    case INDEX_op_st_vec:
3074        tcg_out_st(s, type, a0, a1, a2);
3075        break;
3076    case INDEX_op_dupm_vec:
3077        tcg_out_dupm_vec(s, type, vece, a0, a1, a2);
3078        break;
3079
3080    case INDEX_op_x86_shufps_vec:
3081        insn = OPC_SHUFPS;
3082        sub = args[3];
3083        goto gen_simd_imm8;
3084    case INDEX_op_x86_blend_vec:
3085        if (vece == MO_16) {
3086            insn = OPC_PBLENDW;
3087        } else if (vece == MO_32) {
3088            insn = (have_avx2 ? OPC_VPBLENDD : OPC_BLENDPS);
3089        } else {
3090            g_assert_not_reached();
3091        }
3092        sub = args[3];
3093        goto gen_simd_imm8;
3094    case INDEX_op_x86_vperm2i128_vec:
3095        insn = OPC_VPERM2I128;
3096        sub = args[3];
3097        goto gen_simd_imm8;
3098    case INDEX_op_x86_vpshldi_vec:
3099        insn = vpshldi_insn[vece];
3100        sub = args[3];
3101        goto gen_simd_imm8;
3102
3103    case INDEX_op_not_vec:
3104        insn = OPC_VPTERNLOGQ;
3105        a2 = a1;
3106        sub = 0x33; /* !B */
3107        goto gen_simd_imm8;
3108    case INDEX_op_nor_vec:
3109        insn = OPC_VPTERNLOGQ;
3110        sub = 0x11; /* norCB */
3111        goto gen_simd_imm8;
3112    case INDEX_op_nand_vec:
3113        insn = OPC_VPTERNLOGQ;
3114        sub = 0x77; /* nandCB */
3115        goto gen_simd_imm8;
3116    case INDEX_op_eqv_vec:
3117        insn = OPC_VPTERNLOGQ;
3118        sub = 0x99; /* xnorCB */
3119        goto gen_simd_imm8;
3120    case INDEX_op_orc_vec:
3121        insn = OPC_VPTERNLOGQ;
3122        sub = 0xdd; /* orB!C */
3123        goto gen_simd_imm8;
3124
3125    case INDEX_op_bitsel_vec:
3126        insn = OPC_VPTERNLOGQ;
3127        a3 = args[3];
3128        if (a0 == a1) {
3129            a1 = a2;
3130            a2 = a3;
3131            sub = 0xca; /* A?B:C */
3132        } else if (a0 == a2) {
3133            a2 = a3;
3134            sub = 0xe2; /* B?A:C */
3135        } else {
3136            tcg_out_mov(s, type, a0, a3);
3137            sub = 0xb8; /* B?C:A */
3138        }
3139        goto gen_simd_imm8;
3140
3141    gen_simd_imm8:
3142        tcg_debug_assert(insn != OPC_UD2);
3143        if (type == TCG_TYPE_V256) {
3144            insn |= P_VEXL;
3145        }
3146        tcg_out_vex_modrm(s, insn, a0, a1, a2);
3147        tcg_out8(s, sub);
3148        break;
3149
3150    case INDEX_op_x86_vpblendvb_vec:
3151        insn = OPC_VPBLENDVB;
3152        if (type == TCG_TYPE_V256) {
3153            insn |= P_VEXL;
3154        }
3155        tcg_out_vex_modrm(s, insn, a0, a1, a2);
3156        tcg_out8(s, args[3] << 4);
3157        break;
3158
3159    case INDEX_op_x86_psrldq_vec:
3160        tcg_out_vex_modrm(s, OPC_GRP14, 3, a0, a1);
3161        tcg_out8(s, a2);
3162        break;
3163
3164    case INDEX_op_mov_vec:  /* Always emitted via tcg_out_mov.  */
3165    case INDEX_op_dup_vec:  /* Always emitted via tcg_out_dup_vec.  */
3166    default:
3167        g_assert_not_reached();
3168    }
3169}
3170
3171static TCGConstraintSetIndex tcg_target_op_def(TCGOpcode op)
3172{
3173    switch (op) {
3174    case INDEX_op_goto_ptr:
3175        return C_O0_I1(r);
3176
3177    case INDEX_op_ld8u_i32:
3178    case INDEX_op_ld8u_i64:
3179    case INDEX_op_ld8s_i32:
3180    case INDEX_op_ld8s_i64:
3181    case INDEX_op_ld16u_i32:
3182    case INDEX_op_ld16u_i64:
3183    case INDEX_op_ld16s_i32:
3184    case INDEX_op_ld16s_i64:
3185    case INDEX_op_ld_i32:
3186    case INDEX_op_ld32u_i64:
3187    case INDEX_op_ld32s_i64:
3188    case INDEX_op_ld_i64:
3189        return C_O1_I1(r, r);
3190
3191    case INDEX_op_st8_i32:
3192    case INDEX_op_st8_i64:
3193        return C_O0_I2(qi, r);
3194
3195    case INDEX_op_st16_i32:
3196    case INDEX_op_st16_i64:
3197    case INDEX_op_st_i32:
3198    case INDEX_op_st32_i64:
3199        return C_O0_I2(ri, r);
3200
3201    case INDEX_op_st_i64:
3202        return C_O0_I2(re, r);
3203
3204    case INDEX_op_add_i32:
3205    case INDEX_op_add_i64:
3206        return C_O1_I2(r, r, re);
3207
3208    case INDEX_op_sub_i32:
3209    case INDEX_op_sub_i64:
3210    case INDEX_op_mul_i32:
3211    case INDEX_op_mul_i64:
3212    case INDEX_op_or_i32:
3213    case INDEX_op_or_i64:
3214    case INDEX_op_xor_i32:
3215    case INDEX_op_xor_i64:
3216        return C_O1_I2(r, 0, re);
3217
3218    case INDEX_op_and_i32:
3219    case INDEX_op_and_i64:
3220        return C_O1_I2(r, 0, reZ);
3221
3222    case INDEX_op_andc_i32:
3223    case INDEX_op_andc_i64:
3224        return C_O1_I2(r, r, rI);
3225
3226    case INDEX_op_shl_i32:
3227    case INDEX_op_shl_i64:
3228    case INDEX_op_shr_i32:
3229    case INDEX_op_shr_i64:
3230    case INDEX_op_sar_i32:
3231    case INDEX_op_sar_i64:
3232        return have_bmi2 ? C_O1_I2(r, r, ri) : C_O1_I2(r, 0, ci);
3233
3234    case INDEX_op_rotl_i32:
3235    case INDEX_op_rotl_i64:
3236    case INDEX_op_rotr_i32:
3237    case INDEX_op_rotr_i64:
3238        return C_O1_I2(r, 0, ci);
3239
3240    case INDEX_op_brcond_i32:
3241    case INDEX_op_brcond_i64:
3242        return C_O0_I2(r, re);
3243
3244    case INDEX_op_bswap16_i32:
3245    case INDEX_op_bswap16_i64:
3246    case INDEX_op_bswap32_i32:
3247    case INDEX_op_bswap32_i64:
3248    case INDEX_op_bswap64_i64:
3249    case INDEX_op_neg_i32:
3250    case INDEX_op_neg_i64:
3251    case INDEX_op_not_i32:
3252    case INDEX_op_not_i64:
3253    case INDEX_op_extrh_i64_i32:
3254        return C_O1_I1(r, 0);
3255
3256    case INDEX_op_ext8s_i32:
3257    case INDEX_op_ext8s_i64:
3258    case INDEX_op_ext8u_i32:
3259    case INDEX_op_ext8u_i64:
3260        return C_O1_I1(r, q);
3261
3262    case INDEX_op_ext16s_i32:
3263    case INDEX_op_ext16s_i64:
3264    case INDEX_op_ext16u_i32:
3265    case INDEX_op_ext16u_i64:
3266    case INDEX_op_ext32s_i64:
3267    case INDEX_op_ext32u_i64:
3268    case INDEX_op_ext_i32_i64:
3269    case INDEX_op_extu_i32_i64:
3270    case INDEX_op_extrl_i64_i32:
3271    case INDEX_op_extract_i32:
3272    case INDEX_op_extract_i64:
3273    case INDEX_op_sextract_i32:
3274    case INDEX_op_ctpop_i32:
3275    case INDEX_op_ctpop_i64:
3276        return C_O1_I1(r, r);
3277
3278    case INDEX_op_extract2_i32:
3279    case INDEX_op_extract2_i64:
3280        return C_O1_I2(r, 0, r);
3281
3282    case INDEX_op_deposit_i32:
3283    case INDEX_op_deposit_i64:
3284        return C_O1_I2(Q, 0, Q);
3285
3286    case INDEX_op_setcond_i32:
3287    case INDEX_op_setcond_i64:
3288        return C_O1_I2(q, r, re);
3289
3290    case INDEX_op_movcond_i32:
3291    case INDEX_op_movcond_i64:
3292        return C_O1_I4(r, r, re, r, 0);
3293
3294    case INDEX_op_div2_i32:
3295    case INDEX_op_div2_i64:
3296    case INDEX_op_divu2_i32:
3297    case INDEX_op_divu2_i64:
3298        return C_O2_I3(a, d, 0, 1, r);
3299
3300    case INDEX_op_mulu2_i32:
3301    case INDEX_op_mulu2_i64:
3302    case INDEX_op_muls2_i32:
3303    case INDEX_op_muls2_i64:
3304        return C_O2_I2(a, d, a, r);
3305
3306    case INDEX_op_add2_i32:
3307    case INDEX_op_add2_i64:
3308    case INDEX_op_sub2_i32:
3309    case INDEX_op_sub2_i64:
3310        return C_O2_I4(r, r, 0, 1, re, re);
3311
3312    case INDEX_op_ctz_i32:
3313    case INDEX_op_ctz_i64:
3314        return have_bmi1 ? C_N1_I2(r, r, rW) : C_N1_I2(r, r, r);
3315
3316    case INDEX_op_clz_i32:
3317    case INDEX_op_clz_i64:
3318        return have_lzcnt ? C_N1_I2(r, r, rW) : C_N1_I2(r, r, r);
3319
3320    case INDEX_op_qemu_ld_i32:
3321        return (TARGET_LONG_BITS <= TCG_TARGET_REG_BITS
3322                ? C_O1_I1(r, L) : C_O1_I2(r, L, L));
3323
3324    case INDEX_op_qemu_st_i32:
3325        return (TARGET_LONG_BITS <= TCG_TARGET_REG_BITS
3326                ? C_O0_I2(L, L) : C_O0_I3(L, L, L));
3327    case INDEX_op_qemu_st8_i32:
3328        return (TARGET_LONG_BITS <= TCG_TARGET_REG_BITS
3329                ? C_O0_I2(s, L) : C_O0_I3(s, L, L));
3330
3331    case INDEX_op_qemu_ld_i64:
3332        return (TCG_TARGET_REG_BITS == 64 ? C_O1_I1(r, L)
3333                : TARGET_LONG_BITS <= TCG_TARGET_REG_BITS ? C_O2_I1(r, r, L)
3334                : C_O2_I2(r, r, L, L));
3335
3336    case INDEX_op_qemu_st_i64:
3337        return (TCG_TARGET_REG_BITS == 64 ? C_O0_I2(L, L)
3338                : TARGET_LONG_BITS <= TCG_TARGET_REG_BITS ? C_O0_I3(L, L, L)
3339                : C_O0_I4(L, L, L, L));
3340
3341    case INDEX_op_brcond2_i32:
3342        return C_O0_I4(r, r, ri, ri);
3343
3344    case INDEX_op_setcond2_i32:
3345        return C_O1_I4(r, r, r, ri, ri);
3346
3347    case INDEX_op_ld_vec:
3348    case INDEX_op_dupm_vec:
3349        return C_O1_I1(x, r);
3350
3351    case INDEX_op_st_vec:
3352        return C_O0_I2(x, r);
3353
3354    case INDEX_op_add_vec:
3355    case INDEX_op_sub_vec:
3356    case INDEX_op_mul_vec:
3357    case INDEX_op_and_vec:
3358    case INDEX_op_or_vec:
3359    case INDEX_op_xor_vec:
3360    case INDEX_op_andc_vec:
3361    case INDEX_op_orc_vec:
3362    case INDEX_op_nand_vec:
3363    case INDEX_op_nor_vec:
3364    case INDEX_op_eqv_vec:
3365    case INDEX_op_ssadd_vec:
3366    case INDEX_op_usadd_vec:
3367    case INDEX_op_sssub_vec:
3368    case INDEX_op_ussub_vec:
3369    case INDEX_op_smin_vec:
3370    case INDEX_op_umin_vec:
3371    case INDEX_op_smax_vec:
3372    case INDEX_op_umax_vec:
3373    case INDEX_op_shlv_vec:
3374    case INDEX_op_shrv_vec:
3375    case INDEX_op_sarv_vec:
3376    case INDEX_op_rotlv_vec:
3377    case INDEX_op_rotrv_vec:
3378    case INDEX_op_shls_vec:
3379    case INDEX_op_shrs_vec:
3380    case INDEX_op_sars_vec:
3381    case INDEX_op_cmp_vec:
3382    case INDEX_op_x86_shufps_vec:
3383    case INDEX_op_x86_blend_vec:
3384    case INDEX_op_x86_packss_vec:
3385    case INDEX_op_x86_packus_vec:
3386    case INDEX_op_x86_vperm2i128_vec:
3387    case INDEX_op_x86_punpckl_vec:
3388    case INDEX_op_x86_punpckh_vec:
3389    case INDEX_op_x86_vpshldi_vec:
3390#if TCG_TARGET_REG_BITS == 32
3391    case INDEX_op_dup2_vec:
3392#endif
3393        return C_O1_I2(x, x, x);
3394
3395    case INDEX_op_abs_vec:
3396    case INDEX_op_dup_vec:
3397    case INDEX_op_not_vec:
3398    case INDEX_op_shli_vec:
3399    case INDEX_op_shri_vec:
3400    case INDEX_op_sari_vec:
3401    case INDEX_op_rotli_vec:
3402    case INDEX_op_x86_psrldq_vec:
3403        return C_O1_I1(x, x);
3404
3405    case INDEX_op_x86_vpshldv_vec:
3406    case INDEX_op_x86_vpshrdv_vec:
3407        return C_O1_I3(x, 0, x, x);
3408
3409    case INDEX_op_bitsel_vec:
3410    case INDEX_op_x86_vpblendvb_vec:
3411        return C_O1_I3(x, x, x, x);
3412
3413    default:
3414        g_assert_not_reached();
3415    }
3416}
3417
3418int tcg_can_emit_vec_op(TCGOpcode opc, TCGType type, unsigned vece)
3419{
3420    switch (opc) {
3421    case INDEX_op_add_vec:
3422    case INDEX_op_sub_vec:
3423    case INDEX_op_and_vec:
3424    case INDEX_op_or_vec:
3425    case INDEX_op_xor_vec:
3426    case INDEX_op_andc_vec:
3427    case INDEX_op_orc_vec:
3428    case INDEX_op_nand_vec:
3429    case INDEX_op_nor_vec:
3430    case INDEX_op_eqv_vec:
3431    case INDEX_op_not_vec:
3432    case INDEX_op_bitsel_vec:
3433        return 1;
3434    case INDEX_op_cmp_vec:
3435    case INDEX_op_cmpsel_vec:
3436        return -1;
3437
3438    case INDEX_op_rotli_vec:
3439        return have_avx512vl && vece >= MO_32 ? 1 : -1;
3440
3441    case INDEX_op_shli_vec:
3442    case INDEX_op_shri_vec:
3443        /* We must expand the operation for MO_8.  */
3444        return vece == MO_8 ? -1 : 1;
3445
3446    case INDEX_op_sari_vec:
3447        switch (vece) {
3448        case MO_8:
3449            return -1;
3450        case MO_16:
3451        case MO_32:
3452            return 1;
3453        case MO_64:
3454            if (have_avx512vl) {
3455                return 1;
3456            }
3457            /*
3458             * We can emulate this for MO_64, but it does not pay off
3459             * unless we're producing at least 4 values.
3460             */
3461            return type >= TCG_TYPE_V256 ? -1 : 0;
3462        }
3463        return 0;
3464
3465    case INDEX_op_shls_vec:
3466    case INDEX_op_shrs_vec:
3467        return vece >= MO_16;
3468    case INDEX_op_sars_vec:
3469        switch (vece) {
3470        case MO_16:
3471        case MO_32:
3472            return 1;
3473        case MO_64:
3474            return have_avx512vl;
3475        }
3476        return 0;
3477    case INDEX_op_rotls_vec:
3478        return vece >= MO_16 ? -1 : 0;
3479
3480    case INDEX_op_shlv_vec:
3481    case INDEX_op_shrv_vec:
3482        switch (vece) {
3483        case MO_16:
3484            return have_avx512bw;
3485        case MO_32:
3486        case MO_64:
3487            return have_avx2;
3488        }
3489        return 0;
3490    case INDEX_op_sarv_vec:
3491        switch (vece) {
3492        case MO_16:
3493            return have_avx512bw;
3494        case MO_32:
3495            return have_avx2;
3496        case MO_64:
3497            return have_avx512vl;
3498        }
3499        return 0;
3500    case INDEX_op_rotlv_vec:
3501    case INDEX_op_rotrv_vec:
3502        switch (vece) {
3503        case MO_16:
3504            return have_avx512vbmi2 ? -1 : 0;
3505        case MO_32:
3506        case MO_64:
3507            return have_avx512vl ? 1 : have_avx2 ? -1 : 0;
3508        }
3509        return 0;
3510
3511    case INDEX_op_mul_vec:
3512        switch (vece) {
3513        case MO_8:
3514            return -1;
3515        case MO_64:
3516            return have_avx512dq;
3517        }
3518        return 1;
3519
3520    case INDEX_op_ssadd_vec:
3521    case INDEX_op_usadd_vec:
3522    case INDEX_op_sssub_vec:
3523    case INDEX_op_ussub_vec:
3524        return vece <= MO_16;
3525    case INDEX_op_smin_vec:
3526    case INDEX_op_smax_vec:
3527    case INDEX_op_umin_vec:
3528    case INDEX_op_umax_vec:
3529    case INDEX_op_abs_vec:
3530        return vece <= MO_32 || have_avx512vl;
3531
3532    default:
3533        return 0;
3534    }
3535}
3536
3537static void expand_vec_shi(TCGType type, unsigned vece, TCGOpcode opc,
3538                           TCGv_vec v0, TCGv_vec v1, TCGArg imm)
3539{
3540    TCGv_vec t1, t2;
3541
3542    tcg_debug_assert(vece == MO_8);
3543
3544    t1 = tcg_temp_new_vec(type);
3545    t2 = tcg_temp_new_vec(type);
3546
3547    /*
3548     * Unpack to W, shift, and repack.  Tricky bits:
3549     * (1) Use punpck*bw x,x to produce DDCCBBAA,
3550     *     i.e. duplicate in other half of the 16-bit lane.
3551     * (2) For right-shift, add 8 so that the high half of the lane
3552     *     becomes zero.  For left-shift, and left-rotate, we must
3553     *     shift up and down again.
3554     * (3) Step 2 leaves high half zero such that PACKUSWB
3555     *     (pack with unsigned saturation) does not modify
3556     *     the quantity.
3557     */
3558    vec_gen_3(INDEX_op_x86_punpckl_vec, type, MO_8,
3559              tcgv_vec_arg(t1), tcgv_vec_arg(v1), tcgv_vec_arg(v1));
3560    vec_gen_3(INDEX_op_x86_punpckh_vec, type, MO_8,
3561              tcgv_vec_arg(t2), tcgv_vec_arg(v1), tcgv_vec_arg(v1));
3562
3563    if (opc != INDEX_op_rotli_vec) {
3564        imm += 8;
3565    }
3566    if (opc == INDEX_op_shri_vec) {
3567        tcg_gen_shri_vec(MO_16, t1, t1, imm);
3568        tcg_gen_shri_vec(MO_16, t2, t2, imm);
3569    } else {
3570        tcg_gen_shli_vec(MO_16, t1, t1, imm);
3571        tcg_gen_shli_vec(MO_16, t2, t2, imm);
3572        tcg_gen_shri_vec(MO_16, t1, t1, 8);
3573        tcg_gen_shri_vec(MO_16, t2, t2, 8);
3574    }
3575
3576    vec_gen_3(INDEX_op_x86_packus_vec, type, MO_8,
3577              tcgv_vec_arg(v0), tcgv_vec_arg(t1), tcgv_vec_arg(t2));
3578    tcg_temp_free_vec(t1);
3579    tcg_temp_free_vec(t2);
3580}
3581
3582static void expand_vec_sari(TCGType type, unsigned vece,
3583                            TCGv_vec v0, TCGv_vec v1, TCGArg imm)
3584{
3585    TCGv_vec t1, t2;
3586
3587    switch (vece) {
3588    case MO_8:
3589        /* Unpack to W, shift, and repack, as in expand_vec_shi.  */
3590        t1 = tcg_temp_new_vec(type);
3591        t2 = tcg_temp_new_vec(type);
3592        vec_gen_3(INDEX_op_x86_punpckl_vec, type, MO_8,
3593                  tcgv_vec_arg(t1), tcgv_vec_arg(v1), tcgv_vec_arg(v1));
3594        vec_gen_3(INDEX_op_x86_punpckh_vec, type, MO_8,
3595                  tcgv_vec_arg(t2), tcgv_vec_arg(v1), tcgv_vec_arg(v1));
3596        tcg_gen_sari_vec(MO_16, t1, t1, imm + 8);
3597        tcg_gen_sari_vec(MO_16, t2, t2, imm + 8);
3598        vec_gen_3(INDEX_op_x86_packss_vec, type, MO_8,
3599                  tcgv_vec_arg(v0), tcgv_vec_arg(t1), tcgv_vec_arg(t2));
3600        tcg_temp_free_vec(t1);
3601        tcg_temp_free_vec(t2);
3602        break;
3603
3604    case MO_64:
3605        if (imm <= 32) {
3606            /*
3607             * We can emulate a small sign extend by performing an arithmetic
3608             * 32-bit shift and overwriting the high half of a 64-bit logical
3609             * shift.  Note that the ISA says shift of 32 is valid, but TCG
3610             * does not, so we have to bound the smaller shift -- we get the
3611             * same result in the high half either way.
3612             */
3613            t1 = tcg_temp_new_vec(type);
3614            tcg_gen_sari_vec(MO_32, t1, v1, MIN(imm, 31));
3615            tcg_gen_shri_vec(MO_64, v0, v1, imm);
3616            vec_gen_4(INDEX_op_x86_blend_vec, type, MO_32,
3617                      tcgv_vec_arg(v0), tcgv_vec_arg(v0),
3618                      tcgv_vec_arg(t1), 0xaa);
3619            tcg_temp_free_vec(t1);
3620        } else {
3621            /* Otherwise we will need to use a compare vs 0 to produce
3622             * the sign-extend, shift and merge.
3623             */
3624            t1 = tcg_const_zeros_vec(type);
3625            tcg_gen_cmp_vec(TCG_COND_GT, MO_64, t1, t1, v1);
3626            tcg_gen_shri_vec(MO_64, v0, v1, imm);
3627            tcg_gen_shli_vec(MO_64, t1, t1, 64 - imm);
3628            tcg_gen_or_vec(MO_64, v0, v0, t1);
3629            tcg_temp_free_vec(t1);
3630        }
3631        break;
3632
3633    default:
3634        g_assert_not_reached();
3635    }
3636}
3637
3638static void expand_vec_rotli(TCGType type, unsigned vece,
3639                             TCGv_vec v0, TCGv_vec v1, TCGArg imm)
3640{
3641    TCGv_vec t;
3642
3643    if (vece == MO_8) {
3644        expand_vec_shi(type, vece, INDEX_op_rotli_vec, v0, v1, imm);
3645        return;
3646    }
3647
3648    if (have_avx512vbmi2) {
3649        vec_gen_4(INDEX_op_x86_vpshldi_vec, type, vece,
3650                  tcgv_vec_arg(v0), tcgv_vec_arg(v1), tcgv_vec_arg(v1), imm);
3651        return;
3652    }
3653
3654    t = tcg_temp_new_vec(type);
3655    tcg_gen_shli_vec(vece, t, v1, imm);
3656    tcg_gen_shri_vec(vece, v0, v1, (8 << vece) - imm);
3657    tcg_gen_or_vec(vece, v0, v0, t);
3658    tcg_temp_free_vec(t);
3659}
3660
3661static void expand_vec_rotv(TCGType type, unsigned vece, TCGv_vec v0,
3662                            TCGv_vec v1, TCGv_vec sh, bool right)
3663{
3664    TCGv_vec t;
3665
3666    if (have_avx512vbmi2) {
3667        vec_gen_4(right ? INDEX_op_x86_vpshrdv_vec : INDEX_op_x86_vpshldv_vec,
3668                  type, vece, tcgv_vec_arg(v0), tcgv_vec_arg(v1),
3669                  tcgv_vec_arg(v1), tcgv_vec_arg(sh));
3670        return;
3671    }
3672
3673    t = tcg_temp_new_vec(type);
3674    tcg_gen_dupi_vec(vece, t, 8 << vece);
3675    tcg_gen_sub_vec(vece, t, t, sh);
3676    if (right) {
3677        tcg_gen_shlv_vec(vece, t, v1, t);
3678        tcg_gen_shrv_vec(vece, v0, v1, sh);
3679    } else {
3680        tcg_gen_shrv_vec(vece, t, v1, t);
3681        tcg_gen_shlv_vec(vece, v0, v1, sh);
3682    }
3683    tcg_gen_or_vec(vece, v0, v0, t);
3684    tcg_temp_free_vec(t);
3685}
3686
3687static void expand_vec_rotls(TCGType type, unsigned vece,
3688                             TCGv_vec v0, TCGv_vec v1, TCGv_i32 lsh)
3689{
3690    TCGv_vec t = tcg_temp_new_vec(type);
3691
3692    tcg_debug_assert(vece != MO_8);
3693
3694    if (vece >= MO_32 ? have_avx512vl : have_avx512vbmi2) {
3695        tcg_gen_dup_i32_vec(vece, t, lsh);
3696        if (vece >= MO_32) {
3697            tcg_gen_rotlv_vec(vece, v0, v1, t);
3698        } else {
3699            expand_vec_rotv(type, vece, v0, v1, t, false);
3700        }
3701    } else {
3702        TCGv_i32 rsh = tcg_temp_new_i32();
3703
3704        tcg_gen_neg_i32(rsh, lsh);
3705        tcg_gen_andi_i32(rsh, rsh, (8 << vece) - 1);
3706        tcg_gen_shls_vec(vece, t, v1, lsh);
3707        tcg_gen_shrs_vec(vece, v0, v1, rsh);
3708        tcg_gen_or_vec(vece, v0, v0, t);
3709
3710        tcg_temp_free_i32(rsh);
3711    }
3712
3713    tcg_temp_free_vec(t);
3714}
3715
3716static void expand_vec_mul(TCGType type, unsigned vece,
3717                           TCGv_vec v0, TCGv_vec v1, TCGv_vec v2)
3718{
3719    TCGv_vec t1, t2, t3, t4, zero;
3720
3721    tcg_debug_assert(vece == MO_8);
3722
3723    /*
3724     * Unpack v1 bytes to words, 0 | x.
3725     * Unpack v2 bytes to words, y | 0.
3726     * This leaves the 8-bit result, x * y, with 8 bits of right padding.
3727     * Shift logical right by 8 bits to clear the high 8 bytes before
3728     * using an unsigned saturated pack.
3729     *
3730     * The difference between the V64, V128 and V256 cases is merely how
3731     * we distribute the expansion between temporaries.
3732     */
3733    switch (type) {
3734    case TCG_TYPE_V64:
3735        t1 = tcg_temp_new_vec(TCG_TYPE_V128);
3736        t2 = tcg_temp_new_vec(TCG_TYPE_V128);
3737        zero = tcg_constant_vec(TCG_TYPE_V128, MO_8, 0);
3738        vec_gen_3(INDEX_op_x86_punpckl_vec, TCG_TYPE_V128, MO_8,
3739                  tcgv_vec_arg(t1), tcgv_vec_arg(v1), tcgv_vec_arg(zero));
3740        vec_gen_3(INDEX_op_x86_punpckl_vec, TCG_TYPE_V128, MO_8,
3741                  tcgv_vec_arg(t2), tcgv_vec_arg(zero), tcgv_vec_arg(v2));
3742        tcg_gen_mul_vec(MO_16, t1, t1, t2);
3743        tcg_gen_shri_vec(MO_16, t1, t1, 8);
3744        vec_gen_3(INDEX_op_x86_packus_vec, TCG_TYPE_V128, MO_8,
3745                  tcgv_vec_arg(v0), tcgv_vec_arg(t1), tcgv_vec_arg(t1));
3746        tcg_temp_free_vec(t1);
3747        tcg_temp_free_vec(t2);
3748        break;
3749
3750    case TCG_TYPE_V128:
3751    case TCG_TYPE_V256:
3752        t1 = tcg_temp_new_vec(type);
3753        t2 = tcg_temp_new_vec(type);
3754        t3 = tcg_temp_new_vec(type);
3755        t4 = tcg_temp_new_vec(type);
3756        zero = tcg_constant_vec(TCG_TYPE_V128, MO_8, 0);
3757        vec_gen_3(INDEX_op_x86_punpckl_vec, type, MO_8,
3758                  tcgv_vec_arg(t1), tcgv_vec_arg(v1), tcgv_vec_arg(zero));
3759        vec_gen_3(INDEX_op_x86_punpckl_vec, type, MO_8,
3760                  tcgv_vec_arg(t2), tcgv_vec_arg(zero), tcgv_vec_arg(v2));
3761        vec_gen_3(INDEX_op_x86_punpckh_vec, type, MO_8,
3762                  tcgv_vec_arg(t3), tcgv_vec_arg(v1), tcgv_vec_arg(zero));
3763        vec_gen_3(INDEX_op_x86_punpckh_vec, type, MO_8,
3764                  tcgv_vec_arg(t4), tcgv_vec_arg(zero), tcgv_vec_arg(v2));
3765        tcg_gen_mul_vec(MO_16, t1, t1, t2);
3766        tcg_gen_mul_vec(MO_16, t3, t3, t4);
3767        tcg_gen_shri_vec(MO_16, t1, t1, 8);
3768        tcg_gen_shri_vec(MO_16, t3, t3, 8);
3769        vec_gen_3(INDEX_op_x86_packus_vec, type, MO_8,
3770                  tcgv_vec_arg(v0), tcgv_vec_arg(t1), tcgv_vec_arg(t3));
3771        tcg_temp_free_vec(t1);
3772        tcg_temp_free_vec(t2);
3773        tcg_temp_free_vec(t3);
3774        tcg_temp_free_vec(t4);
3775        break;
3776
3777    default:
3778        g_assert_not_reached();
3779    }
3780}
3781
3782static bool expand_vec_cmp_noinv(TCGType type, unsigned vece, TCGv_vec v0,
3783                                 TCGv_vec v1, TCGv_vec v2, TCGCond cond)
3784{
3785    enum {
3786        NEED_INV  = 1,
3787        NEED_SWAP = 2,
3788        NEED_BIAS = 4,
3789        NEED_UMIN = 8,
3790        NEED_UMAX = 16,
3791    };
3792    TCGv_vec t1, t2, t3;
3793    uint8_t fixup;
3794
3795    switch (cond) {
3796    case TCG_COND_EQ:
3797    case TCG_COND_GT:
3798        fixup = 0;
3799        break;
3800    case TCG_COND_NE:
3801    case TCG_COND_LE:
3802        fixup = NEED_INV;
3803        break;
3804    case TCG_COND_LT:
3805        fixup = NEED_SWAP;
3806        break;
3807    case TCG_COND_GE:
3808        fixup = NEED_SWAP | NEED_INV;
3809        break;
3810    case TCG_COND_LEU:
3811        if (tcg_can_emit_vec_op(INDEX_op_umin_vec, type, vece)) {
3812            fixup = NEED_UMIN;
3813        } else {
3814            fixup = NEED_BIAS | NEED_INV;
3815        }
3816        break;
3817    case TCG_COND_GTU:
3818        if (tcg_can_emit_vec_op(INDEX_op_umin_vec, type, vece)) {
3819            fixup = NEED_UMIN | NEED_INV;
3820        } else {
3821            fixup = NEED_BIAS;
3822        }
3823        break;
3824    case TCG_COND_GEU:
3825        if (tcg_can_emit_vec_op(INDEX_op_umax_vec, type, vece)) {
3826            fixup = NEED_UMAX;
3827        } else {
3828            fixup = NEED_BIAS | NEED_SWAP | NEED_INV;
3829        }
3830        break;
3831    case TCG_COND_LTU:
3832        if (tcg_can_emit_vec_op(INDEX_op_umax_vec, type, vece)) {
3833            fixup = NEED_UMAX | NEED_INV;
3834        } else {
3835            fixup = NEED_BIAS | NEED_SWAP;
3836        }
3837        break;
3838    default:
3839        g_assert_not_reached();
3840    }
3841
3842    if (fixup & NEED_INV) {
3843        cond = tcg_invert_cond(cond);
3844    }
3845    if (fixup & NEED_SWAP) {
3846        t1 = v1, v1 = v2, v2 = t1;
3847        cond = tcg_swap_cond(cond);
3848    }
3849
3850    t1 = t2 = NULL;
3851    if (fixup & (NEED_UMIN | NEED_UMAX)) {
3852        t1 = tcg_temp_new_vec(type);
3853        if (fixup & NEED_UMIN) {
3854            tcg_gen_umin_vec(vece, t1, v1, v2);
3855        } else {
3856            tcg_gen_umax_vec(vece, t1, v1, v2);
3857        }
3858        v2 = t1;
3859        cond = TCG_COND_EQ;
3860    } else if (fixup & NEED_BIAS) {
3861        t1 = tcg_temp_new_vec(type);
3862        t2 = tcg_temp_new_vec(type);
3863        t3 = tcg_constant_vec(type, vece, 1ull << ((8 << vece) - 1));
3864        tcg_gen_sub_vec(vece, t1, v1, t3);
3865        tcg_gen_sub_vec(vece, t2, v2, t3);
3866        v1 = t1;
3867        v2 = t2;
3868        cond = tcg_signed_cond(cond);
3869    }
3870
3871    tcg_debug_assert(cond == TCG_COND_EQ || cond == TCG_COND_GT);
3872    /* Expand directly; do not recurse.  */
3873    vec_gen_4(INDEX_op_cmp_vec, type, vece,
3874              tcgv_vec_arg(v0), tcgv_vec_arg(v1), tcgv_vec_arg(v2), cond);
3875
3876    if (t1) {
3877        tcg_temp_free_vec(t1);
3878        if (t2) {
3879            tcg_temp_free_vec(t2);
3880        }
3881    }
3882    return fixup & NEED_INV;
3883}
3884
3885static void expand_vec_cmp(TCGType type, unsigned vece, TCGv_vec v0,
3886                           TCGv_vec v1, TCGv_vec v2, TCGCond cond)
3887{
3888    if (expand_vec_cmp_noinv(type, vece, v0, v1, v2, cond)) {
3889        tcg_gen_not_vec(vece, v0, v0);
3890    }
3891}
3892
3893static void expand_vec_cmpsel(TCGType type, unsigned vece, TCGv_vec v0,
3894                              TCGv_vec c1, TCGv_vec c2,
3895                              TCGv_vec v3, TCGv_vec v4, TCGCond cond)
3896{
3897    TCGv_vec t = tcg_temp_new_vec(type);
3898
3899    if (expand_vec_cmp_noinv(type, vece, t, c1, c2, cond)) {
3900        /* Invert the sense of the compare by swapping arguments.  */
3901        TCGv_vec x;
3902        x = v3, v3 = v4, v4 = x;
3903    }
3904    vec_gen_4(INDEX_op_x86_vpblendvb_vec, type, vece,
3905              tcgv_vec_arg(v0), tcgv_vec_arg(v4),
3906              tcgv_vec_arg(v3), tcgv_vec_arg(t));
3907    tcg_temp_free_vec(t);
3908}
3909
3910void tcg_expand_vec_op(TCGOpcode opc, TCGType type, unsigned vece,
3911                       TCGArg a0, ...)
3912{
3913    va_list va;
3914    TCGArg a2;
3915    TCGv_vec v0, v1, v2, v3, v4;
3916
3917    va_start(va, a0);
3918    v0 = temp_tcgv_vec(arg_temp(a0));
3919    v1 = temp_tcgv_vec(arg_temp(va_arg(va, TCGArg)));
3920    a2 = va_arg(va, TCGArg);
3921
3922    switch (opc) {
3923    case INDEX_op_shli_vec:
3924    case INDEX_op_shri_vec:
3925        expand_vec_shi(type, vece, opc, v0, v1, a2);
3926        break;
3927
3928    case INDEX_op_sari_vec:
3929        expand_vec_sari(type, vece, v0, v1, a2);
3930        break;
3931
3932    case INDEX_op_rotli_vec:
3933        expand_vec_rotli(type, vece, v0, v1, a2);
3934        break;
3935
3936    case INDEX_op_rotls_vec:
3937        expand_vec_rotls(type, vece, v0, v1, temp_tcgv_i32(arg_temp(a2)));
3938        break;
3939
3940    case INDEX_op_rotlv_vec:
3941        v2 = temp_tcgv_vec(arg_temp(a2));
3942        expand_vec_rotv(type, vece, v0, v1, v2, false);
3943        break;
3944    case INDEX_op_rotrv_vec:
3945        v2 = temp_tcgv_vec(arg_temp(a2));
3946        expand_vec_rotv(type, vece, v0, v1, v2, true);
3947        break;
3948
3949    case INDEX_op_mul_vec:
3950        v2 = temp_tcgv_vec(arg_temp(a2));
3951        expand_vec_mul(type, vece, v0, v1, v2);
3952        break;
3953
3954    case INDEX_op_cmp_vec:
3955        v2 = temp_tcgv_vec(arg_temp(a2));
3956        expand_vec_cmp(type, vece, v0, v1, v2, va_arg(va, TCGArg));
3957        break;
3958
3959    case INDEX_op_cmpsel_vec:
3960        v2 = temp_tcgv_vec(arg_temp(a2));
3961        v3 = temp_tcgv_vec(arg_temp(va_arg(va, TCGArg)));
3962        v4 = temp_tcgv_vec(arg_temp(va_arg(va, TCGArg)));
3963        expand_vec_cmpsel(type, vece, v0, v1, v2, v3, v4, va_arg(va, TCGArg));
3964        break;
3965
3966    default:
3967        break;
3968    }
3969
3970    va_end(va);
3971}
3972
3973static const int tcg_target_callee_save_regs[] = {
3974#if TCG_TARGET_REG_BITS == 64
3975    TCG_REG_RBP,
3976    TCG_REG_RBX,
3977#if defined(_WIN64)
3978    TCG_REG_RDI,
3979    TCG_REG_RSI,
3980#endif
3981    TCG_REG_R12,
3982    TCG_REG_R13,
3983    TCG_REG_R14, /* Currently used for the global env. */
3984    TCG_REG_R15,
3985#else
3986    TCG_REG_EBP, /* Currently used for the global env. */
3987    TCG_REG_EBX,
3988    TCG_REG_ESI,
3989    TCG_REG_EDI,
3990#endif
3991};
3992
3993/* Compute frame size via macros, to share between tcg_target_qemu_prologue
3994   and tcg_register_jit.  */
3995
3996#define PUSH_SIZE \
3997    ((1 + ARRAY_SIZE(tcg_target_callee_save_regs)) \
3998     * (TCG_TARGET_REG_BITS / 8))
3999
4000#define FRAME_SIZE \
4001    ((PUSH_SIZE \
4002      + TCG_STATIC_CALL_ARGS_SIZE \
4003      + CPU_TEMP_BUF_NLONGS * sizeof(long) \
4004      + TCG_TARGET_STACK_ALIGN - 1) \
4005     & ~(TCG_TARGET_STACK_ALIGN - 1))
4006
4007/* Generate global QEMU prologue and epilogue code */
4008static void tcg_target_qemu_prologue(TCGContext *s)
4009{
4010    int i, stack_addend;
4011
4012    /* TB prologue */
4013
4014    /* Reserve some stack space, also for TCG temps.  */
4015    stack_addend = FRAME_SIZE - PUSH_SIZE;
4016    tcg_set_frame(s, TCG_REG_CALL_STACK, TCG_STATIC_CALL_ARGS_SIZE,
4017                  CPU_TEMP_BUF_NLONGS * sizeof(long));
4018
4019    /* Save all callee saved registers.  */
4020    for (i = 0; i < ARRAY_SIZE(tcg_target_callee_save_regs); i++) {
4021        tcg_out_push(s, tcg_target_callee_save_regs[i]);
4022    }
4023
4024#if TCG_TARGET_REG_BITS == 32
4025    tcg_out_ld(s, TCG_TYPE_PTR, TCG_AREG0, TCG_REG_ESP,
4026               (ARRAY_SIZE(tcg_target_callee_save_regs) + 1) * 4);
4027    tcg_out_addi(s, TCG_REG_ESP, -stack_addend);
4028    /* jmp *tb.  */
4029    tcg_out_modrm_offset(s, OPC_GRP5, EXT5_JMPN_Ev, TCG_REG_ESP,
4030                         (ARRAY_SIZE(tcg_target_callee_save_regs) + 2) * 4
4031                         + stack_addend);
4032#else
4033# if !defined(CONFIG_SOFTMMU) && TCG_TARGET_REG_BITS == 64
4034    if (guest_base) {
4035        int seg = setup_guest_base_seg();
4036        if (seg != 0) {
4037            x86_guest_base_seg = seg;
4038        } else if (guest_base == (int32_t)guest_base) {
4039            x86_guest_base_offset = guest_base;
4040        } else {
4041            /* Choose R12 because, as a base, it requires a SIB byte. */
4042            x86_guest_base_index = TCG_REG_R12;
4043            tcg_out_movi(s, TCG_TYPE_PTR, x86_guest_base_index, guest_base);
4044            tcg_regset_set_reg(s->reserved_regs, x86_guest_base_index);
4045        }
4046    }
4047# endif
4048    tcg_out_mov(s, TCG_TYPE_PTR, TCG_AREG0, tcg_target_call_iarg_regs[0]);
4049    tcg_out_addi(s, TCG_REG_ESP, -stack_addend);
4050    /* jmp *tb.  */
4051    tcg_out_modrm(s, OPC_GRP5, EXT5_JMPN_Ev, tcg_target_call_iarg_regs[1]);
4052#endif
4053
4054    /*
4055     * Return path for goto_ptr. Set return value to 0, a-la exit_tb,
4056     * and fall through to the rest of the epilogue.
4057     */
4058    tcg_code_gen_epilogue = tcg_splitwx_to_rx(s->code_ptr);
4059    tcg_out_movi(s, TCG_TYPE_REG, TCG_REG_EAX, 0);
4060
4061    /* TB epilogue */
4062    tb_ret_addr = tcg_splitwx_to_rx(s->code_ptr);
4063
4064    tcg_out_addi(s, TCG_REG_CALL_STACK, stack_addend);
4065
4066    if (have_avx2) {
4067        tcg_out_vex_opc(s, OPC_VZEROUPPER, 0, 0, 0, 0);
4068    }
4069    for (i = ARRAY_SIZE(tcg_target_callee_save_regs) - 1; i >= 0; i--) {
4070        tcg_out_pop(s, tcg_target_callee_save_regs[i]);
4071    }
4072    tcg_out_opc(s, OPC_RET, 0, 0, 0);
4073}
4074
4075static void tcg_out_nop_fill(tcg_insn_unit *p, int count)
4076{
4077    memset(p, 0x90, count);
4078}
4079
4080static void tcg_target_init(TCGContext *s)
4081{
4082#ifdef CONFIG_CPUID_H
4083    unsigned a, b, c, d, b7 = 0, c7 = 0;
4084    unsigned max = __get_cpuid_max(0, 0);
4085
4086    if (max >= 7) {
4087        /* BMI1 is available on AMD Piledriver and Intel Haswell CPUs.  */
4088        __cpuid_count(7, 0, a, b7, c7, d);
4089        have_bmi1 = (b7 & bit_BMI) != 0;
4090        have_bmi2 = (b7 & bit_BMI2) != 0;
4091    }
4092
4093    if (max >= 1) {
4094        __cpuid(1, a, b, c, d);
4095#ifndef have_cmov
4096        /* For 32-bit, 99% certainty that we're running on hardware that
4097           supports cmov, but we still need to check.  In case cmov is not
4098           available, we'll use a small forward branch.  */
4099        have_cmov = (d & bit_CMOV) != 0;
4100#endif
4101
4102        /* MOVBE is only available on Intel Atom and Haswell CPUs, so we
4103           need to probe for it.  */
4104        have_movbe = (c & bit_MOVBE) != 0;
4105        have_popcnt = (c & bit_POPCNT) != 0;
4106
4107        /* There are a number of things we must check before we can be
4108           sure of not hitting invalid opcode.  */
4109        if (c & bit_OSXSAVE) {
4110            unsigned xcrl, xcrh;
4111            /* The xgetbv instruction is not available to older versions of
4112             * the assembler, so we encode the instruction manually.
4113             */
4114            asm(".byte 0x0f, 0x01, 0xd0" : "=a" (xcrl), "=d" (xcrh) : "c" (0));
4115            if ((xcrl & 6) == 6) {
4116                have_avx1 = (c & bit_AVX) != 0;
4117                have_avx2 = (b7 & bit_AVX2) != 0;
4118
4119                /*
4120                 * There are interesting instructions in AVX512, so long
4121                 * as we have AVX512VL, which indicates support for EVEX
4122                 * on sizes smaller than 512 bits.  We are required to
4123                 * check that OPMASK and all extended ZMM state are enabled
4124                 * even if we're not using them -- the insns will fault.
4125                 */
4126                if ((xcrl & 0xe0) == 0xe0
4127                    && (b7 & bit_AVX512F)
4128                    && (b7 & bit_AVX512VL)) {
4129                    have_avx512vl = true;
4130                    have_avx512bw = (b7 & bit_AVX512BW) != 0;
4131                    have_avx512dq = (b7 & bit_AVX512DQ) != 0;
4132                    have_avx512vbmi2 = (c7 & bit_AVX512VBMI2) != 0;
4133                }
4134            }
4135        }
4136    }
4137
4138    max = __get_cpuid_max(0x8000000, 0);
4139    if (max >= 1) {
4140        __cpuid(0x80000001, a, b, c, d);
4141        /* LZCNT was introduced with AMD Barcelona and Intel Haswell CPUs.  */
4142        have_lzcnt = (c & bit_LZCNT) != 0;
4143    }
4144#endif /* CONFIG_CPUID_H */
4145
4146    tcg_target_available_regs[TCG_TYPE_I32] = ALL_GENERAL_REGS;
4147    if (TCG_TARGET_REG_BITS == 64) {
4148        tcg_target_available_regs[TCG_TYPE_I64] = ALL_GENERAL_REGS;
4149    }
4150    if (have_avx1) {
4151        tcg_target_available_regs[TCG_TYPE_V64] = ALL_VECTOR_REGS;
4152        tcg_target_available_regs[TCG_TYPE_V128] = ALL_VECTOR_REGS;
4153    }
4154    if (have_avx2) {
4155        tcg_target_available_regs[TCG_TYPE_V256] = ALL_VECTOR_REGS;
4156    }
4157
4158    tcg_target_call_clobber_regs = ALL_VECTOR_REGS;
4159    tcg_regset_set_reg(tcg_target_call_clobber_regs, TCG_REG_EAX);
4160    tcg_regset_set_reg(tcg_target_call_clobber_regs, TCG_REG_EDX);
4161    tcg_regset_set_reg(tcg_target_call_clobber_regs, TCG_REG_ECX);
4162    if (TCG_TARGET_REG_BITS == 64) {
4163#if !defined(_WIN64)
4164        tcg_regset_set_reg(tcg_target_call_clobber_regs, TCG_REG_RDI);
4165        tcg_regset_set_reg(tcg_target_call_clobber_regs, TCG_REG_RSI);
4166#endif
4167        tcg_regset_set_reg(tcg_target_call_clobber_regs, TCG_REG_R8);
4168        tcg_regset_set_reg(tcg_target_call_clobber_regs, TCG_REG_R9);
4169        tcg_regset_set_reg(tcg_target_call_clobber_regs, TCG_REG_R10);
4170        tcg_regset_set_reg(tcg_target_call_clobber_regs, TCG_REG_R11);
4171    }
4172
4173    s->reserved_regs = 0;
4174    tcg_regset_set_reg(s->reserved_regs, TCG_REG_CALL_STACK);
4175}
4176
4177typedef struct {
4178    DebugFrameHeader h;
4179    uint8_t fde_def_cfa[4];
4180    uint8_t fde_reg_ofs[14];
4181} DebugFrame;
4182
4183/* We're expecting a 2 byte uleb128 encoded value.  */
4184QEMU_BUILD_BUG_ON(FRAME_SIZE >= (1 << 14));
4185
4186#if !defined(__ELF__)
4187    /* Host machine without ELF. */
4188#elif TCG_TARGET_REG_BITS == 64
4189#define ELF_HOST_MACHINE EM_X86_64
4190static const DebugFrame debug_frame = {
4191    .h.cie.len = sizeof(DebugFrameCIE)-4, /* length after .len member */
4192    .h.cie.id = -1,
4193    .h.cie.version = 1,
4194    .h.cie.code_align = 1,
4195    .h.cie.data_align = 0x78,             /* sleb128 -8 */
4196    .h.cie.return_column = 16,
4197
4198    /* Total FDE size does not include the "len" member.  */
4199    .h.fde.len = sizeof(DebugFrame) - offsetof(DebugFrame, h.fde.cie_offset),
4200
4201    .fde_def_cfa = {
4202        12, 7,                          /* DW_CFA_def_cfa %rsp, ... */
4203        (FRAME_SIZE & 0x7f) | 0x80,     /* ... uleb128 FRAME_SIZE */
4204        (FRAME_SIZE >> 7)
4205    },
4206    .fde_reg_ofs = {
4207        0x90, 1,                        /* DW_CFA_offset, %rip, -8 */
4208        /* The following ordering must match tcg_target_callee_save_regs.  */
4209        0x86, 2,                        /* DW_CFA_offset, %rbp, -16 */
4210        0x83, 3,                        /* DW_CFA_offset, %rbx, -24 */
4211        0x8c, 4,                        /* DW_CFA_offset, %r12, -32 */
4212        0x8d, 5,                        /* DW_CFA_offset, %r13, -40 */
4213        0x8e, 6,                        /* DW_CFA_offset, %r14, -48 */
4214        0x8f, 7,                        /* DW_CFA_offset, %r15, -56 */
4215    }
4216};
4217#else
4218#define ELF_HOST_MACHINE EM_386
4219static const DebugFrame debug_frame = {
4220    .h.cie.len = sizeof(DebugFrameCIE)-4, /* length after .len member */
4221    .h.cie.id = -1,
4222    .h.cie.version = 1,
4223    .h.cie.code_align = 1,
4224    .h.cie.data_align = 0x7c,             /* sleb128 -4 */
4225    .h.cie.return_column = 8,
4226
4227    /* Total FDE size does not include the "len" member.  */
4228    .h.fde.len = sizeof(DebugFrame) - offsetof(DebugFrame, h.fde.cie_offset),
4229
4230    .fde_def_cfa = {
4231        12, 4,                          /* DW_CFA_def_cfa %esp, ... */
4232        (FRAME_SIZE & 0x7f) | 0x80,     /* ... uleb128 FRAME_SIZE */
4233        (FRAME_SIZE >> 7)
4234    },
4235    .fde_reg_ofs = {
4236        0x88, 1,                        /* DW_CFA_offset, %eip, -4 */
4237        /* The following ordering must match tcg_target_callee_save_regs.  */
4238        0x85, 2,                        /* DW_CFA_offset, %ebp, -8 */
4239        0x83, 3,                        /* DW_CFA_offset, %ebx, -12 */
4240        0x86, 4,                        /* DW_CFA_offset, %esi, -16 */
4241        0x87, 5,                        /* DW_CFA_offset, %edi, -20 */
4242    }
4243};
4244#endif
4245
4246#if defined(ELF_HOST_MACHINE)
4247void tcg_register_jit(const void *buf, size_t buf_size)
4248{
4249    tcg_register_jit_int(buf, buf_size, &debug_frame, sizeof(debug_frame));
4250}
4251#endif
4252