1/*
2 *  MMX/3DNow!/SSE/SSE2/SSE3/SSSE3/SSE4/PNI support
3 *
4 *  Copyright (c) 2005 Fabrice Bellard
5 *
6 * This library is free software; you can redistribute it and/or
7 * modify it under the terms of the GNU Lesser General Public
8 * License as published by the Free Software Foundation; either
9 * version 2.1 of the License, or (at your option) any later version.
10 *
11 * This library is distributed in the hope that it will be useful,
12 * but WITHOUT ANY WARRANTY; without even the implied warranty of
13 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
14 * Lesser General Public License for more details.
15 *
16 * You should have received a copy of the GNU Lesser General Public
17 * License along with this library; if not, see <http://www.gnu.org/licenses/>.
18 */
19#if SHIFT == 0
20#define Reg MMXReg
21#define SUFFIX _mmx
22#else
23#define Reg ZMMReg
24#if SHIFT == 1
25#define SUFFIX _xmm
26#else
27#define SUFFIX _ymm
28#endif
29#endif
30
31#define dh_alias_Reg ptr
32#define dh_alias_ZMMReg ptr
33#define dh_alias_MMXReg ptr
34#define dh_ctype_Reg Reg *
35#define dh_ctype_ZMMReg ZMMReg *
36#define dh_ctype_MMXReg MMXReg *
37#define dh_typecode_Reg dh_typecode_ptr
38#define dh_typecode_ZMMReg dh_typecode_ptr
39#define dh_typecode_MMXReg dh_typecode_ptr
40
41DEF_HELPER_4(glue(psrlw, SUFFIX), void, env, Reg, Reg, Reg)
42DEF_HELPER_4(glue(psraw, SUFFIX), void, env, Reg, Reg, Reg)
43DEF_HELPER_4(glue(psllw, SUFFIX), void, env, Reg, Reg, Reg)
44DEF_HELPER_4(glue(psrld, SUFFIX), void, env, Reg, Reg, Reg)
45DEF_HELPER_4(glue(psrad, SUFFIX), void, env, Reg, Reg, Reg)
46DEF_HELPER_4(glue(pslld, SUFFIX), void, env, Reg, Reg, Reg)
47DEF_HELPER_4(glue(psrlq, SUFFIX), void, env, Reg, Reg, Reg)
48DEF_HELPER_4(glue(psllq, SUFFIX), void, env, Reg, Reg, Reg)
49
50#if SHIFT >= 1
51DEF_HELPER_4(glue(psrldq, SUFFIX), void, env, Reg, Reg, Reg)
52DEF_HELPER_4(glue(pslldq, SUFFIX), void, env, Reg, Reg, Reg)
53#endif
54
55#define SSE_HELPER_B(name, F)\
56    DEF_HELPER_4(glue(name, SUFFIX), void, env, Reg, Reg, Reg)
57
58#define SSE_HELPER_W(name, F)\
59    DEF_HELPER_4(glue(name, SUFFIX), void, env, Reg, Reg, Reg)
60
61#define SSE_HELPER_L(name, F)\
62    DEF_HELPER_4(glue(name, SUFFIX), void, env, Reg, Reg, Reg)
63
64#define SSE_HELPER_Q(name, F)\
65    DEF_HELPER_4(glue(name, SUFFIX), void, env, Reg, Reg, Reg)
66
67#if SHIFT == 0
68DEF_HELPER_3(glue(pmulhrw, SUFFIX), void, env, Reg, Reg)
69#endif
70SSE_HELPER_W(pmulhuw, FMULHUW)
71SSE_HELPER_W(pmulhw, FMULHW)
72
73SSE_HELPER_B(pavgb, FAVG)
74SSE_HELPER_W(pavgw, FAVG)
75
76DEF_HELPER_4(glue(pmuludq, SUFFIX), void, env, Reg, Reg, Reg)
77DEF_HELPER_4(glue(pmaddwd, SUFFIX), void, env, Reg, Reg, Reg)
78
79DEF_HELPER_4(glue(psadbw, SUFFIX), void, env, Reg, Reg, Reg)
80#if SHIFT < 2
81DEF_HELPER_4(glue(maskmov, SUFFIX), void, env, Reg, Reg, tl)
82#endif
83
84#if SHIFT == 0
85DEF_HELPER_3(glue(pshufw, SUFFIX), void, Reg, Reg, int)
86#else
87DEF_HELPER_3(glue(pshufd, SUFFIX), void, Reg, Reg, int)
88DEF_HELPER_3(glue(pshuflw, SUFFIX), void, Reg, Reg, int)
89DEF_HELPER_3(glue(pshufhw, SUFFIX), void, Reg, Reg, int)
90#endif
91
92#if SHIFT >= 1
93/* FPU ops */
94/* XXX: not accurate */
95
96#define SSE_HELPER_P4(name)                                             \
97    DEF_HELPER_4(glue(name ## ps, SUFFIX), void, env, Reg, Reg, Reg)    \
98    DEF_HELPER_4(glue(name ## pd, SUFFIX), void, env, Reg, Reg, Reg)
99
100#define SSE_HELPER_P3(name, ...)                                        \
101    DEF_HELPER_3(glue(name ## ps, SUFFIX), void, env, Reg, Reg)         \
102    DEF_HELPER_3(glue(name ## pd, SUFFIX), void, env, Reg, Reg)
103
104#if SHIFT == 1
105#define SSE_HELPER_S4(name)                                             \
106    SSE_HELPER_P4(name)                                                 \
107    DEF_HELPER_4(name ## ss, void, env, Reg, Reg, Reg)                  \
108    DEF_HELPER_4(name ## sd, void, env, Reg, Reg, Reg)
109#define SSE_HELPER_S3(name)                                             \
110    SSE_HELPER_P3(name)                                                 \
111    DEF_HELPER_4(name ## ss, void, env, Reg, Reg, Reg)                  \
112    DEF_HELPER_4(name ## sd, void, env, Reg, Reg, Reg)
113#else
114#define SSE_HELPER_S4(name, ...) SSE_HELPER_P4(name)
115#define SSE_HELPER_S3(name, ...) SSE_HELPER_P3(name)
116#endif
117
118DEF_HELPER_4(glue(shufps, SUFFIX), void, Reg, Reg, Reg, int)
119DEF_HELPER_4(glue(shufpd, SUFFIX), void, Reg, Reg, Reg, int)
120
121SSE_HELPER_S4(add)
122SSE_HELPER_S4(sub)
123SSE_HELPER_S4(mul)
124SSE_HELPER_S4(div)
125SSE_HELPER_S4(min)
126SSE_HELPER_S4(max)
127
128SSE_HELPER_S3(sqrt)
129
130DEF_HELPER_3(glue(cvtps2pd, SUFFIX), void, env, Reg, Reg)
131DEF_HELPER_3(glue(cvtpd2ps, SUFFIX), void, env, Reg, Reg)
132DEF_HELPER_3(glue(cvtdq2ps, SUFFIX), void, env, Reg, Reg)
133DEF_HELPER_3(glue(cvtdq2pd, SUFFIX), void, env, Reg, Reg)
134
135DEF_HELPER_3(glue(cvtps2dq, SUFFIX), void, env, ZMMReg, ZMMReg)
136DEF_HELPER_3(glue(cvtpd2dq, SUFFIX), void, env, ZMMReg, ZMMReg)
137
138DEF_HELPER_3(glue(cvttps2dq, SUFFIX), void, env, ZMMReg, ZMMReg)
139DEF_HELPER_3(glue(cvttpd2dq, SUFFIX), void, env, ZMMReg, ZMMReg)
140
141#if SHIFT == 1
142DEF_HELPER_4(cvtss2sd, void, env, Reg, Reg, Reg)
143DEF_HELPER_4(cvtsd2ss, void, env, Reg, Reg, Reg)
144DEF_HELPER_3(cvtpi2ps, void, env, ZMMReg, MMXReg)
145DEF_HELPER_3(cvtpi2pd, void, env, ZMMReg, MMXReg)
146DEF_HELPER_3(cvtsi2ss, void, env, ZMMReg, i32)
147DEF_HELPER_3(cvtsi2sd, void, env, ZMMReg, i32)
148
149#ifdef TARGET_X86_64
150DEF_HELPER_3(cvtsq2ss, void, env, ZMMReg, i64)
151DEF_HELPER_3(cvtsq2sd, void, env, ZMMReg, i64)
152#endif
153
154DEF_HELPER_3(cvtps2pi, void, env, MMXReg, ZMMReg)
155DEF_HELPER_3(cvtpd2pi, void, env, MMXReg, ZMMReg)
156DEF_HELPER_2(cvtss2si, s32, env, ZMMReg)
157DEF_HELPER_2(cvtsd2si, s32, env, ZMMReg)
158#ifdef TARGET_X86_64
159DEF_HELPER_2(cvtss2sq, s64, env, ZMMReg)
160DEF_HELPER_2(cvtsd2sq, s64, env, ZMMReg)
161#endif
162
163DEF_HELPER_3(cvttps2pi, void, env, MMXReg, ZMMReg)
164DEF_HELPER_3(cvttpd2pi, void, env, MMXReg, ZMMReg)
165DEF_HELPER_2(cvttss2si, s32, env, ZMMReg)
166DEF_HELPER_2(cvttsd2si, s32, env, ZMMReg)
167#ifdef TARGET_X86_64
168DEF_HELPER_2(cvttss2sq, s64, env, ZMMReg)
169DEF_HELPER_2(cvttsd2sq, s64, env, ZMMReg)
170#endif
171#endif
172
173DEF_HELPER_3(glue(rsqrtps, SUFFIX), void, env, ZMMReg, ZMMReg)
174DEF_HELPER_3(glue(rcpps, SUFFIX), void, env, ZMMReg, ZMMReg)
175
176#if SHIFT == 1
177DEF_HELPER_4(rsqrtss, void, env, ZMMReg, ZMMReg, ZMMReg)
178DEF_HELPER_4(rcpss, void, env, ZMMReg, ZMMReg, ZMMReg)
179DEF_HELPER_3(extrq_r, void, env, ZMMReg, ZMMReg)
180DEF_HELPER_4(extrq_i, void, env, ZMMReg, int, int)
181DEF_HELPER_3(insertq_r, void, env, ZMMReg, ZMMReg)
182DEF_HELPER_5(insertq_i, void, env, ZMMReg, ZMMReg, int, int)
183#endif
184
185SSE_HELPER_P4(hadd)
186SSE_HELPER_P4(hsub)
187SSE_HELPER_P4(addsub)
188
189#define SSE_HELPER_CMP(name, F, C) SSE_HELPER_S4(name)
190
191SSE_HELPER_CMP(cmpeq, FPU_CMPQ, FPU_EQ)
192SSE_HELPER_CMP(cmplt, FPU_CMPS, FPU_LT)
193SSE_HELPER_CMP(cmple, FPU_CMPS, FPU_LE)
194SSE_HELPER_CMP(cmpunord, FPU_CMPQ,  FPU_UNORD)
195SSE_HELPER_CMP(cmpneq, FPU_CMPQ, !FPU_EQ)
196SSE_HELPER_CMP(cmpnlt, FPU_CMPS, !FPU_LT)
197SSE_HELPER_CMP(cmpnle, FPU_CMPS, !FPU_LE)
198SSE_HELPER_CMP(cmpord, FPU_CMPQ, !FPU_UNORD)
199
200SSE_HELPER_CMP(cmpequ, FPU_CMPQ, FPU_EQU)
201SSE_HELPER_CMP(cmpnge, FPU_CMPS, !FPU_GE)
202SSE_HELPER_CMP(cmpngt, FPU_CMPS, !FPU_GT)
203SSE_HELPER_CMP(cmpfalse, FPU_CMPQ,  FPU_FALSE)
204SSE_HELPER_CMP(cmpnequ, FPU_CMPQ, !FPU_EQU)
205SSE_HELPER_CMP(cmpge, FPU_CMPS, FPU_GE)
206SSE_HELPER_CMP(cmpgt, FPU_CMPS, FPU_GT)
207SSE_HELPER_CMP(cmptrue, FPU_CMPQ,  !FPU_FALSE)
208
209SSE_HELPER_CMP(cmpeqs, FPU_CMPS, FPU_EQ)
210SSE_HELPER_CMP(cmpltq, FPU_CMPQ, FPU_LT)
211SSE_HELPER_CMP(cmpleq, FPU_CMPQ, FPU_LE)
212SSE_HELPER_CMP(cmpunords, FPU_CMPS,  FPU_UNORD)
213SSE_HELPER_CMP(cmpneqq, FPU_CMPS, !FPU_EQ)
214SSE_HELPER_CMP(cmpnltq, FPU_CMPQ, !FPU_LT)
215SSE_HELPER_CMP(cmpnleq, FPU_CMPQ, !FPU_LE)
216SSE_HELPER_CMP(cmpords, FPU_CMPS, !FPU_UNORD)
217
218SSE_HELPER_CMP(cmpequs, FPU_CMPS, FPU_EQU)
219SSE_HELPER_CMP(cmpngeq, FPU_CMPQ, !FPU_GE)
220SSE_HELPER_CMP(cmpngtq, FPU_CMPQ, !FPU_GT)
221SSE_HELPER_CMP(cmpfalses, FPU_CMPS,  FPU_FALSE)
222SSE_HELPER_CMP(cmpnequs, FPU_CMPS, !FPU_EQU)
223SSE_HELPER_CMP(cmpgeq, FPU_CMPQ, FPU_GE)
224SSE_HELPER_CMP(cmpgtq, FPU_CMPQ, FPU_GT)
225SSE_HELPER_CMP(cmptrues, FPU_CMPS,  !FPU_FALSE)
226
227#if SHIFT == 1
228DEF_HELPER_3(ucomiss, void, env, Reg, Reg)
229DEF_HELPER_3(comiss, void, env, Reg, Reg)
230DEF_HELPER_3(ucomisd, void, env, Reg, Reg)
231DEF_HELPER_3(comisd, void, env, Reg, Reg)
232#endif
233
234DEF_HELPER_2(glue(movmskps, SUFFIX), i32, env, Reg)
235DEF_HELPER_2(glue(movmskpd, SUFFIX), i32, env, Reg)
236#endif
237
238DEF_HELPER_4(glue(packsswb, SUFFIX), void, env, Reg, Reg, Reg)
239DEF_HELPER_4(glue(packuswb, SUFFIX), void, env, Reg, Reg, Reg)
240DEF_HELPER_4(glue(packssdw, SUFFIX), void, env, Reg, Reg, Reg)
241#define UNPCK_OP(name, base)                                       \
242    DEF_HELPER_4(glue(punpck ## name ## bw, SUFFIX), void, env, Reg, Reg, Reg) \
243    DEF_HELPER_4(glue(punpck ## name ## wd, SUFFIX), void, env, Reg, Reg, Reg) \
244    DEF_HELPER_4(glue(punpck ## name ## dq, SUFFIX), void, env, Reg, Reg, Reg)
245
246UNPCK_OP(l, 0)
247UNPCK_OP(h, 1)
248
249#if SHIFT >= 1
250DEF_HELPER_4(glue(punpcklqdq, SUFFIX), void, env, Reg, Reg, Reg)
251DEF_HELPER_4(glue(punpckhqdq, SUFFIX), void, env, Reg, Reg, Reg)
252#endif
253
254/* 3DNow! float ops */
255#if SHIFT == 0
256DEF_HELPER_3(pi2fd, void, env, MMXReg, MMXReg)
257DEF_HELPER_3(pi2fw, void, env, MMXReg, MMXReg)
258DEF_HELPER_3(pf2id, void, env, MMXReg, MMXReg)
259DEF_HELPER_3(pf2iw, void, env, MMXReg, MMXReg)
260DEF_HELPER_3(pfacc, void, env, MMXReg, MMXReg)
261DEF_HELPER_3(pfadd, void, env, MMXReg, MMXReg)
262DEF_HELPER_3(pfcmpeq, void, env, MMXReg, MMXReg)
263DEF_HELPER_3(pfcmpge, void, env, MMXReg, MMXReg)
264DEF_HELPER_3(pfcmpgt, void, env, MMXReg, MMXReg)
265DEF_HELPER_3(pfmax, void, env, MMXReg, MMXReg)
266DEF_HELPER_3(pfmin, void, env, MMXReg, MMXReg)
267DEF_HELPER_3(pfmul, void, env, MMXReg, MMXReg)
268DEF_HELPER_3(pfnacc, void, env, MMXReg, MMXReg)
269DEF_HELPER_3(pfpnacc, void, env, MMXReg, MMXReg)
270DEF_HELPER_3(pfrcp, void, env, MMXReg, MMXReg)
271DEF_HELPER_3(pfrsqrt, void, env, MMXReg, MMXReg)
272DEF_HELPER_3(pfsub, void, env, MMXReg, MMXReg)
273DEF_HELPER_3(pfsubr, void, env, MMXReg, MMXReg)
274DEF_HELPER_3(pswapd, void, env, MMXReg, MMXReg)
275#endif
276
277/* SSSE3 op helpers */
278DEF_HELPER_4(glue(phaddw, SUFFIX), void, env, Reg, Reg, Reg)
279DEF_HELPER_4(glue(phaddd, SUFFIX), void, env, Reg, Reg, Reg)
280DEF_HELPER_4(glue(phaddsw, SUFFIX), void, env, Reg, Reg, Reg)
281DEF_HELPER_4(glue(phsubw, SUFFIX), void, env, Reg, Reg, Reg)
282DEF_HELPER_4(glue(phsubd, SUFFIX), void, env, Reg, Reg, Reg)
283DEF_HELPER_4(glue(phsubsw, SUFFIX), void, env, Reg, Reg, Reg)
284DEF_HELPER_4(glue(pmaddubsw, SUFFIX), void, env, Reg, Reg, Reg)
285DEF_HELPER_4(glue(pmulhrsw, SUFFIX), void, env, Reg, Reg, Reg)
286DEF_HELPER_4(glue(pshufb, SUFFIX), void, env, Reg, Reg, Reg)
287DEF_HELPER_4(glue(psignb, SUFFIX), void, env, Reg, Reg, Reg)
288DEF_HELPER_4(glue(psignw, SUFFIX), void, env, Reg, Reg, Reg)
289DEF_HELPER_4(glue(psignd, SUFFIX), void, env, Reg, Reg, Reg)
290DEF_HELPER_5(glue(palignr, SUFFIX), void, env, Reg, Reg, Reg, i32)
291
292/* SSE4.1 op helpers */
293#if SHIFT >= 1
294DEF_HELPER_5(glue(pblendvb, SUFFIX), void, env, Reg, Reg, Reg, Reg)
295DEF_HELPER_5(glue(blendvps, SUFFIX), void, env, Reg, Reg, Reg, Reg)
296DEF_HELPER_5(glue(blendvpd, SUFFIX), void, env, Reg, Reg, Reg, Reg)
297DEF_HELPER_3(glue(ptest, SUFFIX), void, env, Reg, Reg)
298DEF_HELPER_3(glue(pmovsxbw, SUFFIX), void, env, Reg, Reg)
299DEF_HELPER_3(glue(pmovsxbd, SUFFIX), void, env, Reg, Reg)
300DEF_HELPER_3(glue(pmovsxbq, SUFFIX), void, env, Reg, Reg)
301DEF_HELPER_3(glue(pmovsxwd, SUFFIX), void, env, Reg, Reg)
302DEF_HELPER_3(glue(pmovsxwq, SUFFIX), void, env, Reg, Reg)
303DEF_HELPER_3(glue(pmovsxdq, SUFFIX), void, env, Reg, Reg)
304DEF_HELPER_3(glue(pmovzxbw, SUFFIX), void, env, Reg, Reg)
305DEF_HELPER_3(glue(pmovzxbd, SUFFIX), void, env, Reg, Reg)
306DEF_HELPER_3(glue(pmovzxbq, SUFFIX), void, env, Reg, Reg)
307DEF_HELPER_3(glue(pmovzxwd, SUFFIX), void, env, Reg, Reg)
308DEF_HELPER_3(glue(pmovzxwq, SUFFIX), void, env, Reg, Reg)
309DEF_HELPER_3(glue(pmovzxdq, SUFFIX), void, env, Reg, Reg)
310DEF_HELPER_3(glue(pmovsldup, SUFFIX), void, env, Reg, Reg)
311DEF_HELPER_3(glue(pmovshdup, SUFFIX), void, env, Reg, Reg)
312DEF_HELPER_3(glue(pmovdldup, SUFFIX), void, env, Reg, Reg)
313DEF_HELPER_4(glue(pmuldq, SUFFIX), void, env, Reg, Reg, Reg)
314DEF_HELPER_4(glue(packusdw, SUFFIX), void, env, Reg, Reg, Reg)
315#if SHIFT == 1
316DEF_HELPER_3(glue(phminposuw, SUFFIX), void, env, Reg, Reg)
317#endif
318DEF_HELPER_4(glue(roundps, SUFFIX), void, env, Reg, Reg, i32)
319DEF_HELPER_4(glue(roundpd, SUFFIX), void, env, Reg, Reg, i32)
320#if SHIFT == 1
321DEF_HELPER_5(roundss_xmm, void, env, Reg, Reg, Reg, i32)
322DEF_HELPER_5(roundsd_xmm, void, env, Reg, Reg, Reg, i32)
323#endif
324DEF_HELPER_5(glue(blendps, SUFFIX), void, env, Reg, Reg, Reg, i32)
325DEF_HELPER_5(glue(blendpd, SUFFIX), void, env, Reg, Reg, Reg, i32)
326DEF_HELPER_5(glue(pblendw, SUFFIX), void, env, Reg, Reg, Reg, i32)
327DEF_HELPER_5(glue(dpps, SUFFIX), void, env, Reg, Reg, Reg, i32)
328#if SHIFT == 1
329DEF_HELPER_5(glue(dppd, SUFFIX), void, env, Reg, Reg, Reg, i32)
330#endif
331DEF_HELPER_5(glue(mpsadbw, SUFFIX), void, env, Reg, Reg, Reg, i32)
332#endif
333
334/* SSE4.2 op helpers */
335#if SHIFT == 1
336DEF_HELPER_4(glue(pcmpestri, SUFFIX), void, env, Reg, Reg, i32)
337DEF_HELPER_4(glue(pcmpestrm, SUFFIX), void, env, Reg, Reg, i32)
338DEF_HELPER_4(glue(pcmpistri, SUFFIX), void, env, Reg, Reg, i32)
339DEF_HELPER_4(glue(pcmpistrm, SUFFIX), void, env, Reg, Reg, i32)
340DEF_HELPER_3(crc32, tl, i32, tl, i32)
341#endif
342
343/* AES-NI op helpers */
344#if SHIFT >= 1
345DEF_HELPER_4(glue(aesdec, SUFFIX), void, env, Reg, Reg, Reg)
346DEF_HELPER_4(glue(aesdeclast, SUFFIX), void, env, Reg, Reg, Reg)
347DEF_HELPER_4(glue(aesenc, SUFFIX), void, env, Reg, Reg, Reg)
348DEF_HELPER_4(glue(aesenclast, SUFFIX), void, env, Reg, Reg, Reg)
349#if SHIFT == 1
350DEF_HELPER_3(glue(aesimc, SUFFIX), void, env, Reg, Reg)
351DEF_HELPER_4(glue(aeskeygenassist, SUFFIX), void, env, Reg, Reg, i32)
352#endif
353DEF_HELPER_5(glue(pclmulqdq, SUFFIX), void, env, Reg, Reg, Reg, i32)
354#endif
355
356/* F16C helpers */
357#if SHIFT >= 1
358DEF_HELPER_3(glue(cvtph2ps, SUFFIX), void, env, Reg, Reg)
359DEF_HELPER_4(glue(cvtps2ph, SUFFIX), void, env, Reg, Reg, int)
360#endif
361
362/* FMA3 helpers */
363#if SHIFT == 1
364DEF_HELPER_6(fma4ss, void, env, Reg, Reg, Reg, Reg, int)
365DEF_HELPER_6(fma4sd, void, env, Reg, Reg, Reg, Reg, int)
366#endif
367
368#if SHIFT >= 1
369DEF_HELPER_7(glue(fma4ps, SUFFIX), void, env, Reg, Reg, Reg, Reg, int, int)
370DEF_HELPER_7(glue(fma4pd, SUFFIX), void, env, Reg, Reg, Reg, Reg, int, int)
371#endif
372
373/* AVX helpers */
374#if SHIFT >= 1
375DEF_HELPER_4(glue(vpermilpd, SUFFIX), void, env, Reg, Reg, Reg)
376DEF_HELPER_4(glue(vpermilps, SUFFIX), void, env, Reg, Reg, Reg)
377DEF_HELPER_3(glue(vpermilpd_imm, SUFFIX), void, Reg, Reg, i32)
378DEF_HELPER_3(glue(vpermilps_imm, SUFFIX), void, Reg, Reg, i32)
379DEF_HELPER_4(glue(vpsrlvd, SUFFIX), void, env, Reg, Reg, Reg)
380DEF_HELPER_4(glue(vpsravd, SUFFIX), void, env, Reg, Reg, Reg)
381DEF_HELPER_4(glue(vpsllvd, SUFFIX), void, env, Reg, Reg, Reg)
382DEF_HELPER_4(glue(vpsrlvq, SUFFIX), void, env, Reg, Reg, Reg)
383DEF_HELPER_4(glue(vpsravq, SUFFIX), void, env, Reg, Reg, Reg)
384DEF_HELPER_4(glue(vpsllvq, SUFFIX), void, env, Reg, Reg, Reg)
385DEF_HELPER_3(glue(vtestps, SUFFIX), void, env, Reg, Reg)
386DEF_HELPER_3(glue(vtestpd, SUFFIX), void, env, Reg, Reg)
387DEF_HELPER_4(glue(vpmaskmovd_st, SUFFIX), void, env, Reg, Reg, tl)
388DEF_HELPER_4(glue(vpmaskmovq_st, SUFFIX), void, env, Reg, Reg, tl)
389DEF_HELPER_4(glue(vpmaskmovd, SUFFIX), void, env, Reg, Reg, Reg)
390DEF_HELPER_4(glue(vpmaskmovq, SUFFIX), void, env, Reg, Reg, Reg)
391DEF_HELPER_6(glue(vpgatherdd, SUFFIX), void, env, Reg, Reg, Reg, tl, i32)
392DEF_HELPER_6(glue(vpgatherdq, SUFFIX), void, env, Reg, Reg, Reg, tl, i32)
393DEF_HELPER_6(glue(vpgatherqd, SUFFIX), void, env, Reg, Reg, Reg, tl, i32)
394DEF_HELPER_6(glue(vpgatherqq, SUFFIX), void, env, Reg, Reg, Reg, tl, i32)
395#if SHIFT == 2
396DEF_HELPER_3(vpermd_ymm, void, Reg, Reg, Reg)
397DEF_HELPER_4(vpermdq_ymm, void, Reg, Reg, Reg, i32)
398DEF_HELPER_3(vpermq_ymm, void, Reg, Reg, i32)
399#endif
400#endif
401
402/* SHA helpers */
403#if SHIFT == 1
404DEF_HELPER_3(sha1rnds4_f0, void, Reg, Reg, Reg)
405DEF_HELPER_3(sha1rnds4_f1, void, Reg, Reg, Reg)
406DEF_HELPER_3(sha1rnds4_f2, void, Reg, Reg, Reg)
407DEF_HELPER_3(sha1rnds4_f3, void, Reg, Reg, Reg)
408DEF_HELPER_3(sha1nexte, void, Reg, Reg, Reg)
409DEF_HELPER_3(sha1msg1, void, Reg, Reg, Reg)
410DEF_HELPER_3(sha1msg2, void, Reg, Reg, Reg)
411DEF_HELPER_5(sha256rnds2, void, Reg, Reg, Reg, i32, i32)
412DEF_HELPER_3(sha256msg1, void, Reg, Reg, Reg)
413DEF_HELPER_3(sha256msg2, void, Reg, Reg, Reg)
414#endif
415
416#undef SHIFT
417#undef Reg
418#undef SUFFIX
419
420#undef SSE_HELPER_B
421#undef SSE_HELPER_W
422#undef SSE_HELPER_L
423#undef SSE_HELPER_Q
424#undef SSE_HELPER_S3
425#undef SSE_HELPER_S4
426#undef SSE_HELPER_P3
427#undef SSE_HELPER_P4
428#undef SSE_HELPER_CMP
429#undef UNPCK_OP
430