1/* SPDX-License-Identifier: GPL-2.0-or-later */
2/*
3 * ARIA Cipher 32-way parallel algorithm (AVX2)
4 *
5 * Copyright (c) 2022 Taehee Yoo <ap420073@gmail.com>
6 *
7 */
8
9#include <linux/linkage.h>
10#include <asm/frame.h>
11#include <asm/asm-offsets.h>
12#include <linux/cfi_types.h>
13
14/* register macros */
15#define CTX %rdi
16
17#define ymm0_x xmm0
18#define ymm1_x xmm1
19#define ymm2_x xmm2
20#define ymm3_x xmm3
21#define ymm4_x xmm4
22#define ymm5_x xmm5
23#define ymm6_x xmm6
24#define ymm7_x xmm7
25#define ymm8_x xmm8
26#define ymm9_x xmm9
27#define ymm10_x xmm10
28#define ymm11_x xmm11
29#define ymm12_x xmm12
30#define ymm13_x xmm13
31#define ymm14_x xmm14
32#define ymm15_x xmm15
33
34#define BV8(a0, a1, a2, a3, a4, a5, a6, a7)		\
35	( (((a0) & 1) << 0) |				\
36	  (((a1) & 1) << 1) |				\
37	  (((a2) & 1) << 2) |				\
38	  (((a3) & 1) << 3) |				\
39	  (((a4) & 1) << 4) |				\
40	  (((a5) & 1) << 5) |				\
41	  (((a6) & 1) << 6) |				\
42	  (((a7) & 1) << 7) )
43
44#define BM8X8(l0, l1, l2, l3, l4, l5, l6, l7)		\
45	( ((l7) << (0 * 8)) |				\
46	  ((l6) << (1 * 8)) |				\
47	  ((l5) << (2 * 8)) |				\
48	  ((l4) << (3 * 8)) |				\
49	  ((l3) << (4 * 8)) |				\
50	  ((l2) << (5 * 8)) |				\
51	  ((l1) << (6 * 8)) |				\
52	  ((l0) << (7 * 8)) )
53
54#define inc_le128(x, minus_one, tmp)			\
55	vpcmpeqq minus_one, x, tmp;			\
56	vpsubq minus_one, x, x;				\
57	vpslldq $8, tmp, tmp;				\
58	vpsubq tmp, x, x;
59
60#define filter_8bit(x, lo_t, hi_t, mask4bit, tmp0)	\
61	vpand x, mask4bit, tmp0;			\
62	vpandn x, mask4bit, x;				\
63	vpsrld $4, x, x;				\
64							\
65	vpshufb tmp0, lo_t, tmp0;			\
66	vpshufb x, hi_t, x;				\
67	vpxor tmp0, x, x;
68
69#define transpose_4x4(x0, x1, x2, x3, t1, t2)		\
70	vpunpckhdq x1, x0, t2;				\
71	vpunpckldq x1, x0, x0;				\
72							\
73	vpunpckldq x3, x2, t1;				\
74	vpunpckhdq x3, x2, x2;				\
75							\
76	vpunpckhqdq t1, x0, x1;				\
77	vpunpcklqdq t1, x0, x0;				\
78							\
79	vpunpckhqdq x2, t2, x3;				\
80	vpunpcklqdq x2, t2, x2;
81
82#define byteslice_16x16b(a0, b0, c0, d0,		\
83			 a1, b1, c1, d1,		\
84			 a2, b2, c2, d2,		\
85			 a3, b3, c3, d3,		\
86			 st0, st1)			\
87	vmovdqu d2, st0;				\
88	vmovdqu d3, st1;				\
89	transpose_4x4(a0, a1, a2, a3, d2, d3);		\
90	transpose_4x4(b0, b1, b2, b3, d2, d3);		\
91	vmovdqu st0, d2;				\
92	vmovdqu st1, d3;				\
93							\
94	vmovdqu a0, st0;				\
95	vmovdqu a1, st1;				\
96	transpose_4x4(c0, c1, c2, c3, a0, a1);		\
97	transpose_4x4(d0, d1, d2, d3, a0, a1);		\
98							\
99	vbroadcasti128 .Lshufb_16x16b, a0;		\
100	vmovdqu st1, a1;				\
101	vpshufb a0, a2, a2;				\
102	vpshufb a0, a3, a3;				\
103	vpshufb a0, b0, b0;				\
104	vpshufb a0, b1, b1;				\
105	vpshufb a0, b2, b2;				\
106	vpshufb a0, b3, b3;				\
107	vpshufb a0, a1, a1;				\
108	vpshufb a0, c0, c0;				\
109	vpshufb a0, c1, c1;				\
110	vpshufb a0, c2, c2;				\
111	vpshufb a0, c3, c3;				\
112	vpshufb a0, d0, d0;				\
113	vpshufb a0, d1, d1;				\
114	vpshufb a0, d2, d2;				\
115	vpshufb a0, d3, d3;				\
116	vmovdqu d3, st1;				\
117	vmovdqu st0, d3;				\
118	vpshufb a0, d3, a0;				\
119	vmovdqu d2, st0;				\
120							\
121	transpose_4x4(a0, b0, c0, d0, d2, d3);		\
122	transpose_4x4(a1, b1, c1, d1, d2, d3);		\
123	vmovdqu st0, d2;				\
124	vmovdqu st1, d3;				\
125							\
126	vmovdqu b0, st0;				\
127	vmovdqu b1, st1;				\
128	transpose_4x4(a2, b2, c2, d2, b0, b1);		\
129	transpose_4x4(a3, b3, c3, d3, b0, b1);		\
130	vmovdqu st0, b0;				\
131	vmovdqu st1, b1;				\
132	/* does not adjust output bytes inside vectors */
133
134#define debyteslice_16x16b(a0, b0, c0, d0,		\
135			   a1, b1, c1, d1,		\
136			   a2, b2, c2, d2,		\
137			   a3, b3, c3, d3,		\
138			   st0, st1)			\
139	vmovdqu d2, st0;				\
140	vmovdqu d3, st1;				\
141	transpose_4x4(a0, a1, a2, a3, d2, d3);		\
142	transpose_4x4(b0, b1, b2, b3, d2, d3);		\
143	vmovdqu st0, d2;				\
144	vmovdqu st1, d3;				\
145							\
146	vmovdqu a0, st0;				\
147	vmovdqu a1, st1;				\
148	transpose_4x4(c0, c1, c2, c3, a0, a1);		\
149	transpose_4x4(d0, d1, d2, d3, a0, a1);		\
150							\
151	vbroadcasti128 .Lshufb_16x16b, a0;		\
152	vmovdqu st1, a1;				\
153	vpshufb a0, a2, a2;				\
154	vpshufb a0, a3, a3;				\
155	vpshufb a0, b0, b0;				\
156	vpshufb a0, b1, b1;				\
157	vpshufb a0, b2, b2;				\
158	vpshufb a0, b3, b3;				\
159	vpshufb a0, a1, a1;				\
160	vpshufb a0, c0, c0;				\
161	vpshufb a0, c1, c1;				\
162	vpshufb a0, c2, c2;				\
163	vpshufb a0, c3, c3;				\
164	vpshufb a0, d0, d0;				\
165	vpshufb a0, d1, d1;				\
166	vpshufb a0, d2, d2;				\
167	vpshufb a0, d3, d3;				\
168	vmovdqu d3, st1;				\
169	vmovdqu st0, d3;				\
170	vpshufb a0, d3, a0;				\
171	vmovdqu d2, st0;				\
172							\
173	transpose_4x4(c0, d0, a0, b0, d2, d3);		\
174	transpose_4x4(c1, d1, a1, b1, d2, d3);		\
175	vmovdqu st0, d2;				\
176	vmovdqu st1, d3;				\
177							\
178	vmovdqu b0, st0;				\
179	vmovdqu b1, st1;				\
180	transpose_4x4(c2, d2, a2, b2, b0, b1);		\
181	transpose_4x4(c3, d3, a3, b3, b0, b1);		\
182	vmovdqu st0, b0;				\
183	vmovdqu st1, b1;				\
184	/* does not adjust output bytes inside vectors */
185
186/* load blocks to registers and apply pre-whitening */
187#define inpack16_pre(x0, x1, x2, x3,			\
188		     x4, x5, x6, x7,			\
189		     y0, y1, y2, y3,			\
190		     y4, y5, y6, y7,			\
191		     rio)				\
192	vmovdqu (0 * 32)(rio), x0;			\
193	vmovdqu (1 * 32)(rio), x1;			\
194	vmovdqu (2 * 32)(rio), x2;			\
195	vmovdqu (3 * 32)(rio), x3;			\
196	vmovdqu (4 * 32)(rio), x4;			\
197	vmovdqu (5 * 32)(rio), x5;			\
198	vmovdqu (6 * 32)(rio), x6;			\
199	vmovdqu (7 * 32)(rio), x7;			\
200	vmovdqu (8 * 32)(rio), y0;			\
201	vmovdqu (9 * 32)(rio), y1;			\
202	vmovdqu (10 * 32)(rio), y2;			\
203	vmovdqu (11 * 32)(rio), y3;			\
204	vmovdqu (12 * 32)(rio), y4;			\
205	vmovdqu (13 * 32)(rio), y5;			\
206	vmovdqu (14 * 32)(rio), y6;			\
207	vmovdqu (15 * 32)(rio), y7;
208
209/* byteslice pre-whitened blocks and store to temporary memory */
210#define inpack16_post(x0, x1, x2, x3,			\
211		      x4, x5, x6, x7,			\
212		      y0, y1, y2, y3,			\
213		      y4, y5, y6, y7,			\
214		      mem_ab, mem_cd)			\
215	byteslice_16x16b(x0, x1, x2, x3,		\
216			 x4, x5, x6, x7,		\
217			 y0, y1, y2, y3,		\
218			 y4, y5, y6, y7,		\
219			 (mem_ab), (mem_cd));		\
220							\
221	vmovdqu x0, 0 * 32(mem_ab);			\
222	vmovdqu x1, 1 * 32(mem_ab);			\
223	vmovdqu x2, 2 * 32(mem_ab);			\
224	vmovdqu x3, 3 * 32(mem_ab);			\
225	vmovdqu x4, 4 * 32(mem_ab);			\
226	vmovdqu x5, 5 * 32(mem_ab);			\
227	vmovdqu x6, 6 * 32(mem_ab);			\
228	vmovdqu x7, 7 * 32(mem_ab);			\
229	vmovdqu y0, 0 * 32(mem_cd);			\
230	vmovdqu y1, 1 * 32(mem_cd);			\
231	vmovdqu y2, 2 * 32(mem_cd);			\
232	vmovdqu y3, 3 * 32(mem_cd);			\
233	vmovdqu y4, 4 * 32(mem_cd);			\
234	vmovdqu y5, 5 * 32(mem_cd);			\
235	vmovdqu y6, 6 * 32(mem_cd);			\
236	vmovdqu y7, 7 * 32(mem_cd);
237
238#define write_output(x0, x1, x2, x3,			\
239		     x4, x5, x6, x7,			\
240		     y0, y1, y2, y3,			\
241		     y4, y5, y6, y7,			\
242		     mem)				\
243	vmovdqu x0, 0 * 32(mem);			\
244	vmovdqu x1, 1 * 32(mem);			\
245	vmovdqu x2, 2 * 32(mem);			\
246	vmovdqu x3, 3 * 32(mem);			\
247	vmovdqu x4, 4 * 32(mem);			\
248	vmovdqu x5, 5 * 32(mem);			\
249	vmovdqu x6, 6 * 32(mem);			\
250	vmovdqu x7, 7 * 32(mem);			\
251	vmovdqu y0, 8 * 32(mem);			\
252	vmovdqu y1, 9 * 32(mem);			\
253	vmovdqu y2, 10 * 32(mem);			\
254	vmovdqu y3, 11 * 32(mem);			\
255	vmovdqu y4, 12 * 32(mem);			\
256	vmovdqu y5, 13 * 32(mem);			\
257	vmovdqu y6, 14 * 32(mem);			\
258	vmovdqu y7, 15 * 32(mem);			\
259
260#define aria_store_state_8way(x0, x1, x2, x3,		\
261			      x4, x5, x6, x7,		\
262			      mem_tmp, idx)		\
263	vmovdqu x0, ((idx + 0) * 32)(mem_tmp);		\
264	vmovdqu x1, ((idx + 1) * 32)(mem_tmp);		\
265	vmovdqu x2, ((idx + 2) * 32)(mem_tmp);		\
266	vmovdqu x3, ((idx + 3) * 32)(mem_tmp);		\
267	vmovdqu x4, ((idx + 4) * 32)(mem_tmp);		\
268	vmovdqu x5, ((idx + 5) * 32)(mem_tmp);		\
269	vmovdqu x6, ((idx + 6) * 32)(mem_tmp);		\
270	vmovdqu x7, ((idx + 7) * 32)(mem_tmp);
271
272#define aria_load_state_8way(x0, x1, x2, x3,		\
273			     x4, x5, x6, x7,		\
274			     mem_tmp, idx)		\
275	vmovdqu ((idx + 0) * 32)(mem_tmp), x0;		\
276	vmovdqu ((idx + 1) * 32)(mem_tmp), x1;		\
277	vmovdqu ((idx + 2) * 32)(mem_tmp), x2;		\
278	vmovdqu ((idx + 3) * 32)(mem_tmp), x3;		\
279	vmovdqu ((idx + 4) * 32)(mem_tmp), x4;		\
280	vmovdqu ((idx + 5) * 32)(mem_tmp), x5;		\
281	vmovdqu ((idx + 6) * 32)(mem_tmp), x6;		\
282	vmovdqu ((idx + 7) * 32)(mem_tmp), x7;
283
284#define aria_ark_8way(x0, x1, x2, x3,			\
285		      x4, x5, x6, x7,			\
286		      t0, rk, idx, round)		\
287	/* AddRoundKey */                               \
288	vpbroadcastb ((round * 16) + idx + 3)(rk), t0;	\
289	vpxor t0, x0, x0;				\
290	vpbroadcastb ((round * 16) + idx + 2)(rk), t0;	\
291	vpxor t0, x1, x1;				\
292	vpbroadcastb ((round * 16) + idx + 1)(rk), t0;	\
293	vpxor t0, x2, x2;				\
294	vpbroadcastb ((round * 16) + idx + 0)(rk), t0;	\
295	vpxor t0, x3, x3;				\
296	vpbroadcastb ((round * 16) + idx + 7)(rk), t0;	\
297	vpxor t0, x4, x4;				\
298	vpbroadcastb ((round * 16) + idx + 6)(rk), t0;	\
299	vpxor t0, x5, x5;				\
300	vpbroadcastb ((round * 16) + idx + 5)(rk), t0;	\
301	vpxor t0, x6, x6;				\
302	vpbroadcastb ((round * 16) + idx + 4)(rk), t0;	\
303	vpxor t0, x7, x7;
304
305#ifdef CONFIG_AS_GFNI
306#define aria_sbox_8way_gfni(x0, x1, x2, x3,		\
307			    x4, x5, x6, x7,		\
308			    t0, t1, t2, t3,		\
309			    t4, t5, t6, t7)		\
310	vpbroadcastq .Ltf_s2_bitmatrix, t0;		\
311	vpbroadcastq .Ltf_inv_bitmatrix, t1;		\
312	vpbroadcastq .Ltf_id_bitmatrix, t2;		\
313	vpbroadcastq .Ltf_aff_bitmatrix, t3;		\
314	vpbroadcastq .Ltf_x2_bitmatrix, t4;		\
315	vgf2p8affineinvqb $(tf_s2_const), t0, x1, x1;	\
316	vgf2p8affineinvqb $(tf_s2_const), t0, x5, x5;	\
317	vgf2p8affineqb $(tf_inv_const), t1, x2, x2;	\
318	vgf2p8affineqb $(tf_inv_const), t1, x6, x6;	\
319	vgf2p8affineinvqb $0, t2, x2, x2;		\
320	vgf2p8affineinvqb $0, t2, x6, x6;		\
321	vgf2p8affineinvqb $(tf_aff_const), t3, x0, x0;	\
322	vgf2p8affineinvqb $(tf_aff_const), t3, x4, x4;	\
323	vgf2p8affineqb $(tf_x2_const), t4, x3, x3;	\
324	vgf2p8affineqb $(tf_x2_const), t4, x7, x7;	\
325	vgf2p8affineinvqb $0, t2, x3, x3;		\
326	vgf2p8affineinvqb $0, t2, x7, x7
327
328#endif /* CONFIG_AS_GFNI */
329#define aria_sbox_8way(x0, x1, x2, x3,			\
330		       x4, x5, x6, x7,			\
331		       t0, t1, t2, t3,			\
332		       t4, t5, t6, t7)			\
333	vpxor t7, t7, t7;				\
334	vpxor t6, t6, t6;				\
335	vbroadcasti128 .Linv_shift_row, t0;		\
336	vbroadcasti128 .Lshift_row, t1;			\
337	vbroadcasti128 .Ltf_lo__inv_aff__and__s2, t2;	\
338	vbroadcasti128 .Ltf_hi__inv_aff__and__s2, t3;	\
339	vbroadcasti128 .Ltf_lo__x2__and__fwd_aff, t4;	\
340	vbroadcasti128 .Ltf_hi__x2__and__fwd_aff, t5;	\
341							\
342	vextracti128 $1, x0, t6##_x;			\
343	vaesenclast t7##_x, x0##_x, x0##_x;		\
344	vaesenclast t7##_x, t6##_x, t6##_x;		\
345	vinserti128 $1, t6##_x, x0, x0;			\
346							\
347	vextracti128 $1, x4, t6##_x;			\
348	vaesenclast t7##_x, x4##_x, x4##_x;		\
349	vaesenclast t7##_x, t6##_x, t6##_x;		\
350	vinserti128 $1, t6##_x, x4, x4;			\
351							\
352	vextracti128 $1, x1, t6##_x;			\
353	vaesenclast t7##_x, x1##_x, x1##_x;		\
354	vaesenclast t7##_x, t6##_x, t6##_x;		\
355	vinserti128 $1, t6##_x, x1, x1;			\
356							\
357	vextracti128 $1, x5, t6##_x;			\
358	vaesenclast t7##_x, x5##_x, x5##_x;		\
359	vaesenclast t7##_x, t6##_x, t6##_x;		\
360	vinserti128 $1, t6##_x, x5, x5;			\
361							\
362	vextracti128 $1, x2, t6##_x;			\
363	vaesdeclast t7##_x, x2##_x, x2##_x;		\
364	vaesdeclast t7##_x, t6##_x, t6##_x;		\
365	vinserti128 $1, t6##_x, x2, x2;			\
366							\
367	vextracti128 $1, x6, t6##_x;			\
368	vaesdeclast t7##_x, x6##_x, x6##_x;		\
369	vaesdeclast t7##_x, t6##_x, t6##_x;		\
370	vinserti128 $1, t6##_x, x6, x6;			\
371							\
372	vpbroadcastd .L0f0f0f0f, t6;			\
373							\
374	/* AES inverse shift rows */			\
375	vpshufb t0, x0, x0;				\
376	vpshufb t0, x4, x4;				\
377	vpshufb t0, x1, x1;				\
378	vpshufb t0, x5, x5;				\
379	vpshufb t1, x3, x3;				\
380	vpshufb t1, x7, x7;				\
381	vpshufb t1, x2, x2;				\
382	vpshufb t1, x6, x6;				\
383							\
384	/* affine transformation for S2 */		\
385	filter_8bit(x1, t2, t3, t6, t0);		\
386	/* affine transformation for S2 */		\
387	filter_8bit(x5, t2, t3, t6, t0);		\
388							\
389	/* affine transformation for X2 */		\
390	filter_8bit(x3, t4, t5, t6, t0);		\
391	/* affine transformation for X2 */		\
392	filter_8bit(x7, t4, t5, t6, t0);		\
393							\
394	vpxor t6, t6, t6;				\
395	vextracti128 $1, x3, t6##_x;			\
396	vaesdeclast t7##_x, x3##_x, x3##_x;		\
397	vaesdeclast t7##_x, t6##_x, t6##_x;		\
398	vinserti128 $1, t6##_x, x3, x3;			\
399							\
400	vextracti128 $1, x7, t6##_x;			\
401	vaesdeclast t7##_x, x7##_x, x7##_x;		\
402	vaesdeclast t7##_x, t6##_x, t6##_x;		\
403	vinserti128 $1, t6##_x, x7, x7;			\
404
405#define aria_diff_m(x0, x1, x2, x3,			\
406		    t0, t1, t2, t3)			\
407	/* T = rotr32(X, 8); */				\
408	/* X ^= T */					\
409	vpxor x0, x3, t0;				\
410	vpxor x1, x0, t1;				\
411	vpxor x2, x1, t2;				\
412	vpxor x3, x2, t3;				\
413	/* X = T ^ rotr(X, 16); */			\
414	vpxor t2, x0, x0;				\
415	vpxor x1, t3, t3;				\
416	vpxor t0, x2, x2;				\
417	vpxor t1, x3, x1;				\
418	vmovdqu t3, x3;
419
420#define aria_diff_word(x0, x1, x2, x3,			\
421		       x4, x5, x6, x7,			\
422		       y0, y1, y2, y3,			\
423		       y4, y5, y6, y7)			\
424	/* t1 ^= t2; */					\
425	vpxor y0, x4, x4;				\
426	vpxor y1, x5, x5;				\
427	vpxor y2, x6, x6;				\
428	vpxor y3, x7, x7;				\
429							\
430	/* t2 ^= t3; */					\
431	vpxor y4, y0, y0;				\
432	vpxor y5, y1, y1;				\
433	vpxor y6, y2, y2;				\
434	vpxor y7, y3, y3;				\
435							\
436	/* t0 ^= t1; */					\
437	vpxor x4, x0, x0;				\
438	vpxor x5, x1, x1;				\
439	vpxor x6, x2, x2;				\
440	vpxor x7, x3, x3;				\
441							\
442	/* t3 ^= t1; */					\
443	vpxor x4, y4, y4;				\
444	vpxor x5, y5, y5;				\
445	vpxor x6, y6, y6;				\
446	vpxor x7, y7, y7;				\
447							\
448	/* t2 ^= t0; */					\
449	vpxor x0, y0, y0;				\
450	vpxor x1, y1, y1;				\
451	vpxor x2, y2, y2;				\
452	vpxor x3, y3, y3;				\
453							\
454	/* t1 ^= t2; */					\
455	vpxor y0, x4, x4;				\
456	vpxor y1, x5, x5;				\
457	vpxor y2, x6, x6;				\
458	vpxor y3, x7, x7;
459
460#define aria_fe(x0, x1, x2, x3,				\
461		x4, x5, x6, x7,				\
462		y0, y1, y2, y3,				\
463		y4, y5, y6, y7,				\
464		mem_tmp, rk, round)			\
465	aria_ark_8way(x0, x1, x2, x3, x4, x5, x6, x7,	\
466		      y0, rk, 8, round);		\
467							\
468	aria_sbox_8way(x2, x3, x0, x1, x6, x7, x4, x5,	\
469		       y0, y1, y2, y3, y4, y5, y6, y7);	\
470							\
471	aria_diff_m(x0, x1, x2, x3, y0, y1, y2, y3);	\
472	aria_diff_m(x4, x5, x6, x7, y0, y1, y2, y3);	\
473	aria_store_state_8way(x0, x1, x2, x3,		\
474			      x4, x5, x6, x7,		\
475			      mem_tmp, 8);		\
476							\
477	aria_load_state_8way(x0, x1, x2, x3,		\
478			     x4, x5, x6, x7,		\
479			     mem_tmp, 0);		\
480	aria_ark_8way(x0, x1, x2, x3, x4, x5, x6, x7,	\
481		      y0, rk, 0, round);		\
482							\
483	aria_sbox_8way(x2, x3, x0, x1, x6, x7, x4, x5,	\
484		       y0, y1, y2, y3, y4, y5, y6, y7);	\
485							\
486	aria_diff_m(x0, x1, x2, x3, y0, y1, y2, y3);	\
487	aria_diff_m(x4, x5, x6, x7, y0, y1, y2, y3);	\
488	aria_store_state_8way(x0, x1, x2, x3,		\
489			      x4, x5, x6, x7,		\
490			      mem_tmp, 0);		\
491	aria_load_state_8way(y0, y1, y2, y3,		\
492			     y4, y5, y6, y7,		\
493			     mem_tmp, 8);		\
494	aria_diff_word(x0, x1, x2, x3,			\
495		       x4, x5, x6, x7,			\
496		       y0, y1, y2, y3,			\
497		       y4, y5, y6, y7);			\
498	/* aria_diff_byte()				\
499	 * T3 = ABCD -> BADC				\
500	 * T3 = y4, y5, y6, y7 -> y5, y4, y7, y6	\
501	 * T0 = ABCD -> CDAB				\
502	 * T0 = x0, x1, x2, x3 -> x2, x3, x0, x1	\
503	 * T1 = ABCD -> DCBA				\
504	 * T1 = x4, x5, x6, x7 -> x7, x6, x5, x4	\
505	 */						\
506	aria_diff_word(x2, x3, x0, x1,			\
507		       x7, x6, x5, x4,			\
508		       y0, y1, y2, y3,			\
509		       y5, y4, y7, y6);			\
510	aria_store_state_8way(x3, x2, x1, x0,		\
511			      x6, x7, x4, x5,		\
512			      mem_tmp, 0);
513
514#define aria_fo(x0, x1, x2, x3,				\
515		x4, x5, x6, x7,				\
516		y0, y1, y2, y3,				\
517		y4, y5, y6, y7,				\
518		mem_tmp, rk, round)			\
519	aria_ark_8way(x0, x1, x2, x3, x4, x5, x6, x7,	\
520		      y0, rk, 8, round);		\
521							\
522	aria_sbox_8way(x0, x1, x2, x3, x4, x5, x6, x7,	\
523		       y0, y1, y2, y3, y4, y5, y6, y7);	\
524							\
525	aria_diff_m(x0, x1, x2, x3, y0, y1, y2, y3);	\
526	aria_diff_m(x4, x5, x6, x7, y0, y1, y2, y3);	\
527	aria_store_state_8way(x0, x1, x2, x3,		\
528			      x4, x5, x6, x7,		\
529			      mem_tmp, 8);		\
530							\
531	aria_load_state_8way(x0, x1, x2, x3,		\
532			     x4, x5, x6, x7,		\
533			     mem_tmp, 0);		\
534	aria_ark_8way(x0, x1, x2, x3, x4, x5, x6, x7,	\
535		      y0, rk, 0, round);		\
536							\
537	aria_sbox_8way(x0, x1, x2, x3, x4, x5, x6, x7,	\
538		       y0, y1, y2, y3, y4, y5, y6, y7);	\
539							\
540	aria_diff_m(x0, x1, x2, x3, y0, y1, y2, y3);	\
541	aria_diff_m(x4, x5, x6, x7, y0, y1, y2, y3);	\
542	aria_store_state_8way(x0, x1, x2, x3,		\
543			      x4, x5, x6, x7,		\
544			      mem_tmp, 0);		\
545	aria_load_state_8way(y0, y1, y2, y3,		\
546			     y4, y5, y6, y7,		\
547			     mem_tmp, 8);		\
548	aria_diff_word(x0, x1, x2, x3,			\
549		       x4, x5, x6, x7,			\
550		       y0, y1, y2, y3,			\
551		       y4, y5, y6, y7);			\
552	/* aria_diff_byte()				\
553	 * T1 = ABCD -> BADC				\
554	 * T1 = x4, x5, x6, x7 -> x5, x4, x7, x6	\
555	 * T2 = ABCD -> CDAB				\
556	 * T2 = y0, y1, y2, y3, -> y2, y3, y0, y1	\
557	 * T3 = ABCD -> DCBA				\
558	 * T3 = y4, y5, y6, y7 -> y7, y6, y5, y4	\
559	 */						\
560	aria_diff_word(x0, x1, x2, x3,			\
561		       x5, x4, x7, x6,			\
562		       y2, y3, y0, y1,			\
563		       y7, y6, y5, y4);			\
564	aria_store_state_8way(x3, x2, x1, x0,		\
565			      x6, x7, x4, x5,		\
566			      mem_tmp, 0);
567
568#define aria_ff(x0, x1, x2, x3,				\
569		x4, x5, x6, x7,				\
570		y0, y1, y2, y3,				\
571		y4, y5, y6, y7,				\
572		mem_tmp, rk, round, last_round)		\
573	aria_ark_8way(x0, x1, x2, x3, x4, x5, x6, x7,	\
574		      y0, rk, 8, round);		\
575							\
576	aria_sbox_8way(x2, x3, x0, x1, x6, x7, x4, x5,	\
577		       y0, y1, y2, y3, y4, y5, y6, y7);	\
578							\
579	aria_ark_8way(x0, x1, x2, x3, x4, x5, x6, x7,	\
580		      y0, rk, 8, last_round);		\
581							\
582	aria_store_state_8way(x0, x1, x2, x3,		\
583			      x4, x5, x6, x7,		\
584			      mem_tmp, 8);		\
585							\
586	aria_load_state_8way(x0, x1, x2, x3,		\
587			     x4, x5, x6, x7,		\
588			     mem_tmp, 0);		\
589	aria_ark_8way(x0, x1, x2, x3, x4, x5, x6, x7,	\
590		      y0, rk, 0, round);		\
591							\
592	aria_sbox_8way(x2, x3, x0, x1, x6, x7, x4, x5,	\
593		       y0, y1, y2, y3, y4, y5, y6, y7);	\
594							\
595	aria_ark_8way(x0, x1, x2, x3, x4, x5, x6, x7,	\
596		      y0, rk, 0, last_round);		\
597							\
598	aria_load_state_8way(y0, y1, y2, y3,		\
599			     y4, y5, y6, y7,		\
600			     mem_tmp, 8);
601#ifdef CONFIG_AS_GFNI
602#define aria_fe_gfni(x0, x1, x2, x3,			\
603		     x4, x5, x6, x7,			\
604		     y0, y1, y2, y3,			\
605		     y4, y5, y6, y7,			\
606		     mem_tmp, rk, round)		\
607	aria_ark_8way(x0, x1, x2, x3, x4, x5, x6, x7,	\
608		      y0, rk, 8, round);		\
609							\
610	aria_sbox_8way_gfni(x2, x3, x0, x1,		\
611			    x6, x7, x4, x5,		\
612			    y0, y1, y2, y3,		\
613			    y4, y5, y6, y7);		\
614							\
615	aria_diff_m(x0, x1, x2, x3, y0, y1, y2, y3);	\
616	aria_diff_m(x4, x5, x6, x7, y0, y1, y2, y3);	\
617	aria_store_state_8way(x0, x1, x2, x3,		\
618			      x4, x5, x6, x7,		\
619			      mem_tmp, 8);		\
620							\
621	aria_load_state_8way(x0, x1, x2, x3,		\
622			     x4, x5, x6, x7,		\
623			     mem_tmp, 0);		\
624	aria_ark_8way(x0, x1, x2, x3, x4, x5, x6, x7,	\
625		      y0, rk, 0, round);		\
626							\
627	aria_sbox_8way_gfni(x2, x3, x0, x1,		\
628			    x6, x7, x4, x5,		\
629			    y0, y1, y2, y3,		\
630			    y4, y5, y6, y7);		\
631							\
632	aria_diff_m(x0, x1, x2, x3, y0, y1, y2, y3);	\
633	aria_diff_m(x4, x5, x6, x7, y0, y1, y2, y3);	\
634	aria_store_state_8way(x0, x1, x2, x3,		\
635			      x4, x5, x6, x7,		\
636			      mem_tmp, 0);		\
637	aria_load_state_8way(y0, y1, y2, y3,		\
638			     y4, y5, y6, y7,		\
639			     mem_tmp, 8);		\
640	aria_diff_word(x0, x1, x2, x3,			\
641		       x4, x5, x6, x7,			\
642		       y0, y1, y2, y3,			\
643		       y4, y5, y6, y7);			\
644	/* aria_diff_byte()				\
645	 * T3 = ABCD -> BADC				\
646	 * T3 = y4, y5, y6, y7 -> y5, y4, y7, y6	\
647	 * T0 = ABCD -> CDAB				\
648	 * T0 = x0, x1, x2, x3 -> x2, x3, x0, x1	\
649	 * T1 = ABCD -> DCBA				\
650	 * T1 = x4, x5, x6, x7 -> x7, x6, x5, x4	\
651	 */						\
652	aria_diff_word(x2, x3, x0, x1,			\
653		       x7, x6, x5, x4,			\
654		       y0, y1, y2, y3,			\
655		       y5, y4, y7, y6);			\
656	aria_store_state_8way(x3, x2, x1, x0,		\
657			      x6, x7, x4, x5,		\
658			      mem_tmp, 0);
659
660#define aria_fo_gfni(x0, x1, x2, x3,			\
661		     x4, x5, x6, x7,			\
662		     y0, y1, y2, y3,			\
663		     y4, y5, y6, y7,			\
664		     mem_tmp, rk, round)		\
665	aria_ark_8way(x0, x1, x2, x3, x4, x5, x6, x7,	\
666		      y0, rk, 8, round);		\
667							\
668	aria_sbox_8way_gfni(x0, x1, x2, x3,		\
669			    x4, x5, x6, x7,		\
670			    y0, y1, y2, y3,		\
671			    y4, y5, y6, y7);		\
672							\
673	aria_diff_m(x0, x1, x2, x3, y0, y1, y2, y3);	\
674	aria_diff_m(x4, x5, x6, x7, y0, y1, y2, y3);	\
675	aria_store_state_8way(x0, x1, x2, x3,		\
676			      x4, x5, x6, x7,		\
677			      mem_tmp, 8);		\
678							\
679	aria_load_state_8way(x0, x1, x2, x3,		\
680			     x4, x5, x6, x7,		\
681			     mem_tmp, 0);		\
682	aria_ark_8way(x0, x1, x2, x3, x4, x5, x6, x7,	\
683		      y0, rk, 0, round);		\
684							\
685	aria_sbox_8way_gfni(x0, x1, x2, x3,		\
686			    x4, x5, x6, x7,		\
687			    y0, y1, y2, y3,		\
688			    y4, y5, y6, y7);		\
689							\
690	aria_diff_m(x0, x1, x2, x3, y0, y1, y2, y3);	\
691	aria_diff_m(x4, x5, x6, x7, y0, y1, y2, y3);	\
692	aria_store_state_8way(x0, x1, x2, x3,		\
693			      x4, x5, x6, x7,		\
694			      mem_tmp, 0);		\
695	aria_load_state_8way(y0, y1, y2, y3,		\
696			     y4, y5, y6, y7,		\
697			     mem_tmp, 8);		\
698	aria_diff_word(x0, x1, x2, x3,			\
699		       x4, x5, x6, x7,			\
700		       y0, y1, y2, y3,			\
701		       y4, y5, y6, y7);			\
702	/* aria_diff_byte()				\
703	 * T1 = ABCD -> BADC				\
704	 * T1 = x4, x5, x6, x7 -> x5, x4, x7, x6	\
705	 * T2 = ABCD -> CDAB				\
706	 * T2 = y0, y1, y2, y3, -> y2, y3, y0, y1	\
707	 * T3 = ABCD -> DCBA				\
708	 * T3 = y4, y5, y6, y7 -> y7, y6, y5, y4	\
709	 */						\
710	aria_diff_word(x0, x1, x2, x3,			\
711		       x5, x4, x7, x6,			\
712		       y2, y3, y0, y1,			\
713		       y7, y6, y5, y4);			\
714	aria_store_state_8way(x3, x2, x1, x0,		\
715			      x6, x7, x4, x5,		\
716			      mem_tmp, 0);
717
718#define aria_ff_gfni(x0, x1, x2, x3,			\
719		x4, x5, x6, x7,				\
720		y0, y1, y2, y3,				\
721		y4, y5, y6, y7,				\
722		mem_tmp, rk, round, last_round)		\
723	aria_ark_8way(x0, x1, x2, x3, x4, x5, x6, x7,	\
724		      y0, rk, 8, round);		\
725							\
726	aria_sbox_8way_gfni(x2, x3, x0, x1,		\
727			    x6, x7, x4, x5,		\
728			    y0, y1, y2, y3,		\
729			    y4, y5, y6, y7);		\
730							\
731	aria_ark_8way(x0, x1, x2, x3, x4, x5, x6, x7,	\
732		      y0, rk, 8, last_round);		\
733							\
734	aria_store_state_8way(x0, x1, x2, x3,		\
735			      x4, x5, x6, x7,		\
736			      mem_tmp, 8);		\
737							\
738	aria_load_state_8way(x0, x1, x2, x3,		\
739			     x4, x5, x6, x7,		\
740			     mem_tmp, 0);		\
741	aria_ark_8way(x0, x1, x2, x3, x4, x5, x6, x7,	\
742		      y0, rk, 0, round);		\
743							\
744	aria_sbox_8way_gfni(x2, x3, x0, x1,		\
745			    x6, x7, x4, x5,		\
746			    y0, y1, y2, y3,		\
747			    y4, y5, y6, y7);		\
748							\
749	aria_ark_8way(x0, x1, x2, x3, x4, x5, x6, x7,	\
750		      y0, rk, 0, last_round);		\
751							\
752	aria_load_state_8way(y0, y1, y2, y3,		\
753			     y4, y5, y6, y7,		\
754			     mem_tmp, 8);
755#endif /* CONFIG_AS_GFNI */
756
757.section        .rodata.cst32.shufb_16x16b, "aM", @progbits, 32
758.align 32
759#define SHUFB_BYTES(idx) \
760	0 + (idx), 4 + (idx), 8 + (idx), 12 + (idx)
761.Lshufb_16x16b:
762	.byte SHUFB_BYTES(0), SHUFB_BYTES(1), SHUFB_BYTES(2), SHUFB_BYTES(3)
763	.byte SHUFB_BYTES(0), SHUFB_BYTES(1), SHUFB_BYTES(2), SHUFB_BYTES(3)
764
765.section	.rodata.cst16, "aM", @progbits, 16
766.align 16
767/* For isolating SubBytes from AESENCLAST, inverse shift row */
768.Linv_shift_row:
769	.byte 0x00, 0x0d, 0x0a, 0x07, 0x04, 0x01, 0x0e, 0x0b
770	.byte 0x08, 0x05, 0x02, 0x0f, 0x0c, 0x09, 0x06, 0x03
771.Lshift_row:
772	.byte 0x00, 0x05, 0x0a, 0x0f, 0x04, 0x09, 0x0e, 0x03
773	.byte 0x08, 0x0d, 0x02, 0x07, 0x0c, 0x01, 0x06, 0x0b
774/* For CTR-mode IV byteswap */
775.Lbswap128_mask:
776	.byte 0x0f, 0x0e, 0x0d, 0x0c, 0x0b, 0x0a, 0x09, 0x08
777	.byte 0x07, 0x06, 0x05, 0x04, 0x03, 0x02, 0x01, 0x00
778
779/* AES inverse affine and S2 combined:
780 *      1 1 0 0 0 0 0 1     x0     0
781 *      0 1 0 0 1 0 0 0     x1     0
782 *      1 1 0 0 1 1 1 1     x2     0
783 *      0 1 1 0 1 0 0 1     x3     1
784 *      0 1 0 0 1 1 0 0  *  x4  +  0
785 *      0 1 0 1 1 0 0 0     x5     0
786 *      0 0 0 0 0 1 0 1     x6     0
787 *      1 1 1 0 0 1 1 1     x7     1
788 */
789.Ltf_lo__inv_aff__and__s2:
790	.octa 0x92172DA81A9FA520B2370D883ABF8500
791.Ltf_hi__inv_aff__and__s2:
792	.octa 0x2B15FFC1AF917B45E6D8320C625CB688
793
794/* X2 and AES forward affine combined:
795 *      1 0 1 1 0 0 0 1     x0     0
796 *      0 1 1 1 1 0 1 1     x1     0
797 *      0 0 0 1 1 0 1 0     x2     1
798 *      0 1 0 0 0 1 0 0     x3     0
799 *      0 0 1 1 1 0 1 1  *  x4  +  0
800 *      0 1 0 0 1 0 0 0     x5     0
801 *      1 1 0 1 0 0 1 1     x6     0
802 *      0 1 0 0 1 0 1 0     x7     0
803 */
804.Ltf_lo__x2__and__fwd_aff:
805	.octa 0xEFAE0544FCBD1657B8F95213ABEA4100
806.Ltf_hi__x2__and__fwd_aff:
807	.octa 0x3F893781E95FE1576CDA64D2BA0CB204
808
809#ifdef CONFIG_AS_GFNI
810.section	.rodata.cst8, "aM", @progbits, 8
811.align 8
812/* AES affine: */
813#define tf_aff_const BV8(1, 1, 0, 0, 0, 1, 1, 0)
814.Ltf_aff_bitmatrix:
815	.quad BM8X8(BV8(1, 0, 0, 0, 1, 1, 1, 1),
816		    BV8(1, 1, 0, 0, 0, 1, 1, 1),
817		    BV8(1, 1, 1, 0, 0, 0, 1, 1),
818		    BV8(1, 1, 1, 1, 0, 0, 0, 1),
819		    BV8(1, 1, 1, 1, 1, 0, 0, 0),
820		    BV8(0, 1, 1, 1, 1, 1, 0, 0),
821		    BV8(0, 0, 1, 1, 1, 1, 1, 0),
822		    BV8(0, 0, 0, 1, 1, 1, 1, 1))
823
824/* AES inverse affine: */
825#define tf_inv_const BV8(1, 0, 1, 0, 0, 0, 0, 0)
826.Ltf_inv_bitmatrix:
827	.quad BM8X8(BV8(0, 0, 1, 0, 0, 1, 0, 1),
828		    BV8(1, 0, 0, 1, 0, 0, 1, 0),
829		    BV8(0, 1, 0, 0, 1, 0, 0, 1),
830		    BV8(1, 0, 1, 0, 0, 1, 0, 0),
831		    BV8(0, 1, 0, 1, 0, 0, 1, 0),
832		    BV8(0, 0, 1, 0, 1, 0, 0, 1),
833		    BV8(1, 0, 0, 1, 0, 1, 0, 0),
834		    BV8(0, 1, 0, 0, 1, 0, 1, 0))
835
836/* S2: */
837#define tf_s2_const BV8(0, 1, 0, 0, 0, 1, 1, 1)
838.Ltf_s2_bitmatrix:
839	.quad BM8X8(BV8(0, 1, 0, 1, 0, 1, 1, 1),
840		    BV8(0, 0, 1, 1, 1, 1, 1, 1),
841		    BV8(1, 1, 1, 0, 1, 1, 0, 1),
842		    BV8(1, 1, 0, 0, 0, 0, 1, 1),
843		    BV8(0, 1, 0, 0, 0, 0, 1, 1),
844		    BV8(1, 1, 0, 0, 1, 1, 1, 0),
845		    BV8(0, 1, 1, 0, 0, 0, 1, 1),
846		    BV8(1, 1, 1, 1, 0, 1, 1, 0))
847
848/* X2: */
849#define tf_x2_const BV8(0, 0, 1, 1, 0, 1, 0, 0)
850.Ltf_x2_bitmatrix:
851	.quad BM8X8(BV8(0, 0, 0, 1, 1, 0, 0, 0),
852		    BV8(0, 0, 1, 0, 0, 1, 1, 0),
853		    BV8(0, 0, 0, 0, 1, 0, 1, 0),
854		    BV8(1, 1, 1, 0, 0, 0, 1, 1),
855		    BV8(1, 1, 1, 0, 1, 1, 0, 0),
856		    BV8(0, 1, 1, 0, 1, 0, 1, 1),
857		    BV8(1, 0, 1, 1, 1, 1, 0, 1),
858		    BV8(1, 0, 0, 1, 0, 0, 1, 1))
859
860/* Identity matrix: */
861.Ltf_id_bitmatrix:
862	.quad BM8X8(BV8(1, 0, 0, 0, 0, 0, 0, 0),
863		    BV8(0, 1, 0, 0, 0, 0, 0, 0),
864		    BV8(0, 0, 1, 0, 0, 0, 0, 0),
865		    BV8(0, 0, 0, 1, 0, 0, 0, 0),
866		    BV8(0, 0, 0, 0, 1, 0, 0, 0),
867		    BV8(0, 0, 0, 0, 0, 1, 0, 0),
868		    BV8(0, 0, 0, 0, 0, 0, 1, 0),
869		    BV8(0, 0, 0, 0, 0, 0, 0, 1))
870
871#endif /* CONFIG_AS_GFNI */
872
873/* 4-bit mask */
874.section	.rodata.cst4.L0f0f0f0f, "aM", @progbits, 4
875.align 4
876.L0f0f0f0f:
877	.long 0x0f0f0f0f
878
879.text
880
881SYM_FUNC_START_LOCAL(__aria_aesni_avx2_crypt_32way)
882	/* input:
883	 *      %r9: rk
884	 *      %rsi: dst
885	 *      %rdx: src
886	 *      %ymm0..%ymm15: byte-sliced blocks
887	 */
888
889	FRAME_BEGIN
890
891	movq %rsi, %rax;
892	leaq 8 * 32(%rax), %r8;
893
894	inpack16_post(%ymm0, %ymm1, %ymm2, %ymm3, %ymm4, %ymm5, %ymm6, %ymm7,
895		      %ymm8, %ymm9, %ymm10, %ymm11, %ymm12, %ymm13, %ymm14,
896		      %ymm15, %rax, %r8);
897	aria_fo(%ymm8, %ymm9, %ymm10, %ymm11, %ymm12, %ymm13, %ymm14, %ymm15,
898		%ymm0, %ymm1, %ymm2, %ymm3, %ymm4, %ymm5, %ymm6, %ymm7,
899		%rax, %r9, 0);
900	aria_fe(%ymm1, %ymm0, %ymm3, %ymm2, %ymm4, %ymm5, %ymm6, %ymm7,
901		%ymm8, %ymm9, %ymm10, %ymm11, %ymm12, %ymm13, %ymm14,
902		%ymm15, %rax, %r9, 1);
903	aria_fo(%ymm9, %ymm8, %ymm11, %ymm10, %ymm12, %ymm13, %ymm14, %ymm15,
904		%ymm0, %ymm1, %ymm2, %ymm3, %ymm4, %ymm5, %ymm6, %ymm7,
905		%rax, %r9, 2);
906	aria_fe(%ymm1, %ymm0, %ymm3, %ymm2, %ymm4, %ymm5, %ymm6, %ymm7,
907		%ymm8, %ymm9, %ymm10, %ymm11, %ymm12, %ymm13, %ymm14,
908		%ymm15, %rax, %r9, 3);
909	aria_fo(%ymm9, %ymm8, %ymm11, %ymm10, %ymm12, %ymm13, %ymm14, %ymm15,
910		%ymm0, %ymm1, %ymm2, %ymm3, %ymm4, %ymm5, %ymm6, %ymm7,
911		%rax, %r9, 4);
912	aria_fe(%ymm1, %ymm0, %ymm3, %ymm2, %ymm4, %ymm5, %ymm6, %ymm7,
913		%ymm8, %ymm9, %ymm10, %ymm11, %ymm12, %ymm13, %ymm14,
914		%ymm15, %rax, %r9, 5);
915	aria_fo(%ymm9, %ymm8, %ymm11, %ymm10, %ymm12, %ymm13, %ymm14, %ymm15,
916		%ymm0, %ymm1, %ymm2, %ymm3, %ymm4, %ymm5, %ymm6, %ymm7,
917		%rax, %r9, 6);
918	aria_fe(%ymm1, %ymm0, %ymm3, %ymm2, %ymm4, %ymm5, %ymm6, %ymm7,
919		%ymm8, %ymm9, %ymm10, %ymm11, %ymm12, %ymm13, %ymm14,
920		%ymm15, %rax, %r9, 7);
921	aria_fo(%ymm9, %ymm8, %ymm11, %ymm10, %ymm12, %ymm13, %ymm14, %ymm15,
922		%ymm0, %ymm1, %ymm2, %ymm3, %ymm4, %ymm5, %ymm6, %ymm7,
923		%rax, %r9, 8);
924	aria_fe(%ymm1, %ymm0, %ymm3, %ymm2, %ymm4, %ymm5, %ymm6, %ymm7,
925		%ymm8, %ymm9, %ymm10, %ymm11, %ymm12, %ymm13, %ymm14,
926		%ymm15, %rax, %r9, 9);
927	aria_fo(%ymm9, %ymm8, %ymm11, %ymm10, %ymm12, %ymm13, %ymm14, %ymm15,
928		%ymm0, %ymm1, %ymm2, %ymm3, %ymm4, %ymm5, %ymm6, %ymm7,
929		%rax, %r9, 10);
930	cmpl $12, ARIA_CTX_rounds(CTX);
931	jne .Laria_192;
932	aria_ff(%ymm1, %ymm0, %ymm3, %ymm2, %ymm4, %ymm5, %ymm6, %ymm7,
933		%ymm8, %ymm9, %ymm10, %ymm11, %ymm12, %ymm13, %ymm14,
934		%ymm15, %rax, %r9, 11, 12);
935	jmp .Laria_end;
936.Laria_192:
937	aria_fe(%ymm1, %ymm0, %ymm3, %ymm2, %ymm4, %ymm5, %ymm6, %ymm7,
938		%ymm8, %ymm9, %ymm10, %ymm11, %ymm12, %ymm13, %ymm14,
939		%ymm15, %rax, %r9, 11);
940	aria_fo(%ymm9, %ymm8, %ymm11, %ymm10, %ymm12, %ymm13, %ymm14, %ymm15,
941		%ymm0, %ymm1, %ymm2, %ymm3, %ymm4, %ymm5, %ymm6, %ymm7,
942		%rax, %r9, 12);
943	cmpl $14, ARIA_CTX_rounds(CTX);
944	jne .Laria_256;
945	aria_ff(%ymm1, %ymm0, %ymm3, %ymm2, %ymm4, %ymm5, %ymm6, %ymm7,
946		%ymm8, %ymm9, %ymm10, %ymm11, %ymm12, %ymm13, %ymm14,
947		%ymm15, %rax, %r9, 13, 14);
948	jmp .Laria_end;
949.Laria_256:
950	aria_fe(%ymm1, %ymm0, %ymm3, %ymm2, %ymm4, %ymm5, %ymm6, %ymm7,
951		%ymm8, %ymm9, %ymm10, %ymm11, %ymm12, %ymm13, %ymm14,
952		%ymm15, %rax, %r9, 13);
953	aria_fo(%ymm9, %ymm8, %ymm11, %ymm10, %ymm12, %ymm13, %ymm14, %ymm15,
954		%ymm0, %ymm1, %ymm2, %ymm3, %ymm4, %ymm5, %ymm6, %ymm7,
955		%rax, %r9, 14);
956	aria_ff(%ymm1, %ymm0, %ymm3, %ymm2, %ymm4, %ymm5, %ymm6, %ymm7,
957		%ymm8, %ymm9, %ymm10, %ymm11, %ymm12, %ymm13, %ymm14,
958		%ymm15, %rax, %r9, 15, 16);
959.Laria_end:
960	debyteslice_16x16b(%ymm8, %ymm12, %ymm1, %ymm4,
961			   %ymm9, %ymm13, %ymm0, %ymm5,
962			   %ymm10, %ymm14, %ymm3, %ymm6,
963			   %ymm11, %ymm15, %ymm2, %ymm7,
964			   (%rax), (%r8));
965
966	FRAME_END
967	RET;
968SYM_FUNC_END(__aria_aesni_avx2_crypt_32way)
969
970SYM_TYPED_FUNC_START(aria_aesni_avx2_encrypt_32way)
971	/* input:
972	 *      %rdi: ctx, CTX
973	 *      %rsi: dst
974	 *      %rdx: src
975	 */
976
977	FRAME_BEGIN
978
979	leaq ARIA_CTX_enc_key(CTX), %r9;
980
981	inpack16_pre(%ymm0, %ymm1, %ymm2, %ymm3, %ymm4, %ymm5, %ymm6, %ymm7,
982		     %ymm8, %ymm9, %ymm10, %ymm11, %ymm12, %ymm13, %ymm14,
983		     %ymm15, %rdx);
984
985	call __aria_aesni_avx2_crypt_32way;
986
987	write_output(%ymm1, %ymm0, %ymm3, %ymm2, %ymm4, %ymm5, %ymm6, %ymm7,
988		     %ymm8, %ymm9, %ymm10, %ymm11, %ymm12, %ymm13, %ymm14,
989		     %ymm15, %rax);
990
991	FRAME_END
992	RET;
993SYM_FUNC_END(aria_aesni_avx2_encrypt_32way)
994
995SYM_TYPED_FUNC_START(aria_aesni_avx2_decrypt_32way)
996	/* input:
997	 *      %rdi: ctx, CTX
998	 *      %rsi: dst
999	 *      %rdx: src
1000	 */
1001
1002	FRAME_BEGIN
1003
1004	leaq ARIA_CTX_dec_key(CTX), %r9;
1005
1006	inpack16_pre(%ymm0, %ymm1, %ymm2, %ymm3, %ymm4, %ymm5, %ymm6, %ymm7,
1007		     %ymm8, %ymm9, %ymm10, %ymm11, %ymm12, %ymm13, %ymm14,
1008		     %ymm15, %rdx);
1009
1010	call __aria_aesni_avx2_crypt_32way;
1011
1012	write_output(%ymm1, %ymm0, %ymm3, %ymm2, %ymm4, %ymm5, %ymm6, %ymm7,
1013		     %ymm8, %ymm9, %ymm10, %ymm11, %ymm12, %ymm13, %ymm14,
1014		     %ymm15, %rax);
1015
1016	FRAME_END
1017	RET;
1018SYM_FUNC_END(aria_aesni_avx2_decrypt_32way)
1019
1020SYM_FUNC_START_LOCAL(__aria_aesni_avx2_ctr_gen_keystream_32way)
1021	/* input:
1022	 *      %rdi: ctx
1023	 *      %rsi: dst
1024	 *      %rdx: src
1025	 *      %rcx: keystream
1026	 *      %r8: iv (big endian, 128bit)
1027	 */
1028
1029	FRAME_BEGIN
1030	movq 8(%r8), %r11;
1031	bswapq %r11;
1032
1033	vbroadcasti128 .Lbswap128_mask (%rip), %ymm6;
1034	vpcmpeqd %ymm0, %ymm0, %ymm0;
1035	vpsrldq $8, %ymm0, %ymm0;   /* ab: -1:0 ; cd: -1:0 */
1036	vpaddq %ymm0, %ymm0, %ymm5; /* ab: -2:0 ; cd: -2:0 */
1037
1038	/* load IV and byteswap */
1039	vmovdqu (%r8), %xmm7;
1040	vpshufb %xmm6, %xmm7, %xmm7;
1041	vmovdqa %xmm7, %xmm3;
1042	inc_le128(%xmm7, %xmm0, %xmm4);
1043	vinserti128 $1, %xmm7, %ymm3, %ymm3;
1044	vpshufb %ymm6, %ymm3, %ymm8; /* +1 ; +0 */
1045
1046	/* check need for handling 64-bit overflow and carry */
1047	cmpq $(0xffffffffffffffff - 32), %r11;
1048	ja .Lhandle_ctr_carry;
1049
1050	/* construct IVs */
1051	vpsubq %ymm5, %ymm3, %ymm3; /* +3 ; +2 */
1052	vpshufb %ymm6, %ymm3, %ymm9;
1053	vpsubq %ymm5, %ymm3, %ymm3; /* +5 ; +4 */
1054	vpshufb %ymm6, %ymm3, %ymm10;
1055	vpsubq %ymm5, %ymm3, %ymm3; /* +7 ; +6 */
1056	vpshufb %ymm6, %ymm3, %ymm11;
1057	vpsubq %ymm5, %ymm3, %ymm3; /* +9 ; +8 */
1058	vpshufb %ymm6, %ymm3, %ymm12;
1059	vpsubq %ymm5, %ymm3, %ymm3; /* +11 ; +10 */
1060	vpshufb %ymm6, %ymm3, %ymm13;
1061	vpsubq %ymm5, %ymm3, %ymm3; /* +13 ; +12 */
1062	vpshufb %ymm6, %ymm3, %ymm14;
1063	vpsubq %ymm5, %ymm3, %ymm3; /* +15 ; +14 */
1064	vpshufb %ymm6, %ymm3, %ymm15;
1065	vmovdqu %ymm8, (0 * 32)(%rcx);
1066	vmovdqu %ymm9, (1 * 32)(%rcx);
1067	vmovdqu %ymm10, (2 * 32)(%rcx);
1068	vmovdqu %ymm11, (3 * 32)(%rcx);
1069	vmovdqu %ymm12, (4 * 32)(%rcx);
1070	vmovdqu %ymm13, (5 * 32)(%rcx);
1071	vmovdqu %ymm14, (6 * 32)(%rcx);
1072	vmovdqu %ymm15, (7 * 32)(%rcx);
1073
1074	vpsubq %ymm5, %ymm3, %ymm3; /* +17 ; +16 */
1075	vpshufb %ymm6, %ymm3, %ymm8;
1076	vpsubq %ymm5, %ymm3, %ymm3; /* +19 ; +18 */
1077	vpshufb %ymm6, %ymm3, %ymm9;
1078	vpsubq %ymm5, %ymm3, %ymm3; /* +21 ; +20 */
1079	vpshufb %ymm6, %ymm3, %ymm10;
1080	vpsubq %ymm5, %ymm3, %ymm3; /* +23 ; +22 */
1081	vpshufb %ymm6, %ymm3, %ymm11;
1082	vpsubq %ymm5, %ymm3, %ymm3; /* +25 ; +24 */
1083	vpshufb %ymm6, %ymm3, %ymm12;
1084	vpsubq %ymm5, %ymm3, %ymm3; /* +27 ; +26 */
1085	vpshufb %ymm6, %ymm3, %ymm13;
1086	vpsubq %ymm5, %ymm3, %ymm3; /* +29 ; +28 */
1087	vpshufb %ymm6, %ymm3, %ymm14;
1088	vpsubq %ymm5, %ymm3, %ymm3; /* +31 ; +30 */
1089	vpshufb %ymm6, %ymm3, %ymm15;
1090	vpsubq %ymm5, %ymm3, %ymm3; /* +32 */
1091	vpshufb %xmm6, %xmm3, %xmm3;
1092	vmovdqu %xmm3, (%r8);
1093	vmovdqu (0 * 32)(%rcx), %ymm0;
1094	vmovdqu (1 * 32)(%rcx), %ymm1;
1095	vmovdqu (2 * 32)(%rcx), %ymm2;
1096	vmovdqu (3 * 32)(%rcx), %ymm3;
1097	vmovdqu (4 * 32)(%rcx), %ymm4;
1098	vmovdqu (5 * 32)(%rcx), %ymm5;
1099	vmovdqu (6 * 32)(%rcx), %ymm6;
1100	vmovdqu (7 * 32)(%rcx), %ymm7;
1101	jmp .Lctr_carry_done;
1102
1103	.Lhandle_ctr_carry:
1104	/* construct IVs */
1105	inc_le128(%ymm3, %ymm0, %ymm4);
1106	inc_le128(%ymm3, %ymm0, %ymm4);
1107	vpshufb %ymm6, %ymm3, %ymm9; /* +3 ; +2 */
1108	inc_le128(%ymm3, %ymm0, %ymm4);
1109	inc_le128(%ymm3, %ymm0, %ymm4);
1110	vpshufb %ymm6, %ymm3, %ymm10; /* +5 ; +4 */
1111	inc_le128(%ymm3, %ymm0, %ymm4);
1112	inc_le128(%ymm3, %ymm0, %ymm4);
1113	vpshufb %ymm6, %ymm3, %ymm11; /* +7 ; +6 */
1114	inc_le128(%ymm3, %ymm0, %ymm4);
1115	inc_le128(%ymm3, %ymm0, %ymm4);
1116	vpshufb %ymm6, %ymm3, %ymm12; /* +9 ; +8 */
1117	inc_le128(%ymm3, %ymm0, %ymm4);
1118	inc_le128(%ymm3, %ymm0, %ymm4);
1119	vpshufb %ymm6, %ymm3, %ymm13; /* +11 ; +10 */
1120	inc_le128(%ymm3, %ymm0, %ymm4);
1121	inc_le128(%ymm3, %ymm0, %ymm4);
1122	vpshufb %ymm6, %ymm3, %ymm14; /* +13 ; +12 */
1123	inc_le128(%ymm3, %ymm0, %ymm4);
1124	inc_le128(%ymm3, %ymm0, %ymm4);
1125	vpshufb %ymm6, %ymm3, %ymm15; /* +15 ; +14 */
1126	vmovdqu %ymm8, (0 * 32)(%rcx);
1127	vmovdqu %ymm9, (1 * 32)(%rcx);
1128	vmovdqu %ymm10, (2 * 32)(%rcx);
1129	vmovdqu %ymm11, (3 * 32)(%rcx);
1130	vmovdqu %ymm12, (4 * 32)(%rcx);
1131	vmovdqu %ymm13, (5 * 32)(%rcx);
1132	vmovdqu %ymm14, (6 * 32)(%rcx);
1133	vmovdqu %ymm15, (7 * 32)(%rcx);
1134
1135	inc_le128(%ymm3, %ymm0, %ymm4);
1136	inc_le128(%ymm3, %ymm0, %ymm4);
1137	vpshufb %ymm6, %ymm3, %ymm8; /* +17 ; +16 */
1138	inc_le128(%ymm3, %ymm0, %ymm4);
1139	inc_le128(%ymm3, %ymm0, %ymm4);
1140	vpshufb %ymm6, %ymm3, %ymm9; /* +19 ; +18 */
1141	inc_le128(%ymm3, %ymm0, %ymm4);
1142	inc_le128(%ymm3, %ymm0, %ymm4);
1143	vpshufb %ymm6, %ymm3, %ymm10; /* +21 ; +20 */
1144	inc_le128(%ymm3, %ymm0, %ymm4);
1145	inc_le128(%ymm3, %ymm0, %ymm4);
1146	vpshufb %ymm6, %ymm3, %ymm11; /* +23 ; +22 */
1147	inc_le128(%ymm3, %ymm0, %ymm4);
1148	inc_le128(%ymm3, %ymm0, %ymm4);
1149	vpshufb %ymm6, %ymm3, %ymm12; /* +25 ; +24 */
1150	inc_le128(%ymm3, %ymm0, %ymm4);
1151	inc_le128(%ymm3, %ymm0, %ymm4);
1152	vpshufb %ymm6, %ymm3, %ymm13; /* +27 ; +26 */
1153	inc_le128(%ymm3, %ymm0, %ymm4);
1154	inc_le128(%ymm3, %ymm0, %ymm4);
1155	vpshufb %ymm6, %ymm3, %ymm14; /* +29 ; +28 */
1156	inc_le128(%ymm3, %ymm0, %ymm4);
1157	inc_le128(%ymm3, %ymm0, %ymm4);
1158	vpshufb %ymm6, %ymm3, %ymm15; /* +31 ; +30 */
1159	inc_le128(%ymm3, %ymm0, %ymm4);
1160	vextracti128 $1, %ymm3, %xmm3;
1161	vpshufb %xmm6, %xmm3, %xmm3; /* +32 */
1162	vmovdqu %xmm3, (%r8);
1163	vmovdqu (0 * 32)(%rcx), %ymm0;
1164	vmovdqu (1 * 32)(%rcx), %ymm1;
1165	vmovdqu (2 * 32)(%rcx), %ymm2;
1166	vmovdqu (3 * 32)(%rcx), %ymm3;
1167	vmovdqu (4 * 32)(%rcx), %ymm4;
1168	vmovdqu (5 * 32)(%rcx), %ymm5;
1169	vmovdqu (6 * 32)(%rcx), %ymm6;
1170	vmovdqu (7 * 32)(%rcx), %ymm7;
1171
1172	.Lctr_carry_done:
1173
1174	FRAME_END
1175	RET;
1176SYM_FUNC_END(__aria_aesni_avx2_ctr_gen_keystream_32way)
1177
1178SYM_TYPED_FUNC_START(aria_aesni_avx2_ctr_crypt_32way)
1179	/* input:
1180	 *      %rdi: ctx
1181	 *      %rsi: dst
1182	 *      %rdx: src
1183	 *      %rcx: keystream
1184	 *      %r8: iv (big endian, 128bit)
1185	 */
1186	FRAME_BEGIN
1187
1188	call __aria_aesni_avx2_ctr_gen_keystream_32way;
1189
1190	leaq (%rsi), %r10;
1191	leaq (%rdx), %r11;
1192	leaq (%rcx), %rsi;
1193	leaq (%rcx), %rdx;
1194	leaq ARIA_CTX_enc_key(CTX), %r9;
1195
1196	call __aria_aesni_avx2_crypt_32way;
1197
1198	vpxor (0 * 32)(%r11), %ymm1, %ymm1;
1199	vpxor (1 * 32)(%r11), %ymm0, %ymm0;
1200	vpxor (2 * 32)(%r11), %ymm3, %ymm3;
1201	vpxor (3 * 32)(%r11), %ymm2, %ymm2;
1202	vpxor (4 * 32)(%r11), %ymm4, %ymm4;
1203	vpxor (5 * 32)(%r11), %ymm5, %ymm5;
1204	vpxor (6 * 32)(%r11), %ymm6, %ymm6;
1205	vpxor (7 * 32)(%r11), %ymm7, %ymm7;
1206	vpxor (8 * 32)(%r11), %ymm8, %ymm8;
1207	vpxor (9 * 32)(%r11), %ymm9, %ymm9;
1208	vpxor (10 * 32)(%r11), %ymm10, %ymm10;
1209	vpxor (11 * 32)(%r11), %ymm11, %ymm11;
1210	vpxor (12 * 32)(%r11), %ymm12, %ymm12;
1211	vpxor (13 * 32)(%r11), %ymm13, %ymm13;
1212	vpxor (14 * 32)(%r11), %ymm14, %ymm14;
1213	vpxor (15 * 32)(%r11), %ymm15, %ymm15;
1214	write_output(%ymm1, %ymm0, %ymm3, %ymm2, %ymm4, %ymm5, %ymm6, %ymm7,
1215		     %ymm8, %ymm9, %ymm10, %ymm11, %ymm12, %ymm13, %ymm14,
1216		     %ymm15, %r10);
1217
1218	FRAME_END
1219	RET;
1220SYM_FUNC_END(aria_aesni_avx2_ctr_crypt_32way)
1221
1222#ifdef CONFIG_AS_GFNI
1223SYM_FUNC_START_LOCAL(__aria_aesni_avx2_gfni_crypt_32way)
1224	/* input:
1225	 *      %r9: rk
1226	 *      %rsi: dst
1227	 *      %rdx: src
1228	 *      %ymm0..%ymm15: 16 byte-sliced blocks
1229	 */
1230
1231	FRAME_BEGIN
1232
1233	movq %rsi, %rax;
1234	leaq 8 * 32(%rax), %r8;
1235
1236	inpack16_post(%ymm0, %ymm1, %ymm2, %ymm3,
1237		      %ymm4, %ymm5, %ymm6, %ymm7,
1238		      %ymm8, %ymm9, %ymm10, %ymm11,
1239		      %ymm12, %ymm13, %ymm14,
1240		      %ymm15, %rax, %r8);
1241	aria_fo_gfni(%ymm8, %ymm9, %ymm10, %ymm11,
1242		     %ymm12, %ymm13, %ymm14, %ymm15,
1243		     %ymm0, %ymm1, %ymm2, %ymm3,
1244		     %ymm4, %ymm5, %ymm6, %ymm7,
1245		     %rax, %r9, 0);
1246	aria_fe_gfni(%ymm1, %ymm0, %ymm3, %ymm2,
1247		     %ymm4, %ymm5, %ymm6, %ymm7,
1248		     %ymm8, %ymm9, %ymm10, %ymm11,
1249		     %ymm12, %ymm13, %ymm14,
1250		     %ymm15, %rax, %r9, 1);
1251	aria_fo_gfni(%ymm9, %ymm8, %ymm11, %ymm10,
1252		     %ymm12, %ymm13, %ymm14, %ymm15,
1253		     %ymm0, %ymm1, %ymm2, %ymm3,
1254		     %ymm4, %ymm5, %ymm6, %ymm7,
1255		     %rax, %r9, 2);
1256	aria_fe_gfni(%ymm1, %ymm0, %ymm3, %ymm2,
1257		     %ymm4, %ymm5, %ymm6, %ymm7,
1258		     %ymm8, %ymm9, %ymm10, %ymm11,
1259		     %ymm12, %ymm13, %ymm14,
1260		     %ymm15, %rax, %r9, 3);
1261	aria_fo_gfni(%ymm9, %ymm8, %ymm11, %ymm10,
1262		     %ymm12, %ymm13, %ymm14, %ymm15,
1263		     %ymm0, %ymm1, %ymm2, %ymm3,
1264		     %ymm4, %ymm5, %ymm6, %ymm7,
1265		     %rax, %r9, 4);
1266	aria_fe_gfni(%ymm1, %ymm0, %ymm3, %ymm2,
1267		     %ymm4, %ymm5, %ymm6, %ymm7,
1268		     %ymm8, %ymm9, %ymm10, %ymm11,
1269		     %ymm12, %ymm13, %ymm14,
1270		     %ymm15, %rax, %r9, 5);
1271	aria_fo_gfni(%ymm9, %ymm8, %ymm11, %ymm10,
1272		     %ymm12, %ymm13, %ymm14, %ymm15,
1273		     %ymm0, %ymm1, %ymm2, %ymm3,
1274		     %ymm4, %ymm5, %ymm6, %ymm7,
1275		     %rax, %r9, 6);
1276	aria_fe_gfni(%ymm1, %ymm0, %ymm3, %ymm2,
1277		     %ymm4, %ymm5, %ymm6, %ymm7,
1278		     %ymm8, %ymm9, %ymm10, %ymm11,
1279		     %ymm12, %ymm13, %ymm14,
1280		     %ymm15, %rax, %r9, 7);
1281	aria_fo_gfni(%ymm9, %ymm8, %ymm11, %ymm10,
1282		     %ymm12, %ymm13, %ymm14, %ymm15,
1283		     %ymm0, %ymm1, %ymm2, %ymm3,
1284		     %ymm4, %ymm5, %ymm6, %ymm7,
1285		     %rax, %r9, 8);
1286	aria_fe_gfni(%ymm1, %ymm0, %ymm3, %ymm2,
1287		     %ymm4, %ymm5, %ymm6, %ymm7,
1288		     %ymm8, %ymm9, %ymm10, %ymm11,
1289		     %ymm12, %ymm13, %ymm14,
1290		     %ymm15, %rax, %r9, 9);
1291	aria_fo_gfni(%ymm9, %ymm8, %ymm11, %ymm10,
1292		     %ymm12, %ymm13, %ymm14, %ymm15,
1293		     %ymm0, %ymm1, %ymm2, %ymm3,
1294		     %ymm4, %ymm5, %ymm6, %ymm7,
1295		     %rax, %r9, 10);
1296	cmpl $12, ARIA_CTX_rounds(CTX);
1297	jne .Laria_gfni_192;
1298	aria_ff_gfni(%ymm1, %ymm0, %ymm3, %ymm2, %ymm4, %ymm5, %ymm6, %ymm7,
1299		%ymm8, %ymm9, %ymm10, %ymm11, %ymm12, %ymm13, %ymm14,
1300		%ymm15, %rax, %r9, 11, 12);
1301	jmp .Laria_gfni_end;
1302.Laria_gfni_192:
1303	aria_fe_gfni(%ymm1, %ymm0, %ymm3, %ymm2,
1304		     %ymm4, %ymm5, %ymm6, %ymm7,
1305		     %ymm8, %ymm9, %ymm10, %ymm11,
1306		     %ymm12, %ymm13, %ymm14,
1307		     %ymm15, %rax, %r9, 11);
1308	aria_fo_gfni(%ymm9, %ymm8, %ymm11, %ymm10,
1309		     %ymm12, %ymm13, %ymm14, %ymm15,
1310		     %ymm0, %ymm1, %ymm2, %ymm3,
1311		     %ymm4, %ymm5, %ymm6, %ymm7,
1312		     %rax, %r9, 12);
1313	cmpl $14, ARIA_CTX_rounds(CTX);
1314	jne .Laria_gfni_256;
1315	aria_ff_gfni(%ymm1, %ymm0, %ymm3, %ymm2,
1316		     %ymm4, %ymm5, %ymm6, %ymm7,
1317		     %ymm8, %ymm9, %ymm10, %ymm11,
1318		     %ymm12, %ymm13, %ymm14,
1319		     %ymm15, %rax, %r9, 13, 14);
1320	jmp .Laria_gfni_end;
1321.Laria_gfni_256:
1322	aria_fe_gfni(%ymm1, %ymm0, %ymm3, %ymm2,
1323		     %ymm4, %ymm5, %ymm6, %ymm7,
1324		     %ymm8, %ymm9, %ymm10, %ymm11,
1325		     %ymm12, %ymm13, %ymm14,
1326		     %ymm15, %rax, %r9, 13);
1327	aria_fo_gfni(%ymm9, %ymm8, %ymm11, %ymm10,
1328		     %ymm12, %ymm13, %ymm14, %ymm15,
1329		     %ymm0, %ymm1, %ymm2, %ymm3,
1330		     %ymm4, %ymm5, %ymm6, %ymm7,
1331		     %rax, %r9, 14);
1332	aria_ff_gfni(%ymm1, %ymm0, %ymm3, %ymm2,
1333		     %ymm4, %ymm5, %ymm6, %ymm7,
1334		     %ymm8, %ymm9, %ymm10, %ymm11,
1335		     %ymm12, %ymm13, %ymm14,
1336		     %ymm15, %rax, %r9, 15, 16);
1337.Laria_gfni_end:
1338	debyteslice_16x16b(%ymm8, %ymm12, %ymm1, %ymm4,
1339			   %ymm9, %ymm13, %ymm0, %ymm5,
1340			   %ymm10, %ymm14, %ymm3, %ymm6,
1341			   %ymm11, %ymm15, %ymm2, %ymm7,
1342			   (%rax), (%r8));
1343
1344	FRAME_END
1345	RET;
1346SYM_FUNC_END(__aria_aesni_avx2_gfni_crypt_32way)
1347
1348SYM_TYPED_FUNC_START(aria_aesni_avx2_gfni_encrypt_32way)
1349	/* input:
1350	 *      %rdi: ctx, CTX
1351	 *      %rsi: dst
1352	 *      %rdx: src
1353	 */
1354
1355	FRAME_BEGIN
1356
1357	leaq ARIA_CTX_enc_key(CTX), %r9;
1358
1359	inpack16_pre(%ymm0, %ymm1, %ymm2, %ymm3, %ymm4, %ymm5, %ymm6, %ymm7,
1360		     %ymm8, %ymm9, %ymm10, %ymm11, %ymm12, %ymm13, %ymm14,
1361		     %ymm15, %rdx);
1362
1363	call __aria_aesni_avx2_gfni_crypt_32way;
1364
1365	write_output(%ymm1, %ymm0, %ymm3, %ymm2, %ymm4, %ymm5, %ymm6, %ymm7,
1366		     %ymm8, %ymm9, %ymm10, %ymm11, %ymm12, %ymm13, %ymm14,
1367		     %ymm15, %rax);
1368
1369	FRAME_END
1370	RET;
1371SYM_FUNC_END(aria_aesni_avx2_gfni_encrypt_32way)
1372
1373SYM_TYPED_FUNC_START(aria_aesni_avx2_gfni_decrypt_32way)
1374	/* input:
1375	 *      %rdi: ctx, CTX
1376	 *      %rsi: dst
1377	 *      %rdx: src
1378	 */
1379
1380	FRAME_BEGIN
1381
1382	leaq ARIA_CTX_dec_key(CTX), %r9;
1383
1384	inpack16_pre(%ymm0, %ymm1, %ymm2, %ymm3, %ymm4, %ymm5, %ymm6, %ymm7,
1385		     %ymm8, %ymm9, %ymm10, %ymm11, %ymm12, %ymm13, %ymm14,
1386		     %ymm15, %rdx);
1387
1388	call __aria_aesni_avx2_gfni_crypt_32way;
1389
1390	write_output(%ymm1, %ymm0, %ymm3, %ymm2, %ymm4, %ymm5, %ymm6, %ymm7,
1391		     %ymm8, %ymm9, %ymm10, %ymm11, %ymm12, %ymm13, %ymm14,
1392		     %ymm15, %rax);
1393
1394	FRAME_END
1395	RET;
1396SYM_FUNC_END(aria_aesni_avx2_gfni_decrypt_32way)
1397
1398SYM_TYPED_FUNC_START(aria_aesni_avx2_gfni_ctr_crypt_32way)
1399	/* input:
1400	 *      %rdi: ctx
1401	 *      %rsi: dst
1402	 *      %rdx: src
1403	 *      %rcx: keystream
1404	 *      %r8: iv (big endian, 128bit)
1405	 */
1406	FRAME_BEGIN
1407
1408	call __aria_aesni_avx2_ctr_gen_keystream_32way
1409
1410	leaq (%rsi), %r10;
1411	leaq (%rdx), %r11;
1412	leaq (%rcx), %rsi;
1413	leaq (%rcx), %rdx;
1414	leaq ARIA_CTX_enc_key(CTX), %r9;
1415
1416	call __aria_aesni_avx2_gfni_crypt_32way;
1417
1418	vpxor (0 * 32)(%r11), %ymm1, %ymm1;
1419	vpxor (1 * 32)(%r11), %ymm0, %ymm0;
1420	vpxor (2 * 32)(%r11), %ymm3, %ymm3;
1421	vpxor (3 * 32)(%r11), %ymm2, %ymm2;
1422	vpxor (4 * 32)(%r11), %ymm4, %ymm4;
1423	vpxor (5 * 32)(%r11), %ymm5, %ymm5;
1424	vpxor (6 * 32)(%r11), %ymm6, %ymm6;
1425	vpxor (7 * 32)(%r11), %ymm7, %ymm7;
1426	vpxor (8 * 32)(%r11), %ymm8, %ymm8;
1427	vpxor (9 * 32)(%r11), %ymm9, %ymm9;
1428	vpxor (10 * 32)(%r11), %ymm10, %ymm10;
1429	vpxor (11 * 32)(%r11), %ymm11, %ymm11;
1430	vpxor (12 * 32)(%r11), %ymm12, %ymm12;
1431	vpxor (13 * 32)(%r11), %ymm13, %ymm13;
1432	vpxor (14 * 32)(%r11), %ymm14, %ymm14;
1433	vpxor (15 * 32)(%r11), %ymm15, %ymm15;
1434	write_output(%ymm1, %ymm0, %ymm3, %ymm2, %ymm4, %ymm5, %ymm6, %ymm7,
1435		     %ymm8, %ymm9, %ymm10, %ymm11, %ymm12, %ymm13, %ymm14,
1436		     %ymm15, %r10);
1437
1438	FRAME_END
1439	RET;
1440SYM_FUNC_END(aria_aesni_avx2_gfni_ctr_crypt_32way)
1441#endif /* CONFIG_AS_GFNI */
1442