1/* SPDX-License-Identifier: GPL-2.0-only */
2/*
3 * Bit sliced AES using NEON instructions
4 *
5 * Copyright (C) 2016 Linaro Ltd <ard.biesheuvel@linaro.org>
6 */
7
8/*
9 * The algorithm implemented here is described in detail by the paper
10 * 'Faster and Timing-Attack Resistant AES-GCM' by Emilia Kaesper and
11 * Peter Schwabe (https://eprint.iacr.org/2009/129.pdf)
12 *
13 * This implementation is based primarily on the OpenSSL implementation
14 * for 32-bit ARM written by Andy Polyakov <appro@openssl.org>
15 */
16
17#include <linux/linkage.h>
18#include <asm/assembler.h>
19
20	.text
21
22	rounds		.req	x11
23	bskey		.req	x12
24
25	.macro		in_bs_ch, b0, b1, b2, b3, b4, b5, b6, b7
26	eor		\b2, \b2, \b1
27	eor		\b5, \b5, \b6
28	eor		\b3, \b3, \b0
29	eor		\b6, \b6, \b2
30	eor		\b5, \b5, \b0
31	eor		\b6, \b6, \b3
32	eor		\b3, \b3, \b7
33	eor		\b7, \b7, \b5
34	eor		\b3, \b3, \b4
35	eor		\b4, \b4, \b5
36	eor		\b2, \b2, \b7
37	eor		\b3, \b3, \b1
38	eor		\b1, \b1, \b5
39	.endm
40
41	.macro		out_bs_ch, b0, b1, b2, b3, b4, b5, b6, b7
42	eor		\b0, \b0, \b6
43	eor		\b1, \b1, \b4
44	eor		\b4, \b4, \b6
45	eor		\b2, \b2, \b0
46	eor		\b6, \b6, \b1
47	eor		\b1, \b1, \b5
48	eor		\b5, \b5, \b3
49	eor		\b3, \b3, \b7
50	eor		\b7, \b7, \b5
51	eor		\b2, \b2, \b5
52	eor		\b4, \b4, \b7
53	.endm
54
55	.macro		inv_in_bs_ch, b6, b1, b2, b4, b7, b0, b3, b5
56	eor		\b1, \b1, \b7
57	eor		\b4, \b4, \b7
58	eor		\b7, \b7, \b5
59	eor		\b1, \b1, \b3
60	eor		\b2, \b2, \b5
61	eor		\b3, \b3, \b7
62	eor		\b6, \b6, \b1
63	eor		\b2, \b2, \b0
64	eor		\b5, \b5, \b3
65	eor		\b4, \b4, \b6
66	eor		\b0, \b0, \b6
67	eor		\b1, \b1, \b4
68	.endm
69
70	.macro		inv_out_bs_ch, b6, b5, b0, b3, b7, b1, b4, b2
71	eor		\b1, \b1, \b5
72	eor		\b2, \b2, \b7
73	eor		\b3, \b3, \b1
74	eor		\b4, \b4, \b5
75	eor		\b7, \b7, \b5
76	eor		\b3, \b3, \b4
77	eor 		\b5, \b5, \b0
78	eor		\b3, \b3, \b7
79	eor		\b6, \b6, \b2
80	eor		\b2, \b2, \b1
81	eor		\b6, \b6, \b3
82	eor		\b3, \b3, \b0
83	eor		\b5, \b5, \b6
84	.endm
85
86	.macro		mul_gf4, x0, x1, y0, y1, t0, t1
87	eor 		\t0, \y0, \y1
88	and		\t0, \t0, \x0
89	eor		\x0, \x0, \x1
90	and		\t1, \x1, \y0
91	and		\x0, \x0, \y1
92	eor		\x1, \t1, \t0
93	eor		\x0, \x0, \t1
94	.endm
95
96	.macro		mul_gf4_n_gf4, x0, x1, y0, y1, t0, x2, x3, y2, y3, t1
97	eor		\t0, \y0, \y1
98	eor 		\t1, \y2, \y3
99	and		\t0, \t0, \x0
100	and		\t1, \t1, \x2
101	eor		\x0, \x0, \x1
102	eor		\x2, \x2, \x3
103	and		\x1, \x1, \y0
104	and		\x3, \x3, \y2
105	and		\x0, \x0, \y1
106	and		\x2, \x2, \y3
107	eor		\x1, \x1, \x0
108	eor		\x2, \x2, \x3
109	eor		\x0, \x0, \t0
110	eor		\x3, \x3, \t1
111	.endm
112
113	.macro		mul_gf16_2, x0, x1, x2, x3, x4, x5, x6, x7, \
114				    y0, y1, y2, y3, t0, t1, t2, t3
115	eor		\t0, \x0, \x2
116	eor		\t1, \x1, \x3
117	mul_gf4  	\x0, \x1, \y0, \y1, \t2, \t3
118	eor		\y0, \y0, \y2
119	eor		\y1, \y1, \y3
120	mul_gf4_n_gf4	\t0, \t1, \y0, \y1, \t3, \x2, \x3, \y2, \y3, \t2
121	eor		\x0, \x0, \t0
122	eor		\x2, \x2, \t0
123	eor		\x1, \x1, \t1
124	eor		\x3, \x3, \t1
125	eor		\t0, \x4, \x6
126	eor		\t1, \x5, \x7
127	mul_gf4_n_gf4	\t0, \t1, \y0, \y1, \t3, \x6, \x7, \y2, \y3, \t2
128	eor		\y0, \y0, \y2
129	eor		\y1, \y1, \y3
130	mul_gf4  	\x4, \x5, \y0, \y1, \t2, \t3
131	eor		\x4, \x4, \t0
132	eor		\x6, \x6, \t0
133	eor		\x5, \x5, \t1
134	eor		\x7, \x7, \t1
135	.endm
136
137	.macro		inv_gf256, x0, x1, x2, x3, x4, x5, x6, x7, \
138				   t0, t1, t2, t3, s0, s1, s2, s3
139	eor		\t3, \x4, \x6
140	eor		\t0, \x5, \x7
141	eor		\t1, \x1, \x3
142	eor		\s1, \x7, \x6
143	eor		\s0, \x0, \x2
144	eor		\s3, \t3, \t0
145	orr		\t2, \t0, \t1
146	and		\s2, \t3, \s0
147	orr		\t3, \t3, \s0
148	eor		\s0, \s0, \t1
149	and		\t0, \t0, \t1
150	eor		\t1, \x3, \x2
151	and		\s3, \s3, \s0
152	and		\s1, \s1, \t1
153	eor		\t1, \x4, \x5
154	eor		\s0, \x1, \x0
155	eor		\t3, \t3, \s1
156	eor		\t2, \t2, \s1
157	and		\s1, \t1, \s0
158	orr		\t1, \t1, \s0
159	eor		\t3, \t3, \s3
160	eor		\t0, \t0, \s1
161	eor		\t2, \t2, \s2
162	eor		\t1, \t1, \s3
163	eor		\t0, \t0, \s2
164	and		\s0, \x7, \x3
165	eor		\t1, \t1, \s2
166	and		\s1, \x6, \x2
167	and		\s2, \x5, \x1
168	orr		\s3, \x4, \x0
169	eor		\t3, \t3, \s0
170	eor		\t1, \t1, \s2
171	eor		\s0, \t0, \s3
172	eor		\t2, \t2, \s1
173	and		\s2, \t3, \t1
174	eor		\s1, \t2, \s2
175	eor		\s3, \s0, \s2
176	bsl		\s1, \t1, \s0
177	not		\t0, \s0
178	bsl		\s0, \s1, \s3
179	bsl		\t0, \s1, \s3
180	bsl		\s3, \t3, \t2
181	eor		\t3, \t3, \t2
182	and		\s2, \s0, \s3
183	eor		\t1, \t1, \t0
184	eor		\s2, \s2, \t3
185	mul_gf16_2	\x0, \x1, \x2, \x3, \x4, \x5, \x6, \x7, \
186			\s3, \s2, \s1, \t1, \s0, \t0, \t2, \t3
187	.endm
188
189	.macro		sbox, b0, b1, b2, b3, b4, b5, b6, b7, \
190			      t0, t1, t2, t3, s0, s1, s2, s3
191	in_bs_ch	\b0\().16b, \b1\().16b, \b2\().16b, \b3\().16b, \
192			\b4\().16b, \b5\().16b, \b6\().16b, \b7\().16b
193	inv_gf256	\b6\().16b, \b5\().16b, \b0\().16b, \b3\().16b, \
194			\b7\().16b, \b1\().16b, \b4\().16b, \b2\().16b, \
195			\t0\().16b, \t1\().16b, \t2\().16b, \t3\().16b, \
196			\s0\().16b, \s1\().16b, \s2\().16b, \s3\().16b
197	out_bs_ch	\b7\().16b, \b1\().16b, \b4\().16b, \b2\().16b, \
198			\b6\().16b, \b5\().16b, \b0\().16b, \b3\().16b
199	.endm
200
201	.macro		inv_sbox, b0, b1, b2, b3, b4, b5, b6, b7, \
202				  t0, t1, t2, t3, s0, s1, s2, s3
203	inv_in_bs_ch	\b0\().16b, \b1\().16b, \b2\().16b, \b3\().16b, \
204			\b4\().16b, \b5\().16b, \b6\().16b, \b7\().16b
205	inv_gf256	\b5\().16b, \b1\().16b, \b2\().16b, \b6\().16b, \
206			\b3\().16b, \b7\().16b, \b0\().16b, \b4\().16b, \
207			\t0\().16b, \t1\().16b, \t2\().16b, \t3\().16b, \
208			\s0\().16b, \s1\().16b, \s2\().16b, \s3\().16b
209	inv_out_bs_ch	\b3\().16b, \b7\().16b, \b0\().16b, \b4\().16b, \
210			\b5\().16b, \b1\().16b, \b2\().16b, \b6\().16b
211	.endm
212
213	.macro		enc_next_rk
214	ldp		q16, q17, [bskey], #128
215	ldp		q18, q19, [bskey, #-96]
216	ldp		q20, q21, [bskey, #-64]
217	ldp		q22, q23, [bskey, #-32]
218	.endm
219
220	.macro		dec_next_rk
221	ldp		q16, q17, [bskey, #-128]!
222	ldp		q18, q19, [bskey, #32]
223	ldp		q20, q21, [bskey, #64]
224	ldp		q22, q23, [bskey, #96]
225	.endm
226
227	.macro		add_round_key, x0, x1, x2, x3, x4, x5, x6, x7
228	eor		\x0\().16b, \x0\().16b, v16.16b
229	eor		\x1\().16b, \x1\().16b, v17.16b
230	eor		\x2\().16b, \x2\().16b, v18.16b
231	eor		\x3\().16b, \x3\().16b, v19.16b
232	eor		\x4\().16b, \x4\().16b, v20.16b
233	eor		\x5\().16b, \x5\().16b, v21.16b
234	eor		\x6\().16b, \x6\().16b, v22.16b
235	eor		\x7\().16b, \x7\().16b, v23.16b
236	.endm
237
238	.macro		shift_rows, x0, x1, x2, x3, x4, x5, x6, x7, mask
239	tbl		\x0\().16b, {\x0\().16b}, \mask\().16b
240	tbl		\x1\().16b, {\x1\().16b}, \mask\().16b
241	tbl		\x2\().16b, {\x2\().16b}, \mask\().16b
242	tbl		\x3\().16b, {\x3\().16b}, \mask\().16b
243	tbl		\x4\().16b, {\x4\().16b}, \mask\().16b
244	tbl		\x5\().16b, {\x5\().16b}, \mask\().16b
245	tbl		\x6\().16b, {\x6\().16b}, \mask\().16b
246	tbl		\x7\().16b, {\x7\().16b}, \mask\().16b
247	.endm
248
249	.macro		mix_cols, x0, x1, x2, x3, x4, x5, x6, x7, \
250				  t0, t1, t2, t3, t4, t5, t6, t7, inv
251	ext		\t0\().16b, \x0\().16b, \x0\().16b, #12
252	ext		\t1\().16b, \x1\().16b, \x1\().16b, #12
253	eor		\x0\().16b, \x0\().16b, \t0\().16b
254	ext		\t2\().16b, \x2\().16b, \x2\().16b, #12
255	eor		\x1\().16b, \x1\().16b, \t1\().16b
256	ext		\t3\().16b, \x3\().16b, \x3\().16b, #12
257	eor		\x2\().16b, \x2\().16b, \t2\().16b
258	ext		\t4\().16b, \x4\().16b, \x4\().16b, #12
259	eor		\x3\().16b, \x3\().16b, \t3\().16b
260	ext		\t5\().16b, \x5\().16b, \x5\().16b, #12
261	eor		\x4\().16b, \x4\().16b, \t4\().16b
262	ext		\t6\().16b, \x6\().16b, \x6\().16b, #12
263	eor		\x5\().16b, \x5\().16b, \t5\().16b
264	ext		\t7\().16b, \x7\().16b, \x7\().16b, #12
265	eor		\x6\().16b, \x6\().16b, \t6\().16b
266	eor		\t1\().16b, \t1\().16b, \x0\().16b
267	eor		\x7\().16b, \x7\().16b, \t7\().16b
268	ext		\x0\().16b, \x0\().16b, \x0\().16b, #8
269	eor		\t2\().16b, \t2\().16b, \x1\().16b
270	eor		\t0\().16b, \t0\().16b, \x7\().16b
271	eor		\t1\().16b, \t1\().16b, \x7\().16b
272	ext		\x1\().16b, \x1\().16b, \x1\().16b, #8
273	eor		\t5\().16b, \t5\().16b, \x4\().16b
274	eor		\x0\().16b, \x0\().16b, \t0\().16b
275	eor		\t6\().16b, \t6\().16b, \x5\().16b
276	eor		\x1\().16b, \x1\().16b, \t1\().16b
277	ext		\t0\().16b, \x4\().16b, \x4\().16b, #8
278	eor		\t4\().16b, \t4\().16b, \x3\().16b
279	ext		\t1\().16b, \x5\().16b, \x5\().16b, #8
280	eor		\t7\().16b, \t7\().16b, \x6\().16b
281	ext		\x4\().16b, \x3\().16b, \x3\().16b, #8
282	eor		\t3\().16b, \t3\().16b, \x2\().16b
283	ext		\x5\().16b, \x7\().16b, \x7\().16b, #8
284	eor		\t4\().16b, \t4\().16b, \x7\().16b
285	ext		\x3\().16b, \x6\().16b, \x6\().16b, #8
286	eor		\t3\().16b, \t3\().16b, \x7\().16b
287	ext		\x6\().16b, \x2\().16b, \x2\().16b, #8
288	eor		\x7\().16b, \t1\().16b, \t5\().16b
289	.ifb		\inv
290	eor		\x2\().16b, \t0\().16b, \t4\().16b
291	eor		\x4\().16b, \x4\().16b, \t3\().16b
292	eor		\x5\().16b, \x5\().16b, \t7\().16b
293	eor		\x3\().16b, \x3\().16b, \t6\().16b
294	eor		\x6\().16b, \x6\().16b, \t2\().16b
295	.else
296	eor		\t3\().16b, \t3\().16b, \x4\().16b
297	eor		\x5\().16b, \x5\().16b, \t7\().16b
298	eor		\x2\().16b, \x3\().16b, \t6\().16b
299	eor		\x3\().16b, \t0\().16b, \t4\().16b
300	eor		\x4\().16b, \x6\().16b, \t2\().16b
301	mov		\x6\().16b, \t3\().16b
302	.endif
303	.endm
304
305	.macro		inv_mix_cols, x0, x1, x2, x3, x4, x5, x6, x7, \
306				      t0, t1, t2, t3, t4, t5, t6, t7
307	ext		\t0\().16b, \x0\().16b, \x0\().16b, #8
308	ext		\t6\().16b, \x6\().16b, \x6\().16b, #8
309	ext		\t7\().16b, \x7\().16b, \x7\().16b, #8
310	eor		\t0\().16b, \t0\().16b, \x0\().16b
311	ext		\t1\().16b, \x1\().16b, \x1\().16b, #8
312	eor		\t6\().16b, \t6\().16b, \x6\().16b
313	ext		\t2\().16b, \x2\().16b, \x2\().16b, #8
314	eor		\t7\().16b, \t7\().16b, \x7\().16b
315	ext		\t3\().16b, \x3\().16b, \x3\().16b, #8
316	eor		\t1\().16b, \t1\().16b, \x1\().16b
317	ext		\t4\().16b, \x4\().16b, \x4\().16b, #8
318	eor		\t2\().16b, \t2\().16b, \x2\().16b
319	ext		\t5\().16b, \x5\().16b, \x5\().16b, #8
320	eor		\t3\().16b, \t3\().16b, \x3\().16b
321	eor		\t4\().16b, \t4\().16b, \x4\().16b
322	eor		\t5\().16b, \t5\().16b, \x5\().16b
323	eor		\x0\().16b, \x0\().16b, \t6\().16b
324	eor		\x1\().16b, \x1\().16b, \t6\().16b
325	eor		\x2\().16b, \x2\().16b, \t0\().16b
326	eor		\x4\().16b, \x4\().16b, \t2\().16b
327	eor		\x3\().16b, \x3\().16b, \t1\().16b
328	eor		\x1\().16b, \x1\().16b, \t7\().16b
329	eor		\x2\().16b, \x2\().16b, \t7\().16b
330	eor		\x4\().16b, \x4\().16b, \t6\().16b
331	eor		\x5\().16b, \x5\().16b, \t3\().16b
332	eor		\x3\().16b, \x3\().16b, \t6\().16b
333	eor		\x6\().16b, \x6\().16b, \t4\().16b
334	eor		\x4\().16b, \x4\().16b, \t7\().16b
335	eor		\x5\().16b, \x5\().16b, \t7\().16b
336	eor		\x7\().16b, \x7\().16b, \t5\().16b
337	mix_cols	\x0, \x1, \x2, \x3, \x4, \x5, \x6, \x7, \
338			\t0, \t1, \t2, \t3, \t4, \t5, \t6, \t7, 1
339	.endm
340
341	.macro		swapmove_2x, a0, b0, a1, b1, n, mask, t0, t1
342	ushr		\t0\().2d, \b0\().2d, #\n
343	ushr		\t1\().2d, \b1\().2d, #\n
344	eor		\t0\().16b, \t0\().16b, \a0\().16b
345	eor		\t1\().16b, \t1\().16b, \a1\().16b
346	and		\t0\().16b, \t0\().16b, \mask\().16b
347	and		\t1\().16b, \t1\().16b, \mask\().16b
348	eor		\a0\().16b, \a0\().16b, \t0\().16b
349	shl		\t0\().2d, \t0\().2d, #\n
350	eor		\a1\().16b, \a1\().16b, \t1\().16b
351	shl		\t1\().2d, \t1\().2d, #\n
352	eor		\b0\().16b, \b0\().16b, \t0\().16b
353	eor		\b1\().16b, \b1\().16b, \t1\().16b
354	.endm
355
356	.macro		bitslice, x7, x6, x5, x4, x3, x2, x1, x0, t0, t1, t2, t3
357	movi		\t0\().16b, #0x55
358	movi		\t1\().16b, #0x33
359	swapmove_2x	\x0, \x1, \x2, \x3, 1, \t0, \t2, \t3
360	swapmove_2x	\x4, \x5, \x6, \x7, 1, \t0, \t2, \t3
361	movi		\t0\().16b, #0x0f
362	swapmove_2x	\x0, \x2, \x1, \x3, 2, \t1, \t2, \t3
363	swapmove_2x	\x4, \x6, \x5, \x7, 2, \t1, \t2, \t3
364	swapmove_2x	\x0, \x4, \x1, \x5, 4, \t0, \t2, \t3
365	swapmove_2x	\x2, \x6, \x3, \x7, 4, \t0, \t2, \t3
366	.endm
367
368
369	.align		6
370M0:	.octa		0x0004080c0105090d02060a0e03070b0f
371
372M0SR:	.octa		0x0004080c05090d010a0e02060f03070b
373SR:	.octa		0x0f0e0d0c0a09080b0504070600030201
374SRM0:	.octa		0x01060b0c0207080d0304090e00050a0f
375
376M0ISR:	.octa		0x0004080c0d0105090a0e0206070b0f03
377ISR:	.octa		0x0f0e0d0c080b0a090504070602010003
378ISRM0:	.octa		0x0306090c00070a0d01040b0e0205080f
379
380	/*
381	 * void aesbs_convert_key(u8 out[], u32 const rk[], int rounds)
382	 */
383SYM_FUNC_START(aesbs_convert_key)
384	ld1		{v7.4s}, [x1], #16		// load round 0 key
385	ld1		{v17.4s}, [x1], #16		// load round 1 key
386
387	movi		v8.16b,  #0x01			// bit masks
388	movi		v9.16b,  #0x02
389	movi		v10.16b, #0x04
390	movi		v11.16b, #0x08
391	movi		v12.16b, #0x10
392	movi		v13.16b, #0x20
393	movi		v14.16b, #0x40
394	movi		v15.16b, #0x80
395	ldr		q16, M0
396
397	sub		x2, x2, #1
398	str		q7, [x0], #16		// save round 0 key
399
400.Lkey_loop:
401	tbl		v7.16b ,{v17.16b}, v16.16b
402	ld1		{v17.4s}, [x1], #16		// load next round key
403
404	cmtst		v0.16b, v7.16b, v8.16b
405	cmtst		v1.16b, v7.16b, v9.16b
406	cmtst		v2.16b, v7.16b, v10.16b
407	cmtst		v3.16b, v7.16b, v11.16b
408	cmtst		v4.16b, v7.16b, v12.16b
409	cmtst		v5.16b, v7.16b, v13.16b
410	cmtst		v6.16b, v7.16b, v14.16b
411	cmtst		v7.16b, v7.16b, v15.16b
412	not		v0.16b, v0.16b
413	not		v1.16b, v1.16b
414	not		v5.16b, v5.16b
415	not		v6.16b, v6.16b
416
417	subs		x2, x2, #1
418	stp		q0, q1, [x0], #128
419	stp		q2, q3, [x0, #-96]
420	stp		q4, q5, [x0, #-64]
421	stp		q6, q7, [x0, #-32]
422	b.ne		.Lkey_loop
423
424	movi		v7.16b, #0x63			// compose .L63
425	eor		v17.16b, v17.16b, v7.16b
426	str		q17, [x0]
427	ret
428SYM_FUNC_END(aesbs_convert_key)
429
430	.align		4
431SYM_FUNC_START_LOCAL(aesbs_encrypt8)
432	ldr		q9, [bskey], #16		// round 0 key
433	ldr		q8, M0SR
434	ldr		q24, SR
435
436	eor		v10.16b, v0.16b, v9.16b		// xor with round0 key
437	eor		v11.16b, v1.16b, v9.16b
438	tbl		v0.16b, {v10.16b}, v8.16b
439	eor		v12.16b, v2.16b, v9.16b
440	tbl		v1.16b, {v11.16b}, v8.16b
441	eor		v13.16b, v3.16b, v9.16b
442	tbl		v2.16b, {v12.16b}, v8.16b
443	eor		v14.16b, v4.16b, v9.16b
444	tbl		v3.16b, {v13.16b}, v8.16b
445	eor		v15.16b, v5.16b, v9.16b
446	tbl		v4.16b, {v14.16b}, v8.16b
447	eor		v10.16b, v6.16b, v9.16b
448	tbl		v5.16b, {v15.16b}, v8.16b
449	eor		v11.16b, v7.16b, v9.16b
450	tbl		v6.16b, {v10.16b}, v8.16b
451	tbl		v7.16b, {v11.16b}, v8.16b
452
453	bitslice	v0, v1, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11
454
455	sub		rounds, rounds, #1
456	b		.Lenc_sbox
457
458.Lenc_loop:
459	shift_rows	v0, v1, v2, v3, v4, v5, v6, v7, v24
460.Lenc_sbox:
461	sbox		v0, v1, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, \
462								v13, v14, v15
463	subs		rounds, rounds, #1
464	b.cc		.Lenc_done
465
466	enc_next_rk
467
468	mix_cols	v0, v1, v4, v6, v3, v7, v2, v5, v8, v9, v10, v11, v12, \
469								v13, v14, v15
470
471	add_round_key	v0, v1, v2, v3, v4, v5, v6, v7
472
473	b.ne		.Lenc_loop
474	ldr		q24, SRM0
475	b		.Lenc_loop
476
477.Lenc_done:
478	ldr		q12, [bskey]			// last round key
479
480	bitslice	v0, v1, v4, v6, v3, v7, v2, v5, v8, v9, v10, v11
481
482	eor		v0.16b, v0.16b, v12.16b
483	eor		v1.16b, v1.16b, v12.16b
484	eor		v4.16b, v4.16b, v12.16b
485	eor		v6.16b, v6.16b, v12.16b
486	eor		v3.16b, v3.16b, v12.16b
487	eor		v7.16b, v7.16b, v12.16b
488	eor		v2.16b, v2.16b, v12.16b
489	eor		v5.16b, v5.16b, v12.16b
490	ret
491SYM_FUNC_END(aesbs_encrypt8)
492
493	.align		4
494SYM_FUNC_START_LOCAL(aesbs_decrypt8)
495	lsl		x9, rounds, #7
496	add		bskey, bskey, x9
497
498	ldr		q9, [bskey, #-112]!		// round 0 key
499	ldr		q8, M0ISR
500	ldr		q24, ISR
501
502	eor		v10.16b, v0.16b, v9.16b		// xor with round0 key
503	eor		v11.16b, v1.16b, v9.16b
504	tbl		v0.16b, {v10.16b}, v8.16b
505	eor		v12.16b, v2.16b, v9.16b
506	tbl		v1.16b, {v11.16b}, v8.16b
507	eor		v13.16b, v3.16b, v9.16b
508	tbl		v2.16b, {v12.16b}, v8.16b
509	eor		v14.16b, v4.16b, v9.16b
510	tbl		v3.16b, {v13.16b}, v8.16b
511	eor		v15.16b, v5.16b, v9.16b
512	tbl		v4.16b, {v14.16b}, v8.16b
513	eor		v10.16b, v6.16b, v9.16b
514	tbl		v5.16b, {v15.16b}, v8.16b
515	eor		v11.16b, v7.16b, v9.16b
516	tbl		v6.16b, {v10.16b}, v8.16b
517	tbl		v7.16b, {v11.16b}, v8.16b
518
519	bitslice	v0, v1, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11
520
521	sub		rounds, rounds, #1
522	b		.Ldec_sbox
523
524.Ldec_loop:
525	shift_rows	v0, v1, v2, v3, v4, v5, v6, v7, v24
526.Ldec_sbox:
527	inv_sbox	v0, v1, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, \
528								v13, v14, v15
529	subs		rounds, rounds, #1
530	b.cc		.Ldec_done
531
532	dec_next_rk
533
534	add_round_key	v0, v1, v6, v4, v2, v7, v3, v5
535
536	inv_mix_cols	v0, v1, v6, v4, v2, v7, v3, v5, v8, v9, v10, v11, v12, \
537								v13, v14, v15
538
539	b.ne		.Ldec_loop
540	ldr		q24, ISRM0
541	b		.Ldec_loop
542.Ldec_done:
543	ldr		q12, [bskey, #-16]		// last round key
544
545	bitslice	v0, v1, v6, v4, v2, v7, v3, v5, v8, v9, v10, v11
546
547	eor		v0.16b, v0.16b, v12.16b
548	eor		v1.16b, v1.16b, v12.16b
549	eor		v6.16b, v6.16b, v12.16b
550	eor		v4.16b, v4.16b, v12.16b
551	eor		v2.16b, v2.16b, v12.16b
552	eor		v7.16b, v7.16b, v12.16b
553	eor		v3.16b, v3.16b, v12.16b
554	eor		v5.16b, v5.16b, v12.16b
555	ret
556SYM_FUNC_END(aesbs_decrypt8)
557
558	/*
559	 * aesbs_ecb_encrypt(u8 out[], u8 const in[], u8 const rk[], int rounds,
560	 *		     int blocks)
561	 * aesbs_ecb_decrypt(u8 out[], u8 const in[], u8 const rk[], int rounds,
562	 *		     int blocks)
563	 */
564	.macro		__ecb_crypt, do8, o0, o1, o2, o3, o4, o5, o6, o7
565	frame_push	5
566
567	mov		x19, x0
568	mov		x20, x1
569	mov		x21, x2
570	mov		x22, x3
571	mov		x23, x4
572
57399:	mov		x5, #1
574	lsl		x5, x5, x23
575	subs		w23, w23, #8
576	csel		x23, x23, xzr, pl
577	csel		x5, x5, xzr, mi
578
579	ld1		{v0.16b}, [x20], #16
580	tbnz		x5, #1, 0f
581	ld1		{v1.16b}, [x20], #16
582	tbnz		x5, #2, 0f
583	ld1		{v2.16b}, [x20], #16
584	tbnz		x5, #3, 0f
585	ld1		{v3.16b}, [x20], #16
586	tbnz		x5, #4, 0f
587	ld1		{v4.16b}, [x20], #16
588	tbnz		x5, #5, 0f
589	ld1		{v5.16b}, [x20], #16
590	tbnz		x5, #6, 0f
591	ld1		{v6.16b}, [x20], #16
592	tbnz		x5, #7, 0f
593	ld1		{v7.16b}, [x20], #16
594
5950:	mov		bskey, x21
596	mov		rounds, x22
597	bl		\do8
598
599	st1		{\o0\().16b}, [x19], #16
600	tbnz		x5, #1, 1f
601	st1		{\o1\().16b}, [x19], #16
602	tbnz		x5, #2, 1f
603	st1		{\o2\().16b}, [x19], #16
604	tbnz		x5, #3, 1f
605	st1		{\o3\().16b}, [x19], #16
606	tbnz		x5, #4, 1f
607	st1		{\o4\().16b}, [x19], #16
608	tbnz		x5, #5, 1f
609	st1		{\o5\().16b}, [x19], #16
610	tbnz		x5, #6, 1f
611	st1		{\o6\().16b}, [x19], #16
612	tbnz		x5, #7, 1f
613	st1		{\o7\().16b}, [x19], #16
614
615	cbz		x23, 1f
616	cond_yield_neon
617	b		99b
618
6191:	frame_pop
620	ret
621	.endm
622
623	.align		4
624SYM_FUNC_START(aesbs_ecb_encrypt)
625	__ecb_crypt	aesbs_encrypt8, v0, v1, v4, v6, v3, v7, v2, v5
626SYM_FUNC_END(aesbs_ecb_encrypt)
627
628	.align		4
629SYM_FUNC_START(aesbs_ecb_decrypt)
630	__ecb_crypt	aesbs_decrypt8, v0, v1, v6, v4, v2, v7, v3, v5
631SYM_FUNC_END(aesbs_ecb_decrypt)
632
633	/*
634	 * aesbs_cbc_decrypt(u8 out[], u8 const in[], u8 const rk[], int rounds,
635	 *		     int blocks, u8 iv[])
636	 */
637	.align		4
638SYM_FUNC_START(aesbs_cbc_decrypt)
639	frame_push	6
640
641	mov		x19, x0
642	mov		x20, x1
643	mov		x21, x2
644	mov		x22, x3
645	mov		x23, x4
646	mov		x24, x5
647
64899:	mov		x6, #1
649	lsl		x6, x6, x23
650	subs		w23, w23, #8
651	csel		x23, x23, xzr, pl
652	csel		x6, x6, xzr, mi
653
654	ld1		{v0.16b}, [x20], #16
655	mov		v25.16b, v0.16b
656	tbnz		x6, #1, 0f
657	ld1		{v1.16b}, [x20], #16
658	mov		v26.16b, v1.16b
659	tbnz		x6, #2, 0f
660	ld1		{v2.16b}, [x20], #16
661	mov		v27.16b, v2.16b
662	tbnz		x6, #3, 0f
663	ld1		{v3.16b}, [x20], #16
664	mov		v28.16b, v3.16b
665	tbnz		x6, #4, 0f
666	ld1		{v4.16b}, [x20], #16
667	mov		v29.16b, v4.16b
668	tbnz		x6, #5, 0f
669	ld1		{v5.16b}, [x20], #16
670	mov		v30.16b, v5.16b
671	tbnz		x6, #6, 0f
672	ld1		{v6.16b}, [x20], #16
673	mov		v31.16b, v6.16b
674	tbnz		x6, #7, 0f
675	ld1		{v7.16b}, [x20]
676
6770:	mov		bskey, x21
678	mov		rounds, x22
679	bl		aesbs_decrypt8
680
681	ld1		{v24.16b}, [x24]		// load IV
682
683	eor		v1.16b, v1.16b, v25.16b
684	eor		v6.16b, v6.16b, v26.16b
685	eor		v4.16b, v4.16b, v27.16b
686	eor		v2.16b, v2.16b, v28.16b
687	eor		v7.16b, v7.16b, v29.16b
688	eor		v0.16b, v0.16b, v24.16b
689	eor		v3.16b, v3.16b, v30.16b
690	eor		v5.16b, v5.16b, v31.16b
691
692	st1		{v0.16b}, [x19], #16
693	mov		v24.16b, v25.16b
694	tbnz		x6, #1, 1f
695	st1		{v1.16b}, [x19], #16
696	mov		v24.16b, v26.16b
697	tbnz		x6, #2, 1f
698	st1		{v6.16b}, [x19], #16
699	mov		v24.16b, v27.16b
700	tbnz		x6, #3, 1f
701	st1		{v4.16b}, [x19], #16
702	mov		v24.16b, v28.16b
703	tbnz		x6, #4, 1f
704	st1		{v2.16b}, [x19], #16
705	mov		v24.16b, v29.16b
706	tbnz		x6, #5, 1f
707	st1		{v7.16b}, [x19], #16
708	mov		v24.16b, v30.16b
709	tbnz		x6, #6, 1f
710	st1		{v3.16b}, [x19], #16
711	mov		v24.16b, v31.16b
712	tbnz		x6, #7, 1f
713	ld1		{v24.16b}, [x20], #16
714	st1		{v5.16b}, [x19], #16
7151:	st1		{v24.16b}, [x24]		// store IV
716
717	cbz		x23, 2f
718	cond_yield_neon
719	b		99b
720
7212:	frame_pop
722	ret
723SYM_FUNC_END(aesbs_cbc_decrypt)
724
725	.macro		next_tweak, out, in, const, tmp
726	sshr		\tmp\().2d,  \in\().2d,   #63
727	and		\tmp\().16b, \tmp\().16b, \const\().16b
728	add		\out\().2d,  \in\().2d,   \in\().2d
729	ext		\tmp\().16b, \tmp\().16b, \tmp\().16b, #8
730	eor		\out\().16b, \out\().16b, \tmp\().16b
731	.endm
732
733	/*
734	 * aesbs_xts_encrypt(u8 out[], u8 const in[], u8 const rk[], int rounds,
735	 *		     int blocks, u8 iv[])
736	 * aesbs_xts_decrypt(u8 out[], u8 const in[], u8 const rk[], int rounds,
737	 *		     int blocks, u8 iv[])
738	 */
739SYM_FUNC_START_LOCAL(__xts_crypt8)
740	mov		x6, #1
741	lsl		x6, x6, x23
742	subs		w23, w23, #8
743	csel		x23, x23, xzr, pl
744	csel		x6, x6, xzr, mi
745
746	ld1		{v0.16b}, [x20], #16
747	next_tweak	v26, v25, v30, v31
748	eor		v0.16b, v0.16b, v25.16b
749	tbnz		x6, #1, 0f
750
751	ld1		{v1.16b}, [x20], #16
752	next_tweak	v27, v26, v30, v31
753	eor		v1.16b, v1.16b, v26.16b
754	tbnz		x6, #2, 0f
755
756	ld1		{v2.16b}, [x20], #16
757	next_tweak	v28, v27, v30, v31
758	eor		v2.16b, v2.16b, v27.16b
759	tbnz		x6, #3, 0f
760
761	ld1		{v3.16b}, [x20], #16
762	next_tweak	v29, v28, v30, v31
763	eor		v3.16b, v3.16b, v28.16b
764	tbnz		x6, #4, 0f
765
766	ld1		{v4.16b}, [x20], #16
767	str		q29, [sp, #.Lframe_local_offset]
768	eor		v4.16b, v4.16b, v29.16b
769	next_tweak	v29, v29, v30, v31
770	tbnz		x6, #5, 0f
771
772	ld1		{v5.16b}, [x20], #16
773	str		q29, [sp, #.Lframe_local_offset + 16]
774	eor		v5.16b, v5.16b, v29.16b
775	next_tweak	v29, v29, v30, v31
776	tbnz		x6, #6, 0f
777
778	ld1		{v6.16b}, [x20], #16
779	str		q29, [sp, #.Lframe_local_offset + 32]
780	eor		v6.16b, v6.16b, v29.16b
781	next_tweak	v29, v29, v30, v31
782	tbnz		x6, #7, 0f
783
784	ld1		{v7.16b}, [x20], #16
785	str		q29, [sp, #.Lframe_local_offset + 48]
786	eor		v7.16b, v7.16b, v29.16b
787	next_tweak	v29, v29, v30, v31
788
7890:	mov		bskey, x21
790	mov		rounds, x22
791	br		x7
792SYM_FUNC_END(__xts_crypt8)
793
794	.macro		__xts_crypt, do8, o0, o1, o2, o3, o4, o5, o6, o7
795	frame_push	6, 64
796
797	mov		x19, x0
798	mov		x20, x1
799	mov		x21, x2
800	mov		x22, x3
801	mov		x23, x4
802	mov		x24, x5
803
8040:	movi		v30.2s, #0x1
805	movi		v25.2s, #0x87
806	uzp1		v30.4s, v30.4s, v25.4s
807	ld1		{v25.16b}, [x24]
808
80999:	adr		x7, \do8
810	bl		__xts_crypt8
811
812	ldp		q16, q17, [sp, #.Lframe_local_offset]
813	ldp		q18, q19, [sp, #.Lframe_local_offset + 32]
814
815	eor		\o0\().16b, \o0\().16b, v25.16b
816	eor		\o1\().16b, \o1\().16b, v26.16b
817	eor		\o2\().16b, \o2\().16b, v27.16b
818	eor		\o3\().16b, \o3\().16b, v28.16b
819
820	st1		{\o0\().16b}, [x19], #16
821	mov		v25.16b, v26.16b
822	tbnz		x6, #1, 1f
823	st1		{\o1\().16b}, [x19], #16
824	mov		v25.16b, v27.16b
825	tbnz		x6, #2, 1f
826	st1		{\o2\().16b}, [x19], #16
827	mov		v25.16b, v28.16b
828	tbnz		x6, #3, 1f
829	st1		{\o3\().16b}, [x19], #16
830	mov		v25.16b, v29.16b
831	tbnz		x6, #4, 1f
832
833	eor		\o4\().16b, \o4\().16b, v16.16b
834	eor		\o5\().16b, \o5\().16b, v17.16b
835	eor		\o6\().16b, \o6\().16b, v18.16b
836	eor		\o7\().16b, \o7\().16b, v19.16b
837
838	st1		{\o4\().16b}, [x19], #16
839	tbnz		x6, #5, 1f
840	st1		{\o5\().16b}, [x19], #16
841	tbnz		x6, #6, 1f
842	st1		{\o6\().16b}, [x19], #16
843	tbnz		x6, #7, 1f
844	st1		{\o7\().16b}, [x19], #16
845
846	cbz		x23, 1f
847	st1		{v25.16b}, [x24]
848
849	cond_yield_neon	0b
850	b		99b
851
8521:	st1		{v25.16b}, [x24]
853	frame_pop
854	ret
855	.endm
856
857SYM_FUNC_START(aesbs_xts_encrypt)
858	__xts_crypt	aesbs_encrypt8, v0, v1, v4, v6, v3, v7, v2, v5
859SYM_FUNC_END(aesbs_xts_encrypt)
860
861SYM_FUNC_START(aesbs_xts_decrypt)
862	__xts_crypt	aesbs_decrypt8, v0, v1, v6, v4, v2, v7, v3, v5
863SYM_FUNC_END(aesbs_xts_decrypt)
864
865	.macro		next_ctr, v
866	mov		\v\().d[1], x8
867	adds		x8, x8, #1
868	mov		\v\().d[0], x7
869	adc		x7, x7, xzr
870	rev64		\v\().16b, \v\().16b
871	.endm
872
873	/*
874	 * aesbs_ctr_encrypt(u8 out[], u8 const in[], u8 const rk[],
875	 *		     int rounds, int blocks, u8 iv[], u8 final[])
876	 */
877SYM_FUNC_START(aesbs_ctr_encrypt)
878	frame_push	8
879
880	mov		x19, x0
881	mov		x20, x1
882	mov		x21, x2
883	mov		x22, x3
884	mov		x23, x4
885	mov		x24, x5
886	mov		x25, x6
887
888	cmp		x25, #0
889	cset		x26, ne
890	add		x23, x23, x26		// do one extra block if final
891
89298:	ldp		x7, x8, [x24]
893	ld1		{v0.16b}, [x24]
894CPU_LE(	rev		x7, x7		)
895CPU_LE(	rev		x8, x8		)
896	adds		x8, x8, #1
897	adc		x7, x7, xzr
898
89999:	mov		x9, #1
900	lsl		x9, x9, x23
901	subs		w23, w23, #8
902	csel		x23, x23, xzr, pl
903	csel		x9, x9, xzr, le
904
905	tbnz		x9, #1, 0f
906	next_ctr	v1
907	tbnz		x9, #2, 0f
908	next_ctr	v2
909	tbnz		x9, #3, 0f
910	next_ctr	v3
911	tbnz		x9, #4, 0f
912	next_ctr	v4
913	tbnz		x9, #5, 0f
914	next_ctr	v5
915	tbnz		x9, #6, 0f
916	next_ctr	v6
917	tbnz		x9, #7, 0f
918	next_ctr	v7
919
9200:	mov		bskey, x21
921	mov		rounds, x22
922	bl		aesbs_encrypt8
923
924	lsr		x9, x9, x26		// disregard the extra block
925	tbnz		x9, #0, 0f
926
927	ld1		{v8.16b}, [x20], #16
928	eor		v0.16b, v0.16b, v8.16b
929	st1		{v0.16b}, [x19], #16
930	tbnz		x9, #1, 1f
931
932	ld1		{v9.16b}, [x20], #16
933	eor		v1.16b, v1.16b, v9.16b
934	st1		{v1.16b}, [x19], #16
935	tbnz		x9, #2, 2f
936
937	ld1		{v10.16b}, [x20], #16
938	eor		v4.16b, v4.16b, v10.16b
939	st1		{v4.16b}, [x19], #16
940	tbnz		x9, #3, 3f
941
942	ld1		{v11.16b}, [x20], #16
943	eor		v6.16b, v6.16b, v11.16b
944	st1		{v6.16b}, [x19], #16
945	tbnz		x9, #4, 4f
946
947	ld1		{v12.16b}, [x20], #16
948	eor		v3.16b, v3.16b, v12.16b
949	st1		{v3.16b}, [x19], #16
950	tbnz		x9, #5, 5f
951
952	ld1		{v13.16b}, [x20], #16
953	eor		v7.16b, v7.16b, v13.16b
954	st1		{v7.16b}, [x19], #16
955	tbnz		x9, #6, 6f
956
957	ld1		{v14.16b}, [x20], #16
958	eor		v2.16b, v2.16b, v14.16b
959	st1		{v2.16b}, [x19], #16
960	tbnz		x9, #7, 7f
961
962	ld1		{v15.16b}, [x20], #16
963	eor		v5.16b, v5.16b, v15.16b
964	st1		{v5.16b}, [x19], #16
965
9668:	next_ctr	v0
967	st1		{v0.16b}, [x24]
968	cbz		x23, .Lctr_done
969
970	cond_yield_neon	98b
971	b		99b
972
973.Lctr_done:
974	frame_pop
975	ret
976
977	/*
978	 * If we are handling the tail of the input (x6 != NULL), return the
979	 * final keystream block back to the caller.
980	 */
9810:	cbz		x25, 8b
982	st1		{v0.16b}, [x25]
983	b		8b
9841:	cbz		x25, 8b
985	st1		{v1.16b}, [x25]
986	b		8b
9872:	cbz		x25, 8b
988	st1		{v4.16b}, [x25]
989	b		8b
9903:	cbz		x25, 8b
991	st1		{v6.16b}, [x25]
992	b		8b
9934:	cbz		x25, 8b
994	st1		{v3.16b}, [x25]
995	b		8b
9965:	cbz		x25, 8b
997	st1		{v7.16b}, [x25]
998	b		8b
9996:	cbz		x25, 8b
1000	st1		{v2.16b}, [x25]
1001	b		8b
10027:	cbz		x25, 8b
1003	st1		{v5.16b}, [x25]
1004	b		8b
1005SYM_FUNC_END(aesbs_ctr_encrypt)
1006