1/* SPDX-License-Identifier: GPL-2.0-only */
2/*
3 * AES-NI + SSE2 implementation of AEGIS-128
4 *
5 * Copyright (c) 2017-2018 Ondrej Mosnacek <omosnacek@gmail.com>
6 * Copyright (C) 2017-2018 Red Hat, Inc. All rights reserved.
7 */
8
9#include <linux/linkage.h>
10#include <linux/cfi_types.h>
11#include <asm/frame.h>
12
13#define STATE0	%xmm0
14#define STATE1	%xmm1
15#define STATE2	%xmm2
16#define STATE3	%xmm3
17#define STATE4	%xmm4
18#define KEY	%xmm5
19#define MSG	%xmm5
20#define T0	%xmm6
21#define T1	%xmm7
22
23#define STATEP	%rdi
24#define LEN	%rsi
25#define SRC	%rdx
26#define DST	%rcx
27
28.section .rodata.cst16.aegis128_const, "aM", @progbits, 32
29.align 16
30.Laegis128_const_0:
31	.byte 0x00, 0x01, 0x01, 0x02, 0x03, 0x05, 0x08, 0x0d
32	.byte 0x15, 0x22, 0x37, 0x59, 0x90, 0xe9, 0x79, 0x62
33.Laegis128_const_1:
34	.byte 0xdb, 0x3d, 0x18, 0x55, 0x6d, 0xc2, 0x2f, 0xf1
35	.byte 0x20, 0x11, 0x31, 0x42, 0x73, 0xb5, 0x28, 0xdd
36
37.section .rodata.cst16.aegis128_counter, "aM", @progbits, 16
38.align 16
39.Laegis128_counter:
40	.byte 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07
41	.byte 0x08, 0x09, 0x0a, 0x0b, 0x0c, 0x0d, 0x0e, 0x0f
42
43.text
44
45/*
46 * aegis128_update
47 * input:
48 *   STATE[0-4] - input state
49 * output:
50 *   STATE[0-4] - output state (shifted positions)
51 * changed:
52 *   T0
53 */
54.macro aegis128_update
55	movdqa STATE4, T0
56	aesenc STATE0, STATE4
57	aesenc STATE1, STATE0
58	aesenc STATE2, STATE1
59	aesenc STATE3, STATE2
60	aesenc T0,     STATE3
61.endm
62
63/*
64 * __load_partial: internal ABI
65 * input:
66 *   LEN - bytes
67 *   SRC - src
68 * output:
69 *   MSG  - message block
70 * changed:
71 *   T0
72 *   %r8
73 *   %r9
74 */
75SYM_FUNC_START_LOCAL(__load_partial)
76	xor %r9d, %r9d
77	pxor MSG, MSG
78
79	mov LEN, %r8
80	and $0x1, %r8
81	jz .Lld_partial_1
82
83	mov LEN, %r8
84	and $0x1E, %r8
85	add SRC, %r8
86	mov (%r8), %r9b
87
88.Lld_partial_1:
89	mov LEN, %r8
90	and $0x2, %r8
91	jz .Lld_partial_2
92
93	mov LEN, %r8
94	and $0x1C, %r8
95	add SRC, %r8
96	shl $0x10, %r9
97	mov (%r8), %r9w
98
99.Lld_partial_2:
100	mov LEN, %r8
101	and $0x4, %r8
102	jz .Lld_partial_4
103
104	mov LEN, %r8
105	and $0x18, %r8
106	add SRC, %r8
107	shl $32, %r9
108	mov (%r8), %r8d
109	xor %r8, %r9
110
111.Lld_partial_4:
112	movq %r9, MSG
113
114	mov LEN, %r8
115	and $0x8, %r8
116	jz .Lld_partial_8
117
118	mov LEN, %r8
119	and $0x10, %r8
120	add SRC, %r8
121	pslldq $8, MSG
122	movq (%r8), T0
123	pxor T0, MSG
124
125.Lld_partial_8:
126	RET
127SYM_FUNC_END(__load_partial)
128
129/*
130 * __store_partial: internal ABI
131 * input:
132 *   LEN - bytes
133 *   DST - dst
134 * output:
135 *   T0   - message block
136 * changed:
137 *   %r8
138 *   %r9
139 *   %r10
140 */
141SYM_FUNC_START_LOCAL(__store_partial)
142	mov LEN, %r8
143	mov DST, %r9
144
145	movq T0, %r10
146
147	cmp $8, %r8
148	jl .Lst_partial_8
149
150	mov %r10, (%r9)
151	psrldq $8, T0
152	movq T0, %r10
153
154	sub $8, %r8
155	add $8, %r9
156
157.Lst_partial_8:
158	cmp $4, %r8
159	jl .Lst_partial_4
160
161	mov %r10d, (%r9)
162	shr $32, %r10
163
164	sub $4, %r8
165	add $4, %r9
166
167.Lst_partial_4:
168	cmp $2, %r8
169	jl .Lst_partial_2
170
171	mov %r10w, (%r9)
172	shr $0x10, %r10
173
174	sub $2, %r8
175	add $2, %r9
176
177.Lst_partial_2:
178	cmp $1, %r8
179	jl .Lst_partial_1
180
181	mov %r10b, (%r9)
182
183.Lst_partial_1:
184	RET
185SYM_FUNC_END(__store_partial)
186
187/*
188 * void crypto_aegis128_aesni_init(void *state, const void *key, const void *iv);
189 */
190SYM_FUNC_START(crypto_aegis128_aesni_init)
191	FRAME_BEGIN
192
193	/* load IV: */
194	movdqu (%rdx), T1
195
196	/* load key: */
197	movdqa (%rsi), KEY
198	pxor KEY, T1
199	movdqa T1, STATE0
200	movdqa KEY, STATE3
201	movdqa KEY, STATE4
202
203	/* load the constants: */
204	movdqa .Laegis128_const_0, STATE2
205	movdqa .Laegis128_const_1, STATE1
206	pxor STATE2, STATE3
207	pxor STATE1, STATE4
208
209	/* update 10 times with KEY / KEY xor IV: */
210	aegis128_update; pxor KEY, STATE4
211	aegis128_update; pxor T1,  STATE3
212	aegis128_update; pxor KEY, STATE2
213	aegis128_update; pxor T1,  STATE1
214	aegis128_update; pxor KEY, STATE0
215	aegis128_update; pxor T1,  STATE4
216	aegis128_update; pxor KEY, STATE3
217	aegis128_update; pxor T1,  STATE2
218	aegis128_update; pxor KEY, STATE1
219	aegis128_update; pxor T1,  STATE0
220
221	/* store the state: */
222	movdqu STATE0, 0x00(STATEP)
223	movdqu STATE1, 0x10(STATEP)
224	movdqu STATE2, 0x20(STATEP)
225	movdqu STATE3, 0x30(STATEP)
226	movdqu STATE4, 0x40(STATEP)
227
228	FRAME_END
229	RET
230SYM_FUNC_END(crypto_aegis128_aesni_init)
231
232/*
233 * void crypto_aegis128_aesni_ad(void *state, unsigned int length,
234 *                               const void *data);
235 */
236SYM_FUNC_START(crypto_aegis128_aesni_ad)
237	FRAME_BEGIN
238
239	cmp $0x10, LEN
240	jb .Lad_out
241
242	/* load the state: */
243	movdqu 0x00(STATEP), STATE0
244	movdqu 0x10(STATEP), STATE1
245	movdqu 0x20(STATEP), STATE2
246	movdqu 0x30(STATEP), STATE3
247	movdqu 0x40(STATEP), STATE4
248
249	mov SRC, %r8
250	and $0xF, %r8
251	jnz .Lad_u_loop
252
253.align 8
254.Lad_a_loop:
255	movdqa 0x00(SRC), MSG
256	aegis128_update
257	pxor MSG, STATE4
258	sub $0x10, LEN
259	cmp $0x10, LEN
260	jl .Lad_out_1
261
262	movdqa 0x10(SRC), MSG
263	aegis128_update
264	pxor MSG, STATE3
265	sub $0x10, LEN
266	cmp $0x10, LEN
267	jl .Lad_out_2
268
269	movdqa 0x20(SRC), MSG
270	aegis128_update
271	pxor MSG, STATE2
272	sub $0x10, LEN
273	cmp $0x10, LEN
274	jl .Lad_out_3
275
276	movdqa 0x30(SRC), MSG
277	aegis128_update
278	pxor MSG, STATE1
279	sub $0x10, LEN
280	cmp $0x10, LEN
281	jl .Lad_out_4
282
283	movdqa 0x40(SRC), MSG
284	aegis128_update
285	pxor MSG, STATE0
286	sub $0x10, LEN
287	cmp $0x10, LEN
288	jl .Lad_out_0
289
290	add $0x50, SRC
291	jmp .Lad_a_loop
292
293.align 8
294.Lad_u_loop:
295	movdqu 0x00(SRC), MSG
296	aegis128_update
297	pxor MSG, STATE4
298	sub $0x10, LEN
299	cmp $0x10, LEN
300	jl .Lad_out_1
301
302	movdqu 0x10(SRC), MSG
303	aegis128_update
304	pxor MSG, STATE3
305	sub $0x10, LEN
306	cmp $0x10, LEN
307	jl .Lad_out_2
308
309	movdqu 0x20(SRC), MSG
310	aegis128_update
311	pxor MSG, STATE2
312	sub $0x10, LEN
313	cmp $0x10, LEN
314	jl .Lad_out_3
315
316	movdqu 0x30(SRC), MSG
317	aegis128_update
318	pxor MSG, STATE1
319	sub $0x10, LEN
320	cmp $0x10, LEN
321	jl .Lad_out_4
322
323	movdqu 0x40(SRC), MSG
324	aegis128_update
325	pxor MSG, STATE0
326	sub $0x10, LEN
327	cmp $0x10, LEN
328	jl .Lad_out_0
329
330	add $0x50, SRC
331	jmp .Lad_u_loop
332
333	/* store the state: */
334.Lad_out_0:
335	movdqu STATE0, 0x00(STATEP)
336	movdqu STATE1, 0x10(STATEP)
337	movdqu STATE2, 0x20(STATEP)
338	movdqu STATE3, 0x30(STATEP)
339	movdqu STATE4, 0x40(STATEP)
340	FRAME_END
341	RET
342
343.Lad_out_1:
344	movdqu STATE4, 0x00(STATEP)
345	movdqu STATE0, 0x10(STATEP)
346	movdqu STATE1, 0x20(STATEP)
347	movdqu STATE2, 0x30(STATEP)
348	movdqu STATE3, 0x40(STATEP)
349	FRAME_END
350	RET
351
352.Lad_out_2:
353	movdqu STATE3, 0x00(STATEP)
354	movdqu STATE4, 0x10(STATEP)
355	movdqu STATE0, 0x20(STATEP)
356	movdqu STATE1, 0x30(STATEP)
357	movdqu STATE2, 0x40(STATEP)
358	FRAME_END
359	RET
360
361.Lad_out_3:
362	movdqu STATE2, 0x00(STATEP)
363	movdqu STATE3, 0x10(STATEP)
364	movdqu STATE4, 0x20(STATEP)
365	movdqu STATE0, 0x30(STATEP)
366	movdqu STATE1, 0x40(STATEP)
367	FRAME_END
368	RET
369
370.Lad_out_4:
371	movdqu STATE1, 0x00(STATEP)
372	movdqu STATE2, 0x10(STATEP)
373	movdqu STATE3, 0x20(STATEP)
374	movdqu STATE4, 0x30(STATEP)
375	movdqu STATE0, 0x40(STATEP)
376	FRAME_END
377	RET
378
379.Lad_out:
380	FRAME_END
381	RET
382SYM_FUNC_END(crypto_aegis128_aesni_ad)
383
384.macro encrypt_block a s0 s1 s2 s3 s4 i
385	movdq\a (\i * 0x10)(SRC), MSG
386	movdqa MSG, T0
387	pxor \s1, T0
388	pxor \s4, T0
389	movdqa \s2, T1
390	pand \s3, T1
391	pxor T1, T0
392	movdq\a T0, (\i * 0x10)(DST)
393
394	aegis128_update
395	pxor MSG, \s4
396
397	sub $0x10, LEN
398	cmp $0x10, LEN
399	jl .Lenc_out_\i
400.endm
401
402/*
403 * void crypto_aegis128_aesni_enc(void *state, unsigned int length,
404 *                                const void *src, void *dst);
405 */
406SYM_TYPED_FUNC_START(crypto_aegis128_aesni_enc)
407	FRAME_BEGIN
408
409	cmp $0x10, LEN
410	jb .Lenc_out
411
412	/* load the state: */
413	movdqu 0x00(STATEP), STATE0
414	movdqu 0x10(STATEP), STATE1
415	movdqu 0x20(STATEP), STATE2
416	movdqu 0x30(STATEP), STATE3
417	movdqu 0x40(STATEP), STATE4
418
419	mov  SRC,  %r8
420	or   DST,  %r8
421	and $0xF, %r8
422	jnz .Lenc_u_loop
423
424.align 8
425.Lenc_a_loop:
426	encrypt_block a STATE0 STATE1 STATE2 STATE3 STATE4 0
427	encrypt_block a STATE4 STATE0 STATE1 STATE2 STATE3 1
428	encrypt_block a STATE3 STATE4 STATE0 STATE1 STATE2 2
429	encrypt_block a STATE2 STATE3 STATE4 STATE0 STATE1 3
430	encrypt_block a STATE1 STATE2 STATE3 STATE4 STATE0 4
431
432	add $0x50, SRC
433	add $0x50, DST
434	jmp .Lenc_a_loop
435
436.align 8
437.Lenc_u_loop:
438	encrypt_block u STATE0 STATE1 STATE2 STATE3 STATE4 0
439	encrypt_block u STATE4 STATE0 STATE1 STATE2 STATE3 1
440	encrypt_block u STATE3 STATE4 STATE0 STATE1 STATE2 2
441	encrypt_block u STATE2 STATE3 STATE4 STATE0 STATE1 3
442	encrypt_block u STATE1 STATE2 STATE3 STATE4 STATE0 4
443
444	add $0x50, SRC
445	add $0x50, DST
446	jmp .Lenc_u_loop
447
448	/* store the state: */
449.Lenc_out_0:
450	movdqu STATE4, 0x00(STATEP)
451	movdqu STATE0, 0x10(STATEP)
452	movdqu STATE1, 0x20(STATEP)
453	movdqu STATE2, 0x30(STATEP)
454	movdqu STATE3, 0x40(STATEP)
455	FRAME_END
456	RET
457
458.Lenc_out_1:
459	movdqu STATE3, 0x00(STATEP)
460	movdqu STATE4, 0x10(STATEP)
461	movdqu STATE0, 0x20(STATEP)
462	movdqu STATE1, 0x30(STATEP)
463	movdqu STATE2, 0x40(STATEP)
464	FRAME_END
465	RET
466
467.Lenc_out_2:
468	movdqu STATE2, 0x00(STATEP)
469	movdqu STATE3, 0x10(STATEP)
470	movdqu STATE4, 0x20(STATEP)
471	movdqu STATE0, 0x30(STATEP)
472	movdqu STATE1, 0x40(STATEP)
473	FRAME_END
474	RET
475
476.Lenc_out_3:
477	movdqu STATE1, 0x00(STATEP)
478	movdqu STATE2, 0x10(STATEP)
479	movdqu STATE3, 0x20(STATEP)
480	movdqu STATE4, 0x30(STATEP)
481	movdqu STATE0, 0x40(STATEP)
482	FRAME_END
483	RET
484
485.Lenc_out_4:
486	movdqu STATE0, 0x00(STATEP)
487	movdqu STATE1, 0x10(STATEP)
488	movdqu STATE2, 0x20(STATEP)
489	movdqu STATE3, 0x30(STATEP)
490	movdqu STATE4, 0x40(STATEP)
491	FRAME_END
492	RET
493
494.Lenc_out:
495	FRAME_END
496	RET
497SYM_FUNC_END(crypto_aegis128_aesni_enc)
498
499/*
500 * void crypto_aegis128_aesni_enc_tail(void *state, unsigned int length,
501 *                                     const void *src, void *dst);
502 */
503SYM_TYPED_FUNC_START(crypto_aegis128_aesni_enc_tail)
504	FRAME_BEGIN
505
506	/* load the state: */
507	movdqu 0x00(STATEP), STATE0
508	movdqu 0x10(STATEP), STATE1
509	movdqu 0x20(STATEP), STATE2
510	movdqu 0x30(STATEP), STATE3
511	movdqu 0x40(STATEP), STATE4
512
513	/* encrypt message: */
514	call __load_partial
515
516	movdqa MSG, T0
517	pxor STATE1, T0
518	pxor STATE4, T0
519	movdqa STATE2, T1
520	pand STATE3, T1
521	pxor T1, T0
522
523	call __store_partial
524
525	aegis128_update
526	pxor MSG, STATE4
527
528	/* store the state: */
529	movdqu STATE4, 0x00(STATEP)
530	movdqu STATE0, 0x10(STATEP)
531	movdqu STATE1, 0x20(STATEP)
532	movdqu STATE2, 0x30(STATEP)
533	movdqu STATE3, 0x40(STATEP)
534
535	FRAME_END
536	RET
537SYM_FUNC_END(crypto_aegis128_aesni_enc_tail)
538
539.macro decrypt_block a s0 s1 s2 s3 s4 i
540	movdq\a (\i * 0x10)(SRC), MSG
541	pxor \s1, MSG
542	pxor \s4, MSG
543	movdqa \s2, T1
544	pand \s3, T1
545	pxor T1, MSG
546	movdq\a MSG, (\i * 0x10)(DST)
547
548	aegis128_update
549	pxor MSG, \s4
550
551	sub $0x10, LEN
552	cmp $0x10, LEN
553	jl .Ldec_out_\i
554.endm
555
556/*
557 * void crypto_aegis128_aesni_dec(void *state, unsigned int length,
558 *                                const void *src, void *dst);
559 */
560SYM_TYPED_FUNC_START(crypto_aegis128_aesni_dec)
561	FRAME_BEGIN
562
563	cmp $0x10, LEN
564	jb .Ldec_out
565
566	/* load the state: */
567	movdqu 0x00(STATEP), STATE0
568	movdqu 0x10(STATEP), STATE1
569	movdqu 0x20(STATEP), STATE2
570	movdqu 0x30(STATEP), STATE3
571	movdqu 0x40(STATEP), STATE4
572
573	mov  SRC, %r8
574	or   DST, %r8
575	and $0xF, %r8
576	jnz .Ldec_u_loop
577
578.align 8
579.Ldec_a_loop:
580	decrypt_block a STATE0 STATE1 STATE2 STATE3 STATE4 0
581	decrypt_block a STATE4 STATE0 STATE1 STATE2 STATE3 1
582	decrypt_block a STATE3 STATE4 STATE0 STATE1 STATE2 2
583	decrypt_block a STATE2 STATE3 STATE4 STATE0 STATE1 3
584	decrypt_block a STATE1 STATE2 STATE3 STATE4 STATE0 4
585
586	add $0x50, SRC
587	add $0x50, DST
588	jmp .Ldec_a_loop
589
590.align 8
591.Ldec_u_loop:
592	decrypt_block u STATE0 STATE1 STATE2 STATE3 STATE4 0
593	decrypt_block u STATE4 STATE0 STATE1 STATE2 STATE3 1
594	decrypt_block u STATE3 STATE4 STATE0 STATE1 STATE2 2
595	decrypt_block u STATE2 STATE3 STATE4 STATE0 STATE1 3
596	decrypt_block u STATE1 STATE2 STATE3 STATE4 STATE0 4
597
598	add $0x50, SRC
599	add $0x50, DST
600	jmp .Ldec_u_loop
601
602	/* store the state: */
603.Ldec_out_0:
604	movdqu STATE4, 0x00(STATEP)
605	movdqu STATE0, 0x10(STATEP)
606	movdqu STATE1, 0x20(STATEP)
607	movdqu STATE2, 0x30(STATEP)
608	movdqu STATE3, 0x40(STATEP)
609	FRAME_END
610	RET
611
612.Ldec_out_1:
613	movdqu STATE3, 0x00(STATEP)
614	movdqu STATE4, 0x10(STATEP)
615	movdqu STATE0, 0x20(STATEP)
616	movdqu STATE1, 0x30(STATEP)
617	movdqu STATE2, 0x40(STATEP)
618	FRAME_END
619	RET
620
621.Ldec_out_2:
622	movdqu STATE2, 0x00(STATEP)
623	movdqu STATE3, 0x10(STATEP)
624	movdqu STATE4, 0x20(STATEP)
625	movdqu STATE0, 0x30(STATEP)
626	movdqu STATE1, 0x40(STATEP)
627	FRAME_END
628	RET
629
630.Ldec_out_3:
631	movdqu STATE1, 0x00(STATEP)
632	movdqu STATE2, 0x10(STATEP)
633	movdqu STATE3, 0x20(STATEP)
634	movdqu STATE4, 0x30(STATEP)
635	movdqu STATE0, 0x40(STATEP)
636	FRAME_END
637	RET
638
639.Ldec_out_4:
640	movdqu STATE0, 0x00(STATEP)
641	movdqu STATE1, 0x10(STATEP)
642	movdqu STATE2, 0x20(STATEP)
643	movdqu STATE3, 0x30(STATEP)
644	movdqu STATE4, 0x40(STATEP)
645	FRAME_END
646	RET
647
648.Ldec_out:
649	FRAME_END
650	RET
651SYM_FUNC_END(crypto_aegis128_aesni_dec)
652
653/*
654 * void crypto_aegis128_aesni_dec_tail(void *state, unsigned int length,
655 *                                     const void *src, void *dst);
656 */
657SYM_TYPED_FUNC_START(crypto_aegis128_aesni_dec_tail)
658	FRAME_BEGIN
659
660	/* load the state: */
661	movdqu 0x00(STATEP), STATE0
662	movdqu 0x10(STATEP), STATE1
663	movdqu 0x20(STATEP), STATE2
664	movdqu 0x30(STATEP), STATE3
665	movdqu 0x40(STATEP), STATE4
666
667	/* decrypt message: */
668	call __load_partial
669
670	pxor STATE1, MSG
671	pxor STATE4, MSG
672	movdqa STATE2, T1
673	pand STATE3, T1
674	pxor T1, MSG
675
676	movdqa MSG, T0
677	call __store_partial
678
679	/* mask with byte count: */
680	movq LEN, T0
681	punpcklbw T0, T0
682	punpcklbw T0, T0
683	punpcklbw T0, T0
684	punpcklbw T0, T0
685	movdqa .Laegis128_counter, T1
686	pcmpgtb T1, T0
687	pand T0, MSG
688
689	aegis128_update
690	pxor MSG, STATE4
691
692	/* store the state: */
693	movdqu STATE4, 0x00(STATEP)
694	movdqu STATE0, 0x10(STATEP)
695	movdqu STATE1, 0x20(STATEP)
696	movdqu STATE2, 0x30(STATEP)
697	movdqu STATE3, 0x40(STATEP)
698
699	FRAME_END
700	RET
701SYM_FUNC_END(crypto_aegis128_aesni_dec_tail)
702
703/*
704 * void crypto_aegis128_aesni_final(void *state, void *tag_xor,
705 *                                  u64 assoclen, u64 cryptlen);
706 */
707SYM_FUNC_START(crypto_aegis128_aesni_final)
708	FRAME_BEGIN
709
710	/* load the state: */
711	movdqu 0x00(STATEP), STATE0
712	movdqu 0x10(STATEP), STATE1
713	movdqu 0x20(STATEP), STATE2
714	movdqu 0x30(STATEP), STATE3
715	movdqu 0x40(STATEP), STATE4
716
717	/* prepare length block: */
718	movq %rdx, MSG
719	movq %rcx, T0
720	pslldq $8, T0
721	pxor T0, MSG
722	psllq $3, MSG /* multiply by 8 (to get bit count) */
723
724	pxor STATE3, MSG
725
726	/* update state: */
727	aegis128_update; pxor MSG, STATE4
728	aegis128_update; pxor MSG, STATE3
729	aegis128_update; pxor MSG, STATE2
730	aegis128_update; pxor MSG, STATE1
731	aegis128_update; pxor MSG, STATE0
732	aegis128_update; pxor MSG, STATE4
733	aegis128_update; pxor MSG, STATE3
734
735	/* xor tag: */
736	movdqu (%rsi), MSG
737
738	pxor STATE0, MSG
739	pxor STATE1, MSG
740	pxor STATE2, MSG
741	pxor STATE3, MSG
742	pxor STATE4, MSG
743
744	movdqu MSG, (%rsi)
745
746	FRAME_END
747	RET
748SYM_FUNC_END(crypto_aegis128_aesni_final)
749