1/* SPDX-License-Identifier: GPL-2.0-or-later */
2/*
3 * Cast5 Cipher 16-way parallel algorithm (AVX/x86_64)
4 *
5 * Copyright (C) 2012 Johannes Goetzfried
6 *     <Johannes.Goetzfried@informatik.stud.uni-erlangen.de>
7 *
8 * Copyright © 2012 Jussi Kivilinna <jussi.kivilinna@mbnet.fi>
9 */
10
11#include <linux/linkage.h>
12#include <asm/frame.h>
13
14.file "cast5-avx-x86_64-asm_64.S"
15
16.extern cast_s1
17.extern cast_s2
18.extern cast_s3
19.extern cast_s4
20
21/* structure of crypto context */
22#define km	0
23#define kr	(16*4)
24#define rr	((16*4)+16)
25
26/* s-boxes */
27#define s1	cast_s1
28#define s2	cast_s2
29#define s3	cast_s3
30#define s4	cast_s4
31
32/**********************************************************************
33  16-way AVX cast5
34 **********************************************************************/
35#define CTX %r15
36
37#define RL1 %xmm0
38#define RR1 %xmm1
39#define RL2 %xmm2
40#define RR2 %xmm3
41#define RL3 %xmm4
42#define RR3 %xmm5
43#define RL4 %xmm6
44#define RR4 %xmm7
45
46#define RX %xmm8
47
48#define RKM  %xmm9
49#define RKR  %xmm10
50#define RKRF %xmm11
51#define RKRR %xmm12
52
53#define R32  %xmm13
54#define R1ST %xmm14
55
56#define RTMP %xmm15
57
58#define RID1  %rdi
59#define RID1d %edi
60#define RID2  %rsi
61#define RID2d %esi
62
63#define RGI1   %rdx
64#define RGI1bl %dl
65#define RGI1bh %dh
66#define RGI2   %rcx
67#define RGI2bl %cl
68#define RGI2bh %ch
69
70#define RGI3   %rax
71#define RGI3bl %al
72#define RGI3bh %ah
73#define RGI4   %rbx
74#define RGI4bl %bl
75#define RGI4bh %bh
76
77#define RFS1  %r8
78#define RFS1d %r8d
79#define RFS2  %r9
80#define RFS2d %r9d
81#define RFS3  %r10
82#define RFS3d %r10d
83
84
85#define lookup_32bit(src, dst, op1, op2, op3, interleave_op, il_reg) \
86	movzbl		src ## bh,     RID1d;    \
87	leaq		s1(%rip),      RID2;     \
88	movl		(RID2,RID1,4), dst ## d; \
89	movzbl		src ## bl,     RID2d;    \
90	leaq		s2(%rip),      RID1;     \
91	op1		(RID1,RID2,4), dst ## d; \
92	shrq $16,	src;                     \
93	movzbl		src ## bh,     RID1d;    \
94	leaq		s3(%rip),      RID2;     \
95	op2		(RID2,RID1,4), dst ## d; \
96	movzbl		src ## bl,     RID2d;    \
97	interleave_op(il_reg);			 \
98	leaq		s4(%rip),      RID1;     \
99	op3		(RID1,RID2,4), dst ## d;
100
101#define dummy(d) /* do nothing */
102
103#define shr_next(reg) \
104	shrq $16,	reg;
105
106#define F_head(a, x, gi1, gi2, op0) \
107	op0	a,	RKM,  x;                 \
108	vpslld	RKRF,	x,    RTMP;              \
109	vpsrld	RKRR,	x,    x;                 \
110	vpor	RTMP,	x,    x;                 \
111	\
112	vmovq		x,    gi1;               \
113	vpextrq $1,	x,    gi2;
114
115#define F_tail(a, x, gi1, gi2, op1, op2, op3) \
116	lookup_32bit(##gi1, RFS1, op1, op2, op3, shr_next, ##gi1); \
117	lookup_32bit(##gi2, RFS3, op1, op2, op3, shr_next, ##gi2); \
118	\
119	lookup_32bit(##gi1, RFS2, op1, op2, op3, dummy, none);     \
120	shlq $32,	RFS2;                                      \
121	orq		RFS1, RFS2;                                \
122	lookup_32bit(##gi2, RFS1, op1, op2, op3, dummy, none);     \
123	shlq $32,	RFS1;                                      \
124	orq		RFS1, RFS3;                                \
125	\
126	vmovq		RFS2, x;                                   \
127	vpinsrq $1,	RFS3, x, x;
128
129#define F_2(a1, b1, a2, b2, op0, op1, op2, op3) \
130	F_head(b1, RX, RGI1, RGI2, op0);              \
131	F_head(b2, RX, RGI3, RGI4, op0);              \
132	\
133	F_tail(b1, RX, RGI1, RGI2, op1, op2, op3);    \
134	F_tail(b2, RTMP, RGI3, RGI4, op1, op2, op3);  \
135	\
136	vpxor		a1, RX,   a1;                 \
137	vpxor		a2, RTMP, a2;
138
139#define F1_2(a1, b1, a2, b2) \
140	F_2(a1, b1, a2, b2, vpaddd, xorl, subl, addl)
141#define F2_2(a1, b1, a2, b2) \
142	F_2(a1, b1, a2, b2, vpxor, subl, addl, xorl)
143#define F3_2(a1, b1, a2, b2) \
144	F_2(a1, b1, a2, b2, vpsubd, addl, xorl, subl)
145
146#define subround(a1, b1, a2, b2, f) \
147	F ## f ## _2(a1, b1, a2, b2);
148
149#define round(l, r, n, f) \
150	vbroadcastss 	(km+(4*n))(CTX), RKM;        \
151	vpand		R1ST,            RKR,  RKRF; \
152	vpsubq		RKRF,            R32,  RKRR; \
153	vpsrldq $1,	RKR,             RKR;        \
154	subround(l ## 1, r ## 1, l ## 2, r ## 2, f); \
155	subround(l ## 3, r ## 3, l ## 4, r ## 4, f);
156
157#define enc_preload_rkr() \
158	vbroadcastss	.L16_mask(%rip),          RKR;      \
159	/* add 16-bit rotation to key rotations (mod 32) */ \
160	vpxor		kr(CTX),                  RKR, RKR;
161
162#define dec_preload_rkr() \
163	vbroadcastss	.L16_mask(%rip),          RKR;      \
164	/* add 16-bit rotation to key rotations (mod 32) */ \
165	vpxor		kr(CTX),                  RKR, RKR; \
166	vpshufb		.Lbswap128_mask(%rip),    RKR, RKR;
167
168#define transpose_2x4(x0, x1, t0, t1) \
169	vpunpckldq		x1, x0, t0; \
170	vpunpckhdq		x1, x0, t1; \
171	\
172	vpunpcklqdq		t1, t0, x0; \
173	vpunpckhqdq		t1, t0, x1;
174
175#define inpack_blocks(x0, x1, t0, t1, rmask) \
176	vpshufb rmask, 	x0,	x0; \
177	vpshufb rmask, 	x1,	x1; \
178	\
179	transpose_2x4(x0, x1, t0, t1)
180
181#define outunpack_blocks(x0, x1, t0, t1, rmask) \
182	transpose_2x4(x0, x1, t0, t1) \
183	\
184	vpshufb rmask,	x0, x0;           \
185	vpshufb rmask,	x1, x1;
186
187.section	.rodata.cst16.bswap_mask, "aM", @progbits, 16
188.align 16
189.Lbswap_mask:
190	.byte 3, 2, 1, 0, 7, 6, 5, 4, 11, 10, 9, 8, 15, 14, 13, 12
191.section	.rodata.cst16.bswap128_mask, "aM", @progbits, 16
192.align 16
193.Lbswap128_mask:
194	.byte 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0
195.section	.rodata.cst16.bswap_iv_mask, "aM", @progbits, 16
196.align 16
197.Lbswap_iv_mask:
198	.byte 7, 6, 5, 4, 3, 2, 1, 0, 7, 6, 5, 4, 3, 2, 1, 0
199
200.section	.rodata.cst4.16_mask, "aM", @progbits, 4
201.align 4
202.L16_mask:
203	.byte 16, 16, 16, 16
204.section	.rodata.cst4.32_mask, "aM", @progbits, 4
205.align 4
206.L32_mask:
207	.byte 32, 0, 0, 0
208.section	.rodata.cst4.first_mask, "aM", @progbits, 4
209.align 4
210.Lfirst_mask:
211	.byte 0x1f, 0, 0, 0
212
213.text
214
215SYM_FUNC_START_LOCAL(__cast5_enc_blk16)
216	/* input:
217	 *	%rdi: ctx
218	 *	RL1: blocks 1 and 2
219	 *	RR1: blocks 3 and 4
220	 *	RL2: blocks 5 and 6
221	 *	RR2: blocks 7 and 8
222	 *	RL3: blocks 9 and 10
223	 *	RR3: blocks 11 and 12
224	 *	RL4: blocks 13 and 14
225	 *	RR4: blocks 15 and 16
226	 * output:
227	 *	RL1: encrypted blocks 1 and 2
228	 *	RR1: encrypted blocks 3 and 4
229	 *	RL2: encrypted blocks 5 and 6
230	 *	RR2: encrypted blocks 7 and 8
231	 *	RL3: encrypted blocks 9 and 10
232	 *	RR3: encrypted blocks 11 and 12
233	 *	RL4: encrypted blocks 13 and 14
234	 *	RR4: encrypted blocks 15 and 16
235	 */
236
237	pushq %r15;
238	pushq %rbx;
239
240	movq %rdi, CTX;
241
242	vmovdqa .Lbswap_mask(%rip), RKM;
243	vmovd .Lfirst_mask(%rip), R1ST;
244	vmovd .L32_mask(%rip), R32;
245	enc_preload_rkr();
246
247	inpack_blocks(RL1, RR1, RTMP, RX, RKM);
248	inpack_blocks(RL2, RR2, RTMP, RX, RKM);
249	inpack_blocks(RL3, RR3, RTMP, RX, RKM);
250	inpack_blocks(RL4, RR4, RTMP, RX, RKM);
251
252	round(RL, RR, 0, 1);
253	round(RR, RL, 1, 2);
254	round(RL, RR, 2, 3);
255	round(RR, RL, 3, 1);
256	round(RL, RR, 4, 2);
257	round(RR, RL, 5, 3);
258	round(RL, RR, 6, 1);
259	round(RR, RL, 7, 2);
260	round(RL, RR, 8, 3);
261	round(RR, RL, 9, 1);
262	round(RL, RR, 10, 2);
263	round(RR, RL, 11, 3);
264
265	movzbl rr(CTX), %eax;
266	testl %eax, %eax;
267	jnz .L__skip_enc;
268
269	round(RL, RR, 12, 1);
270	round(RR, RL, 13, 2);
271	round(RL, RR, 14, 3);
272	round(RR, RL, 15, 1);
273
274.L__skip_enc:
275	popq %rbx;
276	popq %r15;
277
278	vmovdqa .Lbswap_mask(%rip), RKM;
279
280	outunpack_blocks(RR1, RL1, RTMP, RX, RKM);
281	outunpack_blocks(RR2, RL2, RTMP, RX, RKM);
282	outunpack_blocks(RR3, RL3, RTMP, RX, RKM);
283	outunpack_blocks(RR4, RL4, RTMP, RX, RKM);
284
285	RET;
286SYM_FUNC_END(__cast5_enc_blk16)
287
288SYM_FUNC_START_LOCAL(__cast5_dec_blk16)
289	/* input:
290	 *	%rdi: ctx
291	 *	RL1: encrypted blocks 1 and 2
292	 *	RR1: encrypted blocks 3 and 4
293	 *	RL2: encrypted blocks 5 and 6
294	 *	RR2: encrypted blocks 7 and 8
295	 *	RL3: encrypted blocks 9 and 10
296	 *	RR3: encrypted blocks 11 and 12
297	 *	RL4: encrypted blocks 13 and 14
298	 *	RR4: encrypted blocks 15 and 16
299	 * output:
300	 *	RL1: decrypted blocks 1 and 2
301	 *	RR1: decrypted blocks 3 and 4
302	 *	RL2: decrypted blocks 5 and 6
303	 *	RR2: decrypted blocks 7 and 8
304	 *	RL3: decrypted blocks 9 and 10
305	 *	RR3: decrypted blocks 11 and 12
306	 *	RL4: decrypted blocks 13 and 14
307	 *	RR4: decrypted blocks 15 and 16
308	 */
309
310	pushq %r15;
311	pushq %rbx;
312
313	movq %rdi, CTX;
314
315	vmovdqa .Lbswap_mask(%rip), RKM;
316	vmovd .Lfirst_mask(%rip), R1ST;
317	vmovd .L32_mask(%rip), R32;
318	dec_preload_rkr();
319
320	inpack_blocks(RL1, RR1, RTMP, RX, RKM);
321	inpack_blocks(RL2, RR2, RTMP, RX, RKM);
322	inpack_blocks(RL3, RR3, RTMP, RX, RKM);
323	inpack_blocks(RL4, RR4, RTMP, RX, RKM);
324
325	movzbl rr(CTX), %eax;
326	testl %eax, %eax;
327	jnz .L__skip_dec;
328
329	round(RL, RR, 15, 1);
330	round(RR, RL, 14, 3);
331	round(RL, RR, 13, 2);
332	round(RR, RL, 12, 1);
333
334.L__dec_tail:
335	round(RL, RR, 11, 3);
336	round(RR, RL, 10, 2);
337	round(RL, RR, 9, 1);
338	round(RR, RL, 8, 3);
339	round(RL, RR, 7, 2);
340	round(RR, RL, 6, 1);
341	round(RL, RR, 5, 3);
342	round(RR, RL, 4, 2);
343	round(RL, RR, 3, 1);
344	round(RR, RL, 2, 3);
345	round(RL, RR, 1, 2);
346	round(RR, RL, 0, 1);
347
348	vmovdqa .Lbswap_mask(%rip), RKM;
349	popq %rbx;
350	popq %r15;
351
352	outunpack_blocks(RR1, RL1, RTMP, RX, RKM);
353	outunpack_blocks(RR2, RL2, RTMP, RX, RKM);
354	outunpack_blocks(RR3, RL3, RTMP, RX, RKM);
355	outunpack_blocks(RR4, RL4, RTMP, RX, RKM);
356
357	RET;
358
359.L__skip_dec:
360	vpsrldq $4, RKR, RKR;
361	jmp .L__dec_tail;
362SYM_FUNC_END(__cast5_dec_blk16)
363
364SYM_FUNC_START(cast5_ecb_enc_16way)
365	/* input:
366	 *	%rdi: ctx
367	 *	%rsi: dst
368	 *	%rdx: src
369	 */
370	FRAME_BEGIN
371	pushq %r15;
372
373	movq %rdi, CTX;
374	movq %rsi, %r11;
375
376	vmovdqu (0*4*4)(%rdx), RL1;
377	vmovdqu (1*4*4)(%rdx), RR1;
378	vmovdqu (2*4*4)(%rdx), RL2;
379	vmovdqu (3*4*4)(%rdx), RR2;
380	vmovdqu (4*4*4)(%rdx), RL3;
381	vmovdqu (5*4*4)(%rdx), RR3;
382	vmovdqu (6*4*4)(%rdx), RL4;
383	vmovdqu (7*4*4)(%rdx), RR4;
384
385	call __cast5_enc_blk16;
386
387	vmovdqu RR1, (0*4*4)(%r11);
388	vmovdqu RL1, (1*4*4)(%r11);
389	vmovdqu RR2, (2*4*4)(%r11);
390	vmovdqu RL2, (3*4*4)(%r11);
391	vmovdqu RR3, (4*4*4)(%r11);
392	vmovdqu RL3, (5*4*4)(%r11);
393	vmovdqu RR4, (6*4*4)(%r11);
394	vmovdqu RL4, (7*4*4)(%r11);
395
396	popq %r15;
397	FRAME_END
398	RET;
399SYM_FUNC_END(cast5_ecb_enc_16way)
400
401SYM_FUNC_START(cast5_ecb_dec_16way)
402	/* input:
403	 *	%rdi: ctx
404	 *	%rsi: dst
405	 *	%rdx: src
406	 */
407
408	FRAME_BEGIN
409	pushq %r15;
410
411	movq %rdi, CTX;
412	movq %rsi, %r11;
413
414	vmovdqu (0*4*4)(%rdx), RL1;
415	vmovdqu (1*4*4)(%rdx), RR1;
416	vmovdqu (2*4*4)(%rdx), RL2;
417	vmovdqu (3*4*4)(%rdx), RR2;
418	vmovdqu (4*4*4)(%rdx), RL3;
419	vmovdqu (5*4*4)(%rdx), RR3;
420	vmovdqu (6*4*4)(%rdx), RL4;
421	vmovdqu (7*4*4)(%rdx), RR4;
422
423	call __cast5_dec_blk16;
424
425	vmovdqu RR1, (0*4*4)(%r11);
426	vmovdqu RL1, (1*4*4)(%r11);
427	vmovdqu RR2, (2*4*4)(%r11);
428	vmovdqu RL2, (3*4*4)(%r11);
429	vmovdqu RR3, (4*4*4)(%r11);
430	vmovdqu RL3, (5*4*4)(%r11);
431	vmovdqu RR4, (6*4*4)(%r11);
432	vmovdqu RL4, (7*4*4)(%r11);
433
434	popq %r15;
435	FRAME_END
436	RET;
437SYM_FUNC_END(cast5_ecb_dec_16way)
438
439SYM_FUNC_START(cast5_cbc_dec_16way)
440	/* input:
441	 *	%rdi: ctx
442	 *	%rsi: dst
443	 *	%rdx: src
444	 */
445	FRAME_BEGIN
446	pushq %r12;
447	pushq %r15;
448
449	movq %rdi, CTX;
450	movq %rsi, %r11;
451	movq %rdx, %r12;
452
453	vmovdqu (0*16)(%rdx), RL1;
454	vmovdqu (1*16)(%rdx), RR1;
455	vmovdqu (2*16)(%rdx), RL2;
456	vmovdqu (3*16)(%rdx), RR2;
457	vmovdqu (4*16)(%rdx), RL3;
458	vmovdqu (5*16)(%rdx), RR3;
459	vmovdqu (6*16)(%rdx), RL4;
460	vmovdqu (7*16)(%rdx), RR4;
461
462	call __cast5_dec_blk16;
463
464	/* xor with src */
465	vmovq (%r12), RX;
466	vpshufd $0x4f, RX, RX;
467	vpxor RX, RR1, RR1;
468	vpxor 0*16+8(%r12), RL1, RL1;
469	vpxor 1*16+8(%r12), RR2, RR2;
470	vpxor 2*16+8(%r12), RL2, RL2;
471	vpxor 3*16+8(%r12), RR3, RR3;
472	vpxor 4*16+8(%r12), RL3, RL3;
473	vpxor 5*16+8(%r12), RR4, RR4;
474	vpxor 6*16+8(%r12), RL4, RL4;
475
476	vmovdqu RR1, (0*16)(%r11);
477	vmovdqu RL1, (1*16)(%r11);
478	vmovdqu RR2, (2*16)(%r11);
479	vmovdqu RL2, (3*16)(%r11);
480	vmovdqu RR3, (4*16)(%r11);
481	vmovdqu RL3, (5*16)(%r11);
482	vmovdqu RR4, (6*16)(%r11);
483	vmovdqu RL4, (7*16)(%r11);
484
485	popq %r15;
486	popq %r12;
487	FRAME_END
488	RET;
489SYM_FUNC_END(cast5_cbc_dec_16way)
490
491SYM_FUNC_START(cast5_ctr_16way)
492	/* input:
493	 *	%rdi: ctx
494	 *	%rsi: dst
495	 *	%rdx: src
496	 *	%rcx: iv (big endian, 64bit)
497	 */
498	FRAME_BEGIN
499	pushq %r12;
500	pushq %r15;
501
502	movq %rdi, CTX;
503	movq %rsi, %r11;
504	movq %rdx, %r12;
505
506	vpcmpeqd RTMP, RTMP, RTMP;
507	vpsrldq $8, RTMP, RTMP; /* low: -1, high: 0 */
508
509	vpcmpeqd RKR, RKR, RKR;
510	vpaddq RKR, RKR, RKR; /* low: -2, high: -2 */
511	vmovdqa .Lbswap_iv_mask(%rip), R1ST;
512	vmovdqa .Lbswap128_mask(%rip), RKM;
513
514	/* load IV and byteswap */
515	vmovq (%rcx), RX;
516	vpshufb R1ST, RX, RX;
517
518	/* construct IVs */
519	vpsubq RTMP, RX, RX;  /* le: IV1, IV0 */
520	vpshufb RKM, RX, RL1; /* be: IV0, IV1 */
521	vpsubq RKR, RX, RX;
522	vpshufb RKM, RX, RR1; /* be: IV2, IV3 */
523	vpsubq RKR, RX, RX;
524	vpshufb RKM, RX, RL2; /* be: IV4, IV5 */
525	vpsubq RKR, RX, RX;
526	vpshufb RKM, RX, RR2; /* be: IV6, IV7 */
527	vpsubq RKR, RX, RX;
528	vpshufb RKM, RX, RL3; /* be: IV8, IV9 */
529	vpsubq RKR, RX, RX;
530	vpshufb RKM, RX, RR3; /* be: IV10, IV11 */
531	vpsubq RKR, RX, RX;
532	vpshufb RKM, RX, RL4; /* be: IV12, IV13 */
533	vpsubq RKR, RX, RX;
534	vpshufb RKM, RX, RR4; /* be: IV14, IV15 */
535
536	/* store last IV */
537	vpsubq RTMP, RX, RX; /* le: IV16, IV14 */
538	vpshufb R1ST, RX, RX; /* be: IV16, IV16 */
539	vmovq RX, (%rcx);
540
541	call __cast5_enc_blk16;
542
543	/* dst = src ^ iv */
544	vpxor (0*16)(%r12), RR1, RR1;
545	vpxor (1*16)(%r12), RL1, RL1;
546	vpxor (2*16)(%r12), RR2, RR2;
547	vpxor (3*16)(%r12), RL2, RL2;
548	vpxor (4*16)(%r12), RR3, RR3;
549	vpxor (5*16)(%r12), RL3, RL3;
550	vpxor (6*16)(%r12), RR4, RR4;
551	vpxor (7*16)(%r12), RL4, RL4;
552	vmovdqu RR1, (0*16)(%r11);
553	vmovdqu RL1, (1*16)(%r11);
554	vmovdqu RR2, (2*16)(%r11);
555	vmovdqu RL2, (3*16)(%r11);
556	vmovdqu RR3, (4*16)(%r11);
557	vmovdqu RL3, (5*16)(%r11);
558	vmovdqu RR4, (6*16)(%r11);
559	vmovdqu RL4, (7*16)(%r11);
560
561	popq %r15;
562	popq %r12;
563	FRAME_END
564	RET;
565SYM_FUNC_END(cast5_ctr_16way)
566