1/*
2 * Twofish Cipher 8-way parallel algorithm (AVX/x86_64)
3 *
4 * Copyright (C) 2012 Johannes Goetzfried
5 *     <Johannes.Goetzfried@informatik.stud.uni-erlangen.de>
6 *
7 * Copyright © 2012-2013 Jussi Kivilinna <jussi.kivilinna@iki.fi>
8 *
9 * This program is free software; you can redistribute it and/or modify
10 * it under the terms of the GNU General Public License as published by
11 * the Free Software Foundation; either version 2 of the License, or
12 * (at your option) any later version.
13 *
14 * This program is distributed in the hope that it will be useful,
15 * but WITHOUT ANY WARRANTY; without even the implied warranty of
16 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
17 * GNU General Public License for more details.
18 *
19 * You should have received a copy of the GNU General Public License
20 * along with this program; if not, write to the Free Software
21 * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307
22 * USA
23 *
24 */
25
26#include <linux/linkage.h>
27#include <asm/frame.h>
28#include "glue_helper-asm-avx.S"
29
30.file "twofish-avx-x86_64-asm_64.S"
31
32.section	.rodata.cst16.bswap128_mask, "aM", @progbits, 16
33.align 16
34.Lbswap128_mask:
35	.byte 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0
36
37.section	.rodata.cst16.xts_gf128mul_and_shl1_mask, "aM", @progbits, 16
38.align 16
39.Lxts_gf128mul_and_shl1_mask:
40	.byte 0x87, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0
41
42.text
43
44/* structure of crypto context */
45#define s0	0
46#define s1	1024
47#define s2	2048
48#define s3	3072
49#define w	4096
50#define k	4128
51
52/**********************************************************************
53  8-way AVX twofish
54 **********************************************************************/
55#define CTX %rdi
56
57#define RA1 %xmm0
58#define RB1 %xmm1
59#define RC1 %xmm2
60#define RD1 %xmm3
61
62#define RA2 %xmm4
63#define RB2 %xmm5
64#define RC2 %xmm6
65#define RD2 %xmm7
66
67#define RX0 %xmm8
68#define RY0 %xmm9
69
70#define RX1 %xmm10
71#define RY1 %xmm11
72
73#define RK1 %xmm12
74#define RK2 %xmm13
75
76#define RT %xmm14
77#define RR %xmm15
78
79#define RID1  %rbp
80#define RID1d %ebp
81#define RID2  %rsi
82#define RID2d %esi
83
84#define RGI1   %rdx
85#define RGI1bl %dl
86#define RGI1bh %dh
87#define RGI2   %rcx
88#define RGI2bl %cl
89#define RGI2bh %ch
90
91#define RGI3   %rax
92#define RGI3bl %al
93#define RGI3bh %ah
94#define RGI4   %rbx
95#define RGI4bl %bl
96#define RGI4bh %bh
97
98#define RGS1  %r8
99#define RGS1d %r8d
100#define RGS2  %r9
101#define RGS2d %r9d
102#define RGS3  %r10
103#define RGS3d %r10d
104
105
106#define lookup_32bit(t0, t1, t2, t3, src, dst, interleave_op, il_reg) \
107	movzbl		src ## bl,        RID1d;     \
108	movzbl		src ## bh,        RID2d;     \
109	shrq $16,	src;                         \
110	movl		t0(CTX, RID1, 4), dst ## d;  \
111	movl		t1(CTX, RID2, 4), RID2d;     \
112	movzbl		src ## bl,        RID1d;     \
113	xorl		RID2d,            dst ## d;  \
114	movzbl		src ## bh,        RID2d;     \
115	interleave_op(il_reg);			     \
116	xorl		t2(CTX, RID1, 4), dst ## d;  \
117	xorl		t3(CTX, RID2, 4), dst ## d;
118
119#define dummy(d) /* do nothing */
120
121#define shr_next(reg) \
122	shrq $16,	reg;
123
124#define G(gi1, gi2, x, t0, t1, t2, t3) \
125	lookup_32bit(t0, t1, t2, t3, ##gi1, RGS1, shr_next, ##gi1);  \
126	lookup_32bit(t0, t1, t2, t3, ##gi2, RGS3, shr_next, ##gi2);  \
127	\
128	lookup_32bit(t0, t1, t2, t3, ##gi1, RGS2, dummy, none);      \
129	shlq $32,	RGS2;                                        \
130	orq		RGS1, RGS2;                                  \
131	lookup_32bit(t0, t1, t2, t3, ##gi2, RGS1, dummy, none);      \
132	shlq $32,	RGS1;                                        \
133	orq		RGS1, RGS3;
134
135#define round_head_2(a, b, x1, y1, x2, y2) \
136	vmovq		b ## 1, RGI3;           \
137	vpextrq $1,	b ## 1, RGI4;           \
138	\
139	G(RGI1, RGI2, x1, s0, s1, s2, s3);      \
140	vmovq		a ## 2, RGI1;           \
141	vpextrq $1,	a ## 2, RGI2;           \
142	vmovq		RGS2, x1;               \
143	vpinsrq $1,	RGS3, x1, x1;           \
144	\
145	G(RGI3, RGI4, y1, s1, s2, s3, s0);      \
146	vmovq		b ## 2, RGI3;           \
147	vpextrq $1,	b ## 2, RGI4;           \
148	vmovq		RGS2, y1;               \
149	vpinsrq $1,	RGS3, y1, y1;           \
150	\
151	G(RGI1, RGI2, x2, s0, s1, s2, s3);      \
152	vmovq		RGS2, x2;               \
153	vpinsrq $1,	RGS3, x2, x2;           \
154	\
155	G(RGI3, RGI4, y2, s1, s2, s3, s0);      \
156	vmovq		RGS2, y2;               \
157	vpinsrq $1,	RGS3, y2, y2;
158
159#define encround_tail(a, b, c, d, x, y, prerotate) \
160	vpaddd			x, y,   x; \
161	vpaddd			x, RK1, RT;\
162	prerotate(b);			   \
163	vpxor			RT, c,  c; \
164	vpaddd			y, x,   y; \
165	vpaddd			y, RK2, y; \
166	vpsrld $1,		c, RT;     \
167	vpslld $(32 - 1),	c, c;      \
168	vpor			c, RT,  c; \
169	vpxor			d, y,   d; \
170
171#define decround_tail(a, b, c, d, x, y, prerotate) \
172	vpaddd			x, y,   x; \
173	vpaddd			x, RK1, RT;\
174	prerotate(a);			   \
175	vpxor			RT, c,  c; \
176	vpaddd			y, x,   y; \
177	vpaddd			y, RK2, y; \
178	vpxor			d, y,   d; \
179	vpsrld $1,		d, y;      \
180	vpslld $(32 - 1),	d, d;      \
181	vpor			d, y,   d; \
182
183#define rotate_1l(x) \
184	vpslld $1,		x, RR;     \
185	vpsrld $(32 - 1),	x, x;      \
186	vpor			x, RR,  x;
187
188#define preload_rgi(c) \
189	vmovq			c, RGI1; \
190	vpextrq $1,		c, RGI2;
191
192#define encrypt_round(n, a, b, c, d, preload, prerotate) \
193	vbroadcastss (k+4*(2*(n)))(CTX),   RK1;                  \
194	vbroadcastss (k+4*(2*(n)+1))(CTX), RK2;                  \
195	round_head_2(a, b, RX0, RY0, RX1, RY1);                  \
196	encround_tail(a ## 1, b ## 1, c ## 1, d ## 1, RX0, RY0, prerotate); \
197	preload(c ## 1);                                         \
198	encround_tail(a ## 2, b ## 2, c ## 2, d ## 2, RX1, RY1, prerotate);
199
200#define decrypt_round(n, a, b, c, d, preload, prerotate) \
201	vbroadcastss (k+4*(2*(n)))(CTX),   RK1;                  \
202	vbroadcastss (k+4*(2*(n)+1))(CTX), RK2;                  \
203	round_head_2(a, b, RX0, RY0, RX1, RY1);                  \
204	decround_tail(a ## 1, b ## 1, c ## 1, d ## 1, RX0, RY0, prerotate); \
205	preload(c ## 1);                                         \
206	decround_tail(a ## 2, b ## 2, c ## 2, d ## 2, RX1, RY1, prerotate);
207
208#define encrypt_cycle(n) \
209	encrypt_round((2*n), RA, RB, RC, RD, preload_rgi, rotate_1l); \
210	encrypt_round(((2*n) + 1), RC, RD, RA, RB, preload_rgi, rotate_1l);
211
212#define encrypt_cycle_last(n) \
213	encrypt_round((2*n), RA, RB, RC, RD, preload_rgi, rotate_1l); \
214	encrypt_round(((2*n) + 1), RC, RD, RA, RB, dummy, dummy);
215
216#define decrypt_cycle(n) \
217	decrypt_round(((2*n) + 1), RC, RD, RA, RB, preload_rgi, rotate_1l); \
218	decrypt_round((2*n), RA, RB, RC, RD, preload_rgi, rotate_1l);
219
220#define decrypt_cycle_last(n) \
221	decrypt_round(((2*n) + 1), RC, RD, RA, RB, preload_rgi, rotate_1l); \
222	decrypt_round((2*n), RA, RB, RC, RD, dummy, dummy);
223
224#define transpose_4x4(x0, x1, x2, x3, t0, t1, t2) \
225	vpunpckldq		x1, x0, t0; \
226	vpunpckhdq		x1, x0, t2; \
227	vpunpckldq		x3, x2, t1; \
228	vpunpckhdq		x3, x2, x3; \
229	\
230	vpunpcklqdq		t1, t0, x0; \
231	vpunpckhqdq		t1, t0, x1; \
232	vpunpcklqdq		x3, t2, x2; \
233	vpunpckhqdq		x3, t2, x3;
234
235#define inpack_blocks(x0, x1, x2, x3, wkey, t0, t1, t2) \
236	vpxor		x0, wkey, x0; \
237	vpxor		x1, wkey, x1; \
238	vpxor		x2, wkey, x2; \
239	vpxor		x3, wkey, x3; \
240	\
241	transpose_4x4(x0, x1, x2, x3, t0, t1, t2)
242
243#define outunpack_blocks(x0, x1, x2, x3, wkey, t0, t1, t2) \
244	transpose_4x4(x0, x1, x2, x3, t0, t1, t2) \
245	\
246	vpxor		x0, wkey, x0; \
247	vpxor		x1, wkey, x1; \
248	vpxor		x2, wkey, x2; \
249	vpxor		x3, wkey, x3;
250
251.align 8
252__twofish_enc_blk8:
253	/* input:
254	 *	%rdi: ctx, CTX
255	 *	RA1, RB1, RC1, RD1, RA2, RB2, RC2, RD2: blocks
256	 * output:
257	 *	RC1, RD1, RA1, RB1, RC2, RD2, RA2, RB2: encrypted blocks
258	 */
259
260	vmovdqu w(CTX), RK1;
261
262	pushq %rbp;
263	pushq %rbx;
264	pushq %rcx;
265
266	inpack_blocks(RA1, RB1, RC1, RD1, RK1, RX0, RY0, RK2);
267	preload_rgi(RA1);
268	rotate_1l(RD1);
269	inpack_blocks(RA2, RB2, RC2, RD2, RK1, RX0, RY0, RK2);
270	rotate_1l(RD2);
271
272	encrypt_cycle(0);
273	encrypt_cycle(1);
274	encrypt_cycle(2);
275	encrypt_cycle(3);
276	encrypt_cycle(4);
277	encrypt_cycle(5);
278	encrypt_cycle(6);
279	encrypt_cycle_last(7);
280
281	vmovdqu (w+4*4)(CTX), RK1;
282
283	popq %rcx;
284	popq %rbx;
285	popq %rbp;
286
287	outunpack_blocks(RC1, RD1, RA1, RB1, RK1, RX0, RY0, RK2);
288	outunpack_blocks(RC2, RD2, RA2, RB2, RK1, RX0, RY0, RK2);
289
290	ret;
291ENDPROC(__twofish_enc_blk8)
292
293.align 8
294__twofish_dec_blk8:
295	/* input:
296	 *	%rdi: ctx, CTX
297	 *	RC1, RD1, RA1, RB1, RC2, RD2, RA2, RB2: encrypted blocks
298	 * output:
299	 *	RA1, RB1, RC1, RD1, RA2, RB2, RC2, RD2: decrypted blocks
300	 */
301
302	vmovdqu (w+4*4)(CTX), RK1;
303
304	pushq %rbp;
305	pushq %rbx;
306
307	inpack_blocks(RC1, RD1, RA1, RB1, RK1, RX0, RY0, RK2);
308	preload_rgi(RC1);
309	rotate_1l(RA1);
310	inpack_blocks(RC2, RD2, RA2, RB2, RK1, RX0, RY0, RK2);
311	rotate_1l(RA2);
312
313	decrypt_cycle(7);
314	decrypt_cycle(6);
315	decrypt_cycle(5);
316	decrypt_cycle(4);
317	decrypt_cycle(3);
318	decrypt_cycle(2);
319	decrypt_cycle(1);
320	decrypt_cycle_last(0);
321
322	vmovdqu (w)(CTX), RK1;
323
324	popq %rbx;
325	popq %rbp;
326
327	outunpack_blocks(RA1, RB1, RC1, RD1, RK1, RX0, RY0, RK2);
328	outunpack_blocks(RA2, RB2, RC2, RD2, RK1, RX0, RY0, RK2);
329
330	ret;
331ENDPROC(__twofish_dec_blk8)
332
333ENTRY(twofish_ecb_enc_8way)
334	/* input:
335	 *	%rdi: ctx, CTX
336	 *	%rsi: dst
337	 *	%rdx: src
338	 */
339	FRAME_BEGIN
340
341	movq %rsi, %r11;
342
343	load_8way(%rdx, RA1, RB1, RC1, RD1, RA2, RB2, RC2, RD2);
344
345	call __twofish_enc_blk8;
346
347	store_8way(%r11, RC1, RD1, RA1, RB1, RC2, RD2, RA2, RB2);
348
349	FRAME_END
350	ret;
351ENDPROC(twofish_ecb_enc_8way)
352
353ENTRY(twofish_ecb_dec_8way)
354	/* input:
355	 *	%rdi: ctx, CTX
356	 *	%rsi: dst
357	 *	%rdx: src
358	 */
359	FRAME_BEGIN
360
361	movq %rsi, %r11;
362
363	load_8way(%rdx, RC1, RD1, RA1, RB1, RC2, RD2, RA2, RB2);
364
365	call __twofish_dec_blk8;
366
367	store_8way(%r11, RA1, RB1, RC1, RD1, RA2, RB2, RC2, RD2);
368
369	FRAME_END
370	ret;
371ENDPROC(twofish_ecb_dec_8way)
372
373ENTRY(twofish_cbc_dec_8way)
374	/* input:
375	 *	%rdi: ctx, CTX
376	 *	%rsi: dst
377	 *	%rdx: src
378	 */
379	FRAME_BEGIN
380
381	pushq %r12;
382
383	movq %rsi, %r11;
384	movq %rdx, %r12;
385
386	load_8way(%rdx, RC1, RD1, RA1, RB1, RC2, RD2, RA2, RB2);
387
388	call __twofish_dec_blk8;
389
390	store_cbc_8way(%r12, %r11, RA1, RB1, RC1, RD1, RA2, RB2, RC2, RD2);
391
392	popq %r12;
393
394	FRAME_END
395	ret;
396ENDPROC(twofish_cbc_dec_8way)
397
398ENTRY(twofish_ctr_8way)
399	/* input:
400	 *	%rdi: ctx, CTX
401	 *	%rsi: dst
402	 *	%rdx: src
403	 *	%rcx: iv (little endian, 128bit)
404	 */
405	FRAME_BEGIN
406
407	pushq %r12;
408
409	movq %rsi, %r11;
410	movq %rdx, %r12;
411
412	load_ctr_8way(%rcx, .Lbswap128_mask, RA1, RB1, RC1, RD1, RA2, RB2, RC2,
413		      RD2, RX0, RX1, RY0);
414
415	call __twofish_enc_blk8;
416
417	store_ctr_8way(%r12, %r11, RC1, RD1, RA1, RB1, RC2, RD2, RA2, RB2);
418
419	popq %r12;
420
421	FRAME_END
422	ret;
423ENDPROC(twofish_ctr_8way)
424
425ENTRY(twofish_xts_enc_8way)
426	/* input:
427	 *	%rdi: ctx, CTX
428	 *	%rsi: dst
429	 *	%rdx: src
430	 *	%rcx: iv (t ⊕ αⁿ ∈ GF(2¹²⁸))
431	 */
432	FRAME_BEGIN
433
434	movq %rsi, %r11;
435
436	/* regs <= src, dst <= IVs, regs <= regs xor IVs */
437	load_xts_8way(%rcx, %rdx, %rsi, RA1, RB1, RC1, RD1, RA2, RB2, RC2, RD2,
438		      RX0, RX1, RY0, .Lxts_gf128mul_and_shl1_mask);
439
440	call __twofish_enc_blk8;
441
442	/* dst <= regs xor IVs(in dst) */
443	store_xts_8way(%r11, RC1, RD1, RA1, RB1, RC2, RD2, RA2, RB2);
444
445	FRAME_END
446	ret;
447ENDPROC(twofish_xts_enc_8way)
448
449ENTRY(twofish_xts_dec_8way)
450	/* input:
451	 *	%rdi: ctx, CTX
452	 *	%rsi: dst
453	 *	%rdx: src
454	 *	%rcx: iv (t ⊕ αⁿ ∈ GF(2¹²⁸))
455	 */
456	FRAME_BEGIN
457
458	movq %rsi, %r11;
459
460	/* regs <= src, dst <= IVs, regs <= regs xor IVs */
461	load_xts_8way(%rcx, %rdx, %rsi, RC1, RD1, RA1, RB1, RC2, RD2, RA2, RB2,
462		      RX0, RX1, RY0, .Lxts_gf128mul_and_shl1_mask);
463
464	call __twofish_dec_blk8;
465
466	/* dst <= regs xor IVs(in dst) */
467	store_xts_8way(%r11, RA1, RB1, RC1, RD1, RA2, RB2, RC2, RD2);
468
469	FRAME_END
470	ret;
471ENDPROC(twofish_xts_dec_8way)
472