1/*
2 * Twofish Cipher 8-way parallel algorithm (AVX/x86_64)
3 *
4 * Copyright (C) 2012 Johannes Goetzfried
5 *     <Johannes.Goetzfried@informatik.stud.uni-erlangen.de>
6 *
7 * Copyright © 2012-2013 Jussi Kivilinna <jussi.kivilinna@iki.fi>
8 *
9 * This program is free software; you can redistribute it and/or modify
10 * it under the terms of the GNU General Public License as published by
11 * the Free Software Foundation; either version 2 of the License, or
12 * (at your option) any later version.
13 *
14 * This program is distributed in the hope that it will be useful,
15 * but WITHOUT ANY WARRANTY; without even the implied warranty of
16 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
17 * GNU General Public License for more details.
18 *
19 * You should have received a copy of the GNU General Public License
20 * along with this program; if not, write to the Free Software
21 * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307
22 * USA
23 *
24 */
25
26#include <linux/linkage.h>
27#include "glue_helper-asm-avx.S"
28
29.file "twofish-avx-x86_64-asm_64.S"
30
31.data
32.align 16
33
34.Lbswap128_mask:
35	.byte 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0
36.Lxts_gf128mul_and_shl1_mask:
37	.byte 0x87, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0
38
39.text
40
41/* structure of crypto context */
42#define s0	0
43#define s1	1024
44#define s2	2048
45#define s3	3072
46#define w	4096
47#define k	4128
48
49/**********************************************************************
50  8-way AVX twofish
51 **********************************************************************/
52#define CTX %rdi
53
54#define RA1 %xmm0
55#define RB1 %xmm1
56#define RC1 %xmm2
57#define RD1 %xmm3
58
59#define RA2 %xmm4
60#define RB2 %xmm5
61#define RC2 %xmm6
62#define RD2 %xmm7
63
64#define RX0 %xmm8
65#define RY0 %xmm9
66
67#define RX1 %xmm10
68#define RY1 %xmm11
69
70#define RK1 %xmm12
71#define RK2 %xmm13
72
73#define RT %xmm14
74#define RR %xmm15
75
76#define RID1  %rbp
77#define RID1d %ebp
78#define RID2  %rsi
79#define RID2d %esi
80
81#define RGI1   %rdx
82#define RGI1bl %dl
83#define RGI1bh %dh
84#define RGI2   %rcx
85#define RGI2bl %cl
86#define RGI2bh %ch
87
88#define RGI3   %rax
89#define RGI3bl %al
90#define RGI3bh %ah
91#define RGI4   %rbx
92#define RGI4bl %bl
93#define RGI4bh %bh
94
95#define RGS1  %r8
96#define RGS1d %r8d
97#define RGS2  %r9
98#define RGS2d %r9d
99#define RGS3  %r10
100#define RGS3d %r10d
101
102
103#define lookup_32bit(t0, t1, t2, t3, src, dst, interleave_op, il_reg) \
104	movzbl		src ## bl,        RID1d;     \
105	movzbl		src ## bh,        RID2d;     \
106	shrq $16,	src;                         \
107	movl		t0(CTX, RID1, 4), dst ## d;  \
108	movl		t1(CTX, RID2, 4), RID2d;     \
109	movzbl		src ## bl,        RID1d;     \
110	xorl		RID2d,            dst ## d;  \
111	movzbl		src ## bh,        RID2d;     \
112	interleave_op(il_reg);			     \
113	xorl		t2(CTX, RID1, 4), dst ## d;  \
114	xorl		t3(CTX, RID2, 4), dst ## d;
115
116#define dummy(d) /* do nothing */
117
118#define shr_next(reg) \
119	shrq $16,	reg;
120
121#define G(gi1, gi2, x, t0, t1, t2, t3) \
122	lookup_32bit(t0, t1, t2, t3, ##gi1, RGS1, shr_next, ##gi1);  \
123	lookup_32bit(t0, t1, t2, t3, ##gi2, RGS3, shr_next, ##gi2);  \
124	\
125	lookup_32bit(t0, t1, t2, t3, ##gi1, RGS2, dummy, none);      \
126	shlq $32,	RGS2;                                        \
127	orq		RGS1, RGS2;                                  \
128	lookup_32bit(t0, t1, t2, t3, ##gi2, RGS1, dummy, none);      \
129	shlq $32,	RGS1;                                        \
130	orq		RGS1, RGS3;
131
132#define round_head_2(a, b, x1, y1, x2, y2) \
133	vmovq		b ## 1, RGI3;           \
134	vpextrq $1,	b ## 1, RGI4;           \
135	\
136	G(RGI1, RGI2, x1, s0, s1, s2, s3);      \
137	vmovq		a ## 2, RGI1;           \
138	vpextrq $1,	a ## 2, RGI2;           \
139	vmovq		RGS2, x1;               \
140	vpinsrq $1,	RGS3, x1, x1;           \
141	\
142	G(RGI3, RGI4, y1, s1, s2, s3, s0);      \
143	vmovq		b ## 2, RGI3;           \
144	vpextrq $1,	b ## 2, RGI4;           \
145	vmovq		RGS2, y1;               \
146	vpinsrq $1,	RGS3, y1, y1;           \
147	\
148	G(RGI1, RGI2, x2, s0, s1, s2, s3);      \
149	vmovq		RGS2, x2;               \
150	vpinsrq $1,	RGS3, x2, x2;           \
151	\
152	G(RGI3, RGI4, y2, s1, s2, s3, s0);      \
153	vmovq		RGS2, y2;               \
154	vpinsrq $1,	RGS3, y2, y2;
155
156#define encround_tail(a, b, c, d, x, y, prerotate) \
157	vpaddd			x, y,   x; \
158	vpaddd			x, RK1, RT;\
159	prerotate(b);			   \
160	vpxor			RT, c,  c; \
161	vpaddd			y, x,   y; \
162	vpaddd			y, RK2, y; \
163	vpsrld $1,		c, RT;     \
164	vpslld $(32 - 1),	c, c;      \
165	vpor			c, RT,  c; \
166	vpxor			d, y,   d; \
167
168#define decround_tail(a, b, c, d, x, y, prerotate) \
169	vpaddd			x, y,   x; \
170	vpaddd			x, RK1, RT;\
171	prerotate(a);			   \
172	vpxor			RT, c,  c; \
173	vpaddd			y, x,   y; \
174	vpaddd			y, RK2, y; \
175	vpxor			d, y,   d; \
176	vpsrld $1,		d, y;      \
177	vpslld $(32 - 1),	d, d;      \
178	vpor			d, y,   d; \
179
180#define rotate_1l(x) \
181	vpslld $1,		x, RR;     \
182	vpsrld $(32 - 1),	x, x;      \
183	vpor			x, RR,  x;
184
185#define preload_rgi(c) \
186	vmovq			c, RGI1; \
187	vpextrq $1,		c, RGI2;
188
189#define encrypt_round(n, a, b, c, d, preload, prerotate) \
190	vbroadcastss (k+4*(2*(n)))(CTX),   RK1;                  \
191	vbroadcastss (k+4*(2*(n)+1))(CTX), RK2;                  \
192	round_head_2(a, b, RX0, RY0, RX1, RY1);                  \
193	encround_tail(a ## 1, b ## 1, c ## 1, d ## 1, RX0, RY0, prerotate); \
194	preload(c ## 1);                                         \
195	encround_tail(a ## 2, b ## 2, c ## 2, d ## 2, RX1, RY1, prerotate);
196
197#define decrypt_round(n, a, b, c, d, preload, prerotate) \
198	vbroadcastss (k+4*(2*(n)))(CTX),   RK1;                  \
199	vbroadcastss (k+4*(2*(n)+1))(CTX), RK2;                  \
200	round_head_2(a, b, RX0, RY0, RX1, RY1);                  \
201	decround_tail(a ## 1, b ## 1, c ## 1, d ## 1, RX0, RY0, prerotate); \
202	preload(c ## 1);                                         \
203	decround_tail(a ## 2, b ## 2, c ## 2, d ## 2, RX1, RY1, prerotate);
204
205#define encrypt_cycle(n) \
206	encrypt_round((2*n), RA, RB, RC, RD, preload_rgi, rotate_1l); \
207	encrypt_round(((2*n) + 1), RC, RD, RA, RB, preload_rgi, rotate_1l);
208
209#define encrypt_cycle_last(n) \
210	encrypt_round((2*n), RA, RB, RC, RD, preload_rgi, rotate_1l); \
211	encrypt_round(((2*n) + 1), RC, RD, RA, RB, dummy, dummy);
212
213#define decrypt_cycle(n) \
214	decrypt_round(((2*n) + 1), RC, RD, RA, RB, preload_rgi, rotate_1l); \
215	decrypt_round((2*n), RA, RB, RC, RD, preload_rgi, rotate_1l);
216
217#define decrypt_cycle_last(n) \
218	decrypt_round(((2*n) + 1), RC, RD, RA, RB, preload_rgi, rotate_1l); \
219	decrypt_round((2*n), RA, RB, RC, RD, dummy, dummy);
220
221#define transpose_4x4(x0, x1, x2, x3, t0, t1, t2) \
222	vpunpckldq		x1, x0, t0; \
223	vpunpckhdq		x1, x0, t2; \
224	vpunpckldq		x3, x2, t1; \
225	vpunpckhdq		x3, x2, x3; \
226	\
227	vpunpcklqdq		t1, t0, x0; \
228	vpunpckhqdq		t1, t0, x1; \
229	vpunpcklqdq		x3, t2, x2; \
230	vpunpckhqdq		x3, t2, x3;
231
232#define inpack_blocks(x0, x1, x2, x3, wkey, t0, t1, t2) \
233	vpxor		x0, wkey, x0; \
234	vpxor		x1, wkey, x1; \
235	vpxor		x2, wkey, x2; \
236	vpxor		x3, wkey, x3; \
237	\
238	transpose_4x4(x0, x1, x2, x3, t0, t1, t2)
239
240#define outunpack_blocks(x0, x1, x2, x3, wkey, t0, t1, t2) \
241	transpose_4x4(x0, x1, x2, x3, t0, t1, t2) \
242	\
243	vpxor		x0, wkey, x0; \
244	vpxor		x1, wkey, x1; \
245	vpxor		x2, wkey, x2; \
246	vpxor		x3, wkey, x3;
247
248.align 8
249__twofish_enc_blk8:
250	/* input:
251	 *	%rdi: ctx, CTX
252	 *	RA1, RB1, RC1, RD1, RA2, RB2, RC2, RD2: blocks
253	 * output:
254	 *	RC1, RD1, RA1, RB1, RC2, RD2, RA2, RB2: encrypted blocks
255	 */
256
257	vmovdqu w(CTX), RK1;
258
259	pushq %rbp;
260	pushq %rbx;
261	pushq %rcx;
262
263	inpack_blocks(RA1, RB1, RC1, RD1, RK1, RX0, RY0, RK2);
264	preload_rgi(RA1);
265	rotate_1l(RD1);
266	inpack_blocks(RA2, RB2, RC2, RD2, RK1, RX0, RY0, RK2);
267	rotate_1l(RD2);
268
269	encrypt_cycle(0);
270	encrypt_cycle(1);
271	encrypt_cycle(2);
272	encrypt_cycle(3);
273	encrypt_cycle(4);
274	encrypt_cycle(5);
275	encrypt_cycle(6);
276	encrypt_cycle_last(7);
277
278	vmovdqu (w+4*4)(CTX), RK1;
279
280	popq %rcx;
281	popq %rbx;
282	popq %rbp;
283
284	outunpack_blocks(RC1, RD1, RA1, RB1, RK1, RX0, RY0, RK2);
285	outunpack_blocks(RC2, RD2, RA2, RB2, RK1, RX0, RY0, RK2);
286
287	ret;
288ENDPROC(__twofish_enc_blk8)
289
290.align 8
291__twofish_dec_blk8:
292	/* input:
293	 *	%rdi: ctx, CTX
294	 *	RC1, RD1, RA1, RB1, RC2, RD2, RA2, RB2: encrypted blocks
295	 * output:
296	 *	RA1, RB1, RC1, RD1, RA2, RB2, RC2, RD2: decrypted blocks
297	 */
298
299	vmovdqu (w+4*4)(CTX), RK1;
300
301	pushq %rbp;
302	pushq %rbx;
303
304	inpack_blocks(RC1, RD1, RA1, RB1, RK1, RX0, RY0, RK2);
305	preload_rgi(RC1);
306	rotate_1l(RA1);
307	inpack_blocks(RC2, RD2, RA2, RB2, RK1, RX0, RY0, RK2);
308	rotate_1l(RA2);
309
310	decrypt_cycle(7);
311	decrypt_cycle(6);
312	decrypt_cycle(5);
313	decrypt_cycle(4);
314	decrypt_cycle(3);
315	decrypt_cycle(2);
316	decrypt_cycle(1);
317	decrypt_cycle_last(0);
318
319	vmovdqu (w)(CTX), RK1;
320
321	popq %rbx;
322	popq %rbp;
323
324	outunpack_blocks(RA1, RB1, RC1, RD1, RK1, RX0, RY0, RK2);
325	outunpack_blocks(RA2, RB2, RC2, RD2, RK1, RX0, RY0, RK2);
326
327	ret;
328ENDPROC(__twofish_dec_blk8)
329
330ENTRY(twofish_ecb_enc_8way)
331	/* input:
332	 *	%rdi: ctx, CTX
333	 *	%rsi: dst
334	 *	%rdx: src
335	 */
336
337	movq %rsi, %r11;
338
339	load_8way(%rdx, RA1, RB1, RC1, RD1, RA2, RB2, RC2, RD2);
340
341	call __twofish_enc_blk8;
342
343	store_8way(%r11, RC1, RD1, RA1, RB1, RC2, RD2, RA2, RB2);
344
345	ret;
346ENDPROC(twofish_ecb_enc_8way)
347
348ENTRY(twofish_ecb_dec_8way)
349	/* input:
350	 *	%rdi: ctx, CTX
351	 *	%rsi: dst
352	 *	%rdx: src
353	 */
354
355	movq %rsi, %r11;
356
357	load_8way(%rdx, RC1, RD1, RA1, RB1, RC2, RD2, RA2, RB2);
358
359	call __twofish_dec_blk8;
360
361	store_8way(%r11, RA1, RB1, RC1, RD1, RA2, RB2, RC2, RD2);
362
363	ret;
364ENDPROC(twofish_ecb_dec_8way)
365
366ENTRY(twofish_cbc_dec_8way)
367	/* input:
368	 *	%rdi: ctx, CTX
369	 *	%rsi: dst
370	 *	%rdx: src
371	 */
372
373	pushq %r12;
374
375	movq %rsi, %r11;
376	movq %rdx, %r12;
377
378	load_8way(%rdx, RC1, RD1, RA1, RB1, RC2, RD2, RA2, RB2);
379
380	call __twofish_dec_blk8;
381
382	store_cbc_8way(%r12, %r11, RA1, RB1, RC1, RD1, RA2, RB2, RC2, RD2);
383
384	popq %r12;
385
386	ret;
387ENDPROC(twofish_cbc_dec_8way)
388
389ENTRY(twofish_ctr_8way)
390	/* input:
391	 *	%rdi: ctx, CTX
392	 *	%rsi: dst
393	 *	%rdx: src
394	 *	%rcx: iv (little endian, 128bit)
395	 */
396
397	pushq %r12;
398
399	movq %rsi, %r11;
400	movq %rdx, %r12;
401
402	load_ctr_8way(%rcx, .Lbswap128_mask, RA1, RB1, RC1, RD1, RA2, RB2, RC2,
403		      RD2, RX0, RX1, RY0);
404
405	call __twofish_enc_blk8;
406
407	store_ctr_8way(%r12, %r11, RC1, RD1, RA1, RB1, RC2, RD2, RA2, RB2);
408
409	popq %r12;
410
411	ret;
412ENDPROC(twofish_ctr_8way)
413
414ENTRY(twofish_xts_enc_8way)
415	/* input:
416	 *	%rdi: ctx, CTX
417	 *	%rsi: dst
418	 *	%rdx: src
419	 *	%rcx: iv (t ⊕ αⁿ ∈ GF(2¹²⁸))
420	 */
421
422	movq %rsi, %r11;
423
424	/* regs <= src, dst <= IVs, regs <= regs xor IVs */
425	load_xts_8way(%rcx, %rdx, %rsi, RA1, RB1, RC1, RD1, RA2, RB2, RC2, RD2,
426		      RX0, RX1, RY0, .Lxts_gf128mul_and_shl1_mask);
427
428	call __twofish_enc_blk8;
429
430	/* dst <= regs xor IVs(in dst) */
431	store_xts_8way(%r11, RC1, RD1, RA1, RB1, RC2, RD2, RA2, RB2);
432
433	ret;
434ENDPROC(twofish_xts_enc_8way)
435
436ENTRY(twofish_xts_dec_8way)
437	/* input:
438	 *	%rdi: ctx, CTX
439	 *	%rsi: dst
440	 *	%rdx: src
441	 *	%rcx: iv (t ⊕ αⁿ ∈ GF(2¹²⁸))
442	 */
443
444	movq %rsi, %r11;
445
446	/* regs <= src, dst <= IVs, regs <= regs xor IVs */
447	load_xts_8way(%rcx, %rdx, %rsi, RC1, RD1, RA1, RB1, RC2, RD2, RA2, RB2,
448		      RX0, RX1, RY0, .Lxts_gf128mul_and_shl1_mask);
449
450	call __twofish_dec_blk8;
451
452	/* dst <= regs xor IVs(in dst) */
453	store_xts_8way(%r11, RA1, RB1, RC1, RD1, RA2, RB2, RC2, RD2);
454
455	ret;
456ENDPROC(twofish_xts_dec_8way)
457