1 /*
2  * Glue Code for AVX assembler version of Twofish Cipher
3  *
4  * Copyright (C) 2012 Johannes Goetzfried
5  *     <Johannes.Goetzfried@informatik.stud.uni-erlangen.de>
6  *
7  * Copyright © 2013 Jussi Kivilinna <jussi.kivilinna@iki.fi>
8  *
9  * This program is free software; you can redistribute it and/or modify
10  * it under the terms of the GNU General Public License as published by
11  * the Free Software Foundation; either version 2 of the License, or
12  * (at your option) any later version.
13  *
14  * This program is distributed in the hope that it will be useful,
15  * but WITHOUT ANY WARRANTY; without even the implied warranty of
16  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
17  * GNU General Public License for more details.
18  *
19  * You should have received a copy of the GNU General Public License
20  * along with this program; if not, write to the Free Software
21  * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307
22  * USA
23  *
24  */
25 
26 #include <linux/module.h>
27 #include <linux/hardirq.h>
28 #include <linux/types.h>
29 #include <linux/crypto.h>
30 #include <linux/err.h>
31 #include <crypto/algapi.h>
32 #include <crypto/twofish.h>
33 #include <crypto/cryptd.h>
34 #include <crypto/b128ops.h>
35 #include <crypto/ctr.h>
36 #include <crypto/lrw.h>
37 #include <crypto/xts.h>
38 #include <asm/i387.h>
39 #include <asm/xcr.h>
40 #include <asm/xsave.h>
41 #include <asm/crypto/twofish.h>
42 #include <asm/crypto/ablk_helper.h>
43 #include <asm/crypto/glue_helper.h>
44 #include <crypto/scatterwalk.h>
45 #include <linux/workqueue.h>
46 #include <linux/spinlock.h>
47 
48 #define TWOFISH_PARALLEL_BLOCKS 8
49 
50 /* 8-way parallel cipher functions */
51 asmlinkage void twofish_ecb_enc_8way(struct twofish_ctx *ctx, u8 *dst,
52 				     const u8 *src);
53 EXPORT_SYMBOL_GPL(twofish_ecb_enc_8way);
54 
55 asmlinkage void twofish_ecb_dec_8way(struct twofish_ctx *ctx, u8 *dst,
56 				     const u8 *src);
57 EXPORT_SYMBOL_GPL(twofish_ecb_dec_8way);
58 
59 asmlinkage void twofish_cbc_dec_8way(struct twofish_ctx *ctx, u8 *dst,
60 				     const u8 *src);
61 EXPORT_SYMBOL_GPL(twofish_cbc_dec_8way);
62 
63 asmlinkage void twofish_ctr_8way(struct twofish_ctx *ctx, u8 *dst,
64 				 const u8 *src, le128 *iv);
65 EXPORT_SYMBOL_GPL(twofish_ctr_8way);
66 
67 asmlinkage void twofish_xts_enc_8way(struct twofish_ctx *ctx, u8 *dst,
68 				     const u8 *src, le128 *iv);
69 EXPORT_SYMBOL_GPL(twofish_xts_enc_8way);
70 asmlinkage void twofish_xts_dec_8way(struct twofish_ctx *ctx, u8 *dst,
71 				     const u8 *src, le128 *iv);
72 EXPORT_SYMBOL_GPL(twofish_xts_dec_8way);
73 
74 static inline void twofish_enc_blk_3way(struct twofish_ctx *ctx, u8 *dst,
75 					const u8 *src)
76 {
77 	__twofish_enc_blk_3way(ctx, dst, src, false);
78 }
79 
80 void twofish_xts_enc(void *ctx, u128 *dst, const u128 *src, le128 *iv)
81 {
82 	glue_xts_crypt_128bit_one(ctx, dst, src, iv,
83 				  GLUE_FUNC_CAST(twofish_enc_blk));
84 }
85 EXPORT_SYMBOL_GPL(twofish_xts_enc);
86 
87 void twofish_xts_dec(void *ctx, u128 *dst, const u128 *src, le128 *iv)
88 {
89 	glue_xts_crypt_128bit_one(ctx, dst, src, iv,
90 				  GLUE_FUNC_CAST(twofish_dec_blk));
91 }
92 EXPORT_SYMBOL_GPL(twofish_xts_dec);
93 
94 
95 static const struct common_glue_ctx twofish_enc = {
96 	.num_funcs = 3,
97 	.fpu_blocks_limit = TWOFISH_PARALLEL_BLOCKS,
98 
99 	.funcs = { {
100 		.num_blocks = TWOFISH_PARALLEL_BLOCKS,
101 		.fn_u = { .ecb = GLUE_FUNC_CAST(twofish_ecb_enc_8way) }
102 	}, {
103 		.num_blocks = 3,
104 		.fn_u = { .ecb = GLUE_FUNC_CAST(twofish_enc_blk_3way) }
105 	}, {
106 		.num_blocks = 1,
107 		.fn_u = { .ecb = GLUE_FUNC_CAST(twofish_enc_blk) }
108 	} }
109 };
110 
111 static const struct common_glue_ctx twofish_ctr = {
112 	.num_funcs = 3,
113 	.fpu_blocks_limit = TWOFISH_PARALLEL_BLOCKS,
114 
115 	.funcs = { {
116 		.num_blocks = TWOFISH_PARALLEL_BLOCKS,
117 		.fn_u = { .ctr = GLUE_CTR_FUNC_CAST(twofish_ctr_8way) }
118 	}, {
119 		.num_blocks = 3,
120 		.fn_u = { .ctr = GLUE_CTR_FUNC_CAST(twofish_enc_blk_ctr_3way) }
121 	}, {
122 		.num_blocks = 1,
123 		.fn_u = { .ctr = GLUE_CTR_FUNC_CAST(twofish_enc_blk_ctr) }
124 	} }
125 };
126 
127 static const struct common_glue_ctx twofish_enc_xts = {
128 	.num_funcs = 2,
129 	.fpu_blocks_limit = TWOFISH_PARALLEL_BLOCKS,
130 
131 	.funcs = { {
132 		.num_blocks = TWOFISH_PARALLEL_BLOCKS,
133 		.fn_u = { .xts = GLUE_XTS_FUNC_CAST(twofish_xts_enc_8way) }
134 	}, {
135 		.num_blocks = 1,
136 		.fn_u = { .xts = GLUE_XTS_FUNC_CAST(twofish_xts_enc) }
137 	} }
138 };
139 
140 static const struct common_glue_ctx twofish_dec = {
141 	.num_funcs = 3,
142 	.fpu_blocks_limit = TWOFISH_PARALLEL_BLOCKS,
143 
144 	.funcs = { {
145 		.num_blocks = TWOFISH_PARALLEL_BLOCKS,
146 		.fn_u = { .ecb = GLUE_FUNC_CAST(twofish_ecb_dec_8way) }
147 	}, {
148 		.num_blocks = 3,
149 		.fn_u = { .ecb = GLUE_FUNC_CAST(twofish_dec_blk_3way) }
150 	}, {
151 		.num_blocks = 1,
152 		.fn_u = { .ecb = GLUE_FUNC_CAST(twofish_dec_blk) }
153 	} }
154 };
155 
156 static const struct common_glue_ctx twofish_dec_cbc = {
157 	.num_funcs = 3,
158 	.fpu_blocks_limit = TWOFISH_PARALLEL_BLOCKS,
159 
160 	.funcs = { {
161 		.num_blocks = TWOFISH_PARALLEL_BLOCKS,
162 		.fn_u = { .cbc = GLUE_CBC_FUNC_CAST(twofish_cbc_dec_8way) }
163 	}, {
164 		.num_blocks = 3,
165 		.fn_u = { .cbc = GLUE_CBC_FUNC_CAST(twofish_dec_blk_cbc_3way) }
166 	}, {
167 		.num_blocks = 1,
168 		.fn_u = { .cbc = GLUE_CBC_FUNC_CAST(twofish_dec_blk) }
169 	} }
170 };
171 
172 static const struct common_glue_ctx twofish_dec_xts = {
173 	.num_funcs = 2,
174 	.fpu_blocks_limit = TWOFISH_PARALLEL_BLOCKS,
175 
176 	.funcs = { {
177 		.num_blocks = TWOFISH_PARALLEL_BLOCKS,
178 		.fn_u = { .xts = GLUE_XTS_FUNC_CAST(twofish_xts_dec_8way) }
179 	}, {
180 		.num_blocks = 1,
181 		.fn_u = { .xts = GLUE_XTS_FUNC_CAST(twofish_xts_dec) }
182 	} }
183 };
184 
185 static int ecb_encrypt(struct blkcipher_desc *desc, struct scatterlist *dst,
186 		       struct scatterlist *src, unsigned int nbytes)
187 {
188 	return glue_ecb_crypt_128bit(&twofish_enc, desc, dst, src, nbytes);
189 }
190 
191 static int ecb_decrypt(struct blkcipher_desc *desc, struct scatterlist *dst,
192 		       struct scatterlist *src, unsigned int nbytes)
193 {
194 	return glue_ecb_crypt_128bit(&twofish_dec, desc, dst, src, nbytes);
195 }
196 
197 static int cbc_encrypt(struct blkcipher_desc *desc, struct scatterlist *dst,
198 		       struct scatterlist *src, unsigned int nbytes)
199 {
200 	return glue_cbc_encrypt_128bit(GLUE_FUNC_CAST(twofish_enc_blk), desc,
201 				       dst, src, nbytes);
202 }
203 
204 static int cbc_decrypt(struct blkcipher_desc *desc, struct scatterlist *dst,
205 		       struct scatterlist *src, unsigned int nbytes)
206 {
207 	return glue_cbc_decrypt_128bit(&twofish_dec_cbc, desc, dst, src,
208 				       nbytes);
209 }
210 
211 static int ctr_crypt(struct blkcipher_desc *desc, struct scatterlist *dst,
212 		     struct scatterlist *src, unsigned int nbytes)
213 {
214 	return glue_ctr_crypt_128bit(&twofish_ctr, desc, dst, src, nbytes);
215 }
216 
217 static inline bool twofish_fpu_begin(bool fpu_enabled, unsigned int nbytes)
218 {
219 	return glue_fpu_begin(TF_BLOCK_SIZE, TWOFISH_PARALLEL_BLOCKS, NULL,
220 			      fpu_enabled, nbytes);
221 }
222 
223 static inline void twofish_fpu_end(bool fpu_enabled)
224 {
225 	glue_fpu_end(fpu_enabled);
226 }
227 
228 struct crypt_priv {
229 	struct twofish_ctx *ctx;
230 	bool fpu_enabled;
231 };
232 
233 static void encrypt_callback(void *priv, u8 *srcdst, unsigned int nbytes)
234 {
235 	const unsigned int bsize = TF_BLOCK_SIZE;
236 	struct crypt_priv *ctx = priv;
237 	int i;
238 
239 	ctx->fpu_enabled = twofish_fpu_begin(ctx->fpu_enabled, nbytes);
240 
241 	if (nbytes == bsize * TWOFISH_PARALLEL_BLOCKS) {
242 		twofish_ecb_enc_8way(ctx->ctx, srcdst, srcdst);
243 		return;
244 	}
245 
246 	for (i = 0; i < nbytes / (bsize * 3); i++, srcdst += bsize * 3)
247 		twofish_enc_blk_3way(ctx->ctx, srcdst, srcdst);
248 
249 	nbytes %= bsize * 3;
250 
251 	for (i = 0; i < nbytes / bsize; i++, srcdst += bsize)
252 		twofish_enc_blk(ctx->ctx, srcdst, srcdst);
253 }
254 
255 static void decrypt_callback(void *priv, u8 *srcdst, unsigned int nbytes)
256 {
257 	const unsigned int bsize = TF_BLOCK_SIZE;
258 	struct crypt_priv *ctx = priv;
259 	int i;
260 
261 	ctx->fpu_enabled = twofish_fpu_begin(ctx->fpu_enabled, nbytes);
262 
263 	if (nbytes == bsize * TWOFISH_PARALLEL_BLOCKS) {
264 		twofish_ecb_dec_8way(ctx->ctx, srcdst, srcdst);
265 		return;
266 	}
267 
268 	for (i = 0; i < nbytes / (bsize * 3); i++, srcdst += bsize * 3)
269 		twofish_dec_blk_3way(ctx->ctx, srcdst, srcdst);
270 
271 	nbytes %= bsize * 3;
272 
273 	for (i = 0; i < nbytes / bsize; i++, srcdst += bsize)
274 		twofish_dec_blk(ctx->ctx, srcdst, srcdst);
275 }
276 
277 static int lrw_encrypt(struct blkcipher_desc *desc, struct scatterlist *dst,
278 		       struct scatterlist *src, unsigned int nbytes)
279 {
280 	struct twofish_lrw_ctx *ctx = crypto_blkcipher_ctx(desc->tfm);
281 	be128 buf[TWOFISH_PARALLEL_BLOCKS];
282 	struct crypt_priv crypt_ctx = {
283 		.ctx = &ctx->twofish_ctx,
284 		.fpu_enabled = false,
285 	};
286 	struct lrw_crypt_req req = {
287 		.tbuf = buf,
288 		.tbuflen = sizeof(buf),
289 
290 		.table_ctx = &ctx->lrw_table,
291 		.crypt_ctx = &crypt_ctx,
292 		.crypt_fn = encrypt_callback,
293 	};
294 	int ret;
295 
296 	desc->flags &= ~CRYPTO_TFM_REQ_MAY_SLEEP;
297 	ret = lrw_crypt(desc, dst, src, nbytes, &req);
298 	twofish_fpu_end(crypt_ctx.fpu_enabled);
299 
300 	return ret;
301 }
302 
303 static int lrw_decrypt(struct blkcipher_desc *desc, struct scatterlist *dst,
304 		       struct scatterlist *src, unsigned int nbytes)
305 {
306 	struct twofish_lrw_ctx *ctx = crypto_blkcipher_ctx(desc->tfm);
307 	be128 buf[TWOFISH_PARALLEL_BLOCKS];
308 	struct crypt_priv crypt_ctx = {
309 		.ctx = &ctx->twofish_ctx,
310 		.fpu_enabled = false,
311 	};
312 	struct lrw_crypt_req req = {
313 		.tbuf = buf,
314 		.tbuflen = sizeof(buf),
315 
316 		.table_ctx = &ctx->lrw_table,
317 		.crypt_ctx = &crypt_ctx,
318 		.crypt_fn = decrypt_callback,
319 	};
320 	int ret;
321 
322 	desc->flags &= ~CRYPTO_TFM_REQ_MAY_SLEEP;
323 	ret = lrw_crypt(desc, dst, src, nbytes, &req);
324 	twofish_fpu_end(crypt_ctx.fpu_enabled);
325 
326 	return ret;
327 }
328 
329 static int xts_encrypt(struct blkcipher_desc *desc, struct scatterlist *dst,
330 		       struct scatterlist *src, unsigned int nbytes)
331 {
332 	struct twofish_xts_ctx *ctx = crypto_blkcipher_ctx(desc->tfm);
333 
334 	return glue_xts_crypt_128bit(&twofish_enc_xts, desc, dst, src, nbytes,
335 				     XTS_TWEAK_CAST(twofish_enc_blk),
336 				     &ctx->tweak_ctx, &ctx->crypt_ctx);
337 }
338 
339 static int xts_decrypt(struct blkcipher_desc *desc, struct scatterlist *dst,
340 		       struct scatterlist *src, unsigned int nbytes)
341 {
342 	struct twofish_xts_ctx *ctx = crypto_blkcipher_ctx(desc->tfm);
343 
344 	return glue_xts_crypt_128bit(&twofish_dec_xts, desc, dst, src, nbytes,
345 				     XTS_TWEAK_CAST(twofish_enc_blk),
346 				     &ctx->tweak_ctx, &ctx->crypt_ctx);
347 }
348 
349 static struct crypto_alg twofish_algs[10] = { {
350 	.cra_name		= "__ecb-twofish-avx",
351 	.cra_driver_name	= "__driver-ecb-twofish-avx",
352 	.cra_priority		= 0,
353 	.cra_flags		= CRYPTO_ALG_TYPE_BLKCIPHER,
354 	.cra_blocksize		= TF_BLOCK_SIZE,
355 	.cra_ctxsize		= sizeof(struct twofish_ctx),
356 	.cra_alignmask		= 0,
357 	.cra_type		= &crypto_blkcipher_type,
358 	.cra_module		= THIS_MODULE,
359 	.cra_u = {
360 		.blkcipher = {
361 			.min_keysize	= TF_MIN_KEY_SIZE,
362 			.max_keysize	= TF_MAX_KEY_SIZE,
363 			.setkey		= twofish_setkey,
364 			.encrypt	= ecb_encrypt,
365 			.decrypt	= ecb_decrypt,
366 		},
367 	},
368 }, {
369 	.cra_name		= "__cbc-twofish-avx",
370 	.cra_driver_name	= "__driver-cbc-twofish-avx",
371 	.cra_priority		= 0,
372 	.cra_flags		= CRYPTO_ALG_TYPE_BLKCIPHER,
373 	.cra_blocksize		= TF_BLOCK_SIZE,
374 	.cra_ctxsize		= sizeof(struct twofish_ctx),
375 	.cra_alignmask		= 0,
376 	.cra_type		= &crypto_blkcipher_type,
377 	.cra_module		= THIS_MODULE,
378 	.cra_u = {
379 		.blkcipher = {
380 			.min_keysize	= TF_MIN_KEY_SIZE,
381 			.max_keysize	= TF_MAX_KEY_SIZE,
382 			.setkey		= twofish_setkey,
383 			.encrypt	= cbc_encrypt,
384 			.decrypt	= cbc_decrypt,
385 		},
386 	},
387 }, {
388 	.cra_name		= "__ctr-twofish-avx",
389 	.cra_driver_name	= "__driver-ctr-twofish-avx",
390 	.cra_priority		= 0,
391 	.cra_flags		= CRYPTO_ALG_TYPE_BLKCIPHER,
392 	.cra_blocksize		= 1,
393 	.cra_ctxsize		= sizeof(struct twofish_ctx),
394 	.cra_alignmask		= 0,
395 	.cra_type		= &crypto_blkcipher_type,
396 	.cra_module		= THIS_MODULE,
397 	.cra_u = {
398 		.blkcipher = {
399 			.min_keysize	= TF_MIN_KEY_SIZE,
400 			.max_keysize	= TF_MAX_KEY_SIZE,
401 			.ivsize		= TF_BLOCK_SIZE,
402 			.setkey		= twofish_setkey,
403 			.encrypt	= ctr_crypt,
404 			.decrypt	= ctr_crypt,
405 		},
406 	},
407 }, {
408 	.cra_name		= "__lrw-twofish-avx",
409 	.cra_driver_name	= "__driver-lrw-twofish-avx",
410 	.cra_priority		= 0,
411 	.cra_flags		= CRYPTO_ALG_TYPE_BLKCIPHER,
412 	.cra_blocksize		= TF_BLOCK_SIZE,
413 	.cra_ctxsize		= sizeof(struct twofish_lrw_ctx),
414 	.cra_alignmask		= 0,
415 	.cra_type		= &crypto_blkcipher_type,
416 	.cra_module		= THIS_MODULE,
417 	.cra_exit		= lrw_twofish_exit_tfm,
418 	.cra_u = {
419 		.blkcipher = {
420 			.min_keysize	= TF_MIN_KEY_SIZE +
421 					  TF_BLOCK_SIZE,
422 			.max_keysize	= TF_MAX_KEY_SIZE +
423 					  TF_BLOCK_SIZE,
424 			.ivsize		= TF_BLOCK_SIZE,
425 			.setkey		= lrw_twofish_setkey,
426 			.encrypt	= lrw_encrypt,
427 			.decrypt	= lrw_decrypt,
428 		},
429 	},
430 }, {
431 	.cra_name		= "__xts-twofish-avx",
432 	.cra_driver_name	= "__driver-xts-twofish-avx",
433 	.cra_priority		= 0,
434 	.cra_flags		= CRYPTO_ALG_TYPE_BLKCIPHER,
435 	.cra_blocksize		= TF_BLOCK_SIZE,
436 	.cra_ctxsize		= sizeof(struct twofish_xts_ctx),
437 	.cra_alignmask		= 0,
438 	.cra_type		= &crypto_blkcipher_type,
439 	.cra_module		= THIS_MODULE,
440 	.cra_u = {
441 		.blkcipher = {
442 			.min_keysize	= TF_MIN_KEY_SIZE * 2,
443 			.max_keysize	= TF_MAX_KEY_SIZE * 2,
444 			.ivsize		= TF_BLOCK_SIZE,
445 			.setkey		= xts_twofish_setkey,
446 			.encrypt	= xts_encrypt,
447 			.decrypt	= xts_decrypt,
448 		},
449 	},
450 }, {
451 	.cra_name		= "ecb(twofish)",
452 	.cra_driver_name	= "ecb-twofish-avx",
453 	.cra_priority		= 400,
454 	.cra_flags		= CRYPTO_ALG_TYPE_ABLKCIPHER | CRYPTO_ALG_ASYNC,
455 	.cra_blocksize		= TF_BLOCK_SIZE,
456 	.cra_ctxsize		= sizeof(struct async_helper_ctx),
457 	.cra_alignmask		= 0,
458 	.cra_type		= &crypto_ablkcipher_type,
459 	.cra_module		= THIS_MODULE,
460 	.cra_init		= ablk_init,
461 	.cra_exit		= ablk_exit,
462 	.cra_u = {
463 		.ablkcipher = {
464 			.min_keysize	= TF_MIN_KEY_SIZE,
465 			.max_keysize	= TF_MAX_KEY_SIZE,
466 			.setkey		= ablk_set_key,
467 			.encrypt	= ablk_encrypt,
468 			.decrypt	= ablk_decrypt,
469 		},
470 	},
471 }, {
472 	.cra_name		= "cbc(twofish)",
473 	.cra_driver_name	= "cbc-twofish-avx",
474 	.cra_priority		= 400,
475 	.cra_flags		= CRYPTO_ALG_TYPE_ABLKCIPHER | CRYPTO_ALG_ASYNC,
476 	.cra_blocksize		= TF_BLOCK_SIZE,
477 	.cra_ctxsize		= sizeof(struct async_helper_ctx),
478 	.cra_alignmask		= 0,
479 	.cra_type		= &crypto_ablkcipher_type,
480 	.cra_module		= THIS_MODULE,
481 	.cra_init		= ablk_init,
482 	.cra_exit		= ablk_exit,
483 	.cra_u = {
484 		.ablkcipher = {
485 			.min_keysize	= TF_MIN_KEY_SIZE,
486 			.max_keysize	= TF_MAX_KEY_SIZE,
487 			.ivsize		= TF_BLOCK_SIZE,
488 			.setkey		= ablk_set_key,
489 			.encrypt	= __ablk_encrypt,
490 			.decrypt	= ablk_decrypt,
491 		},
492 	},
493 }, {
494 	.cra_name		= "ctr(twofish)",
495 	.cra_driver_name	= "ctr-twofish-avx",
496 	.cra_priority		= 400,
497 	.cra_flags		= CRYPTO_ALG_TYPE_ABLKCIPHER | CRYPTO_ALG_ASYNC,
498 	.cra_blocksize		= 1,
499 	.cra_ctxsize		= sizeof(struct async_helper_ctx),
500 	.cra_alignmask		= 0,
501 	.cra_type		= &crypto_ablkcipher_type,
502 	.cra_module		= THIS_MODULE,
503 	.cra_init		= ablk_init,
504 	.cra_exit		= ablk_exit,
505 	.cra_u = {
506 		.ablkcipher = {
507 			.min_keysize	= TF_MIN_KEY_SIZE,
508 			.max_keysize	= TF_MAX_KEY_SIZE,
509 			.ivsize		= TF_BLOCK_SIZE,
510 			.setkey		= ablk_set_key,
511 			.encrypt	= ablk_encrypt,
512 			.decrypt	= ablk_encrypt,
513 			.geniv		= "chainiv",
514 		},
515 	},
516 }, {
517 	.cra_name		= "lrw(twofish)",
518 	.cra_driver_name	= "lrw-twofish-avx",
519 	.cra_priority		= 400,
520 	.cra_flags		= CRYPTO_ALG_TYPE_ABLKCIPHER | CRYPTO_ALG_ASYNC,
521 	.cra_blocksize		= TF_BLOCK_SIZE,
522 	.cra_ctxsize		= sizeof(struct async_helper_ctx),
523 	.cra_alignmask		= 0,
524 	.cra_type		= &crypto_ablkcipher_type,
525 	.cra_module		= THIS_MODULE,
526 	.cra_init		= ablk_init,
527 	.cra_exit		= ablk_exit,
528 	.cra_u = {
529 		.ablkcipher = {
530 			.min_keysize	= TF_MIN_KEY_SIZE +
531 					  TF_BLOCK_SIZE,
532 			.max_keysize	= TF_MAX_KEY_SIZE +
533 					  TF_BLOCK_SIZE,
534 			.ivsize		= TF_BLOCK_SIZE,
535 			.setkey		= ablk_set_key,
536 			.encrypt	= ablk_encrypt,
537 			.decrypt	= ablk_decrypt,
538 		},
539 	},
540 }, {
541 	.cra_name		= "xts(twofish)",
542 	.cra_driver_name	= "xts-twofish-avx",
543 	.cra_priority		= 400,
544 	.cra_flags		= CRYPTO_ALG_TYPE_ABLKCIPHER | CRYPTO_ALG_ASYNC,
545 	.cra_blocksize		= TF_BLOCK_SIZE,
546 	.cra_ctxsize		= sizeof(struct async_helper_ctx),
547 	.cra_alignmask		= 0,
548 	.cra_type		= &crypto_ablkcipher_type,
549 	.cra_module		= THIS_MODULE,
550 	.cra_init		= ablk_init,
551 	.cra_exit		= ablk_exit,
552 	.cra_u = {
553 		.ablkcipher = {
554 			.min_keysize	= TF_MIN_KEY_SIZE * 2,
555 			.max_keysize	= TF_MAX_KEY_SIZE * 2,
556 			.ivsize		= TF_BLOCK_SIZE,
557 			.setkey		= ablk_set_key,
558 			.encrypt	= ablk_encrypt,
559 			.decrypt	= ablk_decrypt,
560 		},
561 	},
562 } };
563 
564 static int __init twofish_init(void)
565 {
566 	u64 xcr0;
567 
568 	if (!cpu_has_avx || !cpu_has_osxsave) {
569 		printk(KERN_INFO "AVX instructions are not detected.\n");
570 		return -ENODEV;
571 	}
572 
573 	xcr0 = xgetbv(XCR_XFEATURE_ENABLED_MASK);
574 	if ((xcr0 & (XSTATE_SSE | XSTATE_YMM)) != (XSTATE_SSE | XSTATE_YMM)) {
575 		printk(KERN_INFO "AVX detected but unusable.\n");
576 		return -ENODEV;
577 	}
578 
579 	return crypto_register_algs(twofish_algs, ARRAY_SIZE(twofish_algs));
580 }
581 
582 static void __exit twofish_exit(void)
583 {
584 	crypto_unregister_algs(twofish_algs, ARRAY_SIZE(twofish_algs));
585 }
586 
587 module_init(twofish_init);
588 module_exit(twofish_exit);
589 
590 MODULE_DESCRIPTION("Twofish Cipher Algorithm, AVX optimized");
591 MODULE_LICENSE("GPL");
592 MODULE_ALIAS("twofish");
593