1 /*
2  * Glue Code for SSE2 assembler versions of Serpent Cipher
3  *
4  * Copyright (c) 2011 Jussi Kivilinna <jussi.kivilinna@mbnet.fi>
5  *
6  * Glue code based on aesni-intel_glue.c by:
7  *  Copyright (C) 2008, Intel Corp.
8  *    Author: Huang Ying <ying.huang@intel.com>
9  *
10  * CBC & ECB parts based on code (crypto/cbc.c,ecb.c) by:
11  *   Copyright (c) 2006 Herbert Xu <herbert@gondor.apana.org.au>
12  * CTR part based on code (crypto/ctr.c) by:
13  *   (C) Copyright IBM Corp. 2007 - Joy Latten <latten@us.ibm.com>
14  *
15  * This program is free software; you can redistribute it and/or modify
16  * it under the terms of the GNU General Public License as published by
17  * the Free Software Foundation; either version 2 of the License, or
18  * (at your option) any later version.
19  *
20  * This program is distributed in the hope that it will be useful,
21  * but WITHOUT ANY WARRANTY; without even the implied warranty of
22  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
23  * GNU General Public License for more details.
24  *
25  * You should have received a copy of the GNU General Public License
26  * along with this program; if not, write to the Free Software
27  * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307
28  * USA
29  *
30  */
31 
32 #include <linux/module.h>
33 #include <linux/hardirq.h>
34 #include <linux/types.h>
35 #include <linux/crypto.h>
36 #include <linux/err.h>
37 #include <crypto/ablk_helper.h>
38 #include <crypto/algapi.h>
39 #include <crypto/serpent.h>
40 #include <crypto/cryptd.h>
41 #include <crypto/b128ops.h>
42 #include <crypto/ctr.h>
43 #include <crypto/lrw.h>
44 #include <crypto/xts.h>
45 #include <asm/crypto/serpent-sse2.h>
46 #include <asm/crypto/glue_helper.h>
47 
48 static void serpent_decrypt_cbc_xway(void *ctx, u128 *dst, const u128 *src)
49 {
50 	u128 ivs[SERPENT_PARALLEL_BLOCKS - 1];
51 	unsigned int j;
52 
53 	for (j = 0; j < SERPENT_PARALLEL_BLOCKS - 1; j++)
54 		ivs[j] = src[j];
55 
56 	serpent_dec_blk_xway(ctx, (u8 *)dst, (u8 *)src);
57 
58 	for (j = 0; j < SERPENT_PARALLEL_BLOCKS - 1; j++)
59 		u128_xor(dst + (j + 1), dst + (j + 1), ivs + j);
60 }
61 
62 static void serpent_crypt_ctr(void *ctx, u128 *dst, const u128 *src, le128 *iv)
63 {
64 	be128 ctrblk;
65 
66 	le128_to_be128(&ctrblk, iv);
67 	le128_inc(iv);
68 
69 	__serpent_encrypt(ctx, (u8 *)&ctrblk, (u8 *)&ctrblk);
70 	u128_xor(dst, src, (u128 *)&ctrblk);
71 }
72 
73 static void serpent_crypt_ctr_xway(void *ctx, u128 *dst, const u128 *src,
74 				   le128 *iv)
75 {
76 	be128 ctrblks[SERPENT_PARALLEL_BLOCKS];
77 	unsigned int i;
78 
79 	for (i = 0; i < SERPENT_PARALLEL_BLOCKS; i++) {
80 		if (dst != src)
81 			dst[i] = src[i];
82 
83 		le128_to_be128(&ctrblks[i], iv);
84 		le128_inc(iv);
85 	}
86 
87 	serpent_enc_blk_xway_xor(ctx, (u8 *)dst, (u8 *)ctrblks);
88 }
89 
90 static const struct common_glue_ctx serpent_enc = {
91 	.num_funcs = 2,
92 	.fpu_blocks_limit = SERPENT_PARALLEL_BLOCKS,
93 
94 	.funcs = { {
95 		.num_blocks = SERPENT_PARALLEL_BLOCKS,
96 		.fn_u = { .ecb = GLUE_FUNC_CAST(serpent_enc_blk_xway) }
97 	}, {
98 		.num_blocks = 1,
99 		.fn_u = { .ecb = GLUE_FUNC_CAST(__serpent_encrypt) }
100 	} }
101 };
102 
103 static const struct common_glue_ctx serpent_ctr = {
104 	.num_funcs = 2,
105 	.fpu_blocks_limit = SERPENT_PARALLEL_BLOCKS,
106 
107 	.funcs = { {
108 		.num_blocks = SERPENT_PARALLEL_BLOCKS,
109 		.fn_u = { .ctr = GLUE_CTR_FUNC_CAST(serpent_crypt_ctr_xway) }
110 	}, {
111 		.num_blocks = 1,
112 		.fn_u = { .ctr = GLUE_CTR_FUNC_CAST(serpent_crypt_ctr) }
113 	} }
114 };
115 
116 static const struct common_glue_ctx serpent_dec = {
117 	.num_funcs = 2,
118 	.fpu_blocks_limit = SERPENT_PARALLEL_BLOCKS,
119 
120 	.funcs = { {
121 		.num_blocks = SERPENT_PARALLEL_BLOCKS,
122 		.fn_u = { .ecb = GLUE_FUNC_CAST(serpent_dec_blk_xway) }
123 	}, {
124 		.num_blocks = 1,
125 		.fn_u = { .ecb = GLUE_FUNC_CAST(__serpent_decrypt) }
126 	} }
127 };
128 
129 static const struct common_glue_ctx serpent_dec_cbc = {
130 	.num_funcs = 2,
131 	.fpu_blocks_limit = SERPENT_PARALLEL_BLOCKS,
132 
133 	.funcs = { {
134 		.num_blocks = SERPENT_PARALLEL_BLOCKS,
135 		.fn_u = { .cbc = GLUE_CBC_FUNC_CAST(serpent_decrypt_cbc_xway) }
136 	}, {
137 		.num_blocks = 1,
138 		.fn_u = { .cbc = GLUE_CBC_FUNC_CAST(__serpent_decrypt) }
139 	} }
140 };
141 
142 static int ecb_encrypt(struct blkcipher_desc *desc, struct scatterlist *dst,
143 		       struct scatterlist *src, unsigned int nbytes)
144 {
145 	return glue_ecb_crypt_128bit(&serpent_enc, desc, dst, src, nbytes);
146 }
147 
148 static int ecb_decrypt(struct blkcipher_desc *desc, struct scatterlist *dst,
149 		       struct scatterlist *src, unsigned int nbytes)
150 {
151 	return glue_ecb_crypt_128bit(&serpent_dec, desc, dst, src, nbytes);
152 }
153 
154 static int cbc_encrypt(struct blkcipher_desc *desc, struct scatterlist *dst,
155 		       struct scatterlist *src, unsigned int nbytes)
156 {
157 	return glue_cbc_encrypt_128bit(GLUE_FUNC_CAST(__serpent_encrypt), desc,
158 				     dst, src, nbytes);
159 }
160 
161 static int cbc_decrypt(struct blkcipher_desc *desc, struct scatterlist *dst,
162 		       struct scatterlist *src, unsigned int nbytes)
163 {
164 	return glue_cbc_decrypt_128bit(&serpent_dec_cbc, desc, dst, src,
165 				       nbytes);
166 }
167 
168 static int ctr_crypt(struct blkcipher_desc *desc, struct scatterlist *dst,
169 		     struct scatterlist *src, unsigned int nbytes)
170 {
171 	return glue_ctr_crypt_128bit(&serpent_ctr, desc, dst, src, nbytes);
172 }
173 
174 static inline bool serpent_fpu_begin(bool fpu_enabled, unsigned int nbytes)
175 {
176 	return glue_fpu_begin(SERPENT_BLOCK_SIZE, SERPENT_PARALLEL_BLOCKS,
177 			      NULL, fpu_enabled, nbytes);
178 }
179 
180 static inline void serpent_fpu_end(bool fpu_enabled)
181 {
182 	glue_fpu_end(fpu_enabled);
183 }
184 
185 struct crypt_priv {
186 	struct serpent_ctx *ctx;
187 	bool fpu_enabled;
188 };
189 
190 static void encrypt_callback(void *priv, u8 *srcdst, unsigned int nbytes)
191 {
192 	const unsigned int bsize = SERPENT_BLOCK_SIZE;
193 	struct crypt_priv *ctx = priv;
194 	int i;
195 
196 	ctx->fpu_enabled = serpent_fpu_begin(ctx->fpu_enabled, nbytes);
197 
198 	if (nbytes == bsize * SERPENT_PARALLEL_BLOCKS) {
199 		serpent_enc_blk_xway(ctx->ctx, srcdst, srcdst);
200 		return;
201 	}
202 
203 	for (i = 0; i < nbytes / bsize; i++, srcdst += bsize)
204 		__serpent_encrypt(ctx->ctx, srcdst, srcdst);
205 }
206 
207 static void decrypt_callback(void *priv, u8 *srcdst, unsigned int nbytes)
208 {
209 	const unsigned int bsize = SERPENT_BLOCK_SIZE;
210 	struct crypt_priv *ctx = priv;
211 	int i;
212 
213 	ctx->fpu_enabled = serpent_fpu_begin(ctx->fpu_enabled, nbytes);
214 
215 	if (nbytes == bsize * SERPENT_PARALLEL_BLOCKS) {
216 		serpent_dec_blk_xway(ctx->ctx, srcdst, srcdst);
217 		return;
218 	}
219 
220 	for (i = 0; i < nbytes / bsize; i++, srcdst += bsize)
221 		__serpent_decrypt(ctx->ctx, srcdst, srcdst);
222 }
223 
224 struct serpent_lrw_ctx {
225 	struct lrw_table_ctx lrw_table;
226 	struct serpent_ctx serpent_ctx;
227 };
228 
229 static int lrw_serpent_setkey(struct crypto_tfm *tfm, const u8 *key,
230 			      unsigned int keylen)
231 {
232 	struct serpent_lrw_ctx *ctx = crypto_tfm_ctx(tfm);
233 	int err;
234 
235 	err = __serpent_setkey(&ctx->serpent_ctx, key, keylen -
236 							SERPENT_BLOCK_SIZE);
237 	if (err)
238 		return err;
239 
240 	return lrw_init_table(&ctx->lrw_table, key + keylen -
241 						SERPENT_BLOCK_SIZE);
242 }
243 
244 static int lrw_encrypt(struct blkcipher_desc *desc, struct scatterlist *dst,
245 		       struct scatterlist *src, unsigned int nbytes)
246 {
247 	struct serpent_lrw_ctx *ctx = crypto_blkcipher_ctx(desc->tfm);
248 	be128 buf[SERPENT_PARALLEL_BLOCKS];
249 	struct crypt_priv crypt_ctx = {
250 		.ctx = &ctx->serpent_ctx,
251 		.fpu_enabled = false,
252 	};
253 	struct lrw_crypt_req req = {
254 		.tbuf = buf,
255 		.tbuflen = sizeof(buf),
256 
257 		.table_ctx = &ctx->lrw_table,
258 		.crypt_ctx = &crypt_ctx,
259 		.crypt_fn = encrypt_callback,
260 	};
261 	int ret;
262 
263 	desc->flags &= ~CRYPTO_TFM_REQ_MAY_SLEEP;
264 	ret = lrw_crypt(desc, dst, src, nbytes, &req);
265 	serpent_fpu_end(crypt_ctx.fpu_enabled);
266 
267 	return ret;
268 }
269 
270 static int lrw_decrypt(struct blkcipher_desc *desc, struct scatterlist *dst,
271 		       struct scatterlist *src, unsigned int nbytes)
272 {
273 	struct serpent_lrw_ctx *ctx = crypto_blkcipher_ctx(desc->tfm);
274 	be128 buf[SERPENT_PARALLEL_BLOCKS];
275 	struct crypt_priv crypt_ctx = {
276 		.ctx = &ctx->serpent_ctx,
277 		.fpu_enabled = false,
278 	};
279 	struct lrw_crypt_req req = {
280 		.tbuf = buf,
281 		.tbuflen = sizeof(buf),
282 
283 		.table_ctx = &ctx->lrw_table,
284 		.crypt_ctx = &crypt_ctx,
285 		.crypt_fn = decrypt_callback,
286 	};
287 	int ret;
288 
289 	desc->flags &= ~CRYPTO_TFM_REQ_MAY_SLEEP;
290 	ret = lrw_crypt(desc, dst, src, nbytes, &req);
291 	serpent_fpu_end(crypt_ctx.fpu_enabled);
292 
293 	return ret;
294 }
295 
296 static void lrw_exit_tfm(struct crypto_tfm *tfm)
297 {
298 	struct serpent_lrw_ctx *ctx = crypto_tfm_ctx(tfm);
299 
300 	lrw_free_table(&ctx->lrw_table);
301 }
302 
303 struct serpent_xts_ctx {
304 	struct serpent_ctx tweak_ctx;
305 	struct serpent_ctx crypt_ctx;
306 };
307 
308 static int xts_serpent_setkey(struct crypto_tfm *tfm, const u8 *key,
309 			      unsigned int keylen)
310 {
311 	struct serpent_xts_ctx *ctx = crypto_tfm_ctx(tfm);
312 	u32 *flags = &tfm->crt_flags;
313 	int err;
314 
315 	/* key consists of keys of equal size concatenated, therefore
316 	 * the length must be even
317 	 */
318 	if (keylen % 2) {
319 		*flags |= CRYPTO_TFM_RES_BAD_KEY_LEN;
320 		return -EINVAL;
321 	}
322 
323 	/* first half of xts-key is for crypt */
324 	err = __serpent_setkey(&ctx->crypt_ctx, key, keylen / 2);
325 	if (err)
326 		return err;
327 
328 	/* second half of xts-key is for tweak */
329 	return __serpent_setkey(&ctx->tweak_ctx, key + keylen / 2, keylen / 2);
330 }
331 
332 static int xts_encrypt(struct blkcipher_desc *desc, struct scatterlist *dst,
333 		       struct scatterlist *src, unsigned int nbytes)
334 {
335 	struct serpent_xts_ctx *ctx = crypto_blkcipher_ctx(desc->tfm);
336 	be128 buf[SERPENT_PARALLEL_BLOCKS];
337 	struct crypt_priv crypt_ctx = {
338 		.ctx = &ctx->crypt_ctx,
339 		.fpu_enabled = false,
340 	};
341 	struct xts_crypt_req req = {
342 		.tbuf = buf,
343 		.tbuflen = sizeof(buf),
344 
345 		.tweak_ctx = &ctx->tweak_ctx,
346 		.tweak_fn = XTS_TWEAK_CAST(__serpent_encrypt),
347 		.crypt_ctx = &crypt_ctx,
348 		.crypt_fn = encrypt_callback,
349 	};
350 	int ret;
351 
352 	desc->flags &= ~CRYPTO_TFM_REQ_MAY_SLEEP;
353 	ret = xts_crypt(desc, dst, src, nbytes, &req);
354 	serpent_fpu_end(crypt_ctx.fpu_enabled);
355 
356 	return ret;
357 }
358 
359 static int xts_decrypt(struct blkcipher_desc *desc, struct scatterlist *dst,
360 		       struct scatterlist *src, unsigned int nbytes)
361 {
362 	struct serpent_xts_ctx *ctx = crypto_blkcipher_ctx(desc->tfm);
363 	be128 buf[SERPENT_PARALLEL_BLOCKS];
364 	struct crypt_priv crypt_ctx = {
365 		.ctx = &ctx->crypt_ctx,
366 		.fpu_enabled = false,
367 	};
368 	struct xts_crypt_req req = {
369 		.tbuf = buf,
370 		.tbuflen = sizeof(buf),
371 
372 		.tweak_ctx = &ctx->tweak_ctx,
373 		.tweak_fn = XTS_TWEAK_CAST(__serpent_encrypt),
374 		.crypt_ctx = &crypt_ctx,
375 		.crypt_fn = decrypt_callback,
376 	};
377 	int ret;
378 
379 	desc->flags &= ~CRYPTO_TFM_REQ_MAY_SLEEP;
380 	ret = xts_crypt(desc, dst, src, nbytes, &req);
381 	serpent_fpu_end(crypt_ctx.fpu_enabled);
382 
383 	return ret;
384 }
385 
386 static struct crypto_alg serpent_algs[10] = { {
387 	.cra_name		= "__ecb-serpent-sse2",
388 	.cra_driver_name	= "__driver-ecb-serpent-sse2",
389 	.cra_priority		= 0,
390 	.cra_flags		= CRYPTO_ALG_TYPE_BLKCIPHER |
391 				  CRYPTO_ALG_INTERNAL,
392 	.cra_blocksize		= SERPENT_BLOCK_SIZE,
393 	.cra_ctxsize		= sizeof(struct serpent_ctx),
394 	.cra_alignmask		= 0,
395 	.cra_type		= &crypto_blkcipher_type,
396 	.cra_module		= THIS_MODULE,
397 	.cra_u = {
398 		.blkcipher = {
399 			.min_keysize	= SERPENT_MIN_KEY_SIZE,
400 			.max_keysize	= SERPENT_MAX_KEY_SIZE,
401 			.setkey		= serpent_setkey,
402 			.encrypt	= ecb_encrypt,
403 			.decrypt	= ecb_decrypt,
404 		},
405 	},
406 }, {
407 	.cra_name		= "__cbc-serpent-sse2",
408 	.cra_driver_name	= "__driver-cbc-serpent-sse2",
409 	.cra_priority		= 0,
410 	.cra_flags		= CRYPTO_ALG_TYPE_BLKCIPHER |
411 				  CRYPTO_ALG_INTERNAL,
412 	.cra_blocksize		= SERPENT_BLOCK_SIZE,
413 	.cra_ctxsize		= sizeof(struct serpent_ctx),
414 	.cra_alignmask		= 0,
415 	.cra_type		= &crypto_blkcipher_type,
416 	.cra_module		= THIS_MODULE,
417 	.cra_u = {
418 		.blkcipher = {
419 			.min_keysize	= SERPENT_MIN_KEY_SIZE,
420 			.max_keysize	= SERPENT_MAX_KEY_SIZE,
421 			.setkey		= serpent_setkey,
422 			.encrypt	= cbc_encrypt,
423 			.decrypt	= cbc_decrypt,
424 		},
425 	},
426 }, {
427 	.cra_name		= "__ctr-serpent-sse2",
428 	.cra_driver_name	= "__driver-ctr-serpent-sse2",
429 	.cra_priority		= 0,
430 	.cra_flags		= CRYPTO_ALG_TYPE_BLKCIPHER |
431 				  CRYPTO_ALG_INTERNAL,
432 	.cra_blocksize		= 1,
433 	.cra_ctxsize		= sizeof(struct serpent_ctx),
434 	.cra_alignmask		= 0,
435 	.cra_type		= &crypto_blkcipher_type,
436 	.cra_module		= THIS_MODULE,
437 	.cra_u = {
438 		.blkcipher = {
439 			.min_keysize	= SERPENT_MIN_KEY_SIZE,
440 			.max_keysize	= SERPENT_MAX_KEY_SIZE,
441 			.ivsize		= SERPENT_BLOCK_SIZE,
442 			.setkey		= serpent_setkey,
443 			.encrypt	= ctr_crypt,
444 			.decrypt	= ctr_crypt,
445 		},
446 	},
447 }, {
448 	.cra_name		= "__lrw-serpent-sse2",
449 	.cra_driver_name	= "__driver-lrw-serpent-sse2",
450 	.cra_priority		= 0,
451 	.cra_flags		= CRYPTO_ALG_TYPE_BLKCIPHER |
452 				  CRYPTO_ALG_INTERNAL,
453 	.cra_blocksize		= SERPENT_BLOCK_SIZE,
454 	.cra_ctxsize		= sizeof(struct serpent_lrw_ctx),
455 	.cra_alignmask		= 0,
456 	.cra_type		= &crypto_blkcipher_type,
457 	.cra_module		= THIS_MODULE,
458 	.cra_exit		= lrw_exit_tfm,
459 	.cra_u = {
460 		.blkcipher = {
461 			.min_keysize	= SERPENT_MIN_KEY_SIZE +
462 					  SERPENT_BLOCK_SIZE,
463 			.max_keysize	= SERPENT_MAX_KEY_SIZE +
464 					  SERPENT_BLOCK_SIZE,
465 			.ivsize		= SERPENT_BLOCK_SIZE,
466 			.setkey		= lrw_serpent_setkey,
467 			.encrypt	= lrw_encrypt,
468 			.decrypt	= lrw_decrypt,
469 		},
470 	},
471 }, {
472 	.cra_name		= "__xts-serpent-sse2",
473 	.cra_driver_name	= "__driver-xts-serpent-sse2",
474 	.cra_priority		= 0,
475 	.cra_flags		= CRYPTO_ALG_TYPE_BLKCIPHER |
476 				  CRYPTO_ALG_INTERNAL,
477 	.cra_blocksize		= SERPENT_BLOCK_SIZE,
478 	.cra_ctxsize		= sizeof(struct serpent_xts_ctx),
479 	.cra_alignmask		= 0,
480 	.cra_type		= &crypto_blkcipher_type,
481 	.cra_module		= THIS_MODULE,
482 	.cra_u = {
483 		.blkcipher = {
484 			.min_keysize	= SERPENT_MIN_KEY_SIZE * 2,
485 			.max_keysize	= SERPENT_MAX_KEY_SIZE * 2,
486 			.ivsize		= SERPENT_BLOCK_SIZE,
487 			.setkey		= xts_serpent_setkey,
488 			.encrypt	= xts_encrypt,
489 			.decrypt	= xts_decrypt,
490 		},
491 	},
492 }, {
493 	.cra_name		= "ecb(serpent)",
494 	.cra_driver_name	= "ecb-serpent-sse2",
495 	.cra_priority		= 400,
496 	.cra_flags		= CRYPTO_ALG_TYPE_ABLKCIPHER | CRYPTO_ALG_ASYNC,
497 	.cra_blocksize		= SERPENT_BLOCK_SIZE,
498 	.cra_ctxsize		= sizeof(struct async_helper_ctx),
499 	.cra_alignmask		= 0,
500 	.cra_type		= &crypto_ablkcipher_type,
501 	.cra_module		= THIS_MODULE,
502 	.cra_init		= ablk_init,
503 	.cra_exit		= ablk_exit,
504 	.cra_u = {
505 		.ablkcipher = {
506 			.min_keysize	= SERPENT_MIN_KEY_SIZE,
507 			.max_keysize	= SERPENT_MAX_KEY_SIZE,
508 			.setkey		= ablk_set_key,
509 			.encrypt	= ablk_encrypt,
510 			.decrypt	= ablk_decrypt,
511 		},
512 	},
513 }, {
514 	.cra_name		= "cbc(serpent)",
515 	.cra_driver_name	= "cbc-serpent-sse2",
516 	.cra_priority		= 400,
517 	.cra_flags		= CRYPTO_ALG_TYPE_ABLKCIPHER | CRYPTO_ALG_ASYNC,
518 	.cra_blocksize		= SERPENT_BLOCK_SIZE,
519 	.cra_ctxsize		= sizeof(struct async_helper_ctx),
520 	.cra_alignmask		= 0,
521 	.cra_type		= &crypto_ablkcipher_type,
522 	.cra_module		= THIS_MODULE,
523 	.cra_init		= ablk_init,
524 	.cra_exit		= ablk_exit,
525 	.cra_u = {
526 		.ablkcipher = {
527 			.min_keysize	= SERPENT_MIN_KEY_SIZE,
528 			.max_keysize	= SERPENT_MAX_KEY_SIZE,
529 			.ivsize		= SERPENT_BLOCK_SIZE,
530 			.setkey		= ablk_set_key,
531 			.encrypt	= __ablk_encrypt,
532 			.decrypt	= ablk_decrypt,
533 		},
534 	},
535 }, {
536 	.cra_name		= "ctr(serpent)",
537 	.cra_driver_name	= "ctr-serpent-sse2",
538 	.cra_priority		= 400,
539 	.cra_flags		= CRYPTO_ALG_TYPE_ABLKCIPHER | CRYPTO_ALG_ASYNC,
540 	.cra_blocksize		= 1,
541 	.cra_ctxsize		= sizeof(struct async_helper_ctx),
542 	.cra_alignmask		= 0,
543 	.cra_type		= &crypto_ablkcipher_type,
544 	.cra_module		= THIS_MODULE,
545 	.cra_init		= ablk_init,
546 	.cra_exit		= ablk_exit,
547 	.cra_u = {
548 		.ablkcipher = {
549 			.min_keysize	= SERPENT_MIN_KEY_SIZE,
550 			.max_keysize	= SERPENT_MAX_KEY_SIZE,
551 			.ivsize		= SERPENT_BLOCK_SIZE,
552 			.setkey		= ablk_set_key,
553 			.encrypt	= ablk_encrypt,
554 			.decrypt	= ablk_encrypt,
555 			.geniv		= "chainiv",
556 		},
557 	},
558 }, {
559 	.cra_name		= "lrw(serpent)",
560 	.cra_driver_name	= "lrw-serpent-sse2",
561 	.cra_priority		= 400,
562 	.cra_flags		= CRYPTO_ALG_TYPE_ABLKCIPHER | CRYPTO_ALG_ASYNC,
563 	.cra_blocksize		= SERPENT_BLOCK_SIZE,
564 	.cra_ctxsize		= sizeof(struct async_helper_ctx),
565 	.cra_alignmask		= 0,
566 	.cra_type		= &crypto_ablkcipher_type,
567 	.cra_module		= THIS_MODULE,
568 	.cra_init		= ablk_init,
569 	.cra_exit		= ablk_exit,
570 	.cra_u = {
571 		.ablkcipher = {
572 			.min_keysize	= SERPENT_MIN_KEY_SIZE +
573 					  SERPENT_BLOCK_SIZE,
574 			.max_keysize	= SERPENT_MAX_KEY_SIZE +
575 					  SERPENT_BLOCK_SIZE,
576 			.ivsize		= SERPENT_BLOCK_SIZE,
577 			.setkey		= ablk_set_key,
578 			.encrypt	= ablk_encrypt,
579 			.decrypt	= ablk_decrypt,
580 		},
581 	},
582 }, {
583 	.cra_name		= "xts(serpent)",
584 	.cra_driver_name	= "xts-serpent-sse2",
585 	.cra_priority		= 400,
586 	.cra_flags		= CRYPTO_ALG_TYPE_ABLKCIPHER | CRYPTO_ALG_ASYNC,
587 	.cra_blocksize		= SERPENT_BLOCK_SIZE,
588 	.cra_ctxsize		= sizeof(struct async_helper_ctx),
589 	.cra_alignmask		= 0,
590 	.cra_type		= &crypto_ablkcipher_type,
591 	.cra_module		= THIS_MODULE,
592 	.cra_init		= ablk_init,
593 	.cra_exit		= ablk_exit,
594 	.cra_u = {
595 		.ablkcipher = {
596 			.min_keysize	= SERPENT_MIN_KEY_SIZE * 2,
597 			.max_keysize	= SERPENT_MAX_KEY_SIZE * 2,
598 			.ivsize		= SERPENT_BLOCK_SIZE,
599 			.setkey		= ablk_set_key,
600 			.encrypt	= ablk_encrypt,
601 			.decrypt	= ablk_decrypt,
602 		},
603 	},
604 } };
605 
606 static int __init serpent_sse2_init(void)
607 {
608 	if (!cpu_has_xmm2) {
609 		printk(KERN_INFO "SSE2 instructions are not detected.\n");
610 		return -ENODEV;
611 	}
612 
613 	return crypto_register_algs(serpent_algs, ARRAY_SIZE(serpent_algs));
614 }
615 
616 static void __exit serpent_sse2_exit(void)
617 {
618 	crypto_unregister_algs(serpent_algs, ARRAY_SIZE(serpent_algs));
619 }
620 
621 module_init(serpent_sse2_init);
622 module_exit(serpent_sse2_exit);
623 
624 MODULE_DESCRIPTION("Serpent Cipher Algorithm, SSE2 optimized");
625 MODULE_LICENSE("GPL");
626 MODULE_ALIAS_CRYPTO("serpent");
627