1 /* 2 * Glue Code for AVX assembler versions of Serpent Cipher 3 * 4 * Copyright (C) 2012 Johannes Goetzfried 5 * <Johannes.Goetzfried@informatik.stud.uni-erlangen.de> 6 * 7 * Glue code based on serpent_sse2_glue.c by: 8 * Copyright (C) 2011 Jussi Kivilinna <jussi.kivilinna@mbnet.fi> 9 * 10 * This program is free software; you can redistribute it and/or modify 11 * it under the terms of the GNU General Public License as published by 12 * the Free Software Foundation; either version 2 of the License, or 13 * (at your option) any later version. 14 * 15 * This program is distributed in the hope that it will be useful, 16 * but WITHOUT ANY WARRANTY; without even the implied warranty of 17 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 18 * GNU General Public License for more details. 19 * 20 * You should have received a copy of the GNU General Public License 21 * along with this program; if not, write to the Free Software 22 * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 23 * USA 24 * 25 */ 26 27 #include <linux/module.h> 28 #include <linux/hardirq.h> 29 #include <linux/types.h> 30 #include <linux/crypto.h> 31 #include <linux/err.h> 32 #include <crypto/algapi.h> 33 #include <crypto/serpent.h> 34 #include <crypto/cryptd.h> 35 #include <crypto/b128ops.h> 36 #include <crypto/ctr.h> 37 #include <crypto/lrw.h> 38 #include <crypto/xts.h> 39 #include <asm/xcr.h> 40 #include <asm/xsave.h> 41 #include <asm/serpent-avx.h> 42 #include <asm/crypto/ablk_helper.h> 43 #include <asm/crypto/glue_helper.h> 44 45 static void serpent_decrypt_cbc_xway(void *ctx, u128 *dst, const u128 *src) 46 { 47 u128 ivs[SERPENT_PARALLEL_BLOCKS - 1]; 48 unsigned int j; 49 50 for (j = 0; j < SERPENT_PARALLEL_BLOCKS - 1; j++) 51 ivs[j] = src[j]; 52 53 serpent_dec_blk_xway(ctx, (u8 *)dst, (u8 *)src); 54 55 for (j = 0; j < SERPENT_PARALLEL_BLOCKS - 1; j++) 56 u128_xor(dst + (j + 1), dst + (j + 1), ivs + j); 57 } 58 59 static void serpent_crypt_ctr(void *ctx, u128 *dst, const u128 *src, u128 *iv) 60 { 61 be128 ctrblk; 62 63 u128_to_be128(&ctrblk, iv); 64 u128_inc(iv); 65 66 __serpent_encrypt(ctx, (u8 *)&ctrblk, (u8 *)&ctrblk); 67 u128_xor(dst, src, (u128 *)&ctrblk); 68 } 69 70 static void serpent_crypt_ctr_xway(void *ctx, u128 *dst, const u128 *src, 71 u128 *iv) 72 { 73 be128 ctrblks[SERPENT_PARALLEL_BLOCKS]; 74 unsigned int i; 75 76 for (i = 0; i < SERPENT_PARALLEL_BLOCKS; i++) { 77 if (dst != src) 78 dst[i] = src[i]; 79 80 u128_to_be128(&ctrblks[i], iv); 81 u128_inc(iv); 82 } 83 84 serpent_enc_blk_xway_xor(ctx, (u8 *)dst, (u8 *)ctrblks); 85 } 86 87 static const struct common_glue_ctx serpent_enc = { 88 .num_funcs = 2, 89 .fpu_blocks_limit = SERPENT_PARALLEL_BLOCKS, 90 91 .funcs = { { 92 .num_blocks = SERPENT_PARALLEL_BLOCKS, 93 .fn_u = { .ecb = GLUE_FUNC_CAST(serpent_enc_blk_xway) } 94 }, { 95 .num_blocks = 1, 96 .fn_u = { .ecb = GLUE_FUNC_CAST(__serpent_encrypt) } 97 } } 98 }; 99 100 static const struct common_glue_ctx serpent_ctr = { 101 .num_funcs = 2, 102 .fpu_blocks_limit = SERPENT_PARALLEL_BLOCKS, 103 104 .funcs = { { 105 .num_blocks = SERPENT_PARALLEL_BLOCKS, 106 .fn_u = { .ctr = GLUE_CTR_FUNC_CAST(serpent_crypt_ctr_xway) } 107 }, { 108 .num_blocks = 1, 109 .fn_u = { .ctr = GLUE_CTR_FUNC_CAST(serpent_crypt_ctr) } 110 } } 111 }; 112 113 static const struct common_glue_ctx serpent_dec = { 114 .num_funcs = 2, 115 .fpu_blocks_limit = SERPENT_PARALLEL_BLOCKS, 116 117 .funcs = { { 118 .num_blocks = SERPENT_PARALLEL_BLOCKS, 119 .fn_u = { .ecb = GLUE_FUNC_CAST(serpent_dec_blk_xway) } 120 }, { 121 .num_blocks = 1, 122 .fn_u = { .ecb = GLUE_FUNC_CAST(__serpent_decrypt) } 123 } } 124 }; 125 126 static const struct common_glue_ctx serpent_dec_cbc = { 127 .num_funcs = 2, 128 .fpu_blocks_limit = SERPENT_PARALLEL_BLOCKS, 129 130 .funcs = { { 131 .num_blocks = SERPENT_PARALLEL_BLOCKS, 132 .fn_u = { .cbc = GLUE_CBC_FUNC_CAST(serpent_decrypt_cbc_xway) } 133 }, { 134 .num_blocks = 1, 135 .fn_u = { .cbc = GLUE_CBC_FUNC_CAST(__serpent_decrypt) } 136 } } 137 }; 138 139 static int ecb_encrypt(struct blkcipher_desc *desc, struct scatterlist *dst, 140 struct scatterlist *src, unsigned int nbytes) 141 { 142 return glue_ecb_crypt_128bit(&serpent_enc, desc, dst, src, nbytes); 143 } 144 145 static int ecb_decrypt(struct blkcipher_desc *desc, struct scatterlist *dst, 146 struct scatterlist *src, unsigned int nbytes) 147 { 148 return glue_ecb_crypt_128bit(&serpent_dec, desc, dst, src, nbytes); 149 } 150 151 static int cbc_encrypt(struct blkcipher_desc *desc, struct scatterlist *dst, 152 struct scatterlist *src, unsigned int nbytes) 153 { 154 return glue_cbc_encrypt_128bit(GLUE_FUNC_CAST(__serpent_encrypt), desc, 155 dst, src, nbytes); 156 } 157 158 static int cbc_decrypt(struct blkcipher_desc *desc, struct scatterlist *dst, 159 struct scatterlist *src, unsigned int nbytes) 160 { 161 return glue_cbc_decrypt_128bit(&serpent_dec_cbc, desc, dst, src, 162 nbytes); 163 } 164 165 static int ctr_crypt(struct blkcipher_desc *desc, struct scatterlist *dst, 166 struct scatterlist *src, unsigned int nbytes) 167 { 168 return glue_ctr_crypt_128bit(&serpent_ctr, desc, dst, src, nbytes); 169 } 170 171 static inline bool serpent_fpu_begin(bool fpu_enabled, unsigned int nbytes) 172 { 173 return glue_fpu_begin(SERPENT_BLOCK_SIZE, SERPENT_PARALLEL_BLOCKS, 174 NULL, fpu_enabled, nbytes); 175 } 176 177 static inline void serpent_fpu_end(bool fpu_enabled) 178 { 179 glue_fpu_end(fpu_enabled); 180 } 181 182 struct crypt_priv { 183 struct serpent_ctx *ctx; 184 bool fpu_enabled; 185 }; 186 187 static void encrypt_callback(void *priv, u8 *srcdst, unsigned int nbytes) 188 { 189 const unsigned int bsize = SERPENT_BLOCK_SIZE; 190 struct crypt_priv *ctx = priv; 191 int i; 192 193 ctx->fpu_enabled = serpent_fpu_begin(ctx->fpu_enabled, nbytes); 194 195 if (nbytes == bsize * SERPENT_PARALLEL_BLOCKS) { 196 serpent_enc_blk_xway(ctx->ctx, srcdst, srcdst); 197 return; 198 } 199 200 for (i = 0; i < nbytes / bsize; i++, srcdst += bsize) 201 __serpent_encrypt(ctx->ctx, srcdst, srcdst); 202 } 203 204 static void decrypt_callback(void *priv, u8 *srcdst, unsigned int nbytes) 205 { 206 const unsigned int bsize = SERPENT_BLOCK_SIZE; 207 struct crypt_priv *ctx = priv; 208 int i; 209 210 ctx->fpu_enabled = serpent_fpu_begin(ctx->fpu_enabled, nbytes); 211 212 if (nbytes == bsize * SERPENT_PARALLEL_BLOCKS) { 213 serpent_dec_blk_xway(ctx->ctx, srcdst, srcdst); 214 return; 215 } 216 217 for (i = 0; i < nbytes / bsize; i++, srcdst += bsize) 218 __serpent_decrypt(ctx->ctx, srcdst, srcdst); 219 } 220 221 struct serpent_lrw_ctx { 222 struct lrw_table_ctx lrw_table; 223 struct serpent_ctx serpent_ctx; 224 }; 225 226 static int lrw_serpent_setkey(struct crypto_tfm *tfm, const u8 *key, 227 unsigned int keylen) 228 { 229 struct serpent_lrw_ctx *ctx = crypto_tfm_ctx(tfm); 230 int err; 231 232 err = __serpent_setkey(&ctx->serpent_ctx, key, keylen - 233 SERPENT_BLOCK_SIZE); 234 if (err) 235 return err; 236 237 return lrw_init_table(&ctx->lrw_table, key + keylen - 238 SERPENT_BLOCK_SIZE); 239 } 240 241 static int lrw_encrypt(struct blkcipher_desc *desc, struct scatterlist *dst, 242 struct scatterlist *src, unsigned int nbytes) 243 { 244 struct serpent_lrw_ctx *ctx = crypto_blkcipher_ctx(desc->tfm); 245 be128 buf[SERPENT_PARALLEL_BLOCKS]; 246 struct crypt_priv crypt_ctx = { 247 .ctx = &ctx->serpent_ctx, 248 .fpu_enabled = false, 249 }; 250 struct lrw_crypt_req req = { 251 .tbuf = buf, 252 .tbuflen = sizeof(buf), 253 254 .table_ctx = &ctx->lrw_table, 255 .crypt_ctx = &crypt_ctx, 256 .crypt_fn = encrypt_callback, 257 }; 258 int ret; 259 260 desc->flags &= ~CRYPTO_TFM_REQ_MAY_SLEEP; 261 ret = lrw_crypt(desc, dst, src, nbytes, &req); 262 serpent_fpu_end(crypt_ctx.fpu_enabled); 263 264 return ret; 265 } 266 267 static int lrw_decrypt(struct blkcipher_desc *desc, struct scatterlist *dst, 268 struct scatterlist *src, unsigned int nbytes) 269 { 270 struct serpent_lrw_ctx *ctx = crypto_blkcipher_ctx(desc->tfm); 271 be128 buf[SERPENT_PARALLEL_BLOCKS]; 272 struct crypt_priv crypt_ctx = { 273 .ctx = &ctx->serpent_ctx, 274 .fpu_enabled = false, 275 }; 276 struct lrw_crypt_req req = { 277 .tbuf = buf, 278 .tbuflen = sizeof(buf), 279 280 .table_ctx = &ctx->lrw_table, 281 .crypt_ctx = &crypt_ctx, 282 .crypt_fn = decrypt_callback, 283 }; 284 int ret; 285 286 desc->flags &= ~CRYPTO_TFM_REQ_MAY_SLEEP; 287 ret = lrw_crypt(desc, dst, src, nbytes, &req); 288 serpent_fpu_end(crypt_ctx.fpu_enabled); 289 290 return ret; 291 } 292 293 static void lrw_exit_tfm(struct crypto_tfm *tfm) 294 { 295 struct serpent_lrw_ctx *ctx = crypto_tfm_ctx(tfm); 296 297 lrw_free_table(&ctx->lrw_table); 298 } 299 300 struct serpent_xts_ctx { 301 struct serpent_ctx tweak_ctx; 302 struct serpent_ctx crypt_ctx; 303 }; 304 305 static int xts_serpent_setkey(struct crypto_tfm *tfm, const u8 *key, 306 unsigned int keylen) 307 { 308 struct serpent_xts_ctx *ctx = crypto_tfm_ctx(tfm); 309 u32 *flags = &tfm->crt_flags; 310 int err; 311 312 /* key consists of keys of equal size concatenated, therefore 313 * the length must be even 314 */ 315 if (keylen % 2) { 316 *flags |= CRYPTO_TFM_RES_BAD_KEY_LEN; 317 return -EINVAL; 318 } 319 320 /* first half of xts-key is for crypt */ 321 err = __serpent_setkey(&ctx->crypt_ctx, key, keylen / 2); 322 if (err) 323 return err; 324 325 /* second half of xts-key is for tweak */ 326 return __serpent_setkey(&ctx->tweak_ctx, key + keylen / 2, keylen / 2); 327 } 328 329 static int xts_encrypt(struct blkcipher_desc *desc, struct scatterlist *dst, 330 struct scatterlist *src, unsigned int nbytes) 331 { 332 struct serpent_xts_ctx *ctx = crypto_blkcipher_ctx(desc->tfm); 333 be128 buf[SERPENT_PARALLEL_BLOCKS]; 334 struct crypt_priv crypt_ctx = { 335 .ctx = &ctx->crypt_ctx, 336 .fpu_enabled = false, 337 }; 338 struct xts_crypt_req req = { 339 .tbuf = buf, 340 .tbuflen = sizeof(buf), 341 342 .tweak_ctx = &ctx->tweak_ctx, 343 .tweak_fn = XTS_TWEAK_CAST(__serpent_encrypt), 344 .crypt_ctx = &crypt_ctx, 345 .crypt_fn = encrypt_callback, 346 }; 347 int ret; 348 349 desc->flags &= ~CRYPTO_TFM_REQ_MAY_SLEEP; 350 ret = xts_crypt(desc, dst, src, nbytes, &req); 351 serpent_fpu_end(crypt_ctx.fpu_enabled); 352 353 return ret; 354 } 355 356 static int xts_decrypt(struct blkcipher_desc *desc, struct scatterlist *dst, 357 struct scatterlist *src, unsigned int nbytes) 358 { 359 struct serpent_xts_ctx *ctx = crypto_blkcipher_ctx(desc->tfm); 360 be128 buf[SERPENT_PARALLEL_BLOCKS]; 361 struct crypt_priv crypt_ctx = { 362 .ctx = &ctx->crypt_ctx, 363 .fpu_enabled = false, 364 }; 365 struct xts_crypt_req req = { 366 .tbuf = buf, 367 .tbuflen = sizeof(buf), 368 369 .tweak_ctx = &ctx->tweak_ctx, 370 .tweak_fn = XTS_TWEAK_CAST(__serpent_encrypt), 371 .crypt_ctx = &crypt_ctx, 372 .crypt_fn = decrypt_callback, 373 }; 374 int ret; 375 376 desc->flags &= ~CRYPTO_TFM_REQ_MAY_SLEEP; 377 ret = xts_crypt(desc, dst, src, nbytes, &req); 378 serpent_fpu_end(crypt_ctx.fpu_enabled); 379 380 return ret; 381 } 382 383 static struct crypto_alg serpent_algs[10] = { { 384 .cra_name = "__ecb-serpent-avx", 385 .cra_driver_name = "__driver-ecb-serpent-avx", 386 .cra_priority = 0, 387 .cra_flags = CRYPTO_ALG_TYPE_BLKCIPHER, 388 .cra_blocksize = SERPENT_BLOCK_SIZE, 389 .cra_ctxsize = sizeof(struct serpent_ctx), 390 .cra_alignmask = 0, 391 .cra_type = &crypto_blkcipher_type, 392 .cra_module = THIS_MODULE, 393 .cra_list = LIST_HEAD_INIT(serpent_algs[0].cra_list), 394 .cra_u = { 395 .blkcipher = { 396 .min_keysize = SERPENT_MIN_KEY_SIZE, 397 .max_keysize = SERPENT_MAX_KEY_SIZE, 398 .setkey = serpent_setkey, 399 .encrypt = ecb_encrypt, 400 .decrypt = ecb_decrypt, 401 }, 402 }, 403 }, { 404 .cra_name = "__cbc-serpent-avx", 405 .cra_driver_name = "__driver-cbc-serpent-avx", 406 .cra_priority = 0, 407 .cra_flags = CRYPTO_ALG_TYPE_BLKCIPHER, 408 .cra_blocksize = SERPENT_BLOCK_SIZE, 409 .cra_ctxsize = sizeof(struct serpent_ctx), 410 .cra_alignmask = 0, 411 .cra_type = &crypto_blkcipher_type, 412 .cra_module = THIS_MODULE, 413 .cra_list = LIST_HEAD_INIT(serpent_algs[1].cra_list), 414 .cra_u = { 415 .blkcipher = { 416 .min_keysize = SERPENT_MIN_KEY_SIZE, 417 .max_keysize = SERPENT_MAX_KEY_SIZE, 418 .setkey = serpent_setkey, 419 .encrypt = cbc_encrypt, 420 .decrypt = cbc_decrypt, 421 }, 422 }, 423 }, { 424 .cra_name = "__ctr-serpent-avx", 425 .cra_driver_name = "__driver-ctr-serpent-avx", 426 .cra_priority = 0, 427 .cra_flags = CRYPTO_ALG_TYPE_BLKCIPHER, 428 .cra_blocksize = 1, 429 .cra_ctxsize = sizeof(struct serpent_ctx), 430 .cra_alignmask = 0, 431 .cra_type = &crypto_blkcipher_type, 432 .cra_module = THIS_MODULE, 433 .cra_list = LIST_HEAD_INIT(serpent_algs[2].cra_list), 434 .cra_u = { 435 .blkcipher = { 436 .min_keysize = SERPENT_MIN_KEY_SIZE, 437 .max_keysize = SERPENT_MAX_KEY_SIZE, 438 .ivsize = SERPENT_BLOCK_SIZE, 439 .setkey = serpent_setkey, 440 .encrypt = ctr_crypt, 441 .decrypt = ctr_crypt, 442 }, 443 }, 444 }, { 445 .cra_name = "__lrw-serpent-avx", 446 .cra_driver_name = "__driver-lrw-serpent-avx", 447 .cra_priority = 0, 448 .cra_flags = CRYPTO_ALG_TYPE_BLKCIPHER, 449 .cra_blocksize = SERPENT_BLOCK_SIZE, 450 .cra_ctxsize = sizeof(struct serpent_lrw_ctx), 451 .cra_alignmask = 0, 452 .cra_type = &crypto_blkcipher_type, 453 .cra_module = THIS_MODULE, 454 .cra_list = LIST_HEAD_INIT(serpent_algs[3].cra_list), 455 .cra_exit = lrw_exit_tfm, 456 .cra_u = { 457 .blkcipher = { 458 .min_keysize = SERPENT_MIN_KEY_SIZE + 459 SERPENT_BLOCK_SIZE, 460 .max_keysize = SERPENT_MAX_KEY_SIZE + 461 SERPENT_BLOCK_SIZE, 462 .ivsize = SERPENT_BLOCK_SIZE, 463 .setkey = lrw_serpent_setkey, 464 .encrypt = lrw_encrypt, 465 .decrypt = lrw_decrypt, 466 }, 467 }, 468 }, { 469 .cra_name = "__xts-serpent-avx", 470 .cra_driver_name = "__driver-xts-serpent-avx", 471 .cra_priority = 0, 472 .cra_flags = CRYPTO_ALG_TYPE_BLKCIPHER, 473 .cra_blocksize = SERPENT_BLOCK_SIZE, 474 .cra_ctxsize = sizeof(struct serpent_xts_ctx), 475 .cra_alignmask = 0, 476 .cra_type = &crypto_blkcipher_type, 477 .cra_module = THIS_MODULE, 478 .cra_list = LIST_HEAD_INIT(serpent_algs[4].cra_list), 479 .cra_u = { 480 .blkcipher = { 481 .min_keysize = SERPENT_MIN_KEY_SIZE * 2, 482 .max_keysize = SERPENT_MAX_KEY_SIZE * 2, 483 .ivsize = SERPENT_BLOCK_SIZE, 484 .setkey = xts_serpent_setkey, 485 .encrypt = xts_encrypt, 486 .decrypt = xts_decrypt, 487 }, 488 }, 489 }, { 490 .cra_name = "ecb(serpent)", 491 .cra_driver_name = "ecb-serpent-avx", 492 .cra_priority = 500, 493 .cra_flags = CRYPTO_ALG_TYPE_ABLKCIPHER | CRYPTO_ALG_ASYNC, 494 .cra_blocksize = SERPENT_BLOCK_SIZE, 495 .cra_ctxsize = sizeof(struct async_helper_ctx), 496 .cra_alignmask = 0, 497 .cra_type = &crypto_ablkcipher_type, 498 .cra_module = THIS_MODULE, 499 .cra_list = LIST_HEAD_INIT(serpent_algs[5].cra_list), 500 .cra_init = ablk_init, 501 .cra_exit = ablk_exit, 502 .cra_u = { 503 .ablkcipher = { 504 .min_keysize = SERPENT_MIN_KEY_SIZE, 505 .max_keysize = SERPENT_MAX_KEY_SIZE, 506 .setkey = ablk_set_key, 507 .encrypt = ablk_encrypt, 508 .decrypt = ablk_decrypt, 509 }, 510 }, 511 }, { 512 .cra_name = "cbc(serpent)", 513 .cra_driver_name = "cbc-serpent-avx", 514 .cra_priority = 500, 515 .cra_flags = CRYPTO_ALG_TYPE_ABLKCIPHER | CRYPTO_ALG_ASYNC, 516 .cra_blocksize = SERPENT_BLOCK_SIZE, 517 .cra_ctxsize = sizeof(struct async_helper_ctx), 518 .cra_alignmask = 0, 519 .cra_type = &crypto_ablkcipher_type, 520 .cra_module = THIS_MODULE, 521 .cra_list = LIST_HEAD_INIT(serpent_algs[6].cra_list), 522 .cra_init = ablk_init, 523 .cra_exit = ablk_exit, 524 .cra_u = { 525 .ablkcipher = { 526 .min_keysize = SERPENT_MIN_KEY_SIZE, 527 .max_keysize = SERPENT_MAX_KEY_SIZE, 528 .ivsize = SERPENT_BLOCK_SIZE, 529 .setkey = ablk_set_key, 530 .encrypt = __ablk_encrypt, 531 .decrypt = ablk_decrypt, 532 }, 533 }, 534 }, { 535 .cra_name = "ctr(serpent)", 536 .cra_driver_name = "ctr-serpent-avx", 537 .cra_priority = 500, 538 .cra_flags = CRYPTO_ALG_TYPE_ABLKCIPHER | CRYPTO_ALG_ASYNC, 539 .cra_blocksize = 1, 540 .cra_ctxsize = sizeof(struct async_helper_ctx), 541 .cra_alignmask = 0, 542 .cra_type = &crypto_ablkcipher_type, 543 .cra_module = THIS_MODULE, 544 .cra_list = LIST_HEAD_INIT(serpent_algs[7].cra_list), 545 .cra_init = ablk_init, 546 .cra_exit = ablk_exit, 547 .cra_u = { 548 .ablkcipher = { 549 .min_keysize = SERPENT_MIN_KEY_SIZE, 550 .max_keysize = SERPENT_MAX_KEY_SIZE, 551 .ivsize = SERPENT_BLOCK_SIZE, 552 .setkey = ablk_set_key, 553 .encrypt = ablk_encrypt, 554 .decrypt = ablk_encrypt, 555 .geniv = "chainiv", 556 }, 557 }, 558 }, { 559 .cra_name = "lrw(serpent)", 560 .cra_driver_name = "lrw-serpent-avx", 561 .cra_priority = 500, 562 .cra_flags = CRYPTO_ALG_TYPE_ABLKCIPHER | CRYPTO_ALG_ASYNC, 563 .cra_blocksize = SERPENT_BLOCK_SIZE, 564 .cra_ctxsize = sizeof(struct async_helper_ctx), 565 .cra_alignmask = 0, 566 .cra_type = &crypto_ablkcipher_type, 567 .cra_module = THIS_MODULE, 568 .cra_list = LIST_HEAD_INIT(serpent_algs[8].cra_list), 569 .cra_init = ablk_init, 570 .cra_exit = ablk_exit, 571 .cra_u = { 572 .ablkcipher = { 573 .min_keysize = SERPENT_MIN_KEY_SIZE + 574 SERPENT_BLOCK_SIZE, 575 .max_keysize = SERPENT_MAX_KEY_SIZE + 576 SERPENT_BLOCK_SIZE, 577 .ivsize = SERPENT_BLOCK_SIZE, 578 .setkey = ablk_set_key, 579 .encrypt = ablk_encrypt, 580 .decrypt = ablk_decrypt, 581 }, 582 }, 583 }, { 584 .cra_name = "xts(serpent)", 585 .cra_driver_name = "xts-serpent-avx", 586 .cra_priority = 500, 587 .cra_flags = CRYPTO_ALG_TYPE_ABLKCIPHER | CRYPTO_ALG_ASYNC, 588 .cra_blocksize = SERPENT_BLOCK_SIZE, 589 .cra_ctxsize = sizeof(struct async_helper_ctx), 590 .cra_alignmask = 0, 591 .cra_type = &crypto_ablkcipher_type, 592 .cra_module = THIS_MODULE, 593 .cra_list = LIST_HEAD_INIT(serpent_algs[9].cra_list), 594 .cra_init = ablk_init, 595 .cra_exit = ablk_exit, 596 .cra_u = { 597 .ablkcipher = { 598 .min_keysize = SERPENT_MIN_KEY_SIZE * 2, 599 .max_keysize = SERPENT_MAX_KEY_SIZE * 2, 600 .ivsize = SERPENT_BLOCK_SIZE, 601 .setkey = ablk_set_key, 602 .encrypt = ablk_encrypt, 603 .decrypt = ablk_decrypt, 604 }, 605 }, 606 } }; 607 608 static int __init serpent_init(void) 609 { 610 u64 xcr0; 611 612 if (!cpu_has_avx || !cpu_has_osxsave) { 613 printk(KERN_INFO "AVX instructions are not detected.\n"); 614 return -ENODEV; 615 } 616 617 xcr0 = xgetbv(XCR_XFEATURE_ENABLED_MASK); 618 if ((xcr0 & (XSTATE_SSE | XSTATE_YMM)) != (XSTATE_SSE | XSTATE_YMM)) { 619 printk(KERN_INFO "AVX detected but unusable.\n"); 620 return -ENODEV; 621 } 622 623 return crypto_register_algs(serpent_algs, ARRAY_SIZE(serpent_algs)); 624 } 625 626 static void __exit serpent_exit(void) 627 { 628 crypto_unregister_algs(serpent_algs, ARRAY_SIZE(serpent_algs)); 629 } 630 631 module_init(serpent_init); 632 module_exit(serpent_exit); 633 634 MODULE_DESCRIPTION("Serpent Cipher Algorithm, AVX optimized"); 635 MODULE_LICENSE("GPL"); 636 MODULE_ALIAS("serpent"); 637