1 /* 2 * Glue Code for AVX assembler version of Twofish Cipher 3 * 4 * Copyright (C) 2012 Johannes Goetzfried 5 * <Johannes.Goetzfried@informatik.stud.uni-erlangen.de> 6 * 7 * This program is free software; you can redistribute it and/or modify 8 * it under the terms of the GNU General Public License as published by 9 * the Free Software Foundation; either version 2 of the License, or 10 * (at your option) any later version. 11 * 12 * This program is distributed in the hope that it will be useful, 13 * but WITHOUT ANY WARRANTY; without even the implied warranty of 14 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 15 * GNU General Public License for more details. 16 * 17 * You should have received a copy of the GNU General Public License 18 * along with this program; if not, write to the Free Software 19 * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 20 * USA 21 * 22 */ 23 24 #include <linux/module.h> 25 #include <linux/hardirq.h> 26 #include <linux/types.h> 27 #include <linux/crypto.h> 28 #include <linux/err.h> 29 #include <crypto/algapi.h> 30 #include <crypto/twofish.h> 31 #include <crypto/cryptd.h> 32 #include <crypto/b128ops.h> 33 #include <crypto/ctr.h> 34 #include <crypto/lrw.h> 35 #include <crypto/xts.h> 36 #include <asm/i387.h> 37 #include <asm/xcr.h> 38 #include <asm/xsave.h> 39 #include <asm/crypto/twofish.h> 40 #include <asm/crypto/ablk_helper.h> 41 #include <asm/crypto/glue_helper.h> 42 #include <crypto/scatterwalk.h> 43 #include <linux/workqueue.h> 44 #include <linux/spinlock.h> 45 46 #define TWOFISH_PARALLEL_BLOCKS 8 47 48 static inline void twofish_enc_blk_3way(struct twofish_ctx *ctx, u8 *dst, 49 const u8 *src) 50 { 51 __twofish_enc_blk_3way(ctx, dst, src, false); 52 } 53 54 /* 8-way parallel cipher functions */ 55 asmlinkage void __twofish_enc_blk_8way(struct twofish_ctx *ctx, u8 *dst, 56 const u8 *src, bool xor); 57 asmlinkage void twofish_dec_blk_8way(struct twofish_ctx *ctx, u8 *dst, 58 const u8 *src); 59 60 static inline void twofish_enc_blk_xway(struct twofish_ctx *ctx, u8 *dst, 61 const u8 *src) 62 { 63 __twofish_enc_blk_8way(ctx, dst, src, false); 64 } 65 66 static inline void twofish_enc_blk_xway_xor(struct twofish_ctx *ctx, u8 *dst, 67 const u8 *src) 68 { 69 __twofish_enc_blk_8way(ctx, dst, src, true); 70 } 71 72 static inline void twofish_dec_blk_xway(struct twofish_ctx *ctx, u8 *dst, 73 const u8 *src) 74 { 75 twofish_dec_blk_8way(ctx, dst, src); 76 } 77 78 static void twofish_dec_blk_cbc_xway(void *ctx, u128 *dst, const u128 *src) 79 { 80 u128 ivs[TWOFISH_PARALLEL_BLOCKS - 1]; 81 unsigned int j; 82 83 for (j = 0; j < TWOFISH_PARALLEL_BLOCKS - 1; j++) 84 ivs[j] = src[j]; 85 86 twofish_dec_blk_xway(ctx, (u8 *)dst, (u8 *)src); 87 88 for (j = 0; j < TWOFISH_PARALLEL_BLOCKS - 1; j++) 89 u128_xor(dst + (j + 1), dst + (j + 1), ivs + j); 90 } 91 92 static void twofish_enc_blk_ctr_xway(void *ctx, u128 *dst, const u128 *src, 93 u128 *iv) 94 { 95 be128 ctrblks[TWOFISH_PARALLEL_BLOCKS]; 96 unsigned int i; 97 98 for (i = 0; i < TWOFISH_PARALLEL_BLOCKS; i++) { 99 if (dst != src) 100 dst[i] = src[i]; 101 102 u128_to_be128(&ctrblks[i], iv); 103 u128_inc(iv); 104 } 105 106 twofish_enc_blk_xway_xor(ctx, (u8 *)dst, (u8 *)ctrblks); 107 } 108 109 static const struct common_glue_ctx twofish_enc = { 110 .num_funcs = 3, 111 .fpu_blocks_limit = TWOFISH_PARALLEL_BLOCKS, 112 113 .funcs = { { 114 .num_blocks = TWOFISH_PARALLEL_BLOCKS, 115 .fn_u = { .ecb = GLUE_FUNC_CAST(twofish_enc_blk_xway) } 116 }, { 117 .num_blocks = 3, 118 .fn_u = { .ecb = GLUE_FUNC_CAST(twofish_enc_blk_3way) } 119 }, { 120 .num_blocks = 1, 121 .fn_u = { .ecb = GLUE_FUNC_CAST(twofish_enc_blk) } 122 } } 123 }; 124 125 static const struct common_glue_ctx twofish_ctr = { 126 .num_funcs = 3, 127 .fpu_blocks_limit = TWOFISH_PARALLEL_BLOCKS, 128 129 .funcs = { { 130 .num_blocks = TWOFISH_PARALLEL_BLOCKS, 131 .fn_u = { .ctr = GLUE_CTR_FUNC_CAST(twofish_enc_blk_ctr_xway) } 132 }, { 133 .num_blocks = 3, 134 .fn_u = { .ctr = GLUE_CTR_FUNC_CAST(twofish_enc_blk_ctr_3way) } 135 }, { 136 .num_blocks = 1, 137 .fn_u = { .ctr = GLUE_CTR_FUNC_CAST(twofish_enc_blk_ctr) } 138 } } 139 }; 140 141 static const struct common_glue_ctx twofish_dec = { 142 .num_funcs = 3, 143 .fpu_blocks_limit = TWOFISH_PARALLEL_BLOCKS, 144 145 .funcs = { { 146 .num_blocks = TWOFISH_PARALLEL_BLOCKS, 147 .fn_u = { .ecb = GLUE_FUNC_CAST(twofish_dec_blk_xway) } 148 }, { 149 .num_blocks = 3, 150 .fn_u = { .ecb = GLUE_FUNC_CAST(twofish_dec_blk_3way) } 151 }, { 152 .num_blocks = 1, 153 .fn_u = { .ecb = GLUE_FUNC_CAST(twofish_dec_blk) } 154 } } 155 }; 156 157 static const struct common_glue_ctx twofish_dec_cbc = { 158 .num_funcs = 3, 159 .fpu_blocks_limit = TWOFISH_PARALLEL_BLOCKS, 160 161 .funcs = { { 162 .num_blocks = TWOFISH_PARALLEL_BLOCKS, 163 .fn_u = { .cbc = GLUE_CBC_FUNC_CAST(twofish_dec_blk_cbc_xway) } 164 }, { 165 .num_blocks = 3, 166 .fn_u = { .cbc = GLUE_CBC_FUNC_CAST(twofish_dec_blk_cbc_3way) } 167 }, { 168 .num_blocks = 1, 169 .fn_u = { .cbc = GLUE_CBC_FUNC_CAST(twofish_dec_blk) } 170 } } 171 }; 172 173 static int ecb_encrypt(struct blkcipher_desc *desc, struct scatterlist *dst, 174 struct scatterlist *src, unsigned int nbytes) 175 { 176 return glue_ecb_crypt_128bit(&twofish_enc, desc, dst, src, nbytes); 177 } 178 179 static int ecb_decrypt(struct blkcipher_desc *desc, struct scatterlist *dst, 180 struct scatterlist *src, unsigned int nbytes) 181 { 182 return glue_ecb_crypt_128bit(&twofish_dec, desc, dst, src, nbytes); 183 } 184 185 static int cbc_encrypt(struct blkcipher_desc *desc, struct scatterlist *dst, 186 struct scatterlist *src, unsigned int nbytes) 187 { 188 return glue_cbc_encrypt_128bit(GLUE_FUNC_CAST(twofish_enc_blk), desc, 189 dst, src, nbytes); 190 } 191 192 static int cbc_decrypt(struct blkcipher_desc *desc, struct scatterlist *dst, 193 struct scatterlist *src, unsigned int nbytes) 194 { 195 return glue_cbc_decrypt_128bit(&twofish_dec_cbc, desc, dst, src, 196 nbytes); 197 } 198 199 static int ctr_crypt(struct blkcipher_desc *desc, struct scatterlist *dst, 200 struct scatterlist *src, unsigned int nbytes) 201 { 202 return glue_ctr_crypt_128bit(&twofish_ctr, desc, dst, src, nbytes); 203 } 204 205 static inline bool twofish_fpu_begin(bool fpu_enabled, unsigned int nbytes) 206 { 207 return glue_fpu_begin(TF_BLOCK_SIZE, TWOFISH_PARALLEL_BLOCKS, NULL, 208 fpu_enabled, nbytes); 209 } 210 211 static inline void twofish_fpu_end(bool fpu_enabled) 212 { 213 glue_fpu_end(fpu_enabled); 214 } 215 216 struct crypt_priv { 217 struct twofish_ctx *ctx; 218 bool fpu_enabled; 219 }; 220 221 static void encrypt_callback(void *priv, u8 *srcdst, unsigned int nbytes) 222 { 223 const unsigned int bsize = TF_BLOCK_SIZE; 224 struct crypt_priv *ctx = priv; 225 int i; 226 227 ctx->fpu_enabled = twofish_fpu_begin(ctx->fpu_enabled, nbytes); 228 229 if (nbytes == bsize * TWOFISH_PARALLEL_BLOCKS) { 230 twofish_enc_blk_xway(ctx->ctx, srcdst, srcdst); 231 return; 232 } 233 234 for (i = 0; i < nbytes / (bsize * 3); i++, srcdst += bsize * 3) 235 twofish_enc_blk_3way(ctx->ctx, srcdst, srcdst); 236 237 nbytes %= bsize * 3; 238 239 for (i = 0; i < nbytes / bsize; i++, srcdst += bsize) 240 twofish_enc_blk(ctx->ctx, srcdst, srcdst); 241 } 242 243 static void decrypt_callback(void *priv, u8 *srcdst, unsigned int nbytes) 244 { 245 const unsigned int bsize = TF_BLOCK_SIZE; 246 struct crypt_priv *ctx = priv; 247 int i; 248 249 ctx->fpu_enabled = twofish_fpu_begin(ctx->fpu_enabled, nbytes); 250 251 if (nbytes == bsize * TWOFISH_PARALLEL_BLOCKS) { 252 twofish_dec_blk_xway(ctx->ctx, srcdst, srcdst); 253 return; 254 } 255 256 for (i = 0; i < nbytes / (bsize * 3); i++, srcdst += bsize * 3) 257 twofish_dec_blk_3way(ctx->ctx, srcdst, srcdst); 258 259 nbytes %= bsize * 3; 260 261 for (i = 0; i < nbytes / bsize; i++, srcdst += bsize) 262 twofish_dec_blk(ctx->ctx, srcdst, srcdst); 263 } 264 265 static int lrw_encrypt(struct blkcipher_desc *desc, struct scatterlist *dst, 266 struct scatterlist *src, unsigned int nbytes) 267 { 268 struct twofish_lrw_ctx *ctx = crypto_blkcipher_ctx(desc->tfm); 269 be128 buf[TWOFISH_PARALLEL_BLOCKS]; 270 struct crypt_priv crypt_ctx = { 271 .ctx = &ctx->twofish_ctx, 272 .fpu_enabled = false, 273 }; 274 struct lrw_crypt_req req = { 275 .tbuf = buf, 276 .tbuflen = sizeof(buf), 277 278 .table_ctx = &ctx->lrw_table, 279 .crypt_ctx = &crypt_ctx, 280 .crypt_fn = encrypt_callback, 281 }; 282 int ret; 283 284 desc->flags &= ~CRYPTO_TFM_REQ_MAY_SLEEP; 285 ret = lrw_crypt(desc, dst, src, nbytes, &req); 286 twofish_fpu_end(crypt_ctx.fpu_enabled); 287 288 return ret; 289 } 290 291 static int lrw_decrypt(struct blkcipher_desc *desc, struct scatterlist *dst, 292 struct scatterlist *src, unsigned int nbytes) 293 { 294 struct twofish_lrw_ctx *ctx = crypto_blkcipher_ctx(desc->tfm); 295 be128 buf[TWOFISH_PARALLEL_BLOCKS]; 296 struct crypt_priv crypt_ctx = { 297 .ctx = &ctx->twofish_ctx, 298 .fpu_enabled = false, 299 }; 300 struct lrw_crypt_req req = { 301 .tbuf = buf, 302 .tbuflen = sizeof(buf), 303 304 .table_ctx = &ctx->lrw_table, 305 .crypt_ctx = &crypt_ctx, 306 .crypt_fn = decrypt_callback, 307 }; 308 int ret; 309 310 desc->flags &= ~CRYPTO_TFM_REQ_MAY_SLEEP; 311 ret = lrw_crypt(desc, dst, src, nbytes, &req); 312 twofish_fpu_end(crypt_ctx.fpu_enabled); 313 314 return ret; 315 } 316 317 static int xts_encrypt(struct blkcipher_desc *desc, struct scatterlist *dst, 318 struct scatterlist *src, unsigned int nbytes) 319 { 320 struct twofish_xts_ctx *ctx = crypto_blkcipher_ctx(desc->tfm); 321 be128 buf[TWOFISH_PARALLEL_BLOCKS]; 322 struct crypt_priv crypt_ctx = { 323 .ctx = &ctx->crypt_ctx, 324 .fpu_enabled = false, 325 }; 326 struct xts_crypt_req req = { 327 .tbuf = buf, 328 .tbuflen = sizeof(buf), 329 330 .tweak_ctx = &ctx->tweak_ctx, 331 .tweak_fn = XTS_TWEAK_CAST(twofish_enc_blk), 332 .crypt_ctx = &crypt_ctx, 333 .crypt_fn = encrypt_callback, 334 }; 335 int ret; 336 337 desc->flags &= ~CRYPTO_TFM_REQ_MAY_SLEEP; 338 ret = xts_crypt(desc, dst, src, nbytes, &req); 339 twofish_fpu_end(crypt_ctx.fpu_enabled); 340 341 return ret; 342 } 343 344 static int xts_decrypt(struct blkcipher_desc *desc, struct scatterlist *dst, 345 struct scatterlist *src, unsigned int nbytes) 346 { 347 struct twofish_xts_ctx *ctx = crypto_blkcipher_ctx(desc->tfm); 348 be128 buf[TWOFISH_PARALLEL_BLOCKS]; 349 struct crypt_priv crypt_ctx = { 350 .ctx = &ctx->crypt_ctx, 351 .fpu_enabled = false, 352 }; 353 struct xts_crypt_req req = { 354 .tbuf = buf, 355 .tbuflen = sizeof(buf), 356 357 .tweak_ctx = &ctx->tweak_ctx, 358 .tweak_fn = XTS_TWEAK_CAST(twofish_enc_blk), 359 .crypt_ctx = &crypt_ctx, 360 .crypt_fn = decrypt_callback, 361 }; 362 int ret; 363 364 desc->flags &= ~CRYPTO_TFM_REQ_MAY_SLEEP; 365 ret = xts_crypt(desc, dst, src, nbytes, &req); 366 twofish_fpu_end(crypt_ctx.fpu_enabled); 367 368 return ret; 369 } 370 371 static struct crypto_alg twofish_algs[10] = { { 372 .cra_name = "__ecb-twofish-avx", 373 .cra_driver_name = "__driver-ecb-twofish-avx", 374 .cra_priority = 0, 375 .cra_flags = CRYPTO_ALG_TYPE_BLKCIPHER, 376 .cra_blocksize = TF_BLOCK_SIZE, 377 .cra_ctxsize = sizeof(struct twofish_ctx), 378 .cra_alignmask = 0, 379 .cra_type = &crypto_blkcipher_type, 380 .cra_module = THIS_MODULE, 381 .cra_list = LIST_HEAD_INIT(twofish_algs[0].cra_list), 382 .cra_u = { 383 .blkcipher = { 384 .min_keysize = TF_MIN_KEY_SIZE, 385 .max_keysize = TF_MAX_KEY_SIZE, 386 .setkey = twofish_setkey, 387 .encrypt = ecb_encrypt, 388 .decrypt = ecb_decrypt, 389 }, 390 }, 391 }, { 392 .cra_name = "__cbc-twofish-avx", 393 .cra_driver_name = "__driver-cbc-twofish-avx", 394 .cra_priority = 0, 395 .cra_flags = CRYPTO_ALG_TYPE_BLKCIPHER, 396 .cra_blocksize = TF_BLOCK_SIZE, 397 .cra_ctxsize = sizeof(struct twofish_ctx), 398 .cra_alignmask = 0, 399 .cra_type = &crypto_blkcipher_type, 400 .cra_module = THIS_MODULE, 401 .cra_list = LIST_HEAD_INIT(twofish_algs[1].cra_list), 402 .cra_u = { 403 .blkcipher = { 404 .min_keysize = TF_MIN_KEY_SIZE, 405 .max_keysize = TF_MAX_KEY_SIZE, 406 .setkey = twofish_setkey, 407 .encrypt = cbc_encrypt, 408 .decrypt = cbc_decrypt, 409 }, 410 }, 411 }, { 412 .cra_name = "__ctr-twofish-avx", 413 .cra_driver_name = "__driver-ctr-twofish-avx", 414 .cra_priority = 0, 415 .cra_flags = CRYPTO_ALG_TYPE_BLKCIPHER, 416 .cra_blocksize = 1, 417 .cra_ctxsize = sizeof(struct twofish_ctx), 418 .cra_alignmask = 0, 419 .cra_type = &crypto_blkcipher_type, 420 .cra_module = THIS_MODULE, 421 .cra_list = LIST_HEAD_INIT(twofish_algs[2].cra_list), 422 .cra_u = { 423 .blkcipher = { 424 .min_keysize = TF_MIN_KEY_SIZE, 425 .max_keysize = TF_MAX_KEY_SIZE, 426 .ivsize = TF_BLOCK_SIZE, 427 .setkey = twofish_setkey, 428 .encrypt = ctr_crypt, 429 .decrypt = ctr_crypt, 430 }, 431 }, 432 }, { 433 .cra_name = "__lrw-twofish-avx", 434 .cra_driver_name = "__driver-lrw-twofish-avx", 435 .cra_priority = 0, 436 .cra_flags = CRYPTO_ALG_TYPE_BLKCIPHER, 437 .cra_blocksize = TF_BLOCK_SIZE, 438 .cra_ctxsize = sizeof(struct twofish_lrw_ctx), 439 .cra_alignmask = 0, 440 .cra_type = &crypto_blkcipher_type, 441 .cra_module = THIS_MODULE, 442 .cra_list = LIST_HEAD_INIT(twofish_algs[3].cra_list), 443 .cra_exit = lrw_twofish_exit_tfm, 444 .cra_u = { 445 .blkcipher = { 446 .min_keysize = TF_MIN_KEY_SIZE + 447 TF_BLOCK_SIZE, 448 .max_keysize = TF_MAX_KEY_SIZE + 449 TF_BLOCK_SIZE, 450 .ivsize = TF_BLOCK_SIZE, 451 .setkey = lrw_twofish_setkey, 452 .encrypt = lrw_encrypt, 453 .decrypt = lrw_decrypt, 454 }, 455 }, 456 }, { 457 .cra_name = "__xts-twofish-avx", 458 .cra_driver_name = "__driver-xts-twofish-avx", 459 .cra_priority = 0, 460 .cra_flags = CRYPTO_ALG_TYPE_BLKCIPHER, 461 .cra_blocksize = TF_BLOCK_SIZE, 462 .cra_ctxsize = sizeof(struct twofish_xts_ctx), 463 .cra_alignmask = 0, 464 .cra_type = &crypto_blkcipher_type, 465 .cra_module = THIS_MODULE, 466 .cra_list = LIST_HEAD_INIT(twofish_algs[4].cra_list), 467 .cra_u = { 468 .blkcipher = { 469 .min_keysize = TF_MIN_KEY_SIZE * 2, 470 .max_keysize = TF_MAX_KEY_SIZE * 2, 471 .ivsize = TF_BLOCK_SIZE, 472 .setkey = xts_twofish_setkey, 473 .encrypt = xts_encrypt, 474 .decrypt = xts_decrypt, 475 }, 476 }, 477 }, { 478 .cra_name = "ecb(twofish)", 479 .cra_driver_name = "ecb-twofish-avx", 480 .cra_priority = 400, 481 .cra_flags = CRYPTO_ALG_TYPE_ABLKCIPHER | CRYPTO_ALG_ASYNC, 482 .cra_blocksize = TF_BLOCK_SIZE, 483 .cra_ctxsize = sizeof(struct async_helper_ctx), 484 .cra_alignmask = 0, 485 .cra_type = &crypto_ablkcipher_type, 486 .cra_module = THIS_MODULE, 487 .cra_list = LIST_HEAD_INIT(twofish_algs[5].cra_list), 488 .cra_init = ablk_init, 489 .cra_exit = ablk_exit, 490 .cra_u = { 491 .ablkcipher = { 492 .min_keysize = TF_MIN_KEY_SIZE, 493 .max_keysize = TF_MAX_KEY_SIZE, 494 .setkey = ablk_set_key, 495 .encrypt = ablk_encrypt, 496 .decrypt = ablk_decrypt, 497 }, 498 }, 499 }, { 500 .cra_name = "cbc(twofish)", 501 .cra_driver_name = "cbc-twofish-avx", 502 .cra_priority = 400, 503 .cra_flags = CRYPTO_ALG_TYPE_ABLKCIPHER | CRYPTO_ALG_ASYNC, 504 .cra_blocksize = TF_BLOCK_SIZE, 505 .cra_ctxsize = sizeof(struct async_helper_ctx), 506 .cra_alignmask = 0, 507 .cra_type = &crypto_ablkcipher_type, 508 .cra_module = THIS_MODULE, 509 .cra_list = LIST_HEAD_INIT(twofish_algs[6].cra_list), 510 .cra_init = ablk_init, 511 .cra_exit = ablk_exit, 512 .cra_u = { 513 .ablkcipher = { 514 .min_keysize = TF_MIN_KEY_SIZE, 515 .max_keysize = TF_MAX_KEY_SIZE, 516 .ivsize = TF_BLOCK_SIZE, 517 .setkey = ablk_set_key, 518 .encrypt = __ablk_encrypt, 519 .decrypt = ablk_decrypt, 520 }, 521 }, 522 }, { 523 .cra_name = "ctr(twofish)", 524 .cra_driver_name = "ctr-twofish-avx", 525 .cra_priority = 400, 526 .cra_flags = CRYPTO_ALG_TYPE_ABLKCIPHER | CRYPTO_ALG_ASYNC, 527 .cra_blocksize = 1, 528 .cra_ctxsize = sizeof(struct async_helper_ctx), 529 .cra_alignmask = 0, 530 .cra_type = &crypto_ablkcipher_type, 531 .cra_module = THIS_MODULE, 532 .cra_list = LIST_HEAD_INIT(twofish_algs[7].cra_list), 533 .cra_init = ablk_init, 534 .cra_exit = ablk_exit, 535 .cra_u = { 536 .ablkcipher = { 537 .min_keysize = TF_MIN_KEY_SIZE, 538 .max_keysize = TF_MAX_KEY_SIZE, 539 .ivsize = TF_BLOCK_SIZE, 540 .setkey = ablk_set_key, 541 .encrypt = ablk_encrypt, 542 .decrypt = ablk_encrypt, 543 .geniv = "chainiv", 544 }, 545 }, 546 }, { 547 .cra_name = "lrw(twofish)", 548 .cra_driver_name = "lrw-twofish-avx", 549 .cra_priority = 400, 550 .cra_flags = CRYPTO_ALG_TYPE_ABLKCIPHER | CRYPTO_ALG_ASYNC, 551 .cra_blocksize = TF_BLOCK_SIZE, 552 .cra_ctxsize = sizeof(struct async_helper_ctx), 553 .cra_alignmask = 0, 554 .cra_type = &crypto_ablkcipher_type, 555 .cra_module = THIS_MODULE, 556 .cra_list = LIST_HEAD_INIT(twofish_algs[8].cra_list), 557 .cra_init = ablk_init, 558 .cra_exit = ablk_exit, 559 .cra_u = { 560 .ablkcipher = { 561 .min_keysize = TF_MIN_KEY_SIZE + 562 TF_BLOCK_SIZE, 563 .max_keysize = TF_MAX_KEY_SIZE + 564 TF_BLOCK_SIZE, 565 .ivsize = TF_BLOCK_SIZE, 566 .setkey = ablk_set_key, 567 .encrypt = ablk_encrypt, 568 .decrypt = ablk_decrypt, 569 }, 570 }, 571 }, { 572 .cra_name = "xts(twofish)", 573 .cra_driver_name = "xts-twofish-avx", 574 .cra_priority = 400, 575 .cra_flags = CRYPTO_ALG_TYPE_ABLKCIPHER | CRYPTO_ALG_ASYNC, 576 .cra_blocksize = TF_BLOCK_SIZE, 577 .cra_ctxsize = sizeof(struct async_helper_ctx), 578 .cra_alignmask = 0, 579 .cra_type = &crypto_ablkcipher_type, 580 .cra_module = THIS_MODULE, 581 .cra_list = LIST_HEAD_INIT(twofish_algs[9].cra_list), 582 .cra_init = ablk_init, 583 .cra_exit = ablk_exit, 584 .cra_u = { 585 .ablkcipher = { 586 .min_keysize = TF_MIN_KEY_SIZE * 2, 587 .max_keysize = TF_MAX_KEY_SIZE * 2, 588 .ivsize = TF_BLOCK_SIZE, 589 .setkey = ablk_set_key, 590 .encrypt = ablk_encrypt, 591 .decrypt = ablk_decrypt, 592 }, 593 }, 594 } }; 595 596 static int __init twofish_init(void) 597 { 598 u64 xcr0; 599 600 if (!cpu_has_avx || !cpu_has_osxsave) { 601 printk(KERN_INFO "AVX instructions are not detected.\n"); 602 return -ENODEV; 603 } 604 605 xcr0 = xgetbv(XCR_XFEATURE_ENABLED_MASK); 606 if ((xcr0 & (XSTATE_SSE | XSTATE_YMM)) != (XSTATE_SSE | XSTATE_YMM)) { 607 printk(KERN_INFO "AVX detected but unusable.\n"); 608 return -ENODEV; 609 } 610 611 return crypto_register_algs(twofish_algs, ARRAY_SIZE(twofish_algs)); 612 } 613 614 static void __exit twofish_exit(void) 615 { 616 crypto_unregister_algs(twofish_algs, ARRAY_SIZE(twofish_algs)); 617 } 618 619 module_init(twofish_init); 620 module_exit(twofish_exit); 621 622 MODULE_DESCRIPTION("Twofish Cipher Algorithm, AVX optimized"); 623 MODULE_LICENSE("GPL"); 624 MODULE_ALIAS("twofish"); 625