1 /* 2 * Glue Code for AVX assembler version of Twofish Cipher 3 * 4 * Copyright (C) 2012 Johannes Goetzfried 5 * <Johannes.Goetzfried@informatik.stud.uni-erlangen.de> 6 * 7 * This program is free software; you can redistribute it and/or modify 8 * it under the terms of the GNU General Public License as published by 9 * the Free Software Foundation; either version 2 of the License, or 10 * (at your option) any later version. 11 * 12 * This program is distributed in the hope that it will be useful, 13 * but WITHOUT ANY WARRANTY; without even the implied warranty of 14 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 15 * GNU General Public License for more details. 16 * 17 * You should have received a copy of the GNU General Public License 18 * along with this program; if not, write to the Free Software 19 * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 20 * USA 21 * 22 */ 23 24 #include <linux/module.h> 25 #include <linux/hardirq.h> 26 #include <linux/types.h> 27 #include <linux/crypto.h> 28 #include <linux/err.h> 29 #include <crypto/algapi.h> 30 #include <crypto/twofish.h> 31 #include <crypto/cryptd.h> 32 #include <crypto/b128ops.h> 33 #include <crypto/ctr.h> 34 #include <crypto/lrw.h> 35 #include <crypto/xts.h> 36 #include <asm/i387.h> 37 #include <asm/xcr.h> 38 #include <asm/xsave.h> 39 #include <asm/crypto/twofish.h> 40 #include <asm/crypto/ablk_helper.h> 41 #include <asm/crypto/glue_helper.h> 42 #include <crypto/scatterwalk.h> 43 #include <linux/workqueue.h> 44 #include <linux/spinlock.h> 45 46 #define TWOFISH_PARALLEL_BLOCKS 8 47 48 /* 8-way parallel cipher functions */ 49 asmlinkage void twofish_ecb_enc_8way(struct twofish_ctx *ctx, u8 *dst, 50 const u8 *src); 51 asmlinkage void twofish_ecb_dec_8way(struct twofish_ctx *ctx, u8 *dst, 52 const u8 *src); 53 54 asmlinkage void twofish_cbc_dec_8way(struct twofish_ctx *ctx, u8 *dst, 55 const u8 *src); 56 asmlinkage void twofish_ctr_8way(struct twofish_ctx *ctx, u8 *dst, 57 const u8 *src, le128 *iv); 58 59 static inline void twofish_enc_blk_3way(struct twofish_ctx *ctx, u8 *dst, 60 const u8 *src) 61 { 62 __twofish_enc_blk_3way(ctx, dst, src, false); 63 } 64 65 66 static const struct common_glue_ctx twofish_enc = { 67 .num_funcs = 3, 68 .fpu_blocks_limit = TWOFISH_PARALLEL_BLOCKS, 69 70 .funcs = { { 71 .num_blocks = TWOFISH_PARALLEL_BLOCKS, 72 .fn_u = { .ecb = GLUE_FUNC_CAST(twofish_ecb_enc_8way) } 73 }, { 74 .num_blocks = 3, 75 .fn_u = { .ecb = GLUE_FUNC_CAST(twofish_enc_blk_3way) } 76 }, { 77 .num_blocks = 1, 78 .fn_u = { .ecb = GLUE_FUNC_CAST(twofish_enc_blk) } 79 } } 80 }; 81 82 static const struct common_glue_ctx twofish_ctr = { 83 .num_funcs = 3, 84 .fpu_blocks_limit = TWOFISH_PARALLEL_BLOCKS, 85 86 .funcs = { { 87 .num_blocks = TWOFISH_PARALLEL_BLOCKS, 88 .fn_u = { .ctr = GLUE_CTR_FUNC_CAST(twofish_ctr_8way) } 89 }, { 90 .num_blocks = 3, 91 .fn_u = { .ctr = GLUE_CTR_FUNC_CAST(twofish_enc_blk_ctr_3way) } 92 }, { 93 .num_blocks = 1, 94 .fn_u = { .ctr = GLUE_CTR_FUNC_CAST(twofish_enc_blk_ctr) } 95 } } 96 }; 97 98 static const struct common_glue_ctx twofish_dec = { 99 .num_funcs = 3, 100 .fpu_blocks_limit = TWOFISH_PARALLEL_BLOCKS, 101 102 .funcs = { { 103 .num_blocks = TWOFISH_PARALLEL_BLOCKS, 104 .fn_u = { .ecb = GLUE_FUNC_CAST(twofish_ecb_dec_8way) } 105 }, { 106 .num_blocks = 3, 107 .fn_u = { .ecb = GLUE_FUNC_CAST(twofish_dec_blk_3way) } 108 }, { 109 .num_blocks = 1, 110 .fn_u = { .ecb = GLUE_FUNC_CAST(twofish_dec_blk) } 111 } } 112 }; 113 114 static const struct common_glue_ctx twofish_dec_cbc = { 115 .num_funcs = 3, 116 .fpu_blocks_limit = TWOFISH_PARALLEL_BLOCKS, 117 118 .funcs = { { 119 .num_blocks = TWOFISH_PARALLEL_BLOCKS, 120 .fn_u = { .cbc = GLUE_CBC_FUNC_CAST(twofish_cbc_dec_8way) } 121 }, { 122 .num_blocks = 3, 123 .fn_u = { .cbc = GLUE_CBC_FUNC_CAST(twofish_dec_blk_cbc_3way) } 124 }, { 125 .num_blocks = 1, 126 .fn_u = { .cbc = GLUE_CBC_FUNC_CAST(twofish_dec_blk) } 127 } } 128 }; 129 130 static int ecb_encrypt(struct blkcipher_desc *desc, struct scatterlist *dst, 131 struct scatterlist *src, unsigned int nbytes) 132 { 133 return glue_ecb_crypt_128bit(&twofish_enc, desc, dst, src, nbytes); 134 } 135 136 static int ecb_decrypt(struct blkcipher_desc *desc, struct scatterlist *dst, 137 struct scatterlist *src, unsigned int nbytes) 138 { 139 return glue_ecb_crypt_128bit(&twofish_dec, desc, dst, src, nbytes); 140 } 141 142 static int cbc_encrypt(struct blkcipher_desc *desc, struct scatterlist *dst, 143 struct scatterlist *src, unsigned int nbytes) 144 { 145 return glue_cbc_encrypt_128bit(GLUE_FUNC_CAST(twofish_enc_blk), desc, 146 dst, src, nbytes); 147 } 148 149 static int cbc_decrypt(struct blkcipher_desc *desc, struct scatterlist *dst, 150 struct scatterlist *src, unsigned int nbytes) 151 { 152 return glue_cbc_decrypt_128bit(&twofish_dec_cbc, desc, dst, src, 153 nbytes); 154 } 155 156 static int ctr_crypt(struct blkcipher_desc *desc, struct scatterlist *dst, 157 struct scatterlist *src, unsigned int nbytes) 158 { 159 return glue_ctr_crypt_128bit(&twofish_ctr, desc, dst, src, nbytes); 160 } 161 162 static inline bool twofish_fpu_begin(bool fpu_enabled, unsigned int nbytes) 163 { 164 return glue_fpu_begin(TF_BLOCK_SIZE, TWOFISH_PARALLEL_BLOCKS, NULL, 165 fpu_enabled, nbytes); 166 } 167 168 static inline void twofish_fpu_end(bool fpu_enabled) 169 { 170 glue_fpu_end(fpu_enabled); 171 } 172 173 struct crypt_priv { 174 struct twofish_ctx *ctx; 175 bool fpu_enabled; 176 }; 177 178 static void encrypt_callback(void *priv, u8 *srcdst, unsigned int nbytes) 179 { 180 const unsigned int bsize = TF_BLOCK_SIZE; 181 struct crypt_priv *ctx = priv; 182 int i; 183 184 ctx->fpu_enabled = twofish_fpu_begin(ctx->fpu_enabled, nbytes); 185 186 if (nbytes == bsize * TWOFISH_PARALLEL_BLOCKS) { 187 twofish_ecb_enc_8way(ctx->ctx, srcdst, srcdst); 188 return; 189 } 190 191 for (i = 0; i < nbytes / (bsize * 3); i++, srcdst += bsize * 3) 192 twofish_enc_blk_3way(ctx->ctx, srcdst, srcdst); 193 194 nbytes %= bsize * 3; 195 196 for (i = 0; i < nbytes / bsize; i++, srcdst += bsize) 197 twofish_enc_blk(ctx->ctx, srcdst, srcdst); 198 } 199 200 static void decrypt_callback(void *priv, u8 *srcdst, unsigned int nbytes) 201 { 202 const unsigned int bsize = TF_BLOCK_SIZE; 203 struct crypt_priv *ctx = priv; 204 int i; 205 206 ctx->fpu_enabled = twofish_fpu_begin(ctx->fpu_enabled, nbytes); 207 208 if (nbytes == bsize * TWOFISH_PARALLEL_BLOCKS) { 209 twofish_ecb_dec_8way(ctx->ctx, srcdst, srcdst); 210 return; 211 } 212 213 for (i = 0; i < nbytes / (bsize * 3); i++, srcdst += bsize * 3) 214 twofish_dec_blk_3way(ctx->ctx, srcdst, srcdst); 215 216 nbytes %= bsize * 3; 217 218 for (i = 0; i < nbytes / bsize; i++, srcdst += bsize) 219 twofish_dec_blk(ctx->ctx, srcdst, srcdst); 220 } 221 222 static int lrw_encrypt(struct blkcipher_desc *desc, struct scatterlist *dst, 223 struct scatterlist *src, unsigned int nbytes) 224 { 225 struct twofish_lrw_ctx *ctx = crypto_blkcipher_ctx(desc->tfm); 226 be128 buf[TWOFISH_PARALLEL_BLOCKS]; 227 struct crypt_priv crypt_ctx = { 228 .ctx = &ctx->twofish_ctx, 229 .fpu_enabled = false, 230 }; 231 struct lrw_crypt_req req = { 232 .tbuf = buf, 233 .tbuflen = sizeof(buf), 234 235 .table_ctx = &ctx->lrw_table, 236 .crypt_ctx = &crypt_ctx, 237 .crypt_fn = encrypt_callback, 238 }; 239 int ret; 240 241 desc->flags &= ~CRYPTO_TFM_REQ_MAY_SLEEP; 242 ret = lrw_crypt(desc, dst, src, nbytes, &req); 243 twofish_fpu_end(crypt_ctx.fpu_enabled); 244 245 return ret; 246 } 247 248 static int lrw_decrypt(struct blkcipher_desc *desc, struct scatterlist *dst, 249 struct scatterlist *src, unsigned int nbytes) 250 { 251 struct twofish_lrw_ctx *ctx = crypto_blkcipher_ctx(desc->tfm); 252 be128 buf[TWOFISH_PARALLEL_BLOCKS]; 253 struct crypt_priv crypt_ctx = { 254 .ctx = &ctx->twofish_ctx, 255 .fpu_enabled = false, 256 }; 257 struct lrw_crypt_req req = { 258 .tbuf = buf, 259 .tbuflen = sizeof(buf), 260 261 .table_ctx = &ctx->lrw_table, 262 .crypt_ctx = &crypt_ctx, 263 .crypt_fn = decrypt_callback, 264 }; 265 int ret; 266 267 desc->flags &= ~CRYPTO_TFM_REQ_MAY_SLEEP; 268 ret = lrw_crypt(desc, dst, src, nbytes, &req); 269 twofish_fpu_end(crypt_ctx.fpu_enabled); 270 271 return ret; 272 } 273 274 static int xts_encrypt(struct blkcipher_desc *desc, struct scatterlist *dst, 275 struct scatterlist *src, unsigned int nbytes) 276 { 277 struct twofish_xts_ctx *ctx = crypto_blkcipher_ctx(desc->tfm); 278 be128 buf[TWOFISH_PARALLEL_BLOCKS]; 279 struct crypt_priv crypt_ctx = { 280 .ctx = &ctx->crypt_ctx, 281 .fpu_enabled = false, 282 }; 283 struct xts_crypt_req req = { 284 .tbuf = buf, 285 .tbuflen = sizeof(buf), 286 287 .tweak_ctx = &ctx->tweak_ctx, 288 .tweak_fn = XTS_TWEAK_CAST(twofish_enc_blk), 289 .crypt_ctx = &crypt_ctx, 290 .crypt_fn = encrypt_callback, 291 }; 292 int ret; 293 294 desc->flags &= ~CRYPTO_TFM_REQ_MAY_SLEEP; 295 ret = xts_crypt(desc, dst, src, nbytes, &req); 296 twofish_fpu_end(crypt_ctx.fpu_enabled); 297 298 return ret; 299 } 300 301 static int xts_decrypt(struct blkcipher_desc *desc, struct scatterlist *dst, 302 struct scatterlist *src, unsigned int nbytes) 303 { 304 struct twofish_xts_ctx *ctx = crypto_blkcipher_ctx(desc->tfm); 305 be128 buf[TWOFISH_PARALLEL_BLOCKS]; 306 struct crypt_priv crypt_ctx = { 307 .ctx = &ctx->crypt_ctx, 308 .fpu_enabled = false, 309 }; 310 struct xts_crypt_req req = { 311 .tbuf = buf, 312 .tbuflen = sizeof(buf), 313 314 .tweak_ctx = &ctx->tweak_ctx, 315 .tweak_fn = XTS_TWEAK_CAST(twofish_enc_blk), 316 .crypt_ctx = &crypt_ctx, 317 .crypt_fn = decrypt_callback, 318 }; 319 int ret; 320 321 desc->flags &= ~CRYPTO_TFM_REQ_MAY_SLEEP; 322 ret = xts_crypt(desc, dst, src, nbytes, &req); 323 twofish_fpu_end(crypt_ctx.fpu_enabled); 324 325 return ret; 326 } 327 328 static struct crypto_alg twofish_algs[10] = { { 329 .cra_name = "__ecb-twofish-avx", 330 .cra_driver_name = "__driver-ecb-twofish-avx", 331 .cra_priority = 0, 332 .cra_flags = CRYPTO_ALG_TYPE_BLKCIPHER, 333 .cra_blocksize = TF_BLOCK_SIZE, 334 .cra_ctxsize = sizeof(struct twofish_ctx), 335 .cra_alignmask = 0, 336 .cra_type = &crypto_blkcipher_type, 337 .cra_module = THIS_MODULE, 338 .cra_u = { 339 .blkcipher = { 340 .min_keysize = TF_MIN_KEY_SIZE, 341 .max_keysize = TF_MAX_KEY_SIZE, 342 .setkey = twofish_setkey, 343 .encrypt = ecb_encrypt, 344 .decrypt = ecb_decrypt, 345 }, 346 }, 347 }, { 348 .cra_name = "__cbc-twofish-avx", 349 .cra_driver_name = "__driver-cbc-twofish-avx", 350 .cra_priority = 0, 351 .cra_flags = CRYPTO_ALG_TYPE_BLKCIPHER, 352 .cra_blocksize = TF_BLOCK_SIZE, 353 .cra_ctxsize = sizeof(struct twofish_ctx), 354 .cra_alignmask = 0, 355 .cra_type = &crypto_blkcipher_type, 356 .cra_module = THIS_MODULE, 357 .cra_u = { 358 .blkcipher = { 359 .min_keysize = TF_MIN_KEY_SIZE, 360 .max_keysize = TF_MAX_KEY_SIZE, 361 .setkey = twofish_setkey, 362 .encrypt = cbc_encrypt, 363 .decrypt = cbc_decrypt, 364 }, 365 }, 366 }, { 367 .cra_name = "__ctr-twofish-avx", 368 .cra_driver_name = "__driver-ctr-twofish-avx", 369 .cra_priority = 0, 370 .cra_flags = CRYPTO_ALG_TYPE_BLKCIPHER, 371 .cra_blocksize = 1, 372 .cra_ctxsize = sizeof(struct twofish_ctx), 373 .cra_alignmask = 0, 374 .cra_type = &crypto_blkcipher_type, 375 .cra_module = THIS_MODULE, 376 .cra_u = { 377 .blkcipher = { 378 .min_keysize = TF_MIN_KEY_SIZE, 379 .max_keysize = TF_MAX_KEY_SIZE, 380 .ivsize = TF_BLOCK_SIZE, 381 .setkey = twofish_setkey, 382 .encrypt = ctr_crypt, 383 .decrypt = ctr_crypt, 384 }, 385 }, 386 }, { 387 .cra_name = "__lrw-twofish-avx", 388 .cra_driver_name = "__driver-lrw-twofish-avx", 389 .cra_priority = 0, 390 .cra_flags = CRYPTO_ALG_TYPE_BLKCIPHER, 391 .cra_blocksize = TF_BLOCK_SIZE, 392 .cra_ctxsize = sizeof(struct twofish_lrw_ctx), 393 .cra_alignmask = 0, 394 .cra_type = &crypto_blkcipher_type, 395 .cra_module = THIS_MODULE, 396 .cra_exit = lrw_twofish_exit_tfm, 397 .cra_u = { 398 .blkcipher = { 399 .min_keysize = TF_MIN_KEY_SIZE + 400 TF_BLOCK_SIZE, 401 .max_keysize = TF_MAX_KEY_SIZE + 402 TF_BLOCK_SIZE, 403 .ivsize = TF_BLOCK_SIZE, 404 .setkey = lrw_twofish_setkey, 405 .encrypt = lrw_encrypt, 406 .decrypt = lrw_decrypt, 407 }, 408 }, 409 }, { 410 .cra_name = "__xts-twofish-avx", 411 .cra_driver_name = "__driver-xts-twofish-avx", 412 .cra_priority = 0, 413 .cra_flags = CRYPTO_ALG_TYPE_BLKCIPHER, 414 .cra_blocksize = TF_BLOCK_SIZE, 415 .cra_ctxsize = sizeof(struct twofish_xts_ctx), 416 .cra_alignmask = 0, 417 .cra_type = &crypto_blkcipher_type, 418 .cra_module = THIS_MODULE, 419 .cra_u = { 420 .blkcipher = { 421 .min_keysize = TF_MIN_KEY_SIZE * 2, 422 .max_keysize = TF_MAX_KEY_SIZE * 2, 423 .ivsize = TF_BLOCK_SIZE, 424 .setkey = xts_twofish_setkey, 425 .encrypt = xts_encrypt, 426 .decrypt = xts_decrypt, 427 }, 428 }, 429 }, { 430 .cra_name = "ecb(twofish)", 431 .cra_driver_name = "ecb-twofish-avx", 432 .cra_priority = 400, 433 .cra_flags = CRYPTO_ALG_TYPE_ABLKCIPHER | CRYPTO_ALG_ASYNC, 434 .cra_blocksize = TF_BLOCK_SIZE, 435 .cra_ctxsize = sizeof(struct async_helper_ctx), 436 .cra_alignmask = 0, 437 .cra_type = &crypto_ablkcipher_type, 438 .cra_module = THIS_MODULE, 439 .cra_init = ablk_init, 440 .cra_exit = ablk_exit, 441 .cra_u = { 442 .ablkcipher = { 443 .min_keysize = TF_MIN_KEY_SIZE, 444 .max_keysize = TF_MAX_KEY_SIZE, 445 .setkey = ablk_set_key, 446 .encrypt = ablk_encrypt, 447 .decrypt = ablk_decrypt, 448 }, 449 }, 450 }, { 451 .cra_name = "cbc(twofish)", 452 .cra_driver_name = "cbc-twofish-avx", 453 .cra_priority = 400, 454 .cra_flags = CRYPTO_ALG_TYPE_ABLKCIPHER | CRYPTO_ALG_ASYNC, 455 .cra_blocksize = TF_BLOCK_SIZE, 456 .cra_ctxsize = sizeof(struct async_helper_ctx), 457 .cra_alignmask = 0, 458 .cra_type = &crypto_ablkcipher_type, 459 .cra_module = THIS_MODULE, 460 .cra_init = ablk_init, 461 .cra_exit = ablk_exit, 462 .cra_u = { 463 .ablkcipher = { 464 .min_keysize = TF_MIN_KEY_SIZE, 465 .max_keysize = TF_MAX_KEY_SIZE, 466 .ivsize = TF_BLOCK_SIZE, 467 .setkey = ablk_set_key, 468 .encrypt = __ablk_encrypt, 469 .decrypt = ablk_decrypt, 470 }, 471 }, 472 }, { 473 .cra_name = "ctr(twofish)", 474 .cra_driver_name = "ctr-twofish-avx", 475 .cra_priority = 400, 476 .cra_flags = CRYPTO_ALG_TYPE_ABLKCIPHER | CRYPTO_ALG_ASYNC, 477 .cra_blocksize = 1, 478 .cra_ctxsize = sizeof(struct async_helper_ctx), 479 .cra_alignmask = 0, 480 .cra_type = &crypto_ablkcipher_type, 481 .cra_module = THIS_MODULE, 482 .cra_init = ablk_init, 483 .cra_exit = ablk_exit, 484 .cra_u = { 485 .ablkcipher = { 486 .min_keysize = TF_MIN_KEY_SIZE, 487 .max_keysize = TF_MAX_KEY_SIZE, 488 .ivsize = TF_BLOCK_SIZE, 489 .setkey = ablk_set_key, 490 .encrypt = ablk_encrypt, 491 .decrypt = ablk_encrypt, 492 .geniv = "chainiv", 493 }, 494 }, 495 }, { 496 .cra_name = "lrw(twofish)", 497 .cra_driver_name = "lrw-twofish-avx", 498 .cra_priority = 400, 499 .cra_flags = CRYPTO_ALG_TYPE_ABLKCIPHER | CRYPTO_ALG_ASYNC, 500 .cra_blocksize = TF_BLOCK_SIZE, 501 .cra_ctxsize = sizeof(struct async_helper_ctx), 502 .cra_alignmask = 0, 503 .cra_type = &crypto_ablkcipher_type, 504 .cra_module = THIS_MODULE, 505 .cra_init = ablk_init, 506 .cra_exit = ablk_exit, 507 .cra_u = { 508 .ablkcipher = { 509 .min_keysize = TF_MIN_KEY_SIZE + 510 TF_BLOCK_SIZE, 511 .max_keysize = TF_MAX_KEY_SIZE + 512 TF_BLOCK_SIZE, 513 .ivsize = TF_BLOCK_SIZE, 514 .setkey = ablk_set_key, 515 .encrypt = ablk_encrypt, 516 .decrypt = ablk_decrypt, 517 }, 518 }, 519 }, { 520 .cra_name = "xts(twofish)", 521 .cra_driver_name = "xts-twofish-avx", 522 .cra_priority = 400, 523 .cra_flags = CRYPTO_ALG_TYPE_ABLKCIPHER | CRYPTO_ALG_ASYNC, 524 .cra_blocksize = TF_BLOCK_SIZE, 525 .cra_ctxsize = sizeof(struct async_helper_ctx), 526 .cra_alignmask = 0, 527 .cra_type = &crypto_ablkcipher_type, 528 .cra_module = THIS_MODULE, 529 .cra_init = ablk_init, 530 .cra_exit = ablk_exit, 531 .cra_u = { 532 .ablkcipher = { 533 .min_keysize = TF_MIN_KEY_SIZE * 2, 534 .max_keysize = TF_MAX_KEY_SIZE * 2, 535 .ivsize = TF_BLOCK_SIZE, 536 .setkey = ablk_set_key, 537 .encrypt = ablk_encrypt, 538 .decrypt = ablk_decrypt, 539 }, 540 }, 541 } }; 542 543 static int __init twofish_init(void) 544 { 545 u64 xcr0; 546 547 if (!cpu_has_avx || !cpu_has_osxsave) { 548 printk(KERN_INFO "AVX instructions are not detected.\n"); 549 return -ENODEV; 550 } 551 552 xcr0 = xgetbv(XCR_XFEATURE_ENABLED_MASK); 553 if ((xcr0 & (XSTATE_SSE | XSTATE_YMM)) != (XSTATE_SSE | XSTATE_YMM)) { 554 printk(KERN_INFO "AVX detected but unusable.\n"); 555 return -ENODEV; 556 } 557 558 return crypto_register_algs(twofish_algs, ARRAY_SIZE(twofish_algs)); 559 } 560 561 static void __exit twofish_exit(void) 562 { 563 crypto_unregister_algs(twofish_algs, ARRAY_SIZE(twofish_algs)); 564 } 565 566 module_init(twofish_init); 567 module_exit(twofish_exit); 568 569 MODULE_DESCRIPTION("Twofish Cipher Algorithm, AVX optimized"); 570 MODULE_LICENSE("GPL"); 571 MODULE_ALIAS("twofish"); 572