1 /* 2 * Support for Intel AES-NI instructions. This file contains glue 3 * code, the real AES implementation is in intel-aes_asm.S. 4 * 5 * Copyright (C) 2008, Intel Corp. 6 * Author: Huang Ying <ying.huang@intel.com> 7 * 8 * This program is free software; you can redistribute it and/or modify 9 * it under the terms of the GNU General Public License as published by 10 * the Free Software Foundation; either version 2 of the License, or 11 * (at your option) any later version. 12 */ 13 14 #include <linux/hardirq.h> 15 #include <linux/types.h> 16 #include <linux/crypto.h> 17 #include <linux/err.h> 18 #include <crypto/algapi.h> 19 #include <crypto/aes.h> 20 #include <crypto/cryptd.h> 21 #include <crypto/ctr.h> 22 #include <asm/i387.h> 23 #include <asm/aes.h> 24 25 #if defined(CONFIG_CRYPTO_CTR) || defined(CONFIG_CRYPTO_CTR_MODULE) 26 #define HAS_CTR 27 #endif 28 29 #if defined(CONFIG_CRYPTO_LRW) || defined(CONFIG_CRYPTO_LRW_MODULE) 30 #define HAS_LRW 31 #endif 32 33 #if defined(CONFIG_CRYPTO_PCBC) || defined(CONFIG_CRYPTO_PCBC_MODULE) 34 #define HAS_PCBC 35 #endif 36 37 #if defined(CONFIG_CRYPTO_XTS) || defined(CONFIG_CRYPTO_XTS_MODULE) 38 #define HAS_XTS 39 #endif 40 41 struct async_aes_ctx { 42 struct cryptd_ablkcipher *cryptd_tfm; 43 }; 44 45 #define AESNI_ALIGN 16 46 #define AES_BLOCK_MASK (~(AES_BLOCK_SIZE-1)) 47 48 asmlinkage int aesni_set_key(struct crypto_aes_ctx *ctx, const u8 *in_key, 49 unsigned int key_len); 50 asmlinkage void aesni_enc(struct crypto_aes_ctx *ctx, u8 *out, 51 const u8 *in); 52 asmlinkage void aesni_dec(struct crypto_aes_ctx *ctx, u8 *out, 53 const u8 *in); 54 asmlinkage void aesni_ecb_enc(struct crypto_aes_ctx *ctx, u8 *out, 55 const u8 *in, unsigned int len); 56 asmlinkage void aesni_ecb_dec(struct crypto_aes_ctx *ctx, u8 *out, 57 const u8 *in, unsigned int len); 58 asmlinkage void aesni_cbc_enc(struct crypto_aes_ctx *ctx, u8 *out, 59 const u8 *in, unsigned int len, u8 *iv); 60 asmlinkage void aesni_cbc_dec(struct crypto_aes_ctx *ctx, u8 *out, 61 const u8 *in, unsigned int len, u8 *iv); 62 asmlinkage void aesni_ctr_enc(struct crypto_aes_ctx *ctx, u8 *out, 63 const u8 *in, unsigned int len, u8 *iv); 64 65 static inline struct crypto_aes_ctx *aes_ctx(void *raw_ctx) 66 { 67 unsigned long addr = (unsigned long)raw_ctx; 68 unsigned long align = AESNI_ALIGN; 69 70 if (align <= crypto_tfm_ctx_alignment()) 71 align = 1; 72 return (struct crypto_aes_ctx *)ALIGN(addr, align); 73 } 74 75 static int aes_set_key_common(struct crypto_tfm *tfm, void *raw_ctx, 76 const u8 *in_key, unsigned int key_len) 77 { 78 struct crypto_aes_ctx *ctx = aes_ctx(raw_ctx); 79 u32 *flags = &tfm->crt_flags; 80 int err; 81 82 if (key_len != AES_KEYSIZE_128 && key_len != AES_KEYSIZE_192 && 83 key_len != AES_KEYSIZE_256) { 84 *flags |= CRYPTO_TFM_RES_BAD_KEY_LEN; 85 return -EINVAL; 86 } 87 88 if (!irq_fpu_usable()) 89 err = crypto_aes_expand_key(ctx, in_key, key_len); 90 else { 91 kernel_fpu_begin(); 92 err = aesni_set_key(ctx, in_key, key_len); 93 kernel_fpu_end(); 94 } 95 96 return err; 97 } 98 99 static int aes_set_key(struct crypto_tfm *tfm, const u8 *in_key, 100 unsigned int key_len) 101 { 102 return aes_set_key_common(tfm, crypto_tfm_ctx(tfm), in_key, key_len); 103 } 104 105 static void aes_encrypt(struct crypto_tfm *tfm, u8 *dst, const u8 *src) 106 { 107 struct crypto_aes_ctx *ctx = aes_ctx(crypto_tfm_ctx(tfm)); 108 109 if (!irq_fpu_usable()) 110 crypto_aes_encrypt_x86(ctx, dst, src); 111 else { 112 kernel_fpu_begin(); 113 aesni_enc(ctx, dst, src); 114 kernel_fpu_end(); 115 } 116 } 117 118 static void aes_decrypt(struct crypto_tfm *tfm, u8 *dst, const u8 *src) 119 { 120 struct crypto_aes_ctx *ctx = aes_ctx(crypto_tfm_ctx(tfm)); 121 122 if (!irq_fpu_usable()) 123 crypto_aes_decrypt_x86(ctx, dst, src); 124 else { 125 kernel_fpu_begin(); 126 aesni_dec(ctx, dst, src); 127 kernel_fpu_end(); 128 } 129 } 130 131 static struct crypto_alg aesni_alg = { 132 .cra_name = "aes", 133 .cra_driver_name = "aes-aesni", 134 .cra_priority = 300, 135 .cra_flags = CRYPTO_ALG_TYPE_CIPHER, 136 .cra_blocksize = AES_BLOCK_SIZE, 137 .cra_ctxsize = sizeof(struct crypto_aes_ctx)+AESNI_ALIGN-1, 138 .cra_alignmask = 0, 139 .cra_module = THIS_MODULE, 140 .cra_list = LIST_HEAD_INIT(aesni_alg.cra_list), 141 .cra_u = { 142 .cipher = { 143 .cia_min_keysize = AES_MIN_KEY_SIZE, 144 .cia_max_keysize = AES_MAX_KEY_SIZE, 145 .cia_setkey = aes_set_key, 146 .cia_encrypt = aes_encrypt, 147 .cia_decrypt = aes_decrypt 148 } 149 } 150 }; 151 152 static void __aes_encrypt(struct crypto_tfm *tfm, u8 *dst, const u8 *src) 153 { 154 struct crypto_aes_ctx *ctx = aes_ctx(crypto_tfm_ctx(tfm)); 155 156 aesni_enc(ctx, dst, src); 157 } 158 159 static void __aes_decrypt(struct crypto_tfm *tfm, u8 *dst, const u8 *src) 160 { 161 struct crypto_aes_ctx *ctx = aes_ctx(crypto_tfm_ctx(tfm)); 162 163 aesni_dec(ctx, dst, src); 164 } 165 166 static struct crypto_alg __aesni_alg = { 167 .cra_name = "__aes-aesni", 168 .cra_driver_name = "__driver-aes-aesni", 169 .cra_priority = 0, 170 .cra_flags = CRYPTO_ALG_TYPE_CIPHER, 171 .cra_blocksize = AES_BLOCK_SIZE, 172 .cra_ctxsize = sizeof(struct crypto_aes_ctx)+AESNI_ALIGN-1, 173 .cra_alignmask = 0, 174 .cra_module = THIS_MODULE, 175 .cra_list = LIST_HEAD_INIT(__aesni_alg.cra_list), 176 .cra_u = { 177 .cipher = { 178 .cia_min_keysize = AES_MIN_KEY_SIZE, 179 .cia_max_keysize = AES_MAX_KEY_SIZE, 180 .cia_setkey = aes_set_key, 181 .cia_encrypt = __aes_encrypt, 182 .cia_decrypt = __aes_decrypt 183 } 184 } 185 }; 186 187 static int ecb_encrypt(struct blkcipher_desc *desc, 188 struct scatterlist *dst, struct scatterlist *src, 189 unsigned int nbytes) 190 { 191 struct crypto_aes_ctx *ctx = aes_ctx(crypto_blkcipher_ctx(desc->tfm)); 192 struct blkcipher_walk walk; 193 int err; 194 195 blkcipher_walk_init(&walk, dst, src, nbytes); 196 err = blkcipher_walk_virt(desc, &walk); 197 desc->flags &= ~CRYPTO_TFM_REQ_MAY_SLEEP; 198 199 kernel_fpu_begin(); 200 while ((nbytes = walk.nbytes)) { 201 aesni_ecb_enc(ctx, walk.dst.virt.addr, walk.src.virt.addr, 202 nbytes & AES_BLOCK_MASK); 203 nbytes &= AES_BLOCK_SIZE - 1; 204 err = blkcipher_walk_done(desc, &walk, nbytes); 205 } 206 kernel_fpu_end(); 207 208 return err; 209 } 210 211 static int ecb_decrypt(struct blkcipher_desc *desc, 212 struct scatterlist *dst, struct scatterlist *src, 213 unsigned int nbytes) 214 { 215 struct crypto_aes_ctx *ctx = aes_ctx(crypto_blkcipher_ctx(desc->tfm)); 216 struct blkcipher_walk walk; 217 int err; 218 219 blkcipher_walk_init(&walk, dst, src, nbytes); 220 err = blkcipher_walk_virt(desc, &walk); 221 desc->flags &= ~CRYPTO_TFM_REQ_MAY_SLEEP; 222 223 kernel_fpu_begin(); 224 while ((nbytes = walk.nbytes)) { 225 aesni_ecb_dec(ctx, walk.dst.virt.addr, walk.src.virt.addr, 226 nbytes & AES_BLOCK_MASK); 227 nbytes &= AES_BLOCK_SIZE - 1; 228 err = blkcipher_walk_done(desc, &walk, nbytes); 229 } 230 kernel_fpu_end(); 231 232 return err; 233 } 234 235 static struct crypto_alg blk_ecb_alg = { 236 .cra_name = "__ecb-aes-aesni", 237 .cra_driver_name = "__driver-ecb-aes-aesni", 238 .cra_priority = 0, 239 .cra_flags = CRYPTO_ALG_TYPE_BLKCIPHER, 240 .cra_blocksize = AES_BLOCK_SIZE, 241 .cra_ctxsize = sizeof(struct crypto_aes_ctx)+AESNI_ALIGN-1, 242 .cra_alignmask = 0, 243 .cra_type = &crypto_blkcipher_type, 244 .cra_module = THIS_MODULE, 245 .cra_list = LIST_HEAD_INIT(blk_ecb_alg.cra_list), 246 .cra_u = { 247 .blkcipher = { 248 .min_keysize = AES_MIN_KEY_SIZE, 249 .max_keysize = AES_MAX_KEY_SIZE, 250 .setkey = aes_set_key, 251 .encrypt = ecb_encrypt, 252 .decrypt = ecb_decrypt, 253 }, 254 }, 255 }; 256 257 static int cbc_encrypt(struct blkcipher_desc *desc, 258 struct scatterlist *dst, struct scatterlist *src, 259 unsigned int nbytes) 260 { 261 struct crypto_aes_ctx *ctx = aes_ctx(crypto_blkcipher_ctx(desc->tfm)); 262 struct blkcipher_walk walk; 263 int err; 264 265 blkcipher_walk_init(&walk, dst, src, nbytes); 266 err = blkcipher_walk_virt(desc, &walk); 267 desc->flags &= ~CRYPTO_TFM_REQ_MAY_SLEEP; 268 269 kernel_fpu_begin(); 270 while ((nbytes = walk.nbytes)) { 271 aesni_cbc_enc(ctx, walk.dst.virt.addr, walk.src.virt.addr, 272 nbytes & AES_BLOCK_MASK, walk.iv); 273 nbytes &= AES_BLOCK_SIZE - 1; 274 err = blkcipher_walk_done(desc, &walk, nbytes); 275 } 276 kernel_fpu_end(); 277 278 return err; 279 } 280 281 static int cbc_decrypt(struct blkcipher_desc *desc, 282 struct scatterlist *dst, struct scatterlist *src, 283 unsigned int nbytes) 284 { 285 struct crypto_aes_ctx *ctx = aes_ctx(crypto_blkcipher_ctx(desc->tfm)); 286 struct blkcipher_walk walk; 287 int err; 288 289 blkcipher_walk_init(&walk, dst, src, nbytes); 290 err = blkcipher_walk_virt(desc, &walk); 291 desc->flags &= ~CRYPTO_TFM_REQ_MAY_SLEEP; 292 293 kernel_fpu_begin(); 294 while ((nbytes = walk.nbytes)) { 295 aesni_cbc_dec(ctx, walk.dst.virt.addr, walk.src.virt.addr, 296 nbytes & AES_BLOCK_MASK, walk.iv); 297 nbytes &= AES_BLOCK_SIZE - 1; 298 err = blkcipher_walk_done(desc, &walk, nbytes); 299 } 300 kernel_fpu_end(); 301 302 return err; 303 } 304 305 static struct crypto_alg blk_cbc_alg = { 306 .cra_name = "__cbc-aes-aesni", 307 .cra_driver_name = "__driver-cbc-aes-aesni", 308 .cra_priority = 0, 309 .cra_flags = CRYPTO_ALG_TYPE_BLKCIPHER, 310 .cra_blocksize = AES_BLOCK_SIZE, 311 .cra_ctxsize = sizeof(struct crypto_aes_ctx)+AESNI_ALIGN-1, 312 .cra_alignmask = 0, 313 .cra_type = &crypto_blkcipher_type, 314 .cra_module = THIS_MODULE, 315 .cra_list = LIST_HEAD_INIT(blk_cbc_alg.cra_list), 316 .cra_u = { 317 .blkcipher = { 318 .min_keysize = AES_MIN_KEY_SIZE, 319 .max_keysize = AES_MAX_KEY_SIZE, 320 .setkey = aes_set_key, 321 .encrypt = cbc_encrypt, 322 .decrypt = cbc_decrypt, 323 }, 324 }, 325 }; 326 327 static void ctr_crypt_final(struct crypto_aes_ctx *ctx, 328 struct blkcipher_walk *walk) 329 { 330 u8 *ctrblk = walk->iv; 331 u8 keystream[AES_BLOCK_SIZE]; 332 u8 *src = walk->src.virt.addr; 333 u8 *dst = walk->dst.virt.addr; 334 unsigned int nbytes = walk->nbytes; 335 336 aesni_enc(ctx, keystream, ctrblk); 337 crypto_xor(keystream, src, nbytes); 338 memcpy(dst, keystream, nbytes); 339 crypto_inc(ctrblk, AES_BLOCK_SIZE); 340 } 341 342 static int ctr_crypt(struct blkcipher_desc *desc, 343 struct scatterlist *dst, struct scatterlist *src, 344 unsigned int nbytes) 345 { 346 struct crypto_aes_ctx *ctx = aes_ctx(crypto_blkcipher_ctx(desc->tfm)); 347 struct blkcipher_walk walk; 348 int err; 349 350 blkcipher_walk_init(&walk, dst, src, nbytes); 351 err = blkcipher_walk_virt_block(desc, &walk, AES_BLOCK_SIZE); 352 desc->flags &= ~CRYPTO_TFM_REQ_MAY_SLEEP; 353 354 kernel_fpu_begin(); 355 while ((nbytes = walk.nbytes) >= AES_BLOCK_SIZE) { 356 aesni_ctr_enc(ctx, walk.dst.virt.addr, walk.src.virt.addr, 357 nbytes & AES_BLOCK_MASK, walk.iv); 358 nbytes &= AES_BLOCK_SIZE - 1; 359 err = blkcipher_walk_done(desc, &walk, nbytes); 360 } 361 if (walk.nbytes) { 362 ctr_crypt_final(ctx, &walk); 363 err = blkcipher_walk_done(desc, &walk, 0); 364 } 365 kernel_fpu_end(); 366 367 return err; 368 } 369 370 static struct crypto_alg blk_ctr_alg = { 371 .cra_name = "__ctr-aes-aesni", 372 .cra_driver_name = "__driver-ctr-aes-aesni", 373 .cra_priority = 0, 374 .cra_flags = CRYPTO_ALG_TYPE_BLKCIPHER, 375 .cra_blocksize = 1, 376 .cra_ctxsize = sizeof(struct crypto_aes_ctx)+AESNI_ALIGN-1, 377 .cra_alignmask = 0, 378 .cra_type = &crypto_blkcipher_type, 379 .cra_module = THIS_MODULE, 380 .cra_list = LIST_HEAD_INIT(blk_ctr_alg.cra_list), 381 .cra_u = { 382 .blkcipher = { 383 .min_keysize = AES_MIN_KEY_SIZE, 384 .max_keysize = AES_MAX_KEY_SIZE, 385 .ivsize = AES_BLOCK_SIZE, 386 .setkey = aes_set_key, 387 .encrypt = ctr_crypt, 388 .decrypt = ctr_crypt, 389 }, 390 }, 391 }; 392 393 static int ablk_set_key(struct crypto_ablkcipher *tfm, const u8 *key, 394 unsigned int key_len) 395 { 396 struct async_aes_ctx *ctx = crypto_ablkcipher_ctx(tfm); 397 struct crypto_ablkcipher *child = &ctx->cryptd_tfm->base; 398 int err; 399 400 crypto_ablkcipher_clear_flags(child, CRYPTO_TFM_REQ_MASK); 401 crypto_ablkcipher_set_flags(child, crypto_ablkcipher_get_flags(tfm) 402 & CRYPTO_TFM_REQ_MASK); 403 err = crypto_ablkcipher_setkey(child, key, key_len); 404 crypto_ablkcipher_set_flags(tfm, crypto_ablkcipher_get_flags(child) 405 & CRYPTO_TFM_RES_MASK); 406 return err; 407 } 408 409 static int ablk_encrypt(struct ablkcipher_request *req) 410 { 411 struct crypto_ablkcipher *tfm = crypto_ablkcipher_reqtfm(req); 412 struct async_aes_ctx *ctx = crypto_ablkcipher_ctx(tfm); 413 414 if (!irq_fpu_usable()) { 415 struct ablkcipher_request *cryptd_req = 416 ablkcipher_request_ctx(req); 417 memcpy(cryptd_req, req, sizeof(*req)); 418 ablkcipher_request_set_tfm(cryptd_req, &ctx->cryptd_tfm->base); 419 return crypto_ablkcipher_encrypt(cryptd_req); 420 } else { 421 struct blkcipher_desc desc; 422 desc.tfm = cryptd_ablkcipher_child(ctx->cryptd_tfm); 423 desc.info = req->info; 424 desc.flags = 0; 425 return crypto_blkcipher_crt(desc.tfm)->encrypt( 426 &desc, req->dst, req->src, req->nbytes); 427 } 428 } 429 430 static int ablk_decrypt(struct ablkcipher_request *req) 431 { 432 struct crypto_ablkcipher *tfm = crypto_ablkcipher_reqtfm(req); 433 struct async_aes_ctx *ctx = crypto_ablkcipher_ctx(tfm); 434 435 if (!irq_fpu_usable()) { 436 struct ablkcipher_request *cryptd_req = 437 ablkcipher_request_ctx(req); 438 memcpy(cryptd_req, req, sizeof(*req)); 439 ablkcipher_request_set_tfm(cryptd_req, &ctx->cryptd_tfm->base); 440 return crypto_ablkcipher_decrypt(cryptd_req); 441 } else { 442 struct blkcipher_desc desc; 443 desc.tfm = cryptd_ablkcipher_child(ctx->cryptd_tfm); 444 desc.info = req->info; 445 desc.flags = 0; 446 return crypto_blkcipher_crt(desc.tfm)->decrypt( 447 &desc, req->dst, req->src, req->nbytes); 448 } 449 } 450 451 static void ablk_exit(struct crypto_tfm *tfm) 452 { 453 struct async_aes_ctx *ctx = crypto_tfm_ctx(tfm); 454 455 cryptd_free_ablkcipher(ctx->cryptd_tfm); 456 } 457 458 static void ablk_init_common(struct crypto_tfm *tfm, 459 struct cryptd_ablkcipher *cryptd_tfm) 460 { 461 struct async_aes_ctx *ctx = crypto_tfm_ctx(tfm); 462 463 ctx->cryptd_tfm = cryptd_tfm; 464 tfm->crt_ablkcipher.reqsize = sizeof(struct ablkcipher_request) + 465 crypto_ablkcipher_reqsize(&cryptd_tfm->base); 466 } 467 468 static int ablk_ecb_init(struct crypto_tfm *tfm) 469 { 470 struct cryptd_ablkcipher *cryptd_tfm; 471 472 cryptd_tfm = cryptd_alloc_ablkcipher("__driver-ecb-aes-aesni", 0, 0); 473 if (IS_ERR(cryptd_tfm)) 474 return PTR_ERR(cryptd_tfm); 475 ablk_init_common(tfm, cryptd_tfm); 476 return 0; 477 } 478 479 static struct crypto_alg ablk_ecb_alg = { 480 .cra_name = "ecb(aes)", 481 .cra_driver_name = "ecb-aes-aesni", 482 .cra_priority = 400, 483 .cra_flags = CRYPTO_ALG_TYPE_ABLKCIPHER|CRYPTO_ALG_ASYNC, 484 .cra_blocksize = AES_BLOCK_SIZE, 485 .cra_ctxsize = sizeof(struct async_aes_ctx), 486 .cra_alignmask = 0, 487 .cra_type = &crypto_ablkcipher_type, 488 .cra_module = THIS_MODULE, 489 .cra_list = LIST_HEAD_INIT(ablk_ecb_alg.cra_list), 490 .cra_init = ablk_ecb_init, 491 .cra_exit = ablk_exit, 492 .cra_u = { 493 .ablkcipher = { 494 .min_keysize = AES_MIN_KEY_SIZE, 495 .max_keysize = AES_MAX_KEY_SIZE, 496 .setkey = ablk_set_key, 497 .encrypt = ablk_encrypt, 498 .decrypt = ablk_decrypt, 499 }, 500 }, 501 }; 502 503 static int ablk_cbc_init(struct crypto_tfm *tfm) 504 { 505 struct cryptd_ablkcipher *cryptd_tfm; 506 507 cryptd_tfm = cryptd_alloc_ablkcipher("__driver-cbc-aes-aesni", 0, 0); 508 if (IS_ERR(cryptd_tfm)) 509 return PTR_ERR(cryptd_tfm); 510 ablk_init_common(tfm, cryptd_tfm); 511 return 0; 512 } 513 514 static struct crypto_alg ablk_cbc_alg = { 515 .cra_name = "cbc(aes)", 516 .cra_driver_name = "cbc-aes-aesni", 517 .cra_priority = 400, 518 .cra_flags = CRYPTO_ALG_TYPE_ABLKCIPHER|CRYPTO_ALG_ASYNC, 519 .cra_blocksize = AES_BLOCK_SIZE, 520 .cra_ctxsize = sizeof(struct async_aes_ctx), 521 .cra_alignmask = 0, 522 .cra_type = &crypto_ablkcipher_type, 523 .cra_module = THIS_MODULE, 524 .cra_list = LIST_HEAD_INIT(ablk_cbc_alg.cra_list), 525 .cra_init = ablk_cbc_init, 526 .cra_exit = ablk_exit, 527 .cra_u = { 528 .ablkcipher = { 529 .min_keysize = AES_MIN_KEY_SIZE, 530 .max_keysize = AES_MAX_KEY_SIZE, 531 .ivsize = AES_BLOCK_SIZE, 532 .setkey = ablk_set_key, 533 .encrypt = ablk_encrypt, 534 .decrypt = ablk_decrypt, 535 }, 536 }, 537 }; 538 539 static int ablk_ctr_init(struct crypto_tfm *tfm) 540 { 541 struct cryptd_ablkcipher *cryptd_tfm; 542 543 cryptd_tfm = cryptd_alloc_ablkcipher("__driver-ctr-aes-aesni", 0, 0); 544 if (IS_ERR(cryptd_tfm)) 545 return PTR_ERR(cryptd_tfm); 546 ablk_init_common(tfm, cryptd_tfm); 547 return 0; 548 } 549 550 static struct crypto_alg ablk_ctr_alg = { 551 .cra_name = "ctr(aes)", 552 .cra_driver_name = "ctr-aes-aesni", 553 .cra_priority = 400, 554 .cra_flags = CRYPTO_ALG_TYPE_ABLKCIPHER|CRYPTO_ALG_ASYNC, 555 .cra_blocksize = 1, 556 .cra_ctxsize = sizeof(struct async_aes_ctx), 557 .cra_alignmask = 0, 558 .cra_type = &crypto_ablkcipher_type, 559 .cra_module = THIS_MODULE, 560 .cra_list = LIST_HEAD_INIT(ablk_ctr_alg.cra_list), 561 .cra_init = ablk_ctr_init, 562 .cra_exit = ablk_exit, 563 .cra_u = { 564 .ablkcipher = { 565 .min_keysize = AES_MIN_KEY_SIZE, 566 .max_keysize = AES_MAX_KEY_SIZE, 567 .ivsize = AES_BLOCK_SIZE, 568 .setkey = ablk_set_key, 569 .encrypt = ablk_encrypt, 570 .decrypt = ablk_encrypt, 571 .geniv = "chainiv", 572 }, 573 }, 574 }; 575 576 #ifdef HAS_CTR 577 static int ablk_rfc3686_ctr_init(struct crypto_tfm *tfm) 578 { 579 struct cryptd_ablkcipher *cryptd_tfm; 580 581 cryptd_tfm = cryptd_alloc_ablkcipher( 582 "rfc3686(__driver-ctr-aes-aesni)", 0, 0); 583 if (IS_ERR(cryptd_tfm)) 584 return PTR_ERR(cryptd_tfm); 585 ablk_init_common(tfm, cryptd_tfm); 586 return 0; 587 } 588 589 static struct crypto_alg ablk_rfc3686_ctr_alg = { 590 .cra_name = "rfc3686(ctr(aes))", 591 .cra_driver_name = "rfc3686-ctr-aes-aesni", 592 .cra_priority = 400, 593 .cra_flags = CRYPTO_ALG_TYPE_ABLKCIPHER|CRYPTO_ALG_ASYNC, 594 .cra_blocksize = 1, 595 .cra_ctxsize = sizeof(struct async_aes_ctx), 596 .cra_alignmask = 0, 597 .cra_type = &crypto_ablkcipher_type, 598 .cra_module = THIS_MODULE, 599 .cra_list = LIST_HEAD_INIT(ablk_rfc3686_ctr_alg.cra_list), 600 .cra_init = ablk_rfc3686_ctr_init, 601 .cra_exit = ablk_exit, 602 .cra_u = { 603 .ablkcipher = { 604 .min_keysize = AES_MIN_KEY_SIZE+CTR_RFC3686_NONCE_SIZE, 605 .max_keysize = AES_MAX_KEY_SIZE+CTR_RFC3686_NONCE_SIZE, 606 .ivsize = CTR_RFC3686_IV_SIZE, 607 .setkey = ablk_set_key, 608 .encrypt = ablk_encrypt, 609 .decrypt = ablk_decrypt, 610 .geniv = "seqiv", 611 }, 612 }, 613 }; 614 #endif 615 616 #ifdef HAS_LRW 617 static int ablk_lrw_init(struct crypto_tfm *tfm) 618 { 619 struct cryptd_ablkcipher *cryptd_tfm; 620 621 cryptd_tfm = cryptd_alloc_ablkcipher("fpu(lrw(__driver-aes-aesni))", 622 0, 0); 623 if (IS_ERR(cryptd_tfm)) 624 return PTR_ERR(cryptd_tfm); 625 ablk_init_common(tfm, cryptd_tfm); 626 return 0; 627 } 628 629 static struct crypto_alg ablk_lrw_alg = { 630 .cra_name = "lrw(aes)", 631 .cra_driver_name = "lrw-aes-aesni", 632 .cra_priority = 400, 633 .cra_flags = CRYPTO_ALG_TYPE_ABLKCIPHER|CRYPTO_ALG_ASYNC, 634 .cra_blocksize = AES_BLOCK_SIZE, 635 .cra_ctxsize = sizeof(struct async_aes_ctx), 636 .cra_alignmask = 0, 637 .cra_type = &crypto_ablkcipher_type, 638 .cra_module = THIS_MODULE, 639 .cra_list = LIST_HEAD_INIT(ablk_lrw_alg.cra_list), 640 .cra_init = ablk_lrw_init, 641 .cra_exit = ablk_exit, 642 .cra_u = { 643 .ablkcipher = { 644 .min_keysize = AES_MIN_KEY_SIZE + AES_BLOCK_SIZE, 645 .max_keysize = AES_MAX_KEY_SIZE + AES_BLOCK_SIZE, 646 .ivsize = AES_BLOCK_SIZE, 647 .setkey = ablk_set_key, 648 .encrypt = ablk_encrypt, 649 .decrypt = ablk_decrypt, 650 }, 651 }, 652 }; 653 #endif 654 655 #ifdef HAS_PCBC 656 static int ablk_pcbc_init(struct crypto_tfm *tfm) 657 { 658 struct cryptd_ablkcipher *cryptd_tfm; 659 660 cryptd_tfm = cryptd_alloc_ablkcipher("fpu(pcbc(__driver-aes-aesni))", 661 0, 0); 662 if (IS_ERR(cryptd_tfm)) 663 return PTR_ERR(cryptd_tfm); 664 ablk_init_common(tfm, cryptd_tfm); 665 return 0; 666 } 667 668 static struct crypto_alg ablk_pcbc_alg = { 669 .cra_name = "pcbc(aes)", 670 .cra_driver_name = "pcbc-aes-aesni", 671 .cra_priority = 400, 672 .cra_flags = CRYPTO_ALG_TYPE_ABLKCIPHER|CRYPTO_ALG_ASYNC, 673 .cra_blocksize = AES_BLOCK_SIZE, 674 .cra_ctxsize = sizeof(struct async_aes_ctx), 675 .cra_alignmask = 0, 676 .cra_type = &crypto_ablkcipher_type, 677 .cra_module = THIS_MODULE, 678 .cra_list = LIST_HEAD_INIT(ablk_pcbc_alg.cra_list), 679 .cra_init = ablk_pcbc_init, 680 .cra_exit = ablk_exit, 681 .cra_u = { 682 .ablkcipher = { 683 .min_keysize = AES_MIN_KEY_SIZE, 684 .max_keysize = AES_MAX_KEY_SIZE, 685 .ivsize = AES_BLOCK_SIZE, 686 .setkey = ablk_set_key, 687 .encrypt = ablk_encrypt, 688 .decrypt = ablk_decrypt, 689 }, 690 }, 691 }; 692 #endif 693 694 #ifdef HAS_XTS 695 static int ablk_xts_init(struct crypto_tfm *tfm) 696 { 697 struct cryptd_ablkcipher *cryptd_tfm; 698 699 cryptd_tfm = cryptd_alloc_ablkcipher("fpu(xts(__driver-aes-aesni))", 700 0, 0); 701 if (IS_ERR(cryptd_tfm)) 702 return PTR_ERR(cryptd_tfm); 703 ablk_init_common(tfm, cryptd_tfm); 704 return 0; 705 } 706 707 static struct crypto_alg ablk_xts_alg = { 708 .cra_name = "xts(aes)", 709 .cra_driver_name = "xts-aes-aesni", 710 .cra_priority = 400, 711 .cra_flags = CRYPTO_ALG_TYPE_ABLKCIPHER|CRYPTO_ALG_ASYNC, 712 .cra_blocksize = AES_BLOCK_SIZE, 713 .cra_ctxsize = sizeof(struct async_aes_ctx), 714 .cra_alignmask = 0, 715 .cra_type = &crypto_ablkcipher_type, 716 .cra_module = THIS_MODULE, 717 .cra_list = LIST_HEAD_INIT(ablk_xts_alg.cra_list), 718 .cra_init = ablk_xts_init, 719 .cra_exit = ablk_exit, 720 .cra_u = { 721 .ablkcipher = { 722 .min_keysize = 2 * AES_MIN_KEY_SIZE, 723 .max_keysize = 2 * AES_MAX_KEY_SIZE, 724 .ivsize = AES_BLOCK_SIZE, 725 .setkey = ablk_set_key, 726 .encrypt = ablk_encrypt, 727 .decrypt = ablk_decrypt, 728 }, 729 }, 730 }; 731 #endif 732 733 static int __init aesni_init(void) 734 { 735 int err; 736 737 if (!cpu_has_aes) { 738 printk(KERN_INFO "Intel AES-NI instructions are not detected.\n"); 739 return -ENODEV; 740 } 741 if ((err = crypto_register_alg(&aesni_alg))) 742 goto aes_err; 743 if ((err = crypto_register_alg(&__aesni_alg))) 744 goto __aes_err; 745 if ((err = crypto_register_alg(&blk_ecb_alg))) 746 goto blk_ecb_err; 747 if ((err = crypto_register_alg(&blk_cbc_alg))) 748 goto blk_cbc_err; 749 if ((err = crypto_register_alg(&blk_ctr_alg))) 750 goto blk_ctr_err; 751 if ((err = crypto_register_alg(&ablk_ecb_alg))) 752 goto ablk_ecb_err; 753 if ((err = crypto_register_alg(&ablk_cbc_alg))) 754 goto ablk_cbc_err; 755 if ((err = crypto_register_alg(&ablk_ctr_alg))) 756 goto ablk_ctr_err; 757 #ifdef HAS_CTR 758 if ((err = crypto_register_alg(&ablk_rfc3686_ctr_alg))) 759 goto ablk_rfc3686_ctr_err; 760 #endif 761 #ifdef HAS_LRW 762 if ((err = crypto_register_alg(&ablk_lrw_alg))) 763 goto ablk_lrw_err; 764 #endif 765 #ifdef HAS_PCBC 766 if ((err = crypto_register_alg(&ablk_pcbc_alg))) 767 goto ablk_pcbc_err; 768 #endif 769 #ifdef HAS_XTS 770 if ((err = crypto_register_alg(&ablk_xts_alg))) 771 goto ablk_xts_err; 772 #endif 773 774 return err; 775 776 #ifdef HAS_XTS 777 ablk_xts_err: 778 #endif 779 #ifdef HAS_PCBC 780 crypto_unregister_alg(&ablk_pcbc_alg); 781 ablk_pcbc_err: 782 #endif 783 #ifdef HAS_LRW 784 crypto_unregister_alg(&ablk_lrw_alg); 785 ablk_lrw_err: 786 #endif 787 #ifdef HAS_CTR 788 crypto_unregister_alg(&ablk_rfc3686_ctr_alg); 789 ablk_rfc3686_ctr_err: 790 #endif 791 crypto_unregister_alg(&ablk_ctr_alg); 792 ablk_ctr_err: 793 crypto_unregister_alg(&ablk_cbc_alg); 794 ablk_cbc_err: 795 crypto_unregister_alg(&ablk_ecb_alg); 796 ablk_ecb_err: 797 crypto_unregister_alg(&blk_ctr_alg); 798 blk_ctr_err: 799 crypto_unregister_alg(&blk_cbc_alg); 800 blk_cbc_err: 801 crypto_unregister_alg(&blk_ecb_alg); 802 blk_ecb_err: 803 crypto_unregister_alg(&__aesni_alg); 804 __aes_err: 805 crypto_unregister_alg(&aesni_alg); 806 aes_err: 807 return err; 808 } 809 810 static void __exit aesni_exit(void) 811 { 812 #ifdef HAS_XTS 813 crypto_unregister_alg(&ablk_xts_alg); 814 #endif 815 #ifdef HAS_PCBC 816 crypto_unregister_alg(&ablk_pcbc_alg); 817 #endif 818 #ifdef HAS_LRW 819 crypto_unregister_alg(&ablk_lrw_alg); 820 #endif 821 #ifdef HAS_CTR 822 crypto_unregister_alg(&ablk_rfc3686_ctr_alg); 823 #endif 824 crypto_unregister_alg(&ablk_ctr_alg); 825 crypto_unregister_alg(&ablk_cbc_alg); 826 crypto_unregister_alg(&ablk_ecb_alg); 827 crypto_unregister_alg(&blk_ctr_alg); 828 crypto_unregister_alg(&blk_cbc_alg); 829 crypto_unregister_alg(&blk_ecb_alg); 830 crypto_unregister_alg(&__aesni_alg); 831 crypto_unregister_alg(&aesni_alg); 832 } 833 834 module_init(aesni_init); 835 module_exit(aesni_exit); 836 837 MODULE_DESCRIPTION("Rijndael (AES) Cipher Algorithm, Intel AES-NI instructions optimized"); 838 MODULE_LICENSE("GPL"); 839 MODULE_ALIAS("aes"); 840