1 /* 2 * Support for Intel AES-NI instructions. This file contains glue 3 * code, the real AES implementation is in intel-aes_asm.S. 4 * 5 * Copyright (C) 2008, Intel Corp. 6 * Author: Huang Ying <ying.huang@intel.com> 7 * 8 * Added RFC4106 AES-GCM support for 128-bit keys under the AEAD 9 * interface for 64-bit kernels. 10 * Authors: Adrian Hoban <adrian.hoban@intel.com> 11 * Gabriele Paoloni <gabriele.paoloni@intel.com> 12 * Tadeusz Struk (tadeusz.struk@intel.com) 13 * Aidan O'Mahony (aidan.o.mahony@intel.com) 14 * Copyright (c) 2010, Intel Corporation. 15 * 16 * This program is free software; you can redistribute it and/or modify 17 * it under the terms of the GNU General Public License as published by 18 * the Free Software Foundation; either version 2 of the License, or 19 * (at your option) any later version. 20 */ 21 22 #include <linux/hardirq.h> 23 #include <linux/types.h> 24 #include <linux/crypto.h> 25 #include <linux/err.h> 26 #include <crypto/algapi.h> 27 #include <crypto/aes.h> 28 #include <crypto/cryptd.h> 29 #include <crypto/ctr.h> 30 #include <asm/i387.h> 31 #include <asm/aes.h> 32 #include <crypto/scatterwalk.h> 33 #include <crypto/internal/aead.h> 34 #include <linux/workqueue.h> 35 #include <linux/spinlock.h> 36 37 #if defined(CONFIG_CRYPTO_CTR) || defined(CONFIG_CRYPTO_CTR_MODULE) 38 #define HAS_CTR 39 #endif 40 41 #if defined(CONFIG_CRYPTO_LRW) || defined(CONFIG_CRYPTO_LRW_MODULE) 42 #define HAS_LRW 43 #endif 44 45 #if defined(CONFIG_CRYPTO_PCBC) || defined(CONFIG_CRYPTO_PCBC_MODULE) 46 #define HAS_PCBC 47 #endif 48 49 #if defined(CONFIG_CRYPTO_XTS) || defined(CONFIG_CRYPTO_XTS_MODULE) 50 #define HAS_XTS 51 #endif 52 53 struct async_aes_ctx { 54 struct cryptd_ablkcipher *cryptd_tfm; 55 }; 56 57 /* This data is stored at the end of the crypto_tfm struct. 58 * It's a type of per "session" data storage location. 59 * This needs to be 16 byte aligned. 60 */ 61 struct aesni_rfc4106_gcm_ctx { 62 u8 hash_subkey[16]; 63 struct crypto_aes_ctx aes_key_expanded; 64 u8 nonce[4]; 65 struct cryptd_aead *cryptd_tfm; 66 }; 67 68 struct aesni_gcm_set_hash_subkey_result { 69 int err; 70 struct completion completion; 71 }; 72 73 struct aesni_hash_subkey_req_data { 74 u8 iv[16]; 75 struct aesni_gcm_set_hash_subkey_result result; 76 struct scatterlist sg; 77 }; 78 79 #define AESNI_ALIGN (16) 80 #define AES_BLOCK_MASK (~(AES_BLOCK_SIZE-1)) 81 #define RFC4106_HASH_SUBKEY_SIZE 16 82 83 asmlinkage int aesni_set_key(struct crypto_aes_ctx *ctx, const u8 *in_key, 84 unsigned int key_len); 85 asmlinkage void aesni_enc(struct crypto_aes_ctx *ctx, u8 *out, 86 const u8 *in); 87 asmlinkage void aesni_dec(struct crypto_aes_ctx *ctx, u8 *out, 88 const u8 *in); 89 asmlinkage void aesni_ecb_enc(struct crypto_aes_ctx *ctx, u8 *out, 90 const u8 *in, unsigned int len); 91 asmlinkage void aesni_ecb_dec(struct crypto_aes_ctx *ctx, u8 *out, 92 const u8 *in, unsigned int len); 93 asmlinkage void aesni_cbc_enc(struct crypto_aes_ctx *ctx, u8 *out, 94 const u8 *in, unsigned int len, u8 *iv); 95 asmlinkage void aesni_cbc_dec(struct crypto_aes_ctx *ctx, u8 *out, 96 const u8 *in, unsigned int len, u8 *iv); 97 #ifdef CONFIG_X86_64 98 asmlinkage void aesni_ctr_enc(struct crypto_aes_ctx *ctx, u8 *out, 99 const u8 *in, unsigned int len, u8 *iv); 100 101 /* asmlinkage void aesni_gcm_enc() 102 * void *ctx, AES Key schedule. Starts on a 16 byte boundary. 103 * u8 *out, Ciphertext output. Encrypt in-place is allowed. 104 * const u8 *in, Plaintext input 105 * unsigned long plaintext_len, Length of data in bytes for encryption. 106 * u8 *iv, Pre-counter block j0: 4 byte salt (from Security Association) 107 * concatenated with 8 byte Initialisation Vector (from IPSec ESP 108 * Payload) concatenated with 0x00000001. 16-byte aligned pointer. 109 * u8 *hash_subkey, the Hash sub key input. Data starts on a 16-byte boundary. 110 * const u8 *aad, Additional Authentication Data (AAD) 111 * unsigned long aad_len, Length of AAD in bytes. With RFC4106 this 112 * is going to be 8 or 12 bytes 113 * u8 *auth_tag, Authenticated Tag output. 114 * unsigned long auth_tag_len), Authenticated Tag Length in bytes. 115 * Valid values are 16 (most likely), 12 or 8. 116 */ 117 asmlinkage void aesni_gcm_enc(void *ctx, u8 *out, 118 const u8 *in, unsigned long plaintext_len, u8 *iv, 119 u8 *hash_subkey, const u8 *aad, unsigned long aad_len, 120 u8 *auth_tag, unsigned long auth_tag_len); 121 122 /* asmlinkage void aesni_gcm_dec() 123 * void *ctx, AES Key schedule. Starts on a 16 byte boundary. 124 * u8 *out, Plaintext output. Decrypt in-place is allowed. 125 * const u8 *in, Ciphertext input 126 * unsigned long ciphertext_len, Length of data in bytes for decryption. 127 * u8 *iv, Pre-counter block j0: 4 byte salt (from Security Association) 128 * concatenated with 8 byte Initialisation Vector (from IPSec ESP 129 * Payload) concatenated with 0x00000001. 16-byte aligned pointer. 130 * u8 *hash_subkey, the Hash sub key input. Data starts on a 16-byte boundary. 131 * const u8 *aad, Additional Authentication Data (AAD) 132 * unsigned long aad_len, Length of AAD in bytes. With RFC4106 this is going 133 * to be 8 or 12 bytes 134 * u8 *auth_tag, Authenticated Tag output. 135 * unsigned long auth_tag_len) Authenticated Tag Length in bytes. 136 * Valid values are 16 (most likely), 12 or 8. 137 */ 138 asmlinkage void aesni_gcm_dec(void *ctx, u8 *out, 139 const u8 *in, unsigned long ciphertext_len, u8 *iv, 140 u8 *hash_subkey, const u8 *aad, unsigned long aad_len, 141 u8 *auth_tag, unsigned long auth_tag_len); 142 143 static inline struct 144 aesni_rfc4106_gcm_ctx *aesni_rfc4106_gcm_ctx_get(struct crypto_aead *tfm) 145 { 146 return 147 (struct aesni_rfc4106_gcm_ctx *) 148 PTR_ALIGN((u8 *) 149 crypto_tfm_ctx(crypto_aead_tfm(tfm)), AESNI_ALIGN); 150 } 151 #endif 152 153 static inline struct crypto_aes_ctx *aes_ctx(void *raw_ctx) 154 { 155 unsigned long addr = (unsigned long)raw_ctx; 156 unsigned long align = AESNI_ALIGN; 157 158 if (align <= crypto_tfm_ctx_alignment()) 159 align = 1; 160 return (struct crypto_aes_ctx *)ALIGN(addr, align); 161 } 162 163 static int aes_set_key_common(struct crypto_tfm *tfm, void *raw_ctx, 164 const u8 *in_key, unsigned int key_len) 165 { 166 struct crypto_aes_ctx *ctx = aes_ctx(raw_ctx); 167 u32 *flags = &tfm->crt_flags; 168 int err; 169 170 if (key_len != AES_KEYSIZE_128 && key_len != AES_KEYSIZE_192 && 171 key_len != AES_KEYSIZE_256) { 172 *flags |= CRYPTO_TFM_RES_BAD_KEY_LEN; 173 return -EINVAL; 174 } 175 176 if (!irq_fpu_usable()) 177 err = crypto_aes_expand_key(ctx, in_key, key_len); 178 else { 179 kernel_fpu_begin(); 180 err = aesni_set_key(ctx, in_key, key_len); 181 kernel_fpu_end(); 182 } 183 184 return err; 185 } 186 187 static int aes_set_key(struct crypto_tfm *tfm, const u8 *in_key, 188 unsigned int key_len) 189 { 190 return aes_set_key_common(tfm, crypto_tfm_ctx(tfm), in_key, key_len); 191 } 192 193 static void aes_encrypt(struct crypto_tfm *tfm, u8 *dst, const u8 *src) 194 { 195 struct crypto_aes_ctx *ctx = aes_ctx(crypto_tfm_ctx(tfm)); 196 197 if (!irq_fpu_usable()) 198 crypto_aes_encrypt_x86(ctx, dst, src); 199 else { 200 kernel_fpu_begin(); 201 aesni_enc(ctx, dst, src); 202 kernel_fpu_end(); 203 } 204 } 205 206 static void aes_decrypt(struct crypto_tfm *tfm, u8 *dst, const u8 *src) 207 { 208 struct crypto_aes_ctx *ctx = aes_ctx(crypto_tfm_ctx(tfm)); 209 210 if (!irq_fpu_usable()) 211 crypto_aes_decrypt_x86(ctx, dst, src); 212 else { 213 kernel_fpu_begin(); 214 aesni_dec(ctx, dst, src); 215 kernel_fpu_end(); 216 } 217 } 218 219 static struct crypto_alg aesni_alg = { 220 .cra_name = "aes", 221 .cra_driver_name = "aes-aesni", 222 .cra_priority = 300, 223 .cra_flags = CRYPTO_ALG_TYPE_CIPHER, 224 .cra_blocksize = AES_BLOCK_SIZE, 225 .cra_ctxsize = sizeof(struct crypto_aes_ctx)+AESNI_ALIGN-1, 226 .cra_alignmask = 0, 227 .cra_module = THIS_MODULE, 228 .cra_list = LIST_HEAD_INIT(aesni_alg.cra_list), 229 .cra_u = { 230 .cipher = { 231 .cia_min_keysize = AES_MIN_KEY_SIZE, 232 .cia_max_keysize = AES_MAX_KEY_SIZE, 233 .cia_setkey = aes_set_key, 234 .cia_encrypt = aes_encrypt, 235 .cia_decrypt = aes_decrypt 236 } 237 } 238 }; 239 240 static void __aes_encrypt(struct crypto_tfm *tfm, u8 *dst, const u8 *src) 241 { 242 struct crypto_aes_ctx *ctx = aes_ctx(crypto_tfm_ctx(tfm)); 243 244 aesni_enc(ctx, dst, src); 245 } 246 247 static void __aes_decrypt(struct crypto_tfm *tfm, u8 *dst, const u8 *src) 248 { 249 struct crypto_aes_ctx *ctx = aes_ctx(crypto_tfm_ctx(tfm)); 250 251 aesni_dec(ctx, dst, src); 252 } 253 254 static struct crypto_alg __aesni_alg = { 255 .cra_name = "__aes-aesni", 256 .cra_driver_name = "__driver-aes-aesni", 257 .cra_priority = 0, 258 .cra_flags = CRYPTO_ALG_TYPE_CIPHER, 259 .cra_blocksize = AES_BLOCK_SIZE, 260 .cra_ctxsize = sizeof(struct crypto_aes_ctx)+AESNI_ALIGN-1, 261 .cra_alignmask = 0, 262 .cra_module = THIS_MODULE, 263 .cra_list = LIST_HEAD_INIT(__aesni_alg.cra_list), 264 .cra_u = { 265 .cipher = { 266 .cia_min_keysize = AES_MIN_KEY_SIZE, 267 .cia_max_keysize = AES_MAX_KEY_SIZE, 268 .cia_setkey = aes_set_key, 269 .cia_encrypt = __aes_encrypt, 270 .cia_decrypt = __aes_decrypt 271 } 272 } 273 }; 274 275 static int ecb_encrypt(struct blkcipher_desc *desc, 276 struct scatterlist *dst, struct scatterlist *src, 277 unsigned int nbytes) 278 { 279 struct crypto_aes_ctx *ctx = aes_ctx(crypto_blkcipher_ctx(desc->tfm)); 280 struct blkcipher_walk walk; 281 int err; 282 283 blkcipher_walk_init(&walk, dst, src, nbytes); 284 err = blkcipher_walk_virt(desc, &walk); 285 desc->flags &= ~CRYPTO_TFM_REQ_MAY_SLEEP; 286 287 kernel_fpu_begin(); 288 while ((nbytes = walk.nbytes)) { 289 aesni_ecb_enc(ctx, walk.dst.virt.addr, walk.src.virt.addr, 290 nbytes & AES_BLOCK_MASK); 291 nbytes &= AES_BLOCK_SIZE - 1; 292 err = blkcipher_walk_done(desc, &walk, nbytes); 293 } 294 kernel_fpu_end(); 295 296 return err; 297 } 298 299 static int ecb_decrypt(struct blkcipher_desc *desc, 300 struct scatterlist *dst, struct scatterlist *src, 301 unsigned int nbytes) 302 { 303 struct crypto_aes_ctx *ctx = aes_ctx(crypto_blkcipher_ctx(desc->tfm)); 304 struct blkcipher_walk walk; 305 int err; 306 307 blkcipher_walk_init(&walk, dst, src, nbytes); 308 err = blkcipher_walk_virt(desc, &walk); 309 desc->flags &= ~CRYPTO_TFM_REQ_MAY_SLEEP; 310 311 kernel_fpu_begin(); 312 while ((nbytes = walk.nbytes)) { 313 aesni_ecb_dec(ctx, walk.dst.virt.addr, walk.src.virt.addr, 314 nbytes & AES_BLOCK_MASK); 315 nbytes &= AES_BLOCK_SIZE - 1; 316 err = blkcipher_walk_done(desc, &walk, nbytes); 317 } 318 kernel_fpu_end(); 319 320 return err; 321 } 322 323 static struct crypto_alg blk_ecb_alg = { 324 .cra_name = "__ecb-aes-aesni", 325 .cra_driver_name = "__driver-ecb-aes-aesni", 326 .cra_priority = 0, 327 .cra_flags = CRYPTO_ALG_TYPE_BLKCIPHER, 328 .cra_blocksize = AES_BLOCK_SIZE, 329 .cra_ctxsize = sizeof(struct crypto_aes_ctx)+AESNI_ALIGN-1, 330 .cra_alignmask = 0, 331 .cra_type = &crypto_blkcipher_type, 332 .cra_module = THIS_MODULE, 333 .cra_list = LIST_HEAD_INIT(blk_ecb_alg.cra_list), 334 .cra_u = { 335 .blkcipher = { 336 .min_keysize = AES_MIN_KEY_SIZE, 337 .max_keysize = AES_MAX_KEY_SIZE, 338 .setkey = aes_set_key, 339 .encrypt = ecb_encrypt, 340 .decrypt = ecb_decrypt, 341 }, 342 }, 343 }; 344 345 static int cbc_encrypt(struct blkcipher_desc *desc, 346 struct scatterlist *dst, struct scatterlist *src, 347 unsigned int nbytes) 348 { 349 struct crypto_aes_ctx *ctx = aes_ctx(crypto_blkcipher_ctx(desc->tfm)); 350 struct blkcipher_walk walk; 351 int err; 352 353 blkcipher_walk_init(&walk, dst, src, nbytes); 354 err = blkcipher_walk_virt(desc, &walk); 355 desc->flags &= ~CRYPTO_TFM_REQ_MAY_SLEEP; 356 357 kernel_fpu_begin(); 358 while ((nbytes = walk.nbytes)) { 359 aesni_cbc_enc(ctx, walk.dst.virt.addr, walk.src.virt.addr, 360 nbytes & AES_BLOCK_MASK, walk.iv); 361 nbytes &= AES_BLOCK_SIZE - 1; 362 err = blkcipher_walk_done(desc, &walk, nbytes); 363 } 364 kernel_fpu_end(); 365 366 return err; 367 } 368 369 static int cbc_decrypt(struct blkcipher_desc *desc, 370 struct scatterlist *dst, struct scatterlist *src, 371 unsigned int nbytes) 372 { 373 struct crypto_aes_ctx *ctx = aes_ctx(crypto_blkcipher_ctx(desc->tfm)); 374 struct blkcipher_walk walk; 375 int err; 376 377 blkcipher_walk_init(&walk, dst, src, nbytes); 378 err = blkcipher_walk_virt(desc, &walk); 379 desc->flags &= ~CRYPTO_TFM_REQ_MAY_SLEEP; 380 381 kernel_fpu_begin(); 382 while ((nbytes = walk.nbytes)) { 383 aesni_cbc_dec(ctx, walk.dst.virt.addr, walk.src.virt.addr, 384 nbytes & AES_BLOCK_MASK, walk.iv); 385 nbytes &= AES_BLOCK_SIZE - 1; 386 err = blkcipher_walk_done(desc, &walk, nbytes); 387 } 388 kernel_fpu_end(); 389 390 return err; 391 } 392 393 static struct crypto_alg blk_cbc_alg = { 394 .cra_name = "__cbc-aes-aesni", 395 .cra_driver_name = "__driver-cbc-aes-aesni", 396 .cra_priority = 0, 397 .cra_flags = CRYPTO_ALG_TYPE_BLKCIPHER, 398 .cra_blocksize = AES_BLOCK_SIZE, 399 .cra_ctxsize = sizeof(struct crypto_aes_ctx)+AESNI_ALIGN-1, 400 .cra_alignmask = 0, 401 .cra_type = &crypto_blkcipher_type, 402 .cra_module = THIS_MODULE, 403 .cra_list = LIST_HEAD_INIT(blk_cbc_alg.cra_list), 404 .cra_u = { 405 .blkcipher = { 406 .min_keysize = AES_MIN_KEY_SIZE, 407 .max_keysize = AES_MAX_KEY_SIZE, 408 .setkey = aes_set_key, 409 .encrypt = cbc_encrypt, 410 .decrypt = cbc_decrypt, 411 }, 412 }, 413 }; 414 415 #ifdef CONFIG_X86_64 416 static void ctr_crypt_final(struct crypto_aes_ctx *ctx, 417 struct blkcipher_walk *walk) 418 { 419 u8 *ctrblk = walk->iv; 420 u8 keystream[AES_BLOCK_SIZE]; 421 u8 *src = walk->src.virt.addr; 422 u8 *dst = walk->dst.virt.addr; 423 unsigned int nbytes = walk->nbytes; 424 425 aesni_enc(ctx, keystream, ctrblk); 426 crypto_xor(keystream, src, nbytes); 427 memcpy(dst, keystream, nbytes); 428 crypto_inc(ctrblk, AES_BLOCK_SIZE); 429 } 430 431 static int ctr_crypt(struct blkcipher_desc *desc, 432 struct scatterlist *dst, struct scatterlist *src, 433 unsigned int nbytes) 434 { 435 struct crypto_aes_ctx *ctx = aes_ctx(crypto_blkcipher_ctx(desc->tfm)); 436 struct blkcipher_walk walk; 437 int err; 438 439 blkcipher_walk_init(&walk, dst, src, nbytes); 440 err = blkcipher_walk_virt_block(desc, &walk, AES_BLOCK_SIZE); 441 desc->flags &= ~CRYPTO_TFM_REQ_MAY_SLEEP; 442 443 kernel_fpu_begin(); 444 while ((nbytes = walk.nbytes) >= AES_BLOCK_SIZE) { 445 aesni_ctr_enc(ctx, walk.dst.virt.addr, walk.src.virt.addr, 446 nbytes & AES_BLOCK_MASK, walk.iv); 447 nbytes &= AES_BLOCK_SIZE - 1; 448 err = blkcipher_walk_done(desc, &walk, nbytes); 449 } 450 if (walk.nbytes) { 451 ctr_crypt_final(ctx, &walk); 452 err = blkcipher_walk_done(desc, &walk, 0); 453 } 454 kernel_fpu_end(); 455 456 return err; 457 } 458 459 static struct crypto_alg blk_ctr_alg = { 460 .cra_name = "__ctr-aes-aesni", 461 .cra_driver_name = "__driver-ctr-aes-aesni", 462 .cra_priority = 0, 463 .cra_flags = CRYPTO_ALG_TYPE_BLKCIPHER, 464 .cra_blocksize = 1, 465 .cra_ctxsize = sizeof(struct crypto_aes_ctx)+AESNI_ALIGN-1, 466 .cra_alignmask = 0, 467 .cra_type = &crypto_blkcipher_type, 468 .cra_module = THIS_MODULE, 469 .cra_list = LIST_HEAD_INIT(blk_ctr_alg.cra_list), 470 .cra_u = { 471 .blkcipher = { 472 .min_keysize = AES_MIN_KEY_SIZE, 473 .max_keysize = AES_MAX_KEY_SIZE, 474 .ivsize = AES_BLOCK_SIZE, 475 .setkey = aes_set_key, 476 .encrypt = ctr_crypt, 477 .decrypt = ctr_crypt, 478 }, 479 }, 480 }; 481 #endif 482 483 static int ablk_set_key(struct crypto_ablkcipher *tfm, const u8 *key, 484 unsigned int key_len) 485 { 486 struct async_aes_ctx *ctx = crypto_ablkcipher_ctx(tfm); 487 struct crypto_ablkcipher *child = &ctx->cryptd_tfm->base; 488 int err; 489 490 crypto_ablkcipher_clear_flags(child, CRYPTO_TFM_REQ_MASK); 491 crypto_ablkcipher_set_flags(child, crypto_ablkcipher_get_flags(tfm) 492 & CRYPTO_TFM_REQ_MASK); 493 err = crypto_ablkcipher_setkey(child, key, key_len); 494 crypto_ablkcipher_set_flags(tfm, crypto_ablkcipher_get_flags(child) 495 & CRYPTO_TFM_RES_MASK); 496 return err; 497 } 498 499 static int ablk_encrypt(struct ablkcipher_request *req) 500 { 501 struct crypto_ablkcipher *tfm = crypto_ablkcipher_reqtfm(req); 502 struct async_aes_ctx *ctx = crypto_ablkcipher_ctx(tfm); 503 504 if (!irq_fpu_usable()) { 505 struct ablkcipher_request *cryptd_req = 506 ablkcipher_request_ctx(req); 507 memcpy(cryptd_req, req, sizeof(*req)); 508 ablkcipher_request_set_tfm(cryptd_req, &ctx->cryptd_tfm->base); 509 return crypto_ablkcipher_encrypt(cryptd_req); 510 } else { 511 struct blkcipher_desc desc; 512 desc.tfm = cryptd_ablkcipher_child(ctx->cryptd_tfm); 513 desc.info = req->info; 514 desc.flags = 0; 515 return crypto_blkcipher_crt(desc.tfm)->encrypt( 516 &desc, req->dst, req->src, req->nbytes); 517 } 518 } 519 520 static int ablk_decrypt(struct ablkcipher_request *req) 521 { 522 struct crypto_ablkcipher *tfm = crypto_ablkcipher_reqtfm(req); 523 struct async_aes_ctx *ctx = crypto_ablkcipher_ctx(tfm); 524 525 if (!irq_fpu_usable()) { 526 struct ablkcipher_request *cryptd_req = 527 ablkcipher_request_ctx(req); 528 memcpy(cryptd_req, req, sizeof(*req)); 529 ablkcipher_request_set_tfm(cryptd_req, &ctx->cryptd_tfm->base); 530 return crypto_ablkcipher_decrypt(cryptd_req); 531 } else { 532 struct blkcipher_desc desc; 533 desc.tfm = cryptd_ablkcipher_child(ctx->cryptd_tfm); 534 desc.info = req->info; 535 desc.flags = 0; 536 return crypto_blkcipher_crt(desc.tfm)->decrypt( 537 &desc, req->dst, req->src, req->nbytes); 538 } 539 } 540 541 static void ablk_exit(struct crypto_tfm *tfm) 542 { 543 struct async_aes_ctx *ctx = crypto_tfm_ctx(tfm); 544 545 cryptd_free_ablkcipher(ctx->cryptd_tfm); 546 } 547 548 static void ablk_init_common(struct crypto_tfm *tfm, 549 struct cryptd_ablkcipher *cryptd_tfm) 550 { 551 struct async_aes_ctx *ctx = crypto_tfm_ctx(tfm); 552 553 ctx->cryptd_tfm = cryptd_tfm; 554 tfm->crt_ablkcipher.reqsize = sizeof(struct ablkcipher_request) + 555 crypto_ablkcipher_reqsize(&cryptd_tfm->base); 556 } 557 558 static int ablk_ecb_init(struct crypto_tfm *tfm) 559 { 560 struct cryptd_ablkcipher *cryptd_tfm; 561 562 cryptd_tfm = cryptd_alloc_ablkcipher("__driver-ecb-aes-aesni", 0, 0); 563 if (IS_ERR(cryptd_tfm)) 564 return PTR_ERR(cryptd_tfm); 565 ablk_init_common(tfm, cryptd_tfm); 566 return 0; 567 } 568 569 static struct crypto_alg ablk_ecb_alg = { 570 .cra_name = "ecb(aes)", 571 .cra_driver_name = "ecb-aes-aesni", 572 .cra_priority = 400, 573 .cra_flags = CRYPTO_ALG_TYPE_ABLKCIPHER|CRYPTO_ALG_ASYNC, 574 .cra_blocksize = AES_BLOCK_SIZE, 575 .cra_ctxsize = sizeof(struct async_aes_ctx), 576 .cra_alignmask = 0, 577 .cra_type = &crypto_ablkcipher_type, 578 .cra_module = THIS_MODULE, 579 .cra_list = LIST_HEAD_INIT(ablk_ecb_alg.cra_list), 580 .cra_init = ablk_ecb_init, 581 .cra_exit = ablk_exit, 582 .cra_u = { 583 .ablkcipher = { 584 .min_keysize = AES_MIN_KEY_SIZE, 585 .max_keysize = AES_MAX_KEY_SIZE, 586 .setkey = ablk_set_key, 587 .encrypt = ablk_encrypt, 588 .decrypt = ablk_decrypt, 589 }, 590 }, 591 }; 592 593 static int ablk_cbc_init(struct crypto_tfm *tfm) 594 { 595 struct cryptd_ablkcipher *cryptd_tfm; 596 597 cryptd_tfm = cryptd_alloc_ablkcipher("__driver-cbc-aes-aesni", 0, 0); 598 if (IS_ERR(cryptd_tfm)) 599 return PTR_ERR(cryptd_tfm); 600 ablk_init_common(tfm, cryptd_tfm); 601 return 0; 602 } 603 604 static struct crypto_alg ablk_cbc_alg = { 605 .cra_name = "cbc(aes)", 606 .cra_driver_name = "cbc-aes-aesni", 607 .cra_priority = 400, 608 .cra_flags = CRYPTO_ALG_TYPE_ABLKCIPHER|CRYPTO_ALG_ASYNC, 609 .cra_blocksize = AES_BLOCK_SIZE, 610 .cra_ctxsize = sizeof(struct async_aes_ctx), 611 .cra_alignmask = 0, 612 .cra_type = &crypto_ablkcipher_type, 613 .cra_module = THIS_MODULE, 614 .cra_list = LIST_HEAD_INIT(ablk_cbc_alg.cra_list), 615 .cra_init = ablk_cbc_init, 616 .cra_exit = ablk_exit, 617 .cra_u = { 618 .ablkcipher = { 619 .min_keysize = AES_MIN_KEY_SIZE, 620 .max_keysize = AES_MAX_KEY_SIZE, 621 .ivsize = AES_BLOCK_SIZE, 622 .setkey = ablk_set_key, 623 .encrypt = ablk_encrypt, 624 .decrypt = ablk_decrypt, 625 }, 626 }, 627 }; 628 629 #ifdef CONFIG_X86_64 630 static int ablk_ctr_init(struct crypto_tfm *tfm) 631 { 632 struct cryptd_ablkcipher *cryptd_tfm; 633 634 cryptd_tfm = cryptd_alloc_ablkcipher("__driver-ctr-aes-aesni", 0, 0); 635 if (IS_ERR(cryptd_tfm)) 636 return PTR_ERR(cryptd_tfm); 637 ablk_init_common(tfm, cryptd_tfm); 638 return 0; 639 } 640 641 static struct crypto_alg ablk_ctr_alg = { 642 .cra_name = "ctr(aes)", 643 .cra_driver_name = "ctr-aes-aesni", 644 .cra_priority = 400, 645 .cra_flags = CRYPTO_ALG_TYPE_ABLKCIPHER|CRYPTO_ALG_ASYNC, 646 .cra_blocksize = 1, 647 .cra_ctxsize = sizeof(struct async_aes_ctx), 648 .cra_alignmask = 0, 649 .cra_type = &crypto_ablkcipher_type, 650 .cra_module = THIS_MODULE, 651 .cra_list = LIST_HEAD_INIT(ablk_ctr_alg.cra_list), 652 .cra_init = ablk_ctr_init, 653 .cra_exit = ablk_exit, 654 .cra_u = { 655 .ablkcipher = { 656 .min_keysize = AES_MIN_KEY_SIZE, 657 .max_keysize = AES_MAX_KEY_SIZE, 658 .ivsize = AES_BLOCK_SIZE, 659 .setkey = ablk_set_key, 660 .encrypt = ablk_encrypt, 661 .decrypt = ablk_encrypt, 662 .geniv = "chainiv", 663 }, 664 }, 665 }; 666 667 #ifdef HAS_CTR 668 static int ablk_rfc3686_ctr_init(struct crypto_tfm *tfm) 669 { 670 struct cryptd_ablkcipher *cryptd_tfm; 671 672 cryptd_tfm = cryptd_alloc_ablkcipher( 673 "rfc3686(__driver-ctr-aes-aesni)", 0, 0); 674 if (IS_ERR(cryptd_tfm)) 675 return PTR_ERR(cryptd_tfm); 676 ablk_init_common(tfm, cryptd_tfm); 677 return 0; 678 } 679 680 static struct crypto_alg ablk_rfc3686_ctr_alg = { 681 .cra_name = "rfc3686(ctr(aes))", 682 .cra_driver_name = "rfc3686-ctr-aes-aesni", 683 .cra_priority = 400, 684 .cra_flags = CRYPTO_ALG_TYPE_ABLKCIPHER|CRYPTO_ALG_ASYNC, 685 .cra_blocksize = 1, 686 .cra_ctxsize = sizeof(struct async_aes_ctx), 687 .cra_alignmask = 0, 688 .cra_type = &crypto_ablkcipher_type, 689 .cra_module = THIS_MODULE, 690 .cra_list = LIST_HEAD_INIT(ablk_rfc3686_ctr_alg.cra_list), 691 .cra_init = ablk_rfc3686_ctr_init, 692 .cra_exit = ablk_exit, 693 .cra_u = { 694 .ablkcipher = { 695 .min_keysize = AES_MIN_KEY_SIZE+CTR_RFC3686_NONCE_SIZE, 696 .max_keysize = AES_MAX_KEY_SIZE+CTR_RFC3686_NONCE_SIZE, 697 .ivsize = CTR_RFC3686_IV_SIZE, 698 .setkey = ablk_set_key, 699 .encrypt = ablk_encrypt, 700 .decrypt = ablk_decrypt, 701 .geniv = "seqiv", 702 }, 703 }, 704 }; 705 #endif 706 #endif 707 708 #ifdef HAS_LRW 709 static int ablk_lrw_init(struct crypto_tfm *tfm) 710 { 711 struct cryptd_ablkcipher *cryptd_tfm; 712 713 cryptd_tfm = cryptd_alloc_ablkcipher("fpu(lrw(__driver-aes-aesni))", 714 0, 0); 715 if (IS_ERR(cryptd_tfm)) 716 return PTR_ERR(cryptd_tfm); 717 ablk_init_common(tfm, cryptd_tfm); 718 return 0; 719 } 720 721 static struct crypto_alg ablk_lrw_alg = { 722 .cra_name = "lrw(aes)", 723 .cra_driver_name = "lrw-aes-aesni", 724 .cra_priority = 400, 725 .cra_flags = CRYPTO_ALG_TYPE_ABLKCIPHER|CRYPTO_ALG_ASYNC, 726 .cra_blocksize = AES_BLOCK_SIZE, 727 .cra_ctxsize = sizeof(struct async_aes_ctx), 728 .cra_alignmask = 0, 729 .cra_type = &crypto_ablkcipher_type, 730 .cra_module = THIS_MODULE, 731 .cra_list = LIST_HEAD_INIT(ablk_lrw_alg.cra_list), 732 .cra_init = ablk_lrw_init, 733 .cra_exit = ablk_exit, 734 .cra_u = { 735 .ablkcipher = { 736 .min_keysize = AES_MIN_KEY_SIZE + AES_BLOCK_SIZE, 737 .max_keysize = AES_MAX_KEY_SIZE + AES_BLOCK_SIZE, 738 .ivsize = AES_BLOCK_SIZE, 739 .setkey = ablk_set_key, 740 .encrypt = ablk_encrypt, 741 .decrypt = ablk_decrypt, 742 }, 743 }, 744 }; 745 #endif 746 747 #ifdef HAS_PCBC 748 static int ablk_pcbc_init(struct crypto_tfm *tfm) 749 { 750 struct cryptd_ablkcipher *cryptd_tfm; 751 752 cryptd_tfm = cryptd_alloc_ablkcipher("fpu(pcbc(__driver-aes-aesni))", 753 0, 0); 754 if (IS_ERR(cryptd_tfm)) 755 return PTR_ERR(cryptd_tfm); 756 ablk_init_common(tfm, cryptd_tfm); 757 return 0; 758 } 759 760 static struct crypto_alg ablk_pcbc_alg = { 761 .cra_name = "pcbc(aes)", 762 .cra_driver_name = "pcbc-aes-aesni", 763 .cra_priority = 400, 764 .cra_flags = CRYPTO_ALG_TYPE_ABLKCIPHER|CRYPTO_ALG_ASYNC, 765 .cra_blocksize = AES_BLOCK_SIZE, 766 .cra_ctxsize = sizeof(struct async_aes_ctx), 767 .cra_alignmask = 0, 768 .cra_type = &crypto_ablkcipher_type, 769 .cra_module = THIS_MODULE, 770 .cra_list = LIST_HEAD_INIT(ablk_pcbc_alg.cra_list), 771 .cra_init = ablk_pcbc_init, 772 .cra_exit = ablk_exit, 773 .cra_u = { 774 .ablkcipher = { 775 .min_keysize = AES_MIN_KEY_SIZE, 776 .max_keysize = AES_MAX_KEY_SIZE, 777 .ivsize = AES_BLOCK_SIZE, 778 .setkey = ablk_set_key, 779 .encrypt = ablk_encrypt, 780 .decrypt = ablk_decrypt, 781 }, 782 }, 783 }; 784 #endif 785 786 #ifdef HAS_XTS 787 static int ablk_xts_init(struct crypto_tfm *tfm) 788 { 789 struct cryptd_ablkcipher *cryptd_tfm; 790 791 cryptd_tfm = cryptd_alloc_ablkcipher("fpu(xts(__driver-aes-aesni))", 792 0, 0); 793 if (IS_ERR(cryptd_tfm)) 794 return PTR_ERR(cryptd_tfm); 795 ablk_init_common(tfm, cryptd_tfm); 796 return 0; 797 } 798 799 static struct crypto_alg ablk_xts_alg = { 800 .cra_name = "xts(aes)", 801 .cra_driver_name = "xts-aes-aesni", 802 .cra_priority = 400, 803 .cra_flags = CRYPTO_ALG_TYPE_ABLKCIPHER|CRYPTO_ALG_ASYNC, 804 .cra_blocksize = AES_BLOCK_SIZE, 805 .cra_ctxsize = sizeof(struct async_aes_ctx), 806 .cra_alignmask = 0, 807 .cra_type = &crypto_ablkcipher_type, 808 .cra_module = THIS_MODULE, 809 .cra_list = LIST_HEAD_INIT(ablk_xts_alg.cra_list), 810 .cra_init = ablk_xts_init, 811 .cra_exit = ablk_exit, 812 .cra_u = { 813 .ablkcipher = { 814 .min_keysize = 2 * AES_MIN_KEY_SIZE, 815 .max_keysize = 2 * AES_MAX_KEY_SIZE, 816 .ivsize = AES_BLOCK_SIZE, 817 .setkey = ablk_set_key, 818 .encrypt = ablk_encrypt, 819 .decrypt = ablk_decrypt, 820 }, 821 }, 822 }; 823 #endif 824 825 #ifdef CONFIG_X86_64 826 static int rfc4106_init(struct crypto_tfm *tfm) 827 { 828 struct cryptd_aead *cryptd_tfm; 829 struct aesni_rfc4106_gcm_ctx *ctx = (struct aesni_rfc4106_gcm_ctx *) 830 PTR_ALIGN((u8 *)crypto_tfm_ctx(tfm), AESNI_ALIGN); 831 cryptd_tfm = cryptd_alloc_aead("__driver-gcm-aes-aesni", 0, 0); 832 if (IS_ERR(cryptd_tfm)) 833 return PTR_ERR(cryptd_tfm); 834 ctx->cryptd_tfm = cryptd_tfm; 835 tfm->crt_aead.reqsize = sizeof(struct aead_request) 836 + crypto_aead_reqsize(&cryptd_tfm->base); 837 return 0; 838 } 839 840 static void rfc4106_exit(struct crypto_tfm *tfm) 841 { 842 struct aesni_rfc4106_gcm_ctx *ctx = 843 (struct aesni_rfc4106_gcm_ctx *) 844 PTR_ALIGN((u8 *)crypto_tfm_ctx(tfm), AESNI_ALIGN); 845 if (!IS_ERR(ctx->cryptd_tfm)) 846 cryptd_free_aead(ctx->cryptd_tfm); 847 return; 848 } 849 850 static void 851 rfc4106_set_hash_subkey_done(struct crypto_async_request *req, int err) 852 { 853 struct aesni_gcm_set_hash_subkey_result *result = req->data; 854 855 if (err == -EINPROGRESS) 856 return; 857 result->err = err; 858 complete(&result->completion); 859 } 860 861 static int 862 rfc4106_set_hash_subkey(u8 *hash_subkey, const u8 *key, unsigned int key_len) 863 { 864 struct crypto_ablkcipher *ctr_tfm; 865 struct ablkcipher_request *req; 866 int ret = -EINVAL; 867 struct aesni_hash_subkey_req_data *req_data; 868 869 ctr_tfm = crypto_alloc_ablkcipher("ctr(aes)", 0, 0); 870 if (IS_ERR(ctr_tfm)) 871 return PTR_ERR(ctr_tfm); 872 873 crypto_ablkcipher_clear_flags(ctr_tfm, ~0); 874 875 ret = crypto_ablkcipher_setkey(ctr_tfm, key, key_len); 876 if (ret) { 877 crypto_free_ablkcipher(ctr_tfm); 878 return ret; 879 } 880 881 req = ablkcipher_request_alloc(ctr_tfm, GFP_KERNEL); 882 if (!req) { 883 crypto_free_ablkcipher(ctr_tfm); 884 return -EINVAL; 885 } 886 887 req_data = kmalloc(sizeof(*req_data), GFP_KERNEL); 888 if (!req_data) { 889 crypto_free_ablkcipher(ctr_tfm); 890 return -ENOMEM; 891 } 892 memset(req_data->iv, 0, sizeof(req_data->iv)); 893 894 /* Clear the data in the hash sub key container to zero.*/ 895 /* We want to cipher all zeros to create the hash sub key. */ 896 memset(hash_subkey, 0, RFC4106_HASH_SUBKEY_SIZE); 897 898 init_completion(&req_data->result.completion); 899 sg_init_one(&req_data->sg, hash_subkey, RFC4106_HASH_SUBKEY_SIZE); 900 ablkcipher_request_set_tfm(req, ctr_tfm); 901 ablkcipher_request_set_callback(req, CRYPTO_TFM_REQ_MAY_SLEEP | 902 CRYPTO_TFM_REQ_MAY_BACKLOG, 903 rfc4106_set_hash_subkey_done, 904 &req_data->result); 905 906 ablkcipher_request_set_crypt(req, &req_data->sg, 907 &req_data->sg, RFC4106_HASH_SUBKEY_SIZE, req_data->iv); 908 909 ret = crypto_ablkcipher_encrypt(req); 910 if (ret == -EINPROGRESS || ret == -EBUSY) { 911 ret = wait_for_completion_interruptible 912 (&req_data->result.completion); 913 if (!ret) 914 ret = req_data->result.err; 915 } 916 ablkcipher_request_free(req); 917 kfree(req_data); 918 crypto_free_ablkcipher(ctr_tfm); 919 return ret; 920 } 921 922 static int rfc4106_set_key(struct crypto_aead *parent, const u8 *key, 923 unsigned int key_len) 924 { 925 int ret = 0; 926 struct crypto_tfm *tfm = crypto_aead_tfm(parent); 927 struct aesni_rfc4106_gcm_ctx *ctx = aesni_rfc4106_gcm_ctx_get(parent); 928 u8 *new_key_mem = NULL; 929 930 if (key_len < 4) { 931 crypto_tfm_set_flags(tfm, CRYPTO_TFM_RES_BAD_KEY_LEN); 932 return -EINVAL; 933 } 934 /*Account for 4 byte nonce at the end.*/ 935 key_len -= 4; 936 if (key_len != AES_KEYSIZE_128) { 937 crypto_tfm_set_flags(tfm, CRYPTO_TFM_RES_BAD_KEY_LEN); 938 return -EINVAL; 939 } 940 941 memcpy(ctx->nonce, key + key_len, sizeof(ctx->nonce)); 942 /*This must be on a 16 byte boundary!*/ 943 if ((unsigned long)(&(ctx->aes_key_expanded.key_enc[0])) % AESNI_ALIGN) 944 return -EINVAL; 945 946 if ((unsigned long)key % AESNI_ALIGN) { 947 /*key is not aligned: use an auxuliar aligned pointer*/ 948 new_key_mem = kmalloc(key_len+AESNI_ALIGN, GFP_KERNEL); 949 if (!new_key_mem) 950 return -ENOMEM; 951 952 new_key_mem = PTR_ALIGN(new_key_mem, AESNI_ALIGN); 953 memcpy(new_key_mem, key, key_len); 954 key = new_key_mem; 955 } 956 957 if (!irq_fpu_usable()) 958 ret = crypto_aes_expand_key(&(ctx->aes_key_expanded), 959 key, key_len); 960 else { 961 kernel_fpu_begin(); 962 ret = aesni_set_key(&(ctx->aes_key_expanded), key, key_len); 963 kernel_fpu_end(); 964 } 965 /*This must be on a 16 byte boundary!*/ 966 if ((unsigned long)(&(ctx->hash_subkey[0])) % AESNI_ALIGN) { 967 ret = -EINVAL; 968 goto exit; 969 } 970 ret = rfc4106_set_hash_subkey(ctx->hash_subkey, key, key_len); 971 exit: 972 kfree(new_key_mem); 973 return ret; 974 } 975 976 /* This is the Integrity Check Value (aka the authentication tag length and can 977 * be 8, 12 or 16 bytes long. */ 978 static int rfc4106_set_authsize(struct crypto_aead *parent, 979 unsigned int authsize) 980 { 981 struct aesni_rfc4106_gcm_ctx *ctx = aesni_rfc4106_gcm_ctx_get(parent); 982 struct crypto_aead *cryptd_child = cryptd_aead_child(ctx->cryptd_tfm); 983 984 switch (authsize) { 985 case 8: 986 case 12: 987 case 16: 988 break; 989 default: 990 return -EINVAL; 991 } 992 crypto_aead_crt(parent)->authsize = authsize; 993 crypto_aead_crt(cryptd_child)->authsize = authsize; 994 return 0; 995 } 996 997 static int rfc4106_encrypt(struct aead_request *req) 998 { 999 int ret; 1000 struct crypto_aead *tfm = crypto_aead_reqtfm(req); 1001 struct aesni_rfc4106_gcm_ctx *ctx = aesni_rfc4106_gcm_ctx_get(tfm); 1002 struct crypto_aead *cryptd_child = cryptd_aead_child(ctx->cryptd_tfm); 1003 1004 if (!irq_fpu_usable()) { 1005 struct aead_request *cryptd_req = 1006 (struct aead_request *) aead_request_ctx(req); 1007 memcpy(cryptd_req, req, sizeof(*req)); 1008 aead_request_set_tfm(cryptd_req, &ctx->cryptd_tfm->base); 1009 return crypto_aead_encrypt(cryptd_req); 1010 } else { 1011 kernel_fpu_begin(); 1012 ret = cryptd_child->base.crt_aead.encrypt(req); 1013 kernel_fpu_end(); 1014 return ret; 1015 } 1016 } 1017 1018 static int rfc4106_decrypt(struct aead_request *req) 1019 { 1020 int ret; 1021 struct crypto_aead *tfm = crypto_aead_reqtfm(req); 1022 struct aesni_rfc4106_gcm_ctx *ctx = aesni_rfc4106_gcm_ctx_get(tfm); 1023 struct crypto_aead *cryptd_child = cryptd_aead_child(ctx->cryptd_tfm); 1024 1025 if (!irq_fpu_usable()) { 1026 struct aead_request *cryptd_req = 1027 (struct aead_request *) aead_request_ctx(req); 1028 memcpy(cryptd_req, req, sizeof(*req)); 1029 aead_request_set_tfm(cryptd_req, &ctx->cryptd_tfm->base); 1030 return crypto_aead_decrypt(cryptd_req); 1031 } else { 1032 kernel_fpu_begin(); 1033 ret = cryptd_child->base.crt_aead.decrypt(req); 1034 kernel_fpu_end(); 1035 return ret; 1036 } 1037 } 1038 1039 static struct crypto_alg rfc4106_alg = { 1040 .cra_name = "rfc4106(gcm(aes))", 1041 .cra_driver_name = "rfc4106-gcm-aesni", 1042 .cra_priority = 400, 1043 .cra_flags = CRYPTO_ALG_TYPE_AEAD | CRYPTO_ALG_ASYNC, 1044 .cra_blocksize = 1, 1045 .cra_ctxsize = sizeof(struct aesni_rfc4106_gcm_ctx) + AESNI_ALIGN, 1046 .cra_alignmask = 0, 1047 .cra_type = &crypto_nivaead_type, 1048 .cra_module = THIS_MODULE, 1049 .cra_list = LIST_HEAD_INIT(rfc4106_alg.cra_list), 1050 .cra_init = rfc4106_init, 1051 .cra_exit = rfc4106_exit, 1052 .cra_u = { 1053 .aead = { 1054 .setkey = rfc4106_set_key, 1055 .setauthsize = rfc4106_set_authsize, 1056 .encrypt = rfc4106_encrypt, 1057 .decrypt = rfc4106_decrypt, 1058 .geniv = "seqiv", 1059 .ivsize = 8, 1060 .maxauthsize = 16, 1061 }, 1062 }, 1063 }; 1064 1065 static int __driver_rfc4106_encrypt(struct aead_request *req) 1066 { 1067 u8 one_entry_in_sg = 0; 1068 u8 *src, *dst, *assoc; 1069 __be32 counter = cpu_to_be32(1); 1070 struct crypto_aead *tfm = crypto_aead_reqtfm(req); 1071 struct aesni_rfc4106_gcm_ctx *ctx = aesni_rfc4106_gcm_ctx_get(tfm); 1072 void *aes_ctx = &(ctx->aes_key_expanded); 1073 unsigned long auth_tag_len = crypto_aead_authsize(tfm); 1074 u8 iv_tab[16+AESNI_ALIGN]; 1075 u8* iv = (u8 *) PTR_ALIGN((u8 *)iv_tab, AESNI_ALIGN); 1076 struct scatter_walk src_sg_walk; 1077 struct scatter_walk assoc_sg_walk; 1078 struct scatter_walk dst_sg_walk; 1079 unsigned int i; 1080 1081 /* Assuming we are supporting rfc4106 64-bit extended */ 1082 /* sequence numbers We need to have the AAD length equal */ 1083 /* to 8 or 12 bytes */ 1084 if (unlikely(req->assoclen != 8 && req->assoclen != 12)) 1085 return -EINVAL; 1086 /* IV below built */ 1087 for (i = 0; i < 4; i++) 1088 *(iv+i) = ctx->nonce[i]; 1089 for (i = 0; i < 8; i++) 1090 *(iv+4+i) = req->iv[i]; 1091 *((__be32 *)(iv+12)) = counter; 1092 1093 if ((sg_is_last(req->src)) && (sg_is_last(req->assoc))) { 1094 one_entry_in_sg = 1; 1095 scatterwalk_start(&src_sg_walk, req->src); 1096 scatterwalk_start(&assoc_sg_walk, req->assoc); 1097 src = scatterwalk_map(&src_sg_walk, 0); 1098 assoc = scatterwalk_map(&assoc_sg_walk, 0); 1099 dst = src; 1100 if (unlikely(req->src != req->dst)) { 1101 scatterwalk_start(&dst_sg_walk, req->dst); 1102 dst = scatterwalk_map(&dst_sg_walk, 0); 1103 } 1104 1105 } else { 1106 /* Allocate memory for src, dst, assoc */ 1107 src = kmalloc(req->cryptlen + auth_tag_len + req->assoclen, 1108 GFP_ATOMIC); 1109 if (unlikely(!src)) 1110 return -ENOMEM; 1111 assoc = (src + req->cryptlen + auth_tag_len); 1112 scatterwalk_map_and_copy(src, req->src, 0, req->cryptlen, 0); 1113 scatterwalk_map_and_copy(assoc, req->assoc, 0, 1114 req->assoclen, 0); 1115 dst = src; 1116 } 1117 1118 aesni_gcm_enc(aes_ctx, dst, src, (unsigned long)req->cryptlen, iv, 1119 ctx->hash_subkey, assoc, (unsigned long)req->assoclen, dst 1120 + ((unsigned long)req->cryptlen), auth_tag_len); 1121 1122 /* The authTag (aka the Integrity Check Value) needs to be written 1123 * back to the packet. */ 1124 if (one_entry_in_sg) { 1125 if (unlikely(req->src != req->dst)) { 1126 scatterwalk_unmap(dst, 0); 1127 scatterwalk_done(&dst_sg_walk, 0, 0); 1128 } 1129 scatterwalk_unmap(src, 0); 1130 scatterwalk_unmap(assoc, 0); 1131 scatterwalk_done(&src_sg_walk, 0, 0); 1132 scatterwalk_done(&assoc_sg_walk, 0, 0); 1133 } else { 1134 scatterwalk_map_and_copy(dst, req->dst, 0, 1135 req->cryptlen + auth_tag_len, 1); 1136 kfree(src); 1137 } 1138 return 0; 1139 } 1140 1141 static int __driver_rfc4106_decrypt(struct aead_request *req) 1142 { 1143 u8 one_entry_in_sg = 0; 1144 u8 *src, *dst, *assoc; 1145 unsigned long tempCipherLen = 0; 1146 __be32 counter = cpu_to_be32(1); 1147 int retval = 0; 1148 struct crypto_aead *tfm = crypto_aead_reqtfm(req); 1149 struct aesni_rfc4106_gcm_ctx *ctx = aesni_rfc4106_gcm_ctx_get(tfm); 1150 void *aes_ctx = &(ctx->aes_key_expanded); 1151 unsigned long auth_tag_len = crypto_aead_authsize(tfm); 1152 u8 iv_and_authTag[32+AESNI_ALIGN]; 1153 u8 *iv = (u8 *) PTR_ALIGN((u8 *)iv_and_authTag, AESNI_ALIGN); 1154 u8 *authTag = iv + 16; 1155 struct scatter_walk src_sg_walk; 1156 struct scatter_walk assoc_sg_walk; 1157 struct scatter_walk dst_sg_walk; 1158 unsigned int i; 1159 1160 if (unlikely((req->cryptlen < auth_tag_len) || 1161 (req->assoclen != 8 && req->assoclen != 12))) 1162 return -EINVAL; 1163 /* Assuming we are supporting rfc4106 64-bit extended */ 1164 /* sequence numbers We need to have the AAD length */ 1165 /* equal to 8 or 12 bytes */ 1166 1167 tempCipherLen = (unsigned long)(req->cryptlen - auth_tag_len); 1168 /* IV below built */ 1169 for (i = 0; i < 4; i++) 1170 *(iv+i) = ctx->nonce[i]; 1171 for (i = 0; i < 8; i++) 1172 *(iv+4+i) = req->iv[i]; 1173 *((__be32 *)(iv+12)) = counter; 1174 1175 if ((sg_is_last(req->src)) && (sg_is_last(req->assoc))) { 1176 one_entry_in_sg = 1; 1177 scatterwalk_start(&src_sg_walk, req->src); 1178 scatterwalk_start(&assoc_sg_walk, req->assoc); 1179 src = scatterwalk_map(&src_sg_walk, 0); 1180 assoc = scatterwalk_map(&assoc_sg_walk, 0); 1181 dst = src; 1182 if (unlikely(req->src != req->dst)) { 1183 scatterwalk_start(&dst_sg_walk, req->dst); 1184 dst = scatterwalk_map(&dst_sg_walk, 0); 1185 } 1186 1187 } else { 1188 /* Allocate memory for src, dst, assoc */ 1189 src = kmalloc(req->cryptlen + req->assoclen, GFP_ATOMIC); 1190 if (!src) 1191 return -ENOMEM; 1192 assoc = (src + req->cryptlen + auth_tag_len); 1193 scatterwalk_map_and_copy(src, req->src, 0, req->cryptlen, 0); 1194 scatterwalk_map_and_copy(assoc, req->assoc, 0, 1195 req->assoclen, 0); 1196 dst = src; 1197 } 1198 1199 aesni_gcm_dec(aes_ctx, dst, src, tempCipherLen, iv, 1200 ctx->hash_subkey, assoc, (unsigned long)req->assoclen, 1201 authTag, auth_tag_len); 1202 1203 /* Compare generated tag with passed in tag. */ 1204 retval = memcmp(src + tempCipherLen, authTag, auth_tag_len) ? 1205 -EBADMSG : 0; 1206 1207 if (one_entry_in_sg) { 1208 if (unlikely(req->src != req->dst)) { 1209 scatterwalk_unmap(dst, 0); 1210 scatterwalk_done(&dst_sg_walk, 0, 0); 1211 } 1212 scatterwalk_unmap(src, 0); 1213 scatterwalk_unmap(assoc, 0); 1214 scatterwalk_done(&src_sg_walk, 0, 0); 1215 scatterwalk_done(&assoc_sg_walk, 0, 0); 1216 } else { 1217 scatterwalk_map_and_copy(dst, req->dst, 0, req->cryptlen, 1); 1218 kfree(src); 1219 } 1220 return retval; 1221 } 1222 1223 static struct crypto_alg __rfc4106_alg = { 1224 .cra_name = "__gcm-aes-aesni", 1225 .cra_driver_name = "__driver-gcm-aes-aesni", 1226 .cra_priority = 0, 1227 .cra_flags = CRYPTO_ALG_TYPE_AEAD, 1228 .cra_blocksize = 1, 1229 .cra_ctxsize = sizeof(struct aesni_rfc4106_gcm_ctx) + AESNI_ALIGN, 1230 .cra_alignmask = 0, 1231 .cra_type = &crypto_aead_type, 1232 .cra_module = THIS_MODULE, 1233 .cra_list = LIST_HEAD_INIT(__rfc4106_alg.cra_list), 1234 .cra_u = { 1235 .aead = { 1236 .encrypt = __driver_rfc4106_encrypt, 1237 .decrypt = __driver_rfc4106_decrypt, 1238 }, 1239 }, 1240 }; 1241 #endif 1242 1243 static int __init aesni_init(void) 1244 { 1245 int err; 1246 1247 if (!cpu_has_aes) { 1248 printk(KERN_INFO "Intel AES-NI instructions are not detected.\n"); 1249 return -ENODEV; 1250 } 1251 1252 if ((err = crypto_register_alg(&aesni_alg))) 1253 goto aes_err; 1254 if ((err = crypto_register_alg(&__aesni_alg))) 1255 goto __aes_err; 1256 if ((err = crypto_register_alg(&blk_ecb_alg))) 1257 goto blk_ecb_err; 1258 if ((err = crypto_register_alg(&blk_cbc_alg))) 1259 goto blk_cbc_err; 1260 if ((err = crypto_register_alg(&ablk_ecb_alg))) 1261 goto ablk_ecb_err; 1262 if ((err = crypto_register_alg(&ablk_cbc_alg))) 1263 goto ablk_cbc_err; 1264 #ifdef CONFIG_X86_64 1265 if ((err = crypto_register_alg(&blk_ctr_alg))) 1266 goto blk_ctr_err; 1267 if ((err = crypto_register_alg(&ablk_ctr_alg))) 1268 goto ablk_ctr_err; 1269 if ((err = crypto_register_alg(&__rfc4106_alg))) 1270 goto __aead_gcm_err; 1271 if ((err = crypto_register_alg(&rfc4106_alg))) 1272 goto aead_gcm_err; 1273 #ifdef HAS_CTR 1274 if ((err = crypto_register_alg(&ablk_rfc3686_ctr_alg))) 1275 goto ablk_rfc3686_ctr_err; 1276 #endif 1277 #endif 1278 #ifdef HAS_LRW 1279 if ((err = crypto_register_alg(&ablk_lrw_alg))) 1280 goto ablk_lrw_err; 1281 #endif 1282 #ifdef HAS_PCBC 1283 if ((err = crypto_register_alg(&ablk_pcbc_alg))) 1284 goto ablk_pcbc_err; 1285 #endif 1286 #ifdef HAS_XTS 1287 if ((err = crypto_register_alg(&ablk_xts_alg))) 1288 goto ablk_xts_err; 1289 #endif 1290 return err; 1291 1292 #ifdef HAS_XTS 1293 ablk_xts_err: 1294 #endif 1295 #ifdef HAS_PCBC 1296 crypto_unregister_alg(&ablk_pcbc_alg); 1297 ablk_pcbc_err: 1298 #endif 1299 #ifdef HAS_LRW 1300 crypto_unregister_alg(&ablk_lrw_alg); 1301 ablk_lrw_err: 1302 #endif 1303 #ifdef CONFIG_X86_64 1304 #ifdef HAS_CTR 1305 crypto_unregister_alg(&ablk_rfc3686_ctr_alg); 1306 ablk_rfc3686_ctr_err: 1307 #endif 1308 crypto_unregister_alg(&rfc4106_alg); 1309 aead_gcm_err: 1310 crypto_unregister_alg(&__rfc4106_alg); 1311 __aead_gcm_err: 1312 crypto_unregister_alg(&ablk_ctr_alg); 1313 ablk_ctr_err: 1314 crypto_unregister_alg(&blk_ctr_alg); 1315 blk_ctr_err: 1316 #endif 1317 crypto_unregister_alg(&ablk_cbc_alg); 1318 ablk_cbc_err: 1319 crypto_unregister_alg(&ablk_ecb_alg); 1320 ablk_ecb_err: 1321 crypto_unregister_alg(&blk_cbc_alg); 1322 blk_cbc_err: 1323 crypto_unregister_alg(&blk_ecb_alg); 1324 blk_ecb_err: 1325 crypto_unregister_alg(&__aesni_alg); 1326 __aes_err: 1327 crypto_unregister_alg(&aesni_alg); 1328 aes_err: 1329 return err; 1330 } 1331 1332 static void __exit aesni_exit(void) 1333 { 1334 #ifdef HAS_XTS 1335 crypto_unregister_alg(&ablk_xts_alg); 1336 #endif 1337 #ifdef HAS_PCBC 1338 crypto_unregister_alg(&ablk_pcbc_alg); 1339 #endif 1340 #ifdef HAS_LRW 1341 crypto_unregister_alg(&ablk_lrw_alg); 1342 #endif 1343 #ifdef CONFIG_X86_64 1344 #ifdef HAS_CTR 1345 crypto_unregister_alg(&ablk_rfc3686_ctr_alg); 1346 #endif 1347 crypto_unregister_alg(&rfc4106_alg); 1348 crypto_unregister_alg(&__rfc4106_alg); 1349 crypto_unregister_alg(&ablk_ctr_alg); 1350 crypto_unregister_alg(&blk_ctr_alg); 1351 #endif 1352 crypto_unregister_alg(&ablk_cbc_alg); 1353 crypto_unregister_alg(&ablk_ecb_alg); 1354 crypto_unregister_alg(&blk_cbc_alg); 1355 crypto_unregister_alg(&blk_ecb_alg); 1356 crypto_unregister_alg(&__aesni_alg); 1357 crypto_unregister_alg(&aesni_alg); 1358 } 1359 1360 module_init(aesni_init); 1361 module_exit(aesni_exit); 1362 1363 MODULE_DESCRIPTION("Rijndael (AES) Cipher Algorithm, Intel AES-NI instructions optimized"); 1364 MODULE_LICENSE("GPL"); 1365 MODULE_ALIAS("aes"); 1366