1 /* 2 * Support for Intel AES-NI instructions. This file contains glue 3 * code, the real AES implementation is in intel-aes_asm.S. 4 * 5 * Copyright (C) 2008, Intel Corp. 6 * Author: Huang Ying <ying.huang@intel.com> 7 * 8 * Added RFC4106 AES-GCM support for 128-bit keys under the AEAD 9 * interface for 64-bit kernels. 10 * Authors: Adrian Hoban <adrian.hoban@intel.com> 11 * Gabriele Paoloni <gabriele.paoloni@intel.com> 12 * Tadeusz Struk (tadeusz.struk@intel.com) 13 * Aidan O'Mahony (aidan.o.mahony@intel.com) 14 * Copyright (c) 2010, Intel Corporation. 15 * 16 * This program is free software; you can redistribute it and/or modify 17 * it under the terms of the GNU General Public License as published by 18 * the Free Software Foundation; either version 2 of the License, or 19 * (at your option) any later version. 20 */ 21 22 #include <linux/hardirq.h> 23 #include <linux/types.h> 24 #include <linux/crypto.h> 25 #include <linux/module.h> 26 #include <linux/err.h> 27 #include <crypto/algapi.h> 28 #include <crypto/aes.h> 29 #include <crypto/cryptd.h> 30 #include <crypto/ctr.h> 31 #include <asm/cpu_device_id.h> 32 #include <asm/i387.h> 33 #include <asm/aes.h> 34 #include <crypto/scatterwalk.h> 35 #include <crypto/internal/aead.h> 36 #include <linux/workqueue.h> 37 #include <linux/spinlock.h> 38 39 #if defined(CONFIG_CRYPTO_CTR) || defined(CONFIG_CRYPTO_CTR_MODULE) 40 #define HAS_CTR 41 #endif 42 43 #if defined(CONFIG_CRYPTO_LRW) || defined(CONFIG_CRYPTO_LRW_MODULE) 44 #define HAS_LRW 45 #endif 46 47 #if defined(CONFIG_CRYPTO_PCBC) || defined(CONFIG_CRYPTO_PCBC_MODULE) 48 #define HAS_PCBC 49 #endif 50 51 #if defined(CONFIG_CRYPTO_XTS) || defined(CONFIG_CRYPTO_XTS_MODULE) 52 #define HAS_XTS 53 #endif 54 55 struct async_aes_ctx { 56 struct cryptd_ablkcipher *cryptd_tfm; 57 }; 58 59 /* This data is stored at the end of the crypto_tfm struct. 60 * It's a type of per "session" data storage location. 61 * This needs to be 16 byte aligned. 62 */ 63 struct aesni_rfc4106_gcm_ctx { 64 u8 hash_subkey[16]; 65 struct crypto_aes_ctx aes_key_expanded; 66 u8 nonce[4]; 67 struct cryptd_aead *cryptd_tfm; 68 }; 69 70 struct aesni_gcm_set_hash_subkey_result { 71 int err; 72 struct completion completion; 73 }; 74 75 struct aesni_hash_subkey_req_data { 76 u8 iv[16]; 77 struct aesni_gcm_set_hash_subkey_result result; 78 struct scatterlist sg; 79 }; 80 81 #define AESNI_ALIGN (16) 82 #define AES_BLOCK_MASK (~(AES_BLOCK_SIZE-1)) 83 #define RFC4106_HASH_SUBKEY_SIZE 16 84 85 asmlinkage int aesni_set_key(struct crypto_aes_ctx *ctx, const u8 *in_key, 86 unsigned int key_len); 87 asmlinkage void aesni_enc(struct crypto_aes_ctx *ctx, u8 *out, 88 const u8 *in); 89 asmlinkage void aesni_dec(struct crypto_aes_ctx *ctx, u8 *out, 90 const u8 *in); 91 asmlinkage void aesni_ecb_enc(struct crypto_aes_ctx *ctx, u8 *out, 92 const u8 *in, unsigned int len); 93 asmlinkage void aesni_ecb_dec(struct crypto_aes_ctx *ctx, u8 *out, 94 const u8 *in, unsigned int len); 95 asmlinkage void aesni_cbc_enc(struct crypto_aes_ctx *ctx, u8 *out, 96 const u8 *in, unsigned int len, u8 *iv); 97 asmlinkage void aesni_cbc_dec(struct crypto_aes_ctx *ctx, u8 *out, 98 const u8 *in, unsigned int len, u8 *iv); 99 100 int crypto_fpu_init(void); 101 void crypto_fpu_exit(void); 102 103 #ifdef CONFIG_X86_64 104 asmlinkage void aesni_ctr_enc(struct crypto_aes_ctx *ctx, u8 *out, 105 const u8 *in, unsigned int len, u8 *iv); 106 107 /* asmlinkage void aesni_gcm_enc() 108 * void *ctx, AES Key schedule. Starts on a 16 byte boundary. 109 * u8 *out, Ciphertext output. Encrypt in-place is allowed. 110 * const u8 *in, Plaintext input 111 * unsigned long plaintext_len, Length of data in bytes for encryption. 112 * u8 *iv, Pre-counter block j0: 4 byte salt (from Security Association) 113 * concatenated with 8 byte Initialisation Vector (from IPSec ESP 114 * Payload) concatenated with 0x00000001. 16-byte aligned pointer. 115 * u8 *hash_subkey, the Hash sub key input. Data starts on a 16-byte boundary. 116 * const u8 *aad, Additional Authentication Data (AAD) 117 * unsigned long aad_len, Length of AAD in bytes. With RFC4106 this 118 * is going to be 8 or 12 bytes 119 * u8 *auth_tag, Authenticated Tag output. 120 * unsigned long auth_tag_len), Authenticated Tag Length in bytes. 121 * Valid values are 16 (most likely), 12 or 8. 122 */ 123 asmlinkage void aesni_gcm_enc(void *ctx, u8 *out, 124 const u8 *in, unsigned long plaintext_len, u8 *iv, 125 u8 *hash_subkey, const u8 *aad, unsigned long aad_len, 126 u8 *auth_tag, unsigned long auth_tag_len); 127 128 /* asmlinkage void aesni_gcm_dec() 129 * void *ctx, AES Key schedule. Starts on a 16 byte boundary. 130 * u8 *out, Plaintext output. Decrypt in-place is allowed. 131 * const u8 *in, Ciphertext input 132 * unsigned long ciphertext_len, Length of data in bytes for decryption. 133 * u8 *iv, Pre-counter block j0: 4 byte salt (from Security Association) 134 * concatenated with 8 byte Initialisation Vector (from IPSec ESP 135 * Payload) concatenated with 0x00000001. 16-byte aligned pointer. 136 * u8 *hash_subkey, the Hash sub key input. Data starts on a 16-byte boundary. 137 * const u8 *aad, Additional Authentication Data (AAD) 138 * unsigned long aad_len, Length of AAD in bytes. With RFC4106 this is going 139 * to be 8 or 12 bytes 140 * u8 *auth_tag, Authenticated Tag output. 141 * unsigned long auth_tag_len) Authenticated Tag Length in bytes. 142 * Valid values are 16 (most likely), 12 or 8. 143 */ 144 asmlinkage void aesni_gcm_dec(void *ctx, u8 *out, 145 const u8 *in, unsigned long ciphertext_len, u8 *iv, 146 u8 *hash_subkey, const u8 *aad, unsigned long aad_len, 147 u8 *auth_tag, unsigned long auth_tag_len); 148 149 static inline struct 150 aesni_rfc4106_gcm_ctx *aesni_rfc4106_gcm_ctx_get(struct crypto_aead *tfm) 151 { 152 return 153 (struct aesni_rfc4106_gcm_ctx *) 154 PTR_ALIGN((u8 *) 155 crypto_tfm_ctx(crypto_aead_tfm(tfm)), AESNI_ALIGN); 156 } 157 #endif 158 159 static inline struct crypto_aes_ctx *aes_ctx(void *raw_ctx) 160 { 161 unsigned long addr = (unsigned long)raw_ctx; 162 unsigned long align = AESNI_ALIGN; 163 164 if (align <= crypto_tfm_ctx_alignment()) 165 align = 1; 166 return (struct crypto_aes_ctx *)ALIGN(addr, align); 167 } 168 169 static int aes_set_key_common(struct crypto_tfm *tfm, void *raw_ctx, 170 const u8 *in_key, unsigned int key_len) 171 { 172 struct crypto_aes_ctx *ctx = aes_ctx(raw_ctx); 173 u32 *flags = &tfm->crt_flags; 174 int err; 175 176 if (key_len != AES_KEYSIZE_128 && key_len != AES_KEYSIZE_192 && 177 key_len != AES_KEYSIZE_256) { 178 *flags |= CRYPTO_TFM_RES_BAD_KEY_LEN; 179 return -EINVAL; 180 } 181 182 if (!irq_fpu_usable()) 183 err = crypto_aes_expand_key(ctx, in_key, key_len); 184 else { 185 kernel_fpu_begin(); 186 err = aesni_set_key(ctx, in_key, key_len); 187 kernel_fpu_end(); 188 } 189 190 return err; 191 } 192 193 static int aes_set_key(struct crypto_tfm *tfm, const u8 *in_key, 194 unsigned int key_len) 195 { 196 return aes_set_key_common(tfm, crypto_tfm_ctx(tfm), in_key, key_len); 197 } 198 199 static void aes_encrypt(struct crypto_tfm *tfm, u8 *dst, const u8 *src) 200 { 201 struct crypto_aes_ctx *ctx = aes_ctx(crypto_tfm_ctx(tfm)); 202 203 if (!irq_fpu_usable()) 204 crypto_aes_encrypt_x86(ctx, dst, src); 205 else { 206 kernel_fpu_begin(); 207 aesni_enc(ctx, dst, src); 208 kernel_fpu_end(); 209 } 210 } 211 212 static void aes_decrypt(struct crypto_tfm *tfm, u8 *dst, const u8 *src) 213 { 214 struct crypto_aes_ctx *ctx = aes_ctx(crypto_tfm_ctx(tfm)); 215 216 if (!irq_fpu_usable()) 217 crypto_aes_decrypt_x86(ctx, dst, src); 218 else { 219 kernel_fpu_begin(); 220 aesni_dec(ctx, dst, src); 221 kernel_fpu_end(); 222 } 223 } 224 225 static struct crypto_alg aesni_alg = { 226 .cra_name = "aes", 227 .cra_driver_name = "aes-aesni", 228 .cra_priority = 300, 229 .cra_flags = CRYPTO_ALG_TYPE_CIPHER, 230 .cra_blocksize = AES_BLOCK_SIZE, 231 .cra_ctxsize = sizeof(struct crypto_aes_ctx)+AESNI_ALIGN-1, 232 .cra_alignmask = 0, 233 .cra_module = THIS_MODULE, 234 .cra_list = LIST_HEAD_INIT(aesni_alg.cra_list), 235 .cra_u = { 236 .cipher = { 237 .cia_min_keysize = AES_MIN_KEY_SIZE, 238 .cia_max_keysize = AES_MAX_KEY_SIZE, 239 .cia_setkey = aes_set_key, 240 .cia_encrypt = aes_encrypt, 241 .cia_decrypt = aes_decrypt 242 } 243 } 244 }; 245 246 static void __aes_encrypt(struct crypto_tfm *tfm, u8 *dst, const u8 *src) 247 { 248 struct crypto_aes_ctx *ctx = aes_ctx(crypto_tfm_ctx(tfm)); 249 250 aesni_enc(ctx, dst, src); 251 } 252 253 static void __aes_decrypt(struct crypto_tfm *tfm, u8 *dst, const u8 *src) 254 { 255 struct crypto_aes_ctx *ctx = aes_ctx(crypto_tfm_ctx(tfm)); 256 257 aesni_dec(ctx, dst, src); 258 } 259 260 static struct crypto_alg __aesni_alg = { 261 .cra_name = "__aes-aesni", 262 .cra_driver_name = "__driver-aes-aesni", 263 .cra_priority = 0, 264 .cra_flags = CRYPTO_ALG_TYPE_CIPHER, 265 .cra_blocksize = AES_BLOCK_SIZE, 266 .cra_ctxsize = sizeof(struct crypto_aes_ctx)+AESNI_ALIGN-1, 267 .cra_alignmask = 0, 268 .cra_module = THIS_MODULE, 269 .cra_list = LIST_HEAD_INIT(__aesni_alg.cra_list), 270 .cra_u = { 271 .cipher = { 272 .cia_min_keysize = AES_MIN_KEY_SIZE, 273 .cia_max_keysize = AES_MAX_KEY_SIZE, 274 .cia_setkey = aes_set_key, 275 .cia_encrypt = __aes_encrypt, 276 .cia_decrypt = __aes_decrypt 277 } 278 } 279 }; 280 281 static int ecb_encrypt(struct blkcipher_desc *desc, 282 struct scatterlist *dst, struct scatterlist *src, 283 unsigned int nbytes) 284 { 285 struct crypto_aes_ctx *ctx = aes_ctx(crypto_blkcipher_ctx(desc->tfm)); 286 struct blkcipher_walk walk; 287 int err; 288 289 blkcipher_walk_init(&walk, dst, src, nbytes); 290 err = blkcipher_walk_virt(desc, &walk); 291 desc->flags &= ~CRYPTO_TFM_REQ_MAY_SLEEP; 292 293 kernel_fpu_begin(); 294 while ((nbytes = walk.nbytes)) { 295 aesni_ecb_enc(ctx, walk.dst.virt.addr, walk.src.virt.addr, 296 nbytes & AES_BLOCK_MASK); 297 nbytes &= AES_BLOCK_SIZE - 1; 298 err = blkcipher_walk_done(desc, &walk, nbytes); 299 } 300 kernel_fpu_end(); 301 302 return err; 303 } 304 305 static int ecb_decrypt(struct blkcipher_desc *desc, 306 struct scatterlist *dst, struct scatterlist *src, 307 unsigned int nbytes) 308 { 309 struct crypto_aes_ctx *ctx = aes_ctx(crypto_blkcipher_ctx(desc->tfm)); 310 struct blkcipher_walk walk; 311 int err; 312 313 blkcipher_walk_init(&walk, dst, src, nbytes); 314 err = blkcipher_walk_virt(desc, &walk); 315 desc->flags &= ~CRYPTO_TFM_REQ_MAY_SLEEP; 316 317 kernel_fpu_begin(); 318 while ((nbytes = walk.nbytes)) { 319 aesni_ecb_dec(ctx, walk.dst.virt.addr, walk.src.virt.addr, 320 nbytes & AES_BLOCK_MASK); 321 nbytes &= AES_BLOCK_SIZE - 1; 322 err = blkcipher_walk_done(desc, &walk, nbytes); 323 } 324 kernel_fpu_end(); 325 326 return err; 327 } 328 329 static struct crypto_alg blk_ecb_alg = { 330 .cra_name = "__ecb-aes-aesni", 331 .cra_driver_name = "__driver-ecb-aes-aesni", 332 .cra_priority = 0, 333 .cra_flags = CRYPTO_ALG_TYPE_BLKCIPHER, 334 .cra_blocksize = AES_BLOCK_SIZE, 335 .cra_ctxsize = sizeof(struct crypto_aes_ctx)+AESNI_ALIGN-1, 336 .cra_alignmask = 0, 337 .cra_type = &crypto_blkcipher_type, 338 .cra_module = THIS_MODULE, 339 .cra_list = LIST_HEAD_INIT(blk_ecb_alg.cra_list), 340 .cra_u = { 341 .blkcipher = { 342 .min_keysize = AES_MIN_KEY_SIZE, 343 .max_keysize = AES_MAX_KEY_SIZE, 344 .setkey = aes_set_key, 345 .encrypt = ecb_encrypt, 346 .decrypt = ecb_decrypt, 347 }, 348 }, 349 }; 350 351 static int cbc_encrypt(struct blkcipher_desc *desc, 352 struct scatterlist *dst, struct scatterlist *src, 353 unsigned int nbytes) 354 { 355 struct crypto_aes_ctx *ctx = aes_ctx(crypto_blkcipher_ctx(desc->tfm)); 356 struct blkcipher_walk walk; 357 int err; 358 359 blkcipher_walk_init(&walk, dst, src, nbytes); 360 err = blkcipher_walk_virt(desc, &walk); 361 desc->flags &= ~CRYPTO_TFM_REQ_MAY_SLEEP; 362 363 kernel_fpu_begin(); 364 while ((nbytes = walk.nbytes)) { 365 aesni_cbc_enc(ctx, walk.dst.virt.addr, walk.src.virt.addr, 366 nbytes & AES_BLOCK_MASK, walk.iv); 367 nbytes &= AES_BLOCK_SIZE - 1; 368 err = blkcipher_walk_done(desc, &walk, nbytes); 369 } 370 kernel_fpu_end(); 371 372 return err; 373 } 374 375 static int cbc_decrypt(struct blkcipher_desc *desc, 376 struct scatterlist *dst, struct scatterlist *src, 377 unsigned int nbytes) 378 { 379 struct crypto_aes_ctx *ctx = aes_ctx(crypto_blkcipher_ctx(desc->tfm)); 380 struct blkcipher_walk walk; 381 int err; 382 383 blkcipher_walk_init(&walk, dst, src, nbytes); 384 err = blkcipher_walk_virt(desc, &walk); 385 desc->flags &= ~CRYPTO_TFM_REQ_MAY_SLEEP; 386 387 kernel_fpu_begin(); 388 while ((nbytes = walk.nbytes)) { 389 aesni_cbc_dec(ctx, walk.dst.virt.addr, walk.src.virt.addr, 390 nbytes & AES_BLOCK_MASK, walk.iv); 391 nbytes &= AES_BLOCK_SIZE - 1; 392 err = blkcipher_walk_done(desc, &walk, nbytes); 393 } 394 kernel_fpu_end(); 395 396 return err; 397 } 398 399 static struct crypto_alg blk_cbc_alg = { 400 .cra_name = "__cbc-aes-aesni", 401 .cra_driver_name = "__driver-cbc-aes-aesni", 402 .cra_priority = 0, 403 .cra_flags = CRYPTO_ALG_TYPE_BLKCIPHER, 404 .cra_blocksize = AES_BLOCK_SIZE, 405 .cra_ctxsize = sizeof(struct crypto_aes_ctx)+AESNI_ALIGN-1, 406 .cra_alignmask = 0, 407 .cra_type = &crypto_blkcipher_type, 408 .cra_module = THIS_MODULE, 409 .cra_list = LIST_HEAD_INIT(blk_cbc_alg.cra_list), 410 .cra_u = { 411 .blkcipher = { 412 .min_keysize = AES_MIN_KEY_SIZE, 413 .max_keysize = AES_MAX_KEY_SIZE, 414 .setkey = aes_set_key, 415 .encrypt = cbc_encrypt, 416 .decrypt = cbc_decrypt, 417 }, 418 }, 419 }; 420 421 #ifdef CONFIG_X86_64 422 static void ctr_crypt_final(struct crypto_aes_ctx *ctx, 423 struct blkcipher_walk *walk) 424 { 425 u8 *ctrblk = walk->iv; 426 u8 keystream[AES_BLOCK_SIZE]; 427 u8 *src = walk->src.virt.addr; 428 u8 *dst = walk->dst.virt.addr; 429 unsigned int nbytes = walk->nbytes; 430 431 aesni_enc(ctx, keystream, ctrblk); 432 crypto_xor(keystream, src, nbytes); 433 memcpy(dst, keystream, nbytes); 434 crypto_inc(ctrblk, AES_BLOCK_SIZE); 435 } 436 437 static int ctr_crypt(struct blkcipher_desc *desc, 438 struct scatterlist *dst, struct scatterlist *src, 439 unsigned int nbytes) 440 { 441 struct crypto_aes_ctx *ctx = aes_ctx(crypto_blkcipher_ctx(desc->tfm)); 442 struct blkcipher_walk walk; 443 int err; 444 445 blkcipher_walk_init(&walk, dst, src, nbytes); 446 err = blkcipher_walk_virt_block(desc, &walk, AES_BLOCK_SIZE); 447 desc->flags &= ~CRYPTO_TFM_REQ_MAY_SLEEP; 448 449 kernel_fpu_begin(); 450 while ((nbytes = walk.nbytes) >= AES_BLOCK_SIZE) { 451 aesni_ctr_enc(ctx, walk.dst.virt.addr, walk.src.virt.addr, 452 nbytes & AES_BLOCK_MASK, walk.iv); 453 nbytes &= AES_BLOCK_SIZE - 1; 454 err = blkcipher_walk_done(desc, &walk, nbytes); 455 } 456 if (walk.nbytes) { 457 ctr_crypt_final(ctx, &walk); 458 err = blkcipher_walk_done(desc, &walk, 0); 459 } 460 kernel_fpu_end(); 461 462 return err; 463 } 464 465 static struct crypto_alg blk_ctr_alg = { 466 .cra_name = "__ctr-aes-aesni", 467 .cra_driver_name = "__driver-ctr-aes-aesni", 468 .cra_priority = 0, 469 .cra_flags = CRYPTO_ALG_TYPE_BLKCIPHER, 470 .cra_blocksize = 1, 471 .cra_ctxsize = sizeof(struct crypto_aes_ctx)+AESNI_ALIGN-1, 472 .cra_alignmask = 0, 473 .cra_type = &crypto_blkcipher_type, 474 .cra_module = THIS_MODULE, 475 .cra_list = LIST_HEAD_INIT(blk_ctr_alg.cra_list), 476 .cra_u = { 477 .blkcipher = { 478 .min_keysize = AES_MIN_KEY_SIZE, 479 .max_keysize = AES_MAX_KEY_SIZE, 480 .ivsize = AES_BLOCK_SIZE, 481 .setkey = aes_set_key, 482 .encrypt = ctr_crypt, 483 .decrypt = ctr_crypt, 484 }, 485 }, 486 }; 487 #endif 488 489 static int ablk_set_key(struct crypto_ablkcipher *tfm, const u8 *key, 490 unsigned int key_len) 491 { 492 struct async_aes_ctx *ctx = crypto_ablkcipher_ctx(tfm); 493 struct crypto_ablkcipher *child = &ctx->cryptd_tfm->base; 494 int err; 495 496 crypto_ablkcipher_clear_flags(child, CRYPTO_TFM_REQ_MASK); 497 crypto_ablkcipher_set_flags(child, crypto_ablkcipher_get_flags(tfm) 498 & CRYPTO_TFM_REQ_MASK); 499 err = crypto_ablkcipher_setkey(child, key, key_len); 500 crypto_ablkcipher_set_flags(tfm, crypto_ablkcipher_get_flags(child) 501 & CRYPTO_TFM_RES_MASK); 502 return err; 503 } 504 505 static int ablk_encrypt(struct ablkcipher_request *req) 506 { 507 struct crypto_ablkcipher *tfm = crypto_ablkcipher_reqtfm(req); 508 struct async_aes_ctx *ctx = crypto_ablkcipher_ctx(tfm); 509 510 if (!irq_fpu_usable()) { 511 struct ablkcipher_request *cryptd_req = 512 ablkcipher_request_ctx(req); 513 memcpy(cryptd_req, req, sizeof(*req)); 514 ablkcipher_request_set_tfm(cryptd_req, &ctx->cryptd_tfm->base); 515 return crypto_ablkcipher_encrypt(cryptd_req); 516 } else { 517 struct blkcipher_desc desc; 518 desc.tfm = cryptd_ablkcipher_child(ctx->cryptd_tfm); 519 desc.info = req->info; 520 desc.flags = 0; 521 return crypto_blkcipher_crt(desc.tfm)->encrypt( 522 &desc, req->dst, req->src, req->nbytes); 523 } 524 } 525 526 static int ablk_decrypt(struct ablkcipher_request *req) 527 { 528 struct crypto_ablkcipher *tfm = crypto_ablkcipher_reqtfm(req); 529 struct async_aes_ctx *ctx = crypto_ablkcipher_ctx(tfm); 530 531 if (!irq_fpu_usable()) { 532 struct ablkcipher_request *cryptd_req = 533 ablkcipher_request_ctx(req); 534 memcpy(cryptd_req, req, sizeof(*req)); 535 ablkcipher_request_set_tfm(cryptd_req, &ctx->cryptd_tfm->base); 536 return crypto_ablkcipher_decrypt(cryptd_req); 537 } else { 538 struct blkcipher_desc desc; 539 desc.tfm = cryptd_ablkcipher_child(ctx->cryptd_tfm); 540 desc.info = req->info; 541 desc.flags = 0; 542 return crypto_blkcipher_crt(desc.tfm)->decrypt( 543 &desc, req->dst, req->src, req->nbytes); 544 } 545 } 546 547 static void ablk_exit(struct crypto_tfm *tfm) 548 { 549 struct async_aes_ctx *ctx = crypto_tfm_ctx(tfm); 550 551 cryptd_free_ablkcipher(ctx->cryptd_tfm); 552 } 553 554 static void ablk_init_common(struct crypto_tfm *tfm, 555 struct cryptd_ablkcipher *cryptd_tfm) 556 { 557 struct async_aes_ctx *ctx = crypto_tfm_ctx(tfm); 558 559 ctx->cryptd_tfm = cryptd_tfm; 560 tfm->crt_ablkcipher.reqsize = sizeof(struct ablkcipher_request) + 561 crypto_ablkcipher_reqsize(&cryptd_tfm->base); 562 } 563 564 static int ablk_ecb_init(struct crypto_tfm *tfm) 565 { 566 struct cryptd_ablkcipher *cryptd_tfm; 567 568 cryptd_tfm = cryptd_alloc_ablkcipher("__driver-ecb-aes-aesni", 0, 0); 569 if (IS_ERR(cryptd_tfm)) 570 return PTR_ERR(cryptd_tfm); 571 ablk_init_common(tfm, cryptd_tfm); 572 return 0; 573 } 574 575 static struct crypto_alg ablk_ecb_alg = { 576 .cra_name = "ecb(aes)", 577 .cra_driver_name = "ecb-aes-aesni", 578 .cra_priority = 400, 579 .cra_flags = CRYPTO_ALG_TYPE_ABLKCIPHER|CRYPTO_ALG_ASYNC, 580 .cra_blocksize = AES_BLOCK_SIZE, 581 .cra_ctxsize = sizeof(struct async_aes_ctx), 582 .cra_alignmask = 0, 583 .cra_type = &crypto_ablkcipher_type, 584 .cra_module = THIS_MODULE, 585 .cra_list = LIST_HEAD_INIT(ablk_ecb_alg.cra_list), 586 .cra_init = ablk_ecb_init, 587 .cra_exit = ablk_exit, 588 .cra_u = { 589 .ablkcipher = { 590 .min_keysize = AES_MIN_KEY_SIZE, 591 .max_keysize = AES_MAX_KEY_SIZE, 592 .setkey = ablk_set_key, 593 .encrypt = ablk_encrypt, 594 .decrypt = ablk_decrypt, 595 }, 596 }, 597 }; 598 599 static int ablk_cbc_init(struct crypto_tfm *tfm) 600 { 601 struct cryptd_ablkcipher *cryptd_tfm; 602 603 cryptd_tfm = cryptd_alloc_ablkcipher("__driver-cbc-aes-aesni", 0, 0); 604 if (IS_ERR(cryptd_tfm)) 605 return PTR_ERR(cryptd_tfm); 606 ablk_init_common(tfm, cryptd_tfm); 607 return 0; 608 } 609 610 static struct crypto_alg ablk_cbc_alg = { 611 .cra_name = "cbc(aes)", 612 .cra_driver_name = "cbc-aes-aesni", 613 .cra_priority = 400, 614 .cra_flags = CRYPTO_ALG_TYPE_ABLKCIPHER|CRYPTO_ALG_ASYNC, 615 .cra_blocksize = AES_BLOCK_SIZE, 616 .cra_ctxsize = sizeof(struct async_aes_ctx), 617 .cra_alignmask = 0, 618 .cra_type = &crypto_ablkcipher_type, 619 .cra_module = THIS_MODULE, 620 .cra_list = LIST_HEAD_INIT(ablk_cbc_alg.cra_list), 621 .cra_init = ablk_cbc_init, 622 .cra_exit = ablk_exit, 623 .cra_u = { 624 .ablkcipher = { 625 .min_keysize = AES_MIN_KEY_SIZE, 626 .max_keysize = AES_MAX_KEY_SIZE, 627 .ivsize = AES_BLOCK_SIZE, 628 .setkey = ablk_set_key, 629 .encrypt = ablk_encrypt, 630 .decrypt = ablk_decrypt, 631 }, 632 }, 633 }; 634 635 #ifdef CONFIG_X86_64 636 static int ablk_ctr_init(struct crypto_tfm *tfm) 637 { 638 struct cryptd_ablkcipher *cryptd_tfm; 639 640 cryptd_tfm = cryptd_alloc_ablkcipher("__driver-ctr-aes-aesni", 0, 0); 641 if (IS_ERR(cryptd_tfm)) 642 return PTR_ERR(cryptd_tfm); 643 ablk_init_common(tfm, cryptd_tfm); 644 return 0; 645 } 646 647 static struct crypto_alg ablk_ctr_alg = { 648 .cra_name = "ctr(aes)", 649 .cra_driver_name = "ctr-aes-aesni", 650 .cra_priority = 400, 651 .cra_flags = CRYPTO_ALG_TYPE_ABLKCIPHER|CRYPTO_ALG_ASYNC, 652 .cra_blocksize = 1, 653 .cra_ctxsize = sizeof(struct async_aes_ctx), 654 .cra_alignmask = 0, 655 .cra_type = &crypto_ablkcipher_type, 656 .cra_module = THIS_MODULE, 657 .cra_list = LIST_HEAD_INIT(ablk_ctr_alg.cra_list), 658 .cra_init = ablk_ctr_init, 659 .cra_exit = ablk_exit, 660 .cra_u = { 661 .ablkcipher = { 662 .min_keysize = AES_MIN_KEY_SIZE, 663 .max_keysize = AES_MAX_KEY_SIZE, 664 .ivsize = AES_BLOCK_SIZE, 665 .setkey = ablk_set_key, 666 .encrypt = ablk_encrypt, 667 .decrypt = ablk_encrypt, 668 .geniv = "chainiv", 669 }, 670 }, 671 }; 672 673 #ifdef HAS_CTR 674 static int ablk_rfc3686_ctr_init(struct crypto_tfm *tfm) 675 { 676 struct cryptd_ablkcipher *cryptd_tfm; 677 678 cryptd_tfm = cryptd_alloc_ablkcipher( 679 "rfc3686(__driver-ctr-aes-aesni)", 0, 0); 680 if (IS_ERR(cryptd_tfm)) 681 return PTR_ERR(cryptd_tfm); 682 ablk_init_common(tfm, cryptd_tfm); 683 return 0; 684 } 685 686 static struct crypto_alg ablk_rfc3686_ctr_alg = { 687 .cra_name = "rfc3686(ctr(aes))", 688 .cra_driver_name = "rfc3686-ctr-aes-aesni", 689 .cra_priority = 400, 690 .cra_flags = CRYPTO_ALG_TYPE_ABLKCIPHER|CRYPTO_ALG_ASYNC, 691 .cra_blocksize = 1, 692 .cra_ctxsize = sizeof(struct async_aes_ctx), 693 .cra_alignmask = 0, 694 .cra_type = &crypto_ablkcipher_type, 695 .cra_module = THIS_MODULE, 696 .cra_list = LIST_HEAD_INIT(ablk_rfc3686_ctr_alg.cra_list), 697 .cra_init = ablk_rfc3686_ctr_init, 698 .cra_exit = ablk_exit, 699 .cra_u = { 700 .ablkcipher = { 701 .min_keysize = AES_MIN_KEY_SIZE+CTR_RFC3686_NONCE_SIZE, 702 .max_keysize = AES_MAX_KEY_SIZE+CTR_RFC3686_NONCE_SIZE, 703 .ivsize = CTR_RFC3686_IV_SIZE, 704 .setkey = ablk_set_key, 705 .encrypt = ablk_encrypt, 706 .decrypt = ablk_decrypt, 707 .geniv = "seqiv", 708 }, 709 }, 710 }; 711 #endif 712 #endif 713 714 #ifdef HAS_LRW 715 static int ablk_lrw_init(struct crypto_tfm *tfm) 716 { 717 struct cryptd_ablkcipher *cryptd_tfm; 718 719 cryptd_tfm = cryptd_alloc_ablkcipher("fpu(lrw(__driver-aes-aesni))", 720 0, 0); 721 if (IS_ERR(cryptd_tfm)) 722 return PTR_ERR(cryptd_tfm); 723 ablk_init_common(tfm, cryptd_tfm); 724 return 0; 725 } 726 727 static struct crypto_alg ablk_lrw_alg = { 728 .cra_name = "lrw(aes)", 729 .cra_driver_name = "lrw-aes-aesni", 730 .cra_priority = 400, 731 .cra_flags = CRYPTO_ALG_TYPE_ABLKCIPHER|CRYPTO_ALG_ASYNC, 732 .cra_blocksize = AES_BLOCK_SIZE, 733 .cra_ctxsize = sizeof(struct async_aes_ctx), 734 .cra_alignmask = 0, 735 .cra_type = &crypto_ablkcipher_type, 736 .cra_module = THIS_MODULE, 737 .cra_list = LIST_HEAD_INIT(ablk_lrw_alg.cra_list), 738 .cra_init = ablk_lrw_init, 739 .cra_exit = ablk_exit, 740 .cra_u = { 741 .ablkcipher = { 742 .min_keysize = AES_MIN_KEY_SIZE + AES_BLOCK_SIZE, 743 .max_keysize = AES_MAX_KEY_SIZE + AES_BLOCK_SIZE, 744 .ivsize = AES_BLOCK_SIZE, 745 .setkey = ablk_set_key, 746 .encrypt = ablk_encrypt, 747 .decrypt = ablk_decrypt, 748 }, 749 }, 750 }; 751 #endif 752 753 #ifdef HAS_PCBC 754 static int ablk_pcbc_init(struct crypto_tfm *tfm) 755 { 756 struct cryptd_ablkcipher *cryptd_tfm; 757 758 cryptd_tfm = cryptd_alloc_ablkcipher("fpu(pcbc(__driver-aes-aesni))", 759 0, 0); 760 if (IS_ERR(cryptd_tfm)) 761 return PTR_ERR(cryptd_tfm); 762 ablk_init_common(tfm, cryptd_tfm); 763 return 0; 764 } 765 766 static struct crypto_alg ablk_pcbc_alg = { 767 .cra_name = "pcbc(aes)", 768 .cra_driver_name = "pcbc-aes-aesni", 769 .cra_priority = 400, 770 .cra_flags = CRYPTO_ALG_TYPE_ABLKCIPHER|CRYPTO_ALG_ASYNC, 771 .cra_blocksize = AES_BLOCK_SIZE, 772 .cra_ctxsize = sizeof(struct async_aes_ctx), 773 .cra_alignmask = 0, 774 .cra_type = &crypto_ablkcipher_type, 775 .cra_module = THIS_MODULE, 776 .cra_list = LIST_HEAD_INIT(ablk_pcbc_alg.cra_list), 777 .cra_init = ablk_pcbc_init, 778 .cra_exit = ablk_exit, 779 .cra_u = { 780 .ablkcipher = { 781 .min_keysize = AES_MIN_KEY_SIZE, 782 .max_keysize = AES_MAX_KEY_SIZE, 783 .ivsize = AES_BLOCK_SIZE, 784 .setkey = ablk_set_key, 785 .encrypt = ablk_encrypt, 786 .decrypt = ablk_decrypt, 787 }, 788 }, 789 }; 790 #endif 791 792 #ifdef HAS_XTS 793 static int ablk_xts_init(struct crypto_tfm *tfm) 794 { 795 struct cryptd_ablkcipher *cryptd_tfm; 796 797 cryptd_tfm = cryptd_alloc_ablkcipher("fpu(xts(__driver-aes-aesni))", 798 0, 0); 799 if (IS_ERR(cryptd_tfm)) 800 return PTR_ERR(cryptd_tfm); 801 ablk_init_common(tfm, cryptd_tfm); 802 return 0; 803 } 804 805 static struct crypto_alg ablk_xts_alg = { 806 .cra_name = "xts(aes)", 807 .cra_driver_name = "xts-aes-aesni", 808 .cra_priority = 400, 809 .cra_flags = CRYPTO_ALG_TYPE_ABLKCIPHER|CRYPTO_ALG_ASYNC, 810 .cra_blocksize = AES_BLOCK_SIZE, 811 .cra_ctxsize = sizeof(struct async_aes_ctx), 812 .cra_alignmask = 0, 813 .cra_type = &crypto_ablkcipher_type, 814 .cra_module = THIS_MODULE, 815 .cra_list = LIST_HEAD_INIT(ablk_xts_alg.cra_list), 816 .cra_init = ablk_xts_init, 817 .cra_exit = ablk_exit, 818 .cra_u = { 819 .ablkcipher = { 820 .min_keysize = 2 * AES_MIN_KEY_SIZE, 821 .max_keysize = 2 * AES_MAX_KEY_SIZE, 822 .ivsize = AES_BLOCK_SIZE, 823 .setkey = ablk_set_key, 824 .encrypt = ablk_encrypt, 825 .decrypt = ablk_decrypt, 826 }, 827 }, 828 }; 829 #endif 830 831 #ifdef CONFIG_X86_64 832 static int rfc4106_init(struct crypto_tfm *tfm) 833 { 834 struct cryptd_aead *cryptd_tfm; 835 struct aesni_rfc4106_gcm_ctx *ctx = (struct aesni_rfc4106_gcm_ctx *) 836 PTR_ALIGN((u8 *)crypto_tfm_ctx(tfm), AESNI_ALIGN); 837 struct crypto_aead *cryptd_child; 838 struct aesni_rfc4106_gcm_ctx *child_ctx; 839 cryptd_tfm = cryptd_alloc_aead("__driver-gcm-aes-aesni", 0, 0); 840 if (IS_ERR(cryptd_tfm)) 841 return PTR_ERR(cryptd_tfm); 842 843 cryptd_child = cryptd_aead_child(cryptd_tfm); 844 child_ctx = aesni_rfc4106_gcm_ctx_get(cryptd_child); 845 memcpy(child_ctx, ctx, sizeof(*ctx)); 846 ctx->cryptd_tfm = cryptd_tfm; 847 tfm->crt_aead.reqsize = sizeof(struct aead_request) 848 + crypto_aead_reqsize(&cryptd_tfm->base); 849 return 0; 850 } 851 852 static void rfc4106_exit(struct crypto_tfm *tfm) 853 { 854 struct aesni_rfc4106_gcm_ctx *ctx = 855 (struct aesni_rfc4106_gcm_ctx *) 856 PTR_ALIGN((u8 *)crypto_tfm_ctx(tfm), AESNI_ALIGN); 857 if (!IS_ERR(ctx->cryptd_tfm)) 858 cryptd_free_aead(ctx->cryptd_tfm); 859 return; 860 } 861 862 static void 863 rfc4106_set_hash_subkey_done(struct crypto_async_request *req, int err) 864 { 865 struct aesni_gcm_set_hash_subkey_result *result = req->data; 866 867 if (err == -EINPROGRESS) 868 return; 869 result->err = err; 870 complete(&result->completion); 871 } 872 873 static int 874 rfc4106_set_hash_subkey(u8 *hash_subkey, const u8 *key, unsigned int key_len) 875 { 876 struct crypto_ablkcipher *ctr_tfm; 877 struct ablkcipher_request *req; 878 int ret = -EINVAL; 879 struct aesni_hash_subkey_req_data *req_data; 880 881 ctr_tfm = crypto_alloc_ablkcipher("ctr(aes)", 0, 0); 882 if (IS_ERR(ctr_tfm)) 883 return PTR_ERR(ctr_tfm); 884 885 crypto_ablkcipher_clear_flags(ctr_tfm, ~0); 886 887 ret = crypto_ablkcipher_setkey(ctr_tfm, key, key_len); 888 if (ret) 889 goto out_free_ablkcipher; 890 891 ret = -ENOMEM; 892 req = ablkcipher_request_alloc(ctr_tfm, GFP_KERNEL); 893 if (!req) 894 goto out_free_ablkcipher; 895 896 req_data = kmalloc(sizeof(*req_data), GFP_KERNEL); 897 if (!req_data) 898 goto out_free_request; 899 900 memset(req_data->iv, 0, sizeof(req_data->iv)); 901 902 /* Clear the data in the hash sub key container to zero.*/ 903 /* We want to cipher all zeros to create the hash sub key. */ 904 memset(hash_subkey, 0, RFC4106_HASH_SUBKEY_SIZE); 905 906 init_completion(&req_data->result.completion); 907 sg_init_one(&req_data->sg, hash_subkey, RFC4106_HASH_SUBKEY_SIZE); 908 ablkcipher_request_set_tfm(req, ctr_tfm); 909 ablkcipher_request_set_callback(req, CRYPTO_TFM_REQ_MAY_SLEEP | 910 CRYPTO_TFM_REQ_MAY_BACKLOG, 911 rfc4106_set_hash_subkey_done, 912 &req_data->result); 913 914 ablkcipher_request_set_crypt(req, &req_data->sg, 915 &req_data->sg, RFC4106_HASH_SUBKEY_SIZE, req_data->iv); 916 917 ret = crypto_ablkcipher_encrypt(req); 918 if (ret == -EINPROGRESS || ret == -EBUSY) { 919 ret = wait_for_completion_interruptible 920 (&req_data->result.completion); 921 if (!ret) 922 ret = req_data->result.err; 923 } 924 kfree(req_data); 925 out_free_request: 926 ablkcipher_request_free(req); 927 out_free_ablkcipher: 928 crypto_free_ablkcipher(ctr_tfm); 929 return ret; 930 } 931 932 static int rfc4106_set_key(struct crypto_aead *parent, const u8 *key, 933 unsigned int key_len) 934 { 935 int ret = 0; 936 struct crypto_tfm *tfm = crypto_aead_tfm(parent); 937 struct aesni_rfc4106_gcm_ctx *ctx = aesni_rfc4106_gcm_ctx_get(parent); 938 struct crypto_aead *cryptd_child = cryptd_aead_child(ctx->cryptd_tfm); 939 struct aesni_rfc4106_gcm_ctx *child_ctx = 940 aesni_rfc4106_gcm_ctx_get(cryptd_child); 941 u8 *new_key_mem = NULL; 942 943 if (key_len < 4) { 944 crypto_tfm_set_flags(tfm, CRYPTO_TFM_RES_BAD_KEY_LEN); 945 return -EINVAL; 946 } 947 /*Account for 4 byte nonce at the end.*/ 948 key_len -= 4; 949 if (key_len != AES_KEYSIZE_128) { 950 crypto_tfm_set_flags(tfm, CRYPTO_TFM_RES_BAD_KEY_LEN); 951 return -EINVAL; 952 } 953 954 memcpy(ctx->nonce, key + key_len, sizeof(ctx->nonce)); 955 /*This must be on a 16 byte boundary!*/ 956 if ((unsigned long)(&(ctx->aes_key_expanded.key_enc[0])) % AESNI_ALIGN) 957 return -EINVAL; 958 959 if ((unsigned long)key % AESNI_ALIGN) { 960 /*key is not aligned: use an auxuliar aligned pointer*/ 961 new_key_mem = kmalloc(key_len+AESNI_ALIGN, GFP_KERNEL); 962 if (!new_key_mem) 963 return -ENOMEM; 964 965 new_key_mem = PTR_ALIGN(new_key_mem, AESNI_ALIGN); 966 memcpy(new_key_mem, key, key_len); 967 key = new_key_mem; 968 } 969 970 if (!irq_fpu_usable()) 971 ret = crypto_aes_expand_key(&(ctx->aes_key_expanded), 972 key, key_len); 973 else { 974 kernel_fpu_begin(); 975 ret = aesni_set_key(&(ctx->aes_key_expanded), key, key_len); 976 kernel_fpu_end(); 977 } 978 /*This must be on a 16 byte boundary!*/ 979 if ((unsigned long)(&(ctx->hash_subkey[0])) % AESNI_ALIGN) { 980 ret = -EINVAL; 981 goto exit; 982 } 983 ret = rfc4106_set_hash_subkey(ctx->hash_subkey, key, key_len); 984 memcpy(child_ctx, ctx, sizeof(*ctx)); 985 exit: 986 kfree(new_key_mem); 987 return ret; 988 } 989 990 /* This is the Integrity Check Value (aka the authentication tag length and can 991 * be 8, 12 or 16 bytes long. */ 992 static int rfc4106_set_authsize(struct crypto_aead *parent, 993 unsigned int authsize) 994 { 995 struct aesni_rfc4106_gcm_ctx *ctx = aesni_rfc4106_gcm_ctx_get(parent); 996 struct crypto_aead *cryptd_child = cryptd_aead_child(ctx->cryptd_tfm); 997 998 switch (authsize) { 999 case 8: 1000 case 12: 1001 case 16: 1002 break; 1003 default: 1004 return -EINVAL; 1005 } 1006 crypto_aead_crt(parent)->authsize = authsize; 1007 crypto_aead_crt(cryptd_child)->authsize = authsize; 1008 return 0; 1009 } 1010 1011 static int rfc4106_encrypt(struct aead_request *req) 1012 { 1013 int ret; 1014 struct crypto_aead *tfm = crypto_aead_reqtfm(req); 1015 struct aesni_rfc4106_gcm_ctx *ctx = aesni_rfc4106_gcm_ctx_get(tfm); 1016 1017 if (!irq_fpu_usable()) { 1018 struct aead_request *cryptd_req = 1019 (struct aead_request *) aead_request_ctx(req); 1020 memcpy(cryptd_req, req, sizeof(*req)); 1021 aead_request_set_tfm(cryptd_req, &ctx->cryptd_tfm->base); 1022 return crypto_aead_encrypt(cryptd_req); 1023 } else { 1024 struct crypto_aead *cryptd_child = cryptd_aead_child(ctx->cryptd_tfm); 1025 kernel_fpu_begin(); 1026 ret = cryptd_child->base.crt_aead.encrypt(req); 1027 kernel_fpu_end(); 1028 return ret; 1029 } 1030 } 1031 1032 static int rfc4106_decrypt(struct aead_request *req) 1033 { 1034 int ret; 1035 struct crypto_aead *tfm = crypto_aead_reqtfm(req); 1036 struct aesni_rfc4106_gcm_ctx *ctx = aesni_rfc4106_gcm_ctx_get(tfm); 1037 1038 if (!irq_fpu_usable()) { 1039 struct aead_request *cryptd_req = 1040 (struct aead_request *) aead_request_ctx(req); 1041 memcpy(cryptd_req, req, sizeof(*req)); 1042 aead_request_set_tfm(cryptd_req, &ctx->cryptd_tfm->base); 1043 return crypto_aead_decrypt(cryptd_req); 1044 } else { 1045 struct crypto_aead *cryptd_child = cryptd_aead_child(ctx->cryptd_tfm); 1046 kernel_fpu_begin(); 1047 ret = cryptd_child->base.crt_aead.decrypt(req); 1048 kernel_fpu_end(); 1049 return ret; 1050 } 1051 } 1052 1053 static struct crypto_alg rfc4106_alg = { 1054 .cra_name = "rfc4106(gcm(aes))", 1055 .cra_driver_name = "rfc4106-gcm-aesni", 1056 .cra_priority = 400, 1057 .cra_flags = CRYPTO_ALG_TYPE_AEAD | CRYPTO_ALG_ASYNC, 1058 .cra_blocksize = 1, 1059 .cra_ctxsize = sizeof(struct aesni_rfc4106_gcm_ctx) + AESNI_ALIGN, 1060 .cra_alignmask = 0, 1061 .cra_type = &crypto_nivaead_type, 1062 .cra_module = THIS_MODULE, 1063 .cra_list = LIST_HEAD_INIT(rfc4106_alg.cra_list), 1064 .cra_init = rfc4106_init, 1065 .cra_exit = rfc4106_exit, 1066 .cra_u = { 1067 .aead = { 1068 .setkey = rfc4106_set_key, 1069 .setauthsize = rfc4106_set_authsize, 1070 .encrypt = rfc4106_encrypt, 1071 .decrypt = rfc4106_decrypt, 1072 .geniv = "seqiv", 1073 .ivsize = 8, 1074 .maxauthsize = 16, 1075 }, 1076 }, 1077 }; 1078 1079 static int __driver_rfc4106_encrypt(struct aead_request *req) 1080 { 1081 u8 one_entry_in_sg = 0; 1082 u8 *src, *dst, *assoc; 1083 __be32 counter = cpu_to_be32(1); 1084 struct crypto_aead *tfm = crypto_aead_reqtfm(req); 1085 struct aesni_rfc4106_gcm_ctx *ctx = aesni_rfc4106_gcm_ctx_get(tfm); 1086 void *aes_ctx = &(ctx->aes_key_expanded); 1087 unsigned long auth_tag_len = crypto_aead_authsize(tfm); 1088 u8 iv_tab[16+AESNI_ALIGN]; 1089 u8* iv = (u8 *) PTR_ALIGN((u8 *)iv_tab, AESNI_ALIGN); 1090 struct scatter_walk src_sg_walk; 1091 struct scatter_walk assoc_sg_walk; 1092 struct scatter_walk dst_sg_walk; 1093 unsigned int i; 1094 1095 /* Assuming we are supporting rfc4106 64-bit extended */ 1096 /* sequence numbers We need to have the AAD length equal */ 1097 /* to 8 or 12 bytes */ 1098 if (unlikely(req->assoclen != 8 && req->assoclen != 12)) 1099 return -EINVAL; 1100 /* IV below built */ 1101 for (i = 0; i < 4; i++) 1102 *(iv+i) = ctx->nonce[i]; 1103 for (i = 0; i < 8; i++) 1104 *(iv+4+i) = req->iv[i]; 1105 *((__be32 *)(iv+12)) = counter; 1106 1107 if ((sg_is_last(req->src)) && (sg_is_last(req->assoc))) { 1108 one_entry_in_sg = 1; 1109 scatterwalk_start(&src_sg_walk, req->src); 1110 scatterwalk_start(&assoc_sg_walk, req->assoc); 1111 src = scatterwalk_map(&src_sg_walk); 1112 assoc = scatterwalk_map(&assoc_sg_walk); 1113 dst = src; 1114 if (unlikely(req->src != req->dst)) { 1115 scatterwalk_start(&dst_sg_walk, req->dst); 1116 dst = scatterwalk_map(&dst_sg_walk); 1117 } 1118 1119 } else { 1120 /* Allocate memory for src, dst, assoc */ 1121 src = kmalloc(req->cryptlen + auth_tag_len + req->assoclen, 1122 GFP_ATOMIC); 1123 if (unlikely(!src)) 1124 return -ENOMEM; 1125 assoc = (src + req->cryptlen + auth_tag_len); 1126 scatterwalk_map_and_copy(src, req->src, 0, req->cryptlen, 0); 1127 scatterwalk_map_and_copy(assoc, req->assoc, 0, 1128 req->assoclen, 0); 1129 dst = src; 1130 } 1131 1132 aesni_gcm_enc(aes_ctx, dst, src, (unsigned long)req->cryptlen, iv, 1133 ctx->hash_subkey, assoc, (unsigned long)req->assoclen, dst 1134 + ((unsigned long)req->cryptlen), auth_tag_len); 1135 1136 /* The authTag (aka the Integrity Check Value) needs to be written 1137 * back to the packet. */ 1138 if (one_entry_in_sg) { 1139 if (unlikely(req->src != req->dst)) { 1140 scatterwalk_unmap(dst); 1141 scatterwalk_done(&dst_sg_walk, 0, 0); 1142 } 1143 scatterwalk_unmap(src); 1144 scatterwalk_unmap(assoc); 1145 scatterwalk_done(&src_sg_walk, 0, 0); 1146 scatterwalk_done(&assoc_sg_walk, 0, 0); 1147 } else { 1148 scatterwalk_map_and_copy(dst, req->dst, 0, 1149 req->cryptlen + auth_tag_len, 1); 1150 kfree(src); 1151 } 1152 return 0; 1153 } 1154 1155 static int __driver_rfc4106_decrypt(struct aead_request *req) 1156 { 1157 u8 one_entry_in_sg = 0; 1158 u8 *src, *dst, *assoc; 1159 unsigned long tempCipherLen = 0; 1160 __be32 counter = cpu_to_be32(1); 1161 int retval = 0; 1162 struct crypto_aead *tfm = crypto_aead_reqtfm(req); 1163 struct aesni_rfc4106_gcm_ctx *ctx = aesni_rfc4106_gcm_ctx_get(tfm); 1164 void *aes_ctx = &(ctx->aes_key_expanded); 1165 unsigned long auth_tag_len = crypto_aead_authsize(tfm); 1166 u8 iv_and_authTag[32+AESNI_ALIGN]; 1167 u8 *iv = (u8 *) PTR_ALIGN((u8 *)iv_and_authTag, AESNI_ALIGN); 1168 u8 *authTag = iv + 16; 1169 struct scatter_walk src_sg_walk; 1170 struct scatter_walk assoc_sg_walk; 1171 struct scatter_walk dst_sg_walk; 1172 unsigned int i; 1173 1174 if (unlikely((req->cryptlen < auth_tag_len) || 1175 (req->assoclen != 8 && req->assoclen != 12))) 1176 return -EINVAL; 1177 /* Assuming we are supporting rfc4106 64-bit extended */ 1178 /* sequence numbers We need to have the AAD length */ 1179 /* equal to 8 or 12 bytes */ 1180 1181 tempCipherLen = (unsigned long)(req->cryptlen - auth_tag_len); 1182 /* IV below built */ 1183 for (i = 0; i < 4; i++) 1184 *(iv+i) = ctx->nonce[i]; 1185 for (i = 0; i < 8; i++) 1186 *(iv+4+i) = req->iv[i]; 1187 *((__be32 *)(iv+12)) = counter; 1188 1189 if ((sg_is_last(req->src)) && (sg_is_last(req->assoc))) { 1190 one_entry_in_sg = 1; 1191 scatterwalk_start(&src_sg_walk, req->src); 1192 scatterwalk_start(&assoc_sg_walk, req->assoc); 1193 src = scatterwalk_map(&src_sg_walk); 1194 assoc = scatterwalk_map(&assoc_sg_walk); 1195 dst = src; 1196 if (unlikely(req->src != req->dst)) { 1197 scatterwalk_start(&dst_sg_walk, req->dst); 1198 dst = scatterwalk_map(&dst_sg_walk); 1199 } 1200 1201 } else { 1202 /* Allocate memory for src, dst, assoc */ 1203 src = kmalloc(req->cryptlen + req->assoclen, GFP_ATOMIC); 1204 if (!src) 1205 return -ENOMEM; 1206 assoc = (src + req->cryptlen + auth_tag_len); 1207 scatterwalk_map_and_copy(src, req->src, 0, req->cryptlen, 0); 1208 scatterwalk_map_and_copy(assoc, req->assoc, 0, 1209 req->assoclen, 0); 1210 dst = src; 1211 } 1212 1213 aesni_gcm_dec(aes_ctx, dst, src, tempCipherLen, iv, 1214 ctx->hash_subkey, assoc, (unsigned long)req->assoclen, 1215 authTag, auth_tag_len); 1216 1217 /* Compare generated tag with passed in tag. */ 1218 retval = memcmp(src + tempCipherLen, authTag, auth_tag_len) ? 1219 -EBADMSG : 0; 1220 1221 if (one_entry_in_sg) { 1222 if (unlikely(req->src != req->dst)) { 1223 scatterwalk_unmap(dst); 1224 scatterwalk_done(&dst_sg_walk, 0, 0); 1225 } 1226 scatterwalk_unmap(src); 1227 scatterwalk_unmap(assoc); 1228 scatterwalk_done(&src_sg_walk, 0, 0); 1229 scatterwalk_done(&assoc_sg_walk, 0, 0); 1230 } else { 1231 scatterwalk_map_and_copy(dst, req->dst, 0, req->cryptlen, 1); 1232 kfree(src); 1233 } 1234 return retval; 1235 } 1236 1237 static struct crypto_alg __rfc4106_alg = { 1238 .cra_name = "__gcm-aes-aesni", 1239 .cra_driver_name = "__driver-gcm-aes-aesni", 1240 .cra_priority = 0, 1241 .cra_flags = CRYPTO_ALG_TYPE_AEAD, 1242 .cra_blocksize = 1, 1243 .cra_ctxsize = sizeof(struct aesni_rfc4106_gcm_ctx) + AESNI_ALIGN, 1244 .cra_alignmask = 0, 1245 .cra_type = &crypto_aead_type, 1246 .cra_module = THIS_MODULE, 1247 .cra_list = LIST_HEAD_INIT(__rfc4106_alg.cra_list), 1248 .cra_u = { 1249 .aead = { 1250 .encrypt = __driver_rfc4106_encrypt, 1251 .decrypt = __driver_rfc4106_decrypt, 1252 }, 1253 }, 1254 }; 1255 #endif 1256 1257 1258 static const struct x86_cpu_id aesni_cpu_id[] = { 1259 X86_FEATURE_MATCH(X86_FEATURE_AES), 1260 {} 1261 }; 1262 MODULE_DEVICE_TABLE(x86cpu, aesni_cpu_id); 1263 1264 static int __init aesni_init(void) 1265 { 1266 int err; 1267 1268 if (!x86_match_cpu(aesni_cpu_id)) 1269 return -ENODEV; 1270 1271 if ((err = crypto_fpu_init())) 1272 goto fpu_err; 1273 if ((err = crypto_register_alg(&aesni_alg))) 1274 goto aes_err; 1275 if ((err = crypto_register_alg(&__aesni_alg))) 1276 goto __aes_err; 1277 if ((err = crypto_register_alg(&blk_ecb_alg))) 1278 goto blk_ecb_err; 1279 if ((err = crypto_register_alg(&blk_cbc_alg))) 1280 goto blk_cbc_err; 1281 if ((err = crypto_register_alg(&ablk_ecb_alg))) 1282 goto ablk_ecb_err; 1283 if ((err = crypto_register_alg(&ablk_cbc_alg))) 1284 goto ablk_cbc_err; 1285 #ifdef CONFIG_X86_64 1286 if ((err = crypto_register_alg(&blk_ctr_alg))) 1287 goto blk_ctr_err; 1288 if ((err = crypto_register_alg(&ablk_ctr_alg))) 1289 goto ablk_ctr_err; 1290 if ((err = crypto_register_alg(&__rfc4106_alg))) 1291 goto __aead_gcm_err; 1292 if ((err = crypto_register_alg(&rfc4106_alg))) 1293 goto aead_gcm_err; 1294 #ifdef HAS_CTR 1295 if ((err = crypto_register_alg(&ablk_rfc3686_ctr_alg))) 1296 goto ablk_rfc3686_ctr_err; 1297 #endif 1298 #endif 1299 #ifdef HAS_LRW 1300 if ((err = crypto_register_alg(&ablk_lrw_alg))) 1301 goto ablk_lrw_err; 1302 #endif 1303 #ifdef HAS_PCBC 1304 if ((err = crypto_register_alg(&ablk_pcbc_alg))) 1305 goto ablk_pcbc_err; 1306 #endif 1307 #ifdef HAS_XTS 1308 if ((err = crypto_register_alg(&ablk_xts_alg))) 1309 goto ablk_xts_err; 1310 #endif 1311 return err; 1312 1313 #ifdef HAS_XTS 1314 ablk_xts_err: 1315 #endif 1316 #ifdef HAS_PCBC 1317 crypto_unregister_alg(&ablk_pcbc_alg); 1318 ablk_pcbc_err: 1319 #endif 1320 #ifdef HAS_LRW 1321 crypto_unregister_alg(&ablk_lrw_alg); 1322 ablk_lrw_err: 1323 #endif 1324 #ifdef CONFIG_X86_64 1325 #ifdef HAS_CTR 1326 crypto_unregister_alg(&ablk_rfc3686_ctr_alg); 1327 ablk_rfc3686_ctr_err: 1328 #endif 1329 crypto_unregister_alg(&rfc4106_alg); 1330 aead_gcm_err: 1331 crypto_unregister_alg(&__rfc4106_alg); 1332 __aead_gcm_err: 1333 crypto_unregister_alg(&ablk_ctr_alg); 1334 ablk_ctr_err: 1335 crypto_unregister_alg(&blk_ctr_alg); 1336 blk_ctr_err: 1337 #endif 1338 crypto_unregister_alg(&ablk_cbc_alg); 1339 ablk_cbc_err: 1340 crypto_unregister_alg(&ablk_ecb_alg); 1341 ablk_ecb_err: 1342 crypto_unregister_alg(&blk_cbc_alg); 1343 blk_cbc_err: 1344 crypto_unregister_alg(&blk_ecb_alg); 1345 blk_ecb_err: 1346 crypto_unregister_alg(&__aesni_alg); 1347 __aes_err: 1348 crypto_unregister_alg(&aesni_alg); 1349 aes_err: 1350 fpu_err: 1351 return err; 1352 } 1353 1354 static void __exit aesni_exit(void) 1355 { 1356 #ifdef HAS_XTS 1357 crypto_unregister_alg(&ablk_xts_alg); 1358 #endif 1359 #ifdef HAS_PCBC 1360 crypto_unregister_alg(&ablk_pcbc_alg); 1361 #endif 1362 #ifdef HAS_LRW 1363 crypto_unregister_alg(&ablk_lrw_alg); 1364 #endif 1365 #ifdef CONFIG_X86_64 1366 #ifdef HAS_CTR 1367 crypto_unregister_alg(&ablk_rfc3686_ctr_alg); 1368 #endif 1369 crypto_unregister_alg(&rfc4106_alg); 1370 crypto_unregister_alg(&__rfc4106_alg); 1371 crypto_unregister_alg(&ablk_ctr_alg); 1372 crypto_unregister_alg(&blk_ctr_alg); 1373 #endif 1374 crypto_unregister_alg(&ablk_cbc_alg); 1375 crypto_unregister_alg(&ablk_ecb_alg); 1376 crypto_unregister_alg(&blk_cbc_alg); 1377 crypto_unregister_alg(&blk_ecb_alg); 1378 crypto_unregister_alg(&__aesni_alg); 1379 crypto_unregister_alg(&aesni_alg); 1380 1381 crypto_fpu_exit(); 1382 } 1383 1384 module_init(aesni_init); 1385 module_exit(aesni_exit); 1386 1387 MODULE_DESCRIPTION("Rijndael (AES) Cipher Algorithm, Intel AES-NI instructions optimized"); 1388 MODULE_LICENSE("GPL"); 1389 MODULE_ALIAS("aes"); 1390