1 /* 2 * Support for Intel AES-NI instructions. This file contains glue 3 * code, the real AES implementation is in intel-aes_asm.S. 4 * 5 * Copyright (C) 2008, Intel Corp. 6 * Author: Huang Ying <ying.huang@intel.com> 7 * 8 * Added RFC4106 AES-GCM support for 128-bit keys under the AEAD 9 * interface for 64-bit kernels. 10 * Authors: Adrian Hoban <adrian.hoban@intel.com> 11 * Gabriele Paoloni <gabriele.paoloni@intel.com> 12 * Tadeusz Struk (tadeusz.struk@intel.com) 13 * Aidan O'Mahony (aidan.o.mahony@intel.com) 14 * Copyright (c) 2010, Intel Corporation. 15 * 16 * This program is free software; you can redistribute it and/or modify 17 * it under the terms of the GNU General Public License as published by 18 * the Free Software Foundation; either version 2 of the License, or 19 * (at your option) any later version. 20 */ 21 22 #include <linux/hardirq.h> 23 #include <linux/types.h> 24 #include <linux/crypto.h> 25 #include <linux/module.h> 26 #include <linux/err.h> 27 #include <crypto/algapi.h> 28 #include <crypto/aes.h> 29 #include <crypto/cryptd.h> 30 #include <crypto/ctr.h> 31 #include <crypto/b128ops.h> 32 #include <crypto/lrw.h> 33 #include <crypto/xts.h> 34 #include <asm/cpu_device_id.h> 35 #include <asm/i387.h> 36 #include <asm/crypto/aes.h> 37 #include <asm/crypto/ablk_helper.h> 38 #include <crypto/scatterwalk.h> 39 #include <crypto/internal/aead.h> 40 #include <linux/workqueue.h> 41 #include <linux/spinlock.h> 42 43 #if defined(CONFIG_CRYPTO_CTR) || defined(CONFIG_CRYPTO_CTR_MODULE) 44 #define HAS_CTR 45 #endif 46 47 #if defined(CONFIG_CRYPTO_PCBC) || defined(CONFIG_CRYPTO_PCBC_MODULE) 48 #define HAS_PCBC 49 #endif 50 51 /* This data is stored at the end of the crypto_tfm struct. 52 * It's a type of per "session" data storage location. 53 * This needs to be 16 byte aligned. 54 */ 55 struct aesni_rfc4106_gcm_ctx { 56 u8 hash_subkey[16]; 57 struct crypto_aes_ctx aes_key_expanded; 58 u8 nonce[4]; 59 struct cryptd_aead *cryptd_tfm; 60 }; 61 62 struct aesni_gcm_set_hash_subkey_result { 63 int err; 64 struct completion completion; 65 }; 66 67 struct aesni_hash_subkey_req_data { 68 u8 iv[16]; 69 struct aesni_gcm_set_hash_subkey_result result; 70 struct scatterlist sg; 71 }; 72 73 #define AESNI_ALIGN (16) 74 #define AES_BLOCK_MASK (~(AES_BLOCK_SIZE-1)) 75 #define RFC4106_HASH_SUBKEY_SIZE 16 76 77 struct aesni_lrw_ctx { 78 struct lrw_table_ctx lrw_table; 79 u8 raw_aes_ctx[sizeof(struct crypto_aes_ctx) + AESNI_ALIGN - 1]; 80 }; 81 82 struct aesni_xts_ctx { 83 u8 raw_tweak_ctx[sizeof(struct crypto_aes_ctx) + AESNI_ALIGN - 1]; 84 u8 raw_crypt_ctx[sizeof(struct crypto_aes_ctx) + AESNI_ALIGN - 1]; 85 }; 86 87 asmlinkage int aesni_set_key(struct crypto_aes_ctx *ctx, const u8 *in_key, 88 unsigned int key_len); 89 asmlinkage void aesni_enc(struct crypto_aes_ctx *ctx, u8 *out, 90 const u8 *in); 91 asmlinkage void aesni_dec(struct crypto_aes_ctx *ctx, u8 *out, 92 const u8 *in); 93 asmlinkage void aesni_ecb_enc(struct crypto_aes_ctx *ctx, u8 *out, 94 const u8 *in, unsigned int len); 95 asmlinkage void aesni_ecb_dec(struct crypto_aes_ctx *ctx, u8 *out, 96 const u8 *in, unsigned int len); 97 asmlinkage void aesni_cbc_enc(struct crypto_aes_ctx *ctx, u8 *out, 98 const u8 *in, unsigned int len, u8 *iv); 99 asmlinkage void aesni_cbc_dec(struct crypto_aes_ctx *ctx, u8 *out, 100 const u8 *in, unsigned int len, u8 *iv); 101 102 int crypto_fpu_init(void); 103 void crypto_fpu_exit(void); 104 105 #ifdef CONFIG_X86_64 106 asmlinkage void aesni_ctr_enc(struct crypto_aes_ctx *ctx, u8 *out, 107 const u8 *in, unsigned int len, u8 *iv); 108 109 /* asmlinkage void aesni_gcm_enc() 110 * void *ctx, AES Key schedule. Starts on a 16 byte boundary. 111 * u8 *out, Ciphertext output. Encrypt in-place is allowed. 112 * const u8 *in, Plaintext input 113 * unsigned long plaintext_len, Length of data in bytes for encryption. 114 * u8 *iv, Pre-counter block j0: 4 byte salt (from Security Association) 115 * concatenated with 8 byte Initialisation Vector (from IPSec ESP 116 * Payload) concatenated with 0x00000001. 16-byte aligned pointer. 117 * u8 *hash_subkey, the Hash sub key input. Data starts on a 16-byte boundary. 118 * const u8 *aad, Additional Authentication Data (AAD) 119 * unsigned long aad_len, Length of AAD in bytes. With RFC4106 this 120 * is going to be 8 or 12 bytes 121 * u8 *auth_tag, Authenticated Tag output. 122 * unsigned long auth_tag_len), Authenticated Tag Length in bytes. 123 * Valid values are 16 (most likely), 12 or 8. 124 */ 125 asmlinkage void aesni_gcm_enc(void *ctx, u8 *out, 126 const u8 *in, unsigned long plaintext_len, u8 *iv, 127 u8 *hash_subkey, const u8 *aad, unsigned long aad_len, 128 u8 *auth_tag, unsigned long auth_tag_len); 129 130 /* asmlinkage void aesni_gcm_dec() 131 * void *ctx, AES Key schedule. Starts on a 16 byte boundary. 132 * u8 *out, Plaintext output. Decrypt in-place is allowed. 133 * const u8 *in, Ciphertext input 134 * unsigned long ciphertext_len, Length of data in bytes for decryption. 135 * u8 *iv, Pre-counter block j0: 4 byte salt (from Security Association) 136 * concatenated with 8 byte Initialisation Vector (from IPSec ESP 137 * Payload) concatenated with 0x00000001. 16-byte aligned pointer. 138 * u8 *hash_subkey, the Hash sub key input. Data starts on a 16-byte boundary. 139 * const u8 *aad, Additional Authentication Data (AAD) 140 * unsigned long aad_len, Length of AAD in bytes. With RFC4106 this is going 141 * to be 8 or 12 bytes 142 * u8 *auth_tag, Authenticated Tag output. 143 * unsigned long auth_tag_len) Authenticated Tag Length in bytes. 144 * Valid values are 16 (most likely), 12 or 8. 145 */ 146 asmlinkage void aesni_gcm_dec(void *ctx, u8 *out, 147 const u8 *in, unsigned long ciphertext_len, u8 *iv, 148 u8 *hash_subkey, const u8 *aad, unsigned long aad_len, 149 u8 *auth_tag, unsigned long auth_tag_len); 150 151 static inline struct 152 aesni_rfc4106_gcm_ctx *aesni_rfc4106_gcm_ctx_get(struct crypto_aead *tfm) 153 { 154 return 155 (struct aesni_rfc4106_gcm_ctx *) 156 PTR_ALIGN((u8 *) 157 crypto_tfm_ctx(crypto_aead_tfm(tfm)), AESNI_ALIGN); 158 } 159 #endif 160 161 static inline struct crypto_aes_ctx *aes_ctx(void *raw_ctx) 162 { 163 unsigned long addr = (unsigned long)raw_ctx; 164 unsigned long align = AESNI_ALIGN; 165 166 if (align <= crypto_tfm_ctx_alignment()) 167 align = 1; 168 return (struct crypto_aes_ctx *)ALIGN(addr, align); 169 } 170 171 static int aes_set_key_common(struct crypto_tfm *tfm, void *raw_ctx, 172 const u8 *in_key, unsigned int key_len) 173 { 174 struct crypto_aes_ctx *ctx = aes_ctx(raw_ctx); 175 u32 *flags = &tfm->crt_flags; 176 int err; 177 178 if (key_len != AES_KEYSIZE_128 && key_len != AES_KEYSIZE_192 && 179 key_len != AES_KEYSIZE_256) { 180 *flags |= CRYPTO_TFM_RES_BAD_KEY_LEN; 181 return -EINVAL; 182 } 183 184 if (!irq_fpu_usable()) 185 err = crypto_aes_expand_key(ctx, in_key, key_len); 186 else { 187 kernel_fpu_begin(); 188 err = aesni_set_key(ctx, in_key, key_len); 189 kernel_fpu_end(); 190 } 191 192 return err; 193 } 194 195 static int aes_set_key(struct crypto_tfm *tfm, const u8 *in_key, 196 unsigned int key_len) 197 { 198 return aes_set_key_common(tfm, crypto_tfm_ctx(tfm), in_key, key_len); 199 } 200 201 static void aes_encrypt(struct crypto_tfm *tfm, u8 *dst, const u8 *src) 202 { 203 struct crypto_aes_ctx *ctx = aes_ctx(crypto_tfm_ctx(tfm)); 204 205 if (!irq_fpu_usable()) 206 crypto_aes_encrypt_x86(ctx, dst, src); 207 else { 208 kernel_fpu_begin(); 209 aesni_enc(ctx, dst, src); 210 kernel_fpu_end(); 211 } 212 } 213 214 static void aes_decrypt(struct crypto_tfm *tfm, u8 *dst, const u8 *src) 215 { 216 struct crypto_aes_ctx *ctx = aes_ctx(crypto_tfm_ctx(tfm)); 217 218 if (!irq_fpu_usable()) 219 crypto_aes_decrypt_x86(ctx, dst, src); 220 else { 221 kernel_fpu_begin(); 222 aesni_dec(ctx, dst, src); 223 kernel_fpu_end(); 224 } 225 } 226 227 static void __aes_encrypt(struct crypto_tfm *tfm, u8 *dst, const u8 *src) 228 { 229 struct crypto_aes_ctx *ctx = aes_ctx(crypto_tfm_ctx(tfm)); 230 231 aesni_enc(ctx, dst, src); 232 } 233 234 static void __aes_decrypt(struct crypto_tfm *tfm, u8 *dst, const u8 *src) 235 { 236 struct crypto_aes_ctx *ctx = aes_ctx(crypto_tfm_ctx(tfm)); 237 238 aesni_dec(ctx, dst, src); 239 } 240 241 static int ecb_encrypt(struct blkcipher_desc *desc, 242 struct scatterlist *dst, struct scatterlist *src, 243 unsigned int nbytes) 244 { 245 struct crypto_aes_ctx *ctx = aes_ctx(crypto_blkcipher_ctx(desc->tfm)); 246 struct blkcipher_walk walk; 247 int err; 248 249 blkcipher_walk_init(&walk, dst, src, nbytes); 250 err = blkcipher_walk_virt(desc, &walk); 251 desc->flags &= ~CRYPTO_TFM_REQ_MAY_SLEEP; 252 253 kernel_fpu_begin(); 254 while ((nbytes = walk.nbytes)) { 255 aesni_ecb_enc(ctx, walk.dst.virt.addr, walk.src.virt.addr, 256 nbytes & AES_BLOCK_MASK); 257 nbytes &= AES_BLOCK_SIZE - 1; 258 err = blkcipher_walk_done(desc, &walk, nbytes); 259 } 260 kernel_fpu_end(); 261 262 return err; 263 } 264 265 static int ecb_decrypt(struct blkcipher_desc *desc, 266 struct scatterlist *dst, struct scatterlist *src, 267 unsigned int nbytes) 268 { 269 struct crypto_aes_ctx *ctx = aes_ctx(crypto_blkcipher_ctx(desc->tfm)); 270 struct blkcipher_walk walk; 271 int err; 272 273 blkcipher_walk_init(&walk, dst, src, nbytes); 274 err = blkcipher_walk_virt(desc, &walk); 275 desc->flags &= ~CRYPTO_TFM_REQ_MAY_SLEEP; 276 277 kernel_fpu_begin(); 278 while ((nbytes = walk.nbytes)) { 279 aesni_ecb_dec(ctx, walk.dst.virt.addr, walk.src.virt.addr, 280 nbytes & AES_BLOCK_MASK); 281 nbytes &= AES_BLOCK_SIZE - 1; 282 err = blkcipher_walk_done(desc, &walk, nbytes); 283 } 284 kernel_fpu_end(); 285 286 return err; 287 } 288 289 static int cbc_encrypt(struct blkcipher_desc *desc, 290 struct scatterlist *dst, struct scatterlist *src, 291 unsigned int nbytes) 292 { 293 struct crypto_aes_ctx *ctx = aes_ctx(crypto_blkcipher_ctx(desc->tfm)); 294 struct blkcipher_walk walk; 295 int err; 296 297 blkcipher_walk_init(&walk, dst, src, nbytes); 298 err = blkcipher_walk_virt(desc, &walk); 299 desc->flags &= ~CRYPTO_TFM_REQ_MAY_SLEEP; 300 301 kernel_fpu_begin(); 302 while ((nbytes = walk.nbytes)) { 303 aesni_cbc_enc(ctx, walk.dst.virt.addr, walk.src.virt.addr, 304 nbytes & AES_BLOCK_MASK, walk.iv); 305 nbytes &= AES_BLOCK_SIZE - 1; 306 err = blkcipher_walk_done(desc, &walk, nbytes); 307 } 308 kernel_fpu_end(); 309 310 return err; 311 } 312 313 static int cbc_decrypt(struct blkcipher_desc *desc, 314 struct scatterlist *dst, struct scatterlist *src, 315 unsigned int nbytes) 316 { 317 struct crypto_aes_ctx *ctx = aes_ctx(crypto_blkcipher_ctx(desc->tfm)); 318 struct blkcipher_walk walk; 319 int err; 320 321 blkcipher_walk_init(&walk, dst, src, nbytes); 322 err = blkcipher_walk_virt(desc, &walk); 323 desc->flags &= ~CRYPTO_TFM_REQ_MAY_SLEEP; 324 325 kernel_fpu_begin(); 326 while ((nbytes = walk.nbytes)) { 327 aesni_cbc_dec(ctx, walk.dst.virt.addr, walk.src.virt.addr, 328 nbytes & AES_BLOCK_MASK, walk.iv); 329 nbytes &= AES_BLOCK_SIZE - 1; 330 err = blkcipher_walk_done(desc, &walk, nbytes); 331 } 332 kernel_fpu_end(); 333 334 return err; 335 } 336 337 #ifdef CONFIG_X86_64 338 static void ctr_crypt_final(struct crypto_aes_ctx *ctx, 339 struct blkcipher_walk *walk) 340 { 341 u8 *ctrblk = walk->iv; 342 u8 keystream[AES_BLOCK_SIZE]; 343 u8 *src = walk->src.virt.addr; 344 u8 *dst = walk->dst.virt.addr; 345 unsigned int nbytes = walk->nbytes; 346 347 aesni_enc(ctx, keystream, ctrblk); 348 crypto_xor(keystream, src, nbytes); 349 memcpy(dst, keystream, nbytes); 350 crypto_inc(ctrblk, AES_BLOCK_SIZE); 351 } 352 353 static int ctr_crypt(struct blkcipher_desc *desc, 354 struct scatterlist *dst, struct scatterlist *src, 355 unsigned int nbytes) 356 { 357 struct crypto_aes_ctx *ctx = aes_ctx(crypto_blkcipher_ctx(desc->tfm)); 358 struct blkcipher_walk walk; 359 int err; 360 361 blkcipher_walk_init(&walk, dst, src, nbytes); 362 err = blkcipher_walk_virt_block(desc, &walk, AES_BLOCK_SIZE); 363 desc->flags &= ~CRYPTO_TFM_REQ_MAY_SLEEP; 364 365 kernel_fpu_begin(); 366 while ((nbytes = walk.nbytes) >= AES_BLOCK_SIZE) { 367 aesni_ctr_enc(ctx, walk.dst.virt.addr, walk.src.virt.addr, 368 nbytes & AES_BLOCK_MASK, walk.iv); 369 nbytes &= AES_BLOCK_SIZE - 1; 370 err = blkcipher_walk_done(desc, &walk, nbytes); 371 } 372 if (walk.nbytes) { 373 ctr_crypt_final(ctx, &walk); 374 err = blkcipher_walk_done(desc, &walk, 0); 375 } 376 kernel_fpu_end(); 377 378 return err; 379 } 380 #endif 381 382 static int ablk_ecb_init(struct crypto_tfm *tfm) 383 { 384 return ablk_init_common(tfm, "__driver-ecb-aes-aesni"); 385 } 386 387 static int ablk_cbc_init(struct crypto_tfm *tfm) 388 { 389 return ablk_init_common(tfm, "__driver-cbc-aes-aesni"); 390 } 391 392 #ifdef CONFIG_X86_64 393 static int ablk_ctr_init(struct crypto_tfm *tfm) 394 { 395 return ablk_init_common(tfm, "__driver-ctr-aes-aesni"); 396 } 397 398 #ifdef HAS_CTR 399 static int ablk_rfc3686_ctr_init(struct crypto_tfm *tfm) 400 { 401 return ablk_init_common(tfm, "rfc3686(__driver-ctr-aes-aesni)"); 402 } 403 #endif 404 #endif 405 406 #ifdef HAS_PCBC 407 static int ablk_pcbc_init(struct crypto_tfm *tfm) 408 { 409 return ablk_init_common(tfm, "fpu(pcbc(__driver-aes-aesni))"); 410 } 411 #endif 412 413 static void lrw_xts_encrypt_callback(void *ctx, u8 *blks, unsigned int nbytes) 414 { 415 aesni_ecb_enc(ctx, blks, blks, nbytes); 416 } 417 418 static void lrw_xts_decrypt_callback(void *ctx, u8 *blks, unsigned int nbytes) 419 { 420 aesni_ecb_dec(ctx, blks, blks, nbytes); 421 } 422 423 static int lrw_aesni_setkey(struct crypto_tfm *tfm, const u8 *key, 424 unsigned int keylen) 425 { 426 struct aesni_lrw_ctx *ctx = crypto_tfm_ctx(tfm); 427 int err; 428 429 err = aes_set_key_common(tfm, ctx->raw_aes_ctx, key, 430 keylen - AES_BLOCK_SIZE); 431 if (err) 432 return err; 433 434 return lrw_init_table(&ctx->lrw_table, key + keylen - AES_BLOCK_SIZE); 435 } 436 437 static void lrw_aesni_exit_tfm(struct crypto_tfm *tfm) 438 { 439 struct aesni_lrw_ctx *ctx = crypto_tfm_ctx(tfm); 440 441 lrw_free_table(&ctx->lrw_table); 442 } 443 444 static int lrw_encrypt(struct blkcipher_desc *desc, struct scatterlist *dst, 445 struct scatterlist *src, unsigned int nbytes) 446 { 447 struct aesni_lrw_ctx *ctx = crypto_blkcipher_ctx(desc->tfm); 448 be128 buf[8]; 449 struct lrw_crypt_req req = { 450 .tbuf = buf, 451 .tbuflen = sizeof(buf), 452 453 .table_ctx = &ctx->lrw_table, 454 .crypt_ctx = aes_ctx(ctx->raw_aes_ctx), 455 .crypt_fn = lrw_xts_encrypt_callback, 456 }; 457 int ret; 458 459 desc->flags &= ~CRYPTO_TFM_REQ_MAY_SLEEP; 460 461 kernel_fpu_begin(); 462 ret = lrw_crypt(desc, dst, src, nbytes, &req); 463 kernel_fpu_end(); 464 465 return ret; 466 } 467 468 static int lrw_decrypt(struct blkcipher_desc *desc, struct scatterlist *dst, 469 struct scatterlist *src, unsigned int nbytes) 470 { 471 struct aesni_lrw_ctx *ctx = crypto_blkcipher_ctx(desc->tfm); 472 be128 buf[8]; 473 struct lrw_crypt_req req = { 474 .tbuf = buf, 475 .tbuflen = sizeof(buf), 476 477 .table_ctx = &ctx->lrw_table, 478 .crypt_ctx = aes_ctx(ctx->raw_aes_ctx), 479 .crypt_fn = lrw_xts_decrypt_callback, 480 }; 481 int ret; 482 483 desc->flags &= ~CRYPTO_TFM_REQ_MAY_SLEEP; 484 485 kernel_fpu_begin(); 486 ret = lrw_crypt(desc, dst, src, nbytes, &req); 487 kernel_fpu_end(); 488 489 return ret; 490 } 491 492 static int xts_aesni_setkey(struct crypto_tfm *tfm, const u8 *key, 493 unsigned int keylen) 494 { 495 struct aesni_xts_ctx *ctx = crypto_tfm_ctx(tfm); 496 u32 *flags = &tfm->crt_flags; 497 int err; 498 499 /* key consists of keys of equal size concatenated, therefore 500 * the length must be even 501 */ 502 if (keylen % 2) { 503 *flags |= CRYPTO_TFM_RES_BAD_KEY_LEN; 504 return -EINVAL; 505 } 506 507 /* first half of xts-key is for crypt */ 508 err = aes_set_key_common(tfm, ctx->raw_crypt_ctx, key, keylen / 2); 509 if (err) 510 return err; 511 512 /* second half of xts-key is for tweak */ 513 return aes_set_key_common(tfm, ctx->raw_tweak_ctx, key + keylen / 2, 514 keylen / 2); 515 } 516 517 518 static void aesni_xts_tweak(void *ctx, u8 *out, const u8 *in) 519 { 520 aesni_enc(ctx, out, in); 521 } 522 523 static int xts_encrypt(struct blkcipher_desc *desc, struct scatterlist *dst, 524 struct scatterlist *src, unsigned int nbytes) 525 { 526 struct aesni_xts_ctx *ctx = crypto_blkcipher_ctx(desc->tfm); 527 be128 buf[8]; 528 struct xts_crypt_req req = { 529 .tbuf = buf, 530 .tbuflen = sizeof(buf), 531 532 .tweak_ctx = aes_ctx(ctx->raw_tweak_ctx), 533 .tweak_fn = aesni_xts_tweak, 534 .crypt_ctx = aes_ctx(ctx->raw_crypt_ctx), 535 .crypt_fn = lrw_xts_encrypt_callback, 536 }; 537 int ret; 538 539 desc->flags &= ~CRYPTO_TFM_REQ_MAY_SLEEP; 540 541 kernel_fpu_begin(); 542 ret = xts_crypt(desc, dst, src, nbytes, &req); 543 kernel_fpu_end(); 544 545 return ret; 546 } 547 548 static int xts_decrypt(struct blkcipher_desc *desc, struct scatterlist *dst, 549 struct scatterlist *src, unsigned int nbytes) 550 { 551 struct aesni_xts_ctx *ctx = crypto_blkcipher_ctx(desc->tfm); 552 be128 buf[8]; 553 struct xts_crypt_req req = { 554 .tbuf = buf, 555 .tbuflen = sizeof(buf), 556 557 .tweak_ctx = aes_ctx(ctx->raw_tweak_ctx), 558 .tweak_fn = aesni_xts_tweak, 559 .crypt_ctx = aes_ctx(ctx->raw_crypt_ctx), 560 .crypt_fn = lrw_xts_decrypt_callback, 561 }; 562 int ret; 563 564 desc->flags &= ~CRYPTO_TFM_REQ_MAY_SLEEP; 565 566 kernel_fpu_begin(); 567 ret = xts_crypt(desc, dst, src, nbytes, &req); 568 kernel_fpu_end(); 569 570 return ret; 571 } 572 573 #ifdef CONFIG_X86_64 574 static int rfc4106_init(struct crypto_tfm *tfm) 575 { 576 struct cryptd_aead *cryptd_tfm; 577 struct aesni_rfc4106_gcm_ctx *ctx = (struct aesni_rfc4106_gcm_ctx *) 578 PTR_ALIGN((u8 *)crypto_tfm_ctx(tfm), AESNI_ALIGN); 579 struct crypto_aead *cryptd_child; 580 struct aesni_rfc4106_gcm_ctx *child_ctx; 581 cryptd_tfm = cryptd_alloc_aead("__driver-gcm-aes-aesni", 0, 0); 582 if (IS_ERR(cryptd_tfm)) 583 return PTR_ERR(cryptd_tfm); 584 585 cryptd_child = cryptd_aead_child(cryptd_tfm); 586 child_ctx = aesni_rfc4106_gcm_ctx_get(cryptd_child); 587 memcpy(child_ctx, ctx, sizeof(*ctx)); 588 ctx->cryptd_tfm = cryptd_tfm; 589 tfm->crt_aead.reqsize = sizeof(struct aead_request) 590 + crypto_aead_reqsize(&cryptd_tfm->base); 591 return 0; 592 } 593 594 static void rfc4106_exit(struct crypto_tfm *tfm) 595 { 596 struct aesni_rfc4106_gcm_ctx *ctx = 597 (struct aesni_rfc4106_gcm_ctx *) 598 PTR_ALIGN((u8 *)crypto_tfm_ctx(tfm), AESNI_ALIGN); 599 if (!IS_ERR(ctx->cryptd_tfm)) 600 cryptd_free_aead(ctx->cryptd_tfm); 601 return; 602 } 603 604 static void 605 rfc4106_set_hash_subkey_done(struct crypto_async_request *req, int err) 606 { 607 struct aesni_gcm_set_hash_subkey_result *result = req->data; 608 609 if (err == -EINPROGRESS) 610 return; 611 result->err = err; 612 complete(&result->completion); 613 } 614 615 static int 616 rfc4106_set_hash_subkey(u8 *hash_subkey, const u8 *key, unsigned int key_len) 617 { 618 struct crypto_ablkcipher *ctr_tfm; 619 struct ablkcipher_request *req; 620 int ret = -EINVAL; 621 struct aesni_hash_subkey_req_data *req_data; 622 623 ctr_tfm = crypto_alloc_ablkcipher("ctr(aes)", 0, 0); 624 if (IS_ERR(ctr_tfm)) 625 return PTR_ERR(ctr_tfm); 626 627 crypto_ablkcipher_clear_flags(ctr_tfm, ~0); 628 629 ret = crypto_ablkcipher_setkey(ctr_tfm, key, key_len); 630 if (ret) 631 goto out_free_ablkcipher; 632 633 ret = -ENOMEM; 634 req = ablkcipher_request_alloc(ctr_tfm, GFP_KERNEL); 635 if (!req) 636 goto out_free_ablkcipher; 637 638 req_data = kmalloc(sizeof(*req_data), GFP_KERNEL); 639 if (!req_data) 640 goto out_free_request; 641 642 memset(req_data->iv, 0, sizeof(req_data->iv)); 643 644 /* Clear the data in the hash sub key container to zero.*/ 645 /* We want to cipher all zeros to create the hash sub key. */ 646 memset(hash_subkey, 0, RFC4106_HASH_SUBKEY_SIZE); 647 648 init_completion(&req_data->result.completion); 649 sg_init_one(&req_data->sg, hash_subkey, RFC4106_HASH_SUBKEY_SIZE); 650 ablkcipher_request_set_tfm(req, ctr_tfm); 651 ablkcipher_request_set_callback(req, CRYPTO_TFM_REQ_MAY_SLEEP | 652 CRYPTO_TFM_REQ_MAY_BACKLOG, 653 rfc4106_set_hash_subkey_done, 654 &req_data->result); 655 656 ablkcipher_request_set_crypt(req, &req_data->sg, 657 &req_data->sg, RFC4106_HASH_SUBKEY_SIZE, req_data->iv); 658 659 ret = crypto_ablkcipher_encrypt(req); 660 if (ret == -EINPROGRESS || ret == -EBUSY) { 661 ret = wait_for_completion_interruptible 662 (&req_data->result.completion); 663 if (!ret) 664 ret = req_data->result.err; 665 } 666 kfree(req_data); 667 out_free_request: 668 ablkcipher_request_free(req); 669 out_free_ablkcipher: 670 crypto_free_ablkcipher(ctr_tfm); 671 return ret; 672 } 673 674 static int rfc4106_set_key(struct crypto_aead *parent, const u8 *key, 675 unsigned int key_len) 676 { 677 int ret = 0; 678 struct crypto_tfm *tfm = crypto_aead_tfm(parent); 679 struct aesni_rfc4106_gcm_ctx *ctx = aesni_rfc4106_gcm_ctx_get(parent); 680 struct crypto_aead *cryptd_child = cryptd_aead_child(ctx->cryptd_tfm); 681 struct aesni_rfc4106_gcm_ctx *child_ctx = 682 aesni_rfc4106_gcm_ctx_get(cryptd_child); 683 u8 *new_key_align, *new_key_mem = NULL; 684 685 if (key_len < 4) { 686 crypto_tfm_set_flags(tfm, CRYPTO_TFM_RES_BAD_KEY_LEN); 687 return -EINVAL; 688 } 689 /*Account for 4 byte nonce at the end.*/ 690 key_len -= 4; 691 if (key_len != AES_KEYSIZE_128) { 692 crypto_tfm_set_flags(tfm, CRYPTO_TFM_RES_BAD_KEY_LEN); 693 return -EINVAL; 694 } 695 696 memcpy(ctx->nonce, key + key_len, sizeof(ctx->nonce)); 697 /*This must be on a 16 byte boundary!*/ 698 if ((unsigned long)(&(ctx->aes_key_expanded.key_enc[0])) % AESNI_ALIGN) 699 return -EINVAL; 700 701 if ((unsigned long)key % AESNI_ALIGN) { 702 /*key is not aligned: use an auxuliar aligned pointer*/ 703 new_key_mem = kmalloc(key_len+AESNI_ALIGN, GFP_KERNEL); 704 if (!new_key_mem) 705 return -ENOMEM; 706 707 new_key_align = PTR_ALIGN(new_key_mem, AESNI_ALIGN); 708 memcpy(new_key_align, key, key_len); 709 key = new_key_align; 710 } 711 712 if (!irq_fpu_usable()) 713 ret = crypto_aes_expand_key(&(ctx->aes_key_expanded), 714 key, key_len); 715 else { 716 kernel_fpu_begin(); 717 ret = aesni_set_key(&(ctx->aes_key_expanded), key, key_len); 718 kernel_fpu_end(); 719 } 720 /*This must be on a 16 byte boundary!*/ 721 if ((unsigned long)(&(ctx->hash_subkey[0])) % AESNI_ALIGN) { 722 ret = -EINVAL; 723 goto exit; 724 } 725 ret = rfc4106_set_hash_subkey(ctx->hash_subkey, key, key_len); 726 memcpy(child_ctx, ctx, sizeof(*ctx)); 727 exit: 728 kfree(new_key_mem); 729 return ret; 730 } 731 732 /* This is the Integrity Check Value (aka the authentication tag length and can 733 * be 8, 12 or 16 bytes long. */ 734 static int rfc4106_set_authsize(struct crypto_aead *parent, 735 unsigned int authsize) 736 { 737 struct aesni_rfc4106_gcm_ctx *ctx = aesni_rfc4106_gcm_ctx_get(parent); 738 struct crypto_aead *cryptd_child = cryptd_aead_child(ctx->cryptd_tfm); 739 740 switch (authsize) { 741 case 8: 742 case 12: 743 case 16: 744 break; 745 default: 746 return -EINVAL; 747 } 748 crypto_aead_crt(parent)->authsize = authsize; 749 crypto_aead_crt(cryptd_child)->authsize = authsize; 750 return 0; 751 } 752 753 static int rfc4106_encrypt(struct aead_request *req) 754 { 755 int ret; 756 struct crypto_aead *tfm = crypto_aead_reqtfm(req); 757 struct aesni_rfc4106_gcm_ctx *ctx = aesni_rfc4106_gcm_ctx_get(tfm); 758 759 if (!irq_fpu_usable()) { 760 struct aead_request *cryptd_req = 761 (struct aead_request *) aead_request_ctx(req); 762 memcpy(cryptd_req, req, sizeof(*req)); 763 aead_request_set_tfm(cryptd_req, &ctx->cryptd_tfm->base); 764 return crypto_aead_encrypt(cryptd_req); 765 } else { 766 struct crypto_aead *cryptd_child = cryptd_aead_child(ctx->cryptd_tfm); 767 kernel_fpu_begin(); 768 ret = cryptd_child->base.crt_aead.encrypt(req); 769 kernel_fpu_end(); 770 return ret; 771 } 772 } 773 774 static int rfc4106_decrypt(struct aead_request *req) 775 { 776 int ret; 777 struct crypto_aead *tfm = crypto_aead_reqtfm(req); 778 struct aesni_rfc4106_gcm_ctx *ctx = aesni_rfc4106_gcm_ctx_get(tfm); 779 780 if (!irq_fpu_usable()) { 781 struct aead_request *cryptd_req = 782 (struct aead_request *) aead_request_ctx(req); 783 memcpy(cryptd_req, req, sizeof(*req)); 784 aead_request_set_tfm(cryptd_req, &ctx->cryptd_tfm->base); 785 return crypto_aead_decrypt(cryptd_req); 786 } else { 787 struct crypto_aead *cryptd_child = cryptd_aead_child(ctx->cryptd_tfm); 788 kernel_fpu_begin(); 789 ret = cryptd_child->base.crt_aead.decrypt(req); 790 kernel_fpu_end(); 791 return ret; 792 } 793 } 794 795 static int __driver_rfc4106_encrypt(struct aead_request *req) 796 { 797 u8 one_entry_in_sg = 0; 798 u8 *src, *dst, *assoc; 799 __be32 counter = cpu_to_be32(1); 800 struct crypto_aead *tfm = crypto_aead_reqtfm(req); 801 struct aesni_rfc4106_gcm_ctx *ctx = aesni_rfc4106_gcm_ctx_get(tfm); 802 void *aes_ctx = &(ctx->aes_key_expanded); 803 unsigned long auth_tag_len = crypto_aead_authsize(tfm); 804 u8 iv_tab[16+AESNI_ALIGN]; 805 u8* iv = (u8 *) PTR_ALIGN((u8 *)iv_tab, AESNI_ALIGN); 806 struct scatter_walk src_sg_walk; 807 struct scatter_walk assoc_sg_walk; 808 struct scatter_walk dst_sg_walk; 809 unsigned int i; 810 811 /* Assuming we are supporting rfc4106 64-bit extended */ 812 /* sequence numbers We need to have the AAD length equal */ 813 /* to 8 or 12 bytes */ 814 if (unlikely(req->assoclen != 8 && req->assoclen != 12)) 815 return -EINVAL; 816 /* IV below built */ 817 for (i = 0; i < 4; i++) 818 *(iv+i) = ctx->nonce[i]; 819 for (i = 0; i < 8; i++) 820 *(iv+4+i) = req->iv[i]; 821 *((__be32 *)(iv+12)) = counter; 822 823 if ((sg_is_last(req->src)) && (sg_is_last(req->assoc))) { 824 one_entry_in_sg = 1; 825 scatterwalk_start(&src_sg_walk, req->src); 826 scatterwalk_start(&assoc_sg_walk, req->assoc); 827 src = scatterwalk_map(&src_sg_walk); 828 assoc = scatterwalk_map(&assoc_sg_walk); 829 dst = src; 830 if (unlikely(req->src != req->dst)) { 831 scatterwalk_start(&dst_sg_walk, req->dst); 832 dst = scatterwalk_map(&dst_sg_walk); 833 } 834 835 } else { 836 /* Allocate memory for src, dst, assoc */ 837 src = kmalloc(req->cryptlen + auth_tag_len + req->assoclen, 838 GFP_ATOMIC); 839 if (unlikely(!src)) 840 return -ENOMEM; 841 assoc = (src + req->cryptlen + auth_tag_len); 842 scatterwalk_map_and_copy(src, req->src, 0, req->cryptlen, 0); 843 scatterwalk_map_and_copy(assoc, req->assoc, 0, 844 req->assoclen, 0); 845 dst = src; 846 } 847 848 aesni_gcm_enc(aes_ctx, dst, src, (unsigned long)req->cryptlen, iv, 849 ctx->hash_subkey, assoc, (unsigned long)req->assoclen, dst 850 + ((unsigned long)req->cryptlen), auth_tag_len); 851 852 /* The authTag (aka the Integrity Check Value) needs to be written 853 * back to the packet. */ 854 if (one_entry_in_sg) { 855 if (unlikely(req->src != req->dst)) { 856 scatterwalk_unmap(dst); 857 scatterwalk_done(&dst_sg_walk, 0, 0); 858 } 859 scatterwalk_unmap(src); 860 scatterwalk_unmap(assoc); 861 scatterwalk_done(&src_sg_walk, 0, 0); 862 scatterwalk_done(&assoc_sg_walk, 0, 0); 863 } else { 864 scatterwalk_map_and_copy(dst, req->dst, 0, 865 req->cryptlen + auth_tag_len, 1); 866 kfree(src); 867 } 868 return 0; 869 } 870 871 static int __driver_rfc4106_decrypt(struct aead_request *req) 872 { 873 u8 one_entry_in_sg = 0; 874 u8 *src, *dst, *assoc; 875 unsigned long tempCipherLen = 0; 876 __be32 counter = cpu_to_be32(1); 877 int retval = 0; 878 struct crypto_aead *tfm = crypto_aead_reqtfm(req); 879 struct aesni_rfc4106_gcm_ctx *ctx = aesni_rfc4106_gcm_ctx_get(tfm); 880 void *aes_ctx = &(ctx->aes_key_expanded); 881 unsigned long auth_tag_len = crypto_aead_authsize(tfm); 882 u8 iv_and_authTag[32+AESNI_ALIGN]; 883 u8 *iv = (u8 *) PTR_ALIGN((u8 *)iv_and_authTag, AESNI_ALIGN); 884 u8 *authTag = iv + 16; 885 struct scatter_walk src_sg_walk; 886 struct scatter_walk assoc_sg_walk; 887 struct scatter_walk dst_sg_walk; 888 unsigned int i; 889 890 if (unlikely((req->cryptlen < auth_tag_len) || 891 (req->assoclen != 8 && req->assoclen != 12))) 892 return -EINVAL; 893 /* Assuming we are supporting rfc4106 64-bit extended */ 894 /* sequence numbers We need to have the AAD length */ 895 /* equal to 8 or 12 bytes */ 896 897 tempCipherLen = (unsigned long)(req->cryptlen - auth_tag_len); 898 /* IV below built */ 899 for (i = 0; i < 4; i++) 900 *(iv+i) = ctx->nonce[i]; 901 for (i = 0; i < 8; i++) 902 *(iv+4+i) = req->iv[i]; 903 *((__be32 *)(iv+12)) = counter; 904 905 if ((sg_is_last(req->src)) && (sg_is_last(req->assoc))) { 906 one_entry_in_sg = 1; 907 scatterwalk_start(&src_sg_walk, req->src); 908 scatterwalk_start(&assoc_sg_walk, req->assoc); 909 src = scatterwalk_map(&src_sg_walk); 910 assoc = scatterwalk_map(&assoc_sg_walk); 911 dst = src; 912 if (unlikely(req->src != req->dst)) { 913 scatterwalk_start(&dst_sg_walk, req->dst); 914 dst = scatterwalk_map(&dst_sg_walk); 915 } 916 917 } else { 918 /* Allocate memory for src, dst, assoc */ 919 src = kmalloc(req->cryptlen + req->assoclen, GFP_ATOMIC); 920 if (!src) 921 return -ENOMEM; 922 assoc = (src + req->cryptlen + auth_tag_len); 923 scatterwalk_map_and_copy(src, req->src, 0, req->cryptlen, 0); 924 scatterwalk_map_and_copy(assoc, req->assoc, 0, 925 req->assoclen, 0); 926 dst = src; 927 } 928 929 aesni_gcm_dec(aes_ctx, dst, src, tempCipherLen, iv, 930 ctx->hash_subkey, assoc, (unsigned long)req->assoclen, 931 authTag, auth_tag_len); 932 933 /* Compare generated tag with passed in tag. */ 934 retval = memcmp(src + tempCipherLen, authTag, auth_tag_len) ? 935 -EBADMSG : 0; 936 937 if (one_entry_in_sg) { 938 if (unlikely(req->src != req->dst)) { 939 scatterwalk_unmap(dst); 940 scatterwalk_done(&dst_sg_walk, 0, 0); 941 } 942 scatterwalk_unmap(src); 943 scatterwalk_unmap(assoc); 944 scatterwalk_done(&src_sg_walk, 0, 0); 945 scatterwalk_done(&assoc_sg_walk, 0, 0); 946 } else { 947 scatterwalk_map_and_copy(dst, req->dst, 0, req->cryptlen, 1); 948 kfree(src); 949 } 950 return retval; 951 } 952 #endif 953 954 static struct crypto_alg aesni_algs[] = { { 955 .cra_name = "aes", 956 .cra_driver_name = "aes-aesni", 957 .cra_priority = 300, 958 .cra_flags = CRYPTO_ALG_TYPE_CIPHER, 959 .cra_blocksize = AES_BLOCK_SIZE, 960 .cra_ctxsize = sizeof(struct crypto_aes_ctx) + 961 AESNI_ALIGN - 1, 962 .cra_alignmask = 0, 963 .cra_module = THIS_MODULE, 964 .cra_u = { 965 .cipher = { 966 .cia_min_keysize = AES_MIN_KEY_SIZE, 967 .cia_max_keysize = AES_MAX_KEY_SIZE, 968 .cia_setkey = aes_set_key, 969 .cia_encrypt = aes_encrypt, 970 .cia_decrypt = aes_decrypt 971 } 972 } 973 }, { 974 .cra_name = "__aes-aesni", 975 .cra_driver_name = "__driver-aes-aesni", 976 .cra_priority = 0, 977 .cra_flags = CRYPTO_ALG_TYPE_CIPHER, 978 .cra_blocksize = AES_BLOCK_SIZE, 979 .cra_ctxsize = sizeof(struct crypto_aes_ctx) + 980 AESNI_ALIGN - 1, 981 .cra_alignmask = 0, 982 .cra_module = THIS_MODULE, 983 .cra_u = { 984 .cipher = { 985 .cia_min_keysize = AES_MIN_KEY_SIZE, 986 .cia_max_keysize = AES_MAX_KEY_SIZE, 987 .cia_setkey = aes_set_key, 988 .cia_encrypt = __aes_encrypt, 989 .cia_decrypt = __aes_decrypt 990 } 991 } 992 }, { 993 .cra_name = "__ecb-aes-aesni", 994 .cra_driver_name = "__driver-ecb-aes-aesni", 995 .cra_priority = 0, 996 .cra_flags = CRYPTO_ALG_TYPE_BLKCIPHER, 997 .cra_blocksize = AES_BLOCK_SIZE, 998 .cra_ctxsize = sizeof(struct crypto_aes_ctx) + 999 AESNI_ALIGN - 1, 1000 .cra_alignmask = 0, 1001 .cra_type = &crypto_blkcipher_type, 1002 .cra_module = THIS_MODULE, 1003 .cra_u = { 1004 .blkcipher = { 1005 .min_keysize = AES_MIN_KEY_SIZE, 1006 .max_keysize = AES_MAX_KEY_SIZE, 1007 .setkey = aes_set_key, 1008 .encrypt = ecb_encrypt, 1009 .decrypt = ecb_decrypt, 1010 }, 1011 }, 1012 }, { 1013 .cra_name = "__cbc-aes-aesni", 1014 .cra_driver_name = "__driver-cbc-aes-aesni", 1015 .cra_priority = 0, 1016 .cra_flags = CRYPTO_ALG_TYPE_BLKCIPHER, 1017 .cra_blocksize = AES_BLOCK_SIZE, 1018 .cra_ctxsize = sizeof(struct crypto_aes_ctx) + 1019 AESNI_ALIGN - 1, 1020 .cra_alignmask = 0, 1021 .cra_type = &crypto_blkcipher_type, 1022 .cra_module = THIS_MODULE, 1023 .cra_u = { 1024 .blkcipher = { 1025 .min_keysize = AES_MIN_KEY_SIZE, 1026 .max_keysize = AES_MAX_KEY_SIZE, 1027 .setkey = aes_set_key, 1028 .encrypt = cbc_encrypt, 1029 .decrypt = cbc_decrypt, 1030 }, 1031 }, 1032 }, { 1033 .cra_name = "ecb(aes)", 1034 .cra_driver_name = "ecb-aes-aesni", 1035 .cra_priority = 400, 1036 .cra_flags = CRYPTO_ALG_TYPE_ABLKCIPHER | CRYPTO_ALG_ASYNC, 1037 .cra_blocksize = AES_BLOCK_SIZE, 1038 .cra_ctxsize = sizeof(struct async_helper_ctx), 1039 .cra_alignmask = 0, 1040 .cra_type = &crypto_ablkcipher_type, 1041 .cra_module = THIS_MODULE, 1042 .cra_init = ablk_ecb_init, 1043 .cra_exit = ablk_exit, 1044 .cra_u = { 1045 .ablkcipher = { 1046 .min_keysize = AES_MIN_KEY_SIZE, 1047 .max_keysize = AES_MAX_KEY_SIZE, 1048 .setkey = ablk_set_key, 1049 .encrypt = ablk_encrypt, 1050 .decrypt = ablk_decrypt, 1051 }, 1052 }, 1053 }, { 1054 .cra_name = "cbc(aes)", 1055 .cra_driver_name = "cbc-aes-aesni", 1056 .cra_priority = 400, 1057 .cra_flags = CRYPTO_ALG_TYPE_ABLKCIPHER | CRYPTO_ALG_ASYNC, 1058 .cra_blocksize = AES_BLOCK_SIZE, 1059 .cra_ctxsize = sizeof(struct async_helper_ctx), 1060 .cra_alignmask = 0, 1061 .cra_type = &crypto_ablkcipher_type, 1062 .cra_module = THIS_MODULE, 1063 .cra_init = ablk_cbc_init, 1064 .cra_exit = ablk_exit, 1065 .cra_u = { 1066 .ablkcipher = { 1067 .min_keysize = AES_MIN_KEY_SIZE, 1068 .max_keysize = AES_MAX_KEY_SIZE, 1069 .ivsize = AES_BLOCK_SIZE, 1070 .setkey = ablk_set_key, 1071 .encrypt = ablk_encrypt, 1072 .decrypt = ablk_decrypt, 1073 }, 1074 }, 1075 #ifdef CONFIG_X86_64 1076 }, { 1077 .cra_name = "__ctr-aes-aesni", 1078 .cra_driver_name = "__driver-ctr-aes-aesni", 1079 .cra_priority = 0, 1080 .cra_flags = CRYPTO_ALG_TYPE_BLKCIPHER, 1081 .cra_blocksize = 1, 1082 .cra_ctxsize = sizeof(struct crypto_aes_ctx) + 1083 AESNI_ALIGN - 1, 1084 .cra_alignmask = 0, 1085 .cra_type = &crypto_blkcipher_type, 1086 .cra_module = THIS_MODULE, 1087 .cra_u = { 1088 .blkcipher = { 1089 .min_keysize = AES_MIN_KEY_SIZE, 1090 .max_keysize = AES_MAX_KEY_SIZE, 1091 .ivsize = AES_BLOCK_SIZE, 1092 .setkey = aes_set_key, 1093 .encrypt = ctr_crypt, 1094 .decrypt = ctr_crypt, 1095 }, 1096 }, 1097 }, { 1098 .cra_name = "ctr(aes)", 1099 .cra_driver_name = "ctr-aes-aesni", 1100 .cra_priority = 400, 1101 .cra_flags = CRYPTO_ALG_TYPE_ABLKCIPHER | CRYPTO_ALG_ASYNC, 1102 .cra_blocksize = 1, 1103 .cra_ctxsize = sizeof(struct async_helper_ctx), 1104 .cra_alignmask = 0, 1105 .cra_type = &crypto_ablkcipher_type, 1106 .cra_module = THIS_MODULE, 1107 .cra_init = ablk_ctr_init, 1108 .cra_exit = ablk_exit, 1109 .cra_u = { 1110 .ablkcipher = { 1111 .min_keysize = AES_MIN_KEY_SIZE, 1112 .max_keysize = AES_MAX_KEY_SIZE, 1113 .ivsize = AES_BLOCK_SIZE, 1114 .setkey = ablk_set_key, 1115 .encrypt = ablk_encrypt, 1116 .decrypt = ablk_encrypt, 1117 .geniv = "chainiv", 1118 }, 1119 }, 1120 }, { 1121 .cra_name = "__gcm-aes-aesni", 1122 .cra_driver_name = "__driver-gcm-aes-aesni", 1123 .cra_priority = 0, 1124 .cra_flags = CRYPTO_ALG_TYPE_AEAD, 1125 .cra_blocksize = 1, 1126 .cra_ctxsize = sizeof(struct aesni_rfc4106_gcm_ctx) + 1127 AESNI_ALIGN, 1128 .cra_alignmask = 0, 1129 .cra_type = &crypto_aead_type, 1130 .cra_module = THIS_MODULE, 1131 .cra_u = { 1132 .aead = { 1133 .encrypt = __driver_rfc4106_encrypt, 1134 .decrypt = __driver_rfc4106_decrypt, 1135 }, 1136 }, 1137 }, { 1138 .cra_name = "rfc4106(gcm(aes))", 1139 .cra_driver_name = "rfc4106-gcm-aesni", 1140 .cra_priority = 400, 1141 .cra_flags = CRYPTO_ALG_TYPE_AEAD | CRYPTO_ALG_ASYNC, 1142 .cra_blocksize = 1, 1143 .cra_ctxsize = sizeof(struct aesni_rfc4106_gcm_ctx) + 1144 AESNI_ALIGN, 1145 .cra_alignmask = 0, 1146 .cra_type = &crypto_nivaead_type, 1147 .cra_module = THIS_MODULE, 1148 .cra_init = rfc4106_init, 1149 .cra_exit = rfc4106_exit, 1150 .cra_u = { 1151 .aead = { 1152 .setkey = rfc4106_set_key, 1153 .setauthsize = rfc4106_set_authsize, 1154 .encrypt = rfc4106_encrypt, 1155 .decrypt = rfc4106_decrypt, 1156 .geniv = "seqiv", 1157 .ivsize = 8, 1158 .maxauthsize = 16, 1159 }, 1160 }, 1161 #ifdef HAS_CTR 1162 }, { 1163 .cra_name = "rfc3686(ctr(aes))", 1164 .cra_driver_name = "rfc3686-ctr-aes-aesni", 1165 .cra_priority = 400, 1166 .cra_flags = CRYPTO_ALG_TYPE_ABLKCIPHER | CRYPTO_ALG_ASYNC, 1167 .cra_blocksize = 1, 1168 .cra_ctxsize = sizeof(struct async_helper_ctx), 1169 .cra_alignmask = 0, 1170 .cra_type = &crypto_ablkcipher_type, 1171 .cra_module = THIS_MODULE, 1172 .cra_init = ablk_rfc3686_ctr_init, 1173 .cra_exit = ablk_exit, 1174 .cra_u = { 1175 .ablkcipher = { 1176 .min_keysize = AES_MIN_KEY_SIZE + 1177 CTR_RFC3686_NONCE_SIZE, 1178 .max_keysize = AES_MAX_KEY_SIZE + 1179 CTR_RFC3686_NONCE_SIZE, 1180 .ivsize = CTR_RFC3686_IV_SIZE, 1181 .setkey = ablk_set_key, 1182 .encrypt = ablk_encrypt, 1183 .decrypt = ablk_decrypt, 1184 .geniv = "seqiv", 1185 }, 1186 }, 1187 #endif 1188 #endif 1189 #ifdef HAS_PCBC 1190 }, { 1191 .cra_name = "pcbc(aes)", 1192 .cra_driver_name = "pcbc-aes-aesni", 1193 .cra_priority = 400, 1194 .cra_flags = CRYPTO_ALG_TYPE_ABLKCIPHER | CRYPTO_ALG_ASYNC, 1195 .cra_blocksize = AES_BLOCK_SIZE, 1196 .cra_ctxsize = sizeof(struct async_helper_ctx), 1197 .cra_alignmask = 0, 1198 .cra_type = &crypto_ablkcipher_type, 1199 .cra_module = THIS_MODULE, 1200 .cra_init = ablk_pcbc_init, 1201 .cra_exit = ablk_exit, 1202 .cra_u = { 1203 .ablkcipher = { 1204 .min_keysize = AES_MIN_KEY_SIZE, 1205 .max_keysize = AES_MAX_KEY_SIZE, 1206 .ivsize = AES_BLOCK_SIZE, 1207 .setkey = ablk_set_key, 1208 .encrypt = ablk_encrypt, 1209 .decrypt = ablk_decrypt, 1210 }, 1211 }, 1212 #endif 1213 }, { 1214 .cra_name = "__lrw-aes-aesni", 1215 .cra_driver_name = "__driver-lrw-aes-aesni", 1216 .cra_priority = 0, 1217 .cra_flags = CRYPTO_ALG_TYPE_BLKCIPHER, 1218 .cra_blocksize = AES_BLOCK_SIZE, 1219 .cra_ctxsize = sizeof(struct aesni_lrw_ctx), 1220 .cra_alignmask = 0, 1221 .cra_type = &crypto_blkcipher_type, 1222 .cra_module = THIS_MODULE, 1223 .cra_exit = lrw_aesni_exit_tfm, 1224 .cra_u = { 1225 .blkcipher = { 1226 .min_keysize = AES_MIN_KEY_SIZE + AES_BLOCK_SIZE, 1227 .max_keysize = AES_MAX_KEY_SIZE + AES_BLOCK_SIZE, 1228 .ivsize = AES_BLOCK_SIZE, 1229 .setkey = lrw_aesni_setkey, 1230 .encrypt = lrw_encrypt, 1231 .decrypt = lrw_decrypt, 1232 }, 1233 }, 1234 }, { 1235 .cra_name = "__xts-aes-aesni", 1236 .cra_driver_name = "__driver-xts-aes-aesni", 1237 .cra_priority = 0, 1238 .cra_flags = CRYPTO_ALG_TYPE_BLKCIPHER, 1239 .cra_blocksize = AES_BLOCK_SIZE, 1240 .cra_ctxsize = sizeof(struct aesni_xts_ctx), 1241 .cra_alignmask = 0, 1242 .cra_type = &crypto_blkcipher_type, 1243 .cra_module = THIS_MODULE, 1244 .cra_u = { 1245 .blkcipher = { 1246 .min_keysize = 2 * AES_MIN_KEY_SIZE, 1247 .max_keysize = 2 * AES_MAX_KEY_SIZE, 1248 .ivsize = AES_BLOCK_SIZE, 1249 .setkey = xts_aesni_setkey, 1250 .encrypt = xts_encrypt, 1251 .decrypt = xts_decrypt, 1252 }, 1253 }, 1254 }, { 1255 .cra_name = "lrw(aes)", 1256 .cra_driver_name = "lrw-aes-aesni", 1257 .cra_priority = 400, 1258 .cra_flags = CRYPTO_ALG_TYPE_ABLKCIPHER | CRYPTO_ALG_ASYNC, 1259 .cra_blocksize = AES_BLOCK_SIZE, 1260 .cra_ctxsize = sizeof(struct async_helper_ctx), 1261 .cra_alignmask = 0, 1262 .cra_type = &crypto_ablkcipher_type, 1263 .cra_module = THIS_MODULE, 1264 .cra_init = ablk_init, 1265 .cra_exit = ablk_exit, 1266 .cra_u = { 1267 .ablkcipher = { 1268 .min_keysize = AES_MIN_KEY_SIZE + AES_BLOCK_SIZE, 1269 .max_keysize = AES_MAX_KEY_SIZE + AES_BLOCK_SIZE, 1270 .ivsize = AES_BLOCK_SIZE, 1271 .setkey = ablk_set_key, 1272 .encrypt = ablk_encrypt, 1273 .decrypt = ablk_decrypt, 1274 }, 1275 }, 1276 }, { 1277 .cra_name = "xts(aes)", 1278 .cra_driver_name = "xts-aes-aesni", 1279 .cra_priority = 400, 1280 .cra_flags = CRYPTO_ALG_TYPE_ABLKCIPHER | CRYPTO_ALG_ASYNC, 1281 .cra_blocksize = AES_BLOCK_SIZE, 1282 .cra_ctxsize = sizeof(struct async_helper_ctx), 1283 .cra_alignmask = 0, 1284 .cra_type = &crypto_ablkcipher_type, 1285 .cra_module = THIS_MODULE, 1286 .cra_init = ablk_init, 1287 .cra_exit = ablk_exit, 1288 .cra_u = { 1289 .ablkcipher = { 1290 .min_keysize = 2 * AES_MIN_KEY_SIZE, 1291 .max_keysize = 2 * AES_MAX_KEY_SIZE, 1292 .ivsize = AES_BLOCK_SIZE, 1293 .setkey = ablk_set_key, 1294 .encrypt = ablk_encrypt, 1295 .decrypt = ablk_decrypt, 1296 }, 1297 }, 1298 } }; 1299 1300 1301 static const struct x86_cpu_id aesni_cpu_id[] = { 1302 X86_FEATURE_MATCH(X86_FEATURE_AES), 1303 {} 1304 }; 1305 MODULE_DEVICE_TABLE(x86cpu, aesni_cpu_id); 1306 1307 static int __init aesni_init(void) 1308 { 1309 int err; 1310 1311 if (!x86_match_cpu(aesni_cpu_id)) 1312 return -ENODEV; 1313 1314 err = crypto_fpu_init(); 1315 if (err) 1316 return err; 1317 1318 return crypto_register_algs(aesni_algs, ARRAY_SIZE(aesni_algs)); 1319 } 1320 1321 static void __exit aesni_exit(void) 1322 { 1323 crypto_unregister_algs(aesni_algs, ARRAY_SIZE(aesni_algs)); 1324 1325 crypto_fpu_exit(); 1326 } 1327 1328 module_init(aesni_init); 1329 module_exit(aesni_exit); 1330 1331 MODULE_DESCRIPTION("Rijndael (AES) Cipher Algorithm, Intel AES-NI instructions optimized"); 1332 MODULE_LICENSE("GPL"); 1333 MODULE_ALIAS("aes"); 1334