1 /* 2 * Support for Intel AES-NI instructions. This file contains glue 3 * code, the real AES implementation is in intel-aes_asm.S. 4 * 5 * Copyright (C) 2008, Intel Corp. 6 * Author: Huang Ying <ying.huang@intel.com> 7 * 8 * Added RFC4106 AES-GCM support for 128-bit keys under the AEAD 9 * interface for 64-bit kernels. 10 * Authors: Adrian Hoban <adrian.hoban@intel.com> 11 * Gabriele Paoloni <gabriele.paoloni@intel.com> 12 * Tadeusz Struk (tadeusz.struk@intel.com) 13 * Aidan O'Mahony (aidan.o.mahony@intel.com) 14 * Copyright (c) 2010, Intel Corporation. 15 * 16 * This program is free software; you can redistribute it and/or modify 17 * it under the terms of the GNU General Public License as published by 18 * the Free Software Foundation; either version 2 of the License, or 19 * (at your option) any later version. 20 */ 21 22 #include <linux/hardirq.h> 23 #include <linux/types.h> 24 #include <linux/crypto.h> 25 #include <linux/module.h> 26 #include <linux/err.h> 27 #include <crypto/algapi.h> 28 #include <crypto/aes.h> 29 #include <crypto/cryptd.h> 30 #include <crypto/ctr.h> 31 #include <crypto/b128ops.h> 32 #include <crypto/lrw.h> 33 #include <crypto/xts.h> 34 #include <asm/cpu_device_id.h> 35 #include <asm/i387.h> 36 #include <asm/crypto/aes.h> 37 #include <asm/crypto/ablk_helper.h> 38 #include <crypto/scatterwalk.h> 39 #include <crypto/internal/aead.h> 40 #include <linux/workqueue.h> 41 #include <linux/spinlock.h> 42 43 #if defined(CONFIG_CRYPTO_PCBC) || defined(CONFIG_CRYPTO_PCBC_MODULE) 44 #define HAS_PCBC 45 #endif 46 47 /* This data is stored at the end of the crypto_tfm struct. 48 * It's a type of per "session" data storage location. 49 * This needs to be 16 byte aligned. 50 */ 51 struct aesni_rfc4106_gcm_ctx { 52 u8 hash_subkey[16]; 53 struct crypto_aes_ctx aes_key_expanded; 54 u8 nonce[4]; 55 struct cryptd_aead *cryptd_tfm; 56 }; 57 58 struct aesni_gcm_set_hash_subkey_result { 59 int err; 60 struct completion completion; 61 }; 62 63 struct aesni_hash_subkey_req_data { 64 u8 iv[16]; 65 struct aesni_gcm_set_hash_subkey_result result; 66 struct scatterlist sg; 67 }; 68 69 #define AESNI_ALIGN (16) 70 #define AES_BLOCK_MASK (~(AES_BLOCK_SIZE-1)) 71 #define RFC4106_HASH_SUBKEY_SIZE 16 72 73 struct aesni_lrw_ctx { 74 struct lrw_table_ctx lrw_table; 75 u8 raw_aes_ctx[sizeof(struct crypto_aes_ctx) + AESNI_ALIGN - 1]; 76 }; 77 78 struct aesni_xts_ctx { 79 u8 raw_tweak_ctx[sizeof(struct crypto_aes_ctx) + AESNI_ALIGN - 1]; 80 u8 raw_crypt_ctx[sizeof(struct crypto_aes_ctx) + AESNI_ALIGN - 1]; 81 }; 82 83 asmlinkage int aesni_set_key(struct crypto_aes_ctx *ctx, const u8 *in_key, 84 unsigned int key_len); 85 asmlinkage void aesni_enc(struct crypto_aes_ctx *ctx, u8 *out, 86 const u8 *in); 87 asmlinkage void aesni_dec(struct crypto_aes_ctx *ctx, u8 *out, 88 const u8 *in); 89 asmlinkage void aesni_ecb_enc(struct crypto_aes_ctx *ctx, u8 *out, 90 const u8 *in, unsigned int len); 91 asmlinkage void aesni_ecb_dec(struct crypto_aes_ctx *ctx, u8 *out, 92 const u8 *in, unsigned int len); 93 asmlinkage void aesni_cbc_enc(struct crypto_aes_ctx *ctx, u8 *out, 94 const u8 *in, unsigned int len, u8 *iv); 95 asmlinkage void aesni_cbc_dec(struct crypto_aes_ctx *ctx, u8 *out, 96 const u8 *in, unsigned int len, u8 *iv); 97 98 int crypto_fpu_init(void); 99 void crypto_fpu_exit(void); 100 101 #ifdef CONFIG_X86_64 102 asmlinkage void aesni_ctr_enc(struct crypto_aes_ctx *ctx, u8 *out, 103 const u8 *in, unsigned int len, u8 *iv); 104 105 /* asmlinkage void aesni_gcm_enc() 106 * void *ctx, AES Key schedule. Starts on a 16 byte boundary. 107 * u8 *out, Ciphertext output. Encrypt in-place is allowed. 108 * const u8 *in, Plaintext input 109 * unsigned long plaintext_len, Length of data in bytes for encryption. 110 * u8 *iv, Pre-counter block j0: 4 byte salt (from Security Association) 111 * concatenated with 8 byte Initialisation Vector (from IPSec ESP 112 * Payload) concatenated with 0x00000001. 16-byte aligned pointer. 113 * u8 *hash_subkey, the Hash sub key input. Data starts on a 16-byte boundary. 114 * const u8 *aad, Additional Authentication Data (AAD) 115 * unsigned long aad_len, Length of AAD in bytes. With RFC4106 this 116 * is going to be 8 or 12 bytes 117 * u8 *auth_tag, Authenticated Tag output. 118 * unsigned long auth_tag_len), Authenticated Tag Length in bytes. 119 * Valid values are 16 (most likely), 12 or 8. 120 */ 121 asmlinkage void aesni_gcm_enc(void *ctx, u8 *out, 122 const u8 *in, unsigned long plaintext_len, u8 *iv, 123 u8 *hash_subkey, const u8 *aad, unsigned long aad_len, 124 u8 *auth_tag, unsigned long auth_tag_len); 125 126 /* asmlinkage void aesni_gcm_dec() 127 * void *ctx, AES Key schedule. Starts on a 16 byte boundary. 128 * u8 *out, Plaintext output. Decrypt in-place is allowed. 129 * const u8 *in, Ciphertext input 130 * unsigned long ciphertext_len, Length of data in bytes for decryption. 131 * u8 *iv, Pre-counter block j0: 4 byte salt (from Security Association) 132 * concatenated with 8 byte Initialisation Vector (from IPSec ESP 133 * Payload) concatenated with 0x00000001. 16-byte aligned pointer. 134 * u8 *hash_subkey, the Hash sub key input. Data starts on a 16-byte boundary. 135 * const u8 *aad, Additional Authentication Data (AAD) 136 * unsigned long aad_len, Length of AAD in bytes. With RFC4106 this is going 137 * to be 8 or 12 bytes 138 * u8 *auth_tag, Authenticated Tag output. 139 * unsigned long auth_tag_len) Authenticated Tag Length in bytes. 140 * Valid values are 16 (most likely), 12 or 8. 141 */ 142 asmlinkage void aesni_gcm_dec(void *ctx, u8 *out, 143 const u8 *in, unsigned long ciphertext_len, u8 *iv, 144 u8 *hash_subkey, const u8 *aad, unsigned long aad_len, 145 u8 *auth_tag, unsigned long auth_tag_len); 146 147 static inline struct 148 aesni_rfc4106_gcm_ctx *aesni_rfc4106_gcm_ctx_get(struct crypto_aead *tfm) 149 { 150 return 151 (struct aesni_rfc4106_gcm_ctx *) 152 PTR_ALIGN((u8 *) 153 crypto_tfm_ctx(crypto_aead_tfm(tfm)), AESNI_ALIGN); 154 } 155 #endif 156 157 static inline struct crypto_aes_ctx *aes_ctx(void *raw_ctx) 158 { 159 unsigned long addr = (unsigned long)raw_ctx; 160 unsigned long align = AESNI_ALIGN; 161 162 if (align <= crypto_tfm_ctx_alignment()) 163 align = 1; 164 return (struct crypto_aes_ctx *)ALIGN(addr, align); 165 } 166 167 static int aes_set_key_common(struct crypto_tfm *tfm, void *raw_ctx, 168 const u8 *in_key, unsigned int key_len) 169 { 170 struct crypto_aes_ctx *ctx = aes_ctx(raw_ctx); 171 u32 *flags = &tfm->crt_flags; 172 int err; 173 174 if (key_len != AES_KEYSIZE_128 && key_len != AES_KEYSIZE_192 && 175 key_len != AES_KEYSIZE_256) { 176 *flags |= CRYPTO_TFM_RES_BAD_KEY_LEN; 177 return -EINVAL; 178 } 179 180 if (!irq_fpu_usable()) 181 err = crypto_aes_expand_key(ctx, in_key, key_len); 182 else { 183 kernel_fpu_begin(); 184 err = aesni_set_key(ctx, in_key, key_len); 185 kernel_fpu_end(); 186 } 187 188 return err; 189 } 190 191 static int aes_set_key(struct crypto_tfm *tfm, const u8 *in_key, 192 unsigned int key_len) 193 { 194 return aes_set_key_common(tfm, crypto_tfm_ctx(tfm), in_key, key_len); 195 } 196 197 static void aes_encrypt(struct crypto_tfm *tfm, u8 *dst, const u8 *src) 198 { 199 struct crypto_aes_ctx *ctx = aes_ctx(crypto_tfm_ctx(tfm)); 200 201 if (!irq_fpu_usable()) 202 crypto_aes_encrypt_x86(ctx, dst, src); 203 else { 204 kernel_fpu_begin(); 205 aesni_enc(ctx, dst, src); 206 kernel_fpu_end(); 207 } 208 } 209 210 static void aes_decrypt(struct crypto_tfm *tfm, u8 *dst, const u8 *src) 211 { 212 struct crypto_aes_ctx *ctx = aes_ctx(crypto_tfm_ctx(tfm)); 213 214 if (!irq_fpu_usable()) 215 crypto_aes_decrypt_x86(ctx, dst, src); 216 else { 217 kernel_fpu_begin(); 218 aesni_dec(ctx, dst, src); 219 kernel_fpu_end(); 220 } 221 } 222 223 static void __aes_encrypt(struct crypto_tfm *tfm, u8 *dst, const u8 *src) 224 { 225 struct crypto_aes_ctx *ctx = aes_ctx(crypto_tfm_ctx(tfm)); 226 227 aesni_enc(ctx, dst, src); 228 } 229 230 static void __aes_decrypt(struct crypto_tfm *tfm, u8 *dst, const u8 *src) 231 { 232 struct crypto_aes_ctx *ctx = aes_ctx(crypto_tfm_ctx(tfm)); 233 234 aesni_dec(ctx, dst, src); 235 } 236 237 static int ecb_encrypt(struct blkcipher_desc *desc, 238 struct scatterlist *dst, struct scatterlist *src, 239 unsigned int nbytes) 240 { 241 struct crypto_aes_ctx *ctx = aes_ctx(crypto_blkcipher_ctx(desc->tfm)); 242 struct blkcipher_walk walk; 243 int err; 244 245 blkcipher_walk_init(&walk, dst, src, nbytes); 246 err = blkcipher_walk_virt(desc, &walk); 247 desc->flags &= ~CRYPTO_TFM_REQ_MAY_SLEEP; 248 249 kernel_fpu_begin(); 250 while ((nbytes = walk.nbytes)) { 251 aesni_ecb_enc(ctx, walk.dst.virt.addr, walk.src.virt.addr, 252 nbytes & AES_BLOCK_MASK); 253 nbytes &= AES_BLOCK_SIZE - 1; 254 err = blkcipher_walk_done(desc, &walk, nbytes); 255 } 256 kernel_fpu_end(); 257 258 return err; 259 } 260 261 static int ecb_decrypt(struct blkcipher_desc *desc, 262 struct scatterlist *dst, struct scatterlist *src, 263 unsigned int nbytes) 264 { 265 struct crypto_aes_ctx *ctx = aes_ctx(crypto_blkcipher_ctx(desc->tfm)); 266 struct blkcipher_walk walk; 267 int err; 268 269 blkcipher_walk_init(&walk, dst, src, nbytes); 270 err = blkcipher_walk_virt(desc, &walk); 271 desc->flags &= ~CRYPTO_TFM_REQ_MAY_SLEEP; 272 273 kernel_fpu_begin(); 274 while ((nbytes = walk.nbytes)) { 275 aesni_ecb_dec(ctx, walk.dst.virt.addr, walk.src.virt.addr, 276 nbytes & AES_BLOCK_MASK); 277 nbytes &= AES_BLOCK_SIZE - 1; 278 err = blkcipher_walk_done(desc, &walk, nbytes); 279 } 280 kernel_fpu_end(); 281 282 return err; 283 } 284 285 static int cbc_encrypt(struct blkcipher_desc *desc, 286 struct scatterlist *dst, struct scatterlist *src, 287 unsigned int nbytes) 288 { 289 struct crypto_aes_ctx *ctx = aes_ctx(crypto_blkcipher_ctx(desc->tfm)); 290 struct blkcipher_walk walk; 291 int err; 292 293 blkcipher_walk_init(&walk, dst, src, nbytes); 294 err = blkcipher_walk_virt(desc, &walk); 295 desc->flags &= ~CRYPTO_TFM_REQ_MAY_SLEEP; 296 297 kernel_fpu_begin(); 298 while ((nbytes = walk.nbytes)) { 299 aesni_cbc_enc(ctx, walk.dst.virt.addr, walk.src.virt.addr, 300 nbytes & AES_BLOCK_MASK, walk.iv); 301 nbytes &= AES_BLOCK_SIZE - 1; 302 err = blkcipher_walk_done(desc, &walk, nbytes); 303 } 304 kernel_fpu_end(); 305 306 return err; 307 } 308 309 static int cbc_decrypt(struct blkcipher_desc *desc, 310 struct scatterlist *dst, struct scatterlist *src, 311 unsigned int nbytes) 312 { 313 struct crypto_aes_ctx *ctx = aes_ctx(crypto_blkcipher_ctx(desc->tfm)); 314 struct blkcipher_walk walk; 315 int err; 316 317 blkcipher_walk_init(&walk, dst, src, nbytes); 318 err = blkcipher_walk_virt(desc, &walk); 319 desc->flags &= ~CRYPTO_TFM_REQ_MAY_SLEEP; 320 321 kernel_fpu_begin(); 322 while ((nbytes = walk.nbytes)) { 323 aesni_cbc_dec(ctx, walk.dst.virt.addr, walk.src.virt.addr, 324 nbytes & AES_BLOCK_MASK, walk.iv); 325 nbytes &= AES_BLOCK_SIZE - 1; 326 err = blkcipher_walk_done(desc, &walk, nbytes); 327 } 328 kernel_fpu_end(); 329 330 return err; 331 } 332 333 #ifdef CONFIG_X86_64 334 static void ctr_crypt_final(struct crypto_aes_ctx *ctx, 335 struct blkcipher_walk *walk) 336 { 337 u8 *ctrblk = walk->iv; 338 u8 keystream[AES_BLOCK_SIZE]; 339 u8 *src = walk->src.virt.addr; 340 u8 *dst = walk->dst.virt.addr; 341 unsigned int nbytes = walk->nbytes; 342 343 aesni_enc(ctx, keystream, ctrblk); 344 crypto_xor(keystream, src, nbytes); 345 memcpy(dst, keystream, nbytes); 346 crypto_inc(ctrblk, AES_BLOCK_SIZE); 347 } 348 349 static int ctr_crypt(struct blkcipher_desc *desc, 350 struct scatterlist *dst, struct scatterlist *src, 351 unsigned int nbytes) 352 { 353 struct crypto_aes_ctx *ctx = aes_ctx(crypto_blkcipher_ctx(desc->tfm)); 354 struct blkcipher_walk walk; 355 int err; 356 357 blkcipher_walk_init(&walk, dst, src, nbytes); 358 err = blkcipher_walk_virt_block(desc, &walk, AES_BLOCK_SIZE); 359 desc->flags &= ~CRYPTO_TFM_REQ_MAY_SLEEP; 360 361 kernel_fpu_begin(); 362 while ((nbytes = walk.nbytes) >= AES_BLOCK_SIZE) { 363 aesni_ctr_enc(ctx, walk.dst.virt.addr, walk.src.virt.addr, 364 nbytes & AES_BLOCK_MASK, walk.iv); 365 nbytes &= AES_BLOCK_SIZE - 1; 366 err = blkcipher_walk_done(desc, &walk, nbytes); 367 } 368 if (walk.nbytes) { 369 ctr_crypt_final(ctx, &walk); 370 err = blkcipher_walk_done(desc, &walk, 0); 371 } 372 kernel_fpu_end(); 373 374 return err; 375 } 376 #endif 377 378 static int ablk_ecb_init(struct crypto_tfm *tfm) 379 { 380 return ablk_init_common(tfm, "__driver-ecb-aes-aesni"); 381 } 382 383 static int ablk_cbc_init(struct crypto_tfm *tfm) 384 { 385 return ablk_init_common(tfm, "__driver-cbc-aes-aesni"); 386 } 387 388 #ifdef CONFIG_X86_64 389 static int ablk_ctr_init(struct crypto_tfm *tfm) 390 { 391 return ablk_init_common(tfm, "__driver-ctr-aes-aesni"); 392 } 393 394 #endif 395 396 #ifdef HAS_PCBC 397 static int ablk_pcbc_init(struct crypto_tfm *tfm) 398 { 399 return ablk_init_common(tfm, "fpu(pcbc(__driver-aes-aesni))"); 400 } 401 #endif 402 403 static void lrw_xts_encrypt_callback(void *ctx, u8 *blks, unsigned int nbytes) 404 { 405 aesni_ecb_enc(ctx, blks, blks, nbytes); 406 } 407 408 static void lrw_xts_decrypt_callback(void *ctx, u8 *blks, unsigned int nbytes) 409 { 410 aesni_ecb_dec(ctx, blks, blks, nbytes); 411 } 412 413 static int lrw_aesni_setkey(struct crypto_tfm *tfm, const u8 *key, 414 unsigned int keylen) 415 { 416 struct aesni_lrw_ctx *ctx = crypto_tfm_ctx(tfm); 417 int err; 418 419 err = aes_set_key_common(tfm, ctx->raw_aes_ctx, key, 420 keylen - AES_BLOCK_SIZE); 421 if (err) 422 return err; 423 424 return lrw_init_table(&ctx->lrw_table, key + keylen - AES_BLOCK_SIZE); 425 } 426 427 static void lrw_aesni_exit_tfm(struct crypto_tfm *tfm) 428 { 429 struct aesni_lrw_ctx *ctx = crypto_tfm_ctx(tfm); 430 431 lrw_free_table(&ctx->lrw_table); 432 } 433 434 static int lrw_encrypt(struct blkcipher_desc *desc, struct scatterlist *dst, 435 struct scatterlist *src, unsigned int nbytes) 436 { 437 struct aesni_lrw_ctx *ctx = crypto_blkcipher_ctx(desc->tfm); 438 be128 buf[8]; 439 struct lrw_crypt_req req = { 440 .tbuf = buf, 441 .tbuflen = sizeof(buf), 442 443 .table_ctx = &ctx->lrw_table, 444 .crypt_ctx = aes_ctx(ctx->raw_aes_ctx), 445 .crypt_fn = lrw_xts_encrypt_callback, 446 }; 447 int ret; 448 449 desc->flags &= ~CRYPTO_TFM_REQ_MAY_SLEEP; 450 451 kernel_fpu_begin(); 452 ret = lrw_crypt(desc, dst, src, nbytes, &req); 453 kernel_fpu_end(); 454 455 return ret; 456 } 457 458 static int lrw_decrypt(struct blkcipher_desc *desc, struct scatterlist *dst, 459 struct scatterlist *src, unsigned int nbytes) 460 { 461 struct aesni_lrw_ctx *ctx = crypto_blkcipher_ctx(desc->tfm); 462 be128 buf[8]; 463 struct lrw_crypt_req req = { 464 .tbuf = buf, 465 .tbuflen = sizeof(buf), 466 467 .table_ctx = &ctx->lrw_table, 468 .crypt_ctx = aes_ctx(ctx->raw_aes_ctx), 469 .crypt_fn = lrw_xts_decrypt_callback, 470 }; 471 int ret; 472 473 desc->flags &= ~CRYPTO_TFM_REQ_MAY_SLEEP; 474 475 kernel_fpu_begin(); 476 ret = lrw_crypt(desc, dst, src, nbytes, &req); 477 kernel_fpu_end(); 478 479 return ret; 480 } 481 482 static int xts_aesni_setkey(struct crypto_tfm *tfm, const u8 *key, 483 unsigned int keylen) 484 { 485 struct aesni_xts_ctx *ctx = crypto_tfm_ctx(tfm); 486 u32 *flags = &tfm->crt_flags; 487 int err; 488 489 /* key consists of keys of equal size concatenated, therefore 490 * the length must be even 491 */ 492 if (keylen % 2) { 493 *flags |= CRYPTO_TFM_RES_BAD_KEY_LEN; 494 return -EINVAL; 495 } 496 497 /* first half of xts-key is for crypt */ 498 err = aes_set_key_common(tfm, ctx->raw_crypt_ctx, key, keylen / 2); 499 if (err) 500 return err; 501 502 /* second half of xts-key is for tweak */ 503 return aes_set_key_common(tfm, ctx->raw_tweak_ctx, key + keylen / 2, 504 keylen / 2); 505 } 506 507 508 static void aesni_xts_tweak(void *ctx, u8 *out, const u8 *in) 509 { 510 aesni_enc(ctx, out, in); 511 } 512 513 static int xts_encrypt(struct blkcipher_desc *desc, struct scatterlist *dst, 514 struct scatterlist *src, unsigned int nbytes) 515 { 516 struct aesni_xts_ctx *ctx = crypto_blkcipher_ctx(desc->tfm); 517 be128 buf[8]; 518 struct xts_crypt_req req = { 519 .tbuf = buf, 520 .tbuflen = sizeof(buf), 521 522 .tweak_ctx = aes_ctx(ctx->raw_tweak_ctx), 523 .tweak_fn = aesni_xts_tweak, 524 .crypt_ctx = aes_ctx(ctx->raw_crypt_ctx), 525 .crypt_fn = lrw_xts_encrypt_callback, 526 }; 527 int ret; 528 529 desc->flags &= ~CRYPTO_TFM_REQ_MAY_SLEEP; 530 531 kernel_fpu_begin(); 532 ret = xts_crypt(desc, dst, src, nbytes, &req); 533 kernel_fpu_end(); 534 535 return ret; 536 } 537 538 static int xts_decrypt(struct blkcipher_desc *desc, struct scatterlist *dst, 539 struct scatterlist *src, unsigned int nbytes) 540 { 541 struct aesni_xts_ctx *ctx = crypto_blkcipher_ctx(desc->tfm); 542 be128 buf[8]; 543 struct xts_crypt_req req = { 544 .tbuf = buf, 545 .tbuflen = sizeof(buf), 546 547 .tweak_ctx = aes_ctx(ctx->raw_tweak_ctx), 548 .tweak_fn = aesni_xts_tweak, 549 .crypt_ctx = aes_ctx(ctx->raw_crypt_ctx), 550 .crypt_fn = lrw_xts_decrypt_callback, 551 }; 552 int ret; 553 554 desc->flags &= ~CRYPTO_TFM_REQ_MAY_SLEEP; 555 556 kernel_fpu_begin(); 557 ret = xts_crypt(desc, dst, src, nbytes, &req); 558 kernel_fpu_end(); 559 560 return ret; 561 } 562 563 #ifdef CONFIG_X86_64 564 static int rfc4106_init(struct crypto_tfm *tfm) 565 { 566 struct cryptd_aead *cryptd_tfm; 567 struct aesni_rfc4106_gcm_ctx *ctx = (struct aesni_rfc4106_gcm_ctx *) 568 PTR_ALIGN((u8 *)crypto_tfm_ctx(tfm), AESNI_ALIGN); 569 struct crypto_aead *cryptd_child; 570 struct aesni_rfc4106_gcm_ctx *child_ctx; 571 cryptd_tfm = cryptd_alloc_aead("__driver-gcm-aes-aesni", 0, 0); 572 if (IS_ERR(cryptd_tfm)) 573 return PTR_ERR(cryptd_tfm); 574 575 cryptd_child = cryptd_aead_child(cryptd_tfm); 576 child_ctx = aesni_rfc4106_gcm_ctx_get(cryptd_child); 577 memcpy(child_ctx, ctx, sizeof(*ctx)); 578 ctx->cryptd_tfm = cryptd_tfm; 579 tfm->crt_aead.reqsize = sizeof(struct aead_request) 580 + crypto_aead_reqsize(&cryptd_tfm->base); 581 return 0; 582 } 583 584 static void rfc4106_exit(struct crypto_tfm *tfm) 585 { 586 struct aesni_rfc4106_gcm_ctx *ctx = 587 (struct aesni_rfc4106_gcm_ctx *) 588 PTR_ALIGN((u8 *)crypto_tfm_ctx(tfm), AESNI_ALIGN); 589 if (!IS_ERR(ctx->cryptd_tfm)) 590 cryptd_free_aead(ctx->cryptd_tfm); 591 return; 592 } 593 594 static void 595 rfc4106_set_hash_subkey_done(struct crypto_async_request *req, int err) 596 { 597 struct aesni_gcm_set_hash_subkey_result *result = req->data; 598 599 if (err == -EINPROGRESS) 600 return; 601 result->err = err; 602 complete(&result->completion); 603 } 604 605 static int 606 rfc4106_set_hash_subkey(u8 *hash_subkey, const u8 *key, unsigned int key_len) 607 { 608 struct crypto_ablkcipher *ctr_tfm; 609 struct ablkcipher_request *req; 610 int ret = -EINVAL; 611 struct aesni_hash_subkey_req_data *req_data; 612 613 ctr_tfm = crypto_alloc_ablkcipher("ctr(aes)", 0, 0); 614 if (IS_ERR(ctr_tfm)) 615 return PTR_ERR(ctr_tfm); 616 617 crypto_ablkcipher_clear_flags(ctr_tfm, ~0); 618 619 ret = crypto_ablkcipher_setkey(ctr_tfm, key, key_len); 620 if (ret) 621 goto out_free_ablkcipher; 622 623 ret = -ENOMEM; 624 req = ablkcipher_request_alloc(ctr_tfm, GFP_KERNEL); 625 if (!req) 626 goto out_free_ablkcipher; 627 628 req_data = kmalloc(sizeof(*req_data), GFP_KERNEL); 629 if (!req_data) 630 goto out_free_request; 631 632 memset(req_data->iv, 0, sizeof(req_data->iv)); 633 634 /* Clear the data in the hash sub key container to zero.*/ 635 /* We want to cipher all zeros to create the hash sub key. */ 636 memset(hash_subkey, 0, RFC4106_HASH_SUBKEY_SIZE); 637 638 init_completion(&req_data->result.completion); 639 sg_init_one(&req_data->sg, hash_subkey, RFC4106_HASH_SUBKEY_SIZE); 640 ablkcipher_request_set_tfm(req, ctr_tfm); 641 ablkcipher_request_set_callback(req, CRYPTO_TFM_REQ_MAY_SLEEP | 642 CRYPTO_TFM_REQ_MAY_BACKLOG, 643 rfc4106_set_hash_subkey_done, 644 &req_data->result); 645 646 ablkcipher_request_set_crypt(req, &req_data->sg, 647 &req_data->sg, RFC4106_HASH_SUBKEY_SIZE, req_data->iv); 648 649 ret = crypto_ablkcipher_encrypt(req); 650 if (ret == -EINPROGRESS || ret == -EBUSY) { 651 ret = wait_for_completion_interruptible 652 (&req_data->result.completion); 653 if (!ret) 654 ret = req_data->result.err; 655 } 656 kfree(req_data); 657 out_free_request: 658 ablkcipher_request_free(req); 659 out_free_ablkcipher: 660 crypto_free_ablkcipher(ctr_tfm); 661 return ret; 662 } 663 664 static int rfc4106_set_key(struct crypto_aead *parent, const u8 *key, 665 unsigned int key_len) 666 { 667 int ret = 0; 668 struct crypto_tfm *tfm = crypto_aead_tfm(parent); 669 struct aesni_rfc4106_gcm_ctx *ctx = aesni_rfc4106_gcm_ctx_get(parent); 670 struct crypto_aead *cryptd_child = cryptd_aead_child(ctx->cryptd_tfm); 671 struct aesni_rfc4106_gcm_ctx *child_ctx = 672 aesni_rfc4106_gcm_ctx_get(cryptd_child); 673 u8 *new_key_align, *new_key_mem = NULL; 674 675 if (key_len < 4) { 676 crypto_tfm_set_flags(tfm, CRYPTO_TFM_RES_BAD_KEY_LEN); 677 return -EINVAL; 678 } 679 /*Account for 4 byte nonce at the end.*/ 680 key_len -= 4; 681 if (key_len != AES_KEYSIZE_128) { 682 crypto_tfm_set_flags(tfm, CRYPTO_TFM_RES_BAD_KEY_LEN); 683 return -EINVAL; 684 } 685 686 memcpy(ctx->nonce, key + key_len, sizeof(ctx->nonce)); 687 /*This must be on a 16 byte boundary!*/ 688 if ((unsigned long)(&(ctx->aes_key_expanded.key_enc[0])) % AESNI_ALIGN) 689 return -EINVAL; 690 691 if ((unsigned long)key % AESNI_ALIGN) { 692 /*key is not aligned: use an auxuliar aligned pointer*/ 693 new_key_mem = kmalloc(key_len+AESNI_ALIGN, GFP_KERNEL); 694 if (!new_key_mem) 695 return -ENOMEM; 696 697 new_key_align = PTR_ALIGN(new_key_mem, AESNI_ALIGN); 698 memcpy(new_key_align, key, key_len); 699 key = new_key_align; 700 } 701 702 if (!irq_fpu_usable()) 703 ret = crypto_aes_expand_key(&(ctx->aes_key_expanded), 704 key, key_len); 705 else { 706 kernel_fpu_begin(); 707 ret = aesni_set_key(&(ctx->aes_key_expanded), key, key_len); 708 kernel_fpu_end(); 709 } 710 /*This must be on a 16 byte boundary!*/ 711 if ((unsigned long)(&(ctx->hash_subkey[0])) % AESNI_ALIGN) { 712 ret = -EINVAL; 713 goto exit; 714 } 715 ret = rfc4106_set_hash_subkey(ctx->hash_subkey, key, key_len); 716 memcpy(child_ctx, ctx, sizeof(*ctx)); 717 exit: 718 kfree(new_key_mem); 719 return ret; 720 } 721 722 /* This is the Integrity Check Value (aka the authentication tag length and can 723 * be 8, 12 or 16 bytes long. */ 724 static int rfc4106_set_authsize(struct crypto_aead *parent, 725 unsigned int authsize) 726 { 727 struct aesni_rfc4106_gcm_ctx *ctx = aesni_rfc4106_gcm_ctx_get(parent); 728 struct crypto_aead *cryptd_child = cryptd_aead_child(ctx->cryptd_tfm); 729 730 switch (authsize) { 731 case 8: 732 case 12: 733 case 16: 734 break; 735 default: 736 return -EINVAL; 737 } 738 crypto_aead_crt(parent)->authsize = authsize; 739 crypto_aead_crt(cryptd_child)->authsize = authsize; 740 return 0; 741 } 742 743 static int rfc4106_encrypt(struct aead_request *req) 744 { 745 int ret; 746 struct crypto_aead *tfm = crypto_aead_reqtfm(req); 747 struct aesni_rfc4106_gcm_ctx *ctx = aesni_rfc4106_gcm_ctx_get(tfm); 748 749 if (!irq_fpu_usable()) { 750 struct aead_request *cryptd_req = 751 (struct aead_request *) aead_request_ctx(req); 752 memcpy(cryptd_req, req, sizeof(*req)); 753 aead_request_set_tfm(cryptd_req, &ctx->cryptd_tfm->base); 754 return crypto_aead_encrypt(cryptd_req); 755 } else { 756 struct crypto_aead *cryptd_child = cryptd_aead_child(ctx->cryptd_tfm); 757 kernel_fpu_begin(); 758 ret = cryptd_child->base.crt_aead.encrypt(req); 759 kernel_fpu_end(); 760 return ret; 761 } 762 } 763 764 static int rfc4106_decrypt(struct aead_request *req) 765 { 766 int ret; 767 struct crypto_aead *tfm = crypto_aead_reqtfm(req); 768 struct aesni_rfc4106_gcm_ctx *ctx = aesni_rfc4106_gcm_ctx_get(tfm); 769 770 if (!irq_fpu_usable()) { 771 struct aead_request *cryptd_req = 772 (struct aead_request *) aead_request_ctx(req); 773 memcpy(cryptd_req, req, sizeof(*req)); 774 aead_request_set_tfm(cryptd_req, &ctx->cryptd_tfm->base); 775 return crypto_aead_decrypt(cryptd_req); 776 } else { 777 struct crypto_aead *cryptd_child = cryptd_aead_child(ctx->cryptd_tfm); 778 kernel_fpu_begin(); 779 ret = cryptd_child->base.crt_aead.decrypt(req); 780 kernel_fpu_end(); 781 return ret; 782 } 783 } 784 785 static int __driver_rfc4106_encrypt(struct aead_request *req) 786 { 787 u8 one_entry_in_sg = 0; 788 u8 *src, *dst, *assoc; 789 __be32 counter = cpu_to_be32(1); 790 struct crypto_aead *tfm = crypto_aead_reqtfm(req); 791 struct aesni_rfc4106_gcm_ctx *ctx = aesni_rfc4106_gcm_ctx_get(tfm); 792 void *aes_ctx = &(ctx->aes_key_expanded); 793 unsigned long auth_tag_len = crypto_aead_authsize(tfm); 794 u8 iv_tab[16+AESNI_ALIGN]; 795 u8* iv = (u8 *) PTR_ALIGN((u8 *)iv_tab, AESNI_ALIGN); 796 struct scatter_walk src_sg_walk; 797 struct scatter_walk assoc_sg_walk; 798 struct scatter_walk dst_sg_walk; 799 unsigned int i; 800 801 /* Assuming we are supporting rfc4106 64-bit extended */ 802 /* sequence numbers We need to have the AAD length equal */ 803 /* to 8 or 12 bytes */ 804 if (unlikely(req->assoclen != 8 && req->assoclen != 12)) 805 return -EINVAL; 806 /* IV below built */ 807 for (i = 0; i < 4; i++) 808 *(iv+i) = ctx->nonce[i]; 809 for (i = 0; i < 8; i++) 810 *(iv+4+i) = req->iv[i]; 811 *((__be32 *)(iv+12)) = counter; 812 813 if ((sg_is_last(req->src)) && (sg_is_last(req->assoc))) { 814 one_entry_in_sg = 1; 815 scatterwalk_start(&src_sg_walk, req->src); 816 scatterwalk_start(&assoc_sg_walk, req->assoc); 817 src = scatterwalk_map(&src_sg_walk); 818 assoc = scatterwalk_map(&assoc_sg_walk); 819 dst = src; 820 if (unlikely(req->src != req->dst)) { 821 scatterwalk_start(&dst_sg_walk, req->dst); 822 dst = scatterwalk_map(&dst_sg_walk); 823 } 824 825 } else { 826 /* Allocate memory for src, dst, assoc */ 827 src = kmalloc(req->cryptlen + auth_tag_len + req->assoclen, 828 GFP_ATOMIC); 829 if (unlikely(!src)) 830 return -ENOMEM; 831 assoc = (src + req->cryptlen + auth_tag_len); 832 scatterwalk_map_and_copy(src, req->src, 0, req->cryptlen, 0); 833 scatterwalk_map_and_copy(assoc, req->assoc, 0, 834 req->assoclen, 0); 835 dst = src; 836 } 837 838 aesni_gcm_enc(aes_ctx, dst, src, (unsigned long)req->cryptlen, iv, 839 ctx->hash_subkey, assoc, (unsigned long)req->assoclen, dst 840 + ((unsigned long)req->cryptlen), auth_tag_len); 841 842 /* The authTag (aka the Integrity Check Value) needs to be written 843 * back to the packet. */ 844 if (one_entry_in_sg) { 845 if (unlikely(req->src != req->dst)) { 846 scatterwalk_unmap(dst); 847 scatterwalk_done(&dst_sg_walk, 0, 0); 848 } 849 scatterwalk_unmap(src); 850 scatterwalk_unmap(assoc); 851 scatterwalk_done(&src_sg_walk, 0, 0); 852 scatterwalk_done(&assoc_sg_walk, 0, 0); 853 } else { 854 scatterwalk_map_and_copy(dst, req->dst, 0, 855 req->cryptlen + auth_tag_len, 1); 856 kfree(src); 857 } 858 return 0; 859 } 860 861 static int __driver_rfc4106_decrypt(struct aead_request *req) 862 { 863 u8 one_entry_in_sg = 0; 864 u8 *src, *dst, *assoc; 865 unsigned long tempCipherLen = 0; 866 __be32 counter = cpu_to_be32(1); 867 int retval = 0; 868 struct crypto_aead *tfm = crypto_aead_reqtfm(req); 869 struct aesni_rfc4106_gcm_ctx *ctx = aesni_rfc4106_gcm_ctx_get(tfm); 870 void *aes_ctx = &(ctx->aes_key_expanded); 871 unsigned long auth_tag_len = crypto_aead_authsize(tfm); 872 u8 iv_and_authTag[32+AESNI_ALIGN]; 873 u8 *iv = (u8 *) PTR_ALIGN((u8 *)iv_and_authTag, AESNI_ALIGN); 874 u8 *authTag = iv + 16; 875 struct scatter_walk src_sg_walk; 876 struct scatter_walk assoc_sg_walk; 877 struct scatter_walk dst_sg_walk; 878 unsigned int i; 879 880 if (unlikely((req->cryptlen < auth_tag_len) || 881 (req->assoclen != 8 && req->assoclen != 12))) 882 return -EINVAL; 883 /* Assuming we are supporting rfc4106 64-bit extended */ 884 /* sequence numbers We need to have the AAD length */ 885 /* equal to 8 or 12 bytes */ 886 887 tempCipherLen = (unsigned long)(req->cryptlen - auth_tag_len); 888 /* IV below built */ 889 for (i = 0; i < 4; i++) 890 *(iv+i) = ctx->nonce[i]; 891 for (i = 0; i < 8; i++) 892 *(iv+4+i) = req->iv[i]; 893 *((__be32 *)(iv+12)) = counter; 894 895 if ((sg_is_last(req->src)) && (sg_is_last(req->assoc))) { 896 one_entry_in_sg = 1; 897 scatterwalk_start(&src_sg_walk, req->src); 898 scatterwalk_start(&assoc_sg_walk, req->assoc); 899 src = scatterwalk_map(&src_sg_walk); 900 assoc = scatterwalk_map(&assoc_sg_walk); 901 dst = src; 902 if (unlikely(req->src != req->dst)) { 903 scatterwalk_start(&dst_sg_walk, req->dst); 904 dst = scatterwalk_map(&dst_sg_walk); 905 } 906 907 } else { 908 /* Allocate memory for src, dst, assoc */ 909 src = kmalloc(req->cryptlen + req->assoclen, GFP_ATOMIC); 910 if (!src) 911 return -ENOMEM; 912 assoc = (src + req->cryptlen + auth_tag_len); 913 scatterwalk_map_and_copy(src, req->src, 0, req->cryptlen, 0); 914 scatterwalk_map_and_copy(assoc, req->assoc, 0, 915 req->assoclen, 0); 916 dst = src; 917 } 918 919 aesni_gcm_dec(aes_ctx, dst, src, tempCipherLen, iv, 920 ctx->hash_subkey, assoc, (unsigned long)req->assoclen, 921 authTag, auth_tag_len); 922 923 /* Compare generated tag with passed in tag. */ 924 retval = memcmp(src + tempCipherLen, authTag, auth_tag_len) ? 925 -EBADMSG : 0; 926 927 if (one_entry_in_sg) { 928 if (unlikely(req->src != req->dst)) { 929 scatterwalk_unmap(dst); 930 scatterwalk_done(&dst_sg_walk, 0, 0); 931 } 932 scatterwalk_unmap(src); 933 scatterwalk_unmap(assoc); 934 scatterwalk_done(&src_sg_walk, 0, 0); 935 scatterwalk_done(&assoc_sg_walk, 0, 0); 936 } else { 937 scatterwalk_map_and_copy(dst, req->dst, 0, req->cryptlen, 1); 938 kfree(src); 939 } 940 return retval; 941 } 942 #endif 943 944 static struct crypto_alg aesni_algs[] = { { 945 .cra_name = "aes", 946 .cra_driver_name = "aes-aesni", 947 .cra_priority = 300, 948 .cra_flags = CRYPTO_ALG_TYPE_CIPHER, 949 .cra_blocksize = AES_BLOCK_SIZE, 950 .cra_ctxsize = sizeof(struct crypto_aes_ctx) + 951 AESNI_ALIGN - 1, 952 .cra_alignmask = 0, 953 .cra_module = THIS_MODULE, 954 .cra_u = { 955 .cipher = { 956 .cia_min_keysize = AES_MIN_KEY_SIZE, 957 .cia_max_keysize = AES_MAX_KEY_SIZE, 958 .cia_setkey = aes_set_key, 959 .cia_encrypt = aes_encrypt, 960 .cia_decrypt = aes_decrypt 961 } 962 } 963 }, { 964 .cra_name = "__aes-aesni", 965 .cra_driver_name = "__driver-aes-aesni", 966 .cra_priority = 0, 967 .cra_flags = CRYPTO_ALG_TYPE_CIPHER, 968 .cra_blocksize = AES_BLOCK_SIZE, 969 .cra_ctxsize = sizeof(struct crypto_aes_ctx) + 970 AESNI_ALIGN - 1, 971 .cra_alignmask = 0, 972 .cra_module = THIS_MODULE, 973 .cra_u = { 974 .cipher = { 975 .cia_min_keysize = AES_MIN_KEY_SIZE, 976 .cia_max_keysize = AES_MAX_KEY_SIZE, 977 .cia_setkey = aes_set_key, 978 .cia_encrypt = __aes_encrypt, 979 .cia_decrypt = __aes_decrypt 980 } 981 } 982 }, { 983 .cra_name = "__ecb-aes-aesni", 984 .cra_driver_name = "__driver-ecb-aes-aesni", 985 .cra_priority = 0, 986 .cra_flags = CRYPTO_ALG_TYPE_BLKCIPHER, 987 .cra_blocksize = AES_BLOCK_SIZE, 988 .cra_ctxsize = sizeof(struct crypto_aes_ctx) + 989 AESNI_ALIGN - 1, 990 .cra_alignmask = 0, 991 .cra_type = &crypto_blkcipher_type, 992 .cra_module = THIS_MODULE, 993 .cra_u = { 994 .blkcipher = { 995 .min_keysize = AES_MIN_KEY_SIZE, 996 .max_keysize = AES_MAX_KEY_SIZE, 997 .setkey = aes_set_key, 998 .encrypt = ecb_encrypt, 999 .decrypt = ecb_decrypt, 1000 }, 1001 }, 1002 }, { 1003 .cra_name = "__cbc-aes-aesni", 1004 .cra_driver_name = "__driver-cbc-aes-aesni", 1005 .cra_priority = 0, 1006 .cra_flags = CRYPTO_ALG_TYPE_BLKCIPHER, 1007 .cra_blocksize = AES_BLOCK_SIZE, 1008 .cra_ctxsize = sizeof(struct crypto_aes_ctx) + 1009 AESNI_ALIGN - 1, 1010 .cra_alignmask = 0, 1011 .cra_type = &crypto_blkcipher_type, 1012 .cra_module = THIS_MODULE, 1013 .cra_u = { 1014 .blkcipher = { 1015 .min_keysize = AES_MIN_KEY_SIZE, 1016 .max_keysize = AES_MAX_KEY_SIZE, 1017 .setkey = aes_set_key, 1018 .encrypt = cbc_encrypt, 1019 .decrypt = cbc_decrypt, 1020 }, 1021 }, 1022 }, { 1023 .cra_name = "ecb(aes)", 1024 .cra_driver_name = "ecb-aes-aesni", 1025 .cra_priority = 400, 1026 .cra_flags = CRYPTO_ALG_TYPE_ABLKCIPHER | CRYPTO_ALG_ASYNC, 1027 .cra_blocksize = AES_BLOCK_SIZE, 1028 .cra_ctxsize = sizeof(struct async_helper_ctx), 1029 .cra_alignmask = 0, 1030 .cra_type = &crypto_ablkcipher_type, 1031 .cra_module = THIS_MODULE, 1032 .cra_init = ablk_ecb_init, 1033 .cra_exit = ablk_exit, 1034 .cra_u = { 1035 .ablkcipher = { 1036 .min_keysize = AES_MIN_KEY_SIZE, 1037 .max_keysize = AES_MAX_KEY_SIZE, 1038 .setkey = ablk_set_key, 1039 .encrypt = ablk_encrypt, 1040 .decrypt = ablk_decrypt, 1041 }, 1042 }, 1043 }, { 1044 .cra_name = "cbc(aes)", 1045 .cra_driver_name = "cbc-aes-aesni", 1046 .cra_priority = 400, 1047 .cra_flags = CRYPTO_ALG_TYPE_ABLKCIPHER | CRYPTO_ALG_ASYNC, 1048 .cra_blocksize = AES_BLOCK_SIZE, 1049 .cra_ctxsize = sizeof(struct async_helper_ctx), 1050 .cra_alignmask = 0, 1051 .cra_type = &crypto_ablkcipher_type, 1052 .cra_module = THIS_MODULE, 1053 .cra_init = ablk_cbc_init, 1054 .cra_exit = ablk_exit, 1055 .cra_u = { 1056 .ablkcipher = { 1057 .min_keysize = AES_MIN_KEY_SIZE, 1058 .max_keysize = AES_MAX_KEY_SIZE, 1059 .ivsize = AES_BLOCK_SIZE, 1060 .setkey = ablk_set_key, 1061 .encrypt = ablk_encrypt, 1062 .decrypt = ablk_decrypt, 1063 }, 1064 }, 1065 #ifdef CONFIG_X86_64 1066 }, { 1067 .cra_name = "__ctr-aes-aesni", 1068 .cra_driver_name = "__driver-ctr-aes-aesni", 1069 .cra_priority = 0, 1070 .cra_flags = CRYPTO_ALG_TYPE_BLKCIPHER, 1071 .cra_blocksize = 1, 1072 .cra_ctxsize = sizeof(struct crypto_aes_ctx) + 1073 AESNI_ALIGN - 1, 1074 .cra_alignmask = 0, 1075 .cra_type = &crypto_blkcipher_type, 1076 .cra_module = THIS_MODULE, 1077 .cra_u = { 1078 .blkcipher = { 1079 .min_keysize = AES_MIN_KEY_SIZE, 1080 .max_keysize = AES_MAX_KEY_SIZE, 1081 .ivsize = AES_BLOCK_SIZE, 1082 .setkey = aes_set_key, 1083 .encrypt = ctr_crypt, 1084 .decrypt = ctr_crypt, 1085 }, 1086 }, 1087 }, { 1088 .cra_name = "ctr(aes)", 1089 .cra_driver_name = "ctr-aes-aesni", 1090 .cra_priority = 400, 1091 .cra_flags = CRYPTO_ALG_TYPE_ABLKCIPHER | CRYPTO_ALG_ASYNC, 1092 .cra_blocksize = 1, 1093 .cra_ctxsize = sizeof(struct async_helper_ctx), 1094 .cra_alignmask = 0, 1095 .cra_type = &crypto_ablkcipher_type, 1096 .cra_module = THIS_MODULE, 1097 .cra_init = ablk_ctr_init, 1098 .cra_exit = ablk_exit, 1099 .cra_u = { 1100 .ablkcipher = { 1101 .min_keysize = AES_MIN_KEY_SIZE, 1102 .max_keysize = AES_MAX_KEY_SIZE, 1103 .ivsize = AES_BLOCK_SIZE, 1104 .setkey = ablk_set_key, 1105 .encrypt = ablk_encrypt, 1106 .decrypt = ablk_encrypt, 1107 .geniv = "chainiv", 1108 }, 1109 }, 1110 }, { 1111 .cra_name = "__gcm-aes-aesni", 1112 .cra_driver_name = "__driver-gcm-aes-aesni", 1113 .cra_priority = 0, 1114 .cra_flags = CRYPTO_ALG_TYPE_AEAD, 1115 .cra_blocksize = 1, 1116 .cra_ctxsize = sizeof(struct aesni_rfc4106_gcm_ctx) + 1117 AESNI_ALIGN, 1118 .cra_alignmask = 0, 1119 .cra_type = &crypto_aead_type, 1120 .cra_module = THIS_MODULE, 1121 .cra_u = { 1122 .aead = { 1123 .encrypt = __driver_rfc4106_encrypt, 1124 .decrypt = __driver_rfc4106_decrypt, 1125 }, 1126 }, 1127 }, { 1128 .cra_name = "rfc4106(gcm(aes))", 1129 .cra_driver_name = "rfc4106-gcm-aesni", 1130 .cra_priority = 400, 1131 .cra_flags = CRYPTO_ALG_TYPE_AEAD | CRYPTO_ALG_ASYNC, 1132 .cra_blocksize = 1, 1133 .cra_ctxsize = sizeof(struct aesni_rfc4106_gcm_ctx) + 1134 AESNI_ALIGN, 1135 .cra_alignmask = 0, 1136 .cra_type = &crypto_nivaead_type, 1137 .cra_module = THIS_MODULE, 1138 .cra_init = rfc4106_init, 1139 .cra_exit = rfc4106_exit, 1140 .cra_u = { 1141 .aead = { 1142 .setkey = rfc4106_set_key, 1143 .setauthsize = rfc4106_set_authsize, 1144 .encrypt = rfc4106_encrypt, 1145 .decrypt = rfc4106_decrypt, 1146 .geniv = "seqiv", 1147 .ivsize = 8, 1148 .maxauthsize = 16, 1149 }, 1150 }, 1151 #endif 1152 #ifdef HAS_PCBC 1153 }, { 1154 .cra_name = "pcbc(aes)", 1155 .cra_driver_name = "pcbc-aes-aesni", 1156 .cra_priority = 400, 1157 .cra_flags = CRYPTO_ALG_TYPE_ABLKCIPHER | CRYPTO_ALG_ASYNC, 1158 .cra_blocksize = AES_BLOCK_SIZE, 1159 .cra_ctxsize = sizeof(struct async_helper_ctx), 1160 .cra_alignmask = 0, 1161 .cra_type = &crypto_ablkcipher_type, 1162 .cra_module = THIS_MODULE, 1163 .cra_init = ablk_pcbc_init, 1164 .cra_exit = ablk_exit, 1165 .cra_u = { 1166 .ablkcipher = { 1167 .min_keysize = AES_MIN_KEY_SIZE, 1168 .max_keysize = AES_MAX_KEY_SIZE, 1169 .ivsize = AES_BLOCK_SIZE, 1170 .setkey = ablk_set_key, 1171 .encrypt = ablk_encrypt, 1172 .decrypt = ablk_decrypt, 1173 }, 1174 }, 1175 #endif 1176 }, { 1177 .cra_name = "__lrw-aes-aesni", 1178 .cra_driver_name = "__driver-lrw-aes-aesni", 1179 .cra_priority = 0, 1180 .cra_flags = CRYPTO_ALG_TYPE_BLKCIPHER, 1181 .cra_blocksize = AES_BLOCK_SIZE, 1182 .cra_ctxsize = sizeof(struct aesni_lrw_ctx), 1183 .cra_alignmask = 0, 1184 .cra_type = &crypto_blkcipher_type, 1185 .cra_module = THIS_MODULE, 1186 .cra_exit = lrw_aesni_exit_tfm, 1187 .cra_u = { 1188 .blkcipher = { 1189 .min_keysize = AES_MIN_KEY_SIZE + AES_BLOCK_SIZE, 1190 .max_keysize = AES_MAX_KEY_SIZE + AES_BLOCK_SIZE, 1191 .ivsize = AES_BLOCK_SIZE, 1192 .setkey = lrw_aesni_setkey, 1193 .encrypt = lrw_encrypt, 1194 .decrypt = lrw_decrypt, 1195 }, 1196 }, 1197 }, { 1198 .cra_name = "__xts-aes-aesni", 1199 .cra_driver_name = "__driver-xts-aes-aesni", 1200 .cra_priority = 0, 1201 .cra_flags = CRYPTO_ALG_TYPE_BLKCIPHER, 1202 .cra_blocksize = AES_BLOCK_SIZE, 1203 .cra_ctxsize = sizeof(struct aesni_xts_ctx), 1204 .cra_alignmask = 0, 1205 .cra_type = &crypto_blkcipher_type, 1206 .cra_module = THIS_MODULE, 1207 .cra_u = { 1208 .blkcipher = { 1209 .min_keysize = 2 * AES_MIN_KEY_SIZE, 1210 .max_keysize = 2 * AES_MAX_KEY_SIZE, 1211 .ivsize = AES_BLOCK_SIZE, 1212 .setkey = xts_aesni_setkey, 1213 .encrypt = xts_encrypt, 1214 .decrypt = xts_decrypt, 1215 }, 1216 }, 1217 }, { 1218 .cra_name = "lrw(aes)", 1219 .cra_driver_name = "lrw-aes-aesni", 1220 .cra_priority = 400, 1221 .cra_flags = CRYPTO_ALG_TYPE_ABLKCIPHER | CRYPTO_ALG_ASYNC, 1222 .cra_blocksize = AES_BLOCK_SIZE, 1223 .cra_ctxsize = sizeof(struct async_helper_ctx), 1224 .cra_alignmask = 0, 1225 .cra_type = &crypto_ablkcipher_type, 1226 .cra_module = THIS_MODULE, 1227 .cra_init = ablk_init, 1228 .cra_exit = ablk_exit, 1229 .cra_u = { 1230 .ablkcipher = { 1231 .min_keysize = AES_MIN_KEY_SIZE + AES_BLOCK_SIZE, 1232 .max_keysize = AES_MAX_KEY_SIZE + AES_BLOCK_SIZE, 1233 .ivsize = AES_BLOCK_SIZE, 1234 .setkey = ablk_set_key, 1235 .encrypt = ablk_encrypt, 1236 .decrypt = ablk_decrypt, 1237 }, 1238 }, 1239 }, { 1240 .cra_name = "xts(aes)", 1241 .cra_driver_name = "xts-aes-aesni", 1242 .cra_priority = 400, 1243 .cra_flags = CRYPTO_ALG_TYPE_ABLKCIPHER | CRYPTO_ALG_ASYNC, 1244 .cra_blocksize = AES_BLOCK_SIZE, 1245 .cra_ctxsize = sizeof(struct async_helper_ctx), 1246 .cra_alignmask = 0, 1247 .cra_type = &crypto_ablkcipher_type, 1248 .cra_module = THIS_MODULE, 1249 .cra_init = ablk_init, 1250 .cra_exit = ablk_exit, 1251 .cra_u = { 1252 .ablkcipher = { 1253 .min_keysize = 2 * AES_MIN_KEY_SIZE, 1254 .max_keysize = 2 * AES_MAX_KEY_SIZE, 1255 .ivsize = AES_BLOCK_SIZE, 1256 .setkey = ablk_set_key, 1257 .encrypt = ablk_encrypt, 1258 .decrypt = ablk_decrypt, 1259 }, 1260 }, 1261 } }; 1262 1263 1264 static const struct x86_cpu_id aesni_cpu_id[] = { 1265 X86_FEATURE_MATCH(X86_FEATURE_AES), 1266 {} 1267 }; 1268 MODULE_DEVICE_TABLE(x86cpu, aesni_cpu_id); 1269 1270 static int __init aesni_init(void) 1271 { 1272 int err; 1273 1274 if (!x86_match_cpu(aesni_cpu_id)) 1275 return -ENODEV; 1276 1277 err = crypto_fpu_init(); 1278 if (err) 1279 return err; 1280 1281 return crypto_register_algs(aesni_algs, ARRAY_SIZE(aesni_algs)); 1282 } 1283 1284 static void __exit aesni_exit(void) 1285 { 1286 crypto_unregister_algs(aesni_algs, ARRAY_SIZE(aesni_algs)); 1287 1288 crypto_fpu_exit(); 1289 } 1290 1291 module_init(aesni_init); 1292 module_exit(aesni_exit); 1293 1294 MODULE_DESCRIPTION("Rijndael (AES) Cipher Algorithm, Intel AES-NI instructions optimized"); 1295 MODULE_LICENSE("GPL"); 1296 MODULE_ALIAS("aes"); 1297