1 /* 2 * Support for Intel AES-NI instructions. This file contains glue 3 * code, the real AES implementation is in intel-aes_asm.S. 4 * 5 * Copyright (C) 2008, Intel Corp. 6 * Author: Huang Ying <ying.huang@intel.com> 7 * 8 * Added RFC4106 AES-GCM support for 128-bit keys under the AEAD 9 * interface for 64-bit kernels. 10 * Authors: Adrian Hoban <adrian.hoban@intel.com> 11 * Gabriele Paoloni <gabriele.paoloni@intel.com> 12 * Tadeusz Struk (tadeusz.struk@intel.com) 13 * Aidan O'Mahony (aidan.o.mahony@intel.com) 14 * Copyright (c) 2010, Intel Corporation. 15 * 16 * This program is free software; you can redistribute it and/or modify 17 * it under the terms of the GNU General Public License as published by 18 * the Free Software Foundation; either version 2 of the License, or 19 * (at your option) any later version. 20 */ 21 22 #include <linux/hardirq.h> 23 #include <linux/types.h> 24 #include <linux/crypto.h> 25 #include <linux/module.h> 26 #include <linux/err.h> 27 #include <crypto/algapi.h> 28 #include <crypto/aes.h> 29 #include <crypto/cryptd.h> 30 #include <crypto/ctr.h> 31 #include <crypto/b128ops.h> 32 #include <crypto/lrw.h> 33 #include <crypto/xts.h> 34 #include <asm/cpu_device_id.h> 35 #include <asm/i387.h> 36 #include <asm/crypto/aes.h> 37 #include <crypto/ablk_helper.h> 38 #include <crypto/scatterwalk.h> 39 #include <crypto/internal/aead.h> 40 #include <linux/workqueue.h> 41 #include <linux/spinlock.h> 42 #ifdef CONFIG_X86_64 43 #include <asm/crypto/glue_helper.h> 44 #endif 45 46 #if defined(CONFIG_CRYPTO_PCBC) || defined(CONFIG_CRYPTO_PCBC_MODULE) 47 #define HAS_PCBC 48 #endif 49 50 /* This data is stored at the end of the crypto_tfm struct. 51 * It's a type of per "session" data storage location. 52 * This needs to be 16 byte aligned. 53 */ 54 struct aesni_rfc4106_gcm_ctx { 55 u8 hash_subkey[16]; 56 struct crypto_aes_ctx aes_key_expanded; 57 u8 nonce[4]; 58 struct cryptd_aead *cryptd_tfm; 59 }; 60 61 struct aesni_gcm_set_hash_subkey_result { 62 int err; 63 struct completion completion; 64 }; 65 66 struct aesni_hash_subkey_req_data { 67 u8 iv[16]; 68 struct aesni_gcm_set_hash_subkey_result result; 69 struct scatterlist sg; 70 }; 71 72 #define AESNI_ALIGN (16) 73 #define AES_BLOCK_MASK (~(AES_BLOCK_SIZE-1)) 74 #define RFC4106_HASH_SUBKEY_SIZE 16 75 76 struct aesni_lrw_ctx { 77 struct lrw_table_ctx lrw_table; 78 u8 raw_aes_ctx[sizeof(struct crypto_aes_ctx) + AESNI_ALIGN - 1]; 79 }; 80 81 struct aesni_xts_ctx { 82 u8 raw_tweak_ctx[sizeof(struct crypto_aes_ctx) + AESNI_ALIGN - 1]; 83 u8 raw_crypt_ctx[sizeof(struct crypto_aes_ctx) + AESNI_ALIGN - 1]; 84 }; 85 86 asmlinkage int aesni_set_key(struct crypto_aes_ctx *ctx, const u8 *in_key, 87 unsigned int key_len); 88 asmlinkage void aesni_enc(struct crypto_aes_ctx *ctx, u8 *out, 89 const u8 *in); 90 asmlinkage void aesni_dec(struct crypto_aes_ctx *ctx, u8 *out, 91 const u8 *in); 92 asmlinkage void aesni_ecb_enc(struct crypto_aes_ctx *ctx, u8 *out, 93 const u8 *in, unsigned int len); 94 asmlinkage void aesni_ecb_dec(struct crypto_aes_ctx *ctx, u8 *out, 95 const u8 *in, unsigned int len); 96 asmlinkage void aesni_cbc_enc(struct crypto_aes_ctx *ctx, u8 *out, 97 const u8 *in, unsigned int len, u8 *iv); 98 asmlinkage void aesni_cbc_dec(struct crypto_aes_ctx *ctx, u8 *out, 99 const u8 *in, unsigned int len, u8 *iv); 100 101 int crypto_fpu_init(void); 102 void crypto_fpu_exit(void); 103 104 #ifdef CONFIG_X86_64 105 asmlinkage void aesni_ctr_enc(struct crypto_aes_ctx *ctx, u8 *out, 106 const u8 *in, unsigned int len, u8 *iv); 107 108 asmlinkage void aesni_xts_crypt8(struct crypto_aes_ctx *ctx, u8 *out, 109 const u8 *in, bool enc, u8 *iv); 110 111 /* asmlinkage void aesni_gcm_enc() 112 * void *ctx, AES Key schedule. Starts on a 16 byte boundary. 113 * u8 *out, Ciphertext output. Encrypt in-place is allowed. 114 * const u8 *in, Plaintext input 115 * unsigned long plaintext_len, Length of data in bytes for encryption. 116 * u8 *iv, Pre-counter block j0: 4 byte salt (from Security Association) 117 * concatenated with 8 byte Initialisation Vector (from IPSec ESP 118 * Payload) concatenated with 0x00000001. 16-byte aligned pointer. 119 * u8 *hash_subkey, the Hash sub key input. Data starts on a 16-byte boundary. 120 * const u8 *aad, Additional Authentication Data (AAD) 121 * unsigned long aad_len, Length of AAD in bytes. With RFC4106 this 122 * is going to be 8 or 12 bytes 123 * u8 *auth_tag, Authenticated Tag output. 124 * unsigned long auth_tag_len), Authenticated Tag Length in bytes. 125 * Valid values are 16 (most likely), 12 or 8. 126 */ 127 asmlinkage void aesni_gcm_enc(void *ctx, u8 *out, 128 const u8 *in, unsigned long plaintext_len, u8 *iv, 129 u8 *hash_subkey, const u8 *aad, unsigned long aad_len, 130 u8 *auth_tag, unsigned long auth_tag_len); 131 132 /* asmlinkage void aesni_gcm_dec() 133 * void *ctx, AES Key schedule. Starts on a 16 byte boundary. 134 * u8 *out, Plaintext output. Decrypt in-place is allowed. 135 * const u8 *in, Ciphertext input 136 * unsigned long ciphertext_len, Length of data in bytes for decryption. 137 * u8 *iv, Pre-counter block j0: 4 byte salt (from Security Association) 138 * concatenated with 8 byte Initialisation Vector (from IPSec ESP 139 * Payload) concatenated with 0x00000001. 16-byte aligned pointer. 140 * u8 *hash_subkey, the Hash sub key input. Data starts on a 16-byte boundary. 141 * const u8 *aad, Additional Authentication Data (AAD) 142 * unsigned long aad_len, Length of AAD in bytes. With RFC4106 this is going 143 * to be 8 or 12 bytes 144 * u8 *auth_tag, Authenticated Tag output. 145 * unsigned long auth_tag_len) Authenticated Tag Length in bytes. 146 * Valid values are 16 (most likely), 12 or 8. 147 */ 148 asmlinkage void aesni_gcm_dec(void *ctx, u8 *out, 149 const u8 *in, unsigned long ciphertext_len, u8 *iv, 150 u8 *hash_subkey, const u8 *aad, unsigned long aad_len, 151 u8 *auth_tag, unsigned long auth_tag_len); 152 153 static inline struct 154 aesni_rfc4106_gcm_ctx *aesni_rfc4106_gcm_ctx_get(struct crypto_aead *tfm) 155 { 156 return 157 (struct aesni_rfc4106_gcm_ctx *) 158 PTR_ALIGN((u8 *) 159 crypto_tfm_ctx(crypto_aead_tfm(tfm)), AESNI_ALIGN); 160 } 161 #endif 162 163 static inline struct crypto_aes_ctx *aes_ctx(void *raw_ctx) 164 { 165 unsigned long addr = (unsigned long)raw_ctx; 166 unsigned long align = AESNI_ALIGN; 167 168 if (align <= crypto_tfm_ctx_alignment()) 169 align = 1; 170 return (struct crypto_aes_ctx *)ALIGN(addr, align); 171 } 172 173 static int aes_set_key_common(struct crypto_tfm *tfm, void *raw_ctx, 174 const u8 *in_key, unsigned int key_len) 175 { 176 struct crypto_aes_ctx *ctx = aes_ctx(raw_ctx); 177 u32 *flags = &tfm->crt_flags; 178 int err; 179 180 if (key_len != AES_KEYSIZE_128 && key_len != AES_KEYSIZE_192 && 181 key_len != AES_KEYSIZE_256) { 182 *flags |= CRYPTO_TFM_RES_BAD_KEY_LEN; 183 return -EINVAL; 184 } 185 186 if (!irq_fpu_usable()) 187 err = crypto_aes_expand_key(ctx, in_key, key_len); 188 else { 189 kernel_fpu_begin(); 190 err = aesni_set_key(ctx, in_key, key_len); 191 kernel_fpu_end(); 192 } 193 194 return err; 195 } 196 197 static int aes_set_key(struct crypto_tfm *tfm, const u8 *in_key, 198 unsigned int key_len) 199 { 200 return aes_set_key_common(tfm, crypto_tfm_ctx(tfm), in_key, key_len); 201 } 202 203 static void aes_encrypt(struct crypto_tfm *tfm, u8 *dst, const u8 *src) 204 { 205 struct crypto_aes_ctx *ctx = aes_ctx(crypto_tfm_ctx(tfm)); 206 207 if (!irq_fpu_usable()) 208 crypto_aes_encrypt_x86(ctx, dst, src); 209 else { 210 kernel_fpu_begin(); 211 aesni_enc(ctx, dst, src); 212 kernel_fpu_end(); 213 } 214 } 215 216 static void aes_decrypt(struct crypto_tfm *tfm, u8 *dst, const u8 *src) 217 { 218 struct crypto_aes_ctx *ctx = aes_ctx(crypto_tfm_ctx(tfm)); 219 220 if (!irq_fpu_usable()) 221 crypto_aes_decrypt_x86(ctx, dst, src); 222 else { 223 kernel_fpu_begin(); 224 aesni_dec(ctx, dst, src); 225 kernel_fpu_end(); 226 } 227 } 228 229 static void __aes_encrypt(struct crypto_tfm *tfm, u8 *dst, const u8 *src) 230 { 231 struct crypto_aes_ctx *ctx = aes_ctx(crypto_tfm_ctx(tfm)); 232 233 aesni_enc(ctx, dst, src); 234 } 235 236 static void __aes_decrypt(struct crypto_tfm *tfm, u8 *dst, const u8 *src) 237 { 238 struct crypto_aes_ctx *ctx = aes_ctx(crypto_tfm_ctx(tfm)); 239 240 aesni_dec(ctx, dst, src); 241 } 242 243 static int ecb_encrypt(struct blkcipher_desc *desc, 244 struct scatterlist *dst, struct scatterlist *src, 245 unsigned int nbytes) 246 { 247 struct crypto_aes_ctx *ctx = aes_ctx(crypto_blkcipher_ctx(desc->tfm)); 248 struct blkcipher_walk walk; 249 int err; 250 251 blkcipher_walk_init(&walk, dst, src, nbytes); 252 err = blkcipher_walk_virt(desc, &walk); 253 desc->flags &= ~CRYPTO_TFM_REQ_MAY_SLEEP; 254 255 kernel_fpu_begin(); 256 while ((nbytes = walk.nbytes)) { 257 aesni_ecb_enc(ctx, walk.dst.virt.addr, walk.src.virt.addr, 258 nbytes & AES_BLOCK_MASK); 259 nbytes &= AES_BLOCK_SIZE - 1; 260 err = blkcipher_walk_done(desc, &walk, nbytes); 261 } 262 kernel_fpu_end(); 263 264 return err; 265 } 266 267 static int ecb_decrypt(struct blkcipher_desc *desc, 268 struct scatterlist *dst, struct scatterlist *src, 269 unsigned int nbytes) 270 { 271 struct crypto_aes_ctx *ctx = aes_ctx(crypto_blkcipher_ctx(desc->tfm)); 272 struct blkcipher_walk walk; 273 int err; 274 275 blkcipher_walk_init(&walk, dst, src, nbytes); 276 err = blkcipher_walk_virt(desc, &walk); 277 desc->flags &= ~CRYPTO_TFM_REQ_MAY_SLEEP; 278 279 kernel_fpu_begin(); 280 while ((nbytes = walk.nbytes)) { 281 aesni_ecb_dec(ctx, walk.dst.virt.addr, walk.src.virt.addr, 282 nbytes & AES_BLOCK_MASK); 283 nbytes &= AES_BLOCK_SIZE - 1; 284 err = blkcipher_walk_done(desc, &walk, nbytes); 285 } 286 kernel_fpu_end(); 287 288 return err; 289 } 290 291 static int cbc_encrypt(struct blkcipher_desc *desc, 292 struct scatterlist *dst, struct scatterlist *src, 293 unsigned int nbytes) 294 { 295 struct crypto_aes_ctx *ctx = aes_ctx(crypto_blkcipher_ctx(desc->tfm)); 296 struct blkcipher_walk walk; 297 int err; 298 299 blkcipher_walk_init(&walk, dst, src, nbytes); 300 err = blkcipher_walk_virt(desc, &walk); 301 desc->flags &= ~CRYPTO_TFM_REQ_MAY_SLEEP; 302 303 kernel_fpu_begin(); 304 while ((nbytes = walk.nbytes)) { 305 aesni_cbc_enc(ctx, walk.dst.virt.addr, walk.src.virt.addr, 306 nbytes & AES_BLOCK_MASK, walk.iv); 307 nbytes &= AES_BLOCK_SIZE - 1; 308 err = blkcipher_walk_done(desc, &walk, nbytes); 309 } 310 kernel_fpu_end(); 311 312 return err; 313 } 314 315 static int cbc_decrypt(struct blkcipher_desc *desc, 316 struct scatterlist *dst, struct scatterlist *src, 317 unsigned int nbytes) 318 { 319 struct crypto_aes_ctx *ctx = aes_ctx(crypto_blkcipher_ctx(desc->tfm)); 320 struct blkcipher_walk walk; 321 int err; 322 323 blkcipher_walk_init(&walk, dst, src, nbytes); 324 err = blkcipher_walk_virt(desc, &walk); 325 desc->flags &= ~CRYPTO_TFM_REQ_MAY_SLEEP; 326 327 kernel_fpu_begin(); 328 while ((nbytes = walk.nbytes)) { 329 aesni_cbc_dec(ctx, walk.dst.virt.addr, walk.src.virt.addr, 330 nbytes & AES_BLOCK_MASK, walk.iv); 331 nbytes &= AES_BLOCK_SIZE - 1; 332 err = blkcipher_walk_done(desc, &walk, nbytes); 333 } 334 kernel_fpu_end(); 335 336 return err; 337 } 338 339 #ifdef CONFIG_X86_64 340 static void ctr_crypt_final(struct crypto_aes_ctx *ctx, 341 struct blkcipher_walk *walk) 342 { 343 u8 *ctrblk = walk->iv; 344 u8 keystream[AES_BLOCK_SIZE]; 345 u8 *src = walk->src.virt.addr; 346 u8 *dst = walk->dst.virt.addr; 347 unsigned int nbytes = walk->nbytes; 348 349 aesni_enc(ctx, keystream, ctrblk); 350 crypto_xor(keystream, src, nbytes); 351 memcpy(dst, keystream, nbytes); 352 crypto_inc(ctrblk, AES_BLOCK_SIZE); 353 } 354 355 static int ctr_crypt(struct blkcipher_desc *desc, 356 struct scatterlist *dst, struct scatterlist *src, 357 unsigned int nbytes) 358 { 359 struct crypto_aes_ctx *ctx = aes_ctx(crypto_blkcipher_ctx(desc->tfm)); 360 struct blkcipher_walk walk; 361 int err; 362 363 blkcipher_walk_init(&walk, dst, src, nbytes); 364 err = blkcipher_walk_virt_block(desc, &walk, AES_BLOCK_SIZE); 365 desc->flags &= ~CRYPTO_TFM_REQ_MAY_SLEEP; 366 367 kernel_fpu_begin(); 368 while ((nbytes = walk.nbytes) >= AES_BLOCK_SIZE) { 369 aesni_ctr_enc(ctx, walk.dst.virt.addr, walk.src.virt.addr, 370 nbytes & AES_BLOCK_MASK, walk.iv); 371 nbytes &= AES_BLOCK_SIZE - 1; 372 err = blkcipher_walk_done(desc, &walk, nbytes); 373 } 374 if (walk.nbytes) { 375 ctr_crypt_final(ctx, &walk); 376 err = blkcipher_walk_done(desc, &walk, 0); 377 } 378 kernel_fpu_end(); 379 380 return err; 381 } 382 #endif 383 384 static int ablk_ecb_init(struct crypto_tfm *tfm) 385 { 386 return ablk_init_common(tfm, "__driver-ecb-aes-aesni"); 387 } 388 389 static int ablk_cbc_init(struct crypto_tfm *tfm) 390 { 391 return ablk_init_common(tfm, "__driver-cbc-aes-aesni"); 392 } 393 394 #ifdef CONFIG_X86_64 395 static int ablk_ctr_init(struct crypto_tfm *tfm) 396 { 397 return ablk_init_common(tfm, "__driver-ctr-aes-aesni"); 398 } 399 400 #endif 401 402 #ifdef HAS_PCBC 403 static int ablk_pcbc_init(struct crypto_tfm *tfm) 404 { 405 return ablk_init_common(tfm, "fpu(pcbc(__driver-aes-aesni))"); 406 } 407 #endif 408 409 static void lrw_xts_encrypt_callback(void *ctx, u8 *blks, unsigned int nbytes) 410 { 411 aesni_ecb_enc(ctx, blks, blks, nbytes); 412 } 413 414 static void lrw_xts_decrypt_callback(void *ctx, u8 *blks, unsigned int nbytes) 415 { 416 aesni_ecb_dec(ctx, blks, blks, nbytes); 417 } 418 419 static int lrw_aesni_setkey(struct crypto_tfm *tfm, const u8 *key, 420 unsigned int keylen) 421 { 422 struct aesni_lrw_ctx *ctx = crypto_tfm_ctx(tfm); 423 int err; 424 425 err = aes_set_key_common(tfm, ctx->raw_aes_ctx, key, 426 keylen - AES_BLOCK_SIZE); 427 if (err) 428 return err; 429 430 return lrw_init_table(&ctx->lrw_table, key + keylen - AES_BLOCK_SIZE); 431 } 432 433 static void lrw_aesni_exit_tfm(struct crypto_tfm *tfm) 434 { 435 struct aesni_lrw_ctx *ctx = crypto_tfm_ctx(tfm); 436 437 lrw_free_table(&ctx->lrw_table); 438 } 439 440 static int lrw_encrypt(struct blkcipher_desc *desc, struct scatterlist *dst, 441 struct scatterlist *src, unsigned int nbytes) 442 { 443 struct aesni_lrw_ctx *ctx = crypto_blkcipher_ctx(desc->tfm); 444 be128 buf[8]; 445 struct lrw_crypt_req req = { 446 .tbuf = buf, 447 .tbuflen = sizeof(buf), 448 449 .table_ctx = &ctx->lrw_table, 450 .crypt_ctx = aes_ctx(ctx->raw_aes_ctx), 451 .crypt_fn = lrw_xts_encrypt_callback, 452 }; 453 int ret; 454 455 desc->flags &= ~CRYPTO_TFM_REQ_MAY_SLEEP; 456 457 kernel_fpu_begin(); 458 ret = lrw_crypt(desc, dst, src, nbytes, &req); 459 kernel_fpu_end(); 460 461 return ret; 462 } 463 464 static int lrw_decrypt(struct blkcipher_desc *desc, struct scatterlist *dst, 465 struct scatterlist *src, unsigned int nbytes) 466 { 467 struct aesni_lrw_ctx *ctx = crypto_blkcipher_ctx(desc->tfm); 468 be128 buf[8]; 469 struct lrw_crypt_req req = { 470 .tbuf = buf, 471 .tbuflen = sizeof(buf), 472 473 .table_ctx = &ctx->lrw_table, 474 .crypt_ctx = aes_ctx(ctx->raw_aes_ctx), 475 .crypt_fn = lrw_xts_decrypt_callback, 476 }; 477 int ret; 478 479 desc->flags &= ~CRYPTO_TFM_REQ_MAY_SLEEP; 480 481 kernel_fpu_begin(); 482 ret = lrw_crypt(desc, dst, src, nbytes, &req); 483 kernel_fpu_end(); 484 485 return ret; 486 } 487 488 static int xts_aesni_setkey(struct crypto_tfm *tfm, const u8 *key, 489 unsigned int keylen) 490 { 491 struct aesni_xts_ctx *ctx = crypto_tfm_ctx(tfm); 492 u32 *flags = &tfm->crt_flags; 493 int err; 494 495 /* key consists of keys of equal size concatenated, therefore 496 * the length must be even 497 */ 498 if (keylen % 2) { 499 *flags |= CRYPTO_TFM_RES_BAD_KEY_LEN; 500 return -EINVAL; 501 } 502 503 /* first half of xts-key is for crypt */ 504 err = aes_set_key_common(tfm, ctx->raw_crypt_ctx, key, keylen / 2); 505 if (err) 506 return err; 507 508 /* second half of xts-key is for tweak */ 509 return aes_set_key_common(tfm, ctx->raw_tweak_ctx, key + keylen / 2, 510 keylen / 2); 511 } 512 513 514 static void aesni_xts_tweak(void *ctx, u8 *out, const u8 *in) 515 { 516 aesni_enc(ctx, out, in); 517 } 518 519 #ifdef CONFIG_X86_64 520 521 static void aesni_xts_enc(void *ctx, u128 *dst, const u128 *src, le128 *iv) 522 { 523 glue_xts_crypt_128bit_one(ctx, dst, src, iv, GLUE_FUNC_CAST(aesni_enc)); 524 } 525 526 static void aesni_xts_dec(void *ctx, u128 *dst, const u128 *src, le128 *iv) 527 { 528 glue_xts_crypt_128bit_one(ctx, dst, src, iv, GLUE_FUNC_CAST(aesni_dec)); 529 } 530 531 static void aesni_xts_enc8(void *ctx, u128 *dst, const u128 *src, le128 *iv) 532 { 533 aesni_xts_crypt8(ctx, (u8 *)dst, (const u8 *)src, true, (u8 *)iv); 534 } 535 536 static void aesni_xts_dec8(void *ctx, u128 *dst, const u128 *src, le128 *iv) 537 { 538 aesni_xts_crypt8(ctx, (u8 *)dst, (const u8 *)src, false, (u8 *)iv); 539 } 540 541 static const struct common_glue_ctx aesni_enc_xts = { 542 .num_funcs = 2, 543 .fpu_blocks_limit = 1, 544 545 .funcs = { { 546 .num_blocks = 8, 547 .fn_u = { .xts = GLUE_XTS_FUNC_CAST(aesni_xts_enc8) } 548 }, { 549 .num_blocks = 1, 550 .fn_u = { .xts = GLUE_XTS_FUNC_CAST(aesni_xts_enc) } 551 } } 552 }; 553 554 static const struct common_glue_ctx aesni_dec_xts = { 555 .num_funcs = 2, 556 .fpu_blocks_limit = 1, 557 558 .funcs = { { 559 .num_blocks = 8, 560 .fn_u = { .xts = GLUE_XTS_FUNC_CAST(aesni_xts_dec8) } 561 }, { 562 .num_blocks = 1, 563 .fn_u = { .xts = GLUE_XTS_FUNC_CAST(aesni_xts_dec) } 564 } } 565 }; 566 567 static int xts_encrypt(struct blkcipher_desc *desc, struct scatterlist *dst, 568 struct scatterlist *src, unsigned int nbytes) 569 { 570 struct aesni_xts_ctx *ctx = crypto_blkcipher_ctx(desc->tfm); 571 572 return glue_xts_crypt_128bit(&aesni_enc_xts, desc, dst, src, nbytes, 573 XTS_TWEAK_CAST(aesni_xts_tweak), 574 aes_ctx(ctx->raw_tweak_ctx), 575 aes_ctx(ctx->raw_crypt_ctx)); 576 } 577 578 static int xts_decrypt(struct blkcipher_desc *desc, struct scatterlist *dst, 579 struct scatterlist *src, unsigned int nbytes) 580 { 581 struct aesni_xts_ctx *ctx = crypto_blkcipher_ctx(desc->tfm); 582 583 return glue_xts_crypt_128bit(&aesni_dec_xts, desc, dst, src, nbytes, 584 XTS_TWEAK_CAST(aesni_xts_tweak), 585 aes_ctx(ctx->raw_tweak_ctx), 586 aes_ctx(ctx->raw_crypt_ctx)); 587 } 588 589 #else 590 591 static int xts_encrypt(struct blkcipher_desc *desc, struct scatterlist *dst, 592 struct scatterlist *src, unsigned int nbytes) 593 { 594 struct aesni_xts_ctx *ctx = crypto_blkcipher_ctx(desc->tfm); 595 be128 buf[8]; 596 struct xts_crypt_req req = { 597 .tbuf = buf, 598 .tbuflen = sizeof(buf), 599 600 .tweak_ctx = aes_ctx(ctx->raw_tweak_ctx), 601 .tweak_fn = aesni_xts_tweak, 602 .crypt_ctx = aes_ctx(ctx->raw_crypt_ctx), 603 .crypt_fn = lrw_xts_encrypt_callback, 604 }; 605 int ret; 606 607 desc->flags &= ~CRYPTO_TFM_REQ_MAY_SLEEP; 608 609 kernel_fpu_begin(); 610 ret = xts_crypt(desc, dst, src, nbytes, &req); 611 kernel_fpu_end(); 612 613 return ret; 614 } 615 616 static int xts_decrypt(struct blkcipher_desc *desc, struct scatterlist *dst, 617 struct scatterlist *src, unsigned int nbytes) 618 { 619 struct aesni_xts_ctx *ctx = crypto_blkcipher_ctx(desc->tfm); 620 be128 buf[8]; 621 struct xts_crypt_req req = { 622 .tbuf = buf, 623 .tbuflen = sizeof(buf), 624 625 .tweak_ctx = aes_ctx(ctx->raw_tweak_ctx), 626 .tweak_fn = aesni_xts_tweak, 627 .crypt_ctx = aes_ctx(ctx->raw_crypt_ctx), 628 .crypt_fn = lrw_xts_decrypt_callback, 629 }; 630 int ret; 631 632 desc->flags &= ~CRYPTO_TFM_REQ_MAY_SLEEP; 633 634 kernel_fpu_begin(); 635 ret = xts_crypt(desc, dst, src, nbytes, &req); 636 kernel_fpu_end(); 637 638 return ret; 639 } 640 641 #endif 642 643 #ifdef CONFIG_X86_64 644 static int rfc4106_init(struct crypto_tfm *tfm) 645 { 646 struct cryptd_aead *cryptd_tfm; 647 struct aesni_rfc4106_gcm_ctx *ctx = (struct aesni_rfc4106_gcm_ctx *) 648 PTR_ALIGN((u8 *)crypto_tfm_ctx(tfm), AESNI_ALIGN); 649 struct crypto_aead *cryptd_child; 650 struct aesni_rfc4106_gcm_ctx *child_ctx; 651 cryptd_tfm = cryptd_alloc_aead("__driver-gcm-aes-aesni", 0, 0); 652 if (IS_ERR(cryptd_tfm)) 653 return PTR_ERR(cryptd_tfm); 654 655 cryptd_child = cryptd_aead_child(cryptd_tfm); 656 child_ctx = aesni_rfc4106_gcm_ctx_get(cryptd_child); 657 memcpy(child_ctx, ctx, sizeof(*ctx)); 658 ctx->cryptd_tfm = cryptd_tfm; 659 tfm->crt_aead.reqsize = sizeof(struct aead_request) 660 + crypto_aead_reqsize(&cryptd_tfm->base); 661 return 0; 662 } 663 664 static void rfc4106_exit(struct crypto_tfm *tfm) 665 { 666 struct aesni_rfc4106_gcm_ctx *ctx = 667 (struct aesni_rfc4106_gcm_ctx *) 668 PTR_ALIGN((u8 *)crypto_tfm_ctx(tfm), AESNI_ALIGN); 669 if (!IS_ERR(ctx->cryptd_tfm)) 670 cryptd_free_aead(ctx->cryptd_tfm); 671 return; 672 } 673 674 static void 675 rfc4106_set_hash_subkey_done(struct crypto_async_request *req, int err) 676 { 677 struct aesni_gcm_set_hash_subkey_result *result = req->data; 678 679 if (err == -EINPROGRESS) 680 return; 681 result->err = err; 682 complete(&result->completion); 683 } 684 685 static int 686 rfc4106_set_hash_subkey(u8 *hash_subkey, const u8 *key, unsigned int key_len) 687 { 688 struct crypto_ablkcipher *ctr_tfm; 689 struct ablkcipher_request *req; 690 int ret = -EINVAL; 691 struct aesni_hash_subkey_req_data *req_data; 692 693 ctr_tfm = crypto_alloc_ablkcipher("ctr(aes)", 0, 0); 694 if (IS_ERR(ctr_tfm)) 695 return PTR_ERR(ctr_tfm); 696 697 crypto_ablkcipher_clear_flags(ctr_tfm, ~0); 698 699 ret = crypto_ablkcipher_setkey(ctr_tfm, key, key_len); 700 if (ret) 701 goto out_free_ablkcipher; 702 703 ret = -ENOMEM; 704 req = ablkcipher_request_alloc(ctr_tfm, GFP_KERNEL); 705 if (!req) 706 goto out_free_ablkcipher; 707 708 req_data = kmalloc(sizeof(*req_data), GFP_KERNEL); 709 if (!req_data) 710 goto out_free_request; 711 712 memset(req_data->iv, 0, sizeof(req_data->iv)); 713 714 /* Clear the data in the hash sub key container to zero.*/ 715 /* We want to cipher all zeros to create the hash sub key. */ 716 memset(hash_subkey, 0, RFC4106_HASH_SUBKEY_SIZE); 717 718 init_completion(&req_data->result.completion); 719 sg_init_one(&req_data->sg, hash_subkey, RFC4106_HASH_SUBKEY_SIZE); 720 ablkcipher_request_set_tfm(req, ctr_tfm); 721 ablkcipher_request_set_callback(req, CRYPTO_TFM_REQ_MAY_SLEEP | 722 CRYPTO_TFM_REQ_MAY_BACKLOG, 723 rfc4106_set_hash_subkey_done, 724 &req_data->result); 725 726 ablkcipher_request_set_crypt(req, &req_data->sg, 727 &req_data->sg, RFC4106_HASH_SUBKEY_SIZE, req_data->iv); 728 729 ret = crypto_ablkcipher_encrypt(req); 730 if (ret == -EINPROGRESS || ret == -EBUSY) { 731 ret = wait_for_completion_interruptible 732 (&req_data->result.completion); 733 if (!ret) 734 ret = req_data->result.err; 735 } 736 kfree(req_data); 737 out_free_request: 738 ablkcipher_request_free(req); 739 out_free_ablkcipher: 740 crypto_free_ablkcipher(ctr_tfm); 741 return ret; 742 } 743 744 static int rfc4106_set_key(struct crypto_aead *parent, const u8 *key, 745 unsigned int key_len) 746 { 747 int ret = 0; 748 struct crypto_tfm *tfm = crypto_aead_tfm(parent); 749 struct aesni_rfc4106_gcm_ctx *ctx = aesni_rfc4106_gcm_ctx_get(parent); 750 struct crypto_aead *cryptd_child = cryptd_aead_child(ctx->cryptd_tfm); 751 struct aesni_rfc4106_gcm_ctx *child_ctx = 752 aesni_rfc4106_gcm_ctx_get(cryptd_child); 753 u8 *new_key_align, *new_key_mem = NULL; 754 755 if (key_len < 4) { 756 crypto_tfm_set_flags(tfm, CRYPTO_TFM_RES_BAD_KEY_LEN); 757 return -EINVAL; 758 } 759 /*Account for 4 byte nonce at the end.*/ 760 key_len -= 4; 761 if (key_len != AES_KEYSIZE_128) { 762 crypto_tfm_set_flags(tfm, CRYPTO_TFM_RES_BAD_KEY_LEN); 763 return -EINVAL; 764 } 765 766 memcpy(ctx->nonce, key + key_len, sizeof(ctx->nonce)); 767 /*This must be on a 16 byte boundary!*/ 768 if ((unsigned long)(&(ctx->aes_key_expanded.key_enc[0])) % AESNI_ALIGN) 769 return -EINVAL; 770 771 if ((unsigned long)key % AESNI_ALIGN) { 772 /*key is not aligned: use an auxuliar aligned pointer*/ 773 new_key_mem = kmalloc(key_len+AESNI_ALIGN, GFP_KERNEL); 774 if (!new_key_mem) 775 return -ENOMEM; 776 777 new_key_align = PTR_ALIGN(new_key_mem, AESNI_ALIGN); 778 memcpy(new_key_align, key, key_len); 779 key = new_key_align; 780 } 781 782 if (!irq_fpu_usable()) 783 ret = crypto_aes_expand_key(&(ctx->aes_key_expanded), 784 key, key_len); 785 else { 786 kernel_fpu_begin(); 787 ret = aesni_set_key(&(ctx->aes_key_expanded), key, key_len); 788 kernel_fpu_end(); 789 } 790 /*This must be on a 16 byte boundary!*/ 791 if ((unsigned long)(&(ctx->hash_subkey[0])) % AESNI_ALIGN) { 792 ret = -EINVAL; 793 goto exit; 794 } 795 ret = rfc4106_set_hash_subkey(ctx->hash_subkey, key, key_len); 796 memcpy(child_ctx, ctx, sizeof(*ctx)); 797 exit: 798 kfree(new_key_mem); 799 return ret; 800 } 801 802 /* This is the Integrity Check Value (aka the authentication tag length and can 803 * be 8, 12 or 16 bytes long. */ 804 static int rfc4106_set_authsize(struct crypto_aead *parent, 805 unsigned int authsize) 806 { 807 struct aesni_rfc4106_gcm_ctx *ctx = aesni_rfc4106_gcm_ctx_get(parent); 808 struct crypto_aead *cryptd_child = cryptd_aead_child(ctx->cryptd_tfm); 809 810 switch (authsize) { 811 case 8: 812 case 12: 813 case 16: 814 break; 815 default: 816 return -EINVAL; 817 } 818 crypto_aead_crt(parent)->authsize = authsize; 819 crypto_aead_crt(cryptd_child)->authsize = authsize; 820 return 0; 821 } 822 823 static int rfc4106_encrypt(struct aead_request *req) 824 { 825 int ret; 826 struct crypto_aead *tfm = crypto_aead_reqtfm(req); 827 struct aesni_rfc4106_gcm_ctx *ctx = aesni_rfc4106_gcm_ctx_get(tfm); 828 829 if (!irq_fpu_usable()) { 830 struct aead_request *cryptd_req = 831 (struct aead_request *) aead_request_ctx(req); 832 memcpy(cryptd_req, req, sizeof(*req)); 833 aead_request_set_tfm(cryptd_req, &ctx->cryptd_tfm->base); 834 return crypto_aead_encrypt(cryptd_req); 835 } else { 836 struct crypto_aead *cryptd_child = cryptd_aead_child(ctx->cryptd_tfm); 837 kernel_fpu_begin(); 838 ret = cryptd_child->base.crt_aead.encrypt(req); 839 kernel_fpu_end(); 840 return ret; 841 } 842 } 843 844 static int rfc4106_decrypt(struct aead_request *req) 845 { 846 int ret; 847 struct crypto_aead *tfm = crypto_aead_reqtfm(req); 848 struct aesni_rfc4106_gcm_ctx *ctx = aesni_rfc4106_gcm_ctx_get(tfm); 849 850 if (!irq_fpu_usable()) { 851 struct aead_request *cryptd_req = 852 (struct aead_request *) aead_request_ctx(req); 853 memcpy(cryptd_req, req, sizeof(*req)); 854 aead_request_set_tfm(cryptd_req, &ctx->cryptd_tfm->base); 855 return crypto_aead_decrypt(cryptd_req); 856 } else { 857 struct crypto_aead *cryptd_child = cryptd_aead_child(ctx->cryptd_tfm); 858 kernel_fpu_begin(); 859 ret = cryptd_child->base.crt_aead.decrypt(req); 860 kernel_fpu_end(); 861 return ret; 862 } 863 } 864 865 static int __driver_rfc4106_encrypt(struct aead_request *req) 866 { 867 u8 one_entry_in_sg = 0; 868 u8 *src, *dst, *assoc; 869 __be32 counter = cpu_to_be32(1); 870 struct crypto_aead *tfm = crypto_aead_reqtfm(req); 871 struct aesni_rfc4106_gcm_ctx *ctx = aesni_rfc4106_gcm_ctx_get(tfm); 872 void *aes_ctx = &(ctx->aes_key_expanded); 873 unsigned long auth_tag_len = crypto_aead_authsize(tfm); 874 u8 iv_tab[16+AESNI_ALIGN]; 875 u8* iv = (u8 *) PTR_ALIGN((u8 *)iv_tab, AESNI_ALIGN); 876 struct scatter_walk src_sg_walk; 877 struct scatter_walk assoc_sg_walk; 878 struct scatter_walk dst_sg_walk; 879 unsigned int i; 880 881 /* Assuming we are supporting rfc4106 64-bit extended */ 882 /* sequence numbers We need to have the AAD length equal */ 883 /* to 8 or 12 bytes */ 884 if (unlikely(req->assoclen != 8 && req->assoclen != 12)) 885 return -EINVAL; 886 /* IV below built */ 887 for (i = 0; i < 4; i++) 888 *(iv+i) = ctx->nonce[i]; 889 for (i = 0; i < 8; i++) 890 *(iv+4+i) = req->iv[i]; 891 *((__be32 *)(iv+12)) = counter; 892 893 if ((sg_is_last(req->src)) && (sg_is_last(req->assoc))) { 894 one_entry_in_sg = 1; 895 scatterwalk_start(&src_sg_walk, req->src); 896 scatterwalk_start(&assoc_sg_walk, req->assoc); 897 src = scatterwalk_map(&src_sg_walk); 898 assoc = scatterwalk_map(&assoc_sg_walk); 899 dst = src; 900 if (unlikely(req->src != req->dst)) { 901 scatterwalk_start(&dst_sg_walk, req->dst); 902 dst = scatterwalk_map(&dst_sg_walk); 903 } 904 905 } else { 906 /* Allocate memory for src, dst, assoc */ 907 src = kmalloc(req->cryptlen + auth_tag_len + req->assoclen, 908 GFP_ATOMIC); 909 if (unlikely(!src)) 910 return -ENOMEM; 911 assoc = (src + req->cryptlen + auth_tag_len); 912 scatterwalk_map_and_copy(src, req->src, 0, req->cryptlen, 0); 913 scatterwalk_map_and_copy(assoc, req->assoc, 0, 914 req->assoclen, 0); 915 dst = src; 916 } 917 918 aesni_gcm_enc(aes_ctx, dst, src, (unsigned long)req->cryptlen, iv, 919 ctx->hash_subkey, assoc, (unsigned long)req->assoclen, dst 920 + ((unsigned long)req->cryptlen), auth_tag_len); 921 922 /* The authTag (aka the Integrity Check Value) needs to be written 923 * back to the packet. */ 924 if (one_entry_in_sg) { 925 if (unlikely(req->src != req->dst)) { 926 scatterwalk_unmap(dst); 927 scatterwalk_done(&dst_sg_walk, 0, 0); 928 } 929 scatterwalk_unmap(src); 930 scatterwalk_unmap(assoc); 931 scatterwalk_done(&src_sg_walk, 0, 0); 932 scatterwalk_done(&assoc_sg_walk, 0, 0); 933 } else { 934 scatterwalk_map_and_copy(dst, req->dst, 0, 935 req->cryptlen + auth_tag_len, 1); 936 kfree(src); 937 } 938 return 0; 939 } 940 941 static int __driver_rfc4106_decrypt(struct aead_request *req) 942 { 943 u8 one_entry_in_sg = 0; 944 u8 *src, *dst, *assoc; 945 unsigned long tempCipherLen = 0; 946 __be32 counter = cpu_to_be32(1); 947 int retval = 0; 948 struct crypto_aead *tfm = crypto_aead_reqtfm(req); 949 struct aesni_rfc4106_gcm_ctx *ctx = aesni_rfc4106_gcm_ctx_get(tfm); 950 void *aes_ctx = &(ctx->aes_key_expanded); 951 unsigned long auth_tag_len = crypto_aead_authsize(tfm); 952 u8 iv_and_authTag[32+AESNI_ALIGN]; 953 u8 *iv = (u8 *) PTR_ALIGN((u8 *)iv_and_authTag, AESNI_ALIGN); 954 u8 *authTag = iv + 16; 955 struct scatter_walk src_sg_walk; 956 struct scatter_walk assoc_sg_walk; 957 struct scatter_walk dst_sg_walk; 958 unsigned int i; 959 960 if (unlikely((req->cryptlen < auth_tag_len) || 961 (req->assoclen != 8 && req->assoclen != 12))) 962 return -EINVAL; 963 /* Assuming we are supporting rfc4106 64-bit extended */ 964 /* sequence numbers We need to have the AAD length */ 965 /* equal to 8 or 12 bytes */ 966 967 tempCipherLen = (unsigned long)(req->cryptlen - auth_tag_len); 968 /* IV below built */ 969 for (i = 0; i < 4; i++) 970 *(iv+i) = ctx->nonce[i]; 971 for (i = 0; i < 8; i++) 972 *(iv+4+i) = req->iv[i]; 973 *((__be32 *)(iv+12)) = counter; 974 975 if ((sg_is_last(req->src)) && (sg_is_last(req->assoc))) { 976 one_entry_in_sg = 1; 977 scatterwalk_start(&src_sg_walk, req->src); 978 scatterwalk_start(&assoc_sg_walk, req->assoc); 979 src = scatterwalk_map(&src_sg_walk); 980 assoc = scatterwalk_map(&assoc_sg_walk); 981 dst = src; 982 if (unlikely(req->src != req->dst)) { 983 scatterwalk_start(&dst_sg_walk, req->dst); 984 dst = scatterwalk_map(&dst_sg_walk); 985 } 986 987 } else { 988 /* Allocate memory for src, dst, assoc */ 989 src = kmalloc(req->cryptlen + req->assoclen, GFP_ATOMIC); 990 if (!src) 991 return -ENOMEM; 992 assoc = (src + req->cryptlen + auth_tag_len); 993 scatterwalk_map_and_copy(src, req->src, 0, req->cryptlen, 0); 994 scatterwalk_map_and_copy(assoc, req->assoc, 0, 995 req->assoclen, 0); 996 dst = src; 997 } 998 999 aesni_gcm_dec(aes_ctx, dst, src, tempCipherLen, iv, 1000 ctx->hash_subkey, assoc, (unsigned long)req->assoclen, 1001 authTag, auth_tag_len); 1002 1003 /* Compare generated tag with passed in tag. */ 1004 retval = memcmp(src + tempCipherLen, authTag, auth_tag_len) ? 1005 -EBADMSG : 0; 1006 1007 if (one_entry_in_sg) { 1008 if (unlikely(req->src != req->dst)) { 1009 scatterwalk_unmap(dst); 1010 scatterwalk_done(&dst_sg_walk, 0, 0); 1011 } 1012 scatterwalk_unmap(src); 1013 scatterwalk_unmap(assoc); 1014 scatterwalk_done(&src_sg_walk, 0, 0); 1015 scatterwalk_done(&assoc_sg_walk, 0, 0); 1016 } else { 1017 scatterwalk_map_and_copy(dst, req->dst, 0, req->cryptlen, 1); 1018 kfree(src); 1019 } 1020 return retval; 1021 } 1022 #endif 1023 1024 static struct crypto_alg aesni_algs[] = { { 1025 .cra_name = "aes", 1026 .cra_driver_name = "aes-aesni", 1027 .cra_priority = 300, 1028 .cra_flags = CRYPTO_ALG_TYPE_CIPHER, 1029 .cra_blocksize = AES_BLOCK_SIZE, 1030 .cra_ctxsize = sizeof(struct crypto_aes_ctx) + 1031 AESNI_ALIGN - 1, 1032 .cra_alignmask = 0, 1033 .cra_module = THIS_MODULE, 1034 .cra_u = { 1035 .cipher = { 1036 .cia_min_keysize = AES_MIN_KEY_SIZE, 1037 .cia_max_keysize = AES_MAX_KEY_SIZE, 1038 .cia_setkey = aes_set_key, 1039 .cia_encrypt = aes_encrypt, 1040 .cia_decrypt = aes_decrypt 1041 } 1042 } 1043 }, { 1044 .cra_name = "__aes-aesni", 1045 .cra_driver_name = "__driver-aes-aesni", 1046 .cra_priority = 0, 1047 .cra_flags = CRYPTO_ALG_TYPE_CIPHER, 1048 .cra_blocksize = AES_BLOCK_SIZE, 1049 .cra_ctxsize = sizeof(struct crypto_aes_ctx) + 1050 AESNI_ALIGN - 1, 1051 .cra_alignmask = 0, 1052 .cra_module = THIS_MODULE, 1053 .cra_u = { 1054 .cipher = { 1055 .cia_min_keysize = AES_MIN_KEY_SIZE, 1056 .cia_max_keysize = AES_MAX_KEY_SIZE, 1057 .cia_setkey = aes_set_key, 1058 .cia_encrypt = __aes_encrypt, 1059 .cia_decrypt = __aes_decrypt 1060 } 1061 } 1062 }, { 1063 .cra_name = "__ecb-aes-aesni", 1064 .cra_driver_name = "__driver-ecb-aes-aesni", 1065 .cra_priority = 0, 1066 .cra_flags = CRYPTO_ALG_TYPE_BLKCIPHER, 1067 .cra_blocksize = AES_BLOCK_SIZE, 1068 .cra_ctxsize = sizeof(struct crypto_aes_ctx) + 1069 AESNI_ALIGN - 1, 1070 .cra_alignmask = 0, 1071 .cra_type = &crypto_blkcipher_type, 1072 .cra_module = THIS_MODULE, 1073 .cra_u = { 1074 .blkcipher = { 1075 .min_keysize = AES_MIN_KEY_SIZE, 1076 .max_keysize = AES_MAX_KEY_SIZE, 1077 .setkey = aes_set_key, 1078 .encrypt = ecb_encrypt, 1079 .decrypt = ecb_decrypt, 1080 }, 1081 }, 1082 }, { 1083 .cra_name = "__cbc-aes-aesni", 1084 .cra_driver_name = "__driver-cbc-aes-aesni", 1085 .cra_priority = 0, 1086 .cra_flags = CRYPTO_ALG_TYPE_BLKCIPHER, 1087 .cra_blocksize = AES_BLOCK_SIZE, 1088 .cra_ctxsize = sizeof(struct crypto_aes_ctx) + 1089 AESNI_ALIGN - 1, 1090 .cra_alignmask = 0, 1091 .cra_type = &crypto_blkcipher_type, 1092 .cra_module = THIS_MODULE, 1093 .cra_u = { 1094 .blkcipher = { 1095 .min_keysize = AES_MIN_KEY_SIZE, 1096 .max_keysize = AES_MAX_KEY_SIZE, 1097 .setkey = aes_set_key, 1098 .encrypt = cbc_encrypt, 1099 .decrypt = cbc_decrypt, 1100 }, 1101 }, 1102 }, { 1103 .cra_name = "ecb(aes)", 1104 .cra_driver_name = "ecb-aes-aesni", 1105 .cra_priority = 400, 1106 .cra_flags = CRYPTO_ALG_TYPE_ABLKCIPHER | CRYPTO_ALG_ASYNC, 1107 .cra_blocksize = AES_BLOCK_SIZE, 1108 .cra_ctxsize = sizeof(struct async_helper_ctx), 1109 .cra_alignmask = 0, 1110 .cra_type = &crypto_ablkcipher_type, 1111 .cra_module = THIS_MODULE, 1112 .cra_init = ablk_ecb_init, 1113 .cra_exit = ablk_exit, 1114 .cra_u = { 1115 .ablkcipher = { 1116 .min_keysize = AES_MIN_KEY_SIZE, 1117 .max_keysize = AES_MAX_KEY_SIZE, 1118 .setkey = ablk_set_key, 1119 .encrypt = ablk_encrypt, 1120 .decrypt = ablk_decrypt, 1121 }, 1122 }, 1123 }, { 1124 .cra_name = "cbc(aes)", 1125 .cra_driver_name = "cbc-aes-aesni", 1126 .cra_priority = 400, 1127 .cra_flags = CRYPTO_ALG_TYPE_ABLKCIPHER | CRYPTO_ALG_ASYNC, 1128 .cra_blocksize = AES_BLOCK_SIZE, 1129 .cra_ctxsize = sizeof(struct async_helper_ctx), 1130 .cra_alignmask = 0, 1131 .cra_type = &crypto_ablkcipher_type, 1132 .cra_module = THIS_MODULE, 1133 .cra_init = ablk_cbc_init, 1134 .cra_exit = ablk_exit, 1135 .cra_u = { 1136 .ablkcipher = { 1137 .min_keysize = AES_MIN_KEY_SIZE, 1138 .max_keysize = AES_MAX_KEY_SIZE, 1139 .ivsize = AES_BLOCK_SIZE, 1140 .setkey = ablk_set_key, 1141 .encrypt = ablk_encrypt, 1142 .decrypt = ablk_decrypt, 1143 }, 1144 }, 1145 #ifdef CONFIG_X86_64 1146 }, { 1147 .cra_name = "__ctr-aes-aesni", 1148 .cra_driver_name = "__driver-ctr-aes-aesni", 1149 .cra_priority = 0, 1150 .cra_flags = CRYPTO_ALG_TYPE_BLKCIPHER, 1151 .cra_blocksize = 1, 1152 .cra_ctxsize = sizeof(struct crypto_aes_ctx) + 1153 AESNI_ALIGN - 1, 1154 .cra_alignmask = 0, 1155 .cra_type = &crypto_blkcipher_type, 1156 .cra_module = THIS_MODULE, 1157 .cra_u = { 1158 .blkcipher = { 1159 .min_keysize = AES_MIN_KEY_SIZE, 1160 .max_keysize = AES_MAX_KEY_SIZE, 1161 .ivsize = AES_BLOCK_SIZE, 1162 .setkey = aes_set_key, 1163 .encrypt = ctr_crypt, 1164 .decrypt = ctr_crypt, 1165 }, 1166 }, 1167 }, { 1168 .cra_name = "ctr(aes)", 1169 .cra_driver_name = "ctr-aes-aesni", 1170 .cra_priority = 400, 1171 .cra_flags = CRYPTO_ALG_TYPE_ABLKCIPHER | CRYPTO_ALG_ASYNC, 1172 .cra_blocksize = 1, 1173 .cra_ctxsize = sizeof(struct async_helper_ctx), 1174 .cra_alignmask = 0, 1175 .cra_type = &crypto_ablkcipher_type, 1176 .cra_module = THIS_MODULE, 1177 .cra_init = ablk_ctr_init, 1178 .cra_exit = ablk_exit, 1179 .cra_u = { 1180 .ablkcipher = { 1181 .min_keysize = AES_MIN_KEY_SIZE, 1182 .max_keysize = AES_MAX_KEY_SIZE, 1183 .ivsize = AES_BLOCK_SIZE, 1184 .setkey = ablk_set_key, 1185 .encrypt = ablk_encrypt, 1186 .decrypt = ablk_encrypt, 1187 .geniv = "chainiv", 1188 }, 1189 }, 1190 }, { 1191 .cra_name = "__gcm-aes-aesni", 1192 .cra_driver_name = "__driver-gcm-aes-aesni", 1193 .cra_priority = 0, 1194 .cra_flags = CRYPTO_ALG_TYPE_AEAD, 1195 .cra_blocksize = 1, 1196 .cra_ctxsize = sizeof(struct aesni_rfc4106_gcm_ctx) + 1197 AESNI_ALIGN, 1198 .cra_alignmask = 0, 1199 .cra_type = &crypto_aead_type, 1200 .cra_module = THIS_MODULE, 1201 .cra_u = { 1202 .aead = { 1203 .encrypt = __driver_rfc4106_encrypt, 1204 .decrypt = __driver_rfc4106_decrypt, 1205 }, 1206 }, 1207 }, { 1208 .cra_name = "rfc4106(gcm(aes))", 1209 .cra_driver_name = "rfc4106-gcm-aesni", 1210 .cra_priority = 400, 1211 .cra_flags = CRYPTO_ALG_TYPE_AEAD | CRYPTO_ALG_ASYNC, 1212 .cra_blocksize = 1, 1213 .cra_ctxsize = sizeof(struct aesni_rfc4106_gcm_ctx) + 1214 AESNI_ALIGN, 1215 .cra_alignmask = 0, 1216 .cra_type = &crypto_nivaead_type, 1217 .cra_module = THIS_MODULE, 1218 .cra_init = rfc4106_init, 1219 .cra_exit = rfc4106_exit, 1220 .cra_u = { 1221 .aead = { 1222 .setkey = rfc4106_set_key, 1223 .setauthsize = rfc4106_set_authsize, 1224 .encrypt = rfc4106_encrypt, 1225 .decrypt = rfc4106_decrypt, 1226 .geniv = "seqiv", 1227 .ivsize = 8, 1228 .maxauthsize = 16, 1229 }, 1230 }, 1231 #endif 1232 #ifdef HAS_PCBC 1233 }, { 1234 .cra_name = "pcbc(aes)", 1235 .cra_driver_name = "pcbc-aes-aesni", 1236 .cra_priority = 400, 1237 .cra_flags = CRYPTO_ALG_TYPE_ABLKCIPHER | CRYPTO_ALG_ASYNC, 1238 .cra_blocksize = AES_BLOCK_SIZE, 1239 .cra_ctxsize = sizeof(struct async_helper_ctx), 1240 .cra_alignmask = 0, 1241 .cra_type = &crypto_ablkcipher_type, 1242 .cra_module = THIS_MODULE, 1243 .cra_init = ablk_pcbc_init, 1244 .cra_exit = ablk_exit, 1245 .cra_u = { 1246 .ablkcipher = { 1247 .min_keysize = AES_MIN_KEY_SIZE, 1248 .max_keysize = AES_MAX_KEY_SIZE, 1249 .ivsize = AES_BLOCK_SIZE, 1250 .setkey = ablk_set_key, 1251 .encrypt = ablk_encrypt, 1252 .decrypt = ablk_decrypt, 1253 }, 1254 }, 1255 #endif 1256 }, { 1257 .cra_name = "__lrw-aes-aesni", 1258 .cra_driver_name = "__driver-lrw-aes-aesni", 1259 .cra_priority = 0, 1260 .cra_flags = CRYPTO_ALG_TYPE_BLKCIPHER, 1261 .cra_blocksize = AES_BLOCK_SIZE, 1262 .cra_ctxsize = sizeof(struct aesni_lrw_ctx), 1263 .cra_alignmask = 0, 1264 .cra_type = &crypto_blkcipher_type, 1265 .cra_module = THIS_MODULE, 1266 .cra_exit = lrw_aesni_exit_tfm, 1267 .cra_u = { 1268 .blkcipher = { 1269 .min_keysize = AES_MIN_KEY_SIZE + AES_BLOCK_SIZE, 1270 .max_keysize = AES_MAX_KEY_SIZE + AES_BLOCK_SIZE, 1271 .ivsize = AES_BLOCK_SIZE, 1272 .setkey = lrw_aesni_setkey, 1273 .encrypt = lrw_encrypt, 1274 .decrypt = lrw_decrypt, 1275 }, 1276 }, 1277 }, { 1278 .cra_name = "__xts-aes-aesni", 1279 .cra_driver_name = "__driver-xts-aes-aesni", 1280 .cra_priority = 0, 1281 .cra_flags = CRYPTO_ALG_TYPE_BLKCIPHER, 1282 .cra_blocksize = AES_BLOCK_SIZE, 1283 .cra_ctxsize = sizeof(struct aesni_xts_ctx), 1284 .cra_alignmask = 0, 1285 .cra_type = &crypto_blkcipher_type, 1286 .cra_module = THIS_MODULE, 1287 .cra_u = { 1288 .blkcipher = { 1289 .min_keysize = 2 * AES_MIN_KEY_SIZE, 1290 .max_keysize = 2 * AES_MAX_KEY_SIZE, 1291 .ivsize = AES_BLOCK_SIZE, 1292 .setkey = xts_aesni_setkey, 1293 .encrypt = xts_encrypt, 1294 .decrypt = xts_decrypt, 1295 }, 1296 }, 1297 }, { 1298 .cra_name = "lrw(aes)", 1299 .cra_driver_name = "lrw-aes-aesni", 1300 .cra_priority = 400, 1301 .cra_flags = CRYPTO_ALG_TYPE_ABLKCIPHER | CRYPTO_ALG_ASYNC, 1302 .cra_blocksize = AES_BLOCK_SIZE, 1303 .cra_ctxsize = sizeof(struct async_helper_ctx), 1304 .cra_alignmask = 0, 1305 .cra_type = &crypto_ablkcipher_type, 1306 .cra_module = THIS_MODULE, 1307 .cra_init = ablk_init, 1308 .cra_exit = ablk_exit, 1309 .cra_u = { 1310 .ablkcipher = { 1311 .min_keysize = AES_MIN_KEY_SIZE + AES_BLOCK_SIZE, 1312 .max_keysize = AES_MAX_KEY_SIZE + AES_BLOCK_SIZE, 1313 .ivsize = AES_BLOCK_SIZE, 1314 .setkey = ablk_set_key, 1315 .encrypt = ablk_encrypt, 1316 .decrypt = ablk_decrypt, 1317 }, 1318 }, 1319 }, { 1320 .cra_name = "xts(aes)", 1321 .cra_driver_name = "xts-aes-aesni", 1322 .cra_priority = 400, 1323 .cra_flags = CRYPTO_ALG_TYPE_ABLKCIPHER | CRYPTO_ALG_ASYNC, 1324 .cra_blocksize = AES_BLOCK_SIZE, 1325 .cra_ctxsize = sizeof(struct async_helper_ctx), 1326 .cra_alignmask = 0, 1327 .cra_type = &crypto_ablkcipher_type, 1328 .cra_module = THIS_MODULE, 1329 .cra_init = ablk_init, 1330 .cra_exit = ablk_exit, 1331 .cra_u = { 1332 .ablkcipher = { 1333 .min_keysize = 2 * AES_MIN_KEY_SIZE, 1334 .max_keysize = 2 * AES_MAX_KEY_SIZE, 1335 .ivsize = AES_BLOCK_SIZE, 1336 .setkey = ablk_set_key, 1337 .encrypt = ablk_encrypt, 1338 .decrypt = ablk_decrypt, 1339 }, 1340 }, 1341 } }; 1342 1343 1344 static const struct x86_cpu_id aesni_cpu_id[] = { 1345 X86_FEATURE_MATCH(X86_FEATURE_AES), 1346 {} 1347 }; 1348 MODULE_DEVICE_TABLE(x86cpu, aesni_cpu_id); 1349 1350 static int __init aesni_init(void) 1351 { 1352 int err; 1353 1354 if (!x86_match_cpu(aesni_cpu_id)) 1355 return -ENODEV; 1356 1357 err = crypto_fpu_init(); 1358 if (err) 1359 return err; 1360 1361 return crypto_register_algs(aesni_algs, ARRAY_SIZE(aesni_algs)); 1362 } 1363 1364 static void __exit aesni_exit(void) 1365 { 1366 crypto_unregister_algs(aesni_algs, ARRAY_SIZE(aesni_algs)); 1367 1368 crypto_fpu_exit(); 1369 } 1370 1371 module_init(aesni_init); 1372 module_exit(aesni_exit); 1373 1374 MODULE_DESCRIPTION("Rijndael (AES) Cipher Algorithm, Intel AES-NI instructions optimized"); 1375 MODULE_LICENSE("GPL"); 1376 MODULE_ALIAS("aes"); 1377