1 /* 2 * Support for Intel AES-NI instructions. This file contains glue 3 * code, the real AES implementation is in intel-aes_asm.S. 4 * 5 * Copyright (C) 2008, Intel Corp. 6 * Author: Huang Ying <ying.huang@intel.com> 7 * 8 * Added RFC4106 AES-GCM support for 128-bit keys under the AEAD 9 * interface for 64-bit kernels. 10 * Authors: Adrian Hoban <adrian.hoban@intel.com> 11 * Gabriele Paoloni <gabriele.paoloni@intel.com> 12 * Tadeusz Struk (tadeusz.struk@intel.com) 13 * Aidan O'Mahony (aidan.o.mahony@intel.com) 14 * Copyright (c) 2010, Intel Corporation. 15 * 16 * This program is free software; you can redistribute it and/or modify 17 * it under the terms of the GNU General Public License as published by 18 * the Free Software Foundation; either version 2 of the License, or 19 * (at your option) any later version. 20 */ 21 22 #include <linux/hardirq.h> 23 #include <linux/types.h> 24 #include <linux/module.h> 25 #include <linux/err.h> 26 #include <crypto/algapi.h> 27 #include <crypto/aes.h> 28 #include <crypto/cryptd.h> 29 #include <crypto/ctr.h> 30 #include <crypto/b128ops.h> 31 #include <crypto/gcm.h> 32 #include <crypto/xts.h> 33 #include <asm/cpu_device_id.h> 34 #include <asm/fpu/api.h> 35 #include <asm/crypto/aes.h> 36 #include <crypto/scatterwalk.h> 37 #include <crypto/internal/aead.h> 38 #include <crypto/internal/simd.h> 39 #include <crypto/internal/skcipher.h> 40 #include <linux/workqueue.h> 41 #include <linux/spinlock.h> 42 #ifdef CONFIG_X86_64 43 #include <asm/crypto/glue_helper.h> 44 #endif 45 46 47 #define AESNI_ALIGN 16 48 #define AESNI_ALIGN_ATTR __attribute__ ((__aligned__(AESNI_ALIGN))) 49 #define AES_BLOCK_MASK (~(AES_BLOCK_SIZE - 1)) 50 #define RFC4106_HASH_SUBKEY_SIZE 16 51 #define AESNI_ALIGN_EXTRA ((AESNI_ALIGN - 1) & ~(CRYPTO_MINALIGN - 1)) 52 #define CRYPTO_AES_CTX_SIZE (sizeof(struct crypto_aes_ctx) + AESNI_ALIGN_EXTRA) 53 #define XTS_AES_CTX_SIZE (sizeof(struct aesni_xts_ctx) + AESNI_ALIGN_EXTRA) 54 55 /* This data is stored at the end of the crypto_tfm struct. 56 * It's a type of per "session" data storage location. 57 * This needs to be 16 byte aligned. 58 */ 59 struct aesni_rfc4106_gcm_ctx { 60 u8 hash_subkey[16] AESNI_ALIGN_ATTR; 61 struct crypto_aes_ctx aes_key_expanded AESNI_ALIGN_ATTR; 62 u8 nonce[4]; 63 }; 64 65 struct generic_gcmaes_ctx { 66 u8 hash_subkey[16] AESNI_ALIGN_ATTR; 67 struct crypto_aes_ctx aes_key_expanded AESNI_ALIGN_ATTR; 68 }; 69 70 struct aesni_xts_ctx { 71 u8 raw_tweak_ctx[sizeof(struct crypto_aes_ctx)] AESNI_ALIGN_ATTR; 72 u8 raw_crypt_ctx[sizeof(struct crypto_aes_ctx)] AESNI_ALIGN_ATTR; 73 }; 74 75 #define GCM_BLOCK_LEN 16 76 77 struct gcm_context_data { 78 /* init, update and finalize context data */ 79 u8 aad_hash[GCM_BLOCK_LEN]; 80 u64 aad_length; 81 u64 in_length; 82 u8 partial_block_enc_key[GCM_BLOCK_LEN]; 83 u8 orig_IV[GCM_BLOCK_LEN]; 84 u8 current_counter[GCM_BLOCK_LEN]; 85 u64 partial_block_len; 86 u64 unused; 87 u8 hash_keys[GCM_BLOCK_LEN * 16]; 88 }; 89 90 asmlinkage int aesni_set_key(struct crypto_aes_ctx *ctx, const u8 *in_key, 91 unsigned int key_len); 92 asmlinkage void aesni_enc(struct crypto_aes_ctx *ctx, u8 *out, 93 const u8 *in); 94 asmlinkage void aesni_dec(struct crypto_aes_ctx *ctx, u8 *out, 95 const u8 *in); 96 asmlinkage void aesni_ecb_enc(struct crypto_aes_ctx *ctx, u8 *out, 97 const u8 *in, unsigned int len); 98 asmlinkage void aesni_ecb_dec(struct crypto_aes_ctx *ctx, u8 *out, 99 const u8 *in, unsigned int len); 100 asmlinkage void aesni_cbc_enc(struct crypto_aes_ctx *ctx, u8 *out, 101 const u8 *in, unsigned int len, u8 *iv); 102 asmlinkage void aesni_cbc_dec(struct crypto_aes_ctx *ctx, u8 *out, 103 const u8 *in, unsigned int len, u8 *iv); 104 105 #define AVX_GEN2_OPTSIZE 640 106 #define AVX_GEN4_OPTSIZE 4096 107 108 #ifdef CONFIG_X86_64 109 110 static void (*aesni_ctr_enc_tfm)(struct crypto_aes_ctx *ctx, u8 *out, 111 const u8 *in, unsigned int len, u8 *iv); 112 asmlinkage void aesni_ctr_enc(struct crypto_aes_ctx *ctx, u8 *out, 113 const u8 *in, unsigned int len, u8 *iv); 114 115 asmlinkage void aesni_xts_crypt8(struct crypto_aes_ctx *ctx, u8 *out, 116 const u8 *in, bool enc, u8 *iv); 117 118 /* asmlinkage void aesni_gcm_enc() 119 * void *ctx, AES Key schedule. Starts on a 16 byte boundary. 120 * struct gcm_context_data. May be uninitialized. 121 * u8 *out, Ciphertext output. Encrypt in-place is allowed. 122 * const u8 *in, Plaintext input 123 * unsigned long plaintext_len, Length of data in bytes for encryption. 124 * u8 *iv, Pre-counter block j0: 12 byte IV concatenated with 0x00000001. 125 * 16-byte aligned pointer. 126 * u8 *hash_subkey, the Hash sub key input. Data starts on a 16-byte boundary. 127 * const u8 *aad, Additional Authentication Data (AAD) 128 * unsigned long aad_len, Length of AAD in bytes. 129 * u8 *auth_tag, Authenticated Tag output. 130 * unsigned long auth_tag_len), Authenticated Tag Length in bytes. 131 * Valid values are 16 (most likely), 12 or 8. 132 */ 133 asmlinkage void aesni_gcm_enc(void *ctx, 134 struct gcm_context_data *gdata, u8 *out, 135 const u8 *in, unsigned long plaintext_len, u8 *iv, 136 u8 *hash_subkey, const u8 *aad, unsigned long aad_len, 137 u8 *auth_tag, unsigned long auth_tag_len); 138 139 /* asmlinkage void aesni_gcm_dec() 140 * void *ctx, AES Key schedule. Starts on a 16 byte boundary. 141 * struct gcm_context_data. May be uninitialized. 142 * u8 *out, Plaintext output. Decrypt in-place is allowed. 143 * const u8 *in, Ciphertext input 144 * unsigned long ciphertext_len, Length of data in bytes for decryption. 145 * u8 *iv, Pre-counter block j0: 12 byte IV concatenated with 0x00000001. 146 * 16-byte aligned pointer. 147 * u8 *hash_subkey, the Hash sub key input. Data starts on a 16-byte boundary. 148 * const u8 *aad, Additional Authentication Data (AAD) 149 * unsigned long aad_len, Length of AAD in bytes. With RFC4106 this is going 150 * to be 8 or 12 bytes 151 * u8 *auth_tag, Authenticated Tag output. 152 * unsigned long auth_tag_len) Authenticated Tag Length in bytes. 153 * Valid values are 16 (most likely), 12 or 8. 154 */ 155 asmlinkage void aesni_gcm_dec(void *ctx, 156 struct gcm_context_data *gdata, u8 *out, 157 const u8 *in, unsigned long ciphertext_len, u8 *iv, 158 u8 *hash_subkey, const u8 *aad, unsigned long aad_len, 159 u8 *auth_tag, unsigned long auth_tag_len); 160 161 /* Scatter / Gather routines, with args similar to above */ 162 asmlinkage void aesni_gcm_init(void *ctx, 163 struct gcm_context_data *gdata, 164 u8 *iv, 165 u8 *hash_subkey, const u8 *aad, 166 unsigned long aad_len); 167 asmlinkage void aesni_gcm_enc_update(void *ctx, 168 struct gcm_context_data *gdata, u8 *out, 169 const u8 *in, unsigned long plaintext_len); 170 asmlinkage void aesni_gcm_dec_update(void *ctx, 171 struct gcm_context_data *gdata, u8 *out, 172 const u8 *in, 173 unsigned long ciphertext_len); 174 asmlinkage void aesni_gcm_finalize(void *ctx, 175 struct gcm_context_data *gdata, 176 u8 *auth_tag, unsigned long auth_tag_len); 177 178 static struct aesni_gcm_tfm_s { 179 void (*init)(void *ctx, 180 struct gcm_context_data *gdata, 181 u8 *iv, 182 u8 *hash_subkey, const u8 *aad, 183 unsigned long aad_len); 184 void (*enc_update)(void *ctx, 185 struct gcm_context_data *gdata, u8 *out, 186 const u8 *in, 187 unsigned long plaintext_len); 188 void (*dec_update)(void *ctx, 189 struct gcm_context_data *gdata, u8 *out, 190 const u8 *in, 191 unsigned long ciphertext_len); 192 void (*finalize)(void *ctx, 193 struct gcm_context_data *gdata, 194 u8 *auth_tag, unsigned long auth_tag_len); 195 } *aesni_gcm_tfm; 196 197 struct aesni_gcm_tfm_s aesni_gcm_tfm_sse = { 198 .init = &aesni_gcm_init, 199 .enc_update = &aesni_gcm_enc_update, 200 .dec_update = &aesni_gcm_dec_update, 201 .finalize = &aesni_gcm_finalize, 202 }; 203 204 #ifdef CONFIG_AS_AVX 205 asmlinkage void aes_ctr_enc_128_avx_by8(const u8 *in, u8 *iv, 206 void *keys, u8 *out, unsigned int num_bytes); 207 asmlinkage void aes_ctr_enc_192_avx_by8(const u8 *in, u8 *iv, 208 void *keys, u8 *out, unsigned int num_bytes); 209 asmlinkage void aes_ctr_enc_256_avx_by8(const u8 *in, u8 *iv, 210 void *keys, u8 *out, unsigned int num_bytes); 211 /* 212 * asmlinkage void aesni_gcm_init_avx_gen2() 213 * gcm_data *my_ctx_data, context data 214 * u8 *hash_subkey, the Hash sub key input. Data starts on a 16-byte boundary. 215 */ 216 asmlinkage void aesni_gcm_init_avx_gen2(void *my_ctx_data, 217 struct gcm_context_data *gdata, 218 u8 *iv, 219 u8 *hash_subkey, 220 const u8 *aad, 221 unsigned long aad_len); 222 223 asmlinkage void aesni_gcm_enc_update_avx_gen2(void *ctx, 224 struct gcm_context_data *gdata, u8 *out, 225 const u8 *in, unsigned long plaintext_len); 226 asmlinkage void aesni_gcm_dec_update_avx_gen2(void *ctx, 227 struct gcm_context_data *gdata, u8 *out, 228 const u8 *in, 229 unsigned long ciphertext_len); 230 asmlinkage void aesni_gcm_finalize_avx_gen2(void *ctx, 231 struct gcm_context_data *gdata, 232 u8 *auth_tag, unsigned long auth_tag_len); 233 234 asmlinkage void aesni_gcm_enc_avx_gen2(void *ctx, 235 struct gcm_context_data *gdata, u8 *out, 236 const u8 *in, unsigned long plaintext_len, u8 *iv, 237 const u8 *aad, unsigned long aad_len, 238 u8 *auth_tag, unsigned long auth_tag_len); 239 240 asmlinkage void aesni_gcm_dec_avx_gen2(void *ctx, 241 struct gcm_context_data *gdata, u8 *out, 242 const u8 *in, unsigned long ciphertext_len, u8 *iv, 243 const u8 *aad, unsigned long aad_len, 244 u8 *auth_tag, unsigned long auth_tag_len); 245 246 struct aesni_gcm_tfm_s aesni_gcm_tfm_avx_gen2 = { 247 .init = &aesni_gcm_init_avx_gen2, 248 .enc_update = &aesni_gcm_enc_update_avx_gen2, 249 .dec_update = &aesni_gcm_dec_update_avx_gen2, 250 .finalize = &aesni_gcm_finalize_avx_gen2, 251 }; 252 253 #endif 254 255 #ifdef CONFIG_AS_AVX2 256 /* 257 * asmlinkage void aesni_gcm_init_avx_gen4() 258 * gcm_data *my_ctx_data, context data 259 * u8 *hash_subkey, the Hash sub key input. Data starts on a 16-byte boundary. 260 */ 261 asmlinkage void aesni_gcm_init_avx_gen4(void *my_ctx_data, 262 struct gcm_context_data *gdata, 263 u8 *iv, 264 u8 *hash_subkey, 265 const u8 *aad, 266 unsigned long aad_len); 267 268 asmlinkage void aesni_gcm_enc_update_avx_gen4(void *ctx, 269 struct gcm_context_data *gdata, u8 *out, 270 const u8 *in, unsigned long plaintext_len); 271 asmlinkage void aesni_gcm_dec_update_avx_gen4(void *ctx, 272 struct gcm_context_data *gdata, u8 *out, 273 const u8 *in, 274 unsigned long ciphertext_len); 275 asmlinkage void aesni_gcm_finalize_avx_gen4(void *ctx, 276 struct gcm_context_data *gdata, 277 u8 *auth_tag, unsigned long auth_tag_len); 278 279 asmlinkage void aesni_gcm_enc_avx_gen4(void *ctx, 280 struct gcm_context_data *gdata, u8 *out, 281 const u8 *in, unsigned long plaintext_len, u8 *iv, 282 const u8 *aad, unsigned long aad_len, 283 u8 *auth_tag, unsigned long auth_tag_len); 284 285 asmlinkage void aesni_gcm_dec_avx_gen4(void *ctx, 286 struct gcm_context_data *gdata, u8 *out, 287 const u8 *in, unsigned long ciphertext_len, u8 *iv, 288 const u8 *aad, unsigned long aad_len, 289 u8 *auth_tag, unsigned long auth_tag_len); 290 291 struct aesni_gcm_tfm_s aesni_gcm_tfm_avx_gen4 = { 292 .init = &aesni_gcm_init_avx_gen4, 293 .enc_update = &aesni_gcm_enc_update_avx_gen4, 294 .dec_update = &aesni_gcm_dec_update_avx_gen4, 295 .finalize = &aesni_gcm_finalize_avx_gen4, 296 }; 297 298 #endif 299 300 static inline struct 301 aesni_rfc4106_gcm_ctx *aesni_rfc4106_gcm_ctx_get(struct crypto_aead *tfm) 302 { 303 unsigned long align = AESNI_ALIGN; 304 305 if (align <= crypto_tfm_ctx_alignment()) 306 align = 1; 307 return PTR_ALIGN(crypto_aead_ctx(tfm), align); 308 } 309 310 static inline struct 311 generic_gcmaes_ctx *generic_gcmaes_ctx_get(struct crypto_aead *tfm) 312 { 313 unsigned long align = AESNI_ALIGN; 314 315 if (align <= crypto_tfm_ctx_alignment()) 316 align = 1; 317 return PTR_ALIGN(crypto_aead_ctx(tfm), align); 318 } 319 #endif 320 321 static inline struct crypto_aes_ctx *aes_ctx(void *raw_ctx) 322 { 323 unsigned long addr = (unsigned long)raw_ctx; 324 unsigned long align = AESNI_ALIGN; 325 326 if (align <= crypto_tfm_ctx_alignment()) 327 align = 1; 328 return (struct crypto_aes_ctx *)ALIGN(addr, align); 329 } 330 331 static int aes_set_key_common(struct crypto_tfm *tfm, void *raw_ctx, 332 const u8 *in_key, unsigned int key_len) 333 { 334 struct crypto_aes_ctx *ctx = aes_ctx(raw_ctx); 335 u32 *flags = &tfm->crt_flags; 336 int err; 337 338 if (key_len != AES_KEYSIZE_128 && key_len != AES_KEYSIZE_192 && 339 key_len != AES_KEYSIZE_256) { 340 *flags |= CRYPTO_TFM_RES_BAD_KEY_LEN; 341 return -EINVAL; 342 } 343 344 if (!irq_fpu_usable()) 345 err = crypto_aes_expand_key(ctx, in_key, key_len); 346 else { 347 kernel_fpu_begin(); 348 err = aesni_set_key(ctx, in_key, key_len); 349 kernel_fpu_end(); 350 } 351 352 return err; 353 } 354 355 static int aes_set_key(struct crypto_tfm *tfm, const u8 *in_key, 356 unsigned int key_len) 357 { 358 return aes_set_key_common(tfm, crypto_tfm_ctx(tfm), in_key, key_len); 359 } 360 361 static void aes_encrypt(struct crypto_tfm *tfm, u8 *dst, const u8 *src) 362 { 363 struct crypto_aes_ctx *ctx = aes_ctx(crypto_tfm_ctx(tfm)); 364 365 if (!irq_fpu_usable()) 366 crypto_aes_encrypt_x86(ctx, dst, src); 367 else { 368 kernel_fpu_begin(); 369 aesni_enc(ctx, dst, src); 370 kernel_fpu_end(); 371 } 372 } 373 374 static void aes_decrypt(struct crypto_tfm *tfm, u8 *dst, const u8 *src) 375 { 376 struct crypto_aes_ctx *ctx = aes_ctx(crypto_tfm_ctx(tfm)); 377 378 if (!irq_fpu_usable()) 379 crypto_aes_decrypt_x86(ctx, dst, src); 380 else { 381 kernel_fpu_begin(); 382 aesni_dec(ctx, dst, src); 383 kernel_fpu_end(); 384 } 385 } 386 387 static void __aes_encrypt(struct crypto_tfm *tfm, u8 *dst, const u8 *src) 388 { 389 struct crypto_aes_ctx *ctx = aes_ctx(crypto_tfm_ctx(tfm)); 390 391 aesni_enc(ctx, dst, src); 392 } 393 394 static void __aes_decrypt(struct crypto_tfm *tfm, u8 *dst, const u8 *src) 395 { 396 struct crypto_aes_ctx *ctx = aes_ctx(crypto_tfm_ctx(tfm)); 397 398 aesni_dec(ctx, dst, src); 399 } 400 401 static int aesni_skcipher_setkey(struct crypto_skcipher *tfm, const u8 *key, 402 unsigned int len) 403 { 404 return aes_set_key_common(crypto_skcipher_tfm(tfm), 405 crypto_skcipher_ctx(tfm), key, len); 406 } 407 408 static int ecb_encrypt(struct skcipher_request *req) 409 { 410 struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req); 411 struct crypto_aes_ctx *ctx = aes_ctx(crypto_skcipher_ctx(tfm)); 412 struct skcipher_walk walk; 413 unsigned int nbytes; 414 int err; 415 416 err = skcipher_walk_virt(&walk, req, true); 417 418 kernel_fpu_begin(); 419 while ((nbytes = walk.nbytes)) { 420 aesni_ecb_enc(ctx, walk.dst.virt.addr, walk.src.virt.addr, 421 nbytes & AES_BLOCK_MASK); 422 nbytes &= AES_BLOCK_SIZE - 1; 423 err = skcipher_walk_done(&walk, nbytes); 424 } 425 kernel_fpu_end(); 426 427 return err; 428 } 429 430 static int ecb_decrypt(struct skcipher_request *req) 431 { 432 struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req); 433 struct crypto_aes_ctx *ctx = aes_ctx(crypto_skcipher_ctx(tfm)); 434 struct skcipher_walk walk; 435 unsigned int nbytes; 436 int err; 437 438 err = skcipher_walk_virt(&walk, req, true); 439 440 kernel_fpu_begin(); 441 while ((nbytes = walk.nbytes)) { 442 aesni_ecb_dec(ctx, walk.dst.virt.addr, walk.src.virt.addr, 443 nbytes & AES_BLOCK_MASK); 444 nbytes &= AES_BLOCK_SIZE - 1; 445 err = skcipher_walk_done(&walk, nbytes); 446 } 447 kernel_fpu_end(); 448 449 return err; 450 } 451 452 static int cbc_encrypt(struct skcipher_request *req) 453 { 454 struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req); 455 struct crypto_aes_ctx *ctx = aes_ctx(crypto_skcipher_ctx(tfm)); 456 struct skcipher_walk walk; 457 unsigned int nbytes; 458 int err; 459 460 err = skcipher_walk_virt(&walk, req, true); 461 462 kernel_fpu_begin(); 463 while ((nbytes = walk.nbytes)) { 464 aesni_cbc_enc(ctx, walk.dst.virt.addr, walk.src.virt.addr, 465 nbytes & AES_BLOCK_MASK, walk.iv); 466 nbytes &= AES_BLOCK_SIZE - 1; 467 err = skcipher_walk_done(&walk, nbytes); 468 } 469 kernel_fpu_end(); 470 471 return err; 472 } 473 474 static int cbc_decrypt(struct skcipher_request *req) 475 { 476 struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req); 477 struct crypto_aes_ctx *ctx = aes_ctx(crypto_skcipher_ctx(tfm)); 478 struct skcipher_walk walk; 479 unsigned int nbytes; 480 int err; 481 482 err = skcipher_walk_virt(&walk, req, true); 483 484 kernel_fpu_begin(); 485 while ((nbytes = walk.nbytes)) { 486 aesni_cbc_dec(ctx, walk.dst.virt.addr, walk.src.virt.addr, 487 nbytes & AES_BLOCK_MASK, walk.iv); 488 nbytes &= AES_BLOCK_SIZE - 1; 489 err = skcipher_walk_done(&walk, nbytes); 490 } 491 kernel_fpu_end(); 492 493 return err; 494 } 495 496 #ifdef CONFIG_X86_64 497 static void ctr_crypt_final(struct crypto_aes_ctx *ctx, 498 struct skcipher_walk *walk) 499 { 500 u8 *ctrblk = walk->iv; 501 u8 keystream[AES_BLOCK_SIZE]; 502 u8 *src = walk->src.virt.addr; 503 u8 *dst = walk->dst.virt.addr; 504 unsigned int nbytes = walk->nbytes; 505 506 aesni_enc(ctx, keystream, ctrblk); 507 crypto_xor_cpy(dst, keystream, src, nbytes); 508 509 crypto_inc(ctrblk, AES_BLOCK_SIZE); 510 } 511 512 #ifdef CONFIG_AS_AVX 513 static void aesni_ctr_enc_avx_tfm(struct crypto_aes_ctx *ctx, u8 *out, 514 const u8 *in, unsigned int len, u8 *iv) 515 { 516 /* 517 * based on key length, override with the by8 version 518 * of ctr mode encryption/decryption for improved performance 519 * aes_set_key_common() ensures that key length is one of 520 * {128,192,256} 521 */ 522 if (ctx->key_length == AES_KEYSIZE_128) 523 aes_ctr_enc_128_avx_by8(in, iv, (void *)ctx, out, len); 524 else if (ctx->key_length == AES_KEYSIZE_192) 525 aes_ctr_enc_192_avx_by8(in, iv, (void *)ctx, out, len); 526 else 527 aes_ctr_enc_256_avx_by8(in, iv, (void *)ctx, out, len); 528 } 529 #endif 530 531 static int ctr_crypt(struct skcipher_request *req) 532 { 533 struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req); 534 struct crypto_aes_ctx *ctx = aes_ctx(crypto_skcipher_ctx(tfm)); 535 struct skcipher_walk walk; 536 unsigned int nbytes; 537 int err; 538 539 err = skcipher_walk_virt(&walk, req, true); 540 541 kernel_fpu_begin(); 542 while ((nbytes = walk.nbytes) >= AES_BLOCK_SIZE) { 543 aesni_ctr_enc_tfm(ctx, walk.dst.virt.addr, walk.src.virt.addr, 544 nbytes & AES_BLOCK_MASK, walk.iv); 545 nbytes &= AES_BLOCK_SIZE - 1; 546 err = skcipher_walk_done(&walk, nbytes); 547 } 548 if (walk.nbytes) { 549 ctr_crypt_final(ctx, &walk); 550 err = skcipher_walk_done(&walk, 0); 551 } 552 kernel_fpu_end(); 553 554 return err; 555 } 556 557 static int xts_aesni_setkey(struct crypto_skcipher *tfm, const u8 *key, 558 unsigned int keylen) 559 { 560 struct aesni_xts_ctx *ctx = crypto_skcipher_ctx(tfm); 561 int err; 562 563 err = xts_verify_key(tfm, key, keylen); 564 if (err) 565 return err; 566 567 keylen /= 2; 568 569 /* first half of xts-key is for crypt */ 570 err = aes_set_key_common(crypto_skcipher_tfm(tfm), ctx->raw_crypt_ctx, 571 key, keylen); 572 if (err) 573 return err; 574 575 /* second half of xts-key is for tweak */ 576 return aes_set_key_common(crypto_skcipher_tfm(tfm), ctx->raw_tweak_ctx, 577 key + keylen, keylen); 578 } 579 580 581 static void aesni_xts_tweak(void *ctx, u8 *out, const u8 *in) 582 { 583 aesni_enc(ctx, out, in); 584 } 585 586 static void aesni_xts_enc(void *ctx, u128 *dst, const u128 *src, le128 *iv) 587 { 588 glue_xts_crypt_128bit_one(ctx, dst, src, iv, GLUE_FUNC_CAST(aesni_enc)); 589 } 590 591 static void aesni_xts_dec(void *ctx, u128 *dst, const u128 *src, le128 *iv) 592 { 593 glue_xts_crypt_128bit_one(ctx, dst, src, iv, GLUE_FUNC_CAST(aesni_dec)); 594 } 595 596 static void aesni_xts_enc8(void *ctx, u128 *dst, const u128 *src, le128 *iv) 597 { 598 aesni_xts_crypt8(ctx, (u8 *)dst, (const u8 *)src, true, (u8 *)iv); 599 } 600 601 static void aesni_xts_dec8(void *ctx, u128 *dst, const u128 *src, le128 *iv) 602 { 603 aesni_xts_crypt8(ctx, (u8 *)dst, (const u8 *)src, false, (u8 *)iv); 604 } 605 606 static const struct common_glue_ctx aesni_enc_xts = { 607 .num_funcs = 2, 608 .fpu_blocks_limit = 1, 609 610 .funcs = { { 611 .num_blocks = 8, 612 .fn_u = { .xts = GLUE_XTS_FUNC_CAST(aesni_xts_enc8) } 613 }, { 614 .num_blocks = 1, 615 .fn_u = { .xts = GLUE_XTS_FUNC_CAST(aesni_xts_enc) } 616 } } 617 }; 618 619 static const struct common_glue_ctx aesni_dec_xts = { 620 .num_funcs = 2, 621 .fpu_blocks_limit = 1, 622 623 .funcs = { { 624 .num_blocks = 8, 625 .fn_u = { .xts = GLUE_XTS_FUNC_CAST(aesni_xts_dec8) } 626 }, { 627 .num_blocks = 1, 628 .fn_u = { .xts = GLUE_XTS_FUNC_CAST(aesni_xts_dec) } 629 } } 630 }; 631 632 static int xts_encrypt(struct skcipher_request *req) 633 { 634 struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req); 635 struct aesni_xts_ctx *ctx = crypto_skcipher_ctx(tfm); 636 637 return glue_xts_req_128bit(&aesni_enc_xts, req, 638 XTS_TWEAK_CAST(aesni_xts_tweak), 639 aes_ctx(ctx->raw_tweak_ctx), 640 aes_ctx(ctx->raw_crypt_ctx)); 641 } 642 643 static int xts_decrypt(struct skcipher_request *req) 644 { 645 struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req); 646 struct aesni_xts_ctx *ctx = crypto_skcipher_ctx(tfm); 647 648 return glue_xts_req_128bit(&aesni_dec_xts, req, 649 XTS_TWEAK_CAST(aesni_xts_tweak), 650 aes_ctx(ctx->raw_tweak_ctx), 651 aes_ctx(ctx->raw_crypt_ctx)); 652 } 653 654 static int rfc4106_init(struct crypto_aead *aead) 655 { 656 struct cryptd_aead *cryptd_tfm; 657 struct cryptd_aead **ctx = crypto_aead_ctx(aead); 658 659 cryptd_tfm = cryptd_alloc_aead("__driver-gcm-aes-aesni", 660 CRYPTO_ALG_INTERNAL, 661 CRYPTO_ALG_INTERNAL); 662 if (IS_ERR(cryptd_tfm)) 663 return PTR_ERR(cryptd_tfm); 664 665 *ctx = cryptd_tfm; 666 crypto_aead_set_reqsize(aead, crypto_aead_reqsize(&cryptd_tfm->base)); 667 return 0; 668 } 669 670 static void rfc4106_exit(struct crypto_aead *aead) 671 { 672 struct cryptd_aead **ctx = crypto_aead_ctx(aead); 673 674 cryptd_free_aead(*ctx); 675 } 676 677 static int 678 rfc4106_set_hash_subkey(u8 *hash_subkey, const u8 *key, unsigned int key_len) 679 { 680 struct crypto_cipher *tfm; 681 int ret; 682 683 tfm = crypto_alloc_cipher("aes", 0, 0); 684 if (IS_ERR(tfm)) 685 return PTR_ERR(tfm); 686 687 ret = crypto_cipher_setkey(tfm, key, key_len); 688 if (ret) 689 goto out_free_cipher; 690 691 /* Clear the data in the hash sub key container to zero.*/ 692 /* We want to cipher all zeros to create the hash sub key. */ 693 memset(hash_subkey, 0, RFC4106_HASH_SUBKEY_SIZE); 694 695 crypto_cipher_encrypt_one(tfm, hash_subkey, hash_subkey); 696 697 out_free_cipher: 698 crypto_free_cipher(tfm); 699 return ret; 700 } 701 702 static int common_rfc4106_set_key(struct crypto_aead *aead, const u8 *key, 703 unsigned int key_len) 704 { 705 struct aesni_rfc4106_gcm_ctx *ctx = aesni_rfc4106_gcm_ctx_get(aead); 706 707 if (key_len < 4) { 708 crypto_aead_set_flags(aead, CRYPTO_TFM_RES_BAD_KEY_LEN); 709 return -EINVAL; 710 } 711 /*Account for 4 byte nonce at the end.*/ 712 key_len -= 4; 713 714 memcpy(ctx->nonce, key + key_len, sizeof(ctx->nonce)); 715 716 return aes_set_key_common(crypto_aead_tfm(aead), 717 &ctx->aes_key_expanded, key, key_len) ?: 718 rfc4106_set_hash_subkey(ctx->hash_subkey, key, key_len); 719 } 720 721 static int gcmaes_wrapper_set_key(struct crypto_aead *parent, const u8 *key, 722 unsigned int key_len) 723 { 724 struct cryptd_aead **ctx = crypto_aead_ctx(parent); 725 struct cryptd_aead *cryptd_tfm = *ctx; 726 727 return crypto_aead_setkey(&cryptd_tfm->base, key, key_len); 728 } 729 730 static int common_rfc4106_set_authsize(struct crypto_aead *aead, 731 unsigned int authsize) 732 { 733 switch (authsize) { 734 case 8: 735 case 12: 736 case 16: 737 break; 738 default: 739 return -EINVAL; 740 } 741 742 return 0; 743 } 744 745 /* This is the Integrity Check Value (aka the authentication tag length and can 746 * be 8, 12 or 16 bytes long. */ 747 static int gcmaes_wrapper_set_authsize(struct crypto_aead *parent, 748 unsigned int authsize) 749 { 750 struct cryptd_aead **ctx = crypto_aead_ctx(parent); 751 struct cryptd_aead *cryptd_tfm = *ctx; 752 753 return crypto_aead_setauthsize(&cryptd_tfm->base, authsize); 754 } 755 756 static int generic_gcmaes_set_authsize(struct crypto_aead *tfm, 757 unsigned int authsize) 758 { 759 switch (authsize) { 760 case 4: 761 case 8: 762 case 12: 763 case 13: 764 case 14: 765 case 15: 766 case 16: 767 break; 768 default: 769 return -EINVAL; 770 } 771 772 return 0; 773 } 774 775 static int gcmaes_crypt_by_sg(bool enc, struct aead_request *req, 776 unsigned int assoclen, u8 *hash_subkey, 777 u8 *iv, void *aes_ctx) 778 { 779 struct crypto_aead *tfm = crypto_aead_reqtfm(req); 780 unsigned long auth_tag_len = crypto_aead_authsize(tfm); 781 struct aesni_gcm_tfm_s *gcm_tfm = aesni_gcm_tfm; 782 struct gcm_context_data data AESNI_ALIGN_ATTR; 783 struct scatter_walk dst_sg_walk = {}; 784 unsigned long left = req->cryptlen; 785 unsigned long len, srclen, dstlen; 786 struct scatter_walk assoc_sg_walk; 787 struct scatter_walk src_sg_walk; 788 struct scatterlist src_start[2]; 789 struct scatterlist dst_start[2]; 790 struct scatterlist *src_sg; 791 struct scatterlist *dst_sg; 792 u8 *src, *dst, *assoc; 793 u8 *assocmem = NULL; 794 u8 authTag[16]; 795 796 if (!enc) 797 left -= auth_tag_len; 798 799 #ifdef CONFIG_AS_AVX2 800 if (left < AVX_GEN4_OPTSIZE && gcm_tfm == &aesni_gcm_tfm_avx_gen4) 801 gcm_tfm = &aesni_gcm_tfm_avx_gen2; 802 #endif 803 #ifdef CONFIG_AS_AVX 804 if (left < AVX_GEN2_OPTSIZE && gcm_tfm == &aesni_gcm_tfm_avx_gen2) 805 gcm_tfm = &aesni_gcm_tfm_sse; 806 #endif 807 808 /* Linearize assoc, if not already linear */ 809 if (req->src->length >= assoclen && req->src->length && 810 (!PageHighMem(sg_page(req->src)) || 811 req->src->offset + req->src->length <= PAGE_SIZE)) { 812 scatterwalk_start(&assoc_sg_walk, req->src); 813 assoc = scatterwalk_map(&assoc_sg_walk); 814 } else { 815 /* assoc can be any length, so must be on heap */ 816 assocmem = kmalloc(assoclen, GFP_ATOMIC); 817 if (unlikely(!assocmem)) 818 return -ENOMEM; 819 assoc = assocmem; 820 821 scatterwalk_map_and_copy(assoc, req->src, 0, assoclen, 0); 822 } 823 824 src_sg = scatterwalk_ffwd(src_start, req->src, req->assoclen); 825 scatterwalk_start(&src_sg_walk, src_sg); 826 if (req->src != req->dst) { 827 dst_sg = scatterwalk_ffwd(dst_start, req->dst, req->assoclen); 828 scatterwalk_start(&dst_sg_walk, dst_sg); 829 } 830 831 kernel_fpu_begin(); 832 gcm_tfm->init(aes_ctx, &data, iv, 833 hash_subkey, assoc, assoclen); 834 if (req->src != req->dst) { 835 while (left) { 836 src = scatterwalk_map(&src_sg_walk); 837 dst = scatterwalk_map(&dst_sg_walk); 838 srclen = scatterwalk_clamp(&src_sg_walk, left); 839 dstlen = scatterwalk_clamp(&dst_sg_walk, left); 840 len = min(srclen, dstlen); 841 if (len) { 842 if (enc) 843 gcm_tfm->enc_update(aes_ctx, &data, 844 dst, src, len); 845 else 846 gcm_tfm->dec_update(aes_ctx, &data, 847 dst, src, len); 848 } 849 left -= len; 850 851 scatterwalk_unmap(src); 852 scatterwalk_unmap(dst); 853 scatterwalk_advance(&src_sg_walk, len); 854 scatterwalk_advance(&dst_sg_walk, len); 855 scatterwalk_done(&src_sg_walk, 0, left); 856 scatterwalk_done(&dst_sg_walk, 1, left); 857 } 858 } else { 859 while (left) { 860 dst = src = scatterwalk_map(&src_sg_walk); 861 len = scatterwalk_clamp(&src_sg_walk, left); 862 if (len) { 863 if (enc) 864 gcm_tfm->enc_update(aes_ctx, &data, 865 src, src, len); 866 else 867 gcm_tfm->dec_update(aes_ctx, &data, 868 src, src, len); 869 } 870 left -= len; 871 scatterwalk_unmap(src); 872 scatterwalk_advance(&src_sg_walk, len); 873 scatterwalk_done(&src_sg_walk, 1, left); 874 } 875 } 876 gcm_tfm->finalize(aes_ctx, &data, authTag, auth_tag_len); 877 kernel_fpu_end(); 878 879 if (!assocmem) 880 scatterwalk_unmap(assoc); 881 else 882 kfree(assocmem); 883 884 if (!enc) { 885 u8 authTagMsg[16]; 886 887 /* Copy out original authTag */ 888 scatterwalk_map_and_copy(authTagMsg, req->src, 889 req->assoclen + req->cryptlen - 890 auth_tag_len, 891 auth_tag_len, 0); 892 893 /* Compare generated tag with passed in tag. */ 894 return crypto_memneq(authTagMsg, authTag, auth_tag_len) ? 895 -EBADMSG : 0; 896 } 897 898 /* Copy in the authTag */ 899 scatterwalk_map_and_copy(authTag, req->dst, 900 req->assoclen + req->cryptlen, 901 auth_tag_len, 1); 902 903 return 0; 904 } 905 906 static int gcmaes_encrypt(struct aead_request *req, unsigned int assoclen, 907 u8 *hash_subkey, u8 *iv, void *aes_ctx) 908 { 909 return gcmaes_crypt_by_sg(true, req, assoclen, hash_subkey, iv, 910 aes_ctx); 911 } 912 913 static int gcmaes_decrypt(struct aead_request *req, unsigned int assoclen, 914 u8 *hash_subkey, u8 *iv, void *aes_ctx) 915 { 916 return gcmaes_crypt_by_sg(false, req, assoclen, hash_subkey, iv, 917 aes_ctx); 918 } 919 920 static int helper_rfc4106_encrypt(struct aead_request *req) 921 { 922 struct crypto_aead *tfm = crypto_aead_reqtfm(req); 923 struct aesni_rfc4106_gcm_ctx *ctx = aesni_rfc4106_gcm_ctx_get(tfm); 924 void *aes_ctx = &(ctx->aes_key_expanded); 925 u8 iv[16] __attribute__ ((__aligned__(AESNI_ALIGN))); 926 unsigned int i; 927 __be32 counter = cpu_to_be32(1); 928 929 /* Assuming we are supporting rfc4106 64-bit extended */ 930 /* sequence numbers We need to have the AAD length equal */ 931 /* to 16 or 20 bytes */ 932 if (unlikely(req->assoclen != 16 && req->assoclen != 20)) 933 return -EINVAL; 934 935 /* IV below built */ 936 for (i = 0; i < 4; i++) 937 *(iv+i) = ctx->nonce[i]; 938 for (i = 0; i < 8; i++) 939 *(iv+4+i) = req->iv[i]; 940 *((__be32 *)(iv+12)) = counter; 941 942 return gcmaes_encrypt(req, req->assoclen - 8, ctx->hash_subkey, iv, 943 aes_ctx); 944 } 945 946 static int helper_rfc4106_decrypt(struct aead_request *req) 947 { 948 __be32 counter = cpu_to_be32(1); 949 struct crypto_aead *tfm = crypto_aead_reqtfm(req); 950 struct aesni_rfc4106_gcm_ctx *ctx = aesni_rfc4106_gcm_ctx_get(tfm); 951 void *aes_ctx = &(ctx->aes_key_expanded); 952 u8 iv[16] __attribute__ ((__aligned__(AESNI_ALIGN))); 953 unsigned int i; 954 955 if (unlikely(req->assoclen != 16 && req->assoclen != 20)) 956 return -EINVAL; 957 958 /* Assuming we are supporting rfc4106 64-bit extended */ 959 /* sequence numbers We need to have the AAD length */ 960 /* equal to 16 or 20 bytes */ 961 962 /* IV below built */ 963 for (i = 0; i < 4; i++) 964 *(iv+i) = ctx->nonce[i]; 965 for (i = 0; i < 8; i++) 966 *(iv+4+i) = req->iv[i]; 967 *((__be32 *)(iv+12)) = counter; 968 969 return gcmaes_decrypt(req, req->assoclen - 8, ctx->hash_subkey, iv, 970 aes_ctx); 971 } 972 973 static int gcmaes_wrapper_encrypt(struct aead_request *req) 974 { 975 struct crypto_aead *tfm = crypto_aead_reqtfm(req); 976 struct cryptd_aead **ctx = crypto_aead_ctx(tfm); 977 struct cryptd_aead *cryptd_tfm = *ctx; 978 979 tfm = &cryptd_tfm->base; 980 if (irq_fpu_usable() && (!in_atomic() || 981 !cryptd_aead_queued(cryptd_tfm))) 982 tfm = cryptd_aead_child(cryptd_tfm); 983 984 aead_request_set_tfm(req, tfm); 985 986 return crypto_aead_encrypt(req); 987 } 988 989 static int gcmaes_wrapper_decrypt(struct aead_request *req) 990 { 991 struct crypto_aead *tfm = crypto_aead_reqtfm(req); 992 struct cryptd_aead **ctx = crypto_aead_ctx(tfm); 993 struct cryptd_aead *cryptd_tfm = *ctx; 994 995 tfm = &cryptd_tfm->base; 996 if (irq_fpu_usable() && (!in_atomic() || 997 !cryptd_aead_queued(cryptd_tfm))) 998 tfm = cryptd_aead_child(cryptd_tfm); 999 1000 aead_request_set_tfm(req, tfm); 1001 1002 return crypto_aead_decrypt(req); 1003 } 1004 #endif 1005 1006 static struct crypto_alg aesni_algs[] = { { 1007 .cra_name = "aes", 1008 .cra_driver_name = "aes-aesni", 1009 .cra_priority = 300, 1010 .cra_flags = CRYPTO_ALG_TYPE_CIPHER, 1011 .cra_blocksize = AES_BLOCK_SIZE, 1012 .cra_ctxsize = CRYPTO_AES_CTX_SIZE, 1013 .cra_module = THIS_MODULE, 1014 .cra_u = { 1015 .cipher = { 1016 .cia_min_keysize = AES_MIN_KEY_SIZE, 1017 .cia_max_keysize = AES_MAX_KEY_SIZE, 1018 .cia_setkey = aes_set_key, 1019 .cia_encrypt = aes_encrypt, 1020 .cia_decrypt = aes_decrypt 1021 } 1022 } 1023 }, { 1024 .cra_name = "__aes", 1025 .cra_driver_name = "__aes-aesni", 1026 .cra_priority = 300, 1027 .cra_flags = CRYPTO_ALG_TYPE_CIPHER | CRYPTO_ALG_INTERNAL, 1028 .cra_blocksize = AES_BLOCK_SIZE, 1029 .cra_ctxsize = CRYPTO_AES_CTX_SIZE, 1030 .cra_module = THIS_MODULE, 1031 .cra_u = { 1032 .cipher = { 1033 .cia_min_keysize = AES_MIN_KEY_SIZE, 1034 .cia_max_keysize = AES_MAX_KEY_SIZE, 1035 .cia_setkey = aes_set_key, 1036 .cia_encrypt = __aes_encrypt, 1037 .cia_decrypt = __aes_decrypt 1038 } 1039 } 1040 } }; 1041 1042 static struct skcipher_alg aesni_skciphers[] = { 1043 { 1044 .base = { 1045 .cra_name = "__ecb(aes)", 1046 .cra_driver_name = "__ecb-aes-aesni", 1047 .cra_priority = 400, 1048 .cra_flags = CRYPTO_ALG_INTERNAL, 1049 .cra_blocksize = AES_BLOCK_SIZE, 1050 .cra_ctxsize = CRYPTO_AES_CTX_SIZE, 1051 .cra_module = THIS_MODULE, 1052 }, 1053 .min_keysize = AES_MIN_KEY_SIZE, 1054 .max_keysize = AES_MAX_KEY_SIZE, 1055 .setkey = aesni_skcipher_setkey, 1056 .encrypt = ecb_encrypt, 1057 .decrypt = ecb_decrypt, 1058 }, { 1059 .base = { 1060 .cra_name = "__cbc(aes)", 1061 .cra_driver_name = "__cbc-aes-aesni", 1062 .cra_priority = 400, 1063 .cra_flags = CRYPTO_ALG_INTERNAL, 1064 .cra_blocksize = AES_BLOCK_SIZE, 1065 .cra_ctxsize = CRYPTO_AES_CTX_SIZE, 1066 .cra_module = THIS_MODULE, 1067 }, 1068 .min_keysize = AES_MIN_KEY_SIZE, 1069 .max_keysize = AES_MAX_KEY_SIZE, 1070 .ivsize = AES_BLOCK_SIZE, 1071 .setkey = aesni_skcipher_setkey, 1072 .encrypt = cbc_encrypt, 1073 .decrypt = cbc_decrypt, 1074 #ifdef CONFIG_X86_64 1075 }, { 1076 .base = { 1077 .cra_name = "__ctr(aes)", 1078 .cra_driver_name = "__ctr-aes-aesni", 1079 .cra_priority = 400, 1080 .cra_flags = CRYPTO_ALG_INTERNAL, 1081 .cra_blocksize = 1, 1082 .cra_ctxsize = CRYPTO_AES_CTX_SIZE, 1083 .cra_module = THIS_MODULE, 1084 }, 1085 .min_keysize = AES_MIN_KEY_SIZE, 1086 .max_keysize = AES_MAX_KEY_SIZE, 1087 .ivsize = AES_BLOCK_SIZE, 1088 .chunksize = AES_BLOCK_SIZE, 1089 .setkey = aesni_skcipher_setkey, 1090 .encrypt = ctr_crypt, 1091 .decrypt = ctr_crypt, 1092 }, { 1093 .base = { 1094 .cra_name = "__xts(aes)", 1095 .cra_driver_name = "__xts-aes-aesni", 1096 .cra_priority = 401, 1097 .cra_flags = CRYPTO_ALG_INTERNAL, 1098 .cra_blocksize = AES_BLOCK_SIZE, 1099 .cra_ctxsize = XTS_AES_CTX_SIZE, 1100 .cra_module = THIS_MODULE, 1101 }, 1102 .min_keysize = 2 * AES_MIN_KEY_SIZE, 1103 .max_keysize = 2 * AES_MAX_KEY_SIZE, 1104 .ivsize = AES_BLOCK_SIZE, 1105 .setkey = xts_aesni_setkey, 1106 .encrypt = xts_encrypt, 1107 .decrypt = xts_decrypt, 1108 #endif 1109 } 1110 }; 1111 1112 static 1113 struct simd_skcipher_alg *aesni_simd_skciphers[ARRAY_SIZE(aesni_skciphers)]; 1114 1115 #ifdef CONFIG_X86_64 1116 static int generic_gcmaes_set_key(struct crypto_aead *aead, const u8 *key, 1117 unsigned int key_len) 1118 { 1119 struct generic_gcmaes_ctx *ctx = generic_gcmaes_ctx_get(aead); 1120 1121 return aes_set_key_common(crypto_aead_tfm(aead), 1122 &ctx->aes_key_expanded, key, key_len) ?: 1123 rfc4106_set_hash_subkey(ctx->hash_subkey, key, key_len); 1124 } 1125 1126 static int generic_gcmaes_encrypt(struct aead_request *req) 1127 { 1128 struct crypto_aead *tfm = crypto_aead_reqtfm(req); 1129 struct generic_gcmaes_ctx *ctx = generic_gcmaes_ctx_get(tfm); 1130 void *aes_ctx = &(ctx->aes_key_expanded); 1131 u8 iv[16] __attribute__ ((__aligned__(AESNI_ALIGN))); 1132 __be32 counter = cpu_to_be32(1); 1133 1134 memcpy(iv, req->iv, 12); 1135 *((__be32 *)(iv+12)) = counter; 1136 1137 return gcmaes_encrypt(req, req->assoclen, ctx->hash_subkey, iv, 1138 aes_ctx); 1139 } 1140 1141 static int generic_gcmaes_decrypt(struct aead_request *req) 1142 { 1143 __be32 counter = cpu_to_be32(1); 1144 struct crypto_aead *tfm = crypto_aead_reqtfm(req); 1145 struct generic_gcmaes_ctx *ctx = generic_gcmaes_ctx_get(tfm); 1146 void *aes_ctx = &(ctx->aes_key_expanded); 1147 u8 iv[16] __attribute__ ((__aligned__(AESNI_ALIGN))); 1148 1149 memcpy(iv, req->iv, 12); 1150 *((__be32 *)(iv+12)) = counter; 1151 1152 return gcmaes_decrypt(req, req->assoclen, ctx->hash_subkey, iv, 1153 aes_ctx); 1154 } 1155 1156 static int generic_gcmaes_init(struct crypto_aead *aead) 1157 { 1158 struct cryptd_aead *cryptd_tfm; 1159 struct cryptd_aead **ctx = crypto_aead_ctx(aead); 1160 1161 cryptd_tfm = cryptd_alloc_aead("__driver-generic-gcm-aes-aesni", 1162 CRYPTO_ALG_INTERNAL, 1163 CRYPTO_ALG_INTERNAL); 1164 if (IS_ERR(cryptd_tfm)) 1165 return PTR_ERR(cryptd_tfm); 1166 1167 *ctx = cryptd_tfm; 1168 crypto_aead_set_reqsize(aead, crypto_aead_reqsize(&cryptd_tfm->base)); 1169 1170 return 0; 1171 } 1172 1173 static void generic_gcmaes_exit(struct crypto_aead *aead) 1174 { 1175 struct cryptd_aead **ctx = crypto_aead_ctx(aead); 1176 1177 cryptd_free_aead(*ctx); 1178 } 1179 1180 static struct aead_alg aesni_aead_algs[] = { { 1181 .setkey = common_rfc4106_set_key, 1182 .setauthsize = common_rfc4106_set_authsize, 1183 .encrypt = helper_rfc4106_encrypt, 1184 .decrypt = helper_rfc4106_decrypt, 1185 .ivsize = GCM_RFC4106_IV_SIZE, 1186 .maxauthsize = 16, 1187 .base = { 1188 .cra_name = "__gcm-aes-aesni", 1189 .cra_driver_name = "__driver-gcm-aes-aesni", 1190 .cra_flags = CRYPTO_ALG_INTERNAL, 1191 .cra_blocksize = 1, 1192 .cra_ctxsize = sizeof(struct aesni_rfc4106_gcm_ctx), 1193 .cra_alignmask = AESNI_ALIGN - 1, 1194 .cra_module = THIS_MODULE, 1195 }, 1196 }, { 1197 .init = rfc4106_init, 1198 .exit = rfc4106_exit, 1199 .setkey = gcmaes_wrapper_set_key, 1200 .setauthsize = gcmaes_wrapper_set_authsize, 1201 .encrypt = gcmaes_wrapper_encrypt, 1202 .decrypt = gcmaes_wrapper_decrypt, 1203 .ivsize = GCM_RFC4106_IV_SIZE, 1204 .maxauthsize = 16, 1205 .base = { 1206 .cra_name = "rfc4106(gcm(aes))", 1207 .cra_driver_name = "rfc4106-gcm-aesni", 1208 .cra_priority = 400, 1209 .cra_flags = CRYPTO_ALG_ASYNC, 1210 .cra_blocksize = 1, 1211 .cra_ctxsize = sizeof(struct cryptd_aead *), 1212 .cra_module = THIS_MODULE, 1213 }, 1214 }, { 1215 .setkey = generic_gcmaes_set_key, 1216 .setauthsize = generic_gcmaes_set_authsize, 1217 .encrypt = generic_gcmaes_encrypt, 1218 .decrypt = generic_gcmaes_decrypt, 1219 .ivsize = GCM_AES_IV_SIZE, 1220 .maxauthsize = 16, 1221 .base = { 1222 .cra_name = "__generic-gcm-aes-aesni", 1223 .cra_driver_name = "__driver-generic-gcm-aes-aesni", 1224 .cra_priority = 0, 1225 .cra_flags = CRYPTO_ALG_INTERNAL, 1226 .cra_blocksize = 1, 1227 .cra_ctxsize = sizeof(struct generic_gcmaes_ctx), 1228 .cra_alignmask = AESNI_ALIGN - 1, 1229 .cra_module = THIS_MODULE, 1230 }, 1231 }, { 1232 .init = generic_gcmaes_init, 1233 .exit = generic_gcmaes_exit, 1234 .setkey = gcmaes_wrapper_set_key, 1235 .setauthsize = gcmaes_wrapper_set_authsize, 1236 .encrypt = gcmaes_wrapper_encrypt, 1237 .decrypt = gcmaes_wrapper_decrypt, 1238 .ivsize = GCM_AES_IV_SIZE, 1239 .maxauthsize = 16, 1240 .base = { 1241 .cra_name = "gcm(aes)", 1242 .cra_driver_name = "generic-gcm-aesni", 1243 .cra_priority = 400, 1244 .cra_flags = CRYPTO_ALG_ASYNC, 1245 .cra_blocksize = 1, 1246 .cra_ctxsize = sizeof(struct cryptd_aead *), 1247 .cra_module = THIS_MODULE, 1248 }, 1249 } }; 1250 #else 1251 static struct aead_alg aesni_aead_algs[0]; 1252 #endif 1253 1254 1255 static const struct x86_cpu_id aesni_cpu_id[] = { 1256 X86_FEATURE_MATCH(X86_FEATURE_AES), 1257 {} 1258 }; 1259 MODULE_DEVICE_TABLE(x86cpu, aesni_cpu_id); 1260 1261 static void aesni_free_simds(void) 1262 { 1263 int i; 1264 1265 for (i = 0; i < ARRAY_SIZE(aesni_simd_skciphers) && 1266 aesni_simd_skciphers[i]; i++) 1267 simd_skcipher_free(aesni_simd_skciphers[i]); 1268 } 1269 1270 static int __init aesni_init(void) 1271 { 1272 struct simd_skcipher_alg *simd; 1273 const char *basename; 1274 const char *algname; 1275 const char *drvname; 1276 int err; 1277 int i; 1278 1279 if (!x86_match_cpu(aesni_cpu_id)) 1280 return -ENODEV; 1281 #ifdef CONFIG_X86_64 1282 #ifdef CONFIG_AS_AVX2 1283 if (boot_cpu_has(X86_FEATURE_AVX2)) { 1284 pr_info("AVX2 version of gcm_enc/dec engaged.\n"); 1285 aesni_gcm_tfm = &aesni_gcm_tfm_avx_gen4; 1286 } else 1287 #endif 1288 #ifdef CONFIG_AS_AVX 1289 if (boot_cpu_has(X86_FEATURE_AVX)) { 1290 pr_info("AVX version of gcm_enc/dec engaged.\n"); 1291 aesni_gcm_tfm = &aesni_gcm_tfm_avx_gen2; 1292 } else 1293 #endif 1294 { 1295 pr_info("SSE version of gcm_enc/dec engaged.\n"); 1296 aesni_gcm_tfm = &aesni_gcm_tfm_sse; 1297 } 1298 aesni_ctr_enc_tfm = aesni_ctr_enc; 1299 #ifdef CONFIG_AS_AVX 1300 if (boot_cpu_has(X86_FEATURE_AVX)) { 1301 /* optimize performance of ctr mode encryption transform */ 1302 aesni_ctr_enc_tfm = aesni_ctr_enc_avx_tfm; 1303 pr_info("AES CTR mode by8 optimization enabled\n"); 1304 } 1305 #endif 1306 #endif 1307 1308 err = crypto_register_algs(aesni_algs, ARRAY_SIZE(aesni_algs)); 1309 if (err) 1310 return err; 1311 1312 err = crypto_register_skciphers(aesni_skciphers, 1313 ARRAY_SIZE(aesni_skciphers)); 1314 if (err) 1315 goto unregister_algs; 1316 1317 err = crypto_register_aeads(aesni_aead_algs, 1318 ARRAY_SIZE(aesni_aead_algs)); 1319 if (err) 1320 goto unregister_skciphers; 1321 1322 for (i = 0; i < ARRAY_SIZE(aesni_skciphers); i++) { 1323 algname = aesni_skciphers[i].base.cra_name + 2; 1324 drvname = aesni_skciphers[i].base.cra_driver_name + 2; 1325 basename = aesni_skciphers[i].base.cra_driver_name; 1326 simd = simd_skcipher_create_compat(algname, drvname, basename); 1327 err = PTR_ERR(simd); 1328 if (IS_ERR(simd)) 1329 goto unregister_simds; 1330 1331 aesni_simd_skciphers[i] = simd; 1332 } 1333 1334 return 0; 1335 1336 unregister_simds: 1337 aesni_free_simds(); 1338 crypto_unregister_aeads(aesni_aead_algs, ARRAY_SIZE(aesni_aead_algs)); 1339 unregister_skciphers: 1340 crypto_unregister_skciphers(aesni_skciphers, 1341 ARRAY_SIZE(aesni_skciphers)); 1342 unregister_algs: 1343 crypto_unregister_algs(aesni_algs, ARRAY_SIZE(aesni_algs)); 1344 return err; 1345 } 1346 1347 static void __exit aesni_exit(void) 1348 { 1349 aesni_free_simds(); 1350 crypto_unregister_aeads(aesni_aead_algs, ARRAY_SIZE(aesni_aead_algs)); 1351 crypto_unregister_skciphers(aesni_skciphers, 1352 ARRAY_SIZE(aesni_skciphers)); 1353 crypto_unregister_algs(aesni_algs, ARRAY_SIZE(aesni_algs)); 1354 } 1355 1356 late_initcall(aesni_init); 1357 module_exit(aesni_exit); 1358 1359 MODULE_DESCRIPTION("Rijndael (AES) Cipher Algorithm, Intel AES-NI instructions optimized"); 1360 MODULE_LICENSE("GPL"); 1361 MODULE_ALIAS_CRYPTO("aes"); 1362