1 /* 2 * Support for Intel AES-NI instructions. This file contains glue 3 * code, the real AES implementation is in intel-aes_asm.S. 4 * 5 * Copyright (C) 2008, Intel Corp. 6 * Author: Huang Ying <ying.huang@intel.com> 7 * 8 * Added RFC4106 AES-GCM support for 128-bit keys under the AEAD 9 * interface for 64-bit kernels. 10 * Authors: Adrian Hoban <adrian.hoban@intel.com> 11 * Gabriele Paoloni <gabriele.paoloni@intel.com> 12 * Tadeusz Struk (tadeusz.struk@intel.com) 13 * Aidan O'Mahony (aidan.o.mahony@intel.com) 14 * Copyright (c) 2010, Intel Corporation. 15 * 16 * This program is free software; you can redistribute it and/or modify 17 * it under the terms of the GNU General Public License as published by 18 * the Free Software Foundation; either version 2 of the License, or 19 * (at your option) any later version. 20 */ 21 22 #include <linux/hardirq.h> 23 #include <linux/types.h> 24 #include <linux/module.h> 25 #include <linux/err.h> 26 #include <crypto/algapi.h> 27 #include <crypto/aes.h> 28 #include <crypto/cryptd.h> 29 #include <crypto/ctr.h> 30 #include <crypto/b128ops.h> 31 #include <crypto/gcm.h> 32 #include <crypto/xts.h> 33 #include <asm/cpu_device_id.h> 34 #include <asm/fpu/api.h> 35 #include <asm/crypto/aes.h> 36 #include <crypto/scatterwalk.h> 37 #include <crypto/internal/aead.h> 38 #include <crypto/internal/simd.h> 39 #include <crypto/internal/skcipher.h> 40 #include <linux/workqueue.h> 41 #include <linux/spinlock.h> 42 #ifdef CONFIG_X86_64 43 #include <asm/crypto/glue_helper.h> 44 #endif 45 46 47 #define AESNI_ALIGN 16 48 #define AESNI_ALIGN_ATTR __attribute__ ((__aligned__(AESNI_ALIGN))) 49 #define AES_BLOCK_MASK (~(AES_BLOCK_SIZE - 1)) 50 #define RFC4106_HASH_SUBKEY_SIZE 16 51 #define AESNI_ALIGN_EXTRA ((AESNI_ALIGN - 1) & ~(CRYPTO_MINALIGN - 1)) 52 #define CRYPTO_AES_CTX_SIZE (sizeof(struct crypto_aes_ctx) + AESNI_ALIGN_EXTRA) 53 #define XTS_AES_CTX_SIZE (sizeof(struct aesni_xts_ctx) + AESNI_ALIGN_EXTRA) 54 55 /* This data is stored at the end of the crypto_tfm struct. 56 * It's a type of per "session" data storage location. 57 * This needs to be 16 byte aligned. 58 */ 59 struct aesni_rfc4106_gcm_ctx { 60 u8 hash_subkey[16] AESNI_ALIGN_ATTR; 61 struct crypto_aes_ctx aes_key_expanded AESNI_ALIGN_ATTR; 62 u8 nonce[4]; 63 }; 64 65 struct generic_gcmaes_ctx { 66 u8 hash_subkey[16] AESNI_ALIGN_ATTR; 67 struct crypto_aes_ctx aes_key_expanded AESNI_ALIGN_ATTR; 68 }; 69 70 struct aesni_xts_ctx { 71 u8 raw_tweak_ctx[sizeof(struct crypto_aes_ctx)] AESNI_ALIGN_ATTR; 72 u8 raw_crypt_ctx[sizeof(struct crypto_aes_ctx)] AESNI_ALIGN_ATTR; 73 }; 74 75 asmlinkage int aesni_set_key(struct crypto_aes_ctx *ctx, const u8 *in_key, 76 unsigned int key_len); 77 asmlinkage void aesni_enc(struct crypto_aes_ctx *ctx, u8 *out, 78 const u8 *in); 79 asmlinkage void aesni_dec(struct crypto_aes_ctx *ctx, u8 *out, 80 const u8 *in); 81 asmlinkage void aesni_ecb_enc(struct crypto_aes_ctx *ctx, u8 *out, 82 const u8 *in, unsigned int len); 83 asmlinkage void aesni_ecb_dec(struct crypto_aes_ctx *ctx, u8 *out, 84 const u8 *in, unsigned int len); 85 asmlinkage void aesni_cbc_enc(struct crypto_aes_ctx *ctx, u8 *out, 86 const u8 *in, unsigned int len, u8 *iv); 87 asmlinkage void aesni_cbc_dec(struct crypto_aes_ctx *ctx, u8 *out, 88 const u8 *in, unsigned int len, u8 *iv); 89 90 int crypto_fpu_init(void); 91 void crypto_fpu_exit(void); 92 93 #define AVX_GEN2_OPTSIZE 640 94 #define AVX_GEN4_OPTSIZE 4096 95 96 #ifdef CONFIG_X86_64 97 98 static void (*aesni_ctr_enc_tfm)(struct crypto_aes_ctx *ctx, u8 *out, 99 const u8 *in, unsigned int len, u8 *iv); 100 asmlinkage void aesni_ctr_enc(struct crypto_aes_ctx *ctx, u8 *out, 101 const u8 *in, unsigned int len, u8 *iv); 102 103 asmlinkage void aesni_xts_crypt8(struct crypto_aes_ctx *ctx, u8 *out, 104 const u8 *in, bool enc, u8 *iv); 105 106 /* asmlinkage void aesni_gcm_enc() 107 * void *ctx, AES Key schedule. Starts on a 16 byte boundary. 108 * u8 *out, Ciphertext output. Encrypt in-place is allowed. 109 * const u8 *in, Plaintext input 110 * unsigned long plaintext_len, Length of data in bytes for encryption. 111 * u8 *iv, Pre-counter block j0: 12 byte IV concatenated with 0x00000001. 112 * 16-byte aligned pointer. 113 * u8 *hash_subkey, the Hash sub key input. Data starts on a 16-byte boundary. 114 * const u8 *aad, Additional Authentication Data (AAD) 115 * unsigned long aad_len, Length of AAD in bytes. 116 * u8 *auth_tag, Authenticated Tag output. 117 * unsigned long auth_tag_len), Authenticated Tag Length in bytes. 118 * Valid values are 16 (most likely), 12 or 8. 119 */ 120 asmlinkage void aesni_gcm_enc(void *ctx, u8 *out, 121 const u8 *in, unsigned long plaintext_len, u8 *iv, 122 u8 *hash_subkey, const u8 *aad, unsigned long aad_len, 123 u8 *auth_tag, unsigned long auth_tag_len); 124 125 /* asmlinkage void aesni_gcm_dec() 126 * void *ctx, AES Key schedule. Starts on a 16 byte boundary. 127 * u8 *out, Plaintext output. Decrypt in-place is allowed. 128 * const u8 *in, Ciphertext input 129 * unsigned long ciphertext_len, Length of data in bytes for decryption. 130 * u8 *iv, Pre-counter block j0: 12 byte IV concatenated with 0x00000001. 131 * 16-byte aligned pointer. 132 * u8 *hash_subkey, the Hash sub key input. Data starts on a 16-byte boundary. 133 * const u8 *aad, Additional Authentication Data (AAD) 134 * unsigned long aad_len, Length of AAD in bytes. With RFC4106 this is going 135 * to be 8 or 12 bytes 136 * u8 *auth_tag, Authenticated Tag output. 137 * unsigned long auth_tag_len) Authenticated Tag Length in bytes. 138 * Valid values are 16 (most likely), 12 or 8. 139 */ 140 asmlinkage void aesni_gcm_dec(void *ctx, u8 *out, 141 const u8 *in, unsigned long ciphertext_len, u8 *iv, 142 u8 *hash_subkey, const u8 *aad, unsigned long aad_len, 143 u8 *auth_tag, unsigned long auth_tag_len); 144 145 146 #ifdef CONFIG_AS_AVX 147 asmlinkage void aes_ctr_enc_128_avx_by8(const u8 *in, u8 *iv, 148 void *keys, u8 *out, unsigned int num_bytes); 149 asmlinkage void aes_ctr_enc_192_avx_by8(const u8 *in, u8 *iv, 150 void *keys, u8 *out, unsigned int num_bytes); 151 asmlinkage void aes_ctr_enc_256_avx_by8(const u8 *in, u8 *iv, 152 void *keys, u8 *out, unsigned int num_bytes); 153 /* 154 * asmlinkage void aesni_gcm_precomp_avx_gen2() 155 * gcm_data *my_ctx_data, context data 156 * u8 *hash_subkey, the Hash sub key input. Data starts on a 16-byte boundary. 157 */ 158 asmlinkage void aesni_gcm_precomp_avx_gen2(void *my_ctx_data, u8 *hash_subkey); 159 160 asmlinkage void aesni_gcm_enc_avx_gen2(void *ctx, u8 *out, 161 const u8 *in, unsigned long plaintext_len, u8 *iv, 162 const u8 *aad, unsigned long aad_len, 163 u8 *auth_tag, unsigned long auth_tag_len); 164 165 asmlinkage void aesni_gcm_dec_avx_gen2(void *ctx, u8 *out, 166 const u8 *in, unsigned long ciphertext_len, u8 *iv, 167 const u8 *aad, unsigned long aad_len, 168 u8 *auth_tag, unsigned long auth_tag_len); 169 170 static void aesni_gcm_enc_avx(void *ctx, u8 *out, 171 const u8 *in, unsigned long plaintext_len, u8 *iv, 172 u8 *hash_subkey, const u8 *aad, unsigned long aad_len, 173 u8 *auth_tag, unsigned long auth_tag_len) 174 { 175 struct crypto_aes_ctx *aes_ctx = (struct crypto_aes_ctx*)ctx; 176 if ((plaintext_len < AVX_GEN2_OPTSIZE) || (aes_ctx-> key_length != AES_KEYSIZE_128)){ 177 aesni_gcm_enc(ctx, out, in, plaintext_len, iv, hash_subkey, aad, 178 aad_len, auth_tag, auth_tag_len); 179 } else { 180 aesni_gcm_precomp_avx_gen2(ctx, hash_subkey); 181 aesni_gcm_enc_avx_gen2(ctx, out, in, plaintext_len, iv, aad, 182 aad_len, auth_tag, auth_tag_len); 183 } 184 } 185 186 static void aesni_gcm_dec_avx(void *ctx, u8 *out, 187 const u8 *in, unsigned long ciphertext_len, u8 *iv, 188 u8 *hash_subkey, const u8 *aad, unsigned long aad_len, 189 u8 *auth_tag, unsigned long auth_tag_len) 190 { 191 struct crypto_aes_ctx *aes_ctx = (struct crypto_aes_ctx*)ctx; 192 if ((ciphertext_len < AVX_GEN2_OPTSIZE) || (aes_ctx-> key_length != AES_KEYSIZE_128)) { 193 aesni_gcm_dec(ctx, out, in, ciphertext_len, iv, hash_subkey, aad, 194 aad_len, auth_tag, auth_tag_len); 195 } else { 196 aesni_gcm_precomp_avx_gen2(ctx, hash_subkey); 197 aesni_gcm_dec_avx_gen2(ctx, out, in, ciphertext_len, iv, aad, 198 aad_len, auth_tag, auth_tag_len); 199 } 200 } 201 #endif 202 203 #ifdef CONFIG_AS_AVX2 204 /* 205 * asmlinkage void aesni_gcm_precomp_avx_gen4() 206 * gcm_data *my_ctx_data, context data 207 * u8 *hash_subkey, the Hash sub key input. Data starts on a 16-byte boundary. 208 */ 209 asmlinkage void aesni_gcm_precomp_avx_gen4(void *my_ctx_data, u8 *hash_subkey); 210 211 asmlinkage void aesni_gcm_enc_avx_gen4(void *ctx, u8 *out, 212 const u8 *in, unsigned long plaintext_len, u8 *iv, 213 const u8 *aad, unsigned long aad_len, 214 u8 *auth_tag, unsigned long auth_tag_len); 215 216 asmlinkage void aesni_gcm_dec_avx_gen4(void *ctx, u8 *out, 217 const u8 *in, unsigned long ciphertext_len, u8 *iv, 218 const u8 *aad, unsigned long aad_len, 219 u8 *auth_tag, unsigned long auth_tag_len); 220 221 static void aesni_gcm_enc_avx2(void *ctx, u8 *out, 222 const u8 *in, unsigned long plaintext_len, u8 *iv, 223 u8 *hash_subkey, const u8 *aad, unsigned long aad_len, 224 u8 *auth_tag, unsigned long auth_tag_len) 225 { 226 struct crypto_aes_ctx *aes_ctx = (struct crypto_aes_ctx*)ctx; 227 if ((plaintext_len < AVX_GEN2_OPTSIZE) || (aes_ctx-> key_length != AES_KEYSIZE_128)) { 228 aesni_gcm_enc(ctx, out, in, plaintext_len, iv, hash_subkey, aad, 229 aad_len, auth_tag, auth_tag_len); 230 } else if (plaintext_len < AVX_GEN4_OPTSIZE) { 231 aesni_gcm_precomp_avx_gen2(ctx, hash_subkey); 232 aesni_gcm_enc_avx_gen2(ctx, out, in, plaintext_len, iv, aad, 233 aad_len, auth_tag, auth_tag_len); 234 } else { 235 aesni_gcm_precomp_avx_gen4(ctx, hash_subkey); 236 aesni_gcm_enc_avx_gen4(ctx, out, in, plaintext_len, iv, aad, 237 aad_len, auth_tag, auth_tag_len); 238 } 239 } 240 241 static void aesni_gcm_dec_avx2(void *ctx, u8 *out, 242 const u8 *in, unsigned long ciphertext_len, u8 *iv, 243 u8 *hash_subkey, const u8 *aad, unsigned long aad_len, 244 u8 *auth_tag, unsigned long auth_tag_len) 245 { 246 struct crypto_aes_ctx *aes_ctx = (struct crypto_aes_ctx*)ctx; 247 if ((ciphertext_len < AVX_GEN2_OPTSIZE) || (aes_ctx-> key_length != AES_KEYSIZE_128)) { 248 aesni_gcm_dec(ctx, out, in, ciphertext_len, iv, hash_subkey, 249 aad, aad_len, auth_tag, auth_tag_len); 250 } else if (ciphertext_len < AVX_GEN4_OPTSIZE) { 251 aesni_gcm_precomp_avx_gen2(ctx, hash_subkey); 252 aesni_gcm_dec_avx_gen2(ctx, out, in, ciphertext_len, iv, aad, 253 aad_len, auth_tag, auth_tag_len); 254 } else { 255 aesni_gcm_precomp_avx_gen4(ctx, hash_subkey); 256 aesni_gcm_dec_avx_gen4(ctx, out, in, ciphertext_len, iv, aad, 257 aad_len, auth_tag, auth_tag_len); 258 } 259 } 260 #endif 261 262 static void (*aesni_gcm_enc_tfm)(void *ctx, u8 *out, 263 const u8 *in, unsigned long plaintext_len, u8 *iv, 264 u8 *hash_subkey, const u8 *aad, unsigned long aad_len, 265 u8 *auth_tag, unsigned long auth_tag_len); 266 267 static void (*aesni_gcm_dec_tfm)(void *ctx, u8 *out, 268 const u8 *in, unsigned long ciphertext_len, u8 *iv, 269 u8 *hash_subkey, const u8 *aad, unsigned long aad_len, 270 u8 *auth_tag, unsigned long auth_tag_len); 271 272 static inline struct 273 aesni_rfc4106_gcm_ctx *aesni_rfc4106_gcm_ctx_get(struct crypto_aead *tfm) 274 { 275 unsigned long align = AESNI_ALIGN; 276 277 if (align <= crypto_tfm_ctx_alignment()) 278 align = 1; 279 return PTR_ALIGN(crypto_aead_ctx(tfm), align); 280 } 281 282 static inline struct 283 generic_gcmaes_ctx *generic_gcmaes_ctx_get(struct crypto_aead *tfm) 284 { 285 unsigned long align = AESNI_ALIGN; 286 287 if (align <= crypto_tfm_ctx_alignment()) 288 align = 1; 289 return PTR_ALIGN(crypto_aead_ctx(tfm), align); 290 } 291 #endif 292 293 static inline struct crypto_aes_ctx *aes_ctx(void *raw_ctx) 294 { 295 unsigned long addr = (unsigned long)raw_ctx; 296 unsigned long align = AESNI_ALIGN; 297 298 if (align <= crypto_tfm_ctx_alignment()) 299 align = 1; 300 return (struct crypto_aes_ctx *)ALIGN(addr, align); 301 } 302 303 static int aes_set_key_common(struct crypto_tfm *tfm, void *raw_ctx, 304 const u8 *in_key, unsigned int key_len) 305 { 306 struct crypto_aes_ctx *ctx = aes_ctx(raw_ctx); 307 u32 *flags = &tfm->crt_flags; 308 int err; 309 310 if (key_len != AES_KEYSIZE_128 && key_len != AES_KEYSIZE_192 && 311 key_len != AES_KEYSIZE_256) { 312 *flags |= CRYPTO_TFM_RES_BAD_KEY_LEN; 313 return -EINVAL; 314 } 315 316 if (!irq_fpu_usable()) 317 err = crypto_aes_expand_key(ctx, in_key, key_len); 318 else { 319 kernel_fpu_begin(); 320 err = aesni_set_key(ctx, in_key, key_len); 321 kernel_fpu_end(); 322 } 323 324 return err; 325 } 326 327 static int aes_set_key(struct crypto_tfm *tfm, const u8 *in_key, 328 unsigned int key_len) 329 { 330 return aes_set_key_common(tfm, crypto_tfm_ctx(tfm), in_key, key_len); 331 } 332 333 static void aes_encrypt(struct crypto_tfm *tfm, u8 *dst, const u8 *src) 334 { 335 struct crypto_aes_ctx *ctx = aes_ctx(crypto_tfm_ctx(tfm)); 336 337 if (!irq_fpu_usable()) 338 crypto_aes_encrypt_x86(ctx, dst, src); 339 else { 340 kernel_fpu_begin(); 341 aesni_enc(ctx, dst, src); 342 kernel_fpu_end(); 343 } 344 } 345 346 static void aes_decrypt(struct crypto_tfm *tfm, u8 *dst, const u8 *src) 347 { 348 struct crypto_aes_ctx *ctx = aes_ctx(crypto_tfm_ctx(tfm)); 349 350 if (!irq_fpu_usable()) 351 crypto_aes_decrypt_x86(ctx, dst, src); 352 else { 353 kernel_fpu_begin(); 354 aesni_dec(ctx, dst, src); 355 kernel_fpu_end(); 356 } 357 } 358 359 static void __aes_encrypt(struct crypto_tfm *tfm, u8 *dst, const u8 *src) 360 { 361 struct crypto_aes_ctx *ctx = aes_ctx(crypto_tfm_ctx(tfm)); 362 363 aesni_enc(ctx, dst, src); 364 } 365 366 static void __aes_decrypt(struct crypto_tfm *tfm, u8 *dst, const u8 *src) 367 { 368 struct crypto_aes_ctx *ctx = aes_ctx(crypto_tfm_ctx(tfm)); 369 370 aesni_dec(ctx, dst, src); 371 } 372 373 static int aesni_skcipher_setkey(struct crypto_skcipher *tfm, const u8 *key, 374 unsigned int len) 375 { 376 return aes_set_key_common(crypto_skcipher_tfm(tfm), 377 crypto_skcipher_ctx(tfm), key, len); 378 } 379 380 static int ecb_encrypt(struct skcipher_request *req) 381 { 382 struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req); 383 struct crypto_aes_ctx *ctx = aes_ctx(crypto_skcipher_ctx(tfm)); 384 struct skcipher_walk walk; 385 unsigned int nbytes; 386 int err; 387 388 err = skcipher_walk_virt(&walk, req, true); 389 390 kernel_fpu_begin(); 391 while ((nbytes = walk.nbytes)) { 392 aesni_ecb_enc(ctx, walk.dst.virt.addr, walk.src.virt.addr, 393 nbytes & AES_BLOCK_MASK); 394 nbytes &= AES_BLOCK_SIZE - 1; 395 err = skcipher_walk_done(&walk, nbytes); 396 } 397 kernel_fpu_end(); 398 399 return err; 400 } 401 402 static int ecb_decrypt(struct skcipher_request *req) 403 { 404 struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req); 405 struct crypto_aes_ctx *ctx = aes_ctx(crypto_skcipher_ctx(tfm)); 406 struct skcipher_walk walk; 407 unsigned int nbytes; 408 int err; 409 410 err = skcipher_walk_virt(&walk, req, true); 411 412 kernel_fpu_begin(); 413 while ((nbytes = walk.nbytes)) { 414 aesni_ecb_dec(ctx, walk.dst.virt.addr, walk.src.virt.addr, 415 nbytes & AES_BLOCK_MASK); 416 nbytes &= AES_BLOCK_SIZE - 1; 417 err = skcipher_walk_done(&walk, nbytes); 418 } 419 kernel_fpu_end(); 420 421 return err; 422 } 423 424 static int cbc_encrypt(struct skcipher_request *req) 425 { 426 struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req); 427 struct crypto_aes_ctx *ctx = aes_ctx(crypto_skcipher_ctx(tfm)); 428 struct skcipher_walk walk; 429 unsigned int nbytes; 430 int err; 431 432 err = skcipher_walk_virt(&walk, req, true); 433 434 kernel_fpu_begin(); 435 while ((nbytes = walk.nbytes)) { 436 aesni_cbc_enc(ctx, walk.dst.virt.addr, walk.src.virt.addr, 437 nbytes & AES_BLOCK_MASK, walk.iv); 438 nbytes &= AES_BLOCK_SIZE - 1; 439 err = skcipher_walk_done(&walk, nbytes); 440 } 441 kernel_fpu_end(); 442 443 return err; 444 } 445 446 static int cbc_decrypt(struct skcipher_request *req) 447 { 448 struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req); 449 struct crypto_aes_ctx *ctx = aes_ctx(crypto_skcipher_ctx(tfm)); 450 struct skcipher_walk walk; 451 unsigned int nbytes; 452 int err; 453 454 err = skcipher_walk_virt(&walk, req, true); 455 456 kernel_fpu_begin(); 457 while ((nbytes = walk.nbytes)) { 458 aesni_cbc_dec(ctx, walk.dst.virt.addr, walk.src.virt.addr, 459 nbytes & AES_BLOCK_MASK, walk.iv); 460 nbytes &= AES_BLOCK_SIZE - 1; 461 err = skcipher_walk_done(&walk, nbytes); 462 } 463 kernel_fpu_end(); 464 465 return err; 466 } 467 468 #ifdef CONFIG_X86_64 469 static void ctr_crypt_final(struct crypto_aes_ctx *ctx, 470 struct skcipher_walk *walk) 471 { 472 u8 *ctrblk = walk->iv; 473 u8 keystream[AES_BLOCK_SIZE]; 474 u8 *src = walk->src.virt.addr; 475 u8 *dst = walk->dst.virt.addr; 476 unsigned int nbytes = walk->nbytes; 477 478 aesni_enc(ctx, keystream, ctrblk); 479 crypto_xor_cpy(dst, keystream, src, nbytes); 480 481 crypto_inc(ctrblk, AES_BLOCK_SIZE); 482 } 483 484 #ifdef CONFIG_AS_AVX 485 static void aesni_ctr_enc_avx_tfm(struct crypto_aes_ctx *ctx, u8 *out, 486 const u8 *in, unsigned int len, u8 *iv) 487 { 488 /* 489 * based on key length, override with the by8 version 490 * of ctr mode encryption/decryption for improved performance 491 * aes_set_key_common() ensures that key length is one of 492 * {128,192,256} 493 */ 494 if (ctx->key_length == AES_KEYSIZE_128) 495 aes_ctr_enc_128_avx_by8(in, iv, (void *)ctx, out, len); 496 else if (ctx->key_length == AES_KEYSIZE_192) 497 aes_ctr_enc_192_avx_by8(in, iv, (void *)ctx, out, len); 498 else 499 aes_ctr_enc_256_avx_by8(in, iv, (void *)ctx, out, len); 500 } 501 #endif 502 503 static int ctr_crypt(struct skcipher_request *req) 504 { 505 struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req); 506 struct crypto_aes_ctx *ctx = aes_ctx(crypto_skcipher_ctx(tfm)); 507 struct skcipher_walk walk; 508 unsigned int nbytes; 509 int err; 510 511 err = skcipher_walk_virt(&walk, req, true); 512 513 kernel_fpu_begin(); 514 while ((nbytes = walk.nbytes) >= AES_BLOCK_SIZE) { 515 aesni_ctr_enc_tfm(ctx, walk.dst.virt.addr, walk.src.virt.addr, 516 nbytes & AES_BLOCK_MASK, walk.iv); 517 nbytes &= AES_BLOCK_SIZE - 1; 518 err = skcipher_walk_done(&walk, nbytes); 519 } 520 if (walk.nbytes) { 521 ctr_crypt_final(ctx, &walk); 522 err = skcipher_walk_done(&walk, 0); 523 } 524 kernel_fpu_end(); 525 526 return err; 527 } 528 529 static int xts_aesni_setkey(struct crypto_skcipher *tfm, const u8 *key, 530 unsigned int keylen) 531 { 532 struct aesni_xts_ctx *ctx = crypto_skcipher_ctx(tfm); 533 int err; 534 535 err = xts_verify_key(tfm, key, keylen); 536 if (err) 537 return err; 538 539 keylen /= 2; 540 541 /* first half of xts-key is for crypt */ 542 err = aes_set_key_common(crypto_skcipher_tfm(tfm), ctx->raw_crypt_ctx, 543 key, keylen); 544 if (err) 545 return err; 546 547 /* second half of xts-key is for tweak */ 548 return aes_set_key_common(crypto_skcipher_tfm(tfm), ctx->raw_tweak_ctx, 549 key + keylen, keylen); 550 } 551 552 553 static void aesni_xts_tweak(void *ctx, u8 *out, const u8 *in) 554 { 555 aesni_enc(ctx, out, in); 556 } 557 558 static void aesni_xts_enc(void *ctx, u128 *dst, const u128 *src, le128 *iv) 559 { 560 glue_xts_crypt_128bit_one(ctx, dst, src, iv, GLUE_FUNC_CAST(aesni_enc)); 561 } 562 563 static void aesni_xts_dec(void *ctx, u128 *dst, const u128 *src, le128 *iv) 564 { 565 glue_xts_crypt_128bit_one(ctx, dst, src, iv, GLUE_FUNC_CAST(aesni_dec)); 566 } 567 568 static void aesni_xts_enc8(void *ctx, u128 *dst, const u128 *src, le128 *iv) 569 { 570 aesni_xts_crypt8(ctx, (u8 *)dst, (const u8 *)src, true, (u8 *)iv); 571 } 572 573 static void aesni_xts_dec8(void *ctx, u128 *dst, const u128 *src, le128 *iv) 574 { 575 aesni_xts_crypt8(ctx, (u8 *)dst, (const u8 *)src, false, (u8 *)iv); 576 } 577 578 static const struct common_glue_ctx aesni_enc_xts = { 579 .num_funcs = 2, 580 .fpu_blocks_limit = 1, 581 582 .funcs = { { 583 .num_blocks = 8, 584 .fn_u = { .xts = GLUE_XTS_FUNC_CAST(aesni_xts_enc8) } 585 }, { 586 .num_blocks = 1, 587 .fn_u = { .xts = GLUE_XTS_FUNC_CAST(aesni_xts_enc) } 588 } } 589 }; 590 591 static const struct common_glue_ctx aesni_dec_xts = { 592 .num_funcs = 2, 593 .fpu_blocks_limit = 1, 594 595 .funcs = { { 596 .num_blocks = 8, 597 .fn_u = { .xts = GLUE_XTS_FUNC_CAST(aesni_xts_dec8) } 598 }, { 599 .num_blocks = 1, 600 .fn_u = { .xts = GLUE_XTS_FUNC_CAST(aesni_xts_dec) } 601 } } 602 }; 603 604 static int xts_encrypt(struct skcipher_request *req) 605 { 606 struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req); 607 struct aesni_xts_ctx *ctx = crypto_skcipher_ctx(tfm); 608 609 return glue_xts_req_128bit(&aesni_enc_xts, req, 610 XTS_TWEAK_CAST(aesni_xts_tweak), 611 aes_ctx(ctx->raw_tweak_ctx), 612 aes_ctx(ctx->raw_crypt_ctx)); 613 } 614 615 static int xts_decrypt(struct skcipher_request *req) 616 { 617 struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req); 618 struct aesni_xts_ctx *ctx = crypto_skcipher_ctx(tfm); 619 620 return glue_xts_req_128bit(&aesni_dec_xts, req, 621 XTS_TWEAK_CAST(aesni_xts_tweak), 622 aes_ctx(ctx->raw_tweak_ctx), 623 aes_ctx(ctx->raw_crypt_ctx)); 624 } 625 626 static int rfc4106_init(struct crypto_aead *aead) 627 { 628 struct cryptd_aead *cryptd_tfm; 629 struct cryptd_aead **ctx = crypto_aead_ctx(aead); 630 631 cryptd_tfm = cryptd_alloc_aead("__driver-gcm-aes-aesni", 632 CRYPTO_ALG_INTERNAL, 633 CRYPTO_ALG_INTERNAL); 634 if (IS_ERR(cryptd_tfm)) 635 return PTR_ERR(cryptd_tfm); 636 637 *ctx = cryptd_tfm; 638 crypto_aead_set_reqsize(aead, crypto_aead_reqsize(&cryptd_tfm->base)); 639 return 0; 640 } 641 642 static void rfc4106_exit(struct crypto_aead *aead) 643 { 644 struct cryptd_aead **ctx = crypto_aead_ctx(aead); 645 646 cryptd_free_aead(*ctx); 647 } 648 649 static int 650 rfc4106_set_hash_subkey(u8 *hash_subkey, const u8 *key, unsigned int key_len) 651 { 652 struct crypto_cipher *tfm; 653 int ret; 654 655 tfm = crypto_alloc_cipher("aes", 0, 0); 656 if (IS_ERR(tfm)) 657 return PTR_ERR(tfm); 658 659 ret = crypto_cipher_setkey(tfm, key, key_len); 660 if (ret) 661 goto out_free_cipher; 662 663 /* Clear the data in the hash sub key container to zero.*/ 664 /* We want to cipher all zeros to create the hash sub key. */ 665 memset(hash_subkey, 0, RFC4106_HASH_SUBKEY_SIZE); 666 667 crypto_cipher_encrypt_one(tfm, hash_subkey, hash_subkey); 668 669 out_free_cipher: 670 crypto_free_cipher(tfm); 671 return ret; 672 } 673 674 static int common_rfc4106_set_key(struct crypto_aead *aead, const u8 *key, 675 unsigned int key_len) 676 { 677 struct aesni_rfc4106_gcm_ctx *ctx = aesni_rfc4106_gcm_ctx_get(aead); 678 679 if (key_len < 4) { 680 crypto_aead_set_flags(aead, CRYPTO_TFM_RES_BAD_KEY_LEN); 681 return -EINVAL; 682 } 683 /*Account for 4 byte nonce at the end.*/ 684 key_len -= 4; 685 686 memcpy(ctx->nonce, key + key_len, sizeof(ctx->nonce)); 687 688 return aes_set_key_common(crypto_aead_tfm(aead), 689 &ctx->aes_key_expanded, key, key_len) ?: 690 rfc4106_set_hash_subkey(ctx->hash_subkey, key, key_len); 691 } 692 693 static int gcmaes_wrapper_set_key(struct crypto_aead *parent, const u8 *key, 694 unsigned int key_len) 695 { 696 struct cryptd_aead **ctx = crypto_aead_ctx(parent); 697 struct cryptd_aead *cryptd_tfm = *ctx; 698 699 return crypto_aead_setkey(&cryptd_tfm->base, key, key_len); 700 } 701 702 static int common_rfc4106_set_authsize(struct crypto_aead *aead, 703 unsigned int authsize) 704 { 705 switch (authsize) { 706 case 8: 707 case 12: 708 case 16: 709 break; 710 default: 711 return -EINVAL; 712 } 713 714 return 0; 715 } 716 717 /* This is the Integrity Check Value (aka the authentication tag length and can 718 * be 8, 12 or 16 bytes long. */ 719 static int gcmaes_wrapper_set_authsize(struct crypto_aead *parent, 720 unsigned int authsize) 721 { 722 struct cryptd_aead **ctx = crypto_aead_ctx(parent); 723 struct cryptd_aead *cryptd_tfm = *ctx; 724 725 return crypto_aead_setauthsize(&cryptd_tfm->base, authsize); 726 } 727 728 static int generic_gcmaes_set_authsize(struct crypto_aead *tfm, 729 unsigned int authsize) 730 { 731 switch (authsize) { 732 case 4: 733 case 8: 734 case 12: 735 case 13: 736 case 14: 737 case 15: 738 case 16: 739 break; 740 default: 741 return -EINVAL; 742 } 743 744 return 0; 745 } 746 747 static int gcmaes_encrypt(struct aead_request *req, unsigned int assoclen, 748 u8 *hash_subkey, u8 *iv, void *aes_ctx) 749 { 750 u8 one_entry_in_sg = 0; 751 u8 *src, *dst, *assoc; 752 struct crypto_aead *tfm = crypto_aead_reqtfm(req); 753 unsigned long auth_tag_len = crypto_aead_authsize(tfm); 754 struct scatter_walk src_sg_walk; 755 struct scatter_walk dst_sg_walk = {}; 756 757 if (sg_is_last(req->src) && 758 (!PageHighMem(sg_page(req->src)) || 759 req->src->offset + req->src->length <= PAGE_SIZE) && 760 sg_is_last(req->dst) && 761 (!PageHighMem(sg_page(req->dst)) || 762 req->dst->offset + req->dst->length <= PAGE_SIZE)) { 763 one_entry_in_sg = 1; 764 scatterwalk_start(&src_sg_walk, req->src); 765 assoc = scatterwalk_map(&src_sg_walk); 766 src = assoc + req->assoclen; 767 dst = src; 768 if (unlikely(req->src != req->dst)) { 769 scatterwalk_start(&dst_sg_walk, req->dst); 770 dst = scatterwalk_map(&dst_sg_walk) + req->assoclen; 771 } 772 } else { 773 /* Allocate memory for src, dst, assoc */ 774 assoc = kmalloc(req->cryptlen + auth_tag_len + req->assoclen, 775 GFP_ATOMIC); 776 if (unlikely(!assoc)) 777 return -ENOMEM; 778 scatterwalk_map_and_copy(assoc, req->src, 0, 779 req->assoclen + req->cryptlen, 0); 780 src = assoc + req->assoclen; 781 dst = src; 782 } 783 784 kernel_fpu_begin(); 785 aesni_gcm_enc_tfm(aes_ctx, dst, src, req->cryptlen, iv, 786 hash_subkey, assoc, assoclen, 787 dst + req->cryptlen, auth_tag_len); 788 kernel_fpu_end(); 789 790 /* The authTag (aka the Integrity Check Value) needs to be written 791 * back to the packet. */ 792 if (one_entry_in_sg) { 793 if (unlikely(req->src != req->dst)) { 794 scatterwalk_unmap(dst - req->assoclen); 795 scatterwalk_advance(&dst_sg_walk, req->dst->length); 796 scatterwalk_done(&dst_sg_walk, 1, 0); 797 } 798 scatterwalk_unmap(assoc); 799 scatterwalk_advance(&src_sg_walk, req->src->length); 800 scatterwalk_done(&src_sg_walk, req->src == req->dst, 0); 801 } else { 802 scatterwalk_map_and_copy(dst, req->dst, req->assoclen, 803 req->cryptlen + auth_tag_len, 1); 804 kfree(assoc); 805 } 806 return 0; 807 } 808 809 static int gcmaes_decrypt(struct aead_request *req, unsigned int assoclen, 810 u8 *hash_subkey, u8 *iv, void *aes_ctx) 811 { 812 u8 one_entry_in_sg = 0; 813 u8 *src, *dst, *assoc; 814 unsigned long tempCipherLen = 0; 815 struct crypto_aead *tfm = crypto_aead_reqtfm(req); 816 unsigned long auth_tag_len = crypto_aead_authsize(tfm); 817 u8 authTag[16]; 818 struct scatter_walk src_sg_walk; 819 struct scatter_walk dst_sg_walk = {}; 820 int retval = 0; 821 822 tempCipherLen = (unsigned long)(req->cryptlen - auth_tag_len); 823 824 if (sg_is_last(req->src) && 825 (!PageHighMem(sg_page(req->src)) || 826 req->src->offset + req->src->length <= PAGE_SIZE) && 827 sg_is_last(req->dst) && req->dst->length && 828 (!PageHighMem(sg_page(req->dst)) || 829 req->dst->offset + req->dst->length <= PAGE_SIZE)) { 830 one_entry_in_sg = 1; 831 scatterwalk_start(&src_sg_walk, req->src); 832 assoc = scatterwalk_map(&src_sg_walk); 833 src = assoc + req->assoclen; 834 dst = src; 835 if (unlikely(req->src != req->dst)) { 836 scatterwalk_start(&dst_sg_walk, req->dst); 837 dst = scatterwalk_map(&dst_sg_walk) + req->assoclen; 838 } 839 } else { 840 /* Allocate memory for src, dst, assoc */ 841 assoc = kmalloc(req->cryptlen + req->assoclen, GFP_ATOMIC); 842 if (!assoc) 843 return -ENOMEM; 844 scatterwalk_map_and_copy(assoc, req->src, 0, 845 req->assoclen + req->cryptlen, 0); 846 src = assoc + req->assoclen; 847 dst = src; 848 } 849 850 851 kernel_fpu_begin(); 852 aesni_gcm_dec_tfm(aes_ctx, dst, src, tempCipherLen, iv, 853 hash_subkey, assoc, assoclen, 854 authTag, auth_tag_len); 855 kernel_fpu_end(); 856 857 /* Compare generated tag with passed in tag. */ 858 retval = crypto_memneq(src + tempCipherLen, authTag, auth_tag_len) ? 859 -EBADMSG : 0; 860 861 if (one_entry_in_sg) { 862 if (unlikely(req->src != req->dst)) { 863 scatterwalk_unmap(dst - req->assoclen); 864 scatterwalk_advance(&dst_sg_walk, req->dst->length); 865 scatterwalk_done(&dst_sg_walk, 1, 0); 866 } 867 scatterwalk_unmap(assoc); 868 scatterwalk_advance(&src_sg_walk, req->src->length); 869 scatterwalk_done(&src_sg_walk, req->src == req->dst, 0); 870 } else { 871 scatterwalk_map_and_copy(dst, req->dst, req->assoclen, 872 tempCipherLen, 1); 873 kfree(assoc); 874 } 875 return retval; 876 877 } 878 879 static int helper_rfc4106_encrypt(struct aead_request *req) 880 { 881 struct crypto_aead *tfm = crypto_aead_reqtfm(req); 882 struct aesni_rfc4106_gcm_ctx *ctx = aesni_rfc4106_gcm_ctx_get(tfm); 883 void *aes_ctx = &(ctx->aes_key_expanded); 884 u8 iv[16] __attribute__ ((__aligned__(AESNI_ALIGN))); 885 unsigned int i; 886 __be32 counter = cpu_to_be32(1); 887 888 /* Assuming we are supporting rfc4106 64-bit extended */ 889 /* sequence numbers We need to have the AAD length equal */ 890 /* to 16 or 20 bytes */ 891 if (unlikely(req->assoclen != 16 && req->assoclen != 20)) 892 return -EINVAL; 893 894 /* IV below built */ 895 for (i = 0; i < 4; i++) 896 *(iv+i) = ctx->nonce[i]; 897 for (i = 0; i < 8; i++) 898 *(iv+4+i) = req->iv[i]; 899 *((__be32 *)(iv+12)) = counter; 900 901 return gcmaes_encrypt(req, req->assoclen - 8, ctx->hash_subkey, iv, 902 aes_ctx); 903 } 904 905 static int helper_rfc4106_decrypt(struct aead_request *req) 906 { 907 __be32 counter = cpu_to_be32(1); 908 struct crypto_aead *tfm = crypto_aead_reqtfm(req); 909 struct aesni_rfc4106_gcm_ctx *ctx = aesni_rfc4106_gcm_ctx_get(tfm); 910 void *aes_ctx = &(ctx->aes_key_expanded); 911 u8 iv[16] __attribute__ ((__aligned__(AESNI_ALIGN))); 912 unsigned int i; 913 914 if (unlikely(req->assoclen != 16 && req->assoclen != 20)) 915 return -EINVAL; 916 917 /* Assuming we are supporting rfc4106 64-bit extended */ 918 /* sequence numbers We need to have the AAD length */ 919 /* equal to 16 or 20 bytes */ 920 921 /* IV below built */ 922 for (i = 0; i < 4; i++) 923 *(iv+i) = ctx->nonce[i]; 924 for (i = 0; i < 8; i++) 925 *(iv+4+i) = req->iv[i]; 926 *((__be32 *)(iv+12)) = counter; 927 928 return gcmaes_decrypt(req, req->assoclen - 8, ctx->hash_subkey, iv, 929 aes_ctx); 930 } 931 932 static int gcmaes_wrapper_encrypt(struct aead_request *req) 933 { 934 struct crypto_aead *tfm = crypto_aead_reqtfm(req); 935 struct cryptd_aead **ctx = crypto_aead_ctx(tfm); 936 struct cryptd_aead *cryptd_tfm = *ctx; 937 938 tfm = &cryptd_tfm->base; 939 if (irq_fpu_usable() && (!in_atomic() || 940 !cryptd_aead_queued(cryptd_tfm))) 941 tfm = cryptd_aead_child(cryptd_tfm); 942 943 aead_request_set_tfm(req, tfm); 944 945 return crypto_aead_encrypt(req); 946 } 947 948 static int gcmaes_wrapper_decrypt(struct aead_request *req) 949 { 950 struct crypto_aead *tfm = crypto_aead_reqtfm(req); 951 struct cryptd_aead **ctx = crypto_aead_ctx(tfm); 952 struct cryptd_aead *cryptd_tfm = *ctx; 953 954 tfm = &cryptd_tfm->base; 955 if (irq_fpu_usable() && (!in_atomic() || 956 !cryptd_aead_queued(cryptd_tfm))) 957 tfm = cryptd_aead_child(cryptd_tfm); 958 959 aead_request_set_tfm(req, tfm); 960 961 return crypto_aead_decrypt(req); 962 } 963 #endif 964 965 static struct crypto_alg aesni_algs[] = { { 966 .cra_name = "aes", 967 .cra_driver_name = "aes-aesni", 968 .cra_priority = 300, 969 .cra_flags = CRYPTO_ALG_TYPE_CIPHER, 970 .cra_blocksize = AES_BLOCK_SIZE, 971 .cra_ctxsize = CRYPTO_AES_CTX_SIZE, 972 .cra_module = THIS_MODULE, 973 .cra_u = { 974 .cipher = { 975 .cia_min_keysize = AES_MIN_KEY_SIZE, 976 .cia_max_keysize = AES_MAX_KEY_SIZE, 977 .cia_setkey = aes_set_key, 978 .cia_encrypt = aes_encrypt, 979 .cia_decrypt = aes_decrypt 980 } 981 } 982 }, { 983 .cra_name = "__aes", 984 .cra_driver_name = "__aes-aesni", 985 .cra_priority = 300, 986 .cra_flags = CRYPTO_ALG_TYPE_CIPHER | CRYPTO_ALG_INTERNAL, 987 .cra_blocksize = AES_BLOCK_SIZE, 988 .cra_ctxsize = CRYPTO_AES_CTX_SIZE, 989 .cra_module = THIS_MODULE, 990 .cra_u = { 991 .cipher = { 992 .cia_min_keysize = AES_MIN_KEY_SIZE, 993 .cia_max_keysize = AES_MAX_KEY_SIZE, 994 .cia_setkey = aes_set_key, 995 .cia_encrypt = __aes_encrypt, 996 .cia_decrypt = __aes_decrypt 997 } 998 } 999 } }; 1000 1001 static struct skcipher_alg aesni_skciphers[] = { 1002 { 1003 .base = { 1004 .cra_name = "__ecb(aes)", 1005 .cra_driver_name = "__ecb-aes-aesni", 1006 .cra_priority = 400, 1007 .cra_flags = CRYPTO_ALG_INTERNAL, 1008 .cra_blocksize = AES_BLOCK_SIZE, 1009 .cra_ctxsize = CRYPTO_AES_CTX_SIZE, 1010 .cra_module = THIS_MODULE, 1011 }, 1012 .min_keysize = AES_MIN_KEY_SIZE, 1013 .max_keysize = AES_MAX_KEY_SIZE, 1014 .setkey = aesni_skcipher_setkey, 1015 .encrypt = ecb_encrypt, 1016 .decrypt = ecb_decrypt, 1017 }, { 1018 .base = { 1019 .cra_name = "__cbc(aes)", 1020 .cra_driver_name = "__cbc-aes-aesni", 1021 .cra_priority = 400, 1022 .cra_flags = CRYPTO_ALG_INTERNAL, 1023 .cra_blocksize = AES_BLOCK_SIZE, 1024 .cra_ctxsize = CRYPTO_AES_CTX_SIZE, 1025 .cra_module = THIS_MODULE, 1026 }, 1027 .min_keysize = AES_MIN_KEY_SIZE, 1028 .max_keysize = AES_MAX_KEY_SIZE, 1029 .ivsize = AES_BLOCK_SIZE, 1030 .setkey = aesni_skcipher_setkey, 1031 .encrypt = cbc_encrypt, 1032 .decrypt = cbc_decrypt, 1033 #ifdef CONFIG_X86_64 1034 }, { 1035 .base = { 1036 .cra_name = "__ctr(aes)", 1037 .cra_driver_name = "__ctr-aes-aesni", 1038 .cra_priority = 400, 1039 .cra_flags = CRYPTO_ALG_INTERNAL, 1040 .cra_blocksize = 1, 1041 .cra_ctxsize = CRYPTO_AES_CTX_SIZE, 1042 .cra_module = THIS_MODULE, 1043 }, 1044 .min_keysize = AES_MIN_KEY_SIZE, 1045 .max_keysize = AES_MAX_KEY_SIZE, 1046 .ivsize = AES_BLOCK_SIZE, 1047 .chunksize = AES_BLOCK_SIZE, 1048 .setkey = aesni_skcipher_setkey, 1049 .encrypt = ctr_crypt, 1050 .decrypt = ctr_crypt, 1051 }, { 1052 .base = { 1053 .cra_name = "__xts(aes)", 1054 .cra_driver_name = "__xts-aes-aesni", 1055 .cra_priority = 401, 1056 .cra_flags = CRYPTO_ALG_INTERNAL, 1057 .cra_blocksize = AES_BLOCK_SIZE, 1058 .cra_ctxsize = XTS_AES_CTX_SIZE, 1059 .cra_module = THIS_MODULE, 1060 }, 1061 .min_keysize = 2 * AES_MIN_KEY_SIZE, 1062 .max_keysize = 2 * AES_MAX_KEY_SIZE, 1063 .ivsize = AES_BLOCK_SIZE, 1064 .setkey = xts_aesni_setkey, 1065 .encrypt = xts_encrypt, 1066 .decrypt = xts_decrypt, 1067 #endif 1068 } 1069 }; 1070 1071 static 1072 struct simd_skcipher_alg *aesni_simd_skciphers[ARRAY_SIZE(aesni_skciphers)]; 1073 1074 static struct { 1075 const char *algname; 1076 const char *drvname; 1077 const char *basename; 1078 struct simd_skcipher_alg *simd; 1079 } aesni_simd_skciphers2[] = { 1080 #if (defined(MODULE) && IS_ENABLED(CONFIG_CRYPTO_PCBC)) || \ 1081 IS_BUILTIN(CONFIG_CRYPTO_PCBC) 1082 { 1083 .algname = "pcbc(aes)", 1084 .drvname = "pcbc-aes-aesni", 1085 .basename = "fpu(pcbc(__aes-aesni))", 1086 }, 1087 #endif 1088 }; 1089 1090 #ifdef CONFIG_X86_64 1091 static int generic_gcmaes_set_key(struct crypto_aead *aead, const u8 *key, 1092 unsigned int key_len) 1093 { 1094 struct generic_gcmaes_ctx *ctx = generic_gcmaes_ctx_get(aead); 1095 1096 return aes_set_key_common(crypto_aead_tfm(aead), 1097 &ctx->aes_key_expanded, key, key_len) ?: 1098 rfc4106_set_hash_subkey(ctx->hash_subkey, key, key_len); 1099 } 1100 1101 static int generic_gcmaes_encrypt(struct aead_request *req) 1102 { 1103 struct crypto_aead *tfm = crypto_aead_reqtfm(req); 1104 struct generic_gcmaes_ctx *ctx = generic_gcmaes_ctx_get(tfm); 1105 void *aes_ctx = &(ctx->aes_key_expanded); 1106 u8 iv[16] __attribute__ ((__aligned__(AESNI_ALIGN))); 1107 __be32 counter = cpu_to_be32(1); 1108 1109 memcpy(iv, req->iv, 12); 1110 *((__be32 *)(iv+12)) = counter; 1111 1112 return gcmaes_encrypt(req, req->assoclen, ctx->hash_subkey, iv, 1113 aes_ctx); 1114 } 1115 1116 static int generic_gcmaes_decrypt(struct aead_request *req) 1117 { 1118 __be32 counter = cpu_to_be32(1); 1119 struct crypto_aead *tfm = crypto_aead_reqtfm(req); 1120 struct generic_gcmaes_ctx *ctx = generic_gcmaes_ctx_get(tfm); 1121 void *aes_ctx = &(ctx->aes_key_expanded); 1122 u8 iv[16] __attribute__ ((__aligned__(AESNI_ALIGN))); 1123 1124 memcpy(iv, req->iv, 12); 1125 *((__be32 *)(iv+12)) = counter; 1126 1127 return gcmaes_decrypt(req, req->assoclen, ctx->hash_subkey, iv, 1128 aes_ctx); 1129 } 1130 1131 static int generic_gcmaes_init(struct crypto_aead *aead) 1132 { 1133 struct cryptd_aead *cryptd_tfm; 1134 struct cryptd_aead **ctx = crypto_aead_ctx(aead); 1135 1136 cryptd_tfm = cryptd_alloc_aead("__driver-generic-gcm-aes-aesni", 1137 CRYPTO_ALG_INTERNAL, 1138 CRYPTO_ALG_INTERNAL); 1139 if (IS_ERR(cryptd_tfm)) 1140 return PTR_ERR(cryptd_tfm); 1141 1142 *ctx = cryptd_tfm; 1143 crypto_aead_set_reqsize(aead, crypto_aead_reqsize(&cryptd_tfm->base)); 1144 1145 return 0; 1146 } 1147 1148 static void generic_gcmaes_exit(struct crypto_aead *aead) 1149 { 1150 struct cryptd_aead **ctx = crypto_aead_ctx(aead); 1151 1152 cryptd_free_aead(*ctx); 1153 } 1154 1155 static struct aead_alg aesni_aead_algs[] = { { 1156 .setkey = common_rfc4106_set_key, 1157 .setauthsize = common_rfc4106_set_authsize, 1158 .encrypt = helper_rfc4106_encrypt, 1159 .decrypt = helper_rfc4106_decrypt, 1160 .ivsize = GCM_RFC4106_IV_SIZE, 1161 .maxauthsize = 16, 1162 .base = { 1163 .cra_name = "__gcm-aes-aesni", 1164 .cra_driver_name = "__driver-gcm-aes-aesni", 1165 .cra_flags = CRYPTO_ALG_INTERNAL, 1166 .cra_blocksize = 1, 1167 .cra_ctxsize = sizeof(struct aesni_rfc4106_gcm_ctx), 1168 .cra_alignmask = AESNI_ALIGN - 1, 1169 .cra_module = THIS_MODULE, 1170 }, 1171 }, { 1172 .init = rfc4106_init, 1173 .exit = rfc4106_exit, 1174 .setkey = gcmaes_wrapper_set_key, 1175 .setauthsize = gcmaes_wrapper_set_authsize, 1176 .encrypt = gcmaes_wrapper_encrypt, 1177 .decrypt = gcmaes_wrapper_decrypt, 1178 .ivsize = GCM_RFC4106_IV_SIZE, 1179 .maxauthsize = 16, 1180 .base = { 1181 .cra_name = "rfc4106(gcm(aes))", 1182 .cra_driver_name = "rfc4106-gcm-aesni", 1183 .cra_priority = 400, 1184 .cra_flags = CRYPTO_ALG_ASYNC, 1185 .cra_blocksize = 1, 1186 .cra_ctxsize = sizeof(struct cryptd_aead *), 1187 .cra_module = THIS_MODULE, 1188 }, 1189 }, { 1190 .setkey = generic_gcmaes_set_key, 1191 .setauthsize = generic_gcmaes_set_authsize, 1192 .encrypt = generic_gcmaes_encrypt, 1193 .decrypt = generic_gcmaes_decrypt, 1194 .ivsize = GCM_AES_IV_SIZE, 1195 .maxauthsize = 16, 1196 .base = { 1197 .cra_name = "__generic-gcm-aes-aesni", 1198 .cra_driver_name = "__driver-generic-gcm-aes-aesni", 1199 .cra_priority = 0, 1200 .cra_flags = CRYPTO_ALG_INTERNAL, 1201 .cra_blocksize = 1, 1202 .cra_ctxsize = sizeof(struct generic_gcmaes_ctx), 1203 .cra_alignmask = AESNI_ALIGN - 1, 1204 .cra_module = THIS_MODULE, 1205 }, 1206 }, { 1207 .init = generic_gcmaes_init, 1208 .exit = generic_gcmaes_exit, 1209 .setkey = gcmaes_wrapper_set_key, 1210 .setauthsize = gcmaes_wrapper_set_authsize, 1211 .encrypt = gcmaes_wrapper_encrypt, 1212 .decrypt = gcmaes_wrapper_decrypt, 1213 .ivsize = GCM_AES_IV_SIZE, 1214 .maxauthsize = 16, 1215 .base = { 1216 .cra_name = "gcm(aes)", 1217 .cra_driver_name = "generic-gcm-aesni", 1218 .cra_priority = 400, 1219 .cra_flags = CRYPTO_ALG_ASYNC, 1220 .cra_blocksize = 1, 1221 .cra_ctxsize = sizeof(struct cryptd_aead *), 1222 .cra_module = THIS_MODULE, 1223 }, 1224 } }; 1225 #else 1226 static struct aead_alg aesni_aead_algs[0]; 1227 #endif 1228 1229 1230 static const struct x86_cpu_id aesni_cpu_id[] = { 1231 X86_FEATURE_MATCH(X86_FEATURE_AES), 1232 {} 1233 }; 1234 MODULE_DEVICE_TABLE(x86cpu, aesni_cpu_id); 1235 1236 static void aesni_free_simds(void) 1237 { 1238 int i; 1239 1240 for (i = 0; i < ARRAY_SIZE(aesni_simd_skciphers) && 1241 aesni_simd_skciphers[i]; i++) 1242 simd_skcipher_free(aesni_simd_skciphers[i]); 1243 1244 for (i = 0; i < ARRAY_SIZE(aesni_simd_skciphers2); i++) 1245 if (aesni_simd_skciphers2[i].simd) 1246 simd_skcipher_free(aesni_simd_skciphers2[i].simd); 1247 } 1248 1249 static int __init aesni_init(void) 1250 { 1251 struct simd_skcipher_alg *simd; 1252 const char *basename; 1253 const char *algname; 1254 const char *drvname; 1255 int err; 1256 int i; 1257 1258 if (!x86_match_cpu(aesni_cpu_id)) 1259 return -ENODEV; 1260 #ifdef CONFIG_X86_64 1261 #ifdef CONFIG_AS_AVX2 1262 if (boot_cpu_has(X86_FEATURE_AVX2)) { 1263 pr_info("AVX2 version of gcm_enc/dec engaged.\n"); 1264 aesni_gcm_enc_tfm = aesni_gcm_enc_avx2; 1265 aesni_gcm_dec_tfm = aesni_gcm_dec_avx2; 1266 } else 1267 #endif 1268 #ifdef CONFIG_AS_AVX 1269 if (boot_cpu_has(X86_FEATURE_AVX)) { 1270 pr_info("AVX version of gcm_enc/dec engaged.\n"); 1271 aesni_gcm_enc_tfm = aesni_gcm_enc_avx; 1272 aesni_gcm_dec_tfm = aesni_gcm_dec_avx; 1273 } else 1274 #endif 1275 { 1276 pr_info("SSE version of gcm_enc/dec engaged.\n"); 1277 aesni_gcm_enc_tfm = aesni_gcm_enc; 1278 aesni_gcm_dec_tfm = aesni_gcm_dec; 1279 } 1280 aesni_ctr_enc_tfm = aesni_ctr_enc; 1281 #ifdef CONFIG_AS_AVX 1282 if (boot_cpu_has(X86_FEATURE_AVX)) { 1283 /* optimize performance of ctr mode encryption transform */ 1284 aesni_ctr_enc_tfm = aesni_ctr_enc_avx_tfm; 1285 pr_info("AES CTR mode by8 optimization enabled\n"); 1286 } 1287 #endif 1288 #endif 1289 1290 err = crypto_fpu_init(); 1291 if (err) 1292 return err; 1293 1294 err = crypto_register_algs(aesni_algs, ARRAY_SIZE(aesni_algs)); 1295 if (err) 1296 goto fpu_exit; 1297 1298 err = crypto_register_skciphers(aesni_skciphers, 1299 ARRAY_SIZE(aesni_skciphers)); 1300 if (err) 1301 goto unregister_algs; 1302 1303 err = crypto_register_aeads(aesni_aead_algs, 1304 ARRAY_SIZE(aesni_aead_algs)); 1305 if (err) 1306 goto unregister_skciphers; 1307 1308 for (i = 0; i < ARRAY_SIZE(aesni_skciphers); i++) { 1309 algname = aesni_skciphers[i].base.cra_name + 2; 1310 drvname = aesni_skciphers[i].base.cra_driver_name + 2; 1311 basename = aesni_skciphers[i].base.cra_driver_name; 1312 simd = simd_skcipher_create_compat(algname, drvname, basename); 1313 err = PTR_ERR(simd); 1314 if (IS_ERR(simd)) 1315 goto unregister_simds; 1316 1317 aesni_simd_skciphers[i] = simd; 1318 } 1319 1320 for (i = 0; i < ARRAY_SIZE(aesni_simd_skciphers2); i++) { 1321 algname = aesni_simd_skciphers2[i].algname; 1322 drvname = aesni_simd_skciphers2[i].drvname; 1323 basename = aesni_simd_skciphers2[i].basename; 1324 simd = simd_skcipher_create_compat(algname, drvname, basename); 1325 err = PTR_ERR(simd); 1326 if (IS_ERR(simd)) 1327 continue; 1328 1329 aesni_simd_skciphers2[i].simd = simd; 1330 } 1331 1332 return 0; 1333 1334 unregister_simds: 1335 aesni_free_simds(); 1336 crypto_unregister_aeads(aesni_aead_algs, ARRAY_SIZE(aesni_aead_algs)); 1337 unregister_skciphers: 1338 crypto_unregister_skciphers(aesni_skciphers, 1339 ARRAY_SIZE(aesni_skciphers)); 1340 unregister_algs: 1341 crypto_unregister_algs(aesni_algs, ARRAY_SIZE(aesni_algs)); 1342 fpu_exit: 1343 crypto_fpu_exit(); 1344 return err; 1345 } 1346 1347 static void __exit aesni_exit(void) 1348 { 1349 aesni_free_simds(); 1350 crypto_unregister_aeads(aesni_aead_algs, ARRAY_SIZE(aesni_aead_algs)); 1351 crypto_unregister_skciphers(aesni_skciphers, 1352 ARRAY_SIZE(aesni_skciphers)); 1353 crypto_unregister_algs(aesni_algs, ARRAY_SIZE(aesni_algs)); 1354 1355 crypto_fpu_exit(); 1356 } 1357 1358 late_initcall(aesni_init); 1359 module_exit(aesni_exit); 1360 1361 MODULE_DESCRIPTION("Rijndael (AES) Cipher Algorithm, Intel AES-NI instructions optimized"); 1362 MODULE_LICENSE("GPL"); 1363 MODULE_ALIAS_CRYPTO("aes"); 1364