1 /* 2 * Support for Intel AES-NI instructions. This file contains glue 3 * code, the real AES implementation is in intel-aes_asm.S. 4 * 5 * Copyright (C) 2008, Intel Corp. 6 * Author: Huang Ying <ying.huang@intel.com> 7 * 8 * Added RFC4106 AES-GCM support for 128-bit keys under the AEAD 9 * interface for 64-bit kernels. 10 * Authors: Adrian Hoban <adrian.hoban@intel.com> 11 * Gabriele Paoloni <gabriele.paoloni@intel.com> 12 * Tadeusz Struk (tadeusz.struk@intel.com) 13 * Aidan O'Mahony (aidan.o.mahony@intel.com) 14 * Copyright (c) 2010, Intel Corporation. 15 * 16 * This program is free software; you can redistribute it and/or modify 17 * it under the terms of the GNU General Public License as published by 18 * the Free Software Foundation; either version 2 of the License, or 19 * (at your option) any later version. 20 */ 21 22 #include <linux/hardirq.h> 23 #include <linux/types.h> 24 #include <linux/crypto.h> 25 #include <linux/module.h> 26 #include <linux/err.h> 27 #include <crypto/algapi.h> 28 #include <crypto/aes.h> 29 #include <crypto/cryptd.h> 30 #include <crypto/ctr.h> 31 #include <crypto/b128ops.h> 32 #include <crypto/lrw.h> 33 #include <crypto/xts.h> 34 #include <asm/cpu_device_id.h> 35 #include <asm/fpu/api.h> 36 #include <asm/crypto/aes.h> 37 #include <crypto/ablk_helper.h> 38 #include <crypto/scatterwalk.h> 39 #include <crypto/internal/aead.h> 40 #include <linux/workqueue.h> 41 #include <linux/spinlock.h> 42 #ifdef CONFIG_X86_64 43 #include <asm/crypto/glue_helper.h> 44 #endif 45 46 47 #define AESNI_ALIGN 16 48 #define AES_BLOCK_MASK (~(AES_BLOCK_SIZE - 1)) 49 #define RFC4106_HASH_SUBKEY_SIZE 16 50 51 /* This data is stored at the end of the crypto_tfm struct. 52 * It's a type of per "session" data storage location. 53 * This needs to be 16 byte aligned. 54 */ 55 struct aesni_rfc4106_gcm_ctx { 56 u8 hash_subkey[16] __attribute__ ((__aligned__(AESNI_ALIGN))); 57 struct crypto_aes_ctx aes_key_expanded 58 __attribute__ ((__aligned__(AESNI_ALIGN))); 59 u8 nonce[4]; 60 }; 61 62 struct aesni_lrw_ctx { 63 struct lrw_table_ctx lrw_table; 64 u8 raw_aes_ctx[sizeof(struct crypto_aes_ctx) + AESNI_ALIGN - 1]; 65 }; 66 67 struct aesni_xts_ctx { 68 u8 raw_tweak_ctx[sizeof(struct crypto_aes_ctx) + AESNI_ALIGN - 1]; 69 u8 raw_crypt_ctx[sizeof(struct crypto_aes_ctx) + AESNI_ALIGN - 1]; 70 }; 71 72 asmlinkage int aesni_set_key(struct crypto_aes_ctx *ctx, const u8 *in_key, 73 unsigned int key_len); 74 asmlinkage void aesni_enc(struct crypto_aes_ctx *ctx, u8 *out, 75 const u8 *in); 76 asmlinkage void aesni_dec(struct crypto_aes_ctx *ctx, u8 *out, 77 const u8 *in); 78 asmlinkage void aesni_ecb_enc(struct crypto_aes_ctx *ctx, u8 *out, 79 const u8 *in, unsigned int len); 80 asmlinkage void aesni_ecb_dec(struct crypto_aes_ctx *ctx, u8 *out, 81 const u8 *in, unsigned int len); 82 asmlinkage void aesni_cbc_enc(struct crypto_aes_ctx *ctx, u8 *out, 83 const u8 *in, unsigned int len, u8 *iv); 84 asmlinkage void aesni_cbc_dec(struct crypto_aes_ctx *ctx, u8 *out, 85 const u8 *in, unsigned int len, u8 *iv); 86 87 int crypto_fpu_init(void); 88 void crypto_fpu_exit(void); 89 90 #define AVX_GEN2_OPTSIZE 640 91 #define AVX_GEN4_OPTSIZE 4096 92 93 #ifdef CONFIG_X86_64 94 95 static void (*aesni_ctr_enc_tfm)(struct crypto_aes_ctx *ctx, u8 *out, 96 const u8 *in, unsigned int len, u8 *iv); 97 asmlinkage void aesni_ctr_enc(struct crypto_aes_ctx *ctx, u8 *out, 98 const u8 *in, unsigned int len, u8 *iv); 99 100 asmlinkage void aesni_xts_crypt8(struct crypto_aes_ctx *ctx, u8 *out, 101 const u8 *in, bool enc, u8 *iv); 102 103 /* asmlinkage void aesni_gcm_enc() 104 * void *ctx, AES Key schedule. Starts on a 16 byte boundary. 105 * u8 *out, Ciphertext output. Encrypt in-place is allowed. 106 * const u8 *in, Plaintext input 107 * unsigned long plaintext_len, Length of data in bytes for encryption. 108 * u8 *iv, Pre-counter block j0: 4 byte salt (from Security Association) 109 * concatenated with 8 byte Initialisation Vector (from IPSec ESP 110 * Payload) concatenated with 0x00000001. 16-byte aligned pointer. 111 * u8 *hash_subkey, the Hash sub key input. Data starts on a 16-byte boundary. 112 * const u8 *aad, Additional Authentication Data (AAD) 113 * unsigned long aad_len, Length of AAD in bytes. With RFC4106 this 114 * is going to be 8 or 12 bytes 115 * u8 *auth_tag, Authenticated Tag output. 116 * unsigned long auth_tag_len), Authenticated Tag Length in bytes. 117 * Valid values are 16 (most likely), 12 or 8. 118 */ 119 asmlinkage void aesni_gcm_enc(void *ctx, u8 *out, 120 const u8 *in, unsigned long plaintext_len, u8 *iv, 121 u8 *hash_subkey, const u8 *aad, unsigned long aad_len, 122 u8 *auth_tag, unsigned long auth_tag_len); 123 124 /* asmlinkage void aesni_gcm_dec() 125 * void *ctx, AES Key schedule. Starts on a 16 byte boundary. 126 * u8 *out, Plaintext output. Decrypt in-place is allowed. 127 * const u8 *in, Ciphertext input 128 * unsigned long ciphertext_len, Length of data in bytes for decryption. 129 * u8 *iv, Pre-counter block j0: 4 byte salt (from Security Association) 130 * concatenated with 8 byte Initialisation Vector (from IPSec ESP 131 * Payload) concatenated with 0x00000001. 16-byte aligned pointer. 132 * u8 *hash_subkey, the Hash sub key input. Data starts on a 16-byte boundary. 133 * const u8 *aad, Additional Authentication Data (AAD) 134 * unsigned long aad_len, Length of AAD in bytes. With RFC4106 this is going 135 * to be 8 or 12 bytes 136 * u8 *auth_tag, Authenticated Tag output. 137 * unsigned long auth_tag_len) Authenticated Tag Length in bytes. 138 * Valid values are 16 (most likely), 12 or 8. 139 */ 140 asmlinkage void aesni_gcm_dec(void *ctx, u8 *out, 141 const u8 *in, unsigned long ciphertext_len, u8 *iv, 142 u8 *hash_subkey, const u8 *aad, unsigned long aad_len, 143 u8 *auth_tag, unsigned long auth_tag_len); 144 145 146 #ifdef CONFIG_AS_AVX 147 asmlinkage void aes_ctr_enc_128_avx_by8(const u8 *in, u8 *iv, 148 void *keys, u8 *out, unsigned int num_bytes); 149 asmlinkage void aes_ctr_enc_192_avx_by8(const u8 *in, u8 *iv, 150 void *keys, u8 *out, unsigned int num_bytes); 151 asmlinkage void aes_ctr_enc_256_avx_by8(const u8 *in, u8 *iv, 152 void *keys, u8 *out, unsigned int num_bytes); 153 /* 154 * asmlinkage void aesni_gcm_precomp_avx_gen2() 155 * gcm_data *my_ctx_data, context data 156 * u8 *hash_subkey, the Hash sub key input. Data starts on a 16-byte boundary. 157 */ 158 asmlinkage void aesni_gcm_precomp_avx_gen2(void *my_ctx_data, u8 *hash_subkey); 159 160 asmlinkage void aesni_gcm_enc_avx_gen2(void *ctx, u8 *out, 161 const u8 *in, unsigned long plaintext_len, u8 *iv, 162 const u8 *aad, unsigned long aad_len, 163 u8 *auth_tag, unsigned long auth_tag_len); 164 165 asmlinkage void aesni_gcm_dec_avx_gen2(void *ctx, u8 *out, 166 const u8 *in, unsigned long ciphertext_len, u8 *iv, 167 const u8 *aad, unsigned long aad_len, 168 u8 *auth_tag, unsigned long auth_tag_len); 169 170 static void aesni_gcm_enc_avx(void *ctx, u8 *out, 171 const u8 *in, unsigned long plaintext_len, u8 *iv, 172 u8 *hash_subkey, const u8 *aad, unsigned long aad_len, 173 u8 *auth_tag, unsigned long auth_tag_len) 174 { 175 struct crypto_aes_ctx *aes_ctx = (struct crypto_aes_ctx*)ctx; 176 if ((plaintext_len < AVX_GEN2_OPTSIZE) || (aes_ctx-> key_length != AES_KEYSIZE_128)){ 177 aesni_gcm_enc(ctx, out, in, plaintext_len, iv, hash_subkey, aad, 178 aad_len, auth_tag, auth_tag_len); 179 } else { 180 aesni_gcm_precomp_avx_gen2(ctx, hash_subkey); 181 aesni_gcm_enc_avx_gen2(ctx, out, in, plaintext_len, iv, aad, 182 aad_len, auth_tag, auth_tag_len); 183 } 184 } 185 186 static void aesni_gcm_dec_avx(void *ctx, u8 *out, 187 const u8 *in, unsigned long ciphertext_len, u8 *iv, 188 u8 *hash_subkey, const u8 *aad, unsigned long aad_len, 189 u8 *auth_tag, unsigned long auth_tag_len) 190 { 191 struct crypto_aes_ctx *aes_ctx = (struct crypto_aes_ctx*)ctx; 192 if ((ciphertext_len < AVX_GEN2_OPTSIZE) || (aes_ctx-> key_length != AES_KEYSIZE_128)) { 193 aesni_gcm_dec(ctx, out, in, ciphertext_len, iv, hash_subkey, aad, 194 aad_len, auth_tag, auth_tag_len); 195 } else { 196 aesni_gcm_precomp_avx_gen2(ctx, hash_subkey); 197 aesni_gcm_dec_avx_gen2(ctx, out, in, ciphertext_len, iv, aad, 198 aad_len, auth_tag, auth_tag_len); 199 } 200 } 201 #endif 202 203 #ifdef CONFIG_AS_AVX2 204 /* 205 * asmlinkage void aesni_gcm_precomp_avx_gen4() 206 * gcm_data *my_ctx_data, context data 207 * u8 *hash_subkey, the Hash sub key input. Data starts on a 16-byte boundary. 208 */ 209 asmlinkage void aesni_gcm_precomp_avx_gen4(void *my_ctx_data, u8 *hash_subkey); 210 211 asmlinkage void aesni_gcm_enc_avx_gen4(void *ctx, u8 *out, 212 const u8 *in, unsigned long plaintext_len, u8 *iv, 213 const u8 *aad, unsigned long aad_len, 214 u8 *auth_tag, unsigned long auth_tag_len); 215 216 asmlinkage void aesni_gcm_dec_avx_gen4(void *ctx, u8 *out, 217 const u8 *in, unsigned long ciphertext_len, u8 *iv, 218 const u8 *aad, unsigned long aad_len, 219 u8 *auth_tag, unsigned long auth_tag_len); 220 221 static void aesni_gcm_enc_avx2(void *ctx, u8 *out, 222 const u8 *in, unsigned long plaintext_len, u8 *iv, 223 u8 *hash_subkey, const u8 *aad, unsigned long aad_len, 224 u8 *auth_tag, unsigned long auth_tag_len) 225 { 226 struct crypto_aes_ctx *aes_ctx = (struct crypto_aes_ctx*)ctx; 227 if ((plaintext_len < AVX_GEN2_OPTSIZE) || (aes_ctx-> key_length != AES_KEYSIZE_128)) { 228 aesni_gcm_enc(ctx, out, in, plaintext_len, iv, hash_subkey, aad, 229 aad_len, auth_tag, auth_tag_len); 230 } else if (plaintext_len < AVX_GEN4_OPTSIZE) { 231 aesni_gcm_precomp_avx_gen2(ctx, hash_subkey); 232 aesni_gcm_enc_avx_gen2(ctx, out, in, plaintext_len, iv, aad, 233 aad_len, auth_tag, auth_tag_len); 234 } else { 235 aesni_gcm_precomp_avx_gen4(ctx, hash_subkey); 236 aesni_gcm_enc_avx_gen4(ctx, out, in, plaintext_len, iv, aad, 237 aad_len, auth_tag, auth_tag_len); 238 } 239 } 240 241 static void aesni_gcm_dec_avx2(void *ctx, u8 *out, 242 const u8 *in, unsigned long ciphertext_len, u8 *iv, 243 u8 *hash_subkey, const u8 *aad, unsigned long aad_len, 244 u8 *auth_tag, unsigned long auth_tag_len) 245 { 246 struct crypto_aes_ctx *aes_ctx = (struct crypto_aes_ctx*)ctx; 247 if ((ciphertext_len < AVX_GEN2_OPTSIZE) || (aes_ctx-> key_length != AES_KEYSIZE_128)) { 248 aesni_gcm_dec(ctx, out, in, ciphertext_len, iv, hash_subkey, 249 aad, aad_len, auth_tag, auth_tag_len); 250 } else if (ciphertext_len < AVX_GEN4_OPTSIZE) { 251 aesni_gcm_precomp_avx_gen2(ctx, hash_subkey); 252 aesni_gcm_dec_avx_gen2(ctx, out, in, ciphertext_len, iv, aad, 253 aad_len, auth_tag, auth_tag_len); 254 } else { 255 aesni_gcm_precomp_avx_gen4(ctx, hash_subkey); 256 aesni_gcm_dec_avx_gen4(ctx, out, in, ciphertext_len, iv, aad, 257 aad_len, auth_tag, auth_tag_len); 258 } 259 } 260 #endif 261 262 static void (*aesni_gcm_enc_tfm)(void *ctx, u8 *out, 263 const u8 *in, unsigned long plaintext_len, u8 *iv, 264 u8 *hash_subkey, const u8 *aad, unsigned long aad_len, 265 u8 *auth_tag, unsigned long auth_tag_len); 266 267 static void (*aesni_gcm_dec_tfm)(void *ctx, u8 *out, 268 const u8 *in, unsigned long ciphertext_len, u8 *iv, 269 u8 *hash_subkey, const u8 *aad, unsigned long aad_len, 270 u8 *auth_tag, unsigned long auth_tag_len); 271 272 static inline struct 273 aesni_rfc4106_gcm_ctx *aesni_rfc4106_gcm_ctx_get(struct crypto_aead *tfm) 274 { 275 unsigned long align = AESNI_ALIGN; 276 277 if (align <= crypto_tfm_ctx_alignment()) 278 align = 1; 279 return PTR_ALIGN(crypto_aead_ctx(tfm), align); 280 } 281 #endif 282 283 static inline struct crypto_aes_ctx *aes_ctx(void *raw_ctx) 284 { 285 unsigned long addr = (unsigned long)raw_ctx; 286 unsigned long align = AESNI_ALIGN; 287 288 if (align <= crypto_tfm_ctx_alignment()) 289 align = 1; 290 return (struct crypto_aes_ctx *)ALIGN(addr, align); 291 } 292 293 static int aes_set_key_common(struct crypto_tfm *tfm, void *raw_ctx, 294 const u8 *in_key, unsigned int key_len) 295 { 296 struct crypto_aes_ctx *ctx = aes_ctx(raw_ctx); 297 u32 *flags = &tfm->crt_flags; 298 int err; 299 300 if (key_len != AES_KEYSIZE_128 && key_len != AES_KEYSIZE_192 && 301 key_len != AES_KEYSIZE_256) { 302 *flags |= CRYPTO_TFM_RES_BAD_KEY_LEN; 303 return -EINVAL; 304 } 305 306 if (!irq_fpu_usable()) 307 err = crypto_aes_expand_key(ctx, in_key, key_len); 308 else { 309 kernel_fpu_begin(); 310 err = aesni_set_key(ctx, in_key, key_len); 311 kernel_fpu_end(); 312 } 313 314 return err; 315 } 316 317 static int aes_set_key(struct crypto_tfm *tfm, const u8 *in_key, 318 unsigned int key_len) 319 { 320 return aes_set_key_common(tfm, crypto_tfm_ctx(tfm), in_key, key_len); 321 } 322 323 static void aes_encrypt(struct crypto_tfm *tfm, u8 *dst, const u8 *src) 324 { 325 struct crypto_aes_ctx *ctx = aes_ctx(crypto_tfm_ctx(tfm)); 326 327 if (!irq_fpu_usable()) 328 crypto_aes_encrypt_x86(ctx, dst, src); 329 else { 330 kernel_fpu_begin(); 331 aesni_enc(ctx, dst, src); 332 kernel_fpu_end(); 333 } 334 } 335 336 static void aes_decrypt(struct crypto_tfm *tfm, u8 *dst, const u8 *src) 337 { 338 struct crypto_aes_ctx *ctx = aes_ctx(crypto_tfm_ctx(tfm)); 339 340 if (!irq_fpu_usable()) 341 crypto_aes_decrypt_x86(ctx, dst, src); 342 else { 343 kernel_fpu_begin(); 344 aesni_dec(ctx, dst, src); 345 kernel_fpu_end(); 346 } 347 } 348 349 static void __aes_encrypt(struct crypto_tfm *tfm, u8 *dst, const u8 *src) 350 { 351 struct crypto_aes_ctx *ctx = aes_ctx(crypto_tfm_ctx(tfm)); 352 353 aesni_enc(ctx, dst, src); 354 } 355 356 static void __aes_decrypt(struct crypto_tfm *tfm, u8 *dst, const u8 *src) 357 { 358 struct crypto_aes_ctx *ctx = aes_ctx(crypto_tfm_ctx(tfm)); 359 360 aesni_dec(ctx, dst, src); 361 } 362 363 static int ecb_encrypt(struct blkcipher_desc *desc, 364 struct scatterlist *dst, struct scatterlist *src, 365 unsigned int nbytes) 366 { 367 struct crypto_aes_ctx *ctx = aes_ctx(crypto_blkcipher_ctx(desc->tfm)); 368 struct blkcipher_walk walk; 369 int err; 370 371 blkcipher_walk_init(&walk, dst, src, nbytes); 372 err = blkcipher_walk_virt(desc, &walk); 373 desc->flags &= ~CRYPTO_TFM_REQ_MAY_SLEEP; 374 375 kernel_fpu_begin(); 376 while ((nbytes = walk.nbytes)) { 377 aesni_ecb_enc(ctx, walk.dst.virt.addr, walk.src.virt.addr, 378 nbytes & AES_BLOCK_MASK); 379 nbytes &= AES_BLOCK_SIZE - 1; 380 err = blkcipher_walk_done(desc, &walk, nbytes); 381 } 382 kernel_fpu_end(); 383 384 return err; 385 } 386 387 static int ecb_decrypt(struct blkcipher_desc *desc, 388 struct scatterlist *dst, struct scatterlist *src, 389 unsigned int nbytes) 390 { 391 struct crypto_aes_ctx *ctx = aes_ctx(crypto_blkcipher_ctx(desc->tfm)); 392 struct blkcipher_walk walk; 393 int err; 394 395 blkcipher_walk_init(&walk, dst, src, nbytes); 396 err = blkcipher_walk_virt(desc, &walk); 397 desc->flags &= ~CRYPTO_TFM_REQ_MAY_SLEEP; 398 399 kernel_fpu_begin(); 400 while ((nbytes = walk.nbytes)) { 401 aesni_ecb_dec(ctx, walk.dst.virt.addr, walk.src.virt.addr, 402 nbytes & AES_BLOCK_MASK); 403 nbytes &= AES_BLOCK_SIZE - 1; 404 err = blkcipher_walk_done(desc, &walk, nbytes); 405 } 406 kernel_fpu_end(); 407 408 return err; 409 } 410 411 static int cbc_encrypt(struct blkcipher_desc *desc, 412 struct scatterlist *dst, struct scatterlist *src, 413 unsigned int nbytes) 414 { 415 struct crypto_aes_ctx *ctx = aes_ctx(crypto_blkcipher_ctx(desc->tfm)); 416 struct blkcipher_walk walk; 417 int err; 418 419 blkcipher_walk_init(&walk, dst, src, nbytes); 420 err = blkcipher_walk_virt(desc, &walk); 421 desc->flags &= ~CRYPTO_TFM_REQ_MAY_SLEEP; 422 423 kernel_fpu_begin(); 424 while ((nbytes = walk.nbytes)) { 425 aesni_cbc_enc(ctx, walk.dst.virt.addr, walk.src.virt.addr, 426 nbytes & AES_BLOCK_MASK, walk.iv); 427 nbytes &= AES_BLOCK_SIZE - 1; 428 err = blkcipher_walk_done(desc, &walk, nbytes); 429 } 430 kernel_fpu_end(); 431 432 return err; 433 } 434 435 static int cbc_decrypt(struct blkcipher_desc *desc, 436 struct scatterlist *dst, struct scatterlist *src, 437 unsigned int nbytes) 438 { 439 struct crypto_aes_ctx *ctx = aes_ctx(crypto_blkcipher_ctx(desc->tfm)); 440 struct blkcipher_walk walk; 441 int err; 442 443 blkcipher_walk_init(&walk, dst, src, nbytes); 444 err = blkcipher_walk_virt(desc, &walk); 445 desc->flags &= ~CRYPTO_TFM_REQ_MAY_SLEEP; 446 447 kernel_fpu_begin(); 448 while ((nbytes = walk.nbytes)) { 449 aesni_cbc_dec(ctx, walk.dst.virt.addr, walk.src.virt.addr, 450 nbytes & AES_BLOCK_MASK, walk.iv); 451 nbytes &= AES_BLOCK_SIZE - 1; 452 err = blkcipher_walk_done(desc, &walk, nbytes); 453 } 454 kernel_fpu_end(); 455 456 return err; 457 } 458 459 #ifdef CONFIG_X86_64 460 static void ctr_crypt_final(struct crypto_aes_ctx *ctx, 461 struct blkcipher_walk *walk) 462 { 463 u8 *ctrblk = walk->iv; 464 u8 keystream[AES_BLOCK_SIZE]; 465 u8 *src = walk->src.virt.addr; 466 u8 *dst = walk->dst.virt.addr; 467 unsigned int nbytes = walk->nbytes; 468 469 aesni_enc(ctx, keystream, ctrblk); 470 crypto_xor(keystream, src, nbytes); 471 memcpy(dst, keystream, nbytes); 472 crypto_inc(ctrblk, AES_BLOCK_SIZE); 473 } 474 475 #ifdef CONFIG_AS_AVX 476 static void aesni_ctr_enc_avx_tfm(struct crypto_aes_ctx *ctx, u8 *out, 477 const u8 *in, unsigned int len, u8 *iv) 478 { 479 /* 480 * based on key length, override with the by8 version 481 * of ctr mode encryption/decryption for improved performance 482 * aes_set_key_common() ensures that key length is one of 483 * {128,192,256} 484 */ 485 if (ctx->key_length == AES_KEYSIZE_128) 486 aes_ctr_enc_128_avx_by8(in, iv, (void *)ctx, out, len); 487 else if (ctx->key_length == AES_KEYSIZE_192) 488 aes_ctr_enc_192_avx_by8(in, iv, (void *)ctx, out, len); 489 else 490 aes_ctr_enc_256_avx_by8(in, iv, (void *)ctx, out, len); 491 } 492 #endif 493 494 static int ctr_crypt(struct blkcipher_desc *desc, 495 struct scatterlist *dst, struct scatterlist *src, 496 unsigned int nbytes) 497 { 498 struct crypto_aes_ctx *ctx = aes_ctx(crypto_blkcipher_ctx(desc->tfm)); 499 struct blkcipher_walk walk; 500 int err; 501 502 blkcipher_walk_init(&walk, dst, src, nbytes); 503 err = blkcipher_walk_virt_block(desc, &walk, AES_BLOCK_SIZE); 504 desc->flags &= ~CRYPTO_TFM_REQ_MAY_SLEEP; 505 506 kernel_fpu_begin(); 507 while ((nbytes = walk.nbytes) >= AES_BLOCK_SIZE) { 508 aesni_ctr_enc_tfm(ctx, walk.dst.virt.addr, walk.src.virt.addr, 509 nbytes & AES_BLOCK_MASK, walk.iv); 510 nbytes &= AES_BLOCK_SIZE - 1; 511 err = blkcipher_walk_done(desc, &walk, nbytes); 512 } 513 if (walk.nbytes) { 514 ctr_crypt_final(ctx, &walk); 515 err = blkcipher_walk_done(desc, &walk, 0); 516 } 517 kernel_fpu_end(); 518 519 return err; 520 } 521 #endif 522 523 static int ablk_ecb_init(struct crypto_tfm *tfm) 524 { 525 return ablk_init_common(tfm, "__driver-ecb-aes-aesni"); 526 } 527 528 static int ablk_cbc_init(struct crypto_tfm *tfm) 529 { 530 return ablk_init_common(tfm, "__driver-cbc-aes-aesni"); 531 } 532 533 #ifdef CONFIG_X86_64 534 static int ablk_ctr_init(struct crypto_tfm *tfm) 535 { 536 return ablk_init_common(tfm, "__driver-ctr-aes-aesni"); 537 } 538 539 #endif 540 541 #if IS_ENABLED(CONFIG_CRYPTO_PCBC) 542 static int ablk_pcbc_init(struct crypto_tfm *tfm) 543 { 544 return ablk_init_common(tfm, "fpu(pcbc(__driver-aes-aesni))"); 545 } 546 #endif 547 548 static void lrw_xts_encrypt_callback(void *ctx, u8 *blks, unsigned int nbytes) 549 { 550 aesni_ecb_enc(ctx, blks, blks, nbytes); 551 } 552 553 static void lrw_xts_decrypt_callback(void *ctx, u8 *blks, unsigned int nbytes) 554 { 555 aesni_ecb_dec(ctx, blks, blks, nbytes); 556 } 557 558 static int lrw_aesni_setkey(struct crypto_tfm *tfm, const u8 *key, 559 unsigned int keylen) 560 { 561 struct aesni_lrw_ctx *ctx = crypto_tfm_ctx(tfm); 562 int err; 563 564 err = aes_set_key_common(tfm, ctx->raw_aes_ctx, key, 565 keylen - AES_BLOCK_SIZE); 566 if (err) 567 return err; 568 569 return lrw_init_table(&ctx->lrw_table, key + keylen - AES_BLOCK_SIZE); 570 } 571 572 static void lrw_aesni_exit_tfm(struct crypto_tfm *tfm) 573 { 574 struct aesni_lrw_ctx *ctx = crypto_tfm_ctx(tfm); 575 576 lrw_free_table(&ctx->lrw_table); 577 } 578 579 static int lrw_encrypt(struct blkcipher_desc *desc, struct scatterlist *dst, 580 struct scatterlist *src, unsigned int nbytes) 581 { 582 struct aesni_lrw_ctx *ctx = crypto_blkcipher_ctx(desc->tfm); 583 be128 buf[8]; 584 struct lrw_crypt_req req = { 585 .tbuf = buf, 586 .tbuflen = sizeof(buf), 587 588 .table_ctx = &ctx->lrw_table, 589 .crypt_ctx = aes_ctx(ctx->raw_aes_ctx), 590 .crypt_fn = lrw_xts_encrypt_callback, 591 }; 592 int ret; 593 594 desc->flags &= ~CRYPTO_TFM_REQ_MAY_SLEEP; 595 596 kernel_fpu_begin(); 597 ret = lrw_crypt(desc, dst, src, nbytes, &req); 598 kernel_fpu_end(); 599 600 return ret; 601 } 602 603 static int lrw_decrypt(struct blkcipher_desc *desc, struct scatterlist *dst, 604 struct scatterlist *src, unsigned int nbytes) 605 { 606 struct aesni_lrw_ctx *ctx = crypto_blkcipher_ctx(desc->tfm); 607 be128 buf[8]; 608 struct lrw_crypt_req req = { 609 .tbuf = buf, 610 .tbuflen = sizeof(buf), 611 612 .table_ctx = &ctx->lrw_table, 613 .crypt_ctx = aes_ctx(ctx->raw_aes_ctx), 614 .crypt_fn = lrw_xts_decrypt_callback, 615 }; 616 int ret; 617 618 desc->flags &= ~CRYPTO_TFM_REQ_MAY_SLEEP; 619 620 kernel_fpu_begin(); 621 ret = lrw_crypt(desc, dst, src, nbytes, &req); 622 kernel_fpu_end(); 623 624 return ret; 625 } 626 627 static int xts_aesni_setkey(struct crypto_tfm *tfm, const u8 *key, 628 unsigned int keylen) 629 { 630 struct aesni_xts_ctx *ctx = crypto_tfm_ctx(tfm); 631 int err; 632 633 err = xts_check_key(tfm, key, keylen); 634 if (err) 635 return err; 636 637 /* first half of xts-key is for crypt */ 638 err = aes_set_key_common(tfm, ctx->raw_crypt_ctx, key, keylen / 2); 639 if (err) 640 return err; 641 642 /* second half of xts-key is for tweak */ 643 return aes_set_key_common(tfm, ctx->raw_tweak_ctx, key + keylen / 2, 644 keylen / 2); 645 } 646 647 648 static void aesni_xts_tweak(void *ctx, u8 *out, const u8 *in) 649 { 650 aesni_enc(ctx, out, in); 651 } 652 653 #ifdef CONFIG_X86_64 654 655 static void aesni_xts_enc(void *ctx, u128 *dst, const u128 *src, le128 *iv) 656 { 657 glue_xts_crypt_128bit_one(ctx, dst, src, iv, GLUE_FUNC_CAST(aesni_enc)); 658 } 659 660 static void aesni_xts_dec(void *ctx, u128 *dst, const u128 *src, le128 *iv) 661 { 662 glue_xts_crypt_128bit_one(ctx, dst, src, iv, GLUE_FUNC_CAST(aesni_dec)); 663 } 664 665 static void aesni_xts_enc8(void *ctx, u128 *dst, const u128 *src, le128 *iv) 666 { 667 aesni_xts_crypt8(ctx, (u8 *)dst, (const u8 *)src, true, (u8 *)iv); 668 } 669 670 static void aesni_xts_dec8(void *ctx, u128 *dst, const u128 *src, le128 *iv) 671 { 672 aesni_xts_crypt8(ctx, (u8 *)dst, (const u8 *)src, false, (u8 *)iv); 673 } 674 675 static const struct common_glue_ctx aesni_enc_xts = { 676 .num_funcs = 2, 677 .fpu_blocks_limit = 1, 678 679 .funcs = { { 680 .num_blocks = 8, 681 .fn_u = { .xts = GLUE_XTS_FUNC_CAST(aesni_xts_enc8) } 682 }, { 683 .num_blocks = 1, 684 .fn_u = { .xts = GLUE_XTS_FUNC_CAST(aesni_xts_enc) } 685 } } 686 }; 687 688 static const struct common_glue_ctx aesni_dec_xts = { 689 .num_funcs = 2, 690 .fpu_blocks_limit = 1, 691 692 .funcs = { { 693 .num_blocks = 8, 694 .fn_u = { .xts = GLUE_XTS_FUNC_CAST(aesni_xts_dec8) } 695 }, { 696 .num_blocks = 1, 697 .fn_u = { .xts = GLUE_XTS_FUNC_CAST(aesni_xts_dec) } 698 } } 699 }; 700 701 static int xts_encrypt(struct blkcipher_desc *desc, struct scatterlist *dst, 702 struct scatterlist *src, unsigned int nbytes) 703 { 704 struct aesni_xts_ctx *ctx = crypto_blkcipher_ctx(desc->tfm); 705 706 return glue_xts_crypt_128bit(&aesni_enc_xts, desc, dst, src, nbytes, 707 XTS_TWEAK_CAST(aesni_xts_tweak), 708 aes_ctx(ctx->raw_tweak_ctx), 709 aes_ctx(ctx->raw_crypt_ctx)); 710 } 711 712 static int xts_decrypt(struct blkcipher_desc *desc, struct scatterlist *dst, 713 struct scatterlist *src, unsigned int nbytes) 714 { 715 struct aesni_xts_ctx *ctx = crypto_blkcipher_ctx(desc->tfm); 716 717 return glue_xts_crypt_128bit(&aesni_dec_xts, desc, dst, src, nbytes, 718 XTS_TWEAK_CAST(aesni_xts_tweak), 719 aes_ctx(ctx->raw_tweak_ctx), 720 aes_ctx(ctx->raw_crypt_ctx)); 721 } 722 723 #else 724 725 static int xts_encrypt(struct blkcipher_desc *desc, struct scatterlist *dst, 726 struct scatterlist *src, unsigned int nbytes) 727 { 728 struct aesni_xts_ctx *ctx = crypto_blkcipher_ctx(desc->tfm); 729 be128 buf[8]; 730 struct xts_crypt_req req = { 731 .tbuf = buf, 732 .tbuflen = sizeof(buf), 733 734 .tweak_ctx = aes_ctx(ctx->raw_tweak_ctx), 735 .tweak_fn = aesni_xts_tweak, 736 .crypt_ctx = aes_ctx(ctx->raw_crypt_ctx), 737 .crypt_fn = lrw_xts_encrypt_callback, 738 }; 739 int ret; 740 741 desc->flags &= ~CRYPTO_TFM_REQ_MAY_SLEEP; 742 743 kernel_fpu_begin(); 744 ret = xts_crypt(desc, dst, src, nbytes, &req); 745 kernel_fpu_end(); 746 747 return ret; 748 } 749 750 static int xts_decrypt(struct blkcipher_desc *desc, struct scatterlist *dst, 751 struct scatterlist *src, unsigned int nbytes) 752 { 753 struct aesni_xts_ctx *ctx = crypto_blkcipher_ctx(desc->tfm); 754 be128 buf[8]; 755 struct xts_crypt_req req = { 756 .tbuf = buf, 757 .tbuflen = sizeof(buf), 758 759 .tweak_ctx = aes_ctx(ctx->raw_tweak_ctx), 760 .tweak_fn = aesni_xts_tweak, 761 .crypt_ctx = aes_ctx(ctx->raw_crypt_ctx), 762 .crypt_fn = lrw_xts_decrypt_callback, 763 }; 764 int ret; 765 766 desc->flags &= ~CRYPTO_TFM_REQ_MAY_SLEEP; 767 768 kernel_fpu_begin(); 769 ret = xts_crypt(desc, dst, src, nbytes, &req); 770 kernel_fpu_end(); 771 772 return ret; 773 } 774 775 #endif 776 777 #ifdef CONFIG_X86_64 778 static int rfc4106_init(struct crypto_aead *aead) 779 { 780 struct cryptd_aead *cryptd_tfm; 781 struct cryptd_aead **ctx = crypto_aead_ctx(aead); 782 783 cryptd_tfm = cryptd_alloc_aead("__driver-gcm-aes-aesni", 784 CRYPTO_ALG_INTERNAL, 785 CRYPTO_ALG_INTERNAL); 786 if (IS_ERR(cryptd_tfm)) 787 return PTR_ERR(cryptd_tfm); 788 789 *ctx = cryptd_tfm; 790 crypto_aead_set_reqsize(aead, crypto_aead_reqsize(&cryptd_tfm->base)); 791 return 0; 792 } 793 794 static void rfc4106_exit(struct crypto_aead *aead) 795 { 796 struct cryptd_aead **ctx = crypto_aead_ctx(aead); 797 798 cryptd_free_aead(*ctx); 799 } 800 801 static int 802 rfc4106_set_hash_subkey(u8 *hash_subkey, const u8 *key, unsigned int key_len) 803 { 804 struct crypto_cipher *tfm; 805 int ret; 806 807 tfm = crypto_alloc_cipher("aes", 0, 0); 808 if (IS_ERR(tfm)) 809 return PTR_ERR(tfm); 810 811 ret = crypto_cipher_setkey(tfm, key, key_len); 812 if (ret) 813 goto out_free_cipher; 814 815 /* Clear the data in the hash sub key container to zero.*/ 816 /* We want to cipher all zeros to create the hash sub key. */ 817 memset(hash_subkey, 0, RFC4106_HASH_SUBKEY_SIZE); 818 819 crypto_cipher_encrypt_one(tfm, hash_subkey, hash_subkey); 820 821 out_free_cipher: 822 crypto_free_cipher(tfm); 823 return ret; 824 } 825 826 static int common_rfc4106_set_key(struct crypto_aead *aead, const u8 *key, 827 unsigned int key_len) 828 { 829 struct aesni_rfc4106_gcm_ctx *ctx = aesni_rfc4106_gcm_ctx_get(aead); 830 831 if (key_len < 4) { 832 crypto_aead_set_flags(aead, CRYPTO_TFM_RES_BAD_KEY_LEN); 833 return -EINVAL; 834 } 835 /*Account for 4 byte nonce at the end.*/ 836 key_len -= 4; 837 838 memcpy(ctx->nonce, key + key_len, sizeof(ctx->nonce)); 839 840 return aes_set_key_common(crypto_aead_tfm(aead), 841 &ctx->aes_key_expanded, key, key_len) ?: 842 rfc4106_set_hash_subkey(ctx->hash_subkey, key, key_len); 843 } 844 845 static int rfc4106_set_key(struct crypto_aead *parent, const u8 *key, 846 unsigned int key_len) 847 { 848 struct cryptd_aead **ctx = crypto_aead_ctx(parent); 849 struct cryptd_aead *cryptd_tfm = *ctx; 850 851 return crypto_aead_setkey(&cryptd_tfm->base, key, key_len); 852 } 853 854 static int common_rfc4106_set_authsize(struct crypto_aead *aead, 855 unsigned int authsize) 856 { 857 switch (authsize) { 858 case 8: 859 case 12: 860 case 16: 861 break; 862 default: 863 return -EINVAL; 864 } 865 866 return 0; 867 } 868 869 /* This is the Integrity Check Value (aka the authentication tag length and can 870 * be 8, 12 or 16 bytes long. */ 871 static int rfc4106_set_authsize(struct crypto_aead *parent, 872 unsigned int authsize) 873 { 874 struct cryptd_aead **ctx = crypto_aead_ctx(parent); 875 struct cryptd_aead *cryptd_tfm = *ctx; 876 877 return crypto_aead_setauthsize(&cryptd_tfm->base, authsize); 878 } 879 880 static int helper_rfc4106_encrypt(struct aead_request *req) 881 { 882 u8 one_entry_in_sg = 0; 883 u8 *src, *dst, *assoc; 884 __be32 counter = cpu_to_be32(1); 885 struct crypto_aead *tfm = crypto_aead_reqtfm(req); 886 struct aesni_rfc4106_gcm_ctx *ctx = aesni_rfc4106_gcm_ctx_get(tfm); 887 void *aes_ctx = &(ctx->aes_key_expanded); 888 unsigned long auth_tag_len = crypto_aead_authsize(tfm); 889 u8 iv[16] __attribute__ ((__aligned__(AESNI_ALIGN))); 890 struct scatter_walk src_sg_walk; 891 struct scatter_walk dst_sg_walk; 892 unsigned int i; 893 894 /* Assuming we are supporting rfc4106 64-bit extended */ 895 /* sequence numbers We need to have the AAD length equal */ 896 /* to 16 or 20 bytes */ 897 if (unlikely(req->assoclen != 16 && req->assoclen != 20)) 898 return -EINVAL; 899 900 /* IV below built */ 901 for (i = 0; i < 4; i++) 902 *(iv+i) = ctx->nonce[i]; 903 for (i = 0; i < 8; i++) 904 *(iv+4+i) = req->iv[i]; 905 *((__be32 *)(iv+12)) = counter; 906 907 if (sg_is_last(req->src) && 908 req->src->offset + req->src->length <= PAGE_SIZE && 909 sg_is_last(req->dst) && 910 req->dst->offset + req->dst->length <= PAGE_SIZE) { 911 one_entry_in_sg = 1; 912 scatterwalk_start(&src_sg_walk, req->src); 913 assoc = scatterwalk_map(&src_sg_walk); 914 src = assoc + req->assoclen; 915 dst = src; 916 if (unlikely(req->src != req->dst)) { 917 scatterwalk_start(&dst_sg_walk, req->dst); 918 dst = scatterwalk_map(&dst_sg_walk) + req->assoclen; 919 } 920 } else { 921 /* Allocate memory for src, dst, assoc */ 922 assoc = kmalloc(req->cryptlen + auth_tag_len + req->assoclen, 923 GFP_ATOMIC); 924 if (unlikely(!assoc)) 925 return -ENOMEM; 926 scatterwalk_map_and_copy(assoc, req->src, 0, 927 req->assoclen + req->cryptlen, 0); 928 src = assoc + req->assoclen; 929 dst = src; 930 } 931 932 kernel_fpu_begin(); 933 aesni_gcm_enc_tfm(aes_ctx, dst, src, req->cryptlen, iv, 934 ctx->hash_subkey, assoc, req->assoclen - 8, 935 dst + req->cryptlen, auth_tag_len); 936 kernel_fpu_end(); 937 938 /* The authTag (aka the Integrity Check Value) needs to be written 939 * back to the packet. */ 940 if (one_entry_in_sg) { 941 if (unlikely(req->src != req->dst)) { 942 scatterwalk_unmap(dst - req->assoclen); 943 scatterwalk_advance(&dst_sg_walk, req->dst->length); 944 scatterwalk_done(&dst_sg_walk, 1, 0); 945 } 946 scatterwalk_unmap(assoc); 947 scatterwalk_advance(&src_sg_walk, req->src->length); 948 scatterwalk_done(&src_sg_walk, req->src == req->dst, 0); 949 } else { 950 scatterwalk_map_and_copy(dst, req->dst, req->assoclen, 951 req->cryptlen + auth_tag_len, 1); 952 kfree(assoc); 953 } 954 return 0; 955 } 956 957 static int helper_rfc4106_decrypt(struct aead_request *req) 958 { 959 u8 one_entry_in_sg = 0; 960 u8 *src, *dst, *assoc; 961 unsigned long tempCipherLen = 0; 962 __be32 counter = cpu_to_be32(1); 963 int retval = 0; 964 struct crypto_aead *tfm = crypto_aead_reqtfm(req); 965 struct aesni_rfc4106_gcm_ctx *ctx = aesni_rfc4106_gcm_ctx_get(tfm); 966 void *aes_ctx = &(ctx->aes_key_expanded); 967 unsigned long auth_tag_len = crypto_aead_authsize(tfm); 968 u8 iv[16] __attribute__ ((__aligned__(AESNI_ALIGN))); 969 u8 authTag[16]; 970 struct scatter_walk src_sg_walk; 971 struct scatter_walk dst_sg_walk; 972 unsigned int i; 973 974 if (unlikely(req->assoclen != 16 && req->assoclen != 20)) 975 return -EINVAL; 976 977 /* Assuming we are supporting rfc4106 64-bit extended */ 978 /* sequence numbers We need to have the AAD length */ 979 /* equal to 16 or 20 bytes */ 980 981 tempCipherLen = (unsigned long)(req->cryptlen - auth_tag_len); 982 /* IV below built */ 983 for (i = 0; i < 4; i++) 984 *(iv+i) = ctx->nonce[i]; 985 for (i = 0; i < 8; i++) 986 *(iv+4+i) = req->iv[i]; 987 *((__be32 *)(iv+12)) = counter; 988 989 if (sg_is_last(req->src) && 990 req->src->offset + req->src->length <= PAGE_SIZE && 991 sg_is_last(req->dst) && 992 req->dst->offset + req->dst->length <= PAGE_SIZE) { 993 one_entry_in_sg = 1; 994 scatterwalk_start(&src_sg_walk, req->src); 995 assoc = scatterwalk_map(&src_sg_walk); 996 src = assoc + req->assoclen; 997 dst = src; 998 if (unlikely(req->src != req->dst)) { 999 scatterwalk_start(&dst_sg_walk, req->dst); 1000 dst = scatterwalk_map(&dst_sg_walk) + req->assoclen; 1001 } 1002 1003 } else { 1004 /* Allocate memory for src, dst, assoc */ 1005 assoc = kmalloc(req->cryptlen + req->assoclen, GFP_ATOMIC); 1006 if (!assoc) 1007 return -ENOMEM; 1008 scatterwalk_map_and_copy(assoc, req->src, 0, 1009 req->assoclen + req->cryptlen, 0); 1010 src = assoc + req->assoclen; 1011 dst = src; 1012 } 1013 1014 kernel_fpu_begin(); 1015 aesni_gcm_dec_tfm(aes_ctx, dst, src, tempCipherLen, iv, 1016 ctx->hash_subkey, assoc, req->assoclen - 8, 1017 authTag, auth_tag_len); 1018 kernel_fpu_end(); 1019 1020 /* Compare generated tag with passed in tag. */ 1021 retval = crypto_memneq(src + tempCipherLen, authTag, auth_tag_len) ? 1022 -EBADMSG : 0; 1023 1024 if (one_entry_in_sg) { 1025 if (unlikely(req->src != req->dst)) { 1026 scatterwalk_unmap(dst - req->assoclen); 1027 scatterwalk_advance(&dst_sg_walk, req->dst->length); 1028 scatterwalk_done(&dst_sg_walk, 1, 0); 1029 } 1030 scatterwalk_unmap(assoc); 1031 scatterwalk_advance(&src_sg_walk, req->src->length); 1032 scatterwalk_done(&src_sg_walk, req->src == req->dst, 0); 1033 } else { 1034 scatterwalk_map_and_copy(dst, req->dst, req->assoclen, 1035 tempCipherLen, 1); 1036 kfree(assoc); 1037 } 1038 return retval; 1039 } 1040 1041 static int rfc4106_encrypt(struct aead_request *req) 1042 { 1043 struct crypto_aead *tfm = crypto_aead_reqtfm(req); 1044 struct cryptd_aead **ctx = crypto_aead_ctx(tfm); 1045 struct cryptd_aead *cryptd_tfm = *ctx; 1046 1047 tfm = &cryptd_tfm->base; 1048 if (irq_fpu_usable() && (!in_atomic() || 1049 !cryptd_aead_queued(cryptd_tfm))) 1050 tfm = cryptd_aead_child(cryptd_tfm); 1051 1052 aead_request_set_tfm(req, tfm); 1053 1054 return crypto_aead_encrypt(req); 1055 } 1056 1057 static int rfc4106_decrypt(struct aead_request *req) 1058 { 1059 struct crypto_aead *tfm = crypto_aead_reqtfm(req); 1060 struct cryptd_aead **ctx = crypto_aead_ctx(tfm); 1061 struct cryptd_aead *cryptd_tfm = *ctx; 1062 1063 tfm = &cryptd_tfm->base; 1064 if (irq_fpu_usable() && (!in_atomic() || 1065 !cryptd_aead_queued(cryptd_tfm))) 1066 tfm = cryptd_aead_child(cryptd_tfm); 1067 1068 aead_request_set_tfm(req, tfm); 1069 1070 return crypto_aead_decrypt(req); 1071 } 1072 #endif 1073 1074 static struct crypto_alg aesni_algs[] = { { 1075 .cra_name = "aes", 1076 .cra_driver_name = "aes-aesni", 1077 .cra_priority = 300, 1078 .cra_flags = CRYPTO_ALG_TYPE_CIPHER, 1079 .cra_blocksize = AES_BLOCK_SIZE, 1080 .cra_ctxsize = sizeof(struct crypto_aes_ctx) + 1081 AESNI_ALIGN - 1, 1082 .cra_alignmask = 0, 1083 .cra_module = THIS_MODULE, 1084 .cra_u = { 1085 .cipher = { 1086 .cia_min_keysize = AES_MIN_KEY_SIZE, 1087 .cia_max_keysize = AES_MAX_KEY_SIZE, 1088 .cia_setkey = aes_set_key, 1089 .cia_encrypt = aes_encrypt, 1090 .cia_decrypt = aes_decrypt 1091 } 1092 } 1093 }, { 1094 .cra_name = "__aes-aesni", 1095 .cra_driver_name = "__driver-aes-aesni", 1096 .cra_priority = 0, 1097 .cra_flags = CRYPTO_ALG_TYPE_CIPHER | CRYPTO_ALG_INTERNAL, 1098 .cra_blocksize = AES_BLOCK_SIZE, 1099 .cra_ctxsize = sizeof(struct crypto_aes_ctx) + 1100 AESNI_ALIGN - 1, 1101 .cra_alignmask = 0, 1102 .cra_module = THIS_MODULE, 1103 .cra_u = { 1104 .cipher = { 1105 .cia_min_keysize = AES_MIN_KEY_SIZE, 1106 .cia_max_keysize = AES_MAX_KEY_SIZE, 1107 .cia_setkey = aes_set_key, 1108 .cia_encrypt = __aes_encrypt, 1109 .cia_decrypt = __aes_decrypt 1110 } 1111 } 1112 }, { 1113 .cra_name = "__ecb-aes-aesni", 1114 .cra_driver_name = "__driver-ecb-aes-aesni", 1115 .cra_priority = 0, 1116 .cra_flags = CRYPTO_ALG_TYPE_BLKCIPHER | 1117 CRYPTO_ALG_INTERNAL, 1118 .cra_blocksize = AES_BLOCK_SIZE, 1119 .cra_ctxsize = sizeof(struct crypto_aes_ctx) + 1120 AESNI_ALIGN - 1, 1121 .cra_alignmask = 0, 1122 .cra_type = &crypto_blkcipher_type, 1123 .cra_module = THIS_MODULE, 1124 .cra_u = { 1125 .blkcipher = { 1126 .min_keysize = AES_MIN_KEY_SIZE, 1127 .max_keysize = AES_MAX_KEY_SIZE, 1128 .setkey = aes_set_key, 1129 .encrypt = ecb_encrypt, 1130 .decrypt = ecb_decrypt, 1131 }, 1132 }, 1133 }, { 1134 .cra_name = "__cbc-aes-aesni", 1135 .cra_driver_name = "__driver-cbc-aes-aesni", 1136 .cra_priority = 0, 1137 .cra_flags = CRYPTO_ALG_TYPE_BLKCIPHER | 1138 CRYPTO_ALG_INTERNAL, 1139 .cra_blocksize = AES_BLOCK_SIZE, 1140 .cra_ctxsize = sizeof(struct crypto_aes_ctx) + 1141 AESNI_ALIGN - 1, 1142 .cra_alignmask = 0, 1143 .cra_type = &crypto_blkcipher_type, 1144 .cra_module = THIS_MODULE, 1145 .cra_u = { 1146 .blkcipher = { 1147 .min_keysize = AES_MIN_KEY_SIZE, 1148 .max_keysize = AES_MAX_KEY_SIZE, 1149 .setkey = aes_set_key, 1150 .encrypt = cbc_encrypt, 1151 .decrypt = cbc_decrypt, 1152 }, 1153 }, 1154 }, { 1155 .cra_name = "ecb(aes)", 1156 .cra_driver_name = "ecb-aes-aesni", 1157 .cra_priority = 400, 1158 .cra_flags = CRYPTO_ALG_TYPE_ABLKCIPHER | CRYPTO_ALG_ASYNC, 1159 .cra_blocksize = AES_BLOCK_SIZE, 1160 .cra_ctxsize = sizeof(struct async_helper_ctx), 1161 .cra_alignmask = 0, 1162 .cra_type = &crypto_ablkcipher_type, 1163 .cra_module = THIS_MODULE, 1164 .cra_init = ablk_ecb_init, 1165 .cra_exit = ablk_exit, 1166 .cra_u = { 1167 .ablkcipher = { 1168 .min_keysize = AES_MIN_KEY_SIZE, 1169 .max_keysize = AES_MAX_KEY_SIZE, 1170 .setkey = ablk_set_key, 1171 .encrypt = ablk_encrypt, 1172 .decrypt = ablk_decrypt, 1173 }, 1174 }, 1175 }, { 1176 .cra_name = "cbc(aes)", 1177 .cra_driver_name = "cbc-aes-aesni", 1178 .cra_priority = 400, 1179 .cra_flags = CRYPTO_ALG_TYPE_ABLKCIPHER | CRYPTO_ALG_ASYNC, 1180 .cra_blocksize = AES_BLOCK_SIZE, 1181 .cra_ctxsize = sizeof(struct async_helper_ctx), 1182 .cra_alignmask = 0, 1183 .cra_type = &crypto_ablkcipher_type, 1184 .cra_module = THIS_MODULE, 1185 .cra_init = ablk_cbc_init, 1186 .cra_exit = ablk_exit, 1187 .cra_u = { 1188 .ablkcipher = { 1189 .min_keysize = AES_MIN_KEY_SIZE, 1190 .max_keysize = AES_MAX_KEY_SIZE, 1191 .ivsize = AES_BLOCK_SIZE, 1192 .setkey = ablk_set_key, 1193 .encrypt = ablk_encrypt, 1194 .decrypt = ablk_decrypt, 1195 }, 1196 }, 1197 #ifdef CONFIG_X86_64 1198 }, { 1199 .cra_name = "__ctr-aes-aesni", 1200 .cra_driver_name = "__driver-ctr-aes-aesni", 1201 .cra_priority = 0, 1202 .cra_flags = CRYPTO_ALG_TYPE_BLKCIPHER | 1203 CRYPTO_ALG_INTERNAL, 1204 .cra_blocksize = 1, 1205 .cra_ctxsize = sizeof(struct crypto_aes_ctx) + 1206 AESNI_ALIGN - 1, 1207 .cra_alignmask = 0, 1208 .cra_type = &crypto_blkcipher_type, 1209 .cra_module = THIS_MODULE, 1210 .cra_u = { 1211 .blkcipher = { 1212 .min_keysize = AES_MIN_KEY_SIZE, 1213 .max_keysize = AES_MAX_KEY_SIZE, 1214 .ivsize = AES_BLOCK_SIZE, 1215 .setkey = aes_set_key, 1216 .encrypt = ctr_crypt, 1217 .decrypt = ctr_crypt, 1218 }, 1219 }, 1220 }, { 1221 .cra_name = "ctr(aes)", 1222 .cra_driver_name = "ctr-aes-aesni", 1223 .cra_priority = 400, 1224 .cra_flags = CRYPTO_ALG_TYPE_ABLKCIPHER | CRYPTO_ALG_ASYNC, 1225 .cra_blocksize = 1, 1226 .cra_ctxsize = sizeof(struct async_helper_ctx), 1227 .cra_alignmask = 0, 1228 .cra_type = &crypto_ablkcipher_type, 1229 .cra_module = THIS_MODULE, 1230 .cra_init = ablk_ctr_init, 1231 .cra_exit = ablk_exit, 1232 .cra_u = { 1233 .ablkcipher = { 1234 .min_keysize = AES_MIN_KEY_SIZE, 1235 .max_keysize = AES_MAX_KEY_SIZE, 1236 .ivsize = AES_BLOCK_SIZE, 1237 .setkey = ablk_set_key, 1238 .encrypt = ablk_encrypt, 1239 .decrypt = ablk_encrypt, 1240 .geniv = "chainiv", 1241 }, 1242 }, 1243 #endif 1244 #if IS_ENABLED(CONFIG_CRYPTO_PCBC) 1245 }, { 1246 .cra_name = "pcbc(aes)", 1247 .cra_driver_name = "pcbc-aes-aesni", 1248 .cra_priority = 400, 1249 .cra_flags = CRYPTO_ALG_TYPE_ABLKCIPHER | CRYPTO_ALG_ASYNC, 1250 .cra_blocksize = AES_BLOCK_SIZE, 1251 .cra_ctxsize = sizeof(struct async_helper_ctx), 1252 .cra_alignmask = 0, 1253 .cra_type = &crypto_ablkcipher_type, 1254 .cra_module = THIS_MODULE, 1255 .cra_init = ablk_pcbc_init, 1256 .cra_exit = ablk_exit, 1257 .cra_u = { 1258 .ablkcipher = { 1259 .min_keysize = AES_MIN_KEY_SIZE, 1260 .max_keysize = AES_MAX_KEY_SIZE, 1261 .ivsize = AES_BLOCK_SIZE, 1262 .setkey = ablk_set_key, 1263 .encrypt = ablk_encrypt, 1264 .decrypt = ablk_decrypt, 1265 }, 1266 }, 1267 #endif 1268 }, { 1269 .cra_name = "__lrw-aes-aesni", 1270 .cra_driver_name = "__driver-lrw-aes-aesni", 1271 .cra_priority = 0, 1272 .cra_flags = CRYPTO_ALG_TYPE_BLKCIPHER | 1273 CRYPTO_ALG_INTERNAL, 1274 .cra_blocksize = AES_BLOCK_SIZE, 1275 .cra_ctxsize = sizeof(struct aesni_lrw_ctx), 1276 .cra_alignmask = 0, 1277 .cra_type = &crypto_blkcipher_type, 1278 .cra_module = THIS_MODULE, 1279 .cra_exit = lrw_aesni_exit_tfm, 1280 .cra_u = { 1281 .blkcipher = { 1282 .min_keysize = AES_MIN_KEY_SIZE + AES_BLOCK_SIZE, 1283 .max_keysize = AES_MAX_KEY_SIZE + AES_BLOCK_SIZE, 1284 .ivsize = AES_BLOCK_SIZE, 1285 .setkey = lrw_aesni_setkey, 1286 .encrypt = lrw_encrypt, 1287 .decrypt = lrw_decrypt, 1288 }, 1289 }, 1290 }, { 1291 .cra_name = "__xts-aes-aesni", 1292 .cra_driver_name = "__driver-xts-aes-aesni", 1293 .cra_priority = 0, 1294 .cra_flags = CRYPTO_ALG_TYPE_BLKCIPHER | 1295 CRYPTO_ALG_INTERNAL, 1296 .cra_blocksize = AES_BLOCK_SIZE, 1297 .cra_ctxsize = sizeof(struct aesni_xts_ctx), 1298 .cra_alignmask = 0, 1299 .cra_type = &crypto_blkcipher_type, 1300 .cra_module = THIS_MODULE, 1301 .cra_u = { 1302 .blkcipher = { 1303 .min_keysize = 2 * AES_MIN_KEY_SIZE, 1304 .max_keysize = 2 * AES_MAX_KEY_SIZE, 1305 .ivsize = AES_BLOCK_SIZE, 1306 .setkey = xts_aesni_setkey, 1307 .encrypt = xts_encrypt, 1308 .decrypt = xts_decrypt, 1309 }, 1310 }, 1311 }, { 1312 .cra_name = "lrw(aes)", 1313 .cra_driver_name = "lrw-aes-aesni", 1314 .cra_priority = 400, 1315 .cra_flags = CRYPTO_ALG_TYPE_ABLKCIPHER | CRYPTO_ALG_ASYNC, 1316 .cra_blocksize = AES_BLOCK_SIZE, 1317 .cra_ctxsize = sizeof(struct async_helper_ctx), 1318 .cra_alignmask = 0, 1319 .cra_type = &crypto_ablkcipher_type, 1320 .cra_module = THIS_MODULE, 1321 .cra_init = ablk_init, 1322 .cra_exit = ablk_exit, 1323 .cra_u = { 1324 .ablkcipher = { 1325 .min_keysize = AES_MIN_KEY_SIZE + AES_BLOCK_SIZE, 1326 .max_keysize = AES_MAX_KEY_SIZE + AES_BLOCK_SIZE, 1327 .ivsize = AES_BLOCK_SIZE, 1328 .setkey = ablk_set_key, 1329 .encrypt = ablk_encrypt, 1330 .decrypt = ablk_decrypt, 1331 }, 1332 }, 1333 }, { 1334 .cra_name = "xts(aes)", 1335 .cra_driver_name = "xts-aes-aesni", 1336 .cra_priority = 400, 1337 .cra_flags = CRYPTO_ALG_TYPE_ABLKCIPHER | CRYPTO_ALG_ASYNC, 1338 .cra_blocksize = AES_BLOCK_SIZE, 1339 .cra_ctxsize = sizeof(struct async_helper_ctx), 1340 .cra_alignmask = 0, 1341 .cra_type = &crypto_ablkcipher_type, 1342 .cra_module = THIS_MODULE, 1343 .cra_init = ablk_init, 1344 .cra_exit = ablk_exit, 1345 .cra_u = { 1346 .ablkcipher = { 1347 .min_keysize = 2 * AES_MIN_KEY_SIZE, 1348 .max_keysize = 2 * AES_MAX_KEY_SIZE, 1349 .ivsize = AES_BLOCK_SIZE, 1350 .setkey = ablk_set_key, 1351 .encrypt = ablk_encrypt, 1352 .decrypt = ablk_decrypt, 1353 }, 1354 }, 1355 } }; 1356 1357 #ifdef CONFIG_X86_64 1358 static struct aead_alg aesni_aead_algs[] = { { 1359 .setkey = common_rfc4106_set_key, 1360 .setauthsize = common_rfc4106_set_authsize, 1361 .encrypt = helper_rfc4106_encrypt, 1362 .decrypt = helper_rfc4106_decrypt, 1363 .ivsize = 8, 1364 .maxauthsize = 16, 1365 .base = { 1366 .cra_name = "__gcm-aes-aesni", 1367 .cra_driver_name = "__driver-gcm-aes-aesni", 1368 .cra_flags = CRYPTO_ALG_INTERNAL, 1369 .cra_blocksize = 1, 1370 .cra_ctxsize = sizeof(struct aesni_rfc4106_gcm_ctx), 1371 .cra_alignmask = AESNI_ALIGN - 1, 1372 .cra_module = THIS_MODULE, 1373 }, 1374 }, { 1375 .init = rfc4106_init, 1376 .exit = rfc4106_exit, 1377 .setkey = rfc4106_set_key, 1378 .setauthsize = rfc4106_set_authsize, 1379 .encrypt = rfc4106_encrypt, 1380 .decrypt = rfc4106_decrypt, 1381 .ivsize = 8, 1382 .maxauthsize = 16, 1383 .base = { 1384 .cra_name = "rfc4106(gcm(aes))", 1385 .cra_driver_name = "rfc4106-gcm-aesni", 1386 .cra_priority = 400, 1387 .cra_flags = CRYPTO_ALG_ASYNC, 1388 .cra_blocksize = 1, 1389 .cra_ctxsize = sizeof(struct cryptd_aead *), 1390 .cra_module = THIS_MODULE, 1391 }, 1392 } }; 1393 #else 1394 static struct aead_alg aesni_aead_algs[0]; 1395 #endif 1396 1397 1398 static const struct x86_cpu_id aesni_cpu_id[] = { 1399 X86_FEATURE_MATCH(X86_FEATURE_AES), 1400 {} 1401 }; 1402 MODULE_DEVICE_TABLE(x86cpu, aesni_cpu_id); 1403 1404 static int __init aesni_init(void) 1405 { 1406 int err; 1407 1408 if (!x86_match_cpu(aesni_cpu_id)) 1409 return -ENODEV; 1410 #ifdef CONFIG_X86_64 1411 #ifdef CONFIG_AS_AVX2 1412 if (boot_cpu_has(X86_FEATURE_AVX2)) { 1413 pr_info("AVX2 version of gcm_enc/dec engaged.\n"); 1414 aesni_gcm_enc_tfm = aesni_gcm_enc_avx2; 1415 aesni_gcm_dec_tfm = aesni_gcm_dec_avx2; 1416 } else 1417 #endif 1418 #ifdef CONFIG_AS_AVX 1419 if (boot_cpu_has(X86_FEATURE_AVX)) { 1420 pr_info("AVX version of gcm_enc/dec engaged.\n"); 1421 aesni_gcm_enc_tfm = aesni_gcm_enc_avx; 1422 aesni_gcm_dec_tfm = aesni_gcm_dec_avx; 1423 } else 1424 #endif 1425 { 1426 pr_info("SSE version of gcm_enc/dec engaged.\n"); 1427 aesni_gcm_enc_tfm = aesni_gcm_enc; 1428 aesni_gcm_dec_tfm = aesni_gcm_dec; 1429 } 1430 aesni_ctr_enc_tfm = aesni_ctr_enc; 1431 #ifdef CONFIG_AS_AVX 1432 if (boot_cpu_has(X86_FEATURE_AVX)) { 1433 /* optimize performance of ctr mode encryption transform */ 1434 aesni_ctr_enc_tfm = aesni_ctr_enc_avx_tfm; 1435 pr_info("AES CTR mode by8 optimization enabled\n"); 1436 } 1437 #endif 1438 #endif 1439 1440 err = crypto_fpu_init(); 1441 if (err) 1442 return err; 1443 1444 err = crypto_register_algs(aesni_algs, ARRAY_SIZE(aesni_algs)); 1445 if (err) 1446 goto fpu_exit; 1447 1448 err = crypto_register_aeads(aesni_aead_algs, 1449 ARRAY_SIZE(aesni_aead_algs)); 1450 if (err) 1451 goto unregister_algs; 1452 1453 return err; 1454 1455 unregister_algs: 1456 crypto_unregister_algs(aesni_algs, ARRAY_SIZE(aesni_algs)); 1457 fpu_exit: 1458 crypto_fpu_exit(); 1459 return err; 1460 } 1461 1462 static void __exit aesni_exit(void) 1463 { 1464 crypto_unregister_aeads(aesni_aead_algs, ARRAY_SIZE(aesni_aead_algs)); 1465 crypto_unregister_algs(aesni_algs, ARRAY_SIZE(aesni_algs)); 1466 1467 crypto_fpu_exit(); 1468 } 1469 1470 late_initcall(aesni_init); 1471 module_exit(aesni_exit); 1472 1473 MODULE_DESCRIPTION("Rijndael (AES) Cipher Algorithm, Intel AES-NI instructions optimized"); 1474 MODULE_LICENSE("GPL"); 1475 MODULE_ALIAS_CRYPTO("aes"); 1476