1 /* 2 * Support for Intel AES-NI instructions. This file contains glue 3 * code, the real AES implementation is in intel-aes_asm.S. 4 * 5 * Copyright (C) 2008, Intel Corp. 6 * Author: Huang Ying <ying.huang@intel.com> 7 * 8 * Added RFC4106 AES-GCM support for 128-bit keys under the AEAD 9 * interface for 64-bit kernels. 10 * Authors: Adrian Hoban <adrian.hoban@intel.com> 11 * Gabriele Paoloni <gabriele.paoloni@intel.com> 12 * Tadeusz Struk (tadeusz.struk@intel.com) 13 * Aidan O'Mahony (aidan.o.mahony@intel.com) 14 * Copyright (c) 2010, Intel Corporation. 15 * 16 * This program is free software; you can redistribute it and/or modify 17 * it under the terms of the GNU General Public License as published by 18 * the Free Software Foundation; either version 2 of the License, or 19 * (at your option) any later version. 20 */ 21 22 #include <linux/hardirq.h> 23 #include <linux/types.h> 24 #include <linux/crypto.h> 25 #include <linux/module.h> 26 #include <linux/err.h> 27 #include <crypto/algapi.h> 28 #include <crypto/aes.h> 29 #include <crypto/cryptd.h> 30 #include <crypto/ctr.h> 31 #include <crypto/b128ops.h> 32 #include <crypto/lrw.h> 33 #include <crypto/xts.h> 34 #include <asm/cpu_device_id.h> 35 #include <asm/i387.h> 36 #include <asm/crypto/aes.h> 37 #include <crypto/ablk_helper.h> 38 #include <crypto/scatterwalk.h> 39 #include <crypto/internal/aead.h> 40 #include <linux/workqueue.h> 41 #include <linux/spinlock.h> 42 #ifdef CONFIG_X86_64 43 #include <asm/crypto/glue_helper.h> 44 #endif 45 46 #if defined(CONFIG_CRYPTO_PCBC) || defined(CONFIG_CRYPTO_PCBC_MODULE) 47 #define HAS_PCBC 48 #endif 49 50 /* This data is stored at the end of the crypto_tfm struct. 51 * It's a type of per "session" data storage location. 52 * This needs to be 16 byte aligned. 53 */ 54 struct aesni_rfc4106_gcm_ctx { 55 u8 hash_subkey[16]; 56 struct crypto_aes_ctx aes_key_expanded; 57 u8 nonce[4]; 58 struct cryptd_aead *cryptd_tfm; 59 }; 60 61 struct aesni_gcm_set_hash_subkey_result { 62 int err; 63 struct completion completion; 64 }; 65 66 struct aesni_hash_subkey_req_data { 67 u8 iv[16]; 68 struct aesni_gcm_set_hash_subkey_result result; 69 struct scatterlist sg; 70 }; 71 72 #define AESNI_ALIGN (16) 73 #define AES_BLOCK_MASK (~(AES_BLOCK_SIZE-1)) 74 #define RFC4106_HASH_SUBKEY_SIZE 16 75 76 struct aesni_lrw_ctx { 77 struct lrw_table_ctx lrw_table; 78 u8 raw_aes_ctx[sizeof(struct crypto_aes_ctx) + AESNI_ALIGN - 1]; 79 }; 80 81 struct aesni_xts_ctx { 82 u8 raw_tweak_ctx[sizeof(struct crypto_aes_ctx) + AESNI_ALIGN - 1]; 83 u8 raw_crypt_ctx[sizeof(struct crypto_aes_ctx) + AESNI_ALIGN - 1]; 84 }; 85 86 asmlinkage int aesni_set_key(struct crypto_aes_ctx *ctx, const u8 *in_key, 87 unsigned int key_len); 88 asmlinkage void aesni_enc(struct crypto_aes_ctx *ctx, u8 *out, 89 const u8 *in); 90 asmlinkage void aesni_dec(struct crypto_aes_ctx *ctx, u8 *out, 91 const u8 *in); 92 asmlinkage void aesni_ecb_enc(struct crypto_aes_ctx *ctx, u8 *out, 93 const u8 *in, unsigned int len); 94 asmlinkage void aesni_ecb_dec(struct crypto_aes_ctx *ctx, u8 *out, 95 const u8 *in, unsigned int len); 96 asmlinkage void aesni_cbc_enc(struct crypto_aes_ctx *ctx, u8 *out, 97 const u8 *in, unsigned int len, u8 *iv); 98 asmlinkage void aesni_cbc_dec(struct crypto_aes_ctx *ctx, u8 *out, 99 const u8 *in, unsigned int len, u8 *iv); 100 101 int crypto_fpu_init(void); 102 void crypto_fpu_exit(void); 103 104 #define AVX_GEN2_OPTSIZE 640 105 #define AVX_GEN4_OPTSIZE 4096 106 107 #ifdef CONFIG_X86_64 108 109 static void (*aesni_ctr_enc_tfm)(struct crypto_aes_ctx *ctx, u8 *out, 110 const u8 *in, unsigned int len, u8 *iv); 111 asmlinkage void aesni_ctr_enc(struct crypto_aes_ctx *ctx, u8 *out, 112 const u8 *in, unsigned int len, u8 *iv); 113 114 asmlinkage void aesni_xts_crypt8(struct crypto_aes_ctx *ctx, u8 *out, 115 const u8 *in, bool enc, u8 *iv); 116 117 /* asmlinkage void aesni_gcm_enc() 118 * void *ctx, AES Key schedule. Starts on a 16 byte boundary. 119 * u8 *out, Ciphertext output. Encrypt in-place is allowed. 120 * const u8 *in, Plaintext input 121 * unsigned long plaintext_len, Length of data in bytes for encryption. 122 * u8 *iv, Pre-counter block j0: 4 byte salt (from Security Association) 123 * concatenated with 8 byte Initialisation Vector (from IPSec ESP 124 * Payload) concatenated with 0x00000001. 16-byte aligned pointer. 125 * u8 *hash_subkey, the Hash sub key input. Data starts on a 16-byte boundary. 126 * const u8 *aad, Additional Authentication Data (AAD) 127 * unsigned long aad_len, Length of AAD in bytes. With RFC4106 this 128 * is going to be 8 or 12 bytes 129 * u8 *auth_tag, Authenticated Tag output. 130 * unsigned long auth_tag_len), Authenticated Tag Length in bytes. 131 * Valid values are 16 (most likely), 12 or 8. 132 */ 133 asmlinkage void aesni_gcm_enc(void *ctx, u8 *out, 134 const u8 *in, unsigned long plaintext_len, u8 *iv, 135 u8 *hash_subkey, const u8 *aad, unsigned long aad_len, 136 u8 *auth_tag, unsigned long auth_tag_len); 137 138 /* asmlinkage void aesni_gcm_dec() 139 * void *ctx, AES Key schedule. Starts on a 16 byte boundary. 140 * u8 *out, Plaintext output. Decrypt in-place is allowed. 141 * const u8 *in, Ciphertext input 142 * unsigned long ciphertext_len, Length of data in bytes for decryption. 143 * u8 *iv, Pre-counter block j0: 4 byte salt (from Security Association) 144 * concatenated with 8 byte Initialisation Vector (from IPSec ESP 145 * Payload) concatenated with 0x00000001. 16-byte aligned pointer. 146 * u8 *hash_subkey, the Hash sub key input. Data starts on a 16-byte boundary. 147 * const u8 *aad, Additional Authentication Data (AAD) 148 * unsigned long aad_len, Length of AAD in bytes. With RFC4106 this is going 149 * to be 8 or 12 bytes 150 * u8 *auth_tag, Authenticated Tag output. 151 * unsigned long auth_tag_len) Authenticated Tag Length in bytes. 152 * Valid values are 16 (most likely), 12 or 8. 153 */ 154 asmlinkage void aesni_gcm_dec(void *ctx, u8 *out, 155 const u8 *in, unsigned long ciphertext_len, u8 *iv, 156 u8 *hash_subkey, const u8 *aad, unsigned long aad_len, 157 u8 *auth_tag, unsigned long auth_tag_len); 158 159 160 #ifdef CONFIG_AS_AVX 161 asmlinkage void aes_ctr_enc_128_avx_by8(const u8 *in, u8 *iv, 162 void *keys, u8 *out, unsigned int num_bytes); 163 asmlinkage void aes_ctr_enc_192_avx_by8(const u8 *in, u8 *iv, 164 void *keys, u8 *out, unsigned int num_bytes); 165 asmlinkage void aes_ctr_enc_256_avx_by8(const u8 *in, u8 *iv, 166 void *keys, u8 *out, unsigned int num_bytes); 167 /* 168 * asmlinkage void aesni_gcm_precomp_avx_gen2() 169 * gcm_data *my_ctx_data, context data 170 * u8 *hash_subkey, the Hash sub key input. Data starts on a 16-byte boundary. 171 */ 172 asmlinkage void aesni_gcm_precomp_avx_gen2(void *my_ctx_data, u8 *hash_subkey); 173 174 asmlinkage void aesni_gcm_enc_avx_gen2(void *ctx, u8 *out, 175 const u8 *in, unsigned long plaintext_len, u8 *iv, 176 const u8 *aad, unsigned long aad_len, 177 u8 *auth_tag, unsigned long auth_tag_len); 178 179 asmlinkage void aesni_gcm_dec_avx_gen2(void *ctx, u8 *out, 180 const u8 *in, unsigned long ciphertext_len, u8 *iv, 181 const u8 *aad, unsigned long aad_len, 182 u8 *auth_tag, unsigned long auth_tag_len); 183 184 static void aesni_gcm_enc_avx(void *ctx, u8 *out, 185 const u8 *in, unsigned long plaintext_len, u8 *iv, 186 u8 *hash_subkey, const u8 *aad, unsigned long aad_len, 187 u8 *auth_tag, unsigned long auth_tag_len) 188 { 189 if (plaintext_len < AVX_GEN2_OPTSIZE) { 190 aesni_gcm_enc(ctx, out, in, plaintext_len, iv, hash_subkey, aad, 191 aad_len, auth_tag, auth_tag_len); 192 } else { 193 aesni_gcm_precomp_avx_gen2(ctx, hash_subkey); 194 aesni_gcm_enc_avx_gen2(ctx, out, in, plaintext_len, iv, aad, 195 aad_len, auth_tag, auth_tag_len); 196 } 197 } 198 199 static void aesni_gcm_dec_avx(void *ctx, u8 *out, 200 const u8 *in, unsigned long ciphertext_len, u8 *iv, 201 u8 *hash_subkey, const u8 *aad, unsigned long aad_len, 202 u8 *auth_tag, unsigned long auth_tag_len) 203 { 204 if (ciphertext_len < AVX_GEN2_OPTSIZE) { 205 aesni_gcm_dec(ctx, out, in, ciphertext_len, iv, hash_subkey, aad, 206 aad_len, auth_tag, auth_tag_len); 207 } else { 208 aesni_gcm_precomp_avx_gen2(ctx, hash_subkey); 209 aesni_gcm_dec_avx_gen2(ctx, out, in, ciphertext_len, iv, aad, 210 aad_len, auth_tag, auth_tag_len); 211 } 212 } 213 #endif 214 215 #ifdef CONFIG_AS_AVX2 216 /* 217 * asmlinkage void aesni_gcm_precomp_avx_gen4() 218 * gcm_data *my_ctx_data, context data 219 * u8 *hash_subkey, the Hash sub key input. Data starts on a 16-byte boundary. 220 */ 221 asmlinkage void aesni_gcm_precomp_avx_gen4(void *my_ctx_data, u8 *hash_subkey); 222 223 asmlinkage void aesni_gcm_enc_avx_gen4(void *ctx, u8 *out, 224 const u8 *in, unsigned long plaintext_len, u8 *iv, 225 const u8 *aad, unsigned long aad_len, 226 u8 *auth_tag, unsigned long auth_tag_len); 227 228 asmlinkage void aesni_gcm_dec_avx_gen4(void *ctx, u8 *out, 229 const u8 *in, unsigned long ciphertext_len, u8 *iv, 230 const u8 *aad, unsigned long aad_len, 231 u8 *auth_tag, unsigned long auth_tag_len); 232 233 static void aesni_gcm_enc_avx2(void *ctx, u8 *out, 234 const u8 *in, unsigned long plaintext_len, u8 *iv, 235 u8 *hash_subkey, const u8 *aad, unsigned long aad_len, 236 u8 *auth_tag, unsigned long auth_tag_len) 237 { 238 if (plaintext_len < AVX_GEN2_OPTSIZE) { 239 aesni_gcm_enc(ctx, out, in, plaintext_len, iv, hash_subkey, aad, 240 aad_len, auth_tag, auth_tag_len); 241 } else if (plaintext_len < AVX_GEN4_OPTSIZE) { 242 aesni_gcm_precomp_avx_gen2(ctx, hash_subkey); 243 aesni_gcm_enc_avx_gen2(ctx, out, in, plaintext_len, iv, aad, 244 aad_len, auth_tag, auth_tag_len); 245 } else { 246 aesni_gcm_precomp_avx_gen4(ctx, hash_subkey); 247 aesni_gcm_enc_avx_gen4(ctx, out, in, plaintext_len, iv, aad, 248 aad_len, auth_tag, auth_tag_len); 249 } 250 } 251 252 static void aesni_gcm_dec_avx2(void *ctx, u8 *out, 253 const u8 *in, unsigned long ciphertext_len, u8 *iv, 254 u8 *hash_subkey, const u8 *aad, unsigned long aad_len, 255 u8 *auth_tag, unsigned long auth_tag_len) 256 { 257 if (ciphertext_len < AVX_GEN2_OPTSIZE) { 258 aesni_gcm_dec(ctx, out, in, ciphertext_len, iv, hash_subkey, 259 aad, aad_len, auth_tag, auth_tag_len); 260 } else if (ciphertext_len < AVX_GEN4_OPTSIZE) { 261 aesni_gcm_precomp_avx_gen2(ctx, hash_subkey); 262 aesni_gcm_dec_avx_gen2(ctx, out, in, ciphertext_len, iv, aad, 263 aad_len, auth_tag, auth_tag_len); 264 } else { 265 aesni_gcm_precomp_avx_gen4(ctx, hash_subkey); 266 aesni_gcm_dec_avx_gen4(ctx, out, in, ciphertext_len, iv, aad, 267 aad_len, auth_tag, auth_tag_len); 268 } 269 } 270 #endif 271 272 static void (*aesni_gcm_enc_tfm)(void *ctx, u8 *out, 273 const u8 *in, unsigned long plaintext_len, u8 *iv, 274 u8 *hash_subkey, const u8 *aad, unsigned long aad_len, 275 u8 *auth_tag, unsigned long auth_tag_len); 276 277 static void (*aesni_gcm_dec_tfm)(void *ctx, u8 *out, 278 const u8 *in, unsigned long ciphertext_len, u8 *iv, 279 u8 *hash_subkey, const u8 *aad, unsigned long aad_len, 280 u8 *auth_tag, unsigned long auth_tag_len); 281 282 static inline struct 283 aesni_rfc4106_gcm_ctx *aesni_rfc4106_gcm_ctx_get(struct crypto_aead *tfm) 284 { 285 return 286 (struct aesni_rfc4106_gcm_ctx *) 287 PTR_ALIGN((u8 *) 288 crypto_tfm_ctx(crypto_aead_tfm(tfm)), AESNI_ALIGN); 289 } 290 #endif 291 292 static inline struct crypto_aes_ctx *aes_ctx(void *raw_ctx) 293 { 294 unsigned long addr = (unsigned long)raw_ctx; 295 unsigned long align = AESNI_ALIGN; 296 297 if (align <= crypto_tfm_ctx_alignment()) 298 align = 1; 299 return (struct crypto_aes_ctx *)ALIGN(addr, align); 300 } 301 302 static int aes_set_key_common(struct crypto_tfm *tfm, void *raw_ctx, 303 const u8 *in_key, unsigned int key_len) 304 { 305 struct crypto_aes_ctx *ctx = aes_ctx(raw_ctx); 306 u32 *flags = &tfm->crt_flags; 307 int err; 308 309 if (key_len != AES_KEYSIZE_128 && key_len != AES_KEYSIZE_192 && 310 key_len != AES_KEYSIZE_256) { 311 *flags |= CRYPTO_TFM_RES_BAD_KEY_LEN; 312 return -EINVAL; 313 } 314 315 if (!irq_fpu_usable()) 316 err = crypto_aes_expand_key(ctx, in_key, key_len); 317 else { 318 kernel_fpu_begin(); 319 err = aesni_set_key(ctx, in_key, key_len); 320 kernel_fpu_end(); 321 } 322 323 return err; 324 } 325 326 static int aes_set_key(struct crypto_tfm *tfm, const u8 *in_key, 327 unsigned int key_len) 328 { 329 return aes_set_key_common(tfm, crypto_tfm_ctx(tfm), in_key, key_len); 330 } 331 332 static void aes_encrypt(struct crypto_tfm *tfm, u8 *dst, const u8 *src) 333 { 334 struct crypto_aes_ctx *ctx = aes_ctx(crypto_tfm_ctx(tfm)); 335 336 if (!irq_fpu_usable()) 337 crypto_aes_encrypt_x86(ctx, dst, src); 338 else { 339 kernel_fpu_begin(); 340 aesni_enc(ctx, dst, src); 341 kernel_fpu_end(); 342 } 343 } 344 345 static void aes_decrypt(struct crypto_tfm *tfm, u8 *dst, const u8 *src) 346 { 347 struct crypto_aes_ctx *ctx = aes_ctx(crypto_tfm_ctx(tfm)); 348 349 if (!irq_fpu_usable()) 350 crypto_aes_decrypt_x86(ctx, dst, src); 351 else { 352 kernel_fpu_begin(); 353 aesni_dec(ctx, dst, src); 354 kernel_fpu_end(); 355 } 356 } 357 358 static void __aes_encrypt(struct crypto_tfm *tfm, u8 *dst, const u8 *src) 359 { 360 struct crypto_aes_ctx *ctx = aes_ctx(crypto_tfm_ctx(tfm)); 361 362 aesni_enc(ctx, dst, src); 363 } 364 365 static void __aes_decrypt(struct crypto_tfm *tfm, u8 *dst, const u8 *src) 366 { 367 struct crypto_aes_ctx *ctx = aes_ctx(crypto_tfm_ctx(tfm)); 368 369 aesni_dec(ctx, dst, src); 370 } 371 372 static int ecb_encrypt(struct blkcipher_desc *desc, 373 struct scatterlist *dst, struct scatterlist *src, 374 unsigned int nbytes) 375 { 376 struct crypto_aes_ctx *ctx = aes_ctx(crypto_blkcipher_ctx(desc->tfm)); 377 struct blkcipher_walk walk; 378 int err; 379 380 blkcipher_walk_init(&walk, dst, src, nbytes); 381 err = blkcipher_walk_virt(desc, &walk); 382 desc->flags &= ~CRYPTO_TFM_REQ_MAY_SLEEP; 383 384 kernel_fpu_begin(); 385 while ((nbytes = walk.nbytes)) { 386 aesni_ecb_enc(ctx, walk.dst.virt.addr, walk.src.virt.addr, 387 nbytes & AES_BLOCK_MASK); 388 nbytes &= AES_BLOCK_SIZE - 1; 389 err = blkcipher_walk_done(desc, &walk, nbytes); 390 } 391 kernel_fpu_end(); 392 393 return err; 394 } 395 396 static int ecb_decrypt(struct blkcipher_desc *desc, 397 struct scatterlist *dst, struct scatterlist *src, 398 unsigned int nbytes) 399 { 400 struct crypto_aes_ctx *ctx = aes_ctx(crypto_blkcipher_ctx(desc->tfm)); 401 struct blkcipher_walk walk; 402 int err; 403 404 blkcipher_walk_init(&walk, dst, src, nbytes); 405 err = blkcipher_walk_virt(desc, &walk); 406 desc->flags &= ~CRYPTO_TFM_REQ_MAY_SLEEP; 407 408 kernel_fpu_begin(); 409 while ((nbytes = walk.nbytes)) { 410 aesni_ecb_dec(ctx, walk.dst.virt.addr, walk.src.virt.addr, 411 nbytes & AES_BLOCK_MASK); 412 nbytes &= AES_BLOCK_SIZE - 1; 413 err = blkcipher_walk_done(desc, &walk, nbytes); 414 } 415 kernel_fpu_end(); 416 417 return err; 418 } 419 420 static int cbc_encrypt(struct blkcipher_desc *desc, 421 struct scatterlist *dst, struct scatterlist *src, 422 unsigned int nbytes) 423 { 424 struct crypto_aes_ctx *ctx = aes_ctx(crypto_blkcipher_ctx(desc->tfm)); 425 struct blkcipher_walk walk; 426 int err; 427 428 blkcipher_walk_init(&walk, dst, src, nbytes); 429 err = blkcipher_walk_virt(desc, &walk); 430 desc->flags &= ~CRYPTO_TFM_REQ_MAY_SLEEP; 431 432 kernel_fpu_begin(); 433 while ((nbytes = walk.nbytes)) { 434 aesni_cbc_enc(ctx, walk.dst.virt.addr, walk.src.virt.addr, 435 nbytes & AES_BLOCK_MASK, walk.iv); 436 nbytes &= AES_BLOCK_SIZE - 1; 437 err = blkcipher_walk_done(desc, &walk, nbytes); 438 } 439 kernel_fpu_end(); 440 441 return err; 442 } 443 444 static int cbc_decrypt(struct blkcipher_desc *desc, 445 struct scatterlist *dst, struct scatterlist *src, 446 unsigned int nbytes) 447 { 448 struct crypto_aes_ctx *ctx = aes_ctx(crypto_blkcipher_ctx(desc->tfm)); 449 struct blkcipher_walk walk; 450 int err; 451 452 blkcipher_walk_init(&walk, dst, src, nbytes); 453 err = blkcipher_walk_virt(desc, &walk); 454 desc->flags &= ~CRYPTO_TFM_REQ_MAY_SLEEP; 455 456 kernel_fpu_begin(); 457 while ((nbytes = walk.nbytes)) { 458 aesni_cbc_dec(ctx, walk.dst.virt.addr, walk.src.virt.addr, 459 nbytes & AES_BLOCK_MASK, walk.iv); 460 nbytes &= AES_BLOCK_SIZE - 1; 461 err = blkcipher_walk_done(desc, &walk, nbytes); 462 } 463 kernel_fpu_end(); 464 465 return err; 466 } 467 468 #ifdef CONFIG_X86_64 469 static void ctr_crypt_final(struct crypto_aes_ctx *ctx, 470 struct blkcipher_walk *walk) 471 { 472 u8 *ctrblk = walk->iv; 473 u8 keystream[AES_BLOCK_SIZE]; 474 u8 *src = walk->src.virt.addr; 475 u8 *dst = walk->dst.virt.addr; 476 unsigned int nbytes = walk->nbytes; 477 478 aesni_enc(ctx, keystream, ctrblk); 479 crypto_xor(keystream, src, nbytes); 480 memcpy(dst, keystream, nbytes); 481 crypto_inc(ctrblk, AES_BLOCK_SIZE); 482 } 483 484 #ifdef CONFIG_AS_AVX 485 static void aesni_ctr_enc_avx_tfm(struct crypto_aes_ctx *ctx, u8 *out, 486 const u8 *in, unsigned int len, u8 *iv) 487 { 488 /* 489 * based on key length, override with the by8 version 490 * of ctr mode encryption/decryption for improved performance 491 * aes_set_key_common() ensures that key length is one of 492 * {128,192,256} 493 */ 494 if (ctx->key_length == AES_KEYSIZE_128) 495 aes_ctr_enc_128_avx_by8(in, iv, (void *)ctx, out, len); 496 else if (ctx->key_length == AES_KEYSIZE_192) 497 aes_ctr_enc_192_avx_by8(in, iv, (void *)ctx, out, len); 498 else 499 aes_ctr_enc_256_avx_by8(in, iv, (void *)ctx, out, len); 500 } 501 #endif 502 503 static int ctr_crypt(struct blkcipher_desc *desc, 504 struct scatterlist *dst, struct scatterlist *src, 505 unsigned int nbytes) 506 { 507 struct crypto_aes_ctx *ctx = aes_ctx(crypto_blkcipher_ctx(desc->tfm)); 508 struct blkcipher_walk walk; 509 int err; 510 511 blkcipher_walk_init(&walk, dst, src, nbytes); 512 err = blkcipher_walk_virt_block(desc, &walk, AES_BLOCK_SIZE); 513 desc->flags &= ~CRYPTO_TFM_REQ_MAY_SLEEP; 514 515 kernel_fpu_begin(); 516 while ((nbytes = walk.nbytes) >= AES_BLOCK_SIZE) { 517 aesni_ctr_enc_tfm(ctx, walk.dst.virt.addr, walk.src.virt.addr, 518 nbytes & AES_BLOCK_MASK, walk.iv); 519 nbytes &= AES_BLOCK_SIZE - 1; 520 err = blkcipher_walk_done(desc, &walk, nbytes); 521 } 522 if (walk.nbytes) { 523 ctr_crypt_final(ctx, &walk); 524 err = blkcipher_walk_done(desc, &walk, 0); 525 } 526 kernel_fpu_end(); 527 528 return err; 529 } 530 #endif 531 532 static int ablk_ecb_init(struct crypto_tfm *tfm) 533 { 534 return ablk_init_common(tfm, "__driver-ecb-aes-aesni"); 535 } 536 537 static int ablk_cbc_init(struct crypto_tfm *tfm) 538 { 539 return ablk_init_common(tfm, "__driver-cbc-aes-aesni"); 540 } 541 542 #ifdef CONFIG_X86_64 543 static int ablk_ctr_init(struct crypto_tfm *tfm) 544 { 545 return ablk_init_common(tfm, "__driver-ctr-aes-aesni"); 546 } 547 548 #endif 549 550 #ifdef HAS_PCBC 551 static int ablk_pcbc_init(struct crypto_tfm *tfm) 552 { 553 return ablk_init_common(tfm, "fpu(pcbc(__driver-aes-aesni))"); 554 } 555 #endif 556 557 static void lrw_xts_encrypt_callback(void *ctx, u8 *blks, unsigned int nbytes) 558 { 559 aesni_ecb_enc(ctx, blks, blks, nbytes); 560 } 561 562 static void lrw_xts_decrypt_callback(void *ctx, u8 *blks, unsigned int nbytes) 563 { 564 aesni_ecb_dec(ctx, blks, blks, nbytes); 565 } 566 567 static int lrw_aesni_setkey(struct crypto_tfm *tfm, const u8 *key, 568 unsigned int keylen) 569 { 570 struct aesni_lrw_ctx *ctx = crypto_tfm_ctx(tfm); 571 int err; 572 573 err = aes_set_key_common(tfm, ctx->raw_aes_ctx, key, 574 keylen - AES_BLOCK_SIZE); 575 if (err) 576 return err; 577 578 return lrw_init_table(&ctx->lrw_table, key + keylen - AES_BLOCK_SIZE); 579 } 580 581 static void lrw_aesni_exit_tfm(struct crypto_tfm *tfm) 582 { 583 struct aesni_lrw_ctx *ctx = crypto_tfm_ctx(tfm); 584 585 lrw_free_table(&ctx->lrw_table); 586 } 587 588 static int lrw_encrypt(struct blkcipher_desc *desc, struct scatterlist *dst, 589 struct scatterlist *src, unsigned int nbytes) 590 { 591 struct aesni_lrw_ctx *ctx = crypto_blkcipher_ctx(desc->tfm); 592 be128 buf[8]; 593 struct lrw_crypt_req req = { 594 .tbuf = buf, 595 .tbuflen = sizeof(buf), 596 597 .table_ctx = &ctx->lrw_table, 598 .crypt_ctx = aes_ctx(ctx->raw_aes_ctx), 599 .crypt_fn = lrw_xts_encrypt_callback, 600 }; 601 int ret; 602 603 desc->flags &= ~CRYPTO_TFM_REQ_MAY_SLEEP; 604 605 kernel_fpu_begin(); 606 ret = lrw_crypt(desc, dst, src, nbytes, &req); 607 kernel_fpu_end(); 608 609 return ret; 610 } 611 612 static int lrw_decrypt(struct blkcipher_desc *desc, struct scatterlist *dst, 613 struct scatterlist *src, unsigned int nbytes) 614 { 615 struct aesni_lrw_ctx *ctx = crypto_blkcipher_ctx(desc->tfm); 616 be128 buf[8]; 617 struct lrw_crypt_req req = { 618 .tbuf = buf, 619 .tbuflen = sizeof(buf), 620 621 .table_ctx = &ctx->lrw_table, 622 .crypt_ctx = aes_ctx(ctx->raw_aes_ctx), 623 .crypt_fn = lrw_xts_decrypt_callback, 624 }; 625 int ret; 626 627 desc->flags &= ~CRYPTO_TFM_REQ_MAY_SLEEP; 628 629 kernel_fpu_begin(); 630 ret = lrw_crypt(desc, dst, src, nbytes, &req); 631 kernel_fpu_end(); 632 633 return ret; 634 } 635 636 static int xts_aesni_setkey(struct crypto_tfm *tfm, const u8 *key, 637 unsigned int keylen) 638 { 639 struct aesni_xts_ctx *ctx = crypto_tfm_ctx(tfm); 640 u32 *flags = &tfm->crt_flags; 641 int err; 642 643 /* key consists of keys of equal size concatenated, therefore 644 * the length must be even 645 */ 646 if (keylen % 2) { 647 *flags |= CRYPTO_TFM_RES_BAD_KEY_LEN; 648 return -EINVAL; 649 } 650 651 /* first half of xts-key is for crypt */ 652 err = aes_set_key_common(tfm, ctx->raw_crypt_ctx, key, keylen / 2); 653 if (err) 654 return err; 655 656 /* second half of xts-key is for tweak */ 657 return aes_set_key_common(tfm, ctx->raw_tweak_ctx, key + keylen / 2, 658 keylen / 2); 659 } 660 661 662 static void aesni_xts_tweak(void *ctx, u8 *out, const u8 *in) 663 { 664 aesni_enc(ctx, out, in); 665 } 666 667 #ifdef CONFIG_X86_64 668 669 static void aesni_xts_enc(void *ctx, u128 *dst, const u128 *src, le128 *iv) 670 { 671 glue_xts_crypt_128bit_one(ctx, dst, src, iv, GLUE_FUNC_CAST(aesni_enc)); 672 } 673 674 static void aesni_xts_dec(void *ctx, u128 *dst, const u128 *src, le128 *iv) 675 { 676 glue_xts_crypt_128bit_one(ctx, dst, src, iv, GLUE_FUNC_CAST(aesni_dec)); 677 } 678 679 static void aesni_xts_enc8(void *ctx, u128 *dst, const u128 *src, le128 *iv) 680 { 681 aesni_xts_crypt8(ctx, (u8 *)dst, (const u8 *)src, true, (u8 *)iv); 682 } 683 684 static void aesni_xts_dec8(void *ctx, u128 *dst, const u128 *src, le128 *iv) 685 { 686 aesni_xts_crypt8(ctx, (u8 *)dst, (const u8 *)src, false, (u8 *)iv); 687 } 688 689 static const struct common_glue_ctx aesni_enc_xts = { 690 .num_funcs = 2, 691 .fpu_blocks_limit = 1, 692 693 .funcs = { { 694 .num_blocks = 8, 695 .fn_u = { .xts = GLUE_XTS_FUNC_CAST(aesni_xts_enc8) } 696 }, { 697 .num_blocks = 1, 698 .fn_u = { .xts = GLUE_XTS_FUNC_CAST(aesni_xts_enc) } 699 } } 700 }; 701 702 static const struct common_glue_ctx aesni_dec_xts = { 703 .num_funcs = 2, 704 .fpu_blocks_limit = 1, 705 706 .funcs = { { 707 .num_blocks = 8, 708 .fn_u = { .xts = GLUE_XTS_FUNC_CAST(aesni_xts_dec8) } 709 }, { 710 .num_blocks = 1, 711 .fn_u = { .xts = GLUE_XTS_FUNC_CAST(aesni_xts_dec) } 712 } } 713 }; 714 715 static int xts_encrypt(struct blkcipher_desc *desc, struct scatterlist *dst, 716 struct scatterlist *src, unsigned int nbytes) 717 { 718 struct aesni_xts_ctx *ctx = crypto_blkcipher_ctx(desc->tfm); 719 720 return glue_xts_crypt_128bit(&aesni_enc_xts, desc, dst, src, nbytes, 721 XTS_TWEAK_CAST(aesni_xts_tweak), 722 aes_ctx(ctx->raw_tweak_ctx), 723 aes_ctx(ctx->raw_crypt_ctx)); 724 } 725 726 static int xts_decrypt(struct blkcipher_desc *desc, struct scatterlist *dst, 727 struct scatterlist *src, unsigned int nbytes) 728 { 729 struct aesni_xts_ctx *ctx = crypto_blkcipher_ctx(desc->tfm); 730 731 return glue_xts_crypt_128bit(&aesni_dec_xts, desc, dst, src, nbytes, 732 XTS_TWEAK_CAST(aesni_xts_tweak), 733 aes_ctx(ctx->raw_tweak_ctx), 734 aes_ctx(ctx->raw_crypt_ctx)); 735 } 736 737 #else 738 739 static int xts_encrypt(struct blkcipher_desc *desc, struct scatterlist *dst, 740 struct scatterlist *src, unsigned int nbytes) 741 { 742 struct aesni_xts_ctx *ctx = crypto_blkcipher_ctx(desc->tfm); 743 be128 buf[8]; 744 struct xts_crypt_req req = { 745 .tbuf = buf, 746 .tbuflen = sizeof(buf), 747 748 .tweak_ctx = aes_ctx(ctx->raw_tweak_ctx), 749 .tweak_fn = aesni_xts_tweak, 750 .crypt_ctx = aes_ctx(ctx->raw_crypt_ctx), 751 .crypt_fn = lrw_xts_encrypt_callback, 752 }; 753 int ret; 754 755 desc->flags &= ~CRYPTO_TFM_REQ_MAY_SLEEP; 756 757 kernel_fpu_begin(); 758 ret = xts_crypt(desc, dst, src, nbytes, &req); 759 kernel_fpu_end(); 760 761 return ret; 762 } 763 764 static int xts_decrypt(struct blkcipher_desc *desc, struct scatterlist *dst, 765 struct scatterlist *src, unsigned int nbytes) 766 { 767 struct aesni_xts_ctx *ctx = crypto_blkcipher_ctx(desc->tfm); 768 be128 buf[8]; 769 struct xts_crypt_req req = { 770 .tbuf = buf, 771 .tbuflen = sizeof(buf), 772 773 .tweak_ctx = aes_ctx(ctx->raw_tweak_ctx), 774 .tweak_fn = aesni_xts_tweak, 775 .crypt_ctx = aes_ctx(ctx->raw_crypt_ctx), 776 .crypt_fn = lrw_xts_decrypt_callback, 777 }; 778 int ret; 779 780 desc->flags &= ~CRYPTO_TFM_REQ_MAY_SLEEP; 781 782 kernel_fpu_begin(); 783 ret = xts_crypt(desc, dst, src, nbytes, &req); 784 kernel_fpu_end(); 785 786 return ret; 787 } 788 789 #endif 790 791 #ifdef CONFIG_X86_64 792 static int rfc4106_init(struct crypto_tfm *tfm) 793 { 794 struct cryptd_aead *cryptd_tfm; 795 struct aesni_rfc4106_gcm_ctx *ctx = (struct aesni_rfc4106_gcm_ctx *) 796 PTR_ALIGN((u8 *)crypto_tfm_ctx(tfm), AESNI_ALIGN); 797 struct crypto_aead *cryptd_child; 798 struct aesni_rfc4106_gcm_ctx *child_ctx; 799 cryptd_tfm = cryptd_alloc_aead("__driver-gcm-aes-aesni", 0, 0); 800 if (IS_ERR(cryptd_tfm)) 801 return PTR_ERR(cryptd_tfm); 802 803 cryptd_child = cryptd_aead_child(cryptd_tfm); 804 child_ctx = aesni_rfc4106_gcm_ctx_get(cryptd_child); 805 memcpy(child_ctx, ctx, sizeof(*ctx)); 806 ctx->cryptd_tfm = cryptd_tfm; 807 tfm->crt_aead.reqsize = sizeof(struct aead_request) 808 + crypto_aead_reqsize(&cryptd_tfm->base); 809 return 0; 810 } 811 812 static void rfc4106_exit(struct crypto_tfm *tfm) 813 { 814 struct aesni_rfc4106_gcm_ctx *ctx = 815 (struct aesni_rfc4106_gcm_ctx *) 816 PTR_ALIGN((u8 *)crypto_tfm_ctx(tfm), AESNI_ALIGN); 817 if (!IS_ERR(ctx->cryptd_tfm)) 818 cryptd_free_aead(ctx->cryptd_tfm); 819 return; 820 } 821 822 static void 823 rfc4106_set_hash_subkey_done(struct crypto_async_request *req, int err) 824 { 825 struct aesni_gcm_set_hash_subkey_result *result = req->data; 826 827 if (err == -EINPROGRESS) 828 return; 829 result->err = err; 830 complete(&result->completion); 831 } 832 833 static int 834 rfc4106_set_hash_subkey(u8 *hash_subkey, const u8 *key, unsigned int key_len) 835 { 836 struct crypto_ablkcipher *ctr_tfm; 837 struct ablkcipher_request *req; 838 int ret = -EINVAL; 839 struct aesni_hash_subkey_req_data *req_data; 840 841 ctr_tfm = crypto_alloc_ablkcipher("ctr(aes)", 0, 0); 842 if (IS_ERR(ctr_tfm)) 843 return PTR_ERR(ctr_tfm); 844 845 crypto_ablkcipher_clear_flags(ctr_tfm, ~0); 846 847 ret = crypto_ablkcipher_setkey(ctr_tfm, key, key_len); 848 if (ret) 849 goto out_free_ablkcipher; 850 851 ret = -ENOMEM; 852 req = ablkcipher_request_alloc(ctr_tfm, GFP_KERNEL); 853 if (!req) 854 goto out_free_ablkcipher; 855 856 req_data = kmalloc(sizeof(*req_data), GFP_KERNEL); 857 if (!req_data) 858 goto out_free_request; 859 860 memset(req_data->iv, 0, sizeof(req_data->iv)); 861 862 /* Clear the data in the hash sub key container to zero.*/ 863 /* We want to cipher all zeros to create the hash sub key. */ 864 memset(hash_subkey, 0, RFC4106_HASH_SUBKEY_SIZE); 865 866 init_completion(&req_data->result.completion); 867 sg_init_one(&req_data->sg, hash_subkey, RFC4106_HASH_SUBKEY_SIZE); 868 ablkcipher_request_set_tfm(req, ctr_tfm); 869 ablkcipher_request_set_callback(req, CRYPTO_TFM_REQ_MAY_SLEEP | 870 CRYPTO_TFM_REQ_MAY_BACKLOG, 871 rfc4106_set_hash_subkey_done, 872 &req_data->result); 873 874 ablkcipher_request_set_crypt(req, &req_data->sg, 875 &req_data->sg, RFC4106_HASH_SUBKEY_SIZE, req_data->iv); 876 877 ret = crypto_ablkcipher_encrypt(req); 878 if (ret == -EINPROGRESS || ret == -EBUSY) { 879 ret = wait_for_completion_interruptible 880 (&req_data->result.completion); 881 if (!ret) 882 ret = req_data->result.err; 883 } 884 kfree(req_data); 885 out_free_request: 886 ablkcipher_request_free(req); 887 out_free_ablkcipher: 888 crypto_free_ablkcipher(ctr_tfm); 889 return ret; 890 } 891 892 static int rfc4106_set_key(struct crypto_aead *parent, const u8 *key, 893 unsigned int key_len) 894 { 895 int ret = 0; 896 struct crypto_tfm *tfm = crypto_aead_tfm(parent); 897 struct aesni_rfc4106_gcm_ctx *ctx = aesni_rfc4106_gcm_ctx_get(parent); 898 struct crypto_aead *cryptd_child = cryptd_aead_child(ctx->cryptd_tfm); 899 struct aesni_rfc4106_gcm_ctx *child_ctx = 900 aesni_rfc4106_gcm_ctx_get(cryptd_child); 901 u8 *new_key_align, *new_key_mem = NULL; 902 903 if (key_len < 4) { 904 crypto_tfm_set_flags(tfm, CRYPTO_TFM_RES_BAD_KEY_LEN); 905 return -EINVAL; 906 } 907 /*Account for 4 byte nonce at the end.*/ 908 key_len -= 4; 909 if (key_len != AES_KEYSIZE_128) { 910 crypto_tfm_set_flags(tfm, CRYPTO_TFM_RES_BAD_KEY_LEN); 911 return -EINVAL; 912 } 913 914 memcpy(ctx->nonce, key + key_len, sizeof(ctx->nonce)); 915 /*This must be on a 16 byte boundary!*/ 916 if ((unsigned long)(&(ctx->aes_key_expanded.key_enc[0])) % AESNI_ALIGN) 917 return -EINVAL; 918 919 if ((unsigned long)key % AESNI_ALIGN) { 920 /*key is not aligned: use an auxuliar aligned pointer*/ 921 new_key_mem = kmalloc(key_len+AESNI_ALIGN, GFP_KERNEL); 922 if (!new_key_mem) 923 return -ENOMEM; 924 925 new_key_align = PTR_ALIGN(new_key_mem, AESNI_ALIGN); 926 memcpy(new_key_align, key, key_len); 927 key = new_key_align; 928 } 929 930 if (!irq_fpu_usable()) 931 ret = crypto_aes_expand_key(&(ctx->aes_key_expanded), 932 key, key_len); 933 else { 934 kernel_fpu_begin(); 935 ret = aesni_set_key(&(ctx->aes_key_expanded), key, key_len); 936 kernel_fpu_end(); 937 } 938 /*This must be on a 16 byte boundary!*/ 939 if ((unsigned long)(&(ctx->hash_subkey[0])) % AESNI_ALIGN) { 940 ret = -EINVAL; 941 goto exit; 942 } 943 ret = rfc4106_set_hash_subkey(ctx->hash_subkey, key, key_len); 944 memcpy(child_ctx, ctx, sizeof(*ctx)); 945 exit: 946 kfree(new_key_mem); 947 return ret; 948 } 949 950 /* This is the Integrity Check Value (aka the authentication tag length and can 951 * be 8, 12 or 16 bytes long. */ 952 static int rfc4106_set_authsize(struct crypto_aead *parent, 953 unsigned int authsize) 954 { 955 struct aesni_rfc4106_gcm_ctx *ctx = aesni_rfc4106_gcm_ctx_get(parent); 956 struct crypto_aead *cryptd_child = cryptd_aead_child(ctx->cryptd_tfm); 957 958 switch (authsize) { 959 case 8: 960 case 12: 961 case 16: 962 break; 963 default: 964 return -EINVAL; 965 } 966 crypto_aead_crt(parent)->authsize = authsize; 967 crypto_aead_crt(cryptd_child)->authsize = authsize; 968 return 0; 969 } 970 971 static int rfc4106_encrypt(struct aead_request *req) 972 { 973 int ret; 974 struct crypto_aead *tfm = crypto_aead_reqtfm(req); 975 struct aesni_rfc4106_gcm_ctx *ctx = aesni_rfc4106_gcm_ctx_get(tfm); 976 977 if (!irq_fpu_usable()) { 978 struct aead_request *cryptd_req = 979 (struct aead_request *) aead_request_ctx(req); 980 memcpy(cryptd_req, req, sizeof(*req)); 981 aead_request_set_tfm(cryptd_req, &ctx->cryptd_tfm->base); 982 return crypto_aead_encrypt(cryptd_req); 983 } else { 984 struct crypto_aead *cryptd_child = cryptd_aead_child(ctx->cryptd_tfm); 985 kernel_fpu_begin(); 986 ret = cryptd_child->base.crt_aead.encrypt(req); 987 kernel_fpu_end(); 988 return ret; 989 } 990 } 991 992 static int rfc4106_decrypt(struct aead_request *req) 993 { 994 int ret; 995 struct crypto_aead *tfm = crypto_aead_reqtfm(req); 996 struct aesni_rfc4106_gcm_ctx *ctx = aesni_rfc4106_gcm_ctx_get(tfm); 997 998 if (!irq_fpu_usable()) { 999 struct aead_request *cryptd_req = 1000 (struct aead_request *) aead_request_ctx(req); 1001 memcpy(cryptd_req, req, sizeof(*req)); 1002 aead_request_set_tfm(cryptd_req, &ctx->cryptd_tfm->base); 1003 return crypto_aead_decrypt(cryptd_req); 1004 } else { 1005 struct crypto_aead *cryptd_child = cryptd_aead_child(ctx->cryptd_tfm); 1006 kernel_fpu_begin(); 1007 ret = cryptd_child->base.crt_aead.decrypt(req); 1008 kernel_fpu_end(); 1009 return ret; 1010 } 1011 } 1012 1013 static int __driver_rfc4106_encrypt(struct aead_request *req) 1014 { 1015 u8 one_entry_in_sg = 0; 1016 u8 *src, *dst, *assoc; 1017 __be32 counter = cpu_to_be32(1); 1018 struct crypto_aead *tfm = crypto_aead_reqtfm(req); 1019 struct aesni_rfc4106_gcm_ctx *ctx = aesni_rfc4106_gcm_ctx_get(tfm); 1020 void *aes_ctx = &(ctx->aes_key_expanded); 1021 unsigned long auth_tag_len = crypto_aead_authsize(tfm); 1022 u8 iv_tab[16+AESNI_ALIGN]; 1023 u8* iv = (u8 *) PTR_ALIGN((u8 *)iv_tab, AESNI_ALIGN); 1024 struct scatter_walk src_sg_walk; 1025 struct scatter_walk assoc_sg_walk; 1026 struct scatter_walk dst_sg_walk; 1027 unsigned int i; 1028 1029 /* Assuming we are supporting rfc4106 64-bit extended */ 1030 /* sequence numbers We need to have the AAD length equal */ 1031 /* to 8 or 12 bytes */ 1032 if (unlikely(req->assoclen != 8 && req->assoclen != 12)) 1033 return -EINVAL; 1034 /* IV below built */ 1035 for (i = 0; i < 4; i++) 1036 *(iv+i) = ctx->nonce[i]; 1037 for (i = 0; i < 8; i++) 1038 *(iv+4+i) = req->iv[i]; 1039 *((__be32 *)(iv+12)) = counter; 1040 1041 if ((sg_is_last(req->src)) && (sg_is_last(req->assoc))) { 1042 one_entry_in_sg = 1; 1043 scatterwalk_start(&src_sg_walk, req->src); 1044 scatterwalk_start(&assoc_sg_walk, req->assoc); 1045 src = scatterwalk_map(&src_sg_walk); 1046 assoc = scatterwalk_map(&assoc_sg_walk); 1047 dst = src; 1048 if (unlikely(req->src != req->dst)) { 1049 scatterwalk_start(&dst_sg_walk, req->dst); 1050 dst = scatterwalk_map(&dst_sg_walk); 1051 } 1052 1053 } else { 1054 /* Allocate memory for src, dst, assoc */ 1055 src = kmalloc(req->cryptlen + auth_tag_len + req->assoclen, 1056 GFP_ATOMIC); 1057 if (unlikely(!src)) 1058 return -ENOMEM; 1059 assoc = (src + req->cryptlen + auth_tag_len); 1060 scatterwalk_map_and_copy(src, req->src, 0, req->cryptlen, 0); 1061 scatterwalk_map_and_copy(assoc, req->assoc, 0, 1062 req->assoclen, 0); 1063 dst = src; 1064 } 1065 1066 aesni_gcm_enc_tfm(aes_ctx, dst, src, (unsigned long)req->cryptlen, iv, 1067 ctx->hash_subkey, assoc, (unsigned long)req->assoclen, dst 1068 + ((unsigned long)req->cryptlen), auth_tag_len); 1069 1070 /* The authTag (aka the Integrity Check Value) needs to be written 1071 * back to the packet. */ 1072 if (one_entry_in_sg) { 1073 if (unlikely(req->src != req->dst)) { 1074 scatterwalk_unmap(dst); 1075 scatterwalk_done(&dst_sg_walk, 0, 0); 1076 } 1077 scatterwalk_unmap(src); 1078 scatterwalk_unmap(assoc); 1079 scatterwalk_done(&src_sg_walk, 0, 0); 1080 scatterwalk_done(&assoc_sg_walk, 0, 0); 1081 } else { 1082 scatterwalk_map_and_copy(dst, req->dst, 0, 1083 req->cryptlen + auth_tag_len, 1); 1084 kfree(src); 1085 } 1086 return 0; 1087 } 1088 1089 static int __driver_rfc4106_decrypt(struct aead_request *req) 1090 { 1091 u8 one_entry_in_sg = 0; 1092 u8 *src, *dst, *assoc; 1093 unsigned long tempCipherLen = 0; 1094 __be32 counter = cpu_to_be32(1); 1095 int retval = 0; 1096 struct crypto_aead *tfm = crypto_aead_reqtfm(req); 1097 struct aesni_rfc4106_gcm_ctx *ctx = aesni_rfc4106_gcm_ctx_get(tfm); 1098 void *aes_ctx = &(ctx->aes_key_expanded); 1099 unsigned long auth_tag_len = crypto_aead_authsize(tfm); 1100 u8 iv_and_authTag[32+AESNI_ALIGN]; 1101 u8 *iv = (u8 *) PTR_ALIGN((u8 *)iv_and_authTag, AESNI_ALIGN); 1102 u8 *authTag = iv + 16; 1103 struct scatter_walk src_sg_walk; 1104 struct scatter_walk assoc_sg_walk; 1105 struct scatter_walk dst_sg_walk; 1106 unsigned int i; 1107 1108 if (unlikely((req->cryptlen < auth_tag_len) || 1109 (req->assoclen != 8 && req->assoclen != 12))) 1110 return -EINVAL; 1111 /* Assuming we are supporting rfc4106 64-bit extended */ 1112 /* sequence numbers We need to have the AAD length */ 1113 /* equal to 8 or 12 bytes */ 1114 1115 tempCipherLen = (unsigned long)(req->cryptlen - auth_tag_len); 1116 /* IV below built */ 1117 for (i = 0; i < 4; i++) 1118 *(iv+i) = ctx->nonce[i]; 1119 for (i = 0; i < 8; i++) 1120 *(iv+4+i) = req->iv[i]; 1121 *((__be32 *)(iv+12)) = counter; 1122 1123 if ((sg_is_last(req->src)) && (sg_is_last(req->assoc))) { 1124 one_entry_in_sg = 1; 1125 scatterwalk_start(&src_sg_walk, req->src); 1126 scatterwalk_start(&assoc_sg_walk, req->assoc); 1127 src = scatterwalk_map(&src_sg_walk); 1128 assoc = scatterwalk_map(&assoc_sg_walk); 1129 dst = src; 1130 if (unlikely(req->src != req->dst)) { 1131 scatterwalk_start(&dst_sg_walk, req->dst); 1132 dst = scatterwalk_map(&dst_sg_walk); 1133 } 1134 1135 } else { 1136 /* Allocate memory for src, dst, assoc */ 1137 src = kmalloc(req->cryptlen + req->assoclen, GFP_ATOMIC); 1138 if (!src) 1139 return -ENOMEM; 1140 assoc = (src + req->cryptlen + auth_tag_len); 1141 scatterwalk_map_and_copy(src, req->src, 0, req->cryptlen, 0); 1142 scatterwalk_map_and_copy(assoc, req->assoc, 0, 1143 req->assoclen, 0); 1144 dst = src; 1145 } 1146 1147 aesni_gcm_dec_tfm(aes_ctx, dst, src, tempCipherLen, iv, 1148 ctx->hash_subkey, assoc, (unsigned long)req->assoclen, 1149 authTag, auth_tag_len); 1150 1151 /* Compare generated tag with passed in tag. */ 1152 retval = crypto_memneq(src + tempCipherLen, authTag, auth_tag_len) ? 1153 -EBADMSG : 0; 1154 1155 if (one_entry_in_sg) { 1156 if (unlikely(req->src != req->dst)) { 1157 scatterwalk_unmap(dst); 1158 scatterwalk_done(&dst_sg_walk, 0, 0); 1159 } 1160 scatterwalk_unmap(src); 1161 scatterwalk_unmap(assoc); 1162 scatterwalk_done(&src_sg_walk, 0, 0); 1163 scatterwalk_done(&assoc_sg_walk, 0, 0); 1164 } else { 1165 scatterwalk_map_and_copy(dst, req->dst, 0, req->cryptlen, 1); 1166 kfree(src); 1167 } 1168 return retval; 1169 } 1170 #endif 1171 1172 static struct crypto_alg aesni_algs[] = { { 1173 .cra_name = "aes", 1174 .cra_driver_name = "aes-aesni", 1175 .cra_priority = 300, 1176 .cra_flags = CRYPTO_ALG_TYPE_CIPHER, 1177 .cra_blocksize = AES_BLOCK_SIZE, 1178 .cra_ctxsize = sizeof(struct crypto_aes_ctx) + 1179 AESNI_ALIGN - 1, 1180 .cra_alignmask = 0, 1181 .cra_module = THIS_MODULE, 1182 .cra_u = { 1183 .cipher = { 1184 .cia_min_keysize = AES_MIN_KEY_SIZE, 1185 .cia_max_keysize = AES_MAX_KEY_SIZE, 1186 .cia_setkey = aes_set_key, 1187 .cia_encrypt = aes_encrypt, 1188 .cia_decrypt = aes_decrypt 1189 } 1190 } 1191 }, { 1192 .cra_name = "__aes-aesni", 1193 .cra_driver_name = "__driver-aes-aesni", 1194 .cra_priority = 0, 1195 .cra_flags = CRYPTO_ALG_TYPE_CIPHER, 1196 .cra_blocksize = AES_BLOCK_SIZE, 1197 .cra_ctxsize = sizeof(struct crypto_aes_ctx) + 1198 AESNI_ALIGN - 1, 1199 .cra_alignmask = 0, 1200 .cra_module = THIS_MODULE, 1201 .cra_u = { 1202 .cipher = { 1203 .cia_min_keysize = AES_MIN_KEY_SIZE, 1204 .cia_max_keysize = AES_MAX_KEY_SIZE, 1205 .cia_setkey = aes_set_key, 1206 .cia_encrypt = __aes_encrypt, 1207 .cia_decrypt = __aes_decrypt 1208 } 1209 } 1210 }, { 1211 .cra_name = "__ecb-aes-aesni", 1212 .cra_driver_name = "__driver-ecb-aes-aesni", 1213 .cra_priority = 0, 1214 .cra_flags = CRYPTO_ALG_TYPE_BLKCIPHER, 1215 .cra_blocksize = AES_BLOCK_SIZE, 1216 .cra_ctxsize = sizeof(struct crypto_aes_ctx) + 1217 AESNI_ALIGN - 1, 1218 .cra_alignmask = 0, 1219 .cra_type = &crypto_blkcipher_type, 1220 .cra_module = THIS_MODULE, 1221 .cra_u = { 1222 .blkcipher = { 1223 .min_keysize = AES_MIN_KEY_SIZE, 1224 .max_keysize = AES_MAX_KEY_SIZE, 1225 .setkey = aes_set_key, 1226 .encrypt = ecb_encrypt, 1227 .decrypt = ecb_decrypt, 1228 }, 1229 }, 1230 }, { 1231 .cra_name = "__cbc-aes-aesni", 1232 .cra_driver_name = "__driver-cbc-aes-aesni", 1233 .cra_priority = 0, 1234 .cra_flags = CRYPTO_ALG_TYPE_BLKCIPHER, 1235 .cra_blocksize = AES_BLOCK_SIZE, 1236 .cra_ctxsize = sizeof(struct crypto_aes_ctx) + 1237 AESNI_ALIGN - 1, 1238 .cra_alignmask = 0, 1239 .cra_type = &crypto_blkcipher_type, 1240 .cra_module = THIS_MODULE, 1241 .cra_u = { 1242 .blkcipher = { 1243 .min_keysize = AES_MIN_KEY_SIZE, 1244 .max_keysize = AES_MAX_KEY_SIZE, 1245 .setkey = aes_set_key, 1246 .encrypt = cbc_encrypt, 1247 .decrypt = cbc_decrypt, 1248 }, 1249 }, 1250 }, { 1251 .cra_name = "ecb(aes)", 1252 .cra_driver_name = "ecb-aes-aesni", 1253 .cra_priority = 400, 1254 .cra_flags = CRYPTO_ALG_TYPE_ABLKCIPHER | CRYPTO_ALG_ASYNC, 1255 .cra_blocksize = AES_BLOCK_SIZE, 1256 .cra_ctxsize = sizeof(struct async_helper_ctx), 1257 .cra_alignmask = 0, 1258 .cra_type = &crypto_ablkcipher_type, 1259 .cra_module = THIS_MODULE, 1260 .cra_init = ablk_ecb_init, 1261 .cra_exit = ablk_exit, 1262 .cra_u = { 1263 .ablkcipher = { 1264 .min_keysize = AES_MIN_KEY_SIZE, 1265 .max_keysize = AES_MAX_KEY_SIZE, 1266 .setkey = ablk_set_key, 1267 .encrypt = ablk_encrypt, 1268 .decrypt = ablk_decrypt, 1269 }, 1270 }, 1271 }, { 1272 .cra_name = "cbc(aes)", 1273 .cra_driver_name = "cbc-aes-aesni", 1274 .cra_priority = 400, 1275 .cra_flags = CRYPTO_ALG_TYPE_ABLKCIPHER | CRYPTO_ALG_ASYNC, 1276 .cra_blocksize = AES_BLOCK_SIZE, 1277 .cra_ctxsize = sizeof(struct async_helper_ctx), 1278 .cra_alignmask = 0, 1279 .cra_type = &crypto_ablkcipher_type, 1280 .cra_module = THIS_MODULE, 1281 .cra_init = ablk_cbc_init, 1282 .cra_exit = ablk_exit, 1283 .cra_u = { 1284 .ablkcipher = { 1285 .min_keysize = AES_MIN_KEY_SIZE, 1286 .max_keysize = AES_MAX_KEY_SIZE, 1287 .ivsize = AES_BLOCK_SIZE, 1288 .setkey = ablk_set_key, 1289 .encrypt = ablk_encrypt, 1290 .decrypt = ablk_decrypt, 1291 }, 1292 }, 1293 #ifdef CONFIG_X86_64 1294 }, { 1295 .cra_name = "__ctr-aes-aesni", 1296 .cra_driver_name = "__driver-ctr-aes-aesni", 1297 .cra_priority = 0, 1298 .cra_flags = CRYPTO_ALG_TYPE_BLKCIPHER, 1299 .cra_blocksize = 1, 1300 .cra_ctxsize = sizeof(struct crypto_aes_ctx) + 1301 AESNI_ALIGN - 1, 1302 .cra_alignmask = 0, 1303 .cra_type = &crypto_blkcipher_type, 1304 .cra_module = THIS_MODULE, 1305 .cra_u = { 1306 .blkcipher = { 1307 .min_keysize = AES_MIN_KEY_SIZE, 1308 .max_keysize = AES_MAX_KEY_SIZE, 1309 .ivsize = AES_BLOCK_SIZE, 1310 .setkey = aes_set_key, 1311 .encrypt = ctr_crypt, 1312 .decrypt = ctr_crypt, 1313 }, 1314 }, 1315 }, { 1316 .cra_name = "ctr(aes)", 1317 .cra_driver_name = "ctr-aes-aesni", 1318 .cra_priority = 400, 1319 .cra_flags = CRYPTO_ALG_TYPE_ABLKCIPHER | CRYPTO_ALG_ASYNC, 1320 .cra_blocksize = 1, 1321 .cra_ctxsize = sizeof(struct async_helper_ctx), 1322 .cra_alignmask = 0, 1323 .cra_type = &crypto_ablkcipher_type, 1324 .cra_module = THIS_MODULE, 1325 .cra_init = ablk_ctr_init, 1326 .cra_exit = ablk_exit, 1327 .cra_u = { 1328 .ablkcipher = { 1329 .min_keysize = AES_MIN_KEY_SIZE, 1330 .max_keysize = AES_MAX_KEY_SIZE, 1331 .ivsize = AES_BLOCK_SIZE, 1332 .setkey = ablk_set_key, 1333 .encrypt = ablk_encrypt, 1334 .decrypt = ablk_encrypt, 1335 .geniv = "chainiv", 1336 }, 1337 }, 1338 }, { 1339 .cra_name = "__gcm-aes-aesni", 1340 .cra_driver_name = "__driver-gcm-aes-aesni", 1341 .cra_priority = 0, 1342 .cra_flags = CRYPTO_ALG_TYPE_AEAD, 1343 .cra_blocksize = 1, 1344 .cra_ctxsize = sizeof(struct aesni_rfc4106_gcm_ctx) + 1345 AESNI_ALIGN, 1346 .cra_alignmask = 0, 1347 .cra_type = &crypto_aead_type, 1348 .cra_module = THIS_MODULE, 1349 .cra_u = { 1350 .aead = { 1351 .encrypt = __driver_rfc4106_encrypt, 1352 .decrypt = __driver_rfc4106_decrypt, 1353 }, 1354 }, 1355 }, { 1356 .cra_name = "rfc4106(gcm(aes))", 1357 .cra_driver_name = "rfc4106-gcm-aesni", 1358 .cra_priority = 400, 1359 .cra_flags = CRYPTO_ALG_TYPE_AEAD | CRYPTO_ALG_ASYNC, 1360 .cra_blocksize = 1, 1361 .cra_ctxsize = sizeof(struct aesni_rfc4106_gcm_ctx) + 1362 AESNI_ALIGN, 1363 .cra_alignmask = 0, 1364 .cra_type = &crypto_nivaead_type, 1365 .cra_module = THIS_MODULE, 1366 .cra_init = rfc4106_init, 1367 .cra_exit = rfc4106_exit, 1368 .cra_u = { 1369 .aead = { 1370 .setkey = rfc4106_set_key, 1371 .setauthsize = rfc4106_set_authsize, 1372 .encrypt = rfc4106_encrypt, 1373 .decrypt = rfc4106_decrypt, 1374 .geniv = "seqiv", 1375 .ivsize = 8, 1376 .maxauthsize = 16, 1377 }, 1378 }, 1379 #endif 1380 #ifdef HAS_PCBC 1381 }, { 1382 .cra_name = "pcbc(aes)", 1383 .cra_driver_name = "pcbc-aes-aesni", 1384 .cra_priority = 400, 1385 .cra_flags = CRYPTO_ALG_TYPE_ABLKCIPHER | CRYPTO_ALG_ASYNC, 1386 .cra_blocksize = AES_BLOCK_SIZE, 1387 .cra_ctxsize = sizeof(struct async_helper_ctx), 1388 .cra_alignmask = 0, 1389 .cra_type = &crypto_ablkcipher_type, 1390 .cra_module = THIS_MODULE, 1391 .cra_init = ablk_pcbc_init, 1392 .cra_exit = ablk_exit, 1393 .cra_u = { 1394 .ablkcipher = { 1395 .min_keysize = AES_MIN_KEY_SIZE, 1396 .max_keysize = AES_MAX_KEY_SIZE, 1397 .ivsize = AES_BLOCK_SIZE, 1398 .setkey = ablk_set_key, 1399 .encrypt = ablk_encrypt, 1400 .decrypt = ablk_decrypt, 1401 }, 1402 }, 1403 #endif 1404 }, { 1405 .cra_name = "__lrw-aes-aesni", 1406 .cra_driver_name = "__driver-lrw-aes-aesni", 1407 .cra_priority = 0, 1408 .cra_flags = CRYPTO_ALG_TYPE_BLKCIPHER, 1409 .cra_blocksize = AES_BLOCK_SIZE, 1410 .cra_ctxsize = sizeof(struct aesni_lrw_ctx), 1411 .cra_alignmask = 0, 1412 .cra_type = &crypto_blkcipher_type, 1413 .cra_module = THIS_MODULE, 1414 .cra_exit = lrw_aesni_exit_tfm, 1415 .cra_u = { 1416 .blkcipher = { 1417 .min_keysize = AES_MIN_KEY_SIZE + AES_BLOCK_SIZE, 1418 .max_keysize = AES_MAX_KEY_SIZE + AES_BLOCK_SIZE, 1419 .ivsize = AES_BLOCK_SIZE, 1420 .setkey = lrw_aesni_setkey, 1421 .encrypt = lrw_encrypt, 1422 .decrypt = lrw_decrypt, 1423 }, 1424 }, 1425 }, { 1426 .cra_name = "__xts-aes-aesni", 1427 .cra_driver_name = "__driver-xts-aes-aesni", 1428 .cra_priority = 0, 1429 .cra_flags = CRYPTO_ALG_TYPE_BLKCIPHER, 1430 .cra_blocksize = AES_BLOCK_SIZE, 1431 .cra_ctxsize = sizeof(struct aesni_xts_ctx), 1432 .cra_alignmask = 0, 1433 .cra_type = &crypto_blkcipher_type, 1434 .cra_module = THIS_MODULE, 1435 .cra_u = { 1436 .blkcipher = { 1437 .min_keysize = 2 * AES_MIN_KEY_SIZE, 1438 .max_keysize = 2 * AES_MAX_KEY_SIZE, 1439 .ivsize = AES_BLOCK_SIZE, 1440 .setkey = xts_aesni_setkey, 1441 .encrypt = xts_encrypt, 1442 .decrypt = xts_decrypt, 1443 }, 1444 }, 1445 }, { 1446 .cra_name = "lrw(aes)", 1447 .cra_driver_name = "lrw-aes-aesni", 1448 .cra_priority = 400, 1449 .cra_flags = CRYPTO_ALG_TYPE_ABLKCIPHER | CRYPTO_ALG_ASYNC, 1450 .cra_blocksize = AES_BLOCK_SIZE, 1451 .cra_ctxsize = sizeof(struct async_helper_ctx), 1452 .cra_alignmask = 0, 1453 .cra_type = &crypto_ablkcipher_type, 1454 .cra_module = THIS_MODULE, 1455 .cra_init = ablk_init, 1456 .cra_exit = ablk_exit, 1457 .cra_u = { 1458 .ablkcipher = { 1459 .min_keysize = AES_MIN_KEY_SIZE + AES_BLOCK_SIZE, 1460 .max_keysize = AES_MAX_KEY_SIZE + AES_BLOCK_SIZE, 1461 .ivsize = AES_BLOCK_SIZE, 1462 .setkey = ablk_set_key, 1463 .encrypt = ablk_encrypt, 1464 .decrypt = ablk_decrypt, 1465 }, 1466 }, 1467 }, { 1468 .cra_name = "xts(aes)", 1469 .cra_driver_name = "xts-aes-aesni", 1470 .cra_priority = 400, 1471 .cra_flags = CRYPTO_ALG_TYPE_ABLKCIPHER | CRYPTO_ALG_ASYNC, 1472 .cra_blocksize = AES_BLOCK_SIZE, 1473 .cra_ctxsize = sizeof(struct async_helper_ctx), 1474 .cra_alignmask = 0, 1475 .cra_type = &crypto_ablkcipher_type, 1476 .cra_module = THIS_MODULE, 1477 .cra_init = ablk_init, 1478 .cra_exit = ablk_exit, 1479 .cra_u = { 1480 .ablkcipher = { 1481 .min_keysize = 2 * AES_MIN_KEY_SIZE, 1482 .max_keysize = 2 * AES_MAX_KEY_SIZE, 1483 .ivsize = AES_BLOCK_SIZE, 1484 .setkey = ablk_set_key, 1485 .encrypt = ablk_encrypt, 1486 .decrypt = ablk_decrypt, 1487 }, 1488 }, 1489 } }; 1490 1491 1492 static const struct x86_cpu_id aesni_cpu_id[] = { 1493 X86_FEATURE_MATCH(X86_FEATURE_AES), 1494 {} 1495 }; 1496 MODULE_DEVICE_TABLE(x86cpu, aesni_cpu_id); 1497 1498 static int __init aesni_init(void) 1499 { 1500 int err; 1501 1502 if (!x86_match_cpu(aesni_cpu_id)) 1503 return -ENODEV; 1504 #ifdef CONFIG_X86_64 1505 #ifdef CONFIG_AS_AVX2 1506 if (boot_cpu_has(X86_FEATURE_AVX2)) { 1507 pr_info("AVX2 version of gcm_enc/dec engaged.\n"); 1508 aesni_gcm_enc_tfm = aesni_gcm_enc_avx2; 1509 aesni_gcm_dec_tfm = aesni_gcm_dec_avx2; 1510 } else 1511 #endif 1512 #ifdef CONFIG_AS_AVX 1513 if (boot_cpu_has(X86_FEATURE_AVX)) { 1514 pr_info("AVX version of gcm_enc/dec engaged.\n"); 1515 aesni_gcm_enc_tfm = aesni_gcm_enc_avx; 1516 aesni_gcm_dec_tfm = aesni_gcm_dec_avx; 1517 } else 1518 #endif 1519 { 1520 pr_info("SSE version of gcm_enc/dec engaged.\n"); 1521 aesni_gcm_enc_tfm = aesni_gcm_enc; 1522 aesni_gcm_dec_tfm = aesni_gcm_dec; 1523 } 1524 aesni_ctr_enc_tfm = aesni_ctr_enc; 1525 #ifdef CONFIG_AS_AVX 1526 if (cpu_has_avx) { 1527 /* optimize performance of ctr mode encryption transform */ 1528 aesni_ctr_enc_tfm = aesni_ctr_enc_avx_tfm; 1529 pr_info("AES CTR mode by8 optimization enabled\n"); 1530 } 1531 #endif 1532 #endif 1533 1534 err = crypto_fpu_init(); 1535 if (err) 1536 return err; 1537 1538 return crypto_register_algs(aesni_algs, ARRAY_SIZE(aesni_algs)); 1539 } 1540 1541 static void __exit aesni_exit(void) 1542 { 1543 crypto_unregister_algs(aesni_algs, ARRAY_SIZE(aesni_algs)); 1544 1545 crypto_fpu_exit(); 1546 } 1547 1548 module_init(aesni_init); 1549 module_exit(aesni_exit); 1550 1551 MODULE_DESCRIPTION("Rijndael (AES) Cipher Algorithm, Intel AES-NI instructions optimized"); 1552 MODULE_LICENSE("GPL"); 1553 MODULE_ALIAS("aes"); 1554