1 // SPDX-License-Identifier: GPL-2.0-or-later 2 /* 3 * Support for Intel AES-NI instructions. This file contains glue 4 * code, the real AES implementation is in intel-aes_asm.S. 5 * 6 * Copyright (C) 2008, Intel Corp. 7 * Author: Huang Ying <ying.huang@intel.com> 8 * 9 * Added RFC4106 AES-GCM support for 128-bit keys under the AEAD 10 * interface for 64-bit kernels. 11 * Authors: Adrian Hoban <adrian.hoban@intel.com> 12 * Gabriele Paoloni <gabriele.paoloni@intel.com> 13 * Tadeusz Struk (tadeusz.struk@intel.com) 14 * Aidan O'Mahony (aidan.o.mahony@intel.com) 15 * Copyright (c) 2010, Intel Corporation. 16 */ 17 18 #include <linux/hardirq.h> 19 #include <linux/types.h> 20 #include <linux/module.h> 21 #include <linux/err.h> 22 #include <crypto/algapi.h> 23 #include <crypto/aes.h> 24 #include <crypto/ctr.h> 25 #include <crypto/b128ops.h> 26 #include <crypto/gcm.h> 27 #include <crypto/xts.h> 28 #include <asm/cpu_device_id.h> 29 #include <asm/simd.h> 30 #include <crypto/scatterwalk.h> 31 #include <crypto/internal/aead.h> 32 #include <crypto/internal/simd.h> 33 #include <crypto/internal/skcipher.h> 34 #include <linux/workqueue.h> 35 #include <linux/spinlock.h> 36 37 38 #define AESNI_ALIGN 16 39 #define AESNI_ALIGN_ATTR __attribute__ ((__aligned__(AESNI_ALIGN))) 40 #define AES_BLOCK_MASK (~(AES_BLOCK_SIZE - 1)) 41 #define RFC4106_HASH_SUBKEY_SIZE 16 42 #define AESNI_ALIGN_EXTRA ((AESNI_ALIGN - 1) & ~(CRYPTO_MINALIGN - 1)) 43 #define CRYPTO_AES_CTX_SIZE (sizeof(struct crypto_aes_ctx) + AESNI_ALIGN_EXTRA) 44 #define XTS_AES_CTX_SIZE (sizeof(struct aesni_xts_ctx) + AESNI_ALIGN_EXTRA) 45 46 /* This data is stored at the end of the crypto_tfm struct. 47 * It's a type of per "session" data storage location. 48 * This needs to be 16 byte aligned. 49 */ 50 struct aesni_rfc4106_gcm_ctx { 51 u8 hash_subkey[16] AESNI_ALIGN_ATTR; 52 struct crypto_aes_ctx aes_key_expanded AESNI_ALIGN_ATTR; 53 u8 nonce[4]; 54 }; 55 56 struct generic_gcmaes_ctx { 57 u8 hash_subkey[16] AESNI_ALIGN_ATTR; 58 struct crypto_aes_ctx aes_key_expanded AESNI_ALIGN_ATTR; 59 }; 60 61 struct aesni_xts_ctx { 62 u8 raw_tweak_ctx[sizeof(struct crypto_aes_ctx)] AESNI_ALIGN_ATTR; 63 u8 raw_crypt_ctx[sizeof(struct crypto_aes_ctx)] AESNI_ALIGN_ATTR; 64 }; 65 66 #define GCM_BLOCK_LEN 16 67 68 struct gcm_context_data { 69 /* init, update and finalize context data */ 70 u8 aad_hash[GCM_BLOCK_LEN]; 71 u64 aad_length; 72 u64 in_length; 73 u8 partial_block_enc_key[GCM_BLOCK_LEN]; 74 u8 orig_IV[GCM_BLOCK_LEN]; 75 u8 current_counter[GCM_BLOCK_LEN]; 76 u64 partial_block_len; 77 u64 unused; 78 u8 hash_keys[GCM_BLOCK_LEN * 16]; 79 }; 80 81 asmlinkage int aesni_set_key(struct crypto_aes_ctx *ctx, const u8 *in_key, 82 unsigned int key_len); 83 asmlinkage void aesni_enc(const void *ctx, u8 *out, const u8 *in); 84 asmlinkage void aesni_dec(const void *ctx, u8 *out, const u8 *in); 85 asmlinkage void aesni_ecb_enc(struct crypto_aes_ctx *ctx, u8 *out, 86 const u8 *in, unsigned int len); 87 asmlinkage void aesni_ecb_dec(struct crypto_aes_ctx *ctx, u8 *out, 88 const u8 *in, unsigned int len); 89 asmlinkage void aesni_cbc_enc(struct crypto_aes_ctx *ctx, u8 *out, 90 const u8 *in, unsigned int len, u8 *iv); 91 asmlinkage void aesni_cbc_dec(struct crypto_aes_ctx *ctx, u8 *out, 92 const u8 *in, unsigned int len, u8 *iv); 93 asmlinkage void aesni_cts_cbc_enc(struct crypto_aes_ctx *ctx, u8 *out, 94 const u8 *in, unsigned int len, u8 *iv); 95 asmlinkage void aesni_cts_cbc_dec(struct crypto_aes_ctx *ctx, u8 *out, 96 const u8 *in, unsigned int len, u8 *iv); 97 98 #define AVX_GEN2_OPTSIZE 640 99 #define AVX_GEN4_OPTSIZE 4096 100 101 asmlinkage void aesni_xts_encrypt(const struct crypto_aes_ctx *ctx, u8 *out, 102 const u8 *in, unsigned int len, u8 *iv); 103 104 asmlinkage void aesni_xts_decrypt(const struct crypto_aes_ctx *ctx, u8 *out, 105 const u8 *in, unsigned int len, u8 *iv); 106 107 #ifdef CONFIG_X86_64 108 109 static void (*aesni_ctr_enc_tfm)(struct crypto_aes_ctx *ctx, u8 *out, 110 const u8 *in, unsigned int len, u8 *iv); 111 asmlinkage void aesni_ctr_enc(struct crypto_aes_ctx *ctx, u8 *out, 112 const u8 *in, unsigned int len, u8 *iv); 113 114 /* asmlinkage void aesni_gcm_enc() 115 * void *ctx, AES Key schedule. Starts on a 16 byte boundary. 116 * struct gcm_context_data. May be uninitialized. 117 * u8 *out, Ciphertext output. Encrypt in-place is allowed. 118 * const u8 *in, Plaintext input 119 * unsigned long plaintext_len, Length of data in bytes for encryption. 120 * u8 *iv, Pre-counter block j0: 12 byte IV concatenated with 0x00000001. 121 * 16-byte aligned pointer. 122 * u8 *hash_subkey, the Hash sub key input. Data starts on a 16-byte boundary. 123 * const u8 *aad, Additional Authentication Data (AAD) 124 * unsigned long aad_len, Length of AAD in bytes. 125 * u8 *auth_tag, Authenticated Tag output. 126 * unsigned long auth_tag_len), Authenticated Tag Length in bytes. 127 * Valid values are 16 (most likely), 12 or 8. 128 */ 129 asmlinkage void aesni_gcm_enc(void *ctx, 130 struct gcm_context_data *gdata, u8 *out, 131 const u8 *in, unsigned long plaintext_len, u8 *iv, 132 u8 *hash_subkey, const u8 *aad, unsigned long aad_len, 133 u8 *auth_tag, unsigned long auth_tag_len); 134 135 /* asmlinkage void aesni_gcm_dec() 136 * void *ctx, AES Key schedule. Starts on a 16 byte boundary. 137 * struct gcm_context_data. May be uninitialized. 138 * u8 *out, Plaintext output. Decrypt in-place is allowed. 139 * const u8 *in, Ciphertext input 140 * unsigned long ciphertext_len, Length of data in bytes for decryption. 141 * u8 *iv, Pre-counter block j0: 12 byte IV concatenated with 0x00000001. 142 * 16-byte aligned pointer. 143 * u8 *hash_subkey, the Hash sub key input. Data starts on a 16-byte boundary. 144 * const u8 *aad, Additional Authentication Data (AAD) 145 * unsigned long aad_len, Length of AAD in bytes. With RFC4106 this is going 146 * to be 8 or 12 bytes 147 * u8 *auth_tag, Authenticated Tag output. 148 * unsigned long auth_tag_len) Authenticated Tag Length in bytes. 149 * Valid values are 16 (most likely), 12 or 8. 150 */ 151 asmlinkage void aesni_gcm_dec(void *ctx, 152 struct gcm_context_data *gdata, u8 *out, 153 const u8 *in, unsigned long ciphertext_len, u8 *iv, 154 u8 *hash_subkey, const u8 *aad, unsigned long aad_len, 155 u8 *auth_tag, unsigned long auth_tag_len); 156 157 /* Scatter / Gather routines, with args similar to above */ 158 asmlinkage void aesni_gcm_init(void *ctx, 159 struct gcm_context_data *gdata, 160 u8 *iv, 161 u8 *hash_subkey, const u8 *aad, 162 unsigned long aad_len); 163 asmlinkage void aesni_gcm_enc_update(void *ctx, 164 struct gcm_context_data *gdata, u8 *out, 165 const u8 *in, unsigned long plaintext_len); 166 asmlinkage void aesni_gcm_dec_update(void *ctx, 167 struct gcm_context_data *gdata, u8 *out, 168 const u8 *in, 169 unsigned long ciphertext_len); 170 asmlinkage void aesni_gcm_finalize(void *ctx, 171 struct gcm_context_data *gdata, 172 u8 *auth_tag, unsigned long auth_tag_len); 173 174 static const struct aesni_gcm_tfm_s { 175 void (*init)(void *ctx, struct gcm_context_data *gdata, u8 *iv, 176 u8 *hash_subkey, const u8 *aad, unsigned long aad_len); 177 void (*enc_update)(void *ctx, struct gcm_context_data *gdata, u8 *out, 178 const u8 *in, unsigned long plaintext_len); 179 void (*dec_update)(void *ctx, struct gcm_context_data *gdata, u8 *out, 180 const u8 *in, unsigned long ciphertext_len); 181 void (*finalize)(void *ctx, struct gcm_context_data *gdata, 182 u8 *auth_tag, unsigned long auth_tag_len); 183 } *aesni_gcm_tfm; 184 185 static const struct aesni_gcm_tfm_s aesni_gcm_tfm_sse = { 186 .init = &aesni_gcm_init, 187 .enc_update = &aesni_gcm_enc_update, 188 .dec_update = &aesni_gcm_dec_update, 189 .finalize = &aesni_gcm_finalize, 190 }; 191 192 asmlinkage void aes_ctr_enc_128_avx_by8(const u8 *in, u8 *iv, 193 void *keys, u8 *out, unsigned int num_bytes); 194 asmlinkage void aes_ctr_enc_192_avx_by8(const u8 *in, u8 *iv, 195 void *keys, u8 *out, unsigned int num_bytes); 196 asmlinkage void aes_ctr_enc_256_avx_by8(const u8 *in, u8 *iv, 197 void *keys, u8 *out, unsigned int num_bytes); 198 /* 199 * asmlinkage void aesni_gcm_init_avx_gen2() 200 * gcm_data *my_ctx_data, context data 201 * u8 *hash_subkey, the Hash sub key input. Data starts on a 16-byte boundary. 202 */ 203 asmlinkage void aesni_gcm_init_avx_gen2(void *my_ctx_data, 204 struct gcm_context_data *gdata, 205 u8 *iv, 206 u8 *hash_subkey, 207 const u8 *aad, 208 unsigned long aad_len); 209 210 asmlinkage void aesni_gcm_enc_update_avx_gen2(void *ctx, 211 struct gcm_context_data *gdata, u8 *out, 212 const u8 *in, unsigned long plaintext_len); 213 asmlinkage void aesni_gcm_dec_update_avx_gen2(void *ctx, 214 struct gcm_context_data *gdata, u8 *out, 215 const u8 *in, 216 unsigned long ciphertext_len); 217 asmlinkage void aesni_gcm_finalize_avx_gen2(void *ctx, 218 struct gcm_context_data *gdata, 219 u8 *auth_tag, unsigned long auth_tag_len); 220 221 asmlinkage void aesni_gcm_enc_avx_gen2(void *ctx, 222 struct gcm_context_data *gdata, u8 *out, 223 const u8 *in, unsigned long plaintext_len, u8 *iv, 224 const u8 *aad, unsigned long aad_len, 225 u8 *auth_tag, unsigned long auth_tag_len); 226 227 asmlinkage void aesni_gcm_dec_avx_gen2(void *ctx, 228 struct gcm_context_data *gdata, u8 *out, 229 const u8 *in, unsigned long ciphertext_len, u8 *iv, 230 const u8 *aad, unsigned long aad_len, 231 u8 *auth_tag, unsigned long auth_tag_len); 232 233 static const struct aesni_gcm_tfm_s aesni_gcm_tfm_avx_gen2 = { 234 .init = &aesni_gcm_init_avx_gen2, 235 .enc_update = &aesni_gcm_enc_update_avx_gen2, 236 .dec_update = &aesni_gcm_dec_update_avx_gen2, 237 .finalize = &aesni_gcm_finalize_avx_gen2, 238 }; 239 240 /* 241 * asmlinkage void aesni_gcm_init_avx_gen4() 242 * gcm_data *my_ctx_data, context data 243 * u8 *hash_subkey, the Hash sub key input. Data starts on a 16-byte boundary. 244 */ 245 asmlinkage void aesni_gcm_init_avx_gen4(void *my_ctx_data, 246 struct gcm_context_data *gdata, 247 u8 *iv, 248 u8 *hash_subkey, 249 const u8 *aad, 250 unsigned long aad_len); 251 252 asmlinkage void aesni_gcm_enc_update_avx_gen4(void *ctx, 253 struct gcm_context_data *gdata, u8 *out, 254 const u8 *in, unsigned long plaintext_len); 255 asmlinkage void aesni_gcm_dec_update_avx_gen4(void *ctx, 256 struct gcm_context_data *gdata, u8 *out, 257 const u8 *in, 258 unsigned long ciphertext_len); 259 asmlinkage void aesni_gcm_finalize_avx_gen4(void *ctx, 260 struct gcm_context_data *gdata, 261 u8 *auth_tag, unsigned long auth_tag_len); 262 263 asmlinkage void aesni_gcm_enc_avx_gen4(void *ctx, 264 struct gcm_context_data *gdata, u8 *out, 265 const u8 *in, unsigned long plaintext_len, u8 *iv, 266 const u8 *aad, unsigned long aad_len, 267 u8 *auth_tag, unsigned long auth_tag_len); 268 269 asmlinkage void aesni_gcm_dec_avx_gen4(void *ctx, 270 struct gcm_context_data *gdata, u8 *out, 271 const u8 *in, unsigned long ciphertext_len, u8 *iv, 272 const u8 *aad, unsigned long aad_len, 273 u8 *auth_tag, unsigned long auth_tag_len); 274 275 static const struct aesni_gcm_tfm_s aesni_gcm_tfm_avx_gen4 = { 276 .init = &aesni_gcm_init_avx_gen4, 277 .enc_update = &aesni_gcm_enc_update_avx_gen4, 278 .dec_update = &aesni_gcm_dec_update_avx_gen4, 279 .finalize = &aesni_gcm_finalize_avx_gen4, 280 }; 281 282 static inline struct 283 aesni_rfc4106_gcm_ctx *aesni_rfc4106_gcm_ctx_get(struct crypto_aead *tfm) 284 { 285 unsigned long align = AESNI_ALIGN; 286 287 if (align <= crypto_tfm_ctx_alignment()) 288 align = 1; 289 return PTR_ALIGN(crypto_aead_ctx(tfm), align); 290 } 291 292 static inline struct 293 generic_gcmaes_ctx *generic_gcmaes_ctx_get(struct crypto_aead *tfm) 294 { 295 unsigned long align = AESNI_ALIGN; 296 297 if (align <= crypto_tfm_ctx_alignment()) 298 align = 1; 299 return PTR_ALIGN(crypto_aead_ctx(tfm), align); 300 } 301 #endif 302 303 static inline struct crypto_aes_ctx *aes_ctx(void *raw_ctx) 304 { 305 unsigned long addr = (unsigned long)raw_ctx; 306 unsigned long align = AESNI_ALIGN; 307 308 if (align <= crypto_tfm_ctx_alignment()) 309 align = 1; 310 return (struct crypto_aes_ctx *)ALIGN(addr, align); 311 } 312 313 static int aes_set_key_common(struct crypto_tfm *tfm, void *raw_ctx, 314 const u8 *in_key, unsigned int key_len) 315 { 316 struct crypto_aes_ctx *ctx = aes_ctx(raw_ctx); 317 int err; 318 319 if (key_len != AES_KEYSIZE_128 && key_len != AES_KEYSIZE_192 && 320 key_len != AES_KEYSIZE_256) 321 return -EINVAL; 322 323 if (!crypto_simd_usable()) 324 err = aes_expandkey(ctx, in_key, key_len); 325 else { 326 kernel_fpu_begin(); 327 err = aesni_set_key(ctx, in_key, key_len); 328 kernel_fpu_end(); 329 } 330 331 return err; 332 } 333 334 static int aes_set_key(struct crypto_tfm *tfm, const u8 *in_key, 335 unsigned int key_len) 336 { 337 return aes_set_key_common(tfm, crypto_tfm_ctx(tfm), in_key, key_len); 338 } 339 340 static void aesni_encrypt(struct crypto_tfm *tfm, u8 *dst, const u8 *src) 341 { 342 struct crypto_aes_ctx *ctx = aes_ctx(crypto_tfm_ctx(tfm)); 343 344 if (!crypto_simd_usable()) { 345 aes_encrypt(ctx, dst, src); 346 } else { 347 kernel_fpu_begin(); 348 aesni_enc(ctx, dst, src); 349 kernel_fpu_end(); 350 } 351 } 352 353 static void aesni_decrypt(struct crypto_tfm *tfm, u8 *dst, const u8 *src) 354 { 355 struct crypto_aes_ctx *ctx = aes_ctx(crypto_tfm_ctx(tfm)); 356 357 if (!crypto_simd_usable()) { 358 aes_decrypt(ctx, dst, src); 359 } else { 360 kernel_fpu_begin(); 361 aesni_dec(ctx, dst, src); 362 kernel_fpu_end(); 363 } 364 } 365 366 static int aesni_skcipher_setkey(struct crypto_skcipher *tfm, const u8 *key, 367 unsigned int len) 368 { 369 return aes_set_key_common(crypto_skcipher_tfm(tfm), 370 crypto_skcipher_ctx(tfm), key, len); 371 } 372 373 static int ecb_encrypt(struct skcipher_request *req) 374 { 375 struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req); 376 struct crypto_aes_ctx *ctx = aes_ctx(crypto_skcipher_ctx(tfm)); 377 struct skcipher_walk walk; 378 unsigned int nbytes; 379 int err; 380 381 err = skcipher_walk_virt(&walk, req, true); 382 383 kernel_fpu_begin(); 384 while ((nbytes = walk.nbytes)) { 385 aesni_ecb_enc(ctx, walk.dst.virt.addr, walk.src.virt.addr, 386 nbytes & AES_BLOCK_MASK); 387 nbytes &= AES_BLOCK_SIZE - 1; 388 err = skcipher_walk_done(&walk, nbytes); 389 } 390 kernel_fpu_end(); 391 392 return err; 393 } 394 395 static int ecb_decrypt(struct skcipher_request *req) 396 { 397 struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req); 398 struct crypto_aes_ctx *ctx = aes_ctx(crypto_skcipher_ctx(tfm)); 399 struct skcipher_walk walk; 400 unsigned int nbytes; 401 int err; 402 403 err = skcipher_walk_virt(&walk, req, true); 404 405 kernel_fpu_begin(); 406 while ((nbytes = walk.nbytes)) { 407 aesni_ecb_dec(ctx, walk.dst.virt.addr, walk.src.virt.addr, 408 nbytes & AES_BLOCK_MASK); 409 nbytes &= AES_BLOCK_SIZE - 1; 410 err = skcipher_walk_done(&walk, nbytes); 411 } 412 kernel_fpu_end(); 413 414 return err; 415 } 416 417 static int cbc_encrypt(struct skcipher_request *req) 418 { 419 struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req); 420 struct crypto_aes_ctx *ctx = aes_ctx(crypto_skcipher_ctx(tfm)); 421 struct skcipher_walk walk; 422 unsigned int nbytes; 423 int err; 424 425 err = skcipher_walk_virt(&walk, req, true); 426 427 kernel_fpu_begin(); 428 while ((nbytes = walk.nbytes)) { 429 aesni_cbc_enc(ctx, walk.dst.virt.addr, walk.src.virt.addr, 430 nbytes & AES_BLOCK_MASK, walk.iv); 431 nbytes &= AES_BLOCK_SIZE - 1; 432 err = skcipher_walk_done(&walk, nbytes); 433 } 434 kernel_fpu_end(); 435 436 return err; 437 } 438 439 static int cbc_decrypt(struct skcipher_request *req) 440 { 441 struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req); 442 struct crypto_aes_ctx *ctx = aes_ctx(crypto_skcipher_ctx(tfm)); 443 struct skcipher_walk walk; 444 unsigned int nbytes; 445 int err; 446 447 err = skcipher_walk_virt(&walk, req, true); 448 449 kernel_fpu_begin(); 450 while ((nbytes = walk.nbytes)) { 451 aesni_cbc_dec(ctx, walk.dst.virt.addr, walk.src.virt.addr, 452 nbytes & AES_BLOCK_MASK, walk.iv); 453 nbytes &= AES_BLOCK_SIZE - 1; 454 err = skcipher_walk_done(&walk, nbytes); 455 } 456 kernel_fpu_end(); 457 458 return err; 459 } 460 461 static int cts_cbc_encrypt(struct skcipher_request *req) 462 { 463 struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req); 464 struct crypto_aes_ctx *ctx = aes_ctx(crypto_skcipher_ctx(tfm)); 465 int cbc_blocks = DIV_ROUND_UP(req->cryptlen, AES_BLOCK_SIZE) - 2; 466 struct scatterlist *src = req->src, *dst = req->dst; 467 struct scatterlist sg_src[2], sg_dst[2]; 468 struct skcipher_request subreq; 469 struct skcipher_walk walk; 470 int err; 471 472 skcipher_request_set_tfm(&subreq, tfm); 473 skcipher_request_set_callback(&subreq, skcipher_request_flags(req), 474 NULL, NULL); 475 476 if (req->cryptlen <= AES_BLOCK_SIZE) { 477 if (req->cryptlen < AES_BLOCK_SIZE) 478 return -EINVAL; 479 cbc_blocks = 1; 480 } 481 482 if (cbc_blocks > 0) { 483 skcipher_request_set_crypt(&subreq, req->src, req->dst, 484 cbc_blocks * AES_BLOCK_SIZE, 485 req->iv); 486 487 err = cbc_encrypt(&subreq); 488 if (err) 489 return err; 490 491 if (req->cryptlen == AES_BLOCK_SIZE) 492 return 0; 493 494 dst = src = scatterwalk_ffwd(sg_src, req->src, subreq.cryptlen); 495 if (req->dst != req->src) 496 dst = scatterwalk_ffwd(sg_dst, req->dst, 497 subreq.cryptlen); 498 } 499 500 /* handle ciphertext stealing */ 501 skcipher_request_set_crypt(&subreq, src, dst, 502 req->cryptlen - cbc_blocks * AES_BLOCK_SIZE, 503 req->iv); 504 505 err = skcipher_walk_virt(&walk, &subreq, false); 506 if (err) 507 return err; 508 509 kernel_fpu_begin(); 510 aesni_cts_cbc_enc(ctx, walk.dst.virt.addr, walk.src.virt.addr, 511 walk.nbytes, walk.iv); 512 kernel_fpu_end(); 513 514 return skcipher_walk_done(&walk, 0); 515 } 516 517 static int cts_cbc_decrypt(struct skcipher_request *req) 518 { 519 struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req); 520 struct crypto_aes_ctx *ctx = aes_ctx(crypto_skcipher_ctx(tfm)); 521 int cbc_blocks = DIV_ROUND_UP(req->cryptlen, AES_BLOCK_SIZE) - 2; 522 struct scatterlist *src = req->src, *dst = req->dst; 523 struct scatterlist sg_src[2], sg_dst[2]; 524 struct skcipher_request subreq; 525 struct skcipher_walk walk; 526 int err; 527 528 skcipher_request_set_tfm(&subreq, tfm); 529 skcipher_request_set_callback(&subreq, skcipher_request_flags(req), 530 NULL, NULL); 531 532 if (req->cryptlen <= AES_BLOCK_SIZE) { 533 if (req->cryptlen < AES_BLOCK_SIZE) 534 return -EINVAL; 535 cbc_blocks = 1; 536 } 537 538 if (cbc_blocks > 0) { 539 skcipher_request_set_crypt(&subreq, req->src, req->dst, 540 cbc_blocks * AES_BLOCK_SIZE, 541 req->iv); 542 543 err = cbc_decrypt(&subreq); 544 if (err) 545 return err; 546 547 if (req->cryptlen == AES_BLOCK_SIZE) 548 return 0; 549 550 dst = src = scatterwalk_ffwd(sg_src, req->src, subreq.cryptlen); 551 if (req->dst != req->src) 552 dst = scatterwalk_ffwd(sg_dst, req->dst, 553 subreq.cryptlen); 554 } 555 556 /* handle ciphertext stealing */ 557 skcipher_request_set_crypt(&subreq, src, dst, 558 req->cryptlen - cbc_blocks * AES_BLOCK_SIZE, 559 req->iv); 560 561 err = skcipher_walk_virt(&walk, &subreq, false); 562 if (err) 563 return err; 564 565 kernel_fpu_begin(); 566 aesni_cts_cbc_dec(ctx, walk.dst.virt.addr, walk.src.virt.addr, 567 walk.nbytes, walk.iv); 568 kernel_fpu_end(); 569 570 return skcipher_walk_done(&walk, 0); 571 } 572 573 #ifdef CONFIG_X86_64 574 static void ctr_crypt_final(struct crypto_aes_ctx *ctx, 575 struct skcipher_walk *walk) 576 { 577 u8 *ctrblk = walk->iv; 578 u8 keystream[AES_BLOCK_SIZE]; 579 u8 *src = walk->src.virt.addr; 580 u8 *dst = walk->dst.virt.addr; 581 unsigned int nbytes = walk->nbytes; 582 583 aesni_enc(ctx, keystream, ctrblk); 584 crypto_xor_cpy(dst, keystream, src, nbytes); 585 586 crypto_inc(ctrblk, AES_BLOCK_SIZE); 587 } 588 589 static void aesni_ctr_enc_avx_tfm(struct crypto_aes_ctx *ctx, u8 *out, 590 const u8 *in, unsigned int len, u8 *iv) 591 { 592 /* 593 * based on key length, override with the by8 version 594 * of ctr mode encryption/decryption for improved performance 595 * aes_set_key_common() ensures that key length is one of 596 * {128,192,256} 597 */ 598 if (ctx->key_length == AES_KEYSIZE_128) 599 aes_ctr_enc_128_avx_by8(in, iv, (void *)ctx, out, len); 600 else if (ctx->key_length == AES_KEYSIZE_192) 601 aes_ctr_enc_192_avx_by8(in, iv, (void *)ctx, out, len); 602 else 603 aes_ctr_enc_256_avx_by8(in, iv, (void *)ctx, out, len); 604 } 605 606 static int ctr_crypt(struct skcipher_request *req) 607 { 608 struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req); 609 struct crypto_aes_ctx *ctx = aes_ctx(crypto_skcipher_ctx(tfm)); 610 struct skcipher_walk walk; 611 unsigned int nbytes; 612 int err; 613 614 err = skcipher_walk_virt(&walk, req, true); 615 616 kernel_fpu_begin(); 617 while ((nbytes = walk.nbytes) >= AES_BLOCK_SIZE) { 618 aesni_ctr_enc_tfm(ctx, walk.dst.virt.addr, walk.src.virt.addr, 619 nbytes & AES_BLOCK_MASK, walk.iv); 620 nbytes &= AES_BLOCK_SIZE - 1; 621 err = skcipher_walk_done(&walk, nbytes); 622 } 623 if (walk.nbytes) { 624 ctr_crypt_final(ctx, &walk); 625 err = skcipher_walk_done(&walk, 0); 626 } 627 kernel_fpu_end(); 628 629 return err; 630 } 631 632 static int 633 rfc4106_set_hash_subkey(u8 *hash_subkey, const u8 *key, unsigned int key_len) 634 { 635 struct crypto_aes_ctx ctx; 636 int ret; 637 638 ret = aes_expandkey(&ctx, key, key_len); 639 if (ret) 640 return ret; 641 642 /* Clear the data in the hash sub key container to zero.*/ 643 /* We want to cipher all zeros to create the hash sub key. */ 644 memset(hash_subkey, 0, RFC4106_HASH_SUBKEY_SIZE); 645 646 aes_encrypt(&ctx, hash_subkey, hash_subkey); 647 648 memzero_explicit(&ctx, sizeof(ctx)); 649 return 0; 650 } 651 652 static int common_rfc4106_set_key(struct crypto_aead *aead, const u8 *key, 653 unsigned int key_len) 654 { 655 struct aesni_rfc4106_gcm_ctx *ctx = aesni_rfc4106_gcm_ctx_get(aead); 656 657 if (key_len < 4) 658 return -EINVAL; 659 660 /*Account for 4 byte nonce at the end.*/ 661 key_len -= 4; 662 663 memcpy(ctx->nonce, key + key_len, sizeof(ctx->nonce)); 664 665 return aes_set_key_common(crypto_aead_tfm(aead), 666 &ctx->aes_key_expanded, key, key_len) ?: 667 rfc4106_set_hash_subkey(ctx->hash_subkey, key, key_len); 668 } 669 670 /* This is the Integrity Check Value (aka the authentication tag) length and can 671 * be 8, 12 or 16 bytes long. */ 672 static int common_rfc4106_set_authsize(struct crypto_aead *aead, 673 unsigned int authsize) 674 { 675 switch (authsize) { 676 case 8: 677 case 12: 678 case 16: 679 break; 680 default: 681 return -EINVAL; 682 } 683 684 return 0; 685 } 686 687 static int generic_gcmaes_set_authsize(struct crypto_aead *tfm, 688 unsigned int authsize) 689 { 690 switch (authsize) { 691 case 4: 692 case 8: 693 case 12: 694 case 13: 695 case 14: 696 case 15: 697 case 16: 698 break; 699 default: 700 return -EINVAL; 701 } 702 703 return 0; 704 } 705 706 static int gcmaes_crypt_by_sg(bool enc, struct aead_request *req, 707 unsigned int assoclen, u8 *hash_subkey, 708 u8 *iv, void *aes_ctx) 709 { 710 struct crypto_aead *tfm = crypto_aead_reqtfm(req); 711 unsigned long auth_tag_len = crypto_aead_authsize(tfm); 712 const struct aesni_gcm_tfm_s *gcm_tfm = aesni_gcm_tfm; 713 struct gcm_context_data data AESNI_ALIGN_ATTR; 714 struct scatter_walk dst_sg_walk = {}; 715 unsigned long left = req->cryptlen; 716 unsigned long len, srclen, dstlen; 717 struct scatter_walk assoc_sg_walk; 718 struct scatter_walk src_sg_walk; 719 struct scatterlist src_start[2]; 720 struct scatterlist dst_start[2]; 721 struct scatterlist *src_sg; 722 struct scatterlist *dst_sg; 723 u8 *src, *dst, *assoc; 724 u8 *assocmem = NULL; 725 u8 authTag[16]; 726 727 if (!enc) 728 left -= auth_tag_len; 729 730 if (left < AVX_GEN4_OPTSIZE && gcm_tfm == &aesni_gcm_tfm_avx_gen4) 731 gcm_tfm = &aesni_gcm_tfm_avx_gen2; 732 if (left < AVX_GEN2_OPTSIZE && gcm_tfm == &aesni_gcm_tfm_avx_gen2) 733 gcm_tfm = &aesni_gcm_tfm_sse; 734 735 /* Linearize assoc, if not already linear */ 736 if (req->src->length >= assoclen && req->src->length && 737 (!PageHighMem(sg_page(req->src)) || 738 req->src->offset + req->src->length <= PAGE_SIZE)) { 739 scatterwalk_start(&assoc_sg_walk, req->src); 740 assoc = scatterwalk_map(&assoc_sg_walk); 741 } else { 742 /* assoc can be any length, so must be on heap */ 743 assocmem = kmalloc(assoclen, GFP_ATOMIC); 744 if (unlikely(!assocmem)) 745 return -ENOMEM; 746 assoc = assocmem; 747 748 scatterwalk_map_and_copy(assoc, req->src, 0, assoclen, 0); 749 } 750 751 if (left) { 752 src_sg = scatterwalk_ffwd(src_start, req->src, req->assoclen); 753 scatterwalk_start(&src_sg_walk, src_sg); 754 if (req->src != req->dst) { 755 dst_sg = scatterwalk_ffwd(dst_start, req->dst, 756 req->assoclen); 757 scatterwalk_start(&dst_sg_walk, dst_sg); 758 } 759 } 760 761 kernel_fpu_begin(); 762 gcm_tfm->init(aes_ctx, &data, iv, 763 hash_subkey, assoc, assoclen); 764 if (req->src != req->dst) { 765 while (left) { 766 src = scatterwalk_map(&src_sg_walk); 767 dst = scatterwalk_map(&dst_sg_walk); 768 srclen = scatterwalk_clamp(&src_sg_walk, left); 769 dstlen = scatterwalk_clamp(&dst_sg_walk, left); 770 len = min(srclen, dstlen); 771 if (len) { 772 if (enc) 773 gcm_tfm->enc_update(aes_ctx, &data, 774 dst, src, len); 775 else 776 gcm_tfm->dec_update(aes_ctx, &data, 777 dst, src, len); 778 } 779 left -= len; 780 781 scatterwalk_unmap(src); 782 scatterwalk_unmap(dst); 783 scatterwalk_advance(&src_sg_walk, len); 784 scatterwalk_advance(&dst_sg_walk, len); 785 scatterwalk_done(&src_sg_walk, 0, left); 786 scatterwalk_done(&dst_sg_walk, 1, left); 787 } 788 } else { 789 while (left) { 790 dst = src = scatterwalk_map(&src_sg_walk); 791 len = scatterwalk_clamp(&src_sg_walk, left); 792 if (len) { 793 if (enc) 794 gcm_tfm->enc_update(aes_ctx, &data, 795 src, src, len); 796 else 797 gcm_tfm->dec_update(aes_ctx, &data, 798 src, src, len); 799 } 800 left -= len; 801 scatterwalk_unmap(src); 802 scatterwalk_advance(&src_sg_walk, len); 803 scatterwalk_done(&src_sg_walk, 1, left); 804 } 805 } 806 gcm_tfm->finalize(aes_ctx, &data, authTag, auth_tag_len); 807 kernel_fpu_end(); 808 809 if (!assocmem) 810 scatterwalk_unmap(assoc); 811 else 812 kfree(assocmem); 813 814 if (!enc) { 815 u8 authTagMsg[16]; 816 817 /* Copy out original authTag */ 818 scatterwalk_map_and_copy(authTagMsg, req->src, 819 req->assoclen + req->cryptlen - 820 auth_tag_len, 821 auth_tag_len, 0); 822 823 /* Compare generated tag with passed in tag. */ 824 return crypto_memneq(authTagMsg, authTag, auth_tag_len) ? 825 -EBADMSG : 0; 826 } 827 828 /* Copy in the authTag */ 829 scatterwalk_map_and_copy(authTag, req->dst, 830 req->assoclen + req->cryptlen, 831 auth_tag_len, 1); 832 833 return 0; 834 } 835 836 static int gcmaes_encrypt(struct aead_request *req, unsigned int assoclen, 837 u8 *hash_subkey, u8 *iv, void *aes_ctx) 838 { 839 return gcmaes_crypt_by_sg(true, req, assoclen, hash_subkey, iv, 840 aes_ctx); 841 } 842 843 static int gcmaes_decrypt(struct aead_request *req, unsigned int assoclen, 844 u8 *hash_subkey, u8 *iv, void *aes_ctx) 845 { 846 return gcmaes_crypt_by_sg(false, req, assoclen, hash_subkey, iv, 847 aes_ctx); 848 } 849 850 static int helper_rfc4106_encrypt(struct aead_request *req) 851 { 852 struct crypto_aead *tfm = crypto_aead_reqtfm(req); 853 struct aesni_rfc4106_gcm_ctx *ctx = aesni_rfc4106_gcm_ctx_get(tfm); 854 void *aes_ctx = &(ctx->aes_key_expanded); 855 u8 iv[16] __attribute__ ((__aligned__(AESNI_ALIGN))); 856 unsigned int i; 857 __be32 counter = cpu_to_be32(1); 858 859 /* Assuming we are supporting rfc4106 64-bit extended */ 860 /* sequence numbers We need to have the AAD length equal */ 861 /* to 16 or 20 bytes */ 862 if (unlikely(req->assoclen != 16 && req->assoclen != 20)) 863 return -EINVAL; 864 865 /* IV below built */ 866 for (i = 0; i < 4; i++) 867 *(iv+i) = ctx->nonce[i]; 868 for (i = 0; i < 8; i++) 869 *(iv+4+i) = req->iv[i]; 870 *((__be32 *)(iv+12)) = counter; 871 872 return gcmaes_encrypt(req, req->assoclen - 8, ctx->hash_subkey, iv, 873 aes_ctx); 874 } 875 876 static int helper_rfc4106_decrypt(struct aead_request *req) 877 { 878 __be32 counter = cpu_to_be32(1); 879 struct crypto_aead *tfm = crypto_aead_reqtfm(req); 880 struct aesni_rfc4106_gcm_ctx *ctx = aesni_rfc4106_gcm_ctx_get(tfm); 881 void *aes_ctx = &(ctx->aes_key_expanded); 882 u8 iv[16] __attribute__ ((__aligned__(AESNI_ALIGN))); 883 unsigned int i; 884 885 if (unlikely(req->assoclen != 16 && req->assoclen != 20)) 886 return -EINVAL; 887 888 /* Assuming we are supporting rfc4106 64-bit extended */ 889 /* sequence numbers We need to have the AAD length */ 890 /* equal to 16 or 20 bytes */ 891 892 /* IV below built */ 893 for (i = 0; i < 4; i++) 894 *(iv+i) = ctx->nonce[i]; 895 for (i = 0; i < 8; i++) 896 *(iv+4+i) = req->iv[i]; 897 *((__be32 *)(iv+12)) = counter; 898 899 return gcmaes_decrypt(req, req->assoclen - 8, ctx->hash_subkey, iv, 900 aes_ctx); 901 } 902 #endif 903 904 static int xts_aesni_setkey(struct crypto_skcipher *tfm, const u8 *key, 905 unsigned int keylen) 906 { 907 struct aesni_xts_ctx *ctx = crypto_skcipher_ctx(tfm); 908 int err; 909 910 err = xts_verify_key(tfm, key, keylen); 911 if (err) 912 return err; 913 914 keylen /= 2; 915 916 /* first half of xts-key is for crypt */ 917 err = aes_set_key_common(crypto_skcipher_tfm(tfm), ctx->raw_crypt_ctx, 918 key, keylen); 919 if (err) 920 return err; 921 922 /* second half of xts-key is for tweak */ 923 return aes_set_key_common(crypto_skcipher_tfm(tfm), ctx->raw_tweak_ctx, 924 key + keylen, keylen); 925 } 926 927 static int xts_crypt(struct skcipher_request *req, bool encrypt) 928 { 929 struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req); 930 struct aesni_xts_ctx *ctx = crypto_skcipher_ctx(tfm); 931 int tail = req->cryptlen % AES_BLOCK_SIZE; 932 struct skcipher_request subreq; 933 struct skcipher_walk walk; 934 int err; 935 936 if (req->cryptlen < AES_BLOCK_SIZE) 937 return -EINVAL; 938 939 err = skcipher_walk_virt(&walk, req, false); 940 941 if (unlikely(tail > 0 && walk.nbytes < walk.total)) { 942 int blocks = DIV_ROUND_UP(req->cryptlen, AES_BLOCK_SIZE) - 2; 943 944 skcipher_walk_abort(&walk); 945 946 skcipher_request_set_tfm(&subreq, tfm); 947 skcipher_request_set_callback(&subreq, 948 skcipher_request_flags(req), 949 NULL, NULL); 950 skcipher_request_set_crypt(&subreq, req->src, req->dst, 951 blocks * AES_BLOCK_SIZE, req->iv); 952 req = &subreq; 953 err = skcipher_walk_virt(&walk, req, false); 954 } else { 955 tail = 0; 956 } 957 958 kernel_fpu_begin(); 959 960 /* calculate first value of T */ 961 aesni_enc(aes_ctx(ctx->raw_tweak_ctx), walk.iv, walk.iv); 962 963 while (walk.nbytes > 0) { 964 int nbytes = walk.nbytes; 965 966 if (nbytes < walk.total) 967 nbytes &= ~(AES_BLOCK_SIZE - 1); 968 969 if (encrypt) 970 aesni_xts_encrypt(aes_ctx(ctx->raw_crypt_ctx), 971 walk.dst.virt.addr, walk.src.virt.addr, 972 nbytes, walk.iv); 973 else 974 aesni_xts_decrypt(aes_ctx(ctx->raw_crypt_ctx), 975 walk.dst.virt.addr, walk.src.virt.addr, 976 nbytes, walk.iv); 977 kernel_fpu_end(); 978 979 err = skcipher_walk_done(&walk, walk.nbytes - nbytes); 980 981 if (walk.nbytes > 0) 982 kernel_fpu_begin(); 983 } 984 985 if (unlikely(tail > 0 && !err)) { 986 struct scatterlist sg_src[2], sg_dst[2]; 987 struct scatterlist *src, *dst; 988 989 dst = src = scatterwalk_ffwd(sg_src, req->src, req->cryptlen); 990 if (req->dst != req->src) 991 dst = scatterwalk_ffwd(sg_dst, req->dst, req->cryptlen); 992 993 skcipher_request_set_crypt(req, src, dst, AES_BLOCK_SIZE + tail, 994 req->iv); 995 996 err = skcipher_walk_virt(&walk, &subreq, false); 997 if (err) 998 return err; 999 1000 kernel_fpu_begin(); 1001 if (encrypt) 1002 aesni_xts_encrypt(aes_ctx(ctx->raw_crypt_ctx), 1003 walk.dst.virt.addr, walk.src.virt.addr, 1004 walk.nbytes, walk.iv); 1005 else 1006 aesni_xts_decrypt(aes_ctx(ctx->raw_crypt_ctx), 1007 walk.dst.virt.addr, walk.src.virt.addr, 1008 walk.nbytes, walk.iv); 1009 kernel_fpu_end(); 1010 1011 err = skcipher_walk_done(&walk, 0); 1012 } 1013 return err; 1014 } 1015 1016 static int xts_encrypt(struct skcipher_request *req) 1017 { 1018 return xts_crypt(req, true); 1019 } 1020 1021 static int xts_decrypt(struct skcipher_request *req) 1022 { 1023 return xts_crypt(req, false); 1024 } 1025 1026 static struct crypto_alg aesni_cipher_alg = { 1027 .cra_name = "aes", 1028 .cra_driver_name = "aes-aesni", 1029 .cra_priority = 300, 1030 .cra_flags = CRYPTO_ALG_TYPE_CIPHER, 1031 .cra_blocksize = AES_BLOCK_SIZE, 1032 .cra_ctxsize = CRYPTO_AES_CTX_SIZE, 1033 .cra_module = THIS_MODULE, 1034 .cra_u = { 1035 .cipher = { 1036 .cia_min_keysize = AES_MIN_KEY_SIZE, 1037 .cia_max_keysize = AES_MAX_KEY_SIZE, 1038 .cia_setkey = aes_set_key, 1039 .cia_encrypt = aesni_encrypt, 1040 .cia_decrypt = aesni_decrypt 1041 } 1042 } 1043 }; 1044 1045 static struct skcipher_alg aesni_skciphers[] = { 1046 { 1047 .base = { 1048 .cra_name = "__ecb(aes)", 1049 .cra_driver_name = "__ecb-aes-aesni", 1050 .cra_priority = 400, 1051 .cra_flags = CRYPTO_ALG_INTERNAL, 1052 .cra_blocksize = AES_BLOCK_SIZE, 1053 .cra_ctxsize = CRYPTO_AES_CTX_SIZE, 1054 .cra_module = THIS_MODULE, 1055 }, 1056 .min_keysize = AES_MIN_KEY_SIZE, 1057 .max_keysize = AES_MAX_KEY_SIZE, 1058 .setkey = aesni_skcipher_setkey, 1059 .encrypt = ecb_encrypt, 1060 .decrypt = ecb_decrypt, 1061 }, { 1062 .base = { 1063 .cra_name = "__cbc(aes)", 1064 .cra_driver_name = "__cbc-aes-aesni", 1065 .cra_priority = 400, 1066 .cra_flags = CRYPTO_ALG_INTERNAL, 1067 .cra_blocksize = AES_BLOCK_SIZE, 1068 .cra_ctxsize = CRYPTO_AES_CTX_SIZE, 1069 .cra_module = THIS_MODULE, 1070 }, 1071 .min_keysize = AES_MIN_KEY_SIZE, 1072 .max_keysize = AES_MAX_KEY_SIZE, 1073 .ivsize = AES_BLOCK_SIZE, 1074 .setkey = aesni_skcipher_setkey, 1075 .encrypt = cbc_encrypt, 1076 .decrypt = cbc_decrypt, 1077 }, { 1078 .base = { 1079 .cra_name = "__cts(cbc(aes))", 1080 .cra_driver_name = "__cts-cbc-aes-aesni", 1081 .cra_priority = 400, 1082 .cra_flags = CRYPTO_ALG_INTERNAL, 1083 .cra_blocksize = AES_BLOCK_SIZE, 1084 .cra_ctxsize = CRYPTO_AES_CTX_SIZE, 1085 .cra_module = THIS_MODULE, 1086 }, 1087 .min_keysize = AES_MIN_KEY_SIZE, 1088 .max_keysize = AES_MAX_KEY_SIZE, 1089 .ivsize = AES_BLOCK_SIZE, 1090 .walksize = 2 * AES_BLOCK_SIZE, 1091 .setkey = aesni_skcipher_setkey, 1092 .encrypt = cts_cbc_encrypt, 1093 .decrypt = cts_cbc_decrypt, 1094 #ifdef CONFIG_X86_64 1095 }, { 1096 .base = { 1097 .cra_name = "__ctr(aes)", 1098 .cra_driver_name = "__ctr-aes-aesni", 1099 .cra_priority = 400, 1100 .cra_flags = CRYPTO_ALG_INTERNAL, 1101 .cra_blocksize = 1, 1102 .cra_ctxsize = CRYPTO_AES_CTX_SIZE, 1103 .cra_module = THIS_MODULE, 1104 }, 1105 .min_keysize = AES_MIN_KEY_SIZE, 1106 .max_keysize = AES_MAX_KEY_SIZE, 1107 .ivsize = AES_BLOCK_SIZE, 1108 .chunksize = AES_BLOCK_SIZE, 1109 .setkey = aesni_skcipher_setkey, 1110 .encrypt = ctr_crypt, 1111 .decrypt = ctr_crypt, 1112 #endif 1113 }, { 1114 .base = { 1115 .cra_name = "__xts(aes)", 1116 .cra_driver_name = "__xts-aes-aesni", 1117 .cra_priority = 401, 1118 .cra_flags = CRYPTO_ALG_INTERNAL, 1119 .cra_blocksize = AES_BLOCK_SIZE, 1120 .cra_ctxsize = XTS_AES_CTX_SIZE, 1121 .cra_module = THIS_MODULE, 1122 }, 1123 .min_keysize = 2 * AES_MIN_KEY_SIZE, 1124 .max_keysize = 2 * AES_MAX_KEY_SIZE, 1125 .ivsize = AES_BLOCK_SIZE, 1126 .walksize = 2 * AES_BLOCK_SIZE, 1127 .setkey = xts_aesni_setkey, 1128 .encrypt = xts_encrypt, 1129 .decrypt = xts_decrypt, 1130 } 1131 }; 1132 1133 static 1134 struct simd_skcipher_alg *aesni_simd_skciphers[ARRAY_SIZE(aesni_skciphers)]; 1135 1136 #ifdef CONFIG_X86_64 1137 static int generic_gcmaes_set_key(struct crypto_aead *aead, const u8 *key, 1138 unsigned int key_len) 1139 { 1140 struct generic_gcmaes_ctx *ctx = generic_gcmaes_ctx_get(aead); 1141 1142 return aes_set_key_common(crypto_aead_tfm(aead), 1143 &ctx->aes_key_expanded, key, key_len) ?: 1144 rfc4106_set_hash_subkey(ctx->hash_subkey, key, key_len); 1145 } 1146 1147 static int generic_gcmaes_encrypt(struct aead_request *req) 1148 { 1149 struct crypto_aead *tfm = crypto_aead_reqtfm(req); 1150 struct generic_gcmaes_ctx *ctx = generic_gcmaes_ctx_get(tfm); 1151 void *aes_ctx = &(ctx->aes_key_expanded); 1152 u8 iv[16] __attribute__ ((__aligned__(AESNI_ALIGN))); 1153 __be32 counter = cpu_to_be32(1); 1154 1155 memcpy(iv, req->iv, 12); 1156 *((__be32 *)(iv+12)) = counter; 1157 1158 return gcmaes_encrypt(req, req->assoclen, ctx->hash_subkey, iv, 1159 aes_ctx); 1160 } 1161 1162 static int generic_gcmaes_decrypt(struct aead_request *req) 1163 { 1164 __be32 counter = cpu_to_be32(1); 1165 struct crypto_aead *tfm = crypto_aead_reqtfm(req); 1166 struct generic_gcmaes_ctx *ctx = generic_gcmaes_ctx_get(tfm); 1167 void *aes_ctx = &(ctx->aes_key_expanded); 1168 u8 iv[16] __attribute__ ((__aligned__(AESNI_ALIGN))); 1169 1170 memcpy(iv, req->iv, 12); 1171 *((__be32 *)(iv+12)) = counter; 1172 1173 return gcmaes_decrypt(req, req->assoclen, ctx->hash_subkey, iv, 1174 aes_ctx); 1175 } 1176 1177 static struct aead_alg aesni_aeads[] = { { 1178 .setkey = common_rfc4106_set_key, 1179 .setauthsize = common_rfc4106_set_authsize, 1180 .encrypt = helper_rfc4106_encrypt, 1181 .decrypt = helper_rfc4106_decrypt, 1182 .ivsize = GCM_RFC4106_IV_SIZE, 1183 .maxauthsize = 16, 1184 .base = { 1185 .cra_name = "__rfc4106(gcm(aes))", 1186 .cra_driver_name = "__rfc4106-gcm-aesni", 1187 .cra_priority = 400, 1188 .cra_flags = CRYPTO_ALG_INTERNAL, 1189 .cra_blocksize = 1, 1190 .cra_ctxsize = sizeof(struct aesni_rfc4106_gcm_ctx), 1191 .cra_alignmask = AESNI_ALIGN - 1, 1192 .cra_module = THIS_MODULE, 1193 }, 1194 }, { 1195 .setkey = generic_gcmaes_set_key, 1196 .setauthsize = generic_gcmaes_set_authsize, 1197 .encrypt = generic_gcmaes_encrypt, 1198 .decrypt = generic_gcmaes_decrypt, 1199 .ivsize = GCM_AES_IV_SIZE, 1200 .maxauthsize = 16, 1201 .base = { 1202 .cra_name = "__gcm(aes)", 1203 .cra_driver_name = "__generic-gcm-aesni", 1204 .cra_priority = 400, 1205 .cra_flags = CRYPTO_ALG_INTERNAL, 1206 .cra_blocksize = 1, 1207 .cra_ctxsize = sizeof(struct generic_gcmaes_ctx), 1208 .cra_alignmask = AESNI_ALIGN - 1, 1209 .cra_module = THIS_MODULE, 1210 }, 1211 } }; 1212 #else 1213 static struct aead_alg aesni_aeads[0]; 1214 #endif 1215 1216 static struct simd_aead_alg *aesni_simd_aeads[ARRAY_SIZE(aesni_aeads)]; 1217 1218 static const struct x86_cpu_id aesni_cpu_id[] = { 1219 X86_MATCH_FEATURE(X86_FEATURE_AES, NULL), 1220 {} 1221 }; 1222 MODULE_DEVICE_TABLE(x86cpu, aesni_cpu_id); 1223 1224 static int __init aesni_init(void) 1225 { 1226 int err; 1227 1228 if (!x86_match_cpu(aesni_cpu_id)) 1229 return -ENODEV; 1230 #ifdef CONFIG_X86_64 1231 if (boot_cpu_has(X86_FEATURE_AVX2)) { 1232 pr_info("AVX2 version of gcm_enc/dec engaged.\n"); 1233 aesni_gcm_tfm = &aesni_gcm_tfm_avx_gen4; 1234 } else 1235 if (boot_cpu_has(X86_FEATURE_AVX)) { 1236 pr_info("AVX version of gcm_enc/dec engaged.\n"); 1237 aesni_gcm_tfm = &aesni_gcm_tfm_avx_gen2; 1238 } else { 1239 pr_info("SSE version of gcm_enc/dec engaged.\n"); 1240 aesni_gcm_tfm = &aesni_gcm_tfm_sse; 1241 } 1242 aesni_ctr_enc_tfm = aesni_ctr_enc; 1243 if (boot_cpu_has(X86_FEATURE_AVX)) { 1244 /* optimize performance of ctr mode encryption transform */ 1245 aesni_ctr_enc_tfm = aesni_ctr_enc_avx_tfm; 1246 pr_info("AES CTR mode by8 optimization enabled\n"); 1247 } 1248 #endif 1249 1250 err = crypto_register_alg(&aesni_cipher_alg); 1251 if (err) 1252 return err; 1253 1254 err = simd_register_skciphers_compat(aesni_skciphers, 1255 ARRAY_SIZE(aesni_skciphers), 1256 aesni_simd_skciphers); 1257 if (err) 1258 goto unregister_cipher; 1259 1260 err = simd_register_aeads_compat(aesni_aeads, ARRAY_SIZE(aesni_aeads), 1261 aesni_simd_aeads); 1262 if (err) 1263 goto unregister_skciphers; 1264 1265 return 0; 1266 1267 unregister_skciphers: 1268 simd_unregister_skciphers(aesni_skciphers, ARRAY_SIZE(aesni_skciphers), 1269 aesni_simd_skciphers); 1270 unregister_cipher: 1271 crypto_unregister_alg(&aesni_cipher_alg); 1272 return err; 1273 } 1274 1275 static void __exit aesni_exit(void) 1276 { 1277 simd_unregister_aeads(aesni_aeads, ARRAY_SIZE(aesni_aeads), 1278 aesni_simd_aeads); 1279 simd_unregister_skciphers(aesni_skciphers, ARRAY_SIZE(aesni_skciphers), 1280 aesni_simd_skciphers); 1281 crypto_unregister_alg(&aesni_cipher_alg); 1282 } 1283 1284 late_initcall(aesni_init); 1285 module_exit(aesni_exit); 1286 1287 MODULE_DESCRIPTION("Rijndael (AES) Cipher Algorithm, Intel AES-NI instructions optimized"); 1288 MODULE_LICENSE("GPL"); 1289 MODULE_ALIAS_CRYPTO("aes"); 1290