1 // SPDX-License-Identifier: GPL-2.0-or-later 2 /* 3 * Support for Intel AES-NI instructions. This file contains glue 4 * code, the real AES implementation is in intel-aes_asm.S. 5 * 6 * Copyright (C) 2008, Intel Corp. 7 * Author: Huang Ying <ying.huang@intel.com> 8 * 9 * Added RFC4106 AES-GCM support for 128-bit keys under the AEAD 10 * interface for 64-bit kernels. 11 * Authors: Adrian Hoban <adrian.hoban@intel.com> 12 * Gabriele Paoloni <gabriele.paoloni@intel.com> 13 * Tadeusz Struk (tadeusz.struk@intel.com) 14 * Aidan O'Mahony (aidan.o.mahony@intel.com) 15 * Copyright (c) 2010, Intel Corporation. 16 */ 17 18 #include <linux/hardirq.h> 19 #include <linux/types.h> 20 #include <linux/module.h> 21 #include <linux/err.h> 22 #include <crypto/algapi.h> 23 #include <crypto/aes.h> 24 #include <crypto/ctr.h> 25 #include <crypto/b128ops.h> 26 #include <crypto/gcm.h> 27 #include <crypto/xts.h> 28 #include <asm/cpu_device_id.h> 29 #include <asm/crypto/aes.h> 30 #include <asm/simd.h> 31 #include <crypto/scatterwalk.h> 32 #include <crypto/internal/aead.h> 33 #include <crypto/internal/simd.h> 34 #include <crypto/internal/skcipher.h> 35 #include <linux/workqueue.h> 36 #include <linux/spinlock.h> 37 #ifdef CONFIG_X86_64 38 #include <asm/crypto/glue_helper.h> 39 #endif 40 41 42 #define AESNI_ALIGN 16 43 #define AESNI_ALIGN_ATTR __attribute__ ((__aligned__(AESNI_ALIGN))) 44 #define AES_BLOCK_MASK (~(AES_BLOCK_SIZE - 1)) 45 #define RFC4106_HASH_SUBKEY_SIZE 16 46 #define AESNI_ALIGN_EXTRA ((AESNI_ALIGN - 1) & ~(CRYPTO_MINALIGN - 1)) 47 #define CRYPTO_AES_CTX_SIZE (sizeof(struct crypto_aes_ctx) + AESNI_ALIGN_EXTRA) 48 #define XTS_AES_CTX_SIZE (sizeof(struct aesni_xts_ctx) + AESNI_ALIGN_EXTRA) 49 50 /* This data is stored at the end of the crypto_tfm struct. 51 * It's a type of per "session" data storage location. 52 * This needs to be 16 byte aligned. 53 */ 54 struct aesni_rfc4106_gcm_ctx { 55 u8 hash_subkey[16] AESNI_ALIGN_ATTR; 56 struct crypto_aes_ctx aes_key_expanded AESNI_ALIGN_ATTR; 57 u8 nonce[4]; 58 }; 59 60 struct generic_gcmaes_ctx { 61 u8 hash_subkey[16] AESNI_ALIGN_ATTR; 62 struct crypto_aes_ctx aes_key_expanded AESNI_ALIGN_ATTR; 63 }; 64 65 struct aesni_xts_ctx { 66 u8 raw_tweak_ctx[sizeof(struct crypto_aes_ctx)] AESNI_ALIGN_ATTR; 67 u8 raw_crypt_ctx[sizeof(struct crypto_aes_ctx)] AESNI_ALIGN_ATTR; 68 }; 69 70 #define GCM_BLOCK_LEN 16 71 72 struct gcm_context_data { 73 /* init, update and finalize context data */ 74 u8 aad_hash[GCM_BLOCK_LEN]; 75 u64 aad_length; 76 u64 in_length; 77 u8 partial_block_enc_key[GCM_BLOCK_LEN]; 78 u8 orig_IV[GCM_BLOCK_LEN]; 79 u8 current_counter[GCM_BLOCK_LEN]; 80 u64 partial_block_len; 81 u64 unused; 82 u8 hash_keys[GCM_BLOCK_LEN * 16]; 83 }; 84 85 asmlinkage int aesni_set_key(struct crypto_aes_ctx *ctx, const u8 *in_key, 86 unsigned int key_len); 87 asmlinkage void aesni_enc(struct crypto_aes_ctx *ctx, u8 *out, 88 const u8 *in); 89 asmlinkage void aesni_dec(struct crypto_aes_ctx *ctx, u8 *out, 90 const u8 *in); 91 asmlinkage void aesni_ecb_enc(struct crypto_aes_ctx *ctx, u8 *out, 92 const u8 *in, unsigned int len); 93 asmlinkage void aesni_ecb_dec(struct crypto_aes_ctx *ctx, u8 *out, 94 const u8 *in, unsigned int len); 95 asmlinkage void aesni_cbc_enc(struct crypto_aes_ctx *ctx, u8 *out, 96 const u8 *in, unsigned int len, u8 *iv); 97 asmlinkage void aesni_cbc_dec(struct crypto_aes_ctx *ctx, u8 *out, 98 const u8 *in, unsigned int len, u8 *iv); 99 100 #define AVX_GEN2_OPTSIZE 640 101 #define AVX_GEN4_OPTSIZE 4096 102 103 #ifdef CONFIG_X86_64 104 105 static void (*aesni_ctr_enc_tfm)(struct crypto_aes_ctx *ctx, u8 *out, 106 const u8 *in, unsigned int len, u8 *iv); 107 asmlinkage void aesni_ctr_enc(struct crypto_aes_ctx *ctx, u8 *out, 108 const u8 *in, unsigned int len, u8 *iv); 109 110 asmlinkage void aesni_xts_crypt8(struct crypto_aes_ctx *ctx, u8 *out, 111 const u8 *in, bool enc, u8 *iv); 112 113 /* asmlinkage void aesni_gcm_enc() 114 * void *ctx, AES Key schedule. Starts on a 16 byte boundary. 115 * struct gcm_context_data. May be uninitialized. 116 * u8 *out, Ciphertext output. Encrypt in-place is allowed. 117 * const u8 *in, Plaintext input 118 * unsigned long plaintext_len, Length of data in bytes for encryption. 119 * u8 *iv, Pre-counter block j0: 12 byte IV concatenated with 0x00000001. 120 * 16-byte aligned pointer. 121 * u8 *hash_subkey, the Hash sub key input. Data starts on a 16-byte boundary. 122 * const u8 *aad, Additional Authentication Data (AAD) 123 * unsigned long aad_len, Length of AAD in bytes. 124 * u8 *auth_tag, Authenticated Tag output. 125 * unsigned long auth_tag_len), Authenticated Tag Length in bytes. 126 * Valid values are 16 (most likely), 12 or 8. 127 */ 128 asmlinkage void aesni_gcm_enc(void *ctx, 129 struct gcm_context_data *gdata, u8 *out, 130 const u8 *in, unsigned long plaintext_len, u8 *iv, 131 u8 *hash_subkey, const u8 *aad, unsigned long aad_len, 132 u8 *auth_tag, unsigned long auth_tag_len); 133 134 /* asmlinkage void aesni_gcm_dec() 135 * void *ctx, AES Key schedule. Starts on a 16 byte boundary. 136 * struct gcm_context_data. May be uninitialized. 137 * u8 *out, Plaintext output. Decrypt in-place is allowed. 138 * const u8 *in, Ciphertext input 139 * unsigned long ciphertext_len, Length of data in bytes for decryption. 140 * u8 *iv, Pre-counter block j0: 12 byte IV concatenated with 0x00000001. 141 * 16-byte aligned pointer. 142 * u8 *hash_subkey, the Hash sub key input. Data starts on a 16-byte boundary. 143 * const u8 *aad, Additional Authentication Data (AAD) 144 * unsigned long aad_len, Length of AAD in bytes. With RFC4106 this is going 145 * to be 8 or 12 bytes 146 * u8 *auth_tag, Authenticated Tag output. 147 * unsigned long auth_tag_len) Authenticated Tag Length in bytes. 148 * Valid values are 16 (most likely), 12 or 8. 149 */ 150 asmlinkage void aesni_gcm_dec(void *ctx, 151 struct gcm_context_data *gdata, u8 *out, 152 const u8 *in, unsigned long ciphertext_len, u8 *iv, 153 u8 *hash_subkey, const u8 *aad, unsigned long aad_len, 154 u8 *auth_tag, unsigned long auth_tag_len); 155 156 /* Scatter / Gather routines, with args similar to above */ 157 asmlinkage void aesni_gcm_init(void *ctx, 158 struct gcm_context_data *gdata, 159 u8 *iv, 160 u8 *hash_subkey, const u8 *aad, 161 unsigned long aad_len); 162 asmlinkage void aesni_gcm_enc_update(void *ctx, 163 struct gcm_context_data *gdata, u8 *out, 164 const u8 *in, unsigned long plaintext_len); 165 asmlinkage void aesni_gcm_dec_update(void *ctx, 166 struct gcm_context_data *gdata, u8 *out, 167 const u8 *in, 168 unsigned long ciphertext_len); 169 asmlinkage void aesni_gcm_finalize(void *ctx, 170 struct gcm_context_data *gdata, 171 u8 *auth_tag, unsigned long auth_tag_len); 172 173 static const struct aesni_gcm_tfm_s { 174 void (*init)(void *ctx, struct gcm_context_data *gdata, u8 *iv, 175 u8 *hash_subkey, const u8 *aad, unsigned long aad_len); 176 void (*enc_update)(void *ctx, struct gcm_context_data *gdata, u8 *out, 177 const u8 *in, unsigned long plaintext_len); 178 void (*dec_update)(void *ctx, struct gcm_context_data *gdata, u8 *out, 179 const u8 *in, unsigned long ciphertext_len); 180 void (*finalize)(void *ctx, struct gcm_context_data *gdata, 181 u8 *auth_tag, unsigned long auth_tag_len); 182 } *aesni_gcm_tfm; 183 184 static const struct aesni_gcm_tfm_s aesni_gcm_tfm_sse = { 185 .init = &aesni_gcm_init, 186 .enc_update = &aesni_gcm_enc_update, 187 .dec_update = &aesni_gcm_dec_update, 188 .finalize = &aesni_gcm_finalize, 189 }; 190 191 #ifdef CONFIG_AS_AVX 192 asmlinkage void aes_ctr_enc_128_avx_by8(const u8 *in, u8 *iv, 193 void *keys, u8 *out, unsigned int num_bytes); 194 asmlinkage void aes_ctr_enc_192_avx_by8(const u8 *in, u8 *iv, 195 void *keys, u8 *out, unsigned int num_bytes); 196 asmlinkage void aes_ctr_enc_256_avx_by8(const u8 *in, u8 *iv, 197 void *keys, u8 *out, unsigned int num_bytes); 198 /* 199 * asmlinkage void aesni_gcm_init_avx_gen2() 200 * gcm_data *my_ctx_data, context data 201 * u8 *hash_subkey, the Hash sub key input. Data starts on a 16-byte boundary. 202 */ 203 asmlinkage void aesni_gcm_init_avx_gen2(void *my_ctx_data, 204 struct gcm_context_data *gdata, 205 u8 *iv, 206 u8 *hash_subkey, 207 const u8 *aad, 208 unsigned long aad_len); 209 210 asmlinkage void aesni_gcm_enc_update_avx_gen2(void *ctx, 211 struct gcm_context_data *gdata, u8 *out, 212 const u8 *in, unsigned long plaintext_len); 213 asmlinkage void aesni_gcm_dec_update_avx_gen2(void *ctx, 214 struct gcm_context_data *gdata, u8 *out, 215 const u8 *in, 216 unsigned long ciphertext_len); 217 asmlinkage void aesni_gcm_finalize_avx_gen2(void *ctx, 218 struct gcm_context_data *gdata, 219 u8 *auth_tag, unsigned long auth_tag_len); 220 221 asmlinkage void aesni_gcm_enc_avx_gen2(void *ctx, 222 struct gcm_context_data *gdata, u8 *out, 223 const u8 *in, unsigned long plaintext_len, u8 *iv, 224 const u8 *aad, unsigned long aad_len, 225 u8 *auth_tag, unsigned long auth_tag_len); 226 227 asmlinkage void aesni_gcm_dec_avx_gen2(void *ctx, 228 struct gcm_context_data *gdata, u8 *out, 229 const u8 *in, unsigned long ciphertext_len, u8 *iv, 230 const u8 *aad, unsigned long aad_len, 231 u8 *auth_tag, unsigned long auth_tag_len); 232 233 static const struct aesni_gcm_tfm_s aesni_gcm_tfm_avx_gen2 = { 234 .init = &aesni_gcm_init_avx_gen2, 235 .enc_update = &aesni_gcm_enc_update_avx_gen2, 236 .dec_update = &aesni_gcm_dec_update_avx_gen2, 237 .finalize = &aesni_gcm_finalize_avx_gen2, 238 }; 239 240 #endif 241 242 #ifdef CONFIG_AS_AVX2 243 /* 244 * asmlinkage void aesni_gcm_init_avx_gen4() 245 * gcm_data *my_ctx_data, context data 246 * u8 *hash_subkey, the Hash sub key input. Data starts on a 16-byte boundary. 247 */ 248 asmlinkage void aesni_gcm_init_avx_gen4(void *my_ctx_data, 249 struct gcm_context_data *gdata, 250 u8 *iv, 251 u8 *hash_subkey, 252 const u8 *aad, 253 unsigned long aad_len); 254 255 asmlinkage void aesni_gcm_enc_update_avx_gen4(void *ctx, 256 struct gcm_context_data *gdata, u8 *out, 257 const u8 *in, unsigned long plaintext_len); 258 asmlinkage void aesni_gcm_dec_update_avx_gen4(void *ctx, 259 struct gcm_context_data *gdata, u8 *out, 260 const u8 *in, 261 unsigned long ciphertext_len); 262 asmlinkage void aesni_gcm_finalize_avx_gen4(void *ctx, 263 struct gcm_context_data *gdata, 264 u8 *auth_tag, unsigned long auth_tag_len); 265 266 asmlinkage void aesni_gcm_enc_avx_gen4(void *ctx, 267 struct gcm_context_data *gdata, u8 *out, 268 const u8 *in, unsigned long plaintext_len, u8 *iv, 269 const u8 *aad, unsigned long aad_len, 270 u8 *auth_tag, unsigned long auth_tag_len); 271 272 asmlinkage void aesni_gcm_dec_avx_gen4(void *ctx, 273 struct gcm_context_data *gdata, u8 *out, 274 const u8 *in, unsigned long ciphertext_len, u8 *iv, 275 const u8 *aad, unsigned long aad_len, 276 u8 *auth_tag, unsigned long auth_tag_len); 277 278 static const struct aesni_gcm_tfm_s aesni_gcm_tfm_avx_gen4 = { 279 .init = &aesni_gcm_init_avx_gen4, 280 .enc_update = &aesni_gcm_enc_update_avx_gen4, 281 .dec_update = &aesni_gcm_dec_update_avx_gen4, 282 .finalize = &aesni_gcm_finalize_avx_gen4, 283 }; 284 285 #endif 286 287 static inline struct 288 aesni_rfc4106_gcm_ctx *aesni_rfc4106_gcm_ctx_get(struct crypto_aead *tfm) 289 { 290 unsigned long align = AESNI_ALIGN; 291 292 if (align <= crypto_tfm_ctx_alignment()) 293 align = 1; 294 return PTR_ALIGN(crypto_aead_ctx(tfm), align); 295 } 296 297 static inline struct 298 generic_gcmaes_ctx *generic_gcmaes_ctx_get(struct crypto_aead *tfm) 299 { 300 unsigned long align = AESNI_ALIGN; 301 302 if (align <= crypto_tfm_ctx_alignment()) 303 align = 1; 304 return PTR_ALIGN(crypto_aead_ctx(tfm), align); 305 } 306 #endif 307 308 static inline struct crypto_aes_ctx *aes_ctx(void *raw_ctx) 309 { 310 unsigned long addr = (unsigned long)raw_ctx; 311 unsigned long align = AESNI_ALIGN; 312 313 if (align <= crypto_tfm_ctx_alignment()) 314 align = 1; 315 return (struct crypto_aes_ctx *)ALIGN(addr, align); 316 } 317 318 static int aes_set_key_common(struct crypto_tfm *tfm, void *raw_ctx, 319 const u8 *in_key, unsigned int key_len) 320 { 321 struct crypto_aes_ctx *ctx = aes_ctx(raw_ctx); 322 u32 *flags = &tfm->crt_flags; 323 int err; 324 325 if (key_len != AES_KEYSIZE_128 && key_len != AES_KEYSIZE_192 && 326 key_len != AES_KEYSIZE_256) { 327 *flags |= CRYPTO_TFM_RES_BAD_KEY_LEN; 328 return -EINVAL; 329 } 330 331 if (!crypto_simd_usable()) 332 err = crypto_aes_expand_key(ctx, in_key, key_len); 333 else { 334 kernel_fpu_begin(); 335 err = aesni_set_key(ctx, in_key, key_len); 336 kernel_fpu_end(); 337 } 338 339 return err; 340 } 341 342 static int aes_set_key(struct crypto_tfm *tfm, const u8 *in_key, 343 unsigned int key_len) 344 { 345 return aes_set_key_common(tfm, crypto_tfm_ctx(tfm), in_key, key_len); 346 } 347 348 static void aes_encrypt(struct crypto_tfm *tfm, u8 *dst, const u8 *src) 349 { 350 struct crypto_aes_ctx *ctx = aes_ctx(crypto_tfm_ctx(tfm)); 351 352 if (!crypto_simd_usable()) 353 crypto_aes_encrypt_x86(ctx, dst, src); 354 else { 355 kernel_fpu_begin(); 356 aesni_enc(ctx, dst, src); 357 kernel_fpu_end(); 358 } 359 } 360 361 static void aes_decrypt(struct crypto_tfm *tfm, u8 *dst, const u8 *src) 362 { 363 struct crypto_aes_ctx *ctx = aes_ctx(crypto_tfm_ctx(tfm)); 364 365 if (!crypto_simd_usable()) 366 crypto_aes_decrypt_x86(ctx, dst, src); 367 else { 368 kernel_fpu_begin(); 369 aesni_dec(ctx, dst, src); 370 kernel_fpu_end(); 371 } 372 } 373 374 static int aesni_skcipher_setkey(struct crypto_skcipher *tfm, const u8 *key, 375 unsigned int len) 376 { 377 return aes_set_key_common(crypto_skcipher_tfm(tfm), 378 crypto_skcipher_ctx(tfm), key, len); 379 } 380 381 static int ecb_encrypt(struct skcipher_request *req) 382 { 383 struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req); 384 struct crypto_aes_ctx *ctx = aes_ctx(crypto_skcipher_ctx(tfm)); 385 struct skcipher_walk walk; 386 unsigned int nbytes; 387 int err; 388 389 err = skcipher_walk_virt(&walk, req, true); 390 391 kernel_fpu_begin(); 392 while ((nbytes = walk.nbytes)) { 393 aesni_ecb_enc(ctx, walk.dst.virt.addr, walk.src.virt.addr, 394 nbytes & AES_BLOCK_MASK); 395 nbytes &= AES_BLOCK_SIZE - 1; 396 err = skcipher_walk_done(&walk, nbytes); 397 } 398 kernel_fpu_end(); 399 400 return err; 401 } 402 403 static int ecb_decrypt(struct skcipher_request *req) 404 { 405 struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req); 406 struct crypto_aes_ctx *ctx = aes_ctx(crypto_skcipher_ctx(tfm)); 407 struct skcipher_walk walk; 408 unsigned int nbytes; 409 int err; 410 411 err = skcipher_walk_virt(&walk, req, true); 412 413 kernel_fpu_begin(); 414 while ((nbytes = walk.nbytes)) { 415 aesni_ecb_dec(ctx, walk.dst.virt.addr, walk.src.virt.addr, 416 nbytes & AES_BLOCK_MASK); 417 nbytes &= AES_BLOCK_SIZE - 1; 418 err = skcipher_walk_done(&walk, nbytes); 419 } 420 kernel_fpu_end(); 421 422 return err; 423 } 424 425 static int cbc_encrypt(struct skcipher_request *req) 426 { 427 struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req); 428 struct crypto_aes_ctx *ctx = aes_ctx(crypto_skcipher_ctx(tfm)); 429 struct skcipher_walk walk; 430 unsigned int nbytes; 431 int err; 432 433 err = skcipher_walk_virt(&walk, req, true); 434 435 kernel_fpu_begin(); 436 while ((nbytes = walk.nbytes)) { 437 aesni_cbc_enc(ctx, walk.dst.virt.addr, walk.src.virt.addr, 438 nbytes & AES_BLOCK_MASK, walk.iv); 439 nbytes &= AES_BLOCK_SIZE - 1; 440 err = skcipher_walk_done(&walk, nbytes); 441 } 442 kernel_fpu_end(); 443 444 return err; 445 } 446 447 static int cbc_decrypt(struct skcipher_request *req) 448 { 449 struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req); 450 struct crypto_aes_ctx *ctx = aes_ctx(crypto_skcipher_ctx(tfm)); 451 struct skcipher_walk walk; 452 unsigned int nbytes; 453 int err; 454 455 err = skcipher_walk_virt(&walk, req, true); 456 457 kernel_fpu_begin(); 458 while ((nbytes = walk.nbytes)) { 459 aesni_cbc_dec(ctx, walk.dst.virt.addr, walk.src.virt.addr, 460 nbytes & AES_BLOCK_MASK, walk.iv); 461 nbytes &= AES_BLOCK_SIZE - 1; 462 err = skcipher_walk_done(&walk, nbytes); 463 } 464 kernel_fpu_end(); 465 466 return err; 467 } 468 469 #ifdef CONFIG_X86_64 470 static void ctr_crypt_final(struct crypto_aes_ctx *ctx, 471 struct skcipher_walk *walk) 472 { 473 u8 *ctrblk = walk->iv; 474 u8 keystream[AES_BLOCK_SIZE]; 475 u8 *src = walk->src.virt.addr; 476 u8 *dst = walk->dst.virt.addr; 477 unsigned int nbytes = walk->nbytes; 478 479 aesni_enc(ctx, keystream, ctrblk); 480 crypto_xor_cpy(dst, keystream, src, nbytes); 481 482 crypto_inc(ctrblk, AES_BLOCK_SIZE); 483 } 484 485 #ifdef CONFIG_AS_AVX 486 static void aesni_ctr_enc_avx_tfm(struct crypto_aes_ctx *ctx, u8 *out, 487 const u8 *in, unsigned int len, u8 *iv) 488 { 489 /* 490 * based on key length, override with the by8 version 491 * of ctr mode encryption/decryption for improved performance 492 * aes_set_key_common() ensures that key length is one of 493 * {128,192,256} 494 */ 495 if (ctx->key_length == AES_KEYSIZE_128) 496 aes_ctr_enc_128_avx_by8(in, iv, (void *)ctx, out, len); 497 else if (ctx->key_length == AES_KEYSIZE_192) 498 aes_ctr_enc_192_avx_by8(in, iv, (void *)ctx, out, len); 499 else 500 aes_ctr_enc_256_avx_by8(in, iv, (void *)ctx, out, len); 501 } 502 #endif 503 504 static int ctr_crypt(struct skcipher_request *req) 505 { 506 struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req); 507 struct crypto_aes_ctx *ctx = aes_ctx(crypto_skcipher_ctx(tfm)); 508 struct skcipher_walk walk; 509 unsigned int nbytes; 510 int err; 511 512 err = skcipher_walk_virt(&walk, req, true); 513 514 kernel_fpu_begin(); 515 while ((nbytes = walk.nbytes) >= AES_BLOCK_SIZE) { 516 aesni_ctr_enc_tfm(ctx, walk.dst.virt.addr, walk.src.virt.addr, 517 nbytes & AES_BLOCK_MASK, walk.iv); 518 nbytes &= AES_BLOCK_SIZE - 1; 519 err = skcipher_walk_done(&walk, nbytes); 520 } 521 if (walk.nbytes) { 522 ctr_crypt_final(ctx, &walk); 523 err = skcipher_walk_done(&walk, 0); 524 } 525 kernel_fpu_end(); 526 527 return err; 528 } 529 530 static int xts_aesni_setkey(struct crypto_skcipher *tfm, const u8 *key, 531 unsigned int keylen) 532 { 533 struct aesni_xts_ctx *ctx = crypto_skcipher_ctx(tfm); 534 int err; 535 536 err = xts_verify_key(tfm, key, keylen); 537 if (err) 538 return err; 539 540 keylen /= 2; 541 542 /* first half of xts-key is for crypt */ 543 err = aes_set_key_common(crypto_skcipher_tfm(tfm), ctx->raw_crypt_ctx, 544 key, keylen); 545 if (err) 546 return err; 547 548 /* second half of xts-key is for tweak */ 549 return aes_set_key_common(crypto_skcipher_tfm(tfm), ctx->raw_tweak_ctx, 550 key + keylen, keylen); 551 } 552 553 554 static void aesni_xts_tweak(void *ctx, u8 *out, const u8 *in) 555 { 556 aesni_enc(ctx, out, in); 557 } 558 559 static void aesni_xts_enc(void *ctx, u128 *dst, const u128 *src, le128 *iv) 560 { 561 glue_xts_crypt_128bit_one(ctx, dst, src, iv, GLUE_FUNC_CAST(aesni_enc)); 562 } 563 564 static void aesni_xts_dec(void *ctx, u128 *dst, const u128 *src, le128 *iv) 565 { 566 glue_xts_crypt_128bit_one(ctx, dst, src, iv, GLUE_FUNC_CAST(aesni_dec)); 567 } 568 569 static void aesni_xts_enc8(void *ctx, u128 *dst, const u128 *src, le128 *iv) 570 { 571 aesni_xts_crypt8(ctx, (u8 *)dst, (const u8 *)src, true, (u8 *)iv); 572 } 573 574 static void aesni_xts_dec8(void *ctx, u128 *dst, const u128 *src, le128 *iv) 575 { 576 aesni_xts_crypt8(ctx, (u8 *)dst, (const u8 *)src, false, (u8 *)iv); 577 } 578 579 static const struct common_glue_ctx aesni_enc_xts = { 580 .num_funcs = 2, 581 .fpu_blocks_limit = 1, 582 583 .funcs = { { 584 .num_blocks = 8, 585 .fn_u = { .xts = GLUE_XTS_FUNC_CAST(aesni_xts_enc8) } 586 }, { 587 .num_blocks = 1, 588 .fn_u = { .xts = GLUE_XTS_FUNC_CAST(aesni_xts_enc) } 589 } } 590 }; 591 592 static const struct common_glue_ctx aesni_dec_xts = { 593 .num_funcs = 2, 594 .fpu_blocks_limit = 1, 595 596 .funcs = { { 597 .num_blocks = 8, 598 .fn_u = { .xts = GLUE_XTS_FUNC_CAST(aesni_xts_dec8) } 599 }, { 600 .num_blocks = 1, 601 .fn_u = { .xts = GLUE_XTS_FUNC_CAST(aesni_xts_dec) } 602 } } 603 }; 604 605 static int xts_encrypt(struct skcipher_request *req) 606 { 607 struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req); 608 struct aesni_xts_ctx *ctx = crypto_skcipher_ctx(tfm); 609 610 return glue_xts_req_128bit(&aesni_enc_xts, req, 611 XTS_TWEAK_CAST(aesni_xts_tweak), 612 aes_ctx(ctx->raw_tweak_ctx), 613 aes_ctx(ctx->raw_crypt_ctx)); 614 } 615 616 static int xts_decrypt(struct skcipher_request *req) 617 { 618 struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req); 619 struct aesni_xts_ctx *ctx = crypto_skcipher_ctx(tfm); 620 621 return glue_xts_req_128bit(&aesni_dec_xts, req, 622 XTS_TWEAK_CAST(aesni_xts_tweak), 623 aes_ctx(ctx->raw_tweak_ctx), 624 aes_ctx(ctx->raw_crypt_ctx)); 625 } 626 627 static int 628 rfc4106_set_hash_subkey(u8 *hash_subkey, const u8 *key, unsigned int key_len) 629 { 630 struct crypto_cipher *tfm; 631 int ret; 632 633 tfm = crypto_alloc_cipher("aes", 0, 0); 634 if (IS_ERR(tfm)) 635 return PTR_ERR(tfm); 636 637 ret = crypto_cipher_setkey(tfm, key, key_len); 638 if (ret) 639 goto out_free_cipher; 640 641 /* Clear the data in the hash sub key container to zero.*/ 642 /* We want to cipher all zeros to create the hash sub key. */ 643 memset(hash_subkey, 0, RFC4106_HASH_SUBKEY_SIZE); 644 645 crypto_cipher_encrypt_one(tfm, hash_subkey, hash_subkey); 646 647 out_free_cipher: 648 crypto_free_cipher(tfm); 649 return ret; 650 } 651 652 static int common_rfc4106_set_key(struct crypto_aead *aead, const u8 *key, 653 unsigned int key_len) 654 { 655 struct aesni_rfc4106_gcm_ctx *ctx = aesni_rfc4106_gcm_ctx_get(aead); 656 657 if (key_len < 4) { 658 crypto_aead_set_flags(aead, CRYPTO_TFM_RES_BAD_KEY_LEN); 659 return -EINVAL; 660 } 661 /*Account for 4 byte nonce at the end.*/ 662 key_len -= 4; 663 664 memcpy(ctx->nonce, key + key_len, sizeof(ctx->nonce)); 665 666 return aes_set_key_common(crypto_aead_tfm(aead), 667 &ctx->aes_key_expanded, key, key_len) ?: 668 rfc4106_set_hash_subkey(ctx->hash_subkey, key, key_len); 669 } 670 671 /* This is the Integrity Check Value (aka the authentication tag) length and can 672 * be 8, 12 or 16 bytes long. */ 673 static int common_rfc4106_set_authsize(struct crypto_aead *aead, 674 unsigned int authsize) 675 { 676 switch (authsize) { 677 case 8: 678 case 12: 679 case 16: 680 break; 681 default: 682 return -EINVAL; 683 } 684 685 return 0; 686 } 687 688 static int generic_gcmaes_set_authsize(struct crypto_aead *tfm, 689 unsigned int authsize) 690 { 691 switch (authsize) { 692 case 4: 693 case 8: 694 case 12: 695 case 13: 696 case 14: 697 case 15: 698 case 16: 699 break; 700 default: 701 return -EINVAL; 702 } 703 704 return 0; 705 } 706 707 static int gcmaes_crypt_by_sg(bool enc, struct aead_request *req, 708 unsigned int assoclen, u8 *hash_subkey, 709 u8 *iv, void *aes_ctx) 710 { 711 struct crypto_aead *tfm = crypto_aead_reqtfm(req); 712 unsigned long auth_tag_len = crypto_aead_authsize(tfm); 713 const struct aesni_gcm_tfm_s *gcm_tfm = aesni_gcm_tfm; 714 struct gcm_context_data data AESNI_ALIGN_ATTR; 715 struct scatter_walk dst_sg_walk = {}; 716 unsigned long left = req->cryptlen; 717 unsigned long len, srclen, dstlen; 718 struct scatter_walk assoc_sg_walk; 719 struct scatter_walk src_sg_walk; 720 struct scatterlist src_start[2]; 721 struct scatterlist dst_start[2]; 722 struct scatterlist *src_sg; 723 struct scatterlist *dst_sg; 724 u8 *src, *dst, *assoc; 725 u8 *assocmem = NULL; 726 u8 authTag[16]; 727 728 if (!enc) 729 left -= auth_tag_len; 730 731 #ifdef CONFIG_AS_AVX2 732 if (left < AVX_GEN4_OPTSIZE && gcm_tfm == &aesni_gcm_tfm_avx_gen4) 733 gcm_tfm = &aesni_gcm_tfm_avx_gen2; 734 #endif 735 #ifdef CONFIG_AS_AVX 736 if (left < AVX_GEN2_OPTSIZE && gcm_tfm == &aesni_gcm_tfm_avx_gen2) 737 gcm_tfm = &aesni_gcm_tfm_sse; 738 #endif 739 740 /* Linearize assoc, if not already linear */ 741 if (req->src->length >= assoclen && req->src->length && 742 (!PageHighMem(sg_page(req->src)) || 743 req->src->offset + req->src->length <= PAGE_SIZE)) { 744 scatterwalk_start(&assoc_sg_walk, req->src); 745 assoc = scatterwalk_map(&assoc_sg_walk); 746 } else { 747 /* assoc can be any length, so must be on heap */ 748 assocmem = kmalloc(assoclen, GFP_ATOMIC); 749 if (unlikely(!assocmem)) 750 return -ENOMEM; 751 assoc = assocmem; 752 753 scatterwalk_map_and_copy(assoc, req->src, 0, assoclen, 0); 754 } 755 756 if (left) { 757 src_sg = scatterwalk_ffwd(src_start, req->src, req->assoclen); 758 scatterwalk_start(&src_sg_walk, src_sg); 759 if (req->src != req->dst) { 760 dst_sg = scatterwalk_ffwd(dst_start, req->dst, 761 req->assoclen); 762 scatterwalk_start(&dst_sg_walk, dst_sg); 763 } 764 } 765 766 kernel_fpu_begin(); 767 gcm_tfm->init(aes_ctx, &data, iv, 768 hash_subkey, assoc, assoclen); 769 if (req->src != req->dst) { 770 while (left) { 771 src = scatterwalk_map(&src_sg_walk); 772 dst = scatterwalk_map(&dst_sg_walk); 773 srclen = scatterwalk_clamp(&src_sg_walk, left); 774 dstlen = scatterwalk_clamp(&dst_sg_walk, left); 775 len = min(srclen, dstlen); 776 if (len) { 777 if (enc) 778 gcm_tfm->enc_update(aes_ctx, &data, 779 dst, src, len); 780 else 781 gcm_tfm->dec_update(aes_ctx, &data, 782 dst, src, len); 783 } 784 left -= len; 785 786 scatterwalk_unmap(src); 787 scatterwalk_unmap(dst); 788 scatterwalk_advance(&src_sg_walk, len); 789 scatterwalk_advance(&dst_sg_walk, len); 790 scatterwalk_done(&src_sg_walk, 0, left); 791 scatterwalk_done(&dst_sg_walk, 1, left); 792 } 793 } else { 794 while (left) { 795 dst = src = scatterwalk_map(&src_sg_walk); 796 len = scatterwalk_clamp(&src_sg_walk, left); 797 if (len) { 798 if (enc) 799 gcm_tfm->enc_update(aes_ctx, &data, 800 src, src, len); 801 else 802 gcm_tfm->dec_update(aes_ctx, &data, 803 src, src, len); 804 } 805 left -= len; 806 scatterwalk_unmap(src); 807 scatterwalk_advance(&src_sg_walk, len); 808 scatterwalk_done(&src_sg_walk, 1, left); 809 } 810 } 811 gcm_tfm->finalize(aes_ctx, &data, authTag, auth_tag_len); 812 kernel_fpu_end(); 813 814 if (!assocmem) 815 scatterwalk_unmap(assoc); 816 else 817 kfree(assocmem); 818 819 if (!enc) { 820 u8 authTagMsg[16]; 821 822 /* Copy out original authTag */ 823 scatterwalk_map_and_copy(authTagMsg, req->src, 824 req->assoclen + req->cryptlen - 825 auth_tag_len, 826 auth_tag_len, 0); 827 828 /* Compare generated tag with passed in tag. */ 829 return crypto_memneq(authTagMsg, authTag, auth_tag_len) ? 830 -EBADMSG : 0; 831 } 832 833 /* Copy in the authTag */ 834 scatterwalk_map_and_copy(authTag, req->dst, 835 req->assoclen + req->cryptlen, 836 auth_tag_len, 1); 837 838 return 0; 839 } 840 841 static int gcmaes_encrypt(struct aead_request *req, unsigned int assoclen, 842 u8 *hash_subkey, u8 *iv, void *aes_ctx) 843 { 844 return gcmaes_crypt_by_sg(true, req, assoclen, hash_subkey, iv, 845 aes_ctx); 846 } 847 848 static int gcmaes_decrypt(struct aead_request *req, unsigned int assoclen, 849 u8 *hash_subkey, u8 *iv, void *aes_ctx) 850 { 851 return gcmaes_crypt_by_sg(false, req, assoclen, hash_subkey, iv, 852 aes_ctx); 853 } 854 855 static int helper_rfc4106_encrypt(struct aead_request *req) 856 { 857 struct crypto_aead *tfm = crypto_aead_reqtfm(req); 858 struct aesni_rfc4106_gcm_ctx *ctx = aesni_rfc4106_gcm_ctx_get(tfm); 859 void *aes_ctx = &(ctx->aes_key_expanded); 860 u8 iv[16] __attribute__ ((__aligned__(AESNI_ALIGN))); 861 unsigned int i; 862 __be32 counter = cpu_to_be32(1); 863 864 /* Assuming we are supporting rfc4106 64-bit extended */ 865 /* sequence numbers We need to have the AAD length equal */ 866 /* to 16 or 20 bytes */ 867 if (unlikely(req->assoclen != 16 && req->assoclen != 20)) 868 return -EINVAL; 869 870 /* IV below built */ 871 for (i = 0; i < 4; i++) 872 *(iv+i) = ctx->nonce[i]; 873 for (i = 0; i < 8; i++) 874 *(iv+4+i) = req->iv[i]; 875 *((__be32 *)(iv+12)) = counter; 876 877 return gcmaes_encrypt(req, req->assoclen - 8, ctx->hash_subkey, iv, 878 aes_ctx); 879 } 880 881 static int helper_rfc4106_decrypt(struct aead_request *req) 882 { 883 __be32 counter = cpu_to_be32(1); 884 struct crypto_aead *tfm = crypto_aead_reqtfm(req); 885 struct aesni_rfc4106_gcm_ctx *ctx = aesni_rfc4106_gcm_ctx_get(tfm); 886 void *aes_ctx = &(ctx->aes_key_expanded); 887 u8 iv[16] __attribute__ ((__aligned__(AESNI_ALIGN))); 888 unsigned int i; 889 890 if (unlikely(req->assoclen != 16 && req->assoclen != 20)) 891 return -EINVAL; 892 893 /* Assuming we are supporting rfc4106 64-bit extended */ 894 /* sequence numbers We need to have the AAD length */ 895 /* equal to 16 or 20 bytes */ 896 897 /* IV below built */ 898 for (i = 0; i < 4; i++) 899 *(iv+i) = ctx->nonce[i]; 900 for (i = 0; i < 8; i++) 901 *(iv+4+i) = req->iv[i]; 902 *((__be32 *)(iv+12)) = counter; 903 904 return gcmaes_decrypt(req, req->assoclen - 8, ctx->hash_subkey, iv, 905 aes_ctx); 906 } 907 #endif 908 909 static struct crypto_alg aesni_cipher_alg = { 910 .cra_name = "aes", 911 .cra_driver_name = "aes-aesni", 912 .cra_priority = 300, 913 .cra_flags = CRYPTO_ALG_TYPE_CIPHER, 914 .cra_blocksize = AES_BLOCK_SIZE, 915 .cra_ctxsize = CRYPTO_AES_CTX_SIZE, 916 .cra_module = THIS_MODULE, 917 .cra_u = { 918 .cipher = { 919 .cia_min_keysize = AES_MIN_KEY_SIZE, 920 .cia_max_keysize = AES_MAX_KEY_SIZE, 921 .cia_setkey = aes_set_key, 922 .cia_encrypt = aes_encrypt, 923 .cia_decrypt = aes_decrypt 924 } 925 } 926 }; 927 928 static struct skcipher_alg aesni_skciphers[] = { 929 { 930 .base = { 931 .cra_name = "__ecb(aes)", 932 .cra_driver_name = "__ecb-aes-aesni", 933 .cra_priority = 400, 934 .cra_flags = CRYPTO_ALG_INTERNAL, 935 .cra_blocksize = AES_BLOCK_SIZE, 936 .cra_ctxsize = CRYPTO_AES_CTX_SIZE, 937 .cra_module = THIS_MODULE, 938 }, 939 .min_keysize = AES_MIN_KEY_SIZE, 940 .max_keysize = AES_MAX_KEY_SIZE, 941 .setkey = aesni_skcipher_setkey, 942 .encrypt = ecb_encrypt, 943 .decrypt = ecb_decrypt, 944 }, { 945 .base = { 946 .cra_name = "__cbc(aes)", 947 .cra_driver_name = "__cbc-aes-aesni", 948 .cra_priority = 400, 949 .cra_flags = CRYPTO_ALG_INTERNAL, 950 .cra_blocksize = AES_BLOCK_SIZE, 951 .cra_ctxsize = CRYPTO_AES_CTX_SIZE, 952 .cra_module = THIS_MODULE, 953 }, 954 .min_keysize = AES_MIN_KEY_SIZE, 955 .max_keysize = AES_MAX_KEY_SIZE, 956 .ivsize = AES_BLOCK_SIZE, 957 .setkey = aesni_skcipher_setkey, 958 .encrypt = cbc_encrypt, 959 .decrypt = cbc_decrypt, 960 #ifdef CONFIG_X86_64 961 }, { 962 .base = { 963 .cra_name = "__ctr(aes)", 964 .cra_driver_name = "__ctr-aes-aesni", 965 .cra_priority = 400, 966 .cra_flags = CRYPTO_ALG_INTERNAL, 967 .cra_blocksize = 1, 968 .cra_ctxsize = CRYPTO_AES_CTX_SIZE, 969 .cra_module = THIS_MODULE, 970 }, 971 .min_keysize = AES_MIN_KEY_SIZE, 972 .max_keysize = AES_MAX_KEY_SIZE, 973 .ivsize = AES_BLOCK_SIZE, 974 .chunksize = AES_BLOCK_SIZE, 975 .setkey = aesni_skcipher_setkey, 976 .encrypt = ctr_crypt, 977 .decrypt = ctr_crypt, 978 }, { 979 .base = { 980 .cra_name = "__xts(aes)", 981 .cra_driver_name = "__xts-aes-aesni", 982 .cra_priority = 401, 983 .cra_flags = CRYPTO_ALG_INTERNAL, 984 .cra_blocksize = AES_BLOCK_SIZE, 985 .cra_ctxsize = XTS_AES_CTX_SIZE, 986 .cra_module = THIS_MODULE, 987 }, 988 .min_keysize = 2 * AES_MIN_KEY_SIZE, 989 .max_keysize = 2 * AES_MAX_KEY_SIZE, 990 .ivsize = AES_BLOCK_SIZE, 991 .setkey = xts_aesni_setkey, 992 .encrypt = xts_encrypt, 993 .decrypt = xts_decrypt, 994 #endif 995 } 996 }; 997 998 static 999 struct simd_skcipher_alg *aesni_simd_skciphers[ARRAY_SIZE(aesni_skciphers)]; 1000 1001 #ifdef CONFIG_X86_64 1002 static int generic_gcmaes_set_key(struct crypto_aead *aead, const u8 *key, 1003 unsigned int key_len) 1004 { 1005 struct generic_gcmaes_ctx *ctx = generic_gcmaes_ctx_get(aead); 1006 1007 return aes_set_key_common(crypto_aead_tfm(aead), 1008 &ctx->aes_key_expanded, key, key_len) ?: 1009 rfc4106_set_hash_subkey(ctx->hash_subkey, key, key_len); 1010 } 1011 1012 static int generic_gcmaes_encrypt(struct aead_request *req) 1013 { 1014 struct crypto_aead *tfm = crypto_aead_reqtfm(req); 1015 struct generic_gcmaes_ctx *ctx = generic_gcmaes_ctx_get(tfm); 1016 void *aes_ctx = &(ctx->aes_key_expanded); 1017 u8 iv[16] __attribute__ ((__aligned__(AESNI_ALIGN))); 1018 __be32 counter = cpu_to_be32(1); 1019 1020 memcpy(iv, req->iv, 12); 1021 *((__be32 *)(iv+12)) = counter; 1022 1023 return gcmaes_encrypt(req, req->assoclen, ctx->hash_subkey, iv, 1024 aes_ctx); 1025 } 1026 1027 static int generic_gcmaes_decrypt(struct aead_request *req) 1028 { 1029 __be32 counter = cpu_to_be32(1); 1030 struct crypto_aead *tfm = crypto_aead_reqtfm(req); 1031 struct generic_gcmaes_ctx *ctx = generic_gcmaes_ctx_get(tfm); 1032 void *aes_ctx = &(ctx->aes_key_expanded); 1033 u8 iv[16] __attribute__ ((__aligned__(AESNI_ALIGN))); 1034 1035 memcpy(iv, req->iv, 12); 1036 *((__be32 *)(iv+12)) = counter; 1037 1038 return gcmaes_decrypt(req, req->assoclen, ctx->hash_subkey, iv, 1039 aes_ctx); 1040 } 1041 1042 static struct aead_alg aesni_aeads[] = { { 1043 .setkey = common_rfc4106_set_key, 1044 .setauthsize = common_rfc4106_set_authsize, 1045 .encrypt = helper_rfc4106_encrypt, 1046 .decrypt = helper_rfc4106_decrypt, 1047 .ivsize = GCM_RFC4106_IV_SIZE, 1048 .maxauthsize = 16, 1049 .base = { 1050 .cra_name = "__rfc4106(gcm(aes))", 1051 .cra_driver_name = "__rfc4106-gcm-aesni", 1052 .cra_priority = 400, 1053 .cra_flags = CRYPTO_ALG_INTERNAL, 1054 .cra_blocksize = 1, 1055 .cra_ctxsize = sizeof(struct aesni_rfc4106_gcm_ctx), 1056 .cra_alignmask = AESNI_ALIGN - 1, 1057 .cra_module = THIS_MODULE, 1058 }, 1059 }, { 1060 .setkey = generic_gcmaes_set_key, 1061 .setauthsize = generic_gcmaes_set_authsize, 1062 .encrypt = generic_gcmaes_encrypt, 1063 .decrypt = generic_gcmaes_decrypt, 1064 .ivsize = GCM_AES_IV_SIZE, 1065 .maxauthsize = 16, 1066 .base = { 1067 .cra_name = "__gcm(aes)", 1068 .cra_driver_name = "__generic-gcm-aesni", 1069 .cra_priority = 400, 1070 .cra_flags = CRYPTO_ALG_INTERNAL, 1071 .cra_blocksize = 1, 1072 .cra_ctxsize = sizeof(struct generic_gcmaes_ctx), 1073 .cra_alignmask = AESNI_ALIGN - 1, 1074 .cra_module = THIS_MODULE, 1075 }, 1076 } }; 1077 #else 1078 static struct aead_alg aesni_aeads[0]; 1079 #endif 1080 1081 static struct simd_aead_alg *aesni_simd_aeads[ARRAY_SIZE(aesni_aeads)]; 1082 1083 static const struct x86_cpu_id aesni_cpu_id[] = { 1084 X86_FEATURE_MATCH(X86_FEATURE_AES), 1085 {} 1086 }; 1087 MODULE_DEVICE_TABLE(x86cpu, aesni_cpu_id); 1088 1089 static int __init aesni_init(void) 1090 { 1091 int err; 1092 1093 if (!x86_match_cpu(aesni_cpu_id)) 1094 return -ENODEV; 1095 #ifdef CONFIG_X86_64 1096 #ifdef CONFIG_AS_AVX2 1097 if (boot_cpu_has(X86_FEATURE_AVX2)) { 1098 pr_info("AVX2 version of gcm_enc/dec engaged.\n"); 1099 aesni_gcm_tfm = &aesni_gcm_tfm_avx_gen4; 1100 } else 1101 #endif 1102 #ifdef CONFIG_AS_AVX 1103 if (boot_cpu_has(X86_FEATURE_AVX)) { 1104 pr_info("AVX version of gcm_enc/dec engaged.\n"); 1105 aesni_gcm_tfm = &aesni_gcm_tfm_avx_gen2; 1106 } else 1107 #endif 1108 { 1109 pr_info("SSE version of gcm_enc/dec engaged.\n"); 1110 aesni_gcm_tfm = &aesni_gcm_tfm_sse; 1111 } 1112 aesni_ctr_enc_tfm = aesni_ctr_enc; 1113 #ifdef CONFIG_AS_AVX 1114 if (boot_cpu_has(X86_FEATURE_AVX)) { 1115 /* optimize performance of ctr mode encryption transform */ 1116 aesni_ctr_enc_tfm = aesni_ctr_enc_avx_tfm; 1117 pr_info("AES CTR mode by8 optimization enabled\n"); 1118 } 1119 #endif 1120 #endif 1121 1122 err = crypto_register_alg(&aesni_cipher_alg); 1123 if (err) 1124 return err; 1125 1126 err = simd_register_skciphers_compat(aesni_skciphers, 1127 ARRAY_SIZE(aesni_skciphers), 1128 aesni_simd_skciphers); 1129 if (err) 1130 goto unregister_cipher; 1131 1132 err = simd_register_aeads_compat(aesni_aeads, ARRAY_SIZE(aesni_aeads), 1133 aesni_simd_aeads); 1134 if (err) 1135 goto unregister_skciphers; 1136 1137 return 0; 1138 1139 unregister_skciphers: 1140 simd_unregister_skciphers(aesni_skciphers, ARRAY_SIZE(aesni_skciphers), 1141 aesni_simd_skciphers); 1142 unregister_cipher: 1143 crypto_unregister_alg(&aesni_cipher_alg); 1144 return err; 1145 } 1146 1147 static void __exit aesni_exit(void) 1148 { 1149 simd_unregister_aeads(aesni_aeads, ARRAY_SIZE(aesni_aeads), 1150 aesni_simd_aeads); 1151 simd_unregister_skciphers(aesni_skciphers, ARRAY_SIZE(aesni_skciphers), 1152 aesni_simd_skciphers); 1153 crypto_unregister_alg(&aesni_cipher_alg); 1154 } 1155 1156 late_initcall(aesni_init); 1157 module_exit(aesni_exit); 1158 1159 MODULE_DESCRIPTION("Rijndael (AES) Cipher Algorithm, Intel AES-NI instructions optimized"); 1160 MODULE_LICENSE("GPL"); 1161 MODULE_ALIAS_CRYPTO("aes"); 1162