1 // SPDX-License-Identifier: GPL-2.0-or-later
2 /*
3  * Support for Intel AES-NI instructions. This file contains glue
4  * code, the real AES implementation is in intel-aes_asm.S.
5  *
6  * Copyright (C) 2008, Intel Corp.
7  *    Author: Huang Ying <ying.huang@intel.com>
8  *
9  * Added RFC4106 AES-GCM support for 128-bit keys under the AEAD
10  * interface for 64-bit kernels.
11  *    Authors: Adrian Hoban <adrian.hoban@intel.com>
12  *             Gabriele Paoloni <gabriele.paoloni@intel.com>
13  *             Tadeusz Struk (tadeusz.struk@intel.com)
14  *             Aidan O'Mahony (aidan.o.mahony@intel.com)
15  *    Copyright (c) 2010, Intel Corporation.
16  */
17 
18 #include <linux/hardirq.h>
19 #include <linux/types.h>
20 #include <linux/module.h>
21 #include <linux/err.h>
22 #include <crypto/algapi.h>
23 #include <crypto/aes.h>
24 #include <crypto/ctr.h>
25 #include <crypto/b128ops.h>
26 #include <crypto/gcm.h>
27 #include <crypto/xts.h>
28 #include <asm/cpu_device_id.h>
29 #include <asm/simd.h>
30 #include <crypto/scatterwalk.h>
31 #include <crypto/internal/aead.h>
32 #include <crypto/internal/simd.h>
33 #include <crypto/internal/skcipher.h>
34 #include <linux/workqueue.h>
35 #include <linux/spinlock.h>
36 #ifdef CONFIG_X86_64
37 #include <asm/crypto/glue_helper.h>
38 #endif
39 
40 
41 #define AESNI_ALIGN	16
42 #define AESNI_ALIGN_ATTR __attribute__ ((__aligned__(AESNI_ALIGN)))
43 #define AES_BLOCK_MASK	(~(AES_BLOCK_SIZE - 1))
44 #define RFC4106_HASH_SUBKEY_SIZE 16
45 #define AESNI_ALIGN_EXTRA ((AESNI_ALIGN - 1) & ~(CRYPTO_MINALIGN - 1))
46 #define CRYPTO_AES_CTX_SIZE (sizeof(struct crypto_aes_ctx) + AESNI_ALIGN_EXTRA)
47 #define XTS_AES_CTX_SIZE (sizeof(struct aesni_xts_ctx) + AESNI_ALIGN_EXTRA)
48 
49 /* This data is stored at the end of the crypto_tfm struct.
50  * It's a type of per "session" data storage location.
51  * This needs to be 16 byte aligned.
52  */
53 struct aesni_rfc4106_gcm_ctx {
54 	u8 hash_subkey[16] AESNI_ALIGN_ATTR;
55 	struct crypto_aes_ctx aes_key_expanded AESNI_ALIGN_ATTR;
56 	u8 nonce[4];
57 };
58 
59 struct generic_gcmaes_ctx {
60 	u8 hash_subkey[16] AESNI_ALIGN_ATTR;
61 	struct crypto_aes_ctx aes_key_expanded AESNI_ALIGN_ATTR;
62 };
63 
64 struct aesni_xts_ctx {
65 	u8 raw_tweak_ctx[sizeof(struct crypto_aes_ctx)] AESNI_ALIGN_ATTR;
66 	u8 raw_crypt_ctx[sizeof(struct crypto_aes_ctx)] AESNI_ALIGN_ATTR;
67 };
68 
69 #define GCM_BLOCK_LEN 16
70 
71 struct gcm_context_data {
72 	/* init, update and finalize context data */
73 	u8 aad_hash[GCM_BLOCK_LEN];
74 	u64 aad_length;
75 	u64 in_length;
76 	u8 partial_block_enc_key[GCM_BLOCK_LEN];
77 	u8 orig_IV[GCM_BLOCK_LEN];
78 	u8 current_counter[GCM_BLOCK_LEN];
79 	u64 partial_block_len;
80 	u64 unused;
81 	u8 hash_keys[GCM_BLOCK_LEN * 16];
82 };
83 
84 asmlinkage int aesni_set_key(struct crypto_aes_ctx *ctx, const u8 *in_key,
85 			     unsigned int key_len);
86 asmlinkage void aesni_enc(struct crypto_aes_ctx *ctx, u8 *out,
87 			  const u8 *in);
88 asmlinkage void aesni_dec(struct crypto_aes_ctx *ctx, u8 *out,
89 			  const u8 *in);
90 asmlinkage void aesni_ecb_enc(struct crypto_aes_ctx *ctx, u8 *out,
91 			      const u8 *in, unsigned int len);
92 asmlinkage void aesni_ecb_dec(struct crypto_aes_ctx *ctx, u8 *out,
93 			      const u8 *in, unsigned int len);
94 asmlinkage void aesni_cbc_enc(struct crypto_aes_ctx *ctx, u8 *out,
95 			      const u8 *in, unsigned int len, u8 *iv);
96 asmlinkage void aesni_cbc_dec(struct crypto_aes_ctx *ctx, u8 *out,
97 			      const u8 *in, unsigned int len, u8 *iv);
98 
99 #define AVX_GEN2_OPTSIZE 640
100 #define AVX_GEN4_OPTSIZE 4096
101 
102 #ifdef CONFIG_X86_64
103 
104 static void (*aesni_ctr_enc_tfm)(struct crypto_aes_ctx *ctx, u8 *out,
105 			      const u8 *in, unsigned int len, u8 *iv);
106 asmlinkage void aesni_ctr_enc(struct crypto_aes_ctx *ctx, u8 *out,
107 			      const u8 *in, unsigned int len, u8 *iv);
108 
109 asmlinkage void aesni_xts_crypt8(struct crypto_aes_ctx *ctx, u8 *out,
110 				 const u8 *in, bool enc, u8 *iv);
111 
112 /* asmlinkage void aesni_gcm_enc()
113  * void *ctx,  AES Key schedule. Starts on a 16 byte boundary.
114  * struct gcm_context_data.  May be uninitialized.
115  * u8 *out, Ciphertext output. Encrypt in-place is allowed.
116  * const u8 *in, Plaintext input
117  * unsigned long plaintext_len, Length of data in bytes for encryption.
118  * u8 *iv, Pre-counter block j0: 12 byte IV concatenated with 0x00000001.
119  *         16-byte aligned pointer.
120  * u8 *hash_subkey, the Hash sub key input. Data starts on a 16-byte boundary.
121  * const u8 *aad, Additional Authentication Data (AAD)
122  * unsigned long aad_len, Length of AAD in bytes.
123  * u8 *auth_tag, Authenticated Tag output.
124  * unsigned long auth_tag_len), Authenticated Tag Length in bytes.
125  *          Valid values are 16 (most likely), 12 or 8.
126  */
127 asmlinkage void aesni_gcm_enc(void *ctx,
128 			struct gcm_context_data *gdata, u8 *out,
129 			const u8 *in, unsigned long plaintext_len, u8 *iv,
130 			u8 *hash_subkey, const u8 *aad, unsigned long aad_len,
131 			u8 *auth_tag, unsigned long auth_tag_len);
132 
133 /* asmlinkage void aesni_gcm_dec()
134  * void *ctx, AES Key schedule. Starts on a 16 byte boundary.
135  * struct gcm_context_data.  May be uninitialized.
136  * u8 *out, Plaintext output. Decrypt in-place is allowed.
137  * const u8 *in, Ciphertext input
138  * unsigned long ciphertext_len, Length of data in bytes for decryption.
139  * u8 *iv, Pre-counter block j0: 12 byte IV concatenated with 0x00000001.
140  *         16-byte aligned pointer.
141  * u8 *hash_subkey, the Hash sub key input. Data starts on a 16-byte boundary.
142  * const u8 *aad, Additional Authentication Data (AAD)
143  * unsigned long aad_len, Length of AAD in bytes. With RFC4106 this is going
144  * to be 8 or 12 bytes
145  * u8 *auth_tag, Authenticated Tag output.
146  * unsigned long auth_tag_len) Authenticated Tag Length in bytes.
147  * Valid values are 16 (most likely), 12 or 8.
148  */
149 asmlinkage void aesni_gcm_dec(void *ctx,
150 			struct gcm_context_data *gdata, u8 *out,
151 			const u8 *in, unsigned long ciphertext_len, u8 *iv,
152 			u8 *hash_subkey, const u8 *aad, unsigned long aad_len,
153 			u8 *auth_tag, unsigned long auth_tag_len);
154 
155 /* Scatter / Gather routines, with args similar to above */
156 asmlinkage void aesni_gcm_init(void *ctx,
157 			       struct gcm_context_data *gdata,
158 			       u8 *iv,
159 			       u8 *hash_subkey, const u8 *aad,
160 			       unsigned long aad_len);
161 asmlinkage void aesni_gcm_enc_update(void *ctx,
162 				     struct gcm_context_data *gdata, u8 *out,
163 				     const u8 *in, unsigned long plaintext_len);
164 asmlinkage void aesni_gcm_dec_update(void *ctx,
165 				     struct gcm_context_data *gdata, u8 *out,
166 				     const u8 *in,
167 				     unsigned long ciphertext_len);
168 asmlinkage void aesni_gcm_finalize(void *ctx,
169 				   struct gcm_context_data *gdata,
170 				   u8 *auth_tag, unsigned long auth_tag_len);
171 
172 static const struct aesni_gcm_tfm_s {
173 	void (*init)(void *ctx, struct gcm_context_data *gdata, u8 *iv,
174 		     u8 *hash_subkey, const u8 *aad, unsigned long aad_len);
175 	void (*enc_update)(void *ctx, struct gcm_context_data *gdata, u8 *out,
176 			   const u8 *in, unsigned long plaintext_len);
177 	void (*dec_update)(void *ctx, struct gcm_context_data *gdata, u8 *out,
178 			   const u8 *in, unsigned long ciphertext_len);
179 	void (*finalize)(void *ctx, struct gcm_context_data *gdata,
180 			 u8 *auth_tag, unsigned long auth_tag_len);
181 } *aesni_gcm_tfm;
182 
183 static const struct aesni_gcm_tfm_s aesni_gcm_tfm_sse = {
184 	.init = &aesni_gcm_init,
185 	.enc_update = &aesni_gcm_enc_update,
186 	.dec_update = &aesni_gcm_dec_update,
187 	.finalize = &aesni_gcm_finalize,
188 };
189 
190 #ifdef CONFIG_AS_AVX
191 asmlinkage void aes_ctr_enc_128_avx_by8(const u8 *in, u8 *iv,
192 		void *keys, u8 *out, unsigned int num_bytes);
193 asmlinkage void aes_ctr_enc_192_avx_by8(const u8 *in, u8 *iv,
194 		void *keys, u8 *out, unsigned int num_bytes);
195 asmlinkage void aes_ctr_enc_256_avx_by8(const u8 *in, u8 *iv,
196 		void *keys, u8 *out, unsigned int num_bytes);
197 /*
198  * asmlinkage void aesni_gcm_init_avx_gen2()
199  * gcm_data *my_ctx_data, context data
200  * u8 *hash_subkey,  the Hash sub key input. Data starts on a 16-byte boundary.
201  */
202 asmlinkage void aesni_gcm_init_avx_gen2(void *my_ctx_data,
203 					struct gcm_context_data *gdata,
204 					u8 *iv,
205 					u8 *hash_subkey,
206 					const u8 *aad,
207 					unsigned long aad_len);
208 
209 asmlinkage void aesni_gcm_enc_update_avx_gen2(void *ctx,
210 				     struct gcm_context_data *gdata, u8 *out,
211 				     const u8 *in, unsigned long plaintext_len);
212 asmlinkage void aesni_gcm_dec_update_avx_gen2(void *ctx,
213 				     struct gcm_context_data *gdata, u8 *out,
214 				     const u8 *in,
215 				     unsigned long ciphertext_len);
216 asmlinkage void aesni_gcm_finalize_avx_gen2(void *ctx,
217 				   struct gcm_context_data *gdata,
218 				   u8 *auth_tag, unsigned long auth_tag_len);
219 
220 asmlinkage void aesni_gcm_enc_avx_gen2(void *ctx,
221 				struct gcm_context_data *gdata, u8 *out,
222 			const u8 *in, unsigned long plaintext_len, u8 *iv,
223 			const u8 *aad, unsigned long aad_len,
224 			u8 *auth_tag, unsigned long auth_tag_len);
225 
226 asmlinkage void aesni_gcm_dec_avx_gen2(void *ctx,
227 				struct gcm_context_data *gdata, u8 *out,
228 			const u8 *in, unsigned long ciphertext_len, u8 *iv,
229 			const u8 *aad, unsigned long aad_len,
230 			u8 *auth_tag, unsigned long auth_tag_len);
231 
232 static const struct aesni_gcm_tfm_s aesni_gcm_tfm_avx_gen2 = {
233 	.init = &aesni_gcm_init_avx_gen2,
234 	.enc_update = &aesni_gcm_enc_update_avx_gen2,
235 	.dec_update = &aesni_gcm_dec_update_avx_gen2,
236 	.finalize = &aesni_gcm_finalize_avx_gen2,
237 };
238 
239 #endif
240 
241 #ifdef CONFIG_AS_AVX2
242 /*
243  * asmlinkage void aesni_gcm_init_avx_gen4()
244  * gcm_data *my_ctx_data, context data
245  * u8 *hash_subkey,  the Hash sub key input. Data starts on a 16-byte boundary.
246  */
247 asmlinkage void aesni_gcm_init_avx_gen4(void *my_ctx_data,
248 					struct gcm_context_data *gdata,
249 					u8 *iv,
250 					u8 *hash_subkey,
251 					const u8 *aad,
252 					unsigned long aad_len);
253 
254 asmlinkage void aesni_gcm_enc_update_avx_gen4(void *ctx,
255 				     struct gcm_context_data *gdata, u8 *out,
256 				     const u8 *in, unsigned long plaintext_len);
257 asmlinkage void aesni_gcm_dec_update_avx_gen4(void *ctx,
258 				     struct gcm_context_data *gdata, u8 *out,
259 				     const u8 *in,
260 				     unsigned long ciphertext_len);
261 asmlinkage void aesni_gcm_finalize_avx_gen4(void *ctx,
262 				   struct gcm_context_data *gdata,
263 				   u8 *auth_tag, unsigned long auth_tag_len);
264 
265 asmlinkage void aesni_gcm_enc_avx_gen4(void *ctx,
266 				struct gcm_context_data *gdata, u8 *out,
267 			const u8 *in, unsigned long plaintext_len, u8 *iv,
268 			const u8 *aad, unsigned long aad_len,
269 			u8 *auth_tag, unsigned long auth_tag_len);
270 
271 asmlinkage void aesni_gcm_dec_avx_gen4(void *ctx,
272 				struct gcm_context_data *gdata, u8 *out,
273 			const u8 *in, unsigned long ciphertext_len, u8 *iv,
274 			const u8 *aad, unsigned long aad_len,
275 			u8 *auth_tag, unsigned long auth_tag_len);
276 
277 static const struct aesni_gcm_tfm_s aesni_gcm_tfm_avx_gen4 = {
278 	.init = &aesni_gcm_init_avx_gen4,
279 	.enc_update = &aesni_gcm_enc_update_avx_gen4,
280 	.dec_update = &aesni_gcm_dec_update_avx_gen4,
281 	.finalize = &aesni_gcm_finalize_avx_gen4,
282 };
283 
284 #endif
285 
286 static inline struct
287 aesni_rfc4106_gcm_ctx *aesni_rfc4106_gcm_ctx_get(struct crypto_aead *tfm)
288 {
289 	unsigned long align = AESNI_ALIGN;
290 
291 	if (align <= crypto_tfm_ctx_alignment())
292 		align = 1;
293 	return PTR_ALIGN(crypto_aead_ctx(tfm), align);
294 }
295 
296 static inline struct
297 generic_gcmaes_ctx *generic_gcmaes_ctx_get(struct crypto_aead *tfm)
298 {
299 	unsigned long align = AESNI_ALIGN;
300 
301 	if (align <= crypto_tfm_ctx_alignment())
302 		align = 1;
303 	return PTR_ALIGN(crypto_aead_ctx(tfm), align);
304 }
305 #endif
306 
307 static inline struct crypto_aes_ctx *aes_ctx(void *raw_ctx)
308 {
309 	unsigned long addr = (unsigned long)raw_ctx;
310 	unsigned long align = AESNI_ALIGN;
311 
312 	if (align <= crypto_tfm_ctx_alignment())
313 		align = 1;
314 	return (struct crypto_aes_ctx *)ALIGN(addr, align);
315 }
316 
317 static int aes_set_key_common(struct crypto_tfm *tfm, void *raw_ctx,
318 			      const u8 *in_key, unsigned int key_len)
319 {
320 	struct crypto_aes_ctx *ctx = aes_ctx(raw_ctx);
321 	u32 *flags = &tfm->crt_flags;
322 	int err;
323 
324 	if (key_len != AES_KEYSIZE_128 && key_len != AES_KEYSIZE_192 &&
325 	    key_len != AES_KEYSIZE_256) {
326 		*flags |= CRYPTO_TFM_RES_BAD_KEY_LEN;
327 		return -EINVAL;
328 	}
329 
330 	if (!crypto_simd_usable())
331 		err = aes_expandkey(ctx, in_key, key_len);
332 	else {
333 		kernel_fpu_begin();
334 		err = aesni_set_key(ctx, in_key, key_len);
335 		kernel_fpu_end();
336 	}
337 
338 	return err;
339 }
340 
341 static int aes_set_key(struct crypto_tfm *tfm, const u8 *in_key,
342 		       unsigned int key_len)
343 {
344 	return aes_set_key_common(tfm, crypto_tfm_ctx(tfm), in_key, key_len);
345 }
346 
347 static void aesni_encrypt(struct crypto_tfm *tfm, u8 *dst, const u8 *src)
348 {
349 	struct crypto_aes_ctx *ctx = aes_ctx(crypto_tfm_ctx(tfm));
350 
351 	if (!crypto_simd_usable()) {
352 		aes_encrypt(ctx, dst, src);
353 	} else {
354 		kernel_fpu_begin();
355 		aesni_enc(ctx, dst, src);
356 		kernel_fpu_end();
357 	}
358 }
359 
360 static void aesni_decrypt(struct crypto_tfm *tfm, u8 *dst, const u8 *src)
361 {
362 	struct crypto_aes_ctx *ctx = aes_ctx(crypto_tfm_ctx(tfm));
363 
364 	if (!crypto_simd_usable()) {
365 		aes_decrypt(ctx, dst, src);
366 	} else {
367 		kernel_fpu_begin();
368 		aesni_dec(ctx, dst, src);
369 		kernel_fpu_end();
370 	}
371 }
372 
373 static int aesni_skcipher_setkey(struct crypto_skcipher *tfm, const u8 *key,
374 			         unsigned int len)
375 {
376 	return aes_set_key_common(crypto_skcipher_tfm(tfm),
377 				  crypto_skcipher_ctx(tfm), key, len);
378 }
379 
380 static int ecb_encrypt(struct skcipher_request *req)
381 {
382 	struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req);
383 	struct crypto_aes_ctx *ctx = aes_ctx(crypto_skcipher_ctx(tfm));
384 	struct skcipher_walk walk;
385 	unsigned int nbytes;
386 	int err;
387 
388 	err = skcipher_walk_virt(&walk, req, true);
389 
390 	kernel_fpu_begin();
391 	while ((nbytes = walk.nbytes)) {
392 		aesni_ecb_enc(ctx, walk.dst.virt.addr, walk.src.virt.addr,
393 			      nbytes & AES_BLOCK_MASK);
394 		nbytes &= AES_BLOCK_SIZE - 1;
395 		err = skcipher_walk_done(&walk, nbytes);
396 	}
397 	kernel_fpu_end();
398 
399 	return err;
400 }
401 
402 static int ecb_decrypt(struct skcipher_request *req)
403 {
404 	struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req);
405 	struct crypto_aes_ctx *ctx = aes_ctx(crypto_skcipher_ctx(tfm));
406 	struct skcipher_walk walk;
407 	unsigned int nbytes;
408 	int err;
409 
410 	err = skcipher_walk_virt(&walk, req, true);
411 
412 	kernel_fpu_begin();
413 	while ((nbytes = walk.nbytes)) {
414 		aesni_ecb_dec(ctx, walk.dst.virt.addr, walk.src.virt.addr,
415 			      nbytes & AES_BLOCK_MASK);
416 		nbytes &= AES_BLOCK_SIZE - 1;
417 		err = skcipher_walk_done(&walk, nbytes);
418 	}
419 	kernel_fpu_end();
420 
421 	return err;
422 }
423 
424 static int cbc_encrypt(struct skcipher_request *req)
425 {
426 	struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req);
427 	struct crypto_aes_ctx *ctx = aes_ctx(crypto_skcipher_ctx(tfm));
428 	struct skcipher_walk walk;
429 	unsigned int nbytes;
430 	int err;
431 
432 	err = skcipher_walk_virt(&walk, req, true);
433 
434 	kernel_fpu_begin();
435 	while ((nbytes = walk.nbytes)) {
436 		aesni_cbc_enc(ctx, walk.dst.virt.addr, walk.src.virt.addr,
437 			      nbytes & AES_BLOCK_MASK, walk.iv);
438 		nbytes &= AES_BLOCK_SIZE - 1;
439 		err = skcipher_walk_done(&walk, nbytes);
440 	}
441 	kernel_fpu_end();
442 
443 	return err;
444 }
445 
446 static int cbc_decrypt(struct skcipher_request *req)
447 {
448 	struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req);
449 	struct crypto_aes_ctx *ctx = aes_ctx(crypto_skcipher_ctx(tfm));
450 	struct skcipher_walk walk;
451 	unsigned int nbytes;
452 	int err;
453 
454 	err = skcipher_walk_virt(&walk, req, true);
455 
456 	kernel_fpu_begin();
457 	while ((nbytes = walk.nbytes)) {
458 		aesni_cbc_dec(ctx, walk.dst.virt.addr, walk.src.virt.addr,
459 			      nbytes & AES_BLOCK_MASK, walk.iv);
460 		nbytes &= AES_BLOCK_SIZE - 1;
461 		err = skcipher_walk_done(&walk, nbytes);
462 	}
463 	kernel_fpu_end();
464 
465 	return err;
466 }
467 
468 #ifdef CONFIG_X86_64
469 static void ctr_crypt_final(struct crypto_aes_ctx *ctx,
470 			    struct skcipher_walk *walk)
471 {
472 	u8 *ctrblk = walk->iv;
473 	u8 keystream[AES_BLOCK_SIZE];
474 	u8 *src = walk->src.virt.addr;
475 	u8 *dst = walk->dst.virt.addr;
476 	unsigned int nbytes = walk->nbytes;
477 
478 	aesni_enc(ctx, keystream, ctrblk);
479 	crypto_xor_cpy(dst, keystream, src, nbytes);
480 
481 	crypto_inc(ctrblk, AES_BLOCK_SIZE);
482 }
483 
484 #ifdef CONFIG_AS_AVX
485 static void aesni_ctr_enc_avx_tfm(struct crypto_aes_ctx *ctx, u8 *out,
486 			      const u8 *in, unsigned int len, u8 *iv)
487 {
488 	/*
489 	 * based on key length, override with the by8 version
490 	 * of ctr mode encryption/decryption for improved performance
491 	 * aes_set_key_common() ensures that key length is one of
492 	 * {128,192,256}
493 	 */
494 	if (ctx->key_length == AES_KEYSIZE_128)
495 		aes_ctr_enc_128_avx_by8(in, iv, (void *)ctx, out, len);
496 	else if (ctx->key_length == AES_KEYSIZE_192)
497 		aes_ctr_enc_192_avx_by8(in, iv, (void *)ctx, out, len);
498 	else
499 		aes_ctr_enc_256_avx_by8(in, iv, (void *)ctx, out, len);
500 }
501 #endif
502 
503 static int ctr_crypt(struct skcipher_request *req)
504 {
505 	struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req);
506 	struct crypto_aes_ctx *ctx = aes_ctx(crypto_skcipher_ctx(tfm));
507 	struct skcipher_walk walk;
508 	unsigned int nbytes;
509 	int err;
510 
511 	err = skcipher_walk_virt(&walk, req, true);
512 
513 	kernel_fpu_begin();
514 	while ((nbytes = walk.nbytes) >= AES_BLOCK_SIZE) {
515 		aesni_ctr_enc_tfm(ctx, walk.dst.virt.addr, walk.src.virt.addr,
516 			              nbytes & AES_BLOCK_MASK, walk.iv);
517 		nbytes &= AES_BLOCK_SIZE - 1;
518 		err = skcipher_walk_done(&walk, nbytes);
519 	}
520 	if (walk.nbytes) {
521 		ctr_crypt_final(ctx, &walk);
522 		err = skcipher_walk_done(&walk, 0);
523 	}
524 	kernel_fpu_end();
525 
526 	return err;
527 }
528 
529 static int xts_aesni_setkey(struct crypto_skcipher *tfm, const u8 *key,
530 			    unsigned int keylen)
531 {
532 	struct aesni_xts_ctx *ctx = crypto_skcipher_ctx(tfm);
533 	int err;
534 
535 	err = xts_verify_key(tfm, key, keylen);
536 	if (err)
537 		return err;
538 
539 	keylen /= 2;
540 
541 	/* first half of xts-key is for crypt */
542 	err = aes_set_key_common(crypto_skcipher_tfm(tfm), ctx->raw_crypt_ctx,
543 				 key, keylen);
544 	if (err)
545 		return err;
546 
547 	/* second half of xts-key is for tweak */
548 	return aes_set_key_common(crypto_skcipher_tfm(tfm), ctx->raw_tweak_ctx,
549 				  key + keylen, keylen);
550 }
551 
552 
553 static void aesni_xts_tweak(void *ctx, u8 *out, const u8 *in)
554 {
555 	aesni_enc(ctx, out, in);
556 }
557 
558 static void aesni_xts_enc(void *ctx, u128 *dst, const u128 *src, le128 *iv)
559 {
560 	glue_xts_crypt_128bit_one(ctx, dst, src, iv, GLUE_FUNC_CAST(aesni_enc));
561 }
562 
563 static void aesni_xts_dec(void *ctx, u128 *dst, const u128 *src, le128 *iv)
564 {
565 	glue_xts_crypt_128bit_one(ctx, dst, src, iv, GLUE_FUNC_CAST(aesni_dec));
566 }
567 
568 static void aesni_xts_enc8(void *ctx, u128 *dst, const u128 *src, le128 *iv)
569 {
570 	aesni_xts_crypt8(ctx, (u8 *)dst, (const u8 *)src, true, (u8 *)iv);
571 }
572 
573 static void aesni_xts_dec8(void *ctx, u128 *dst, const u128 *src, le128 *iv)
574 {
575 	aesni_xts_crypt8(ctx, (u8 *)dst, (const u8 *)src, false, (u8 *)iv);
576 }
577 
578 static const struct common_glue_ctx aesni_enc_xts = {
579 	.num_funcs = 2,
580 	.fpu_blocks_limit = 1,
581 
582 	.funcs = { {
583 		.num_blocks = 8,
584 		.fn_u = { .xts = GLUE_XTS_FUNC_CAST(aesni_xts_enc8) }
585 	}, {
586 		.num_blocks = 1,
587 		.fn_u = { .xts = GLUE_XTS_FUNC_CAST(aesni_xts_enc) }
588 	} }
589 };
590 
591 static const struct common_glue_ctx aesni_dec_xts = {
592 	.num_funcs = 2,
593 	.fpu_blocks_limit = 1,
594 
595 	.funcs = { {
596 		.num_blocks = 8,
597 		.fn_u = { .xts = GLUE_XTS_FUNC_CAST(aesni_xts_dec8) }
598 	}, {
599 		.num_blocks = 1,
600 		.fn_u = { .xts = GLUE_XTS_FUNC_CAST(aesni_xts_dec) }
601 	} }
602 };
603 
604 static int xts_encrypt(struct skcipher_request *req)
605 {
606 	struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req);
607 	struct aesni_xts_ctx *ctx = crypto_skcipher_ctx(tfm);
608 
609 	return glue_xts_req_128bit(&aesni_enc_xts, req,
610 				   XTS_TWEAK_CAST(aesni_xts_tweak),
611 				   aes_ctx(ctx->raw_tweak_ctx),
612 				   aes_ctx(ctx->raw_crypt_ctx),
613 				   false);
614 }
615 
616 static int xts_decrypt(struct skcipher_request *req)
617 {
618 	struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req);
619 	struct aesni_xts_ctx *ctx = crypto_skcipher_ctx(tfm);
620 
621 	return glue_xts_req_128bit(&aesni_dec_xts, req,
622 				   XTS_TWEAK_CAST(aesni_xts_tweak),
623 				   aes_ctx(ctx->raw_tweak_ctx),
624 				   aes_ctx(ctx->raw_crypt_ctx),
625 				   true);
626 }
627 
628 static int
629 rfc4106_set_hash_subkey(u8 *hash_subkey, const u8 *key, unsigned int key_len)
630 {
631 	struct crypto_aes_ctx ctx;
632 	int ret;
633 
634 	ret = aes_expandkey(&ctx, key, key_len);
635 	if (ret)
636 		return ret;
637 
638 	/* Clear the data in the hash sub key container to zero.*/
639 	/* We want to cipher all zeros to create the hash sub key. */
640 	memset(hash_subkey, 0, RFC4106_HASH_SUBKEY_SIZE);
641 
642 	aes_encrypt(&ctx, hash_subkey, hash_subkey);
643 
644 	memzero_explicit(&ctx, sizeof(ctx));
645 	return 0;
646 }
647 
648 static int common_rfc4106_set_key(struct crypto_aead *aead, const u8 *key,
649 				  unsigned int key_len)
650 {
651 	struct aesni_rfc4106_gcm_ctx *ctx = aesni_rfc4106_gcm_ctx_get(aead);
652 
653 	if (key_len < 4) {
654 		crypto_aead_set_flags(aead, CRYPTO_TFM_RES_BAD_KEY_LEN);
655 		return -EINVAL;
656 	}
657 	/*Account for 4 byte nonce at the end.*/
658 	key_len -= 4;
659 
660 	memcpy(ctx->nonce, key + key_len, sizeof(ctx->nonce));
661 
662 	return aes_set_key_common(crypto_aead_tfm(aead),
663 				  &ctx->aes_key_expanded, key, key_len) ?:
664 	       rfc4106_set_hash_subkey(ctx->hash_subkey, key, key_len);
665 }
666 
667 /* This is the Integrity Check Value (aka the authentication tag) length and can
668  * be 8, 12 or 16 bytes long. */
669 static int common_rfc4106_set_authsize(struct crypto_aead *aead,
670 				       unsigned int authsize)
671 {
672 	switch (authsize) {
673 	case 8:
674 	case 12:
675 	case 16:
676 		break;
677 	default:
678 		return -EINVAL;
679 	}
680 
681 	return 0;
682 }
683 
684 static int generic_gcmaes_set_authsize(struct crypto_aead *tfm,
685 				       unsigned int authsize)
686 {
687 	switch (authsize) {
688 	case 4:
689 	case 8:
690 	case 12:
691 	case 13:
692 	case 14:
693 	case 15:
694 	case 16:
695 		break;
696 	default:
697 		return -EINVAL;
698 	}
699 
700 	return 0;
701 }
702 
703 static int gcmaes_crypt_by_sg(bool enc, struct aead_request *req,
704 			      unsigned int assoclen, u8 *hash_subkey,
705 			      u8 *iv, void *aes_ctx)
706 {
707 	struct crypto_aead *tfm = crypto_aead_reqtfm(req);
708 	unsigned long auth_tag_len = crypto_aead_authsize(tfm);
709 	const struct aesni_gcm_tfm_s *gcm_tfm = aesni_gcm_tfm;
710 	struct gcm_context_data data AESNI_ALIGN_ATTR;
711 	struct scatter_walk dst_sg_walk = {};
712 	unsigned long left = req->cryptlen;
713 	unsigned long len, srclen, dstlen;
714 	struct scatter_walk assoc_sg_walk;
715 	struct scatter_walk src_sg_walk;
716 	struct scatterlist src_start[2];
717 	struct scatterlist dst_start[2];
718 	struct scatterlist *src_sg;
719 	struct scatterlist *dst_sg;
720 	u8 *src, *dst, *assoc;
721 	u8 *assocmem = NULL;
722 	u8 authTag[16];
723 
724 	if (!enc)
725 		left -= auth_tag_len;
726 
727 #ifdef CONFIG_AS_AVX2
728 	if (left < AVX_GEN4_OPTSIZE && gcm_tfm == &aesni_gcm_tfm_avx_gen4)
729 		gcm_tfm = &aesni_gcm_tfm_avx_gen2;
730 #endif
731 #ifdef CONFIG_AS_AVX
732 	if (left < AVX_GEN2_OPTSIZE && gcm_tfm == &aesni_gcm_tfm_avx_gen2)
733 		gcm_tfm = &aesni_gcm_tfm_sse;
734 #endif
735 
736 	/* Linearize assoc, if not already linear */
737 	if (req->src->length >= assoclen && req->src->length &&
738 		(!PageHighMem(sg_page(req->src)) ||
739 			req->src->offset + req->src->length <= PAGE_SIZE)) {
740 		scatterwalk_start(&assoc_sg_walk, req->src);
741 		assoc = scatterwalk_map(&assoc_sg_walk);
742 	} else {
743 		/* assoc can be any length, so must be on heap */
744 		assocmem = kmalloc(assoclen, GFP_ATOMIC);
745 		if (unlikely(!assocmem))
746 			return -ENOMEM;
747 		assoc = assocmem;
748 
749 		scatterwalk_map_and_copy(assoc, req->src, 0, assoclen, 0);
750 	}
751 
752 	if (left) {
753 		src_sg = scatterwalk_ffwd(src_start, req->src, req->assoclen);
754 		scatterwalk_start(&src_sg_walk, src_sg);
755 		if (req->src != req->dst) {
756 			dst_sg = scatterwalk_ffwd(dst_start, req->dst,
757 						  req->assoclen);
758 			scatterwalk_start(&dst_sg_walk, dst_sg);
759 		}
760 	}
761 
762 	kernel_fpu_begin();
763 	gcm_tfm->init(aes_ctx, &data, iv,
764 		hash_subkey, assoc, assoclen);
765 	if (req->src != req->dst) {
766 		while (left) {
767 			src = scatterwalk_map(&src_sg_walk);
768 			dst = scatterwalk_map(&dst_sg_walk);
769 			srclen = scatterwalk_clamp(&src_sg_walk, left);
770 			dstlen = scatterwalk_clamp(&dst_sg_walk, left);
771 			len = min(srclen, dstlen);
772 			if (len) {
773 				if (enc)
774 					gcm_tfm->enc_update(aes_ctx, &data,
775 							     dst, src, len);
776 				else
777 					gcm_tfm->dec_update(aes_ctx, &data,
778 							     dst, src, len);
779 			}
780 			left -= len;
781 
782 			scatterwalk_unmap(src);
783 			scatterwalk_unmap(dst);
784 			scatterwalk_advance(&src_sg_walk, len);
785 			scatterwalk_advance(&dst_sg_walk, len);
786 			scatterwalk_done(&src_sg_walk, 0, left);
787 			scatterwalk_done(&dst_sg_walk, 1, left);
788 		}
789 	} else {
790 		while (left) {
791 			dst = src = scatterwalk_map(&src_sg_walk);
792 			len = scatterwalk_clamp(&src_sg_walk, left);
793 			if (len) {
794 				if (enc)
795 					gcm_tfm->enc_update(aes_ctx, &data,
796 							     src, src, len);
797 				else
798 					gcm_tfm->dec_update(aes_ctx, &data,
799 							     src, src, len);
800 			}
801 			left -= len;
802 			scatterwalk_unmap(src);
803 			scatterwalk_advance(&src_sg_walk, len);
804 			scatterwalk_done(&src_sg_walk, 1, left);
805 		}
806 	}
807 	gcm_tfm->finalize(aes_ctx, &data, authTag, auth_tag_len);
808 	kernel_fpu_end();
809 
810 	if (!assocmem)
811 		scatterwalk_unmap(assoc);
812 	else
813 		kfree(assocmem);
814 
815 	if (!enc) {
816 		u8 authTagMsg[16];
817 
818 		/* Copy out original authTag */
819 		scatterwalk_map_and_copy(authTagMsg, req->src,
820 					 req->assoclen + req->cryptlen -
821 					 auth_tag_len,
822 					 auth_tag_len, 0);
823 
824 		/* Compare generated tag with passed in tag. */
825 		return crypto_memneq(authTagMsg, authTag, auth_tag_len) ?
826 			-EBADMSG : 0;
827 	}
828 
829 	/* Copy in the authTag */
830 	scatterwalk_map_and_copy(authTag, req->dst,
831 				 req->assoclen + req->cryptlen,
832 				 auth_tag_len, 1);
833 
834 	return 0;
835 }
836 
837 static int gcmaes_encrypt(struct aead_request *req, unsigned int assoclen,
838 			  u8 *hash_subkey, u8 *iv, void *aes_ctx)
839 {
840 	return gcmaes_crypt_by_sg(true, req, assoclen, hash_subkey, iv,
841 				aes_ctx);
842 }
843 
844 static int gcmaes_decrypt(struct aead_request *req, unsigned int assoclen,
845 			  u8 *hash_subkey, u8 *iv, void *aes_ctx)
846 {
847 	return gcmaes_crypt_by_sg(false, req, assoclen, hash_subkey, iv,
848 				aes_ctx);
849 }
850 
851 static int helper_rfc4106_encrypt(struct aead_request *req)
852 {
853 	struct crypto_aead *tfm = crypto_aead_reqtfm(req);
854 	struct aesni_rfc4106_gcm_ctx *ctx = aesni_rfc4106_gcm_ctx_get(tfm);
855 	void *aes_ctx = &(ctx->aes_key_expanded);
856 	u8 iv[16] __attribute__ ((__aligned__(AESNI_ALIGN)));
857 	unsigned int i;
858 	__be32 counter = cpu_to_be32(1);
859 
860 	/* Assuming we are supporting rfc4106 64-bit extended */
861 	/* sequence numbers We need to have the AAD length equal */
862 	/* to 16 or 20 bytes */
863 	if (unlikely(req->assoclen != 16 && req->assoclen != 20))
864 		return -EINVAL;
865 
866 	/* IV below built */
867 	for (i = 0; i < 4; i++)
868 		*(iv+i) = ctx->nonce[i];
869 	for (i = 0; i < 8; i++)
870 		*(iv+4+i) = req->iv[i];
871 	*((__be32 *)(iv+12)) = counter;
872 
873 	return gcmaes_encrypt(req, req->assoclen - 8, ctx->hash_subkey, iv,
874 			      aes_ctx);
875 }
876 
877 static int helper_rfc4106_decrypt(struct aead_request *req)
878 {
879 	__be32 counter = cpu_to_be32(1);
880 	struct crypto_aead *tfm = crypto_aead_reqtfm(req);
881 	struct aesni_rfc4106_gcm_ctx *ctx = aesni_rfc4106_gcm_ctx_get(tfm);
882 	void *aes_ctx = &(ctx->aes_key_expanded);
883 	u8 iv[16] __attribute__ ((__aligned__(AESNI_ALIGN)));
884 	unsigned int i;
885 
886 	if (unlikely(req->assoclen != 16 && req->assoclen != 20))
887 		return -EINVAL;
888 
889 	/* Assuming we are supporting rfc4106 64-bit extended */
890 	/* sequence numbers We need to have the AAD length */
891 	/* equal to 16 or 20 bytes */
892 
893 	/* IV below built */
894 	for (i = 0; i < 4; i++)
895 		*(iv+i) = ctx->nonce[i];
896 	for (i = 0; i < 8; i++)
897 		*(iv+4+i) = req->iv[i];
898 	*((__be32 *)(iv+12)) = counter;
899 
900 	return gcmaes_decrypt(req, req->assoclen - 8, ctx->hash_subkey, iv,
901 			      aes_ctx);
902 }
903 #endif
904 
905 static struct crypto_alg aesni_cipher_alg = {
906 	.cra_name		= "aes",
907 	.cra_driver_name	= "aes-aesni",
908 	.cra_priority		= 300,
909 	.cra_flags		= CRYPTO_ALG_TYPE_CIPHER,
910 	.cra_blocksize		= AES_BLOCK_SIZE,
911 	.cra_ctxsize		= CRYPTO_AES_CTX_SIZE,
912 	.cra_module		= THIS_MODULE,
913 	.cra_u	= {
914 		.cipher	= {
915 			.cia_min_keysize	= AES_MIN_KEY_SIZE,
916 			.cia_max_keysize	= AES_MAX_KEY_SIZE,
917 			.cia_setkey		= aes_set_key,
918 			.cia_encrypt		= aesni_encrypt,
919 			.cia_decrypt		= aesni_decrypt
920 		}
921 	}
922 };
923 
924 static struct skcipher_alg aesni_skciphers[] = {
925 	{
926 		.base = {
927 			.cra_name		= "__ecb(aes)",
928 			.cra_driver_name	= "__ecb-aes-aesni",
929 			.cra_priority		= 400,
930 			.cra_flags		= CRYPTO_ALG_INTERNAL,
931 			.cra_blocksize		= AES_BLOCK_SIZE,
932 			.cra_ctxsize		= CRYPTO_AES_CTX_SIZE,
933 			.cra_module		= THIS_MODULE,
934 		},
935 		.min_keysize	= AES_MIN_KEY_SIZE,
936 		.max_keysize	= AES_MAX_KEY_SIZE,
937 		.setkey		= aesni_skcipher_setkey,
938 		.encrypt	= ecb_encrypt,
939 		.decrypt	= ecb_decrypt,
940 	}, {
941 		.base = {
942 			.cra_name		= "__cbc(aes)",
943 			.cra_driver_name	= "__cbc-aes-aesni",
944 			.cra_priority		= 400,
945 			.cra_flags		= CRYPTO_ALG_INTERNAL,
946 			.cra_blocksize		= AES_BLOCK_SIZE,
947 			.cra_ctxsize		= CRYPTO_AES_CTX_SIZE,
948 			.cra_module		= THIS_MODULE,
949 		},
950 		.min_keysize	= AES_MIN_KEY_SIZE,
951 		.max_keysize	= AES_MAX_KEY_SIZE,
952 		.ivsize		= AES_BLOCK_SIZE,
953 		.setkey		= aesni_skcipher_setkey,
954 		.encrypt	= cbc_encrypt,
955 		.decrypt	= cbc_decrypt,
956 #ifdef CONFIG_X86_64
957 	}, {
958 		.base = {
959 			.cra_name		= "__ctr(aes)",
960 			.cra_driver_name	= "__ctr-aes-aesni",
961 			.cra_priority		= 400,
962 			.cra_flags		= CRYPTO_ALG_INTERNAL,
963 			.cra_blocksize		= 1,
964 			.cra_ctxsize		= CRYPTO_AES_CTX_SIZE,
965 			.cra_module		= THIS_MODULE,
966 		},
967 		.min_keysize	= AES_MIN_KEY_SIZE,
968 		.max_keysize	= AES_MAX_KEY_SIZE,
969 		.ivsize		= AES_BLOCK_SIZE,
970 		.chunksize	= AES_BLOCK_SIZE,
971 		.setkey		= aesni_skcipher_setkey,
972 		.encrypt	= ctr_crypt,
973 		.decrypt	= ctr_crypt,
974 	}, {
975 		.base = {
976 			.cra_name		= "__xts(aes)",
977 			.cra_driver_name	= "__xts-aes-aesni",
978 			.cra_priority		= 401,
979 			.cra_flags		= CRYPTO_ALG_INTERNAL,
980 			.cra_blocksize		= AES_BLOCK_SIZE,
981 			.cra_ctxsize		= XTS_AES_CTX_SIZE,
982 			.cra_module		= THIS_MODULE,
983 		},
984 		.min_keysize	= 2 * AES_MIN_KEY_SIZE,
985 		.max_keysize	= 2 * AES_MAX_KEY_SIZE,
986 		.ivsize		= AES_BLOCK_SIZE,
987 		.setkey		= xts_aesni_setkey,
988 		.encrypt	= xts_encrypt,
989 		.decrypt	= xts_decrypt,
990 #endif
991 	}
992 };
993 
994 static
995 struct simd_skcipher_alg *aesni_simd_skciphers[ARRAY_SIZE(aesni_skciphers)];
996 
997 #ifdef CONFIG_X86_64
998 static int generic_gcmaes_set_key(struct crypto_aead *aead, const u8 *key,
999 				  unsigned int key_len)
1000 {
1001 	struct generic_gcmaes_ctx *ctx = generic_gcmaes_ctx_get(aead);
1002 
1003 	return aes_set_key_common(crypto_aead_tfm(aead),
1004 				  &ctx->aes_key_expanded, key, key_len) ?:
1005 	       rfc4106_set_hash_subkey(ctx->hash_subkey, key, key_len);
1006 }
1007 
1008 static int generic_gcmaes_encrypt(struct aead_request *req)
1009 {
1010 	struct crypto_aead *tfm = crypto_aead_reqtfm(req);
1011 	struct generic_gcmaes_ctx *ctx = generic_gcmaes_ctx_get(tfm);
1012 	void *aes_ctx = &(ctx->aes_key_expanded);
1013 	u8 iv[16] __attribute__ ((__aligned__(AESNI_ALIGN)));
1014 	__be32 counter = cpu_to_be32(1);
1015 
1016 	memcpy(iv, req->iv, 12);
1017 	*((__be32 *)(iv+12)) = counter;
1018 
1019 	return gcmaes_encrypt(req, req->assoclen, ctx->hash_subkey, iv,
1020 			      aes_ctx);
1021 }
1022 
1023 static int generic_gcmaes_decrypt(struct aead_request *req)
1024 {
1025 	__be32 counter = cpu_to_be32(1);
1026 	struct crypto_aead *tfm = crypto_aead_reqtfm(req);
1027 	struct generic_gcmaes_ctx *ctx = generic_gcmaes_ctx_get(tfm);
1028 	void *aes_ctx = &(ctx->aes_key_expanded);
1029 	u8 iv[16] __attribute__ ((__aligned__(AESNI_ALIGN)));
1030 
1031 	memcpy(iv, req->iv, 12);
1032 	*((__be32 *)(iv+12)) = counter;
1033 
1034 	return gcmaes_decrypt(req, req->assoclen, ctx->hash_subkey, iv,
1035 			      aes_ctx);
1036 }
1037 
1038 static struct aead_alg aesni_aeads[] = { {
1039 	.setkey			= common_rfc4106_set_key,
1040 	.setauthsize		= common_rfc4106_set_authsize,
1041 	.encrypt		= helper_rfc4106_encrypt,
1042 	.decrypt		= helper_rfc4106_decrypt,
1043 	.ivsize			= GCM_RFC4106_IV_SIZE,
1044 	.maxauthsize		= 16,
1045 	.base = {
1046 		.cra_name		= "__rfc4106(gcm(aes))",
1047 		.cra_driver_name	= "__rfc4106-gcm-aesni",
1048 		.cra_priority		= 400,
1049 		.cra_flags		= CRYPTO_ALG_INTERNAL,
1050 		.cra_blocksize		= 1,
1051 		.cra_ctxsize		= sizeof(struct aesni_rfc4106_gcm_ctx),
1052 		.cra_alignmask		= AESNI_ALIGN - 1,
1053 		.cra_module		= THIS_MODULE,
1054 	},
1055 }, {
1056 	.setkey			= generic_gcmaes_set_key,
1057 	.setauthsize		= generic_gcmaes_set_authsize,
1058 	.encrypt		= generic_gcmaes_encrypt,
1059 	.decrypt		= generic_gcmaes_decrypt,
1060 	.ivsize			= GCM_AES_IV_SIZE,
1061 	.maxauthsize		= 16,
1062 	.base = {
1063 		.cra_name		= "__gcm(aes)",
1064 		.cra_driver_name	= "__generic-gcm-aesni",
1065 		.cra_priority		= 400,
1066 		.cra_flags		= CRYPTO_ALG_INTERNAL,
1067 		.cra_blocksize		= 1,
1068 		.cra_ctxsize		= sizeof(struct generic_gcmaes_ctx),
1069 		.cra_alignmask		= AESNI_ALIGN - 1,
1070 		.cra_module		= THIS_MODULE,
1071 	},
1072 } };
1073 #else
1074 static struct aead_alg aesni_aeads[0];
1075 #endif
1076 
1077 static struct simd_aead_alg *aesni_simd_aeads[ARRAY_SIZE(aesni_aeads)];
1078 
1079 static const struct x86_cpu_id aesni_cpu_id[] = {
1080 	X86_FEATURE_MATCH(X86_FEATURE_AES),
1081 	{}
1082 };
1083 MODULE_DEVICE_TABLE(x86cpu, aesni_cpu_id);
1084 
1085 static int __init aesni_init(void)
1086 {
1087 	int err;
1088 
1089 	if (!x86_match_cpu(aesni_cpu_id))
1090 		return -ENODEV;
1091 #ifdef CONFIG_X86_64
1092 #ifdef CONFIG_AS_AVX2
1093 	if (boot_cpu_has(X86_FEATURE_AVX2)) {
1094 		pr_info("AVX2 version of gcm_enc/dec engaged.\n");
1095 		aesni_gcm_tfm = &aesni_gcm_tfm_avx_gen4;
1096 	} else
1097 #endif
1098 #ifdef CONFIG_AS_AVX
1099 	if (boot_cpu_has(X86_FEATURE_AVX)) {
1100 		pr_info("AVX version of gcm_enc/dec engaged.\n");
1101 		aesni_gcm_tfm = &aesni_gcm_tfm_avx_gen2;
1102 	} else
1103 #endif
1104 	{
1105 		pr_info("SSE version of gcm_enc/dec engaged.\n");
1106 		aesni_gcm_tfm = &aesni_gcm_tfm_sse;
1107 	}
1108 	aesni_ctr_enc_tfm = aesni_ctr_enc;
1109 #ifdef CONFIG_AS_AVX
1110 	if (boot_cpu_has(X86_FEATURE_AVX)) {
1111 		/* optimize performance of ctr mode encryption transform */
1112 		aesni_ctr_enc_tfm = aesni_ctr_enc_avx_tfm;
1113 		pr_info("AES CTR mode by8 optimization enabled\n");
1114 	}
1115 #endif
1116 #endif
1117 
1118 	err = crypto_register_alg(&aesni_cipher_alg);
1119 	if (err)
1120 		return err;
1121 
1122 	err = simd_register_skciphers_compat(aesni_skciphers,
1123 					     ARRAY_SIZE(aesni_skciphers),
1124 					     aesni_simd_skciphers);
1125 	if (err)
1126 		goto unregister_cipher;
1127 
1128 	err = simd_register_aeads_compat(aesni_aeads, ARRAY_SIZE(aesni_aeads),
1129 					 aesni_simd_aeads);
1130 	if (err)
1131 		goto unregister_skciphers;
1132 
1133 	return 0;
1134 
1135 unregister_skciphers:
1136 	simd_unregister_skciphers(aesni_skciphers, ARRAY_SIZE(aesni_skciphers),
1137 				  aesni_simd_skciphers);
1138 unregister_cipher:
1139 	crypto_unregister_alg(&aesni_cipher_alg);
1140 	return err;
1141 }
1142 
1143 static void __exit aesni_exit(void)
1144 {
1145 	simd_unregister_aeads(aesni_aeads, ARRAY_SIZE(aesni_aeads),
1146 			      aesni_simd_aeads);
1147 	simd_unregister_skciphers(aesni_skciphers, ARRAY_SIZE(aesni_skciphers),
1148 				  aesni_simd_skciphers);
1149 	crypto_unregister_alg(&aesni_cipher_alg);
1150 }
1151 
1152 late_initcall(aesni_init);
1153 module_exit(aesni_exit);
1154 
1155 MODULE_DESCRIPTION("Rijndael (AES) Cipher Algorithm, Intel AES-NI instructions optimized");
1156 MODULE_LICENSE("GPL");
1157 MODULE_ALIAS_CRYPTO("aes");
1158