1 // SPDX-License-Identifier: GPL-2.0-or-later
2 /*
3  * Support for Intel AES-NI instructions. This file contains glue
4  * code, the real AES implementation is in intel-aes_asm.S.
5  *
6  * Copyright (C) 2008, Intel Corp.
7  *    Author: Huang Ying <ying.huang@intel.com>
8  *
9  * Added RFC4106 AES-GCM support for 128-bit keys under the AEAD
10  * interface for 64-bit kernels.
11  *    Authors: Adrian Hoban <adrian.hoban@intel.com>
12  *             Gabriele Paoloni <gabriele.paoloni@intel.com>
13  *             Tadeusz Struk (tadeusz.struk@intel.com)
14  *             Aidan O'Mahony (aidan.o.mahony@intel.com)
15  *    Copyright (c) 2010, Intel Corporation.
16  */
17 
18 #include <linux/hardirq.h>
19 #include <linux/types.h>
20 #include <linux/module.h>
21 #include <linux/err.h>
22 #include <crypto/algapi.h>
23 #include <crypto/aes.h>
24 #include <crypto/ctr.h>
25 #include <crypto/b128ops.h>
26 #include <crypto/gcm.h>
27 #include <crypto/xts.h>
28 #include <asm/cpu_device_id.h>
29 #include <asm/simd.h>
30 #include <crypto/scatterwalk.h>
31 #include <crypto/internal/aead.h>
32 #include <crypto/internal/simd.h>
33 #include <crypto/internal/skcipher.h>
34 #include <linux/jump_label.h>
35 #include <linux/workqueue.h>
36 #include <linux/spinlock.h>
37 #include <linux/static_call.h>
38 
39 
40 #define AESNI_ALIGN	16
41 #define AESNI_ALIGN_ATTR __attribute__ ((__aligned__(AESNI_ALIGN)))
42 #define AES_BLOCK_MASK	(~(AES_BLOCK_SIZE - 1))
43 #define RFC4106_HASH_SUBKEY_SIZE 16
44 #define AESNI_ALIGN_EXTRA ((AESNI_ALIGN - 1) & ~(CRYPTO_MINALIGN - 1))
45 #define CRYPTO_AES_CTX_SIZE (sizeof(struct crypto_aes_ctx) + AESNI_ALIGN_EXTRA)
46 #define XTS_AES_CTX_SIZE (sizeof(struct aesni_xts_ctx) + AESNI_ALIGN_EXTRA)
47 
48 /* This data is stored at the end of the crypto_tfm struct.
49  * It's a type of per "session" data storage location.
50  * This needs to be 16 byte aligned.
51  */
52 struct aesni_rfc4106_gcm_ctx {
53 	u8 hash_subkey[16] AESNI_ALIGN_ATTR;
54 	struct crypto_aes_ctx aes_key_expanded AESNI_ALIGN_ATTR;
55 	u8 nonce[4];
56 };
57 
58 struct generic_gcmaes_ctx {
59 	u8 hash_subkey[16] AESNI_ALIGN_ATTR;
60 	struct crypto_aes_ctx aes_key_expanded AESNI_ALIGN_ATTR;
61 };
62 
63 struct aesni_xts_ctx {
64 	u8 raw_tweak_ctx[sizeof(struct crypto_aes_ctx)] AESNI_ALIGN_ATTR;
65 	u8 raw_crypt_ctx[sizeof(struct crypto_aes_ctx)] AESNI_ALIGN_ATTR;
66 };
67 
68 #define GCM_BLOCK_LEN 16
69 
70 struct gcm_context_data {
71 	/* init, update and finalize context data */
72 	u8 aad_hash[GCM_BLOCK_LEN];
73 	u64 aad_length;
74 	u64 in_length;
75 	u8 partial_block_enc_key[GCM_BLOCK_LEN];
76 	u8 orig_IV[GCM_BLOCK_LEN];
77 	u8 current_counter[GCM_BLOCK_LEN];
78 	u64 partial_block_len;
79 	u64 unused;
80 	u8 hash_keys[GCM_BLOCK_LEN * 16];
81 };
82 
83 asmlinkage int aesni_set_key(struct crypto_aes_ctx *ctx, const u8 *in_key,
84 			     unsigned int key_len);
85 asmlinkage void aesni_enc(const void *ctx, u8 *out, const u8 *in);
86 asmlinkage void aesni_dec(const void *ctx, u8 *out, const u8 *in);
87 asmlinkage void aesni_ecb_enc(struct crypto_aes_ctx *ctx, u8 *out,
88 			      const u8 *in, unsigned int len);
89 asmlinkage void aesni_ecb_dec(struct crypto_aes_ctx *ctx, u8 *out,
90 			      const u8 *in, unsigned int len);
91 asmlinkage void aesni_cbc_enc(struct crypto_aes_ctx *ctx, u8 *out,
92 			      const u8 *in, unsigned int len, u8 *iv);
93 asmlinkage void aesni_cbc_dec(struct crypto_aes_ctx *ctx, u8 *out,
94 			      const u8 *in, unsigned int len, u8 *iv);
95 asmlinkage void aesni_cts_cbc_enc(struct crypto_aes_ctx *ctx, u8 *out,
96 				  const u8 *in, unsigned int len, u8 *iv);
97 asmlinkage void aesni_cts_cbc_dec(struct crypto_aes_ctx *ctx, u8 *out,
98 				  const u8 *in, unsigned int len, u8 *iv);
99 
100 #define AVX_GEN2_OPTSIZE 640
101 #define AVX_GEN4_OPTSIZE 4096
102 
103 asmlinkage void aesni_xts_encrypt(const struct crypto_aes_ctx *ctx, u8 *out,
104 				  const u8 *in, unsigned int len, u8 *iv);
105 
106 asmlinkage void aesni_xts_decrypt(const struct crypto_aes_ctx *ctx, u8 *out,
107 				  const u8 *in, unsigned int len, u8 *iv);
108 
109 #ifdef CONFIG_X86_64
110 
111 asmlinkage void aesni_ctr_enc(struct crypto_aes_ctx *ctx, u8 *out,
112 			      const u8 *in, unsigned int len, u8 *iv);
113 DEFINE_STATIC_CALL(aesni_ctr_enc_tfm, aesni_ctr_enc);
114 
115 /* Scatter / Gather routines, with args similar to above */
116 asmlinkage void aesni_gcm_init(void *ctx,
117 			       struct gcm_context_data *gdata,
118 			       u8 *iv,
119 			       u8 *hash_subkey, const u8 *aad,
120 			       unsigned long aad_len);
121 asmlinkage void aesni_gcm_enc_update(void *ctx,
122 				     struct gcm_context_data *gdata, u8 *out,
123 				     const u8 *in, unsigned long plaintext_len);
124 asmlinkage void aesni_gcm_dec_update(void *ctx,
125 				     struct gcm_context_data *gdata, u8 *out,
126 				     const u8 *in,
127 				     unsigned long ciphertext_len);
128 asmlinkage void aesni_gcm_finalize(void *ctx,
129 				   struct gcm_context_data *gdata,
130 				   u8 *auth_tag, unsigned long auth_tag_len);
131 
132 asmlinkage void aes_ctr_enc_128_avx_by8(const u8 *in, u8 *iv,
133 		void *keys, u8 *out, unsigned int num_bytes);
134 asmlinkage void aes_ctr_enc_192_avx_by8(const u8 *in, u8 *iv,
135 		void *keys, u8 *out, unsigned int num_bytes);
136 asmlinkage void aes_ctr_enc_256_avx_by8(const u8 *in, u8 *iv,
137 		void *keys, u8 *out, unsigned int num_bytes);
138 
139 
140 asmlinkage void aes_xctr_enc_128_avx_by8(const u8 *in, const u8 *iv,
141 	const void *keys, u8 *out, unsigned int num_bytes,
142 	unsigned int byte_ctr);
143 
144 asmlinkage void aes_xctr_enc_192_avx_by8(const u8 *in, const u8 *iv,
145 	const void *keys, u8 *out, unsigned int num_bytes,
146 	unsigned int byte_ctr);
147 
148 asmlinkage void aes_xctr_enc_256_avx_by8(const u8 *in, const u8 *iv,
149 	const void *keys, u8 *out, unsigned int num_bytes,
150 	unsigned int byte_ctr);
151 
152 /*
153  * asmlinkage void aesni_gcm_init_avx_gen2()
154  * gcm_data *my_ctx_data, context data
155  * u8 *hash_subkey,  the Hash sub key input. Data starts on a 16-byte boundary.
156  */
157 asmlinkage void aesni_gcm_init_avx_gen2(void *my_ctx_data,
158 					struct gcm_context_data *gdata,
159 					u8 *iv,
160 					u8 *hash_subkey,
161 					const u8 *aad,
162 					unsigned long aad_len);
163 
164 asmlinkage void aesni_gcm_enc_update_avx_gen2(void *ctx,
165 				     struct gcm_context_data *gdata, u8 *out,
166 				     const u8 *in, unsigned long plaintext_len);
167 asmlinkage void aesni_gcm_dec_update_avx_gen2(void *ctx,
168 				     struct gcm_context_data *gdata, u8 *out,
169 				     const u8 *in,
170 				     unsigned long ciphertext_len);
171 asmlinkage void aesni_gcm_finalize_avx_gen2(void *ctx,
172 				   struct gcm_context_data *gdata,
173 				   u8 *auth_tag, unsigned long auth_tag_len);
174 
175 /*
176  * asmlinkage void aesni_gcm_init_avx_gen4()
177  * gcm_data *my_ctx_data, context data
178  * u8 *hash_subkey,  the Hash sub key input. Data starts on a 16-byte boundary.
179  */
180 asmlinkage void aesni_gcm_init_avx_gen4(void *my_ctx_data,
181 					struct gcm_context_data *gdata,
182 					u8 *iv,
183 					u8 *hash_subkey,
184 					const u8 *aad,
185 					unsigned long aad_len);
186 
187 asmlinkage void aesni_gcm_enc_update_avx_gen4(void *ctx,
188 				     struct gcm_context_data *gdata, u8 *out,
189 				     const u8 *in, unsigned long plaintext_len);
190 asmlinkage void aesni_gcm_dec_update_avx_gen4(void *ctx,
191 				     struct gcm_context_data *gdata, u8 *out,
192 				     const u8 *in,
193 				     unsigned long ciphertext_len);
194 asmlinkage void aesni_gcm_finalize_avx_gen4(void *ctx,
195 				   struct gcm_context_data *gdata,
196 				   u8 *auth_tag, unsigned long auth_tag_len);
197 
198 static __ro_after_init DEFINE_STATIC_KEY_FALSE(gcm_use_avx);
199 static __ro_after_init DEFINE_STATIC_KEY_FALSE(gcm_use_avx2);
200 
201 static inline struct
202 aesni_rfc4106_gcm_ctx *aesni_rfc4106_gcm_ctx_get(struct crypto_aead *tfm)
203 {
204 	unsigned long align = AESNI_ALIGN;
205 
206 	if (align <= crypto_tfm_ctx_alignment())
207 		align = 1;
208 	return PTR_ALIGN(crypto_aead_ctx(tfm), align);
209 }
210 
211 static inline struct
212 generic_gcmaes_ctx *generic_gcmaes_ctx_get(struct crypto_aead *tfm)
213 {
214 	unsigned long align = AESNI_ALIGN;
215 
216 	if (align <= crypto_tfm_ctx_alignment())
217 		align = 1;
218 	return PTR_ALIGN(crypto_aead_ctx(tfm), align);
219 }
220 #endif
221 
222 static inline struct crypto_aes_ctx *aes_ctx(void *raw_ctx)
223 {
224 	unsigned long addr = (unsigned long)raw_ctx;
225 	unsigned long align = AESNI_ALIGN;
226 
227 	if (align <= crypto_tfm_ctx_alignment())
228 		align = 1;
229 	return (struct crypto_aes_ctx *)ALIGN(addr, align);
230 }
231 
232 static int aes_set_key_common(struct crypto_aes_ctx *ctx,
233 			      const u8 *in_key, unsigned int key_len)
234 {
235 	int err;
236 
237 	if (key_len != AES_KEYSIZE_128 && key_len != AES_KEYSIZE_192 &&
238 	    key_len != AES_KEYSIZE_256)
239 		return -EINVAL;
240 
241 	if (!crypto_simd_usable())
242 		err = aes_expandkey(ctx, in_key, key_len);
243 	else {
244 		kernel_fpu_begin();
245 		err = aesni_set_key(ctx, in_key, key_len);
246 		kernel_fpu_end();
247 	}
248 
249 	return err;
250 }
251 
252 static int aes_set_key(struct crypto_tfm *tfm, const u8 *in_key,
253 		       unsigned int key_len)
254 {
255 	return aes_set_key_common(aes_ctx(crypto_tfm_ctx(tfm)), in_key,
256 				  key_len);
257 }
258 
259 static void aesni_encrypt(struct crypto_tfm *tfm, u8 *dst, const u8 *src)
260 {
261 	struct crypto_aes_ctx *ctx = aes_ctx(crypto_tfm_ctx(tfm));
262 
263 	if (!crypto_simd_usable()) {
264 		aes_encrypt(ctx, dst, src);
265 	} else {
266 		kernel_fpu_begin();
267 		aesni_enc(ctx, dst, src);
268 		kernel_fpu_end();
269 	}
270 }
271 
272 static void aesni_decrypt(struct crypto_tfm *tfm, u8 *dst, const u8 *src)
273 {
274 	struct crypto_aes_ctx *ctx = aes_ctx(crypto_tfm_ctx(tfm));
275 
276 	if (!crypto_simd_usable()) {
277 		aes_decrypt(ctx, dst, src);
278 	} else {
279 		kernel_fpu_begin();
280 		aesni_dec(ctx, dst, src);
281 		kernel_fpu_end();
282 	}
283 }
284 
285 static int aesni_skcipher_setkey(struct crypto_skcipher *tfm, const u8 *key,
286 			         unsigned int len)
287 {
288 	return aes_set_key_common(aes_ctx(crypto_skcipher_ctx(tfm)), key, len);
289 }
290 
291 static int ecb_encrypt(struct skcipher_request *req)
292 {
293 	struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req);
294 	struct crypto_aes_ctx *ctx = aes_ctx(crypto_skcipher_ctx(tfm));
295 	struct skcipher_walk walk;
296 	unsigned int nbytes;
297 	int err;
298 
299 	err = skcipher_walk_virt(&walk, req, false);
300 
301 	while ((nbytes = walk.nbytes)) {
302 		kernel_fpu_begin();
303 		aesni_ecb_enc(ctx, walk.dst.virt.addr, walk.src.virt.addr,
304 			      nbytes & AES_BLOCK_MASK);
305 		kernel_fpu_end();
306 		nbytes &= AES_BLOCK_SIZE - 1;
307 		err = skcipher_walk_done(&walk, nbytes);
308 	}
309 
310 	return err;
311 }
312 
313 static int ecb_decrypt(struct skcipher_request *req)
314 {
315 	struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req);
316 	struct crypto_aes_ctx *ctx = aes_ctx(crypto_skcipher_ctx(tfm));
317 	struct skcipher_walk walk;
318 	unsigned int nbytes;
319 	int err;
320 
321 	err = skcipher_walk_virt(&walk, req, false);
322 
323 	while ((nbytes = walk.nbytes)) {
324 		kernel_fpu_begin();
325 		aesni_ecb_dec(ctx, walk.dst.virt.addr, walk.src.virt.addr,
326 			      nbytes & AES_BLOCK_MASK);
327 		kernel_fpu_end();
328 		nbytes &= AES_BLOCK_SIZE - 1;
329 		err = skcipher_walk_done(&walk, nbytes);
330 	}
331 
332 	return err;
333 }
334 
335 static int cbc_encrypt(struct skcipher_request *req)
336 {
337 	struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req);
338 	struct crypto_aes_ctx *ctx = aes_ctx(crypto_skcipher_ctx(tfm));
339 	struct skcipher_walk walk;
340 	unsigned int nbytes;
341 	int err;
342 
343 	err = skcipher_walk_virt(&walk, req, false);
344 
345 	while ((nbytes = walk.nbytes)) {
346 		kernel_fpu_begin();
347 		aesni_cbc_enc(ctx, walk.dst.virt.addr, walk.src.virt.addr,
348 			      nbytes & AES_BLOCK_MASK, walk.iv);
349 		kernel_fpu_end();
350 		nbytes &= AES_BLOCK_SIZE - 1;
351 		err = skcipher_walk_done(&walk, nbytes);
352 	}
353 
354 	return err;
355 }
356 
357 static int cbc_decrypt(struct skcipher_request *req)
358 {
359 	struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req);
360 	struct crypto_aes_ctx *ctx = aes_ctx(crypto_skcipher_ctx(tfm));
361 	struct skcipher_walk walk;
362 	unsigned int nbytes;
363 	int err;
364 
365 	err = skcipher_walk_virt(&walk, req, false);
366 
367 	while ((nbytes = walk.nbytes)) {
368 		kernel_fpu_begin();
369 		aesni_cbc_dec(ctx, walk.dst.virt.addr, walk.src.virt.addr,
370 			      nbytes & AES_BLOCK_MASK, walk.iv);
371 		kernel_fpu_end();
372 		nbytes &= AES_BLOCK_SIZE - 1;
373 		err = skcipher_walk_done(&walk, nbytes);
374 	}
375 
376 	return err;
377 }
378 
379 static int cts_cbc_encrypt(struct skcipher_request *req)
380 {
381 	struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req);
382 	struct crypto_aes_ctx *ctx = aes_ctx(crypto_skcipher_ctx(tfm));
383 	int cbc_blocks = DIV_ROUND_UP(req->cryptlen, AES_BLOCK_SIZE) - 2;
384 	struct scatterlist *src = req->src, *dst = req->dst;
385 	struct scatterlist sg_src[2], sg_dst[2];
386 	struct skcipher_request subreq;
387 	struct skcipher_walk walk;
388 	int err;
389 
390 	skcipher_request_set_tfm(&subreq, tfm);
391 	skcipher_request_set_callback(&subreq, skcipher_request_flags(req),
392 				      NULL, NULL);
393 
394 	if (req->cryptlen <= AES_BLOCK_SIZE) {
395 		if (req->cryptlen < AES_BLOCK_SIZE)
396 			return -EINVAL;
397 		cbc_blocks = 1;
398 	}
399 
400 	if (cbc_blocks > 0) {
401 		skcipher_request_set_crypt(&subreq, req->src, req->dst,
402 					   cbc_blocks * AES_BLOCK_SIZE,
403 					   req->iv);
404 
405 		err = cbc_encrypt(&subreq);
406 		if (err)
407 			return err;
408 
409 		if (req->cryptlen == AES_BLOCK_SIZE)
410 			return 0;
411 
412 		dst = src = scatterwalk_ffwd(sg_src, req->src, subreq.cryptlen);
413 		if (req->dst != req->src)
414 			dst = scatterwalk_ffwd(sg_dst, req->dst,
415 					       subreq.cryptlen);
416 	}
417 
418 	/* handle ciphertext stealing */
419 	skcipher_request_set_crypt(&subreq, src, dst,
420 				   req->cryptlen - cbc_blocks * AES_BLOCK_SIZE,
421 				   req->iv);
422 
423 	err = skcipher_walk_virt(&walk, &subreq, false);
424 	if (err)
425 		return err;
426 
427 	kernel_fpu_begin();
428 	aesni_cts_cbc_enc(ctx, walk.dst.virt.addr, walk.src.virt.addr,
429 			  walk.nbytes, walk.iv);
430 	kernel_fpu_end();
431 
432 	return skcipher_walk_done(&walk, 0);
433 }
434 
435 static int cts_cbc_decrypt(struct skcipher_request *req)
436 {
437 	struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req);
438 	struct crypto_aes_ctx *ctx = aes_ctx(crypto_skcipher_ctx(tfm));
439 	int cbc_blocks = DIV_ROUND_UP(req->cryptlen, AES_BLOCK_SIZE) - 2;
440 	struct scatterlist *src = req->src, *dst = req->dst;
441 	struct scatterlist sg_src[2], sg_dst[2];
442 	struct skcipher_request subreq;
443 	struct skcipher_walk walk;
444 	int err;
445 
446 	skcipher_request_set_tfm(&subreq, tfm);
447 	skcipher_request_set_callback(&subreq, skcipher_request_flags(req),
448 				      NULL, NULL);
449 
450 	if (req->cryptlen <= AES_BLOCK_SIZE) {
451 		if (req->cryptlen < AES_BLOCK_SIZE)
452 			return -EINVAL;
453 		cbc_blocks = 1;
454 	}
455 
456 	if (cbc_blocks > 0) {
457 		skcipher_request_set_crypt(&subreq, req->src, req->dst,
458 					   cbc_blocks * AES_BLOCK_SIZE,
459 					   req->iv);
460 
461 		err = cbc_decrypt(&subreq);
462 		if (err)
463 			return err;
464 
465 		if (req->cryptlen == AES_BLOCK_SIZE)
466 			return 0;
467 
468 		dst = src = scatterwalk_ffwd(sg_src, req->src, subreq.cryptlen);
469 		if (req->dst != req->src)
470 			dst = scatterwalk_ffwd(sg_dst, req->dst,
471 					       subreq.cryptlen);
472 	}
473 
474 	/* handle ciphertext stealing */
475 	skcipher_request_set_crypt(&subreq, src, dst,
476 				   req->cryptlen - cbc_blocks * AES_BLOCK_SIZE,
477 				   req->iv);
478 
479 	err = skcipher_walk_virt(&walk, &subreq, false);
480 	if (err)
481 		return err;
482 
483 	kernel_fpu_begin();
484 	aesni_cts_cbc_dec(ctx, walk.dst.virt.addr, walk.src.virt.addr,
485 			  walk.nbytes, walk.iv);
486 	kernel_fpu_end();
487 
488 	return skcipher_walk_done(&walk, 0);
489 }
490 
491 #ifdef CONFIG_X86_64
492 static void aesni_ctr_enc_avx_tfm(struct crypto_aes_ctx *ctx, u8 *out,
493 			      const u8 *in, unsigned int len, u8 *iv)
494 {
495 	/*
496 	 * based on key length, override with the by8 version
497 	 * of ctr mode encryption/decryption for improved performance
498 	 * aes_set_key_common() ensures that key length is one of
499 	 * {128,192,256}
500 	 */
501 	if (ctx->key_length == AES_KEYSIZE_128)
502 		aes_ctr_enc_128_avx_by8(in, iv, (void *)ctx, out, len);
503 	else if (ctx->key_length == AES_KEYSIZE_192)
504 		aes_ctr_enc_192_avx_by8(in, iv, (void *)ctx, out, len);
505 	else
506 		aes_ctr_enc_256_avx_by8(in, iv, (void *)ctx, out, len);
507 }
508 
509 static int ctr_crypt(struct skcipher_request *req)
510 {
511 	struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req);
512 	struct crypto_aes_ctx *ctx = aes_ctx(crypto_skcipher_ctx(tfm));
513 	u8 keystream[AES_BLOCK_SIZE];
514 	struct skcipher_walk walk;
515 	unsigned int nbytes;
516 	int err;
517 
518 	err = skcipher_walk_virt(&walk, req, false);
519 
520 	while ((nbytes = walk.nbytes) > 0) {
521 		kernel_fpu_begin();
522 		if (nbytes & AES_BLOCK_MASK)
523 			static_call(aesni_ctr_enc_tfm)(ctx, walk.dst.virt.addr,
524 						       walk.src.virt.addr,
525 						       nbytes & AES_BLOCK_MASK,
526 						       walk.iv);
527 		nbytes &= ~AES_BLOCK_MASK;
528 
529 		if (walk.nbytes == walk.total && nbytes > 0) {
530 			aesni_enc(ctx, keystream, walk.iv);
531 			crypto_xor_cpy(walk.dst.virt.addr + walk.nbytes - nbytes,
532 				       walk.src.virt.addr + walk.nbytes - nbytes,
533 				       keystream, nbytes);
534 			crypto_inc(walk.iv, AES_BLOCK_SIZE);
535 			nbytes = 0;
536 		}
537 		kernel_fpu_end();
538 		err = skcipher_walk_done(&walk, nbytes);
539 	}
540 	return err;
541 }
542 
543 static void aesni_xctr_enc_avx_tfm(struct crypto_aes_ctx *ctx, u8 *out,
544 				   const u8 *in, unsigned int len, u8 *iv,
545 				   unsigned int byte_ctr)
546 {
547 	if (ctx->key_length == AES_KEYSIZE_128)
548 		aes_xctr_enc_128_avx_by8(in, iv, (void *)ctx, out, len,
549 					 byte_ctr);
550 	else if (ctx->key_length == AES_KEYSIZE_192)
551 		aes_xctr_enc_192_avx_by8(in, iv, (void *)ctx, out, len,
552 					 byte_ctr);
553 	else
554 		aes_xctr_enc_256_avx_by8(in, iv, (void *)ctx, out, len,
555 					 byte_ctr);
556 }
557 
558 static int xctr_crypt(struct skcipher_request *req)
559 {
560 	struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req);
561 	struct crypto_aes_ctx *ctx = aes_ctx(crypto_skcipher_ctx(tfm));
562 	u8 keystream[AES_BLOCK_SIZE];
563 	struct skcipher_walk walk;
564 	unsigned int nbytes;
565 	unsigned int byte_ctr = 0;
566 	int err;
567 	__le32 block[AES_BLOCK_SIZE / sizeof(__le32)];
568 
569 	err = skcipher_walk_virt(&walk, req, false);
570 
571 	while ((nbytes = walk.nbytes) > 0) {
572 		kernel_fpu_begin();
573 		if (nbytes & AES_BLOCK_MASK)
574 			aesni_xctr_enc_avx_tfm(ctx, walk.dst.virt.addr,
575 				walk.src.virt.addr, nbytes & AES_BLOCK_MASK,
576 				walk.iv, byte_ctr);
577 		nbytes &= ~AES_BLOCK_MASK;
578 		byte_ctr += walk.nbytes - nbytes;
579 
580 		if (walk.nbytes == walk.total && nbytes > 0) {
581 			memcpy(block, walk.iv, AES_BLOCK_SIZE);
582 			block[0] ^= cpu_to_le32(1 + byte_ctr / AES_BLOCK_SIZE);
583 			aesni_enc(ctx, keystream, (u8 *)block);
584 			crypto_xor_cpy(walk.dst.virt.addr + walk.nbytes -
585 				       nbytes, walk.src.virt.addr + walk.nbytes
586 				       - nbytes, keystream, nbytes);
587 			byte_ctr += nbytes;
588 			nbytes = 0;
589 		}
590 		kernel_fpu_end();
591 		err = skcipher_walk_done(&walk, nbytes);
592 	}
593 	return err;
594 }
595 
596 static int
597 rfc4106_set_hash_subkey(u8 *hash_subkey, const u8 *key, unsigned int key_len)
598 {
599 	struct crypto_aes_ctx ctx;
600 	int ret;
601 
602 	ret = aes_expandkey(&ctx, key, key_len);
603 	if (ret)
604 		return ret;
605 
606 	/* Clear the data in the hash sub key container to zero.*/
607 	/* We want to cipher all zeros to create the hash sub key. */
608 	memset(hash_subkey, 0, RFC4106_HASH_SUBKEY_SIZE);
609 
610 	aes_encrypt(&ctx, hash_subkey, hash_subkey);
611 
612 	memzero_explicit(&ctx, sizeof(ctx));
613 	return 0;
614 }
615 
616 static int common_rfc4106_set_key(struct crypto_aead *aead, const u8 *key,
617 				  unsigned int key_len)
618 {
619 	struct aesni_rfc4106_gcm_ctx *ctx = aesni_rfc4106_gcm_ctx_get(aead);
620 
621 	if (key_len < 4)
622 		return -EINVAL;
623 
624 	/*Account for 4 byte nonce at the end.*/
625 	key_len -= 4;
626 
627 	memcpy(ctx->nonce, key + key_len, sizeof(ctx->nonce));
628 
629 	return aes_set_key_common(&ctx->aes_key_expanded, key, key_len) ?:
630 	       rfc4106_set_hash_subkey(ctx->hash_subkey, key, key_len);
631 }
632 
633 /* This is the Integrity Check Value (aka the authentication tag) length and can
634  * be 8, 12 or 16 bytes long. */
635 static int common_rfc4106_set_authsize(struct crypto_aead *aead,
636 				       unsigned int authsize)
637 {
638 	switch (authsize) {
639 	case 8:
640 	case 12:
641 	case 16:
642 		break;
643 	default:
644 		return -EINVAL;
645 	}
646 
647 	return 0;
648 }
649 
650 static int generic_gcmaes_set_authsize(struct crypto_aead *tfm,
651 				       unsigned int authsize)
652 {
653 	switch (authsize) {
654 	case 4:
655 	case 8:
656 	case 12:
657 	case 13:
658 	case 14:
659 	case 15:
660 	case 16:
661 		break;
662 	default:
663 		return -EINVAL;
664 	}
665 
666 	return 0;
667 }
668 
669 static int gcmaes_crypt_by_sg(bool enc, struct aead_request *req,
670 			      unsigned int assoclen, u8 *hash_subkey,
671 			      u8 *iv, void *aes_ctx, u8 *auth_tag,
672 			      unsigned long auth_tag_len)
673 {
674 	u8 databuf[sizeof(struct gcm_context_data) + (AESNI_ALIGN - 8)] __aligned(8);
675 	struct gcm_context_data *data = PTR_ALIGN((void *)databuf, AESNI_ALIGN);
676 	unsigned long left = req->cryptlen;
677 	struct scatter_walk assoc_sg_walk;
678 	struct skcipher_walk walk;
679 	bool do_avx, do_avx2;
680 	u8 *assocmem = NULL;
681 	u8 *assoc;
682 	int err;
683 
684 	if (!enc)
685 		left -= auth_tag_len;
686 
687 	do_avx = (left >= AVX_GEN2_OPTSIZE);
688 	do_avx2 = (left >= AVX_GEN4_OPTSIZE);
689 
690 	/* Linearize assoc, if not already linear */
691 	if (req->src->length >= assoclen && req->src->length) {
692 		scatterwalk_start(&assoc_sg_walk, req->src);
693 		assoc = scatterwalk_map(&assoc_sg_walk);
694 	} else {
695 		gfp_t flags = (req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP) ?
696 			      GFP_KERNEL : GFP_ATOMIC;
697 
698 		/* assoc can be any length, so must be on heap */
699 		assocmem = kmalloc(assoclen, flags);
700 		if (unlikely(!assocmem))
701 			return -ENOMEM;
702 		assoc = assocmem;
703 
704 		scatterwalk_map_and_copy(assoc, req->src, 0, assoclen, 0);
705 	}
706 
707 	kernel_fpu_begin();
708 	if (static_branch_likely(&gcm_use_avx2) && do_avx2)
709 		aesni_gcm_init_avx_gen4(aes_ctx, data, iv, hash_subkey, assoc,
710 					assoclen);
711 	else if (static_branch_likely(&gcm_use_avx) && do_avx)
712 		aesni_gcm_init_avx_gen2(aes_ctx, data, iv, hash_subkey, assoc,
713 					assoclen);
714 	else
715 		aesni_gcm_init(aes_ctx, data, iv, hash_subkey, assoc, assoclen);
716 	kernel_fpu_end();
717 
718 	if (!assocmem)
719 		scatterwalk_unmap(assoc);
720 	else
721 		kfree(assocmem);
722 
723 	err = enc ? skcipher_walk_aead_encrypt(&walk, req, false)
724 		  : skcipher_walk_aead_decrypt(&walk, req, false);
725 
726 	while (walk.nbytes > 0) {
727 		kernel_fpu_begin();
728 		if (static_branch_likely(&gcm_use_avx2) && do_avx2) {
729 			if (enc)
730 				aesni_gcm_enc_update_avx_gen4(aes_ctx, data,
731 							      walk.dst.virt.addr,
732 							      walk.src.virt.addr,
733 							      walk.nbytes);
734 			else
735 				aesni_gcm_dec_update_avx_gen4(aes_ctx, data,
736 							      walk.dst.virt.addr,
737 							      walk.src.virt.addr,
738 							      walk.nbytes);
739 		} else if (static_branch_likely(&gcm_use_avx) && do_avx) {
740 			if (enc)
741 				aesni_gcm_enc_update_avx_gen2(aes_ctx, data,
742 							      walk.dst.virt.addr,
743 							      walk.src.virt.addr,
744 							      walk.nbytes);
745 			else
746 				aesni_gcm_dec_update_avx_gen2(aes_ctx, data,
747 							      walk.dst.virt.addr,
748 							      walk.src.virt.addr,
749 							      walk.nbytes);
750 		} else if (enc) {
751 			aesni_gcm_enc_update(aes_ctx, data, walk.dst.virt.addr,
752 					     walk.src.virt.addr, walk.nbytes);
753 		} else {
754 			aesni_gcm_dec_update(aes_ctx, data, walk.dst.virt.addr,
755 					     walk.src.virt.addr, walk.nbytes);
756 		}
757 		kernel_fpu_end();
758 
759 		err = skcipher_walk_done(&walk, 0);
760 	}
761 
762 	if (err)
763 		return err;
764 
765 	kernel_fpu_begin();
766 	if (static_branch_likely(&gcm_use_avx2) && do_avx2)
767 		aesni_gcm_finalize_avx_gen4(aes_ctx, data, auth_tag,
768 					    auth_tag_len);
769 	else if (static_branch_likely(&gcm_use_avx) && do_avx)
770 		aesni_gcm_finalize_avx_gen2(aes_ctx, data, auth_tag,
771 					    auth_tag_len);
772 	else
773 		aesni_gcm_finalize(aes_ctx, data, auth_tag, auth_tag_len);
774 	kernel_fpu_end();
775 
776 	return 0;
777 }
778 
779 static int gcmaes_encrypt(struct aead_request *req, unsigned int assoclen,
780 			  u8 *hash_subkey, u8 *iv, void *aes_ctx)
781 {
782 	struct crypto_aead *tfm = crypto_aead_reqtfm(req);
783 	unsigned long auth_tag_len = crypto_aead_authsize(tfm);
784 	u8 auth_tag[16];
785 	int err;
786 
787 	err = gcmaes_crypt_by_sg(true, req, assoclen, hash_subkey, iv, aes_ctx,
788 				 auth_tag, auth_tag_len);
789 	if (err)
790 		return err;
791 
792 	scatterwalk_map_and_copy(auth_tag, req->dst,
793 				 req->assoclen + req->cryptlen,
794 				 auth_tag_len, 1);
795 	return 0;
796 }
797 
798 static int gcmaes_decrypt(struct aead_request *req, unsigned int assoclen,
799 			  u8 *hash_subkey, u8 *iv, void *aes_ctx)
800 {
801 	struct crypto_aead *tfm = crypto_aead_reqtfm(req);
802 	unsigned long auth_tag_len = crypto_aead_authsize(tfm);
803 	u8 auth_tag_msg[16];
804 	u8 auth_tag[16];
805 	int err;
806 
807 	err = gcmaes_crypt_by_sg(false, req, assoclen, hash_subkey, iv, aes_ctx,
808 				 auth_tag, auth_tag_len);
809 	if (err)
810 		return err;
811 
812 	/* Copy out original auth_tag */
813 	scatterwalk_map_and_copy(auth_tag_msg, req->src,
814 				 req->assoclen + req->cryptlen - auth_tag_len,
815 				 auth_tag_len, 0);
816 
817 	/* Compare generated tag with passed in tag. */
818 	if (crypto_memneq(auth_tag_msg, auth_tag, auth_tag_len)) {
819 		memzero_explicit(auth_tag, sizeof(auth_tag));
820 		return -EBADMSG;
821 	}
822 	return 0;
823 }
824 
825 static int helper_rfc4106_encrypt(struct aead_request *req)
826 {
827 	struct crypto_aead *tfm = crypto_aead_reqtfm(req);
828 	struct aesni_rfc4106_gcm_ctx *ctx = aesni_rfc4106_gcm_ctx_get(tfm);
829 	void *aes_ctx = &(ctx->aes_key_expanded);
830 	u8 ivbuf[16 + (AESNI_ALIGN - 8)] __aligned(8);
831 	u8 *iv = PTR_ALIGN(&ivbuf[0], AESNI_ALIGN);
832 	unsigned int i;
833 	__be32 counter = cpu_to_be32(1);
834 
835 	/* Assuming we are supporting rfc4106 64-bit extended */
836 	/* sequence numbers We need to have the AAD length equal */
837 	/* to 16 or 20 bytes */
838 	if (unlikely(req->assoclen != 16 && req->assoclen != 20))
839 		return -EINVAL;
840 
841 	/* IV below built */
842 	for (i = 0; i < 4; i++)
843 		*(iv+i) = ctx->nonce[i];
844 	for (i = 0; i < 8; i++)
845 		*(iv+4+i) = req->iv[i];
846 	*((__be32 *)(iv+12)) = counter;
847 
848 	return gcmaes_encrypt(req, req->assoclen - 8, ctx->hash_subkey, iv,
849 			      aes_ctx);
850 }
851 
852 static int helper_rfc4106_decrypt(struct aead_request *req)
853 {
854 	__be32 counter = cpu_to_be32(1);
855 	struct crypto_aead *tfm = crypto_aead_reqtfm(req);
856 	struct aesni_rfc4106_gcm_ctx *ctx = aesni_rfc4106_gcm_ctx_get(tfm);
857 	void *aes_ctx = &(ctx->aes_key_expanded);
858 	u8 ivbuf[16 + (AESNI_ALIGN - 8)] __aligned(8);
859 	u8 *iv = PTR_ALIGN(&ivbuf[0], AESNI_ALIGN);
860 	unsigned int i;
861 
862 	if (unlikely(req->assoclen != 16 && req->assoclen != 20))
863 		return -EINVAL;
864 
865 	/* Assuming we are supporting rfc4106 64-bit extended */
866 	/* sequence numbers We need to have the AAD length */
867 	/* equal to 16 or 20 bytes */
868 
869 	/* IV below built */
870 	for (i = 0; i < 4; i++)
871 		*(iv+i) = ctx->nonce[i];
872 	for (i = 0; i < 8; i++)
873 		*(iv+4+i) = req->iv[i];
874 	*((__be32 *)(iv+12)) = counter;
875 
876 	return gcmaes_decrypt(req, req->assoclen - 8, ctx->hash_subkey, iv,
877 			      aes_ctx);
878 }
879 #endif
880 
881 static int xts_aesni_setkey(struct crypto_skcipher *tfm, const u8 *key,
882 			    unsigned int keylen)
883 {
884 	struct aesni_xts_ctx *ctx = crypto_skcipher_ctx(tfm);
885 	int err;
886 
887 	err = xts_verify_key(tfm, key, keylen);
888 	if (err)
889 		return err;
890 
891 	keylen /= 2;
892 
893 	/* first half of xts-key is for crypt */
894 	err = aes_set_key_common(aes_ctx(ctx->raw_crypt_ctx), key, keylen);
895 	if (err)
896 		return err;
897 
898 	/* second half of xts-key is for tweak */
899 	return aes_set_key_common(aes_ctx(ctx->raw_tweak_ctx), key + keylen,
900 				  keylen);
901 }
902 
903 static int xts_crypt(struct skcipher_request *req, bool encrypt)
904 {
905 	struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req);
906 	struct aesni_xts_ctx *ctx = crypto_skcipher_ctx(tfm);
907 	int tail = req->cryptlen % AES_BLOCK_SIZE;
908 	struct skcipher_request subreq;
909 	struct skcipher_walk walk;
910 	int err;
911 
912 	if (req->cryptlen < AES_BLOCK_SIZE)
913 		return -EINVAL;
914 
915 	err = skcipher_walk_virt(&walk, req, false);
916 	if (!walk.nbytes)
917 		return err;
918 
919 	if (unlikely(tail > 0 && walk.nbytes < walk.total)) {
920 		int blocks = DIV_ROUND_UP(req->cryptlen, AES_BLOCK_SIZE) - 2;
921 
922 		skcipher_walk_abort(&walk);
923 
924 		skcipher_request_set_tfm(&subreq, tfm);
925 		skcipher_request_set_callback(&subreq,
926 					      skcipher_request_flags(req),
927 					      NULL, NULL);
928 		skcipher_request_set_crypt(&subreq, req->src, req->dst,
929 					   blocks * AES_BLOCK_SIZE, req->iv);
930 		req = &subreq;
931 
932 		err = skcipher_walk_virt(&walk, req, false);
933 		if (!walk.nbytes)
934 			return err;
935 	} else {
936 		tail = 0;
937 	}
938 
939 	kernel_fpu_begin();
940 
941 	/* calculate first value of T */
942 	aesni_enc(aes_ctx(ctx->raw_tweak_ctx), walk.iv, walk.iv);
943 
944 	while (walk.nbytes > 0) {
945 		int nbytes = walk.nbytes;
946 
947 		if (nbytes < walk.total)
948 			nbytes &= ~(AES_BLOCK_SIZE - 1);
949 
950 		if (encrypt)
951 			aesni_xts_encrypt(aes_ctx(ctx->raw_crypt_ctx),
952 					  walk.dst.virt.addr, walk.src.virt.addr,
953 					  nbytes, walk.iv);
954 		else
955 			aesni_xts_decrypt(aes_ctx(ctx->raw_crypt_ctx),
956 					  walk.dst.virt.addr, walk.src.virt.addr,
957 					  nbytes, walk.iv);
958 		kernel_fpu_end();
959 
960 		err = skcipher_walk_done(&walk, walk.nbytes - nbytes);
961 
962 		if (walk.nbytes > 0)
963 			kernel_fpu_begin();
964 	}
965 
966 	if (unlikely(tail > 0 && !err)) {
967 		struct scatterlist sg_src[2], sg_dst[2];
968 		struct scatterlist *src, *dst;
969 
970 		dst = src = scatterwalk_ffwd(sg_src, req->src, req->cryptlen);
971 		if (req->dst != req->src)
972 			dst = scatterwalk_ffwd(sg_dst, req->dst, req->cryptlen);
973 
974 		skcipher_request_set_crypt(req, src, dst, AES_BLOCK_SIZE + tail,
975 					   req->iv);
976 
977 		err = skcipher_walk_virt(&walk, &subreq, false);
978 		if (err)
979 			return err;
980 
981 		kernel_fpu_begin();
982 		if (encrypt)
983 			aesni_xts_encrypt(aes_ctx(ctx->raw_crypt_ctx),
984 					  walk.dst.virt.addr, walk.src.virt.addr,
985 					  walk.nbytes, walk.iv);
986 		else
987 			aesni_xts_decrypt(aes_ctx(ctx->raw_crypt_ctx),
988 					  walk.dst.virt.addr, walk.src.virt.addr,
989 					  walk.nbytes, walk.iv);
990 		kernel_fpu_end();
991 
992 		err = skcipher_walk_done(&walk, 0);
993 	}
994 	return err;
995 }
996 
997 static int xts_encrypt(struct skcipher_request *req)
998 {
999 	return xts_crypt(req, true);
1000 }
1001 
1002 static int xts_decrypt(struct skcipher_request *req)
1003 {
1004 	return xts_crypt(req, false);
1005 }
1006 
1007 static struct crypto_alg aesni_cipher_alg = {
1008 	.cra_name		= "aes",
1009 	.cra_driver_name	= "aes-aesni",
1010 	.cra_priority		= 300,
1011 	.cra_flags		= CRYPTO_ALG_TYPE_CIPHER,
1012 	.cra_blocksize		= AES_BLOCK_SIZE,
1013 	.cra_ctxsize		= CRYPTO_AES_CTX_SIZE,
1014 	.cra_module		= THIS_MODULE,
1015 	.cra_u	= {
1016 		.cipher	= {
1017 			.cia_min_keysize	= AES_MIN_KEY_SIZE,
1018 			.cia_max_keysize	= AES_MAX_KEY_SIZE,
1019 			.cia_setkey		= aes_set_key,
1020 			.cia_encrypt		= aesni_encrypt,
1021 			.cia_decrypt		= aesni_decrypt
1022 		}
1023 	}
1024 };
1025 
1026 static struct skcipher_alg aesni_skciphers[] = {
1027 	{
1028 		.base = {
1029 			.cra_name		= "__ecb(aes)",
1030 			.cra_driver_name	= "__ecb-aes-aesni",
1031 			.cra_priority		= 400,
1032 			.cra_flags		= CRYPTO_ALG_INTERNAL,
1033 			.cra_blocksize		= AES_BLOCK_SIZE,
1034 			.cra_ctxsize		= CRYPTO_AES_CTX_SIZE,
1035 			.cra_module		= THIS_MODULE,
1036 		},
1037 		.min_keysize	= AES_MIN_KEY_SIZE,
1038 		.max_keysize	= AES_MAX_KEY_SIZE,
1039 		.setkey		= aesni_skcipher_setkey,
1040 		.encrypt	= ecb_encrypt,
1041 		.decrypt	= ecb_decrypt,
1042 	}, {
1043 		.base = {
1044 			.cra_name		= "__cbc(aes)",
1045 			.cra_driver_name	= "__cbc-aes-aesni",
1046 			.cra_priority		= 400,
1047 			.cra_flags		= CRYPTO_ALG_INTERNAL,
1048 			.cra_blocksize		= AES_BLOCK_SIZE,
1049 			.cra_ctxsize		= CRYPTO_AES_CTX_SIZE,
1050 			.cra_module		= THIS_MODULE,
1051 		},
1052 		.min_keysize	= AES_MIN_KEY_SIZE,
1053 		.max_keysize	= AES_MAX_KEY_SIZE,
1054 		.ivsize		= AES_BLOCK_SIZE,
1055 		.setkey		= aesni_skcipher_setkey,
1056 		.encrypt	= cbc_encrypt,
1057 		.decrypt	= cbc_decrypt,
1058 	}, {
1059 		.base = {
1060 			.cra_name		= "__cts(cbc(aes))",
1061 			.cra_driver_name	= "__cts-cbc-aes-aesni",
1062 			.cra_priority		= 400,
1063 			.cra_flags		= CRYPTO_ALG_INTERNAL,
1064 			.cra_blocksize		= AES_BLOCK_SIZE,
1065 			.cra_ctxsize		= CRYPTO_AES_CTX_SIZE,
1066 			.cra_module		= THIS_MODULE,
1067 		},
1068 		.min_keysize	= AES_MIN_KEY_SIZE,
1069 		.max_keysize	= AES_MAX_KEY_SIZE,
1070 		.ivsize		= AES_BLOCK_SIZE,
1071 		.walksize	= 2 * AES_BLOCK_SIZE,
1072 		.setkey		= aesni_skcipher_setkey,
1073 		.encrypt	= cts_cbc_encrypt,
1074 		.decrypt	= cts_cbc_decrypt,
1075 #ifdef CONFIG_X86_64
1076 	}, {
1077 		.base = {
1078 			.cra_name		= "__ctr(aes)",
1079 			.cra_driver_name	= "__ctr-aes-aesni",
1080 			.cra_priority		= 400,
1081 			.cra_flags		= CRYPTO_ALG_INTERNAL,
1082 			.cra_blocksize		= 1,
1083 			.cra_ctxsize		= CRYPTO_AES_CTX_SIZE,
1084 			.cra_module		= THIS_MODULE,
1085 		},
1086 		.min_keysize	= AES_MIN_KEY_SIZE,
1087 		.max_keysize	= AES_MAX_KEY_SIZE,
1088 		.ivsize		= AES_BLOCK_SIZE,
1089 		.chunksize	= AES_BLOCK_SIZE,
1090 		.setkey		= aesni_skcipher_setkey,
1091 		.encrypt	= ctr_crypt,
1092 		.decrypt	= ctr_crypt,
1093 #endif
1094 	}, {
1095 		.base = {
1096 			.cra_name		= "__xts(aes)",
1097 			.cra_driver_name	= "__xts-aes-aesni",
1098 			.cra_priority		= 401,
1099 			.cra_flags		= CRYPTO_ALG_INTERNAL,
1100 			.cra_blocksize		= AES_BLOCK_SIZE,
1101 			.cra_ctxsize		= XTS_AES_CTX_SIZE,
1102 			.cra_module		= THIS_MODULE,
1103 		},
1104 		.min_keysize	= 2 * AES_MIN_KEY_SIZE,
1105 		.max_keysize	= 2 * AES_MAX_KEY_SIZE,
1106 		.ivsize		= AES_BLOCK_SIZE,
1107 		.walksize	= 2 * AES_BLOCK_SIZE,
1108 		.setkey		= xts_aesni_setkey,
1109 		.encrypt	= xts_encrypt,
1110 		.decrypt	= xts_decrypt,
1111 	}
1112 };
1113 
1114 static
1115 struct simd_skcipher_alg *aesni_simd_skciphers[ARRAY_SIZE(aesni_skciphers)];
1116 
1117 #ifdef CONFIG_X86_64
1118 /*
1119  * XCTR does not have a non-AVX implementation, so it must be enabled
1120  * conditionally.
1121  */
1122 static struct skcipher_alg aesni_xctr = {
1123 	.base = {
1124 		.cra_name		= "__xctr(aes)",
1125 		.cra_driver_name	= "__xctr-aes-aesni",
1126 		.cra_priority		= 400,
1127 		.cra_flags		= CRYPTO_ALG_INTERNAL,
1128 		.cra_blocksize		= 1,
1129 		.cra_ctxsize		= CRYPTO_AES_CTX_SIZE,
1130 		.cra_module		= THIS_MODULE,
1131 	},
1132 	.min_keysize	= AES_MIN_KEY_SIZE,
1133 	.max_keysize	= AES_MAX_KEY_SIZE,
1134 	.ivsize		= AES_BLOCK_SIZE,
1135 	.chunksize	= AES_BLOCK_SIZE,
1136 	.setkey		= aesni_skcipher_setkey,
1137 	.encrypt	= xctr_crypt,
1138 	.decrypt	= xctr_crypt,
1139 };
1140 
1141 static struct simd_skcipher_alg *aesni_simd_xctr;
1142 #endif /* CONFIG_X86_64 */
1143 
1144 #ifdef CONFIG_X86_64
1145 static int generic_gcmaes_set_key(struct crypto_aead *aead, const u8 *key,
1146 				  unsigned int key_len)
1147 {
1148 	struct generic_gcmaes_ctx *ctx = generic_gcmaes_ctx_get(aead);
1149 
1150 	return aes_set_key_common(&ctx->aes_key_expanded, key, key_len) ?:
1151 	       rfc4106_set_hash_subkey(ctx->hash_subkey, key, key_len);
1152 }
1153 
1154 static int generic_gcmaes_encrypt(struct aead_request *req)
1155 {
1156 	struct crypto_aead *tfm = crypto_aead_reqtfm(req);
1157 	struct generic_gcmaes_ctx *ctx = generic_gcmaes_ctx_get(tfm);
1158 	void *aes_ctx = &(ctx->aes_key_expanded);
1159 	u8 ivbuf[16 + (AESNI_ALIGN - 8)] __aligned(8);
1160 	u8 *iv = PTR_ALIGN(&ivbuf[0], AESNI_ALIGN);
1161 	__be32 counter = cpu_to_be32(1);
1162 
1163 	memcpy(iv, req->iv, 12);
1164 	*((__be32 *)(iv+12)) = counter;
1165 
1166 	return gcmaes_encrypt(req, req->assoclen, ctx->hash_subkey, iv,
1167 			      aes_ctx);
1168 }
1169 
1170 static int generic_gcmaes_decrypt(struct aead_request *req)
1171 {
1172 	__be32 counter = cpu_to_be32(1);
1173 	struct crypto_aead *tfm = crypto_aead_reqtfm(req);
1174 	struct generic_gcmaes_ctx *ctx = generic_gcmaes_ctx_get(tfm);
1175 	void *aes_ctx = &(ctx->aes_key_expanded);
1176 	u8 ivbuf[16 + (AESNI_ALIGN - 8)] __aligned(8);
1177 	u8 *iv = PTR_ALIGN(&ivbuf[0], AESNI_ALIGN);
1178 
1179 	memcpy(iv, req->iv, 12);
1180 	*((__be32 *)(iv+12)) = counter;
1181 
1182 	return gcmaes_decrypt(req, req->assoclen, ctx->hash_subkey, iv,
1183 			      aes_ctx);
1184 }
1185 
1186 static struct aead_alg aesni_aeads[] = { {
1187 	.setkey			= common_rfc4106_set_key,
1188 	.setauthsize		= common_rfc4106_set_authsize,
1189 	.encrypt		= helper_rfc4106_encrypt,
1190 	.decrypt		= helper_rfc4106_decrypt,
1191 	.ivsize			= GCM_RFC4106_IV_SIZE,
1192 	.maxauthsize		= 16,
1193 	.base = {
1194 		.cra_name		= "__rfc4106(gcm(aes))",
1195 		.cra_driver_name	= "__rfc4106-gcm-aesni",
1196 		.cra_priority		= 400,
1197 		.cra_flags		= CRYPTO_ALG_INTERNAL,
1198 		.cra_blocksize		= 1,
1199 		.cra_ctxsize		= sizeof(struct aesni_rfc4106_gcm_ctx),
1200 		.cra_alignmask		= 0,
1201 		.cra_module		= THIS_MODULE,
1202 	},
1203 }, {
1204 	.setkey			= generic_gcmaes_set_key,
1205 	.setauthsize		= generic_gcmaes_set_authsize,
1206 	.encrypt		= generic_gcmaes_encrypt,
1207 	.decrypt		= generic_gcmaes_decrypt,
1208 	.ivsize			= GCM_AES_IV_SIZE,
1209 	.maxauthsize		= 16,
1210 	.base = {
1211 		.cra_name		= "__gcm(aes)",
1212 		.cra_driver_name	= "__generic-gcm-aesni",
1213 		.cra_priority		= 400,
1214 		.cra_flags		= CRYPTO_ALG_INTERNAL,
1215 		.cra_blocksize		= 1,
1216 		.cra_ctxsize		= sizeof(struct generic_gcmaes_ctx),
1217 		.cra_alignmask		= 0,
1218 		.cra_module		= THIS_MODULE,
1219 	},
1220 } };
1221 #else
1222 static struct aead_alg aesni_aeads[0];
1223 #endif
1224 
1225 static struct simd_aead_alg *aesni_simd_aeads[ARRAY_SIZE(aesni_aeads)];
1226 
1227 static const struct x86_cpu_id aesni_cpu_id[] = {
1228 	X86_MATCH_FEATURE(X86_FEATURE_AES, NULL),
1229 	{}
1230 };
1231 MODULE_DEVICE_TABLE(x86cpu, aesni_cpu_id);
1232 
1233 static int __init aesni_init(void)
1234 {
1235 	int err;
1236 
1237 	if (!x86_match_cpu(aesni_cpu_id))
1238 		return -ENODEV;
1239 #ifdef CONFIG_X86_64
1240 	if (boot_cpu_has(X86_FEATURE_AVX2)) {
1241 		pr_info("AVX2 version of gcm_enc/dec engaged.\n");
1242 		static_branch_enable(&gcm_use_avx);
1243 		static_branch_enable(&gcm_use_avx2);
1244 	} else
1245 	if (boot_cpu_has(X86_FEATURE_AVX)) {
1246 		pr_info("AVX version of gcm_enc/dec engaged.\n");
1247 		static_branch_enable(&gcm_use_avx);
1248 	} else {
1249 		pr_info("SSE version of gcm_enc/dec engaged.\n");
1250 	}
1251 	if (boot_cpu_has(X86_FEATURE_AVX)) {
1252 		/* optimize performance of ctr mode encryption transform */
1253 		static_call_update(aesni_ctr_enc_tfm, aesni_ctr_enc_avx_tfm);
1254 		pr_info("AES CTR mode by8 optimization enabled\n");
1255 	}
1256 #endif /* CONFIG_X86_64 */
1257 
1258 	err = crypto_register_alg(&aesni_cipher_alg);
1259 	if (err)
1260 		return err;
1261 
1262 	err = simd_register_skciphers_compat(aesni_skciphers,
1263 					     ARRAY_SIZE(aesni_skciphers),
1264 					     aesni_simd_skciphers);
1265 	if (err)
1266 		goto unregister_cipher;
1267 
1268 	err = simd_register_aeads_compat(aesni_aeads, ARRAY_SIZE(aesni_aeads),
1269 					 aesni_simd_aeads);
1270 	if (err)
1271 		goto unregister_skciphers;
1272 
1273 #ifdef CONFIG_X86_64
1274 	if (boot_cpu_has(X86_FEATURE_AVX))
1275 		err = simd_register_skciphers_compat(&aesni_xctr, 1,
1276 						     &aesni_simd_xctr);
1277 	if (err)
1278 		goto unregister_aeads;
1279 #endif /* CONFIG_X86_64 */
1280 
1281 	return 0;
1282 
1283 #ifdef CONFIG_X86_64
1284 unregister_aeads:
1285 	simd_unregister_aeads(aesni_aeads, ARRAY_SIZE(aesni_aeads),
1286 				aesni_simd_aeads);
1287 #endif /* CONFIG_X86_64 */
1288 
1289 unregister_skciphers:
1290 	simd_unregister_skciphers(aesni_skciphers, ARRAY_SIZE(aesni_skciphers),
1291 				  aesni_simd_skciphers);
1292 unregister_cipher:
1293 	crypto_unregister_alg(&aesni_cipher_alg);
1294 	return err;
1295 }
1296 
1297 static void __exit aesni_exit(void)
1298 {
1299 	simd_unregister_aeads(aesni_aeads, ARRAY_SIZE(aesni_aeads),
1300 			      aesni_simd_aeads);
1301 	simd_unregister_skciphers(aesni_skciphers, ARRAY_SIZE(aesni_skciphers),
1302 				  aesni_simd_skciphers);
1303 	crypto_unregister_alg(&aesni_cipher_alg);
1304 #ifdef CONFIG_X86_64
1305 	if (boot_cpu_has(X86_FEATURE_AVX))
1306 		simd_unregister_skciphers(&aesni_xctr, 1, &aesni_simd_xctr);
1307 #endif /* CONFIG_X86_64 */
1308 }
1309 
1310 late_initcall(aesni_init);
1311 module_exit(aesni_exit);
1312 
1313 MODULE_DESCRIPTION("Rijndael (AES) Cipher Algorithm, Intel AES-NI instructions optimized");
1314 MODULE_LICENSE("GPL");
1315 MODULE_ALIAS_CRYPTO("aes");
1316