1 /*
2  * Support for Intel AES-NI instructions. This file contains glue
3  * code, the real AES implementation is in intel-aes_asm.S.
4  *
5  * Copyright (C) 2008, Intel Corp.
6  *    Author: Huang Ying <ying.huang@intel.com>
7  *
8  * Added RFC4106 AES-GCM support for 128-bit keys under the AEAD
9  * interface for 64-bit kernels.
10  *    Authors: Adrian Hoban <adrian.hoban@intel.com>
11  *             Gabriele Paoloni <gabriele.paoloni@intel.com>
12  *             Tadeusz Struk (tadeusz.struk@intel.com)
13  *             Aidan O'Mahony (aidan.o.mahony@intel.com)
14  *    Copyright (c) 2010, Intel Corporation.
15  *
16  * This program is free software; you can redistribute it and/or modify
17  * it under the terms of the GNU General Public License as published by
18  * the Free Software Foundation; either version 2 of the License, or
19  * (at your option) any later version.
20  */
21 
22 #include <linux/hardirq.h>
23 #include <linux/types.h>
24 #include <linux/module.h>
25 #include <linux/err.h>
26 #include <crypto/algapi.h>
27 #include <crypto/aes.h>
28 #include <crypto/cryptd.h>
29 #include <crypto/ctr.h>
30 #include <crypto/b128ops.h>
31 #include <crypto/gcm.h>
32 #include <crypto/xts.h>
33 #include <asm/cpu_device_id.h>
34 #include <asm/fpu/api.h>
35 #include <asm/crypto/aes.h>
36 #include <crypto/scatterwalk.h>
37 #include <crypto/internal/aead.h>
38 #include <crypto/internal/simd.h>
39 #include <crypto/internal/skcipher.h>
40 #include <linux/workqueue.h>
41 #include <linux/spinlock.h>
42 #ifdef CONFIG_X86_64
43 #include <asm/crypto/glue_helper.h>
44 #endif
45 
46 
47 #define AESNI_ALIGN	16
48 #define AESNI_ALIGN_ATTR __attribute__ ((__aligned__(AESNI_ALIGN)))
49 #define AES_BLOCK_MASK	(~(AES_BLOCK_SIZE - 1))
50 #define RFC4106_HASH_SUBKEY_SIZE 16
51 #define AESNI_ALIGN_EXTRA ((AESNI_ALIGN - 1) & ~(CRYPTO_MINALIGN - 1))
52 #define CRYPTO_AES_CTX_SIZE (sizeof(struct crypto_aes_ctx) + AESNI_ALIGN_EXTRA)
53 #define XTS_AES_CTX_SIZE (sizeof(struct aesni_xts_ctx) + AESNI_ALIGN_EXTRA)
54 
55 /* This data is stored at the end of the crypto_tfm struct.
56  * It's a type of per "session" data storage location.
57  * This needs to be 16 byte aligned.
58  */
59 struct aesni_rfc4106_gcm_ctx {
60 	u8 hash_subkey[16] AESNI_ALIGN_ATTR;
61 	struct crypto_aes_ctx aes_key_expanded AESNI_ALIGN_ATTR;
62 	u8 nonce[4];
63 };
64 
65 struct generic_gcmaes_ctx {
66 	u8 hash_subkey[16] AESNI_ALIGN_ATTR;
67 	struct crypto_aes_ctx aes_key_expanded AESNI_ALIGN_ATTR;
68 };
69 
70 struct aesni_xts_ctx {
71 	u8 raw_tweak_ctx[sizeof(struct crypto_aes_ctx)] AESNI_ALIGN_ATTR;
72 	u8 raw_crypt_ctx[sizeof(struct crypto_aes_ctx)] AESNI_ALIGN_ATTR;
73 };
74 
75 #define GCM_BLOCK_LEN 16
76 
77 struct gcm_context_data {
78 	/* init, update and finalize context data */
79 	u8 aad_hash[GCM_BLOCK_LEN];
80 	u64 aad_length;
81 	u64 in_length;
82 	u8 partial_block_enc_key[GCM_BLOCK_LEN];
83 	u8 orig_IV[GCM_BLOCK_LEN];
84 	u8 current_counter[GCM_BLOCK_LEN];
85 	u64 partial_block_len;
86 	u64 unused;
87 	u8 hash_keys[GCM_BLOCK_LEN * 16];
88 };
89 
90 asmlinkage int aesni_set_key(struct crypto_aes_ctx *ctx, const u8 *in_key,
91 			     unsigned int key_len);
92 asmlinkage void aesni_enc(struct crypto_aes_ctx *ctx, u8 *out,
93 			  const u8 *in);
94 asmlinkage void aesni_dec(struct crypto_aes_ctx *ctx, u8 *out,
95 			  const u8 *in);
96 asmlinkage void aesni_ecb_enc(struct crypto_aes_ctx *ctx, u8 *out,
97 			      const u8 *in, unsigned int len);
98 asmlinkage void aesni_ecb_dec(struct crypto_aes_ctx *ctx, u8 *out,
99 			      const u8 *in, unsigned int len);
100 asmlinkage void aesni_cbc_enc(struct crypto_aes_ctx *ctx, u8 *out,
101 			      const u8 *in, unsigned int len, u8 *iv);
102 asmlinkage void aesni_cbc_dec(struct crypto_aes_ctx *ctx, u8 *out,
103 			      const u8 *in, unsigned int len, u8 *iv);
104 
105 #define AVX_GEN2_OPTSIZE 640
106 #define AVX_GEN4_OPTSIZE 4096
107 
108 #ifdef CONFIG_X86_64
109 
110 static void (*aesni_ctr_enc_tfm)(struct crypto_aes_ctx *ctx, u8 *out,
111 			      const u8 *in, unsigned int len, u8 *iv);
112 asmlinkage void aesni_ctr_enc(struct crypto_aes_ctx *ctx, u8 *out,
113 			      const u8 *in, unsigned int len, u8 *iv);
114 
115 asmlinkage void aesni_xts_crypt8(struct crypto_aes_ctx *ctx, u8 *out,
116 				 const u8 *in, bool enc, u8 *iv);
117 
118 /* asmlinkage void aesni_gcm_enc()
119  * void *ctx,  AES Key schedule. Starts on a 16 byte boundary.
120  * struct gcm_context_data.  May be uninitialized.
121  * u8 *out, Ciphertext output. Encrypt in-place is allowed.
122  * const u8 *in, Plaintext input
123  * unsigned long plaintext_len, Length of data in bytes for encryption.
124  * u8 *iv, Pre-counter block j0: 12 byte IV concatenated with 0x00000001.
125  *         16-byte aligned pointer.
126  * u8 *hash_subkey, the Hash sub key input. Data starts on a 16-byte boundary.
127  * const u8 *aad, Additional Authentication Data (AAD)
128  * unsigned long aad_len, Length of AAD in bytes.
129  * u8 *auth_tag, Authenticated Tag output.
130  * unsigned long auth_tag_len), Authenticated Tag Length in bytes.
131  *          Valid values are 16 (most likely), 12 or 8.
132  */
133 asmlinkage void aesni_gcm_enc(void *ctx,
134 			struct gcm_context_data *gdata, u8 *out,
135 			const u8 *in, unsigned long plaintext_len, u8 *iv,
136 			u8 *hash_subkey, const u8 *aad, unsigned long aad_len,
137 			u8 *auth_tag, unsigned long auth_tag_len);
138 
139 /* asmlinkage void aesni_gcm_dec()
140  * void *ctx, AES Key schedule. Starts on a 16 byte boundary.
141  * struct gcm_context_data.  May be uninitialized.
142  * u8 *out, Plaintext output. Decrypt in-place is allowed.
143  * const u8 *in, Ciphertext input
144  * unsigned long ciphertext_len, Length of data in bytes for decryption.
145  * u8 *iv, Pre-counter block j0: 12 byte IV concatenated with 0x00000001.
146  *         16-byte aligned pointer.
147  * u8 *hash_subkey, the Hash sub key input. Data starts on a 16-byte boundary.
148  * const u8 *aad, Additional Authentication Data (AAD)
149  * unsigned long aad_len, Length of AAD in bytes. With RFC4106 this is going
150  * to be 8 or 12 bytes
151  * u8 *auth_tag, Authenticated Tag output.
152  * unsigned long auth_tag_len) Authenticated Tag Length in bytes.
153  * Valid values are 16 (most likely), 12 or 8.
154  */
155 asmlinkage void aesni_gcm_dec(void *ctx,
156 			struct gcm_context_data *gdata, u8 *out,
157 			const u8 *in, unsigned long ciphertext_len, u8 *iv,
158 			u8 *hash_subkey, const u8 *aad, unsigned long aad_len,
159 			u8 *auth_tag, unsigned long auth_tag_len);
160 
161 /* Scatter / Gather routines, with args similar to above */
162 asmlinkage void aesni_gcm_init(void *ctx,
163 			       struct gcm_context_data *gdata,
164 			       u8 *iv,
165 			       u8 *hash_subkey, const u8 *aad,
166 			       unsigned long aad_len);
167 asmlinkage void aesni_gcm_enc_update(void *ctx,
168 				     struct gcm_context_data *gdata, u8 *out,
169 				     const u8 *in, unsigned long plaintext_len);
170 asmlinkage void aesni_gcm_dec_update(void *ctx,
171 				     struct gcm_context_data *gdata, u8 *out,
172 				     const u8 *in,
173 				     unsigned long ciphertext_len);
174 asmlinkage void aesni_gcm_finalize(void *ctx,
175 				   struct gcm_context_data *gdata,
176 				   u8 *auth_tag, unsigned long auth_tag_len);
177 
178 static const struct aesni_gcm_tfm_s {
179 	void (*init)(void *ctx, struct gcm_context_data *gdata, u8 *iv,
180 		     u8 *hash_subkey, const u8 *aad, unsigned long aad_len);
181 	void (*enc_update)(void *ctx, struct gcm_context_data *gdata, u8 *out,
182 			   const u8 *in, unsigned long plaintext_len);
183 	void (*dec_update)(void *ctx, struct gcm_context_data *gdata, u8 *out,
184 			   const u8 *in, unsigned long ciphertext_len);
185 	void (*finalize)(void *ctx, struct gcm_context_data *gdata,
186 			 u8 *auth_tag, unsigned long auth_tag_len);
187 } *aesni_gcm_tfm;
188 
189 static const struct aesni_gcm_tfm_s aesni_gcm_tfm_sse = {
190 	.init = &aesni_gcm_init,
191 	.enc_update = &aesni_gcm_enc_update,
192 	.dec_update = &aesni_gcm_dec_update,
193 	.finalize = &aesni_gcm_finalize,
194 };
195 
196 #ifdef CONFIG_AS_AVX
197 asmlinkage void aes_ctr_enc_128_avx_by8(const u8 *in, u8 *iv,
198 		void *keys, u8 *out, unsigned int num_bytes);
199 asmlinkage void aes_ctr_enc_192_avx_by8(const u8 *in, u8 *iv,
200 		void *keys, u8 *out, unsigned int num_bytes);
201 asmlinkage void aes_ctr_enc_256_avx_by8(const u8 *in, u8 *iv,
202 		void *keys, u8 *out, unsigned int num_bytes);
203 /*
204  * asmlinkage void aesni_gcm_init_avx_gen2()
205  * gcm_data *my_ctx_data, context data
206  * u8 *hash_subkey,  the Hash sub key input. Data starts on a 16-byte boundary.
207  */
208 asmlinkage void aesni_gcm_init_avx_gen2(void *my_ctx_data,
209 					struct gcm_context_data *gdata,
210 					u8 *iv,
211 					u8 *hash_subkey,
212 					const u8 *aad,
213 					unsigned long aad_len);
214 
215 asmlinkage void aesni_gcm_enc_update_avx_gen2(void *ctx,
216 				     struct gcm_context_data *gdata, u8 *out,
217 				     const u8 *in, unsigned long plaintext_len);
218 asmlinkage void aesni_gcm_dec_update_avx_gen2(void *ctx,
219 				     struct gcm_context_data *gdata, u8 *out,
220 				     const u8 *in,
221 				     unsigned long ciphertext_len);
222 asmlinkage void aesni_gcm_finalize_avx_gen2(void *ctx,
223 				   struct gcm_context_data *gdata,
224 				   u8 *auth_tag, unsigned long auth_tag_len);
225 
226 asmlinkage void aesni_gcm_enc_avx_gen2(void *ctx,
227 				struct gcm_context_data *gdata, u8 *out,
228 			const u8 *in, unsigned long plaintext_len, u8 *iv,
229 			const u8 *aad, unsigned long aad_len,
230 			u8 *auth_tag, unsigned long auth_tag_len);
231 
232 asmlinkage void aesni_gcm_dec_avx_gen2(void *ctx,
233 				struct gcm_context_data *gdata, u8 *out,
234 			const u8 *in, unsigned long ciphertext_len, u8 *iv,
235 			const u8 *aad, unsigned long aad_len,
236 			u8 *auth_tag, unsigned long auth_tag_len);
237 
238 static const struct aesni_gcm_tfm_s aesni_gcm_tfm_avx_gen2 = {
239 	.init = &aesni_gcm_init_avx_gen2,
240 	.enc_update = &aesni_gcm_enc_update_avx_gen2,
241 	.dec_update = &aesni_gcm_dec_update_avx_gen2,
242 	.finalize = &aesni_gcm_finalize_avx_gen2,
243 };
244 
245 #endif
246 
247 #ifdef CONFIG_AS_AVX2
248 /*
249  * asmlinkage void aesni_gcm_init_avx_gen4()
250  * gcm_data *my_ctx_data, context data
251  * u8 *hash_subkey,  the Hash sub key input. Data starts on a 16-byte boundary.
252  */
253 asmlinkage void aesni_gcm_init_avx_gen4(void *my_ctx_data,
254 					struct gcm_context_data *gdata,
255 					u8 *iv,
256 					u8 *hash_subkey,
257 					const u8 *aad,
258 					unsigned long aad_len);
259 
260 asmlinkage void aesni_gcm_enc_update_avx_gen4(void *ctx,
261 				     struct gcm_context_data *gdata, u8 *out,
262 				     const u8 *in, unsigned long plaintext_len);
263 asmlinkage void aesni_gcm_dec_update_avx_gen4(void *ctx,
264 				     struct gcm_context_data *gdata, u8 *out,
265 				     const u8 *in,
266 				     unsigned long ciphertext_len);
267 asmlinkage void aesni_gcm_finalize_avx_gen4(void *ctx,
268 				   struct gcm_context_data *gdata,
269 				   u8 *auth_tag, unsigned long auth_tag_len);
270 
271 asmlinkage void aesni_gcm_enc_avx_gen4(void *ctx,
272 				struct gcm_context_data *gdata, u8 *out,
273 			const u8 *in, unsigned long plaintext_len, u8 *iv,
274 			const u8 *aad, unsigned long aad_len,
275 			u8 *auth_tag, unsigned long auth_tag_len);
276 
277 asmlinkage void aesni_gcm_dec_avx_gen4(void *ctx,
278 				struct gcm_context_data *gdata, u8 *out,
279 			const u8 *in, unsigned long ciphertext_len, u8 *iv,
280 			const u8 *aad, unsigned long aad_len,
281 			u8 *auth_tag, unsigned long auth_tag_len);
282 
283 static const struct aesni_gcm_tfm_s aesni_gcm_tfm_avx_gen4 = {
284 	.init = &aesni_gcm_init_avx_gen4,
285 	.enc_update = &aesni_gcm_enc_update_avx_gen4,
286 	.dec_update = &aesni_gcm_dec_update_avx_gen4,
287 	.finalize = &aesni_gcm_finalize_avx_gen4,
288 };
289 
290 #endif
291 
292 static inline struct
293 aesni_rfc4106_gcm_ctx *aesni_rfc4106_gcm_ctx_get(struct crypto_aead *tfm)
294 {
295 	unsigned long align = AESNI_ALIGN;
296 
297 	if (align <= crypto_tfm_ctx_alignment())
298 		align = 1;
299 	return PTR_ALIGN(crypto_aead_ctx(tfm), align);
300 }
301 
302 static inline struct
303 generic_gcmaes_ctx *generic_gcmaes_ctx_get(struct crypto_aead *tfm)
304 {
305 	unsigned long align = AESNI_ALIGN;
306 
307 	if (align <= crypto_tfm_ctx_alignment())
308 		align = 1;
309 	return PTR_ALIGN(crypto_aead_ctx(tfm), align);
310 }
311 #endif
312 
313 static inline struct crypto_aes_ctx *aes_ctx(void *raw_ctx)
314 {
315 	unsigned long addr = (unsigned long)raw_ctx;
316 	unsigned long align = AESNI_ALIGN;
317 
318 	if (align <= crypto_tfm_ctx_alignment())
319 		align = 1;
320 	return (struct crypto_aes_ctx *)ALIGN(addr, align);
321 }
322 
323 static int aes_set_key_common(struct crypto_tfm *tfm, void *raw_ctx,
324 			      const u8 *in_key, unsigned int key_len)
325 {
326 	struct crypto_aes_ctx *ctx = aes_ctx(raw_ctx);
327 	u32 *flags = &tfm->crt_flags;
328 	int err;
329 
330 	if (key_len != AES_KEYSIZE_128 && key_len != AES_KEYSIZE_192 &&
331 	    key_len != AES_KEYSIZE_256) {
332 		*flags |= CRYPTO_TFM_RES_BAD_KEY_LEN;
333 		return -EINVAL;
334 	}
335 
336 	if (!irq_fpu_usable())
337 		err = crypto_aes_expand_key(ctx, in_key, key_len);
338 	else {
339 		kernel_fpu_begin();
340 		err = aesni_set_key(ctx, in_key, key_len);
341 		kernel_fpu_end();
342 	}
343 
344 	return err;
345 }
346 
347 static int aes_set_key(struct crypto_tfm *tfm, const u8 *in_key,
348 		       unsigned int key_len)
349 {
350 	return aes_set_key_common(tfm, crypto_tfm_ctx(tfm), in_key, key_len);
351 }
352 
353 static void aes_encrypt(struct crypto_tfm *tfm, u8 *dst, const u8 *src)
354 {
355 	struct crypto_aes_ctx *ctx = aes_ctx(crypto_tfm_ctx(tfm));
356 
357 	if (!irq_fpu_usable())
358 		crypto_aes_encrypt_x86(ctx, dst, src);
359 	else {
360 		kernel_fpu_begin();
361 		aesni_enc(ctx, dst, src);
362 		kernel_fpu_end();
363 	}
364 }
365 
366 static void aes_decrypt(struct crypto_tfm *tfm, u8 *dst, const u8 *src)
367 {
368 	struct crypto_aes_ctx *ctx = aes_ctx(crypto_tfm_ctx(tfm));
369 
370 	if (!irq_fpu_usable())
371 		crypto_aes_decrypt_x86(ctx, dst, src);
372 	else {
373 		kernel_fpu_begin();
374 		aesni_dec(ctx, dst, src);
375 		kernel_fpu_end();
376 	}
377 }
378 
379 static void __aes_encrypt(struct crypto_tfm *tfm, u8 *dst, const u8 *src)
380 {
381 	struct crypto_aes_ctx *ctx = aes_ctx(crypto_tfm_ctx(tfm));
382 
383 	aesni_enc(ctx, dst, src);
384 }
385 
386 static void __aes_decrypt(struct crypto_tfm *tfm, u8 *dst, const u8 *src)
387 {
388 	struct crypto_aes_ctx *ctx = aes_ctx(crypto_tfm_ctx(tfm));
389 
390 	aesni_dec(ctx, dst, src);
391 }
392 
393 static int aesni_skcipher_setkey(struct crypto_skcipher *tfm, const u8 *key,
394 			         unsigned int len)
395 {
396 	return aes_set_key_common(crypto_skcipher_tfm(tfm),
397 				  crypto_skcipher_ctx(tfm), key, len);
398 }
399 
400 static int ecb_encrypt(struct skcipher_request *req)
401 {
402 	struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req);
403 	struct crypto_aes_ctx *ctx = aes_ctx(crypto_skcipher_ctx(tfm));
404 	struct skcipher_walk walk;
405 	unsigned int nbytes;
406 	int err;
407 
408 	err = skcipher_walk_virt(&walk, req, true);
409 
410 	kernel_fpu_begin();
411 	while ((nbytes = walk.nbytes)) {
412 		aesni_ecb_enc(ctx, walk.dst.virt.addr, walk.src.virt.addr,
413 			      nbytes & AES_BLOCK_MASK);
414 		nbytes &= AES_BLOCK_SIZE - 1;
415 		err = skcipher_walk_done(&walk, nbytes);
416 	}
417 	kernel_fpu_end();
418 
419 	return err;
420 }
421 
422 static int ecb_decrypt(struct skcipher_request *req)
423 {
424 	struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req);
425 	struct crypto_aes_ctx *ctx = aes_ctx(crypto_skcipher_ctx(tfm));
426 	struct skcipher_walk walk;
427 	unsigned int nbytes;
428 	int err;
429 
430 	err = skcipher_walk_virt(&walk, req, true);
431 
432 	kernel_fpu_begin();
433 	while ((nbytes = walk.nbytes)) {
434 		aesni_ecb_dec(ctx, walk.dst.virt.addr, walk.src.virt.addr,
435 			      nbytes & AES_BLOCK_MASK);
436 		nbytes &= AES_BLOCK_SIZE - 1;
437 		err = skcipher_walk_done(&walk, nbytes);
438 	}
439 	kernel_fpu_end();
440 
441 	return err;
442 }
443 
444 static int cbc_encrypt(struct skcipher_request *req)
445 {
446 	struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req);
447 	struct crypto_aes_ctx *ctx = aes_ctx(crypto_skcipher_ctx(tfm));
448 	struct skcipher_walk walk;
449 	unsigned int nbytes;
450 	int err;
451 
452 	err = skcipher_walk_virt(&walk, req, true);
453 
454 	kernel_fpu_begin();
455 	while ((nbytes = walk.nbytes)) {
456 		aesni_cbc_enc(ctx, walk.dst.virt.addr, walk.src.virt.addr,
457 			      nbytes & AES_BLOCK_MASK, walk.iv);
458 		nbytes &= AES_BLOCK_SIZE - 1;
459 		err = skcipher_walk_done(&walk, nbytes);
460 	}
461 	kernel_fpu_end();
462 
463 	return err;
464 }
465 
466 static int cbc_decrypt(struct skcipher_request *req)
467 {
468 	struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req);
469 	struct crypto_aes_ctx *ctx = aes_ctx(crypto_skcipher_ctx(tfm));
470 	struct skcipher_walk walk;
471 	unsigned int nbytes;
472 	int err;
473 
474 	err = skcipher_walk_virt(&walk, req, true);
475 
476 	kernel_fpu_begin();
477 	while ((nbytes = walk.nbytes)) {
478 		aesni_cbc_dec(ctx, walk.dst.virt.addr, walk.src.virt.addr,
479 			      nbytes & AES_BLOCK_MASK, walk.iv);
480 		nbytes &= AES_BLOCK_SIZE - 1;
481 		err = skcipher_walk_done(&walk, nbytes);
482 	}
483 	kernel_fpu_end();
484 
485 	return err;
486 }
487 
488 #ifdef CONFIG_X86_64
489 static void ctr_crypt_final(struct crypto_aes_ctx *ctx,
490 			    struct skcipher_walk *walk)
491 {
492 	u8 *ctrblk = walk->iv;
493 	u8 keystream[AES_BLOCK_SIZE];
494 	u8 *src = walk->src.virt.addr;
495 	u8 *dst = walk->dst.virt.addr;
496 	unsigned int nbytes = walk->nbytes;
497 
498 	aesni_enc(ctx, keystream, ctrblk);
499 	crypto_xor_cpy(dst, keystream, src, nbytes);
500 
501 	crypto_inc(ctrblk, AES_BLOCK_SIZE);
502 }
503 
504 #ifdef CONFIG_AS_AVX
505 static void aesni_ctr_enc_avx_tfm(struct crypto_aes_ctx *ctx, u8 *out,
506 			      const u8 *in, unsigned int len, u8 *iv)
507 {
508 	/*
509 	 * based on key length, override with the by8 version
510 	 * of ctr mode encryption/decryption for improved performance
511 	 * aes_set_key_common() ensures that key length is one of
512 	 * {128,192,256}
513 	 */
514 	if (ctx->key_length == AES_KEYSIZE_128)
515 		aes_ctr_enc_128_avx_by8(in, iv, (void *)ctx, out, len);
516 	else if (ctx->key_length == AES_KEYSIZE_192)
517 		aes_ctr_enc_192_avx_by8(in, iv, (void *)ctx, out, len);
518 	else
519 		aes_ctr_enc_256_avx_by8(in, iv, (void *)ctx, out, len);
520 }
521 #endif
522 
523 static int ctr_crypt(struct skcipher_request *req)
524 {
525 	struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req);
526 	struct crypto_aes_ctx *ctx = aes_ctx(crypto_skcipher_ctx(tfm));
527 	struct skcipher_walk walk;
528 	unsigned int nbytes;
529 	int err;
530 
531 	err = skcipher_walk_virt(&walk, req, true);
532 
533 	kernel_fpu_begin();
534 	while ((nbytes = walk.nbytes) >= AES_BLOCK_SIZE) {
535 		aesni_ctr_enc_tfm(ctx, walk.dst.virt.addr, walk.src.virt.addr,
536 			              nbytes & AES_BLOCK_MASK, walk.iv);
537 		nbytes &= AES_BLOCK_SIZE - 1;
538 		err = skcipher_walk_done(&walk, nbytes);
539 	}
540 	if (walk.nbytes) {
541 		ctr_crypt_final(ctx, &walk);
542 		err = skcipher_walk_done(&walk, 0);
543 	}
544 	kernel_fpu_end();
545 
546 	return err;
547 }
548 
549 static int xts_aesni_setkey(struct crypto_skcipher *tfm, const u8 *key,
550 			    unsigned int keylen)
551 {
552 	struct aesni_xts_ctx *ctx = crypto_skcipher_ctx(tfm);
553 	int err;
554 
555 	err = xts_verify_key(tfm, key, keylen);
556 	if (err)
557 		return err;
558 
559 	keylen /= 2;
560 
561 	/* first half of xts-key is for crypt */
562 	err = aes_set_key_common(crypto_skcipher_tfm(tfm), ctx->raw_crypt_ctx,
563 				 key, keylen);
564 	if (err)
565 		return err;
566 
567 	/* second half of xts-key is for tweak */
568 	return aes_set_key_common(crypto_skcipher_tfm(tfm), ctx->raw_tweak_ctx,
569 				  key + keylen, keylen);
570 }
571 
572 
573 static void aesni_xts_tweak(void *ctx, u8 *out, const u8 *in)
574 {
575 	aesni_enc(ctx, out, in);
576 }
577 
578 static void aesni_xts_enc(void *ctx, u128 *dst, const u128 *src, le128 *iv)
579 {
580 	glue_xts_crypt_128bit_one(ctx, dst, src, iv, GLUE_FUNC_CAST(aesni_enc));
581 }
582 
583 static void aesni_xts_dec(void *ctx, u128 *dst, const u128 *src, le128 *iv)
584 {
585 	glue_xts_crypt_128bit_one(ctx, dst, src, iv, GLUE_FUNC_CAST(aesni_dec));
586 }
587 
588 static void aesni_xts_enc8(void *ctx, u128 *dst, const u128 *src, le128 *iv)
589 {
590 	aesni_xts_crypt8(ctx, (u8 *)dst, (const u8 *)src, true, (u8 *)iv);
591 }
592 
593 static void aesni_xts_dec8(void *ctx, u128 *dst, const u128 *src, le128 *iv)
594 {
595 	aesni_xts_crypt8(ctx, (u8 *)dst, (const u8 *)src, false, (u8 *)iv);
596 }
597 
598 static const struct common_glue_ctx aesni_enc_xts = {
599 	.num_funcs = 2,
600 	.fpu_blocks_limit = 1,
601 
602 	.funcs = { {
603 		.num_blocks = 8,
604 		.fn_u = { .xts = GLUE_XTS_FUNC_CAST(aesni_xts_enc8) }
605 	}, {
606 		.num_blocks = 1,
607 		.fn_u = { .xts = GLUE_XTS_FUNC_CAST(aesni_xts_enc) }
608 	} }
609 };
610 
611 static const struct common_glue_ctx aesni_dec_xts = {
612 	.num_funcs = 2,
613 	.fpu_blocks_limit = 1,
614 
615 	.funcs = { {
616 		.num_blocks = 8,
617 		.fn_u = { .xts = GLUE_XTS_FUNC_CAST(aesni_xts_dec8) }
618 	}, {
619 		.num_blocks = 1,
620 		.fn_u = { .xts = GLUE_XTS_FUNC_CAST(aesni_xts_dec) }
621 	} }
622 };
623 
624 static int xts_encrypt(struct skcipher_request *req)
625 {
626 	struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req);
627 	struct aesni_xts_ctx *ctx = crypto_skcipher_ctx(tfm);
628 
629 	return glue_xts_req_128bit(&aesni_enc_xts, req,
630 				   XTS_TWEAK_CAST(aesni_xts_tweak),
631 				   aes_ctx(ctx->raw_tweak_ctx),
632 				   aes_ctx(ctx->raw_crypt_ctx));
633 }
634 
635 static int xts_decrypt(struct skcipher_request *req)
636 {
637 	struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req);
638 	struct aesni_xts_ctx *ctx = crypto_skcipher_ctx(tfm);
639 
640 	return glue_xts_req_128bit(&aesni_dec_xts, req,
641 				   XTS_TWEAK_CAST(aesni_xts_tweak),
642 				   aes_ctx(ctx->raw_tweak_ctx),
643 				   aes_ctx(ctx->raw_crypt_ctx));
644 }
645 
646 static int rfc4106_init(struct crypto_aead *aead)
647 {
648 	struct cryptd_aead *cryptd_tfm;
649 	struct cryptd_aead **ctx = crypto_aead_ctx(aead);
650 
651 	cryptd_tfm = cryptd_alloc_aead("__driver-gcm-aes-aesni",
652 				       CRYPTO_ALG_INTERNAL,
653 				       CRYPTO_ALG_INTERNAL);
654 	if (IS_ERR(cryptd_tfm))
655 		return PTR_ERR(cryptd_tfm);
656 
657 	*ctx = cryptd_tfm;
658 	crypto_aead_set_reqsize(aead, crypto_aead_reqsize(&cryptd_tfm->base));
659 	return 0;
660 }
661 
662 static void rfc4106_exit(struct crypto_aead *aead)
663 {
664 	struct cryptd_aead **ctx = crypto_aead_ctx(aead);
665 
666 	cryptd_free_aead(*ctx);
667 }
668 
669 static int
670 rfc4106_set_hash_subkey(u8 *hash_subkey, const u8 *key, unsigned int key_len)
671 {
672 	struct crypto_cipher *tfm;
673 	int ret;
674 
675 	tfm = crypto_alloc_cipher("aes", 0, 0);
676 	if (IS_ERR(tfm))
677 		return PTR_ERR(tfm);
678 
679 	ret = crypto_cipher_setkey(tfm, key, key_len);
680 	if (ret)
681 		goto out_free_cipher;
682 
683 	/* Clear the data in the hash sub key container to zero.*/
684 	/* We want to cipher all zeros to create the hash sub key. */
685 	memset(hash_subkey, 0, RFC4106_HASH_SUBKEY_SIZE);
686 
687 	crypto_cipher_encrypt_one(tfm, hash_subkey, hash_subkey);
688 
689 out_free_cipher:
690 	crypto_free_cipher(tfm);
691 	return ret;
692 }
693 
694 static int common_rfc4106_set_key(struct crypto_aead *aead, const u8 *key,
695 				  unsigned int key_len)
696 {
697 	struct aesni_rfc4106_gcm_ctx *ctx = aesni_rfc4106_gcm_ctx_get(aead);
698 
699 	if (key_len < 4) {
700 		crypto_aead_set_flags(aead, CRYPTO_TFM_RES_BAD_KEY_LEN);
701 		return -EINVAL;
702 	}
703 	/*Account for 4 byte nonce at the end.*/
704 	key_len -= 4;
705 
706 	memcpy(ctx->nonce, key + key_len, sizeof(ctx->nonce));
707 
708 	return aes_set_key_common(crypto_aead_tfm(aead),
709 				  &ctx->aes_key_expanded, key, key_len) ?:
710 	       rfc4106_set_hash_subkey(ctx->hash_subkey, key, key_len);
711 }
712 
713 static int gcmaes_wrapper_set_key(struct crypto_aead *parent, const u8 *key,
714 				  unsigned int key_len)
715 {
716 	struct cryptd_aead **ctx = crypto_aead_ctx(parent);
717 	struct cryptd_aead *cryptd_tfm = *ctx;
718 
719 	return crypto_aead_setkey(&cryptd_tfm->base, key, key_len);
720 }
721 
722 static int common_rfc4106_set_authsize(struct crypto_aead *aead,
723 				       unsigned int authsize)
724 {
725 	switch (authsize) {
726 	case 8:
727 	case 12:
728 	case 16:
729 		break;
730 	default:
731 		return -EINVAL;
732 	}
733 
734 	return 0;
735 }
736 
737 /* This is the Integrity Check Value (aka the authentication tag length and can
738  * be 8, 12 or 16 bytes long. */
739 static int gcmaes_wrapper_set_authsize(struct crypto_aead *parent,
740 				       unsigned int authsize)
741 {
742 	struct cryptd_aead **ctx = crypto_aead_ctx(parent);
743 	struct cryptd_aead *cryptd_tfm = *ctx;
744 
745 	return crypto_aead_setauthsize(&cryptd_tfm->base, authsize);
746 }
747 
748 static int generic_gcmaes_set_authsize(struct crypto_aead *tfm,
749 				       unsigned int authsize)
750 {
751 	switch (authsize) {
752 	case 4:
753 	case 8:
754 	case 12:
755 	case 13:
756 	case 14:
757 	case 15:
758 	case 16:
759 		break;
760 	default:
761 		return -EINVAL;
762 	}
763 
764 	return 0;
765 }
766 
767 static int gcmaes_crypt_by_sg(bool enc, struct aead_request *req,
768 			      unsigned int assoclen, u8 *hash_subkey,
769 			      u8 *iv, void *aes_ctx)
770 {
771 	struct crypto_aead *tfm = crypto_aead_reqtfm(req);
772 	unsigned long auth_tag_len = crypto_aead_authsize(tfm);
773 	const struct aesni_gcm_tfm_s *gcm_tfm = aesni_gcm_tfm;
774 	struct gcm_context_data data AESNI_ALIGN_ATTR;
775 	struct scatter_walk dst_sg_walk = {};
776 	unsigned long left = req->cryptlen;
777 	unsigned long len, srclen, dstlen;
778 	struct scatter_walk assoc_sg_walk;
779 	struct scatter_walk src_sg_walk;
780 	struct scatterlist src_start[2];
781 	struct scatterlist dst_start[2];
782 	struct scatterlist *src_sg;
783 	struct scatterlist *dst_sg;
784 	u8 *src, *dst, *assoc;
785 	u8 *assocmem = NULL;
786 	u8 authTag[16];
787 
788 	if (!enc)
789 		left -= auth_tag_len;
790 
791 #ifdef CONFIG_AS_AVX2
792 	if (left < AVX_GEN4_OPTSIZE && gcm_tfm == &aesni_gcm_tfm_avx_gen4)
793 		gcm_tfm = &aesni_gcm_tfm_avx_gen2;
794 #endif
795 #ifdef CONFIG_AS_AVX
796 	if (left < AVX_GEN2_OPTSIZE && gcm_tfm == &aesni_gcm_tfm_avx_gen2)
797 		gcm_tfm = &aesni_gcm_tfm_sse;
798 #endif
799 
800 	/* Linearize assoc, if not already linear */
801 	if (req->src->length >= assoclen && req->src->length &&
802 		(!PageHighMem(sg_page(req->src)) ||
803 			req->src->offset + req->src->length <= PAGE_SIZE)) {
804 		scatterwalk_start(&assoc_sg_walk, req->src);
805 		assoc = scatterwalk_map(&assoc_sg_walk);
806 	} else {
807 		/* assoc can be any length, so must be on heap */
808 		assocmem = kmalloc(assoclen, GFP_ATOMIC);
809 		if (unlikely(!assocmem))
810 			return -ENOMEM;
811 		assoc = assocmem;
812 
813 		scatterwalk_map_and_copy(assoc, req->src, 0, assoclen, 0);
814 	}
815 
816 	if (left) {
817 		src_sg = scatterwalk_ffwd(src_start, req->src, req->assoclen);
818 		scatterwalk_start(&src_sg_walk, src_sg);
819 		if (req->src != req->dst) {
820 			dst_sg = scatterwalk_ffwd(dst_start, req->dst,
821 						  req->assoclen);
822 			scatterwalk_start(&dst_sg_walk, dst_sg);
823 		}
824 	}
825 
826 	kernel_fpu_begin();
827 	gcm_tfm->init(aes_ctx, &data, iv,
828 		hash_subkey, assoc, assoclen);
829 	if (req->src != req->dst) {
830 		while (left) {
831 			src = scatterwalk_map(&src_sg_walk);
832 			dst = scatterwalk_map(&dst_sg_walk);
833 			srclen = scatterwalk_clamp(&src_sg_walk, left);
834 			dstlen = scatterwalk_clamp(&dst_sg_walk, left);
835 			len = min(srclen, dstlen);
836 			if (len) {
837 				if (enc)
838 					gcm_tfm->enc_update(aes_ctx, &data,
839 							     dst, src, len);
840 				else
841 					gcm_tfm->dec_update(aes_ctx, &data,
842 							     dst, src, len);
843 			}
844 			left -= len;
845 
846 			scatterwalk_unmap(src);
847 			scatterwalk_unmap(dst);
848 			scatterwalk_advance(&src_sg_walk, len);
849 			scatterwalk_advance(&dst_sg_walk, len);
850 			scatterwalk_done(&src_sg_walk, 0, left);
851 			scatterwalk_done(&dst_sg_walk, 1, left);
852 		}
853 	} else {
854 		while (left) {
855 			dst = src = scatterwalk_map(&src_sg_walk);
856 			len = scatterwalk_clamp(&src_sg_walk, left);
857 			if (len) {
858 				if (enc)
859 					gcm_tfm->enc_update(aes_ctx, &data,
860 							     src, src, len);
861 				else
862 					gcm_tfm->dec_update(aes_ctx, &data,
863 							     src, src, len);
864 			}
865 			left -= len;
866 			scatterwalk_unmap(src);
867 			scatterwalk_advance(&src_sg_walk, len);
868 			scatterwalk_done(&src_sg_walk, 1, left);
869 		}
870 	}
871 	gcm_tfm->finalize(aes_ctx, &data, authTag, auth_tag_len);
872 	kernel_fpu_end();
873 
874 	if (!assocmem)
875 		scatterwalk_unmap(assoc);
876 	else
877 		kfree(assocmem);
878 
879 	if (!enc) {
880 		u8 authTagMsg[16];
881 
882 		/* Copy out original authTag */
883 		scatterwalk_map_and_copy(authTagMsg, req->src,
884 					 req->assoclen + req->cryptlen -
885 					 auth_tag_len,
886 					 auth_tag_len, 0);
887 
888 		/* Compare generated tag with passed in tag. */
889 		return crypto_memneq(authTagMsg, authTag, auth_tag_len) ?
890 			-EBADMSG : 0;
891 	}
892 
893 	/* Copy in the authTag */
894 	scatterwalk_map_and_copy(authTag, req->dst,
895 				 req->assoclen + req->cryptlen,
896 				 auth_tag_len, 1);
897 
898 	return 0;
899 }
900 
901 static int gcmaes_encrypt(struct aead_request *req, unsigned int assoclen,
902 			  u8 *hash_subkey, u8 *iv, void *aes_ctx)
903 {
904 	return gcmaes_crypt_by_sg(true, req, assoclen, hash_subkey, iv,
905 				aes_ctx);
906 }
907 
908 static int gcmaes_decrypt(struct aead_request *req, unsigned int assoclen,
909 			  u8 *hash_subkey, u8 *iv, void *aes_ctx)
910 {
911 	return gcmaes_crypt_by_sg(false, req, assoclen, hash_subkey, iv,
912 				aes_ctx);
913 }
914 
915 static int helper_rfc4106_encrypt(struct aead_request *req)
916 {
917 	struct crypto_aead *tfm = crypto_aead_reqtfm(req);
918 	struct aesni_rfc4106_gcm_ctx *ctx = aesni_rfc4106_gcm_ctx_get(tfm);
919 	void *aes_ctx = &(ctx->aes_key_expanded);
920 	u8 iv[16] __attribute__ ((__aligned__(AESNI_ALIGN)));
921 	unsigned int i;
922 	__be32 counter = cpu_to_be32(1);
923 
924 	/* Assuming we are supporting rfc4106 64-bit extended */
925 	/* sequence numbers We need to have the AAD length equal */
926 	/* to 16 or 20 bytes */
927 	if (unlikely(req->assoclen != 16 && req->assoclen != 20))
928 		return -EINVAL;
929 
930 	/* IV below built */
931 	for (i = 0; i < 4; i++)
932 		*(iv+i) = ctx->nonce[i];
933 	for (i = 0; i < 8; i++)
934 		*(iv+4+i) = req->iv[i];
935 	*((__be32 *)(iv+12)) = counter;
936 
937 	return gcmaes_encrypt(req, req->assoclen - 8, ctx->hash_subkey, iv,
938 			      aes_ctx);
939 }
940 
941 static int helper_rfc4106_decrypt(struct aead_request *req)
942 {
943 	__be32 counter = cpu_to_be32(1);
944 	struct crypto_aead *tfm = crypto_aead_reqtfm(req);
945 	struct aesni_rfc4106_gcm_ctx *ctx = aesni_rfc4106_gcm_ctx_get(tfm);
946 	void *aes_ctx = &(ctx->aes_key_expanded);
947 	u8 iv[16] __attribute__ ((__aligned__(AESNI_ALIGN)));
948 	unsigned int i;
949 
950 	if (unlikely(req->assoclen != 16 && req->assoclen != 20))
951 		return -EINVAL;
952 
953 	/* Assuming we are supporting rfc4106 64-bit extended */
954 	/* sequence numbers We need to have the AAD length */
955 	/* equal to 16 or 20 bytes */
956 
957 	/* IV below built */
958 	for (i = 0; i < 4; i++)
959 		*(iv+i) = ctx->nonce[i];
960 	for (i = 0; i < 8; i++)
961 		*(iv+4+i) = req->iv[i];
962 	*((__be32 *)(iv+12)) = counter;
963 
964 	return gcmaes_decrypt(req, req->assoclen - 8, ctx->hash_subkey, iv,
965 			      aes_ctx);
966 }
967 
968 static int gcmaes_wrapper_encrypt(struct aead_request *req)
969 {
970 	struct crypto_aead *tfm = crypto_aead_reqtfm(req);
971 	struct cryptd_aead **ctx = crypto_aead_ctx(tfm);
972 	struct cryptd_aead *cryptd_tfm = *ctx;
973 
974 	tfm = &cryptd_tfm->base;
975 	if (irq_fpu_usable() && (!in_atomic() ||
976 				 !cryptd_aead_queued(cryptd_tfm)))
977 		tfm = cryptd_aead_child(cryptd_tfm);
978 
979 	aead_request_set_tfm(req, tfm);
980 
981 	return crypto_aead_encrypt(req);
982 }
983 
984 static int gcmaes_wrapper_decrypt(struct aead_request *req)
985 {
986 	struct crypto_aead *tfm = crypto_aead_reqtfm(req);
987 	struct cryptd_aead **ctx = crypto_aead_ctx(tfm);
988 	struct cryptd_aead *cryptd_tfm = *ctx;
989 
990 	tfm = &cryptd_tfm->base;
991 	if (irq_fpu_usable() && (!in_atomic() ||
992 				 !cryptd_aead_queued(cryptd_tfm)))
993 		tfm = cryptd_aead_child(cryptd_tfm);
994 
995 	aead_request_set_tfm(req, tfm);
996 
997 	return crypto_aead_decrypt(req);
998 }
999 #endif
1000 
1001 static struct crypto_alg aesni_algs[] = { {
1002 	.cra_name		= "aes",
1003 	.cra_driver_name	= "aes-aesni",
1004 	.cra_priority		= 300,
1005 	.cra_flags		= CRYPTO_ALG_TYPE_CIPHER,
1006 	.cra_blocksize		= AES_BLOCK_SIZE,
1007 	.cra_ctxsize		= CRYPTO_AES_CTX_SIZE,
1008 	.cra_module		= THIS_MODULE,
1009 	.cra_u	= {
1010 		.cipher	= {
1011 			.cia_min_keysize	= AES_MIN_KEY_SIZE,
1012 			.cia_max_keysize	= AES_MAX_KEY_SIZE,
1013 			.cia_setkey		= aes_set_key,
1014 			.cia_encrypt		= aes_encrypt,
1015 			.cia_decrypt		= aes_decrypt
1016 		}
1017 	}
1018 }, {
1019 	.cra_name		= "__aes",
1020 	.cra_driver_name	= "__aes-aesni",
1021 	.cra_priority		= 300,
1022 	.cra_flags		= CRYPTO_ALG_TYPE_CIPHER | CRYPTO_ALG_INTERNAL,
1023 	.cra_blocksize		= AES_BLOCK_SIZE,
1024 	.cra_ctxsize		= CRYPTO_AES_CTX_SIZE,
1025 	.cra_module		= THIS_MODULE,
1026 	.cra_u	= {
1027 		.cipher	= {
1028 			.cia_min_keysize	= AES_MIN_KEY_SIZE,
1029 			.cia_max_keysize	= AES_MAX_KEY_SIZE,
1030 			.cia_setkey		= aes_set_key,
1031 			.cia_encrypt		= __aes_encrypt,
1032 			.cia_decrypt		= __aes_decrypt
1033 		}
1034 	}
1035 } };
1036 
1037 static struct skcipher_alg aesni_skciphers[] = {
1038 	{
1039 		.base = {
1040 			.cra_name		= "__ecb(aes)",
1041 			.cra_driver_name	= "__ecb-aes-aesni",
1042 			.cra_priority		= 400,
1043 			.cra_flags		= CRYPTO_ALG_INTERNAL,
1044 			.cra_blocksize		= AES_BLOCK_SIZE,
1045 			.cra_ctxsize		= CRYPTO_AES_CTX_SIZE,
1046 			.cra_module		= THIS_MODULE,
1047 		},
1048 		.min_keysize	= AES_MIN_KEY_SIZE,
1049 		.max_keysize	= AES_MAX_KEY_SIZE,
1050 		.setkey		= aesni_skcipher_setkey,
1051 		.encrypt	= ecb_encrypt,
1052 		.decrypt	= ecb_decrypt,
1053 	}, {
1054 		.base = {
1055 			.cra_name		= "__cbc(aes)",
1056 			.cra_driver_name	= "__cbc-aes-aesni",
1057 			.cra_priority		= 400,
1058 			.cra_flags		= CRYPTO_ALG_INTERNAL,
1059 			.cra_blocksize		= AES_BLOCK_SIZE,
1060 			.cra_ctxsize		= CRYPTO_AES_CTX_SIZE,
1061 			.cra_module		= THIS_MODULE,
1062 		},
1063 		.min_keysize	= AES_MIN_KEY_SIZE,
1064 		.max_keysize	= AES_MAX_KEY_SIZE,
1065 		.ivsize		= AES_BLOCK_SIZE,
1066 		.setkey		= aesni_skcipher_setkey,
1067 		.encrypt	= cbc_encrypt,
1068 		.decrypt	= cbc_decrypt,
1069 #ifdef CONFIG_X86_64
1070 	}, {
1071 		.base = {
1072 			.cra_name		= "__ctr(aes)",
1073 			.cra_driver_name	= "__ctr-aes-aesni",
1074 			.cra_priority		= 400,
1075 			.cra_flags		= CRYPTO_ALG_INTERNAL,
1076 			.cra_blocksize		= 1,
1077 			.cra_ctxsize		= CRYPTO_AES_CTX_SIZE,
1078 			.cra_module		= THIS_MODULE,
1079 		},
1080 		.min_keysize	= AES_MIN_KEY_SIZE,
1081 		.max_keysize	= AES_MAX_KEY_SIZE,
1082 		.ivsize		= AES_BLOCK_SIZE,
1083 		.chunksize	= AES_BLOCK_SIZE,
1084 		.setkey		= aesni_skcipher_setkey,
1085 		.encrypt	= ctr_crypt,
1086 		.decrypt	= ctr_crypt,
1087 	}, {
1088 		.base = {
1089 			.cra_name		= "__xts(aes)",
1090 			.cra_driver_name	= "__xts-aes-aesni",
1091 			.cra_priority		= 401,
1092 			.cra_flags		= CRYPTO_ALG_INTERNAL,
1093 			.cra_blocksize		= AES_BLOCK_SIZE,
1094 			.cra_ctxsize		= XTS_AES_CTX_SIZE,
1095 			.cra_module		= THIS_MODULE,
1096 		},
1097 		.min_keysize	= 2 * AES_MIN_KEY_SIZE,
1098 		.max_keysize	= 2 * AES_MAX_KEY_SIZE,
1099 		.ivsize		= AES_BLOCK_SIZE,
1100 		.setkey		= xts_aesni_setkey,
1101 		.encrypt	= xts_encrypt,
1102 		.decrypt	= xts_decrypt,
1103 #endif
1104 	}
1105 };
1106 
1107 static
1108 struct simd_skcipher_alg *aesni_simd_skciphers[ARRAY_SIZE(aesni_skciphers)];
1109 
1110 #ifdef CONFIG_X86_64
1111 static int generic_gcmaes_set_key(struct crypto_aead *aead, const u8 *key,
1112 				  unsigned int key_len)
1113 {
1114 	struct generic_gcmaes_ctx *ctx = generic_gcmaes_ctx_get(aead);
1115 
1116 	return aes_set_key_common(crypto_aead_tfm(aead),
1117 				  &ctx->aes_key_expanded, key, key_len) ?:
1118 	       rfc4106_set_hash_subkey(ctx->hash_subkey, key, key_len);
1119 }
1120 
1121 static int generic_gcmaes_encrypt(struct aead_request *req)
1122 {
1123 	struct crypto_aead *tfm = crypto_aead_reqtfm(req);
1124 	struct generic_gcmaes_ctx *ctx = generic_gcmaes_ctx_get(tfm);
1125 	void *aes_ctx = &(ctx->aes_key_expanded);
1126 	u8 iv[16] __attribute__ ((__aligned__(AESNI_ALIGN)));
1127 	__be32 counter = cpu_to_be32(1);
1128 
1129 	memcpy(iv, req->iv, 12);
1130 	*((__be32 *)(iv+12)) = counter;
1131 
1132 	return gcmaes_encrypt(req, req->assoclen, ctx->hash_subkey, iv,
1133 			      aes_ctx);
1134 }
1135 
1136 static int generic_gcmaes_decrypt(struct aead_request *req)
1137 {
1138 	__be32 counter = cpu_to_be32(1);
1139 	struct crypto_aead *tfm = crypto_aead_reqtfm(req);
1140 	struct generic_gcmaes_ctx *ctx = generic_gcmaes_ctx_get(tfm);
1141 	void *aes_ctx = &(ctx->aes_key_expanded);
1142 	u8 iv[16] __attribute__ ((__aligned__(AESNI_ALIGN)));
1143 
1144 	memcpy(iv, req->iv, 12);
1145 	*((__be32 *)(iv+12)) = counter;
1146 
1147 	return gcmaes_decrypt(req, req->assoclen, ctx->hash_subkey, iv,
1148 			      aes_ctx);
1149 }
1150 
1151 static int generic_gcmaes_init(struct crypto_aead *aead)
1152 {
1153 	struct cryptd_aead *cryptd_tfm;
1154 	struct cryptd_aead **ctx = crypto_aead_ctx(aead);
1155 
1156 	cryptd_tfm = cryptd_alloc_aead("__driver-generic-gcm-aes-aesni",
1157 				       CRYPTO_ALG_INTERNAL,
1158 				       CRYPTO_ALG_INTERNAL);
1159 	if (IS_ERR(cryptd_tfm))
1160 		return PTR_ERR(cryptd_tfm);
1161 
1162 	*ctx = cryptd_tfm;
1163 	crypto_aead_set_reqsize(aead, crypto_aead_reqsize(&cryptd_tfm->base));
1164 
1165 	return 0;
1166 }
1167 
1168 static void generic_gcmaes_exit(struct crypto_aead *aead)
1169 {
1170 	struct cryptd_aead **ctx = crypto_aead_ctx(aead);
1171 
1172 	cryptd_free_aead(*ctx);
1173 }
1174 
1175 static struct aead_alg aesni_aead_algs[] = { {
1176 	.setkey			= common_rfc4106_set_key,
1177 	.setauthsize		= common_rfc4106_set_authsize,
1178 	.encrypt		= helper_rfc4106_encrypt,
1179 	.decrypt		= helper_rfc4106_decrypt,
1180 	.ivsize			= GCM_RFC4106_IV_SIZE,
1181 	.maxauthsize		= 16,
1182 	.base = {
1183 		.cra_name		= "__gcm-aes-aesni",
1184 		.cra_driver_name	= "__driver-gcm-aes-aesni",
1185 		.cra_flags		= CRYPTO_ALG_INTERNAL,
1186 		.cra_blocksize		= 1,
1187 		.cra_ctxsize		= sizeof(struct aesni_rfc4106_gcm_ctx),
1188 		.cra_alignmask		= AESNI_ALIGN - 1,
1189 		.cra_module		= THIS_MODULE,
1190 	},
1191 }, {
1192 	.init			= rfc4106_init,
1193 	.exit			= rfc4106_exit,
1194 	.setkey			= gcmaes_wrapper_set_key,
1195 	.setauthsize		= gcmaes_wrapper_set_authsize,
1196 	.encrypt		= gcmaes_wrapper_encrypt,
1197 	.decrypt		= gcmaes_wrapper_decrypt,
1198 	.ivsize			= GCM_RFC4106_IV_SIZE,
1199 	.maxauthsize		= 16,
1200 	.base = {
1201 		.cra_name		= "rfc4106(gcm(aes))",
1202 		.cra_driver_name	= "rfc4106-gcm-aesni",
1203 		.cra_priority		= 400,
1204 		.cra_flags		= CRYPTO_ALG_ASYNC,
1205 		.cra_blocksize		= 1,
1206 		.cra_ctxsize		= sizeof(struct cryptd_aead *),
1207 		.cra_module		= THIS_MODULE,
1208 	},
1209 }, {
1210 	.setkey			= generic_gcmaes_set_key,
1211 	.setauthsize		= generic_gcmaes_set_authsize,
1212 	.encrypt		= generic_gcmaes_encrypt,
1213 	.decrypt		= generic_gcmaes_decrypt,
1214 	.ivsize			= GCM_AES_IV_SIZE,
1215 	.maxauthsize		= 16,
1216 	.base = {
1217 		.cra_name		= "__generic-gcm-aes-aesni",
1218 		.cra_driver_name	= "__driver-generic-gcm-aes-aesni",
1219 		.cra_priority		= 0,
1220 		.cra_flags		= CRYPTO_ALG_INTERNAL,
1221 		.cra_blocksize		= 1,
1222 		.cra_ctxsize		= sizeof(struct generic_gcmaes_ctx),
1223 		.cra_alignmask		= AESNI_ALIGN - 1,
1224 		.cra_module		= THIS_MODULE,
1225 	},
1226 }, {
1227 	.init			= generic_gcmaes_init,
1228 	.exit			= generic_gcmaes_exit,
1229 	.setkey			= gcmaes_wrapper_set_key,
1230 	.setauthsize		= gcmaes_wrapper_set_authsize,
1231 	.encrypt		= gcmaes_wrapper_encrypt,
1232 	.decrypt		= gcmaes_wrapper_decrypt,
1233 	.ivsize			= GCM_AES_IV_SIZE,
1234 	.maxauthsize		= 16,
1235 	.base = {
1236 		.cra_name		= "gcm(aes)",
1237 		.cra_driver_name	= "generic-gcm-aesni",
1238 		.cra_priority		= 400,
1239 		.cra_flags		= CRYPTO_ALG_ASYNC,
1240 		.cra_blocksize		= 1,
1241 		.cra_ctxsize		= sizeof(struct cryptd_aead *),
1242 		.cra_module		= THIS_MODULE,
1243 	},
1244 } };
1245 #else
1246 static struct aead_alg aesni_aead_algs[0];
1247 #endif
1248 
1249 
1250 static const struct x86_cpu_id aesni_cpu_id[] = {
1251 	X86_FEATURE_MATCH(X86_FEATURE_AES),
1252 	{}
1253 };
1254 MODULE_DEVICE_TABLE(x86cpu, aesni_cpu_id);
1255 
1256 static void aesni_free_simds(void)
1257 {
1258 	int i;
1259 
1260 	for (i = 0; i < ARRAY_SIZE(aesni_simd_skciphers) &&
1261 		    aesni_simd_skciphers[i]; i++)
1262 		simd_skcipher_free(aesni_simd_skciphers[i]);
1263 }
1264 
1265 static int __init aesni_init(void)
1266 {
1267 	struct simd_skcipher_alg *simd;
1268 	const char *basename;
1269 	const char *algname;
1270 	const char *drvname;
1271 	int err;
1272 	int i;
1273 
1274 	if (!x86_match_cpu(aesni_cpu_id))
1275 		return -ENODEV;
1276 #ifdef CONFIG_X86_64
1277 #ifdef CONFIG_AS_AVX2
1278 	if (boot_cpu_has(X86_FEATURE_AVX2)) {
1279 		pr_info("AVX2 version of gcm_enc/dec engaged.\n");
1280 		aesni_gcm_tfm = &aesni_gcm_tfm_avx_gen4;
1281 	} else
1282 #endif
1283 #ifdef CONFIG_AS_AVX
1284 	if (boot_cpu_has(X86_FEATURE_AVX)) {
1285 		pr_info("AVX version of gcm_enc/dec engaged.\n");
1286 		aesni_gcm_tfm = &aesni_gcm_tfm_avx_gen2;
1287 	} else
1288 #endif
1289 	{
1290 		pr_info("SSE version of gcm_enc/dec engaged.\n");
1291 		aesni_gcm_tfm = &aesni_gcm_tfm_sse;
1292 	}
1293 	aesni_ctr_enc_tfm = aesni_ctr_enc;
1294 #ifdef CONFIG_AS_AVX
1295 	if (boot_cpu_has(X86_FEATURE_AVX)) {
1296 		/* optimize performance of ctr mode encryption transform */
1297 		aesni_ctr_enc_tfm = aesni_ctr_enc_avx_tfm;
1298 		pr_info("AES CTR mode by8 optimization enabled\n");
1299 	}
1300 #endif
1301 #endif
1302 
1303 	err = crypto_register_algs(aesni_algs, ARRAY_SIZE(aesni_algs));
1304 	if (err)
1305 		return err;
1306 
1307 	err = crypto_register_skciphers(aesni_skciphers,
1308 					ARRAY_SIZE(aesni_skciphers));
1309 	if (err)
1310 		goto unregister_algs;
1311 
1312 	err = crypto_register_aeads(aesni_aead_algs,
1313 				    ARRAY_SIZE(aesni_aead_algs));
1314 	if (err)
1315 		goto unregister_skciphers;
1316 
1317 	for (i = 0; i < ARRAY_SIZE(aesni_skciphers); i++) {
1318 		algname = aesni_skciphers[i].base.cra_name + 2;
1319 		drvname = aesni_skciphers[i].base.cra_driver_name + 2;
1320 		basename = aesni_skciphers[i].base.cra_driver_name;
1321 		simd = simd_skcipher_create_compat(algname, drvname, basename);
1322 		err = PTR_ERR(simd);
1323 		if (IS_ERR(simd))
1324 			goto unregister_simds;
1325 
1326 		aesni_simd_skciphers[i] = simd;
1327 	}
1328 
1329 	return 0;
1330 
1331 unregister_simds:
1332 	aesni_free_simds();
1333 	crypto_unregister_aeads(aesni_aead_algs, ARRAY_SIZE(aesni_aead_algs));
1334 unregister_skciphers:
1335 	crypto_unregister_skciphers(aesni_skciphers,
1336 				    ARRAY_SIZE(aesni_skciphers));
1337 unregister_algs:
1338 	crypto_unregister_algs(aesni_algs, ARRAY_SIZE(aesni_algs));
1339 	return err;
1340 }
1341 
1342 static void __exit aesni_exit(void)
1343 {
1344 	aesni_free_simds();
1345 	crypto_unregister_aeads(aesni_aead_algs, ARRAY_SIZE(aesni_aead_algs));
1346 	crypto_unregister_skciphers(aesni_skciphers,
1347 				    ARRAY_SIZE(aesni_skciphers));
1348 	crypto_unregister_algs(aesni_algs, ARRAY_SIZE(aesni_algs));
1349 }
1350 
1351 late_initcall(aesni_init);
1352 module_exit(aesni_exit);
1353 
1354 MODULE_DESCRIPTION("Rijndael (AES) Cipher Algorithm, Intel AES-NI instructions optimized");
1355 MODULE_LICENSE("GPL");
1356 MODULE_ALIAS_CRYPTO("aes");
1357