1 // SPDX-License-Identifier: GPL-2.0-or-later
2 /*
3 * Support for Intel AES-NI instructions. This file contains glue
4 * code, the real AES implementation is in intel-aes_asm.S.
5 *
6 * Copyright (C) 2008, Intel Corp.
7 * Author: Huang Ying <ying.huang@intel.com>
8 *
9 * Added RFC4106 AES-GCM support for 128-bit keys under the AEAD
10 * interface for 64-bit kernels.
11 * Authors: Adrian Hoban <adrian.hoban@intel.com>
12 * Gabriele Paoloni <gabriele.paoloni@intel.com>
13 * Tadeusz Struk (tadeusz.struk@intel.com)
14 * Aidan O'Mahony (aidan.o.mahony@intel.com)
15 * Copyright (c) 2010, Intel Corporation.
16 */
17
18 #include <linux/hardirq.h>
19 #include <linux/types.h>
20 #include <linux/module.h>
21 #include <linux/err.h>
22 #include <crypto/algapi.h>
23 #include <crypto/aes.h>
24 #include <crypto/ctr.h>
25 #include <crypto/b128ops.h>
26 #include <crypto/gcm.h>
27 #include <crypto/xts.h>
28 #include <asm/cpu_device_id.h>
29 #include <asm/simd.h>
30 #include <crypto/scatterwalk.h>
31 #include <crypto/internal/aead.h>
32 #include <crypto/internal/simd.h>
33 #include <crypto/internal/skcipher.h>
34 #include <linux/jump_label.h>
35 #include <linux/workqueue.h>
36 #include <linux/spinlock.h>
37 #include <linux/static_call.h>
38
39
40 #define AESNI_ALIGN 16
41 #define AESNI_ALIGN_ATTR __attribute__ ((__aligned__(AESNI_ALIGN)))
42 #define AES_BLOCK_MASK (~(AES_BLOCK_SIZE - 1))
43 #define RFC4106_HASH_SUBKEY_SIZE 16
44 #define AESNI_ALIGN_EXTRA ((AESNI_ALIGN - 1) & ~(CRYPTO_MINALIGN - 1))
45 #define CRYPTO_AES_CTX_SIZE (sizeof(struct crypto_aes_ctx) + AESNI_ALIGN_EXTRA)
46 #define XTS_AES_CTX_SIZE (sizeof(struct aesni_xts_ctx) + AESNI_ALIGN_EXTRA)
47
48 /* This data is stored at the end of the crypto_tfm struct.
49 * It's a type of per "session" data storage location.
50 * This needs to be 16 byte aligned.
51 */
52 struct aesni_rfc4106_gcm_ctx {
53 u8 hash_subkey[16] AESNI_ALIGN_ATTR;
54 struct crypto_aes_ctx aes_key_expanded AESNI_ALIGN_ATTR;
55 u8 nonce[4];
56 };
57
58 struct generic_gcmaes_ctx {
59 u8 hash_subkey[16] AESNI_ALIGN_ATTR;
60 struct crypto_aes_ctx aes_key_expanded AESNI_ALIGN_ATTR;
61 };
62
63 struct aesni_xts_ctx {
64 u8 raw_tweak_ctx[sizeof(struct crypto_aes_ctx)] AESNI_ALIGN_ATTR;
65 u8 raw_crypt_ctx[sizeof(struct crypto_aes_ctx)] AESNI_ALIGN_ATTR;
66 };
67
68 #define GCM_BLOCK_LEN 16
69
70 struct gcm_context_data {
71 /* init, update and finalize context data */
72 u8 aad_hash[GCM_BLOCK_LEN];
73 u64 aad_length;
74 u64 in_length;
75 u8 partial_block_enc_key[GCM_BLOCK_LEN];
76 u8 orig_IV[GCM_BLOCK_LEN];
77 u8 current_counter[GCM_BLOCK_LEN];
78 u64 partial_block_len;
79 u64 unused;
80 u8 hash_keys[GCM_BLOCK_LEN * 16];
81 };
82
83 asmlinkage int aesni_set_key(struct crypto_aes_ctx *ctx, const u8 *in_key,
84 unsigned int key_len);
85 asmlinkage void aesni_enc(const void *ctx, u8 *out, const u8 *in);
86 asmlinkage void aesni_dec(const void *ctx, u8 *out, const u8 *in);
87 asmlinkage void aesni_ecb_enc(struct crypto_aes_ctx *ctx, u8 *out,
88 const u8 *in, unsigned int len);
89 asmlinkage void aesni_ecb_dec(struct crypto_aes_ctx *ctx, u8 *out,
90 const u8 *in, unsigned int len);
91 asmlinkage void aesni_cbc_enc(struct crypto_aes_ctx *ctx, u8 *out,
92 const u8 *in, unsigned int len, u8 *iv);
93 asmlinkage void aesni_cbc_dec(struct crypto_aes_ctx *ctx, u8 *out,
94 const u8 *in, unsigned int len, u8 *iv);
95 asmlinkage void aesni_cts_cbc_enc(struct crypto_aes_ctx *ctx, u8 *out,
96 const u8 *in, unsigned int len, u8 *iv);
97 asmlinkage void aesni_cts_cbc_dec(struct crypto_aes_ctx *ctx, u8 *out,
98 const u8 *in, unsigned int len, u8 *iv);
99
100 #define AVX_GEN2_OPTSIZE 640
101 #define AVX_GEN4_OPTSIZE 4096
102
103 asmlinkage void aesni_xts_encrypt(const struct crypto_aes_ctx *ctx, u8 *out,
104 const u8 *in, unsigned int len, u8 *iv);
105
106 asmlinkage void aesni_xts_decrypt(const struct crypto_aes_ctx *ctx, u8 *out,
107 const u8 *in, unsigned int len, u8 *iv);
108
109 #ifdef CONFIG_X86_64
110
111 asmlinkage void aesni_ctr_enc(struct crypto_aes_ctx *ctx, u8 *out,
112 const u8 *in, unsigned int len, u8 *iv);
113 DEFINE_STATIC_CALL(aesni_ctr_enc_tfm, aesni_ctr_enc);
114
115 /* Scatter / Gather routines, with args similar to above */
116 asmlinkage void aesni_gcm_init(void *ctx,
117 struct gcm_context_data *gdata,
118 u8 *iv,
119 u8 *hash_subkey, const u8 *aad,
120 unsigned long aad_len);
121 asmlinkage void aesni_gcm_enc_update(void *ctx,
122 struct gcm_context_data *gdata, u8 *out,
123 const u8 *in, unsigned long plaintext_len);
124 asmlinkage void aesni_gcm_dec_update(void *ctx,
125 struct gcm_context_data *gdata, u8 *out,
126 const u8 *in,
127 unsigned long ciphertext_len);
128 asmlinkage void aesni_gcm_finalize(void *ctx,
129 struct gcm_context_data *gdata,
130 u8 *auth_tag, unsigned long auth_tag_len);
131
132 asmlinkage void aes_ctr_enc_128_avx_by8(const u8 *in, u8 *iv,
133 void *keys, u8 *out, unsigned int num_bytes);
134 asmlinkage void aes_ctr_enc_192_avx_by8(const u8 *in, u8 *iv,
135 void *keys, u8 *out, unsigned int num_bytes);
136 asmlinkage void aes_ctr_enc_256_avx_by8(const u8 *in, u8 *iv,
137 void *keys, u8 *out, unsigned int num_bytes);
138
139
140 asmlinkage void aes_xctr_enc_128_avx_by8(const u8 *in, const u8 *iv,
141 const void *keys, u8 *out, unsigned int num_bytes,
142 unsigned int byte_ctr);
143
144 asmlinkage void aes_xctr_enc_192_avx_by8(const u8 *in, const u8 *iv,
145 const void *keys, u8 *out, unsigned int num_bytes,
146 unsigned int byte_ctr);
147
148 asmlinkage void aes_xctr_enc_256_avx_by8(const u8 *in, const u8 *iv,
149 const void *keys, u8 *out, unsigned int num_bytes,
150 unsigned int byte_ctr);
151
152 /*
153 * asmlinkage void aesni_gcm_init_avx_gen2()
154 * gcm_data *my_ctx_data, context data
155 * u8 *hash_subkey, the Hash sub key input. Data starts on a 16-byte boundary.
156 */
157 asmlinkage void aesni_gcm_init_avx_gen2(void *my_ctx_data,
158 struct gcm_context_data *gdata,
159 u8 *iv,
160 u8 *hash_subkey,
161 const u8 *aad,
162 unsigned long aad_len);
163
164 asmlinkage void aesni_gcm_enc_update_avx_gen2(void *ctx,
165 struct gcm_context_data *gdata, u8 *out,
166 const u8 *in, unsigned long plaintext_len);
167 asmlinkage void aesni_gcm_dec_update_avx_gen2(void *ctx,
168 struct gcm_context_data *gdata, u8 *out,
169 const u8 *in,
170 unsigned long ciphertext_len);
171 asmlinkage void aesni_gcm_finalize_avx_gen2(void *ctx,
172 struct gcm_context_data *gdata,
173 u8 *auth_tag, unsigned long auth_tag_len);
174
175 /*
176 * asmlinkage void aesni_gcm_init_avx_gen4()
177 * gcm_data *my_ctx_data, context data
178 * u8 *hash_subkey, the Hash sub key input. Data starts on a 16-byte boundary.
179 */
180 asmlinkage void aesni_gcm_init_avx_gen4(void *my_ctx_data,
181 struct gcm_context_data *gdata,
182 u8 *iv,
183 u8 *hash_subkey,
184 const u8 *aad,
185 unsigned long aad_len);
186
187 asmlinkage void aesni_gcm_enc_update_avx_gen4(void *ctx,
188 struct gcm_context_data *gdata, u8 *out,
189 const u8 *in, unsigned long plaintext_len);
190 asmlinkage void aesni_gcm_dec_update_avx_gen4(void *ctx,
191 struct gcm_context_data *gdata, u8 *out,
192 const u8 *in,
193 unsigned long ciphertext_len);
194 asmlinkage void aesni_gcm_finalize_avx_gen4(void *ctx,
195 struct gcm_context_data *gdata,
196 u8 *auth_tag, unsigned long auth_tag_len);
197
198 static __ro_after_init DEFINE_STATIC_KEY_FALSE(gcm_use_avx);
199 static __ro_after_init DEFINE_STATIC_KEY_FALSE(gcm_use_avx2);
200
201 static inline struct
aesni_rfc4106_gcm_ctx_get(struct crypto_aead * tfm)202 aesni_rfc4106_gcm_ctx *aesni_rfc4106_gcm_ctx_get(struct crypto_aead *tfm)
203 {
204 unsigned long align = AESNI_ALIGN;
205
206 if (align <= crypto_tfm_ctx_alignment())
207 align = 1;
208 return PTR_ALIGN(crypto_aead_ctx(tfm), align);
209 }
210
211 static inline struct
generic_gcmaes_ctx_get(struct crypto_aead * tfm)212 generic_gcmaes_ctx *generic_gcmaes_ctx_get(struct crypto_aead *tfm)
213 {
214 unsigned long align = AESNI_ALIGN;
215
216 if (align <= crypto_tfm_ctx_alignment())
217 align = 1;
218 return PTR_ALIGN(crypto_aead_ctx(tfm), align);
219 }
220 #endif
221
aes_ctx(void * raw_ctx)222 static inline struct crypto_aes_ctx *aes_ctx(void *raw_ctx)
223 {
224 unsigned long addr = (unsigned long)raw_ctx;
225 unsigned long align = AESNI_ALIGN;
226
227 if (align <= crypto_tfm_ctx_alignment())
228 align = 1;
229 return (struct crypto_aes_ctx *)ALIGN(addr, align);
230 }
231
aes_set_key_common(struct crypto_aes_ctx * ctx,const u8 * in_key,unsigned int key_len)232 static int aes_set_key_common(struct crypto_aes_ctx *ctx,
233 const u8 *in_key, unsigned int key_len)
234 {
235 int err;
236
237 if (key_len != AES_KEYSIZE_128 && key_len != AES_KEYSIZE_192 &&
238 key_len != AES_KEYSIZE_256)
239 return -EINVAL;
240
241 if (!crypto_simd_usable())
242 err = aes_expandkey(ctx, in_key, key_len);
243 else {
244 kernel_fpu_begin();
245 err = aesni_set_key(ctx, in_key, key_len);
246 kernel_fpu_end();
247 }
248
249 return err;
250 }
251
aes_set_key(struct crypto_tfm * tfm,const u8 * in_key,unsigned int key_len)252 static int aes_set_key(struct crypto_tfm *tfm, const u8 *in_key,
253 unsigned int key_len)
254 {
255 return aes_set_key_common(aes_ctx(crypto_tfm_ctx(tfm)), in_key,
256 key_len);
257 }
258
aesni_encrypt(struct crypto_tfm * tfm,u8 * dst,const u8 * src)259 static void aesni_encrypt(struct crypto_tfm *tfm, u8 *dst, const u8 *src)
260 {
261 struct crypto_aes_ctx *ctx = aes_ctx(crypto_tfm_ctx(tfm));
262
263 if (!crypto_simd_usable()) {
264 aes_encrypt(ctx, dst, src);
265 } else {
266 kernel_fpu_begin();
267 aesni_enc(ctx, dst, src);
268 kernel_fpu_end();
269 }
270 }
271
aesni_decrypt(struct crypto_tfm * tfm,u8 * dst,const u8 * src)272 static void aesni_decrypt(struct crypto_tfm *tfm, u8 *dst, const u8 *src)
273 {
274 struct crypto_aes_ctx *ctx = aes_ctx(crypto_tfm_ctx(tfm));
275
276 if (!crypto_simd_usable()) {
277 aes_decrypt(ctx, dst, src);
278 } else {
279 kernel_fpu_begin();
280 aesni_dec(ctx, dst, src);
281 kernel_fpu_end();
282 }
283 }
284
aesni_skcipher_setkey(struct crypto_skcipher * tfm,const u8 * key,unsigned int len)285 static int aesni_skcipher_setkey(struct crypto_skcipher *tfm, const u8 *key,
286 unsigned int len)
287 {
288 return aes_set_key_common(aes_ctx(crypto_skcipher_ctx(tfm)), key, len);
289 }
290
ecb_encrypt(struct skcipher_request * req)291 static int ecb_encrypt(struct skcipher_request *req)
292 {
293 struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req);
294 struct crypto_aes_ctx *ctx = aes_ctx(crypto_skcipher_ctx(tfm));
295 struct skcipher_walk walk;
296 unsigned int nbytes;
297 int err;
298
299 err = skcipher_walk_virt(&walk, req, false);
300
301 while ((nbytes = walk.nbytes)) {
302 kernel_fpu_begin();
303 aesni_ecb_enc(ctx, walk.dst.virt.addr, walk.src.virt.addr,
304 nbytes & AES_BLOCK_MASK);
305 kernel_fpu_end();
306 nbytes &= AES_BLOCK_SIZE - 1;
307 err = skcipher_walk_done(&walk, nbytes);
308 }
309
310 return err;
311 }
312
ecb_decrypt(struct skcipher_request * req)313 static int ecb_decrypt(struct skcipher_request *req)
314 {
315 struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req);
316 struct crypto_aes_ctx *ctx = aes_ctx(crypto_skcipher_ctx(tfm));
317 struct skcipher_walk walk;
318 unsigned int nbytes;
319 int err;
320
321 err = skcipher_walk_virt(&walk, req, false);
322
323 while ((nbytes = walk.nbytes)) {
324 kernel_fpu_begin();
325 aesni_ecb_dec(ctx, walk.dst.virt.addr, walk.src.virt.addr,
326 nbytes & AES_BLOCK_MASK);
327 kernel_fpu_end();
328 nbytes &= AES_BLOCK_SIZE - 1;
329 err = skcipher_walk_done(&walk, nbytes);
330 }
331
332 return err;
333 }
334
cbc_encrypt(struct skcipher_request * req)335 static int cbc_encrypt(struct skcipher_request *req)
336 {
337 struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req);
338 struct crypto_aes_ctx *ctx = aes_ctx(crypto_skcipher_ctx(tfm));
339 struct skcipher_walk walk;
340 unsigned int nbytes;
341 int err;
342
343 err = skcipher_walk_virt(&walk, req, false);
344
345 while ((nbytes = walk.nbytes)) {
346 kernel_fpu_begin();
347 aesni_cbc_enc(ctx, walk.dst.virt.addr, walk.src.virt.addr,
348 nbytes & AES_BLOCK_MASK, walk.iv);
349 kernel_fpu_end();
350 nbytes &= AES_BLOCK_SIZE - 1;
351 err = skcipher_walk_done(&walk, nbytes);
352 }
353
354 return err;
355 }
356
cbc_decrypt(struct skcipher_request * req)357 static int cbc_decrypt(struct skcipher_request *req)
358 {
359 struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req);
360 struct crypto_aes_ctx *ctx = aes_ctx(crypto_skcipher_ctx(tfm));
361 struct skcipher_walk walk;
362 unsigned int nbytes;
363 int err;
364
365 err = skcipher_walk_virt(&walk, req, false);
366
367 while ((nbytes = walk.nbytes)) {
368 kernel_fpu_begin();
369 aesni_cbc_dec(ctx, walk.dst.virt.addr, walk.src.virt.addr,
370 nbytes & AES_BLOCK_MASK, walk.iv);
371 kernel_fpu_end();
372 nbytes &= AES_BLOCK_SIZE - 1;
373 err = skcipher_walk_done(&walk, nbytes);
374 }
375
376 return err;
377 }
378
cts_cbc_encrypt(struct skcipher_request * req)379 static int cts_cbc_encrypt(struct skcipher_request *req)
380 {
381 struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req);
382 struct crypto_aes_ctx *ctx = aes_ctx(crypto_skcipher_ctx(tfm));
383 int cbc_blocks = DIV_ROUND_UP(req->cryptlen, AES_BLOCK_SIZE) - 2;
384 struct scatterlist *src = req->src, *dst = req->dst;
385 struct scatterlist sg_src[2], sg_dst[2];
386 struct skcipher_request subreq;
387 struct skcipher_walk walk;
388 int err;
389
390 skcipher_request_set_tfm(&subreq, tfm);
391 skcipher_request_set_callback(&subreq, skcipher_request_flags(req),
392 NULL, NULL);
393
394 if (req->cryptlen <= AES_BLOCK_SIZE) {
395 if (req->cryptlen < AES_BLOCK_SIZE)
396 return -EINVAL;
397 cbc_blocks = 1;
398 }
399
400 if (cbc_blocks > 0) {
401 skcipher_request_set_crypt(&subreq, req->src, req->dst,
402 cbc_blocks * AES_BLOCK_SIZE,
403 req->iv);
404
405 err = cbc_encrypt(&subreq);
406 if (err)
407 return err;
408
409 if (req->cryptlen == AES_BLOCK_SIZE)
410 return 0;
411
412 dst = src = scatterwalk_ffwd(sg_src, req->src, subreq.cryptlen);
413 if (req->dst != req->src)
414 dst = scatterwalk_ffwd(sg_dst, req->dst,
415 subreq.cryptlen);
416 }
417
418 /* handle ciphertext stealing */
419 skcipher_request_set_crypt(&subreq, src, dst,
420 req->cryptlen - cbc_blocks * AES_BLOCK_SIZE,
421 req->iv);
422
423 err = skcipher_walk_virt(&walk, &subreq, false);
424 if (err)
425 return err;
426
427 kernel_fpu_begin();
428 aesni_cts_cbc_enc(ctx, walk.dst.virt.addr, walk.src.virt.addr,
429 walk.nbytes, walk.iv);
430 kernel_fpu_end();
431
432 return skcipher_walk_done(&walk, 0);
433 }
434
cts_cbc_decrypt(struct skcipher_request * req)435 static int cts_cbc_decrypt(struct skcipher_request *req)
436 {
437 struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req);
438 struct crypto_aes_ctx *ctx = aes_ctx(crypto_skcipher_ctx(tfm));
439 int cbc_blocks = DIV_ROUND_UP(req->cryptlen, AES_BLOCK_SIZE) - 2;
440 struct scatterlist *src = req->src, *dst = req->dst;
441 struct scatterlist sg_src[2], sg_dst[2];
442 struct skcipher_request subreq;
443 struct skcipher_walk walk;
444 int err;
445
446 skcipher_request_set_tfm(&subreq, tfm);
447 skcipher_request_set_callback(&subreq, skcipher_request_flags(req),
448 NULL, NULL);
449
450 if (req->cryptlen <= AES_BLOCK_SIZE) {
451 if (req->cryptlen < AES_BLOCK_SIZE)
452 return -EINVAL;
453 cbc_blocks = 1;
454 }
455
456 if (cbc_blocks > 0) {
457 skcipher_request_set_crypt(&subreq, req->src, req->dst,
458 cbc_blocks * AES_BLOCK_SIZE,
459 req->iv);
460
461 err = cbc_decrypt(&subreq);
462 if (err)
463 return err;
464
465 if (req->cryptlen == AES_BLOCK_SIZE)
466 return 0;
467
468 dst = src = scatterwalk_ffwd(sg_src, req->src, subreq.cryptlen);
469 if (req->dst != req->src)
470 dst = scatterwalk_ffwd(sg_dst, req->dst,
471 subreq.cryptlen);
472 }
473
474 /* handle ciphertext stealing */
475 skcipher_request_set_crypt(&subreq, src, dst,
476 req->cryptlen - cbc_blocks * AES_BLOCK_SIZE,
477 req->iv);
478
479 err = skcipher_walk_virt(&walk, &subreq, false);
480 if (err)
481 return err;
482
483 kernel_fpu_begin();
484 aesni_cts_cbc_dec(ctx, walk.dst.virt.addr, walk.src.virt.addr,
485 walk.nbytes, walk.iv);
486 kernel_fpu_end();
487
488 return skcipher_walk_done(&walk, 0);
489 }
490
491 #ifdef CONFIG_X86_64
aesni_ctr_enc_avx_tfm(struct crypto_aes_ctx * ctx,u8 * out,const u8 * in,unsigned int len,u8 * iv)492 static void aesni_ctr_enc_avx_tfm(struct crypto_aes_ctx *ctx, u8 *out,
493 const u8 *in, unsigned int len, u8 *iv)
494 {
495 /*
496 * based on key length, override with the by8 version
497 * of ctr mode encryption/decryption for improved performance
498 * aes_set_key_common() ensures that key length is one of
499 * {128,192,256}
500 */
501 if (ctx->key_length == AES_KEYSIZE_128)
502 aes_ctr_enc_128_avx_by8(in, iv, (void *)ctx, out, len);
503 else if (ctx->key_length == AES_KEYSIZE_192)
504 aes_ctr_enc_192_avx_by8(in, iv, (void *)ctx, out, len);
505 else
506 aes_ctr_enc_256_avx_by8(in, iv, (void *)ctx, out, len);
507 }
508
ctr_crypt(struct skcipher_request * req)509 static int ctr_crypt(struct skcipher_request *req)
510 {
511 struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req);
512 struct crypto_aes_ctx *ctx = aes_ctx(crypto_skcipher_ctx(tfm));
513 u8 keystream[AES_BLOCK_SIZE];
514 struct skcipher_walk walk;
515 unsigned int nbytes;
516 int err;
517
518 err = skcipher_walk_virt(&walk, req, false);
519
520 while ((nbytes = walk.nbytes) > 0) {
521 kernel_fpu_begin();
522 if (nbytes & AES_BLOCK_MASK)
523 static_call(aesni_ctr_enc_tfm)(ctx, walk.dst.virt.addr,
524 walk.src.virt.addr,
525 nbytes & AES_BLOCK_MASK,
526 walk.iv);
527 nbytes &= ~AES_BLOCK_MASK;
528
529 if (walk.nbytes == walk.total && nbytes > 0) {
530 aesni_enc(ctx, keystream, walk.iv);
531 crypto_xor_cpy(walk.dst.virt.addr + walk.nbytes - nbytes,
532 walk.src.virt.addr + walk.nbytes - nbytes,
533 keystream, nbytes);
534 crypto_inc(walk.iv, AES_BLOCK_SIZE);
535 nbytes = 0;
536 }
537 kernel_fpu_end();
538 err = skcipher_walk_done(&walk, nbytes);
539 }
540 return err;
541 }
542
aesni_xctr_enc_avx_tfm(struct crypto_aes_ctx * ctx,u8 * out,const u8 * in,unsigned int len,u8 * iv,unsigned int byte_ctr)543 static void aesni_xctr_enc_avx_tfm(struct crypto_aes_ctx *ctx, u8 *out,
544 const u8 *in, unsigned int len, u8 *iv,
545 unsigned int byte_ctr)
546 {
547 if (ctx->key_length == AES_KEYSIZE_128)
548 aes_xctr_enc_128_avx_by8(in, iv, (void *)ctx, out, len,
549 byte_ctr);
550 else if (ctx->key_length == AES_KEYSIZE_192)
551 aes_xctr_enc_192_avx_by8(in, iv, (void *)ctx, out, len,
552 byte_ctr);
553 else
554 aes_xctr_enc_256_avx_by8(in, iv, (void *)ctx, out, len,
555 byte_ctr);
556 }
557
xctr_crypt(struct skcipher_request * req)558 static int xctr_crypt(struct skcipher_request *req)
559 {
560 struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req);
561 struct crypto_aes_ctx *ctx = aes_ctx(crypto_skcipher_ctx(tfm));
562 u8 keystream[AES_BLOCK_SIZE];
563 struct skcipher_walk walk;
564 unsigned int nbytes;
565 unsigned int byte_ctr = 0;
566 int err;
567 __le32 block[AES_BLOCK_SIZE / sizeof(__le32)];
568
569 err = skcipher_walk_virt(&walk, req, false);
570
571 while ((nbytes = walk.nbytes) > 0) {
572 kernel_fpu_begin();
573 if (nbytes & AES_BLOCK_MASK)
574 aesni_xctr_enc_avx_tfm(ctx, walk.dst.virt.addr,
575 walk.src.virt.addr, nbytes & AES_BLOCK_MASK,
576 walk.iv, byte_ctr);
577 nbytes &= ~AES_BLOCK_MASK;
578 byte_ctr += walk.nbytes - nbytes;
579
580 if (walk.nbytes == walk.total && nbytes > 0) {
581 memcpy(block, walk.iv, AES_BLOCK_SIZE);
582 block[0] ^= cpu_to_le32(1 + byte_ctr / AES_BLOCK_SIZE);
583 aesni_enc(ctx, keystream, (u8 *)block);
584 crypto_xor_cpy(walk.dst.virt.addr + walk.nbytes -
585 nbytes, walk.src.virt.addr + walk.nbytes
586 - nbytes, keystream, nbytes);
587 byte_ctr += nbytes;
588 nbytes = 0;
589 }
590 kernel_fpu_end();
591 err = skcipher_walk_done(&walk, nbytes);
592 }
593 return err;
594 }
595
596 static int
rfc4106_set_hash_subkey(u8 * hash_subkey,const u8 * key,unsigned int key_len)597 rfc4106_set_hash_subkey(u8 *hash_subkey, const u8 *key, unsigned int key_len)
598 {
599 struct crypto_aes_ctx ctx;
600 int ret;
601
602 ret = aes_expandkey(&ctx, key, key_len);
603 if (ret)
604 return ret;
605
606 /* Clear the data in the hash sub key container to zero.*/
607 /* We want to cipher all zeros to create the hash sub key. */
608 memset(hash_subkey, 0, RFC4106_HASH_SUBKEY_SIZE);
609
610 aes_encrypt(&ctx, hash_subkey, hash_subkey);
611
612 memzero_explicit(&ctx, sizeof(ctx));
613 return 0;
614 }
615
common_rfc4106_set_key(struct crypto_aead * aead,const u8 * key,unsigned int key_len)616 static int common_rfc4106_set_key(struct crypto_aead *aead, const u8 *key,
617 unsigned int key_len)
618 {
619 struct aesni_rfc4106_gcm_ctx *ctx = aesni_rfc4106_gcm_ctx_get(aead);
620
621 if (key_len < 4)
622 return -EINVAL;
623
624 /*Account for 4 byte nonce at the end.*/
625 key_len -= 4;
626
627 memcpy(ctx->nonce, key + key_len, sizeof(ctx->nonce));
628
629 return aes_set_key_common(&ctx->aes_key_expanded, key, key_len) ?:
630 rfc4106_set_hash_subkey(ctx->hash_subkey, key, key_len);
631 }
632
633 /* This is the Integrity Check Value (aka the authentication tag) length and can
634 * be 8, 12 or 16 bytes long. */
common_rfc4106_set_authsize(struct crypto_aead * aead,unsigned int authsize)635 static int common_rfc4106_set_authsize(struct crypto_aead *aead,
636 unsigned int authsize)
637 {
638 switch (authsize) {
639 case 8:
640 case 12:
641 case 16:
642 break;
643 default:
644 return -EINVAL;
645 }
646
647 return 0;
648 }
649
generic_gcmaes_set_authsize(struct crypto_aead * tfm,unsigned int authsize)650 static int generic_gcmaes_set_authsize(struct crypto_aead *tfm,
651 unsigned int authsize)
652 {
653 switch (authsize) {
654 case 4:
655 case 8:
656 case 12:
657 case 13:
658 case 14:
659 case 15:
660 case 16:
661 break;
662 default:
663 return -EINVAL;
664 }
665
666 return 0;
667 }
668
gcmaes_crypt_by_sg(bool enc,struct aead_request * req,unsigned int assoclen,u8 * hash_subkey,u8 * iv,void * aes_ctx,u8 * auth_tag,unsigned long auth_tag_len)669 static int gcmaes_crypt_by_sg(bool enc, struct aead_request *req,
670 unsigned int assoclen, u8 *hash_subkey,
671 u8 *iv, void *aes_ctx, u8 *auth_tag,
672 unsigned long auth_tag_len)
673 {
674 u8 databuf[sizeof(struct gcm_context_data) + (AESNI_ALIGN - 8)] __aligned(8);
675 struct gcm_context_data *data = PTR_ALIGN((void *)databuf, AESNI_ALIGN);
676 unsigned long left = req->cryptlen;
677 struct scatter_walk assoc_sg_walk;
678 struct skcipher_walk walk;
679 bool do_avx, do_avx2;
680 u8 *assocmem = NULL;
681 u8 *assoc;
682 int err;
683
684 if (!enc)
685 left -= auth_tag_len;
686
687 do_avx = (left >= AVX_GEN2_OPTSIZE);
688 do_avx2 = (left >= AVX_GEN4_OPTSIZE);
689
690 /* Linearize assoc, if not already linear */
691 if (req->src->length >= assoclen && req->src->length) {
692 scatterwalk_start(&assoc_sg_walk, req->src);
693 assoc = scatterwalk_map(&assoc_sg_walk);
694 } else {
695 gfp_t flags = (req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP) ?
696 GFP_KERNEL : GFP_ATOMIC;
697
698 /* assoc can be any length, so must be on heap */
699 assocmem = kmalloc(assoclen, flags);
700 if (unlikely(!assocmem))
701 return -ENOMEM;
702 assoc = assocmem;
703
704 scatterwalk_map_and_copy(assoc, req->src, 0, assoclen, 0);
705 }
706
707 kernel_fpu_begin();
708 if (static_branch_likely(&gcm_use_avx2) && do_avx2)
709 aesni_gcm_init_avx_gen4(aes_ctx, data, iv, hash_subkey, assoc,
710 assoclen);
711 else if (static_branch_likely(&gcm_use_avx) && do_avx)
712 aesni_gcm_init_avx_gen2(aes_ctx, data, iv, hash_subkey, assoc,
713 assoclen);
714 else
715 aesni_gcm_init(aes_ctx, data, iv, hash_subkey, assoc, assoclen);
716 kernel_fpu_end();
717
718 if (!assocmem)
719 scatterwalk_unmap(assoc);
720 else
721 kfree(assocmem);
722
723 err = enc ? skcipher_walk_aead_encrypt(&walk, req, false)
724 : skcipher_walk_aead_decrypt(&walk, req, false);
725
726 while (walk.nbytes > 0) {
727 kernel_fpu_begin();
728 if (static_branch_likely(&gcm_use_avx2) && do_avx2) {
729 if (enc)
730 aesni_gcm_enc_update_avx_gen4(aes_ctx, data,
731 walk.dst.virt.addr,
732 walk.src.virt.addr,
733 walk.nbytes);
734 else
735 aesni_gcm_dec_update_avx_gen4(aes_ctx, data,
736 walk.dst.virt.addr,
737 walk.src.virt.addr,
738 walk.nbytes);
739 } else if (static_branch_likely(&gcm_use_avx) && do_avx) {
740 if (enc)
741 aesni_gcm_enc_update_avx_gen2(aes_ctx, data,
742 walk.dst.virt.addr,
743 walk.src.virt.addr,
744 walk.nbytes);
745 else
746 aesni_gcm_dec_update_avx_gen2(aes_ctx, data,
747 walk.dst.virt.addr,
748 walk.src.virt.addr,
749 walk.nbytes);
750 } else if (enc) {
751 aesni_gcm_enc_update(aes_ctx, data, walk.dst.virt.addr,
752 walk.src.virt.addr, walk.nbytes);
753 } else {
754 aesni_gcm_dec_update(aes_ctx, data, walk.dst.virt.addr,
755 walk.src.virt.addr, walk.nbytes);
756 }
757 kernel_fpu_end();
758
759 err = skcipher_walk_done(&walk, 0);
760 }
761
762 if (err)
763 return err;
764
765 kernel_fpu_begin();
766 if (static_branch_likely(&gcm_use_avx2) && do_avx2)
767 aesni_gcm_finalize_avx_gen4(aes_ctx, data, auth_tag,
768 auth_tag_len);
769 else if (static_branch_likely(&gcm_use_avx) && do_avx)
770 aesni_gcm_finalize_avx_gen2(aes_ctx, data, auth_tag,
771 auth_tag_len);
772 else
773 aesni_gcm_finalize(aes_ctx, data, auth_tag, auth_tag_len);
774 kernel_fpu_end();
775
776 return 0;
777 }
778
gcmaes_encrypt(struct aead_request * req,unsigned int assoclen,u8 * hash_subkey,u8 * iv,void * aes_ctx)779 static int gcmaes_encrypt(struct aead_request *req, unsigned int assoclen,
780 u8 *hash_subkey, u8 *iv, void *aes_ctx)
781 {
782 struct crypto_aead *tfm = crypto_aead_reqtfm(req);
783 unsigned long auth_tag_len = crypto_aead_authsize(tfm);
784 u8 auth_tag[16];
785 int err;
786
787 err = gcmaes_crypt_by_sg(true, req, assoclen, hash_subkey, iv, aes_ctx,
788 auth_tag, auth_tag_len);
789 if (err)
790 return err;
791
792 scatterwalk_map_and_copy(auth_tag, req->dst,
793 req->assoclen + req->cryptlen,
794 auth_tag_len, 1);
795 return 0;
796 }
797
gcmaes_decrypt(struct aead_request * req,unsigned int assoclen,u8 * hash_subkey,u8 * iv,void * aes_ctx)798 static int gcmaes_decrypt(struct aead_request *req, unsigned int assoclen,
799 u8 *hash_subkey, u8 *iv, void *aes_ctx)
800 {
801 struct crypto_aead *tfm = crypto_aead_reqtfm(req);
802 unsigned long auth_tag_len = crypto_aead_authsize(tfm);
803 u8 auth_tag_msg[16];
804 u8 auth_tag[16];
805 int err;
806
807 err = gcmaes_crypt_by_sg(false, req, assoclen, hash_subkey, iv, aes_ctx,
808 auth_tag, auth_tag_len);
809 if (err)
810 return err;
811
812 /* Copy out original auth_tag */
813 scatterwalk_map_and_copy(auth_tag_msg, req->src,
814 req->assoclen + req->cryptlen - auth_tag_len,
815 auth_tag_len, 0);
816
817 /* Compare generated tag with passed in tag. */
818 if (crypto_memneq(auth_tag_msg, auth_tag, auth_tag_len)) {
819 memzero_explicit(auth_tag, sizeof(auth_tag));
820 return -EBADMSG;
821 }
822 return 0;
823 }
824
helper_rfc4106_encrypt(struct aead_request * req)825 static int helper_rfc4106_encrypt(struct aead_request *req)
826 {
827 struct crypto_aead *tfm = crypto_aead_reqtfm(req);
828 struct aesni_rfc4106_gcm_ctx *ctx = aesni_rfc4106_gcm_ctx_get(tfm);
829 void *aes_ctx = &(ctx->aes_key_expanded);
830 u8 ivbuf[16 + (AESNI_ALIGN - 8)] __aligned(8);
831 u8 *iv = PTR_ALIGN(&ivbuf[0], AESNI_ALIGN);
832 unsigned int i;
833 __be32 counter = cpu_to_be32(1);
834
835 /* Assuming we are supporting rfc4106 64-bit extended */
836 /* sequence numbers We need to have the AAD length equal */
837 /* to 16 or 20 bytes */
838 if (unlikely(req->assoclen != 16 && req->assoclen != 20))
839 return -EINVAL;
840
841 /* IV below built */
842 for (i = 0; i < 4; i++)
843 *(iv+i) = ctx->nonce[i];
844 for (i = 0; i < 8; i++)
845 *(iv+4+i) = req->iv[i];
846 *((__be32 *)(iv+12)) = counter;
847
848 return gcmaes_encrypt(req, req->assoclen - 8, ctx->hash_subkey, iv,
849 aes_ctx);
850 }
851
helper_rfc4106_decrypt(struct aead_request * req)852 static int helper_rfc4106_decrypt(struct aead_request *req)
853 {
854 __be32 counter = cpu_to_be32(1);
855 struct crypto_aead *tfm = crypto_aead_reqtfm(req);
856 struct aesni_rfc4106_gcm_ctx *ctx = aesni_rfc4106_gcm_ctx_get(tfm);
857 void *aes_ctx = &(ctx->aes_key_expanded);
858 u8 ivbuf[16 + (AESNI_ALIGN - 8)] __aligned(8);
859 u8 *iv = PTR_ALIGN(&ivbuf[0], AESNI_ALIGN);
860 unsigned int i;
861
862 if (unlikely(req->assoclen != 16 && req->assoclen != 20))
863 return -EINVAL;
864
865 /* Assuming we are supporting rfc4106 64-bit extended */
866 /* sequence numbers We need to have the AAD length */
867 /* equal to 16 or 20 bytes */
868
869 /* IV below built */
870 for (i = 0; i < 4; i++)
871 *(iv+i) = ctx->nonce[i];
872 for (i = 0; i < 8; i++)
873 *(iv+4+i) = req->iv[i];
874 *((__be32 *)(iv+12)) = counter;
875
876 return gcmaes_decrypt(req, req->assoclen - 8, ctx->hash_subkey, iv,
877 aes_ctx);
878 }
879 #endif
880
xts_aesni_setkey(struct crypto_skcipher * tfm,const u8 * key,unsigned int keylen)881 static int xts_aesni_setkey(struct crypto_skcipher *tfm, const u8 *key,
882 unsigned int keylen)
883 {
884 struct aesni_xts_ctx *ctx = crypto_skcipher_ctx(tfm);
885 int err;
886
887 err = xts_verify_key(tfm, key, keylen);
888 if (err)
889 return err;
890
891 keylen /= 2;
892
893 /* first half of xts-key is for crypt */
894 err = aes_set_key_common(aes_ctx(ctx->raw_crypt_ctx), key, keylen);
895 if (err)
896 return err;
897
898 /* second half of xts-key is for tweak */
899 return aes_set_key_common(aes_ctx(ctx->raw_tweak_ctx), key + keylen,
900 keylen);
901 }
902
xts_crypt(struct skcipher_request * req,bool encrypt)903 static int xts_crypt(struct skcipher_request *req, bool encrypt)
904 {
905 struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req);
906 struct aesni_xts_ctx *ctx = crypto_skcipher_ctx(tfm);
907 int tail = req->cryptlen % AES_BLOCK_SIZE;
908 struct skcipher_request subreq;
909 struct skcipher_walk walk;
910 int err;
911
912 if (req->cryptlen < AES_BLOCK_SIZE)
913 return -EINVAL;
914
915 err = skcipher_walk_virt(&walk, req, false);
916 if (!walk.nbytes)
917 return err;
918
919 if (unlikely(tail > 0 && walk.nbytes < walk.total)) {
920 int blocks = DIV_ROUND_UP(req->cryptlen, AES_BLOCK_SIZE) - 2;
921
922 skcipher_walk_abort(&walk);
923
924 skcipher_request_set_tfm(&subreq, tfm);
925 skcipher_request_set_callback(&subreq,
926 skcipher_request_flags(req),
927 NULL, NULL);
928 skcipher_request_set_crypt(&subreq, req->src, req->dst,
929 blocks * AES_BLOCK_SIZE, req->iv);
930 req = &subreq;
931
932 err = skcipher_walk_virt(&walk, req, false);
933 if (!walk.nbytes)
934 return err;
935 } else {
936 tail = 0;
937 }
938
939 kernel_fpu_begin();
940
941 /* calculate first value of T */
942 aesni_enc(aes_ctx(ctx->raw_tweak_ctx), walk.iv, walk.iv);
943
944 while (walk.nbytes > 0) {
945 int nbytes = walk.nbytes;
946
947 if (nbytes < walk.total)
948 nbytes &= ~(AES_BLOCK_SIZE - 1);
949
950 if (encrypt)
951 aesni_xts_encrypt(aes_ctx(ctx->raw_crypt_ctx),
952 walk.dst.virt.addr, walk.src.virt.addr,
953 nbytes, walk.iv);
954 else
955 aesni_xts_decrypt(aes_ctx(ctx->raw_crypt_ctx),
956 walk.dst.virt.addr, walk.src.virt.addr,
957 nbytes, walk.iv);
958 kernel_fpu_end();
959
960 err = skcipher_walk_done(&walk, walk.nbytes - nbytes);
961
962 if (walk.nbytes > 0)
963 kernel_fpu_begin();
964 }
965
966 if (unlikely(tail > 0 && !err)) {
967 struct scatterlist sg_src[2], sg_dst[2];
968 struct scatterlist *src, *dst;
969
970 dst = src = scatterwalk_ffwd(sg_src, req->src, req->cryptlen);
971 if (req->dst != req->src)
972 dst = scatterwalk_ffwd(sg_dst, req->dst, req->cryptlen);
973
974 skcipher_request_set_crypt(req, src, dst, AES_BLOCK_SIZE + tail,
975 req->iv);
976
977 err = skcipher_walk_virt(&walk, &subreq, false);
978 if (err)
979 return err;
980
981 kernel_fpu_begin();
982 if (encrypt)
983 aesni_xts_encrypt(aes_ctx(ctx->raw_crypt_ctx),
984 walk.dst.virt.addr, walk.src.virt.addr,
985 walk.nbytes, walk.iv);
986 else
987 aesni_xts_decrypt(aes_ctx(ctx->raw_crypt_ctx),
988 walk.dst.virt.addr, walk.src.virt.addr,
989 walk.nbytes, walk.iv);
990 kernel_fpu_end();
991
992 err = skcipher_walk_done(&walk, 0);
993 }
994 return err;
995 }
996
xts_encrypt(struct skcipher_request * req)997 static int xts_encrypt(struct skcipher_request *req)
998 {
999 return xts_crypt(req, true);
1000 }
1001
xts_decrypt(struct skcipher_request * req)1002 static int xts_decrypt(struct skcipher_request *req)
1003 {
1004 return xts_crypt(req, false);
1005 }
1006
1007 static struct crypto_alg aesni_cipher_alg = {
1008 .cra_name = "aes",
1009 .cra_driver_name = "aes-aesni",
1010 .cra_priority = 300,
1011 .cra_flags = CRYPTO_ALG_TYPE_CIPHER,
1012 .cra_blocksize = AES_BLOCK_SIZE,
1013 .cra_ctxsize = CRYPTO_AES_CTX_SIZE,
1014 .cra_module = THIS_MODULE,
1015 .cra_u = {
1016 .cipher = {
1017 .cia_min_keysize = AES_MIN_KEY_SIZE,
1018 .cia_max_keysize = AES_MAX_KEY_SIZE,
1019 .cia_setkey = aes_set_key,
1020 .cia_encrypt = aesni_encrypt,
1021 .cia_decrypt = aesni_decrypt
1022 }
1023 }
1024 };
1025
1026 static struct skcipher_alg aesni_skciphers[] = {
1027 {
1028 .base = {
1029 .cra_name = "__ecb(aes)",
1030 .cra_driver_name = "__ecb-aes-aesni",
1031 .cra_priority = 400,
1032 .cra_flags = CRYPTO_ALG_INTERNAL,
1033 .cra_blocksize = AES_BLOCK_SIZE,
1034 .cra_ctxsize = CRYPTO_AES_CTX_SIZE,
1035 .cra_module = THIS_MODULE,
1036 },
1037 .min_keysize = AES_MIN_KEY_SIZE,
1038 .max_keysize = AES_MAX_KEY_SIZE,
1039 .setkey = aesni_skcipher_setkey,
1040 .encrypt = ecb_encrypt,
1041 .decrypt = ecb_decrypt,
1042 }, {
1043 .base = {
1044 .cra_name = "__cbc(aes)",
1045 .cra_driver_name = "__cbc-aes-aesni",
1046 .cra_priority = 400,
1047 .cra_flags = CRYPTO_ALG_INTERNAL,
1048 .cra_blocksize = AES_BLOCK_SIZE,
1049 .cra_ctxsize = CRYPTO_AES_CTX_SIZE,
1050 .cra_module = THIS_MODULE,
1051 },
1052 .min_keysize = AES_MIN_KEY_SIZE,
1053 .max_keysize = AES_MAX_KEY_SIZE,
1054 .ivsize = AES_BLOCK_SIZE,
1055 .setkey = aesni_skcipher_setkey,
1056 .encrypt = cbc_encrypt,
1057 .decrypt = cbc_decrypt,
1058 }, {
1059 .base = {
1060 .cra_name = "__cts(cbc(aes))",
1061 .cra_driver_name = "__cts-cbc-aes-aesni",
1062 .cra_priority = 400,
1063 .cra_flags = CRYPTO_ALG_INTERNAL,
1064 .cra_blocksize = AES_BLOCK_SIZE,
1065 .cra_ctxsize = CRYPTO_AES_CTX_SIZE,
1066 .cra_module = THIS_MODULE,
1067 },
1068 .min_keysize = AES_MIN_KEY_SIZE,
1069 .max_keysize = AES_MAX_KEY_SIZE,
1070 .ivsize = AES_BLOCK_SIZE,
1071 .walksize = 2 * AES_BLOCK_SIZE,
1072 .setkey = aesni_skcipher_setkey,
1073 .encrypt = cts_cbc_encrypt,
1074 .decrypt = cts_cbc_decrypt,
1075 #ifdef CONFIG_X86_64
1076 }, {
1077 .base = {
1078 .cra_name = "__ctr(aes)",
1079 .cra_driver_name = "__ctr-aes-aesni",
1080 .cra_priority = 400,
1081 .cra_flags = CRYPTO_ALG_INTERNAL,
1082 .cra_blocksize = 1,
1083 .cra_ctxsize = CRYPTO_AES_CTX_SIZE,
1084 .cra_module = THIS_MODULE,
1085 },
1086 .min_keysize = AES_MIN_KEY_SIZE,
1087 .max_keysize = AES_MAX_KEY_SIZE,
1088 .ivsize = AES_BLOCK_SIZE,
1089 .chunksize = AES_BLOCK_SIZE,
1090 .setkey = aesni_skcipher_setkey,
1091 .encrypt = ctr_crypt,
1092 .decrypt = ctr_crypt,
1093 #endif
1094 }, {
1095 .base = {
1096 .cra_name = "__xts(aes)",
1097 .cra_driver_name = "__xts-aes-aesni",
1098 .cra_priority = 401,
1099 .cra_flags = CRYPTO_ALG_INTERNAL,
1100 .cra_blocksize = AES_BLOCK_SIZE,
1101 .cra_ctxsize = XTS_AES_CTX_SIZE,
1102 .cra_module = THIS_MODULE,
1103 },
1104 .min_keysize = 2 * AES_MIN_KEY_SIZE,
1105 .max_keysize = 2 * AES_MAX_KEY_SIZE,
1106 .ivsize = AES_BLOCK_SIZE,
1107 .walksize = 2 * AES_BLOCK_SIZE,
1108 .setkey = xts_aesni_setkey,
1109 .encrypt = xts_encrypt,
1110 .decrypt = xts_decrypt,
1111 }
1112 };
1113
1114 static
1115 struct simd_skcipher_alg *aesni_simd_skciphers[ARRAY_SIZE(aesni_skciphers)];
1116
1117 #ifdef CONFIG_X86_64
1118 /*
1119 * XCTR does not have a non-AVX implementation, so it must be enabled
1120 * conditionally.
1121 */
1122 static struct skcipher_alg aesni_xctr = {
1123 .base = {
1124 .cra_name = "__xctr(aes)",
1125 .cra_driver_name = "__xctr-aes-aesni",
1126 .cra_priority = 400,
1127 .cra_flags = CRYPTO_ALG_INTERNAL,
1128 .cra_blocksize = 1,
1129 .cra_ctxsize = CRYPTO_AES_CTX_SIZE,
1130 .cra_module = THIS_MODULE,
1131 },
1132 .min_keysize = AES_MIN_KEY_SIZE,
1133 .max_keysize = AES_MAX_KEY_SIZE,
1134 .ivsize = AES_BLOCK_SIZE,
1135 .chunksize = AES_BLOCK_SIZE,
1136 .setkey = aesni_skcipher_setkey,
1137 .encrypt = xctr_crypt,
1138 .decrypt = xctr_crypt,
1139 };
1140
1141 static struct simd_skcipher_alg *aesni_simd_xctr;
1142 #endif /* CONFIG_X86_64 */
1143
1144 #ifdef CONFIG_X86_64
generic_gcmaes_set_key(struct crypto_aead * aead,const u8 * key,unsigned int key_len)1145 static int generic_gcmaes_set_key(struct crypto_aead *aead, const u8 *key,
1146 unsigned int key_len)
1147 {
1148 struct generic_gcmaes_ctx *ctx = generic_gcmaes_ctx_get(aead);
1149
1150 return aes_set_key_common(&ctx->aes_key_expanded, key, key_len) ?:
1151 rfc4106_set_hash_subkey(ctx->hash_subkey, key, key_len);
1152 }
1153
generic_gcmaes_encrypt(struct aead_request * req)1154 static int generic_gcmaes_encrypt(struct aead_request *req)
1155 {
1156 struct crypto_aead *tfm = crypto_aead_reqtfm(req);
1157 struct generic_gcmaes_ctx *ctx = generic_gcmaes_ctx_get(tfm);
1158 void *aes_ctx = &(ctx->aes_key_expanded);
1159 u8 ivbuf[16 + (AESNI_ALIGN - 8)] __aligned(8);
1160 u8 *iv = PTR_ALIGN(&ivbuf[0], AESNI_ALIGN);
1161 __be32 counter = cpu_to_be32(1);
1162
1163 memcpy(iv, req->iv, 12);
1164 *((__be32 *)(iv+12)) = counter;
1165
1166 return gcmaes_encrypt(req, req->assoclen, ctx->hash_subkey, iv,
1167 aes_ctx);
1168 }
1169
generic_gcmaes_decrypt(struct aead_request * req)1170 static int generic_gcmaes_decrypt(struct aead_request *req)
1171 {
1172 __be32 counter = cpu_to_be32(1);
1173 struct crypto_aead *tfm = crypto_aead_reqtfm(req);
1174 struct generic_gcmaes_ctx *ctx = generic_gcmaes_ctx_get(tfm);
1175 void *aes_ctx = &(ctx->aes_key_expanded);
1176 u8 ivbuf[16 + (AESNI_ALIGN - 8)] __aligned(8);
1177 u8 *iv = PTR_ALIGN(&ivbuf[0], AESNI_ALIGN);
1178
1179 memcpy(iv, req->iv, 12);
1180 *((__be32 *)(iv+12)) = counter;
1181
1182 return gcmaes_decrypt(req, req->assoclen, ctx->hash_subkey, iv,
1183 aes_ctx);
1184 }
1185
1186 static struct aead_alg aesni_aeads[] = { {
1187 .setkey = common_rfc4106_set_key,
1188 .setauthsize = common_rfc4106_set_authsize,
1189 .encrypt = helper_rfc4106_encrypt,
1190 .decrypt = helper_rfc4106_decrypt,
1191 .ivsize = GCM_RFC4106_IV_SIZE,
1192 .maxauthsize = 16,
1193 .base = {
1194 .cra_name = "__rfc4106(gcm(aes))",
1195 .cra_driver_name = "__rfc4106-gcm-aesni",
1196 .cra_priority = 400,
1197 .cra_flags = CRYPTO_ALG_INTERNAL,
1198 .cra_blocksize = 1,
1199 .cra_ctxsize = sizeof(struct aesni_rfc4106_gcm_ctx),
1200 .cra_alignmask = 0,
1201 .cra_module = THIS_MODULE,
1202 },
1203 }, {
1204 .setkey = generic_gcmaes_set_key,
1205 .setauthsize = generic_gcmaes_set_authsize,
1206 .encrypt = generic_gcmaes_encrypt,
1207 .decrypt = generic_gcmaes_decrypt,
1208 .ivsize = GCM_AES_IV_SIZE,
1209 .maxauthsize = 16,
1210 .base = {
1211 .cra_name = "__gcm(aes)",
1212 .cra_driver_name = "__generic-gcm-aesni",
1213 .cra_priority = 400,
1214 .cra_flags = CRYPTO_ALG_INTERNAL,
1215 .cra_blocksize = 1,
1216 .cra_ctxsize = sizeof(struct generic_gcmaes_ctx),
1217 .cra_alignmask = 0,
1218 .cra_module = THIS_MODULE,
1219 },
1220 } };
1221 #else
1222 static struct aead_alg aesni_aeads[0];
1223 #endif
1224
1225 static struct simd_aead_alg *aesni_simd_aeads[ARRAY_SIZE(aesni_aeads)];
1226
1227 static const struct x86_cpu_id aesni_cpu_id[] = {
1228 X86_MATCH_FEATURE(X86_FEATURE_AES, NULL),
1229 {}
1230 };
1231 MODULE_DEVICE_TABLE(x86cpu, aesni_cpu_id);
1232
aesni_init(void)1233 static int __init aesni_init(void)
1234 {
1235 int err;
1236
1237 if (!x86_match_cpu(aesni_cpu_id))
1238 return -ENODEV;
1239 #ifdef CONFIG_X86_64
1240 if (boot_cpu_has(X86_FEATURE_AVX2)) {
1241 pr_info("AVX2 version of gcm_enc/dec engaged.\n");
1242 static_branch_enable(&gcm_use_avx);
1243 static_branch_enable(&gcm_use_avx2);
1244 } else
1245 if (boot_cpu_has(X86_FEATURE_AVX)) {
1246 pr_info("AVX version of gcm_enc/dec engaged.\n");
1247 static_branch_enable(&gcm_use_avx);
1248 } else {
1249 pr_info("SSE version of gcm_enc/dec engaged.\n");
1250 }
1251 if (boot_cpu_has(X86_FEATURE_AVX)) {
1252 /* optimize performance of ctr mode encryption transform */
1253 static_call_update(aesni_ctr_enc_tfm, aesni_ctr_enc_avx_tfm);
1254 pr_info("AES CTR mode by8 optimization enabled\n");
1255 }
1256 #endif /* CONFIG_X86_64 */
1257
1258 err = crypto_register_alg(&aesni_cipher_alg);
1259 if (err)
1260 return err;
1261
1262 err = simd_register_skciphers_compat(aesni_skciphers,
1263 ARRAY_SIZE(aesni_skciphers),
1264 aesni_simd_skciphers);
1265 if (err)
1266 goto unregister_cipher;
1267
1268 err = simd_register_aeads_compat(aesni_aeads, ARRAY_SIZE(aesni_aeads),
1269 aesni_simd_aeads);
1270 if (err)
1271 goto unregister_skciphers;
1272
1273 #ifdef CONFIG_X86_64
1274 if (boot_cpu_has(X86_FEATURE_AVX))
1275 err = simd_register_skciphers_compat(&aesni_xctr, 1,
1276 &aesni_simd_xctr);
1277 if (err)
1278 goto unregister_aeads;
1279 #endif /* CONFIG_X86_64 */
1280
1281 return 0;
1282
1283 #ifdef CONFIG_X86_64
1284 unregister_aeads:
1285 simd_unregister_aeads(aesni_aeads, ARRAY_SIZE(aesni_aeads),
1286 aesni_simd_aeads);
1287 #endif /* CONFIG_X86_64 */
1288
1289 unregister_skciphers:
1290 simd_unregister_skciphers(aesni_skciphers, ARRAY_SIZE(aesni_skciphers),
1291 aesni_simd_skciphers);
1292 unregister_cipher:
1293 crypto_unregister_alg(&aesni_cipher_alg);
1294 return err;
1295 }
1296
aesni_exit(void)1297 static void __exit aesni_exit(void)
1298 {
1299 simd_unregister_aeads(aesni_aeads, ARRAY_SIZE(aesni_aeads),
1300 aesni_simd_aeads);
1301 simd_unregister_skciphers(aesni_skciphers, ARRAY_SIZE(aesni_skciphers),
1302 aesni_simd_skciphers);
1303 crypto_unregister_alg(&aesni_cipher_alg);
1304 #ifdef CONFIG_X86_64
1305 if (boot_cpu_has(X86_FEATURE_AVX))
1306 simd_unregister_skciphers(&aesni_xctr, 1, &aesni_simd_xctr);
1307 #endif /* CONFIG_X86_64 */
1308 }
1309
1310 late_initcall(aesni_init);
1311 module_exit(aesni_exit);
1312
1313 MODULE_DESCRIPTION("Rijndael (AES) Cipher Algorithm, Intel AES-NI instructions optimized");
1314 MODULE_LICENSE("GPL");
1315 MODULE_ALIAS_CRYPTO("aes");
1316