xref: /openbmc/linux/arch/arm64/crypto/aes-glue.c (revision 62e59c4e)
1 /*
2  * linux/arch/arm64/crypto/aes-glue.c - wrapper code for ARMv8 AES
3  *
4  * Copyright (C) 2013 - 2017 Linaro Ltd <ard.biesheuvel@linaro.org>
5  *
6  * This program is free software; you can redistribute it and/or modify
7  * it under the terms of the GNU General Public License version 2 as
8  * published by the Free Software Foundation.
9  */
10 
11 #include <asm/neon.h>
12 #include <asm/hwcap.h>
13 #include <asm/simd.h>
14 #include <crypto/aes.h>
15 #include <crypto/internal/hash.h>
16 #include <crypto/internal/simd.h>
17 #include <crypto/internal/skcipher.h>
18 #include <crypto/scatterwalk.h>
19 #include <linux/module.h>
20 #include <linux/cpufeature.h>
21 #include <crypto/xts.h>
22 
23 #include "aes-ce-setkey.h"
24 #include "aes-ctr-fallback.h"
25 
26 #ifdef USE_V8_CRYPTO_EXTENSIONS
27 #define MODE			"ce"
28 #define PRIO			300
29 #define aes_setkey		ce_aes_setkey
30 #define aes_expandkey		ce_aes_expandkey
31 #define aes_ecb_encrypt		ce_aes_ecb_encrypt
32 #define aes_ecb_decrypt		ce_aes_ecb_decrypt
33 #define aes_cbc_encrypt		ce_aes_cbc_encrypt
34 #define aes_cbc_decrypt		ce_aes_cbc_decrypt
35 #define aes_cbc_cts_encrypt	ce_aes_cbc_cts_encrypt
36 #define aes_cbc_cts_decrypt	ce_aes_cbc_cts_decrypt
37 #define aes_ctr_encrypt		ce_aes_ctr_encrypt
38 #define aes_xts_encrypt		ce_aes_xts_encrypt
39 #define aes_xts_decrypt		ce_aes_xts_decrypt
40 #define aes_mac_update		ce_aes_mac_update
41 MODULE_DESCRIPTION("AES-ECB/CBC/CTR/XTS using ARMv8 Crypto Extensions");
42 #else
43 #define MODE			"neon"
44 #define PRIO			200
45 #define aes_setkey		crypto_aes_set_key
46 #define aes_expandkey		crypto_aes_expand_key
47 #define aes_ecb_encrypt		neon_aes_ecb_encrypt
48 #define aes_ecb_decrypt		neon_aes_ecb_decrypt
49 #define aes_cbc_encrypt		neon_aes_cbc_encrypt
50 #define aes_cbc_decrypt		neon_aes_cbc_decrypt
51 #define aes_cbc_cts_encrypt	neon_aes_cbc_cts_encrypt
52 #define aes_cbc_cts_decrypt	neon_aes_cbc_cts_decrypt
53 #define aes_ctr_encrypt		neon_aes_ctr_encrypt
54 #define aes_xts_encrypt		neon_aes_xts_encrypt
55 #define aes_xts_decrypt		neon_aes_xts_decrypt
56 #define aes_mac_update		neon_aes_mac_update
57 MODULE_DESCRIPTION("AES-ECB/CBC/CTR/XTS using ARMv8 NEON");
58 MODULE_ALIAS_CRYPTO("ecb(aes)");
59 MODULE_ALIAS_CRYPTO("cbc(aes)");
60 MODULE_ALIAS_CRYPTO("ctr(aes)");
61 MODULE_ALIAS_CRYPTO("xts(aes)");
62 MODULE_ALIAS_CRYPTO("cmac(aes)");
63 MODULE_ALIAS_CRYPTO("xcbc(aes)");
64 MODULE_ALIAS_CRYPTO("cbcmac(aes)");
65 #endif
66 
67 MODULE_AUTHOR("Ard Biesheuvel <ard.biesheuvel@linaro.org>");
68 MODULE_LICENSE("GPL v2");
69 
70 /* defined in aes-modes.S */
71 asmlinkage void aes_ecb_encrypt(u8 out[], u8 const in[], u32 const rk[],
72 				int rounds, int blocks);
73 asmlinkage void aes_ecb_decrypt(u8 out[], u8 const in[], u32 const rk[],
74 				int rounds, int blocks);
75 
76 asmlinkage void aes_cbc_encrypt(u8 out[], u8 const in[], u32 const rk[],
77 				int rounds, int blocks, u8 iv[]);
78 asmlinkage void aes_cbc_decrypt(u8 out[], u8 const in[], u32 const rk[],
79 				int rounds, int blocks, u8 iv[]);
80 
81 asmlinkage void aes_cbc_cts_encrypt(u8 out[], u8 const in[], u32 const rk[],
82 				int rounds, int bytes, u8 const iv[]);
83 asmlinkage void aes_cbc_cts_decrypt(u8 out[], u8 const in[], u32 const rk[],
84 				int rounds, int bytes, u8 const iv[]);
85 
86 asmlinkage void aes_ctr_encrypt(u8 out[], u8 const in[], u32 const rk[],
87 				int rounds, int blocks, u8 ctr[]);
88 
89 asmlinkage void aes_xts_encrypt(u8 out[], u8 const in[], u32 const rk1[],
90 				int rounds, int blocks, u32 const rk2[], u8 iv[],
91 				int first);
92 asmlinkage void aes_xts_decrypt(u8 out[], u8 const in[], u32 const rk1[],
93 				int rounds, int blocks, u32 const rk2[], u8 iv[],
94 				int first);
95 
96 asmlinkage void aes_mac_update(u8 const in[], u32 const rk[], int rounds,
97 			       int blocks, u8 dg[], int enc_before,
98 			       int enc_after);
99 
100 struct cts_cbc_req_ctx {
101 	struct scatterlist sg_src[2];
102 	struct scatterlist sg_dst[2];
103 	struct skcipher_request subreq;
104 };
105 
106 struct crypto_aes_xts_ctx {
107 	struct crypto_aes_ctx key1;
108 	struct crypto_aes_ctx __aligned(8) key2;
109 };
110 
111 struct mac_tfm_ctx {
112 	struct crypto_aes_ctx key;
113 	u8 __aligned(8) consts[];
114 };
115 
116 struct mac_desc_ctx {
117 	unsigned int len;
118 	u8 dg[AES_BLOCK_SIZE];
119 };
120 
121 static int skcipher_aes_setkey(struct crypto_skcipher *tfm, const u8 *in_key,
122 			       unsigned int key_len)
123 {
124 	return aes_setkey(crypto_skcipher_tfm(tfm), in_key, key_len);
125 }
126 
127 static int xts_set_key(struct crypto_skcipher *tfm, const u8 *in_key,
128 		       unsigned int key_len)
129 {
130 	struct crypto_aes_xts_ctx *ctx = crypto_skcipher_ctx(tfm);
131 	int ret;
132 
133 	ret = xts_verify_key(tfm, in_key, key_len);
134 	if (ret)
135 		return ret;
136 
137 	ret = aes_expandkey(&ctx->key1, in_key, key_len / 2);
138 	if (!ret)
139 		ret = aes_expandkey(&ctx->key2, &in_key[key_len / 2],
140 				    key_len / 2);
141 	if (!ret)
142 		return 0;
143 
144 	crypto_skcipher_set_flags(tfm, CRYPTO_TFM_RES_BAD_KEY_LEN);
145 	return -EINVAL;
146 }
147 
148 static int ecb_encrypt(struct skcipher_request *req)
149 {
150 	struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req);
151 	struct crypto_aes_ctx *ctx = crypto_skcipher_ctx(tfm);
152 	int err, rounds = 6 + ctx->key_length / 4;
153 	struct skcipher_walk walk;
154 	unsigned int blocks;
155 
156 	err = skcipher_walk_virt(&walk, req, false);
157 
158 	while ((blocks = (walk.nbytes / AES_BLOCK_SIZE))) {
159 		kernel_neon_begin();
160 		aes_ecb_encrypt(walk.dst.virt.addr, walk.src.virt.addr,
161 				ctx->key_enc, rounds, blocks);
162 		kernel_neon_end();
163 		err = skcipher_walk_done(&walk, walk.nbytes % AES_BLOCK_SIZE);
164 	}
165 	return err;
166 }
167 
168 static int ecb_decrypt(struct skcipher_request *req)
169 {
170 	struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req);
171 	struct crypto_aes_ctx *ctx = crypto_skcipher_ctx(tfm);
172 	int err, rounds = 6 + ctx->key_length / 4;
173 	struct skcipher_walk walk;
174 	unsigned int blocks;
175 
176 	err = skcipher_walk_virt(&walk, req, false);
177 
178 	while ((blocks = (walk.nbytes / AES_BLOCK_SIZE))) {
179 		kernel_neon_begin();
180 		aes_ecb_decrypt(walk.dst.virt.addr, walk.src.virt.addr,
181 				ctx->key_dec, rounds, blocks);
182 		kernel_neon_end();
183 		err = skcipher_walk_done(&walk, walk.nbytes % AES_BLOCK_SIZE);
184 	}
185 	return err;
186 }
187 
188 static int cbc_encrypt(struct skcipher_request *req)
189 {
190 	struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req);
191 	struct crypto_aes_ctx *ctx = crypto_skcipher_ctx(tfm);
192 	int err, rounds = 6 + ctx->key_length / 4;
193 	struct skcipher_walk walk;
194 	unsigned int blocks;
195 
196 	err = skcipher_walk_virt(&walk, req, false);
197 
198 	while ((blocks = (walk.nbytes / AES_BLOCK_SIZE))) {
199 		kernel_neon_begin();
200 		aes_cbc_encrypt(walk.dst.virt.addr, walk.src.virt.addr,
201 				ctx->key_enc, rounds, blocks, walk.iv);
202 		kernel_neon_end();
203 		err = skcipher_walk_done(&walk, walk.nbytes % AES_BLOCK_SIZE);
204 	}
205 	return err;
206 }
207 
208 static int cbc_decrypt(struct skcipher_request *req)
209 {
210 	struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req);
211 	struct crypto_aes_ctx *ctx = crypto_skcipher_ctx(tfm);
212 	int err, rounds = 6 + ctx->key_length / 4;
213 	struct skcipher_walk walk;
214 	unsigned int blocks;
215 
216 	err = skcipher_walk_virt(&walk, req, false);
217 
218 	while ((blocks = (walk.nbytes / AES_BLOCK_SIZE))) {
219 		kernel_neon_begin();
220 		aes_cbc_decrypt(walk.dst.virt.addr, walk.src.virt.addr,
221 				ctx->key_dec, rounds, blocks, walk.iv);
222 		kernel_neon_end();
223 		err = skcipher_walk_done(&walk, walk.nbytes % AES_BLOCK_SIZE);
224 	}
225 	return err;
226 }
227 
228 static int cts_cbc_init_tfm(struct crypto_skcipher *tfm)
229 {
230 	crypto_skcipher_set_reqsize(tfm, sizeof(struct cts_cbc_req_ctx));
231 	return 0;
232 }
233 
234 static int cts_cbc_encrypt(struct skcipher_request *req)
235 {
236 	struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req);
237 	struct crypto_aes_ctx *ctx = crypto_skcipher_ctx(tfm);
238 	struct cts_cbc_req_ctx *rctx = skcipher_request_ctx(req);
239 	int err, rounds = 6 + ctx->key_length / 4;
240 	int cbc_blocks = DIV_ROUND_UP(req->cryptlen, AES_BLOCK_SIZE) - 2;
241 	struct scatterlist *src = req->src, *dst = req->dst;
242 	struct skcipher_walk walk;
243 
244 	skcipher_request_set_tfm(&rctx->subreq, tfm);
245 
246 	if (req->cryptlen <= AES_BLOCK_SIZE) {
247 		if (req->cryptlen < AES_BLOCK_SIZE)
248 			return -EINVAL;
249 		cbc_blocks = 1;
250 	}
251 
252 	if (cbc_blocks > 0) {
253 		unsigned int blocks;
254 
255 		skcipher_request_set_crypt(&rctx->subreq, req->src, req->dst,
256 					   cbc_blocks * AES_BLOCK_SIZE,
257 					   req->iv);
258 
259 		err = skcipher_walk_virt(&walk, &rctx->subreq, false);
260 
261 		while ((blocks = (walk.nbytes / AES_BLOCK_SIZE))) {
262 			kernel_neon_begin();
263 			aes_cbc_encrypt(walk.dst.virt.addr, walk.src.virt.addr,
264 					ctx->key_enc, rounds, blocks, walk.iv);
265 			kernel_neon_end();
266 			err = skcipher_walk_done(&walk,
267 						 walk.nbytes % AES_BLOCK_SIZE);
268 		}
269 		if (err)
270 			return err;
271 
272 		if (req->cryptlen == AES_BLOCK_SIZE)
273 			return 0;
274 
275 		dst = src = scatterwalk_ffwd(rctx->sg_src, req->src,
276 					     rctx->subreq.cryptlen);
277 		if (req->dst != req->src)
278 			dst = scatterwalk_ffwd(rctx->sg_dst, req->dst,
279 					       rctx->subreq.cryptlen);
280 	}
281 
282 	/* handle ciphertext stealing */
283 	skcipher_request_set_crypt(&rctx->subreq, src, dst,
284 				   req->cryptlen - cbc_blocks * AES_BLOCK_SIZE,
285 				   req->iv);
286 
287 	err = skcipher_walk_virt(&walk, &rctx->subreq, false);
288 	if (err)
289 		return err;
290 
291 	kernel_neon_begin();
292 	aes_cbc_cts_encrypt(walk.dst.virt.addr, walk.src.virt.addr,
293 			    ctx->key_enc, rounds, walk.nbytes, walk.iv);
294 	kernel_neon_end();
295 
296 	return skcipher_walk_done(&walk, 0);
297 }
298 
299 static int cts_cbc_decrypt(struct skcipher_request *req)
300 {
301 	struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req);
302 	struct crypto_aes_ctx *ctx = crypto_skcipher_ctx(tfm);
303 	struct cts_cbc_req_ctx *rctx = skcipher_request_ctx(req);
304 	int err, rounds = 6 + ctx->key_length / 4;
305 	int cbc_blocks = DIV_ROUND_UP(req->cryptlen, AES_BLOCK_SIZE) - 2;
306 	struct scatterlist *src = req->src, *dst = req->dst;
307 	struct skcipher_walk walk;
308 
309 	skcipher_request_set_tfm(&rctx->subreq, tfm);
310 
311 	if (req->cryptlen <= AES_BLOCK_SIZE) {
312 		if (req->cryptlen < AES_BLOCK_SIZE)
313 			return -EINVAL;
314 		cbc_blocks = 1;
315 	}
316 
317 	if (cbc_blocks > 0) {
318 		unsigned int blocks;
319 
320 		skcipher_request_set_crypt(&rctx->subreq, req->src, req->dst,
321 					   cbc_blocks * AES_BLOCK_SIZE,
322 					   req->iv);
323 
324 		err = skcipher_walk_virt(&walk, &rctx->subreq, false);
325 
326 		while ((blocks = (walk.nbytes / AES_BLOCK_SIZE))) {
327 			kernel_neon_begin();
328 			aes_cbc_decrypt(walk.dst.virt.addr, walk.src.virt.addr,
329 					ctx->key_dec, rounds, blocks, walk.iv);
330 			kernel_neon_end();
331 			err = skcipher_walk_done(&walk,
332 						 walk.nbytes % AES_BLOCK_SIZE);
333 		}
334 		if (err)
335 			return err;
336 
337 		if (req->cryptlen == AES_BLOCK_SIZE)
338 			return 0;
339 
340 		dst = src = scatterwalk_ffwd(rctx->sg_src, req->src,
341 					     rctx->subreq.cryptlen);
342 		if (req->dst != req->src)
343 			dst = scatterwalk_ffwd(rctx->sg_dst, req->dst,
344 					       rctx->subreq.cryptlen);
345 	}
346 
347 	/* handle ciphertext stealing */
348 	skcipher_request_set_crypt(&rctx->subreq, src, dst,
349 				   req->cryptlen - cbc_blocks * AES_BLOCK_SIZE,
350 				   req->iv);
351 
352 	err = skcipher_walk_virt(&walk, &rctx->subreq, false);
353 	if (err)
354 		return err;
355 
356 	kernel_neon_begin();
357 	aes_cbc_cts_decrypt(walk.dst.virt.addr, walk.src.virt.addr,
358 			    ctx->key_dec, rounds, walk.nbytes, walk.iv);
359 	kernel_neon_end();
360 
361 	return skcipher_walk_done(&walk, 0);
362 }
363 
364 static int ctr_encrypt(struct skcipher_request *req)
365 {
366 	struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req);
367 	struct crypto_aes_ctx *ctx = crypto_skcipher_ctx(tfm);
368 	int err, rounds = 6 + ctx->key_length / 4;
369 	struct skcipher_walk walk;
370 	int blocks;
371 
372 	err = skcipher_walk_virt(&walk, req, false);
373 
374 	while ((blocks = (walk.nbytes / AES_BLOCK_SIZE))) {
375 		kernel_neon_begin();
376 		aes_ctr_encrypt(walk.dst.virt.addr, walk.src.virt.addr,
377 				ctx->key_enc, rounds, blocks, walk.iv);
378 		kernel_neon_end();
379 		err = skcipher_walk_done(&walk, walk.nbytes % AES_BLOCK_SIZE);
380 	}
381 	if (walk.nbytes) {
382 		u8 __aligned(8) tail[AES_BLOCK_SIZE];
383 		unsigned int nbytes = walk.nbytes;
384 		u8 *tdst = walk.dst.virt.addr;
385 		u8 *tsrc = walk.src.virt.addr;
386 
387 		/*
388 		 * Tell aes_ctr_encrypt() to process a tail block.
389 		 */
390 		blocks = -1;
391 
392 		kernel_neon_begin();
393 		aes_ctr_encrypt(tail, NULL, ctx->key_enc, rounds,
394 				blocks, walk.iv);
395 		kernel_neon_end();
396 		crypto_xor_cpy(tdst, tsrc, tail, nbytes);
397 		err = skcipher_walk_done(&walk, 0);
398 	}
399 
400 	return err;
401 }
402 
403 static int ctr_encrypt_sync(struct skcipher_request *req)
404 {
405 	struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req);
406 	struct crypto_aes_ctx *ctx = crypto_skcipher_ctx(tfm);
407 
408 	if (!may_use_simd())
409 		return aes_ctr_encrypt_fallback(ctx, req);
410 
411 	return ctr_encrypt(req);
412 }
413 
414 static int xts_encrypt(struct skcipher_request *req)
415 {
416 	struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req);
417 	struct crypto_aes_xts_ctx *ctx = crypto_skcipher_ctx(tfm);
418 	int err, first, rounds = 6 + ctx->key1.key_length / 4;
419 	struct skcipher_walk walk;
420 	unsigned int blocks;
421 
422 	err = skcipher_walk_virt(&walk, req, false);
423 
424 	for (first = 1; (blocks = (walk.nbytes / AES_BLOCK_SIZE)); first = 0) {
425 		kernel_neon_begin();
426 		aes_xts_encrypt(walk.dst.virt.addr, walk.src.virt.addr,
427 				ctx->key1.key_enc, rounds, blocks,
428 				ctx->key2.key_enc, walk.iv, first);
429 		kernel_neon_end();
430 		err = skcipher_walk_done(&walk, walk.nbytes % AES_BLOCK_SIZE);
431 	}
432 
433 	return err;
434 }
435 
436 static int xts_decrypt(struct skcipher_request *req)
437 {
438 	struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req);
439 	struct crypto_aes_xts_ctx *ctx = crypto_skcipher_ctx(tfm);
440 	int err, first, rounds = 6 + ctx->key1.key_length / 4;
441 	struct skcipher_walk walk;
442 	unsigned int blocks;
443 
444 	err = skcipher_walk_virt(&walk, req, false);
445 
446 	for (first = 1; (blocks = (walk.nbytes / AES_BLOCK_SIZE)); first = 0) {
447 		kernel_neon_begin();
448 		aes_xts_decrypt(walk.dst.virt.addr, walk.src.virt.addr,
449 				ctx->key1.key_dec, rounds, blocks,
450 				ctx->key2.key_enc, walk.iv, first);
451 		kernel_neon_end();
452 		err = skcipher_walk_done(&walk, walk.nbytes % AES_BLOCK_SIZE);
453 	}
454 
455 	return err;
456 }
457 
458 static struct skcipher_alg aes_algs[] = { {
459 	.base = {
460 		.cra_name		= "__ecb(aes)",
461 		.cra_driver_name	= "__ecb-aes-" MODE,
462 		.cra_priority		= PRIO,
463 		.cra_flags		= CRYPTO_ALG_INTERNAL,
464 		.cra_blocksize		= AES_BLOCK_SIZE,
465 		.cra_ctxsize		= sizeof(struct crypto_aes_ctx),
466 		.cra_module		= THIS_MODULE,
467 	},
468 	.min_keysize	= AES_MIN_KEY_SIZE,
469 	.max_keysize	= AES_MAX_KEY_SIZE,
470 	.setkey		= skcipher_aes_setkey,
471 	.encrypt	= ecb_encrypt,
472 	.decrypt	= ecb_decrypt,
473 }, {
474 	.base = {
475 		.cra_name		= "__cbc(aes)",
476 		.cra_driver_name	= "__cbc-aes-" MODE,
477 		.cra_priority		= PRIO,
478 		.cra_flags		= CRYPTO_ALG_INTERNAL,
479 		.cra_blocksize		= AES_BLOCK_SIZE,
480 		.cra_ctxsize		= sizeof(struct crypto_aes_ctx),
481 		.cra_module		= THIS_MODULE,
482 	},
483 	.min_keysize	= AES_MIN_KEY_SIZE,
484 	.max_keysize	= AES_MAX_KEY_SIZE,
485 	.ivsize		= AES_BLOCK_SIZE,
486 	.setkey		= skcipher_aes_setkey,
487 	.encrypt	= cbc_encrypt,
488 	.decrypt	= cbc_decrypt,
489 }, {
490 	.base = {
491 		.cra_name		= "__cts(cbc(aes))",
492 		.cra_driver_name	= "__cts-cbc-aes-" MODE,
493 		.cra_priority		= PRIO,
494 		.cra_flags		= CRYPTO_ALG_INTERNAL,
495 		.cra_blocksize		= AES_BLOCK_SIZE,
496 		.cra_ctxsize		= sizeof(struct crypto_aes_ctx),
497 		.cra_module		= THIS_MODULE,
498 	},
499 	.min_keysize	= AES_MIN_KEY_SIZE,
500 	.max_keysize	= AES_MAX_KEY_SIZE,
501 	.ivsize		= AES_BLOCK_SIZE,
502 	.walksize	= 2 * AES_BLOCK_SIZE,
503 	.setkey		= skcipher_aes_setkey,
504 	.encrypt	= cts_cbc_encrypt,
505 	.decrypt	= cts_cbc_decrypt,
506 	.init		= cts_cbc_init_tfm,
507 }, {
508 	.base = {
509 		.cra_name		= "__ctr(aes)",
510 		.cra_driver_name	= "__ctr-aes-" MODE,
511 		.cra_priority		= PRIO,
512 		.cra_flags		= CRYPTO_ALG_INTERNAL,
513 		.cra_blocksize		= 1,
514 		.cra_ctxsize		= sizeof(struct crypto_aes_ctx),
515 		.cra_module		= THIS_MODULE,
516 	},
517 	.min_keysize	= AES_MIN_KEY_SIZE,
518 	.max_keysize	= AES_MAX_KEY_SIZE,
519 	.ivsize		= AES_BLOCK_SIZE,
520 	.chunksize	= AES_BLOCK_SIZE,
521 	.setkey		= skcipher_aes_setkey,
522 	.encrypt	= ctr_encrypt,
523 	.decrypt	= ctr_encrypt,
524 }, {
525 	.base = {
526 		.cra_name		= "ctr(aes)",
527 		.cra_driver_name	= "ctr-aes-" MODE,
528 		.cra_priority		= PRIO - 1,
529 		.cra_blocksize		= 1,
530 		.cra_ctxsize		= sizeof(struct crypto_aes_ctx),
531 		.cra_module		= THIS_MODULE,
532 	},
533 	.min_keysize	= AES_MIN_KEY_SIZE,
534 	.max_keysize	= AES_MAX_KEY_SIZE,
535 	.ivsize		= AES_BLOCK_SIZE,
536 	.chunksize	= AES_BLOCK_SIZE,
537 	.setkey		= skcipher_aes_setkey,
538 	.encrypt	= ctr_encrypt_sync,
539 	.decrypt	= ctr_encrypt_sync,
540 }, {
541 	.base = {
542 		.cra_name		= "__xts(aes)",
543 		.cra_driver_name	= "__xts-aes-" MODE,
544 		.cra_priority		= PRIO,
545 		.cra_flags		= CRYPTO_ALG_INTERNAL,
546 		.cra_blocksize		= AES_BLOCK_SIZE,
547 		.cra_ctxsize		= sizeof(struct crypto_aes_xts_ctx),
548 		.cra_module		= THIS_MODULE,
549 	},
550 	.min_keysize	= 2 * AES_MIN_KEY_SIZE,
551 	.max_keysize	= 2 * AES_MAX_KEY_SIZE,
552 	.ivsize		= AES_BLOCK_SIZE,
553 	.setkey		= xts_set_key,
554 	.encrypt	= xts_encrypt,
555 	.decrypt	= xts_decrypt,
556 } };
557 
558 static int cbcmac_setkey(struct crypto_shash *tfm, const u8 *in_key,
559 			 unsigned int key_len)
560 {
561 	struct mac_tfm_ctx *ctx = crypto_shash_ctx(tfm);
562 	int err;
563 
564 	err = aes_expandkey(&ctx->key, in_key, key_len);
565 	if (err)
566 		crypto_shash_set_flags(tfm, CRYPTO_TFM_RES_BAD_KEY_LEN);
567 
568 	return err;
569 }
570 
571 static void cmac_gf128_mul_by_x(be128 *y, const be128 *x)
572 {
573 	u64 a = be64_to_cpu(x->a);
574 	u64 b = be64_to_cpu(x->b);
575 
576 	y->a = cpu_to_be64((a << 1) | (b >> 63));
577 	y->b = cpu_to_be64((b << 1) ^ ((a >> 63) ? 0x87 : 0));
578 }
579 
580 static int cmac_setkey(struct crypto_shash *tfm, const u8 *in_key,
581 		       unsigned int key_len)
582 {
583 	struct mac_tfm_ctx *ctx = crypto_shash_ctx(tfm);
584 	be128 *consts = (be128 *)ctx->consts;
585 	int rounds = 6 + key_len / 4;
586 	int err;
587 
588 	err = cbcmac_setkey(tfm, in_key, key_len);
589 	if (err)
590 		return err;
591 
592 	/* encrypt the zero vector */
593 	kernel_neon_begin();
594 	aes_ecb_encrypt(ctx->consts, (u8[AES_BLOCK_SIZE]){}, ctx->key.key_enc,
595 			rounds, 1);
596 	kernel_neon_end();
597 
598 	cmac_gf128_mul_by_x(consts, consts);
599 	cmac_gf128_mul_by_x(consts + 1, consts);
600 
601 	return 0;
602 }
603 
604 static int xcbc_setkey(struct crypto_shash *tfm, const u8 *in_key,
605 		       unsigned int key_len)
606 {
607 	static u8 const ks[3][AES_BLOCK_SIZE] = {
608 		{ [0 ... AES_BLOCK_SIZE - 1] = 0x1 },
609 		{ [0 ... AES_BLOCK_SIZE - 1] = 0x2 },
610 		{ [0 ... AES_BLOCK_SIZE - 1] = 0x3 },
611 	};
612 
613 	struct mac_tfm_ctx *ctx = crypto_shash_ctx(tfm);
614 	int rounds = 6 + key_len / 4;
615 	u8 key[AES_BLOCK_SIZE];
616 	int err;
617 
618 	err = cbcmac_setkey(tfm, in_key, key_len);
619 	if (err)
620 		return err;
621 
622 	kernel_neon_begin();
623 	aes_ecb_encrypt(key, ks[0], ctx->key.key_enc, rounds, 1);
624 	aes_ecb_encrypt(ctx->consts, ks[1], ctx->key.key_enc, rounds, 2);
625 	kernel_neon_end();
626 
627 	return cbcmac_setkey(tfm, key, sizeof(key));
628 }
629 
630 static int mac_init(struct shash_desc *desc)
631 {
632 	struct mac_desc_ctx *ctx = shash_desc_ctx(desc);
633 
634 	memset(ctx->dg, 0, AES_BLOCK_SIZE);
635 	ctx->len = 0;
636 
637 	return 0;
638 }
639 
640 static void mac_do_update(struct crypto_aes_ctx *ctx, u8 const in[], int blocks,
641 			  u8 dg[], int enc_before, int enc_after)
642 {
643 	int rounds = 6 + ctx->key_length / 4;
644 
645 	if (may_use_simd()) {
646 		kernel_neon_begin();
647 		aes_mac_update(in, ctx->key_enc, rounds, blocks, dg, enc_before,
648 			       enc_after);
649 		kernel_neon_end();
650 	} else {
651 		if (enc_before)
652 			__aes_arm64_encrypt(ctx->key_enc, dg, dg, rounds);
653 
654 		while (blocks--) {
655 			crypto_xor(dg, in, AES_BLOCK_SIZE);
656 			in += AES_BLOCK_SIZE;
657 
658 			if (blocks || enc_after)
659 				__aes_arm64_encrypt(ctx->key_enc, dg, dg,
660 						    rounds);
661 		}
662 	}
663 }
664 
665 static int mac_update(struct shash_desc *desc, const u8 *p, unsigned int len)
666 {
667 	struct mac_tfm_ctx *tctx = crypto_shash_ctx(desc->tfm);
668 	struct mac_desc_ctx *ctx = shash_desc_ctx(desc);
669 
670 	while (len > 0) {
671 		unsigned int l;
672 
673 		if ((ctx->len % AES_BLOCK_SIZE) == 0 &&
674 		    (ctx->len + len) > AES_BLOCK_SIZE) {
675 
676 			int blocks = len / AES_BLOCK_SIZE;
677 
678 			len %= AES_BLOCK_SIZE;
679 
680 			mac_do_update(&tctx->key, p, blocks, ctx->dg,
681 				      (ctx->len != 0), (len != 0));
682 
683 			p += blocks * AES_BLOCK_SIZE;
684 
685 			if (!len) {
686 				ctx->len = AES_BLOCK_SIZE;
687 				break;
688 			}
689 			ctx->len = 0;
690 		}
691 
692 		l = min(len, AES_BLOCK_SIZE - ctx->len);
693 
694 		if (l <= AES_BLOCK_SIZE) {
695 			crypto_xor(ctx->dg + ctx->len, p, l);
696 			ctx->len += l;
697 			len -= l;
698 			p += l;
699 		}
700 	}
701 
702 	return 0;
703 }
704 
705 static int cbcmac_final(struct shash_desc *desc, u8 *out)
706 {
707 	struct mac_tfm_ctx *tctx = crypto_shash_ctx(desc->tfm);
708 	struct mac_desc_ctx *ctx = shash_desc_ctx(desc);
709 
710 	mac_do_update(&tctx->key, NULL, 0, ctx->dg, 1, 0);
711 
712 	memcpy(out, ctx->dg, AES_BLOCK_SIZE);
713 
714 	return 0;
715 }
716 
717 static int cmac_final(struct shash_desc *desc, u8 *out)
718 {
719 	struct mac_tfm_ctx *tctx = crypto_shash_ctx(desc->tfm);
720 	struct mac_desc_ctx *ctx = shash_desc_ctx(desc);
721 	u8 *consts = tctx->consts;
722 
723 	if (ctx->len != AES_BLOCK_SIZE) {
724 		ctx->dg[ctx->len] ^= 0x80;
725 		consts += AES_BLOCK_SIZE;
726 	}
727 
728 	mac_do_update(&tctx->key, consts, 1, ctx->dg, 0, 1);
729 
730 	memcpy(out, ctx->dg, AES_BLOCK_SIZE);
731 
732 	return 0;
733 }
734 
735 static struct shash_alg mac_algs[] = { {
736 	.base.cra_name		= "cmac(aes)",
737 	.base.cra_driver_name	= "cmac-aes-" MODE,
738 	.base.cra_priority	= PRIO,
739 	.base.cra_blocksize	= AES_BLOCK_SIZE,
740 	.base.cra_ctxsize	= sizeof(struct mac_tfm_ctx) +
741 				  2 * AES_BLOCK_SIZE,
742 	.base.cra_module	= THIS_MODULE,
743 
744 	.digestsize		= AES_BLOCK_SIZE,
745 	.init			= mac_init,
746 	.update			= mac_update,
747 	.final			= cmac_final,
748 	.setkey			= cmac_setkey,
749 	.descsize		= sizeof(struct mac_desc_ctx),
750 }, {
751 	.base.cra_name		= "xcbc(aes)",
752 	.base.cra_driver_name	= "xcbc-aes-" MODE,
753 	.base.cra_priority	= PRIO,
754 	.base.cra_blocksize	= AES_BLOCK_SIZE,
755 	.base.cra_ctxsize	= sizeof(struct mac_tfm_ctx) +
756 				  2 * AES_BLOCK_SIZE,
757 	.base.cra_module	= THIS_MODULE,
758 
759 	.digestsize		= AES_BLOCK_SIZE,
760 	.init			= mac_init,
761 	.update			= mac_update,
762 	.final			= cmac_final,
763 	.setkey			= xcbc_setkey,
764 	.descsize		= sizeof(struct mac_desc_ctx),
765 }, {
766 	.base.cra_name		= "cbcmac(aes)",
767 	.base.cra_driver_name	= "cbcmac-aes-" MODE,
768 	.base.cra_priority	= PRIO,
769 	.base.cra_blocksize	= 1,
770 	.base.cra_ctxsize	= sizeof(struct mac_tfm_ctx),
771 	.base.cra_module	= THIS_MODULE,
772 
773 	.digestsize		= AES_BLOCK_SIZE,
774 	.init			= mac_init,
775 	.update			= mac_update,
776 	.final			= cbcmac_final,
777 	.setkey			= cbcmac_setkey,
778 	.descsize		= sizeof(struct mac_desc_ctx),
779 } };
780 
781 static struct simd_skcipher_alg *aes_simd_algs[ARRAY_SIZE(aes_algs)];
782 
783 static void aes_exit(void)
784 {
785 	int i;
786 
787 	for (i = 0; i < ARRAY_SIZE(aes_simd_algs); i++)
788 		if (aes_simd_algs[i])
789 			simd_skcipher_free(aes_simd_algs[i]);
790 
791 	crypto_unregister_shashes(mac_algs, ARRAY_SIZE(mac_algs));
792 	crypto_unregister_skciphers(aes_algs, ARRAY_SIZE(aes_algs));
793 }
794 
795 static int __init aes_init(void)
796 {
797 	struct simd_skcipher_alg *simd;
798 	const char *basename;
799 	const char *algname;
800 	const char *drvname;
801 	int err;
802 	int i;
803 
804 	err = crypto_register_skciphers(aes_algs, ARRAY_SIZE(aes_algs));
805 	if (err)
806 		return err;
807 
808 	err = crypto_register_shashes(mac_algs, ARRAY_SIZE(mac_algs));
809 	if (err)
810 		goto unregister_ciphers;
811 
812 	for (i = 0; i < ARRAY_SIZE(aes_algs); i++) {
813 		if (!(aes_algs[i].base.cra_flags & CRYPTO_ALG_INTERNAL))
814 			continue;
815 
816 		algname = aes_algs[i].base.cra_name + 2;
817 		drvname = aes_algs[i].base.cra_driver_name + 2;
818 		basename = aes_algs[i].base.cra_driver_name;
819 		simd = simd_skcipher_create_compat(algname, drvname, basename);
820 		err = PTR_ERR(simd);
821 		if (IS_ERR(simd))
822 			goto unregister_simds;
823 
824 		aes_simd_algs[i] = simd;
825 	}
826 
827 	return 0;
828 
829 unregister_simds:
830 	aes_exit();
831 	return err;
832 unregister_ciphers:
833 	crypto_unregister_skciphers(aes_algs, ARRAY_SIZE(aes_algs));
834 	return err;
835 }
836 
837 #ifdef USE_V8_CRYPTO_EXTENSIONS
838 module_cpu_feature_match(AES, aes_init);
839 #else
840 module_init(aes_init);
841 EXPORT_SYMBOL(neon_aes_ecb_encrypt);
842 EXPORT_SYMBOL(neon_aes_cbc_encrypt);
843 #endif
844 module_exit(aes_exit);
845