1d2912cb1SThomas Gleixner // SPDX-License-Identifier: GPL-2.0-only
21abee99eSArd Biesheuvel /*
31abee99eSArd Biesheuvel  * Bit sliced AES using NEON instructions
41abee99eSArd Biesheuvel  *
5ec808bbeSArd Biesheuvel  * Copyright (C) 2016 - 2017 Linaro Ltd <ard.biesheuvel@linaro.org>
61abee99eSArd Biesheuvel  */
71abee99eSArd Biesheuvel 
81abee99eSArd Biesheuvel #include <asm/neon.h>
9ec808bbeSArd Biesheuvel #include <asm/simd.h>
101abee99eSArd Biesheuvel #include <crypto/aes.h>
11ff6f4115SArd Biesheuvel #include <crypto/ctr.h>
121abee99eSArd Biesheuvel #include <crypto/internal/simd.h>
131abee99eSArd Biesheuvel #include <crypto/internal/skcipher.h>
1467cfa5d3SArd Biesheuvel #include <crypto/scatterwalk.h>
151abee99eSArd Biesheuvel #include <crypto/xts.h>
161abee99eSArd Biesheuvel #include <linux/module.h>
171abee99eSArd Biesheuvel 
181abee99eSArd Biesheuvel MODULE_AUTHOR("Ard Biesheuvel <ard.biesheuvel@linaro.org>");
191abee99eSArd Biesheuvel MODULE_LICENSE("GPL v2");
201abee99eSArd Biesheuvel 
211abee99eSArd Biesheuvel MODULE_ALIAS_CRYPTO("ecb(aes)");
221abee99eSArd Biesheuvel MODULE_ALIAS_CRYPTO("cbc(aes)");
231abee99eSArd Biesheuvel MODULE_ALIAS_CRYPTO("ctr(aes)");
241abee99eSArd Biesheuvel MODULE_ALIAS_CRYPTO("xts(aes)");
251abee99eSArd Biesheuvel 
261abee99eSArd Biesheuvel asmlinkage void aesbs_convert_key(u8 out[], u32 const rk[], int rounds);
271abee99eSArd Biesheuvel 
281abee99eSArd Biesheuvel asmlinkage void aesbs_ecb_encrypt(u8 out[], u8 const in[], u8 const rk[],
291abee99eSArd Biesheuvel 				  int rounds, int blocks);
301abee99eSArd Biesheuvel asmlinkage void aesbs_ecb_decrypt(u8 out[], u8 const in[], u8 const rk[],
311abee99eSArd Biesheuvel 				  int rounds, int blocks);
321abee99eSArd Biesheuvel 
331abee99eSArd Biesheuvel asmlinkage void aesbs_cbc_decrypt(u8 out[], u8 const in[], u8 const rk[],
341abee99eSArd Biesheuvel 				  int rounds, int blocks, u8 iv[]);
351abee99eSArd Biesheuvel 
361abee99eSArd Biesheuvel asmlinkage void aesbs_ctr_encrypt(u8 out[], u8 const in[], u8 const rk[],
37fc074e13SArd Biesheuvel 				  int rounds, int blocks, u8 iv[]);
381abee99eSArd Biesheuvel 
391abee99eSArd Biesheuvel asmlinkage void aesbs_xts_encrypt(u8 out[], u8 const in[], u8 const rk[],
401abee99eSArd Biesheuvel 				  int rounds, int blocks, u8 iv[]);
411abee99eSArd Biesheuvel asmlinkage void aesbs_xts_decrypt(u8 out[], u8 const in[], u8 const rk[],
421abee99eSArd Biesheuvel 				  int rounds, int blocks, u8 iv[]);
431abee99eSArd Biesheuvel 
4412fcd923SArd Biesheuvel /* borrowed from aes-neon-blk.ko */
4512fcd923SArd Biesheuvel asmlinkage void neon_aes_ecb_encrypt(u8 out[], u8 const in[], u32 const rk[],
4668338174SArd Biesheuvel 				     int rounds, int blocks);
4712fcd923SArd Biesheuvel asmlinkage void neon_aes_cbc_encrypt(u8 out[], u8 const in[], u32 const rk[],
4868338174SArd Biesheuvel 				     int rounds, int blocks, u8 iv[]);
49fc074e13SArd Biesheuvel asmlinkage void neon_aes_ctr_encrypt(u8 out[], u8 const in[], u32 const rk[],
50fc074e13SArd Biesheuvel 				     int rounds, int bytes, u8 ctr[]);
5167cfa5d3SArd Biesheuvel asmlinkage void neon_aes_xts_encrypt(u8 out[], u8 const in[],
5267cfa5d3SArd Biesheuvel 				     u32 const rk1[], int rounds, int bytes,
5367cfa5d3SArd Biesheuvel 				     u32 const rk2[], u8 iv[], int first);
5467cfa5d3SArd Biesheuvel asmlinkage void neon_aes_xts_decrypt(u8 out[], u8 const in[],
5567cfa5d3SArd Biesheuvel 				     u32 const rk1[], int rounds, int bytes,
5667cfa5d3SArd Biesheuvel 				     u32 const rk2[], u8 iv[], int first);
571abee99eSArd Biesheuvel 
581abee99eSArd Biesheuvel struct aesbs_ctx {
591abee99eSArd Biesheuvel 	u8	rk[13 * (8 * AES_BLOCK_SIZE) + 32];
601abee99eSArd Biesheuvel 	int	rounds;
611abee99eSArd Biesheuvel } __aligned(AES_BLOCK_SIZE);
621abee99eSArd Biesheuvel 
63fc074e13SArd Biesheuvel struct aesbs_cbc_ctr_ctx {
641abee99eSArd Biesheuvel 	struct aesbs_ctx	key;
651abee99eSArd Biesheuvel 	u32			enc[AES_MAX_KEYLENGTH_U32];
661abee99eSArd Biesheuvel };
671abee99eSArd Biesheuvel 
681abee99eSArd Biesheuvel struct aesbs_xts_ctx {
691abee99eSArd Biesheuvel 	struct aesbs_ctx	key;
701abee99eSArd Biesheuvel 	u32			twkey[AES_MAX_KEYLENGTH_U32];
7167cfa5d3SArd Biesheuvel 	struct crypto_aes_ctx	cts;
721abee99eSArd Biesheuvel };
731abee99eSArd Biesheuvel 
741abee99eSArd Biesheuvel static int aesbs_setkey(struct crypto_skcipher *tfm, const u8 *in_key,
751abee99eSArd Biesheuvel 			unsigned int key_len)
761abee99eSArd Biesheuvel {
771abee99eSArd Biesheuvel 	struct aesbs_ctx *ctx = crypto_skcipher_ctx(tfm);
781abee99eSArd Biesheuvel 	struct crypto_aes_ctx rk;
791abee99eSArd Biesheuvel 	int err;
801abee99eSArd Biesheuvel 
81f68df543SArd Biesheuvel 	err = aes_expandkey(&rk, in_key, key_len);
821abee99eSArd Biesheuvel 	if (err)
831abee99eSArd Biesheuvel 		return err;
841abee99eSArd Biesheuvel 
851abee99eSArd Biesheuvel 	ctx->rounds = 6 + key_len / 4;
861abee99eSArd Biesheuvel 
871abee99eSArd Biesheuvel 	kernel_neon_begin();
881abee99eSArd Biesheuvel 	aesbs_convert_key(ctx->rk, rk.key_enc, ctx->rounds);
891abee99eSArd Biesheuvel 	kernel_neon_end();
901abee99eSArd Biesheuvel 
911abee99eSArd Biesheuvel 	return 0;
921abee99eSArd Biesheuvel }
931abee99eSArd Biesheuvel 
941abee99eSArd Biesheuvel static int __ecb_crypt(struct skcipher_request *req,
951abee99eSArd Biesheuvel 		       void (*fn)(u8 out[], u8 const in[], u8 const rk[],
961abee99eSArd Biesheuvel 				  int rounds, int blocks))
971abee99eSArd Biesheuvel {
981abee99eSArd Biesheuvel 	struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req);
991abee99eSArd Biesheuvel 	struct aesbs_ctx *ctx = crypto_skcipher_ctx(tfm);
1001abee99eSArd Biesheuvel 	struct skcipher_walk walk;
1011abee99eSArd Biesheuvel 	int err;
1021abee99eSArd Biesheuvel 
10378ad7b08SArd Biesheuvel 	err = skcipher_walk_virt(&walk, req, false);
1041abee99eSArd Biesheuvel 
1051abee99eSArd Biesheuvel 	while (walk.nbytes >= AES_BLOCK_SIZE) {
1061abee99eSArd Biesheuvel 		unsigned int blocks = walk.nbytes / AES_BLOCK_SIZE;
1071abee99eSArd Biesheuvel 
1081abee99eSArd Biesheuvel 		if (walk.nbytes < walk.total)
1091abee99eSArd Biesheuvel 			blocks = round_down(blocks,
1101abee99eSArd Biesheuvel 					    walk.stride / AES_BLOCK_SIZE);
1111abee99eSArd Biesheuvel 
11278ad7b08SArd Biesheuvel 		kernel_neon_begin();
1131abee99eSArd Biesheuvel 		fn(walk.dst.virt.addr, walk.src.virt.addr, ctx->rk,
1141abee99eSArd Biesheuvel 		   ctx->rounds, blocks);
11578ad7b08SArd Biesheuvel 		kernel_neon_end();
1161abee99eSArd Biesheuvel 		err = skcipher_walk_done(&walk,
1171abee99eSArd Biesheuvel 					 walk.nbytes - blocks * AES_BLOCK_SIZE);
1181abee99eSArd Biesheuvel 	}
1191abee99eSArd Biesheuvel 
1201abee99eSArd Biesheuvel 	return err;
1211abee99eSArd Biesheuvel }
1221abee99eSArd Biesheuvel 
1231abee99eSArd Biesheuvel static int ecb_encrypt(struct skcipher_request *req)
1241abee99eSArd Biesheuvel {
1251abee99eSArd Biesheuvel 	return __ecb_crypt(req, aesbs_ecb_encrypt);
1261abee99eSArd Biesheuvel }
1271abee99eSArd Biesheuvel 
1281abee99eSArd Biesheuvel static int ecb_decrypt(struct skcipher_request *req)
1291abee99eSArd Biesheuvel {
1301abee99eSArd Biesheuvel 	return __ecb_crypt(req, aesbs_ecb_decrypt);
1311abee99eSArd Biesheuvel }
1321abee99eSArd Biesheuvel 
133fc074e13SArd Biesheuvel static int aesbs_cbc_ctr_setkey(struct crypto_skcipher *tfm, const u8 *in_key,
1341abee99eSArd Biesheuvel 			    unsigned int key_len)
1351abee99eSArd Biesheuvel {
136fc074e13SArd Biesheuvel 	struct aesbs_cbc_ctr_ctx *ctx = crypto_skcipher_ctx(tfm);
1371abee99eSArd Biesheuvel 	struct crypto_aes_ctx rk;
1381abee99eSArd Biesheuvel 	int err;
1391abee99eSArd Biesheuvel 
140f68df543SArd Biesheuvel 	err = aes_expandkey(&rk, in_key, key_len);
1411abee99eSArd Biesheuvel 	if (err)
1421abee99eSArd Biesheuvel 		return err;
1431abee99eSArd Biesheuvel 
1441abee99eSArd Biesheuvel 	ctx->key.rounds = 6 + key_len / 4;
1451abee99eSArd Biesheuvel 
1461abee99eSArd Biesheuvel 	memcpy(ctx->enc, rk.key_enc, sizeof(ctx->enc));
1471abee99eSArd Biesheuvel 
1481abee99eSArd Biesheuvel 	kernel_neon_begin();
1491abee99eSArd Biesheuvel 	aesbs_convert_key(ctx->key.rk, rk.key_enc, ctx->key.rounds);
1501abee99eSArd Biesheuvel 	kernel_neon_end();
15182ff493eSTorsten Duwe 	memzero_explicit(&rk, sizeof(rk));
1521abee99eSArd Biesheuvel 
1531abee99eSArd Biesheuvel 	return 0;
1541abee99eSArd Biesheuvel }
1551abee99eSArd Biesheuvel 
1561abee99eSArd Biesheuvel static int cbc_encrypt(struct skcipher_request *req)
1571abee99eSArd Biesheuvel {
15812fcd923SArd Biesheuvel 	struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req);
159fc074e13SArd Biesheuvel 	struct aesbs_cbc_ctr_ctx *ctx = crypto_skcipher_ctx(tfm);
16012fcd923SArd Biesheuvel 	struct skcipher_walk walk;
16168338174SArd Biesheuvel 	int err;
16212fcd923SArd Biesheuvel 
16378ad7b08SArd Biesheuvel 	err = skcipher_walk_virt(&walk, req, false);
16412fcd923SArd Biesheuvel 
16512fcd923SArd Biesheuvel 	while (walk.nbytes >= AES_BLOCK_SIZE) {
16612fcd923SArd Biesheuvel 		unsigned int blocks = walk.nbytes / AES_BLOCK_SIZE;
16712fcd923SArd Biesheuvel 
16812fcd923SArd Biesheuvel 		/* fall back to the non-bitsliced NEON implementation */
16978ad7b08SArd Biesheuvel 		kernel_neon_begin();
17012fcd923SArd Biesheuvel 		neon_aes_cbc_encrypt(walk.dst.virt.addr, walk.src.virt.addr,
17168338174SArd Biesheuvel 				     ctx->enc, ctx->key.rounds, blocks,
17268338174SArd Biesheuvel 				     walk.iv);
17378ad7b08SArd Biesheuvel 		kernel_neon_end();
17412fcd923SArd Biesheuvel 		err = skcipher_walk_done(&walk, walk.nbytes % AES_BLOCK_SIZE);
17512fcd923SArd Biesheuvel 	}
17612fcd923SArd Biesheuvel 	return err;
1771abee99eSArd Biesheuvel }
1781abee99eSArd Biesheuvel 
1791abee99eSArd Biesheuvel static int cbc_decrypt(struct skcipher_request *req)
1801abee99eSArd Biesheuvel {
1811abee99eSArd Biesheuvel 	struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req);
182fc074e13SArd Biesheuvel 	struct aesbs_cbc_ctr_ctx *ctx = crypto_skcipher_ctx(tfm);
1831abee99eSArd Biesheuvel 	struct skcipher_walk walk;
1841abee99eSArd Biesheuvel 	int err;
1851abee99eSArd Biesheuvel 
18678ad7b08SArd Biesheuvel 	err = skcipher_walk_virt(&walk, req, false);
1871abee99eSArd Biesheuvel 
1881abee99eSArd Biesheuvel 	while (walk.nbytes >= AES_BLOCK_SIZE) {
1891abee99eSArd Biesheuvel 		unsigned int blocks = walk.nbytes / AES_BLOCK_SIZE;
1901abee99eSArd Biesheuvel 
1911abee99eSArd Biesheuvel 		if (walk.nbytes < walk.total)
1921abee99eSArd Biesheuvel 			blocks = round_down(blocks,
1931abee99eSArd Biesheuvel 					    walk.stride / AES_BLOCK_SIZE);
1941abee99eSArd Biesheuvel 
19578ad7b08SArd Biesheuvel 		kernel_neon_begin();
1961abee99eSArd Biesheuvel 		aesbs_cbc_decrypt(walk.dst.virt.addr, walk.src.virt.addr,
1971abee99eSArd Biesheuvel 				  ctx->key.rk, ctx->key.rounds, blocks,
1981abee99eSArd Biesheuvel 				  walk.iv);
19978ad7b08SArd Biesheuvel 		kernel_neon_end();
2001abee99eSArd Biesheuvel 		err = skcipher_walk_done(&walk,
2011abee99eSArd Biesheuvel 					 walk.nbytes - blocks * AES_BLOCK_SIZE);
2021abee99eSArd Biesheuvel 	}
2031abee99eSArd Biesheuvel 
2041abee99eSArd Biesheuvel 	return err;
2051abee99eSArd Biesheuvel }
2061abee99eSArd Biesheuvel 
2071abee99eSArd Biesheuvel static int ctr_encrypt(struct skcipher_request *req)
2081abee99eSArd Biesheuvel {
2091abee99eSArd Biesheuvel 	struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req);
210fc074e13SArd Biesheuvel 	struct aesbs_cbc_ctr_ctx *ctx = crypto_skcipher_ctx(tfm);
2111abee99eSArd Biesheuvel 	struct skcipher_walk walk;
2121abee99eSArd Biesheuvel 	int err;
2131abee99eSArd Biesheuvel 
21478ad7b08SArd Biesheuvel 	err = skcipher_walk_virt(&walk, req, false);
2151abee99eSArd Biesheuvel 
2161abee99eSArd Biesheuvel 	while (walk.nbytes > 0) {
217fc074e13SArd Biesheuvel 		int blocks = (walk.nbytes / AES_BLOCK_SIZE) & ~7;
218fc074e13SArd Biesheuvel 		int nbytes = walk.nbytes % (8 * AES_BLOCK_SIZE);
219fc074e13SArd Biesheuvel 		const u8 *src = walk.src.virt.addr;
220fc074e13SArd Biesheuvel 		u8 *dst = walk.dst.virt.addr;
2211abee99eSArd Biesheuvel 
22278ad7b08SArd Biesheuvel 		kernel_neon_begin();
223fc074e13SArd Biesheuvel 		if (blocks >= 8) {
224fc074e13SArd Biesheuvel 			aesbs_ctr_encrypt(dst, src, ctx->key.rk, ctx->key.rounds,
225fc074e13SArd Biesheuvel 					  blocks, walk.iv);
226fc074e13SArd Biesheuvel 			dst += blocks * AES_BLOCK_SIZE;
227fc074e13SArd Biesheuvel 			src += blocks * AES_BLOCK_SIZE;
2281abee99eSArd Biesheuvel 		}
229fc074e13SArd Biesheuvel 		if (nbytes && walk.nbytes == walk.total) {
230fc074e13SArd Biesheuvel 			neon_aes_ctr_encrypt(dst, src, ctx->enc, ctx->key.rounds,
231fc074e13SArd Biesheuvel 					     nbytes, walk.iv);
232fc074e13SArd Biesheuvel 			nbytes = 0;
233fc074e13SArd Biesheuvel 		}
234fc074e13SArd Biesheuvel 		kernel_neon_end();
235fc074e13SArd Biesheuvel 		err = skcipher_walk_done(&walk, nbytes);
2361abee99eSArd Biesheuvel 	}
2371abee99eSArd Biesheuvel 	return err;
2381abee99eSArd Biesheuvel }
2391abee99eSArd Biesheuvel 
2401abee99eSArd Biesheuvel static int aesbs_xts_setkey(struct crypto_skcipher *tfm, const u8 *in_key,
2411abee99eSArd Biesheuvel 			    unsigned int key_len)
2421abee99eSArd Biesheuvel {
2431abee99eSArd Biesheuvel 	struct aesbs_xts_ctx *ctx = crypto_skcipher_ctx(tfm);
2441abee99eSArd Biesheuvel 	struct crypto_aes_ctx rk;
2451abee99eSArd Biesheuvel 	int err;
2461abee99eSArd Biesheuvel 
2471abee99eSArd Biesheuvel 	err = xts_verify_key(tfm, in_key, key_len);
2481abee99eSArd Biesheuvel 	if (err)
2491abee99eSArd Biesheuvel 		return err;
2501abee99eSArd Biesheuvel 
2511abee99eSArd Biesheuvel 	key_len /= 2;
25267cfa5d3SArd Biesheuvel 	err = aes_expandkey(&ctx->cts, in_key, key_len);
25367cfa5d3SArd Biesheuvel 	if (err)
25467cfa5d3SArd Biesheuvel 		return err;
25567cfa5d3SArd Biesheuvel 
256f68df543SArd Biesheuvel 	err = aes_expandkey(&rk, in_key + key_len, key_len);
2571abee99eSArd Biesheuvel 	if (err)
2581abee99eSArd Biesheuvel 		return err;
2591abee99eSArd Biesheuvel 
2601abee99eSArd Biesheuvel 	memcpy(ctx->twkey, rk.key_enc, sizeof(ctx->twkey));
2611abee99eSArd Biesheuvel 
2621abee99eSArd Biesheuvel 	return aesbs_setkey(tfm, in_key, key_len);
2631abee99eSArd Biesheuvel }
2641abee99eSArd Biesheuvel 
26567cfa5d3SArd Biesheuvel static int __xts_crypt(struct skcipher_request *req, bool encrypt,
2661abee99eSArd Biesheuvel 		       void (*fn)(u8 out[], u8 const in[], u8 const rk[],
2671abee99eSArd Biesheuvel 				  int rounds, int blocks, u8 iv[]))
2681abee99eSArd Biesheuvel {
2691abee99eSArd Biesheuvel 	struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req);
2701abee99eSArd Biesheuvel 	struct aesbs_xts_ctx *ctx = crypto_skcipher_ctx(tfm);
27167cfa5d3SArd Biesheuvel 	int tail = req->cryptlen % (8 * AES_BLOCK_SIZE);
27267cfa5d3SArd Biesheuvel 	struct scatterlist sg_src[2], sg_dst[2];
27367cfa5d3SArd Biesheuvel 	struct skcipher_request subreq;
27467cfa5d3SArd Biesheuvel 	struct scatterlist *src, *dst;
2751abee99eSArd Biesheuvel 	struct skcipher_walk walk;
27667cfa5d3SArd Biesheuvel 	int nbytes, err;
27767cfa5d3SArd Biesheuvel 	int first = 1;
27867cfa5d3SArd Biesheuvel 	u8 *out, *in;
27967cfa5d3SArd Biesheuvel 
28067cfa5d3SArd Biesheuvel 	if (req->cryptlen < AES_BLOCK_SIZE)
28167cfa5d3SArd Biesheuvel 		return -EINVAL;
28267cfa5d3SArd Biesheuvel 
28367cfa5d3SArd Biesheuvel 	/* ensure that the cts tail is covered by a single step */
28467cfa5d3SArd Biesheuvel 	if (unlikely(tail > 0 && tail < AES_BLOCK_SIZE)) {
28567cfa5d3SArd Biesheuvel 		int xts_blocks = DIV_ROUND_UP(req->cryptlen,
28667cfa5d3SArd Biesheuvel 					      AES_BLOCK_SIZE) - 2;
28767cfa5d3SArd Biesheuvel 
28867cfa5d3SArd Biesheuvel 		skcipher_request_set_tfm(&subreq, tfm);
28967cfa5d3SArd Biesheuvel 		skcipher_request_set_callback(&subreq,
29067cfa5d3SArd Biesheuvel 					      skcipher_request_flags(req),
29167cfa5d3SArd Biesheuvel 					      NULL, NULL);
29267cfa5d3SArd Biesheuvel 		skcipher_request_set_crypt(&subreq, req->src, req->dst,
29367cfa5d3SArd Biesheuvel 					   xts_blocks * AES_BLOCK_SIZE,
29467cfa5d3SArd Biesheuvel 					   req->iv);
29567cfa5d3SArd Biesheuvel 		req = &subreq;
29667cfa5d3SArd Biesheuvel 	} else {
29767cfa5d3SArd Biesheuvel 		tail = 0;
29867cfa5d3SArd Biesheuvel 	}
2991abee99eSArd Biesheuvel 
30078ad7b08SArd Biesheuvel 	err = skcipher_walk_virt(&walk, req, false);
3014a8108b7SEric Biggers 	if (err)
3024a8108b7SEric Biggers 		return err;
3031abee99eSArd Biesheuvel 
3041abee99eSArd Biesheuvel 	while (walk.nbytes >= AES_BLOCK_SIZE) {
305*dfc6031eSArd Biesheuvel 		int blocks = (walk.nbytes / AES_BLOCK_SIZE) & ~7;
30667cfa5d3SArd Biesheuvel 		out = walk.dst.virt.addr;
30767cfa5d3SArd Biesheuvel 		in = walk.src.virt.addr;
30867cfa5d3SArd Biesheuvel 		nbytes = walk.nbytes;
30967cfa5d3SArd Biesheuvel 
31078ad7b08SArd Biesheuvel 		kernel_neon_begin();
311*dfc6031eSArd Biesheuvel 		if (blocks >= 8) {
312*dfc6031eSArd Biesheuvel 			if (first == 1)
31367cfa5d3SArd Biesheuvel 				neon_aes_ecb_encrypt(walk.iv, walk.iv,
31467cfa5d3SArd Biesheuvel 						     ctx->twkey,
31567cfa5d3SArd Biesheuvel 						     ctx->key.rounds, 1);
316*dfc6031eSArd Biesheuvel 			first = 2;
31767cfa5d3SArd Biesheuvel 
31867cfa5d3SArd Biesheuvel 			fn(out, in, ctx->key.rk, ctx->key.rounds, blocks,
31967cfa5d3SArd Biesheuvel 			   walk.iv);
32067cfa5d3SArd Biesheuvel 
32167cfa5d3SArd Biesheuvel 			out += blocks * AES_BLOCK_SIZE;
32267cfa5d3SArd Biesheuvel 			in += blocks * AES_BLOCK_SIZE;
32367cfa5d3SArd Biesheuvel 			nbytes -= blocks * AES_BLOCK_SIZE;
3241abee99eSArd Biesheuvel 		}
325*dfc6031eSArd Biesheuvel 		if (walk.nbytes == walk.total && nbytes > 0) {
326*dfc6031eSArd Biesheuvel 			if (encrypt)
327*dfc6031eSArd Biesheuvel 				neon_aes_xts_encrypt(out, in, ctx->cts.key_enc,
328*dfc6031eSArd Biesheuvel 						     ctx->key.rounds, nbytes,
329*dfc6031eSArd Biesheuvel 						     ctx->twkey, walk.iv, first);
330*dfc6031eSArd Biesheuvel 			else
331*dfc6031eSArd Biesheuvel 				neon_aes_xts_decrypt(out, in, ctx->cts.key_dec,
332*dfc6031eSArd Biesheuvel 						     ctx->key.rounds, nbytes,
333*dfc6031eSArd Biesheuvel 						     ctx->twkey, walk.iv, first);
334*dfc6031eSArd Biesheuvel 			nbytes = first = 0;
335*dfc6031eSArd Biesheuvel 		}
33667cfa5d3SArd Biesheuvel 		kernel_neon_end();
3379b537997SYunfeng Ye 		err = skcipher_walk_done(&walk, nbytes);
33867cfa5d3SArd Biesheuvel 	}
33967cfa5d3SArd Biesheuvel 
34067cfa5d3SArd Biesheuvel 	if (err || likely(!tail))
3411abee99eSArd Biesheuvel 		return err;
34267cfa5d3SArd Biesheuvel 
34367cfa5d3SArd Biesheuvel 	/* handle ciphertext stealing */
34467cfa5d3SArd Biesheuvel 	dst = src = scatterwalk_ffwd(sg_src, req->src, req->cryptlen);
34567cfa5d3SArd Biesheuvel 	if (req->dst != req->src)
34667cfa5d3SArd Biesheuvel 		dst = scatterwalk_ffwd(sg_dst, req->dst, req->cryptlen);
34767cfa5d3SArd Biesheuvel 
34867cfa5d3SArd Biesheuvel 	skcipher_request_set_crypt(req, src, dst, AES_BLOCK_SIZE + tail,
34967cfa5d3SArd Biesheuvel 				   req->iv);
35067cfa5d3SArd Biesheuvel 
35167cfa5d3SArd Biesheuvel 	err = skcipher_walk_virt(&walk, req, false);
35267cfa5d3SArd Biesheuvel 	if (err)
35367cfa5d3SArd Biesheuvel 		return err;
35467cfa5d3SArd Biesheuvel 
35567cfa5d3SArd Biesheuvel 	out = walk.dst.virt.addr;
35667cfa5d3SArd Biesheuvel 	in = walk.src.virt.addr;
35767cfa5d3SArd Biesheuvel 	nbytes = walk.nbytes;
35867cfa5d3SArd Biesheuvel 
35967cfa5d3SArd Biesheuvel 	kernel_neon_begin();
36067cfa5d3SArd Biesheuvel 	if (encrypt)
36167cfa5d3SArd Biesheuvel 		neon_aes_xts_encrypt(out, in, ctx->cts.key_enc, ctx->key.rounds,
362*dfc6031eSArd Biesheuvel 				     nbytes, ctx->twkey, walk.iv, first);
36367cfa5d3SArd Biesheuvel 	else
36467cfa5d3SArd Biesheuvel 		neon_aes_xts_decrypt(out, in, ctx->cts.key_dec, ctx->key.rounds,
365*dfc6031eSArd Biesheuvel 				     nbytes, ctx->twkey, walk.iv, first);
36667cfa5d3SArd Biesheuvel 	kernel_neon_end();
36767cfa5d3SArd Biesheuvel 
36867cfa5d3SArd Biesheuvel 	return skcipher_walk_done(&walk, 0);
3691abee99eSArd Biesheuvel }
3701abee99eSArd Biesheuvel 
3711abee99eSArd Biesheuvel static int xts_encrypt(struct skcipher_request *req)
3721abee99eSArd Biesheuvel {
37367cfa5d3SArd Biesheuvel 	return __xts_crypt(req, true, aesbs_xts_encrypt);
3741abee99eSArd Biesheuvel }
3751abee99eSArd Biesheuvel 
3761abee99eSArd Biesheuvel static int xts_decrypt(struct skcipher_request *req)
3771abee99eSArd Biesheuvel {
37867cfa5d3SArd Biesheuvel 	return __xts_crypt(req, false, aesbs_xts_decrypt);
3791abee99eSArd Biesheuvel }
3801abee99eSArd Biesheuvel 
3811abee99eSArd Biesheuvel static struct skcipher_alg aes_algs[] = { {
38296c34e14SArd Biesheuvel 	.base.cra_name		= "ecb(aes)",
38396c34e14SArd Biesheuvel 	.base.cra_driver_name	= "ecb-aes-neonbs",
3841abee99eSArd Biesheuvel 	.base.cra_priority	= 250,
3851abee99eSArd Biesheuvel 	.base.cra_blocksize	= AES_BLOCK_SIZE,
3861abee99eSArd Biesheuvel 	.base.cra_ctxsize	= sizeof(struct aesbs_ctx),
3871abee99eSArd Biesheuvel 	.base.cra_module	= THIS_MODULE,
3881abee99eSArd Biesheuvel 
3891abee99eSArd Biesheuvel 	.min_keysize		= AES_MIN_KEY_SIZE,
3901abee99eSArd Biesheuvel 	.max_keysize		= AES_MAX_KEY_SIZE,
3911abee99eSArd Biesheuvel 	.walksize		= 8 * AES_BLOCK_SIZE,
3921abee99eSArd Biesheuvel 	.setkey			= aesbs_setkey,
3931abee99eSArd Biesheuvel 	.encrypt		= ecb_encrypt,
3941abee99eSArd Biesheuvel 	.decrypt		= ecb_decrypt,
3951abee99eSArd Biesheuvel }, {
39696c34e14SArd Biesheuvel 	.base.cra_name		= "cbc(aes)",
39796c34e14SArd Biesheuvel 	.base.cra_driver_name	= "cbc-aes-neonbs",
3981abee99eSArd Biesheuvel 	.base.cra_priority	= 250,
3991abee99eSArd Biesheuvel 	.base.cra_blocksize	= AES_BLOCK_SIZE,
400fc074e13SArd Biesheuvel 	.base.cra_ctxsize	= sizeof(struct aesbs_cbc_ctr_ctx),
4011abee99eSArd Biesheuvel 	.base.cra_module	= THIS_MODULE,
4021abee99eSArd Biesheuvel 
4031abee99eSArd Biesheuvel 	.min_keysize		= AES_MIN_KEY_SIZE,
4041abee99eSArd Biesheuvel 	.max_keysize		= AES_MAX_KEY_SIZE,
4051abee99eSArd Biesheuvel 	.walksize		= 8 * AES_BLOCK_SIZE,
4061abee99eSArd Biesheuvel 	.ivsize			= AES_BLOCK_SIZE,
407fc074e13SArd Biesheuvel 	.setkey			= aesbs_cbc_ctr_setkey,
4081abee99eSArd Biesheuvel 	.encrypt		= cbc_encrypt,
4091abee99eSArd Biesheuvel 	.decrypt		= cbc_decrypt,
4101abee99eSArd Biesheuvel }, {
41196c34e14SArd Biesheuvel 	.base.cra_name		= "ctr(aes)",
41296c34e14SArd Biesheuvel 	.base.cra_driver_name	= "ctr-aes-neonbs",
4131abee99eSArd Biesheuvel 	.base.cra_priority	= 250,
4141abee99eSArd Biesheuvel 	.base.cra_blocksize	= 1,
415fc074e13SArd Biesheuvel 	.base.cra_ctxsize	= sizeof(struct aesbs_cbc_ctr_ctx),
4161abee99eSArd Biesheuvel 	.base.cra_module	= THIS_MODULE,
4171abee99eSArd Biesheuvel 
4181abee99eSArd Biesheuvel 	.min_keysize		= AES_MIN_KEY_SIZE,
4191abee99eSArd Biesheuvel 	.max_keysize		= AES_MAX_KEY_SIZE,
4201abee99eSArd Biesheuvel 	.chunksize		= AES_BLOCK_SIZE,
4211abee99eSArd Biesheuvel 	.walksize		= 8 * AES_BLOCK_SIZE,
4221abee99eSArd Biesheuvel 	.ivsize			= AES_BLOCK_SIZE,
423fc074e13SArd Biesheuvel 	.setkey			= aesbs_cbc_ctr_setkey,
4241abee99eSArd Biesheuvel 	.encrypt		= ctr_encrypt,
4251abee99eSArd Biesheuvel 	.decrypt		= ctr_encrypt,
4261abee99eSArd Biesheuvel }, {
42796c34e14SArd Biesheuvel 	.base.cra_name		= "xts(aes)",
42896c34e14SArd Biesheuvel 	.base.cra_driver_name	= "xts-aes-neonbs",
4291abee99eSArd Biesheuvel 	.base.cra_priority	= 250,
4301abee99eSArd Biesheuvel 	.base.cra_blocksize	= AES_BLOCK_SIZE,
4311abee99eSArd Biesheuvel 	.base.cra_ctxsize	= sizeof(struct aesbs_xts_ctx),
4321abee99eSArd Biesheuvel 	.base.cra_module	= THIS_MODULE,
4331abee99eSArd Biesheuvel 
4341abee99eSArd Biesheuvel 	.min_keysize		= 2 * AES_MIN_KEY_SIZE,
4351abee99eSArd Biesheuvel 	.max_keysize		= 2 * AES_MAX_KEY_SIZE,
4361abee99eSArd Biesheuvel 	.walksize		= 8 * AES_BLOCK_SIZE,
4371abee99eSArd Biesheuvel 	.ivsize			= AES_BLOCK_SIZE,
4381abee99eSArd Biesheuvel 	.setkey			= aesbs_xts_setkey,
4391abee99eSArd Biesheuvel 	.encrypt		= xts_encrypt,
4401abee99eSArd Biesheuvel 	.decrypt		= xts_decrypt,
4411abee99eSArd Biesheuvel } };
4421abee99eSArd Biesheuvel 
4431abee99eSArd Biesheuvel static void aes_exit(void)
4441abee99eSArd Biesheuvel {
4451abee99eSArd Biesheuvel 	crypto_unregister_skciphers(aes_algs, ARRAY_SIZE(aes_algs));
4461abee99eSArd Biesheuvel }
4471abee99eSArd Biesheuvel 
4481abee99eSArd Biesheuvel static int __init aes_init(void)
4491abee99eSArd Biesheuvel {
450aaba098fSAndrew Murray 	if (!cpu_have_named_feature(ASIMD))
4511abee99eSArd Biesheuvel 		return -ENODEV;
4521abee99eSArd Biesheuvel 
45396c34e14SArd Biesheuvel 	return crypto_register_skciphers(aes_algs, ARRAY_SIZE(aes_algs));
4541abee99eSArd Biesheuvel }
4551abee99eSArd Biesheuvel 
4561abee99eSArd Biesheuvel module_init(aes_init);
4571abee99eSArd Biesheuvel module_exit(aes_exit);
458