1 // SPDX-License-Identifier: GPL-2.0-only
2 /*
3  * aes-ccm-glue.c - AES-CCM transform for ARMv8 with Crypto Extensions
4  *
5  * Copyright (C) 2013 - 2017 Linaro Ltd <ard.biesheuvel@linaro.org>
6  */
7 
8 #include <asm/neon.h>
9 #include <asm/simd.h>
10 #include <asm/unaligned.h>
11 #include <crypto/aes.h>
12 #include <crypto/scatterwalk.h>
13 #include <crypto/internal/aead.h>
14 #include <crypto/internal/simd.h>
15 #include <crypto/internal/skcipher.h>
16 #include <linux/module.h>
17 
18 #include "aes-ce-setkey.h"
19 
20 static int num_rounds(struct crypto_aes_ctx *ctx)
21 {
22 	/*
23 	 * # of rounds specified by AES:
24 	 * 128 bit key		10 rounds
25 	 * 192 bit key		12 rounds
26 	 * 256 bit key		14 rounds
27 	 * => n byte key	=> 6 + (n/4) rounds
28 	 */
29 	return 6 + ctx->key_length / 4;
30 }
31 
32 asmlinkage void ce_aes_ccm_auth_data(u8 mac[], u8 const in[], u32 abytes,
33 				     u32 *macp, u32 const rk[], u32 rounds);
34 
35 asmlinkage void ce_aes_ccm_encrypt(u8 out[], u8 const in[], u32 cbytes,
36 				   u32 const rk[], u32 rounds, u8 mac[],
37 				   u8 ctr[]);
38 
39 asmlinkage void ce_aes_ccm_decrypt(u8 out[], u8 const in[], u32 cbytes,
40 				   u32 const rk[], u32 rounds, u8 mac[],
41 				   u8 ctr[]);
42 
43 asmlinkage void ce_aes_ccm_final(u8 mac[], u8 const ctr[], u32 const rk[],
44 				 u32 rounds);
45 
46 asmlinkage void __aes_arm64_encrypt(u32 *rk, u8 *out, const u8 *in, int rounds);
47 
48 static int ccm_setkey(struct crypto_aead *tfm, const u8 *in_key,
49 		      unsigned int key_len)
50 {
51 	struct crypto_aes_ctx *ctx = crypto_aead_ctx(tfm);
52 	int ret;
53 
54 	ret = ce_aes_expandkey(ctx, in_key, key_len);
55 	if (!ret)
56 		return 0;
57 
58 	tfm->base.crt_flags |= CRYPTO_TFM_RES_BAD_KEY_LEN;
59 	return -EINVAL;
60 }
61 
62 static int ccm_setauthsize(struct crypto_aead *tfm, unsigned int authsize)
63 {
64 	if ((authsize & 1) || authsize < 4)
65 		return -EINVAL;
66 	return 0;
67 }
68 
69 static int ccm_init_mac(struct aead_request *req, u8 maciv[], u32 msglen)
70 {
71 	struct crypto_aead *aead = crypto_aead_reqtfm(req);
72 	__be32 *n = (__be32 *)&maciv[AES_BLOCK_SIZE - 8];
73 	u32 l = req->iv[0] + 1;
74 
75 	/* verify that CCM dimension 'L' is set correctly in the IV */
76 	if (l < 2 || l > 8)
77 		return -EINVAL;
78 
79 	/* verify that msglen can in fact be represented in L bytes */
80 	if (l < 4 && msglen >> (8 * l))
81 		return -EOVERFLOW;
82 
83 	/*
84 	 * Even if the CCM spec allows L values of up to 8, the Linux cryptoapi
85 	 * uses a u32 type to represent msglen so the top 4 bytes are always 0.
86 	 */
87 	n[0] = 0;
88 	n[1] = cpu_to_be32(msglen);
89 
90 	memcpy(maciv, req->iv, AES_BLOCK_SIZE - l);
91 
92 	/*
93 	 * Meaning of byte 0 according to CCM spec (RFC 3610/NIST 800-38C)
94 	 * - bits 0..2	: max # of bytes required to represent msglen, minus 1
95 	 *                (already set by caller)
96 	 * - bits 3..5	: size of auth tag (1 => 4 bytes, 2 => 6 bytes, etc)
97 	 * - bit 6	: indicates presence of authenticate-only data
98 	 */
99 	maciv[0] |= (crypto_aead_authsize(aead) - 2) << 2;
100 	if (req->assoclen)
101 		maciv[0] |= 0x40;
102 
103 	memset(&req->iv[AES_BLOCK_SIZE - l], 0, l);
104 	return 0;
105 }
106 
107 static void ccm_update_mac(struct crypto_aes_ctx *key, u8 mac[], u8 const in[],
108 			   u32 abytes, u32 *macp)
109 {
110 	if (crypto_simd_usable()) {
111 		kernel_neon_begin();
112 		ce_aes_ccm_auth_data(mac, in, abytes, macp, key->key_enc,
113 				     num_rounds(key));
114 		kernel_neon_end();
115 	} else {
116 		if (*macp > 0 && *macp < AES_BLOCK_SIZE) {
117 			int added = min(abytes, AES_BLOCK_SIZE - *macp);
118 
119 			crypto_xor(&mac[*macp], in, added);
120 
121 			*macp += added;
122 			in += added;
123 			abytes -= added;
124 		}
125 
126 		while (abytes >= AES_BLOCK_SIZE) {
127 			__aes_arm64_encrypt(key->key_enc, mac, mac,
128 					    num_rounds(key));
129 			crypto_xor(mac, in, AES_BLOCK_SIZE);
130 
131 			in += AES_BLOCK_SIZE;
132 			abytes -= AES_BLOCK_SIZE;
133 		}
134 
135 		if (abytes > 0) {
136 			__aes_arm64_encrypt(key->key_enc, mac, mac,
137 					    num_rounds(key));
138 			crypto_xor(mac, in, abytes);
139 			*macp = abytes;
140 		}
141 	}
142 }
143 
144 static void ccm_calculate_auth_mac(struct aead_request *req, u8 mac[])
145 {
146 	struct crypto_aead *aead = crypto_aead_reqtfm(req);
147 	struct crypto_aes_ctx *ctx = crypto_aead_ctx(aead);
148 	struct __packed { __be16 l; __be32 h; u16 len; } ltag;
149 	struct scatter_walk walk;
150 	u32 len = req->assoclen;
151 	u32 macp = 0;
152 
153 	/* prepend the AAD with a length tag */
154 	if (len < 0xff00) {
155 		ltag.l = cpu_to_be16(len);
156 		ltag.len = 2;
157 	} else  {
158 		ltag.l = cpu_to_be16(0xfffe);
159 		put_unaligned_be32(len, &ltag.h);
160 		ltag.len = 6;
161 	}
162 
163 	ccm_update_mac(ctx, mac, (u8 *)&ltag, ltag.len, &macp);
164 	scatterwalk_start(&walk, req->src);
165 
166 	do {
167 		u32 n = scatterwalk_clamp(&walk, len);
168 		u8 *p;
169 
170 		if (!n) {
171 			scatterwalk_start(&walk, sg_next(walk.sg));
172 			n = scatterwalk_clamp(&walk, len);
173 		}
174 		p = scatterwalk_map(&walk);
175 		ccm_update_mac(ctx, mac, p, n, &macp);
176 		len -= n;
177 
178 		scatterwalk_unmap(p);
179 		scatterwalk_advance(&walk, n);
180 		scatterwalk_done(&walk, 0, len);
181 	} while (len);
182 }
183 
184 static int ccm_crypt_fallback(struct skcipher_walk *walk, u8 mac[], u8 iv0[],
185 			      struct crypto_aes_ctx *ctx, bool enc)
186 {
187 	u8 buf[AES_BLOCK_SIZE];
188 	int err = 0;
189 
190 	while (walk->nbytes) {
191 		int blocks = walk->nbytes / AES_BLOCK_SIZE;
192 		u32 tail = walk->nbytes % AES_BLOCK_SIZE;
193 		u8 *dst = walk->dst.virt.addr;
194 		u8 *src = walk->src.virt.addr;
195 		u32 nbytes = walk->nbytes;
196 
197 		if (nbytes == walk->total && tail > 0) {
198 			blocks++;
199 			tail = 0;
200 		}
201 
202 		do {
203 			u32 bsize = AES_BLOCK_SIZE;
204 
205 			if (nbytes < AES_BLOCK_SIZE)
206 				bsize = nbytes;
207 
208 			crypto_inc(walk->iv, AES_BLOCK_SIZE);
209 			__aes_arm64_encrypt(ctx->key_enc, buf, walk->iv,
210 					    num_rounds(ctx));
211 			__aes_arm64_encrypt(ctx->key_enc, mac, mac,
212 					    num_rounds(ctx));
213 			if (enc)
214 				crypto_xor(mac, src, bsize);
215 			crypto_xor_cpy(dst, src, buf, bsize);
216 			if (!enc)
217 				crypto_xor(mac, dst, bsize);
218 			dst += bsize;
219 			src += bsize;
220 			nbytes -= bsize;
221 		} while (--blocks);
222 
223 		err = skcipher_walk_done(walk, tail);
224 	}
225 
226 	if (!err) {
227 		__aes_arm64_encrypt(ctx->key_enc, buf, iv0, num_rounds(ctx));
228 		__aes_arm64_encrypt(ctx->key_enc, mac, mac, num_rounds(ctx));
229 		crypto_xor(mac, buf, AES_BLOCK_SIZE);
230 	}
231 	return err;
232 }
233 
234 static int ccm_encrypt(struct aead_request *req)
235 {
236 	struct crypto_aead *aead = crypto_aead_reqtfm(req);
237 	struct crypto_aes_ctx *ctx = crypto_aead_ctx(aead);
238 	struct skcipher_walk walk;
239 	u8 __aligned(8) mac[AES_BLOCK_SIZE];
240 	u8 buf[AES_BLOCK_SIZE];
241 	u32 len = req->cryptlen;
242 	int err;
243 
244 	err = ccm_init_mac(req, mac, len);
245 	if (err)
246 		return err;
247 
248 	if (req->assoclen)
249 		ccm_calculate_auth_mac(req, mac);
250 
251 	/* preserve the original iv for the final round */
252 	memcpy(buf, req->iv, AES_BLOCK_SIZE);
253 
254 	err = skcipher_walk_aead_encrypt(&walk, req, false);
255 
256 	if (crypto_simd_usable()) {
257 		while (walk.nbytes) {
258 			u32 tail = walk.nbytes % AES_BLOCK_SIZE;
259 
260 			if (walk.nbytes == walk.total)
261 				tail = 0;
262 
263 			kernel_neon_begin();
264 			ce_aes_ccm_encrypt(walk.dst.virt.addr,
265 					   walk.src.virt.addr,
266 					   walk.nbytes - tail, ctx->key_enc,
267 					   num_rounds(ctx), mac, walk.iv);
268 			kernel_neon_end();
269 
270 			err = skcipher_walk_done(&walk, tail);
271 		}
272 		if (!err) {
273 			kernel_neon_begin();
274 			ce_aes_ccm_final(mac, buf, ctx->key_enc,
275 					 num_rounds(ctx));
276 			kernel_neon_end();
277 		}
278 	} else {
279 		err = ccm_crypt_fallback(&walk, mac, buf, ctx, true);
280 	}
281 	if (err)
282 		return err;
283 
284 	/* copy authtag to end of dst */
285 	scatterwalk_map_and_copy(mac, req->dst, req->assoclen + req->cryptlen,
286 				 crypto_aead_authsize(aead), 1);
287 
288 	return 0;
289 }
290 
291 static int ccm_decrypt(struct aead_request *req)
292 {
293 	struct crypto_aead *aead = crypto_aead_reqtfm(req);
294 	struct crypto_aes_ctx *ctx = crypto_aead_ctx(aead);
295 	unsigned int authsize = crypto_aead_authsize(aead);
296 	struct skcipher_walk walk;
297 	u8 __aligned(8) mac[AES_BLOCK_SIZE];
298 	u8 buf[AES_BLOCK_SIZE];
299 	u32 len = req->cryptlen - authsize;
300 	int err;
301 
302 	err = ccm_init_mac(req, mac, len);
303 	if (err)
304 		return err;
305 
306 	if (req->assoclen)
307 		ccm_calculate_auth_mac(req, mac);
308 
309 	/* preserve the original iv for the final round */
310 	memcpy(buf, req->iv, AES_BLOCK_SIZE);
311 
312 	err = skcipher_walk_aead_decrypt(&walk, req, false);
313 
314 	if (crypto_simd_usable()) {
315 		while (walk.nbytes) {
316 			u32 tail = walk.nbytes % AES_BLOCK_SIZE;
317 
318 			if (walk.nbytes == walk.total)
319 				tail = 0;
320 
321 			kernel_neon_begin();
322 			ce_aes_ccm_decrypt(walk.dst.virt.addr,
323 					   walk.src.virt.addr,
324 					   walk.nbytes - tail, ctx->key_enc,
325 					   num_rounds(ctx), mac, walk.iv);
326 			kernel_neon_end();
327 
328 			err = skcipher_walk_done(&walk, tail);
329 		}
330 		if (!err) {
331 			kernel_neon_begin();
332 			ce_aes_ccm_final(mac, buf, ctx->key_enc,
333 					 num_rounds(ctx));
334 			kernel_neon_end();
335 		}
336 	} else {
337 		err = ccm_crypt_fallback(&walk, mac, buf, ctx, false);
338 	}
339 
340 	if (err)
341 		return err;
342 
343 	/* compare calculated auth tag with the stored one */
344 	scatterwalk_map_and_copy(buf, req->src,
345 				 req->assoclen + req->cryptlen - authsize,
346 				 authsize, 0);
347 
348 	if (crypto_memneq(mac, buf, authsize))
349 		return -EBADMSG;
350 	return 0;
351 }
352 
353 static struct aead_alg ccm_aes_alg = {
354 	.base = {
355 		.cra_name		= "ccm(aes)",
356 		.cra_driver_name	= "ccm-aes-ce",
357 		.cra_priority		= 300,
358 		.cra_blocksize		= 1,
359 		.cra_ctxsize		= sizeof(struct crypto_aes_ctx),
360 		.cra_module		= THIS_MODULE,
361 	},
362 	.ivsize		= AES_BLOCK_SIZE,
363 	.chunksize	= AES_BLOCK_SIZE,
364 	.maxauthsize	= AES_BLOCK_SIZE,
365 	.setkey		= ccm_setkey,
366 	.setauthsize	= ccm_setauthsize,
367 	.encrypt	= ccm_encrypt,
368 	.decrypt	= ccm_decrypt,
369 };
370 
371 static int __init aes_mod_init(void)
372 {
373 	if (!cpu_have_named_feature(AES))
374 		return -ENODEV;
375 	return crypto_register_aead(&ccm_aes_alg);
376 }
377 
378 static void __exit aes_mod_exit(void)
379 {
380 	crypto_unregister_aead(&ccm_aes_alg);
381 }
382 
383 module_init(aes_mod_init);
384 module_exit(aes_mod_exit);
385 
386 MODULE_DESCRIPTION("Synchronous AES in CCM mode using ARMv8 Crypto Extensions");
387 MODULE_AUTHOR("Ard Biesheuvel <ard.biesheuvel@linaro.org>");
388 MODULE_LICENSE("GPL v2");
389 MODULE_ALIAS_CRYPTO("ccm(aes)");
390