xref: /openbmc/linux/arch/arm/crypto/aes-ce-glue.c (revision 5d0e4d78)
1 /*
2  * aes-ce-glue.c - wrapper code for ARMv8 AES
3  *
4  * Copyright (C) 2015 Linaro Ltd <ard.biesheuvel@linaro.org>
5  *
6  * This program is free software; you can redistribute it and/or modify
7  * it under the terms of the GNU General Public License version 2 as
8  * published by the Free Software Foundation.
9  */
10 
11 #include <asm/hwcap.h>
12 #include <asm/neon.h>
13 #include <asm/hwcap.h>
14 #include <crypto/aes.h>
15 #include <crypto/internal/simd.h>
16 #include <crypto/internal/skcipher.h>
17 #include <linux/cpufeature.h>
18 #include <linux/module.h>
19 #include <crypto/xts.h>
20 
21 MODULE_DESCRIPTION("AES-ECB/CBC/CTR/XTS using ARMv8 Crypto Extensions");
22 MODULE_AUTHOR("Ard Biesheuvel <ard.biesheuvel@linaro.org>");
23 MODULE_LICENSE("GPL v2");
24 
25 /* defined in aes-ce-core.S */
26 asmlinkage u32 ce_aes_sub(u32 input);
27 asmlinkage void ce_aes_invert(void *dst, void *src);
28 
29 asmlinkage void ce_aes_ecb_encrypt(u8 out[], u8 const in[], u8 const rk[],
30 				   int rounds, int blocks);
31 asmlinkage void ce_aes_ecb_decrypt(u8 out[], u8 const in[], u8 const rk[],
32 				   int rounds, int blocks);
33 
34 asmlinkage void ce_aes_cbc_encrypt(u8 out[], u8 const in[], u8 const rk[],
35 				   int rounds, int blocks, u8 iv[]);
36 asmlinkage void ce_aes_cbc_decrypt(u8 out[], u8 const in[], u8 const rk[],
37 				   int rounds, int blocks, u8 iv[]);
38 
39 asmlinkage void ce_aes_ctr_encrypt(u8 out[], u8 const in[], u8 const rk[],
40 				   int rounds, int blocks, u8 ctr[]);
41 
42 asmlinkage void ce_aes_xts_encrypt(u8 out[], u8 const in[], u8 const rk1[],
43 				   int rounds, int blocks, u8 iv[],
44 				   u8 const rk2[], int first);
45 asmlinkage void ce_aes_xts_decrypt(u8 out[], u8 const in[], u8 const rk1[],
46 				   int rounds, int blocks, u8 iv[],
47 				   u8 const rk2[], int first);
48 
49 struct aes_block {
50 	u8 b[AES_BLOCK_SIZE];
51 };
52 
53 static int num_rounds(struct crypto_aes_ctx *ctx)
54 {
55 	/*
56 	 * # of rounds specified by AES:
57 	 * 128 bit key		10 rounds
58 	 * 192 bit key		12 rounds
59 	 * 256 bit key		14 rounds
60 	 * => n byte key	=> 6 + (n/4) rounds
61 	 */
62 	return 6 + ctx->key_length / 4;
63 }
64 
65 static int ce_aes_expandkey(struct crypto_aes_ctx *ctx, const u8 *in_key,
66 			    unsigned int key_len)
67 {
68 	/*
69 	 * The AES key schedule round constants
70 	 */
71 	static u8 const rcon[] = {
72 		0x01, 0x02, 0x04, 0x08, 0x10, 0x20, 0x40, 0x80, 0x1b, 0x36,
73 	};
74 
75 	u32 kwords = key_len / sizeof(u32);
76 	struct aes_block *key_enc, *key_dec;
77 	int i, j;
78 
79 	if (key_len != AES_KEYSIZE_128 &&
80 	    key_len != AES_KEYSIZE_192 &&
81 	    key_len != AES_KEYSIZE_256)
82 		return -EINVAL;
83 
84 	memcpy(ctx->key_enc, in_key, key_len);
85 	ctx->key_length = key_len;
86 
87 	kernel_neon_begin();
88 	for (i = 0; i < sizeof(rcon); i++) {
89 		u32 *rki = ctx->key_enc + (i * kwords);
90 		u32 *rko = rki + kwords;
91 
92 #ifndef CONFIG_CPU_BIG_ENDIAN
93 		rko[0] = ror32(ce_aes_sub(rki[kwords - 1]), 8);
94 		rko[0] = rko[0] ^ rki[0] ^ rcon[i];
95 #else
96 		rko[0] = rol32(ce_aes_sub(rki[kwords - 1]), 8);
97 		rko[0] = rko[0] ^ rki[0] ^ (rcon[i] << 24);
98 #endif
99 		rko[1] = rko[0] ^ rki[1];
100 		rko[2] = rko[1] ^ rki[2];
101 		rko[3] = rko[2] ^ rki[3];
102 
103 		if (key_len == AES_KEYSIZE_192) {
104 			if (i >= 7)
105 				break;
106 			rko[4] = rko[3] ^ rki[4];
107 			rko[5] = rko[4] ^ rki[5];
108 		} else if (key_len == AES_KEYSIZE_256) {
109 			if (i >= 6)
110 				break;
111 			rko[4] = ce_aes_sub(rko[3]) ^ rki[4];
112 			rko[5] = rko[4] ^ rki[5];
113 			rko[6] = rko[5] ^ rki[6];
114 			rko[7] = rko[6] ^ rki[7];
115 		}
116 	}
117 
118 	/*
119 	 * Generate the decryption keys for the Equivalent Inverse Cipher.
120 	 * This involves reversing the order of the round keys, and applying
121 	 * the Inverse Mix Columns transformation on all but the first and
122 	 * the last one.
123 	 */
124 	key_enc = (struct aes_block *)ctx->key_enc;
125 	key_dec = (struct aes_block *)ctx->key_dec;
126 	j = num_rounds(ctx);
127 
128 	key_dec[0] = key_enc[j];
129 	for (i = 1, j--; j > 0; i++, j--)
130 		ce_aes_invert(key_dec + i, key_enc + j);
131 	key_dec[i] = key_enc[0];
132 
133 	kernel_neon_end();
134 	return 0;
135 }
136 
137 static int ce_aes_setkey(struct crypto_skcipher *tfm, const u8 *in_key,
138 			 unsigned int key_len)
139 {
140 	struct crypto_aes_ctx *ctx = crypto_skcipher_ctx(tfm);
141 	int ret;
142 
143 	ret = ce_aes_expandkey(ctx, in_key, key_len);
144 	if (!ret)
145 		return 0;
146 
147 	crypto_skcipher_set_flags(tfm, CRYPTO_TFM_RES_BAD_KEY_LEN);
148 	return -EINVAL;
149 }
150 
151 struct crypto_aes_xts_ctx {
152 	struct crypto_aes_ctx key1;
153 	struct crypto_aes_ctx __aligned(8) key2;
154 };
155 
156 static int xts_set_key(struct crypto_skcipher *tfm, const u8 *in_key,
157 		       unsigned int key_len)
158 {
159 	struct crypto_aes_xts_ctx *ctx = crypto_skcipher_ctx(tfm);
160 	int ret;
161 
162 	ret = xts_verify_key(tfm, in_key, key_len);
163 	if (ret)
164 		return ret;
165 
166 	ret = ce_aes_expandkey(&ctx->key1, in_key, key_len / 2);
167 	if (!ret)
168 		ret = ce_aes_expandkey(&ctx->key2, &in_key[key_len / 2],
169 				       key_len / 2);
170 	if (!ret)
171 		return 0;
172 
173 	crypto_skcipher_set_flags(tfm, CRYPTO_TFM_RES_BAD_KEY_LEN);
174 	return -EINVAL;
175 }
176 
177 static int ecb_encrypt(struct skcipher_request *req)
178 {
179 	struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req);
180 	struct crypto_aes_ctx *ctx = crypto_skcipher_ctx(tfm);
181 	struct skcipher_walk walk;
182 	unsigned int blocks;
183 	int err;
184 
185 	err = skcipher_walk_virt(&walk, req, true);
186 
187 	kernel_neon_begin();
188 	while ((blocks = (walk.nbytes / AES_BLOCK_SIZE))) {
189 		ce_aes_ecb_encrypt(walk.dst.virt.addr, walk.src.virt.addr,
190 				   (u8 *)ctx->key_enc, num_rounds(ctx), blocks);
191 		err = skcipher_walk_done(&walk, walk.nbytes % AES_BLOCK_SIZE);
192 	}
193 	kernel_neon_end();
194 	return err;
195 }
196 
197 static int ecb_decrypt(struct skcipher_request *req)
198 {
199 	struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req);
200 	struct crypto_aes_ctx *ctx = crypto_skcipher_ctx(tfm);
201 	struct skcipher_walk walk;
202 	unsigned int blocks;
203 	int err;
204 
205 	err = skcipher_walk_virt(&walk, req, true);
206 
207 	kernel_neon_begin();
208 	while ((blocks = (walk.nbytes / AES_BLOCK_SIZE))) {
209 		ce_aes_ecb_decrypt(walk.dst.virt.addr, walk.src.virt.addr,
210 				   (u8 *)ctx->key_dec, num_rounds(ctx), blocks);
211 		err = skcipher_walk_done(&walk, walk.nbytes % AES_BLOCK_SIZE);
212 	}
213 	kernel_neon_end();
214 	return err;
215 }
216 
217 static int cbc_encrypt(struct skcipher_request *req)
218 {
219 	struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req);
220 	struct crypto_aes_ctx *ctx = crypto_skcipher_ctx(tfm);
221 	struct skcipher_walk walk;
222 	unsigned int blocks;
223 	int err;
224 
225 	err = skcipher_walk_virt(&walk, req, true);
226 
227 	kernel_neon_begin();
228 	while ((blocks = (walk.nbytes / AES_BLOCK_SIZE))) {
229 		ce_aes_cbc_encrypt(walk.dst.virt.addr, walk.src.virt.addr,
230 				   (u8 *)ctx->key_enc, num_rounds(ctx), blocks,
231 				   walk.iv);
232 		err = skcipher_walk_done(&walk, walk.nbytes % AES_BLOCK_SIZE);
233 	}
234 	kernel_neon_end();
235 	return err;
236 }
237 
238 static int cbc_decrypt(struct skcipher_request *req)
239 {
240 	struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req);
241 	struct crypto_aes_ctx *ctx = crypto_skcipher_ctx(tfm);
242 	struct skcipher_walk walk;
243 	unsigned int blocks;
244 	int err;
245 
246 	err = skcipher_walk_virt(&walk, req, true);
247 
248 	kernel_neon_begin();
249 	while ((blocks = (walk.nbytes / AES_BLOCK_SIZE))) {
250 		ce_aes_cbc_decrypt(walk.dst.virt.addr, walk.src.virt.addr,
251 				   (u8 *)ctx->key_dec, num_rounds(ctx), blocks,
252 				   walk.iv);
253 		err = skcipher_walk_done(&walk, walk.nbytes % AES_BLOCK_SIZE);
254 	}
255 	kernel_neon_end();
256 	return err;
257 }
258 
259 static int ctr_encrypt(struct skcipher_request *req)
260 {
261 	struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req);
262 	struct crypto_aes_ctx *ctx = crypto_skcipher_ctx(tfm);
263 	struct skcipher_walk walk;
264 	int err, blocks;
265 
266 	err = skcipher_walk_virt(&walk, req, true);
267 
268 	kernel_neon_begin();
269 	while ((blocks = (walk.nbytes / AES_BLOCK_SIZE))) {
270 		ce_aes_ctr_encrypt(walk.dst.virt.addr, walk.src.virt.addr,
271 				   (u8 *)ctx->key_enc, num_rounds(ctx), blocks,
272 				   walk.iv);
273 		err = skcipher_walk_done(&walk, walk.nbytes % AES_BLOCK_SIZE);
274 	}
275 	if (walk.nbytes) {
276 		u8 __aligned(8) tail[AES_BLOCK_SIZE];
277 		unsigned int nbytes = walk.nbytes;
278 		u8 *tdst = walk.dst.virt.addr;
279 		u8 *tsrc = walk.src.virt.addr;
280 
281 		/*
282 		 * Tell aes_ctr_encrypt() to process a tail block.
283 		 */
284 		blocks = -1;
285 
286 		ce_aes_ctr_encrypt(tail, NULL, (u8 *)ctx->key_enc,
287 				   num_rounds(ctx), blocks, walk.iv);
288 		if (tdst != tsrc)
289 			memcpy(tdst, tsrc, nbytes);
290 		crypto_xor(tdst, tail, nbytes);
291 		err = skcipher_walk_done(&walk, 0);
292 	}
293 	kernel_neon_end();
294 
295 	return err;
296 }
297 
298 static int xts_encrypt(struct skcipher_request *req)
299 {
300 	struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req);
301 	struct crypto_aes_xts_ctx *ctx = crypto_skcipher_ctx(tfm);
302 	int err, first, rounds = num_rounds(&ctx->key1);
303 	struct skcipher_walk walk;
304 	unsigned int blocks;
305 
306 	err = skcipher_walk_virt(&walk, req, true);
307 
308 	kernel_neon_begin();
309 	for (first = 1; (blocks = (walk.nbytes / AES_BLOCK_SIZE)); first = 0) {
310 		ce_aes_xts_encrypt(walk.dst.virt.addr, walk.src.virt.addr,
311 				   (u8 *)ctx->key1.key_enc, rounds, blocks,
312 				   walk.iv, (u8 *)ctx->key2.key_enc, first);
313 		err = skcipher_walk_done(&walk, walk.nbytes % AES_BLOCK_SIZE);
314 	}
315 	kernel_neon_end();
316 
317 	return err;
318 }
319 
320 static int xts_decrypt(struct skcipher_request *req)
321 {
322 	struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req);
323 	struct crypto_aes_xts_ctx *ctx = crypto_skcipher_ctx(tfm);
324 	int err, first, rounds = num_rounds(&ctx->key1);
325 	struct skcipher_walk walk;
326 	unsigned int blocks;
327 
328 	err = skcipher_walk_virt(&walk, req, true);
329 
330 	kernel_neon_begin();
331 	for (first = 1; (blocks = (walk.nbytes / AES_BLOCK_SIZE)); first = 0) {
332 		ce_aes_xts_decrypt(walk.dst.virt.addr, walk.src.virt.addr,
333 				   (u8 *)ctx->key1.key_dec, rounds, blocks,
334 				   walk.iv, (u8 *)ctx->key2.key_enc, first);
335 		err = skcipher_walk_done(&walk, walk.nbytes % AES_BLOCK_SIZE);
336 	}
337 	kernel_neon_end();
338 
339 	return err;
340 }
341 
342 static struct skcipher_alg aes_algs[] = { {
343 	.base = {
344 		.cra_name		= "__ecb(aes)",
345 		.cra_driver_name	= "__ecb-aes-ce",
346 		.cra_priority		= 300,
347 		.cra_flags		= CRYPTO_ALG_INTERNAL,
348 		.cra_blocksize		= AES_BLOCK_SIZE,
349 		.cra_ctxsize		= sizeof(struct crypto_aes_ctx),
350 		.cra_module		= THIS_MODULE,
351 	},
352 	.min_keysize	= AES_MIN_KEY_SIZE,
353 	.max_keysize	= AES_MAX_KEY_SIZE,
354 	.setkey		= ce_aes_setkey,
355 	.encrypt	= ecb_encrypt,
356 	.decrypt	= ecb_decrypt,
357 }, {
358 	.base = {
359 		.cra_name		= "__cbc(aes)",
360 		.cra_driver_name	= "__cbc-aes-ce",
361 		.cra_priority		= 300,
362 		.cra_flags		= CRYPTO_ALG_INTERNAL,
363 		.cra_blocksize		= AES_BLOCK_SIZE,
364 		.cra_ctxsize		= sizeof(struct crypto_aes_ctx),
365 		.cra_module		= THIS_MODULE,
366 	},
367 	.min_keysize	= AES_MIN_KEY_SIZE,
368 	.max_keysize	= AES_MAX_KEY_SIZE,
369 	.ivsize		= AES_BLOCK_SIZE,
370 	.setkey		= ce_aes_setkey,
371 	.encrypt	= cbc_encrypt,
372 	.decrypt	= cbc_decrypt,
373 }, {
374 	.base = {
375 		.cra_name		= "__ctr(aes)",
376 		.cra_driver_name	= "__ctr-aes-ce",
377 		.cra_priority		= 300,
378 		.cra_flags		= CRYPTO_ALG_INTERNAL,
379 		.cra_blocksize		= 1,
380 		.cra_ctxsize		= sizeof(struct crypto_aes_ctx),
381 		.cra_module		= THIS_MODULE,
382 	},
383 	.min_keysize	= AES_MIN_KEY_SIZE,
384 	.max_keysize	= AES_MAX_KEY_SIZE,
385 	.ivsize		= AES_BLOCK_SIZE,
386 	.chunksize	= AES_BLOCK_SIZE,
387 	.setkey		= ce_aes_setkey,
388 	.encrypt	= ctr_encrypt,
389 	.decrypt	= ctr_encrypt,
390 }, {
391 	.base = {
392 		.cra_name		= "__xts(aes)",
393 		.cra_driver_name	= "__xts-aes-ce",
394 		.cra_priority		= 300,
395 		.cra_flags		= CRYPTO_ALG_INTERNAL,
396 		.cra_blocksize		= AES_BLOCK_SIZE,
397 		.cra_ctxsize		= sizeof(struct crypto_aes_xts_ctx),
398 		.cra_module		= THIS_MODULE,
399 	},
400 	.min_keysize	= 2 * AES_MIN_KEY_SIZE,
401 	.max_keysize	= 2 * AES_MAX_KEY_SIZE,
402 	.ivsize		= AES_BLOCK_SIZE,
403 	.setkey		= xts_set_key,
404 	.encrypt	= xts_encrypt,
405 	.decrypt	= xts_decrypt,
406 } };
407 
408 static struct simd_skcipher_alg *aes_simd_algs[ARRAY_SIZE(aes_algs)];
409 
410 static void aes_exit(void)
411 {
412 	int i;
413 
414 	for (i = 0; i < ARRAY_SIZE(aes_simd_algs) && aes_simd_algs[i]; i++)
415 		simd_skcipher_free(aes_simd_algs[i]);
416 
417 	crypto_unregister_skciphers(aes_algs, ARRAY_SIZE(aes_algs));
418 }
419 
420 static int __init aes_init(void)
421 {
422 	struct simd_skcipher_alg *simd;
423 	const char *basename;
424 	const char *algname;
425 	const char *drvname;
426 	int err;
427 	int i;
428 
429 	err = crypto_register_skciphers(aes_algs, ARRAY_SIZE(aes_algs));
430 	if (err)
431 		return err;
432 
433 	for (i = 0; i < ARRAY_SIZE(aes_algs); i++) {
434 		algname = aes_algs[i].base.cra_name + 2;
435 		drvname = aes_algs[i].base.cra_driver_name + 2;
436 		basename = aes_algs[i].base.cra_driver_name;
437 		simd = simd_skcipher_create_compat(algname, drvname, basename);
438 		err = PTR_ERR(simd);
439 		if (IS_ERR(simd))
440 			goto unregister_simds;
441 
442 		aes_simd_algs[i] = simd;
443 	}
444 
445 	return 0;
446 
447 unregister_simds:
448 	aes_exit();
449 	return err;
450 }
451 
452 module_cpu_feature_match(AES, aes_init);
453 module_exit(aes_exit);
454