1 // SPDX-License-Identifier: GPL-2.0-or-later
2 /*
3  * Glue code for AES implementation for SPE instructions (PPC)
4  *
5  * Based on generic implementation. The assembler module takes care
6  * about the SPE registers so it can run from interrupt context.
7  *
8  * Copyright (c) 2015 Markus Stockhausen <stockhausen@collogia.de>
9  */
10 
11 #include <crypto/aes.h>
12 #include <linux/module.h>
13 #include <linux/init.h>
14 #include <linux/types.h>
15 #include <linux/errno.h>
16 #include <linux/crypto.h>
17 #include <asm/byteorder.h>
18 #include <asm/switch_to.h>
19 #include <crypto/algapi.h>
20 #include <crypto/internal/skcipher.h>
21 #include <crypto/xts.h>
22 #include <crypto/gf128mul.h>
23 #include <crypto/scatterwalk.h>
24 
25 /*
26  * MAX_BYTES defines the number of bytes that are allowed to be processed
27  * between preempt_disable() and preempt_enable(). e500 cores can issue two
28  * instructions per clock cycle using one 32/64 bit unit (SU1) and one 32
29  * bit unit (SU2). One of these can be a memory access that is executed via
30  * a single load and store unit (LSU). XTS-AES-256 takes ~780 operations per
31  * 16 byte block block or 25 cycles per byte. Thus 768 bytes of input data
32  * will need an estimated maximum of 20,000 cycles. Headroom for cache misses
33  * included. Even with the low end model clocked at 667 MHz this equals to a
34  * critical time window of less than 30us. The value has been chosen to
35  * process a 512 byte disk block in one or a large 1400 bytes IPsec network
36  * packet in two runs.
37  *
38  */
39 #define MAX_BYTES 768
40 
41 struct ppc_aes_ctx {
42 	u32 key_enc[AES_MAX_KEYLENGTH_U32];
43 	u32 key_dec[AES_MAX_KEYLENGTH_U32];
44 	u32 rounds;
45 };
46 
47 struct ppc_xts_ctx {
48 	u32 key_enc[AES_MAX_KEYLENGTH_U32];
49 	u32 key_dec[AES_MAX_KEYLENGTH_U32];
50 	u32 key_twk[AES_MAX_KEYLENGTH_U32];
51 	u32 rounds;
52 };
53 
54 extern void ppc_encrypt_aes(u8 *out, const u8 *in, u32 *key_enc, u32 rounds);
55 extern void ppc_decrypt_aes(u8 *out, const u8 *in, u32 *key_dec, u32 rounds);
56 extern void ppc_encrypt_ecb(u8 *out, const u8 *in, u32 *key_enc, u32 rounds,
57 			    u32 bytes);
58 extern void ppc_decrypt_ecb(u8 *out, const u8 *in, u32 *key_dec, u32 rounds,
59 			    u32 bytes);
60 extern void ppc_encrypt_cbc(u8 *out, const u8 *in, u32 *key_enc, u32 rounds,
61 			    u32 bytes, u8 *iv);
62 extern void ppc_decrypt_cbc(u8 *out, const u8 *in, u32 *key_dec, u32 rounds,
63 			    u32 bytes, u8 *iv);
64 extern void ppc_crypt_ctr  (u8 *out, const u8 *in, u32 *key_enc, u32 rounds,
65 			    u32 bytes, u8 *iv);
66 extern void ppc_encrypt_xts(u8 *out, const u8 *in, u32 *key_enc, u32 rounds,
67 			    u32 bytes, u8 *iv, u32 *key_twk);
68 extern void ppc_decrypt_xts(u8 *out, const u8 *in, u32 *key_dec, u32 rounds,
69 			    u32 bytes, u8 *iv, u32 *key_twk);
70 
71 extern void ppc_expand_key_128(u32 *key_enc, const u8 *key);
72 extern void ppc_expand_key_192(u32 *key_enc, const u8 *key);
73 extern void ppc_expand_key_256(u32 *key_enc, const u8 *key);
74 
75 extern void ppc_generate_decrypt_key(u32 *key_dec,u32 *key_enc,
76 				     unsigned int key_len);
77 
78 static void spe_begin(void)
79 {
80 	/* disable preemption and save users SPE registers if required */
81 	preempt_disable();
82 	enable_kernel_spe();
83 }
84 
85 static void spe_end(void)
86 {
87 	disable_kernel_spe();
88 	/* reenable preemption */
89 	preempt_enable();
90 }
91 
92 static int ppc_aes_setkey(struct crypto_tfm *tfm, const u8 *in_key,
93 		unsigned int key_len)
94 {
95 	struct ppc_aes_ctx *ctx = crypto_tfm_ctx(tfm);
96 
97 	switch (key_len) {
98 	case AES_KEYSIZE_128:
99 		ctx->rounds = 4;
100 		ppc_expand_key_128(ctx->key_enc, in_key);
101 		break;
102 	case AES_KEYSIZE_192:
103 		ctx->rounds = 5;
104 		ppc_expand_key_192(ctx->key_enc, in_key);
105 		break;
106 	case AES_KEYSIZE_256:
107 		ctx->rounds = 6;
108 		ppc_expand_key_256(ctx->key_enc, in_key);
109 		break;
110 	default:
111 		return -EINVAL;
112 	}
113 
114 	ppc_generate_decrypt_key(ctx->key_dec, ctx->key_enc, key_len);
115 
116 	return 0;
117 }
118 
119 static int ppc_aes_setkey_skcipher(struct crypto_skcipher *tfm,
120 				   const u8 *in_key, unsigned int key_len)
121 {
122 	return ppc_aes_setkey(crypto_skcipher_tfm(tfm), in_key, key_len);
123 }
124 
125 static int ppc_xts_setkey(struct crypto_skcipher *tfm, const u8 *in_key,
126 		   unsigned int key_len)
127 {
128 	struct ppc_xts_ctx *ctx = crypto_skcipher_ctx(tfm);
129 	int err;
130 
131 	err = xts_verify_key(tfm, in_key, key_len);
132 	if (err)
133 		return err;
134 
135 	key_len >>= 1;
136 
137 	switch (key_len) {
138 	case AES_KEYSIZE_128:
139 		ctx->rounds = 4;
140 		ppc_expand_key_128(ctx->key_enc, in_key);
141 		ppc_expand_key_128(ctx->key_twk, in_key + AES_KEYSIZE_128);
142 		break;
143 	case AES_KEYSIZE_192:
144 		ctx->rounds = 5;
145 		ppc_expand_key_192(ctx->key_enc, in_key);
146 		ppc_expand_key_192(ctx->key_twk, in_key + AES_KEYSIZE_192);
147 		break;
148 	case AES_KEYSIZE_256:
149 		ctx->rounds = 6;
150 		ppc_expand_key_256(ctx->key_enc, in_key);
151 		ppc_expand_key_256(ctx->key_twk, in_key + AES_KEYSIZE_256);
152 		break;
153 	default:
154 		return -EINVAL;
155 	}
156 
157 	ppc_generate_decrypt_key(ctx->key_dec, ctx->key_enc, key_len);
158 
159 	return 0;
160 }
161 
162 static void ppc_aes_encrypt(struct crypto_tfm *tfm, u8 *out, const u8 *in)
163 {
164 	struct ppc_aes_ctx *ctx = crypto_tfm_ctx(tfm);
165 
166 	spe_begin();
167 	ppc_encrypt_aes(out, in, ctx->key_enc, ctx->rounds);
168 	spe_end();
169 }
170 
171 static void ppc_aes_decrypt(struct crypto_tfm *tfm, u8 *out, const u8 *in)
172 {
173 	struct ppc_aes_ctx *ctx = crypto_tfm_ctx(tfm);
174 
175 	spe_begin();
176 	ppc_decrypt_aes(out, in, ctx->key_dec, ctx->rounds);
177 	spe_end();
178 }
179 
180 static int ppc_ecb_crypt(struct skcipher_request *req, bool enc)
181 {
182 	struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req);
183 	struct ppc_aes_ctx *ctx = crypto_skcipher_ctx(tfm);
184 	struct skcipher_walk walk;
185 	unsigned int nbytes;
186 	int err;
187 
188 	err = skcipher_walk_virt(&walk, req, false);
189 
190 	while ((nbytes = walk.nbytes) != 0) {
191 		nbytes = min_t(unsigned int, nbytes, MAX_BYTES);
192 		nbytes = round_down(nbytes, AES_BLOCK_SIZE);
193 
194 		spe_begin();
195 		if (enc)
196 			ppc_encrypt_ecb(walk.dst.virt.addr, walk.src.virt.addr,
197 					ctx->key_enc, ctx->rounds, nbytes);
198 		else
199 			ppc_decrypt_ecb(walk.dst.virt.addr, walk.src.virt.addr,
200 					ctx->key_dec, ctx->rounds, nbytes);
201 		spe_end();
202 
203 		err = skcipher_walk_done(&walk, walk.nbytes - nbytes);
204 	}
205 
206 	return err;
207 }
208 
209 static int ppc_ecb_encrypt(struct skcipher_request *req)
210 {
211 	return ppc_ecb_crypt(req, true);
212 }
213 
214 static int ppc_ecb_decrypt(struct skcipher_request *req)
215 {
216 	return ppc_ecb_crypt(req, false);
217 }
218 
219 static int ppc_cbc_crypt(struct skcipher_request *req, bool enc)
220 {
221 	struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req);
222 	struct ppc_aes_ctx *ctx = crypto_skcipher_ctx(tfm);
223 	struct skcipher_walk walk;
224 	unsigned int nbytes;
225 	int err;
226 
227 	err = skcipher_walk_virt(&walk, req, false);
228 
229 	while ((nbytes = walk.nbytes) != 0) {
230 		nbytes = min_t(unsigned int, nbytes, MAX_BYTES);
231 		nbytes = round_down(nbytes, AES_BLOCK_SIZE);
232 
233 		spe_begin();
234 		if (enc)
235 			ppc_encrypt_cbc(walk.dst.virt.addr, walk.src.virt.addr,
236 					ctx->key_enc, ctx->rounds, nbytes,
237 					walk.iv);
238 		else
239 			ppc_decrypt_cbc(walk.dst.virt.addr, walk.src.virt.addr,
240 					ctx->key_dec, ctx->rounds, nbytes,
241 					walk.iv);
242 		spe_end();
243 
244 		err = skcipher_walk_done(&walk, walk.nbytes - nbytes);
245 	}
246 
247 	return err;
248 }
249 
250 static int ppc_cbc_encrypt(struct skcipher_request *req)
251 {
252 	return ppc_cbc_crypt(req, true);
253 }
254 
255 static int ppc_cbc_decrypt(struct skcipher_request *req)
256 {
257 	return ppc_cbc_crypt(req, false);
258 }
259 
260 static int ppc_ctr_crypt(struct skcipher_request *req)
261 {
262 	struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req);
263 	struct ppc_aes_ctx *ctx = crypto_skcipher_ctx(tfm);
264 	struct skcipher_walk walk;
265 	unsigned int nbytes;
266 	int err;
267 
268 	err = skcipher_walk_virt(&walk, req, false);
269 
270 	while ((nbytes = walk.nbytes) != 0) {
271 		nbytes = min_t(unsigned int, nbytes, MAX_BYTES);
272 		if (nbytes < walk.total)
273 			nbytes = round_down(nbytes, AES_BLOCK_SIZE);
274 
275 		spe_begin();
276 		ppc_crypt_ctr(walk.dst.virt.addr, walk.src.virt.addr,
277 			      ctx->key_enc, ctx->rounds, nbytes, walk.iv);
278 		spe_end();
279 
280 		err = skcipher_walk_done(&walk, walk.nbytes - nbytes);
281 	}
282 
283 	return err;
284 }
285 
286 static int ppc_xts_crypt(struct skcipher_request *req, bool enc)
287 {
288 	struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req);
289 	struct ppc_xts_ctx *ctx = crypto_skcipher_ctx(tfm);
290 	struct skcipher_walk walk;
291 	unsigned int nbytes;
292 	int err;
293 	u32 *twk;
294 
295 	err = skcipher_walk_virt(&walk, req, false);
296 	twk = ctx->key_twk;
297 
298 	while ((nbytes = walk.nbytes) != 0) {
299 		nbytes = min_t(unsigned int, nbytes, MAX_BYTES);
300 		nbytes = round_down(nbytes, AES_BLOCK_SIZE);
301 
302 		spe_begin();
303 		if (enc)
304 			ppc_encrypt_xts(walk.dst.virt.addr, walk.src.virt.addr,
305 					ctx->key_enc, ctx->rounds, nbytes,
306 					walk.iv, twk);
307 		else
308 			ppc_decrypt_xts(walk.dst.virt.addr, walk.src.virt.addr,
309 					ctx->key_dec, ctx->rounds, nbytes,
310 					walk.iv, twk);
311 		spe_end();
312 
313 		twk = NULL;
314 		err = skcipher_walk_done(&walk, walk.nbytes - nbytes);
315 	}
316 
317 	return err;
318 }
319 
320 static int ppc_xts_encrypt(struct skcipher_request *req)
321 {
322 	struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req);
323 	struct ppc_xts_ctx *ctx = crypto_skcipher_ctx(tfm);
324 	int tail = req->cryptlen % AES_BLOCK_SIZE;
325 	int offset = req->cryptlen - tail - AES_BLOCK_SIZE;
326 	struct skcipher_request subreq;
327 	u8 b[2][AES_BLOCK_SIZE];
328 	int err;
329 
330 	if (req->cryptlen < AES_BLOCK_SIZE)
331 		return -EINVAL;
332 
333 	if (tail) {
334 		subreq = *req;
335 		skcipher_request_set_crypt(&subreq, req->src, req->dst,
336 					   req->cryptlen - tail, req->iv);
337 		req = &subreq;
338 	}
339 
340 	err = ppc_xts_crypt(req, true);
341 	if (err || !tail)
342 		return err;
343 
344 	scatterwalk_map_and_copy(b[0], req->dst, offset, AES_BLOCK_SIZE, 0);
345 	memcpy(b[1], b[0], tail);
346 	scatterwalk_map_and_copy(b[0], req->src, offset + AES_BLOCK_SIZE, tail, 0);
347 
348 	spe_begin();
349 	ppc_encrypt_xts(b[0], b[0], ctx->key_enc, ctx->rounds, AES_BLOCK_SIZE,
350 			req->iv, NULL);
351 	spe_end();
352 
353 	scatterwalk_map_and_copy(b[0], req->dst, offset, AES_BLOCK_SIZE + tail, 1);
354 
355 	return 0;
356 }
357 
358 static int ppc_xts_decrypt(struct skcipher_request *req)
359 {
360 	struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req);
361 	struct ppc_xts_ctx *ctx = crypto_skcipher_ctx(tfm);
362 	int tail = req->cryptlen % AES_BLOCK_SIZE;
363 	int offset = req->cryptlen - tail - AES_BLOCK_SIZE;
364 	struct skcipher_request subreq;
365 	u8 b[3][AES_BLOCK_SIZE];
366 	le128 twk;
367 	int err;
368 
369 	if (req->cryptlen < AES_BLOCK_SIZE)
370 		return -EINVAL;
371 
372 	if (tail) {
373 		subreq = *req;
374 		skcipher_request_set_crypt(&subreq, req->src, req->dst,
375 					   offset, req->iv);
376 		req = &subreq;
377 	}
378 
379 	err = ppc_xts_crypt(req, false);
380 	if (err || !tail)
381 		return err;
382 
383 	scatterwalk_map_and_copy(b[1], req->src, offset, AES_BLOCK_SIZE + tail, 0);
384 
385 	spe_begin();
386 	if (!offset)
387 		ppc_encrypt_ecb(req->iv, req->iv, ctx->key_twk, ctx->rounds,
388 				AES_BLOCK_SIZE);
389 
390 	gf128mul_x_ble(&twk, (le128 *)req->iv);
391 
392 	ppc_decrypt_xts(b[1], b[1], ctx->key_dec, ctx->rounds, AES_BLOCK_SIZE,
393 			(u8 *)&twk, NULL);
394 	memcpy(b[0], b[2], tail);
395 	memcpy(b[0] + tail, b[1] + tail, AES_BLOCK_SIZE - tail);
396 	ppc_decrypt_xts(b[0], b[0], ctx->key_dec, ctx->rounds, AES_BLOCK_SIZE,
397 			req->iv, NULL);
398 	spe_end();
399 
400 	scatterwalk_map_and_copy(b[0], req->dst, offset, AES_BLOCK_SIZE + tail, 1);
401 
402 	return 0;
403 }
404 
405 /*
406  * Algorithm definitions. Disabling alignment (cra_alignmask=0) was chosen
407  * because the e500 platform can handle unaligned reads/writes very efficently.
408  * This improves IPsec thoughput by another few percent. Additionally we assume
409  * that AES context is always aligned to at least 8 bytes because it is created
410  * with kmalloc() in the crypto infrastructure
411  */
412 
413 static struct crypto_alg aes_cipher_alg = {
414 	.cra_name		=	"aes",
415 	.cra_driver_name	=	"aes-ppc-spe",
416 	.cra_priority		=	300,
417 	.cra_flags		=	CRYPTO_ALG_TYPE_CIPHER,
418 	.cra_blocksize		=	AES_BLOCK_SIZE,
419 	.cra_ctxsize		=	sizeof(struct ppc_aes_ctx),
420 	.cra_alignmask		=	0,
421 	.cra_module		=	THIS_MODULE,
422 	.cra_u			=	{
423 		.cipher = {
424 			.cia_min_keysize	=	AES_MIN_KEY_SIZE,
425 			.cia_max_keysize	=	AES_MAX_KEY_SIZE,
426 			.cia_setkey		=	ppc_aes_setkey,
427 			.cia_encrypt		=	ppc_aes_encrypt,
428 			.cia_decrypt		=	ppc_aes_decrypt
429 		}
430 	}
431 };
432 
433 static struct skcipher_alg aes_skcipher_algs[] = {
434 	{
435 		.base.cra_name		=	"ecb(aes)",
436 		.base.cra_driver_name	=	"ecb-ppc-spe",
437 		.base.cra_priority	=	300,
438 		.base.cra_blocksize	=	AES_BLOCK_SIZE,
439 		.base.cra_ctxsize	=	sizeof(struct ppc_aes_ctx),
440 		.base.cra_module	=	THIS_MODULE,
441 		.min_keysize		=	AES_MIN_KEY_SIZE,
442 		.max_keysize		=	AES_MAX_KEY_SIZE,
443 		.setkey			=	ppc_aes_setkey_skcipher,
444 		.encrypt		=	ppc_ecb_encrypt,
445 		.decrypt		=	ppc_ecb_decrypt,
446 	}, {
447 		.base.cra_name		=	"cbc(aes)",
448 		.base.cra_driver_name	=	"cbc-ppc-spe",
449 		.base.cra_priority	=	300,
450 		.base.cra_blocksize	=	AES_BLOCK_SIZE,
451 		.base.cra_ctxsize	=	sizeof(struct ppc_aes_ctx),
452 		.base.cra_module	=	THIS_MODULE,
453 		.min_keysize		=	AES_MIN_KEY_SIZE,
454 		.max_keysize		=	AES_MAX_KEY_SIZE,
455 		.ivsize			=	AES_BLOCK_SIZE,
456 		.setkey			=	ppc_aes_setkey_skcipher,
457 		.encrypt		=	ppc_cbc_encrypt,
458 		.decrypt		=	ppc_cbc_decrypt,
459 	}, {
460 		.base.cra_name		=	"ctr(aes)",
461 		.base.cra_driver_name	=	"ctr-ppc-spe",
462 		.base.cra_priority	=	300,
463 		.base.cra_blocksize	=	1,
464 		.base.cra_ctxsize	=	sizeof(struct ppc_aes_ctx),
465 		.base.cra_module	=	THIS_MODULE,
466 		.min_keysize		=	AES_MIN_KEY_SIZE,
467 		.max_keysize		=	AES_MAX_KEY_SIZE,
468 		.ivsize			=	AES_BLOCK_SIZE,
469 		.setkey			=	ppc_aes_setkey_skcipher,
470 		.encrypt		=	ppc_ctr_crypt,
471 		.decrypt		=	ppc_ctr_crypt,
472 		.chunksize		=	AES_BLOCK_SIZE,
473 	}, {
474 		.base.cra_name		=	"xts(aes)",
475 		.base.cra_driver_name	=	"xts-ppc-spe",
476 		.base.cra_priority	=	300,
477 		.base.cra_blocksize	=	AES_BLOCK_SIZE,
478 		.base.cra_ctxsize	=	sizeof(struct ppc_xts_ctx),
479 		.base.cra_module	=	THIS_MODULE,
480 		.min_keysize		=	AES_MIN_KEY_SIZE * 2,
481 		.max_keysize		=	AES_MAX_KEY_SIZE * 2,
482 		.ivsize			=	AES_BLOCK_SIZE,
483 		.setkey			=	ppc_xts_setkey,
484 		.encrypt		=	ppc_xts_encrypt,
485 		.decrypt		=	ppc_xts_decrypt,
486 	}
487 };
488 
489 static int __init ppc_aes_mod_init(void)
490 {
491 	int err;
492 
493 	err = crypto_register_alg(&aes_cipher_alg);
494 	if (err)
495 		return err;
496 
497 	err = crypto_register_skciphers(aes_skcipher_algs,
498 					ARRAY_SIZE(aes_skcipher_algs));
499 	if (err)
500 		crypto_unregister_alg(&aes_cipher_alg);
501 	return err;
502 }
503 
504 static void __exit ppc_aes_mod_fini(void)
505 {
506 	crypto_unregister_alg(&aes_cipher_alg);
507 	crypto_unregister_skciphers(aes_skcipher_algs,
508 				    ARRAY_SIZE(aes_skcipher_algs));
509 }
510 
511 module_init(ppc_aes_mod_init);
512 module_exit(ppc_aes_mod_fini);
513 
514 MODULE_LICENSE("GPL");
515 MODULE_DESCRIPTION("AES-ECB/CBC/CTR/XTS, SPE optimized");
516 
517 MODULE_ALIAS_CRYPTO("aes");
518 MODULE_ALIAS_CRYPTO("ecb(aes)");
519 MODULE_ALIAS_CRYPTO("cbc(aes)");
520 MODULE_ALIAS_CRYPTO("ctr(aes)");
521 MODULE_ALIAS_CRYPTO("xts(aes)");
522 MODULE_ALIAS_CRYPTO("aes-ppc-spe");
523