xref: /openbmc/linux/drivers/crypto/geode-aes.c (revision 51b67a6e) 
1 // SPDX-License-Identifier: GPL-2.0-or-later
2  /* Copyright (C) 2004-2006, Advanced Micro Devices, Inc.
3   */
4 
5 #include <linux/module.h>
6 #include <linux/kernel.h>
7 #include <linux/pci.h>
8 #include <linux/pci_ids.h>
9 #include <linux/crypto.h>
10 #include <linux/spinlock.h>
11 #include <crypto/algapi.h>
12 #include <crypto/aes.h>
13 
14 #include <linux/io.h>
15 #include <linux/delay.h>
16 
17 #include "geode-aes.h"
18 
19 /* Static structures */
20 
21 static void __iomem *_iobase;
22 static spinlock_t lock;
23 
24 /* Write a 128 bit field (either a writable key or IV) */
25 static inline void
26 _writefield(u32 offset, void *value)
27 {
28 	int i;
29 
30 	for (i = 0; i < 4; i++)
31 		iowrite32(((u32 *) value)[i], _iobase + offset + (i * 4));
32 }
33 
34 /* Read a 128 bit field (either a writable key or IV) */
35 static inline void
36 _readfield(u32 offset, void *value)
37 {
38 	int i;
39 
40 	for (i = 0; i < 4; i++)
41 		((u32 *) value)[i] = ioread32(_iobase + offset + (i * 4));
42 }
43 
44 static int
45 do_crypt(void *src, void *dst, int len, u32 flags)
46 {
47 	u32 status;
48 	u32 counter = AES_OP_TIMEOUT;
49 
50 	iowrite32(virt_to_phys(src), _iobase + AES_SOURCEA_REG);
51 	iowrite32(virt_to_phys(dst), _iobase + AES_DSTA_REG);
52 	iowrite32(len,  _iobase + AES_LENA_REG);
53 
54 	/* Start the operation */
55 	iowrite32(AES_CTRL_START | flags, _iobase + AES_CTRLA_REG);
56 
57 	do {
58 		status = ioread32(_iobase + AES_INTR_REG);
59 		cpu_relax();
60 	} while (!(status & AES_INTRA_PENDING) && --counter);
61 
62 	/* Clear the event */
63 	iowrite32((status & 0xFF) | AES_INTRA_PENDING, _iobase + AES_INTR_REG);
64 	return counter ? 0 : 1;
65 }
66 
67 static unsigned int
68 geode_aes_crypt(struct geode_aes_op *op)
69 {
70 	u32 flags = 0;
71 	unsigned long iflags;
72 	int ret;
73 
74 	if (op->len == 0)
75 		return 0;
76 
77 	/* If the source and destination is the same, then
78 	 * we need to turn on the coherent flags, otherwise
79 	 * we don't need to worry
80 	 */
81 
82 	flags |= (AES_CTRL_DCA | AES_CTRL_SCA);
83 
84 	if (op->dir == AES_DIR_ENCRYPT)
85 		flags |= AES_CTRL_ENCRYPT;
86 
87 	/* Start the critical section */
88 
89 	spin_lock_irqsave(&lock, iflags);
90 
91 	if (op->mode == AES_MODE_CBC) {
92 		flags |= AES_CTRL_CBC;
93 		_writefield(AES_WRITEIV0_REG, op->iv);
94 	}
95 
96 	if (!(op->flags & AES_FLAGS_HIDDENKEY)) {
97 		flags |= AES_CTRL_WRKEY;
98 		_writefield(AES_WRITEKEY0_REG, op->key);
99 	}
100 
101 	ret = do_crypt(op->src, op->dst, op->len, flags);
102 	BUG_ON(ret);
103 
104 	if (op->mode == AES_MODE_CBC)
105 		_readfield(AES_WRITEIV0_REG, op->iv);
106 
107 	spin_unlock_irqrestore(&lock, iflags);
108 
109 	return op->len;
110 }
111 
112 /* CRYPTO-API Functions */
113 
114 static int geode_setkey_cip(struct crypto_tfm *tfm, const u8 *key,
115 		unsigned int len)
116 {
117 	struct geode_aes_op *op = crypto_tfm_ctx(tfm);
118 	unsigned int ret;
119 
120 	op->keylen = len;
121 
122 	if (len == AES_KEYSIZE_128) {
123 		memcpy(op->key, key, len);
124 		return 0;
125 	}
126 
127 	if (len != AES_KEYSIZE_192 && len != AES_KEYSIZE_256) {
128 		/* not supported at all */
129 		tfm->crt_flags |= CRYPTO_TFM_RES_BAD_KEY_LEN;
130 		return -EINVAL;
131 	}
132 
133 	/*
134 	 * The requested key size is not supported by HW, do a fallback
135 	 */
136 	op->fallback.cip->base.crt_flags &= ~CRYPTO_TFM_REQ_MASK;
137 	op->fallback.cip->base.crt_flags |= (tfm->crt_flags & CRYPTO_TFM_REQ_MASK);
138 
139 	ret = crypto_cipher_setkey(op->fallback.cip, key, len);
140 	if (ret) {
141 		tfm->crt_flags &= ~CRYPTO_TFM_RES_MASK;
142 		tfm->crt_flags |= (op->fallback.cip->base.crt_flags & CRYPTO_TFM_RES_MASK);
143 	}
144 	return ret;
145 }
146 
147 static int geode_setkey_blk(struct crypto_tfm *tfm, const u8 *key,
148 		unsigned int len)
149 {
150 	struct geode_aes_op *op = crypto_tfm_ctx(tfm);
151 	unsigned int ret;
152 
153 	op->keylen = len;
154 
155 	if (len == AES_KEYSIZE_128) {
156 		memcpy(op->key, key, len);
157 		return 0;
158 	}
159 
160 	if (len != AES_KEYSIZE_192 && len != AES_KEYSIZE_256) {
161 		/* not supported at all */
162 		tfm->crt_flags |= CRYPTO_TFM_RES_BAD_KEY_LEN;
163 		return -EINVAL;
164 	}
165 
166 	/*
167 	 * The requested key size is not supported by HW, do a fallback
168 	 */
169 	op->fallback.blk->base.crt_flags &= ~CRYPTO_TFM_REQ_MASK;
170 	op->fallback.blk->base.crt_flags |= (tfm->crt_flags & CRYPTO_TFM_REQ_MASK);
171 
172 	ret = crypto_blkcipher_setkey(op->fallback.blk, key, len);
173 	if (ret) {
174 		tfm->crt_flags &= ~CRYPTO_TFM_RES_MASK;
175 		tfm->crt_flags |= (op->fallback.blk->base.crt_flags & CRYPTO_TFM_RES_MASK);
176 	}
177 	return ret;
178 }
179 
180 static int fallback_blk_dec(struct blkcipher_desc *desc,
181 		struct scatterlist *dst, struct scatterlist *src,
182 		unsigned int nbytes)
183 {
184 	unsigned int ret;
185 	struct crypto_blkcipher *tfm;
186 	struct geode_aes_op *op = crypto_blkcipher_ctx(desc->tfm);
187 
188 	tfm = desc->tfm;
189 	desc->tfm = op->fallback.blk;
190 
191 	ret = crypto_blkcipher_decrypt_iv(desc, dst, src, nbytes);
192 
193 	desc->tfm = tfm;
194 	return ret;
195 }
196 static int fallback_blk_enc(struct blkcipher_desc *desc,
197 		struct scatterlist *dst, struct scatterlist *src,
198 		unsigned int nbytes)
199 {
200 	unsigned int ret;
201 	struct crypto_blkcipher *tfm;
202 	struct geode_aes_op *op = crypto_blkcipher_ctx(desc->tfm);
203 
204 	tfm = desc->tfm;
205 	desc->tfm = op->fallback.blk;
206 
207 	ret = crypto_blkcipher_encrypt_iv(desc, dst, src, nbytes);
208 
209 	desc->tfm = tfm;
210 	return ret;
211 }
212 
213 static void
214 geode_encrypt(struct crypto_tfm *tfm, u8 *out, const u8 *in)
215 {
216 	struct geode_aes_op *op = crypto_tfm_ctx(tfm);
217 
218 	if (unlikely(op->keylen != AES_KEYSIZE_128)) {
219 		crypto_cipher_encrypt_one(op->fallback.cip, out, in);
220 		return;
221 	}
222 
223 	op->src = (void *) in;
224 	op->dst = (void *) out;
225 	op->mode = AES_MODE_ECB;
226 	op->flags = 0;
227 	op->len = AES_BLOCK_SIZE;
228 	op->dir = AES_DIR_ENCRYPT;
229 
230 	geode_aes_crypt(op);
231 }
232 
233 
234 static void
235 geode_decrypt(struct crypto_tfm *tfm, u8 *out, const u8 *in)
236 {
237 	struct geode_aes_op *op = crypto_tfm_ctx(tfm);
238 
239 	if (unlikely(op->keylen != AES_KEYSIZE_128)) {
240 		crypto_cipher_decrypt_one(op->fallback.cip, out, in);
241 		return;
242 	}
243 
244 	op->src = (void *) in;
245 	op->dst = (void *) out;
246 	op->mode = AES_MODE_ECB;
247 	op->flags = 0;
248 	op->len = AES_BLOCK_SIZE;
249 	op->dir = AES_DIR_DECRYPT;
250 
251 	geode_aes_crypt(op);
252 }
253 
254 static int fallback_init_cip(struct crypto_tfm *tfm)
255 {
256 	const char *name = crypto_tfm_alg_name(tfm);
257 	struct geode_aes_op *op = crypto_tfm_ctx(tfm);
258 
259 	op->fallback.cip = crypto_alloc_cipher(name, 0,
260 					       CRYPTO_ALG_NEED_FALLBACK);
261 
262 	if (IS_ERR(op->fallback.cip)) {
263 		printk(KERN_ERR "Error allocating fallback algo %s\n", name);
264 		return PTR_ERR(op->fallback.cip);
265 	}
266 
267 	return 0;
268 }
269 
270 static void fallback_exit_cip(struct crypto_tfm *tfm)
271 {
272 	struct geode_aes_op *op = crypto_tfm_ctx(tfm);
273 
274 	crypto_free_cipher(op->fallback.cip);
275 	op->fallback.cip = NULL;
276 }
277 
278 static struct crypto_alg geode_alg = {
279 	.cra_name			=	"aes",
280 	.cra_driver_name	=	"geode-aes",
281 	.cra_priority		=	300,
282 	.cra_alignmask		=	15,
283 	.cra_flags			=	CRYPTO_ALG_TYPE_CIPHER |
284 							CRYPTO_ALG_NEED_FALLBACK,
285 	.cra_init			=	fallback_init_cip,
286 	.cra_exit			=	fallback_exit_cip,
287 	.cra_blocksize		=	AES_BLOCK_SIZE,
288 	.cra_ctxsize		=	sizeof(struct geode_aes_op),
289 	.cra_module			=	THIS_MODULE,
290 	.cra_u				=	{
291 		.cipher	=	{
292 			.cia_min_keysize	=	AES_MIN_KEY_SIZE,
293 			.cia_max_keysize	=	AES_MAX_KEY_SIZE,
294 			.cia_setkey			=	geode_setkey_cip,
295 			.cia_encrypt		=	geode_encrypt,
296 			.cia_decrypt		=	geode_decrypt
297 		}
298 	}
299 };
300 
301 static int
302 geode_cbc_decrypt(struct blkcipher_desc *desc,
303 		  struct scatterlist *dst, struct scatterlist *src,
304 		  unsigned int nbytes)
305 {
306 	struct geode_aes_op *op = crypto_blkcipher_ctx(desc->tfm);
307 	struct blkcipher_walk walk;
308 	int err, ret;
309 
310 	if (unlikely(op->keylen != AES_KEYSIZE_128))
311 		return fallback_blk_dec(desc, dst, src, nbytes);
312 
313 	blkcipher_walk_init(&walk, dst, src, nbytes);
314 	err = blkcipher_walk_virt(desc, &walk);
315 	op->iv = walk.iv;
316 
317 	while ((nbytes = walk.nbytes)) {
318 		op->src = walk.src.virt.addr,
319 		op->dst = walk.dst.virt.addr;
320 		op->mode = AES_MODE_CBC;
321 		op->len = nbytes - (nbytes % AES_BLOCK_SIZE);
322 		op->dir = AES_DIR_DECRYPT;
323 
324 		ret = geode_aes_crypt(op);
325 
326 		nbytes -= ret;
327 		err = blkcipher_walk_done(desc, &walk, nbytes);
328 	}
329 
330 	return err;
331 }
332 
333 static int
334 geode_cbc_encrypt(struct blkcipher_desc *desc,
335 		  struct scatterlist *dst, struct scatterlist *src,
336 		  unsigned int nbytes)
337 {
338 	struct geode_aes_op *op = crypto_blkcipher_ctx(desc->tfm);
339 	struct blkcipher_walk walk;
340 	int err, ret;
341 
342 	if (unlikely(op->keylen != AES_KEYSIZE_128))
343 		return fallback_blk_enc(desc, dst, src, nbytes);
344 
345 	blkcipher_walk_init(&walk, dst, src, nbytes);
346 	err = blkcipher_walk_virt(desc, &walk);
347 	op->iv = walk.iv;
348 
349 	while ((nbytes = walk.nbytes)) {
350 		op->src = walk.src.virt.addr,
351 		op->dst = walk.dst.virt.addr;
352 		op->mode = AES_MODE_CBC;
353 		op->len = nbytes - (nbytes % AES_BLOCK_SIZE);
354 		op->dir = AES_DIR_ENCRYPT;
355 
356 		ret = geode_aes_crypt(op);
357 		nbytes -= ret;
358 		err = blkcipher_walk_done(desc, &walk, nbytes);
359 	}
360 
361 	return err;
362 }
363 
364 static int fallback_init_blk(struct crypto_tfm *tfm)
365 {
366 	const char *name = crypto_tfm_alg_name(tfm);
367 	struct geode_aes_op *op = crypto_tfm_ctx(tfm);
368 
369 	op->fallback.blk = crypto_alloc_blkcipher(name, 0,
370 			CRYPTO_ALG_ASYNC | CRYPTO_ALG_NEED_FALLBACK);
371 
372 	if (IS_ERR(op->fallback.blk)) {
373 		printk(KERN_ERR "Error allocating fallback algo %s\n", name);
374 		return PTR_ERR(op->fallback.blk);
375 	}
376 
377 	return 0;
378 }
379 
380 static void fallback_exit_blk(struct crypto_tfm *tfm)
381 {
382 	struct geode_aes_op *op = crypto_tfm_ctx(tfm);
383 
384 	crypto_free_blkcipher(op->fallback.blk);
385 	op->fallback.blk = NULL;
386 }
387 
388 static struct crypto_alg geode_cbc_alg = {
389 	.cra_name		=	"cbc(aes)",
390 	.cra_driver_name	=	"cbc-aes-geode",
391 	.cra_priority		=	400,
392 	.cra_flags			=	CRYPTO_ALG_TYPE_BLKCIPHER |
393 						CRYPTO_ALG_KERN_DRIVER_ONLY |
394 						CRYPTO_ALG_NEED_FALLBACK,
395 	.cra_init			=	fallback_init_blk,
396 	.cra_exit			=	fallback_exit_blk,
397 	.cra_blocksize		=	AES_BLOCK_SIZE,
398 	.cra_ctxsize		=	sizeof(struct geode_aes_op),
399 	.cra_alignmask		=	15,
400 	.cra_type			=	&crypto_blkcipher_type,
401 	.cra_module			=	THIS_MODULE,
402 	.cra_u				=	{
403 		.blkcipher	=	{
404 			.min_keysize	=	AES_MIN_KEY_SIZE,
405 			.max_keysize	=	AES_MAX_KEY_SIZE,
406 			.setkey			=	geode_setkey_blk,
407 			.encrypt		=	geode_cbc_encrypt,
408 			.decrypt		=	geode_cbc_decrypt,
409 			.ivsize			=	AES_BLOCK_SIZE,
410 		}
411 	}
412 };
413 
414 static int
415 geode_ecb_decrypt(struct blkcipher_desc *desc,
416 		  struct scatterlist *dst, struct scatterlist *src,
417 		  unsigned int nbytes)
418 {
419 	struct geode_aes_op *op = crypto_blkcipher_ctx(desc->tfm);
420 	struct blkcipher_walk walk;
421 	int err, ret;
422 
423 	if (unlikely(op->keylen != AES_KEYSIZE_128))
424 		return fallback_blk_dec(desc, dst, src, nbytes);
425 
426 	blkcipher_walk_init(&walk, dst, src, nbytes);
427 	err = blkcipher_walk_virt(desc, &walk);
428 
429 	while ((nbytes = walk.nbytes)) {
430 		op->src = walk.src.virt.addr,
431 		op->dst = walk.dst.virt.addr;
432 		op->mode = AES_MODE_ECB;
433 		op->len = nbytes - (nbytes % AES_BLOCK_SIZE);
434 		op->dir = AES_DIR_DECRYPT;
435 
436 		ret = geode_aes_crypt(op);
437 		nbytes -= ret;
438 		err = blkcipher_walk_done(desc, &walk, nbytes);
439 	}
440 
441 	return err;
442 }
443 
444 static int
445 geode_ecb_encrypt(struct blkcipher_desc *desc,
446 		  struct scatterlist *dst, struct scatterlist *src,
447 		  unsigned int nbytes)
448 {
449 	struct geode_aes_op *op = crypto_blkcipher_ctx(desc->tfm);
450 	struct blkcipher_walk walk;
451 	int err, ret;
452 
453 	if (unlikely(op->keylen != AES_KEYSIZE_128))
454 		return fallback_blk_enc(desc, dst, src, nbytes);
455 
456 	blkcipher_walk_init(&walk, dst, src, nbytes);
457 	err = blkcipher_walk_virt(desc, &walk);
458 
459 	while ((nbytes = walk.nbytes)) {
460 		op->src = walk.src.virt.addr,
461 		op->dst = walk.dst.virt.addr;
462 		op->mode = AES_MODE_ECB;
463 		op->len = nbytes - (nbytes % AES_BLOCK_SIZE);
464 		op->dir = AES_DIR_ENCRYPT;
465 
466 		ret = geode_aes_crypt(op);
467 		nbytes -= ret;
468 		ret =  blkcipher_walk_done(desc, &walk, nbytes);
469 	}
470 
471 	return err;
472 }
473 
474 static struct crypto_alg geode_ecb_alg = {
475 	.cra_name			=	"ecb(aes)",
476 	.cra_driver_name	=	"ecb-aes-geode",
477 	.cra_priority		=	400,
478 	.cra_flags			=	CRYPTO_ALG_TYPE_BLKCIPHER |
479 						CRYPTO_ALG_KERN_DRIVER_ONLY |
480 						CRYPTO_ALG_NEED_FALLBACK,
481 	.cra_init			=	fallback_init_blk,
482 	.cra_exit			=	fallback_exit_blk,
483 	.cra_blocksize		=	AES_BLOCK_SIZE,
484 	.cra_ctxsize		=	sizeof(struct geode_aes_op),
485 	.cra_alignmask		=	15,
486 	.cra_type			=	&crypto_blkcipher_type,
487 	.cra_module			=	THIS_MODULE,
488 	.cra_u				=	{
489 		.blkcipher	=	{
490 			.min_keysize	=	AES_MIN_KEY_SIZE,
491 			.max_keysize	=	AES_MAX_KEY_SIZE,
492 			.setkey			=	geode_setkey_blk,
493 			.encrypt		=	geode_ecb_encrypt,
494 			.decrypt		=	geode_ecb_decrypt,
495 		}
496 	}
497 };
498 
499 static void geode_aes_remove(struct pci_dev *dev)
500 {
501 	crypto_unregister_alg(&geode_alg);
502 	crypto_unregister_alg(&geode_ecb_alg);
503 	crypto_unregister_alg(&geode_cbc_alg);
504 
505 	pci_iounmap(dev, _iobase);
506 	_iobase = NULL;
507 
508 	pci_release_regions(dev);
509 	pci_disable_device(dev);
510 }
511 
512 
513 static int geode_aes_probe(struct pci_dev *dev, const struct pci_device_id *id)
514 {
515 	int ret;
516 
517 	ret = pci_enable_device(dev);
518 	if (ret)
519 		return ret;
520 
521 	ret = pci_request_regions(dev, "geode-aes");
522 	if (ret)
523 		goto eenable;
524 
525 	_iobase = pci_iomap(dev, 0, 0);
526 
527 	if (_iobase == NULL) {
528 		ret = -ENOMEM;
529 		goto erequest;
530 	}
531 
532 	spin_lock_init(&lock);
533 
534 	/* Clear any pending activity */
535 	iowrite32(AES_INTR_PENDING | AES_INTR_MASK, _iobase + AES_INTR_REG);
536 
537 	ret = crypto_register_alg(&geode_alg);
538 	if (ret)
539 		goto eiomap;
540 
541 	ret = crypto_register_alg(&geode_ecb_alg);
542 	if (ret)
543 		goto ealg;
544 
545 	ret = crypto_register_alg(&geode_cbc_alg);
546 	if (ret)
547 		goto eecb;
548 
549 	dev_notice(&dev->dev, "GEODE AES engine enabled.\n");
550 	return 0;
551 
552  eecb:
553 	crypto_unregister_alg(&geode_ecb_alg);
554 
555  ealg:
556 	crypto_unregister_alg(&geode_alg);
557 
558  eiomap:
559 	pci_iounmap(dev, _iobase);
560 
561  erequest:
562 	pci_release_regions(dev);
563 
564  eenable:
565 	pci_disable_device(dev);
566 
567 	dev_err(&dev->dev, "GEODE AES initialization failed.\n");
568 	return ret;
569 }
570 
571 static struct pci_device_id geode_aes_tbl[] = {
572 	{ PCI_VDEVICE(AMD, PCI_DEVICE_ID_AMD_LX_AES), },
573 	{ 0, }
574 };
575 
576 MODULE_DEVICE_TABLE(pci, geode_aes_tbl);
577 
578 static struct pci_driver geode_aes_driver = {
579 	.name = "Geode LX AES",
580 	.id_table = geode_aes_tbl,
581 	.probe = geode_aes_probe,
582 	.remove = geode_aes_remove,
583 };
584 
585 module_pci_driver(geode_aes_driver);
586 
587 MODULE_AUTHOR("Advanced Micro Devices, Inc.");
588 MODULE_DESCRIPTION("Geode LX Hardware AES driver");
589 MODULE_LICENSE("GPL");
590