1 // SPDX-License-Identifier: GPL-2.0-only
2 /*
3  * Intel IXP4xx NPE-C crypto driver
4  *
5  * Copyright (C) 2008 Christian Hohnstaedt <chohnstaedt@innominate.com>
6  */
7 
8 #include <linux/platform_device.h>
9 #include <linux/dma-mapping.h>
10 #include <linux/dmapool.h>
11 #include <linux/crypto.h>
12 #include <linux/kernel.h>
13 #include <linux/rtnetlink.h>
14 #include <linux/interrupt.h>
15 #include <linux/spinlock.h>
16 #include <linux/gfp.h>
17 #include <linux/module.h>
18 #include <linux/of.h>
19 
20 #include <crypto/ctr.h>
21 #include <crypto/internal/des.h>
22 #include <crypto/aes.h>
23 #include <crypto/hmac.h>
24 #include <crypto/sha1.h>
25 #include <crypto/algapi.h>
26 #include <crypto/internal/aead.h>
27 #include <crypto/internal/skcipher.h>
28 #include <crypto/authenc.h>
29 #include <crypto/scatterwalk.h>
30 
31 #include <linux/soc/ixp4xx/npe.h>
32 #include <linux/soc/ixp4xx/qmgr.h>
33 
34 /* Intermittent includes, delete this after v5.14-rc1 */
35 #include <linux/soc/ixp4xx/cpu.h>
36 
37 #define MAX_KEYLEN 32
38 
39 /* hash: cfgword + 2 * digestlen; crypt: keylen + cfgword */
40 #define NPE_CTX_LEN 80
41 #define AES_BLOCK128 16
42 
43 #define NPE_OP_HASH_VERIFY   0x01
44 #define NPE_OP_CCM_ENABLE    0x04
45 #define NPE_OP_CRYPT_ENABLE  0x08
46 #define NPE_OP_HASH_ENABLE   0x10
47 #define NPE_OP_NOT_IN_PLACE  0x20
48 #define NPE_OP_HMAC_DISABLE  0x40
49 #define NPE_OP_CRYPT_ENCRYPT 0x80
50 
51 #define NPE_OP_CCM_GEN_MIC   0xcc
52 #define NPE_OP_HASH_GEN_ICV  0x50
53 #define NPE_OP_ENC_GEN_KEY   0xc9
54 
55 #define MOD_ECB     0x0000
56 #define MOD_CTR     0x1000
57 #define MOD_CBC_ENC 0x2000
58 #define MOD_CBC_DEC 0x3000
59 #define MOD_CCM_ENC 0x4000
60 #define MOD_CCM_DEC 0x5000
61 
62 #define KEYLEN_128  4
63 #define KEYLEN_192  6
64 #define KEYLEN_256  8
65 
66 #define CIPH_DECR   0x0000
67 #define CIPH_ENCR   0x0400
68 
69 #define MOD_DES     0x0000
70 #define MOD_TDEA2   0x0100
71 #define MOD_3DES   0x0200
72 #define MOD_AES     0x0800
73 #define MOD_AES128  (0x0800 | KEYLEN_128)
74 #define MOD_AES192  (0x0900 | KEYLEN_192)
75 #define MOD_AES256  (0x0a00 | KEYLEN_256)
76 
77 #define MAX_IVLEN   16
78 #define NPE_QLEN    16
79 /* Space for registering when the first
80  * NPE_QLEN crypt_ctl are busy */
81 #define NPE_QLEN_TOTAL 64
82 
83 #define CTL_FLAG_UNUSED		0x0000
84 #define CTL_FLAG_USED		0x1000
85 #define CTL_FLAG_PERFORM_ABLK	0x0001
86 #define CTL_FLAG_GEN_ICV	0x0002
87 #define CTL_FLAG_GEN_REVAES	0x0004
88 #define CTL_FLAG_PERFORM_AEAD	0x0008
89 #define CTL_FLAG_MASK		0x000f
90 
91 #define HMAC_PAD_BLOCKLEN SHA1_BLOCK_SIZE
92 
93 #define MD5_DIGEST_SIZE   16
94 
95 struct buffer_desc {
96 	u32 phys_next;
97 #ifdef __ARMEB__
98 	u16 buf_len;
99 	u16 pkt_len;
100 #else
101 	u16 pkt_len;
102 	u16 buf_len;
103 #endif
104 	dma_addr_t phys_addr;
105 	u32 __reserved[4];
106 	struct buffer_desc *next;
107 	enum dma_data_direction dir;
108 };
109 
110 struct crypt_ctl {
111 #ifdef __ARMEB__
112 	u8 mode;		/* NPE_OP_*  operation mode */
113 	u8 init_len;
114 	u16 reserved;
115 #else
116 	u16 reserved;
117 	u8 init_len;
118 	u8 mode;		/* NPE_OP_*  operation mode */
119 #endif
120 	u8 iv[MAX_IVLEN];	/* IV for CBC mode or CTR IV for CTR mode */
121 	u32 icv_rev_aes;	/* icv or rev aes */
122 	u32 src_buf;
123 	u32 dst_buf;
124 #ifdef __ARMEB__
125 	u16 auth_offs;		/* Authentication start offset */
126 	u16 auth_len;		/* Authentication data length */
127 	u16 crypt_offs;		/* Cryption start offset */
128 	u16 crypt_len;		/* Cryption data length */
129 #else
130 	u16 auth_len;		/* Authentication data length */
131 	u16 auth_offs;		/* Authentication start offset */
132 	u16 crypt_len;		/* Cryption data length */
133 	u16 crypt_offs;		/* Cryption start offset */
134 #endif
135 	u32 aadAddr;		/* Additional Auth Data Addr for CCM mode */
136 	u32 crypto_ctx;		/* NPE Crypto Param structure address */
137 
138 	/* Used by Host: 4*4 bytes*/
139 	unsigned int ctl_flags;
140 	union {
141 		struct skcipher_request *ablk_req;
142 		struct aead_request *aead_req;
143 		struct crypto_tfm *tfm;
144 	} data;
145 	struct buffer_desc *regist_buf;
146 	u8 *regist_ptr;
147 };
148 
149 struct ablk_ctx {
150 	struct buffer_desc *src;
151 	struct buffer_desc *dst;
152 	u8 iv[MAX_IVLEN];
153 	bool encrypt;
154 	struct skcipher_request fallback_req;   // keep at the end
155 };
156 
157 struct aead_ctx {
158 	struct buffer_desc *src;
159 	struct buffer_desc *dst;
160 	struct scatterlist ivlist;
161 	/* used when the hmac is not on one sg entry */
162 	u8 *hmac_virt;
163 	int encrypt;
164 };
165 
166 struct ix_hash_algo {
167 	u32 cfgword;
168 	unsigned char *icv;
169 };
170 
171 struct ix_sa_dir {
172 	unsigned char *npe_ctx;
173 	dma_addr_t npe_ctx_phys;
174 	int npe_ctx_idx;
175 	u8 npe_mode;
176 };
177 
178 struct ixp_ctx {
179 	struct ix_sa_dir encrypt;
180 	struct ix_sa_dir decrypt;
181 	int authkey_len;
182 	u8 authkey[MAX_KEYLEN];
183 	int enckey_len;
184 	u8 enckey[MAX_KEYLEN];
185 	u8 salt[MAX_IVLEN];
186 	u8 nonce[CTR_RFC3686_NONCE_SIZE];
187 	unsigned int salted;
188 	atomic_t configuring;
189 	struct completion completion;
190 	struct crypto_skcipher *fallback_tfm;
191 };
192 
193 struct ixp_alg {
194 	struct skcipher_alg crypto;
195 	const struct ix_hash_algo *hash;
196 	u32 cfg_enc;
197 	u32 cfg_dec;
198 
199 	int registered;
200 };
201 
202 struct ixp_aead_alg {
203 	struct aead_alg crypto;
204 	const struct ix_hash_algo *hash;
205 	u32 cfg_enc;
206 	u32 cfg_dec;
207 
208 	int registered;
209 };
210 
211 static const struct ix_hash_algo hash_alg_md5 = {
212 	.cfgword	= 0xAA010004,
213 	.icv		= "\x01\x23\x45\x67\x89\xAB\xCD\xEF"
214 			  "\xFE\xDC\xBA\x98\x76\x54\x32\x10",
215 };
216 
217 static const struct ix_hash_algo hash_alg_sha1 = {
218 	.cfgword	= 0x00000005,
219 	.icv		= "\x67\x45\x23\x01\xEF\xCD\xAB\x89\x98\xBA"
220 			  "\xDC\xFE\x10\x32\x54\x76\xC3\xD2\xE1\xF0",
221 };
222 
223 static struct npe *npe_c;
224 
225 static unsigned int send_qid;
226 static unsigned int recv_qid;
227 static struct dma_pool *buffer_pool;
228 static struct dma_pool *ctx_pool;
229 
230 static struct crypt_ctl *crypt_virt;
231 static dma_addr_t crypt_phys;
232 
233 static int support_aes = 1;
234 
235 static struct platform_device *pdev;
236 
237 static inline dma_addr_t crypt_virt2phys(struct crypt_ctl *virt)
238 {
239 	return crypt_phys + (virt - crypt_virt) * sizeof(struct crypt_ctl);
240 }
241 
242 static inline struct crypt_ctl *crypt_phys2virt(dma_addr_t phys)
243 {
244 	return crypt_virt + (phys - crypt_phys) / sizeof(struct crypt_ctl);
245 }
246 
247 static inline u32 cipher_cfg_enc(struct crypto_tfm *tfm)
248 {
249 	return container_of(tfm->__crt_alg, struct ixp_alg, crypto.base)->cfg_enc;
250 }
251 
252 static inline u32 cipher_cfg_dec(struct crypto_tfm *tfm)
253 {
254 	return container_of(tfm->__crt_alg, struct ixp_alg, crypto.base)->cfg_dec;
255 }
256 
257 static inline const struct ix_hash_algo *ix_hash(struct crypto_tfm *tfm)
258 {
259 	return container_of(tfm->__crt_alg, struct ixp_alg, crypto.base)->hash;
260 }
261 
262 static int setup_crypt_desc(void)
263 {
264 	struct device *dev = &pdev->dev;
265 
266 	BUILD_BUG_ON(!(IS_ENABLED(CONFIG_COMPILE_TEST) &&
267 		       IS_ENABLED(CONFIG_64BIT)) &&
268 		     sizeof(struct crypt_ctl) != 64);
269 	crypt_virt = dma_alloc_coherent(dev,
270 					NPE_QLEN * sizeof(struct crypt_ctl),
271 					&crypt_phys, GFP_ATOMIC);
272 	if (!crypt_virt)
273 		return -ENOMEM;
274 	return 0;
275 }
276 
277 static DEFINE_SPINLOCK(desc_lock);
278 static struct crypt_ctl *get_crypt_desc(void)
279 {
280 	int i;
281 	static int idx;
282 	unsigned long flags;
283 
284 	spin_lock_irqsave(&desc_lock, flags);
285 
286 	if (unlikely(!crypt_virt))
287 		setup_crypt_desc();
288 	if (unlikely(!crypt_virt)) {
289 		spin_unlock_irqrestore(&desc_lock, flags);
290 		return NULL;
291 	}
292 	i = idx;
293 	if (crypt_virt[i].ctl_flags == CTL_FLAG_UNUSED) {
294 		if (++idx >= NPE_QLEN)
295 			idx = 0;
296 		crypt_virt[i].ctl_flags = CTL_FLAG_USED;
297 		spin_unlock_irqrestore(&desc_lock, flags);
298 		return crypt_virt + i;
299 	} else {
300 		spin_unlock_irqrestore(&desc_lock, flags);
301 		return NULL;
302 	}
303 }
304 
305 static DEFINE_SPINLOCK(emerg_lock);
306 static struct crypt_ctl *get_crypt_desc_emerg(void)
307 {
308 	int i;
309 	static int idx = NPE_QLEN;
310 	struct crypt_ctl *desc;
311 	unsigned long flags;
312 
313 	desc = get_crypt_desc();
314 	if (desc)
315 		return desc;
316 	if (unlikely(!crypt_virt))
317 		return NULL;
318 
319 	spin_lock_irqsave(&emerg_lock, flags);
320 	i = idx;
321 	if (crypt_virt[i].ctl_flags == CTL_FLAG_UNUSED) {
322 		if (++idx >= NPE_QLEN_TOTAL)
323 			idx = NPE_QLEN;
324 		crypt_virt[i].ctl_flags = CTL_FLAG_USED;
325 		spin_unlock_irqrestore(&emerg_lock, flags);
326 		return crypt_virt + i;
327 	} else {
328 		spin_unlock_irqrestore(&emerg_lock, flags);
329 		return NULL;
330 	}
331 }
332 
333 static void free_buf_chain(struct device *dev, struct buffer_desc *buf,
334 			   dma_addr_t phys)
335 {
336 	while (buf) {
337 		struct buffer_desc *buf1;
338 		u32 phys1;
339 
340 		buf1 = buf->next;
341 		phys1 = buf->phys_next;
342 		dma_unmap_single(dev, buf->phys_addr, buf->buf_len, buf->dir);
343 		dma_pool_free(buffer_pool, buf, phys);
344 		buf = buf1;
345 		phys = phys1;
346 	}
347 }
348 
349 static struct tasklet_struct crypto_done_tasklet;
350 
351 static void finish_scattered_hmac(struct crypt_ctl *crypt)
352 {
353 	struct aead_request *req = crypt->data.aead_req;
354 	struct aead_ctx *req_ctx = aead_request_ctx(req);
355 	struct crypto_aead *tfm = crypto_aead_reqtfm(req);
356 	int authsize = crypto_aead_authsize(tfm);
357 	int decryptlen = req->assoclen + req->cryptlen - authsize;
358 
359 	if (req_ctx->encrypt) {
360 		scatterwalk_map_and_copy(req_ctx->hmac_virt, req->dst,
361 					 decryptlen, authsize, 1);
362 	}
363 	dma_pool_free(buffer_pool, req_ctx->hmac_virt, crypt->icv_rev_aes);
364 }
365 
366 static void one_packet(dma_addr_t phys)
367 {
368 	struct device *dev = &pdev->dev;
369 	struct crypt_ctl *crypt;
370 	struct ixp_ctx *ctx;
371 	int failed;
372 
373 	failed = phys & 0x1 ? -EBADMSG : 0;
374 	phys &= ~0x3;
375 	crypt = crypt_phys2virt(phys);
376 
377 	switch (crypt->ctl_flags & CTL_FLAG_MASK) {
378 	case CTL_FLAG_PERFORM_AEAD: {
379 		struct aead_request *req = crypt->data.aead_req;
380 		struct aead_ctx *req_ctx = aead_request_ctx(req);
381 
382 		free_buf_chain(dev, req_ctx->src, crypt->src_buf);
383 		free_buf_chain(dev, req_ctx->dst, crypt->dst_buf);
384 		if (req_ctx->hmac_virt)
385 			finish_scattered_hmac(crypt);
386 
387 		aead_request_complete(req, failed);
388 		break;
389 	}
390 	case CTL_FLAG_PERFORM_ABLK: {
391 		struct skcipher_request *req = crypt->data.ablk_req;
392 		struct ablk_ctx *req_ctx = skcipher_request_ctx(req);
393 		struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req);
394 		unsigned int ivsize = crypto_skcipher_ivsize(tfm);
395 		unsigned int offset;
396 
397 		if (ivsize > 0) {
398 			offset = req->cryptlen - ivsize;
399 			if (req_ctx->encrypt) {
400 				scatterwalk_map_and_copy(req->iv, req->dst,
401 							 offset, ivsize, 0);
402 			} else {
403 				memcpy(req->iv, req_ctx->iv, ivsize);
404 				memzero_explicit(req_ctx->iv, ivsize);
405 			}
406 		}
407 
408 		if (req_ctx->dst)
409 			free_buf_chain(dev, req_ctx->dst, crypt->dst_buf);
410 
411 		free_buf_chain(dev, req_ctx->src, crypt->src_buf);
412 		skcipher_request_complete(req, failed);
413 		break;
414 	}
415 	case CTL_FLAG_GEN_ICV:
416 		ctx = crypto_tfm_ctx(crypt->data.tfm);
417 		dma_pool_free(ctx_pool, crypt->regist_ptr,
418 			      crypt->regist_buf->phys_addr);
419 		dma_pool_free(buffer_pool, crypt->regist_buf, crypt->src_buf);
420 		if (atomic_dec_and_test(&ctx->configuring))
421 			complete(&ctx->completion);
422 		break;
423 	case CTL_FLAG_GEN_REVAES:
424 		ctx = crypto_tfm_ctx(crypt->data.tfm);
425 		*(__be32 *)ctx->decrypt.npe_ctx &= cpu_to_be32(~CIPH_ENCR);
426 		if (atomic_dec_and_test(&ctx->configuring))
427 			complete(&ctx->completion);
428 		break;
429 	default:
430 		BUG();
431 	}
432 	crypt->ctl_flags = CTL_FLAG_UNUSED;
433 }
434 
435 static void irqhandler(void *_unused)
436 {
437 	tasklet_schedule(&crypto_done_tasklet);
438 }
439 
440 static void crypto_done_action(unsigned long arg)
441 {
442 	int i;
443 
444 	for (i = 0; i < 4; i++) {
445 		dma_addr_t phys = qmgr_get_entry(recv_qid);
446 		if (!phys)
447 			return;
448 		one_packet(phys);
449 	}
450 	tasklet_schedule(&crypto_done_tasklet);
451 }
452 
453 static int init_ixp_crypto(struct device *dev)
454 {
455 	struct device_node *np = dev->of_node;
456 	u32 msg[2] = { 0, 0 };
457 	int ret = -ENODEV;
458 	u32 npe_id;
459 
460 	dev_info(dev, "probing...\n");
461 
462 	/* Locate the NPE and queue manager to use from device tree */
463 	if (IS_ENABLED(CONFIG_OF) && np) {
464 		struct of_phandle_args queue_spec;
465 		struct of_phandle_args npe_spec;
466 
467 		ret = of_parse_phandle_with_fixed_args(np, "intel,npe-handle",
468 						       1, 0, &npe_spec);
469 		if (ret) {
470 			dev_err(dev, "no NPE engine specified\n");
471 			return -ENODEV;
472 		}
473 		npe_id = npe_spec.args[0];
474 
475 		ret = of_parse_phandle_with_fixed_args(np, "queue-rx", 1, 0,
476 						       &queue_spec);
477 		if (ret) {
478 			dev_err(dev, "no rx queue phandle\n");
479 			return -ENODEV;
480 		}
481 		recv_qid = queue_spec.args[0];
482 
483 		ret = of_parse_phandle_with_fixed_args(np, "queue-txready", 1, 0,
484 						       &queue_spec);
485 		if (ret) {
486 			dev_err(dev, "no txready queue phandle\n");
487 			return -ENODEV;
488 		}
489 		send_qid = queue_spec.args[0];
490 	} else {
491 		/*
492 		 * Hardcoded engine when using platform data, this goes away
493 		 * when we switch to using DT only.
494 		 */
495 		npe_id = 2;
496 		send_qid = 29;
497 		recv_qid = 30;
498 	}
499 
500 	npe_c = npe_request(npe_id);
501 	if (!npe_c)
502 		return ret;
503 
504 	if (!npe_running(npe_c)) {
505 		ret = npe_load_firmware(npe_c, npe_name(npe_c), dev);
506 		if (ret)
507 			goto npe_release;
508 		if (npe_recv_message(npe_c, msg, "STATUS_MSG"))
509 			goto npe_error;
510 	} else {
511 		if (npe_send_message(npe_c, msg, "STATUS_MSG"))
512 			goto npe_error;
513 
514 		if (npe_recv_message(npe_c, msg, "STATUS_MSG"))
515 			goto npe_error;
516 	}
517 
518 	switch ((msg[1] >> 16) & 0xff) {
519 	case 3:
520 		dev_warn(dev, "Firmware of %s lacks AES support\n", npe_name(npe_c));
521 		support_aes = 0;
522 		break;
523 	case 4:
524 	case 5:
525 		support_aes = 1;
526 		break;
527 	default:
528 		dev_err(dev, "Firmware of %s lacks crypto support\n", npe_name(npe_c));
529 		ret = -ENODEV;
530 		goto npe_release;
531 	}
532 	/* buffer_pool will also be used to sometimes store the hmac,
533 	 * so assure it is large enough
534 	 */
535 	BUILD_BUG_ON(SHA1_DIGEST_SIZE > sizeof(struct buffer_desc));
536 	buffer_pool = dma_pool_create("buffer", dev, sizeof(struct buffer_desc),
537 				      32, 0);
538 	ret = -ENOMEM;
539 	if (!buffer_pool)
540 		goto err;
541 
542 	ctx_pool = dma_pool_create("context", dev, NPE_CTX_LEN, 16, 0);
543 	if (!ctx_pool)
544 		goto err;
545 
546 	ret = qmgr_request_queue(send_qid, NPE_QLEN_TOTAL, 0, 0,
547 				 "ixp_crypto:out", NULL);
548 	if (ret)
549 		goto err;
550 	ret = qmgr_request_queue(recv_qid, NPE_QLEN, 0, 0,
551 				 "ixp_crypto:in", NULL);
552 	if (ret) {
553 		qmgr_release_queue(send_qid);
554 		goto err;
555 	}
556 	qmgr_set_irq(recv_qid, QUEUE_IRQ_SRC_NOT_EMPTY, irqhandler, NULL);
557 	tasklet_init(&crypto_done_tasklet, crypto_done_action, 0);
558 
559 	qmgr_enable_irq(recv_qid);
560 	return 0;
561 
562 npe_error:
563 	dev_err(dev, "%s not responding\n", npe_name(npe_c));
564 	ret = -EIO;
565 err:
566 	dma_pool_destroy(ctx_pool);
567 	dma_pool_destroy(buffer_pool);
568 npe_release:
569 	npe_release(npe_c);
570 	return ret;
571 }
572 
573 static void release_ixp_crypto(struct device *dev)
574 {
575 	qmgr_disable_irq(recv_qid);
576 	tasklet_kill(&crypto_done_tasklet);
577 
578 	qmgr_release_queue(send_qid);
579 	qmgr_release_queue(recv_qid);
580 
581 	dma_pool_destroy(ctx_pool);
582 	dma_pool_destroy(buffer_pool);
583 
584 	npe_release(npe_c);
585 
586 	if (crypt_virt)
587 		dma_free_coherent(dev, NPE_QLEN * sizeof(struct crypt_ctl),
588 				  crypt_virt, crypt_phys);
589 }
590 
591 static void reset_sa_dir(struct ix_sa_dir *dir)
592 {
593 	memset(dir->npe_ctx, 0, NPE_CTX_LEN);
594 	dir->npe_ctx_idx = 0;
595 	dir->npe_mode = 0;
596 }
597 
598 static int init_sa_dir(struct ix_sa_dir *dir)
599 {
600 	dir->npe_ctx = dma_pool_alloc(ctx_pool, GFP_KERNEL, &dir->npe_ctx_phys);
601 	if (!dir->npe_ctx)
602 		return -ENOMEM;
603 
604 	reset_sa_dir(dir);
605 	return 0;
606 }
607 
608 static void free_sa_dir(struct ix_sa_dir *dir)
609 {
610 	memset(dir->npe_ctx, 0, NPE_CTX_LEN);
611 	dma_pool_free(ctx_pool, dir->npe_ctx, dir->npe_ctx_phys);
612 }
613 
614 static int init_tfm(struct crypto_tfm *tfm)
615 {
616 	struct ixp_ctx *ctx = crypto_tfm_ctx(tfm);
617 	int ret;
618 
619 	atomic_set(&ctx->configuring, 0);
620 	ret = init_sa_dir(&ctx->encrypt);
621 	if (ret)
622 		return ret;
623 	ret = init_sa_dir(&ctx->decrypt);
624 	if (ret)
625 		free_sa_dir(&ctx->encrypt);
626 
627 	return ret;
628 }
629 
630 static int init_tfm_ablk(struct crypto_skcipher *tfm)
631 {
632 	struct crypto_tfm *ctfm = crypto_skcipher_tfm(tfm);
633 	struct ixp_ctx *ctx = crypto_tfm_ctx(ctfm);
634 	const char *name = crypto_tfm_alg_name(ctfm);
635 
636 	ctx->fallback_tfm = crypto_alloc_skcipher(name, 0, CRYPTO_ALG_NEED_FALLBACK);
637 	if (IS_ERR(ctx->fallback_tfm)) {
638 		pr_err("ERROR: Cannot allocate fallback for %s %ld\n",
639 			name, PTR_ERR(ctx->fallback_tfm));
640 		return PTR_ERR(ctx->fallback_tfm);
641 	}
642 
643 	pr_info("Fallback for %s is %s\n",
644 		 crypto_tfm_alg_driver_name(&tfm->base),
645 		 crypto_tfm_alg_driver_name(crypto_skcipher_tfm(ctx->fallback_tfm))
646 		 );
647 
648 	crypto_skcipher_set_reqsize(tfm, sizeof(struct ablk_ctx) + crypto_skcipher_reqsize(ctx->fallback_tfm));
649 	return init_tfm(crypto_skcipher_tfm(tfm));
650 }
651 
652 static int init_tfm_aead(struct crypto_aead *tfm)
653 {
654 	crypto_aead_set_reqsize(tfm, sizeof(struct aead_ctx));
655 	return init_tfm(crypto_aead_tfm(tfm));
656 }
657 
658 static void exit_tfm(struct crypto_tfm *tfm)
659 {
660 	struct ixp_ctx *ctx = crypto_tfm_ctx(tfm);
661 
662 	free_sa_dir(&ctx->encrypt);
663 	free_sa_dir(&ctx->decrypt);
664 }
665 
666 static void exit_tfm_ablk(struct crypto_skcipher *tfm)
667 {
668 	struct crypto_tfm *ctfm = crypto_skcipher_tfm(tfm);
669 	struct ixp_ctx *ctx = crypto_tfm_ctx(ctfm);
670 
671 	crypto_free_skcipher(ctx->fallback_tfm);
672 	exit_tfm(crypto_skcipher_tfm(tfm));
673 }
674 
675 static void exit_tfm_aead(struct crypto_aead *tfm)
676 {
677 	exit_tfm(crypto_aead_tfm(tfm));
678 }
679 
680 static int register_chain_var(struct crypto_tfm *tfm, u8 xpad, u32 target,
681 			      int init_len, u32 ctx_addr, const u8 *key,
682 			      int key_len)
683 {
684 	struct ixp_ctx *ctx = crypto_tfm_ctx(tfm);
685 	struct crypt_ctl *crypt;
686 	struct buffer_desc *buf;
687 	int i;
688 	u8 *pad;
689 	dma_addr_t pad_phys, buf_phys;
690 
691 	BUILD_BUG_ON(NPE_CTX_LEN < HMAC_PAD_BLOCKLEN);
692 	pad = dma_pool_alloc(ctx_pool, GFP_KERNEL, &pad_phys);
693 	if (!pad)
694 		return -ENOMEM;
695 	buf = dma_pool_alloc(buffer_pool, GFP_KERNEL, &buf_phys);
696 	if (!buf) {
697 		dma_pool_free(ctx_pool, pad, pad_phys);
698 		return -ENOMEM;
699 	}
700 	crypt = get_crypt_desc_emerg();
701 	if (!crypt) {
702 		dma_pool_free(ctx_pool, pad, pad_phys);
703 		dma_pool_free(buffer_pool, buf, buf_phys);
704 		return -EAGAIN;
705 	}
706 
707 	memcpy(pad, key, key_len);
708 	memset(pad + key_len, 0, HMAC_PAD_BLOCKLEN - key_len);
709 	for (i = 0; i < HMAC_PAD_BLOCKLEN; i++)
710 		pad[i] ^= xpad;
711 
712 	crypt->data.tfm = tfm;
713 	crypt->regist_ptr = pad;
714 	crypt->regist_buf = buf;
715 
716 	crypt->auth_offs = 0;
717 	crypt->auth_len = HMAC_PAD_BLOCKLEN;
718 	crypt->crypto_ctx = ctx_addr;
719 	crypt->src_buf = buf_phys;
720 	crypt->icv_rev_aes = target;
721 	crypt->mode = NPE_OP_HASH_GEN_ICV;
722 	crypt->init_len = init_len;
723 	crypt->ctl_flags |= CTL_FLAG_GEN_ICV;
724 
725 	buf->next = NULL;
726 	buf->buf_len = HMAC_PAD_BLOCKLEN;
727 	buf->pkt_len = 0;
728 	buf->phys_addr = pad_phys;
729 
730 	atomic_inc(&ctx->configuring);
731 	qmgr_put_entry(send_qid, crypt_virt2phys(crypt));
732 	BUG_ON(qmgr_stat_overflow(send_qid));
733 	return 0;
734 }
735 
736 static int setup_auth(struct crypto_tfm *tfm, int encrypt, unsigned int authsize,
737 		      const u8 *key, int key_len, unsigned int digest_len)
738 {
739 	u32 itarget, otarget, npe_ctx_addr;
740 	unsigned char *cinfo;
741 	int init_len, ret = 0;
742 	u32 cfgword;
743 	struct ix_sa_dir *dir;
744 	struct ixp_ctx *ctx = crypto_tfm_ctx(tfm);
745 	const struct ix_hash_algo *algo;
746 
747 	dir = encrypt ? &ctx->encrypt : &ctx->decrypt;
748 	cinfo = dir->npe_ctx + dir->npe_ctx_idx;
749 	algo = ix_hash(tfm);
750 
751 	/* write cfg word to cryptinfo */
752 	cfgword = algo->cfgword | (authsize << 6); /* (authsize/4) << 8 */
753 #ifndef __ARMEB__
754 	cfgword ^= 0xAA000000; /* change the "byte swap" flags */
755 #endif
756 	*(__be32 *)cinfo = cpu_to_be32(cfgword);
757 	cinfo += sizeof(cfgword);
758 
759 	/* write ICV to cryptinfo */
760 	memcpy(cinfo, algo->icv, digest_len);
761 	cinfo += digest_len;
762 
763 	itarget = dir->npe_ctx_phys + dir->npe_ctx_idx
764 				+ sizeof(algo->cfgword);
765 	otarget = itarget + digest_len;
766 	init_len = cinfo - (dir->npe_ctx + dir->npe_ctx_idx);
767 	npe_ctx_addr = dir->npe_ctx_phys + dir->npe_ctx_idx;
768 
769 	dir->npe_ctx_idx += init_len;
770 	dir->npe_mode |= NPE_OP_HASH_ENABLE;
771 
772 	if (!encrypt)
773 		dir->npe_mode |= NPE_OP_HASH_VERIFY;
774 
775 	ret = register_chain_var(tfm, HMAC_OPAD_VALUE, otarget,
776 				 init_len, npe_ctx_addr, key, key_len);
777 	if (ret)
778 		return ret;
779 	return register_chain_var(tfm, HMAC_IPAD_VALUE, itarget,
780 				  init_len, npe_ctx_addr, key, key_len);
781 }
782 
783 static int gen_rev_aes_key(struct crypto_tfm *tfm)
784 {
785 	struct crypt_ctl *crypt;
786 	struct ixp_ctx *ctx = crypto_tfm_ctx(tfm);
787 	struct ix_sa_dir *dir = &ctx->decrypt;
788 
789 	crypt = get_crypt_desc_emerg();
790 	if (!crypt)
791 		return -EAGAIN;
792 
793 	*(__be32 *)dir->npe_ctx |= cpu_to_be32(CIPH_ENCR);
794 
795 	crypt->data.tfm = tfm;
796 	crypt->crypt_offs = 0;
797 	crypt->crypt_len = AES_BLOCK128;
798 	crypt->src_buf = 0;
799 	crypt->crypto_ctx = dir->npe_ctx_phys;
800 	crypt->icv_rev_aes = dir->npe_ctx_phys + sizeof(u32);
801 	crypt->mode = NPE_OP_ENC_GEN_KEY;
802 	crypt->init_len = dir->npe_ctx_idx;
803 	crypt->ctl_flags |= CTL_FLAG_GEN_REVAES;
804 
805 	atomic_inc(&ctx->configuring);
806 	qmgr_put_entry(send_qid, crypt_virt2phys(crypt));
807 	BUG_ON(qmgr_stat_overflow(send_qid));
808 	return 0;
809 }
810 
811 static int setup_cipher(struct crypto_tfm *tfm, int encrypt, const u8 *key,
812 			int key_len)
813 {
814 	u8 *cinfo;
815 	u32 cipher_cfg;
816 	u32 keylen_cfg = 0;
817 	struct ix_sa_dir *dir;
818 	struct ixp_ctx *ctx = crypto_tfm_ctx(tfm);
819 	int err;
820 
821 	dir = encrypt ? &ctx->encrypt : &ctx->decrypt;
822 	cinfo = dir->npe_ctx;
823 
824 	if (encrypt) {
825 		cipher_cfg = cipher_cfg_enc(tfm);
826 		dir->npe_mode |= NPE_OP_CRYPT_ENCRYPT;
827 	} else {
828 		cipher_cfg = cipher_cfg_dec(tfm);
829 	}
830 	if (cipher_cfg & MOD_AES) {
831 		switch (key_len) {
832 		case 16:
833 			keylen_cfg = MOD_AES128;
834 			break;
835 		case 24:
836 			keylen_cfg = MOD_AES192;
837 			break;
838 		case 32:
839 			keylen_cfg = MOD_AES256;
840 			break;
841 		default:
842 			return -EINVAL;
843 		}
844 		cipher_cfg |= keylen_cfg;
845 	} else {
846 		err = crypto_des_verify_key(tfm, key);
847 		if (err)
848 			return err;
849 	}
850 	/* write cfg word to cryptinfo */
851 	*(__be32 *)cinfo = cpu_to_be32(cipher_cfg);
852 	cinfo += sizeof(cipher_cfg);
853 
854 	/* write cipher key to cryptinfo */
855 	memcpy(cinfo, key, key_len);
856 	/* NPE wants keylen set to DES3_EDE_KEY_SIZE even for single DES */
857 	if (key_len < DES3_EDE_KEY_SIZE && !(cipher_cfg & MOD_AES)) {
858 		memset(cinfo + key_len, 0, DES3_EDE_KEY_SIZE - key_len);
859 		key_len = DES3_EDE_KEY_SIZE;
860 	}
861 	dir->npe_ctx_idx = sizeof(cipher_cfg) + key_len;
862 	dir->npe_mode |= NPE_OP_CRYPT_ENABLE;
863 	if ((cipher_cfg & MOD_AES) && !encrypt)
864 		return gen_rev_aes_key(tfm);
865 
866 	return 0;
867 }
868 
869 static struct buffer_desc *chainup_buffers(struct device *dev,
870 		struct scatterlist *sg,	unsigned int nbytes,
871 		struct buffer_desc *buf, gfp_t flags,
872 		enum dma_data_direction dir)
873 {
874 	for (; nbytes > 0; sg = sg_next(sg)) {
875 		unsigned int len = min(nbytes, sg->length);
876 		struct buffer_desc *next_buf;
877 		dma_addr_t next_buf_phys;
878 		void *ptr;
879 
880 		nbytes -= len;
881 		ptr = sg_virt(sg);
882 		next_buf = dma_pool_alloc(buffer_pool, flags, &next_buf_phys);
883 		if (!next_buf) {
884 			buf = NULL;
885 			break;
886 		}
887 		sg_dma_address(sg) = dma_map_single(dev, ptr, len, dir);
888 		buf->next = next_buf;
889 		buf->phys_next = next_buf_phys;
890 		buf = next_buf;
891 
892 		buf->phys_addr = sg_dma_address(sg);
893 		buf->buf_len = len;
894 		buf->dir = dir;
895 	}
896 	buf->next = NULL;
897 	buf->phys_next = 0;
898 	return buf;
899 }
900 
901 static int ablk_setkey(struct crypto_skcipher *tfm, const u8 *key,
902 		       unsigned int key_len)
903 {
904 	struct ixp_ctx *ctx = crypto_skcipher_ctx(tfm);
905 	int ret;
906 
907 	init_completion(&ctx->completion);
908 	atomic_inc(&ctx->configuring);
909 
910 	reset_sa_dir(&ctx->encrypt);
911 	reset_sa_dir(&ctx->decrypt);
912 
913 	ctx->encrypt.npe_mode = NPE_OP_HMAC_DISABLE;
914 	ctx->decrypt.npe_mode = NPE_OP_HMAC_DISABLE;
915 
916 	ret = setup_cipher(&tfm->base, 0, key, key_len);
917 	if (ret)
918 		goto out;
919 	ret = setup_cipher(&tfm->base, 1, key, key_len);
920 out:
921 	if (!atomic_dec_and_test(&ctx->configuring))
922 		wait_for_completion(&ctx->completion);
923 	if (ret)
924 		return ret;
925 	crypto_skcipher_clear_flags(ctx->fallback_tfm, CRYPTO_TFM_REQ_MASK);
926 	crypto_skcipher_set_flags(ctx->fallback_tfm, tfm->base.crt_flags & CRYPTO_TFM_REQ_MASK);
927 
928 	return crypto_skcipher_setkey(ctx->fallback_tfm, key, key_len);
929 }
930 
931 static int ablk_des3_setkey(struct crypto_skcipher *tfm, const u8 *key,
932 			    unsigned int key_len)
933 {
934 	return verify_skcipher_des3_key(tfm, key) ?:
935 	       ablk_setkey(tfm, key, key_len);
936 }
937 
938 static int ablk_rfc3686_setkey(struct crypto_skcipher *tfm, const u8 *key,
939 			       unsigned int key_len)
940 {
941 	struct ixp_ctx *ctx = crypto_skcipher_ctx(tfm);
942 
943 	/* the nonce is stored in bytes at end of key */
944 	if (key_len < CTR_RFC3686_NONCE_SIZE)
945 		return -EINVAL;
946 
947 	memcpy(ctx->nonce, key + (key_len - CTR_RFC3686_NONCE_SIZE),
948 	       CTR_RFC3686_NONCE_SIZE);
949 
950 	key_len -= CTR_RFC3686_NONCE_SIZE;
951 	return ablk_setkey(tfm, key, key_len);
952 }
953 
954 static int ixp4xx_cipher_fallback(struct skcipher_request *areq, int encrypt)
955 {
956 	struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(areq);
957 	struct ixp_ctx *op = crypto_skcipher_ctx(tfm);
958 	struct ablk_ctx *rctx = skcipher_request_ctx(areq);
959 	int err;
960 
961 	skcipher_request_set_tfm(&rctx->fallback_req, op->fallback_tfm);
962 	skcipher_request_set_callback(&rctx->fallback_req, areq->base.flags,
963 				      areq->base.complete, areq->base.data);
964 	skcipher_request_set_crypt(&rctx->fallback_req, areq->src, areq->dst,
965 				   areq->cryptlen, areq->iv);
966 	if (encrypt)
967 		err = crypto_skcipher_encrypt(&rctx->fallback_req);
968 	else
969 		err = crypto_skcipher_decrypt(&rctx->fallback_req);
970 	return err;
971 }
972 
973 static int ablk_perform(struct skcipher_request *req, int encrypt)
974 {
975 	struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req);
976 	struct ixp_ctx *ctx = crypto_skcipher_ctx(tfm);
977 	unsigned int ivsize = crypto_skcipher_ivsize(tfm);
978 	struct ix_sa_dir *dir;
979 	struct crypt_ctl *crypt;
980 	unsigned int nbytes = req->cryptlen;
981 	enum dma_data_direction src_direction = DMA_BIDIRECTIONAL;
982 	struct ablk_ctx *req_ctx = skcipher_request_ctx(req);
983 	struct buffer_desc src_hook;
984 	struct device *dev = &pdev->dev;
985 	unsigned int offset;
986 	gfp_t flags = req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP ?
987 				GFP_KERNEL : GFP_ATOMIC;
988 
989 	if (sg_nents(req->src) > 1 || sg_nents(req->dst) > 1)
990 		return ixp4xx_cipher_fallback(req, encrypt);
991 
992 	if (qmgr_stat_full(send_qid))
993 		return -EAGAIN;
994 	if (atomic_read(&ctx->configuring))
995 		return -EAGAIN;
996 
997 	dir = encrypt ? &ctx->encrypt : &ctx->decrypt;
998 	req_ctx->encrypt = encrypt;
999 
1000 	crypt = get_crypt_desc();
1001 	if (!crypt)
1002 		return -ENOMEM;
1003 
1004 	crypt->data.ablk_req = req;
1005 	crypt->crypto_ctx = dir->npe_ctx_phys;
1006 	crypt->mode = dir->npe_mode;
1007 	crypt->init_len = dir->npe_ctx_idx;
1008 
1009 	crypt->crypt_offs = 0;
1010 	crypt->crypt_len = nbytes;
1011 
1012 	BUG_ON(ivsize && !req->iv);
1013 	memcpy(crypt->iv, req->iv, ivsize);
1014 	if (ivsize > 0 && !encrypt) {
1015 		offset = req->cryptlen - ivsize;
1016 		scatterwalk_map_and_copy(req_ctx->iv, req->src, offset, ivsize, 0);
1017 	}
1018 	if (req->src != req->dst) {
1019 		struct buffer_desc dst_hook;
1020 
1021 		crypt->mode |= NPE_OP_NOT_IN_PLACE;
1022 		/* This was never tested by Intel
1023 		 * for more than one dst buffer, I think. */
1024 		req_ctx->dst = NULL;
1025 		if (!chainup_buffers(dev, req->dst, nbytes, &dst_hook,
1026 				     flags, DMA_FROM_DEVICE))
1027 			goto free_buf_dest;
1028 		src_direction = DMA_TO_DEVICE;
1029 		req_ctx->dst = dst_hook.next;
1030 		crypt->dst_buf = dst_hook.phys_next;
1031 	} else {
1032 		req_ctx->dst = NULL;
1033 	}
1034 	req_ctx->src = NULL;
1035 	if (!chainup_buffers(dev, req->src, nbytes, &src_hook, flags,
1036 			     src_direction))
1037 		goto free_buf_src;
1038 
1039 	req_ctx->src = src_hook.next;
1040 	crypt->src_buf = src_hook.phys_next;
1041 	crypt->ctl_flags |= CTL_FLAG_PERFORM_ABLK;
1042 	qmgr_put_entry(send_qid, crypt_virt2phys(crypt));
1043 	BUG_ON(qmgr_stat_overflow(send_qid));
1044 	return -EINPROGRESS;
1045 
1046 free_buf_src:
1047 	free_buf_chain(dev, req_ctx->src, crypt->src_buf);
1048 free_buf_dest:
1049 	if (req->src != req->dst)
1050 		free_buf_chain(dev, req_ctx->dst, crypt->dst_buf);
1051 
1052 	crypt->ctl_flags = CTL_FLAG_UNUSED;
1053 	return -ENOMEM;
1054 }
1055 
1056 static int ablk_encrypt(struct skcipher_request *req)
1057 {
1058 	return ablk_perform(req, 1);
1059 }
1060 
1061 static int ablk_decrypt(struct skcipher_request *req)
1062 {
1063 	return ablk_perform(req, 0);
1064 }
1065 
1066 static int ablk_rfc3686_crypt(struct skcipher_request *req)
1067 {
1068 	struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req);
1069 	struct ixp_ctx *ctx = crypto_skcipher_ctx(tfm);
1070 	u8 iv[CTR_RFC3686_BLOCK_SIZE];
1071 	u8 *info = req->iv;
1072 	int ret;
1073 
1074 	/* set up counter block */
1075 	memcpy(iv, ctx->nonce, CTR_RFC3686_NONCE_SIZE);
1076 	memcpy(iv + CTR_RFC3686_NONCE_SIZE, info, CTR_RFC3686_IV_SIZE);
1077 
1078 	/* initialize counter portion of counter block */
1079 	*(__be32 *)(iv + CTR_RFC3686_NONCE_SIZE + CTR_RFC3686_IV_SIZE) =
1080 		cpu_to_be32(1);
1081 
1082 	req->iv = iv;
1083 	ret = ablk_perform(req, 1);
1084 	req->iv = info;
1085 	return ret;
1086 }
1087 
1088 static int aead_perform(struct aead_request *req, int encrypt,
1089 			int cryptoffset, int eff_cryptlen, u8 *iv)
1090 {
1091 	struct crypto_aead *tfm = crypto_aead_reqtfm(req);
1092 	struct ixp_ctx *ctx = crypto_aead_ctx(tfm);
1093 	unsigned int ivsize = crypto_aead_ivsize(tfm);
1094 	unsigned int authsize = crypto_aead_authsize(tfm);
1095 	struct ix_sa_dir *dir;
1096 	struct crypt_ctl *crypt;
1097 	unsigned int cryptlen;
1098 	struct buffer_desc *buf, src_hook;
1099 	struct aead_ctx *req_ctx = aead_request_ctx(req);
1100 	struct device *dev = &pdev->dev;
1101 	gfp_t flags = req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP ?
1102 				GFP_KERNEL : GFP_ATOMIC;
1103 	enum dma_data_direction src_direction = DMA_BIDIRECTIONAL;
1104 	unsigned int lastlen;
1105 
1106 	if (qmgr_stat_full(send_qid))
1107 		return -EAGAIN;
1108 	if (atomic_read(&ctx->configuring))
1109 		return -EAGAIN;
1110 
1111 	if (encrypt) {
1112 		dir = &ctx->encrypt;
1113 		cryptlen = req->cryptlen;
1114 	} else {
1115 		dir = &ctx->decrypt;
1116 		/* req->cryptlen includes the authsize when decrypting */
1117 		cryptlen = req->cryptlen - authsize;
1118 		eff_cryptlen -= authsize;
1119 	}
1120 	crypt = get_crypt_desc();
1121 	if (!crypt)
1122 		return -ENOMEM;
1123 
1124 	crypt->data.aead_req = req;
1125 	crypt->crypto_ctx = dir->npe_ctx_phys;
1126 	crypt->mode = dir->npe_mode;
1127 	crypt->init_len = dir->npe_ctx_idx;
1128 
1129 	crypt->crypt_offs = cryptoffset;
1130 	crypt->crypt_len = eff_cryptlen;
1131 
1132 	crypt->auth_offs = 0;
1133 	crypt->auth_len = req->assoclen + cryptlen;
1134 	BUG_ON(ivsize && !req->iv);
1135 	memcpy(crypt->iv, req->iv, ivsize);
1136 
1137 	buf = chainup_buffers(dev, req->src, crypt->auth_len,
1138 			      &src_hook, flags, src_direction);
1139 	req_ctx->src = src_hook.next;
1140 	crypt->src_buf = src_hook.phys_next;
1141 	if (!buf)
1142 		goto free_buf_src;
1143 
1144 	lastlen = buf->buf_len;
1145 	if (lastlen >= authsize)
1146 		crypt->icv_rev_aes = buf->phys_addr +
1147 				     buf->buf_len - authsize;
1148 
1149 	req_ctx->dst = NULL;
1150 
1151 	if (req->src != req->dst) {
1152 		struct buffer_desc dst_hook;
1153 
1154 		crypt->mode |= NPE_OP_NOT_IN_PLACE;
1155 		src_direction = DMA_TO_DEVICE;
1156 
1157 		buf = chainup_buffers(dev, req->dst, crypt->auth_len,
1158 				      &dst_hook, flags, DMA_FROM_DEVICE);
1159 		req_ctx->dst = dst_hook.next;
1160 		crypt->dst_buf = dst_hook.phys_next;
1161 
1162 		if (!buf)
1163 			goto free_buf_dst;
1164 
1165 		if (encrypt) {
1166 			lastlen = buf->buf_len;
1167 			if (lastlen >= authsize)
1168 				crypt->icv_rev_aes = buf->phys_addr +
1169 						     buf->buf_len - authsize;
1170 		}
1171 	}
1172 
1173 	if (unlikely(lastlen < authsize)) {
1174 		dma_addr_t dma;
1175 		/* The 12 hmac bytes are scattered,
1176 		 * we need to copy them into a safe buffer */
1177 		req_ctx->hmac_virt = dma_pool_alloc(buffer_pool, flags, &dma);
1178 		if (unlikely(!req_ctx->hmac_virt))
1179 			goto free_buf_dst;
1180 		crypt->icv_rev_aes = dma;
1181 		if (!encrypt) {
1182 			scatterwalk_map_and_copy(req_ctx->hmac_virt,
1183 						 req->src, cryptlen, authsize, 0);
1184 		}
1185 		req_ctx->encrypt = encrypt;
1186 	} else {
1187 		req_ctx->hmac_virt = NULL;
1188 	}
1189 
1190 	crypt->ctl_flags |= CTL_FLAG_PERFORM_AEAD;
1191 	qmgr_put_entry(send_qid, crypt_virt2phys(crypt));
1192 	BUG_ON(qmgr_stat_overflow(send_qid));
1193 	return -EINPROGRESS;
1194 
1195 free_buf_dst:
1196 	free_buf_chain(dev, req_ctx->dst, crypt->dst_buf);
1197 free_buf_src:
1198 	free_buf_chain(dev, req_ctx->src, crypt->src_buf);
1199 	crypt->ctl_flags = CTL_FLAG_UNUSED;
1200 	return -ENOMEM;
1201 }
1202 
1203 static int aead_setup(struct crypto_aead *tfm, unsigned int authsize)
1204 {
1205 	struct ixp_ctx *ctx = crypto_aead_ctx(tfm);
1206 	unsigned int digest_len = crypto_aead_maxauthsize(tfm);
1207 	int ret;
1208 
1209 	if (!ctx->enckey_len && !ctx->authkey_len)
1210 		return 0;
1211 	init_completion(&ctx->completion);
1212 	atomic_inc(&ctx->configuring);
1213 
1214 	reset_sa_dir(&ctx->encrypt);
1215 	reset_sa_dir(&ctx->decrypt);
1216 
1217 	ret = setup_cipher(&tfm->base, 0, ctx->enckey, ctx->enckey_len);
1218 	if (ret)
1219 		goto out;
1220 	ret = setup_cipher(&tfm->base, 1, ctx->enckey, ctx->enckey_len);
1221 	if (ret)
1222 		goto out;
1223 	ret = setup_auth(&tfm->base, 0, authsize, ctx->authkey,
1224 			 ctx->authkey_len, digest_len);
1225 	if (ret)
1226 		goto out;
1227 	ret = setup_auth(&tfm->base, 1, authsize,  ctx->authkey,
1228 			 ctx->authkey_len, digest_len);
1229 out:
1230 	if (!atomic_dec_and_test(&ctx->configuring))
1231 		wait_for_completion(&ctx->completion);
1232 	return ret;
1233 }
1234 
1235 static int aead_setauthsize(struct crypto_aead *tfm, unsigned int authsize)
1236 {
1237 	int max = crypto_aead_maxauthsize(tfm) >> 2;
1238 
1239 	if ((authsize >> 2) < 1 || (authsize >> 2) > max || (authsize & 3))
1240 		return -EINVAL;
1241 	return aead_setup(tfm, authsize);
1242 }
1243 
1244 static int aead_setkey(struct crypto_aead *tfm, const u8 *key,
1245 		       unsigned int keylen)
1246 {
1247 	struct ixp_ctx *ctx = crypto_aead_ctx(tfm);
1248 	struct crypto_authenc_keys keys;
1249 
1250 	if (crypto_authenc_extractkeys(&keys, key, keylen) != 0)
1251 		goto badkey;
1252 
1253 	if (keys.authkeylen > sizeof(ctx->authkey))
1254 		goto badkey;
1255 
1256 	if (keys.enckeylen > sizeof(ctx->enckey))
1257 		goto badkey;
1258 
1259 	memcpy(ctx->authkey, keys.authkey, keys.authkeylen);
1260 	memcpy(ctx->enckey, keys.enckey, keys.enckeylen);
1261 	ctx->authkey_len = keys.authkeylen;
1262 	ctx->enckey_len = keys.enckeylen;
1263 
1264 	memzero_explicit(&keys, sizeof(keys));
1265 	return aead_setup(tfm, crypto_aead_authsize(tfm));
1266 badkey:
1267 	memzero_explicit(&keys, sizeof(keys));
1268 	return -EINVAL;
1269 }
1270 
1271 static int des3_aead_setkey(struct crypto_aead *tfm, const u8 *key,
1272 			    unsigned int keylen)
1273 {
1274 	struct ixp_ctx *ctx = crypto_aead_ctx(tfm);
1275 	struct crypto_authenc_keys keys;
1276 	int err;
1277 
1278 	err = crypto_authenc_extractkeys(&keys, key, keylen);
1279 	if (unlikely(err))
1280 		goto badkey;
1281 
1282 	err = -EINVAL;
1283 	if (keys.authkeylen > sizeof(ctx->authkey))
1284 		goto badkey;
1285 
1286 	err = verify_aead_des3_key(tfm, keys.enckey, keys.enckeylen);
1287 	if (err)
1288 		goto badkey;
1289 
1290 	memcpy(ctx->authkey, keys.authkey, keys.authkeylen);
1291 	memcpy(ctx->enckey, keys.enckey, keys.enckeylen);
1292 	ctx->authkey_len = keys.authkeylen;
1293 	ctx->enckey_len = keys.enckeylen;
1294 
1295 	memzero_explicit(&keys, sizeof(keys));
1296 	return aead_setup(tfm, crypto_aead_authsize(tfm));
1297 badkey:
1298 	memzero_explicit(&keys, sizeof(keys));
1299 	return err;
1300 }
1301 
1302 static int aead_encrypt(struct aead_request *req)
1303 {
1304 	return aead_perform(req, 1, req->assoclen, req->cryptlen, req->iv);
1305 }
1306 
1307 static int aead_decrypt(struct aead_request *req)
1308 {
1309 	return aead_perform(req, 0, req->assoclen, req->cryptlen, req->iv);
1310 }
1311 
1312 static struct ixp_alg ixp4xx_algos[] = {
1313 {
1314 	.crypto	= {
1315 		.base.cra_name		= "cbc(des)",
1316 		.base.cra_blocksize	= DES_BLOCK_SIZE,
1317 
1318 		.min_keysize		= DES_KEY_SIZE,
1319 		.max_keysize		= DES_KEY_SIZE,
1320 		.ivsize			= DES_BLOCK_SIZE,
1321 	},
1322 	.cfg_enc = CIPH_ENCR | MOD_DES | MOD_CBC_ENC | KEYLEN_192,
1323 	.cfg_dec = CIPH_DECR | MOD_DES | MOD_CBC_DEC | KEYLEN_192,
1324 
1325 }, {
1326 	.crypto	= {
1327 		.base.cra_name		= "ecb(des)",
1328 		.base.cra_blocksize	= DES_BLOCK_SIZE,
1329 		.min_keysize		= DES_KEY_SIZE,
1330 		.max_keysize		= DES_KEY_SIZE,
1331 	},
1332 	.cfg_enc = CIPH_ENCR | MOD_DES | MOD_ECB | KEYLEN_192,
1333 	.cfg_dec = CIPH_DECR | MOD_DES | MOD_ECB | KEYLEN_192,
1334 }, {
1335 	.crypto	= {
1336 		.base.cra_name		= "cbc(des3_ede)",
1337 		.base.cra_blocksize	= DES3_EDE_BLOCK_SIZE,
1338 
1339 		.min_keysize		= DES3_EDE_KEY_SIZE,
1340 		.max_keysize		= DES3_EDE_KEY_SIZE,
1341 		.ivsize			= DES3_EDE_BLOCK_SIZE,
1342 		.setkey			= ablk_des3_setkey,
1343 	},
1344 	.cfg_enc = CIPH_ENCR | MOD_3DES | MOD_CBC_ENC | KEYLEN_192,
1345 	.cfg_dec = CIPH_DECR | MOD_3DES | MOD_CBC_DEC | KEYLEN_192,
1346 }, {
1347 	.crypto	= {
1348 		.base.cra_name		= "ecb(des3_ede)",
1349 		.base.cra_blocksize	= DES3_EDE_BLOCK_SIZE,
1350 
1351 		.min_keysize		= DES3_EDE_KEY_SIZE,
1352 		.max_keysize		= DES3_EDE_KEY_SIZE,
1353 		.setkey			= ablk_des3_setkey,
1354 	},
1355 	.cfg_enc = CIPH_ENCR | MOD_3DES | MOD_ECB | KEYLEN_192,
1356 	.cfg_dec = CIPH_DECR | MOD_3DES | MOD_ECB | KEYLEN_192,
1357 }, {
1358 	.crypto	= {
1359 		.base.cra_name		= "cbc(aes)",
1360 		.base.cra_blocksize	= AES_BLOCK_SIZE,
1361 
1362 		.min_keysize		= AES_MIN_KEY_SIZE,
1363 		.max_keysize		= AES_MAX_KEY_SIZE,
1364 		.ivsize			= AES_BLOCK_SIZE,
1365 	},
1366 	.cfg_enc = CIPH_ENCR | MOD_AES | MOD_CBC_ENC,
1367 	.cfg_dec = CIPH_DECR | MOD_AES | MOD_CBC_DEC,
1368 }, {
1369 	.crypto	= {
1370 		.base.cra_name		= "ecb(aes)",
1371 		.base.cra_blocksize	= AES_BLOCK_SIZE,
1372 
1373 		.min_keysize		= AES_MIN_KEY_SIZE,
1374 		.max_keysize		= AES_MAX_KEY_SIZE,
1375 	},
1376 	.cfg_enc = CIPH_ENCR | MOD_AES | MOD_ECB,
1377 	.cfg_dec = CIPH_DECR | MOD_AES | MOD_ECB,
1378 }, {
1379 	.crypto	= {
1380 		.base.cra_name		= "ctr(aes)",
1381 		.base.cra_blocksize	= 1,
1382 
1383 		.min_keysize		= AES_MIN_KEY_SIZE,
1384 		.max_keysize		= AES_MAX_KEY_SIZE,
1385 		.ivsize			= AES_BLOCK_SIZE,
1386 	},
1387 	.cfg_enc = CIPH_ENCR | MOD_AES | MOD_CTR,
1388 	.cfg_dec = CIPH_ENCR | MOD_AES | MOD_CTR,
1389 }, {
1390 	.crypto	= {
1391 		.base.cra_name		= "rfc3686(ctr(aes))",
1392 		.base.cra_blocksize	= 1,
1393 
1394 		.min_keysize		= AES_MIN_KEY_SIZE,
1395 		.max_keysize		= AES_MAX_KEY_SIZE,
1396 		.ivsize			= AES_BLOCK_SIZE,
1397 		.setkey			= ablk_rfc3686_setkey,
1398 		.encrypt		= ablk_rfc3686_crypt,
1399 		.decrypt		= ablk_rfc3686_crypt,
1400 	},
1401 	.cfg_enc = CIPH_ENCR | MOD_AES | MOD_CTR,
1402 	.cfg_dec = CIPH_ENCR | MOD_AES | MOD_CTR,
1403 } };
1404 
1405 static struct ixp_aead_alg ixp4xx_aeads[] = {
1406 {
1407 	.crypto	= {
1408 		.base = {
1409 			.cra_name	= "authenc(hmac(md5),cbc(des))",
1410 			.cra_blocksize	= DES_BLOCK_SIZE,
1411 		},
1412 		.ivsize		= DES_BLOCK_SIZE,
1413 		.maxauthsize	= MD5_DIGEST_SIZE,
1414 	},
1415 	.hash = &hash_alg_md5,
1416 	.cfg_enc = CIPH_ENCR | MOD_DES | MOD_CBC_ENC | KEYLEN_192,
1417 	.cfg_dec = CIPH_DECR | MOD_DES | MOD_CBC_DEC | KEYLEN_192,
1418 }, {
1419 	.crypto	= {
1420 		.base = {
1421 			.cra_name	= "authenc(hmac(md5),cbc(des3_ede))",
1422 			.cra_blocksize	= DES3_EDE_BLOCK_SIZE,
1423 		},
1424 		.ivsize		= DES3_EDE_BLOCK_SIZE,
1425 		.maxauthsize	= MD5_DIGEST_SIZE,
1426 		.setkey		= des3_aead_setkey,
1427 	},
1428 	.hash = &hash_alg_md5,
1429 	.cfg_enc = CIPH_ENCR | MOD_3DES | MOD_CBC_ENC | KEYLEN_192,
1430 	.cfg_dec = CIPH_DECR | MOD_3DES | MOD_CBC_DEC | KEYLEN_192,
1431 }, {
1432 	.crypto	= {
1433 		.base = {
1434 			.cra_name	= "authenc(hmac(sha1),cbc(des))",
1435 			.cra_blocksize	= DES_BLOCK_SIZE,
1436 		},
1437 			.ivsize		= DES_BLOCK_SIZE,
1438 			.maxauthsize	= SHA1_DIGEST_SIZE,
1439 	},
1440 	.hash = &hash_alg_sha1,
1441 	.cfg_enc = CIPH_ENCR | MOD_DES | MOD_CBC_ENC | KEYLEN_192,
1442 	.cfg_dec = CIPH_DECR | MOD_DES | MOD_CBC_DEC | KEYLEN_192,
1443 }, {
1444 	.crypto	= {
1445 		.base = {
1446 			.cra_name	= "authenc(hmac(sha1),cbc(des3_ede))",
1447 			.cra_blocksize	= DES3_EDE_BLOCK_SIZE,
1448 		},
1449 		.ivsize		= DES3_EDE_BLOCK_SIZE,
1450 		.maxauthsize	= SHA1_DIGEST_SIZE,
1451 		.setkey		= des3_aead_setkey,
1452 	},
1453 	.hash = &hash_alg_sha1,
1454 	.cfg_enc = CIPH_ENCR | MOD_3DES | MOD_CBC_ENC | KEYLEN_192,
1455 	.cfg_dec = CIPH_DECR | MOD_3DES | MOD_CBC_DEC | KEYLEN_192,
1456 }, {
1457 	.crypto	= {
1458 		.base = {
1459 			.cra_name	= "authenc(hmac(md5),cbc(aes))",
1460 			.cra_blocksize	= AES_BLOCK_SIZE,
1461 		},
1462 		.ivsize		= AES_BLOCK_SIZE,
1463 		.maxauthsize	= MD5_DIGEST_SIZE,
1464 	},
1465 	.hash = &hash_alg_md5,
1466 	.cfg_enc = CIPH_ENCR | MOD_AES | MOD_CBC_ENC,
1467 	.cfg_dec = CIPH_DECR | MOD_AES | MOD_CBC_DEC,
1468 }, {
1469 	.crypto	= {
1470 		.base = {
1471 			.cra_name	= "authenc(hmac(sha1),cbc(aes))",
1472 			.cra_blocksize	= AES_BLOCK_SIZE,
1473 		},
1474 		.ivsize		= AES_BLOCK_SIZE,
1475 		.maxauthsize	= SHA1_DIGEST_SIZE,
1476 	},
1477 	.hash = &hash_alg_sha1,
1478 	.cfg_enc = CIPH_ENCR | MOD_AES | MOD_CBC_ENC,
1479 	.cfg_dec = CIPH_DECR | MOD_AES | MOD_CBC_DEC,
1480 } };
1481 
1482 #define IXP_POSTFIX "-ixp4xx"
1483 
1484 static int ixp_crypto_probe(struct platform_device *_pdev)
1485 {
1486 	struct device *dev = &_pdev->dev;
1487 	int num = ARRAY_SIZE(ixp4xx_algos);
1488 	int i, err;
1489 
1490 	pdev = _pdev;
1491 
1492 	err = init_ixp_crypto(dev);
1493 	if (err)
1494 		return err;
1495 
1496 	for (i = 0; i < num; i++) {
1497 		struct skcipher_alg *cra = &ixp4xx_algos[i].crypto;
1498 
1499 		if (snprintf(cra->base.cra_driver_name, CRYPTO_MAX_ALG_NAME,
1500 			     "%s"IXP_POSTFIX, cra->base.cra_name) >=
1501 			     CRYPTO_MAX_ALG_NAME)
1502 			continue;
1503 		if (!support_aes && (ixp4xx_algos[i].cfg_enc & MOD_AES))
1504 			continue;
1505 
1506 		/* block ciphers */
1507 		cra->base.cra_flags = CRYPTO_ALG_KERN_DRIVER_ONLY |
1508 				      CRYPTO_ALG_ASYNC |
1509 				      CRYPTO_ALG_ALLOCATES_MEMORY |
1510 				      CRYPTO_ALG_NEED_FALLBACK;
1511 		if (!cra->setkey)
1512 			cra->setkey = ablk_setkey;
1513 		if (!cra->encrypt)
1514 			cra->encrypt = ablk_encrypt;
1515 		if (!cra->decrypt)
1516 			cra->decrypt = ablk_decrypt;
1517 		cra->init = init_tfm_ablk;
1518 		cra->exit = exit_tfm_ablk;
1519 
1520 		cra->base.cra_ctxsize = sizeof(struct ixp_ctx);
1521 		cra->base.cra_module = THIS_MODULE;
1522 		cra->base.cra_alignmask = 3;
1523 		cra->base.cra_priority = 300;
1524 		if (crypto_register_skcipher(cra))
1525 			dev_err(&pdev->dev, "Failed to register '%s'\n",
1526 				cra->base.cra_name);
1527 		else
1528 			ixp4xx_algos[i].registered = 1;
1529 	}
1530 
1531 	for (i = 0; i < ARRAY_SIZE(ixp4xx_aeads); i++) {
1532 		struct aead_alg *cra = &ixp4xx_aeads[i].crypto;
1533 
1534 		if (snprintf(cra->base.cra_driver_name, CRYPTO_MAX_ALG_NAME,
1535 			     "%s"IXP_POSTFIX, cra->base.cra_name) >=
1536 		    CRYPTO_MAX_ALG_NAME)
1537 			continue;
1538 		if (!support_aes && (ixp4xx_algos[i].cfg_enc & MOD_AES))
1539 			continue;
1540 
1541 		/* authenc */
1542 		cra->base.cra_flags = CRYPTO_ALG_KERN_DRIVER_ONLY |
1543 				      CRYPTO_ALG_ASYNC |
1544 				      CRYPTO_ALG_ALLOCATES_MEMORY;
1545 		cra->setkey = cra->setkey ?: aead_setkey;
1546 		cra->setauthsize = aead_setauthsize;
1547 		cra->encrypt = aead_encrypt;
1548 		cra->decrypt = aead_decrypt;
1549 		cra->init = init_tfm_aead;
1550 		cra->exit = exit_tfm_aead;
1551 
1552 		cra->base.cra_ctxsize = sizeof(struct ixp_ctx);
1553 		cra->base.cra_module = THIS_MODULE;
1554 		cra->base.cra_alignmask = 3;
1555 		cra->base.cra_priority = 300;
1556 
1557 		if (crypto_register_aead(cra))
1558 			dev_err(&pdev->dev, "Failed to register '%s'\n",
1559 				cra->base.cra_driver_name);
1560 		else
1561 			ixp4xx_aeads[i].registered = 1;
1562 	}
1563 	return 0;
1564 }
1565 
1566 static int ixp_crypto_remove(struct platform_device *pdev)
1567 {
1568 	int num = ARRAY_SIZE(ixp4xx_algos);
1569 	int i;
1570 
1571 	for (i = 0; i < ARRAY_SIZE(ixp4xx_aeads); i++) {
1572 		if (ixp4xx_aeads[i].registered)
1573 			crypto_unregister_aead(&ixp4xx_aeads[i].crypto);
1574 	}
1575 
1576 	for (i = 0; i < num; i++) {
1577 		if (ixp4xx_algos[i].registered)
1578 			crypto_unregister_skcipher(&ixp4xx_algos[i].crypto);
1579 	}
1580 	release_ixp_crypto(&pdev->dev);
1581 
1582 	return 0;
1583 }
1584 static const struct of_device_id ixp4xx_crypto_of_match[] = {
1585 	{
1586 		.compatible = "intel,ixp4xx-crypto",
1587 	},
1588 	{},
1589 };
1590 
1591 static struct platform_driver ixp_crypto_driver = {
1592 	.probe = ixp_crypto_probe,
1593 	.remove = ixp_crypto_remove,
1594 	.driver = {
1595 		.name = "ixp4xx_crypto",
1596 		.of_match_table = ixp4xx_crypto_of_match,
1597 	},
1598 };
1599 module_platform_driver(ixp_crypto_driver);
1600 
1601 MODULE_LICENSE("GPL");
1602 MODULE_AUTHOR("Christian Hohnstaedt <chohnstaedt@innominate.com>");
1603 MODULE_DESCRIPTION("IXP4xx hardware crypto");
1604 
1605