xref: /openbmc/linux/drivers/crypto/omap-sham.c (revision e2f1cf25)
1 /*
2  * Cryptographic API.
3  *
4  * Support for OMAP SHA1/MD5 HW acceleration.
5  *
6  * Copyright (c) 2010 Nokia Corporation
7  * Author: Dmitry Kasatkin <dmitry.kasatkin@nokia.com>
8  * Copyright (c) 2011 Texas Instruments Incorporated
9  *
10  * This program is free software; you can redistribute it and/or modify
11  * it under the terms of the GNU General Public License version 2 as published
12  * by the Free Software Foundation.
13  *
14  * Some ideas are from old omap-sha1-md5.c driver.
15  */
16 
17 #define pr_fmt(fmt) "%s: " fmt, __func__
18 
19 #include <linux/err.h>
20 #include <linux/device.h>
21 #include <linux/module.h>
22 #include <linux/init.h>
23 #include <linux/errno.h>
24 #include <linux/interrupt.h>
25 #include <linux/kernel.h>
26 #include <linux/irq.h>
27 #include <linux/io.h>
28 #include <linux/platform_device.h>
29 #include <linux/scatterlist.h>
30 #include <linux/dma-mapping.h>
31 #include <linux/dmaengine.h>
32 #include <linux/omap-dma.h>
33 #include <linux/pm_runtime.h>
34 #include <linux/of.h>
35 #include <linux/of_device.h>
36 #include <linux/of_address.h>
37 #include <linux/of_irq.h>
38 #include <linux/delay.h>
39 #include <linux/crypto.h>
40 #include <linux/cryptohash.h>
41 #include <crypto/scatterwalk.h>
42 #include <crypto/algapi.h>
43 #include <crypto/sha.h>
44 #include <crypto/hash.h>
45 #include <crypto/internal/hash.h>
46 
47 #define MD5_DIGEST_SIZE			16
48 
49 #define SHA_REG_IDIGEST(dd, x)		((dd)->pdata->idigest_ofs + ((x)*0x04))
50 #define SHA_REG_DIN(dd, x)		((dd)->pdata->din_ofs + ((x) * 0x04))
51 #define SHA_REG_DIGCNT(dd)		((dd)->pdata->digcnt_ofs)
52 
53 #define SHA_REG_ODIGEST(dd, x)		((dd)->pdata->odigest_ofs + (x * 0x04))
54 
55 #define SHA_REG_CTRL			0x18
56 #define SHA_REG_CTRL_LENGTH		(0xFFFFFFFF << 5)
57 #define SHA_REG_CTRL_CLOSE_HASH		(1 << 4)
58 #define SHA_REG_CTRL_ALGO_CONST		(1 << 3)
59 #define SHA_REG_CTRL_ALGO		(1 << 2)
60 #define SHA_REG_CTRL_INPUT_READY	(1 << 1)
61 #define SHA_REG_CTRL_OUTPUT_READY	(1 << 0)
62 
63 #define SHA_REG_REV(dd)			((dd)->pdata->rev_ofs)
64 
65 #define SHA_REG_MASK(dd)		((dd)->pdata->mask_ofs)
66 #define SHA_REG_MASK_DMA_EN		(1 << 3)
67 #define SHA_REG_MASK_IT_EN		(1 << 2)
68 #define SHA_REG_MASK_SOFTRESET		(1 << 1)
69 #define SHA_REG_AUTOIDLE		(1 << 0)
70 
71 #define SHA_REG_SYSSTATUS(dd)		((dd)->pdata->sysstatus_ofs)
72 #define SHA_REG_SYSSTATUS_RESETDONE	(1 << 0)
73 
74 #define SHA_REG_MODE(dd)		((dd)->pdata->mode_ofs)
75 #define SHA_REG_MODE_HMAC_OUTER_HASH	(1 << 7)
76 #define SHA_REG_MODE_HMAC_KEY_PROC	(1 << 5)
77 #define SHA_REG_MODE_CLOSE_HASH		(1 << 4)
78 #define SHA_REG_MODE_ALGO_CONSTANT	(1 << 3)
79 
80 #define SHA_REG_MODE_ALGO_MASK		(7 << 0)
81 #define SHA_REG_MODE_ALGO_MD5_128	(0 << 1)
82 #define SHA_REG_MODE_ALGO_SHA1_160	(1 << 1)
83 #define SHA_REG_MODE_ALGO_SHA2_224	(2 << 1)
84 #define SHA_REG_MODE_ALGO_SHA2_256	(3 << 1)
85 #define SHA_REG_MODE_ALGO_SHA2_384	(1 << 0)
86 #define SHA_REG_MODE_ALGO_SHA2_512	(3 << 0)
87 
88 #define SHA_REG_LENGTH(dd)		((dd)->pdata->length_ofs)
89 
90 #define SHA_REG_IRQSTATUS		0x118
91 #define SHA_REG_IRQSTATUS_CTX_RDY	(1 << 3)
92 #define SHA_REG_IRQSTATUS_PARTHASH_RDY (1 << 2)
93 #define SHA_REG_IRQSTATUS_INPUT_RDY	(1 << 1)
94 #define SHA_REG_IRQSTATUS_OUTPUT_RDY	(1 << 0)
95 
96 #define SHA_REG_IRQENA			0x11C
97 #define SHA_REG_IRQENA_CTX_RDY		(1 << 3)
98 #define SHA_REG_IRQENA_PARTHASH_RDY	(1 << 2)
99 #define SHA_REG_IRQENA_INPUT_RDY	(1 << 1)
100 #define SHA_REG_IRQENA_OUTPUT_RDY	(1 << 0)
101 
102 #define DEFAULT_TIMEOUT_INTERVAL	HZ
103 
104 /* mostly device flags */
105 #define FLAGS_BUSY		0
106 #define FLAGS_FINAL		1
107 #define FLAGS_DMA_ACTIVE	2
108 #define FLAGS_OUTPUT_READY	3
109 #define FLAGS_INIT		4
110 #define FLAGS_CPU		5
111 #define FLAGS_DMA_READY		6
112 #define FLAGS_AUTO_XOR		7
113 #define FLAGS_BE32_SHA1		8
114 /* context flags */
115 #define FLAGS_FINUP		16
116 #define FLAGS_SG		17
117 
118 #define FLAGS_MODE_SHIFT	18
119 #define FLAGS_MODE_MASK		(SHA_REG_MODE_ALGO_MASK	<< FLAGS_MODE_SHIFT)
120 #define FLAGS_MODE_MD5		(SHA_REG_MODE_ALGO_MD5_128 << FLAGS_MODE_SHIFT)
121 #define FLAGS_MODE_SHA1		(SHA_REG_MODE_ALGO_SHA1_160 << FLAGS_MODE_SHIFT)
122 #define FLAGS_MODE_SHA224	(SHA_REG_MODE_ALGO_SHA2_224 << FLAGS_MODE_SHIFT)
123 #define FLAGS_MODE_SHA256	(SHA_REG_MODE_ALGO_SHA2_256 << FLAGS_MODE_SHIFT)
124 #define FLAGS_MODE_SHA384	(SHA_REG_MODE_ALGO_SHA2_384 << FLAGS_MODE_SHIFT)
125 #define FLAGS_MODE_SHA512	(SHA_REG_MODE_ALGO_SHA2_512 << FLAGS_MODE_SHIFT)
126 
127 #define FLAGS_HMAC		21
128 #define FLAGS_ERROR		22
129 
130 #define OP_UPDATE		1
131 #define OP_FINAL		2
132 
133 #define OMAP_ALIGN_MASK		(sizeof(u32)-1)
134 #define OMAP_ALIGNED		__attribute__((aligned(sizeof(u32))))
135 
136 #define BUFLEN			PAGE_SIZE
137 
138 struct omap_sham_dev;
139 
140 struct omap_sham_reqctx {
141 	struct omap_sham_dev	*dd;
142 	unsigned long		flags;
143 	unsigned long		op;
144 
145 	u8			digest[SHA512_DIGEST_SIZE] OMAP_ALIGNED;
146 	size_t			digcnt;
147 	size_t			bufcnt;
148 	size_t			buflen;
149 	dma_addr_t		dma_addr;
150 
151 	/* walk state */
152 	struct scatterlist	*sg;
153 	struct scatterlist	sgl;
154 	unsigned int		offset;	/* offset in current sg */
155 	unsigned int		total;	/* total request */
156 
157 	u8			buffer[0] OMAP_ALIGNED;
158 };
159 
160 struct omap_sham_hmac_ctx {
161 	struct crypto_shash	*shash;
162 	u8			ipad[SHA512_BLOCK_SIZE] OMAP_ALIGNED;
163 	u8			opad[SHA512_BLOCK_SIZE] OMAP_ALIGNED;
164 };
165 
166 struct omap_sham_ctx {
167 	struct omap_sham_dev	*dd;
168 
169 	unsigned long		flags;
170 
171 	/* fallback stuff */
172 	struct crypto_shash	*fallback;
173 
174 	struct omap_sham_hmac_ctx base[0];
175 };
176 
177 #define OMAP_SHAM_QUEUE_LENGTH	1
178 
179 struct omap_sham_algs_info {
180 	struct ahash_alg	*algs_list;
181 	unsigned int		size;
182 	unsigned int		registered;
183 };
184 
185 struct omap_sham_pdata {
186 	struct omap_sham_algs_info	*algs_info;
187 	unsigned int	algs_info_size;
188 	unsigned long	flags;
189 	int		digest_size;
190 
191 	void		(*copy_hash)(struct ahash_request *req, int out);
192 	void		(*write_ctrl)(struct omap_sham_dev *dd, size_t length,
193 				      int final, int dma);
194 	void		(*trigger)(struct omap_sham_dev *dd, size_t length);
195 	int		(*poll_irq)(struct omap_sham_dev *dd);
196 	irqreturn_t	(*intr_hdlr)(int irq, void *dev_id);
197 
198 	u32		odigest_ofs;
199 	u32		idigest_ofs;
200 	u32		din_ofs;
201 	u32		digcnt_ofs;
202 	u32		rev_ofs;
203 	u32		mask_ofs;
204 	u32		sysstatus_ofs;
205 	u32		mode_ofs;
206 	u32		length_ofs;
207 
208 	u32		major_mask;
209 	u32		major_shift;
210 	u32		minor_mask;
211 	u32		minor_shift;
212 };
213 
214 struct omap_sham_dev {
215 	struct list_head	list;
216 	unsigned long		phys_base;
217 	struct device		*dev;
218 	void __iomem		*io_base;
219 	int			irq;
220 	spinlock_t		lock;
221 	int			err;
222 	unsigned int		dma;
223 	struct dma_chan		*dma_lch;
224 	struct tasklet_struct	done_task;
225 	u8			polling_mode;
226 
227 	unsigned long		flags;
228 	struct crypto_queue	queue;
229 	struct ahash_request	*req;
230 
231 	const struct omap_sham_pdata	*pdata;
232 };
233 
234 struct omap_sham_drv {
235 	struct list_head	dev_list;
236 	spinlock_t		lock;
237 	unsigned long		flags;
238 };
239 
240 static struct omap_sham_drv sham = {
241 	.dev_list = LIST_HEAD_INIT(sham.dev_list),
242 	.lock = __SPIN_LOCK_UNLOCKED(sham.lock),
243 };
244 
245 static inline u32 omap_sham_read(struct omap_sham_dev *dd, u32 offset)
246 {
247 	return __raw_readl(dd->io_base + offset);
248 }
249 
250 static inline void omap_sham_write(struct omap_sham_dev *dd,
251 					u32 offset, u32 value)
252 {
253 	__raw_writel(value, dd->io_base + offset);
254 }
255 
256 static inline void omap_sham_write_mask(struct omap_sham_dev *dd, u32 address,
257 					u32 value, u32 mask)
258 {
259 	u32 val;
260 
261 	val = omap_sham_read(dd, address);
262 	val &= ~mask;
263 	val |= value;
264 	omap_sham_write(dd, address, val);
265 }
266 
267 static inline int omap_sham_wait(struct omap_sham_dev *dd, u32 offset, u32 bit)
268 {
269 	unsigned long timeout = jiffies + DEFAULT_TIMEOUT_INTERVAL;
270 
271 	while (!(omap_sham_read(dd, offset) & bit)) {
272 		if (time_is_before_jiffies(timeout))
273 			return -ETIMEDOUT;
274 	}
275 
276 	return 0;
277 }
278 
279 static void omap_sham_copy_hash_omap2(struct ahash_request *req, int out)
280 {
281 	struct omap_sham_reqctx *ctx = ahash_request_ctx(req);
282 	struct omap_sham_dev *dd = ctx->dd;
283 	u32 *hash = (u32 *)ctx->digest;
284 	int i;
285 
286 	for (i = 0; i < dd->pdata->digest_size / sizeof(u32); i++) {
287 		if (out)
288 			hash[i] = omap_sham_read(dd, SHA_REG_IDIGEST(dd, i));
289 		else
290 			omap_sham_write(dd, SHA_REG_IDIGEST(dd, i), hash[i]);
291 	}
292 }
293 
294 static void omap_sham_copy_hash_omap4(struct ahash_request *req, int out)
295 {
296 	struct omap_sham_reqctx *ctx = ahash_request_ctx(req);
297 	struct omap_sham_dev *dd = ctx->dd;
298 	int i;
299 
300 	if (ctx->flags & BIT(FLAGS_HMAC)) {
301 		struct crypto_ahash *tfm = crypto_ahash_reqtfm(dd->req);
302 		struct omap_sham_ctx *tctx = crypto_ahash_ctx(tfm);
303 		struct omap_sham_hmac_ctx *bctx = tctx->base;
304 		u32 *opad = (u32 *)bctx->opad;
305 
306 		for (i = 0; i < dd->pdata->digest_size / sizeof(u32); i++) {
307 			if (out)
308 				opad[i] = omap_sham_read(dd,
309 						SHA_REG_ODIGEST(dd, i));
310 			else
311 				omap_sham_write(dd, SHA_REG_ODIGEST(dd, i),
312 						opad[i]);
313 		}
314 	}
315 
316 	omap_sham_copy_hash_omap2(req, out);
317 }
318 
319 static void omap_sham_copy_ready_hash(struct ahash_request *req)
320 {
321 	struct omap_sham_reqctx *ctx = ahash_request_ctx(req);
322 	u32 *in = (u32 *)ctx->digest;
323 	u32 *hash = (u32 *)req->result;
324 	int i, d, big_endian = 0;
325 
326 	if (!hash)
327 		return;
328 
329 	switch (ctx->flags & FLAGS_MODE_MASK) {
330 	case FLAGS_MODE_MD5:
331 		d = MD5_DIGEST_SIZE / sizeof(u32);
332 		break;
333 	case FLAGS_MODE_SHA1:
334 		/* OMAP2 SHA1 is big endian */
335 		if (test_bit(FLAGS_BE32_SHA1, &ctx->dd->flags))
336 			big_endian = 1;
337 		d = SHA1_DIGEST_SIZE / sizeof(u32);
338 		break;
339 	case FLAGS_MODE_SHA224:
340 		d = SHA224_DIGEST_SIZE / sizeof(u32);
341 		break;
342 	case FLAGS_MODE_SHA256:
343 		d = SHA256_DIGEST_SIZE / sizeof(u32);
344 		break;
345 	case FLAGS_MODE_SHA384:
346 		d = SHA384_DIGEST_SIZE / sizeof(u32);
347 		break;
348 	case FLAGS_MODE_SHA512:
349 		d = SHA512_DIGEST_SIZE / sizeof(u32);
350 		break;
351 	default:
352 		d = 0;
353 	}
354 
355 	if (big_endian)
356 		for (i = 0; i < d; i++)
357 			hash[i] = be32_to_cpu(in[i]);
358 	else
359 		for (i = 0; i < d; i++)
360 			hash[i] = le32_to_cpu(in[i]);
361 }
362 
363 static int omap_sham_hw_init(struct omap_sham_dev *dd)
364 {
365 	int err;
366 
367 	err = pm_runtime_get_sync(dd->dev);
368 	if (err < 0) {
369 		dev_err(dd->dev, "failed to get sync: %d\n", err);
370 		return err;
371 	}
372 
373 	if (!test_bit(FLAGS_INIT, &dd->flags)) {
374 		set_bit(FLAGS_INIT, &dd->flags);
375 		dd->err = 0;
376 	}
377 
378 	return 0;
379 }
380 
381 static void omap_sham_write_ctrl_omap2(struct omap_sham_dev *dd, size_t length,
382 				 int final, int dma)
383 {
384 	struct omap_sham_reqctx *ctx = ahash_request_ctx(dd->req);
385 	u32 val = length << 5, mask;
386 
387 	if (likely(ctx->digcnt))
388 		omap_sham_write(dd, SHA_REG_DIGCNT(dd), ctx->digcnt);
389 
390 	omap_sham_write_mask(dd, SHA_REG_MASK(dd),
391 		SHA_REG_MASK_IT_EN | (dma ? SHA_REG_MASK_DMA_EN : 0),
392 		SHA_REG_MASK_IT_EN | SHA_REG_MASK_DMA_EN);
393 	/*
394 	 * Setting ALGO_CONST only for the first iteration
395 	 * and CLOSE_HASH only for the last one.
396 	 */
397 	if ((ctx->flags & FLAGS_MODE_MASK) == FLAGS_MODE_SHA1)
398 		val |= SHA_REG_CTRL_ALGO;
399 	if (!ctx->digcnt)
400 		val |= SHA_REG_CTRL_ALGO_CONST;
401 	if (final)
402 		val |= SHA_REG_CTRL_CLOSE_HASH;
403 
404 	mask = SHA_REG_CTRL_ALGO_CONST | SHA_REG_CTRL_CLOSE_HASH |
405 			SHA_REG_CTRL_ALGO | SHA_REG_CTRL_LENGTH;
406 
407 	omap_sham_write_mask(dd, SHA_REG_CTRL, val, mask);
408 }
409 
410 static void omap_sham_trigger_omap2(struct omap_sham_dev *dd, size_t length)
411 {
412 }
413 
414 static int omap_sham_poll_irq_omap2(struct omap_sham_dev *dd)
415 {
416 	return omap_sham_wait(dd, SHA_REG_CTRL, SHA_REG_CTRL_INPUT_READY);
417 }
418 
419 static int get_block_size(struct omap_sham_reqctx *ctx)
420 {
421 	int d;
422 
423 	switch (ctx->flags & FLAGS_MODE_MASK) {
424 	case FLAGS_MODE_MD5:
425 	case FLAGS_MODE_SHA1:
426 		d = SHA1_BLOCK_SIZE;
427 		break;
428 	case FLAGS_MODE_SHA224:
429 	case FLAGS_MODE_SHA256:
430 		d = SHA256_BLOCK_SIZE;
431 		break;
432 	case FLAGS_MODE_SHA384:
433 	case FLAGS_MODE_SHA512:
434 		d = SHA512_BLOCK_SIZE;
435 		break;
436 	default:
437 		d = 0;
438 	}
439 
440 	return d;
441 }
442 
443 static void omap_sham_write_n(struct omap_sham_dev *dd, u32 offset,
444 				    u32 *value, int count)
445 {
446 	for (; count--; value++, offset += 4)
447 		omap_sham_write(dd, offset, *value);
448 }
449 
450 static void omap_sham_write_ctrl_omap4(struct omap_sham_dev *dd, size_t length,
451 				 int final, int dma)
452 {
453 	struct omap_sham_reqctx *ctx = ahash_request_ctx(dd->req);
454 	u32 val, mask;
455 
456 	/*
457 	 * Setting ALGO_CONST only for the first iteration and
458 	 * CLOSE_HASH only for the last one. Note that flags mode bits
459 	 * correspond to algorithm encoding in mode register.
460 	 */
461 	val = (ctx->flags & FLAGS_MODE_MASK) >> (FLAGS_MODE_SHIFT);
462 	if (!ctx->digcnt) {
463 		struct crypto_ahash *tfm = crypto_ahash_reqtfm(dd->req);
464 		struct omap_sham_ctx *tctx = crypto_ahash_ctx(tfm);
465 		struct omap_sham_hmac_ctx *bctx = tctx->base;
466 		int bs, nr_dr;
467 
468 		val |= SHA_REG_MODE_ALGO_CONSTANT;
469 
470 		if (ctx->flags & BIT(FLAGS_HMAC)) {
471 			bs = get_block_size(ctx);
472 			nr_dr = bs / (2 * sizeof(u32));
473 			val |= SHA_REG_MODE_HMAC_KEY_PROC;
474 			omap_sham_write_n(dd, SHA_REG_ODIGEST(dd, 0),
475 					  (u32 *)bctx->ipad, nr_dr);
476 			omap_sham_write_n(dd, SHA_REG_IDIGEST(dd, 0),
477 					  (u32 *)bctx->ipad + nr_dr, nr_dr);
478 			ctx->digcnt += bs;
479 		}
480 	}
481 
482 	if (final) {
483 		val |= SHA_REG_MODE_CLOSE_HASH;
484 
485 		if (ctx->flags & BIT(FLAGS_HMAC))
486 			val |= SHA_REG_MODE_HMAC_OUTER_HASH;
487 	}
488 
489 	mask = SHA_REG_MODE_ALGO_CONSTANT | SHA_REG_MODE_CLOSE_HASH |
490 	       SHA_REG_MODE_ALGO_MASK | SHA_REG_MODE_HMAC_OUTER_HASH |
491 	       SHA_REG_MODE_HMAC_KEY_PROC;
492 
493 	dev_dbg(dd->dev, "ctrl: %08x, flags: %08lx\n", val, ctx->flags);
494 	omap_sham_write_mask(dd, SHA_REG_MODE(dd), val, mask);
495 	omap_sham_write(dd, SHA_REG_IRQENA, SHA_REG_IRQENA_OUTPUT_RDY);
496 	omap_sham_write_mask(dd, SHA_REG_MASK(dd),
497 			     SHA_REG_MASK_IT_EN |
498 				     (dma ? SHA_REG_MASK_DMA_EN : 0),
499 			     SHA_REG_MASK_IT_EN | SHA_REG_MASK_DMA_EN);
500 }
501 
502 static void omap_sham_trigger_omap4(struct omap_sham_dev *dd, size_t length)
503 {
504 	omap_sham_write(dd, SHA_REG_LENGTH(dd), length);
505 }
506 
507 static int omap_sham_poll_irq_omap4(struct omap_sham_dev *dd)
508 {
509 	return omap_sham_wait(dd, SHA_REG_IRQSTATUS,
510 			      SHA_REG_IRQSTATUS_INPUT_RDY);
511 }
512 
513 static int omap_sham_xmit_cpu(struct omap_sham_dev *dd, const u8 *buf,
514 			      size_t length, int final)
515 {
516 	struct omap_sham_reqctx *ctx = ahash_request_ctx(dd->req);
517 	int count, len32, bs32, offset = 0;
518 	const u32 *buffer = (const u32 *)buf;
519 
520 	dev_dbg(dd->dev, "xmit_cpu: digcnt: %d, length: %d, final: %d\n",
521 						ctx->digcnt, length, final);
522 
523 	dd->pdata->write_ctrl(dd, length, final, 0);
524 	dd->pdata->trigger(dd, length);
525 
526 	/* should be non-zero before next lines to disable clocks later */
527 	ctx->digcnt += length;
528 
529 	if (final)
530 		set_bit(FLAGS_FINAL, &dd->flags); /* catch last interrupt */
531 
532 	set_bit(FLAGS_CPU, &dd->flags);
533 
534 	len32 = DIV_ROUND_UP(length, sizeof(u32));
535 	bs32 = get_block_size(ctx) / sizeof(u32);
536 
537 	while (len32) {
538 		if (dd->pdata->poll_irq(dd))
539 			return -ETIMEDOUT;
540 
541 		for (count = 0; count < min(len32, bs32); count++, offset++)
542 			omap_sham_write(dd, SHA_REG_DIN(dd, count),
543 					buffer[offset]);
544 		len32 -= min(len32, bs32);
545 	}
546 
547 	return -EINPROGRESS;
548 }
549 
550 static void omap_sham_dma_callback(void *param)
551 {
552 	struct omap_sham_dev *dd = param;
553 
554 	set_bit(FLAGS_DMA_READY, &dd->flags);
555 	tasklet_schedule(&dd->done_task);
556 }
557 
558 static int omap_sham_xmit_dma(struct omap_sham_dev *dd, dma_addr_t dma_addr,
559 			      size_t length, int final, int is_sg)
560 {
561 	struct omap_sham_reqctx *ctx = ahash_request_ctx(dd->req);
562 	struct dma_async_tx_descriptor *tx;
563 	struct dma_slave_config cfg;
564 	int len32, ret, dma_min = get_block_size(ctx);
565 
566 	dev_dbg(dd->dev, "xmit_dma: digcnt: %d, length: %d, final: %d\n",
567 						ctx->digcnt, length, final);
568 
569 	memset(&cfg, 0, sizeof(cfg));
570 
571 	cfg.dst_addr = dd->phys_base + SHA_REG_DIN(dd, 0);
572 	cfg.dst_addr_width = DMA_SLAVE_BUSWIDTH_4_BYTES;
573 	cfg.dst_maxburst = dma_min / DMA_SLAVE_BUSWIDTH_4_BYTES;
574 
575 	ret = dmaengine_slave_config(dd->dma_lch, &cfg);
576 	if (ret) {
577 		pr_err("omap-sham: can't configure dmaengine slave: %d\n", ret);
578 		return ret;
579 	}
580 
581 	len32 = DIV_ROUND_UP(length, dma_min) * dma_min;
582 
583 	if (is_sg) {
584 		/*
585 		 * The SG entry passed in may not have the 'length' member
586 		 * set correctly so use a local SG entry (sgl) with the
587 		 * proper value for 'length' instead.  If this is not done,
588 		 * the dmaengine may try to DMA the incorrect amount of data.
589 		 */
590 		sg_init_table(&ctx->sgl, 1);
591 		ctx->sgl.page_link = ctx->sg->page_link;
592 		ctx->sgl.offset = ctx->sg->offset;
593 		sg_dma_len(&ctx->sgl) = len32;
594 		sg_dma_address(&ctx->sgl) = sg_dma_address(ctx->sg);
595 
596 		tx = dmaengine_prep_slave_sg(dd->dma_lch, &ctx->sgl, 1,
597 			DMA_MEM_TO_DEV, DMA_PREP_INTERRUPT | DMA_CTRL_ACK);
598 	} else {
599 		tx = dmaengine_prep_slave_single(dd->dma_lch, dma_addr, len32,
600 			DMA_MEM_TO_DEV, DMA_PREP_INTERRUPT | DMA_CTRL_ACK);
601 	}
602 
603 	if (!tx) {
604 		dev_err(dd->dev, "prep_slave_sg/single() failed\n");
605 		return -EINVAL;
606 	}
607 
608 	tx->callback = omap_sham_dma_callback;
609 	tx->callback_param = dd;
610 
611 	dd->pdata->write_ctrl(dd, length, final, 1);
612 
613 	ctx->digcnt += length;
614 
615 	if (final)
616 		set_bit(FLAGS_FINAL, &dd->flags); /* catch last interrupt */
617 
618 	set_bit(FLAGS_DMA_ACTIVE, &dd->flags);
619 
620 	dmaengine_submit(tx);
621 	dma_async_issue_pending(dd->dma_lch);
622 
623 	dd->pdata->trigger(dd, length);
624 
625 	return -EINPROGRESS;
626 }
627 
628 static size_t omap_sham_append_buffer(struct omap_sham_reqctx *ctx,
629 				const u8 *data, size_t length)
630 {
631 	size_t count = min(length, ctx->buflen - ctx->bufcnt);
632 
633 	count = min(count, ctx->total);
634 	if (count <= 0)
635 		return 0;
636 	memcpy(ctx->buffer + ctx->bufcnt, data, count);
637 	ctx->bufcnt += count;
638 
639 	return count;
640 }
641 
642 static size_t omap_sham_append_sg(struct omap_sham_reqctx *ctx)
643 {
644 	size_t count;
645 	const u8 *vaddr;
646 
647 	while (ctx->sg) {
648 		vaddr = kmap_atomic(sg_page(ctx->sg));
649 		vaddr += ctx->sg->offset;
650 
651 		count = omap_sham_append_buffer(ctx,
652 				vaddr + ctx->offset,
653 				ctx->sg->length - ctx->offset);
654 
655 		kunmap_atomic((void *)vaddr);
656 
657 		if (!count)
658 			break;
659 		ctx->offset += count;
660 		ctx->total -= count;
661 		if (ctx->offset == ctx->sg->length) {
662 			ctx->sg = sg_next(ctx->sg);
663 			if (ctx->sg)
664 				ctx->offset = 0;
665 			else
666 				ctx->total = 0;
667 		}
668 	}
669 
670 	return 0;
671 }
672 
673 static int omap_sham_xmit_dma_map(struct omap_sham_dev *dd,
674 					struct omap_sham_reqctx *ctx,
675 					size_t length, int final)
676 {
677 	int ret;
678 
679 	ctx->dma_addr = dma_map_single(dd->dev, ctx->buffer, ctx->buflen,
680 				       DMA_TO_DEVICE);
681 	if (dma_mapping_error(dd->dev, ctx->dma_addr)) {
682 		dev_err(dd->dev, "dma %u bytes error\n", ctx->buflen);
683 		return -EINVAL;
684 	}
685 
686 	ctx->flags &= ~BIT(FLAGS_SG);
687 
688 	ret = omap_sham_xmit_dma(dd, ctx->dma_addr, length, final, 0);
689 	if (ret != -EINPROGRESS)
690 		dma_unmap_single(dd->dev, ctx->dma_addr, ctx->buflen,
691 				 DMA_TO_DEVICE);
692 
693 	return ret;
694 }
695 
696 static int omap_sham_update_dma_slow(struct omap_sham_dev *dd)
697 {
698 	struct omap_sham_reqctx *ctx = ahash_request_ctx(dd->req);
699 	unsigned int final;
700 	size_t count;
701 
702 	omap_sham_append_sg(ctx);
703 
704 	final = (ctx->flags & BIT(FLAGS_FINUP)) && !ctx->total;
705 
706 	dev_dbg(dd->dev, "slow: bufcnt: %u, digcnt: %d, final: %d\n",
707 					 ctx->bufcnt, ctx->digcnt, final);
708 
709 	if (final || (ctx->bufcnt == ctx->buflen && ctx->total)) {
710 		count = ctx->bufcnt;
711 		ctx->bufcnt = 0;
712 		return omap_sham_xmit_dma_map(dd, ctx, count, final);
713 	}
714 
715 	return 0;
716 }
717 
718 /* Start address alignment */
719 #define SG_AA(sg)	(IS_ALIGNED(sg->offset, sizeof(u32)))
720 /* SHA1 block size alignment */
721 #define SG_SA(sg, bs)	(IS_ALIGNED(sg->length, bs))
722 
723 static int omap_sham_update_dma_start(struct omap_sham_dev *dd)
724 {
725 	struct omap_sham_reqctx *ctx = ahash_request_ctx(dd->req);
726 	unsigned int length, final, tail;
727 	struct scatterlist *sg;
728 	int ret, bs;
729 
730 	if (!ctx->total)
731 		return 0;
732 
733 	if (ctx->bufcnt || ctx->offset)
734 		return omap_sham_update_dma_slow(dd);
735 
736 	/*
737 	 * Don't use the sg interface when the transfer size is less
738 	 * than the number of elements in a DMA frame.  Otherwise,
739 	 * the dmaengine infrastructure will calculate that it needs
740 	 * to transfer 0 frames which ultimately fails.
741 	 */
742 	if (ctx->total < get_block_size(ctx))
743 		return omap_sham_update_dma_slow(dd);
744 
745 	dev_dbg(dd->dev, "fast: digcnt: %d, bufcnt: %u, total: %u\n",
746 			ctx->digcnt, ctx->bufcnt, ctx->total);
747 
748 	sg = ctx->sg;
749 	bs = get_block_size(ctx);
750 
751 	if (!SG_AA(sg))
752 		return omap_sham_update_dma_slow(dd);
753 
754 	if (!sg_is_last(sg) && !SG_SA(sg, bs))
755 		/* size is not BLOCK_SIZE aligned */
756 		return omap_sham_update_dma_slow(dd);
757 
758 	length = min(ctx->total, sg->length);
759 
760 	if (sg_is_last(sg)) {
761 		if (!(ctx->flags & BIT(FLAGS_FINUP))) {
762 			/* not last sg must be BLOCK_SIZE aligned */
763 			tail = length & (bs - 1);
764 			/* without finup() we need one block to close hash */
765 			if (!tail)
766 				tail = bs;
767 			length -= tail;
768 		}
769 	}
770 
771 	if (!dma_map_sg(dd->dev, ctx->sg, 1, DMA_TO_DEVICE)) {
772 		dev_err(dd->dev, "dma_map_sg  error\n");
773 		return -EINVAL;
774 	}
775 
776 	ctx->flags |= BIT(FLAGS_SG);
777 
778 	ctx->total -= length;
779 	ctx->offset = length; /* offset where to start slow */
780 
781 	final = (ctx->flags & BIT(FLAGS_FINUP)) && !ctx->total;
782 
783 	ret = omap_sham_xmit_dma(dd, sg_dma_address(ctx->sg), length, final, 1);
784 	if (ret != -EINPROGRESS)
785 		dma_unmap_sg(dd->dev, ctx->sg, 1, DMA_TO_DEVICE);
786 
787 	return ret;
788 }
789 
790 static int omap_sham_update_cpu(struct omap_sham_dev *dd)
791 {
792 	struct omap_sham_reqctx *ctx = ahash_request_ctx(dd->req);
793 	int bufcnt, final;
794 
795 	if (!ctx->total)
796 		return 0;
797 
798 	omap_sham_append_sg(ctx);
799 
800 	final = (ctx->flags & BIT(FLAGS_FINUP)) && !ctx->total;
801 
802 	dev_dbg(dd->dev, "cpu: bufcnt: %u, digcnt: %d, final: %d\n",
803 		ctx->bufcnt, ctx->digcnt, final);
804 
805 	if (final || (ctx->bufcnt == ctx->buflen && ctx->total)) {
806 		bufcnt = ctx->bufcnt;
807 		ctx->bufcnt = 0;
808 		return omap_sham_xmit_cpu(dd, ctx->buffer, bufcnt, final);
809 	}
810 
811 	return 0;
812 }
813 
814 static int omap_sham_update_dma_stop(struct omap_sham_dev *dd)
815 {
816 	struct omap_sham_reqctx *ctx = ahash_request_ctx(dd->req);
817 
818 	dmaengine_terminate_all(dd->dma_lch);
819 
820 	if (ctx->flags & BIT(FLAGS_SG)) {
821 		dma_unmap_sg(dd->dev, ctx->sg, 1, DMA_TO_DEVICE);
822 		if (ctx->sg->length == ctx->offset) {
823 			ctx->sg = sg_next(ctx->sg);
824 			if (ctx->sg)
825 				ctx->offset = 0;
826 		}
827 	} else {
828 		dma_unmap_single(dd->dev, ctx->dma_addr, ctx->buflen,
829 				 DMA_TO_DEVICE);
830 	}
831 
832 	return 0;
833 }
834 
835 static int omap_sham_init(struct ahash_request *req)
836 {
837 	struct crypto_ahash *tfm = crypto_ahash_reqtfm(req);
838 	struct omap_sham_ctx *tctx = crypto_ahash_ctx(tfm);
839 	struct omap_sham_reqctx *ctx = ahash_request_ctx(req);
840 	struct omap_sham_dev *dd = NULL, *tmp;
841 	int bs = 0;
842 
843 	spin_lock_bh(&sham.lock);
844 	if (!tctx->dd) {
845 		list_for_each_entry(tmp, &sham.dev_list, list) {
846 			dd = tmp;
847 			break;
848 		}
849 		tctx->dd = dd;
850 	} else {
851 		dd = tctx->dd;
852 	}
853 	spin_unlock_bh(&sham.lock);
854 
855 	ctx->dd = dd;
856 
857 	ctx->flags = 0;
858 
859 	dev_dbg(dd->dev, "init: digest size: %d\n",
860 		crypto_ahash_digestsize(tfm));
861 
862 	switch (crypto_ahash_digestsize(tfm)) {
863 	case MD5_DIGEST_SIZE:
864 		ctx->flags |= FLAGS_MODE_MD5;
865 		bs = SHA1_BLOCK_SIZE;
866 		break;
867 	case SHA1_DIGEST_SIZE:
868 		ctx->flags |= FLAGS_MODE_SHA1;
869 		bs = SHA1_BLOCK_SIZE;
870 		break;
871 	case SHA224_DIGEST_SIZE:
872 		ctx->flags |= FLAGS_MODE_SHA224;
873 		bs = SHA224_BLOCK_SIZE;
874 		break;
875 	case SHA256_DIGEST_SIZE:
876 		ctx->flags |= FLAGS_MODE_SHA256;
877 		bs = SHA256_BLOCK_SIZE;
878 		break;
879 	case SHA384_DIGEST_SIZE:
880 		ctx->flags |= FLAGS_MODE_SHA384;
881 		bs = SHA384_BLOCK_SIZE;
882 		break;
883 	case SHA512_DIGEST_SIZE:
884 		ctx->flags |= FLAGS_MODE_SHA512;
885 		bs = SHA512_BLOCK_SIZE;
886 		break;
887 	}
888 
889 	ctx->bufcnt = 0;
890 	ctx->digcnt = 0;
891 	ctx->buflen = BUFLEN;
892 
893 	if (tctx->flags & BIT(FLAGS_HMAC)) {
894 		if (!test_bit(FLAGS_AUTO_XOR, &dd->flags)) {
895 			struct omap_sham_hmac_ctx *bctx = tctx->base;
896 
897 			memcpy(ctx->buffer, bctx->ipad, bs);
898 			ctx->bufcnt = bs;
899 		}
900 
901 		ctx->flags |= BIT(FLAGS_HMAC);
902 	}
903 
904 	return 0;
905 
906 }
907 
908 static int omap_sham_update_req(struct omap_sham_dev *dd)
909 {
910 	struct ahash_request *req = dd->req;
911 	struct omap_sham_reqctx *ctx = ahash_request_ctx(req);
912 	int err;
913 
914 	dev_dbg(dd->dev, "update_req: total: %u, digcnt: %d, finup: %d\n",
915 		 ctx->total, ctx->digcnt, (ctx->flags & BIT(FLAGS_FINUP)) != 0);
916 
917 	if (ctx->flags & BIT(FLAGS_CPU))
918 		err = omap_sham_update_cpu(dd);
919 	else
920 		err = omap_sham_update_dma_start(dd);
921 
922 	/* wait for dma completion before can take more data */
923 	dev_dbg(dd->dev, "update: err: %d, digcnt: %d\n", err, ctx->digcnt);
924 
925 	return err;
926 }
927 
928 static int omap_sham_final_req(struct omap_sham_dev *dd)
929 {
930 	struct ahash_request *req = dd->req;
931 	struct omap_sham_reqctx *ctx = ahash_request_ctx(req);
932 	int err = 0, use_dma = 1;
933 
934 	if ((ctx->bufcnt <= get_block_size(ctx)) || dd->polling_mode)
935 		/*
936 		 * faster to handle last block with cpu or
937 		 * use cpu when dma is not present.
938 		 */
939 		use_dma = 0;
940 
941 	if (use_dma)
942 		err = omap_sham_xmit_dma_map(dd, ctx, ctx->bufcnt, 1);
943 	else
944 		err = omap_sham_xmit_cpu(dd, ctx->buffer, ctx->bufcnt, 1);
945 
946 	ctx->bufcnt = 0;
947 
948 	dev_dbg(dd->dev, "final_req: err: %d\n", err);
949 
950 	return err;
951 }
952 
953 static int omap_sham_finish_hmac(struct ahash_request *req)
954 {
955 	struct omap_sham_ctx *tctx = crypto_tfm_ctx(req->base.tfm);
956 	struct omap_sham_hmac_ctx *bctx = tctx->base;
957 	int bs = crypto_shash_blocksize(bctx->shash);
958 	int ds = crypto_shash_digestsize(bctx->shash);
959 	SHASH_DESC_ON_STACK(shash, bctx->shash);
960 
961 	shash->tfm = bctx->shash;
962 	shash->flags = 0; /* not CRYPTO_TFM_REQ_MAY_SLEEP */
963 
964 	return crypto_shash_init(shash) ?:
965 	       crypto_shash_update(shash, bctx->opad, bs) ?:
966 	       crypto_shash_finup(shash, req->result, ds, req->result);
967 }
968 
969 static int omap_sham_finish(struct ahash_request *req)
970 {
971 	struct omap_sham_reqctx *ctx = ahash_request_ctx(req);
972 	struct omap_sham_dev *dd = ctx->dd;
973 	int err = 0;
974 
975 	if (ctx->digcnt) {
976 		omap_sham_copy_ready_hash(req);
977 		if ((ctx->flags & BIT(FLAGS_HMAC)) &&
978 				!test_bit(FLAGS_AUTO_XOR, &dd->flags))
979 			err = omap_sham_finish_hmac(req);
980 	}
981 
982 	dev_dbg(dd->dev, "digcnt: %d, bufcnt: %d\n", ctx->digcnt, ctx->bufcnt);
983 
984 	return err;
985 }
986 
987 static void omap_sham_finish_req(struct ahash_request *req, int err)
988 {
989 	struct omap_sham_reqctx *ctx = ahash_request_ctx(req);
990 	struct omap_sham_dev *dd = ctx->dd;
991 
992 	if (!err) {
993 		dd->pdata->copy_hash(req, 1);
994 		if (test_bit(FLAGS_FINAL, &dd->flags))
995 			err = omap_sham_finish(req);
996 	} else {
997 		ctx->flags |= BIT(FLAGS_ERROR);
998 	}
999 
1000 	/* atomic operation is not needed here */
1001 	dd->flags &= ~(BIT(FLAGS_BUSY) | BIT(FLAGS_FINAL) | BIT(FLAGS_CPU) |
1002 			BIT(FLAGS_DMA_READY) | BIT(FLAGS_OUTPUT_READY));
1003 
1004 	pm_runtime_put(dd->dev);
1005 
1006 	if (req->base.complete)
1007 		req->base.complete(&req->base, err);
1008 
1009 	/* handle new request */
1010 	tasklet_schedule(&dd->done_task);
1011 }
1012 
1013 static int omap_sham_handle_queue(struct omap_sham_dev *dd,
1014 				  struct ahash_request *req)
1015 {
1016 	struct crypto_async_request *async_req, *backlog;
1017 	struct omap_sham_reqctx *ctx;
1018 	unsigned long flags;
1019 	int err = 0, ret = 0;
1020 
1021 	spin_lock_irqsave(&dd->lock, flags);
1022 	if (req)
1023 		ret = ahash_enqueue_request(&dd->queue, req);
1024 	if (test_bit(FLAGS_BUSY, &dd->flags)) {
1025 		spin_unlock_irqrestore(&dd->lock, flags);
1026 		return ret;
1027 	}
1028 	backlog = crypto_get_backlog(&dd->queue);
1029 	async_req = crypto_dequeue_request(&dd->queue);
1030 	if (async_req)
1031 		set_bit(FLAGS_BUSY, &dd->flags);
1032 	spin_unlock_irqrestore(&dd->lock, flags);
1033 
1034 	if (!async_req)
1035 		return ret;
1036 
1037 	if (backlog)
1038 		backlog->complete(backlog, -EINPROGRESS);
1039 
1040 	req = ahash_request_cast(async_req);
1041 	dd->req = req;
1042 	ctx = ahash_request_ctx(req);
1043 
1044 	dev_dbg(dd->dev, "handling new req, op: %lu, nbytes: %d\n",
1045 						ctx->op, req->nbytes);
1046 
1047 	err = omap_sham_hw_init(dd);
1048 	if (err)
1049 		goto err1;
1050 
1051 	if (ctx->digcnt)
1052 		/* request has changed - restore hash */
1053 		dd->pdata->copy_hash(req, 0);
1054 
1055 	if (ctx->op == OP_UPDATE) {
1056 		err = omap_sham_update_req(dd);
1057 		if (err != -EINPROGRESS && (ctx->flags & BIT(FLAGS_FINUP)))
1058 			/* no final() after finup() */
1059 			err = omap_sham_final_req(dd);
1060 	} else if (ctx->op == OP_FINAL) {
1061 		err = omap_sham_final_req(dd);
1062 	}
1063 err1:
1064 	if (err != -EINPROGRESS)
1065 		/* done_task will not finish it, so do it here */
1066 		omap_sham_finish_req(req, err);
1067 
1068 	dev_dbg(dd->dev, "exit, err: %d\n", err);
1069 
1070 	return ret;
1071 }
1072 
1073 static int omap_sham_enqueue(struct ahash_request *req, unsigned int op)
1074 {
1075 	struct omap_sham_reqctx *ctx = ahash_request_ctx(req);
1076 	struct omap_sham_ctx *tctx = crypto_tfm_ctx(req->base.tfm);
1077 	struct omap_sham_dev *dd = tctx->dd;
1078 
1079 	ctx->op = op;
1080 
1081 	return omap_sham_handle_queue(dd, req);
1082 }
1083 
1084 static int omap_sham_update(struct ahash_request *req)
1085 {
1086 	struct omap_sham_reqctx *ctx = ahash_request_ctx(req);
1087 	struct omap_sham_dev *dd = ctx->dd;
1088 	int bs = get_block_size(ctx);
1089 
1090 	if (!req->nbytes)
1091 		return 0;
1092 
1093 	ctx->total = req->nbytes;
1094 	ctx->sg = req->src;
1095 	ctx->offset = 0;
1096 
1097 	if (ctx->flags & BIT(FLAGS_FINUP)) {
1098 		if ((ctx->digcnt + ctx->bufcnt + ctx->total) < 9) {
1099 			/*
1100 			* OMAP HW accel works only with buffers >= 9
1101 			* will switch to bypass in final()
1102 			* final has the same request and data
1103 			*/
1104 			omap_sham_append_sg(ctx);
1105 			return 0;
1106 		} else if ((ctx->bufcnt + ctx->total <= bs) ||
1107 			   dd->polling_mode) {
1108 			/*
1109 			 * faster to use CPU for short transfers or
1110 			 * use cpu when dma is not present.
1111 			 */
1112 			ctx->flags |= BIT(FLAGS_CPU);
1113 		}
1114 	} else if (ctx->bufcnt + ctx->total < ctx->buflen) {
1115 		omap_sham_append_sg(ctx);
1116 		return 0;
1117 	}
1118 
1119 	if (dd->polling_mode)
1120 		ctx->flags |= BIT(FLAGS_CPU);
1121 
1122 	return omap_sham_enqueue(req, OP_UPDATE);
1123 }
1124 
1125 static int omap_sham_shash_digest(struct crypto_shash *tfm, u32 flags,
1126 				  const u8 *data, unsigned int len, u8 *out)
1127 {
1128 	SHASH_DESC_ON_STACK(shash, tfm);
1129 
1130 	shash->tfm = tfm;
1131 	shash->flags = flags & CRYPTO_TFM_REQ_MAY_SLEEP;
1132 
1133 	return crypto_shash_digest(shash, data, len, out);
1134 }
1135 
1136 static int omap_sham_final_shash(struct ahash_request *req)
1137 {
1138 	struct omap_sham_ctx *tctx = crypto_tfm_ctx(req->base.tfm);
1139 	struct omap_sham_reqctx *ctx = ahash_request_ctx(req);
1140 
1141 	return omap_sham_shash_digest(tctx->fallback, req->base.flags,
1142 				      ctx->buffer, ctx->bufcnt, req->result);
1143 }
1144 
1145 static int omap_sham_final(struct ahash_request *req)
1146 {
1147 	struct omap_sham_reqctx *ctx = ahash_request_ctx(req);
1148 
1149 	ctx->flags |= BIT(FLAGS_FINUP);
1150 
1151 	if (ctx->flags & BIT(FLAGS_ERROR))
1152 		return 0; /* uncompleted hash is not needed */
1153 
1154 	/* OMAP HW accel works only with buffers >= 9 */
1155 	/* HMAC is always >= 9 because ipad == block size */
1156 	if ((ctx->digcnt + ctx->bufcnt) < 9)
1157 		return omap_sham_final_shash(req);
1158 	else if (ctx->bufcnt)
1159 		return omap_sham_enqueue(req, OP_FINAL);
1160 
1161 	/* copy ready hash (+ finalize hmac) */
1162 	return omap_sham_finish(req);
1163 }
1164 
1165 static int omap_sham_finup(struct ahash_request *req)
1166 {
1167 	struct omap_sham_reqctx *ctx = ahash_request_ctx(req);
1168 	int err1, err2;
1169 
1170 	ctx->flags |= BIT(FLAGS_FINUP);
1171 
1172 	err1 = omap_sham_update(req);
1173 	if (err1 == -EINPROGRESS || err1 == -EBUSY)
1174 		return err1;
1175 	/*
1176 	 * final() has to be always called to cleanup resources
1177 	 * even if udpate() failed, except EINPROGRESS
1178 	 */
1179 	err2 = omap_sham_final(req);
1180 
1181 	return err1 ?: err2;
1182 }
1183 
1184 static int omap_sham_digest(struct ahash_request *req)
1185 {
1186 	return omap_sham_init(req) ?: omap_sham_finup(req);
1187 }
1188 
1189 static int omap_sham_setkey(struct crypto_ahash *tfm, const u8 *key,
1190 		      unsigned int keylen)
1191 {
1192 	struct omap_sham_ctx *tctx = crypto_ahash_ctx(tfm);
1193 	struct omap_sham_hmac_ctx *bctx = tctx->base;
1194 	int bs = crypto_shash_blocksize(bctx->shash);
1195 	int ds = crypto_shash_digestsize(bctx->shash);
1196 	struct omap_sham_dev *dd = NULL, *tmp;
1197 	int err, i;
1198 
1199 	spin_lock_bh(&sham.lock);
1200 	if (!tctx->dd) {
1201 		list_for_each_entry(tmp, &sham.dev_list, list) {
1202 			dd = tmp;
1203 			break;
1204 		}
1205 		tctx->dd = dd;
1206 	} else {
1207 		dd = tctx->dd;
1208 	}
1209 	spin_unlock_bh(&sham.lock);
1210 
1211 	err = crypto_shash_setkey(tctx->fallback, key, keylen);
1212 	if (err)
1213 		return err;
1214 
1215 	if (keylen > bs) {
1216 		err = omap_sham_shash_digest(bctx->shash,
1217 				crypto_shash_get_flags(bctx->shash),
1218 				key, keylen, bctx->ipad);
1219 		if (err)
1220 			return err;
1221 		keylen = ds;
1222 	} else {
1223 		memcpy(bctx->ipad, key, keylen);
1224 	}
1225 
1226 	memset(bctx->ipad + keylen, 0, bs - keylen);
1227 
1228 	if (!test_bit(FLAGS_AUTO_XOR, &dd->flags)) {
1229 		memcpy(bctx->opad, bctx->ipad, bs);
1230 
1231 		for (i = 0; i < bs; i++) {
1232 			bctx->ipad[i] ^= 0x36;
1233 			bctx->opad[i] ^= 0x5c;
1234 		}
1235 	}
1236 
1237 	return err;
1238 }
1239 
1240 static int omap_sham_cra_init_alg(struct crypto_tfm *tfm, const char *alg_base)
1241 {
1242 	struct omap_sham_ctx *tctx = crypto_tfm_ctx(tfm);
1243 	const char *alg_name = crypto_tfm_alg_name(tfm);
1244 
1245 	/* Allocate a fallback and abort if it failed. */
1246 	tctx->fallback = crypto_alloc_shash(alg_name, 0,
1247 					    CRYPTO_ALG_NEED_FALLBACK);
1248 	if (IS_ERR(tctx->fallback)) {
1249 		pr_err("omap-sham: fallback driver '%s' "
1250 				"could not be loaded.\n", alg_name);
1251 		return PTR_ERR(tctx->fallback);
1252 	}
1253 
1254 	crypto_ahash_set_reqsize(__crypto_ahash_cast(tfm),
1255 				 sizeof(struct omap_sham_reqctx) + BUFLEN);
1256 
1257 	if (alg_base) {
1258 		struct omap_sham_hmac_ctx *bctx = tctx->base;
1259 		tctx->flags |= BIT(FLAGS_HMAC);
1260 		bctx->shash = crypto_alloc_shash(alg_base, 0,
1261 						CRYPTO_ALG_NEED_FALLBACK);
1262 		if (IS_ERR(bctx->shash)) {
1263 			pr_err("omap-sham: base driver '%s' "
1264 					"could not be loaded.\n", alg_base);
1265 			crypto_free_shash(tctx->fallback);
1266 			return PTR_ERR(bctx->shash);
1267 		}
1268 
1269 	}
1270 
1271 	return 0;
1272 }
1273 
1274 static int omap_sham_cra_init(struct crypto_tfm *tfm)
1275 {
1276 	return omap_sham_cra_init_alg(tfm, NULL);
1277 }
1278 
1279 static int omap_sham_cra_sha1_init(struct crypto_tfm *tfm)
1280 {
1281 	return omap_sham_cra_init_alg(tfm, "sha1");
1282 }
1283 
1284 static int omap_sham_cra_sha224_init(struct crypto_tfm *tfm)
1285 {
1286 	return omap_sham_cra_init_alg(tfm, "sha224");
1287 }
1288 
1289 static int omap_sham_cra_sha256_init(struct crypto_tfm *tfm)
1290 {
1291 	return omap_sham_cra_init_alg(tfm, "sha256");
1292 }
1293 
1294 static int omap_sham_cra_md5_init(struct crypto_tfm *tfm)
1295 {
1296 	return omap_sham_cra_init_alg(tfm, "md5");
1297 }
1298 
1299 static int omap_sham_cra_sha384_init(struct crypto_tfm *tfm)
1300 {
1301 	return omap_sham_cra_init_alg(tfm, "sha384");
1302 }
1303 
1304 static int omap_sham_cra_sha512_init(struct crypto_tfm *tfm)
1305 {
1306 	return omap_sham_cra_init_alg(tfm, "sha512");
1307 }
1308 
1309 static void omap_sham_cra_exit(struct crypto_tfm *tfm)
1310 {
1311 	struct omap_sham_ctx *tctx = crypto_tfm_ctx(tfm);
1312 
1313 	crypto_free_shash(tctx->fallback);
1314 	tctx->fallback = NULL;
1315 
1316 	if (tctx->flags & BIT(FLAGS_HMAC)) {
1317 		struct omap_sham_hmac_ctx *bctx = tctx->base;
1318 		crypto_free_shash(bctx->shash);
1319 	}
1320 }
1321 
1322 static struct ahash_alg algs_sha1_md5[] = {
1323 {
1324 	.init		= omap_sham_init,
1325 	.update		= omap_sham_update,
1326 	.final		= omap_sham_final,
1327 	.finup		= omap_sham_finup,
1328 	.digest		= omap_sham_digest,
1329 	.halg.digestsize	= SHA1_DIGEST_SIZE,
1330 	.halg.base	= {
1331 		.cra_name		= "sha1",
1332 		.cra_driver_name	= "omap-sha1",
1333 		.cra_priority		= 100,
1334 		.cra_flags		= CRYPTO_ALG_TYPE_AHASH |
1335 						CRYPTO_ALG_KERN_DRIVER_ONLY |
1336 						CRYPTO_ALG_ASYNC |
1337 						CRYPTO_ALG_NEED_FALLBACK,
1338 		.cra_blocksize		= SHA1_BLOCK_SIZE,
1339 		.cra_ctxsize		= sizeof(struct omap_sham_ctx),
1340 		.cra_alignmask		= 0,
1341 		.cra_module		= THIS_MODULE,
1342 		.cra_init		= omap_sham_cra_init,
1343 		.cra_exit		= omap_sham_cra_exit,
1344 	}
1345 },
1346 {
1347 	.init		= omap_sham_init,
1348 	.update		= omap_sham_update,
1349 	.final		= omap_sham_final,
1350 	.finup		= omap_sham_finup,
1351 	.digest		= omap_sham_digest,
1352 	.halg.digestsize	= MD5_DIGEST_SIZE,
1353 	.halg.base	= {
1354 		.cra_name		= "md5",
1355 		.cra_driver_name	= "omap-md5",
1356 		.cra_priority		= 100,
1357 		.cra_flags		= CRYPTO_ALG_TYPE_AHASH |
1358 						CRYPTO_ALG_KERN_DRIVER_ONLY |
1359 						CRYPTO_ALG_ASYNC |
1360 						CRYPTO_ALG_NEED_FALLBACK,
1361 		.cra_blocksize		= SHA1_BLOCK_SIZE,
1362 		.cra_ctxsize		= sizeof(struct omap_sham_ctx),
1363 		.cra_alignmask		= OMAP_ALIGN_MASK,
1364 		.cra_module		= THIS_MODULE,
1365 		.cra_init		= omap_sham_cra_init,
1366 		.cra_exit		= omap_sham_cra_exit,
1367 	}
1368 },
1369 {
1370 	.init		= omap_sham_init,
1371 	.update		= omap_sham_update,
1372 	.final		= omap_sham_final,
1373 	.finup		= omap_sham_finup,
1374 	.digest		= omap_sham_digest,
1375 	.setkey		= omap_sham_setkey,
1376 	.halg.digestsize	= SHA1_DIGEST_SIZE,
1377 	.halg.base	= {
1378 		.cra_name		= "hmac(sha1)",
1379 		.cra_driver_name	= "omap-hmac-sha1",
1380 		.cra_priority		= 100,
1381 		.cra_flags		= CRYPTO_ALG_TYPE_AHASH |
1382 						CRYPTO_ALG_KERN_DRIVER_ONLY |
1383 						CRYPTO_ALG_ASYNC |
1384 						CRYPTO_ALG_NEED_FALLBACK,
1385 		.cra_blocksize		= SHA1_BLOCK_SIZE,
1386 		.cra_ctxsize		= sizeof(struct omap_sham_ctx) +
1387 					sizeof(struct omap_sham_hmac_ctx),
1388 		.cra_alignmask		= OMAP_ALIGN_MASK,
1389 		.cra_module		= THIS_MODULE,
1390 		.cra_init		= omap_sham_cra_sha1_init,
1391 		.cra_exit		= omap_sham_cra_exit,
1392 	}
1393 },
1394 {
1395 	.init		= omap_sham_init,
1396 	.update		= omap_sham_update,
1397 	.final		= omap_sham_final,
1398 	.finup		= omap_sham_finup,
1399 	.digest		= omap_sham_digest,
1400 	.setkey		= omap_sham_setkey,
1401 	.halg.digestsize	= MD5_DIGEST_SIZE,
1402 	.halg.base	= {
1403 		.cra_name		= "hmac(md5)",
1404 		.cra_driver_name	= "omap-hmac-md5",
1405 		.cra_priority		= 100,
1406 		.cra_flags		= CRYPTO_ALG_TYPE_AHASH |
1407 						CRYPTO_ALG_KERN_DRIVER_ONLY |
1408 						CRYPTO_ALG_ASYNC |
1409 						CRYPTO_ALG_NEED_FALLBACK,
1410 		.cra_blocksize		= SHA1_BLOCK_SIZE,
1411 		.cra_ctxsize		= sizeof(struct omap_sham_ctx) +
1412 					sizeof(struct omap_sham_hmac_ctx),
1413 		.cra_alignmask		= OMAP_ALIGN_MASK,
1414 		.cra_module		= THIS_MODULE,
1415 		.cra_init		= omap_sham_cra_md5_init,
1416 		.cra_exit		= omap_sham_cra_exit,
1417 	}
1418 }
1419 };
1420 
1421 /* OMAP4 has some algs in addition to what OMAP2 has */
1422 static struct ahash_alg algs_sha224_sha256[] = {
1423 {
1424 	.init		= omap_sham_init,
1425 	.update		= omap_sham_update,
1426 	.final		= omap_sham_final,
1427 	.finup		= omap_sham_finup,
1428 	.digest		= omap_sham_digest,
1429 	.halg.digestsize	= SHA224_DIGEST_SIZE,
1430 	.halg.base	= {
1431 		.cra_name		= "sha224",
1432 		.cra_driver_name	= "omap-sha224",
1433 		.cra_priority		= 100,
1434 		.cra_flags		= CRYPTO_ALG_TYPE_AHASH |
1435 						CRYPTO_ALG_ASYNC |
1436 						CRYPTO_ALG_NEED_FALLBACK,
1437 		.cra_blocksize		= SHA224_BLOCK_SIZE,
1438 		.cra_ctxsize		= sizeof(struct omap_sham_ctx),
1439 		.cra_alignmask		= 0,
1440 		.cra_module		= THIS_MODULE,
1441 		.cra_init		= omap_sham_cra_init,
1442 		.cra_exit		= omap_sham_cra_exit,
1443 	}
1444 },
1445 {
1446 	.init		= omap_sham_init,
1447 	.update		= omap_sham_update,
1448 	.final		= omap_sham_final,
1449 	.finup		= omap_sham_finup,
1450 	.digest		= omap_sham_digest,
1451 	.halg.digestsize	= SHA256_DIGEST_SIZE,
1452 	.halg.base	= {
1453 		.cra_name		= "sha256",
1454 		.cra_driver_name	= "omap-sha256",
1455 		.cra_priority		= 100,
1456 		.cra_flags		= CRYPTO_ALG_TYPE_AHASH |
1457 						CRYPTO_ALG_ASYNC |
1458 						CRYPTO_ALG_NEED_FALLBACK,
1459 		.cra_blocksize		= SHA256_BLOCK_SIZE,
1460 		.cra_ctxsize		= sizeof(struct omap_sham_ctx),
1461 		.cra_alignmask		= 0,
1462 		.cra_module		= THIS_MODULE,
1463 		.cra_init		= omap_sham_cra_init,
1464 		.cra_exit		= omap_sham_cra_exit,
1465 	}
1466 },
1467 {
1468 	.init		= omap_sham_init,
1469 	.update		= omap_sham_update,
1470 	.final		= omap_sham_final,
1471 	.finup		= omap_sham_finup,
1472 	.digest		= omap_sham_digest,
1473 	.setkey		= omap_sham_setkey,
1474 	.halg.digestsize	= SHA224_DIGEST_SIZE,
1475 	.halg.base	= {
1476 		.cra_name		= "hmac(sha224)",
1477 		.cra_driver_name	= "omap-hmac-sha224",
1478 		.cra_priority		= 100,
1479 		.cra_flags		= CRYPTO_ALG_TYPE_AHASH |
1480 						CRYPTO_ALG_ASYNC |
1481 						CRYPTO_ALG_NEED_FALLBACK,
1482 		.cra_blocksize		= SHA224_BLOCK_SIZE,
1483 		.cra_ctxsize		= sizeof(struct omap_sham_ctx) +
1484 					sizeof(struct omap_sham_hmac_ctx),
1485 		.cra_alignmask		= OMAP_ALIGN_MASK,
1486 		.cra_module		= THIS_MODULE,
1487 		.cra_init		= omap_sham_cra_sha224_init,
1488 		.cra_exit		= omap_sham_cra_exit,
1489 	}
1490 },
1491 {
1492 	.init		= omap_sham_init,
1493 	.update		= omap_sham_update,
1494 	.final		= omap_sham_final,
1495 	.finup		= omap_sham_finup,
1496 	.digest		= omap_sham_digest,
1497 	.setkey		= omap_sham_setkey,
1498 	.halg.digestsize	= SHA256_DIGEST_SIZE,
1499 	.halg.base	= {
1500 		.cra_name		= "hmac(sha256)",
1501 		.cra_driver_name	= "omap-hmac-sha256",
1502 		.cra_priority		= 100,
1503 		.cra_flags		= CRYPTO_ALG_TYPE_AHASH |
1504 						CRYPTO_ALG_ASYNC |
1505 						CRYPTO_ALG_NEED_FALLBACK,
1506 		.cra_blocksize		= SHA256_BLOCK_SIZE,
1507 		.cra_ctxsize		= sizeof(struct omap_sham_ctx) +
1508 					sizeof(struct omap_sham_hmac_ctx),
1509 		.cra_alignmask		= OMAP_ALIGN_MASK,
1510 		.cra_module		= THIS_MODULE,
1511 		.cra_init		= omap_sham_cra_sha256_init,
1512 		.cra_exit		= omap_sham_cra_exit,
1513 	}
1514 },
1515 };
1516 
1517 static struct ahash_alg algs_sha384_sha512[] = {
1518 {
1519 	.init		= omap_sham_init,
1520 	.update		= omap_sham_update,
1521 	.final		= omap_sham_final,
1522 	.finup		= omap_sham_finup,
1523 	.digest		= omap_sham_digest,
1524 	.halg.digestsize	= SHA384_DIGEST_SIZE,
1525 	.halg.base	= {
1526 		.cra_name		= "sha384",
1527 		.cra_driver_name	= "omap-sha384",
1528 		.cra_priority		= 100,
1529 		.cra_flags		= CRYPTO_ALG_TYPE_AHASH |
1530 						CRYPTO_ALG_ASYNC |
1531 						CRYPTO_ALG_NEED_FALLBACK,
1532 		.cra_blocksize		= SHA384_BLOCK_SIZE,
1533 		.cra_ctxsize		= sizeof(struct omap_sham_ctx),
1534 		.cra_alignmask		= 0,
1535 		.cra_module		= THIS_MODULE,
1536 		.cra_init		= omap_sham_cra_init,
1537 		.cra_exit		= omap_sham_cra_exit,
1538 	}
1539 },
1540 {
1541 	.init		= omap_sham_init,
1542 	.update		= omap_sham_update,
1543 	.final		= omap_sham_final,
1544 	.finup		= omap_sham_finup,
1545 	.digest		= omap_sham_digest,
1546 	.halg.digestsize	= SHA512_DIGEST_SIZE,
1547 	.halg.base	= {
1548 		.cra_name		= "sha512",
1549 		.cra_driver_name	= "omap-sha512",
1550 		.cra_priority		= 100,
1551 		.cra_flags		= CRYPTO_ALG_TYPE_AHASH |
1552 						CRYPTO_ALG_ASYNC |
1553 						CRYPTO_ALG_NEED_FALLBACK,
1554 		.cra_blocksize		= SHA512_BLOCK_SIZE,
1555 		.cra_ctxsize		= sizeof(struct omap_sham_ctx),
1556 		.cra_alignmask		= 0,
1557 		.cra_module		= THIS_MODULE,
1558 		.cra_init		= omap_sham_cra_init,
1559 		.cra_exit		= omap_sham_cra_exit,
1560 	}
1561 },
1562 {
1563 	.init		= omap_sham_init,
1564 	.update		= omap_sham_update,
1565 	.final		= omap_sham_final,
1566 	.finup		= omap_sham_finup,
1567 	.digest		= omap_sham_digest,
1568 	.setkey		= omap_sham_setkey,
1569 	.halg.digestsize	= SHA384_DIGEST_SIZE,
1570 	.halg.base	= {
1571 		.cra_name		= "hmac(sha384)",
1572 		.cra_driver_name	= "omap-hmac-sha384",
1573 		.cra_priority		= 100,
1574 		.cra_flags		= CRYPTO_ALG_TYPE_AHASH |
1575 						CRYPTO_ALG_ASYNC |
1576 						CRYPTO_ALG_NEED_FALLBACK,
1577 		.cra_blocksize		= SHA384_BLOCK_SIZE,
1578 		.cra_ctxsize		= sizeof(struct omap_sham_ctx) +
1579 					sizeof(struct omap_sham_hmac_ctx),
1580 		.cra_alignmask		= OMAP_ALIGN_MASK,
1581 		.cra_module		= THIS_MODULE,
1582 		.cra_init		= omap_sham_cra_sha384_init,
1583 		.cra_exit		= omap_sham_cra_exit,
1584 	}
1585 },
1586 {
1587 	.init		= omap_sham_init,
1588 	.update		= omap_sham_update,
1589 	.final		= omap_sham_final,
1590 	.finup		= omap_sham_finup,
1591 	.digest		= omap_sham_digest,
1592 	.setkey		= omap_sham_setkey,
1593 	.halg.digestsize	= SHA512_DIGEST_SIZE,
1594 	.halg.base	= {
1595 		.cra_name		= "hmac(sha512)",
1596 		.cra_driver_name	= "omap-hmac-sha512",
1597 		.cra_priority		= 100,
1598 		.cra_flags		= CRYPTO_ALG_TYPE_AHASH |
1599 						CRYPTO_ALG_ASYNC |
1600 						CRYPTO_ALG_NEED_FALLBACK,
1601 		.cra_blocksize		= SHA512_BLOCK_SIZE,
1602 		.cra_ctxsize		= sizeof(struct omap_sham_ctx) +
1603 					sizeof(struct omap_sham_hmac_ctx),
1604 		.cra_alignmask		= OMAP_ALIGN_MASK,
1605 		.cra_module		= THIS_MODULE,
1606 		.cra_init		= omap_sham_cra_sha512_init,
1607 		.cra_exit		= omap_sham_cra_exit,
1608 	}
1609 },
1610 };
1611 
1612 static void omap_sham_done_task(unsigned long data)
1613 {
1614 	struct omap_sham_dev *dd = (struct omap_sham_dev *)data;
1615 	int err = 0;
1616 
1617 	if (!test_bit(FLAGS_BUSY, &dd->flags)) {
1618 		omap_sham_handle_queue(dd, NULL);
1619 		return;
1620 	}
1621 
1622 	if (test_bit(FLAGS_CPU, &dd->flags)) {
1623 		if (test_and_clear_bit(FLAGS_OUTPUT_READY, &dd->flags)) {
1624 			/* hash or semi-hash ready */
1625 			err = omap_sham_update_cpu(dd);
1626 			if (err != -EINPROGRESS)
1627 				goto finish;
1628 		}
1629 	} else if (test_bit(FLAGS_DMA_READY, &dd->flags)) {
1630 		if (test_and_clear_bit(FLAGS_DMA_ACTIVE, &dd->flags)) {
1631 			omap_sham_update_dma_stop(dd);
1632 			if (dd->err) {
1633 				err = dd->err;
1634 				goto finish;
1635 			}
1636 		}
1637 		if (test_and_clear_bit(FLAGS_OUTPUT_READY, &dd->flags)) {
1638 			/* hash or semi-hash ready */
1639 			clear_bit(FLAGS_DMA_READY, &dd->flags);
1640 			err = omap_sham_update_dma_start(dd);
1641 			if (err != -EINPROGRESS)
1642 				goto finish;
1643 		}
1644 	}
1645 
1646 	return;
1647 
1648 finish:
1649 	dev_dbg(dd->dev, "update done: err: %d\n", err);
1650 	/* finish curent request */
1651 	omap_sham_finish_req(dd->req, err);
1652 }
1653 
1654 static irqreturn_t omap_sham_irq_common(struct omap_sham_dev *dd)
1655 {
1656 	if (!test_bit(FLAGS_BUSY, &dd->flags)) {
1657 		dev_warn(dd->dev, "Interrupt when no active requests.\n");
1658 	} else {
1659 		set_bit(FLAGS_OUTPUT_READY, &dd->flags);
1660 		tasklet_schedule(&dd->done_task);
1661 	}
1662 
1663 	return IRQ_HANDLED;
1664 }
1665 
1666 static irqreturn_t omap_sham_irq_omap2(int irq, void *dev_id)
1667 {
1668 	struct omap_sham_dev *dd = dev_id;
1669 
1670 	if (unlikely(test_bit(FLAGS_FINAL, &dd->flags)))
1671 		/* final -> allow device to go to power-saving mode */
1672 		omap_sham_write_mask(dd, SHA_REG_CTRL, 0, SHA_REG_CTRL_LENGTH);
1673 
1674 	omap_sham_write_mask(dd, SHA_REG_CTRL, SHA_REG_CTRL_OUTPUT_READY,
1675 				 SHA_REG_CTRL_OUTPUT_READY);
1676 	omap_sham_read(dd, SHA_REG_CTRL);
1677 
1678 	return omap_sham_irq_common(dd);
1679 }
1680 
1681 static irqreturn_t omap_sham_irq_omap4(int irq, void *dev_id)
1682 {
1683 	struct omap_sham_dev *dd = dev_id;
1684 
1685 	omap_sham_write_mask(dd, SHA_REG_MASK(dd), 0, SHA_REG_MASK_IT_EN);
1686 
1687 	return omap_sham_irq_common(dd);
1688 }
1689 
1690 static struct omap_sham_algs_info omap_sham_algs_info_omap2[] = {
1691 	{
1692 		.algs_list	= algs_sha1_md5,
1693 		.size		= ARRAY_SIZE(algs_sha1_md5),
1694 	},
1695 };
1696 
1697 static const struct omap_sham_pdata omap_sham_pdata_omap2 = {
1698 	.algs_info	= omap_sham_algs_info_omap2,
1699 	.algs_info_size	= ARRAY_SIZE(omap_sham_algs_info_omap2),
1700 	.flags		= BIT(FLAGS_BE32_SHA1),
1701 	.digest_size	= SHA1_DIGEST_SIZE,
1702 	.copy_hash	= omap_sham_copy_hash_omap2,
1703 	.write_ctrl	= omap_sham_write_ctrl_omap2,
1704 	.trigger	= omap_sham_trigger_omap2,
1705 	.poll_irq	= omap_sham_poll_irq_omap2,
1706 	.intr_hdlr	= omap_sham_irq_omap2,
1707 	.idigest_ofs	= 0x00,
1708 	.din_ofs	= 0x1c,
1709 	.digcnt_ofs	= 0x14,
1710 	.rev_ofs	= 0x5c,
1711 	.mask_ofs	= 0x60,
1712 	.sysstatus_ofs	= 0x64,
1713 	.major_mask	= 0xf0,
1714 	.major_shift	= 4,
1715 	.minor_mask	= 0x0f,
1716 	.minor_shift	= 0,
1717 };
1718 
1719 #ifdef CONFIG_OF
1720 static struct omap_sham_algs_info omap_sham_algs_info_omap4[] = {
1721 	{
1722 		.algs_list	= algs_sha1_md5,
1723 		.size		= ARRAY_SIZE(algs_sha1_md5),
1724 	},
1725 	{
1726 		.algs_list	= algs_sha224_sha256,
1727 		.size		= ARRAY_SIZE(algs_sha224_sha256),
1728 	},
1729 };
1730 
1731 static const struct omap_sham_pdata omap_sham_pdata_omap4 = {
1732 	.algs_info	= omap_sham_algs_info_omap4,
1733 	.algs_info_size	= ARRAY_SIZE(omap_sham_algs_info_omap4),
1734 	.flags		= BIT(FLAGS_AUTO_XOR),
1735 	.digest_size	= SHA256_DIGEST_SIZE,
1736 	.copy_hash	= omap_sham_copy_hash_omap4,
1737 	.write_ctrl	= omap_sham_write_ctrl_omap4,
1738 	.trigger	= omap_sham_trigger_omap4,
1739 	.poll_irq	= omap_sham_poll_irq_omap4,
1740 	.intr_hdlr	= omap_sham_irq_omap4,
1741 	.idigest_ofs	= 0x020,
1742 	.odigest_ofs	= 0x0,
1743 	.din_ofs	= 0x080,
1744 	.digcnt_ofs	= 0x040,
1745 	.rev_ofs	= 0x100,
1746 	.mask_ofs	= 0x110,
1747 	.sysstatus_ofs	= 0x114,
1748 	.mode_ofs	= 0x44,
1749 	.length_ofs	= 0x48,
1750 	.major_mask	= 0x0700,
1751 	.major_shift	= 8,
1752 	.minor_mask	= 0x003f,
1753 	.minor_shift	= 0,
1754 };
1755 
1756 static struct omap_sham_algs_info omap_sham_algs_info_omap5[] = {
1757 	{
1758 		.algs_list	= algs_sha1_md5,
1759 		.size		= ARRAY_SIZE(algs_sha1_md5),
1760 	},
1761 	{
1762 		.algs_list	= algs_sha224_sha256,
1763 		.size		= ARRAY_SIZE(algs_sha224_sha256),
1764 	},
1765 	{
1766 		.algs_list	= algs_sha384_sha512,
1767 		.size		= ARRAY_SIZE(algs_sha384_sha512),
1768 	},
1769 };
1770 
1771 static const struct omap_sham_pdata omap_sham_pdata_omap5 = {
1772 	.algs_info	= omap_sham_algs_info_omap5,
1773 	.algs_info_size	= ARRAY_SIZE(omap_sham_algs_info_omap5),
1774 	.flags		= BIT(FLAGS_AUTO_XOR),
1775 	.digest_size	= SHA512_DIGEST_SIZE,
1776 	.copy_hash	= omap_sham_copy_hash_omap4,
1777 	.write_ctrl	= omap_sham_write_ctrl_omap4,
1778 	.trigger	= omap_sham_trigger_omap4,
1779 	.poll_irq	= omap_sham_poll_irq_omap4,
1780 	.intr_hdlr	= omap_sham_irq_omap4,
1781 	.idigest_ofs	= 0x240,
1782 	.odigest_ofs	= 0x200,
1783 	.din_ofs	= 0x080,
1784 	.digcnt_ofs	= 0x280,
1785 	.rev_ofs	= 0x100,
1786 	.mask_ofs	= 0x110,
1787 	.sysstatus_ofs	= 0x114,
1788 	.mode_ofs	= 0x284,
1789 	.length_ofs	= 0x288,
1790 	.major_mask	= 0x0700,
1791 	.major_shift	= 8,
1792 	.minor_mask	= 0x003f,
1793 	.minor_shift	= 0,
1794 };
1795 
1796 static const struct of_device_id omap_sham_of_match[] = {
1797 	{
1798 		.compatible	= "ti,omap2-sham",
1799 		.data		= &omap_sham_pdata_omap2,
1800 	},
1801 	{
1802 		.compatible	= "ti,omap3-sham",
1803 		.data		= &omap_sham_pdata_omap2,
1804 	},
1805 	{
1806 		.compatible	= "ti,omap4-sham",
1807 		.data		= &omap_sham_pdata_omap4,
1808 	},
1809 	{
1810 		.compatible	= "ti,omap5-sham",
1811 		.data		= &omap_sham_pdata_omap5,
1812 	},
1813 	{},
1814 };
1815 MODULE_DEVICE_TABLE(of, omap_sham_of_match);
1816 
1817 static int omap_sham_get_res_of(struct omap_sham_dev *dd,
1818 		struct device *dev, struct resource *res)
1819 {
1820 	struct device_node *node = dev->of_node;
1821 	const struct of_device_id *match;
1822 	int err = 0;
1823 
1824 	match = of_match_device(of_match_ptr(omap_sham_of_match), dev);
1825 	if (!match) {
1826 		dev_err(dev, "no compatible OF match\n");
1827 		err = -EINVAL;
1828 		goto err;
1829 	}
1830 
1831 	err = of_address_to_resource(node, 0, res);
1832 	if (err < 0) {
1833 		dev_err(dev, "can't translate OF node address\n");
1834 		err = -EINVAL;
1835 		goto err;
1836 	}
1837 
1838 	dd->irq = irq_of_parse_and_map(node, 0);
1839 	if (!dd->irq) {
1840 		dev_err(dev, "can't translate OF irq value\n");
1841 		err = -EINVAL;
1842 		goto err;
1843 	}
1844 
1845 	dd->dma = -1; /* Dummy value that's unused */
1846 	dd->pdata = match->data;
1847 
1848 err:
1849 	return err;
1850 }
1851 #else
1852 static const struct of_device_id omap_sham_of_match[] = {
1853 	{},
1854 };
1855 
1856 static int omap_sham_get_res_of(struct omap_sham_dev *dd,
1857 		struct device *dev, struct resource *res)
1858 {
1859 	return -EINVAL;
1860 }
1861 #endif
1862 
1863 static int omap_sham_get_res_pdev(struct omap_sham_dev *dd,
1864 		struct platform_device *pdev, struct resource *res)
1865 {
1866 	struct device *dev = &pdev->dev;
1867 	struct resource *r;
1868 	int err = 0;
1869 
1870 	/* Get the base address */
1871 	r = platform_get_resource(pdev, IORESOURCE_MEM, 0);
1872 	if (!r) {
1873 		dev_err(dev, "no MEM resource info\n");
1874 		err = -ENODEV;
1875 		goto err;
1876 	}
1877 	memcpy(res, r, sizeof(*res));
1878 
1879 	/* Get the IRQ */
1880 	dd->irq = platform_get_irq(pdev, 0);
1881 	if (dd->irq < 0) {
1882 		dev_err(dev, "no IRQ resource info\n");
1883 		err = dd->irq;
1884 		goto err;
1885 	}
1886 
1887 	/* Get the DMA */
1888 	r = platform_get_resource(pdev, IORESOURCE_DMA, 0);
1889 	if (!r) {
1890 		dev_err(dev, "no DMA resource info\n");
1891 		err = -ENODEV;
1892 		goto err;
1893 	}
1894 	dd->dma = r->start;
1895 
1896 	/* Only OMAP2/3 can be non-DT */
1897 	dd->pdata = &omap_sham_pdata_omap2;
1898 
1899 err:
1900 	return err;
1901 }
1902 
1903 static int omap_sham_probe(struct platform_device *pdev)
1904 {
1905 	struct omap_sham_dev *dd;
1906 	struct device *dev = &pdev->dev;
1907 	struct resource res;
1908 	dma_cap_mask_t mask;
1909 	int err, i, j;
1910 	u32 rev;
1911 
1912 	dd = devm_kzalloc(dev, sizeof(struct omap_sham_dev), GFP_KERNEL);
1913 	if (dd == NULL) {
1914 		dev_err(dev, "unable to alloc data struct.\n");
1915 		err = -ENOMEM;
1916 		goto data_err;
1917 	}
1918 	dd->dev = dev;
1919 	platform_set_drvdata(pdev, dd);
1920 
1921 	INIT_LIST_HEAD(&dd->list);
1922 	spin_lock_init(&dd->lock);
1923 	tasklet_init(&dd->done_task, omap_sham_done_task, (unsigned long)dd);
1924 	crypto_init_queue(&dd->queue, OMAP_SHAM_QUEUE_LENGTH);
1925 
1926 	err = (dev->of_node) ? omap_sham_get_res_of(dd, dev, &res) :
1927 			       omap_sham_get_res_pdev(dd, pdev, &res);
1928 	if (err)
1929 		goto data_err;
1930 
1931 	dd->io_base = devm_ioremap_resource(dev, &res);
1932 	if (IS_ERR(dd->io_base)) {
1933 		err = PTR_ERR(dd->io_base);
1934 		goto data_err;
1935 	}
1936 	dd->phys_base = res.start;
1937 
1938 	err = devm_request_irq(dev, dd->irq, dd->pdata->intr_hdlr,
1939 			       IRQF_TRIGGER_NONE, dev_name(dev), dd);
1940 	if (err) {
1941 		dev_err(dev, "unable to request irq %d, err = %d\n",
1942 			dd->irq, err);
1943 		goto data_err;
1944 	}
1945 
1946 	dma_cap_zero(mask);
1947 	dma_cap_set(DMA_SLAVE, mask);
1948 
1949 	dd->dma_lch = dma_request_slave_channel_compat(mask, omap_dma_filter_fn,
1950 						       &dd->dma, dev, "rx");
1951 	if (!dd->dma_lch) {
1952 		dd->polling_mode = 1;
1953 		dev_dbg(dev, "using polling mode instead of dma\n");
1954 	}
1955 
1956 	dd->flags |= dd->pdata->flags;
1957 
1958 	pm_runtime_enable(dev);
1959 	pm_runtime_irq_safe(dev);
1960 
1961 	err = pm_runtime_get_sync(dev);
1962 	if (err < 0) {
1963 		dev_err(dev, "failed to get sync: %d\n", err);
1964 		goto err_pm;
1965 	}
1966 
1967 	rev = omap_sham_read(dd, SHA_REG_REV(dd));
1968 	pm_runtime_put_sync(&pdev->dev);
1969 
1970 	dev_info(dev, "hw accel on OMAP rev %u.%u\n",
1971 		(rev & dd->pdata->major_mask) >> dd->pdata->major_shift,
1972 		(rev & dd->pdata->minor_mask) >> dd->pdata->minor_shift);
1973 
1974 	spin_lock(&sham.lock);
1975 	list_add_tail(&dd->list, &sham.dev_list);
1976 	spin_unlock(&sham.lock);
1977 
1978 	for (i = 0; i < dd->pdata->algs_info_size; i++) {
1979 		for (j = 0; j < dd->pdata->algs_info[i].size; j++) {
1980 			err = crypto_register_ahash(
1981 					&dd->pdata->algs_info[i].algs_list[j]);
1982 			if (err)
1983 				goto err_algs;
1984 
1985 			dd->pdata->algs_info[i].registered++;
1986 		}
1987 	}
1988 
1989 	return 0;
1990 
1991 err_algs:
1992 	for (i = dd->pdata->algs_info_size - 1; i >= 0; i--)
1993 		for (j = dd->pdata->algs_info[i].registered - 1; j >= 0; j--)
1994 			crypto_unregister_ahash(
1995 					&dd->pdata->algs_info[i].algs_list[j]);
1996 err_pm:
1997 	pm_runtime_disable(dev);
1998 	if (dd->dma_lch)
1999 		dma_release_channel(dd->dma_lch);
2000 data_err:
2001 	dev_err(dev, "initialization failed.\n");
2002 
2003 	return err;
2004 }
2005 
2006 static int omap_sham_remove(struct platform_device *pdev)
2007 {
2008 	static struct omap_sham_dev *dd;
2009 	int i, j;
2010 
2011 	dd = platform_get_drvdata(pdev);
2012 	if (!dd)
2013 		return -ENODEV;
2014 	spin_lock(&sham.lock);
2015 	list_del(&dd->list);
2016 	spin_unlock(&sham.lock);
2017 	for (i = dd->pdata->algs_info_size - 1; i >= 0; i--)
2018 		for (j = dd->pdata->algs_info[i].registered - 1; j >= 0; j--)
2019 			crypto_unregister_ahash(
2020 					&dd->pdata->algs_info[i].algs_list[j]);
2021 	tasklet_kill(&dd->done_task);
2022 	pm_runtime_disable(&pdev->dev);
2023 
2024 	if (dd->dma_lch)
2025 		dma_release_channel(dd->dma_lch);
2026 
2027 	return 0;
2028 }
2029 
2030 #ifdef CONFIG_PM_SLEEP
2031 static int omap_sham_suspend(struct device *dev)
2032 {
2033 	pm_runtime_put_sync(dev);
2034 	return 0;
2035 }
2036 
2037 static int omap_sham_resume(struct device *dev)
2038 {
2039 	int err = pm_runtime_get_sync(dev);
2040 	if (err < 0) {
2041 		dev_err(dev, "failed to get sync: %d\n", err);
2042 		return err;
2043 	}
2044 	return 0;
2045 }
2046 #endif
2047 
2048 static SIMPLE_DEV_PM_OPS(omap_sham_pm_ops, omap_sham_suspend, omap_sham_resume);
2049 
2050 static struct platform_driver omap_sham_driver = {
2051 	.probe	= omap_sham_probe,
2052 	.remove	= omap_sham_remove,
2053 	.driver	= {
2054 		.name	= "omap-sham",
2055 		.pm	= &omap_sham_pm_ops,
2056 		.of_match_table	= omap_sham_of_match,
2057 	},
2058 };
2059 
2060 module_platform_driver(omap_sham_driver);
2061 
2062 MODULE_DESCRIPTION("OMAP SHA1/MD5 hw acceleration support.");
2063 MODULE_LICENSE("GPL v2");
2064 MODULE_AUTHOR("Dmitry Kasatkin");
2065 MODULE_ALIAS("platform:omap-sham");
2066