xref: /openbmc/linux/drivers/crypto/img-hash.c (revision 2c684d89)
1 /*
2  * Copyright (c) 2014 Imagination Technologies
3  * Authors:  Will Thomas, James Hartley
4  *
5  * This program is free software; you can redistribute it and/or modify
6  * it under the terms of the GNU General Public License version 2 as published
7  * by the Free Software Foundation.
8  *
9  *	Interface structure taken from omap-sham driver
10  */
11 
12 #include <linux/clk.h>
13 #include <linux/dmaengine.h>
14 #include <linux/interrupt.h>
15 #include <linux/io.h>
16 #include <linux/kernel.h>
17 #include <linux/module.h>
18 #include <linux/of_device.h>
19 #include <linux/platform_device.h>
20 #include <linux/scatterlist.h>
21 
22 #include <crypto/internal/hash.h>
23 #include <crypto/md5.h>
24 #include <crypto/sha.h>
25 
26 #define CR_RESET			0
27 #define CR_RESET_SET			1
28 #define CR_RESET_UNSET			0
29 
30 #define CR_MESSAGE_LENGTH_H		0x4
31 #define CR_MESSAGE_LENGTH_L		0x8
32 
33 #define CR_CONTROL			0xc
34 #define CR_CONTROL_BYTE_ORDER_3210	0
35 #define CR_CONTROL_BYTE_ORDER_0123	1
36 #define CR_CONTROL_BYTE_ORDER_2310	2
37 #define CR_CONTROL_BYTE_ORDER_1032	3
38 #define CR_CONTROL_BYTE_ORDER_SHIFT	8
39 #define CR_CONTROL_ALGO_MD5	0
40 #define CR_CONTROL_ALGO_SHA1	1
41 #define CR_CONTROL_ALGO_SHA224	2
42 #define CR_CONTROL_ALGO_SHA256	3
43 
44 #define CR_INTSTAT			0x10
45 #define CR_INTENAB			0x14
46 #define CR_INTCLEAR			0x18
47 #define CR_INT_RESULTS_AVAILABLE	BIT(0)
48 #define CR_INT_NEW_RESULTS_SET		BIT(1)
49 #define CR_INT_RESULT_READ_ERR		BIT(2)
50 #define CR_INT_MESSAGE_WRITE_ERROR	BIT(3)
51 #define CR_INT_STATUS			BIT(8)
52 
53 #define CR_RESULT_QUEUE		0x1c
54 #define CR_RSD0				0x40
55 #define CR_CORE_REV			0x50
56 #define CR_CORE_DES1		0x60
57 #define CR_CORE_DES2		0x70
58 
59 #define DRIVER_FLAGS_BUSY		BIT(0)
60 #define DRIVER_FLAGS_FINAL		BIT(1)
61 #define DRIVER_FLAGS_DMA_ACTIVE		BIT(2)
62 #define DRIVER_FLAGS_OUTPUT_READY	BIT(3)
63 #define DRIVER_FLAGS_INIT		BIT(4)
64 #define DRIVER_FLAGS_CPU		BIT(5)
65 #define DRIVER_FLAGS_DMA_READY		BIT(6)
66 #define DRIVER_FLAGS_ERROR		BIT(7)
67 #define DRIVER_FLAGS_SG			BIT(8)
68 #define DRIVER_FLAGS_SHA1		BIT(18)
69 #define DRIVER_FLAGS_SHA224		BIT(19)
70 #define DRIVER_FLAGS_SHA256		BIT(20)
71 #define DRIVER_FLAGS_MD5		BIT(21)
72 
73 #define IMG_HASH_QUEUE_LENGTH		20
74 #define IMG_HASH_DMA_THRESHOLD		64
75 
76 #ifdef __LITTLE_ENDIAN
77 #define IMG_HASH_BYTE_ORDER		CR_CONTROL_BYTE_ORDER_3210
78 #else
79 #define IMG_HASH_BYTE_ORDER		CR_CONTROL_BYTE_ORDER_0123
80 #endif
81 
82 struct img_hash_dev;
83 
84 struct img_hash_request_ctx {
85 	struct img_hash_dev	*hdev;
86 	u8 digest[SHA256_DIGEST_SIZE] __aligned(sizeof(u32));
87 	unsigned long		flags;
88 	size_t			digsize;
89 
90 	dma_addr_t		dma_addr;
91 	size_t			dma_ct;
92 
93 	/* sg root */
94 	struct scatterlist	*sgfirst;
95 	/* walk state */
96 	struct scatterlist	*sg;
97 	size_t			nents;
98 	size_t			offset;
99 	unsigned int		total;
100 	size_t			sent;
101 
102 	unsigned long		op;
103 
104 	size_t			bufcnt;
105 	u8 buffer[0] __aligned(sizeof(u32));
106 	struct ahash_request	fallback_req;
107 };
108 
109 struct img_hash_ctx {
110 	struct img_hash_dev	*hdev;
111 	unsigned long		flags;
112 	struct crypto_ahash	*fallback;
113 };
114 
115 struct img_hash_dev {
116 	struct list_head	list;
117 	struct device		*dev;
118 	struct clk		*hash_clk;
119 	struct clk		*sys_clk;
120 	void __iomem		*io_base;
121 
122 	phys_addr_t		bus_addr;
123 	void __iomem		*cpu_addr;
124 
125 	spinlock_t		lock;
126 	int			err;
127 	struct tasklet_struct	done_task;
128 	struct tasklet_struct	dma_task;
129 
130 	unsigned long		flags;
131 	struct crypto_queue	queue;
132 	struct ahash_request	*req;
133 
134 	struct dma_chan		*dma_lch;
135 };
136 
137 struct img_hash_drv {
138 	struct list_head dev_list;
139 	spinlock_t lock;
140 };
141 
142 static struct img_hash_drv img_hash = {
143 	.dev_list = LIST_HEAD_INIT(img_hash.dev_list),
144 	.lock = __SPIN_LOCK_UNLOCKED(img_hash.lock),
145 };
146 
147 static inline u32 img_hash_read(struct img_hash_dev *hdev, u32 offset)
148 {
149 	return readl_relaxed(hdev->io_base + offset);
150 }
151 
152 static inline void img_hash_write(struct img_hash_dev *hdev,
153 				  u32 offset, u32 value)
154 {
155 	writel_relaxed(value, hdev->io_base + offset);
156 }
157 
158 static inline u32 img_hash_read_result_queue(struct img_hash_dev *hdev)
159 {
160 	return be32_to_cpu(img_hash_read(hdev, CR_RESULT_QUEUE));
161 }
162 
163 static void img_hash_start(struct img_hash_dev *hdev, bool dma)
164 {
165 	struct img_hash_request_ctx *ctx = ahash_request_ctx(hdev->req);
166 	u32 cr = IMG_HASH_BYTE_ORDER << CR_CONTROL_BYTE_ORDER_SHIFT;
167 
168 	if (ctx->flags & DRIVER_FLAGS_MD5)
169 		cr |= CR_CONTROL_ALGO_MD5;
170 	else if (ctx->flags & DRIVER_FLAGS_SHA1)
171 		cr |= CR_CONTROL_ALGO_SHA1;
172 	else if (ctx->flags & DRIVER_FLAGS_SHA224)
173 		cr |= CR_CONTROL_ALGO_SHA224;
174 	else if (ctx->flags & DRIVER_FLAGS_SHA256)
175 		cr |= CR_CONTROL_ALGO_SHA256;
176 	dev_dbg(hdev->dev, "Starting hash process\n");
177 	img_hash_write(hdev, CR_CONTROL, cr);
178 
179 	/*
180 	 * The hardware block requires two cycles between writing the control
181 	 * register and writing the first word of data in non DMA mode, to
182 	 * ensure the first data write is not grouped in burst with the control
183 	 * register write a read is issued to 'flush' the bus.
184 	 */
185 	if (!dma)
186 		img_hash_read(hdev, CR_CONTROL);
187 }
188 
189 static int img_hash_xmit_cpu(struct img_hash_dev *hdev, const u8 *buf,
190 			     size_t length, int final)
191 {
192 	u32 count, len32;
193 	const u32 *buffer = (const u32 *)buf;
194 
195 	dev_dbg(hdev->dev, "xmit_cpu:  length: %zu bytes\n", length);
196 
197 	if (final)
198 		hdev->flags |= DRIVER_FLAGS_FINAL;
199 
200 	len32 = DIV_ROUND_UP(length, sizeof(u32));
201 
202 	for (count = 0; count < len32; count++)
203 		writel_relaxed(buffer[count], hdev->cpu_addr);
204 
205 	return -EINPROGRESS;
206 }
207 
208 static void img_hash_dma_callback(void *data)
209 {
210 	struct img_hash_dev *hdev = (struct img_hash_dev *)data;
211 	struct img_hash_request_ctx *ctx = ahash_request_ctx(hdev->req);
212 
213 	if (ctx->bufcnt) {
214 		img_hash_xmit_cpu(hdev, ctx->buffer, ctx->bufcnt, 0);
215 		ctx->bufcnt = 0;
216 	}
217 	if (ctx->sg)
218 		tasklet_schedule(&hdev->dma_task);
219 }
220 
221 static int img_hash_xmit_dma(struct img_hash_dev *hdev, struct scatterlist *sg)
222 {
223 	struct dma_async_tx_descriptor *desc;
224 	struct img_hash_request_ctx *ctx = ahash_request_ctx(hdev->req);
225 
226 	ctx->dma_ct = dma_map_sg(hdev->dev, sg, 1, DMA_MEM_TO_DEV);
227 	if (ctx->dma_ct == 0) {
228 		dev_err(hdev->dev, "Invalid DMA sg\n");
229 		hdev->err = -EINVAL;
230 		return -EINVAL;
231 	}
232 
233 	desc = dmaengine_prep_slave_sg(hdev->dma_lch,
234 				       sg,
235 				       ctx->dma_ct,
236 				       DMA_MEM_TO_DEV,
237 				       DMA_PREP_INTERRUPT | DMA_CTRL_ACK);
238 	if (!desc) {
239 		dev_err(hdev->dev, "Null DMA descriptor\n");
240 		hdev->err = -EINVAL;
241 		dma_unmap_sg(hdev->dev, sg, 1, DMA_MEM_TO_DEV);
242 		return -EINVAL;
243 	}
244 	desc->callback = img_hash_dma_callback;
245 	desc->callback_param = hdev;
246 	dmaengine_submit(desc);
247 	dma_async_issue_pending(hdev->dma_lch);
248 
249 	return 0;
250 }
251 
252 static int img_hash_write_via_cpu(struct img_hash_dev *hdev)
253 {
254 	struct img_hash_request_ctx *ctx = ahash_request_ctx(hdev->req);
255 
256 	ctx->bufcnt = sg_copy_to_buffer(hdev->req->src, sg_nents(ctx->sg),
257 					ctx->buffer, hdev->req->nbytes);
258 
259 	ctx->total = hdev->req->nbytes;
260 	ctx->bufcnt = 0;
261 
262 	hdev->flags |= (DRIVER_FLAGS_CPU | DRIVER_FLAGS_FINAL);
263 
264 	img_hash_start(hdev, false);
265 
266 	return img_hash_xmit_cpu(hdev, ctx->buffer, ctx->total, 1);
267 }
268 
269 static int img_hash_finish(struct ahash_request *req)
270 {
271 	struct img_hash_request_ctx *ctx = ahash_request_ctx(req);
272 
273 	if (!req->result)
274 		return -EINVAL;
275 
276 	memcpy(req->result, ctx->digest, ctx->digsize);
277 
278 	return 0;
279 }
280 
281 static void img_hash_copy_hash(struct ahash_request *req)
282 {
283 	struct img_hash_request_ctx *ctx = ahash_request_ctx(req);
284 	u32 *hash = (u32 *)ctx->digest;
285 	int i;
286 
287 	for (i = (ctx->digsize / sizeof(u32)) - 1; i >= 0; i--)
288 		hash[i] = img_hash_read_result_queue(ctx->hdev);
289 }
290 
291 static void img_hash_finish_req(struct ahash_request *req, int err)
292 {
293 	struct img_hash_request_ctx *ctx = ahash_request_ctx(req);
294 	struct img_hash_dev *hdev =  ctx->hdev;
295 
296 	if (!err) {
297 		img_hash_copy_hash(req);
298 		if (DRIVER_FLAGS_FINAL & hdev->flags)
299 			err = img_hash_finish(req);
300 	} else {
301 		dev_warn(hdev->dev, "Hash failed with error %d\n", err);
302 		ctx->flags |= DRIVER_FLAGS_ERROR;
303 	}
304 
305 	hdev->flags &= ~(DRIVER_FLAGS_DMA_READY | DRIVER_FLAGS_OUTPUT_READY |
306 		DRIVER_FLAGS_CPU | DRIVER_FLAGS_BUSY | DRIVER_FLAGS_FINAL);
307 
308 	if (req->base.complete)
309 		req->base.complete(&req->base, err);
310 }
311 
312 static int img_hash_write_via_dma(struct img_hash_dev *hdev)
313 {
314 	struct img_hash_request_ctx *ctx = ahash_request_ctx(hdev->req);
315 
316 	img_hash_start(hdev, true);
317 
318 	dev_dbg(hdev->dev, "xmit dma size: %d\n", ctx->total);
319 
320 	if (!ctx->total)
321 		hdev->flags |= DRIVER_FLAGS_FINAL;
322 
323 	hdev->flags |= DRIVER_FLAGS_DMA_ACTIVE | DRIVER_FLAGS_FINAL;
324 
325 	tasklet_schedule(&hdev->dma_task);
326 
327 	return -EINPROGRESS;
328 }
329 
330 static int img_hash_dma_init(struct img_hash_dev *hdev)
331 {
332 	struct dma_slave_config dma_conf;
333 	int err = -EINVAL;
334 
335 	hdev->dma_lch = dma_request_slave_channel(hdev->dev, "tx");
336 	if (!hdev->dma_lch) {
337 		dev_err(hdev->dev, "Couldn't acquire a slave DMA channel.\n");
338 		return -EBUSY;
339 	}
340 	dma_conf.direction = DMA_MEM_TO_DEV;
341 	dma_conf.dst_addr = hdev->bus_addr;
342 	dma_conf.dst_addr_width = DMA_SLAVE_BUSWIDTH_4_BYTES;
343 	dma_conf.dst_maxburst = 16;
344 	dma_conf.device_fc = false;
345 
346 	err = dmaengine_slave_config(hdev->dma_lch,  &dma_conf);
347 	if (err) {
348 		dev_err(hdev->dev, "Couldn't configure DMA slave.\n");
349 		dma_release_channel(hdev->dma_lch);
350 		return err;
351 	}
352 
353 	return 0;
354 }
355 
356 static void img_hash_dma_task(unsigned long d)
357 {
358 	struct img_hash_dev *hdev = (struct img_hash_dev *)d;
359 	struct img_hash_request_ctx *ctx = ahash_request_ctx(hdev->req);
360 	u8 *addr;
361 	size_t nbytes, bleft, wsend, len, tbc;
362 	struct scatterlist tsg;
363 
364 	if (!ctx->sg)
365 		return;
366 
367 	addr = sg_virt(ctx->sg);
368 	nbytes = ctx->sg->length - ctx->offset;
369 
370 	/*
371 	 * The hash accelerator does not support a data valid mask. This means
372 	 * that if each dma (i.e. per page) is not a multiple of 4 bytes, the
373 	 * padding bytes in the last word written by that dma would erroneously
374 	 * be included in the hash. To avoid this we round down the transfer,
375 	 * and add the excess to the start of the next dma. It does not matter
376 	 * that the final dma may not be a multiple of 4 bytes as the hashing
377 	 * block is programmed to accept the correct number of bytes.
378 	 */
379 
380 	bleft = nbytes % 4;
381 	wsend = (nbytes / 4);
382 
383 	if (wsend) {
384 		sg_init_one(&tsg, addr + ctx->offset, wsend * 4);
385 		if (img_hash_xmit_dma(hdev, &tsg)) {
386 			dev_err(hdev->dev, "DMA failed, falling back to CPU");
387 			ctx->flags |= DRIVER_FLAGS_CPU;
388 			hdev->err = 0;
389 			img_hash_xmit_cpu(hdev, addr + ctx->offset,
390 					  wsend * 4, 0);
391 			ctx->sent += wsend * 4;
392 			wsend = 0;
393 		} else {
394 			ctx->sent += wsend * 4;
395 		}
396 	}
397 
398 	if (bleft) {
399 		ctx->bufcnt = sg_pcopy_to_buffer(ctx->sgfirst, ctx->nents,
400 						 ctx->buffer, bleft, ctx->sent);
401 		tbc = 0;
402 		ctx->sg = sg_next(ctx->sg);
403 		while (ctx->sg && (ctx->bufcnt < 4)) {
404 			len = ctx->sg->length;
405 			if (likely(len > (4 - ctx->bufcnt)))
406 				len = 4 - ctx->bufcnt;
407 			tbc = sg_pcopy_to_buffer(ctx->sgfirst, ctx->nents,
408 						 ctx->buffer + ctx->bufcnt, len,
409 					ctx->sent + ctx->bufcnt);
410 			ctx->bufcnt += tbc;
411 			if (tbc >= ctx->sg->length) {
412 				ctx->sg = sg_next(ctx->sg);
413 				tbc = 0;
414 			}
415 		}
416 
417 		ctx->sent += ctx->bufcnt;
418 		ctx->offset = tbc;
419 
420 		if (!wsend)
421 			img_hash_dma_callback(hdev);
422 	} else {
423 		ctx->offset = 0;
424 		ctx->sg = sg_next(ctx->sg);
425 	}
426 }
427 
428 static int img_hash_write_via_dma_stop(struct img_hash_dev *hdev)
429 {
430 	struct img_hash_request_ctx *ctx = ahash_request_ctx(hdev->req);
431 
432 	if (ctx->flags & DRIVER_FLAGS_SG)
433 		dma_unmap_sg(hdev->dev, ctx->sg, ctx->dma_ct, DMA_TO_DEVICE);
434 
435 	return 0;
436 }
437 
438 static int img_hash_process_data(struct img_hash_dev *hdev)
439 {
440 	struct ahash_request *req = hdev->req;
441 	struct img_hash_request_ctx *ctx = ahash_request_ctx(req);
442 	int err = 0;
443 
444 	ctx->bufcnt = 0;
445 
446 	if (req->nbytes >= IMG_HASH_DMA_THRESHOLD) {
447 		dev_dbg(hdev->dev, "process data request(%d bytes) using DMA\n",
448 			req->nbytes);
449 		err = img_hash_write_via_dma(hdev);
450 	} else {
451 		dev_dbg(hdev->dev, "process data request(%d bytes) using CPU\n",
452 			req->nbytes);
453 		err = img_hash_write_via_cpu(hdev);
454 	}
455 	return err;
456 }
457 
458 static int img_hash_hw_init(struct img_hash_dev *hdev)
459 {
460 	unsigned long long nbits;
461 	u32 u, l;
462 
463 	img_hash_write(hdev, CR_RESET, CR_RESET_SET);
464 	img_hash_write(hdev, CR_RESET, CR_RESET_UNSET);
465 	img_hash_write(hdev, CR_INTENAB, CR_INT_NEW_RESULTS_SET);
466 
467 	nbits = (u64)hdev->req->nbytes << 3;
468 	u = nbits >> 32;
469 	l = nbits;
470 	img_hash_write(hdev, CR_MESSAGE_LENGTH_H, u);
471 	img_hash_write(hdev, CR_MESSAGE_LENGTH_L, l);
472 
473 	if (!(DRIVER_FLAGS_INIT & hdev->flags)) {
474 		hdev->flags |= DRIVER_FLAGS_INIT;
475 		hdev->err = 0;
476 	}
477 	dev_dbg(hdev->dev, "hw initialized, nbits: %llx\n", nbits);
478 	return 0;
479 }
480 
481 static int img_hash_init(struct ahash_request *req)
482 {
483 	struct crypto_ahash *tfm = crypto_ahash_reqtfm(req);
484 	struct img_hash_request_ctx *rctx = ahash_request_ctx(req);
485 	struct img_hash_ctx *ctx = crypto_ahash_ctx(tfm);
486 
487 	ahash_request_set_tfm(&rctx->fallback_req, ctx->fallback);
488 	rctx->fallback_req.base.flags =	req->base.flags
489 		& CRYPTO_TFM_REQ_MAY_SLEEP;
490 
491 	return crypto_ahash_init(&rctx->fallback_req);
492 }
493 
494 static int img_hash_handle_queue(struct img_hash_dev *hdev,
495 				 struct ahash_request *req)
496 {
497 	struct crypto_async_request *async_req, *backlog;
498 	struct img_hash_request_ctx *ctx;
499 	unsigned long flags;
500 	int err = 0, res = 0;
501 
502 	spin_lock_irqsave(&hdev->lock, flags);
503 
504 	if (req)
505 		res = ahash_enqueue_request(&hdev->queue, req);
506 
507 	if (DRIVER_FLAGS_BUSY & hdev->flags) {
508 		spin_unlock_irqrestore(&hdev->lock, flags);
509 		return res;
510 	}
511 
512 	backlog = crypto_get_backlog(&hdev->queue);
513 	async_req = crypto_dequeue_request(&hdev->queue);
514 	if (async_req)
515 		hdev->flags |= DRIVER_FLAGS_BUSY;
516 
517 	spin_unlock_irqrestore(&hdev->lock, flags);
518 
519 	if (!async_req)
520 		return res;
521 
522 	if (backlog)
523 		backlog->complete(backlog, -EINPROGRESS);
524 
525 	req = ahash_request_cast(async_req);
526 	hdev->req = req;
527 
528 	ctx = ahash_request_ctx(req);
529 
530 	dev_info(hdev->dev, "processing req, op: %lu, bytes: %d\n",
531 		 ctx->op, req->nbytes);
532 
533 	err = img_hash_hw_init(hdev);
534 
535 	if (!err)
536 		err = img_hash_process_data(hdev);
537 
538 	if (err != -EINPROGRESS) {
539 		/* done_task will not finish so do it here */
540 		img_hash_finish_req(req, err);
541 	}
542 	return res;
543 }
544 
545 static int img_hash_update(struct ahash_request *req)
546 {
547 	struct img_hash_request_ctx *rctx = ahash_request_ctx(req);
548 	struct crypto_ahash *tfm = crypto_ahash_reqtfm(req);
549 	struct img_hash_ctx *ctx = crypto_ahash_ctx(tfm);
550 
551 	ahash_request_set_tfm(&rctx->fallback_req, ctx->fallback);
552 	rctx->fallback_req.base.flags = req->base.flags
553 		& CRYPTO_TFM_REQ_MAY_SLEEP;
554 	rctx->fallback_req.nbytes = req->nbytes;
555 	rctx->fallback_req.src = req->src;
556 
557 	return crypto_ahash_update(&rctx->fallback_req);
558 }
559 
560 static int img_hash_final(struct ahash_request *req)
561 {
562 	struct img_hash_request_ctx *rctx = ahash_request_ctx(req);
563 	struct crypto_ahash *tfm = crypto_ahash_reqtfm(req);
564 	struct img_hash_ctx *ctx = crypto_ahash_ctx(tfm);
565 
566 	ahash_request_set_tfm(&rctx->fallback_req, ctx->fallback);
567 	rctx->fallback_req.base.flags = req->base.flags
568 		& CRYPTO_TFM_REQ_MAY_SLEEP;
569 	rctx->fallback_req.result = req->result;
570 
571 	return crypto_ahash_final(&rctx->fallback_req);
572 }
573 
574 static int img_hash_finup(struct ahash_request *req)
575 {
576 	struct img_hash_request_ctx *rctx = ahash_request_ctx(req);
577 	struct crypto_ahash *tfm = crypto_ahash_reqtfm(req);
578 	struct img_hash_ctx *ctx = crypto_ahash_ctx(tfm);
579 
580 	ahash_request_set_tfm(&rctx->fallback_req, ctx->fallback);
581 	rctx->fallback_req.base.flags = req->base.flags
582 		& CRYPTO_TFM_REQ_MAY_SLEEP;
583 	rctx->fallback_req.nbytes = req->nbytes;
584 	rctx->fallback_req.src = req->src;
585 	rctx->fallback_req.result = req->result;
586 
587 	return crypto_ahash_finup(&rctx->fallback_req);
588 }
589 
590 static int img_hash_digest(struct ahash_request *req)
591 {
592 	struct crypto_ahash *tfm = crypto_ahash_reqtfm(req);
593 	struct img_hash_ctx *tctx = crypto_ahash_ctx(tfm);
594 	struct img_hash_request_ctx *ctx = ahash_request_ctx(req);
595 	struct img_hash_dev *hdev = NULL;
596 	struct img_hash_dev *tmp;
597 	int err;
598 
599 	spin_lock(&img_hash.lock);
600 	if (!tctx->hdev) {
601 		list_for_each_entry(tmp, &img_hash.dev_list, list) {
602 			hdev = tmp;
603 			break;
604 		}
605 		tctx->hdev = hdev;
606 
607 	} else {
608 		hdev = tctx->hdev;
609 	}
610 
611 	spin_unlock(&img_hash.lock);
612 	ctx->hdev = hdev;
613 	ctx->flags = 0;
614 	ctx->digsize = crypto_ahash_digestsize(tfm);
615 
616 	switch (ctx->digsize) {
617 	case SHA1_DIGEST_SIZE:
618 		ctx->flags |= DRIVER_FLAGS_SHA1;
619 		break;
620 	case SHA256_DIGEST_SIZE:
621 		ctx->flags |= DRIVER_FLAGS_SHA256;
622 		break;
623 	case SHA224_DIGEST_SIZE:
624 		ctx->flags |= DRIVER_FLAGS_SHA224;
625 		break;
626 	case MD5_DIGEST_SIZE:
627 		ctx->flags |= DRIVER_FLAGS_MD5;
628 		break;
629 	default:
630 		return -EINVAL;
631 	}
632 
633 	ctx->bufcnt = 0;
634 	ctx->offset = 0;
635 	ctx->sent = 0;
636 	ctx->total = req->nbytes;
637 	ctx->sg = req->src;
638 	ctx->sgfirst = req->src;
639 	ctx->nents = sg_nents(ctx->sg);
640 
641 	err = img_hash_handle_queue(tctx->hdev, req);
642 
643 	return err;
644 }
645 
646 static int img_hash_cra_init(struct crypto_tfm *tfm)
647 {
648 	struct img_hash_ctx *ctx = crypto_tfm_ctx(tfm);
649 	const char *alg_name = crypto_tfm_alg_name(tfm);
650 	int err = -ENOMEM;
651 
652 	ctx->fallback = crypto_alloc_ahash(alg_name, 0,
653 					   CRYPTO_ALG_NEED_FALLBACK);
654 	if (IS_ERR(ctx->fallback)) {
655 		pr_err("img_hash: Could not load fallback driver.\n");
656 		err = PTR_ERR(ctx->fallback);
657 		goto err;
658 	}
659 	crypto_ahash_set_reqsize(__crypto_ahash_cast(tfm),
660 				 sizeof(struct img_hash_request_ctx) +
661 				 IMG_HASH_DMA_THRESHOLD);
662 
663 	return 0;
664 
665 err:
666 	return err;
667 }
668 
669 static void img_hash_cra_exit(struct crypto_tfm *tfm)
670 {
671 	struct img_hash_ctx *tctx = crypto_tfm_ctx(tfm);
672 
673 	crypto_free_ahash(tctx->fallback);
674 }
675 
676 static irqreturn_t img_irq_handler(int irq, void *dev_id)
677 {
678 	struct img_hash_dev *hdev = dev_id;
679 	u32 reg;
680 
681 	reg = img_hash_read(hdev, CR_INTSTAT);
682 	img_hash_write(hdev, CR_INTCLEAR, reg);
683 
684 	if (reg & CR_INT_NEW_RESULTS_SET) {
685 		dev_dbg(hdev->dev, "IRQ CR_INT_NEW_RESULTS_SET\n");
686 		if (DRIVER_FLAGS_BUSY & hdev->flags) {
687 			hdev->flags |= DRIVER_FLAGS_OUTPUT_READY;
688 			if (!(DRIVER_FLAGS_CPU & hdev->flags))
689 				hdev->flags |= DRIVER_FLAGS_DMA_READY;
690 			tasklet_schedule(&hdev->done_task);
691 		} else {
692 			dev_warn(hdev->dev,
693 				 "HASH interrupt when no active requests.\n");
694 		}
695 	} else if (reg & CR_INT_RESULTS_AVAILABLE) {
696 		dev_warn(hdev->dev,
697 			 "IRQ triggered before the hash had completed\n");
698 	} else if (reg & CR_INT_RESULT_READ_ERR) {
699 		dev_warn(hdev->dev,
700 			 "Attempt to read from an empty result queue\n");
701 	} else if (reg & CR_INT_MESSAGE_WRITE_ERROR) {
702 		dev_warn(hdev->dev,
703 			 "Data written before the hardware was configured\n");
704 	}
705 	return IRQ_HANDLED;
706 }
707 
708 static struct ahash_alg img_algs[] = {
709 	{
710 		.init = img_hash_init,
711 		.update = img_hash_update,
712 		.final = img_hash_final,
713 		.finup = img_hash_finup,
714 		.digest = img_hash_digest,
715 		.halg = {
716 			.digestsize = MD5_DIGEST_SIZE,
717 			.base = {
718 				.cra_name = "md5",
719 				.cra_driver_name = "img-md5",
720 				.cra_priority = 300,
721 				.cra_flags =
722 				CRYPTO_ALG_ASYNC |
723 				CRYPTO_ALG_NEED_FALLBACK,
724 				.cra_blocksize = MD5_HMAC_BLOCK_SIZE,
725 				.cra_ctxsize = sizeof(struct img_hash_ctx),
726 				.cra_init = img_hash_cra_init,
727 				.cra_exit = img_hash_cra_exit,
728 				.cra_module = THIS_MODULE,
729 			}
730 		}
731 	},
732 	{
733 		.init = img_hash_init,
734 		.update = img_hash_update,
735 		.final = img_hash_final,
736 		.finup = img_hash_finup,
737 		.digest = img_hash_digest,
738 		.halg = {
739 			.digestsize = SHA1_DIGEST_SIZE,
740 			.base = {
741 				.cra_name = "sha1",
742 				.cra_driver_name = "img-sha1",
743 				.cra_priority = 300,
744 				.cra_flags =
745 				CRYPTO_ALG_ASYNC |
746 				CRYPTO_ALG_NEED_FALLBACK,
747 				.cra_blocksize = SHA1_BLOCK_SIZE,
748 				.cra_ctxsize = sizeof(struct img_hash_ctx),
749 				.cra_init = img_hash_cra_init,
750 				.cra_exit = img_hash_cra_exit,
751 				.cra_module = THIS_MODULE,
752 			}
753 		}
754 	},
755 	{
756 		.init = img_hash_init,
757 		.update = img_hash_update,
758 		.final = img_hash_final,
759 		.finup = img_hash_finup,
760 		.digest = img_hash_digest,
761 		.halg = {
762 			.digestsize = SHA224_DIGEST_SIZE,
763 			.base = {
764 				.cra_name = "sha224",
765 				.cra_driver_name = "img-sha224",
766 				.cra_priority = 300,
767 				.cra_flags =
768 				CRYPTO_ALG_ASYNC |
769 				CRYPTO_ALG_NEED_FALLBACK,
770 				.cra_blocksize = SHA224_BLOCK_SIZE,
771 				.cra_ctxsize = sizeof(struct img_hash_ctx),
772 				.cra_init = img_hash_cra_init,
773 				.cra_exit = img_hash_cra_exit,
774 				.cra_module = THIS_MODULE,
775 			}
776 		}
777 	},
778 	{
779 		.init = img_hash_init,
780 		.update = img_hash_update,
781 		.final = img_hash_final,
782 		.finup = img_hash_finup,
783 		.digest = img_hash_digest,
784 		.halg = {
785 			.digestsize = SHA256_DIGEST_SIZE,
786 			.base = {
787 				.cra_name = "sha256",
788 				.cra_driver_name = "img-sha256",
789 				.cra_priority = 300,
790 				.cra_flags =
791 				CRYPTO_ALG_ASYNC |
792 				CRYPTO_ALG_NEED_FALLBACK,
793 				.cra_blocksize = SHA256_BLOCK_SIZE,
794 				.cra_ctxsize = sizeof(struct img_hash_ctx),
795 				.cra_init = img_hash_cra_init,
796 				.cra_exit = img_hash_cra_exit,
797 				.cra_module = THIS_MODULE,
798 			}
799 		}
800 	}
801 };
802 
803 static int img_register_algs(struct img_hash_dev *hdev)
804 {
805 	int i, err;
806 
807 	for (i = 0; i < ARRAY_SIZE(img_algs); i++) {
808 		err = crypto_register_ahash(&img_algs[i]);
809 		if (err)
810 			goto err_reg;
811 	}
812 	return 0;
813 
814 err_reg:
815 	for (; i--; )
816 		crypto_unregister_ahash(&img_algs[i]);
817 
818 	return err;
819 }
820 
821 static int img_unregister_algs(struct img_hash_dev *hdev)
822 {
823 	int i;
824 
825 	for (i = 0; i < ARRAY_SIZE(img_algs); i++)
826 		crypto_unregister_ahash(&img_algs[i]);
827 	return 0;
828 }
829 
830 static void img_hash_done_task(unsigned long data)
831 {
832 	struct img_hash_dev *hdev = (struct img_hash_dev *)data;
833 	int err = 0;
834 
835 	if (hdev->err == -EINVAL) {
836 		err = hdev->err;
837 		goto finish;
838 	}
839 
840 	if (!(DRIVER_FLAGS_BUSY & hdev->flags)) {
841 		img_hash_handle_queue(hdev, NULL);
842 		return;
843 	}
844 
845 	if (DRIVER_FLAGS_CPU & hdev->flags) {
846 		if (DRIVER_FLAGS_OUTPUT_READY & hdev->flags) {
847 			hdev->flags &= ~DRIVER_FLAGS_OUTPUT_READY;
848 			goto finish;
849 		}
850 	} else if (DRIVER_FLAGS_DMA_READY & hdev->flags) {
851 		if (DRIVER_FLAGS_DMA_ACTIVE & hdev->flags) {
852 			hdev->flags &= ~DRIVER_FLAGS_DMA_ACTIVE;
853 			img_hash_write_via_dma_stop(hdev);
854 			if (hdev->err) {
855 				err = hdev->err;
856 				goto finish;
857 			}
858 		}
859 		if (DRIVER_FLAGS_OUTPUT_READY & hdev->flags) {
860 			hdev->flags &= ~(DRIVER_FLAGS_DMA_READY |
861 					DRIVER_FLAGS_OUTPUT_READY);
862 			goto finish;
863 		}
864 	}
865 	return;
866 
867 finish:
868 	img_hash_finish_req(hdev->req, err);
869 }
870 
871 static const struct of_device_id img_hash_match[] = {
872 	{ .compatible = "img,hash-accelerator" },
873 	{}
874 };
875 MODULE_DEVICE_TABLE(of, img_hash_match);
876 
877 static int img_hash_probe(struct platform_device *pdev)
878 {
879 	struct img_hash_dev *hdev;
880 	struct device *dev = &pdev->dev;
881 	struct resource *hash_res;
882 	int	irq;
883 	int err;
884 
885 	hdev = devm_kzalloc(dev, sizeof(*hdev), GFP_KERNEL);
886 	if (hdev == NULL)
887 		return -ENOMEM;
888 
889 	spin_lock_init(&hdev->lock);
890 
891 	hdev->dev = dev;
892 
893 	platform_set_drvdata(pdev, hdev);
894 
895 	INIT_LIST_HEAD(&hdev->list);
896 
897 	tasklet_init(&hdev->done_task, img_hash_done_task, (unsigned long)hdev);
898 	tasklet_init(&hdev->dma_task, img_hash_dma_task, (unsigned long)hdev);
899 
900 	crypto_init_queue(&hdev->queue, IMG_HASH_QUEUE_LENGTH);
901 
902 	/* Register bank */
903 	hash_res = platform_get_resource(pdev, IORESOURCE_MEM, 0);
904 
905 	hdev->io_base = devm_ioremap_resource(dev, hash_res);
906 	if (IS_ERR(hdev->io_base)) {
907 		err = PTR_ERR(hdev->io_base);
908 		dev_err(dev, "can't ioremap, returned %d\n", err);
909 
910 		goto res_err;
911 	}
912 
913 	/* Write port (DMA or CPU) */
914 	hash_res = platform_get_resource(pdev, IORESOURCE_MEM, 1);
915 	hdev->cpu_addr = devm_ioremap_resource(dev, hash_res);
916 	if (IS_ERR(hdev->cpu_addr)) {
917 		dev_err(dev, "can't ioremap write port\n");
918 		err = PTR_ERR(hdev->cpu_addr);
919 		goto res_err;
920 	}
921 	hdev->bus_addr = hash_res->start;
922 
923 	irq = platform_get_irq(pdev, 0);
924 	if (irq < 0) {
925 		dev_err(dev, "no IRQ resource info\n");
926 		err = irq;
927 		goto res_err;
928 	}
929 
930 	err = devm_request_irq(dev, irq, img_irq_handler, 0,
931 			       dev_name(dev), hdev);
932 	if (err) {
933 		dev_err(dev, "unable to request irq\n");
934 		goto res_err;
935 	}
936 	dev_dbg(dev, "using IRQ channel %d\n", irq);
937 
938 	hdev->hash_clk = devm_clk_get(&pdev->dev, "hash");
939 	if (IS_ERR(hdev->hash_clk)) {
940 		dev_err(dev, "clock initialization failed.\n");
941 		err = PTR_ERR(hdev->hash_clk);
942 		goto res_err;
943 	}
944 
945 	hdev->sys_clk = devm_clk_get(&pdev->dev, "sys");
946 	if (IS_ERR(hdev->sys_clk)) {
947 		dev_err(dev, "clock initialization failed.\n");
948 		err = PTR_ERR(hdev->sys_clk);
949 		goto res_err;
950 	}
951 
952 	err = clk_prepare_enable(hdev->hash_clk);
953 	if (err)
954 		goto res_err;
955 
956 	err = clk_prepare_enable(hdev->sys_clk);
957 	if (err)
958 		goto clk_err;
959 
960 	err = img_hash_dma_init(hdev);
961 	if (err)
962 		goto dma_err;
963 
964 	dev_dbg(dev, "using %s for DMA transfers\n",
965 		dma_chan_name(hdev->dma_lch));
966 
967 	spin_lock(&img_hash.lock);
968 	list_add_tail(&hdev->list, &img_hash.dev_list);
969 	spin_unlock(&img_hash.lock);
970 
971 	err = img_register_algs(hdev);
972 	if (err)
973 		goto err_algs;
974 	dev_dbg(dev, "Img MD5/SHA1/SHA224/SHA256 Hardware accelerator initialized\n");
975 
976 	return 0;
977 
978 err_algs:
979 	spin_lock(&img_hash.lock);
980 	list_del(&hdev->list);
981 	spin_unlock(&img_hash.lock);
982 	dma_release_channel(hdev->dma_lch);
983 dma_err:
984 	clk_disable_unprepare(hdev->sys_clk);
985 clk_err:
986 	clk_disable_unprepare(hdev->hash_clk);
987 res_err:
988 	tasklet_kill(&hdev->done_task);
989 	tasklet_kill(&hdev->dma_task);
990 
991 	return err;
992 }
993 
994 static int img_hash_remove(struct platform_device *pdev)
995 {
996 	static struct img_hash_dev *hdev;
997 
998 	hdev = platform_get_drvdata(pdev);
999 	spin_lock(&img_hash.lock);
1000 	list_del(&hdev->list);
1001 	spin_unlock(&img_hash.lock);
1002 
1003 	img_unregister_algs(hdev);
1004 
1005 	tasklet_kill(&hdev->done_task);
1006 	tasklet_kill(&hdev->dma_task);
1007 
1008 	dma_release_channel(hdev->dma_lch);
1009 
1010 	clk_disable_unprepare(hdev->hash_clk);
1011 	clk_disable_unprepare(hdev->sys_clk);
1012 
1013 	return 0;
1014 }
1015 
1016 static struct platform_driver img_hash_driver = {
1017 	.probe		= img_hash_probe,
1018 	.remove		= img_hash_remove,
1019 	.driver		= {
1020 		.name	= "img-hash-accelerator",
1021 		.of_match_table	= of_match_ptr(img_hash_match),
1022 	}
1023 };
1024 module_platform_driver(img_hash_driver);
1025 
1026 MODULE_LICENSE("GPL v2");
1027 MODULE_DESCRIPTION("Imgtec SHA1/224/256 & MD5 hw accelerator driver");
1028 MODULE_AUTHOR("Will Thomas.");
1029 MODULE_AUTHOR("James Hartley <james.hartley@imgtec.com>");
1030