1 // SPDX-License-Identifier: GPL-2.0-only
2 /*
3  * This file is part of STM32 Crypto driver for Linux.
4  *
5  * Copyright (C) 2017, STMicroelectronics - All Rights Reserved
6  * Author(s): Lionel DEBIEVE <lionel.debieve@st.com> for STMicroelectronics.
7  */
8 
9 #include <crypto/engine.h>
10 #include <crypto/internal/hash.h>
11 #include <crypto/md5.h>
12 #include <crypto/scatterwalk.h>
13 #include <crypto/sha1.h>
14 #include <crypto/sha2.h>
15 #include <crypto/sha3.h>
16 #include <linux/clk.h>
17 #include <linux/delay.h>
18 #include <linux/dma-mapping.h>
19 #include <linux/dmaengine.h>
20 #include <linux/interrupt.h>
21 #include <linux/iopoll.h>
22 #include <linux/kernel.h>
23 #include <linux/module.h>
24 #include <linux/of.h>
25 #include <linux/platform_device.h>
26 #include <linux/pm_runtime.h>
27 #include <linux/reset.h>
28 #include <linux/string.h>
29 
30 #define HASH_CR				0x00
31 #define HASH_DIN			0x04
32 #define HASH_STR			0x08
33 #define HASH_UX500_HREG(x)		(0x0c + ((x) * 0x04))
34 #define HASH_IMR			0x20
35 #define HASH_SR				0x24
36 #define HASH_CSR(x)			(0x0F8 + ((x) * 0x04))
37 #define HASH_HREG(x)			(0x310 + ((x) * 0x04))
38 #define HASH_HWCFGR			0x3F0
39 #define HASH_VER			0x3F4
40 #define HASH_ID				0x3F8
41 
42 /* Control Register */
43 #define HASH_CR_INIT			BIT(2)
44 #define HASH_CR_DMAE			BIT(3)
45 #define HASH_CR_DATATYPE_POS		4
46 #define HASH_CR_MODE			BIT(6)
47 #define HASH_CR_ALGO_POS		7
48 #define HASH_CR_MDMAT			BIT(13)
49 #define HASH_CR_DMAA			BIT(14)
50 #define HASH_CR_LKEY			BIT(16)
51 
52 /* Interrupt */
53 #define HASH_DINIE			BIT(0)
54 #define HASH_DCIE			BIT(1)
55 
56 /* Interrupt Mask */
57 #define HASH_MASK_CALC_COMPLETION	BIT(0)
58 #define HASH_MASK_DATA_INPUT		BIT(1)
59 
60 /* Status Flags */
61 #define HASH_SR_DATA_INPUT_READY	BIT(0)
62 #define HASH_SR_OUTPUT_READY		BIT(1)
63 #define HASH_SR_DMA_ACTIVE		BIT(2)
64 #define HASH_SR_BUSY			BIT(3)
65 
66 /* STR Register */
67 #define HASH_STR_NBLW_MASK		GENMASK(4, 0)
68 #define HASH_STR_DCAL			BIT(8)
69 
70 /* HWCFGR Register */
71 #define HASH_HWCFG_DMA_MASK		GENMASK(3, 0)
72 
73 /* Context swap register */
74 #define HASH_CSR_NB_SHA256_HMAC		54
75 #define HASH_CSR_NB_SHA256		38
76 #define HASH_CSR_NB_SHA512_HMAC		103
77 #define HASH_CSR_NB_SHA512		91
78 #define HASH_CSR_NB_SHA3_HMAC		88
79 #define HASH_CSR_NB_SHA3		72
80 #define HASH_CSR_NB_MAX			HASH_CSR_NB_SHA512_HMAC
81 
82 #define HASH_FLAGS_INIT			BIT(0)
83 #define HASH_FLAGS_OUTPUT_READY		BIT(1)
84 #define HASH_FLAGS_CPU			BIT(2)
85 #define HASH_FLAGS_DMA_ACTIVE		BIT(3)
86 #define HASH_FLAGS_HMAC_INIT		BIT(4)
87 #define HASH_FLAGS_HMAC_FINAL		BIT(5)
88 #define HASH_FLAGS_HMAC_KEY		BIT(6)
89 #define HASH_FLAGS_SHA3_MODE		BIT(7)
90 #define HASH_FLAGS_FINAL		BIT(15)
91 #define HASH_FLAGS_FINUP		BIT(16)
92 #define HASH_FLAGS_ALGO_MASK		GENMASK(20, 17)
93 #define HASH_FLAGS_ALGO_SHIFT		17
94 #define HASH_FLAGS_ERRORS		BIT(21)
95 #define HASH_FLAGS_EMPTY		BIT(22)
96 #define HASH_FLAGS_HMAC			BIT(23)
97 
98 #define HASH_OP_UPDATE			1
99 #define HASH_OP_FINAL			2
100 
101 #define HASH_BURST_LEVEL		4
102 
103 enum stm32_hash_data_format {
104 	HASH_DATA_32_BITS		= 0x0,
105 	HASH_DATA_16_BITS		= 0x1,
106 	HASH_DATA_8_BITS		= 0x2,
107 	HASH_DATA_1_BIT			= 0x3
108 };
109 
110 #define HASH_BUFLEN			(SHA3_224_BLOCK_SIZE + 4)
111 #define HASH_MAX_KEY_SIZE		(SHA512_BLOCK_SIZE * 8)
112 
113 enum stm32_hash_algo {
114 	HASH_SHA1			= 0,
115 	HASH_MD5			= 1,
116 	HASH_SHA224			= 2,
117 	HASH_SHA256			= 3,
118 	HASH_SHA3_224			= 4,
119 	HASH_SHA3_256			= 5,
120 	HASH_SHA3_384			= 6,
121 	HASH_SHA3_512			= 7,
122 	HASH_SHA384			= 12,
123 	HASH_SHA512			= 15,
124 };
125 
126 enum ux500_hash_algo {
127 	HASH_SHA256_UX500		= 0,
128 	HASH_SHA1_UX500			= 1,
129 };
130 
131 #define HASH_AUTOSUSPEND_DELAY		50
132 
133 struct stm32_hash_ctx {
134 	struct stm32_hash_dev	*hdev;
135 	struct crypto_shash	*xtfm;
136 	unsigned long		flags;
137 
138 	u8			key[HASH_MAX_KEY_SIZE];
139 	int			keylen;
140 };
141 
142 struct stm32_hash_state {
143 	u32			flags;
144 
145 	u16			bufcnt;
146 	u16			blocklen;
147 
148 	u8 buffer[HASH_BUFLEN] __aligned(4);
149 
150 	/* hash state */
151 	u32			hw_context[3 + HASH_CSR_NB_MAX];
152 };
153 
154 struct stm32_hash_request_ctx {
155 	struct stm32_hash_dev	*hdev;
156 	unsigned long		op;
157 
158 	u8 digest[SHA512_DIGEST_SIZE] __aligned(sizeof(u32));
159 	size_t			digcnt;
160 
161 	/* DMA */
162 	struct scatterlist	*sg;
163 	unsigned int		offset;
164 	unsigned int		total;
165 	struct scatterlist	sg_key;
166 
167 	dma_addr_t		dma_addr;
168 	size_t			dma_ct;
169 	int			nents;
170 
171 	u8			data_type;
172 
173 	struct stm32_hash_state state;
174 };
175 
176 struct stm32_hash_algs_info {
177 	struct ahash_engine_alg	*algs_list;
178 	size_t			size;
179 };
180 
181 struct stm32_hash_pdata {
182 	const int				alg_shift;
183 	const struct stm32_hash_algs_info	*algs_info;
184 	size_t					algs_info_size;
185 	bool					has_sr;
186 	bool					has_mdmat;
187 	bool					broken_emptymsg;
188 	bool					ux500;
189 };
190 
191 struct stm32_hash_dev {
192 	struct list_head	list;
193 	struct device		*dev;
194 	struct clk		*clk;
195 	struct reset_control	*rst;
196 	void __iomem		*io_base;
197 	phys_addr_t		phys_base;
198 	u32			dma_mode;
199 	bool			polled;
200 
201 	struct ahash_request	*req;
202 	struct crypto_engine	*engine;
203 
204 	unsigned long		flags;
205 
206 	struct dma_chan		*dma_lch;
207 	struct completion	dma_completion;
208 
209 	const struct stm32_hash_pdata	*pdata;
210 };
211 
212 struct stm32_hash_drv {
213 	struct list_head	dev_list;
214 	spinlock_t		lock; /* List protection access */
215 };
216 
217 static struct stm32_hash_drv stm32_hash = {
218 	.dev_list = LIST_HEAD_INIT(stm32_hash.dev_list),
219 	.lock = __SPIN_LOCK_UNLOCKED(stm32_hash.lock),
220 };
221 
222 static void stm32_hash_dma_callback(void *param);
223 
224 static inline u32 stm32_hash_read(struct stm32_hash_dev *hdev, u32 offset)
225 {
226 	return readl_relaxed(hdev->io_base + offset);
227 }
228 
229 static inline void stm32_hash_write(struct stm32_hash_dev *hdev,
230 				    u32 offset, u32 value)
231 {
232 	writel_relaxed(value, hdev->io_base + offset);
233 }
234 
235 static inline int stm32_hash_wait_busy(struct stm32_hash_dev *hdev)
236 {
237 	u32 status;
238 
239 	/* The Ux500 lacks the special status register, we poll the DCAL bit instead */
240 	if (!hdev->pdata->has_sr)
241 		return readl_relaxed_poll_timeout(hdev->io_base + HASH_STR, status,
242 						  !(status & HASH_STR_DCAL), 10, 10000);
243 
244 	return readl_relaxed_poll_timeout(hdev->io_base + HASH_SR, status,
245 				   !(status & HASH_SR_BUSY), 10, 10000);
246 }
247 
248 static void stm32_hash_set_nblw(struct stm32_hash_dev *hdev, int length)
249 {
250 	u32 reg;
251 
252 	reg = stm32_hash_read(hdev, HASH_STR);
253 	reg &= ~(HASH_STR_NBLW_MASK);
254 	reg |= (8U * ((length) % 4U));
255 	stm32_hash_write(hdev, HASH_STR, reg);
256 }
257 
258 static int stm32_hash_write_key(struct stm32_hash_dev *hdev)
259 {
260 	struct crypto_ahash *tfm = crypto_ahash_reqtfm(hdev->req);
261 	struct stm32_hash_ctx *ctx = crypto_ahash_ctx(tfm);
262 	u32 reg;
263 	int keylen = ctx->keylen;
264 	void *key = ctx->key;
265 
266 	if (keylen) {
267 		stm32_hash_set_nblw(hdev, keylen);
268 
269 		while (keylen > 0) {
270 			stm32_hash_write(hdev, HASH_DIN, *(u32 *)key);
271 			keylen -= 4;
272 			key += 4;
273 		}
274 
275 		reg = stm32_hash_read(hdev, HASH_STR);
276 		reg |= HASH_STR_DCAL;
277 		stm32_hash_write(hdev, HASH_STR, reg);
278 
279 		return -EINPROGRESS;
280 	}
281 
282 	return 0;
283 }
284 
285 static void stm32_hash_write_ctrl(struct stm32_hash_dev *hdev)
286 {
287 	struct stm32_hash_request_ctx *rctx = ahash_request_ctx(hdev->req);
288 	struct crypto_ahash *tfm = crypto_ahash_reqtfm(hdev->req);
289 	struct stm32_hash_ctx *ctx = crypto_ahash_ctx(tfm);
290 	struct stm32_hash_state *state = &rctx->state;
291 	u32 alg = (state->flags & HASH_FLAGS_ALGO_MASK) >> HASH_FLAGS_ALGO_SHIFT;
292 
293 	u32 reg = HASH_CR_INIT;
294 
295 	if (!(hdev->flags & HASH_FLAGS_INIT)) {
296 		if (hdev->pdata->ux500) {
297 			reg |= ((alg & BIT(0)) << HASH_CR_ALGO_POS);
298 		} else {
299 			if (hdev->pdata->alg_shift == HASH_CR_ALGO_POS)
300 				reg |= ((alg & BIT(1)) << 17) |
301 				       ((alg & BIT(0)) << HASH_CR_ALGO_POS);
302 			else
303 				reg |= alg << hdev->pdata->alg_shift;
304 		}
305 
306 		reg |= (rctx->data_type << HASH_CR_DATATYPE_POS);
307 
308 		if (state->flags & HASH_FLAGS_HMAC) {
309 			hdev->flags |= HASH_FLAGS_HMAC;
310 			reg |= HASH_CR_MODE;
311 			if (ctx->keylen > crypto_ahash_blocksize(tfm))
312 				reg |= HASH_CR_LKEY;
313 		}
314 
315 		if (!hdev->polled)
316 			stm32_hash_write(hdev, HASH_IMR, HASH_DCIE);
317 
318 		stm32_hash_write(hdev, HASH_CR, reg);
319 
320 		hdev->flags |= HASH_FLAGS_INIT;
321 
322 		/*
323 		 * After first block + 1 words are fill up,
324 		 * we only need to fill 1 block to start partial computation
325 		 */
326 		rctx->state.blocklen -= sizeof(u32);
327 
328 		dev_dbg(hdev->dev, "Write Control %x\n", reg);
329 	}
330 }
331 
332 static void stm32_hash_append_sg(struct stm32_hash_request_ctx *rctx)
333 {
334 	struct stm32_hash_state *state = &rctx->state;
335 	size_t count;
336 
337 	while ((state->bufcnt < state->blocklen) && rctx->total) {
338 		count = min(rctx->sg->length - rctx->offset, rctx->total);
339 		count = min_t(size_t, count, state->blocklen - state->bufcnt);
340 
341 		if (count <= 0) {
342 			if ((rctx->sg->length == 0) && !sg_is_last(rctx->sg)) {
343 				rctx->sg = sg_next(rctx->sg);
344 				continue;
345 			} else {
346 				break;
347 			}
348 		}
349 
350 		scatterwalk_map_and_copy(state->buffer + state->bufcnt,
351 					 rctx->sg, rctx->offset, count, 0);
352 
353 		state->bufcnt += count;
354 		rctx->offset += count;
355 		rctx->total -= count;
356 
357 		if (rctx->offset == rctx->sg->length) {
358 			rctx->sg = sg_next(rctx->sg);
359 			if (rctx->sg)
360 				rctx->offset = 0;
361 			else
362 				rctx->total = 0;
363 		}
364 	}
365 }
366 
367 static int stm32_hash_xmit_cpu(struct stm32_hash_dev *hdev,
368 			       const u8 *buf, size_t length, int final)
369 {
370 	struct stm32_hash_request_ctx *rctx = ahash_request_ctx(hdev->req);
371 	struct stm32_hash_state *state = &rctx->state;
372 	unsigned int count, len32;
373 	const u32 *buffer = (const u32 *)buf;
374 	u32 reg;
375 
376 	if (final) {
377 		hdev->flags |= HASH_FLAGS_FINAL;
378 
379 		/* Do not process empty messages if hw is buggy. */
380 		if (!(hdev->flags & HASH_FLAGS_INIT) && !length &&
381 		    hdev->pdata->broken_emptymsg) {
382 			state->flags |= HASH_FLAGS_EMPTY;
383 			return 0;
384 		}
385 	}
386 
387 	len32 = DIV_ROUND_UP(length, sizeof(u32));
388 
389 	dev_dbg(hdev->dev, "%s: length: %zd, final: %x len32 %i\n",
390 		__func__, length, final, len32);
391 
392 	hdev->flags |= HASH_FLAGS_CPU;
393 
394 	stm32_hash_write_ctrl(hdev);
395 
396 	if (stm32_hash_wait_busy(hdev))
397 		return -ETIMEDOUT;
398 
399 	if ((hdev->flags & HASH_FLAGS_HMAC) &&
400 	    (!(hdev->flags & HASH_FLAGS_HMAC_KEY))) {
401 		hdev->flags |= HASH_FLAGS_HMAC_KEY;
402 		stm32_hash_write_key(hdev);
403 		if (stm32_hash_wait_busy(hdev))
404 			return -ETIMEDOUT;
405 	}
406 
407 	for (count = 0; count < len32; count++)
408 		stm32_hash_write(hdev, HASH_DIN, buffer[count]);
409 
410 	if (final) {
411 		if (stm32_hash_wait_busy(hdev))
412 			return -ETIMEDOUT;
413 
414 		stm32_hash_set_nblw(hdev, length);
415 		reg = stm32_hash_read(hdev, HASH_STR);
416 		reg |= HASH_STR_DCAL;
417 		stm32_hash_write(hdev, HASH_STR, reg);
418 		if (hdev->flags & HASH_FLAGS_HMAC) {
419 			if (stm32_hash_wait_busy(hdev))
420 				return -ETIMEDOUT;
421 			stm32_hash_write_key(hdev);
422 		}
423 		return -EINPROGRESS;
424 	}
425 
426 	return 0;
427 }
428 
429 static int hash_swap_reg(struct stm32_hash_request_ctx *rctx)
430 {
431 	struct stm32_hash_state *state = &rctx->state;
432 
433 	switch ((state->flags & HASH_FLAGS_ALGO_MASK) >>
434 		HASH_FLAGS_ALGO_SHIFT) {
435 	case HASH_MD5:
436 	case HASH_SHA1:
437 	case HASH_SHA224:
438 	case HASH_SHA256:
439 		if (state->flags & HASH_FLAGS_HMAC)
440 			return HASH_CSR_NB_SHA256_HMAC;
441 		else
442 			return HASH_CSR_NB_SHA256;
443 		break;
444 
445 	case HASH_SHA384:
446 	case HASH_SHA512:
447 		if (state->flags & HASH_FLAGS_HMAC)
448 			return HASH_CSR_NB_SHA512_HMAC;
449 		else
450 			return HASH_CSR_NB_SHA512;
451 		break;
452 
453 	case HASH_SHA3_224:
454 	case HASH_SHA3_256:
455 	case HASH_SHA3_384:
456 	case HASH_SHA3_512:
457 		if (state->flags & HASH_FLAGS_HMAC)
458 			return HASH_CSR_NB_SHA3_HMAC;
459 		else
460 			return HASH_CSR_NB_SHA3;
461 		break;
462 
463 	default:
464 		return -EINVAL;
465 	}
466 }
467 
468 static int stm32_hash_update_cpu(struct stm32_hash_dev *hdev)
469 {
470 	struct stm32_hash_request_ctx *rctx = ahash_request_ctx(hdev->req);
471 	struct stm32_hash_state *state = &rctx->state;
472 	u32 *preg = state->hw_context;
473 	int bufcnt, err = 0, final;
474 	int i, swap_reg;
475 
476 	dev_dbg(hdev->dev, "%s flags %x\n", __func__, state->flags);
477 
478 	final = state->flags & HASH_FLAGS_FINAL;
479 
480 	while ((rctx->total >= state->blocklen) ||
481 	       (state->bufcnt + rctx->total >= state->blocklen)) {
482 		stm32_hash_append_sg(rctx);
483 		bufcnt = state->bufcnt;
484 		state->bufcnt = 0;
485 		err = stm32_hash_xmit_cpu(hdev, state->buffer, bufcnt, 0);
486 		if (err)
487 			return err;
488 	}
489 
490 	stm32_hash_append_sg(rctx);
491 
492 	if (final) {
493 		bufcnt = state->bufcnt;
494 		state->bufcnt = 0;
495 		return stm32_hash_xmit_cpu(hdev, state->buffer, bufcnt, 1);
496 	}
497 
498 	if (!(hdev->flags & HASH_FLAGS_INIT))
499 		return 0;
500 
501 	if (stm32_hash_wait_busy(hdev))
502 		return -ETIMEDOUT;
503 
504 	swap_reg = hash_swap_reg(rctx);
505 
506 	if (!hdev->pdata->ux500)
507 		*preg++ = stm32_hash_read(hdev, HASH_IMR);
508 	*preg++ = stm32_hash_read(hdev, HASH_STR);
509 	*preg++ = stm32_hash_read(hdev, HASH_CR);
510 	for (i = 0; i < swap_reg; i++)
511 		*preg++ = stm32_hash_read(hdev, HASH_CSR(i));
512 
513 	state->flags |= HASH_FLAGS_INIT;
514 
515 	return err;
516 }
517 
518 static int stm32_hash_xmit_dma(struct stm32_hash_dev *hdev,
519 			       struct scatterlist *sg, int length, int mdma)
520 {
521 	struct dma_async_tx_descriptor *in_desc;
522 	dma_cookie_t cookie;
523 	u32 reg;
524 	int err;
525 
526 	in_desc = dmaengine_prep_slave_sg(hdev->dma_lch, sg, 1,
527 					  DMA_MEM_TO_DEV, DMA_PREP_INTERRUPT |
528 					  DMA_CTRL_ACK);
529 	if (!in_desc) {
530 		dev_err(hdev->dev, "dmaengine_prep_slave error\n");
531 		return -ENOMEM;
532 	}
533 
534 	reinit_completion(&hdev->dma_completion);
535 	in_desc->callback = stm32_hash_dma_callback;
536 	in_desc->callback_param = hdev;
537 
538 	hdev->flags |= HASH_FLAGS_FINAL;
539 	hdev->flags |= HASH_FLAGS_DMA_ACTIVE;
540 
541 	reg = stm32_hash_read(hdev, HASH_CR);
542 
543 	if (hdev->pdata->has_mdmat) {
544 		if (mdma)
545 			reg |= HASH_CR_MDMAT;
546 		else
547 			reg &= ~HASH_CR_MDMAT;
548 	}
549 	reg |= HASH_CR_DMAE;
550 
551 	stm32_hash_write(hdev, HASH_CR, reg);
552 
553 	stm32_hash_set_nblw(hdev, length);
554 
555 	cookie = dmaengine_submit(in_desc);
556 	err = dma_submit_error(cookie);
557 	if (err)
558 		return -ENOMEM;
559 
560 	dma_async_issue_pending(hdev->dma_lch);
561 
562 	if (!wait_for_completion_timeout(&hdev->dma_completion,
563 					 msecs_to_jiffies(100)))
564 		err = -ETIMEDOUT;
565 
566 	if (dma_async_is_tx_complete(hdev->dma_lch, cookie,
567 				     NULL, NULL) != DMA_COMPLETE)
568 		err = -ETIMEDOUT;
569 
570 	if (err) {
571 		dev_err(hdev->dev, "DMA Error %i\n", err);
572 		dmaengine_terminate_all(hdev->dma_lch);
573 		return err;
574 	}
575 
576 	return -EINPROGRESS;
577 }
578 
579 static void stm32_hash_dma_callback(void *param)
580 {
581 	struct stm32_hash_dev *hdev = param;
582 
583 	complete(&hdev->dma_completion);
584 }
585 
586 static int stm32_hash_hmac_dma_send(struct stm32_hash_dev *hdev)
587 {
588 	struct stm32_hash_request_ctx *rctx = ahash_request_ctx(hdev->req);
589 	struct crypto_ahash *tfm = crypto_ahash_reqtfm(hdev->req);
590 	struct stm32_hash_ctx *ctx = crypto_ahash_ctx(tfm);
591 	int err;
592 
593 	if (ctx->keylen < rctx->state.blocklen || hdev->dma_mode == 1) {
594 		err = stm32_hash_write_key(hdev);
595 		if (stm32_hash_wait_busy(hdev))
596 			return -ETIMEDOUT;
597 	} else {
598 		if (!(hdev->flags & HASH_FLAGS_HMAC_KEY))
599 			sg_init_one(&rctx->sg_key, ctx->key,
600 				    ALIGN(ctx->keylen, sizeof(u32)));
601 
602 		rctx->dma_ct = dma_map_sg(hdev->dev, &rctx->sg_key, 1,
603 					  DMA_TO_DEVICE);
604 		if (rctx->dma_ct == 0) {
605 			dev_err(hdev->dev, "dma_map_sg error\n");
606 			return -ENOMEM;
607 		}
608 
609 		err = stm32_hash_xmit_dma(hdev, &rctx->sg_key, ctx->keylen, 0);
610 
611 		dma_unmap_sg(hdev->dev, &rctx->sg_key, 1, DMA_TO_DEVICE);
612 	}
613 
614 	return err;
615 }
616 
617 static int stm32_hash_dma_init(struct stm32_hash_dev *hdev)
618 {
619 	struct dma_slave_config dma_conf;
620 	struct dma_chan *chan;
621 	int err;
622 
623 	memset(&dma_conf, 0, sizeof(dma_conf));
624 
625 	dma_conf.direction = DMA_MEM_TO_DEV;
626 	dma_conf.dst_addr = hdev->phys_base + HASH_DIN;
627 	dma_conf.dst_addr_width = DMA_SLAVE_BUSWIDTH_4_BYTES;
628 	dma_conf.src_maxburst = HASH_BURST_LEVEL;
629 	dma_conf.dst_maxburst = HASH_BURST_LEVEL;
630 	dma_conf.device_fc = false;
631 
632 	chan = dma_request_chan(hdev->dev, "in");
633 	if (IS_ERR(chan))
634 		return PTR_ERR(chan);
635 
636 	hdev->dma_lch = chan;
637 
638 	err = dmaengine_slave_config(hdev->dma_lch, &dma_conf);
639 	if (err) {
640 		dma_release_channel(hdev->dma_lch);
641 		hdev->dma_lch = NULL;
642 		dev_err(hdev->dev, "Couldn't configure DMA slave.\n");
643 		return err;
644 	}
645 
646 	init_completion(&hdev->dma_completion);
647 
648 	return 0;
649 }
650 
651 static int stm32_hash_dma_send(struct stm32_hash_dev *hdev)
652 {
653 	struct stm32_hash_request_ctx *rctx = ahash_request_ctx(hdev->req);
654 	u32 *buffer = (void *)rctx->state.buffer;
655 	struct scatterlist sg[1], *tsg;
656 	int err = 0, reg, ncp = 0;
657 	unsigned int i, len = 0, bufcnt = 0;
658 	bool is_last = false;
659 
660 	rctx->sg = hdev->req->src;
661 	rctx->total = hdev->req->nbytes;
662 
663 	rctx->nents = sg_nents(rctx->sg);
664 	if (rctx->nents < 0)
665 		return -EINVAL;
666 
667 	stm32_hash_write_ctrl(hdev);
668 
669 	if (hdev->flags & HASH_FLAGS_HMAC) {
670 		err = stm32_hash_hmac_dma_send(hdev);
671 		if (err != -EINPROGRESS)
672 			return err;
673 	}
674 
675 	for_each_sg(rctx->sg, tsg, rctx->nents, i) {
676 		sg[0] = *tsg;
677 		len = sg->length;
678 
679 		if (sg_is_last(sg) || (bufcnt + sg[0].length) >= rctx->total) {
680 			sg->length = rctx->total - bufcnt;
681 			is_last = true;
682 			if (hdev->dma_mode == 1) {
683 				len = (ALIGN(sg->length, 16) - 16);
684 
685 				ncp = sg_pcopy_to_buffer(
686 					rctx->sg, rctx->nents,
687 					rctx->state.buffer, sg->length - len,
688 					rctx->total - sg->length + len);
689 
690 				sg->length = len;
691 			} else {
692 				if (!(IS_ALIGNED(sg->length, sizeof(u32)))) {
693 					len = sg->length;
694 					sg->length = ALIGN(sg->length,
695 							   sizeof(u32));
696 				}
697 			}
698 		}
699 
700 		rctx->dma_ct = dma_map_sg(hdev->dev, sg, 1,
701 					  DMA_TO_DEVICE);
702 		if (rctx->dma_ct == 0) {
703 			dev_err(hdev->dev, "dma_map_sg error\n");
704 			return -ENOMEM;
705 		}
706 
707 		err = stm32_hash_xmit_dma(hdev, sg, len, !is_last);
708 
709 		bufcnt += sg[0].length;
710 		dma_unmap_sg(hdev->dev, sg, 1, DMA_TO_DEVICE);
711 
712 		if (err == -ENOMEM)
713 			return err;
714 		if (is_last)
715 			break;
716 	}
717 
718 	if (hdev->dma_mode == 1) {
719 		if (stm32_hash_wait_busy(hdev))
720 			return -ETIMEDOUT;
721 		reg = stm32_hash_read(hdev, HASH_CR);
722 		reg &= ~HASH_CR_DMAE;
723 		reg |= HASH_CR_DMAA;
724 		stm32_hash_write(hdev, HASH_CR, reg);
725 
726 		if (ncp) {
727 			memset(buffer + ncp, 0,
728 			       DIV_ROUND_UP(ncp, sizeof(u32)) - ncp);
729 			writesl(hdev->io_base + HASH_DIN, buffer,
730 				DIV_ROUND_UP(ncp, sizeof(u32)));
731 		}
732 		stm32_hash_set_nblw(hdev, ncp);
733 		reg = stm32_hash_read(hdev, HASH_STR);
734 		reg |= HASH_STR_DCAL;
735 		stm32_hash_write(hdev, HASH_STR, reg);
736 		err = -EINPROGRESS;
737 	}
738 
739 	if (hdev->flags & HASH_FLAGS_HMAC) {
740 		if (stm32_hash_wait_busy(hdev))
741 			return -ETIMEDOUT;
742 		err = stm32_hash_hmac_dma_send(hdev);
743 	}
744 
745 	return err;
746 }
747 
748 static struct stm32_hash_dev *stm32_hash_find_dev(struct stm32_hash_ctx *ctx)
749 {
750 	struct stm32_hash_dev *hdev = NULL, *tmp;
751 
752 	spin_lock_bh(&stm32_hash.lock);
753 	if (!ctx->hdev) {
754 		list_for_each_entry(tmp, &stm32_hash.dev_list, list) {
755 			hdev = tmp;
756 			break;
757 		}
758 		ctx->hdev = hdev;
759 	} else {
760 		hdev = ctx->hdev;
761 	}
762 
763 	spin_unlock_bh(&stm32_hash.lock);
764 
765 	return hdev;
766 }
767 
768 static bool stm32_hash_dma_aligned_data(struct ahash_request *req)
769 {
770 	struct scatterlist *sg;
771 	struct stm32_hash_request_ctx *rctx = ahash_request_ctx(req);
772 	struct stm32_hash_ctx *ctx = crypto_ahash_ctx(crypto_ahash_reqtfm(req));
773 	struct stm32_hash_dev *hdev = stm32_hash_find_dev(ctx);
774 	int i;
775 
776 	if (!hdev->dma_lch || req->nbytes <= rctx->state.blocklen)
777 		return false;
778 
779 	if (sg_nents(req->src) > 1) {
780 		if (hdev->dma_mode == 1)
781 			return false;
782 		for_each_sg(req->src, sg, sg_nents(req->src), i) {
783 			if ((!IS_ALIGNED(sg->length, sizeof(u32))) &&
784 			    (!sg_is_last(sg)))
785 				return false;
786 		}
787 	}
788 
789 	if (req->src->offset % 4)
790 		return false;
791 
792 	return true;
793 }
794 
795 static int stm32_hash_init(struct ahash_request *req)
796 {
797 	struct crypto_ahash *tfm = crypto_ahash_reqtfm(req);
798 	struct stm32_hash_ctx *ctx = crypto_ahash_ctx(tfm);
799 	struct stm32_hash_request_ctx *rctx = ahash_request_ctx(req);
800 	struct stm32_hash_dev *hdev = stm32_hash_find_dev(ctx);
801 	struct stm32_hash_state *state = &rctx->state;
802 	bool sha3_mode = ctx->flags & HASH_FLAGS_SHA3_MODE;
803 
804 	rctx->hdev = hdev;
805 
806 	state->flags = HASH_FLAGS_CPU;
807 
808 	if (sha3_mode)
809 		state->flags |= HASH_FLAGS_SHA3_MODE;
810 
811 	rctx->digcnt = crypto_ahash_digestsize(tfm);
812 	switch (rctx->digcnt) {
813 	case MD5_DIGEST_SIZE:
814 		state->flags |= HASH_MD5 << HASH_FLAGS_ALGO_SHIFT;
815 		break;
816 	case SHA1_DIGEST_SIZE:
817 		if (hdev->pdata->ux500)
818 			state->flags |= HASH_SHA1_UX500 << HASH_FLAGS_ALGO_SHIFT;
819 		else
820 			state->flags |= HASH_SHA1 << HASH_FLAGS_ALGO_SHIFT;
821 		break;
822 	case SHA224_DIGEST_SIZE:
823 		if (sha3_mode)
824 			state->flags |= HASH_SHA3_224 << HASH_FLAGS_ALGO_SHIFT;
825 		else
826 			state->flags |= HASH_SHA224 << HASH_FLAGS_ALGO_SHIFT;
827 		break;
828 	case SHA256_DIGEST_SIZE:
829 		if (sha3_mode) {
830 			state->flags |= HASH_SHA3_256 << HASH_FLAGS_ALGO_SHIFT;
831 		} else {
832 			if (hdev->pdata->ux500)
833 				state->flags |= HASH_SHA256_UX500 << HASH_FLAGS_ALGO_SHIFT;
834 			else
835 				state->flags |= HASH_SHA256 << HASH_FLAGS_ALGO_SHIFT;
836 		}
837 		break;
838 	case SHA384_DIGEST_SIZE:
839 		if (sha3_mode)
840 			state->flags |= HASH_SHA3_384 << HASH_FLAGS_ALGO_SHIFT;
841 		else
842 			state->flags |= HASH_SHA384 << HASH_FLAGS_ALGO_SHIFT;
843 		break;
844 	case SHA512_DIGEST_SIZE:
845 		if (sha3_mode)
846 			state->flags |= HASH_SHA3_512 << HASH_FLAGS_ALGO_SHIFT;
847 		else
848 			state->flags |= HASH_SHA512 << HASH_FLAGS_ALGO_SHIFT;
849 		break;
850 	default:
851 		return -EINVAL;
852 	}
853 
854 	rctx->state.bufcnt = 0;
855 	rctx->state.blocklen = crypto_ahash_blocksize(tfm) + sizeof(u32);
856 	if (rctx->state.blocklen > HASH_BUFLEN) {
857 		dev_err(hdev->dev, "Error, block too large");
858 		return -EINVAL;
859 	}
860 	rctx->total = 0;
861 	rctx->offset = 0;
862 	rctx->data_type = HASH_DATA_8_BITS;
863 
864 	if (ctx->flags & HASH_FLAGS_HMAC)
865 		state->flags |= HASH_FLAGS_HMAC;
866 
867 	dev_dbg(hdev->dev, "%s Flags %x\n", __func__, state->flags);
868 
869 	return 0;
870 }
871 
872 static int stm32_hash_update_req(struct stm32_hash_dev *hdev)
873 {
874 	struct stm32_hash_request_ctx *rctx = ahash_request_ctx(hdev->req);
875 	struct stm32_hash_state *state = &rctx->state;
876 
877 	if (!(state->flags & HASH_FLAGS_CPU))
878 		return stm32_hash_dma_send(hdev);
879 
880 	return stm32_hash_update_cpu(hdev);
881 }
882 
883 static int stm32_hash_final_req(struct stm32_hash_dev *hdev)
884 {
885 	struct ahash_request *req = hdev->req;
886 	struct stm32_hash_request_ctx *rctx = ahash_request_ctx(req);
887 	struct stm32_hash_state *state = &rctx->state;
888 	int buflen = state->bufcnt;
889 
890 	if (state->flags & HASH_FLAGS_FINUP)
891 		return stm32_hash_update_req(hdev);
892 
893 	state->bufcnt = 0;
894 
895 	return stm32_hash_xmit_cpu(hdev, state->buffer, buflen, 1);
896 }
897 
898 static void stm32_hash_emptymsg_fallback(struct ahash_request *req)
899 {
900 	struct crypto_ahash *ahash = crypto_ahash_reqtfm(req);
901 	struct stm32_hash_ctx *ctx = crypto_ahash_ctx(ahash);
902 	struct stm32_hash_request_ctx *rctx = ahash_request_ctx(req);
903 	struct stm32_hash_dev *hdev = rctx->hdev;
904 	int ret;
905 
906 	dev_dbg(hdev->dev, "use fallback message size 0 key size %d\n",
907 		ctx->keylen);
908 
909 	if (!ctx->xtfm) {
910 		dev_err(hdev->dev, "no fallback engine\n");
911 		return;
912 	}
913 
914 	if (ctx->keylen) {
915 		ret = crypto_shash_setkey(ctx->xtfm, ctx->key, ctx->keylen);
916 		if (ret) {
917 			dev_err(hdev->dev, "failed to set key ret=%d\n", ret);
918 			return;
919 		}
920 	}
921 
922 	ret = crypto_shash_tfm_digest(ctx->xtfm, NULL, 0, rctx->digest);
923 	if (ret)
924 		dev_err(hdev->dev, "shash digest error\n");
925 }
926 
927 static void stm32_hash_copy_hash(struct ahash_request *req)
928 {
929 	struct crypto_ahash *tfm = crypto_ahash_reqtfm(req);
930 	struct stm32_hash_request_ctx *rctx = ahash_request_ctx(req);
931 	struct stm32_hash_state *state = &rctx->state;
932 	struct stm32_hash_dev *hdev = rctx->hdev;
933 	__be32 *hash = (void *)rctx->digest;
934 	unsigned int i, hashsize;
935 
936 	if (hdev->pdata->broken_emptymsg && (state->flags & HASH_FLAGS_EMPTY))
937 		return stm32_hash_emptymsg_fallback(req);
938 
939 	hashsize = crypto_ahash_digestsize(tfm);
940 
941 	for (i = 0; i < hashsize / sizeof(u32); i++) {
942 		if (hdev->pdata->ux500)
943 			hash[i] = cpu_to_be32(stm32_hash_read(hdev,
944 					      HASH_UX500_HREG(i)));
945 		else
946 			hash[i] = cpu_to_be32(stm32_hash_read(hdev,
947 					      HASH_HREG(i)));
948 	}
949 }
950 
951 static int stm32_hash_finish(struct ahash_request *req)
952 {
953 	struct stm32_hash_request_ctx *rctx = ahash_request_ctx(req);
954 	u32 reg;
955 
956 	reg = stm32_hash_read(rctx->hdev, HASH_SR);
957 	reg &= ~HASH_SR_OUTPUT_READY;
958 	stm32_hash_write(rctx->hdev, HASH_SR, reg);
959 
960 	if (!req->result)
961 		return -EINVAL;
962 
963 	memcpy(req->result, rctx->digest, rctx->digcnt);
964 
965 	return 0;
966 }
967 
968 static void stm32_hash_finish_req(struct ahash_request *req, int err)
969 {
970 	struct stm32_hash_request_ctx *rctx = ahash_request_ctx(req);
971 	struct stm32_hash_dev *hdev = rctx->hdev;
972 
973 	if (!err && (HASH_FLAGS_FINAL & hdev->flags)) {
974 		stm32_hash_copy_hash(req);
975 		err = stm32_hash_finish(req);
976 	}
977 
978 	pm_runtime_mark_last_busy(hdev->dev);
979 	pm_runtime_put_autosuspend(hdev->dev);
980 
981 	crypto_finalize_hash_request(hdev->engine, req, err);
982 }
983 
984 static int stm32_hash_handle_queue(struct stm32_hash_dev *hdev,
985 				   struct ahash_request *req)
986 {
987 	return crypto_transfer_hash_request_to_engine(hdev->engine, req);
988 }
989 
990 static int stm32_hash_one_request(struct crypto_engine *engine, void *areq)
991 {
992 	struct ahash_request *req = container_of(areq, struct ahash_request,
993 						 base);
994 	struct stm32_hash_ctx *ctx = crypto_ahash_ctx(crypto_ahash_reqtfm(req));
995 	struct stm32_hash_request_ctx *rctx = ahash_request_ctx(req);
996 	struct stm32_hash_dev *hdev = stm32_hash_find_dev(ctx);
997 	struct stm32_hash_state *state = &rctx->state;
998 	int swap_reg;
999 	int err = 0;
1000 
1001 	if (!hdev)
1002 		return -ENODEV;
1003 
1004 	dev_dbg(hdev->dev, "processing new req, op: %lu, nbytes %d\n",
1005 		rctx->op, req->nbytes);
1006 
1007 	pm_runtime_get_sync(hdev->dev);
1008 
1009 	hdev->req = req;
1010 	hdev->flags = 0;
1011 	swap_reg = hash_swap_reg(rctx);
1012 
1013 	if (state->flags & HASH_FLAGS_INIT) {
1014 		u32 *preg = rctx->state.hw_context;
1015 		u32 reg;
1016 		int i;
1017 
1018 		if (!hdev->pdata->ux500)
1019 			stm32_hash_write(hdev, HASH_IMR, *preg++);
1020 		stm32_hash_write(hdev, HASH_STR, *preg++);
1021 		stm32_hash_write(hdev, HASH_CR, *preg);
1022 		reg = *preg++ | HASH_CR_INIT;
1023 		stm32_hash_write(hdev, HASH_CR, reg);
1024 
1025 		for (i = 0; i < swap_reg; i++)
1026 			stm32_hash_write(hdev, HASH_CSR(i), *preg++);
1027 
1028 		hdev->flags |= HASH_FLAGS_INIT;
1029 
1030 		if (state->flags & HASH_FLAGS_HMAC)
1031 			hdev->flags |= HASH_FLAGS_HMAC |
1032 				       HASH_FLAGS_HMAC_KEY;
1033 	}
1034 
1035 	if (rctx->op == HASH_OP_UPDATE)
1036 		err = stm32_hash_update_req(hdev);
1037 	else if (rctx->op == HASH_OP_FINAL)
1038 		err = stm32_hash_final_req(hdev);
1039 
1040 	/* If we have an IRQ, wait for that, else poll for completion */
1041 	if (err == -EINPROGRESS && hdev->polled) {
1042 		if (stm32_hash_wait_busy(hdev))
1043 			err = -ETIMEDOUT;
1044 		else {
1045 			hdev->flags |= HASH_FLAGS_OUTPUT_READY;
1046 			err = 0;
1047 		}
1048 	}
1049 
1050 	if (err != -EINPROGRESS)
1051 	/* done task will not finish it, so do it here */
1052 		stm32_hash_finish_req(req, err);
1053 
1054 	return 0;
1055 }
1056 
1057 static int stm32_hash_enqueue(struct ahash_request *req, unsigned int op)
1058 {
1059 	struct stm32_hash_request_ctx *rctx = ahash_request_ctx(req);
1060 	struct stm32_hash_ctx *ctx = crypto_tfm_ctx(req->base.tfm);
1061 	struct stm32_hash_dev *hdev = ctx->hdev;
1062 
1063 	rctx->op = op;
1064 
1065 	return stm32_hash_handle_queue(hdev, req);
1066 }
1067 
1068 static int stm32_hash_update(struct ahash_request *req)
1069 {
1070 	struct stm32_hash_request_ctx *rctx = ahash_request_ctx(req);
1071 	struct stm32_hash_state *state = &rctx->state;
1072 
1073 	if (!req->nbytes || !(state->flags & HASH_FLAGS_CPU))
1074 		return 0;
1075 
1076 	rctx->total = req->nbytes;
1077 	rctx->sg = req->src;
1078 	rctx->offset = 0;
1079 
1080 	if ((state->bufcnt + rctx->total < state->blocklen)) {
1081 		stm32_hash_append_sg(rctx);
1082 		return 0;
1083 	}
1084 
1085 	return stm32_hash_enqueue(req, HASH_OP_UPDATE);
1086 }
1087 
1088 static int stm32_hash_final(struct ahash_request *req)
1089 {
1090 	struct stm32_hash_request_ctx *rctx = ahash_request_ctx(req);
1091 	struct stm32_hash_state *state = &rctx->state;
1092 
1093 	state->flags |= HASH_FLAGS_FINAL;
1094 
1095 	return stm32_hash_enqueue(req, HASH_OP_FINAL);
1096 }
1097 
1098 static int stm32_hash_finup(struct ahash_request *req)
1099 {
1100 	struct stm32_hash_request_ctx *rctx = ahash_request_ctx(req);
1101 	struct stm32_hash_ctx *ctx = crypto_ahash_ctx(crypto_ahash_reqtfm(req));
1102 	struct stm32_hash_dev *hdev = stm32_hash_find_dev(ctx);
1103 	struct stm32_hash_state *state = &rctx->state;
1104 
1105 	if (!req->nbytes)
1106 		goto out;
1107 
1108 	state->flags |= HASH_FLAGS_FINUP;
1109 	rctx->total = req->nbytes;
1110 	rctx->sg = req->src;
1111 	rctx->offset = 0;
1112 
1113 	if (hdev->dma_lch && stm32_hash_dma_aligned_data(req))
1114 		state->flags &= ~HASH_FLAGS_CPU;
1115 
1116 out:
1117 	return stm32_hash_final(req);
1118 }
1119 
1120 static int stm32_hash_digest(struct ahash_request *req)
1121 {
1122 	return stm32_hash_init(req) ?: stm32_hash_finup(req);
1123 }
1124 
1125 static int stm32_hash_export(struct ahash_request *req, void *out)
1126 {
1127 	struct stm32_hash_request_ctx *rctx = ahash_request_ctx(req);
1128 
1129 	memcpy(out, &rctx->state, sizeof(rctx->state));
1130 
1131 	return 0;
1132 }
1133 
1134 static int stm32_hash_import(struct ahash_request *req, const void *in)
1135 {
1136 	struct stm32_hash_request_ctx *rctx = ahash_request_ctx(req);
1137 
1138 	stm32_hash_init(req);
1139 	memcpy(&rctx->state, in, sizeof(rctx->state));
1140 
1141 	return 0;
1142 }
1143 
1144 static int stm32_hash_setkey(struct crypto_ahash *tfm,
1145 			     const u8 *key, unsigned int keylen)
1146 {
1147 	struct stm32_hash_ctx *ctx = crypto_ahash_ctx(tfm);
1148 
1149 	if (keylen <= HASH_MAX_KEY_SIZE) {
1150 		memcpy(ctx->key, key, keylen);
1151 		ctx->keylen = keylen;
1152 	} else {
1153 		return -ENOMEM;
1154 	}
1155 
1156 	return 0;
1157 }
1158 
1159 static int stm32_hash_init_fallback(struct crypto_tfm *tfm)
1160 {
1161 	struct stm32_hash_ctx *ctx = crypto_tfm_ctx(tfm);
1162 	struct stm32_hash_dev *hdev = stm32_hash_find_dev(ctx);
1163 	const char *name = crypto_tfm_alg_name(tfm);
1164 	struct crypto_shash *xtfm;
1165 
1166 	/* The fallback is only needed on Ux500 */
1167 	if (!hdev->pdata->ux500)
1168 		return 0;
1169 
1170 	xtfm = crypto_alloc_shash(name, 0, CRYPTO_ALG_NEED_FALLBACK);
1171 	if (IS_ERR(xtfm)) {
1172 		dev_err(hdev->dev, "failed to allocate %s fallback\n",
1173 			name);
1174 		return PTR_ERR(xtfm);
1175 	}
1176 	dev_info(hdev->dev, "allocated %s fallback\n", name);
1177 	ctx->xtfm = xtfm;
1178 
1179 	return 0;
1180 }
1181 
1182 static int stm32_hash_cra_init_algs(struct crypto_tfm *tfm, u32 algs_flags)
1183 {
1184 	struct stm32_hash_ctx *ctx = crypto_tfm_ctx(tfm);
1185 
1186 	crypto_ahash_set_reqsize(__crypto_ahash_cast(tfm),
1187 				 sizeof(struct stm32_hash_request_ctx));
1188 
1189 	ctx->keylen = 0;
1190 
1191 	if (algs_flags)
1192 		ctx->flags |= algs_flags;
1193 
1194 	return stm32_hash_init_fallback(tfm);
1195 }
1196 
1197 static int stm32_hash_cra_init(struct crypto_tfm *tfm)
1198 {
1199 	return stm32_hash_cra_init_algs(tfm, 0);
1200 }
1201 
1202 static int stm32_hash_cra_hmac_init(struct crypto_tfm *tfm)
1203 {
1204 	return stm32_hash_cra_init_algs(tfm, HASH_FLAGS_HMAC);
1205 }
1206 
1207 static int stm32_hash_cra_sha3_init(struct crypto_tfm *tfm)
1208 {
1209 	return stm32_hash_cra_init_algs(tfm, HASH_FLAGS_SHA3_MODE);
1210 }
1211 
1212 static int stm32_hash_cra_sha3_hmac_init(struct crypto_tfm *tfm)
1213 {
1214 	return stm32_hash_cra_init_algs(tfm, HASH_FLAGS_SHA3_MODE |
1215 					HASH_FLAGS_HMAC);
1216 }
1217 
1218 
1219 static void stm32_hash_cra_exit(struct crypto_tfm *tfm)
1220 {
1221 	struct stm32_hash_ctx *ctx = crypto_tfm_ctx(tfm);
1222 
1223 	if (ctx->xtfm)
1224 		crypto_free_shash(ctx->xtfm);
1225 }
1226 
1227 static irqreturn_t stm32_hash_irq_thread(int irq, void *dev_id)
1228 {
1229 	struct stm32_hash_dev *hdev = dev_id;
1230 
1231 	if (HASH_FLAGS_CPU & hdev->flags) {
1232 		if (HASH_FLAGS_OUTPUT_READY & hdev->flags) {
1233 			hdev->flags &= ~HASH_FLAGS_OUTPUT_READY;
1234 			goto finish;
1235 		}
1236 	} else if (HASH_FLAGS_DMA_ACTIVE & hdev->flags) {
1237 		hdev->flags &= ~HASH_FLAGS_DMA_ACTIVE;
1238 			goto finish;
1239 	}
1240 
1241 	return IRQ_HANDLED;
1242 
1243 finish:
1244 	/* Finish current request */
1245 	stm32_hash_finish_req(hdev->req, 0);
1246 
1247 	return IRQ_HANDLED;
1248 }
1249 
1250 static irqreturn_t stm32_hash_irq_handler(int irq, void *dev_id)
1251 {
1252 	struct stm32_hash_dev *hdev = dev_id;
1253 	u32 reg;
1254 
1255 	reg = stm32_hash_read(hdev, HASH_SR);
1256 	if (reg & HASH_SR_OUTPUT_READY) {
1257 		hdev->flags |= HASH_FLAGS_OUTPUT_READY;
1258 		/* Disable IT*/
1259 		stm32_hash_write(hdev, HASH_IMR, 0);
1260 		return IRQ_WAKE_THREAD;
1261 	}
1262 
1263 	return IRQ_NONE;
1264 }
1265 
1266 static struct ahash_engine_alg algs_md5[] = {
1267 	{
1268 		.base.init = stm32_hash_init,
1269 		.base.update = stm32_hash_update,
1270 		.base.final = stm32_hash_final,
1271 		.base.finup = stm32_hash_finup,
1272 		.base.digest = stm32_hash_digest,
1273 		.base.export = stm32_hash_export,
1274 		.base.import = stm32_hash_import,
1275 		.base.halg = {
1276 			.digestsize = MD5_DIGEST_SIZE,
1277 			.statesize = sizeof(struct stm32_hash_state),
1278 			.base = {
1279 				.cra_name = "md5",
1280 				.cra_driver_name = "stm32-md5",
1281 				.cra_priority = 200,
1282 				.cra_flags = CRYPTO_ALG_ASYNC |
1283 					CRYPTO_ALG_KERN_DRIVER_ONLY,
1284 				.cra_blocksize = MD5_HMAC_BLOCK_SIZE,
1285 				.cra_ctxsize = sizeof(struct stm32_hash_ctx),
1286 				.cra_alignmask = 3,
1287 				.cra_init = stm32_hash_cra_init,
1288 				.cra_exit = stm32_hash_cra_exit,
1289 				.cra_module = THIS_MODULE,
1290 			}
1291 		},
1292 		.op = {
1293 			.do_one_request = stm32_hash_one_request,
1294 		},
1295 	},
1296 	{
1297 		.base.init = stm32_hash_init,
1298 		.base.update = stm32_hash_update,
1299 		.base.final = stm32_hash_final,
1300 		.base.finup = stm32_hash_finup,
1301 		.base.digest = stm32_hash_digest,
1302 		.base.export = stm32_hash_export,
1303 		.base.import = stm32_hash_import,
1304 		.base.setkey = stm32_hash_setkey,
1305 		.base.halg = {
1306 			.digestsize = MD5_DIGEST_SIZE,
1307 			.statesize = sizeof(struct stm32_hash_state),
1308 			.base = {
1309 				.cra_name = "hmac(md5)",
1310 				.cra_driver_name = "stm32-hmac-md5",
1311 				.cra_priority = 200,
1312 				.cra_flags = CRYPTO_ALG_ASYNC |
1313 					CRYPTO_ALG_KERN_DRIVER_ONLY,
1314 				.cra_blocksize = MD5_HMAC_BLOCK_SIZE,
1315 				.cra_ctxsize = sizeof(struct stm32_hash_ctx),
1316 				.cra_alignmask = 3,
1317 				.cra_init = stm32_hash_cra_hmac_init,
1318 				.cra_exit = stm32_hash_cra_exit,
1319 				.cra_module = THIS_MODULE,
1320 			}
1321 		},
1322 		.op = {
1323 			.do_one_request = stm32_hash_one_request,
1324 		},
1325 	}
1326 };
1327 
1328 static struct ahash_engine_alg algs_sha1[] = {
1329 	{
1330 		.base.init = stm32_hash_init,
1331 		.base.update = stm32_hash_update,
1332 		.base.final = stm32_hash_final,
1333 		.base.finup = stm32_hash_finup,
1334 		.base.digest = stm32_hash_digest,
1335 		.base.export = stm32_hash_export,
1336 		.base.import = stm32_hash_import,
1337 		.base.halg = {
1338 			.digestsize = SHA1_DIGEST_SIZE,
1339 			.statesize = sizeof(struct stm32_hash_state),
1340 			.base = {
1341 				.cra_name = "sha1",
1342 				.cra_driver_name = "stm32-sha1",
1343 				.cra_priority = 200,
1344 				.cra_flags = CRYPTO_ALG_ASYNC |
1345 					CRYPTO_ALG_KERN_DRIVER_ONLY,
1346 				.cra_blocksize = SHA1_BLOCK_SIZE,
1347 				.cra_ctxsize = sizeof(struct stm32_hash_ctx),
1348 				.cra_alignmask = 3,
1349 				.cra_init = stm32_hash_cra_init,
1350 				.cra_exit = stm32_hash_cra_exit,
1351 				.cra_module = THIS_MODULE,
1352 			}
1353 		},
1354 		.op = {
1355 			.do_one_request = stm32_hash_one_request,
1356 		},
1357 	},
1358 	{
1359 		.base.init = stm32_hash_init,
1360 		.base.update = stm32_hash_update,
1361 		.base.final = stm32_hash_final,
1362 		.base.finup = stm32_hash_finup,
1363 		.base.digest = stm32_hash_digest,
1364 		.base.export = stm32_hash_export,
1365 		.base.import = stm32_hash_import,
1366 		.base.setkey = stm32_hash_setkey,
1367 		.base.halg = {
1368 			.digestsize = SHA1_DIGEST_SIZE,
1369 			.statesize = sizeof(struct stm32_hash_state),
1370 			.base = {
1371 				.cra_name = "hmac(sha1)",
1372 				.cra_driver_name = "stm32-hmac-sha1",
1373 				.cra_priority = 200,
1374 				.cra_flags = CRYPTO_ALG_ASYNC |
1375 					CRYPTO_ALG_KERN_DRIVER_ONLY,
1376 				.cra_blocksize = SHA1_BLOCK_SIZE,
1377 				.cra_ctxsize = sizeof(struct stm32_hash_ctx),
1378 				.cra_alignmask = 3,
1379 				.cra_init = stm32_hash_cra_hmac_init,
1380 				.cra_exit = stm32_hash_cra_exit,
1381 				.cra_module = THIS_MODULE,
1382 			}
1383 		},
1384 		.op = {
1385 			.do_one_request = stm32_hash_one_request,
1386 		},
1387 	},
1388 };
1389 
1390 static struct ahash_engine_alg algs_sha224[] = {
1391 	{
1392 		.base.init = stm32_hash_init,
1393 		.base.update = stm32_hash_update,
1394 		.base.final = stm32_hash_final,
1395 		.base.finup = stm32_hash_finup,
1396 		.base.digest = stm32_hash_digest,
1397 		.base.export = stm32_hash_export,
1398 		.base.import = stm32_hash_import,
1399 		.base.halg = {
1400 			.digestsize = SHA224_DIGEST_SIZE,
1401 			.statesize = sizeof(struct stm32_hash_state),
1402 			.base = {
1403 				.cra_name = "sha224",
1404 				.cra_driver_name = "stm32-sha224",
1405 				.cra_priority = 200,
1406 				.cra_flags = CRYPTO_ALG_ASYNC |
1407 					CRYPTO_ALG_KERN_DRIVER_ONLY,
1408 				.cra_blocksize = SHA224_BLOCK_SIZE,
1409 				.cra_ctxsize = sizeof(struct stm32_hash_ctx),
1410 				.cra_alignmask = 3,
1411 				.cra_init = stm32_hash_cra_init,
1412 				.cra_exit = stm32_hash_cra_exit,
1413 				.cra_module = THIS_MODULE,
1414 			}
1415 		},
1416 		.op = {
1417 			.do_one_request = stm32_hash_one_request,
1418 		},
1419 	},
1420 	{
1421 		.base.init = stm32_hash_init,
1422 		.base.update = stm32_hash_update,
1423 		.base.final = stm32_hash_final,
1424 		.base.finup = stm32_hash_finup,
1425 		.base.digest = stm32_hash_digest,
1426 		.base.setkey = stm32_hash_setkey,
1427 		.base.export = stm32_hash_export,
1428 		.base.import = stm32_hash_import,
1429 		.base.halg = {
1430 			.digestsize = SHA224_DIGEST_SIZE,
1431 			.statesize = sizeof(struct stm32_hash_state),
1432 			.base = {
1433 				.cra_name = "hmac(sha224)",
1434 				.cra_driver_name = "stm32-hmac-sha224",
1435 				.cra_priority = 200,
1436 				.cra_flags = CRYPTO_ALG_ASYNC |
1437 					CRYPTO_ALG_KERN_DRIVER_ONLY,
1438 				.cra_blocksize = SHA224_BLOCK_SIZE,
1439 				.cra_ctxsize = sizeof(struct stm32_hash_ctx),
1440 				.cra_alignmask = 3,
1441 				.cra_init = stm32_hash_cra_hmac_init,
1442 				.cra_exit = stm32_hash_cra_exit,
1443 				.cra_module = THIS_MODULE,
1444 			}
1445 		},
1446 		.op = {
1447 			.do_one_request = stm32_hash_one_request,
1448 		},
1449 	},
1450 };
1451 
1452 static struct ahash_engine_alg algs_sha256[] = {
1453 	{
1454 		.base.init = stm32_hash_init,
1455 		.base.update = stm32_hash_update,
1456 		.base.final = stm32_hash_final,
1457 		.base.finup = stm32_hash_finup,
1458 		.base.digest = stm32_hash_digest,
1459 		.base.export = stm32_hash_export,
1460 		.base.import = stm32_hash_import,
1461 		.base.halg = {
1462 			.digestsize = SHA256_DIGEST_SIZE,
1463 			.statesize = sizeof(struct stm32_hash_state),
1464 			.base = {
1465 				.cra_name = "sha256",
1466 				.cra_driver_name = "stm32-sha256",
1467 				.cra_priority = 200,
1468 				.cra_flags = CRYPTO_ALG_ASYNC |
1469 					CRYPTO_ALG_KERN_DRIVER_ONLY,
1470 				.cra_blocksize = SHA256_BLOCK_SIZE,
1471 				.cra_ctxsize = sizeof(struct stm32_hash_ctx),
1472 				.cra_alignmask = 3,
1473 				.cra_init = stm32_hash_cra_init,
1474 				.cra_exit = stm32_hash_cra_exit,
1475 				.cra_module = THIS_MODULE,
1476 			}
1477 		},
1478 		.op = {
1479 			.do_one_request = stm32_hash_one_request,
1480 		},
1481 	},
1482 	{
1483 		.base.init = stm32_hash_init,
1484 		.base.update = stm32_hash_update,
1485 		.base.final = stm32_hash_final,
1486 		.base.finup = stm32_hash_finup,
1487 		.base.digest = stm32_hash_digest,
1488 		.base.export = stm32_hash_export,
1489 		.base.import = stm32_hash_import,
1490 		.base.setkey = stm32_hash_setkey,
1491 		.base.halg = {
1492 			.digestsize = SHA256_DIGEST_SIZE,
1493 			.statesize = sizeof(struct stm32_hash_state),
1494 			.base = {
1495 				.cra_name = "hmac(sha256)",
1496 				.cra_driver_name = "stm32-hmac-sha256",
1497 				.cra_priority = 200,
1498 				.cra_flags = CRYPTO_ALG_ASYNC |
1499 					CRYPTO_ALG_KERN_DRIVER_ONLY,
1500 				.cra_blocksize = SHA256_BLOCK_SIZE,
1501 				.cra_ctxsize = sizeof(struct stm32_hash_ctx),
1502 				.cra_alignmask = 3,
1503 				.cra_init = stm32_hash_cra_hmac_init,
1504 				.cra_exit = stm32_hash_cra_exit,
1505 				.cra_module = THIS_MODULE,
1506 			}
1507 		},
1508 		.op = {
1509 			.do_one_request = stm32_hash_one_request,
1510 		},
1511 	},
1512 };
1513 
1514 static struct ahash_engine_alg algs_sha384_sha512[] = {
1515 	{
1516 		.base.init = stm32_hash_init,
1517 		.base.update = stm32_hash_update,
1518 		.base.final = stm32_hash_final,
1519 		.base.finup = stm32_hash_finup,
1520 		.base.digest = stm32_hash_digest,
1521 		.base.export = stm32_hash_export,
1522 		.base.import = stm32_hash_import,
1523 		.base.halg = {
1524 			.digestsize = SHA384_DIGEST_SIZE,
1525 			.statesize = sizeof(struct stm32_hash_state),
1526 			.base = {
1527 				.cra_name = "sha384",
1528 				.cra_driver_name = "stm32-sha384",
1529 				.cra_priority = 200,
1530 				.cra_flags = CRYPTO_ALG_ASYNC |
1531 					CRYPTO_ALG_KERN_DRIVER_ONLY,
1532 				.cra_blocksize = SHA384_BLOCK_SIZE,
1533 				.cra_ctxsize = sizeof(struct stm32_hash_ctx),
1534 				.cra_alignmask = 3,
1535 				.cra_init = stm32_hash_cra_init,
1536 				.cra_exit = stm32_hash_cra_exit,
1537 				.cra_module = THIS_MODULE,
1538 			}
1539 		},
1540 		.op = {
1541 			.do_one_request = stm32_hash_one_request,
1542 		},
1543 	},
1544 	{
1545 		.base.init = stm32_hash_init,
1546 		.base.update = stm32_hash_update,
1547 		.base.final = stm32_hash_final,
1548 		.base.finup = stm32_hash_finup,
1549 		.base.digest = stm32_hash_digest,
1550 		.base.setkey = stm32_hash_setkey,
1551 		.base.export = stm32_hash_export,
1552 		.base.import = stm32_hash_import,
1553 		.base.halg = {
1554 			.digestsize = SHA384_DIGEST_SIZE,
1555 			.statesize = sizeof(struct stm32_hash_state),
1556 			.base = {
1557 				.cra_name = "hmac(sha384)",
1558 				.cra_driver_name = "stm32-hmac-sha384",
1559 				.cra_priority = 200,
1560 				.cra_flags = CRYPTO_ALG_ASYNC |
1561 					CRYPTO_ALG_KERN_DRIVER_ONLY,
1562 				.cra_blocksize = SHA384_BLOCK_SIZE,
1563 				.cra_ctxsize = sizeof(struct stm32_hash_ctx),
1564 				.cra_alignmask = 3,
1565 				.cra_init = stm32_hash_cra_hmac_init,
1566 				.cra_exit = stm32_hash_cra_exit,
1567 				.cra_module = THIS_MODULE,
1568 			}
1569 		},
1570 		.op = {
1571 			.do_one_request = stm32_hash_one_request,
1572 		},
1573 	},
1574 	{
1575 		.base.init = stm32_hash_init,
1576 		.base.update = stm32_hash_update,
1577 		.base.final = stm32_hash_final,
1578 		.base.finup = stm32_hash_finup,
1579 		.base.digest = stm32_hash_digest,
1580 		.base.export = stm32_hash_export,
1581 		.base.import = stm32_hash_import,
1582 		.base.halg = {
1583 			.digestsize = SHA512_DIGEST_SIZE,
1584 			.statesize = sizeof(struct stm32_hash_state),
1585 			.base = {
1586 				.cra_name = "sha512",
1587 				.cra_driver_name = "stm32-sha512",
1588 				.cra_priority = 200,
1589 				.cra_flags = CRYPTO_ALG_ASYNC |
1590 					CRYPTO_ALG_KERN_DRIVER_ONLY,
1591 				.cra_blocksize = SHA512_BLOCK_SIZE,
1592 				.cra_ctxsize = sizeof(struct stm32_hash_ctx),
1593 				.cra_alignmask = 3,
1594 				.cra_init = stm32_hash_cra_init,
1595 				.cra_exit = stm32_hash_cra_exit,
1596 				.cra_module = THIS_MODULE,
1597 			}
1598 		},
1599 		.op = {
1600 			.do_one_request = stm32_hash_one_request,
1601 		},
1602 	},
1603 	{
1604 		.base.init = stm32_hash_init,
1605 		.base.update = stm32_hash_update,
1606 		.base.final = stm32_hash_final,
1607 		.base.finup = stm32_hash_finup,
1608 		.base.digest = stm32_hash_digest,
1609 		.base.export = stm32_hash_export,
1610 		.base.import = stm32_hash_import,
1611 		.base.setkey = stm32_hash_setkey,
1612 		.base.halg = {
1613 			.digestsize = SHA512_DIGEST_SIZE,
1614 			.statesize = sizeof(struct stm32_hash_state),
1615 			.base = {
1616 				.cra_name = "hmac(sha512)",
1617 				.cra_driver_name = "stm32-hmac-sha512",
1618 				.cra_priority = 200,
1619 				.cra_flags = CRYPTO_ALG_ASYNC |
1620 					CRYPTO_ALG_KERN_DRIVER_ONLY,
1621 				.cra_blocksize = SHA512_BLOCK_SIZE,
1622 				.cra_ctxsize = sizeof(struct stm32_hash_ctx),
1623 				.cra_alignmask = 3,
1624 				.cra_init = stm32_hash_cra_hmac_init,
1625 				.cra_exit = stm32_hash_cra_exit,
1626 				.cra_module = THIS_MODULE,
1627 			}
1628 		},
1629 		.op = {
1630 			.do_one_request = stm32_hash_one_request,
1631 		},
1632 	},
1633 };
1634 
1635 static struct ahash_engine_alg algs_sha3[] = {
1636 	{
1637 		.base.init = stm32_hash_init,
1638 		.base.update = stm32_hash_update,
1639 		.base.final = stm32_hash_final,
1640 		.base.finup = stm32_hash_finup,
1641 		.base.digest = stm32_hash_digest,
1642 		.base.export = stm32_hash_export,
1643 		.base.import = stm32_hash_import,
1644 		.base.halg = {
1645 			.digestsize = SHA3_224_DIGEST_SIZE,
1646 			.statesize = sizeof(struct stm32_hash_state),
1647 			.base = {
1648 				.cra_name = "sha3-224",
1649 				.cra_driver_name = "stm32-sha3-224",
1650 				.cra_priority = 200,
1651 				.cra_flags = CRYPTO_ALG_ASYNC |
1652 					CRYPTO_ALG_KERN_DRIVER_ONLY,
1653 				.cra_blocksize = SHA3_224_BLOCK_SIZE,
1654 				.cra_ctxsize = sizeof(struct stm32_hash_ctx),
1655 				.cra_alignmask = 3,
1656 				.cra_init = stm32_hash_cra_sha3_init,
1657 				.cra_exit = stm32_hash_cra_exit,
1658 				.cra_module = THIS_MODULE,
1659 			}
1660 		},
1661 		.op = {
1662 			.do_one_request = stm32_hash_one_request,
1663 		},
1664 	},
1665 	{
1666 		.base.init = stm32_hash_init,
1667 		.base.update = stm32_hash_update,
1668 		.base.final = stm32_hash_final,
1669 		.base.finup = stm32_hash_finup,
1670 		.base.digest = stm32_hash_digest,
1671 		.base.export = stm32_hash_export,
1672 		.base.import = stm32_hash_import,
1673 		.base.setkey = stm32_hash_setkey,
1674 		.base.halg = {
1675 			.digestsize = SHA3_224_DIGEST_SIZE,
1676 			.statesize = sizeof(struct stm32_hash_state),
1677 			.base = {
1678 				.cra_name = "hmac(sha3-224)",
1679 				.cra_driver_name = "stm32-hmac-sha3-224",
1680 				.cra_priority = 200,
1681 				.cra_flags = CRYPTO_ALG_ASYNC |
1682 					CRYPTO_ALG_KERN_DRIVER_ONLY,
1683 				.cra_blocksize = SHA3_224_BLOCK_SIZE,
1684 				.cra_ctxsize = sizeof(struct stm32_hash_ctx),
1685 				.cra_alignmask = 3,
1686 				.cra_init = stm32_hash_cra_sha3_hmac_init,
1687 				.cra_exit = stm32_hash_cra_exit,
1688 				.cra_module = THIS_MODULE,
1689 			}
1690 		},
1691 		.op = {
1692 			.do_one_request = stm32_hash_one_request,
1693 		},
1694 	},
1695 	{
1696 		.base.init = stm32_hash_init,
1697 		.base.update = stm32_hash_update,
1698 		.base.final = stm32_hash_final,
1699 		.base.finup = stm32_hash_finup,
1700 		.base.digest = stm32_hash_digest,
1701 		.base.export = stm32_hash_export,
1702 		.base.import = stm32_hash_import,
1703 		.base.halg = {
1704 			.digestsize = SHA3_256_DIGEST_SIZE,
1705 			.statesize = sizeof(struct stm32_hash_state),
1706 			.base = {
1707 				.cra_name = "sha3-256",
1708 				.cra_driver_name = "stm32-sha3-256",
1709 				.cra_priority = 200,
1710 				.cra_flags = CRYPTO_ALG_ASYNC |
1711 					CRYPTO_ALG_KERN_DRIVER_ONLY,
1712 				.cra_blocksize = SHA3_256_BLOCK_SIZE,
1713 				.cra_ctxsize = sizeof(struct stm32_hash_ctx),
1714 				.cra_alignmask = 3,
1715 				.cra_init = stm32_hash_cra_sha3_init,
1716 				.cra_exit = stm32_hash_cra_exit,
1717 				.cra_module = THIS_MODULE,
1718 			}
1719 		},
1720 		.op = {
1721 			.do_one_request = stm32_hash_one_request,
1722 		},
1723 	},
1724 	{
1725 		.base.init = stm32_hash_init,
1726 		.base.update = stm32_hash_update,
1727 		.base.final = stm32_hash_final,
1728 		.base.finup = stm32_hash_finup,
1729 		.base.digest = stm32_hash_digest,
1730 		.base.export = stm32_hash_export,
1731 		.base.import = stm32_hash_import,
1732 		.base.setkey = stm32_hash_setkey,
1733 		.base.halg = {
1734 			.digestsize = SHA3_256_DIGEST_SIZE,
1735 			.statesize = sizeof(struct stm32_hash_state),
1736 			.base = {
1737 				.cra_name = "hmac(sha3-256)",
1738 				.cra_driver_name = "stm32-hmac-sha3-256",
1739 				.cra_priority = 200,
1740 				.cra_flags = CRYPTO_ALG_ASYNC |
1741 					CRYPTO_ALG_KERN_DRIVER_ONLY,
1742 				.cra_blocksize = SHA3_256_BLOCK_SIZE,
1743 				.cra_ctxsize = sizeof(struct stm32_hash_ctx),
1744 				.cra_alignmask = 3,
1745 				.cra_init = stm32_hash_cra_sha3_hmac_init,
1746 				.cra_exit = stm32_hash_cra_exit,
1747 				.cra_module = THIS_MODULE,
1748 			}
1749 		},
1750 		.op = {
1751 			.do_one_request = stm32_hash_one_request,
1752 		},
1753 	},
1754 	{
1755 		.base.init = stm32_hash_init,
1756 		.base.update = stm32_hash_update,
1757 		.base.final = stm32_hash_final,
1758 		.base.finup = stm32_hash_finup,
1759 		.base.digest = stm32_hash_digest,
1760 		.base.export = stm32_hash_export,
1761 		.base.import = stm32_hash_import,
1762 		.base.halg = {
1763 			.digestsize = SHA3_384_DIGEST_SIZE,
1764 			.statesize = sizeof(struct stm32_hash_state),
1765 			.base = {
1766 				.cra_name = "sha3-384",
1767 				.cra_driver_name = "stm32-sha3-384",
1768 				.cra_priority = 200,
1769 				.cra_flags = CRYPTO_ALG_ASYNC |
1770 					CRYPTO_ALG_KERN_DRIVER_ONLY,
1771 				.cra_blocksize = SHA3_384_BLOCK_SIZE,
1772 				.cra_ctxsize = sizeof(struct stm32_hash_ctx),
1773 				.cra_alignmask = 3,
1774 				.cra_init = stm32_hash_cra_sha3_init,
1775 				.cra_exit = stm32_hash_cra_exit,
1776 				.cra_module = THIS_MODULE,
1777 			}
1778 		},
1779 		.op = {
1780 			.do_one_request = stm32_hash_one_request,
1781 		},
1782 	},
1783 	{
1784 		.base.init = stm32_hash_init,
1785 		.base.update = stm32_hash_update,
1786 		.base.final = stm32_hash_final,
1787 		.base.finup = stm32_hash_finup,
1788 		.base.digest = stm32_hash_digest,
1789 		.base.export = stm32_hash_export,
1790 		.base.import = stm32_hash_import,
1791 		.base.setkey = stm32_hash_setkey,
1792 		.base.halg = {
1793 			.digestsize = SHA3_384_DIGEST_SIZE,
1794 			.statesize = sizeof(struct stm32_hash_state),
1795 			.base = {
1796 				.cra_name = "hmac(sha3-384)",
1797 				.cra_driver_name = "stm32-hmac-sha3-384",
1798 				.cra_priority = 200,
1799 				.cra_flags = CRYPTO_ALG_ASYNC |
1800 					CRYPTO_ALG_KERN_DRIVER_ONLY,
1801 				.cra_blocksize = SHA3_384_BLOCK_SIZE,
1802 				.cra_ctxsize = sizeof(struct stm32_hash_ctx),
1803 				.cra_alignmask = 3,
1804 				.cra_init = stm32_hash_cra_sha3_hmac_init,
1805 				.cra_exit = stm32_hash_cra_exit,
1806 				.cra_module = THIS_MODULE,
1807 			}
1808 		},
1809 		.op = {
1810 			.do_one_request = stm32_hash_one_request,
1811 		},
1812 	},
1813 	{
1814 		.base.init = stm32_hash_init,
1815 		.base.update = stm32_hash_update,
1816 		.base.final = stm32_hash_final,
1817 		.base.finup = stm32_hash_finup,
1818 		.base.digest = stm32_hash_digest,
1819 		.base.export = stm32_hash_export,
1820 		.base.import = stm32_hash_import,
1821 		.base.halg = {
1822 			.digestsize = SHA3_512_DIGEST_SIZE,
1823 			.statesize = sizeof(struct stm32_hash_state),
1824 			.base = {
1825 				.cra_name = "sha3-512",
1826 				.cra_driver_name = "stm32-sha3-512",
1827 				.cra_priority = 200,
1828 				.cra_flags = CRYPTO_ALG_ASYNC |
1829 					CRYPTO_ALG_KERN_DRIVER_ONLY,
1830 				.cra_blocksize = SHA3_512_BLOCK_SIZE,
1831 				.cra_ctxsize = sizeof(struct stm32_hash_ctx),
1832 				.cra_alignmask = 3,
1833 				.cra_init = stm32_hash_cra_sha3_init,
1834 				.cra_exit = stm32_hash_cra_exit,
1835 				.cra_module = THIS_MODULE,
1836 			}
1837 		},
1838 		.op = {
1839 			.do_one_request = stm32_hash_one_request,
1840 		},
1841 	},
1842 	{
1843 		.base.init = stm32_hash_init,
1844 		.base.update = stm32_hash_update,
1845 		.base.final = stm32_hash_final,
1846 		.base.finup = stm32_hash_finup,
1847 		.base.digest = stm32_hash_digest,
1848 		.base.export = stm32_hash_export,
1849 		.base.import = stm32_hash_import,
1850 		.base.setkey = stm32_hash_setkey,
1851 		.base.halg = {
1852 			.digestsize = SHA3_512_DIGEST_SIZE,
1853 			.statesize = sizeof(struct stm32_hash_state),
1854 			.base = {
1855 				.cra_name = "hmac(sha3-512)",
1856 				.cra_driver_name = "stm32-hmac-sha3-512",
1857 				.cra_priority = 200,
1858 				.cra_flags = CRYPTO_ALG_ASYNC |
1859 					CRYPTO_ALG_KERN_DRIVER_ONLY,
1860 				.cra_blocksize = SHA3_512_BLOCK_SIZE,
1861 				.cra_ctxsize = sizeof(struct stm32_hash_ctx),
1862 				.cra_alignmask = 3,
1863 				.cra_init = stm32_hash_cra_sha3_hmac_init,
1864 				.cra_exit = stm32_hash_cra_exit,
1865 				.cra_module = THIS_MODULE,
1866 			}
1867 		},
1868 		.op = {
1869 			.do_one_request = stm32_hash_one_request,
1870 		},
1871 	}
1872 };
1873 
1874 static int stm32_hash_register_algs(struct stm32_hash_dev *hdev)
1875 {
1876 	unsigned int i, j;
1877 	int err;
1878 
1879 	for (i = 0; i < hdev->pdata->algs_info_size; i++) {
1880 		for (j = 0; j < hdev->pdata->algs_info[i].size; j++) {
1881 			err = crypto_engine_register_ahash(
1882 				&hdev->pdata->algs_info[i].algs_list[j]);
1883 			if (err)
1884 				goto err_algs;
1885 		}
1886 	}
1887 
1888 	return 0;
1889 err_algs:
1890 	dev_err(hdev->dev, "Algo %d : %d failed\n", i, j);
1891 	for (; i--; ) {
1892 		for (; j--;)
1893 			crypto_engine_unregister_ahash(
1894 				&hdev->pdata->algs_info[i].algs_list[j]);
1895 	}
1896 
1897 	return err;
1898 }
1899 
1900 static int stm32_hash_unregister_algs(struct stm32_hash_dev *hdev)
1901 {
1902 	unsigned int i, j;
1903 
1904 	for (i = 0; i < hdev->pdata->algs_info_size; i++) {
1905 		for (j = 0; j < hdev->pdata->algs_info[i].size; j++)
1906 			crypto_engine_unregister_ahash(
1907 				&hdev->pdata->algs_info[i].algs_list[j]);
1908 	}
1909 
1910 	return 0;
1911 }
1912 
1913 static struct stm32_hash_algs_info stm32_hash_algs_info_ux500[] = {
1914 	{
1915 		.algs_list	= algs_sha1,
1916 		.size		= ARRAY_SIZE(algs_sha1),
1917 	},
1918 	{
1919 		.algs_list	= algs_sha256,
1920 		.size		= ARRAY_SIZE(algs_sha256),
1921 	},
1922 };
1923 
1924 static const struct stm32_hash_pdata stm32_hash_pdata_ux500 = {
1925 	.alg_shift	= 7,
1926 	.algs_info	= stm32_hash_algs_info_ux500,
1927 	.algs_info_size	= ARRAY_SIZE(stm32_hash_algs_info_ux500),
1928 	.broken_emptymsg = true,
1929 	.ux500		= true,
1930 };
1931 
1932 static struct stm32_hash_algs_info stm32_hash_algs_info_stm32f4[] = {
1933 	{
1934 		.algs_list	= algs_md5,
1935 		.size		= ARRAY_SIZE(algs_md5),
1936 	},
1937 	{
1938 		.algs_list	= algs_sha1,
1939 		.size		= ARRAY_SIZE(algs_sha1),
1940 	},
1941 };
1942 
1943 static const struct stm32_hash_pdata stm32_hash_pdata_stm32f4 = {
1944 	.alg_shift	= 7,
1945 	.algs_info	= stm32_hash_algs_info_stm32f4,
1946 	.algs_info_size	= ARRAY_SIZE(stm32_hash_algs_info_stm32f4),
1947 	.has_sr		= true,
1948 	.has_mdmat	= true,
1949 };
1950 
1951 static struct stm32_hash_algs_info stm32_hash_algs_info_stm32f7[] = {
1952 	{
1953 		.algs_list	= algs_md5,
1954 		.size		= ARRAY_SIZE(algs_md5),
1955 	},
1956 	{
1957 		.algs_list	= algs_sha1,
1958 		.size		= ARRAY_SIZE(algs_sha1),
1959 	},
1960 	{
1961 		.algs_list	= algs_sha224,
1962 		.size		= ARRAY_SIZE(algs_sha224),
1963 	},
1964 	{
1965 		.algs_list	= algs_sha256,
1966 		.size		= ARRAY_SIZE(algs_sha256),
1967 	},
1968 };
1969 
1970 static const struct stm32_hash_pdata stm32_hash_pdata_stm32f7 = {
1971 	.alg_shift	= 7,
1972 	.algs_info	= stm32_hash_algs_info_stm32f7,
1973 	.algs_info_size	= ARRAY_SIZE(stm32_hash_algs_info_stm32f7),
1974 	.has_sr		= true,
1975 	.has_mdmat	= true,
1976 };
1977 
1978 static struct stm32_hash_algs_info stm32_hash_algs_info_stm32mp13[] = {
1979 	{
1980 		.algs_list	= algs_sha1,
1981 		.size		= ARRAY_SIZE(algs_sha1),
1982 	},
1983 	{
1984 		.algs_list	= algs_sha224,
1985 		.size		= ARRAY_SIZE(algs_sha224),
1986 	},
1987 	{
1988 		.algs_list	= algs_sha256,
1989 		.size		= ARRAY_SIZE(algs_sha256),
1990 	},
1991 	{
1992 		.algs_list	= algs_sha384_sha512,
1993 		.size		= ARRAY_SIZE(algs_sha384_sha512),
1994 	},
1995 	{
1996 		.algs_list	= algs_sha3,
1997 		.size		= ARRAY_SIZE(algs_sha3),
1998 	},
1999 };
2000 
2001 static const struct stm32_hash_pdata stm32_hash_pdata_stm32mp13 = {
2002 	.alg_shift	= 17,
2003 	.algs_info	= stm32_hash_algs_info_stm32mp13,
2004 	.algs_info_size	= ARRAY_SIZE(stm32_hash_algs_info_stm32mp13),
2005 	.has_sr		= true,
2006 	.has_mdmat	= true,
2007 };
2008 
2009 static const struct of_device_id stm32_hash_of_match[] = {
2010 	{ .compatible = "stericsson,ux500-hash", .data = &stm32_hash_pdata_ux500 },
2011 	{ .compatible = "st,stm32f456-hash", .data = &stm32_hash_pdata_stm32f4 },
2012 	{ .compatible = "st,stm32f756-hash", .data = &stm32_hash_pdata_stm32f7 },
2013 	{ .compatible = "st,stm32mp13-hash", .data = &stm32_hash_pdata_stm32mp13 },
2014 	{},
2015 };
2016 
2017 MODULE_DEVICE_TABLE(of, stm32_hash_of_match);
2018 
2019 static int stm32_hash_get_of_match(struct stm32_hash_dev *hdev,
2020 				   struct device *dev)
2021 {
2022 	hdev->pdata = of_device_get_match_data(dev);
2023 	if (!hdev->pdata) {
2024 		dev_err(dev, "no compatible OF match\n");
2025 		return -EINVAL;
2026 	}
2027 
2028 	return 0;
2029 }
2030 
2031 static int stm32_hash_probe(struct platform_device *pdev)
2032 {
2033 	struct stm32_hash_dev *hdev;
2034 	struct device *dev = &pdev->dev;
2035 	struct resource *res;
2036 	int ret, irq;
2037 
2038 	hdev = devm_kzalloc(dev, sizeof(*hdev), GFP_KERNEL);
2039 	if (!hdev)
2040 		return -ENOMEM;
2041 
2042 	hdev->io_base = devm_platform_get_and_ioremap_resource(pdev, 0, &res);
2043 	if (IS_ERR(hdev->io_base))
2044 		return PTR_ERR(hdev->io_base);
2045 
2046 	hdev->phys_base = res->start;
2047 
2048 	ret = stm32_hash_get_of_match(hdev, dev);
2049 	if (ret)
2050 		return ret;
2051 
2052 	irq = platform_get_irq_optional(pdev, 0);
2053 	if (irq < 0 && irq != -ENXIO)
2054 		return irq;
2055 
2056 	if (irq > 0) {
2057 		ret = devm_request_threaded_irq(dev, irq,
2058 						stm32_hash_irq_handler,
2059 						stm32_hash_irq_thread,
2060 						IRQF_ONESHOT,
2061 						dev_name(dev), hdev);
2062 		if (ret) {
2063 			dev_err(dev, "Cannot grab IRQ\n");
2064 			return ret;
2065 		}
2066 	} else {
2067 		dev_info(dev, "No IRQ, use polling mode\n");
2068 		hdev->polled = true;
2069 	}
2070 
2071 	hdev->clk = devm_clk_get(&pdev->dev, NULL);
2072 	if (IS_ERR(hdev->clk))
2073 		return dev_err_probe(dev, PTR_ERR(hdev->clk),
2074 				     "failed to get clock for hash\n");
2075 
2076 	ret = clk_prepare_enable(hdev->clk);
2077 	if (ret) {
2078 		dev_err(dev, "failed to enable hash clock (%d)\n", ret);
2079 		return ret;
2080 	}
2081 
2082 	pm_runtime_set_autosuspend_delay(dev, HASH_AUTOSUSPEND_DELAY);
2083 	pm_runtime_use_autosuspend(dev);
2084 
2085 	pm_runtime_get_noresume(dev);
2086 	pm_runtime_set_active(dev);
2087 	pm_runtime_enable(dev);
2088 
2089 	hdev->rst = devm_reset_control_get(&pdev->dev, NULL);
2090 	if (IS_ERR(hdev->rst)) {
2091 		if (PTR_ERR(hdev->rst) == -EPROBE_DEFER) {
2092 			ret = -EPROBE_DEFER;
2093 			goto err_reset;
2094 		}
2095 	} else {
2096 		reset_control_assert(hdev->rst);
2097 		udelay(2);
2098 		reset_control_deassert(hdev->rst);
2099 	}
2100 
2101 	hdev->dev = dev;
2102 
2103 	platform_set_drvdata(pdev, hdev);
2104 
2105 	ret = stm32_hash_dma_init(hdev);
2106 	switch (ret) {
2107 	case 0:
2108 		break;
2109 	case -ENOENT:
2110 	case -ENODEV:
2111 		dev_info(dev, "DMA mode not available\n");
2112 		break;
2113 	default:
2114 		dev_err(dev, "DMA init error %d\n", ret);
2115 		goto err_dma;
2116 	}
2117 
2118 	spin_lock(&stm32_hash.lock);
2119 	list_add_tail(&hdev->list, &stm32_hash.dev_list);
2120 	spin_unlock(&stm32_hash.lock);
2121 
2122 	/* Initialize crypto engine */
2123 	hdev->engine = crypto_engine_alloc_init(dev, 1);
2124 	if (!hdev->engine) {
2125 		ret = -ENOMEM;
2126 		goto err_engine;
2127 	}
2128 
2129 	ret = crypto_engine_start(hdev->engine);
2130 	if (ret)
2131 		goto err_engine_start;
2132 
2133 	if (hdev->pdata->ux500)
2134 		/* FIXME: implement DMA mode for Ux500 */
2135 		hdev->dma_mode = 0;
2136 	else
2137 		hdev->dma_mode = stm32_hash_read(hdev, HASH_HWCFGR) & HASH_HWCFG_DMA_MASK;
2138 
2139 	/* Register algos */
2140 	ret = stm32_hash_register_algs(hdev);
2141 	if (ret)
2142 		goto err_algs;
2143 
2144 	dev_info(dev, "Init HASH done HW ver %x DMA mode %u\n",
2145 		 stm32_hash_read(hdev, HASH_VER), hdev->dma_mode);
2146 
2147 	pm_runtime_put_sync(dev);
2148 
2149 	return 0;
2150 
2151 err_algs:
2152 err_engine_start:
2153 	crypto_engine_exit(hdev->engine);
2154 err_engine:
2155 	spin_lock(&stm32_hash.lock);
2156 	list_del(&hdev->list);
2157 	spin_unlock(&stm32_hash.lock);
2158 err_dma:
2159 	if (hdev->dma_lch)
2160 		dma_release_channel(hdev->dma_lch);
2161 err_reset:
2162 	pm_runtime_disable(dev);
2163 	pm_runtime_put_noidle(dev);
2164 
2165 	clk_disable_unprepare(hdev->clk);
2166 
2167 	return ret;
2168 }
2169 
2170 static void stm32_hash_remove(struct platform_device *pdev)
2171 {
2172 	struct stm32_hash_dev *hdev = platform_get_drvdata(pdev);
2173 	int ret;
2174 
2175 	ret = pm_runtime_get_sync(hdev->dev);
2176 
2177 	stm32_hash_unregister_algs(hdev);
2178 
2179 	crypto_engine_exit(hdev->engine);
2180 
2181 	spin_lock(&stm32_hash.lock);
2182 	list_del(&hdev->list);
2183 	spin_unlock(&stm32_hash.lock);
2184 
2185 	if (hdev->dma_lch)
2186 		dma_release_channel(hdev->dma_lch);
2187 
2188 	pm_runtime_disable(hdev->dev);
2189 	pm_runtime_put_noidle(hdev->dev);
2190 
2191 	if (ret >= 0)
2192 		clk_disable_unprepare(hdev->clk);
2193 }
2194 
2195 #ifdef CONFIG_PM
2196 static int stm32_hash_runtime_suspend(struct device *dev)
2197 {
2198 	struct stm32_hash_dev *hdev = dev_get_drvdata(dev);
2199 
2200 	clk_disable_unprepare(hdev->clk);
2201 
2202 	return 0;
2203 }
2204 
2205 static int stm32_hash_runtime_resume(struct device *dev)
2206 {
2207 	struct stm32_hash_dev *hdev = dev_get_drvdata(dev);
2208 	int ret;
2209 
2210 	ret = clk_prepare_enable(hdev->clk);
2211 	if (ret) {
2212 		dev_err(hdev->dev, "Failed to prepare_enable clock\n");
2213 		return ret;
2214 	}
2215 
2216 	return 0;
2217 }
2218 #endif
2219 
2220 static const struct dev_pm_ops stm32_hash_pm_ops = {
2221 	SET_SYSTEM_SLEEP_PM_OPS(pm_runtime_force_suspend,
2222 				pm_runtime_force_resume)
2223 	SET_RUNTIME_PM_OPS(stm32_hash_runtime_suspend,
2224 			   stm32_hash_runtime_resume, NULL)
2225 };
2226 
2227 static struct platform_driver stm32_hash_driver = {
2228 	.probe		= stm32_hash_probe,
2229 	.remove_new	= stm32_hash_remove,
2230 	.driver		= {
2231 		.name	= "stm32-hash",
2232 		.pm = &stm32_hash_pm_ops,
2233 		.of_match_table	= stm32_hash_of_match,
2234 	}
2235 };
2236 
2237 module_platform_driver(stm32_hash_driver);
2238 
2239 MODULE_DESCRIPTION("STM32 SHA1/SHA2/SHA3 & MD5 (HMAC) hw accelerator driver");
2240 MODULE_AUTHOR("Lionel Debieve <lionel.debieve@st.com>");
2241 MODULE_LICENSE("GPL v2");
2242