xref: /openbmc/linux/drivers/crypto/s5p-sss.c (revision ae40e94f)
1 // SPDX-License-Identifier: GPL-2.0
2 //
3 // Cryptographic API.
4 //
5 // Support for Samsung S5PV210 and Exynos HW acceleration.
6 //
7 // Copyright (C) 2011 NetUP Inc. All rights reserved.
8 // Copyright (c) 2017 Samsung Electronics Co., Ltd. All rights reserved.
9 //
10 // Hash part based on omap-sham.c driver.
11 
12 #include <linux/clk.h>
13 #include <linux/crypto.h>
14 #include <linux/dma-mapping.h>
15 #include <linux/err.h>
16 #include <linux/errno.h>
17 #include <linux/init.h>
18 #include <linux/interrupt.h>
19 #include <linux/io.h>
20 #include <linux/kernel.h>
21 #include <linux/module.h>
22 #include <linux/of.h>
23 #include <linux/platform_device.h>
24 #include <linux/scatterlist.h>
25 
26 #include <crypto/ctr.h>
27 #include <crypto/aes.h>
28 #include <crypto/algapi.h>
29 #include <crypto/scatterwalk.h>
30 
31 #include <crypto/hash.h>
32 #include <crypto/md5.h>
33 #include <crypto/sha.h>
34 #include <crypto/internal/hash.h>
35 
36 #define _SBF(s, v)			((v) << (s))
37 
38 /* Feed control registers */
39 #define SSS_REG_FCINTSTAT		0x0000
40 #define SSS_FCINTSTAT_HPARTINT		BIT(7)
41 #define SSS_FCINTSTAT_HDONEINT		BIT(5)
42 #define SSS_FCINTSTAT_BRDMAINT		BIT(3)
43 #define SSS_FCINTSTAT_BTDMAINT		BIT(2)
44 #define SSS_FCINTSTAT_HRDMAINT		BIT(1)
45 #define SSS_FCINTSTAT_PKDMAINT		BIT(0)
46 
47 #define SSS_REG_FCINTENSET		0x0004
48 #define SSS_FCINTENSET_HPARTINTENSET	BIT(7)
49 #define SSS_FCINTENSET_HDONEINTENSET	BIT(5)
50 #define SSS_FCINTENSET_BRDMAINTENSET	BIT(3)
51 #define SSS_FCINTENSET_BTDMAINTENSET	BIT(2)
52 #define SSS_FCINTENSET_HRDMAINTENSET	BIT(1)
53 #define SSS_FCINTENSET_PKDMAINTENSET	BIT(0)
54 
55 #define SSS_REG_FCINTENCLR		0x0008
56 #define SSS_FCINTENCLR_HPARTINTENCLR	BIT(7)
57 #define SSS_FCINTENCLR_HDONEINTENCLR	BIT(5)
58 #define SSS_FCINTENCLR_BRDMAINTENCLR	BIT(3)
59 #define SSS_FCINTENCLR_BTDMAINTENCLR	BIT(2)
60 #define SSS_FCINTENCLR_HRDMAINTENCLR	BIT(1)
61 #define SSS_FCINTENCLR_PKDMAINTENCLR	BIT(0)
62 
63 #define SSS_REG_FCINTPEND		0x000C
64 #define SSS_FCINTPEND_HPARTINTP		BIT(7)
65 #define SSS_FCINTPEND_HDONEINTP		BIT(5)
66 #define SSS_FCINTPEND_BRDMAINTP		BIT(3)
67 #define SSS_FCINTPEND_BTDMAINTP		BIT(2)
68 #define SSS_FCINTPEND_HRDMAINTP		BIT(1)
69 #define SSS_FCINTPEND_PKDMAINTP		BIT(0)
70 
71 #define SSS_REG_FCFIFOSTAT		0x0010
72 #define SSS_FCFIFOSTAT_BRFIFOFUL	BIT(7)
73 #define SSS_FCFIFOSTAT_BRFIFOEMP	BIT(6)
74 #define SSS_FCFIFOSTAT_BTFIFOFUL	BIT(5)
75 #define SSS_FCFIFOSTAT_BTFIFOEMP	BIT(4)
76 #define SSS_FCFIFOSTAT_HRFIFOFUL	BIT(3)
77 #define SSS_FCFIFOSTAT_HRFIFOEMP	BIT(2)
78 #define SSS_FCFIFOSTAT_PKFIFOFUL	BIT(1)
79 #define SSS_FCFIFOSTAT_PKFIFOEMP	BIT(0)
80 
81 #define SSS_REG_FCFIFOCTRL		0x0014
82 #define SSS_FCFIFOCTRL_DESSEL		BIT(2)
83 #define SSS_HASHIN_INDEPENDENT		_SBF(0, 0x00)
84 #define SSS_HASHIN_CIPHER_INPUT		_SBF(0, 0x01)
85 #define SSS_HASHIN_CIPHER_OUTPUT	_SBF(0, 0x02)
86 #define SSS_HASHIN_MASK			_SBF(0, 0x03)
87 
88 #define SSS_REG_FCBRDMAS		0x0020
89 #define SSS_REG_FCBRDMAL		0x0024
90 #define SSS_REG_FCBRDMAC		0x0028
91 #define SSS_FCBRDMAC_BYTESWAP		BIT(1)
92 #define SSS_FCBRDMAC_FLUSH		BIT(0)
93 
94 #define SSS_REG_FCBTDMAS		0x0030
95 #define SSS_REG_FCBTDMAL		0x0034
96 #define SSS_REG_FCBTDMAC		0x0038
97 #define SSS_FCBTDMAC_BYTESWAP		BIT(1)
98 #define SSS_FCBTDMAC_FLUSH		BIT(0)
99 
100 #define SSS_REG_FCHRDMAS		0x0040
101 #define SSS_REG_FCHRDMAL		0x0044
102 #define SSS_REG_FCHRDMAC		0x0048
103 #define SSS_FCHRDMAC_BYTESWAP		BIT(1)
104 #define SSS_FCHRDMAC_FLUSH		BIT(0)
105 
106 #define SSS_REG_FCPKDMAS		0x0050
107 #define SSS_REG_FCPKDMAL		0x0054
108 #define SSS_REG_FCPKDMAC		0x0058
109 #define SSS_FCPKDMAC_BYTESWAP		BIT(3)
110 #define SSS_FCPKDMAC_DESCEND		BIT(2)
111 #define SSS_FCPKDMAC_TRANSMIT		BIT(1)
112 #define SSS_FCPKDMAC_FLUSH		BIT(0)
113 
114 #define SSS_REG_FCPKDMAO		0x005C
115 
116 /* AES registers */
117 #define SSS_REG_AES_CONTROL		0x00
118 #define SSS_AES_BYTESWAP_DI		BIT(11)
119 #define SSS_AES_BYTESWAP_DO		BIT(10)
120 #define SSS_AES_BYTESWAP_IV		BIT(9)
121 #define SSS_AES_BYTESWAP_CNT		BIT(8)
122 #define SSS_AES_BYTESWAP_KEY		BIT(7)
123 #define SSS_AES_KEY_CHANGE_MODE		BIT(6)
124 #define SSS_AES_KEY_SIZE_128		_SBF(4, 0x00)
125 #define SSS_AES_KEY_SIZE_192		_SBF(4, 0x01)
126 #define SSS_AES_KEY_SIZE_256		_SBF(4, 0x02)
127 #define SSS_AES_FIFO_MODE		BIT(3)
128 #define SSS_AES_CHAIN_MODE_ECB		_SBF(1, 0x00)
129 #define SSS_AES_CHAIN_MODE_CBC		_SBF(1, 0x01)
130 #define SSS_AES_CHAIN_MODE_CTR		_SBF(1, 0x02)
131 #define SSS_AES_MODE_DECRYPT		BIT(0)
132 
133 #define SSS_REG_AES_STATUS		0x04
134 #define SSS_AES_BUSY			BIT(2)
135 #define SSS_AES_INPUT_READY		BIT(1)
136 #define SSS_AES_OUTPUT_READY		BIT(0)
137 
138 #define SSS_REG_AES_IN_DATA(s)		(0x10 + (s << 2))
139 #define SSS_REG_AES_OUT_DATA(s)		(0x20 + (s << 2))
140 #define SSS_REG_AES_IV_DATA(s)		(0x30 + (s << 2))
141 #define SSS_REG_AES_CNT_DATA(s)		(0x40 + (s << 2))
142 #define SSS_REG_AES_KEY_DATA(s)		(0x80 + (s << 2))
143 
144 #define SSS_REG(dev, reg)		((dev)->ioaddr + (SSS_REG_##reg))
145 #define SSS_READ(dev, reg)		__raw_readl(SSS_REG(dev, reg))
146 #define SSS_WRITE(dev, reg, val)	__raw_writel((val), SSS_REG(dev, reg))
147 
148 #define SSS_AES_REG(dev, reg)		((dev)->aes_ioaddr + SSS_REG_##reg)
149 #define SSS_AES_WRITE(dev, reg, val)    __raw_writel((val), \
150 						SSS_AES_REG(dev, reg))
151 
152 /* HW engine modes */
153 #define FLAGS_AES_DECRYPT		BIT(0)
154 #define FLAGS_AES_MODE_MASK		_SBF(1, 0x03)
155 #define FLAGS_AES_CBC			_SBF(1, 0x01)
156 #define FLAGS_AES_CTR			_SBF(1, 0x02)
157 
158 #define AES_KEY_LEN			16
159 #define CRYPTO_QUEUE_LEN		1
160 
161 /* HASH registers */
162 #define SSS_REG_HASH_CTRL		0x00
163 
164 #define SSS_HASH_USER_IV_EN		BIT(5)
165 #define SSS_HASH_INIT_BIT		BIT(4)
166 #define SSS_HASH_ENGINE_SHA1		_SBF(1, 0x00)
167 #define SSS_HASH_ENGINE_MD5		_SBF(1, 0x01)
168 #define SSS_HASH_ENGINE_SHA256		_SBF(1, 0x02)
169 
170 #define SSS_HASH_ENGINE_MASK		_SBF(1, 0x03)
171 
172 #define SSS_REG_HASH_CTRL_PAUSE		0x04
173 
174 #define SSS_HASH_PAUSE			BIT(0)
175 
176 #define SSS_REG_HASH_CTRL_FIFO		0x08
177 
178 #define SSS_HASH_FIFO_MODE_DMA		BIT(0)
179 #define SSS_HASH_FIFO_MODE_CPU          0
180 
181 #define SSS_REG_HASH_CTRL_SWAP		0x0C
182 
183 #define SSS_HASH_BYTESWAP_DI		BIT(3)
184 #define SSS_HASH_BYTESWAP_DO		BIT(2)
185 #define SSS_HASH_BYTESWAP_IV		BIT(1)
186 #define SSS_HASH_BYTESWAP_KEY		BIT(0)
187 
188 #define SSS_REG_HASH_STATUS		0x10
189 
190 #define SSS_HASH_STATUS_MSG_DONE	BIT(6)
191 #define SSS_HASH_STATUS_PARTIAL_DONE	BIT(4)
192 #define SSS_HASH_STATUS_BUFFER_READY	BIT(0)
193 
194 #define SSS_REG_HASH_MSG_SIZE_LOW	0x20
195 #define SSS_REG_HASH_MSG_SIZE_HIGH	0x24
196 
197 #define SSS_REG_HASH_PRE_MSG_SIZE_LOW	0x28
198 #define SSS_REG_HASH_PRE_MSG_SIZE_HIGH	0x2C
199 
200 #define SSS_REG_HASH_IV(s)		(0xB0 + ((s) << 2))
201 #define SSS_REG_HASH_OUT(s)		(0x100 + ((s) << 2))
202 
203 #define HASH_BLOCK_SIZE			64
204 #define HASH_REG_SIZEOF			4
205 #define HASH_MD5_MAX_REG		(MD5_DIGEST_SIZE / HASH_REG_SIZEOF)
206 #define HASH_SHA1_MAX_REG		(SHA1_DIGEST_SIZE / HASH_REG_SIZEOF)
207 #define HASH_SHA256_MAX_REG		(SHA256_DIGEST_SIZE / HASH_REG_SIZEOF)
208 
209 /*
210  * HASH bit numbers, used by device, setting in dev->hash_flags with
211  * functions set_bit(), clear_bit() or tested with test_bit() or BIT(),
212  * to keep HASH state BUSY or FREE, or to signal state from irq_handler
213  * to hash_tasklet. SGS keep track of allocated memory for scatterlist
214  */
215 #define HASH_FLAGS_BUSY		0
216 #define HASH_FLAGS_FINAL	1
217 #define HASH_FLAGS_DMA_ACTIVE	2
218 #define HASH_FLAGS_OUTPUT_READY	3
219 #define HASH_FLAGS_DMA_READY	4
220 #define HASH_FLAGS_SGS_COPIED	5
221 #define HASH_FLAGS_SGS_ALLOCED	6
222 
223 /* HASH HW constants */
224 #define BUFLEN			HASH_BLOCK_SIZE
225 
226 #define SSS_HASH_DMA_LEN_ALIGN	8
227 #define SSS_HASH_DMA_ALIGN_MASK	(SSS_HASH_DMA_LEN_ALIGN - 1)
228 
229 #define SSS_HASH_QUEUE_LENGTH	10
230 
231 /**
232  * struct samsung_aes_variant - platform specific SSS driver data
233  * @aes_offset: AES register offset from SSS module's base.
234  * @hash_offset: HASH register offset from SSS module's base.
235  * @clk_names: names of clocks needed to run SSS IP
236  *
237  * Specifies platform specific configuration of SSS module.
238  * Note: A structure for driver specific platform data is used for future
239  * expansion of its usage.
240  */
241 struct samsung_aes_variant {
242 	unsigned int			aes_offset;
243 	unsigned int			hash_offset;
244 	const char			*clk_names[2];
245 };
246 
247 struct s5p_aes_reqctx {
248 	unsigned long			mode;
249 };
250 
251 struct s5p_aes_ctx {
252 	struct s5p_aes_dev		*dev;
253 
254 	u8				aes_key[AES_MAX_KEY_SIZE];
255 	u8				nonce[CTR_RFC3686_NONCE_SIZE];
256 	int				keylen;
257 };
258 
259 /**
260  * struct s5p_aes_dev - Crypto device state container
261  * @dev:	Associated device
262  * @clk:	Clock for accessing hardware
263  * @ioaddr:	Mapped IO memory region
264  * @aes_ioaddr:	Per-varian offset for AES block IO memory
265  * @irq_fc:	Feed control interrupt line
266  * @req:	Crypto request currently handled by the device
267  * @ctx:	Configuration for currently handled crypto request
268  * @sg_src:	Scatter list with source data for currently handled block
269  *		in device.  This is DMA-mapped into device.
270  * @sg_dst:	Scatter list with destination data for currently handled block
271  *		in device. This is DMA-mapped into device.
272  * @sg_src_cpy:	In case of unaligned access, copied scatter list
273  *		with source data.
274  * @sg_dst_cpy:	In case of unaligned access, copied scatter list
275  *		with destination data.
276  * @tasklet:	New request scheduling jib
277  * @queue:	Crypto queue
278  * @busy:	Indicates whether the device is currently handling some request
279  *		thus it uses some of the fields from this state, like:
280  *		req, ctx, sg_src/dst (and copies).  This essentially
281  *		protects against concurrent access to these fields.
282  * @lock:	Lock for protecting both access to device hardware registers
283  *		and fields related to current request (including the busy field).
284  * @res:	Resources for hash.
285  * @io_hash_base: Per-variant offset for HASH block IO memory.
286  * @hash_lock:	Lock for protecting hash_req, hash_queue and hash_flags
287  *		variable.
288  * @hash_flags:	Flags for current HASH op.
289  * @hash_queue:	Async hash queue.
290  * @hash_tasklet: New HASH request scheduling job.
291  * @xmit_buf:	Buffer for current HASH request transfer into SSS block.
292  * @hash_req:	Current request sending to SSS HASH block.
293  * @hash_sg_iter: Scatterlist transferred through DMA into SSS HASH block.
294  * @hash_sg_cnt: Counter for hash_sg_iter.
295  *
296  * @use_hash:	true if HASH algs enabled
297  */
298 struct s5p_aes_dev {
299 	struct device			*dev;
300 	struct clk			*clk;
301 	struct clk			*pclk;
302 	void __iomem			*ioaddr;
303 	void __iomem			*aes_ioaddr;
304 	int				irq_fc;
305 
306 	struct ablkcipher_request	*req;
307 	struct s5p_aes_ctx		*ctx;
308 	struct scatterlist		*sg_src;
309 	struct scatterlist		*sg_dst;
310 
311 	struct scatterlist		*sg_src_cpy;
312 	struct scatterlist		*sg_dst_cpy;
313 
314 	struct tasklet_struct		tasklet;
315 	struct crypto_queue		queue;
316 	bool				busy;
317 	spinlock_t			lock;
318 
319 	struct resource			*res;
320 	void __iomem			*io_hash_base;
321 
322 	spinlock_t			hash_lock; /* protect hash_ vars */
323 	unsigned long			hash_flags;
324 	struct crypto_queue		hash_queue;
325 	struct tasklet_struct		hash_tasklet;
326 
327 	u8				xmit_buf[BUFLEN];
328 	struct ahash_request		*hash_req;
329 	struct scatterlist		*hash_sg_iter;
330 	unsigned int			hash_sg_cnt;
331 
332 	bool				use_hash;
333 };
334 
335 /**
336  * struct s5p_hash_reqctx - HASH request context
337  * @dd:		Associated device
338  * @op_update:	Current request operation (OP_UPDATE or OP_FINAL)
339  * @digcnt:	Number of bytes processed by HW (without buffer[] ones)
340  * @digest:	Digest message or IV for partial result
341  * @nregs:	Number of HW registers for digest or IV read/write
342  * @engine:	Bits for selecting type of HASH in SSS block
343  * @sg:		sg for DMA transfer
344  * @sg_len:	Length of sg for DMA transfer
345  * @sgl[]:	sg for joining buffer and req->src scatterlist
346  * @skip:	Skip offset in req->src for current op
347  * @total:	Total number of bytes for current request
348  * @finup:	Keep state for finup or final.
349  * @error:	Keep track of error.
350  * @bufcnt:	Number of bytes holded in buffer[]
351  * @buffer[]:	For byte(s) from end of req->src in UPDATE op
352  */
353 struct s5p_hash_reqctx {
354 	struct s5p_aes_dev	*dd;
355 	bool			op_update;
356 
357 	u64			digcnt;
358 	u8			digest[SHA256_DIGEST_SIZE];
359 
360 	unsigned int		nregs; /* digest_size / sizeof(reg) */
361 	u32			engine;
362 
363 	struct scatterlist	*sg;
364 	unsigned int		sg_len;
365 	struct scatterlist	sgl[2];
366 	unsigned int		skip;
367 	unsigned int		total;
368 	bool			finup;
369 	bool			error;
370 
371 	u32			bufcnt;
372 	u8			buffer[0];
373 };
374 
375 /**
376  * struct s5p_hash_ctx - HASH transformation context
377  * @dd:		Associated device
378  * @flags:	Bits for algorithm HASH.
379  * @fallback:	Software transformation for zero message or size < BUFLEN.
380  */
381 struct s5p_hash_ctx {
382 	struct s5p_aes_dev	*dd;
383 	unsigned long		flags;
384 	struct crypto_shash	*fallback;
385 };
386 
387 static const struct samsung_aes_variant s5p_aes_data = {
388 	.aes_offset	= 0x4000,
389 	.hash_offset	= 0x6000,
390 	.clk_names	= { "secss", },
391 };
392 
393 static const struct samsung_aes_variant exynos_aes_data = {
394 	.aes_offset	= 0x200,
395 	.hash_offset	= 0x400,
396 	.clk_names	= { "secss", },
397 };
398 
399 static const struct samsung_aes_variant exynos5433_slim_aes_data = {
400 	.aes_offset	= 0x400,
401 	.hash_offset	= 0x800,
402 	.clk_names	= { "pclk", "aclk", },
403 };
404 
405 static const struct of_device_id s5p_sss_dt_match[] = {
406 	{
407 		.compatible = "samsung,s5pv210-secss",
408 		.data = &s5p_aes_data,
409 	},
410 	{
411 		.compatible = "samsung,exynos4210-secss",
412 		.data = &exynos_aes_data,
413 	},
414 	{
415 		.compatible = "samsung,exynos5433-slim-sss",
416 		.data = &exynos5433_slim_aes_data,
417 	},
418 	{ },
419 };
420 MODULE_DEVICE_TABLE(of, s5p_sss_dt_match);
421 
422 static inline const struct samsung_aes_variant *find_s5p_sss_version
423 				   (const struct platform_device *pdev)
424 {
425 	if (IS_ENABLED(CONFIG_OF) && (pdev->dev.of_node)) {
426 		const struct of_device_id *match;
427 
428 		match = of_match_node(s5p_sss_dt_match,
429 					pdev->dev.of_node);
430 		return (const struct samsung_aes_variant *)match->data;
431 	}
432 	return (const struct samsung_aes_variant *)
433 			platform_get_device_id(pdev)->driver_data;
434 }
435 
436 static struct s5p_aes_dev *s5p_dev;
437 
438 static void s5p_set_dma_indata(struct s5p_aes_dev *dev,
439 			       const struct scatterlist *sg)
440 {
441 	SSS_WRITE(dev, FCBRDMAS, sg_dma_address(sg));
442 	SSS_WRITE(dev, FCBRDMAL, sg_dma_len(sg));
443 }
444 
445 static void s5p_set_dma_outdata(struct s5p_aes_dev *dev,
446 				const struct scatterlist *sg)
447 {
448 	SSS_WRITE(dev, FCBTDMAS, sg_dma_address(sg));
449 	SSS_WRITE(dev, FCBTDMAL, sg_dma_len(sg));
450 }
451 
452 static void s5p_free_sg_cpy(struct s5p_aes_dev *dev, struct scatterlist **sg)
453 {
454 	int len;
455 
456 	if (!*sg)
457 		return;
458 
459 	len = ALIGN(dev->req->nbytes, AES_BLOCK_SIZE);
460 	free_pages((unsigned long)sg_virt(*sg), get_order(len));
461 
462 	kfree(*sg);
463 	*sg = NULL;
464 }
465 
466 static void s5p_sg_copy_buf(void *buf, struct scatterlist *sg,
467 			    unsigned int nbytes, int out)
468 {
469 	struct scatter_walk walk;
470 
471 	if (!nbytes)
472 		return;
473 
474 	scatterwalk_start(&walk, sg);
475 	scatterwalk_copychunks(buf, &walk, nbytes, out);
476 	scatterwalk_done(&walk, out, 0);
477 }
478 
479 static void s5p_sg_done(struct s5p_aes_dev *dev)
480 {
481 	struct ablkcipher_request *req = dev->req;
482 	struct s5p_aes_reqctx *reqctx = ablkcipher_request_ctx(req);
483 
484 	if (dev->sg_dst_cpy) {
485 		dev_dbg(dev->dev,
486 			"Copying %d bytes of output data back to original place\n",
487 			dev->req->nbytes);
488 		s5p_sg_copy_buf(sg_virt(dev->sg_dst_cpy), dev->req->dst,
489 				dev->req->nbytes, 1);
490 	}
491 	s5p_free_sg_cpy(dev, &dev->sg_src_cpy);
492 	s5p_free_sg_cpy(dev, &dev->sg_dst_cpy);
493 	if (reqctx->mode & FLAGS_AES_CBC)
494 		memcpy_fromio(req->info, dev->aes_ioaddr + SSS_REG_AES_IV_DATA(0), AES_BLOCK_SIZE);
495 
496 	else if (reqctx->mode & FLAGS_AES_CTR)
497 		memcpy_fromio(req->info, dev->aes_ioaddr + SSS_REG_AES_CNT_DATA(0), AES_BLOCK_SIZE);
498 }
499 
500 /* Calls the completion. Cannot be called with dev->lock hold. */
501 static void s5p_aes_complete(struct ablkcipher_request *req, int err)
502 {
503 	req->base.complete(&req->base, err);
504 }
505 
506 static void s5p_unset_outdata(struct s5p_aes_dev *dev)
507 {
508 	dma_unmap_sg(dev->dev, dev->sg_dst, 1, DMA_FROM_DEVICE);
509 }
510 
511 static void s5p_unset_indata(struct s5p_aes_dev *dev)
512 {
513 	dma_unmap_sg(dev->dev, dev->sg_src, 1, DMA_TO_DEVICE);
514 }
515 
516 static int s5p_make_sg_cpy(struct s5p_aes_dev *dev, struct scatterlist *src,
517 			   struct scatterlist **dst)
518 {
519 	void *pages;
520 	int len;
521 
522 	*dst = kmalloc(sizeof(**dst), GFP_ATOMIC);
523 	if (!*dst)
524 		return -ENOMEM;
525 
526 	len = ALIGN(dev->req->nbytes, AES_BLOCK_SIZE);
527 	pages = (void *)__get_free_pages(GFP_ATOMIC, get_order(len));
528 	if (!pages) {
529 		kfree(*dst);
530 		*dst = NULL;
531 		return -ENOMEM;
532 	}
533 
534 	s5p_sg_copy_buf(pages, src, dev->req->nbytes, 0);
535 
536 	sg_init_table(*dst, 1);
537 	sg_set_buf(*dst, pages, len);
538 
539 	return 0;
540 }
541 
542 static int s5p_set_outdata(struct s5p_aes_dev *dev, struct scatterlist *sg)
543 {
544 	if (!sg->length)
545 		return -EINVAL;
546 
547 	if (!dma_map_sg(dev->dev, sg, 1, DMA_FROM_DEVICE))
548 		return -ENOMEM;
549 
550 	dev->sg_dst = sg;
551 
552 	return 0;
553 }
554 
555 static int s5p_set_indata(struct s5p_aes_dev *dev, struct scatterlist *sg)
556 {
557 	if (!sg->length)
558 		return -EINVAL;
559 
560 	if (!dma_map_sg(dev->dev, sg, 1, DMA_TO_DEVICE))
561 		return -ENOMEM;
562 
563 	dev->sg_src = sg;
564 
565 	return 0;
566 }
567 
568 /*
569  * Returns -ERRNO on error (mapping of new data failed).
570  * On success returns:
571  *  - 0 if there is no more data,
572  *  - 1 if new transmitting (output) data is ready and its address+length
573  *     have to be written to device (by calling s5p_set_dma_outdata()).
574  */
575 static int s5p_aes_tx(struct s5p_aes_dev *dev)
576 {
577 	int ret = 0;
578 
579 	s5p_unset_outdata(dev);
580 
581 	if (!sg_is_last(dev->sg_dst)) {
582 		ret = s5p_set_outdata(dev, sg_next(dev->sg_dst));
583 		if (!ret)
584 			ret = 1;
585 	}
586 
587 	return ret;
588 }
589 
590 /*
591  * Returns -ERRNO on error (mapping of new data failed).
592  * On success returns:
593  *  - 0 if there is no more data,
594  *  - 1 if new receiving (input) data is ready and its address+length
595  *     have to be written to device (by calling s5p_set_dma_indata()).
596  */
597 static int s5p_aes_rx(struct s5p_aes_dev *dev/*, bool *set_dma*/)
598 {
599 	int ret = 0;
600 
601 	s5p_unset_indata(dev);
602 
603 	if (!sg_is_last(dev->sg_src)) {
604 		ret = s5p_set_indata(dev, sg_next(dev->sg_src));
605 		if (!ret)
606 			ret = 1;
607 	}
608 
609 	return ret;
610 }
611 
612 static inline u32 s5p_hash_read(struct s5p_aes_dev *dd, u32 offset)
613 {
614 	return __raw_readl(dd->io_hash_base + offset);
615 }
616 
617 static inline void s5p_hash_write(struct s5p_aes_dev *dd,
618 				  u32 offset, u32 value)
619 {
620 	__raw_writel(value, dd->io_hash_base + offset);
621 }
622 
623 /**
624  * s5p_set_dma_hashdata() - start DMA with sg
625  * @dev:	device
626  * @sg:		scatterlist ready to DMA transmit
627  */
628 static void s5p_set_dma_hashdata(struct s5p_aes_dev *dev,
629 				 const struct scatterlist *sg)
630 {
631 	dev->hash_sg_cnt--;
632 	SSS_WRITE(dev, FCHRDMAS, sg_dma_address(sg));
633 	SSS_WRITE(dev, FCHRDMAL, sg_dma_len(sg)); /* DMA starts */
634 }
635 
636 /**
637  * s5p_hash_rx() - get next hash_sg_iter
638  * @dev:	device
639  *
640  * Return:
641  * 2	if there is no more data and it is UPDATE op
642  * 1	if new receiving (input) data is ready and can be written to device
643  * 0	if there is no more data and it is FINAL op
644  */
645 static int s5p_hash_rx(struct s5p_aes_dev *dev)
646 {
647 	if (dev->hash_sg_cnt > 0) {
648 		dev->hash_sg_iter = sg_next(dev->hash_sg_iter);
649 		return 1;
650 	}
651 
652 	set_bit(HASH_FLAGS_DMA_READY, &dev->hash_flags);
653 	if (test_bit(HASH_FLAGS_FINAL, &dev->hash_flags))
654 		return 0;
655 
656 	return 2;
657 }
658 
659 static irqreturn_t s5p_aes_interrupt(int irq, void *dev_id)
660 {
661 	struct platform_device *pdev = dev_id;
662 	struct s5p_aes_dev *dev = platform_get_drvdata(pdev);
663 	struct ablkcipher_request *req;
664 	int err_dma_tx = 0;
665 	int err_dma_rx = 0;
666 	int err_dma_hx = 0;
667 	bool tx_end = false;
668 	bool hx_end = false;
669 	unsigned long flags;
670 	u32 status, st_bits;
671 	int err;
672 
673 	spin_lock_irqsave(&dev->lock, flags);
674 
675 	/*
676 	 * Handle rx or tx interrupt. If there is still data (scatterlist did not
677 	 * reach end), then map next scatterlist entry.
678 	 * In case of such mapping error, s5p_aes_complete() should be called.
679 	 *
680 	 * If there is no more data in tx scatter list, call s5p_aes_complete()
681 	 * and schedule new tasklet.
682 	 *
683 	 * Handle hx interrupt. If there is still data map next entry.
684 	 */
685 	status = SSS_READ(dev, FCINTSTAT);
686 	if (status & SSS_FCINTSTAT_BRDMAINT)
687 		err_dma_rx = s5p_aes_rx(dev);
688 
689 	if (status & SSS_FCINTSTAT_BTDMAINT) {
690 		if (sg_is_last(dev->sg_dst))
691 			tx_end = true;
692 		err_dma_tx = s5p_aes_tx(dev);
693 	}
694 
695 	if (status & SSS_FCINTSTAT_HRDMAINT)
696 		err_dma_hx = s5p_hash_rx(dev);
697 
698 	st_bits = status & (SSS_FCINTSTAT_BRDMAINT | SSS_FCINTSTAT_BTDMAINT |
699 				SSS_FCINTSTAT_HRDMAINT);
700 	/* clear DMA bits */
701 	SSS_WRITE(dev, FCINTPEND, st_bits);
702 
703 	/* clear HASH irq bits */
704 	if (status & (SSS_FCINTSTAT_HDONEINT | SSS_FCINTSTAT_HPARTINT)) {
705 		/* cannot have both HPART and HDONE */
706 		if (status & SSS_FCINTSTAT_HPARTINT)
707 			st_bits = SSS_HASH_STATUS_PARTIAL_DONE;
708 
709 		if (status & SSS_FCINTSTAT_HDONEINT)
710 			st_bits = SSS_HASH_STATUS_MSG_DONE;
711 
712 		set_bit(HASH_FLAGS_OUTPUT_READY, &dev->hash_flags);
713 		s5p_hash_write(dev, SSS_REG_HASH_STATUS, st_bits);
714 		hx_end = true;
715 		/* when DONE or PART, do not handle HASH DMA */
716 		err_dma_hx = 0;
717 	}
718 
719 	if (err_dma_rx < 0) {
720 		err = err_dma_rx;
721 		goto error;
722 	}
723 	if (err_dma_tx < 0) {
724 		err = err_dma_tx;
725 		goto error;
726 	}
727 
728 	if (tx_end) {
729 		s5p_sg_done(dev);
730 		if (err_dma_hx == 1)
731 			s5p_set_dma_hashdata(dev, dev->hash_sg_iter);
732 
733 		spin_unlock_irqrestore(&dev->lock, flags);
734 
735 		s5p_aes_complete(dev->req, 0);
736 		/* Device is still busy */
737 		tasklet_schedule(&dev->tasklet);
738 	} else {
739 		/*
740 		 * Writing length of DMA block (either receiving or
741 		 * transmitting) will start the operation immediately, so this
742 		 * should be done at the end (even after clearing pending
743 		 * interrupts to not miss the interrupt).
744 		 */
745 		if (err_dma_tx == 1)
746 			s5p_set_dma_outdata(dev, dev->sg_dst);
747 		if (err_dma_rx == 1)
748 			s5p_set_dma_indata(dev, dev->sg_src);
749 		if (err_dma_hx == 1)
750 			s5p_set_dma_hashdata(dev, dev->hash_sg_iter);
751 
752 		spin_unlock_irqrestore(&dev->lock, flags);
753 	}
754 
755 	goto hash_irq_end;
756 
757 error:
758 	s5p_sg_done(dev);
759 	dev->busy = false;
760 	req = dev->req;
761 	if (err_dma_hx == 1)
762 		s5p_set_dma_hashdata(dev, dev->hash_sg_iter);
763 
764 	spin_unlock_irqrestore(&dev->lock, flags);
765 	s5p_aes_complete(req, err);
766 
767 hash_irq_end:
768 	/*
769 	 * Note about else if:
770 	 *   when hash_sg_iter reaches end and its UPDATE op,
771 	 *   issue SSS_HASH_PAUSE and wait for HPART irq
772 	 */
773 	if (hx_end)
774 		tasklet_schedule(&dev->hash_tasklet);
775 	else if (err_dma_hx == 2)
776 		s5p_hash_write(dev, SSS_REG_HASH_CTRL_PAUSE,
777 			       SSS_HASH_PAUSE);
778 
779 	return IRQ_HANDLED;
780 }
781 
782 /**
783  * s5p_hash_read_msg() - read message or IV from HW
784  * @req:	AHASH request
785  */
786 static void s5p_hash_read_msg(struct ahash_request *req)
787 {
788 	struct s5p_hash_reqctx *ctx = ahash_request_ctx(req);
789 	struct s5p_aes_dev *dd = ctx->dd;
790 	u32 *hash = (u32 *)ctx->digest;
791 	unsigned int i;
792 
793 	for (i = 0; i < ctx->nregs; i++)
794 		hash[i] = s5p_hash_read(dd, SSS_REG_HASH_OUT(i));
795 }
796 
797 /**
798  * s5p_hash_write_ctx_iv() - write IV for next partial/finup op.
799  * @dd:		device
800  * @ctx:	request context
801  */
802 static void s5p_hash_write_ctx_iv(struct s5p_aes_dev *dd,
803 				  const struct s5p_hash_reqctx *ctx)
804 {
805 	const u32 *hash = (const u32 *)ctx->digest;
806 	unsigned int i;
807 
808 	for (i = 0; i < ctx->nregs; i++)
809 		s5p_hash_write(dd, SSS_REG_HASH_IV(i), hash[i]);
810 }
811 
812 /**
813  * s5p_hash_write_iv() - write IV for next partial/finup op.
814  * @req:	AHASH request
815  */
816 static void s5p_hash_write_iv(struct ahash_request *req)
817 {
818 	struct s5p_hash_reqctx *ctx = ahash_request_ctx(req);
819 
820 	s5p_hash_write_ctx_iv(ctx->dd, ctx);
821 }
822 
823 /**
824  * s5p_hash_copy_result() - copy digest into req->result
825  * @req:	AHASH request
826  */
827 static void s5p_hash_copy_result(struct ahash_request *req)
828 {
829 	const struct s5p_hash_reqctx *ctx = ahash_request_ctx(req);
830 
831 	if (!req->result)
832 		return;
833 
834 	memcpy(req->result, ctx->digest, ctx->nregs * HASH_REG_SIZEOF);
835 }
836 
837 /**
838  * s5p_hash_dma_flush() - flush HASH DMA
839  * @dev:	secss device
840  */
841 static void s5p_hash_dma_flush(struct s5p_aes_dev *dev)
842 {
843 	SSS_WRITE(dev, FCHRDMAC, SSS_FCHRDMAC_FLUSH);
844 }
845 
846 /**
847  * s5p_hash_dma_enable() - enable DMA mode for HASH
848  * @dev:	secss device
849  *
850  * enable DMA mode for HASH
851  */
852 static void s5p_hash_dma_enable(struct s5p_aes_dev *dev)
853 {
854 	s5p_hash_write(dev, SSS_REG_HASH_CTRL_FIFO, SSS_HASH_FIFO_MODE_DMA);
855 }
856 
857 /**
858  * s5p_hash_irq_disable() - disable irq HASH signals
859  * @dev:	secss device
860  * @flags:	bitfield with irq's to be disabled
861  */
862 static void s5p_hash_irq_disable(struct s5p_aes_dev *dev, u32 flags)
863 {
864 	SSS_WRITE(dev, FCINTENCLR, flags);
865 }
866 
867 /**
868  * s5p_hash_irq_enable() - enable irq signals
869  * @dev:	secss device
870  * @flags:	bitfield with irq's to be enabled
871  */
872 static void s5p_hash_irq_enable(struct s5p_aes_dev *dev, int flags)
873 {
874 	SSS_WRITE(dev, FCINTENSET, flags);
875 }
876 
877 /**
878  * s5p_hash_set_flow() - set flow inside SecSS AES/DES with/without HASH
879  * @dev:	secss device
880  * @hashflow:	HASH stream flow with/without crypto AES/DES
881  */
882 static void s5p_hash_set_flow(struct s5p_aes_dev *dev, u32 hashflow)
883 {
884 	unsigned long flags;
885 	u32 flow;
886 
887 	spin_lock_irqsave(&dev->lock, flags);
888 
889 	flow = SSS_READ(dev, FCFIFOCTRL);
890 	flow &= ~SSS_HASHIN_MASK;
891 	flow |= hashflow;
892 	SSS_WRITE(dev, FCFIFOCTRL, flow);
893 
894 	spin_unlock_irqrestore(&dev->lock, flags);
895 }
896 
897 /**
898  * s5p_ahash_dma_init() - enable DMA and set HASH flow inside SecSS
899  * @dev:	secss device
900  * @hashflow:	HASH stream flow with/without AES/DES
901  *
902  * flush HASH DMA and enable DMA, set HASH stream flow inside SecSS HW,
903  * enable HASH irq's HRDMA, HDONE, HPART
904  */
905 static void s5p_ahash_dma_init(struct s5p_aes_dev *dev, u32 hashflow)
906 {
907 	s5p_hash_irq_disable(dev, SSS_FCINTENCLR_HRDMAINTENCLR |
908 			     SSS_FCINTENCLR_HDONEINTENCLR |
909 			     SSS_FCINTENCLR_HPARTINTENCLR);
910 	s5p_hash_dma_flush(dev);
911 
912 	s5p_hash_dma_enable(dev);
913 	s5p_hash_set_flow(dev, hashflow & SSS_HASHIN_MASK);
914 	s5p_hash_irq_enable(dev, SSS_FCINTENSET_HRDMAINTENSET |
915 			    SSS_FCINTENSET_HDONEINTENSET |
916 			    SSS_FCINTENSET_HPARTINTENSET);
917 }
918 
919 /**
920  * s5p_hash_write_ctrl() - prepare HASH block in SecSS for processing
921  * @dd:		secss device
922  * @length:	length for request
923  * @final:	true if final op
924  *
925  * Prepare SSS HASH block for processing bytes in DMA mode. If it is called
926  * after previous updates, fill up IV words. For final, calculate and set
927  * lengths for HASH so SecSS can finalize hash. For partial, set SSS HASH
928  * length as 2^63 so it will be never reached and set to zero prelow and
929  * prehigh.
930  *
931  * This function does not start DMA transfer.
932  */
933 static void s5p_hash_write_ctrl(struct s5p_aes_dev *dd, size_t length,
934 				bool final)
935 {
936 	struct s5p_hash_reqctx *ctx = ahash_request_ctx(dd->hash_req);
937 	u32 prelow, prehigh, low, high;
938 	u32 configflags, swapflags;
939 	u64 tmplen;
940 
941 	configflags = ctx->engine | SSS_HASH_INIT_BIT;
942 
943 	if (likely(ctx->digcnt)) {
944 		s5p_hash_write_ctx_iv(dd, ctx);
945 		configflags |= SSS_HASH_USER_IV_EN;
946 	}
947 
948 	if (final) {
949 		/* number of bytes for last part */
950 		low = length;
951 		high = 0;
952 		/* total number of bits prev hashed */
953 		tmplen = ctx->digcnt * 8;
954 		prelow = (u32)tmplen;
955 		prehigh = (u32)(tmplen >> 32);
956 	} else {
957 		prelow = 0;
958 		prehigh = 0;
959 		low = 0;
960 		high = BIT(31);
961 	}
962 
963 	swapflags = SSS_HASH_BYTESWAP_DI | SSS_HASH_BYTESWAP_DO |
964 		    SSS_HASH_BYTESWAP_IV | SSS_HASH_BYTESWAP_KEY;
965 
966 	s5p_hash_write(dd, SSS_REG_HASH_MSG_SIZE_LOW, low);
967 	s5p_hash_write(dd, SSS_REG_HASH_MSG_SIZE_HIGH, high);
968 	s5p_hash_write(dd, SSS_REG_HASH_PRE_MSG_SIZE_LOW, prelow);
969 	s5p_hash_write(dd, SSS_REG_HASH_PRE_MSG_SIZE_HIGH, prehigh);
970 
971 	s5p_hash_write(dd, SSS_REG_HASH_CTRL_SWAP, swapflags);
972 	s5p_hash_write(dd, SSS_REG_HASH_CTRL, configflags);
973 }
974 
975 /**
976  * s5p_hash_xmit_dma() - start DMA hash processing
977  * @dd:		secss device
978  * @length:	length for request
979  * @final:	true if final op
980  *
981  * Update digcnt here, as it is needed for finup/final op.
982  */
983 static int s5p_hash_xmit_dma(struct s5p_aes_dev *dd, size_t length,
984 			     bool final)
985 {
986 	struct s5p_hash_reqctx *ctx = ahash_request_ctx(dd->hash_req);
987 	unsigned int cnt;
988 
989 	cnt = dma_map_sg(dd->dev, ctx->sg, ctx->sg_len, DMA_TO_DEVICE);
990 	if (!cnt) {
991 		dev_err(dd->dev, "dma_map_sg error\n");
992 		ctx->error = true;
993 		return -EINVAL;
994 	}
995 
996 	set_bit(HASH_FLAGS_DMA_ACTIVE, &dd->hash_flags);
997 	dd->hash_sg_iter = ctx->sg;
998 	dd->hash_sg_cnt = cnt;
999 	s5p_hash_write_ctrl(dd, length, final);
1000 	ctx->digcnt += length;
1001 	ctx->total -= length;
1002 
1003 	/* catch last interrupt */
1004 	if (final)
1005 		set_bit(HASH_FLAGS_FINAL, &dd->hash_flags);
1006 
1007 	s5p_set_dma_hashdata(dd, dd->hash_sg_iter); /* DMA starts */
1008 
1009 	return -EINPROGRESS;
1010 }
1011 
1012 /**
1013  * s5p_hash_copy_sgs() - copy request's bytes into new buffer
1014  * @ctx:	request context
1015  * @sg:		source scatterlist request
1016  * @new_len:	number of bytes to process from sg
1017  *
1018  * Allocate new buffer, copy data for HASH into it. If there was xmit_buf
1019  * filled, copy it first, then copy data from sg into it. Prepare one sgl[0]
1020  * with allocated buffer.
1021  *
1022  * Set bit in dd->hash_flag so we can free it after irq ends processing.
1023  */
1024 static int s5p_hash_copy_sgs(struct s5p_hash_reqctx *ctx,
1025 			     struct scatterlist *sg, unsigned int new_len)
1026 {
1027 	unsigned int pages, len;
1028 	void *buf;
1029 
1030 	len = new_len + ctx->bufcnt;
1031 	pages = get_order(len);
1032 
1033 	buf = (void *)__get_free_pages(GFP_ATOMIC, pages);
1034 	if (!buf) {
1035 		dev_err(ctx->dd->dev, "alloc pages for unaligned case.\n");
1036 		ctx->error = true;
1037 		return -ENOMEM;
1038 	}
1039 
1040 	if (ctx->bufcnt)
1041 		memcpy(buf, ctx->dd->xmit_buf, ctx->bufcnt);
1042 
1043 	scatterwalk_map_and_copy(buf + ctx->bufcnt, sg, ctx->skip,
1044 				 new_len, 0);
1045 	sg_init_table(ctx->sgl, 1);
1046 	sg_set_buf(ctx->sgl, buf, len);
1047 	ctx->sg = ctx->sgl;
1048 	ctx->sg_len = 1;
1049 	ctx->bufcnt = 0;
1050 	ctx->skip = 0;
1051 	set_bit(HASH_FLAGS_SGS_COPIED, &ctx->dd->hash_flags);
1052 
1053 	return 0;
1054 }
1055 
1056 /**
1057  * s5p_hash_copy_sg_lists() - copy sg list and make fixes in copy
1058  * @ctx:	request context
1059  * @sg:		source scatterlist request
1060  * @new_len:	number of bytes to process from sg
1061  *
1062  * Allocate new scatterlist table, copy data for HASH into it. If there was
1063  * xmit_buf filled, prepare it first, then copy page, length and offset from
1064  * source sg into it, adjusting begin and/or end for skip offset and
1065  * hash_later value.
1066  *
1067  * Resulting sg table will be assigned to ctx->sg. Set flag so we can free
1068  * it after irq ends processing.
1069  */
1070 static int s5p_hash_copy_sg_lists(struct s5p_hash_reqctx *ctx,
1071 				  struct scatterlist *sg, unsigned int new_len)
1072 {
1073 	unsigned int skip = ctx->skip, n = sg_nents(sg);
1074 	struct scatterlist *tmp;
1075 	unsigned int len;
1076 
1077 	if (ctx->bufcnt)
1078 		n++;
1079 
1080 	ctx->sg = kmalloc_array(n, sizeof(*sg), GFP_KERNEL);
1081 	if (!ctx->sg) {
1082 		ctx->error = true;
1083 		return -ENOMEM;
1084 	}
1085 
1086 	sg_init_table(ctx->sg, n);
1087 
1088 	tmp = ctx->sg;
1089 
1090 	ctx->sg_len = 0;
1091 
1092 	if (ctx->bufcnt) {
1093 		sg_set_buf(tmp, ctx->dd->xmit_buf, ctx->bufcnt);
1094 		tmp = sg_next(tmp);
1095 		ctx->sg_len++;
1096 	}
1097 
1098 	while (sg && skip >= sg->length) {
1099 		skip -= sg->length;
1100 		sg = sg_next(sg);
1101 	}
1102 
1103 	while (sg && new_len) {
1104 		len = sg->length - skip;
1105 		if (new_len < len)
1106 			len = new_len;
1107 
1108 		new_len -= len;
1109 		sg_set_page(tmp, sg_page(sg), len, sg->offset + skip);
1110 		skip = 0;
1111 		if (new_len <= 0)
1112 			sg_mark_end(tmp);
1113 
1114 		tmp = sg_next(tmp);
1115 		ctx->sg_len++;
1116 		sg = sg_next(sg);
1117 	}
1118 
1119 	set_bit(HASH_FLAGS_SGS_ALLOCED, &ctx->dd->hash_flags);
1120 
1121 	return 0;
1122 }
1123 
1124 /**
1125  * s5p_hash_prepare_sgs() - prepare sg for processing
1126  * @ctx:	request context
1127  * @sg:		source scatterlist request
1128  * @nbytes:	number of bytes to process from sg
1129  * @final:	final flag
1130  *
1131  * Check two conditions: (1) if buffers in sg have len aligned data, and (2)
1132  * sg table have good aligned elements (list_ok). If one of this checks fails,
1133  * then either (1) allocates new buffer for data with s5p_hash_copy_sgs, copy
1134  * data into this buffer and prepare request in sgl, or (2) allocates new sg
1135  * table and prepare sg elements.
1136  *
1137  * For digest or finup all conditions can be good, and we may not need any
1138  * fixes.
1139  */
1140 static int s5p_hash_prepare_sgs(struct s5p_hash_reqctx *ctx,
1141 				struct scatterlist *sg,
1142 				unsigned int new_len, bool final)
1143 {
1144 	unsigned int skip = ctx->skip, nbytes = new_len, n = 0;
1145 	bool aligned = true, list_ok = true;
1146 	struct scatterlist *sg_tmp = sg;
1147 
1148 	if (!sg || !sg->length || !new_len)
1149 		return 0;
1150 
1151 	if (skip || !final)
1152 		list_ok = false;
1153 
1154 	while (nbytes > 0 && sg_tmp) {
1155 		n++;
1156 		if (skip >= sg_tmp->length) {
1157 			skip -= sg_tmp->length;
1158 			if (!sg_tmp->length) {
1159 				aligned = false;
1160 				break;
1161 			}
1162 		} else {
1163 			if (!IS_ALIGNED(sg_tmp->length - skip, BUFLEN)) {
1164 				aligned = false;
1165 				break;
1166 			}
1167 
1168 			if (nbytes < sg_tmp->length - skip) {
1169 				list_ok = false;
1170 				break;
1171 			}
1172 
1173 			nbytes -= sg_tmp->length - skip;
1174 			skip = 0;
1175 		}
1176 
1177 		sg_tmp = sg_next(sg_tmp);
1178 	}
1179 
1180 	if (!aligned)
1181 		return s5p_hash_copy_sgs(ctx, sg, new_len);
1182 	else if (!list_ok)
1183 		return s5p_hash_copy_sg_lists(ctx, sg, new_len);
1184 
1185 	/*
1186 	 * Have aligned data from previous operation and/or current
1187 	 * Note: will enter here only if (digest or finup) and aligned
1188 	 */
1189 	if (ctx->bufcnt) {
1190 		ctx->sg_len = n;
1191 		sg_init_table(ctx->sgl, 2);
1192 		sg_set_buf(ctx->sgl, ctx->dd->xmit_buf, ctx->bufcnt);
1193 		sg_chain(ctx->sgl, 2, sg);
1194 		ctx->sg = ctx->sgl;
1195 		ctx->sg_len++;
1196 	} else {
1197 		ctx->sg = sg;
1198 		ctx->sg_len = n;
1199 	}
1200 
1201 	return 0;
1202 }
1203 
1204 /**
1205  * s5p_hash_prepare_request() - prepare request for processing
1206  * @req:	AHASH request
1207  * @update:	true if UPDATE op
1208  *
1209  * Note 1: we can have update flag _and_ final flag at the same time.
1210  * Note 2: we enter here when digcnt > BUFLEN (=HASH_BLOCK_SIZE) or
1211  *	   either req->nbytes or ctx->bufcnt + req->nbytes is > BUFLEN or
1212  *	   we have final op
1213  */
1214 static int s5p_hash_prepare_request(struct ahash_request *req, bool update)
1215 {
1216 	struct s5p_hash_reqctx *ctx = ahash_request_ctx(req);
1217 	bool final = ctx->finup;
1218 	int xmit_len, hash_later, nbytes;
1219 	int ret;
1220 
1221 	if (update)
1222 		nbytes = req->nbytes;
1223 	else
1224 		nbytes = 0;
1225 
1226 	ctx->total = nbytes + ctx->bufcnt;
1227 	if (!ctx->total)
1228 		return 0;
1229 
1230 	if (nbytes && (!IS_ALIGNED(ctx->bufcnt, BUFLEN))) {
1231 		/* bytes left from previous request, so fill up to BUFLEN */
1232 		int len = BUFLEN - ctx->bufcnt % BUFLEN;
1233 
1234 		if (len > nbytes)
1235 			len = nbytes;
1236 
1237 		scatterwalk_map_and_copy(ctx->buffer + ctx->bufcnt, req->src,
1238 					 0, len, 0);
1239 		ctx->bufcnt += len;
1240 		nbytes -= len;
1241 		ctx->skip = len;
1242 	} else {
1243 		ctx->skip = 0;
1244 	}
1245 
1246 	if (ctx->bufcnt)
1247 		memcpy(ctx->dd->xmit_buf, ctx->buffer, ctx->bufcnt);
1248 
1249 	xmit_len = ctx->total;
1250 	if (final) {
1251 		hash_later = 0;
1252 	} else {
1253 		if (IS_ALIGNED(xmit_len, BUFLEN))
1254 			xmit_len -= BUFLEN;
1255 		else
1256 			xmit_len -= xmit_len & (BUFLEN - 1);
1257 
1258 		hash_later = ctx->total - xmit_len;
1259 		/* copy hash_later bytes from end of req->src */
1260 		/* previous bytes are in xmit_buf, so no overwrite */
1261 		scatterwalk_map_and_copy(ctx->buffer, req->src,
1262 					 req->nbytes - hash_later,
1263 					 hash_later, 0);
1264 	}
1265 
1266 	if (xmit_len > BUFLEN) {
1267 		ret = s5p_hash_prepare_sgs(ctx, req->src, nbytes - hash_later,
1268 					   final);
1269 		if (ret)
1270 			return ret;
1271 	} else {
1272 		/* have buffered data only */
1273 		if (unlikely(!ctx->bufcnt)) {
1274 			/* first update didn't fill up buffer */
1275 			scatterwalk_map_and_copy(ctx->dd->xmit_buf, req->src,
1276 						 0, xmit_len, 0);
1277 		}
1278 
1279 		sg_init_table(ctx->sgl, 1);
1280 		sg_set_buf(ctx->sgl, ctx->dd->xmit_buf, xmit_len);
1281 
1282 		ctx->sg = ctx->sgl;
1283 		ctx->sg_len = 1;
1284 	}
1285 
1286 	ctx->bufcnt = hash_later;
1287 	if (!final)
1288 		ctx->total = xmit_len;
1289 
1290 	return 0;
1291 }
1292 
1293 /**
1294  * s5p_hash_update_dma_stop() - unmap DMA
1295  * @dd:		secss device
1296  *
1297  * Unmap scatterlist ctx->sg.
1298  */
1299 static void s5p_hash_update_dma_stop(struct s5p_aes_dev *dd)
1300 {
1301 	const struct s5p_hash_reqctx *ctx = ahash_request_ctx(dd->hash_req);
1302 
1303 	dma_unmap_sg(dd->dev, ctx->sg, ctx->sg_len, DMA_TO_DEVICE);
1304 	clear_bit(HASH_FLAGS_DMA_ACTIVE, &dd->hash_flags);
1305 }
1306 
1307 /**
1308  * s5p_hash_finish() - copy calculated digest to crypto layer
1309  * @req:	AHASH request
1310  */
1311 static void s5p_hash_finish(struct ahash_request *req)
1312 {
1313 	struct s5p_hash_reqctx *ctx = ahash_request_ctx(req);
1314 	struct s5p_aes_dev *dd = ctx->dd;
1315 
1316 	if (ctx->digcnt)
1317 		s5p_hash_copy_result(req);
1318 
1319 	dev_dbg(dd->dev, "hash_finish digcnt: %lld\n", ctx->digcnt);
1320 }
1321 
1322 /**
1323  * s5p_hash_finish_req() - finish request
1324  * @req:	AHASH request
1325  * @err:	error
1326  */
1327 static void s5p_hash_finish_req(struct ahash_request *req, int err)
1328 {
1329 	struct s5p_hash_reqctx *ctx = ahash_request_ctx(req);
1330 	struct s5p_aes_dev *dd = ctx->dd;
1331 	unsigned long flags;
1332 
1333 	if (test_bit(HASH_FLAGS_SGS_COPIED, &dd->hash_flags))
1334 		free_pages((unsigned long)sg_virt(ctx->sg),
1335 			   get_order(ctx->sg->length));
1336 
1337 	if (test_bit(HASH_FLAGS_SGS_ALLOCED, &dd->hash_flags))
1338 		kfree(ctx->sg);
1339 
1340 	ctx->sg = NULL;
1341 	dd->hash_flags &= ~(BIT(HASH_FLAGS_SGS_ALLOCED) |
1342 			    BIT(HASH_FLAGS_SGS_COPIED));
1343 
1344 	if (!err && !ctx->error) {
1345 		s5p_hash_read_msg(req);
1346 		if (test_bit(HASH_FLAGS_FINAL, &dd->hash_flags))
1347 			s5p_hash_finish(req);
1348 	} else {
1349 		ctx->error = true;
1350 	}
1351 
1352 	spin_lock_irqsave(&dd->hash_lock, flags);
1353 	dd->hash_flags &= ~(BIT(HASH_FLAGS_BUSY) | BIT(HASH_FLAGS_FINAL) |
1354 			    BIT(HASH_FLAGS_DMA_READY) |
1355 			    BIT(HASH_FLAGS_OUTPUT_READY));
1356 	spin_unlock_irqrestore(&dd->hash_lock, flags);
1357 
1358 	if (req->base.complete)
1359 		req->base.complete(&req->base, err);
1360 }
1361 
1362 /**
1363  * s5p_hash_handle_queue() - handle hash queue
1364  * @dd:		device s5p_aes_dev
1365  * @req:	AHASH request
1366  *
1367  * If req!=NULL enqueue it on dd->queue, if FLAGS_BUSY is not set on the
1368  * device then processes the first request from the dd->queue
1369  *
1370  * Returns: see s5p_hash_final below.
1371  */
1372 static int s5p_hash_handle_queue(struct s5p_aes_dev *dd,
1373 				 struct ahash_request *req)
1374 {
1375 	struct crypto_async_request *async_req, *backlog;
1376 	struct s5p_hash_reqctx *ctx;
1377 	unsigned long flags;
1378 	int err = 0, ret = 0;
1379 
1380 retry:
1381 	spin_lock_irqsave(&dd->hash_lock, flags);
1382 	if (req)
1383 		ret = ahash_enqueue_request(&dd->hash_queue, req);
1384 
1385 	if (test_bit(HASH_FLAGS_BUSY, &dd->hash_flags)) {
1386 		spin_unlock_irqrestore(&dd->hash_lock, flags);
1387 		return ret;
1388 	}
1389 
1390 	backlog = crypto_get_backlog(&dd->hash_queue);
1391 	async_req = crypto_dequeue_request(&dd->hash_queue);
1392 	if (async_req)
1393 		set_bit(HASH_FLAGS_BUSY, &dd->hash_flags);
1394 
1395 	spin_unlock_irqrestore(&dd->hash_lock, flags);
1396 
1397 	if (!async_req)
1398 		return ret;
1399 
1400 	if (backlog)
1401 		backlog->complete(backlog, -EINPROGRESS);
1402 
1403 	req = ahash_request_cast(async_req);
1404 	dd->hash_req = req;
1405 	ctx = ahash_request_ctx(req);
1406 
1407 	err = s5p_hash_prepare_request(req, ctx->op_update);
1408 	if (err || !ctx->total)
1409 		goto out;
1410 
1411 	dev_dbg(dd->dev, "handling new req, op_update: %u, nbytes: %d\n",
1412 		ctx->op_update, req->nbytes);
1413 
1414 	s5p_ahash_dma_init(dd, SSS_HASHIN_INDEPENDENT);
1415 	if (ctx->digcnt)
1416 		s5p_hash_write_iv(req); /* restore hash IV */
1417 
1418 	if (ctx->op_update) { /* HASH_OP_UPDATE */
1419 		err = s5p_hash_xmit_dma(dd, ctx->total, ctx->finup);
1420 		if (err != -EINPROGRESS && ctx->finup && !ctx->error)
1421 			/* no final() after finup() */
1422 			err = s5p_hash_xmit_dma(dd, ctx->total, true);
1423 	} else { /* HASH_OP_FINAL */
1424 		err = s5p_hash_xmit_dma(dd, ctx->total, true);
1425 	}
1426 out:
1427 	if (err != -EINPROGRESS) {
1428 		/* hash_tasklet_cb will not finish it, so do it here */
1429 		s5p_hash_finish_req(req, err);
1430 		req = NULL;
1431 
1432 		/*
1433 		 * Execute next request immediately if there is anything
1434 		 * in queue.
1435 		 */
1436 		goto retry;
1437 	}
1438 
1439 	return ret;
1440 }
1441 
1442 /**
1443  * s5p_hash_tasklet_cb() - hash tasklet
1444  * @data:	ptr to s5p_aes_dev
1445  */
1446 static void s5p_hash_tasklet_cb(unsigned long data)
1447 {
1448 	struct s5p_aes_dev *dd = (struct s5p_aes_dev *)data;
1449 
1450 	if (!test_bit(HASH_FLAGS_BUSY, &dd->hash_flags)) {
1451 		s5p_hash_handle_queue(dd, NULL);
1452 		return;
1453 	}
1454 
1455 	if (test_bit(HASH_FLAGS_DMA_READY, &dd->hash_flags)) {
1456 		if (test_and_clear_bit(HASH_FLAGS_DMA_ACTIVE,
1457 				       &dd->hash_flags)) {
1458 			s5p_hash_update_dma_stop(dd);
1459 		}
1460 
1461 		if (test_and_clear_bit(HASH_FLAGS_OUTPUT_READY,
1462 				       &dd->hash_flags)) {
1463 			/* hash or semi-hash ready */
1464 			clear_bit(HASH_FLAGS_DMA_READY, &dd->hash_flags);
1465 			goto finish;
1466 		}
1467 	}
1468 
1469 	return;
1470 
1471 finish:
1472 	/* finish curent request */
1473 	s5p_hash_finish_req(dd->hash_req, 0);
1474 
1475 	/* If we are not busy, process next req */
1476 	if (!test_bit(HASH_FLAGS_BUSY, &dd->hash_flags))
1477 		s5p_hash_handle_queue(dd, NULL);
1478 }
1479 
1480 /**
1481  * s5p_hash_enqueue() - enqueue request
1482  * @req:	AHASH request
1483  * @op:		operation UPDATE (true) or FINAL (false)
1484  *
1485  * Returns: see s5p_hash_final below.
1486  */
1487 static int s5p_hash_enqueue(struct ahash_request *req, bool op)
1488 {
1489 	struct s5p_hash_reqctx *ctx = ahash_request_ctx(req);
1490 	struct s5p_hash_ctx *tctx = crypto_tfm_ctx(req->base.tfm);
1491 
1492 	ctx->op_update = op;
1493 
1494 	return s5p_hash_handle_queue(tctx->dd, req);
1495 }
1496 
1497 /**
1498  * s5p_hash_update() - process the hash input data
1499  * @req:	AHASH request
1500  *
1501  * If request will fit in buffer, copy it and return immediately
1502  * else enqueue it with OP_UPDATE.
1503  *
1504  * Returns: see s5p_hash_final below.
1505  */
1506 static int s5p_hash_update(struct ahash_request *req)
1507 {
1508 	struct s5p_hash_reqctx *ctx = ahash_request_ctx(req);
1509 
1510 	if (!req->nbytes)
1511 		return 0;
1512 
1513 	if (ctx->bufcnt + req->nbytes <= BUFLEN) {
1514 		scatterwalk_map_and_copy(ctx->buffer + ctx->bufcnt, req->src,
1515 					 0, req->nbytes, 0);
1516 		ctx->bufcnt += req->nbytes;
1517 		return 0;
1518 	}
1519 
1520 	return s5p_hash_enqueue(req, true); /* HASH_OP_UPDATE */
1521 }
1522 
1523 /**
1524  * s5p_hash_shash_digest() - calculate shash digest
1525  * @tfm:	crypto transformation
1526  * @flags:	tfm flags
1527  * @data:	input data
1528  * @len:	length of data
1529  * @out:	output buffer
1530  */
1531 static int s5p_hash_shash_digest(struct crypto_shash *tfm, u32 flags,
1532 				 const u8 *data, unsigned int len, u8 *out)
1533 {
1534 	SHASH_DESC_ON_STACK(shash, tfm);
1535 
1536 	shash->tfm = tfm;
1537 	shash->flags = flags & ~CRYPTO_TFM_REQ_MAY_SLEEP;
1538 
1539 	return crypto_shash_digest(shash, data, len, out);
1540 }
1541 
1542 /**
1543  * s5p_hash_final_shash() - calculate shash digest
1544  * @req:	AHASH request
1545  */
1546 static int s5p_hash_final_shash(struct ahash_request *req)
1547 {
1548 	struct s5p_hash_ctx *tctx = crypto_tfm_ctx(req->base.tfm);
1549 	struct s5p_hash_reqctx *ctx = ahash_request_ctx(req);
1550 
1551 	return s5p_hash_shash_digest(tctx->fallback, req->base.flags,
1552 				     ctx->buffer, ctx->bufcnt, req->result);
1553 }
1554 
1555 /**
1556  * s5p_hash_final() - close up hash and calculate digest
1557  * @req:	AHASH request
1558  *
1559  * Note: in final req->src do not have any data, and req->nbytes can be
1560  * non-zero.
1561  *
1562  * If there were no input data processed yet and the buffered hash data is
1563  * less than BUFLEN (64) then calculate the final hash immediately by using
1564  * SW algorithm fallback.
1565  *
1566  * Otherwise enqueues the current AHASH request with OP_FINAL operation op
1567  * and finalize hash message in HW. Note that if digcnt!=0 then there were
1568  * previous update op, so there are always some buffered bytes in ctx->buffer,
1569  * which means that ctx->bufcnt!=0
1570  *
1571  * Returns:
1572  * 0 if the request has been processed immediately,
1573  * -EINPROGRESS if the operation has been queued for later execution or is set
1574  *		to processing by HW,
1575  * -EBUSY if queue is full and request should be resubmitted later,
1576  * other negative values denotes an error.
1577  */
1578 static int s5p_hash_final(struct ahash_request *req)
1579 {
1580 	struct s5p_hash_reqctx *ctx = ahash_request_ctx(req);
1581 
1582 	ctx->finup = true;
1583 	if (ctx->error)
1584 		return -EINVAL; /* uncompleted hash is not needed */
1585 
1586 	if (!ctx->digcnt && ctx->bufcnt < BUFLEN)
1587 		return s5p_hash_final_shash(req);
1588 
1589 	return s5p_hash_enqueue(req, false); /* HASH_OP_FINAL */
1590 }
1591 
1592 /**
1593  * s5p_hash_finup() - process last req->src and calculate digest
1594  * @req:	AHASH request containing the last update data
1595  *
1596  * Return values: see s5p_hash_final above.
1597  */
1598 static int s5p_hash_finup(struct ahash_request *req)
1599 {
1600 	struct s5p_hash_reqctx *ctx = ahash_request_ctx(req);
1601 	int err1, err2;
1602 
1603 	ctx->finup = true;
1604 
1605 	err1 = s5p_hash_update(req);
1606 	if (err1 == -EINPROGRESS || err1 == -EBUSY)
1607 		return err1;
1608 
1609 	/*
1610 	 * final() has to be always called to cleanup resources even if
1611 	 * update() failed, except EINPROGRESS or calculate digest for small
1612 	 * size
1613 	 */
1614 	err2 = s5p_hash_final(req);
1615 
1616 	return err1 ?: err2;
1617 }
1618 
1619 /**
1620  * s5p_hash_init() - initialize AHASH request contex
1621  * @req:	AHASH request
1622  *
1623  * Init async hash request context.
1624  */
1625 static int s5p_hash_init(struct ahash_request *req)
1626 {
1627 	struct s5p_hash_reqctx *ctx = ahash_request_ctx(req);
1628 	struct crypto_ahash *tfm = crypto_ahash_reqtfm(req);
1629 	struct s5p_hash_ctx *tctx = crypto_ahash_ctx(tfm);
1630 
1631 	ctx->dd = tctx->dd;
1632 	ctx->error = false;
1633 	ctx->finup = false;
1634 	ctx->bufcnt = 0;
1635 	ctx->digcnt = 0;
1636 	ctx->total = 0;
1637 	ctx->skip = 0;
1638 
1639 	dev_dbg(tctx->dd->dev, "init: digest size: %d\n",
1640 		crypto_ahash_digestsize(tfm));
1641 
1642 	switch (crypto_ahash_digestsize(tfm)) {
1643 	case MD5_DIGEST_SIZE:
1644 		ctx->engine = SSS_HASH_ENGINE_MD5;
1645 		ctx->nregs = HASH_MD5_MAX_REG;
1646 		break;
1647 	case SHA1_DIGEST_SIZE:
1648 		ctx->engine = SSS_HASH_ENGINE_SHA1;
1649 		ctx->nregs = HASH_SHA1_MAX_REG;
1650 		break;
1651 	case SHA256_DIGEST_SIZE:
1652 		ctx->engine = SSS_HASH_ENGINE_SHA256;
1653 		ctx->nregs = HASH_SHA256_MAX_REG;
1654 		break;
1655 	default:
1656 		ctx->error = true;
1657 		return -EINVAL;
1658 	}
1659 
1660 	return 0;
1661 }
1662 
1663 /**
1664  * s5p_hash_digest - calculate digest from req->src
1665  * @req:	AHASH request
1666  *
1667  * Return values: see s5p_hash_final above.
1668  */
1669 static int s5p_hash_digest(struct ahash_request *req)
1670 {
1671 	return s5p_hash_init(req) ?: s5p_hash_finup(req);
1672 }
1673 
1674 /**
1675  * s5p_hash_cra_init_alg - init crypto alg transformation
1676  * @tfm:	crypto transformation
1677  */
1678 static int s5p_hash_cra_init_alg(struct crypto_tfm *tfm)
1679 {
1680 	struct s5p_hash_ctx *tctx = crypto_tfm_ctx(tfm);
1681 	const char *alg_name = crypto_tfm_alg_name(tfm);
1682 
1683 	tctx->dd = s5p_dev;
1684 	/* Allocate a fallback and abort if it failed. */
1685 	tctx->fallback = crypto_alloc_shash(alg_name, 0,
1686 					    CRYPTO_ALG_NEED_FALLBACK);
1687 	if (IS_ERR(tctx->fallback)) {
1688 		pr_err("fallback alloc fails for '%s'\n", alg_name);
1689 		return PTR_ERR(tctx->fallback);
1690 	}
1691 
1692 	crypto_ahash_set_reqsize(__crypto_ahash_cast(tfm),
1693 				 sizeof(struct s5p_hash_reqctx) + BUFLEN);
1694 
1695 	return 0;
1696 }
1697 
1698 /**
1699  * s5p_hash_cra_init - init crypto tfm
1700  * @tfm:	crypto transformation
1701  */
1702 static int s5p_hash_cra_init(struct crypto_tfm *tfm)
1703 {
1704 	return s5p_hash_cra_init_alg(tfm);
1705 }
1706 
1707 /**
1708  * s5p_hash_cra_exit - exit crypto tfm
1709  * @tfm:	crypto transformation
1710  *
1711  * free allocated fallback
1712  */
1713 static void s5p_hash_cra_exit(struct crypto_tfm *tfm)
1714 {
1715 	struct s5p_hash_ctx *tctx = crypto_tfm_ctx(tfm);
1716 
1717 	crypto_free_shash(tctx->fallback);
1718 	tctx->fallback = NULL;
1719 }
1720 
1721 /**
1722  * s5p_hash_export - export hash state
1723  * @req:	AHASH request
1724  * @out:	buffer for exported state
1725  */
1726 static int s5p_hash_export(struct ahash_request *req, void *out)
1727 {
1728 	const struct s5p_hash_reqctx *ctx = ahash_request_ctx(req);
1729 
1730 	memcpy(out, ctx, sizeof(*ctx) + ctx->bufcnt);
1731 
1732 	return 0;
1733 }
1734 
1735 /**
1736  * s5p_hash_import - import hash state
1737  * @req:	AHASH request
1738  * @in:		buffer with state to be imported from
1739  */
1740 static int s5p_hash_import(struct ahash_request *req, const void *in)
1741 {
1742 	struct s5p_hash_reqctx *ctx = ahash_request_ctx(req);
1743 	struct crypto_ahash *tfm = crypto_ahash_reqtfm(req);
1744 	struct s5p_hash_ctx *tctx = crypto_ahash_ctx(tfm);
1745 	const struct s5p_hash_reqctx *ctx_in = in;
1746 
1747 	memcpy(ctx, in, sizeof(*ctx) + BUFLEN);
1748 	if (ctx_in->bufcnt > BUFLEN) {
1749 		ctx->error = true;
1750 		return -EINVAL;
1751 	}
1752 
1753 	ctx->dd = tctx->dd;
1754 	ctx->error = false;
1755 
1756 	return 0;
1757 }
1758 
1759 static struct ahash_alg algs_sha1_md5_sha256[] = {
1760 {
1761 	.init		= s5p_hash_init,
1762 	.update		= s5p_hash_update,
1763 	.final		= s5p_hash_final,
1764 	.finup		= s5p_hash_finup,
1765 	.digest		= s5p_hash_digest,
1766 	.export		= s5p_hash_export,
1767 	.import		= s5p_hash_import,
1768 	.halg.statesize = sizeof(struct s5p_hash_reqctx) + BUFLEN,
1769 	.halg.digestsize	= SHA1_DIGEST_SIZE,
1770 	.halg.base	= {
1771 		.cra_name		= "sha1",
1772 		.cra_driver_name	= "exynos-sha1",
1773 		.cra_priority		= 100,
1774 		.cra_flags		= CRYPTO_ALG_KERN_DRIVER_ONLY |
1775 					  CRYPTO_ALG_ASYNC |
1776 					  CRYPTO_ALG_NEED_FALLBACK,
1777 		.cra_blocksize		= HASH_BLOCK_SIZE,
1778 		.cra_ctxsize		= sizeof(struct s5p_hash_ctx),
1779 		.cra_alignmask		= SSS_HASH_DMA_ALIGN_MASK,
1780 		.cra_module		= THIS_MODULE,
1781 		.cra_init		= s5p_hash_cra_init,
1782 		.cra_exit		= s5p_hash_cra_exit,
1783 	}
1784 },
1785 {
1786 	.init		= s5p_hash_init,
1787 	.update		= s5p_hash_update,
1788 	.final		= s5p_hash_final,
1789 	.finup		= s5p_hash_finup,
1790 	.digest		= s5p_hash_digest,
1791 	.export		= s5p_hash_export,
1792 	.import		= s5p_hash_import,
1793 	.halg.statesize = sizeof(struct s5p_hash_reqctx) + BUFLEN,
1794 	.halg.digestsize	= MD5_DIGEST_SIZE,
1795 	.halg.base	= {
1796 		.cra_name		= "md5",
1797 		.cra_driver_name	= "exynos-md5",
1798 		.cra_priority		= 100,
1799 		.cra_flags		= CRYPTO_ALG_KERN_DRIVER_ONLY |
1800 					  CRYPTO_ALG_ASYNC |
1801 					  CRYPTO_ALG_NEED_FALLBACK,
1802 		.cra_blocksize		= HASH_BLOCK_SIZE,
1803 		.cra_ctxsize		= sizeof(struct s5p_hash_ctx),
1804 		.cra_alignmask		= SSS_HASH_DMA_ALIGN_MASK,
1805 		.cra_module		= THIS_MODULE,
1806 		.cra_init		= s5p_hash_cra_init,
1807 		.cra_exit		= s5p_hash_cra_exit,
1808 	}
1809 },
1810 {
1811 	.init		= s5p_hash_init,
1812 	.update		= s5p_hash_update,
1813 	.final		= s5p_hash_final,
1814 	.finup		= s5p_hash_finup,
1815 	.digest		= s5p_hash_digest,
1816 	.export		= s5p_hash_export,
1817 	.import		= s5p_hash_import,
1818 	.halg.statesize = sizeof(struct s5p_hash_reqctx) + BUFLEN,
1819 	.halg.digestsize	= SHA256_DIGEST_SIZE,
1820 	.halg.base	= {
1821 		.cra_name		= "sha256",
1822 		.cra_driver_name	= "exynos-sha256",
1823 		.cra_priority		= 100,
1824 		.cra_flags		= CRYPTO_ALG_KERN_DRIVER_ONLY |
1825 					  CRYPTO_ALG_ASYNC |
1826 					  CRYPTO_ALG_NEED_FALLBACK,
1827 		.cra_blocksize		= HASH_BLOCK_SIZE,
1828 		.cra_ctxsize		= sizeof(struct s5p_hash_ctx),
1829 		.cra_alignmask		= SSS_HASH_DMA_ALIGN_MASK,
1830 		.cra_module		= THIS_MODULE,
1831 		.cra_init		= s5p_hash_cra_init,
1832 		.cra_exit		= s5p_hash_cra_exit,
1833 	}
1834 }
1835 
1836 };
1837 
1838 static void s5p_set_aes(struct s5p_aes_dev *dev,
1839 			const u8 *key, const u8 *iv, const u8 *ctr,
1840 			unsigned int keylen)
1841 {
1842 	void __iomem *keystart;
1843 
1844 	if (iv)
1845 		memcpy_toio(dev->aes_ioaddr + SSS_REG_AES_IV_DATA(0), iv,
1846 			    AES_BLOCK_SIZE);
1847 
1848 	if (ctr)
1849 		memcpy_toio(dev->aes_ioaddr + SSS_REG_AES_CNT_DATA(0), ctr,
1850 			    AES_BLOCK_SIZE);
1851 
1852 	if (keylen == AES_KEYSIZE_256)
1853 		keystart = dev->aes_ioaddr + SSS_REG_AES_KEY_DATA(0);
1854 	else if (keylen == AES_KEYSIZE_192)
1855 		keystart = dev->aes_ioaddr + SSS_REG_AES_KEY_DATA(2);
1856 	else
1857 		keystart = dev->aes_ioaddr + SSS_REG_AES_KEY_DATA(4);
1858 
1859 	memcpy_toio(keystart, key, keylen);
1860 }
1861 
1862 static bool s5p_is_sg_aligned(struct scatterlist *sg)
1863 {
1864 	while (sg) {
1865 		if (!IS_ALIGNED(sg->length, AES_BLOCK_SIZE))
1866 			return false;
1867 		sg = sg_next(sg);
1868 	}
1869 
1870 	return true;
1871 }
1872 
1873 static int s5p_set_indata_start(struct s5p_aes_dev *dev,
1874 				struct ablkcipher_request *req)
1875 {
1876 	struct scatterlist *sg;
1877 	int err;
1878 
1879 	dev->sg_src_cpy = NULL;
1880 	sg = req->src;
1881 	if (!s5p_is_sg_aligned(sg)) {
1882 		dev_dbg(dev->dev,
1883 			"At least one unaligned source scatter list, making a copy\n");
1884 		err = s5p_make_sg_cpy(dev, sg, &dev->sg_src_cpy);
1885 		if (err)
1886 			return err;
1887 
1888 		sg = dev->sg_src_cpy;
1889 	}
1890 
1891 	err = s5p_set_indata(dev, sg);
1892 	if (err) {
1893 		s5p_free_sg_cpy(dev, &dev->sg_src_cpy);
1894 		return err;
1895 	}
1896 
1897 	return 0;
1898 }
1899 
1900 static int s5p_set_outdata_start(struct s5p_aes_dev *dev,
1901 				 struct ablkcipher_request *req)
1902 {
1903 	struct scatterlist *sg;
1904 	int err;
1905 
1906 	dev->sg_dst_cpy = NULL;
1907 	sg = req->dst;
1908 	if (!s5p_is_sg_aligned(sg)) {
1909 		dev_dbg(dev->dev,
1910 			"At least one unaligned dest scatter list, making a copy\n");
1911 		err = s5p_make_sg_cpy(dev, sg, &dev->sg_dst_cpy);
1912 		if (err)
1913 			return err;
1914 
1915 		sg = dev->sg_dst_cpy;
1916 	}
1917 
1918 	err = s5p_set_outdata(dev, sg);
1919 	if (err) {
1920 		s5p_free_sg_cpy(dev, &dev->sg_dst_cpy);
1921 		return err;
1922 	}
1923 
1924 	return 0;
1925 }
1926 
1927 static void s5p_aes_crypt_start(struct s5p_aes_dev *dev, unsigned long mode)
1928 {
1929 	struct ablkcipher_request *req = dev->req;
1930 	u32 aes_control;
1931 	unsigned long flags;
1932 	int err;
1933 	u8 *iv, *ctr;
1934 
1935 	/* This sets bit [13:12] to 00, which selects 128-bit counter */
1936 	aes_control = SSS_AES_KEY_CHANGE_MODE;
1937 	if (mode & FLAGS_AES_DECRYPT)
1938 		aes_control |= SSS_AES_MODE_DECRYPT;
1939 
1940 	if ((mode & FLAGS_AES_MODE_MASK) == FLAGS_AES_CBC) {
1941 		aes_control |= SSS_AES_CHAIN_MODE_CBC;
1942 		iv = req->info;
1943 		ctr = NULL;
1944 	} else if ((mode & FLAGS_AES_MODE_MASK) == FLAGS_AES_CTR) {
1945 		aes_control |= SSS_AES_CHAIN_MODE_CTR;
1946 		iv = NULL;
1947 		ctr = req->info;
1948 	} else {
1949 		iv = NULL; /* AES_ECB */
1950 		ctr = NULL;
1951 	}
1952 
1953 	if (dev->ctx->keylen == AES_KEYSIZE_192)
1954 		aes_control |= SSS_AES_KEY_SIZE_192;
1955 	else if (dev->ctx->keylen == AES_KEYSIZE_256)
1956 		aes_control |= SSS_AES_KEY_SIZE_256;
1957 
1958 	aes_control |= SSS_AES_FIFO_MODE;
1959 
1960 	/* as a variant it is possible to use byte swapping on DMA side */
1961 	aes_control |= SSS_AES_BYTESWAP_DI
1962 		    |  SSS_AES_BYTESWAP_DO
1963 		    |  SSS_AES_BYTESWAP_IV
1964 		    |  SSS_AES_BYTESWAP_KEY
1965 		    |  SSS_AES_BYTESWAP_CNT;
1966 
1967 	spin_lock_irqsave(&dev->lock, flags);
1968 
1969 	SSS_WRITE(dev, FCINTENCLR,
1970 		  SSS_FCINTENCLR_BTDMAINTENCLR | SSS_FCINTENCLR_BRDMAINTENCLR);
1971 	SSS_WRITE(dev, FCFIFOCTRL, 0x00);
1972 
1973 	err = s5p_set_indata_start(dev, req);
1974 	if (err)
1975 		goto indata_error;
1976 
1977 	err = s5p_set_outdata_start(dev, req);
1978 	if (err)
1979 		goto outdata_error;
1980 
1981 	SSS_AES_WRITE(dev, AES_CONTROL, aes_control);
1982 	s5p_set_aes(dev, dev->ctx->aes_key, iv, ctr, dev->ctx->keylen);
1983 
1984 	s5p_set_dma_indata(dev,  dev->sg_src);
1985 	s5p_set_dma_outdata(dev, dev->sg_dst);
1986 
1987 	SSS_WRITE(dev, FCINTENSET,
1988 		  SSS_FCINTENSET_BTDMAINTENSET | SSS_FCINTENSET_BRDMAINTENSET);
1989 
1990 	spin_unlock_irqrestore(&dev->lock, flags);
1991 
1992 	return;
1993 
1994 outdata_error:
1995 	s5p_unset_indata(dev);
1996 
1997 indata_error:
1998 	s5p_sg_done(dev);
1999 	dev->busy = false;
2000 	spin_unlock_irqrestore(&dev->lock, flags);
2001 	s5p_aes_complete(req, err);
2002 }
2003 
2004 static void s5p_tasklet_cb(unsigned long data)
2005 {
2006 	struct s5p_aes_dev *dev = (struct s5p_aes_dev *)data;
2007 	struct crypto_async_request *async_req, *backlog;
2008 	struct s5p_aes_reqctx *reqctx;
2009 	unsigned long flags;
2010 
2011 	spin_lock_irqsave(&dev->lock, flags);
2012 	backlog   = crypto_get_backlog(&dev->queue);
2013 	async_req = crypto_dequeue_request(&dev->queue);
2014 
2015 	if (!async_req) {
2016 		dev->busy = false;
2017 		spin_unlock_irqrestore(&dev->lock, flags);
2018 		return;
2019 	}
2020 	spin_unlock_irqrestore(&dev->lock, flags);
2021 
2022 	if (backlog)
2023 		backlog->complete(backlog, -EINPROGRESS);
2024 
2025 	dev->req = ablkcipher_request_cast(async_req);
2026 	dev->ctx = crypto_tfm_ctx(dev->req->base.tfm);
2027 	reqctx   = ablkcipher_request_ctx(dev->req);
2028 
2029 	s5p_aes_crypt_start(dev, reqctx->mode);
2030 }
2031 
2032 static int s5p_aes_handle_req(struct s5p_aes_dev *dev,
2033 			      struct ablkcipher_request *req)
2034 {
2035 	unsigned long flags;
2036 	int err;
2037 
2038 	spin_lock_irqsave(&dev->lock, flags);
2039 	err = ablkcipher_enqueue_request(&dev->queue, req);
2040 	if (dev->busy) {
2041 		spin_unlock_irqrestore(&dev->lock, flags);
2042 		return err;
2043 	}
2044 	dev->busy = true;
2045 
2046 	spin_unlock_irqrestore(&dev->lock, flags);
2047 
2048 	tasklet_schedule(&dev->tasklet);
2049 
2050 	return err;
2051 }
2052 
2053 static int s5p_aes_crypt(struct ablkcipher_request *req, unsigned long mode)
2054 {
2055 	struct crypto_ablkcipher *tfm = crypto_ablkcipher_reqtfm(req);
2056 	struct s5p_aes_reqctx *reqctx = ablkcipher_request_ctx(req);
2057 	struct s5p_aes_ctx *ctx = crypto_ablkcipher_ctx(tfm);
2058 	struct s5p_aes_dev *dev = ctx->dev;
2059 
2060 	if (!IS_ALIGNED(req->nbytes, AES_BLOCK_SIZE) &&
2061 			((mode & FLAGS_AES_MODE_MASK) != FLAGS_AES_CTR)) {
2062 		dev_err(dev->dev, "request size is not exact amount of AES blocks\n");
2063 		return -EINVAL;
2064 	}
2065 
2066 	reqctx->mode = mode;
2067 
2068 	return s5p_aes_handle_req(dev, req);
2069 }
2070 
2071 static int s5p_aes_setkey(struct crypto_ablkcipher *cipher,
2072 			  const u8 *key, unsigned int keylen)
2073 {
2074 	struct crypto_tfm *tfm = crypto_ablkcipher_tfm(cipher);
2075 	struct s5p_aes_ctx *ctx = crypto_tfm_ctx(tfm);
2076 
2077 	if (keylen != AES_KEYSIZE_128 &&
2078 	    keylen != AES_KEYSIZE_192 &&
2079 	    keylen != AES_KEYSIZE_256)
2080 		return -EINVAL;
2081 
2082 	memcpy(ctx->aes_key, key, keylen);
2083 	ctx->keylen = keylen;
2084 
2085 	return 0;
2086 }
2087 
2088 static int s5p_aes_ecb_encrypt(struct ablkcipher_request *req)
2089 {
2090 	return s5p_aes_crypt(req, 0);
2091 }
2092 
2093 static int s5p_aes_ecb_decrypt(struct ablkcipher_request *req)
2094 {
2095 	return s5p_aes_crypt(req, FLAGS_AES_DECRYPT);
2096 }
2097 
2098 static int s5p_aes_cbc_encrypt(struct ablkcipher_request *req)
2099 {
2100 	return s5p_aes_crypt(req, FLAGS_AES_CBC);
2101 }
2102 
2103 static int s5p_aes_cbc_decrypt(struct ablkcipher_request *req)
2104 {
2105 	return s5p_aes_crypt(req, FLAGS_AES_DECRYPT | FLAGS_AES_CBC);
2106 }
2107 
2108 static int s5p_aes_ctr_crypt(struct ablkcipher_request *req)
2109 {
2110 	return s5p_aes_crypt(req, FLAGS_AES_CTR);
2111 }
2112 
2113 static int s5p_aes_cra_init(struct crypto_tfm *tfm)
2114 {
2115 	struct s5p_aes_ctx *ctx = crypto_tfm_ctx(tfm);
2116 
2117 	ctx->dev = s5p_dev;
2118 	tfm->crt_ablkcipher.reqsize = sizeof(struct s5p_aes_reqctx);
2119 
2120 	return 0;
2121 }
2122 
2123 static struct crypto_alg algs[] = {
2124 	{
2125 		.cra_name		= "ecb(aes)",
2126 		.cra_driver_name	= "ecb-aes-s5p",
2127 		.cra_priority		= 100,
2128 		.cra_flags		= CRYPTO_ALG_TYPE_ABLKCIPHER |
2129 					  CRYPTO_ALG_ASYNC |
2130 					  CRYPTO_ALG_KERN_DRIVER_ONLY,
2131 		.cra_blocksize		= AES_BLOCK_SIZE,
2132 		.cra_ctxsize		= sizeof(struct s5p_aes_ctx),
2133 		.cra_alignmask		= 0x0f,
2134 		.cra_type		= &crypto_ablkcipher_type,
2135 		.cra_module		= THIS_MODULE,
2136 		.cra_init		= s5p_aes_cra_init,
2137 		.cra_u.ablkcipher = {
2138 			.min_keysize	= AES_MIN_KEY_SIZE,
2139 			.max_keysize	= AES_MAX_KEY_SIZE,
2140 			.setkey		= s5p_aes_setkey,
2141 			.encrypt	= s5p_aes_ecb_encrypt,
2142 			.decrypt	= s5p_aes_ecb_decrypt,
2143 		}
2144 	},
2145 	{
2146 		.cra_name		= "cbc(aes)",
2147 		.cra_driver_name	= "cbc-aes-s5p",
2148 		.cra_priority		= 100,
2149 		.cra_flags		= CRYPTO_ALG_TYPE_ABLKCIPHER |
2150 					  CRYPTO_ALG_ASYNC |
2151 					  CRYPTO_ALG_KERN_DRIVER_ONLY,
2152 		.cra_blocksize		= AES_BLOCK_SIZE,
2153 		.cra_ctxsize		= sizeof(struct s5p_aes_ctx),
2154 		.cra_alignmask		= 0x0f,
2155 		.cra_type		= &crypto_ablkcipher_type,
2156 		.cra_module		= THIS_MODULE,
2157 		.cra_init		= s5p_aes_cra_init,
2158 		.cra_u.ablkcipher = {
2159 			.min_keysize	= AES_MIN_KEY_SIZE,
2160 			.max_keysize	= AES_MAX_KEY_SIZE,
2161 			.ivsize		= AES_BLOCK_SIZE,
2162 			.setkey		= s5p_aes_setkey,
2163 			.encrypt	= s5p_aes_cbc_encrypt,
2164 			.decrypt	= s5p_aes_cbc_decrypt,
2165 		}
2166 	},
2167 	{
2168 		.cra_name		= "ctr(aes)",
2169 		.cra_driver_name	= "ctr-aes-s5p",
2170 		.cra_priority		= 100,
2171 		.cra_flags		= CRYPTO_ALG_TYPE_ABLKCIPHER |
2172 					  CRYPTO_ALG_ASYNC |
2173 					  CRYPTO_ALG_KERN_DRIVER_ONLY,
2174 		.cra_blocksize		= AES_BLOCK_SIZE,
2175 		.cra_ctxsize		= sizeof(struct s5p_aes_ctx),
2176 		.cra_alignmask		= 0x0f,
2177 		.cra_type		= &crypto_ablkcipher_type,
2178 		.cra_module		= THIS_MODULE,
2179 		.cra_init		= s5p_aes_cra_init,
2180 		.cra_u.ablkcipher = {
2181 			.min_keysize	= AES_MIN_KEY_SIZE,
2182 			.max_keysize	= AES_MAX_KEY_SIZE,
2183 			.ivsize		= AES_BLOCK_SIZE,
2184 			.setkey		= s5p_aes_setkey,
2185 			.encrypt	= s5p_aes_ctr_crypt,
2186 			.decrypt	= s5p_aes_ctr_crypt,
2187 		}
2188 	},
2189 };
2190 
2191 static int s5p_aes_probe(struct platform_device *pdev)
2192 {
2193 	struct device *dev = &pdev->dev;
2194 	int i, j, err = -ENODEV;
2195 	const struct samsung_aes_variant *variant;
2196 	struct s5p_aes_dev *pdata;
2197 	struct resource *res;
2198 	unsigned int hash_i;
2199 
2200 	if (s5p_dev)
2201 		return -EEXIST;
2202 
2203 	pdata = devm_kzalloc(dev, sizeof(*pdata), GFP_KERNEL);
2204 	if (!pdata)
2205 		return -ENOMEM;
2206 
2207 	variant = find_s5p_sss_version(pdev);
2208 	res = platform_get_resource(pdev, IORESOURCE_MEM, 0);
2209 
2210 	/*
2211 	 * Note: HASH and PRNG uses the same registers in secss, avoid
2212 	 * overwrite each other. This will drop HASH when CONFIG_EXYNOS_RNG
2213 	 * is enabled in config. We need larger size for HASH registers in
2214 	 * secss, current describe only AES/DES
2215 	 */
2216 	if (IS_ENABLED(CONFIG_CRYPTO_DEV_EXYNOS_HASH)) {
2217 		if (variant == &exynos_aes_data) {
2218 			res->end += 0x300;
2219 			pdata->use_hash = true;
2220 		}
2221 	}
2222 
2223 	pdata->res = res;
2224 	pdata->ioaddr = devm_ioremap_resource(&pdev->dev, res);
2225 	if (IS_ERR(pdata->ioaddr)) {
2226 		if (!pdata->use_hash)
2227 			return PTR_ERR(pdata->ioaddr);
2228 		/* try AES without HASH */
2229 		res->end -= 0x300;
2230 		pdata->use_hash = false;
2231 		pdata->ioaddr = devm_ioremap_resource(&pdev->dev, res);
2232 		if (IS_ERR(pdata->ioaddr))
2233 			return PTR_ERR(pdata->ioaddr);
2234 	}
2235 
2236 	pdata->clk = devm_clk_get(dev, variant->clk_names[0]);
2237 	if (IS_ERR(pdata->clk)) {
2238 		dev_err(dev, "failed to find secss clock %s\n",
2239 			variant->clk_names[0]);
2240 		return -ENOENT;
2241 	}
2242 
2243 	err = clk_prepare_enable(pdata->clk);
2244 	if (err < 0) {
2245 		dev_err(dev, "Enabling clock %s failed, err %d\n",
2246 			variant->clk_names[0], err);
2247 		return err;
2248 	}
2249 
2250 	if (variant->clk_names[1]) {
2251 		pdata->pclk = devm_clk_get(dev, variant->clk_names[1]);
2252 		if (IS_ERR(pdata->pclk)) {
2253 			dev_err(dev, "failed to find clock %s\n",
2254 				variant->clk_names[1]);
2255 			err = -ENOENT;
2256 			goto err_clk;
2257 		}
2258 
2259 		err = clk_prepare_enable(pdata->pclk);
2260 		if (err < 0) {
2261 			dev_err(dev, "Enabling clock %s failed, err %d\n",
2262 				variant->clk_names[0], err);
2263 			goto err_clk;
2264 		}
2265 	} else {
2266 		pdata->pclk = NULL;
2267 	}
2268 
2269 	spin_lock_init(&pdata->lock);
2270 	spin_lock_init(&pdata->hash_lock);
2271 
2272 	pdata->aes_ioaddr = pdata->ioaddr + variant->aes_offset;
2273 	pdata->io_hash_base = pdata->ioaddr + variant->hash_offset;
2274 
2275 	pdata->irq_fc = platform_get_irq(pdev, 0);
2276 	if (pdata->irq_fc < 0) {
2277 		err = pdata->irq_fc;
2278 		dev_warn(dev, "feed control interrupt is not available.\n");
2279 		goto err_irq;
2280 	}
2281 	err = devm_request_threaded_irq(dev, pdata->irq_fc, NULL,
2282 					s5p_aes_interrupt, IRQF_ONESHOT,
2283 					pdev->name, pdev);
2284 	if (err < 0) {
2285 		dev_warn(dev, "feed control interrupt is not available.\n");
2286 		goto err_irq;
2287 	}
2288 
2289 	pdata->busy = false;
2290 	pdata->dev = dev;
2291 	platform_set_drvdata(pdev, pdata);
2292 	s5p_dev = pdata;
2293 
2294 	tasklet_init(&pdata->tasklet, s5p_tasklet_cb, (unsigned long)pdata);
2295 	crypto_init_queue(&pdata->queue, CRYPTO_QUEUE_LEN);
2296 
2297 	for (i = 0; i < ARRAY_SIZE(algs); i++) {
2298 		err = crypto_register_alg(&algs[i]);
2299 		if (err)
2300 			goto err_algs;
2301 	}
2302 
2303 	if (pdata->use_hash) {
2304 		tasklet_init(&pdata->hash_tasklet, s5p_hash_tasklet_cb,
2305 			     (unsigned long)pdata);
2306 		crypto_init_queue(&pdata->hash_queue, SSS_HASH_QUEUE_LENGTH);
2307 
2308 		for (hash_i = 0; hash_i < ARRAY_SIZE(algs_sha1_md5_sha256);
2309 		     hash_i++) {
2310 			struct ahash_alg *alg;
2311 
2312 			alg = &algs_sha1_md5_sha256[hash_i];
2313 			err = crypto_register_ahash(alg);
2314 			if (err) {
2315 				dev_err(dev, "can't register '%s': %d\n",
2316 					alg->halg.base.cra_driver_name, err);
2317 				goto err_hash;
2318 			}
2319 		}
2320 	}
2321 
2322 	dev_info(dev, "s5p-sss driver registered\n");
2323 
2324 	return 0;
2325 
2326 err_hash:
2327 	for (j = hash_i - 1; j >= 0; j--)
2328 		crypto_unregister_ahash(&algs_sha1_md5_sha256[j]);
2329 
2330 	tasklet_kill(&pdata->hash_tasklet);
2331 	res->end -= 0x300;
2332 
2333 err_algs:
2334 	if (i < ARRAY_SIZE(algs))
2335 		dev_err(dev, "can't register '%s': %d\n", algs[i].cra_name,
2336 			err);
2337 
2338 	for (j = 0; j < i; j++)
2339 		crypto_unregister_alg(&algs[j]);
2340 
2341 	tasklet_kill(&pdata->tasklet);
2342 
2343 err_irq:
2344 	if (pdata->pclk)
2345 		clk_disable_unprepare(pdata->pclk);
2346 
2347 err_clk:
2348 	clk_disable_unprepare(pdata->clk);
2349 	s5p_dev = NULL;
2350 
2351 	return err;
2352 }
2353 
2354 static int s5p_aes_remove(struct platform_device *pdev)
2355 {
2356 	struct s5p_aes_dev *pdata = platform_get_drvdata(pdev);
2357 	int i;
2358 
2359 	if (!pdata)
2360 		return -ENODEV;
2361 
2362 	for (i = 0; i < ARRAY_SIZE(algs); i++)
2363 		crypto_unregister_alg(&algs[i]);
2364 
2365 	tasklet_kill(&pdata->tasklet);
2366 	if (pdata->use_hash) {
2367 		for (i = ARRAY_SIZE(algs_sha1_md5_sha256) - 1; i >= 0; i--)
2368 			crypto_unregister_ahash(&algs_sha1_md5_sha256[i]);
2369 
2370 		pdata->res->end -= 0x300;
2371 		tasklet_kill(&pdata->hash_tasklet);
2372 		pdata->use_hash = false;
2373 	}
2374 
2375 	if (pdata->pclk)
2376 		clk_disable_unprepare(pdata->pclk);
2377 
2378 	clk_disable_unprepare(pdata->clk);
2379 	s5p_dev = NULL;
2380 
2381 	return 0;
2382 }
2383 
2384 static struct platform_driver s5p_aes_crypto = {
2385 	.probe	= s5p_aes_probe,
2386 	.remove	= s5p_aes_remove,
2387 	.driver	= {
2388 		.name	= "s5p-secss",
2389 		.of_match_table = s5p_sss_dt_match,
2390 	},
2391 };
2392 
2393 module_platform_driver(s5p_aes_crypto);
2394 
2395 MODULE_DESCRIPTION("S5PV210 AES hw acceleration support.");
2396 MODULE_LICENSE("GPL v2");
2397 MODULE_AUTHOR("Vladimir Zapolskiy <vzapolskiy@gmail.com>");
2398 MODULE_AUTHOR("Kamil Konieczny <k.konieczny@partner.samsung.com>");
2399