xref: /openbmc/linux/drivers/crypto/omap-sham.c (revision b0cc7491)
1 // SPDX-License-Identifier: GPL-2.0-only
2 /*
3  * Cryptographic API.
4  *
5  * Support for OMAP SHA1/MD5 HW acceleration.
6  *
7  * Copyright (c) 2010 Nokia Corporation
8  * Author: Dmitry Kasatkin <dmitry.kasatkin@nokia.com>
9  * Copyright (c) 2011 Texas Instruments Incorporated
10  *
11  * Some ideas are from old omap-sha1-md5.c driver.
12  */
13 
14 #define pr_fmt(fmt) "%s: " fmt, __func__
15 
16 #include <crypto/engine.h>
17 #include <crypto/hmac.h>
18 #include <crypto/internal/hash.h>
19 #include <crypto/scatterwalk.h>
20 #include <crypto/sha1.h>
21 #include <crypto/sha2.h>
22 #include <linux/err.h>
23 #include <linux/device.h>
24 #include <linux/dma-mapping.h>
25 #include <linux/dmaengine.h>
26 #include <linux/init.h>
27 #include <linux/interrupt.h>
28 #include <linux/io.h>
29 #include <linux/irq.h>
30 #include <linux/kernel.h>
31 #include <linux/module.h>
32 #include <linux/of.h>
33 #include <linux/of_address.h>
34 #include <linux/of_irq.h>
35 #include <linux/platform_device.h>
36 #include <linux/pm_runtime.h>
37 #include <linux/scatterlist.h>
38 #include <linux/slab.h>
39 #include <linux/string.h>
40 
41 #define MD5_DIGEST_SIZE			16
42 
43 #define SHA_REG_IDIGEST(dd, x)		((dd)->pdata->idigest_ofs + ((x)*0x04))
44 #define SHA_REG_DIN(dd, x)		((dd)->pdata->din_ofs + ((x) * 0x04))
45 #define SHA_REG_DIGCNT(dd)		((dd)->pdata->digcnt_ofs)
46 
47 #define SHA_REG_ODIGEST(dd, x)		((dd)->pdata->odigest_ofs + (x * 0x04))
48 
49 #define SHA_REG_CTRL			0x18
50 #define SHA_REG_CTRL_LENGTH		(0xFFFFFFFF << 5)
51 #define SHA_REG_CTRL_CLOSE_HASH		(1 << 4)
52 #define SHA_REG_CTRL_ALGO_CONST		(1 << 3)
53 #define SHA_REG_CTRL_ALGO		(1 << 2)
54 #define SHA_REG_CTRL_INPUT_READY	(1 << 1)
55 #define SHA_REG_CTRL_OUTPUT_READY	(1 << 0)
56 
57 #define SHA_REG_REV(dd)			((dd)->pdata->rev_ofs)
58 
59 #define SHA_REG_MASK(dd)		((dd)->pdata->mask_ofs)
60 #define SHA_REG_MASK_DMA_EN		(1 << 3)
61 #define SHA_REG_MASK_IT_EN		(1 << 2)
62 #define SHA_REG_MASK_SOFTRESET		(1 << 1)
63 #define SHA_REG_AUTOIDLE		(1 << 0)
64 
65 #define SHA_REG_SYSSTATUS(dd)		((dd)->pdata->sysstatus_ofs)
66 #define SHA_REG_SYSSTATUS_RESETDONE	(1 << 0)
67 
68 #define SHA_REG_MODE(dd)		((dd)->pdata->mode_ofs)
69 #define SHA_REG_MODE_HMAC_OUTER_HASH	(1 << 7)
70 #define SHA_REG_MODE_HMAC_KEY_PROC	(1 << 5)
71 #define SHA_REG_MODE_CLOSE_HASH		(1 << 4)
72 #define SHA_REG_MODE_ALGO_CONSTANT	(1 << 3)
73 
74 #define SHA_REG_MODE_ALGO_MASK		(7 << 0)
75 #define SHA_REG_MODE_ALGO_MD5_128	(0 << 1)
76 #define SHA_REG_MODE_ALGO_SHA1_160	(1 << 1)
77 #define SHA_REG_MODE_ALGO_SHA2_224	(2 << 1)
78 #define SHA_REG_MODE_ALGO_SHA2_256	(3 << 1)
79 #define SHA_REG_MODE_ALGO_SHA2_384	(1 << 0)
80 #define SHA_REG_MODE_ALGO_SHA2_512	(3 << 0)
81 
82 #define SHA_REG_LENGTH(dd)		((dd)->pdata->length_ofs)
83 
84 #define SHA_REG_IRQSTATUS		0x118
85 #define SHA_REG_IRQSTATUS_CTX_RDY	(1 << 3)
86 #define SHA_REG_IRQSTATUS_PARTHASH_RDY (1 << 2)
87 #define SHA_REG_IRQSTATUS_INPUT_RDY	(1 << 1)
88 #define SHA_REG_IRQSTATUS_OUTPUT_RDY	(1 << 0)
89 
90 #define SHA_REG_IRQENA			0x11C
91 #define SHA_REG_IRQENA_CTX_RDY		(1 << 3)
92 #define SHA_REG_IRQENA_PARTHASH_RDY	(1 << 2)
93 #define SHA_REG_IRQENA_INPUT_RDY	(1 << 1)
94 #define SHA_REG_IRQENA_OUTPUT_RDY	(1 << 0)
95 
96 #define DEFAULT_TIMEOUT_INTERVAL	HZ
97 
98 #define DEFAULT_AUTOSUSPEND_DELAY	1000
99 
100 /* mostly device flags */
101 #define FLAGS_FINAL		1
102 #define FLAGS_DMA_ACTIVE	2
103 #define FLAGS_OUTPUT_READY	3
104 #define FLAGS_CPU		5
105 #define FLAGS_DMA_READY		6
106 #define FLAGS_AUTO_XOR		7
107 #define FLAGS_BE32_SHA1		8
108 #define FLAGS_SGS_COPIED	9
109 #define FLAGS_SGS_ALLOCED	10
110 #define FLAGS_HUGE		11
111 
112 /* context flags */
113 #define FLAGS_FINUP		16
114 
115 #define FLAGS_MODE_SHIFT	18
116 #define FLAGS_MODE_MASK		(SHA_REG_MODE_ALGO_MASK	<< FLAGS_MODE_SHIFT)
117 #define FLAGS_MODE_MD5		(SHA_REG_MODE_ALGO_MD5_128 << FLAGS_MODE_SHIFT)
118 #define FLAGS_MODE_SHA1		(SHA_REG_MODE_ALGO_SHA1_160 << FLAGS_MODE_SHIFT)
119 #define FLAGS_MODE_SHA224	(SHA_REG_MODE_ALGO_SHA2_224 << FLAGS_MODE_SHIFT)
120 #define FLAGS_MODE_SHA256	(SHA_REG_MODE_ALGO_SHA2_256 << FLAGS_MODE_SHIFT)
121 #define FLAGS_MODE_SHA384	(SHA_REG_MODE_ALGO_SHA2_384 << FLAGS_MODE_SHIFT)
122 #define FLAGS_MODE_SHA512	(SHA_REG_MODE_ALGO_SHA2_512 << FLAGS_MODE_SHIFT)
123 
124 #define FLAGS_HMAC		21
125 #define FLAGS_ERROR		22
126 
127 #define OP_UPDATE		1
128 #define OP_FINAL		2
129 
130 #define OMAP_ALIGN_MASK		(sizeof(u32)-1)
131 #define OMAP_ALIGNED		__attribute__((aligned(sizeof(u32))))
132 
133 #define BUFLEN			SHA512_BLOCK_SIZE
134 #define OMAP_SHA_DMA_THRESHOLD	256
135 
136 #define OMAP_SHA_MAX_DMA_LEN	(1024 * 2048)
137 
138 struct omap_sham_dev;
139 
140 struct omap_sham_reqctx {
141 	struct omap_sham_dev	*dd;
142 	unsigned long		flags;
143 	u8			op;
144 
145 	u8			digest[SHA512_DIGEST_SIZE] OMAP_ALIGNED;
146 	size_t			digcnt;
147 	size_t			bufcnt;
148 	size_t			buflen;
149 
150 	/* walk state */
151 	struct scatterlist	*sg;
152 	struct scatterlist	sgl[2];
153 	int			offset;	/* offset in current sg */
154 	int			sg_len;
155 	unsigned int		total;	/* total request */
156 
157 	u8			buffer[] OMAP_ALIGNED;
158 };
159 
160 struct omap_sham_hmac_ctx {
161 	struct crypto_shash	*shash;
162 	u8			ipad[SHA512_BLOCK_SIZE] OMAP_ALIGNED;
163 	u8			opad[SHA512_BLOCK_SIZE] OMAP_ALIGNED;
164 };
165 
166 struct omap_sham_ctx {
167 	unsigned long		flags;
168 
169 	/* fallback stuff */
170 	struct crypto_shash	*fallback;
171 
172 	struct omap_sham_hmac_ctx base[];
173 };
174 
175 #define OMAP_SHAM_QUEUE_LENGTH	10
176 
177 struct omap_sham_algs_info {
178 	struct ahash_engine_alg	*algs_list;
179 	unsigned int		size;
180 	unsigned int		registered;
181 };
182 
183 struct omap_sham_pdata {
184 	struct omap_sham_algs_info	*algs_info;
185 	unsigned int	algs_info_size;
186 	unsigned long	flags;
187 	int		digest_size;
188 
189 	void		(*copy_hash)(struct ahash_request *req, int out);
190 	void		(*write_ctrl)(struct omap_sham_dev *dd, size_t length,
191 				      int final, int dma);
192 	void		(*trigger)(struct omap_sham_dev *dd, size_t length);
193 	int		(*poll_irq)(struct omap_sham_dev *dd);
194 	irqreturn_t	(*intr_hdlr)(int irq, void *dev_id);
195 
196 	u32		odigest_ofs;
197 	u32		idigest_ofs;
198 	u32		din_ofs;
199 	u32		digcnt_ofs;
200 	u32		rev_ofs;
201 	u32		mask_ofs;
202 	u32		sysstatus_ofs;
203 	u32		mode_ofs;
204 	u32		length_ofs;
205 
206 	u32		major_mask;
207 	u32		major_shift;
208 	u32		minor_mask;
209 	u32		minor_shift;
210 };
211 
212 struct omap_sham_dev {
213 	struct list_head	list;
214 	unsigned long		phys_base;
215 	struct device		*dev;
216 	void __iomem		*io_base;
217 	int			irq;
218 	int			err;
219 	struct dma_chan		*dma_lch;
220 	struct tasklet_struct	done_task;
221 	u8			polling_mode;
222 	u8			xmit_buf[BUFLEN] OMAP_ALIGNED;
223 
224 	unsigned long		flags;
225 	int			fallback_sz;
226 	struct crypto_queue	queue;
227 	struct ahash_request	*req;
228 	struct crypto_engine	*engine;
229 
230 	const struct omap_sham_pdata	*pdata;
231 };
232 
233 struct omap_sham_drv {
234 	struct list_head	dev_list;
235 	spinlock_t		lock;
236 	unsigned long		flags;
237 };
238 
239 static struct omap_sham_drv sham = {
240 	.dev_list = LIST_HEAD_INIT(sham.dev_list),
241 	.lock = __SPIN_LOCK_UNLOCKED(sham.lock),
242 };
243 
244 static int omap_sham_enqueue(struct ahash_request *req, unsigned int op);
245 static void omap_sham_finish_req(struct ahash_request *req, int err);
246 
omap_sham_read(struct omap_sham_dev * dd,u32 offset)247 static inline u32 omap_sham_read(struct omap_sham_dev *dd, u32 offset)
248 {
249 	return __raw_readl(dd->io_base + offset);
250 }
251 
omap_sham_write(struct omap_sham_dev * dd,u32 offset,u32 value)252 static inline void omap_sham_write(struct omap_sham_dev *dd,
253 					u32 offset, u32 value)
254 {
255 	__raw_writel(value, dd->io_base + offset);
256 }
257 
omap_sham_write_mask(struct omap_sham_dev * dd,u32 address,u32 value,u32 mask)258 static inline void omap_sham_write_mask(struct omap_sham_dev *dd, u32 address,
259 					u32 value, u32 mask)
260 {
261 	u32 val;
262 
263 	val = omap_sham_read(dd, address);
264 	val &= ~mask;
265 	val |= value;
266 	omap_sham_write(dd, address, val);
267 }
268 
omap_sham_wait(struct omap_sham_dev * dd,u32 offset,u32 bit)269 static inline int omap_sham_wait(struct omap_sham_dev *dd, u32 offset, u32 bit)
270 {
271 	unsigned long timeout = jiffies + DEFAULT_TIMEOUT_INTERVAL;
272 
273 	while (!(omap_sham_read(dd, offset) & bit)) {
274 		if (time_is_before_jiffies(timeout))
275 			return -ETIMEDOUT;
276 	}
277 
278 	return 0;
279 }
280 
omap_sham_copy_hash_omap2(struct ahash_request * req,int out)281 static void omap_sham_copy_hash_omap2(struct ahash_request *req, int out)
282 {
283 	struct omap_sham_reqctx *ctx = ahash_request_ctx(req);
284 	struct omap_sham_dev *dd = ctx->dd;
285 	u32 *hash = (u32 *)ctx->digest;
286 	int i;
287 
288 	for (i = 0; i < dd->pdata->digest_size / sizeof(u32); i++) {
289 		if (out)
290 			hash[i] = omap_sham_read(dd, SHA_REG_IDIGEST(dd, i));
291 		else
292 			omap_sham_write(dd, SHA_REG_IDIGEST(dd, i), hash[i]);
293 	}
294 }
295 
omap_sham_copy_hash_omap4(struct ahash_request * req,int out)296 static void omap_sham_copy_hash_omap4(struct ahash_request *req, int out)
297 {
298 	struct omap_sham_reqctx *ctx = ahash_request_ctx(req);
299 	struct omap_sham_dev *dd = ctx->dd;
300 	int i;
301 
302 	if (ctx->flags & BIT(FLAGS_HMAC)) {
303 		struct crypto_ahash *tfm = crypto_ahash_reqtfm(dd->req);
304 		struct omap_sham_ctx *tctx = crypto_ahash_ctx(tfm);
305 		struct omap_sham_hmac_ctx *bctx = tctx->base;
306 		u32 *opad = (u32 *)bctx->opad;
307 
308 		for (i = 0; i < dd->pdata->digest_size / sizeof(u32); i++) {
309 			if (out)
310 				opad[i] = omap_sham_read(dd,
311 						SHA_REG_ODIGEST(dd, i));
312 			else
313 				omap_sham_write(dd, SHA_REG_ODIGEST(dd, i),
314 						opad[i]);
315 		}
316 	}
317 
318 	omap_sham_copy_hash_omap2(req, out);
319 }
320 
omap_sham_copy_ready_hash(struct ahash_request * req)321 static void omap_sham_copy_ready_hash(struct ahash_request *req)
322 {
323 	struct omap_sham_reqctx *ctx = ahash_request_ctx(req);
324 	u32 *in = (u32 *)ctx->digest;
325 	u32 *hash = (u32 *)req->result;
326 	int i, d, big_endian = 0;
327 
328 	if (!hash)
329 		return;
330 
331 	switch (ctx->flags & FLAGS_MODE_MASK) {
332 	case FLAGS_MODE_MD5:
333 		d = MD5_DIGEST_SIZE / sizeof(u32);
334 		break;
335 	case FLAGS_MODE_SHA1:
336 		/* OMAP2 SHA1 is big endian */
337 		if (test_bit(FLAGS_BE32_SHA1, &ctx->dd->flags))
338 			big_endian = 1;
339 		d = SHA1_DIGEST_SIZE / sizeof(u32);
340 		break;
341 	case FLAGS_MODE_SHA224:
342 		d = SHA224_DIGEST_SIZE / sizeof(u32);
343 		break;
344 	case FLAGS_MODE_SHA256:
345 		d = SHA256_DIGEST_SIZE / sizeof(u32);
346 		break;
347 	case FLAGS_MODE_SHA384:
348 		d = SHA384_DIGEST_SIZE / sizeof(u32);
349 		break;
350 	case FLAGS_MODE_SHA512:
351 		d = SHA512_DIGEST_SIZE / sizeof(u32);
352 		break;
353 	default:
354 		d = 0;
355 	}
356 
357 	if (big_endian)
358 		for (i = 0; i < d; i++)
359 			hash[i] = be32_to_cpup((__be32 *)in + i);
360 	else
361 		for (i = 0; i < d; i++)
362 			hash[i] = le32_to_cpup((__le32 *)in + i);
363 }
364 
omap_sham_write_ctrl_omap2(struct omap_sham_dev * dd,size_t length,int final,int dma)365 static void omap_sham_write_ctrl_omap2(struct omap_sham_dev *dd, size_t length,
366 				 int final, int dma)
367 {
368 	struct omap_sham_reqctx *ctx = ahash_request_ctx(dd->req);
369 	u32 val = length << 5, mask;
370 
371 	if (likely(ctx->digcnt))
372 		omap_sham_write(dd, SHA_REG_DIGCNT(dd), ctx->digcnt);
373 
374 	omap_sham_write_mask(dd, SHA_REG_MASK(dd),
375 		SHA_REG_MASK_IT_EN | (dma ? SHA_REG_MASK_DMA_EN : 0),
376 		SHA_REG_MASK_IT_EN | SHA_REG_MASK_DMA_EN);
377 	/*
378 	 * Setting ALGO_CONST only for the first iteration
379 	 * and CLOSE_HASH only for the last one.
380 	 */
381 	if ((ctx->flags & FLAGS_MODE_MASK) == FLAGS_MODE_SHA1)
382 		val |= SHA_REG_CTRL_ALGO;
383 	if (!ctx->digcnt)
384 		val |= SHA_REG_CTRL_ALGO_CONST;
385 	if (final)
386 		val |= SHA_REG_CTRL_CLOSE_HASH;
387 
388 	mask = SHA_REG_CTRL_ALGO_CONST | SHA_REG_CTRL_CLOSE_HASH |
389 			SHA_REG_CTRL_ALGO | SHA_REG_CTRL_LENGTH;
390 
391 	omap_sham_write_mask(dd, SHA_REG_CTRL, val, mask);
392 }
393 
omap_sham_trigger_omap2(struct omap_sham_dev * dd,size_t length)394 static void omap_sham_trigger_omap2(struct omap_sham_dev *dd, size_t length)
395 {
396 }
397 
omap_sham_poll_irq_omap2(struct omap_sham_dev * dd)398 static int omap_sham_poll_irq_omap2(struct omap_sham_dev *dd)
399 {
400 	return omap_sham_wait(dd, SHA_REG_CTRL, SHA_REG_CTRL_INPUT_READY);
401 }
402 
get_block_size(struct omap_sham_reqctx * ctx)403 static int get_block_size(struct omap_sham_reqctx *ctx)
404 {
405 	int d;
406 
407 	switch (ctx->flags & FLAGS_MODE_MASK) {
408 	case FLAGS_MODE_MD5:
409 	case FLAGS_MODE_SHA1:
410 		d = SHA1_BLOCK_SIZE;
411 		break;
412 	case FLAGS_MODE_SHA224:
413 	case FLAGS_MODE_SHA256:
414 		d = SHA256_BLOCK_SIZE;
415 		break;
416 	case FLAGS_MODE_SHA384:
417 	case FLAGS_MODE_SHA512:
418 		d = SHA512_BLOCK_SIZE;
419 		break;
420 	default:
421 		d = 0;
422 	}
423 
424 	return d;
425 }
426 
omap_sham_write_n(struct omap_sham_dev * dd,u32 offset,u32 * value,int count)427 static void omap_sham_write_n(struct omap_sham_dev *dd, u32 offset,
428 				    u32 *value, int count)
429 {
430 	for (; count--; value++, offset += 4)
431 		omap_sham_write(dd, offset, *value);
432 }
433 
omap_sham_write_ctrl_omap4(struct omap_sham_dev * dd,size_t length,int final,int dma)434 static void omap_sham_write_ctrl_omap4(struct omap_sham_dev *dd, size_t length,
435 				 int final, int dma)
436 {
437 	struct omap_sham_reqctx *ctx = ahash_request_ctx(dd->req);
438 	u32 val, mask;
439 
440 	if (likely(ctx->digcnt))
441 		omap_sham_write(dd, SHA_REG_DIGCNT(dd), ctx->digcnt);
442 
443 	/*
444 	 * Setting ALGO_CONST only for the first iteration and
445 	 * CLOSE_HASH only for the last one. Note that flags mode bits
446 	 * correspond to algorithm encoding in mode register.
447 	 */
448 	val = (ctx->flags & FLAGS_MODE_MASK) >> (FLAGS_MODE_SHIFT);
449 	if (!ctx->digcnt) {
450 		struct crypto_ahash *tfm = crypto_ahash_reqtfm(dd->req);
451 		struct omap_sham_ctx *tctx = crypto_ahash_ctx(tfm);
452 		struct omap_sham_hmac_ctx *bctx = tctx->base;
453 		int bs, nr_dr;
454 
455 		val |= SHA_REG_MODE_ALGO_CONSTANT;
456 
457 		if (ctx->flags & BIT(FLAGS_HMAC)) {
458 			bs = get_block_size(ctx);
459 			nr_dr = bs / (2 * sizeof(u32));
460 			val |= SHA_REG_MODE_HMAC_KEY_PROC;
461 			omap_sham_write_n(dd, SHA_REG_ODIGEST(dd, 0),
462 					  (u32 *)bctx->ipad, nr_dr);
463 			omap_sham_write_n(dd, SHA_REG_IDIGEST(dd, 0),
464 					  (u32 *)bctx->ipad + nr_dr, nr_dr);
465 			ctx->digcnt += bs;
466 		}
467 	}
468 
469 	if (final) {
470 		val |= SHA_REG_MODE_CLOSE_HASH;
471 
472 		if (ctx->flags & BIT(FLAGS_HMAC))
473 			val |= SHA_REG_MODE_HMAC_OUTER_HASH;
474 	}
475 
476 	mask = SHA_REG_MODE_ALGO_CONSTANT | SHA_REG_MODE_CLOSE_HASH |
477 	       SHA_REG_MODE_ALGO_MASK | SHA_REG_MODE_HMAC_OUTER_HASH |
478 	       SHA_REG_MODE_HMAC_KEY_PROC;
479 
480 	dev_dbg(dd->dev, "ctrl: %08x, flags: %08lx\n", val, ctx->flags);
481 	omap_sham_write_mask(dd, SHA_REG_MODE(dd), val, mask);
482 	omap_sham_write(dd, SHA_REG_IRQENA, SHA_REG_IRQENA_OUTPUT_RDY);
483 	omap_sham_write_mask(dd, SHA_REG_MASK(dd),
484 			     SHA_REG_MASK_IT_EN |
485 				     (dma ? SHA_REG_MASK_DMA_EN : 0),
486 			     SHA_REG_MASK_IT_EN | SHA_REG_MASK_DMA_EN);
487 }
488 
omap_sham_trigger_omap4(struct omap_sham_dev * dd,size_t length)489 static void omap_sham_trigger_omap4(struct omap_sham_dev *dd, size_t length)
490 {
491 	omap_sham_write(dd, SHA_REG_LENGTH(dd), length);
492 }
493 
omap_sham_poll_irq_omap4(struct omap_sham_dev * dd)494 static int omap_sham_poll_irq_omap4(struct omap_sham_dev *dd)
495 {
496 	return omap_sham_wait(dd, SHA_REG_IRQSTATUS,
497 			      SHA_REG_IRQSTATUS_INPUT_RDY);
498 }
499 
omap_sham_xmit_cpu(struct omap_sham_dev * dd,size_t length,int final)500 static int omap_sham_xmit_cpu(struct omap_sham_dev *dd, size_t length,
501 			      int final)
502 {
503 	struct omap_sham_reqctx *ctx = ahash_request_ctx(dd->req);
504 	int count, len32, bs32, offset = 0;
505 	const u32 *buffer;
506 	int mlen;
507 	struct sg_mapping_iter mi;
508 
509 	dev_dbg(dd->dev, "xmit_cpu: digcnt: %zd, length: %zd, final: %d\n",
510 						ctx->digcnt, length, final);
511 
512 	dd->pdata->write_ctrl(dd, length, final, 0);
513 	dd->pdata->trigger(dd, length);
514 
515 	/* should be non-zero before next lines to disable clocks later */
516 	ctx->digcnt += length;
517 	ctx->total -= length;
518 
519 	if (final)
520 		set_bit(FLAGS_FINAL, &dd->flags); /* catch last interrupt */
521 
522 	set_bit(FLAGS_CPU, &dd->flags);
523 
524 	len32 = DIV_ROUND_UP(length, sizeof(u32));
525 	bs32 = get_block_size(ctx) / sizeof(u32);
526 
527 	sg_miter_start(&mi, ctx->sg, ctx->sg_len,
528 		       SG_MITER_FROM_SG | SG_MITER_ATOMIC);
529 
530 	mlen = 0;
531 
532 	while (len32) {
533 		if (dd->pdata->poll_irq(dd))
534 			return -ETIMEDOUT;
535 
536 		for (count = 0; count < min(len32, bs32); count++, offset++) {
537 			if (!mlen) {
538 				sg_miter_next(&mi);
539 				mlen = mi.length;
540 				if (!mlen) {
541 					pr_err("sg miter failure.\n");
542 					return -EINVAL;
543 				}
544 				offset = 0;
545 				buffer = mi.addr;
546 			}
547 			omap_sham_write(dd, SHA_REG_DIN(dd, count),
548 					buffer[offset]);
549 			mlen -= 4;
550 		}
551 		len32 -= min(len32, bs32);
552 	}
553 
554 	sg_miter_stop(&mi);
555 
556 	return -EINPROGRESS;
557 }
558 
omap_sham_dma_callback(void * param)559 static void omap_sham_dma_callback(void *param)
560 {
561 	struct omap_sham_dev *dd = param;
562 
563 	set_bit(FLAGS_DMA_READY, &dd->flags);
564 	tasklet_schedule(&dd->done_task);
565 }
566 
omap_sham_xmit_dma(struct omap_sham_dev * dd,size_t length,int final)567 static int omap_sham_xmit_dma(struct omap_sham_dev *dd, size_t length,
568 			      int final)
569 {
570 	struct omap_sham_reqctx *ctx = ahash_request_ctx(dd->req);
571 	struct dma_async_tx_descriptor *tx;
572 	struct dma_slave_config cfg;
573 	int ret;
574 
575 	dev_dbg(dd->dev, "xmit_dma: digcnt: %zd, length: %zd, final: %d\n",
576 						ctx->digcnt, length, final);
577 
578 	if (!dma_map_sg(dd->dev, ctx->sg, ctx->sg_len, DMA_TO_DEVICE)) {
579 		dev_err(dd->dev, "dma_map_sg error\n");
580 		return -EINVAL;
581 	}
582 
583 	memset(&cfg, 0, sizeof(cfg));
584 
585 	cfg.dst_addr = dd->phys_base + SHA_REG_DIN(dd, 0);
586 	cfg.dst_addr_width = DMA_SLAVE_BUSWIDTH_4_BYTES;
587 	cfg.dst_maxburst = get_block_size(ctx) / DMA_SLAVE_BUSWIDTH_4_BYTES;
588 
589 	ret = dmaengine_slave_config(dd->dma_lch, &cfg);
590 	if (ret) {
591 		pr_err("omap-sham: can't configure dmaengine slave: %d\n", ret);
592 		return ret;
593 	}
594 
595 	tx = dmaengine_prep_slave_sg(dd->dma_lch, ctx->sg, ctx->sg_len,
596 				     DMA_MEM_TO_DEV,
597 				     DMA_PREP_INTERRUPT | DMA_CTRL_ACK);
598 
599 	if (!tx) {
600 		dev_err(dd->dev, "prep_slave_sg failed\n");
601 		return -EINVAL;
602 	}
603 
604 	tx->callback = omap_sham_dma_callback;
605 	tx->callback_param = dd;
606 
607 	dd->pdata->write_ctrl(dd, length, final, 1);
608 
609 	ctx->digcnt += length;
610 	ctx->total -= length;
611 
612 	if (final)
613 		set_bit(FLAGS_FINAL, &dd->flags); /* catch last interrupt */
614 
615 	set_bit(FLAGS_DMA_ACTIVE, &dd->flags);
616 
617 	dmaengine_submit(tx);
618 	dma_async_issue_pending(dd->dma_lch);
619 
620 	dd->pdata->trigger(dd, length);
621 
622 	return -EINPROGRESS;
623 }
624 
omap_sham_copy_sg_lists(struct omap_sham_reqctx * ctx,struct scatterlist * sg,int bs,int new_len)625 static int omap_sham_copy_sg_lists(struct omap_sham_reqctx *ctx,
626 				   struct scatterlist *sg, int bs, int new_len)
627 {
628 	int n = sg_nents(sg);
629 	struct scatterlist *tmp;
630 	int offset = ctx->offset;
631 
632 	ctx->total = new_len;
633 
634 	if (ctx->bufcnt)
635 		n++;
636 
637 	ctx->sg = kmalloc_array(n, sizeof(*sg), GFP_KERNEL);
638 	if (!ctx->sg)
639 		return -ENOMEM;
640 
641 	sg_init_table(ctx->sg, n);
642 
643 	tmp = ctx->sg;
644 
645 	ctx->sg_len = 0;
646 
647 	if (ctx->bufcnt) {
648 		sg_set_buf(tmp, ctx->dd->xmit_buf, ctx->bufcnt);
649 		tmp = sg_next(tmp);
650 		ctx->sg_len++;
651 		new_len -= ctx->bufcnt;
652 	}
653 
654 	while (sg && new_len) {
655 		int len = sg->length - offset;
656 
657 		if (len <= 0) {
658 			offset -= sg->length;
659 			sg = sg_next(sg);
660 			continue;
661 		}
662 
663 		if (new_len < len)
664 			len = new_len;
665 
666 		if (len > 0) {
667 			new_len -= len;
668 			sg_set_page(tmp, sg_page(sg), len, sg->offset + offset);
669 			offset = 0;
670 			ctx->offset = 0;
671 			ctx->sg_len++;
672 			if (new_len <= 0)
673 				break;
674 			tmp = sg_next(tmp);
675 		}
676 
677 		sg = sg_next(sg);
678 	}
679 
680 	if (tmp)
681 		sg_mark_end(tmp);
682 
683 	set_bit(FLAGS_SGS_ALLOCED, &ctx->dd->flags);
684 
685 	ctx->offset += new_len - ctx->bufcnt;
686 	ctx->bufcnt = 0;
687 
688 	return 0;
689 }
690 
omap_sham_copy_sgs(struct omap_sham_reqctx * ctx,struct scatterlist * sg,int bs,unsigned int new_len)691 static int omap_sham_copy_sgs(struct omap_sham_reqctx *ctx,
692 			      struct scatterlist *sg, int bs,
693 			      unsigned int new_len)
694 {
695 	int pages;
696 	void *buf;
697 
698 	pages = get_order(new_len);
699 
700 	buf = (void *)__get_free_pages(GFP_ATOMIC, pages);
701 	if (!buf) {
702 		pr_err("Couldn't allocate pages for unaligned cases.\n");
703 		return -ENOMEM;
704 	}
705 
706 	if (ctx->bufcnt)
707 		memcpy(buf, ctx->dd->xmit_buf, ctx->bufcnt);
708 
709 	scatterwalk_map_and_copy(buf + ctx->bufcnt, sg, ctx->offset,
710 				 min(new_len, ctx->total) - ctx->bufcnt, 0);
711 	sg_init_table(ctx->sgl, 1);
712 	sg_set_buf(ctx->sgl, buf, new_len);
713 	ctx->sg = ctx->sgl;
714 	set_bit(FLAGS_SGS_COPIED, &ctx->dd->flags);
715 	ctx->sg_len = 1;
716 	ctx->offset += new_len - ctx->bufcnt;
717 	ctx->bufcnt = 0;
718 	ctx->total = new_len;
719 
720 	return 0;
721 }
722 
omap_sham_align_sgs(struct scatterlist * sg,int nbytes,int bs,bool final,struct omap_sham_reqctx * rctx)723 static int omap_sham_align_sgs(struct scatterlist *sg,
724 			       int nbytes, int bs, bool final,
725 			       struct omap_sham_reqctx *rctx)
726 {
727 	int n = 0;
728 	bool aligned = true;
729 	bool list_ok = true;
730 	struct scatterlist *sg_tmp = sg;
731 	int new_len;
732 	int offset = rctx->offset;
733 	int bufcnt = rctx->bufcnt;
734 
735 	if (!sg || !sg->length || !nbytes) {
736 		if (bufcnt) {
737 			bufcnt = DIV_ROUND_UP(bufcnt, bs) * bs;
738 			sg_init_table(rctx->sgl, 1);
739 			sg_set_buf(rctx->sgl, rctx->dd->xmit_buf, bufcnt);
740 			rctx->sg = rctx->sgl;
741 			rctx->sg_len = 1;
742 		}
743 
744 		return 0;
745 	}
746 
747 	new_len = nbytes;
748 
749 	if (offset)
750 		list_ok = false;
751 
752 	if (final)
753 		new_len = DIV_ROUND_UP(new_len, bs) * bs;
754 	else
755 		new_len = (new_len - 1) / bs * bs;
756 
757 	if (!new_len)
758 		return 0;
759 
760 	if (nbytes != new_len)
761 		list_ok = false;
762 
763 	while (nbytes > 0 && sg_tmp) {
764 		n++;
765 
766 		if (bufcnt) {
767 			if (!IS_ALIGNED(bufcnt, bs)) {
768 				aligned = false;
769 				break;
770 			}
771 			nbytes -= bufcnt;
772 			bufcnt = 0;
773 			if (!nbytes)
774 				list_ok = false;
775 
776 			continue;
777 		}
778 
779 #ifdef CONFIG_ZONE_DMA
780 		if (page_zonenum(sg_page(sg_tmp)) != ZONE_DMA) {
781 			aligned = false;
782 			break;
783 		}
784 #endif
785 
786 		if (offset < sg_tmp->length) {
787 			if (!IS_ALIGNED(offset + sg_tmp->offset, 4)) {
788 				aligned = false;
789 				break;
790 			}
791 
792 			if (!IS_ALIGNED(sg_tmp->length - offset, bs)) {
793 				aligned = false;
794 				break;
795 			}
796 		}
797 
798 		if (offset) {
799 			offset -= sg_tmp->length;
800 			if (offset < 0) {
801 				nbytes += offset;
802 				offset = 0;
803 			}
804 		} else {
805 			nbytes -= sg_tmp->length;
806 		}
807 
808 		sg_tmp = sg_next(sg_tmp);
809 
810 		if (nbytes < 0) {
811 			list_ok = false;
812 			break;
813 		}
814 	}
815 
816 	if (new_len > OMAP_SHA_MAX_DMA_LEN) {
817 		new_len = OMAP_SHA_MAX_DMA_LEN;
818 		aligned = false;
819 	}
820 
821 	if (!aligned)
822 		return omap_sham_copy_sgs(rctx, sg, bs, new_len);
823 	else if (!list_ok)
824 		return omap_sham_copy_sg_lists(rctx, sg, bs, new_len);
825 
826 	rctx->total = new_len;
827 	rctx->offset += new_len;
828 	rctx->sg_len = n;
829 	if (rctx->bufcnt) {
830 		sg_init_table(rctx->sgl, 2);
831 		sg_set_buf(rctx->sgl, rctx->dd->xmit_buf, rctx->bufcnt);
832 		sg_chain(rctx->sgl, 2, sg);
833 		rctx->sg = rctx->sgl;
834 	} else {
835 		rctx->sg = sg;
836 	}
837 
838 	return 0;
839 }
840 
omap_sham_prepare_request(struct crypto_engine * engine,void * areq)841 static int omap_sham_prepare_request(struct crypto_engine *engine, void *areq)
842 {
843 	struct ahash_request *req = container_of(areq, struct ahash_request,
844 						 base);
845 	struct omap_sham_reqctx *rctx = ahash_request_ctx(req);
846 	int bs;
847 	int ret;
848 	unsigned int nbytes;
849 	bool final = rctx->flags & BIT(FLAGS_FINUP);
850 	bool update = rctx->op == OP_UPDATE;
851 	int hash_later;
852 
853 	bs = get_block_size(rctx);
854 
855 	nbytes = rctx->bufcnt;
856 
857 	if (update)
858 		nbytes += req->nbytes - rctx->offset;
859 
860 	dev_dbg(rctx->dd->dev,
861 		"%s: nbytes=%d, bs=%d, total=%d, offset=%d, bufcnt=%zd\n",
862 		__func__, nbytes, bs, rctx->total, rctx->offset,
863 		rctx->bufcnt);
864 
865 	if (!nbytes)
866 		return 0;
867 
868 	rctx->total = nbytes;
869 
870 	if (update && req->nbytes && (!IS_ALIGNED(rctx->bufcnt, bs))) {
871 		int len = bs - rctx->bufcnt % bs;
872 
873 		if (len > req->nbytes)
874 			len = req->nbytes;
875 		scatterwalk_map_and_copy(rctx->buffer + rctx->bufcnt, req->src,
876 					 0, len, 0);
877 		rctx->bufcnt += len;
878 		rctx->offset = len;
879 	}
880 
881 	if (rctx->bufcnt)
882 		memcpy(rctx->dd->xmit_buf, rctx->buffer, rctx->bufcnt);
883 
884 	ret = omap_sham_align_sgs(req->src, nbytes, bs, final, rctx);
885 	if (ret)
886 		return ret;
887 
888 	hash_later = nbytes - rctx->total;
889 	if (hash_later < 0)
890 		hash_later = 0;
891 
892 	if (hash_later && hash_later <= rctx->buflen) {
893 		scatterwalk_map_and_copy(rctx->buffer,
894 					 req->src,
895 					 req->nbytes - hash_later,
896 					 hash_later, 0);
897 
898 		rctx->bufcnt = hash_later;
899 	} else {
900 		rctx->bufcnt = 0;
901 	}
902 
903 	if (hash_later > rctx->buflen)
904 		set_bit(FLAGS_HUGE, &rctx->dd->flags);
905 
906 	rctx->total = min(nbytes, rctx->total);
907 
908 	return 0;
909 }
910 
omap_sham_update_dma_stop(struct omap_sham_dev * dd)911 static int omap_sham_update_dma_stop(struct omap_sham_dev *dd)
912 {
913 	struct omap_sham_reqctx *ctx = ahash_request_ctx(dd->req);
914 
915 	dma_unmap_sg(dd->dev, ctx->sg, ctx->sg_len, DMA_TO_DEVICE);
916 
917 	clear_bit(FLAGS_DMA_ACTIVE, &dd->flags);
918 
919 	return 0;
920 }
921 
omap_sham_find_dev(struct omap_sham_reqctx * ctx)922 static struct omap_sham_dev *omap_sham_find_dev(struct omap_sham_reqctx *ctx)
923 {
924 	struct omap_sham_dev *dd;
925 
926 	if (ctx->dd)
927 		return ctx->dd;
928 
929 	spin_lock_bh(&sham.lock);
930 	dd = list_first_entry(&sham.dev_list, struct omap_sham_dev, list);
931 	list_move_tail(&dd->list, &sham.dev_list);
932 	ctx->dd = dd;
933 	spin_unlock_bh(&sham.lock);
934 
935 	return dd;
936 }
937 
omap_sham_init(struct ahash_request * req)938 static int omap_sham_init(struct ahash_request *req)
939 {
940 	struct crypto_ahash *tfm = crypto_ahash_reqtfm(req);
941 	struct omap_sham_ctx *tctx = crypto_ahash_ctx(tfm);
942 	struct omap_sham_reqctx *ctx = ahash_request_ctx(req);
943 	struct omap_sham_dev *dd;
944 	int bs = 0;
945 
946 	ctx->dd = NULL;
947 
948 	dd = omap_sham_find_dev(ctx);
949 	if (!dd)
950 		return -ENODEV;
951 
952 	ctx->flags = 0;
953 
954 	dev_dbg(dd->dev, "init: digest size: %d\n",
955 		crypto_ahash_digestsize(tfm));
956 
957 	switch (crypto_ahash_digestsize(tfm)) {
958 	case MD5_DIGEST_SIZE:
959 		ctx->flags |= FLAGS_MODE_MD5;
960 		bs = SHA1_BLOCK_SIZE;
961 		break;
962 	case SHA1_DIGEST_SIZE:
963 		ctx->flags |= FLAGS_MODE_SHA1;
964 		bs = SHA1_BLOCK_SIZE;
965 		break;
966 	case SHA224_DIGEST_SIZE:
967 		ctx->flags |= FLAGS_MODE_SHA224;
968 		bs = SHA224_BLOCK_SIZE;
969 		break;
970 	case SHA256_DIGEST_SIZE:
971 		ctx->flags |= FLAGS_MODE_SHA256;
972 		bs = SHA256_BLOCK_SIZE;
973 		break;
974 	case SHA384_DIGEST_SIZE:
975 		ctx->flags |= FLAGS_MODE_SHA384;
976 		bs = SHA384_BLOCK_SIZE;
977 		break;
978 	case SHA512_DIGEST_SIZE:
979 		ctx->flags |= FLAGS_MODE_SHA512;
980 		bs = SHA512_BLOCK_SIZE;
981 		break;
982 	}
983 
984 	ctx->bufcnt = 0;
985 	ctx->digcnt = 0;
986 	ctx->total = 0;
987 	ctx->offset = 0;
988 	ctx->buflen = BUFLEN;
989 
990 	if (tctx->flags & BIT(FLAGS_HMAC)) {
991 		if (!test_bit(FLAGS_AUTO_XOR, &dd->flags)) {
992 			struct omap_sham_hmac_ctx *bctx = tctx->base;
993 
994 			memcpy(ctx->buffer, bctx->ipad, bs);
995 			ctx->bufcnt = bs;
996 		}
997 
998 		ctx->flags |= BIT(FLAGS_HMAC);
999 	}
1000 
1001 	return 0;
1002 
1003 }
1004 
omap_sham_update_req(struct omap_sham_dev * dd)1005 static int omap_sham_update_req(struct omap_sham_dev *dd)
1006 {
1007 	struct ahash_request *req = dd->req;
1008 	struct omap_sham_reqctx *ctx = ahash_request_ctx(req);
1009 	int err;
1010 	bool final = (ctx->flags & BIT(FLAGS_FINUP)) &&
1011 		!(dd->flags & BIT(FLAGS_HUGE));
1012 
1013 	dev_dbg(dd->dev, "update_req: total: %u, digcnt: %zd, final: %d",
1014 		ctx->total, ctx->digcnt, final);
1015 
1016 	if (ctx->total < get_block_size(ctx) ||
1017 	    ctx->total < dd->fallback_sz)
1018 		ctx->flags |= BIT(FLAGS_CPU);
1019 
1020 	if (ctx->flags & BIT(FLAGS_CPU))
1021 		err = omap_sham_xmit_cpu(dd, ctx->total, final);
1022 	else
1023 		err = omap_sham_xmit_dma(dd, ctx->total, final);
1024 
1025 	/* wait for dma completion before can take more data */
1026 	dev_dbg(dd->dev, "update: err: %d, digcnt: %zd\n", err, ctx->digcnt);
1027 
1028 	return err;
1029 }
1030 
omap_sham_final_req(struct omap_sham_dev * dd)1031 static int omap_sham_final_req(struct omap_sham_dev *dd)
1032 {
1033 	struct ahash_request *req = dd->req;
1034 	struct omap_sham_reqctx *ctx = ahash_request_ctx(req);
1035 	int err = 0, use_dma = 1;
1036 
1037 	if (dd->flags & BIT(FLAGS_HUGE))
1038 		return 0;
1039 
1040 	if ((ctx->total <= get_block_size(ctx)) || dd->polling_mode)
1041 		/*
1042 		 * faster to handle last block with cpu or
1043 		 * use cpu when dma is not present.
1044 		 */
1045 		use_dma = 0;
1046 
1047 	if (use_dma)
1048 		err = omap_sham_xmit_dma(dd, ctx->total, 1);
1049 	else
1050 		err = omap_sham_xmit_cpu(dd, ctx->total, 1);
1051 
1052 	ctx->bufcnt = 0;
1053 
1054 	dev_dbg(dd->dev, "final_req: err: %d\n", err);
1055 
1056 	return err;
1057 }
1058 
omap_sham_hash_one_req(struct crypto_engine * engine,void * areq)1059 static int omap_sham_hash_one_req(struct crypto_engine *engine, void *areq)
1060 {
1061 	struct ahash_request *req = container_of(areq, struct ahash_request,
1062 						 base);
1063 	struct omap_sham_reqctx *ctx = ahash_request_ctx(req);
1064 	struct omap_sham_dev *dd = ctx->dd;
1065 	int err;
1066 	bool final = (ctx->flags & BIT(FLAGS_FINUP)) &&
1067 			!(dd->flags & BIT(FLAGS_HUGE));
1068 
1069 	dev_dbg(dd->dev, "hash-one: op: %u, total: %u, digcnt: %zd, final: %d",
1070 		ctx->op, ctx->total, ctx->digcnt, final);
1071 
1072 	err = omap_sham_prepare_request(engine, areq);
1073 	if (err)
1074 		return err;
1075 
1076 	err = pm_runtime_resume_and_get(dd->dev);
1077 	if (err < 0) {
1078 		dev_err(dd->dev, "failed to get sync: %d\n", err);
1079 		return err;
1080 	}
1081 
1082 	dd->err = 0;
1083 	dd->req = req;
1084 
1085 	if (ctx->digcnt)
1086 		dd->pdata->copy_hash(req, 0);
1087 
1088 	if (ctx->op == OP_UPDATE)
1089 		err = omap_sham_update_req(dd);
1090 	else if (ctx->op == OP_FINAL)
1091 		err = omap_sham_final_req(dd);
1092 
1093 	if (err != -EINPROGRESS)
1094 		omap_sham_finish_req(req, err);
1095 
1096 	return 0;
1097 }
1098 
omap_sham_finish_hmac(struct ahash_request * req)1099 static int omap_sham_finish_hmac(struct ahash_request *req)
1100 {
1101 	struct omap_sham_ctx *tctx = crypto_tfm_ctx(req->base.tfm);
1102 	struct omap_sham_hmac_ctx *bctx = tctx->base;
1103 	int bs = crypto_shash_blocksize(bctx->shash);
1104 	int ds = crypto_shash_digestsize(bctx->shash);
1105 	SHASH_DESC_ON_STACK(shash, bctx->shash);
1106 
1107 	shash->tfm = bctx->shash;
1108 
1109 	return crypto_shash_init(shash) ?:
1110 	       crypto_shash_update(shash, bctx->opad, bs) ?:
1111 	       crypto_shash_finup(shash, req->result, ds, req->result);
1112 }
1113 
omap_sham_finish(struct ahash_request * req)1114 static int omap_sham_finish(struct ahash_request *req)
1115 {
1116 	struct omap_sham_reqctx *ctx = ahash_request_ctx(req);
1117 	struct omap_sham_dev *dd = ctx->dd;
1118 	int err = 0;
1119 
1120 	if (ctx->digcnt) {
1121 		omap_sham_copy_ready_hash(req);
1122 		if ((ctx->flags & BIT(FLAGS_HMAC)) &&
1123 				!test_bit(FLAGS_AUTO_XOR, &dd->flags))
1124 			err = omap_sham_finish_hmac(req);
1125 	}
1126 
1127 	dev_dbg(dd->dev, "digcnt: %zd, bufcnt: %zd\n", ctx->digcnt, ctx->bufcnt);
1128 
1129 	return err;
1130 }
1131 
omap_sham_finish_req(struct ahash_request * req,int err)1132 static void omap_sham_finish_req(struct ahash_request *req, int err)
1133 {
1134 	struct omap_sham_reqctx *ctx = ahash_request_ctx(req);
1135 	struct omap_sham_dev *dd = ctx->dd;
1136 
1137 	if (test_bit(FLAGS_SGS_COPIED, &dd->flags))
1138 		free_pages((unsigned long)sg_virt(ctx->sg),
1139 			   get_order(ctx->sg->length));
1140 
1141 	if (test_bit(FLAGS_SGS_ALLOCED, &dd->flags))
1142 		kfree(ctx->sg);
1143 
1144 	ctx->sg = NULL;
1145 
1146 	dd->flags &= ~(BIT(FLAGS_SGS_ALLOCED) | BIT(FLAGS_SGS_COPIED) |
1147 		       BIT(FLAGS_CPU) | BIT(FLAGS_DMA_READY) |
1148 		       BIT(FLAGS_OUTPUT_READY));
1149 
1150 	if (!err)
1151 		dd->pdata->copy_hash(req, 1);
1152 
1153 	if (dd->flags & BIT(FLAGS_HUGE)) {
1154 		/* Re-enqueue the request */
1155 		omap_sham_enqueue(req, ctx->op);
1156 		return;
1157 	}
1158 
1159 	if (!err) {
1160 		if (test_bit(FLAGS_FINAL, &dd->flags))
1161 			err = omap_sham_finish(req);
1162 	} else {
1163 		ctx->flags |= BIT(FLAGS_ERROR);
1164 	}
1165 
1166 	/* atomic operation is not needed here */
1167 	dd->flags &= ~(BIT(FLAGS_FINAL) | BIT(FLAGS_CPU) |
1168 			BIT(FLAGS_DMA_READY) | BIT(FLAGS_OUTPUT_READY));
1169 
1170 	pm_runtime_mark_last_busy(dd->dev);
1171 	pm_runtime_put_autosuspend(dd->dev);
1172 
1173 	ctx->offset = 0;
1174 
1175 	crypto_finalize_hash_request(dd->engine, req, err);
1176 }
1177 
omap_sham_handle_queue(struct omap_sham_dev * dd,struct ahash_request * req)1178 static int omap_sham_handle_queue(struct omap_sham_dev *dd,
1179 				  struct ahash_request *req)
1180 {
1181 	return crypto_transfer_hash_request_to_engine(dd->engine, req);
1182 }
1183 
omap_sham_enqueue(struct ahash_request * req,unsigned int op)1184 static int omap_sham_enqueue(struct ahash_request *req, unsigned int op)
1185 {
1186 	struct omap_sham_reqctx *ctx = ahash_request_ctx(req);
1187 	struct omap_sham_dev *dd = ctx->dd;
1188 
1189 	ctx->op = op;
1190 
1191 	return omap_sham_handle_queue(dd, req);
1192 }
1193 
omap_sham_update(struct ahash_request * req)1194 static int omap_sham_update(struct ahash_request *req)
1195 {
1196 	struct omap_sham_reqctx *ctx = ahash_request_ctx(req);
1197 	struct omap_sham_dev *dd = omap_sham_find_dev(ctx);
1198 
1199 	if (!req->nbytes)
1200 		return 0;
1201 
1202 	if (ctx->bufcnt + req->nbytes <= ctx->buflen) {
1203 		scatterwalk_map_and_copy(ctx->buffer + ctx->bufcnt, req->src,
1204 					 0, req->nbytes, 0);
1205 		ctx->bufcnt += req->nbytes;
1206 		return 0;
1207 	}
1208 
1209 	if (dd->polling_mode)
1210 		ctx->flags |= BIT(FLAGS_CPU);
1211 
1212 	return omap_sham_enqueue(req, OP_UPDATE);
1213 }
1214 
omap_sham_final_shash(struct ahash_request * req)1215 static int omap_sham_final_shash(struct ahash_request *req)
1216 {
1217 	struct omap_sham_ctx *tctx = crypto_tfm_ctx(req->base.tfm);
1218 	struct omap_sham_reqctx *ctx = ahash_request_ctx(req);
1219 	int offset = 0;
1220 
1221 	/*
1222 	 * If we are running HMAC on limited hardware support, skip
1223 	 * the ipad in the beginning of the buffer if we are going for
1224 	 * software fallback algorithm.
1225 	 */
1226 	if (test_bit(FLAGS_HMAC, &ctx->flags) &&
1227 	    !test_bit(FLAGS_AUTO_XOR, &ctx->dd->flags))
1228 		offset = get_block_size(ctx);
1229 
1230 	return crypto_shash_tfm_digest(tctx->fallback, ctx->buffer + offset,
1231 				       ctx->bufcnt - offset, req->result);
1232 }
1233 
omap_sham_final(struct ahash_request * req)1234 static int omap_sham_final(struct ahash_request *req)
1235 {
1236 	struct omap_sham_reqctx *ctx = ahash_request_ctx(req);
1237 
1238 	ctx->flags |= BIT(FLAGS_FINUP);
1239 
1240 	if (ctx->flags & BIT(FLAGS_ERROR))
1241 		return 0; /* uncompleted hash is not needed */
1242 
1243 	/*
1244 	 * OMAP HW accel works only with buffers >= 9.
1245 	 * HMAC is always >= 9 because ipad == block size.
1246 	 * If buffersize is less than fallback_sz, we use fallback
1247 	 * SW encoding, as using DMA + HW in this case doesn't provide
1248 	 * any benefit.
1249 	 */
1250 	if (!ctx->digcnt && ctx->bufcnt < ctx->dd->fallback_sz)
1251 		return omap_sham_final_shash(req);
1252 	else if (ctx->bufcnt)
1253 		return omap_sham_enqueue(req, OP_FINAL);
1254 
1255 	/* copy ready hash (+ finalize hmac) */
1256 	return omap_sham_finish(req);
1257 }
1258 
omap_sham_finup(struct ahash_request * req)1259 static int omap_sham_finup(struct ahash_request *req)
1260 {
1261 	struct omap_sham_reqctx *ctx = ahash_request_ctx(req);
1262 	int err1, err2;
1263 
1264 	ctx->flags |= BIT(FLAGS_FINUP);
1265 
1266 	err1 = omap_sham_update(req);
1267 	if (err1 == -EINPROGRESS || err1 == -EBUSY)
1268 		return err1;
1269 	/*
1270 	 * final() has to be always called to cleanup resources
1271 	 * even if udpate() failed, except EINPROGRESS
1272 	 */
1273 	err2 = omap_sham_final(req);
1274 
1275 	return err1 ?: err2;
1276 }
1277 
omap_sham_digest(struct ahash_request * req)1278 static int omap_sham_digest(struct ahash_request *req)
1279 {
1280 	return omap_sham_init(req) ?: omap_sham_finup(req);
1281 }
1282 
omap_sham_setkey(struct crypto_ahash * tfm,const u8 * key,unsigned int keylen)1283 static int omap_sham_setkey(struct crypto_ahash *tfm, const u8 *key,
1284 		      unsigned int keylen)
1285 {
1286 	struct omap_sham_ctx *tctx = crypto_ahash_ctx(tfm);
1287 	struct omap_sham_hmac_ctx *bctx = tctx->base;
1288 	int bs = crypto_shash_blocksize(bctx->shash);
1289 	int ds = crypto_shash_digestsize(bctx->shash);
1290 	int err, i;
1291 
1292 	err = crypto_shash_setkey(tctx->fallback, key, keylen);
1293 	if (err)
1294 		return err;
1295 
1296 	if (keylen > bs) {
1297 		err = crypto_shash_tfm_digest(bctx->shash, key, keylen,
1298 					      bctx->ipad);
1299 		if (err)
1300 			return err;
1301 		keylen = ds;
1302 	} else {
1303 		memcpy(bctx->ipad, key, keylen);
1304 	}
1305 
1306 	memset(bctx->ipad + keylen, 0, bs - keylen);
1307 
1308 	if (!test_bit(FLAGS_AUTO_XOR, &sham.flags)) {
1309 		memcpy(bctx->opad, bctx->ipad, bs);
1310 
1311 		for (i = 0; i < bs; i++) {
1312 			bctx->ipad[i] ^= HMAC_IPAD_VALUE;
1313 			bctx->opad[i] ^= HMAC_OPAD_VALUE;
1314 		}
1315 	}
1316 
1317 	return err;
1318 }
1319 
omap_sham_cra_init_alg(struct crypto_tfm * tfm,const char * alg_base)1320 static int omap_sham_cra_init_alg(struct crypto_tfm *tfm, const char *alg_base)
1321 {
1322 	struct omap_sham_ctx *tctx = crypto_tfm_ctx(tfm);
1323 	const char *alg_name = crypto_tfm_alg_name(tfm);
1324 
1325 	/* Allocate a fallback and abort if it failed. */
1326 	tctx->fallback = crypto_alloc_shash(alg_name, 0,
1327 					    CRYPTO_ALG_NEED_FALLBACK);
1328 	if (IS_ERR(tctx->fallback)) {
1329 		pr_err("omap-sham: fallback driver '%s' "
1330 				"could not be loaded.\n", alg_name);
1331 		return PTR_ERR(tctx->fallback);
1332 	}
1333 
1334 	crypto_ahash_set_reqsize(__crypto_ahash_cast(tfm),
1335 				 sizeof(struct omap_sham_reqctx) + BUFLEN);
1336 
1337 	if (alg_base) {
1338 		struct omap_sham_hmac_ctx *bctx = tctx->base;
1339 		tctx->flags |= BIT(FLAGS_HMAC);
1340 		bctx->shash = crypto_alloc_shash(alg_base, 0,
1341 						CRYPTO_ALG_NEED_FALLBACK);
1342 		if (IS_ERR(bctx->shash)) {
1343 			pr_err("omap-sham: base driver '%s' "
1344 					"could not be loaded.\n", alg_base);
1345 			crypto_free_shash(tctx->fallback);
1346 			return PTR_ERR(bctx->shash);
1347 		}
1348 
1349 	}
1350 
1351 	return 0;
1352 }
1353 
omap_sham_cra_init(struct crypto_tfm * tfm)1354 static int omap_sham_cra_init(struct crypto_tfm *tfm)
1355 {
1356 	return omap_sham_cra_init_alg(tfm, NULL);
1357 }
1358 
omap_sham_cra_sha1_init(struct crypto_tfm * tfm)1359 static int omap_sham_cra_sha1_init(struct crypto_tfm *tfm)
1360 {
1361 	return omap_sham_cra_init_alg(tfm, "sha1");
1362 }
1363 
omap_sham_cra_sha224_init(struct crypto_tfm * tfm)1364 static int omap_sham_cra_sha224_init(struct crypto_tfm *tfm)
1365 {
1366 	return omap_sham_cra_init_alg(tfm, "sha224");
1367 }
1368 
omap_sham_cra_sha256_init(struct crypto_tfm * tfm)1369 static int omap_sham_cra_sha256_init(struct crypto_tfm *tfm)
1370 {
1371 	return omap_sham_cra_init_alg(tfm, "sha256");
1372 }
1373 
omap_sham_cra_md5_init(struct crypto_tfm * tfm)1374 static int omap_sham_cra_md5_init(struct crypto_tfm *tfm)
1375 {
1376 	return omap_sham_cra_init_alg(tfm, "md5");
1377 }
1378 
omap_sham_cra_sha384_init(struct crypto_tfm * tfm)1379 static int omap_sham_cra_sha384_init(struct crypto_tfm *tfm)
1380 {
1381 	return omap_sham_cra_init_alg(tfm, "sha384");
1382 }
1383 
omap_sham_cra_sha512_init(struct crypto_tfm * tfm)1384 static int omap_sham_cra_sha512_init(struct crypto_tfm *tfm)
1385 {
1386 	return omap_sham_cra_init_alg(tfm, "sha512");
1387 }
1388 
omap_sham_cra_exit(struct crypto_tfm * tfm)1389 static void omap_sham_cra_exit(struct crypto_tfm *tfm)
1390 {
1391 	struct omap_sham_ctx *tctx = crypto_tfm_ctx(tfm);
1392 
1393 	crypto_free_shash(tctx->fallback);
1394 	tctx->fallback = NULL;
1395 
1396 	if (tctx->flags & BIT(FLAGS_HMAC)) {
1397 		struct omap_sham_hmac_ctx *bctx = tctx->base;
1398 		crypto_free_shash(bctx->shash);
1399 	}
1400 }
1401 
omap_sham_export(struct ahash_request * req,void * out)1402 static int omap_sham_export(struct ahash_request *req, void *out)
1403 {
1404 	struct omap_sham_reqctx *rctx = ahash_request_ctx(req);
1405 
1406 	memcpy(out, rctx, sizeof(*rctx) + rctx->bufcnt);
1407 
1408 	return 0;
1409 }
1410 
omap_sham_import(struct ahash_request * req,const void * in)1411 static int omap_sham_import(struct ahash_request *req, const void *in)
1412 {
1413 	struct omap_sham_reqctx *rctx = ahash_request_ctx(req);
1414 	const struct omap_sham_reqctx *ctx_in = in;
1415 
1416 	memcpy(rctx, in, sizeof(*rctx) + ctx_in->bufcnt);
1417 
1418 	return 0;
1419 }
1420 
1421 static struct ahash_engine_alg algs_sha1_md5[] = {
1422 {
1423 	.base.init		= omap_sham_init,
1424 	.base.update		= omap_sham_update,
1425 	.base.final		= omap_sham_final,
1426 	.base.finup		= omap_sham_finup,
1427 	.base.digest		= omap_sham_digest,
1428 	.base.halg.digestsize	= SHA1_DIGEST_SIZE,
1429 	.base.halg.base	= {
1430 		.cra_name		= "sha1",
1431 		.cra_driver_name	= "omap-sha1",
1432 		.cra_priority		= 400,
1433 		.cra_flags		= CRYPTO_ALG_KERN_DRIVER_ONLY |
1434 						CRYPTO_ALG_ASYNC |
1435 						CRYPTO_ALG_NEED_FALLBACK,
1436 		.cra_blocksize		= SHA1_BLOCK_SIZE,
1437 		.cra_ctxsize		= sizeof(struct omap_sham_ctx),
1438 		.cra_alignmask		= OMAP_ALIGN_MASK,
1439 		.cra_module		= THIS_MODULE,
1440 		.cra_init		= omap_sham_cra_init,
1441 		.cra_exit		= omap_sham_cra_exit,
1442 	},
1443 	.op.do_one_request = omap_sham_hash_one_req,
1444 },
1445 {
1446 	.base.init		= omap_sham_init,
1447 	.base.update		= omap_sham_update,
1448 	.base.final		= omap_sham_final,
1449 	.base.finup		= omap_sham_finup,
1450 	.base.digest		= omap_sham_digest,
1451 	.base.halg.digestsize	= MD5_DIGEST_SIZE,
1452 	.base.halg.base	= {
1453 		.cra_name		= "md5",
1454 		.cra_driver_name	= "omap-md5",
1455 		.cra_priority		= 400,
1456 		.cra_flags		= CRYPTO_ALG_KERN_DRIVER_ONLY |
1457 						CRYPTO_ALG_ASYNC |
1458 						CRYPTO_ALG_NEED_FALLBACK,
1459 		.cra_blocksize		= SHA1_BLOCK_SIZE,
1460 		.cra_ctxsize		= sizeof(struct omap_sham_ctx),
1461 		.cra_alignmask		= OMAP_ALIGN_MASK,
1462 		.cra_module		= THIS_MODULE,
1463 		.cra_init		= omap_sham_cra_init,
1464 		.cra_exit		= omap_sham_cra_exit,
1465 	},
1466 	.op.do_one_request = omap_sham_hash_one_req,
1467 },
1468 {
1469 	.base.init		= omap_sham_init,
1470 	.base.update		= omap_sham_update,
1471 	.base.final		= omap_sham_final,
1472 	.base.finup		= omap_sham_finup,
1473 	.base.digest		= omap_sham_digest,
1474 	.base.setkey		= omap_sham_setkey,
1475 	.base.halg.digestsize	= SHA1_DIGEST_SIZE,
1476 	.base.halg.base	= {
1477 		.cra_name		= "hmac(sha1)",
1478 		.cra_driver_name	= "omap-hmac-sha1",
1479 		.cra_priority		= 400,
1480 		.cra_flags		= CRYPTO_ALG_KERN_DRIVER_ONLY |
1481 						CRYPTO_ALG_ASYNC |
1482 						CRYPTO_ALG_NEED_FALLBACK,
1483 		.cra_blocksize		= SHA1_BLOCK_SIZE,
1484 		.cra_ctxsize		= sizeof(struct omap_sham_ctx) +
1485 					sizeof(struct omap_sham_hmac_ctx),
1486 		.cra_alignmask		= OMAP_ALIGN_MASK,
1487 		.cra_module		= THIS_MODULE,
1488 		.cra_init		= omap_sham_cra_sha1_init,
1489 		.cra_exit		= omap_sham_cra_exit,
1490 	},
1491 	.op.do_one_request = omap_sham_hash_one_req,
1492 },
1493 {
1494 	.base.init		= omap_sham_init,
1495 	.base.update		= omap_sham_update,
1496 	.base.final		= omap_sham_final,
1497 	.base.finup		= omap_sham_finup,
1498 	.base.digest		= omap_sham_digest,
1499 	.base.setkey		= omap_sham_setkey,
1500 	.base.halg.digestsize	= MD5_DIGEST_SIZE,
1501 	.base.halg.base	= {
1502 		.cra_name		= "hmac(md5)",
1503 		.cra_driver_name	= "omap-hmac-md5",
1504 		.cra_priority		= 400,
1505 		.cra_flags		= CRYPTO_ALG_KERN_DRIVER_ONLY |
1506 						CRYPTO_ALG_ASYNC |
1507 						CRYPTO_ALG_NEED_FALLBACK,
1508 		.cra_blocksize		= SHA1_BLOCK_SIZE,
1509 		.cra_ctxsize		= sizeof(struct omap_sham_ctx) +
1510 					sizeof(struct omap_sham_hmac_ctx),
1511 		.cra_alignmask		= OMAP_ALIGN_MASK,
1512 		.cra_module		= THIS_MODULE,
1513 		.cra_init		= omap_sham_cra_md5_init,
1514 		.cra_exit		= omap_sham_cra_exit,
1515 	},
1516 	.op.do_one_request = omap_sham_hash_one_req,
1517 }
1518 };
1519 
1520 /* OMAP4 has some algs in addition to what OMAP2 has */
1521 static struct ahash_engine_alg algs_sha224_sha256[] = {
1522 {
1523 	.base.init		= omap_sham_init,
1524 	.base.update		= omap_sham_update,
1525 	.base.final		= omap_sham_final,
1526 	.base.finup		= omap_sham_finup,
1527 	.base.digest		= omap_sham_digest,
1528 	.base.halg.digestsize	= SHA224_DIGEST_SIZE,
1529 	.base.halg.base	= {
1530 		.cra_name		= "sha224",
1531 		.cra_driver_name	= "omap-sha224",
1532 		.cra_priority		= 400,
1533 		.cra_flags		= CRYPTO_ALG_KERN_DRIVER_ONLY |
1534 						CRYPTO_ALG_ASYNC |
1535 						CRYPTO_ALG_NEED_FALLBACK,
1536 		.cra_blocksize		= SHA224_BLOCK_SIZE,
1537 		.cra_ctxsize		= sizeof(struct omap_sham_ctx),
1538 		.cra_alignmask		= OMAP_ALIGN_MASK,
1539 		.cra_module		= THIS_MODULE,
1540 		.cra_init		= omap_sham_cra_init,
1541 		.cra_exit		= omap_sham_cra_exit,
1542 	},
1543 	.op.do_one_request = omap_sham_hash_one_req,
1544 },
1545 {
1546 	.base.init		= omap_sham_init,
1547 	.base.update		= omap_sham_update,
1548 	.base.final		= omap_sham_final,
1549 	.base.finup		= omap_sham_finup,
1550 	.base.digest		= omap_sham_digest,
1551 	.base.halg.digestsize	= SHA256_DIGEST_SIZE,
1552 	.base.halg.base	= {
1553 		.cra_name		= "sha256",
1554 		.cra_driver_name	= "omap-sha256",
1555 		.cra_priority		= 400,
1556 		.cra_flags		= CRYPTO_ALG_KERN_DRIVER_ONLY |
1557 						CRYPTO_ALG_ASYNC |
1558 						CRYPTO_ALG_NEED_FALLBACK,
1559 		.cra_blocksize		= SHA256_BLOCK_SIZE,
1560 		.cra_ctxsize		= sizeof(struct omap_sham_ctx),
1561 		.cra_alignmask		= OMAP_ALIGN_MASK,
1562 		.cra_module		= THIS_MODULE,
1563 		.cra_init		= omap_sham_cra_init,
1564 		.cra_exit		= omap_sham_cra_exit,
1565 	},
1566 	.op.do_one_request = omap_sham_hash_one_req,
1567 },
1568 {
1569 	.base.init		= omap_sham_init,
1570 	.base.update		= omap_sham_update,
1571 	.base.final		= omap_sham_final,
1572 	.base.finup		= omap_sham_finup,
1573 	.base.digest		= omap_sham_digest,
1574 	.base.setkey		= omap_sham_setkey,
1575 	.base.halg.digestsize	= SHA224_DIGEST_SIZE,
1576 	.base.halg.base	= {
1577 		.cra_name		= "hmac(sha224)",
1578 		.cra_driver_name	= "omap-hmac-sha224",
1579 		.cra_priority		= 400,
1580 		.cra_flags		= CRYPTO_ALG_KERN_DRIVER_ONLY |
1581 						CRYPTO_ALG_ASYNC |
1582 						CRYPTO_ALG_NEED_FALLBACK,
1583 		.cra_blocksize		= SHA224_BLOCK_SIZE,
1584 		.cra_ctxsize		= sizeof(struct omap_sham_ctx) +
1585 					sizeof(struct omap_sham_hmac_ctx),
1586 		.cra_alignmask		= OMAP_ALIGN_MASK,
1587 		.cra_module		= THIS_MODULE,
1588 		.cra_init		= omap_sham_cra_sha224_init,
1589 		.cra_exit		= omap_sham_cra_exit,
1590 	},
1591 	.op.do_one_request = omap_sham_hash_one_req,
1592 },
1593 {
1594 	.base.init		= omap_sham_init,
1595 	.base.update		= omap_sham_update,
1596 	.base.final		= omap_sham_final,
1597 	.base.finup		= omap_sham_finup,
1598 	.base.digest		= omap_sham_digest,
1599 	.base.setkey		= omap_sham_setkey,
1600 	.base.halg.digestsize	= SHA256_DIGEST_SIZE,
1601 	.base.halg.base	= {
1602 		.cra_name		= "hmac(sha256)",
1603 		.cra_driver_name	= "omap-hmac-sha256",
1604 		.cra_priority		= 400,
1605 		.cra_flags		= CRYPTO_ALG_KERN_DRIVER_ONLY |
1606 						CRYPTO_ALG_ASYNC |
1607 						CRYPTO_ALG_NEED_FALLBACK,
1608 		.cra_blocksize		= SHA256_BLOCK_SIZE,
1609 		.cra_ctxsize		= sizeof(struct omap_sham_ctx) +
1610 					sizeof(struct omap_sham_hmac_ctx),
1611 		.cra_alignmask		= OMAP_ALIGN_MASK,
1612 		.cra_module		= THIS_MODULE,
1613 		.cra_init		= omap_sham_cra_sha256_init,
1614 		.cra_exit		= omap_sham_cra_exit,
1615 	},
1616 	.op.do_one_request = omap_sham_hash_one_req,
1617 },
1618 };
1619 
1620 static struct ahash_engine_alg algs_sha384_sha512[] = {
1621 {
1622 	.base.init		= omap_sham_init,
1623 	.base.update		= omap_sham_update,
1624 	.base.final		= omap_sham_final,
1625 	.base.finup		= omap_sham_finup,
1626 	.base.digest		= omap_sham_digest,
1627 	.base.halg.digestsize	= SHA384_DIGEST_SIZE,
1628 	.base.halg.base	= {
1629 		.cra_name		= "sha384",
1630 		.cra_driver_name	= "omap-sha384",
1631 		.cra_priority		= 400,
1632 		.cra_flags		= CRYPTO_ALG_KERN_DRIVER_ONLY |
1633 						CRYPTO_ALG_ASYNC |
1634 						CRYPTO_ALG_NEED_FALLBACK,
1635 		.cra_blocksize		= SHA384_BLOCK_SIZE,
1636 		.cra_ctxsize		= sizeof(struct omap_sham_ctx),
1637 		.cra_alignmask		= OMAP_ALIGN_MASK,
1638 		.cra_module		= THIS_MODULE,
1639 		.cra_init		= omap_sham_cra_init,
1640 		.cra_exit		= omap_sham_cra_exit,
1641 	},
1642 	.op.do_one_request = omap_sham_hash_one_req,
1643 },
1644 {
1645 	.base.init		= omap_sham_init,
1646 	.base.update		= omap_sham_update,
1647 	.base.final		= omap_sham_final,
1648 	.base.finup		= omap_sham_finup,
1649 	.base.digest		= omap_sham_digest,
1650 	.base.halg.digestsize	= SHA512_DIGEST_SIZE,
1651 	.base.halg.base	= {
1652 		.cra_name		= "sha512",
1653 		.cra_driver_name	= "omap-sha512",
1654 		.cra_priority		= 400,
1655 		.cra_flags		= CRYPTO_ALG_KERN_DRIVER_ONLY |
1656 						CRYPTO_ALG_ASYNC |
1657 						CRYPTO_ALG_NEED_FALLBACK,
1658 		.cra_blocksize		= SHA512_BLOCK_SIZE,
1659 		.cra_ctxsize		= sizeof(struct omap_sham_ctx),
1660 		.cra_alignmask		= OMAP_ALIGN_MASK,
1661 		.cra_module		= THIS_MODULE,
1662 		.cra_init		= omap_sham_cra_init,
1663 		.cra_exit		= omap_sham_cra_exit,
1664 	},
1665 	.op.do_one_request = omap_sham_hash_one_req,
1666 },
1667 {
1668 	.base.init		= omap_sham_init,
1669 	.base.update		= omap_sham_update,
1670 	.base.final		= omap_sham_final,
1671 	.base.finup		= omap_sham_finup,
1672 	.base.digest		= omap_sham_digest,
1673 	.base.setkey		= omap_sham_setkey,
1674 	.base.halg.digestsize	= SHA384_DIGEST_SIZE,
1675 	.base.halg.base	= {
1676 		.cra_name		= "hmac(sha384)",
1677 		.cra_driver_name	= "omap-hmac-sha384",
1678 		.cra_priority		= 400,
1679 		.cra_flags		= CRYPTO_ALG_KERN_DRIVER_ONLY |
1680 						CRYPTO_ALG_ASYNC |
1681 						CRYPTO_ALG_NEED_FALLBACK,
1682 		.cra_blocksize		= SHA384_BLOCK_SIZE,
1683 		.cra_ctxsize		= sizeof(struct omap_sham_ctx) +
1684 					sizeof(struct omap_sham_hmac_ctx),
1685 		.cra_alignmask		= OMAP_ALIGN_MASK,
1686 		.cra_module		= THIS_MODULE,
1687 		.cra_init		= omap_sham_cra_sha384_init,
1688 		.cra_exit		= omap_sham_cra_exit,
1689 	},
1690 	.op.do_one_request = omap_sham_hash_one_req,
1691 },
1692 {
1693 	.base.init		= omap_sham_init,
1694 	.base.update		= omap_sham_update,
1695 	.base.final		= omap_sham_final,
1696 	.base.finup		= omap_sham_finup,
1697 	.base.digest		= omap_sham_digest,
1698 	.base.setkey		= omap_sham_setkey,
1699 	.base.halg.digestsize	= SHA512_DIGEST_SIZE,
1700 	.base.halg.base	= {
1701 		.cra_name		= "hmac(sha512)",
1702 		.cra_driver_name	= "omap-hmac-sha512",
1703 		.cra_priority		= 400,
1704 		.cra_flags		= CRYPTO_ALG_KERN_DRIVER_ONLY |
1705 						CRYPTO_ALG_ASYNC |
1706 						CRYPTO_ALG_NEED_FALLBACK,
1707 		.cra_blocksize		= SHA512_BLOCK_SIZE,
1708 		.cra_ctxsize		= sizeof(struct omap_sham_ctx) +
1709 					sizeof(struct omap_sham_hmac_ctx),
1710 		.cra_alignmask		= OMAP_ALIGN_MASK,
1711 		.cra_module		= THIS_MODULE,
1712 		.cra_init		= omap_sham_cra_sha512_init,
1713 		.cra_exit		= omap_sham_cra_exit,
1714 	},
1715 	.op.do_one_request = omap_sham_hash_one_req,
1716 },
1717 };
1718 
omap_sham_done_task(unsigned long data)1719 static void omap_sham_done_task(unsigned long data)
1720 {
1721 	struct omap_sham_dev *dd = (struct omap_sham_dev *)data;
1722 	int err = 0;
1723 
1724 	dev_dbg(dd->dev, "%s: flags=%lx\n", __func__, dd->flags);
1725 
1726 	if (test_bit(FLAGS_CPU, &dd->flags)) {
1727 		if (test_and_clear_bit(FLAGS_OUTPUT_READY, &dd->flags))
1728 			goto finish;
1729 	} else if (test_bit(FLAGS_DMA_READY, &dd->flags)) {
1730 		if (test_bit(FLAGS_DMA_ACTIVE, &dd->flags)) {
1731 			omap_sham_update_dma_stop(dd);
1732 			if (dd->err) {
1733 				err = dd->err;
1734 				goto finish;
1735 			}
1736 		}
1737 		if (test_and_clear_bit(FLAGS_OUTPUT_READY, &dd->flags)) {
1738 			/* hash or semi-hash ready */
1739 			clear_bit(FLAGS_DMA_READY, &dd->flags);
1740 			goto finish;
1741 		}
1742 	}
1743 
1744 	return;
1745 
1746 finish:
1747 	dev_dbg(dd->dev, "update done: err: %d\n", err);
1748 	/* finish curent request */
1749 	omap_sham_finish_req(dd->req, err);
1750 }
1751 
omap_sham_irq_common(struct omap_sham_dev * dd)1752 static irqreturn_t omap_sham_irq_common(struct omap_sham_dev *dd)
1753 {
1754 	set_bit(FLAGS_OUTPUT_READY, &dd->flags);
1755 	tasklet_schedule(&dd->done_task);
1756 
1757 	return IRQ_HANDLED;
1758 }
1759 
omap_sham_irq_omap2(int irq,void * dev_id)1760 static irqreturn_t omap_sham_irq_omap2(int irq, void *dev_id)
1761 {
1762 	struct omap_sham_dev *dd = dev_id;
1763 
1764 	if (unlikely(test_bit(FLAGS_FINAL, &dd->flags)))
1765 		/* final -> allow device to go to power-saving mode */
1766 		omap_sham_write_mask(dd, SHA_REG_CTRL, 0, SHA_REG_CTRL_LENGTH);
1767 
1768 	omap_sham_write_mask(dd, SHA_REG_CTRL, SHA_REG_CTRL_OUTPUT_READY,
1769 				 SHA_REG_CTRL_OUTPUT_READY);
1770 	omap_sham_read(dd, SHA_REG_CTRL);
1771 
1772 	return omap_sham_irq_common(dd);
1773 }
1774 
omap_sham_irq_omap4(int irq,void * dev_id)1775 static irqreturn_t omap_sham_irq_omap4(int irq, void *dev_id)
1776 {
1777 	struct omap_sham_dev *dd = dev_id;
1778 
1779 	omap_sham_write_mask(dd, SHA_REG_MASK(dd), 0, SHA_REG_MASK_IT_EN);
1780 
1781 	return omap_sham_irq_common(dd);
1782 }
1783 
1784 static struct omap_sham_algs_info omap_sham_algs_info_omap2[] = {
1785 	{
1786 		.algs_list	= algs_sha1_md5,
1787 		.size		= ARRAY_SIZE(algs_sha1_md5),
1788 	},
1789 };
1790 
1791 static const struct omap_sham_pdata omap_sham_pdata_omap2 = {
1792 	.algs_info	= omap_sham_algs_info_omap2,
1793 	.algs_info_size	= ARRAY_SIZE(omap_sham_algs_info_omap2),
1794 	.flags		= BIT(FLAGS_BE32_SHA1),
1795 	.digest_size	= SHA1_DIGEST_SIZE,
1796 	.copy_hash	= omap_sham_copy_hash_omap2,
1797 	.write_ctrl	= omap_sham_write_ctrl_omap2,
1798 	.trigger	= omap_sham_trigger_omap2,
1799 	.poll_irq	= omap_sham_poll_irq_omap2,
1800 	.intr_hdlr	= omap_sham_irq_omap2,
1801 	.idigest_ofs	= 0x00,
1802 	.din_ofs	= 0x1c,
1803 	.digcnt_ofs	= 0x14,
1804 	.rev_ofs	= 0x5c,
1805 	.mask_ofs	= 0x60,
1806 	.sysstatus_ofs	= 0x64,
1807 	.major_mask	= 0xf0,
1808 	.major_shift	= 4,
1809 	.minor_mask	= 0x0f,
1810 	.minor_shift	= 0,
1811 };
1812 
1813 #ifdef CONFIG_OF
1814 static struct omap_sham_algs_info omap_sham_algs_info_omap4[] = {
1815 	{
1816 		.algs_list	= algs_sha1_md5,
1817 		.size		= ARRAY_SIZE(algs_sha1_md5),
1818 	},
1819 	{
1820 		.algs_list	= algs_sha224_sha256,
1821 		.size		= ARRAY_SIZE(algs_sha224_sha256),
1822 	},
1823 };
1824 
1825 static const struct omap_sham_pdata omap_sham_pdata_omap4 = {
1826 	.algs_info	= omap_sham_algs_info_omap4,
1827 	.algs_info_size	= ARRAY_SIZE(omap_sham_algs_info_omap4),
1828 	.flags		= BIT(FLAGS_AUTO_XOR),
1829 	.digest_size	= SHA256_DIGEST_SIZE,
1830 	.copy_hash	= omap_sham_copy_hash_omap4,
1831 	.write_ctrl	= omap_sham_write_ctrl_omap4,
1832 	.trigger	= omap_sham_trigger_omap4,
1833 	.poll_irq	= omap_sham_poll_irq_omap4,
1834 	.intr_hdlr	= omap_sham_irq_omap4,
1835 	.idigest_ofs	= 0x020,
1836 	.odigest_ofs	= 0x0,
1837 	.din_ofs	= 0x080,
1838 	.digcnt_ofs	= 0x040,
1839 	.rev_ofs	= 0x100,
1840 	.mask_ofs	= 0x110,
1841 	.sysstatus_ofs	= 0x114,
1842 	.mode_ofs	= 0x44,
1843 	.length_ofs	= 0x48,
1844 	.major_mask	= 0x0700,
1845 	.major_shift	= 8,
1846 	.minor_mask	= 0x003f,
1847 	.minor_shift	= 0,
1848 };
1849 
1850 static struct omap_sham_algs_info omap_sham_algs_info_omap5[] = {
1851 	{
1852 		.algs_list	= algs_sha1_md5,
1853 		.size		= ARRAY_SIZE(algs_sha1_md5),
1854 	},
1855 	{
1856 		.algs_list	= algs_sha224_sha256,
1857 		.size		= ARRAY_SIZE(algs_sha224_sha256),
1858 	},
1859 	{
1860 		.algs_list	= algs_sha384_sha512,
1861 		.size		= ARRAY_SIZE(algs_sha384_sha512),
1862 	},
1863 };
1864 
1865 static const struct omap_sham_pdata omap_sham_pdata_omap5 = {
1866 	.algs_info	= omap_sham_algs_info_omap5,
1867 	.algs_info_size	= ARRAY_SIZE(omap_sham_algs_info_omap5),
1868 	.flags		= BIT(FLAGS_AUTO_XOR),
1869 	.digest_size	= SHA512_DIGEST_SIZE,
1870 	.copy_hash	= omap_sham_copy_hash_omap4,
1871 	.write_ctrl	= omap_sham_write_ctrl_omap4,
1872 	.trigger	= omap_sham_trigger_omap4,
1873 	.poll_irq	= omap_sham_poll_irq_omap4,
1874 	.intr_hdlr	= omap_sham_irq_omap4,
1875 	.idigest_ofs	= 0x240,
1876 	.odigest_ofs	= 0x200,
1877 	.din_ofs	= 0x080,
1878 	.digcnt_ofs	= 0x280,
1879 	.rev_ofs	= 0x100,
1880 	.mask_ofs	= 0x110,
1881 	.sysstatus_ofs	= 0x114,
1882 	.mode_ofs	= 0x284,
1883 	.length_ofs	= 0x288,
1884 	.major_mask	= 0x0700,
1885 	.major_shift	= 8,
1886 	.minor_mask	= 0x003f,
1887 	.minor_shift	= 0,
1888 };
1889 
1890 static const struct of_device_id omap_sham_of_match[] = {
1891 	{
1892 		.compatible	= "ti,omap2-sham",
1893 		.data		= &omap_sham_pdata_omap2,
1894 	},
1895 	{
1896 		.compatible	= "ti,omap3-sham",
1897 		.data		= &omap_sham_pdata_omap2,
1898 	},
1899 	{
1900 		.compatible	= "ti,omap4-sham",
1901 		.data		= &omap_sham_pdata_omap4,
1902 	},
1903 	{
1904 		.compatible	= "ti,omap5-sham",
1905 		.data		= &omap_sham_pdata_omap5,
1906 	},
1907 	{},
1908 };
1909 MODULE_DEVICE_TABLE(of, omap_sham_of_match);
1910 
omap_sham_get_res_of(struct omap_sham_dev * dd,struct device * dev,struct resource * res)1911 static int omap_sham_get_res_of(struct omap_sham_dev *dd,
1912 		struct device *dev, struct resource *res)
1913 {
1914 	struct device_node *node = dev->of_node;
1915 	int err = 0;
1916 
1917 	dd->pdata = of_device_get_match_data(dev);
1918 	if (!dd->pdata) {
1919 		dev_err(dev, "no compatible OF match\n");
1920 		err = -EINVAL;
1921 		goto err;
1922 	}
1923 
1924 	err = of_address_to_resource(node, 0, res);
1925 	if (err < 0) {
1926 		dev_err(dev, "can't translate OF node address\n");
1927 		err = -EINVAL;
1928 		goto err;
1929 	}
1930 
1931 	dd->irq = irq_of_parse_and_map(node, 0);
1932 	if (!dd->irq) {
1933 		dev_err(dev, "can't translate OF irq value\n");
1934 		err = -EINVAL;
1935 		goto err;
1936 	}
1937 
1938 err:
1939 	return err;
1940 }
1941 #else
1942 static const struct of_device_id omap_sham_of_match[] = {
1943 	{},
1944 };
1945 
omap_sham_get_res_of(struct omap_sham_dev * dd,struct device * dev,struct resource * res)1946 static int omap_sham_get_res_of(struct omap_sham_dev *dd,
1947 		struct device *dev, struct resource *res)
1948 {
1949 	return -EINVAL;
1950 }
1951 #endif
1952 
omap_sham_get_res_pdev(struct omap_sham_dev * dd,struct platform_device * pdev,struct resource * res)1953 static int omap_sham_get_res_pdev(struct omap_sham_dev *dd,
1954 		struct platform_device *pdev, struct resource *res)
1955 {
1956 	struct device *dev = &pdev->dev;
1957 	struct resource *r;
1958 	int err = 0;
1959 
1960 	/* Get the base address */
1961 	r = platform_get_resource(pdev, IORESOURCE_MEM, 0);
1962 	if (!r) {
1963 		dev_err(dev, "no MEM resource info\n");
1964 		err = -ENODEV;
1965 		goto err;
1966 	}
1967 	memcpy(res, r, sizeof(*res));
1968 
1969 	/* Get the IRQ */
1970 	dd->irq = platform_get_irq(pdev, 0);
1971 	if (dd->irq < 0) {
1972 		err = dd->irq;
1973 		goto err;
1974 	}
1975 
1976 	/* Only OMAP2/3 can be non-DT */
1977 	dd->pdata = &omap_sham_pdata_omap2;
1978 
1979 err:
1980 	return err;
1981 }
1982 
fallback_show(struct device * dev,struct device_attribute * attr,char * buf)1983 static ssize_t fallback_show(struct device *dev, struct device_attribute *attr,
1984 			     char *buf)
1985 {
1986 	struct omap_sham_dev *dd = dev_get_drvdata(dev);
1987 
1988 	return sprintf(buf, "%d\n", dd->fallback_sz);
1989 }
1990 
fallback_store(struct device * dev,struct device_attribute * attr,const char * buf,size_t size)1991 static ssize_t fallback_store(struct device *dev, struct device_attribute *attr,
1992 			      const char *buf, size_t size)
1993 {
1994 	struct omap_sham_dev *dd = dev_get_drvdata(dev);
1995 	ssize_t status;
1996 	long value;
1997 
1998 	status = kstrtol(buf, 0, &value);
1999 	if (status)
2000 		return status;
2001 
2002 	/* HW accelerator only works with buffers > 9 */
2003 	if (value < 9) {
2004 		dev_err(dev, "minimum fallback size 9\n");
2005 		return -EINVAL;
2006 	}
2007 
2008 	dd->fallback_sz = value;
2009 
2010 	return size;
2011 }
2012 
queue_len_show(struct device * dev,struct device_attribute * attr,char * buf)2013 static ssize_t queue_len_show(struct device *dev, struct device_attribute *attr,
2014 			      char *buf)
2015 {
2016 	struct omap_sham_dev *dd = dev_get_drvdata(dev);
2017 
2018 	return sprintf(buf, "%d\n", dd->queue.max_qlen);
2019 }
2020 
queue_len_store(struct device * dev,struct device_attribute * attr,const char * buf,size_t size)2021 static ssize_t queue_len_store(struct device *dev,
2022 			       struct device_attribute *attr, const char *buf,
2023 			       size_t size)
2024 {
2025 	struct omap_sham_dev *dd = dev_get_drvdata(dev);
2026 	ssize_t status;
2027 	long value;
2028 
2029 	status = kstrtol(buf, 0, &value);
2030 	if (status)
2031 		return status;
2032 
2033 	if (value < 1)
2034 		return -EINVAL;
2035 
2036 	/*
2037 	 * Changing the queue size in fly is safe, if size becomes smaller
2038 	 * than current size, it will just not accept new entries until
2039 	 * it has shrank enough.
2040 	 */
2041 	dd->queue.max_qlen = value;
2042 
2043 	return size;
2044 }
2045 
2046 static DEVICE_ATTR_RW(queue_len);
2047 static DEVICE_ATTR_RW(fallback);
2048 
2049 static struct attribute *omap_sham_attrs[] = {
2050 	&dev_attr_queue_len.attr,
2051 	&dev_attr_fallback.attr,
2052 	NULL,
2053 };
2054 
2055 static const struct attribute_group omap_sham_attr_group = {
2056 	.attrs = omap_sham_attrs,
2057 };
2058 
omap_sham_probe(struct platform_device * pdev)2059 static int omap_sham_probe(struct platform_device *pdev)
2060 {
2061 	struct omap_sham_dev *dd;
2062 	struct device *dev = &pdev->dev;
2063 	struct resource res;
2064 	dma_cap_mask_t mask;
2065 	int err, i, j;
2066 	u32 rev;
2067 
2068 	dd = devm_kzalloc(dev, sizeof(struct omap_sham_dev), GFP_KERNEL);
2069 	if (dd == NULL) {
2070 		dev_err(dev, "unable to alloc data struct.\n");
2071 		err = -ENOMEM;
2072 		goto data_err;
2073 	}
2074 	dd->dev = dev;
2075 	platform_set_drvdata(pdev, dd);
2076 
2077 	INIT_LIST_HEAD(&dd->list);
2078 	tasklet_init(&dd->done_task, omap_sham_done_task, (unsigned long)dd);
2079 	crypto_init_queue(&dd->queue, OMAP_SHAM_QUEUE_LENGTH);
2080 
2081 	err = (dev->of_node) ? omap_sham_get_res_of(dd, dev, &res) :
2082 			       omap_sham_get_res_pdev(dd, pdev, &res);
2083 	if (err)
2084 		goto data_err;
2085 
2086 	dd->io_base = devm_ioremap_resource(dev, &res);
2087 	if (IS_ERR(dd->io_base)) {
2088 		err = PTR_ERR(dd->io_base);
2089 		goto data_err;
2090 	}
2091 	dd->phys_base = res.start;
2092 
2093 	err = devm_request_irq(dev, dd->irq, dd->pdata->intr_hdlr,
2094 			       IRQF_TRIGGER_NONE, dev_name(dev), dd);
2095 	if (err) {
2096 		dev_err(dev, "unable to request irq %d, err = %d\n",
2097 			dd->irq, err);
2098 		goto data_err;
2099 	}
2100 
2101 	dma_cap_zero(mask);
2102 	dma_cap_set(DMA_SLAVE, mask);
2103 
2104 	dd->dma_lch = dma_request_chan(dev, "rx");
2105 	if (IS_ERR(dd->dma_lch)) {
2106 		err = PTR_ERR(dd->dma_lch);
2107 		if (err == -EPROBE_DEFER)
2108 			goto data_err;
2109 
2110 		dd->polling_mode = 1;
2111 		dev_dbg(dev, "using polling mode instead of dma\n");
2112 	}
2113 
2114 	dd->flags |= dd->pdata->flags;
2115 	sham.flags |= dd->pdata->flags;
2116 
2117 	pm_runtime_use_autosuspend(dev);
2118 	pm_runtime_set_autosuspend_delay(dev, DEFAULT_AUTOSUSPEND_DELAY);
2119 
2120 	dd->fallback_sz = OMAP_SHA_DMA_THRESHOLD;
2121 
2122 	pm_runtime_enable(dev);
2123 
2124 	err = pm_runtime_resume_and_get(dev);
2125 	if (err < 0) {
2126 		dev_err(dev, "failed to get sync: %d\n", err);
2127 		goto err_pm;
2128 	}
2129 
2130 	rev = omap_sham_read(dd, SHA_REG_REV(dd));
2131 	pm_runtime_put_sync(&pdev->dev);
2132 
2133 	dev_info(dev, "hw accel on OMAP rev %u.%u\n",
2134 		(rev & dd->pdata->major_mask) >> dd->pdata->major_shift,
2135 		(rev & dd->pdata->minor_mask) >> dd->pdata->minor_shift);
2136 
2137 	spin_lock_bh(&sham.lock);
2138 	list_add_tail(&dd->list, &sham.dev_list);
2139 	spin_unlock_bh(&sham.lock);
2140 
2141 	dd->engine = crypto_engine_alloc_init(dev, 1);
2142 	if (!dd->engine) {
2143 		err = -ENOMEM;
2144 		goto err_engine;
2145 	}
2146 
2147 	err = crypto_engine_start(dd->engine);
2148 	if (err)
2149 		goto err_engine_start;
2150 
2151 	for (i = 0; i < dd->pdata->algs_info_size; i++) {
2152 		if (dd->pdata->algs_info[i].registered)
2153 			break;
2154 
2155 		for (j = 0; j < dd->pdata->algs_info[i].size; j++) {
2156 			struct ahash_engine_alg *ealg;
2157 			struct ahash_alg *alg;
2158 
2159 			ealg = &dd->pdata->algs_info[i].algs_list[j];
2160 			alg = &ealg->base;
2161 			alg->export = omap_sham_export;
2162 			alg->import = omap_sham_import;
2163 			alg->halg.statesize = sizeof(struct omap_sham_reqctx) +
2164 					      BUFLEN;
2165 			err = crypto_engine_register_ahash(ealg);
2166 			if (err)
2167 				goto err_algs;
2168 
2169 			dd->pdata->algs_info[i].registered++;
2170 		}
2171 	}
2172 
2173 	err = sysfs_create_group(&dev->kobj, &omap_sham_attr_group);
2174 	if (err) {
2175 		dev_err(dev, "could not create sysfs device attrs\n");
2176 		goto err_algs;
2177 	}
2178 
2179 	return 0;
2180 
2181 err_algs:
2182 	for (i = dd->pdata->algs_info_size - 1; i >= 0; i--)
2183 		for (j = dd->pdata->algs_info[i].registered - 1; j >= 0; j--)
2184 			crypto_engine_unregister_ahash(
2185 					&dd->pdata->algs_info[i].algs_list[j]);
2186 err_engine_start:
2187 	crypto_engine_exit(dd->engine);
2188 err_engine:
2189 	spin_lock_bh(&sham.lock);
2190 	list_del(&dd->list);
2191 	spin_unlock_bh(&sham.lock);
2192 err_pm:
2193 	pm_runtime_dont_use_autosuspend(dev);
2194 	pm_runtime_disable(dev);
2195 	if (!dd->polling_mode)
2196 		dma_release_channel(dd->dma_lch);
2197 data_err:
2198 	dev_err(dev, "initialization failed.\n");
2199 
2200 	return err;
2201 }
2202 
omap_sham_remove(struct platform_device * pdev)2203 static int omap_sham_remove(struct platform_device *pdev)
2204 {
2205 	struct omap_sham_dev *dd;
2206 	int i, j;
2207 
2208 	dd = platform_get_drvdata(pdev);
2209 
2210 	spin_lock_bh(&sham.lock);
2211 	list_del(&dd->list);
2212 	spin_unlock_bh(&sham.lock);
2213 	for (i = dd->pdata->algs_info_size - 1; i >= 0; i--)
2214 		for (j = dd->pdata->algs_info[i].registered - 1; j >= 0; j--) {
2215 			crypto_engine_unregister_ahash(
2216 					&dd->pdata->algs_info[i].algs_list[j]);
2217 			dd->pdata->algs_info[i].registered--;
2218 		}
2219 	tasklet_kill(&dd->done_task);
2220 	pm_runtime_dont_use_autosuspend(&pdev->dev);
2221 	pm_runtime_disable(&pdev->dev);
2222 
2223 	if (!dd->polling_mode)
2224 		dma_release_channel(dd->dma_lch);
2225 
2226 	sysfs_remove_group(&dd->dev->kobj, &omap_sham_attr_group);
2227 
2228 	return 0;
2229 }
2230 
2231 static struct platform_driver omap_sham_driver = {
2232 	.probe	= omap_sham_probe,
2233 	.remove	= omap_sham_remove,
2234 	.driver	= {
2235 		.name	= "omap-sham",
2236 		.of_match_table	= omap_sham_of_match,
2237 	},
2238 };
2239 
2240 module_platform_driver(omap_sham_driver);
2241 
2242 MODULE_DESCRIPTION("OMAP SHA1/MD5 hw acceleration support.");
2243 MODULE_LICENSE("GPL v2");
2244 MODULE_AUTHOR("Dmitry Kasatkin");
2245 MODULE_ALIAS("platform:omap-sham");
2246