1 // SPDX-License-Identifier: GPL-2.0
2 /*
3  * sun8i-ss-hash.c - hardware cryptographic offloader for
4  * Allwinner A80/A83T SoC
5  *
6  * Copyright (C) 2015-2020 Corentin Labbe <clabbe@baylibre.com>
7  *
8  * This file add support for MD5 and SHA1/SHA224/SHA256.
9  *
10  * You could find the datasheet in Documentation/arm/sunxi.rst
11  */
12 #include <linux/bottom_half.h>
13 #include <linux/dma-mapping.h>
14 #include <linux/pm_runtime.h>
15 #include <linux/scatterlist.h>
16 #include <crypto/internal/hash.h>
17 #include <crypto/hmac.h>
18 #include <crypto/scatterwalk.h>
19 #include <crypto/sha1.h>
20 #include <crypto/sha2.h>
21 #include <crypto/md5.h>
22 #include "sun8i-ss.h"
23 
24 static int sun8i_ss_hashkey(struct sun8i_ss_hash_tfm_ctx *tfmctx, const u8 *key,
25 			    unsigned int keylen)
26 {
27 	struct crypto_shash *xtfm;
28 	struct shash_desc *sdesc;
29 	size_t len;
30 	int ret = 0;
31 
32 	xtfm = crypto_alloc_shash("sha1", 0, CRYPTO_ALG_NEED_FALLBACK);
33 	if (IS_ERR(xtfm))
34 		return PTR_ERR(xtfm);
35 
36 	len = sizeof(*sdesc) + crypto_shash_descsize(xtfm);
37 	sdesc = kmalloc(len, GFP_KERNEL);
38 	if (!sdesc) {
39 		ret = -ENOMEM;
40 		goto err_hashkey_sdesc;
41 	}
42 	sdesc->tfm = xtfm;
43 
44 	ret = crypto_shash_init(sdesc);
45 	if (ret) {
46 		dev_err(tfmctx->ss->dev, "shash init error ret=%d\n", ret);
47 		goto err_hashkey;
48 	}
49 	ret = crypto_shash_finup(sdesc, key, keylen, tfmctx->key);
50 	if (ret)
51 		dev_err(tfmctx->ss->dev, "shash finup error\n");
52 err_hashkey:
53 	kfree(sdesc);
54 err_hashkey_sdesc:
55 	crypto_free_shash(xtfm);
56 	return ret;
57 }
58 
59 int sun8i_ss_hmac_setkey(struct crypto_ahash *ahash, const u8 *key,
60 			 unsigned int keylen)
61 {
62 	struct sun8i_ss_hash_tfm_ctx *tfmctx = crypto_ahash_ctx(ahash);
63 	struct ahash_alg *alg = __crypto_ahash_alg(ahash->base.__crt_alg);
64 	struct sun8i_ss_alg_template *algt;
65 	int digestsize, i;
66 	int bs = crypto_ahash_blocksize(ahash);
67 	int ret;
68 
69 	algt = container_of(alg, struct sun8i_ss_alg_template, alg.hash);
70 	digestsize = algt->alg.hash.halg.digestsize;
71 
72 	if (keylen > bs) {
73 		ret = sun8i_ss_hashkey(tfmctx, key, keylen);
74 		if (ret)
75 			return ret;
76 		tfmctx->keylen = digestsize;
77 	} else {
78 		tfmctx->keylen = keylen;
79 		memcpy(tfmctx->key, key, keylen);
80 	}
81 
82 	tfmctx->ipad = kzalloc(bs, GFP_KERNEL | GFP_DMA);
83 	if (!tfmctx->ipad)
84 		return -ENOMEM;
85 	tfmctx->opad = kzalloc(bs, GFP_KERNEL | GFP_DMA);
86 	if (!tfmctx->opad) {
87 		ret = -ENOMEM;
88 		goto err_opad;
89 	}
90 
91 	memset(tfmctx->key + tfmctx->keylen, 0, bs - tfmctx->keylen);
92 	memcpy(tfmctx->ipad, tfmctx->key, tfmctx->keylen);
93 	memcpy(tfmctx->opad, tfmctx->key, tfmctx->keylen);
94 	for (i = 0; i < bs; i++) {
95 		tfmctx->ipad[i] ^= HMAC_IPAD_VALUE;
96 		tfmctx->opad[i] ^= HMAC_OPAD_VALUE;
97 	}
98 
99 	ret = crypto_ahash_setkey(tfmctx->fallback_tfm, key, keylen);
100 	if (!ret)
101 		return 0;
102 
103 	memzero_explicit(tfmctx->key, keylen);
104 	kfree_sensitive(tfmctx->opad);
105 err_opad:
106 	kfree_sensitive(tfmctx->ipad);
107 	return ret;
108 }
109 
110 int sun8i_ss_hash_crainit(struct crypto_tfm *tfm)
111 {
112 	struct sun8i_ss_hash_tfm_ctx *op = crypto_tfm_ctx(tfm);
113 	struct ahash_alg *alg = __crypto_ahash_alg(tfm->__crt_alg);
114 	struct sun8i_ss_alg_template *algt;
115 	int err;
116 
117 	memset(op, 0, sizeof(struct sun8i_ss_hash_tfm_ctx));
118 
119 	algt = container_of(alg, struct sun8i_ss_alg_template, alg.hash);
120 	op->ss = algt->ss;
121 
122 	op->enginectx.op.do_one_request = sun8i_ss_hash_run;
123 	op->enginectx.op.prepare_request = NULL;
124 	op->enginectx.op.unprepare_request = NULL;
125 
126 	/* FALLBACK */
127 	op->fallback_tfm = crypto_alloc_ahash(crypto_tfm_alg_name(tfm), 0,
128 					      CRYPTO_ALG_NEED_FALLBACK);
129 	if (IS_ERR(op->fallback_tfm)) {
130 		dev_err(algt->ss->dev, "Fallback driver could no be loaded\n");
131 		return PTR_ERR(op->fallback_tfm);
132 	}
133 
134 	if (algt->alg.hash.halg.statesize < crypto_ahash_statesize(op->fallback_tfm))
135 		algt->alg.hash.halg.statesize = crypto_ahash_statesize(op->fallback_tfm);
136 
137 	crypto_ahash_set_reqsize(__crypto_ahash_cast(tfm),
138 				 sizeof(struct sun8i_ss_hash_reqctx) +
139 				 crypto_ahash_reqsize(op->fallback_tfm));
140 
141 	memcpy(algt->fbname, crypto_tfm_alg_driver_name(&op->fallback_tfm->base), CRYPTO_MAX_ALG_NAME);
142 
143 	err = pm_runtime_get_sync(op->ss->dev);
144 	if (err < 0)
145 		goto error_pm;
146 	return 0;
147 error_pm:
148 	pm_runtime_put_noidle(op->ss->dev);
149 	crypto_free_ahash(op->fallback_tfm);
150 	return err;
151 }
152 
153 void sun8i_ss_hash_craexit(struct crypto_tfm *tfm)
154 {
155 	struct sun8i_ss_hash_tfm_ctx *tfmctx = crypto_tfm_ctx(tfm);
156 
157 	kfree_sensitive(tfmctx->ipad);
158 	kfree_sensitive(tfmctx->opad);
159 
160 	crypto_free_ahash(tfmctx->fallback_tfm);
161 	pm_runtime_put_sync_suspend(tfmctx->ss->dev);
162 }
163 
164 int sun8i_ss_hash_init(struct ahash_request *areq)
165 {
166 	struct sun8i_ss_hash_reqctx *rctx = ahash_request_ctx(areq);
167 	struct crypto_ahash *tfm = crypto_ahash_reqtfm(areq);
168 	struct sun8i_ss_hash_tfm_ctx *tfmctx = crypto_ahash_ctx(tfm);
169 
170 	memset(rctx, 0, sizeof(struct sun8i_ss_hash_reqctx));
171 
172 	ahash_request_set_tfm(&rctx->fallback_req, tfmctx->fallback_tfm);
173 	rctx->fallback_req.base.flags = areq->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP;
174 
175 	return crypto_ahash_init(&rctx->fallback_req);
176 }
177 
178 int sun8i_ss_hash_export(struct ahash_request *areq, void *out)
179 {
180 	struct sun8i_ss_hash_reqctx *rctx = ahash_request_ctx(areq);
181 	struct crypto_ahash *tfm = crypto_ahash_reqtfm(areq);
182 	struct sun8i_ss_hash_tfm_ctx *tfmctx = crypto_ahash_ctx(tfm);
183 
184 	ahash_request_set_tfm(&rctx->fallback_req, tfmctx->fallback_tfm);
185 	rctx->fallback_req.base.flags = areq->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP;
186 
187 	return crypto_ahash_export(&rctx->fallback_req, out);
188 }
189 
190 int sun8i_ss_hash_import(struct ahash_request *areq, const void *in)
191 {
192 	struct sun8i_ss_hash_reqctx *rctx = ahash_request_ctx(areq);
193 	struct crypto_ahash *tfm = crypto_ahash_reqtfm(areq);
194 	struct sun8i_ss_hash_tfm_ctx *tfmctx = crypto_ahash_ctx(tfm);
195 
196 	ahash_request_set_tfm(&rctx->fallback_req, tfmctx->fallback_tfm);
197 	rctx->fallback_req.base.flags = areq->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP;
198 
199 	return crypto_ahash_import(&rctx->fallback_req, in);
200 }
201 
202 int sun8i_ss_hash_final(struct ahash_request *areq)
203 {
204 	struct sun8i_ss_hash_reqctx *rctx = ahash_request_ctx(areq);
205 	struct crypto_ahash *tfm = crypto_ahash_reqtfm(areq);
206 	struct sun8i_ss_hash_tfm_ctx *tfmctx = crypto_ahash_ctx(tfm);
207 #ifdef CONFIG_CRYPTO_DEV_SUN8I_SS_DEBUG
208 	struct ahash_alg *alg = __crypto_ahash_alg(tfm->base.__crt_alg);
209 	struct sun8i_ss_alg_template *algt;
210 #endif
211 
212 	ahash_request_set_tfm(&rctx->fallback_req, tfmctx->fallback_tfm);
213 	rctx->fallback_req.base.flags = areq->base.flags &
214 					CRYPTO_TFM_REQ_MAY_SLEEP;
215 	rctx->fallback_req.result = areq->result;
216 
217 #ifdef CONFIG_CRYPTO_DEV_SUN8I_SS_DEBUG
218 	algt = container_of(alg, struct sun8i_ss_alg_template, alg.hash);
219 	algt->stat_fb++;
220 #endif
221 
222 	return crypto_ahash_final(&rctx->fallback_req);
223 }
224 
225 int sun8i_ss_hash_update(struct ahash_request *areq)
226 {
227 	struct sun8i_ss_hash_reqctx *rctx = ahash_request_ctx(areq);
228 	struct crypto_ahash *tfm = crypto_ahash_reqtfm(areq);
229 	struct sun8i_ss_hash_tfm_ctx *tfmctx = crypto_ahash_ctx(tfm);
230 
231 	ahash_request_set_tfm(&rctx->fallback_req, tfmctx->fallback_tfm);
232 	rctx->fallback_req.base.flags = areq->base.flags &
233 					CRYPTO_TFM_REQ_MAY_SLEEP;
234 	rctx->fallback_req.nbytes = areq->nbytes;
235 	rctx->fallback_req.src = areq->src;
236 
237 	return crypto_ahash_update(&rctx->fallback_req);
238 }
239 
240 int sun8i_ss_hash_finup(struct ahash_request *areq)
241 {
242 	struct sun8i_ss_hash_reqctx *rctx = ahash_request_ctx(areq);
243 	struct crypto_ahash *tfm = crypto_ahash_reqtfm(areq);
244 	struct sun8i_ss_hash_tfm_ctx *tfmctx = crypto_ahash_ctx(tfm);
245 #ifdef CONFIG_CRYPTO_DEV_SUN8I_SS_DEBUG
246 	struct ahash_alg *alg = __crypto_ahash_alg(tfm->base.__crt_alg);
247 	struct sun8i_ss_alg_template *algt;
248 #endif
249 
250 	ahash_request_set_tfm(&rctx->fallback_req, tfmctx->fallback_tfm);
251 	rctx->fallback_req.base.flags = areq->base.flags &
252 					CRYPTO_TFM_REQ_MAY_SLEEP;
253 
254 	rctx->fallback_req.nbytes = areq->nbytes;
255 	rctx->fallback_req.src = areq->src;
256 	rctx->fallback_req.result = areq->result;
257 #ifdef CONFIG_CRYPTO_DEV_SUN8I_SS_DEBUG
258 	algt = container_of(alg, struct sun8i_ss_alg_template, alg.hash);
259 	algt->stat_fb++;
260 #endif
261 
262 	return crypto_ahash_finup(&rctx->fallback_req);
263 }
264 
265 static int sun8i_ss_hash_digest_fb(struct ahash_request *areq)
266 {
267 	struct sun8i_ss_hash_reqctx *rctx = ahash_request_ctx(areq);
268 	struct crypto_ahash *tfm = crypto_ahash_reqtfm(areq);
269 	struct sun8i_ss_hash_tfm_ctx *tfmctx = crypto_ahash_ctx(tfm);
270 #ifdef CONFIG_CRYPTO_DEV_SUN8I_SS_DEBUG
271 	struct ahash_alg *alg = __crypto_ahash_alg(tfm->base.__crt_alg);
272 	struct sun8i_ss_alg_template *algt;
273 #endif
274 
275 	ahash_request_set_tfm(&rctx->fallback_req, tfmctx->fallback_tfm);
276 	rctx->fallback_req.base.flags = areq->base.flags &
277 					CRYPTO_TFM_REQ_MAY_SLEEP;
278 
279 	rctx->fallback_req.nbytes = areq->nbytes;
280 	rctx->fallback_req.src = areq->src;
281 	rctx->fallback_req.result = areq->result;
282 #ifdef CONFIG_CRYPTO_DEV_SUN8I_SS_DEBUG
283 	algt = container_of(alg, struct sun8i_ss_alg_template, alg.hash);
284 	algt->stat_fb++;
285 #endif
286 
287 	return crypto_ahash_digest(&rctx->fallback_req);
288 }
289 
290 static int sun8i_ss_run_hash_task(struct sun8i_ss_dev *ss,
291 				  struct sun8i_ss_hash_reqctx *rctx,
292 				  const char *name)
293 {
294 	int flow = rctx->flow;
295 	u32 v = SS_START;
296 	int i;
297 
298 #ifdef CONFIG_CRYPTO_DEV_SUN8I_SS_DEBUG
299 	ss->flows[flow].stat_req++;
300 #endif
301 
302 	/* choose between stream0/stream1 */
303 	if (flow)
304 		v |= SS_FLOW1;
305 	else
306 		v |= SS_FLOW0;
307 
308 	v |= rctx->method;
309 
310 	for (i = 0; i < MAX_SG; i++) {
311 		if (!rctx->t_dst[i].addr)
312 			break;
313 
314 		mutex_lock(&ss->mlock);
315 		if (i > 0) {
316 			v |= BIT(17);
317 			writel(rctx->t_dst[i - 1].addr, ss->base + SS_KEY_ADR_REG);
318 			writel(rctx->t_dst[i - 1].addr, ss->base + SS_IV_ADR_REG);
319 		}
320 
321 		dev_dbg(ss->dev,
322 			"Processing SG %d on flow %d %s ctl=%x %d to %d method=%x src=%x dst=%x\n",
323 			i, flow, name, v,
324 			rctx->t_src[i].len, rctx->t_dst[i].len,
325 			rctx->method, rctx->t_src[i].addr, rctx->t_dst[i].addr);
326 
327 		writel(rctx->t_src[i].addr, ss->base + SS_SRC_ADR_REG);
328 		writel(rctx->t_dst[i].addr, ss->base + SS_DST_ADR_REG);
329 		writel(rctx->t_src[i].len, ss->base + SS_LEN_ADR_REG);
330 		writel(BIT(0) | BIT(1), ss->base + SS_INT_CTL_REG);
331 
332 		reinit_completion(&ss->flows[flow].complete);
333 		ss->flows[flow].status = 0;
334 		wmb();
335 
336 		writel(v, ss->base + SS_CTL_REG);
337 		mutex_unlock(&ss->mlock);
338 		wait_for_completion_interruptible_timeout(&ss->flows[flow].complete,
339 							  msecs_to_jiffies(2000));
340 		if (ss->flows[flow].status == 0) {
341 			dev_err(ss->dev, "DMA timeout for %s\n", name);
342 			return -EFAULT;
343 		}
344 	}
345 
346 	return 0;
347 }
348 
349 static bool sun8i_ss_hash_need_fallback(struct ahash_request *areq)
350 {
351 	struct crypto_ahash *tfm = crypto_ahash_reqtfm(areq);
352 	struct ahash_alg *alg = __crypto_ahash_alg(tfm->base.__crt_alg);
353 	struct sun8i_ss_alg_template *algt;
354 	struct scatterlist *sg;
355 
356 	algt = container_of(alg, struct sun8i_ss_alg_template, alg.hash);
357 
358 	if (areq->nbytes == 0) {
359 		algt->stat_fb_len++;
360 		return true;
361 	}
362 
363 	if (areq->nbytes >= MAX_PAD_SIZE - 64) {
364 		algt->stat_fb_len++;
365 		return true;
366 	}
367 
368 	/* we need to reserve one SG for the padding one */
369 	if (sg_nents(areq->src) > MAX_SG - 1) {
370 		algt->stat_fb_sgnum++;
371 		return true;
372 	}
373 
374 	sg = areq->src;
375 	while (sg) {
376 		/* SS can operate hash only on full block size
377 		 * since SS support only MD5,sha1,sha224 and sha256, blocksize
378 		 * is always 64
379 		 */
380 		/* Only the last block could be bounced to the pad buffer */
381 		if (sg->length % 64 && sg_next(sg)) {
382 			algt->stat_fb_sglen++;
383 			return true;
384 		}
385 		if (!IS_ALIGNED(sg->offset, sizeof(u32))) {
386 			algt->stat_fb_align++;
387 			return true;
388 		}
389 		if (sg->length % 4) {
390 			algt->stat_fb_sglen++;
391 			return true;
392 		}
393 		sg = sg_next(sg);
394 	}
395 	return false;
396 }
397 
398 int sun8i_ss_hash_digest(struct ahash_request *areq)
399 {
400 	struct crypto_ahash *tfm = crypto_ahash_reqtfm(areq);
401 	struct ahash_alg *alg = __crypto_ahash_alg(tfm->base.__crt_alg);
402 	struct sun8i_ss_hash_reqctx *rctx = ahash_request_ctx(areq);
403 	struct sun8i_ss_alg_template *algt;
404 	struct sun8i_ss_dev *ss;
405 	struct crypto_engine *engine;
406 	int e;
407 
408 	if (sun8i_ss_hash_need_fallback(areq))
409 		return sun8i_ss_hash_digest_fb(areq);
410 
411 	algt = container_of(alg, struct sun8i_ss_alg_template, alg.hash);
412 	ss = algt->ss;
413 
414 	e = sun8i_ss_get_engine_number(ss);
415 	rctx->flow = e;
416 	engine = ss->flows[e].engine;
417 
418 	return crypto_transfer_hash_request_to_engine(engine, areq);
419 }
420 
421 static u64 hash_pad(__le32 *buf, unsigned int bufsize, u64 padi, u64 byte_count, bool le, int bs)
422 {
423 	u64 fill, min_fill, j, k;
424 	__be64 *bebits;
425 	__le64 *lebits;
426 
427 	j = padi;
428 	buf[j++] = cpu_to_le32(0x80);
429 
430 	if (bs == 64) {
431 		fill = 64 - (byte_count % 64);
432 		min_fill = 2 * sizeof(u32) + sizeof(u32);
433 	} else {
434 		fill = 128 - (byte_count % 128);
435 		min_fill = 4 * sizeof(u32) + sizeof(u32);
436 	}
437 
438 	if (fill < min_fill)
439 		fill += bs;
440 
441 	k = j;
442 	j += (fill - min_fill) / sizeof(u32);
443 	if (j * 4 > bufsize) {
444 		pr_err("%s OVERFLOW %llu\n", __func__, j);
445 		return 0;
446 	}
447 	for (; k < j; k++)
448 		buf[k] = 0;
449 
450 	if (le) {
451 		/* MD5 */
452 		lebits = (__le64 *)&buf[j];
453 		*lebits = cpu_to_le64(byte_count << 3);
454 		j += 2;
455 	} else {
456 		if (bs == 64) {
457 			/* sha1 sha224 sha256 */
458 			bebits = (__be64 *)&buf[j];
459 			*bebits = cpu_to_be64(byte_count << 3);
460 			j += 2;
461 		} else {
462 			/* sha384 sha512*/
463 			bebits = (__be64 *)&buf[j];
464 			*bebits = cpu_to_be64(byte_count >> 61);
465 			j += 2;
466 			bebits = (__be64 *)&buf[j];
467 			*bebits = cpu_to_be64(byte_count << 3);
468 			j += 2;
469 		}
470 	}
471 	if (j * 4 > bufsize) {
472 		pr_err("%s OVERFLOW %llu\n", __func__, j);
473 		return 0;
474 	}
475 
476 	return j;
477 }
478 
479 /* sun8i_ss_hash_run - run an ahash request
480  * Send the data of the request to the SS along with an extra SG with padding
481  */
482 int sun8i_ss_hash_run(struct crypto_engine *engine, void *breq)
483 {
484 	struct ahash_request *areq = container_of(breq, struct ahash_request, base);
485 	struct crypto_ahash *tfm = crypto_ahash_reqtfm(areq);
486 	struct sun8i_ss_hash_tfm_ctx *tfmctx = crypto_ahash_ctx(tfm);
487 	struct ahash_alg *alg = __crypto_ahash_alg(tfm->base.__crt_alg);
488 	struct sun8i_ss_hash_reqctx *rctx = ahash_request_ctx(areq);
489 	struct sun8i_ss_alg_template *algt;
490 	struct sun8i_ss_dev *ss;
491 	struct scatterlist *sg;
492 	int bs = crypto_ahash_blocksize(tfm);
493 	int nr_sgs, err, digestsize;
494 	unsigned int len;
495 	u64 byte_count;
496 	void *pad, *result;
497 	int j, i, k, todo;
498 	dma_addr_t addr_res, addr_pad, addr_xpad;
499 	__le32 *bf;
500 	/* HMAC step:
501 	 * 0: normal hashing
502 	 * 1: IPAD
503 	 * 2: OPAD
504 	 */
505 	int hmac = 0;
506 
507 	algt = container_of(alg, struct sun8i_ss_alg_template, alg.hash);
508 	ss = algt->ss;
509 
510 	digestsize = algt->alg.hash.halg.digestsize;
511 	if (digestsize == SHA224_DIGEST_SIZE)
512 		digestsize = SHA256_DIGEST_SIZE;
513 
514 	result = ss->flows[rctx->flow].result;
515 	pad = ss->flows[rctx->flow].pad;
516 	bf = (__le32 *)pad;
517 
518 	for (i = 0; i < MAX_SG; i++) {
519 		rctx->t_dst[i].addr = 0;
520 		rctx->t_dst[i].len = 0;
521 	}
522 
523 #ifdef CONFIG_CRYPTO_DEV_SUN8I_SS_DEBUG
524 	algt->stat_req++;
525 #endif
526 
527 	rctx->method = ss->variant->alg_hash[algt->ss_algo_id];
528 
529 	nr_sgs = dma_map_sg(ss->dev, areq->src, sg_nents(areq->src), DMA_TO_DEVICE);
530 	if (nr_sgs <= 0 || nr_sgs > MAX_SG) {
531 		dev_err(ss->dev, "Invalid sg number %d\n", nr_sgs);
532 		err = -EINVAL;
533 		goto theend;
534 	}
535 
536 	addr_res = dma_map_single(ss->dev, result, digestsize, DMA_FROM_DEVICE);
537 	if (dma_mapping_error(ss->dev, addr_res)) {
538 		dev_err(ss->dev, "DMA map dest\n");
539 		err = -EINVAL;
540 		goto err_dma_result;
541 	}
542 
543 	j = 0;
544 	len = areq->nbytes;
545 	sg = areq->src;
546 	i = 0;
547 	while (len > 0 && sg) {
548 		if (sg_dma_len(sg) == 0) {
549 			sg = sg_next(sg);
550 			continue;
551 		}
552 		todo = min(len, sg_dma_len(sg));
553 		/* only the last SG could be with a size not modulo64 */
554 		if (todo % 64 == 0) {
555 			rctx->t_src[i].addr = sg_dma_address(sg);
556 			rctx->t_src[i].len = todo / 4;
557 			rctx->t_dst[i].addr = addr_res;
558 			rctx->t_dst[i].len = digestsize / 4;
559 			len -= todo;
560 		} else {
561 			scatterwalk_map_and_copy(bf, sg, 0, todo, 0);
562 			j += todo / 4;
563 			len -= todo;
564 		}
565 		sg = sg_next(sg);
566 		i++;
567 	}
568 	if (len > 0) {
569 		dev_err(ss->dev, "remaining len %d\n", len);
570 		err = -EINVAL;
571 		goto theend;
572 	}
573 
574 	if (j > 0)
575 		i--;
576 
577 retry:
578 	byte_count = areq->nbytes;
579 	if (tfmctx->keylen && hmac == 0) {
580 		hmac = 1;
581 		/* shift all SG one slot up, to free slot 0 for IPAD */
582 		for (k = 6; k >= 0; k--) {
583 			rctx->t_src[k + 1].addr = rctx->t_src[k].addr;
584 			rctx->t_src[k + 1].len = rctx->t_src[k].len;
585 			rctx->t_dst[k + 1].addr = rctx->t_dst[k].addr;
586 			rctx->t_dst[k + 1].len = rctx->t_dst[k].len;
587 		}
588 		addr_xpad = dma_map_single(ss->dev, tfmctx->ipad, bs, DMA_TO_DEVICE);
589 		err = dma_mapping_error(ss->dev, addr_xpad);
590 		if (err) {
591 			dev_err(ss->dev, "Fail to create DMA mapping of ipad\n");
592 			goto err_dma_xpad;
593 		}
594 		rctx->t_src[0].addr = addr_xpad;
595 		rctx->t_src[0].len = bs / 4;
596 		rctx->t_dst[0].addr = addr_res;
597 		rctx->t_dst[0].len = digestsize / 4;
598 		i++;
599 		byte_count = areq->nbytes + bs;
600 	}
601 	if (tfmctx->keylen && hmac == 2) {
602 		for (i = 0; i < MAX_SG; i++) {
603 			rctx->t_src[i].addr = 0;
604 			rctx->t_src[i].len = 0;
605 			rctx->t_dst[i].addr = 0;
606 			rctx->t_dst[i].len = 0;
607 		}
608 
609 		addr_res = dma_map_single(ss->dev, result, digestsize, DMA_FROM_DEVICE);
610 		if (dma_mapping_error(ss->dev, addr_res)) {
611 			dev_err(ss->dev, "Fail to create DMA mapping of result\n");
612 			err = -EINVAL;
613 			goto err_dma_result;
614 		}
615 		addr_xpad = dma_map_single(ss->dev, tfmctx->opad, bs, DMA_TO_DEVICE);
616 		err = dma_mapping_error(ss->dev, addr_xpad);
617 		if (err) {
618 			dev_err(ss->dev, "Fail to create DMA mapping of opad\n");
619 			goto err_dma_xpad;
620 		}
621 		rctx->t_src[0].addr = addr_xpad;
622 		rctx->t_src[0].len = bs / 4;
623 
624 		memcpy(bf, result, digestsize);
625 		j = digestsize / 4;
626 		i = 1;
627 		byte_count = digestsize + bs;
628 
629 		rctx->t_dst[0].addr = addr_res;
630 		rctx->t_dst[0].len = digestsize / 4;
631 	}
632 
633 	switch (algt->ss_algo_id) {
634 	case SS_ID_HASH_MD5:
635 		j = hash_pad(bf, 4096, j, byte_count, true, bs);
636 		break;
637 	case SS_ID_HASH_SHA1:
638 	case SS_ID_HASH_SHA224:
639 	case SS_ID_HASH_SHA256:
640 		j = hash_pad(bf, 4096, j, byte_count, false, bs);
641 		break;
642 	}
643 	if (!j) {
644 		err = -EINVAL;
645 		goto theend;
646 	}
647 
648 	addr_pad = dma_map_single(ss->dev, pad, j * 4, DMA_TO_DEVICE);
649 	if (dma_mapping_error(ss->dev, addr_pad)) {
650 		dev_err(ss->dev, "DMA error on padding SG\n");
651 		err = -EINVAL;
652 		goto err_dma_pad;
653 	}
654 	rctx->t_src[i].addr = addr_pad;
655 	rctx->t_src[i].len = j;
656 	rctx->t_dst[i].addr = addr_res;
657 	rctx->t_dst[i].len = digestsize / 4;
658 
659 	err = sun8i_ss_run_hash_task(ss, rctx, crypto_tfm_alg_name(areq->base.tfm));
660 
661 	/*
662 	 * mini helper for checking dma map/unmap
663 	 * flow start for hmac = 0 (and HMAC = 1)
664 	 * HMAC = 0
665 	 * MAP src
666 	 * MAP res
667 	 *
668 	 * retry:
669 	 * if hmac then hmac = 1
670 	 *	MAP xpad (ipad)
671 	 * if hmac == 2
672 	 *	MAP res
673 	 *	MAP xpad (opad)
674 	 * MAP pad
675 	 * ACTION!
676 	 * UNMAP pad
677 	 * if hmac
678 	 *	UNMAP xpad
679 	 * UNMAP res
680 	 * if hmac < 2
681 	 *	UNMAP SRC
682 	 *
683 	 * if hmac = 1 then hmac = 2 goto retry
684 	 */
685 
686 	dma_unmap_single(ss->dev, addr_pad, j * 4, DMA_TO_DEVICE);
687 
688 err_dma_pad:
689 	if (hmac > 0)
690 		dma_unmap_single(ss->dev, addr_xpad, bs, DMA_TO_DEVICE);
691 err_dma_xpad:
692 	dma_unmap_single(ss->dev, addr_res, digestsize, DMA_FROM_DEVICE);
693 err_dma_result:
694 	if (hmac < 2)
695 		dma_unmap_sg(ss->dev, areq->src, sg_nents(areq->src),
696 			     DMA_TO_DEVICE);
697 	if (hmac == 1 && !err) {
698 		hmac = 2;
699 		goto retry;
700 	}
701 
702 	if (!err)
703 		memcpy(areq->result, result, algt->alg.hash.halg.digestsize);
704 theend:
705 	local_bh_disable();
706 	crypto_finalize_hash_request(engine, breq, err);
707 	local_bh_enable();
708 	return 0;
709 }
710