1 /*
2  * Copyright (C) 2017 Marvell
3  *
4  * Antoine Tenart <antoine.tenart@free-electrons.com>
5  *
6  * This file is licensed under the terms of the GNU General Public
7  * License version 2. This program is licensed "as is" without any
8  * warranty of any kind, whether express or implied.
9  */
10 
11 #include <crypto/hmac.h>
12 #include <crypto/sha.h>
13 #include <linux/device.h>
14 #include <linux/dma-mapping.h>
15 #include <linux/dmapool.h>
16 
17 
18 #include "safexcel.h"
19 
20 struct safexcel_ahash_ctx {
21 	struct safexcel_context base;
22 	struct safexcel_crypto_priv *priv;
23 
24 	u32 alg;
25 	u32 digest;
26 
27 	u32 ipad[SHA1_DIGEST_SIZE / sizeof(u32)];
28 	u32 opad[SHA1_DIGEST_SIZE / sizeof(u32)];
29 };
30 
31 struct safexcel_ahash_req {
32 	bool last_req;
33 	bool finish;
34 	bool hmac;
35 
36 	u8 state_sz;    /* expected sate size, only set once */
37 	u32 state[SHA256_DIGEST_SIZE / sizeof(u32)];
38 
39 	u64 len;
40 	u64 processed;
41 
42 	u8 cache[SHA256_BLOCK_SIZE] __aligned(sizeof(u32));
43 	u8 cache_next[SHA256_BLOCK_SIZE] __aligned(sizeof(u32));
44 };
45 
46 struct safexcel_ahash_export_state {
47 	u64 len;
48 	u64 processed;
49 
50 	u32 state[SHA256_DIGEST_SIZE / sizeof(u32)];
51 	u8 cache[SHA256_BLOCK_SIZE];
52 };
53 
54 static void safexcel_hash_token(struct safexcel_command_desc *cdesc,
55 				u32 input_length, u32 result_length)
56 {
57 	struct safexcel_token *token =
58 		(struct safexcel_token *)cdesc->control_data.token;
59 
60 	token[0].opcode = EIP197_TOKEN_OPCODE_DIRECTION;
61 	token[0].packet_length = input_length;
62 	token[0].stat = EIP197_TOKEN_STAT_LAST_HASH;
63 	token[0].instructions = EIP197_TOKEN_INS_TYPE_HASH;
64 
65 	token[1].opcode = EIP197_TOKEN_OPCODE_INSERT;
66 	token[1].packet_length = result_length;
67 	token[1].stat = EIP197_TOKEN_STAT_LAST_HASH |
68 			EIP197_TOKEN_STAT_LAST_PACKET;
69 	token[1].instructions = EIP197_TOKEN_INS_TYPE_OUTPUT |
70 				EIP197_TOKEN_INS_INSERT_HASH_DIGEST;
71 }
72 
73 static void safexcel_context_control(struct safexcel_ahash_ctx *ctx,
74 				     struct safexcel_ahash_req *req,
75 				     struct safexcel_command_desc *cdesc,
76 				     unsigned int digestsize,
77 				     unsigned int blocksize)
78 {
79 	int i;
80 
81 	cdesc->control_data.control0 |= CONTEXT_CONTROL_TYPE_HASH_OUT;
82 	cdesc->control_data.control0 |= ctx->alg;
83 	cdesc->control_data.control0 |= ctx->digest;
84 
85 	if (ctx->digest == CONTEXT_CONTROL_DIGEST_PRECOMPUTED) {
86 		if (req->processed) {
87 			if (ctx->alg == CONTEXT_CONTROL_CRYPTO_ALG_SHA1)
88 				cdesc->control_data.control0 |= CONTEXT_CONTROL_SIZE(6);
89 			else if (ctx->alg == CONTEXT_CONTROL_CRYPTO_ALG_SHA224 ||
90 				 ctx->alg == CONTEXT_CONTROL_CRYPTO_ALG_SHA256)
91 				cdesc->control_data.control0 |= CONTEXT_CONTROL_SIZE(9);
92 
93 			cdesc->control_data.control1 |= CONTEXT_CONTROL_DIGEST_CNT;
94 		} else {
95 			cdesc->control_data.control0 |= CONTEXT_CONTROL_RESTART_HASH;
96 		}
97 
98 		if (!req->finish)
99 			cdesc->control_data.control0 |= CONTEXT_CONTROL_NO_FINISH_HASH;
100 
101 		/*
102 		 * Copy the input digest if needed, and setup the context
103 		 * fields. Do this now as we need it to setup the first command
104 		 * descriptor.
105 		 */
106 		if (req->processed) {
107 			for (i = 0; i < digestsize / sizeof(u32); i++)
108 				ctx->base.ctxr->data[i] = cpu_to_le32(req->state[i]);
109 
110 			if (req->finish)
111 				ctx->base.ctxr->data[i] = cpu_to_le32(req->processed / blocksize);
112 		}
113 	} else if (ctx->digest == CONTEXT_CONTROL_DIGEST_HMAC) {
114 		cdesc->control_data.control0 |= CONTEXT_CONTROL_SIZE(10);
115 
116 		memcpy(ctx->base.ctxr->data, ctx->ipad, digestsize);
117 		memcpy(ctx->base.ctxr->data + digestsize / sizeof(u32),
118 		       ctx->opad, digestsize);
119 	}
120 }
121 
122 static int safexcel_handle_result(struct safexcel_crypto_priv *priv, int ring,
123 				  struct crypto_async_request *async,
124 				  bool *should_complete, int *ret)
125 {
126 	struct safexcel_result_desc *rdesc;
127 	struct ahash_request *areq = ahash_request_cast(async);
128 	struct crypto_ahash *ahash = crypto_ahash_reqtfm(areq);
129 	struct safexcel_ahash_req *sreq = ahash_request_ctx(areq);
130 	int cache_len, result_sz = sreq->state_sz;
131 
132 	*ret = 0;
133 
134 	spin_lock_bh(&priv->ring[ring].egress_lock);
135 	rdesc = safexcel_ring_next_rptr(priv, &priv->ring[ring].rdr);
136 	if (IS_ERR(rdesc)) {
137 		dev_err(priv->dev,
138 			"hash: result: could not retrieve the result descriptor\n");
139 		*ret = PTR_ERR(rdesc);
140 	} else if (rdesc->result_data.error_code) {
141 		dev_err(priv->dev,
142 			"hash: result: result descriptor error (%d)\n",
143 			rdesc->result_data.error_code);
144 		*ret = -EINVAL;
145 	}
146 
147 	safexcel_complete(priv, ring);
148 	spin_unlock_bh(&priv->ring[ring].egress_lock);
149 
150 	if (sreq->finish)
151 		result_sz = crypto_ahash_digestsize(ahash);
152 	memcpy(sreq->state, areq->result, result_sz);
153 
154 	dma_unmap_sg(priv->dev, areq->src,
155 		     sg_nents_for_len(areq->src, areq->nbytes), DMA_TO_DEVICE);
156 
157 	safexcel_free_context(priv, async, sreq->state_sz);
158 
159 	cache_len = sreq->len - sreq->processed;
160 	if (cache_len)
161 		memcpy(sreq->cache, sreq->cache_next, cache_len);
162 
163 	*should_complete = true;
164 
165 	return 1;
166 }
167 
168 static int safexcel_ahash_send(struct crypto_async_request *async, int ring,
169 			       struct safexcel_request *request, int *commands,
170 			       int *results)
171 {
172 	struct ahash_request *areq = ahash_request_cast(async);
173 	struct crypto_ahash *ahash = crypto_ahash_reqtfm(areq);
174 	struct safexcel_ahash_req *req = ahash_request_ctx(areq);
175 	struct safexcel_ahash_ctx *ctx = crypto_ahash_ctx(crypto_ahash_reqtfm(areq));
176 	struct safexcel_crypto_priv *priv = ctx->priv;
177 	struct safexcel_command_desc *cdesc, *first_cdesc = NULL;
178 	struct safexcel_result_desc *rdesc;
179 	struct scatterlist *sg;
180 	int i, nents, queued, len, cache_len, extra, n_cdesc = 0, ret = 0;
181 
182 	queued = len = req->len - req->processed;
183 	if (queued < crypto_ahash_blocksize(ahash))
184 		cache_len = queued;
185 	else
186 		cache_len = queued - areq->nbytes;
187 
188 	/*
189 	 * If this is not the last request and the queued data does not fit
190 	 * into full blocks, cache it for the next send() call.
191 	 */
192 	extra = queued & (crypto_ahash_blocksize(ahash) - 1);
193 	if (!req->last_req && extra) {
194 		sg_pcopy_to_buffer(areq->src, sg_nents(areq->src),
195 				   req->cache_next, extra, areq->nbytes - extra);
196 
197 		queued -= extra;
198 		len -= extra;
199 	}
200 
201 	spin_lock_bh(&priv->ring[ring].egress_lock);
202 
203 	/* Add a command descriptor for the cached data, if any */
204 	if (cache_len) {
205 		ctx->base.cache = kzalloc(cache_len, EIP197_GFP_FLAGS(*async));
206 		if (!ctx->base.cache) {
207 			ret = -ENOMEM;
208 			goto unlock;
209 		}
210 		memcpy(ctx->base.cache, req->cache, cache_len);
211 		ctx->base.cache_dma = dma_map_single(priv->dev, ctx->base.cache,
212 						     cache_len, DMA_TO_DEVICE);
213 		if (dma_mapping_error(priv->dev, ctx->base.cache_dma)) {
214 			ret = -EINVAL;
215 			goto free_cache;
216 		}
217 
218 		ctx->base.cache_sz = cache_len;
219 		first_cdesc = safexcel_add_cdesc(priv, ring, 1,
220 						 (cache_len == len),
221 						 ctx->base.cache_dma,
222 						 cache_len, len,
223 						 ctx->base.ctxr_dma);
224 		if (IS_ERR(first_cdesc)) {
225 			ret = PTR_ERR(first_cdesc);
226 			goto unmap_cache;
227 		}
228 		n_cdesc++;
229 
230 		queued -= cache_len;
231 		if (!queued)
232 			goto send_command;
233 	}
234 
235 	/* Now handle the current ahash request buffer(s) */
236 	nents = dma_map_sg(priv->dev, areq->src,
237 		       sg_nents_for_len(areq->src, areq->nbytes),
238 		       DMA_TO_DEVICE);
239 	if (!nents) {
240 		ret = -ENOMEM;
241 		goto cdesc_rollback;
242 	}
243 
244 	for_each_sg(areq->src, sg, nents, i) {
245 		int sglen = sg_dma_len(sg);
246 
247 		/* Do not overflow the request */
248 		if (queued - sglen < 0)
249 			sglen = queued;
250 
251 		cdesc = safexcel_add_cdesc(priv, ring, !n_cdesc,
252 					   !(queued - sglen), sg_dma_address(sg),
253 					   sglen, len, ctx->base.ctxr_dma);
254 		if (IS_ERR(cdesc)) {
255 			ret = PTR_ERR(cdesc);
256 			goto cdesc_rollback;
257 		}
258 		n_cdesc++;
259 
260 		if (n_cdesc == 1)
261 			first_cdesc = cdesc;
262 
263 		queued -= sglen;
264 		if (!queued)
265 			break;
266 	}
267 
268 send_command:
269 	/* Setup the context options */
270 	safexcel_context_control(ctx, req, first_cdesc, req->state_sz,
271 				 crypto_ahash_blocksize(ahash));
272 
273 	/* Add the token */
274 	safexcel_hash_token(first_cdesc, len, req->state_sz);
275 
276 	ctx->base.result_dma = dma_map_single(priv->dev, areq->result,
277 					      req->state_sz, DMA_FROM_DEVICE);
278 	if (dma_mapping_error(priv->dev, ctx->base.result_dma)) {
279 		ret = -EINVAL;
280 		goto cdesc_rollback;
281 	}
282 
283 	/* Add a result descriptor */
284 	rdesc = safexcel_add_rdesc(priv, ring, 1, 1, ctx->base.result_dma,
285 				   req->state_sz);
286 	if (IS_ERR(rdesc)) {
287 		ret = PTR_ERR(rdesc);
288 		goto cdesc_rollback;
289 	}
290 
291 	spin_unlock_bh(&priv->ring[ring].egress_lock);
292 
293 	req->processed += len;
294 	request->req = &areq->base;
295 	ctx->base.handle_result = safexcel_handle_result;
296 
297 	*commands = n_cdesc;
298 	*results = 1;
299 	return 0;
300 
301 cdesc_rollback:
302 	for (i = 0; i < n_cdesc; i++)
303 		safexcel_ring_rollback_wptr(priv, &priv->ring[ring].cdr);
304 unmap_cache:
305 	if (ctx->base.cache_dma) {
306 		dma_unmap_single(priv->dev, ctx->base.cache_dma,
307 				 ctx->base.cache_sz, DMA_TO_DEVICE);
308 		ctx->base.cache_sz = 0;
309 	}
310 free_cache:
311 	if (ctx->base.cache) {
312 		kfree(ctx->base.cache);
313 		ctx->base.cache = NULL;
314 	}
315 
316 unlock:
317 	spin_unlock_bh(&priv->ring[ring].egress_lock);
318 	return ret;
319 }
320 
321 static inline bool safexcel_ahash_needs_inv_get(struct ahash_request *areq)
322 {
323 	struct safexcel_ahash_ctx *ctx = crypto_ahash_ctx(crypto_ahash_reqtfm(areq));
324 	struct safexcel_ahash_req *req = ahash_request_ctx(areq);
325 	struct crypto_ahash *ahash = crypto_ahash_reqtfm(areq);
326 	unsigned int state_w_sz = req->state_sz / sizeof(u32);
327 	int i;
328 
329 	for (i = 0; i < state_w_sz; i++)
330 		if (ctx->base.ctxr->data[i] != cpu_to_le32(req->state[i]))
331 			return true;
332 
333 	if (ctx->base.ctxr->data[state_w_sz] !=
334 	    cpu_to_le32(req->processed / crypto_ahash_blocksize(ahash)))
335 		return true;
336 
337 	return false;
338 }
339 
340 static int safexcel_handle_inv_result(struct safexcel_crypto_priv *priv,
341 				      int ring,
342 				      struct crypto_async_request *async,
343 				      bool *should_complete, int *ret)
344 {
345 	struct safexcel_result_desc *rdesc;
346 	struct ahash_request *areq = ahash_request_cast(async);
347 	struct crypto_ahash *ahash = crypto_ahash_reqtfm(areq);
348 	struct safexcel_ahash_ctx *ctx = crypto_ahash_ctx(ahash);
349 	int enq_ret;
350 
351 	*ret = 0;
352 
353 	spin_lock_bh(&priv->ring[ring].egress_lock);
354 	rdesc = safexcel_ring_next_rptr(priv, &priv->ring[ring].rdr);
355 	if (IS_ERR(rdesc)) {
356 		dev_err(priv->dev,
357 			"hash: invalidate: could not retrieve the result descriptor\n");
358 		*ret = PTR_ERR(rdesc);
359 	} else if (rdesc->result_data.error_code) {
360 		dev_err(priv->dev,
361 			"hash: invalidate: result descriptor error (%d)\n",
362 			rdesc->result_data.error_code);
363 		*ret = -EINVAL;
364 	}
365 
366 	safexcel_complete(priv, ring);
367 	spin_unlock_bh(&priv->ring[ring].egress_lock);
368 
369 	if (ctx->base.exit_inv) {
370 		dma_pool_free(priv->context_pool, ctx->base.ctxr,
371 			      ctx->base.ctxr_dma);
372 
373 		*should_complete = true;
374 		return 1;
375 	}
376 
377 	ring = safexcel_select_ring(priv);
378 	ctx->base.ring = ring;
379 	ctx->base.needs_inv = false;
380 	ctx->base.send = safexcel_ahash_send;
381 
382 	spin_lock_bh(&priv->ring[ring].queue_lock);
383 	enq_ret = crypto_enqueue_request(&priv->ring[ring].queue, async);
384 	spin_unlock_bh(&priv->ring[ring].queue_lock);
385 
386 	if (enq_ret != -EINPROGRESS)
387 		*ret = enq_ret;
388 
389 	if (!priv->ring[ring].need_dequeue)
390 		safexcel_dequeue(priv, ring);
391 
392 	*should_complete = false;
393 
394 	return 1;
395 }
396 
397 static int safexcel_ahash_send_inv(struct crypto_async_request *async,
398 				   int ring, struct safexcel_request *request,
399 				   int *commands, int *results)
400 {
401 	struct ahash_request *areq = ahash_request_cast(async);
402 	struct safexcel_ahash_ctx *ctx = crypto_ahash_ctx(crypto_ahash_reqtfm(areq));
403 	int ret;
404 
405 	ctx->base.handle_result = safexcel_handle_inv_result;
406 	ret = safexcel_invalidate_cache(async, &ctx->base, ctx->priv,
407 					ctx->base.ctxr_dma, ring, request);
408 	if (unlikely(ret))
409 		return ret;
410 
411 	*commands = 1;
412 	*results = 1;
413 
414 	return 0;
415 }
416 
417 static int safexcel_ahash_exit_inv(struct crypto_tfm *tfm)
418 {
419 	struct safexcel_ahash_ctx *ctx = crypto_tfm_ctx(tfm);
420 	struct safexcel_crypto_priv *priv = ctx->priv;
421 	struct ahash_request req;
422 	struct safexcel_inv_result result = { 0 };
423 	int ring = ctx->base.ring;
424 
425 	memset(&req, 0, sizeof(struct ahash_request));
426 
427 	/* create invalidation request */
428 	init_completion(&result.completion);
429 	ahash_request_set_callback(&req, CRYPTO_TFM_REQ_MAY_BACKLOG,
430 				   safexcel_inv_complete, &result);
431 
432 	ahash_request_set_tfm(&req, __crypto_ahash_cast(tfm));
433 	ctx = crypto_tfm_ctx(req.base.tfm);
434 	ctx->base.exit_inv = true;
435 	ctx->base.send = safexcel_ahash_send_inv;
436 
437 	spin_lock_bh(&priv->ring[ring].queue_lock);
438 	crypto_enqueue_request(&priv->ring[ring].queue, &req.base);
439 	spin_unlock_bh(&priv->ring[ring].queue_lock);
440 
441 	if (!priv->ring[ring].need_dequeue)
442 		safexcel_dequeue(priv, ring);
443 
444 	wait_for_completion_interruptible(&result.completion);
445 
446 	if (result.error) {
447 		dev_warn(priv->dev, "hash: completion error (%d)\n",
448 			 result.error);
449 		return result.error;
450 	}
451 
452 	return 0;
453 }
454 
455 static int safexcel_ahash_cache(struct ahash_request *areq)
456 {
457 	struct safexcel_ahash_req *req = ahash_request_ctx(areq);
458 	struct crypto_ahash *ahash = crypto_ahash_reqtfm(areq);
459 	int queued, cache_len;
460 
461 	cache_len = req->len - areq->nbytes - req->processed;
462 	queued = req->len - req->processed;
463 
464 	/*
465 	 * In case there isn't enough bytes to proceed (less than a
466 	 * block size), cache the data until we have enough.
467 	 */
468 	if (cache_len + areq->nbytes <= crypto_ahash_blocksize(ahash)) {
469 		sg_pcopy_to_buffer(areq->src, sg_nents(areq->src),
470 				   req->cache + cache_len,
471 				   areq->nbytes, 0);
472 		return areq->nbytes;
473 	}
474 
475 	/* We could'nt cache all the data */
476 	return -E2BIG;
477 }
478 
479 static int safexcel_ahash_enqueue(struct ahash_request *areq)
480 {
481 	struct safexcel_ahash_ctx *ctx = crypto_ahash_ctx(crypto_ahash_reqtfm(areq));
482 	struct safexcel_ahash_req *req = ahash_request_ctx(areq);
483 	struct safexcel_crypto_priv *priv = ctx->priv;
484 	int ret, ring;
485 
486 	ctx->base.send = safexcel_ahash_send;
487 
488 	if (req->processed && ctx->digest == CONTEXT_CONTROL_DIGEST_PRECOMPUTED)
489 		ctx->base.needs_inv = safexcel_ahash_needs_inv_get(areq);
490 
491 	if (ctx->base.ctxr) {
492 		if (ctx->base.needs_inv)
493 			ctx->base.send = safexcel_ahash_send_inv;
494 	} else {
495 		ctx->base.ring = safexcel_select_ring(priv);
496 		ctx->base.ctxr = dma_pool_zalloc(priv->context_pool,
497 						 EIP197_GFP_FLAGS(areq->base),
498 						 &ctx->base.ctxr_dma);
499 		if (!ctx->base.ctxr)
500 			return -ENOMEM;
501 	}
502 
503 	ring = ctx->base.ring;
504 
505 	spin_lock_bh(&priv->ring[ring].queue_lock);
506 	ret = crypto_enqueue_request(&priv->ring[ring].queue, &areq->base);
507 	spin_unlock_bh(&priv->ring[ring].queue_lock);
508 
509 	if (!priv->ring[ring].need_dequeue)
510 		safexcel_dequeue(priv, ring);
511 
512 	return ret;
513 }
514 
515 static int safexcel_ahash_update(struct ahash_request *areq)
516 {
517 	struct safexcel_ahash_ctx *ctx = crypto_ahash_ctx(crypto_ahash_reqtfm(areq));
518 	struct safexcel_ahash_req *req = ahash_request_ctx(areq);
519 	struct crypto_ahash *ahash = crypto_ahash_reqtfm(areq);
520 
521 	/* If the request is 0 length, do nothing */
522 	if (!areq->nbytes)
523 		return 0;
524 
525 	req->len += areq->nbytes;
526 
527 	safexcel_ahash_cache(areq);
528 
529 	/*
530 	 * We're not doing partial updates when performing an hmac request.
531 	 * Everything will be handled by the final() call.
532 	 */
533 	if (ctx->digest == CONTEXT_CONTROL_DIGEST_HMAC)
534 		return 0;
535 
536 	if (req->hmac)
537 		return safexcel_ahash_enqueue(areq);
538 
539 	if (!req->last_req &&
540 	    req->len - req->processed > crypto_ahash_blocksize(ahash))
541 		return safexcel_ahash_enqueue(areq);
542 
543 	return 0;
544 }
545 
546 static int safexcel_ahash_final(struct ahash_request *areq)
547 {
548 	struct safexcel_ahash_req *req = ahash_request_ctx(areq);
549 	struct safexcel_ahash_ctx *ctx = crypto_ahash_ctx(crypto_ahash_reqtfm(areq));
550 
551 	req->last_req = true;
552 	req->finish = true;
553 
554 	/* If we have an overall 0 length request */
555 	if (!(req->len + areq->nbytes)) {
556 		if (ctx->alg == CONTEXT_CONTROL_CRYPTO_ALG_SHA1)
557 			memcpy(areq->result, sha1_zero_message_hash,
558 			       SHA1_DIGEST_SIZE);
559 		else if (ctx->alg == CONTEXT_CONTROL_CRYPTO_ALG_SHA224)
560 			memcpy(areq->result, sha224_zero_message_hash,
561 			       SHA224_DIGEST_SIZE);
562 		else if (ctx->alg == CONTEXT_CONTROL_CRYPTO_ALG_SHA256)
563 			memcpy(areq->result, sha256_zero_message_hash,
564 			       SHA256_DIGEST_SIZE);
565 
566 		return 0;
567 	}
568 
569 	return safexcel_ahash_enqueue(areq);
570 }
571 
572 static int safexcel_ahash_finup(struct ahash_request *areq)
573 {
574 	struct safexcel_ahash_req *req = ahash_request_ctx(areq);
575 
576 	req->last_req = true;
577 	req->finish = true;
578 
579 	safexcel_ahash_update(areq);
580 	return safexcel_ahash_final(areq);
581 }
582 
583 static int safexcel_ahash_export(struct ahash_request *areq, void *out)
584 {
585 	struct crypto_ahash *ahash = crypto_ahash_reqtfm(areq);
586 	struct safexcel_ahash_req *req = ahash_request_ctx(areq);
587 	struct safexcel_ahash_export_state *export = out;
588 
589 	export->len = req->len;
590 	export->processed = req->processed;
591 
592 	memcpy(export->state, req->state, req->state_sz);
593 	memset(export->cache, 0, crypto_ahash_blocksize(ahash));
594 	memcpy(export->cache, req->cache, crypto_ahash_blocksize(ahash));
595 
596 	return 0;
597 }
598 
599 static int safexcel_ahash_import(struct ahash_request *areq, const void *in)
600 {
601 	struct crypto_ahash *ahash = crypto_ahash_reqtfm(areq);
602 	struct safexcel_ahash_req *req = ahash_request_ctx(areq);
603 	const struct safexcel_ahash_export_state *export = in;
604 	int ret;
605 
606 	ret = crypto_ahash_init(areq);
607 	if (ret)
608 		return ret;
609 
610 	req->len = export->len;
611 	req->processed = export->processed;
612 
613 	memcpy(req->cache, export->cache, crypto_ahash_blocksize(ahash));
614 	memcpy(req->state, export->state, req->state_sz);
615 
616 	return 0;
617 }
618 
619 static int safexcel_ahash_cra_init(struct crypto_tfm *tfm)
620 {
621 	struct safexcel_ahash_ctx *ctx = crypto_tfm_ctx(tfm);
622 	struct safexcel_alg_template *tmpl =
623 		container_of(__crypto_ahash_alg(tfm->__crt_alg),
624 			     struct safexcel_alg_template, alg.ahash);
625 
626 	ctx->priv = tmpl->priv;
627 
628 	crypto_ahash_set_reqsize(__crypto_ahash_cast(tfm),
629 				 sizeof(struct safexcel_ahash_req));
630 	return 0;
631 }
632 
633 static int safexcel_sha1_init(struct ahash_request *areq)
634 {
635 	struct safexcel_ahash_ctx *ctx = crypto_ahash_ctx(crypto_ahash_reqtfm(areq));
636 	struct safexcel_ahash_req *req = ahash_request_ctx(areq);
637 
638 	memset(req, 0, sizeof(*req));
639 
640 	req->state[0] = SHA1_H0;
641 	req->state[1] = SHA1_H1;
642 	req->state[2] = SHA1_H2;
643 	req->state[3] = SHA1_H3;
644 	req->state[4] = SHA1_H4;
645 
646 	ctx->alg = CONTEXT_CONTROL_CRYPTO_ALG_SHA1;
647 	ctx->digest = CONTEXT_CONTROL_DIGEST_PRECOMPUTED;
648 	req->state_sz = SHA1_DIGEST_SIZE;
649 
650 	return 0;
651 }
652 
653 static int safexcel_sha1_digest(struct ahash_request *areq)
654 {
655 	int ret = safexcel_sha1_init(areq);
656 
657 	if (ret)
658 		return ret;
659 
660 	return safexcel_ahash_finup(areq);
661 }
662 
663 static void safexcel_ahash_cra_exit(struct crypto_tfm *tfm)
664 {
665 	struct safexcel_ahash_ctx *ctx = crypto_tfm_ctx(tfm);
666 	struct safexcel_crypto_priv *priv = ctx->priv;
667 	int ret;
668 
669 	/* context not allocated, skip invalidation */
670 	if (!ctx->base.ctxr)
671 		return;
672 
673 	ret = safexcel_ahash_exit_inv(tfm);
674 	if (ret)
675 		dev_warn(priv->dev, "hash: invalidation error %d\n", ret);
676 }
677 
678 struct safexcel_alg_template safexcel_alg_sha1 = {
679 	.type = SAFEXCEL_ALG_TYPE_AHASH,
680 	.alg.ahash = {
681 		.init = safexcel_sha1_init,
682 		.update = safexcel_ahash_update,
683 		.final = safexcel_ahash_final,
684 		.finup = safexcel_ahash_finup,
685 		.digest = safexcel_sha1_digest,
686 		.export = safexcel_ahash_export,
687 		.import = safexcel_ahash_import,
688 		.halg = {
689 			.digestsize = SHA1_DIGEST_SIZE,
690 			.statesize = sizeof(struct safexcel_ahash_export_state),
691 			.base = {
692 				.cra_name = "sha1",
693 				.cra_driver_name = "safexcel-sha1",
694 				.cra_priority = 300,
695 				.cra_flags = CRYPTO_ALG_ASYNC |
696 					     CRYPTO_ALG_KERN_DRIVER_ONLY,
697 				.cra_blocksize = SHA1_BLOCK_SIZE,
698 				.cra_ctxsize = sizeof(struct safexcel_ahash_ctx),
699 				.cra_init = safexcel_ahash_cra_init,
700 				.cra_exit = safexcel_ahash_cra_exit,
701 				.cra_module = THIS_MODULE,
702 			},
703 		},
704 	},
705 };
706 
707 static int safexcel_hmac_sha1_init(struct ahash_request *areq)
708 {
709 	struct safexcel_ahash_ctx *ctx = crypto_ahash_ctx(crypto_ahash_reqtfm(areq));
710 
711 	safexcel_sha1_init(areq);
712 	ctx->digest = CONTEXT_CONTROL_DIGEST_HMAC;
713 	return 0;
714 }
715 
716 static int safexcel_hmac_sha1_digest(struct ahash_request *areq)
717 {
718 	int ret = safexcel_hmac_sha1_init(areq);
719 
720 	if (ret)
721 		return ret;
722 
723 	return safexcel_ahash_finup(areq);
724 }
725 
726 struct safexcel_ahash_result {
727 	struct completion completion;
728 	int error;
729 };
730 
731 static void safexcel_ahash_complete(struct crypto_async_request *req, int error)
732 {
733 	struct safexcel_ahash_result *result = req->data;
734 
735 	if (error == -EINPROGRESS)
736 		return;
737 
738 	result->error = error;
739 	complete(&result->completion);
740 }
741 
742 static int safexcel_hmac_init_pad(struct ahash_request *areq,
743 				  unsigned int blocksize, const u8 *key,
744 				  unsigned int keylen, u8 *ipad, u8 *opad)
745 {
746 	struct safexcel_ahash_result result;
747 	struct scatterlist sg;
748 	int ret, i;
749 	u8 *keydup;
750 
751 	if (keylen <= blocksize) {
752 		memcpy(ipad, key, keylen);
753 	} else {
754 		keydup = kmemdup(key, keylen, GFP_KERNEL);
755 		if (!keydup)
756 			return -ENOMEM;
757 
758 		ahash_request_set_callback(areq, CRYPTO_TFM_REQ_MAY_BACKLOG,
759 					   safexcel_ahash_complete, &result);
760 		sg_init_one(&sg, keydup, keylen);
761 		ahash_request_set_crypt(areq, &sg, ipad, keylen);
762 		init_completion(&result.completion);
763 
764 		ret = crypto_ahash_digest(areq);
765 		if (ret == -EINPROGRESS) {
766 			wait_for_completion_interruptible(&result.completion);
767 			ret = result.error;
768 		}
769 
770 		/* Avoid leaking */
771 		memzero_explicit(keydup, keylen);
772 		kfree(keydup);
773 
774 		if (ret)
775 			return ret;
776 
777 		keylen = crypto_ahash_digestsize(crypto_ahash_reqtfm(areq));
778 	}
779 
780 	memset(ipad + keylen, 0, blocksize - keylen);
781 	memcpy(opad, ipad, blocksize);
782 
783 	for (i = 0; i < blocksize; i++) {
784 		ipad[i] ^= HMAC_IPAD_VALUE;
785 		opad[i] ^= HMAC_OPAD_VALUE;
786 	}
787 
788 	return 0;
789 }
790 
791 static int safexcel_hmac_init_iv(struct ahash_request *areq,
792 				 unsigned int blocksize, u8 *pad, void *state)
793 {
794 	struct safexcel_ahash_result result;
795 	struct safexcel_ahash_req *req;
796 	struct scatterlist sg;
797 	int ret;
798 
799 	ahash_request_set_callback(areq, CRYPTO_TFM_REQ_MAY_BACKLOG,
800 				   safexcel_ahash_complete, &result);
801 	sg_init_one(&sg, pad, blocksize);
802 	ahash_request_set_crypt(areq, &sg, pad, blocksize);
803 	init_completion(&result.completion);
804 
805 	ret = crypto_ahash_init(areq);
806 	if (ret)
807 		return ret;
808 
809 	req = ahash_request_ctx(areq);
810 	req->hmac = true;
811 	req->last_req = true;
812 
813 	ret = crypto_ahash_update(areq);
814 	if (ret && ret != -EINPROGRESS)
815 		return ret;
816 
817 	wait_for_completion_interruptible(&result.completion);
818 	if (result.error)
819 		return result.error;
820 
821 	return crypto_ahash_export(areq, state);
822 }
823 
824 static int safexcel_hmac_setkey(const char *alg, const u8 *key,
825 				unsigned int keylen, void *istate, void *ostate)
826 {
827 	struct ahash_request *areq;
828 	struct crypto_ahash *tfm;
829 	unsigned int blocksize;
830 	u8 *ipad, *opad;
831 	int ret;
832 
833 	tfm = crypto_alloc_ahash(alg, CRYPTO_ALG_TYPE_AHASH,
834 				 CRYPTO_ALG_TYPE_AHASH_MASK);
835 	if (IS_ERR(tfm))
836 		return PTR_ERR(tfm);
837 
838 	areq = ahash_request_alloc(tfm, GFP_KERNEL);
839 	if (!areq) {
840 		ret = -ENOMEM;
841 		goto free_ahash;
842 	}
843 
844 	crypto_ahash_clear_flags(tfm, ~0);
845 	blocksize = crypto_tfm_alg_blocksize(crypto_ahash_tfm(tfm));
846 
847 	ipad = kzalloc(2 * blocksize, GFP_KERNEL);
848 	if (!ipad) {
849 		ret = -ENOMEM;
850 		goto free_request;
851 	}
852 
853 	opad = ipad + blocksize;
854 
855 	ret = safexcel_hmac_init_pad(areq, blocksize, key, keylen, ipad, opad);
856 	if (ret)
857 		goto free_ipad;
858 
859 	ret = safexcel_hmac_init_iv(areq, blocksize, ipad, istate);
860 	if (ret)
861 		goto free_ipad;
862 
863 	ret = safexcel_hmac_init_iv(areq, blocksize, opad, ostate);
864 
865 free_ipad:
866 	kfree(ipad);
867 free_request:
868 	ahash_request_free(areq);
869 free_ahash:
870 	crypto_free_ahash(tfm);
871 
872 	return ret;
873 }
874 
875 static int safexcel_hmac_sha1_setkey(struct crypto_ahash *tfm, const u8 *key,
876 				     unsigned int keylen)
877 {
878 	struct safexcel_ahash_ctx *ctx = crypto_tfm_ctx(crypto_ahash_tfm(tfm));
879 	struct safexcel_ahash_export_state istate, ostate;
880 	int ret, i;
881 
882 	ret = safexcel_hmac_setkey("safexcel-sha1", key, keylen, &istate, &ostate);
883 	if (ret)
884 		return ret;
885 
886 	memcpy(ctx->ipad, &istate.state, SHA1_DIGEST_SIZE);
887 	memcpy(ctx->opad, &ostate.state, SHA1_DIGEST_SIZE);
888 
889 	for (i = 0; i < ARRAY_SIZE(istate.state); i++) {
890 		if (ctx->ipad[i] != le32_to_cpu(istate.state[i]) ||
891 		    ctx->opad[i] != le32_to_cpu(ostate.state[i])) {
892 			ctx->base.needs_inv = true;
893 			break;
894 		}
895 	}
896 
897 	return 0;
898 }
899 
900 struct safexcel_alg_template safexcel_alg_hmac_sha1 = {
901 	.type = SAFEXCEL_ALG_TYPE_AHASH,
902 	.alg.ahash = {
903 		.init = safexcel_hmac_sha1_init,
904 		.update = safexcel_ahash_update,
905 		.final = safexcel_ahash_final,
906 		.finup = safexcel_ahash_finup,
907 		.digest = safexcel_hmac_sha1_digest,
908 		.setkey = safexcel_hmac_sha1_setkey,
909 		.export = safexcel_ahash_export,
910 		.import = safexcel_ahash_import,
911 		.halg = {
912 			.digestsize = SHA1_DIGEST_SIZE,
913 			.statesize = sizeof(struct safexcel_ahash_export_state),
914 			.base = {
915 				.cra_name = "hmac(sha1)",
916 				.cra_driver_name = "safexcel-hmac-sha1",
917 				.cra_priority = 300,
918 				.cra_flags = CRYPTO_ALG_ASYNC |
919 					     CRYPTO_ALG_KERN_DRIVER_ONLY,
920 				.cra_blocksize = SHA1_BLOCK_SIZE,
921 				.cra_ctxsize = sizeof(struct safexcel_ahash_ctx),
922 				.cra_init = safexcel_ahash_cra_init,
923 				.cra_exit = safexcel_ahash_cra_exit,
924 				.cra_module = THIS_MODULE,
925 			},
926 		},
927 	},
928 };
929 
930 static int safexcel_sha256_init(struct ahash_request *areq)
931 {
932 	struct safexcel_ahash_ctx *ctx = crypto_ahash_ctx(crypto_ahash_reqtfm(areq));
933 	struct safexcel_ahash_req *req = ahash_request_ctx(areq);
934 
935 	memset(req, 0, sizeof(*req));
936 
937 	req->state[0] = SHA256_H0;
938 	req->state[1] = SHA256_H1;
939 	req->state[2] = SHA256_H2;
940 	req->state[3] = SHA256_H3;
941 	req->state[4] = SHA256_H4;
942 	req->state[5] = SHA256_H5;
943 	req->state[6] = SHA256_H6;
944 	req->state[7] = SHA256_H7;
945 
946 	ctx->alg = CONTEXT_CONTROL_CRYPTO_ALG_SHA256;
947 	ctx->digest = CONTEXT_CONTROL_DIGEST_PRECOMPUTED;
948 	req->state_sz = SHA256_DIGEST_SIZE;
949 
950 	return 0;
951 }
952 
953 static int safexcel_sha256_digest(struct ahash_request *areq)
954 {
955 	int ret = safexcel_sha256_init(areq);
956 
957 	if (ret)
958 		return ret;
959 
960 	return safexcel_ahash_finup(areq);
961 }
962 
963 struct safexcel_alg_template safexcel_alg_sha256 = {
964 	.type = SAFEXCEL_ALG_TYPE_AHASH,
965 	.alg.ahash = {
966 		.init = safexcel_sha256_init,
967 		.update = safexcel_ahash_update,
968 		.final = safexcel_ahash_final,
969 		.finup = safexcel_ahash_finup,
970 		.digest = safexcel_sha256_digest,
971 		.export = safexcel_ahash_export,
972 		.import = safexcel_ahash_import,
973 		.halg = {
974 			.digestsize = SHA256_DIGEST_SIZE,
975 			.statesize = sizeof(struct safexcel_ahash_export_state),
976 			.base = {
977 				.cra_name = "sha256",
978 				.cra_driver_name = "safexcel-sha256",
979 				.cra_priority = 300,
980 				.cra_flags = CRYPTO_ALG_ASYNC |
981 					     CRYPTO_ALG_KERN_DRIVER_ONLY,
982 				.cra_blocksize = SHA256_BLOCK_SIZE,
983 				.cra_ctxsize = sizeof(struct safexcel_ahash_ctx),
984 				.cra_init = safexcel_ahash_cra_init,
985 				.cra_exit = safexcel_ahash_cra_exit,
986 				.cra_module = THIS_MODULE,
987 			},
988 		},
989 	},
990 };
991 
992 static int safexcel_sha224_init(struct ahash_request *areq)
993 {
994 	struct safexcel_ahash_ctx *ctx = crypto_ahash_ctx(crypto_ahash_reqtfm(areq));
995 	struct safexcel_ahash_req *req = ahash_request_ctx(areq);
996 
997 	memset(req, 0, sizeof(*req));
998 
999 	req->state[0] = SHA224_H0;
1000 	req->state[1] = SHA224_H1;
1001 	req->state[2] = SHA224_H2;
1002 	req->state[3] = SHA224_H3;
1003 	req->state[4] = SHA224_H4;
1004 	req->state[5] = SHA224_H5;
1005 	req->state[6] = SHA224_H6;
1006 	req->state[7] = SHA224_H7;
1007 
1008 	ctx->alg = CONTEXT_CONTROL_CRYPTO_ALG_SHA224;
1009 	ctx->digest = CONTEXT_CONTROL_DIGEST_PRECOMPUTED;
1010 	req->state_sz = SHA256_DIGEST_SIZE;
1011 
1012 	return 0;
1013 }
1014 
1015 static int safexcel_sha224_digest(struct ahash_request *areq)
1016 {
1017 	int ret = safexcel_sha224_init(areq);
1018 
1019 	if (ret)
1020 		return ret;
1021 
1022 	return safexcel_ahash_finup(areq);
1023 }
1024 
1025 struct safexcel_alg_template safexcel_alg_sha224 = {
1026 	.type = SAFEXCEL_ALG_TYPE_AHASH,
1027 	.alg.ahash = {
1028 		.init = safexcel_sha224_init,
1029 		.update = safexcel_ahash_update,
1030 		.final = safexcel_ahash_final,
1031 		.finup = safexcel_ahash_finup,
1032 		.digest = safexcel_sha224_digest,
1033 		.export = safexcel_ahash_export,
1034 		.import = safexcel_ahash_import,
1035 		.halg = {
1036 			.digestsize = SHA224_DIGEST_SIZE,
1037 			.statesize = sizeof(struct safexcel_ahash_export_state),
1038 			.base = {
1039 				.cra_name = "sha224",
1040 				.cra_driver_name = "safexcel-sha224",
1041 				.cra_priority = 300,
1042 				.cra_flags = CRYPTO_ALG_ASYNC |
1043 					     CRYPTO_ALG_KERN_DRIVER_ONLY,
1044 				.cra_blocksize = SHA224_BLOCK_SIZE,
1045 				.cra_ctxsize = sizeof(struct safexcel_ahash_ctx),
1046 				.cra_init = safexcel_ahash_cra_init,
1047 				.cra_exit = safexcel_ahash_cra_exit,
1048 				.cra_module = THIS_MODULE,
1049 			},
1050 		},
1051 	},
1052 };
1053