1 // SPDX-License-Identifier: GPL-2.0
2 /*
3  * Copyright (C) 2017 Marvell
4  *
5  * Antoine Tenart <antoine.tenart@free-electrons.com>
6  */
7 
8 #include <crypto/aes.h>
9 #include <crypto/hmac.h>
10 #include <crypto/md5.h>
11 #include <crypto/sha1.h>
12 #include <crypto/sha2.h>
13 #include <crypto/sha3.h>
14 #include <crypto/skcipher.h>
15 #include <crypto/sm3.h>
16 #include <crypto/internal/cipher.h>
17 #include <linux/device.h>
18 #include <linux/dma-mapping.h>
19 #include <linux/dmapool.h>
20 
21 #include "safexcel.h"
22 
23 struct safexcel_ahash_ctx {
24 	struct safexcel_context base;
25 
26 	u32 alg;
27 	u8  key_sz;
28 	bool cbcmac;
29 	bool do_fallback;
30 	bool fb_init_done;
31 	bool fb_do_setkey;
32 
33 	struct crypto_aes_ctx *aes;
34 	struct crypto_ahash *fback;
35 	struct crypto_shash *shpre;
36 	struct shash_desc *shdesc;
37 };
38 
39 struct safexcel_ahash_req {
40 	bool last_req;
41 	bool finish;
42 	bool hmac;
43 	bool needs_inv;
44 	bool hmac_zlen;
45 	bool len_is_le;
46 	bool not_first;
47 	bool xcbcmac;
48 
49 	int nents;
50 	dma_addr_t result_dma;
51 
52 	u32 digest;
53 
54 	u8 state_sz;    /* expected state size, only set once */
55 	u8 block_sz;    /* block size, only set once */
56 	u8 digest_sz;   /* output digest size, only set once */
57 	__le32 state[SHA3_512_BLOCK_SIZE /
58 		     sizeof(__le32)] __aligned(sizeof(__le32));
59 
60 	u64 len;
61 	u64 processed;
62 
63 	u8 cache[HASH_CACHE_SIZE] __aligned(sizeof(u32));
64 	dma_addr_t cache_dma;
65 	unsigned int cache_sz;
66 
67 	u8 cache_next[HASH_CACHE_SIZE] __aligned(sizeof(u32));
68 };
69 
70 static inline u64 safexcel_queued_len(struct safexcel_ahash_req *req)
71 {
72 	return req->len - req->processed;
73 }
74 
75 static void safexcel_hash_token(struct safexcel_command_desc *cdesc,
76 				u32 input_length, u32 result_length,
77 				bool cbcmac)
78 {
79 	struct safexcel_token *token =
80 		(struct safexcel_token *)cdesc->control_data.token;
81 
82 	token[0].opcode = EIP197_TOKEN_OPCODE_DIRECTION;
83 	token[0].packet_length = input_length;
84 	token[0].instructions = EIP197_TOKEN_INS_TYPE_HASH;
85 
86 	input_length &= 15;
87 	if (unlikely(cbcmac && input_length)) {
88 		token[0].stat =  0;
89 		token[1].opcode = EIP197_TOKEN_OPCODE_INSERT;
90 		token[1].packet_length = 16 - input_length;
91 		token[1].stat = EIP197_TOKEN_STAT_LAST_HASH;
92 		token[1].instructions = EIP197_TOKEN_INS_TYPE_HASH;
93 	} else {
94 		token[0].stat = EIP197_TOKEN_STAT_LAST_HASH;
95 		eip197_noop_token(&token[1]);
96 	}
97 
98 	token[2].opcode = EIP197_TOKEN_OPCODE_INSERT;
99 	token[2].stat = EIP197_TOKEN_STAT_LAST_HASH |
100 			EIP197_TOKEN_STAT_LAST_PACKET;
101 	token[2].packet_length = result_length;
102 	token[2].instructions = EIP197_TOKEN_INS_TYPE_OUTPUT |
103 				EIP197_TOKEN_INS_INSERT_HASH_DIGEST;
104 
105 	eip197_noop_token(&token[3]);
106 }
107 
108 static void safexcel_context_control(struct safexcel_ahash_ctx *ctx,
109 				     struct safexcel_ahash_req *req,
110 				     struct safexcel_command_desc *cdesc)
111 {
112 	struct safexcel_crypto_priv *priv = ctx->base.priv;
113 	u64 count = 0;
114 
115 	cdesc->control_data.control0 = ctx->alg;
116 	cdesc->control_data.control1 = 0;
117 
118 	/*
119 	 * Copy the input digest if needed, and setup the context
120 	 * fields. Do this now as we need it to setup the first command
121 	 * descriptor.
122 	 */
123 	if (unlikely(req->digest == CONTEXT_CONTROL_DIGEST_XCM)) {
124 		if (req->xcbcmac)
125 			memcpy(ctx->base.ctxr->data, &ctx->base.ipad, ctx->key_sz);
126 		else
127 			memcpy(ctx->base.ctxr->data, req->state, req->state_sz);
128 
129 		if (!req->finish && req->xcbcmac)
130 			cdesc->control_data.control0 |=
131 				CONTEXT_CONTROL_DIGEST_XCM |
132 				CONTEXT_CONTROL_TYPE_HASH_OUT  |
133 				CONTEXT_CONTROL_NO_FINISH_HASH |
134 				CONTEXT_CONTROL_SIZE(req->state_sz /
135 						     sizeof(u32));
136 		else
137 			cdesc->control_data.control0 |=
138 				CONTEXT_CONTROL_DIGEST_XCM |
139 				CONTEXT_CONTROL_TYPE_HASH_OUT  |
140 				CONTEXT_CONTROL_SIZE(req->state_sz /
141 						     sizeof(u32));
142 		return;
143 	} else if (!req->processed) {
144 		/* First - and possibly only - block of basic hash only */
145 		if (req->finish)
146 			cdesc->control_data.control0 |= req->digest |
147 				CONTEXT_CONTROL_TYPE_HASH_OUT |
148 				CONTEXT_CONTROL_RESTART_HASH  |
149 				/* ensure its not 0! */
150 				CONTEXT_CONTROL_SIZE(1);
151 		else
152 			cdesc->control_data.control0 |= req->digest |
153 				CONTEXT_CONTROL_TYPE_HASH_OUT  |
154 				CONTEXT_CONTROL_RESTART_HASH   |
155 				CONTEXT_CONTROL_NO_FINISH_HASH |
156 				/* ensure its not 0! */
157 				CONTEXT_CONTROL_SIZE(1);
158 		return;
159 	}
160 
161 	/* Hash continuation or HMAC, setup (inner) digest from state */
162 	memcpy(ctx->base.ctxr->data, req->state, req->state_sz);
163 
164 	if (req->finish) {
165 		/* Compute digest count for hash/HMAC finish operations */
166 		if ((req->digest == CONTEXT_CONTROL_DIGEST_PRECOMPUTED) ||
167 		    req->hmac_zlen || (req->processed != req->block_sz)) {
168 			count = req->processed / EIP197_COUNTER_BLOCK_SIZE;
169 
170 			/* This is a hardware limitation, as the
171 			 * counter must fit into an u32. This represents
172 			 * a fairly big amount of input data, so we
173 			 * shouldn't see this.
174 			 */
175 			if (unlikely(count & 0xffffffff00000000ULL)) {
176 				dev_warn(priv->dev,
177 					 "Input data is too big\n");
178 				return;
179 			}
180 		}
181 
182 		if ((req->digest == CONTEXT_CONTROL_DIGEST_PRECOMPUTED) ||
183 		    /* Special case: zero length HMAC */
184 		    req->hmac_zlen ||
185 		    /* PE HW < 4.4 cannot do HMAC continue, fake using hash */
186 		    (req->processed != req->block_sz)) {
187 			/* Basic hash continue operation, need digest + cnt */
188 			cdesc->control_data.control0 |=
189 				CONTEXT_CONTROL_SIZE((req->state_sz >> 2) + 1) |
190 				CONTEXT_CONTROL_TYPE_HASH_OUT |
191 				CONTEXT_CONTROL_DIGEST_PRECOMPUTED;
192 			/* For zero-len HMAC, don't finalize, already padded! */
193 			if (req->hmac_zlen)
194 				cdesc->control_data.control0 |=
195 					CONTEXT_CONTROL_NO_FINISH_HASH;
196 			cdesc->control_data.control1 |=
197 				CONTEXT_CONTROL_DIGEST_CNT;
198 			ctx->base.ctxr->data[req->state_sz >> 2] =
199 				cpu_to_le32(count);
200 			req->digest = CONTEXT_CONTROL_DIGEST_PRECOMPUTED;
201 
202 			/* Clear zero-length HMAC flag for next operation! */
203 			req->hmac_zlen = false;
204 		} else { /* HMAC */
205 			/* Need outer digest for HMAC finalization */
206 			memcpy(ctx->base.ctxr->data + (req->state_sz >> 2),
207 			       &ctx->base.opad, req->state_sz);
208 
209 			/* Single pass HMAC - no digest count */
210 			cdesc->control_data.control0 |=
211 				CONTEXT_CONTROL_SIZE(req->state_sz >> 1) |
212 				CONTEXT_CONTROL_TYPE_HASH_OUT |
213 				CONTEXT_CONTROL_DIGEST_HMAC;
214 		}
215 	} else { /* Hash continuation, do not finish yet */
216 		cdesc->control_data.control0 |=
217 			CONTEXT_CONTROL_SIZE(req->state_sz >> 2) |
218 			CONTEXT_CONTROL_DIGEST_PRECOMPUTED |
219 			CONTEXT_CONTROL_TYPE_HASH_OUT |
220 			CONTEXT_CONTROL_NO_FINISH_HASH;
221 	}
222 }
223 
224 static int safexcel_ahash_enqueue(struct ahash_request *areq);
225 
226 static int safexcel_handle_req_result(struct safexcel_crypto_priv *priv,
227 				      int ring,
228 				      struct crypto_async_request *async,
229 				      bool *should_complete, int *ret)
230 {
231 	struct safexcel_result_desc *rdesc;
232 	struct ahash_request *areq = ahash_request_cast(async);
233 	struct crypto_ahash *ahash = crypto_ahash_reqtfm(areq);
234 	struct safexcel_ahash_req *sreq = ahash_request_ctx_dma(areq);
235 	struct safexcel_ahash_ctx *ctx = crypto_ahash_ctx(ahash);
236 	u64 cache_len;
237 
238 	*ret = 0;
239 
240 	rdesc = safexcel_ring_next_rptr(priv, &priv->ring[ring].rdr);
241 	if (IS_ERR(rdesc)) {
242 		dev_err(priv->dev,
243 			"hash: result: could not retrieve the result descriptor\n");
244 		*ret = PTR_ERR(rdesc);
245 	} else {
246 		*ret = safexcel_rdesc_check_errors(priv, rdesc);
247 	}
248 
249 	safexcel_complete(priv, ring);
250 
251 	if (sreq->nents) {
252 		dma_unmap_sg(priv->dev, areq->src, sreq->nents, DMA_TO_DEVICE);
253 		sreq->nents = 0;
254 	}
255 
256 	if (sreq->result_dma) {
257 		dma_unmap_single(priv->dev, sreq->result_dma, sreq->digest_sz,
258 				 DMA_FROM_DEVICE);
259 		sreq->result_dma = 0;
260 	}
261 
262 	if (sreq->cache_dma) {
263 		dma_unmap_single(priv->dev, sreq->cache_dma, sreq->cache_sz,
264 				 DMA_TO_DEVICE);
265 		sreq->cache_dma = 0;
266 		sreq->cache_sz = 0;
267 	}
268 
269 	if (sreq->finish) {
270 		if (sreq->hmac &&
271 		    (sreq->digest != CONTEXT_CONTROL_DIGEST_HMAC)) {
272 			/* Faking HMAC using hash - need to do outer hash */
273 			memcpy(sreq->cache, sreq->state,
274 			       crypto_ahash_digestsize(ahash));
275 
276 			memcpy(sreq->state, &ctx->base.opad, sreq->digest_sz);
277 
278 			sreq->len = sreq->block_sz +
279 				    crypto_ahash_digestsize(ahash);
280 			sreq->processed = sreq->block_sz;
281 			sreq->hmac = 0;
282 
283 			if (priv->flags & EIP197_TRC_CACHE)
284 				ctx->base.needs_inv = true;
285 			areq->nbytes = 0;
286 			safexcel_ahash_enqueue(areq);
287 
288 			*should_complete = false; /* Not done yet */
289 			return 1;
290 		}
291 
292 		if (unlikely(sreq->digest == CONTEXT_CONTROL_DIGEST_XCM &&
293 			     ctx->alg == CONTEXT_CONTROL_CRYPTO_ALG_CRC32)) {
294 			/* Undo final XOR with 0xffffffff ...*/
295 			*(__le32 *)areq->result = ~sreq->state[0];
296 		} else {
297 			memcpy(areq->result, sreq->state,
298 			       crypto_ahash_digestsize(ahash));
299 		}
300 	}
301 
302 	cache_len = safexcel_queued_len(sreq);
303 	if (cache_len)
304 		memcpy(sreq->cache, sreq->cache_next, cache_len);
305 
306 	*should_complete = true;
307 
308 	return 1;
309 }
310 
311 static int safexcel_ahash_send_req(struct crypto_async_request *async, int ring,
312 				   int *commands, int *results)
313 {
314 	struct ahash_request *areq = ahash_request_cast(async);
315 	struct safexcel_ahash_req *req = ahash_request_ctx_dma(areq);
316 	struct safexcel_ahash_ctx *ctx = crypto_ahash_ctx(crypto_ahash_reqtfm(areq));
317 	struct safexcel_crypto_priv *priv = ctx->base.priv;
318 	struct safexcel_command_desc *cdesc, *first_cdesc = NULL;
319 	struct safexcel_result_desc *rdesc;
320 	struct scatterlist *sg;
321 	struct safexcel_token *dmmy;
322 	int i, extra = 0, n_cdesc = 0, ret = 0, cache_len, skip = 0;
323 	u64 queued, len;
324 
325 	queued = safexcel_queued_len(req);
326 	if (queued <= HASH_CACHE_SIZE)
327 		cache_len = queued;
328 	else
329 		cache_len = queued - areq->nbytes;
330 
331 	if (!req->finish && !req->last_req) {
332 		/* If this is not the last request and the queued data does not
333 		 * fit into full cache blocks, cache it for the next send call.
334 		 */
335 		extra = queued & (HASH_CACHE_SIZE - 1);
336 
337 		/* If this is not the last request and the queued data
338 		 * is a multiple of a block, cache the last one for now.
339 		 */
340 		if (!extra)
341 			extra = HASH_CACHE_SIZE;
342 
343 		sg_pcopy_to_buffer(areq->src, sg_nents(areq->src),
344 				   req->cache_next, extra,
345 				   areq->nbytes - extra);
346 
347 		queued -= extra;
348 
349 		if (!queued) {
350 			*commands = 0;
351 			*results = 0;
352 			return 0;
353 		}
354 
355 		extra = 0;
356 	}
357 
358 	if (unlikely(req->xcbcmac && req->processed > AES_BLOCK_SIZE)) {
359 		if (unlikely(cache_len < AES_BLOCK_SIZE)) {
360 			/*
361 			 * Cache contains less than 1 full block, complete.
362 			 */
363 			extra = AES_BLOCK_SIZE - cache_len;
364 			if (queued > cache_len) {
365 				/* More data follows: borrow bytes */
366 				u64 tmp = queued - cache_len;
367 
368 				skip = min_t(u64, tmp, extra);
369 				sg_pcopy_to_buffer(areq->src,
370 					sg_nents(areq->src),
371 					req->cache + cache_len,
372 					skip, 0);
373 			}
374 			extra -= skip;
375 			memset(req->cache + cache_len + skip, 0, extra);
376 			if (!ctx->cbcmac && extra) {
377 				// 10- padding for XCBCMAC & CMAC
378 				req->cache[cache_len + skip] = 0x80;
379 				// HW will use K2 iso K3 - compensate!
380 				for (i = 0; i < AES_BLOCK_SIZE / 4; i++) {
381 					u32 *cache = (void *)req->cache;
382 					u32 *ipad = ctx->base.ipad.word;
383 					u32 x;
384 
385 					x = ipad[i] ^ ipad[i + 4];
386 					cache[i] ^= swab32(x);
387 				}
388 			}
389 			cache_len = AES_BLOCK_SIZE;
390 			queued = queued + extra;
391 		}
392 
393 		/* XCBC continue: XOR previous result into 1st word */
394 		crypto_xor(req->cache, (const u8 *)req->state, AES_BLOCK_SIZE);
395 	}
396 
397 	len = queued;
398 	/* Add a command descriptor for the cached data, if any */
399 	if (cache_len) {
400 		req->cache_dma = dma_map_single(priv->dev, req->cache,
401 						cache_len, DMA_TO_DEVICE);
402 		if (dma_mapping_error(priv->dev, req->cache_dma))
403 			return -EINVAL;
404 
405 		req->cache_sz = cache_len;
406 		first_cdesc = safexcel_add_cdesc(priv, ring, 1,
407 						 (cache_len == len),
408 						 req->cache_dma, cache_len,
409 						 len, ctx->base.ctxr_dma,
410 						 &dmmy);
411 		if (IS_ERR(first_cdesc)) {
412 			ret = PTR_ERR(first_cdesc);
413 			goto unmap_cache;
414 		}
415 		n_cdesc++;
416 
417 		queued -= cache_len;
418 		if (!queued)
419 			goto send_command;
420 	}
421 
422 	/* Now handle the current ahash request buffer(s) */
423 	req->nents = dma_map_sg(priv->dev, areq->src,
424 				sg_nents_for_len(areq->src,
425 						 areq->nbytes),
426 				DMA_TO_DEVICE);
427 	if (!req->nents) {
428 		ret = -ENOMEM;
429 		goto cdesc_rollback;
430 	}
431 
432 	for_each_sg(areq->src, sg, req->nents, i) {
433 		int sglen = sg_dma_len(sg);
434 
435 		if (unlikely(sglen <= skip)) {
436 			skip -= sglen;
437 			continue;
438 		}
439 
440 		/* Do not overflow the request */
441 		if ((queued + skip) <= sglen)
442 			sglen = queued;
443 		else
444 			sglen -= skip;
445 
446 		cdesc = safexcel_add_cdesc(priv, ring, !n_cdesc,
447 					   !(queued - sglen),
448 					   sg_dma_address(sg) + skip, sglen,
449 					   len, ctx->base.ctxr_dma, &dmmy);
450 		if (IS_ERR(cdesc)) {
451 			ret = PTR_ERR(cdesc);
452 			goto unmap_sg;
453 		}
454 
455 		if (!n_cdesc)
456 			first_cdesc = cdesc;
457 		n_cdesc++;
458 
459 		queued -= sglen;
460 		if (!queued)
461 			break;
462 		skip = 0;
463 	}
464 
465 send_command:
466 	/* Setup the context options */
467 	safexcel_context_control(ctx, req, first_cdesc);
468 
469 	/* Add the token */
470 	safexcel_hash_token(first_cdesc, len, req->digest_sz, ctx->cbcmac);
471 
472 	req->result_dma = dma_map_single(priv->dev, req->state, req->digest_sz,
473 					 DMA_FROM_DEVICE);
474 	if (dma_mapping_error(priv->dev, req->result_dma)) {
475 		ret = -EINVAL;
476 		goto unmap_sg;
477 	}
478 
479 	/* Add a result descriptor */
480 	rdesc = safexcel_add_rdesc(priv, ring, 1, 1, req->result_dma,
481 				   req->digest_sz);
482 	if (IS_ERR(rdesc)) {
483 		ret = PTR_ERR(rdesc);
484 		goto unmap_result;
485 	}
486 
487 	safexcel_rdr_req_set(priv, ring, rdesc, &areq->base);
488 
489 	req->processed += len - extra;
490 
491 	*commands = n_cdesc;
492 	*results = 1;
493 	return 0;
494 
495 unmap_result:
496 	dma_unmap_single(priv->dev, req->result_dma, req->digest_sz,
497 			 DMA_FROM_DEVICE);
498 unmap_sg:
499 	if (req->nents) {
500 		dma_unmap_sg(priv->dev, areq->src, req->nents, DMA_TO_DEVICE);
501 		req->nents = 0;
502 	}
503 cdesc_rollback:
504 	for (i = 0; i < n_cdesc; i++)
505 		safexcel_ring_rollback_wptr(priv, &priv->ring[ring].cdr);
506 unmap_cache:
507 	if (req->cache_dma) {
508 		dma_unmap_single(priv->dev, req->cache_dma, req->cache_sz,
509 				 DMA_TO_DEVICE);
510 		req->cache_dma = 0;
511 		req->cache_sz = 0;
512 	}
513 
514 	return ret;
515 }
516 
517 static int safexcel_handle_inv_result(struct safexcel_crypto_priv *priv,
518 				      int ring,
519 				      struct crypto_async_request *async,
520 				      bool *should_complete, int *ret)
521 {
522 	struct safexcel_result_desc *rdesc;
523 	struct ahash_request *areq = ahash_request_cast(async);
524 	struct crypto_ahash *ahash = crypto_ahash_reqtfm(areq);
525 	struct safexcel_ahash_ctx *ctx = crypto_ahash_ctx(ahash);
526 	int enq_ret;
527 
528 	*ret = 0;
529 
530 	rdesc = safexcel_ring_next_rptr(priv, &priv->ring[ring].rdr);
531 	if (IS_ERR(rdesc)) {
532 		dev_err(priv->dev,
533 			"hash: invalidate: could not retrieve the result descriptor\n");
534 		*ret = PTR_ERR(rdesc);
535 	} else {
536 		*ret = safexcel_rdesc_check_errors(priv, rdesc);
537 	}
538 
539 	safexcel_complete(priv, ring);
540 
541 	if (ctx->base.exit_inv) {
542 		dma_pool_free(priv->context_pool, ctx->base.ctxr,
543 			      ctx->base.ctxr_dma);
544 
545 		*should_complete = true;
546 		return 1;
547 	}
548 
549 	ring = safexcel_select_ring(priv);
550 	ctx->base.ring = ring;
551 
552 	spin_lock_bh(&priv->ring[ring].queue_lock);
553 	enq_ret = crypto_enqueue_request(&priv->ring[ring].queue, async);
554 	spin_unlock_bh(&priv->ring[ring].queue_lock);
555 
556 	if (enq_ret != -EINPROGRESS)
557 		*ret = enq_ret;
558 
559 	queue_work(priv->ring[ring].workqueue,
560 		   &priv->ring[ring].work_data.work);
561 
562 	*should_complete = false;
563 
564 	return 1;
565 }
566 
567 static int safexcel_handle_result(struct safexcel_crypto_priv *priv, int ring,
568 				  struct crypto_async_request *async,
569 				  bool *should_complete, int *ret)
570 {
571 	struct ahash_request *areq = ahash_request_cast(async);
572 	struct safexcel_ahash_req *req = ahash_request_ctx_dma(areq);
573 	int err;
574 
575 	BUG_ON(!(priv->flags & EIP197_TRC_CACHE) && req->needs_inv);
576 
577 	if (req->needs_inv) {
578 		req->needs_inv = false;
579 		err = safexcel_handle_inv_result(priv, ring, async,
580 						 should_complete, ret);
581 	} else {
582 		err = safexcel_handle_req_result(priv, ring, async,
583 						 should_complete, ret);
584 	}
585 
586 	return err;
587 }
588 
589 static int safexcel_ahash_send_inv(struct crypto_async_request *async,
590 				   int ring, int *commands, int *results)
591 {
592 	struct ahash_request *areq = ahash_request_cast(async);
593 	struct safexcel_ahash_ctx *ctx = crypto_ahash_ctx(crypto_ahash_reqtfm(areq));
594 	int ret;
595 
596 	ret = safexcel_invalidate_cache(async, ctx->base.priv,
597 					ctx->base.ctxr_dma, ring);
598 	if (unlikely(ret))
599 		return ret;
600 
601 	*commands = 1;
602 	*results = 1;
603 
604 	return 0;
605 }
606 
607 static int safexcel_ahash_send(struct crypto_async_request *async,
608 			       int ring, int *commands, int *results)
609 {
610 	struct ahash_request *areq = ahash_request_cast(async);
611 	struct safexcel_ahash_req *req = ahash_request_ctx_dma(areq);
612 	int ret;
613 
614 	if (req->needs_inv)
615 		ret = safexcel_ahash_send_inv(async, ring, commands, results);
616 	else
617 		ret = safexcel_ahash_send_req(async, ring, commands, results);
618 
619 	return ret;
620 }
621 
622 static int safexcel_ahash_exit_inv(struct crypto_tfm *tfm)
623 {
624 	struct safexcel_ahash_ctx *ctx = crypto_tfm_ctx(tfm);
625 	struct safexcel_crypto_priv *priv = ctx->base.priv;
626 	EIP197_REQUEST_ON_STACK(req, ahash, EIP197_AHASH_REQ_SIZE);
627 	struct safexcel_ahash_req *rctx = ahash_request_ctx_dma(req);
628 	struct safexcel_inv_result result = {};
629 	int ring = ctx->base.ring;
630 
631 	memset(req, 0, EIP197_AHASH_REQ_SIZE);
632 
633 	/* create invalidation request */
634 	init_completion(&result.completion);
635 	ahash_request_set_callback(req, CRYPTO_TFM_REQ_MAY_BACKLOG,
636 				   safexcel_inv_complete, &result);
637 
638 	ahash_request_set_tfm(req, __crypto_ahash_cast(tfm));
639 	ctx = crypto_tfm_ctx(req->base.tfm);
640 	ctx->base.exit_inv = true;
641 	rctx->needs_inv = true;
642 
643 	spin_lock_bh(&priv->ring[ring].queue_lock);
644 	crypto_enqueue_request(&priv->ring[ring].queue, &req->base);
645 	spin_unlock_bh(&priv->ring[ring].queue_lock);
646 
647 	queue_work(priv->ring[ring].workqueue,
648 		   &priv->ring[ring].work_data.work);
649 
650 	wait_for_completion(&result.completion);
651 
652 	if (result.error) {
653 		dev_warn(priv->dev, "hash: completion error (%d)\n",
654 			 result.error);
655 		return result.error;
656 	}
657 
658 	return 0;
659 }
660 
661 /* safexcel_ahash_cache: cache data until at least one request can be sent to
662  * the engine, aka. when there is at least 1 block size in the pipe.
663  */
664 static int safexcel_ahash_cache(struct ahash_request *areq)
665 {
666 	struct safexcel_ahash_req *req = ahash_request_ctx_dma(areq);
667 	u64 cache_len;
668 
669 	/* cache_len: everything accepted by the driver but not sent yet,
670 	 * tot sz handled by update() - last req sz - tot sz handled by send()
671 	 */
672 	cache_len = safexcel_queued_len(req);
673 
674 	/*
675 	 * In case there isn't enough bytes to proceed (less than a
676 	 * block size), cache the data until we have enough.
677 	 */
678 	if (cache_len + areq->nbytes <= HASH_CACHE_SIZE) {
679 		sg_pcopy_to_buffer(areq->src, sg_nents(areq->src),
680 				   req->cache + cache_len,
681 				   areq->nbytes, 0);
682 		return 0;
683 	}
684 
685 	/* We couldn't cache all the data */
686 	return -E2BIG;
687 }
688 
689 static int safexcel_ahash_enqueue(struct ahash_request *areq)
690 {
691 	struct safexcel_ahash_ctx *ctx = crypto_ahash_ctx(crypto_ahash_reqtfm(areq));
692 	struct safexcel_ahash_req *req = ahash_request_ctx_dma(areq);
693 	struct safexcel_crypto_priv *priv = ctx->base.priv;
694 	int ret, ring;
695 
696 	req->needs_inv = false;
697 
698 	if (ctx->base.ctxr) {
699 		if (priv->flags & EIP197_TRC_CACHE && !ctx->base.needs_inv &&
700 		     /* invalidate for *any* non-XCBC continuation */
701 		   ((req->not_first && !req->xcbcmac) ||
702 		     /* invalidate if (i)digest changed */
703 		     memcmp(ctx->base.ctxr->data, req->state, req->state_sz) ||
704 		     /* invalidate for HMAC finish with odigest changed */
705 		     (req->finish && req->hmac &&
706 		      memcmp(ctx->base.ctxr->data + (req->state_sz>>2),
707 			     &ctx->base.opad, req->state_sz))))
708 			/*
709 			 * We're still setting needs_inv here, even though it is
710 			 * cleared right away, because the needs_inv flag can be
711 			 * set in other functions and we want to keep the same
712 			 * logic.
713 			 */
714 			ctx->base.needs_inv = true;
715 
716 		if (ctx->base.needs_inv) {
717 			ctx->base.needs_inv = false;
718 			req->needs_inv = true;
719 		}
720 	} else {
721 		ctx->base.ring = safexcel_select_ring(priv);
722 		ctx->base.ctxr = dma_pool_zalloc(priv->context_pool,
723 						 EIP197_GFP_FLAGS(areq->base),
724 						 &ctx->base.ctxr_dma);
725 		if (!ctx->base.ctxr)
726 			return -ENOMEM;
727 	}
728 	req->not_first = true;
729 
730 	ring = ctx->base.ring;
731 
732 	spin_lock_bh(&priv->ring[ring].queue_lock);
733 	ret = crypto_enqueue_request(&priv->ring[ring].queue, &areq->base);
734 	spin_unlock_bh(&priv->ring[ring].queue_lock);
735 
736 	queue_work(priv->ring[ring].workqueue,
737 		   &priv->ring[ring].work_data.work);
738 
739 	return ret;
740 }
741 
742 static int safexcel_ahash_update(struct ahash_request *areq)
743 {
744 	struct safexcel_ahash_req *req = ahash_request_ctx_dma(areq);
745 	int ret;
746 
747 	/* If the request is 0 length, do nothing */
748 	if (!areq->nbytes)
749 		return 0;
750 
751 	/* Add request to the cache if it fits */
752 	ret = safexcel_ahash_cache(areq);
753 
754 	/* Update total request length */
755 	req->len += areq->nbytes;
756 
757 	/* If not all data could fit into the cache, go process the excess.
758 	 * Also go process immediately for an HMAC IV precompute, which
759 	 * will never be finished at all, but needs to be processed anyway.
760 	 */
761 	if ((ret && !req->finish) || req->last_req)
762 		return safexcel_ahash_enqueue(areq);
763 
764 	return 0;
765 }
766 
767 static int safexcel_ahash_final(struct ahash_request *areq)
768 {
769 	struct safexcel_ahash_req *req = ahash_request_ctx_dma(areq);
770 	struct safexcel_ahash_ctx *ctx = crypto_ahash_ctx(crypto_ahash_reqtfm(areq));
771 
772 	req->finish = true;
773 
774 	if (unlikely(!req->len && !areq->nbytes)) {
775 		/*
776 		 * If we have an overall 0 length *hash* request:
777 		 * The HW cannot do 0 length hash, so we provide the correct
778 		 * result directly here.
779 		 */
780 		if (ctx->alg == CONTEXT_CONTROL_CRYPTO_ALG_MD5)
781 			memcpy(areq->result, md5_zero_message_hash,
782 			       MD5_DIGEST_SIZE);
783 		else if (ctx->alg == CONTEXT_CONTROL_CRYPTO_ALG_SHA1)
784 			memcpy(areq->result, sha1_zero_message_hash,
785 			       SHA1_DIGEST_SIZE);
786 		else if (ctx->alg == CONTEXT_CONTROL_CRYPTO_ALG_SHA224)
787 			memcpy(areq->result, sha224_zero_message_hash,
788 			       SHA224_DIGEST_SIZE);
789 		else if (ctx->alg == CONTEXT_CONTROL_CRYPTO_ALG_SHA256)
790 			memcpy(areq->result, sha256_zero_message_hash,
791 			       SHA256_DIGEST_SIZE);
792 		else if (ctx->alg == CONTEXT_CONTROL_CRYPTO_ALG_SHA384)
793 			memcpy(areq->result, sha384_zero_message_hash,
794 			       SHA384_DIGEST_SIZE);
795 		else if (ctx->alg == CONTEXT_CONTROL_CRYPTO_ALG_SHA512)
796 			memcpy(areq->result, sha512_zero_message_hash,
797 			       SHA512_DIGEST_SIZE);
798 		else if (ctx->alg == CONTEXT_CONTROL_CRYPTO_ALG_SM3) {
799 			memcpy(areq->result,
800 			       EIP197_SM3_ZEROM_HASH, SM3_DIGEST_SIZE);
801 		}
802 
803 		return 0;
804 	} else if (unlikely(req->digest == CONTEXT_CONTROL_DIGEST_XCM &&
805 			    ctx->alg == CONTEXT_CONTROL_CRYPTO_ALG_MD5 &&
806 			    req->len == sizeof(u32) && !areq->nbytes)) {
807 		/* Zero length CRC32 */
808 		memcpy(areq->result, &ctx->base.ipad, sizeof(u32));
809 		return 0;
810 	} else if (unlikely(ctx->cbcmac && req->len == AES_BLOCK_SIZE &&
811 			    !areq->nbytes)) {
812 		/* Zero length CBC MAC */
813 		memset(areq->result, 0, AES_BLOCK_SIZE);
814 		return 0;
815 	} else if (unlikely(req->xcbcmac && req->len == AES_BLOCK_SIZE &&
816 			    !areq->nbytes)) {
817 		/* Zero length (X)CBC/CMAC */
818 		int i;
819 
820 		for (i = 0; i < AES_BLOCK_SIZE / sizeof(u32); i++) {
821 			u32 *result = (void *)areq->result;
822 
823 			/* K3 */
824 			result[i] = swab32(ctx->base.ipad.word[i + 4]);
825 		}
826 		areq->result[0] ^= 0x80;			// 10- padding
827 		aes_encrypt(ctx->aes, areq->result, areq->result);
828 		return 0;
829 	} else if (unlikely(req->hmac &&
830 			    (req->len == req->block_sz) &&
831 			    !areq->nbytes)) {
832 		/*
833 		 * If we have an overall 0 length *HMAC* request:
834 		 * For HMAC, we need to finalize the inner digest
835 		 * and then perform the outer hash.
836 		 */
837 
838 		/* generate pad block in the cache */
839 		/* start with a hash block of all zeroes */
840 		memset(req->cache, 0, req->block_sz);
841 		/* set the first byte to 0x80 to 'append a 1 bit' */
842 		req->cache[0] = 0x80;
843 		/* add the length in bits in the last 2 bytes */
844 		if (req->len_is_le) {
845 			/* Little endian length word (e.g. MD5) */
846 			req->cache[req->block_sz-8] = (req->block_sz << 3) &
847 						      255;
848 			req->cache[req->block_sz-7] = (req->block_sz >> 5);
849 		} else {
850 			/* Big endian length word (e.g. any SHA) */
851 			req->cache[req->block_sz-2] = (req->block_sz >> 5);
852 			req->cache[req->block_sz-1] = (req->block_sz << 3) &
853 						      255;
854 		}
855 
856 		req->len += req->block_sz; /* plus 1 hash block */
857 
858 		/* Set special zero-length HMAC flag */
859 		req->hmac_zlen = true;
860 
861 		/* Finalize HMAC */
862 		req->digest = CONTEXT_CONTROL_DIGEST_HMAC;
863 	} else if (req->hmac) {
864 		/* Finalize HMAC */
865 		req->digest = CONTEXT_CONTROL_DIGEST_HMAC;
866 	}
867 
868 	return safexcel_ahash_enqueue(areq);
869 }
870 
871 static int safexcel_ahash_finup(struct ahash_request *areq)
872 {
873 	struct safexcel_ahash_req *req = ahash_request_ctx_dma(areq);
874 
875 	req->finish = true;
876 
877 	safexcel_ahash_update(areq);
878 	return safexcel_ahash_final(areq);
879 }
880 
881 static int safexcel_ahash_export(struct ahash_request *areq, void *out)
882 {
883 	struct safexcel_ahash_req *req = ahash_request_ctx_dma(areq);
884 	struct safexcel_ahash_export_state *export = out;
885 
886 	export->len = req->len;
887 	export->processed = req->processed;
888 
889 	export->digest = req->digest;
890 
891 	memcpy(export->state, req->state, req->state_sz);
892 	memcpy(export->cache, req->cache, HASH_CACHE_SIZE);
893 
894 	return 0;
895 }
896 
897 static int safexcel_ahash_import(struct ahash_request *areq, const void *in)
898 {
899 	struct safexcel_ahash_req *req = ahash_request_ctx_dma(areq);
900 	const struct safexcel_ahash_export_state *export = in;
901 	int ret;
902 
903 	ret = crypto_ahash_init(areq);
904 	if (ret)
905 		return ret;
906 
907 	req->len = export->len;
908 	req->processed = export->processed;
909 
910 	req->digest = export->digest;
911 
912 	memcpy(req->cache, export->cache, HASH_CACHE_SIZE);
913 	memcpy(req->state, export->state, req->state_sz);
914 
915 	return 0;
916 }
917 
918 static int safexcel_ahash_cra_init(struct crypto_tfm *tfm)
919 {
920 	struct safexcel_ahash_ctx *ctx = crypto_tfm_ctx(tfm);
921 	struct safexcel_alg_template *tmpl =
922 		container_of(__crypto_ahash_alg(tfm->__crt_alg),
923 			     struct safexcel_alg_template, alg.ahash);
924 
925 	ctx->base.priv = tmpl->priv;
926 	ctx->base.send = safexcel_ahash_send;
927 	ctx->base.handle_result = safexcel_handle_result;
928 	ctx->fb_do_setkey = false;
929 
930 	crypto_ahash_set_reqsize_dma(__crypto_ahash_cast(tfm),
931 				     sizeof(struct safexcel_ahash_req));
932 	return 0;
933 }
934 
935 static int safexcel_sha1_init(struct ahash_request *areq)
936 {
937 	struct safexcel_ahash_ctx *ctx = crypto_ahash_ctx(crypto_ahash_reqtfm(areq));
938 	struct safexcel_ahash_req *req = ahash_request_ctx_dma(areq);
939 
940 	memset(req, 0, sizeof(*req));
941 
942 	ctx->alg = CONTEXT_CONTROL_CRYPTO_ALG_SHA1;
943 	req->digest = CONTEXT_CONTROL_DIGEST_PRECOMPUTED;
944 	req->state_sz = SHA1_DIGEST_SIZE;
945 	req->digest_sz = SHA1_DIGEST_SIZE;
946 	req->block_sz = SHA1_BLOCK_SIZE;
947 
948 	return 0;
949 }
950 
951 static int safexcel_sha1_digest(struct ahash_request *areq)
952 {
953 	int ret = safexcel_sha1_init(areq);
954 
955 	if (ret)
956 		return ret;
957 
958 	return safexcel_ahash_finup(areq);
959 }
960 
961 static void safexcel_ahash_cra_exit(struct crypto_tfm *tfm)
962 {
963 	struct safexcel_ahash_ctx *ctx = crypto_tfm_ctx(tfm);
964 	struct safexcel_crypto_priv *priv = ctx->base.priv;
965 	int ret;
966 
967 	/* context not allocated, skip invalidation */
968 	if (!ctx->base.ctxr)
969 		return;
970 
971 	if (priv->flags & EIP197_TRC_CACHE) {
972 		ret = safexcel_ahash_exit_inv(tfm);
973 		if (ret)
974 			dev_warn(priv->dev, "hash: invalidation error %d\n", ret);
975 	} else {
976 		dma_pool_free(priv->context_pool, ctx->base.ctxr,
977 			      ctx->base.ctxr_dma);
978 	}
979 }
980 
981 struct safexcel_alg_template safexcel_alg_sha1 = {
982 	.type = SAFEXCEL_ALG_TYPE_AHASH,
983 	.algo_mask = SAFEXCEL_ALG_SHA1,
984 	.alg.ahash = {
985 		.init = safexcel_sha1_init,
986 		.update = safexcel_ahash_update,
987 		.final = safexcel_ahash_final,
988 		.finup = safexcel_ahash_finup,
989 		.digest = safexcel_sha1_digest,
990 		.export = safexcel_ahash_export,
991 		.import = safexcel_ahash_import,
992 		.halg = {
993 			.digestsize = SHA1_DIGEST_SIZE,
994 			.statesize = sizeof(struct safexcel_ahash_export_state),
995 			.base = {
996 				.cra_name = "sha1",
997 				.cra_driver_name = "safexcel-sha1",
998 				.cra_priority = SAFEXCEL_CRA_PRIORITY,
999 				.cra_flags = CRYPTO_ALG_ASYNC |
1000 					     CRYPTO_ALG_ALLOCATES_MEMORY |
1001 					     CRYPTO_ALG_KERN_DRIVER_ONLY,
1002 				.cra_blocksize = SHA1_BLOCK_SIZE,
1003 				.cra_ctxsize = sizeof(struct safexcel_ahash_ctx),
1004 				.cra_init = safexcel_ahash_cra_init,
1005 				.cra_exit = safexcel_ahash_cra_exit,
1006 				.cra_module = THIS_MODULE,
1007 			},
1008 		},
1009 	},
1010 };
1011 
1012 static int safexcel_hmac_sha1_init(struct ahash_request *areq)
1013 {
1014 	struct safexcel_ahash_ctx *ctx = crypto_ahash_ctx(crypto_ahash_reqtfm(areq));
1015 	struct safexcel_ahash_req *req = ahash_request_ctx_dma(areq);
1016 
1017 	memset(req, 0, sizeof(*req));
1018 
1019 	/* Start from ipad precompute */
1020 	memcpy(req->state, &ctx->base.ipad, SHA1_DIGEST_SIZE);
1021 	/* Already processed the key^ipad part now! */
1022 	req->len	= SHA1_BLOCK_SIZE;
1023 	req->processed	= SHA1_BLOCK_SIZE;
1024 
1025 	ctx->alg = CONTEXT_CONTROL_CRYPTO_ALG_SHA1;
1026 	req->digest = CONTEXT_CONTROL_DIGEST_PRECOMPUTED;
1027 	req->state_sz = SHA1_DIGEST_SIZE;
1028 	req->digest_sz = SHA1_DIGEST_SIZE;
1029 	req->block_sz = SHA1_BLOCK_SIZE;
1030 	req->hmac = true;
1031 
1032 	return 0;
1033 }
1034 
1035 static int safexcel_hmac_sha1_digest(struct ahash_request *areq)
1036 {
1037 	int ret = safexcel_hmac_sha1_init(areq);
1038 
1039 	if (ret)
1040 		return ret;
1041 
1042 	return safexcel_ahash_finup(areq);
1043 }
1044 
1045 struct safexcel_ahash_result {
1046 	struct completion completion;
1047 	int error;
1048 };
1049 
1050 static void safexcel_ahash_complete(struct crypto_async_request *req, int error)
1051 {
1052 	struct safexcel_ahash_result *result = req->data;
1053 
1054 	if (error == -EINPROGRESS)
1055 		return;
1056 
1057 	result->error = error;
1058 	complete(&result->completion);
1059 }
1060 
1061 static int safexcel_hmac_init_pad(struct ahash_request *areq,
1062 				  unsigned int blocksize, const u8 *key,
1063 				  unsigned int keylen, u8 *ipad, u8 *opad)
1064 {
1065 	struct safexcel_ahash_result result;
1066 	struct scatterlist sg;
1067 	int ret, i;
1068 	u8 *keydup;
1069 
1070 	if (keylen <= blocksize) {
1071 		memcpy(ipad, key, keylen);
1072 	} else {
1073 		keydup = kmemdup(key, keylen, GFP_KERNEL);
1074 		if (!keydup)
1075 			return -ENOMEM;
1076 
1077 		ahash_request_set_callback(areq, CRYPTO_TFM_REQ_MAY_BACKLOG,
1078 					   safexcel_ahash_complete, &result);
1079 		sg_init_one(&sg, keydup, keylen);
1080 		ahash_request_set_crypt(areq, &sg, ipad, keylen);
1081 		init_completion(&result.completion);
1082 
1083 		ret = crypto_ahash_digest(areq);
1084 		if (ret == -EINPROGRESS || ret == -EBUSY) {
1085 			wait_for_completion_interruptible(&result.completion);
1086 			ret = result.error;
1087 		}
1088 
1089 		/* Avoid leaking */
1090 		kfree_sensitive(keydup);
1091 
1092 		if (ret)
1093 			return ret;
1094 
1095 		keylen = crypto_ahash_digestsize(crypto_ahash_reqtfm(areq));
1096 	}
1097 
1098 	memset(ipad + keylen, 0, blocksize - keylen);
1099 	memcpy(opad, ipad, blocksize);
1100 
1101 	for (i = 0; i < blocksize; i++) {
1102 		ipad[i] ^= HMAC_IPAD_VALUE;
1103 		opad[i] ^= HMAC_OPAD_VALUE;
1104 	}
1105 
1106 	return 0;
1107 }
1108 
1109 static int safexcel_hmac_init_iv(struct ahash_request *areq,
1110 				 unsigned int blocksize, u8 *pad, void *state)
1111 {
1112 	struct safexcel_ahash_result result;
1113 	struct safexcel_ahash_req *req;
1114 	struct scatterlist sg;
1115 	int ret;
1116 
1117 	ahash_request_set_callback(areq, CRYPTO_TFM_REQ_MAY_BACKLOG,
1118 				   safexcel_ahash_complete, &result);
1119 	sg_init_one(&sg, pad, blocksize);
1120 	ahash_request_set_crypt(areq, &sg, pad, blocksize);
1121 	init_completion(&result.completion);
1122 
1123 	ret = crypto_ahash_init(areq);
1124 	if (ret)
1125 		return ret;
1126 
1127 	req = ahash_request_ctx_dma(areq);
1128 	req->hmac = true;
1129 	req->last_req = true;
1130 
1131 	ret = crypto_ahash_update(areq);
1132 	if (ret && ret != -EINPROGRESS && ret != -EBUSY)
1133 		return ret;
1134 
1135 	wait_for_completion_interruptible(&result.completion);
1136 	if (result.error)
1137 		return result.error;
1138 
1139 	return crypto_ahash_export(areq, state);
1140 }
1141 
1142 static int __safexcel_hmac_setkey(const char *alg, const u8 *key,
1143 				  unsigned int keylen,
1144 				  void *istate, void *ostate)
1145 {
1146 	struct ahash_request *areq;
1147 	struct crypto_ahash *tfm;
1148 	unsigned int blocksize;
1149 	u8 *ipad, *opad;
1150 	int ret;
1151 
1152 	tfm = crypto_alloc_ahash(alg, 0, 0);
1153 	if (IS_ERR(tfm))
1154 		return PTR_ERR(tfm);
1155 
1156 	areq = ahash_request_alloc(tfm, GFP_KERNEL);
1157 	if (!areq) {
1158 		ret = -ENOMEM;
1159 		goto free_ahash;
1160 	}
1161 
1162 	crypto_ahash_clear_flags(tfm, ~0);
1163 	blocksize = crypto_tfm_alg_blocksize(crypto_ahash_tfm(tfm));
1164 
1165 	ipad = kcalloc(2, blocksize, GFP_KERNEL);
1166 	if (!ipad) {
1167 		ret = -ENOMEM;
1168 		goto free_request;
1169 	}
1170 
1171 	opad = ipad + blocksize;
1172 
1173 	ret = safexcel_hmac_init_pad(areq, blocksize, key, keylen, ipad, opad);
1174 	if (ret)
1175 		goto free_ipad;
1176 
1177 	ret = safexcel_hmac_init_iv(areq, blocksize, ipad, istate);
1178 	if (ret)
1179 		goto free_ipad;
1180 
1181 	ret = safexcel_hmac_init_iv(areq, blocksize, opad, ostate);
1182 
1183 free_ipad:
1184 	kfree(ipad);
1185 free_request:
1186 	ahash_request_free(areq);
1187 free_ahash:
1188 	crypto_free_ahash(tfm);
1189 
1190 	return ret;
1191 }
1192 
1193 int safexcel_hmac_setkey(struct safexcel_context *base, const u8 *key,
1194 			 unsigned int keylen, const char *alg,
1195 			 unsigned int state_sz)
1196 {
1197 	struct safexcel_crypto_priv *priv = base->priv;
1198 	struct safexcel_ahash_export_state istate, ostate;
1199 	int ret;
1200 
1201 	ret = __safexcel_hmac_setkey(alg, key, keylen, &istate, &ostate);
1202 	if (ret)
1203 		return ret;
1204 
1205 	if (priv->flags & EIP197_TRC_CACHE && base->ctxr &&
1206 	    (memcmp(&base->ipad, istate.state, state_sz) ||
1207 	     memcmp(&base->opad, ostate.state, state_sz)))
1208 		base->needs_inv = true;
1209 
1210 	memcpy(&base->ipad, &istate.state, state_sz);
1211 	memcpy(&base->opad, &ostate.state, state_sz);
1212 
1213 	return 0;
1214 }
1215 
1216 static int safexcel_hmac_alg_setkey(struct crypto_ahash *tfm, const u8 *key,
1217 				    unsigned int keylen, const char *alg,
1218 				    unsigned int state_sz)
1219 {
1220 	struct safexcel_ahash_ctx *ctx = crypto_ahash_ctx(tfm);
1221 
1222 	return safexcel_hmac_setkey(&ctx->base, key, keylen, alg, state_sz);
1223 }
1224 
1225 static int safexcel_hmac_sha1_setkey(struct crypto_ahash *tfm, const u8 *key,
1226 				     unsigned int keylen)
1227 {
1228 	return safexcel_hmac_alg_setkey(tfm, key, keylen, "safexcel-sha1",
1229 					SHA1_DIGEST_SIZE);
1230 }
1231 
1232 struct safexcel_alg_template safexcel_alg_hmac_sha1 = {
1233 	.type = SAFEXCEL_ALG_TYPE_AHASH,
1234 	.algo_mask = SAFEXCEL_ALG_SHA1,
1235 	.alg.ahash = {
1236 		.init = safexcel_hmac_sha1_init,
1237 		.update = safexcel_ahash_update,
1238 		.final = safexcel_ahash_final,
1239 		.finup = safexcel_ahash_finup,
1240 		.digest = safexcel_hmac_sha1_digest,
1241 		.setkey = safexcel_hmac_sha1_setkey,
1242 		.export = safexcel_ahash_export,
1243 		.import = safexcel_ahash_import,
1244 		.halg = {
1245 			.digestsize = SHA1_DIGEST_SIZE,
1246 			.statesize = sizeof(struct safexcel_ahash_export_state),
1247 			.base = {
1248 				.cra_name = "hmac(sha1)",
1249 				.cra_driver_name = "safexcel-hmac-sha1",
1250 				.cra_priority = SAFEXCEL_CRA_PRIORITY,
1251 				.cra_flags = CRYPTO_ALG_ASYNC |
1252 					     CRYPTO_ALG_ALLOCATES_MEMORY |
1253 					     CRYPTO_ALG_KERN_DRIVER_ONLY,
1254 				.cra_blocksize = SHA1_BLOCK_SIZE,
1255 				.cra_ctxsize = sizeof(struct safexcel_ahash_ctx),
1256 				.cra_init = safexcel_ahash_cra_init,
1257 				.cra_exit = safexcel_ahash_cra_exit,
1258 				.cra_module = THIS_MODULE,
1259 			},
1260 		},
1261 	},
1262 };
1263 
1264 static int safexcel_sha256_init(struct ahash_request *areq)
1265 {
1266 	struct safexcel_ahash_ctx *ctx = crypto_ahash_ctx(crypto_ahash_reqtfm(areq));
1267 	struct safexcel_ahash_req *req = ahash_request_ctx_dma(areq);
1268 
1269 	memset(req, 0, sizeof(*req));
1270 
1271 	ctx->alg = CONTEXT_CONTROL_CRYPTO_ALG_SHA256;
1272 	req->digest = CONTEXT_CONTROL_DIGEST_PRECOMPUTED;
1273 	req->state_sz = SHA256_DIGEST_SIZE;
1274 	req->digest_sz = SHA256_DIGEST_SIZE;
1275 	req->block_sz = SHA256_BLOCK_SIZE;
1276 
1277 	return 0;
1278 }
1279 
1280 static int safexcel_sha256_digest(struct ahash_request *areq)
1281 {
1282 	int ret = safexcel_sha256_init(areq);
1283 
1284 	if (ret)
1285 		return ret;
1286 
1287 	return safexcel_ahash_finup(areq);
1288 }
1289 
1290 struct safexcel_alg_template safexcel_alg_sha256 = {
1291 	.type = SAFEXCEL_ALG_TYPE_AHASH,
1292 	.algo_mask = SAFEXCEL_ALG_SHA2_256,
1293 	.alg.ahash = {
1294 		.init = safexcel_sha256_init,
1295 		.update = safexcel_ahash_update,
1296 		.final = safexcel_ahash_final,
1297 		.finup = safexcel_ahash_finup,
1298 		.digest = safexcel_sha256_digest,
1299 		.export = safexcel_ahash_export,
1300 		.import = safexcel_ahash_import,
1301 		.halg = {
1302 			.digestsize = SHA256_DIGEST_SIZE,
1303 			.statesize = sizeof(struct safexcel_ahash_export_state),
1304 			.base = {
1305 				.cra_name = "sha256",
1306 				.cra_driver_name = "safexcel-sha256",
1307 				.cra_priority = SAFEXCEL_CRA_PRIORITY,
1308 				.cra_flags = CRYPTO_ALG_ASYNC |
1309 					     CRYPTO_ALG_ALLOCATES_MEMORY |
1310 					     CRYPTO_ALG_KERN_DRIVER_ONLY,
1311 				.cra_blocksize = SHA256_BLOCK_SIZE,
1312 				.cra_ctxsize = sizeof(struct safexcel_ahash_ctx),
1313 				.cra_init = safexcel_ahash_cra_init,
1314 				.cra_exit = safexcel_ahash_cra_exit,
1315 				.cra_module = THIS_MODULE,
1316 			},
1317 		},
1318 	},
1319 };
1320 
1321 static int safexcel_sha224_init(struct ahash_request *areq)
1322 {
1323 	struct safexcel_ahash_ctx *ctx = crypto_ahash_ctx(crypto_ahash_reqtfm(areq));
1324 	struct safexcel_ahash_req *req = ahash_request_ctx_dma(areq);
1325 
1326 	memset(req, 0, sizeof(*req));
1327 
1328 	ctx->alg = CONTEXT_CONTROL_CRYPTO_ALG_SHA224;
1329 	req->digest = CONTEXT_CONTROL_DIGEST_PRECOMPUTED;
1330 	req->state_sz = SHA256_DIGEST_SIZE;
1331 	req->digest_sz = SHA256_DIGEST_SIZE;
1332 	req->block_sz = SHA256_BLOCK_SIZE;
1333 
1334 	return 0;
1335 }
1336 
1337 static int safexcel_sha224_digest(struct ahash_request *areq)
1338 {
1339 	int ret = safexcel_sha224_init(areq);
1340 
1341 	if (ret)
1342 		return ret;
1343 
1344 	return safexcel_ahash_finup(areq);
1345 }
1346 
1347 struct safexcel_alg_template safexcel_alg_sha224 = {
1348 	.type = SAFEXCEL_ALG_TYPE_AHASH,
1349 	.algo_mask = SAFEXCEL_ALG_SHA2_256,
1350 	.alg.ahash = {
1351 		.init = safexcel_sha224_init,
1352 		.update = safexcel_ahash_update,
1353 		.final = safexcel_ahash_final,
1354 		.finup = safexcel_ahash_finup,
1355 		.digest = safexcel_sha224_digest,
1356 		.export = safexcel_ahash_export,
1357 		.import = safexcel_ahash_import,
1358 		.halg = {
1359 			.digestsize = SHA224_DIGEST_SIZE,
1360 			.statesize = sizeof(struct safexcel_ahash_export_state),
1361 			.base = {
1362 				.cra_name = "sha224",
1363 				.cra_driver_name = "safexcel-sha224",
1364 				.cra_priority = SAFEXCEL_CRA_PRIORITY,
1365 				.cra_flags = CRYPTO_ALG_ASYNC |
1366 					     CRYPTO_ALG_ALLOCATES_MEMORY |
1367 					     CRYPTO_ALG_KERN_DRIVER_ONLY,
1368 				.cra_blocksize = SHA224_BLOCK_SIZE,
1369 				.cra_ctxsize = sizeof(struct safexcel_ahash_ctx),
1370 				.cra_init = safexcel_ahash_cra_init,
1371 				.cra_exit = safexcel_ahash_cra_exit,
1372 				.cra_module = THIS_MODULE,
1373 			},
1374 		},
1375 	},
1376 };
1377 
1378 static int safexcel_hmac_sha224_setkey(struct crypto_ahash *tfm, const u8 *key,
1379 				       unsigned int keylen)
1380 {
1381 	return safexcel_hmac_alg_setkey(tfm, key, keylen, "safexcel-sha224",
1382 					SHA256_DIGEST_SIZE);
1383 }
1384 
1385 static int safexcel_hmac_sha224_init(struct ahash_request *areq)
1386 {
1387 	struct safexcel_ahash_ctx *ctx = crypto_ahash_ctx(crypto_ahash_reqtfm(areq));
1388 	struct safexcel_ahash_req *req = ahash_request_ctx_dma(areq);
1389 
1390 	memset(req, 0, sizeof(*req));
1391 
1392 	/* Start from ipad precompute */
1393 	memcpy(req->state, &ctx->base.ipad, SHA256_DIGEST_SIZE);
1394 	/* Already processed the key^ipad part now! */
1395 	req->len	= SHA256_BLOCK_SIZE;
1396 	req->processed	= SHA256_BLOCK_SIZE;
1397 
1398 	ctx->alg = CONTEXT_CONTROL_CRYPTO_ALG_SHA224;
1399 	req->digest = CONTEXT_CONTROL_DIGEST_PRECOMPUTED;
1400 	req->state_sz = SHA256_DIGEST_SIZE;
1401 	req->digest_sz = SHA256_DIGEST_SIZE;
1402 	req->block_sz = SHA256_BLOCK_SIZE;
1403 	req->hmac = true;
1404 
1405 	return 0;
1406 }
1407 
1408 static int safexcel_hmac_sha224_digest(struct ahash_request *areq)
1409 {
1410 	int ret = safexcel_hmac_sha224_init(areq);
1411 
1412 	if (ret)
1413 		return ret;
1414 
1415 	return safexcel_ahash_finup(areq);
1416 }
1417 
1418 struct safexcel_alg_template safexcel_alg_hmac_sha224 = {
1419 	.type = SAFEXCEL_ALG_TYPE_AHASH,
1420 	.algo_mask = SAFEXCEL_ALG_SHA2_256,
1421 	.alg.ahash = {
1422 		.init = safexcel_hmac_sha224_init,
1423 		.update = safexcel_ahash_update,
1424 		.final = safexcel_ahash_final,
1425 		.finup = safexcel_ahash_finup,
1426 		.digest = safexcel_hmac_sha224_digest,
1427 		.setkey = safexcel_hmac_sha224_setkey,
1428 		.export = safexcel_ahash_export,
1429 		.import = safexcel_ahash_import,
1430 		.halg = {
1431 			.digestsize = SHA224_DIGEST_SIZE,
1432 			.statesize = sizeof(struct safexcel_ahash_export_state),
1433 			.base = {
1434 				.cra_name = "hmac(sha224)",
1435 				.cra_driver_name = "safexcel-hmac-sha224",
1436 				.cra_priority = SAFEXCEL_CRA_PRIORITY,
1437 				.cra_flags = CRYPTO_ALG_ASYNC |
1438 					     CRYPTO_ALG_ALLOCATES_MEMORY |
1439 					     CRYPTO_ALG_KERN_DRIVER_ONLY,
1440 				.cra_blocksize = SHA224_BLOCK_SIZE,
1441 				.cra_ctxsize = sizeof(struct safexcel_ahash_ctx),
1442 				.cra_init = safexcel_ahash_cra_init,
1443 				.cra_exit = safexcel_ahash_cra_exit,
1444 				.cra_module = THIS_MODULE,
1445 			},
1446 		},
1447 	},
1448 };
1449 
1450 static int safexcel_hmac_sha256_setkey(struct crypto_ahash *tfm, const u8 *key,
1451 				     unsigned int keylen)
1452 {
1453 	return safexcel_hmac_alg_setkey(tfm, key, keylen, "safexcel-sha256",
1454 					SHA256_DIGEST_SIZE);
1455 }
1456 
1457 static int safexcel_hmac_sha256_init(struct ahash_request *areq)
1458 {
1459 	struct safexcel_ahash_ctx *ctx = crypto_ahash_ctx(crypto_ahash_reqtfm(areq));
1460 	struct safexcel_ahash_req *req = ahash_request_ctx_dma(areq);
1461 
1462 	memset(req, 0, sizeof(*req));
1463 
1464 	/* Start from ipad precompute */
1465 	memcpy(req->state, &ctx->base.ipad, SHA256_DIGEST_SIZE);
1466 	/* Already processed the key^ipad part now! */
1467 	req->len	= SHA256_BLOCK_SIZE;
1468 	req->processed	= SHA256_BLOCK_SIZE;
1469 
1470 	ctx->alg = CONTEXT_CONTROL_CRYPTO_ALG_SHA256;
1471 	req->digest = CONTEXT_CONTROL_DIGEST_PRECOMPUTED;
1472 	req->state_sz = SHA256_DIGEST_SIZE;
1473 	req->digest_sz = SHA256_DIGEST_SIZE;
1474 	req->block_sz = SHA256_BLOCK_SIZE;
1475 	req->hmac = true;
1476 
1477 	return 0;
1478 }
1479 
1480 static int safexcel_hmac_sha256_digest(struct ahash_request *areq)
1481 {
1482 	int ret = safexcel_hmac_sha256_init(areq);
1483 
1484 	if (ret)
1485 		return ret;
1486 
1487 	return safexcel_ahash_finup(areq);
1488 }
1489 
1490 struct safexcel_alg_template safexcel_alg_hmac_sha256 = {
1491 	.type = SAFEXCEL_ALG_TYPE_AHASH,
1492 	.algo_mask = SAFEXCEL_ALG_SHA2_256,
1493 	.alg.ahash = {
1494 		.init = safexcel_hmac_sha256_init,
1495 		.update = safexcel_ahash_update,
1496 		.final = safexcel_ahash_final,
1497 		.finup = safexcel_ahash_finup,
1498 		.digest = safexcel_hmac_sha256_digest,
1499 		.setkey = safexcel_hmac_sha256_setkey,
1500 		.export = safexcel_ahash_export,
1501 		.import = safexcel_ahash_import,
1502 		.halg = {
1503 			.digestsize = SHA256_DIGEST_SIZE,
1504 			.statesize = sizeof(struct safexcel_ahash_export_state),
1505 			.base = {
1506 				.cra_name = "hmac(sha256)",
1507 				.cra_driver_name = "safexcel-hmac-sha256",
1508 				.cra_priority = SAFEXCEL_CRA_PRIORITY,
1509 				.cra_flags = CRYPTO_ALG_ASYNC |
1510 					     CRYPTO_ALG_ALLOCATES_MEMORY |
1511 					     CRYPTO_ALG_KERN_DRIVER_ONLY,
1512 				.cra_blocksize = SHA256_BLOCK_SIZE,
1513 				.cra_ctxsize = sizeof(struct safexcel_ahash_ctx),
1514 				.cra_init = safexcel_ahash_cra_init,
1515 				.cra_exit = safexcel_ahash_cra_exit,
1516 				.cra_module = THIS_MODULE,
1517 			},
1518 		},
1519 	},
1520 };
1521 
1522 static int safexcel_sha512_init(struct ahash_request *areq)
1523 {
1524 	struct safexcel_ahash_ctx *ctx = crypto_ahash_ctx(crypto_ahash_reqtfm(areq));
1525 	struct safexcel_ahash_req *req = ahash_request_ctx_dma(areq);
1526 
1527 	memset(req, 0, sizeof(*req));
1528 
1529 	ctx->alg = CONTEXT_CONTROL_CRYPTO_ALG_SHA512;
1530 	req->digest = CONTEXT_CONTROL_DIGEST_PRECOMPUTED;
1531 	req->state_sz = SHA512_DIGEST_SIZE;
1532 	req->digest_sz = SHA512_DIGEST_SIZE;
1533 	req->block_sz = SHA512_BLOCK_SIZE;
1534 
1535 	return 0;
1536 }
1537 
1538 static int safexcel_sha512_digest(struct ahash_request *areq)
1539 {
1540 	int ret = safexcel_sha512_init(areq);
1541 
1542 	if (ret)
1543 		return ret;
1544 
1545 	return safexcel_ahash_finup(areq);
1546 }
1547 
1548 struct safexcel_alg_template safexcel_alg_sha512 = {
1549 	.type = SAFEXCEL_ALG_TYPE_AHASH,
1550 	.algo_mask = SAFEXCEL_ALG_SHA2_512,
1551 	.alg.ahash = {
1552 		.init = safexcel_sha512_init,
1553 		.update = safexcel_ahash_update,
1554 		.final = safexcel_ahash_final,
1555 		.finup = safexcel_ahash_finup,
1556 		.digest = safexcel_sha512_digest,
1557 		.export = safexcel_ahash_export,
1558 		.import = safexcel_ahash_import,
1559 		.halg = {
1560 			.digestsize = SHA512_DIGEST_SIZE,
1561 			.statesize = sizeof(struct safexcel_ahash_export_state),
1562 			.base = {
1563 				.cra_name = "sha512",
1564 				.cra_driver_name = "safexcel-sha512",
1565 				.cra_priority = SAFEXCEL_CRA_PRIORITY,
1566 				.cra_flags = CRYPTO_ALG_ASYNC |
1567 					     CRYPTO_ALG_ALLOCATES_MEMORY |
1568 					     CRYPTO_ALG_KERN_DRIVER_ONLY,
1569 				.cra_blocksize = SHA512_BLOCK_SIZE,
1570 				.cra_ctxsize = sizeof(struct safexcel_ahash_ctx),
1571 				.cra_init = safexcel_ahash_cra_init,
1572 				.cra_exit = safexcel_ahash_cra_exit,
1573 				.cra_module = THIS_MODULE,
1574 			},
1575 		},
1576 	},
1577 };
1578 
1579 static int safexcel_sha384_init(struct ahash_request *areq)
1580 {
1581 	struct safexcel_ahash_ctx *ctx = crypto_ahash_ctx(crypto_ahash_reqtfm(areq));
1582 	struct safexcel_ahash_req *req = ahash_request_ctx_dma(areq);
1583 
1584 	memset(req, 0, sizeof(*req));
1585 
1586 	ctx->alg = CONTEXT_CONTROL_CRYPTO_ALG_SHA384;
1587 	req->digest = CONTEXT_CONTROL_DIGEST_PRECOMPUTED;
1588 	req->state_sz = SHA512_DIGEST_SIZE;
1589 	req->digest_sz = SHA512_DIGEST_SIZE;
1590 	req->block_sz = SHA512_BLOCK_SIZE;
1591 
1592 	return 0;
1593 }
1594 
1595 static int safexcel_sha384_digest(struct ahash_request *areq)
1596 {
1597 	int ret = safexcel_sha384_init(areq);
1598 
1599 	if (ret)
1600 		return ret;
1601 
1602 	return safexcel_ahash_finup(areq);
1603 }
1604 
1605 struct safexcel_alg_template safexcel_alg_sha384 = {
1606 	.type = SAFEXCEL_ALG_TYPE_AHASH,
1607 	.algo_mask = SAFEXCEL_ALG_SHA2_512,
1608 	.alg.ahash = {
1609 		.init = safexcel_sha384_init,
1610 		.update = safexcel_ahash_update,
1611 		.final = safexcel_ahash_final,
1612 		.finup = safexcel_ahash_finup,
1613 		.digest = safexcel_sha384_digest,
1614 		.export = safexcel_ahash_export,
1615 		.import = safexcel_ahash_import,
1616 		.halg = {
1617 			.digestsize = SHA384_DIGEST_SIZE,
1618 			.statesize = sizeof(struct safexcel_ahash_export_state),
1619 			.base = {
1620 				.cra_name = "sha384",
1621 				.cra_driver_name = "safexcel-sha384",
1622 				.cra_priority = SAFEXCEL_CRA_PRIORITY,
1623 				.cra_flags = CRYPTO_ALG_ASYNC |
1624 					     CRYPTO_ALG_ALLOCATES_MEMORY |
1625 					     CRYPTO_ALG_KERN_DRIVER_ONLY,
1626 				.cra_blocksize = SHA384_BLOCK_SIZE,
1627 				.cra_ctxsize = sizeof(struct safexcel_ahash_ctx),
1628 				.cra_init = safexcel_ahash_cra_init,
1629 				.cra_exit = safexcel_ahash_cra_exit,
1630 				.cra_module = THIS_MODULE,
1631 			},
1632 		},
1633 	},
1634 };
1635 
1636 static int safexcel_hmac_sha512_setkey(struct crypto_ahash *tfm, const u8 *key,
1637 				       unsigned int keylen)
1638 {
1639 	return safexcel_hmac_alg_setkey(tfm, key, keylen, "safexcel-sha512",
1640 					SHA512_DIGEST_SIZE);
1641 }
1642 
1643 static int safexcel_hmac_sha512_init(struct ahash_request *areq)
1644 {
1645 	struct safexcel_ahash_ctx *ctx = crypto_ahash_ctx(crypto_ahash_reqtfm(areq));
1646 	struct safexcel_ahash_req *req = ahash_request_ctx_dma(areq);
1647 
1648 	memset(req, 0, sizeof(*req));
1649 
1650 	/* Start from ipad precompute */
1651 	memcpy(req->state, &ctx->base.ipad, SHA512_DIGEST_SIZE);
1652 	/* Already processed the key^ipad part now! */
1653 	req->len	= SHA512_BLOCK_SIZE;
1654 	req->processed	= SHA512_BLOCK_SIZE;
1655 
1656 	ctx->alg = CONTEXT_CONTROL_CRYPTO_ALG_SHA512;
1657 	req->digest = CONTEXT_CONTROL_DIGEST_PRECOMPUTED;
1658 	req->state_sz = SHA512_DIGEST_SIZE;
1659 	req->digest_sz = SHA512_DIGEST_SIZE;
1660 	req->block_sz = SHA512_BLOCK_SIZE;
1661 	req->hmac = true;
1662 
1663 	return 0;
1664 }
1665 
1666 static int safexcel_hmac_sha512_digest(struct ahash_request *areq)
1667 {
1668 	int ret = safexcel_hmac_sha512_init(areq);
1669 
1670 	if (ret)
1671 		return ret;
1672 
1673 	return safexcel_ahash_finup(areq);
1674 }
1675 
1676 struct safexcel_alg_template safexcel_alg_hmac_sha512 = {
1677 	.type = SAFEXCEL_ALG_TYPE_AHASH,
1678 	.algo_mask = SAFEXCEL_ALG_SHA2_512,
1679 	.alg.ahash = {
1680 		.init = safexcel_hmac_sha512_init,
1681 		.update = safexcel_ahash_update,
1682 		.final = safexcel_ahash_final,
1683 		.finup = safexcel_ahash_finup,
1684 		.digest = safexcel_hmac_sha512_digest,
1685 		.setkey = safexcel_hmac_sha512_setkey,
1686 		.export = safexcel_ahash_export,
1687 		.import = safexcel_ahash_import,
1688 		.halg = {
1689 			.digestsize = SHA512_DIGEST_SIZE,
1690 			.statesize = sizeof(struct safexcel_ahash_export_state),
1691 			.base = {
1692 				.cra_name = "hmac(sha512)",
1693 				.cra_driver_name = "safexcel-hmac-sha512",
1694 				.cra_priority = SAFEXCEL_CRA_PRIORITY,
1695 				.cra_flags = CRYPTO_ALG_ASYNC |
1696 					     CRYPTO_ALG_ALLOCATES_MEMORY |
1697 					     CRYPTO_ALG_KERN_DRIVER_ONLY,
1698 				.cra_blocksize = SHA512_BLOCK_SIZE,
1699 				.cra_ctxsize = sizeof(struct safexcel_ahash_ctx),
1700 				.cra_init = safexcel_ahash_cra_init,
1701 				.cra_exit = safexcel_ahash_cra_exit,
1702 				.cra_module = THIS_MODULE,
1703 			},
1704 		},
1705 	},
1706 };
1707 
1708 static int safexcel_hmac_sha384_setkey(struct crypto_ahash *tfm, const u8 *key,
1709 				       unsigned int keylen)
1710 {
1711 	return safexcel_hmac_alg_setkey(tfm, key, keylen, "safexcel-sha384",
1712 					SHA512_DIGEST_SIZE);
1713 }
1714 
1715 static int safexcel_hmac_sha384_init(struct ahash_request *areq)
1716 {
1717 	struct safexcel_ahash_ctx *ctx = crypto_ahash_ctx(crypto_ahash_reqtfm(areq));
1718 	struct safexcel_ahash_req *req = ahash_request_ctx_dma(areq);
1719 
1720 	memset(req, 0, sizeof(*req));
1721 
1722 	/* Start from ipad precompute */
1723 	memcpy(req->state, &ctx->base.ipad, SHA512_DIGEST_SIZE);
1724 	/* Already processed the key^ipad part now! */
1725 	req->len	= SHA512_BLOCK_SIZE;
1726 	req->processed	= SHA512_BLOCK_SIZE;
1727 
1728 	ctx->alg = CONTEXT_CONTROL_CRYPTO_ALG_SHA384;
1729 	req->digest = CONTEXT_CONTROL_DIGEST_PRECOMPUTED;
1730 	req->state_sz = SHA512_DIGEST_SIZE;
1731 	req->digest_sz = SHA512_DIGEST_SIZE;
1732 	req->block_sz = SHA512_BLOCK_SIZE;
1733 	req->hmac = true;
1734 
1735 	return 0;
1736 }
1737 
1738 static int safexcel_hmac_sha384_digest(struct ahash_request *areq)
1739 {
1740 	int ret = safexcel_hmac_sha384_init(areq);
1741 
1742 	if (ret)
1743 		return ret;
1744 
1745 	return safexcel_ahash_finup(areq);
1746 }
1747 
1748 struct safexcel_alg_template safexcel_alg_hmac_sha384 = {
1749 	.type = SAFEXCEL_ALG_TYPE_AHASH,
1750 	.algo_mask = SAFEXCEL_ALG_SHA2_512,
1751 	.alg.ahash = {
1752 		.init = safexcel_hmac_sha384_init,
1753 		.update = safexcel_ahash_update,
1754 		.final = safexcel_ahash_final,
1755 		.finup = safexcel_ahash_finup,
1756 		.digest = safexcel_hmac_sha384_digest,
1757 		.setkey = safexcel_hmac_sha384_setkey,
1758 		.export = safexcel_ahash_export,
1759 		.import = safexcel_ahash_import,
1760 		.halg = {
1761 			.digestsize = SHA384_DIGEST_SIZE,
1762 			.statesize = sizeof(struct safexcel_ahash_export_state),
1763 			.base = {
1764 				.cra_name = "hmac(sha384)",
1765 				.cra_driver_name = "safexcel-hmac-sha384",
1766 				.cra_priority = SAFEXCEL_CRA_PRIORITY,
1767 				.cra_flags = CRYPTO_ALG_ASYNC |
1768 					     CRYPTO_ALG_ALLOCATES_MEMORY |
1769 					     CRYPTO_ALG_KERN_DRIVER_ONLY,
1770 				.cra_blocksize = SHA384_BLOCK_SIZE,
1771 				.cra_ctxsize = sizeof(struct safexcel_ahash_ctx),
1772 				.cra_init = safexcel_ahash_cra_init,
1773 				.cra_exit = safexcel_ahash_cra_exit,
1774 				.cra_module = THIS_MODULE,
1775 			},
1776 		},
1777 	},
1778 };
1779 
1780 static int safexcel_md5_init(struct ahash_request *areq)
1781 {
1782 	struct safexcel_ahash_ctx *ctx = crypto_ahash_ctx(crypto_ahash_reqtfm(areq));
1783 	struct safexcel_ahash_req *req = ahash_request_ctx_dma(areq);
1784 
1785 	memset(req, 0, sizeof(*req));
1786 
1787 	ctx->alg = CONTEXT_CONTROL_CRYPTO_ALG_MD5;
1788 	req->digest = CONTEXT_CONTROL_DIGEST_PRECOMPUTED;
1789 	req->state_sz = MD5_DIGEST_SIZE;
1790 	req->digest_sz = MD5_DIGEST_SIZE;
1791 	req->block_sz = MD5_HMAC_BLOCK_SIZE;
1792 
1793 	return 0;
1794 }
1795 
1796 static int safexcel_md5_digest(struct ahash_request *areq)
1797 {
1798 	int ret = safexcel_md5_init(areq);
1799 
1800 	if (ret)
1801 		return ret;
1802 
1803 	return safexcel_ahash_finup(areq);
1804 }
1805 
1806 struct safexcel_alg_template safexcel_alg_md5 = {
1807 	.type = SAFEXCEL_ALG_TYPE_AHASH,
1808 	.algo_mask = SAFEXCEL_ALG_MD5,
1809 	.alg.ahash = {
1810 		.init = safexcel_md5_init,
1811 		.update = safexcel_ahash_update,
1812 		.final = safexcel_ahash_final,
1813 		.finup = safexcel_ahash_finup,
1814 		.digest = safexcel_md5_digest,
1815 		.export = safexcel_ahash_export,
1816 		.import = safexcel_ahash_import,
1817 		.halg = {
1818 			.digestsize = MD5_DIGEST_SIZE,
1819 			.statesize = sizeof(struct safexcel_ahash_export_state),
1820 			.base = {
1821 				.cra_name = "md5",
1822 				.cra_driver_name = "safexcel-md5",
1823 				.cra_priority = SAFEXCEL_CRA_PRIORITY,
1824 				.cra_flags = CRYPTO_ALG_ASYNC |
1825 					     CRYPTO_ALG_ALLOCATES_MEMORY |
1826 					     CRYPTO_ALG_KERN_DRIVER_ONLY,
1827 				.cra_blocksize = MD5_HMAC_BLOCK_SIZE,
1828 				.cra_ctxsize = sizeof(struct safexcel_ahash_ctx),
1829 				.cra_init = safexcel_ahash_cra_init,
1830 				.cra_exit = safexcel_ahash_cra_exit,
1831 				.cra_module = THIS_MODULE,
1832 			},
1833 		},
1834 	},
1835 };
1836 
1837 static int safexcel_hmac_md5_init(struct ahash_request *areq)
1838 {
1839 	struct safexcel_ahash_ctx *ctx = crypto_ahash_ctx(crypto_ahash_reqtfm(areq));
1840 	struct safexcel_ahash_req *req = ahash_request_ctx_dma(areq);
1841 
1842 	memset(req, 0, sizeof(*req));
1843 
1844 	/* Start from ipad precompute */
1845 	memcpy(req->state, &ctx->base.ipad, MD5_DIGEST_SIZE);
1846 	/* Already processed the key^ipad part now! */
1847 	req->len	= MD5_HMAC_BLOCK_SIZE;
1848 	req->processed	= MD5_HMAC_BLOCK_SIZE;
1849 
1850 	ctx->alg = CONTEXT_CONTROL_CRYPTO_ALG_MD5;
1851 	req->digest = CONTEXT_CONTROL_DIGEST_PRECOMPUTED;
1852 	req->state_sz = MD5_DIGEST_SIZE;
1853 	req->digest_sz = MD5_DIGEST_SIZE;
1854 	req->block_sz = MD5_HMAC_BLOCK_SIZE;
1855 	req->len_is_le = true; /* MD5 is little endian! ... */
1856 	req->hmac = true;
1857 
1858 	return 0;
1859 }
1860 
1861 static int safexcel_hmac_md5_setkey(struct crypto_ahash *tfm, const u8 *key,
1862 				     unsigned int keylen)
1863 {
1864 	return safexcel_hmac_alg_setkey(tfm, key, keylen, "safexcel-md5",
1865 					MD5_DIGEST_SIZE);
1866 }
1867 
1868 static int safexcel_hmac_md5_digest(struct ahash_request *areq)
1869 {
1870 	int ret = safexcel_hmac_md5_init(areq);
1871 
1872 	if (ret)
1873 		return ret;
1874 
1875 	return safexcel_ahash_finup(areq);
1876 }
1877 
1878 struct safexcel_alg_template safexcel_alg_hmac_md5 = {
1879 	.type = SAFEXCEL_ALG_TYPE_AHASH,
1880 	.algo_mask = SAFEXCEL_ALG_MD5,
1881 	.alg.ahash = {
1882 		.init = safexcel_hmac_md5_init,
1883 		.update = safexcel_ahash_update,
1884 		.final = safexcel_ahash_final,
1885 		.finup = safexcel_ahash_finup,
1886 		.digest = safexcel_hmac_md5_digest,
1887 		.setkey = safexcel_hmac_md5_setkey,
1888 		.export = safexcel_ahash_export,
1889 		.import = safexcel_ahash_import,
1890 		.halg = {
1891 			.digestsize = MD5_DIGEST_SIZE,
1892 			.statesize = sizeof(struct safexcel_ahash_export_state),
1893 			.base = {
1894 				.cra_name = "hmac(md5)",
1895 				.cra_driver_name = "safexcel-hmac-md5",
1896 				.cra_priority = SAFEXCEL_CRA_PRIORITY,
1897 				.cra_flags = CRYPTO_ALG_ASYNC |
1898 					     CRYPTO_ALG_ALLOCATES_MEMORY |
1899 					     CRYPTO_ALG_KERN_DRIVER_ONLY,
1900 				.cra_blocksize = MD5_HMAC_BLOCK_SIZE,
1901 				.cra_ctxsize = sizeof(struct safexcel_ahash_ctx),
1902 				.cra_init = safexcel_ahash_cra_init,
1903 				.cra_exit = safexcel_ahash_cra_exit,
1904 				.cra_module = THIS_MODULE,
1905 			},
1906 		},
1907 	},
1908 };
1909 
1910 static int safexcel_crc32_cra_init(struct crypto_tfm *tfm)
1911 {
1912 	struct safexcel_ahash_ctx *ctx = crypto_tfm_ctx(tfm);
1913 	int ret = safexcel_ahash_cra_init(tfm);
1914 
1915 	/* Default 'key' is all zeroes */
1916 	memset(&ctx->base.ipad, 0, sizeof(u32));
1917 	return ret;
1918 }
1919 
1920 static int safexcel_crc32_init(struct ahash_request *areq)
1921 {
1922 	struct safexcel_ahash_ctx *ctx = crypto_ahash_ctx(crypto_ahash_reqtfm(areq));
1923 	struct safexcel_ahash_req *req = ahash_request_ctx_dma(areq);
1924 
1925 	memset(req, 0, sizeof(*req));
1926 
1927 	/* Start from loaded key */
1928 	req->state[0]	= cpu_to_le32(~ctx->base.ipad.word[0]);
1929 	/* Set processed to non-zero to enable invalidation detection */
1930 	req->len	= sizeof(u32);
1931 	req->processed	= sizeof(u32);
1932 
1933 	ctx->alg = CONTEXT_CONTROL_CRYPTO_ALG_CRC32;
1934 	req->digest = CONTEXT_CONTROL_DIGEST_XCM;
1935 	req->state_sz = sizeof(u32);
1936 	req->digest_sz = sizeof(u32);
1937 	req->block_sz = sizeof(u32);
1938 
1939 	return 0;
1940 }
1941 
1942 static int safexcel_crc32_setkey(struct crypto_ahash *tfm, const u8 *key,
1943 				 unsigned int keylen)
1944 {
1945 	struct safexcel_ahash_ctx *ctx = crypto_tfm_ctx(crypto_ahash_tfm(tfm));
1946 
1947 	if (keylen != sizeof(u32))
1948 		return -EINVAL;
1949 
1950 	memcpy(&ctx->base.ipad, key, sizeof(u32));
1951 	return 0;
1952 }
1953 
1954 static int safexcel_crc32_digest(struct ahash_request *areq)
1955 {
1956 	return safexcel_crc32_init(areq) ?: safexcel_ahash_finup(areq);
1957 }
1958 
1959 struct safexcel_alg_template safexcel_alg_crc32 = {
1960 	.type = SAFEXCEL_ALG_TYPE_AHASH,
1961 	.algo_mask = 0,
1962 	.alg.ahash = {
1963 		.init = safexcel_crc32_init,
1964 		.update = safexcel_ahash_update,
1965 		.final = safexcel_ahash_final,
1966 		.finup = safexcel_ahash_finup,
1967 		.digest = safexcel_crc32_digest,
1968 		.setkey = safexcel_crc32_setkey,
1969 		.export = safexcel_ahash_export,
1970 		.import = safexcel_ahash_import,
1971 		.halg = {
1972 			.digestsize = sizeof(u32),
1973 			.statesize = sizeof(struct safexcel_ahash_export_state),
1974 			.base = {
1975 				.cra_name = "crc32",
1976 				.cra_driver_name = "safexcel-crc32",
1977 				.cra_priority = SAFEXCEL_CRA_PRIORITY,
1978 				.cra_flags = CRYPTO_ALG_OPTIONAL_KEY |
1979 					     CRYPTO_ALG_ASYNC |
1980 					     CRYPTO_ALG_ALLOCATES_MEMORY |
1981 					     CRYPTO_ALG_KERN_DRIVER_ONLY,
1982 				.cra_blocksize = 1,
1983 				.cra_ctxsize = sizeof(struct safexcel_ahash_ctx),
1984 				.cra_init = safexcel_crc32_cra_init,
1985 				.cra_exit = safexcel_ahash_cra_exit,
1986 				.cra_module = THIS_MODULE,
1987 			},
1988 		},
1989 	},
1990 };
1991 
1992 static int safexcel_cbcmac_init(struct ahash_request *areq)
1993 {
1994 	struct safexcel_ahash_ctx *ctx = crypto_ahash_ctx(crypto_ahash_reqtfm(areq));
1995 	struct safexcel_ahash_req *req = ahash_request_ctx_dma(areq);
1996 
1997 	memset(req, 0, sizeof(*req));
1998 
1999 	/* Start from loaded keys */
2000 	memcpy(req->state, &ctx->base.ipad, ctx->key_sz);
2001 	/* Set processed to non-zero to enable invalidation detection */
2002 	req->len	= AES_BLOCK_SIZE;
2003 	req->processed	= AES_BLOCK_SIZE;
2004 
2005 	req->digest   = CONTEXT_CONTROL_DIGEST_XCM;
2006 	req->state_sz = ctx->key_sz;
2007 	req->digest_sz = AES_BLOCK_SIZE;
2008 	req->block_sz = AES_BLOCK_SIZE;
2009 	req->xcbcmac  = true;
2010 
2011 	return 0;
2012 }
2013 
2014 static int safexcel_cbcmac_setkey(struct crypto_ahash *tfm, const u8 *key,
2015 				 unsigned int len)
2016 {
2017 	struct safexcel_ahash_ctx *ctx = crypto_tfm_ctx(crypto_ahash_tfm(tfm));
2018 	struct crypto_aes_ctx aes;
2019 	int ret, i;
2020 
2021 	ret = aes_expandkey(&aes, key, len);
2022 	if (ret)
2023 		return ret;
2024 
2025 	memset(&ctx->base.ipad, 0, 2 * AES_BLOCK_SIZE);
2026 	for (i = 0; i < len / sizeof(u32); i++)
2027 		ctx->base.ipad.be[i + 8] = cpu_to_be32(aes.key_enc[i]);
2028 
2029 	if (len == AES_KEYSIZE_192) {
2030 		ctx->alg    = CONTEXT_CONTROL_CRYPTO_ALG_XCBC192;
2031 		ctx->key_sz = AES_MAX_KEY_SIZE + 2 * AES_BLOCK_SIZE;
2032 	} else if (len == AES_KEYSIZE_256) {
2033 		ctx->alg    = CONTEXT_CONTROL_CRYPTO_ALG_XCBC256;
2034 		ctx->key_sz = AES_MAX_KEY_SIZE + 2 * AES_BLOCK_SIZE;
2035 	} else {
2036 		ctx->alg    = CONTEXT_CONTROL_CRYPTO_ALG_XCBC128;
2037 		ctx->key_sz = AES_MIN_KEY_SIZE + 2 * AES_BLOCK_SIZE;
2038 	}
2039 	ctx->cbcmac  = true;
2040 
2041 	memzero_explicit(&aes, sizeof(aes));
2042 	return 0;
2043 }
2044 
2045 static int safexcel_cbcmac_digest(struct ahash_request *areq)
2046 {
2047 	return safexcel_cbcmac_init(areq) ?: safexcel_ahash_finup(areq);
2048 }
2049 
2050 struct safexcel_alg_template safexcel_alg_cbcmac = {
2051 	.type = SAFEXCEL_ALG_TYPE_AHASH,
2052 	.algo_mask = 0,
2053 	.alg.ahash = {
2054 		.init = safexcel_cbcmac_init,
2055 		.update = safexcel_ahash_update,
2056 		.final = safexcel_ahash_final,
2057 		.finup = safexcel_ahash_finup,
2058 		.digest = safexcel_cbcmac_digest,
2059 		.setkey = safexcel_cbcmac_setkey,
2060 		.export = safexcel_ahash_export,
2061 		.import = safexcel_ahash_import,
2062 		.halg = {
2063 			.digestsize = AES_BLOCK_SIZE,
2064 			.statesize = sizeof(struct safexcel_ahash_export_state),
2065 			.base = {
2066 				.cra_name = "cbcmac(aes)",
2067 				.cra_driver_name = "safexcel-cbcmac-aes",
2068 				.cra_priority = SAFEXCEL_CRA_PRIORITY,
2069 				.cra_flags = CRYPTO_ALG_ASYNC |
2070 					     CRYPTO_ALG_ALLOCATES_MEMORY |
2071 					     CRYPTO_ALG_KERN_DRIVER_ONLY,
2072 				.cra_blocksize = 1,
2073 				.cra_ctxsize = sizeof(struct safexcel_ahash_ctx),
2074 				.cra_init = safexcel_ahash_cra_init,
2075 				.cra_exit = safexcel_ahash_cra_exit,
2076 				.cra_module = THIS_MODULE,
2077 			},
2078 		},
2079 	},
2080 };
2081 
2082 static int safexcel_xcbcmac_setkey(struct crypto_ahash *tfm, const u8 *key,
2083 				 unsigned int len)
2084 {
2085 	struct safexcel_ahash_ctx *ctx = crypto_tfm_ctx(crypto_ahash_tfm(tfm));
2086 	u32 key_tmp[3 * AES_BLOCK_SIZE / sizeof(u32)];
2087 	int ret, i;
2088 
2089 	ret = aes_expandkey(ctx->aes, key, len);
2090 	if (ret)
2091 		return ret;
2092 
2093 	/* precompute the XCBC key material */
2094 	aes_encrypt(ctx->aes, (u8 *)key_tmp + 2 * AES_BLOCK_SIZE,
2095 		    "\x1\x1\x1\x1\x1\x1\x1\x1\x1\x1\x1\x1\x1\x1\x1\x1");
2096 	aes_encrypt(ctx->aes, (u8 *)key_tmp,
2097 		    "\x2\x2\x2\x2\x2\x2\x2\x2\x2\x2\x2\x2\x2\x2\x2\x2");
2098 	aes_encrypt(ctx->aes, (u8 *)key_tmp + AES_BLOCK_SIZE,
2099 		    "\x3\x3\x3\x3\x3\x3\x3\x3\x3\x3\x3\x3\x3\x3\x3\x3");
2100 	for (i = 0; i < 3 * AES_BLOCK_SIZE / sizeof(u32); i++)
2101 		ctx->base.ipad.word[i] = swab32(key_tmp[i]);
2102 
2103 	ret = aes_expandkey(ctx->aes,
2104 			    (u8 *)key_tmp + 2 * AES_BLOCK_SIZE,
2105 			    AES_MIN_KEY_SIZE);
2106 	if (ret)
2107 		return ret;
2108 
2109 	ctx->alg    = CONTEXT_CONTROL_CRYPTO_ALG_XCBC128;
2110 	ctx->key_sz = AES_MIN_KEY_SIZE + 2 * AES_BLOCK_SIZE;
2111 	ctx->cbcmac = false;
2112 
2113 	return 0;
2114 }
2115 
2116 static int safexcel_xcbcmac_cra_init(struct crypto_tfm *tfm)
2117 {
2118 	struct safexcel_ahash_ctx *ctx = crypto_tfm_ctx(tfm);
2119 
2120 	safexcel_ahash_cra_init(tfm);
2121 	ctx->aes = kmalloc(sizeof(*ctx->aes), GFP_KERNEL);
2122 	return PTR_ERR_OR_ZERO(ctx->aes);
2123 }
2124 
2125 static void safexcel_xcbcmac_cra_exit(struct crypto_tfm *tfm)
2126 {
2127 	struct safexcel_ahash_ctx *ctx = crypto_tfm_ctx(tfm);
2128 
2129 	kfree(ctx->aes);
2130 	safexcel_ahash_cra_exit(tfm);
2131 }
2132 
2133 struct safexcel_alg_template safexcel_alg_xcbcmac = {
2134 	.type = SAFEXCEL_ALG_TYPE_AHASH,
2135 	.algo_mask = 0,
2136 	.alg.ahash = {
2137 		.init = safexcel_cbcmac_init,
2138 		.update = safexcel_ahash_update,
2139 		.final = safexcel_ahash_final,
2140 		.finup = safexcel_ahash_finup,
2141 		.digest = safexcel_cbcmac_digest,
2142 		.setkey = safexcel_xcbcmac_setkey,
2143 		.export = safexcel_ahash_export,
2144 		.import = safexcel_ahash_import,
2145 		.halg = {
2146 			.digestsize = AES_BLOCK_SIZE,
2147 			.statesize = sizeof(struct safexcel_ahash_export_state),
2148 			.base = {
2149 				.cra_name = "xcbc(aes)",
2150 				.cra_driver_name = "safexcel-xcbc-aes",
2151 				.cra_priority = SAFEXCEL_CRA_PRIORITY,
2152 				.cra_flags = CRYPTO_ALG_ASYNC |
2153 					     CRYPTO_ALG_ALLOCATES_MEMORY |
2154 					     CRYPTO_ALG_KERN_DRIVER_ONLY,
2155 				.cra_blocksize = AES_BLOCK_SIZE,
2156 				.cra_ctxsize = sizeof(struct safexcel_ahash_ctx),
2157 				.cra_init = safexcel_xcbcmac_cra_init,
2158 				.cra_exit = safexcel_xcbcmac_cra_exit,
2159 				.cra_module = THIS_MODULE,
2160 			},
2161 		},
2162 	},
2163 };
2164 
2165 static int safexcel_cmac_setkey(struct crypto_ahash *tfm, const u8 *key,
2166 				unsigned int len)
2167 {
2168 	struct safexcel_ahash_ctx *ctx = crypto_tfm_ctx(crypto_ahash_tfm(tfm));
2169 	__be64 consts[4];
2170 	u64 _const[2];
2171 	u8 msb_mask, gfmask;
2172 	int ret, i;
2173 
2174 	/* precompute the CMAC key material */
2175 	ret = aes_expandkey(ctx->aes, key, len);
2176 	if (ret)
2177 		return ret;
2178 
2179 	for (i = 0; i < len / sizeof(u32); i++)
2180 		ctx->base.ipad.word[i + 8] = swab32(ctx->aes->key_enc[i]);
2181 
2182 	/* code below borrowed from crypto/cmac.c */
2183 	/* encrypt the zero block */
2184 	memset(consts, 0, AES_BLOCK_SIZE);
2185 	aes_encrypt(ctx->aes, (u8 *)consts, (u8 *)consts);
2186 
2187 	gfmask = 0x87;
2188 	_const[0] = be64_to_cpu(consts[1]);
2189 	_const[1] = be64_to_cpu(consts[0]);
2190 
2191 	/* gf(2^128) multiply zero-ciphertext with u and u^2 */
2192 	for (i = 0; i < 4; i += 2) {
2193 		msb_mask = ((s64)_const[1] >> 63) & gfmask;
2194 		_const[1] = (_const[1] << 1) | (_const[0] >> 63);
2195 		_const[0] = (_const[0] << 1) ^ msb_mask;
2196 
2197 		consts[i + 0] = cpu_to_be64(_const[1]);
2198 		consts[i + 1] = cpu_to_be64(_const[0]);
2199 	}
2200 	/* end of code borrowed from crypto/cmac.c */
2201 
2202 	for (i = 0; i < 2 * AES_BLOCK_SIZE / sizeof(u32); i++)
2203 		ctx->base.ipad.be[i] = cpu_to_be32(((u32 *)consts)[i]);
2204 
2205 	if (len == AES_KEYSIZE_192) {
2206 		ctx->alg    = CONTEXT_CONTROL_CRYPTO_ALG_XCBC192;
2207 		ctx->key_sz = AES_MAX_KEY_SIZE + 2 * AES_BLOCK_SIZE;
2208 	} else if (len == AES_KEYSIZE_256) {
2209 		ctx->alg    = CONTEXT_CONTROL_CRYPTO_ALG_XCBC256;
2210 		ctx->key_sz = AES_MAX_KEY_SIZE + 2 * AES_BLOCK_SIZE;
2211 	} else {
2212 		ctx->alg    = CONTEXT_CONTROL_CRYPTO_ALG_XCBC128;
2213 		ctx->key_sz = AES_MIN_KEY_SIZE + 2 * AES_BLOCK_SIZE;
2214 	}
2215 	ctx->cbcmac = false;
2216 
2217 	return 0;
2218 }
2219 
2220 struct safexcel_alg_template safexcel_alg_cmac = {
2221 	.type = SAFEXCEL_ALG_TYPE_AHASH,
2222 	.algo_mask = 0,
2223 	.alg.ahash = {
2224 		.init = safexcel_cbcmac_init,
2225 		.update = safexcel_ahash_update,
2226 		.final = safexcel_ahash_final,
2227 		.finup = safexcel_ahash_finup,
2228 		.digest = safexcel_cbcmac_digest,
2229 		.setkey = safexcel_cmac_setkey,
2230 		.export = safexcel_ahash_export,
2231 		.import = safexcel_ahash_import,
2232 		.halg = {
2233 			.digestsize = AES_BLOCK_SIZE,
2234 			.statesize = sizeof(struct safexcel_ahash_export_state),
2235 			.base = {
2236 				.cra_name = "cmac(aes)",
2237 				.cra_driver_name = "safexcel-cmac-aes",
2238 				.cra_priority = SAFEXCEL_CRA_PRIORITY,
2239 				.cra_flags = CRYPTO_ALG_ASYNC |
2240 					     CRYPTO_ALG_ALLOCATES_MEMORY |
2241 					     CRYPTO_ALG_KERN_DRIVER_ONLY,
2242 				.cra_blocksize = AES_BLOCK_SIZE,
2243 				.cra_ctxsize = sizeof(struct safexcel_ahash_ctx),
2244 				.cra_init = safexcel_xcbcmac_cra_init,
2245 				.cra_exit = safexcel_xcbcmac_cra_exit,
2246 				.cra_module = THIS_MODULE,
2247 			},
2248 		},
2249 	},
2250 };
2251 
2252 static int safexcel_sm3_init(struct ahash_request *areq)
2253 {
2254 	struct safexcel_ahash_ctx *ctx = crypto_ahash_ctx(crypto_ahash_reqtfm(areq));
2255 	struct safexcel_ahash_req *req = ahash_request_ctx_dma(areq);
2256 
2257 	memset(req, 0, sizeof(*req));
2258 
2259 	ctx->alg = CONTEXT_CONTROL_CRYPTO_ALG_SM3;
2260 	req->digest = CONTEXT_CONTROL_DIGEST_PRECOMPUTED;
2261 	req->state_sz = SM3_DIGEST_SIZE;
2262 	req->digest_sz = SM3_DIGEST_SIZE;
2263 	req->block_sz = SM3_BLOCK_SIZE;
2264 
2265 	return 0;
2266 }
2267 
2268 static int safexcel_sm3_digest(struct ahash_request *areq)
2269 {
2270 	int ret = safexcel_sm3_init(areq);
2271 
2272 	if (ret)
2273 		return ret;
2274 
2275 	return safexcel_ahash_finup(areq);
2276 }
2277 
2278 struct safexcel_alg_template safexcel_alg_sm3 = {
2279 	.type = SAFEXCEL_ALG_TYPE_AHASH,
2280 	.algo_mask = SAFEXCEL_ALG_SM3,
2281 	.alg.ahash = {
2282 		.init = safexcel_sm3_init,
2283 		.update = safexcel_ahash_update,
2284 		.final = safexcel_ahash_final,
2285 		.finup = safexcel_ahash_finup,
2286 		.digest = safexcel_sm3_digest,
2287 		.export = safexcel_ahash_export,
2288 		.import = safexcel_ahash_import,
2289 		.halg = {
2290 			.digestsize = SM3_DIGEST_SIZE,
2291 			.statesize = sizeof(struct safexcel_ahash_export_state),
2292 			.base = {
2293 				.cra_name = "sm3",
2294 				.cra_driver_name = "safexcel-sm3",
2295 				.cra_priority = SAFEXCEL_CRA_PRIORITY,
2296 				.cra_flags = CRYPTO_ALG_ASYNC |
2297 					     CRYPTO_ALG_ALLOCATES_MEMORY |
2298 					     CRYPTO_ALG_KERN_DRIVER_ONLY,
2299 				.cra_blocksize = SM3_BLOCK_SIZE,
2300 				.cra_ctxsize = sizeof(struct safexcel_ahash_ctx),
2301 				.cra_init = safexcel_ahash_cra_init,
2302 				.cra_exit = safexcel_ahash_cra_exit,
2303 				.cra_module = THIS_MODULE,
2304 			},
2305 		},
2306 	},
2307 };
2308 
2309 static int safexcel_hmac_sm3_setkey(struct crypto_ahash *tfm, const u8 *key,
2310 				    unsigned int keylen)
2311 {
2312 	return safexcel_hmac_alg_setkey(tfm, key, keylen, "safexcel-sm3",
2313 					SM3_DIGEST_SIZE);
2314 }
2315 
2316 static int safexcel_hmac_sm3_init(struct ahash_request *areq)
2317 {
2318 	struct safexcel_ahash_ctx *ctx = crypto_ahash_ctx(crypto_ahash_reqtfm(areq));
2319 	struct safexcel_ahash_req *req = ahash_request_ctx_dma(areq);
2320 
2321 	memset(req, 0, sizeof(*req));
2322 
2323 	/* Start from ipad precompute */
2324 	memcpy(req->state, &ctx->base.ipad, SM3_DIGEST_SIZE);
2325 	/* Already processed the key^ipad part now! */
2326 	req->len	= SM3_BLOCK_SIZE;
2327 	req->processed	= SM3_BLOCK_SIZE;
2328 
2329 	ctx->alg = CONTEXT_CONTROL_CRYPTO_ALG_SM3;
2330 	req->digest = CONTEXT_CONTROL_DIGEST_PRECOMPUTED;
2331 	req->state_sz = SM3_DIGEST_SIZE;
2332 	req->digest_sz = SM3_DIGEST_SIZE;
2333 	req->block_sz = SM3_BLOCK_SIZE;
2334 	req->hmac = true;
2335 
2336 	return 0;
2337 }
2338 
2339 static int safexcel_hmac_sm3_digest(struct ahash_request *areq)
2340 {
2341 	int ret = safexcel_hmac_sm3_init(areq);
2342 
2343 	if (ret)
2344 		return ret;
2345 
2346 	return safexcel_ahash_finup(areq);
2347 }
2348 
2349 struct safexcel_alg_template safexcel_alg_hmac_sm3 = {
2350 	.type = SAFEXCEL_ALG_TYPE_AHASH,
2351 	.algo_mask = SAFEXCEL_ALG_SM3,
2352 	.alg.ahash = {
2353 		.init = safexcel_hmac_sm3_init,
2354 		.update = safexcel_ahash_update,
2355 		.final = safexcel_ahash_final,
2356 		.finup = safexcel_ahash_finup,
2357 		.digest = safexcel_hmac_sm3_digest,
2358 		.setkey = safexcel_hmac_sm3_setkey,
2359 		.export = safexcel_ahash_export,
2360 		.import = safexcel_ahash_import,
2361 		.halg = {
2362 			.digestsize = SM3_DIGEST_SIZE,
2363 			.statesize = sizeof(struct safexcel_ahash_export_state),
2364 			.base = {
2365 				.cra_name = "hmac(sm3)",
2366 				.cra_driver_name = "safexcel-hmac-sm3",
2367 				.cra_priority = SAFEXCEL_CRA_PRIORITY,
2368 				.cra_flags = CRYPTO_ALG_ASYNC |
2369 					     CRYPTO_ALG_ALLOCATES_MEMORY |
2370 					     CRYPTO_ALG_KERN_DRIVER_ONLY,
2371 				.cra_blocksize = SM3_BLOCK_SIZE,
2372 				.cra_ctxsize = sizeof(struct safexcel_ahash_ctx),
2373 				.cra_init = safexcel_ahash_cra_init,
2374 				.cra_exit = safexcel_ahash_cra_exit,
2375 				.cra_module = THIS_MODULE,
2376 			},
2377 		},
2378 	},
2379 };
2380 
2381 static int safexcel_sha3_224_init(struct ahash_request *areq)
2382 {
2383 	struct crypto_ahash *tfm = crypto_ahash_reqtfm(areq);
2384 	struct safexcel_ahash_ctx *ctx = crypto_ahash_ctx(tfm);
2385 	struct safexcel_ahash_req *req = ahash_request_ctx_dma(areq);
2386 
2387 	memset(req, 0, sizeof(*req));
2388 
2389 	ctx->alg = CONTEXT_CONTROL_CRYPTO_ALG_SHA3_224;
2390 	req->digest = CONTEXT_CONTROL_DIGEST_INITIAL;
2391 	req->state_sz = SHA3_224_DIGEST_SIZE;
2392 	req->digest_sz = SHA3_224_DIGEST_SIZE;
2393 	req->block_sz = SHA3_224_BLOCK_SIZE;
2394 	ctx->do_fallback = false;
2395 	ctx->fb_init_done = false;
2396 	return 0;
2397 }
2398 
2399 static int safexcel_sha3_fbcheck(struct ahash_request *req)
2400 {
2401 	struct crypto_ahash *tfm = crypto_ahash_reqtfm(req);
2402 	struct safexcel_ahash_ctx *ctx = crypto_ahash_ctx(tfm);
2403 	struct ahash_request *subreq = ahash_request_ctx_dma(req);
2404 	int ret = 0;
2405 
2406 	if (ctx->do_fallback) {
2407 		ahash_request_set_tfm(subreq, ctx->fback);
2408 		ahash_request_set_callback(subreq, req->base.flags,
2409 					   req->base.complete, req->base.data);
2410 		ahash_request_set_crypt(subreq, req->src, req->result,
2411 					req->nbytes);
2412 		if (!ctx->fb_init_done) {
2413 			if (ctx->fb_do_setkey) {
2414 				/* Set fallback cipher HMAC key */
2415 				u8 key[SHA3_224_BLOCK_SIZE];
2416 
2417 				memcpy(key, &ctx->base.ipad,
2418 				       crypto_ahash_blocksize(ctx->fback) / 2);
2419 				memcpy(key +
2420 				       crypto_ahash_blocksize(ctx->fback) / 2,
2421 				       &ctx->base.opad,
2422 				       crypto_ahash_blocksize(ctx->fback) / 2);
2423 				ret = crypto_ahash_setkey(ctx->fback, key,
2424 					crypto_ahash_blocksize(ctx->fback));
2425 				memzero_explicit(key,
2426 					crypto_ahash_blocksize(ctx->fback));
2427 				ctx->fb_do_setkey = false;
2428 			}
2429 			ret = ret ?: crypto_ahash_init(subreq);
2430 			ctx->fb_init_done = true;
2431 		}
2432 	}
2433 	return ret;
2434 }
2435 
2436 static int safexcel_sha3_update(struct ahash_request *req)
2437 {
2438 	struct crypto_ahash *tfm = crypto_ahash_reqtfm(req);
2439 	struct safexcel_ahash_ctx *ctx = crypto_ahash_ctx(tfm);
2440 	struct ahash_request *subreq = ahash_request_ctx_dma(req);
2441 
2442 	ctx->do_fallback = true;
2443 	return safexcel_sha3_fbcheck(req) ?: crypto_ahash_update(subreq);
2444 }
2445 
2446 static int safexcel_sha3_final(struct ahash_request *req)
2447 {
2448 	struct crypto_ahash *tfm = crypto_ahash_reqtfm(req);
2449 	struct safexcel_ahash_ctx *ctx = crypto_ahash_ctx(tfm);
2450 	struct ahash_request *subreq = ahash_request_ctx_dma(req);
2451 
2452 	ctx->do_fallback = true;
2453 	return safexcel_sha3_fbcheck(req) ?: crypto_ahash_final(subreq);
2454 }
2455 
2456 static int safexcel_sha3_finup(struct ahash_request *req)
2457 {
2458 	struct crypto_ahash *tfm = crypto_ahash_reqtfm(req);
2459 	struct safexcel_ahash_ctx *ctx = crypto_ahash_ctx(tfm);
2460 	struct ahash_request *subreq = ahash_request_ctx_dma(req);
2461 
2462 	ctx->do_fallback |= !req->nbytes;
2463 	if (ctx->do_fallback)
2464 		/* Update or ex/import happened or len 0, cannot use the HW */
2465 		return safexcel_sha3_fbcheck(req) ?:
2466 		       crypto_ahash_finup(subreq);
2467 	else
2468 		return safexcel_ahash_finup(req);
2469 }
2470 
2471 static int safexcel_sha3_digest_fallback(struct ahash_request *req)
2472 {
2473 	struct crypto_ahash *tfm = crypto_ahash_reqtfm(req);
2474 	struct safexcel_ahash_ctx *ctx = crypto_ahash_ctx(tfm);
2475 	struct ahash_request *subreq = ahash_request_ctx_dma(req);
2476 
2477 	ctx->do_fallback = true;
2478 	ctx->fb_init_done = false;
2479 	return safexcel_sha3_fbcheck(req) ?: crypto_ahash_finup(subreq);
2480 }
2481 
2482 static int safexcel_sha3_224_digest(struct ahash_request *req)
2483 {
2484 	if (req->nbytes)
2485 		return safexcel_sha3_224_init(req) ?: safexcel_ahash_finup(req);
2486 
2487 	/* HW cannot do zero length hash, use fallback instead */
2488 	return safexcel_sha3_digest_fallback(req);
2489 }
2490 
2491 static int safexcel_sha3_export(struct ahash_request *req, void *out)
2492 {
2493 	struct crypto_ahash *tfm = crypto_ahash_reqtfm(req);
2494 	struct safexcel_ahash_ctx *ctx = crypto_ahash_ctx(tfm);
2495 	struct ahash_request *subreq = ahash_request_ctx_dma(req);
2496 
2497 	ctx->do_fallback = true;
2498 	return safexcel_sha3_fbcheck(req) ?: crypto_ahash_export(subreq, out);
2499 }
2500 
2501 static int safexcel_sha3_import(struct ahash_request *req, const void *in)
2502 {
2503 	struct crypto_ahash *tfm = crypto_ahash_reqtfm(req);
2504 	struct safexcel_ahash_ctx *ctx = crypto_ahash_ctx(tfm);
2505 	struct ahash_request *subreq = ahash_request_ctx_dma(req);
2506 
2507 	ctx->do_fallback = true;
2508 	return safexcel_sha3_fbcheck(req) ?: crypto_ahash_import(subreq, in);
2509 	// return safexcel_ahash_import(req, in);
2510 }
2511 
2512 static int safexcel_sha3_cra_init(struct crypto_tfm *tfm)
2513 {
2514 	struct crypto_ahash *ahash = __crypto_ahash_cast(tfm);
2515 	struct safexcel_ahash_ctx *ctx = crypto_tfm_ctx(tfm);
2516 
2517 	safexcel_ahash_cra_init(tfm);
2518 
2519 	/* Allocate fallback implementation */
2520 	ctx->fback = crypto_alloc_ahash(crypto_tfm_alg_name(tfm), 0,
2521 					CRYPTO_ALG_ASYNC |
2522 					CRYPTO_ALG_NEED_FALLBACK);
2523 	if (IS_ERR(ctx->fback))
2524 		return PTR_ERR(ctx->fback);
2525 
2526 	/* Update statesize from fallback algorithm! */
2527 	crypto_hash_alg_common(ahash)->statesize =
2528 		crypto_ahash_statesize(ctx->fback);
2529 	crypto_ahash_set_reqsize_dma(
2530 		ahash, max(sizeof(struct safexcel_ahash_req),
2531 			   sizeof(struct ahash_request) +
2532 			   crypto_ahash_reqsize(ctx->fback)));
2533 	return 0;
2534 }
2535 
2536 static void safexcel_sha3_cra_exit(struct crypto_tfm *tfm)
2537 {
2538 	struct safexcel_ahash_ctx *ctx = crypto_tfm_ctx(tfm);
2539 
2540 	crypto_free_ahash(ctx->fback);
2541 	safexcel_ahash_cra_exit(tfm);
2542 }
2543 
2544 struct safexcel_alg_template safexcel_alg_sha3_224 = {
2545 	.type = SAFEXCEL_ALG_TYPE_AHASH,
2546 	.algo_mask = SAFEXCEL_ALG_SHA3,
2547 	.alg.ahash = {
2548 		.init = safexcel_sha3_224_init,
2549 		.update = safexcel_sha3_update,
2550 		.final = safexcel_sha3_final,
2551 		.finup = safexcel_sha3_finup,
2552 		.digest = safexcel_sha3_224_digest,
2553 		.export = safexcel_sha3_export,
2554 		.import = safexcel_sha3_import,
2555 		.halg = {
2556 			.digestsize = SHA3_224_DIGEST_SIZE,
2557 			.statesize = sizeof(struct safexcel_ahash_export_state),
2558 			.base = {
2559 				.cra_name = "sha3-224",
2560 				.cra_driver_name = "safexcel-sha3-224",
2561 				.cra_priority = SAFEXCEL_CRA_PRIORITY,
2562 				.cra_flags = CRYPTO_ALG_ASYNC |
2563 					     CRYPTO_ALG_KERN_DRIVER_ONLY |
2564 					     CRYPTO_ALG_NEED_FALLBACK,
2565 				.cra_blocksize = SHA3_224_BLOCK_SIZE,
2566 				.cra_ctxsize = sizeof(struct safexcel_ahash_ctx),
2567 				.cra_init = safexcel_sha3_cra_init,
2568 				.cra_exit = safexcel_sha3_cra_exit,
2569 				.cra_module = THIS_MODULE,
2570 			},
2571 		},
2572 	},
2573 };
2574 
2575 static int safexcel_sha3_256_init(struct ahash_request *areq)
2576 {
2577 	struct crypto_ahash *tfm = crypto_ahash_reqtfm(areq);
2578 	struct safexcel_ahash_ctx *ctx = crypto_ahash_ctx(tfm);
2579 	struct safexcel_ahash_req *req = ahash_request_ctx_dma(areq);
2580 
2581 	memset(req, 0, sizeof(*req));
2582 
2583 	ctx->alg = CONTEXT_CONTROL_CRYPTO_ALG_SHA3_256;
2584 	req->digest = CONTEXT_CONTROL_DIGEST_INITIAL;
2585 	req->state_sz = SHA3_256_DIGEST_SIZE;
2586 	req->digest_sz = SHA3_256_DIGEST_SIZE;
2587 	req->block_sz = SHA3_256_BLOCK_SIZE;
2588 	ctx->do_fallback = false;
2589 	ctx->fb_init_done = false;
2590 	return 0;
2591 }
2592 
2593 static int safexcel_sha3_256_digest(struct ahash_request *req)
2594 {
2595 	if (req->nbytes)
2596 		return safexcel_sha3_256_init(req) ?: safexcel_ahash_finup(req);
2597 
2598 	/* HW cannot do zero length hash, use fallback instead */
2599 	return safexcel_sha3_digest_fallback(req);
2600 }
2601 
2602 struct safexcel_alg_template safexcel_alg_sha3_256 = {
2603 	.type = SAFEXCEL_ALG_TYPE_AHASH,
2604 	.algo_mask = SAFEXCEL_ALG_SHA3,
2605 	.alg.ahash = {
2606 		.init = safexcel_sha3_256_init,
2607 		.update = safexcel_sha3_update,
2608 		.final = safexcel_sha3_final,
2609 		.finup = safexcel_sha3_finup,
2610 		.digest = safexcel_sha3_256_digest,
2611 		.export = safexcel_sha3_export,
2612 		.import = safexcel_sha3_import,
2613 		.halg = {
2614 			.digestsize = SHA3_256_DIGEST_SIZE,
2615 			.statesize = sizeof(struct safexcel_ahash_export_state),
2616 			.base = {
2617 				.cra_name = "sha3-256",
2618 				.cra_driver_name = "safexcel-sha3-256",
2619 				.cra_priority = SAFEXCEL_CRA_PRIORITY,
2620 				.cra_flags = CRYPTO_ALG_ASYNC |
2621 					     CRYPTO_ALG_KERN_DRIVER_ONLY |
2622 					     CRYPTO_ALG_NEED_FALLBACK,
2623 				.cra_blocksize = SHA3_256_BLOCK_SIZE,
2624 				.cra_ctxsize = sizeof(struct safexcel_ahash_ctx),
2625 				.cra_init = safexcel_sha3_cra_init,
2626 				.cra_exit = safexcel_sha3_cra_exit,
2627 				.cra_module = THIS_MODULE,
2628 			},
2629 		},
2630 	},
2631 };
2632 
2633 static int safexcel_sha3_384_init(struct ahash_request *areq)
2634 {
2635 	struct crypto_ahash *tfm = crypto_ahash_reqtfm(areq);
2636 	struct safexcel_ahash_ctx *ctx = crypto_ahash_ctx(tfm);
2637 	struct safexcel_ahash_req *req = ahash_request_ctx_dma(areq);
2638 
2639 	memset(req, 0, sizeof(*req));
2640 
2641 	ctx->alg = CONTEXT_CONTROL_CRYPTO_ALG_SHA3_384;
2642 	req->digest = CONTEXT_CONTROL_DIGEST_INITIAL;
2643 	req->state_sz = SHA3_384_DIGEST_SIZE;
2644 	req->digest_sz = SHA3_384_DIGEST_SIZE;
2645 	req->block_sz = SHA3_384_BLOCK_SIZE;
2646 	ctx->do_fallback = false;
2647 	ctx->fb_init_done = false;
2648 	return 0;
2649 }
2650 
2651 static int safexcel_sha3_384_digest(struct ahash_request *req)
2652 {
2653 	if (req->nbytes)
2654 		return safexcel_sha3_384_init(req) ?: safexcel_ahash_finup(req);
2655 
2656 	/* HW cannot do zero length hash, use fallback instead */
2657 	return safexcel_sha3_digest_fallback(req);
2658 }
2659 
2660 struct safexcel_alg_template safexcel_alg_sha3_384 = {
2661 	.type = SAFEXCEL_ALG_TYPE_AHASH,
2662 	.algo_mask = SAFEXCEL_ALG_SHA3,
2663 	.alg.ahash = {
2664 		.init = safexcel_sha3_384_init,
2665 		.update = safexcel_sha3_update,
2666 		.final = safexcel_sha3_final,
2667 		.finup = safexcel_sha3_finup,
2668 		.digest = safexcel_sha3_384_digest,
2669 		.export = safexcel_sha3_export,
2670 		.import = safexcel_sha3_import,
2671 		.halg = {
2672 			.digestsize = SHA3_384_DIGEST_SIZE,
2673 			.statesize = sizeof(struct safexcel_ahash_export_state),
2674 			.base = {
2675 				.cra_name = "sha3-384",
2676 				.cra_driver_name = "safexcel-sha3-384",
2677 				.cra_priority = SAFEXCEL_CRA_PRIORITY,
2678 				.cra_flags = CRYPTO_ALG_ASYNC |
2679 					     CRYPTO_ALG_KERN_DRIVER_ONLY |
2680 					     CRYPTO_ALG_NEED_FALLBACK,
2681 				.cra_blocksize = SHA3_384_BLOCK_SIZE,
2682 				.cra_ctxsize = sizeof(struct safexcel_ahash_ctx),
2683 				.cra_init = safexcel_sha3_cra_init,
2684 				.cra_exit = safexcel_sha3_cra_exit,
2685 				.cra_module = THIS_MODULE,
2686 			},
2687 		},
2688 	},
2689 };
2690 
2691 static int safexcel_sha3_512_init(struct ahash_request *areq)
2692 {
2693 	struct crypto_ahash *tfm = crypto_ahash_reqtfm(areq);
2694 	struct safexcel_ahash_ctx *ctx = crypto_ahash_ctx(tfm);
2695 	struct safexcel_ahash_req *req = ahash_request_ctx_dma(areq);
2696 
2697 	memset(req, 0, sizeof(*req));
2698 
2699 	ctx->alg = CONTEXT_CONTROL_CRYPTO_ALG_SHA3_512;
2700 	req->digest = CONTEXT_CONTROL_DIGEST_INITIAL;
2701 	req->state_sz = SHA3_512_DIGEST_SIZE;
2702 	req->digest_sz = SHA3_512_DIGEST_SIZE;
2703 	req->block_sz = SHA3_512_BLOCK_SIZE;
2704 	ctx->do_fallback = false;
2705 	ctx->fb_init_done = false;
2706 	return 0;
2707 }
2708 
2709 static int safexcel_sha3_512_digest(struct ahash_request *req)
2710 {
2711 	if (req->nbytes)
2712 		return safexcel_sha3_512_init(req) ?: safexcel_ahash_finup(req);
2713 
2714 	/* HW cannot do zero length hash, use fallback instead */
2715 	return safexcel_sha3_digest_fallback(req);
2716 }
2717 
2718 struct safexcel_alg_template safexcel_alg_sha3_512 = {
2719 	.type = SAFEXCEL_ALG_TYPE_AHASH,
2720 	.algo_mask = SAFEXCEL_ALG_SHA3,
2721 	.alg.ahash = {
2722 		.init = safexcel_sha3_512_init,
2723 		.update = safexcel_sha3_update,
2724 		.final = safexcel_sha3_final,
2725 		.finup = safexcel_sha3_finup,
2726 		.digest = safexcel_sha3_512_digest,
2727 		.export = safexcel_sha3_export,
2728 		.import = safexcel_sha3_import,
2729 		.halg = {
2730 			.digestsize = SHA3_512_DIGEST_SIZE,
2731 			.statesize = sizeof(struct safexcel_ahash_export_state),
2732 			.base = {
2733 				.cra_name = "sha3-512",
2734 				.cra_driver_name = "safexcel-sha3-512",
2735 				.cra_priority = SAFEXCEL_CRA_PRIORITY,
2736 				.cra_flags = CRYPTO_ALG_ASYNC |
2737 					     CRYPTO_ALG_KERN_DRIVER_ONLY |
2738 					     CRYPTO_ALG_NEED_FALLBACK,
2739 				.cra_blocksize = SHA3_512_BLOCK_SIZE,
2740 				.cra_ctxsize = sizeof(struct safexcel_ahash_ctx),
2741 				.cra_init = safexcel_sha3_cra_init,
2742 				.cra_exit = safexcel_sha3_cra_exit,
2743 				.cra_module = THIS_MODULE,
2744 			},
2745 		},
2746 	},
2747 };
2748 
2749 static int safexcel_hmac_sha3_cra_init(struct crypto_tfm *tfm, const char *alg)
2750 {
2751 	struct safexcel_ahash_ctx *ctx = crypto_tfm_ctx(tfm);
2752 	int ret;
2753 
2754 	ret = safexcel_sha3_cra_init(tfm);
2755 	if (ret)
2756 		return ret;
2757 
2758 	/* Allocate precalc basic digest implementation */
2759 	ctx->shpre = crypto_alloc_shash(alg, 0, CRYPTO_ALG_NEED_FALLBACK);
2760 	if (IS_ERR(ctx->shpre))
2761 		return PTR_ERR(ctx->shpre);
2762 
2763 	ctx->shdesc = kmalloc(sizeof(*ctx->shdesc) +
2764 			      crypto_shash_descsize(ctx->shpre), GFP_KERNEL);
2765 	if (!ctx->shdesc) {
2766 		crypto_free_shash(ctx->shpre);
2767 		return -ENOMEM;
2768 	}
2769 	ctx->shdesc->tfm = ctx->shpre;
2770 	return 0;
2771 }
2772 
2773 static void safexcel_hmac_sha3_cra_exit(struct crypto_tfm *tfm)
2774 {
2775 	struct safexcel_ahash_ctx *ctx = crypto_tfm_ctx(tfm);
2776 
2777 	crypto_free_ahash(ctx->fback);
2778 	crypto_free_shash(ctx->shpre);
2779 	kfree(ctx->shdesc);
2780 	safexcel_ahash_cra_exit(tfm);
2781 }
2782 
2783 static int safexcel_hmac_sha3_setkey(struct crypto_ahash *tfm, const u8 *key,
2784 				     unsigned int keylen)
2785 {
2786 	struct safexcel_ahash_ctx *ctx = crypto_ahash_ctx(tfm);
2787 	int ret = 0;
2788 
2789 	if (keylen > crypto_ahash_blocksize(tfm)) {
2790 		/*
2791 		 * If the key is larger than the blocksize, then hash it
2792 		 * first using our fallback cipher
2793 		 */
2794 		ret = crypto_shash_digest(ctx->shdesc, key, keylen,
2795 					  ctx->base.ipad.byte);
2796 		keylen = crypto_shash_digestsize(ctx->shpre);
2797 
2798 		/*
2799 		 * If the digest is larger than half the blocksize, we need to
2800 		 * move the rest to opad due to the way our HMAC infra works.
2801 		 */
2802 		if (keylen > crypto_ahash_blocksize(tfm) / 2)
2803 			/* Buffers overlap, need to use memmove iso memcpy! */
2804 			memmove(&ctx->base.opad,
2805 				ctx->base.ipad.byte +
2806 					crypto_ahash_blocksize(tfm) / 2,
2807 				keylen - crypto_ahash_blocksize(tfm) / 2);
2808 	} else {
2809 		/*
2810 		 * Copy the key to our ipad & opad buffers
2811 		 * Note that ipad and opad each contain one half of the key,
2812 		 * to match the existing HMAC driver infrastructure.
2813 		 */
2814 		if (keylen <= crypto_ahash_blocksize(tfm) / 2) {
2815 			memcpy(&ctx->base.ipad, key, keylen);
2816 		} else {
2817 			memcpy(&ctx->base.ipad, key,
2818 			       crypto_ahash_blocksize(tfm) / 2);
2819 			memcpy(&ctx->base.opad,
2820 			       key + crypto_ahash_blocksize(tfm) / 2,
2821 			       keylen - crypto_ahash_blocksize(tfm) / 2);
2822 		}
2823 	}
2824 
2825 	/* Pad key with zeroes */
2826 	if (keylen <= crypto_ahash_blocksize(tfm) / 2) {
2827 		memset(ctx->base.ipad.byte + keylen, 0,
2828 		       crypto_ahash_blocksize(tfm) / 2 - keylen);
2829 		memset(&ctx->base.opad, 0, crypto_ahash_blocksize(tfm) / 2);
2830 	} else {
2831 		memset(ctx->base.opad.byte + keylen -
2832 		       crypto_ahash_blocksize(tfm) / 2, 0,
2833 		       crypto_ahash_blocksize(tfm) - keylen);
2834 	}
2835 
2836 	/* If doing fallback, still need to set the new key! */
2837 	ctx->fb_do_setkey = true;
2838 	return ret;
2839 }
2840 
2841 static int safexcel_hmac_sha3_224_init(struct ahash_request *areq)
2842 {
2843 	struct crypto_ahash *tfm = crypto_ahash_reqtfm(areq);
2844 	struct safexcel_ahash_ctx *ctx = crypto_ahash_ctx(tfm);
2845 	struct safexcel_ahash_req *req = ahash_request_ctx_dma(areq);
2846 
2847 	memset(req, 0, sizeof(*req));
2848 
2849 	/* Copy (half of) the key */
2850 	memcpy(req->state, &ctx->base.ipad, SHA3_224_BLOCK_SIZE / 2);
2851 	/* Start of HMAC should have len == processed == blocksize */
2852 	req->len	= SHA3_224_BLOCK_SIZE;
2853 	req->processed	= SHA3_224_BLOCK_SIZE;
2854 	ctx->alg = CONTEXT_CONTROL_CRYPTO_ALG_SHA3_224;
2855 	req->digest = CONTEXT_CONTROL_DIGEST_HMAC;
2856 	req->state_sz = SHA3_224_BLOCK_SIZE / 2;
2857 	req->digest_sz = SHA3_224_DIGEST_SIZE;
2858 	req->block_sz = SHA3_224_BLOCK_SIZE;
2859 	req->hmac = true;
2860 	ctx->do_fallback = false;
2861 	ctx->fb_init_done = false;
2862 	return 0;
2863 }
2864 
2865 static int safexcel_hmac_sha3_224_digest(struct ahash_request *req)
2866 {
2867 	if (req->nbytes)
2868 		return safexcel_hmac_sha3_224_init(req) ?:
2869 		       safexcel_ahash_finup(req);
2870 
2871 	/* HW cannot do zero length HMAC, use fallback instead */
2872 	return safexcel_sha3_digest_fallback(req);
2873 }
2874 
2875 static int safexcel_hmac_sha3_224_cra_init(struct crypto_tfm *tfm)
2876 {
2877 	return safexcel_hmac_sha3_cra_init(tfm, "sha3-224");
2878 }
2879 
2880 struct safexcel_alg_template safexcel_alg_hmac_sha3_224 = {
2881 	.type = SAFEXCEL_ALG_TYPE_AHASH,
2882 	.algo_mask = SAFEXCEL_ALG_SHA3,
2883 	.alg.ahash = {
2884 		.init = safexcel_hmac_sha3_224_init,
2885 		.update = safexcel_sha3_update,
2886 		.final = safexcel_sha3_final,
2887 		.finup = safexcel_sha3_finup,
2888 		.digest = safexcel_hmac_sha3_224_digest,
2889 		.setkey = safexcel_hmac_sha3_setkey,
2890 		.export = safexcel_sha3_export,
2891 		.import = safexcel_sha3_import,
2892 		.halg = {
2893 			.digestsize = SHA3_224_DIGEST_SIZE,
2894 			.statesize = sizeof(struct safexcel_ahash_export_state),
2895 			.base = {
2896 				.cra_name = "hmac(sha3-224)",
2897 				.cra_driver_name = "safexcel-hmac-sha3-224",
2898 				.cra_priority = SAFEXCEL_CRA_PRIORITY,
2899 				.cra_flags = CRYPTO_ALG_ASYNC |
2900 					     CRYPTO_ALG_KERN_DRIVER_ONLY |
2901 					     CRYPTO_ALG_NEED_FALLBACK,
2902 				.cra_blocksize = SHA3_224_BLOCK_SIZE,
2903 				.cra_ctxsize = sizeof(struct safexcel_ahash_ctx),
2904 				.cra_init = safexcel_hmac_sha3_224_cra_init,
2905 				.cra_exit = safexcel_hmac_sha3_cra_exit,
2906 				.cra_module = THIS_MODULE,
2907 			},
2908 		},
2909 	},
2910 };
2911 
2912 static int safexcel_hmac_sha3_256_init(struct ahash_request *areq)
2913 {
2914 	struct crypto_ahash *tfm = crypto_ahash_reqtfm(areq);
2915 	struct safexcel_ahash_ctx *ctx = crypto_ahash_ctx(tfm);
2916 	struct safexcel_ahash_req *req = ahash_request_ctx_dma(areq);
2917 
2918 	memset(req, 0, sizeof(*req));
2919 
2920 	/* Copy (half of) the key */
2921 	memcpy(req->state, &ctx->base.ipad, SHA3_256_BLOCK_SIZE / 2);
2922 	/* Start of HMAC should have len == processed == blocksize */
2923 	req->len	= SHA3_256_BLOCK_SIZE;
2924 	req->processed	= SHA3_256_BLOCK_SIZE;
2925 	ctx->alg = CONTEXT_CONTROL_CRYPTO_ALG_SHA3_256;
2926 	req->digest = CONTEXT_CONTROL_DIGEST_HMAC;
2927 	req->state_sz = SHA3_256_BLOCK_SIZE / 2;
2928 	req->digest_sz = SHA3_256_DIGEST_SIZE;
2929 	req->block_sz = SHA3_256_BLOCK_SIZE;
2930 	req->hmac = true;
2931 	ctx->do_fallback = false;
2932 	ctx->fb_init_done = false;
2933 	return 0;
2934 }
2935 
2936 static int safexcel_hmac_sha3_256_digest(struct ahash_request *req)
2937 {
2938 	if (req->nbytes)
2939 		return safexcel_hmac_sha3_256_init(req) ?:
2940 		       safexcel_ahash_finup(req);
2941 
2942 	/* HW cannot do zero length HMAC, use fallback instead */
2943 	return safexcel_sha3_digest_fallback(req);
2944 }
2945 
2946 static int safexcel_hmac_sha3_256_cra_init(struct crypto_tfm *tfm)
2947 {
2948 	return safexcel_hmac_sha3_cra_init(tfm, "sha3-256");
2949 }
2950 
2951 struct safexcel_alg_template safexcel_alg_hmac_sha3_256 = {
2952 	.type = SAFEXCEL_ALG_TYPE_AHASH,
2953 	.algo_mask = SAFEXCEL_ALG_SHA3,
2954 	.alg.ahash = {
2955 		.init = safexcel_hmac_sha3_256_init,
2956 		.update = safexcel_sha3_update,
2957 		.final = safexcel_sha3_final,
2958 		.finup = safexcel_sha3_finup,
2959 		.digest = safexcel_hmac_sha3_256_digest,
2960 		.setkey = safexcel_hmac_sha3_setkey,
2961 		.export = safexcel_sha3_export,
2962 		.import = safexcel_sha3_import,
2963 		.halg = {
2964 			.digestsize = SHA3_256_DIGEST_SIZE,
2965 			.statesize = sizeof(struct safexcel_ahash_export_state),
2966 			.base = {
2967 				.cra_name = "hmac(sha3-256)",
2968 				.cra_driver_name = "safexcel-hmac-sha3-256",
2969 				.cra_priority = SAFEXCEL_CRA_PRIORITY,
2970 				.cra_flags = CRYPTO_ALG_ASYNC |
2971 					     CRYPTO_ALG_KERN_DRIVER_ONLY |
2972 					     CRYPTO_ALG_NEED_FALLBACK,
2973 				.cra_blocksize = SHA3_256_BLOCK_SIZE,
2974 				.cra_ctxsize = sizeof(struct safexcel_ahash_ctx),
2975 				.cra_init = safexcel_hmac_sha3_256_cra_init,
2976 				.cra_exit = safexcel_hmac_sha3_cra_exit,
2977 				.cra_module = THIS_MODULE,
2978 			},
2979 		},
2980 	},
2981 };
2982 
2983 static int safexcel_hmac_sha3_384_init(struct ahash_request *areq)
2984 {
2985 	struct crypto_ahash *tfm = crypto_ahash_reqtfm(areq);
2986 	struct safexcel_ahash_ctx *ctx = crypto_ahash_ctx(tfm);
2987 	struct safexcel_ahash_req *req = ahash_request_ctx_dma(areq);
2988 
2989 	memset(req, 0, sizeof(*req));
2990 
2991 	/* Copy (half of) the key */
2992 	memcpy(req->state, &ctx->base.ipad, SHA3_384_BLOCK_SIZE / 2);
2993 	/* Start of HMAC should have len == processed == blocksize */
2994 	req->len	= SHA3_384_BLOCK_SIZE;
2995 	req->processed	= SHA3_384_BLOCK_SIZE;
2996 	ctx->alg = CONTEXT_CONTROL_CRYPTO_ALG_SHA3_384;
2997 	req->digest = CONTEXT_CONTROL_DIGEST_HMAC;
2998 	req->state_sz = SHA3_384_BLOCK_SIZE / 2;
2999 	req->digest_sz = SHA3_384_DIGEST_SIZE;
3000 	req->block_sz = SHA3_384_BLOCK_SIZE;
3001 	req->hmac = true;
3002 	ctx->do_fallback = false;
3003 	ctx->fb_init_done = false;
3004 	return 0;
3005 }
3006 
3007 static int safexcel_hmac_sha3_384_digest(struct ahash_request *req)
3008 {
3009 	if (req->nbytes)
3010 		return safexcel_hmac_sha3_384_init(req) ?:
3011 		       safexcel_ahash_finup(req);
3012 
3013 	/* HW cannot do zero length HMAC, use fallback instead */
3014 	return safexcel_sha3_digest_fallback(req);
3015 }
3016 
3017 static int safexcel_hmac_sha3_384_cra_init(struct crypto_tfm *tfm)
3018 {
3019 	return safexcel_hmac_sha3_cra_init(tfm, "sha3-384");
3020 }
3021 
3022 struct safexcel_alg_template safexcel_alg_hmac_sha3_384 = {
3023 	.type = SAFEXCEL_ALG_TYPE_AHASH,
3024 	.algo_mask = SAFEXCEL_ALG_SHA3,
3025 	.alg.ahash = {
3026 		.init = safexcel_hmac_sha3_384_init,
3027 		.update = safexcel_sha3_update,
3028 		.final = safexcel_sha3_final,
3029 		.finup = safexcel_sha3_finup,
3030 		.digest = safexcel_hmac_sha3_384_digest,
3031 		.setkey = safexcel_hmac_sha3_setkey,
3032 		.export = safexcel_sha3_export,
3033 		.import = safexcel_sha3_import,
3034 		.halg = {
3035 			.digestsize = SHA3_384_DIGEST_SIZE,
3036 			.statesize = sizeof(struct safexcel_ahash_export_state),
3037 			.base = {
3038 				.cra_name = "hmac(sha3-384)",
3039 				.cra_driver_name = "safexcel-hmac-sha3-384",
3040 				.cra_priority = SAFEXCEL_CRA_PRIORITY,
3041 				.cra_flags = CRYPTO_ALG_ASYNC |
3042 					     CRYPTO_ALG_KERN_DRIVER_ONLY |
3043 					     CRYPTO_ALG_NEED_FALLBACK,
3044 				.cra_blocksize = SHA3_384_BLOCK_SIZE,
3045 				.cra_ctxsize = sizeof(struct safexcel_ahash_ctx),
3046 				.cra_init = safexcel_hmac_sha3_384_cra_init,
3047 				.cra_exit = safexcel_hmac_sha3_cra_exit,
3048 				.cra_module = THIS_MODULE,
3049 			},
3050 		},
3051 	},
3052 };
3053 
3054 static int safexcel_hmac_sha3_512_init(struct ahash_request *areq)
3055 {
3056 	struct crypto_ahash *tfm = crypto_ahash_reqtfm(areq);
3057 	struct safexcel_ahash_ctx *ctx = crypto_ahash_ctx(tfm);
3058 	struct safexcel_ahash_req *req = ahash_request_ctx_dma(areq);
3059 
3060 	memset(req, 0, sizeof(*req));
3061 
3062 	/* Copy (half of) the key */
3063 	memcpy(req->state, &ctx->base.ipad, SHA3_512_BLOCK_SIZE / 2);
3064 	/* Start of HMAC should have len == processed == blocksize */
3065 	req->len	= SHA3_512_BLOCK_SIZE;
3066 	req->processed	= SHA3_512_BLOCK_SIZE;
3067 	ctx->alg = CONTEXT_CONTROL_CRYPTO_ALG_SHA3_512;
3068 	req->digest = CONTEXT_CONTROL_DIGEST_HMAC;
3069 	req->state_sz = SHA3_512_BLOCK_SIZE / 2;
3070 	req->digest_sz = SHA3_512_DIGEST_SIZE;
3071 	req->block_sz = SHA3_512_BLOCK_SIZE;
3072 	req->hmac = true;
3073 	ctx->do_fallback = false;
3074 	ctx->fb_init_done = false;
3075 	return 0;
3076 }
3077 
3078 static int safexcel_hmac_sha3_512_digest(struct ahash_request *req)
3079 {
3080 	if (req->nbytes)
3081 		return safexcel_hmac_sha3_512_init(req) ?:
3082 		       safexcel_ahash_finup(req);
3083 
3084 	/* HW cannot do zero length HMAC, use fallback instead */
3085 	return safexcel_sha3_digest_fallback(req);
3086 }
3087 
3088 static int safexcel_hmac_sha3_512_cra_init(struct crypto_tfm *tfm)
3089 {
3090 	return safexcel_hmac_sha3_cra_init(tfm, "sha3-512");
3091 }
3092 struct safexcel_alg_template safexcel_alg_hmac_sha3_512 = {
3093 	.type = SAFEXCEL_ALG_TYPE_AHASH,
3094 	.algo_mask = SAFEXCEL_ALG_SHA3,
3095 	.alg.ahash = {
3096 		.init = safexcel_hmac_sha3_512_init,
3097 		.update = safexcel_sha3_update,
3098 		.final = safexcel_sha3_final,
3099 		.finup = safexcel_sha3_finup,
3100 		.digest = safexcel_hmac_sha3_512_digest,
3101 		.setkey = safexcel_hmac_sha3_setkey,
3102 		.export = safexcel_sha3_export,
3103 		.import = safexcel_sha3_import,
3104 		.halg = {
3105 			.digestsize = SHA3_512_DIGEST_SIZE,
3106 			.statesize = sizeof(struct safexcel_ahash_export_state),
3107 			.base = {
3108 				.cra_name = "hmac(sha3-512)",
3109 				.cra_driver_name = "safexcel-hmac-sha3-512",
3110 				.cra_priority = SAFEXCEL_CRA_PRIORITY,
3111 				.cra_flags = CRYPTO_ALG_ASYNC |
3112 					     CRYPTO_ALG_KERN_DRIVER_ONLY |
3113 					     CRYPTO_ALG_NEED_FALLBACK,
3114 				.cra_blocksize = SHA3_512_BLOCK_SIZE,
3115 				.cra_ctxsize = sizeof(struct safexcel_ahash_ctx),
3116 				.cra_init = safexcel_hmac_sha3_512_cra_init,
3117 				.cra_exit = safexcel_hmac_sha3_cra_exit,
3118 				.cra_module = THIS_MODULE,
3119 			},
3120 		},
3121 	},
3122 };
3123