1 // SPDX-License-Identifier: GPL-2.0
2 /*
3  * Hash function and HMAC support for StarFive driver
4  *
5  * Copyright (c) 2022 StarFive Technology
6  *
7  */
8 
9 #include <crypto/engine.h>
10 #include <crypto/internal/hash.h>
11 #include <crypto/scatterwalk.h>
12 #include "jh7110-cryp.h"
13 #include <linux/amba/pl080.h>
14 #include <linux/clk.h>
15 #include <linux/dma-direct.h>
16 #include <linux/interrupt.h>
17 #include <linux/iopoll.h>
18 #include <linux/kernel.h>
19 #include <linux/module.h>
20 #include <linux/platform_device.h>
21 #include <linux/pm_runtime.h>
22 #include <linux/reset.h>
23 
24 #define STARFIVE_HASH_REGS_OFFSET	0x300
25 #define STARFIVE_HASH_SHACSR		(STARFIVE_HASH_REGS_OFFSET + 0x0)
26 #define STARFIVE_HASH_SHAWDR		(STARFIVE_HASH_REGS_OFFSET + 0x4)
27 #define STARFIVE_HASH_SHARDR		(STARFIVE_HASH_REGS_OFFSET + 0x8)
28 #define STARFIVE_HASH_SHAWSR		(STARFIVE_HASH_REGS_OFFSET + 0xC)
29 #define STARFIVE_HASH_SHAWLEN3		(STARFIVE_HASH_REGS_OFFSET + 0x10)
30 #define STARFIVE_HASH_SHAWLEN2		(STARFIVE_HASH_REGS_OFFSET + 0x14)
31 #define STARFIVE_HASH_SHAWLEN1		(STARFIVE_HASH_REGS_OFFSET + 0x18)
32 #define STARFIVE_HASH_SHAWLEN0		(STARFIVE_HASH_REGS_OFFSET + 0x1C)
33 #define STARFIVE_HASH_SHAWKR		(STARFIVE_HASH_REGS_OFFSET + 0x20)
34 #define STARFIVE_HASH_SHAWKLEN		(STARFIVE_HASH_REGS_OFFSET + 0x24)
35 
36 #define STARFIVE_HASH_BUFLEN		SHA512_BLOCK_SIZE
37 #define STARFIVE_HASH_RESET		0x2
38 
39 static inline int starfive_hash_wait_busy(struct starfive_cryp_ctx *ctx)
40 {
41 	struct starfive_cryp_dev *cryp = ctx->cryp;
42 	u32 status;
43 
44 	return readl_relaxed_poll_timeout(cryp->base + STARFIVE_HASH_SHACSR, status,
45 					  !(status & STARFIVE_HASH_BUSY), 10, 100000);
46 }
47 
48 static inline int starfive_hash_wait_key_done(struct starfive_cryp_ctx *ctx)
49 {
50 	struct starfive_cryp_dev *cryp = ctx->cryp;
51 	u32 status;
52 
53 	return readl_relaxed_poll_timeout(cryp->base + STARFIVE_HASH_SHACSR, status,
54 					  (status & STARFIVE_HASH_KEY_DONE), 10, 100000);
55 }
56 
57 static int starfive_hash_hmac_key(struct starfive_cryp_ctx *ctx)
58 {
59 	struct starfive_cryp_request_ctx *rctx = ctx->rctx;
60 	struct starfive_cryp_dev *cryp = ctx->cryp;
61 	int klen = ctx->keylen, loop;
62 	unsigned int *key = (unsigned int *)ctx->key;
63 	unsigned char *cl;
64 
65 	writel(ctx->keylen, cryp->base + STARFIVE_HASH_SHAWKLEN);
66 
67 	rctx->csr.hash.hmac = 1;
68 	rctx->csr.hash.key_flag = 1;
69 
70 	writel(rctx->csr.hash.v, cryp->base + STARFIVE_HASH_SHACSR);
71 
72 	for (loop = 0; loop < klen / sizeof(unsigned int); loop++, key++)
73 		writel(*key, cryp->base + STARFIVE_HASH_SHAWKR);
74 
75 	if (klen & 0x3) {
76 		cl = (unsigned char *)key;
77 		for (loop = 0; loop < (klen & 0x3); loop++, cl++)
78 			writeb(*cl, cryp->base + STARFIVE_HASH_SHAWKR);
79 	}
80 
81 	if (starfive_hash_wait_key_done(ctx))
82 		return dev_err_probe(cryp->dev, -ETIMEDOUT, "starfive_hash_wait_key_done error\n");
83 
84 	return 0;
85 }
86 
87 static void starfive_hash_start(void *param)
88 {
89 	struct starfive_cryp_ctx *ctx = param;
90 	struct starfive_cryp_request_ctx *rctx = ctx->rctx;
91 	struct starfive_cryp_dev *cryp = ctx->cryp;
92 	union starfive_alg_cr alg_cr;
93 	union starfive_hash_csr csr;
94 	u32 stat;
95 
96 	dma_unmap_sg(cryp->dev, rctx->in_sg, rctx->in_sg_len, DMA_TO_DEVICE);
97 
98 	alg_cr.v = 0;
99 	alg_cr.clear = 1;
100 
101 	writel(alg_cr.v, cryp->base + STARFIVE_ALG_CR_OFFSET);
102 
103 	csr.v = readl(cryp->base + STARFIVE_HASH_SHACSR);
104 	csr.firstb = 0;
105 	csr.final = 1;
106 
107 	stat = readl(cryp->base + STARFIVE_IE_MASK_OFFSET);
108 	stat &= ~STARFIVE_IE_MASK_HASH_DONE;
109 	writel(stat, cryp->base + STARFIVE_IE_MASK_OFFSET);
110 	writel(csr.v, cryp->base + STARFIVE_HASH_SHACSR);
111 }
112 
113 static int starfive_hash_xmit_dma(struct starfive_cryp_ctx *ctx)
114 {
115 	struct starfive_cryp_request_ctx *rctx = ctx->rctx;
116 	struct starfive_cryp_dev *cryp = ctx->cryp;
117 	struct dma_async_tx_descriptor	*in_desc;
118 	union  starfive_alg_cr alg_cr;
119 	int total_len;
120 	int ret;
121 
122 	if (!rctx->total) {
123 		starfive_hash_start(ctx);
124 		return 0;
125 	}
126 
127 	writel(rctx->total, cryp->base + STARFIVE_DMA_IN_LEN_OFFSET);
128 
129 	total_len = rctx->total;
130 	total_len = (total_len & 0x3) ? (((total_len >> 2) + 1) << 2) : total_len;
131 	sg_dma_len(rctx->in_sg) = total_len;
132 
133 	alg_cr.v = 0;
134 	alg_cr.start = 1;
135 	alg_cr.hash_dma_en = 1;
136 
137 	writel(alg_cr.v, cryp->base + STARFIVE_ALG_CR_OFFSET);
138 
139 	ret = dma_map_sg(cryp->dev, rctx->in_sg, rctx->in_sg_len, DMA_TO_DEVICE);
140 	if (!ret)
141 		return dev_err_probe(cryp->dev, -EINVAL, "dma_map_sg() error\n");
142 
143 	cryp->cfg_in.direction = DMA_MEM_TO_DEV;
144 	cryp->cfg_in.src_addr_width = DMA_SLAVE_BUSWIDTH_4_BYTES;
145 	cryp->cfg_in.dst_addr_width = DMA_SLAVE_BUSWIDTH_4_BYTES;
146 	cryp->cfg_in.src_maxburst = cryp->dma_maxburst;
147 	cryp->cfg_in.dst_maxburst = cryp->dma_maxburst;
148 	cryp->cfg_in.dst_addr = cryp->phys_base + STARFIVE_ALG_FIFO_OFFSET;
149 
150 	dmaengine_slave_config(cryp->tx, &cryp->cfg_in);
151 
152 	in_desc = dmaengine_prep_slave_sg(cryp->tx, rctx->in_sg,
153 					  ret, DMA_MEM_TO_DEV,
154 					  DMA_PREP_INTERRUPT  |  DMA_CTRL_ACK);
155 
156 	if (!in_desc)
157 		return -EINVAL;
158 
159 	in_desc->callback = starfive_hash_start;
160 	in_desc->callback_param = ctx;
161 
162 	dmaengine_submit(in_desc);
163 	dma_async_issue_pending(cryp->tx);
164 
165 	return 0;
166 }
167 
168 static int starfive_hash_xmit(struct starfive_cryp_ctx *ctx)
169 {
170 	struct starfive_cryp_request_ctx *rctx = ctx->rctx;
171 	struct starfive_cryp_dev *cryp = ctx->cryp;
172 	int ret = 0;
173 
174 	rctx->csr.hash.v = 0;
175 	rctx->csr.hash.reset = 1;
176 	writel(rctx->csr.hash.v, cryp->base + STARFIVE_HASH_SHACSR);
177 
178 	if (starfive_hash_wait_busy(ctx))
179 		return dev_err_probe(cryp->dev, -ETIMEDOUT, "Error resetting engine.\n");
180 
181 	rctx->csr.hash.v = 0;
182 	rctx->csr.hash.mode = ctx->hash_mode;
183 	rctx->csr.hash.ie = 1;
184 
185 	if (ctx->is_hmac) {
186 		ret = starfive_hash_hmac_key(ctx);
187 		if (ret)
188 			return ret;
189 	} else {
190 		rctx->csr.hash.start = 1;
191 		rctx->csr.hash.firstb = 1;
192 		writel(rctx->csr.hash.v, cryp->base + STARFIVE_HASH_SHACSR);
193 	}
194 
195 	return starfive_hash_xmit_dma(ctx);
196 }
197 
198 static int starfive_hash_copy_hash(struct ahash_request *req)
199 {
200 	struct starfive_cryp_request_ctx *rctx = ahash_request_ctx(req);
201 	struct starfive_cryp_ctx *ctx = crypto_ahash_ctx(crypto_ahash_reqtfm(req));
202 	int count, *data;
203 	int mlen;
204 
205 	if (!req->result)
206 		return 0;
207 
208 	mlen = rctx->digsize / sizeof(u32);
209 	data = (u32 *)req->result;
210 
211 	for (count = 0; count < mlen; count++)
212 		data[count] = readl(ctx->cryp->base + STARFIVE_HASH_SHARDR);
213 
214 	return 0;
215 }
216 
217 void starfive_hash_done_task(unsigned long param)
218 {
219 	struct starfive_cryp_dev *cryp = (struct starfive_cryp_dev *)param;
220 	int err = cryp->err;
221 
222 	if (!err)
223 		err = starfive_hash_copy_hash(cryp->req.hreq);
224 
225 	/* Reset to clear hash_done in irq register*/
226 	writel(STARFIVE_HASH_RESET, cryp->base + STARFIVE_HASH_SHACSR);
227 
228 	crypto_finalize_hash_request(cryp->engine, cryp->req.hreq, err);
229 }
230 
231 static int starfive_hash_check_aligned(struct scatterlist *sg, size_t total, size_t align)
232 {
233 	int len = 0;
234 
235 	if (!total)
236 		return 0;
237 
238 	if (!IS_ALIGNED(total, align))
239 		return -EINVAL;
240 
241 	while (sg) {
242 		if (!IS_ALIGNED(sg->offset, sizeof(u32)))
243 			return -EINVAL;
244 
245 		if (!IS_ALIGNED(sg->length, align))
246 			return -EINVAL;
247 
248 		len += sg->length;
249 		sg = sg_next(sg);
250 	}
251 
252 	if (len != total)
253 		return -EINVAL;
254 
255 	return 0;
256 }
257 
258 static int starfive_hash_one_request(struct crypto_engine *engine, void *areq)
259 {
260 	struct ahash_request *req = container_of(areq, struct ahash_request,
261 						 base);
262 	struct starfive_cryp_ctx *ctx = crypto_ahash_ctx(crypto_ahash_reqtfm(req));
263 	struct starfive_cryp_dev *cryp = ctx->cryp;
264 
265 	if (!cryp)
266 		return -ENODEV;
267 
268 	return starfive_hash_xmit(ctx);
269 }
270 
271 static int starfive_hash_init(struct ahash_request *req)
272 {
273 	struct crypto_ahash *tfm = crypto_ahash_reqtfm(req);
274 	struct starfive_cryp_request_ctx *rctx = ahash_request_ctx(req);
275 	struct starfive_cryp_ctx *ctx = crypto_ahash_ctx(tfm);
276 
277 	ahash_request_set_tfm(&rctx->ahash_fbk_req, ctx->ahash_fbk);
278 	ahash_request_set_callback(&rctx->ahash_fbk_req,
279 				   req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP,
280 				   req->base.complete, req->base.data);
281 
282 	ahash_request_set_crypt(&rctx->ahash_fbk_req, req->src,
283 				req->result, req->nbytes);
284 
285 	return crypto_ahash_init(&rctx->ahash_fbk_req);
286 }
287 
288 static int starfive_hash_update(struct ahash_request *req)
289 {
290 	struct starfive_cryp_request_ctx *rctx = ahash_request_ctx(req);
291 	struct crypto_ahash *tfm = crypto_ahash_reqtfm(req);
292 	struct starfive_cryp_ctx *ctx = crypto_ahash_ctx(tfm);
293 
294 	ahash_request_set_tfm(&rctx->ahash_fbk_req, ctx->ahash_fbk);
295 	ahash_request_set_callback(&rctx->ahash_fbk_req,
296 				   req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP,
297 				   req->base.complete, req->base.data);
298 
299 	ahash_request_set_crypt(&rctx->ahash_fbk_req, req->src,
300 				req->result, req->nbytes);
301 
302 	return crypto_ahash_update(&rctx->ahash_fbk_req);
303 }
304 
305 static int starfive_hash_final(struct ahash_request *req)
306 {
307 	struct starfive_cryp_request_ctx *rctx = ahash_request_ctx(req);
308 	struct crypto_ahash *tfm = crypto_ahash_reqtfm(req);
309 	struct starfive_cryp_ctx *ctx = crypto_ahash_ctx(tfm);
310 
311 	ahash_request_set_tfm(&rctx->ahash_fbk_req, ctx->ahash_fbk);
312 	ahash_request_set_callback(&rctx->ahash_fbk_req,
313 				   req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP,
314 				   req->base.complete, req->base.data);
315 
316 	ahash_request_set_crypt(&rctx->ahash_fbk_req, req->src,
317 				req->result, req->nbytes);
318 
319 	return crypto_ahash_final(&rctx->ahash_fbk_req);
320 }
321 
322 static int starfive_hash_finup(struct ahash_request *req)
323 {
324 	struct starfive_cryp_request_ctx *rctx = ahash_request_ctx(req);
325 	struct crypto_ahash *tfm = crypto_ahash_reqtfm(req);
326 	struct starfive_cryp_ctx *ctx = crypto_ahash_ctx(tfm);
327 
328 	ahash_request_set_tfm(&rctx->ahash_fbk_req, ctx->ahash_fbk);
329 	ahash_request_set_callback(&rctx->ahash_fbk_req,
330 				   req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP,
331 				   req->base.complete, req->base.data);
332 
333 	ahash_request_set_crypt(&rctx->ahash_fbk_req, req->src,
334 				req->result, req->nbytes);
335 
336 	return crypto_ahash_finup(&rctx->ahash_fbk_req);
337 }
338 
339 static int starfive_hash_digest_fb(struct ahash_request *req)
340 {
341 	struct starfive_cryp_request_ctx *rctx = ahash_request_ctx(req);
342 	struct crypto_ahash *tfm = crypto_ahash_reqtfm(req);
343 	struct starfive_cryp_ctx *ctx = crypto_ahash_ctx(tfm);
344 
345 	ahash_request_set_tfm(&rctx->ahash_fbk_req, ctx->ahash_fbk);
346 	ahash_request_set_callback(&rctx->ahash_fbk_req, req->base.flags,
347 				   req->base.complete, req->base.data);
348 
349 	ahash_request_set_crypt(&rctx->ahash_fbk_req, req->src,
350 				req->result, req->nbytes);
351 
352 	return crypto_ahash_digest(&rctx->ahash_fbk_req);
353 }
354 
355 static int starfive_hash_digest(struct ahash_request *req)
356 {
357 	struct crypto_ahash *tfm = crypto_ahash_reqtfm(req);
358 	struct starfive_cryp_ctx *ctx = crypto_ahash_ctx(tfm);
359 	struct starfive_cryp_request_ctx *rctx = ahash_request_ctx(req);
360 	struct starfive_cryp_dev *cryp = ctx->cryp;
361 
362 	memset(rctx, 0, sizeof(struct starfive_cryp_request_ctx));
363 
364 	cryp->req.hreq = req;
365 	rctx->total = req->nbytes;
366 	rctx->in_sg = req->src;
367 	rctx->blksize = crypto_tfm_alg_blocksize(crypto_ahash_tfm(tfm));
368 	rctx->digsize = crypto_ahash_digestsize(tfm);
369 	rctx->in_sg_len = sg_nents_for_len(rctx->in_sg, rctx->total);
370 	ctx->rctx = rctx;
371 
372 	if (starfive_hash_check_aligned(rctx->in_sg, rctx->total, rctx->blksize))
373 		return starfive_hash_digest_fb(req);
374 
375 	return crypto_transfer_hash_request_to_engine(cryp->engine, req);
376 }
377 
378 static int starfive_hash_export(struct ahash_request *req, void *out)
379 {
380 	struct starfive_cryp_request_ctx *rctx = ahash_request_ctx(req);
381 	struct crypto_ahash *tfm = crypto_ahash_reqtfm(req);
382 	struct starfive_cryp_ctx *ctx = crypto_ahash_ctx(tfm);
383 
384 	ahash_request_set_tfm(&rctx->ahash_fbk_req, ctx->ahash_fbk);
385 	ahash_request_set_callback(&rctx->ahash_fbk_req,
386 				   req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP,
387 				   req->base.complete, req->base.data);
388 
389 	return crypto_ahash_export(&rctx->ahash_fbk_req, out);
390 }
391 
392 static int starfive_hash_import(struct ahash_request *req, const void *in)
393 {
394 	struct starfive_cryp_request_ctx *rctx = ahash_request_ctx(req);
395 	struct crypto_ahash *tfm = crypto_ahash_reqtfm(req);
396 	struct starfive_cryp_ctx *ctx = crypto_ahash_ctx(tfm);
397 
398 	ahash_request_set_tfm(&rctx->ahash_fbk_req, ctx->ahash_fbk);
399 	ahash_request_set_callback(&rctx->ahash_fbk_req,
400 				   req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP,
401 				   req->base.complete, req->base.data);
402 
403 	return crypto_ahash_import(&rctx->ahash_fbk_req, in);
404 }
405 
406 static int starfive_hash_init_tfm(struct crypto_ahash *hash,
407 				  const char *alg_name,
408 				  unsigned int mode)
409 {
410 	struct starfive_cryp_ctx *ctx = crypto_ahash_ctx(hash);
411 
412 	ctx->cryp = starfive_cryp_find_dev(ctx);
413 
414 	if (!ctx->cryp)
415 		return -ENODEV;
416 
417 	ctx->ahash_fbk = crypto_alloc_ahash(alg_name, 0,
418 					    CRYPTO_ALG_NEED_FALLBACK);
419 
420 	if (IS_ERR(ctx->ahash_fbk))
421 		return dev_err_probe(ctx->cryp->dev, PTR_ERR(ctx->ahash_fbk),
422 				     "starfive_hash: Could not load fallback driver.\n");
423 
424 	crypto_ahash_set_statesize(hash, crypto_ahash_statesize(ctx->ahash_fbk));
425 	crypto_ahash_set_reqsize(hash, sizeof(struct starfive_cryp_request_ctx) +
426 				 crypto_ahash_reqsize(ctx->ahash_fbk));
427 
428 	ctx->keylen = 0;
429 	ctx->hash_mode = mode;
430 
431 	return 0;
432 }
433 
434 static void starfive_hash_exit_tfm(struct crypto_ahash *hash)
435 {
436 	struct starfive_cryp_ctx *ctx = crypto_ahash_ctx(hash);
437 
438 	crypto_free_ahash(ctx->ahash_fbk);
439 }
440 
441 static int starfive_hash_long_setkey(struct starfive_cryp_ctx *ctx,
442 				     const u8 *key, unsigned int keylen,
443 				     const char *alg_name)
444 {
445 	struct crypto_wait wait;
446 	struct ahash_request *req;
447 	struct scatterlist sg;
448 	struct crypto_ahash *ahash_tfm;
449 	u8 *buf;
450 	int ret;
451 
452 	ahash_tfm = crypto_alloc_ahash(alg_name, 0, 0);
453 	if (IS_ERR(ahash_tfm))
454 		return PTR_ERR(ahash_tfm);
455 
456 	req = ahash_request_alloc(ahash_tfm, GFP_KERNEL);
457 	if (!req) {
458 		ret = -ENOMEM;
459 		goto err_free_ahash;
460 	}
461 
462 	crypto_init_wait(&wait);
463 	ahash_request_set_callback(req, CRYPTO_TFM_REQ_MAY_BACKLOG,
464 				   crypto_req_done, &wait);
465 	crypto_ahash_clear_flags(ahash_tfm, ~0);
466 
467 	buf = kzalloc(keylen + STARFIVE_HASH_BUFLEN, GFP_KERNEL);
468 	if (!buf) {
469 		ret = -ENOMEM;
470 		goto err_free_req;
471 	}
472 
473 	memcpy(buf, key, keylen);
474 	sg_init_one(&sg, buf, keylen);
475 	ahash_request_set_crypt(req, &sg, ctx->key, keylen);
476 
477 	ret = crypto_wait_req(crypto_ahash_digest(req), &wait);
478 
479 	kfree(buf);
480 err_free_req:
481 	ahash_request_free(req);
482 err_free_ahash:
483 	crypto_free_ahash(ahash_tfm);
484 	return ret;
485 }
486 
487 static int starfive_hash_setkey(struct crypto_ahash *hash,
488 				const u8 *key, unsigned int keylen)
489 {
490 	struct starfive_cryp_ctx *ctx = crypto_ahash_ctx(hash);
491 	unsigned int digestsize = crypto_ahash_digestsize(hash);
492 	unsigned int blocksize = crypto_ahash_blocksize(hash);
493 	const char *alg_name;
494 
495 	crypto_ahash_setkey(ctx->ahash_fbk, key, keylen);
496 
497 	if (keylen <= blocksize) {
498 		memcpy(ctx->key, key, keylen);
499 		ctx->keylen = keylen;
500 		return 0;
501 	}
502 
503 	ctx->keylen = digestsize;
504 
505 	switch (digestsize) {
506 	case SHA224_DIGEST_SIZE:
507 		alg_name = "sha224-starfive";
508 		break;
509 	case SHA256_DIGEST_SIZE:
510 		if (ctx->hash_mode == STARFIVE_HASH_SM3)
511 			alg_name = "sm3-starfive";
512 		else
513 			alg_name = "sha256-starfive";
514 		break;
515 	case SHA384_DIGEST_SIZE:
516 		alg_name = "sha384-starfive";
517 		break;
518 	case SHA512_DIGEST_SIZE:
519 		alg_name = "sha512-starfive";
520 		break;
521 	default:
522 		return -EINVAL;
523 	}
524 
525 	return starfive_hash_long_setkey(ctx, key, keylen, alg_name);
526 }
527 
528 static int starfive_sha224_init_tfm(struct crypto_ahash *hash)
529 {
530 	return starfive_hash_init_tfm(hash, "sha224-generic",
531 				      STARFIVE_HASH_SHA224);
532 }
533 
534 static int starfive_sha256_init_tfm(struct crypto_ahash *hash)
535 {
536 	return starfive_hash_init_tfm(hash, "sha256-generic",
537 				      STARFIVE_HASH_SHA256);
538 }
539 
540 static int starfive_sha384_init_tfm(struct crypto_ahash *hash)
541 {
542 	return starfive_hash_init_tfm(hash, "sha384-generic",
543 				      STARFIVE_HASH_SHA384);
544 }
545 
546 static int starfive_sha512_init_tfm(struct crypto_ahash *hash)
547 {
548 	return starfive_hash_init_tfm(hash, "sha512-generic",
549 				      STARFIVE_HASH_SHA512);
550 }
551 
552 static int starfive_sm3_init_tfm(struct crypto_ahash *hash)
553 {
554 	return starfive_hash_init_tfm(hash, "sm3-generic",
555 				      STARFIVE_HASH_SM3);
556 }
557 
558 static int starfive_hmac_sha224_init_tfm(struct crypto_ahash *hash)
559 {
560 	struct starfive_cryp_ctx *ctx = crypto_ahash_ctx(hash);
561 
562 	ctx->is_hmac = true;
563 
564 	return starfive_hash_init_tfm(hash, "hmac(sha224-generic)",
565 				      STARFIVE_HASH_SHA224);
566 }
567 
568 static int starfive_hmac_sha256_init_tfm(struct crypto_ahash *hash)
569 {
570 	struct starfive_cryp_ctx *ctx = crypto_ahash_ctx(hash);
571 
572 	ctx->is_hmac = true;
573 
574 	return starfive_hash_init_tfm(hash, "hmac(sha256-generic)",
575 				      STARFIVE_HASH_SHA256);
576 }
577 
578 static int starfive_hmac_sha384_init_tfm(struct crypto_ahash *hash)
579 {
580 	struct starfive_cryp_ctx *ctx = crypto_ahash_ctx(hash);
581 
582 	ctx->is_hmac = true;
583 
584 	return starfive_hash_init_tfm(hash, "hmac(sha384-generic)",
585 				      STARFIVE_HASH_SHA384);
586 }
587 
588 static int starfive_hmac_sha512_init_tfm(struct crypto_ahash *hash)
589 {
590 	struct starfive_cryp_ctx *ctx = crypto_ahash_ctx(hash);
591 
592 	ctx->is_hmac = true;
593 
594 	return starfive_hash_init_tfm(hash, "hmac(sha512-generic)",
595 				      STARFIVE_HASH_SHA512);
596 }
597 
598 static int starfive_hmac_sm3_init_tfm(struct crypto_ahash *hash)
599 {
600 	struct starfive_cryp_ctx *ctx = crypto_ahash_ctx(hash);
601 
602 	ctx->is_hmac = true;
603 
604 	return starfive_hash_init_tfm(hash, "hmac(sm3-generic)",
605 				      STARFIVE_HASH_SM3);
606 }
607 
608 static struct ahash_engine_alg algs_sha2_sm3[] = {
609 {
610 	.base.init     = starfive_hash_init,
611 	.base.update   = starfive_hash_update,
612 	.base.final    = starfive_hash_final,
613 	.base.finup    = starfive_hash_finup,
614 	.base.digest   = starfive_hash_digest,
615 	.base.export   = starfive_hash_export,
616 	.base.import   = starfive_hash_import,
617 	.base.init_tfm = starfive_sha224_init_tfm,
618 	.base.exit_tfm = starfive_hash_exit_tfm,
619 	.base.halg = {
620 		.digestsize = SHA224_DIGEST_SIZE,
621 		.statesize  = sizeof(struct sha256_state),
622 		.base = {
623 			.cra_name		= "sha224",
624 			.cra_driver_name	= "sha224-starfive",
625 			.cra_priority		= 200,
626 			.cra_flags		= CRYPTO_ALG_ASYNC |
627 						  CRYPTO_ALG_TYPE_AHASH |
628 						  CRYPTO_ALG_NEED_FALLBACK,
629 			.cra_blocksize		= SHA224_BLOCK_SIZE,
630 			.cra_ctxsize		= sizeof(struct starfive_cryp_ctx),
631 			.cra_alignmask		= 3,
632 			.cra_module		= THIS_MODULE,
633 		}
634 	},
635 	.op = {
636 		.do_one_request = starfive_hash_one_request,
637 	},
638 }, {
639 	.base.init     = starfive_hash_init,
640 	.base.update   = starfive_hash_update,
641 	.base.final    = starfive_hash_final,
642 	.base.finup    = starfive_hash_finup,
643 	.base.digest   = starfive_hash_digest,
644 	.base.export   = starfive_hash_export,
645 	.base.import   = starfive_hash_import,
646 	.base.init_tfm = starfive_hmac_sha224_init_tfm,
647 	.base.exit_tfm = starfive_hash_exit_tfm,
648 	.base.setkey   = starfive_hash_setkey,
649 	.base.halg = {
650 		.digestsize = SHA224_DIGEST_SIZE,
651 		.statesize  = sizeof(struct sha256_state),
652 		.base = {
653 			.cra_name		= "hmac(sha224)",
654 			.cra_driver_name	= "sha224-hmac-starfive",
655 			.cra_priority		= 200,
656 			.cra_flags		= CRYPTO_ALG_ASYNC |
657 						  CRYPTO_ALG_TYPE_AHASH |
658 						  CRYPTO_ALG_NEED_FALLBACK,
659 			.cra_blocksize		= SHA224_BLOCK_SIZE,
660 			.cra_ctxsize		= sizeof(struct starfive_cryp_ctx),
661 			.cra_alignmask		= 3,
662 			.cra_module		= THIS_MODULE,
663 		}
664 	},
665 	.op = {
666 		.do_one_request = starfive_hash_one_request,
667 	},
668 }, {
669 	.base.init     = starfive_hash_init,
670 	.base.update   = starfive_hash_update,
671 	.base.final    = starfive_hash_final,
672 	.base.finup    = starfive_hash_finup,
673 	.base.digest   = starfive_hash_digest,
674 	.base.export   = starfive_hash_export,
675 	.base.import   = starfive_hash_import,
676 	.base.init_tfm = starfive_sha256_init_tfm,
677 	.base.exit_tfm = starfive_hash_exit_tfm,
678 	.base.halg = {
679 		.digestsize = SHA256_DIGEST_SIZE,
680 		.statesize  = sizeof(struct sha256_state),
681 		.base = {
682 			.cra_name		= "sha256",
683 			.cra_driver_name	= "sha256-starfive",
684 			.cra_priority		= 200,
685 			.cra_flags		= CRYPTO_ALG_ASYNC |
686 						  CRYPTO_ALG_TYPE_AHASH |
687 						  CRYPTO_ALG_NEED_FALLBACK,
688 			.cra_blocksize		= SHA256_BLOCK_SIZE,
689 			.cra_ctxsize		= sizeof(struct starfive_cryp_ctx),
690 			.cra_alignmask		= 3,
691 			.cra_module		= THIS_MODULE,
692 		}
693 	},
694 	.op = {
695 		.do_one_request = starfive_hash_one_request,
696 	},
697 }, {
698 	.base.init     = starfive_hash_init,
699 	.base.update   = starfive_hash_update,
700 	.base.final    = starfive_hash_final,
701 	.base.finup    = starfive_hash_finup,
702 	.base.digest   = starfive_hash_digest,
703 	.base.export   = starfive_hash_export,
704 	.base.import   = starfive_hash_import,
705 	.base.init_tfm = starfive_hmac_sha256_init_tfm,
706 	.base.exit_tfm = starfive_hash_exit_tfm,
707 	.base.setkey   = starfive_hash_setkey,
708 	.base.halg = {
709 		.digestsize = SHA256_DIGEST_SIZE,
710 		.statesize  = sizeof(struct sha256_state),
711 		.base = {
712 			.cra_name		= "hmac(sha256)",
713 			.cra_driver_name	= "sha256-hmac-starfive",
714 			.cra_priority		= 200,
715 			.cra_flags		= CRYPTO_ALG_ASYNC |
716 						  CRYPTO_ALG_TYPE_AHASH |
717 						  CRYPTO_ALG_NEED_FALLBACK,
718 			.cra_blocksize		= SHA256_BLOCK_SIZE,
719 			.cra_ctxsize		= sizeof(struct starfive_cryp_ctx),
720 			.cra_alignmask		= 3,
721 			.cra_module		= THIS_MODULE,
722 		}
723 	},
724 	.op = {
725 		.do_one_request = starfive_hash_one_request,
726 	},
727 }, {
728 	.base.init     = starfive_hash_init,
729 	.base.update   = starfive_hash_update,
730 	.base.final    = starfive_hash_final,
731 	.base.finup    = starfive_hash_finup,
732 	.base.digest   = starfive_hash_digest,
733 	.base.export   = starfive_hash_export,
734 	.base.import   = starfive_hash_import,
735 	.base.init_tfm = starfive_sha384_init_tfm,
736 	.base.exit_tfm = starfive_hash_exit_tfm,
737 	.base.halg = {
738 		.digestsize = SHA384_DIGEST_SIZE,
739 		.statesize  = sizeof(struct sha512_state),
740 		.base = {
741 			.cra_name		= "sha384",
742 			.cra_driver_name	= "sha384-starfive",
743 			.cra_priority		= 200,
744 			.cra_flags		= CRYPTO_ALG_ASYNC |
745 						  CRYPTO_ALG_TYPE_AHASH |
746 						  CRYPTO_ALG_NEED_FALLBACK,
747 			.cra_blocksize		= SHA384_BLOCK_SIZE,
748 			.cra_ctxsize		= sizeof(struct starfive_cryp_ctx),
749 			.cra_alignmask		= 3,
750 			.cra_module		= THIS_MODULE,
751 		}
752 	},
753 	.op = {
754 		.do_one_request = starfive_hash_one_request,
755 	},
756 }, {
757 	.base.init     = starfive_hash_init,
758 	.base.update   = starfive_hash_update,
759 	.base.final    = starfive_hash_final,
760 	.base.finup    = starfive_hash_finup,
761 	.base.digest   = starfive_hash_digest,
762 	.base.export   = starfive_hash_export,
763 	.base.import   = starfive_hash_import,
764 	.base.init_tfm = starfive_hmac_sha384_init_tfm,
765 	.base.exit_tfm = starfive_hash_exit_tfm,
766 	.base.setkey   = starfive_hash_setkey,
767 	.base.halg = {
768 		.digestsize = SHA384_DIGEST_SIZE,
769 		.statesize  = sizeof(struct sha512_state),
770 		.base = {
771 			.cra_name		= "hmac(sha384)",
772 			.cra_driver_name	= "sha384-hmac-starfive",
773 			.cra_priority		= 200,
774 			.cra_flags		= CRYPTO_ALG_ASYNC |
775 						  CRYPTO_ALG_TYPE_AHASH |
776 						  CRYPTO_ALG_NEED_FALLBACK,
777 			.cra_blocksize		= SHA384_BLOCK_SIZE,
778 			.cra_ctxsize		= sizeof(struct starfive_cryp_ctx),
779 			.cra_alignmask		= 3,
780 			.cra_module		= THIS_MODULE,
781 		}
782 	},
783 	.op = {
784 		.do_one_request = starfive_hash_one_request,
785 	},
786 }, {
787 	.base.init     = starfive_hash_init,
788 	.base.update   = starfive_hash_update,
789 	.base.final    = starfive_hash_final,
790 	.base.finup    = starfive_hash_finup,
791 	.base.digest   = starfive_hash_digest,
792 	.base.export   = starfive_hash_export,
793 	.base.import   = starfive_hash_import,
794 	.base.init_tfm = starfive_sha512_init_tfm,
795 	.base.exit_tfm = starfive_hash_exit_tfm,
796 	.base.halg = {
797 		.digestsize = SHA512_DIGEST_SIZE,
798 		.statesize  = sizeof(struct sha512_state),
799 		.base = {
800 			.cra_name		= "sha512",
801 			.cra_driver_name	= "sha512-starfive",
802 			.cra_priority		= 200,
803 			.cra_flags		= CRYPTO_ALG_ASYNC |
804 						  CRYPTO_ALG_TYPE_AHASH |
805 						  CRYPTO_ALG_NEED_FALLBACK,
806 			.cra_blocksize		= SHA512_BLOCK_SIZE,
807 			.cra_ctxsize		= sizeof(struct starfive_cryp_ctx),
808 			.cra_alignmask		= 3,
809 			.cra_module		= THIS_MODULE,
810 		}
811 	},
812 	.op = {
813 		.do_one_request = starfive_hash_one_request,
814 	},
815 }, {
816 	.base.init     = starfive_hash_init,
817 	.base.update   = starfive_hash_update,
818 	.base.final    = starfive_hash_final,
819 	.base.finup    = starfive_hash_finup,
820 	.base.digest   = starfive_hash_digest,
821 	.base.export   = starfive_hash_export,
822 	.base.import   = starfive_hash_import,
823 	.base.init_tfm = starfive_hmac_sha512_init_tfm,
824 	.base.exit_tfm = starfive_hash_exit_tfm,
825 	.base.setkey   = starfive_hash_setkey,
826 	.base.halg = {
827 		.digestsize = SHA512_DIGEST_SIZE,
828 		.statesize  = sizeof(struct sha512_state),
829 		.base = {
830 			.cra_name		= "hmac(sha512)",
831 			.cra_driver_name	= "sha512-hmac-starfive",
832 			.cra_priority		= 200,
833 			.cra_flags		= CRYPTO_ALG_ASYNC |
834 						  CRYPTO_ALG_TYPE_AHASH |
835 						  CRYPTO_ALG_NEED_FALLBACK,
836 			.cra_blocksize		= SHA512_BLOCK_SIZE,
837 			.cra_ctxsize		= sizeof(struct starfive_cryp_ctx),
838 			.cra_alignmask		= 3,
839 			.cra_module		= THIS_MODULE,
840 		}
841 	},
842 	.op = {
843 		.do_one_request = starfive_hash_one_request,
844 	},
845 }, {
846 	.base.init     = starfive_hash_init,
847 	.base.update   = starfive_hash_update,
848 	.base.final    = starfive_hash_final,
849 	.base.finup    = starfive_hash_finup,
850 	.base.digest   = starfive_hash_digest,
851 	.base.export   = starfive_hash_export,
852 	.base.import   = starfive_hash_import,
853 	.base.init_tfm = starfive_sm3_init_tfm,
854 	.base.exit_tfm = starfive_hash_exit_tfm,
855 	.base.halg = {
856 		.digestsize = SM3_DIGEST_SIZE,
857 		.statesize  = sizeof(struct sm3_state),
858 		.base = {
859 			.cra_name		= "sm3",
860 			.cra_driver_name	= "sm3-starfive",
861 			.cra_priority		= 200,
862 			.cra_flags		= CRYPTO_ALG_ASYNC |
863 						  CRYPTO_ALG_TYPE_AHASH |
864 						  CRYPTO_ALG_NEED_FALLBACK,
865 			.cra_blocksize		= SM3_BLOCK_SIZE,
866 			.cra_ctxsize		= sizeof(struct starfive_cryp_ctx),
867 			.cra_alignmask		= 3,
868 			.cra_module		= THIS_MODULE,
869 		}
870 	},
871 	.op = {
872 		.do_one_request = starfive_hash_one_request,
873 	},
874 }, {
875 	.base.init	  = starfive_hash_init,
876 	.base.update	  = starfive_hash_update,
877 	.base.final	  = starfive_hash_final,
878 	.base.finup	  = starfive_hash_finup,
879 	.base.digest	  = starfive_hash_digest,
880 	.base.export	  = starfive_hash_export,
881 	.base.import	  = starfive_hash_import,
882 	.base.init_tfm = starfive_hmac_sm3_init_tfm,
883 	.base.exit_tfm = starfive_hash_exit_tfm,
884 	.base.setkey	  = starfive_hash_setkey,
885 	.base.halg = {
886 		.digestsize = SM3_DIGEST_SIZE,
887 		.statesize  = sizeof(struct sm3_state),
888 		.base = {
889 			.cra_name		= "hmac(sm3)",
890 			.cra_driver_name	= "sm3-hmac-starfive",
891 			.cra_priority		= 200,
892 			.cra_flags		= CRYPTO_ALG_ASYNC |
893 						  CRYPTO_ALG_TYPE_AHASH |
894 						  CRYPTO_ALG_NEED_FALLBACK,
895 			.cra_blocksize		= SM3_BLOCK_SIZE,
896 			.cra_ctxsize		= sizeof(struct starfive_cryp_ctx),
897 			.cra_alignmask		= 3,
898 			.cra_module		= THIS_MODULE,
899 		}
900 	},
901 	.op = {
902 		.do_one_request = starfive_hash_one_request,
903 	},
904 },
905 };
906 
907 int starfive_hash_register_algs(void)
908 {
909 	return crypto_engine_register_ahashes(algs_sha2_sm3, ARRAY_SIZE(algs_sha2_sm3));
910 }
911 
912 void starfive_hash_unregister_algs(void)
913 {
914 	crypto_engine_unregister_ahashes(algs_sha2_sm3, ARRAY_SIZE(algs_sha2_sm3));
915 }
916