1 // SPDX-License-Identifier: GPL-2.0
2 /* Copyright (c) 2019 HiSilicon Limited. */
3 #include <crypto/akcipher.h>
4 #include <crypto/dh.h>
5 #include <crypto/internal/akcipher.h>
6 #include <crypto/internal/kpp.h>
7 #include <crypto/internal/rsa.h>
8 #include <crypto/kpp.h>
9 #include <crypto/scatterwalk.h>
10 #include <linux/dma-mapping.h>
11 #include <linux/fips.h>
12 #include <linux/module.h>
13 #include "hpre.h"
14 
15 struct hpre_ctx;
16 
17 #define HPRE_CRYPTO_ALG_PRI	1000
18 #define HPRE_ALIGN_SZ		64
19 #define HPRE_BITS_2_BYTES_SHIFT	3
20 #define HPRE_RSA_512BITS_KSZ	64
21 #define HPRE_RSA_1536BITS_KSZ	192
22 #define HPRE_CRT_PRMS		5
23 #define HPRE_CRT_Q		2
24 #define HPRE_CRT_P		3
25 #define HPRE_CRT_INV		4
26 #define HPRE_DH_G_FLAG		0x02
27 #define HPRE_TRY_SEND_TIMES	100
28 #define HPRE_INVLD_REQ_ID		(-1)
29 #define HPRE_DEV(ctx)		(&((ctx)->qp->qm->pdev->dev))
30 
31 #define HPRE_SQE_ALG_BITS	5
32 #define HPRE_SQE_DONE_SHIFT	30
33 #define HPRE_DH_MAX_P_SZ	512
34 
35 typedef void (*hpre_cb)(struct hpre_ctx *ctx, void *sqe);
36 
37 struct hpre_rsa_ctx {
38 	/* low address: e--->n */
39 	char *pubkey;
40 	dma_addr_t dma_pubkey;
41 
42 	/* low address: d--->n */
43 	char *prikey;
44 	dma_addr_t dma_prikey;
45 
46 	/* low address: dq->dp->q->p->qinv */
47 	char *crt_prikey;
48 	dma_addr_t dma_crt_prikey;
49 
50 	struct crypto_akcipher *soft_tfm;
51 };
52 
53 struct hpre_dh_ctx {
54 	/*
55 	 * If base is g we compute the public key
56 	 *	ya = g^xa mod p; [RFC2631 sec 2.1.1]
57 	 * else if base if the counterpart public key we
58 	 * compute the shared secret
59 	 *	ZZ = yb^xa mod p; [RFC2631 sec 2.1.1]
60 	 */
61 	char *xa_p; /* low address: d--->n, please refer to Hisilicon HPRE UM */
62 	dma_addr_t dma_xa_p;
63 
64 	char *g; /* m */
65 	dma_addr_t dma_g;
66 };
67 
68 struct hpre_ctx {
69 	struct hisi_qp *qp;
70 	struct hpre_asym_request **req_list;
71 	spinlock_t req_lock;
72 	unsigned int key_sz;
73 	bool crt_g2_mode;
74 	struct idr req_idr;
75 	union {
76 		struct hpre_rsa_ctx rsa;
77 		struct hpre_dh_ctx dh;
78 	};
79 };
80 
81 struct hpre_asym_request {
82 	char *src;
83 	char *dst;
84 	struct hpre_sqe req;
85 	struct hpre_ctx *ctx;
86 	union {
87 		struct akcipher_request *rsa;
88 		struct kpp_request *dh;
89 	} areq;
90 	int err;
91 	int req_id;
92 	hpre_cb cb;
93 };
94 
95 static DEFINE_MUTEX(hpre_alg_lock);
96 static unsigned int hpre_active_devs;
97 
98 static int hpre_alloc_req_id(struct hpre_ctx *ctx)
99 {
100 	unsigned long flags;
101 	int id;
102 
103 	spin_lock_irqsave(&ctx->req_lock, flags);
104 	id = idr_alloc(&ctx->req_idr, NULL, 0, QM_Q_DEPTH, GFP_ATOMIC);
105 	spin_unlock_irqrestore(&ctx->req_lock, flags);
106 
107 	return id;
108 }
109 
110 static void hpre_free_req_id(struct hpre_ctx *ctx, int req_id)
111 {
112 	unsigned long flags;
113 
114 	spin_lock_irqsave(&ctx->req_lock, flags);
115 	idr_remove(&ctx->req_idr, req_id);
116 	spin_unlock_irqrestore(&ctx->req_lock, flags);
117 }
118 
119 static int hpre_add_req_to_ctx(struct hpre_asym_request *hpre_req)
120 {
121 	struct hpre_ctx *ctx;
122 	int id;
123 
124 	ctx = hpre_req->ctx;
125 	id = hpre_alloc_req_id(ctx);
126 	if (unlikely(id < 0))
127 		return -EINVAL;
128 
129 	ctx->req_list[id] = hpre_req;
130 	hpre_req->req_id = id;
131 
132 	return id;
133 }
134 
135 static void hpre_rm_req_from_ctx(struct hpre_asym_request *hpre_req)
136 {
137 	struct hpre_ctx *ctx = hpre_req->ctx;
138 	int id = hpre_req->req_id;
139 
140 	if (hpre_req->req_id >= 0) {
141 		hpre_req->req_id = HPRE_INVLD_REQ_ID;
142 		ctx->req_list[id] = NULL;
143 		hpre_free_req_id(ctx, id);
144 	}
145 }
146 
147 static struct hisi_qp *hpre_get_qp_and_start(void)
148 {
149 	struct hisi_qp *qp;
150 	struct hpre *hpre;
151 	int ret;
152 
153 	/* find the proper hpre device, which is near the current CPU core */
154 	hpre = hpre_find_device(cpu_to_node(smp_processor_id()));
155 	if (!hpre) {
156 		pr_err("Can not find proper hpre device!\n");
157 		return ERR_PTR(-ENODEV);
158 	}
159 
160 	qp = hisi_qm_create_qp(&hpre->qm, 0);
161 	if (IS_ERR(qp)) {
162 		pci_err(hpre->qm.pdev, "Can not create qp!\n");
163 		return ERR_PTR(-ENODEV);
164 	}
165 
166 	ret = hisi_qm_start_qp(qp, 0);
167 	if (ret < 0) {
168 		hisi_qm_release_qp(qp);
169 		pci_err(hpre->qm.pdev, "Can not start qp!\n");
170 		return ERR_PTR(-EINVAL);
171 	}
172 
173 	return qp;
174 }
175 
176 static int hpre_get_data_dma_addr(struct hpre_asym_request *hpre_req,
177 				  struct scatterlist *data, unsigned int len,
178 				  int is_src, dma_addr_t *tmp)
179 {
180 	struct hpre_ctx *ctx = hpre_req->ctx;
181 	struct device *dev = HPRE_DEV(ctx);
182 	enum dma_data_direction dma_dir;
183 
184 	if (is_src) {
185 		hpre_req->src = NULL;
186 		dma_dir = DMA_TO_DEVICE;
187 	} else {
188 		hpre_req->dst = NULL;
189 		dma_dir = DMA_FROM_DEVICE;
190 	}
191 	*tmp = dma_map_single(dev, sg_virt(data),
192 			      len, dma_dir);
193 	if (unlikely(dma_mapping_error(dev, *tmp))) {
194 		dev_err(dev, "dma map data err!\n");
195 		return -ENOMEM;
196 	}
197 
198 	return 0;
199 }
200 
201 static int hpre_prepare_dma_buf(struct hpre_asym_request *hpre_req,
202 				struct scatterlist *data, unsigned int len,
203 				int is_src, dma_addr_t *tmp)
204 {
205 	struct hpre_ctx *ctx = hpre_req->ctx;
206 	struct device *dev = HPRE_DEV(ctx);
207 	void *ptr;
208 	int shift;
209 
210 	shift = ctx->key_sz - len;
211 	if (unlikely(shift < 0))
212 		return -EINVAL;
213 
214 	ptr = dma_alloc_coherent(dev, ctx->key_sz, tmp, GFP_KERNEL);
215 	if (unlikely(!ptr))
216 		return -ENOMEM;
217 
218 	if (is_src) {
219 		scatterwalk_map_and_copy(ptr + shift, data, 0, len, 0);
220 		hpre_req->src = ptr;
221 	} else {
222 		hpre_req->dst = ptr;
223 	}
224 
225 	return 0;
226 }
227 
228 static int hpre_hw_data_init(struct hpre_asym_request *hpre_req,
229 			     struct scatterlist *data, unsigned int len,
230 			     int is_src, int is_dh)
231 {
232 	struct hpre_sqe *msg = &hpre_req->req;
233 	struct hpre_ctx *ctx = hpre_req->ctx;
234 	dma_addr_t tmp = 0;
235 	int ret;
236 
237 	/* when the data is dh's source, we should format it */
238 	if ((sg_is_last(data) && len == ctx->key_sz) &&
239 	    ((is_dh && !is_src) || !is_dh))
240 		ret = hpre_get_data_dma_addr(hpre_req, data, len, is_src, &tmp);
241 	else
242 		ret = hpre_prepare_dma_buf(hpre_req, data, len,
243 					  is_src, &tmp);
244 	if (unlikely(ret))
245 		return ret;
246 
247 	if (is_src)
248 		msg->in = cpu_to_le64(tmp);
249 	else
250 		msg->out = cpu_to_le64(tmp);
251 
252 	return 0;
253 }
254 
255 static void hpre_hw_data_clr_all(struct hpre_ctx *ctx,
256 				 struct hpre_asym_request *req,
257 				 struct scatterlist *dst,
258 				 struct scatterlist *src)
259 {
260 	struct device *dev = HPRE_DEV(ctx);
261 	struct hpre_sqe *sqe = &req->req;
262 	dma_addr_t tmp;
263 
264 	tmp = le64_to_cpu(sqe->in);
265 	if (unlikely(!tmp))
266 		return;
267 
268 	if (src) {
269 		if (req->src)
270 			dma_free_coherent(dev, ctx->key_sz,
271 					  req->src, tmp);
272 		else
273 			dma_unmap_single(dev, tmp,
274 					 ctx->key_sz, DMA_TO_DEVICE);
275 	}
276 
277 	tmp = le64_to_cpu(sqe->out);
278 	if (unlikely(!tmp))
279 		return;
280 
281 	if (req->dst) {
282 		if (dst)
283 			scatterwalk_map_and_copy(req->dst, dst, 0,
284 						 ctx->key_sz, 1);
285 		dma_free_coherent(dev, ctx->key_sz, req->dst, tmp);
286 	} else {
287 		dma_unmap_single(dev, tmp, ctx->key_sz, DMA_FROM_DEVICE);
288 	}
289 }
290 
291 static int hpre_alg_res_post_hf(struct hpre_ctx *ctx, struct hpre_sqe *sqe,
292 				void **kreq)
293 {
294 	struct hpre_asym_request *req;
295 	int err, id, done;
296 
297 #define HPRE_NO_HW_ERR		0
298 #define HPRE_HW_TASK_DONE	3
299 #define HREE_HW_ERR_MASK	0x7ff
300 #define HREE_SQE_DONE_MASK	0x3
301 	id = (int)le16_to_cpu(sqe->tag);
302 	req = ctx->req_list[id];
303 	hpre_rm_req_from_ctx(req);
304 	*kreq = req;
305 
306 	err = (le32_to_cpu(sqe->dw0) >> HPRE_SQE_ALG_BITS) &
307 		HREE_HW_ERR_MASK;
308 
309 	done = (le32_to_cpu(sqe->dw0) >> HPRE_SQE_DONE_SHIFT) &
310 		HREE_SQE_DONE_MASK;
311 
312 	if (likely(err == HPRE_NO_HW_ERR && done == HPRE_HW_TASK_DONE))
313 		return  0;
314 
315 	return -EINVAL;
316 }
317 
318 static int hpre_ctx_set(struct hpre_ctx *ctx, struct hisi_qp *qp, int qlen)
319 {
320 	if (!ctx || !qp || qlen < 0)
321 		return -EINVAL;
322 
323 	spin_lock_init(&ctx->req_lock);
324 	ctx->qp = qp;
325 
326 	ctx->req_list = kcalloc(qlen, sizeof(void *), GFP_KERNEL);
327 	if (!ctx->req_list)
328 		return -ENOMEM;
329 	ctx->key_sz = 0;
330 	ctx->crt_g2_mode = false;
331 	idr_init(&ctx->req_idr);
332 
333 	return 0;
334 }
335 
336 static void hpre_ctx_clear(struct hpre_ctx *ctx, bool is_clear_all)
337 {
338 	if (is_clear_all) {
339 		idr_destroy(&ctx->req_idr);
340 		kfree(ctx->req_list);
341 		hisi_qm_release_qp(ctx->qp);
342 	}
343 
344 	ctx->crt_g2_mode = false;
345 	ctx->key_sz = 0;
346 }
347 
348 static void hpre_dh_cb(struct hpre_ctx *ctx, void *resp)
349 {
350 	struct hpre_asym_request *req;
351 	struct kpp_request *areq;
352 	int ret;
353 
354 	ret = hpre_alg_res_post_hf(ctx, resp, (void **)&req);
355 	areq = req->areq.dh;
356 	areq->dst_len = ctx->key_sz;
357 	hpre_hw_data_clr_all(ctx, req, areq->dst, areq->src);
358 	kpp_request_complete(areq, ret);
359 }
360 
361 static void hpre_rsa_cb(struct hpre_ctx *ctx, void *resp)
362 {
363 	struct hpre_asym_request *req;
364 	struct akcipher_request *areq;
365 	int ret;
366 
367 	ret = hpre_alg_res_post_hf(ctx, resp, (void **)&req);
368 	areq = req->areq.rsa;
369 	areq->dst_len = ctx->key_sz;
370 	hpre_hw_data_clr_all(ctx, req, areq->dst, areq->src);
371 	akcipher_request_complete(areq, ret);
372 }
373 
374 static void hpre_alg_cb(struct hisi_qp *qp, void *resp)
375 {
376 	struct hpre_ctx *ctx = qp->qp_ctx;
377 	struct hpre_sqe *sqe = resp;
378 
379 	ctx->req_list[le16_to_cpu(sqe->tag)]->cb(ctx, resp);
380 }
381 
382 static int hpre_ctx_init(struct hpre_ctx *ctx)
383 {
384 	struct hisi_qp *qp;
385 
386 	qp = hpre_get_qp_and_start();
387 	if (IS_ERR(qp))
388 		return PTR_ERR(qp);
389 
390 	qp->qp_ctx = ctx;
391 	qp->req_cb = hpre_alg_cb;
392 
393 	return hpre_ctx_set(ctx, qp, QM_Q_DEPTH);
394 }
395 
396 static int hpre_msg_request_set(struct hpre_ctx *ctx, void *req, bool is_rsa)
397 {
398 	struct hpre_asym_request *h_req;
399 	struct hpre_sqe *msg;
400 	int req_id;
401 	void *tmp;
402 
403 	if (is_rsa) {
404 		struct akcipher_request *akreq = req;
405 
406 		if (akreq->dst_len < ctx->key_sz) {
407 			akreq->dst_len = ctx->key_sz;
408 			return -EOVERFLOW;
409 		}
410 
411 		tmp = akcipher_request_ctx(akreq);
412 		h_req = PTR_ALIGN(tmp, HPRE_ALIGN_SZ);
413 		h_req->cb = hpre_rsa_cb;
414 		h_req->areq.rsa = akreq;
415 		msg = &h_req->req;
416 		memset(msg, 0, sizeof(*msg));
417 	} else {
418 		struct kpp_request *kreq = req;
419 
420 		if (kreq->dst_len < ctx->key_sz) {
421 			kreq->dst_len = ctx->key_sz;
422 			return -EOVERFLOW;
423 		}
424 
425 		tmp = kpp_request_ctx(kreq);
426 		h_req = PTR_ALIGN(tmp, HPRE_ALIGN_SZ);
427 		h_req->cb = hpre_dh_cb;
428 		h_req->areq.dh = kreq;
429 		msg = &h_req->req;
430 		memset(msg, 0, sizeof(*msg));
431 		msg->key = cpu_to_le64((u64)ctx->dh.dma_xa_p);
432 	}
433 
434 	msg->dw0 |= cpu_to_le32(0x1 << HPRE_SQE_DONE_SHIFT);
435 	msg->task_len1 = (ctx->key_sz >> HPRE_BITS_2_BYTES_SHIFT) - 1;
436 	h_req->ctx = ctx;
437 
438 	req_id = hpre_add_req_to_ctx(h_req);
439 	if (req_id < 0)
440 		return -EBUSY;
441 
442 	msg->tag = cpu_to_le16((u16)req_id);
443 
444 	return 0;
445 }
446 
447 #ifdef CONFIG_CRYPTO_DH
448 static int hpre_dh_compute_value(struct kpp_request *req)
449 {
450 	struct crypto_kpp *tfm = crypto_kpp_reqtfm(req);
451 	struct hpre_ctx *ctx = kpp_tfm_ctx(tfm);
452 	void *tmp = kpp_request_ctx(req);
453 	struct hpre_asym_request *hpre_req = PTR_ALIGN(tmp, HPRE_ALIGN_SZ);
454 	struct hpre_sqe *msg = &hpre_req->req;
455 	int ctr = 0;
456 	int ret;
457 
458 	ret = hpre_msg_request_set(ctx, req, false);
459 	if (unlikely(ret))
460 		return ret;
461 
462 	if (req->src) {
463 		ret = hpre_hw_data_init(hpre_req, req->src, req->src_len, 1, 1);
464 		if (unlikely(ret))
465 			goto clear_all;
466 	}
467 
468 	ret = hpre_hw_data_init(hpre_req, req->dst, req->dst_len, 0, 1);
469 	if (unlikely(ret))
470 		goto clear_all;
471 
472 	if (ctx->crt_g2_mode && !req->src)
473 		msg->dw0 = cpu_to_le32(le32_to_cpu(msg->dw0) | HPRE_ALG_DH_G2);
474 	else
475 		msg->dw0 = cpu_to_le32(le32_to_cpu(msg->dw0) | HPRE_ALG_DH);
476 	do {
477 		ret = hisi_qp_send(ctx->qp, msg);
478 	} while (ret == -EBUSY && ctr++ < HPRE_TRY_SEND_TIMES);
479 
480 	/* success */
481 	if (likely(!ret))
482 		return -EINPROGRESS;
483 
484 clear_all:
485 	hpre_rm_req_from_ctx(hpre_req);
486 	hpre_hw_data_clr_all(ctx, hpre_req, req->dst, req->src);
487 
488 	return ret;
489 }
490 
491 static int hpre_is_dh_params_length_valid(unsigned int key_sz)
492 {
493 #define _HPRE_DH_GRP1		768
494 #define _HPRE_DH_GRP2		1024
495 #define _HPRE_DH_GRP5		1536
496 #define _HPRE_DH_GRP14		2048
497 #define _HPRE_DH_GRP15		3072
498 #define _HPRE_DH_GRP16		4096
499 	switch (key_sz) {
500 	case _HPRE_DH_GRP1:
501 	case _HPRE_DH_GRP2:
502 	case _HPRE_DH_GRP5:
503 	case _HPRE_DH_GRP14:
504 	case _HPRE_DH_GRP15:
505 	case _HPRE_DH_GRP16:
506 		return 0;
507 	}
508 
509 	return -EINVAL;
510 }
511 
512 static int hpre_dh_set_params(struct hpre_ctx *ctx, struct dh *params)
513 {
514 	struct device *dev = HPRE_DEV(ctx);
515 	unsigned int sz;
516 
517 	if (params->p_size > HPRE_DH_MAX_P_SZ)
518 		return -EINVAL;
519 
520 	if (hpre_is_dh_params_length_valid(params->p_size <<
521 					   HPRE_BITS_2_BYTES_SHIFT))
522 		return -EINVAL;
523 
524 	sz = ctx->key_sz = params->p_size;
525 	ctx->dh.xa_p = dma_alloc_coherent(dev, sz << 1,
526 					  &ctx->dh.dma_xa_p, GFP_KERNEL);
527 	if (!ctx->dh.xa_p)
528 		return -ENOMEM;
529 
530 	memcpy(ctx->dh.xa_p + sz, params->p, sz);
531 
532 	/* If g equals 2 don't copy it */
533 	if (params->g_size == 1 && *(char *)params->g == HPRE_DH_G_FLAG) {
534 		ctx->crt_g2_mode = true;
535 		return 0;
536 	}
537 
538 	ctx->dh.g = dma_alloc_coherent(dev, sz, &ctx->dh.dma_g, GFP_KERNEL);
539 	if (!ctx->dh.g) {
540 		dma_free_coherent(dev, sz << 1, ctx->dh.xa_p,
541 				  ctx->dh.dma_xa_p);
542 		ctx->dh.xa_p = NULL;
543 		return -ENOMEM;
544 	}
545 
546 	memcpy(ctx->dh.g + (sz - params->g_size), params->g, params->g_size);
547 
548 	return 0;
549 }
550 
551 static void hpre_dh_clear_ctx(struct hpre_ctx *ctx, bool is_clear_all)
552 {
553 	struct device *dev = HPRE_DEV(ctx);
554 	unsigned int sz = ctx->key_sz;
555 
556 	if (is_clear_all)
557 		hisi_qm_stop_qp(ctx->qp);
558 
559 	if (ctx->dh.g) {
560 		dma_free_coherent(dev, sz, ctx->dh.g, ctx->dh.dma_g);
561 		ctx->dh.g = NULL;
562 	}
563 
564 	if (ctx->dh.xa_p) {
565 		memzero_explicit(ctx->dh.xa_p, sz);
566 		dma_free_coherent(dev, sz << 1, ctx->dh.xa_p,
567 				  ctx->dh.dma_xa_p);
568 		ctx->dh.xa_p = NULL;
569 	}
570 
571 	hpre_ctx_clear(ctx, is_clear_all);
572 }
573 
574 static int hpre_dh_set_secret(struct crypto_kpp *tfm, const void *buf,
575 			      unsigned int len)
576 {
577 	struct hpre_ctx *ctx = kpp_tfm_ctx(tfm);
578 	struct dh params;
579 	int ret;
580 
581 	if (crypto_dh_decode_key(buf, len, &params) < 0)
582 		return -EINVAL;
583 
584 	/* Free old secret if any */
585 	hpre_dh_clear_ctx(ctx, false);
586 
587 	ret = hpre_dh_set_params(ctx, &params);
588 	if (ret < 0)
589 		goto err_clear_ctx;
590 
591 	memcpy(ctx->dh.xa_p + (ctx->key_sz - params.key_size), params.key,
592 	       params.key_size);
593 
594 	return 0;
595 
596 err_clear_ctx:
597 	hpre_dh_clear_ctx(ctx, false);
598 	return ret;
599 }
600 
601 static unsigned int hpre_dh_max_size(struct crypto_kpp *tfm)
602 {
603 	struct hpre_ctx *ctx = kpp_tfm_ctx(tfm);
604 
605 	return ctx->key_sz;
606 }
607 
608 static int hpre_dh_init_tfm(struct crypto_kpp *tfm)
609 {
610 	struct hpre_ctx *ctx = kpp_tfm_ctx(tfm);
611 
612 	return hpre_ctx_init(ctx);
613 }
614 
615 static void hpre_dh_exit_tfm(struct crypto_kpp *tfm)
616 {
617 	struct hpre_ctx *ctx = kpp_tfm_ctx(tfm);
618 
619 	hpre_dh_clear_ctx(ctx, true);
620 }
621 #endif
622 
623 static void hpre_rsa_drop_leading_zeros(const char **ptr, size_t *len)
624 {
625 	while (!**ptr && *len) {
626 		(*ptr)++;
627 		(*len)--;
628 	}
629 }
630 
631 static bool hpre_rsa_key_size_is_support(unsigned int len)
632 {
633 	unsigned int bits = len << HPRE_BITS_2_BYTES_SHIFT;
634 
635 #define _RSA_1024BITS_KEY_WDTH		1024
636 #define _RSA_2048BITS_KEY_WDTH		2048
637 #define _RSA_3072BITS_KEY_WDTH		3072
638 #define _RSA_4096BITS_KEY_WDTH		4096
639 
640 	switch (bits) {
641 	case _RSA_1024BITS_KEY_WDTH:
642 	case _RSA_2048BITS_KEY_WDTH:
643 	case _RSA_3072BITS_KEY_WDTH:
644 	case _RSA_4096BITS_KEY_WDTH:
645 		return true;
646 	default:
647 		return false;
648 	}
649 }
650 
651 static int hpre_rsa_enc(struct akcipher_request *req)
652 {
653 	struct crypto_akcipher *tfm = crypto_akcipher_reqtfm(req);
654 	struct hpre_ctx *ctx = akcipher_tfm_ctx(tfm);
655 	void *tmp = akcipher_request_ctx(req);
656 	struct hpre_asym_request *hpre_req = PTR_ALIGN(tmp, HPRE_ALIGN_SZ);
657 	struct hpre_sqe *msg = &hpre_req->req;
658 	int ctr = 0;
659 	int ret;
660 
661 	/* For 512 and 1536 bits key size, use soft tfm instead */
662 	if (ctx->key_sz == HPRE_RSA_512BITS_KSZ ||
663 	    ctx->key_sz == HPRE_RSA_1536BITS_KSZ) {
664 		akcipher_request_set_tfm(req, ctx->rsa.soft_tfm);
665 		ret = crypto_akcipher_encrypt(req);
666 		akcipher_request_set_tfm(req, tfm);
667 		return ret;
668 	}
669 
670 	if (unlikely(!ctx->rsa.pubkey))
671 		return -EINVAL;
672 
673 	ret = hpre_msg_request_set(ctx, req, true);
674 	if (unlikely(ret))
675 		return ret;
676 
677 	msg->dw0 |= cpu_to_le32(HPRE_ALG_NC_NCRT);
678 	msg->key = cpu_to_le64((u64)ctx->rsa.dma_pubkey);
679 
680 	ret = hpre_hw_data_init(hpre_req, req->src, req->src_len, 1, 0);
681 	if (unlikely(ret))
682 		goto clear_all;
683 
684 	ret = hpre_hw_data_init(hpre_req, req->dst, req->dst_len, 0, 0);
685 	if (unlikely(ret))
686 		goto clear_all;
687 
688 	do {
689 		ret = hisi_qp_send(ctx->qp, msg);
690 	} while (ret == -EBUSY && ctr++ < HPRE_TRY_SEND_TIMES);
691 
692 	/* success */
693 	if (likely(!ret))
694 		return -EINPROGRESS;
695 
696 clear_all:
697 	hpre_rm_req_from_ctx(hpre_req);
698 	hpre_hw_data_clr_all(ctx, hpre_req, req->dst, req->src);
699 
700 	return ret;
701 }
702 
703 static int hpre_rsa_dec(struct akcipher_request *req)
704 {
705 	struct crypto_akcipher *tfm = crypto_akcipher_reqtfm(req);
706 	struct hpre_ctx *ctx = akcipher_tfm_ctx(tfm);
707 	void *tmp = akcipher_request_ctx(req);
708 	struct hpre_asym_request *hpre_req = PTR_ALIGN(tmp, HPRE_ALIGN_SZ);
709 	struct hpre_sqe *msg = &hpre_req->req;
710 	int ctr = 0;
711 	int ret;
712 
713 	/* For 512 and 1536 bits key size, use soft tfm instead */
714 	if (ctx->key_sz == HPRE_RSA_512BITS_KSZ ||
715 	    ctx->key_sz == HPRE_RSA_1536BITS_KSZ) {
716 		akcipher_request_set_tfm(req, ctx->rsa.soft_tfm);
717 		ret = crypto_akcipher_decrypt(req);
718 		akcipher_request_set_tfm(req, tfm);
719 		return ret;
720 	}
721 
722 	if (unlikely(!ctx->rsa.prikey))
723 		return -EINVAL;
724 
725 	ret = hpre_msg_request_set(ctx, req, true);
726 	if (unlikely(ret))
727 		return ret;
728 
729 	if (ctx->crt_g2_mode) {
730 		msg->key = cpu_to_le64((u64)ctx->rsa.dma_crt_prikey);
731 		msg->dw0 = cpu_to_le32(le32_to_cpu(msg->dw0) |
732 				       HPRE_ALG_NC_CRT);
733 	} else {
734 		msg->key = cpu_to_le64((u64)ctx->rsa.dma_prikey);
735 		msg->dw0 = cpu_to_le32(le32_to_cpu(msg->dw0) |
736 				       HPRE_ALG_NC_NCRT);
737 	}
738 
739 	ret = hpre_hw_data_init(hpre_req, req->src, req->src_len, 1, 0);
740 	if (unlikely(ret))
741 		goto clear_all;
742 
743 	ret = hpre_hw_data_init(hpre_req, req->dst, req->dst_len, 0, 0);
744 	if (unlikely(ret))
745 		goto clear_all;
746 
747 	do {
748 		ret = hisi_qp_send(ctx->qp, msg);
749 	} while (ret == -EBUSY && ctr++ < HPRE_TRY_SEND_TIMES);
750 
751 	/* success */
752 	if (likely(!ret))
753 		return -EINPROGRESS;
754 
755 clear_all:
756 	hpre_rm_req_from_ctx(hpre_req);
757 	hpre_hw_data_clr_all(ctx, hpre_req, req->dst, req->src);
758 
759 	return ret;
760 }
761 
762 static int hpre_rsa_set_n(struct hpre_ctx *ctx, const char *value,
763 			  size_t vlen, bool private)
764 {
765 	const char *ptr = value;
766 
767 	hpre_rsa_drop_leading_zeros(&ptr, &vlen);
768 
769 	ctx->key_sz = vlen;
770 
771 	/* if invalid key size provided, we use software tfm */
772 	if (!hpre_rsa_key_size_is_support(ctx->key_sz))
773 		return 0;
774 
775 	ctx->rsa.pubkey = dma_alloc_coherent(HPRE_DEV(ctx), vlen << 1,
776 					     &ctx->rsa.dma_pubkey,
777 					     GFP_KERNEL);
778 	if (!ctx->rsa.pubkey)
779 		return -ENOMEM;
780 
781 	if (private) {
782 		ctx->rsa.prikey = dma_alloc_coherent(HPRE_DEV(ctx), vlen << 1,
783 						     &ctx->rsa.dma_prikey,
784 						     GFP_KERNEL);
785 		if (!ctx->rsa.prikey) {
786 			dma_free_coherent(HPRE_DEV(ctx), vlen << 1,
787 					  ctx->rsa.pubkey,
788 					  ctx->rsa.dma_pubkey);
789 			ctx->rsa.pubkey = NULL;
790 			return -ENOMEM;
791 		}
792 		memcpy(ctx->rsa.prikey + vlen, ptr, vlen);
793 	}
794 	memcpy(ctx->rsa.pubkey + vlen, ptr, vlen);
795 
796 	/* Using hardware HPRE to do RSA */
797 	return 1;
798 }
799 
800 static int hpre_rsa_set_e(struct hpre_ctx *ctx, const char *value,
801 			  size_t vlen)
802 {
803 	const char *ptr = value;
804 
805 	hpre_rsa_drop_leading_zeros(&ptr, &vlen);
806 
807 	if (!ctx->key_sz || !vlen || vlen > ctx->key_sz)
808 		return -EINVAL;
809 
810 	memcpy(ctx->rsa.pubkey + ctx->key_sz - vlen, ptr, vlen);
811 
812 	return 0;
813 }
814 
815 static int hpre_rsa_set_d(struct hpre_ctx *ctx, const char *value,
816 			  size_t vlen)
817 {
818 	const char *ptr = value;
819 
820 	hpre_rsa_drop_leading_zeros(&ptr, &vlen);
821 
822 	if (!ctx->key_sz || !vlen || vlen > ctx->key_sz)
823 		return -EINVAL;
824 
825 	memcpy(ctx->rsa.prikey + ctx->key_sz - vlen, ptr, vlen);
826 
827 	return 0;
828 }
829 
830 static int hpre_crt_para_get(char *para, size_t para_sz,
831 			     const char *raw, size_t raw_sz)
832 {
833 	const char *ptr = raw;
834 	size_t len = raw_sz;
835 
836 	hpre_rsa_drop_leading_zeros(&ptr, &len);
837 	if (!len || len > para_sz)
838 		return -EINVAL;
839 
840 	memcpy(para + para_sz - len, ptr, len);
841 
842 	return 0;
843 }
844 
845 static int hpre_rsa_setkey_crt(struct hpre_ctx *ctx, struct rsa_key *rsa_key)
846 {
847 	unsigned int hlf_ksz = ctx->key_sz >> 1;
848 	struct device *dev = HPRE_DEV(ctx);
849 	u64 offset;
850 	int ret;
851 
852 	ctx->rsa.crt_prikey = dma_alloc_coherent(dev, hlf_ksz * HPRE_CRT_PRMS,
853 					&ctx->rsa.dma_crt_prikey,
854 					GFP_KERNEL);
855 	if (!ctx->rsa.crt_prikey)
856 		return -ENOMEM;
857 
858 	ret = hpre_crt_para_get(ctx->rsa.crt_prikey, hlf_ksz,
859 				rsa_key->dq, rsa_key->dq_sz);
860 	if (ret)
861 		goto free_key;
862 
863 	offset = hlf_ksz;
864 	ret = hpre_crt_para_get(ctx->rsa.crt_prikey + offset, hlf_ksz,
865 				rsa_key->dp, rsa_key->dp_sz);
866 	if (ret)
867 		goto free_key;
868 
869 	offset = hlf_ksz * HPRE_CRT_Q;
870 	ret = hpre_crt_para_get(ctx->rsa.crt_prikey + offset, hlf_ksz,
871 				rsa_key->q, rsa_key->q_sz);
872 	if (ret)
873 		goto free_key;
874 
875 	offset = hlf_ksz * HPRE_CRT_P;
876 	ret = hpre_crt_para_get(ctx->rsa.crt_prikey + offset, hlf_ksz,
877 				rsa_key->p, rsa_key->p_sz);
878 	if (ret)
879 		goto free_key;
880 
881 	offset = hlf_ksz * HPRE_CRT_INV;
882 	ret = hpre_crt_para_get(ctx->rsa.crt_prikey + offset, hlf_ksz,
883 				rsa_key->qinv, rsa_key->qinv_sz);
884 	if (ret)
885 		goto free_key;
886 
887 	ctx->crt_g2_mode = true;
888 
889 	return 0;
890 
891 free_key:
892 	offset = hlf_ksz * HPRE_CRT_PRMS;
893 	memzero_explicit(ctx->rsa.crt_prikey, offset);
894 	dma_free_coherent(dev, hlf_ksz * HPRE_CRT_PRMS, ctx->rsa.crt_prikey,
895 			  ctx->rsa.dma_crt_prikey);
896 	ctx->rsa.crt_prikey = NULL;
897 	ctx->crt_g2_mode = false;
898 
899 	return ret;
900 }
901 
902 /* If it is clear all, all the resources of the QP will be cleaned. */
903 static void hpre_rsa_clear_ctx(struct hpre_ctx *ctx, bool is_clear_all)
904 {
905 	unsigned int half_key_sz = ctx->key_sz >> 1;
906 	struct device *dev = HPRE_DEV(ctx);
907 
908 	if (is_clear_all)
909 		hisi_qm_stop_qp(ctx->qp);
910 
911 	if (ctx->rsa.pubkey) {
912 		dma_free_coherent(dev, ctx->key_sz << 1,
913 				  ctx->rsa.pubkey, ctx->rsa.dma_pubkey);
914 		ctx->rsa.pubkey = NULL;
915 	}
916 
917 	if (ctx->rsa.crt_prikey) {
918 		memzero_explicit(ctx->rsa.crt_prikey,
919 				 half_key_sz * HPRE_CRT_PRMS);
920 		dma_free_coherent(dev, half_key_sz * HPRE_CRT_PRMS,
921 				  ctx->rsa.crt_prikey, ctx->rsa.dma_crt_prikey);
922 		ctx->rsa.crt_prikey = NULL;
923 	}
924 
925 	if (ctx->rsa.prikey) {
926 		memzero_explicit(ctx->rsa.prikey, ctx->key_sz);
927 		dma_free_coherent(dev, ctx->key_sz << 1, ctx->rsa.prikey,
928 				  ctx->rsa.dma_prikey);
929 		ctx->rsa.prikey = NULL;
930 	}
931 
932 	hpre_ctx_clear(ctx, is_clear_all);
933 }
934 
935 /*
936  * we should judge if it is CRT or not,
937  * CRT: return true,  N-CRT: return false .
938  */
939 static bool hpre_is_crt_key(struct rsa_key *key)
940 {
941 	u16 len = key->p_sz + key->q_sz + key->dp_sz + key->dq_sz +
942 		  key->qinv_sz;
943 
944 #define LEN_OF_NCRT_PARA	5
945 
946 	/* N-CRT less than 5 parameters */
947 	return len > LEN_OF_NCRT_PARA;
948 }
949 
950 static int hpre_rsa_setkey(struct hpre_ctx *ctx, const void *key,
951 			   unsigned int keylen, bool private)
952 {
953 	struct rsa_key rsa_key;
954 	int ret;
955 
956 	hpre_rsa_clear_ctx(ctx, false);
957 
958 	if (private)
959 		ret = rsa_parse_priv_key(&rsa_key, key, keylen);
960 	else
961 		ret = rsa_parse_pub_key(&rsa_key, key, keylen);
962 	if (ret < 0)
963 		return ret;
964 
965 	ret = hpre_rsa_set_n(ctx, rsa_key.n, rsa_key.n_sz, private);
966 	if (ret <= 0)
967 		return ret;
968 
969 	if (private) {
970 		ret = hpre_rsa_set_d(ctx, rsa_key.d, rsa_key.d_sz);
971 		if (ret < 0)
972 			goto free;
973 
974 		if (hpre_is_crt_key(&rsa_key)) {
975 			ret = hpre_rsa_setkey_crt(ctx, &rsa_key);
976 			if (ret < 0)
977 				goto free;
978 		}
979 	}
980 
981 	ret = hpre_rsa_set_e(ctx, rsa_key.e, rsa_key.e_sz);
982 	if (ret < 0)
983 		goto free;
984 
985 	if ((private && !ctx->rsa.prikey) || !ctx->rsa.pubkey) {
986 		ret = -EINVAL;
987 		goto free;
988 	}
989 
990 	return 0;
991 
992 free:
993 	hpre_rsa_clear_ctx(ctx, false);
994 	return ret;
995 }
996 
997 static int hpre_rsa_setpubkey(struct crypto_akcipher *tfm, const void *key,
998 			      unsigned int keylen)
999 {
1000 	struct hpre_ctx *ctx = akcipher_tfm_ctx(tfm);
1001 	int ret;
1002 
1003 	ret = crypto_akcipher_set_pub_key(ctx->rsa.soft_tfm, key, keylen);
1004 	if (ret)
1005 		return ret;
1006 
1007 	return hpre_rsa_setkey(ctx, key, keylen, false);
1008 }
1009 
1010 static int hpre_rsa_setprivkey(struct crypto_akcipher *tfm, const void *key,
1011 			       unsigned int keylen)
1012 {
1013 	struct hpre_ctx *ctx = akcipher_tfm_ctx(tfm);
1014 	int ret;
1015 
1016 	ret = crypto_akcipher_set_priv_key(ctx->rsa.soft_tfm, key, keylen);
1017 	if (ret)
1018 		return ret;
1019 
1020 	return hpre_rsa_setkey(ctx, key, keylen, true);
1021 }
1022 
1023 static unsigned int hpre_rsa_max_size(struct crypto_akcipher *tfm)
1024 {
1025 	struct hpre_ctx *ctx = akcipher_tfm_ctx(tfm);
1026 
1027 	/* For 512 and 1536 bits key size, use soft tfm instead */
1028 	if (ctx->key_sz == HPRE_RSA_512BITS_KSZ ||
1029 	    ctx->key_sz == HPRE_RSA_1536BITS_KSZ)
1030 		return crypto_akcipher_maxsize(ctx->rsa.soft_tfm);
1031 
1032 	return ctx->key_sz;
1033 }
1034 
1035 static int hpre_rsa_init_tfm(struct crypto_akcipher *tfm)
1036 {
1037 	struct hpre_ctx *ctx = akcipher_tfm_ctx(tfm);
1038 	int ret;
1039 
1040 	ctx->rsa.soft_tfm = crypto_alloc_akcipher("rsa-generic", 0, 0);
1041 	if (IS_ERR(ctx->rsa.soft_tfm)) {
1042 		pr_err("Can not alloc_akcipher!\n");
1043 		return PTR_ERR(ctx->rsa.soft_tfm);
1044 	}
1045 
1046 	ret = hpre_ctx_init(ctx);
1047 	if (ret)
1048 		crypto_free_akcipher(ctx->rsa.soft_tfm);
1049 
1050 	return ret;
1051 }
1052 
1053 static void hpre_rsa_exit_tfm(struct crypto_akcipher *tfm)
1054 {
1055 	struct hpre_ctx *ctx = akcipher_tfm_ctx(tfm);
1056 
1057 	hpre_rsa_clear_ctx(ctx, true);
1058 	crypto_free_akcipher(ctx->rsa.soft_tfm);
1059 }
1060 
1061 static struct akcipher_alg rsa = {
1062 	.sign = hpre_rsa_dec,
1063 	.verify = hpre_rsa_enc,
1064 	.encrypt = hpre_rsa_enc,
1065 	.decrypt = hpre_rsa_dec,
1066 	.set_pub_key = hpre_rsa_setpubkey,
1067 	.set_priv_key = hpre_rsa_setprivkey,
1068 	.max_size = hpre_rsa_max_size,
1069 	.init = hpre_rsa_init_tfm,
1070 	.exit = hpre_rsa_exit_tfm,
1071 	.reqsize = sizeof(struct hpre_asym_request) + HPRE_ALIGN_SZ,
1072 	.base = {
1073 		.cra_ctxsize = sizeof(struct hpre_ctx),
1074 		.cra_priority = HPRE_CRYPTO_ALG_PRI,
1075 		.cra_name = "rsa",
1076 		.cra_driver_name = "hpre-rsa",
1077 		.cra_module = THIS_MODULE,
1078 	},
1079 };
1080 
1081 #ifdef CONFIG_CRYPTO_DH
1082 static struct kpp_alg dh = {
1083 	.set_secret = hpre_dh_set_secret,
1084 	.generate_public_key = hpre_dh_compute_value,
1085 	.compute_shared_secret = hpre_dh_compute_value,
1086 	.max_size = hpre_dh_max_size,
1087 	.init = hpre_dh_init_tfm,
1088 	.exit = hpre_dh_exit_tfm,
1089 	.reqsize = sizeof(struct hpre_asym_request) + HPRE_ALIGN_SZ,
1090 	.base = {
1091 		.cra_ctxsize = sizeof(struct hpre_ctx),
1092 		.cra_priority = HPRE_CRYPTO_ALG_PRI,
1093 		.cra_name = "dh",
1094 		.cra_driver_name = "hpre-dh",
1095 		.cra_module = THIS_MODULE,
1096 	},
1097 };
1098 #endif
1099 
1100 int hpre_algs_register(void)
1101 {
1102 	int ret = 0;
1103 
1104 	mutex_lock(&hpre_alg_lock);
1105 	if (++hpre_active_devs == 1) {
1106 		rsa.base.cra_flags = 0;
1107 		ret = crypto_register_akcipher(&rsa);
1108 		if (ret)
1109 			goto unlock;
1110 #ifdef CONFIG_CRYPTO_DH
1111 		ret = crypto_register_kpp(&dh);
1112 		if (ret) {
1113 			crypto_unregister_akcipher(&rsa);
1114 			goto unlock;
1115 		}
1116 #endif
1117 	}
1118 
1119 unlock:
1120 	mutex_unlock(&hpre_alg_lock);
1121 	return ret;
1122 }
1123 
1124 void hpre_algs_unregister(void)
1125 {
1126 	mutex_lock(&hpre_alg_lock);
1127 	if (--hpre_active_devs == 0) {
1128 		crypto_unregister_akcipher(&rsa);
1129 #ifdef CONFIG_CRYPTO_DH
1130 		crypto_unregister_kpp(&dh);
1131 #endif
1132 	}
1133 	mutex_unlock(&hpre_alg_lock);
1134 }
1135