1 // SPDX-License-Identifier: GPL-2.0
2 /* Copyright (c) 2019 HiSilicon Limited. */
3 #include <crypto/akcipher.h>
4 #include <crypto/curve25519.h>
5 #include <crypto/dh.h>
6 #include <crypto/ecc_curve.h>
7 #include <crypto/ecdh.h>
8 #include <crypto/rng.h>
9 #include <crypto/internal/akcipher.h>
10 #include <crypto/internal/kpp.h>
11 #include <crypto/internal/rsa.h>
12 #include <crypto/kpp.h>
13 #include <crypto/scatterwalk.h>
14 #include <linux/dma-mapping.h>
15 #include <linux/fips.h>
16 #include <linux/module.h>
17 #include <linux/time.h>
18 #include "hpre.h"
19 
20 struct hpre_ctx;
21 
22 #define HPRE_CRYPTO_ALG_PRI	1000
23 #define HPRE_ALIGN_SZ		64
24 #define HPRE_BITS_2_BYTES_SHIFT	3
25 #define HPRE_RSA_512BITS_KSZ	64
26 #define HPRE_RSA_1536BITS_KSZ	192
27 #define HPRE_CRT_PRMS		5
28 #define HPRE_CRT_Q		2
29 #define HPRE_CRT_P		3
30 #define HPRE_CRT_INV		4
31 #define HPRE_DH_G_FLAG		0x02
32 #define HPRE_TRY_SEND_TIMES	100
33 #define HPRE_INVLD_REQ_ID		(-1)
34 
35 #define HPRE_SQE_ALG_BITS	5
36 #define HPRE_SQE_DONE_SHIFT	30
37 #define HPRE_DH_MAX_P_SZ	512
38 
39 #define HPRE_DFX_SEC_TO_US	1000000
40 #define HPRE_DFX_US_TO_NS	1000
41 
42 /* due to nist p521  */
43 #define HPRE_ECC_MAX_KSZ	66
44 
45 /* size in bytes of the n prime */
46 #define HPRE_ECC_NIST_P192_N_SIZE	24
47 #define HPRE_ECC_NIST_P256_N_SIZE	32
48 #define HPRE_ECC_NIST_P384_N_SIZE	48
49 
50 /* size in bytes */
51 #define HPRE_ECC_HW256_KSZ_B	32
52 #define HPRE_ECC_HW384_KSZ_B	48
53 
54 typedef void (*hpre_cb)(struct hpre_ctx *ctx, void *sqe);
55 
56 struct hpre_rsa_ctx {
57 	/* low address: e--->n */
58 	char *pubkey;
59 	dma_addr_t dma_pubkey;
60 
61 	/* low address: d--->n */
62 	char *prikey;
63 	dma_addr_t dma_prikey;
64 
65 	/* low address: dq->dp->q->p->qinv */
66 	char *crt_prikey;
67 	dma_addr_t dma_crt_prikey;
68 
69 	struct crypto_akcipher *soft_tfm;
70 };
71 
72 struct hpre_dh_ctx {
73 	/*
74 	 * If base is g we compute the public key
75 	 *	ya = g^xa mod p; [RFC2631 sec 2.1.1]
76 	 * else if base if the counterpart public key we
77 	 * compute the shared secret
78 	 *	ZZ = yb^xa mod p; [RFC2631 sec 2.1.1]
79 	 * low address: d--->n, please refer to Hisilicon HPRE UM
80 	 */
81 	char *xa_p;
82 	dma_addr_t dma_xa_p;
83 
84 	char *g; /* m */
85 	dma_addr_t dma_g;
86 };
87 
88 struct hpre_ecdh_ctx {
89 	/* low address: p->a->k->b */
90 	unsigned char *p;
91 	dma_addr_t dma_p;
92 
93 	/* low address: x->y */
94 	unsigned char *g;
95 	dma_addr_t dma_g;
96 };
97 
98 struct hpre_curve25519_ctx {
99 	/* low address: p->a->k */
100 	unsigned char *p;
101 	dma_addr_t dma_p;
102 
103 	/* gx coordinate */
104 	unsigned char *g;
105 	dma_addr_t dma_g;
106 };
107 
108 struct hpre_ctx {
109 	struct hisi_qp *qp;
110 	struct device *dev;
111 	struct hpre_asym_request **req_list;
112 	struct hpre *hpre;
113 	spinlock_t req_lock;
114 	unsigned int key_sz;
115 	bool crt_g2_mode;
116 	struct idr req_idr;
117 	union {
118 		struct hpre_rsa_ctx rsa;
119 		struct hpre_dh_ctx dh;
120 		struct hpre_ecdh_ctx ecdh;
121 		struct hpre_curve25519_ctx curve25519;
122 	};
123 	/* for ecc algorithms */
124 	unsigned int curve_id;
125 };
126 
127 struct hpre_asym_request {
128 	char *src;
129 	char *dst;
130 	struct hpre_sqe req;
131 	struct hpre_ctx *ctx;
132 	union {
133 		struct akcipher_request *rsa;
134 		struct kpp_request *dh;
135 		struct kpp_request *ecdh;
136 		struct kpp_request *curve25519;
137 	} areq;
138 	int err;
139 	int req_id;
140 	hpre_cb cb;
141 	struct timespec64 req_time;
142 };
143 
144 static int hpre_alloc_req_id(struct hpre_ctx *ctx)
145 {
146 	unsigned long flags;
147 	int id;
148 
149 	spin_lock_irqsave(&ctx->req_lock, flags);
150 	id = idr_alloc(&ctx->req_idr, NULL, 0, QM_Q_DEPTH, GFP_ATOMIC);
151 	spin_unlock_irqrestore(&ctx->req_lock, flags);
152 
153 	return id;
154 }
155 
156 static void hpre_free_req_id(struct hpre_ctx *ctx, int req_id)
157 {
158 	unsigned long flags;
159 
160 	spin_lock_irqsave(&ctx->req_lock, flags);
161 	idr_remove(&ctx->req_idr, req_id);
162 	spin_unlock_irqrestore(&ctx->req_lock, flags);
163 }
164 
165 static int hpre_add_req_to_ctx(struct hpre_asym_request *hpre_req)
166 {
167 	struct hpre_ctx *ctx;
168 	struct hpre_dfx *dfx;
169 	int id;
170 
171 	ctx = hpre_req->ctx;
172 	id = hpre_alloc_req_id(ctx);
173 	if (unlikely(id < 0))
174 		return -EINVAL;
175 
176 	ctx->req_list[id] = hpre_req;
177 	hpre_req->req_id = id;
178 
179 	dfx = ctx->hpre->debug.dfx;
180 	if (atomic64_read(&dfx[HPRE_OVERTIME_THRHLD].value))
181 		ktime_get_ts64(&hpre_req->req_time);
182 
183 	return id;
184 }
185 
186 static void hpre_rm_req_from_ctx(struct hpre_asym_request *hpre_req)
187 {
188 	struct hpre_ctx *ctx = hpre_req->ctx;
189 	int id = hpre_req->req_id;
190 
191 	if (hpre_req->req_id >= 0) {
192 		hpre_req->req_id = HPRE_INVLD_REQ_ID;
193 		ctx->req_list[id] = NULL;
194 		hpre_free_req_id(ctx, id);
195 	}
196 }
197 
198 static struct hisi_qp *hpre_get_qp_and_start(u8 type)
199 {
200 	struct hisi_qp *qp;
201 	int ret;
202 
203 	qp = hpre_create_qp(type);
204 	if (!qp) {
205 		pr_err("Can not create hpre qp!\n");
206 		return ERR_PTR(-ENODEV);
207 	}
208 
209 	ret = hisi_qm_start_qp(qp, 0);
210 	if (ret < 0) {
211 		hisi_qm_free_qps(&qp, 1);
212 		pci_err(qp->qm->pdev, "Can not start qp!\n");
213 		return ERR_PTR(-EINVAL);
214 	}
215 
216 	return qp;
217 }
218 
219 static int hpre_get_data_dma_addr(struct hpre_asym_request *hpre_req,
220 				  struct scatterlist *data, unsigned int len,
221 				  int is_src, dma_addr_t *tmp)
222 {
223 	struct device *dev = hpre_req->ctx->dev;
224 	enum dma_data_direction dma_dir;
225 
226 	if (is_src) {
227 		hpre_req->src = NULL;
228 		dma_dir = DMA_TO_DEVICE;
229 	} else {
230 		hpre_req->dst = NULL;
231 		dma_dir = DMA_FROM_DEVICE;
232 	}
233 	*tmp = dma_map_single(dev, sg_virt(data), len, dma_dir);
234 	if (unlikely(dma_mapping_error(dev, *tmp))) {
235 		dev_err(dev, "dma map data err!\n");
236 		return -ENOMEM;
237 	}
238 
239 	return 0;
240 }
241 
242 static int hpre_prepare_dma_buf(struct hpre_asym_request *hpre_req,
243 				struct scatterlist *data, unsigned int len,
244 				int is_src, dma_addr_t *tmp)
245 {
246 	struct hpre_ctx *ctx = hpre_req->ctx;
247 	struct device *dev = ctx->dev;
248 	void *ptr;
249 	int shift;
250 
251 	shift = ctx->key_sz - len;
252 	if (unlikely(shift < 0))
253 		return -EINVAL;
254 
255 	ptr = dma_alloc_coherent(dev, ctx->key_sz, tmp, GFP_ATOMIC);
256 	if (unlikely(!ptr))
257 		return -ENOMEM;
258 
259 	if (is_src) {
260 		scatterwalk_map_and_copy(ptr + shift, data, 0, len, 0);
261 		hpre_req->src = ptr;
262 	} else {
263 		hpre_req->dst = ptr;
264 	}
265 
266 	return 0;
267 }
268 
269 static int hpre_hw_data_init(struct hpre_asym_request *hpre_req,
270 			     struct scatterlist *data, unsigned int len,
271 			     int is_src, int is_dh)
272 {
273 	struct hpre_sqe *msg = &hpre_req->req;
274 	struct hpre_ctx *ctx = hpre_req->ctx;
275 	dma_addr_t tmp = 0;
276 	int ret;
277 
278 	/* when the data is dh's source, we should format it */
279 	if ((sg_is_last(data) && len == ctx->key_sz) &&
280 	    ((is_dh && !is_src) || !is_dh))
281 		ret = hpre_get_data_dma_addr(hpre_req, data, len, is_src, &tmp);
282 	else
283 		ret = hpre_prepare_dma_buf(hpre_req, data, len, is_src, &tmp);
284 
285 	if (unlikely(ret))
286 		return ret;
287 
288 	if (is_src)
289 		msg->in = cpu_to_le64(tmp);
290 	else
291 		msg->out = cpu_to_le64(tmp);
292 
293 	return 0;
294 }
295 
296 static void hpre_hw_data_clr_all(struct hpre_ctx *ctx,
297 				 struct hpre_asym_request *req,
298 				 struct scatterlist *dst,
299 				 struct scatterlist *src)
300 {
301 	struct device *dev = ctx->dev;
302 	struct hpre_sqe *sqe = &req->req;
303 	dma_addr_t tmp;
304 
305 	tmp = le64_to_cpu(sqe->in);
306 	if (unlikely(dma_mapping_error(dev, tmp)))
307 		return;
308 
309 	if (src) {
310 		if (req->src)
311 			dma_free_coherent(dev, ctx->key_sz, req->src, tmp);
312 		else
313 			dma_unmap_single(dev, tmp, ctx->key_sz, DMA_TO_DEVICE);
314 	}
315 
316 	tmp = le64_to_cpu(sqe->out);
317 	if (unlikely(dma_mapping_error(dev, tmp)))
318 		return;
319 
320 	if (req->dst) {
321 		if (dst)
322 			scatterwalk_map_and_copy(req->dst, dst, 0,
323 						 ctx->key_sz, 1);
324 		dma_free_coherent(dev, ctx->key_sz, req->dst, tmp);
325 	} else {
326 		dma_unmap_single(dev, tmp, ctx->key_sz, DMA_FROM_DEVICE);
327 	}
328 }
329 
330 static int hpre_alg_res_post_hf(struct hpre_ctx *ctx, struct hpre_sqe *sqe,
331 				void **kreq)
332 {
333 	struct hpre_asym_request *req;
334 	unsigned int err, done, alg;
335 	int id;
336 
337 #define HPRE_NO_HW_ERR		0
338 #define HPRE_HW_TASK_DONE	3
339 #define HREE_HW_ERR_MASK	GENMASK(10, 0)
340 #define HREE_SQE_DONE_MASK	GENMASK(1, 0)
341 #define HREE_ALG_TYPE_MASK	GENMASK(4, 0)
342 	id = (int)le16_to_cpu(sqe->tag);
343 	req = ctx->req_list[id];
344 	hpre_rm_req_from_ctx(req);
345 	*kreq = req;
346 
347 	err = (le32_to_cpu(sqe->dw0) >> HPRE_SQE_ALG_BITS) &
348 		HREE_HW_ERR_MASK;
349 
350 	done = (le32_to_cpu(sqe->dw0) >> HPRE_SQE_DONE_SHIFT) &
351 		HREE_SQE_DONE_MASK;
352 
353 	if (likely(err == HPRE_NO_HW_ERR && done == HPRE_HW_TASK_DONE))
354 		return 0;
355 
356 	alg = le32_to_cpu(sqe->dw0) & HREE_ALG_TYPE_MASK;
357 	dev_err_ratelimited(ctx->dev, "alg[0x%x] error: done[0x%x], etype[0x%x]\n",
358 		alg, done, err);
359 
360 	return -EINVAL;
361 }
362 
363 static int hpre_ctx_set(struct hpre_ctx *ctx, struct hisi_qp *qp, int qlen)
364 {
365 	struct hpre *hpre;
366 
367 	if (!ctx || !qp || qlen < 0)
368 		return -EINVAL;
369 
370 	spin_lock_init(&ctx->req_lock);
371 	ctx->qp = qp;
372 	ctx->dev = &qp->qm->pdev->dev;
373 
374 	hpre = container_of(ctx->qp->qm, struct hpre, qm);
375 	ctx->hpre = hpre;
376 	ctx->req_list = kcalloc(qlen, sizeof(void *), GFP_KERNEL);
377 	if (!ctx->req_list)
378 		return -ENOMEM;
379 	ctx->key_sz = 0;
380 	ctx->crt_g2_mode = false;
381 	idr_init(&ctx->req_idr);
382 
383 	return 0;
384 }
385 
386 static void hpre_ctx_clear(struct hpre_ctx *ctx, bool is_clear_all)
387 {
388 	if (is_clear_all) {
389 		idr_destroy(&ctx->req_idr);
390 		kfree(ctx->req_list);
391 		hisi_qm_free_qps(&ctx->qp, 1);
392 	}
393 
394 	ctx->crt_g2_mode = false;
395 	ctx->key_sz = 0;
396 }
397 
398 static bool hpre_is_bd_timeout(struct hpre_asym_request *req,
399 			       u64 overtime_thrhld)
400 {
401 	struct timespec64 reply_time;
402 	u64 time_use_us;
403 
404 	ktime_get_ts64(&reply_time);
405 	time_use_us = (reply_time.tv_sec - req->req_time.tv_sec) *
406 		HPRE_DFX_SEC_TO_US +
407 		(reply_time.tv_nsec - req->req_time.tv_nsec) /
408 		HPRE_DFX_US_TO_NS;
409 
410 	if (time_use_us <= overtime_thrhld)
411 		return false;
412 
413 	return true;
414 }
415 
416 static void hpre_dh_cb(struct hpre_ctx *ctx, void *resp)
417 {
418 	struct hpre_dfx *dfx = ctx->hpre->debug.dfx;
419 	struct hpre_asym_request *req;
420 	struct kpp_request *areq;
421 	u64 overtime_thrhld;
422 	int ret;
423 
424 	ret = hpre_alg_res_post_hf(ctx, resp, (void **)&req);
425 	areq = req->areq.dh;
426 	areq->dst_len = ctx->key_sz;
427 
428 	overtime_thrhld = atomic64_read(&dfx[HPRE_OVERTIME_THRHLD].value);
429 	if (overtime_thrhld && hpre_is_bd_timeout(req, overtime_thrhld))
430 		atomic64_inc(&dfx[HPRE_OVER_THRHLD_CNT].value);
431 
432 	hpre_hw_data_clr_all(ctx, req, areq->dst, areq->src);
433 	kpp_request_complete(areq, ret);
434 	atomic64_inc(&dfx[HPRE_RECV_CNT].value);
435 }
436 
437 static void hpre_rsa_cb(struct hpre_ctx *ctx, void *resp)
438 {
439 	struct hpre_dfx *dfx = ctx->hpre->debug.dfx;
440 	struct hpre_asym_request *req;
441 	struct akcipher_request *areq;
442 	u64 overtime_thrhld;
443 	int ret;
444 
445 	ret = hpre_alg_res_post_hf(ctx, resp, (void **)&req);
446 
447 	overtime_thrhld = atomic64_read(&dfx[HPRE_OVERTIME_THRHLD].value);
448 	if (overtime_thrhld && hpre_is_bd_timeout(req, overtime_thrhld))
449 		atomic64_inc(&dfx[HPRE_OVER_THRHLD_CNT].value);
450 
451 	areq = req->areq.rsa;
452 	areq->dst_len = ctx->key_sz;
453 	hpre_hw_data_clr_all(ctx, req, areq->dst, areq->src);
454 	akcipher_request_complete(areq, ret);
455 	atomic64_inc(&dfx[HPRE_RECV_CNT].value);
456 }
457 
458 static void hpre_alg_cb(struct hisi_qp *qp, void *resp)
459 {
460 	struct hpre_ctx *ctx = qp->qp_ctx;
461 	struct hpre_dfx *dfx = ctx->hpre->debug.dfx;
462 	struct hpre_sqe *sqe = resp;
463 	struct hpre_asym_request *req = ctx->req_list[le16_to_cpu(sqe->tag)];
464 
465 	if (unlikely(!req)) {
466 		atomic64_inc(&dfx[HPRE_INVALID_REQ_CNT].value);
467 		return;
468 	}
469 
470 	req->cb(ctx, resp);
471 }
472 
473 static void hpre_stop_qp_and_put(struct hisi_qp *qp)
474 {
475 	hisi_qm_stop_qp(qp);
476 	hisi_qm_free_qps(&qp, 1);
477 }
478 
479 static int hpre_ctx_init(struct hpre_ctx *ctx, u8 type)
480 {
481 	struct hisi_qp *qp;
482 	int ret;
483 
484 	qp = hpre_get_qp_and_start(type);
485 	if (IS_ERR(qp))
486 		return PTR_ERR(qp);
487 
488 	qp->qp_ctx = ctx;
489 	qp->req_cb = hpre_alg_cb;
490 
491 	ret = hpre_ctx_set(ctx, qp, QM_Q_DEPTH);
492 	if (ret)
493 		hpre_stop_qp_and_put(qp);
494 
495 	return ret;
496 }
497 
498 static int hpre_msg_request_set(struct hpre_ctx *ctx, void *req, bool is_rsa)
499 {
500 	struct hpre_asym_request *h_req;
501 	struct hpre_sqe *msg;
502 	int req_id;
503 	void *tmp;
504 
505 	if (is_rsa) {
506 		struct akcipher_request *akreq = req;
507 
508 		if (akreq->dst_len < ctx->key_sz) {
509 			akreq->dst_len = ctx->key_sz;
510 			return -EOVERFLOW;
511 		}
512 
513 		tmp = akcipher_request_ctx(akreq);
514 		h_req = PTR_ALIGN(tmp, HPRE_ALIGN_SZ);
515 		h_req->cb = hpre_rsa_cb;
516 		h_req->areq.rsa = akreq;
517 		msg = &h_req->req;
518 		memset(msg, 0, sizeof(*msg));
519 	} else {
520 		struct kpp_request *kreq = req;
521 
522 		if (kreq->dst_len < ctx->key_sz) {
523 			kreq->dst_len = ctx->key_sz;
524 			return -EOVERFLOW;
525 		}
526 
527 		tmp = kpp_request_ctx(kreq);
528 		h_req = PTR_ALIGN(tmp, HPRE_ALIGN_SZ);
529 		h_req->cb = hpre_dh_cb;
530 		h_req->areq.dh = kreq;
531 		msg = &h_req->req;
532 		memset(msg, 0, sizeof(*msg));
533 		msg->key = cpu_to_le64(ctx->dh.dma_xa_p);
534 	}
535 
536 	msg->in = cpu_to_le64(DMA_MAPPING_ERROR);
537 	msg->out = cpu_to_le64(DMA_MAPPING_ERROR);
538 	msg->dw0 |= cpu_to_le32(0x1 << HPRE_SQE_DONE_SHIFT);
539 	msg->task_len1 = (ctx->key_sz >> HPRE_BITS_2_BYTES_SHIFT) - 1;
540 	h_req->ctx = ctx;
541 
542 	req_id = hpre_add_req_to_ctx(h_req);
543 	if (req_id < 0)
544 		return -EBUSY;
545 
546 	msg->tag = cpu_to_le16((u16)req_id);
547 
548 	return 0;
549 }
550 
551 static int hpre_send(struct hpre_ctx *ctx, struct hpre_sqe *msg)
552 {
553 	struct hpre_dfx *dfx = ctx->hpre->debug.dfx;
554 	int ctr = 0;
555 	int ret;
556 
557 	do {
558 		atomic64_inc(&dfx[HPRE_SEND_CNT].value);
559 		ret = hisi_qp_send(ctx->qp, msg);
560 		if (ret != -EBUSY)
561 			break;
562 		atomic64_inc(&dfx[HPRE_SEND_BUSY_CNT].value);
563 	} while (ctr++ < HPRE_TRY_SEND_TIMES);
564 
565 	if (likely(!ret))
566 		return ret;
567 
568 	if (ret != -EBUSY)
569 		atomic64_inc(&dfx[HPRE_SEND_FAIL_CNT].value);
570 
571 	return ret;
572 }
573 
574 static int hpre_dh_compute_value(struct kpp_request *req)
575 {
576 	struct crypto_kpp *tfm = crypto_kpp_reqtfm(req);
577 	struct hpre_ctx *ctx = kpp_tfm_ctx(tfm);
578 	void *tmp = kpp_request_ctx(req);
579 	struct hpre_asym_request *hpre_req = PTR_ALIGN(tmp, HPRE_ALIGN_SZ);
580 	struct hpre_sqe *msg = &hpre_req->req;
581 	int ret;
582 
583 	ret = hpre_msg_request_set(ctx, req, false);
584 	if (unlikely(ret))
585 		return ret;
586 
587 	if (req->src) {
588 		ret = hpre_hw_data_init(hpre_req, req->src, req->src_len, 1, 1);
589 		if (unlikely(ret))
590 			goto clear_all;
591 	} else {
592 		msg->in = cpu_to_le64(ctx->dh.dma_g);
593 	}
594 
595 	ret = hpre_hw_data_init(hpre_req, req->dst, req->dst_len, 0, 1);
596 	if (unlikely(ret))
597 		goto clear_all;
598 
599 	if (ctx->crt_g2_mode && !req->src)
600 		msg->dw0 = cpu_to_le32(le32_to_cpu(msg->dw0) | HPRE_ALG_DH_G2);
601 	else
602 		msg->dw0 = cpu_to_le32(le32_to_cpu(msg->dw0) | HPRE_ALG_DH);
603 
604 	/* success */
605 	ret = hpre_send(ctx, msg);
606 	if (likely(!ret))
607 		return -EINPROGRESS;
608 
609 clear_all:
610 	hpre_rm_req_from_ctx(hpre_req);
611 	hpre_hw_data_clr_all(ctx, hpre_req, req->dst, req->src);
612 
613 	return ret;
614 }
615 
616 static int hpre_is_dh_params_length_valid(unsigned int key_sz)
617 {
618 #define _HPRE_DH_GRP1		768
619 #define _HPRE_DH_GRP2		1024
620 #define _HPRE_DH_GRP5		1536
621 #define _HPRE_DH_GRP14		2048
622 #define _HPRE_DH_GRP15		3072
623 #define _HPRE_DH_GRP16		4096
624 	switch (key_sz) {
625 	case _HPRE_DH_GRP1:
626 	case _HPRE_DH_GRP2:
627 	case _HPRE_DH_GRP5:
628 	case _HPRE_DH_GRP14:
629 	case _HPRE_DH_GRP15:
630 	case _HPRE_DH_GRP16:
631 		return 0;
632 	default:
633 		return -EINVAL;
634 	}
635 }
636 
637 static int hpre_dh_set_params(struct hpre_ctx *ctx, struct dh *params)
638 {
639 	struct device *dev = ctx->dev;
640 	unsigned int sz;
641 
642 	if (params->p_size > HPRE_DH_MAX_P_SZ)
643 		return -EINVAL;
644 
645 	if (hpre_is_dh_params_length_valid(params->p_size <<
646 					   HPRE_BITS_2_BYTES_SHIFT))
647 		return -EINVAL;
648 
649 	sz = ctx->key_sz = params->p_size;
650 	ctx->dh.xa_p = dma_alloc_coherent(dev, sz << 1,
651 					  &ctx->dh.dma_xa_p, GFP_KERNEL);
652 	if (!ctx->dh.xa_p)
653 		return -ENOMEM;
654 
655 	memcpy(ctx->dh.xa_p + sz, params->p, sz);
656 
657 	/* If g equals 2 don't copy it */
658 	if (params->g_size == 1 && *(char *)params->g == HPRE_DH_G_FLAG) {
659 		ctx->crt_g2_mode = true;
660 		return 0;
661 	}
662 
663 	ctx->dh.g = dma_alloc_coherent(dev, sz, &ctx->dh.dma_g, GFP_KERNEL);
664 	if (!ctx->dh.g) {
665 		dma_free_coherent(dev, sz << 1, ctx->dh.xa_p,
666 				  ctx->dh.dma_xa_p);
667 		ctx->dh.xa_p = NULL;
668 		return -ENOMEM;
669 	}
670 
671 	memcpy(ctx->dh.g + (sz - params->g_size), params->g, params->g_size);
672 
673 	return 0;
674 }
675 
676 static void hpre_dh_clear_ctx(struct hpre_ctx *ctx, bool is_clear_all)
677 {
678 	struct device *dev = ctx->dev;
679 	unsigned int sz = ctx->key_sz;
680 
681 	if (is_clear_all)
682 		hisi_qm_stop_qp(ctx->qp);
683 
684 	if (ctx->dh.g) {
685 		dma_free_coherent(dev, sz, ctx->dh.g, ctx->dh.dma_g);
686 		ctx->dh.g = NULL;
687 	}
688 
689 	if (ctx->dh.xa_p) {
690 		memzero_explicit(ctx->dh.xa_p, sz);
691 		dma_free_coherent(dev, sz << 1, ctx->dh.xa_p,
692 				  ctx->dh.dma_xa_p);
693 		ctx->dh.xa_p = NULL;
694 	}
695 
696 	hpre_ctx_clear(ctx, is_clear_all);
697 }
698 
699 static int hpre_dh_set_secret(struct crypto_kpp *tfm, const void *buf,
700 			      unsigned int len)
701 {
702 	struct hpre_ctx *ctx = kpp_tfm_ctx(tfm);
703 	struct dh params;
704 	int ret;
705 
706 	if (crypto_dh_decode_key(buf, len, &params) < 0)
707 		return -EINVAL;
708 
709 	/* Free old secret if any */
710 	hpre_dh_clear_ctx(ctx, false);
711 
712 	ret = hpre_dh_set_params(ctx, &params);
713 	if (ret < 0)
714 		goto err_clear_ctx;
715 
716 	memcpy(ctx->dh.xa_p + (ctx->key_sz - params.key_size), params.key,
717 	       params.key_size);
718 
719 	return 0;
720 
721 err_clear_ctx:
722 	hpre_dh_clear_ctx(ctx, false);
723 	return ret;
724 }
725 
726 static unsigned int hpre_dh_max_size(struct crypto_kpp *tfm)
727 {
728 	struct hpre_ctx *ctx = kpp_tfm_ctx(tfm);
729 
730 	return ctx->key_sz;
731 }
732 
733 static int hpre_dh_init_tfm(struct crypto_kpp *tfm)
734 {
735 	struct hpre_ctx *ctx = kpp_tfm_ctx(tfm);
736 
737 	return hpre_ctx_init(ctx, HPRE_V2_ALG_TYPE);
738 }
739 
740 static void hpre_dh_exit_tfm(struct crypto_kpp *tfm)
741 {
742 	struct hpre_ctx *ctx = kpp_tfm_ctx(tfm);
743 
744 	hpre_dh_clear_ctx(ctx, true);
745 }
746 
747 static void hpre_rsa_drop_leading_zeros(const char **ptr, size_t *len)
748 {
749 	while (!**ptr && *len) {
750 		(*ptr)++;
751 		(*len)--;
752 	}
753 }
754 
755 static bool hpre_rsa_key_size_is_support(unsigned int len)
756 {
757 	unsigned int bits = len << HPRE_BITS_2_BYTES_SHIFT;
758 
759 #define _RSA_1024BITS_KEY_WDTH		1024
760 #define _RSA_2048BITS_KEY_WDTH		2048
761 #define _RSA_3072BITS_KEY_WDTH		3072
762 #define _RSA_4096BITS_KEY_WDTH		4096
763 
764 	switch (bits) {
765 	case _RSA_1024BITS_KEY_WDTH:
766 	case _RSA_2048BITS_KEY_WDTH:
767 	case _RSA_3072BITS_KEY_WDTH:
768 	case _RSA_4096BITS_KEY_WDTH:
769 		return true;
770 	default:
771 		return false;
772 	}
773 }
774 
775 static int hpre_rsa_enc(struct akcipher_request *req)
776 {
777 	struct crypto_akcipher *tfm = crypto_akcipher_reqtfm(req);
778 	struct hpre_ctx *ctx = akcipher_tfm_ctx(tfm);
779 	void *tmp = akcipher_request_ctx(req);
780 	struct hpre_asym_request *hpre_req = PTR_ALIGN(tmp, HPRE_ALIGN_SZ);
781 	struct hpre_sqe *msg = &hpre_req->req;
782 	int ret;
783 
784 	/* For 512 and 1536 bits key size, use soft tfm instead */
785 	if (ctx->key_sz == HPRE_RSA_512BITS_KSZ ||
786 	    ctx->key_sz == HPRE_RSA_1536BITS_KSZ) {
787 		akcipher_request_set_tfm(req, ctx->rsa.soft_tfm);
788 		ret = crypto_akcipher_encrypt(req);
789 		akcipher_request_set_tfm(req, tfm);
790 		return ret;
791 	}
792 
793 	if (unlikely(!ctx->rsa.pubkey))
794 		return -EINVAL;
795 
796 	ret = hpre_msg_request_set(ctx, req, true);
797 	if (unlikely(ret))
798 		return ret;
799 
800 	msg->dw0 |= cpu_to_le32(HPRE_ALG_NC_NCRT);
801 	msg->key = cpu_to_le64(ctx->rsa.dma_pubkey);
802 
803 	ret = hpre_hw_data_init(hpre_req, req->src, req->src_len, 1, 0);
804 	if (unlikely(ret))
805 		goto clear_all;
806 
807 	ret = hpre_hw_data_init(hpre_req, req->dst, req->dst_len, 0, 0);
808 	if (unlikely(ret))
809 		goto clear_all;
810 
811 	/* success */
812 	ret = hpre_send(ctx, msg);
813 	if (likely(!ret))
814 		return -EINPROGRESS;
815 
816 clear_all:
817 	hpre_rm_req_from_ctx(hpre_req);
818 	hpre_hw_data_clr_all(ctx, hpre_req, req->dst, req->src);
819 
820 	return ret;
821 }
822 
823 static int hpre_rsa_dec(struct akcipher_request *req)
824 {
825 	struct crypto_akcipher *tfm = crypto_akcipher_reqtfm(req);
826 	struct hpre_ctx *ctx = akcipher_tfm_ctx(tfm);
827 	void *tmp = akcipher_request_ctx(req);
828 	struct hpre_asym_request *hpre_req = PTR_ALIGN(tmp, HPRE_ALIGN_SZ);
829 	struct hpre_sqe *msg = &hpre_req->req;
830 	int ret;
831 
832 	/* For 512 and 1536 bits key size, use soft tfm instead */
833 	if (ctx->key_sz == HPRE_RSA_512BITS_KSZ ||
834 	    ctx->key_sz == HPRE_RSA_1536BITS_KSZ) {
835 		akcipher_request_set_tfm(req, ctx->rsa.soft_tfm);
836 		ret = crypto_akcipher_decrypt(req);
837 		akcipher_request_set_tfm(req, tfm);
838 		return ret;
839 	}
840 
841 	if (unlikely(!ctx->rsa.prikey))
842 		return -EINVAL;
843 
844 	ret = hpre_msg_request_set(ctx, req, true);
845 	if (unlikely(ret))
846 		return ret;
847 
848 	if (ctx->crt_g2_mode) {
849 		msg->key = cpu_to_le64(ctx->rsa.dma_crt_prikey);
850 		msg->dw0 = cpu_to_le32(le32_to_cpu(msg->dw0) |
851 				       HPRE_ALG_NC_CRT);
852 	} else {
853 		msg->key = cpu_to_le64(ctx->rsa.dma_prikey);
854 		msg->dw0 = cpu_to_le32(le32_to_cpu(msg->dw0) |
855 				       HPRE_ALG_NC_NCRT);
856 	}
857 
858 	ret = hpre_hw_data_init(hpre_req, req->src, req->src_len, 1, 0);
859 	if (unlikely(ret))
860 		goto clear_all;
861 
862 	ret = hpre_hw_data_init(hpre_req, req->dst, req->dst_len, 0, 0);
863 	if (unlikely(ret))
864 		goto clear_all;
865 
866 	/* success */
867 	ret = hpre_send(ctx, msg);
868 	if (likely(!ret))
869 		return -EINPROGRESS;
870 
871 clear_all:
872 	hpre_rm_req_from_ctx(hpre_req);
873 	hpre_hw_data_clr_all(ctx, hpre_req, req->dst, req->src);
874 
875 	return ret;
876 }
877 
878 static int hpre_rsa_set_n(struct hpre_ctx *ctx, const char *value,
879 			  size_t vlen, bool private)
880 {
881 	const char *ptr = value;
882 
883 	hpre_rsa_drop_leading_zeros(&ptr, &vlen);
884 
885 	ctx->key_sz = vlen;
886 
887 	/* if invalid key size provided, we use software tfm */
888 	if (!hpre_rsa_key_size_is_support(ctx->key_sz))
889 		return 0;
890 
891 	ctx->rsa.pubkey = dma_alloc_coherent(ctx->dev, vlen << 1,
892 					     &ctx->rsa.dma_pubkey,
893 					     GFP_KERNEL);
894 	if (!ctx->rsa.pubkey)
895 		return -ENOMEM;
896 
897 	if (private) {
898 		ctx->rsa.prikey = dma_alloc_coherent(ctx->dev, vlen << 1,
899 						     &ctx->rsa.dma_prikey,
900 						     GFP_KERNEL);
901 		if (!ctx->rsa.prikey) {
902 			dma_free_coherent(ctx->dev, vlen << 1,
903 					  ctx->rsa.pubkey,
904 					  ctx->rsa.dma_pubkey);
905 			ctx->rsa.pubkey = NULL;
906 			return -ENOMEM;
907 		}
908 		memcpy(ctx->rsa.prikey + vlen, ptr, vlen);
909 	}
910 	memcpy(ctx->rsa.pubkey + vlen, ptr, vlen);
911 
912 	/* Using hardware HPRE to do RSA */
913 	return 1;
914 }
915 
916 static int hpre_rsa_set_e(struct hpre_ctx *ctx, const char *value,
917 			  size_t vlen)
918 {
919 	const char *ptr = value;
920 
921 	hpre_rsa_drop_leading_zeros(&ptr, &vlen);
922 
923 	if (!ctx->key_sz || !vlen || vlen > ctx->key_sz)
924 		return -EINVAL;
925 
926 	memcpy(ctx->rsa.pubkey + ctx->key_sz - vlen, ptr, vlen);
927 
928 	return 0;
929 }
930 
931 static int hpre_rsa_set_d(struct hpre_ctx *ctx, const char *value,
932 			  size_t vlen)
933 {
934 	const char *ptr = value;
935 
936 	hpre_rsa_drop_leading_zeros(&ptr, &vlen);
937 
938 	if (!ctx->key_sz || !vlen || vlen > ctx->key_sz)
939 		return -EINVAL;
940 
941 	memcpy(ctx->rsa.prikey + ctx->key_sz - vlen, ptr, vlen);
942 
943 	return 0;
944 }
945 
946 static int hpre_crt_para_get(char *para, size_t para_sz,
947 			     const char *raw, size_t raw_sz)
948 {
949 	const char *ptr = raw;
950 	size_t len = raw_sz;
951 
952 	hpre_rsa_drop_leading_zeros(&ptr, &len);
953 	if (!len || len > para_sz)
954 		return -EINVAL;
955 
956 	memcpy(para + para_sz - len, ptr, len);
957 
958 	return 0;
959 }
960 
961 static int hpre_rsa_setkey_crt(struct hpre_ctx *ctx, struct rsa_key *rsa_key)
962 {
963 	unsigned int hlf_ksz = ctx->key_sz >> 1;
964 	struct device *dev = ctx->dev;
965 	u64 offset;
966 	int ret;
967 
968 	ctx->rsa.crt_prikey = dma_alloc_coherent(dev, hlf_ksz * HPRE_CRT_PRMS,
969 					&ctx->rsa.dma_crt_prikey,
970 					GFP_KERNEL);
971 	if (!ctx->rsa.crt_prikey)
972 		return -ENOMEM;
973 
974 	ret = hpre_crt_para_get(ctx->rsa.crt_prikey, hlf_ksz,
975 				rsa_key->dq, rsa_key->dq_sz);
976 	if (ret)
977 		goto free_key;
978 
979 	offset = hlf_ksz;
980 	ret = hpre_crt_para_get(ctx->rsa.crt_prikey + offset, hlf_ksz,
981 				rsa_key->dp, rsa_key->dp_sz);
982 	if (ret)
983 		goto free_key;
984 
985 	offset = hlf_ksz * HPRE_CRT_Q;
986 	ret = hpre_crt_para_get(ctx->rsa.crt_prikey + offset, hlf_ksz,
987 				rsa_key->q, rsa_key->q_sz);
988 	if (ret)
989 		goto free_key;
990 
991 	offset = hlf_ksz * HPRE_CRT_P;
992 	ret = hpre_crt_para_get(ctx->rsa.crt_prikey + offset, hlf_ksz,
993 				rsa_key->p, rsa_key->p_sz);
994 	if (ret)
995 		goto free_key;
996 
997 	offset = hlf_ksz * HPRE_CRT_INV;
998 	ret = hpre_crt_para_get(ctx->rsa.crt_prikey + offset, hlf_ksz,
999 				rsa_key->qinv, rsa_key->qinv_sz);
1000 	if (ret)
1001 		goto free_key;
1002 
1003 	ctx->crt_g2_mode = true;
1004 
1005 	return 0;
1006 
1007 free_key:
1008 	offset = hlf_ksz * HPRE_CRT_PRMS;
1009 	memzero_explicit(ctx->rsa.crt_prikey, offset);
1010 	dma_free_coherent(dev, hlf_ksz * HPRE_CRT_PRMS, ctx->rsa.crt_prikey,
1011 			  ctx->rsa.dma_crt_prikey);
1012 	ctx->rsa.crt_prikey = NULL;
1013 	ctx->crt_g2_mode = false;
1014 
1015 	return ret;
1016 }
1017 
1018 /* If it is clear all, all the resources of the QP will be cleaned. */
1019 static void hpre_rsa_clear_ctx(struct hpre_ctx *ctx, bool is_clear_all)
1020 {
1021 	unsigned int half_key_sz = ctx->key_sz >> 1;
1022 	struct device *dev = ctx->dev;
1023 
1024 	if (is_clear_all)
1025 		hisi_qm_stop_qp(ctx->qp);
1026 
1027 	if (ctx->rsa.pubkey) {
1028 		dma_free_coherent(dev, ctx->key_sz << 1,
1029 				  ctx->rsa.pubkey, ctx->rsa.dma_pubkey);
1030 		ctx->rsa.pubkey = NULL;
1031 	}
1032 
1033 	if (ctx->rsa.crt_prikey) {
1034 		memzero_explicit(ctx->rsa.crt_prikey,
1035 				 half_key_sz * HPRE_CRT_PRMS);
1036 		dma_free_coherent(dev, half_key_sz * HPRE_CRT_PRMS,
1037 				  ctx->rsa.crt_prikey, ctx->rsa.dma_crt_prikey);
1038 		ctx->rsa.crt_prikey = NULL;
1039 	}
1040 
1041 	if (ctx->rsa.prikey) {
1042 		memzero_explicit(ctx->rsa.prikey, ctx->key_sz);
1043 		dma_free_coherent(dev, ctx->key_sz << 1, ctx->rsa.prikey,
1044 				  ctx->rsa.dma_prikey);
1045 		ctx->rsa.prikey = NULL;
1046 	}
1047 
1048 	hpre_ctx_clear(ctx, is_clear_all);
1049 }
1050 
1051 /*
1052  * we should judge if it is CRT or not,
1053  * CRT: return true,  N-CRT: return false .
1054  */
1055 static bool hpre_is_crt_key(struct rsa_key *key)
1056 {
1057 	u16 len = key->p_sz + key->q_sz + key->dp_sz + key->dq_sz +
1058 		  key->qinv_sz;
1059 
1060 #define LEN_OF_NCRT_PARA	5
1061 
1062 	/* N-CRT less than 5 parameters */
1063 	return len > LEN_OF_NCRT_PARA;
1064 }
1065 
1066 static int hpre_rsa_setkey(struct hpre_ctx *ctx, const void *key,
1067 			   unsigned int keylen, bool private)
1068 {
1069 	struct rsa_key rsa_key;
1070 	int ret;
1071 
1072 	hpre_rsa_clear_ctx(ctx, false);
1073 
1074 	if (private)
1075 		ret = rsa_parse_priv_key(&rsa_key, key, keylen);
1076 	else
1077 		ret = rsa_parse_pub_key(&rsa_key, key, keylen);
1078 	if (ret < 0)
1079 		return ret;
1080 
1081 	ret = hpre_rsa_set_n(ctx, rsa_key.n, rsa_key.n_sz, private);
1082 	if (ret <= 0)
1083 		return ret;
1084 
1085 	if (private) {
1086 		ret = hpre_rsa_set_d(ctx, rsa_key.d, rsa_key.d_sz);
1087 		if (ret < 0)
1088 			goto free;
1089 
1090 		if (hpre_is_crt_key(&rsa_key)) {
1091 			ret = hpre_rsa_setkey_crt(ctx, &rsa_key);
1092 			if (ret < 0)
1093 				goto free;
1094 		}
1095 	}
1096 
1097 	ret = hpre_rsa_set_e(ctx, rsa_key.e, rsa_key.e_sz);
1098 	if (ret < 0)
1099 		goto free;
1100 
1101 	if ((private && !ctx->rsa.prikey) || !ctx->rsa.pubkey) {
1102 		ret = -EINVAL;
1103 		goto free;
1104 	}
1105 
1106 	return 0;
1107 
1108 free:
1109 	hpre_rsa_clear_ctx(ctx, false);
1110 	return ret;
1111 }
1112 
1113 static int hpre_rsa_setpubkey(struct crypto_akcipher *tfm, const void *key,
1114 			      unsigned int keylen)
1115 {
1116 	struct hpre_ctx *ctx = akcipher_tfm_ctx(tfm);
1117 	int ret;
1118 
1119 	ret = crypto_akcipher_set_pub_key(ctx->rsa.soft_tfm, key, keylen);
1120 	if (ret)
1121 		return ret;
1122 
1123 	return hpre_rsa_setkey(ctx, key, keylen, false);
1124 }
1125 
1126 static int hpre_rsa_setprivkey(struct crypto_akcipher *tfm, const void *key,
1127 			       unsigned int keylen)
1128 {
1129 	struct hpre_ctx *ctx = akcipher_tfm_ctx(tfm);
1130 	int ret;
1131 
1132 	ret = crypto_akcipher_set_priv_key(ctx->rsa.soft_tfm, key, keylen);
1133 	if (ret)
1134 		return ret;
1135 
1136 	return hpre_rsa_setkey(ctx, key, keylen, true);
1137 }
1138 
1139 static unsigned int hpre_rsa_max_size(struct crypto_akcipher *tfm)
1140 {
1141 	struct hpre_ctx *ctx = akcipher_tfm_ctx(tfm);
1142 
1143 	/* For 512 and 1536 bits key size, use soft tfm instead */
1144 	if (ctx->key_sz == HPRE_RSA_512BITS_KSZ ||
1145 	    ctx->key_sz == HPRE_RSA_1536BITS_KSZ)
1146 		return crypto_akcipher_maxsize(ctx->rsa.soft_tfm);
1147 
1148 	return ctx->key_sz;
1149 }
1150 
1151 static int hpre_rsa_init_tfm(struct crypto_akcipher *tfm)
1152 {
1153 	struct hpre_ctx *ctx = akcipher_tfm_ctx(tfm);
1154 	int ret;
1155 
1156 	ctx->rsa.soft_tfm = crypto_alloc_akcipher("rsa-generic", 0, 0);
1157 	if (IS_ERR(ctx->rsa.soft_tfm)) {
1158 		pr_err("Can not alloc_akcipher!\n");
1159 		return PTR_ERR(ctx->rsa.soft_tfm);
1160 	}
1161 
1162 	ret = hpre_ctx_init(ctx, HPRE_V2_ALG_TYPE);
1163 	if (ret)
1164 		crypto_free_akcipher(ctx->rsa.soft_tfm);
1165 
1166 	return ret;
1167 }
1168 
1169 static void hpre_rsa_exit_tfm(struct crypto_akcipher *tfm)
1170 {
1171 	struct hpre_ctx *ctx = akcipher_tfm_ctx(tfm);
1172 
1173 	hpre_rsa_clear_ctx(ctx, true);
1174 	crypto_free_akcipher(ctx->rsa.soft_tfm);
1175 }
1176 
1177 static void hpre_key_to_big_end(u8 *data, int len)
1178 {
1179 	int i, j;
1180 
1181 	for (i = 0; i < len / 2; i++) {
1182 		j = len - i - 1;
1183 		swap(data[j], data[i]);
1184 	}
1185 }
1186 
1187 static void hpre_ecc_clear_ctx(struct hpre_ctx *ctx, bool is_clear_all,
1188 			       bool is_ecdh)
1189 {
1190 	struct device *dev = ctx->dev;
1191 	unsigned int sz = ctx->key_sz;
1192 	unsigned int shift = sz << 1;
1193 
1194 	if (is_clear_all)
1195 		hisi_qm_stop_qp(ctx->qp);
1196 
1197 	if (is_ecdh && ctx->ecdh.p) {
1198 		/* ecdh: p->a->k->b */
1199 		memzero_explicit(ctx->ecdh.p + shift, sz);
1200 		dma_free_coherent(dev, sz << 3, ctx->ecdh.p, ctx->ecdh.dma_p);
1201 		ctx->ecdh.p = NULL;
1202 	} else if (!is_ecdh && ctx->curve25519.p) {
1203 		/* curve25519: p->a->k */
1204 		memzero_explicit(ctx->curve25519.p + shift, sz);
1205 		dma_free_coherent(dev, sz << 2, ctx->curve25519.p,
1206 				  ctx->curve25519.dma_p);
1207 		ctx->curve25519.p = NULL;
1208 	}
1209 
1210 	hpre_ctx_clear(ctx, is_clear_all);
1211 }
1212 
1213 /*
1214  * The bits of 192/224/256/384/521 are supported by HPRE,
1215  * and convert the bits like:
1216  * bits<=256, bits=256; 256<bits<=384, bits=384; 384<bits<=576, bits=576;
1217  * If the parameter bit width is insufficient, then we fill in the
1218  * high-order zeros by soft, so TASK_LENGTH1 is 0x3/0x5/0x8;
1219  */
1220 static unsigned int hpre_ecdh_supported_curve(unsigned short id)
1221 {
1222 	switch (id) {
1223 	case ECC_CURVE_NIST_P192:
1224 	case ECC_CURVE_NIST_P256:
1225 		return HPRE_ECC_HW256_KSZ_B;
1226 	case ECC_CURVE_NIST_P384:
1227 		return HPRE_ECC_HW384_KSZ_B;
1228 	default:
1229 		break;
1230 	}
1231 
1232 	return 0;
1233 }
1234 
1235 static void fill_curve_param(void *addr, u64 *param, unsigned int cur_sz, u8 ndigits)
1236 {
1237 	unsigned int sz = cur_sz - (ndigits - 1) * sizeof(u64);
1238 	u8 i = 0;
1239 
1240 	while (i < ndigits - 1) {
1241 		memcpy(addr + sizeof(u64) * i, &param[i], sizeof(u64));
1242 		i++;
1243 	}
1244 
1245 	memcpy(addr + sizeof(u64) * i, &param[ndigits - 1], sz);
1246 	hpre_key_to_big_end((u8 *)addr, cur_sz);
1247 }
1248 
1249 static int hpre_ecdh_fill_curve(struct hpre_ctx *ctx, struct ecdh *params,
1250 				unsigned int cur_sz)
1251 {
1252 	unsigned int shifta = ctx->key_sz << 1;
1253 	unsigned int shiftb = ctx->key_sz << 2;
1254 	void *p = ctx->ecdh.p + ctx->key_sz - cur_sz;
1255 	void *a = ctx->ecdh.p + shifta - cur_sz;
1256 	void *b = ctx->ecdh.p + shiftb - cur_sz;
1257 	void *x = ctx->ecdh.g + ctx->key_sz - cur_sz;
1258 	void *y = ctx->ecdh.g + shifta - cur_sz;
1259 	const struct ecc_curve *curve = ecc_get_curve(ctx->curve_id);
1260 	char *n;
1261 
1262 	if (unlikely(!curve))
1263 		return -EINVAL;
1264 
1265 	n = kzalloc(ctx->key_sz, GFP_KERNEL);
1266 	if (!n)
1267 		return -ENOMEM;
1268 
1269 	fill_curve_param(p, curve->p, cur_sz, curve->g.ndigits);
1270 	fill_curve_param(a, curve->a, cur_sz, curve->g.ndigits);
1271 	fill_curve_param(b, curve->b, cur_sz, curve->g.ndigits);
1272 	fill_curve_param(x, curve->g.x, cur_sz, curve->g.ndigits);
1273 	fill_curve_param(y, curve->g.y, cur_sz, curve->g.ndigits);
1274 	fill_curve_param(n, curve->n, cur_sz, curve->g.ndigits);
1275 
1276 	if (params->key_size == cur_sz && memcmp(params->key, n, cur_sz) >= 0) {
1277 		kfree(n);
1278 		return -EINVAL;
1279 	}
1280 
1281 	kfree(n);
1282 	return 0;
1283 }
1284 
1285 static unsigned int hpre_ecdh_get_curvesz(unsigned short id)
1286 {
1287 	switch (id) {
1288 	case ECC_CURVE_NIST_P192:
1289 		return HPRE_ECC_NIST_P192_N_SIZE;
1290 	case ECC_CURVE_NIST_P256:
1291 		return HPRE_ECC_NIST_P256_N_SIZE;
1292 	case ECC_CURVE_NIST_P384:
1293 		return HPRE_ECC_NIST_P384_N_SIZE;
1294 	default:
1295 		break;
1296 	}
1297 
1298 	return 0;
1299 }
1300 
1301 static int hpre_ecdh_set_param(struct hpre_ctx *ctx, struct ecdh *params)
1302 {
1303 	struct device *dev = ctx->dev;
1304 	unsigned int sz, shift, curve_sz;
1305 	int ret;
1306 
1307 	ctx->key_sz = hpre_ecdh_supported_curve(ctx->curve_id);
1308 	if (!ctx->key_sz)
1309 		return -EINVAL;
1310 
1311 	curve_sz = hpre_ecdh_get_curvesz(ctx->curve_id);
1312 	if (!curve_sz || params->key_size > curve_sz)
1313 		return -EINVAL;
1314 
1315 	sz = ctx->key_sz;
1316 
1317 	if (!ctx->ecdh.p) {
1318 		ctx->ecdh.p = dma_alloc_coherent(dev, sz << 3, &ctx->ecdh.dma_p,
1319 						 GFP_KERNEL);
1320 		if (!ctx->ecdh.p)
1321 			return -ENOMEM;
1322 	}
1323 
1324 	shift = sz << 2;
1325 	ctx->ecdh.g = ctx->ecdh.p + shift;
1326 	ctx->ecdh.dma_g = ctx->ecdh.dma_p + shift;
1327 
1328 	ret = hpre_ecdh_fill_curve(ctx, params, curve_sz);
1329 	if (ret) {
1330 		dev_err(dev, "failed to fill curve_param, ret = %d!\n", ret);
1331 		dma_free_coherent(dev, sz << 3, ctx->ecdh.p, ctx->ecdh.dma_p);
1332 		ctx->ecdh.p = NULL;
1333 		return ret;
1334 	}
1335 
1336 	return 0;
1337 }
1338 
1339 static bool hpre_key_is_zero(char *key, unsigned short key_sz)
1340 {
1341 	int i;
1342 
1343 	for (i = 0; i < key_sz; i++)
1344 		if (key[i])
1345 			return false;
1346 
1347 	return true;
1348 }
1349 
1350 static int ecdh_gen_privkey(struct hpre_ctx *ctx, struct ecdh *params)
1351 {
1352 	struct device *dev = ctx->dev;
1353 	int ret;
1354 
1355 	ret = crypto_get_default_rng();
1356 	if (ret) {
1357 		dev_err(dev, "failed to get default rng, ret = %d!\n", ret);
1358 		return ret;
1359 	}
1360 
1361 	ret = crypto_rng_get_bytes(crypto_default_rng, (u8 *)params->key,
1362 				   params->key_size);
1363 	crypto_put_default_rng();
1364 	if (ret)
1365 		dev_err(dev, "failed to get rng, ret = %d!\n", ret);
1366 
1367 	return ret;
1368 }
1369 
1370 static int hpre_ecdh_set_secret(struct crypto_kpp *tfm, const void *buf,
1371 				unsigned int len)
1372 {
1373 	struct hpre_ctx *ctx = kpp_tfm_ctx(tfm);
1374 	struct device *dev = ctx->dev;
1375 	char key[HPRE_ECC_MAX_KSZ];
1376 	unsigned int sz, sz_shift;
1377 	struct ecdh params;
1378 	int ret;
1379 
1380 	if (crypto_ecdh_decode_key(buf, len, &params) < 0) {
1381 		dev_err(dev, "failed to decode ecdh key!\n");
1382 		return -EINVAL;
1383 	}
1384 
1385 	/* Use stdrng to generate private key */
1386 	if (!params.key || !params.key_size) {
1387 		params.key = key;
1388 		params.key_size = hpre_ecdh_get_curvesz(ctx->curve_id);
1389 		ret = ecdh_gen_privkey(ctx, &params);
1390 		if (ret)
1391 			return ret;
1392 	}
1393 
1394 	if (hpre_key_is_zero(params.key, params.key_size)) {
1395 		dev_err(dev, "Invalid hpre key!\n");
1396 		return -EINVAL;
1397 	}
1398 
1399 	hpre_ecc_clear_ctx(ctx, false, true);
1400 
1401 	ret = hpre_ecdh_set_param(ctx, &params);
1402 	if (ret < 0) {
1403 		dev_err(dev, "failed to set hpre param, ret = %d!\n", ret);
1404 		return ret;
1405 	}
1406 
1407 	sz = ctx->key_sz;
1408 	sz_shift = (sz << 1) + sz - params.key_size;
1409 	memcpy(ctx->ecdh.p + sz_shift, params.key, params.key_size);
1410 
1411 	return 0;
1412 }
1413 
1414 static void hpre_ecdh_hw_data_clr_all(struct hpre_ctx *ctx,
1415 				      struct hpre_asym_request *req,
1416 				      struct scatterlist *dst,
1417 				      struct scatterlist *src)
1418 {
1419 	struct device *dev = ctx->dev;
1420 	struct hpre_sqe *sqe = &req->req;
1421 	dma_addr_t dma;
1422 
1423 	dma = le64_to_cpu(sqe->in);
1424 	if (unlikely(dma_mapping_error(dev, dma)))
1425 		return;
1426 
1427 	if (src && req->src)
1428 		dma_free_coherent(dev, ctx->key_sz << 2, req->src, dma);
1429 
1430 	dma = le64_to_cpu(sqe->out);
1431 	if (unlikely(dma_mapping_error(dev, dma)))
1432 		return;
1433 
1434 	if (req->dst)
1435 		dma_free_coherent(dev, ctx->key_sz << 1, req->dst, dma);
1436 	if (dst)
1437 		dma_unmap_single(dev, dma, ctx->key_sz << 1, DMA_FROM_DEVICE);
1438 }
1439 
1440 static void hpre_ecdh_cb(struct hpre_ctx *ctx, void *resp)
1441 {
1442 	unsigned int curve_sz = hpre_ecdh_get_curvesz(ctx->curve_id);
1443 	struct hpre_dfx *dfx = ctx->hpre->debug.dfx;
1444 	struct hpre_asym_request *req = NULL;
1445 	struct kpp_request *areq;
1446 	u64 overtime_thrhld;
1447 	char *p;
1448 	int ret;
1449 
1450 	ret = hpre_alg_res_post_hf(ctx, resp, (void **)&req);
1451 	areq = req->areq.ecdh;
1452 	areq->dst_len = ctx->key_sz << 1;
1453 
1454 	overtime_thrhld = atomic64_read(&dfx[HPRE_OVERTIME_THRHLD].value);
1455 	if (overtime_thrhld && hpre_is_bd_timeout(req, overtime_thrhld))
1456 		atomic64_inc(&dfx[HPRE_OVER_THRHLD_CNT].value);
1457 
1458 	p = sg_virt(areq->dst);
1459 	memmove(p, p + ctx->key_sz - curve_sz, curve_sz);
1460 	memmove(p + curve_sz, p + areq->dst_len - curve_sz, curve_sz);
1461 
1462 	hpre_ecdh_hw_data_clr_all(ctx, req, areq->dst, areq->src);
1463 	kpp_request_complete(areq, ret);
1464 
1465 	atomic64_inc(&dfx[HPRE_RECV_CNT].value);
1466 }
1467 
1468 static int hpre_ecdh_msg_request_set(struct hpre_ctx *ctx,
1469 				     struct kpp_request *req)
1470 {
1471 	struct hpre_asym_request *h_req;
1472 	struct hpre_sqe *msg;
1473 	int req_id;
1474 	void *tmp;
1475 
1476 	if (req->dst_len < ctx->key_sz << 1) {
1477 		req->dst_len = ctx->key_sz << 1;
1478 		return -EINVAL;
1479 	}
1480 
1481 	tmp = kpp_request_ctx(req);
1482 	h_req = PTR_ALIGN(tmp, HPRE_ALIGN_SZ);
1483 	h_req->cb = hpre_ecdh_cb;
1484 	h_req->areq.ecdh = req;
1485 	msg = &h_req->req;
1486 	memset(msg, 0, sizeof(*msg));
1487 	msg->in = cpu_to_le64(DMA_MAPPING_ERROR);
1488 	msg->out = cpu_to_le64(DMA_MAPPING_ERROR);
1489 	msg->key = cpu_to_le64(ctx->ecdh.dma_p);
1490 
1491 	msg->dw0 |= cpu_to_le32(0x1U << HPRE_SQE_DONE_SHIFT);
1492 	msg->task_len1 = (ctx->key_sz >> HPRE_BITS_2_BYTES_SHIFT) - 1;
1493 	h_req->ctx = ctx;
1494 
1495 	req_id = hpre_add_req_to_ctx(h_req);
1496 	if (req_id < 0)
1497 		return -EBUSY;
1498 
1499 	msg->tag = cpu_to_le16((u16)req_id);
1500 	return 0;
1501 }
1502 
1503 static int hpre_ecdh_src_data_init(struct hpre_asym_request *hpre_req,
1504 				   struct scatterlist *data, unsigned int len)
1505 {
1506 	struct hpre_sqe *msg = &hpre_req->req;
1507 	struct hpre_ctx *ctx = hpre_req->ctx;
1508 	struct device *dev = ctx->dev;
1509 	unsigned int tmpshift;
1510 	dma_addr_t dma = 0;
1511 	void *ptr;
1512 	int shift;
1513 
1514 	/* Src_data include gx and gy. */
1515 	shift = ctx->key_sz - (len >> 1);
1516 	if (unlikely(shift < 0))
1517 		return -EINVAL;
1518 
1519 	ptr = dma_alloc_coherent(dev, ctx->key_sz << 2, &dma, GFP_KERNEL);
1520 	if (unlikely(!ptr))
1521 		return -ENOMEM;
1522 
1523 	tmpshift = ctx->key_sz << 1;
1524 	scatterwalk_map_and_copy(ptr + tmpshift, data, 0, len, 0);
1525 	memcpy(ptr + shift, ptr + tmpshift, len >> 1);
1526 	memcpy(ptr + ctx->key_sz + shift, ptr + tmpshift + (len >> 1), len >> 1);
1527 
1528 	hpre_req->src = ptr;
1529 	msg->in = cpu_to_le64(dma);
1530 	return 0;
1531 }
1532 
1533 static int hpre_ecdh_dst_data_init(struct hpre_asym_request *hpre_req,
1534 				   struct scatterlist *data, unsigned int len)
1535 {
1536 	struct hpre_sqe *msg = &hpre_req->req;
1537 	struct hpre_ctx *ctx = hpre_req->ctx;
1538 	struct device *dev = ctx->dev;
1539 	dma_addr_t dma;
1540 
1541 	if (unlikely(!data || !sg_is_last(data) || len != ctx->key_sz << 1)) {
1542 		dev_err(dev, "data or data length is illegal!\n");
1543 		return -EINVAL;
1544 	}
1545 
1546 	hpre_req->dst = NULL;
1547 	dma = dma_map_single(dev, sg_virt(data), len, DMA_FROM_DEVICE);
1548 	if (unlikely(dma_mapping_error(dev, dma))) {
1549 		dev_err(dev, "dma map data err!\n");
1550 		return -ENOMEM;
1551 	}
1552 
1553 	msg->out = cpu_to_le64(dma);
1554 	return 0;
1555 }
1556 
1557 static int hpre_ecdh_compute_value(struct kpp_request *req)
1558 {
1559 	struct crypto_kpp *tfm = crypto_kpp_reqtfm(req);
1560 	struct hpre_ctx *ctx = kpp_tfm_ctx(tfm);
1561 	struct device *dev = ctx->dev;
1562 	void *tmp = kpp_request_ctx(req);
1563 	struct hpre_asym_request *hpre_req = PTR_ALIGN(tmp, HPRE_ALIGN_SZ);
1564 	struct hpre_sqe *msg = &hpre_req->req;
1565 	int ret;
1566 
1567 	ret = hpre_ecdh_msg_request_set(ctx, req);
1568 	if (unlikely(ret)) {
1569 		dev_err(dev, "failed to set ecdh request, ret = %d!\n", ret);
1570 		return ret;
1571 	}
1572 
1573 	if (req->src) {
1574 		ret = hpre_ecdh_src_data_init(hpre_req, req->src, req->src_len);
1575 		if (unlikely(ret)) {
1576 			dev_err(dev, "failed to init src data, ret = %d!\n", ret);
1577 			goto clear_all;
1578 		}
1579 	} else {
1580 		msg->in = cpu_to_le64(ctx->ecdh.dma_g);
1581 	}
1582 
1583 	ret = hpre_ecdh_dst_data_init(hpre_req, req->dst, req->dst_len);
1584 	if (unlikely(ret)) {
1585 		dev_err(dev, "failed to init dst data, ret = %d!\n", ret);
1586 		goto clear_all;
1587 	}
1588 
1589 	msg->dw0 = cpu_to_le32(le32_to_cpu(msg->dw0) | HPRE_ALG_ECC_MUL);
1590 	ret = hpre_send(ctx, msg);
1591 	if (likely(!ret))
1592 		return -EINPROGRESS;
1593 
1594 clear_all:
1595 	hpre_rm_req_from_ctx(hpre_req);
1596 	hpre_ecdh_hw_data_clr_all(ctx, hpre_req, req->dst, req->src);
1597 	return ret;
1598 }
1599 
1600 static unsigned int hpre_ecdh_max_size(struct crypto_kpp *tfm)
1601 {
1602 	struct hpre_ctx *ctx = kpp_tfm_ctx(tfm);
1603 
1604 	/* max size is the pub_key_size, include x and y */
1605 	return ctx->key_sz << 1;
1606 }
1607 
1608 static int hpre_ecdh_nist_p192_init_tfm(struct crypto_kpp *tfm)
1609 {
1610 	struct hpre_ctx *ctx = kpp_tfm_ctx(tfm);
1611 
1612 	ctx->curve_id = ECC_CURVE_NIST_P192;
1613 
1614 	return hpre_ctx_init(ctx, HPRE_V3_ECC_ALG_TYPE);
1615 }
1616 
1617 static int hpre_ecdh_nist_p256_init_tfm(struct crypto_kpp *tfm)
1618 {
1619 	struct hpre_ctx *ctx = kpp_tfm_ctx(tfm);
1620 
1621 	ctx->curve_id = ECC_CURVE_NIST_P256;
1622 
1623 	return hpre_ctx_init(ctx, HPRE_V3_ECC_ALG_TYPE);
1624 }
1625 
1626 static int hpre_ecdh_nist_p384_init_tfm(struct crypto_kpp *tfm)
1627 {
1628 	struct hpre_ctx *ctx = kpp_tfm_ctx(tfm);
1629 
1630 	ctx->curve_id = ECC_CURVE_NIST_P384;
1631 
1632 	return hpre_ctx_init(ctx, HPRE_V3_ECC_ALG_TYPE);
1633 }
1634 
1635 static void hpre_ecdh_exit_tfm(struct crypto_kpp *tfm)
1636 {
1637 	struct hpre_ctx *ctx = kpp_tfm_ctx(tfm);
1638 
1639 	hpre_ecc_clear_ctx(ctx, true, true);
1640 }
1641 
1642 static void hpre_curve25519_fill_curve(struct hpre_ctx *ctx, const void *buf,
1643 				       unsigned int len)
1644 {
1645 	u8 secret[CURVE25519_KEY_SIZE] = { 0 };
1646 	unsigned int sz = ctx->key_sz;
1647 	const struct ecc_curve *curve;
1648 	unsigned int shift = sz << 1;
1649 	void *p;
1650 
1651 	/*
1652 	 * The key from 'buf' is in little-endian, we should preprocess it as
1653 	 * the description in rfc7748: "k[0] &= 248, k[31] &= 127, k[31] |= 64",
1654 	 * then convert it to big endian. Only in this way, the result can be
1655 	 * the same as the software curve-25519 that exists in crypto.
1656 	 */
1657 	memcpy(secret, buf, len);
1658 	curve25519_clamp_secret(secret);
1659 	hpre_key_to_big_end(secret, CURVE25519_KEY_SIZE);
1660 
1661 	p = ctx->curve25519.p + sz - len;
1662 
1663 	curve = ecc_get_curve25519();
1664 
1665 	/* fill curve parameters */
1666 	fill_curve_param(p, curve->p, len, curve->g.ndigits);
1667 	fill_curve_param(p + sz, curve->a, len, curve->g.ndigits);
1668 	memcpy(p + shift, secret, len);
1669 	fill_curve_param(p + shift + sz, curve->g.x, len, curve->g.ndigits);
1670 	memzero_explicit(secret, CURVE25519_KEY_SIZE);
1671 }
1672 
1673 static int hpre_curve25519_set_param(struct hpre_ctx *ctx, const void *buf,
1674 				     unsigned int len)
1675 {
1676 	struct device *dev = ctx->dev;
1677 	unsigned int sz = ctx->key_sz;
1678 	unsigned int shift = sz << 1;
1679 
1680 	/* p->a->k->gx */
1681 	if (!ctx->curve25519.p) {
1682 		ctx->curve25519.p = dma_alloc_coherent(dev, sz << 2,
1683 						       &ctx->curve25519.dma_p,
1684 						       GFP_KERNEL);
1685 		if (!ctx->curve25519.p)
1686 			return -ENOMEM;
1687 	}
1688 
1689 	ctx->curve25519.g = ctx->curve25519.p + shift + sz;
1690 	ctx->curve25519.dma_g = ctx->curve25519.dma_p + shift + sz;
1691 
1692 	hpre_curve25519_fill_curve(ctx, buf, len);
1693 
1694 	return 0;
1695 }
1696 
1697 static int hpre_curve25519_set_secret(struct crypto_kpp *tfm, const void *buf,
1698 				      unsigned int len)
1699 {
1700 	struct hpre_ctx *ctx = kpp_tfm_ctx(tfm);
1701 	struct device *dev = ctx->dev;
1702 	int ret = -EINVAL;
1703 
1704 	if (len != CURVE25519_KEY_SIZE ||
1705 	    !crypto_memneq(buf, curve25519_null_point, CURVE25519_KEY_SIZE)) {
1706 		dev_err(dev, "key is null or key len is not 32bytes!\n");
1707 		return ret;
1708 	}
1709 
1710 	/* Free old secret if any */
1711 	hpre_ecc_clear_ctx(ctx, false, false);
1712 
1713 	ctx->key_sz = CURVE25519_KEY_SIZE;
1714 	ret = hpre_curve25519_set_param(ctx, buf, CURVE25519_KEY_SIZE);
1715 	if (ret) {
1716 		dev_err(dev, "failed to set curve25519 param, ret = %d!\n", ret);
1717 		hpre_ecc_clear_ctx(ctx, false, false);
1718 		return ret;
1719 	}
1720 
1721 	return 0;
1722 }
1723 
1724 static void hpre_curve25519_hw_data_clr_all(struct hpre_ctx *ctx,
1725 					    struct hpre_asym_request *req,
1726 					    struct scatterlist *dst,
1727 					    struct scatterlist *src)
1728 {
1729 	struct device *dev = ctx->dev;
1730 	struct hpre_sqe *sqe = &req->req;
1731 	dma_addr_t dma;
1732 
1733 	dma = le64_to_cpu(sqe->in);
1734 	if (unlikely(dma_mapping_error(dev, dma)))
1735 		return;
1736 
1737 	if (src && req->src)
1738 		dma_free_coherent(dev, ctx->key_sz, req->src, dma);
1739 
1740 	dma = le64_to_cpu(sqe->out);
1741 	if (unlikely(dma_mapping_error(dev, dma)))
1742 		return;
1743 
1744 	if (req->dst)
1745 		dma_free_coherent(dev, ctx->key_sz, req->dst, dma);
1746 	if (dst)
1747 		dma_unmap_single(dev, dma, ctx->key_sz, DMA_FROM_DEVICE);
1748 }
1749 
1750 static void hpre_curve25519_cb(struct hpre_ctx *ctx, void *resp)
1751 {
1752 	struct hpre_dfx *dfx = ctx->hpre->debug.dfx;
1753 	struct hpre_asym_request *req = NULL;
1754 	struct kpp_request *areq;
1755 	u64 overtime_thrhld;
1756 	int ret;
1757 
1758 	ret = hpre_alg_res_post_hf(ctx, resp, (void **)&req);
1759 	areq = req->areq.curve25519;
1760 	areq->dst_len = ctx->key_sz;
1761 
1762 	overtime_thrhld = atomic64_read(&dfx[HPRE_OVERTIME_THRHLD].value);
1763 	if (overtime_thrhld && hpre_is_bd_timeout(req, overtime_thrhld))
1764 		atomic64_inc(&dfx[HPRE_OVER_THRHLD_CNT].value);
1765 
1766 	hpre_key_to_big_end(sg_virt(areq->dst), CURVE25519_KEY_SIZE);
1767 
1768 	hpre_curve25519_hw_data_clr_all(ctx, req, areq->dst, areq->src);
1769 	kpp_request_complete(areq, ret);
1770 
1771 	atomic64_inc(&dfx[HPRE_RECV_CNT].value);
1772 }
1773 
1774 static int hpre_curve25519_msg_request_set(struct hpre_ctx *ctx,
1775 					   struct kpp_request *req)
1776 {
1777 	struct hpre_asym_request *h_req;
1778 	struct hpre_sqe *msg;
1779 	int req_id;
1780 	void *tmp;
1781 
1782 	if (unlikely(req->dst_len < ctx->key_sz)) {
1783 		req->dst_len = ctx->key_sz;
1784 		return -EINVAL;
1785 	}
1786 
1787 	tmp = kpp_request_ctx(req);
1788 	h_req = PTR_ALIGN(tmp, HPRE_ALIGN_SZ);
1789 	h_req->cb = hpre_curve25519_cb;
1790 	h_req->areq.curve25519 = req;
1791 	msg = &h_req->req;
1792 	memset(msg, 0, sizeof(*msg));
1793 	msg->in = cpu_to_le64(DMA_MAPPING_ERROR);
1794 	msg->out = cpu_to_le64(DMA_MAPPING_ERROR);
1795 	msg->key = cpu_to_le64(ctx->curve25519.dma_p);
1796 
1797 	msg->dw0 |= cpu_to_le32(0x1U << HPRE_SQE_DONE_SHIFT);
1798 	msg->task_len1 = (ctx->key_sz >> HPRE_BITS_2_BYTES_SHIFT) - 1;
1799 	h_req->ctx = ctx;
1800 
1801 	req_id = hpre_add_req_to_ctx(h_req);
1802 	if (req_id < 0)
1803 		return -EBUSY;
1804 
1805 	msg->tag = cpu_to_le16((u16)req_id);
1806 	return 0;
1807 }
1808 
1809 static void hpre_curve25519_src_modulo_p(u8 *ptr)
1810 {
1811 	int i;
1812 
1813 	for (i = 0; i < CURVE25519_KEY_SIZE - 1; i++)
1814 		ptr[i] = 0;
1815 
1816 	/* The modulus is ptr's last byte minus '0xed'(last byte of p) */
1817 	ptr[i] -= 0xed;
1818 }
1819 
1820 static int hpre_curve25519_src_init(struct hpre_asym_request *hpre_req,
1821 				    struct scatterlist *data, unsigned int len)
1822 {
1823 	struct hpre_sqe *msg = &hpre_req->req;
1824 	struct hpre_ctx *ctx = hpre_req->ctx;
1825 	struct device *dev = ctx->dev;
1826 	u8 p[CURVE25519_KEY_SIZE] = { 0 };
1827 	const struct ecc_curve *curve;
1828 	dma_addr_t dma = 0;
1829 	u8 *ptr;
1830 
1831 	if (len != CURVE25519_KEY_SIZE) {
1832 		dev_err(dev, "sourc_data len is not 32bytes, len = %u!\n", len);
1833 		return -EINVAL;
1834 	}
1835 
1836 	ptr = dma_alloc_coherent(dev, ctx->key_sz, &dma, GFP_KERNEL);
1837 	if (unlikely(!ptr))
1838 		return -ENOMEM;
1839 
1840 	scatterwalk_map_and_copy(ptr, data, 0, len, 0);
1841 
1842 	if (!crypto_memneq(ptr, curve25519_null_point, CURVE25519_KEY_SIZE)) {
1843 		dev_err(dev, "gx is null!\n");
1844 		goto err;
1845 	}
1846 
1847 	/*
1848 	 * Src_data(gx) is in little-endian order, MSB in the final byte should
1849 	 * be masked as described in RFC7748, then transform it to big-endian
1850 	 * form, then hisi_hpre can use the data.
1851 	 */
1852 	ptr[31] &= 0x7f;
1853 	hpre_key_to_big_end(ptr, CURVE25519_KEY_SIZE);
1854 
1855 	curve = ecc_get_curve25519();
1856 
1857 	fill_curve_param(p, curve->p, CURVE25519_KEY_SIZE, curve->g.ndigits);
1858 
1859 	/*
1860 	 * When src_data equals (2^255 - 19) ~  (2^255 - 1), it is out of p,
1861 	 * we get its modulus to p, and then use it.
1862 	 */
1863 	if (memcmp(ptr, p, ctx->key_sz) == 0) {
1864 		dev_err(dev, "gx is p!\n");
1865 		goto err;
1866 	} else if (memcmp(ptr, p, ctx->key_sz) > 0) {
1867 		hpre_curve25519_src_modulo_p(ptr);
1868 	}
1869 
1870 	hpre_req->src = ptr;
1871 	msg->in = cpu_to_le64(dma);
1872 	return 0;
1873 
1874 err:
1875 	dma_free_coherent(dev, ctx->key_sz, ptr, dma);
1876 	return -EINVAL;
1877 }
1878 
1879 static int hpre_curve25519_dst_init(struct hpre_asym_request *hpre_req,
1880 				    struct scatterlist *data, unsigned int len)
1881 {
1882 	struct hpre_sqe *msg = &hpre_req->req;
1883 	struct hpre_ctx *ctx = hpre_req->ctx;
1884 	struct device *dev = ctx->dev;
1885 	dma_addr_t dma;
1886 
1887 	if (!data || !sg_is_last(data) || len != ctx->key_sz) {
1888 		dev_err(dev, "data or data length is illegal!\n");
1889 		return -EINVAL;
1890 	}
1891 
1892 	hpre_req->dst = NULL;
1893 	dma = dma_map_single(dev, sg_virt(data), len, DMA_FROM_DEVICE);
1894 	if (unlikely(dma_mapping_error(dev, dma))) {
1895 		dev_err(dev, "dma map data err!\n");
1896 		return -ENOMEM;
1897 	}
1898 
1899 	msg->out = cpu_to_le64(dma);
1900 	return 0;
1901 }
1902 
1903 static int hpre_curve25519_compute_value(struct kpp_request *req)
1904 {
1905 	struct crypto_kpp *tfm = crypto_kpp_reqtfm(req);
1906 	struct hpre_ctx *ctx = kpp_tfm_ctx(tfm);
1907 	struct device *dev = ctx->dev;
1908 	void *tmp = kpp_request_ctx(req);
1909 	struct hpre_asym_request *hpre_req = PTR_ALIGN(tmp, HPRE_ALIGN_SZ);
1910 	struct hpre_sqe *msg = &hpre_req->req;
1911 	int ret;
1912 
1913 	ret = hpre_curve25519_msg_request_set(ctx, req);
1914 	if (unlikely(ret)) {
1915 		dev_err(dev, "failed to set curve25519 request, ret = %d!\n", ret);
1916 		return ret;
1917 	}
1918 
1919 	if (req->src) {
1920 		ret = hpre_curve25519_src_init(hpre_req, req->src, req->src_len);
1921 		if (unlikely(ret)) {
1922 			dev_err(dev, "failed to init src data, ret = %d!\n",
1923 				ret);
1924 			goto clear_all;
1925 		}
1926 	} else {
1927 		msg->in = cpu_to_le64(ctx->curve25519.dma_g);
1928 	}
1929 
1930 	ret = hpre_curve25519_dst_init(hpre_req, req->dst, req->dst_len);
1931 	if (unlikely(ret)) {
1932 		dev_err(dev, "failed to init dst data, ret = %d!\n", ret);
1933 		goto clear_all;
1934 	}
1935 
1936 	msg->dw0 = cpu_to_le32(le32_to_cpu(msg->dw0) | HPRE_ALG_CURVE25519_MUL);
1937 	ret = hpre_send(ctx, msg);
1938 	if (likely(!ret))
1939 		return -EINPROGRESS;
1940 
1941 clear_all:
1942 	hpre_rm_req_from_ctx(hpre_req);
1943 	hpre_curve25519_hw_data_clr_all(ctx, hpre_req, req->dst, req->src);
1944 	return ret;
1945 }
1946 
1947 static unsigned int hpre_curve25519_max_size(struct crypto_kpp *tfm)
1948 {
1949 	struct hpre_ctx *ctx = kpp_tfm_ctx(tfm);
1950 
1951 	return ctx->key_sz;
1952 }
1953 
1954 static int hpre_curve25519_init_tfm(struct crypto_kpp *tfm)
1955 {
1956 	struct hpre_ctx *ctx = kpp_tfm_ctx(tfm);
1957 
1958 	return hpre_ctx_init(ctx, HPRE_V3_ECC_ALG_TYPE);
1959 }
1960 
1961 static void hpre_curve25519_exit_tfm(struct crypto_kpp *tfm)
1962 {
1963 	struct hpre_ctx *ctx = kpp_tfm_ctx(tfm);
1964 
1965 	hpre_ecc_clear_ctx(ctx, true, false);
1966 }
1967 
1968 static struct akcipher_alg rsa = {
1969 	.sign = hpre_rsa_dec,
1970 	.verify = hpre_rsa_enc,
1971 	.encrypt = hpre_rsa_enc,
1972 	.decrypt = hpre_rsa_dec,
1973 	.set_pub_key = hpre_rsa_setpubkey,
1974 	.set_priv_key = hpre_rsa_setprivkey,
1975 	.max_size = hpre_rsa_max_size,
1976 	.init = hpre_rsa_init_tfm,
1977 	.exit = hpre_rsa_exit_tfm,
1978 	.reqsize = sizeof(struct hpre_asym_request) + HPRE_ALIGN_SZ,
1979 	.base = {
1980 		.cra_ctxsize = sizeof(struct hpre_ctx),
1981 		.cra_priority = HPRE_CRYPTO_ALG_PRI,
1982 		.cra_name = "rsa",
1983 		.cra_driver_name = "hpre-rsa",
1984 		.cra_module = THIS_MODULE,
1985 	},
1986 };
1987 
1988 static struct kpp_alg dh = {
1989 	.set_secret = hpre_dh_set_secret,
1990 	.generate_public_key = hpre_dh_compute_value,
1991 	.compute_shared_secret = hpre_dh_compute_value,
1992 	.max_size = hpre_dh_max_size,
1993 	.init = hpre_dh_init_tfm,
1994 	.exit = hpre_dh_exit_tfm,
1995 	.reqsize = sizeof(struct hpre_asym_request) + HPRE_ALIGN_SZ,
1996 	.base = {
1997 		.cra_ctxsize = sizeof(struct hpre_ctx),
1998 		.cra_priority = HPRE_CRYPTO_ALG_PRI,
1999 		.cra_name = "dh",
2000 		.cra_driver_name = "hpre-dh",
2001 		.cra_module = THIS_MODULE,
2002 	},
2003 };
2004 
2005 static struct kpp_alg ecdh_nist_p192 = {
2006 	.set_secret = hpre_ecdh_set_secret,
2007 	.generate_public_key = hpre_ecdh_compute_value,
2008 	.compute_shared_secret = hpre_ecdh_compute_value,
2009 	.max_size = hpre_ecdh_max_size,
2010 	.init = hpre_ecdh_nist_p192_init_tfm,
2011 	.exit = hpre_ecdh_exit_tfm,
2012 	.reqsize = sizeof(struct hpre_asym_request) + HPRE_ALIGN_SZ,
2013 	.base = {
2014 		.cra_ctxsize = sizeof(struct hpre_ctx),
2015 		.cra_priority = HPRE_CRYPTO_ALG_PRI,
2016 		.cra_name = "ecdh-nist-p192",
2017 		.cra_driver_name = "hpre-ecdh-nist-p192",
2018 		.cra_module = THIS_MODULE,
2019 	},
2020 };
2021 
2022 static struct kpp_alg ecdh_nist_p256 = {
2023 	.set_secret = hpre_ecdh_set_secret,
2024 	.generate_public_key = hpre_ecdh_compute_value,
2025 	.compute_shared_secret = hpre_ecdh_compute_value,
2026 	.max_size = hpre_ecdh_max_size,
2027 	.init = hpre_ecdh_nist_p256_init_tfm,
2028 	.exit = hpre_ecdh_exit_tfm,
2029 	.reqsize = sizeof(struct hpre_asym_request) + HPRE_ALIGN_SZ,
2030 	.base = {
2031 		.cra_ctxsize = sizeof(struct hpre_ctx),
2032 		.cra_priority = HPRE_CRYPTO_ALG_PRI,
2033 		.cra_name = "ecdh-nist-p256",
2034 		.cra_driver_name = "hpre-ecdh-nist-p256",
2035 		.cra_module = THIS_MODULE,
2036 	},
2037 };
2038 
2039 static struct kpp_alg ecdh_nist_p384 = {
2040 	.set_secret = hpre_ecdh_set_secret,
2041 	.generate_public_key = hpre_ecdh_compute_value,
2042 	.compute_shared_secret = hpre_ecdh_compute_value,
2043 	.max_size = hpre_ecdh_max_size,
2044 	.init = hpre_ecdh_nist_p384_init_tfm,
2045 	.exit = hpre_ecdh_exit_tfm,
2046 	.reqsize = sizeof(struct hpre_asym_request) + HPRE_ALIGN_SZ,
2047 	.base = {
2048 		.cra_ctxsize = sizeof(struct hpre_ctx),
2049 		.cra_priority = HPRE_CRYPTO_ALG_PRI,
2050 		.cra_name = "ecdh-nist-p384",
2051 		.cra_driver_name = "hpre-ecdh-nist-p384",
2052 		.cra_module = THIS_MODULE,
2053 	},
2054 };
2055 
2056 static struct kpp_alg curve25519_alg = {
2057 	.set_secret = hpre_curve25519_set_secret,
2058 	.generate_public_key = hpre_curve25519_compute_value,
2059 	.compute_shared_secret = hpre_curve25519_compute_value,
2060 	.max_size = hpre_curve25519_max_size,
2061 	.init = hpre_curve25519_init_tfm,
2062 	.exit = hpre_curve25519_exit_tfm,
2063 	.reqsize = sizeof(struct hpre_asym_request) + HPRE_ALIGN_SZ,
2064 	.base = {
2065 		.cra_ctxsize = sizeof(struct hpre_ctx),
2066 		.cra_priority = HPRE_CRYPTO_ALG_PRI,
2067 		.cra_name = "curve25519",
2068 		.cra_driver_name = "hpre-curve25519",
2069 		.cra_module = THIS_MODULE,
2070 	},
2071 };
2072 
2073 
2074 static int hpre_register_ecdh(void)
2075 {
2076 	int ret;
2077 
2078 	ret = crypto_register_kpp(&ecdh_nist_p192);
2079 	if (ret)
2080 		return ret;
2081 
2082 	ret = crypto_register_kpp(&ecdh_nist_p256);
2083 	if (ret)
2084 		goto unregister_ecdh_p192;
2085 
2086 	ret = crypto_register_kpp(&ecdh_nist_p384);
2087 	if (ret)
2088 		goto unregister_ecdh_p256;
2089 
2090 	return 0;
2091 
2092 unregister_ecdh_p256:
2093 	crypto_unregister_kpp(&ecdh_nist_p256);
2094 unregister_ecdh_p192:
2095 	crypto_unregister_kpp(&ecdh_nist_p192);
2096 	return ret;
2097 }
2098 
2099 static void hpre_unregister_ecdh(void)
2100 {
2101 	crypto_unregister_kpp(&ecdh_nist_p384);
2102 	crypto_unregister_kpp(&ecdh_nist_p256);
2103 	crypto_unregister_kpp(&ecdh_nist_p192);
2104 }
2105 
2106 int hpre_algs_register(struct hisi_qm *qm)
2107 {
2108 	int ret;
2109 
2110 	rsa.base.cra_flags = 0;
2111 	ret = crypto_register_akcipher(&rsa);
2112 	if (ret)
2113 		return ret;
2114 
2115 	ret = crypto_register_kpp(&dh);
2116 	if (ret)
2117 		goto unreg_rsa;
2118 
2119 	if (qm->ver >= QM_HW_V3) {
2120 		ret = hpre_register_ecdh();
2121 		if (ret)
2122 			goto unreg_dh;
2123 		ret = crypto_register_kpp(&curve25519_alg);
2124 		if (ret)
2125 			goto unreg_ecdh;
2126 	}
2127 	return 0;
2128 
2129 unreg_ecdh:
2130 	hpre_unregister_ecdh();
2131 unreg_dh:
2132 	crypto_unregister_kpp(&dh);
2133 unreg_rsa:
2134 	crypto_unregister_akcipher(&rsa);
2135 	return ret;
2136 }
2137 
2138 void hpre_algs_unregister(struct hisi_qm *qm)
2139 {
2140 	if (qm->ver >= QM_HW_V3) {
2141 		crypto_unregister_kpp(&curve25519_alg);
2142 		hpre_unregister_ecdh();
2143 	}
2144 
2145 	crypto_unregister_kpp(&dh);
2146 	crypto_unregister_akcipher(&rsa);
2147 }
2148