1 // SPDX-License-Identifier: GPL-2.0
2 /* Copyright (c) 2019 HiSilicon Limited. */
3 #include <crypto/akcipher.h>
4 #include <crypto/curve25519.h>
5 #include <crypto/dh.h>
6 #include <crypto/ecc_curve.h>
7 #include <crypto/ecdh.h>
8 #include <crypto/rng.h>
9 #include <crypto/internal/akcipher.h>
10 #include <crypto/internal/kpp.h>
11 #include <crypto/internal/rsa.h>
12 #include <crypto/kpp.h>
13 #include <crypto/scatterwalk.h>
14 #include <linux/dma-mapping.h>
15 #include <linux/fips.h>
16 #include <linux/module.h>
17 #include <linux/time.h>
18 #include "hpre.h"
19
20 struct hpre_ctx;
21
22 #define HPRE_CRYPTO_ALG_PRI 1000
23 #define HPRE_ALIGN_SZ 64
24 #define HPRE_BITS_2_BYTES_SHIFT 3
25 #define HPRE_RSA_512BITS_KSZ 64
26 #define HPRE_RSA_1536BITS_KSZ 192
27 #define HPRE_CRT_PRMS 5
28 #define HPRE_CRT_Q 2
29 #define HPRE_CRT_P 3
30 #define HPRE_CRT_INV 4
31 #define HPRE_DH_G_FLAG 0x02
32 #define HPRE_TRY_SEND_TIMES 100
33 #define HPRE_INVLD_REQ_ID (-1)
34
35 #define HPRE_SQE_ALG_BITS 5
36 #define HPRE_SQE_DONE_SHIFT 30
37 #define HPRE_DH_MAX_P_SZ 512
38
39 #define HPRE_DFX_SEC_TO_US 1000000
40 #define HPRE_DFX_US_TO_NS 1000
41
42 /* due to nist p521 */
43 #define HPRE_ECC_MAX_KSZ 66
44
45 /* size in bytes of the n prime */
46 #define HPRE_ECC_NIST_P192_N_SIZE 24
47 #define HPRE_ECC_NIST_P256_N_SIZE 32
48 #define HPRE_ECC_NIST_P384_N_SIZE 48
49
50 /* size in bytes */
51 #define HPRE_ECC_HW256_KSZ_B 32
52 #define HPRE_ECC_HW384_KSZ_B 48
53
54 /* capability register mask of driver */
55 #define HPRE_DRV_RSA_MASK_CAP BIT(0)
56 #define HPRE_DRV_DH_MASK_CAP BIT(1)
57 #define HPRE_DRV_ECDH_MASK_CAP BIT(2)
58 #define HPRE_DRV_X25519_MASK_CAP BIT(5)
59
60 typedef void (*hpre_cb)(struct hpre_ctx *ctx, void *sqe);
61
62 struct hpre_rsa_ctx {
63 /* low address: e--->n */
64 char *pubkey;
65 dma_addr_t dma_pubkey;
66
67 /* low address: d--->n */
68 char *prikey;
69 dma_addr_t dma_prikey;
70
71 /* low address: dq->dp->q->p->qinv */
72 char *crt_prikey;
73 dma_addr_t dma_crt_prikey;
74
75 struct crypto_akcipher *soft_tfm;
76 };
77
78 struct hpre_dh_ctx {
79 /*
80 * If base is g we compute the public key
81 * ya = g^xa mod p; [RFC2631 sec 2.1.1]
82 * else if base if the counterpart public key we
83 * compute the shared secret
84 * ZZ = yb^xa mod p; [RFC2631 sec 2.1.1]
85 * low address: d--->n, please refer to Hisilicon HPRE UM
86 */
87 char *xa_p;
88 dma_addr_t dma_xa_p;
89
90 char *g; /* m */
91 dma_addr_t dma_g;
92 };
93
94 struct hpre_ecdh_ctx {
95 /* low address: p->a->k->b */
96 unsigned char *p;
97 dma_addr_t dma_p;
98
99 /* low address: x->y */
100 unsigned char *g;
101 dma_addr_t dma_g;
102 };
103
104 struct hpre_curve25519_ctx {
105 /* low address: p->a->k */
106 unsigned char *p;
107 dma_addr_t dma_p;
108
109 /* gx coordinate */
110 unsigned char *g;
111 dma_addr_t dma_g;
112 };
113
114 struct hpre_ctx {
115 struct hisi_qp *qp;
116 struct device *dev;
117 struct hpre_asym_request **req_list;
118 struct hpre *hpre;
119 spinlock_t req_lock;
120 unsigned int key_sz;
121 bool crt_g2_mode;
122 struct idr req_idr;
123 union {
124 struct hpre_rsa_ctx rsa;
125 struct hpre_dh_ctx dh;
126 struct hpre_ecdh_ctx ecdh;
127 struct hpre_curve25519_ctx curve25519;
128 };
129 /* for ecc algorithms */
130 unsigned int curve_id;
131 };
132
133 struct hpre_asym_request {
134 char *src;
135 char *dst;
136 struct hpre_sqe req;
137 struct hpre_ctx *ctx;
138 union {
139 struct akcipher_request *rsa;
140 struct kpp_request *dh;
141 struct kpp_request *ecdh;
142 struct kpp_request *curve25519;
143 } areq;
144 int err;
145 int req_id;
146 hpre_cb cb;
147 struct timespec64 req_time;
148 };
149
hpre_align_sz(void)150 static inline unsigned int hpre_align_sz(void)
151 {
152 return ((crypto_dma_align() - 1) | (HPRE_ALIGN_SZ - 1)) + 1;
153 }
154
hpre_align_pd(void)155 static inline unsigned int hpre_align_pd(void)
156 {
157 return (hpre_align_sz() - 1) & ~(crypto_tfm_ctx_alignment() - 1);
158 }
159
hpre_alloc_req_id(struct hpre_ctx * ctx)160 static int hpre_alloc_req_id(struct hpre_ctx *ctx)
161 {
162 unsigned long flags;
163 int id;
164
165 spin_lock_irqsave(&ctx->req_lock, flags);
166 id = idr_alloc(&ctx->req_idr, NULL, 0, ctx->qp->sq_depth, GFP_ATOMIC);
167 spin_unlock_irqrestore(&ctx->req_lock, flags);
168
169 return id;
170 }
171
hpre_free_req_id(struct hpre_ctx * ctx,int req_id)172 static void hpre_free_req_id(struct hpre_ctx *ctx, int req_id)
173 {
174 unsigned long flags;
175
176 spin_lock_irqsave(&ctx->req_lock, flags);
177 idr_remove(&ctx->req_idr, req_id);
178 spin_unlock_irqrestore(&ctx->req_lock, flags);
179 }
180
hpre_add_req_to_ctx(struct hpre_asym_request * hpre_req)181 static int hpre_add_req_to_ctx(struct hpre_asym_request *hpre_req)
182 {
183 struct hpre_ctx *ctx;
184 struct hpre_dfx *dfx;
185 int id;
186
187 ctx = hpre_req->ctx;
188 id = hpre_alloc_req_id(ctx);
189 if (unlikely(id < 0))
190 return -EINVAL;
191
192 ctx->req_list[id] = hpre_req;
193 hpre_req->req_id = id;
194
195 dfx = ctx->hpre->debug.dfx;
196 if (atomic64_read(&dfx[HPRE_OVERTIME_THRHLD].value))
197 ktime_get_ts64(&hpre_req->req_time);
198
199 return id;
200 }
201
hpre_rm_req_from_ctx(struct hpre_asym_request * hpre_req)202 static void hpre_rm_req_from_ctx(struct hpre_asym_request *hpre_req)
203 {
204 struct hpre_ctx *ctx = hpre_req->ctx;
205 int id = hpre_req->req_id;
206
207 if (hpre_req->req_id >= 0) {
208 hpre_req->req_id = HPRE_INVLD_REQ_ID;
209 ctx->req_list[id] = NULL;
210 hpre_free_req_id(ctx, id);
211 }
212 }
213
hpre_get_qp_and_start(u8 type)214 static struct hisi_qp *hpre_get_qp_and_start(u8 type)
215 {
216 struct hisi_qp *qp;
217 int ret;
218
219 qp = hpre_create_qp(type);
220 if (!qp) {
221 pr_err("Can not create hpre qp!\n");
222 return ERR_PTR(-ENODEV);
223 }
224
225 ret = hisi_qm_start_qp(qp, 0);
226 if (ret < 0) {
227 hisi_qm_free_qps(&qp, 1);
228 pci_err(qp->qm->pdev, "Can not start qp!\n");
229 return ERR_PTR(-EINVAL);
230 }
231
232 return qp;
233 }
234
hpre_get_data_dma_addr(struct hpre_asym_request * hpre_req,struct scatterlist * data,unsigned int len,int is_src,dma_addr_t * tmp)235 static int hpre_get_data_dma_addr(struct hpre_asym_request *hpre_req,
236 struct scatterlist *data, unsigned int len,
237 int is_src, dma_addr_t *tmp)
238 {
239 struct device *dev = hpre_req->ctx->dev;
240 enum dma_data_direction dma_dir;
241
242 if (is_src) {
243 hpre_req->src = NULL;
244 dma_dir = DMA_TO_DEVICE;
245 } else {
246 hpre_req->dst = NULL;
247 dma_dir = DMA_FROM_DEVICE;
248 }
249 *tmp = dma_map_single(dev, sg_virt(data), len, dma_dir);
250 if (unlikely(dma_mapping_error(dev, *tmp))) {
251 dev_err(dev, "dma map data err!\n");
252 return -ENOMEM;
253 }
254
255 return 0;
256 }
257
hpre_prepare_dma_buf(struct hpre_asym_request * hpre_req,struct scatterlist * data,unsigned int len,int is_src,dma_addr_t * tmp)258 static int hpre_prepare_dma_buf(struct hpre_asym_request *hpre_req,
259 struct scatterlist *data, unsigned int len,
260 int is_src, dma_addr_t *tmp)
261 {
262 struct hpre_ctx *ctx = hpre_req->ctx;
263 struct device *dev = ctx->dev;
264 void *ptr;
265 int shift;
266
267 shift = ctx->key_sz - len;
268 if (unlikely(shift < 0))
269 return -EINVAL;
270
271 ptr = dma_alloc_coherent(dev, ctx->key_sz, tmp, GFP_ATOMIC);
272 if (unlikely(!ptr))
273 return -ENOMEM;
274
275 if (is_src) {
276 scatterwalk_map_and_copy(ptr + shift, data, 0, len, 0);
277 hpre_req->src = ptr;
278 } else {
279 hpre_req->dst = ptr;
280 }
281
282 return 0;
283 }
284
hpre_hw_data_init(struct hpre_asym_request * hpre_req,struct scatterlist * data,unsigned int len,int is_src,int is_dh)285 static int hpre_hw_data_init(struct hpre_asym_request *hpre_req,
286 struct scatterlist *data, unsigned int len,
287 int is_src, int is_dh)
288 {
289 struct hpre_sqe *msg = &hpre_req->req;
290 struct hpre_ctx *ctx = hpre_req->ctx;
291 dma_addr_t tmp = 0;
292 int ret;
293
294 /* when the data is dh's source, we should format it */
295 if ((sg_is_last(data) && len == ctx->key_sz) &&
296 ((is_dh && !is_src) || !is_dh))
297 ret = hpre_get_data_dma_addr(hpre_req, data, len, is_src, &tmp);
298 else
299 ret = hpre_prepare_dma_buf(hpre_req, data, len, is_src, &tmp);
300
301 if (unlikely(ret))
302 return ret;
303
304 if (is_src)
305 msg->in = cpu_to_le64(tmp);
306 else
307 msg->out = cpu_to_le64(tmp);
308
309 return 0;
310 }
311
hpre_hw_data_clr_all(struct hpre_ctx * ctx,struct hpre_asym_request * req,struct scatterlist * dst,struct scatterlist * src)312 static void hpre_hw_data_clr_all(struct hpre_ctx *ctx,
313 struct hpre_asym_request *req,
314 struct scatterlist *dst,
315 struct scatterlist *src)
316 {
317 struct device *dev = ctx->dev;
318 struct hpre_sqe *sqe = &req->req;
319 dma_addr_t tmp;
320
321 tmp = le64_to_cpu(sqe->in);
322 if (unlikely(dma_mapping_error(dev, tmp)))
323 return;
324
325 if (src) {
326 if (req->src)
327 dma_free_coherent(dev, ctx->key_sz, req->src, tmp);
328 else
329 dma_unmap_single(dev, tmp, ctx->key_sz, DMA_TO_DEVICE);
330 }
331
332 tmp = le64_to_cpu(sqe->out);
333 if (unlikely(dma_mapping_error(dev, tmp)))
334 return;
335
336 if (req->dst) {
337 if (dst)
338 scatterwalk_map_and_copy(req->dst, dst, 0,
339 ctx->key_sz, 1);
340 dma_free_coherent(dev, ctx->key_sz, req->dst, tmp);
341 } else {
342 dma_unmap_single(dev, tmp, ctx->key_sz, DMA_FROM_DEVICE);
343 }
344 }
345
hpre_alg_res_post_hf(struct hpre_ctx * ctx,struct hpre_sqe * sqe,void ** kreq)346 static int hpre_alg_res_post_hf(struct hpre_ctx *ctx, struct hpre_sqe *sqe,
347 void **kreq)
348 {
349 struct hpre_asym_request *req;
350 unsigned int err, done, alg;
351 int id;
352
353 #define HPRE_NO_HW_ERR 0
354 #define HPRE_HW_TASK_DONE 3
355 #define HREE_HW_ERR_MASK GENMASK(10, 0)
356 #define HREE_SQE_DONE_MASK GENMASK(1, 0)
357 #define HREE_ALG_TYPE_MASK GENMASK(4, 0)
358 id = (int)le16_to_cpu(sqe->tag);
359 req = ctx->req_list[id];
360 hpre_rm_req_from_ctx(req);
361 *kreq = req;
362
363 err = (le32_to_cpu(sqe->dw0) >> HPRE_SQE_ALG_BITS) &
364 HREE_HW_ERR_MASK;
365
366 done = (le32_to_cpu(sqe->dw0) >> HPRE_SQE_DONE_SHIFT) &
367 HREE_SQE_DONE_MASK;
368
369 if (likely(err == HPRE_NO_HW_ERR && done == HPRE_HW_TASK_DONE))
370 return 0;
371
372 alg = le32_to_cpu(sqe->dw0) & HREE_ALG_TYPE_MASK;
373 dev_err_ratelimited(ctx->dev, "alg[0x%x] error: done[0x%x], etype[0x%x]\n",
374 alg, done, err);
375
376 return -EINVAL;
377 }
378
hpre_ctx_set(struct hpre_ctx * ctx,struct hisi_qp * qp,int qlen)379 static int hpre_ctx_set(struct hpre_ctx *ctx, struct hisi_qp *qp, int qlen)
380 {
381 struct hpre *hpre;
382
383 if (!ctx || !qp || qlen < 0)
384 return -EINVAL;
385
386 spin_lock_init(&ctx->req_lock);
387 ctx->qp = qp;
388 ctx->dev = &qp->qm->pdev->dev;
389
390 hpre = container_of(ctx->qp->qm, struct hpre, qm);
391 ctx->hpre = hpre;
392 ctx->req_list = kcalloc(qlen, sizeof(void *), GFP_KERNEL);
393 if (!ctx->req_list)
394 return -ENOMEM;
395 ctx->key_sz = 0;
396 ctx->crt_g2_mode = false;
397 idr_init(&ctx->req_idr);
398
399 return 0;
400 }
401
hpre_ctx_clear(struct hpre_ctx * ctx,bool is_clear_all)402 static void hpre_ctx_clear(struct hpre_ctx *ctx, bool is_clear_all)
403 {
404 if (is_clear_all) {
405 idr_destroy(&ctx->req_idr);
406 kfree(ctx->req_list);
407 hisi_qm_free_qps(&ctx->qp, 1);
408 }
409
410 ctx->crt_g2_mode = false;
411 ctx->key_sz = 0;
412 }
413
hpre_is_bd_timeout(struct hpre_asym_request * req,u64 overtime_thrhld)414 static bool hpre_is_bd_timeout(struct hpre_asym_request *req,
415 u64 overtime_thrhld)
416 {
417 struct timespec64 reply_time;
418 u64 time_use_us;
419
420 ktime_get_ts64(&reply_time);
421 time_use_us = (reply_time.tv_sec - req->req_time.tv_sec) *
422 HPRE_DFX_SEC_TO_US +
423 (reply_time.tv_nsec - req->req_time.tv_nsec) /
424 HPRE_DFX_US_TO_NS;
425
426 if (time_use_us <= overtime_thrhld)
427 return false;
428
429 return true;
430 }
431
hpre_dh_cb(struct hpre_ctx * ctx,void * resp)432 static void hpre_dh_cb(struct hpre_ctx *ctx, void *resp)
433 {
434 struct hpre_dfx *dfx = ctx->hpre->debug.dfx;
435 struct hpre_asym_request *req;
436 struct kpp_request *areq;
437 u64 overtime_thrhld;
438 int ret;
439
440 ret = hpre_alg_res_post_hf(ctx, resp, (void **)&req);
441 areq = req->areq.dh;
442 areq->dst_len = ctx->key_sz;
443
444 overtime_thrhld = atomic64_read(&dfx[HPRE_OVERTIME_THRHLD].value);
445 if (overtime_thrhld && hpre_is_bd_timeout(req, overtime_thrhld))
446 atomic64_inc(&dfx[HPRE_OVER_THRHLD_CNT].value);
447
448 hpre_hw_data_clr_all(ctx, req, areq->dst, areq->src);
449 kpp_request_complete(areq, ret);
450 atomic64_inc(&dfx[HPRE_RECV_CNT].value);
451 }
452
hpre_rsa_cb(struct hpre_ctx * ctx,void * resp)453 static void hpre_rsa_cb(struct hpre_ctx *ctx, void *resp)
454 {
455 struct hpre_dfx *dfx = ctx->hpre->debug.dfx;
456 struct hpre_asym_request *req;
457 struct akcipher_request *areq;
458 u64 overtime_thrhld;
459 int ret;
460
461 ret = hpre_alg_res_post_hf(ctx, resp, (void **)&req);
462
463 overtime_thrhld = atomic64_read(&dfx[HPRE_OVERTIME_THRHLD].value);
464 if (overtime_thrhld && hpre_is_bd_timeout(req, overtime_thrhld))
465 atomic64_inc(&dfx[HPRE_OVER_THRHLD_CNT].value);
466
467 areq = req->areq.rsa;
468 areq->dst_len = ctx->key_sz;
469 hpre_hw_data_clr_all(ctx, req, areq->dst, areq->src);
470 akcipher_request_complete(areq, ret);
471 atomic64_inc(&dfx[HPRE_RECV_CNT].value);
472 }
473
hpre_alg_cb(struct hisi_qp * qp,void * resp)474 static void hpre_alg_cb(struct hisi_qp *qp, void *resp)
475 {
476 struct hpre_ctx *ctx = qp->qp_ctx;
477 struct hpre_dfx *dfx = ctx->hpre->debug.dfx;
478 struct hpre_sqe *sqe = resp;
479 struct hpre_asym_request *req = ctx->req_list[le16_to_cpu(sqe->tag)];
480
481 if (unlikely(!req)) {
482 atomic64_inc(&dfx[HPRE_INVALID_REQ_CNT].value);
483 return;
484 }
485
486 req->cb(ctx, resp);
487 }
488
hpre_stop_qp_and_put(struct hisi_qp * qp)489 static void hpre_stop_qp_and_put(struct hisi_qp *qp)
490 {
491 hisi_qm_stop_qp(qp);
492 hisi_qm_free_qps(&qp, 1);
493 }
494
hpre_ctx_init(struct hpre_ctx * ctx,u8 type)495 static int hpre_ctx_init(struct hpre_ctx *ctx, u8 type)
496 {
497 struct hisi_qp *qp;
498 int ret;
499
500 qp = hpre_get_qp_and_start(type);
501 if (IS_ERR(qp))
502 return PTR_ERR(qp);
503
504 qp->qp_ctx = ctx;
505 qp->req_cb = hpre_alg_cb;
506
507 ret = hpre_ctx_set(ctx, qp, qp->sq_depth);
508 if (ret)
509 hpre_stop_qp_and_put(qp);
510
511 return ret;
512 }
513
hpre_msg_request_set(struct hpre_ctx * ctx,void * req,bool is_rsa)514 static int hpre_msg_request_set(struct hpre_ctx *ctx, void *req, bool is_rsa)
515 {
516 struct hpre_asym_request *h_req;
517 struct hpre_sqe *msg;
518 int req_id;
519 void *tmp;
520
521 if (is_rsa) {
522 struct akcipher_request *akreq = req;
523
524 if (akreq->dst_len < ctx->key_sz) {
525 akreq->dst_len = ctx->key_sz;
526 return -EOVERFLOW;
527 }
528
529 tmp = akcipher_request_ctx(akreq);
530 h_req = PTR_ALIGN(tmp, hpre_align_sz());
531 h_req->cb = hpre_rsa_cb;
532 h_req->areq.rsa = akreq;
533 msg = &h_req->req;
534 memset(msg, 0, sizeof(*msg));
535 } else {
536 struct kpp_request *kreq = req;
537
538 if (kreq->dst_len < ctx->key_sz) {
539 kreq->dst_len = ctx->key_sz;
540 return -EOVERFLOW;
541 }
542
543 tmp = kpp_request_ctx(kreq);
544 h_req = PTR_ALIGN(tmp, hpre_align_sz());
545 h_req->cb = hpre_dh_cb;
546 h_req->areq.dh = kreq;
547 msg = &h_req->req;
548 memset(msg, 0, sizeof(*msg));
549 msg->key = cpu_to_le64(ctx->dh.dma_xa_p);
550 }
551
552 msg->in = cpu_to_le64(DMA_MAPPING_ERROR);
553 msg->out = cpu_to_le64(DMA_MAPPING_ERROR);
554 msg->dw0 |= cpu_to_le32(0x1 << HPRE_SQE_DONE_SHIFT);
555 msg->task_len1 = (ctx->key_sz >> HPRE_BITS_2_BYTES_SHIFT) - 1;
556 h_req->ctx = ctx;
557
558 req_id = hpre_add_req_to_ctx(h_req);
559 if (req_id < 0)
560 return -EBUSY;
561
562 msg->tag = cpu_to_le16((u16)req_id);
563
564 return 0;
565 }
566
hpre_send(struct hpre_ctx * ctx,struct hpre_sqe * msg)567 static int hpre_send(struct hpre_ctx *ctx, struct hpre_sqe *msg)
568 {
569 struct hpre_dfx *dfx = ctx->hpre->debug.dfx;
570 int ctr = 0;
571 int ret;
572
573 do {
574 atomic64_inc(&dfx[HPRE_SEND_CNT].value);
575 ret = hisi_qp_send(ctx->qp, msg);
576 if (ret != -EBUSY)
577 break;
578 atomic64_inc(&dfx[HPRE_SEND_BUSY_CNT].value);
579 } while (ctr++ < HPRE_TRY_SEND_TIMES);
580
581 if (likely(!ret))
582 return ret;
583
584 if (ret != -EBUSY)
585 atomic64_inc(&dfx[HPRE_SEND_FAIL_CNT].value);
586
587 return ret;
588 }
589
hpre_dh_compute_value(struct kpp_request * req)590 static int hpre_dh_compute_value(struct kpp_request *req)
591 {
592 struct crypto_kpp *tfm = crypto_kpp_reqtfm(req);
593 struct hpre_ctx *ctx = kpp_tfm_ctx(tfm);
594 void *tmp = kpp_request_ctx(req);
595 struct hpre_asym_request *hpre_req = PTR_ALIGN(tmp, hpre_align_sz());
596 struct hpre_sqe *msg = &hpre_req->req;
597 int ret;
598
599 ret = hpre_msg_request_set(ctx, req, false);
600 if (unlikely(ret))
601 return ret;
602
603 if (req->src) {
604 ret = hpre_hw_data_init(hpre_req, req->src, req->src_len, 1, 1);
605 if (unlikely(ret))
606 goto clear_all;
607 } else {
608 msg->in = cpu_to_le64(ctx->dh.dma_g);
609 }
610
611 ret = hpre_hw_data_init(hpre_req, req->dst, req->dst_len, 0, 1);
612 if (unlikely(ret))
613 goto clear_all;
614
615 if (ctx->crt_g2_mode && !req->src)
616 msg->dw0 = cpu_to_le32(le32_to_cpu(msg->dw0) | HPRE_ALG_DH_G2);
617 else
618 msg->dw0 = cpu_to_le32(le32_to_cpu(msg->dw0) | HPRE_ALG_DH);
619
620 /* success */
621 ret = hpre_send(ctx, msg);
622 if (likely(!ret))
623 return -EINPROGRESS;
624
625 clear_all:
626 hpre_rm_req_from_ctx(hpre_req);
627 hpre_hw_data_clr_all(ctx, hpre_req, req->dst, req->src);
628
629 return ret;
630 }
631
hpre_is_dh_params_length_valid(unsigned int key_sz)632 static int hpre_is_dh_params_length_valid(unsigned int key_sz)
633 {
634 #define _HPRE_DH_GRP1 768
635 #define _HPRE_DH_GRP2 1024
636 #define _HPRE_DH_GRP5 1536
637 #define _HPRE_DH_GRP14 2048
638 #define _HPRE_DH_GRP15 3072
639 #define _HPRE_DH_GRP16 4096
640 switch (key_sz) {
641 case _HPRE_DH_GRP1:
642 case _HPRE_DH_GRP2:
643 case _HPRE_DH_GRP5:
644 case _HPRE_DH_GRP14:
645 case _HPRE_DH_GRP15:
646 case _HPRE_DH_GRP16:
647 return 0;
648 default:
649 return -EINVAL;
650 }
651 }
652
hpre_dh_set_params(struct hpre_ctx * ctx,struct dh * params)653 static int hpre_dh_set_params(struct hpre_ctx *ctx, struct dh *params)
654 {
655 struct device *dev = ctx->dev;
656 unsigned int sz;
657
658 if (params->p_size > HPRE_DH_MAX_P_SZ)
659 return -EINVAL;
660
661 if (hpre_is_dh_params_length_valid(params->p_size <<
662 HPRE_BITS_2_BYTES_SHIFT))
663 return -EINVAL;
664
665 sz = ctx->key_sz = params->p_size;
666 ctx->dh.xa_p = dma_alloc_coherent(dev, sz << 1,
667 &ctx->dh.dma_xa_p, GFP_KERNEL);
668 if (!ctx->dh.xa_p)
669 return -ENOMEM;
670
671 memcpy(ctx->dh.xa_p + sz, params->p, sz);
672
673 /* If g equals 2 don't copy it */
674 if (params->g_size == 1 && *(char *)params->g == HPRE_DH_G_FLAG) {
675 ctx->crt_g2_mode = true;
676 return 0;
677 }
678
679 ctx->dh.g = dma_alloc_coherent(dev, sz, &ctx->dh.dma_g, GFP_KERNEL);
680 if (!ctx->dh.g) {
681 dma_free_coherent(dev, sz << 1, ctx->dh.xa_p,
682 ctx->dh.dma_xa_p);
683 ctx->dh.xa_p = NULL;
684 return -ENOMEM;
685 }
686
687 memcpy(ctx->dh.g + (sz - params->g_size), params->g, params->g_size);
688
689 return 0;
690 }
691
hpre_dh_clear_ctx(struct hpre_ctx * ctx,bool is_clear_all)692 static void hpre_dh_clear_ctx(struct hpre_ctx *ctx, bool is_clear_all)
693 {
694 struct device *dev = ctx->dev;
695 unsigned int sz = ctx->key_sz;
696
697 if (is_clear_all)
698 hisi_qm_stop_qp(ctx->qp);
699
700 if (ctx->dh.g) {
701 dma_free_coherent(dev, sz, ctx->dh.g, ctx->dh.dma_g);
702 ctx->dh.g = NULL;
703 }
704
705 if (ctx->dh.xa_p) {
706 memzero_explicit(ctx->dh.xa_p, sz);
707 dma_free_coherent(dev, sz << 1, ctx->dh.xa_p,
708 ctx->dh.dma_xa_p);
709 ctx->dh.xa_p = NULL;
710 }
711
712 hpre_ctx_clear(ctx, is_clear_all);
713 }
714
hpre_dh_set_secret(struct crypto_kpp * tfm,const void * buf,unsigned int len)715 static int hpre_dh_set_secret(struct crypto_kpp *tfm, const void *buf,
716 unsigned int len)
717 {
718 struct hpre_ctx *ctx = kpp_tfm_ctx(tfm);
719 struct dh params;
720 int ret;
721
722 if (crypto_dh_decode_key(buf, len, ¶ms) < 0)
723 return -EINVAL;
724
725 /* Free old secret if any */
726 hpre_dh_clear_ctx(ctx, false);
727
728 ret = hpre_dh_set_params(ctx, ¶ms);
729 if (ret < 0)
730 goto err_clear_ctx;
731
732 memcpy(ctx->dh.xa_p + (ctx->key_sz - params.key_size), params.key,
733 params.key_size);
734
735 return 0;
736
737 err_clear_ctx:
738 hpre_dh_clear_ctx(ctx, false);
739 return ret;
740 }
741
hpre_dh_max_size(struct crypto_kpp * tfm)742 static unsigned int hpre_dh_max_size(struct crypto_kpp *tfm)
743 {
744 struct hpre_ctx *ctx = kpp_tfm_ctx(tfm);
745
746 return ctx->key_sz;
747 }
748
hpre_dh_init_tfm(struct crypto_kpp * tfm)749 static int hpre_dh_init_tfm(struct crypto_kpp *tfm)
750 {
751 struct hpre_ctx *ctx = kpp_tfm_ctx(tfm);
752
753 kpp_set_reqsize(tfm, sizeof(struct hpre_asym_request) + hpre_align_pd());
754
755 return hpre_ctx_init(ctx, HPRE_V2_ALG_TYPE);
756 }
757
hpre_dh_exit_tfm(struct crypto_kpp * tfm)758 static void hpre_dh_exit_tfm(struct crypto_kpp *tfm)
759 {
760 struct hpre_ctx *ctx = kpp_tfm_ctx(tfm);
761
762 hpre_dh_clear_ctx(ctx, true);
763 }
764
hpre_rsa_drop_leading_zeros(const char ** ptr,size_t * len)765 static void hpre_rsa_drop_leading_zeros(const char **ptr, size_t *len)
766 {
767 while (!**ptr && *len) {
768 (*ptr)++;
769 (*len)--;
770 }
771 }
772
hpre_rsa_key_size_is_support(unsigned int len)773 static bool hpre_rsa_key_size_is_support(unsigned int len)
774 {
775 unsigned int bits = len << HPRE_BITS_2_BYTES_SHIFT;
776
777 #define _RSA_1024BITS_KEY_WDTH 1024
778 #define _RSA_2048BITS_KEY_WDTH 2048
779 #define _RSA_3072BITS_KEY_WDTH 3072
780 #define _RSA_4096BITS_KEY_WDTH 4096
781
782 switch (bits) {
783 case _RSA_1024BITS_KEY_WDTH:
784 case _RSA_2048BITS_KEY_WDTH:
785 case _RSA_3072BITS_KEY_WDTH:
786 case _RSA_4096BITS_KEY_WDTH:
787 return true;
788 default:
789 return false;
790 }
791 }
792
hpre_rsa_enc(struct akcipher_request * req)793 static int hpre_rsa_enc(struct akcipher_request *req)
794 {
795 struct crypto_akcipher *tfm = crypto_akcipher_reqtfm(req);
796 struct hpre_ctx *ctx = akcipher_tfm_ctx(tfm);
797 void *tmp = akcipher_request_ctx(req);
798 struct hpre_asym_request *hpre_req = PTR_ALIGN(tmp, hpre_align_sz());
799 struct hpre_sqe *msg = &hpre_req->req;
800 int ret;
801
802 /* For 512 and 1536 bits key size, use soft tfm instead */
803 if (ctx->key_sz == HPRE_RSA_512BITS_KSZ ||
804 ctx->key_sz == HPRE_RSA_1536BITS_KSZ) {
805 akcipher_request_set_tfm(req, ctx->rsa.soft_tfm);
806 ret = crypto_akcipher_encrypt(req);
807 akcipher_request_set_tfm(req, tfm);
808 return ret;
809 }
810
811 if (unlikely(!ctx->rsa.pubkey))
812 return -EINVAL;
813
814 ret = hpre_msg_request_set(ctx, req, true);
815 if (unlikely(ret))
816 return ret;
817
818 msg->dw0 |= cpu_to_le32(HPRE_ALG_NC_NCRT);
819 msg->key = cpu_to_le64(ctx->rsa.dma_pubkey);
820
821 ret = hpre_hw_data_init(hpre_req, req->src, req->src_len, 1, 0);
822 if (unlikely(ret))
823 goto clear_all;
824
825 ret = hpre_hw_data_init(hpre_req, req->dst, req->dst_len, 0, 0);
826 if (unlikely(ret))
827 goto clear_all;
828
829 /* success */
830 ret = hpre_send(ctx, msg);
831 if (likely(!ret))
832 return -EINPROGRESS;
833
834 clear_all:
835 hpre_rm_req_from_ctx(hpre_req);
836 hpre_hw_data_clr_all(ctx, hpre_req, req->dst, req->src);
837
838 return ret;
839 }
840
hpre_rsa_dec(struct akcipher_request * req)841 static int hpre_rsa_dec(struct akcipher_request *req)
842 {
843 struct crypto_akcipher *tfm = crypto_akcipher_reqtfm(req);
844 struct hpre_ctx *ctx = akcipher_tfm_ctx(tfm);
845 void *tmp = akcipher_request_ctx(req);
846 struct hpre_asym_request *hpre_req = PTR_ALIGN(tmp, hpre_align_sz());
847 struct hpre_sqe *msg = &hpre_req->req;
848 int ret;
849
850 /* For 512 and 1536 bits key size, use soft tfm instead */
851 if (ctx->key_sz == HPRE_RSA_512BITS_KSZ ||
852 ctx->key_sz == HPRE_RSA_1536BITS_KSZ) {
853 akcipher_request_set_tfm(req, ctx->rsa.soft_tfm);
854 ret = crypto_akcipher_decrypt(req);
855 akcipher_request_set_tfm(req, tfm);
856 return ret;
857 }
858
859 if (unlikely(!ctx->rsa.prikey))
860 return -EINVAL;
861
862 ret = hpre_msg_request_set(ctx, req, true);
863 if (unlikely(ret))
864 return ret;
865
866 if (ctx->crt_g2_mode) {
867 msg->key = cpu_to_le64(ctx->rsa.dma_crt_prikey);
868 msg->dw0 = cpu_to_le32(le32_to_cpu(msg->dw0) |
869 HPRE_ALG_NC_CRT);
870 } else {
871 msg->key = cpu_to_le64(ctx->rsa.dma_prikey);
872 msg->dw0 = cpu_to_le32(le32_to_cpu(msg->dw0) |
873 HPRE_ALG_NC_NCRT);
874 }
875
876 ret = hpre_hw_data_init(hpre_req, req->src, req->src_len, 1, 0);
877 if (unlikely(ret))
878 goto clear_all;
879
880 ret = hpre_hw_data_init(hpre_req, req->dst, req->dst_len, 0, 0);
881 if (unlikely(ret))
882 goto clear_all;
883
884 /* success */
885 ret = hpre_send(ctx, msg);
886 if (likely(!ret))
887 return -EINPROGRESS;
888
889 clear_all:
890 hpre_rm_req_from_ctx(hpre_req);
891 hpre_hw_data_clr_all(ctx, hpre_req, req->dst, req->src);
892
893 return ret;
894 }
895
hpre_rsa_set_n(struct hpre_ctx * ctx,const char * value,size_t vlen,bool private)896 static int hpre_rsa_set_n(struct hpre_ctx *ctx, const char *value,
897 size_t vlen, bool private)
898 {
899 const char *ptr = value;
900
901 hpre_rsa_drop_leading_zeros(&ptr, &vlen);
902
903 ctx->key_sz = vlen;
904
905 /* if invalid key size provided, we use software tfm */
906 if (!hpre_rsa_key_size_is_support(ctx->key_sz))
907 return 0;
908
909 ctx->rsa.pubkey = dma_alloc_coherent(ctx->dev, vlen << 1,
910 &ctx->rsa.dma_pubkey,
911 GFP_KERNEL);
912 if (!ctx->rsa.pubkey)
913 return -ENOMEM;
914
915 if (private) {
916 ctx->rsa.prikey = dma_alloc_coherent(ctx->dev, vlen << 1,
917 &ctx->rsa.dma_prikey,
918 GFP_KERNEL);
919 if (!ctx->rsa.prikey) {
920 dma_free_coherent(ctx->dev, vlen << 1,
921 ctx->rsa.pubkey,
922 ctx->rsa.dma_pubkey);
923 ctx->rsa.pubkey = NULL;
924 return -ENOMEM;
925 }
926 memcpy(ctx->rsa.prikey + vlen, ptr, vlen);
927 }
928 memcpy(ctx->rsa.pubkey + vlen, ptr, vlen);
929
930 /* Using hardware HPRE to do RSA */
931 return 1;
932 }
933
hpre_rsa_set_e(struct hpre_ctx * ctx,const char * value,size_t vlen)934 static int hpre_rsa_set_e(struct hpre_ctx *ctx, const char *value,
935 size_t vlen)
936 {
937 const char *ptr = value;
938
939 hpre_rsa_drop_leading_zeros(&ptr, &vlen);
940
941 if (!ctx->key_sz || !vlen || vlen > ctx->key_sz)
942 return -EINVAL;
943
944 memcpy(ctx->rsa.pubkey + ctx->key_sz - vlen, ptr, vlen);
945
946 return 0;
947 }
948
hpre_rsa_set_d(struct hpre_ctx * ctx,const char * value,size_t vlen)949 static int hpre_rsa_set_d(struct hpre_ctx *ctx, const char *value,
950 size_t vlen)
951 {
952 const char *ptr = value;
953
954 hpre_rsa_drop_leading_zeros(&ptr, &vlen);
955
956 if (!ctx->key_sz || !vlen || vlen > ctx->key_sz)
957 return -EINVAL;
958
959 memcpy(ctx->rsa.prikey + ctx->key_sz - vlen, ptr, vlen);
960
961 return 0;
962 }
963
hpre_crt_para_get(char * para,size_t para_sz,const char * raw,size_t raw_sz)964 static int hpre_crt_para_get(char *para, size_t para_sz,
965 const char *raw, size_t raw_sz)
966 {
967 const char *ptr = raw;
968 size_t len = raw_sz;
969
970 hpre_rsa_drop_leading_zeros(&ptr, &len);
971 if (!len || len > para_sz)
972 return -EINVAL;
973
974 memcpy(para + para_sz - len, ptr, len);
975
976 return 0;
977 }
978
hpre_rsa_setkey_crt(struct hpre_ctx * ctx,struct rsa_key * rsa_key)979 static int hpre_rsa_setkey_crt(struct hpre_ctx *ctx, struct rsa_key *rsa_key)
980 {
981 unsigned int hlf_ksz = ctx->key_sz >> 1;
982 struct device *dev = ctx->dev;
983 u64 offset;
984 int ret;
985
986 ctx->rsa.crt_prikey = dma_alloc_coherent(dev, hlf_ksz * HPRE_CRT_PRMS,
987 &ctx->rsa.dma_crt_prikey,
988 GFP_KERNEL);
989 if (!ctx->rsa.crt_prikey)
990 return -ENOMEM;
991
992 ret = hpre_crt_para_get(ctx->rsa.crt_prikey, hlf_ksz,
993 rsa_key->dq, rsa_key->dq_sz);
994 if (ret)
995 goto free_key;
996
997 offset = hlf_ksz;
998 ret = hpre_crt_para_get(ctx->rsa.crt_prikey + offset, hlf_ksz,
999 rsa_key->dp, rsa_key->dp_sz);
1000 if (ret)
1001 goto free_key;
1002
1003 offset = hlf_ksz * HPRE_CRT_Q;
1004 ret = hpre_crt_para_get(ctx->rsa.crt_prikey + offset, hlf_ksz,
1005 rsa_key->q, rsa_key->q_sz);
1006 if (ret)
1007 goto free_key;
1008
1009 offset = hlf_ksz * HPRE_CRT_P;
1010 ret = hpre_crt_para_get(ctx->rsa.crt_prikey + offset, hlf_ksz,
1011 rsa_key->p, rsa_key->p_sz);
1012 if (ret)
1013 goto free_key;
1014
1015 offset = hlf_ksz * HPRE_CRT_INV;
1016 ret = hpre_crt_para_get(ctx->rsa.crt_prikey + offset, hlf_ksz,
1017 rsa_key->qinv, rsa_key->qinv_sz);
1018 if (ret)
1019 goto free_key;
1020
1021 ctx->crt_g2_mode = true;
1022
1023 return 0;
1024
1025 free_key:
1026 offset = hlf_ksz * HPRE_CRT_PRMS;
1027 memzero_explicit(ctx->rsa.crt_prikey, offset);
1028 dma_free_coherent(dev, hlf_ksz * HPRE_CRT_PRMS, ctx->rsa.crt_prikey,
1029 ctx->rsa.dma_crt_prikey);
1030 ctx->rsa.crt_prikey = NULL;
1031 ctx->crt_g2_mode = false;
1032
1033 return ret;
1034 }
1035
1036 /* If it is clear all, all the resources of the QP will be cleaned. */
hpre_rsa_clear_ctx(struct hpre_ctx * ctx,bool is_clear_all)1037 static void hpre_rsa_clear_ctx(struct hpre_ctx *ctx, bool is_clear_all)
1038 {
1039 unsigned int half_key_sz = ctx->key_sz >> 1;
1040 struct device *dev = ctx->dev;
1041
1042 if (is_clear_all)
1043 hisi_qm_stop_qp(ctx->qp);
1044
1045 if (ctx->rsa.pubkey) {
1046 dma_free_coherent(dev, ctx->key_sz << 1,
1047 ctx->rsa.pubkey, ctx->rsa.dma_pubkey);
1048 ctx->rsa.pubkey = NULL;
1049 }
1050
1051 if (ctx->rsa.crt_prikey) {
1052 memzero_explicit(ctx->rsa.crt_prikey,
1053 half_key_sz * HPRE_CRT_PRMS);
1054 dma_free_coherent(dev, half_key_sz * HPRE_CRT_PRMS,
1055 ctx->rsa.crt_prikey, ctx->rsa.dma_crt_prikey);
1056 ctx->rsa.crt_prikey = NULL;
1057 }
1058
1059 if (ctx->rsa.prikey) {
1060 memzero_explicit(ctx->rsa.prikey, ctx->key_sz);
1061 dma_free_coherent(dev, ctx->key_sz << 1, ctx->rsa.prikey,
1062 ctx->rsa.dma_prikey);
1063 ctx->rsa.prikey = NULL;
1064 }
1065
1066 hpre_ctx_clear(ctx, is_clear_all);
1067 }
1068
1069 /*
1070 * we should judge if it is CRT or not,
1071 * CRT: return true, N-CRT: return false .
1072 */
hpre_is_crt_key(struct rsa_key * key)1073 static bool hpre_is_crt_key(struct rsa_key *key)
1074 {
1075 u16 len = key->p_sz + key->q_sz + key->dp_sz + key->dq_sz +
1076 key->qinv_sz;
1077
1078 #define LEN_OF_NCRT_PARA 5
1079
1080 /* N-CRT less than 5 parameters */
1081 return len > LEN_OF_NCRT_PARA;
1082 }
1083
hpre_rsa_setkey(struct hpre_ctx * ctx,const void * key,unsigned int keylen,bool private)1084 static int hpre_rsa_setkey(struct hpre_ctx *ctx, const void *key,
1085 unsigned int keylen, bool private)
1086 {
1087 struct rsa_key rsa_key;
1088 int ret;
1089
1090 hpre_rsa_clear_ctx(ctx, false);
1091
1092 if (private)
1093 ret = rsa_parse_priv_key(&rsa_key, key, keylen);
1094 else
1095 ret = rsa_parse_pub_key(&rsa_key, key, keylen);
1096 if (ret < 0)
1097 return ret;
1098
1099 ret = hpre_rsa_set_n(ctx, rsa_key.n, rsa_key.n_sz, private);
1100 if (ret <= 0)
1101 return ret;
1102
1103 if (private) {
1104 ret = hpre_rsa_set_d(ctx, rsa_key.d, rsa_key.d_sz);
1105 if (ret < 0)
1106 goto free;
1107
1108 if (hpre_is_crt_key(&rsa_key)) {
1109 ret = hpre_rsa_setkey_crt(ctx, &rsa_key);
1110 if (ret < 0)
1111 goto free;
1112 }
1113 }
1114
1115 ret = hpre_rsa_set_e(ctx, rsa_key.e, rsa_key.e_sz);
1116 if (ret < 0)
1117 goto free;
1118
1119 if ((private && !ctx->rsa.prikey) || !ctx->rsa.pubkey) {
1120 ret = -EINVAL;
1121 goto free;
1122 }
1123
1124 return 0;
1125
1126 free:
1127 hpre_rsa_clear_ctx(ctx, false);
1128 return ret;
1129 }
1130
hpre_rsa_setpubkey(struct crypto_akcipher * tfm,const void * key,unsigned int keylen)1131 static int hpre_rsa_setpubkey(struct crypto_akcipher *tfm, const void *key,
1132 unsigned int keylen)
1133 {
1134 struct hpre_ctx *ctx = akcipher_tfm_ctx(tfm);
1135 int ret;
1136
1137 ret = crypto_akcipher_set_pub_key(ctx->rsa.soft_tfm, key, keylen);
1138 if (ret)
1139 return ret;
1140
1141 return hpre_rsa_setkey(ctx, key, keylen, false);
1142 }
1143
hpre_rsa_setprivkey(struct crypto_akcipher * tfm,const void * key,unsigned int keylen)1144 static int hpre_rsa_setprivkey(struct crypto_akcipher *tfm, const void *key,
1145 unsigned int keylen)
1146 {
1147 struct hpre_ctx *ctx = akcipher_tfm_ctx(tfm);
1148 int ret;
1149
1150 ret = crypto_akcipher_set_priv_key(ctx->rsa.soft_tfm, key, keylen);
1151 if (ret)
1152 return ret;
1153
1154 return hpre_rsa_setkey(ctx, key, keylen, true);
1155 }
1156
hpre_rsa_max_size(struct crypto_akcipher * tfm)1157 static unsigned int hpre_rsa_max_size(struct crypto_akcipher *tfm)
1158 {
1159 struct hpre_ctx *ctx = akcipher_tfm_ctx(tfm);
1160
1161 /* For 512 and 1536 bits key size, use soft tfm instead */
1162 if (ctx->key_sz == HPRE_RSA_512BITS_KSZ ||
1163 ctx->key_sz == HPRE_RSA_1536BITS_KSZ)
1164 return crypto_akcipher_maxsize(ctx->rsa.soft_tfm);
1165
1166 return ctx->key_sz;
1167 }
1168
hpre_rsa_init_tfm(struct crypto_akcipher * tfm)1169 static int hpre_rsa_init_tfm(struct crypto_akcipher *tfm)
1170 {
1171 struct hpre_ctx *ctx = akcipher_tfm_ctx(tfm);
1172 int ret;
1173
1174 ctx->rsa.soft_tfm = crypto_alloc_akcipher("rsa-generic", 0, 0);
1175 if (IS_ERR(ctx->rsa.soft_tfm)) {
1176 pr_err("Can not alloc_akcipher!\n");
1177 return PTR_ERR(ctx->rsa.soft_tfm);
1178 }
1179
1180 akcipher_set_reqsize(tfm, sizeof(struct hpre_asym_request) +
1181 hpre_align_pd());
1182
1183 ret = hpre_ctx_init(ctx, HPRE_V2_ALG_TYPE);
1184 if (ret)
1185 crypto_free_akcipher(ctx->rsa.soft_tfm);
1186
1187 return ret;
1188 }
1189
hpre_rsa_exit_tfm(struct crypto_akcipher * tfm)1190 static void hpre_rsa_exit_tfm(struct crypto_akcipher *tfm)
1191 {
1192 struct hpre_ctx *ctx = akcipher_tfm_ctx(tfm);
1193
1194 hpre_rsa_clear_ctx(ctx, true);
1195 crypto_free_akcipher(ctx->rsa.soft_tfm);
1196 }
1197
hpre_key_to_big_end(u8 * data,int len)1198 static void hpre_key_to_big_end(u8 *data, int len)
1199 {
1200 int i, j;
1201
1202 for (i = 0; i < len / 2; i++) {
1203 j = len - i - 1;
1204 swap(data[j], data[i]);
1205 }
1206 }
1207
hpre_ecc_clear_ctx(struct hpre_ctx * ctx,bool is_clear_all,bool is_ecdh)1208 static void hpre_ecc_clear_ctx(struct hpre_ctx *ctx, bool is_clear_all,
1209 bool is_ecdh)
1210 {
1211 struct device *dev = ctx->dev;
1212 unsigned int sz = ctx->key_sz;
1213 unsigned int shift = sz << 1;
1214
1215 if (is_clear_all)
1216 hisi_qm_stop_qp(ctx->qp);
1217
1218 if (is_ecdh && ctx->ecdh.p) {
1219 /* ecdh: p->a->k->b */
1220 memzero_explicit(ctx->ecdh.p + shift, sz);
1221 dma_free_coherent(dev, sz << 3, ctx->ecdh.p, ctx->ecdh.dma_p);
1222 ctx->ecdh.p = NULL;
1223 } else if (!is_ecdh && ctx->curve25519.p) {
1224 /* curve25519: p->a->k */
1225 memzero_explicit(ctx->curve25519.p + shift, sz);
1226 dma_free_coherent(dev, sz << 2, ctx->curve25519.p,
1227 ctx->curve25519.dma_p);
1228 ctx->curve25519.p = NULL;
1229 }
1230
1231 hpre_ctx_clear(ctx, is_clear_all);
1232 }
1233
1234 /*
1235 * The bits of 192/224/256/384/521 are supported by HPRE,
1236 * and convert the bits like:
1237 * bits<=256, bits=256; 256<bits<=384, bits=384; 384<bits<=576, bits=576;
1238 * If the parameter bit width is insufficient, then we fill in the
1239 * high-order zeros by soft, so TASK_LENGTH1 is 0x3/0x5/0x8;
1240 */
hpre_ecdh_supported_curve(unsigned short id)1241 static unsigned int hpre_ecdh_supported_curve(unsigned short id)
1242 {
1243 switch (id) {
1244 case ECC_CURVE_NIST_P192:
1245 case ECC_CURVE_NIST_P256:
1246 return HPRE_ECC_HW256_KSZ_B;
1247 case ECC_CURVE_NIST_P384:
1248 return HPRE_ECC_HW384_KSZ_B;
1249 default:
1250 break;
1251 }
1252
1253 return 0;
1254 }
1255
fill_curve_param(void * addr,u64 * param,unsigned int cur_sz,u8 ndigits)1256 static void fill_curve_param(void *addr, u64 *param, unsigned int cur_sz, u8 ndigits)
1257 {
1258 unsigned int sz = cur_sz - (ndigits - 1) * sizeof(u64);
1259 u8 i = 0;
1260
1261 while (i < ndigits - 1) {
1262 memcpy(addr + sizeof(u64) * i, ¶m[i], sizeof(u64));
1263 i++;
1264 }
1265
1266 memcpy(addr + sizeof(u64) * i, ¶m[ndigits - 1], sz);
1267 hpre_key_to_big_end((u8 *)addr, cur_sz);
1268 }
1269
hpre_ecdh_fill_curve(struct hpre_ctx * ctx,struct ecdh * params,unsigned int cur_sz)1270 static int hpre_ecdh_fill_curve(struct hpre_ctx *ctx, struct ecdh *params,
1271 unsigned int cur_sz)
1272 {
1273 unsigned int shifta = ctx->key_sz << 1;
1274 unsigned int shiftb = ctx->key_sz << 2;
1275 void *p = ctx->ecdh.p + ctx->key_sz - cur_sz;
1276 void *a = ctx->ecdh.p + shifta - cur_sz;
1277 void *b = ctx->ecdh.p + shiftb - cur_sz;
1278 void *x = ctx->ecdh.g + ctx->key_sz - cur_sz;
1279 void *y = ctx->ecdh.g + shifta - cur_sz;
1280 const struct ecc_curve *curve = ecc_get_curve(ctx->curve_id);
1281 char *n;
1282
1283 if (unlikely(!curve))
1284 return -EINVAL;
1285
1286 n = kzalloc(ctx->key_sz, GFP_KERNEL);
1287 if (!n)
1288 return -ENOMEM;
1289
1290 fill_curve_param(p, curve->p, cur_sz, curve->g.ndigits);
1291 fill_curve_param(a, curve->a, cur_sz, curve->g.ndigits);
1292 fill_curve_param(b, curve->b, cur_sz, curve->g.ndigits);
1293 fill_curve_param(x, curve->g.x, cur_sz, curve->g.ndigits);
1294 fill_curve_param(y, curve->g.y, cur_sz, curve->g.ndigits);
1295 fill_curve_param(n, curve->n, cur_sz, curve->g.ndigits);
1296
1297 if (params->key_size == cur_sz && memcmp(params->key, n, cur_sz) >= 0) {
1298 kfree(n);
1299 return -EINVAL;
1300 }
1301
1302 kfree(n);
1303 return 0;
1304 }
1305
hpre_ecdh_get_curvesz(unsigned short id)1306 static unsigned int hpre_ecdh_get_curvesz(unsigned short id)
1307 {
1308 switch (id) {
1309 case ECC_CURVE_NIST_P192:
1310 return HPRE_ECC_NIST_P192_N_SIZE;
1311 case ECC_CURVE_NIST_P256:
1312 return HPRE_ECC_NIST_P256_N_SIZE;
1313 case ECC_CURVE_NIST_P384:
1314 return HPRE_ECC_NIST_P384_N_SIZE;
1315 default:
1316 break;
1317 }
1318
1319 return 0;
1320 }
1321
hpre_ecdh_set_param(struct hpre_ctx * ctx,struct ecdh * params)1322 static int hpre_ecdh_set_param(struct hpre_ctx *ctx, struct ecdh *params)
1323 {
1324 struct device *dev = ctx->dev;
1325 unsigned int sz, shift, curve_sz;
1326 int ret;
1327
1328 ctx->key_sz = hpre_ecdh_supported_curve(ctx->curve_id);
1329 if (!ctx->key_sz)
1330 return -EINVAL;
1331
1332 curve_sz = hpre_ecdh_get_curvesz(ctx->curve_id);
1333 if (!curve_sz || params->key_size > curve_sz)
1334 return -EINVAL;
1335
1336 sz = ctx->key_sz;
1337
1338 if (!ctx->ecdh.p) {
1339 ctx->ecdh.p = dma_alloc_coherent(dev, sz << 3, &ctx->ecdh.dma_p,
1340 GFP_KERNEL);
1341 if (!ctx->ecdh.p)
1342 return -ENOMEM;
1343 }
1344
1345 shift = sz << 2;
1346 ctx->ecdh.g = ctx->ecdh.p + shift;
1347 ctx->ecdh.dma_g = ctx->ecdh.dma_p + shift;
1348
1349 ret = hpre_ecdh_fill_curve(ctx, params, curve_sz);
1350 if (ret) {
1351 dev_err(dev, "failed to fill curve_param, ret = %d!\n", ret);
1352 dma_free_coherent(dev, sz << 3, ctx->ecdh.p, ctx->ecdh.dma_p);
1353 ctx->ecdh.p = NULL;
1354 return ret;
1355 }
1356
1357 return 0;
1358 }
1359
hpre_key_is_zero(char * key,unsigned short key_sz)1360 static bool hpre_key_is_zero(char *key, unsigned short key_sz)
1361 {
1362 int i;
1363
1364 for (i = 0; i < key_sz; i++)
1365 if (key[i])
1366 return false;
1367
1368 return true;
1369 }
1370
ecdh_gen_privkey(struct hpre_ctx * ctx,struct ecdh * params)1371 static int ecdh_gen_privkey(struct hpre_ctx *ctx, struct ecdh *params)
1372 {
1373 struct device *dev = ctx->dev;
1374 int ret;
1375
1376 ret = crypto_get_default_rng();
1377 if (ret) {
1378 dev_err(dev, "failed to get default rng, ret = %d!\n", ret);
1379 return ret;
1380 }
1381
1382 ret = crypto_rng_get_bytes(crypto_default_rng, (u8 *)params->key,
1383 params->key_size);
1384 crypto_put_default_rng();
1385 if (ret)
1386 dev_err(dev, "failed to get rng, ret = %d!\n", ret);
1387
1388 return ret;
1389 }
1390
hpre_ecdh_set_secret(struct crypto_kpp * tfm,const void * buf,unsigned int len)1391 static int hpre_ecdh_set_secret(struct crypto_kpp *tfm, const void *buf,
1392 unsigned int len)
1393 {
1394 struct hpre_ctx *ctx = kpp_tfm_ctx(tfm);
1395 unsigned int sz, sz_shift, curve_sz;
1396 struct device *dev = ctx->dev;
1397 char key[HPRE_ECC_MAX_KSZ];
1398 struct ecdh params;
1399 int ret;
1400
1401 if (crypto_ecdh_decode_key(buf, len, ¶ms) < 0) {
1402 dev_err(dev, "failed to decode ecdh key!\n");
1403 return -EINVAL;
1404 }
1405
1406 /* Use stdrng to generate private key */
1407 if (!params.key || !params.key_size) {
1408 params.key = key;
1409 curve_sz = hpre_ecdh_get_curvesz(ctx->curve_id);
1410 if (!curve_sz) {
1411 dev_err(dev, "Invalid curve size!\n");
1412 return -EINVAL;
1413 }
1414
1415 params.key_size = curve_sz - 1;
1416 ret = ecdh_gen_privkey(ctx, ¶ms);
1417 if (ret)
1418 return ret;
1419 }
1420
1421 if (hpre_key_is_zero(params.key, params.key_size)) {
1422 dev_err(dev, "Invalid hpre key!\n");
1423 return -EINVAL;
1424 }
1425
1426 hpre_ecc_clear_ctx(ctx, false, true);
1427
1428 ret = hpre_ecdh_set_param(ctx, ¶ms);
1429 if (ret < 0) {
1430 dev_err(dev, "failed to set hpre param, ret = %d!\n", ret);
1431 return ret;
1432 }
1433
1434 sz = ctx->key_sz;
1435 sz_shift = (sz << 1) + sz - params.key_size;
1436 memcpy(ctx->ecdh.p + sz_shift, params.key, params.key_size);
1437
1438 return 0;
1439 }
1440
hpre_ecdh_hw_data_clr_all(struct hpre_ctx * ctx,struct hpre_asym_request * req,struct scatterlist * dst,struct scatterlist * src)1441 static void hpre_ecdh_hw_data_clr_all(struct hpre_ctx *ctx,
1442 struct hpre_asym_request *req,
1443 struct scatterlist *dst,
1444 struct scatterlist *src)
1445 {
1446 struct device *dev = ctx->dev;
1447 struct hpre_sqe *sqe = &req->req;
1448 dma_addr_t dma;
1449
1450 dma = le64_to_cpu(sqe->in);
1451 if (unlikely(dma_mapping_error(dev, dma)))
1452 return;
1453
1454 if (src && req->src)
1455 dma_free_coherent(dev, ctx->key_sz << 2, req->src, dma);
1456
1457 dma = le64_to_cpu(sqe->out);
1458 if (unlikely(dma_mapping_error(dev, dma)))
1459 return;
1460
1461 if (req->dst)
1462 dma_free_coherent(dev, ctx->key_sz << 1, req->dst, dma);
1463 if (dst)
1464 dma_unmap_single(dev, dma, ctx->key_sz << 1, DMA_FROM_DEVICE);
1465 }
1466
hpre_ecdh_cb(struct hpre_ctx * ctx,void * resp)1467 static void hpre_ecdh_cb(struct hpre_ctx *ctx, void *resp)
1468 {
1469 unsigned int curve_sz = hpre_ecdh_get_curvesz(ctx->curve_id);
1470 struct hpre_dfx *dfx = ctx->hpre->debug.dfx;
1471 struct hpre_asym_request *req = NULL;
1472 struct kpp_request *areq;
1473 u64 overtime_thrhld;
1474 char *p;
1475 int ret;
1476
1477 ret = hpre_alg_res_post_hf(ctx, resp, (void **)&req);
1478 areq = req->areq.ecdh;
1479 areq->dst_len = ctx->key_sz << 1;
1480
1481 overtime_thrhld = atomic64_read(&dfx[HPRE_OVERTIME_THRHLD].value);
1482 if (overtime_thrhld && hpre_is_bd_timeout(req, overtime_thrhld))
1483 atomic64_inc(&dfx[HPRE_OVER_THRHLD_CNT].value);
1484
1485 p = sg_virt(areq->dst);
1486 memmove(p, p + ctx->key_sz - curve_sz, curve_sz);
1487 memmove(p + curve_sz, p + areq->dst_len - curve_sz, curve_sz);
1488
1489 hpre_ecdh_hw_data_clr_all(ctx, req, areq->dst, areq->src);
1490 kpp_request_complete(areq, ret);
1491
1492 atomic64_inc(&dfx[HPRE_RECV_CNT].value);
1493 }
1494
hpre_ecdh_msg_request_set(struct hpre_ctx * ctx,struct kpp_request * req)1495 static int hpre_ecdh_msg_request_set(struct hpre_ctx *ctx,
1496 struct kpp_request *req)
1497 {
1498 struct hpre_asym_request *h_req;
1499 struct hpre_sqe *msg;
1500 int req_id;
1501 void *tmp;
1502
1503 if (req->dst_len < ctx->key_sz << 1) {
1504 req->dst_len = ctx->key_sz << 1;
1505 return -EINVAL;
1506 }
1507
1508 tmp = kpp_request_ctx(req);
1509 h_req = PTR_ALIGN(tmp, hpre_align_sz());
1510 h_req->cb = hpre_ecdh_cb;
1511 h_req->areq.ecdh = req;
1512 msg = &h_req->req;
1513 memset(msg, 0, sizeof(*msg));
1514 msg->in = cpu_to_le64(DMA_MAPPING_ERROR);
1515 msg->out = cpu_to_le64(DMA_MAPPING_ERROR);
1516 msg->key = cpu_to_le64(ctx->ecdh.dma_p);
1517
1518 msg->dw0 |= cpu_to_le32(0x1U << HPRE_SQE_DONE_SHIFT);
1519 msg->task_len1 = (ctx->key_sz >> HPRE_BITS_2_BYTES_SHIFT) - 1;
1520 h_req->ctx = ctx;
1521
1522 req_id = hpre_add_req_to_ctx(h_req);
1523 if (req_id < 0)
1524 return -EBUSY;
1525
1526 msg->tag = cpu_to_le16((u16)req_id);
1527 return 0;
1528 }
1529
hpre_ecdh_src_data_init(struct hpre_asym_request * hpre_req,struct scatterlist * data,unsigned int len)1530 static int hpre_ecdh_src_data_init(struct hpre_asym_request *hpre_req,
1531 struct scatterlist *data, unsigned int len)
1532 {
1533 struct hpre_sqe *msg = &hpre_req->req;
1534 struct hpre_ctx *ctx = hpre_req->ctx;
1535 struct device *dev = ctx->dev;
1536 unsigned int tmpshift;
1537 dma_addr_t dma = 0;
1538 void *ptr;
1539 int shift;
1540
1541 /* Src_data include gx and gy. */
1542 shift = ctx->key_sz - (len >> 1);
1543 if (unlikely(shift < 0))
1544 return -EINVAL;
1545
1546 ptr = dma_alloc_coherent(dev, ctx->key_sz << 2, &dma, GFP_KERNEL);
1547 if (unlikely(!ptr))
1548 return -ENOMEM;
1549
1550 tmpshift = ctx->key_sz << 1;
1551 scatterwalk_map_and_copy(ptr + tmpshift, data, 0, len, 0);
1552 memcpy(ptr + shift, ptr + tmpshift, len >> 1);
1553 memcpy(ptr + ctx->key_sz + shift, ptr + tmpshift + (len >> 1), len >> 1);
1554
1555 hpre_req->src = ptr;
1556 msg->in = cpu_to_le64(dma);
1557 return 0;
1558 }
1559
hpre_ecdh_dst_data_init(struct hpre_asym_request * hpre_req,struct scatterlist * data,unsigned int len)1560 static int hpre_ecdh_dst_data_init(struct hpre_asym_request *hpre_req,
1561 struct scatterlist *data, unsigned int len)
1562 {
1563 struct hpre_sqe *msg = &hpre_req->req;
1564 struct hpre_ctx *ctx = hpre_req->ctx;
1565 struct device *dev = ctx->dev;
1566 dma_addr_t dma;
1567
1568 if (unlikely(!data || !sg_is_last(data) || len != ctx->key_sz << 1)) {
1569 dev_err(dev, "data or data length is illegal!\n");
1570 return -EINVAL;
1571 }
1572
1573 hpre_req->dst = NULL;
1574 dma = dma_map_single(dev, sg_virt(data), len, DMA_FROM_DEVICE);
1575 if (unlikely(dma_mapping_error(dev, dma))) {
1576 dev_err(dev, "dma map data err!\n");
1577 return -ENOMEM;
1578 }
1579
1580 msg->out = cpu_to_le64(dma);
1581 return 0;
1582 }
1583
hpre_ecdh_compute_value(struct kpp_request * req)1584 static int hpre_ecdh_compute_value(struct kpp_request *req)
1585 {
1586 struct crypto_kpp *tfm = crypto_kpp_reqtfm(req);
1587 struct hpre_ctx *ctx = kpp_tfm_ctx(tfm);
1588 struct device *dev = ctx->dev;
1589 void *tmp = kpp_request_ctx(req);
1590 struct hpre_asym_request *hpre_req = PTR_ALIGN(tmp, hpre_align_sz());
1591 struct hpre_sqe *msg = &hpre_req->req;
1592 int ret;
1593
1594 ret = hpre_ecdh_msg_request_set(ctx, req);
1595 if (unlikely(ret)) {
1596 dev_err(dev, "failed to set ecdh request, ret = %d!\n", ret);
1597 return ret;
1598 }
1599
1600 if (req->src) {
1601 ret = hpre_ecdh_src_data_init(hpre_req, req->src, req->src_len);
1602 if (unlikely(ret)) {
1603 dev_err(dev, "failed to init src data, ret = %d!\n", ret);
1604 goto clear_all;
1605 }
1606 } else {
1607 msg->in = cpu_to_le64(ctx->ecdh.dma_g);
1608 }
1609
1610 ret = hpre_ecdh_dst_data_init(hpre_req, req->dst, req->dst_len);
1611 if (unlikely(ret)) {
1612 dev_err(dev, "failed to init dst data, ret = %d!\n", ret);
1613 goto clear_all;
1614 }
1615
1616 msg->dw0 = cpu_to_le32(le32_to_cpu(msg->dw0) | HPRE_ALG_ECC_MUL);
1617 ret = hpre_send(ctx, msg);
1618 if (likely(!ret))
1619 return -EINPROGRESS;
1620
1621 clear_all:
1622 hpre_rm_req_from_ctx(hpre_req);
1623 hpre_ecdh_hw_data_clr_all(ctx, hpre_req, req->dst, req->src);
1624 return ret;
1625 }
1626
hpre_ecdh_max_size(struct crypto_kpp * tfm)1627 static unsigned int hpre_ecdh_max_size(struct crypto_kpp *tfm)
1628 {
1629 struct hpre_ctx *ctx = kpp_tfm_ctx(tfm);
1630
1631 /* max size is the pub_key_size, include x and y */
1632 return ctx->key_sz << 1;
1633 }
1634
hpre_ecdh_nist_p192_init_tfm(struct crypto_kpp * tfm)1635 static int hpre_ecdh_nist_p192_init_tfm(struct crypto_kpp *tfm)
1636 {
1637 struct hpre_ctx *ctx = kpp_tfm_ctx(tfm);
1638
1639 ctx->curve_id = ECC_CURVE_NIST_P192;
1640
1641 kpp_set_reqsize(tfm, sizeof(struct hpre_asym_request) + hpre_align_pd());
1642
1643 return hpre_ctx_init(ctx, HPRE_V3_ECC_ALG_TYPE);
1644 }
1645
hpre_ecdh_nist_p256_init_tfm(struct crypto_kpp * tfm)1646 static int hpre_ecdh_nist_p256_init_tfm(struct crypto_kpp *tfm)
1647 {
1648 struct hpre_ctx *ctx = kpp_tfm_ctx(tfm);
1649
1650 ctx->curve_id = ECC_CURVE_NIST_P256;
1651
1652 kpp_set_reqsize(tfm, sizeof(struct hpre_asym_request) + hpre_align_pd());
1653
1654 return hpre_ctx_init(ctx, HPRE_V3_ECC_ALG_TYPE);
1655 }
1656
hpre_ecdh_nist_p384_init_tfm(struct crypto_kpp * tfm)1657 static int hpre_ecdh_nist_p384_init_tfm(struct crypto_kpp *tfm)
1658 {
1659 struct hpre_ctx *ctx = kpp_tfm_ctx(tfm);
1660
1661 ctx->curve_id = ECC_CURVE_NIST_P384;
1662
1663 kpp_set_reqsize(tfm, sizeof(struct hpre_asym_request) + hpre_align_pd());
1664
1665 return hpre_ctx_init(ctx, HPRE_V3_ECC_ALG_TYPE);
1666 }
1667
hpre_ecdh_exit_tfm(struct crypto_kpp * tfm)1668 static void hpre_ecdh_exit_tfm(struct crypto_kpp *tfm)
1669 {
1670 struct hpre_ctx *ctx = kpp_tfm_ctx(tfm);
1671
1672 hpre_ecc_clear_ctx(ctx, true, true);
1673 }
1674
hpre_curve25519_fill_curve(struct hpre_ctx * ctx,const void * buf,unsigned int len)1675 static void hpre_curve25519_fill_curve(struct hpre_ctx *ctx, const void *buf,
1676 unsigned int len)
1677 {
1678 u8 secret[CURVE25519_KEY_SIZE] = { 0 };
1679 unsigned int sz = ctx->key_sz;
1680 const struct ecc_curve *curve;
1681 unsigned int shift = sz << 1;
1682 void *p;
1683
1684 /*
1685 * The key from 'buf' is in little-endian, we should preprocess it as
1686 * the description in rfc7748: "k[0] &= 248, k[31] &= 127, k[31] |= 64",
1687 * then convert it to big endian. Only in this way, the result can be
1688 * the same as the software curve-25519 that exists in crypto.
1689 */
1690 memcpy(secret, buf, len);
1691 curve25519_clamp_secret(secret);
1692 hpre_key_to_big_end(secret, CURVE25519_KEY_SIZE);
1693
1694 p = ctx->curve25519.p + sz - len;
1695
1696 curve = ecc_get_curve25519();
1697
1698 /* fill curve parameters */
1699 fill_curve_param(p, curve->p, len, curve->g.ndigits);
1700 fill_curve_param(p + sz, curve->a, len, curve->g.ndigits);
1701 memcpy(p + shift, secret, len);
1702 fill_curve_param(p + shift + sz, curve->g.x, len, curve->g.ndigits);
1703 memzero_explicit(secret, CURVE25519_KEY_SIZE);
1704 }
1705
hpre_curve25519_set_param(struct hpre_ctx * ctx,const void * buf,unsigned int len)1706 static int hpre_curve25519_set_param(struct hpre_ctx *ctx, const void *buf,
1707 unsigned int len)
1708 {
1709 struct device *dev = ctx->dev;
1710 unsigned int sz = ctx->key_sz;
1711 unsigned int shift = sz << 1;
1712
1713 /* p->a->k->gx */
1714 if (!ctx->curve25519.p) {
1715 ctx->curve25519.p = dma_alloc_coherent(dev, sz << 2,
1716 &ctx->curve25519.dma_p,
1717 GFP_KERNEL);
1718 if (!ctx->curve25519.p)
1719 return -ENOMEM;
1720 }
1721
1722 ctx->curve25519.g = ctx->curve25519.p + shift + sz;
1723 ctx->curve25519.dma_g = ctx->curve25519.dma_p + shift + sz;
1724
1725 hpre_curve25519_fill_curve(ctx, buf, len);
1726
1727 return 0;
1728 }
1729
hpre_curve25519_set_secret(struct crypto_kpp * tfm,const void * buf,unsigned int len)1730 static int hpre_curve25519_set_secret(struct crypto_kpp *tfm, const void *buf,
1731 unsigned int len)
1732 {
1733 struct hpre_ctx *ctx = kpp_tfm_ctx(tfm);
1734 struct device *dev = ctx->dev;
1735 int ret = -EINVAL;
1736
1737 if (len != CURVE25519_KEY_SIZE ||
1738 !crypto_memneq(buf, curve25519_null_point, CURVE25519_KEY_SIZE)) {
1739 dev_err(dev, "key is null or key len is not 32bytes!\n");
1740 return ret;
1741 }
1742
1743 /* Free old secret if any */
1744 hpre_ecc_clear_ctx(ctx, false, false);
1745
1746 ctx->key_sz = CURVE25519_KEY_SIZE;
1747 ret = hpre_curve25519_set_param(ctx, buf, CURVE25519_KEY_SIZE);
1748 if (ret) {
1749 dev_err(dev, "failed to set curve25519 param, ret = %d!\n", ret);
1750 hpre_ecc_clear_ctx(ctx, false, false);
1751 return ret;
1752 }
1753
1754 return 0;
1755 }
1756
hpre_curve25519_hw_data_clr_all(struct hpre_ctx * ctx,struct hpre_asym_request * req,struct scatterlist * dst,struct scatterlist * src)1757 static void hpre_curve25519_hw_data_clr_all(struct hpre_ctx *ctx,
1758 struct hpre_asym_request *req,
1759 struct scatterlist *dst,
1760 struct scatterlist *src)
1761 {
1762 struct device *dev = ctx->dev;
1763 struct hpre_sqe *sqe = &req->req;
1764 dma_addr_t dma;
1765
1766 dma = le64_to_cpu(sqe->in);
1767 if (unlikely(dma_mapping_error(dev, dma)))
1768 return;
1769
1770 if (src && req->src)
1771 dma_free_coherent(dev, ctx->key_sz, req->src, dma);
1772
1773 dma = le64_to_cpu(sqe->out);
1774 if (unlikely(dma_mapping_error(dev, dma)))
1775 return;
1776
1777 if (req->dst)
1778 dma_free_coherent(dev, ctx->key_sz, req->dst, dma);
1779 if (dst)
1780 dma_unmap_single(dev, dma, ctx->key_sz, DMA_FROM_DEVICE);
1781 }
1782
hpre_curve25519_cb(struct hpre_ctx * ctx,void * resp)1783 static void hpre_curve25519_cb(struct hpre_ctx *ctx, void *resp)
1784 {
1785 struct hpre_dfx *dfx = ctx->hpre->debug.dfx;
1786 struct hpre_asym_request *req = NULL;
1787 struct kpp_request *areq;
1788 u64 overtime_thrhld;
1789 int ret;
1790
1791 ret = hpre_alg_res_post_hf(ctx, resp, (void **)&req);
1792 areq = req->areq.curve25519;
1793 areq->dst_len = ctx->key_sz;
1794
1795 overtime_thrhld = atomic64_read(&dfx[HPRE_OVERTIME_THRHLD].value);
1796 if (overtime_thrhld && hpre_is_bd_timeout(req, overtime_thrhld))
1797 atomic64_inc(&dfx[HPRE_OVER_THRHLD_CNT].value);
1798
1799 hpre_key_to_big_end(sg_virt(areq->dst), CURVE25519_KEY_SIZE);
1800
1801 hpre_curve25519_hw_data_clr_all(ctx, req, areq->dst, areq->src);
1802 kpp_request_complete(areq, ret);
1803
1804 atomic64_inc(&dfx[HPRE_RECV_CNT].value);
1805 }
1806
hpre_curve25519_msg_request_set(struct hpre_ctx * ctx,struct kpp_request * req)1807 static int hpre_curve25519_msg_request_set(struct hpre_ctx *ctx,
1808 struct kpp_request *req)
1809 {
1810 struct hpre_asym_request *h_req;
1811 struct hpre_sqe *msg;
1812 int req_id;
1813 void *tmp;
1814
1815 if (unlikely(req->dst_len < ctx->key_sz)) {
1816 req->dst_len = ctx->key_sz;
1817 return -EINVAL;
1818 }
1819
1820 tmp = kpp_request_ctx(req);
1821 h_req = PTR_ALIGN(tmp, hpre_align_sz());
1822 h_req->cb = hpre_curve25519_cb;
1823 h_req->areq.curve25519 = req;
1824 msg = &h_req->req;
1825 memset(msg, 0, sizeof(*msg));
1826 msg->in = cpu_to_le64(DMA_MAPPING_ERROR);
1827 msg->out = cpu_to_le64(DMA_MAPPING_ERROR);
1828 msg->key = cpu_to_le64(ctx->curve25519.dma_p);
1829
1830 msg->dw0 |= cpu_to_le32(0x1U << HPRE_SQE_DONE_SHIFT);
1831 msg->task_len1 = (ctx->key_sz >> HPRE_BITS_2_BYTES_SHIFT) - 1;
1832 h_req->ctx = ctx;
1833
1834 req_id = hpre_add_req_to_ctx(h_req);
1835 if (req_id < 0)
1836 return -EBUSY;
1837
1838 msg->tag = cpu_to_le16((u16)req_id);
1839 return 0;
1840 }
1841
hpre_curve25519_src_modulo_p(u8 * ptr)1842 static void hpre_curve25519_src_modulo_p(u8 *ptr)
1843 {
1844 int i;
1845
1846 for (i = 0; i < CURVE25519_KEY_SIZE - 1; i++)
1847 ptr[i] = 0;
1848
1849 /* The modulus is ptr's last byte minus '0xed'(last byte of p) */
1850 ptr[i] -= 0xed;
1851 }
1852
hpre_curve25519_src_init(struct hpre_asym_request * hpre_req,struct scatterlist * data,unsigned int len)1853 static int hpre_curve25519_src_init(struct hpre_asym_request *hpre_req,
1854 struct scatterlist *data, unsigned int len)
1855 {
1856 struct hpre_sqe *msg = &hpre_req->req;
1857 struct hpre_ctx *ctx = hpre_req->ctx;
1858 struct device *dev = ctx->dev;
1859 u8 p[CURVE25519_KEY_SIZE] = { 0 };
1860 const struct ecc_curve *curve;
1861 dma_addr_t dma = 0;
1862 u8 *ptr;
1863
1864 if (len != CURVE25519_KEY_SIZE) {
1865 dev_err(dev, "sourc_data len is not 32bytes, len = %u!\n", len);
1866 return -EINVAL;
1867 }
1868
1869 ptr = dma_alloc_coherent(dev, ctx->key_sz, &dma, GFP_KERNEL);
1870 if (unlikely(!ptr))
1871 return -ENOMEM;
1872
1873 scatterwalk_map_and_copy(ptr, data, 0, len, 0);
1874
1875 if (!crypto_memneq(ptr, curve25519_null_point, CURVE25519_KEY_SIZE)) {
1876 dev_err(dev, "gx is null!\n");
1877 goto err;
1878 }
1879
1880 /*
1881 * Src_data(gx) is in little-endian order, MSB in the final byte should
1882 * be masked as described in RFC7748, then transform it to big-endian
1883 * form, then hisi_hpre can use the data.
1884 */
1885 ptr[31] &= 0x7f;
1886 hpre_key_to_big_end(ptr, CURVE25519_KEY_SIZE);
1887
1888 curve = ecc_get_curve25519();
1889
1890 fill_curve_param(p, curve->p, CURVE25519_KEY_SIZE, curve->g.ndigits);
1891
1892 /*
1893 * When src_data equals (2^255 - 19) ~ (2^255 - 1), it is out of p,
1894 * we get its modulus to p, and then use it.
1895 */
1896 if (memcmp(ptr, p, ctx->key_sz) == 0) {
1897 dev_err(dev, "gx is p!\n");
1898 goto err;
1899 } else if (memcmp(ptr, p, ctx->key_sz) > 0) {
1900 hpre_curve25519_src_modulo_p(ptr);
1901 }
1902
1903 hpre_req->src = ptr;
1904 msg->in = cpu_to_le64(dma);
1905 return 0;
1906
1907 err:
1908 dma_free_coherent(dev, ctx->key_sz, ptr, dma);
1909 return -EINVAL;
1910 }
1911
hpre_curve25519_dst_init(struct hpre_asym_request * hpre_req,struct scatterlist * data,unsigned int len)1912 static int hpre_curve25519_dst_init(struct hpre_asym_request *hpre_req,
1913 struct scatterlist *data, unsigned int len)
1914 {
1915 struct hpre_sqe *msg = &hpre_req->req;
1916 struct hpre_ctx *ctx = hpre_req->ctx;
1917 struct device *dev = ctx->dev;
1918 dma_addr_t dma;
1919
1920 if (!data || !sg_is_last(data) || len != ctx->key_sz) {
1921 dev_err(dev, "data or data length is illegal!\n");
1922 return -EINVAL;
1923 }
1924
1925 hpre_req->dst = NULL;
1926 dma = dma_map_single(dev, sg_virt(data), len, DMA_FROM_DEVICE);
1927 if (unlikely(dma_mapping_error(dev, dma))) {
1928 dev_err(dev, "dma map data err!\n");
1929 return -ENOMEM;
1930 }
1931
1932 msg->out = cpu_to_le64(dma);
1933 return 0;
1934 }
1935
hpre_curve25519_compute_value(struct kpp_request * req)1936 static int hpre_curve25519_compute_value(struct kpp_request *req)
1937 {
1938 struct crypto_kpp *tfm = crypto_kpp_reqtfm(req);
1939 struct hpre_ctx *ctx = kpp_tfm_ctx(tfm);
1940 struct device *dev = ctx->dev;
1941 void *tmp = kpp_request_ctx(req);
1942 struct hpre_asym_request *hpre_req = PTR_ALIGN(tmp, hpre_align_sz());
1943 struct hpre_sqe *msg = &hpre_req->req;
1944 int ret;
1945
1946 ret = hpre_curve25519_msg_request_set(ctx, req);
1947 if (unlikely(ret)) {
1948 dev_err(dev, "failed to set curve25519 request, ret = %d!\n", ret);
1949 return ret;
1950 }
1951
1952 if (req->src) {
1953 ret = hpre_curve25519_src_init(hpre_req, req->src, req->src_len);
1954 if (unlikely(ret)) {
1955 dev_err(dev, "failed to init src data, ret = %d!\n",
1956 ret);
1957 goto clear_all;
1958 }
1959 } else {
1960 msg->in = cpu_to_le64(ctx->curve25519.dma_g);
1961 }
1962
1963 ret = hpre_curve25519_dst_init(hpre_req, req->dst, req->dst_len);
1964 if (unlikely(ret)) {
1965 dev_err(dev, "failed to init dst data, ret = %d!\n", ret);
1966 goto clear_all;
1967 }
1968
1969 msg->dw0 = cpu_to_le32(le32_to_cpu(msg->dw0) | HPRE_ALG_CURVE25519_MUL);
1970 ret = hpre_send(ctx, msg);
1971 if (likely(!ret))
1972 return -EINPROGRESS;
1973
1974 clear_all:
1975 hpre_rm_req_from_ctx(hpre_req);
1976 hpre_curve25519_hw_data_clr_all(ctx, hpre_req, req->dst, req->src);
1977 return ret;
1978 }
1979
hpre_curve25519_max_size(struct crypto_kpp * tfm)1980 static unsigned int hpre_curve25519_max_size(struct crypto_kpp *tfm)
1981 {
1982 struct hpre_ctx *ctx = kpp_tfm_ctx(tfm);
1983
1984 return ctx->key_sz;
1985 }
1986
hpre_curve25519_init_tfm(struct crypto_kpp * tfm)1987 static int hpre_curve25519_init_tfm(struct crypto_kpp *tfm)
1988 {
1989 struct hpre_ctx *ctx = kpp_tfm_ctx(tfm);
1990
1991 kpp_set_reqsize(tfm, sizeof(struct hpre_asym_request) + hpre_align_pd());
1992
1993 return hpre_ctx_init(ctx, HPRE_V3_ECC_ALG_TYPE);
1994 }
1995
hpre_curve25519_exit_tfm(struct crypto_kpp * tfm)1996 static void hpre_curve25519_exit_tfm(struct crypto_kpp *tfm)
1997 {
1998 struct hpre_ctx *ctx = kpp_tfm_ctx(tfm);
1999
2000 hpre_ecc_clear_ctx(ctx, true, false);
2001 }
2002
2003 static struct akcipher_alg rsa = {
2004 .sign = hpre_rsa_dec,
2005 .verify = hpre_rsa_enc,
2006 .encrypt = hpre_rsa_enc,
2007 .decrypt = hpre_rsa_dec,
2008 .set_pub_key = hpre_rsa_setpubkey,
2009 .set_priv_key = hpre_rsa_setprivkey,
2010 .max_size = hpre_rsa_max_size,
2011 .init = hpre_rsa_init_tfm,
2012 .exit = hpre_rsa_exit_tfm,
2013 .base = {
2014 .cra_ctxsize = sizeof(struct hpre_ctx),
2015 .cra_priority = HPRE_CRYPTO_ALG_PRI,
2016 .cra_name = "rsa",
2017 .cra_driver_name = "hpre-rsa",
2018 .cra_module = THIS_MODULE,
2019 },
2020 };
2021
2022 static struct kpp_alg dh = {
2023 .set_secret = hpre_dh_set_secret,
2024 .generate_public_key = hpre_dh_compute_value,
2025 .compute_shared_secret = hpre_dh_compute_value,
2026 .max_size = hpre_dh_max_size,
2027 .init = hpre_dh_init_tfm,
2028 .exit = hpre_dh_exit_tfm,
2029 .base = {
2030 .cra_ctxsize = sizeof(struct hpre_ctx),
2031 .cra_priority = HPRE_CRYPTO_ALG_PRI,
2032 .cra_name = "dh",
2033 .cra_driver_name = "hpre-dh",
2034 .cra_module = THIS_MODULE,
2035 },
2036 };
2037
2038 static struct kpp_alg ecdh_curves[] = {
2039 {
2040 .set_secret = hpre_ecdh_set_secret,
2041 .generate_public_key = hpre_ecdh_compute_value,
2042 .compute_shared_secret = hpre_ecdh_compute_value,
2043 .max_size = hpre_ecdh_max_size,
2044 .init = hpre_ecdh_nist_p192_init_tfm,
2045 .exit = hpre_ecdh_exit_tfm,
2046 .base = {
2047 .cra_ctxsize = sizeof(struct hpre_ctx),
2048 .cra_priority = HPRE_CRYPTO_ALG_PRI,
2049 .cra_name = "ecdh-nist-p192",
2050 .cra_driver_name = "hpre-ecdh-nist-p192",
2051 .cra_module = THIS_MODULE,
2052 },
2053 }, {
2054 .set_secret = hpre_ecdh_set_secret,
2055 .generate_public_key = hpre_ecdh_compute_value,
2056 .compute_shared_secret = hpre_ecdh_compute_value,
2057 .max_size = hpre_ecdh_max_size,
2058 .init = hpre_ecdh_nist_p256_init_tfm,
2059 .exit = hpre_ecdh_exit_tfm,
2060 .base = {
2061 .cra_ctxsize = sizeof(struct hpre_ctx),
2062 .cra_priority = HPRE_CRYPTO_ALG_PRI,
2063 .cra_name = "ecdh-nist-p256",
2064 .cra_driver_name = "hpre-ecdh-nist-p256",
2065 .cra_module = THIS_MODULE,
2066 },
2067 }, {
2068 .set_secret = hpre_ecdh_set_secret,
2069 .generate_public_key = hpre_ecdh_compute_value,
2070 .compute_shared_secret = hpre_ecdh_compute_value,
2071 .max_size = hpre_ecdh_max_size,
2072 .init = hpre_ecdh_nist_p384_init_tfm,
2073 .exit = hpre_ecdh_exit_tfm,
2074 .base = {
2075 .cra_ctxsize = sizeof(struct hpre_ctx),
2076 .cra_priority = HPRE_CRYPTO_ALG_PRI,
2077 .cra_name = "ecdh-nist-p384",
2078 .cra_driver_name = "hpre-ecdh-nist-p384",
2079 .cra_module = THIS_MODULE,
2080 },
2081 }
2082 };
2083
2084 static struct kpp_alg curve25519_alg = {
2085 .set_secret = hpre_curve25519_set_secret,
2086 .generate_public_key = hpre_curve25519_compute_value,
2087 .compute_shared_secret = hpre_curve25519_compute_value,
2088 .max_size = hpre_curve25519_max_size,
2089 .init = hpre_curve25519_init_tfm,
2090 .exit = hpre_curve25519_exit_tfm,
2091 .base = {
2092 .cra_ctxsize = sizeof(struct hpre_ctx),
2093 .cra_priority = HPRE_CRYPTO_ALG_PRI,
2094 .cra_name = "curve25519",
2095 .cra_driver_name = "hpre-curve25519",
2096 .cra_module = THIS_MODULE,
2097 },
2098 };
2099
hpre_register_rsa(struct hisi_qm * qm)2100 static int hpre_register_rsa(struct hisi_qm *qm)
2101 {
2102 int ret;
2103
2104 if (!hpre_check_alg_support(qm, HPRE_DRV_RSA_MASK_CAP))
2105 return 0;
2106
2107 rsa.base.cra_flags = 0;
2108 ret = crypto_register_akcipher(&rsa);
2109 if (ret)
2110 dev_err(&qm->pdev->dev, "failed to register rsa (%d)!\n", ret);
2111
2112 return ret;
2113 }
2114
hpre_unregister_rsa(struct hisi_qm * qm)2115 static void hpre_unregister_rsa(struct hisi_qm *qm)
2116 {
2117 if (!hpre_check_alg_support(qm, HPRE_DRV_RSA_MASK_CAP))
2118 return;
2119
2120 crypto_unregister_akcipher(&rsa);
2121 }
2122
hpre_register_dh(struct hisi_qm * qm)2123 static int hpre_register_dh(struct hisi_qm *qm)
2124 {
2125 int ret;
2126
2127 if (!hpre_check_alg_support(qm, HPRE_DRV_DH_MASK_CAP))
2128 return 0;
2129
2130 ret = crypto_register_kpp(&dh);
2131 if (ret)
2132 dev_err(&qm->pdev->dev, "failed to register dh (%d)!\n", ret);
2133
2134 return ret;
2135 }
2136
hpre_unregister_dh(struct hisi_qm * qm)2137 static void hpre_unregister_dh(struct hisi_qm *qm)
2138 {
2139 if (!hpre_check_alg_support(qm, HPRE_DRV_DH_MASK_CAP))
2140 return;
2141
2142 crypto_unregister_kpp(&dh);
2143 }
2144
hpre_register_ecdh(struct hisi_qm * qm)2145 static int hpre_register_ecdh(struct hisi_qm *qm)
2146 {
2147 int ret, i;
2148
2149 if (!hpre_check_alg_support(qm, HPRE_DRV_ECDH_MASK_CAP))
2150 return 0;
2151
2152 for (i = 0; i < ARRAY_SIZE(ecdh_curves); i++) {
2153 ret = crypto_register_kpp(&ecdh_curves[i]);
2154 if (ret) {
2155 dev_err(&qm->pdev->dev, "failed to register %s (%d)!\n",
2156 ecdh_curves[i].base.cra_name, ret);
2157 goto unreg_kpp;
2158 }
2159 }
2160
2161 return 0;
2162
2163 unreg_kpp:
2164 for (--i; i >= 0; --i)
2165 crypto_unregister_kpp(&ecdh_curves[i]);
2166
2167 return ret;
2168 }
2169
hpre_unregister_ecdh(struct hisi_qm * qm)2170 static void hpre_unregister_ecdh(struct hisi_qm *qm)
2171 {
2172 int i;
2173
2174 if (!hpre_check_alg_support(qm, HPRE_DRV_ECDH_MASK_CAP))
2175 return;
2176
2177 for (i = ARRAY_SIZE(ecdh_curves) - 1; i >= 0; --i)
2178 crypto_unregister_kpp(&ecdh_curves[i]);
2179 }
2180
hpre_register_x25519(struct hisi_qm * qm)2181 static int hpre_register_x25519(struct hisi_qm *qm)
2182 {
2183 int ret;
2184
2185 if (!hpre_check_alg_support(qm, HPRE_DRV_X25519_MASK_CAP))
2186 return 0;
2187
2188 ret = crypto_register_kpp(&curve25519_alg);
2189 if (ret)
2190 dev_err(&qm->pdev->dev, "failed to register x25519 (%d)!\n", ret);
2191
2192 return ret;
2193 }
2194
hpre_unregister_x25519(struct hisi_qm * qm)2195 static void hpre_unregister_x25519(struct hisi_qm *qm)
2196 {
2197 if (!hpre_check_alg_support(qm, HPRE_DRV_X25519_MASK_CAP))
2198 return;
2199
2200 crypto_unregister_kpp(&curve25519_alg);
2201 }
2202
hpre_algs_register(struct hisi_qm * qm)2203 int hpre_algs_register(struct hisi_qm *qm)
2204 {
2205 int ret;
2206
2207 ret = hpre_register_rsa(qm);
2208 if (ret)
2209 return ret;
2210
2211 ret = hpre_register_dh(qm);
2212 if (ret)
2213 goto unreg_rsa;
2214
2215 ret = hpre_register_ecdh(qm);
2216 if (ret)
2217 goto unreg_dh;
2218
2219 ret = hpre_register_x25519(qm);
2220 if (ret)
2221 goto unreg_ecdh;
2222
2223 return ret;
2224
2225 unreg_ecdh:
2226 hpre_unregister_ecdh(qm);
2227 unreg_dh:
2228 hpre_unregister_dh(qm);
2229 unreg_rsa:
2230 hpre_unregister_rsa(qm);
2231 return ret;
2232 }
2233
hpre_algs_unregister(struct hisi_qm * qm)2234 void hpre_algs_unregister(struct hisi_qm *qm)
2235 {
2236 hpre_unregister_x25519(qm);
2237 hpre_unregister_ecdh(qm);
2238 hpre_unregister_dh(qm);
2239 hpre_unregister_rsa(qm);
2240 }
2241