1 // SPDX-License-Identifier: GPL-2.0 2 /* Copyright (c) 2019 HiSilicon Limited. */ 3 #include <crypto/akcipher.h> 4 #include <crypto/curve25519.h> 5 #include <crypto/dh.h> 6 #include <crypto/ecc_curve.h> 7 #include <crypto/ecdh.h> 8 #include <crypto/rng.h> 9 #include <crypto/internal/akcipher.h> 10 #include <crypto/internal/kpp.h> 11 #include <crypto/internal/rsa.h> 12 #include <crypto/kpp.h> 13 #include <crypto/scatterwalk.h> 14 #include <linux/dma-mapping.h> 15 #include <linux/fips.h> 16 #include <linux/module.h> 17 #include <linux/time.h> 18 #include "hpre.h" 19 20 struct hpre_ctx; 21 22 #define HPRE_CRYPTO_ALG_PRI 1000 23 #define HPRE_ALIGN_SZ 64 24 #define HPRE_BITS_2_BYTES_SHIFT 3 25 #define HPRE_RSA_512BITS_KSZ 64 26 #define HPRE_RSA_1536BITS_KSZ 192 27 #define HPRE_CRT_PRMS 5 28 #define HPRE_CRT_Q 2 29 #define HPRE_CRT_P 3 30 #define HPRE_CRT_INV 4 31 #define HPRE_DH_G_FLAG 0x02 32 #define HPRE_TRY_SEND_TIMES 100 33 #define HPRE_INVLD_REQ_ID (-1) 34 35 #define HPRE_SQE_ALG_BITS 5 36 #define HPRE_SQE_DONE_SHIFT 30 37 #define HPRE_DH_MAX_P_SZ 512 38 39 #define HPRE_DFX_SEC_TO_US 1000000 40 #define HPRE_DFX_US_TO_NS 1000 41 42 /* due to nist p521 */ 43 #define HPRE_ECC_MAX_KSZ 66 44 45 /* size in bytes of the n prime */ 46 #define HPRE_ECC_NIST_P192_N_SIZE 24 47 #define HPRE_ECC_NIST_P256_N_SIZE 32 48 #define HPRE_ECC_NIST_P384_N_SIZE 48 49 50 /* size in bytes */ 51 #define HPRE_ECC_HW256_KSZ_B 32 52 #define HPRE_ECC_HW384_KSZ_B 48 53 54 typedef void (*hpre_cb)(struct hpre_ctx *ctx, void *sqe); 55 56 struct hpre_rsa_ctx { 57 /* low address: e--->n */ 58 char *pubkey; 59 dma_addr_t dma_pubkey; 60 61 /* low address: d--->n */ 62 char *prikey; 63 dma_addr_t dma_prikey; 64 65 /* low address: dq->dp->q->p->qinv */ 66 char *crt_prikey; 67 dma_addr_t dma_crt_prikey; 68 69 struct crypto_akcipher *soft_tfm; 70 }; 71 72 struct hpre_dh_ctx { 73 /* 74 * If base is g we compute the public key 75 * ya = g^xa mod p; [RFC2631 sec 2.1.1] 76 * else if base if the counterpart public key we 77 * compute the shared secret 78 * ZZ = yb^xa mod p; [RFC2631 sec 2.1.1] 79 * low address: d--->n, please refer to Hisilicon HPRE UM 80 */ 81 char *xa_p; 82 dma_addr_t dma_xa_p; 83 84 char *g; /* m */ 85 dma_addr_t dma_g; 86 }; 87 88 struct hpre_ecdh_ctx { 89 /* low address: p->a->k->b */ 90 unsigned char *p; 91 dma_addr_t dma_p; 92 93 /* low address: x->y */ 94 unsigned char *g; 95 dma_addr_t dma_g; 96 }; 97 98 struct hpre_curve25519_ctx { 99 /* low address: p->a->k */ 100 unsigned char *p; 101 dma_addr_t dma_p; 102 103 /* gx coordinate */ 104 unsigned char *g; 105 dma_addr_t dma_g; 106 }; 107 108 struct hpre_ctx { 109 struct hisi_qp *qp; 110 struct device *dev; 111 struct hpre_asym_request **req_list; 112 struct hpre *hpre; 113 spinlock_t req_lock; 114 unsigned int key_sz; 115 bool crt_g2_mode; 116 struct idr req_idr; 117 union { 118 struct hpre_rsa_ctx rsa; 119 struct hpre_dh_ctx dh; 120 struct hpre_ecdh_ctx ecdh; 121 struct hpre_curve25519_ctx curve25519; 122 }; 123 /* for ecc algorithms */ 124 unsigned int curve_id; 125 }; 126 127 struct hpre_asym_request { 128 char *src; 129 char *dst; 130 struct hpre_sqe req; 131 struct hpre_ctx *ctx; 132 union { 133 struct akcipher_request *rsa; 134 struct kpp_request *dh; 135 struct kpp_request *ecdh; 136 struct kpp_request *curve25519; 137 } areq; 138 int err; 139 int req_id; 140 hpre_cb cb; 141 struct timespec64 req_time; 142 }; 143 144 static int hpre_alloc_req_id(struct hpre_ctx *ctx) 145 { 146 unsigned long flags; 147 int id; 148 149 spin_lock_irqsave(&ctx->req_lock, flags); 150 id = idr_alloc(&ctx->req_idr, NULL, 0, QM_Q_DEPTH, GFP_ATOMIC); 151 spin_unlock_irqrestore(&ctx->req_lock, flags); 152 153 return id; 154 } 155 156 static void hpre_free_req_id(struct hpre_ctx *ctx, int req_id) 157 { 158 unsigned long flags; 159 160 spin_lock_irqsave(&ctx->req_lock, flags); 161 idr_remove(&ctx->req_idr, req_id); 162 spin_unlock_irqrestore(&ctx->req_lock, flags); 163 } 164 165 static int hpre_add_req_to_ctx(struct hpre_asym_request *hpre_req) 166 { 167 struct hpre_ctx *ctx; 168 struct hpre_dfx *dfx; 169 int id; 170 171 ctx = hpre_req->ctx; 172 id = hpre_alloc_req_id(ctx); 173 if (unlikely(id < 0)) 174 return -EINVAL; 175 176 ctx->req_list[id] = hpre_req; 177 hpre_req->req_id = id; 178 179 dfx = ctx->hpre->debug.dfx; 180 if (atomic64_read(&dfx[HPRE_OVERTIME_THRHLD].value)) 181 ktime_get_ts64(&hpre_req->req_time); 182 183 return id; 184 } 185 186 static void hpre_rm_req_from_ctx(struct hpre_asym_request *hpre_req) 187 { 188 struct hpre_ctx *ctx = hpre_req->ctx; 189 int id = hpre_req->req_id; 190 191 if (hpre_req->req_id >= 0) { 192 hpre_req->req_id = HPRE_INVLD_REQ_ID; 193 ctx->req_list[id] = NULL; 194 hpre_free_req_id(ctx, id); 195 } 196 } 197 198 static struct hisi_qp *hpre_get_qp_and_start(u8 type) 199 { 200 struct hisi_qp *qp; 201 int ret; 202 203 qp = hpre_create_qp(type); 204 if (!qp) { 205 pr_err("Can not create hpre qp!\n"); 206 return ERR_PTR(-ENODEV); 207 } 208 209 ret = hisi_qm_start_qp(qp, 0); 210 if (ret < 0) { 211 hisi_qm_free_qps(&qp, 1); 212 pci_err(qp->qm->pdev, "Can not start qp!\n"); 213 return ERR_PTR(-EINVAL); 214 } 215 216 return qp; 217 } 218 219 static int hpre_get_data_dma_addr(struct hpre_asym_request *hpre_req, 220 struct scatterlist *data, unsigned int len, 221 int is_src, dma_addr_t *tmp) 222 { 223 struct device *dev = hpre_req->ctx->dev; 224 enum dma_data_direction dma_dir; 225 226 if (is_src) { 227 hpre_req->src = NULL; 228 dma_dir = DMA_TO_DEVICE; 229 } else { 230 hpre_req->dst = NULL; 231 dma_dir = DMA_FROM_DEVICE; 232 } 233 *tmp = dma_map_single(dev, sg_virt(data), len, dma_dir); 234 if (unlikely(dma_mapping_error(dev, *tmp))) { 235 dev_err(dev, "dma map data err!\n"); 236 return -ENOMEM; 237 } 238 239 return 0; 240 } 241 242 static int hpre_prepare_dma_buf(struct hpre_asym_request *hpre_req, 243 struct scatterlist *data, unsigned int len, 244 int is_src, dma_addr_t *tmp) 245 { 246 struct hpre_ctx *ctx = hpre_req->ctx; 247 struct device *dev = ctx->dev; 248 void *ptr; 249 int shift; 250 251 shift = ctx->key_sz - len; 252 if (unlikely(shift < 0)) 253 return -EINVAL; 254 255 ptr = dma_alloc_coherent(dev, ctx->key_sz, tmp, GFP_KERNEL); 256 if (unlikely(!ptr)) 257 return -ENOMEM; 258 259 if (is_src) { 260 scatterwalk_map_and_copy(ptr + shift, data, 0, len, 0); 261 hpre_req->src = ptr; 262 } else { 263 hpre_req->dst = ptr; 264 } 265 266 return 0; 267 } 268 269 static int hpre_hw_data_init(struct hpre_asym_request *hpre_req, 270 struct scatterlist *data, unsigned int len, 271 int is_src, int is_dh) 272 { 273 struct hpre_sqe *msg = &hpre_req->req; 274 struct hpre_ctx *ctx = hpre_req->ctx; 275 dma_addr_t tmp = 0; 276 int ret; 277 278 /* when the data is dh's source, we should format it */ 279 if ((sg_is_last(data) && len == ctx->key_sz) && 280 ((is_dh && !is_src) || !is_dh)) 281 ret = hpre_get_data_dma_addr(hpre_req, data, len, is_src, &tmp); 282 else 283 ret = hpre_prepare_dma_buf(hpre_req, data, len, is_src, &tmp); 284 285 if (unlikely(ret)) 286 return ret; 287 288 if (is_src) 289 msg->in = cpu_to_le64(tmp); 290 else 291 msg->out = cpu_to_le64(tmp); 292 293 return 0; 294 } 295 296 static void hpre_hw_data_clr_all(struct hpre_ctx *ctx, 297 struct hpre_asym_request *req, 298 struct scatterlist *dst, 299 struct scatterlist *src) 300 { 301 struct device *dev = ctx->dev; 302 struct hpre_sqe *sqe = &req->req; 303 dma_addr_t tmp; 304 305 tmp = le64_to_cpu(sqe->in); 306 if (unlikely(dma_mapping_error(dev, tmp))) 307 return; 308 309 if (src) { 310 if (req->src) 311 dma_free_coherent(dev, ctx->key_sz, req->src, tmp); 312 else 313 dma_unmap_single(dev, tmp, ctx->key_sz, DMA_TO_DEVICE); 314 } 315 316 tmp = le64_to_cpu(sqe->out); 317 if (unlikely(dma_mapping_error(dev, tmp))) 318 return; 319 320 if (req->dst) { 321 if (dst) 322 scatterwalk_map_and_copy(req->dst, dst, 0, 323 ctx->key_sz, 1); 324 dma_free_coherent(dev, ctx->key_sz, req->dst, tmp); 325 } else { 326 dma_unmap_single(dev, tmp, ctx->key_sz, DMA_FROM_DEVICE); 327 } 328 } 329 330 static int hpre_alg_res_post_hf(struct hpre_ctx *ctx, struct hpre_sqe *sqe, 331 void **kreq) 332 { 333 struct hpre_asym_request *req; 334 unsigned int err, done, alg; 335 int id; 336 337 #define HPRE_NO_HW_ERR 0 338 #define HPRE_HW_TASK_DONE 3 339 #define HREE_HW_ERR_MASK GENMASK(10, 0) 340 #define HREE_SQE_DONE_MASK GENMASK(1, 0) 341 #define HREE_ALG_TYPE_MASK GENMASK(4, 0) 342 id = (int)le16_to_cpu(sqe->tag); 343 req = ctx->req_list[id]; 344 hpre_rm_req_from_ctx(req); 345 *kreq = req; 346 347 err = (le32_to_cpu(sqe->dw0) >> HPRE_SQE_ALG_BITS) & 348 HREE_HW_ERR_MASK; 349 350 done = (le32_to_cpu(sqe->dw0) >> HPRE_SQE_DONE_SHIFT) & 351 HREE_SQE_DONE_MASK; 352 353 if (likely(err == HPRE_NO_HW_ERR && done == HPRE_HW_TASK_DONE)) 354 return 0; 355 356 alg = le32_to_cpu(sqe->dw0) & HREE_ALG_TYPE_MASK; 357 dev_err_ratelimited(ctx->dev, "alg[0x%x] error: done[0x%x], etype[0x%x]\n", 358 alg, done, err); 359 360 return -EINVAL; 361 } 362 363 static int hpre_ctx_set(struct hpre_ctx *ctx, struct hisi_qp *qp, int qlen) 364 { 365 struct hpre *hpre; 366 367 if (!ctx || !qp || qlen < 0) 368 return -EINVAL; 369 370 spin_lock_init(&ctx->req_lock); 371 ctx->qp = qp; 372 ctx->dev = &qp->qm->pdev->dev; 373 374 hpre = container_of(ctx->qp->qm, struct hpre, qm); 375 ctx->hpre = hpre; 376 ctx->req_list = kcalloc(qlen, sizeof(void *), GFP_KERNEL); 377 if (!ctx->req_list) 378 return -ENOMEM; 379 ctx->key_sz = 0; 380 ctx->crt_g2_mode = false; 381 idr_init(&ctx->req_idr); 382 383 return 0; 384 } 385 386 static void hpre_ctx_clear(struct hpre_ctx *ctx, bool is_clear_all) 387 { 388 if (is_clear_all) { 389 idr_destroy(&ctx->req_idr); 390 kfree(ctx->req_list); 391 hisi_qm_free_qps(&ctx->qp, 1); 392 } 393 394 ctx->crt_g2_mode = false; 395 ctx->key_sz = 0; 396 } 397 398 static bool hpre_is_bd_timeout(struct hpre_asym_request *req, 399 u64 overtime_thrhld) 400 { 401 struct timespec64 reply_time; 402 u64 time_use_us; 403 404 ktime_get_ts64(&reply_time); 405 time_use_us = (reply_time.tv_sec - req->req_time.tv_sec) * 406 HPRE_DFX_SEC_TO_US + 407 (reply_time.tv_nsec - req->req_time.tv_nsec) / 408 HPRE_DFX_US_TO_NS; 409 410 if (time_use_us <= overtime_thrhld) 411 return false; 412 413 return true; 414 } 415 416 static void hpre_dh_cb(struct hpre_ctx *ctx, void *resp) 417 { 418 struct hpre_dfx *dfx = ctx->hpre->debug.dfx; 419 struct hpre_asym_request *req; 420 struct kpp_request *areq; 421 u64 overtime_thrhld; 422 int ret; 423 424 ret = hpre_alg_res_post_hf(ctx, resp, (void **)&req); 425 areq = req->areq.dh; 426 areq->dst_len = ctx->key_sz; 427 428 overtime_thrhld = atomic64_read(&dfx[HPRE_OVERTIME_THRHLD].value); 429 if (overtime_thrhld && hpre_is_bd_timeout(req, overtime_thrhld)) 430 atomic64_inc(&dfx[HPRE_OVER_THRHLD_CNT].value); 431 432 hpre_hw_data_clr_all(ctx, req, areq->dst, areq->src); 433 kpp_request_complete(areq, ret); 434 atomic64_inc(&dfx[HPRE_RECV_CNT].value); 435 } 436 437 static void hpre_rsa_cb(struct hpre_ctx *ctx, void *resp) 438 { 439 struct hpre_dfx *dfx = ctx->hpre->debug.dfx; 440 struct hpre_asym_request *req; 441 struct akcipher_request *areq; 442 u64 overtime_thrhld; 443 int ret; 444 445 ret = hpre_alg_res_post_hf(ctx, resp, (void **)&req); 446 447 overtime_thrhld = atomic64_read(&dfx[HPRE_OVERTIME_THRHLD].value); 448 if (overtime_thrhld && hpre_is_bd_timeout(req, overtime_thrhld)) 449 atomic64_inc(&dfx[HPRE_OVER_THRHLD_CNT].value); 450 451 areq = req->areq.rsa; 452 areq->dst_len = ctx->key_sz; 453 hpre_hw_data_clr_all(ctx, req, areq->dst, areq->src); 454 akcipher_request_complete(areq, ret); 455 atomic64_inc(&dfx[HPRE_RECV_CNT].value); 456 } 457 458 static void hpre_alg_cb(struct hisi_qp *qp, void *resp) 459 { 460 struct hpre_ctx *ctx = qp->qp_ctx; 461 struct hpre_dfx *dfx = ctx->hpre->debug.dfx; 462 struct hpre_sqe *sqe = resp; 463 struct hpre_asym_request *req = ctx->req_list[le16_to_cpu(sqe->tag)]; 464 465 if (unlikely(!req)) { 466 atomic64_inc(&dfx[HPRE_INVALID_REQ_CNT].value); 467 return; 468 } 469 470 req->cb(ctx, resp); 471 } 472 473 static void hpre_stop_qp_and_put(struct hisi_qp *qp) 474 { 475 hisi_qm_stop_qp(qp); 476 hisi_qm_free_qps(&qp, 1); 477 } 478 479 static int hpre_ctx_init(struct hpre_ctx *ctx, u8 type) 480 { 481 struct hisi_qp *qp; 482 int ret; 483 484 qp = hpre_get_qp_and_start(type); 485 if (IS_ERR(qp)) 486 return PTR_ERR(qp); 487 488 qp->qp_ctx = ctx; 489 qp->req_cb = hpre_alg_cb; 490 491 ret = hpre_ctx_set(ctx, qp, QM_Q_DEPTH); 492 if (ret) 493 hpre_stop_qp_and_put(qp); 494 495 return ret; 496 } 497 498 static int hpre_msg_request_set(struct hpre_ctx *ctx, void *req, bool is_rsa) 499 { 500 struct hpre_asym_request *h_req; 501 struct hpre_sqe *msg; 502 int req_id; 503 void *tmp; 504 505 if (is_rsa) { 506 struct akcipher_request *akreq = req; 507 508 if (akreq->dst_len < ctx->key_sz) { 509 akreq->dst_len = ctx->key_sz; 510 return -EOVERFLOW; 511 } 512 513 tmp = akcipher_request_ctx(akreq); 514 h_req = PTR_ALIGN(tmp, HPRE_ALIGN_SZ); 515 h_req->cb = hpre_rsa_cb; 516 h_req->areq.rsa = akreq; 517 msg = &h_req->req; 518 memset(msg, 0, sizeof(*msg)); 519 } else { 520 struct kpp_request *kreq = req; 521 522 if (kreq->dst_len < ctx->key_sz) { 523 kreq->dst_len = ctx->key_sz; 524 return -EOVERFLOW; 525 } 526 527 tmp = kpp_request_ctx(kreq); 528 h_req = PTR_ALIGN(tmp, HPRE_ALIGN_SZ); 529 h_req->cb = hpre_dh_cb; 530 h_req->areq.dh = kreq; 531 msg = &h_req->req; 532 memset(msg, 0, sizeof(*msg)); 533 msg->key = cpu_to_le64(ctx->dh.dma_xa_p); 534 } 535 536 msg->in = cpu_to_le64(DMA_MAPPING_ERROR); 537 msg->out = cpu_to_le64(DMA_MAPPING_ERROR); 538 msg->dw0 |= cpu_to_le32(0x1 << HPRE_SQE_DONE_SHIFT); 539 msg->task_len1 = (ctx->key_sz >> HPRE_BITS_2_BYTES_SHIFT) - 1; 540 h_req->ctx = ctx; 541 542 req_id = hpre_add_req_to_ctx(h_req); 543 if (req_id < 0) 544 return -EBUSY; 545 546 msg->tag = cpu_to_le16((u16)req_id); 547 548 return 0; 549 } 550 551 static int hpre_send(struct hpre_ctx *ctx, struct hpre_sqe *msg) 552 { 553 struct hpre_dfx *dfx = ctx->hpre->debug.dfx; 554 int ctr = 0; 555 int ret; 556 557 do { 558 atomic64_inc(&dfx[HPRE_SEND_CNT].value); 559 ret = hisi_qp_send(ctx->qp, msg); 560 if (ret != -EBUSY) 561 break; 562 atomic64_inc(&dfx[HPRE_SEND_BUSY_CNT].value); 563 } while (ctr++ < HPRE_TRY_SEND_TIMES); 564 565 if (likely(!ret)) 566 return ret; 567 568 if (ret != -EBUSY) 569 atomic64_inc(&dfx[HPRE_SEND_FAIL_CNT].value); 570 571 return ret; 572 } 573 574 static int hpre_dh_compute_value(struct kpp_request *req) 575 { 576 struct crypto_kpp *tfm = crypto_kpp_reqtfm(req); 577 struct hpre_ctx *ctx = kpp_tfm_ctx(tfm); 578 void *tmp = kpp_request_ctx(req); 579 struct hpre_asym_request *hpre_req = PTR_ALIGN(tmp, HPRE_ALIGN_SZ); 580 struct hpre_sqe *msg = &hpre_req->req; 581 int ret; 582 583 ret = hpre_msg_request_set(ctx, req, false); 584 if (unlikely(ret)) 585 return ret; 586 587 if (req->src) { 588 ret = hpre_hw_data_init(hpre_req, req->src, req->src_len, 1, 1); 589 if (unlikely(ret)) 590 goto clear_all; 591 } else { 592 msg->in = cpu_to_le64(ctx->dh.dma_g); 593 } 594 595 ret = hpre_hw_data_init(hpre_req, req->dst, req->dst_len, 0, 1); 596 if (unlikely(ret)) 597 goto clear_all; 598 599 if (ctx->crt_g2_mode && !req->src) 600 msg->dw0 = cpu_to_le32(le32_to_cpu(msg->dw0) | HPRE_ALG_DH_G2); 601 else 602 msg->dw0 = cpu_to_le32(le32_to_cpu(msg->dw0) | HPRE_ALG_DH); 603 604 /* success */ 605 ret = hpre_send(ctx, msg); 606 if (likely(!ret)) 607 return -EINPROGRESS; 608 609 clear_all: 610 hpre_rm_req_from_ctx(hpre_req); 611 hpre_hw_data_clr_all(ctx, hpre_req, req->dst, req->src); 612 613 return ret; 614 } 615 616 static int hpre_is_dh_params_length_valid(unsigned int key_sz) 617 { 618 #define _HPRE_DH_GRP1 768 619 #define _HPRE_DH_GRP2 1024 620 #define _HPRE_DH_GRP5 1536 621 #define _HPRE_DH_GRP14 2048 622 #define _HPRE_DH_GRP15 3072 623 #define _HPRE_DH_GRP16 4096 624 switch (key_sz) { 625 case _HPRE_DH_GRP1: 626 case _HPRE_DH_GRP2: 627 case _HPRE_DH_GRP5: 628 case _HPRE_DH_GRP14: 629 case _HPRE_DH_GRP15: 630 case _HPRE_DH_GRP16: 631 return 0; 632 default: 633 return -EINVAL; 634 } 635 } 636 637 static int hpre_dh_set_params(struct hpre_ctx *ctx, struct dh *params) 638 { 639 struct device *dev = ctx->dev; 640 unsigned int sz; 641 642 if (params->p_size > HPRE_DH_MAX_P_SZ) 643 return -EINVAL; 644 645 if (hpre_is_dh_params_length_valid(params->p_size << 646 HPRE_BITS_2_BYTES_SHIFT)) 647 return -EINVAL; 648 649 sz = ctx->key_sz = params->p_size; 650 ctx->dh.xa_p = dma_alloc_coherent(dev, sz << 1, 651 &ctx->dh.dma_xa_p, GFP_KERNEL); 652 if (!ctx->dh.xa_p) 653 return -ENOMEM; 654 655 memcpy(ctx->dh.xa_p + sz, params->p, sz); 656 657 /* If g equals 2 don't copy it */ 658 if (params->g_size == 1 && *(char *)params->g == HPRE_DH_G_FLAG) { 659 ctx->crt_g2_mode = true; 660 return 0; 661 } 662 663 ctx->dh.g = dma_alloc_coherent(dev, sz, &ctx->dh.dma_g, GFP_KERNEL); 664 if (!ctx->dh.g) { 665 dma_free_coherent(dev, sz << 1, ctx->dh.xa_p, 666 ctx->dh.dma_xa_p); 667 ctx->dh.xa_p = NULL; 668 return -ENOMEM; 669 } 670 671 memcpy(ctx->dh.g + (sz - params->g_size), params->g, params->g_size); 672 673 return 0; 674 } 675 676 static void hpre_dh_clear_ctx(struct hpre_ctx *ctx, bool is_clear_all) 677 { 678 struct device *dev = ctx->dev; 679 unsigned int sz = ctx->key_sz; 680 681 if (is_clear_all) 682 hisi_qm_stop_qp(ctx->qp); 683 684 if (ctx->dh.g) { 685 dma_free_coherent(dev, sz, ctx->dh.g, ctx->dh.dma_g); 686 ctx->dh.g = NULL; 687 } 688 689 if (ctx->dh.xa_p) { 690 memzero_explicit(ctx->dh.xa_p, sz); 691 dma_free_coherent(dev, sz << 1, ctx->dh.xa_p, 692 ctx->dh.dma_xa_p); 693 ctx->dh.xa_p = NULL; 694 } 695 696 hpre_ctx_clear(ctx, is_clear_all); 697 } 698 699 static int hpre_dh_set_secret(struct crypto_kpp *tfm, const void *buf, 700 unsigned int len) 701 { 702 struct hpre_ctx *ctx = kpp_tfm_ctx(tfm); 703 struct dh params; 704 int ret; 705 706 if (crypto_dh_decode_key(buf, len, ¶ms) < 0) 707 return -EINVAL; 708 709 /* Free old secret if any */ 710 hpre_dh_clear_ctx(ctx, false); 711 712 ret = hpre_dh_set_params(ctx, ¶ms); 713 if (ret < 0) 714 goto err_clear_ctx; 715 716 memcpy(ctx->dh.xa_p + (ctx->key_sz - params.key_size), params.key, 717 params.key_size); 718 719 return 0; 720 721 err_clear_ctx: 722 hpre_dh_clear_ctx(ctx, false); 723 return ret; 724 } 725 726 static unsigned int hpre_dh_max_size(struct crypto_kpp *tfm) 727 { 728 struct hpre_ctx *ctx = kpp_tfm_ctx(tfm); 729 730 return ctx->key_sz; 731 } 732 733 static int hpre_dh_init_tfm(struct crypto_kpp *tfm) 734 { 735 struct hpre_ctx *ctx = kpp_tfm_ctx(tfm); 736 737 return hpre_ctx_init(ctx, HPRE_V2_ALG_TYPE); 738 } 739 740 static void hpre_dh_exit_tfm(struct crypto_kpp *tfm) 741 { 742 struct hpre_ctx *ctx = kpp_tfm_ctx(tfm); 743 744 hpre_dh_clear_ctx(ctx, true); 745 } 746 747 static void hpre_rsa_drop_leading_zeros(const char **ptr, size_t *len) 748 { 749 while (!**ptr && *len) { 750 (*ptr)++; 751 (*len)--; 752 } 753 } 754 755 static bool hpre_rsa_key_size_is_support(unsigned int len) 756 { 757 unsigned int bits = len << HPRE_BITS_2_BYTES_SHIFT; 758 759 #define _RSA_1024BITS_KEY_WDTH 1024 760 #define _RSA_2048BITS_KEY_WDTH 2048 761 #define _RSA_3072BITS_KEY_WDTH 3072 762 #define _RSA_4096BITS_KEY_WDTH 4096 763 764 switch (bits) { 765 case _RSA_1024BITS_KEY_WDTH: 766 case _RSA_2048BITS_KEY_WDTH: 767 case _RSA_3072BITS_KEY_WDTH: 768 case _RSA_4096BITS_KEY_WDTH: 769 return true; 770 default: 771 return false; 772 } 773 } 774 775 static int hpre_rsa_enc(struct akcipher_request *req) 776 { 777 struct crypto_akcipher *tfm = crypto_akcipher_reqtfm(req); 778 struct hpre_ctx *ctx = akcipher_tfm_ctx(tfm); 779 void *tmp = akcipher_request_ctx(req); 780 struct hpre_asym_request *hpre_req = PTR_ALIGN(tmp, HPRE_ALIGN_SZ); 781 struct hpre_sqe *msg = &hpre_req->req; 782 int ret; 783 784 /* For 512 and 1536 bits key size, use soft tfm instead */ 785 if (ctx->key_sz == HPRE_RSA_512BITS_KSZ || 786 ctx->key_sz == HPRE_RSA_1536BITS_KSZ) { 787 akcipher_request_set_tfm(req, ctx->rsa.soft_tfm); 788 ret = crypto_akcipher_encrypt(req); 789 akcipher_request_set_tfm(req, tfm); 790 return ret; 791 } 792 793 if (unlikely(!ctx->rsa.pubkey)) 794 return -EINVAL; 795 796 ret = hpre_msg_request_set(ctx, req, true); 797 if (unlikely(ret)) 798 return ret; 799 800 msg->dw0 |= cpu_to_le32(HPRE_ALG_NC_NCRT); 801 msg->key = cpu_to_le64(ctx->rsa.dma_pubkey); 802 803 ret = hpre_hw_data_init(hpre_req, req->src, req->src_len, 1, 0); 804 if (unlikely(ret)) 805 goto clear_all; 806 807 ret = hpre_hw_data_init(hpre_req, req->dst, req->dst_len, 0, 0); 808 if (unlikely(ret)) 809 goto clear_all; 810 811 /* success */ 812 ret = hpre_send(ctx, msg); 813 if (likely(!ret)) 814 return -EINPROGRESS; 815 816 clear_all: 817 hpre_rm_req_from_ctx(hpre_req); 818 hpre_hw_data_clr_all(ctx, hpre_req, req->dst, req->src); 819 820 return ret; 821 } 822 823 static int hpre_rsa_dec(struct akcipher_request *req) 824 { 825 struct crypto_akcipher *tfm = crypto_akcipher_reqtfm(req); 826 struct hpre_ctx *ctx = akcipher_tfm_ctx(tfm); 827 void *tmp = akcipher_request_ctx(req); 828 struct hpre_asym_request *hpre_req = PTR_ALIGN(tmp, HPRE_ALIGN_SZ); 829 struct hpre_sqe *msg = &hpre_req->req; 830 int ret; 831 832 /* For 512 and 1536 bits key size, use soft tfm instead */ 833 if (ctx->key_sz == HPRE_RSA_512BITS_KSZ || 834 ctx->key_sz == HPRE_RSA_1536BITS_KSZ) { 835 akcipher_request_set_tfm(req, ctx->rsa.soft_tfm); 836 ret = crypto_akcipher_decrypt(req); 837 akcipher_request_set_tfm(req, tfm); 838 return ret; 839 } 840 841 if (unlikely(!ctx->rsa.prikey)) 842 return -EINVAL; 843 844 ret = hpre_msg_request_set(ctx, req, true); 845 if (unlikely(ret)) 846 return ret; 847 848 if (ctx->crt_g2_mode) { 849 msg->key = cpu_to_le64(ctx->rsa.dma_crt_prikey); 850 msg->dw0 = cpu_to_le32(le32_to_cpu(msg->dw0) | 851 HPRE_ALG_NC_CRT); 852 } else { 853 msg->key = cpu_to_le64(ctx->rsa.dma_prikey); 854 msg->dw0 = cpu_to_le32(le32_to_cpu(msg->dw0) | 855 HPRE_ALG_NC_NCRT); 856 } 857 858 ret = hpre_hw_data_init(hpre_req, req->src, req->src_len, 1, 0); 859 if (unlikely(ret)) 860 goto clear_all; 861 862 ret = hpre_hw_data_init(hpre_req, req->dst, req->dst_len, 0, 0); 863 if (unlikely(ret)) 864 goto clear_all; 865 866 /* success */ 867 ret = hpre_send(ctx, msg); 868 if (likely(!ret)) 869 return -EINPROGRESS; 870 871 clear_all: 872 hpre_rm_req_from_ctx(hpre_req); 873 hpre_hw_data_clr_all(ctx, hpre_req, req->dst, req->src); 874 875 return ret; 876 } 877 878 static int hpre_rsa_set_n(struct hpre_ctx *ctx, const char *value, 879 size_t vlen, bool private) 880 { 881 const char *ptr = value; 882 883 hpre_rsa_drop_leading_zeros(&ptr, &vlen); 884 885 ctx->key_sz = vlen; 886 887 /* if invalid key size provided, we use software tfm */ 888 if (!hpre_rsa_key_size_is_support(ctx->key_sz)) 889 return 0; 890 891 ctx->rsa.pubkey = dma_alloc_coherent(ctx->dev, vlen << 1, 892 &ctx->rsa.dma_pubkey, 893 GFP_KERNEL); 894 if (!ctx->rsa.pubkey) 895 return -ENOMEM; 896 897 if (private) { 898 ctx->rsa.prikey = dma_alloc_coherent(ctx->dev, vlen << 1, 899 &ctx->rsa.dma_prikey, 900 GFP_KERNEL); 901 if (!ctx->rsa.prikey) { 902 dma_free_coherent(ctx->dev, vlen << 1, 903 ctx->rsa.pubkey, 904 ctx->rsa.dma_pubkey); 905 ctx->rsa.pubkey = NULL; 906 return -ENOMEM; 907 } 908 memcpy(ctx->rsa.prikey + vlen, ptr, vlen); 909 } 910 memcpy(ctx->rsa.pubkey + vlen, ptr, vlen); 911 912 /* Using hardware HPRE to do RSA */ 913 return 1; 914 } 915 916 static int hpre_rsa_set_e(struct hpre_ctx *ctx, const char *value, 917 size_t vlen) 918 { 919 const char *ptr = value; 920 921 hpre_rsa_drop_leading_zeros(&ptr, &vlen); 922 923 if (!ctx->key_sz || !vlen || vlen > ctx->key_sz) 924 return -EINVAL; 925 926 memcpy(ctx->rsa.pubkey + ctx->key_sz - vlen, ptr, vlen); 927 928 return 0; 929 } 930 931 static int hpre_rsa_set_d(struct hpre_ctx *ctx, const char *value, 932 size_t vlen) 933 { 934 const char *ptr = value; 935 936 hpre_rsa_drop_leading_zeros(&ptr, &vlen); 937 938 if (!ctx->key_sz || !vlen || vlen > ctx->key_sz) 939 return -EINVAL; 940 941 memcpy(ctx->rsa.prikey + ctx->key_sz - vlen, ptr, vlen); 942 943 return 0; 944 } 945 946 static int hpre_crt_para_get(char *para, size_t para_sz, 947 const char *raw, size_t raw_sz) 948 { 949 const char *ptr = raw; 950 size_t len = raw_sz; 951 952 hpre_rsa_drop_leading_zeros(&ptr, &len); 953 if (!len || len > para_sz) 954 return -EINVAL; 955 956 memcpy(para + para_sz - len, ptr, len); 957 958 return 0; 959 } 960 961 static int hpre_rsa_setkey_crt(struct hpre_ctx *ctx, struct rsa_key *rsa_key) 962 { 963 unsigned int hlf_ksz = ctx->key_sz >> 1; 964 struct device *dev = ctx->dev; 965 u64 offset; 966 int ret; 967 968 ctx->rsa.crt_prikey = dma_alloc_coherent(dev, hlf_ksz * HPRE_CRT_PRMS, 969 &ctx->rsa.dma_crt_prikey, 970 GFP_KERNEL); 971 if (!ctx->rsa.crt_prikey) 972 return -ENOMEM; 973 974 ret = hpre_crt_para_get(ctx->rsa.crt_prikey, hlf_ksz, 975 rsa_key->dq, rsa_key->dq_sz); 976 if (ret) 977 goto free_key; 978 979 offset = hlf_ksz; 980 ret = hpre_crt_para_get(ctx->rsa.crt_prikey + offset, hlf_ksz, 981 rsa_key->dp, rsa_key->dp_sz); 982 if (ret) 983 goto free_key; 984 985 offset = hlf_ksz * HPRE_CRT_Q; 986 ret = hpre_crt_para_get(ctx->rsa.crt_prikey + offset, hlf_ksz, 987 rsa_key->q, rsa_key->q_sz); 988 if (ret) 989 goto free_key; 990 991 offset = hlf_ksz * HPRE_CRT_P; 992 ret = hpre_crt_para_get(ctx->rsa.crt_prikey + offset, hlf_ksz, 993 rsa_key->p, rsa_key->p_sz); 994 if (ret) 995 goto free_key; 996 997 offset = hlf_ksz * HPRE_CRT_INV; 998 ret = hpre_crt_para_get(ctx->rsa.crt_prikey + offset, hlf_ksz, 999 rsa_key->qinv, rsa_key->qinv_sz); 1000 if (ret) 1001 goto free_key; 1002 1003 ctx->crt_g2_mode = true; 1004 1005 return 0; 1006 1007 free_key: 1008 offset = hlf_ksz * HPRE_CRT_PRMS; 1009 memzero_explicit(ctx->rsa.crt_prikey, offset); 1010 dma_free_coherent(dev, hlf_ksz * HPRE_CRT_PRMS, ctx->rsa.crt_prikey, 1011 ctx->rsa.dma_crt_prikey); 1012 ctx->rsa.crt_prikey = NULL; 1013 ctx->crt_g2_mode = false; 1014 1015 return ret; 1016 } 1017 1018 /* If it is clear all, all the resources of the QP will be cleaned. */ 1019 static void hpre_rsa_clear_ctx(struct hpre_ctx *ctx, bool is_clear_all) 1020 { 1021 unsigned int half_key_sz = ctx->key_sz >> 1; 1022 struct device *dev = ctx->dev; 1023 1024 if (is_clear_all) 1025 hisi_qm_stop_qp(ctx->qp); 1026 1027 if (ctx->rsa.pubkey) { 1028 dma_free_coherent(dev, ctx->key_sz << 1, 1029 ctx->rsa.pubkey, ctx->rsa.dma_pubkey); 1030 ctx->rsa.pubkey = NULL; 1031 } 1032 1033 if (ctx->rsa.crt_prikey) { 1034 memzero_explicit(ctx->rsa.crt_prikey, 1035 half_key_sz * HPRE_CRT_PRMS); 1036 dma_free_coherent(dev, half_key_sz * HPRE_CRT_PRMS, 1037 ctx->rsa.crt_prikey, ctx->rsa.dma_crt_prikey); 1038 ctx->rsa.crt_prikey = NULL; 1039 } 1040 1041 if (ctx->rsa.prikey) { 1042 memzero_explicit(ctx->rsa.prikey, ctx->key_sz); 1043 dma_free_coherent(dev, ctx->key_sz << 1, ctx->rsa.prikey, 1044 ctx->rsa.dma_prikey); 1045 ctx->rsa.prikey = NULL; 1046 } 1047 1048 hpre_ctx_clear(ctx, is_clear_all); 1049 } 1050 1051 /* 1052 * we should judge if it is CRT or not, 1053 * CRT: return true, N-CRT: return false . 1054 */ 1055 static bool hpre_is_crt_key(struct rsa_key *key) 1056 { 1057 u16 len = key->p_sz + key->q_sz + key->dp_sz + key->dq_sz + 1058 key->qinv_sz; 1059 1060 #define LEN_OF_NCRT_PARA 5 1061 1062 /* N-CRT less than 5 parameters */ 1063 return len > LEN_OF_NCRT_PARA; 1064 } 1065 1066 static int hpre_rsa_setkey(struct hpre_ctx *ctx, const void *key, 1067 unsigned int keylen, bool private) 1068 { 1069 struct rsa_key rsa_key; 1070 int ret; 1071 1072 hpre_rsa_clear_ctx(ctx, false); 1073 1074 if (private) 1075 ret = rsa_parse_priv_key(&rsa_key, key, keylen); 1076 else 1077 ret = rsa_parse_pub_key(&rsa_key, key, keylen); 1078 if (ret < 0) 1079 return ret; 1080 1081 ret = hpre_rsa_set_n(ctx, rsa_key.n, rsa_key.n_sz, private); 1082 if (ret <= 0) 1083 return ret; 1084 1085 if (private) { 1086 ret = hpre_rsa_set_d(ctx, rsa_key.d, rsa_key.d_sz); 1087 if (ret < 0) 1088 goto free; 1089 1090 if (hpre_is_crt_key(&rsa_key)) { 1091 ret = hpre_rsa_setkey_crt(ctx, &rsa_key); 1092 if (ret < 0) 1093 goto free; 1094 } 1095 } 1096 1097 ret = hpre_rsa_set_e(ctx, rsa_key.e, rsa_key.e_sz); 1098 if (ret < 0) 1099 goto free; 1100 1101 if ((private && !ctx->rsa.prikey) || !ctx->rsa.pubkey) { 1102 ret = -EINVAL; 1103 goto free; 1104 } 1105 1106 return 0; 1107 1108 free: 1109 hpre_rsa_clear_ctx(ctx, false); 1110 return ret; 1111 } 1112 1113 static int hpre_rsa_setpubkey(struct crypto_akcipher *tfm, const void *key, 1114 unsigned int keylen) 1115 { 1116 struct hpre_ctx *ctx = akcipher_tfm_ctx(tfm); 1117 int ret; 1118 1119 ret = crypto_akcipher_set_pub_key(ctx->rsa.soft_tfm, key, keylen); 1120 if (ret) 1121 return ret; 1122 1123 return hpre_rsa_setkey(ctx, key, keylen, false); 1124 } 1125 1126 static int hpre_rsa_setprivkey(struct crypto_akcipher *tfm, const void *key, 1127 unsigned int keylen) 1128 { 1129 struct hpre_ctx *ctx = akcipher_tfm_ctx(tfm); 1130 int ret; 1131 1132 ret = crypto_akcipher_set_priv_key(ctx->rsa.soft_tfm, key, keylen); 1133 if (ret) 1134 return ret; 1135 1136 return hpre_rsa_setkey(ctx, key, keylen, true); 1137 } 1138 1139 static unsigned int hpre_rsa_max_size(struct crypto_akcipher *tfm) 1140 { 1141 struct hpre_ctx *ctx = akcipher_tfm_ctx(tfm); 1142 1143 /* For 512 and 1536 bits key size, use soft tfm instead */ 1144 if (ctx->key_sz == HPRE_RSA_512BITS_KSZ || 1145 ctx->key_sz == HPRE_RSA_1536BITS_KSZ) 1146 return crypto_akcipher_maxsize(ctx->rsa.soft_tfm); 1147 1148 return ctx->key_sz; 1149 } 1150 1151 static int hpre_rsa_init_tfm(struct crypto_akcipher *tfm) 1152 { 1153 struct hpre_ctx *ctx = akcipher_tfm_ctx(tfm); 1154 int ret; 1155 1156 ctx->rsa.soft_tfm = crypto_alloc_akcipher("rsa-generic", 0, 0); 1157 if (IS_ERR(ctx->rsa.soft_tfm)) { 1158 pr_err("Can not alloc_akcipher!\n"); 1159 return PTR_ERR(ctx->rsa.soft_tfm); 1160 } 1161 1162 ret = hpre_ctx_init(ctx, HPRE_V2_ALG_TYPE); 1163 if (ret) 1164 crypto_free_akcipher(ctx->rsa.soft_tfm); 1165 1166 return ret; 1167 } 1168 1169 static void hpre_rsa_exit_tfm(struct crypto_akcipher *tfm) 1170 { 1171 struct hpre_ctx *ctx = akcipher_tfm_ctx(tfm); 1172 1173 hpre_rsa_clear_ctx(ctx, true); 1174 crypto_free_akcipher(ctx->rsa.soft_tfm); 1175 } 1176 1177 static void hpre_key_to_big_end(u8 *data, int len) 1178 { 1179 int i, j; 1180 u8 tmp; 1181 1182 for (i = 0; i < len / 2; i++) { 1183 j = len - i - 1; 1184 tmp = data[j]; 1185 data[j] = data[i]; 1186 data[i] = tmp; 1187 } 1188 } 1189 1190 static void hpre_ecc_clear_ctx(struct hpre_ctx *ctx, bool is_clear_all, 1191 bool is_ecdh) 1192 { 1193 struct device *dev = ctx->dev; 1194 unsigned int sz = ctx->key_sz; 1195 unsigned int shift = sz << 1; 1196 1197 if (is_clear_all) 1198 hisi_qm_stop_qp(ctx->qp); 1199 1200 if (is_ecdh && ctx->ecdh.p) { 1201 /* ecdh: p->a->k->b */ 1202 memzero_explicit(ctx->ecdh.p + shift, sz); 1203 dma_free_coherent(dev, sz << 3, ctx->ecdh.p, ctx->ecdh.dma_p); 1204 ctx->ecdh.p = NULL; 1205 } else if (!is_ecdh && ctx->curve25519.p) { 1206 /* curve25519: p->a->k */ 1207 memzero_explicit(ctx->curve25519.p + shift, sz); 1208 dma_free_coherent(dev, sz << 2, ctx->curve25519.p, 1209 ctx->curve25519.dma_p); 1210 ctx->curve25519.p = NULL; 1211 } 1212 1213 hpre_ctx_clear(ctx, is_clear_all); 1214 } 1215 1216 /* 1217 * The bits of 192/224/256/384/521 are supported by HPRE, 1218 * and convert the bits like: 1219 * bits<=256, bits=256; 256<bits<=384, bits=384; 384<bits<=576, bits=576; 1220 * If the parameter bit width is insufficient, then we fill in the 1221 * high-order zeros by soft, so TASK_LENGTH1 is 0x3/0x5/0x8; 1222 */ 1223 static unsigned int hpre_ecdh_supported_curve(unsigned short id) 1224 { 1225 switch (id) { 1226 case ECC_CURVE_NIST_P192: 1227 case ECC_CURVE_NIST_P256: 1228 return HPRE_ECC_HW256_KSZ_B; 1229 case ECC_CURVE_NIST_P384: 1230 return HPRE_ECC_HW384_KSZ_B; 1231 default: 1232 break; 1233 } 1234 1235 return 0; 1236 } 1237 1238 static void fill_curve_param(void *addr, u64 *param, unsigned int cur_sz, u8 ndigits) 1239 { 1240 unsigned int sz = cur_sz - (ndigits - 1) * sizeof(u64); 1241 u8 i = 0; 1242 1243 while (i < ndigits - 1) { 1244 memcpy(addr + sizeof(u64) * i, ¶m[i], sizeof(u64)); 1245 i++; 1246 } 1247 1248 memcpy(addr + sizeof(u64) * i, ¶m[ndigits - 1], sz); 1249 hpre_key_to_big_end((u8 *)addr, cur_sz); 1250 } 1251 1252 static int hpre_ecdh_fill_curve(struct hpre_ctx *ctx, struct ecdh *params, 1253 unsigned int cur_sz) 1254 { 1255 unsigned int shifta = ctx->key_sz << 1; 1256 unsigned int shiftb = ctx->key_sz << 2; 1257 void *p = ctx->ecdh.p + ctx->key_sz - cur_sz; 1258 void *a = ctx->ecdh.p + shifta - cur_sz; 1259 void *b = ctx->ecdh.p + shiftb - cur_sz; 1260 void *x = ctx->ecdh.g + ctx->key_sz - cur_sz; 1261 void *y = ctx->ecdh.g + shifta - cur_sz; 1262 const struct ecc_curve *curve = ecc_get_curve(ctx->curve_id); 1263 char *n; 1264 1265 if (unlikely(!curve)) 1266 return -EINVAL; 1267 1268 n = kzalloc(ctx->key_sz, GFP_KERNEL); 1269 if (!n) 1270 return -ENOMEM; 1271 1272 fill_curve_param(p, curve->p, cur_sz, curve->g.ndigits); 1273 fill_curve_param(a, curve->a, cur_sz, curve->g.ndigits); 1274 fill_curve_param(b, curve->b, cur_sz, curve->g.ndigits); 1275 fill_curve_param(x, curve->g.x, cur_sz, curve->g.ndigits); 1276 fill_curve_param(y, curve->g.y, cur_sz, curve->g.ndigits); 1277 fill_curve_param(n, curve->n, cur_sz, curve->g.ndigits); 1278 1279 if (params->key_size == cur_sz && memcmp(params->key, n, cur_sz) >= 0) { 1280 kfree(n); 1281 return -EINVAL; 1282 } 1283 1284 kfree(n); 1285 return 0; 1286 } 1287 1288 static unsigned int hpre_ecdh_get_curvesz(unsigned short id) 1289 { 1290 switch (id) { 1291 case ECC_CURVE_NIST_P192: 1292 return HPRE_ECC_NIST_P192_N_SIZE; 1293 case ECC_CURVE_NIST_P256: 1294 return HPRE_ECC_NIST_P256_N_SIZE; 1295 case ECC_CURVE_NIST_P384: 1296 return HPRE_ECC_NIST_P384_N_SIZE; 1297 default: 1298 break; 1299 } 1300 1301 return 0; 1302 } 1303 1304 static int hpre_ecdh_set_param(struct hpre_ctx *ctx, struct ecdh *params) 1305 { 1306 struct device *dev = ctx->dev; 1307 unsigned int sz, shift, curve_sz; 1308 int ret; 1309 1310 ctx->key_sz = hpre_ecdh_supported_curve(ctx->curve_id); 1311 if (!ctx->key_sz) 1312 return -EINVAL; 1313 1314 curve_sz = hpre_ecdh_get_curvesz(ctx->curve_id); 1315 if (!curve_sz || params->key_size > curve_sz) 1316 return -EINVAL; 1317 1318 sz = ctx->key_sz; 1319 1320 if (!ctx->ecdh.p) { 1321 ctx->ecdh.p = dma_alloc_coherent(dev, sz << 3, &ctx->ecdh.dma_p, 1322 GFP_KERNEL); 1323 if (!ctx->ecdh.p) 1324 return -ENOMEM; 1325 } 1326 1327 shift = sz << 2; 1328 ctx->ecdh.g = ctx->ecdh.p + shift; 1329 ctx->ecdh.dma_g = ctx->ecdh.dma_p + shift; 1330 1331 ret = hpre_ecdh_fill_curve(ctx, params, curve_sz); 1332 if (ret) { 1333 dev_err(dev, "failed to fill curve_param, ret = %d!\n", ret); 1334 dma_free_coherent(dev, sz << 3, ctx->ecdh.p, ctx->ecdh.dma_p); 1335 ctx->ecdh.p = NULL; 1336 return ret; 1337 } 1338 1339 return 0; 1340 } 1341 1342 static bool hpre_key_is_zero(char *key, unsigned short key_sz) 1343 { 1344 int i; 1345 1346 for (i = 0; i < key_sz; i++) 1347 if (key[i]) 1348 return false; 1349 1350 return true; 1351 } 1352 1353 static int ecdh_gen_privkey(struct hpre_ctx *ctx, struct ecdh *params) 1354 { 1355 struct device *dev = ctx->dev; 1356 int ret; 1357 1358 ret = crypto_get_default_rng(); 1359 if (ret) { 1360 dev_err(dev, "failed to get default rng, ret = %d!\n", ret); 1361 return ret; 1362 } 1363 1364 ret = crypto_rng_get_bytes(crypto_default_rng, (u8 *)params->key, 1365 params->key_size); 1366 crypto_put_default_rng(); 1367 if (ret) 1368 dev_err(dev, "failed to get rng, ret = %d!\n", ret); 1369 1370 return ret; 1371 } 1372 1373 static int hpre_ecdh_set_secret(struct crypto_kpp *tfm, const void *buf, 1374 unsigned int len) 1375 { 1376 struct hpre_ctx *ctx = kpp_tfm_ctx(tfm); 1377 struct device *dev = ctx->dev; 1378 char key[HPRE_ECC_MAX_KSZ]; 1379 unsigned int sz, sz_shift; 1380 struct ecdh params; 1381 int ret; 1382 1383 if (crypto_ecdh_decode_key(buf, len, ¶ms) < 0) { 1384 dev_err(dev, "failed to decode ecdh key!\n"); 1385 return -EINVAL; 1386 } 1387 1388 /* Use stdrng to generate private key */ 1389 if (!params.key || !params.key_size) { 1390 params.key = key; 1391 params.key_size = hpre_ecdh_get_curvesz(ctx->curve_id); 1392 ret = ecdh_gen_privkey(ctx, ¶ms); 1393 if (ret) 1394 return ret; 1395 } 1396 1397 if (hpre_key_is_zero(params.key, params.key_size)) { 1398 dev_err(dev, "Invalid hpre key!\n"); 1399 return -EINVAL; 1400 } 1401 1402 hpre_ecc_clear_ctx(ctx, false, true); 1403 1404 ret = hpre_ecdh_set_param(ctx, ¶ms); 1405 if (ret < 0) { 1406 dev_err(dev, "failed to set hpre param, ret = %d!\n", ret); 1407 return ret; 1408 } 1409 1410 sz = ctx->key_sz; 1411 sz_shift = (sz << 1) + sz - params.key_size; 1412 memcpy(ctx->ecdh.p + sz_shift, params.key, params.key_size); 1413 1414 return 0; 1415 } 1416 1417 static void hpre_ecdh_hw_data_clr_all(struct hpre_ctx *ctx, 1418 struct hpre_asym_request *req, 1419 struct scatterlist *dst, 1420 struct scatterlist *src) 1421 { 1422 struct device *dev = ctx->dev; 1423 struct hpre_sqe *sqe = &req->req; 1424 dma_addr_t dma; 1425 1426 dma = le64_to_cpu(sqe->in); 1427 if (unlikely(dma_mapping_error(dev, dma))) 1428 return; 1429 1430 if (src && req->src) 1431 dma_free_coherent(dev, ctx->key_sz << 2, req->src, dma); 1432 1433 dma = le64_to_cpu(sqe->out); 1434 if (unlikely(dma_mapping_error(dev, dma))) 1435 return; 1436 1437 if (req->dst) 1438 dma_free_coherent(dev, ctx->key_sz << 1, req->dst, dma); 1439 if (dst) 1440 dma_unmap_single(dev, dma, ctx->key_sz << 1, DMA_FROM_DEVICE); 1441 } 1442 1443 static void hpre_ecdh_cb(struct hpre_ctx *ctx, void *resp) 1444 { 1445 unsigned int curve_sz = hpre_ecdh_get_curvesz(ctx->curve_id); 1446 struct hpre_dfx *dfx = ctx->hpre->debug.dfx; 1447 struct hpre_asym_request *req = NULL; 1448 struct kpp_request *areq; 1449 u64 overtime_thrhld; 1450 char *p; 1451 int ret; 1452 1453 ret = hpre_alg_res_post_hf(ctx, resp, (void **)&req); 1454 areq = req->areq.ecdh; 1455 areq->dst_len = ctx->key_sz << 1; 1456 1457 overtime_thrhld = atomic64_read(&dfx[HPRE_OVERTIME_THRHLD].value); 1458 if (overtime_thrhld && hpre_is_bd_timeout(req, overtime_thrhld)) 1459 atomic64_inc(&dfx[HPRE_OVER_THRHLD_CNT].value); 1460 1461 p = sg_virt(areq->dst); 1462 memmove(p, p + ctx->key_sz - curve_sz, curve_sz); 1463 memmove(p + curve_sz, p + areq->dst_len - curve_sz, curve_sz); 1464 1465 hpre_ecdh_hw_data_clr_all(ctx, req, areq->dst, areq->src); 1466 kpp_request_complete(areq, ret); 1467 1468 atomic64_inc(&dfx[HPRE_RECV_CNT].value); 1469 } 1470 1471 static int hpre_ecdh_msg_request_set(struct hpre_ctx *ctx, 1472 struct kpp_request *req) 1473 { 1474 struct hpre_asym_request *h_req; 1475 struct hpre_sqe *msg; 1476 int req_id; 1477 void *tmp; 1478 1479 if (req->dst_len < ctx->key_sz << 1) { 1480 req->dst_len = ctx->key_sz << 1; 1481 return -EINVAL; 1482 } 1483 1484 tmp = kpp_request_ctx(req); 1485 h_req = PTR_ALIGN(tmp, HPRE_ALIGN_SZ); 1486 h_req->cb = hpre_ecdh_cb; 1487 h_req->areq.ecdh = req; 1488 msg = &h_req->req; 1489 memset(msg, 0, sizeof(*msg)); 1490 msg->in = cpu_to_le64(DMA_MAPPING_ERROR); 1491 msg->out = cpu_to_le64(DMA_MAPPING_ERROR); 1492 msg->key = cpu_to_le64(ctx->ecdh.dma_p); 1493 1494 msg->dw0 |= cpu_to_le32(0x1U << HPRE_SQE_DONE_SHIFT); 1495 msg->task_len1 = (ctx->key_sz >> HPRE_BITS_2_BYTES_SHIFT) - 1; 1496 h_req->ctx = ctx; 1497 1498 req_id = hpre_add_req_to_ctx(h_req); 1499 if (req_id < 0) 1500 return -EBUSY; 1501 1502 msg->tag = cpu_to_le16((u16)req_id); 1503 return 0; 1504 } 1505 1506 static int hpre_ecdh_src_data_init(struct hpre_asym_request *hpre_req, 1507 struct scatterlist *data, unsigned int len) 1508 { 1509 struct hpre_sqe *msg = &hpre_req->req; 1510 struct hpre_ctx *ctx = hpre_req->ctx; 1511 struct device *dev = ctx->dev; 1512 unsigned int tmpshift; 1513 dma_addr_t dma = 0; 1514 void *ptr; 1515 int shift; 1516 1517 /* Src_data include gx and gy. */ 1518 shift = ctx->key_sz - (len >> 1); 1519 if (unlikely(shift < 0)) 1520 return -EINVAL; 1521 1522 ptr = dma_alloc_coherent(dev, ctx->key_sz << 2, &dma, GFP_KERNEL); 1523 if (unlikely(!ptr)) 1524 return -ENOMEM; 1525 1526 tmpshift = ctx->key_sz << 1; 1527 scatterwalk_map_and_copy(ptr + tmpshift, data, 0, len, 0); 1528 memcpy(ptr + shift, ptr + tmpshift, len >> 1); 1529 memcpy(ptr + ctx->key_sz + shift, ptr + tmpshift + (len >> 1), len >> 1); 1530 1531 hpre_req->src = ptr; 1532 msg->in = cpu_to_le64(dma); 1533 return 0; 1534 } 1535 1536 static int hpre_ecdh_dst_data_init(struct hpre_asym_request *hpre_req, 1537 struct scatterlist *data, unsigned int len) 1538 { 1539 struct hpre_sqe *msg = &hpre_req->req; 1540 struct hpre_ctx *ctx = hpre_req->ctx; 1541 struct device *dev = ctx->dev; 1542 dma_addr_t dma; 1543 1544 if (unlikely(!data || !sg_is_last(data) || len != ctx->key_sz << 1)) { 1545 dev_err(dev, "data or data length is illegal!\n"); 1546 return -EINVAL; 1547 } 1548 1549 hpre_req->dst = NULL; 1550 dma = dma_map_single(dev, sg_virt(data), len, DMA_FROM_DEVICE); 1551 if (unlikely(dma_mapping_error(dev, dma))) { 1552 dev_err(dev, "dma map data err!\n"); 1553 return -ENOMEM; 1554 } 1555 1556 msg->out = cpu_to_le64(dma); 1557 return 0; 1558 } 1559 1560 static int hpre_ecdh_compute_value(struct kpp_request *req) 1561 { 1562 struct crypto_kpp *tfm = crypto_kpp_reqtfm(req); 1563 struct hpre_ctx *ctx = kpp_tfm_ctx(tfm); 1564 struct device *dev = ctx->dev; 1565 void *tmp = kpp_request_ctx(req); 1566 struct hpre_asym_request *hpre_req = PTR_ALIGN(tmp, HPRE_ALIGN_SZ); 1567 struct hpre_sqe *msg = &hpre_req->req; 1568 int ret; 1569 1570 ret = hpre_ecdh_msg_request_set(ctx, req); 1571 if (unlikely(ret)) { 1572 dev_err(dev, "failed to set ecdh request, ret = %d!\n", ret); 1573 return ret; 1574 } 1575 1576 if (req->src) { 1577 ret = hpre_ecdh_src_data_init(hpre_req, req->src, req->src_len); 1578 if (unlikely(ret)) { 1579 dev_err(dev, "failed to init src data, ret = %d!\n", ret); 1580 goto clear_all; 1581 } 1582 } else { 1583 msg->in = cpu_to_le64(ctx->ecdh.dma_g); 1584 } 1585 1586 ret = hpre_ecdh_dst_data_init(hpre_req, req->dst, req->dst_len); 1587 if (unlikely(ret)) { 1588 dev_err(dev, "failed to init dst data, ret = %d!\n", ret); 1589 goto clear_all; 1590 } 1591 1592 msg->dw0 = cpu_to_le32(le32_to_cpu(msg->dw0) | HPRE_ALG_ECC_MUL); 1593 ret = hpre_send(ctx, msg); 1594 if (likely(!ret)) 1595 return -EINPROGRESS; 1596 1597 clear_all: 1598 hpre_rm_req_from_ctx(hpre_req); 1599 hpre_ecdh_hw_data_clr_all(ctx, hpre_req, req->dst, req->src); 1600 return ret; 1601 } 1602 1603 static unsigned int hpre_ecdh_max_size(struct crypto_kpp *tfm) 1604 { 1605 struct hpre_ctx *ctx = kpp_tfm_ctx(tfm); 1606 1607 /* max size is the pub_key_size, include x and y */ 1608 return ctx->key_sz << 1; 1609 } 1610 1611 static int hpre_ecdh_nist_p192_init_tfm(struct crypto_kpp *tfm) 1612 { 1613 struct hpre_ctx *ctx = kpp_tfm_ctx(tfm); 1614 1615 ctx->curve_id = ECC_CURVE_NIST_P192; 1616 1617 return hpre_ctx_init(ctx, HPRE_V3_ECC_ALG_TYPE); 1618 } 1619 1620 static int hpre_ecdh_nist_p256_init_tfm(struct crypto_kpp *tfm) 1621 { 1622 struct hpre_ctx *ctx = kpp_tfm_ctx(tfm); 1623 1624 ctx->curve_id = ECC_CURVE_NIST_P256; 1625 1626 return hpre_ctx_init(ctx, HPRE_V3_ECC_ALG_TYPE); 1627 } 1628 1629 static int hpre_ecdh_nist_p384_init_tfm(struct crypto_kpp *tfm) 1630 { 1631 struct hpre_ctx *ctx = kpp_tfm_ctx(tfm); 1632 1633 ctx->curve_id = ECC_CURVE_NIST_P384; 1634 1635 return hpre_ctx_init(ctx, HPRE_V3_ECC_ALG_TYPE); 1636 } 1637 1638 static void hpre_ecdh_exit_tfm(struct crypto_kpp *tfm) 1639 { 1640 struct hpre_ctx *ctx = kpp_tfm_ctx(tfm); 1641 1642 hpre_ecc_clear_ctx(ctx, true, true); 1643 } 1644 1645 static void hpre_curve25519_fill_curve(struct hpre_ctx *ctx, const void *buf, 1646 unsigned int len) 1647 { 1648 u8 secret[CURVE25519_KEY_SIZE] = { 0 }; 1649 unsigned int sz = ctx->key_sz; 1650 const struct ecc_curve *curve; 1651 unsigned int shift = sz << 1; 1652 void *p; 1653 1654 /* 1655 * The key from 'buf' is in little-endian, we should preprocess it as 1656 * the description in rfc7748: "k[0] &= 248, k[31] &= 127, k[31] |= 64", 1657 * then convert it to big endian. Only in this way, the result can be 1658 * the same as the software curve-25519 that exists in crypto. 1659 */ 1660 memcpy(secret, buf, len); 1661 curve25519_clamp_secret(secret); 1662 hpre_key_to_big_end(secret, CURVE25519_KEY_SIZE); 1663 1664 p = ctx->curve25519.p + sz - len; 1665 1666 curve = ecc_get_curve25519(); 1667 1668 /* fill curve parameters */ 1669 fill_curve_param(p, curve->p, len, curve->g.ndigits); 1670 fill_curve_param(p + sz, curve->a, len, curve->g.ndigits); 1671 memcpy(p + shift, secret, len); 1672 fill_curve_param(p + shift + sz, curve->g.x, len, curve->g.ndigits); 1673 memzero_explicit(secret, CURVE25519_KEY_SIZE); 1674 } 1675 1676 static int hpre_curve25519_set_param(struct hpre_ctx *ctx, const void *buf, 1677 unsigned int len) 1678 { 1679 struct device *dev = ctx->dev; 1680 unsigned int sz = ctx->key_sz; 1681 unsigned int shift = sz << 1; 1682 1683 /* p->a->k->gx */ 1684 if (!ctx->curve25519.p) { 1685 ctx->curve25519.p = dma_alloc_coherent(dev, sz << 2, 1686 &ctx->curve25519.dma_p, 1687 GFP_KERNEL); 1688 if (!ctx->curve25519.p) 1689 return -ENOMEM; 1690 } 1691 1692 ctx->curve25519.g = ctx->curve25519.p + shift + sz; 1693 ctx->curve25519.dma_g = ctx->curve25519.dma_p + shift + sz; 1694 1695 hpre_curve25519_fill_curve(ctx, buf, len); 1696 1697 return 0; 1698 } 1699 1700 static int hpre_curve25519_set_secret(struct crypto_kpp *tfm, const void *buf, 1701 unsigned int len) 1702 { 1703 struct hpre_ctx *ctx = kpp_tfm_ctx(tfm); 1704 struct device *dev = ctx->dev; 1705 int ret = -EINVAL; 1706 1707 if (len != CURVE25519_KEY_SIZE || 1708 !crypto_memneq(buf, curve25519_null_point, CURVE25519_KEY_SIZE)) { 1709 dev_err(dev, "key is null or key len is not 32bytes!\n"); 1710 return ret; 1711 } 1712 1713 /* Free old secret if any */ 1714 hpre_ecc_clear_ctx(ctx, false, false); 1715 1716 ctx->key_sz = CURVE25519_KEY_SIZE; 1717 ret = hpre_curve25519_set_param(ctx, buf, CURVE25519_KEY_SIZE); 1718 if (ret) { 1719 dev_err(dev, "failed to set curve25519 param, ret = %d!\n", ret); 1720 hpre_ecc_clear_ctx(ctx, false, false); 1721 return ret; 1722 } 1723 1724 return 0; 1725 } 1726 1727 static void hpre_curve25519_hw_data_clr_all(struct hpre_ctx *ctx, 1728 struct hpre_asym_request *req, 1729 struct scatterlist *dst, 1730 struct scatterlist *src) 1731 { 1732 struct device *dev = ctx->dev; 1733 struct hpre_sqe *sqe = &req->req; 1734 dma_addr_t dma; 1735 1736 dma = le64_to_cpu(sqe->in); 1737 if (unlikely(dma_mapping_error(dev, dma))) 1738 return; 1739 1740 if (src && req->src) 1741 dma_free_coherent(dev, ctx->key_sz, req->src, dma); 1742 1743 dma = le64_to_cpu(sqe->out); 1744 if (unlikely(dma_mapping_error(dev, dma))) 1745 return; 1746 1747 if (req->dst) 1748 dma_free_coherent(dev, ctx->key_sz, req->dst, dma); 1749 if (dst) 1750 dma_unmap_single(dev, dma, ctx->key_sz, DMA_FROM_DEVICE); 1751 } 1752 1753 static void hpre_curve25519_cb(struct hpre_ctx *ctx, void *resp) 1754 { 1755 struct hpre_dfx *dfx = ctx->hpre->debug.dfx; 1756 struct hpre_asym_request *req = NULL; 1757 struct kpp_request *areq; 1758 u64 overtime_thrhld; 1759 int ret; 1760 1761 ret = hpre_alg_res_post_hf(ctx, resp, (void **)&req); 1762 areq = req->areq.curve25519; 1763 areq->dst_len = ctx->key_sz; 1764 1765 overtime_thrhld = atomic64_read(&dfx[HPRE_OVERTIME_THRHLD].value); 1766 if (overtime_thrhld && hpre_is_bd_timeout(req, overtime_thrhld)) 1767 atomic64_inc(&dfx[HPRE_OVER_THRHLD_CNT].value); 1768 1769 hpre_key_to_big_end(sg_virt(areq->dst), CURVE25519_KEY_SIZE); 1770 1771 hpre_curve25519_hw_data_clr_all(ctx, req, areq->dst, areq->src); 1772 kpp_request_complete(areq, ret); 1773 1774 atomic64_inc(&dfx[HPRE_RECV_CNT].value); 1775 } 1776 1777 static int hpre_curve25519_msg_request_set(struct hpre_ctx *ctx, 1778 struct kpp_request *req) 1779 { 1780 struct hpre_asym_request *h_req; 1781 struct hpre_sqe *msg; 1782 int req_id; 1783 void *tmp; 1784 1785 if (unlikely(req->dst_len < ctx->key_sz)) { 1786 req->dst_len = ctx->key_sz; 1787 return -EINVAL; 1788 } 1789 1790 tmp = kpp_request_ctx(req); 1791 h_req = PTR_ALIGN(tmp, HPRE_ALIGN_SZ); 1792 h_req->cb = hpre_curve25519_cb; 1793 h_req->areq.curve25519 = req; 1794 msg = &h_req->req; 1795 memset(msg, 0, sizeof(*msg)); 1796 msg->in = cpu_to_le64(DMA_MAPPING_ERROR); 1797 msg->out = cpu_to_le64(DMA_MAPPING_ERROR); 1798 msg->key = cpu_to_le64(ctx->curve25519.dma_p); 1799 1800 msg->dw0 |= cpu_to_le32(0x1U << HPRE_SQE_DONE_SHIFT); 1801 msg->task_len1 = (ctx->key_sz >> HPRE_BITS_2_BYTES_SHIFT) - 1; 1802 h_req->ctx = ctx; 1803 1804 req_id = hpre_add_req_to_ctx(h_req); 1805 if (req_id < 0) 1806 return -EBUSY; 1807 1808 msg->tag = cpu_to_le16((u16)req_id); 1809 return 0; 1810 } 1811 1812 static void hpre_curve25519_src_modulo_p(u8 *ptr) 1813 { 1814 int i; 1815 1816 for (i = 0; i < CURVE25519_KEY_SIZE - 1; i++) 1817 ptr[i] = 0; 1818 1819 /* The modulus is ptr's last byte minus '0xed'(last byte of p) */ 1820 ptr[i] -= 0xed; 1821 } 1822 1823 static int hpre_curve25519_src_init(struct hpre_asym_request *hpre_req, 1824 struct scatterlist *data, unsigned int len) 1825 { 1826 struct hpre_sqe *msg = &hpre_req->req; 1827 struct hpre_ctx *ctx = hpre_req->ctx; 1828 struct device *dev = ctx->dev; 1829 u8 p[CURVE25519_KEY_SIZE] = { 0 }; 1830 const struct ecc_curve *curve; 1831 dma_addr_t dma = 0; 1832 u8 *ptr; 1833 1834 if (len != CURVE25519_KEY_SIZE) { 1835 dev_err(dev, "sourc_data len is not 32bytes, len = %u!\n", len); 1836 return -EINVAL; 1837 } 1838 1839 ptr = dma_alloc_coherent(dev, ctx->key_sz, &dma, GFP_KERNEL); 1840 if (unlikely(!ptr)) 1841 return -ENOMEM; 1842 1843 scatterwalk_map_and_copy(ptr, data, 0, len, 0); 1844 1845 if (!crypto_memneq(ptr, curve25519_null_point, CURVE25519_KEY_SIZE)) { 1846 dev_err(dev, "gx is null!\n"); 1847 goto err; 1848 } 1849 1850 /* 1851 * Src_data(gx) is in little-endian order, MSB in the final byte should 1852 * be masked as described in RFC7748, then transform it to big-endian 1853 * form, then hisi_hpre can use the data. 1854 */ 1855 ptr[31] &= 0x7f; 1856 hpre_key_to_big_end(ptr, CURVE25519_KEY_SIZE); 1857 1858 curve = ecc_get_curve25519(); 1859 1860 fill_curve_param(p, curve->p, CURVE25519_KEY_SIZE, curve->g.ndigits); 1861 1862 /* 1863 * When src_data equals (2^255 - 19) ~ (2^255 - 1), it is out of p, 1864 * we get its modulus to p, and then use it. 1865 */ 1866 if (memcmp(ptr, p, ctx->key_sz) == 0) { 1867 dev_err(dev, "gx is p!\n"); 1868 return -EINVAL; 1869 } else if (memcmp(ptr, p, ctx->key_sz) > 0) { 1870 hpre_curve25519_src_modulo_p(ptr); 1871 } 1872 1873 hpre_req->src = ptr; 1874 msg->in = cpu_to_le64(dma); 1875 return 0; 1876 1877 err: 1878 dma_free_coherent(dev, ctx->key_sz, ptr, dma); 1879 return -EINVAL; 1880 } 1881 1882 static int hpre_curve25519_dst_init(struct hpre_asym_request *hpre_req, 1883 struct scatterlist *data, unsigned int len) 1884 { 1885 struct hpre_sqe *msg = &hpre_req->req; 1886 struct hpre_ctx *ctx = hpre_req->ctx; 1887 struct device *dev = ctx->dev; 1888 dma_addr_t dma; 1889 1890 if (!data || !sg_is_last(data) || len != ctx->key_sz) { 1891 dev_err(dev, "data or data length is illegal!\n"); 1892 return -EINVAL; 1893 } 1894 1895 hpre_req->dst = NULL; 1896 dma = dma_map_single(dev, sg_virt(data), len, DMA_FROM_DEVICE); 1897 if (unlikely(dma_mapping_error(dev, dma))) { 1898 dev_err(dev, "dma map data err!\n"); 1899 return -ENOMEM; 1900 } 1901 1902 msg->out = cpu_to_le64(dma); 1903 return 0; 1904 } 1905 1906 static int hpre_curve25519_compute_value(struct kpp_request *req) 1907 { 1908 struct crypto_kpp *tfm = crypto_kpp_reqtfm(req); 1909 struct hpre_ctx *ctx = kpp_tfm_ctx(tfm); 1910 struct device *dev = ctx->dev; 1911 void *tmp = kpp_request_ctx(req); 1912 struct hpre_asym_request *hpre_req = PTR_ALIGN(tmp, HPRE_ALIGN_SZ); 1913 struct hpre_sqe *msg = &hpre_req->req; 1914 int ret; 1915 1916 ret = hpre_curve25519_msg_request_set(ctx, req); 1917 if (unlikely(ret)) { 1918 dev_err(dev, "failed to set curve25519 request, ret = %d!\n", ret); 1919 return ret; 1920 } 1921 1922 if (req->src) { 1923 ret = hpre_curve25519_src_init(hpre_req, req->src, req->src_len); 1924 if (unlikely(ret)) { 1925 dev_err(dev, "failed to init src data, ret = %d!\n", 1926 ret); 1927 goto clear_all; 1928 } 1929 } else { 1930 msg->in = cpu_to_le64(ctx->curve25519.dma_g); 1931 } 1932 1933 ret = hpre_curve25519_dst_init(hpre_req, req->dst, req->dst_len); 1934 if (unlikely(ret)) { 1935 dev_err(dev, "failed to init dst data, ret = %d!\n", ret); 1936 goto clear_all; 1937 } 1938 1939 msg->dw0 = cpu_to_le32(le32_to_cpu(msg->dw0) | HPRE_ALG_CURVE25519_MUL); 1940 ret = hpre_send(ctx, msg); 1941 if (likely(!ret)) 1942 return -EINPROGRESS; 1943 1944 clear_all: 1945 hpre_rm_req_from_ctx(hpre_req); 1946 hpre_curve25519_hw_data_clr_all(ctx, hpre_req, req->dst, req->src); 1947 return ret; 1948 } 1949 1950 static unsigned int hpre_curve25519_max_size(struct crypto_kpp *tfm) 1951 { 1952 struct hpre_ctx *ctx = kpp_tfm_ctx(tfm); 1953 1954 return ctx->key_sz; 1955 } 1956 1957 static int hpre_curve25519_init_tfm(struct crypto_kpp *tfm) 1958 { 1959 struct hpre_ctx *ctx = kpp_tfm_ctx(tfm); 1960 1961 return hpre_ctx_init(ctx, HPRE_V3_ECC_ALG_TYPE); 1962 } 1963 1964 static void hpre_curve25519_exit_tfm(struct crypto_kpp *tfm) 1965 { 1966 struct hpre_ctx *ctx = kpp_tfm_ctx(tfm); 1967 1968 hpre_ecc_clear_ctx(ctx, true, false); 1969 } 1970 1971 static struct akcipher_alg rsa = { 1972 .sign = hpre_rsa_dec, 1973 .verify = hpre_rsa_enc, 1974 .encrypt = hpre_rsa_enc, 1975 .decrypt = hpre_rsa_dec, 1976 .set_pub_key = hpre_rsa_setpubkey, 1977 .set_priv_key = hpre_rsa_setprivkey, 1978 .max_size = hpre_rsa_max_size, 1979 .init = hpre_rsa_init_tfm, 1980 .exit = hpre_rsa_exit_tfm, 1981 .reqsize = sizeof(struct hpre_asym_request) + HPRE_ALIGN_SZ, 1982 .base = { 1983 .cra_ctxsize = sizeof(struct hpre_ctx), 1984 .cra_priority = HPRE_CRYPTO_ALG_PRI, 1985 .cra_name = "rsa", 1986 .cra_driver_name = "hpre-rsa", 1987 .cra_module = THIS_MODULE, 1988 }, 1989 }; 1990 1991 static struct kpp_alg dh = { 1992 .set_secret = hpre_dh_set_secret, 1993 .generate_public_key = hpre_dh_compute_value, 1994 .compute_shared_secret = hpre_dh_compute_value, 1995 .max_size = hpre_dh_max_size, 1996 .init = hpre_dh_init_tfm, 1997 .exit = hpre_dh_exit_tfm, 1998 .reqsize = sizeof(struct hpre_asym_request) + HPRE_ALIGN_SZ, 1999 .base = { 2000 .cra_ctxsize = sizeof(struct hpre_ctx), 2001 .cra_priority = HPRE_CRYPTO_ALG_PRI, 2002 .cra_name = "dh", 2003 .cra_driver_name = "hpre-dh", 2004 .cra_module = THIS_MODULE, 2005 }, 2006 }; 2007 2008 static struct kpp_alg ecdh_nist_p192 = { 2009 .set_secret = hpre_ecdh_set_secret, 2010 .generate_public_key = hpre_ecdh_compute_value, 2011 .compute_shared_secret = hpre_ecdh_compute_value, 2012 .max_size = hpre_ecdh_max_size, 2013 .init = hpre_ecdh_nist_p192_init_tfm, 2014 .exit = hpre_ecdh_exit_tfm, 2015 .reqsize = sizeof(struct hpre_asym_request) + HPRE_ALIGN_SZ, 2016 .base = { 2017 .cra_ctxsize = sizeof(struct hpre_ctx), 2018 .cra_priority = HPRE_CRYPTO_ALG_PRI, 2019 .cra_name = "ecdh-nist-p192", 2020 .cra_driver_name = "hpre-ecdh-nist-p192", 2021 .cra_module = THIS_MODULE, 2022 }, 2023 }; 2024 2025 static struct kpp_alg ecdh_nist_p256 = { 2026 .set_secret = hpre_ecdh_set_secret, 2027 .generate_public_key = hpre_ecdh_compute_value, 2028 .compute_shared_secret = hpre_ecdh_compute_value, 2029 .max_size = hpre_ecdh_max_size, 2030 .init = hpre_ecdh_nist_p256_init_tfm, 2031 .exit = hpre_ecdh_exit_tfm, 2032 .reqsize = sizeof(struct hpre_asym_request) + HPRE_ALIGN_SZ, 2033 .base = { 2034 .cra_ctxsize = sizeof(struct hpre_ctx), 2035 .cra_priority = HPRE_CRYPTO_ALG_PRI, 2036 .cra_name = "ecdh-nist-p256", 2037 .cra_driver_name = "hpre-ecdh-nist-p256", 2038 .cra_module = THIS_MODULE, 2039 }, 2040 }; 2041 2042 static struct kpp_alg ecdh_nist_p384 = { 2043 .set_secret = hpre_ecdh_set_secret, 2044 .generate_public_key = hpre_ecdh_compute_value, 2045 .compute_shared_secret = hpre_ecdh_compute_value, 2046 .max_size = hpre_ecdh_max_size, 2047 .init = hpre_ecdh_nist_p384_init_tfm, 2048 .exit = hpre_ecdh_exit_tfm, 2049 .reqsize = sizeof(struct hpre_asym_request) + HPRE_ALIGN_SZ, 2050 .base = { 2051 .cra_ctxsize = sizeof(struct hpre_ctx), 2052 .cra_priority = HPRE_CRYPTO_ALG_PRI, 2053 .cra_name = "ecdh-nist-p384", 2054 .cra_driver_name = "hpre-ecdh-nist-p384", 2055 .cra_module = THIS_MODULE, 2056 }, 2057 }; 2058 2059 static struct kpp_alg curve25519_alg = { 2060 .set_secret = hpre_curve25519_set_secret, 2061 .generate_public_key = hpre_curve25519_compute_value, 2062 .compute_shared_secret = hpre_curve25519_compute_value, 2063 .max_size = hpre_curve25519_max_size, 2064 .init = hpre_curve25519_init_tfm, 2065 .exit = hpre_curve25519_exit_tfm, 2066 .reqsize = sizeof(struct hpre_asym_request) + HPRE_ALIGN_SZ, 2067 .base = { 2068 .cra_ctxsize = sizeof(struct hpre_ctx), 2069 .cra_priority = HPRE_CRYPTO_ALG_PRI, 2070 .cra_name = "curve25519", 2071 .cra_driver_name = "hpre-curve25519", 2072 .cra_module = THIS_MODULE, 2073 }, 2074 }; 2075 2076 2077 static int hpre_register_ecdh(void) 2078 { 2079 int ret; 2080 2081 ret = crypto_register_kpp(&ecdh_nist_p192); 2082 if (ret) 2083 return ret; 2084 2085 ret = crypto_register_kpp(&ecdh_nist_p256); 2086 if (ret) 2087 goto unregister_ecdh_p192; 2088 2089 ret = crypto_register_kpp(&ecdh_nist_p384); 2090 if (ret) 2091 goto unregister_ecdh_p256; 2092 2093 return 0; 2094 2095 unregister_ecdh_p256: 2096 crypto_unregister_kpp(&ecdh_nist_p256); 2097 unregister_ecdh_p192: 2098 crypto_unregister_kpp(&ecdh_nist_p192); 2099 return ret; 2100 } 2101 2102 static void hpre_unregister_ecdh(void) 2103 { 2104 crypto_unregister_kpp(&ecdh_nist_p384); 2105 crypto_unregister_kpp(&ecdh_nist_p256); 2106 crypto_unregister_kpp(&ecdh_nist_p192); 2107 } 2108 2109 int hpre_algs_register(struct hisi_qm *qm) 2110 { 2111 int ret; 2112 2113 rsa.base.cra_flags = 0; 2114 ret = crypto_register_akcipher(&rsa); 2115 if (ret) 2116 return ret; 2117 2118 ret = crypto_register_kpp(&dh); 2119 if (ret) 2120 goto unreg_rsa; 2121 2122 if (qm->ver >= QM_HW_V3) { 2123 ret = hpre_register_ecdh(); 2124 if (ret) 2125 goto unreg_dh; 2126 ret = crypto_register_kpp(&curve25519_alg); 2127 if (ret) 2128 goto unreg_ecdh; 2129 } 2130 return 0; 2131 2132 unreg_ecdh: 2133 hpre_unregister_ecdh(); 2134 unreg_dh: 2135 crypto_unregister_kpp(&dh); 2136 unreg_rsa: 2137 crypto_unregister_akcipher(&rsa); 2138 return ret; 2139 } 2140 2141 void hpre_algs_unregister(struct hisi_qm *qm) 2142 { 2143 if (qm->ver >= QM_HW_V3) { 2144 crypto_unregister_kpp(&curve25519_alg); 2145 hpre_unregister_ecdh(); 2146 } 2147 2148 crypto_unregister_kpp(&dh); 2149 crypto_unregister_akcipher(&rsa); 2150 } 2151