1 // SPDX-License-Identifier: GPL-2.0 2 /* Copyright (c) 2019 HiSilicon Limited. */ 3 4 #include <crypto/aes.h> 5 #include <crypto/algapi.h> 6 #include <crypto/authenc.h> 7 #include <crypto/des.h> 8 #include <crypto/hash.h> 9 #include <crypto/internal/aead.h> 10 #include <crypto/sha.h> 11 #include <crypto/skcipher.h> 12 #include <crypto/xts.h> 13 #include <linux/crypto.h> 14 #include <linux/dma-mapping.h> 15 #include <linux/idr.h> 16 17 #include "sec.h" 18 #include "sec_crypto.h" 19 20 #define SEC_PRIORITY 4001 21 #define SEC_XTS_MIN_KEY_SIZE (2 * AES_MIN_KEY_SIZE) 22 #define SEC_XTS_MAX_KEY_SIZE (2 * AES_MAX_KEY_SIZE) 23 #define SEC_DES3_2KEY_SIZE (2 * DES_KEY_SIZE) 24 #define SEC_DES3_3KEY_SIZE (3 * DES_KEY_SIZE) 25 26 /* SEC sqe(bd) bit operational relative MACRO */ 27 #define SEC_DE_OFFSET 1 28 #define SEC_CIPHER_OFFSET 4 29 #define SEC_SCENE_OFFSET 3 30 #define SEC_DST_SGL_OFFSET 2 31 #define SEC_SRC_SGL_OFFSET 7 32 #define SEC_CKEY_OFFSET 9 33 #define SEC_CMODE_OFFSET 12 34 #define SEC_AKEY_OFFSET 5 35 #define SEC_AEAD_ALG_OFFSET 11 36 #define SEC_AUTH_OFFSET 6 37 38 #define SEC_FLAG_OFFSET 7 39 #define SEC_FLAG_MASK 0x0780 40 #define SEC_TYPE_MASK 0x0F 41 #define SEC_DONE_MASK 0x0001 42 43 #define SEC_TOTAL_IV_SZ (SEC_IV_SIZE * QM_Q_DEPTH) 44 #define SEC_SGL_SGE_NR 128 45 #define SEC_CTX_DEV(ctx) (&(ctx)->sec->qm.pdev->dev) 46 #define SEC_CIPHER_AUTH 0xfe 47 #define SEC_AUTH_CIPHER 0x1 48 #define SEC_MAX_MAC_LEN 64 49 #define SEC_MAX_AAD_LEN 65535 50 #define SEC_TOTAL_MAC_SZ (SEC_MAX_MAC_LEN * QM_Q_DEPTH) 51 52 #define SEC_PBUF_SZ 512 53 #define SEC_PBUF_IV_OFFSET SEC_PBUF_SZ 54 #define SEC_PBUF_MAC_OFFSET (SEC_PBUF_SZ + SEC_IV_SIZE) 55 #define SEC_PBUF_PKG (SEC_PBUF_SZ + SEC_IV_SIZE + \ 56 SEC_MAX_MAC_LEN * 2) 57 #define SEC_PBUF_NUM (PAGE_SIZE / SEC_PBUF_PKG) 58 #define SEC_PBUF_PAGE_NUM (QM_Q_DEPTH / SEC_PBUF_NUM) 59 #define SEC_PBUF_LEFT_SZ (SEC_PBUF_PKG * (QM_Q_DEPTH - \ 60 SEC_PBUF_PAGE_NUM * SEC_PBUF_NUM)) 61 #define SEC_TOTAL_PBUF_SZ (PAGE_SIZE * SEC_PBUF_PAGE_NUM + \ 62 SEC_PBUF_LEFT_SZ) 63 64 #define SEC_SQE_LEN_RATE 4 65 #define SEC_SQE_CFLAG 2 66 #define SEC_SQE_AEAD_FLAG 3 67 #define SEC_SQE_DONE 0x1 68 69 /* Get an en/de-cipher queue cyclically to balance load over queues of TFM */ 70 static inline int sec_alloc_queue_id(struct sec_ctx *ctx, struct sec_req *req) 71 { 72 if (req->c_req.encrypt) 73 return (u32)atomic_inc_return(&ctx->enc_qcyclic) % 74 ctx->hlf_q_num; 75 76 return (u32)atomic_inc_return(&ctx->dec_qcyclic) % ctx->hlf_q_num + 77 ctx->hlf_q_num; 78 } 79 80 static inline void sec_free_queue_id(struct sec_ctx *ctx, struct sec_req *req) 81 { 82 if (req->c_req.encrypt) 83 atomic_dec(&ctx->enc_qcyclic); 84 else 85 atomic_dec(&ctx->dec_qcyclic); 86 } 87 88 static int sec_alloc_req_id(struct sec_req *req, struct sec_qp_ctx *qp_ctx) 89 { 90 int req_id; 91 92 mutex_lock(&qp_ctx->req_lock); 93 94 req_id = idr_alloc_cyclic(&qp_ctx->req_idr, NULL, 95 0, QM_Q_DEPTH, GFP_ATOMIC); 96 mutex_unlock(&qp_ctx->req_lock); 97 if (unlikely(req_id < 0)) { 98 dev_err(SEC_CTX_DEV(req->ctx), "alloc req id fail!\n"); 99 return req_id; 100 } 101 102 req->qp_ctx = qp_ctx; 103 qp_ctx->req_list[req_id] = req; 104 return req_id; 105 } 106 107 static void sec_free_req_id(struct sec_req *req) 108 { 109 struct sec_qp_ctx *qp_ctx = req->qp_ctx; 110 int req_id = req->req_id; 111 112 if (unlikely(req_id < 0 || req_id >= QM_Q_DEPTH)) { 113 dev_err(SEC_CTX_DEV(req->ctx), "free request id invalid!\n"); 114 return; 115 } 116 117 qp_ctx->req_list[req_id] = NULL; 118 req->qp_ctx = NULL; 119 120 mutex_lock(&qp_ctx->req_lock); 121 idr_remove(&qp_ctx->req_idr, req_id); 122 mutex_unlock(&qp_ctx->req_lock); 123 } 124 125 static int sec_aead_verify(struct sec_req *req) 126 { 127 struct aead_request *aead_req = req->aead_req.aead_req; 128 struct crypto_aead *tfm = crypto_aead_reqtfm(aead_req); 129 size_t authsize = crypto_aead_authsize(tfm); 130 u8 *mac_out = req->aead_req.out_mac; 131 u8 *mac = mac_out + SEC_MAX_MAC_LEN; 132 struct scatterlist *sgl = aead_req->src; 133 size_t sz; 134 135 sz = sg_pcopy_to_buffer(sgl, sg_nents(sgl), mac, authsize, 136 aead_req->cryptlen + aead_req->assoclen - 137 authsize); 138 if (unlikely(sz != authsize || memcmp(mac_out, mac, sz))) { 139 dev_err(SEC_CTX_DEV(req->ctx), "aead verify failure!\n"); 140 return -EBADMSG; 141 } 142 143 return 0; 144 } 145 146 static void sec_req_cb(struct hisi_qp *qp, void *resp) 147 { 148 struct sec_qp_ctx *qp_ctx = qp->qp_ctx; 149 struct sec_dfx *dfx = &qp_ctx->ctx->sec->debug.dfx; 150 struct sec_sqe *bd = resp; 151 struct sec_ctx *ctx; 152 struct sec_req *req; 153 u16 done, flag; 154 int err = 0; 155 u8 type; 156 157 type = bd->type_cipher_auth & SEC_TYPE_MASK; 158 if (unlikely(type != SEC_BD_TYPE2)) { 159 atomic64_inc(&dfx->err_bd_cnt); 160 pr_err("err bd type [%d]\n", type); 161 return; 162 } 163 164 req = qp_ctx->req_list[le16_to_cpu(bd->type2.tag)]; 165 if (unlikely(!req)) { 166 atomic64_inc(&dfx->invalid_req_cnt); 167 atomic_inc(&qp->qp_status.used); 168 return; 169 } 170 req->err_type = bd->type2.error_type; 171 ctx = req->ctx; 172 done = le16_to_cpu(bd->type2.done_flag) & SEC_DONE_MASK; 173 flag = (le16_to_cpu(bd->type2.done_flag) & 174 SEC_FLAG_MASK) >> SEC_FLAG_OFFSET; 175 if (unlikely(req->err_type || done != SEC_SQE_DONE || 176 (ctx->alg_type == SEC_SKCIPHER && flag != SEC_SQE_CFLAG) || 177 (ctx->alg_type == SEC_AEAD && flag != SEC_SQE_AEAD_FLAG))) { 178 dev_err(SEC_CTX_DEV(ctx), 179 "err_type[%d],done[%d],flag[%d]\n", 180 req->err_type, done, flag); 181 err = -EIO; 182 atomic64_inc(&dfx->done_flag_cnt); 183 } 184 185 if (ctx->alg_type == SEC_AEAD && !req->c_req.encrypt) 186 err = sec_aead_verify(req); 187 188 atomic64_inc(&dfx->recv_cnt); 189 190 ctx->req_op->buf_unmap(ctx, req); 191 192 ctx->req_op->callback(ctx, req, err); 193 } 194 195 static int sec_bd_send(struct sec_ctx *ctx, struct sec_req *req) 196 { 197 struct sec_qp_ctx *qp_ctx = req->qp_ctx; 198 int ret; 199 200 if (ctx->fake_req_limit <= 201 atomic_read(&qp_ctx->qp->qp_status.used) && 202 !(req->flag & CRYPTO_TFM_REQ_MAY_BACKLOG)) 203 return -EBUSY; 204 205 mutex_lock(&qp_ctx->req_lock); 206 ret = hisi_qp_send(qp_ctx->qp, &req->sec_sqe); 207 208 if (ctx->fake_req_limit <= 209 atomic_read(&qp_ctx->qp->qp_status.used) && !ret) { 210 list_add_tail(&req->backlog_head, &qp_ctx->backlog); 211 atomic64_inc(&ctx->sec->debug.dfx.send_cnt); 212 atomic64_inc(&ctx->sec->debug.dfx.send_busy_cnt); 213 mutex_unlock(&qp_ctx->req_lock); 214 return -EBUSY; 215 } 216 mutex_unlock(&qp_ctx->req_lock); 217 218 if (unlikely(ret == -EBUSY)) 219 return -ENOBUFS; 220 221 if (likely(!ret)) { 222 ret = -EINPROGRESS; 223 atomic64_inc(&ctx->sec->debug.dfx.send_cnt); 224 } 225 226 return ret; 227 } 228 229 /* Get DMA memory resources */ 230 static int sec_alloc_civ_resource(struct device *dev, struct sec_alg_res *res) 231 { 232 int i; 233 234 res->c_ivin = dma_alloc_coherent(dev, SEC_TOTAL_IV_SZ, 235 &res->c_ivin_dma, GFP_KERNEL); 236 if (!res->c_ivin) 237 return -ENOMEM; 238 239 for (i = 1; i < QM_Q_DEPTH; i++) { 240 res[i].c_ivin_dma = res->c_ivin_dma + i * SEC_IV_SIZE; 241 res[i].c_ivin = res->c_ivin + i * SEC_IV_SIZE; 242 } 243 244 return 0; 245 } 246 247 static void sec_free_civ_resource(struct device *dev, struct sec_alg_res *res) 248 { 249 if (res->c_ivin) 250 dma_free_coherent(dev, SEC_TOTAL_IV_SZ, 251 res->c_ivin, res->c_ivin_dma); 252 } 253 254 static int sec_alloc_mac_resource(struct device *dev, struct sec_alg_res *res) 255 { 256 int i; 257 258 res->out_mac = dma_alloc_coherent(dev, SEC_TOTAL_MAC_SZ << 1, 259 &res->out_mac_dma, GFP_KERNEL); 260 if (!res->out_mac) 261 return -ENOMEM; 262 263 for (i = 1; i < QM_Q_DEPTH; i++) { 264 res[i].out_mac_dma = res->out_mac_dma + 265 i * (SEC_MAX_MAC_LEN << 1); 266 res[i].out_mac = res->out_mac + i * (SEC_MAX_MAC_LEN << 1); 267 } 268 269 return 0; 270 } 271 272 static void sec_free_mac_resource(struct device *dev, struct sec_alg_res *res) 273 { 274 if (res->out_mac) 275 dma_free_coherent(dev, SEC_TOTAL_MAC_SZ << 1, 276 res->out_mac, res->out_mac_dma); 277 } 278 279 static void sec_free_pbuf_resource(struct device *dev, struct sec_alg_res *res) 280 { 281 if (res->pbuf) 282 dma_free_coherent(dev, SEC_TOTAL_PBUF_SZ, 283 res->pbuf, res->pbuf_dma); 284 } 285 286 /* 287 * To improve performance, pbuffer is used for 288 * small packets (< 512Bytes) as IOMMU translation using. 289 */ 290 static int sec_alloc_pbuf_resource(struct device *dev, struct sec_alg_res *res) 291 { 292 int pbuf_page_offset; 293 int i, j, k; 294 295 res->pbuf = dma_alloc_coherent(dev, SEC_TOTAL_PBUF_SZ, 296 &res->pbuf_dma, GFP_KERNEL); 297 if (!res->pbuf) 298 return -ENOMEM; 299 300 /* 301 * SEC_PBUF_PKG contains data pbuf, iv and 302 * out_mac : <SEC_PBUF|SEC_IV|SEC_MAC> 303 * Every PAGE contains six SEC_PBUF_PKG 304 * The sec_qp_ctx contains QM_Q_DEPTH numbers of SEC_PBUF_PKG 305 * So we need SEC_PBUF_PAGE_NUM numbers of PAGE 306 * for the SEC_TOTAL_PBUF_SZ 307 */ 308 for (i = 0; i <= SEC_PBUF_PAGE_NUM; i++) { 309 pbuf_page_offset = PAGE_SIZE * i; 310 for (j = 0; j < SEC_PBUF_NUM; j++) { 311 k = i * SEC_PBUF_NUM + j; 312 if (k == QM_Q_DEPTH) 313 break; 314 res[k].pbuf = res->pbuf + 315 j * SEC_PBUF_PKG + pbuf_page_offset; 316 res[k].pbuf_dma = res->pbuf_dma + 317 j * SEC_PBUF_PKG + pbuf_page_offset; 318 } 319 } 320 return 0; 321 } 322 323 static int sec_alg_resource_alloc(struct sec_ctx *ctx, 324 struct sec_qp_ctx *qp_ctx) 325 { 326 struct device *dev = SEC_CTX_DEV(ctx); 327 struct sec_alg_res *res = qp_ctx->res; 328 int ret; 329 330 ret = sec_alloc_civ_resource(dev, res); 331 if (ret) 332 return ret; 333 334 if (ctx->alg_type == SEC_AEAD) { 335 ret = sec_alloc_mac_resource(dev, res); 336 if (ret) 337 goto alloc_fail; 338 } 339 if (ctx->pbuf_supported) { 340 ret = sec_alloc_pbuf_resource(dev, res); 341 if (ret) { 342 dev_err(dev, "fail to alloc pbuf dma resource!\n"); 343 goto alloc_pbuf_fail; 344 } 345 } 346 347 return 0; 348 alloc_pbuf_fail: 349 if (ctx->alg_type == SEC_AEAD) 350 sec_free_mac_resource(dev, qp_ctx->res); 351 alloc_fail: 352 sec_free_civ_resource(dev, res); 353 354 return ret; 355 } 356 357 static void sec_alg_resource_free(struct sec_ctx *ctx, 358 struct sec_qp_ctx *qp_ctx) 359 { 360 struct device *dev = SEC_CTX_DEV(ctx); 361 362 sec_free_civ_resource(dev, qp_ctx->res); 363 364 if (ctx->pbuf_supported) 365 sec_free_pbuf_resource(dev, qp_ctx->res); 366 if (ctx->alg_type == SEC_AEAD) 367 sec_free_mac_resource(dev, qp_ctx->res); 368 } 369 370 static int sec_create_qp_ctx(struct hisi_qm *qm, struct sec_ctx *ctx, 371 int qp_ctx_id, int alg_type) 372 { 373 struct device *dev = SEC_CTX_DEV(ctx); 374 struct sec_qp_ctx *qp_ctx; 375 struct hisi_qp *qp; 376 int ret = -ENOMEM; 377 378 qp_ctx = &ctx->qp_ctx[qp_ctx_id]; 379 qp = ctx->qps[qp_ctx_id]; 380 qp->req_type = 0; 381 qp->qp_ctx = qp_ctx; 382 qp->req_cb = sec_req_cb; 383 qp_ctx->qp = qp; 384 qp_ctx->ctx = ctx; 385 386 mutex_init(&qp_ctx->req_lock); 387 idr_init(&qp_ctx->req_idr); 388 INIT_LIST_HEAD(&qp_ctx->backlog); 389 390 qp_ctx->c_in_pool = hisi_acc_create_sgl_pool(dev, QM_Q_DEPTH, 391 SEC_SGL_SGE_NR); 392 if (IS_ERR(qp_ctx->c_in_pool)) { 393 dev_err(dev, "fail to create sgl pool for input!\n"); 394 goto err_destroy_idr; 395 } 396 397 qp_ctx->c_out_pool = hisi_acc_create_sgl_pool(dev, QM_Q_DEPTH, 398 SEC_SGL_SGE_NR); 399 if (IS_ERR(qp_ctx->c_out_pool)) { 400 dev_err(dev, "fail to create sgl pool for output!\n"); 401 goto err_free_c_in_pool; 402 } 403 404 ret = sec_alg_resource_alloc(ctx, qp_ctx); 405 if (ret) 406 goto err_free_c_out_pool; 407 408 ret = hisi_qm_start_qp(qp, 0); 409 if (ret < 0) 410 goto err_queue_free; 411 412 return 0; 413 414 err_queue_free: 415 sec_alg_resource_free(ctx, qp_ctx); 416 err_free_c_out_pool: 417 hisi_acc_free_sgl_pool(dev, qp_ctx->c_out_pool); 418 err_free_c_in_pool: 419 hisi_acc_free_sgl_pool(dev, qp_ctx->c_in_pool); 420 err_destroy_idr: 421 idr_destroy(&qp_ctx->req_idr); 422 423 return ret; 424 } 425 426 static void sec_release_qp_ctx(struct sec_ctx *ctx, 427 struct sec_qp_ctx *qp_ctx) 428 { 429 struct device *dev = SEC_CTX_DEV(ctx); 430 431 hisi_qm_stop_qp(qp_ctx->qp); 432 sec_alg_resource_free(ctx, qp_ctx); 433 434 hisi_acc_free_sgl_pool(dev, qp_ctx->c_out_pool); 435 hisi_acc_free_sgl_pool(dev, qp_ctx->c_in_pool); 436 437 idr_destroy(&qp_ctx->req_idr); 438 } 439 440 static int sec_ctx_base_init(struct sec_ctx *ctx) 441 { 442 struct sec_dev *sec; 443 int i, ret; 444 445 ctx->qps = sec_create_qps(); 446 if (!ctx->qps) { 447 pr_err("Can not create sec qps!\n"); 448 return -ENODEV; 449 } 450 451 sec = container_of(ctx->qps[0]->qm, struct sec_dev, qm); 452 ctx->sec = sec; 453 ctx->hlf_q_num = sec->ctx_q_num >> 1; 454 455 ctx->pbuf_supported = ctx->sec->iommu_used; 456 457 /* Half of queue depth is taken as fake requests limit in the queue. */ 458 ctx->fake_req_limit = QM_Q_DEPTH >> 1; 459 ctx->qp_ctx = kcalloc(sec->ctx_q_num, sizeof(struct sec_qp_ctx), 460 GFP_KERNEL); 461 if (!ctx->qp_ctx) { 462 ret = -ENOMEM; 463 goto err_destroy_qps; 464 } 465 466 for (i = 0; i < sec->ctx_q_num; i++) { 467 ret = sec_create_qp_ctx(&sec->qm, ctx, i, 0); 468 if (ret) 469 goto err_sec_release_qp_ctx; 470 } 471 472 return 0; 473 474 err_sec_release_qp_ctx: 475 for (i = i - 1; i >= 0; i--) 476 sec_release_qp_ctx(ctx, &ctx->qp_ctx[i]); 477 478 kfree(ctx->qp_ctx); 479 err_destroy_qps: 480 sec_destroy_qps(ctx->qps, sec->ctx_q_num); 481 482 return ret; 483 } 484 485 static void sec_ctx_base_uninit(struct sec_ctx *ctx) 486 { 487 int i; 488 489 for (i = 0; i < ctx->sec->ctx_q_num; i++) 490 sec_release_qp_ctx(ctx, &ctx->qp_ctx[i]); 491 492 sec_destroy_qps(ctx->qps, ctx->sec->ctx_q_num); 493 kfree(ctx->qp_ctx); 494 } 495 496 static int sec_cipher_init(struct sec_ctx *ctx) 497 { 498 struct sec_cipher_ctx *c_ctx = &ctx->c_ctx; 499 500 c_ctx->c_key = dma_alloc_coherent(SEC_CTX_DEV(ctx), SEC_MAX_KEY_SIZE, 501 &c_ctx->c_key_dma, GFP_KERNEL); 502 if (!c_ctx->c_key) 503 return -ENOMEM; 504 505 return 0; 506 } 507 508 static void sec_cipher_uninit(struct sec_ctx *ctx) 509 { 510 struct sec_cipher_ctx *c_ctx = &ctx->c_ctx; 511 512 memzero_explicit(c_ctx->c_key, SEC_MAX_KEY_SIZE); 513 dma_free_coherent(SEC_CTX_DEV(ctx), SEC_MAX_KEY_SIZE, 514 c_ctx->c_key, c_ctx->c_key_dma); 515 } 516 517 static int sec_auth_init(struct sec_ctx *ctx) 518 { 519 struct sec_auth_ctx *a_ctx = &ctx->a_ctx; 520 521 a_ctx->a_key = dma_alloc_coherent(SEC_CTX_DEV(ctx), SEC_MAX_KEY_SIZE, 522 &a_ctx->a_key_dma, GFP_KERNEL); 523 if (!a_ctx->a_key) 524 return -ENOMEM; 525 526 return 0; 527 } 528 529 static void sec_auth_uninit(struct sec_ctx *ctx) 530 { 531 struct sec_auth_ctx *a_ctx = &ctx->a_ctx; 532 533 memzero_explicit(a_ctx->a_key, SEC_MAX_KEY_SIZE); 534 dma_free_coherent(SEC_CTX_DEV(ctx), SEC_MAX_KEY_SIZE, 535 a_ctx->a_key, a_ctx->a_key_dma); 536 } 537 538 static int sec_skcipher_init(struct crypto_skcipher *tfm) 539 { 540 struct sec_ctx *ctx = crypto_skcipher_ctx(tfm); 541 int ret; 542 543 ctx->alg_type = SEC_SKCIPHER; 544 crypto_skcipher_set_reqsize(tfm, sizeof(struct sec_req)); 545 ctx->c_ctx.ivsize = crypto_skcipher_ivsize(tfm); 546 if (ctx->c_ctx.ivsize > SEC_IV_SIZE) { 547 dev_err(SEC_CTX_DEV(ctx), "get error skcipher iv size!\n"); 548 return -EINVAL; 549 } 550 551 ret = sec_ctx_base_init(ctx); 552 if (ret) 553 return ret; 554 555 ret = sec_cipher_init(ctx); 556 if (ret) 557 goto err_cipher_init; 558 559 return 0; 560 err_cipher_init: 561 sec_ctx_base_uninit(ctx); 562 563 return ret; 564 } 565 566 static void sec_skcipher_uninit(struct crypto_skcipher *tfm) 567 { 568 struct sec_ctx *ctx = crypto_skcipher_ctx(tfm); 569 570 sec_cipher_uninit(ctx); 571 sec_ctx_base_uninit(ctx); 572 } 573 574 static int sec_skcipher_3des_setkey(struct sec_cipher_ctx *c_ctx, 575 const u32 keylen, 576 const enum sec_cmode c_mode) 577 { 578 switch (keylen) { 579 case SEC_DES3_2KEY_SIZE: 580 c_ctx->c_key_len = SEC_CKEY_3DES_2KEY; 581 break; 582 case SEC_DES3_3KEY_SIZE: 583 c_ctx->c_key_len = SEC_CKEY_3DES_3KEY; 584 break; 585 default: 586 return -EINVAL; 587 } 588 589 return 0; 590 } 591 592 static int sec_skcipher_aes_sm4_setkey(struct sec_cipher_ctx *c_ctx, 593 const u32 keylen, 594 const enum sec_cmode c_mode) 595 { 596 if (c_mode == SEC_CMODE_XTS) { 597 switch (keylen) { 598 case SEC_XTS_MIN_KEY_SIZE: 599 c_ctx->c_key_len = SEC_CKEY_128BIT; 600 break; 601 case SEC_XTS_MAX_KEY_SIZE: 602 c_ctx->c_key_len = SEC_CKEY_256BIT; 603 break; 604 default: 605 pr_err("hisi_sec2: xts mode key error!\n"); 606 return -EINVAL; 607 } 608 } else { 609 switch (keylen) { 610 case AES_KEYSIZE_128: 611 c_ctx->c_key_len = SEC_CKEY_128BIT; 612 break; 613 case AES_KEYSIZE_192: 614 c_ctx->c_key_len = SEC_CKEY_192BIT; 615 break; 616 case AES_KEYSIZE_256: 617 c_ctx->c_key_len = SEC_CKEY_256BIT; 618 break; 619 default: 620 pr_err("hisi_sec2: aes key error!\n"); 621 return -EINVAL; 622 } 623 } 624 625 return 0; 626 } 627 628 static int sec_skcipher_setkey(struct crypto_skcipher *tfm, const u8 *key, 629 const u32 keylen, const enum sec_calg c_alg, 630 const enum sec_cmode c_mode) 631 { 632 struct sec_ctx *ctx = crypto_skcipher_ctx(tfm); 633 struct sec_cipher_ctx *c_ctx = &ctx->c_ctx; 634 int ret; 635 636 if (c_mode == SEC_CMODE_XTS) { 637 ret = xts_verify_key(tfm, key, keylen); 638 if (ret) { 639 dev_err(SEC_CTX_DEV(ctx), "xts mode key err!\n"); 640 return ret; 641 } 642 } 643 644 c_ctx->c_alg = c_alg; 645 c_ctx->c_mode = c_mode; 646 647 switch (c_alg) { 648 case SEC_CALG_3DES: 649 ret = sec_skcipher_3des_setkey(c_ctx, keylen, c_mode); 650 break; 651 case SEC_CALG_AES: 652 case SEC_CALG_SM4: 653 ret = sec_skcipher_aes_sm4_setkey(c_ctx, keylen, c_mode); 654 break; 655 default: 656 return -EINVAL; 657 } 658 659 if (ret) { 660 dev_err(SEC_CTX_DEV(ctx), "set sec key err!\n"); 661 return ret; 662 } 663 664 memcpy(c_ctx->c_key, key, keylen); 665 666 return 0; 667 } 668 669 #define GEN_SEC_SETKEY_FUNC(name, c_alg, c_mode) \ 670 static int sec_setkey_##name(struct crypto_skcipher *tfm, const u8 *key,\ 671 u32 keylen) \ 672 { \ 673 return sec_skcipher_setkey(tfm, key, keylen, c_alg, c_mode); \ 674 } 675 676 GEN_SEC_SETKEY_FUNC(aes_ecb, SEC_CALG_AES, SEC_CMODE_ECB) 677 GEN_SEC_SETKEY_FUNC(aes_cbc, SEC_CALG_AES, SEC_CMODE_CBC) 678 GEN_SEC_SETKEY_FUNC(aes_xts, SEC_CALG_AES, SEC_CMODE_XTS) 679 680 GEN_SEC_SETKEY_FUNC(3des_ecb, SEC_CALG_3DES, SEC_CMODE_ECB) 681 GEN_SEC_SETKEY_FUNC(3des_cbc, SEC_CALG_3DES, SEC_CMODE_CBC) 682 683 GEN_SEC_SETKEY_FUNC(sm4_xts, SEC_CALG_SM4, SEC_CMODE_XTS) 684 GEN_SEC_SETKEY_FUNC(sm4_cbc, SEC_CALG_SM4, SEC_CMODE_CBC) 685 686 static int sec_cipher_pbuf_map(struct sec_ctx *ctx, struct sec_req *req, 687 struct scatterlist *src) 688 { 689 struct aead_request *aead_req = req->aead_req.aead_req; 690 struct sec_cipher_req *c_req = &req->c_req; 691 struct sec_qp_ctx *qp_ctx = req->qp_ctx; 692 struct device *dev = SEC_CTX_DEV(ctx); 693 int copy_size, pbuf_length; 694 int req_id = req->req_id; 695 696 if (ctx->alg_type == SEC_AEAD) 697 copy_size = aead_req->cryptlen + aead_req->assoclen; 698 else 699 copy_size = c_req->c_len; 700 701 pbuf_length = sg_copy_to_buffer(src, sg_nents(src), 702 qp_ctx->res[req_id].pbuf, 703 copy_size); 704 705 if (unlikely(pbuf_length != copy_size)) { 706 dev_err(dev, "copy src data to pbuf error!\n"); 707 return -EINVAL; 708 } 709 710 c_req->c_in_dma = qp_ctx->res[req_id].pbuf_dma; 711 712 if (!c_req->c_in_dma) { 713 dev_err(dev, "fail to set pbuffer address!\n"); 714 return -ENOMEM; 715 } 716 717 c_req->c_out_dma = c_req->c_in_dma; 718 719 return 0; 720 } 721 722 static void sec_cipher_pbuf_unmap(struct sec_ctx *ctx, struct sec_req *req, 723 struct scatterlist *dst) 724 { 725 struct aead_request *aead_req = req->aead_req.aead_req; 726 struct sec_cipher_req *c_req = &req->c_req; 727 struct sec_qp_ctx *qp_ctx = req->qp_ctx; 728 struct device *dev = SEC_CTX_DEV(ctx); 729 int copy_size, pbuf_length; 730 int req_id = req->req_id; 731 732 if (ctx->alg_type == SEC_AEAD) 733 copy_size = c_req->c_len + aead_req->assoclen; 734 else 735 copy_size = c_req->c_len; 736 737 pbuf_length = sg_copy_from_buffer(dst, sg_nents(dst), 738 qp_ctx->res[req_id].pbuf, 739 copy_size); 740 741 if (unlikely(pbuf_length != copy_size)) 742 dev_err(dev, "copy pbuf data to dst error!\n"); 743 744 } 745 746 static int sec_cipher_map(struct sec_ctx *ctx, struct sec_req *req, 747 struct scatterlist *src, struct scatterlist *dst) 748 { 749 struct sec_cipher_req *c_req = &req->c_req; 750 struct sec_aead_req *a_req = &req->aead_req; 751 struct sec_qp_ctx *qp_ctx = req->qp_ctx; 752 struct sec_alg_res *res = &qp_ctx->res[req->req_id]; 753 struct device *dev = SEC_CTX_DEV(ctx); 754 int ret; 755 756 if (req->use_pbuf) { 757 ret = sec_cipher_pbuf_map(ctx, req, src); 758 c_req->c_ivin = res->pbuf + SEC_PBUF_IV_OFFSET; 759 c_req->c_ivin_dma = res->pbuf_dma + SEC_PBUF_IV_OFFSET; 760 if (ctx->alg_type == SEC_AEAD) { 761 a_req->out_mac = res->pbuf + SEC_PBUF_MAC_OFFSET; 762 a_req->out_mac_dma = res->pbuf_dma + 763 SEC_PBUF_MAC_OFFSET; 764 } 765 766 return ret; 767 } 768 c_req->c_ivin = res->c_ivin; 769 c_req->c_ivin_dma = res->c_ivin_dma; 770 if (ctx->alg_type == SEC_AEAD) { 771 a_req->out_mac = res->out_mac; 772 a_req->out_mac_dma = res->out_mac_dma; 773 } 774 775 c_req->c_in = hisi_acc_sg_buf_map_to_hw_sgl(dev, src, 776 qp_ctx->c_in_pool, 777 req->req_id, 778 &c_req->c_in_dma); 779 780 if (IS_ERR(c_req->c_in)) { 781 dev_err(dev, "fail to dma map input sgl buffers!\n"); 782 return PTR_ERR(c_req->c_in); 783 } 784 785 if (dst == src) { 786 c_req->c_out = c_req->c_in; 787 c_req->c_out_dma = c_req->c_in_dma; 788 } else { 789 c_req->c_out = hisi_acc_sg_buf_map_to_hw_sgl(dev, dst, 790 qp_ctx->c_out_pool, 791 req->req_id, 792 &c_req->c_out_dma); 793 794 if (IS_ERR(c_req->c_out)) { 795 dev_err(dev, "fail to dma map output sgl buffers!\n"); 796 hisi_acc_sg_buf_unmap(dev, src, c_req->c_in); 797 return PTR_ERR(c_req->c_out); 798 } 799 } 800 801 return 0; 802 } 803 804 static void sec_cipher_unmap(struct sec_ctx *ctx, struct sec_req *req, 805 struct scatterlist *src, struct scatterlist *dst) 806 { 807 struct sec_cipher_req *c_req = &req->c_req; 808 struct device *dev = SEC_CTX_DEV(ctx); 809 810 if (req->use_pbuf) { 811 sec_cipher_pbuf_unmap(ctx, req, dst); 812 } else { 813 if (dst != src) 814 hisi_acc_sg_buf_unmap(dev, src, c_req->c_in); 815 816 hisi_acc_sg_buf_unmap(dev, dst, c_req->c_out); 817 } 818 } 819 820 static int sec_skcipher_sgl_map(struct sec_ctx *ctx, struct sec_req *req) 821 { 822 struct skcipher_request *sq = req->c_req.sk_req; 823 824 return sec_cipher_map(ctx, req, sq->src, sq->dst); 825 } 826 827 static void sec_skcipher_sgl_unmap(struct sec_ctx *ctx, struct sec_req *req) 828 { 829 struct skcipher_request *sq = req->c_req.sk_req; 830 831 sec_cipher_unmap(ctx, req, sq->src, sq->dst); 832 } 833 834 static int sec_aead_aes_set_key(struct sec_cipher_ctx *c_ctx, 835 struct crypto_authenc_keys *keys) 836 { 837 switch (keys->enckeylen) { 838 case AES_KEYSIZE_128: 839 c_ctx->c_key_len = SEC_CKEY_128BIT; 840 break; 841 case AES_KEYSIZE_192: 842 c_ctx->c_key_len = SEC_CKEY_192BIT; 843 break; 844 case AES_KEYSIZE_256: 845 c_ctx->c_key_len = SEC_CKEY_256BIT; 846 break; 847 default: 848 pr_err("hisi_sec2: aead aes key error!\n"); 849 return -EINVAL; 850 } 851 memcpy(c_ctx->c_key, keys->enckey, keys->enckeylen); 852 853 return 0; 854 } 855 856 static int sec_aead_auth_set_key(struct sec_auth_ctx *ctx, 857 struct crypto_authenc_keys *keys) 858 { 859 struct crypto_shash *hash_tfm = ctx->hash_tfm; 860 int blocksize, ret; 861 862 if (!keys->authkeylen) { 863 pr_err("hisi_sec2: aead auth key error!\n"); 864 return -EINVAL; 865 } 866 867 blocksize = crypto_shash_blocksize(hash_tfm); 868 if (keys->authkeylen > blocksize) { 869 ret = crypto_shash_tfm_digest(hash_tfm, keys->authkey, 870 keys->authkeylen, ctx->a_key); 871 if (ret) { 872 pr_err("hisi_sec2: aead auth digest error!\n"); 873 return -EINVAL; 874 } 875 ctx->a_key_len = blocksize; 876 } else { 877 memcpy(ctx->a_key, keys->authkey, keys->authkeylen); 878 ctx->a_key_len = keys->authkeylen; 879 } 880 881 return 0; 882 } 883 884 static int sec_aead_setkey(struct crypto_aead *tfm, const u8 *key, 885 const u32 keylen, const enum sec_hash_alg a_alg, 886 const enum sec_calg c_alg, 887 const enum sec_mac_len mac_len, 888 const enum sec_cmode c_mode) 889 { 890 struct sec_ctx *ctx = crypto_aead_ctx(tfm); 891 struct sec_cipher_ctx *c_ctx = &ctx->c_ctx; 892 struct crypto_authenc_keys keys; 893 int ret; 894 895 ctx->a_ctx.a_alg = a_alg; 896 ctx->c_ctx.c_alg = c_alg; 897 ctx->a_ctx.mac_len = mac_len; 898 c_ctx->c_mode = c_mode; 899 900 if (crypto_authenc_extractkeys(&keys, key, keylen)) 901 goto bad_key; 902 903 ret = sec_aead_aes_set_key(c_ctx, &keys); 904 if (ret) { 905 dev_err(SEC_CTX_DEV(ctx), "set sec cipher key err!\n"); 906 goto bad_key; 907 } 908 909 ret = sec_aead_auth_set_key(&ctx->a_ctx, &keys); 910 if (ret) { 911 dev_err(SEC_CTX_DEV(ctx), "set sec auth key err!\n"); 912 goto bad_key; 913 } 914 915 return 0; 916 bad_key: 917 memzero_explicit(&keys, sizeof(struct crypto_authenc_keys)); 918 919 return -EINVAL; 920 } 921 922 923 #define GEN_SEC_AEAD_SETKEY_FUNC(name, aalg, calg, maclen, cmode) \ 924 static int sec_setkey_##name(struct crypto_aead *tfm, const u8 *key, \ 925 u32 keylen) \ 926 { \ 927 return sec_aead_setkey(tfm, key, keylen, aalg, calg, maclen, cmode);\ 928 } 929 930 GEN_SEC_AEAD_SETKEY_FUNC(aes_cbc_sha1, SEC_A_HMAC_SHA1, 931 SEC_CALG_AES, SEC_HMAC_SHA1_MAC, SEC_CMODE_CBC) 932 GEN_SEC_AEAD_SETKEY_FUNC(aes_cbc_sha256, SEC_A_HMAC_SHA256, 933 SEC_CALG_AES, SEC_HMAC_SHA256_MAC, SEC_CMODE_CBC) 934 GEN_SEC_AEAD_SETKEY_FUNC(aes_cbc_sha512, SEC_A_HMAC_SHA512, 935 SEC_CALG_AES, SEC_HMAC_SHA512_MAC, SEC_CMODE_CBC) 936 937 static int sec_aead_sgl_map(struct sec_ctx *ctx, struct sec_req *req) 938 { 939 struct aead_request *aq = req->aead_req.aead_req; 940 941 return sec_cipher_map(ctx, req, aq->src, aq->dst); 942 } 943 944 static void sec_aead_sgl_unmap(struct sec_ctx *ctx, struct sec_req *req) 945 { 946 struct aead_request *aq = req->aead_req.aead_req; 947 948 sec_cipher_unmap(ctx, req, aq->src, aq->dst); 949 } 950 951 static int sec_request_transfer(struct sec_ctx *ctx, struct sec_req *req) 952 { 953 int ret; 954 955 ret = ctx->req_op->buf_map(ctx, req); 956 if (unlikely(ret)) 957 return ret; 958 959 ctx->req_op->do_transfer(ctx, req); 960 961 ret = ctx->req_op->bd_fill(ctx, req); 962 if (unlikely(ret)) 963 goto unmap_req_buf; 964 965 return ret; 966 967 unmap_req_buf: 968 ctx->req_op->buf_unmap(ctx, req); 969 970 return ret; 971 } 972 973 static void sec_request_untransfer(struct sec_ctx *ctx, struct sec_req *req) 974 { 975 ctx->req_op->buf_unmap(ctx, req); 976 } 977 978 static void sec_skcipher_copy_iv(struct sec_ctx *ctx, struct sec_req *req) 979 { 980 struct skcipher_request *sk_req = req->c_req.sk_req; 981 struct sec_cipher_req *c_req = &req->c_req; 982 983 memcpy(c_req->c_ivin, sk_req->iv, ctx->c_ctx.ivsize); 984 } 985 986 static int sec_skcipher_bd_fill(struct sec_ctx *ctx, struct sec_req *req) 987 { 988 struct sec_cipher_ctx *c_ctx = &ctx->c_ctx; 989 struct sec_cipher_req *c_req = &req->c_req; 990 struct sec_sqe *sec_sqe = &req->sec_sqe; 991 u8 scene, sa_type, da_type; 992 u8 bd_type, cipher; 993 u8 de = 0; 994 995 memset(sec_sqe, 0, sizeof(struct sec_sqe)); 996 997 sec_sqe->type2.c_key_addr = cpu_to_le64(c_ctx->c_key_dma); 998 sec_sqe->type2.c_ivin_addr = cpu_to_le64(c_req->c_ivin_dma); 999 sec_sqe->type2.data_src_addr = cpu_to_le64(c_req->c_in_dma); 1000 sec_sqe->type2.data_dst_addr = cpu_to_le64(c_req->c_out_dma); 1001 1002 sec_sqe->type2.icvw_kmode |= cpu_to_le16(((u16)c_ctx->c_mode) << 1003 SEC_CMODE_OFFSET); 1004 sec_sqe->type2.c_alg = c_ctx->c_alg; 1005 sec_sqe->type2.icvw_kmode |= cpu_to_le16(((u16)c_ctx->c_key_len) << 1006 SEC_CKEY_OFFSET); 1007 1008 bd_type = SEC_BD_TYPE2; 1009 if (c_req->encrypt) 1010 cipher = SEC_CIPHER_ENC << SEC_CIPHER_OFFSET; 1011 else 1012 cipher = SEC_CIPHER_DEC << SEC_CIPHER_OFFSET; 1013 sec_sqe->type_cipher_auth = bd_type | cipher; 1014 1015 if (req->use_pbuf) 1016 sa_type = SEC_PBUF << SEC_SRC_SGL_OFFSET; 1017 else 1018 sa_type = SEC_SGL << SEC_SRC_SGL_OFFSET; 1019 scene = SEC_COMM_SCENE << SEC_SCENE_OFFSET; 1020 if (c_req->c_in_dma != c_req->c_out_dma) 1021 de = 0x1 << SEC_DE_OFFSET; 1022 1023 sec_sqe->sds_sa_type = (de | scene | sa_type); 1024 1025 /* Just set DST address type */ 1026 if (req->use_pbuf) 1027 da_type = SEC_PBUF << SEC_DST_SGL_OFFSET; 1028 else 1029 da_type = SEC_SGL << SEC_DST_SGL_OFFSET; 1030 sec_sqe->sdm_addr_type |= da_type; 1031 1032 sec_sqe->type2.clen_ivhlen |= cpu_to_le32(c_req->c_len); 1033 sec_sqe->type2.tag = cpu_to_le16((u16)req->req_id); 1034 1035 return 0; 1036 } 1037 1038 static void sec_update_iv(struct sec_req *req, enum sec_alg_type alg_type) 1039 { 1040 struct aead_request *aead_req = req->aead_req.aead_req; 1041 struct skcipher_request *sk_req = req->c_req.sk_req; 1042 u32 iv_size = req->ctx->c_ctx.ivsize; 1043 struct scatterlist *sgl; 1044 unsigned int cryptlen; 1045 size_t sz; 1046 u8 *iv; 1047 1048 if (req->c_req.encrypt) 1049 sgl = alg_type == SEC_SKCIPHER ? sk_req->dst : aead_req->dst; 1050 else 1051 sgl = alg_type == SEC_SKCIPHER ? sk_req->src : aead_req->src; 1052 1053 if (alg_type == SEC_SKCIPHER) { 1054 iv = sk_req->iv; 1055 cryptlen = sk_req->cryptlen; 1056 } else { 1057 iv = aead_req->iv; 1058 cryptlen = aead_req->cryptlen; 1059 } 1060 1061 sz = sg_pcopy_to_buffer(sgl, sg_nents(sgl), iv, iv_size, 1062 cryptlen - iv_size); 1063 if (unlikely(sz != iv_size)) 1064 dev_err(SEC_CTX_DEV(req->ctx), "copy output iv error!\n"); 1065 } 1066 1067 static struct sec_req *sec_back_req_clear(struct sec_ctx *ctx, 1068 struct sec_qp_ctx *qp_ctx) 1069 { 1070 struct sec_req *backlog_req = NULL; 1071 1072 mutex_lock(&qp_ctx->req_lock); 1073 if (ctx->fake_req_limit >= 1074 atomic_read(&qp_ctx->qp->qp_status.used) && 1075 !list_empty(&qp_ctx->backlog)) { 1076 backlog_req = list_first_entry(&qp_ctx->backlog, 1077 typeof(*backlog_req), backlog_head); 1078 list_del(&backlog_req->backlog_head); 1079 } 1080 mutex_unlock(&qp_ctx->req_lock); 1081 1082 return backlog_req; 1083 } 1084 1085 static void sec_skcipher_callback(struct sec_ctx *ctx, struct sec_req *req, 1086 int err) 1087 { 1088 struct skcipher_request *sk_req = req->c_req.sk_req; 1089 struct sec_qp_ctx *qp_ctx = req->qp_ctx; 1090 struct skcipher_request *backlog_sk_req; 1091 struct sec_req *backlog_req; 1092 1093 sec_free_req_id(req); 1094 1095 /* IV output at encrypto of CBC mode */ 1096 if (!err && ctx->c_ctx.c_mode == SEC_CMODE_CBC && req->c_req.encrypt) 1097 sec_update_iv(req, SEC_SKCIPHER); 1098 1099 while (1) { 1100 backlog_req = sec_back_req_clear(ctx, qp_ctx); 1101 if (!backlog_req) 1102 break; 1103 1104 backlog_sk_req = backlog_req->c_req.sk_req; 1105 backlog_sk_req->base.complete(&backlog_sk_req->base, 1106 -EINPROGRESS); 1107 atomic64_inc(&ctx->sec->debug.dfx.recv_busy_cnt); 1108 } 1109 1110 1111 sk_req->base.complete(&sk_req->base, err); 1112 } 1113 1114 static void sec_aead_copy_iv(struct sec_ctx *ctx, struct sec_req *req) 1115 { 1116 struct aead_request *aead_req = req->aead_req.aead_req; 1117 struct sec_cipher_req *c_req = &req->c_req; 1118 1119 memcpy(c_req->c_ivin, aead_req->iv, ctx->c_ctx.ivsize); 1120 } 1121 1122 static void sec_auth_bd_fill_ex(struct sec_auth_ctx *ctx, int dir, 1123 struct sec_req *req, struct sec_sqe *sec_sqe) 1124 { 1125 struct sec_aead_req *a_req = &req->aead_req; 1126 struct sec_cipher_req *c_req = &req->c_req; 1127 struct aead_request *aq = a_req->aead_req; 1128 1129 sec_sqe->type2.a_key_addr = cpu_to_le64(ctx->a_key_dma); 1130 1131 sec_sqe->type2.mac_key_alg = 1132 cpu_to_le32(ctx->mac_len / SEC_SQE_LEN_RATE); 1133 1134 sec_sqe->type2.mac_key_alg |= 1135 cpu_to_le32((u32)((ctx->a_key_len) / 1136 SEC_SQE_LEN_RATE) << SEC_AKEY_OFFSET); 1137 1138 sec_sqe->type2.mac_key_alg |= 1139 cpu_to_le32((u32)(ctx->a_alg) << SEC_AEAD_ALG_OFFSET); 1140 1141 sec_sqe->type_cipher_auth |= SEC_AUTH_TYPE1 << SEC_AUTH_OFFSET; 1142 1143 if (dir) 1144 sec_sqe->sds_sa_type &= SEC_CIPHER_AUTH; 1145 else 1146 sec_sqe->sds_sa_type |= SEC_AUTH_CIPHER; 1147 1148 sec_sqe->type2.alen_ivllen = cpu_to_le32(c_req->c_len + aq->assoclen); 1149 1150 sec_sqe->type2.cipher_src_offset = cpu_to_le16((u16)aq->assoclen); 1151 1152 sec_sqe->type2.mac_addr = cpu_to_le64(a_req->out_mac_dma); 1153 } 1154 1155 static int sec_aead_bd_fill(struct sec_ctx *ctx, struct sec_req *req) 1156 { 1157 struct sec_auth_ctx *auth_ctx = &ctx->a_ctx; 1158 struct sec_sqe *sec_sqe = &req->sec_sqe; 1159 int ret; 1160 1161 ret = sec_skcipher_bd_fill(ctx, req); 1162 if (unlikely(ret)) { 1163 dev_err(SEC_CTX_DEV(ctx), "skcipher bd fill is error!\n"); 1164 return ret; 1165 } 1166 1167 sec_auth_bd_fill_ex(auth_ctx, req->c_req.encrypt, req, sec_sqe); 1168 1169 return 0; 1170 } 1171 1172 static void sec_aead_callback(struct sec_ctx *c, struct sec_req *req, int err) 1173 { 1174 struct aead_request *a_req = req->aead_req.aead_req; 1175 struct crypto_aead *tfm = crypto_aead_reqtfm(a_req); 1176 struct sec_aead_req *aead_req = &req->aead_req; 1177 struct sec_cipher_req *c_req = &req->c_req; 1178 size_t authsize = crypto_aead_authsize(tfm); 1179 struct sec_qp_ctx *qp_ctx = req->qp_ctx; 1180 struct aead_request *backlog_aead_req; 1181 struct sec_req *backlog_req; 1182 size_t sz; 1183 1184 if (!err && c->c_ctx.c_mode == SEC_CMODE_CBC && c_req->encrypt) 1185 sec_update_iv(req, SEC_AEAD); 1186 1187 /* Copy output mac */ 1188 if (!err && c_req->encrypt) { 1189 struct scatterlist *sgl = a_req->dst; 1190 1191 sz = sg_pcopy_from_buffer(sgl, sg_nents(sgl), 1192 aead_req->out_mac, 1193 authsize, a_req->cryptlen + 1194 a_req->assoclen); 1195 1196 if (unlikely(sz != authsize)) { 1197 dev_err(SEC_CTX_DEV(req->ctx), "copy out mac err!\n"); 1198 err = -EINVAL; 1199 } 1200 } 1201 1202 sec_free_req_id(req); 1203 1204 while (1) { 1205 backlog_req = sec_back_req_clear(c, qp_ctx); 1206 if (!backlog_req) 1207 break; 1208 1209 backlog_aead_req = backlog_req->aead_req.aead_req; 1210 backlog_aead_req->base.complete(&backlog_aead_req->base, 1211 -EINPROGRESS); 1212 atomic64_inc(&c->sec->debug.dfx.recv_busy_cnt); 1213 } 1214 1215 a_req->base.complete(&a_req->base, err); 1216 } 1217 1218 static void sec_request_uninit(struct sec_ctx *ctx, struct sec_req *req) 1219 { 1220 sec_free_req_id(req); 1221 sec_free_queue_id(ctx, req); 1222 } 1223 1224 static int sec_request_init(struct sec_ctx *ctx, struct sec_req *req) 1225 { 1226 struct sec_qp_ctx *qp_ctx; 1227 int queue_id; 1228 1229 /* To load balance */ 1230 queue_id = sec_alloc_queue_id(ctx, req); 1231 qp_ctx = &ctx->qp_ctx[queue_id]; 1232 1233 req->req_id = sec_alloc_req_id(req, qp_ctx); 1234 if (unlikely(req->req_id < 0)) { 1235 sec_free_queue_id(ctx, req); 1236 return req->req_id; 1237 } 1238 1239 return 0; 1240 } 1241 1242 static int sec_process(struct sec_ctx *ctx, struct sec_req *req) 1243 { 1244 struct sec_cipher_req *c_req = &req->c_req; 1245 int ret; 1246 1247 ret = sec_request_init(ctx, req); 1248 if (unlikely(ret)) 1249 return ret; 1250 1251 ret = sec_request_transfer(ctx, req); 1252 if (unlikely(ret)) 1253 goto err_uninit_req; 1254 1255 /* Output IV as decrypto */ 1256 if (ctx->c_ctx.c_mode == SEC_CMODE_CBC && !req->c_req.encrypt) 1257 sec_update_iv(req, ctx->alg_type); 1258 1259 ret = ctx->req_op->bd_send(ctx, req); 1260 if (unlikely((ret != -EBUSY && ret != -EINPROGRESS) || 1261 (ret == -EBUSY && !(req->flag & CRYPTO_TFM_REQ_MAY_BACKLOG)))) { 1262 dev_err_ratelimited(SEC_CTX_DEV(ctx), "send sec request failed!\n"); 1263 goto err_send_req; 1264 } 1265 1266 return ret; 1267 1268 err_send_req: 1269 /* As failing, restore the IV from user */ 1270 if (ctx->c_ctx.c_mode == SEC_CMODE_CBC && !req->c_req.encrypt) { 1271 if (ctx->alg_type == SEC_SKCIPHER) 1272 memcpy(req->c_req.sk_req->iv, c_req->c_ivin, 1273 ctx->c_ctx.ivsize); 1274 else 1275 memcpy(req->aead_req.aead_req->iv, c_req->c_ivin, 1276 ctx->c_ctx.ivsize); 1277 } 1278 1279 sec_request_untransfer(ctx, req); 1280 err_uninit_req: 1281 sec_request_uninit(ctx, req); 1282 1283 return ret; 1284 } 1285 1286 static const struct sec_req_op sec_skcipher_req_ops = { 1287 .buf_map = sec_skcipher_sgl_map, 1288 .buf_unmap = sec_skcipher_sgl_unmap, 1289 .do_transfer = sec_skcipher_copy_iv, 1290 .bd_fill = sec_skcipher_bd_fill, 1291 .bd_send = sec_bd_send, 1292 .callback = sec_skcipher_callback, 1293 .process = sec_process, 1294 }; 1295 1296 static const struct sec_req_op sec_aead_req_ops = { 1297 .buf_map = sec_aead_sgl_map, 1298 .buf_unmap = sec_aead_sgl_unmap, 1299 .do_transfer = sec_aead_copy_iv, 1300 .bd_fill = sec_aead_bd_fill, 1301 .bd_send = sec_bd_send, 1302 .callback = sec_aead_callback, 1303 .process = sec_process, 1304 }; 1305 1306 static int sec_skcipher_ctx_init(struct crypto_skcipher *tfm) 1307 { 1308 struct sec_ctx *ctx = crypto_skcipher_ctx(tfm); 1309 1310 ctx->req_op = &sec_skcipher_req_ops; 1311 1312 return sec_skcipher_init(tfm); 1313 } 1314 1315 static void sec_skcipher_ctx_exit(struct crypto_skcipher *tfm) 1316 { 1317 sec_skcipher_uninit(tfm); 1318 } 1319 1320 static int sec_aead_init(struct crypto_aead *tfm) 1321 { 1322 struct sec_ctx *ctx = crypto_aead_ctx(tfm); 1323 int ret; 1324 1325 crypto_aead_set_reqsize(tfm, sizeof(struct sec_req)); 1326 ctx->alg_type = SEC_AEAD; 1327 ctx->c_ctx.ivsize = crypto_aead_ivsize(tfm); 1328 if (ctx->c_ctx.ivsize > SEC_IV_SIZE) { 1329 dev_err(SEC_CTX_DEV(ctx), "get error aead iv size!\n"); 1330 return -EINVAL; 1331 } 1332 1333 ctx->req_op = &sec_aead_req_ops; 1334 ret = sec_ctx_base_init(ctx); 1335 if (ret) 1336 return ret; 1337 1338 ret = sec_auth_init(ctx); 1339 if (ret) 1340 goto err_auth_init; 1341 1342 ret = sec_cipher_init(ctx); 1343 if (ret) 1344 goto err_cipher_init; 1345 1346 return ret; 1347 1348 err_cipher_init: 1349 sec_auth_uninit(ctx); 1350 err_auth_init: 1351 sec_ctx_base_uninit(ctx); 1352 1353 return ret; 1354 } 1355 1356 static void sec_aead_exit(struct crypto_aead *tfm) 1357 { 1358 struct sec_ctx *ctx = crypto_aead_ctx(tfm); 1359 1360 sec_cipher_uninit(ctx); 1361 sec_auth_uninit(ctx); 1362 sec_ctx_base_uninit(ctx); 1363 } 1364 1365 static int sec_aead_ctx_init(struct crypto_aead *tfm, const char *hash_name) 1366 { 1367 struct sec_ctx *ctx = crypto_aead_ctx(tfm); 1368 struct sec_auth_ctx *auth_ctx = &ctx->a_ctx; 1369 int ret; 1370 1371 ret = sec_aead_init(tfm); 1372 if (ret) { 1373 pr_err("hisi_sec2: aead init error!\n"); 1374 return ret; 1375 } 1376 1377 auth_ctx->hash_tfm = crypto_alloc_shash(hash_name, 0, 0); 1378 if (IS_ERR(auth_ctx->hash_tfm)) { 1379 dev_err(SEC_CTX_DEV(ctx), "aead alloc shash error!\n"); 1380 sec_aead_exit(tfm); 1381 return PTR_ERR(auth_ctx->hash_tfm); 1382 } 1383 1384 return 0; 1385 } 1386 1387 static void sec_aead_ctx_exit(struct crypto_aead *tfm) 1388 { 1389 struct sec_ctx *ctx = crypto_aead_ctx(tfm); 1390 1391 crypto_free_shash(ctx->a_ctx.hash_tfm); 1392 sec_aead_exit(tfm); 1393 } 1394 1395 static int sec_aead_sha1_ctx_init(struct crypto_aead *tfm) 1396 { 1397 return sec_aead_ctx_init(tfm, "sha1"); 1398 } 1399 1400 static int sec_aead_sha256_ctx_init(struct crypto_aead *tfm) 1401 { 1402 return sec_aead_ctx_init(tfm, "sha256"); 1403 } 1404 1405 static int sec_aead_sha512_ctx_init(struct crypto_aead *tfm) 1406 { 1407 return sec_aead_ctx_init(tfm, "sha512"); 1408 } 1409 1410 static int sec_skcipher_param_check(struct sec_ctx *ctx, struct sec_req *sreq) 1411 { 1412 struct skcipher_request *sk_req = sreq->c_req.sk_req; 1413 struct device *dev = SEC_CTX_DEV(ctx); 1414 u8 c_alg = ctx->c_ctx.c_alg; 1415 1416 if (unlikely(!sk_req->src || !sk_req->dst)) { 1417 dev_err(dev, "skcipher input param error!\n"); 1418 return -EINVAL; 1419 } 1420 sreq->c_req.c_len = sk_req->cryptlen; 1421 1422 if (ctx->pbuf_supported && sk_req->cryptlen <= SEC_PBUF_SZ) 1423 sreq->use_pbuf = true; 1424 else 1425 sreq->use_pbuf = false; 1426 1427 if (c_alg == SEC_CALG_3DES) { 1428 if (unlikely(sk_req->cryptlen & (DES3_EDE_BLOCK_SIZE - 1))) { 1429 dev_err(dev, "skcipher 3des input length error!\n"); 1430 return -EINVAL; 1431 } 1432 return 0; 1433 } else if (c_alg == SEC_CALG_AES || c_alg == SEC_CALG_SM4) { 1434 if (unlikely(sk_req->cryptlen & (AES_BLOCK_SIZE - 1))) { 1435 dev_err(dev, "skcipher aes input length error!\n"); 1436 return -EINVAL; 1437 } 1438 return 0; 1439 } 1440 1441 dev_err(dev, "skcipher algorithm error!\n"); 1442 return -EINVAL; 1443 } 1444 1445 static int sec_skcipher_crypto(struct skcipher_request *sk_req, bool encrypt) 1446 { 1447 struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(sk_req); 1448 struct sec_req *req = skcipher_request_ctx(sk_req); 1449 struct sec_ctx *ctx = crypto_skcipher_ctx(tfm); 1450 int ret; 1451 1452 if (!sk_req->cryptlen) 1453 return 0; 1454 1455 req->flag = sk_req->base.flags; 1456 req->c_req.sk_req = sk_req; 1457 req->c_req.encrypt = encrypt; 1458 req->ctx = ctx; 1459 1460 ret = sec_skcipher_param_check(ctx, req); 1461 if (unlikely(ret)) 1462 return -EINVAL; 1463 1464 return ctx->req_op->process(ctx, req); 1465 } 1466 1467 static int sec_skcipher_encrypt(struct skcipher_request *sk_req) 1468 { 1469 return sec_skcipher_crypto(sk_req, true); 1470 } 1471 1472 static int sec_skcipher_decrypt(struct skcipher_request *sk_req) 1473 { 1474 return sec_skcipher_crypto(sk_req, false); 1475 } 1476 1477 #define SEC_SKCIPHER_GEN_ALG(sec_cra_name, sec_set_key, sec_min_key_size, \ 1478 sec_max_key_size, ctx_init, ctx_exit, blk_size, iv_size)\ 1479 {\ 1480 .base = {\ 1481 .cra_name = sec_cra_name,\ 1482 .cra_driver_name = "hisi_sec_"sec_cra_name,\ 1483 .cra_priority = SEC_PRIORITY,\ 1484 .cra_flags = CRYPTO_ALG_ASYNC | CRYPTO_ALG_ALLOCATES_MEMORY,\ 1485 .cra_blocksize = blk_size,\ 1486 .cra_ctxsize = sizeof(struct sec_ctx),\ 1487 .cra_module = THIS_MODULE,\ 1488 },\ 1489 .init = ctx_init,\ 1490 .exit = ctx_exit,\ 1491 .setkey = sec_set_key,\ 1492 .decrypt = sec_skcipher_decrypt,\ 1493 .encrypt = sec_skcipher_encrypt,\ 1494 .min_keysize = sec_min_key_size,\ 1495 .max_keysize = sec_max_key_size,\ 1496 .ivsize = iv_size,\ 1497 }, 1498 1499 #define SEC_SKCIPHER_ALG(name, key_func, min_key_size, \ 1500 max_key_size, blk_size, iv_size) \ 1501 SEC_SKCIPHER_GEN_ALG(name, key_func, min_key_size, max_key_size, \ 1502 sec_skcipher_ctx_init, sec_skcipher_ctx_exit, blk_size, iv_size) 1503 1504 static struct skcipher_alg sec_skciphers[] = { 1505 SEC_SKCIPHER_ALG("ecb(aes)", sec_setkey_aes_ecb, 1506 AES_MIN_KEY_SIZE, AES_MAX_KEY_SIZE, 1507 AES_BLOCK_SIZE, 0) 1508 1509 SEC_SKCIPHER_ALG("cbc(aes)", sec_setkey_aes_cbc, 1510 AES_MIN_KEY_SIZE, AES_MAX_KEY_SIZE, 1511 AES_BLOCK_SIZE, AES_BLOCK_SIZE) 1512 1513 SEC_SKCIPHER_ALG("xts(aes)", sec_setkey_aes_xts, 1514 SEC_XTS_MIN_KEY_SIZE, SEC_XTS_MAX_KEY_SIZE, 1515 AES_BLOCK_SIZE, AES_BLOCK_SIZE) 1516 1517 SEC_SKCIPHER_ALG("ecb(des3_ede)", sec_setkey_3des_ecb, 1518 SEC_DES3_2KEY_SIZE, SEC_DES3_3KEY_SIZE, 1519 DES3_EDE_BLOCK_SIZE, 0) 1520 1521 SEC_SKCIPHER_ALG("cbc(des3_ede)", sec_setkey_3des_cbc, 1522 SEC_DES3_2KEY_SIZE, SEC_DES3_3KEY_SIZE, 1523 DES3_EDE_BLOCK_SIZE, DES3_EDE_BLOCK_SIZE) 1524 1525 SEC_SKCIPHER_ALG("xts(sm4)", sec_setkey_sm4_xts, 1526 SEC_XTS_MIN_KEY_SIZE, SEC_XTS_MIN_KEY_SIZE, 1527 AES_BLOCK_SIZE, AES_BLOCK_SIZE) 1528 1529 SEC_SKCIPHER_ALG("cbc(sm4)", sec_setkey_sm4_cbc, 1530 AES_MIN_KEY_SIZE, AES_MIN_KEY_SIZE, 1531 AES_BLOCK_SIZE, AES_BLOCK_SIZE) 1532 }; 1533 1534 static int sec_aead_param_check(struct sec_ctx *ctx, struct sec_req *sreq) 1535 { 1536 u8 c_alg = ctx->c_ctx.c_alg; 1537 struct aead_request *req = sreq->aead_req.aead_req; 1538 struct crypto_aead *tfm = crypto_aead_reqtfm(req); 1539 size_t authsize = crypto_aead_authsize(tfm); 1540 1541 if (unlikely(!req->src || !req->dst || !req->cryptlen || 1542 req->assoclen > SEC_MAX_AAD_LEN)) { 1543 dev_err(SEC_CTX_DEV(ctx), "aead input param error!\n"); 1544 return -EINVAL; 1545 } 1546 1547 if (ctx->pbuf_supported && (req->cryptlen + req->assoclen) <= 1548 SEC_PBUF_SZ) 1549 sreq->use_pbuf = true; 1550 else 1551 sreq->use_pbuf = false; 1552 1553 /* Support AES only */ 1554 if (unlikely(c_alg != SEC_CALG_AES)) { 1555 dev_err(SEC_CTX_DEV(ctx), "aead crypto alg error!\n"); 1556 return -EINVAL; 1557 1558 } 1559 if (sreq->c_req.encrypt) 1560 sreq->c_req.c_len = req->cryptlen; 1561 else 1562 sreq->c_req.c_len = req->cryptlen - authsize; 1563 1564 if (unlikely(sreq->c_req.c_len & (AES_BLOCK_SIZE - 1))) { 1565 dev_err(SEC_CTX_DEV(ctx), "aead crypto length error!\n"); 1566 return -EINVAL; 1567 } 1568 1569 return 0; 1570 } 1571 1572 static int sec_aead_crypto(struct aead_request *a_req, bool encrypt) 1573 { 1574 struct crypto_aead *tfm = crypto_aead_reqtfm(a_req); 1575 struct sec_req *req = aead_request_ctx(a_req); 1576 struct sec_ctx *ctx = crypto_aead_ctx(tfm); 1577 int ret; 1578 1579 req->flag = a_req->base.flags; 1580 req->aead_req.aead_req = a_req; 1581 req->c_req.encrypt = encrypt; 1582 req->ctx = ctx; 1583 1584 ret = sec_aead_param_check(ctx, req); 1585 if (unlikely(ret)) 1586 return -EINVAL; 1587 1588 return ctx->req_op->process(ctx, req); 1589 } 1590 1591 static int sec_aead_encrypt(struct aead_request *a_req) 1592 { 1593 return sec_aead_crypto(a_req, true); 1594 } 1595 1596 static int sec_aead_decrypt(struct aead_request *a_req) 1597 { 1598 return sec_aead_crypto(a_req, false); 1599 } 1600 1601 #define SEC_AEAD_GEN_ALG(sec_cra_name, sec_set_key, ctx_init,\ 1602 ctx_exit, blk_size, iv_size, max_authsize)\ 1603 {\ 1604 .base = {\ 1605 .cra_name = sec_cra_name,\ 1606 .cra_driver_name = "hisi_sec_"sec_cra_name,\ 1607 .cra_priority = SEC_PRIORITY,\ 1608 .cra_flags = CRYPTO_ALG_ASYNC | CRYPTO_ALG_ALLOCATES_MEMORY,\ 1609 .cra_blocksize = blk_size,\ 1610 .cra_ctxsize = sizeof(struct sec_ctx),\ 1611 .cra_module = THIS_MODULE,\ 1612 },\ 1613 .init = ctx_init,\ 1614 .exit = ctx_exit,\ 1615 .setkey = sec_set_key,\ 1616 .decrypt = sec_aead_decrypt,\ 1617 .encrypt = sec_aead_encrypt,\ 1618 .ivsize = iv_size,\ 1619 .maxauthsize = max_authsize,\ 1620 } 1621 1622 #define SEC_AEAD_ALG(algname, keyfunc, aead_init, blksize, ivsize, authsize)\ 1623 SEC_AEAD_GEN_ALG(algname, keyfunc, aead_init,\ 1624 sec_aead_ctx_exit, blksize, ivsize, authsize) 1625 1626 static struct aead_alg sec_aeads[] = { 1627 SEC_AEAD_ALG("authenc(hmac(sha1),cbc(aes))", 1628 sec_setkey_aes_cbc_sha1, sec_aead_sha1_ctx_init, 1629 AES_BLOCK_SIZE, AES_BLOCK_SIZE, SHA1_DIGEST_SIZE), 1630 1631 SEC_AEAD_ALG("authenc(hmac(sha256),cbc(aes))", 1632 sec_setkey_aes_cbc_sha256, sec_aead_sha256_ctx_init, 1633 AES_BLOCK_SIZE, AES_BLOCK_SIZE, SHA256_DIGEST_SIZE), 1634 1635 SEC_AEAD_ALG("authenc(hmac(sha512),cbc(aes))", 1636 sec_setkey_aes_cbc_sha512, sec_aead_sha512_ctx_init, 1637 AES_BLOCK_SIZE, AES_BLOCK_SIZE, SHA512_DIGEST_SIZE), 1638 }; 1639 1640 int sec_register_to_crypto(void) 1641 { 1642 int ret; 1643 1644 /* To avoid repeat register */ 1645 ret = crypto_register_skciphers(sec_skciphers, 1646 ARRAY_SIZE(sec_skciphers)); 1647 if (ret) 1648 return ret; 1649 1650 ret = crypto_register_aeads(sec_aeads, ARRAY_SIZE(sec_aeads)); 1651 if (ret) 1652 crypto_unregister_skciphers(sec_skciphers, 1653 ARRAY_SIZE(sec_skciphers)); 1654 return ret; 1655 } 1656 1657 void sec_unregister_from_crypto(void) 1658 { 1659 crypto_unregister_skciphers(sec_skciphers, 1660 ARRAY_SIZE(sec_skciphers)); 1661 crypto_unregister_aeads(sec_aeads, ARRAY_SIZE(sec_aeads)); 1662 } 1663