1 // SPDX-License-Identifier: GPL-2.0 2 /* Copyright (c) 2019 HiSilicon Limited. */ 3 4 #include <crypto/aes.h> 5 #include <crypto/algapi.h> 6 #include <crypto/authenc.h> 7 #include <crypto/des.h> 8 #include <crypto/hash.h> 9 #include <crypto/internal/aead.h> 10 #include <crypto/sha.h> 11 #include <crypto/skcipher.h> 12 #include <crypto/xts.h> 13 #include <linux/crypto.h> 14 #include <linux/dma-mapping.h> 15 #include <linux/idr.h> 16 17 #include "sec.h" 18 #include "sec_crypto.h" 19 20 #define SEC_PRIORITY 4001 21 #define SEC_XTS_MIN_KEY_SIZE (2 * AES_MIN_KEY_SIZE) 22 #define SEC_XTS_MAX_KEY_SIZE (2 * AES_MAX_KEY_SIZE) 23 #define SEC_DES3_2KEY_SIZE (2 * DES_KEY_SIZE) 24 #define SEC_DES3_3KEY_SIZE (3 * DES_KEY_SIZE) 25 26 /* SEC sqe(bd) bit operational relative MACRO */ 27 #define SEC_DE_OFFSET 1 28 #define SEC_CIPHER_OFFSET 4 29 #define SEC_SCENE_OFFSET 3 30 #define SEC_DST_SGL_OFFSET 2 31 #define SEC_SRC_SGL_OFFSET 7 32 #define SEC_CKEY_OFFSET 9 33 #define SEC_CMODE_OFFSET 12 34 #define SEC_AKEY_OFFSET 5 35 #define SEC_AEAD_ALG_OFFSET 11 36 #define SEC_AUTH_OFFSET 6 37 38 #define SEC_FLAG_OFFSET 7 39 #define SEC_FLAG_MASK 0x0780 40 #define SEC_TYPE_MASK 0x0F 41 #define SEC_DONE_MASK 0x0001 42 43 #define SEC_TOTAL_IV_SZ (SEC_IV_SIZE * QM_Q_DEPTH) 44 #define SEC_SGL_SGE_NR 128 45 #define SEC_CTX_DEV(ctx) (&(ctx)->sec->qm.pdev->dev) 46 #define SEC_CIPHER_AUTH 0xfe 47 #define SEC_AUTH_CIPHER 0x1 48 #define SEC_MAX_MAC_LEN 64 49 #define SEC_TOTAL_MAC_SZ (SEC_MAX_MAC_LEN * QM_Q_DEPTH) 50 #define SEC_SQE_LEN_RATE 4 51 #define SEC_SQE_CFLAG 2 52 #define SEC_SQE_AEAD_FLAG 3 53 #define SEC_SQE_DONE 0x1 54 55 static atomic_t sec_active_devs; 56 57 /* Get an en/de-cipher queue cyclically to balance load over queues of TFM */ 58 static inline int sec_alloc_queue_id(struct sec_ctx *ctx, struct sec_req *req) 59 { 60 if (req->c_req.encrypt) 61 return (u32)atomic_inc_return(&ctx->enc_qcyclic) % 62 ctx->hlf_q_num; 63 64 return (u32)atomic_inc_return(&ctx->dec_qcyclic) % ctx->hlf_q_num + 65 ctx->hlf_q_num; 66 } 67 68 static inline void sec_free_queue_id(struct sec_ctx *ctx, struct sec_req *req) 69 { 70 if (req->c_req.encrypt) 71 atomic_dec(&ctx->enc_qcyclic); 72 else 73 atomic_dec(&ctx->dec_qcyclic); 74 } 75 76 static int sec_alloc_req_id(struct sec_req *req, struct sec_qp_ctx *qp_ctx) 77 { 78 int req_id; 79 80 mutex_lock(&qp_ctx->req_lock); 81 82 req_id = idr_alloc_cyclic(&qp_ctx->req_idr, NULL, 83 0, QM_Q_DEPTH, GFP_ATOMIC); 84 mutex_unlock(&qp_ctx->req_lock); 85 if (unlikely(req_id < 0)) { 86 dev_err(SEC_CTX_DEV(req->ctx), "alloc req id fail!\n"); 87 return req_id; 88 } 89 90 req->qp_ctx = qp_ctx; 91 qp_ctx->req_list[req_id] = req; 92 return req_id; 93 } 94 95 static void sec_free_req_id(struct sec_req *req) 96 { 97 struct sec_qp_ctx *qp_ctx = req->qp_ctx; 98 int req_id = req->req_id; 99 100 if (unlikely(req_id < 0 || req_id >= QM_Q_DEPTH)) { 101 dev_err(SEC_CTX_DEV(req->ctx), "free request id invalid!\n"); 102 return; 103 } 104 105 qp_ctx->req_list[req_id] = NULL; 106 req->qp_ctx = NULL; 107 108 mutex_lock(&qp_ctx->req_lock); 109 idr_remove(&qp_ctx->req_idr, req_id); 110 mutex_unlock(&qp_ctx->req_lock); 111 } 112 113 static int sec_aead_verify(struct sec_req *req, struct sec_qp_ctx *qp_ctx) 114 { 115 struct aead_request *aead_req = req->aead_req.aead_req; 116 struct crypto_aead *tfm = crypto_aead_reqtfm(aead_req); 117 u8 *mac_out = qp_ctx->res[req->req_id].out_mac; 118 size_t authsize = crypto_aead_authsize(tfm); 119 u8 *mac = mac_out + SEC_MAX_MAC_LEN; 120 struct scatterlist *sgl = aead_req->src; 121 size_t sz; 122 123 sz = sg_pcopy_to_buffer(sgl, sg_nents(sgl), mac, authsize, 124 aead_req->cryptlen + aead_req->assoclen - 125 authsize); 126 if (unlikely(sz != authsize || memcmp(mac_out, mac, sz))) { 127 dev_err(SEC_CTX_DEV(req->ctx), "aead verify failure!\n"); 128 return -EBADMSG; 129 } 130 131 return 0; 132 } 133 134 static void sec_req_cb(struct hisi_qp *qp, void *resp) 135 { 136 struct sec_qp_ctx *qp_ctx = qp->qp_ctx; 137 struct sec_sqe *bd = resp; 138 struct sec_ctx *ctx; 139 struct sec_req *req; 140 u16 done, flag; 141 int err = 0; 142 u8 type; 143 144 type = bd->type_cipher_auth & SEC_TYPE_MASK; 145 if (unlikely(type != SEC_BD_TYPE2)) { 146 pr_err("err bd type [%d]\n", type); 147 return; 148 } 149 150 req = qp_ctx->req_list[le16_to_cpu(bd->type2.tag)]; 151 req->err_type = bd->type2.error_type; 152 ctx = req->ctx; 153 done = le16_to_cpu(bd->type2.done_flag) & SEC_DONE_MASK; 154 flag = (le16_to_cpu(bd->type2.done_flag) & 155 SEC_FLAG_MASK) >> SEC_FLAG_OFFSET; 156 if (unlikely(req->err_type || done != SEC_SQE_DONE || 157 (ctx->alg_type == SEC_SKCIPHER && flag != SEC_SQE_CFLAG) || 158 (ctx->alg_type == SEC_AEAD && flag != SEC_SQE_AEAD_FLAG))) { 159 dev_err(SEC_CTX_DEV(ctx), 160 "err_type[%d],done[%d],flag[%d]\n", 161 req->err_type, done, flag); 162 err = -EIO; 163 } 164 165 if (ctx->alg_type == SEC_AEAD && !req->c_req.encrypt) 166 err = sec_aead_verify(req, qp_ctx); 167 168 atomic64_inc(&ctx->sec->debug.dfx.recv_cnt); 169 170 ctx->req_op->buf_unmap(ctx, req); 171 172 ctx->req_op->callback(ctx, req, err); 173 } 174 175 static int sec_bd_send(struct sec_ctx *ctx, struct sec_req *req) 176 { 177 struct sec_qp_ctx *qp_ctx = req->qp_ctx; 178 int ret; 179 180 mutex_lock(&qp_ctx->req_lock); 181 ret = hisi_qp_send(qp_ctx->qp, &req->sec_sqe); 182 mutex_unlock(&qp_ctx->req_lock); 183 atomic64_inc(&ctx->sec->debug.dfx.send_cnt); 184 185 if (unlikely(ret == -EBUSY)) 186 return -ENOBUFS; 187 188 if (!ret) { 189 if (req->fake_busy) 190 ret = -EBUSY; 191 else 192 ret = -EINPROGRESS; 193 } 194 195 return ret; 196 } 197 198 /* Get DMA memory resources */ 199 static int sec_alloc_civ_resource(struct device *dev, struct sec_alg_res *res) 200 { 201 int i; 202 203 res->c_ivin = dma_alloc_coherent(dev, SEC_TOTAL_IV_SZ, 204 &res->c_ivin_dma, GFP_KERNEL); 205 if (!res->c_ivin) 206 return -ENOMEM; 207 208 for (i = 1; i < QM_Q_DEPTH; i++) { 209 res[i].c_ivin_dma = res->c_ivin_dma + i * SEC_IV_SIZE; 210 res[i].c_ivin = res->c_ivin + i * SEC_IV_SIZE; 211 } 212 213 return 0; 214 } 215 216 static void sec_free_civ_resource(struct device *dev, struct sec_alg_res *res) 217 { 218 if (res->c_ivin) 219 dma_free_coherent(dev, SEC_TOTAL_IV_SZ, 220 res->c_ivin, res->c_ivin_dma); 221 } 222 223 static int sec_alloc_mac_resource(struct device *dev, struct sec_alg_res *res) 224 { 225 int i; 226 227 res->out_mac = dma_alloc_coherent(dev, SEC_TOTAL_MAC_SZ << 1, 228 &res->out_mac_dma, GFP_KERNEL); 229 if (!res->out_mac) 230 return -ENOMEM; 231 232 for (i = 1; i < QM_Q_DEPTH; i++) { 233 res[i].out_mac_dma = res->out_mac_dma + 234 i * (SEC_MAX_MAC_LEN << 1); 235 res[i].out_mac = res->out_mac + i * (SEC_MAX_MAC_LEN << 1); 236 } 237 238 return 0; 239 } 240 241 static void sec_free_mac_resource(struct device *dev, struct sec_alg_res *res) 242 { 243 if (res->out_mac) 244 dma_free_coherent(dev, SEC_TOTAL_MAC_SZ << 1, 245 res->out_mac, res->out_mac_dma); 246 } 247 248 static int sec_alg_resource_alloc(struct sec_ctx *ctx, 249 struct sec_qp_ctx *qp_ctx) 250 { 251 struct device *dev = SEC_CTX_DEV(ctx); 252 struct sec_alg_res *res = qp_ctx->res; 253 int ret; 254 255 ret = sec_alloc_civ_resource(dev, res); 256 if (ret) 257 return ret; 258 259 if (ctx->alg_type == SEC_AEAD) { 260 ret = sec_alloc_mac_resource(dev, res); 261 if (ret) 262 goto get_fail; 263 } 264 265 return 0; 266 get_fail: 267 sec_free_civ_resource(dev, res); 268 269 return ret; 270 } 271 272 static void sec_alg_resource_free(struct sec_ctx *ctx, 273 struct sec_qp_ctx *qp_ctx) 274 { 275 struct device *dev = SEC_CTX_DEV(ctx); 276 277 sec_free_civ_resource(dev, qp_ctx->res); 278 279 if (ctx->alg_type == SEC_AEAD) 280 sec_free_mac_resource(dev, qp_ctx->res); 281 } 282 283 static int sec_create_qp_ctx(struct hisi_qm *qm, struct sec_ctx *ctx, 284 int qp_ctx_id, int alg_type) 285 { 286 struct device *dev = SEC_CTX_DEV(ctx); 287 struct sec_qp_ctx *qp_ctx; 288 struct hisi_qp *qp; 289 int ret = -ENOMEM; 290 291 qp = hisi_qm_create_qp(qm, alg_type); 292 if (IS_ERR(qp)) 293 return PTR_ERR(qp); 294 295 qp_ctx = &ctx->qp_ctx[qp_ctx_id]; 296 qp->req_type = 0; 297 qp->qp_ctx = qp_ctx; 298 qp->req_cb = sec_req_cb; 299 qp_ctx->qp = qp; 300 qp_ctx->ctx = ctx; 301 302 mutex_init(&qp_ctx->req_lock); 303 atomic_set(&qp_ctx->pending_reqs, 0); 304 idr_init(&qp_ctx->req_idr); 305 306 qp_ctx->c_in_pool = hisi_acc_create_sgl_pool(dev, QM_Q_DEPTH, 307 SEC_SGL_SGE_NR); 308 if (IS_ERR(qp_ctx->c_in_pool)) { 309 dev_err(dev, "fail to create sgl pool for input!\n"); 310 goto err_destroy_idr; 311 } 312 313 qp_ctx->c_out_pool = hisi_acc_create_sgl_pool(dev, QM_Q_DEPTH, 314 SEC_SGL_SGE_NR); 315 if (IS_ERR(qp_ctx->c_out_pool)) { 316 dev_err(dev, "fail to create sgl pool for output!\n"); 317 goto err_free_c_in_pool; 318 } 319 320 ret = sec_alg_resource_alloc(ctx, qp_ctx); 321 if (ret) 322 goto err_free_c_out_pool; 323 324 ret = hisi_qm_start_qp(qp, 0); 325 if (ret < 0) 326 goto err_queue_free; 327 328 return 0; 329 330 err_queue_free: 331 sec_alg_resource_free(ctx, qp_ctx); 332 err_free_c_out_pool: 333 hisi_acc_free_sgl_pool(dev, qp_ctx->c_out_pool); 334 err_free_c_in_pool: 335 hisi_acc_free_sgl_pool(dev, qp_ctx->c_in_pool); 336 err_destroy_idr: 337 idr_destroy(&qp_ctx->req_idr); 338 hisi_qm_release_qp(qp); 339 340 return ret; 341 } 342 343 static void sec_release_qp_ctx(struct sec_ctx *ctx, 344 struct sec_qp_ctx *qp_ctx) 345 { 346 struct device *dev = SEC_CTX_DEV(ctx); 347 348 hisi_qm_stop_qp(qp_ctx->qp); 349 sec_alg_resource_free(ctx, qp_ctx); 350 351 hisi_acc_free_sgl_pool(dev, qp_ctx->c_out_pool); 352 hisi_acc_free_sgl_pool(dev, qp_ctx->c_in_pool); 353 354 idr_destroy(&qp_ctx->req_idr); 355 hisi_qm_release_qp(qp_ctx->qp); 356 } 357 358 static int sec_ctx_base_init(struct sec_ctx *ctx) 359 { 360 struct sec_dev *sec; 361 int i, ret; 362 363 sec = sec_find_device(cpu_to_node(smp_processor_id())); 364 if (!sec) { 365 pr_err("Can not find proper Hisilicon SEC device!\n"); 366 return -ENODEV; 367 } 368 ctx->sec = sec; 369 ctx->hlf_q_num = sec->ctx_q_num >> 1; 370 371 /* Half of queue depth is taken as fake requests limit in the queue. */ 372 ctx->fake_req_limit = QM_Q_DEPTH >> 1; 373 ctx->qp_ctx = kcalloc(sec->ctx_q_num, sizeof(struct sec_qp_ctx), 374 GFP_KERNEL); 375 if (!ctx->qp_ctx) 376 return -ENOMEM; 377 378 for (i = 0; i < sec->ctx_q_num; i++) { 379 ret = sec_create_qp_ctx(&sec->qm, ctx, i, 0); 380 if (ret) 381 goto err_sec_release_qp_ctx; 382 } 383 384 return 0; 385 err_sec_release_qp_ctx: 386 for (i = i - 1; i >= 0; i--) 387 sec_release_qp_ctx(ctx, &ctx->qp_ctx[i]); 388 389 kfree(ctx->qp_ctx); 390 return ret; 391 } 392 393 static void sec_ctx_base_uninit(struct sec_ctx *ctx) 394 { 395 int i; 396 397 for (i = 0; i < ctx->sec->ctx_q_num; i++) 398 sec_release_qp_ctx(ctx, &ctx->qp_ctx[i]); 399 400 kfree(ctx->qp_ctx); 401 } 402 403 static int sec_cipher_init(struct sec_ctx *ctx) 404 { 405 struct sec_cipher_ctx *c_ctx = &ctx->c_ctx; 406 407 c_ctx->c_key = dma_alloc_coherent(SEC_CTX_DEV(ctx), SEC_MAX_KEY_SIZE, 408 &c_ctx->c_key_dma, GFP_KERNEL); 409 if (!c_ctx->c_key) 410 return -ENOMEM; 411 412 return 0; 413 } 414 415 static void sec_cipher_uninit(struct sec_ctx *ctx) 416 { 417 struct sec_cipher_ctx *c_ctx = &ctx->c_ctx; 418 419 memzero_explicit(c_ctx->c_key, SEC_MAX_KEY_SIZE); 420 dma_free_coherent(SEC_CTX_DEV(ctx), SEC_MAX_KEY_SIZE, 421 c_ctx->c_key, c_ctx->c_key_dma); 422 } 423 424 static int sec_auth_init(struct sec_ctx *ctx) 425 { 426 struct sec_auth_ctx *a_ctx = &ctx->a_ctx; 427 428 a_ctx->a_key = dma_alloc_coherent(SEC_CTX_DEV(ctx), SEC_MAX_KEY_SIZE, 429 &a_ctx->a_key_dma, GFP_KERNEL); 430 if (!a_ctx->a_key) 431 return -ENOMEM; 432 433 return 0; 434 } 435 436 static void sec_auth_uninit(struct sec_ctx *ctx) 437 { 438 struct sec_auth_ctx *a_ctx = &ctx->a_ctx; 439 440 memzero_explicit(a_ctx->a_key, SEC_MAX_KEY_SIZE); 441 dma_free_coherent(SEC_CTX_DEV(ctx), SEC_MAX_KEY_SIZE, 442 a_ctx->a_key, a_ctx->a_key_dma); 443 } 444 445 static int sec_skcipher_init(struct crypto_skcipher *tfm) 446 { 447 struct sec_ctx *ctx = crypto_skcipher_ctx(tfm); 448 int ret; 449 450 ctx = crypto_skcipher_ctx(tfm); 451 ctx->alg_type = SEC_SKCIPHER; 452 crypto_skcipher_set_reqsize(tfm, sizeof(struct sec_req)); 453 ctx->c_ctx.ivsize = crypto_skcipher_ivsize(tfm); 454 if (ctx->c_ctx.ivsize > SEC_IV_SIZE) { 455 dev_err(SEC_CTX_DEV(ctx), "get error skcipher iv size!\n"); 456 return -EINVAL; 457 } 458 459 ret = sec_ctx_base_init(ctx); 460 if (ret) 461 return ret; 462 463 ret = sec_cipher_init(ctx); 464 if (ret) 465 goto err_cipher_init; 466 467 return 0; 468 err_cipher_init: 469 sec_ctx_base_uninit(ctx); 470 471 return ret; 472 } 473 474 static void sec_skcipher_uninit(struct crypto_skcipher *tfm) 475 { 476 struct sec_ctx *ctx = crypto_skcipher_ctx(tfm); 477 478 sec_cipher_uninit(ctx); 479 sec_ctx_base_uninit(ctx); 480 } 481 482 static int sec_skcipher_3des_setkey(struct sec_cipher_ctx *c_ctx, 483 const u32 keylen, 484 const enum sec_cmode c_mode) 485 { 486 switch (keylen) { 487 case SEC_DES3_2KEY_SIZE: 488 c_ctx->c_key_len = SEC_CKEY_3DES_2KEY; 489 break; 490 case SEC_DES3_3KEY_SIZE: 491 c_ctx->c_key_len = SEC_CKEY_3DES_3KEY; 492 break; 493 default: 494 return -EINVAL; 495 } 496 497 return 0; 498 } 499 500 static int sec_skcipher_aes_sm4_setkey(struct sec_cipher_ctx *c_ctx, 501 const u32 keylen, 502 const enum sec_cmode c_mode) 503 { 504 if (c_mode == SEC_CMODE_XTS) { 505 switch (keylen) { 506 case SEC_XTS_MIN_KEY_SIZE: 507 c_ctx->c_key_len = SEC_CKEY_128BIT; 508 break; 509 case SEC_XTS_MAX_KEY_SIZE: 510 c_ctx->c_key_len = SEC_CKEY_256BIT; 511 break; 512 default: 513 pr_err("hisi_sec2: xts mode key error!\n"); 514 return -EINVAL; 515 } 516 } else { 517 switch (keylen) { 518 case AES_KEYSIZE_128: 519 c_ctx->c_key_len = SEC_CKEY_128BIT; 520 break; 521 case AES_KEYSIZE_192: 522 c_ctx->c_key_len = SEC_CKEY_192BIT; 523 break; 524 case AES_KEYSIZE_256: 525 c_ctx->c_key_len = SEC_CKEY_256BIT; 526 break; 527 default: 528 pr_err("hisi_sec2: aes key error!\n"); 529 return -EINVAL; 530 } 531 } 532 533 return 0; 534 } 535 536 static int sec_skcipher_setkey(struct crypto_skcipher *tfm, const u8 *key, 537 const u32 keylen, const enum sec_calg c_alg, 538 const enum sec_cmode c_mode) 539 { 540 struct sec_ctx *ctx = crypto_skcipher_ctx(tfm); 541 struct sec_cipher_ctx *c_ctx = &ctx->c_ctx; 542 int ret; 543 544 if (c_mode == SEC_CMODE_XTS) { 545 ret = xts_verify_key(tfm, key, keylen); 546 if (ret) { 547 dev_err(SEC_CTX_DEV(ctx), "xts mode key err!\n"); 548 return ret; 549 } 550 } 551 552 c_ctx->c_alg = c_alg; 553 c_ctx->c_mode = c_mode; 554 555 switch (c_alg) { 556 case SEC_CALG_3DES: 557 ret = sec_skcipher_3des_setkey(c_ctx, keylen, c_mode); 558 break; 559 case SEC_CALG_AES: 560 case SEC_CALG_SM4: 561 ret = sec_skcipher_aes_sm4_setkey(c_ctx, keylen, c_mode); 562 break; 563 default: 564 return -EINVAL; 565 } 566 567 if (ret) { 568 dev_err(SEC_CTX_DEV(ctx), "set sec key err!\n"); 569 return ret; 570 } 571 572 memcpy(c_ctx->c_key, key, keylen); 573 574 return 0; 575 } 576 577 #define GEN_SEC_SETKEY_FUNC(name, c_alg, c_mode) \ 578 static int sec_setkey_##name(struct crypto_skcipher *tfm, const u8 *key,\ 579 u32 keylen) \ 580 { \ 581 return sec_skcipher_setkey(tfm, key, keylen, c_alg, c_mode); \ 582 } 583 584 GEN_SEC_SETKEY_FUNC(aes_ecb, SEC_CALG_AES, SEC_CMODE_ECB) 585 GEN_SEC_SETKEY_FUNC(aes_cbc, SEC_CALG_AES, SEC_CMODE_CBC) 586 GEN_SEC_SETKEY_FUNC(aes_xts, SEC_CALG_AES, SEC_CMODE_XTS) 587 588 GEN_SEC_SETKEY_FUNC(3des_ecb, SEC_CALG_3DES, SEC_CMODE_ECB) 589 GEN_SEC_SETKEY_FUNC(3des_cbc, SEC_CALG_3DES, SEC_CMODE_CBC) 590 591 GEN_SEC_SETKEY_FUNC(sm4_xts, SEC_CALG_SM4, SEC_CMODE_XTS) 592 GEN_SEC_SETKEY_FUNC(sm4_cbc, SEC_CALG_SM4, SEC_CMODE_CBC) 593 594 static int sec_cipher_map(struct device *dev, struct sec_req *req, 595 struct scatterlist *src, struct scatterlist *dst) 596 { 597 struct sec_cipher_req *c_req = &req->c_req; 598 struct sec_qp_ctx *qp_ctx = req->qp_ctx; 599 600 c_req->c_in = hisi_acc_sg_buf_map_to_hw_sgl(dev, src, 601 qp_ctx->c_in_pool, 602 req->req_id, 603 &c_req->c_in_dma); 604 605 if (IS_ERR(c_req->c_in)) { 606 dev_err(dev, "fail to dma map input sgl buffers!\n"); 607 return PTR_ERR(c_req->c_in); 608 } 609 610 if (dst == src) { 611 c_req->c_out = c_req->c_in; 612 c_req->c_out_dma = c_req->c_in_dma; 613 } else { 614 c_req->c_out = hisi_acc_sg_buf_map_to_hw_sgl(dev, dst, 615 qp_ctx->c_out_pool, 616 req->req_id, 617 &c_req->c_out_dma); 618 619 if (IS_ERR(c_req->c_out)) { 620 dev_err(dev, "fail to dma map output sgl buffers!\n"); 621 hisi_acc_sg_buf_unmap(dev, src, c_req->c_in); 622 return PTR_ERR(c_req->c_out); 623 } 624 } 625 626 return 0; 627 } 628 629 static void sec_cipher_unmap(struct device *dev, struct sec_cipher_req *req, 630 struct scatterlist *src, struct scatterlist *dst) 631 { 632 if (dst != src) 633 hisi_acc_sg_buf_unmap(dev, src, req->c_in); 634 635 hisi_acc_sg_buf_unmap(dev, dst, req->c_out); 636 } 637 638 static int sec_skcipher_sgl_map(struct sec_ctx *ctx, struct sec_req *req) 639 { 640 struct skcipher_request *sq = req->c_req.sk_req; 641 642 return sec_cipher_map(SEC_CTX_DEV(ctx), req, sq->src, sq->dst); 643 } 644 645 static void sec_skcipher_sgl_unmap(struct sec_ctx *ctx, struct sec_req *req) 646 { 647 struct device *dev = SEC_CTX_DEV(ctx); 648 struct sec_cipher_req *c_req = &req->c_req; 649 struct skcipher_request *sk_req = c_req->sk_req; 650 651 sec_cipher_unmap(dev, c_req, sk_req->src, sk_req->dst); 652 } 653 654 static int sec_aead_aes_set_key(struct sec_cipher_ctx *c_ctx, 655 struct crypto_authenc_keys *keys) 656 { 657 switch (keys->enckeylen) { 658 case AES_KEYSIZE_128: 659 c_ctx->c_key_len = SEC_CKEY_128BIT; 660 break; 661 case AES_KEYSIZE_192: 662 c_ctx->c_key_len = SEC_CKEY_192BIT; 663 break; 664 case AES_KEYSIZE_256: 665 c_ctx->c_key_len = SEC_CKEY_256BIT; 666 break; 667 default: 668 pr_err("hisi_sec2: aead aes key error!\n"); 669 return -EINVAL; 670 } 671 memcpy(c_ctx->c_key, keys->enckey, keys->enckeylen); 672 673 return 0; 674 } 675 676 static int sec_aead_auth_set_key(struct sec_auth_ctx *ctx, 677 struct crypto_authenc_keys *keys) 678 { 679 struct crypto_shash *hash_tfm = ctx->hash_tfm; 680 SHASH_DESC_ON_STACK(shash, hash_tfm); 681 int blocksize, ret; 682 683 if (!keys->authkeylen) { 684 pr_err("hisi_sec2: aead auth key error!\n"); 685 return -EINVAL; 686 } 687 688 blocksize = crypto_shash_blocksize(hash_tfm); 689 if (keys->authkeylen > blocksize) { 690 ret = crypto_shash_digest(shash, keys->authkey, 691 keys->authkeylen, ctx->a_key); 692 if (ret) { 693 pr_err("hisi_sec2: aead auth digest error!\n"); 694 return -EINVAL; 695 } 696 ctx->a_key_len = blocksize; 697 } else { 698 memcpy(ctx->a_key, keys->authkey, keys->authkeylen); 699 ctx->a_key_len = keys->authkeylen; 700 } 701 702 return 0; 703 } 704 705 static int sec_aead_setkey(struct crypto_aead *tfm, const u8 *key, 706 const u32 keylen, const enum sec_hash_alg a_alg, 707 const enum sec_calg c_alg, 708 const enum sec_mac_len mac_len, 709 const enum sec_cmode c_mode) 710 { 711 struct sec_ctx *ctx = crypto_aead_ctx(tfm); 712 struct sec_cipher_ctx *c_ctx = &ctx->c_ctx; 713 struct crypto_authenc_keys keys; 714 int ret; 715 716 ctx->a_ctx.a_alg = a_alg; 717 ctx->c_ctx.c_alg = c_alg; 718 ctx->a_ctx.mac_len = mac_len; 719 c_ctx->c_mode = c_mode; 720 721 if (crypto_authenc_extractkeys(&keys, key, keylen)) 722 goto bad_key; 723 724 ret = sec_aead_aes_set_key(c_ctx, &keys); 725 if (ret) { 726 dev_err(SEC_CTX_DEV(ctx), "set sec cipher key err!\n"); 727 goto bad_key; 728 } 729 730 ret = sec_aead_auth_set_key(&ctx->a_ctx, &keys); 731 if (ret) { 732 dev_err(SEC_CTX_DEV(ctx), "set sec auth key err!\n"); 733 goto bad_key; 734 } 735 736 return 0; 737 bad_key: 738 memzero_explicit(&keys, sizeof(struct crypto_authenc_keys)); 739 740 return -EINVAL; 741 } 742 743 744 #define GEN_SEC_AEAD_SETKEY_FUNC(name, aalg, calg, maclen, cmode) \ 745 static int sec_setkey_##name(struct crypto_aead *tfm, const u8 *key, \ 746 u32 keylen) \ 747 { \ 748 return sec_aead_setkey(tfm, key, keylen, aalg, calg, maclen, cmode);\ 749 } 750 751 GEN_SEC_AEAD_SETKEY_FUNC(aes_cbc_sha1, SEC_A_HMAC_SHA1, 752 SEC_CALG_AES, SEC_HMAC_SHA1_MAC, SEC_CMODE_CBC) 753 GEN_SEC_AEAD_SETKEY_FUNC(aes_cbc_sha256, SEC_A_HMAC_SHA256, 754 SEC_CALG_AES, SEC_HMAC_SHA256_MAC, SEC_CMODE_CBC) 755 GEN_SEC_AEAD_SETKEY_FUNC(aes_cbc_sha512, SEC_A_HMAC_SHA512, 756 SEC_CALG_AES, SEC_HMAC_SHA512_MAC, SEC_CMODE_CBC) 757 758 static int sec_aead_sgl_map(struct sec_ctx *ctx, struct sec_req *req) 759 { 760 struct aead_request *aq = req->aead_req.aead_req; 761 762 return sec_cipher_map(SEC_CTX_DEV(ctx), req, aq->src, aq->dst); 763 } 764 765 static void sec_aead_sgl_unmap(struct sec_ctx *ctx, struct sec_req *req) 766 { 767 struct device *dev = SEC_CTX_DEV(ctx); 768 struct sec_cipher_req *cq = &req->c_req; 769 struct aead_request *aq = req->aead_req.aead_req; 770 771 sec_cipher_unmap(dev, cq, aq->src, aq->dst); 772 } 773 774 static int sec_request_transfer(struct sec_ctx *ctx, struct sec_req *req) 775 { 776 int ret; 777 778 ret = ctx->req_op->buf_map(ctx, req); 779 if (unlikely(ret)) 780 return ret; 781 782 ctx->req_op->do_transfer(ctx, req); 783 784 ret = ctx->req_op->bd_fill(ctx, req); 785 if (unlikely(ret)) 786 goto unmap_req_buf; 787 788 return ret; 789 790 unmap_req_buf: 791 ctx->req_op->buf_unmap(ctx, req); 792 793 return ret; 794 } 795 796 static void sec_request_untransfer(struct sec_ctx *ctx, struct sec_req *req) 797 { 798 ctx->req_op->buf_unmap(ctx, req); 799 } 800 801 static void sec_skcipher_copy_iv(struct sec_ctx *ctx, struct sec_req *req) 802 { 803 struct skcipher_request *sk_req = req->c_req.sk_req; 804 u8 *c_ivin = req->qp_ctx->res[req->req_id].c_ivin; 805 806 memcpy(c_ivin, sk_req->iv, ctx->c_ctx.ivsize); 807 } 808 809 static int sec_skcipher_bd_fill(struct sec_ctx *ctx, struct sec_req *req) 810 { 811 struct sec_cipher_ctx *c_ctx = &ctx->c_ctx; 812 struct sec_cipher_req *c_req = &req->c_req; 813 struct sec_sqe *sec_sqe = &req->sec_sqe; 814 u8 scene, sa_type, da_type; 815 u8 bd_type, cipher; 816 u8 de = 0; 817 818 memset(sec_sqe, 0, sizeof(struct sec_sqe)); 819 820 sec_sqe->type2.c_key_addr = cpu_to_le64(c_ctx->c_key_dma); 821 sec_sqe->type2.c_ivin_addr = 822 cpu_to_le64(req->qp_ctx->res[req->req_id].c_ivin_dma); 823 sec_sqe->type2.data_src_addr = cpu_to_le64(c_req->c_in_dma); 824 sec_sqe->type2.data_dst_addr = cpu_to_le64(c_req->c_out_dma); 825 826 sec_sqe->type2.icvw_kmode |= cpu_to_le16(((u16)c_ctx->c_mode) << 827 SEC_CMODE_OFFSET); 828 sec_sqe->type2.c_alg = c_ctx->c_alg; 829 sec_sqe->type2.icvw_kmode |= cpu_to_le16(((u16)c_ctx->c_key_len) << 830 SEC_CKEY_OFFSET); 831 832 bd_type = SEC_BD_TYPE2; 833 if (c_req->encrypt) 834 cipher = SEC_CIPHER_ENC << SEC_CIPHER_OFFSET; 835 else 836 cipher = SEC_CIPHER_DEC << SEC_CIPHER_OFFSET; 837 sec_sqe->type_cipher_auth = bd_type | cipher; 838 839 sa_type = SEC_SGL << SEC_SRC_SGL_OFFSET; 840 scene = SEC_COMM_SCENE << SEC_SCENE_OFFSET; 841 if (c_req->c_in_dma != c_req->c_out_dma) 842 de = 0x1 << SEC_DE_OFFSET; 843 844 sec_sqe->sds_sa_type = (de | scene | sa_type); 845 846 /* Just set DST address type */ 847 da_type = SEC_SGL << SEC_DST_SGL_OFFSET; 848 sec_sqe->sdm_addr_type |= da_type; 849 850 sec_sqe->type2.clen_ivhlen |= cpu_to_le32(c_req->c_len); 851 sec_sqe->type2.tag = cpu_to_le16((u16)req->req_id); 852 853 return 0; 854 } 855 856 static void sec_update_iv(struct sec_req *req, enum sec_alg_type alg_type) 857 { 858 struct aead_request *aead_req = req->aead_req.aead_req; 859 struct skcipher_request *sk_req = req->c_req.sk_req; 860 u32 iv_size = req->ctx->c_ctx.ivsize; 861 struct scatterlist *sgl; 862 unsigned int cryptlen; 863 size_t sz; 864 u8 *iv; 865 866 if (req->c_req.encrypt) 867 sgl = alg_type == SEC_SKCIPHER ? sk_req->dst : aead_req->dst; 868 else 869 sgl = alg_type == SEC_SKCIPHER ? sk_req->src : aead_req->src; 870 871 if (alg_type == SEC_SKCIPHER) { 872 iv = sk_req->iv; 873 cryptlen = sk_req->cryptlen; 874 } else { 875 iv = aead_req->iv; 876 cryptlen = aead_req->cryptlen; 877 } 878 879 sz = sg_pcopy_to_buffer(sgl, sg_nents(sgl), iv, iv_size, 880 cryptlen - iv_size); 881 if (unlikely(sz != iv_size)) 882 dev_err(SEC_CTX_DEV(req->ctx), "copy output iv error!\n"); 883 } 884 885 static void sec_skcipher_callback(struct sec_ctx *ctx, struct sec_req *req, 886 int err) 887 { 888 struct skcipher_request *sk_req = req->c_req.sk_req; 889 struct sec_qp_ctx *qp_ctx = req->qp_ctx; 890 891 atomic_dec(&qp_ctx->pending_reqs); 892 sec_free_req_id(req); 893 894 /* IV output at encrypto of CBC mode */ 895 if (!err && ctx->c_ctx.c_mode == SEC_CMODE_CBC && req->c_req.encrypt) 896 sec_update_iv(req, SEC_SKCIPHER); 897 898 if (req->fake_busy) 899 sk_req->base.complete(&sk_req->base, -EINPROGRESS); 900 901 sk_req->base.complete(&sk_req->base, err); 902 } 903 904 static void sec_aead_copy_iv(struct sec_ctx *ctx, struct sec_req *req) 905 { 906 struct aead_request *aead_req = req->aead_req.aead_req; 907 u8 *c_ivin = req->qp_ctx->res[req->req_id].c_ivin; 908 909 memcpy(c_ivin, aead_req->iv, ctx->c_ctx.ivsize); 910 } 911 912 static void sec_auth_bd_fill_ex(struct sec_auth_ctx *ctx, int dir, 913 struct sec_req *req, struct sec_sqe *sec_sqe) 914 { 915 struct sec_aead_req *a_req = &req->aead_req; 916 struct sec_cipher_req *c_req = &req->c_req; 917 struct aead_request *aq = a_req->aead_req; 918 919 sec_sqe->type2.a_key_addr = cpu_to_le64(ctx->a_key_dma); 920 921 sec_sqe->type2.mac_key_alg = 922 cpu_to_le32(ctx->mac_len / SEC_SQE_LEN_RATE); 923 924 sec_sqe->type2.mac_key_alg |= 925 cpu_to_le32((u32)((ctx->a_key_len) / 926 SEC_SQE_LEN_RATE) << SEC_AKEY_OFFSET); 927 928 sec_sqe->type2.mac_key_alg |= 929 cpu_to_le32((u32)(ctx->a_alg) << SEC_AEAD_ALG_OFFSET); 930 931 sec_sqe->type_cipher_auth |= SEC_AUTH_TYPE1 << SEC_AUTH_OFFSET; 932 933 if (dir) 934 sec_sqe->sds_sa_type &= SEC_CIPHER_AUTH; 935 else 936 sec_sqe->sds_sa_type |= SEC_AUTH_CIPHER; 937 938 sec_sqe->type2.alen_ivllen = cpu_to_le32(c_req->c_len + aq->assoclen); 939 940 sec_sqe->type2.cipher_src_offset = cpu_to_le16((u16)aq->assoclen); 941 942 sec_sqe->type2.mac_addr = 943 cpu_to_le64(req->qp_ctx->res[req->req_id].out_mac_dma); 944 } 945 946 static int sec_aead_bd_fill(struct sec_ctx *ctx, struct sec_req *req) 947 { 948 struct sec_auth_ctx *auth_ctx = &ctx->a_ctx; 949 struct sec_sqe *sec_sqe = &req->sec_sqe; 950 int ret; 951 952 ret = sec_skcipher_bd_fill(ctx, req); 953 if (unlikely(ret)) { 954 dev_err(SEC_CTX_DEV(ctx), "skcipher bd fill is error!\n"); 955 return ret; 956 } 957 958 sec_auth_bd_fill_ex(auth_ctx, req->c_req.encrypt, req, sec_sqe); 959 960 return 0; 961 } 962 963 static void sec_aead_callback(struct sec_ctx *c, struct sec_req *req, int err) 964 { 965 struct aead_request *a_req = req->aead_req.aead_req; 966 struct crypto_aead *tfm = crypto_aead_reqtfm(a_req); 967 struct sec_cipher_req *c_req = &req->c_req; 968 size_t authsize = crypto_aead_authsize(tfm); 969 struct sec_qp_ctx *qp_ctx = req->qp_ctx; 970 size_t sz; 971 972 atomic_dec(&qp_ctx->pending_reqs); 973 974 if (!err && c->c_ctx.c_mode == SEC_CMODE_CBC && c_req->encrypt) 975 sec_update_iv(req, SEC_AEAD); 976 977 /* Copy output mac */ 978 if (!err && c_req->encrypt) { 979 struct scatterlist *sgl = a_req->dst; 980 981 sz = sg_pcopy_from_buffer(sgl, sg_nents(sgl), 982 qp_ctx->res[req->req_id].out_mac, 983 authsize, a_req->cryptlen + 984 a_req->assoclen); 985 986 if (unlikely(sz != authsize)) { 987 dev_err(SEC_CTX_DEV(req->ctx), "copy out mac err!\n"); 988 err = -EINVAL; 989 } 990 } 991 992 sec_free_req_id(req); 993 994 if (req->fake_busy) 995 a_req->base.complete(&a_req->base, -EINPROGRESS); 996 997 a_req->base.complete(&a_req->base, err); 998 } 999 1000 static void sec_request_uninit(struct sec_ctx *ctx, struct sec_req *req) 1001 { 1002 struct sec_qp_ctx *qp_ctx = req->qp_ctx; 1003 1004 atomic_dec(&qp_ctx->pending_reqs); 1005 sec_free_req_id(req); 1006 sec_free_queue_id(ctx, req); 1007 } 1008 1009 static int sec_request_init(struct sec_ctx *ctx, struct sec_req *req) 1010 { 1011 struct sec_qp_ctx *qp_ctx; 1012 int queue_id; 1013 1014 /* To load balance */ 1015 queue_id = sec_alloc_queue_id(ctx, req); 1016 qp_ctx = &ctx->qp_ctx[queue_id]; 1017 1018 req->req_id = sec_alloc_req_id(req, qp_ctx); 1019 if (unlikely(req->req_id < 0)) { 1020 sec_free_queue_id(ctx, req); 1021 return req->req_id; 1022 } 1023 1024 if (ctx->fake_req_limit <= atomic_inc_return(&qp_ctx->pending_reqs)) 1025 req->fake_busy = true; 1026 else 1027 req->fake_busy = false; 1028 1029 return 0; 1030 } 1031 1032 static int sec_process(struct sec_ctx *ctx, struct sec_req *req) 1033 { 1034 int ret; 1035 1036 ret = sec_request_init(ctx, req); 1037 if (unlikely(ret)) 1038 return ret; 1039 1040 ret = sec_request_transfer(ctx, req); 1041 if (unlikely(ret)) 1042 goto err_uninit_req; 1043 1044 /* Output IV as decrypto */ 1045 if (ctx->c_ctx.c_mode == SEC_CMODE_CBC && !req->c_req.encrypt) 1046 sec_update_iv(req, ctx->alg_type); 1047 1048 ret = ctx->req_op->bd_send(ctx, req); 1049 if (unlikely(ret != -EBUSY && ret != -EINPROGRESS)) { 1050 dev_err_ratelimited(SEC_CTX_DEV(ctx), "send sec request failed!\n"); 1051 goto err_send_req; 1052 } 1053 1054 return ret; 1055 1056 err_send_req: 1057 /* As failing, restore the IV from user */ 1058 if (ctx->c_ctx.c_mode == SEC_CMODE_CBC && !req->c_req.encrypt) { 1059 if (ctx->alg_type == SEC_SKCIPHER) 1060 memcpy(req->c_req.sk_req->iv, 1061 req->qp_ctx->res[req->req_id].c_ivin, 1062 ctx->c_ctx.ivsize); 1063 else 1064 memcpy(req->aead_req.aead_req->iv, 1065 req->qp_ctx->res[req->req_id].c_ivin, 1066 ctx->c_ctx.ivsize); 1067 } 1068 1069 sec_request_untransfer(ctx, req); 1070 err_uninit_req: 1071 sec_request_uninit(ctx, req); 1072 1073 return ret; 1074 } 1075 1076 static const struct sec_req_op sec_skcipher_req_ops = { 1077 .buf_map = sec_skcipher_sgl_map, 1078 .buf_unmap = sec_skcipher_sgl_unmap, 1079 .do_transfer = sec_skcipher_copy_iv, 1080 .bd_fill = sec_skcipher_bd_fill, 1081 .bd_send = sec_bd_send, 1082 .callback = sec_skcipher_callback, 1083 .process = sec_process, 1084 }; 1085 1086 static const struct sec_req_op sec_aead_req_ops = { 1087 .buf_map = sec_aead_sgl_map, 1088 .buf_unmap = sec_aead_sgl_unmap, 1089 .do_transfer = sec_aead_copy_iv, 1090 .bd_fill = sec_aead_bd_fill, 1091 .bd_send = sec_bd_send, 1092 .callback = sec_aead_callback, 1093 .process = sec_process, 1094 }; 1095 1096 static int sec_skcipher_ctx_init(struct crypto_skcipher *tfm) 1097 { 1098 struct sec_ctx *ctx = crypto_skcipher_ctx(tfm); 1099 1100 ctx->req_op = &sec_skcipher_req_ops; 1101 1102 return sec_skcipher_init(tfm); 1103 } 1104 1105 static void sec_skcipher_ctx_exit(struct crypto_skcipher *tfm) 1106 { 1107 sec_skcipher_uninit(tfm); 1108 } 1109 1110 static int sec_aead_init(struct crypto_aead *tfm) 1111 { 1112 struct sec_ctx *ctx = crypto_aead_ctx(tfm); 1113 int ret; 1114 1115 crypto_aead_set_reqsize(tfm, sizeof(struct sec_req)); 1116 ctx->alg_type = SEC_AEAD; 1117 ctx->c_ctx.ivsize = crypto_aead_ivsize(tfm); 1118 if (ctx->c_ctx.ivsize > SEC_IV_SIZE) { 1119 dev_err(SEC_CTX_DEV(ctx), "get error aead iv size!\n"); 1120 return -EINVAL; 1121 } 1122 1123 ctx->req_op = &sec_aead_req_ops; 1124 ret = sec_ctx_base_init(ctx); 1125 if (ret) 1126 return ret; 1127 1128 ret = sec_auth_init(ctx); 1129 if (ret) 1130 goto err_auth_init; 1131 1132 ret = sec_cipher_init(ctx); 1133 if (ret) 1134 goto err_cipher_init; 1135 1136 return ret; 1137 1138 err_cipher_init: 1139 sec_auth_uninit(ctx); 1140 err_auth_init: 1141 sec_ctx_base_uninit(ctx); 1142 1143 return ret; 1144 } 1145 1146 static void sec_aead_exit(struct crypto_aead *tfm) 1147 { 1148 struct sec_ctx *ctx = crypto_aead_ctx(tfm); 1149 1150 sec_cipher_uninit(ctx); 1151 sec_auth_uninit(ctx); 1152 sec_ctx_base_uninit(ctx); 1153 } 1154 1155 static int sec_aead_ctx_init(struct crypto_aead *tfm, const char *hash_name) 1156 { 1157 struct sec_ctx *ctx = crypto_aead_ctx(tfm); 1158 struct sec_auth_ctx *auth_ctx = &ctx->a_ctx; 1159 int ret; 1160 1161 ret = sec_aead_init(tfm); 1162 if (ret) { 1163 pr_err("hisi_sec2: aead init error!\n"); 1164 return ret; 1165 } 1166 1167 auth_ctx->hash_tfm = crypto_alloc_shash(hash_name, 0, 0); 1168 if (IS_ERR(auth_ctx->hash_tfm)) { 1169 dev_err(SEC_CTX_DEV(ctx), "aead alloc shash error!\n"); 1170 sec_aead_exit(tfm); 1171 return PTR_ERR(auth_ctx->hash_tfm); 1172 } 1173 1174 return 0; 1175 } 1176 1177 static void sec_aead_ctx_exit(struct crypto_aead *tfm) 1178 { 1179 struct sec_ctx *ctx = crypto_aead_ctx(tfm); 1180 1181 crypto_free_shash(ctx->a_ctx.hash_tfm); 1182 sec_aead_exit(tfm); 1183 } 1184 1185 static int sec_aead_sha1_ctx_init(struct crypto_aead *tfm) 1186 { 1187 return sec_aead_ctx_init(tfm, "sha1"); 1188 } 1189 1190 static int sec_aead_sha256_ctx_init(struct crypto_aead *tfm) 1191 { 1192 return sec_aead_ctx_init(tfm, "sha256"); 1193 } 1194 1195 static int sec_aead_sha512_ctx_init(struct crypto_aead *tfm) 1196 { 1197 return sec_aead_ctx_init(tfm, "sha512"); 1198 } 1199 1200 static int sec_skcipher_param_check(struct sec_ctx *ctx, struct sec_req *sreq) 1201 { 1202 struct skcipher_request *sk_req = sreq->c_req.sk_req; 1203 struct device *dev = SEC_CTX_DEV(ctx); 1204 u8 c_alg = ctx->c_ctx.c_alg; 1205 1206 if (unlikely(!sk_req->src || !sk_req->dst)) { 1207 dev_err(dev, "skcipher input param error!\n"); 1208 return -EINVAL; 1209 } 1210 sreq->c_req.c_len = sk_req->cryptlen; 1211 if (c_alg == SEC_CALG_3DES) { 1212 if (unlikely(sk_req->cryptlen & (DES3_EDE_BLOCK_SIZE - 1))) { 1213 dev_err(dev, "skcipher 3des input length error!\n"); 1214 return -EINVAL; 1215 } 1216 return 0; 1217 } else if (c_alg == SEC_CALG_AES || c_alg == SEC_CALG_SM4) { 1218 if (unlikely(sk_req->cryptlen & (AES_BLOCK_SIZE - 1))) { 1219 dev_err(dev, "skcipher aes input length error!\n"); 1220 return -EINVAL; 1221 } 1222 return 0; 1223 } 1224 1225 dev_err(dev, "skcipher algorithm error!\n"); 1226 return -EINVAL; 1227 } 1228 1229 static int sec_skcipher_crypto(struct skcipher_request *sk_req, bool encrypt) 1230 { 1231 struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(sk_req); 1232 struct sec_req *req = skcipher_request_ctx(sk_req); 1233 struct sec_ctx *ctx = crypto_skcipher_ctx(tfm); 1234 int ret; 1235 1236 if (!sk_req->cryptlen) 1237 return 0; 1238 1239 req->c_req.sk_req = sk_req; 1240 req->c_req.encrypt = encrypt; 1241 req->ctx = ctx; 1242 1243 ret = sec_skcipher_param_check(ctx, req); 1244 if (unlikely(ret)) 1245 return -EINVAL; 1246 1247 return ctx->req_op->process(ctx, req); 1248 } 1249 1250 static int sec_skcipher_encrypt(struct skcipher_request *sk_req) 1251 { 1252 return sec_skcipher_crypto(sk_req, true); 1253 } 1254 1255 static int sec_skcipher_decrypt(struct skcipher_request *sk_req) 1256 { 1257 return sec_skcipher_crypto(sk_req, false); 1258 } 1259 1260 #define SEC_SKCIPHER_GEN_ALG(sec_cra_name, sec_set_key, sec_min_key_size, \ 1261 sec_max_key_size, ctx_init, ctx_exit, blk_size, iv_size)\ 1262 {\ 1263 .base = {\ 1264 .cra_name = sec_cra_name,\ 1265 .cra_driver_name = "hisi_sec_"sec_cra_name,\ 1266 .cra_priority = SEC_PRIORITY,\ 1267 .cra_flags = CRYPTO_ALG_ASYNC,\ 1268 .cra_blocksize = blk_size,\ 1269 .cra_ctxsize = sizeof(struct sec_ctx),\ 1270 .cra_module = THIS_MODULE,\ 1271 },\ 1272 .init = ctx_init,\ 1273 .exit = ctx_exit,\ 1274 .setkey = sec_set_key,\ 1275 .decrypt = sec_skcipher_decrypt,\ 1276 .encrypt = sec_skcipher_encrypt,\ 1277 .min_keysize = sec_min_key_size,\ 1278 .max_keysize = sec_max_key_size,\ 1279 .ivsize = iv_size,\ 1280 }, 1281 1282 #define SEC_SKCIPHER_ALG(name, key_func, min_key_size, \ 1283 max_key_size, blk_size, iv_size) \ 1284 SEC_SKCIPHER_GEN_ALG(name, key_func, min_key_size, max_key_size, \ 1285 sec_skcipher_ctx_init, sec_skcipher_ctx_exit, blk_size, iv_size) 1286 1287 static struct skcipher_alg sec_skciphers[] = { 1288 SEC_SKCIPHER_ALG("ecb(aes)", sec_setkey_aes_ecb, 1289 AES_MIN_KEY_SIZE, AES_MAX_KEY_SIZE, 1290 AES_BLOCK_SIZE, 0) 1291 1292 SEC_SKCIPHER_ALG("cbc(aes)", sec_setkey_aes_cbc, 1293 AES_MIN_KEY_SIZE, AES_MAX_KEY_SIZE, 1294 AES_BLOCK_SIZE, AES_BLOCK_SIZE) 1295 1296 SEC_SKCIPHER_ALG("xts(aes)", sec_setkey_aes_xts, 1297 SEC_XTS_MIN_KEY_SIZE, SEC_XTS_MAX_KEY_SIZE, 1298 AES_BLOCK_SIZE, AES_BLOCK_SIZE) 1299 1300 SEC_SKCIPHER_ALG("ecb(des3_ede)", sec_setkey_3des_ecb, 1301 SEC_DES3_2KEY_SIZE, SEC_DES3_3KEY_SIZE, 1302 DES3_EDE_BLOCK_SIZE, 0) 1303 1304 SEC_SKCIPHER_ALG("cbc(des3_ede)", sec_setkey_3des_cbc, 1305 SEC_DES3_2KEY_SIZE, SEC_DES3_3KEY_SIZE, 1306 DES3_EDE_BLOCK_SIZE, DES3_EDE_BLOCK_SIZE) 1307 1308 SEC_SKCIPHER_ALG("xts(sm4)", sec_setkey_sm4_xts, 1309 SEC_XTS_MIN_KEY_SIZE, SEC_XTS_MIN_KEY_SIZE, 1310 AES_BLOCK_SIZE, AES_BLOCK_SIZE) 1311 1312 SEC_SKCIPHER_ALG("cbc(sm4)", sec_setkey_sm4_cbc, 1313 AES_MIN_KEY_SIZE, AES_MIN_KEY_SIZE, 1314 AES_BLOCK_SIZE, AES_BLOCK_SIZE) 1315 }; 1316 1317 static int sec_aead_param_check(struct sec_ctx *ctx, struct sec_req *sreq) 1318 { 1319 u8 c_alg = ctx->c_ctx.c_alg; 1320 struct aead_request *req = sreq->aead_req.aead_req; 1321 struct crypto_aead *tfm = crypto_aead_reqtfm(req); 1322 size_t authsize = crypto_aead_authsize(tfm); 1323 1324 if (unlikely(!req->src || !req->dst || !req->cryptlen)) { 1325 dev_err(SEC_CTX_DEV(ctx), "aead input param error!\n"); 1326 return -EINVAL; 1327 } 1328 1329 /* Support AES only */ 1330 if (unlikely(c_alg != SEC_CALG_AES)) { 1331 dev_err(SEC_CTX_DEV(ctx), "aead crypto alg error!\n"); 1332 return -EINVAL; 1333 1334 } 1335 if (sreq->c_req.encrypt) 1336 sreq->c_req.c_len = req->cryptlen; 1337 else 1338 sreq->c_req.c_len = req->cryptlen - authsize; 1339 1340 if (unlikely(sreq->c_req.c_len & (AES_BLOCK_SIZE - 1))) { 1341 dev_err(SEC_CTX_DEV(ctx), "aead crypto length error!\n"); 1342 return -EINVAL; 1343 } 1344 1345 return 0; 1346 } 1347 1348 static int sec_aead_crypto(struct aead_request *a_req, bool encrypt) 1349 { 1350 struct crypto_aead *tfm = crypto_aead_reqtfm(a_req); 1351 struct sec_req *req = aead_request_ctx(a_req); 1352 struct sec_ctx *ctx = crypto_aead_ctx(tfm); 1353 int ret; 1354 1355 req->aead_req.aead_req = a_req; 1356 req->c_req.encrypt = encrypt; 1357 req->ctx = ctx; 1358 1359 ret = sec_aead_param_check(ctx, req); 1360 if (unlikely(ret)) 1361 return -EINVAL; 1362 1363 return ctx->req_op->process(ctx, req); 1364 } 1365 1366 static int sec_aead_encrypt(struct aead_request *a_req) 1367 { 1368 return sec_aead_crypto(a_req, true); 1369 } 1370 1371 static int sec_aead_decrypt(struct aead_request *a_req) 1372 { 1373 return sec_aead_crypto(a_req, false); 1374 } 1375 1376 #define SEC_AEAD_GEN_ALG(sec_cra_name, sec_set_key, ctx_init,\ 1377 ctx_exit, blk_size, iv_size, max_authsize)\ 1378 {\ 1379 .base = {\ 1380 .cra_name = sec_cra_name,\ 1381 .cra_driver_name = "hisi_sec_"sec_cra_name,\ 1382 .cra_priority = SEC_PRIORITY,\ 1383 .cra_flags = CRYPTO_ALG_ASYNC,\ 1384 .cra_blocksize = blk_size,\ 1385 .cra_ctxsize = sizeof(struct sec_ctx),\ 1386 .cra_module = THIS_MODULE,\ 1387 },\ 1388 .init = ctx_init,\ 1389 .exit = ctx_exit,\ 1390 .setkey = sec_set_key,\ 1391 .decrypt = sec_aead_decrypt,\ 1392 .encrypt = sec_aead_encrypt,\ 1393 .ivsize = iv_size,\ 1394 .maxauthsize = max_authsize,\ 1395 } 1396 1397 #define SEC_AEAD_ALG(algname, keyfunc, aead_init, blksize, ivsize, authsize)\ 1398 SEC_AEAD_GEN_ALG(algname, keyfunc, aead_init,\ 1399 sec_aead_ctx_exit, blksize, ivsize, authsize) 1400 1401 static struct aead_alg sec_aeads[] = { 1402 SEC_AEAD_ALG("authenc(hmac(sha1),cbc(aes))", 1403 sec_setkey_aes_cbc_sha1, sec_aead_sha1_ctx_init, 1404 AES_BLOCK_SIZE, AES_BLOCK_SIZE, SHA1_DIGEST_SIZE), 1405 1406 SEC_AEAD_ALG("authenc(hmac(sha256),cbc(aes))", 1407 sec_setkey_aes_cbc_sha256, sec_aead_sha256_ctx_init, 1408 AES_BLOCK_SIZE, AES_BLOCK_SIZE, SHA256_DIGEST_SIZE), 1409 1410 SEC_AEAD_ALG("authenc(hmac(sha512),cbc(aes))", 1411 sec_setkey_aes_cbc_sha512, sec_aead_sha512_ctx_init, 1412 AES_BLOCK_SIZE, AES_BLOCK_SIZE, SHA512_DIGEST_SIZE), 1413 }; 1414 1415 int sec_register_to_crypto(void) 1416 { 1417 int ret = 0; 1418 1419 /* To avoid repeat register */ 1420 if (atomic_add_return(1, &sec_active_devs) == 1) { 1421 ret = crypto_register_skciphers(sec_skciphers, 1422 ARRAY_SIZE(sec_skciphers)); 1423 if (ret) 1424 return ret; 1425 1426 ret = crypto_register_aeads(sec_aeads, ARRAY_SIZE(sec_aeads)); 1427 if (ret) 1428 goto reg_aead_fail; 1429 } 1430 1431 return ret; 1432 1433 reg_aead_fail: 1434 crypto_unregister_skciphers(sec_skciphers, ARRAY_SIZE(sec_skciphers)); 1435 1436 return ret; 1437 } 1438 1439 void sec_unregister_from_crypto(void) 1440 { 1441 if (atomic_sub_return(1, &sec_active_devs) == 0) { 1442 crypto_unregister_skciphers(sec_skciphers, 1443 ARRAY_SIZE(sec_skciphers)); 1444 crypto_unregister_aeads(sec_aeads, ARRAY_SIZE(sec_aeads)); 1445 } 1446 } 1447