1416d8220SZaibo Xu // SPDX-License-Identifier: GPL-2.0 2416d8220SZaibo Xu /* Copyright (c) 2019 HiSilicon Limited. */ 3416d8220SZaibo Xu 4416d8220SZaibo Xu #include <crypto/aes.h> 56c46a329SKai Ye #include <crypto/aead.h> 6416d8220SZaibo Xu #include <crypto/algapi.h> 72f072d75SZaibo Xu #include <crypto/authenc.h> 8416d8220SZaibo Xu #include <crypto/des.h> 92f072d75SZaibo Xu #include <crypto/hash.h> 102f072d75SZaibo Xu #include <crypto/internal/aead.h> 11ae6ce7b1SKai Ye #include <crypto/internal/des.h> 12a24d22b2SEric Biggers #include <crypto/sha1.h> 13a24d22b2SEric Biggers #include <crypto/sha2.h> 14416d8220SZaibo Xu #include <crypto/skcipher.h> 15416d8220SZaibo Xu #include <crypto/xts.h> 16416d8220SZaibo Xu #include <linux/crypto.h> 17416d8220SZaibo Xu #include <linux/dma-mapping.h> 18416d8220SZaibo Xu #include <linux/idr.h> 19416d8220SZaibo Xu 20416d8220SZaibo Xu #include "sec.h" 21416d8220SZaibo Xu #include "sec_crypto.h" 22416d8220SZaibo Xu 23416d8220SZaibo Xu #define SEC_PRIORITY 4001 24416d8220SZaibo Xu #define SEC_XTS_MIN_KEY_SIZE (2 * AES_MIN_KEY_SIZE) 255652d55aSKai Ye #define SEC_XTS_MID_KEY_SIZE (3 * AES_MIN_KEY_SIZE) 26416d8220SZaibo Xu #define SEC_XTS_MAX_KEY_SIZE (2 * AES_MAX_KEY_SIZE) 27416d8220SZaibo Xu #define SEC_DES3_2KEY_SIZE (2 * DES_KEY_SIZE) 28416d8220SZaibo Xu #define SEC_DES3_3KEY_SIZE (3 * DES_KEY_SIZE) 29416d8220SZaibo Xu 30416d8220SZaibo Xu /* SEC sqe(bd) bit operational relative MACRO */ 31416d8220SZaibo Xu #define SEC_DE_OFFSET 1 32416d8220SZaibo Xu #define SEC_CIPHER_OFFSET 4 33416d8220SZaibo Xu #define SEC_SCENE_OFFSET 3 34416d8220SZaibo Xu #define SEC_DST_SGL_OFFSET 2 35416d8220SZaibo Xu #define SEC_SRC_SGL_OFFSET 7 36416d8220SZaibo Xu #define SEC_CKEY_OFFSET 9 37416d8220SZaibo Xu #define SEC_CMODE_OFFSET 12 382f072d75SZaibo Xu #define SEC_AKEY_OFFSET 5 392f072d75SZaibo Xu #define SEC_AEAD_ALG_OFFSET 11 402f072d75SZaibo Xu #define SEC_AUTH_OFFSET 6 412f072d75SZaibo Xu 42adc3f65aSKai Ye #define SEC_DE_OFFSET_V3 9 43adc3f65aSKai Ye #define SEC_SCENE_OFFSET_V3 5 44adc3f65aSKai Ye #define SEC_CKEY_OFFSET_V3 13 45566f060fSKai Ye #define SEC_CTR_CNT_OFFSET 25 46566f060fSKai Ye #define SEC_CTR_CNT_ROLLOVER 2 47adc3f65aSKai Ye #define SEC_SRC_SGL_OFFSET_V3 11 48adc3f65aSKai Ye #define SEC_DST_SGL_OFFSET_V3 14 49adc3f65aSKai Ye #define SEC_CALG_OFFSET_V3 4 50adc3f65aSKai Ye #define SEC_AKEY_OFFSET_V3 9 51adc3f65aSKai Ye #define SEC_MAC_OFFSET_V3 4 52adc3f65aSKai Ye #define SEC_AUTH_ALG_OFFSET_V3 15 53adc3f65aSKai Ye #define SEC_CIPHER_AUTH_V3 0xbf 54adc3f65aSKai Ye #define SEC_AUTH_CIPHER_V3 0x40 55416d8220SZaibo Xu #define SEC_FLAG_OFFSET 7 56416d8220SZaibo Xu #define SEC_FLAG_MASK 0x0780 57416d8220SZaibo Xu #define SEC_TYPE_MASK 0x0F 58416d8220SZaibo Xu #define SEC_DONE_MASK 0x0001 59668f1ab7SKai Ye #define SEC_ICV_MASK 0x000E 60adc3f65aSKai Ye #define SEC_SQE_LEN_RATE_MASK 0x3 61416d8220SZaibo Xu 62129a9f34SWeili Qian #define SEC_TOTAL_IV_SZ(depth) (SEC_IV_SIZE * (depth)) 63416d8220SZaibo Xu #define SEC_SGL_SGE_NR 128 642f072d75SZaibo Xu #define SEC_CIPHER_AUTH 0xfe 652f072d75SZaibo Xu #define SEC_AUTH_CIPHER 0x1 662f072d75SZaibo Xu #define SEC_MAX_MAC_LEN 64 672514f559SLongfang Liu #define SEC_MAX_AAD_LEN 65535 685e340558SKai Ye #define SEC_MAX_CCM_AAD_LEN 65279 69129a9f34SWeili Qian #define SEC_TOTAL_MAC_SZ(depth) (SEC_MAX_MAC_LEN * (depth)) 7074b58db8SLongfang Liu 7174b58db8SLongfang Liu #define SEC_PBUF_SZ 512 7274b58db8SLongfang Liu #define SEC_PBUF_IV_OFFSET SEC_PBUF_SZ 7374b58db8SLongfang Liu #define SEC_PBUF_MAC_OFFSET (SEC_PBUF_SZ + SEC_IV_SIZE) 7474b58db8SLongfang Liu #define SEC_PBUF_PKG (SEC_PBUF_SZ + SEC_IV_SIZE + \ 7574b58db8SLongfang Liu SEC_MAX_MAC_LEN * 2) 7674b58db8SLongfang Liu #define SEC_PBUF_NUM (PAGE_SIZE / SEC_PBUF_PKG) 77129a9f34SWeili Qian #define SEC_PBUF_PAGE_NUM(depth) ((depth) / SEC_PBUF_NUM) 78129a9f34SWeili Qian #define SEC_PBUF_LEFT_SZ(depth) (SEC_PBUF_PKG * ((depth) - \ 79129a9f34SWeili Qian SEC_PBUF_PAGE_NUM(depth) * SEC_PBUF_NUM)) 80129a9f34SWeili Qian #define SEC_TOTAL_PBUF_SZ(depth) (PAGE_SIZE * SEC_PBUF_PAGE_NUM(depth) + \ 81129a9f34SWeili Qian SEC_PBUF_LEFT_SZ(depth)) 8274b58db8SLongfang Liu 832f072d75SZaibo Xu #define SEC_SQE_LEN_RATE 4 84d6de2a59SZaibo Xu #define SEC_SQE_CFLAG 2 852f072d75SZaibo Xu #define SEC_SQE_AEAD_FLAG 3 86d6de2a59SZaibo Xu #define SEC_SQE_DONE 0x1 87668f1ab7SKai Ye #define SEC_ICV_ERR 0x2 88c16a70c1SKai Ye #define MIN_MAC_LEN 4 89c16a70c1SKai Ye #define MAC_LEN_MASK 0x1U 907b44c0eeSKai Ye #define MAX_INPUT_DATA_LEN 0xFFFE00 917b44c0eeSKai Ye #define BITS_MASK 0xFF 927b44c0eeSKai Ye #define BYTE_BITS 0x8 935652d55aSKai Ye #define SEC_XTS_NAME_SZ 0x3 94c16a70c1SKai Ye #define IV_CM_CAL_NUM 2 95c16a70c1SKai Ye #define IV_CL_MASK 0x7 96c16a70c1SKai Ye #define IV_CL_MIN 2 97c16a70c1SKai Ye #define IV_CL_MID 4 98c16a70c1SKai Ye #define IV_CL_MAX 8 99c16a70c1SKai Ye #define IV_FLAGS_OFFSET 0x6 100c16a70c1SKai Ye #define IV_CM_OFFSET 0x3 101c16a70c1SKai Ye #define IV_LAST_BYTE1 1 102c16a70c1SKai Ye #define IV_LAST_BYTE2 2 103c16a70c1SKai Ye #define IV_LAST_BYTE_MASK 0xFF 104c16a70c1SKai Ye #define IV_CTR_INIT 0x1 105c16a70c1SKai Ye #define IV_BYTE_OFFSET 0x8 106416d8220SZaibo Xu 107921715b6SWenkai Lin struct sec_skcipher { 108921715b6SWenkai Lin u64 alg_msk; 109921715b6SWenkai Lin struct skcipher_alg alg; 110921715b6SWenkai Lin }; 111921715b6SWenkai Lin 112921715b6SWenkai Lin struct sec_aead { 113921715b6SWenkai Lin u64 alg_msk; 114921715b6SWenkai Lin struct aead_alg alg; 115921715b6SWenkai Lin }; 116921715b6SWenkai Lin 117416d8220SZaibo Xu /* Get an en/de-cipher queue cyclically to balance load over queues of TFM */ 118a181647cSZaibo Xu static inline int sec_alloc_queue_id(struct sec_ctx *ctx, struct sec_req *req) 119416d8220SZaibo Xu { 120416d8220SZaibo Xu if (req->c_req.encrypt) 121416d8220SZaibo Xu return (u32)atomic_inc_return(&ctx->enc_qcyclic) % 122416d8220SZaibo Xu ctx->hlf_q_num; 123416d8220SZaibo Xu 124416d8220SZaibo Xu return (u32)atomic_inc_return(&ctx->dec_qcyclic) % ctx->hlf_q_num + 125416d8220SZaibo Xu ctx->hlf_q_num; 126416d8220SZaibo Xu } 127416d8220SZaibo Xu 128a181647cSZaibo Xu static inline void sec_free_queue_id(struct sec_ctx *ctx, struct sec_req *req) 129416d8220SZaibo Xu { 130416d8220SZaibo Xu if (req->c_req.encrypt) 131416d8220SZaibo Xu atomic_dec(&ctx->enc_qcyclic); 132416d8220SZaibo Xu else 133416d8220SZaibo Xu atomic_dec(&ctx->dec_qcyclic); 134416d8220SZaibo Xu } 135416d8220SZaibo Xu 136416d8220SZaibo Xu static int sec_alloc_req_id(struct sec_req *req, struct sec_qp_ctx *qp_ctx) 137416d8220SZaibo Xu { 138416d8220SZaibo Xu int req_id; 139416d8220SZaibo Xu 14002884a4fSZhengchao Shao spin_lock_bh(&qp_ctx->req_lock); 141129a9f34SWeili Qian req_id = idr_alloc_cyclic(&qp_ctx->req_idr, NULL, 0, qp_ctx->qp->sq_depth, GFP_ATOMIC); 14202884a4fSZhengchao Shao spin_unlock_bh(&qp_ctx->req_lock); 143b9c8d897SZaibo Xu if (unlikely(req_id < 0)) { 144a44dce50SLongfang Liu dev_err(req->ctx->dev, "alloc req id fail!\n"); 145416d8220SZaibo Xu return req_id; 146416d8220SZaibo Xu } 147416d8220SZaibo Xu 148416d8220SZaibo Xu req->qp_ctx = qp_ctx; 149416d8220SZaibo Xu qp_ctx->req_list[req_id] = req; 150633e507fSLongfang Liu 151416d8220SZaibo Xu return req_id; 152416d8220SZaibo Xu } 153416d8220SZaibo Xu 154416d8220SZaibo Xu static void sec_free_req_id(struct sec_req *req) 155416d8220SZaibo Xu { 156416d8220SZaibo Xu struct sec_qp_ctx *qp_ctx = req->qp_ctx; 157416d8220SZaibo Xu int req_id = req->req_id; 158416d8220SZaibo Xu 159129a9f34SWeili Qian if (unlikely(req_id < 0 || req_id >= qp_ctx->qp->sq_depth)) { 160a44dce50SLongfang Liu dev_err(req->ctx->dev, "free request id invalid!\n"); 161416d8220SZaibo Xu return; 162416d8220SZaibo Xu } 163416d8220SZaibo Xu 164416d8220SZaibo Xu qp_ctx->req_list[req_id] = NULL; 165416d8220SZaibo Xu req->qp_ctx = NULL; 166416d8220SZaibo Xu 16702884a4fSZhengchao Shao spin_lock_bh(&qp_ctx->req_lock); 168416d8220SZaibo Xu idr_remove(&qp_ctx->req_idr, req_id); 16902884a4fSZhengchao Shao spin_unlock_bh(&qp_ctx->req_lock); 170416d8220SZaibo Xu } 171416d8220SZaibo Xu 172adc3f65aSKai Ye static u8 pre_parse_finished_bd(struct bd_status *status, void *resp) 173adc3f65aSKai Ye { 174adc3f65aSKai Ye struct sec_sqe *bd = resp; 175adc3f65aSKai Ye 176adc3f65aSKai Ye status->done = le16_to_cpu(bd->type2.done_flag) & SEC_DONE_MASK; 177668f1ab7SKai Ye status->icv = (le16_to_cpu(bd->type2.done_flag) & SEC_ICV_MASK) >> 1; 178adc3f65aSKai Ye status->flag = (le16_to_cpu(bd->type2.done_flag) & 179adc3f65aSKai Ye SEC_FLAG_MASK) >> SEC_FLAG_OFFSET; 180adc3f65aSKai Ye status->tag = le16_to_cpu(bd->type2.tag); 181adc3f65aSKai Ye status->err_type = bd->type2.error_type; 182adc3f65aSKai Ye 183adc3f65aSKai Ye return bd->type_cipher_auth & SEC_TYPE_MASK; 184adc3f65aSKai Ye } 185adc3f65aSKai Ye 186adc3f65aSKai Ye static u8 pre_parse_finished_bd3(struct bd_status *status, void *resp) 187adc3f65aSKai Ye { 188adc3f65aSKai Ye struct sec_sqe3 *bd3 = resp; 189adc3f65aSKai Ye 190adc3f65aSKai Ye status->done = le16_to_cpu(bd3->done_flag) & SEC_DONE_MASK; 191668f1ab7SKai Ye status->icv = (le16_to_cpu(bd3->done_flag) & SEC_ICV_MASK) >> 1; 192adc3f65aSKai Ye status->flag = (le16_to_cpu(bd3->done_flag) & 193adc3f65aSKai Ye SEC_FLAG_MASK) >> SEC_FLAG_OFFSET; 194adc3f65aSKai Ye status->tag = le64_to_cpu(bd3->tag); 195adc3f65aSKai Ye status->err_type = bd3->error_type; 196adc3f65aSKai Ye 197adc3f65aSKai Ye return le32_to_cpu(bd3->bd_param) & SEC_TYPE_MASK; 198adc3f65aSKai Ye } 199adc3f65aSKai Ye 200adc3f65aSKai Ye static int sec_cb_status_check(struct sec_req *req, 201adc3f65aSKai Ye struct bd_status *status) 202adc3f65aSKai Ye { 203adc3f65aSKai Ye struct sec_ctx *ctx = req->ctx; 204adc3f65aSKai Ye 205adc3f65aSKai Ye if (unlikely(req->err_type || status->done != SEC_SQE_DONE)) { 206adc3f65aSKai Ye dev_err_ratelimited(ctx->dev, "err_type[%d], done[%u]\n", 207adc3f65aSKai Ye req->err_type, status->done); 208adc3f65aSKai Ye return -EIO; 209adc3f65aSKai Ye } 210adc3f65aSKai Ye 211adc3f65aSKai Ye if (unlikely(ctx->alg_type == SEC_SKCIPHER)) { 212adc3f65aSKai Ye if (unlikely(status->flag != SEC_SQE_CFLAG)) { 213adc3f65aSKai Ye dev_err_ratelimited(ctx->dev, "flag[%u]\n", 214adc3f65aSKai Ye status->flag); 215adc3f65aSKai Ye return -EIO; 216adc3f65aSKai Ye } 217668f1ab7SKai Ye } else if (unlikely(ctx->alg_type == SEC_AEAD)) { 218668f1ab7SKai Ye if (unlikely(status->flag != SEC_SQE_AEAD_FLAG || 219668f1ab7SKai Ye status->icv == SEC_ICV_ERR)) { 220668f1ab7SKai Ye dev_err_ratelimited(ctx->dev, 221668f1ab7SKai Ye "flag[%u], icv[%u]\n", 222668f1ab7SKai Ye status->flag, status->icv); 223668f1ab7SKai Ye return -EBADMSG; 224668f1ab7SKai Ye } 225adc3f65aSKai Ye } 226adc3f65aSKai Ye 227adc3f65aSKai Ye return 0; 228adc3f65aSKai Ye } 229adc3f65aSKai Ye 230416d8220SZaibo Xu static void sec_req_cb(struct hisi_qp *qp, void *resp) 231416d8220SZaibo Xu { 232416d8220SZaibo Xu struct sec_qp_ctx *qp_ctx = qp->qp_ctx; 2338213a1a6SKai Ye struct sec_dfx *dfx = &qp_ctx->ctx->sec->debug.dfx; 234adc3f65aSKai Ye u8 type_supported = qp_ctx->ctx->type_supported; 235adc3f65aSKai Ye struct bd_status status; 236d6de2a59SZaibo Xu struct sec_ctx *ctx; 237d6de2a59SZaibo Xu struct sec_req *req; 238adc3f65aSKai Ye int err; 239416d8220SZaibo Xu u8 type; 240416d8220SZaibo Xu 241adc3f65aSKai Ye if (type_supported == SEC_BD_TYPE2) { 242adc3f65aSKai Ye type = pre_parse_finished_bd(&status, resp); 243adc3f65aSKai Ye req = qp_ctx->req_list[status.tag]; 244adc3f65aSKai Ye } else { 245adc3f65aSKai Ye type = pre_parse_finished_bd3(&status, resp); 246adc3f65aSKai Ye req = (void *)(uintptr_t)status.tag; 247adc3f65aSKai Ye } 248adc3f65aSKai Ye 249adc3f65aSKai Ye if (unlikely(type != type_supported)) { 2508213a1a6SKai Ye atomic64_inc(&dfx->err_bd_cnt); 25149838259SKai Ye pr_err("err bd type [%u]\n", type); 252416d8220SZaibo Xu return; 253416d8220SZaibo Xu } 254416d8220SZaibo Xu 2558213a1a6SKai Ye if (unlikely(!req)) { 2568213a1a6SKai Ye atomic64_inc(&dfx->invalid_req_cnt); 2579597efc3SKai Ye atomic_inc(&qp->qp_status.used); 2588213a1a6SKai Ye return; 2598213a1a6SKai Ye } 260adc3f65aSKai Ye 261adc3f65aSKai Ye req->err_type = status.err_type; 262d6de2a59SZaibo Xu ctx = req->ctx; 263adc3f65aSKai Ye err = sec_cb_status_check(req, &status); 264adc3f65aSKai Ye if (err) 2658213a1a6SKai Ye atomic64_inc(&dfx->done_flag_cnt); 2661e9bc276SZaibo Xu 2678213a1a6SKai Ye atomic64_inc(&dfx->recv_cnt); 268416d8220SZaibo Xu 269d6de2a59SZaibo Xu ctx->req_op->buf_unmap(ctx, req); 270d6de2a59SZaibo Xu 271310ea0acSZaibo Xu ctx->req_op->callback(ctx, req, err); 272416d8220SZaibo Xu } 273416d8220SZaibo Xu 274416d8220SZaibo Xu static int sec_bd_send(struct sec_ctx *ctx, struct sec_req *req) 275416d8220SZaibo Xu { 276416d8220SZaibo Xu struct sec_qp_ctx *qp_ctx = req->qp_ctx; 277416d8220SZaibo Xu int ret; 278416d8220SZaibo Xu 2799597efc3SKai Ye if (ctx->fake_req_limit <= 2809597efc3SKai Ye atomic_read(&qp_ctx->qp->qp_status.used) && 2819597efc3SKai Ye !(req->flag & CRYPTO_TFM_REQ_MAY_BACKLOG)) 2829597efc3SKai Ye return -EBUSY; 2839597efc3SKai Ye 28402884a4fSZhengchao Shao spin_lock_bh(&qp_ctx->req_lock); 285416d8220SZaibo Xu ret = hisi_qp_send(qp_ctx->qp, &req->sec_sqe); 2869597efc3SKai Ye if (ctx->fake_req_limit <= 2879597efc3SKai Ye atomic_read(&qp_ctx->qp->qp_status.used) && !ret) { 2889597efc3SKai Ye list_add_tail(&req->backlog_head, &qp_ctx->backlog); 289cb1eeb75SArnd Bergmann atomic64_inc(&ctx->sec->debug.dfx.send_cnt); 2909597efc3SKai Ye atomic64_inc(&ctx->sec->debug.dfx.send_busy_cnt); 29102884a4fSZhengchao Shao spin_unlock_bh(&qp_ctx->req_lock); 2929597efc3SKai Ye return -EBUSY; 2939597efc3SKai Ye } 29402884a4fSZhengchao Shao spin_unlock_bh(&qp_ctx->req_lock); 295416d8220SZaibo Xu 296b9c8d897SZaibo Xu if (unlikely(ret == -EBUSY)) 297416d8220SZaibo Xu return -ENOBUFS; 298416d8220SZaibo Xu 2999597efc3SKai Ye if (likely(!ret)) { 300416d8220SZaibo Xu ret = -EINPROGRESS; 3019597efc3SKai Ye atomic64_inc(&ctx->sec->debug.dfx.send_cnt); 3028213a1a6SKai Ye } 303416d8220SZaibo Xu 304416d8220SZaibo Xu return ret; 305416d8220SZaibo Xu } 306416d8220SZaibo Xu 3077c7d902aSZaibo Xu /* Get DMA memory resources */ 3087c7d902aSZaibo Xu static int sec_alloc_civ_resource(struct device *dev, struct sec_alg_res *res) 3097c7d902aSZaibo Xu { 310129a9f34SWeili Qian u16 q_depth = res->depth; 3117c7d902aSZaibo Xu int i; 3127c7d902aSZaibo Xu 313129a9f34SWeili Qian res->c_ivin = dma_alloc_coherent(dev, SEC_TOTAL_IV_SZ(q_depth), 3147c7d902aSZaibo Xu &res->c_ivin_dma, GFP_KERNEL); 3157c7d902aSZaibo Xu if (!res->c_ivin) 3167c7d902aSZaibo Xu return -ENOMEM; 3177c7d902aSZaibo Xu 318129a9f34SWeili Qian for (i = 1; i < q_depth; i++) { 3197c7d902aSZaibo Xu res[i].c_ivin_dma = res->c_ivin_dma + i * SEC_IV_SIZE; 3207c7d902aSZaibo Xu res[i].c_ivin = res->c_ivin + i * SEC_IV_SIZE; 3217c7d902aSZaibo Xu } 3227c7d902aSZaibo Xu 3237c7d902aSZaibo Xu return 0; 3247c7d902aSZaibo Xu } 3257c7d902aSZaibo Xu 3267c7d902aSZaibo Xu static void sec_free_civ_resource(struct device *dev, struct sec_alg_res *res) 3277c7d902aSZaibo Xu { 3287c7d902aSZaibo Xu if (res->c_ivin) 329129a9f34SWeili Qian dma_free_coherent(dev, SEC_TOTAL_IV_SZ(res->depth), 3307c7d902aSZaibo Xu res->c_ivin, res->c_ivin_dma); 3317c7d902aSZaibo Xu } 3327c7d902aSZaibo Xu 333c16a70c1SKai Ye static int sec_alloc_aiv_resource(struct device *dev, struct sec_alg_res *res) 334c16a70c1SKai Ye { 335129a9f34SWeili Qian u16 q_depth = res->depth; 336c16a70c1SKai Ye int i; 337c16a70c1SKai Ye 338129a9f34SWeili Qian res->a_ivin = dma_alloc_coherent(dev, SEC_TOTAL_IV_SZ(q_depth), 339c16a70c1SKai Ye &res->a_ivin_dma, GFP_KERNEL); 340c16a70c1SKai Ye if (!res->a_ivin) 341c16a70c1SKai Ye return -ENOMEM; 342c16a70c1SKai Ye 343129a9f34SWeili Qian for (i = 1; i < q_depth; i++) { 344c16a70c1SKai Ye res[i].a_ivin_dma = res->a_ivin_dma + i * SEC_IV_SIZE; 345c16a70c1SKai Ye res[i].a_ivin = res->a_ivin + i * SEC_IV_SIZE; 346c16a70c1SKai Ye } 347c16a70c1SKai Ye 348c16a70c1SKai Ye return 0; 349c16a70c1SKai Ye } 350c16a70c1SKai Ye 351c16a70c1SKai Ye static void sec_free_aiv_resource(struct device *dev, struct sec_alg_res *res) 352c16a70c1SKai Ye { 353c16a70c1SKai Ye if (res->a_ivin) 354129a9f34SWeili Qian dma_free_coherent(dev, SEC_TOTAL_IV_SZ(res->depth), 355c16a70c1SKai Ye res->a_ivin, res->a_ivin_dma); 356c16a70c1SKai Ye } 357c16a70c1SKai Ye 3582f072d75SZaibo Xu static int sec_alloc_mac_resource(struct device *dev, struct sec_alg_res *res) 3592f072d75SZaibo Xu { 360129a9f34SWeili Qian u16 q_depth = res->depth; 3612f072d75SZaibo Xu int i; 3622f072d75SZaibo Xu 363129a9f34SWeili Qian res->out_mac = dma_alloc_coherent(dev, SEC_TOTAL_MAC_SZ(q_depth) << 1, 3642f072d75SZaibo Xu &res->out_mac_dma, GFP_KERNEL); 3652f072d75SZaibo Xu if (!res->out_mac) 3662f072d75SZaibo Xu return -ENOMEM; 3672f072d75SZaibo Xu 368129a9f34SWeili Qian for (i = 1; i < q_depth; i++) { 3692f072d75SZaibo Xu res[i].out_mac_dma = res->out_mac_dma + 3702f072d75SZaibo Xu i * (SEC_MAX_MAC_LEN << 1); 3712f072d75SZaibo Xu res[i].out_mac = res->out_mac + i * (SEC_MAX_MAC_LEN << 1); 3722f072d75SZaibo Xu } 3732f072d75SZaibo Xu 3742f072d75SZaibo Xu return 0; 3752f072d75SZaibo Xu } 3762f072d75SZaibo Xu 3772f072d75SZaibo Xu static void sec_free_mac_resource(struct device *dev, struct sec_alg_res *res) 3782f072d75SZaibo Xu { 3792f072d75SZaibo Xu if (res->out_mac) 380129a9f34SWeili Qian dma_free_coherent(dev, SEC_TOTAL_MAC_SZ(res->depth) << 1, 3812f072d75SZaibo Xu res->out_mac, res->out_mac_dma); 3822f072d75SZaibo Xu } 3832f072d75SZaibo Xu 38474b58db8SLongfang Liu static void sec_free_pbuf_resource(struct device *dev, struct sec_alg_res *res) 38574b58db8SLongfang Liu { 38674b58db8SLongfang Liu if (res->pbuf) 387129a9f34SWeili Qian dma_free_coherent(dev, SEC_TOTAL_PBUF_SZ(res->depth), 38874b58db8SLongfang Liu res->pbuf, res->pbuf_dma); 38974b58db8SLongfang Liu } 39074b58db8SLongfang Liu 39174b58db8SLongfang Liu /* 39274b58db8SLongfang Liu * To improve performance, pbuffer is used for 39374b58db8SLongfang Liu * small packets (< 512Bytes) as IOMMU translation using. 39474b58db8SLongfang Liu */ 39574b58db8SLongfang Liu static int sec_alloc_pbuf_resource(struct device *dev, struct sec_alg_res *res) 39674b58db8SLongfang Liu { 397129a9f34SWeili Qian u16 q_depth = res->depth; 398129a9f34SWeili Qian int size = SEC_PBUF_PAGE_NUM(q_depth); 39974b58db8SLongfang Liu int pbuf_page_offset; 40074b58db8SLongfang Liu int i, j, k; 40174b58db8SLongfang Liu 402129a9f34SWeili Qian res->pbuf = dma_alloc_coherent(dev, SEC_TOTAL_PBUF_SZ(q_depth), 40374b58db8SLongfang Liu &res->pbuf_dma, GFP_KERNEL); 40474b58db8SLongfang Liu if (!res->pbuf) 40574b58db8SLongfang Liu return -ENOMEM; 40674b58db8SLongfang Liu 40774b58db8SLongfang Liu /* 40874b58db8SLongfang Liu * SEC_PBUF_PKG contains data pbuf, iv and 40974b58db8SLongfang Liu * out_mac : <SEC_PBUF|SEC_IV|SEC_MAC> 41074b58db8SLongfang Liu * Every PAGE contains six SEC_PBUF_PKG 41174b58db8SLongfang Liu * The sec_qp_ctx contains QM_Q_DEPTH numbers of SEC_PBUF_PKG 41274b58db8SLongfang Liu * So we need SEC_PBUF_PAGE_NUM numbers of PAGE 41374b58db8SLongfang Liu * for the SEC_TOTAL_PBUF_SZ 41474b58db8SLongfang Liu */ 415129a9f34SWeili Qian for (i = 0; i <= size; i++) { 41674b58db8SLongfang Liu pbuf_page_offset = PAGE_SIZE * i; 41774b58db8SLongfang Liu for (j = 0; j < SEC_PBUF_NUM; j++) { 41874b58db8SLongfang Liu k = i * SEC_PBUF_NUM + j; 419129a9f34SWeili Qian if (k == q_depth) 42074b58db8SLongfang Liu break; 42174b58db8SLongfang Liu res[k].pbuf = res->pbuf + 42274b58db8SLongfang Liu j * SEC_PBUF_PKG + pbuf_page_offset; 42374b58db8SLongfang Liu res[k].pbuf_dma = res->pbuf_dma + 42474b58db8SLongfang Liu j * SEC_PBUF_PKG + pbuf_page_offset; 42574b58db8SLongfang Liu } 42674b58db8SLongfang Liu } 427633e507fSLongfang Liu 42874b58db8SLongfang Liu return 0; 42974b58db8SLongfang Liu } 43074b58db8SLongfang Liu 4317c7d902aSZaibo Xu static int sec_alg_resource_alloc(struct sec_ctx *ctx, 4327c7d902aSZaibo Xu struct sec_qp_ctx *qp_ctx) 4337c7d902aSZaibo Xu { 4342f072d75SZaibo Xu struct sec_alg_res *res = qp_ctx->res; 435a44dce50SLongfang Liu struct device *dev = ctx->dev; 4362f072d75SZaibo Xu int ret; 4377c7d902aSZaibo Xu 4382f072d75SZaibo Xu ret = sec_alloc_civ_resource(dev, res); 4392f072d75SZaibo Xu if (ret) 4402f072d75SZaibo Xu return ret; 4412f072d75SZaibo Xu 4422f072d75SZaibo Xu if (ctx->alg_type == SEC_AEAD) { 443c16a70c1SKai Ye ret = sec_alloc_aiv_resource(dev, res); 444c16a70c1SKai Ye if (ret) 445c16a70c1SKai Ye goto alloc_aiv_fail; 446c16a70c1SKai Ye 4472f072d75SZaibo Xu ret = sec_alloc_mac_resource(dev, res); 4482f072d75SZaibo Xu if (ret) 449c16a70c1SKai Ye goto alloc_mac_fail; 4502f072d75SZaibo Xu } 45174b58db8SLongfang Liu if (ctx->pbuf_supported) { 45274b58db8SLongfang Liu ret = sec_alloc_pbuf_resource(dev, res); 45374b58db8SLongfang Liu if (ret) { 45474b58db8SLongfang Liu dev_err(dev, "fail to alloc pbuf dma resource!\n"); 45524efcec2SLongfang Liu goto alloc_pbuf_fail; 45674b58db8SLongfang Liu } 45774b58db8SLongfang Liu } 4582f072d75SZaibo Xu 4592f072d75SZaibo Xu return 0; 460633e507fSLongfang Liu 46124efcec2SLongfang Liu alloc_pbuf_fail: 46224efcec2SLongfang Liu if (ctx->alg_type == SEC_AEAD) 46324efcec2SLongfang Liu sec_free_mac_resource(dev, qp_ctx->res); 464c16a70c1SKai Ye alloc_mac_fail: 465c16a70c1SKai Ye if (ctx->alg_type == SEC_AEAD) 466c16a70c1SKai Ye sec_free_aiv_resource(dev, res); 467c16a70c1SKai Ye alloc_aiv_fail: 4682f072d75SZaibo Xu sec_free_civ_resource(dev, res); 4692f072d75SZaibo Xu return ret; 4707c7d902aSZaibo Xu } 4717c7d902aSZaibo Xu 4727c7d902aSZaibo Xu static void sec_alg_resource_free(struct sec_ctx *ctx, 4737c7d902aSZaibo Xu struct sec_qp_ctx *qp_ctx) 4747c7d902aSZaibo Xu { 475a44dce50SLongfang Liu struct device *dev = ctx->dev; 4767c7d902aSZaibo Xu 4777c7d902aSZaibo Xu sec_free_civ_resource(dev, qp_ctx->res); 4782f072d75SZaibo Xu 47974b58db8SLongfang Liu if (ctx->pbuf_supported) 48074b58db8SLongfang Liu sec_free_pbuf_resource(dev, qp_ctx->res); 4812f072d75SZaibo Xu if (ctx->alg_type == SEC_AEAD) 4822f072d75SZaibo Xu sec_free_mac_resource(dev, qp_ctx->res); 4837c7d902aSZaibo Xu } 4847c7d902aSZaibo Xu 485129a9f34SWeili Qian static int sec_alloc_qp_ctx_resource(struct hisi_qm *qm, struct sec_ctx *ctx, 486129a9f34SWeili Qian struct sec_qp_ctx *qp_ctx) 487129a9f34SWeili Qian { 488129a9f34SWeili Qian u16 q_depth = qp_ctx->qp->sq_depth; 489129a9f34SWeili Qian struct device *dev = ctx->dev; 490129a9f34SWeili Qian int ret = -ENOMEM; 491129a9f34SWeili Qian 492129a9f34SWeili Qian qp_ctx->req_list = kcalloc(q_depth, sizeof(struct sec_req *), GFP_KERNEL); 493129a9f34SWeili Qian if (!qp_ctx->req_list) 494129a9f34SWeili Qian return ret; 495129a9f34SWeili Qian 496129a9f34SWeili Qian qp_ctx->res = kcalloc(q_depth, sizeof(struct sec_alg_res), GFP_KERNEL); 497129a9f34SWeili Qian if (!qp_ctx->res) 498129a9f34SWeili Qian goto err_free_req_list; 499129a9f34SWeili Qian qp_ctx->res->depth = q_depth; 500129a9f34SWeili Qian 501129a9f34SWeili Qian qp_ctx->c_in_pool = hisi_acc_create_sgl_pool(dev, q_depth, SEC_SGL_SGE_NR); 502129a9f34SWeili Qian if (IS_ERR(qp_ctx->c_in_pool)) { 503129a9f34SWeili Qian dev_err(dev, "fail to create sgl pool for input!\n"); 504129a9f34SWeili Qian goto err_free_res; 505129a9f34SWeili Qian } 506129a9f34SWeili Qian 507129a9f34SWeili Qian qp_ctx->c_out_pool = hisi_acc_create_sgl_pool(dev, q_depth, SEC_SGL_SGE_NR); 508129a9f34SWeili Qian if (IS_ERR(qp_ctx->c_out_pool)) { 509129a9f34SWeili Qian dev_err(dev, "fail to create sgl pool for output!\n"); 510129a9f34SWeili Qian goto err_free_c_in_pool; 511129a9f34SWeili Qian } 512129a9f34SWeili Qian 513129a9f34SWeili Qian ret = sec_alg_resource_alloc(ctx, qp_ctx); 514129a9f34SWeili Qian if (ret) 515129a9f34SWeili Qian goto err_free_c_out_pool; 516129a9f34SWeili Qian 517129a9f34SWeili Qian return 0; 518129a9f34SWeili Qian 519129a9f34SWeili Qian err_free_c_out_pool: 520129a9f34SWeili Qian hisi_acc_free_sgl_pool(dev, qp_ctx->c_out_pool); 521129a9f34SWeili Qian err_free_c_in_pool: 522129a9f34SWeili Qian hisi_acc_free_sgl_pool(dev, qp_ctx->c_in_pool); 523129a9f34SWeili Qian err_free_res: 524129a9f34SWeili Qian kfree(qp_ctx->res); 525129a9f34SWeili Qian err_free_req_list: 526129a9f34SWeili Qian kfree(qp_ctx->req_list); 527129a9f34SWeili Qian return ret; 528129a9f34SWeili Qian } 529129a9f34SWeili Qian 530129a9f34SWeili Qian static void sec_free_qp_ctx_resource(struct sec_ctx *ctx, struct sec_qp_ctx *qp_ctx) 531129a9f34SWeili Qian { 532129a9f34SWeili Qian struct device *dev = ctx->dev; 533129a9f34SWeili Qian 534129a9f34SWeili Qian sec_alg_resource_free(ctx, qp_ctx); 535129a9f34SWeili Qian hisi_acc_free_sgl_pool(dev, qp_ctx->c_out_pool); 536129a9f34SWeili Qian hisi_acc_free_sgl_pool(dev, qp_ctx->c_in_pool); 537129a9f34SWeili Qian kfree(qp_ctx->res); 538129a9f34SWeili Qian kfree(qp_ctx->req_list); 539129a9f34SWeili Qian } 540129a9f34SWeili Qian 541416d8220SZaibo Xu static int sec_create_qp_ctx(struct hisi_qm *qm, struct sec_ctx *ctx, 542416d8220SZaibo Xu int qp_ctx_id, int alg_type) 543416d8220SZaibo Xu { 544416d8220SZaibo Xu struct sec_qp_ctx *qp_ctx; 545416d8220SZaibo Xu struct hisi_qp *qp; 546129a9f34SWeili Qian int ret; 547416d8220SZaibo Xu 548416d8220SZaibo Xu qp_ctx = &ctx->qp_ctx[qp_ctx_id]; 5490b5e43bcSKai Ye qp = ctx->qps[qp_ctx_id]; 550416d8220SZaibo Xu qp->req_type = 0; 551416d8220SZaibo Xu qp->qp_ctx = qp_ctx; 552416d8220SZaibo Xu qp_ctx->qp = qp; 553416d8220SZaibo Xu qp_ctx->ctx = ctx; 554416d8220SZaibo Xu 555adc3f65aSKai Ye qp->req_cb = sec_req_cb; 556adc3f65aSKai Ye 55702884a4fSZhengchao Shao spin_lock_init(&qp_ctx->req_lock); 558416d8220SZaibo Xu idr_init(&qp_ctx->req_idr); 5599597efc3SKai Ye INIT_LIST_HEAD(&qp_ctx->backlog); 560416d8220SZaibo Xu 561129a9f34SWeili Qian ret = sec_alloc_qp_ctx_resource(qm, ctx, qp_ctx); 562416d8220SZaibo Xu if (ret) 563129a9f34SWeili Qian goto err_destroy_idr; 564416d8220SZaibo Xu 565416d8220SZaibo Xu ret = hisi_qm_start_qp(qp, 0); 566416d8220SZaibo Xu if (ret < 0) 567129a9f34SWeili Qian goto err_resource_free; 568416d8220SZaibo Xu 569416d8220SZaibo Xu return 0; 570416d8220SZaibo Xu 571129a9f34SWeili Qian err_resource_free: 572129a9f34SWeili Qian sec_free_qp_ctx_resource(ctx, qp_ctx); 573416d8220SZaibo Xu err_destroy_idr: 574416d8220SZaibo Xu idr_destroy(&qp_ctx->req_idr); 575416d8220SZaibo Xu return ret; 576416d8220SZaibo Xu } 577416d8220SZaibo Xu 578416d8220SZaibo Xu static void sec_release_qp_ctx(struct sec_ctx *ctx, 579416d8220SZaibo Xu struct sec_qp_ctx *qp_ctx) 580416d8220SZaibo Xu { 581416d8220SZaibo Xu hisi_qm_stop_qp(qp_ctx->qp); 582129a9f34SWeili Qian sec_free_qp_ctx_resource(ctx, qp_ctx); 583416d8220SZaibo Xu idr_destroy(&qp_ctx->req_idr); 584416d8220SZaibo Xu } 585416d8220SZaibo Xu 586473a0f96SZaibo Xu static int sec_ctx_base_init(struct sec_ctx *ctx) 587416d8220SZaibo Xu { 588416d8220SZaibo Xu struct sec_dev *sec; 589416d8220SZaibo Xu int i, ret; 590416d8220SZaibo Xu 5910b5e43bcSKai Ye ctx->qps = sec_create_qps(); 5920b5e43bcSKai Ye if (!ctx->qps) { 5930b5e43bcSKai Ye pr_err("Can not create sec qps!\n"); 594416d8220SZaibo Xu return -ENODEV; 595416d8220SZaibo Xu } 5960b5e43bcSKai Ye 5970b5e43bcSKai Ye sec = container_of(ctx->qps[0]->qm, struct sec_dev, qm); 598416d8220SZaibo Xu ctx->sec = sec; 599a44dce50SLongfang Liu ctx->dev = &sec->qm.pdev->dev; 600a718cfceSZaibo Xu ctx->hlf_q_num = sec->ctx_q_num >> 1; 601416d8220SZaibo Xu 60274b58db8SLongfang Liu ctx->pbuf_supported = ctx->sec->iommu_used; 60374b58db8SLongfang Liu 604416d8220SZaibo Xu /* Half of queue depth is taken as fake requests limit in the queue. */ 605129a9f34SWeili Qian ctx->fake_req_limit = ctx->qps[0]->sq_depth >> 1; 606416d8220SZaibo Xu ctx->qp_ctx = kcalloc(sec->ctx_q_num, sizeof(struct sec_qp_ctx), 607416d8220SZaibo Xu GFP_KERNEL); 60824efcec2SLongfang Liu if (!ctx->qp_ctx) { 60924efcec2SLongfang Liu ret = -ENOMEM; 61024efcec2SLongfang Liu goto err_destroy_qps; 61124efcec2SLongfang Liu } 612416d8220SZaibo Xu 613416d8220SZaibo Xu for (i = 0; i < sec->ctx_q_num; i++) { 614473a0f96SZaibo Xu ret = sec_create_qp_ctx(&sec->qm, ctx, i, 0); 615416d8220SZaibo Xu if (ret) 616416d8220SZaibo Xu goto err_sec_release_qp_ctx; 617416d8220SZaibo Xu } 618416d8220SZaibo Xu 619416d8220SZaibo Xu return 0; 62024efcec2SLongfang Liu 621416d8220SZaibo Xu err_sec_release_qp_ctx: 622416d8220SZaibo Xu for (i = i - 1; i >= 0; i--) 623416d8220SZaibo Xu sec_release_qp_ctx(ctx, &ctx->qp_ctx[i]); 624416d8220SZaibo Xu kfree(ctx->qp_ctx); 62524efcec2SLongfang Liu err_destroy_qps: 62624efcec2SLongfang Liu sec_destroy_qps(ctx->qps, sec->ctx_q_num); 627416d8220SZaibo Xu return ret; 628416d8220SZaibo Xu } 629416d8220SZaibo Xu 630473a0f96SZaibo Xu static void sec_ctx_base_uninit(struct sec_ctx *ctx) 631416d8220SZaibo Xu { 632473a0f96SZaibo Xu int i; 633416d8220SZaibo Xu 634416d8220SZaibo Xu for (i = 0; i < ctx->sec->ctx_q_num; i++) 635416d8220SZaibo Xu sec_release_qp_ctx(ctx, &ctx->qp_ctx[i]); 636416d8220SZaibo Xu 6370b5e43bcSKai Ye sec_destroy_qps(ctx->qps, ctx->sec->ctx_q_num); 638416d8220SZaibo Xu kfree(ctx->qp_ctx); 639416d8220SZaibo Xu } 640416d8220SZaibo Xu 641473a0f96SZaibo Xu static int sec_cipher_init(struct sec_ctx *ctx) 642473a0f96SZaibo Xu { 643473a0f96SZaibo Xu struct sec_cipher_ctx *c_ctx = &ctx->c_ctx; 644473a0f96SZaibo Xu 645a44dce50SLongfang Liu c_ctx->c_key = dma_alloc_coherent(ctx->dev, SEC_MAX_KEY_SIZE, 646473a0f96SZaibo Xu &c_ctx->c_key_dma, GFP_KERNEL); 647473a0f96SZaibo Xu if (!c_ctx->c_key) 648473a0f96SZaibo Xu return -ENOMEM; 649473a0f96SZaibo Xu 650473a0f96SZaibo Xu return 0; 651473a0f96SZaibo Xu } 652473a0f96SZaibo Xu 653473a0f96SZaibo Xu static void sec_cipher_uninit(struct sec_ctx *ctx) 654473a0f96SZaibo Xu { 655473a0f96SZaibo Xu struct sec_cipher_ctx *c_ctx = &ctx->c_ctx; 656473a0f96SZaibo Xu 657473a0f96SZaibo Xu memzero_explicit(c_ctx->c_key, SEC_MAX_KEY_SIZE); 658a44dce50SLongfang Liu dma_free_coherent(ctx->dev, SEC_MAX_KEY_SIZE, 659473a0f96SZaibo Xu c_ctx->c_key, c_ctx->c_key_dma); 660473a0f96SZaibo Xu } 661473a0f96SZaibo Xu 6622f072d75SZaibo Xu static int sec_auth_init(struct sec_ctx *ctx) 6632f072d75SZaibo Xu { 6642f072d75SZaibo Xu struct sec_auth_ctx *a_ctx = &ctx->a_ctx; 6652f072d75SZaibo Xu 66645f5d017SKai Ye a_ctx->a_key = dma_alloc_coherent(ctx->dev, SEC_MAX_AKEY_SIZE, 6672f072d75SZaibo Xu &a_ctx->a_key_dma, GFP_KERNEL); 6682f072d75SZaibo Xu if (!a_ctx->a_key) 6692f072d75SZaibo Xu return -ENOMEM; 6702f072d75SZaibo Xu 6712f072d75SZaibo Xu return 0; 6722f072d75SZaibo Xu } 6732f072d75SZaibo Xu 6742f072d75SZaibo Xu static void sec_auth_uninit(struct sec_ctx *ctx) 6752f072d75SZaibo Xu { 6762f072d75SZaibo Xu struct sec_auth_ctx *a_ctx = &ctx->a_ctx; 6772f072d75SZaibo Xu 67845f5d017SKai Ye memzero_explicit(a_ctx->a_key, SEC_MAX_AKEY_SIZE); 67945f5d017SKai Ye dma_free_coherent(ctx->dev, SEC_MAX_AKEY_SIZE, 6802f072d75SZaibo Xu a_ctx->a_key, a_ctx->a_key_dma); 6812f072d75SZaibo Xu } 6822f072d75SZaibo Xu 6835652d55aSKai Ye static int sec_skcipher_fbtfm_init(struct crypto_skcipher *tfm) 6845652d55aSKai Ye { 6855652d55aSKai Ye const char *alg = crypto_tfm_alg_name(&tfm->base); 6865652d55aSKai Ye struct sec_ctx *ctx = crypto_skcipher_ctx(tfm); 6875652d55aSKai Ye struct sec_cipher_ctx *c_ctx = &ctx->c_ctx; 6885652d55aSKai Ye 6895652d55aSKai Ye c_ctx->fallback = false; 690e764d81dSKai Ye 691e764d81dSKai Ye /* Currently, only XTS mode need fallback tfm when using 192bit key */ 6925652d55aSKai Ye if (likely(strncmp(alg, "xts", SEC_XTS_NAME_SZ))) 6935652d55aSKai Ye return 0; 6945652d55aSKai Ye 6955652d55aSKai Ye c_ctx->fbtfm = crypto_alloc_sync_skcipher(alg, 0, 6965652d55aSKai Ye CRYPTO_ALG_NEED_FALLBACK); 6975652d55aSKai Ye if (IS_ERR(c_ctx->fbtfm)) { 698e764d81dSKai Ye pr_err("failed to alloc xts mode fallback tfm!\n"); 6995652d55aSKai Ye return PTR_ERR(c_ctx->fbtfm); 7005652d55aSKai Ye } 7015652d55aSKai Ye 7025652d55aSKai Ye return 0; 7035652d55aSKai Ye } 7045652d55aSKai Ye 705473a0f96SZaibo Xu static int sec_skcipher_init(struct crypto_skcipher *tfm) 706473a0f96SZaibo Xu { 707473a0f96SZaibo Xu struct sec_ctx *ctx = crypto_skcipher_ctx(tfm); 708473a0f96SZaibo Xu int ret; 709473a0f96SZaibo Xu 7102f072d75SZaibo Xu ctx->alg_type = SEC_SKCIPHER; 711473a0f96SZaibo Xu crypto_skcipher_set_reqsize(tfm, sizeof(struct sec_req)); 712473a0f96SZaibo Xu ctx->c_ctx.ivsize = crypto_skcipher_ivsize(tfm); 713473a0f96SZaibo Xu if (ctx->c_ctx.ivsize > SEC_IV_SIZE) { 7144b7aef02SLongfang Liu pr_err("get error skcipher iv size!\n"); 715473a0f96SZaibo Xu return -EINVAL; 716473a0f96SZaibo Xu } 717473a0f96SZaibo Xu 718473a0f96SZaibo Xu ret = sec_ctx_base_init(ctx); 719473a0f96SZaibo Xu if (ret) 720473a0f96SZaibo Xu return ret; 721473a0f96SZaibo Xu 722473a0f96SZaibo Xu ret = sec_cipher_init(ctx); 723473a0f96SZaibo Xu if (ret) 724473a0f96SZaibo Xu goto err_cipher_init; 725473a0f96SZaibo Xu 7265652d55aSKai Ye ret = sec_skcipher_fbtfm_init(tfm); 7275652d55aSKai Ye if (ret) 7285652d55aSKai Ye goto err_fbtfm_init; 7295652d55aSKai Ye 730473a0f96SZaibo Xu return 0; 731633e507fSLongfang Liu 7325652d55aSKai Ye err_fbtfm_init: 7335652d55aSKai Ye sec_cipher_uninit(ctx); 734473a0f96SZaibo Xu err_cipher_init: 735473a0f96SZaibo Xu sec_ctx_base_uninit(ctx); 736473a0f96SZaibo Xu return ret; 737473a0f96SZaibo Xu } 738473a0f96SZaibo Xu 739473a0f96SZaibo Xu static void sec_skcipher_uninit(struct crypto_skcipher *tfm) 740473a0f96SZaibo Xu { 741473a0f96SZaibo Xu struct sec_ctx *ctx = crypto_skcipher_ctx(tfm); 742473a0f96SZaibo Xu 7435652d55aSKai Ye if (ctx->c_ctx.fbtfm) 7445652d55aSKai Ye crypto_free_sync_skcipher(ctx->c_ctx.fbtfm); 7455652d55aSKai Ye 746473a0f96SZaibo Xu sec_cipher_uninit(ctx); 747473a0f96SZaibo Xu sec_ctx_base_uninit(ctx); 748473a0f96SZaibo Xu } 749473a0f96SZaibo Xu 750ae6ce7b1SKai Ye static int sec_skcipher_3des_setkey(struct crypto_skcipher *tfm, const u8 *key, 751416d8220SZaibo Xu const u32 keylen, 752416d8220SZaibo Xu const enum sec_cmode c_mode) 753416d8220SZaibo Xu { 754ae6ce7b1SKai Ye struct sec_ctx *ctx = crypto_skcipher_ctx(tfm); 755ae6ce7b1SKai Ye struct sec_cipher_ctx *c_ctx = &ctx->c_ctx; 756ae6ce7b1SKai Ye int ret; 757ae6ce7b1SKai Ye 758ae6ce7b1SKai Ye ret = verify_skcipher_des3_key(tfm, key); 759ae6ce7b1SKai Ye if (ret) 760ae6ce7b1SKai Ye return ret; 761ae6ce7b1SKai Ye 762416d8220SZaibo Xu switch (keylen) { 763416d8220SZaibo Xu case SEC_DES3_2KEY_SIZE: 764416d8220SZaibo Xu c_ctx->c_key_len = SEC_CKEY_3DES_2KEY; 765416d8220SZaibo Xu break; 766416d8220SZaibo Xu case SEC_DES3_3KEY_SIZE: 767416d8220SZaibo Xu c_ctx->c_key_len = SEC_CKEY_3DES_3KEY; 768416d8220SZaibo Xu break; 769416d8220SZaibo Xu default: 770416d8220SZaibo Xu return -EINVAL; 771416d8220SZaibo Xu } 772416d8220SZaibo Xu 773416d8220SZaibo Xu return 0; 774416d8220SZaibo Xu } 775416d8220SZaibo Xu 776416d8220SZaibo Xu static int sec_skcipher_aes_sm4_setkey(struct sec_cipher_ctx *c_ctx, 777416d8220SZaibo Xu const u32 keylen, 778416d8220SZaibo Xu const enum sec_cmode c_mode) 779416d8220SZaibo Xu { 780416d8220SZaibo Xu if (c_mode == SEC_CMODE_XTS) { 781416d8220SZaibo Xu switch (keylen) { 782416d8220SZaibo Xu case SEC_XTS_MIN_KEY_SIZE: 783416d8220SZaibo Xu c_ctx->c_key_len = SEC_CKEY_128BIT; 784416d8220SZaibo Xu break; 7855652d55aSKai Ye case SEC_XTS_MID_KEY_SIZE: 7865652d55aSKai Ye c_ctx->fallback = true; 7875652d55aSKai Ye break; 788416d8220SZaibo Xu case SEC_XTS_MAX_KEY_SIZE: 789416d8220SZaibo Xu c_ctx->c_key_len = SEC_CKEY_256BIT; 790416d8220SZaibo Xu break; 791416d8220SZaibo Xu default: 792416d8220SZaibo Xu pr_err("hisi_sec2: xts mode key error!\n"); 793416d8220SZaibo Xu return -EINVAL; 794416d8220SZaibo Xu } 795416d8220SZaibo Xu } else { 796adc3f65aSKai Ye if (c_ctx->c_alg == SEC_CALG_SM4 && 797adc3f65aSKai Ye keylen != AES_KEYSIZE_128) { 798adc3f65aSKai Ye pr_err("hisi_sec2: sm4 key error!\n"); 799adc3f65aSKai Ye return -EINVAL; 800adc3f65aSKai Ye } else { 801416d8220SZaibo Xu switch (keylen) { 802416d8220SZaibo Xu case AES_KEYSIZE_128: 803416d8220SZaibo Xu c_ctx->c_key_len = SEC_CKEY_128BIT; 804416d8220SZaibo Xu break; 805416d8220SZaibo Xu case AES_KEYSIZE_192: 806416d8220SZaibo Xu c_ctx->c_key_len = SEC_CKEY_192BIT; 807416d8220SZaibo Xu break; 808416d8220SZaibo Xu case AES_KEYSIZE_256: 809416d8220SZaibo Xu c_ctx->c_key_len = SEC_CKEY_256BIT; 810416d8220SZaibo Xu break; 811416d8220SZaibo Xu default: 812416d8220SZaibo Xu pr_err("hisi_sec2: aes key error!\n"); 813416d8220SZaibo Xu return -EINVAL; 814416d8220SZaibo Xu } 815416d8220SZaibo Xu } 816adc3f65aSKai Ye } 817416d8220SZaibo Xu 818416d8220SZaibo Xu return 0; 819416d8220SZaibo Xu } 820416d8220SZaibo Xu 821416d8220SZaibo Xu static int sec_skcipher_setkey(struct crypto_skcipher *tfm, const u8 *key, 822416d8220SZaibo Xu const u32 keylen, const enum sec_calg c_alg, 823416d8220SZaibo Xu const enum sec_cmode c_mode) 824416d8220SZaibo Xu { 825416d8220SZaibo Xu struct sec_ctx *ctx = crypto_skcipher_ctx(tfm); 826416d8220SZaibo Xu struct sec_cipher_ctx *c_ctx = &ctx->c_ctx; 827a44dce50SLongfang Liu struct device *dev = ctx->dev; 828416d8220SZaibo Xu int ret; 829416d8220SZaibo Xu 830416d8220SZaibo Xu if (c_mode == SEC_CMODE_XTS) { 831416d8220SZaibo Xu ret = xts_verify_key(tfm, key, keylen); 832416d8220SZaibo Xu if (ret) { 833a44dce50SLongfang Liu dev_err(dev, "xts mode key err!\n"); 834416d8220SZaibo Xu return ret; 835416d8220SZaibo Xu } 836416d8220SZaibo Xu } 837416d8220SZaibo Xu 838416d8220SZaibo Xu c_ctx->c_alg = c_alg; 839416d8220SZaibo Xu c_ctx->c_mode = c_mode; 840416d8220SZaibo Xu 841416d8220SZaibo Xu switch (c_alg) { 842416d8220SZaibo Xu case SEC_CALG_3DES: 843ae6ce7b1SKai Ye ret = sec_skcipher_3des_setkey(tfm, key, keylen, c_mode); 844416d8220SZaibo Xu break; 845416d8220SZaibo Xu case SEC_CALG_AES: 846416d8220SZaibo Xu case SEC_CALG_SM4: 847416d8220SZaibo Xu ret = sec_skcipher_aes_sm4_setkey(c_ctx, keylen, c_mode); 848416d8220SZaibo Xu break; 849416d8220SZaibo Xu default: 850416d8220SZaibo Xu return -EINVAL; 851416d8220SZaibo Xu } 852416d8220SZaibo Xu 853416d8220SZaibo Xu if (ret) { 854a44dce50SLongfang Liu dev_err(dev, "set sec key err!\n"); 855416d8220SZaibo Xu return ret; 856416d8220SZaibo Xu } 857416d8220SZaibo Xu 858416d8220SZaibo Xu memcpy(c_ctx->c_key, key, keylen); 859e764d81dSKai Ye if (c_ctx->fallback && c_ctx->fbtfm) { 8605652d55aSKai Ye ret = crypto_sync_skcipher_setkey(c_ctx->fbtfm, key, keylen); 8615652d55aSKai Ye if (ret) { 8625652d55aSKai Ye dev_err(dev, "failed to set fallback skcipher key!\n"); 8635652d55aSKai Ye return ret; 8645652d55aSKai Ye } 8655652d55aSKai Ye } 866416d8220SZaibo Xu return 0; 867416d8220SZaibo Xu } 868416d8220SZaibo Xu 869416d8220SZaibo Xu #define GEN_SEC_SETKEY_FUNC(name, c_alg, c_mode) \ 870416d8220SZaibo Xu static int sec_setkey_##name(struct crypto_skcipher *tfm, const u8 *key,\ 871416d8220SZaibo Xu u32 keylen) \ 872416d8220SZaibo Xu { \ 873416d8220SZaibo Xu return sec_skcipher_setkey(tfm, key, keylen, c_alg, c_mode); \ 874416d8220SZaibo Xu } 875416d8220SZaibo Xu 876416d8220SZaibo Xu GEN_SEC_SETKEY_FUNC(aes_ecb, SEC_CALG_AES, SEC_CMODE_ECB) 877416d8220SZaibo Xu GEN_SEC_SETKEY_FUNC(aes_cbc, SEC_CALG_AES, SEC_CMODE_CBC) 878416d8220SZaibo Xu GEN_SEC_SETKEY_FUNC(aes_xts, SEC_CALG_AES, SEC_CMODE_XTS) 8797b44c0eeSKai Ye GEN_SEC_SETKEY_FUNC(aes_ofb, SEC_CALG_AES, SEC_CMODE_OFB) 8807b44c0eeSKai Ye GEN_SEC_SETKEY_FUNC(aes_cfb, SEC_CALG_AES, SEC_CMODE_CFB) 8817b44c0eeSKai Ye GEN_SEC_SETKEY_FUNC(aes_ctr, SEC_CALG_AES, SEC_CMODE_CTR) 882416d8220SZaibo Xu GEN_SEC_SETKEY_FUNC(3des_ecb, SEC_CALG_3DES, SEC_CMODE_ECB) 883416d8220SZaibo Xu GEN_SEC_SETKEY_FUNC(3des_cbc, SEC_CALG_3DES, SEC_CMODE_CBC) 884416d8220SZaibo Xu GEN_SEC_SETKEY_FUNC(sm4_xts, SEC_CALG_SM4, SEC_CMODE_XTS) 885416d8220SZaibo Xu GEN_SEC_SETKEY_FUNC(sm4_cbc, SEC_CALG_SM4, SEC_CMODE_CBC) 8867b44c0eeSKai Ye GEN_SEC_SETKEY_FUNC(sm4_ofb, SEC_CALG_SM4, SEC_CMODE_OFB) 8877b44c0eeSKai Ye GEN_SEC_SETKEY_FUNC(sm4_cfb, SEC_CALG_SM4, SEC_CMODE_CFB) 8887b44c0eeSKai Ye GEN_SEC_SETKEY_FUNC(sm4_ctr, SEC_CALG_SM4, SEC_CMODE_CTR) 889416d8220SZaibo Xu 89074b58db8SLongfang Liu static int sec_cipher_pbuf_map(struct sec_ctx *ctx, struct sec_req *req, 89174b58db8SLongfang Liu struct scatterlist *src) 89274b58db8SLongfang Liu { 8936c46a329SKai Ye struct sec_aead_req *a_req = &req->aead_req; 8946c46a329SKai Ye struct aead_request *aead_req = a_req->aead_req; 89574b58db8SLongfang Liu struct sec_cipher_req *c_req = &req->c_req; 89674b58db8SLongfang Liu struct sec_qp_ctx *qp_ctx = req->qp_ctx; 897a44dce50SLongfang Liu struct device *dev = ctx->dev; 89874b58db8SLongfang Liu int copy_size, pbuf_length; 89974b58db8SLongfang Liu int req_id = req->req_id; 9006c46a329SKai Ye struct crypto_aead *tfm; 9016c46a329SKai Ye size_t authsize; 9026c46a329SKai Ye u8 *mac_offset; 90374b58db8SLongfang Liu 90474b58db8SLongfang Liu if (ctx->alg_type == SEC_AEAD) 90574b58db8SLongfang Liu copy_size = aead_req->cryptlen + aead_req->assoclen; 90674b58db8SLongfang Liu else 90774b58db8SLongfang Liu copy_size = c_req->c_len; 90874b58db8SLongfang Liu 90974b58db8SLongfang Liu pbuf_length = sg_copy_to_buffer(src, sg_nents(src), 9106c46a329SKai Ye qp_ctx->res[req_id].pbuf, copy_size); 91174b58db8SLongfang Liu if (unlikely(pbuf_length != copy_size)) { 91274b58db8SLongfang Liu dev_err(dev, "copy src data to pbuf error!\n"); 91374b58db8SLongfang Liu return -EINVAL; 91474b58db8SLongfang Liu } 9156c46a329SKai Ye if (!c_req->encrypt && ctx->alg_type == SEC_AEAD) { 9166c46a329SKai Ye tfm = crypto_aead_reqtfm(aead_req); 9176c46a329SKai Ye authsize = crypto_aead_authsize(tfm); 9186c46a329SKai Ye mac_offset = qp_ctx->res[req_id].pbuf + copy_size - authsize; 9196c46a329SKai Ye memcpy(a_req->out_mac, mac_offset, authsize); 9206c46a329SKai Ye } 92174b58db8SLongfang Liu 9229039878aSKai Ye req->in_dma = qp_ctx->res[req_id].pbuf_dma; 9239039878aSKai Ye c_req->c_out_dma = req->in_dma; 92474b58db8SLongfang Liu 92574b58db8SLongfang Liu return 0; 92674b58db8SLongfang Liu } 92774b58db8SLongfang Liu 92874b58db8SLongfang Liu static void sec_cipher_pbuf_unmap(struct sec_ctx *ctx, struct sec_req *req, 92974b58db8SLongfang Liu struct scatterlist *dst) 93074b58db8SLongfang Liu { 93174b58db8SLongfang Liu struct aead_request *aead_req = req->aead_req.aead_req; 93274b58db8SLongfang Liu struct sec_cipher_req *c_req = &req->c_req; 93374b58db8SLongfang Liu struct sec_qp_ctx *qp_ctx = req->qp_ctx; 93474b58db8SLongfang Liu int copy_size, pbuf_length; 93574b58db8SLongfang Liu int req_id = req->req_id; 93674b58db8SLongfang Liu 93774b58db8SLongfang Liu if (ctx->alg_type == SEC_AEAD) 93874b58db8SLongfang Liu copy_size = c_req->c_len + aead_req->assoclen; 93974b58db8SLongfang Liu else 94074b58db8SLongfang Liu copy_size = c_req->c_len; 94174b58db8SLongfang Liu 94274b58db8SLongfang Liu pbuf_length = sg_copy_from_buffer(dst, sg_nents(dst), 943668f1ab7SKai Ye qp_ctx->res[req_id].pbuf, copy_size); 94474b58db8SLongfang Liu if (unlikely(pbuf_length != copy_size)) 945668f1ab7SKai Ye dev_err(ctx->dev, "copy pbuf data to dst error!\n"); 946668f1ab7SKai Ye } 947668f1ab7SKai Ye 948668f1ab7SKai Ye static int sec_aead_mac_init(struct sec_aead_req *req) 949668f1ab7SKai Ye { 950668f1ab7SKai Ye struct aead_request *aead_req = req->aead_req; 951668f1ab7SKai Ye struct crypto_aead *tfm = crypto_aead_reqtfm(aead_req); 952668f1ab7SKai Ye size_t authsize = crypto_aead_authsize(tfm); 953668f1ab7SKai Ye u8 *mac_out = req->out_mac; 954668f1ab7SKai Ye struct scatterlist *sgl = aead_req->src; 955668f1ab7SKai Ye size_t copy_size; 956668f1ab7SKai Ye off_t skip_size; 957668f1ab7SKai Ye 958668f1ab7SKai Ye /* Copy input mac */ 959668f1ab7SKai Ye skip_size = aead_req->assoclen + aead_req->cryptlen - authsize; 960668f1ab7SKai Ye copy_size = sg_pcopy_to_buffer(sgl, sg_nents(sgl), mac_out, 961668f1ab7SKai Ye authsize, skip_size); 962668f1ab7SKai Ye if (unlikely(copy_size != authsize)) 963668f1ab7SKai Ye return -EINVAL; 964668f1ab7SKai Ye 965668f1ab7SKai Ye return 0; 96674b58db8SLongfang Liu } 96774b58db8SLongfang Liu 9682514f559SLongfang Liu static int sec_cipher_map(struct sec_ctx *ctx, struct sec_req *req, 969416d8220SZaibo Xu struct scatterlist *src, struct scatterlist *dst) 970416d8220SZaibo Xu { 971416d8220SZaibo Xu struct sec_cipher_req *c_req = &req->c_req; 9722514f559SLongfang Liu struct sec_aead_req *a_req = &req->aead_req; 973416d8220SZaibo Xu struct sec_qp_ctx *qp_ctx = req->qp_ctx; 9742514f559SLongfang Liu struct sec_alg_res *res = &qp_ctx->res[req->req_id]; 975a44dce50SLongfang Liu struct device *dev = ctx->dev; 97674b58db8SLongfang Liu int ret; 9772514f559SLongfang Liu 97874b58db8SLongfang Liu if (req->use_pbuf) { 97974b58db8SLongfang Liu c_req->c_ivin = res->pbuf + SEC_PBUF_IV_OFFSET; 98074b58db8SLongfang Liu c_req->c_ivin_dma = res->pbuf_dma + SEC_PBUF_IV_OFFSET; 98174b58db8SLongfang Liu if (ctx->alg_type == SEC_AEAD) { 982c16a70c1SKai Ye a_req->a_ivin = res->a_ivin; 983c16a70c1SKai Ye a_req->a_ivin_dma = res->a_ivin_dma; 98474b58db8SLongfang Liu a_req->out_mac = res->pbuf + SEC_PBUF_MAC_OFFSET; 98574b58db8SLongfang Liu a_req->out_mac_dma = res->pbuf_dma + 98674b58db8SLongfang Liu SEC_PBUF_MAC_OFFSET; 98774b58db8SLongfang Liu } 988668f1ab7SKai Ye ret = sec_cipher_pbuf_map(ctx, req, src); 98974b58db8SLongfang Liu 99074b58db8SLongfang Liu return ret; 99174b58db8SLongfang Liu } 9922514f559SLongfang Liu c_req->c_ivin = res->c_ivin; 9932514f559SLongfang Liu c_req->c_ivin_dma = res->c_ivin_dma; 9942514f559SLongfang Liu if (ctx->alg_type == SEC_AEAD) { 995c16a70c1SKai Ye a_req->a_ivin = res->a_ivin; 996c16a70c1SKai Ye a_req->a_ivin_dma = res->a_ivin_dma; 9972514f559SLongfang Liu a_req->out_mac = res->out_mac; 9982514f559SLongfang Liu a_req->out_mac_dma = res->out_mac_dma; 9992514f559SLongfang Liu } 1000416d8220SZaibo Xu 10019039878aSKai Ye req->in = hisi_acc_sg_buf_map_to_hw_sgl(dev, src, 1002416d8220SZaibo Xu qp_ctx->c_in_pool, 1003416d8220SZaibo Xu req->req_id, 10049039878aSKai Ye &req->in_dma); 10059039878aSKai Ye if (IS_ERR(req->in)) { 1006416d8220SZaibo Xu dev_err(dev, "fail to dma map input sgl buffers!\n"); 10079039878aSKai Ye return PTR_ERR(req->in); 1008416d8220SZaibo Xu } 1009416d8220SZaibo Xu 1010668f1ab7SKai Ye if (!c_req->encrypt && ctx->alg_type == SEC_AEAD) { 1011668f1ab7SKai Ye ret = sec_aead_mac_init(a_req); 1012668f1ab7SKai Ye if (unlikely(ret)) { 1013668f1ab7SKai Ye dev_err(dev, "fail to init mac data for ICV!\n"); 1014668f1ab7SKai Ye return ret; 1015668f1ab7SKai Ye } 1016668f1ab7SKai Ye } 10179039878aSKai Ye 1018416d8220SZaibo Xu if (dst == src) { 10199039878aSKai Ye c_req->c_out = req->in; 10209039878aSKai Ye c_req->c_out_dma = req->in_dma; 1021416d8220SZaibo Xu } else { 1022416d8220SZaibo Xu c_req->c_out = hisi_acc_sg_buf_map_to_hw_sgl(dev, dst, 1023416d8220SZaibo Xu qp_ctx->c_out_pool, 1024416d8220SZaibo Xu req->req_id, 1025416d8220SZaibo Xu &c_req->c_out_dma); 1026416d8220SZaibo Xu 1027416d8220SZaibo Xu if (IS_ERR(c_req->c_out)) { 1028416d8220SZaibo Xu dev_err(dev, "fail to dma map output sgl buffers!\n"); 10299039878aSKai Ye hisi_acc_sg_buf_unmap(dev, src, req->in); 1030416d8220SZaibo Xu return PTR_ERR(c_req->c_out); 1031416d8220SZaibo Xu } 1032416d8220SZaibo Xu } 1033416d8220SZaibo Xu 1034416d8220SZaibo Xu return 0; 1035416d8220SZaibo Xu } 1036416d8220SZaibo Xu 10372514f559SLongfang Liu static void sec_cipher_unmap(struct sec_ctx *ctx, struct sec_req *req, 1038a181647cSZaibo Xu struct scatterlist *src, struct scatterlist *dst) 1039a181647cSZaibo Xu { 10402514f559SLongfang Liu struct sec_cipher_req *c_req = &req->c_req; 1041a44dce50SLongfang Liu struct device *dev = ctx->dev; 1042a181647cSZaibo Xu 104374b58db8SLongfang Liu if (req->use_pbuf) { 104474b58db8SLongfang Liu sec_cipher_pbuf_unmap(ctx, req, dst); 104574b58db8SLongfang Liu } else { 10462514f559SLongfang Liu if (dst != src) 10479039878aSKai Ye hisi_acc_sg_buf_unmap(dev, src, req->in); 10482514f559SLongfang Liu 10492514f559SLongfang Liu hisi_acc_sg_buf_unmap(dev, dst, c_req->c_out); 1050a181647cSZaibo Xu } 105174b58db8SLongfang Liu } 1052a181647cSZaibo Xu 1053416d8220SZaibo Xu static int sec_skcipher_sgl_map(struct sec_ctx *ctx, struct sec_req *req) 1054416d8220SZaibo Xu { 1055a181647cSZaibo Xu struct skcipher_request *sq = req->c_req.sk_req; 1056416d8220SZaibo Xu 10572514f559SLongfang Liu return sec_cipher_map(ctx, req, sq->src, sq->dst); 1058416d8220SZaibo Xu } 1059416d8220SZaibo Xu 1060416d8220SZaibo Xu static void sec_skcipher_sgl_unmap(struct sec_ctx *ctx, struct sec_req *req) 1061416d8220SZaibo Xu { 10622514f559SLongfang Liu struct skcipher_request *sq = req->c_req.sk_req; 1063416d8220SZaibo Xu 10642514f559SLongfang Liu sec_cipher_unmap(ctx, req, sq->src, sq->dst); 1065416d8220SZaibo Xu } 1066416d8220SZaibo Xu 10672f072d75SZaibo Xu static int sec_aead_aes_set_key(struct sec_cipher_ctx *c_ctx, 10682f072d75SZaibo Xu struct crypto_authenc_keys *keys) 10692f072d75SZaibo Xu { 10702f072d75SZaibo Xu switch (keys->enckeylen) { 10712f072d75SZaibo Xu case AES_KEYSIZE_128: 10722f072d75SZaibo Xu c_ctx->c_key_len = SEC_CKEY_128BIT; 10732f072d75SZaibo Xu break; 10742f072d75SZaibo Xu case AES_KEYSIZE_192: 10752f072d75SZaibo Xu c_ctx->c_key_len = SEC_CKEY_192BIT; 10762f072d75SZaibo Xu break; 10772f072d75SZaibo Xu case AES_KEYSIZE_256: 10782f072d75SZaibo Xu c_ctx->c_key_len = SEC_CKEY_256BIT; 10792f072d75SZaibo Xu break; 10802f072d75SZaibo Xu default: 10812f072d75SZaibo Xu pr_err("hisi_sec2: aead aes key error!\n"); 10822f072d75SZaibo Xu return -EINVAL; 10832f072d75SZaibo Xu } 10842f072d75SZaibo Xu memcpy(c_ctx->c_key, keys->enckey, keys->enckeylen); 10852f072d75SZaibo Xu 10862f072d75SZaibo Xu return 0; 10872f072d75SZaibo Xu } 10882f072d75SZaibo Xu 10892f072d75SZaibo Xu static int sec_aead_auth_set_key(struct sec_auth_ctx *ctx, 10902f072d75SZaibo Xu struct crypto_authenc_keys *keys) 10912f072d75SZaibo Xu { 10922f072d75SZaibo Xu struct crypto_shash *hash_tfm = ctx->hash_tfm; 10935761498cSKai Ye int blocksize, digestsize, ret; 10942f072d75SZaibo Xu 10952f072d75SZaibo Xu if (!keys->authkeylen) { 10962f072d75SZaibo Xu pr_err("hisi_sec2: aead auth key error!\n"); 10972f072d75SZaibo Xu return -EINVAL; 10982f072d75SZaibo Xu } 10992f072d75SZaibo Xu 11002f072d75SZaibo Xu blocksize = crypto_shash_blocksize(hash_tfm); 11015761498cSKai Ye digestsize = crypto_shash_digestsize(hash_tfm); 11022f072d75SZaibo Xu if (keys->authkeylen > blocksize) { 110361c38e3aSEric Biggers ret = crypto_shash_tfm_digest(hash_tfm, keys->authkey, 11042f072d75SZaibo Xu keys->authkeylen, ctx->a_key); 11052f072d75SZaibo Xu if (ret) { 11062203d3f7SColin Ian King pr_err("hisi_sec2: aead auth digest error!\n"); 11072f072d75SZaibo Xu return -EINVAL; 11082f072d75SZaibo Xu } 11095761498cSKai Ye ctx->a_key_len = digestsize; 11102f072d75SZaibo Xu } else { 11112f072d75SZaibo Xu memcpy(ctx->a_key, keys->authkey, keys->authkeylen); 11122f072d75SZaibo Xu ctx->a_key_len = keys->authkeylen; 11132f072d75SZaibo Xu } 11142f072d75SZaibo Xu 11152f072d75SZaibo Xu return 0; 11162f072d75SZaibo Xu } 11172f072d75SZaibo Xu 11186c46a329SKai Ye static int sec_aead_setauthsize(struct crypto_aead *aead, unsigned int authsize) 11196c46a329SKai Ye { 11206c46a329SKai Ye struct crypto_tfm *tfm = crypto_aead_tfm(aead); 11216c46a329SKai Ye struct sec_ctx *ctx = crypto_tfm_ctx(tfm); 11226c46a329SKai Ye struct sec_auth_ctx *a_ctx = &ctx->a_ctx; 11236c46a329SKai Ye 11246c46a329SKai Ye if (unlikely(a_ctx->fallback_aead_tfm)) 11256c46a329SKai Ye return crypto_aead_setauthsize(a_ctx->fallback_aead_tfm, authsize); 11266c46a329SKai Ye 11276c46a329SKai Ye return 0; 11286c46a329SKai Ye } 11296c46a329SKai Ye 11306c46a329SKai Ye static int sec_aead_fallback_setkey(struct sec_auth_ctx *a_ctx, 11316c46a329SKai Ye struct crypto_aead *tfm, const u8 *key, 11326c46a329SKai Ye unsigned int keylen) 11336c46a329SKai Ye { 11346c46a329SKai Ye crypto_aead_clear_flags(a_ctx->fallback_aead_tfm, CRYPTO_TFM_REQ_MASK); 11356c46a329SKai Ye crypto_aead_set_flags(a_ctx->fallback_aead_tfm, 11366c46a329SKai Ye crypto_aead_get_flags(tfm) & CRYPTO_TFM_REQ_MASK); 11376c46a329SKai Ye return crypto_aead_setkey(a_ctx->fallback_aead_tfm, key, keylen); 11386c46a329SKai Ye } 11396c46a329SKai Ye 11402f072d75SZaibo Xu static int sec_aead_setkey(struct crypto_aead *tfm, const u8 *key, 11412f072d75SZaibo Xu const u32 keylen, const enum sec_hash_alg a_alg, 11422f072d75SZaibo Xu const enum sec_calg c_alg, 11432f072d75SZaibo Xu const enum sec_mac_len mac_len, 11442f072d75SZaibo Xu const enum sec_cmode c_mode) 11452f072d75SZaibo Xu { 11462f072d75SZaibo Xu struct sec_ctx *ctx = crypto_aead_ctx(tfm); 11472f072d75SZaibo Xu struct sec_cipher_ctx *c_ctx = &ctx->c_ctx; 11486c46a329SKai Ye struct sec_auth_ctx *a_ctx = &ctx->a_ctx; 1149a44dce50SLongfang Liu struct device *dev = ctx->dev; 11502f072d75SZaibo Xu struct crypto_authenc_keys keys; 11512f072d75SZaibo Xu int ret; 11522f072d75SZaibo Xu 11532f072d75SZaibo Xu ctx->a_ctx.a_alg = a_alg; 11542f072d75SZaibo Xu ctx->c_ctx.c_alg = c_alg; 11552f072d75SZaibo Xu ctx->a_ctx.mac_len = mac_len; 11562f072d75SZaibo Xu c_ctx->c_mode = c_mode; 11572f072d75SZaibo Xu 1158c16a70c1SKai Ye if (c_mode == SEC_CMODE_CCM || c_mode == SEC_CMODE_GCM) { 1159c16a70c1SKai Ye ret = sec_skcipher_aes_sm4_setkey(c_ctx, keylen, c_mode); 1160c16a70c1SKai Ye if (ret) { 1161c16a70c1SKai Ye dev_err(dev, "set sec aes ccm cipher key err!\n"); 1162c16a70c1SKai Ye return ret; 1163c16a70c1SKai Ye } 1164c16a70c1SKai Ye memcpy(c_ctx->c_key, key, keylen); 1165c16a70c1SKai Ye 11666c46a329SKai Ye if (unlikely(a_ctx->fallback_aead_tfm)) { 11676c46a329SKai Ye ret = sec_aead_fallback_setkey(a_ctx, tfm, key, keylen); 11686c46a329SKai Ye if (ret) 11696c46a329SKai Ye return ret; 11706c46a329SKai Ye } 11716c46a329SKai Ye 1172c16a70c1SKai Ye return 0; 1173c16a70c1SKai Ye } 1174c16a70c1SKai Ye 11752f072d75SZaibo Xu if (crypto_authenc_extractkeys(&keys, key, keylen)) 11762f072d75SZaibo Xu goto bad_key; 11772f072d75SZaibo Xu 11782f072d75SZaibo Xu ret = sec_aead_aes_set_key(c_ctx, &keys); 11792f072d75SZaibo Xu if (ret) { 1180a44dce50SLongfang Liu dev_err(dev, "set sec cipher key err!\n"); 11812f072d75SZaibo Xu goto bad_key; 11822f072d75SZaibo Xu } 11832f072d75SZaibo Xu 11842f072d75SZaibo Xu ret = sec_aead_auth_set_key(&ctx->a_ctx, &keys); 11852f072d75SZaibo Xu if (ret) { 1186a44dce50SLongfang Liu dev_err(dev, "set sec auth key err!\n"); 11872f072d75SZaibo Xu goto bad_key; 11882f072d75SZaibo Xu } 11892f072d75SZaibo Xu 1190adc3f65aSKai Ye if ((ctx->a_ctx.mac_len & SEC_SQE_LEN_RATE_MASK) || 1191adc3f65aSKai Ye (ctx->a_ctx.a_key_len & SEC_SQE_LEN_RATE_MASK)) { 1192adc3f65aSKai Ye dev_err(dev, "MAC or AUTH key length error!\n"); 1193adc3f65aSKai Ye goto bad_key; 1194adc3f65aSKai Ye } 1195adc3f65aSKai Ye 11962f072d75SZaibo Xu return 0; 1197633e507fSLongfang Liu 11982f072d75SZaibo Xu bad_key: 11992f072d75SZaibo Xu memzero_explicit(&keys, sizeof(struct crypto_authenc_keys)); 12002f072d75SZaibo Xu return -EINVAL; 12012f072d75SZaibo Xu } 12022f072d75SZaibo Xu 12032f072d75SZaibo Xu 12042f072d75SZaibo Xu #define GEN_SEC_AEAD_SETKEY_FUNC(name, aalg, calg, maclen, cmode) \ 12052f072d75SZaibo Xu static int sec_setkey_##name(struct crypto_aead *tfm, const u8 *key, \ 12062f072d75SZaibo Xu u32 keylen) \ 12072f072d75SZaibo Xu { \ 12082f072d75SZaibo Xu return sec_aead_setkey(tfm, key, keylen, aalg, calg, maclen, cmode);\ 12092f072d75SZaibo Xu } 12102f072d75SZaibo Xu 12112f072d75SZaibo Xu GEN_SEC_AEAD_SETKEY_FUNC(aes_cbc_sha1, SEC_A_HMAC_SHA1, 12122f072d75SZaibo Xu SEC_CALG_AES, SEC_HMAC_SHA1_MAC, SEC_CMODE_CBC) 12132f072d75SZaibo Xu GEN_SEC_AEAD_SETKEY_FUNC(aes_cbc_sha256, SEC_A_HMAC_SHA256, 12142f072d75SZaibo Xu SEC_CALG_AES, SEC_HMAC_SHA256_MAC, SEC_CMODE_CBC) 12152f072d75SZaibo Xu GEN_SEC_AEAD_SETKEY_FUNC(aes_cbc_sha512, SEC_A_HMAC_SHA512, 12162f072d75SZaibo Xu SEC_CALG_AES, SEC_HMAC_SHA512_MAC, SEC_CMODE_CBC) 1217c16a70c1SKai Ye GEN_SEC_AEAD_SETKEY_FUNC(aes_ccm, 0, SEC_CALG_AES, 1218c16a70c1SKai Ye SEC_HMAC_CCM_MAC, SEC_CMODE_CCM) 1219c16a70c1SKai Ye GEN_SEC_AEAD_SETKEY_FUNC(aes_gcm, 0, SEC_CALG_AES, 1220c16a70c1SKai Ye SEC_HMAC_GCM_MAC, SEC_CMODE_GCM) 1221c16a70c1SKai Ye GEN_SEC_AEAD_SETKEY_FUNC(sm4_ccm, 0, SEC_CALG_SM4, 1222c16a70c1SKai Ye SEC_HMAC_CCM_MAC, SEC_CMODE_CCM) 1223c16a70c1SKai Ye GEN_SEC_AEAD_SETKEY_FUNC(sm4_gcm, 0, SEC_CALG_SM4, 1224c16a70c1SKai Ye SEC_HMAC_GCM_MAC, SEC_CMODE_GCM) 12252f072d75SZaibo Xu 12262f072d75SZaibo Xu static int sec_aead_sgl_map(struct sec_ctx *ctx, struct sec_req *req) 12272f072d75SZaibo Xu { 12282f072d75SZaibo Xu struct aead_request *aq = req->aead_req.aead_req; 12292f072d75SZaibo Xu 12302514f559SLongfang Liu return sec_cipher_map(ctx, req, aq->src, aq->dst); 12312f072d75SZaibo Xu } 12322f072d75SZaibo Xu 12332f072d75SZaibo Xu static void sec_aead_sgl_unmap(struct sec_ctx *ctx, struct sec_req *req) 12342f072d75SZaibo Xu { 12352f072d75SZaibo Xu struct aead_request *aq = req->aead_req.aead_req; 12362f072d75SZaibo Xu 12372514f559SLongfang Liu sec_cipher_unmap(ctx, req, aq->src, aq->dst); 12382f072d75SZaibo Xu } 12392f072d75SZaibo Xu 1240416d8220SZaibo Xu static int sec_request_transfer(struct sec_ctx *ctx, struct sec_req *req) 1241416d8220SZaibo Xu { 1242416d8220SZaibo Xu int ret; 1243416d8220SZaibo Xu 1244416d8220SZaibo Xu ret = ctx->req_op->buf_map(ctx, req); 1245b9c8d897SZaibo Xu if (unlikely(ret)) 1246416d8220SZaibo Xu return ret; 1247416d8220SZaibo Xu 1248416d8220SZaibo Xu ctx->req_op->do_transfer(ctx, req); 1249416d8220SZaibo Xu 1250416d8220SZaibo Xu ret = ctx->req_op->bd_fill(ctx, req); 1251b9c8d897SZaibo Xu if (unlikely(ret)) 1252416d8220SZaibo Xu goto unmap_req_buf; 1253416d8220SZaibo Xu 1254416d8220SZaibo Xu return ret; 1255416d8220SZaibo Xu 1256416d8220SZaibo Xu unmap_req_buf: 1257416d8220SZaibo Xu ctx->req_op->buf_unmap(ctx, req); 1258416d8220SZaibo Xu return ret; 1259416d8220SZaibo Xu } 1260416d8220SZaibo Xu 1261416d8220SZaibo Xu static void sec_request_untransfer(struct sec_ctx *ctx, struct sec_req *req) 1262416d8220SZaibo Xu { 1263416d8220SZaibo Xu ctx->req_op->buf_unmap(ctx, req); 1264416d8220SZaibo Xu } 1265416d8220SZaibo Xu 1266416d8220SZaibo Xu static void sec_skcipher_copy_iv(struct sec_ctx *ctx, struct sec_req *req) 1267416d8220SZaibo Xu { 1268416d8220SZaibo Xu struct skcipher_request *sk_req = req->c_req.sk_req; 12692514f559SLongfang Liu struct sec_cipher_req *c_req = &req->c_req; 1270416d8220SZaibo Xu 12712514f559SLongfang Liu memcpy(c_req->c_ivin, sk_req->iv, ctx->c_ctx.ivsize); 1272416d8220SZaibo Xu } 1273416d8220SZaibo Xu 1274416d8220SZaibo Xu static int sec_skcipher_bd_fill(struct sec_ctx *ctx, struct sec_req *req) 1275416d8220SZaibo Xu { 1276416d8220SZaibo Xu struct sec_cipher_ctx *c_ctx = &ctx->c_ctx; 1277416d8220SZaibo Xu struct sec_cipher_req *c_req = &req->c_req; 1278416d8220SZaibo Xu struct sec_sqe *sec_sqe = &req->sec_sqe; 1279416d8220SZaibo Xu u8 scene, sa_type, da_type; 1280416d8220SZaibo Xu u8 bd_type, cipher; 12817c7d902aSZaibo Xu u8 de = 0; 1282416d8220SZaibo Xu 1283416d8220SZaibo Xu memset(sec_sqe, 0, sizeof(struct sec_sqe)); 1284416d8220SZaibo Xu 1285416d8220SZaibo Xu sec_sqe->type2.c_key_addr = cpu_to_le64(c_ctx->c_key_dma); 12862514f559SLongfang Liu sec_sqe->type2.c_ivin_addr = cpu_to_le64(c_req->c_ivin_dma); 12879039878aSKai Ye sec_sqe->type2.data_src_addr = cpu_to_le64(req->in_dma); 1288416d8220SZaibo Xu sec_sqe->type2.data_dst_addr = cpu_to_le64(c_req->c_out_dma); 1289416d8220SZaibo Xu 1290416d8220SZaibo Xu sec_sqe->type2.icvw_kmode |= cpu_to_le16(((u16)c_ctx->c_mode) << 1291416d8220SZaibo Xu SEC_CMODE_OFFSET); 1292416d8220SZaibo Xu sec_sqe->type2.c_alg = c_ctx->c_alg; 1293416d8220SZaibo Xu sec_sqe->type2.icvw_kmode |= cpu_to_le16(((u16)c_ctx->c_key_len) << 1294416d8220SZaibo Xu SEC_CKEY_OFFSET); 1295416d8220SZaibo Xu 1296416d8220SZaibo Xu bd_type = SEC_BD_TYPE2; 1297416d8220SZaibo Xu if (c_req->encrypt) 1298416d8220SZaibo Xu cipher = SEC_CIPHER_ENC << SEC_CIPHER_OFFSET; 1299416d8220SZaibo Xu else 1300416d8220SZaibo Xu cipher = SEC_CIPHER_DEC << SEC_CIPHER_OFFSET; 1301416d8220SZaibo Xu sec_sqe->type_cipher_auth = bd_type | cipher; 1302416d8220SZaibo Xu 1303adc3f65aSKai Ye /* Set destination and source address type */ 1304adc3f65aSKai Ye if (req->use_pbuf) { 130574b58db8SLongfang Liu sa_type = SEC_PBUF << SEC_SRC_SGL_OFFSET; 1306adc3f65aSKai Ye da_type = SEC_PBUF << SEC_DST_SGL_OFFSET; 1307adc3f65aSKai Ye } else { 1308416d8220SZaibo Xu sa_type = SEC_SGL << SEC_SRC_SGL_OFFSET; 1309adc3f65aSKai Ye da_type = SEC_SGL << SEC_DST_SGL_OFFSET; 1310adc3f65aSKai Ye } 1311adc3f65aSKai Ye 1312adc3f65aSKai Ye sec_sqe->sdm_addr_type |= da_type; 1313416d8220SZaibo Xu scene = SEC_COMM_SCENE << SEC_SCENE_OFFSET; 13149039878aSKai Ye if (req->in_dma != c_req->c_out_dma) 1315416d8220SZaibo Xu de = 0x1 << SEC_DE_OFFSET; 1316416d8220SZaibo Xu 1317416d8220SZaibo Xu sec_sqe->sds_sa_type = (de | scene | sa_type); 1318416d8220SZaibo Xu 1319416d8220SZaibo Xu sec_sqe->type2.clen_ivhlen |= cpu_to_le32(c_req->c_len); 1320416d8220SZaibo Xu sec_sqe->type2.tag = cpu_to_le16((u16)req->req_id); 1321416d8220SZaibo Xu 1322416d8220SZaibo Xu return 0; 1323416d8220SZaibo Xu } 1324416d8220SZaibo Xu 1325adc3f65aSKai Ye static int sec_skcipher_bd_fill_v3(struct sec_ctx *ctx, struct sec_req *req) 1326adc3f65aSKai Ye { 1327adc3f65aSKai Ye struct sec_sqe3 *sec_sqe3 = &req->sec_sqe3; 1328adc3f65aSKai Ye struct sec_cipher_ctx *c_ctx = &ctx->c_ctx; 1329adc3f65aSKai Ye struct sec_cipher_req *c_req = &req->c_req; 1330adc3f65aSKai Ye u32 bd_param = 0; 1331adc3f65aSKai Ye u16 cipher; 1332adc3f65aSKai Ye 1333adc3f65aSKai Ye memset(sec_sqe3, 0, sizeof(struct sec_sqe3)); 1334adc3f65aSKai Ye 1335adc3f65aSKai Ye sec_sqe3->c_key_addr = cpu_to_le64(c_ctx->c_key_dma); 1336adc3f65aSKai Ye sec_sqe3->no_scene.c_ivin_addr = cpu_to_le64(c_req->c_ivin_dma); 13379039878aSKai Ye sec_sqe3->data_src_addr = cpu_to_le64(req->in_dma); 1338adc3f65aSKai Ye sec_sqe3->data_dst_addr = cpu_to_le64(c_req->c_out_dma); 1339adc3f65aSKai Ye 1340adc3f65aSKai Ye sec_sqe3->c_mode_alg = ((u8)c_ctx->c_alg << SEC_CALG_OFFSET_V3) | 1341adc3f65aSKai Ye c_ctx->c_mode; 1342adc3f65aSKai Ye sec_sqe3->c_icv_key |= cpu_to_le16(((u16)c_ctx->c_key_len) << 1343adc3f65aSKai Ye SEC_CKEY_OFFSET_V3); 1344adc3f65aSKai Ye 1345adc3f65aSKai Ye if (c_req->encrypt) 1346adc3f65aSKai Ye cipher = SEC_CIPHER_ENC; 1347adc3f65aSKai Ye else 1348adc3f65aSKai Ye cipher = SEC_CIPHER_DEC; 1349adc3f65aSKai Ye sec_sqe3->c_icv_key |= cpu_to_le16(cipher); 1350adc3f65aSKai Ye 1351566f060fSKai Ye /* Set the CTR counter mode is 128bit rollover */ 1352566f060fSKai Ye sec_sqe3->auth_mac_key = cpu_to_le32((u32)SEC_CTR_CNT_ROLLOVER << 1353566f060fSKai Ye SEC_CTR_CNT_OFFSET); 1354566f060fSKai Ye 1355adc3f65aSKai Ye if (req->use_pbuf) { 1356adc3f65aSKai Ye bd_param |= SEC_PBUF << SEC_SRC_SGL_OFFSET_V3; 1357adc3f65aSKai Ye bd_param |= SEC_PBUF << SEC_DST_SGL_OFFSET_V3; 1358adc3f65aSKai Ye } else { 1359adc3f65aSKai Ye bd_param |= SEC_SGL << SEC_SRC_SGL_OFFSET_V3; 1360adc3f65aSKai Ye bd_param |= SEC_SGL << SEC_DST_SGL_OFFSET_V3; 1361adc3f65aSKai Ye } 1362adc3f65aSKai Ye 1363adc3f65aSKai Ye bd_param |= SEC_COMM_SCENE << SEC_SCENE_OFFSET_V3; 13649039878aSKai Ye if (req->in_dma != c_req->c_out_dma) 1365adc3f65aSKai Ye bd_param |= 0x1 << SEC_DE_OFFSET_V3; 1366adc3f65aSKai Ye 1367adc3f65aSKai Ye bd_param |= SEC_BD_TYPE3; 1368adc3f65aSKai Ye sec_sqe3->bd_param = cpu_to_le32(bd_param); 1369adc3f65aSKai Ye 1370adc3f65aSKai Ye sec_sqe3->c_len_ivin |= cpu_to_le32(c_req->c_len); 1371adc3f65aSKai Ye sec_sqe3->tag = cpu_to_le64(req); 1372adc3f65aSKai Ye 1373adc3f65aSKai Ye return 0; 1374adc3f65aSKai Ye } 1375adc3f65aSKai Ye 13767b44c0eeSKai Ye /* increment counter (128-bit int) */ 13777b44c0eeSKai Ye static void ctr_iv_inc(__u8 *counter, __u8 bits, __u32 nums) 13787b44c0eeSKai Ye { 13797b44c0eeSKai Ye do { 13807b44c0eeSKai Ye --bits; 13817b44c0eeSKai Ye nums += counter[bits]; 13827b44c0eeSKai Ye counter[bits] = nums & BITS_MASK; 13837b44c0eeSKai Ye nums >>= BYTE_BITS; 13847b44c0eeSKai Ye } while (bits && nums); 13857b44c0eeSKai Ye } 13867b44c0eeSKai Ye 13872f072d75SZaibo Xu static void sec_update_iv(struct sec_req *req, enum sec_alg_type alg_type) 1388416d8220SZaibo Xu { 13892f072d75SZaibo Xu struct aead_request *aead_req = req->aead_req.aead_req; 1390416d8220SZaibo Xu struct skcipher_request *sk_req = req->c_req.sk_req; 1391416d8220SZaibo Xu u32 iv_size = req->ctx->c_ctx.ivsize; 1392416d8220SZaibo Xu struct scatterlist *sgl; 13932f072d75SZaibo Xu unsigned int cryptlen; 1394416d8220SZaibo Xu size_t sz; 13952f072d75SZaibo Xu u8 *iv; 1396416d8220SZaibo Xu 1397416d8220SZaibo Xu if (req->c_req.encrypt) 13982f072d75SZaibo Xu sgl = alg_type == SEC_SKCIPHER ? sk_req->dst : aead_req->dst; 1399416d8220SZaibo Xu else 14002f072d75SZaibo Xu sgl = alg_type == SEC_SKCIPHER ? sk_req->src : aead_req->src; 1401416d8220SZaibo Xu 14022f072d75SZaibo Xu if (alg_type == SEC_SKCIPHER) { 14032f072d75SZaibo Xu iv = sk_req->iv; 14042f072d75SZaibo Xu cryptlen = sk_req->cryptlen; 14052f072d75SZaibo Xu } else { 14062f072d75SZaibo Xu iv = aead_req->iv; 14072f072d75SZaibo Xu cryptlen = aead_req->cryptlen; 14082f072d75SZaibo Xu } 14092f072d75SZaibo Xu 14107b44c0eeSKai Ye if (req->ctx->c_ctx.c_mode == SEC_CMODE_CBC) { 14112f072d75SZaibo Xu sz = sg_pcopy_to_buffer(sgl, sg_nents(sgl), iv, iv_size, 14122f072d75SZaibo Xu cryptlen - iv_size); 1413b9c8d897SZaibo Xu if (unlikely(sz != iv_size)) 1414a44dce50SLongfang Liu dev_err(req->ctx->dev, "copy output iv error!\n"); 14157b44c0eeSKai Ye } else { 14167b44c0eeSKai Ye sz = cryptlen / iv_size; 14177b44c0eeSKai Ye if (cryptlen % iv_size) 14187b44c0eeSKai Ye sz += 1; 14197b44c0eeSKai Ye ctr_iv_inc(iv, iv_size, sz); 14207b44c0eeSKai Ye } 1421416d8220SZaibo Xu } 1422416d8220SZaibo Xu 14239597efc3SKai Ye static struct sec_req *sec_back_req_clear(struct sec_ctx *ctx, 14249597efc3SKai Ye struct sec_qp_ctx *qp_ctx) 14259597efc3SKai Ye { 14269597efc3SKai Ye struct sec_req *backlog_req = NULL; 14279597efc3SKai Ye 142802884a4fSZhengchao Shao spin_lock_bh(&qp_ctx->req_lock); 14299597efc3SKai Ye if (ctx->fake_req_limit >= 14309597efc3SKai Ye atomic_read(&qp_ctx->qp->qp_status.used) && 14319597efc3SKai Ye !list_empty(&qp_ctx->backlog)) { 14329597efc3SKai Ye backlog_req = list_first_entry(&qp_ctx->backlog, 14339597efc3SKai Ye typeof(*backlog_req), backlog_head); 14349597efc3SKai Ye list_del(&backlog_req->backlog_head); 14359597efc3SKai Ye } 143602884a4fSZhengchao Shao spin_unlock_bh(&qp_ctx->req_lock); 14379597efc3SKai Ye 14389597efc3SKai Ye return backlog_req; 14399597efc3SKai Ye } 14409597efc3SKai Ye 1441310ea0acSZaibo Xu static void sec_skcipher_callback(struct sec_ctx *ctx, struct sec_req *req, 1442310ea0acSZaibo Xu int err) 1443416d8220SZaibo Xu { 1444416d8220SZaibo Xu struct skcipher_request *sk_req = req->c_req.sk_req; 1445416d8220SZaibo Xu struct sec_qp_ctx *qp_ctx = req->qp_ctx; 14469597efc3SKai Ye struct skcipher_request *backlog_sk_req; 14479597efc3SKai Ye struct sec_req *backlog_req; 1448416d8220SZaibo Xu 1449416d8220SZaibo Xu sec_free_req_id(req); 1450416d8220SZaibo Xu 14517b44c0eeSKai Ye /* IV output at encrypto of CBC/CTR mode */ 14527b44c0eeSKai Ye if (!err && (ctx->c_ctx.c_mode == SEC_CMODE_CBC || 14537b44c0eeSKai Ye ctx->c_ctx.c_mode == SEC_CMODE_CTR) && req->c_req.encrypt) 14542f072d75SZaibo Xu sec_update_iv(req, SEC_SKCIPHER); 1455416d8220SZaibo Xu 14569597efc3SKai Ye while (1) { 14579597efc3SKai Ye backlog_req = sec_back_req_clear(ctx, qp_ctx); 14589597efc3SKai Ye if (!backlog_req) 14599597efc3SKai Ye break; 14609597efc3SKai Ye 14619597efc3SKai Ye backlog_sk_req = backlog_req->c_req.sk_req; 1462e2b537b7SHerbert Xu skcipher_request_complete(backlog_sk_req, -EINPROGRESS); 14639597efc3SKai Ye atomic64_inc(&ctx->sec->debug.dfx.recv_busy_cnt); 14649597efc3SKai Ye } 14659597efc3SKai Ye 1466e2b537b7SHerbert Xu skcipher_request_complete(sk_req, err); 1467416d8220SZaibo Xu } 1468416d8220SZaibo Xu 1469c16a70c1SKai Ye static void set_aead_auth_iv(struct sec_ctx *ctx, struct sec_req *req) 14702f072d75SZaibo Xu { 14712f072d75SZaibo Xu struct aead_request *aead_req = req->aead_req.aead_req; 14722514f559SLongfang Liu struct sec_cipher_req *c_req = &req->c_req; 1473c16a70c1SKai Ye struct sec_aead_req *a_req = &req->aead_req; 1474c16a70c1SKai Ye size_t authsize = ctx->a_ctx.mac_len; 1475c16a70c1SKai Ye u32 data_size = aead_req->cryptlen; 1476c16a70c1SKai Ye u8 flage = 0; 1477c16a70c1SKai Ye u8 cm, cl; 1478c16a70c1SKai Ye 1479c16a70c1SKai Ye /* the specification has been checked in aead_iv_demension_check() */ 1480c16a70c1SKai Ye cl = c_req->c_ivin[0] + 1; 1481c16a70c1SKai Ye c_req->c_ivin[ctx->c_ctx.ivsize - cl] = 0x00; 1482c16a70c1SKai Ye memset(&c_req->c_ivin[ctx->c_ctx.ivsize - cl], 0, cl); 1483c16a70c1SKai Ye c_req->c_ivin[ctx->c_ctx.ivsize - IV_LAST_BYTE1] = IV_CTR_INIT; 1484c16a70c1SKai Ye 1485c16a70c1SKai Ye /* the last 3bit is L' */ 1486c16a70c1SKai Ye flage |= c_req->c_ivin[0] & IV_CL_MASK; 1487c16a70c1SKai Ye 1488c16a70c1SKai Ye /* the M' is bit3~bit5, the Flags is bit6 */ 1489c16a70c1SKai Ye cm = (authsize - IV_CM_CAL_NUM) / IV_CM_CAL_NUM; 1490c16a70c1SKai Ye flage |= cm << IV_CM_OFFSET; 1491c16a70c1SKai Ye if (aead_req->assoclen) 1492c16a70c1SKai Ye flage |= 0x01 << IV_FLAGS_OFFSET; 1493c16a70c1SKai Ye 1494c16a70c1SKai Ye memcpy(a_req->a_ivin, c_req->c_ivin, ctx->c_ctx.ivsize); 1495c16a70c1SKai Ye a_req->a_ivin[0] = flage; 1496c16a70c1SKai Ye 1497c16a70c1SKai Ye /* 1498c16a70c1SKai Ye * the last 32bit is counter's initial number, 1499c16a70c1SKai Ye * but the nonce uses the first 16bit 1500c16a70c1SKai Ye * the tail 16bit fill with the cipher length 1501c16a70c1SKai Ye */ 1502c16a70c1SKai Ye if (!c_req->encrypt) 1503c16a70c1SKai Ye data_size = aead_req->cryptlen - authsize; 1504c16a70c1SKai Ye 1505c16a70c1SKai Ye a_req->a_ivin[ctx->c_ctx.ivsize - IV_LAST_BYTE1] = 1506c16a70c1SKai Ye data_size & IV_LAST_BYTE_MASK; 1507c16a70c1SKai Ye data_size >>= IV_BYTE_OFFSET; 1508c16a70c1SKai Ye a_req->a_ivin[ctx->c_ctx.ivsize - IV_LAST_BYTE2] = 1509c16a70c1SKai Ye data_size & IV_LAST_BYTE_MASK; 1510c16a70c1SKai Ye } 1511c16a70c1SKai Ye 1512c16a70c1SKai Ye static void sec_aead_set_iv(struct sec_ctx *ctx, struct sec_req *req) 1513c16a70c1SKai Ye { 1514c16a70c1SKai Ye struct aead_request *aead_req = req->aead_req.aead_req; 1515c16a70c1SKai Ye struct crypto_aead *tfm = crypto_aead_reqtfm(aead_req); 1516c16a70c1SKai Ye size_t authsize = crypto_aead_authsize(tfm); 1517c16a70c1SKai Ye struct sec_cipher_req *c_req = &req->c_req; 1518c16a70c1SKai Ye struct sec_aead_req *a_req = &req->aead_req; 15192f072d75SZaibo Xu 15202514f559SLongfang Liu memcpy(c_req->c_ivin, aead_req->iv, ctx->c_ctx.ivsize); 1521c16a70c1SKai Ye 1522c16a70c1SKai Ye if (ctx->c_ctx.c_mode == SEC_CMODE_CCM) { 1523c16a70c1SKai Ye /* 1524c16a70c1SKai Ye * CCM 16Byte Cipher_IV: {1B_Flage,13B_IV,2B_counter}, 1525c16a70c1SKai Ye * the counter must set to 0x01 1526c16a70c1SKai Ye */ 1527c16a70c1SKai Ye ctx->a_ctx.mac_len = authsize; 1528c16a70c1SKai Ye /* CCM 16Byte Auth_IV: {1B_AFlage,13B_IV,2B_Ptext_length} */ 1529c16a70c1SKai Ye set_aead_auth_iv(ctx, req); 1530c16a70c1SKai Ye } 1531c16a70c1SKai Ye 1532c16a70c1SKai Ye /* GCM 12Byte Cipher_IV == Auth_IV */ 1533c16a70c1SKai Ye if (ctx->c_ctx.c_mode == SEC_CMODE_GCM) { 1534c16a70c1SKai Ye ctx->a_ctx.mac_len = authsize; 1535c16a70c1SKai Ye memcpy(a_req->a_ivin, c_req->c_ivin, SEC_AIV_SIZE); 1536c16a70c1SKai Ye } 1537c16a70c1SKai Ye } 1538c16a70c1SKai Ye 1539c16a70c1SKai Ye static void sec_auth_bd_fill_xcm(struct sec_auth_ctx *ctx, int dir, 1540c16a70c1SKai Ye struct sec_req *req, struct sec_sqe *sec_sqe) 1541c16a70c1SKai Ye { 1542c16a70c1SKai Ye struct sec_aead_req *a_req = &req->aead_req; 1543c16a70c1SKai Ye struct aead_request *aq = a_req->aead_req; 1544c16a70c1SKai Ye 1545c16a70c1SKai Ye /* C_ICV_Len is MAC size, 0x4 ~ 0x10 */ 1546c16a70c1SKai Ye sec_sqe->type2.icvw_kmode |= cpu_to_le16((u16)ctx->mac_len); 1547c16a70c1SKai Ye 1548c16a70c1SKai Ye /* mode set to CCM/GCM, don't set {A_Alg, AKey_Len, MAC_Len} */ 1549c16a70c1SKai Ye sec_sqe->type2.a_key_addr = sec_sqe->type2.c_key_addr; 1550c16a70c1SKai Ye sec_sqe->type2.a_ivin_addr = cpu_to_le64(a_req->a_ivin_dma); 1551c16a70c1SKai Ye sec_sqe->type_cipher_auth |= SEC_NO_AUTH << SEC_AUTH_OFFSET; 1552c16a70c1SKai Ye 1553c16a70c1SKai Ye if (dir) 1554c16a70c1SKai Ye sec_sqe->sds_sa_type &= SEC_CIPHER_AUTH; 1555c16a70c1SKai Ye else 1556c16a70c1SKai Ye sec_sqe->sds_sa_type |= SEC_AUTH_CIPHER; 1557c16a70c1SKai Ye 1558c16a70c1SKai Ye sec_sqe->type2.alen_ivllen = cpu_to_le32(aq->assoclen); 1559c16a70c1SKai Ye sec_sqe->type2.auth_src_offset = cpu_to_le16(0x0); 1560c16a70c1SKai Ye sec_sqe->type2.cipher_src_offset = cpu_to_le16((u16)aq->assoclen); 1561c16a70c1SKai Ye 1562c16a70c1SKai Ye sec_sqe->type2.mac_addr = cpu_to_le64(a_req->out_mac_dma); 1563c16a70c1SKai Ye } 1564c16a70c1SKai Ye 1565c16a70c1SKai Ye static void sec_auth_bd_fill_xcm_v3(struct sec_auth_ctx *ctx, int dir, 1566c16a70c1SKai Ye struct sec_req *req, struct sec_sqe3 *sqe3) 1567c16a70c1SKai Ye { 1568c16a70c1SKai Ye struct sec_aead_req *a_req = &req->aead_req; 1569c16a70c1SKai Ye struct aead_request *aq = a_req->aead_req; 1570c16a70c1SKai Ye 1571c16a70c1SKai Ye /* C_ICV_Len is MAC size, 0x4 ~ 0x10 */ 1572c16a70c1SKai Ye sqe3->c_icv_key |= cpu_to_le16((u16)ctx->mac_len << SEC_MAC_OFFSET_V3); 1573c16a70c1SKai Ye 1574c16a70c1SKai Ye /* mode set to CCM/GCM, don't set {A_Alg, AKey_Len, MAC_Len} */ 1575c16a70c1SKai Ye sqe3->a_key_addr = sqe3->c_key_addr; 1576c16a70c1SKai Ye sqe3->auth_ivin.a_ivin_addr = cpu_to_le64(a_req->a_ivin_dma); 1577c16a70c1SKai Ye sqe3->auth_mac_key |= SEC_NO_AUTH; 1578c16a70c1SKai Ye 1579c16a70c1SKai Ye if (dir) 1580c16a70c1SKai Ye sqe3->huk_iv_seq &= SEC_CIPHER_AUTH_V3; 1581c16a70c1SKai Ye else 1582c16a70c1SKai Ye sqe3->huk_iv_seq |= SEC_AUTH_CIPHER_V3; 1583c16a70c1SKai Ye 1584c16a70c1SKai Ye sqe3->a_len_key = cpu_to_le32(aq->assoclen); 1585c16a70c1SKai Ye sqe3->auth_src_offset = cpu_to_le16(0x0); 1586c16a70c1SKai Ye sqe3->cipher_src_offset = cpu_to_le16((u16)aq->assoclen); 1587c16a70c1SKai Ye sqe3->mac_addr = cpu_to_le64(a_req->out_mac_dma); 15882f072d75SZaibo Xu } 15892f072d75SZaibo Xu 15902f072d75SZaibo Xu static void sec_auth_bd_fill_ex(struct sec_auth_ctx *ctx, int dir, 15912f072d75SZaibo Xu struct sec_req *req, struct sec_sqe *sec_sqe) 15922f072d75SZaibo Xu { 15932f072d75SZaibo Xu struct sec_aead_req *a_req = &req->aead_req; 15942f072d75SZaibo Xu struct sec_cipher_req *c_req = &req->c_req; 15952f072d75SZaibo Xu struct aead_request *aq = a_req->aead_req; 15962f072d75SZaibo Xu 15972f072d75SZaibo Xu sec_sqe->type2.a_key_addr = cpu_to_le64(ctx->a_key_dma); 15982f072d75SZaibo Xu 15992f072d75SZaibo Xu sec_sqe->type2.mac_key_alg = 16002f072d75SZaibo Xu cpu_to_le32(ctx->mac_len / SEC_SQE_LEN_RATE); 16012f072d75SZaibo Xu 16022f072d75SZaibo Xu sec_sqe->type2.mac_key_alg |= 16032f072d75SZaibo Xu cpu_to_le32((u32)((ctx->a_key_len) / 16042f072d75SZaibo Xu SEC_SQE_LEN_RATE) << SEC_AKEY_OFFSET); 16052f072d75SZaibo Xu 16062f072d75SZaibo Xu sec_sqe->type2.mac_key_alg |= 16072f072d75SZaibo Xu cpu_to_le32((u32)(ctx->a_alg) << SEC_AEAD_ALG_OFFSET); 16082f072d75SZaibo Xu 1609668f1ab7SKai Ye if (dir) { 16102f072d75SZaibo Xu sec_sqe->type_cipher_auth |= SEC_AUTH_TYPE1 << SEC_AUTH_OFFSET; 16112f072d75SZaibo Xu sec_sqe->sds_sa_type &= SEC_CIPHER_AUTH; 1612668f1ab7SKai Ye } else { 1613668f1ab7SKai Ye sec_sqe->type_cipher_auth |= SEC_AUTH_TYPE2 << SEC_AUTH_OFFSET; 16142f072d75SZaibo Xu sec_sqe->sds_sa_type |= SEC_AUTH_CIPHER; 1615668f1ab7SKai Ye } 16162f072d75SZaibo Xu sec_sqe->type2.alen_ivllen = cpu_to_le32(c_req->c_len + aq->assoclen); 16172f072d75SZaibo Xu 16182f072d75SZaibo Xu sec_sqe->type2.cipher_src_offset = cpu_to_le16((u16)aq->assoclen); 16192f072d75SZaibo Xu 16202514f559SLongfang Liu sec_sqe->type2.mac_addr = cpu_to_le64(a_req->out_mac_dma); 16212f072d75SZaibo Xu } 16222f072d75SZaibo Xu 16232f072d75SZaibo Xu static int sec_aead_bd_fill(struct sec_ctx *ctx, struct sec_req *req) 16242f072d75SZaibo Xu { 16252f072d75SZaibo Xu struct sec_auth_ctx *auth_ctx = &ctx->a_ctx; 16262f072d75SZaibo Xu struct sec_sqe *sec_sqe = &req->sec_sqe; 16272f072d75SZaibo Xu int ret; 16282f072d75SZaibo Xu 16292f072d75SZaibo Xu ret = sec_skcipher_bd_fill(ctx, req); 16302f072d75SZaibo Xu if (unlikely(ret)) { 1631a44dce50SLongfang Liu dev_err(ctx->dev, "skcipher bd fill is error!\n"); 16322f072d75SZaibo Xu return ret; 16332f072d75SZaibo Xu } 16342f072d75SZaibo Xu 1635c16a70c1SKai Ye if (ctx->c_ctx.c_mode == SEC_CMODE_CCM || 1636c16a70c1SKai Ye ctx->c_ctx.c_mode == SEC_CMODE_GCM) 1637c16a70c1SKai Ye sec_auth_bd_fill_xcm(auth_ctx, req->c_req.encrypt, req, sec_sqe); 1638c16a70c1SKai Ye else 16392f072d75SZaibo Xu sec_auth_bd_fill_ex(auth_ctx, req->c_req.encrypt, req, sec_sqe); 16402f072d75SZaibo Xu 16412f072d75SZaibo Xu return 0; 16422f072d75SZaibo Xu } 16432f072d75SZaibo Xu 1644adc3f65aSKai Ye static void sec_auth_bd_fill_ex_v3(struct sec_auth_ctx *ctx, int dir, 1645adc3f65aSKai Ye struct sec_req *req, struct sec_sqe3 *sqe3) 1646adc3f65aSKai Ye { 1647adc3f65aSKai Ye struct sec_aead_req *a_req = &req->aead_req; 1648adc3f65aSKai Ye struct sec_cipher_req *c_req = &req->c_req; 1649adc3f65aSKai Ye struct aead_request *aq = a_req->aead_req; 1650adc3f65aSKai Ye 1651adc3f65aSKai Ye sqe3->a_key_addr = cpu_to_le64(ctx->a_key_dma); 1652adc3f65aSKai Ye 1653adc3f65aSKai Ye sqe3->auth_mac_key |= 1654adc3f65aSKai Ye cpu_to_le32((u32)(ctx->mac_len / 1655adc3f65aSKai Ye SEC_SQE_LEN_RATE) << SEC_MAC_OFFSET_V3); 1656adc3f65aSKai Ye 1657adc3f65aSKai Ye sqe3->auth_mac_key |= 1658adc3f65aSKai Ye cpu_to_le32((u32)(ctx->a_key_len / 1659adc3f65aSKai Ye SEC_SQE_LEN_RATE) << SEC_AKEY_OFFSET_V3); 1660adc3f65aSKai Ye 1661adc3f65aSKai Ye sqe3->auth_mac_key |= 1662adc3f65aSKai Ye cpu_to_le32((u32)(ctx->a_alg) << SEC_AUTH_ALG_OFFSET_V3); 1663adc3f65aSKai Ye 1664adc3f65aSKai Ye if (dir) { 1665adc3f65aSKai Ye sqe3->auth_mac_key |= cpu_to_le32((u32)SEC_AUTH_TYPE1); 1666adc3f65aSKai Ye sqe3->huk_iv_seq &= SEC_CIPHER_AUTH_V3; 1667adc3f65aSKai Ye } else { 166860ef3ddeSKai Ye sqe3->auth_mac_key |= cpu_to_le32((u32)SEC_AUTH_TYPE2); 1669adc3f65aSKai Ye sqe3->huk_iv_seq |= SEC_AUTH_CIPHER_V3; 1670adc3f65aSKai Ye } 1671adc3f65aSKai Ye sqe3->a_len_key = cpu_to_le32(c_req->c_len + aq->assoclen); 1672adc3f65aSKai Ye 1673adc3f65aSKai Ye sqe3->cipher_src_offset = cpu_to_le16((u16)aq->assoclen); 1674adc3f65aSKai Ye 1675adc3f65aSKai Ye sqe3->mac_addr = cpu_to_le64(a_req->out_mac_dma); 1676adc3f65aSKai Ye } 1677adc3f65aSKai Ye 1678adc3f65aSKai Ye static int sec_aead_bd_fill_v3(struct sec_ctx *ctx, struct sec_req *req) 1679adc3f65aSKai Ye { 1680adc3f65aSKai Ye struct sec_auth_ctx *auth_ctx = &ctx->a_ctx; 1681adc3f65aSKai Ye struct sec_sqe3 *sec_sqe3 = &req->sec_sqe3; 1682adc3f65aSKai Ye int ret; 1683adc3f65aSKai Ye 1684adc3f65aSKai Ye ret = sec_skcipher_bd_fill_v3(ctx, req); 1685adc3f65aSKai Ye if (unlikely(ret)) { 1686adc3f65aSKai Ye dev_err(ctx->dev, "skcipher bd3 fill is error!\n"); 1687adc3f65aSKai Ye return ret; 1688adc3f65aSKai Ye } 1689adc3f65aSKai Ye 1690c16a70c1SKai Ye if (ctx->c_ctx.c_mode == SEC_CMODE_CCM || 1691c16a70c1SKai Ye ctx->c_ctx.c_mode == SEC_CMODE_GCM) 1692c16a70c1SKai Ye sec_auth_bd_fill_xcm_v3(auth_ctx, req->c_req.encrypt, 1693c16a70c1SKai Ye req, sec_sqe3); 1694c16a70c1SKai Ye else 1695c16a70c1SKai Ye sec_auth_bd_fill_ex_v3(auth_ctx, req->c_req.encrypt, 1696c16a70c1SKai Ye req, sec_sqe3); 1697adc3f65aSKai Ye 1698adc3f65aSKai Ye return 0; 1699adc3f65aSKai Ye } 1700adc3f65aSKai Ye 17012f072d75SZaibo Xu static void sec_aead_callback(struct sec_ctx *c, struct sec_req *req, int err) 17022f072d75SZaibo Xu { 17032f072d75SZaibo Xu struct aead_request *a_req = req->aead_req.aead_req; 17042f072d75SZaibo Xu struct crypto_aead *tfm = crypto_aead_reqtfm(a_req); 17052514f559SLongfang Liu struct sec_aead_req *aead_req = &req->aead_req; 17062f072d75SZaibo Xu struct sec_cipher_req *c_req = &req->c_req; 17072f072d75SZaibo Xu size_t authsize = crypto_aead_authsize(tfm); 17082f072d75SZaibo Xu struct sec_qp_ctx *qp_ctx = req->qp_ctx; 17099597efc3SKai Ye struct aead_request *backlog_aead_req; 17109597efc3SKai Ye struct sec_req *backlog_req; 17112f072d75SZaibo Xu size_t sz; 17122f072d75SZaibo Xu 17132f072d75SZaibo Xu if (!err && c->c_ctx.c_mode == SEC_CMODE_CBC && c_req->encrypt) 17142f072d75SZaibo Xu sec_update_iv(req, SEC_AEAD); 17152f072d75SZaibo Xu 17162f072d75SZaibo Xu /* Copy output mac */ 17172f072d75SZaibo Xu if (!err && c_req->encrypt) { 17182f072d75SZaibo Xu struct scatterlist *sgl = a_req->dst; 17192f072d75SZaibo Xu 17202f072d75SZaibo Xu sz = sg_pcopy_from_buffer(sgl, sg_nents(sgl), 17212514f559SLongfang Liu aead_req->out_mac, 17222f072d75SZaibo Xu authsize, a_req->cryptlen + 17232f072d75SZaibo Xu a_req->assoclen); 17242f072d75SZaibo Xu if (unlikely(sz != authsize)) { 1725a44dce50SLongfang Liu dev_err(c->dev, "copy out mac err!\n"); 17262f072d75SZaibo Xu err = -EINVAL; 17272f072d75SZaibo Xu } 17282f072d75SZaibo Xu } 17292f072d75SZaibo Xu 17302f072d75SZaibo Xu sec_free_req_id(req); 17312f072d75SZaibo Xu 17329597efc3SKai Ye while (1) { 17339597efc3SKai Ye backlog_req = sec_back_req_clear(c, qp_ctx); 17349597efc3SKai Ye if (!backlog_req) 17359597efc3SKai Ye break; 17369597efc3SKai Ye 17379597efc3SKai Ye backlog_aead_req = backlog_req->aead_req.aead_req; 1738e2b537b7SHerbert Xu aead_request_complete(backlog_aead_req, -EINPROGRESS); 17399597efc3SKai Ye atomic64_inc(&c->sec->debug.dfx.recv_busy_cnt); 17409597efc3SKai Ye } 17412f072d75SZaibo Xu 1742e2b537b7SHerbert Xu aead_request_complete(a_req, err); 17432f072d75SZaibo Xu } 17442f072d75SZaibo Xu 1745416d8220SZaibo Xu static void sec_request_uninit(struct sec_ctx *ctx, struct sec_req *req) 1746416d8220SZaibo Xu { 1747416d8220SZaibo Xu sec_free_req_id(req); 1748a181647cSZaibo Xu sec_free_queue_id(ctx, req); 1749416d8220SZaibo Xu } 1750416d8220SZaibo Xu 1751416d8220SZaibo Xu static int sec_request_init(struct sec_ctx *ctx, struct sec_req *req) 1752416d8220SZaibo Xu { 1753416d8220SZaibo Xu struct sec_qp_ctx *qp_ctx; 17547c7d902aSZaibo Xu int queue_id; 1755416d8220SZaibo Xu 1756416d8220SZaibo Xu /* To load balance */ 1757a181647cSZaibo Xu queue_id = sec_alloc_queue_id(ctx, req); 1758a181647cSZaibo Xu qp_ctx = &ctx->qp_ctx[queue_id]; 1759416d8220SZaibo Xu 1760416d8220SZaibo Xu req->req_id = sec_alloc_req_id(req, qp_ctx); 1761b9c8d897SZaibo Xu if (unlikely(req->req_id < 0)) { 1762a181647cSZaibo Xu sec_free_queue_id(ctx, req); 1763416d8220SZaibo Xu return req->req_id; 1764416d8220SZaibo Xu } 1765416d8220SZaibo Xu 17667c7d902aSZaibo Xu return 0; 1767416d8220SZaibo Xu } 1768416d8220SZaibo Xu 1769416d8220SZaibo Xu static int sec_process(struct sec_ctx *ctx, struct sec_req *req) 1770416d8220SZaibo Xu { 17712514f559SLongfang Liu struct sec_cipher_req *c_req = &req->c_req; 1772416d8220SZaibo Xu int ret; 1773416d8220SZaibo Xu 1774416d8220SZaibo Xu ret = sec_request_init(ctx, req); 1775b9c8d897SZaibo Xu if (unlikely(ret)) 1776416d8220SZaibo Xu return ret; 1777416d8220SZaibo Xu 1778416d8220SZaibo Xu ret = sec_request_transfer(ctx, req); 1779b9c8d897SZaibo Xu if (unlikely(ret)) 1780416d8220SZaibo Xu goto err_uninit_req; 1781416d8220SZaibo Xu 1782416d8220SZaibo Xu /* Output IV as decrypto */ 17837b44c0eeSKai Ye if (!req->c_req.encrypt && (ctx->c_ctx.c_mode == SEC_CMODE_CBC || 17847b44c0eeSKai Ye ctx->c_ctx.c_mode == SEC_CMODE_CTR)) 17852f072d75SZaibo Xu sec_update_iv(req, ctx->alg_type); 1786416d8220SZaibo Xu 1787416d8220SZaibo Xu ret = ctx->req_op->bd_send(ctx, req); 17889597efc3SKai Ye if (unlikely((ret != -EBUSY && ret != -EINPROGRESS) || 17899597efc3SKai Ye (ret == -EBUSY && !(req->flag & CRYPTO_TFM_REQ_MAY_BACKLOG)))) { 1790a44dce50SLongfang Liu dev_err_ratelimited(ctx->dev, "send sec request failed!\n"); 1791416d8220SZaibo Xu goto err_send_req; 1792416d8220SZaibo Xu } 1793416d8220SZaibo Xu 1794416d8220SZaibo Xu return ret; 1795416d8220SZaibo Xu 1796416d8220SZaibo Xu err_send_req: 1797416d8220SZaibo Xu /* As failing, restore the IV from user */ 17982f072d75SZaibo Xu if (ctx->c_ctx.c_mode == SEC_CMODE_CBC && !req->c_req.encrypt) { 17992f072d75SZaibo Xu if (ctx->alg_type == SEC_SKCIPHER) 18002514f559SLongfang Liu memcpy(req->c_req.sk_req->iv, c_req->c_ivin, 1801416d8220SZaibo Xu ctx->c_ctx.ivsize); 18022f072d75SZaibo Xu else 18032514f559SLongfang Liu memcpy(req->aead_req.aead_req->iv, c_req->c_ivin, 18042f072d75SZaibo Xu ctx->c_ctx.ivsize); 18052f072d75SZaibo Xu } 1806416d8220SZaibo Xu 1807416d8220SZaibo Xu sec_request_untransfer(ctx, req); 1808416d8220SZaibo Xu err_uninit_req: 1809416d8220SZaibo Xu sec_request_uninit(ctx, req); 1810416d8220SZaibo Xu return ret; 1811416d8220SZaibo Xu } 1812416d8220SZaibo Xu 1813a181647cSZaibo Xu static const struct sec_req_op sec_skcipher_req_ops = { 1814416d8220SZaibo Xu .buf_map = sec_skcipher_sgl_map, 1815416d8220SZaibo Xu .buf_unmap = sec_skcipher_sgl_unmap, 1816416d8220SZaibo Xu .do_transfer = sec_skcipher_copy_iv, 1817416d8220SZaibo Xu .bd_fill = sec_skcipher_bd_fill, 1818416d8220SZaibo Xu .bd_send = sec_bd_send, 1819416d8220SZaibo Xu .callback = sec_skcipher_callback, 1820416d8220SZaibo Xu .process = sec_process, 1821416d8220SZaibo Xu }; 1822416d8220SZaibo Xu 18232f072d75SZaibo Xu static const struct sec_req_op sec_aead_req_ops = { 18242f072d75SZaibo Xu .buf_map = sec_aead_sgl_map, 18252f072d75SZaibo Xu .buf_unmap = sec_aead_sgl_unmap, 1826c16a70c1SKai Ye .do_transfer = sec_aead_set_iv, 18272f072d75SZaibo Xu .bd_fill = sec_aead_bd_fill, 18282f072d75SZaibo Xu .bd_send = sec_bd_send, 18292f072d75SZaibo Xu .callback = sec_aead_callback, 18302f072d75SZaibo Xu .process = sec_process, 18312f072d75SZaibo Xu }; 18322f072d75SZaibo Xu 1833adc3f65aSKai Ye static const struct sec_req_op sec_skcipher_req_ops_v3 = { 1834adc3f65aSKai Ye .buf_map = sec_skcipher_sgl_map, 1835adc3f65aSKai Ye .buf_unmap = sec_skcipher_sgl_unmap, 1836adc3f65aSKai Ye .do_transfer = sec_skcipher_copy_iv, 1837adc3f65aSKai Ye .bd_fill = sec_skcipher_bd_fill_v3, 1838adc3f65aSKai Ye .bd_send = sec_bd_send, 1839adc3f65aSKai Ye .callback = sec_skcipher_callback, 1840adc3f65aSKai Ye .process = sec_process, 1841adc3f65aSKai Ye }; 1842adc3f65aSKai Ye 1843adc3f65aSKai Ye static const struct sec_req_op sec_aead_req_ops_v3 = { 1844adc3f65aSKai Ye .buf_map = sec_aead_sgl_map, 1845adc3f65aSKai Ye .buf_unmap = sec_aead_sgl_unmap, 1846c16a70c1SKai Ye .do_transfer = sec_aead_set_iv, 1847adc3f65aSKai Ye .bd_fill = sec_aead_bd_fill_v3, 1848adc3f65aSKai Ye .bd_send = sec_bd_send, 1849adc3f65aSKai Ye .callback = sec_aead_callback, 1850adc3f65aSKai Ye .process = sec_process, 1851adc3f65aSKai Ye }; 1852adc3f65aSKai Ye 1853416d8220SZaibo Xu static int sec_skcipher_ctx_init(struct crypto_skcipher *tfm) 1854416d8220SZaibo Xu { 1855416d8220SZaibo Xu struct sec_ctx *ctx = crypto_skcipher_ctx(tfm); 1856adc3f65aSKai Ye int ret; 1857416d8220SZaibo Xu 1858adc3f65aSKai Ye ret = sec_skcipher_init(tfm); 1859adc3f65aSKai Ye if (ret) 1860adc3f65aSKai Ye return ret; 1861adc3f65aSKai Ye 1862adc3f65aSKai Ye if (ctx->sec->qm.ver < QM_HW_V3) { 1863adc3f65aSKai Ye ctx->type_supported = SEC_BD_TYPE2; 1864a181647cSZaibo Xu ctx->req_op = &sec_skcipher_req_ops; 1865adc3f65aSKai Ye } else { 1866adc3f65aSKai Ye ctx->type_supported = SEC_BD_TYPE3; 1867adc3f65aSKai Ye ctx->req_op = &sec_skcipher_req_ops_v3; 1868adc3f65aSKai Ye } 1869416d8220SZaibo Xu 1870adc3f65aSKai Ye return ret; 1871416d8220SZaibo Xu } 1872416d8220SZaibo Xu 1873416d8220SZaibo Xu static void sec_skcipher_ctx_exit(struct crypto_skcipher *tfm) 1874416d8220SZaibo Xu { 1875a181647cSZaibo Xu sec_skcipher_uninit(tfm); 1876416d8220SZaibo Xu } 1877416d8220SZaibo Xu 18782f072d75SZaibo Xu static int sec_aead_init(struct crypto_aead *tfm) 18792f072d75SZaibo Xu { 18802f072d75SZaibo Xu struct sec_ctx *ctx = crypto_aead_ctx(tfm); 18812f072d75SZaibo Xu int ret; 18822f072d75SZaibo Xu 18832f072d75SZaibo Xu crypto_aead_set_reqsize(tfm, sizeof(struct sec_req)); 18842f072d75SZaibo Xu ctx->alg_type = SEC_AEAD; 18852f072d75SZaibo Xu ctx->c_ctx.ivsize = crypto_aead_ivsize(tfm); 1886c16a70c1SKai Ye if (ctx->c_ctx.ivsize < SEC_AIV_SIZE || 1887c16a70c1SKai Ye ctx->c_ctx.ivsize > SEC_IV_SIZE) { 1888c16a70c1SKai Ye pr_err("get error aead iv size!\n"); 18892f072d75SZaibo Xu return -EINVAL; 18902f072d75SZaibo Xu } 18912f072d75SZaibo Xu 18922f072d75SZaibo Xu ret = sec_ctx_base_init(ctx); 18932f072d75SZaibo Xu if (ret) 18942f072d75SZaibo Xu return ret; 1895adc3f65aSKai Ye if (ctx->sec->qm.ver < QM_HW_V3) { 1896adc3f65aSKai Ye ctx->type_supported = SEC_BD_TYPE2; 1897adc3f65aSKai Ye ctx->req_op = &sec_aead_req_ops; 1898adc3f65aSKai Ye } else { 1899adc3f65aSKai Ye ctx->type_supported = SEC_BD_TYPE3; 1900adc3f65aSKai Ye ctx->req_op = &sec_aead_req_ops_v3; 1901adc3f65aSKai Ye } 19022f072d75SZaibo Xu 19032f072d75SZaibo Xu ret = sec_auth_init(ctx); 19042f072d75SZaibo Xu if (ret) 19052f072d75SZaibo Xu goto err_auth_init; 19062f072d75SZaibo Xu 19072f072d75SZaibo Xu ret = sec_cipher_init(ctx); 19082f072d75SZaibo Xu if (ret) 19092f072d75SZaibo Xu goto err_cipher_init; 19102f072d75SZaibo Xu 19112f072d75SZaibo Xu return ret; 19122f072d75SZaibo Xu 19132f072d75SZaibo Xu err_cipher_init: 19142f072d75SZaibo Xu sec_auth_uninit(ctx); 19152f072d75SZaibo Xu err_auth_init: 19162f072d75SZaibo Xu sec_ctx_base_uninit(ctx); 19172f072d75SZaibo Xu return ret; 19182f072d75SZaibo Xu } 19192f072d75SZaibo Xu 19202f072d75SZaibo Xu static void sec_aead_exit(struct crypto_aead *tfm) 19212f072d75SZaibo Xu { 19222f072d75SZaibo Xu struct sec_ctx *ctx = crypto_aead_ctx(tfm); 19232f072d75SZaibo Xu 19242f072d75SZaibo Xu sec_cipher_uninit(ctx); 19252f072d75SZaibo Xu sec_auth_uninit(ctx); 19262f072d75SZaibo Xu sec_ctx_base_uninit(ctx); 19272f072d75SZaibo Xu } 19282f072d75SZaibo Xu 19292f072d75SZaibo Xu static int sec_aead_ctx_init(struct crypto_aead *tfm, const char *hash_name) 19302f072d75SZaibo Xu { 19312f072d75SZaibo Xu struct sec_ctx *ctx = crypto_aead_ctx(tfm); 19322f072d75SZaibo Xu struct sec_auth_ctx *auth_ctx = &ctx->a_ctx; 19332f072d75SZaibo Xu int ret; 19342f072d75SZaibo Xu 19352f072d75SZaibo Xu ret = sec_aead_init(tfm); 19362f072d75SZaibo Xu if (ret) { 19372f072d75SZaibo Xu pr_err("hisi_sec2: aead init error!\n"); 19382f072d75SZaibo Xu return ret; 19392f072d75SZaibo Xu } 19402f072d75SZaibo Xu 19412f072d75SZaibo Xu auth_ctx->hash_tfm = crypto_alloc_shash(hash_name, 0, 0); 19422f072d75SZaibo Xu if (IS_ERR(auth_ctx->hash_tfm)) { 1943a44dce50SLongfang Liu dev_err(ctx->dev, "aead alloc shash error!\n"); 19442f072d75SZaibo Xu sec_aead_exit(tfm); 19452f072d75SZaibo Xu return PTR_ERR(auth_ctx->hash_tfm); 19462f072d75SZaibo Xu } 19472f072d75SZaibo Xu 19482f072d75SZaibo Xu return 0; 19492f072d75SZaibo Xu } 19502f072d75SZaibo Xu 19512f072d75SZaibo Xu static void sec_aead_ctx_exit(struct crypto_aead *tfm) 19522f072d75SZaibo Xu { 19532f072d75SZaibo Xu struct sec_ctx *ctx = crypto_aead_ctx(tfm); 19542f072d75SZaibo Xu 19552f072d75SZaibo Xu crypto_free_shash(ctx->a_ctx.hash_tfm); 19562f072d75SZaibo Xu sec_aead_exit(tfm); 19572f072d75SZaibo Xu } 19582f072d75SZaibo Xu 1959c16a70c1SKai Ye static int sec_aead_xcm_ctx_init(struct crypto_aead *tfm) 1960c16a70c1SKai Ye { 19616c46a329SKai Ye struct aead_alg *alg = crypto_aead_alg(tfm); 1962c16a70c1SKai Ye struct sec_ctx *ctx = crypto_aead_ctx(tfm); 19636c46a329SKai Ye struct sec_auth_ctx *a_ctx = &ctx->a_ctx; 19646c46a329SKai Ye const char *aead_name = alg->base.cra_name; 1965c16a70c1SKai Ye int ret; 1966c16a70c1SKai Ye 1967c16a70c1SKai Ye ret = sec_aead_init(tfm); 1968c16a70c1SKai Ye if (ret) { 1969c16a70c1SKai Ye dev_err(ctx->dev, "hisi_sec2: aead xcm init error!\n"); 1970c16a70c1SKai Ye return ret; 1971c16a70c1SKai Ye } 1972c16a70c1SKai Ye 19736c46a329SKai Ye a_ctx->fallback_aead_tfm = crypto_alloc_aead(aead_name, 0, 19746c46a329SKai Ye CRYPTO_ALG_NEED_FALLBACK | 19756c46a329SKai Ye CRYPTO_ALG_ASYNC); 19766c46a329SKai Ye if (IS_ERR(a_ctx->fallback_aead_tfm)) { 19776c46a329SKai Ye dev_err(ctx->dev, "aead driver alloc fallback tfm error!\n"); 19786c46a329SKai Ye sec_aead_exit(tfm); 19796c46a329SKai Ye return PTR_ERR(a_ctx->fallback_aead_tfm); 19806c46a329SKai Ye } 19816c46a329SKai Ye a_ctx->fallback = false; 19826c46a329SKai Ye 1983c16a70c1SKai Ye return 0; 1984c16a70c1SKai Ye } 1985c16a70c1SKai Ye 1986c16a70c1SKai Ye static void sec_aead_xcm_ctx_exit(struct crypto_aead *tfm) 1987c16a70c1SKai Ye { 19886c46a329SKai Ye struct sec_ctx *ctx = crypto_aead_ctx(tfm); 19896c46a329SKai Ye 19906c46a329SKai Ye crypto_free_aead(ctx->a_ctx.fallback_aead_tfm); 1991c16a70c1SKai Ye sec_aead_exit(tfm); 1992c16a70c1SKai Ye } 1993c16a70c1SKai Ye 19942f072d75SZaibo Xu static int sec_aead_sha1_ctx_init(struct crypto_aead *tfm) 19952f072d75SZaibo Xu { 19962f072d75SZaibo Xu return sec_aead_ctx_init(tfm, "sha1"); 19972f072d75SZaibo Xu } 19982f072d75SZaibo Xu 19992f072d75SZaibo Xu static int sec_aead_sha256_ctx_init(struct crypto_aead *tfm) 20002f072d75SZaibo Xu { 20012f072d75SZaibo Xu return sec_aead_ctx_init(tfm, "sha256"); 20022f072d75SZaibo Xu } 20032f072d75SZaibo Xu 20042f072d75SZaibo Xu static int sec_aead_sha512_ctx_init(struct crypto_aead *tfm) 20052f072d75SZaibo Xu { 20062f072d75SZaibo Xu return sec_aead_ctx_init(tfm, "sha512"); 20072f072d75SZaibo Xu } 20082f072d75SZaibo Xu 20092132d4efSKai Ye static int sec_skcipher_cryptlen_check(struct sec_ctx *ctx, 2010059c5342SLongfang Liu struct sec_req *sreq) 2011059c5342SLongfang Liu { 2012059c5342SLongfang Liu u32 cryptlen = sreq->c_req.sk_req->cryptlen; 2013059c5342SLongfang Liu struct device *dev = ctx->dev; 2014059c5342SLongfang Liu u8 c_mode = ctx->c_ctx.c_mode; 2015059c5342SLongfang Liu int ret = 0; 2016059c5342SLongfang Liu 2017059c5342SLongfang Liu switch (c_mode) { 2018059c5342SLongfang Liu case SEC_CMODE_XTS: 2019059c5342SLongfang Liu if (unlikely(cryptlen < AES_BLOCK_SIZE)) { 2020059c5342SLongfang Liu dev_err(dev, "skcipher XTS mode input length error!\n"); 2021059c5342SLongfang Liu ret = -EINVAL; 2022059c5342SLongfang Liu } 2023059c5342SLongfang Liu break; 2024059c5342SLongfang Liu case SEC_CMODE_ECB: 2025059c5342SLongfang Liu case SEC_CMODE_CBC: 2026059c5342SLongfang Liu if (unlikely(cryptlen & (AES_BLOCK_SIZE - 1))) { 2027059c5342SLongfang Liu dev_err(dev, "skcipher AES input length error!\n"); 2028059c5342SLongfang Liu ret = -EINVAL; 2029059c5342SLongfang Liu } 2030059c5342SLongfang Liu break; 20317b44c0eeSKai Ye case SEC_CMODE_CFB: 20327b44c0eeSKai Ye case SEC_CMODE_OFB: 20337b44c0eeSKai Ye case SEC_CMODE_CTR: 20347b44c0eeSKai Ye if (unlikely(ctx->sec->qm.ver < QM_HW_V3)) { 20357b44c0eeSKai Ye dev_err(dev, "skcipher HW version error!\n"); 20367b44c0eeSKai Ye ret = -EINVAL; 20377b44c0eeSKai Ye } 20387b44c0eeSKai Ye break; 2039059c5342SLongfang Liu default: 2040059c5342SLongfang Liu ret = -EINVAL; 2041059c5342SLongfang Liu } 2042059c5342SLongfang Liu 2043059c5342SLongfang Liu return ret; 2044059c5342SLongfang Liu } 2045059c5342SLongfang Liu 2046d6de2a59SZaibo Xu static int sec_skcipher_param_check(struct sec_ctx *ctx, struct sec_req *sreq) 2047416d8220SZaibo Xu { 2048d6de2a59SZaibo Xu struct skcipher_request *sk_req = sreq->c_req.sk_req; 2049a44dce50SLongfang Liu struct device *dev = ctx->dev; 2050d6de2a59SZaibo Xu u8 c_alg = ctx->c_ctx.c_alg; 2051416d8220SZaibo Xu 20527b44c0eeSKai Ye if (unlikely(!sk_req->src || !sk_req->dst || 20537b44c0eeSKai Ye sk_req->cryptlen > MAX_INPUT_DATA_LEN)) { 2054416d8220SZaibo Xu dev_err(dev, "skcipher input param error!\n"); 2055416d8220SZaibo Xu return -EINVAL; 2056416d8220SZaibo Xu } 2057d6de2a59SZaibo Xu sreq->c_req.c_len = sk_req->cryptlen; 205874b58db8SLongfang Liu 205974b58db8SLongfang Liu if (ctx->pbuf_supported && sk_req->cryptlen <= SEC_PBUF_SZ) 206074b58db8SLongfang Liu sreq->use_pbuf = true; 206174b58db8SLongfang Liu else 206274b58db8SLongfang Liu sreq->use_pbuf = false; 206374b58db8SLongfang Liu 2064416d8220SZaibo Xu if (c_alg == SEC_CALG_3DES) { 2065b9c8d897SZaibo Xu if (unlikely(sk_req->cryptlen & (DES3_EDE_BLOCK_SIZE - 1))) { 2066416d8220SZaibo Xu dev_err(dev, "skcipher 3des input length error!\n"); 2067416d8220SZaibo Xu return -EINVAL; 2068416d8220SZaibo Xu } 2069416d8220SZaibo Xu return 0; 2070416d8220SZaibo Xu } else if (c_alg == SEC_CALG_AES || c_alg == SEC_CALG_SM4) { 20712132d4efSKai Ye return sec_skcipher_cryptlen_check(ctx, sreq); 2072416d8220SZaibo Xu } 2073059c5342SLongfang Liu 2074416d8220SZaibo Xu dev_err(dev, "skcipher algorithm error!\n"); 2075633e507fSLongfang Liu 2076416d8220SZaibo Xu return -EINVAL; 2077416d8220SZaibo Xu } 2078416d8220SZaibo Xu 20795652d55aSKai Ye static int sec_skcipher_soft_crypto(struct sec_ctx *ctx, 20805652d55aSKai Ye struct skcipher_request *sreq, bool encrypt) 20815652d55aSKai Ye { 20825652d55aSKai Ye struct sec_cipher_ctx *c_ctx = &ctx->c_ctx; 2083e764d81dSKai Ye SYNC_SKCIPHER_REQUEST_ON_STACK(subreq, c_ctx->fbtfm); 20845652d55aSKai Ye struct device *dev = ctx->dev; 20855652d55aSKai Ye int ret; 20865652d55aSKai Ye 20875652d55aSKai Ye if (!c_ctx->fbtfm) { 2088e764d81dSKai Ye dev_err_ratelimited(dev, "the soft tfm isn't supported in the current system.\n"); 20895652d55aSKai Ye return -EINVAL; 20905652d55aSKai Ye } 20915652d55aSKai Ye 20925652d55aSKai Ye skcipher_request_set_sync_tfm(subreq, c_ctx->fbtfm); 20935652d55aSKai Ye 20945652d55aSKai Ye /* software need sync mode to do crypto */ 20955652d55aSKai Ye skcipher_request_set_callback(subreq, sreq->base.flags, 20965652d55aSKai Ye NULL, NULL); 20975652d55aSKai Ye skcipher_request_set_crypt(subreq, sreq->src, sreq->dst, 20985652d55aSKai Ye sreq->cryptlen, sreq->iv); 20995652d55aSKai Ye if (encrypt) 21005652d55aSKai Ye ret = crypto_skcipher_encrypt(subreq); 21015652d55aSKai Ye else 21025652d55aSKai Ye ret = crypto_skcipher_decrypt(subreq); 21035652d55aSKai Ye 21045652d55aSKai Ye skcipher_request_zero(subreq); 21055652d55aSKai Ye 21065652d55aSKai Ye return ret; 21075652d55aSKai Ye } 21085652d55aSKai Ye 2109416d8220SZaibo Xu static int sec_skcipher_crypto(struct skcipher_request *sk_req, bool encrypt) 2110416d8220SZaibo Xu { 2111416d8220SZaibo Xu struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(sk_req); 2112416d8220SZaibo Xu struct sec_req *req = skcipher_request_ctx(sk_req); 2113416d8220SZaibo Xu struct sec_ctx *ctx = crypto_skcipher_ctx(tfm); 2114416d8220SZaibo Xu int ret; 2115416d8220SZaibo Xu 21165652d55aSKai Ye if (!sk_req->cryptlen) { 21175652d55aSKai Ye if (ctx->c_ctx.c_mode == SEC_CMODE_XTS) 21185652d55aSKai Ye return -EINVAL; 2119416d8220SZaibo Xu return 0; 21205652d55aSKai Ye } 2121416d8220SZaibo Xu 21229597efc3SKai Ye req->flag = sk_req->base.flags; 2123416d8220SZaibo Xu req->c_req.sk_req = sk_req; 2124416d8220SZaibo Xu req->c_req.encrypt = encrypt; 2125416d8220SZaibo Xu req->ctx = ctx; 2126416d8220SZaibo Xu 2127d6de2a59SZaibo Xu ret = sec_skcipher_param_check(ctx, req); 2128d6de2a59SZaibo Xu if (unlikely(ret)) 2129d6de2a59SZaibo Xu return -EINVAL; 2130d6de2a59SZaibo Xu 21315652d55aSKai Ye if (unlikely(ctx->c_ctx.fallback)) 21325652d55aSKai Ye return sec_skcipher_soft_crypto(ctx, sk_req, encrypt); 21335652d55aSKai Ye 2134416d8220SZaibo Xu return ctx->req_op->process(ctx, req); 2135416d8220SZaibo Xu } 2136416d8220SZaibo Xu 2137416d8220SZaibo Xu static int sec_skcipher_encrypt(struct skcipher_request *sk_req) 2138416d8220SZaibo Xu { 2139416d8220SZaibo Xu return sec_skcipher_crypto(sk_req, true); 2140416d8220SZaibo Xu } 2141416d8220SZaibo Xu 2142416d8220SZaibo Xu static int sec_skcipher_decrypt(struct skcipher_request *sk_req) 2143416d8220SZaibo Xu { 2144416d8220SZaibo Xu return sec_skcipher_crypto(sk_req, false); 2145416d8220SZaibo Xu } 2146416d8220SZaibo Xu 2147416d8220SZaibo Xu #define SEC_SKCIPHER_GEN_ALG(sec_cra_name, sec_set_key, sec_min_key_size, \ 2148416d8220SZaibo Xu sec_max_key_size, ctx_init, ctx_exit, blk_size, iv_size)\ 2149416d8220SZaibo Xu {\ 2150416d8220SZaibo Xu .base = {\ 2151416d8220SZaibo Xu .cra_name = sec_cra_name,\ 2152416d8220SZaibo Xu .cra_driver_name = "hisi_sec_"sec_cra_name,\ 2153416d8220SZaibo Xu .cra_priority = SEC_PRIORITY,\ 21545652d55aSKai Ye .cra_flags = CRYPTO_ALG_ASYNC |\ 21555652d55aSKai Ye CRYPTO_ALG_NEED_FALLBACK,\ 2156416d8220SZaibo Xu .cra_blocksize = blk_size,\ 2157416d8220SZaibo Xu .cra_ctxsize = sizeof(struct sec_ctx),\ 2158416d8220SZaibo Xu .cra_module = THIS_MODULE,\ 2159416d8220SZaibo Xu },\ 2160416d8220SZaibo Xu .init = ctx_init,\ 2161416d8220SZaibo Xu .exit = ctx_exit,\ 2162416d8220SZaibo Xu .setkey = sec_set_key,\ 2163416d8220SZaibo Xu .decrypt = sec_skcipher_decrypt,\ 2164416d8220SZaibo Xu .encrypt = sec_skcipher_encrypt,\ 2165416d8220SZaibo Xu .min_keysize = sec_min_key_size,\ 2166416d8220SZaibo Xu .max_keysize = sec_max_key_size,\ 2167416d8220SZaibo Xu .ivsize = iv_size,\ 2168921715b6SWenkai Lin } 2169416d8220SZaibo Xu 2170416d8220SZaibo Xu #define SEC_SKCIPHER_ALG(name, key_func, min_key_size, \ 2171416d8220SZaibo Xu max_key_size, blk_size, iv_size) \ 2172416d8220SZaibo Xu SEC_SKCIPHER_GEN_ALG(name, key_func, min_key_size, max_key_size, \ 2173416d8220SZaibo Xu sec_skcipher_ctx_init, sec_skcipher_ctx_exit, blk_size, iv_size) 2174416d8220SZaibo Xu 2175921715b6SWenkai Lin static struct sec_skcipher sec_skciphers[] = { 2176921715b6SWenkai Lin { 2177921715b6SWenkai Lin .alg_msk = BIT(0), 2178921715b6SWenkai Lin .alg = SEC_SKCIPHER_ALG("ecb(aes)", sec_setkey_aes_ecb, AES_MIN_KEY_SIZE, 2179921715b6SWenkai Lin AES_MAX_KEY_SIZE, AES_BLOCK_SIZE, 0), 2180921715b6SWenkai Lin }, 2181921715b6SWenkai Lin { 2182921715b6SWenkai Lin .alg_msk = BIT(1), 2183921715b6SWenkai Lin .alg = SEC_SKCIPHER_ALG("cbc(aes)", sec_setkey_aes_cbc, AES_MIN_KEY_SIZE, 2184921715b6SWenkai Lin AES_MAX_KEY_SIZE, AES_BLOCK_SIZE, AES_BLOCK_SIZE), 2185921715b6SWenkai Lin }, 2186921715b6SWenkai Lin { 2187921715b6SWenkai Lin .alg_msk = BIT(2), 2188921715b6SWenkai Lin .alg = SEC_SKCIPHER_ALG("ctr(aes)", sec_setkey_aes_ctr, AES_MIN_KEY_SIZE, 2189921715b6SWenkai Lin AES_MAX_KEY_SIZE, SEC_MIN_BLOCK_SZ, AES_BLOCK_SIZE), 2190921715b6SWenkai Lin }, 2191921715b6SWenkai Lin { 2192921715b6SWenkai Lin .alg_msk = BIT(3), 2193921715b6SWenkai Lin .alg = SEC_SKCIPHER_ALG("xts(aes)", sec_setkey_aes_xts, SEC_XTS_MIN_KEY_SIZE, 2194921715b6SWenkai Lin SEC_XTS_MAX_KEY_SIZE, AES_BLOCK_SIZE, AES_BLOCK_SIZE), 2195921715b6SWenkai Lin }, 2196921715b6SWenkai Lin { 2197921715b6SWenkai Lin .alg_msk = BIT(4), 2198921715b6SWenkai Lin .alg = SEC_SKCIPHER_ALG("ofb(aes)", sec_setkey_aes_ofb, AES_MIN_KEY_SIZE, 2199921715b6SWenkai Lin AES_MAX_KEY_SIZE, SEC_MIN_BLOCK_SZ, AES_BLOCK_SIZE), 2200921715b6SWenkai Lin }, 2201921715b6SWenkai Lin { 2202921715b6SWenkai Lin .alg_msk = BIT(5), 2203921715b6SWenkai Lin .alg = SEC_SKCIPHER_ALG("cfb(aes)", sec_setkey_aes_cfb, AES_MIN_KEY_SIZE, 2204921715b6SWenkai Lin AES_MAX_KEY_SIZE, SEC_MIN_BLOCK_SZ, AES_BLOCK_SIZE), 2205921715b6SWenkai Lin }, 2206921715b6SWenkai Lin { 2207921715b6SWenkai Lin .alg_msk = BIT(12), 2208921715b6SWenkai Lin .alg = SEC_SKCIPHER_ALG("cbc(sm4)", sec_setkey_sm4_cbc, AES_MIN_KEY_SIZE, 2209921715b6SWenkai Lin AES_MIN_KEY_SIZE, AES_BLOCK_SIZE, AES_BLOCK_SIZE), 2210921715b6SWenkai Lin }, 2211921715b6SWenkai Lin { 2212921715b6SWenkai Lin .alg_msk = BIT(13), 2213921715b6SWenkai Lin .alg = SEC_SKCIPHER_ALG("ctr(sm4)", sec_setkey_sm4_ctr, AES_MIN_KEY_SIZE, 2214921715b6SWenkai Lin AES_MIN_KEY_SIZE, SEC_MIN_BLOCK_SZ, AES_BLOCK_SIZE), 2215921715b6SWenkai Lin }, 2216921715b6SWenkai Lin { 2217921715b6SWenkai Lin .alg_msk = BIT(14), 2218921715b6SWenkai Lin .alg = SEC_SKCIPHER_ALG("xts(sm4)", sec_setkey_sm4_xts, SEC_XTS_MIN_KEY_SIZE, 2219921715b6SWenkai Lin SEC_XTS_MIN_KEY_SIZE, AES_BLOCK_SIZE, AES_BLOCK_SIZE), 2220921715b6SWenkai Lin }, 2221921715b6SWenkai Lin { 2222921715b6SWenkai Lin .alg_msk = BIT(15), 2223921715b6SWenkai Lin .alg = SEC_SKCIPHER_ALG("ofb(sm4)", sec_setkey_sm4_ofb, AES_MIN_KEY_SIZE, 2224921715b6SWenkai Lin AES_MIN_KEY_SIZE, SEC_MIN_BLOCK_SZ, AES_BLOCK_SIZE), 2225921715b6SWenkai Lin }, 2226921715b6SWenkai Lin { 2227921715b6SWenkai Lin .alg_msk = BIT(16), 2228921715b6SWenkai Lin .alg = SEC_SKCIPHER_ALG("cfb(sm4)", sec_setkey_sm4_cfb, AES_MIN_KEY_SIZE, 2229921715b6SWenkai Lin AES_MIN_KEY_SIZE, SEC_MIN_BLOCK_SZ, AES_BLOCK_SIZE), 2230921715b6SWenkai Lin }, 2231921715b6SWenkai Lin { 2232921715b6SWenkai Lin .alg_msk = BIT(23), 2233921715b6SWenkai Lin .alg = SEC_SKCIPHER_ALG("ecb(des3_ede)", sec_setkey_3des_ecb, SEC_DES3_3KEY_SIZE, 2234921715b6SWenkai Lin SEC_DES3_3KEY_SIZE, DES3_EDE_BLOCK_SIZE, 0), 2235921715b6SWenkai Lin }, 2236921715b6SWenkai Lin { 2237921715b6SWenkai Lin .alg_msk = BIT(24), 2238921715b6SWenkai Lin .alg = SEC_SKCIPHER_ALG("cbc(des3_ede)", sec_setkey_3des_cbc, SEC_DES3_3KEY_SIZE, 2239921715b6SWenkai Lin SEC_DES3_3KEY_SIZE, DES3_EDE_BLOCK_SIZE, 2240921715b6SWenkai Lin DES3_EDE_BLOCK_SIZE), 2241921715b6SWenkai Lin }, 22427b44c0eeSKai Ye }; 22437b44c0eeSKai Ye 2244c16a70c1SKai Ye static int aead_iv_demension_check(struct aead_request *aead_req) 2245c16a70c1SKai Ye { 2246c16a70c1SKai Ye u8 cl; 2247c16a70c1SKai Ye 2248c16a70c1SKai Ye cl = aead_req->iv[0] + 1; 2249c16a70c1SKai Ye if (cl < IV_CL_MIN || cl > IV_CL_MAX) 2250c16a70c1SKai Ye return -EINVAL; 2251c16a70c1SKai Ye 2252c16a70c1SKai Ye if (cl < IV_CL_MID && aead_req->cryptlen >> (BYTE_BITS * cl)) 2253c16a70c1SKai Ye return -EOVERFLOW; 2254c16a70c1SKai Ye 2255c16a70c1SKai Ye return 0; 2256c16a70c1SKai Ye } 2257c16a70c1SKai Ye 2258c16a70c1SKai Ye static int sec_aead_spec_check(struct sec_ctx *ctx, struct sec_req *sreq) 2259c16a70c1SKai Ye { 2260c16a70c1SKai Ye struct aead_request *req = sreq->aead_req.aead_req; 2261c16a70c1SKai Ye struct crypto_aead *tfm = crypto_aead_reqtfm(req); 2262c16a70c1SKai Ye size_t authsize = crypto_aead_authsize(tfm); 2263c16a70c1SKai Ye u8 c_mode = ctx->c_ctx.c_mode; 2264c16a70c1SKai Ye struct device *dev = ctx->dev; 2265c16a70c1SKai Ye int ret; 2266c16a70c1SKai Ye 2267c16a70c1SKai Ye if (unlikely(req->cryptlen + req->assoclen > MAX_INPUT_DATA_LEN || 2268c16a70c1SKai Ye req->assoclen > SEC_MAX_AAD_LEN)) { 2269c16a70c1SKai Ye dev_err(dev, "aead input spec error!\n"); 2270c16a70c1SKai Ye return -EINVAL; 2271c16a70c1SKai Ye } 2272c16a70c1SKai Ye 2273c16a70c1SKai Ye if (unlikely((c_mode == SEC_CMODE_GCM && authsize < DES_BLOCK_SIZE) || 2274c16a70c1SKai Ye (c_mode == SEC_CMODE_CCM && (authsize < MIN_MAC_LEN || 2275c16a70c1SKai Ye authsize & MAC_LEN_MASK)))) { 2276c16a70c1SKai Ye dev_err(dev, "aead input mac length error!\n"); 2277c16a70c1SKai Ye return -EINVAL; 2278c16a70c1SKai Ye } 2279c16a70c1SKai Ye 2280c16a70c1SKai Ye if (c_mode == SEC_CMODE_CCM) { 22815e340558SKai Ye if (unlikely(req->assoclen > SEC_MAX_CCM_AAD_LEN)) { 22825e340558SKai Ye dev_err_ratelimited(dev, "CCM input aad parameter is too long!\n"); 22835e340558SKai Ye return -EINVAL; 22845e340558SKai Ye } 2285c16a70c1SKai Ye ret = aead_iv_demension_check(req); 2286c16a70c1SKai Ye if (ret) { 2287c16a70c1SKai Ye dev_err(dev, "aead input iv param error!\n"); 2288c16a70c1SKai Ye return ret; 2289c16a70c1SKai Ye } 2290c16a70c1SKai Ye } 2291c16a70c1SKai Ye 2292c16a70c1SKai Ye if (sreq->c_req.encrypt) 2293c16a70c1SKai Ye sreq->c_req.c_len = req->cryptlen; 2294c16a70c1SKai Ye else 2295c16a70c1SKai Ye sreq->c_req.c_len = req->cryptlen - authsize; 2296c16a70c1SKai Ye if (c_mode == SEC_CMODE_CBC) { 2297c16a70c1SKai Ye if (unlikely(sreq->c_req.c_len & (AES_BLOCK_SIZE - 1))) { 2298c16a70c1SKai Ye dev_err(dev, "aead crypto length error!\n"); 2299c16a70c1SKai Ye return -EINVAL; 2300c16a70c1SKai Ye } 2301c16a70c1SKai Ye } 2302c16a70c1SKai Ye 2303c16a70c1SKai Ye return 0; 2304c16a70c1SKai Ye } 2305c16a70c1SKai Ye 23062f072d75SZaibo Xu static int sec_aead_param_check(struct sec_ctx *ctx, struct sec_req *sreq) 23072f072d75SZaibo Xu { 23082f072d75SZaibo Xu struct aead_request *req = sreq->aead_req.aead_req; 23092f072d75SZaibo Xu struct crypto_aead *tfm = crypto_aead_reqtfm(req); 23102f072d75SZaibo Xu size_t authsize = crypto_aead_authsize(tfm); 2311a44dce50SLongfang Liu struct device *dev = ctx->dev; 2312a44dce50SLongfang Liu u8 c_alg = ctx->c_ctx.c_alg; 23132f072d75SZaibo Xu 2314c16a70c1SKai Ye if (unlikely(!req->src || !req->dst)) { 2315a44dce50SLongfang Liu dev_err(dev, "aead input param error!\n"); 23162f072d75SZaibo Xu return -EINVAL; 23172f072d75SZaibo Xu } 23182f072d75SZaibo Xu 2319c16a70c1SKai Ye if (ctx->sec->qm.ver == QM_HW_V2) { 2320c16a70c1SKai Ye if (unlikely(!req->cryptlen || (!sreq->c_req.encrypt && 2321c16a70c1SKai Ye req->cryptlen <= authsize))) { 23226c46a329SKai Ye ctx->a_ctx.fallback = true; 2323c16a70c1SKai Ye return -EINVAL; 2324c16a70c1SKai Ye } 2325c16a70c1SKai Ye } 2326c16a70c1SKai Ye 2327c16a70c1SKai Ye /* Support AES or SM4 */ 2328c16a70c1SKai Ye if (unlikely(c_alg != SEC_CALG_AES && c_alg != SEC_CALG_SM4)) { 2329c16a70c1SKai Ye dev_err(dev, "aead crypto alg error!\n"); 2330c16a70c1SKai Ye return -EINVAL; 2331c16a70c1SKai Ye } 2332c16a70c1SKai Ye 2333c16a70c1SKai Ye if (unlikely(sec_aead_spec_check(ctx, sreq))) 2334c16a70c1SKai Ye return -EINVAL; 2335c16a70c1SKai Ye 233674b58db8SLongfang Liu if (ctx->pbuf_supported && (req->cryptlen + req->assoclen) <= 233774b58db8SLongfang Liu SEC_PBUF_SZ) 233874b58db8SLongfang Liu sreq->use_pbuf = true; 233974b58db8SLongfang Liu else 234074b58db8SLongfang Liu sreq->use_pbuf = false; 234174b58db8SLongfang Liu 23422f072d75SZaibo Xu return 0; 23432f072d75SZaibo Xu } 23442f072d75SZaibo Xu 23456c46a329SKai Ye static int sec_aead_soft_crypto(struct sec_ctx *ctx, 23466c46a329SKai Ye struct aead_request *aead_req, 23476c46a329SKai Ye bool encrypt) 23486c46a329SKai Ye { 23496c46a329SKai Ye struct sec_auth_ctx *a_ctx = &ctx->a_ctx; 23506c46a329SKai Ye struct device *dev = ctx->dev; 23510a2a464fSKai Ye struct aead_request *subreq; 23520a2a464fSKai Ye int ret; 23536c46a329SKai Ye 23546c46a329SKai Ye /* Kunpeng920 aead mode not support input 0 size */ 23556c46a329SKai Ye if (!a_ctx->fallback_aead_tfm) { 235684c2c729SColin Ian King dev_err(dev, "aead fallback tfm is NULL!\n"); 23576c46a329SKai Ye return -EINVAL; 23586c46a329SKai Ye } 23596c46a329SKai Ye 23600a2a464fSKai Ye subreq = aead_request_alloc(a_ctx->fallback_aead_tfm, GFP_KERNEL); 23610a2a464fSKai Ye if (!subreq) 23620a2a464fSKai Ye return -ENOMEM; 23630a2a464fSKai Ye 23646c46a329SKai Ye aead_request_set_tfm(subreq, a_ctx->fallback_aead_tfm); 23656c46a329SKai Ye aead_request_set_callback(subreq, aead_req->base.flags, 23666c46a329SKai Ye aead_req->base.complete, aead_req->base.data); 23676c46a329SKai Ye aead_request_set_crypt(subreq, aead_req->src, aead_req->dst, 23686c46a329SKai Ye aead_req->cryptlen, aead_req->iv); 23696c46a329SKai Ye aead_request_set_ad(subreq, aead_req->assoclen); 23706c46a329SKai Ye 23710a2a464fSKai Ye if (encrypt) 23720a2a464fSKai Ye ret = crypto_aead_encrypt(subreq); 23730a2a464fSKai Ye else 23740a2a464fSKai Ye ret = crypto_aead_decrypt(subreq); 23750a2a464fSKai Ye aead_request_free(subreq); 23760a2a464fSKai Ye 23770a2a464fSKai Ye return ret; 23786c46a329SKai Ye } 23796c46a329SKai Ye 23802f072d75SZaibo Xu static int sec_aead_crypto(struct aead_request *a_req, bool encrypt) 23812f072d75SZaibo Xu { 23822f072d75SZaibo Xu struct crypto_aead *tfm = crypto_aead_reqtfm(a_req); 23832f072d75SZaibo Xu struct sec_req *req = aead_request_ctx(a_req); 23842f072d75SZaibo Xu struct sec_ctx *ctx = crypto_aead_ctx(tfm); 23852f072d75SZaibo Xu int ret; 23862f072d75SZaibo Xu 23879597efc3SKai Ye req->flag = a_req->base.flags; 23882f072d75SZaibo Xu req->aead_req.aead_req = a_req; 23892f072d75SZaibo Xu req->c_req.encrypt = encrypt; 23902f072d75SZaibo Xu req->ctx = ctx; 23912f072d75SZaibo Xu 23922f072d75SZaibo Xu ret = sec_aead_param_check(ctx, req); 23936c46a329SKai Ye if (unlikely(ret)) { 23946c46a329SKai Ye if (ctx->a_ctx.fallback) 23956c46a329SKai Ye return sec_aead_soft_crypto(ctx, a_req, encrypt); 23962f072d75SZaibo Xu return -EINVAL; 23976c46a329SKai Ye } 23982f072d75SZaibo Xu 23992f072d75SZaibo Xu return ctx->req_op->process(ctx, req); 24002f072d75SZaibo Xu } 24012f072d75SZaibo Xu 24022f072d75SZaibo Xu static int sec_aead_encrypt(struct aead_request *a_req) 24032f072d75SZaibo Xu { 24042f072d75SZaibo Xu return sec_aead_crypto(a_req, true); 24052f072d75SZaibo Xu } 24062f072d75SZaibo Xu 24072f072d75SZaibo Xu static int sec_aead_decrypt(struct aead_request *a_req) 24082f072d75SZaibo Xu { 24092f072d75SZaibo Xu return sec_aead_crypto(a_req, false); 24102f072d75SZaibo Xu } 24112f072d75SZaibo Xu 2412c16a70c1SKai Ye #define SEC_AEAD_ALG(sec_cra_name, sec_set_key, ctx_init,\ 24132f072d75SZaibo Xu ctx_exit, blk_size, iv_size, max_authsize)\ 24142f072d75SZaibo Xu {\ 24152f072d75SZaibo Xu .base = {\ 24162f072d75SZaibo Xu .cra_name = sec_cra_name,\ 24172f072d75SZaibo Xu .cra_driver_name = "hisi_sec_"sec_cra_name,\ 24182f072d75SZaibo Xu .cra_priority = SEC_PRIORITY,\ 24196c46a329SKai Ye .cra_flags = CRYPTO_ALG_ASYNC |\ 24206c46a329SKai Ye CRYPTO_ALG_NEED_FALLBACK,\ 24212f072d75SZaibo Xu .cra_blocksize = blk_size,\ 24222f072d75SZaibo Xu .cra_ctxsize = sizeof(struct sec_ctx),\ 24232f072d75SZaibo Xu .cra_module = THIS_MODULE,\ 24242f072d75SZaibo Xu },\ 24252f072d75SZaibo Xu .init = ctx_init,\ 24262f072d75SZaibo Xu .exit = ctx_exit,\ 24272f072d75SZaibo Xu .setkey = sec_set_key,\ 24286c46a329SKai Ye .setauthsize = sec_aead_setauthsize,\ 24292f072d75SZaibo Xu .decrypt = sec_aead_decrypt,\ 24302f072d75SZaibo Xu .encrypt = sec_aead_encrypt,\ 24312f072d75SZaibo Xu .ivsize = iv_size,\ 24322f072d75SZaibo Xu .maxauthsize = max_authsize,\ 24332f072d75SZaibo Xu } 24342f072d75SZaibo Xu 2435921715b6SWenkai Lin static struct sec_aead sec_aeads[] = { 2436921715b6SWenkai Lin { 2437921715b6SWenkai Lin .alg_msk = BIT(6), 2438921715b6SWenkai Lin .alg = SEC_AEAD_ALG("ccm(aes)", sec_setkey_aes_ccm, sec_aead_xcm_ctx_init, 2439921715b6SWenkai Lin sec_aead_xcm_ctx_exit, SEC_MIN_BLOCK_SZ, AES_BLOCK_SIZE, 2440921715b6SWenkai Lin AES_BLOCK_SIZE), 2441921715b6SWenkai Lin }, 2442921715b6SWenkai Lin { 2443921715b6SWenkai Lin .alg_msk = BIT(7), 2444921715b6SWenkai Lin .alg = SEC_AEAD_ALG("gcm(aes)", sec_setkey_aes_gcm, sec_aead_xcm_ctx_init, 2445921715b6SWenkai Lin sec_aead_xcm_ctx_exit, SEC_MIN_BLOCK_SZ, SEC_AIV_SIZE, 2446921715b6SWenkai Lin AES_BLOCK_SIZE), 2447921715b6SWenkai Lin }, 2448921715b6SWenkai Lin { 2449921715b6SWenkai Lin .alg_msk = BIT(17), 2450921715b6SWenkai Lin .alg = SEC_AEAD_ALG("ccm(sm4)", sec_setkey_sm4_ccm, sec_aead_xcm_ctx_init, 2451921715b6SWenkai Lin sec_aead_xcm_ctx_exit, SEC_MIN_BLOCK_SZ, AES_BLOCK_SIZE, 2452921715b6SWenkai Lin AES_BLOCK_SIZE), 2453921715b6SWenkai Lin }, 2454921715b6SWenkai Lin { 2455921715b6SWenkai Lin .alg_msk = BIT(18), 2456921715b6SWenkai Lin .alg = SEC_AEAD_ALG("gcm(sm4)", sec_setkey_sm4_gcm, sec_aead_xcm_ctx_init, 2457921715b6SWenkai Lin sec_aead_xcm_ctx_exit, SEC_MIN_BLOCK_SZ, SEC_AIV_SIZE, 2458921715b6SWenkai Lin AES_BLOCK_SIZE), 2459921715b6SWenkai Lin }, 2460921715b6SWenkai Lin { 2461921715b6SWenkai Lin .alg_msk = BIT(43), 2462921715b6SWenkai Lin .alg = SEC_AEAD_ALG("authenc(hmac(sha1),cbc(aes))", sec_setkey_aes_cbc_sha1, 2463921715b6SWenkai Lin sec_aead_sha1_ctx_init, sec_aead_ctx_exit, AES_BLOCK_SIZE, 2464c16a70c1SKai Ye AES_BLOCK_SIZE, SHA1_DIGEST_SIZE), 2465921715b6SWenkai Lin }, 2466921715b6SWenkai Lin { 2467921715b6SWenkai Lin .alg_msk = BIT(44), 2468921715b6SWenkai Lin .alg = SEC_AEAD_ALG("authenc(hmac(sha256),cbc(aes))", sec_setkey_aes_cbc_sha256, 2469921715b6SWenkai Lin sec_aead_sha256_ctx_init, sec_aead_ctx_exit, AES_BLOCK_SIZE, 2470c16a70c1SKai Ye AES_BLOCK_SIZE, SHA256_DIGEST_SIZE), 2471921715b6SWenkai Lin }, 2472921715b6SWenkai Lin { 2473921715b6SWenkai Lin .alg_msk = BIT(45), 2474921715b6SWenkai Lin .alg = SEC_AEAD_ALG("authenc(hmac(sha512),cbc(aes))", sec_setkey_aes_cbc_sha512, 2475921715b6SWenkai Lin sec_aead_sha512_ctx_init, sec_aead_ctx_exit, AES_BLOCK_SIZE, 2476c16a70c1SKai Ye AES_BLOCK_SIZE, SHA512_DIGEST_SIZE), 2477921715b6SWenkai Lin }, 2478c16a70c1SKai Ye }; 2479c16a70c1SKai Ye 2480921715b6SWenkai Lin static void sec_unregister_skcipher(u64 alg_mask, int end) 2481921715b6SWenkai Lin { 2482921715b6SWenkai Lin int i; 2483c16a70c1SKai Ye 2484921715b6SWenkai Lin for (i = 0; i < end; i++) 2485921715b6SWenkai Lin if (sec_skciphers[i].alg_msk & alg_mask) 2486921715b6SWenkai Lin crypto_unregister_skcipher(&sec_skciphers[i].alg); 2487921715b6SWenkai Lin } 2488921715b6SWenkai Lin 2489921715b6SWenkai Lin static int sec_register_skcipher(u64 alg_mask) 2490921715b6SWenkai Lin { 2491921715b6SWenkai Lin int i, ret, count; 2492921715b6SWenkai Lin 2493921715b6SWenkai Lin count = ARRAY_SIZE(sec_skciphers); 2494921715b6SWenkai Lin 2495921715b6SWenkai Lin for (i = 0; i < count; i++) { 2496921715b6SWenkai Lin if (!(sec_skciphers[i].alg_msk & alg_mask)) 2497921715b6SWenkai Lin continue; 2498921715b6SWenkai Lin 2499921715b6SWenkai Lin ret = crypto_register_skcipher(&sec_skciphers[i].alg); 2500921715b6SWenkai Lin if (ret) 2501921715b6SWenkai Lin goto err; 2502921715b6SWenkai Lin } 2503921715b6SWenkai Lin 2504921715b6SWenkai Lin return 0; 2505921715b6SWenkai Lin 2506921715b6SWenkai Lin err: 2507921715b6SWenkai Lin sec_unregister_skcipher(alg_mask, i); 2508921715b6SWenkai Lin 2509921715b6SWenkai Lin return ret; 2510921715b6SWenkai Lin } 2511921715b6SWenkai Lin 2512921715b6SWenkai Lin static void sec_unregister_aead(u64 alg_mask, int end) 2513921715b6SWenkai Lin { 2514921715b6SWenkai Lin int i; 2515921715b6SWenkai Lin 2516921715b6SWenkai Lin for (i = 0; i < end; i++) 2517921715b6SWenkai Lin if (sec_aeads[i].alg_msk & alg_mask) 2518921715b6SWenkai Lin crypto_unregister_aead(&sec_aeads[i].alg); 2519921715b6SWenkai Lin } 2520921715b6SWenkai Lin 2521921715b6SWenkai Lin static int sec_register_aead(u64 alg_mask) 2522921715b6SWenkai Lin { 2523921715b6SWenkai Lin int i, ret, count; 2524921715b6SWenkai Lin 2525921715b6SWenkai Lin count = ARRAY_SIZE(sec_aeads); 2526921715b6SWenkai Lin 2527921715b6SWenkai Lin for (i = 0; i < count; i++) { 2528921715b6SWenkai Lin if (!(sec_aeads[i].alg_msk & alg_mask)) 2529921715b6SWenkai Lin continue; 2530921715b6SWenkai Lin 2531921715b6SWenkai Lin ret = crypto_register_aead(&sec_aeads[i].alg); 2532921715b6SWenkai Lin if (ret) 2533921715b6SWenkai Lin goto err; 2534921715b6SWenkai Lin } 2535921715b6SWenkai Lin 2536921715b6SWenkai Lin return 0; 2537921715b6SWenkai Lin 2538921715b6SWenkai Lin err: 2539921715b6SWenkai Lin sec_unregister_aead(alg_mask, i); 2540921715b6SWenkai Lin 2541921715b6SWenkai Lin return ret; 2542921715b6SWenkai Lin } 25432f072d75SZaibo Xu 25448123455aSMeng Yu int sec_register_to_crypto(struct hisi_qm *qm) 2545416d8220SZaibo Xu { 2546*b06a6d5eSZhiqi Song u64 alg_mask; 2547*b06a6d5eSZhiqi Song int ret = 0; 2548*b06a6d5eSZhiqi Song 2549*b06a6d5eSZhiqi Song alg_mask = sec_get_alg_bitmap(qm, SEC_DRV_ALG_BITMAP_HIGH_IDX, 2550*b06a6d5eSZhiqi Song SEC_DRV_ALG_BITMAP_LOW_IDX); 2551*b06a6d5eSZhiqi Song 2552416d8220SZaibo Xu 2553921715b6SWenkai Lin ret = sec_register_skcipher(alg_mask); 25542f072d75SZaibo Xu if (ret) 25552f072d75SZaibo Xu return ret; 25562f072d75SZaibo Xu 2557921715b6SWenkai Lin ret = sec_register_aead(alg_mask); 25587b44c0eeSKai Ye if (ret) 2559921715b6SWenkai Lin sec_unregister_skcipher(alg_mask, ARRAY_SIZE(sec_skciphers)); 2560c16a70c1SKai Ye 2561416d8220SZaibo Xu return ret; 2562416d8220SZaibo Xu } 2563416d8220SZaibo Xu 25648123455aSMeng Yu void sec_unregister_from_crypto(struct hisi_qm *qm) 2565416d8220SZaibo Xu { 2566*b06a6d5eSZhiqi Song u64 alg_mask; 2567*b06a6d5eSZhiqi Song 2568*b06a6d5eSZhiqi Song alg_mask = sec_get_alg_bitmap(qm, SEC_DRV_ALG_BITMAP_HIGH_IDX, 2569*b06a6d5eSZhiqi Song SEC_DRV_ALG_BITMAP_LOW_IDX); 2570c16a70c1SKai Ye 2571921715b6SWenkai Lin sec_unregister_aead(alg_mask, ARRAY_SIZE(sec_aeads)); 2572921715b6SWenkai Lin sec_unregister_skcipher(alg_mask, ARRAY_SIZE(sec_skciphers)); 25732f072d75SZaibo Xu } 2574