1416d8220SZaibo Xu // SPDX-License-Identifier: GPL-2.0 2416d8220SZaibo Xu /* Copyright (c) 2019 HiSilicon Limited. */ 3416d8220SZaibo Xu 4416d8220SZaibo Xu #include <crypto/aes.h> 5416d8220SZaibo Xu #include <crypto/algapi.h> 62f072d75SZaibo Xu #include <crypto/authenc.h> 7416d8220SZaibo Xu #include <crypto/des.h> 82f072d75SZaibo Xu #include <crypto/hash.h> 92f072d75SZaibo Xu #include <crypto/internal/aead.h> 10ae6ce7b1SKai Ye #include <crypto/internal/des.h> 11a24d22b2SEric Biggers #include <crypto/sha1.h> 12a24d22b2SEric Biggers #include <crypto/sha2.h> 13416d8220SZaibo Xu #include <crypto/skcipher.h> 14416d8220SZaibo Xu #include <crypto/xts.h> 15416d8220SZaibo Xu #include <linux/crypto.h> 16416d8220SZaibo Xu #include <linux/dma-mapping.h> 17416d8220SZaibo Xu #include <linux/idr.h> 18416d8220SZaibo Xu 19416d8220SZaibo Xu #include "sec.h" 20416d8220SZaibo Xu #include "sec_crypto.h" 21416d8220SZaibo Xu 22416d8220SZaibo Xu #define SEC_PRIORITY 4001 23416d8220SZaibo Xu #define SEC_XTS_MIN_KEY_SIZE (2 * AES_MIN_KEY_SIZE) 245652d55aSKai Ye #define SEC_XTS_MID_KEY_SIZE (3 * AES_MIN_KEY_SIZE) 25416d8220SZaibo Xu #define SEC_XTS_MAX_KEY_SIZE (2 * AES_MAX_KEY_SIZE) 26416d8220SZaibo Xu #define SEC_DES3_2KEY_SIZE (2 * DES_KEY_SIZE) 27416d8220SZaibo Xu #define SEC_DES3_3KEY_SIZE (3 * DES_KEY_SIZE) 28416d8220SZaibo Xu 29416d8220SZaibo Xu /* SEC sqe(bd) bit operational relative MACRO */ 30416d8220SZaibo Xu #define SEC_DE_OFFSET 1 31416d8220SZaibo Xu #define SEC_CIPHER_OFFSET 4 32416d8220SZaibo Xu #define SEC_SCENE_OFFSET 3 33416d8220SZaibo Xu #define SEC_DST_SGL_OFFSET 2 34416d8220SZaibo Xu #define SEC_SRC_SGL_OFFSET 7 35416d8220SZaibo Xu #define SEC_CKEY_OFFSET 9 36416d8220SZaibo Xu #define SEC_CMODE_OFFSET 12 372f072d75SZaibo Xu #define SEC_AKEY_OFFSET 5 382f072d75SZaibo Xu #define SEC_AEAD_ALG_OFFSET 11 392f072d75SZaibo Xu #define SEC_AUTH_OFFSET 6 402f072d75SZaibo Xu 41adc3f65aSKai Ye #define SEC_DE_OFFSET_V3 9 42adc3f65aSKai Ye #define SEC_SCENE_OFFSET_V3 5 43adc3f65aSKai Ye #define SEC_CKEY_OFFSET_V3 13 44adc3f65aSKai Ye #define SEC_SRC_SGL_OFFSET_V3 11 45adc3f65aSKai Ye #define SEC_DST_SGL_OFFSET_V3 14 46adc3f65aSKai Ye #define SEC_CALG_OFFSET_V3 4 47adc3f65aSKai Ye #define SEC_AKEY_OFFSET_V3 9 48adc3f65aSKai Ye #define SEC_MAC_OFFSET_V3 4 49adc3f65aSKai Ye #define SEC_AUTH_ALG_OFFSET_V3 15 50adc3f65aSKai Ye #define SEC_CIPHER_AUTH_V3 0xbf 51adc3f65aSKai Ye #define SEC_AUTH_CIPHER_V3 0x40 52416d8220SZaibo Xu #define SEC_FLAG_OFFSET 7 53416d8220SZaibo Xu #define SEC_FLAG_MASK 0x0780 54416d8220SZaibo Xu #define SEC_TYPE_MASK 0x0F 55416d8220SZaibo Xu #define SEC_DONE_MASK 0x0001 56adc3f65aSKai Ye #define SEC_SQE_LEN_RATE_MASK 0x3 57416d8220SZaibo Xu 58416d8220SZaibo Xu #define SEC_TOTAL_IV_SZ (SEC_IV_SIZE * QM_Q_DEPTH) 59416d8220SZaibo Xu #define SEC_SGL_SGE_NR 128 602f072d75SZaibo Xu #define SEC_CIPHER_AUTH 0xfe 612f072d75SZaibo Xu #define SEC_AUTH_CIPHER 0x1 622f072d75SZaibo Xu #define SEC_MAX_MAC_LEN 64 632514f559SLongfang Liu #define SEC_MAX_AAD_LEN 65535 642f072d75SZaibo Xu #define SEC_TOTAL_MAC_SZ (SEC_MAX_MAC_LEN * QM_Q_DEPTH) 6574b58db8SLongfang Liu 6674b58db8SLongfang Liu #define SEC_PBUF_SZ 512 6774b58db8SLongfang Liu #define SEC_PBUF_IV_OFFSET SEC_PBUF_SZ 6874b58db8SLongfang Liu #define SEC_PBUF_MAC_OFFSET (SEC_PBUF_SZ + SEC_IV_SIZE) 6974b58db8SLongfang Liu #define SEC_PBUF_PKG (SEC_PBUF_SZ + SEC_IV_SIZE + \ 7074b58db8SLongfang Liu SEC_MAX_MAC_LEN * 2) 7174b58db8SLongfang Liu #define SEC_PBUF_NUM (PAGE_SIZE / SEC_PBUF_PKG) 7274b58db8SLongfang Liu #define SEC_PBUF_PAGE_NUM (QM_Q_DEPTH / SEC_PBUF_NUM) 7374b58db8SLongfang Liu #define SEC_PBUF_LEFT_SZ (SEC_PBUF_PKG * (QM_Q_DEPTH - \ 7474b58db8SLongfang Liu SEC_PBUF_PAGE_NUM * SEC_PBUF_NUM)) 7574b58db8SLongfang Liu #define SEC_TOTAL_PBUF_SZ (PAGE_SIZE * SEC_PBUF_PAGE_NUM + \ 7674b58db8SLongfang Liu SEC_PBUF_LEFT_SZ) 7774b58db8SLongfang Liu 782f072d75SZaibo Xu #define SEC_SQE_LEN_RATE 4 79d6de2a59SZaibo Xu #define SEC_SQE_CFLAG 2 802f072d75SZaibo Xu #define SEC_SQE_AEAD_FLAG 3 81d6de2a59SZaibo Xu #define SEC_SQE_DONE 0x1 827b44c0eeSKai Ye #define MAX_INPUT_DATA_LEN 0xFFFE00 837b44c0eeSKai Ye #define BITS_MASK 0xFF 847b44c0eeSKai Ye #define BYTE_BITS 0x8 855652d55aSKai Ye #define SEC_XTS_NAME_SZ 0x3 86416d8220SZaibo Xu 87416d8220SZaibo Xu /* Get an en/de-cipher queue cyclically to balance load over queues of TFM */ 88a181647cSZaibo Xu static inline int sec_alloc_queue_id(struct sec_ctx *ctx, struct sec_req *req) 89416d8220SZaibo Xu { 90416d8220SZaibo Xu if (req->c_req.encrypt) 91416d8220SZaibo Xu return (u32)atomic_inc_return(&ctx->enc_qcyclic) % 92416d8220SZaibo Xu ctx->hlf_q_num; 93416d8220SZaibo Xu 94416d8220SZaibo Xu return (u32)atomic_inc_return(&ctx->dec_qcyclic) % ctx->hlf_q_num + 95416d8220SZaibo Xu ctx->hlf_q_num; 96416d8220SZaibo Xu } 97416d8220SZaibo Xu 98a181647cSZaibo Xu static inline void sec_free_queue_id(struct sec_ctx *ctx, struct sec_req *req) 99416d8220SZaibo Xu { 100416d8220SZaibo Xu if (req->c_req.encrypt) 101416d8220SZaibo Xu atomic_dec(&ctx->enc_qcyclic); 102416d8220SZaibo Xu else 103416d8220SZaibo Xu atomic_dec(&ctx->dec_qcyclic); 104416d8220SZaibo Xu } 105416d8220SZaibo Xu 106416d8220SZaibo Xu static int sec_alloc_req_id(struct sec_req *req, struct sec_qp_ctx *qp_ctx) 107416d8220SZaibo Xu { 108416d8220SZaibo Xu int req_id; 109416d8220SZaibo Xu 110416d8220SZaibo Xu mutex_lock(&qp_ctx->req_lock); 111416d8220SZaibo Xu 112416d8220SZaibo Xu req_id = idr_alloc_cyclic(&qp_ctx->req_idr, NULL, 113416d8220SZaibo Xu 0, QM_Q_DEPTH, GFP_ATOMIC); 114416d8220SZaibo Xu mutex_unlock(&qp_ctx->req_lock); 115b9c8d897SZaibo Xu if (unlikely(req_id < 0)) { 116a44dce50SLongfang Liu dev_err(req->ctx->dev, "alloc req id fail!\n"); 117416d8220SZaibo Xu return req_id; 118416d8220SZaibo Xu } 119416d8220SZaibo Xu 120416d8220SZaibo Xu req->qp_ctx = qp_ctx; 121416d8220SZaibo Xu qp_ctx->req_list[req_id] = req; 122633e507fSLongfang Liu 123416d8220SZaibo Xu return req_id; 124416d8220SZaibo Xu } 125416d8220SZaibo Xu 126416d8220SZaibo Xu static void sec_free_req_id(struct sec_req *req) 127416d8220SZaibo Xu { 128416d8220SZaibo Xu struct sec_qp_ctx *qp_ctx = req->qp_ctx; 129416d8220SZaibo Xu int req_id = req->req_id; 130416d8220SZaibo Xu 131b9c8d897SZaibo Xu if (unlikely(req_id < 0 || req_id >= QM_Q_DEPTH)) { 132a44dce50SLongfang Liu dev_err(req->ctx->dev, "free request id invalid!\n"); 133416d8220SZaibo Xu return; 134416d8220SZaibo Xu } 135416d8220SZaibo Xu 136416d8220SZaibo Xu qp_ctx->req_list[req_id] = NULL; 137416d8220SZaibo Xu req->qp_ctx = NULL; 138416d8220SZaibo Xu 139416d8220SZaibo Xu mutex_lock(&qp_ctx->req_lock); 140416d8220SZaibo Xu idr_remove(&qp_ctx->req_idr, req_id); 141416d8220SZaibo Xu mutex_unlock(&qp_ctx->req_lock); 142416d8220SZaibo Xu } 143416d8220SZaibo Xu 1442514f559SLongfang Liu static int sec_aead_verify(struct sec_req *req) 1452f072d75SZaibo Xu { 1462f072d75SZaibo Xu struct aead_request *aead_req = req->aead_req.aead_req; 1472f072d75SZaibo Xu struct crypto_aead *tfm = crypto_aead_reqtfm(aead_req); 1482f072d75SZaibo Xu size_t authsize = crypto_aead_authsize(tfm); 1492514f559SLongfang Liu u8 *mac_out = req->aead_req.out_mac; 1502f072d75SZaibo Xu u8 *mac = mac_out + SEC_MAX_MAC_LEN; 1512f072d75SZaibo Xu struct scatterlist *sgl = aead_req->src; 1522f072d75SZaibo Xu size_t sz; 1532f072d75SZaibo Xu 1542f072d75SZaibo Xu sz = sg_pcopy_to_buffer(sgl, sg_nents(sgl), mac, authsize, 1552f072d75SZaibo Xu aead_req->cryptlen + aead_req->assoclen - 1562f072d75SZaibo Xu authsize); 1572f072d75SZaibo Xu if (unlikely(sz != authsize || memcmp(mac_out, mac, sz))) { 158a44dce50SLongfang Liu dev_err(req->ctx->dev, "aead verify failure!\n"); 1592f072d75SZaibo Xu return -EBADMSG; 1602f072d75SZaibo Xu } 1612f072d75SZaibo Xu 1622f072d75SZaibo Xu return 0; 1632f072d75SZaibo Xu } 1642f072d75SZaibo Xu 165adc3f65aSKai Ye static u8 pre_parse_finished_bd(struct bd_status *status, void *resp) 166adc3f65aSKai Ye { 167adc3f65aSKai Ye struct sec_sqe *bd = resp; 168adc3f65aSKai Ye 169adc3f65aSKai Ye status->done = le16_to_cpu(bd->type2.done_flag) & SEC_DONE_MASK; 170adc3f65aSKai Ye status->flag = (le16_to_cpu(bd->type2.done_flag) & 171adc3f65aSKai Ye SEC_FLAG_MASK) >> SEC_FLAG_OFFSET; 172adc3f65aSKai Ye status->tag = le16_to_cpu(bd->type2.tag); 173adc3f65aSKai Ye status->err_type = bd->type2.error_type; 174adc3f65aSKai Ye 175adc3f65aSKai Ye return bd->type_cipher_auth & SEC_TYPE_MASK; 176adc3f65aSKai Ye } 177adc3f65aSKai Ye 178adc3f65aSKai Ye static u8 pre_parse_finished_bd3(struct bd_status *status, void *resp) 179adc3f65aSKai Ye { 180adc3f65aSKai Ye struct sec_sqe3 *bd3 = resp; 181adc3f65aSKai Ye 182adc3f65aSKai Ye status->done = le16_to_cpu(bd3->done_flag) & SEC_DONE_MASK; 183adc3f65aSKai Ye status->flag = (le16_to_cpu(bd3->done_flag) & 184adc3f65aSKai Ye SEC_FLAG_MASK) >> SEC_FLAG_OFFSET; 185adc3f65aSKai Ye status->tag = le64_to_cpu(bd3->tag); 186adc3f65aSKai Ye status->err_type = bd3->error_type; 187adc3f65aSKai Ye 188adc3f65aSKai Ye return le32_to_cpu(bd3->bd_param) & SEC_TYPE_MASK; 189adc3f65aSKai Ye } 190adc3f65aSKai Ye 191adc3f65aSKai Ye static int sec_cb_status_check(struct sec_req *req, 192adc3f65aSKai Ye struct bd_status *status) 193adc3f65aSKai Ye { 194adc3f65aSKai Ye struct sec_ctx *ctx = req->ctx; 195adc3f65aSKai Ye 196adc3f65aSKai Ye if (unlikely(req->err_type || status->done != SEC_SQE_DONE)) { 197adc3f65aSKai Ye dev_err_ratelimited(ctx->dev, "err_type[%d], done[%u]\n", 198adc3f65aSKai Ye req->err_type, status->done); 199adc3f65aSKai Ye return -EIO; 200adc3f65aSKai Ye } 201adc3f65aSKai Ye 202adc3f65aSKai Ye if (unlikely(ctx->alg_type == SEC_SKCIPHER)) { 203adc3f65aSKai Ye if (unlikely(status->flag != SEC_SQE_CFLAG)) { 204adc3f65aSKai Ye dev_err_ratelimited(ctx->dev, "flag[%u]\n", 205adc3f65aSKai Ye status->flag); 206adc3f65aSKai Ye return -EIO; 207adc3f65aSKai Ye } 208adc3f65aSKai Ye } 209adc3f65aSKai Ye 210adc3f65aSKai Ye return 0; 211adc3f65aSKai Ye } 212adc3f65aSKai Ye 213416d8220SZaibo Xu static void sec_req_cb(struct hisi_qp *qp, void *resp) 214416d8220SZaibo Xu { 215416d8220SZaibo Xu struct sec_qp_ctx *qp_ctx = qp->qp_ctx; 2168213a1a6SKai Ye struct sec_dfx *dfx = &qp_ctx->ctx->sec->debug.dfx; 217adc3f65aSKai Ye u8 type_supported = qp_ctx->ctx->type_supported; 218adc3f65aSKai Ye struct bd_status status; 219d6de2a59SZaibo Xu struct sec_ctx *ctx; 220d6de2a59SZaibo Xu struct sec_req *req; 221adc3f65aSKai Ye int err; 222416d8220SZaibo Xu u8 type; 223416d8220SZaibo Xu 224adc3f65aSKai Ye if (type_supported == SEC_BD_TYPE2) { 225adc3f65aSKai Ye type = pre_parse_finished_bd(&status, resp); 226adc3f65aSKai Ye req = qp_ctx->req_list[status.tag]; 227adc3f65aSKai Ye } else { 228adc3f65aSKai Ye type = pre_parse_finished_bd3(&status, resp); 229adc3f65aSKai Ye req = (void *)(uintptr_t)status.tag; 230adc3f65aSKai Ye } 231adc3f65aSKai Ye 232adc3f65aSKai Ye if (unlikely(type != type_supported)) { 2338213a1a6SKai Ye atomic64_inc(&dfx->err_bd_cnt); 234416d8220SZaibo Xu pr_err("err bd type [%d]\n", type); 235416d8220SZaibo Xu return; 236416d8220SZaibo Xu } 237416d8220SZaibo Xu 2388213a1a6SKai Ye if (unlikely(!req)) { 2398213a1a6SKai Ye atomic64_inc(&dfx->invalid_req_cnt); 2409597efc3SKai Ye atomic_inc(&qp->qp_status.used); 2418213a1a6SKai Ye return; 2428213a1a6SKai Ye } 243adc3f65aSKai Ye 244adc3f65aSKai Ye req->err_type = status.err_type; 245d6de2a59SZaibo Xu ctx = req->ctx; 246adc3f65aSKai Ye err = sec_cb_status_check(req, &status); 247adc3f65aSKai Ye if (err) 2488213a1a6SKai Ye atomic64_inc(&dfx->done_flag_cnt); 2491e9bc276SZaibo Xu 2502f072d75SZaibo Xu if (ctx->alg_type == SEC_AEAD && !req->c_req.encrypt) 2512514f559SLongfang Liu err = sec_aead_verify(req); 2522f072d75SZaibo Xu 2538213a1a6SKai Ye atomic64_inc(&dfx->recv_cnt); 254416d8220SZaibo Xu 255d6de2a59SZaibo Xu ctx->req_op->buf_unmap(ctx, req); 256d6de2a59SZaibo Xu 257310ea0acSZaibo Xu ctx->req_op->callback(ctx, req, err); 258416d8220SZaibo Xu } 259416d8220SZaibo Xu 260416d8220SZaibo Xu static int sec_bd_send(struct sec_ctx *ctx, struct sec_req *req) 261416d8220SZaibo Xu { 262416d8220SZaibo Xu struct sec_qp_ctx *qp_ctx = req->qp_ctx; 263416d8220SZaibo Xu int ret; 264416d8220SZaibo Xu 2659597efc3SKai Ye if (ctx->fake_req_limit <= 2669597efc3SKai Ye atomic_read(&qp_ctx->qp->qp_status.used) && 2679597efc3SKai Ye !(req->flag & CRYPTO_TFM_REQ_MAY_BACKLOG)) 2689597efc3SKai Ye return -EBUSY; 2699597efc3SKai Ye 270416d8220SZaibo Xu mutex_lock(&qp_ctx->req_lock); 271416d8220SZaibo Xu ret = hisi_qp_send(qp_ctx->qp, &req->sec_sqe); 2729597efc3SKai Ye 2739597efc3SKai Ye if (ctx->fake_req_limit <= 2749597efc3SKai Ye atomic_read(&qp_ctx->qp->qp_status.used) && !ret) { 2759597efc3SKai Ye list_add_tail(&req->backlog_head, &qp_ctx->backlog); 276cb1eeb75SArnd Bergmann atomic64_inc(&ctx->sec->debug.dfx.send_cnt); 2779597efc3SKai Ye atomic64_inc(&ctx->sec->debug.dfx.send_busy_cnt); 2789597efc3SKai Ye mutex_unlock(&qp_ctx->req_lock); 2799597efc3SKai Ye return -EBUSY; 2809597efc3SKai Ye } 2819597efc3SKai Ye mutex_unlock(&qp_ctx->req_lock); 282416d8220SZaibo Xu 283b9c8d897SZaibo Xu if (unlikely(ret == -EBUSY)) 284416d8220SZaibo Xu return -ENOBUFS; 285416d8220SZaibo Xu 2869597efc3SKai Ye if (likely(!ret)) { 287416d8220SZaibo Xu ret = -EINPROGRESS; 2889597efc3SKai Ye atomic64_inc(&ctx->sec->debug.dfx.send_cnt); 2898213a1a6SKai Ye } 290416d8220SZaibo Xu 291416d8220SZaibo Xu return ret; 292416d8220SZaibo Xu } 293416d8220SZaibo Xu 2947c7d902aSZaibo Xu /* Get DMA memory resources */ 2957c7d902aSZaibo Xu static int sec_alloc_civ_resource(struct device *dev, struct sec_alg_res *res) 2967c7d902aSZaibo Xu { 2977c7d902aSZaibo Xu int i; 2987c7d902aSZaibo Xu 2997c7d902aSZaibo Xu res->c_ivin = dma_alloc_coherent(dev, SEC_TOTAL_IV_SZ, 3007c7d902aSZaibo Xu &res->c_ivin_dma, GFP_KERNEL); 3017c7d902aSZaibo Xu if (!res->c_ivin) 3027c7d902aSZaibo Xu return -ENOMEM; 3037c7d902aSZaibo Xu 3047c7d902aSZaibo Xu for (i = 1; i < QM_Q_DEPTH; i++) { 3057c7d902aSZaibo Xu res[i].c_ivin_dma = res->c_ivin_dma + i * SEC_IV_SIZE; 3067c7d902aSZaibo Xu res[i].c_ivin = res->c_ivin + i * SEC_IV_SIZE; 3077c7d902aSZaibo Xu } 3087c7d902aSZaibo Xu 3097c7d902aSZaibo Xu return 0; 3107c7d902aSZaibo Xu } 3117c7d902aSZaibo Xu 3127c7d902aSZaibo Xu static void sec_free_civ_resource(struct device *dev, struct sec_alg_res *res) 3137c7d902aSZaibo Xu { 3147c7d902aSZaibo Xu if (res->c_ivin) 3157c7d902aSZaibo Xu dma_free_coherent(dev, SEC_TOTAL_IV_SZ, 3167c7d902aSZaibo Xu res->c_ivin, res->c_ivin_dma); 3177c7d902aSZaibo Xu } 3187c7d902aSZaibo Xu 3192f072d75SZaibo Xu static int sec_alloc_mac_resource(struct device *dev, struct sec_alg_res *res) 3202f072d75SZaibo Xu { 3212f072d75SZaibo Xu int i; 3222f072d75SZaibo Xu 3232f072d75SZaibo Xu res->out_mac = dma_alloc_coherent(dev, SEC_TOTAL_MAC_SZ << 1, 3242f072d75SZaibo Xu &res->out_mac_dma, GFP_KERNEL); 3252f072d75SZaibo Xu if (!res->out_mac) 3262f072d75SZaibo Xu return -ENOMEM; 3272f072d75SZaibo Xu 3282f072d75SZaibo Xu for (i = 1; i < QM_Q_DEPTH; i++) { 3292f072d75SZaibo Xu res[i].out_mac_dma = res->out_mac_dma + 3302f072d75SZaibo Xu i * (SEC_MAX_MAC_LEN << 1); 3312f072d75SZaibo Xu res[i].out_mac = res->out_mac + i * (SEC_MAX_MAC_LEN << 1); 3322f072d75SZaibo Xu } 3332f072d75SZaibo Xu 3342f072d75SZaibo Xu return 0; 3352f072d75SZaibo Xu } 3362f072d75SZaibo Xu 3372f072d75SZaibo Xu static void sec_free_mac_resource(struct device *dev, struct sec_alg_res *res) 3382f072d75SZaibo Xu { 3392f072d75SZaibo Xu if (res->out_mac) 3402f072d75SZaibo Xu dma_free_coherent(dev, SEC_TOTAL_MAC_SZ << 1, 3412f072d75SZaibo Xu res->out_mac, res->out_mac_dma); 3422f072d75SZaibo Xu } 3432f072d75SZaibo Xu 34474b58db8SLongfang Liu static void sec_free_pbuf_resource(struct device *dev, struct sec_alg_res *res) 34574b58db8SLongfang Liu { 34674b58db8SLongfang Liu if (res->pbuf) 34774b58db8SLongfang Liu dma_free_coherent(dev, SEC_TOTAL_PBUF_SZ, 34874b58db8SLongfang Liu res->pbuf, res->pbuf_dma); 34974b58db8SLongfang Liu } 35074b58db8SLongfang Liu 35174b58db8SLongfang Liu /* 35274b58db8SLongfang Liu * To improve performance, pbuffer is used for 35374b58db8SLongfang Liu * small packets (< 512Bytes) as IOMMU translation using. 35474b58db8SLongfang Liu */ 35574b58db8SLongfang Liu static int sec_alloc_pbuf_resource(struct device *dev, struct sec_alg_res *res) 35674b58db8SLongfang Liu { 35774b58db8SLongfang Liu int pbuf_page_offset; 35874b58db8SLongfang Liu int i, j, k; 35974b58db8SLongfang Liu 36074b58db8SLongfang Liu res->pbuf = dma_alloc_coherent(dev, SEC_TOTAL_PBUF_SZ, 36174b58db8SLongfang Liu &res->pbuf_dma, GFP_KERNEL); 36274b58db8SLongfang Liu if (!res->pbuf) 36374b58db8SLongfang Liu return -ENOMEM; 36474b58db8SLongfang Liu 36574b58db8SLongfang Liu /* 36674b58db8SLongfang Liu * SEC_PBUF_PKG contains data pbuf, iv and 36774b58db8SLongfang Liu * out_mac : <SEC_PBUF|SEC_IV|SEC_MAC> 36874b58db8SLongfang Liu * Every PAGE contains six SEC_PBUF_PKG 36974b58db8SLongfang Liu * The sec_qp_ctx contains QM_Q_DEPTH numbers of SEC_PBUF_PKG 37074b58db8SLongfang Liu * So we need SEC_PBUF_PAGE_NUM numbers of PAGE 37174b58db8SLongfang Liu * for the SEC_TOTAL_PBUF_SZ 37274b58db8SLongfang Liu */ 37374b58db8SLongfang Liu for (i = 0; i <= SEC_PBUF_PAGE_NUM; i++) { 37474b58db8SLongfang Liu pbuf_page_offset = PAGE_SIZE * i; 37574b58db8SLongfang Liu for (j = 0; j < SEC_PBUF_NUM; j++) { 37674b58db8SLongfang Liu k = i * SEC_PBUF_NUM + j; 37774b58db8SLongfang Liu if (k == QM_Q_DEPTH) 37874b58db8SLongfang Liu break; 37974b58db8SLongfang Liu res[k].pbuf = res->pbuf + 38074b58db8SLongfang Liu j * SEC_PBUF_PKG + pbuf_page_offset; 38174b58db8SLongfang Liu res[k].pbuf_dma = res->pbuf_dma + 38274b58db8SLongfang Liu j * SEC_PBUF_PKG + pbuf_page_offset; 38374b58db8SLongfang Liu } 38474b58db8SLongfang Liu } 385633e507fSLongfang Liu 38674b58db8SLongfang Liu return 0; 38774b58db8SLongfang Liu } 38874b58db8SLongfang Liu 3897c7d902aSZaibo Xu static int sec_alg_resource_alloc(struct sec_ctx *ctx, 3907c7d902aSZaibo Xu struct sec_qp_ctx *qp_ctx) 3917c7d902aSZaibo Xu { 3922f072d75SZaibo Xu struct sec_alg_res *res = qp_ctx->res; 393a44dce50SLongfang Liu struct device *dev = ctx->dev; 3942f072d75SZaibo Xu int ret; 3957c7d902aSZaibo Xu 3962f072d75SZaibo Xu ret = sec_alloc_civ_resource(dev, res); 3972f072d75SZaibo Xu if (ret) 3982f072d75SZaibo Xu return ret; 3992f072d75SZaibo Xu 4002f072d75SZaibo Xu if (ctx->alg_type == SEC_AEAD) { 4012f072d75SZaibo Xu ret = sec_alloc_mac_resource(dev, res); 4022f072d75SZaibo Xu if (ret) 4032514f559SLongfang Liu goto alloc_fail; 4042f072d75SZaibo Xu } 40574b58db8SLongfang Liu if (ctx->pbuf_supported) { 40674b58db8SLongfang Liu ret = sec_alloc_pbuf_resource(dev, res); 40774b58db8SLongfang Liu if (ret) { 40874b58db8SLongfang Liu dev_err(dev, "fail to alloc pbuf dma resource!\n"); 40924efcec2SLongfang Liu goto alloc_pbuf_fail; 41074b58db8SLongfang Liu } 41174b58db8SLongfang Liu } 4122f072d75SZaibo Xu 4132f072d75SZaibo Xu return 0; 414633e507fSLongfang Liu 41524efcec2SLongfang Liu alloc_pbuf_fail: 41624efcec2SLongfang Liu if (ctx->alg_type == SEC_AEAD) 41724efcec2SLongfang Liu sec_free_mac_resource(dev, qp_ctx->res); 4182514f559SLongfang Liu alloc_fail: 4192f072d75SZaibo Xu sec_free_civ_resource(dev, res); 4202f072d75SZaibo Xu return ret; 4217c7d902aSZaibo Xu } 4227c7d902aSZaibo Xu 4237c7d902aSZaibo Xu static void sec_alg_resource_free(struct sec_ctx *ctx, 4247c7d902aSZaibo Xu struct sec_qp_ctx *qp_ctx) 4257c7d902aSZaibo Xu { 426a44dce50SLongfang Liu struct device *dev = ctx->dev; 4277c7d902aSZaibo Xu 4287c7d902aSZaibo Xu sec_free_civ_resource(dev, qp_ctx->res); 4292f072d75SZaibo Xu 43074b58db8SLongfang Liu if (ctx->pbuf_supported) 43174b58db8SLongfang Liu sec_free_pbuf_resource(dev, qp_ctx->res); 4322f072d75SZaibo Xu if (ctx->alg_type == SEC_AEAD) 4332f072d75SZaibo Xu sec_free_mac_resource(dev, qp_ctx->res); 4347c7d902aSZaibo Xu } 4357c7d902aSZaibo Xu 436416d8220SZaibo Xu static int sec_create_qp_ctx(struct hisi_qm *qm, struct sec_ctx *ctx, 437416d8220SZaibo Xu int qp_ctx_id, int alg_type) 438416d8220SZaibo Xu { 439a44dce50SLongfang Liu struct device *dev = ctx->dev; 440416d8220SZaibo Xu struct sec_qp_ctx *qp_ctx; 441416d8220SZaibo Xu struct hisi_qp *qp; 442416d8220SZaibo Xu int ret = -ENOMEM; 443416d8220SZaibo Xu 444416d8220SZaibo Xu qp_ctx = &ctx->qp_ctx[qp_ctx_id]; 4450b5e43bcSKai Ye qp = ctx->qps[qp_ctx_id]; 446416d8220SZaibo Xu qp->req_type = 0; 447416d8220SZaibo Xu qp->qp_ctx = qp_ctx; 448416d8220SZaibo Xu qp_ctx->qp = qp; 449416d8220SZaibo Xu qp_ctx->ctx = ctx; 450416d8220SZaibo Xu 451adc3f65aSKai Ye qp->req_cb = sec_req_cb; 452adc3f65aSKai Ye 453416d8220SZaibo Xu mutex_init(&qp_ctx->req_lock); 454416d8220SZaibo Xu idr_init(&qp_ctx->req_idr); 4559597efc3SKai Ye INIT_LIST_HEAD(&qp_ctx->backlog); 456416d8220SZaibo Xu 457416d8220SZaibo Xu qp_ctx->c_in_pool = hisi_acc_create_sgl_pool(dev, QM_Q_DEPTH, 458416d8220SZaibo Xu SEC_SGL_SGE_NR); 4598a6b8f4dSDan Carpenter if (IS_ERR(qp_ctx->c_in_pool)) { 460416d8220SZaibo Xu dev_err(dev, "fail to create sgl pool for input!\n"); 4617c7d902aSZaibo Xu goto err_destroy_idr; 462416d8220SZaibo Xu } 463416d8220SZaibo Xu 464416d8220SZaibo Xu qp_ctx->c_out_pool = hisi_acc_create_sgl_pool(dev, QM_Q_DEPTH, 465416d8220SZaibo Xu SEC_SGL_SGE_NR); 4668a6b8f4dSDan Carpenter if (IS_ERR(qp_ctx->c_out_pool)) { 467416d8220SZaibo Xu dev_err(dev, "fail to create sgl pool for output!\n"); 468416d8220SZaibo Xu goto err_free_c_in_pool; 469416d8220SZaibo Xu } 470416d8220SZaibo Xu 4717c7d902aSZaibo Xu ret = sec_alg_resource_alloc(ctx, qp_ctx); 472416d8220SZaibo Xu if (ret) 473416d8220SZaibo Xu goto err_free_c_out_pool; 474416d8220SZaibo Xu 475416d8220SZaibo Xu ret = hisi_qm_start_qp(qp, 0); 476416d8220SZaibo Xu if (ret < 0) 477416d8220SZaibo Xu goto err_queue_free; 478416d8220SZaibo Xu 479416d8220SZaibo Xu return 0; 480416d8220SZaibo Xu 481416d8220SZaibo Xu err_queue_free: 4827c7d902aSZaibo Xu sec_alg_resource_free(ctx, qp_ctx); 483416d8220SZaibo Xu err_free_c_out_pool: 484416d8220SZaibo Xu hisi_acc_free_sgl_pool(dev, qp_ctx->c_out_pool); 485416d8220SZaibo Xu err_free_c_in_pool: 486416d8220SZaibo Xu hisi_acc_free_sgl_pool(dev, qp_ctx->c_in_pool); 487416d8220SZaibo Xu err_destroy_idr: 488416d8220SZaibo Xu idr_destroy(&qp_ctx->req_idr); 489416d8220SZaibo Xu return ret; 490416d8220SZaibo Xu } 491416d8220SZaibo Xu 492416d8220SZaibo Xu static void sec_release_qp_ctx(struct sec_ctx *ctx, 493416d8220SZaibo Xu struct sec_qp_ctx *qp_ctx) 494416d8220SZaibo Xu { 495a44dce50SLongfang Liu struct device *dev = ctx->dev; 496416d8220SZaibo Xu 497416d8220SZaibo Xu hisi_qm_stop_qp(qp_ctx->qp); 4987c7d902aSZaibo Xu sec_alg_resource_free(ctx, qp_ctx); 499416d8220SZaibo Xu 500416d8220SZaibo Xu hisi_acc_free_sgl_pool(dev, qp_ctx->c_out_pool); 501416d8220SZaibo Xu hisi_acc_free_sgl_pool(dev, qp_ctx->c_in_pool); 502416d8220SZaibo Xu 503416d8220SZaibo Xu idr_destroy(&qp_ctx->req_idr); 504416d8220SZaibo Xu } 505416d8220SZaibo Xu 506473a0f96SZaibo Xu static int sec_ctx_base_init(struct sec_ctx *ctx) 507416d8220SZaibo Xu { 508416d8220SZaibo Xu struct sec_dev *sec; 509416d8220SZaibo Xu int i, ret; 510416d8220SZaibo Xu 5110b5e43bcSKai Ye ctx->qps = sec_create_qps(); 5120b5e43bcSKai Ye if (!ctx->qps) { 5130b5e43bcSKai Ye pr_err("Can not create sec qps!\n"); 514416d8220SZaibo Xu return -ENODEV; 515416d8220SZaibo Xu } 5160b5e43bcSKai Ye 5170b5e43bcSKai Ye sec = container_of(ctx->qps[0]->qm, struct sec_dev, qm); 518416d8220SZaibo Xu ctx->sec = sec; 519a44dce50SLongfang Liu ctx->dev = &sec->qm.pdev->dev; 520a718cfceSZaibo Xu ctx->hlf_q_num = sec->ctx_q_num >> 1; 521416d8220SZaibo Xu 52274b58db8SLongfang Liu ctx->pbuf_supported = ctx->sec->iommu_used; 52374b58db8SLongfang Liu 524416d8220SZaibo Xu /* Half of queue depth is taken as fake requests limit in the queue. */ 525a718cfceSZaibo Xu ctx->fake_req_limit = QM_Q_DEPTH >> 1; 526416d8220SZaibo Xu ctx->qp_ctx = kcalloc(sec->ctx_q_num, sizeof(struct sec_qp_ctx), 527416d8220SZaibo Xu GFP_KERNEL); 52824efcec2SLongfang Liu if (!ctx->qp_ctx) { 52924efcec2SLongfang Liu ret = -ENOMEM; 53024efcec2SLongfang Liu goto err_destroy_qps; 53124efcec2SLongfang Liu } 532416d8220SZaibo Xu 533416d8220SZaibo Xu for (i = 0; i < sec->ctx_q_num; i++) { 534473a0f96SZaibo Xu ret = sec_create_qp_ctx(&sec->qm, ctx, i, 0); 535416d8220SZaibo Xu if (ret) 536416d8220SZaibo Xu goto err_sec_release_qp_ctx; 537416d8220SZaibo Xu } 538416d8220SZaibo Xu 539416d8220SZaibo Xu return 0; 54024efcec2SLongfang Liu 541416d8220SZaibo Xu err_sec_release_qp_ctx: 542416d8220SZaibo Xu for (i = i - 1; i >= 0; i--) 543416d8220SZaibo Xu sec_release_qp_ctx(ctx, &ctx->qp_ctx[i]); 544416d8220SZaibo Xu kfree(ctx->qp_ctx); 54524efcec2SLongfang Liu err_destroy_qps: 54624efcec2SLongfang Liu sec_destroy_qps(ctx->qps, sec->ctx_q_num); 547416d8220SZaibo Xu return ret; 548416d8220SZaibo Xu } 549416d8220SZaibo Xu 550473a0f96SZaibo Xu static void sec_ctx_base_uninit(struct sec_ctx *ctx) 551416d8220SZaibo Xu { 552473a0f96SZaibo Xu int i; 553416d8220SZaibo Xu 554416d8220SZaibo Xu for (i = 0; i < ctx->sec->ctx_q_num; i++) 555416d8220SZaibo Xu sec_release_qp_ctx(ctx, &ctx->qp_ctx[i]); 556416d8220SZaibo Xu 5570b5e43bcSKai Ye sec_destroy_qps(ctx->qps, ctx->sec->ctx_q_num); 558416d8220SZaibo Xu kfree(ctx->qp_ctx); 559416d8220SZaibo Xu } 560416d8220SZaibo Xu 561473a0f96SZaibo Xu static int sec_cipher_init(struct sec_ctx *ctx) 562473a0f96SZaibo Xu { 563473a0f96SZaibo Xu struct sec_cipher_ctx *c_ctx = &ctx->c_ctx; 564473a0f96SZaibo Xu 565a44dce50SLongfang Liu c_ctx->c_key = dma_alloc_coherent(ctx->dev, SEC_MAX_KEY_SIZE, 566473a0f96SZaibo Xu &c_ctx->c_key_dma, GFP_KERNEL); 567473a0f96SZaibo Xu if (!c_ctx->c_key) 568473a0f96SZaibo Xu return -ENOMEM; 569473a0f96SZaibo Xu 570473a0f96SZaibo Xu return 0; 571473a0f96SZaibo Xu } 572473a0f96SZaibo Xu 573473a0f96SZaibo Xu static void sec_cipher_uninit(struct sec_ctx *ctx) 574473a0f96SZaibo Xu { 575473a0f96SZaibo Xu struct sec_cipher_ctx *c_ctx = &ctx->c_ctx; 576473a0f96SZaibo Xu 577473a0f96SZaibo Xu memzero_explicit(c_ctx->c_key, SEC_MAX_KEY_SIZE); 578a44dce50SLongfang Liu dma_free_coherent(ctx->dev, SEC_MAX_KEY_SIZE, 579473a0f96SZaibo Xu c_ctx->c_key, c_ctx->c_key_dma); 580473a0f96SZaibo Xu } 581473a0f96SZaibo Xu 5822f072d75SZaibo Xu static int sec_auth_init(struct sec_ctx *ctx) 5832f072d75SZaibo Xu { 5842f072d75SZaibo Xu struct sec_auth_ctx *a_ctx = &ctx->a_ctx; 5852f072d75SZaibo Xu 586a44dce50SLongfang Liu a_ctx->a_key = dma_alloc_coherent(ctx->dev, SEC_MAX_KEY_SIZE, 5872f072d75SZaibo Xu &a_ctx->a_key_dma, GFP_KERNEL); 5882f072d75SZaibo Xu if (!a_ctx->a_key) 5892f072d75SZaibo Xu return -ENOMEM; 5902f072d75SZaibo Xu 5912f072d75SZaibo Xu return 0; 5922f072d75SZaibo Xu } 5932f072d75SZaibo Xu 5942f072d75SZaibo Xu static void sec_auth_uninit(struct sec_ctx *ctx) 5952f072d75SZaibo Xu { 5962f072d75SZaibo Xu struct sec_auth_ctx *a_ctx = &ctx->a_ctx; 5972f072d75SZaibo Xu 5982f072d75SZaibo Xu memzero_explicit(a_ctx->a_key, SEC_MAX_KEY_SIZE); 599a44dce50SLongfang Liu dma_free_coherent(ctx->dev, SEC_MAX_KEY_SIZE, 6002f072d75SZaibo Xu a_ctx->a_key, a_ctx->a_key_dma); 6012f072d75SZaibo Xu } 6022f072d75SZaibo Xu 6035652d55aSKai Ye static int sec_skcipher_fbtfm_init(struct crypto_skcipher *tfm) 6045652d55aSKai Ye { 6055652d55aSKai Ye const char *alg = crypto_tfm_alg_name(&tfm->base); 6065652d55aSKai Ye struct sec_ctx *ctx = crypto_skcipher_ctx(tfm); 6075652d55aSKai Ye struct sec_cipher_ctx *c_ctx = &ctx->c_ctx; 6085652d55aSKai Ye 6095652d55aSKai Ye c_ctx->fallback = false; 6105652d55aSKai Ye if (likely(strncmp(alg, "xts", SEC_XTS_NAME_SZ))) 6115652d55aSKai Ye return 0; 6125652d55aSKai Ye 6135652d55aSKai Ye c_ctx->fbtfm = crypto_alloc_sync_skcipher(alg, 0, 6145652d55aSKai Ye CRYPTO_ALG_NEED_FALLBACK); 6155652d55aSKai Ye if (IS_ERR(c_ctx->fbtfm)) { 6165652d55aSKai Ye pr_err("failed to alloc fallback tfm!\n"); 6175652d55aSKai Ye return PTR_ERR(c_ctx->fbtfm); 6185652d55aSKai Ye } 6195652d55aSKai Ye 6205652d55aSKai Ye return 0; 6215652d55aSKai Ye } 6225652d55aSKai Ye 623473a0f96SZaibo Xu static int sec_skcipher_init(struct crypto_skcipher *tfm) 624473a0f96SZaibo Xu { 625473a0f96SZaibo Xu struct sec_ctx *ctx = crypto_skcipher_ctx(tfm); 626473a0f96SZaibo Xu int ret; 627473a0f96SZaibo Xu 6282f072d75SZaibo Xu ctx->alg_type = SEC_SKCIPHER; 629473a0f96SZaibo Xu crypto_skcipher_set_reqsize(tfm, sizeof(struct sec_req)); 630473a0f96SZaibo Xu ctx->c_ctx.ivsize = crypto_skcipher_ivsize(tfm); 631473a0f96SZaibo Xu if (ctx->c_ctx.ivsize > SEC_IV_SIZE) { 6324b7aef02SLongfang Liu pr_err("get error skcipher iv size!\n"); 633473a0f96SZaibo Xu return -EINVAL; 634473a0f96SZaibo Xu } 635473a0f96SZaibo Xu 636473a0f96SZaibo Xu ret = sec_ctx_base_init(ctx); 637473a0f96SZaibo Xu if (ret) 638473a0f96SZaibo Xu return ret; 639473a0f96SZaibo Xu 640473a0f96SZaibo Xu ret = sec_cipher_init(ctx); 641473a0f96SZaibo Xu if (ret) 642473a0f96SZaibo Xu goto err_cipher_init; 643473a0f96SZaibo Xu 6445652d55aSKai Ye ret = sec_skcipher_fbtfm_init(tfm); 6455652d55aSKai Ye if (ret) 6465652d55aSKai Ye goto err_fbtfm_init; 6475652d55aSKai Ye 648473a0f96SZaibo Xu return 0; 649633e507fSLongfang Liu 6505652d55aSKai Ye err_fbtfm_init: 6515652d55aSKai Ye sec_cipher_uninit(ctx); 652473a0f96SZaibo Xu err_cipher_init: 653473a0f96SZaibo Xu sec_ctx_base_uninit(ctx); 654473a0f96SZaibo Xu return ret; 655473a0f96SZaibo Xu } 656473a0f96SZaibo Xu 657473a0f96SZaibo Xu static void sec_skcipher_uninit(struct crypto_skcipher *tfm) 658473a0f96SZaibo Xu { 659473a0f96SZaibo Xu struct sec_ctx *ctx = crypto_skcipher_ctx(tfm); 660473a0f96SZaibo Xu 6615652d55aSKai Ye if (ctx->c_ctx.fbtfm) 6625652d55aSKai Ye crypto_free_sync_skcipher(ctx->c_ctx.fbtfm); 6635652d55aSKai Ye 664473a0f96SZaibo Xu sec_cipher_uninit(ctx); 665473a0f96SZaibo Xu sec_ctx_base_uninit(ctx); 666473a0f96SZaibo Xu } 667473a0f96SZaibo Xu 668ae6ce7b1SKai Ye static int sec_skcipher_3des_setkey(struct crypto_skcipher *tfm, const u8 *key, 669416d8220SZaibo Xu const u32 keylen, 670416d8220SZaibo Xu const enum sec_cmode c_mode) 671416d8220SZaibo Xu { 672ae6ce7b1SKai Ye struct sec_ctx *ctx = crypto_skcipher_ctx(tfm); 673ae6ce7b1SKai Ye struct sec_cipher_ctx *c_ctx = &ctx->c_ctx; 674ae6ce7b1SKai Ye int ret; 675ae6ce7b1SKai Ye 676ae6ce7b1SKai Ye ret = verify_skcipher_des3_key(tfm, key); 677ae6ce7b1SKai Ye if (ret) 678ae6ce7b1SKai Ye return ret; 679ae6ce7b1SKai Ye 680416d8220SZaibo Xu switch (keylen) { 681416d8220SZaibo Xu case SEC_DES3_2KEY_SIZE: 682416d8220SZaibo Xu c_ctx->c_key_len = SEC_CKEY_3DES_2KEY; 683416d8220SZaibo Xu break; 684416d8220SZaibo Xu case SEC_DES3_3KEY_SIZE: 685416d8220SZaibo Xu c_ctx->c_key_len = SEC_CKEY_3DES_3KEY; 686416d8220SZaibo Xu break; 687416d8220SZaibo Xu default: 688416d8220SZaibo Xu return -EINVAL; 689416d8220SZaibo Xu } 690416d8220SZaibo Xu 691416d8220SZaibo Xu return 0; 692416d8220SZaibo Xu } 693416d8220SZaibo Xu 694416d8220SZaibo Xu static int sec_skcipher_aes_sm4_setkey(struct sec_cipher_ctx *c_ctx, 695416d8220SZaibo Xu const u32 keylen, 696416d8220SZaibo Xu const enum sec_cmode c_mode) 697416d8220SZaibo Xu { 698416d8220SZaibo Xu if (c_mode == SEC_CMODE_XTS) { 699416d8220SZaibo Xu switch (keylen) { 700416d8220SZaibo Xu case SEC_XTS_MIN_KEY_SIZE: 701416d8220SZaibo Xu c_ctx->c_key_len = SEC_CKEY_128BIT; 702416d8220SZaibo Xu break; 7035652d55aSKai Ye case SEC_XTS_MID_KEY_SIZE: 7045652d55aSKai Ye c_ctx->fallback = true; 7055652d55aSKai Ye break; 706416d8220SZaibo Xu case SEC_XTS_MAX_KEY_SIZE: 707416d8220SZaibo Xu c_ctx->c_key_len = SEC_CKEY_256BIT; 708416d8220SZaibo Xu break; 709416d8220SZaibo Xu default: 710416d8220SZaibo Xu pr_err("hisi_sec2: xts mode key error!\n"); 711416d8220SZaibo Xu return -EINVAL; 712416d8220SZaibo Xu } 713416d8220SZaibo Xu } else { 714adc3f65aSKai Ye if (c_ctx->c_alg == SEC_CALG_SM4 && 715adc3f65aSKai Ye keylen != AES_KEYSIZE_128) { 716adc3f65aSKai Ye pr_err("hisi_sec2: sm4 key error!\n"); 717adc3f65aSKai Ye return -EINVAL; 718adc3f65aSKai Ye } else { 719416d8220SZaibo Xu switch (keylen) { 720416d8220SZaibo Xu case AES_KEYSIZE_128: 721416d8220SZaibo Xu c_ctx->c_key_len = SEC_CKEY_128BIT; 722416d8220SZaibo Xu break; 723416d8220SZaibo Xu case AES_KEYSIZE_192: 724416d8220SZaibo Xu c_ctx->c_key_len = SEC_CKEY_192BIT; 725416d8220SZaibo Xu break; 726416d8220SZaibo Xu case AES_KEYSIZE_256: 727416d8220SZaibo Xu c_ctx->c_key_len = SEC_CKEY_256BIT; 728416d8220SZaibo Xu break; 729416d8220SZaibo Xu default: 730416d8220SZaibo Xu pr_err("hisi_sec2: aes key error!\n"); 731416d8220SZaibo Xu return -EINVAL; 732416d8220SZaibo Xu } 733416d8220SZaibo Xu } 734adc3f65aSKai Ye } 735416d8220SZaibo Xu 736416d8220SZaibo Xu return 0; 737416d8220SZaibo Xu } 738416d8220SZaibo Xu 739416d8220SZaibo Xu static int sec_skcipher_setkey(struct crypto_skcipher *tfm, const u8 *key, 740416d8220SZaibo Xu const u32 keylen, const enum sec_calg c_alg, 741416d8220SZaibo Xu const enum sec_cmode c_mode) 742416d8220SZaibo Xu { 743416d8220SZaibo Xu struct sec_ctx *ctx = crypto_skcipher_ctx(tfm); 744416d8220SZaibo Xu struct sec_cipher_ctx *c_ctx = &ctx->c_ctx; 745a44dce50SLongfang Liu struct device *dev = ctx->dev; 746416d8220SZaibo Xu int ret; 747416d8220SZaibo Xu 748416d8220SZaibo Xu if (c_mode == SEC_CMODE_XTS) { 749416d8220SZaibo Xu ret = xts_verify_key(tfm, key, keylen); 750416d8220SZaibo Xu if (ret) { 751a44dce50SLongfang Liu dev_err(dev, "xts mode key err!\n"); 752416d8220SZaibo Xu return ret; 753416d8220SZaibo Xu } 754416d8220SZaibo Xu } 755416d8220SZaibo Xu 756416d8220SZaibo Xu c_ctx->c_alg = c_alg; 757416d8220SZaibo Xu c_ctx->c_mode = c_mode; 758416d8220SZaibo Xu 759416d8220SZaibo Xu switch (c_alg) { 760416d8220SZaibo Xu case SEC_CALG_3DES: 761ae6ce7b1SKai Ye ret = sec_skcipher_3des_setkey(tfm, key, keylen, c_mode); 762416d8220SZaibo Xu break; 763416d8220SZaibo Xu case SEC_CALG_AES: 764416d8220SZaibo Xu case SEC_CALG_SM4: 765416d8220SZaibo Xu ret = sec_skcipher_aes_sm4_setkey(c_ctx, keylen, c_mode); 766416d8220SZaibo Xu break; 767416d8220SZaibo Xu default: 768416d8220SZaibo Xu return -EINVAL; 769416d8220SZaibo Xu } 770416d8220SZaibo Xu 771416d8220SZaibo Xu if (ret) { 772a44dce50SLongfang Liu dev_err(dev, "set sec key err!\n"); 773416d8220SZaibo Xu return ret; 774416d8220SZaibo Xu } 775416d8220SZaibo Xu 776416d8220SZaibo Xu memcpy(c_ctx->c_key, key, keylen); 7775652d55aSKai Ye if (c_ctx->fallback) { 7785652d55aSKai Ye ret = crypto_sync_skcipher_setkey(c_ctx->fbtfm, key, keylen); 7795652d55aSKai Ye if (ret) { 7805652d55aSKai Ye dev_err(dev, "failed to set fallback skcipher key!\n"); 7815652d55aSKai Ye return ret; 7825652d55aSKai Ye } 7835652d55aSKai Ye } 784416d8220SZaibo Xu return 0; 785416d8220SZaibo Xu } 786416d8220SZaibo Xu 787416d8220SZaibo Xu #define GEN_SEC_SETKEY_FUNC(name, c_alg, c_mode) \ 788416d8220SZaibo Xu static int sec_setkey_##name(struct crypto_skcipher *tfm, const u8 *key,\ 789416d8220SZaibo Xu u32 keylen) \ 790416d8220SZaibo Xu { \ 791416d8220SZaibo Xu return sec_skcipher_setkey(tfm, key, keylen, c_alg, c_mode); \ 792416d8220SZaibo Xu } 793416d8220SZaibo Xu 794416d8220SZaibo Xu GEN_SEC_SETKEY_FUNC(aes_ecb, SEC_CALG_AES, SEC_CMODE_ECB) 795416d8220SZaibo Xu GEN_SEC_SETKEY_FUNC(aes_cbc, SEC_CALG_AES, SEC_CMODE_CBC) 796416d8220SZaibo Xu GEN_SEC_SETKEY_FUNC(aes_xts, SEC_CALG_AES, SEC_CMODE_XTS) 7977b44c0eeSKai Ye GEN_SEC_SETKEY_FUNC(aes_ofb, SEC_CALG_AES, SEC_CMODE_OFB) 7987b44c0eeSKai Ye GEN_SEC_SETKEY_FUNC(aes_cfb, SEC_CALG_AES, SEC_CMODE_CFB) 7997b44c0eeSKai Ye GEN_SEC_SETKEY_FUNC(aes_ctr, SEC_CALG_AES, SEC_CMODE_CTR) 800416d8220SZaibo Xu GEN_SEC_SETKEY_FUNC(3des_ecb, SEC_CALG_3DES, SEC_CMODE_ECB) 801416d8220SZaibo Xu GEN_SEC_SETKEY_FUNC(3des_cbc, SEC_CALG_3DES, SEC_CMODE_CBC) 802416d8220SZaibo Xu GEN_SEC_SETKEY_FUNC(sm4_xts, SEC_CALG_SM4, SEC_CMODE_XTS) 803416d8220SZaibo Xu GEN_SEC_SETKEY_FUNC(sm4_cbc, SEC_CALG_SM4, SEC_CMODE_CBC) 8047b44c0eeSKai Ye GEN_SEC_SETKEY_FUNC(sm4_ofb, SEC_CALG_SM4, SEC_CMODE_OFB) 8057b44c0eeSKai Ye GEN_SEC_SETKEY_FUNC(sm4_cfb, SEC_CALG_SM4, SEC_CMODE_CFB) 8067b44c0eeSKai Ye GEN_SEC_SETKEY_FUNC(sm4_ctr, SEC_CALG_SM4, SEC_CMODE_CTR) 807416d8220SZaibo Xu 80874b58db8SLongfang Liu static int sec_cipher_pbuf_map(struct sec_ctx *ctx, struct sec_req *req, 80974b58db8SLongfang Liu struct scatterlist *src) 81074b58db8SLongfang Liu { 81174b58db8SLongfang Liu struct aead_request *aead_req = req->aead_req.aead_req; 81274b58db8SLongfang Liu struct sec_cipher_req *c_req = &req->c_req; 81374b58db8SLongfang Liu struct sec_qp_ctx *qp_ctx = req->qp_ctx; 814a44dce50SLongfang Liu struct device *dev = ctx->dev; 81574b58db8SLongfang Liu int copy_size, pbuf_length; 81674b58db8SLongfang Liu int req_id = req->req_id; 81774b58db8SLongfang Liu 81874b58db8SLongfang Liu if (ctx->alg_type == SEC_AEAD) 81974b58db8SLongfang Liu copy_size = aead_req->cryptlen + aead_req->assoclen; 82074b58db8SLongfang Liu else 82174b58db8SLongfang Liu copy_size = c_req->c_len; 82274b58db8SLongfang Liu 82374b58db8SLongfang Liu pbuf_length = sg_copy_to_buffer(src, sg_nents(src), 82474b58db8SLongfang Liu qp_ctx->res[req_id].pbuf, 82574b58db8SLongfang Liu copy_size); 82674b58db8SLongfang Liu if (unlikely(pbuf_length != copy_size)) { 82774b58db8SLongfang Liu dev_err(dev, "copy src data to pbuf error!\n"); 82874b58db8SLongfang Liu return -EINVAL; 82974b58db8SLongfang Liu } 83074b58db8SLongfang Liu 83174b58db8SLongfang Liu c_req->c_in_dma = qp_ctx->res[req_id].pbuf_dma; 83274b58db8SLongfang Liu c_req->c_out_dma = c_req->c_in_dma; 83374b58db8SLongfang Liu 83474b58db8SLongfang Liu return 0; 83574b58db8SLongfang Liu } 83674b58db8SLongfang Liu 83774b58db8SLongfang Liu static void sec_cipher_pbuf_unmap(struct sec_ctx *ctx, struct sec_req *req, 83874b58db8SLongfang Liu struct scatterlist *dst) 83974b58db8SLongfang Liu { 84074b58db8SLongfang Liu struct aead_request *aead_req = req->aead_req.aead_req; 84174b58db8SLongfang Liu struct sec_cipher_req *c_req = &req->c_req; 84274b58db8SLongfang Liu struct sec_qp_ctx *qp_ctx = req->qp_ctx; 843a44dce50SLongfang Liu struct device *dev = ctx->dev; 84474b58db8SLongfang Liu int copy_size, pbuf_length; 84574b58db8SLongfang Liu int req_id = req->req_id; 84674b58db8SLongfang Liu 84774b58db8SLongfang Liu if (ctx->alg_type == SEC_AEAD) 84874b58db8SLongfang Liu copy_size = c_req->c_len + aead_req->assoclen; 84974b58db8SLongfang Liu else 85074b58db8SLongfang Liu copy_size = c_req->c_len; 85174b58db8SLongfang Liu 85274b58db8SLongfang Liu pbuf_length = sg_copy_from_buffer(dst, sg_nents(dst), 85374b58db8SLongfang Liu qp_ctx->res[req_id].pbuf, 85474b58db8SLongfang Liu copy_size); 85574b58db8SLongfang Liu if (unlikely(pbuf_length != copy_size)) 85674b58db8SLongfang Liu dev_err(dev, "copy pbuf data to dst error!\n"); 85774b58db8SLongfang Liu } 85874b58db8SLongfang Liu 8592514f559SLongfang Liu static int sec_cipher_map(struct sec_ctx *ctx, struct sec_req *req, 860416d8220SZaibo Xu struct scatterlist *src, struct scatterlist *dst) 861416d8220SZaibo Xu { 862416d8220SZaibo Xu struct sec_cipher_req *c_req = &req->c_req; 8632514f559SLongfang Liu struct sec_aead_req *a_req = &req->aead_req; 864416d8220SZaibo Xu struct sec_qp_ctx *qp_ctx = req->qp_ctx; 8652514f559SLongfang Liu struct sec_alg_res *res = &qp_ctx->res[req->req_id]; 866a44dce50SLongfang Liu struct device *dev = ctx->dev; 86774b58db8SLongfang Liu int ret; 8682514f559SLongfang Liu 86974b58db8SLongfang Liu if (req->use_pbuf) { 87074b58db8SLongfang Liu ret = sec_cipher_pbuf_map(ctx, req, src); 87174b58db8SLongfang Liu c_req->c_ivin = res->pbuf + SEC_PBUF_IV_OFFSET; 87274b58db8SLongfang Liu c_req->c_ivin_dma = res->pbuf_dma + SEC_PBUF_IV_OFFSET; 87374b58db8SLongfang Liu if (ctx->alg_type == SEC_AEAD) { 87474b58db8SLongfang Liu a_req->out_mac = res->pbuf + SEC_PBUF_MAC_OFFSET; 87574b58db8SLongfang Liu a_req->out_mac_dma = res->pbuf_dma + 87674b58db8SLongfang Liu SEC_PBUF_MAC_OFFSET; 87774b58db8SLongfang Liu } 87874b58db8SLongfang Liu 87974b58db8SLongfang Liu return ret; 88074b58db8SLongfang Liu } 8812514f559SLongfang Liu c_req->c_ivin = res->c_ivin; 8822514f559SLongfang Liu c_req->c_ivin_dma = res->c_ivin_dma; 8832514f559SLongfang Liu if (ctx->alg_type == SEC_AEAD) { 8842514f559SLongfang Liu a_req->out_mac = res->out_mac; 8852514f559SLongfang Liu a_req->out_mac_dma = res->out_mac_dma; 8862514f559SLongfang Liu } 887416d8220SZaibo Xu 888416d8220SZaibo Xu c_req->c_in = hisi_acc_sg_buf_map_to_hw_sgl(dev, src, 889416d8220SZaibo Xu qp_ctx->c_in_pool, 890416d8220SZaibo Xu req->req_id, 891416d8220SZaibo Xu &c_req->c_in_dma); 892416d8220SZaibo Xu 893416d8220SZaibo Xu if (IS_ERR(c_req->c_in)) { 894416d8220SZaibo Xu dev_err(dev, "fail to dma map input sgl buffers!\n"); 895416d8220SZaibo Xu return PTR_ERR(c_req->c_in); 896416d8220SZaibo Xu } 897416d8220SZaibo Xu 898416d8220SZaibo Xu if (dst == src) { 899416d8220SZaibo Xu c_req->c_out = c_req->c_in; 900416d8220SZaibo Xu c_req->c_out_dma = c_req->c_in_dma; 901416d8220SZaibo Xu } else { 902416d8220SZaibo Xu c_req->c_out = hisi_acc_sg_buf_map_to_hw_sgl(dev, dst, 903416d8220SZaibo Xu qp_ctx->c_out_pool, 904416d8220SZaibo Xu req->req_id, 905416d8220SZaibo Xu &c_req->c_out_dma); 906416d8220SZaibo Xu 907416d8220SZaibo Xu if (IS_ERR(c_req->c_out)) { 908416d8220SZaibo Xu dev_err(dev, "fail to dma map output sgl buffers!\n"); 909416d8220SZaibo Xu hisi_acc_sg_buf_unmap(dev, src, c_req->c_in); 910416d8220SZaibo Xu return PTR_ERR(c_req->c_out); 911416d8220SZaibo Xu } 912416d8220SZaibo Xu } 913416d8220SZaibo Xu 914416d8220SZaibo Xu return 0; 915416d8220SZaibo Xu } 916416d8220SZaibo Xu 9172514f559SLongfang Liu static void sec_cipher_unmap(struct sec_ctx *ctx, struct sec_req *req, 918a181647cSZaibo Xu struct scatterlist *src, struct scatterlist *dst) 919a181647cSZaibo Xu { 9202514f559SLongfang Liu struct sec_cipher_req *c_req = &req->c_req; 921a44dce50SLongfang Liu struct device *dev = ctx->dev; 922a181647cSZaibo Xu 92374b58db8SLongfang Liu if (req->use_pbuf) { 92474b58db8SLongfang Liu sec_cipher_pbuf_unmap(ctx, req, dst); 92574b58db8SLongfang Liu } else { 9262514f559SLongfang Liu if (dst != src) 9272514f559SLongfang Liu hisi_acc_sg_buf_unmap(dev, src, c_req->c_in); 9282514f559SLongfang Liu 9292514f559SLongfang Liu hisi_acc_sg_buf_unmap(dev, dst, c_req->c_out); 930a181647cSZaibo Xu } 93174b58db8SLongfang Liu } 932a181647cSZaibo Xu 933416d8220SZaibo Xu static int sec_skcipher_sgl_map(struct sec_ctx *ctx, struct sec_req *req) 934416d8220SZaibo Xu { 935a181647cSZaibo Xu struct skcipher_request *sq = req->c_req.sk_req; 936416d8220SZaibo Xu 9372514f559SLongfang Liu return sec_cipher_map(ctx, req, sq->src, sq->dst); 938416d8220SZaibo Xu } 939416d8220SZaibo Xu 940416d8220SZaibo Xu static void sec_skcipher_sgl_unmap(struct sec_ctx *ctx, struct sec_req *req) 941416d8220SZaibo Xu { 9422514f559SLongfang Liu struct skcipher_request *sq = req->c_req.sk_req; 943416d8220SZaibo Xu 9442514f559SLongfang Liu sec_cipher_unmap(ctx, req, sq->src, sq->dst); 945416d8220SZaibo Xu } 946416d8220SZaibo Xu 9472f072d75SZaibo Xu static int sec_aead_aes_set_key(struct sec_cipher_ctx *c_ctx, 9482f072d75SZaibo Xu struct crypto_authenc_keys *keys) 9492f072d75SZaibo Xu { 9502f072d75SZaibo Xu switch (keys->enckeylen) { 9512f072d75SZaibo Xu case AES_KEYSIZE_128: 9522f072d75SZaibo Xu c_ctx->c_key_len = SEC_CKEY_128BIT; 9532f072d75SZaibo Xu break; 9542f072d75SZaibo Xu case AES_KEYSIZE_192: 9552f072d75SZaibo Xu c_ctx->c_key_len = SEC_CKEY_192BIT; 9562f072d75SZaibo Xu break; 9572f072d75SZaibo Xu case AES_KEYSIZE_256: 9582f072d75SZaibo Xu c_ctx->c_key_len = SEC_CKEY_256BIT; 9592f072d75SZaibo Xu break; 9602f072d75SZaibo Xu default: 9612f072d75SZaibo Xu pr_err("hisi_sec2: aead aes key error!\n"); 9622f072d75SZaibo Xu return -EINVAL; 9632f072d75SZaibo Xu } 9642f072d75SZaibo Xu memcpy(c_ctx->c_key, keys->enckey, keys->enckeylen); 9652f072d75SZaibo Xu 9662f072d75SZaibo Xu return 0; 9672f072d75SZaibo Xu } 9682f072d75SZaibo Xu 9692f072d75SZaibo Xu static int sec_aead_auth_set_key(struct sec_auth_ctx *ctx, 9702f072d75SZaibo Xu struct crypto_authenc_keys *keys) 9712f072d75SZaibo Xu { 9722f072d75SZaibo Xu struct crypto_shash *hash_tfm = ctx->hash_tfm; 9735761498cSKai Ye int blocksize, digestsize, ret; 9742f072d75SZaibo Xu 9752f072d75SZaibo Xu if (!keys->authkeylen) { 9762f072d75SZaibo Xu pr_err("hisi_sec2: aead auth key error!\n"); 9772f072d75SZaibo Xu return -EINVAL; 9782f072d75SZaibo Xu } 9792f072d75SZaibo Xu 9802f072d75SZaibo Xu blocksize = crypto_shash_blocksize(hash_tfm); 9815761498cSKai Ye digestsize = crypto_shash_digestsize(hash_tfm); 9822f072d75SZaibo Xu if (keys->authkeylen > blocksize) { 98361c38e3aSEric Biggers ret = crypto_shash_tfm_digest(hash_tfm, keys->authkey, 9842f072d75SZaibo Xu keys->authkeylen, ctx->a_key); 9852f072d75SZaibo Xu if (ret) { 9862203d3f7SColin Ian King pr_err("hisi_sec2: aead auth digest error!\n"); 9872f072d75SZaibo Xu return -EINVAL; 9882f072d75SZaibo Xu } 9895761498cSKai Ye ctx->a_key_len = digestsize; 9902f072d75SZaibo Xu } else { 9912f072d75SZaibo Xu memcpy(ctx->a_key, keys->authkey, keys->authkeylen); 9922f072d75SZaibo Xu ctx->a_key_len = keys->authkeylen; 9932f072d75SZaibo Xu } 9942f072d75SZaibo Xu 9952f072d75SZaibo Xu return 0; 9962f072d75SZaibo Xu } 9972f072d75SZaibo Xu 9982f072d75SZaibo Xu static int sec_aead_setkey(struct crypto_aead *tfm, const u8 *key, 9992f072d75SZaibo Xu const u32 keylen, const enum sec_hash_alg a_alg, 10002f072d75SZaibo Xu const enum sec_calg c_alg, 10012f072d75SZaibo Xu const enum sec_mac_len mac_len, 10022f072d75SZaibo Xu const enum sec_cmode c_mode) 10032f072d75SZaibo Xu { 10042f072d75SZaibo Xu struct sec_ctx *ctx = crypto_aead_ctx(tfm); 10052f072d75SZaibo Xu struct sec_cipher_ctx *c_ctx = &ctx->c_ctx; 1006a44dce50SLongfang Liu struct device *dev = ctx->dev; 10072f072d75SZaibo Xu struct crypto_authenc_keys keys; 10082f072d75SZaibo Xu int ret; 10092f072d75SZaibo Xu 10102f072d75SZaibo Xu ctx->a_ctx.a_alg = a_alg; 10112f072d75SZaibo Xu ctx->c_ctx.c_alg = c_alg; 10122f072d75SZaibo Xu ctx->a_ctx.mac_len = mac_len; 10132f072d75SZaibo Xu c_ctx->c_mode = c_mode; 10142f072d75SZaibo Xu 10152f072d75SZaibo Xu if (crypto_authenc_extractkeys(&keys, key, keylen)) 10162f072d75SZaibo Xu goto bad_key; 10172f072d75SZaibo Xu 10182f072d75SZaibo Xu ret = sec_aead_aes_set_key(c_ctx, &keys); 10192f072d75SZaibo Xu if (ret) { 1020a44dce50SLongfang Liu dev_err(dev, "set sec cipher key err!\n"); 10212f072d75SZaibo Xu goto bad_key; 10222f072d75SZaibo Xu } 10232f072d75SZaibo Xu 10242f072d75SZaibo Xu ret = sec_aead_auth_set_key(&ctx->a_ctx, &keys); 10252f072d75SZaibo Xu if (ret) { 1026a44dce50SLongfang Liu dev_err(dev, "set sec auth key err!\n"); 10272f072d75SZaibo Xu goto bad_key; 10282f072d75SZaibo Xu } 10292f072d75SZaibo Xu 1030adc3f65aSKai Ye if ((ctx->a_ctx.mac_len & SEC_SQE_LEN_RATE_MASK) || 1031adc3f65aSKai Ye (ctx->a_ctx.a_key_len & SEC_SQE_LEN_RATE_MASK)) { 1032adc3f65aSKai Ye dev_err(dev, "MAC or AUTH key length error!\n"); 1033adc3f65aSKai Ye goto bad_key; 1034adc3f65aSKai Ye } 1035adc3f65aSKai Ye 10362f072d75SZaibo Xu return 0; 1037633e507fSLongfang Liu 10382f072d75SZaibo Xu bad_key: 10392f072d75SZaibo Xu memzero_explicit(&keys, sizeof(struct crypto_authenc_keys)); 10402f072d75SZaibo Xu return -EINVAL; 10412f072d75SZaibo Xu } 10422f072d75SZaibo Xu 10432f072d75SZaibo Xu 10442f072d75SZaibo Xu #define GEN_SEC_AEAD_SETKEY_FUNC(name, aalg, calg, maclen, cmode) \ 10452f072d75SZaibo Xu static int sec_setkey_##name(struct crypto_aead *tfm, const u8 *key, \ 10462f072d75SZaibo Xu u32 keylen) \ 10472f072d75SZaibo Xu { \ 10482f072d75SZaibo Xu return sec_aead_setkey(tfm, key, keylen, aalg, calg, maclen, cmode);\ 10492f072d75SZaibo Xu } 10502f072d75SZaibo Xu 10512f072d75SZaibo Xu GEN_SEC_AEAD_SETKEY_FUNC(aes_cbc_sha1, SEC_A_HMAC_SHA1, 10522f072d75SZaibo Xu SEC_CALG_AES, SEC_HMAC_SHA1_MAC, SEC_CMODE_CBC) 10532f072d75SZaibo Xu GEN_SEC_AEAD_SETKEY_FUNC(aes_cbc_sha256, SEC_A_HMAC_SHA256, 10542f072d75SZaibo Xu SEC_CALG_AES, SEC_HMAC_SHA256_MAC, SEC_CMODE_CBC) 10552f072d75SZaibo Xu GEN_SEC_AEAD_SETKEY_FUNC(aes_cbc_sha512, SEC_A_HMAC_SHA512, 10562f072d75SZaibo Xu SEC_CALG_AES, SEC_HMAC_SHA512_MAC, SEC_CMODE_CBC) 10572f072d75SZaibo Xu 10582f072d75SZaibo Xu static int sec_aead_sgl_map(struct sec_ctx *ctx, struct sec_req *req) 10592f072d75SZaibo Xu { 10602f072d75SZaibo Xu struct aead_request *aq = req->aead_req.aead_req; 10612f072d75SZaibo Xu 10622514f559SLongfang Liu return sec_cipher_map(ctx, req, aq->src, aq->dst); 10632f072d75SZaibo Xu } 10642f072d75SZaibo Xu 10652f072d75SZaibo Xu static void sec_aead_sgl_unmap(struct sec_ctx *ctx, struct sec_req *req) 10662f072d75SZaibo Xu { 10672f072d75SZaibo Xu struct aead_request *aq = req->aead_req.aead_req; 10682f072d75SZaibo Xu 10692514f559SLongfang Liu sec_cipher_unmap(ctx, req, aq->src, aq->dst); 10702f072d75SZaibo Xu } 10712f072d75SZaibo Xu 1072416d8220SZaibo Xu static int sec_request_transfer(struct sec_ctx *ctx, struct sec_req *req) 1073416d8220SZaibo Xu { 1074416d8220SZaibo Xu int ret; 1075416d8220SZaibo Xu 1076416d8220SZaibo Xu ret = ctx->req_op->buf_map(ctx, req); 1077b9c8d897SZaibo Xu if (unlikely(ret)) 1078416d8220SZaibo Xu return ret; 1079416d8220SZaibo Xu 1080416d8220SZaibo Xu ctx->req_op->do_transfer(ctx, req); 1081416d8220SZaibo Xu 1082416d8220SZaibo Xu ret = ctx->req_op->bd_fill(ctx, req); 1083b9c8d897SZaibo Xu if (unlikely(ret)) 1084416d8220SZaibo Xu goto unmap_req_buf; 1085416d8220SZaibo Xu 1086416d8220SZaibo Xu return ret; 1087416d8220SZaibo Xu 1088416d8220SZaibo Xu unmap_req_buf: 1089416d8220SZaibo Xu ctx->req_op->buf_unmap(ctx, req); 1090416d8220SZaibo Xu return ret; 1091416d8220SZaibo Xu } 1092416d8220SZaibo Xu 1093416d8220SZaibo Xu static void sec_request_untransfer(struct sec_ctx *ctx, struct sec_req *req) 1094416d8220SZaibo Xu { 1095416d8220SZaibo Xu ctx->req_op->buf_unmap(ctx, req); 1096416d8220SZaibo Xu } 1097416d8220SZaibo Xu 1098416d8220SZaibo Xu static void sec_skcipher_copy_iv(struct sec_ctx *ctx, struct sec_req *req) 1099416d8220SZaibo Xu { 1100416d8220SZaibo Xu struct skcipher_request *sk_req = req->c_req.sk_req; 11012514f559SLongfang Liu struct sec_cipher_req *c_req = &req->c_req; 1102416d8220SZaibo Xu 11032514f559SLongfang Liu memcpy(c_req->c_ivin, sk_req->iv, ctx->c_ctx.ivsize); 1104416d8220SZaibo Xu } 1105416d8220SZaibo Xu 1106416d8220SZaibo Xu static int sec_skcipher_bd_fill(struct sec_ctx *ctx, struct sec_req *req) 1107416d8220SZaibo Xu { 1108416d8220SZaibo Xu struct sec_cipher_ctx *c_ctx = &ctx->c_ctx; 1109416d8220SZaibo Xu struct sec_cipher_req *c_req = &req->c_req; 1110416d8220SZaibo Xu struct sec_sqe *sec_sqe = &req->sec_sqe; 1111416d8220SZaibo Xu u8 scene, sa_type, da_type; 1112416d8220SZaibo Xu u8 bd_type, cipher; 11137c7d902aSZaibo Xu u8 de = 0; 1114416d8220SZaibo Xu 1115416d8220SZaibo Xu memset(sec_sqe, 0, sizeof(struct sec_sqe)); 1116416d8220SZaibo Xu 1117416d8220SZaibo Xu sec_sqe->type2.c_key_addr = cpu_to_le64(c_ctx->c_key_dma); 11182514f559SLongfang Liu sec_sqe->type2.c_ivin_addr = cpu_to_le64(c_req->c_ivin_dma); 1119416d8220SZaibo Xu sec_sqe->type2.data_src_addr = cpu_to_le64(c_req->c_in_dma); 1120416d8220SZaibo Xu sec_sqe->type2.data_dst_addr = cpu_to_le64(c_req->c_out_dma); 1121416d8220SZaibo Xu 1122416d8220SZaibo Xu sec_sqe->type2.icvw_kmode |= cpu_to_le16(((u16)c_ctx->c_mode) << 1123416d8220SZaibo Xu SEC_CMODE_OFFSET); 1124416d8220SZaibo Xu sec_sqe->type2.c_alg = c_ctx->c_alg; 1125416d8220SZaibo Xu sec_sqe->type2.icvw_kmode |= cpu_to_le16(((u16)c_ctx->c_key_len) << 1126416d8220SZaibo Xu SEC_CKEY_OFFSET); 1127416d8220SZaibo Xu 1128416d8220SZaibo Xu bd_type = SEC_BD_TYPE2; 1129416d8220SZaibo Xu if (c_req->encrypt) 1130416d8220SZaibo Xu cipher = SEC_CIPHER_ENC << SEC_CIPHER_OFFSET; 1131416d8220SZaibo Xu else 1132416d8220SZaibo Xu cipher = SEC_CIPHER_DEC << SEC_CIPHER_OFFSET; 1133416d8220SZaibo Xu sec_sqe->type_cipher_auth = bd_type | cipher; 1134416d8220SZaibo Xu 1135adc3f65aSKai Ye /* Set destination and source address type */ 1136adc3f65aSKai Ye if (req->use_pbuf) { 113774b58db8SLongfang Liu sa_type = SEC_PBUF << SEC_SRC_SGL_OFFSET; 1138adc3f65aSKai Ye da_type = SEC_PBUF << SEC_DST_SGL_OFFSET; 1139adc3f65aSKai Ye } else { 1140416d8220SZaibo Xu sa_type = SEC_SGL << SEC_SRC_SGL_OFFSET; 1141adc3f65aSKai Ye da_type = SEC_SGL << SEC_DST_SGL_OFFSET; 1142adc3f65aSKai Ye } 1143adc3f65aSKai Ye 1144adc3f65aSKai Ye sec_sqe->sdm_addr_type |= da_type; 1145416d8220SZaibo Xu scene = SEC_COMM_SCENE << SEC_SCENE_OFFSET; 1146416d8220SZaibo Xu if (c_req->c_in_dma != c_req->c_out_dma) 1147416d8220SZaibo Xu de = 0x1 << SEC_DE_OFFSET; 1148416d8220SZaibo Xu 1149416d8220SZaibo Xu sec_sqe->sds_sa_type = (de | scene | sa_type); 1150416d8220SZaibo Xu 1151416d8220SZaibo Xu sec_sqe->type2.clen_ivhlen |= cpu_to_le32(c_req->c_len); 1152416d8220SZaibo Xu sec_sqe->type2.tag = cpu_to_le16((u16)req->req_id); 1153416d8220SZaibo Xu 1154416d8220SZaibo Xu return 0; 1155416d8220SZaibo Xu } 1156416d8220SZaibo Xu 1157adc3f65aSKai Ye static int sec_skcipher_bd_fill_v3(struct sec_ctx *ctx, struct sec_req *req) 1158adc3f65aSKai Ye { 1159adc3f65aSKai Ye struct sec_sqe3 *sec_sqe3 = &req->sec_sqe3; 1160adc3f65aSKai Ye struct sec_cipher_ctx *c_ctx = &ctx->c_ctx; 1161adc3f65aSKai Ye struct sec_cipher_req *c_req = &req->c_req; 1162adc3f65aSKai Ye u32 bd_param = 0; 1163adc3f65aSKai Ye u16 cipher; 1164adc3f65aSKai Ye 1165adc3f65aSKai Ye memset(sec_sqe3, 0, sizeof(struct sec_sqe3)); 1166adc3f65aSKai Ye 1167adc3f65aSKai Ye sec_sqe3->c_key_addr = cpu_to_le64(c_ctx->c_key_dma); 1168adc3f65aSKai Ye sec_sqe3->no_scene.c_ivin_addr = cpu_to_le64(c_req->c_ivin_dma); 1169adc3f65aSKai Ye sec_sqe3->data_src_addr = cpu_to_le64(c_req->c_in_dma); 1170adc3f65aSKai Ye sec_sqe3->data_dst_addr = cpu_to_le64(c_req->c_out_dma); 1171adc3f65aSKai Ye 1172adc3f65aSKai Ye sec_sqe3->c_mode_alg = ((u8)c_ctx->c_alg << SEC_CALG_OFFSET_V3) | 1173adc3f65aSKai Ye c_ctx->c_mode; 1174adc3f65aSKai Ye sec_sqe3->c_icv_key |= cpu_to_le16(((u16)c_ctx->c_key_len) << 1175adc3f65aSKai Ye SEC_CKEY_OFFSET_V3); 1176adc3f65aSKai Ye 1177adc3f65aSKai Ye if (c_req->encrypt) 1178adc3f65aSKai Ye cipher = SEC_CIPHER_ENC; 1179adc3f65aSKai Ye else 1180adc3f65aSKai Ye cipher = SEC_CIPHER_DEC; 1181adc3f65aSKai Ye sec_sqe3->c_icv_key |= cpu_to_le16(cipher); 1182adc3f65aSKai Ye 1183adc3f65aSKai Ye if (req->use_pbuf) { 1184adc3f65aSKai Ye bd_param |= SEC_PBUF << SEC_SRC_SGL_OFFSET_V3; 1185adc3f65aSKai Ye bd_param |= SEC_PBUF << SEC_DST_SGL_OFFSET_V3; 1186adc3f65aSKai Ye } else { 1187adc3f65aSKai Ye bd_param |= SEC_SGL << SEC_SRC_SGL_OFFSET_V3; 1188adc3f65aSKai Ye bd_param |= SEC_SGL << SEC_DST_SGL_OFFSET_V3; 1189adc3f65aSKai Ye } 1190adc3f65aSKai Ye 1191adc3f65aSKai Ye bd_param |= SEC_COMM_SCENE << SEC_SCENE_OFFSET_V3; 1192adc3f65aSKai Ye if (c_req->c_in_dma != c_req->c_out_dma) 1193adc3f65aSKai Ye bd_param |= 0x1 << SEC_DE_OFFSET_V3; 1194adc3f65aSKai Ye 1195adc3f65aSKai Ye bd_param |= SEC_BD_TYPE3; 1196adc3f65aSKai Ye sec_sqe3->bd_param = cpu_to_le32(bd_param); 1197adc3f65aSKai Ye 1198adc3f65aSKai Ye sec_sqe3->c_len_ivin |= cpu_to_le32(c_req->c_len); 1199adc3f65aSKai Ye sec_sqe3->tag = cpu_to_le64(req); 1200adc3f65aSKai Ye 1201adc3f65aSKai Ye return 0; 1202adc3f65aSKai Ye } 1203adc3f65aSKai Ye 12047b44c0eeSKai Ye /* increment counter (128-bit int) */ 12057b44c0eeSKai Ye static void ctr_iv_inc(__u8 *counter, __u8 bits, __u32 nums) 12067b44c0eeSKai Ye { 12077b44c0eeSKai Ye do { 12087b44c0eeSKai Ye --bits; 12097b44c0eeSKai Ye nums += counter[bits]; 12107b44c0eeSKai Ye counter[bits] = nums & BITS_MASK; 12117b44c0eeSKai Ye nums >>= BYTE_BITS; 12127b44c0eeSKai Ye } while (bits && nums); 12137b44c0eeSKai Ye } 12147b44c0eeSKai Ye 12152f072d75SZaibo Xu static void sec_update_iv(struct sec_req *req, enum sec_alg_type alg_type) 1216416d8220SZaibo Xu { 12172f072d75SZaibo Xu struct aead_request *aead_req = req->aead_req.aead_req; 1218416d8220SZaibo Xu struct skcipher_request *sk_req = req->c_req.sk_req; 1219416d8220SZaibo Xu u32 iv_size = req->ctx->c_ctx.ivsize; 1220416d8220SZaibo Xu struct scatterlist *sgl; 12212f072d75SZaibo Xu unsigned int cryptlen; 1222416d8220SZaibo Xu size_t sz; 12232f072d75SZaibo Xu u8 *iv; 1224416d8220SZaibo Xu 1225416d8220SZaibo Xu if (req->c_req.encrypt) 12262f072d75SZaibo Xu sgl = alg_type == SEC_SKCIPHER ? sk_req->dst : aead_req->dst; 1227416d8220SZaibo Xu else 12282f072d75SZaibo Xu sgl = alg_type == SEC_SKCIPHER ? sk_req->src : aead_req->src; 1229416d8220SZaibo Xu 12302f072d75SZaibo Xu if (alg_type == SEC_SKCIPHER) { 12312f072d75SZaibo Xu iv = sk_req->iv; 12322f072d75SZaibo Xu cryptlen = sk_req->cryptlen; 12332f072d75SZaibo Xu } else { 12342f072d75SZaibo Xu iv = aead_req->iv; 12352f072d75SZaibo Xu cryptlen = aead_req->cryptlen; 12362f072d75SZaibo Xu } 12372f072d75SZaibo Xu 12387b44c0eeSKai Ye if (req->ctx->c_ctx.c_mode == SEC_CMODE_CBC) { 12392f072d75SZaibo Xu sz = sg_pcopy_to_buffer(sgl, sg_nents(sgl), iv, iv_size, 12402f072d75SZaibo Xu cryptlen - iv_size); 1241b9c8d897SZaibo Xu if (unlikely(sz != iv_size)) 1242a44dce50SLongfang Liu dev_err(req->ctx->dev, "copy output iv error!\n"); 12437b44c0eeSKai Ye } else { 12447b44c0eeSKai Ye sz = cryptlen / iv_size; 12457b44c0eeSKai Ye if (cryptlen % iv_size) 12467b44c0eeSKai Ye sz += 1; 12477b44c0eeSKai Ye ctr_iv_inc(iv, iv_size, sz); 12487b44c0eeSKai Ye } 1249416d8220SZaibo Xu } 1250416d8220SZaibo Xu 12519597efc3SKai Ye static struct sec_req *sec_back_req_clear(struct sec_ctx *ctx, 12529597efc3SKai Ye struct sec_qp_ctx *qp_ctx) 12539597efc3SKai Ye { 12549597efc3SKai Ye struct sec_req *backlog_req = NULL; 12559597efc3SKai Ye 12569597efc3SKai Ye mutex_lock(&qp_ctx->req_lock); 12579597efc3SKai Ye if (ctx->fake_req_limit >= 12589597efc3SKai Ye atomic_read(&qp_ctx->qp->qp_status.used) && 12599597efc3SKai Ye !list_empty(&qp_ctx->backlog)) { 12609597efc3SKai Ye backlog_req = list_first_entry(&qp_ctx->backlog, 12619597efc3SKai Ye typeof(*backlog_req), backlog_head); 12629597efc3SKai Ye list_del(&backlog_req->backlog_head); 12639597efc3SKai Ye } 12649597efc3SKai Ye mutex_unlock(&qp_ctx->req_lock); 12659597efc3SKai Ye 12669597efc3SKai Ye return backlog_req; 12679597efc3SKai Ye } 12689597efc3SKai Ye 1269310ea0acSZaibo Xu static void sec_skcipher_callback(struct sec_ctx *ctx, struct sec_req *req, 1270310ea0acSZaibo Xu int err) 1271416d8220SZaibo Xu { 1272416d8220SZaibo Xu struct skcipher_request *sk_req = req->c_req.sk_req; 1273416d8220SZaibo Xu struct sec_qp_ctx *qp_ctx = req->qp_ctx; 12749597efc3SKai Ye struct skcipher_request *backlog_sk_req; 12759597efc3SKai Ye struct sec_req *backlog_req; 1276416d8220SZaibo Xu 1277416d8220SZaibo Xu sec_free_req_id(req); 1278416d8220SZaibo Xu 12797b44c0eeSKai Ye /* IV output at encrypto of CBC/CTR mode */ 12807b44c0eeSKai Ye if (!err && (ctx->c_ctx.c_mode == SEC_CMODE_CBC || 12817b44c0eeSKai Ye ctx->c_ctx.c_mode == SEC_CMODE_CTR) && req->c_req.encrypt) 12822f072d75SZaibo Xu sec_update_iv(req, SEC_SKCIPHER); 1283416d8220SZaibo Xu 12849597efc3SKai Ye while (1) { 12859597efc3SKai Ye backlog_req = sec_back_req_clear(ctx, qp_ctx); 12869597efc3SKai Ye if (!backlog_req) 12879597efc3SKai Ye break; 12889597efc3SKai Ye 12899597efc3SKai Ye backlog_sk_req = backlog_req->c_req.sk_req; 12909597efc3SKai Ye backlog_sk_req->base.complete(&backlog_sk_req->base, 12919597efc3SKai Ye -EINPROGRESS); 12929597efc3SKai Ye atomic64_inc(&ctx->sec->debug.dfx.recv_busy_cnt); 12939597efc3SKai Ye } 12949597efc3SKai Ye 1295310ea0acSZaibo Xu sk_req->base.complete(&sk_req->base, err); 1296416d8220SZaibo Xu } 1297416d8220SZaibo Xu 12982f072d75SZaibo Xu static void sec_aead_copy_iv(struct sec_ctx *ctx, struct sec_req *req) 12992f072d75SZaibo Xu { 13002f072d75SZaibo Xu struct aead_request *aead_req = req->aead_req.aead_req; 13012514f559SLongfang Liu struct sec_cipher_req *c_req = &req->c_req; 13022f072d75SZaibo Xu 13032514f559SLongfang Liu memcpy(c_req->c_ivin, aead_req->iv, ctx->c_ctx.ivsize); 13042f072d75SZaibo Xu } 13052f072d75SZaibo Xu 13062f072d75SZaibo Xu static void sec_auth_bd_fill_ex(struct sec_auth_ctx *ctx, int dir, 13072f072d75SZaibo Xu struct sec_req *req, struct sec_sqe *sec_sqe) 13082f072d75SZaibo Xu { 13092f072d75SZaibo Xu struct sec_aead_req *a_req = &req->aead_req; 13102f072d75SZaibo Xu struct sec_cipher_req *c_req = &req->c_req; 13112f072d75SZaibo Xu struct aead_request *aq = a_req->aead_req; 13122f072d75SZaibo Xu 13132f072d75SZaibo Xu sec_sqe->type2.a_key_addr = cpu_to_le64(ctx->a_key_dma); 13142f072d75SZaibo Xu 13152f072d75SZaibo Xu sec_sqe->type2.mac_key_alg = 13162f072d75SZaibo Xu cpu_to_le32(ctx->mac_len / SEC_SQE_LEN_RATE); 13172f072d75SZaibo Xu 13182f072d75SZaibo Xu sec_sqe->type2.mac_key_alg |= 13192f072d75SZaibo Xu cpu_to_le32((u32)((ctx->a_key_len) / 13202f072d75SZaibo Xu SEC_SQE_LEN_RATE) << SEC_AKEY_OFFSET); 13212f072d75SZaibo Xu 13222f072d75SZaibo Xu sec_sqe->type2.mac_key_alg |= 13232f072d75SZaibo Xu cpu_to_le32((u32)(ctx->a_alg) << SEC_AEAD_ALG_OFFSET); 13242f072d75SZaibo Xu 13252f072d75SZaibo Xu sec_sqe->type_cipher_auth |= SEC_AUTH_TYPE1 << SEC_AUTH_OFFSET; 13262f072d75SZaibo Xu 13272f072d75SZaibo Xu if (dir) 13282f072d75SZaibo Xu sec_sqe->sds_sa_type &= SEC_CIPHER_AUTH; 13292f072d75SZaibo Xu else 13302f072d75SZaibo Xu sec_sqe->sds_sa_type |= SEC_AUTH_CIPHER; 13312f072d75SZaibo Xu 13322f072d75SZaibo Xu sec_sqe->type2.alen_ivllen = cpu_to_le32(c_req->c_len + aq->assoclen); 13332f072d75SZaibo Xu 13342f072d75SZaibo Xu sec_sqe->type2.cipher_src_offset = cpu_to_le16((u16)aq->assoclen); 13352f072d75SZaibo Xu 13362514f559SLongfang Liu sec_sqe->type2.mac_addr = cpu_to_le64(a_req->out_mac_dma); 13372f072d75SZaibo Xu } 13382f072d75SZaibo Xu 13392f072d75SZaibo Xu static int sec_aead_bd_fill(struct sec_ctx *ctx, struct sec_req *req) 13402f072d75SZaibo Xu { 13412f072d75SZaibo Xu struct sec_auth_ctx *auth_ctx = &ctx->a_ctx; 13422f072d75SZaibo Xu struct sec_sqe *sec_sqe = &req->sec_sqe; 13432f072d75SZaibo Xu int ret; 13442f072d75SZaibo Xu 13452f072d75SZaibo Xu ret = sec_skcipher_bd_fill(ctx, req); 13462f072d75SZaibo Xu if (unlikely(ret)) { 1347a44dce50SLongfang Liu dev_err(ctx->dev, "skcipher bd fill is error!\n"); 13482f072d75SZaibo Xu return ret; 13492f072d75SZaibo Xu } 13502f072d75SZaibo Xu 13512f072d75SZaibo Xu sec_auth_bd_fill_ex(auth_ctx, req->c_req.encrypt, req, sec_sqe); 13522f072d75SZaibo Xu 13532f072d75SZaibo Xu return 0; 13542f072d75SZaibo Xu } 13552f072d75SZaibo Xu 1356adc3f65aSKai Ye static void sec_auth_bd_fill_ex_v3(struct sec_auth_ctx *ctx, int dir, 1357adc3f65aSKai Ye struct sec_req *req, struct sec_sqe3 *sqe3) 1358adc3f65aSKai Ye { 1359adc3f65aSKai Ye struct sec_aead_req *a_req = &req->aead_req; 1360adc3f65aSKai Ye struct sec_cipher_req *c_req = &req->c_req; 1361adc3f65aSKai Ye struct aead_request *aq = a_req->aead_req; 1362adc3f65aSKai Ye 1363adc3f65aSKai Ye sqe3->a_key_addr = cpu_to_le64(ctx->a_key_dma); 1364adc3f65aSKai Ye 1365adc3f65aSKai Ye sqe3->auth_mac_key |= 1366adc3f65aSKai Ye cpu_to_le32((u32)(ctx->mac_len / 1367adc3f65aSKai Ye SEC_SQE_LEN_RATE) << SEC_MAC_OFFSET_V3); 1368adc3f65aSKai Ye 1369adc3f65aSKai Ye sqe3->auth_mac_key |= 1370adc3f65aSKai Ye cpu_to_le32((u32)(ctx->a_key_len / 1371adc3f65aSKai Ye SEC_SQE_LEN_RATE) << SEC_AKEY_OFFSET_V3); 1372adc3f65aSKai Ye 1373adc3f65aSKai Ye sqe3->auth_mac_key |= 1374adc3f65aSKai Ye cpu_to_le32((u32)(ctx->a_alg) << SEC_AUTH_ALG_OFFSET_V3); 1375adc3f65aSKai Ye 1376adc3f65aSKai Ye if (dir) { 1377adc3f65aSKai Ye sqe3->auth_mac_key |= cpu_to_le32((u32)SEC_AUTH_TYPE1); 1378adc3f65aSKai Ye sqe3->huk_iv_seq &= SEC_CIPHER_AUTH_V3; 1379adc3f65aSKai Ye } else { 1380adc3f65aSKai Ye sqe3->auth_mac_key |= cpu_to_le32((u32)SEC_AUTH_TYPE1); 1381adc3f65aSKai Ye sqe3->huk_iv_seq |= SEC_AUTH_CIPHER_V3; 1382adc3f65aSKai Ye } 1383adc3f65aSKai Ye sqe3->a_len_key = cpu_to_le32(c_req->c_len + aq->assoclen); 1384adc3f65aSKai Ye 1385adc3f65aSKai Ye sqe3->cipher_src_offset = cpu_to_le16((u16)aq->assoclen); 1386adc3f65aSKai Ye 1387adc3f65aSKai Ye sqe3->mac_addr = cpu_to_le64(a_req->out_mac_dma); 1388adc3f65aSKai Ye } 1389adc3f65aSKai Ye 1390adc3f65aSKai Ye static int sec_aead_bd_fill_v3(struct sec_ctx *ctx, struct sec_req *req) 1391adc3f65aSKai Ye { 1392adc3f65aSKai Ye struct sec_auth_ctx *auth_ctx = &ctx->a_ctx; 1393adc3f65aSKai Ye struct sec_sqe3 *sec_sqe3 = &req->sec_sqe3; 1394adc3f65aSKai Ye int ret; 1395adc3f65aSKai Ye 1396adc3f65aSKai Ye ret = sec_skcipher_bd_fill_v3(ctx, req); 1397adc3f65aSKai Ye if (unlikely(ret)) { 1398adc3f65aSKai Ye dev_err(ctx->dev, "skcipher bd3 fill is error!\n"); 1399adc3f65aSKai Ye return ret; 1400adc3f65aSKai Ye } 1401adc3f65aSKai Ye 1402adc3f65aSKai Ye sec_auth_bd_fill_ex_v3(auth_ctx, req->c_req.encrypt, req, sec_sqe3); 1403adc3f65aSKai Ye 1404adc3f65aSKai Ye return 0; 1405adc3f65aSKai Ye } 1406adc3f65aSKai Ye 14072f072d75SZaibo Xu static void sec_aead_callback(struct sec_ctx *c, struct sec_req *req, int err) 14082f072d75SZaibo Xu { 14092f072d75SZaibo Xu struct aead_request *a_req = req->aead_req.aead_req; 14102f072d75SZaibo Xu struct crypto_aead *tfm = crypto_aead_reqtfm(a_req); 14112514f559SLongfang Liu struct sec_aead_req *aead_req = &req->aead_req; 14122f072d75SZaibo Xu struct sec_cipher_req *c_req = &req->c_req; 14132f072d75SZaibo Xu size_t authsize = crypto_aead_authsize(tfm); 14142f072d75SZaibo Xu struct sec_qp_ctx *qp_ctx = req->qp_ctx; 14159597efc3SKai Ye struct aead_request *backlog_aead_req; 14169597efc3SKai Ye struct sec_req *backlog_req; 14172f072d75SZaibo Xu size_t sz; 14182f072d75SZaibo Xu 14192f072d75SZaibo Xu if (!err && c->c_ctx.c_mode == SEC_CMODE_CBC && c_req->encrypt) 14202f072d75SZaibo Xu sec_update_iv(req, SEC_AEAD); 14212f072d75SZaibo Xu 14222f072d75SZaibo Xu /* Copy output mac */ 14232f072d75SZaibo Xu if (!err && c_req->encrypt) { 14242f072d75SZaibo Xu struct scatterlist *sgl = a_req->dst; 14252f072d75SZaibo Xu 14262f072d75SZaibo Xu sz = sg_pcopy_from_buffer(sgl, sg_nents(sgl), 14272514f559SLongfang Liu aead_req->out_mac, 14282f072d75SZaibo Xu authsize, a_req->cryptlen + 14292f072d75SZaibo Xu a_req->assoclen); 14302f072d75SZaibo Xu 14312f072d75SZaibo Xu if (unlikely(sz != authsize)) { 1432a44dce50SLongfang Liu dev_err(c->dev, "copy out mac err!\n"); 14332f072d75SZaibo Xu err = -EINVAL; 14342f072d75SZaibo Xu } 14352f072d75SZaibo Xu } 14362f072d75SZaibo Xu 14372f072d75SZaibo Xu sec_free_req_id(req); 14382f072d75SZaibo Xu 14399597efc3SKai Ye while (1) { 14409597efc3SKai Ye backlog_req = sec_back_req_clear(c, qp_ctx); 14419597efc3SKai Ye if (!backlog_req) 14429597efc3SKai Ye break; 14439597efc3SKai Ye 14449597efc3SKai Ye backlog_aead_req = backlog_req->aead_req.aead_req; 14459597efc3SKai Ye backlog_aead_req->base.complete(&backlog_aead_req->base, 14469597efc3SKai Ye -EINPROGRESS); 14479597efc3SKai Ye atomic64_inc(&c->sec->debug.dfx.recv_busy_cnt); 14489597efc3SKai Ye } 14492f072d75SZaibo Xu 14502f072d75SZaibo Xu a_req->base.complete(&a_req->base, err); 14512f072d75SZaibo Xu } 14522f072d75SZaibo Xu 1453416d8220SZaibo Xu static void sec_request_uninit(struct sec_ctx *ctx, struct sec_req *req) 1454416d8220SZaibo Xu { 1455416d8220SZaibo Xu sec_free_req_id(req); 1456a181647cSZaibo Xu sec_free_queue_id(ctx, req); 1457416d8220SZaibo Xu } 1458416d8220SZaibo Xu 1459416d8220SZaibo Xu static int sec_request_init(struct sec_ctx *ctx, struct sec_req *req) 1460416d8220SZaibo Xu { 1461416d8220SZaibo Xu struct sec_qp_ctx *qp_ctx; 14627c7d902aSZaibo Xu int queue_id; 1463416d8220SZaibo Xu 1464416d8220SZaibo Xu /* To load balance */ 1465a181647cSZaibo Xu queue_id = sec_alloc_queue_id(ctx, req); 1466a181647cSZaibo Xu qp_ctx = &ctx->qp_ctx[queue_id]; 1467416d8220SZaibo Xu 1468416d8220SZaibo Xu req->req_id = sec_alloc_req_id(req, qp_ctx); 1469b9c8d897SZaibo Xu if (unlikely(req->req_id < 0)) { 1470a181647cSZaibo Xu sec_free_queue_id(ctx, req); 1471416d8220SZaibo Xu return req->req_id; 1472416d8220SZaibo Xu } 1473416d8220SZaibo Xu 14747c7d902aSZaibo Xu return 0; 1475416d8220SZaibo Xu } 1476416d8220SZaibo Xu 1477416d8220SZaibo Xu static int sec_process(struct sec_ctx *ctx, struct sec_req *req) 1478416d8220SZaibo Xu { 14792514f559SLongfang Liu struct sec_cipher_req *c_req = &req->c_req; 1480416d8220SZaibo Xu int ret; 1481416d8220SZaibo Xu 1482416d8220SZaibo Xu ret = sec_request_init(ctx, req); 1483b9c8d897SZaibo Xu if (unlikely(ret)) 1484416d8220SZaibo Xu return ret; 1485416d8220SZaibo Xu 1486416d8220SZaibo Xu ret = sec_request_transfer(ctx, req); 1487b9c8d897SZaibo Xu if (unlikely(ret)) 1488416d8220SZaibo Xu goto err_uninit_req; 1489416d8220SZaibo Xu 1490416d8220SZaibo Xu /* Output IV as decrypto */ 14917b44c0eeSKai Ye if (!req->c_req.encrypt && (ctx->c_ctx.c_mode == SEC_CMODE_CBC || 14927b44c0eeSKai Ye ctx->c_ctx.c_mode == SEC_CMODE_CTR)) 14932f072d75SZaibo Xu sec_update_iv(req, ctx->alg_type); 1494416d8220SZaibo Xu 1495416d8220SZaibo Xu ret = ctx->req_op->bd_send(ctx, req); 14969597efc3SKai Ye if (unlikely((ret != -EBUSY && ret != -EINPROGRESS) || 14979597efc3SKai Ye (ret == -EBUSY && !(req->flag & CRYPTO_TFM_REQ_MAY_BACKLOG)))) { 1498a44dce50SLongfang Liu dev_err_ratelimited(ctx->dev, "send sec request failed!\n"); 1499416d8220SZaibo Xu goto err_send_req; 1500416d8220SZaibo Xu } 1501416d8220SZaibo Xu 1502416d8220SZaibo Xu return ret; 1503416d8220SZaibo Xu 1504416d8220SZaibo Xu err_send_req: 1505416d8220SZaibo Xu /* As failing, restore the IV from user */ 15062f072d75SZaibo Xu if (ctx->c_ctx.c_mode == SEC_CMODE_CBC && !req->c_req.encrypt) { 15072f072d75SZaibo Xu if (ctx->alg_type == SEC_SKCIPHER) 15082514f559SLongfang Liu memcpy(req->c_req.sk_req->iv, c_req->c_ivin, 1509416d8220SZaibo Xu ctx->c_ctx.ivsize); 15102f072d75SZaibo Xu else 15112514f559SLongfang Liu memcpy(req->aead_req.aead_req->iv, c_req->c_ivin, 15122f072d75SZaibo Xu ctx->c_ctx.ivsize); 15132f072d75SZaibo Xu } 1514416d8220SZaibo Xu 1515416d8220SZaibo Xu sec_request_untransfer(ctx, req); 1516416d8220SZaibo Xu err_uninit_req: 1517416d8220SZaibo Xu sec_request_uninit(ctx, req); 1518416d8220SZaibo Xu return ret; 1519416d8220SZaibo Xu } 1520416d8220SZaibo Xu 1521a181647cSZaibo Xu static const struct sec_req_op sec_skcipher_req_ops = { 1522416d8220SZaibo Xu .buf_map = sec_skcipher_sgl_map, 1523416d8220SZaibo Xu .buf_unmap = sec_skcipher_sgl_unmap, 1524416d8220SZaibo Xu .do_transfer = sec_skcipher_copy_iv, 1525416d8220SZaibo Xu .bd_fill = sec_skcipher_bd_fill, 1526416d8220SZaibo Xu .bd_send = sec_bd_send, 1527416d8220SZaibo Xu .callback = sec_skcipher_callback, 1528416d8220SZaibo Xu .process = sec_process, 1529416d8220SZaibo Xu }; 1530416d8220SZaibo Xu 15312f072d75SZaibo Xu static const struct sec_req_op sec_aead_req_ops = { 15322f072d75SZaibo Xu .buf_map = sec_aead_sgl_map, 15332f072d75SZaibo Xu .buf_unmap = sec_aead_sgl_unmap, 15342f072d75SZaibo Xu .do_transfer = sec_aead_copy_iv, 15352f072d75SZaibo Xu .bd_fill = sec_aead_bd_fill, 15362f072d75SZaibo Xu .bd_send = sec_bd_send, 15372f072d75SZaibo Xu .callback = sec_aead_callback, 15382f072d75SZaibo Xu .process = sec_process, 15392f072d75SZaibo Xu }; 15402f072d75SZaibo Xu 1541adc3f65aSKai Ye static const struct sec_req_op sec_skcipher_req_ops_v3 = { 1542adc3f65aSKai Ye .buf_map = sec_skcipher_sgl_map, 1543adc3f65aSKai Ye .buf_unmap = sec_skcipher_sgl_unmap, 1544adc3f65aSKai Ye .do_transfer = sec_skcipher_copy_iv, 1545adc3f65aSKai Ye .bd_fill = sec_skcipher_bd_fill_v3, 1546adc3f65aSKai Ye .bd_send = sec_bd_send, 1547adc3f65aSKai Ye .callback = sec_skcipher_callback, 1548adc3f65aSKai Ye .process = sec_process, 1549adc3f65aSKai Ye }; 1550adc3f65aSKai Ye 1551adc3f65aSKai Ye static const struct sec_req_op sec_aead_req_ops_v3 = { 1552adc3f65aSKai Ye .buf_map = sec_aead_sgl_map, 1553adc3f65aSKai Ye .buf_unmap = sec_aead_sgl_unmap, 1554adc3f65aSKai Ye .do_transfer = sec_aead_copy_iv, 1555adc3f65aSKai Ye .bd_fill = sec_aead_bd_fill_v3, 1556adc3f65aSKai Ye .bd_send = sec_bd_send, 1557adc3f65aSKai Ye .callback = sec_aead_callback, 1558adc3f65aSKai Ye .process = sec_process, 1559adc3f65aSKai Ye }; 1560adc3f65aSKai Ye 1561416d8220SZaibo Xu static int sec_skcipher_ctx_init(struct crypto_skcipher *tfm) 1562416d8220SZaibo Xu { 1563416d8220SZaibo Xu struct sec_ctx *ctx = crypto_skcipher_ctx(tfm); 1564adc3f65aSKai Ye int ret; 1565416d8220SZaibo Xu 1566adc3f65aSKai Ye ret = sec_skcipher_init(tfm); 1567adc3f65aSKai Ye if (ret) 1568adc3f65aSKai Ye return ret; 1569adc3f65aSKai Ye 1570adc3f65aSKai Ye if (ctx->sec->qm.ver < QM_HW_V3) { 1571adc3f65aSKai Ye ctx->type_supported = SEC_BD_TYPE2; 1572a181647cSZaibo Xu ctx->req_op = &sec_skcipher_req_ops; 1573adc3f65aSKai Ye } else { 1574adc3f65aSKai Ye ctx->type_supported = SEC_BD_TYPE3; 1575adc3f65aSKai Ye ctx->req_op = &sec_skcipher_req_ops_v3; 1576adc3f65aSKai Ye } 1577416d8220SZaibo Xu 1578adc3f65aSKai Ye return ret; 1579416d8220SZaibo Xu } 1580416d8220SZaibo Xu 1581416d8220SZaibo Xu static void sec_skcipher_ctx_exit(struct crypto_skcipher *tfm) 1582416d8220SZaibo Xu { 1583a181647cSZaibo Xu sec_skcipher_uninit(tfm); 1584416d8220SZaibo Xu } 1585416d8220SZaibo Xu 15862f072d75SZaibo Xu static int sec_aead_init(struct crypto_aead *tfm) 15872f072d75SZaibo Xu { 15882f072d75SZaibo Xu struct sec_ctx *ctx = crypto_aead_ctx(tfm); 15892f072d75SZaibo Xu int ret; 15902f072d75SZaibo Xu 15912f072d75SZaibo Xu crypto_aead_set_reqsize(tfm, sizeof(struct sec_req)); 15922f072d75SZaibo Xu ctx->alg_type = SEC_AEAD; 15932f072d75SZaibo Xu ctx->c_ctx.ivsize = crypto_aead_ivsize(tfm); 15942f072d75SZaibo Xu if (ctx->c_ctx.ivsize > SEC_IV_SIZE) { 1595a44dce50SLongfang Liu dev_err(ctx->dev, "get error aead iv size!\n"); 15962f072d75SZaibo Xu return -EINVAL; 15972f072d75SZaibo Xu } 15982f072d75SZaibo Xu 15992f072d75SZaibo Xu ret = sec_ctx_base_init(ctx); 16002f072d75SZaibo Xu if (ret) 16012f072d75SZaibo Xu return ret; 1602adc3f65aSKai Ye if (ctx->sec->qm.ver < QM_HW_V3) { 1603adc3f65aSKai Ye ctx->type_supported = SEC_BD_TYPE2; 1604adc3f65aSKai Ye ctx->req_op = &sec_aead_req_ops; 1605adc3f65aSKai Ye } else { 1606adc3f65aSKai Ye ctx->type_supported = SEC_BD_TYPE3; 1607adc3f65aSKai Ye ctx->req_op = &sec_aead_req_ops_v3; 1608adc3f65aSKai Ye } 16092f072d75SZaibo Xu 16102f072d75SZaibo Xu ret = sec_auth_init(ctx); 16112f072d75SZaibo Xu if (ret) 16122f072d75SZaibo Xu goto err_auth_init; 16132f072d75SZaibo Xu 16142f072d75SZaibo Xu ret = sec_cipher_init(ctx); 16152f072d75SZaibo Xu if (ret) 16162f072d75SZaibo Xu goto err_cipher_init; 16172f072d75SZaibo Xu 16182f072d75SZaibo Xu return ret; 16192f072d75SZaibo Xu 16202f072d75SZaibo Xu err_cipher_init: 16212f072d75SZaibo Xu sec_auth_uninit(ctx); 16222f072d75SZaibo Xu err_auth_init: 16232f072d75SZaibo Xu sec_ctx_base_uninit(ctx); 16242f072d75SZaibo Xu return ret; 16252f072d75SZaibo Xu } 16262f072d75SZaibo Xu 16272f072d75SZaibo Xu static void sec_aead_exit(struct crypto_aead *tfm) 16282f072d75SZaibo Xu { 16292f072d75SZaibo Xu struct sec_ctx *ctx = crypto_aead_ctx(tfm); 16302f072d75SZaibo Xu 16312f072d75SZaibo Xu sec_cipher_uninit(ctx); 16322f072d75SZaibo Xu sec_auth_uninit(ctx); 16332f072d75SZaibo Xu sec_ctx_base_uninit(ctx); 16342f072d75SZaibo Xu } 16352f072d75SZaibo Xu 16362f072d75SZaibo Xu static int sec_aead_ctx_init(struct crypto_aead *tfm, const char *hash_name) 16372f072d75SZaibo Xu { 16382f072d75SZaibo Xu struct sec_ctx *ctx = crypto_aead_ctx(tfm); 16392f072d75SZaibo Xu struct sec_auth_ctx *auth_ctx = &ctx->a_ctx; 16402f072d75SZaibo Xu int ret; 16412f072d75SZaibo Xu 16422f072d75SZaibo Xu ret = sec_aead_init(tfm); 16432f072d75SZaibo Xu if (ret) { 16442f072d75SZaibo Xu pr_err("hisi_sec2: aead init error!\n"); 16452f072d75SZaibo Xu return ret; 16462f072d75SZaibo Xu } 16472f072d75SZaibo Xu 16482f072d75SZaibo Xu auth_ctx->hash_tfm = crypto_alloc_shash(hash_name, 0, 0); 16492f072d75SZaibo Xu if (IS_ERR(auth_ctx->hash_tfm)) { 1650a44dce50SLongfang Liu dev_err(ctx->dev, "aead alloc shash error!\n"); 16512f072d75SZaibo Xu sec_aead_exit(tfm); 16522f072d75SZaibo Xu return PTR_ERR(auth_ctx->hash_tfm); 16532f072d75SZaibo Xu } 16542f072d75SZaibo Xu 16552f072d75SZaibo Xu return 0; 16562f072d75SZaibo Xu } 16572f072d75SZaibo Xu 16582f072d75SZaibo Xu static void sec_aead_ctx_exit(struct crypto_aead *tfm) 16592f072d75SZaibo Xu { 16602f072d75SZaibo Xu struct sec_ctx *ctx = crypto_aead_ctx(tfm); 16612f072d75SZaibo Xu 16622f072d75SZaibo Xu crypto_free_shash(ctx->a_ctx.hash_tfm); 16632f072d75SZaibo Xu sec_aead_exit(tfm); 16642f072d75SZaibo Xu } 16652f072d75SZaibo Xu 16662f072d75SZaibo Xu static int sec_aead_sha1_ctx_init(struct crypto_aead *tfm) 16672f072d75SZaibo Xu { 16682f072d75SZaibo Xu return sec_aead_ctx_init(tfm, "sha1"); 16692f072d75SZaibo Xu } 16702f072d75SZaibo Xu 16712f072d75SZaibo Xu static int sec_aead_sha256_ctx_init(struct crypto_aead *tfm) 16722f072d75SZaibo Xu { 16732f072d75SZaibo Xu return sec_aead_ctx_init(tfm, "sha256"); 16742f072d75SZaibo Xu } 16752f072d75SZaibo Xu 16762f072d75SZaibo Xu static int sec_aead_sha512_ctx_init(struct crypto_aead *tfm) 16772f072d75SZaibo Xu { 16782f072d75SZaibo Xu return sec_aead_ctx_init(tfm, "sha512"); 16792f072d75SZaibo Xu } 16802f072d75SZaibo Xu 1681059c5342SLongfang Liu 1682059c5342SLongfang Liu static int sec_skcipher_cryptlen_ckeck(struct sec_ctx *ctx, 1683059c5342SLongfang Liu struct sec_req *sreq) 1684059c5342SLongfang Liu { 1685059c5342SLongfang Liu u32 cryptlen = sreq->c_req.sk_req->cryptlen; 1686059c5342SLongfang Liu struct device *dev = ctx->dev; 1687059c5342SLongfang Liu u8 c_mode = ctx->c_ctx.c_mode; 1688059c5342SLongfang Liu int ret = 0; 1689059c5342SLongfang Liu 1690059c5342SLongfang Liu switch (c_mode) { 1691059c5342SLongfang Liu case SEC_CMODE_XTS: 1692059c5342SLongfang Liu if (unlikely(cryptlen < AES_BLOCK_SIZE)) { 1693059c5342SLongfang Liu dev_err(dev, "skcipher XTS mode input length error!\n"); 1694059c5342SLongfang Liu ret = -EINVAL; 1695059c5342SLongfang Liu } 1696059c5342SLongfang Liu break; 1697059c5342SLongfang Liu case SEC_CMODE_ECB: 1698059c5342SLongfang Liu case SEC_CMODE_CBC: 1699059c5342SLongfang Liu if (unlikely(cryptlen & (AES_BLOCK_SIZE - 1))) { 1700059c5342SLongfang Liu dev_err(dev, "skcipher AES input length error!\n"); 1701059c5342SLongfang Liu ret = -EINVAL; 1702059c5342SLongfang Liu } 1703059c5342SLongfang Liu break; 17047b44c0eeSKai Ye case SEC_CMODE_CFB: 17057b44c0eeSKai Ye case SEC_CMODE_OFB: 17067b44c0eeSKai Ye case SEC_CMODE_CTR: 17077b44c0eeSKai Ye if (unlikely(ctx->sec->qm.ver < QM_HW_V3)) { 17087b44c0eeSKai Ye dev_err(dev, "skcipher HW version error!\n"); 17097b44c0eeSKai Ye ret = -EINVAL; 17107b44c0eeSKai Ye } 17117b44c0eeSKai Ye break; 1712059c5342SLongfang Liu default: 1713059c5342SLongfang Liu ret = -EINVAL; 1714059c5342SLongfang Liu } 1715059c5342SLongfang Liu 1716059c5342SLongfang Liu return ret; 1717059c5342SLongfang Liu } 1718059c5342SLongfang Liu 1719d6de2a59SZaibo Xu static int sec_skcipher_param_check(struct sec_ctx *ctx, struct sec_req *sreq) 1720416d8220SZaibo Xu { 1721d6de2a59SZaibo Xu struct skcipher_request *sk_req = sreq->c_req.sk_req; 1722a44dce50SLongfang Liu struct device *dev = ctx->dev; 1723d6de2a59SZaibo Xu u8 c_alg = ctx->c_ctx.c_alg; 1724416d8220SZaibo Xu 17257b44c0eeSKai Ye if (unlikely(!sk_req->src || !sk_req->dst || 17267b44c0eeSKai Ye sk_req->cryptlen > MAX_INPUT_DATA_LEN)) { 1727416d8220SZaibo Xu dev_err(dev, "skcipher input param error!\n"); 1728416d8220SZaibo Xu return -EINVAL; 1729416d8220SZaibo Xu } 1730d6de2a59SZaibo Xu sreq->c_req.c_len = sk_req->cryptlen; 173174b58db8SLongfang Liu 173274b58db8SLongfang Liu if (ctx->pbuf_supported && sk_req->cryptlen <= SEC_PBUF_SZ) 173374b58db8SLongfang Liu sreq->use_pbuf = true; 173474b58db8SLongfang Liu else 173574b58db8SLongfang Liu sreq->use_pbuf = false; 173674b58db8SLongfang Liu 1737416d8220SZaibo Xu if (c_alg == SEC_CALG_3DES) { 1738b9c8d897SZaibo Xu if (unlikely(sk_req->cryptlen & (DES3_EDE_BLOCK_SIZE - 1))) { 1739416d8220SZaibo Xu dev_err(dev, "skcipher 3des input length error!\n"); 1740416d8220SZaibo Xu return -EINVAL; 1741416d8220SZaibo Xu } 1742416d8220SZaibo Xu return 0; 1743416d8220SZaibo Xu } else if (c_alg == SEC_CALG_AES || c_alg == SEC_CALG_SM4) { 1744059c5342SLongfang Liu return sec_skcipher_cryptlen_ckeck(ctx, sreq); 1745416d8220SZaibo Xu } 1746059c5342SLongfang Liu 1747416d8220SZaibo Xu dev_err(dev, "skcipher algorithm error!\n"); 1748633e507fSLongfang Liu 1749416d8220SZaibo Xu return -EINVAL; 1750416d8220SZaibo Xu } 1751416d8220SZaibo Xu 17525652d55aSKai Ye static int sec_skcipher_soft_crypto(struct sec_ctx *ctx, 17535652d55aSKai Ye struct skcipher_request *sreq, bool encrypt) 17545652d55aSKai Ye { 17555652d55aSKai Ye struct sec_cipher_ctx *c_ctx = &ctx->c_ctx; 17565652d55aSKai Ye struct device *dev = ctx->dev; 17575652d55aSKai Ye int ret; 17585652d55aSKai Ye 17595652d55aSKai Ye SYNC_SKCIPHER_REQUEST_ON_STACK(subreq, c_ctx->fbtfm); 17605652d55aSKai Ye 17615652d55aSKai Ye if (!c_ctx->fbtfm) { 17625652d55aSKai Ye dev_err(dev, "failed to check fallback tfm\n"); 17635652d55aSKai Ye return -EINVAL; 17645652d55aSKai Ye } 17655652d55aSKai Ye 17665652d55aSKai Ye skcipher_request_set_sync_tfm(subreq, c_ctx->fbtfm); 17675652d55aSKai Ye 17685652d55aSKai Ye /* software need sync mode to do crypto */ 17695652d55aSKai Ye skcipher_request_set_callback(subreq, sreq->base.flags, 17705652d55aSKai Ye NULL, NULL); 17715652d55aSKai Ye skcipher_request_set_crypt(subreq, sreq->src, sreq->dst, 17725652d55aSKai Ye sreq->cryptlen, sreq->iv); 17735652d55aSKai Ye if (encrypt) 17745652d55aSKai Ye ret = crypto_skcipher_encrypt(subreq); 17755652d55aSKai Ye else 17765652d55aSKai Ye ret = crypto_skcipher_decrypt(subreq); 17775652d55aSKai Ye 17785652d55aSKai Ye skcipher_request_zero(subreq); 17795652d55aSKai Ye 17805652d55aSKai Ye return ret; 17815652d55aSKai Ye } 17825652d55aSKai Ye 1783416d8220SZaibo Xu static int sec_skcipher_crypto(struct skcipher_request *sk_req, bool encrypt) 1784416d8220SZaibo Xu { 1785416d8220SZaibo Xu struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(sk_req); 1786416d8220SZaibo Xu struct sec_req *req = skcipher_request_ctx(sk_req); 1787416d8220SZaibo Xu struct sec_ctx *ctx = crypto_skcipher_ctx(tfm); 1788416d8220SZaibo Xu int ret; 1789416d8220SZaibo Xu 17905652d55aSKai Ye if (!sk_req->cryptlen) { 17915652d55aSKai Ye if (ctx->c_ctx.c_mode == SEC_CMODE_XTS) 17925652d55aSKai Ye return -EINVAL; 1793416d8220SZaibo Xu return 0; 17945652d55aSKai Ye } 1795416d8220SZaibo Xu 17969597efc3SKai Ye req->flag = sk_req->base.flags; 1797416d8220SZaibo Xu req->c_req.sk_req = sk_req; 1798416d8220SZaibo Xu req->c_req.encrypt = encrypt; 1799416d8220SZaibo Xu req->ctx = ctx; 1800416d8220SZaibo Xu 1801d6de2a59SZaibo Xu ret = sec_skcipher_param_check(ctx, req); 1802d6de2a59SZaibo Xu if (unlikely(ret)) 1803d6de2a59SZaibo Xu return -EINVAL; 1804d6de2a59SZaibo Xu 18055652d55aSKai Ye if (unlikely(ctx->c_ctx.fallback)) 18065652d55aSKai Ye return sec_skcipher_soft_crypto(ctx, sk_req, encrypt); 18075652d55aSKai Ye 1808416d8220SZaibo Xu return ctx->req_op->process(ctx, req); 1809416d8220SZaibo Xu } 1810416d8220SZaibo Xu 1811416d8220SZaibo Xu static int sec_skcipher_encrypt(struct skcipher_request *sk_req) 1812416d8220SZaibo Xu { 1813416d8220SZaibo Xu return sec_skcipher_crypto(sk_req, true); 1814416d8220SZaibo Xu } 1815416d8220SZaibo Xu 1816416d8220SZaibo Xu static int sec_skcipher_decrypt(struct skcipher_request *sk_req) 1817416d8220SZaibo Xu { 1818416d8220SZaibo Xu return sec_skcipher_crypto(sk_req, false); 1819416d8220SZaibo Xu } 1820416d8220SZaibo Xu 1821416d8220SZaibo Xu #define SEC_SKCIPHER_GEN_ALG(sec_cra_name, sec_set_key, sec_min_key_size, \ 1822416d8220SZaibo Xu sec_max_key_size, ctx_init, ctx_exit, blk_size, iv_size)\ 1823416d8220SZaibo Xu {\ 1824416d8220SZaibo Xu .base = {\ 1825416d8220SZaibo Xu .cra_name = sec_cra_name,\ 1826416d8220SZaibo Xu .cra_driver_name = "hisi_sec_"sec_cra_name,\ 1827416d8220SZaibo Xu .cra_priority = SEC_PRIORITY,\ 18285652d55aSKai Ye .cra_flags = CRYPTO_ALG_ASYNC |\ 18295652d55aSKai Ye CRYPTO_ALG_ALLOCATES_MEMORY |\ 18305652d55aSKai Ye CRYPTO_ALG_NEED_FALLBACK,\ 1831416d8220SZaibo Xu .cra_blocksize = blk_size,\ 1832416d8220SZaibo Xu .cra_ctxsize = sizeof(struct sec_ctx),\ 1833416d8220SZaibo Xu .cra_module = THIS_MODULE,\ 1834416d8220SZaibo Xu },\ 1835416d8220SZaibo Xu .init = ctx_init,\ 1836416d8220SZaibo Xu .exit = ctx_exit,\ 1837416d8220SZaibo Xu .setkey = sec_set_key,\ 1838416d8220SZaibo Xu .decrypt = sec_skcipher_decrypt,\ 1839416d8220SZaibo Xu .encrypt = sec_skcipher_encrypt,\ 1840416d8220SZaibo Xu .min_keysize = sec_min_key_size,\ 1841416d8220SZaibo Xu .max_keysize = sec_max_key_size,\ 1842416d8220SZaibo Xu .ivsize = iv_size,\ 1843416d8220SZaibo Xu }, 1844416d8220SZaibo Xu 1845416d8220SZaibo Xu #define SEC_SKCIPHER_ALG(name, key_func, min_key_size, \ 1846416d8220SZaibo Xu max_key_size, blk_size, iv_size) \ 1847416d8220SZaibo Xu SEC_SKCIPHER_GEN_ALG(name, key_func, min_key_size, max_key_size, \ 1848416d8220SZaibo Xu sec_skcipher_ctx_init, sec_skcipher_ctx_exit, blk_size, iv_size) 1849416d8220SZaibo Xu 1850a181647cSZaibo Xu static struct skcipher_alg sec_skciphers[] = { 1851416d8220SZaibo Xu SEC_SKCIPHER_ALG("ecb(aes)", sec_setkey_aes_ecb, 1852416d8220SZaibo Xu AES_MIN_KEY_SIZE, AES_MAX_KEY_SIZE, 1853416d8220SZaibo Xu AES_BLOCK_SIZE, 0) 1854416d8220SZaibo Xu 1855416d8220SZaibo Xu SEC_SKCIPHER_ALG("cbc(aes)", sec_setkey_aes_cbc, 1856416d8220SZaibo Xu AES_MIN_KEY_SIZE, AES_MAX_KEY_SIZE, 1857416d8220SZaibo Xu AES_BLOCK_SIZE, AES_BLOCK_SIZE) 1858416d8220SZaibo Xu 1859416d8220SZaibo Xu SEC_SKCIPHER_ALG("xts(aes)", sec_setkey_aes_xts, 1860416d8220SZaibo Xu SEC_XTS_MIN_KEY_SIZE, SEC_XTS_MAX_KEY_SIZE, 1861416d8220SZaibo Xu AES_BLOCK_SIZE, AES_BLOCK_SIZE) 1862416d8220SZaibo Xu 1863416d8220SZaibo Xu SEC_SKCIPHER_ALG("ecb(des3_ede)", sec_setkey_3des_ecb, 1864*6161f40cSKai Ye SEC_DES3_3KEY_SIZE, SEC_DES3_3KEY_SIZE, 1865416d8220SZaibo Xu DES3_EDE_BLOCK_SIZE, 0) 1866416d8220SZaibo Xu 1867416d8220SZaibo Xu SEC_SKCIPHER_ALG("cbc(des3_ede)", sec_setkey_3des_cbc, 1868*6161f40cSKai Ye SEC_DES3_3KEY_SIZE, SEC_DES3_3KEY_SIZE, 1869416d8220SZaibo Xu DES3_EDE_BLOCK_SIZE, DES3_EDE_BLOCK_SIZE) 1870416d8220SZaibo Xu 1871416d8220SZaibo Xu SEC_SKCIPHER_ALG("xts(sm4)", sec_setkey_sm4_xts, 1872416d8220SZaibo Xu SEC_XTS_MIN_KEY_SIZE, SEC_XTS_MIN_KEY_SIZE, 1873416d8220SZaibo Xu AES_BLOCK_SIZE, AES_BLOCK_SIZE) 1874416d8220SZaibo Xu 1875416d8220SZaibo Xu SEC_SKCIPHER_ALG("cbc(sm4)", sec_setkey_sm4_cbc, 1876416d8220SZaibo Xu AES_MIN_KEY_SIZE, AES_MIN_KEY_SIZE, 1877416d8220SZaibo Xu AES_BLOCK_SIZE, AES_BLOCK_SIZE) 1878416d8220SZaibo Xu }; 1879416d8220SZaibo Xu 18807b44c0eeSKai Ye static struct skcipher_alg sec_skciphers_v3[] = { 18817b44c0eeSKai Ye SEC_SKCIPHER_ALG("ofb(aes)", sec_setkey_aes_ofb, 18827b44c0eeSKai Ye AES_MIN_KEY_SIZE, AES_MAX_KEY_SIZE, 18837b44c0eeSKai Ye SEC_MIN_BLOCK_SZ, AES_BLOCK_SIZE) 18847b44c0eeSKai Ye 18857b44c0eeSKai Ye SEC_SKCIPHER_ALG("cfb(aes)", sec_setkey_aes_cfb, 18867b44c0eeSKai Ye AES_MIN_KEY_SIZE, AES_MAX_KEY_SIZE, 18877b44c0eeSKai Ye SEC_MIN_BLOCK_SZ, AES_BLOCK_SIZE) 18887b44c0eeSKai Ye 18897b44c0eeSKai Ye SEC_SKCIPHER_ALG("ctr(aes)", sec_setkey_aes_ctr, 18907b44c0eeSKai Ye AES_MIN_KEY_SIZE, AES_MAX_KEY_SIZE, 18917b44c0eeSKai Ye SEC_MIN_BLOCK_SZ, AES_BLOCK_SIZE) 18927b44c0eeSKai Ye 18937b44c0eeSKai Ye SEC_SKCIPHER_ALG("ofb(sm4)", sec_setkey_sm4_ofb, 18947b44c0eeSKai Ye AES_MIN_KEY_SIZE, AES_MIN_KEY_SIZE, 18957b44c0eeSKai Ye SEC_MIN_BLOCK_SZ, AES_BLOCK_SIZE) 18967b44c0eeSKai Ye 18977b44c0eeSKai Ye SEC_SKCIPHER_ALG("cfb(sm4)", sec_setkey_sm4_cfb, 18987b44c0eeSKai Ye AES_MIN_KEY_SIZE, AES_MIN_KEY_SIZE, 18997b44c0eeSKai Ye SEC_MIN_BLOCK_SZ, AES_BLOCK_SIZE) 19007b44c0eeSKai Ye 19017b44c0eeSKai Ye SEC_SKCIPHER_ALG("ctr(sm4)", sec_setkey_sm4_ctr, 19027b44c0eeSKai Ye AES_MIN_KEY_SIZE, AES_MIN_KEY_SIZE, 19037b44c0eeSKai Ye SEC_MIN_BLOCK_SZ, AES_BLOCK_SIZE) 19047b44c0eeSKai Ye }; 19057b44c0eeSKai Ye 19062f072d75SZaibo Xu static int sec_aead_param_check(struct sec_ctx *ctx, struct sec_req *sreq) 19072f072d75SZaibo Xu { 19082f072d75SZaibo Xu struct aead_request *req = sreq->aead_req.aead_req; 19092f072d75SZaibo Xu struct crypto_aead *tfm = crypto_aead_reqtfm(req); 19102f072d75SZaibo Xu size_t authsize = crypto_aead_authsize(tfm); 1911a44dce50SLongfang Liu struct device *dev = ctx->dev; 1912a44dce50SLongfang Liu u8 c_alg = ctx->c_ctx.c_alg; 19132f072d75SZaibo Xu 19142514f559SLongfang Liu if (unlikely(!req->src || !req->dst || !req->cryptlen || 19152514f559SLongfang Liu req->assoclen > SEC_MAX_AAD_LEN)) { 1916a44dce50SLongfang Liu dev_err(dev, "aead input param error!\n"); 19172f072d75SZaibo Xu return -EINVAL; 19182f072d75SZaibo Xu } 19192f072d75SZaibo Xu 192074b58db8SLongfang Liu if (ctx->pbuf_supported && (req->cryptlen + req->assoclen) <= 192174b58db8SLongfang Liu SEC_PBUF_SZ) 192274b58db8SLongfang Liu sreq->use_pbuf = true; 192374b58db8SLongfang Liu else 192474b58db8SLongfang Liu sreq->use_pbuf = false; 192574b58db8SLongfang Liu 19262f072d75SZaibo Xu /* Support AES only */ 19272f072d75SZaibo Xu if (unlikely(c_alg != SEC_CALG_AES)) { 1928a44dce50SLongfang Liu dev_err(dev, "aead crypto alg error!\n"); 19292f072d75SZaibo Xu return -EINVAL; 19302f072d75SZaibo Xu } 19312f072d75SZaibo Xu if (sreq->c_req.encrypt) 19322f072d75SZaibo Xu sreq->c_req.c_len = req->cryptlen; 19332f072d75SZaibo Xu else 19342f072d75SZaibo Xu sreq->c_req.c_len = req->cryptlen - authsize; 19352f072d75SZaibo Xu 19362f072d75SZaibo Xu if (unlikely(sreq->c_req.c_len & (AES_BLOCK_SIZE - 1))) { 1937a44dce50SLongfang Liu dev_err(dev, "aead crypto length error!\n"); 19382f072d75SZaibo Xu return -EINVAL; 19392f072d75SZaibo Xu } 19402f072d75SZaibo Xu 19412f072d75SZaibo Xu return 0; 19422f072d75SZaibo Xu } 19432f072d75SZaibo Xu 19442f072d75SZaibo Xu static int sec_aead_crypto(struct aead_request *a_req, bool encrypt) 19452f072d75SZaibo Xu { 19462f072d75SZaibo Xu struct crypto_aead *tfm = crypto_aead_reqtfm(a_req); 19472f072d75SZaibo Xu struct sec_req *req = aead_request_ctx(a_req); 19482f072d75SZaibo Xu struct sec_ctx *ctx = crypto_aead_ctx(tfm); 19492f072d75SZaibo Xu int ret; 19502f072d75SZaibo Xu 19519597efc3SKai Ye req->flag = a_req->base.flags; 19522f072d75SZaibo Xu req->aead_req.aead_req = a_req; 19532f072d75SZaibo Xu req->c_req.encrypt = encrypt; 19542f072d75SZaibo Xu req->ctx = ctx; 19552f072d75SZaibo Xu 19562f072d75SZaibo Xu ret = sec_aead_param_check(ctx, req); 19572f072d75SZaibo Xu if (unlikely(ret)) 19582f072d75SZaibo Xu return -EINVAL; 19592f072d75SZaibo Xu 19602f072d75SZaibo Xu return ctx->req_op->process(ctx, req); 19612f072d75SZaibo Xu } 19622f072d75SZaibo Xu 19632f072d75SZaibo Xu static int sec_aead_encrypt(struct aead_request *a_req) 19642f072d75SZaibo Xu { 19652f072d75SZaibo Xu return sec_aead_crypto(a_req, true); 19662f072d75SZaibo Xu } 19672f072d75SZaibo Xu 19682f072d75SZaibo Xu static int sec_aead_decrypt(struct aead_request *a_req) 19692f072d75SZaibo Xu { 19702f072d75SZaibo Xu return sec_aead_crypto(a_req, false); 19712f072d75SZaibo Xu } 19722f072d75SZaibo Xu 19732f072d75SZaibo Xu #define SEC_AEAD_GEN_ALG(sec_cra_name, sec_set_key, ctx_init,\ 19742f072d75SZaibo Xu ctx_exit, blk_size, iv_size, max_authsize)\ 19752f072d75SZaibo Xu {\ 19762f072d75SZaibo Xu .base = {\ 19772f072d75SZaibo Xu .cra_name = sec_cra_name,\ 19782f072d75SZaibo Xu .cra_driver_name = "hisi_sec_"sec_cra_name,\ 19792f072d75SZaibo Xu .cra_priority = SEC_PRIORITY,\ 1980b8aa7dc5SMikulas Patocka .cra_flags = CRYPTO_ALG_ASYNC | CRYPTO_ALG_ALLOCATES_MEMORY,\ 19812f072d75SZaibo Xu .cra_blocksize = blk_size,\ 19822f072d75SZaibo Xu .cra_ctxsize = sizeof(struct sec_ctx),\ 19832f072d75SZaibo Xu .cra_module = THIS_MODULE,\ 19842f072d75SZaibo Xu },\ 19852f072d75SZaibo Xu .init = ctx_init,\ 19862f072d75SZaibo Xu .exit = ctx_exit,\ 19872f072d75SZaibo Xu .setkey = sec_set_key,\ 19882f072d75SZaibo Xu .decrypt = sec_aead_decrypt,\ 19892f072d75SZaibo Xu .encrypt = sec_aead_encrypt,\ 19902f072d75SZaibo Xu .ivsize = iv_size,\ 19912f072d75SZaibo Xu .maxauthsize = max_authsize,\ 19922f072d75SZaibo Xu } 19932f072d75SZaibo Xu 19942f072d75SZaibo Xu #define SEC_AEAD_ALG(algname, keyfunc, aead_init, blksize, ivsize, authsize)\ 19952f072d75SZaibo Xu SEC_AEAD_GEN_ALG(algname, keyfunc, aead_init,\ 19962f072d75SZaibo Xu sec_aead_ctx_exit, blksize, ivsize, authsize) 19972f072d75SZaibo Xu 19982f072d75SZaibo Xu static struct aead_alg sec_aeads[] = { 19992f072d75SZaibo Xu SEC_AEAD_ALG("authenc(hmac(sha1),cbc(aes))", 20002f072d75SZaibo Xu sec_setkey_aes_cbc_sha1, sec_aead_sha1_ctx_init, 20012f072d75SZaibo Xu AES_BLOCK_SIZE, AES_BLOCK_SIZE, SHA1_DIGEST_SIZE), 20022f072d75SZaibo Xu 20032f072d75SZaibo Xu SEC_AEAD_ALG("authenc(hmac(sha256),cbc(aes))", 20042f072d75SZaibo Xu sec_setkey_aes_cbc_sha256, sec_aead_sha256_ctx_init, 20052f072d75SZaibo Xu AES_BLOCK_SIZE, AES_BLOCK_SIZE, SHA256_DIGEST_SIZE), 20062f072d75SZaibo Xu 20072f072d75SZaibo Xu SEC_AEAD_ALG("authenc(hmac(sha512),cbc(aes))", 20082f072d75SZaibo Xu sec_setkey_aes_cbc_sha512, sec_aead_sha512_ctx_init, 20092f072d75SZaibo Xu AES_BLOCK_SIZE, AES_BLOCK_SIZE, SHA512_DIGEST_SIZE), 20102f072d75SZaibo Xu }; 20112f072d75SZaibo Xu 20128123455aSMeng Yu int sec_register_to_crypto(struct hisi_qm *qm) 2013416d8220SZaibo Xu { 20143d29e98dSYang Shen int ret; 2015416d8220SZaibo Xu 2016416d8220SZaibo Xu /* To avoid repeat register */ 2017a181647cSZaibo Xu ret = crypto_register_skciphers(sec_skciphers, 2018a181647cSZaibo Xu ARRAY_SIZE(sec_skciphers)); 20192f072d75SZaibo Xu if (ret) 20202f072d75SZaibo Xu return ret; 20212f072d75SZaibo Xu 20227b44c0eeSKai Ye if (qm->ver > QM_HW_V2) { 20237b44c0eeSKai Ye ret = crypto_register_skciphers(sec_skciphers_v3, 20247b44c0eeSKai Ye ARRAY_SIZE(sec_skciphers_v3)); 20257b44c0eeSKai Ye if (ret) 20267b44c0eeSKai Ye goto reg_skcipher_fail; 20277b44c0eeSKai Ye } 20282f072d75SZaibo Xu ret = crypto_register_aeads(sec_aeads, ARRAY_SIZE(sec_aeads)); 20292f072d75SZaibo Xu if (ret) 20307b44c0eeSKai Ye goto reg_aead_fail; 20317b44c0eeSKai Ye return ret; 20327b44c0eeSKai Ye 20337b44c0eeSKai Ye reg_aead_fail: 20347b44c0eeSKai Ye if (qm->ver > QM_HW_V2) 20357b44c0eeSKai Ye crypto_unregister_skciphers(sec_skciphers_v3, 20367b44c0eeSKai Ye ARRAY_SIZE(sec_skciphers_v3)); 20377b44c0eeSKai Ye reg_skcipher_fail: 20383d29e98dSYang Shen crypto_unregister_skciphers(sec_skciphers, 20393d29e98dSYang Shen ARRAY_SIZE(sec_skciphers)); 2040416d8220SZaibo Xu return ret; 2041416d8220SZaibo Xu } 2042416d8220SZaibo Xu 20438123455aSMeng Yu void sec_unregister_from_crypto(struct hisi_qm *qm) 2044416d8220SZaibo Xu { 20457b44c0eeSKai Ye if (qm->ver > QM_HW_V2) 20467b44c0eeSKai Ye crypto_unregister_skciphers(sec_skciphers_v3, 20477b44c0eeSKai Ye ARRAY_SIZE(sec_skciphers_v3)); 2048a181647cSZaibo Xu crypto_unregister_skciphers(sec_skciphers, 2049a181647cSZaibo Xu ARRAY_SIZE(sec_skciphers)); 20502f072d75SZaibo Xu crypto_unregister_aeads(sec_aeads, ARRAY_SIZE(sec_aeads)); 20512f072d75SZaibo Xu } 2052