1416d8220SZaibo Xu // SPDX-License-Identifier: GPL-2.0
2416d8220SZaibo Xu /* Copyright (c) 2019 HiSilicon Limited. */
3416d8220SZaibo Xu
4416d8220SZaibo Xu #include <crypto/aes.h>
56c46a329SKai Ye #include <crypto/aead.h>
6416d8220SZaibo Xu #include <crypto/algapi.h>
72f072d75SZaibo Xu #include <crypto/authenc.h>
8416d8220SZaibo Xu #include <crypto/des.h>
92f072d75SZaibo Xu #include <crypto/hash.h>
102f072d75SZaibo Xu #include <crypto/internal/aead.h>
11ae6ce7b1SKai Ye #include <crypto/internal/des.h>
12a24d22b2SEric Biggers #include <crypto/sha1.h>
13a24d22b2SEric Biggers #include <crypto/sha2.h>
14416d8220SZaibo Xu #include <crypto/skcipher.h>
15416d8220SZaibo Xu #include <crypto/xts.h>
16416d8220SZaibo Xu #include <linux/crypto.h>
17416d8220SZaibo Xu #include <linux/dma-mapping.h>
18416d8220SZaibo Xu #include <linux/idr.h>
19416d8220SZaibo Xu
20416d8220SZaibo Xu #include "sec.h"
21416d8220SZaibo Xu #include "sec_crypto.h"
22416d8220SZaibo Xu
23416d8220SZaibo Xu #define SEC_PRIORITY 4001
24416d8220SZaibo Xu #define SEC_XTS_MIN_KEY_SIZE (2 * AES_MIN_KEY_SIZE)
255652d55aSKai Ye #define SEC_XTS_MID_KEY_SIZE (3 * AES_MIN_KEY_SIZE)
26416d8220SZaibo Xu #define SEC_XTS_MAX_KEY_SIZE (2 * AES_MAX_KEY_SIZE)
27416d8220SZaibo Xu #define SEC_DES3_2KEY_SIZE (2 * DES_KEY_SIZE)
28416d8220SZaibo Xu #define SEC_DES3_3KEY_SIZE (3 * DES_KEY_SIZE)
29416d8220SZaibo Xu
30416d8220SZaibo Xu /* SEC sqe(bd) bit operational relative MACRO */
31416d8220SZaibo Xu #define SEC_DE_OFFSET 1
32416d8220SZaibo Xu #define SEC_CIPHER_OFFSET 4
33416d8220SZaibo Xu #define SEC_SCENE_OFFSET 3
34416d8220SZaibo Xu #define SEC_DST_SGL_OFFSET 2
35416d8220SZaibo Xu #define SEC_SRC_SGL_OFFSET 7
36416d8220SZaibo Xu #define SEC_CKEY_OFFSET 9
37416d8220SZaibo Xu #define SEC_CMODE_OFFSET 12
382f072d75SZaibo Xu #define SEC_AKEY_OFFSET 5
392f072d75SZaibo Xu #define SEC_AEAD_ALG_OFFSET 11
402f072d75SZaibo Xu #define SEC_AUTH_OFFSET 6
412f072d75SZaibo Xu
42adc3f65aSKai Ye #define SEC_DE_OFFSET_V3 9
43adc3f65aSKai Ye #define SEC_SCENE_OFFSET_V3 5
44adc3f65aSKai Ye #define SEC_CKEY_OFFSET_V3 13
45566f060fSKai Ye #define SEC_CTR_CNT_OFFSET 25
46566f060fSKai Ye #define SEC_CTR_CNT_ROLLOVER 2
47adc3f65aSKai Ye #define SEC_SRC_SGL_OFFSET_V3 11
48adc3f65aSKai Ye #define SEC_DST_SGL_OFFSET_V3 14
49adc3f65aSKai Ye #define SEC_CALG_OFFSET_V3 4
50adc3f65aSKai Ye #define SEC_AKEY_OFFSET_V3 9
51adc3f65aSKai Ye #define SEC_MAC_OFFSET_V3 4
52adc3f65aSKai Ye #define SEC_AUTH_ALG_OFFSET_V3 15
53adc3f65aSKai Ye #define SEC_CIPHER_AUTH_V3 0xbf
54adc3f65aSKai Ye #define SEC_AUTH_CIPHER_V3 0x40
55416d8220SZaibo Xu #define SEC_FLAG_OFFSET 7
56416d8220SZaibo Xu #define SEC_FLAG_MASK 0x0780
57416d8220SZaibo Xu #define SEC_TYPE_MASK 0x0F
58416d8220SZaibo Xu #define SEC_DONE_MASK 0x0001
59668f1ab7SKai Ye #define SEC_ICV_MASK 0x000E
60adc3f65aSKai Ye #define SEC_SQE_LEN_RATE_MASK 0x3
61416d8220SZaibo Xu
62129a9f34SWeili Qian #define SEC_TOTAL_IV_SZ(depth) (SEC_IV_SIZE * (depth))
63416d8220SZaibo Xu #define SEC_SGL_SGE_NR 128
642f072d75SZaibo Xu #define SEC_CIPHER_AUTH 0xfe
652f072d75SZaibo Xu #define SEC_AUTH_CIPHER 0x1
662f072d75SZaibo Xu #define SEC_MAX_MAC_LEN 64
672514f559SLongfang Liu #define SEC_MAX_AAD_LEN 65535
685e340558SKai Ye #define SEC_MAX_CCM_AAD_LEN 65279
69129a9f34SWeili Qian #define SEC_TOTAL_MAC_SZ(depth) (SEC_MAX_MAC_LEN * (depth))
7074b58db8SLongfang Liu
7174b58db8SLongfang Liu #define SEC_PBUF_SZ 512
7274b58db8SLongfang Liu #define SEC_PBUF_IV_OFFSET SEC_PBUF_SZ
7374b58db8SLongfang Liu #define SEC_PBUF_MAC_OFFSET (SEC_PBUF_SZ + SEC_IV_SIZE)
7474b58db8SLongfang Liu #define SEC_PBUF_PKG (SEC_PBUF_SZ + SEC_IV_SIZE + \
7574b58db8SLongfang Liu SEC_MAX_MAC_LEN * 2)
7674b58db8SLongfang Liu #define SEC_PBUF_NUM (PAGE_SIZE / SEC_PBUF_PKG)
77129a9f34SWeili Qian #define SEC_PBUF_PAGE_NUM(depth) ((depth) / SEC_PBUF_NUM)
78129a9f34SWeili Qian #define SEC_PBUF_LEFT_SZ(depth) (SEC_PBUF_PKG * ((depth) - \
79129a9f34SWeili Qian SEC_PBUF_PAGE_NUM(depth) * SEC_PBUF_NUM))
80129a9f34SWeili Qian #define SEC_TOTAL_PBUF_SZ(depth) (PAGE_SIZE * SEC_PBUF_PAGE_NUM(depth) + \
81129a9f34SWeili Qian SEC_PBUF_LEFT_SZ(depth))
8274b58db8SLongfang Liu
832f072d75SZaibo Xu #define SEC_SQE_LEN_RATE 4
84d6de2a59SZaibo Xu #define SEC_SQE_CFLAG 2
852f072d75SZaibo Xu #define SEC_SQE_AEAD_FLAG 3
86d6de2a59SZaibo Xu #define SEC_SQE_DONE 0x1
87668f1ab7SKai Ye #define SEC_ICV_ERR 0x2
88c16a70c1SKai Ye #define MIN_MAC_LEN 4
89c16a70c1SKai Ye #define MAC_LEN_MASK 0x1U
907b44c0eeSKai Ye #define MAX_INPUT_DATA_LEN 0xFFFE00
917b44c0eeSKai Ye #define BITS_MASK 0xFF
927b44c0eeSKai Ye #define BYTE_BITS 0x8
935652d55aSKai Ye #define SEC_XTS_NAME_SZ 0x3
94c16a70c1SKai Ye #define IV_CM_CAL_NUM 2
95c16a70c1SKai Ye #define IV_CL_MASK 0x7
96c16a70c1SKai Ye #define IV_CL_MIN 2
97c16a70c1SKai Ye #define IV_CL_MID 4
98c16a70c1SKai Ye #define IV_CL_MAX 8
99c16a70c1SKai Ye #define IV_FLAGS_OFFSET 0x6
100c16a70c1SKai Ye #define IV_CM_OFFSET 0x3
101c16a70c1SKai Ye #define IV_LAST_BYTE1 1
102c16a70c1SKai Ye #define IV_LAST_BYTE2 2
103c16a70c1SKai Ye #define IV_LAST_BYTE_MASK 0xFF
104c16a70c1SKai Ye #define IV_CTR_INIT 0x1
105c16a70c1SKai Ye #define IV_BYTE_OFFSET 0x8
106416d8220SZaibo Xu
107921715b6SWenkai Lin struct sec_skcipher {
108921715b6SWenkai Lin u64 alg_msk;
109921715b6SWenkai Lin struct skcipher_alg alg;
110921715b6SWenkai Lin };
111921715b6SWenkai Lin
112921715b6SWenkai Lin struct sec_aead {
113921715b6SWenkai Lin u64 alg_msk;
114921715b6SWenkai Lin struct aead_alg alg;
115921715b6SWenkai Lin };
116921715b6SWenkai Lin
117416d8220SZaibo Xu /* Get an en/de-cipher queue cyclically to balance load over queues of TFM */
sec_alloc_queue_id(struct sec_ctx * ctx,struct sec_req * req)118a181647cSZaibo Xu static inline int sec_alloc_queue_id(struct sec_ctx *ctx, struct sec_req *req)
119416d8220SZaibo Xu {
120416d8220SZaibo Xu if (req->c_req.encrypt)
121416d8220SZaibo Xu return (u32)atomic_inc_return(&ctx->enc_qcyclic) %
122416d8220SZaibo Xu ctx->hlf_q_num;
123416d8220SZaibo Xu
124416d8220SZaibo Xu return (u32)atomic_inc_return(&ctx->dec_qcyclic) % ctx->hlf_q_num +
125416d8220SZaibo Xu ctx->hlf_q_num;
126416d8220SZaibo Xu }
127416d8220SZaibo Xu
sec_free_queue_id(struct sec_ctx * ctx,struct sec_req * req)128a181647cSZaibo Xu static inline void sec_free_queue_id(struct sec_ctx *ctx, struct sec_req *req)
129416d8220SZaibo Xu {
130416d8220SZaibo Xu if (req->c_req.encrypt)
131416d8220SZaibo Xu atomic_dec(&ctx->enc_qcyclic);
132416d8220SZaibo Xu else
133416d8220SZaibo Xu atomic_dec(&ctx->dec_qcyclic);
134416d8220SZaibo Xu }
135416d8220SZaibo Xu
sec_alloc_req_id(struct sec_req * req,struct sec_qp_ctx * qp_ctx)136416d8220SZaibo Xu static int sec_alloc_req_id(struct sec_req *req, struct sec_qp_ctx *qp_ctx)
137416d8220SZaibo Xu {
138416d8220SZaibo Xu int req_id;
139416d8220SZaibo Xu
14002884a4fSZhengchao Shao spin_lock_bh(&qp_ctx->req_lock);
141129a9f34SWeili Qian req_id = idr_alloc_cyclic(&qp_ctx->req_idr, NULL, 0, qp_ctx->qp->sq_depth, GFP_ATOMIC);
14202884a4fSZhengchao Shao spin_unlock_bh(&qp_ctx->req_lock);
143b9c8d897SZaibo Xu if (unlikely(req_id < 0)) {
144a44dce50SLongfang Liu dev_err(req->ctx->dev, "alloc req id fail!\n");
145416d8220SZaibo Xu return req_id;
146416d8220SZaibo Xu }
147416d8220SZaibo Xu
148416d8220SZaibo Xu req->qp_ctx = qp_ctx;
149416d8220SZaibo Xu qp_ctx->req_list[req_id] = req;
150633e507fSLongfang Liu
151416d8220SZaibo Xu return req_id;
152416d8220SZaibo Xu }
153416d8220SZaibo Xu
sec_free_req_id(struct sec_req * req)154416d8220SZaibo Xu static void sec_free_req_id(struct sec_req *req)
155416d8220SZaibo Xu {
156416d8220SZaibo Xu struct sec_qp_ctx *qp_ctx = req->qp_ctx;
157416d8220SZaibo Xu int req_id = req->req_id;
158416d8220SZaibo Xu
159129a9f34SWeili Qian if (unlikely(req_id < 0 || req_id >= qp_ctx->qp->sq_depth)) {
160a44dce50SLongfang Liu dev_err(req->ctx->dev, "free request id invalid!\n");
161416d8220SZaibo Xu return;
162416d8220SZaibo Xu }
163416d8220SZaibo Xu
164416d8220SZaibo Xu qp_ctx->req_list[req_id] = NULL;
165416d8220SZaibo Xu req->qp_ctx = NULL;
166416d8220SZaibo Xu
16702884a4fSZhengchao Shao spin_lock_bh(&qp_ctx->req_lock);
168416d8220SZaibo Xu idr_remove(&qp_ctx->req_idr, req_id);
16902884a4fSZhengchao Shao spin_unlock_bh(&qp_ctx->req_lock);
170416d8220SZaibo Xu }
171416d8220SZaibo Xu
pre_parse_finished_bd(struct bd_status * status,void * resp)172adc3f65aSKai Ye static u8 pre_parse_finished_bd(struct bd_status *status, void *resp)
173adc3f65aSKai Ye {
174adc3f65aSKai Ye struct sec_sqe *bd = resp;
175adc3f65aSKai Ye
176adc3f65aSKai Ye status->done = le16_to_cpu(bd->type2.done_flag) & SEC_DONE_MASK;
177668f1ab7SKai Ye status->icv = (le16_to_cpu(bd->type2.done_flag) & SEC_ICV_MASK) >> 1;
178adc3f65aSKai Ye status->flag = (le16_to_cpu(bd->type2.done_flag) &
179adc3f65aSKai Ye SEC_FLAG_MASK) >> SEC_FLAG_OFFSET;
180adc3f65aSKai Ye status->tag = le16_to_cpu(bd->type2.tag);
181adc3f65aSKai Ye status->err_type = bd->type2.error_type;
182adc3f65aSKai Ye
183adc3f65aSKai Ye return bd->type_cipher_auth & SEC_TYPE_MASK;
184adc3f65aSKai Ye }
185adc3f65aSKai Ye
pre_parse_finished_bd3(struct bd_status * status,void * resp)186adc3f65aSKai Ye static u8 pre_parse_finished_bd3(struct bd_status *status, void *resp)
187adc3f65aSKai Ye {
188adc3f65aSKai Ye struct sec_sqe3 *bd3 = resp;
189adc3f65aSKai Ye
190adc3f65aSKai Ye status->done = le16_to_cpu(bd3->done_flag) & SEC_DONE_MASK;
191668f1ab7SKai Ye status->icv = (le16_to_cpu(bd3->done_flag) & SEC_ICV_MASK) >> 1;
192adc3f65aSKai Ye status->flag = (le16_to_cpu(bd3->done_flag) &
193adc3f65aSKai Ye SEC_FLAG_MASK) >> SEC_FLAG_OFFSET;
194adc3f65aSKai Ye status->tag = le64_to_cpu(bd3->tag);
195adc3f65aSKai Ye status->err_type = bd3->error_type;
196adc3f65aSKai Ye
197adc3f65aSKai Ye return le32_to_cpu(bd3->bd_param) & SEC_TYPE_MASK;
198adc3f65aSKai Ye }
199adc3f65aSKai Ye
sec_cb_status_check(struct sec_req * req,struct bd_status * status)200adc3f65aSKai Ye static int sec_cb_status_check(struct sec_req *req,
201adc3f65aSKai Ye struct bd_status *status)
202adc3f65aSKai Ye {
203adc3f65aSKai Ye struct sec_ctx *ctx = req->ctx;
204adc3f65aSKai Ye
205adc3f65aSKai Ye if (unlikely(req->err_type || status->done != SEC_SQE_DONE)) {
206adc3f65aSKai Ye dev_err_ratelimited(ctx->dev, "err_type[%d], done[%u]\n",
207adc3f65aSKai Ye req->err_type, status->done);
208adc3f65aSKai Ye return -EIO;
209adc3f65aSKai Ye }
210adc3f65aSKai Ye
211adc3f65aSKai Ye if (unlikely(ctx->alg_type == SEC_SKCIPHER)) {
212adc3f65aSKai Ye if (unlikely(status->flag != SEC_SQE_CFLAG)) {
213adc3f65aSKai Ye dev_err_ratelimited(ctx->dev, "flag[%u]\n",
214adc3f65aSKai Ye status->flag);
215adc3f65aSKai Ye return -EIO;
216adc3f65aSKai Ye }
217668f1ab7SKai Ye } else if (unlikely(ctx->alg_type == SEC_AEAD)) {
218668f1ab7SKai Ye if (unlikely(status->flag != SEC_SQE_AEAD_FLAG ||
219668f1ab7SKai Ye status->icv == SEC_ICV_ERR)) {
220668f1ab7SKai Ye dev_err_ratelimited(ctx->dev,
221668f1ab7SKai Ye "flag[%u], icv[%u]\n",
222668f1ab7SKai Ye status->flag, status->icv);
223668f1ab7SKai Ye return -EBADMSG;
224668f1ab7SKai Ye }
225adc3f65aSKai Ye }
226adc3f65aSKai Ye
227adc3f65aSKai Ye return 0;
228adc3f65aSKai Ye }
229adc3f65aSKai Ye
sec_req_cb(struct hisi_qp * qp,void * resp)230416d8220SZaibo Xu static void sec_req_cb(struct hisi_qp *qp, void *resp)
231416d8220SZaibo Xu {
232416d8220SZaibo Xu struct sec_qp_ctx *qp_ctx = qp->qp_ctx;
2338213a1a6SKai Ye struct sec_dfx *dfx = &qp_ctx->ctx->sec->debug.dfx;
234adc3f65aSKai Ye u8 type_supported = qp_ctx->ctx->type_supported;
235adc3f65aSKai Ye struct bd_status status;
236d6de2a59SZaibo Xu struct sec_ctx *ctx;
237d6de2a59SZaibo Xu struct sec_req *req;
238adc3f65aSKai Ye int err;
239416d8220SZaibo Xu u8 type;
240416d8220SZaibo Xu
241adc3f65aSKai Ye if (type_supported == SEC_BD_TYPE2) {
242adc3f65aSKai Ye type = pre_parse_finished_bd(&status, resp);
243adc3f65aSKai Ye req = qp_ctx->req_list[status.tag];
244adc3f65aSKai Ye } else {
245adc3f65aSKai Ye type = pre_parse_finished_bd3(&status, resp);
246adc3f65aSKai Ye req = (void *)(uintptr_t)status.tag;
247adc3f65aSKai Ye }
248adc3f65aSKai Ye
249adc3f65aSKai Ye if (unlikely(type != type_supported)) {
2508213a1a6SKai Ye atomic64_inc(&dfx->err_bd_cnt);
25149838259SKai Ye pr_err("err bd type [%u]\n", type);
252416d8220SZaibo Xu return;
253416d8220SZaibo Xu }
254416d8220SZaibo Xu
2558213a1a6SKai Ye if (unlikely(!req)) {
2568213a1a6SKai Ye atomic64_inc(&dfx->invalid_req_cnt);
2579597efc3SKai Ye atomic_inc(&qp->qp_status.used);
2588213a1a6SKai Ye return;
2598213a1a6SKai Ye }
260adc3f65aSKai Ye
261adc3f65aSKai Ye req->err_type = status.err_type;
262d6de2a59SZaibo Xu ctx = req->ctx;
263adc3f65aSKai Ye err = sec_cb_status_check(req, &status);
264adc3f65aSKai Ye if (err)
2658213a1a6SKai Ye atomic64_inc(&dfx->done_flag_cnt);
2661e9bc276SZaibo Xu
2678213a1a6SKai Ye atomic64_inc(&dfx->recv_cnt);
268416d8220SZaibo Xu
269d6de2a59SZaibo Xu ctx->req_op->buf_unmap(ctx, req);
270d6de2a59SZaibo Xu
271310ea0acSZaibo Xu ctx->req_op->callback(ctx, req, err);
272416d8220SZaibo Xu }
273416d8220SZaibo Xu
sec_bd_send(struct sec_ctx * ctx,struct sec_req * req)274416d8220SZaibo Xu static int sec_bd_send(struct sec_ctx *ctx, struct sec_req *req)
275416d8220SZaibo Xu {
276416d8220SZaibo Xu struct sec_qp_ctx *qp_ctx = req->qp_ctx;
277416d8220SZaibo Xu int ret;
278416d8220SZaibo Xu
2799597efc3SKai Ye if (ctx->fake_req_limit <=
2809597efc3SKai Ye atomic_read(&qp_ctx->qp->qp_status.used) &&
2819597efc3SKai Ye !(req->flag & CRYPTO_TFM_REQ_MAY_BACKLOG))
2829597efc3SKai Ye return -EBUSY;
2839597efc3SKai Ye
28402884a4fSZhengchao Shao spin_lock_bh(&qp_ctx->req_lock);
285416d8220SZaibo Xu ret = hisi_qp_send(qp_ctx->qp, &req->sec_sqe);
2869597efc3SKai Ye if (ctx->fake_req_limit <=
2879597efc3SKai Ye atomic_read(&qp_ctx->qp->qp_status.used) && !ret) {
2889597efc3SKai Ye list_add_tail(&req->backlog_head, &qp_ctx->backlog);
289cb1eeb75SArnd Bergmann atomic64_inc(&ctx->sec->debug.dfx.send_cnt);
2909597efc3SKai Ye atomic64_inc(&ctx->sec->debug.dfx.send_busy_cnt);
29102884a4fSZhengchao Shao spin_unlock_bh(&qp_ctx->req_lock);
2929597efc3SKai Ye return -EBUSY;
2939597efc3SKai Ye }
29402884a4fSZhengchao Shao spin_unlock_bh(&qp_ctx->req_lock);
295416d8220SZaibo Xu
296b9c8d897SZaibo Xu if (unlikely(ret == -EBUSY))
297416d8220SZaibo Xu return -ENOBUFS;
298416d8220SZaibo Xu
2999597efc3SKai Ye if (likely(!ret)) {
300416d8220SZaibo Xu ret = -EINPROGRESS;
3019597efc3SKai Ye atomic64_inc(&ctx->sec->debug.dfx.send_cnt);
3028213a1a6SKai Ye }
303416d8220SZaibo Xu
304416d8220SZaibo Xu return ret;
305416d8220SZaibo Xu }
306416d8220SZaibo Xu
3077c7d902aSZaibo Xu /* Get DMA memory resources */
sec_alloc_civ_resource(struct device * dev,struct sec_alg_res * res)3087c7d902aSZaibo Xu static int sec_alloc_civ_resource(struct device *dev, struct sec_alg_res *res)
3097c7d902aSZaibo Xu {
310129a9f34SWeili Qian u16 q_depth = res->depth;
3117c7d902aSZaibo Xu int i;
3127c7d902aSZaibo Xu
313129a9f34SWeili Qian res->c_ivin = dma_alloc_coherent(dev, SEC_TOTAL_IV_SZ(q_depth),
3147c7d902aSZaibo Xu &res->c_ivin_dma, GFP_KERNEL);
3157c7d902aSZaibo Xu if (!res->c_ivin)
3167c7d902aSZaibo Xu return -ENOMEM;
3177c7d902aSZaibo Xu
318129a9f34SWeili Qian for (i = 1; i < q_depth; i++) {
3197c7d902aSZaibo Xu res[i].c_ivin_dma = res->c_ivin_dma + i * SEC_IV_SIZE;
3207c7d902aSZaibo Xu res[i].c_ivin = res->c_ivin + i * SEC_IV_SIZE;
3217c7d902aSZaibo Xu }
3227c7d902aSZaibo Xu
3237c7d902aSZaibo Xu return 0;
3247c7d902aSZaibo Xu }
3257c7d902aSZaibo Xu
sec_free_civ_resource(struct device * dev,struct sec_alg_res * res)3267c7d902aSZaibo Xu static void sec_free_civ_resource(struct device *dev, struct sec_alg_res *res)
3277c7d902aSZaibo Xu {
3287c7d902aSZaibo Xu if (res->c_ivin)
329129a9f34SWeili Qian dma_free_coherent(dev, SEC_TOTAL_IV_SZ(res->depth),
3307c7d902aSZaibo Xu res->c_ivin, res->c_ivin_dma);
3317c7d902aSZaibo Xu }
3327c7d902aSZaibo Xu
sec_alloc_aiv_resource(struct device * dev,struct sec_alg_res * res)333c16a70c1SKai Ye static int sec_alloc_aiv_resource(struct device *dev, struct sec_alg_res *res)
334c16a70c1SKai Ye {
335129a9f34SWeili Qian u16 q_depth = res->depth;
336c16a70c1SKai Ye int i;
337c16a70c1SKai Ye
338129a9f34SWeili Qian res->a_ivin = dma_alloc_coherent(dev, SEC_TOTAL_IV_SZ(q_depth),
339c16a70c1SKai Ye &res->a_ivin_dma, GFP_KERNEL);
340c16a70c1SKai Ye if (!res->a_ivin)
341c16a70c1SKai Ye return -ENOMEM;
342c16a70c1SKai Ye
343129a9f34SWeili Qian for (i = 1; i < q_depth; i++) {
344c16a70c1SKai Ye res[i].a_ivin_dma = res->a_ivin_dma + i * SEC_IV_SIZE;
345c16a70c1SKai Ye res[i].a_ivin = res->a_ivin + i * SEC_IV_SIZE;
346c16a70c1SKai Ye }
347c16a70c1SKai Ye
348c16a70c1SKai Ye return 0;
349c16a70c1SKai Ye }
350c16a70c1SKai Ye
sec_free_aiv_resource(struct device * dev,struct sec_alg_res * res)351c16a70c1SKai Ye static void sec_free_aiv_resource(struct device *dev, struct sec_alg_res *res)
352c16a70c1SKai Ye {
353c16a70c1SKai Ye if (res->a_ivin)
354129a9f34SWeili Qian dma_free_coherent(dev, SEC_TOTAL_IV_SZ(res->depth),
355c16a70c1SKai Ye res->a_ivin, res->a_ivin_dma);
356c16a70c1SKai Ye }
357c16a70c1SKai Ye
sec_alloc_mac_resource(struct device * dev,struct sec_alg_res * res)3582f072d75SZaibo Xu static int sec_alloc_mac_resource(struct device *dev, struct sec_alg_res *res)
3592f072d75SZaibo Xu {
360129a9f34SWeili Qian u16 q_depth = res->depth;
3612f072d75SZaibo Xu int i;
3622f072d75SZaibo Xu
363129a9f34SWeili Qian res->out_mac = dma_alloc_coherent(dev, SEC_TOTAL_MAC_SZ(q_depth) << 1,
3642f072d75SZaibo Xu &res->out_mac_dma, GFP_KERNEL);
3652f072d75SZaibo Xu if (!res->out_mac)
3662f072d75SZaibo Xu return -ENOMEM;
3672f072d75SZaibo Xu
368129a9f34SWeili Qian for (i = 1; i < q_depth; i++) {
3692f072d75SZaibo Xu res[i].out_mac_dma = res->out_mac_dma +
3702f072d75SZaibo Xu i * (SEC_MAX_MAC_LEN << 1);
3712f072d75SZaibo Xu res[i].out_mac = res->out_mac + i * (SEC_MAX_MAC_LEN << 1);
3722f072d75SZaibo Xu }
3732f072d75SZaibo Xu
3742f072d75SZaibo Xu return 0;
3752f072d75SZaibo Xu }
3762f072d75SZaibo Xu
sec_free_mac_resource(struct device * dev,struct sec_alg_res * res)3772f072d75SZaibo Xu static void sec_free_mac_resource(struct device *dev, struct sec_alg_res *res)
3782f072d75SZaibo Xu {
3792f072d75SZaibo Xu if (res->out_mac)
380129a9f34SWeili Qian dma_free_coherent(dev, SEC_TOTAL_MAC_SZ(res->depth) << 1,
3812f072d75SZaibo Xu res->out_mac, res->out_mac_dma);
3822f072d75SZaibo Xu }
3832f072d75SZaibo Xu
sec_free_pbuf_resource(struct device * dev,struct sec_alg_res * res)38474b58db8SLongfang Liu static void sec_free_pbuf_resource(struct device *dev, struct sec_alg_res *res)
38574b58db8SLongfang Liu {
38674b58db8SLongfang Liu if (res->pbuf)
387129a9f34SWeili Qian dma_free_coherent(dev, SEC_TOTAL_PBUF_SZ(res->depth),
38874b58db8SLongfang Liu res->pbuf, res->pbuf_dma);
38974b58db8SLongfang Liu }
39074b58db8SLongfang Liu
39174b58db8SLongfang Liu /*
39274b58db8SLongfang Liu * To improve performance, pbuffer is used for
39374b58db8SLongfang Liu * small packets (< 512Bytes) as IOMMU translation using.
39474b58db8SLongfang Liu */
sec_alloc_pbuf_resource(struct device * dev,struct sec_alg_res * res)39574b58db8SLongfang Liu static int sec_alloc_pbuf_resource(struct device *dev, struct sec_alg_res *res)
39674b58db8SLongfang Liu {
397129a9f34SWeili Qian u16 q_depth = res->depth;
398129a9f34SWeili Qian int size = SEC_PBUF_PAGE_NUM(q_depth);
39974b58db8SLongfang Liu int pbuf_page_offset;
40074b58db8SLongfang Liu int i, j, k;
40174b58db8SLongfang Liu
402129a9f34SWeili Qian res->pbuf = dma_alloc_coherent(dev, SEC_TOTAL_PBUF_SZ(q_depth),
40374b58db8SLongfang Liu &res->pbuf_dma, GFP_KERNEL);
40474b58db8SLongfang Liu if (!res->pbuf)
40574b58db8SLongfang Liu return -ENOMEM;
40674b58db8SLongfang Liu
40774b58db8SLongfang Liu /*
40874b58db8SLongfang Liu * SEC_PBUF_PKG contains data pbuf, iv and
40974b58db8SLongfang Liu * out_mac : <SEC_PBUF|SEC_IV|SEC_MAC>
41074b58db8SLongfang Liu * Every PAGE contains six SEC_PBUF_PKG
41174b58db8SLongfang Liu * The sec_qp_ctx contains QM_Q_DEPTH numbers of SEC_PBUF_PKG
41274b58db8SLongfang Liu * So we need SEC_PBUF_PAGE_NUM numbers of PAGE
41374b58db8SLongfang Liu * for the SEC_TOTAL_PBUF_SZ
41474b58db8SLongfang Liu */
415129a9f34SWeili Qian for (i = 0; i <= size; i++) {
41674b58db8SLongfang Liu pbuf_page_offset = PAGE_SIZE * i;
41774b58db8SLongfang Liu for (j = 0; j < SEC_PBUF_NUM; j++) {
41874b58db8SLongfang Liu k = i * SEC_PBUF_NUM + j;
419129a9f34SWeili Qian if (k == q_depth)
42074b58db8SLongfang Liu break;
42174b58db8SLongfang Liu res[k].pbuf = res->pbuf +
42274b58db8SLongfang Liu j * SEC_PBUF_PKG + pbuf_page_offset;
42374b58db8SLongfang Liu res[k].pbuf_dma = res->pbuf_dma +
42474b58db8SLongfang Liu j * SEC_PBUF_PKG + pbuf_page_offset;
42574b58db8SLongfang Liu }
42674b58db8SLongfang Liu }
427633e507fSLongfang Liu
42874b58db8SLongfang Liu return 0;
42974b58db8SLongfang Liu }
43074b58db8SLongfang Liu
sec_alg_resource_alloc(struct sec_ctx * ctx,struct sec_qp_ctx * qp_ctx)4317c7d902aSZaibo Xu static int sec_alg_resource_alloc(struct sec_ctx *ctx,
4327c7d902aSZaibo Xu struct sec_qp_ctx *qp_ctx)
4337c7d902aSZaibo Xu {
4342f072d75SZaibo Xu struct sec_alg_res *res = qp_ctx->res;
435a44dce50SLongfang Liu struct device *dev = ctx->dev;
4362f072d75SZaibo Xu int ret;
4377c7d902aSZaibo Xu
4382f072d75SZaibo Xu ret = sec_alloc_civ_resource(dev, res);
4392f072d75SZaibo Xu if (ret)
4402f072d75SZaibo Xu return ret;
4412f072d75SZaibo Xu
4422f072d75SZaibo Xu if (ctx->alg_type == SEC_AEAD) {
443c16a70c1SKai Ye ret = sec_alloc_aiv_resource(dev, res);
444c16a70c1SKai Ye if (ret)
445c16a70c1SKai Ye goto alloc_aiv_fail;
446c16a70c1SKai Ye
4472f072d75SZaibo Xu ret = sec_alloc_mac_resource(dev, res);
4482f072d75SZaibo Xu if (ret)
449c16a70c1SKai Ye goto alloc_mac_fail;
4502f072d75SZaibo Xu }
45174b58db8SLongfang Liu if (ctx->pbuf_supported) {
45274b58db8SLongfang Liu ret = sec_alloc_pbuf_resource(dev, res);
45374b58db8SLongfang Liu if (ret) {
45474b58db8SLongfang Liu dev_err(dev, "fail to alloc pbuf dma resource!\n");
45524efcec2SLongfang Liu goto alloc_pbuf_fail;
45674b58db8SLongfang Liu }
45774b58db8SLongfang Liu }
4582f072d75SZaibo Xu
4592f072d75SZaibo Xu return 0;
460633e507fSLongfang Liu
46124efcec2SLongfang Liu alloc_pbuf_fail:
46224efcec2SLongfang Liu if (ctx->alg_type == SEC_AEAD)
46324efcec2SLongfang Liu sec_free_mac_resource(dev, qp_ctx->res);
464c16a70c1SKai Ye alloc_mac_fail:
465c16a70c1SKai Ye if (ctx->alg_type == SEC_AEAD)
466c16a70c1SKai Ye sec_free_aiv_resource(dev, res);
467c16a70c1SKai Ye alloc_aiv_fail:
4682f072d75SZaibo Xu sec_free_civ_resource(dev, res);
4692f072d75SZaibo Xu return ret;
4707c7d902aSZaibo Xu }
4717c7d902aSZaibo Xu
sec_alg_resource_free(struct sec_ctx * ctx,struct sec_qp_ctx * qp_ctx)4727c7d902aSZaibo Xu static void sec_alg_resource_free(struct sec_ctx *ctx,
4737c7d902aSZaibo Xu struct sec_qp_ctx *qp_ctx)
4747c7d902aSZaibo Xu {
475a44dce50SLongfang Liu struct device *dev = ctx->dev;
4767c7d902aSZaibo Xu
4777c7d902aSZaibo Xu sec_free_civ_resource(dev, qp_ctx->res);
4782f072d75SZaibo Xu
47974b58db8SLongfang Liu if (ctx->pbuf_supported)
48074b58db8SLongfang Liu sec_free_pbuf_resource(dev, qp_ctx->res);
481*9f218863SChenghai Huang if (ctx->alg_type == SEC_AEAD) {
4822f072d75SZaibo Xu sec_free_mac_resource(dev, qp_ctx->res);
483*9f218863SChenghai Huang sec_free_aiv_resource(dev, qp_ctx->res);
484*9f218863SChenghai Huang }
4857c7d902aSZaibo Xu }
4867c7d902aSZaibo Xu
sec_alloc_qp_ctx_resource(struct hisi_qm * qm,struct sec_ctx * ctx,struct sec_qp_ctx * qp_ctx)487129a9f34SWeili Qian static int sec_alloc_qp_ctx_resource(struct hisi_qm *qm, struct sec_ctx *ctx,
488129a9f34SWeili Qian struct sec_qp_ctx *qp_ctx)
489129a9f34SWeili Qian {
490129a9f34SWeili Qian u16 q_depth = qp_ctx->qp->sq_depth;
491129a9f34SWeili Qian struct device *dev = ctx->dev;
492129a9f34SWeili Qian int ret = -ENOMEM;
493129a9f34SWeili Qian
494129a9f34SWeili Qian qp_ctx->req_list = kcalloc(q_depth, sizeof(struct sec_req *), GFP_KERNEL);
495129a9f34SWeili Qian if (!qp_ctx->req_list)
496129a9f34SWeili Qian return ret;
497129a9f34SWeili Qian
498129a9f34SWeili Qian qp_ctx->res = kcalloc(q_depth, sizeof(struct sec_alg_res), GFP_KERNEL);
499129a9f34SWeili Qian if (!qp_ctx->res)
500129a9f34SWeili Qian goto err_free_req_list;
501129a9f34SWeili Qian qp_ctx->res->depth = q_depth;
502129a9f34SWeili Qian
503129a9f34SWeili Qian qp_ctx->c_in_pool = hisi_acc_create_sgl_pool(dev, q_depth, SEC_SGL_SGE_NR);
504129a9f34SWeili Qian if (IS_ERR(qp_ctx->c_in_pool)) {
505129a9f34SWeili Qian dev_err(dev, "fail to create sgl pool for input!\n");
506129a9f34SWeili Qian goto err_free_res;
507129a9f34SWeili Qian }
508129a9f34SWeili Qian
509129a9f34SWeili Qian qp_ctx->c_out_pool = hisi_acc_create_sgl_pool(dev, q_depth, SEC_SGL_SGE_NR);
510129a9f34SWeili Qian if (IS_ERR(qp_ctx->c_out_pool)) {
511129a9f34SWeili Qian dev_err(dev, "fail to create sgl pool for output!\n");
512129a9f34SWeili Qian goto err_free_c_in_pool;
513129a9f34SWeili Qian }
514129a9f34SWeili Qian
515129a9f34SWeili Qian ret = sec_alg_resource_alloc(ctx, qp_ctx);
516129a9f34SWeili Qian if (ret)
517129a9f34SWeili Qian goto err_free_c_out_pool;
518129a9f34SWeili Qian
519129a9f34SWeili Qian return 0;
520129a9f34SWeili Qian
521129a9f34SWeili Qian err_free_c_out_pool:
522129a9f34SWeili Qian hisi_acc_free_sgl_pool(dev, qp_ctx->c_out_pool);
523129a9f34SWeili Qian err_free_c_in_pool:
524129a9f34SWeili Qian hisi_acc_free_sgl_pool(dev, qp_ctx->c_in_pool);
525129a9f34SWeili Qian err_free_res:
526129a9f34SWeili Qian kfree(qp_ctx->res);
527129a9f34SWeili Qian err_free_req_list:
528129a9f34SWeili Qian kfree(qp_ctx->req_list);
529129a9f34SWeili Qian return ret;
530129a9f34SWeili Qian }
531129a9f34SWeili Qian
sec_free_qp_ctx_resource(struct sec_ctx * ctx,struct sec_qp_ctx * qp_ctx)532129a9f34SWeili Qian static void sec_free_qp_ctx_resource(struct sec_ctx *ctx, struct sec_qp_ctx *qp_ctx)
533129a9f34SWeili Qian {
534129a9f34SWeili Qian struct device *dev = ctx->dev;
535129a9f34SWeili Qian
536129a9f34SWeili Qian sec_alg_resource_free(ctx, qp_ctx);
537129a9f34SWeili Qian hisi_acc_free_sgl_pool(dev, qp_ctx->c_out_pool);
538129a9f34SWeili Qian hisi_acc_free_sgl_pool(dev, qp_ctx->c_in_pool);
539129a9f34SWeili Qian kfree(qp_ctx->res);
540129a9f34SWeili Qian kfree(qp_ctx->req_list);
541129a9f34SWeili Qian }
542129a9f34SWeili Qian
sec_create_qp_ctx(struct hisi_qm * qm,struct sec_ctx * ctx,int qp_ctx_id,int alg_type)543416d8220SZaibo Xu static int sec_create_qp_ctx(struct hisi_qm *qm, struct sec_ctx *ctx,
544416d8220SZaibo Xu int qp_ctx_id, int alg_type)
545416d8220SZaibo Xu {
546416d8220SZaibo Xu struct sec_qp_ctx *qp_ctx;
547416d8220SZaibo Xu struct hisi_qp *qp;
548129a9f34SWeili Qian int ret;
549416d8220SZaibo Xu
550416d8220SZaibo Xu qp_ctx = &ctx->qp_ctx[qp_ctx_id];
5510b5e43bcSKai Ye qp = ctx->qps[qp_ctx_id];
552416d8220SZaibo Xu qp->req_type = 0;
553416d8220SZaibo Xu qp->qp_ctx = qp_ctx;
554416d8220SZaibo Xu qp_ctx->qp = qp;
555416d8220SZaibo Xu qp_ctx->ctx = ctx;
556416d8220SZaibo Xu
557adc3f65aSKai Ye qp->req_cb = sec_req_cb;
558adc3f65aSKai Ye
55902884a4fSZhengchao Shao spin_lock_init(&qp_ctx->req_lock);
560416d8220SZaibo Xu idr_init(&qp_ctx->req_idr);
5619597efc3SKai Ye INIT_LIST_HEAD(&qp_ctx->backlog);
562416d8220SZaibo Xu
563129a9f34SWeili Qian ret = sec_alloc_qp_ctx_resource(qm, ctx, qp_ctx);
564416d8220SZaibo Xu if (ret)
565129a9f34SWeili Qian goto err_destroy_idr;
566416d8220SZaibo Xu
567416d8220SZaibo Xu ret = hisi_qm_start_qp(qp, 0);
568416d8220SZaibo Xu if (ret < 0)
569129a9f34SWeili Qian goto err_resource_free;
570416d8220SZaibo Xu
571416d8220SZaibo Xu return 0;
572416d8220SZaibo Xu
573129a9f34SWeili Qian err_resource_free:
574129a9f34SWeili Qian sec_free_qp_ctx_resource(ctx, qp_ctx);
575416d8220SZaibo Xu err_destroy_idr:
576416d8220SZaibo Xu idr_destroy(&qp_ctx->req_idr);
577416d8220SZaibo Xu return ret;
578416d8220SZaibo Xu }
579416d8220SZaibo Xu
sec_release_qp_ctx(struct sec_ctx * ctx,struct sec_qp_ctx * qp_ctx)580416d8220SZaibo Xu static void sec_release_qp_ctx(struct sec_ctx *ctx,
581416d8220SZaibo Xu struct sec_qp_ctx *qp_ctx)
582416d8220SZaibo Xu {
583416d8220SZaibo Xu hisi_qm_stop_qp(qp_ctx->qp);
584129a9f34SWeili Qian sec_free_qp_ctx_resource(ctx, qp_ctx);
585416d8220SZaibo Xu idr_destroy(&qp_ctx->req_idr);
586416d8220SZaibo Xu }
587416d8220SZaibo Xu
sec_ctx_base_init(struct sec_ctx * ctx)588473a0f96SZaibo Xu static int sec_ctx_base_init(struct sec_ctx *ctx)
589416d8220SZaibo Xu {
590416d8220SZaibo Xu struct sec_dev *sec;
591416d8220SZaibo Xu int i, ret;
592416d8220SZaibo Xu
5930b5e43bcSKai Ye ctx->qps = sec_create_qps();
5940b5e43bcSKai Ye if (!ctx->qps) {
5950b5e43bcSKai Ye pr_err("Can not create sec qps!\n");
596416d8220SZaibo Xu return -ENODEV;
597416d8220SZaibo Xu }
5980b5e43bcSKai Ye
5990b5e43bcSKai Ye sec = container_of(ctx->qps[0]->qm, struct sec_dev, qm);
600416d8220SZaibo Xu ctx->sec = sec;
601a44dce50SLongfang Liu ctx->dev = &sec->qm.pdev->dev;
602a718cfceSZaibo Xu ctx->hlf_q_num = sec->ctx_q_num >> 1;
603416d8220SZaibo Xu
60474b58db8SLongfang Liu ctx->pbuf_supported = ctx->sec->iommu_used;
60574b58db8SLongfang Liu
606416d8220SZaibo Xu /* Half of queue depth is taken as fake requests limit in the queue. */
607129a9f34SWeili Qian ctx->fake_req_limit = ctx->qps[0]->sq_depth >> 1;
608416d8220SZaibo Xu ctx->qp_ctx = kcalloc(sec->ctx_q_num, sizeof(struct sec_qp_ctx),
609416d8220SZaibo Xu GFP_KERNEL);
61024efcec2SLongfang Liu if (!ctx->qp_ctx) {
61124efcec2SLongfang Liu ret = -ENOMEM;
61224efcec2SLongfang Liu goto err_destroy_qps;
61324efcec2SLongfang Liu }
614416d8220SZaibo Xu
615416d8220SZaibo Xu for (i = 0; i < sec->ctx_q_num; i++) {
616473a0f96SZaibo Xu ret = sec_create_qp_ctx(&sec->qm, ctx, i, 0);
617416d8220SZaibo Xu if (ret)
618416d8220SZaibo Xu goto err_sec_release_qp_ctx;
619416d8220SZaibo Xu }
620416d8220SZaibo Xu
621416d8220SZaibo Xu return 0;
62224efcec2SLongfang Liu
623416d8220SZaibo Xu err_sec_release_qp_ctx:
624416d8220SZaibo Xu for (i = i - 1; i >= 0; i--)
625416d8220SZaibo Xu sec_release_qp_ctx(ctx, &ctx->qp_ctx[i]);
626416d8220SZaibo Xu kfree(ctx->qp_ctx);
62724efcec2SLongfang Liu err_destroy_qps:
62824efcec2SLongfang Liu sec_destroy_qps(ctx->qps, sec->ctx_q_num);
629416d8220SZaibo Xu return ret;
630416d8220SZaibo Xu }
631416d8220SZaibo Xu
sec_ctx_base_uninit(struct sec_ctx * ctx)632473a0f96SZaibo Xu static void sec_ctx_base_uninit(struct sec_ctx *ctx)
633416d8220SZaibo Xu {
634473a0f96SZaibo Xu int i;
635416d8220SZaibo Xu
636416d8220SZaibo Xu for (i = 0; i < ctx->sec->ctx_q_num; i++)
637416d8220SZaibo Xu sec_release_qp_ctx(ctx, &ctx->qp_ctx[i]);
638416d8220SZaibo Xu
6390b5e43bcSKai Ye sec_destroy_qps(ctx->qps, ctx->sec->ctx_q_num);
640416d8220SZaibo Xu kfree(ctx->qp_ctx);
641416d8220SZaibo Xu }
642416d8220SZaibo Xu
sec_cipher_init(struct sec_ctx * ctx)643473a0f96SZaibo Xu static int sec_cipher_init(struct sec_ctx *ctx)
644473a0f96SZaibo Xu {
645473a0f96SZaibo Xu struct sec_cipher_ctx *c_ctx = &ctx->c_ctx;
646473a0f96SZaibo Xu
647a44dce50SLongfang Liu c_ctx->c_key = dma_alloc_coherent(ctx->dev, SEC_MAX_KEY_SIZE,
648473a0f96SZaibo Xu &c_ctx->c_key_dma, GFP_KERNEL);
649473a0f96SZaibo Xu if (!c_ctx->c_key)
650473a0f96SZaibo Xu return -ENOMEM;
651473a0f96SZaibo Xu
652473a0f96SZaibo Xu return 0;
653473a0f96SZaibo Xu }
654473a0f96SZaibo Xu
sec_cipher_uninit(struct sec_ctx * ctx)655473a0f96SZaibo Xu static void sec_cipher_uninit(struct sec_ctx *ctx)
656473a0f96SZaibo Xu {
657473a0f96SZaibo Xu struct sec_cipher_ctx *c_ctx = &ctx->c_ctx;
658473a0f96SZaibo Xu
659473a0f96SZaibo Xu memzero_explicit(c_ctx->c_key, SEC_MAX_KEY_SIZE);
660a44dce50SLongfang Liu dma_free_coherent(ctx->dev, SEC_MAX_KEY_SIZE,
661473a0f96SZaibo Xu c_ctx->c_key, c_ctx->c_key_dma);
662473a0f96SZaibo Xu }
663473a0f96SZaibo Xu
sec_auth_init(struct sec_ctx * ctx)6642f072d75SZaibo Xu static int sec_auth_init(struct sec_ctx *ctx)
6652f072d75SZaibo Xu {
6662f072d75SZaibo Xu struct sec_auth_ctx *a_ctx = &ctx->a_ctx;
6672f072d75SZaibo Xu
66845f5d017SKai Ye a_ctx->a_key = dma_alloc_coherent(ctx->dev, SEC_MAX_AKEY_SIZE,
6692f072d75SZaibo Xu &a_ctx->a_key_dma, GFP_KERNEL);
6702f072d75SZaibo Xu if (!a_ctx->a_key)
6712f072d75SZaibo Xu return -ENOMEM;
6722f072d75SZaibo Xu
6732f072d75SZaibo Xu return 0;
6742f072d75SZaibo Xu }
6752f072d75SZaibo Xu
sec_auth_uninit(struct sec_ctx * ctx)6762f072d75SZaibo Xu static void sec_auth_uninit(struct sec_ctx *ctx)
6772f072d75SZaibo Xu {
6782f072d75SZaibo Xu struct sec_auth_ctx *a_ctx = &ctx->a_ctx;
6792f072d75SZaibo Xu
68045f5d017SKai Ye memzero_explicit(a_ctx->a_key, SEC_MAX_AKEY_SIZE);
68145f5d017SKai Ye dma_free_coherent(ctx->dev, SEC_MAX_AKEY_SIZE,
6822f072d75SZaibo Xu a_ctx->a_key, a_ctx->a_key_dma);
6832f072d75SZaibo Xu }
6842f072d75SZaibo Xu
sec_skcipher_fbtfm_init(struct crypto_skcipher * tfm)6855652d55aSKai Ye static int sec_skcipher_fbtfm_init(struct crypto_skcipher *tfm)
6865652d55aSKai Ye {
6875652d55aSKai Ye const char *alg = crypto_tfm_alg_name(&tfm->base);
6885652d55aSKai Ye struct sec_ctx *ctx = crypto_skcipher_ctx(tfm);
6895652d55aSKai Ye struct sec_cipher_ctx *c_ctx = &ctx->c_ctx;
6905652d55aSKai Ye
6915652d55aSKai Ye c_ctx->fallback = false;
692e764d81dSKai Ye
693e764d81dSKai Ye /* Currently, only XTS mode need fallback tfm when using 192bit key */
6945652d55aSKai Ye if (likely(strncmp(alg, "xts", SEC_XTS_NAME_SZ)))
6955652d55aSKai Ye return 0;
6965652d55aSKai Ye
6975652d55aSKai Ye c_ctx->fbtfm = crypto_alloc_sync_skcipher(alg, 0,
6985652d55aSKai Ye CRYPTO_ALG_NEED_FALLBACK);
6995652d55aSKai Ye if (IS_ERR(c_ctx->fbtfm)) {
700e764d81dSKai Ye pr_err("failed to alloc xts mode fallback tfm!\n");
7015652d55aSKai Ye return PTR_ERR(c_ctx->fbtfm);
7025652d55aSKai Ye }
7035652d55aSKai Ye
7045652d55aSKai Ye return 0;
7055652d55aSKai Ye }
7065652d55aSKai Ye
sec_skcipher_init(struct crypto_skcipher * tfm)707473a0f96SZaibo Xu static int sec_skcipher_init(struct crypto_skcipher *tfm)
708473a0f96SZaibo Xu {
709473a0f96SZaibo Xu struct sec_ctx *ctx = crypto_skcipher_ctx(tfm);
710473a0f96SZaibo Xu int ret;
711473a0f96SZaibo Xu
7122f072d75SZaibo Xu ctx->alg_type = SEC_SKCIPHER;
713473a0f96SZaibo Xu crypto_skcipher_set_reqsize(tfm, sizeof(struct sec_req));
714473a0f96SZaibo Xu ctx->c_ctx.ivsize = crypto_skcipher_ivsize(tfm);
715473a0f96SZaibo Xu if (ctx->c_ctx.ivsize > SEC_IV_SIZE) {
7164b7aef02SLongfang Liu pr_err("get error skcipher iv size!\n");
717473a0f96SZaibo Xu return -EINVAL;
718473a0f96SZaibo Xu }
719473a0f96SZaibo Xu
720473a0f96SZaibo Xu ret = sec_ctx_base_init(ctx);
721473a0f96SZaibo Xu if (ret)
722473a0f96SZaibo Xu return ret;
723473a0f96SZaibo Xu
724473a0f96SZaibo Xu ret = sec_cipher_init(ctx);
725473a0f96SZaibo Xu if (ret)
726473a0f96SZaibo Xu goto err_cipher_init;
727473a0f96SZaibo Xu
7285652d55aSKai Ye ret = sec_skcipher_fbtfm_init(tfm);
7295652d55aSKai Ye if (ret)
7305652d55aSKai Ye goto err_fbtfm_init;
7315652d55aSKai Ye
732473a0f96SZaibo Xu return 0;
733633e507fSLongfang Liu
7345652d55aSKai Ye err_fbtfm_init:
7355652d55aSKai Ye sec_cipher_uninit(ctx);
736473a0f96SZaibo Xu err_cipher_init:
737473a0f96SZaibo Xu sec_ctx_base_uninit(ctx);
738473a0f96SZaibo Xu return ret;
739473a0f96SZaibo Xu }
740473a0f96SZaibo Xu
sec_skcipher_uninit(struct crypto_skcipher * tfm)741473a0f96SZaibo Xu static void sec_skcipher_uninit(struct crypto_skcipher *tfm)
742473a0f96SZaibo Xu {
743473a0f96SZaibo Xu struct sec_ctx *ctx = crypto_skcipher_ctx(tfm);
744473a0f96SZaibo Xu
7455652d55aSKai Ye if (ctx->c_ctx.fbtfm)
7465652d55aSKai Ye crypto_free_sync_skcipher(ctx->c_ctx.fbtfm);
7475652d55aSKai Ye
748473a0f96SZaibo Xu sec_cipher_uninit(ctx);
749473a0f96SZaibo Xu sec_ctx_base_uninit(ctx);
750473a0f96SZaibo Xu }
751473a0f96SZaibo Xu
sec_skcipher_3des_setkey(struct crypto_skcipher * tfm,const u8 * key,const u32 keylen,const enum sec_cmode c_mode)752ae6ce7b1SKai Ye static int sec_skcipher_3des_setkey(struct crypto_skcipher *tfm, const u8 *key,
753416d8220SZaibo Xu const u32 keylen,
754416d8220SZaibo Xu const enum sec_cmode c_mode)
755416d8220SZaibo Xu {
756ae6ce7b1SKai Ye struct sec_ctx *ctx = crypto_skcipher_ctx(tfm);
757ae6ce7b1SKai Ye struct sec_cipher_ctx *c_ctx = &ctx->c_ctx;
758ae6ce7b1SKai Ye int ret;
759ae6ce7b1SKai Ye
760ae6ce7b1SKai Ye ret = verify_skcipher_des3_key(tfm, key);
761ae6ce7b1SKai Ye if (ret)
762ae6ce7b1SKai Ye return ret;
763ae6ce7b1SKai Ye
764416d8220SZaibo Xu switch (keylen) {
765416d8220SZaibo Xu case SEC_DES3_2KEY_SIZE:
766416d8220SZaibo Xu c_ctx->c_key_len = SEC_CKEY_3DES_2KEY;
767416d8220SZaibo Xu break;
768416d8220SZaibo Xu case SEC_DES3_3KEY_SIZE:
769416d8220SZaibo Xu c_ctx->c_key_len = SEC_CKEY_3DES_3KEY;
770416d8220SZaibo Xu break;
771416d8220SZaibo Xu default:
772416d8220SZaibo Xu return -EINVAL;
773416d8220SZaibo Xu }
774416d8220SZaibo Xu
775416d8220SZaibo Xu return 0;
776416d8220SZaibo Xu }
777416d8220SZaibo Xu
sec_skcipher_aes_sm4_setkey(struct sec_cipher_ctx * c_ctx,const u32 keylen,const enum sec_cmode c_mode)778416d8220SZaibo Xu static int sec_skcipher_aes_sm4_setkey(struct sec_cipher_ctx *c_ctx,
779416d8220SZaibo Xu const u32 keylen,
780416d8220SZaibo Xu const enum sec_cmode c_mode)
781416d8220SZaibo Xu {
782416d8220SZaibo Xu if (c_mode == SEC_CMODE_XTS) {
783416d8220SZaibo Xu switch (keylen) {
784416d8220SZaibo Xu case SEC_XTS_MIN_KEY_SIZE:
785416d8220SZaibo Xu c_ctx->c_key_len = SEC_CKEY_128BIT;
786416d8220SZaibo Xu break;
7875652d55aSKai Ye case SEC_XTS_MID_KEY_SIZE:
7885652d55aSKai Ye c_ctx->fallback = true;
7895652d55aSKai Ye break;
790416d8220SZaibo Xu case SEC_XTS_MAX_KEY_SIZE:
791416d8220SZaibo Xu c_ctx->c_key_len = SEC_CKEY_256BIT;
792416d8220SZaibo Xu break;
793416d8220SZaibo Xu default:
794416d8220SZaibo Xu pr_err("hisi_sec2: xts mode key error!\n");
795416d8220SZaibo Xu return -EINVAL;
796416d8220SZaibo Xu }
797416d8220SZaibo Xu } else {
798adc3f65aSKai Ye if (c_ctx->c_alg == SEC_CALG_SM4 &&
799adc3f65aSKai Ye keylen != AES_KEYSIZE_128) {
800adc3f65aSKai Ye pr_err("hisi_sec2: sm4 key error!\n");
801adc3f65aSKai Ye return -EINVAL;
802adc3f65aSKai Ye } else {
803416d8220SZaibo Xu switch (keylen) {
804416d8220SZaibo Xu case AES_KEYSIZE_128:
805416d8220SZaibo Xu c_ctx->c_key_len = SEC_CKEY_128BIT;
806416d8220SZaibo Xu break;
807416d8220SZaibo Xu case AES_KEYSIZE_192:
808416d8220SZaibo Xu c_ctx->c_key_len = SEC_CKEY_192BIT;
809416d8220SZaibo Xu break;
810416d8220SZaibo Xu case AES_KEYSIZE_256:
811416d8220SZaibo Xu c_ctx->c_key_len = SEC_CKEY_256BIT;
812416d8220SZaibo Xu break;
813416d8220SZaibo Xu default:
814416d8220SZaibo Xu pr_err("hisi_sec2: aes key error!\n");
815416d8220SZaibo Xu return -EINVAL;
816416d8220SZaibo Xu }
817416d8220SZaibo Xu }
818adc3f65aSKai Ye }
819416d8220SZaibo Xu
820416d8220SZaibo Xu return 0;
821416d8220SZaibo Xu }
822416d8220SZaibo Xu
sec_skcipher_setkey(struct crypto_skcipher * tfm,const u8 * key,const u32 keylen,const enum sec_calg c_alg,const enum sec_cmode c_mode)823416d8220SZaibo Xu static int sec_skcipher_setkey(struct crypto_skcipher *tfm, const u8 *key,
824416d8220SZaibo Xu const u32 keylen, const enum sec_calg c_alg,
825416d8220SZaibo Xu const enum sec_cmode c_mode)
826416d8220SZaibo Xu {
827416d8220SZaibo Xu struct sec_ctx *ctx = crypto_skcipher_ctx(tfm);
828416d8220SZaibo Xu struct sec_cipher_ctx *c_ctx = &ctx->c_ctx;
829a44dce50SLongfang Liu struct device *dev = ctx->dev;
830416d8220SZaibo Xu int ret;
831416d8220SZaibo Xu
832416d8220SZaibo Xu if (c_mode == SEC_CMODE_XTS) {
833416d8220SZaibo Xu ret = xts_verify_key(tfm, key, keylen);
834416d8220SZaibo Xu if (ret) {
835a44dce50SLongfang Liu dev_err(dev, "xts mode key err!\n");
836416d8220SZaibo Xu return ret;
837416d8220SZaibo Xu }
838416d8220SZaibo Xu }
839416d8220SZaibo Xu
840416d8220SZaibo Xu c_ctx->c_alg = c_alg;
841416d8220SZaibo Xu c_ctx->c_mode = c_mode;
842416d8220SZaibo Xu
843416d8220SZaibo Xu switch (c_alg) {
844416d8220SZaibo Xu case SEC_CALG_3DES:
845ae6ce7b1SKai Ye ret = sec_skcipher_3des_setkey(tfm, key, keylen, c_mode);
846416d8220SZaibo Xu break;
847416d8220SZaibo Xu case SEC_CALG_AES:
848416d8220SZaibo Xu case SEC_CALG_SM4:
849416d8220SZaibo Xu ret = sec_skcipher_aes_sm4_setkey(c_ctx, keylen, c_mode);
850416d8220SZaibo Xu break;
851416d8220SZaibo Xu default:
852416d8220SZaibo Xu return -EINVAL;
853416d8220SZaibo Xu }
854416d8220SZaibo Xu
855416d8220SZaibo Xu if (ret) {
856a44dce50SLongfang Liu dev_err(dev, "set sec key err!\n");
857416d8220SZaibo Xu return ret;
858416d8220SZaibo Xu }
859416d8220SZaibo Xu
860416d8220SZaibo Xu memcpy(c_ctx->c_key, key, keylen);
861e764d81dSKai Ye if (c_ctx->fallback && c_ctx->fbtfm) {
8625652d55aSKai Ye ret = crypto_sync_skcipher_setkey(c_ctx->fbtfm, key, keylen);
8635652d55aSKai Ye if (ret) {
8645652d55aSKai Ye dev_err(dev, "failed to set fallback skcipher key!\n");
8655652d55aSKai Ye return ret;
8665652d55aSKai Ye }
8675652d55aSKai Ye }
868416d8220SZaibo Xu return 0;
869416d8220SZaibo Xu }
870416d8220SZaibo Xu
871416d8220SZaibo Xu #define GEN_SEC_SETKEY_FUNC(name, c_alg, c_mode) \
872416d8220SZaibo Xu static int sec_setkey_##name(struct crypto_skcipher *tfm, const u8 *key,\
873416d8220SZaibo Xu u32 keylen) \
874416d8220SZaibo Xu { \
875416d8220SZaibo Xu return sec_skcipher_setkey(tfm, key, keylen, c_alg, c_mode); \
876416d8220SZaibo Xu }
877416d8220SZaibo Xu
GEN_SEC_SETKEY_FUNC(aes_ecb,SEC_CALG_AES,SEC_CMODE_ECB)878416d8220SZaibo Xu GEN_SEC_SETKEY_FUNC(aes_ecb, SEC_CALG_AES, SEC_CMODE_ECB)
879416d8220SZaibo Xu GEN_SEC_SETKEY_FUNC(aes_cbc, SEC_CALG_AES, SEC_CMODE_CBC)
880416d8220SZaibo Xu GEN_SEC_SETKEY_FUNC(aes_xts, SEC_CALG_AES, SEC_CMODE_XTS)
8817b44c0eeSKai Ye GEN_SEC_SETKEY_FUNC(aes_ofb, SEC_CALG_AES, SEC_CMODE_OFB)
8827b44c0eeSKai Ye GEN_SEC_SETKEY_FUNC(aes_cfb, SEC_CALG_AES, SEC_CMODE_CFB)
8837b44c0eeSKai Ye GEN_SEC_SETKEY_FUNC(aes_ctr, SEC_CALG_AES, SEC_CMODE_CTR)
884416d8220SZaibo Xu GEN_SEC_SETKEY_FUNC(3des_ecb, SEC_CALG_3DES, SEC_CMODE_ECB)
885416d8220SZaibo Xu GEN_SEC_SETKEY_FUNC(3des_cbc, SEC_CALG_3DES, SEC_CMODE_CBC)
886416d8220SZaibo Xu GEN_SEC_SETKEY_FUNC(sm4_xts, SEC_CALG_SM4, SEC_CMODE_XTS)
887416d8220SZaibo Xu GEN_SEC_SETKEY_FUNC(sm4_cbc, SEC_CALG_SM4, SEC_CMODE_CBC)
8887b44c0eeSKai Ye GEN_SEC_SETKEY_FUNC(sm4_ofb, SEC_CALG_SM4, SEC_CMODE_OFB)
8897b44c0eeSKai Ye GEN_SEC_SETKEY_FUNC(sm4_cfb, SEC_CALG_SM4, SEC_CMODE_CFB)
8907b44c0eeSKai Ye GEN_SEC_SETKEY_FUNC(sm4_ctr, SEC_CALG_SM4, SEC_CMODE_CTR)
891416d8220SZaibo Xu
89274b58db8SLongfang Liu static int sec_cipher_pbuf_map(struct sec_ctx *ctx, struct sec_req *req,
89374b58db8SLongfang Liu struct scatterlist *src)
89474b58db8SLongfang Liu {
8956c46a329SKai Ye struct sec_aead_req *a_req = &req->aead_req;
8966c46a329SKai Ye struct aead_request *aead_req = a_req->aead_req;
89774b58db8SLongfang Liu struct sec_cipher_req *c_req = &req->c_req;
89874b58db8SLongfang Liu struct sec_qp_ctx *qp_ctx = req->qp_ctx;
899a44dce50SLongfang Liu struct device *dev = ctx->dev;
90074b58db8SLongfang Liu int copy_size, pbuf_length;
90174b58db8SLongfang Liu int req_id = req->req_id;
9026c46a329SKai Ye struct crypto_aead *tfm;
9036c46a329SKai Ye size_t authsize;
9046c46a329SKai Ye u8 *mac_offset;
90574b58db8SLongfang Liu
90674b58db8SLongfang Liu if (ctx->alg_type == SEC_AEAD)
90774b58db8SLongfang Liu copy_size = aead_req->cryptlen + aead_req->assoclen;
90874b58db8SLongfang Liu else
90974b58db8SLongfang Liu copy_size = c_req->c_len;
91074b58db8SLongfang Liu
91174b58db8SLongfang Liu pbuf_length = sg_copy_to_buffer(src, sg_nents(src),
9126c46a329SKai Ye qp_ctx->res[req_id].pbuf, copy_size);
91374b58db8SLongfang Liu if (unlikely(pbuf_length != copy_size)) {
91474b58db8SLongfang Liu dev_err(dev, "copy src data to pbuf error!\n");
91574b58db8SLongfang Liu return -EINVAL;
91674b58db8SLongfang Liu }
9176c46a329SKai Ye if (!c_req->encrypt && ctx->alg_type == SEC_AEAD) {
9186c46a329SKai Ye tfm = crypto_aead_reqtfm(aead_req);
9196c46a329SKai Ye authsize = crypto_aead_authsize(tfm);
9206c46a329SKai Ye mac_offset = qp_ctx->res[req_id].pbuf + copy_size - authsize;
9216c46a329SKai Ye memcpy(a_req->out_mac, mac_offset, authsize);
9226c46a329SKai Ye }
92374b58db8SLongfang Liu
9249039878aSKai Ye req->in_dma = qp_ctx->res[req_id].pbuf_dma;
9259039878aSKai Ye c_req->c_out_dma = req->in_dma;
92674b58db8SLongfang Liu
92774b58db8SLongfang Liu return 0;
92874b58db8SLongfang Liu }
92974b58db8SLongfang Liu
sec_cipher_pbuf_unmap(struct sec_ctx * ctx,struct sec_req * req,struct scatterlist * dst)93074b58db8SLongfang Liu static void sec_cipher_pbuf_unmap(struct sec_ctx *ctx, struct sec_req *req,
93174b58db8SLongfang Liu struct scatterlist *dst)
93274b58db8SLongfang Liu {
93374b58db8SLongfang Liu struct aead_request *aead_req = req->aead_req.aead_req;
93474b58db8SLongfang Liu struct sec_cipher_req *c_req = &req->c_req;
93574b58db8SLongfang Liu struct sec_qp_ctx *qp_ctx = req->qp_ctx;
93674b58db8SLongfang Liu int copy_size, pbuf_length;
93774b58db8SLongfang Liu int req_id = req->req_id;
93874b58db8SLongfang Liu
93974b58db8SLongfang Liu if (ctx->alg_type == SEC_AEAD)
94074b58db8SLongfang Liu copy_size = c_req->c_len + aead_req->assoclen;
94174b58db8SLongfang Liu else
94274b58db8SLongfang Liu copy_size = c_req->c_len;
94374b58db8SLongfang Liu
94474b58db8SLongfang Liu pbuf_length = sg_copy_from_buffer(dst, sg_nents(dst),
945668f1ab7SKai Ye qp_ctx->res[req_id].pbuf, copy_size);
94674b58db8SLongfang Liu if (unlikely(pbuf_length != copy_size))
947668f1ab7SKai Ye dev_err(ctx->dev, "copy pbuf data to dst error!\n");
948668f1ab7SKai Ye }
949668f1ab7SKai Ye
sec_aead_mac_init(struct sec_aead_req * req)950668f1ab7SKai Ye static int sec_aead_mac_init(struct sec_aead_req *req)
951668f1ab7SKai Ye {
952668f1ab7SKai Ye struct aead_request *aead_req = req->aead_req;
953668f1ab7SKai Ye struct crypto_aead *tfm = crypto_aead_reqtfm(aead_req);
954668f1ab7SKai Ye size_t authsize = crypto_aead_authsize(tfm);
955668f1ab7SKai Ye u8 *mac_out = req->out_mac;
956668f1ab7SKai Ye struct scatterlist *sgl = aead_req->src;
957668f1ab7SKai Ye size_t copy_size;
958668f1ab7SKai Ye off_t skip_size;
959668f1ab7SKai Ye
960668f1ab7SKai Ye /* Copy input mac */
961668f1ab7SKai Ye skip_size = aead_req->assoclen + aead_req->cryptlen - authsize;
962668f1ab7SKai Ye copy_size = sg_pcopy_to_buffer(sgl, sg_nents(sgl), mac_out,
963668f1ab7SKai Ye authsize, skip_size);
964668f1ab7SKai Ye if (unlikely(copy_size != authsize))
965668f1ab7SKai Ye return -EINVAL;
966668f1ab7SKai Ye
967668f1ab7SKai Ye return 0;
96874b58db8SLongfang Liu }
96974b58db8SLongfang Liu
sec_cipher_map(struct sec_ctx * ctx,struct sec_req * req,struct scatterlist * src,struct scatterlist * dst)9702514f559SLongfang Liu static int sec_cipher_map(struct sec_ctx *ctx, struct sec_req *req,
971416d8220SZaibo Xu struct scatterlist *src, struct scatterlist *dst)
972416d8220SZaibo Xu {
973416d8220SZaibo Xu struct sec_cipher_req *c_req = &req->c_req;
9742514f559SLongfang Liu struct sec_aead_req *a_req = &req->aead_req;
975416d8220SZaibo Xu struct sec_qp_ctx *qp_ctx = req->qp_ctx;
9762514f559SLongfang Liu struct sec_alg_res *res = &qp_ctx->res[req->req_id];
977a44dce50SLongfang Liu struct device *dev = ctx->dev;
97874b58db8SLongfang Liu int ret;
9792514f559SLongfang Liu
98074b58db8SLongfang Liu if (req->use_pbuf) {
98174b58db8SLongfang Liu c_req->c_ivin = res->pbuf + SEC_PBUF_IV_OFFSET;
98274b58db8SLongfang Liu c_req->c_ivin_dma = res->pbuf_dma + SEC_PBUF_IV_OFFSET;
98374b58db8SLongfang Liu if (ctx->alg_type == SEC_AEAD) {
984c16a70c1SKai Ye a_req->a_ivin = res->a_ivin;
985c16a70c1SKai Ye a_req->a_ivin_dma = res->a_ivin_dma;
98674b58db8SLongfang Liu a_req->out_mac = res->pbuf + SEC_PBUF_MAC_OFFSET;
98774b58db8SLongfang Liu a_req->out_mac_dma = res->pbuf_dma +
98874b58db8SLongfang Liu SEC_PBUF_MAC_OFFSET;
98974b58db8SLongfang Liu }
990668f1ab7SKai Ye ret = sec_cipher_pbuf_map(ctx, req, src);
99174b58db8SLongfang Liu
99274b58db8SLongfang Liu return ret;
99374b58db8SLongfang Liu }
9942514f559SLongfang Liu c_req->c_ivin = res->c_ivin;
9952514f559SLongfang Liu c_req->c_ivin_dma = res->c_ivin_dma;
9962514f559SLongfang Liu if (ctx->alg_type == SEC_AEAD) {
997c16a70c1SKai Ye a_req->a_ivin = res->a_ivin;
998c16a70c1SKai Ye a_req->a_ivin_dma = res->a_ivin_dma;
9992514f559SLongfang Liu a_req->out_mac = res->out_mac;
10002514f559SLongfang Liu a_req->out_mac_dma = res->out_mac_dma;
10012514f559SLongfang Liu }
1002416d8220SZaibo Xu
10039039878aSKai Ye req->in = hisi_acc_sg_buf_map_to_hw_sgl(dev, src,
1004416d8220SZaibo Xu qp_ctx->c_in_pool,
1005416d8220SZaibo Xu req->req_id,
10069039878aSKai Ye &req->in_dma);
10079039878aSKai Ye if (IS_ERR(req->in)) {
1008416d8220SZaibo Xu dev_err(dev, "fail to dma map input sgl buffers!\n");
10099039878aSKai Ye return PTR_ERR(req->in);
1010416d8220SZaibo Xu }
1011416d8220SZaibo Xu
1012668f1ab7SKai Ye if (!c_req->encrypt && ctx->alg_type == SEC_AEAD) {
1013668f1ab7SKai Ye ret = sec_aead_mac_init(a_req);
1014668f1ab7SKai Ye if (unlikely(ret)) {
1015668f1ab7SKai Ye dev_err(dev, "fail to init mac data for ICV!\n");
1016668f1ab7SKai Ye return ret;
1017668f1ab7SKai Ye }
1018668f1ab7SKai Ye }
10199039878aSKai Ye
1020416d8220SZaibo Xu if (dst == src) {
10219039878aSKai Ye c_req->c_out = req->in;
10229039878aSKai Ye c_req->c_out_dma = req->in_dma;
1023416d8220SZaibo Xu } else {
1024416d8220SZaibo Xu c_req->c_out = hisi_acc_sg_buf_map_to_hw_sgl(dev, dst,
1025416d8220SZaibo Xu qp_ctx->c_out_pool,
1026416d8220SZaibo Xu req->req_id,
1027416d8220SZaibo Xu &c_req->c_out_dma);
1028416d8220SZaibo Xu
1029416d8220SZaibo Xu if (IS_ERR(c_req->c_out)) {
1030416d8220SZaibo Xu dev_err(dev, "fail to dma map output sgl buffers!\n");
10319039878aSKai Ye hisi_acc_sg_buf_unmap(dev, src, req->in);
1032416d8220SZaibo Xu return PTR_ERR(c_req->c_out);
1033416d8220SZaibo Xu }
1034416d8220SZaibo Xu }
1035416d8220SZaibo Xu
1036416d8220SZaibo Xu return 0;
1037416d8220SZaibo Xu }
1038416d8220SZaibo Xu
sec_cipher_unmap(struct sec_ctx * ctx,struct sec_req * req,struct scatterlist * src,struct scatterlist * dst)10392514f559SLongfang Liu static void sec_cipher_unmap(struct sec_ctx *ctx, struct sec_req *req,
1040a181647cSZaibo Xu struct scatterlist *src, struct scatterlist *dst)
1041a181647cSZaibo Xu {
10422514f559SLongfang Liu struct sec_cipher_req *c_req = &req->c_req;
1043a44dce50SLongfang Liu struct device *dev = ctx->dev;
1044a181647cSZaibo Xu
104574b58db8SLongfang Liu if (req->use_pbuf) {
104674b58db8SLongfang Liu sec_cipher_pbuf_unmap(ctx, req, dst);
104774b58db8SLongfang Liu } else {
10482514f559SLongfang Liu if (dst != src)
10499039878aSKai Ye hisi_acc_sg_buf_unmap(dev, src, req->in);
10502514f559SLongfang Liu
10512514f559SLongfang Liu hisi_acc_sg_buf_unmap(dev, dst, c_req->c_out);
1052a181647cSZaibo Xu }
105374b58db8SLongfang Liu }
1054a181647cSZaibo Xu
sec_skcipher_sgl_map(struct sec_ctx * ctx,struct sec_req * req)1055416d8220SZaibo Xu static int sec_skcipher_sgl_map(struct sec_ctx *ctx, struct sec_req *req)
1056416d8220SZaibo Xu {
1057a181647cSZaibo Xu struct skcipher_request *sq = req->c_req.sk_req;
1058416d8220SZaibo Xu
10592514f559SLongfang Liu return sec_cipher_map(ctx, req, sq->src, sq->dst);
1060416d8220SZaibo Xu }
1061416d8220SZaibo Xu
sec_skcipher_sgl_unmap(struct sec_ctx * ctx,struct sec_req * req)1062416d8220SZaibo Xu static void sec_skcipher_sgl_unmap(struct sec_ctx *ctx, struct sec_req *req)
1063416d8220SZaibo Xu {
10642514f559SLongfang Liu struct skcipher_request *sq = req->c_req.sk_req;
1065416d8220SZaibo Xu
10662514f559SLongfang Liu sec_cipher_unmap(ctx, req, sq->src, sq->dst);
1067416d8220SZaibo Xu }
1068416d8220SZaibo Xu
sec_aead_aes_set_key(struct sec_cipher_ctx * c_ctx,struct crypto_authenc_keys * keys)10692f072d75SZaibo Xu static int sec_aead_aes_set_key(struct sec_cipher_ctx *c_ctx,
10702f072d75SZaibo Xu struct crypto_authenc_keys *keys)
10712f072d75SZaibo Xu {
10722f072d75SZaibo Xu switch (keys->enckeylen) {
10732f072d75SZaibo Xu case AES_KEYSIZE_128:
10742f072d75SZaibo Xu c_ctx->c_key_len = SEC_CKEY_128BIT;
10752f072d75SZaibo Xu break;
10762f072d75SZaibo Xu case AES_KEYSIZE_192:
10772f072d75SZaibo Xu c_ctx->c_key_len = SEC_CKEY_192BIT;
10782f072d75SZaibo Xu break;
10792f072d75SZaibo Xu case AES_KEYSIZE_256:
10802f072d75SZaibo Xu c_ctx->c_key_len = SEC_CKEY_256BIT;
10812f072d75SZaibo Xu break;
10822f072d75SZaibo Xu default:
10832f072d75SZaibo Xu pr_err("hisi_sec2: aead aes key error!\n");
10842f072d75SZaibo Xu return -EINVAL;
10852f072d75SZaibo Xu }
10862f072d75SZaibo Xu memcpy(c_ctx->c_key, keys->enckey, keys->enckeylen);
10872f072d75SZaibo Xu
10882f072d75SZaibo Xu return 0;
10892f072d75SZaibo Xu }
10902f072d75SZaibo Xu
sec_aead_auth_set_key(struct sec_auth_ctx * ctx,struct crypto_authenc_keys * keys)10912f072d75SZaibo Xu static int sec_aead_auth_set_key(struct sec_auth_ctx *ctx,
10922f072d75SZaibo Xu struct crypto_authenc_keys *keys)
10932f072d75SZaibo Xu {
10942f072d75SZaibo Xu struct crypto_shash *hash_tfm = ctx->hash_tfm;
10955761498cSKai Ye int blocksize, digestsize, ret;
10962f072d75SZaibo Xu
10972f072d75SZaibo Xu if (!keys->authkeylen) {
10982f072d75SZaibo Xu pr_err("hisi_sec2: aead auth key error!\n");
10992f072d75SZaibo Xu return -EINVAL;
11002f072d75SZaibo Xu }
11012f072d75SZaibo Xu
11022f072d75SZaibo Xu blocksize = crypto_shash_blocksize(hash_tfm);
11035761498cSKai Ye digestsize = crypto_shash_digestsize(hash_tfm);
11042f072d75SZaibo Xu if (keys->authkeylen > blocksize) {
110561c38e3aSEric Biggers ret = crypto_shash_tfm_digest(hash_tfm, keys->authkey,
11062f072d75SZaibo Xu keys->authkeylen, ctx->a_key);
11072f072d75SZaibo Xu if (ret) {
11082203d3f7SColin Ian King pr_err("hisi_sec2: aead auth digest error!\n");
11092f072d75SZaibo Xu return -EINVAL;
11102f072d75SZaibo Xu }
11115761498cSKai Ye ctx->a_key_len = digestsize;
11122f072d75SZaibo Xu } else {
11132f072d75SZaibo Xu memcpy(ctx->a_key, keys->authkey, keys->authkeylen);
11142f072d75SZaibo Xu ctx->a_key_len = keys->authkeylen;
11152f072d75SZaibo Xu }
11162f072d75SZaibo Xu
11172f072d75SZaibo Xu return 0;
11182f072d75SZaibo Xu }
11192f072d75SZaibo Xu
sec_aead_setauthsize(struct crypto_aead * aead,unsigned int authsize)11206c46a329SKai Ye static int sec_aead_setauthsize(struct crypto_aead *aead, unsigned int authsize)
11216c46a329SKai Ye {
11226c46a329SKai Ye struct crypto_tfm *tfm = crypto_aead_tfm(aead);
11236c46a329SKai Ye struct sec_ctx *ctx = crypto_tfm_ctx(tfm);
11246c46a329SKai Ye struct sec_auth_ctx *a_ctx = &ctx->a_ctx;
11256c46a329SKai Ye
11266c46a329SKai Ye if (unlikely(a_ctx->fallback_aead_tfm))
11276c46a329SKai Ye return crypto_aead_setauthsize(a_ctx->fallback_aead_tfm, authsize);
11286c46a329SKai Ye
11296c46a329SKai Ye return 0;
11306c46a329SKai Ye }
11316c46a329SKai Ye
sec_aead_fallback_setkey(struct sec_auth_ctx * a_ctx,struct crypto_aead * tfm,const u8 * key,unsigned int keylen)11326c46a329SKai Ye static int sec_aead_fallback_setkey(struct sec_auth_ctx *a_ctx,
11336c46a329SKai Ye struct crypto_aead *tfm, const u8 *key,
11346c46a329SKai Ye unsigned int keylen)
11356c46a329SKai Ye {
11366c46a329SKai Ye crypto_aead_clear_flags(a_ctx->fallback_aead_tfm, CRYPTO_TFM_REQ_MASK);
11376c46a329SKai Ye crypto_aead_set_flags(a_ctx->fallback_aead_tfm,
11386c46a329SKai Ye crypto_aead_get_flags(tfm) & CRYPTO_TFM_REQ_MASK);
11396c46a329SKai Ye return crypto_aead_setkey(a_ctx->fallback_aead_tfm, key, keylen);
11406c46a329SKai Ye }
11416c46a329SKai Ye
sec_aead_setkey(struct crypto_aead * tfm,const u8 * key,const u32 keylen,const enum sec_hash_alg a_alg,const enum sec_calg c_alg,const enum sec_mac_len mac_len,const enum sec_cmode c_mode)11422f072d75SZaibo Xu static int sec_aead_setkey(struct crypto_aead *tfm, const u8 *key,
11432f072d75SZaibo Xu const u32 keylen, const enum sec_hash_alg a_alg,
11442f072d75SZaibo Xu const enum sec_calg c_alg,
11452f072d75SZaibo Xu const enum sec_mac_len mac_len,
11462f072d75SZaibo Xu const enum sec_cmode c_mode)
11472f072d75SZaibo Xu {
11482f072d75SZaibo Xu struct sec_ctx *ctx = crypto_aead_ctx(tfm);
11492f072d75SZaibo Xu struct sec_cipher_ctx *c_ctx = &ctx->c_ctx;
11506c46a329SKai Ye struct sec_auth_ctx *a_ctx = &ctx->a_ctx;
1151a44dce50SLongfang Liu struct device *dev = ctx->dev;
11522f072d75SZaibo Xu struct crypto_authenc_keys keys;
11532f072d75SZaibo Xu int ret;
11542f072d75SZaibo Xu
11552f072d75SZaibo Xu ctx->a_ctx.a_alg = a_alg;
11562f072d75SZaibo Xu ctx->c_ctx.c_alg = c_alg;
11572f072d75SZaibo Xu ctx->a_ctx.mac_len = mac_len;
11582f072d75SZaibo Xu c_ctx->c_mode = c_mode;
11592f072d75SZaibo Xu
1160c16a70c1SKai Ye if (c_mode == SEC_CMODE_CCM || c_mode == SEC_CMODE_GCM) {
1161c16a70c1SKai Ye ret = sec_skcipher_aes_sm4_setkey(c_ctx, keylen, c_mode);
1162c16a70c1SKai Ye if (ret) {
1163c16a70c1SKai Ye dev_err(dev, "set sec aes ccm cipher key err!\n");
1164c16a70c1SKai Ye return ret;
1165c16a70c1SKai Ye }
1166c16a70c1SKai Ye memcpy(c_ctx->c_key, key, keylen);
1167c16a70c1SKai Ye
11686c46a329SKai Ye if (unlikely(a_ctx->fallback_aead_tfm)) {
11696c46a329SKai Ye ret = sec_aead_fallback_setkey(a_ctx, tfm, key, keylen);
11706c46a329SKai Ye if (ret)
11716c46a329SKai Ye return ret;
11726c46a329SKai Ye }
11736c46a329SKai Ye
1174c16a70c1SKai Ye return 0;
1175c16a70c1SKai Ye }
1176c16a70c1SKai Ye
11772f072d75SZaibo Xu if (crypto_authenc_extractkeys(&keys, key, keylen))
11782f072d75SZaibo Xu goto bad_key;
11792f072d75SZaibo Xu
11802f072d75SZaibo Xu ret = sec_aead_aes_set_key(c_ctx, &keys);
11812f072d75SZaibo Xu if (ret) {
1182a44dce50SLongfang Liu dev_err(dev, "set sec cipher key err!\n");
11832f072d75SZaibo Xu goto bad_key;
11842f072d75SZaibo Xu }
11852f072d75SZaibo Xu
11862f072d75SZaibo Xu ret = sec_aead_auth_set_key(&ctx->a_ctx, &keys);
11872f072d75SZaibo Xu if (ret) {
1188a44dce50SLongfang Liu dev_err(dev, "set sec auth key err!\n");
11892f072d75SZaibo Xu goto bad_key;
11902f072d75SZaibo Xu }
11912f072d75SZaibo Xu
1192adc3f65aSKai Ye if ((ctx->a_ctx.mac_len & SEC_SQE_LEN_RATE_MASK) ||
1193adc3f65aSKai Ye (ctx->a_ctx.a_key_len & SEC_SQE_LEN_RATE_MASK)) {
1194adc3f65aSKai Ye dev_err(dev, "MAC or AUTH key length error!\n");
1195adc3f65aSKai Ye goto bad_key;
1196adc3f65aSKai Ye }
1197adc3f65aSKai Ye
11982f072d75SZaibo Xu return 0;
1199633e507fSLongfang Liu
12002f072d75SZaibo Xu bad_key:
12012f072d75SZaibo Xu memzero_explicit(&keys, sizeof(struct crypto_authenc_keys));
12022f072d75SZaibo Xu return -EINVAL;
12032f072d75SZaibo Xu }
12042f072d75SZaibo Xu
12052f072d75SZaibo Xu
12062f072d75SZaibo Xu #define GEN_SEC_AEAD_SETKEY_FUNC(name, aalg, calg, maclen, cmode) \
12072f072d75SZaibo Xu static int sec_setkey_##name(struct crypto_aead *tfm, const u8 *key, \
12082f072d75SZaibo Xu u32 keylen) \
12092f072d75SZaibo Xu { \
12102f072d75SZaibo Xu return sec_aead_setkey(tfm, key, keylen, aalg, calg, maclen, cmode);\
12112f072d75SZaibo Xu }
12122f072d75SZaibo Xu
GEN_SEC_AEAD_SETKEY_FUNC(aes_cbc_sha1,SEC_A_HMAC_SHA1,SEC_CALG_AES,SEC_HMAC_SHA1_MAC,SEC_CMODE_CBC)12132f072d75SZaibo Xu GEN_SEC_AEAD_SETKEY_FUNC(aes_cbc_sha1, SEC_A_HMAC_SHA1,
12142f072d75SZaibo Xu SEC_CALG_AES, SEC_HMAC_SHA1_MAC, SEC_CMODE_CBC)
12152f072d75SZaibo Xu GEN_SEC_AEAD_SETKEY_FUNC(aes_cbc_sha256, SEC_A_HMAC_SHA256,
12162f072d75SZaibo Xu SEC_CALG_AES, SEC_HMAC_SHA256_MAC, SEC_CMODE_CBC)
12172f072d75SZaibo Xu GEN_SEC_AEAD_SETKEY_FUNC(aes_cbc_sha512, SEC_A_HMAC_SHA512,
12182f072d75SZaibo Xu SEC_CALG_AES, SEC_HMAC_SHA512_MAC, SEC_CMODE_CBC)
1219c16a70c1SKai Ye GEN_SEC_AEAD_SETKEY_FUNC(aes_ccm, 0, SEC_CALG_AES,
1220c16a70c1SKai Ye SEC_HMAC_CCM_MAC, SEC_CMODE_CCM)
1221c16a70c1SKai Ye GEN_SEC_AEAD_SETKEY_FUNC(aes_gcm, 0, SEC_CALG_AES,
1222c16a70c1SKai Ye SEC_HMAC_GCM_MAC, SEC_CMODE_GCM)
1223c16a70c1SKai Ye GEN_SEC_AEAD_SETKEY_FUNC(sm4_ccm, 0, SEC_CALG_SM4,
1224c16a70c1SKai Ye SEC_HMAC_CCM_MAC, SEC_CMODE_CCM)
1225c16a70c1SKai Ye GEN_SEC_AEAD_SETKEY_FUNC(sm4_gcm, 0, SEC_CALG_SM4,
1226c16a70c1SKai Ye SEC_HMAC_GCM_MAC, SEC_CMODE_GCM)
12272f072d75SZaibo Xu
12282f072d75SZaibo Xu static int sec_aead_sgl_map(struct sec_ctx *ctx, struct sec_req *req)
12292f072d75SZaibo Xu {
12302f072d75SZaibo Xu struct aead_request *aq = req->aead_req.aead_req;
12312f072d75SZaibo Xu
12322514f559SLongfang Liu return sec_cipher_map(ctx, req, aq->src, aq->dst);
12332f072d75SZaibo Xu }
12342f072d75SZaibo Xu
sec_aead_sgl_unmap(struct sec_ctx * ctx,struct sec_req * req)12352f072d75SZaibo Xu static void sec_aead_sgl_unmap(struct sec_ctx *ctx, struct sec_req *req)
12362f072d75SZaibo Xu {
12372f072d75SZaibo Xu struct aead_request *aq = req->aead_req.aead_req;
12382f072d75SZaibo Xu
12392514f559SLongfang Liu sec_cipher_unmap(ctx, req, aq->src, aq->dst);
12402f072d75SZaibo Xu }
12412f072d75SZaibo Xu
sec_request_transfer(struct sec_ctx * ctx,struct sec_req * req)1242416d8220SZaibo Xu static int sec_request_transfer(struct sec_ctx *ctx, struct sec_req *req)
1243416d8220SZaibo Xu {
1244416d8220SZaibo Xu int ret;
1245416d8220SZaibo Xu
1246416d8220SZaibo Xu ret = ctx->req_op->buf_map(ctx, req);
1247b9c8d897SZaibo Xu if (unlikely(ret))
1248416d8220SZaibo Xu return ret;
1249416d8220SZaibo Xu
1250416d8220SZaibo Xu ctx->req_op->do_transfer(ctx, req);
1251416d8220SZaibo Xu
1252416d8220SZaibo Xu ret = ctx->req_op->bd_fill(ctx, req);
1253b9c8d897SZaibo Xu if (unlikely(ret))
1254416d8220SZaibo Xu goto unmap_req_buf;
1255416d8220SZaibo Xu
1256416d8220SZaibo Xu return ret;
1257416d8220SZaibo Xu
1258416d8220SZaibo Xu unmap_req_buf:
1259416d8220SZaibo Xu ctx->req_op->buf_unmap(ctx, req);
1260416d8220SZaibo Xu return ret;
1261416d8220SZaibo Xu }
1262416d8220SZaibo Xu
sec_request_untransfer(struct sec_ctx * ctx,struct sec_req * req)1263416d8220SZaibo Xu static void sec_request_untransfer(struct sec_ctx *ctx, struct sec_req *req)
1264416d8220SZaibo Xu {
1265416d8220SZaibo Xu ctx->req_op->buf_unmap(ctx, req);
1266416d8220SZaibo Xu }
1267416d8220SZaibo Xu
sec_skcipher_copy_iv(struct sec_ctx * ctx,struct sec_req * req)1268416d8220SZaibo Xu static void sec_skcipher_copy_iv(struct sec_ctx *ctx, struct sec_req *req)
1269416d8220SZaibo Xu {
1270416d8220SZaibo Xu struct skcipher_request *sk_req = req->c_req.sk_req;
12712514f559SLongfang Liu struct sec_cipher_req *c_req = &req->c_req;
1272416d8220SZaibo Xu
12732514f559SLongfang Liu memcpy(c_req->c_ivin, sk_req->iv, ctx->c_ctx.ivsize);
1274416d8220SZaibo Xu }
1275416d8220SZaibo Xu
sec_skcipher_bd_fill(struct sec_ctx * ctx,struct sec_req * req)1276416d8220SZaibo Xu static int sec_skcipher_bd_fill(struct sec_ctx *ctx, struct sec_req *req)
1277416d8220SZaibo Xu {
1278416d8220SZaibo Xu struct sec_cipher_ctx *c_ctx = &ctx->c_ctx;
1279416d8220SZaibo Xu struct sec_cipher_req *c_req = &req->c_req;
1280416d8220SZaibo Xu struct sec_sqe *sec_sqe = &req->sec_sqe;
1281416d8220SZaibo Xu u8 scene, sa_type, da_type;
1282416d8220SZaibo Xu u8 bd_type, cipher;
12837c7d902aSZaibo Xu u8 de = 0;
1284416d8220SZaibo Xu
1285416d8220SZaibo Xu memset(sec_sqe, 0, sizeof(struct sec_sqe));
1286416d8220SZaibo Xu
1287416d8220SZaibo Xu sec_sqe->type2.c_key_addr = cpu_to_le64(c_ctx->c_key_dma);
12882514f559SLongfang Liu sec_sqe->type2.c_ivin_addr = cpu_to_le64(c_req->c_ivin_dma);
12899039878aSKai Ye sec_sqe->type2.data_src_addr = cpu_to_le64(req->in_dma);
1290416d8220SZaibo Xu sec_sqe->type2.data_dst_addr = cpu_to_le64(c_req->c_out_dma);
1291416d8220SZaibo Xu
1292416d8220SZaibo Xu sec_sqe->type2.icvw_kmode |= cpu_to_le16(((u16)c_ctx->c_mode) <<
1293416d8220SZaibo Xu SEC_CMODE_OFFSET);
1294416d8220SZaibo Xu sec_sqe->type2.c_alg = c_ctx->c_alg;
1295416d8220SZaibo Xu sec_sqe->type2.icvw_kmode |= cpu_to_le16(((u16)c_ctx->c_key_len) <<
1296416d8220SZaibo Xu SEC_CKEY_OFFSET);
1297416d8220SZaibo Xu
1298416d8220SZaibo Xu bd_type = SEC_BD_TYPE2;
1299416d8220SZaibo Xu if (c_req->encrypt)
1300416d8220SZaibo Xu cipher = SEC_CIPHER_ENC << SEC_CIPHER_OFFSET;
1301416d8220SZaibo Xu else
1302416d8220SZaibo Xu cipher = SEC_CIPHER_DEC << SEC_CIPHER_OFFSET;
1303416d8220SZaibo Xu sec_sqe->type_cipher_auth = bd_type | cipher;
1304416d8220SZaibo Xu
1305adc3f65aSKai Ye /* Set destination and source address type */
1306adc3f65aSKai Ye if (req->use_pbuf) {
130774b58db8SLongfang Liu sa_type = SEC_PBUF << SEC_SRC_SGL_OFFSET;
1308adc3f65aSKai Ye da_type = SEC_PBUF << SEC_DST_SGL_OFFSET;
1309adc3f65aSKai Ye } else {
1310416d8220SZaibo Xu sa_type = SEC_SGL << SEC_SRC_SGL_OFFSET;
1311adc3f65aSKai Ye da_type = SEC_SGL << SEC_DST_SGL_OFFSET;
1312adc3f65aSKai Ye }
1313adc3f65aSKai Ye
1314adc3f65aSKai Ye sec_sqe->sdm_addr_type |= da_type;
1315416d8220SZaibo Xu scene = SEC_COMM_SCENE << SEC_SCENE_OFFSET;
13169039878aSKai Ye if (req->in_dma != c_req->c_out_dma)
1317416d8220SZaibo Xu de = 0x1 << SEC_DE_OFFSET;
1318416d8220SZaibo Xu
1319416d8220SZaibo Xu sec_sqe->sds_sa_type = (de | scene | sa_type);
1320416d8220SZaibo Xu
1321416d8220SZaibo Xu sec_sqe->type2.clen_ivhlen |= cpu_to_le32(c_req->c_len);
1322416d8220SZaibo Xu sec_sqe->type2.tag = cpu_to_le16((u16)req->req_id);
1323416d8220SZaibo Xu
1324416d8220SZaibo Xu return 0;
1325416d8220SZaibo Xu }
1326416d8220SZaibo Xu
sec_skcipher_bd_fill_v3(struct sec_ctx * ctx,struct sec_req * req)1327adc3f65aSKai Ye static int sec_skcipher_bd_fill_v3(struct sec_ctx *ctx, struct sec_req *req)
1328adc3f65aSKai Ye {
1329adc3f65aSKai Ye struct sec_sqe3 *sec_sqe3 = &req->sec_sqe3;
1330adc3f65aSKai Ye struct sec_cipher_ctx *c_ctx = &ctx->c_ctx;
1331adc3f65aSKai Ye struct sec_cipher_req *c_req = &req->c_req;
1332adc3f65aSKai Ye u32 bd_param = 0;
1333adc3f65aSKai Ye u16 cipher;
1334adc3f65aSKai Ye
1335adc3f65aSKai Ye memset(sec_sqe3, 0, sizeof(struct sec_sqe3));
1336adc3f65aSKai Ye
1337adc3f65aSKai Ye sec_sqe3->c_key_addr = cpu_to_le64(c_ctx->c_key_dma);
1338adc3f65aSKai Ye sec_sqe3->no_scene.c_ivin_addr = cpu_to_le64(c_req->c_ivin_dma);
13399039878aSKai Ye sec_sqe3->data_src_addr = cpu_to_le64(req->in_dma);
1340adc3f65aSKai Ye sec_sqe3->data_dst_addr = cpu_to_le64(c_req->c_out_dma);
1341adc3f65aSKai Ye
1342adc3f65aSKai Ye sec_sqe3->c_mode_alg = ((u8)c_ctx->c_alg << SEC_CALG_OFFSET_V3) |
1343adc3f65aSKai Ye c_ctx->c_mode;
1344adc3f65aSKai Ye sec_sqe3->c_icv_key |= cpu_to_le16(((u16)c_ctx->c_key_len) <<
1345adc3f65aSKai Ye SEC_CKEY_OFFSET_V3);
1346adc3f65aSKai Ye
1347adc3f65aSKai Ye if (c_req->encrypt)
1348adc3f65aSKai Ye cipher = SEC_CIPHER_ENC;
1349adc3f65aSKai Ye else
1350adc3f65aSKai Ye cipher = SEC_CIPHER_DEC;
1351adc3f65aSKai Ye sec_sqe3->c_icv_key |= cpu_to_le16(cipher);
1352adc3f65aSKai Ye
1353566f060fSKai Ye /* Set the CTR counter mode is 128bit rollover */
1354566f060fSKai Ye sec_sqe3->auth_mac_key = cpu_to_le32((u32)SEC_CTR_CNT_ROLLOVER <<
1355566f060fSKai Ye SEC_CTR_CNT_OFFSET);
1356566f060fSKai Ye
1357adc3f65aSKai Ye if (req->use_pbuf) {
1358adc3f65aSKai Ye bd_param |= SEC_PBUF << SEC_SRC_SGL_OFFSET_V3;
1359adc3f65aSKai Ye bd_param |= SEC_PBUF << SEC_DST_SGL_OFFSET_V3;
1360adc3f65aSKai Ye } else {
1361adc3f65aSKai Ye bd_param |= SEC_SGL << SEC_SRC_SGL_OFFSET_V3;
1362adc3f65aSKai Ye bd_param |= SEC_SGL << SEC_DST_SGL_OFFSET_V3;
1363adc3f65aSKai Ye }
1364adc3f65aSKai Ye
1365adc3f65aSKai Ye bd_param |= SEC_COMM_SCENE << SEC_SCENE_OFFSET_V3;
13669039878aSKai Ye if (req->in_dma != c_req->c_out_dma)
1367adc3f65aSKai Ye bd_param |= 0x1 << SEC_DE_OFFSET_V3;
1368adc3f65aSKai Ye
1369adc3f65aSKai Ye bd_param |= SEC_BD_TYPE3;
1370adc3f65aSKai Ye sec_sqe3->bd_param = cpu_to_le32(bd_param);
1371adc3f65aSKai Ye
1372adc3f65aSKai Ye sec_sqe3->c_len_ivin |= cpu_to_le32(c_req->c_len);
1373adc3f65aSKai Ye sec_sqe3->tag = cpu_to_le64(req);
1374adc3f65aSKai Ye
1375adc3f65aSKai Ye return 0;
1376adc3f65aSKai Ye }
1377adc3f65aSKai Ye
13787b44c0eeSKai Ye /* increment counter (128-bit int) */
ctr_iv_inc(__u8 * counter,__u8 bits,__u32 nums)13797b44c0eeSKai Ye static void ctr_iv_inc(__u8 *counter, __u8 bits, __u32 nums)
13807b44c0eeSKai Ye {
13817b44c0eeSKai Ye do {
13827b44c0eeSKai Ye --bits;
13837b44c0eeSKai Ye nums += counter[bits];
13847b44c0eeSKai Ye counter[bits] = nums & BITS_MASK;
13857b44c0eeSKai Ye nums >>= BYTE_BITS;
13867b44c0eeSKai Ye } while (bits && nums);
13877b44c0eeSKai Ye }
13887b44c0eeSKai Ye
sec_update_iv(struct sec_req * req,enum sec_alg_type alg_type)13892f072d75SZaibo Xu static void sec_update_iv(struct sec_req *req, enum sec_alg_type alg_type)
1390416d8220SZaibo Xu {
13912f072d75SZaibo Xu struct aead_request *aead_req = req->aead_req.aead_req;
1392416d8220SZaibo Xu struct skcipher_request *sk_req = req->c_req.sk_req;
1393416d8220SZaibo Xu u32 iv_size = req->ctx->c_ctx.ivsize;
1394416d8220SZaibo Xu struct scatterlist *sgl;
13952f072d75SZaibo Xu unsigned int cryptlen;
1396416d8220SZaibo Xu size_t sz;
13972f072d75SZaibo Xu u8 *iv;
1398416d8220SZaibo Xu
1399416d8220SZaibo Xu if (req->c_req.encrypt)
14002f072d75SZaibo Xu sgl = alg_type == SEC_SKCIPHER ? sk_req->dst : aead_req->dst;
1401416d8220SZaibo Xu else
14022f072d75SZaibo Xu sgl = alg_type == SEC_SKCIPHER ? sk_req->src : aead_req->src;
1403416d8220SZaibo Xu
14042f072d75SZaibo Xu if (alg_type == SEC_SKCIPHER) {
14052f072d75SZaibo Xu iv = sk_req->iv;
14062f072d75SZaibo Xu cryptlen = sk_req->cryptlen;
14072f072d75SZaibo Xu } else {
14082f072d75SZaibo Xu iv = aead_req->iv;
14092f072d75SZaibo Xu cryptlen = aead_req->cryptlen;
14102f072d75SZaibo Xu }
14112f072d75SZaibo Xu
14127b44c0eeSKai Ye if (req->ctx->c_ctx.c_mode == SEC_CMODE_CBC) {
14132f072d75SZaibo Xu sz = sg_pcopy_to_buffer(sgl, sg_nents(sgl), iv, iv_size,
14142f072d75SZaibo Xu cryptlen - iv_size);
1415b9c8d897SZaibo Xu if (unlikely(sz != iv_size))
1416a44dce50SLongfang Liu dev_err(req->ctx->dev, "copy output iv error!\n");
14177b44c0eeSKai Ye } else {
14187b44c0eeSKai Ye sz = cryptlen / iv_size;
14197b44c0eeSKai Ye if (cryptlen % iv_size)
14207b44c0eeSKai Ye sz += 1;
14217b44c0eeSKai Ye ctr_iv_inc(iv, iv_size, sz);
14227b44c0eeSKai Ye }
1423416d8220SZaibo Xu }
1424416d8220SZaibo Xu
sec_back_req_clear(struct sec_ctx * ctx,struct sec_qp_ctx * qp_ctx)14259597efc3SKai Ye static struct sec_req *sec_back_req_clear(struct sec_ctx *ctx,
14269597efc3SKai Ye struct sec_qp_ctx *qp_ctx)
14279597efc3SKai Ye {
14289597efc3SKai Ye struct sec_req *backlog_req = NULL;
14299597efc3SKai Ye
143002884a4fSZhengchao Shao spin_lock_bh(&qp_ctx->req_lock);
14319597efc3SKai Ye if (ctx->fake_req_limit >=
14329597efc3SKai Ye atomic_read(&qp_ctx->qp->qp_status.used) &&
14339597efc3SKai Ye !list_empty(&qp_ctx->backlog)) {
14349597efc3SKai Ye backlog_req = list_first_entry(&qp_ctx->backlog,
14359597efc3SKai Ye typeof(*backlog_req), backlog_head);
14369597efc3SKai Ye list_del(&backlog_req->backlog_head);
14379597efc3SKai Ye }
143802884a4fSZhengchao Shao spin_unlock_bh(&qp_ctx->req_lock);
14399597efc3SKai Ye
14409597efc3SKai Ye return backlog_req;
14419597efc3SKai Ye }
14429597efc3SKai Ye
sec_skcipher_callback(struct sec_ctx * ctx,struct sec_req * req,int err)1443310ea0acSZaibo Xu static void sec_skcipher_callback(struct sec_ctx *ctx, struct sec_req *req,
1444310ea0acSZaibo Xu int err)
1445416d8220SZaibo Xu {
1446416d8220SZaibo Xu struct skcipher_request *sk_req = req->c_req.sk_req;
1447416d8220SZaibo Xu struct sec_qp_ctx *qp_ctx = req->qp_ctx;
14489597efc3SKai Ye struct skcipher_request *backlog_sk_req;
14499597efc3SKai Ye struct sec_req *backlog_req;
1450416d8220SZaibo Xu
1451416d8220SZaibo Xu sec_free_req_id(req);
1452416d8220SZaibo Xu
14537b44c0eeSKai Ye /* IV output at encrypto of CBC/CTR mode */
14547b44c0eeSKai Ye if (!err && (ctx->c_ctx.c_mode == SEC_CMODE_CBC ||
14557b44c0eeSKai Ye ctx->c_ctx.c_mode == SEC_CMODE_CTR) && req->c_req.encrypt)
14562f072d75SZaibo Xu sec_update_iv(req, SEC_SKCIPHER);
1457416d8220SZaibo Xu
14589597efc3SKai Ye while (1) {
14599597efc3SKai Ye backlog_req = sec_back_req_clear(ctx, qp_ctx);
14609597efc3SKai Ye if (!backlog_req)
14619597efc3SKai Ye break;
14629597efc3SKai Ye
14639597efc3SKai Ye backlog_sk_req = backlog_req->c_req.sk_req;
1464e2b537b7SHerbert Xu skcipher_request_complete(backlog_sk_req, -EINPROGRESS);
14659597efc3SKai Ye atomic64_inc(&ctx->sec->debug.dfx.recv_busy_cnt);
14669597efc3SKai Ye }
14679597efc3SKai Ye
1468e2b537b7SHerbert Xu skcipher_request_complete(sk_req, err);
1469416d8220SZaibo Xu }
1470416d8220SZaibo Xu
set_aead_auth_iv(struct sec_ctx * ctx,struct sec_req * req)1471c16a70c1SKai Ye static void set_aead_auth_iv(struct sec_ctx *ctx, struct sec_req *req)
14722f072d75SZaibo Xu {
14732f072d75SZaibo Xu struct aead_request *aead_req = req->aead_req.aead_req;
14742514f559SLongfang Liu struct sec_cipher_req *c_req = &req->c_req;
1475c16a70c1SKai Ye struct sec_aead_req *a_req = &req->aead_req;
1476c16a70c1SKai Ye size_t authsize = ctx->a_ctx.mac_len;
1477c16a70c1SKai Ye u32 data_size = aead_req->cryptlen;
1478c16a70c1SKai Ye u8 flage = 0;
1479c16a70c1SKai Ye u8 cm, cl;
1480c16a70c1SKai Ye
1481c16a70c1SKai Ye /* the specification has been checked in aead_iv_demension_check() */
1482c16a70c1SKai Ye cl = c_req->c_ivin[0] + 1;
1483c16a70c1SKai Ye c_req->c_ivin[ctx->c_ctx.ivsize - cl] = 0x00;
1484c16a70c1SKai Ye memset(&c_req->c_ivin[ctx->c_ctx.ivsize - cl], 0, cl);
1485c16a70c1SKai Ye c_req->c_ivin[ctx->c_ctx.ivsize - IV_LAST_BYTE1] = IV_CTR_INIT;
1486c16a70c1SKai Ye
1487c16a70c1SKai Ye /* the last 3bit is L' */
1488c16a70c1SKai Ye flage |= c_req->c_ivin[0] & IV_CL_MASK;
1489c16a70c1SKai Ye
1490c16a70c1SKai Ye /* the M' is bit3~bit5, the Flags is bit6 */
1491c16a70c1SKai Ye cm = (authsize - IV_CM_CAL_NUM) / IV_CM_CAL_NUM;
1492c16a70c1SKai Ye flage |= cm << IV_CM_OFFSET;
1493c16a70c1SKai Ye if (aead_req->assoclen)
1494c16a70c1SKai Ye flage |= 0x01 << IV_FLAGS_OFFSET;
1495c16a70c1SKai Ye
1496c16a70c1SKai Ye memcpy(a_req->a_ivin, c_req->c_ivin, ctx->c_ctx.ivsize);
1497c16a70c1SKai Ye a_req->a_ivin[0] = flage;
1498c16a70c1SKai Ye
1499c16a70c1SKai Ye /*
1500c16a70c1SKai Ye * the last 32bit is counter's initial number,
1501c16a70c1SKai Ye * but the nonce uses the first 16bit
1502c16a70c1SKai Ye * the tail 16bit fill with the cipher length
1503c16a70c1SKai Ye */
1504c16a70c1SKai Ye if (!c_req->encrypt)
1505c16a70c1SKai Ye data_size = aead_req->cryptlen - authsize;
1506c16a70c1SKai Ye
1507c16a70c1SKai Ye a_req->a_ivin[ctx->c_ctx.ivsize - IV_LAST_BYTE1] =
1508c16a70c1SKai Ye data_size & IV_LAST_BYTE_MASK;
1509c16a70c1SKai Ye data_size >>= IV_BYTE_OFFSET;
1510c16a70c1SKai Ye a_req->a_ivin[ctx->c_ctx.ivsize - IV_LAST_BYTE2] =
1511c16a70c1SKai Ye data_size & IV_LAST_BYTE_MASK;
1512c16a70c1SKai Ye }
1513c16a70c1SKai Ye
sec_aead_set_iv(struct sec_ctx * ctx,struct sec_req * req)1514c16a70c1SKai Ye static void sec_aead_set_iv(struct sec_ctx *ctx, struct sec_req *req)
1515c16a70c1SKai Ye {
1516c16a70c1SKai Ye struct aead_request *aead_req = req->aead_req.aead_req;
1517c16a70c1SKai Ye struct crypto_aead *tfm = crypto_aead_reqtfm(aead_req);
1518c16a70c1SKai Ye size_t authsize = crypto_aead_authsize(tfm);
1519c16a70c1SKai Ye struct sec_cipher_req *c_req = &req->c_req;
1520c16a70c1SKai Ye struct sec_aead_req *a_req = &req->aead_req;
15212f072d75SZaibo Xu
15222514f559SLongfang Liu memcpy(c_req->c_ivin, aead_req->iv, ctx->c_ctx.ivsize);
1523c16a70c1SKai Ye
1524c16a70c1SKai Ye if (ctx->c_ctx.c_mode == SEC_CMODE_CCM) {
1525c16a70c1SKai Ye /*
1526c16a70c1SKai Ye * CCM 16Byte Cipher_IV: {1B_Flage,13B_IV,2B_counter},
1527c16a70c1SKai Ye * the counter must set to 0x01
1528c16a70c1SKai Ye */
1529c16a70c1SKai Ye ctx->a_ctx.mac_len = authsize;
1530c16a70c1SKai Ye /* CCM 16Byte Auth_IV: {1B_AFlage,13B_IV,2B_Ptext_length} */
1531c16a70c1SKai Ye set_aead_auth_iv(ctx, req);
1532c16a70c1SKai Ye }
1533c16a70c1SKai Ye
1534c16a70c1SKai Ye /* GCM 12Byte Cipher_IV == Auth_IV */
1535c16a70c1SKai Ye if (ctx->c_ctx.c_mode == SEC_CMODE_GCM) {
1536c16a70c1SKai Ye ctx->a_ctx.mac_len = authsize;
1537c16a70c1SKai Ye memcpy(a_req->a_ivin, c_req->c_ivin, SEC_AIV_SIZE);
1538c16a70c1SKai Ye }
1539c16a70c1SKai Ye }
1540c16a70c1SKai Ye
sec_auth_bd_fill_xcm(struct sec_auth_ctx * ctx,int dir,struct sec_req * req,struct sec_sqe * sec_sqe)1541c16a70c1SKai Ye static void sec_auth_bd_fill_xcm(struct sec_auth_ctx *ctx, int dir,
1542c16a70c1SKai Ye struct sec_req *req, struct sec_sqe *sec_sqe)
1543c16a70c1SKai Ye {
1544c16a70c1SKai Ye struct sec_aead_req *a_req = &req->aead_req;
1545c16a70c1SKai Ye struct aead_request *aq = a_req->aead_req;
1546c16a70c1SKai Ye
1547c16a70c1SKai Ye /* C_ICV_Len is MAC size, 0x4 ~ 0x10 */
1548c16a70c1SKai Ye sec_sqe->type2.icvw_kmode |= cpu_to_le16((u16)ctx->mac_len);
1549c16a70c1SKai Ye
1550c16a70c1SKai Ye /* mode set to CCM/GCM, don't set {A_Alg, AKey_Len, MAC_Len} */
1551c16a70c1SKai Ye sec_sqe->type2.a_key_addr = sec_sqe->type2.c_key_addr;
1552c16a70c1SKai Ye sec_sqe->type2.a_ivin_addr = cpu_to_le64(a_req->a_ivin_dma);
1553c16a70c1SKai Ye sec_sqe->type_cipher_auth |= SEC_NO_AUTH << SEC_AUTH_OFFSET;
1554c16a70c1SKai Ye
1555c16a70c1SKai Ye if (dir)
1556c16a70c1SKai Ye sec_sqe->sds_sa_type &= SEC_CIPHER_AUTH;
1557c16a70c1SKai Ye else
1558c16a70c1SKai Ye sec_sqe->sds_sa_type |= SEC_AUTH_CIPHER;
1559c16a70c1SKai Ye
1560c16a70c1SKai Ye sec_sqe->type2.alen_ivllen = cpu_to_le32(aq->assoclen);
1561c16a70c1SKai Ye sec_sqe->type2.auth_src_offset = cpu_to_le16(0x0);
1562c16a70c1SKai Ye sec_sqe->type2.cipher_src_offset = cpu_to_le16((u16)aq->assoclen);
1563c16a70c1SKai Ye
1564c16a70c1SKai Ye sec_sqe->type2.mac_addr = cpu_to_le64(a_req->out_mac_dma);
1565c16a70c1SKai Ye }
1566c16a70c1SKai Ye
sec_auth_bd_fill_xcm_v3(struct sec_auth_ctx * ctx,int dir,struct sec_req * req,struct sec_sqe3 * sqe3)1567c16a70c1SKai Ye static void sec_auth_bd_fill_xcm_v3(struct sec_auth_ctx *ctx, int dir,
1568c16a70c1SKai Ye struct sec_req *req, struct sec_sqe3 *sqe3)
1569c16a70c1SKai Ye {
1570c16a70c1SKai Ye struct sec_aead_req *a_req = &req->aead_req;
1571c16a70c1SKai Ye struct aead_request *aq = a_req->aead_req;
1572c16a70c1SKai Ye
1573c16a70c1SKai Ye /* C_ICV_Len is MAC size, 0x4 ~ 0x10 */
1574c16a70c1SKai Ye sqe3->c_icv_key |= cpu_to_le16((u16)ctx->mac_len << SEC_MAC_OFFSET_V3);
1575c16a70c1SKai Ye
1576c16a70c1SKai Ye /* mode set to CCM/GCM, don't set {A_Alg, AKey_Len, MAC_Len} */
1577c16a70c1SKai Ye sqe3->a_key_addr = sqe3->c_key_addr;
1578c16a70c1SKai Ye sqe3->auth_ivin.a_ivin_addr = cpu_to_le64(a_req->a_ivin_dma);
1579c16a70c1SKai Ye sqe3->auth_mac_key |= SEC_NO_AUTH;
1580c16a70c1SKai Ye
1581c16a70c1SKai Ye if (dir)
1582c16a70c1SKai Ye sqe3->huk_iv_seq &= SEC_CIPHER_AUTH_V3;
1583c16a70c1SKai Ye else
1584c16a70c1SKai Ye sqe3->huk_iv_seq |= SEC_AUTH_CIPHER_V3;
1585c16a70c1SKai Ye
1586c16a70c1SKai Ye sqe3->a_len_key = cpu_to_le32(aq->assoclen);
1587c16a70c1SKai Ye sqe3->auth_src_offset = cpu_to_le16(0x0);
1588c16a70c1SKai Ye sqe3->cipher_src_offset = cpu_to_le16((u16)aq->assoclen);
1589c16a70c1SKai Ye sqe3->mac_addr = cpu_to_le64(a_req->out_mac_dma);
15902f072d75SZaibo Xu }
15912f072d75SZaibo Xu
sec_auth_bd_fill_ex(struct sec_auth_ctx * ctx,int dir,struct sec_req * req,struct sec_sqe * sec_sqe)15922f072d75SZaibo Xu static void sec_auth_bd_fill_ex(struct sec_auth_ctx *ctx, int dir,
15932f072d75SZaibo Xu struct sec_req *req, struct sec_sqe *sec_sqe)
15942f072d75SZaibo Xu {
15952f072d75SZaibo Xu struct sec_aead_req *a_req = &req->aead_req;
15962f072d75SZaibo Xu struct sec_cipher_req *c_req = &req->c_req;
15972f072d75SZaibo Xu struct aead_request *aq = a_req->aead_req;
15982f072d75SZaibo Xu
15992f072d75SZaibo Xu sec_sqe->type2.a_key_addr = cpu_to_le64(ctx->a_key_dma);
16002f072d75SZaibo Xu
16012f072d75SZaibo Xu sec_sqe->type2.mac_key_alg =
16022f072d75SZaibo Xu cpu_to_le32(ctx->mac_len / SEC_SQE_LEN_RATE);
16032f072d75SZaibo Xu
16042f072d75SZaibo Xu sec_sqe->type2.mac_key_alg |=
16052f072d75SZaibo Xu cpu_to_le32((u32)((ctx->a_key_len) /
16062f072d75SZaibo Xu SEC_SQE_LEN_RATE) << SEC_AKEY_OFFSET);
16072f072d75SZaibo Xu
16082f072d75SZaibo Xu sec_sqe->type2.mac_key_alg |=
16092f072d75SZaibo Xu cpu_to_le32((u32)(ctx->a_alg) << SEC_AEAD_ALG_OFFSET);
16102f072d75SZaibo Xu
1611668f1ab7SKai Ye if (dir) {
16122f072d75SZaibo Xu sec_sqe->type_cipher_auth |= SEC_AUTH_TYPE1 << SEC_AUTH_OFFSET;
16132f072d75SZaibo Xu sec_sqe->sds_sa_type &= SEC_CIPHER_AUTH;
1614668f1ab7SKai Ye } else {
1615668f1ab7SKai Ye sec_sqe->type_cipher_auth |= SEC_AUTH_TYPE2 << SEC_AUTH_OFFSET;
16162f072d75SZaibo Xu sec_sqe->sds_sa_type |= SEC_AUTH_CIPHER;
1617668f1ab7SKai Ye }
16182f072d75SZaibo Xu sec_sqe->type2.alen_ivllen = cpu_to_le32(c_req->c_len + aq->assoclen);
16192f072d75SZaibo Xu
16202f072d75SZaibo Xu sec_sqe->type2.cipher_src_offset = cpu_to_le16((u16)aq->assoclen);
16212f072d75SZaibo Xu
16222514f559SLongfang Liu sec_sqe->type2.mac_addr = cpu_to_le64(a_req->out_mac_dma);
16232f072d75SZaibo Xu }
16242f072d75SZaibo Xu
sec_aead_bd_fill(struct sec_ctx * ctx,struct sec_req * req)16252f072d75SZaibo Xu static int sec_aead_bd_fill(struct sec_ctx *ctx, struct sec_req *req)
16262f072d75SZaibo Xu {
16272f072d75SZaibo Xu struct sec_auth_ctx *auth_ctx = &ctx->a_ctx;
16282f072d75SZaibo Xu struct sec_sqe *sec_sqe = &req->sec_sqe;
16292f072d75SZaibo Xu int ret;
16302f072d75SZaibo Xu
16312f072d75SZaibo Xu ret = sec_skcipher_bd_fill(ctx, req);
16322f072d75SZaibo Xu if (unlikely(ret)) {
1633a44dce50SLongfang Liu dev_err(ctx->dev, "skcipher bd fill is error!\n");
16342f072d75SZaibo Xu return ret;
16352f072d75SZaibo Xu }
16362f072d75SZaibo Xu
1637c16a70c1SKai Ye if (ctx->c_ctx.c_mode == SEC_CMODE_CCM ||
1638c16a70c1SKai Ye ctx->c_ctx.c_mode == SEC_CMODE_GCM)
1639c16a70c1SKai Ye sec_auth_bd_fill_xcm(auth_ctx, req->c_req.encrypt, req, sec_sqe);
1640c16a70c1SKai Ye else
16412f072d75SZaibo Xu sec_auth_bd_fill_ex(auth_ctx, req->c_req.encrypt, req, sec_sqe);
16422f072d75SZaibo Xu
16432f072d75SZaibo Xu return 0;
16442f072d75SZaibo Xu }
16452f072d75SZaibo Xu
sec_auth_bd_fill_ex_v3(struct sec_auth_ctx * ctx,int dir,struct sec_req * req,struct sec_sqe3 * sqe3)1646adc3f65aSKai Ye static void sec_auth_bd_fill_ex_v3(struct sec_auth_ctx *ctx, int dir,
1647adc3f65aSKai Ye struct sec_req *req, struct sec_sqe3 *sqe3)
1648adc3f65aSKai Ye {
1649adc3f65aSKai Ye struct sec_aead_req *a_req = &req->aead_req;
1650adc3f65aSKai Ye struct sec_cipher_req *c_req = &req->c_req;
1651adc3f65aSKai Ye struct aead_request *aq = a_req->aead_req;
1652adc3f65aSKai Ye
1653adc3f65aSKai Ye sqe3->a_key_addr = cpu_to_le64(ctx->a_key_dma);
1654adc3f65aSKai Ye
1655adc3f65aSKai Ye sqe3->auth_mac_key |=
1656adc3f65aSKai Ye cpu_to_le32((u32)(ctx->mac_len /
1657adc3f65aSKai Ye SEC_SQE_LEN_RATE) << SEC_MAC_OFFSET_V3);
1658adc3f65aSKai Ye
1659adc3f65aSKai Ye sqe3->auth_mac_key |=
1660adc3f65aSKai Ye cpu_to_le32((u32)(ctx->a_key_len /
1661adc3f65aSKai Ye SEC_SQE_LEN_RATE) << SEC_AKEY_OFFSET_V3);
1662adc3f65aSKai Ye
1663adc3f65aSKai Ye sqe3->auth_mac_key |=
1664adc3f65aSKai Ye cpu_to_le32((u32)(ctx->a_alg) << SEC_AUTH_ALG_OFFSET_V3);
1665adc3f65aSKai Ye
1666adc3f65aSKai Ye if (dir) {
1667adc3f65aSKai Ye sqe3->auth_mac_key |= cpu_to_le32((u32)SEC_AUTH_TYPE1);
1668adc3f65aSKai Ye sqe3->huk_iv_seq &= SEC_CIPHER_AUTH_V3;
1669adc3f65aSKai Ye } else {
167060ef3ddeSKai Ye sqe3->auth_mac_key |= cpu_to_le32((u32)SEC_AUTH_TYPE2);
1671adc3f65aSKai Ye sqe3->huk_iv_seq |= SEC_AUTH_CIPHER_V3;
1672adc3f65aSKai Ye }
1673adc3f65aSKai Ye sqe3->a_len_key = cpu_to_le32(c_req->c_len + aq->assoclen);
1674adc3f65aSKai Ye
1675adc3f65aSKai Ye sqe3->cipher_src_offset = cpu_to_le16((u16)aq->assoclen);
1676adc3f65aSKai Ye
1677adc3f65aSKai Ye sqe3->mac_addr = cpu_to_le64(a_req->out_mac_dma);
1678adc3f65aSKai Ye }
1679adc3f65aSKai Ye
sec_aead_bd_fill_v3(struct sec_ctx * ctx,struct sec_req * req)1680adc3f65aSKai Ye static int sec_aead_bd_fill_v3(struct sec_ctx *ctx, struct sec_req *req)
1681adc3f65aSKai Ye {
1682adc3f65aSKai Ye struct sec_auth_ctx *auth_ctx = &ctx->a_ctx;
1683adc3f65aSKai Ye struct sec_sqe3 *sec_sqe3 = &req->sec_sqe3;
1684adc3f65aSKai Ye int ret;
1685adc3f65aSKai Ye
1686adc3f65aSKai Ye ret = sec_skcipher_bd_fill_v3(ctx, req);
1687adc3f65aSKai Ye if (unlikely(ret)) {
1688adc3f65aSKai Ye dev_err(ctx->dev, "skcipher bd3 fill is error!\n");
1689adc3f65aSKai Ye return ret;
1690adc3f65aSKai Ye }
1691adc3f65aSKai Ye
1692c16a70c1SKai Ye if (ctx->c_ctx.c_mode == SEC_CMODE_CCM ||
1693c16a70c1SKai Ye ctx->c_ctx.c_mode == SEC_CMODE_GCM)
1694c16a70c1SKai Ye sec_auth_bd_fill_xcm_v3(auth_ctx, req->c_req.encrypt,
1695c16a70c1SKai Ye req, sec_sqe3);
1696c16a70c1SKai Ye else
1697c16a70c1SKai Ye sec_auth_bd_fill_ex_v3(auth_ctx, req->c_req.encrypt,
1698c16a70c1SKai Ye req, sec_sqe3);
1699adc3f65aSKai Ye
1700adc3f65aSKai Ye return 0;
1701adc3f65aSKai Ye }
1702adc3f65aSKai Ye
sec_aead_callback(struct sec_ctx * c,struct sec_req * req,int err)17032f072d75SZaibo Xu static void sec_aead_callback(struct sec_ctx *c, struct sec_req *req, int err)
17042f072d75SZaibo Xu {
17052f072d75SZaibo Xu struct aead_request *a_req = req->aead_req.aead_req;
17062f072d75SZaibo Xu struct crypto_aead *tfm = crypto_aead_reqtfm(a_req);
17072514f559SLongfang Liu struct sec_aead_req *aead_req = &req->aead_req;
17082f072d75SZaibo Xu struct sec_cipher_req *c_req = &req->c_req;
17092f072d75SZaibo Xu size_t authsize = crypto_aead_authsize(tfm);
17102f072d75SZaibo Xu struct sec_qp_ctx *qp_ctx = req->qp_ctx;
17119597efc3SKai Ye struct aead_request *backlog_aead_req;
17129597efc3SKai Ye struct sec_req *backlog_req;
17132f072d75SZaibo Xu size_t sz;
17142f072d75SZaibo Xu
17152f072d75SZaibo Xu if (!err && c->c_ctx.c_mode == SEC_CMODE_CBC && c_req->encrypt)
17162f072d75SZaibo Xu sec_update_iv(req, SEC_AEAD);
17172f072d75SZaibo Xu
17182f072d75SZaibo Xu /* Copy output mac */
17192f072d75SZaibo Xu if (!err && c_req->encrypt) {
17202f072d75SZaibo Xu struct scatterlist *sgl = a_req->dst;
17212f072d75SZaibo Xu
17222f072d75SZaibo Xu sz = sg_pcopy_from_buffer(sgl, sg_nents(sgl),
17232514f559SLongfang Liu aead_req->out_mac,
17242f072d75SZaibo Xu authsize, a_req->cryptlen +
17252f072d75SZaibo Xu a_req->assoclen);
17262f072d75SZaibo Xu if (unlikely(sz != authsize)) {
1727a44dce50SLongfang Liu dev_err(c->dev, "copy out mac err!\n");
17282f072d75SZaibo Xu err = -EINVAL;
17292f072d75SZaibo Xu }
17302f072d75SZaibo Xu }
17312f072d75SZaibo Xu
17322f072d75SZaibo Xu sec_free_req_id(req);
17332f072d75SZaibo Xu
17349597efc3SKai Ye while (1) {
17359597efc3SKai Ye backlog_req = sec_back_req_clear(c, qp_ctx);
17369597efc3SKai Ye if (!backlog_req)
17379597efc3SKai Ye break;
17389597efc3SKai Ye
17399597efc3SKai Ye backlog_aead_req = backlog_req->aead_req.aead_req;
1740e2b537b7SHerbert Xu aead_request_complete(backlog_aead_req, -EINPROGRESS);
17419597efc3SKai Ye atomic64_inc(&c->sec->debug.dfx.recv_busy_cnt);
17429597efc3SKai Ye }
17432f072d75SZaibo Xu
1744e2b537b7SHerbert Xu aead_request_complete(a_req, err);
17452f072d75SZaibo Xu }
17462f072d75SZaibo Xu
sec_request_uninit(struct sec_ctx * ctx,struct sec_req * req)1747416d8220SZaibo Xu static void sec_request_uninit(struct sec_ctx *ctx, struct sec_req *req)
1748416d8220SZaibo Xu {
1749416d8220SZaibo Xu sec_free_req_id(req);
1750a181647cSZaibo Xu sec_free_queue_id(ctx, req);
1751416d8220SZaibo Xu }
1752416d8220SZaibo Xu
sec_request_init(struct sec_ctx * ctx,struct sec_req * req)1753416d8220SZaibo Xu static int sec_request_init(struct sec_ctx *ctx, struct sec_req *req)
1754416d8220SZaibo Xu {
1755416d8220SZaibo Xu struct sec_qp_ctx *qp_ctx;
17567c7d902aSZaibo Xu int queue_id;
1757416d8220SZaibo Xu
1758416d8220SZaibo Xu /* To load balance */
1759a181647cSZaibo Xu queue_id = sec_alloc_queue_id(ctx, req);
1760a181647cSZaibo Xu qp_ctx = &ctx->qp_ctx[queue_id];
1761416d8220SZaibo Xu
1762416d8220SZaibo Xu req->req_id = sec_alloc_req_id(req, qp_ctx);
1763b9c8d897SZaibo Xu if (unlikely(req->req_id < 0)) {
1764a181647cSZaibo Xu sec_free_queue_id(ctx, req);
1765416d8220SZaibo Xu return req->req_id;
1766416d8220SZaibo Xu }
1767416d8220SZaibo Xu
17687c7d902aSZaibo Xu return 0;
1769416d8220SZaibo Xu }
1770416d8220SZaibo Xu
sec_process(struct sec_ctx * ctx,struct sec_req * req)1771416d8220SZaibo Xu static int sec_process(struct sec_ctx *ctx, struct sec_req *req)
1772416d8220SZaibo Xu {
17732514f559SLongfang Liu struct sec_cipher_req *c_req = &req->c_req;
1774416d8220SZaibo Xu int ret;
1775416d8220SZaibo Xu
1776416d8220SZaibo Xu ret = sec_request_init(ctx, req);
1777b9c8d897SZaibo Xu if (unlikely(ret))
1778416d8220SZaibo Xu return ret;
1779416d8220SZaibo Xu
1780416d8220SZaibo Xu ret = sec_request_transfer(ctx, req);
1781b9c8d897SZaibo Xu if (unlikely(ret))
1782416d8220SZaibo Xu goto err_uninit_req;
1783416d8220SZaibo Xu
1784416d8220SZaibo Xu /* Output IV as decrypto */
17857b44c0eeSKai Ye if (!req->c_req.encrypt && (ctx->c_ctx.c_mode == SEC_CMODE_CBC ||
17867b44c0eeSKai Ye ctx->c_ctx.c_mode == SEC_CMODE_CTR))
17872f072d75SZaibo Xu sec_update_iv(req, ctx->alg_type);
1788416d8220SZaibo Xu
1789416d8220SZaibo Xu ret = ctx->req_op->bd_send(ctx, req);
17909597efc3SKai Ye if (unlikely((ret != -EBUSY && ret != -EINPROGRESS) ||
17919597efc3SKai Ye (ret == -EBUSY && !(req->flag & CRYPTO_TFM_REQ_MAY_BACKLOG)))) {
1792a44dce50SLongfang Liu dev_err_ratelimited(ctx->dev, "send sec request failed!\n");
1793416d8220SZaibo Xu goto err_send_req;
1794416d8220SZaibo Xu }
1795416d8220SZaibo Xu
1796416d8220SZaibo Xu return ret;
1797416d8220SZaibo Xu
1798416d8220SZaibo Xu err_send_req:
1799416d8220SZaibo Xu /* As failing, restore the IV from user */
18002f072d75SZaibo Xu if (ctx->c_ctx.c_mode == SEC_CMODE_CBC && !req->c_req.encrypt) {
18012f072d75SZaibo Xu if (ctx->alg_type == SEC_SKCIPHER)
18022514f559SLongfang Liu memcpy(req->c_req.sk_req->iv, c_req->c_ivin,
1803416d8220SZaibo Xu ctx->c_ctx.ivsize);
18042f072d75SZaibo Xu else
18052514f559SLongfang Liu memcpy(req->aead_req.aead_req->iv, c_req->c_ivin,
18062f072d75SZaibo Xu ctx->c_ctx.ivsize);
18072f072d75SZaibo Xu }
1808416d8220SZaibo Xu
1809416d8220SZaibo Xu sec_request_untransfer(ctx, req);
1810416d8220SZaibo Xu err_uninit_req:
1811416d8220SZaibo Xu sec_request_uninit(ctx, req);
1812416d8220SZaibo Xu return ret;
1813416d8220SZaibo Xu }
1814416d8220SZaibo Xu
1815a181647cSZaibo Xu static const struct sec_req_op sec_skcipher_req_ops = {
1816416d8220SZaibo Xu .buf_map = sec_skcipher_sgl_map,
1817416d8220SZaibo Xu .buf_unmap = sec_skcipher_sgl_unmap,
1818416d8220SZaibo Xu .do_transfer = sec_skcipher_copy_iv,
1819416d8220SZaibo Xu .bd_fill = sec_skcipher_bd_fill,
1820416d8220SZaibo Xu .bd_send = sec_bd_send,
1821416d8220SZaibo Xu .callback = sec_skcipher_callback,
1822416d8220SZaibo Xu .process = sec_process,
1823416d8220SZaibo Xu };
1824416d8220SZaibo Xu
18252f072d75SZaibo Xu static const struct sec_req_op sec_aead_req_ops = {
18262f072d75SZaibo Xu .buf_map = sec_aead_sgl_map,
18272f072d75SZaibo Xu .buf_unmap = sec_aead_sgl_unmap,
1828c16a70c1SKai Ye .do_transfer = sec_aead_set_iv,
18292f072d75SZaibo Xu .bd_fill = sec_aead_bd_fill,
18302f072d75SZaibo Xu .bd_send = sec_bd_send,
18312f072d75SZaibo Xu .callback = sec_aead_callback,
18322f072d75SZaibo Xu .process = sec_process,
18332f072d75SZaibo Xu };
18342f072d75SZaibo Xu
1835adc3f65aSKai Ye static const struct sec_req_op sec_skcipher_req_ops_v3 = {
1836adc3f65aSKai Ye .buf_map = sec_skcipher_sgl_map,
1837adc3f65aSKai Ye .buf_unmap = sec_skcipher_sgl_unmap,
1838adc3f65aSKai Ye .do_transfer = sec_skcipher_copy_iv,
1839adc3f65aSKai Ye .bd_fill = sec_skcipher_bd_fill_v3,
1840adc3f65aSKai Ye .bd_send = sec_bd_send,
1841adc3f65aSKai Ye .callback = sec_skcipher_callback,
1842adc3f65aSKai Ye .process = sec_process,
1843adc3f65aSKai Ye };
1844adc3f65aSKai Ye
1845adc3f65aSKai Ye static const struct sec_req_op sec_aead_req_ops_v3 = {
1846adc3f65aSKai Ye .buf_map = sec_aead_sgl_map,
1847adc3f65aSKai Ye .buf_unmap = sec_aead_sgl_unmap,
1848c16a70c1SKai Ye .do_transfer = sec_aead_set_iv,
1849adc3f65aSKai Ye .bd_fill = sec_aead_bd_fill_v3,
1850adc3f65aSKai Ye .bd_send = sec_bd_send,
1851adc3f65aSKai Ye .callback = sec_aead_callback,
1852adc3f65aSKai Ye .process = sec_process,
1853adc3f65aSKai Ye };
1854adc3f65aSKai Ye
sec_skcipher_ctx_init(struct crypto_skcipher * tfm)1855416d8220SZaibo Xu static int sec_skcipher_ctx_init(struct crypto_skcipher *tfm)
1856416d8220SZaibo Xu {
1857416d8220SZaibo Xu struct sec_ctx *ctx = crypto_skcipher_ctx(tfm);
1858adc3f65aSKai Ye int ret;
1859416d8220SZaibo Xu
1860adc3f65aSKai Ye ret = sec_skcipher_init(tfm);
1861adc3f65aSKai Ye if (ret)
1862adc3f65aSKai Ye return ret;
1863adc3f65aSKai Ye
1864adc3f65aSKai Ye if (ctx->sec->qm.ver < QM_HW_V3) {
1865adc3f65aSKai Ye ctx->type_supported = SEC_BD_TYPE2;
1866a181647cSZaibo Xu ctx->req_op = &sec_skcipher_req_ops;
1867adc3f65aSKai Ye } else {
1868adc3f65aSKai Ye ctx->type_supported = SEC_BD_TYPE3;
1869adc3f65aSKai Ye ctx->req_op = &sec_skcipher_req_ops_v3;
1870adc3f65aSKai Ye }
1871416d8220SZaibo Xu
1872adc3f65aSKai Ye return ret;
1873416d8220SZaibo Xu }
1874416d8220SZaibo Xu
sec_skcipher_ctx_exit(struct crypto_skcipher * tfm)1875416d8220SZaibo Xu static void sec_skcipher_ctx_exit(struct crypto_skcipher *tfm)
1876416d8220SZaibo Xu {
1877a181647cSZaibo Xu sec_skcipher_uninit(tfm);
1878416d8220SZaibo Xu }
1879416d8220SZaibo Xu
sec_aead_init(struct crypto_aead * tfm)18802f072d75SZaibo Xu static int sec_aead_init(struct crypto_aead *tfm)
18812f072d75SZaibo Xu {
18822f072d75SZaibo Xu struct sec_ctx *ctx = crypto_aead_ctx(tfm);
18832f072d75SZaibo Xu int ret;
18842f072d75SZaibo Xu
18852f072d75SZaibo Xu crypto_aead_set_reqsize(tfm, sizeof(struct sec_req));
18862f072d75SZaibo Xu ctx->alg_type = SEC_AEAD;
18872f072d75SZaibo Xu ctx->c_ctx.ivsize = crypto_aead_ivsize(tfm);
1888c16a70c1SKai Ye if (ctx->c_ctx.ivsize < SEC_AIV_SIZE ||
1889c16a70c1SKai Ye ctx->c_ctx.ivsize > SEC_IV_SIZE) {
1890c16a70c1SKai Ye pr_err("get error aead iv size!\n");
18912f072d75SZaibo Xu return -EINVAL;
18922f072d75SZaibo Xu }
18932f072d75SZaibo Xu
18942f072d75SZaibo Xu ret = sec_ctx_base_init(ctx);
18952f072d75SZaibo Xu if (ret)
18962f072d75SZaibo Xu return ret;
1897adc3f65aSKai Ye if (ctx->sec->qm.ver < QM_HW_V3) {
1898adc3f65aSKai Ye ctx->type_supported = SEC_BD_TYPE2;
1899adc3f65aSKai Ye ctx->req_op = &sec_aead_req_ops;
1900adc3f65aSKai Ye } else {
1901adc3f65aSKai Ye ctx->type_supported = SEC_BD_TYPE3;
1902adc3f65aSKai Ye ctx->req_op = &sec_aead_req_ops_v3;
1903adc3f65aSKai Ye }
19042f072d75SZaibo Xu
19052f072d75SZaibo Xu ret = sec_auth_init(ctx);
19062f072d75SZaibo Xu if (ret)
19072f072d75SZaibo Xu goto err_auth_init;
19082f072d75SZaibo Xu
19092f072d75SZaibo Xu ret = sec_cipher_init(ctx);
19102f072d75SZaibo Xu if (ret)
19112f072d75SZaibo Xu goto err_cipher_init;
19122f072d75SZaibo Xu
19132f072d75SZaibo Xu return ret;
19142f072d75SZaibo Xu
19152f072d75SZaibo Xu err_cipher_init:
19162f072d75SZaibo Xu sec_auth_uninit(ctx);
19172f072d75SZaibo Xu err_auth_init:
19182f072d75SZaibo Xu sec_ctx_base_uninit(ctx);
19192f072d75SZaibo Xu return ret;
19202f072d75SZaibo Xu }
19212f072d75SZaibo Xu
sec_aead_exit(struct crypto_aead * tfm)19222f072d75SZaibo Xu static void sec_aead_exit(struct crypto_aead *tfm)
19232f072d75SZaibo Xu {
19242f072d75SZaibo Xu struct sec_ctx *ctx = crypto_aead_ctx(tfm);
19252f072d75SZaibo Xu
19262f072d75SZaibo Xu sec_cipher_uninit(ctx);
19272f072d75SZaibo Xu sec_auth_uninit(ctx);
19282f072d75SZaibo Xu sec_ctx_base_uninit(ctx);
19292f072d75SZaibo Xu }
19302f072d75SZaibo Xu
sec_aead_ctx_init(struct crypto_aead * tfm,const char * hash_name)19312f072d75SZaibo Xu static int sec_aead_ctx_init(struct crypto_aead *tfm, const char *hash_name)
19322f072d75SZaibo Xu {
19332f072d75SZaibo Xu struct sec_ctx *ctx = crypto_aead_ctx(tfm);
19342f072d75SZaibo Xu struct sec_auth_ctx *auth_ctx = &ctx->a_ctx;
19352f072d75SZaibo Xu int ret;
19362f072d75SZaibo Xu
19372f072d75SZaibo Xu ret = sec_aead_init(tfm);
19382f072d75SZaibo Xu if (ret) {
19392f072d75SZaibo Xu pr_err("hisi_sec2: aead init error!\n");
19402f072d75SZaibo Xu return ret;
19412f072d75SZaibo Xu }
19422f072d75SZaibo Xu
19432f072d75SZaibo Xu auth_ctx->hash_tfm = crypto_alloc_shash(hash_name, 0, 0);
19442f072d75SZaibo Xu if (IS_ERR(auth_ctx->hash_tfm)) {
1945a44dce50SLongfang Liu dev_err(ctx->dev, "aead alloc shash error!\n");
19462f072d75SZaibo Xu sec_aead_exit(tfm);
19472f072d75SZaibo Xu return PTR_ERR(auth_ctx->hash_tfm);
19482f072d75SZaibo Xu }
19492f072d75SZaibo Xu
19502f072d75SZaibo Xu return 0;
19512f072d75SZaibo Xu }
19522f072d75SZaibo Xu
sec_aead_ctx_exit(struct crypto_aead * tfm)19532f072d75SZaibo Xu static void sec_aead_ctx_exit(struct crypto_aead *tfm)
19542f072d75SZaibo Xu {
19552f072d75SZaibo Xu struct sec_ctx *ctx = crypto_aead_ctx(tfm);
19562f072d75SZaibo Xu
19572f072d75SZaibo Xu crypto_free_shash(ctx->a_ctx.hash_tfm);
19582f072d75SZaibo Xu sec_aead_exit(tfm);
19592f072d75SZaibo Xu }
19602f072d75SZaibo Xu
sec_aead_xcm_ctx_init(struct crypto_aead * tfm)1961c16a70c1SKai Ye static int sec_aead_xcm_ctx_init(struct crypto_aead *tfm)
1962c16a70c1SKai Ye {
19636c46a329SKai Ye struct aead_alg *alg = crypto_aead_alg(tfm);
1964c16a70c1SKai Ye struct sec_ctx *ctx = crypto_aead_ctx(tfm);
19656c46a329SKai Ye struct sec_auth_ctx *a_ctx = &ctx->a_ctx;
19666c46a329SKai Ye const char *aead_name = alg->base.cra_name;
1967c16a70c1SKai Ye int ret;
1968c16a70c1SKai Ye
1969c16a70c1SKai Ye ret = sec_aead_init(tfm);
1970c16a70c1SKai Ye if (ret) {
1971c16a70c1SKai Ye dev_err(ctx->dev, "hisi_sec2: aead xcm init error!\n");
1972c16a70c1SKai Ye return ret;
1973c16a70c1SKai Ye }
1974c16a70c1SKai Ye
19756c46a329SKai Ye a_ctx->fallback_aead_tfm = crypto_alloc_aead(aead_name, 0,
19766c46a329SKai Ye CRYPTO_ALG_NEED_FALLBACK |
19776c46a329SKai Ye CRYPTO_ALG_ASYNC);
19786c46a329SKai Ye if (IS_ERR(a_ctx->fallback_aead_tfm)) {
19796c46a329SKai Ye dev_err(ctx->dev, "aead driver alloc fallback tfm error!\n");
19806c46a329SKai Ye sec_aead_exit(tfm);
19816c46a329SKai Ye return PTR_ERR(a_ctx->fallback_aead_tfm);
19826c46a329SKai Ye }
19836c46a329SKai Ye a_ctx->fallback = false;
19846c46a329SKai Ye
1985c16a70c1SKai Ye return 0;
1986c16a70c1SKai Ye }
1987c16a70c1SKai Ye
sec_aead_xcm_ctx_exit(struct crypto_aead * tfm)1988c16a70c1SKai Ye static void sec_aead_xcm_ctx_exit(struct crypto_aead *tfm)
1989c16a70c1SKai Ye {
19906c46a329SKai Ye struct sec_ctx *ctx = crypto_aead_ctx(tfm);
19916c46a329SKai Ye
19926c46a329SKai Ye crypto_free_aead(ctx->a_ctx.fallback_aead_tfm);
1993c16a70c1SKai Ye sec_aead_exit(tfm);
1994c16a70c1SKai Ye }
1995c16a70c1SKai Ye
sec_aead_sha1_ctx_init(struct crypto_aead * tfm)19962f072d75SZaibo Xu static int sec_aead_sha1_ctx_init(struct crypto_aead *tfm)
19972f072d75SZaibo Xu {
19982f072d75SZaibo Xu return sec_aead_ctx_init(tfm, "sha1");
19992f072d75SZaibo Xu }
20002f072d75SZaibo Xu
sec_aead_sha256_ctx_init(struct crypto_aead * tfm)20012f072d75SZaibo Xu static int sec_aead_sha256_ctx_init(struct crypto_aead *tfm)
20022f072d75SZaibo Xu {
20032f072d75SZaibo Xu return sec_aead_ctx_init(tfm, "sha256");
20042f072d75SZaibo Xu }
20052f072d75SZaibo Xu
sec_aead_sha512_ctx_init(struct crypto_aead * tfm)20062f072d75SZaibo Xu static int sec_aead_sha512_ctx_init(struct crypto_aead *tfm)
20072f072d75SZaibo Xu {
20082f072d75SZaibo Xu return sec_aead_ctx_init(tfm, "sha512");
20092f072d75SZaibo Xu }
20102f072d75SZaibo Xu
sec_skcipher_cryptlen_check(struct sec_ctx * ctx,struct sec_req * sreq)20112132d4efSKai Ye static int sec_skcipher_cryptlen_check(struct sec_ctx *ctx,
2012059c5342SLongfang Liu struct sec_req *sreq)
2013059c5342SLongfang Liu {
2014059c5342SLongfang Liu u32 cryptlen = sreq->c_req.sk_req->cryptlen;
2015059c5342SLongfang Liu struct device *dev = ctx->dev;
2016059c5342SLongfang Liu u8 c_mode = ctx->c_ctx.c_mode;
2017059c5342SLongfang Liu int ret = 0;
2018059c5342SLongfang Liu
2019059c5342SLongfang Liu switch (c_mode) {
2020059c5342SLongfang Liu case SEC_CMODE_XTS:
2021059c5342SLongfang Liu if (unlikely(cryptlen < AES_BLOCK_SIZE)) {
2022059c5342SLongfang Liu dev_err(dev, "skcipher XTS mode input length error!\n");
2023059c5342SLongfang Liu ret = -EINVAL;
2024059c5342SLongfang Liu }
2025059c5342SLongfang Liu break;
2026059c5342SLongfang Liu case SEC_CMODE_ECB:
2027059c5342SLongfang Liu case SEC_CMODE_CBC:
2028059c5342SLongfang Liu if (unlikely(cryptlen & (AES_BLOCK_SIZE - 1))) {
2029059c5342SLongfang Liu dev_err(dev, "skcipher AES input length error!\n");
2030059c5342SLongfang Liu ret = -EINVAL;
2031059c5342SLongfang Liu }
2032059c5342SLongfang Liu break;
20337b44c0eeSKai Ye case SEC_CMODE_CFB:
20347b44c0eeSKai Ye case SEC_CMODE_OFB:
20357b44c0eeSKai Ye case SEC_CMODE_CTR:
20367b44c0eeSKai Ye if (unlikely(ctx->sec->qm.ver < QM_HW_V3)) {
20377b44c0eeSKai Ye dev_err(dev, "skcipher HW version error!\n");
20387b44c0eeSKai Ye ret = -EINVAL;
20397b44c0eeSKai Ye }
20407b44c0eeSKai Ye break;
2041059c5342SLongfang Liu default:
2042059c5342SLongfang Liu ret = -EINVAL;
2043059c5342SLongfang Liu }
2044059c5342SLongfang Liu
2045059c5342SLongfang Liu return ret;
2046059c5342SLongfang Liu }
2047059c5342SLongfang Liu
sec_skcipher_param_check(struct sec_ctx * ctx,struct sec_req * sreq)2048d6de2a59SZaibo Xu static int sec_skcipher_param_check(struct sec_ctx *ctx, struct sec_req *sreq)
2049416d8220SZaibo Xu {
2050d6de2a59SZaibo Xu struct skcipher_request *sk_req = sreq->c_req.sk_req;
2051a44dce50SLongfang Liu struct device *dev = ctx->dev;
2052d6de2a59SZaibo Xu u8 c_alg = ctx->c_ctx.c_alg;
2053416d8220SZaibo Xu
20547b44c0eeSKai Ye if (unlikely(!sk_req->src || !sk_req->dst ||
20557b44c0eeSKai Ye sk_req->cryptlen > MAX_INPUT_DATA_LEN)) {
2056416d8220SZaibo Xu dev_err(dev, "skcipher input param error!\n");
2057416d8220SZaibo Xu return -EINVAL;
2058416d8220SZaibo Xu }
2059d6de2a59SZaibo Xu sreq->c_req.c_len = sk_req->cryptlen;
206074b58db8SLongfang Liu
206174b58db8SLongfang Liu if (ctx->pbuf_supported && sk_req->cryptlen <= SEC_PBUF_SZ)
206274b58db8SLongfang Liu sreq->use_pbuf = true;
206374b58db8SLongfang Liu else
206474b58db8SLongfang Liu sreq->use_pbuf = false;
206574b58db8SLongfang Liu
2066416d8220SZaibo Xu if (c_alg == SEC_CALG_3DES) {
2067b9c8d897SZaibo Xu if (unlikely(sk_req->cryptlen & (DES3_EDE_BLOCK_SIZE - 1))) {
2068416d8220SZaibo Xu dev_err(dev, "skcipher 3des input length error!\n");
2069416d8220SZaibo Xu return -EINVAL;
2070416d8220SZaibo Xu }
2071416d8220SZaibo Xu return 0;
2072416d8220SZaibo Xu } else if (c_alg == SEC_CALG_AES || c_alg == SEC_CALG_SM4) {
20732132d4efSKai Ye return sec_skcipher_cryptlen_check(ctx, sreq);
2074416d8220SZaibo Xu }
2075059c5342SLongfang Liu
2076416d8220SZaibo Xu dev_err(dev, "skcipher algorithm error!\n");
2077633e507fSLongfang Liu
2078416d8220SZaibo Xu return -EINVAL;
2079416d8220SZaibo Xu }
2080416d8220SZaibo Xu
sec_skcipher_soft_crypto(struct sec_ctx * ctx,struct skcipher_request * sreq,bool encrypt)20815652d55aSKai Ye static int sec_skcipher_soft_crypto(struct sec_ctx *ctx,
20825652d55aSKai Ye struct skcipher_request *sreq, bool encrypt)
20835652d55aSKai Ye {
20845652d55aSKai Ye struct sec_cipher_ctx *c_ctx = &ctx->c_ctx;
2085e764d81dSKai Ye SYNC_SKCIPHER_REQUEST_ON_STACK(subreq, c_ctx->fbtfm);
20865652d55aSKai Ye struct device *dev = ctx->dev;
20875652d55aSKai Ye int ret;
20885652d55aSKai Ye
20895652d55aSKai Ye if (!c_ctx->fbtfm) {
2090e764d81dSKai Ye dev_err_ratelimited(dev, "the soft tfm isn't supported in the current system.\n");
20915652d55aSKai Ye return -EINVAL;
20925652d55aSKai Ye }
20935652d55aSKai Ye
20945652d55aSKai Ye skcipher_request_set_sync_tfm(subreq, c_ctx->fbtfm);
20955652d55aSKai Ye
20965652d55aSKai Ye /* software need sync mode to do crypto */
20975652d55aSKai Ye skcipher_request_set_callback(subreq, sreq->base.flags,
20985652d55aSKai Ye NULL, NULL);
20995652d55aSKai Ye skcipher_request_set_crypt(subreq, sreq->src, sreq->dst,
21005652d55aSKai Ye sreq->cryptlen, sreq->iv);
21015652d55aSKai Ye if (encrypt)
21025652d55aSKai Ye ret = crypto_skcipher_encrypt(subreq);
21035652d55aSKai Ye else
21045652d55aSKai Ye ret = crypto_skcipher_decrypt(subreq);
21055652d55aSKai Ye
21065652d55aSKai Ye skcipher_request_zero(subreq);
21075652d55aSKai Ye
21085652d55aSKai Ye return ret;
21095652d55aSKai Ye }
21105652d55aSKai Ye
sec_skcipher_crypto(struct skcipher_request * sk_req,bool encrypt)2111416d8220SZaibo Xu static int sec_skcipher_crypto(struct skcipher_request *sk_req, bool encrypt)
2112416d8220SZaibo Xu {
2113416d8220SZaibo Xu struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(sk_req);
2114416d8220SZaibo Xu struct sec_req *req = skcipher_request_ctx(sk_req);
2115416d8220SZaibo Xu struct sec_ctx *ctx = crypto_skcipher_ctx(tfm);
2116416d8220SZaibo Xu int ret;
2117416d8220SZaibo Xu
21185652d55aSKai Ye if (!sk_req->cryptlen) {
21195652d55aSKai Ye if (ctx->c_ctx.c_mode == SEC_CMODE_XTS)
21205652d55aSKai Ye return -EINVAL;
2121416d8220SZaibo Xu return 0;
21225652d55aSKai Ye }
2123416d8220SZaibo Xu
21249597efc3SKai Ye req->flag = sk_req->base.flags;
2125416d8220SZaibo Xu req->c_req.sk_req = sk_req;
2126416d8220SZaibo Xu req->c_req.encrypt = encrypt;
2127416d8220SZaibo Xu req->ctx = ctx;
2128416d8220SZaibo Xu
2129d6de2a59SZaibo Xu ret = sec_skcipher_param_check(ctx, req);
2130d6de2a59SZaibo Xu if (unlikely(ret))
2131d6de2a59SZaibo Xu return -EINVAL;
2132d6de2a59SZaibo Xu
21335652d55aSKai Ye if (unlikely(ctx->c_ctx.fallback))
21345652d55aSKai Ye return sec_skcipher_soft_crypto(ctx, sk_req, encrypt);
21355652d55aSKai Ye
2136416d8220SZaibo Xu return ctx->req_op->process(ctx, req);
2137416d8220SZaibo Xu }
2138416d8220SZaibo Xu
sec_skcipher_encrypt(struct skcipher_request * sk_req)2139416d8220SZaibo Xu static int sec_skcipher_encrypt(struct skcipher_request *sk_req)
2140416d8220SZaibo Xu {
2141416d8220SZaibo Xu return sec_skcipher_crypto(sk_req, true);
2142416d8220SZaibo Xu }
2143416d8220SZaibo Xu
sec_skcipher_decrypt(struct skcipher_request * sk_req)2144416d8220SZaibo Xu static int sec_skcipher_decrypt(struct skcipher_request *sk_req)
2145416d8220SZaibo Xu {
2146416d8220SZaibo Xu return sec_skcipher_crypto(sk_req, false);
2147416d8220SZaibo Xu }
2148416d8220SZaibo Xu
2149416d8220SZaibo Xu #define SEC_SKCIPHER_GEN_ALG(sec_cra_name, sec_set_key, sec_min_key_size, \
2150416d8220SZaibo Xu sec_max_key_size, ctx_init, ctx_exit, blk_size, iv_size)\
2151416d8220SZaibo Xu {\
2152416d8220SZaibo Xu .base = {\
2153416d8220SZaibo Xu .cra_name = sec_cra_name,\
2154416d8220SZaibo Xu .cra_driver_name = "hisi_sec_"sec_cra_name,\
2155416d8220SZaibo Xu .cra_priority = SEC_PRIORITY,\
21565652d55aSKai Ye .cra_flags = CRYPTO_ALG_ASYNC |\
21575652d55aSKai Ye CRYPTO_ALG_NEED_FALLBACK,\
2158416d8220SZaibo Xu .cra_blocksize = blk_size,\
2159416d8220SZaibo Xu .cra_ctxsize = sizeof(struct sec_ctx),\
2160416d8220SZaibo Xu .cra_module = THIS_MODULE,\
2161416d8220SZaibo Xu },\
2162416d8220SZaibo Xu .init = ctx_init,\
2163416d8220SZaibo Xu .exit = ctx_exit,\
2164416d8220SZaibo Xu .setkey = sec_set_key,\
2165416d8220SZaibo Xu .decrypt = sec_skcipher_decrypt,\
2166416d8220SZaibo Xu .encrypt = sec_skcipher_encrypt,\
2167416d8220SZaibo Xu .min_keysize = sec_min_key_size,\
2168416d8220SZaibo Xu .max_keysize = sec_max_key_size,\
2169416d8220SZaibo Xu .ivsize = iv_size,\
2170921715b6SWenkai Lin }
2171416d8220SZaibo Xu
2172416d8220SZaibo Xu #define SEC_SKCIPHER_ALG(name, key_func, min_key_size, \
2173416d8220SZaibo Xu max_key_size, blk_size, iv_size) \
2174416d8220SZaibo Xu SEC_SKCIPHER_GEN_ALG(name, key_func, min_key_size, max_key_size, \
2175416d8220SZaibo Xu sec_skcipher_ctx_init, sec_skcipher_ctx_exit, blk_size, iv_size)
2176416d8220SZaibo Xu
2177921715b6SWenkai Lin static struct sec_skcipher sec_skciphers[] = {
2178921715b6SWenkai Lin {
2179921715b6SWenkai Lin .alg_msk = BIT(0),
2180921715b6SWenkai Lin .alg = SEC_SKCIPHER_ALG("ecb(aes)", sec_setkey_aes_ecb, AES_MIN_KEY_SIZE,
2181921715b6SWenkai Lin AES_MAX_KEY_SIZE, AES_BLOCK_SIZE, 0),
2182921715b6SWenkai Lin },
2183921715b6SWenkai Lin {
2184921715b6SWenkai Lin .alg_msk = BIT(1),
2185921715b6SWenkai Lin .alg = SEC_SKCIPHER_ALG("cbc(aes)", sec_setkey_aes_cbc, AES_MIN_KEY_SIZE,
2186921715b6SWenkai Lin AES_MAX_KEY_SIZE, AES_BLOCK_SIZE, AES_BLOCK_SIZE),
2187921715b6SWenkai Lin },
2188921715b6SWenkai Lin {
2189921715b6SWenkai Lin .alg_msk = BIT(2),
2190921715b6SWenkai Lin .alg = SEC_SKCIPHER_ALG("ctr(aes)", sec_setkey_aes_ctr, AES_MIN_KEY_SIZE,
2191921715b6SWenkai Lin AES_MAX_KEY_SIZE, SEC_MIN_BLOCK_SZ, AES_BLOCK_SIZE),
2192921715b6SWenkai Lin },
2193921715b6SWenkai Lin {
2194921715b6SWenkai Lin .alg_msk = BIT(3),
2195921715b6SWenkai Lin .alg = SEC_SKCIPHER_ALG("xts(aes)", sec_setkey_aes_xts, SEC_XTS_MIN_KEY_SIZE,
2196921715b6SWenkai Lin SEC_XTS_MAX_KEY_SIZE, AES_BLOCK_SIZE, AES_BLOCK_SIZE),
2197921715b6SWenkai Lin },
2198921715b6SWenkai Lin {
2199921715b6SWenkai Lin .alg_msk = BIT(4),
2200921715b6SWenkai Lin .alg = SEC_SKCIPHER_ALG("ofb(aes)", sec_setkey_aes_ofb, AES_MIN_KEY_SIZE,
2201921715b6SWenkai Lin AES_MAX_KEY_SIZE, SEC_MIN_BLOCK_SZ, AES_BLOCK_SIZE),
2202921715b6SWenkai Lin },
2203921715b6SWenkai Lin {
2204921715b6SWenkai Lin .alg_msk = BIT(5),
2205921715b6SWenkai Lin .alg = SEC_SKCIPHER_ALG("cfb(aes)", sec_setkey_aes_cfb, AES_MIN_KEY_SIZE,
2206921715b6SWenkai Lin AES_MAX_KEY_SIZE, SEC_MIN_BLOCK_SZ, AES_BLOCK_SIZE),
2207921715b6SWenkai Lin },
2208921715b6SWenkai Lin {
2209921715b6SWenkai Lin .alg_msk = BIT(12),
2210921715b6SWenkai Lin .alg = SEC_SKCIPHER_ALG("cbc(sm4)", sec_setkey_sm4_cbc, AES_MIN_KEY_SIZE,
2211921715b6SWenkai Lin AES_MIN_KEY_SIZE, AES_BLOCK_SIZE, AES_BLOCK_SIZE),
2212921715b6SWenkai Lin },
2213921715b6SWenkai Lin {
2214921715b6SWenkai Lin .alg_msk = BIT(13),
2215921715b6SWenkai Lin .alg = SEC_SKCIPHER_ALG("ctr(sm4)", sec_setkey_sm4_ctr, AES_MIN_KEY_SIZE,
2216921715b6SWenkai Lin AES_MIN_KEY_SIZE, SEC_MIN_BLOCK_SZ, AES_BLOCK_SIZE),
2217921715b6SWenkai Lin },
2218921715b6SWenkai Lin {
2219921715b6SWenkai Lin .alg_msk = BIT(14),
2220921715b6SWenkai Lin .alg = SEC_SKCIPHER_ALG("xts(sm4)", sec_setkey_sm4_xts, SEC_XTS_MIN_KEY_SIZE,
2221921715b6SWenkai Lin SEC_XTS_MIN_KEY_SIZE, AES_BLOCK_SIZE, AES_BLOCK_SIZE),
2222921715b6SWenkai Lin },
2223921715b6SWenkai Lin {
2224921715b6SWenkai Lin .alg_msk = BIT(15),
2225921715b6SWenkai Lin .alg = SEC_SKCIPHER_ALG("ofb(sm4)", sec_setkey_sm4_ofb, AES_MIN_KEY_SIZE,
2226921715b6SWenkai Lin AES_MIN_KEY_SIZE, SEC_MIN_BLOCK_SZ, AES_BLOCK_SIZE),
2227921715b6SWenkai Lin },
2228921715b6SWenkai Lin {
2229921715b6SWenkai Lin .alg_msk = BIT(16),
2230921715b6SWenkai Lin .alg = SEC_SKCIPHER_ALG("cfb(sm4)", sec_setkey_sm4_cfb, AES_MIN_KEY_SIZE,
2231921715b6SWenkai Lin AES_MIN_KEY_SIZE, SEC_MIN_BLOCK_SZ, AES_BLOCK_SIZE),
2232921715b6SWenkai Lin },
2233921715b6SWenkai Lin {
2234921715b6SWenkai Lin .alg_msk = BIT(23),
2235921715b6SWenkai Lin .alg = SEC_SKCIPHER_ALG("ecb(des3_ede)", sec_setkey_3des_ecb, SEC_DES3_3KEY_SIZE,
2236921715b6SWenkai Lin SEC_DES3_3KEY_SIZE, DES3_EDE_BLOCK_SIZE, 0),
2237921715b6SWenkai Lin },
2238921715b6SWenkai Lin {
2239921715b6SWenkai Lin .alg_msk = BIT(24),
2240921715b6SWenkai Lin .alg = SEC_SKCIPHER_ALG("cbc(des3_ede)", sec_setkey_3des_cbc, SEC_DES3_3KEY_SIZE,
2241921715b6SWenkai Lin SEC_DES3_3KEY_SIZE, DES3_EDE_BLOCK_SIZE,
2242921715b6SWenkai Lin DES3_EDE_BLOCK_SIZE),
2243921715b6SWenkai Lin },
22447b44c0eeSKai Ye };
22457b44c0eeSKai Ye
aead_iv_demension_check(struct aead_request * aead_req)2246c16a70c1SKai Ye static int aead_iv_demension_check(struct aead_request *aead_req)
2247c16a70c1SKai Ye {
2248c16a70c1SKai Ye u8 cl;
2249c16a70c1SKai Ye
2250c16a70c1SKai Ye cl = aead_req->iv[0] + 1;
2251c16a70c1SKai Ye if (cl < IV_CL_MIN || cl > IV_CL_MAX)
2252c16a70c1SKai Ye return -EINVAL;
2253c16a70c1SKai Ye
2254c16a70c1SKai Ye if (cl < IV_CL_MID && aead_req->cryptlen >> (BYTE_BITS * cl))
2255c16a70c1SKai Ye return -EOVERFLOW;
2256c16a70c1SKai Ye
2257c16a70c1SKai Ye return 0;
2258c16a70c1SKai Ye }
2259c16a70c1SKai Ye
sec_aead_spec_check(struct sec_ctx * ctx,struct sec_req * sreq)2260c16a70c1SKai Ye static int sec_aead_spec_check(struct sec_ctx *ctx, struct sec_req *sreq)
2261c16a70c1SKai Ye {
2262c16a70c1SKai Ye struct aead_request *req = sreq->aead_req.aead_req;
2263c16a70c1SKai Ye struct crypto_aead *tfm = crypto_aead_reqtfm(req);
2264c16a70c1SKai Ye size_t authsize = crypto_aead_authsize(tfm);
2265c16a70c1SKai Ye u8 c_mode = ctx->c_ctx.c_mode;
2266c16a70c1SKai Ye struct device *dev = ctx->dev;
2267c16a70c1SKai Ye int ret;
2268c16a70c1SKai Ye
2269c16a70c1SKai Ye if (unlikely(req->cryptlen + req->assoclen > MAX_INPUT_DATA_LEN ||
2270c16a70c1SKai Ye req->assoclen > SEC_MAX_AAD_LEN)) {
2271c16a70c1SKai Ye dev_err(dev, "aead input spec error!\n");
2272c16a70c1SKai Ye return -EINVAL;
2273c16a70c1SKai Ye }
2274c16a70c1SKai Ye
2275c16a70c1SKai Ye if (unlikely((c_mode == SEC_CMODE_GCM && authsize < DES_BLOCK_SIZE) ||
2276c16a70c1SKai Ye (c_mode == SEC_CMODE_CCM && (authsize < MIN_MAC_LEN ||
2277c16a70c1SKai Ye authsize & MAC_LEN_MASK)))) {
2278c16a70c1SKai Ye dev_err(dev, "aead input mac length error!\n");
2279c16a70c1SKai Ye return -EINVAL;
2280c16a70c1SKai Ye }
2281c16a70c1SKai Ye
2282c16a70c1SKai Ye if (c_mode == SEC_CMODE_CCM) {
22835e340558SKai Ye if (unlikely(req->assoclen > SEC_MAX_CCM_AAD_LEN)) {
22845e340558SKai Ye dev_err_ratelimited(dev, "CCM input aad parameter is too long!\n");
22855e340558SKai Ye return -EINVAL;
22865e340558SKai Ye }
2287c16a70c1SKai Ye ret = aead_iv_demension_check(req);
2288c16a70c1SKai Ye if (ret) {
2289c16a70c1SKai Ye dev_err(dev, "aead input iv param error!\n");
2290c16a70c1SKai Ye return ret;
2291c16a70c1SKai Ye }
2292c16a70c1SKai Ye }
2293c16a70c1SKai Ye
2294c16a70c1SKai Ye if (sreq->c_req.encrypt)
2295c16a70c1SKai Ye sreq->c_req.c_len = req->cryptlen;
2296c16a70c1SKai Ye else
2297c16a70c1SKai Ye sreq->c_req.c_len = req->cryptlen - authsize;
2298c16a70c1SKai Ye if (c_mode == SEC_CMODE_CBC) {
2299c16a70c1SKai Ye if (unlikely(sreq->c_req.c_len & (AES_BLOCK_SIZE - 1))) {
2300c16a70c1SKai Ye dev_err(dev, "aead crypto length error!\n");
2301c16a70c1SKai Ye return -EINVAL;
2302c16a70c1SKai Ye }
2303c16a70c1SKai Ye }
2304c16a70c1SKai Ye
2305c16a70c1SKai Ye return 0;
2306c16a70c1SKai Ye }
2307c16a70c1SKai Ye
sec_aead_param_check(struct sec_ctx * ctx,struct sec_req * sreq)23082f072d75SZaibo Xu static int sec_aead_param_check(struct sec_ctx *ctx, struct sec_req *sreq)
23092f072d75SZaibo Xu {
23102f072d75SZaibo Xu struct aead_request *req = sreq->aead_req.aead_req;
23112f072d75SZaibo Xu struct crypto_aead *tfm = crypto_aead_reqtfm(req);
23122f072d75SZaibo Xu size_t authsize = crypto_aead_authsize(tfm);
2313a44dce50SLongfang Liu struct device *dev = ctx->dev;
2314a44dce50SLongfang Liu u8 c_alg = ctx->c_ctx.c_alg;
23152f072d75SZaibo Xu
2316c16a70c1SKai Ye if (unlikely(!req->src || !req->dst)) {
2317a44dce50SLongfang Liu dev_err(dev, "aead input param error!\n");
23182f072d75SZaibo Xu return -EINVAL;
23192f072d75SZaibo Xu }
23202f072d75SZaibo Xu
2321c16a70c1SKai Ye if (ctx->sec->qm.ver == QM_HW_V2) {
2322c16a70c1SKai Ye if (unlikely(!req->cryptlen || (!sreq->c_req.encrypt &&
2323c16a70c1SKai Ye req->cryptlen <= authsize))) {
23246c46a329SKai Ye ctx->a_ctx.fallback = true;
2325c16a70c1SKai Ye return -EINVAL;
2326c16a70c1SKai Ye }
2327c16a70c1SKai Ye }
2328c16a70c1SKai Ye
2329c16a70c1SKai Ye /* Support AES or SM4 */
2330c16a70c1SKai Ye if (unlikely(c_alg != SEC_CALG_AES && c_alg != SEC_CALG_SM4)) {
2331c16a70c1SKai Ye dev_err(dev, "aead crypto alg error!\n");
2332c16a70c1SKai Ye return -EINVAL;
2333c16a70c1SKai Ye }
2334c16a70c1SKai Ye
2335c16a70c1SKai Ye if (unlikely(sec_aead_spec_check(ctx, sreq)))
2336c16a70c1SKai Ye return -EINVAL;
2337c16a70c1SKai Ye
233874b58db8SLongfang Liu if (ctx->pbuf_supported && (req->cryptlen + req->assoclen) <=
233974b58db8SLongfang Liu SEC_PBUF_SZ)
234074b58db8SLongfang Liu sreq->use_pbuf = true;
234174b58db8SLongfang Liu else
234274b58db8SLongfang Liu sreq->use_pbuf = false;
234374b58db8SLongfang Liu
23442f072d75SZaibo Xu return 0;
23452f072d75SZaibo Xu }
23462f072d75SZaibo Xu
sec_aead_soft_crypto(struct sec_ctx * ctx,struct aead_request * aead_req,bool encrypt)23476c46a329SKai Ye static int sec_aead_soft_crypto(struct sec_ctx *ctx,
23486c46a329SKai Ye struct aead_request *aead_req,
23496c46a329SKai Ye bool encrypt)
23506c46a329SKai Ye {
23516c46a329SKai Ye struct sec_auth_ctx *a_ctx = &ctx->a_ctx;
23526c46a329SKai Ye struct device *dev = ctx->dev;
23530a2a464fSKai Ye struct aead_request *subreq;
23540a2a464fSKai Ye int ret;
23556c46a329SKai Ye
23566c46a329SKai Ye /* Kunpeng920 aead mode not support input 0 size */
23576c46a329SKai Ye if (!a_ctx->fallback_aead_tfm) {
235884c2c729SColin Ian King dev_err(dev, "aead fallback tfm is NULL!\n");
23596c46a329SKai Ye return -EINVAL;
23606c46a329SKai Ye }
23616c46a329SKai Ye
23620a2a464fSKai Ye subreq = aead_request_alloc(a_ctx->fallback_aead_tfm, GFP_KERNEL);
23630a2a464fSKai Ye if (!subreq)
23640a2a464fSKai Ye return -ENOMEM;
23650a2a464fSKai Ye
23666c46a329SKai Ye aead_request_set_tfm(subreq, a_ctx->fallback_aead_tfm);
23676c46a329SKai Ye aead_request_set_callback(subreq, aead_req->base.flags,
23686c46a329SKai Ye aead_req->base.complete, aead_req->base.data);
23696c46a329SKai Ye aead_request_set_crypt(subreq, aead_req->src, aead_req->dst,
23706c46a329SKai Ye aead_req->cryptlen, aead_req->iv);
23716c46a329SKai Ye aead_request_set_ad(subreq, aead_req->assoclen);
23726c46a329SKai Ye
23730a2a464fSKai Ye if (encrypt)
23740a2a464fSKai Ye ret = crypto_aead_encrypt(subreq);
23750a2a464fSKai Ye else
23760a2a464fSKai Ye ret = crypto_aead_decrypt(subreq);
23770a2a464fSKai Ye aead_request_free(subreq);
23780a2a464fSKai Ye
23790a2a464fSKai Ye return ret;
23806c46a329SKai Ye }
23816c46a329SKai Ye
sec_aead_crypto(struct aead_request * a_req,bool encrypt)23822f072d75SZaibo Xu static int sec_aead_crypto(struct aead_request *a_req, bool encrypt)
23832f072d75SZaibo Xu {
23842f072d75SZaibo Xu struct crypto_aead *tfm = crypto_aead_reqtfm(a_req);
23852f072d75SZaibo Xu struct sec_req *req = aead_request_ctx(a_req);
23862f072d75SZaibo Xu struct sec_ctx *ctx = crypto_aead_ctx(tfm);
23872f072d75SZaibo Xu int ret;
23882f072d75SZaibo Xu
23899597efc3SKai Ye req->flag = a_req->base.flags;
23902f072d75SZaibo Xu req->aead_req.aead_req = a_req;
23912f072d75SZaibo Xu req->c_req.encrypt = encrypt;
23922f072d75SZaibo Xu req->ctx = ctx;
23932f072d75SZaibo Xu
23942f072d75SZaibo Xu ret = sec_aead_param_check(ctx, req);
23956c46a329SKai Ye if (unlikely(ret)) {
23966c46a329SKai Ye if (ctx->a_ctx.fallback)
23976c46a329SKai Ye return sec_aead_soft_crypto(ctx, a_req, encrypt);
23982f072d75SZaibo Xu return -EINVAL;
23996c46a329SKai Ye }
24002f072d75SZaibo Xu
24012f072d75SZaibo Xu return ctx->req_op->process(ctx, req);
24022f072d75SZaibo Xu }
24032f072d75SZaibo Xu
sec_aead_encrypt(struct aead_request * a_req)24042f072d75SZaibo Xu static int sec_aead_encrypt(struct aead_request *a_req)
24052f072d75SZaibo Xu {
24062f072d75SZaibo Xu return sec_aead_crypto(a_req, true);
24072f072d75SZaibo Xu }
24082f072d75SZaibo Xu
sec_aead_decrypt(struct aead_request * a_req)24092f072d75SZaibo Xu static int sec_aead_decrypt(struct aead_request *a_req)
24102f072d75SZaibo Xu {
24112f072d75SZaibo Xu return sec_aead_crypto(a_req, false);
24122f072d75SZaibo Xu }
24132f072d75SZaibo Xu
2414c16a70c1SKai Ye #define SEC_AEAD_ALG(sec_cra_name, sec_set_key, ctx_init,\
24152f072d75SZaibo Xu ctx_exit, blk_size, iv_size, max_authsize)\
24162f072d75SZaibo Xu {\
24172f072d75SZaibo Xu .base = {\
24182f072d75SZaibo Xu .cra_name = sec_cra_name,\
24192f072d75SZaibo Xu .cra_driver_name = "hisi_sec_"sec_cra_name,\
24202f072d75SZaibo Xu .cra_priority = SEC_PRIORITY,\
24216c46a329SKai Ye .cra_flags = CRYPTO_ALG_ASYNC |\
24226c46a329SKai Ye CRYPTO_ALG_NEED_FALLBACK,\
24232f072d75SZaibo Xu .cra_blocksize = blk_size,\
24242f072d75SZaibo Xu .cra_ctxsize = sizeof(struct sec_ctx),\
24252f072d75SZaibo Xu .cra_module = THIS_MODULE,\
24262f072d75SZaibo Xu },\
24272f072d75SZaibo Xu .init = ctx_init,\
24282f072d75SZaibo Xu .exit = ctx_exit,\
24292f072d75SZaibo Xu .setkey = sec_set_key,\
24306c46a329SKai Ye .setauthsize = sec_aead_setauthsize,\
24312f072d75SZaibo Xu .decrypt = sec_aead_decrypt,\
24322f072d75SZaibo Xu .encrypt = sec_aead_encrypt,\
24332f072d75SZaibo Xu .ivsize = iv_size,\
24342f072d75SZaibo Xu .maxauthsize = max_authsize,\
24352f072d75SZaibo Xu }
24362f072d75SZaibo Xu
2437921715b6SWenkai Lin static struct sec_aead sec_aeads[] = {
2438921715b6SWenkai Lin {
2439921715b6SWenkai Lin .alg_msk = BIT(6),
2440921715b6SWenkai Lin .alg = SEC_AEAD_ALG("ccm(aes)", sec_setkey_aes_ccm, sec_aead_xcm_ctx_init,
2441921715b6SWenkai Lin sec_aead_xcm_ctx_exit, SEC_MIN_BLOCK_SZ, AES_BLOCK_SIZE,
2442921715b6SWenkai Lin AES_BLOCK_SIZE),
2443921715b6SWenkai Lin },
2444921715b6SWenkai Lin {
2445921715b6SWenkai Lin .alg_msk = BIT(7),
2446921715b6SWenkai Lin .alg = SEC_AEAD_ALG("gcm(aes)", sec_setkey_aes_gcm, sec_aead_xcm_ctx_init,
2447921715b6SWenkai Lin sec_aead_xcm_ctx_exit, SEC_MIN_BLOCK_SZ, SEC_AIV_SIZE,
2448921715b6SWenkai Lin AES_BLOCK_SIZE),
2449921715b6SWenkai Lin },
2450921715b6SWenkai Lin {
2451921715b6SWenkai Lin .alg_msk = BIT(17),
2452921715b6SWenkai Lin .alg = SEC_AEAD_ALG("ccm(sm4)", sec_setkey_sm4_ccm, sec_aead_xcm_ctx_init,
2453921715b6SWenkai Lin sec_aead_xcm_ctx_exit, SEC_MIN_BLOCK_SZ, AES_BLOCK_SIZE,
2454921715b6SWenkai Lin AES_BLOCK_SIZE),
2455921715b6SWenkai Lin },
2456921715b6SWenkai Lin {
2457921715b6SWenkai Lin .alg_msk = BIT(18),
2458921715b6SWenkai Lin .alg = SEC_AEAD_ALG("gcm(sm4)", sec_setkey_sm4_gcm, sec_aead_xcm_ctx_init,
2459921715b6SWenkai Lin sec_aead_xcm_ctx_exit, SEC_MIN_BLOCK_SZ, SEC_AIV_SIZE,
2460921715b6SWenkai Lin AES_BLOCK_SIZE),
2461921715b6SWenkai Lin },
2462921715b6SWenkai Lin {
2463921715b6SWenkai Lin .alg_msk = BIT(43),
2464921715b6SWenkai Lin .alg = SEC_AEAD_ALG("authenc(hmac(sha1),cbc(aes))", sec_setkey_aes_cbc_sha1,
2465921715b6SWenkai Lin sec_aead_sha1_ctx_init, sec_aead_ctx_exit, AES_BLOCK_SIZE,
2466c16a70c1SKai Ye AES_BLOCK_SIZE, SHA1_DIGEST_SIZE),
2467921715b6SWenkai Lin },
2468921715b6SWenkai Lin {
2469921715b6SWenkai Lin .alg_msk = BIT(44),
2470921715b6SWenkai Lin .alg = SEC_AEAD_ALG("authenc(hmac(sha256),cbc(aes))", sec_setkey_aes_cbc_sha256,
2471921715b6SWenkai Lin sec_aead_sha256_ctx_init, sec_aead_ctx_exit, AES_BLOCK_SIZE,
2472c16a70c1SKai Ye AES_BLOCK_SIZE, SHA256_DIGEST_SIZE),
2473921715b6SWenkai Lin },
2474921715b6SWenkai Lin {
2475921715b6SWenkai Lin .alg_msk = BIT(45),
2476921715b6SWenkai Lin .alg = SEC_AEAD_ALG("authenc(hmac(sha512),cbc(aes))", sec_setkey_aes_cbc_sha512,
2477921715b6SWenkai Lin sec_aead_sha512_ctx_init, sec_aead_ctx_exit, AES_BLOCK_SIZE,
2478c16a70c1SKai Ye AES_BLOCK_SIZE, SHA512_DIGEST_SIZE),
2479921715b6SWenkai Lin },
2480c16a70c1SKai Ye };
2481c16a70c1SKai Ye
sec_unregister_skcipher(u64 alg_mask,int end)2482921715b6SWenkai Lin static void sec_unregister_skcipher(u64 alg_mask, int end)
2483921715b6SWenkai Lin {
2484921715b6SWenkai Lin int i;
2485c16a70c1SKai Ye
2486921715b6SWenkai Lin for (i = 0; i < end; i++)
2487921715b6SWenkai Lin if (sec_skciphers[i].alg_msk & alg_mask)
2488921715b6SWenkai Lin crypto_unregister_skcipher(&sec_skciphers[i].alg);
2489921715b6SWenkai Lin }
2490921715b6SWenkai Lin
sec_register_skcipher(u64 alg_mask)2491921715b6SWenkai Lin static int sec_register_skcipher(u64 alg_mask)
2492921715b6SWenkai Lin {
2493921715b6SWenkai Lin int i, ret, count;
2494921715b6SWenkai Lin
2495921715b6SWenkai Lin count = ARRAY_SIZE(sec_skciphers);
2496921715b6SWenkai Lin
2497921715b6SWenkai Lin for (i = 0; i < count; i++) {
2498921715b6SWenkai Lin if (!(sec_skciphers[i].alg_msk & alg_mask))
2499921715b6SWenkai Lin continue;
2500921715b6SWenkai Lin
2501921715b6SWenkai Lin ret = crypto_register_skcipher(&sec_skciphers[i].alg);
2502921715b6SWenkai Lin if (ret)
2503921715b6SWenkai Lin goto err;
2504921715b6SWenkai Lin }
2505921715b6SWenkai Lin
2506921715b6SWenkai Lin return 0;
2507921715b6SWenkai Lin
2508921715b6SWenkai Lin err:
2509921715b6SWenkai Lin sec_unregister_skcipher(alg_mask, i);
2510921715b6SWenkai Lin
2511921715b6SWenkai Lin return ret;
2512921715b6SWenkai Lin }
2513921715b6SWenkai Lin
sec_unregister_aead(u64 alg_mask,int end)2514921715b6SWenkai Lin static void sec_unregister_aead(u64 alg_mask, int end)
2515921715b6SWenkai Lin {
2516921715b6SWenkai Lin int i;
2517921715b6SWenkai Lin
2518921715b6SWenkai Lin for (i = 0; i < end; i++)
2519921715b6SWenkai Lin if (sec_aeads[i].alg_msk & alg_mask)
2520921715b6SWenkai Lin crypto_unregister_aead(&sec_aeads[i].alg);
2521921715b6SWenkai Lin }
2522921715b6SWenkai Lin
sec_register_aead(u64 alg_mask)2523921715b6SWenkai Lin static int sec_register_aead(u64 alg_mask)
2524921715b6SWenkai Lin {
2525921715b6SWenkai Lin int i, ret, count;
2526921715b6SWenkai Lin
2527921715b6SWenkai Lin count = ARRAY_SIZE(sec_aeads);
2528921715b6SWenkai Lin
2529921715b6SWenkai Lin for (i = 0; i < count; i++) {
2530921715b6SWenkai Lin if (!(sec_aeads[i].alg_msk & alg_mask))
2531921715b6SWenkai Lin continue;
2532921715b6SWenkai Lin
2533921715b6SWenkai Lin ret = crypto_register_aead(&sec_aeads[i].alg);
2534921715b6SWenkai Lin if (ret)
2535921715b6SWenkai Lin goto err;
2536921715b6SWenkai Lin }
2537921715b6SWenkai Lin
2538921715b6SWenkai Lin return 0;
2539921715b6SWenkai Lin
2540921715b6SWenkai Lin err:
2541921715b6SWenkai Lin sec_unregister_aead(alg_mask, i);
2542921715b6SWenkai Lin
2543921715b6SWenkai Lin return ret;
2544921715b6SWenkai Lin }
25452f072d75SZaibo Xu
sec_register_to_crypto(struct hisi_qm * qm)25468123455aSMeng Yu int sec_register_to_crypto(struct hisi_qm *qm)
2547416d8220SZaibo Xu {
2548b06a6d5eSZhiqi Song u64 alg_mask;
2549b06a6d5eSZhiqi Song int ret = 0;
2550b06a6d5eSZhiqi Song
2551b06a6d5eSZhiqi Song alg_mask = sec_get_alg_bitmap(qm, SEC_DRV_ALG_BITMAP_HIGH_IDX,
2552b06a6d5eSZhiqi Song SEC_DRV_ALG_BITMAP_LOW_IDX);
2553b06a6d5eSZhiqi Song
2554416d8220SZaibo Xu
2555921715b6SWenkai Lin ret = sec_register_skcipher(alg_mask);
25562f072d75SZaibo Xu if (ret)
25572f072d75SZaibo Xu return ret;
25582f072d75SZaibo Xu
2559921715b6SWenkai Lin ret = sec_register_aead(alg_mask);
25607b44c0eeSKai Ye if (ret)
2561921715b6SWenkai Lin sec_unregister_skcipher(alg_mask, ARRAY_SIZE(sec_skciphers));
2562c16a70c1SKai Ye
2563416d8220SZaibo Xu return ret;
2564416d8220SZaibo Xu }
2565416d8220SZaibo Xu
sec_unregister_from_crypto(struct hisi_qm * qm)25668123455aSMeng Yu void sec_unregister_from_crypto(struct hisi_qm *qm)
2567416d8220SZaibo Xu {
2568b06a6d5eSZhiqi Song u64 alg_mask;
2569b06a6d5eSZhiqi Song
2570b06a6d5eSZhiqi Song alg_mask = sec_get_alg_bitmap(qm, SEC_DRV_ALG_BITMAP_HIGH_IDX,
2571b06a6d5eSZhiqi Song SEC_DRV_ALG_BITMAP_LOW_IDX);
2572c16a70c1SKai Ye
2573921715b6SWenkai Lin sec_unregister_aead(alg_mask, ARRAY_SIZE(sec_aeads));
2574921715b6SWenkai Lin sec_unregister_skcipher(alg_mask, ARRAY_SIZE(sec_skciphers));
25752f072d75SZaibo Xu }
2576