1 // SPDX-License-Identifier: GPL-2.0
2 /* Copyright (c) 2019 HiSilicon Limited. */
3 
4 #include <crypto/aes.h>
5 #include <crypto/aead.h>
6 #include <crypto/algapi.h>
7 #include <crypto/authenc.h>
8 #include <crypto/des.h>
9 #include <crypto/hash.h>
10 #include <crypto/internal/aead.h>
11 #include <crypto/internal/des.h>
12 #include <crypto/sha1.h>
13 #include <crypto/sha2.h>
14 #include <crypto/skcipher.h>
15 #include <crypto/xts.h>
16 #include <linux/crypto.h>
17 #include <linux/dma-mapping.h>
18 #include <linux/idr.h>
19 
20 #include "sec.h"
21 #include "sec_crypto.h"
22 
23 #define SEC_PRIORITY		4001
24 #define SEC_XTS_MIN_KEY_SIZE	(2 * AES_MIN_KEY_SIZE)
25 #define SEC_XTS_MID_KEY_SIZE	(3 * AES_MIN_KEY_SIZE)
26 #define SEC_XTS_MAX_KEY_SIZE	(2 * AES_MAX_KEY_SIZE)
27 #define SEC_DES3_2KEY_SIZE	(2 * DES_KEY_SIZE)
28 #define SEC_DES3_3KEY_SIZE	(3 * DES_KEY_SIZE)
29 
30 /* SEC sqe(bd) bit operational relative MACRO */
31 #define SEC_DE_OFFSET		1
32 #define SEC_CIPHER_OFFSET	4
33 #define SEC_SCENE_OFFSET	3
34 #define SEC_DST_SGL_OFFSET	2
35 #define SEC_SRC_SGL_OFFSET	7
36 #define SEC_CKEY_OFFSET		9
37 #define SEC_CMODE_OFFSET	12
38 #define SEC_AKEY_OFFSET         5
39 #define SEC_AEAD_ALG_OFFSET     11
40 #define SEC_AUTH_OFFSET		6
41 
42 #define SEC_DE_OFFSET_V3		9
43 #define SEC_SCENE_OFFSET_V3	5
44 #define SEC_CKEY_OFFSET_V3	13
45 #define SEC_CTR_CNT_OFFSET	25
46 #define SEC_CTR_CNT_ROLLOVER	2
47 #define SEC_SRC_SGL_OFFSET_V3	11
48 #define SEC_DST_SGL_OFFSET_V3	14
49 #define SEC_CALG_OFFSET_V3	4
50 #define SEC_AKEY_OFFSET_V3	9
51 #define SEC_MAC_OFFSET_V3	4
52 #define SEC_AUTH_ALG_OFFSET_V3	15
53 #define SEC_CIPHER_AUTH_V3	0xbf
54 #define SEC_AUTH_CIPHER_V3	0x40
55 #define SEC_FLAG_OFFSET		7
56 #define SEC_FLAG_MASK		0x0780
57 #define SEC_TYPE_MASK		0x0F
58 #define SEC_DONE_MASK		0x0001
59 #define SEC_ICV_MASK		0x000E
60 #define SEC_SQE_LEN_RATE_MASK	0x3
61 
62 #define SEC_TOTAL_IV_SZ(depth)	(SEC_IV_SIZE * (depth))
63 #define SEC_SGL_SGE_NR		128
64 #define SEC_CIPHER_AUTH		0xfe
65 #define SEC_AUTH_CIPHER		0x1
66 #define SEC_MAX_MAC_LEN		64
67 #define SEC_MAX_AAD_LEN		65535
68 #define SEC_MAX_CCM_AAD_LEN	65279
69 #define SEC_TOTAL_MAC_SZ(depth) (SEC_MAX_MAC_LEN * (depth))
70 
71 #define SEC_PBUF_SZ			512
72 #define SEC_PBUF_IV_OFFSET		SEC_PBUF_SZ
73 #define SEC_PBUF_MAC_OFFSET		(SEC_PBUF_SZ + SEC_IV_SIZE)
74 #define SEC_PBUF_PKG		(SEC_PBUF_SZ + SEC_IV_SIZE +	\
75 			SEC_MAX_MAC_LEN * 2)
76 #define SEC_PBUF_NUM		(PAGE_SIZE / SEC_PBUF_PKG)
77 #define SEC_PBUF_PAGE_NUM(depth)	((depth) / SEC_PBUF_NUM)
78 #define SEC_PBUF_LEFT_SZ(depth)		(SEC_PBUF_PKG * ((depth) -	\
79 				SEC_PBUF_PAGE_NUM(depth) * SEC_PBUF_NUM))
80 #define SEC_TOTAL_PBUF_SZ(depth)	(PAGE_SIZE * SEC_PBUF_PAGE_NUM(depth) +	\
81 				SEC_PBUF_LEFT_SZ(depth))
82 
83 #define SEC_SQE_LEN_RATE	4
84 #define SEC_SQE_CFLAG		2
85 #define SEC_SQE_AEAD_FLAG	3
86 #define SEC_SQE_DONE		0x1
87 #define SEC_ICV_ERR		0x2
88 #define MIN_MAC_LEN		4
89 #define MAC_LEN_MASK		0x1U
90 #define MAX_INPUT_DATA_LEN	0xFFFE00
91 #define BITS_MASK		0xFF
92 #define BYTE_BITS		0x8
93 #define SEC_XTS_NAME_SZ		0x3
94 #define IV_CM_CAL_NUM		2
95 #define IV_CL_MASK		0x7
96 #define IV_CL_MIN		2
97 #define IV_CL_MID		4
98 #define IV_CL_MAX		8
99 #define IV_FLAGS_OFFSET	0x6
100 #define IV_CM_OFFSET		0x3
101 #define IV_LAST_BYTE1		1
102 #define IV_LAST_BYTE2		2
103 #define IV_LAST_BYTE_MASK	0xFF
104 #define IV_CTR_INIT		0x1
105 #define IV_BYTE_OFFSET		0x8
106 
107 struct sec_skcipher {
108 	u64 alg_msk;
109 	struct skcipher_alg alg;
110 };
111 
112 struct sec_aead {
113 	u64 alg_msk;
114 	struct aead_alg alg;
115 };
116 
117 /* Get an en/de-cipher queue cyclically to balance load over queues of TFM */
118 static inline int sec_alloc_queue_id(struct sec_ctx *ctx, struct sec_req *req)
119 {
120 	if (req->c_req.encrypt)
121 		return (u32)atomic_inc_return(&ctx->enc_qcyclic) %
122 				 ctx->hlf_q_num;
123 
124 	return (u32)atomic_inc_return(&ctx->dec_qcyclic) % ctx->hlf_q_num +
125 				 ctx->hlf_q_num;
126 }
127 
128 static inline void sec_free_queue_id(struct sec_ctx *ctx, struct sec_req *req)
129 {
130 	if (req->c_req.encrypt)
131 		atomic_dec(&ctx->enc_qcyclic);
132 	else
133 		atomic_dec(&ctx->dec_qcyclic);
134 }
135 
136 static int sec_alloc_req_id(struct sec_req *req, struct sec_qp_ctx *qp_ctx)
137 {
138 	int req_id;
139 
140 	spin_lock_bh(&qp_ctx->req_lock);
141 	req_id = idr_alloc_cyclic(&qp_ctx->req_idr, NULL, 0, qp_ctx->qp->sq_depth, GFP_ATOMIC);
142 	spin_unlock_bh(&qp_ctx->req_lock);
143 	if (unlikely(req_id < 0)) {
144 		dev_err(req->ctx->dev, "alloc req id fail!\n");
145 		return req_id;
146 	}
147 
148 	req->qp_ctx = qp_ctx;
149 	qp_ctx->req_list[req_id] = req;
150 
151 	return req_id;
152 }
153 
154 static void sec_free_req_id(struct sec_req *req)
155 {
156 	struct sec_qp_ctx *qp_ctx = req->qp_ctx;
157 	int req_id = req->req_id;
158 
159 	if (unlikely(req_id < 0 || req_id >= qp_ctx->qp->sq_depth)) {
160 		dev_err(req->ctx->dev, "free request id invalid!\n");
161 		return;
162 	}
163 
164 	qp_ctx->req_list[req_id] = NULL;
165 	req->qp_ctx = NULL;
166 
167 	spin_lock_bh(&qp_ctx->req_lock);
168 	idr_remove(&qp_ctx->req_idr, req_id);
169 	spin_unlock_bh(&qp_ctx->req_lock);
170 }
171 
172 static u8 pre_parse_finished_bd(struct bd_status *status, void *resp)
173 {
174 	struct sec_sqe *bd = resp;
175 
176 	status->done = le16_to_cpu(bd->type2.done_flag) & SEC_DONE_MASK;
177 	status->icv = (le16_to_cpu(bd->type2.done_flag) & SEC_ICV_MASK) >> 1;
178 	status->flag = (le16_to_cpu(bd->type2.done_flag) &
179 					SEC_FLAG_MASK) >> SEC_FLAG_OFFSET;
180 	status->tag = le16_to_cpu(bd->type2.tag);
181 	status->err_type = bd->type2.error_type;
182 
183 	return bd->type_cipher_auth & SEC_TYPE_MASK;
184 }
185 
186 static u8 pre_parse_finished_bd3(struct bd_status *status, void *resp)
187 {
188 	struct sec_sqe3 *bd3 = resp;
189 
190 	status->done = le16_to_cpu(bd3->done_flag) & SEC_DONE_MASK;
191 	status->icv = (le16_to_cpu(bd3->done_flag) & SEC_ICV_MASK) >> 1;
192 	status->flag = (le16_to_cpu(bd3->done_flag) &
193 					SEC_FLAG_MASK) >> SEC_FLAG_OFFSET;
194 	status->tag = le64_to_cpu(bd3->tag);
195 	status->err_type = bd3->error_type;
196 
197 	return le32_to_cpu(bd3->bd_param) & SEC_TYPE_MASK;
198 }
199 
200 static int sec_cb_status_check(struct sec_req *req,
201 			       struct bd_status *status)
202 {
203 	struct sec_ctx *ctx = req->ctx;
204 
205 	if (unlikely(req->err_type || status->done != SEC_SQE_DONE)) {
206 		dev_err_ratelimited(ctx->dev, "err_type[%d], done[%u]\n",
207 				    req->err_type, status->done);
208 		return -EIO;
209 	}
210 
211 	if (unlikely(ctx->alg_type == SEC_SKCIPHER)) {
212 		if (unlikely(status->flag != SEC_SQE_CFLAG)) {
213 			dev_err_ratelimited(ctx->dev, "flag[%u]\n",
214 					    status->flag);
215 			return -EIO;
216 		}
217 	} else if (unlikely(ctx->alg_type == SEC_AEAD)) {
218 		if (unlikely(status->flag != SEC_SQE_AEAD_FLAG ||
219 			     status->icv == SEC_ICV_ERR)) {
220 			dev_err_ratelimited(ctx->dev,
221 					    "flag[%u], icv[%u]\n",
222 					    status->flag, status->icv);
223 			return -EBADMSG;
224 		}
225 	}
226 
227 	return 0;
228 }
229 
230 static void sec_req_cb(struct hisi_qp *qp, void *resp)
231 {
232 	struct sec_qp_ctx *qp_ctx = qp->qp_ctx;
233 	struct sec_dfx *dfx = &qp_ctx->ctx->sec->debug.dfx;
234 	u8 type_supported = qp_ctx->ctx->type_supported;
235 	struct bd_status status;
236 	struct sec_ctx *ctx;
237 	struct sec_req *req;
238 	int err;
239 	u8 type;
240 
241 	if (type_supported == SEC_BD_TYPE2) {
242 		type = pre_parse_finished_bd(&status, resp);
243 		req = qp_ctx->req_list[status.tag];
244 	} else {
245 		type = pre_parse_finished_bd3(&status, resp);
246 		req = (void *)(uintptr_t)status.tag;
247 	}
248 
249 	if (unlikely(type != type_supported)) {
250 		atomic64_inc(&dfx->err_bd_cnt);
251 		pr_err("err bd type [%u]\n", type);
252 		return;
253 	}
254 
255 	if (unlikely(!req)) {
256 		atomic64_inc(&dfx->invalid_req_cnt);
257 		atomic_inc(&qp->qp_status.used);
258 		return;
259 	}
260 
261 	req->err_type = status.err_type;
262 	ctx = req->ctx;
263 	err = sec_cb_status_check(req, &status);
264 	if (err)
265 		atomic64_inc(&dfx->done_flag_cnt);
266 
267 	atomic64_inc(&dfx->recv_cnt);
268 
269 	ctx->req_op->buf_unmap(ctx, req);
270 
271 	ctx->req_op->callback(ctx, req, err);
272 }
273 
274 static int sec_bd_send(struct sec_ctx *ctx, struct sec_req *req)
275 {
276 	struct sec_qp_ctx *qp_ctx = req->qp_ctx;
277 	int ret;
278 
279 	if (ctx->fake_req_limit <=
280 	    atomic_read(&qp_ctx->qp->qp_status.used) &&
281 	    !(req->flag & CRYPTO_TFM_REQ_MAY_BACKLOG))
282 		return -EBUSY;
283 
284 	spin_lock_bh(&qp_ctx->req_lock);
285 	ret = hisi_qp_send(qp_ctx->qp, &req->sec_sqe);
286 	if (ctx->fake_req_limit <=
287 	    atomic_read(&qp_ctx->qp->qp_status.used) && !ret) {
288 		list_add_tail(&req->backlog_head, &qp_ctx->backlog);
289 		atomic64_inc(&ctx->sec->debug.dfx.send_cnt);
290 		atomic64_inc(&ctx->sec->debug.dfx.send_busy_cnt);
291 		spin_unlock_bh(&qp_ctx->req_lock);
292 		return -EBUSY;
293 	}
294 	spin_unlock_bh(&qp_ctx->req_lock);
295 
296 	if (unlikely(ret == -EBUSY))
297 		return -ENOBUFS;
298 
299 	if (likely(!ret)) {
300 		ret = -EINPROGRESS;
301 		atomic64_inc(&ctx->sec->debug.dfx.send_cnt);
302 	}
303 
304 	return ret;
305 }
306 
307 /* Get DMA memory resources */
308 static int sec_alloc_civ_resource(struct device *dev, struct sec_alg_res *res)
309 {
310 	u16 q_depth = res->depth;
311 	int i;
312 
313 	res->c_ivin = dma_alloc_coherent(dev, SEC_TOTAL_IV_SZ(q_depth),
314 					 &res->c_ivin_dma, GFP_KERNEL);
315 	if (!res->c_ivin)
316 		return -ENOMEM;
317 
318 	for (i = 1; i < q_depth; i++) {
319 		res[i].c_ivin_dma = res->c_ivin_dma + i * SEC_IV_SIZE;
320 		res[i].c_ivin = res->c_ivin + i * SEC_IV_SIZE;
321 	}
322 
323 	return 0;
324 }
325 
326 static void sec_free_civ_resource(struct device *dev, struct sec_alg_res *res)
327 {
328 	if (res->c_ivin)
329 		dma_free_coherent(dev, SEC_TOTAL_IV_SZ(res->depth),
330 				  res->c_ivin, res->c_ivin_dma);
331 }
332 
333 static int sec_alloc_aiv_resource(struct device *dev, struct sec_alg_res *res)
334 {
335 	u16 q_depth = res->depth;
336 	int i;
337 
338 	res->a_ivin = dma_alloc_coherent(dev, SEC_TOTAL_IV_SZ(q_depth),
339 					 &res->a_ivin_dma, GFP_KERNEL);
340 	if (!res->a_ivin)
341 		return -ENOMEM;
342 
343 	for (i = 1; i < q_depth; i++) {
344 		res[i].a_ivin_dma = res->a_ivin_dma + i * SEC_IV_SIZE;
345 		res[i].a_ivin = res->a_ivin + i * SEC_IV_SIZE;
346 	}
347 
348 	return 0;
349 }
350 
351 static void sec_free_aiv_resource(struct device *dev, struct sec_alg_res *res)
352 {
353 	if (res->a_ivin)
354 		dma_free_coherent(dev, SEC_TOTAL_IV_SZ(res->depth),
355 				  res->a_ivin, res->a_ivin_dma);
356 }
357 
358 static int sec_alloc_mac_resource(struct device *dev, struct sec_alg_res *res)
359 {
360 	u16 q_depth = res->depth;
361 	int i;
362 
363 	res->out_mac = dma_alloc_coherent(dev, SEC_TOTAL_MAC_SZ(q_depth) << 1,
364 					  &res->out_mac_dma, GFP_KERNEL);
365 	if (!res->out_mac)
366 		return -ENOMEM;
367 
368 	for (i = 1; i < q_depth; i++) {
369 		res[i].out_mac_dma = res->out_mac_dma +
370 				     i * (SEC_MAX_MAC_LEN << 1);
371 		res[i].out_mac = res->out_mac + i * (SEC_MAX_MAC_LEN << 1);
372 	}
373 
374 	return 0;
375 }
376 
377 static void sec_free_mac_resource(struct device *dev, struct sec_alg_res *res)
378 {
379 	if (res->out_mac)
380 		dma_free_coherent(dev, SEC_TOTAL_MAC_SZ(res->depth) << 1,
381 				  res->out_mac, res->out_mac_dma);
382 }
383 
384 static void sec_free_pbuf_resource(struct device *dev, struct sec_alg_res *res)
385 {
386 	if (res->pbuf)
387 		dma_free_coherent(dev, SEC_TOTAL_PBUF_SZ(res->depth),
388 				  res->pbuf, res->pbuf_dma);
389 }
390 
391 /*
392  * To improve performance, pbuffer is used for
393  * small packets (< 512Bytes) as IOMMU translation using.
394  */
395 static int sec_alloc_pbuf_resource(struct device *dev, struct sec_alg_res *res)
396 {
397 	u16 q_depth = res->depth;
398 	int size = SEC_PBUF_PAGE_NUM(q_depth);
399 	int pbuf_page_offset;
400 	int i, j, k;
401 
402 	res->pbuf = dma_alloc_coherent(dev, SEC_TOTAL_PBUF_SZ(q_depth),
403 				&res->pbuf_dma, GFP_KERNEL);
404 	if (!res->pbuf)
405 		return -ENOMEM;
406 
407 	/*
408 	 * SEC_PBUF_PKG contains data pbuf, iv and
409 	 * out_mac : <SEC_PBUF|SEC_IV|SEC_MAC>
410 	 * Every PAGE contains six SEC_PBUF_PKG
411 	 * The sec_qp_ctx contains QM_Q_DEPTH numbers of SEC_PBUF_PKG
412 	 * So we need SEC_PBUF_PAGE_NUM numbers of PAGE
413 	 * for the SEC_TOTAL_PBUF_SZ
414 	 */
415 	for (i = 0; i <= size; i++) {
416 		pbuf_page_offset = PAGE_SIZE * i;
417 		for (j = 0; j < SEC_PBUF_NUM; j++) {
418 			k = i * SEC_PBUF_NUM + j;
419 			if (k == q_depth)
420 				break;
421 			res[k].pbuf = res->pbuf +
422 				j * SEC_PBUF_PKG + pbuf_page_offset;
423 			res[k].pbuf_dma = res->pbuf_dma +
424 				j * SEC_PBUF_PKG + pbuf_page_offset;
425 		}
426 	}
427 
428 	return 0;
429 }
430 
431 static int sec_alg_resource_alloc(struct sec_ctx *ctx,
432 				  struct sec_qp_ctx *qp_ctx)
433 {
434 	struct sec_alg_res *res = qp_ctx->res;
435 	struct device *dev = ctx->dev;
436 	int ret;
437 
438 	ret = sec_alloc_civ_resource(dev, res);
439 	if (ret)
440 		return ret;
441 
442 	if (ctx->alg_type == SEC_AEAD) {
443 		ret = sec_alloc_aiv_resource(dev, res);
444 		if (ret)
445 			goto alloc_aiv_fail;
446 
447 		ret = sec_alloc_mac_resource(dev, res);
448 		if (ret)
449 			goto alloc_mac_fail;
450 	}
451 	if (ctx->pbuf_supported) {
452 		ret = sec_alloc_pbuf_resource(dev, res);
453 		if (ret) {
454 			dev_err(dev, "fail to alloc pbuf dma resource!\n");
455 			goto alloc_pbuf_fail;
456 		}
457 	}
458 
459 	return 0;
460 
461 alloc_pbuf_fail:
462 	if (ctx->alg_type == SEC_AEAD)
463 		sec_free_mac_resource(dev, qp_ctx->res);
464 alloc_mac_fail:
465 	if (ctx->alg_type == SEC_AEAD)
466 		sec_free_aiv_resource(dev, res);
467 alloc_aiv_fail:
468 	sec_free_civ_resource(dev, res);
469 	return ret;
470 }
471 
472 static void sec_alg_resource_free(struct sec_ctx *ctx,
473 				  struct sec_qp_ctx *qp_ctx)
474 {
475 	struct device *dev = ctx->dev;
476 
477 	sec_free_civ_resource(dev, qp_ctx->res);
478 
479 	if (ctx->pbuf_supported)
480 		sec_free_pbuf_resource(dev, qp_ctx->res);
481 	if (ctx->alg_type == SEC_AEAD)
482 		sec_free_mac_resource(dev, qp_ctx->res);
483 }
484 
485 static int sec_alloc_qp_ctx_resource(struct hisi_qm *qm, struct sec_ctx *ctx,
486 				     struct sec_qp_ctx *qp_ctx)
487 {
488 	u16 q_depth = qp_ctx->qp->sq_depth;
489 	struct device *dev = ctx->dev;
490 	int ret = -ENOMEM;
491 
492 	qp_ctx->req_list = kcalloc(q_depth, sizeof(struct sec_req *), GFP_KERNEL);
493 	if (!qp_ctx->req_list)
494 		return ret;
495 
496 	qp_ctx->res = kcalloc(q_depth, sizeof(struct sec_alg_res), GFP_KERNEL);
497 	if (!qp_ctx->res)
498 		goto err_free_req_list;
499 	qp_ctx->res->depth = q_depth;
500 
501 	qp_ctx->c_in_pool = hisi_acc_create_sgl_pool(dev, q_depth, SEC_SGL_SGE_NR);
502 	if (IS_ERR(qp_ctx->c_in_pool)) {
503 		dev_err(dev, "fail to create sgl pool for input!\n");
504 		goto err_free_res;
505 	}
506 
507 	qp_ctx->c_out_pool = hisi_acc_create_sgl_pool(dev, q_depth, SEC_SGL_SGE_NR);
508 	if (IS_ERR(qp_ctx->c_out_pool)) {
509 		dev_err(dev, "fail to create sgl pool for output!\n");
510 		goto err_free_c_in_pool;
511 	}
512 
513 	ret = sec_alg_resource_alloc(ctx, qp_ctx);
514 	if (ret)
515 		goto err_free_c_out_pool;
516 
517 	return 0;
518 
519 err_free_c_out_pool:
520 	hisi_acc_free_sgl_pool(dev, qp_ctx->c_out_pool);
521 err_free_c_in_pool:
522 	hisi_acc_free_sgl_pool(dev, qp_ctx->c_in_pool);
523 err_free_res:
524 	kfree(qp_ctx->res);
525 err_free_req_list:
526 	kfree(qp_ctx->req_list);
527 	return ret;
528 }
529 
530 static void sec_free_qp_ctx_resource(struct sec_ctx *ctx, struct sec_qp_ctx *qp_ctx)
531 {
532 	struct device *dev = ctx->dev;
533 
534 	sec_alg_resource_free(ctx, qp_ctx);
535 	hisi_acc_free_sgl_pool(dev, qp_ctx->c_out_pool);
536 	hisi_acc_free_sgl_pool(dev, qp_ctx->c_in_pool);
537 	kfree(qp_ctx->res);
538 	kfree(qp_ctx->req_list);
539 }
540 
541 static int sec_create_qp_ctx(struct hisi_qm *qm, struct sec_ctx *ctx,
542 			     int qp_ctx_id, int alg_type)
543 {
544 	struct sec_qp_ctx *qp_ctx;
545 	struct hisi_qp *qp;
546 	int ret;
547 
548 	qp_ctx = &ctx->qp_ctx[qp_ctx_id];
549 	qp = ctx->qps[qp_ctx_id];
550 	qp->req_type = 0;
551 	qp->qp_ctx = qp_ctx;
552 	qp_ctx->qp = qp;
553 	qp_ctx->ctx = ctx;
554 
555 	qp->req_cb = sec_req_cb;
556 
557 	spin_lock_init(&qp_ctx->req_lock);
558 	idr_init(&qp_ctx->req_idr);
559 	INIT_LIST_HEAD(&qp_ctx->backlog);
560 
561 	ret = sec_alloc_qp_ctx_resource(qm, ctx, qp_ctx);
562 	if (ret)
563 		goto err_destroy_idr;
564 
565 	ret = hisi_qm_start_qp(qp, 0);
566 	if (ret < 0)
567 		goto err_resource_free;
568 
569 	return 0;
570 
571 err_resource_free:
572 	sec_free_qp_ctx_resource(ctx, qp_ctx);
573 err_destroy_idr:
574 	idr_destroy(&qp_ctx->req_idr);
575 	return ret;
576 }
577 
578 static void sec_release_qp_ctx(struct sec_ctx *ctx,
579 			       struct sec_qp_ctx *qp_ctx)
580 {
581 	hisi_qm_stop_qp(qp_ctx->qp);
582 	sec_free_qp_ctx_resource(ctx, qp_ctx);
583 	idr_destroy(&qp_ctx->req_idr);
584 }
585 
586 static int sec_ctx_base_init(struct sec_ctx *ctx)
587 {
588 	struct sec_dev *sec;
589 	int i, ret;
590 
591 	ctx->qps = sec_create_qps();
592 	if (!ctx->qps) {
593 		pr_err("Can not create sec qps!\n");
594 		return -ENODEV;
595 	}
596 
597 	sec = container_of(ctx->qps[0]->qm, struct sec_dev, qm);
598 	ctx->sec = sec;
599 	ctx->dev = &sec->qm.pdev->dev;
600 	ctx->hlf_q_num = sec->ctx_q_num >> 1;
601 
602 	ctx->pbuf_supported = ctx->sec->iommu_used;
603 
604 	/* Half of queue depth is taken as fake requests limit in the queue. */
605 	ctx->fake_req_limit = ctx->qps[0]->sq_depth >> 1;
606 	ctx->qp_ctx = kcalloc(sec->ctx_q_num, sizeof(struct sec_qp_ctx),
607 			      GFP_KERNEL);
608 	if (!ctx->qp_ctx) {
609 		ret = -ENOMEM;
610 		goto err_destroy_qps;
611 	}
612 
613 	for (i = 0; i < sec->ctx_q_num; i++) {
614 		ret = sec_create_qp_ctx(&sec->qm, ctx, i, 0);
615 		if (ret)
616 			goto err_sec_release_qp_ctx;
617 	}
618 
619 	return 0;
620 
621 err_sec_release_qp_ctx:
622 	for (i = i - 1; i >= 0; i--)
623 		sec_release_qp_ctx(ctx, &ctx->qp_ctx[i]);
624 	kfree(ctx->qp_ctx);
625 err_destroy_qps:
626 	sec_destroy_qps(ctx->qps, sec->ctx_q_num);
627 	return ret;
628 }
629 
630 static void sec_ctx_base_uninit(struct sec_ctx *ctx)
631 {
632 	int i;
633 
634 	for (i = 0; i < ctx->sec->ctx_q_num; i++)
635 		sec_release_qp_ctx(ctx, &ctx->qp_ctx[i]);
636 
637 	sec_destroy_qps(ctx->qps, ctx->sec->ctx_q_num);
638 	kfree(ctx->qp_ctx);
639 }
640 
641 static int sec_cipher_init(struct sec_ctx *ctx)
642 {
643 	struct sec_cipher_ctx *c_ctx = &ctx->c_ctx;
644 
645 	c_ctx->c_key = dma_alloc_coherent(ctx->dev, SEC_MAX_KEY_SIZE,
646 					  &c_ctx->c_key_dma, GFP_KERNEL);
647 	if (!c_ctx->c_key)
648 		return -ENOMEM;
649 
650 	return 0;
651 }
652 
653 static void sec_cipher_uninit(struct sec_ctx *ctx)
654 {
655 	struct sec_cipher_ctx *c_ctx = &ctx->c_ctx;
656 
657 	memzero_explicit(c_ctx->c_key, SEC_MAX_KEY_SIZE);
658 	dma_free_coherent(ctx->dev, SEC_MAX_KEY_SIZE,
659 			  c_ctx->c_key, c_ctx->c_key_dma);
660 }
661 
662 static int sec_auth_init(struct sec_ctx *ctx)
663 {
664 	struct sec_auth_ctx *a_ctx = &ctx->a_ctx;
665 
666 	a_ctx->a_key = dma_alloc_coherent(ctx->dev, SEC_MAX_AKEY_SIZE,
667 					  &a_ctx->a_key_dma, GFP_KERNEL);
668 	if (!a_ctx->a_key)
669 		return -ENOMEM;
670 
671 	return 0;
672 }
673 
674 static void sec_auth_uninit(struct sec_ctx *ctx)
675 {
676 	struct sec_auth_ctx *a_ctx = &ctx->a_ctx;
677 
678 	memzero_explicit(a_ctx->a_key, SEC_MAX_AKEY_SIZE);
679 	dma_free_coherent(ctx->dev, SEC_MAX_AKEY_SIZE,
680 			  a_ctx->a_key, a_ctx->a_key_dma);
681 }
682 
683 static int sec_skcipher_fbtfm_init(struct crypto_skcipher *tfm)
684 {
685 	const char *alg = crypto_tfm_alg_name(&tfm->base);
686 	struct sec_ctx *ctx = crypto_skcipher_ctx(tfm);
687 	struct sec_cipher_ctx *c_ctx = &ctx->c_ctx;
688 
689 	c_ctx->fallback = false;
690 
691 	/* Currently, only XTS mode need fallback tfm when using 192bit key */
692 	if (likely(strncmp(alg, "xts", SEC_XTS_NAME_SZ)))
693 		return 0;
694 
695 	c_ctx->fbtfm = crypto_alloc_sync_skcipher(alg, 0,
696 						  CRYPTO_ALG_NEED_FALLBACK);
697 	if (IS_ERR(c_ctx->fbtfm)) {
698 		pr_err("failed to alloc xts mode fallback tfm!\n");
699 		return PTR_ERR(c_ctx->fbtfm);
700 	}
701 
702 	return 0;
703 }
704 
705 static int sec_skcipher_init(struct crypto_skcipher *tfm)
706 {
707 	struct sec_ctx *ctx = crypto_skcipher_ctx(tfm);
708 	int ret;
709 
710 	ctx->alg_type = SEC_SKCIPHER;
711 	crypto_skcipher_set_reqsize(tfm, sizeof(struct sec_req));
712 	ctx->c_ctx.ivsize = crypto_skcipher_ivsize(tfm);
713 	if (ctx->c_ctx.ivsize > SEC_IV_SIZE) {
714 		pr_err("get error skcipher iv size!\n");
715 		return -EINVAL;
716 	}
717 
718 	ret = sec_ctx_base_init(ctx);
719 	if (ret)
720 		return ret;
721 
722 	ret = sec_cipher_init(ctx);
723 	if (ret)
724 		goto err_cipher_init;
725 
726 	ret = sec_skcipher_fbtfm_init(tfm);
727 	if (ret)
728 		goto err_fbtfm_init;
729 
730 	return 0;
731 
732 err_fbtfm_init:
733 	sec_cipher_uninit(ctx);
734 err_cipher_init:
735 	sec_ctx_base_uninit(ctx);
736 	return ret;
737 }
738 
739 static void sec_skcipher_uninit(struct crypto_skcipher *tfm)
740 {
741 	struct sec_ctx *ctx = crypto_skcipher_ctx(tfm);
742 
743 	if (ctx->c_ctx.fbtfm)
744 		crypto_free_sync_skcipher(ctx->c_ctx.fbtfm);
745 
746 	sec_cipher_uninit(ctx);
747 	sec_ctx_base_uninit(ctx);
748 }
749 
750 static int sec_skcipher_3des_setkey(struct crypto_skcipher *tfm, const u8 *key,
751 				    const u32 keylen,
752 				    const enum sec_cmode c_mode)
753 {
754 	struct sec_ctx *ctx = crypto_skcipher_ctx(tfm);
755 	struct sec_cipher_ctx *c_ctx = &ctx->c_ctx;
756 	int ret;
757 
758 	ret = verify_skcipher_des3_key(tfm, key);
759 	if (ret)
760 		return ret;
761 
762 	switch (keylen) {
763 	case SEC_DES3_2KEY_SIZE:
764 		c_ctx->c_key_len = SEC_CKEY_3DES_2KEY;
765 		break;
766 	case SEC_DES3_3KEY_SIZE:
767 		c_ctx->c_key_len = SEC_CKEY_3DES_3KEY;
768 		break;
769 	default:
770 		return -EINVAL;
771 	}
772 
773 	return 0;
774 }
775 
776 static int sec_skcipher_aes_sm4_setkey(struct sec_cipher_ctx *c_ctx,
777 				       const u32 keylen,
778 				       const enum sec_cmode c_mode)
779 {
780 	if (c_mode == SEC_CMODE_XTS) {
781 		switch (keylen) {
782 		case SEC_XTS_MIN_KEY_SIZE:
783 			c_ctx->c_key_len = SEC_CKEY_128BIT;
784 			break;
785 		case SEC_XTS_MID_KEY_SIZE:
786 			c_ctx->fallback = true;
787 			break;
788 		case SEC_XTS_MAX_KEY_SIZE:
789 			c_ctx->c_key_len = SEC_CKEY_256BIT;
790 			break;
791 		default:
792 			pr_err("hisi_sec2: xts mode key error!\n");
793 			return -EINVAL;
794 		}
795 	} else {
796 		if (c_ctx->c_alg == SEC_CALG_SM4 &&
797 		    keylen != AES_KEYSIZE_128) {
798 			pr_err("hisi_sec2: sm4 key error!\n");
799 			return -EINVAL;
800 		} else {
801 			switch (keylen) {
802 			case AES_KEYSIZE_128:
803 				c_ctx->c_key_len = SEC_CKEY_128BIT;
804 				break;
805 			case AES_KEYSIZE_192:
806 				c_ctx->c_key_len = SEC_CKEY_192BIT;
807 				break;
808 			case AES_KEYSIZE_256:
809 				c_ctx->c_key_len = SEC_CKEY_256BIT;
810 				break;
811 			default:
812 				pr_err("hisi_sec2: aes key error!\n");
813 				return -EINVAL;
814 			}
815 		}
816 	}
817 
818 	return 0;
819 }
820 
821 static int sec_skcipher_setkey(struct crypto_skcipher *tfm, const u8 *key,
822 			       const u32 keylen, const enum sec_calg c_alg,
823 			       const enum sec_cmode c_mode)
824 {
825 	struct sec_ctx *ctx = crypto_skcipher_ctx(tfm);
826 	struct sec_cipher_ctx *c_ctx = &ctx->c_ctx;
827 	struct device *dev = ctx->dev;
828 	int ret;
829 
830 	if (c_mode == SEC_CMODE_XTS) {
831 		ret = xts_verify_key(tfm, key, keylen);
832 		if (ret) {
833 			dev_err(dev, "xts mode key err!\n");
834 			return ret;
835 		}
836 	}
837 
838 	c_ctx->c_alg  = c_alg;
839 	c_ctx->c_mode = c_mode;
840 
841 	switch (c_alg) {
842 	case SEC_CALG_3DES:
843 		ret = sec_skcipher_3des_setkey(tfm, key, keylen, c_mode);
844 		break;
845 	case SEC_CALG_AES:
846 	case SEC_CALG_SM4:
847 		ret = sec_skcipher_aes_sm4_setkey(c_ctx, keylen, c_mode);
848 		break;
849 	default:
850 		return -EINVAL;
851 	}
852 
853 	if (ret) {
854 		dev_err(dev, "set sec key err!\n");
855 		return ret;
856 	}
857 
858 	memcpy(c_ctx->c_key, key, keylen);
859 	if (c_ctx->fallback && c_ctx->fbtfm) {
860 		ret = crypto_sync_skcipher_setkey(c_ctx->fbtfm, key, keylen);
861 		if (ret) {
862 			dev_err(dev, "failed to set fallback skcipher key!\n");
863 			return ret;
864 		}
865 	}
866 	return 0;
867 }
868 
869 #define GEN_SEC_SETKEY_FUNC(name, c_alg, c_mode)			\
870 static int sec_setkey_##name(struct crypto_skcipher *tfm, const u8 *key,\
871 	u32 keylen)							\
872 {									\
873 	return sec_skcipher_setkey(tfm, key, keylen, c_alg, c_mode);	\
874 }
875 
876 GEN_SEC_SETKEY_FUNC(aes_ecb, SEC_CALG_AES, SEC_CMODE_ECB)
877 GEN_SEC_SETKEY_FUNC(aes_cbc, SEC_CALG_AES, SEC_CMODE_CBC)
878 GEN_SEC_SETKEY_FUNC(aes_xts, SEC_CALG_AES, SEC_CMODE_XTS)
879 GEN_SEC_SETKEY_FUNC(aes_ofb, SEC_CALG_AES, SEC_CMODE_OFB)
880 GEN_SEC_SETKEY_FUNC(aes_cfb, SEC_CALG_AES, SEC_CMODE_CFB)
881 GEN_SEC_SETKEY_FUNC(aes_ctr, SEC_CALG_AES, SEC_CMODE_CTR)
882 GEN_SEC_SETKEY_FUNC(3des_ecb, SEC_CALG_3DES, SEC_CMODE_ECB)
883 GEN_SEC_SETKEY_FUNC(3des_cbc, SEC_CALG_3DES, SEC_CMODE_CBC)
884 GEN_SEC_SETKEY_FUNC(sm4_xts, SEC_CALG_SM4, SEC_CMODE_XTS)
885 GEN_SEC_SETKEY_FUNC(sm4_cbc, SEC_CALG_SM4, SEC_CMODE_CBC)
886 GEN_SEC_SETKEY_FUNC(sm4_ofb, SEC_CALG_SM4, SEC_CMODE_OFB)
887 GEN_SEC_SETKEY_FUNC(sm4_cfb, SEC_CALG_SM4, SEC_CMODE_CFB)
888 GEN_SEC_SETKEY_FUNC(sm4_ctr, SEC_CALG_SM4, SEC_CMODE_CTR)
889 
890 static int sec_cipher_pbuf_map(struct sec_ctx *ctx, struct sec_req *req,
891 			struct scatterlist *src)
892 {
893 	struct sec_aead_req *a_req = &req->aead_req;
894 	struct aead_request *aead_req = a_req->aead_req;
895 	struct sec_cipher_req *c_req = &req->c_req;
896 	struct sec_qp_ctx *qp_ctx = req->qp_ctx;
897 	struct device *dev = ctx->dev;
898 	int copy_size, pbuf_length;
899 	int req_id = req->req_id;
900 	struct crypto_aead *tfm;
901 	size_t authsize;
902 	u8 *mac_offset;
903 
904 	if (ctx->alg_type == SEC_AEAD)
905 		copy_size = aead_req->cryptlen + aead_req->assoclen;
906 	else
907 		copy_size = c_req->c_len;
908 
909 	pbuf_length = sg_copy_to_buffer(src, sg_nents(src),
910 			qp_ctx->res[req_id].pbuf, copy_size);
911 	if (unlikely(pbuf_length != copy_size)) {
912 		dev_err(dev, "copy src data to pbuf error!\n");
913 		return -EINVAL;
914 	}
915 	if (!c_req->encrypt && ctx->alg_type == SEC_AEAD) {
916 		tfm = crypto_aead_reqtfm(aead_req);
917 		authsize = crypto_aead_authsize(tfm);
918 		mac_offset = qp_ctx->res[req_id].pbuf + copy_size - authsize;
919 		memcpy(a_req->out_mac, mac_offset, authsize);
920 	}
921 
922 	req->in_dma = qp_ctx->res[req_id].pbuf_dma;
923 	c_req->c_out_dma = req->in_dma;
924 
925 	return 0;
926 }
927 
928 static void sec_cipher_pbuf_unmap(struct sec_ctx *ctx, struct sec_req *req,
929 			struct scatterlist *dst)
930 {
931 	struct aead_request *aead_req = req->aead_req.aead_req;
932 	struct sec_cipher_req *c_req = &req->c_req;
933 	struct sec_qp_ctx *qp_ctx = req->qp_ctx;
934 	int copy_size, pbuf_length;
935 	int req_id = req->req_id;
936 
937 	if (ctx->alg_type == SEC_AEAD)
938 		copy_size = c_req->c_len + aead_req->assoclen;
939 	else
940 		copy_size = c_req->c_len;
941 
942 	pbuf_length = sg_copy_from_buffer(dst, sg_nents(dst),
943 			qp_ctx->res[req_id].pbuf, copy_size);
944 	if (unlikely(pbuf_length != copy_size))
945 		dev_err(ctx->dev, "copy pbuf data to dst error!\n");
946 }
947 
948 static int sec_aead_mac_init(struct sec_aead_req *req)
949 {
950 	struct aead_request *aead_req = req->aead_req;
951 	struct crypto_aead *tfm = crypto_aead_reqtfm(aead_req);
952 	size_t authsize = crypto_aead_authsize(tfm);
953 	u8 *mac_out = req->out_mac;
954 	struct scatterlist *sgl = aead_req->src;
955 	size_t copy_size;
956 	off_t skip_size;
957 
958 	/* Copy input mac */
959 	skip_size = aead_req->assoclen + aead_req->cryptlen - authsize;
960 	copy_size = sg_pcopy_to_buffer(sgl, sg_nents(sgl), mac_out,
961 				       authsize, skip_size);
962 	if (unlikely(copy_size != authsize))
963 		return -EINVAL;
964 
965 	return 0;
966 }
967 
968 static int sec_cipher_map(struct sec_ctx *ctx, struct sec_req *req,
969 			  struct scatterlist *src, struct scatterlist *dst)
970 {
971 	struct sec_cipher_req *c_req = &req->c_req;
972 	struct sec_aead_req *a_req = &req->aead_req;
973 	struct sec_qp_ctx *qp_ctx = req->qp_ctx;
974 	struct sec_alg_res *res = &qp_ctx->res[req->req_id];
975 	struct device *dev = ctx->dev;
976 	int ret;
977 
978 	if (req->use_pbuf) {
979 		c_req->c_ivin = res->pbuf + SEC_PBUF_IV_OFFSET;
980 		c_req->c_ivin_dma = res->pbuf_dma + SEC_PBUF_IV_OFFSET;
981 		if (ctx->alg_type == SEC_AEAD) {
982 			a_req->a_ivin = res->a_ivin;
983 			a_req->a_ivin_dma = res->a_ivin_dma;
984 			a_req->out_mac = res->pbuf + SEC_PBUF_MAC_OFFSET;
985 			a_req->out_mac_dma = res->pbuf_dma +
986 					SEC_PBUF_MAC_OFFSET;
987 		}
988 		ret = sec_cipher_pbuf_map(ctx, req, src);
989 
990 		return ret;
991 	}
992 	c_req->c_ivin = res->c_ivin;
993 	c_req->c_ivin_dma = res->c_ivin_dma;
994 	if (ctx->alg_type == SEC_AEAD) {
995 		a_req->a_ivin = res->a_ivin;
996 		a_req->a_ivin_dma = res->a_ivin_dma;
997 		a_req->out_mac = res->out_mac;
998 		a_req->out_mac_dma = res->out_mac_dma;
999 	}
1000 
1001 	req->in = hisi_acc_sg_buf_map_to_hw_sgl(dev, src,
1002 						qp_ctx->c_in_pool,
1003 						req->req_id,
1004 						&req->in_dma);
1005 	if (IS_ERR(req->in)) {
1006 		dev_err(dev, "fail to dma map input sgl buffers!\n");
1007 		return PTR_ERR(req->in);
1008 	}
1009 
1010 	if (!c_req->encrypt && ctx->alg_type == SEC_AEAD) {
1011 		ret = sec_aead_mac_init(a_req);
1012 		if (unlikely(ret)) {
1013 			dev_err(dev, "fail to init mac data for ICV!\n");
1014 			return ret;
1015 		}
1016 	}
1017 
1018 	if (dst == src) {
1019 		c_req->c_out = req->in;
1020 		c_req->c_out_dma = req->in_dma;
1021 	} else {
1022 		c_req->c_out = hisi_acc_sg_buf_map_to_hw_sgl(dev, dst,
1023 							     qp_ctx->c_out_pool,
1024 							     req->req_id,
1025 							     &c_req->c_out_dma);
1026 
1027 		if (IS_ERR(c_req->c_out)) {
1028 			dev_err(dev, "fail to dma map output sgl buffers!\n");
1029 			hisi_acc_sg_buf_unmap(dev, src, req->in);
1030 			return PTR_ERR(c_req->c_out);
1031 		}
1032 	}
1033 
1034 	return 0;
1035 }
1036 
1037 static void sec_cipher_unmap(struct sec_ctx *ctx, struct sec_req *req,
1038 			     struct scatterlist *src, struct scatterlist *dst)
1039 {
1040 	struct sec_cipher_req *c_req = &req->c_req;
1041 	struct device *dev = ctx->dev;
1042 
1043 	if (req->use_pbuf) {
1044 		sec_cipher_pbuf_unmap(ctx, req, dst);
1045 	} else {
1046 		if (dst != src)
1047 			hisi_acc_sg_buf_unmap(dev, src, req->in);
1048 
1049 		hisi_acc_sg_buf_unmap(dev, dst, c_req->c_out);
1050 	}
1051 }
1052 
1053 static int sec_skcipher_sgl_map(struct sec_ctx *ctx, struct sec_req *req)
1054 {
1055 	struct skcipher_request *sq = req->c_req.sk_req;
1056 
1057 	return sec_cipher_map(ctx, req, sq->src, sq->dst);
1058 }
1059 
1060 static void sec_skcipher_sgl_unmap(struct sec_ctx *ctx, struct sec_req *req)
1061 {
1062 	struct skcipher_request *sq = req->c_req.sk_req;
1063 
1064 	sec_cipher_unmap(ctx, req, sq->src, sq->dst);
1065 }
1066 
1067 static int sec_aead_aes_set_key(struct sec_cipher_ctx *c_ctx,
1068 				struct crypto_authenc_keys *keys)
1069 {
1070 	switch (keys->enckeylen) {
1071 	case AES_KEYSIZE_128:
1072 		c_ctx->c_key_len = SEC_CKEY_128BIT;
1073 		break;
1074 	case AES_KEYSIZE_192:
1075 		c_ctx->c_key_len = SEC_CKEY_192BIT;
1076 		break;
1077 	case AES_KEYSIZE_256:
1078 		c_ctx->c_key_len = SEC_CKEY_256BIT;
1079 		break;
1080 	default:
1081 		pr_err("hisi_sec2: aead aes key error!\n");
1082 		return -EINVAL;
1083 	}
1084 	memcpy(c_ctx->c_key, keys->enckey, keys->enckeylen);
1085 
1086 	return 0;
1087 }
1088 
1089 static int sec_aead_auth_set_key(struct sec_auth_ctx *ctx,
1090 				 struct crypto_authenc_keys *keys)
1091 {
1092 	struct crypto_shash *hash_tfm = ctx->hash_tfm;
1093 	int blocksize, digestsize, ret;
1094 
1095 	if (!keys->authkeylen) {
1096 		pr_err("hisi_sec2: aead auth key error!\n");
1097 		return -EINVAL;
1098 	}
1099 
1100 	blocksize = crypto_shash_blocksize(hash_tfm);
1101 	digestsize = crypto_shash_digestsize(hash_tfm);
1102 	if (keys->authkeylen > blocksize) {
1103 		ret = crypto_shash_tfm_digest(hash_tfm, keys->authkey,
1104 					      keys->authkeylen, ctx->a_key);
1105 		if (ret) {
1106 			pr_err("hisi_sec2: aead auth digest error!\n");
1107 			return -EINVAL;
1108 		}
1109 		ctx->a_key_len = digestsize;
1110 	} else {
1111 		memcpy(ctx->a_key, keys->authkey, keys->authkeylen);
1112 		ctx->a_key_len = keys->authkeylen;
1113 	}
1114 
1115 	return 0;
1116 }
1117 
1118 static int sec_aead_setauthsize(struct crypto_aead *aead, unsigned int authsize)
1119 {
1120 	struct crypto_tfm *tfm = crypto_aead_tfm(aead);
1121 	struct sec_ctx *ctx = crypto_tfm_ctx(tfm);
1122 	struct sec_auth_ctx *a_ctx = &ctx->a_ctx;
1123 
1124 	if (unlikely(a_ctx->fallback_aead_tfm))
1125 		return crypto_aead_setauthsize(a_ctx->fallback_aead_tfm, authsize);
1126 
1127 	return 0;
1128 }
1129 
1130 static int sec_aead_fallback_setkey(struct sec_auth_ctx *a_ctx,
1131 				    struct crypto_aead *tfm, const u8 *key,
1132 				    unsigned int keylen)
1133 {
1134 	crypto_aead_clear_flags(a_ctx->fallback_aead_tfm, CRYPTO_TFM_REQ_MASK);
1135 	crypto_aead_set_flags(a_ctx->fallback_aead_tfm,
1136 			      crypto_aead_get_flags(tfm) & CRYPTO_TFM_REQ_MASK);
1137 	return crypto_aead_setkey(a_ctx->fallback_aead_tfm, key, keylen);
1138 }
1139 
1140 static int sec_aead_setkey(struct crypto_aead *tfm, const u8 *key,
1141 			   const u32 keylen, const enum sec_hash_alg a_alg,
1142 			   const enum sec_calg c_alg,
1143 			   const enum sec_mac_len mac_len,
1144 			   const enum sec_cmode c_mode)
1145 {
1146 	struct sec_ctx *ctx = crypto_aead_ctx(tfm);
1147 	struct sec_cipher_ctx *c_ctx = &ctx->c_ctx;
1148 	struct sec_auth_ctx *a_ctx = &ctx->a_ctx;
1149 	struct device *dev = ctx->dev;
1150 	struct crypto_authenc_keys keys;
1151 	int ret;
1152 
1153 	ctx->a_ctx.a_alg = a_alg;
1154 	ctx->c_ctx.c_alg = c_alg;
1155 	ctx->a_ctx.mac_len = mac_len;
1156 	c_ctx->c_mode = c_mode;
1157 
1158 	if (c_mode == SEC_CMODE_CCM || c_mode == SEC_CMODE_GCM) {
1159 		ret = sec_skcipher_aes_sm4_setkey(c_ctx, keylen, c_mode);
1160 		if (ret) {
1161 			dev_err(dev, "set sec aes ccm cipher key err!\n");
1162 			return ret;
1163 		}
1164 		memcpy(c_ctx->c_key, key, keylen);
1165 
1166 		if (unlikely(a_ctx->fallback_aead_tfm)) {
1167 			ret = sec_aead_fallback_setkey(a_ctx, tfm, key, keylen);
1168 			if (ret)
1169 				return ret;
1170 		}
1171 
1172 		return 0;
1173 	}
1174 
1175 	if (crypto_authenc_extractkeys(&keys, key, keylen))
1176 		goto bad_key;
1177 
1178 	ret = sec_aead_aes_set_key(c_ctx, &keys);
1179 	if (ret) {
1180 		dev_err(dev, "set sec cipher key err!\n");
1181 		goto bad_key;
1182 	}
1183 
1184 	ret = sec_aead_auth_set_key(&ctx->a_ctx, &keys);
1185 	if (ret) {
1186 		dev_err(dev, "set sec auth key err!\n");
1187 		goto bad_key;
1188 	}
1189 
1190 	if ((ctx->a_ctx.mac_len & SEC_SQE_LEN_RATE_MASK)  ||
1191 	    (ctx->a_ctx.a_key_len & SEC_SQE_LEN_RATE_MASK)) {
1192 		dev_err(dev, "MAC or AUTH key length error!\n");
1193 		goto bad_key;
1194 	}
1195 
1196 	return 0;
1197 
1198 bad_key:
1199 	memzero_explicit(&keys, sizeof(struct crypto_authenc_keys));
1200 	return -EINVAL;
1201 }
1202 
1203 
1204 #define GEN_SEC_AEAD_SETKEY_FUNC(name, aalg, calg, maclen, cmode)	\
1205 static int sec_setkey_##name(struct crypto_aead *tfm, const u8 *key,	\
1206 	u32 keylen)							\
1207 {									\
1208 	return sec_aead_setkey(tfm, key, keylen, aalg, calg, maclen, cmode);\
1209 }
1210 
1211 GEN_SEC_AEAD_SETKEY_FUNC(aes_cbc_sha1, SEC_A_HMAC_SHA1,
1212 			 SEC_CALG_AES, SEC_HMAC_SHA1_MAC, SEC_CMODE_CBC)
1213 GEN_SEC_AEAD_SETKEY_FUNC(aes_cbc_sha256, SEC_A_HMAC_SHA256,
1214 			 SEC_CALG_AES, SEC_HMAC_SHA256_MAC, SEC_CMODE_CBC)
1215 GEN_SEC_AEAD_SETKEY_FUNC(aes_cbc_sha512, SEC_A_HMAC_SHA512,
1216 			 SEC_CALG_AES, SEC_HMAC_SHA512_MAC, SEC_CMODE_CBC)
1217 GEN_SEC_AEAD_SETKEY_FUNC(aes_ccm, 0, SEC_CALG_AES,
1218 			 SEC_HMAC_CCM_MAC, SEC_CMODE_CCM)
1219 GEN_SEC_AEAD_SETKEY_FUNC(aes_gcm, 0, SEC_CALG_AES,
1220 			 SEC_HMAC_GCM_MAC, SEC_CMODE_GCM)
1221 GEN_SEC_AEAD_SETKEY_FUNC(sm4_ccm, 0, SEC_CALG_SM4,
1222 			 SEC_HMAC_CCM_MAC, SEC_CMODE_CCM)
1223 GEN_SEC_AEAD_SETKEY_FUNC(sm4_gcm, 0, SEC_CALG_SM4,
1224 			 SEC_HMAC_GCM_MAC, SEC_CMODE_GCM)
1225 
1226 static int sec_aead_sgl_map(struct sec_ctx *ctx, struct sec_req *req)
1227 {
1228 	struct aead_request *aq = req->aead_req.aead_req;
1229 
1230 	return sec_cipher_map(ctx, req, aq->src, aq->dst);
1231 }
1232 
1233 static void sec_aead_sgl_unmap(struct sec_ctx *ctx, struct sec_req *req)
1234 {
1235 	struct aead_request *aq = req->aead_req.aead_req;
1236 
1237 	sec_cipher_unmap(ctx, req, aq->src, aq->dst);
1238 }
1239 
1240 static int sec_request_transfer(struct sec_ctx *ctx, struct sec_req *req)
1241 {
1242 	int ret;
1243 
1244 	ret = ctx->req_op->buf_map(ctx, req);
1245 	if (unlikely(ret))
1246 		return ret;
1247 
1248 	ctx->req_op->do_transfer(ctx, req);
1249 
1250 	ret = ctx->req_op->bd_fill(ctx, req);
1251 	if (unlikely(ret))
1252 		goto unmap_req_buf;
1253 
1254 	return ret;
1255 
1256 unmap_req_buf:
1257 	ctx->req_op->buf_unmap(ctx, req);
1258 	return ret;
1259 }
1260 
1261 static void sec_request_untransfer(struct sec_ctx *ctx, struct sec_req *req)
1262 {
1263 	ctx->req_op->buf_unmap(ctx, req);
1264 }
1265 
1266 static void sec_skcipher_copy_iv(struct sec_ctx *ctx, struct sec_req *req)
1267 {
1268 	struct skcipher_request *sk_req = req->c_req.sk_req;
1269 	struct sec_cipher_req *c_req = &req->c_req;
1270 
1271 	memcpy(c_req->c_ivin, sk_req->iv, ctx->c_ctx.ivsize);
1272 }
1273 
1274 static int sec_skcipher_bd_fill(struct sec_ctx *ctx, struct sec_req *req)
1275 {
1276 	struct sec_cipher_ctx *c_ctx = &ctx->c_ctx;
1277 	struct sec_cipher_req *c_req = &req->c_req;
1278 	struct sec_sqe *sec_sqe = &req->sec_sqe;
1279 	u8 scene, sa_type, da_type;
1280 	u8 bd_type, cipher;
1281 	u8 de = 0;
1282 
1283 	memset(sec_sqe, 0, sizeof(struct sec_sqe));
1284 
1285 	sec_sqe->type2.c_key_addr = cpu_to_le64(c_ctx->c_key_dma);
1286 	sec_sqe->type2.c_ivin_addr = cpu_to_le64(c_req->c_ivin_dma);
1287 	sec_sqe->type2.data_src_addr = cpu_to_le64(req->in_dma);
1288 	sec_sqe->type2.data_dst_addr = cpu_to_le64(c_req->c_out_dma);
1289 
1290 	sec_sqe->type2.icvw_kmode |= cpu_to_le16(((u16)c_ctx->c_mode) <<
1291 						SEC_CMODE_OFFSET);
1292 	sec_sqe->type2.c_alg = c_ctx->c_alg;
1293 	sec_sqe->type2.icvw_kmode |= cpu_to_le16(((u16)c_ctx->c_key_len) <<
1294 						SEC_CKEY_OFFSET);
1295 
1296 	bd_type = SEC_BD_TYPE2;
1297 	if (c_req->encrypt)
1298 		cipher = SEC_CIPHER_ENC << SEC_CIPHER_OFFSET;
1299 	else
1300 		cipher = SEC_CIPHER_DEC << SEC_CIPHER_OFFSET;
1301 	sec_sqe->type_cipher_auth = bd_type | cipher;
1302 
1303 	/* Set destination and source address type */
1304 	if (req->use_pbuf) {
1305 		sa_type = SEC_PBUF << SEC_SRC_SGL_OFFSET;
1306 		da_type = SEC_PBUF << SEC_DST_SGL_OFFSET;
1307 	} else {
1308 		sa_type = SEC_SGL << SEC_SRC_SGL_OFFSET;
1309 		da_type = SEC_SGL << SEC_DST_SGL_OFFSET;
1310 	}
1311 
1312 	sec_sqe->sdm_addr_type |= da_type;
1313 	scene = SEC_COMM_SCENE << SEC_SCENE_OFFSET;
1314 	if (req->in_dma != c_req->c_out_dma)
1315 		de = 0x1 << SEC_DE_OFFSET;
1316 
1317 	sec_sqe->sds_sa_type = (de | scene | sa_type);
1318 
1319 	sec_sqe->type2.clen_ivhlen |= cpu_to_le32(c_req->c_len);
1320 	sec_sqe->type2.tag = cpu_to_le16((u16)req->req_id);
1321 
1322 	return 0;
1323 }
1324 
1325 static int sec_skcipher_bd_fill_v3(struct sec_ctx *ctx, struct sec_req *req)
1326 {
1327 	struct sec_sqe3 *sec_sqe3 = &req->sec_sqe3;
1328 	struct sec_cipher_ctx *c_ctx = &ctx->c_ctx;
1329 	struct sec_cipher_req *c_req = &req->c_req;
1330 	u32 bd_param = 0;
1331 	u16 cipher;
1332 
1333 	memset(sec_sqe3, 0, sizeof(struct sec_sqe3));
1334 
1335 	sec_sqe3->c_key_addr = cpu_to_le64(c_ctx->c_key_dma);
1336 	sec_sqe3->no_scene.c_ivin_addr = cpu_to_le64(c_req->c_ivin_dma);
1337 	sec_sqe3->data_src_addr = cpu_to_le64(req->in_dma);
1338 	sec_sqe3->data_dst_addr = cpu_to_le64(c_req->c_out_dma);
1339 
1340 	sec_sqe3->c_mode_alg = ((u8)c_ctx->c_alg << SEC_CALG_OFFSET_V3) |
1341 						c_ctx->c_mode;
1342 	sec_sqe3->c_icv_key |= cpu_to_le16(((u16)c_ctx->c_key_len) <<
1343 						SEC_CKEY_OFFSET_V3);
1344 
1345 	if (c_req->encrypt)
1346 		cipher = SEC_CIPHER_ENC;
1347 	else
1348 		cipher = SEC_CIPHER_DEC;
1349 	sec_sqe3->c_icv_key |= cpu_to_le16(cipher);
1350 
1351 	/* Set the CTR counter mode is 128bit rollover */
1352 	sec_sqe3->auth_mac_key = cpu_to_le32((u32)SEC_CTR_CNT_ROLLOVER <<
1353 					SEC_CTR_CNT_OFFSET);
1354 
1355 	if (req->use_pbuf) {
1356 		bd_param |= SEC_PBUF << SEC_SRC_SGL_OFFSET_V3;
1357 		bd_param |= SEC_PBUF << SEC_DST_SGL_OFFSET_V3;
1358 	} else {
1359 		bd_param |= SEC_SGL << SEC_SRC_SGL_OFFSET_V3;
1360 		bd_param |= SEC_SGL << SEC_DST_SGL_OFFSET_V3;
1361 	}
1362 
1363 	bd_param |= SEC_COMM_SCENE << SEC_SCENE_OFFSET_V3;
1364 	if (req->in_dma != c_req->c_out_dma)
1365 		bd_param |= 0x1 << SEC_DE_OFFSET_V3;
1366 
1367 	bd_param |= SEC_BD_TYPE3;
1368 	sec_sqe3->bd_param = cpu_to_le32(bd_param);
1369 
1370 	sec_sqe3->c_len_ivin |= cpu_to_le32(c_req->c_len);
1371 	sec_sqe3->tag = cpu_to_le64(req);
1372 
1373 	return 0;
1374 }
1375 
1376 /* increment counter (128-bit int) */
1377 static void ctr_iv_inc(__u8 *counter, __u8 bits, __u32 nums)
1378 {
1379 	do {
1380 		--bits;
1381 		nums += counter[bits];
1382 		counter[bits] = nums & BITS_MASK;
1383 		nums >>= BYTE_BITS;
1384 	} while (bits && nums);
1385 }
1386 
1387 static void sec_update_iv(struct sec_req *req, enum sec_alg_type alg_type)
1388 {
1389 	struct aead_request *aead_req = req->aead_req.aead_req;
1390 	struct skcipher_request *sk_req = req->c_req.sk_req;
1391 	u32 iv_size = req->ctx->c_ctx.ivsize;
1392 	struct scatterlist *sgl;
1393 	unsigned int cryptlen;
1394 	size_t sz;
1395 	u8 *iv;
1396 
1397 	if (req->c_req.encrypt)
1398 		sgl = alg_type == SEC_SKCIPHER ? sk_req->dst : aead_req->dst;
1399 	else
1400 		sgl = alg_type == SEC_SKCIPHER ? sk_req->src : aead_req->src;
1401 
1402 	if (alg_type == SEC_SKCIPHER) {
1403 		iv = sk_req->iv;
1404 		cryptlen = sk_req->cryptlen;
1405 	} else {
1406 		iv = aead_req->iv;
1407 		cryptlen = aead_req->cryptlen;
1408 	}
1409 
1410 	if (req->ctx->c_ctx.c_mode == SEC_CMODE_CBC) {
1411 		sz = sg_pcopy_to_buffer(sgl, sg_nents(sgl), iv, iv_size,
1412 					cryptlen - iv_size);
1413 		if (unlikely(sz != iv_size))
1414 			dev_err(req->ctx->dev, "copy output iv error!\n");
1415 	} else {
1416 		sz = cryptlen / iv_size;
1417 		if (cryptlen % iv_size)
1418 			sz += 1;
1419 		ctr_iv_inc(iv, iv_size, sz);
1420 	}
1421 }
1422 
1423 static struct sec_req *sec_back_req_clear(struct sec_ctx *ctx,
1424 				struct sec_qp_ctx *qp_ctx)
1425 {
1426 	struct sec_req *backlog_req = NULL;
1427 
1428 	spin_lock_bh(&qp_ctx->req_lock);
1429 	if (ctx->fake_req_limit >=
1430 	    atomic_read(&qp_ctx->qp->qp_status.used) &&
1431 	    !list_empty(&qp_ctx->backlog)) {
1432 		backlog_req = list_first_entry(&qp_ctx->backlog,
1433 				typeof(*backlog_req), backlog_head);
1434 		list_del(&backlog_req->backlog_head);
1435 	}
1436 	spin_unlock_bh(&qp_ctx->req_lock);
1437 
1438 	return backlog_req;
1439 }
1440 
1441 static void sec_skcipher_callback(struct sec_ctx *ctx, struct sec_req *req,
1442 				  int err)
1443 {
1444 	struct skcipher_request *sk_req = req->c_req.sk_req;
1445 	struct sec_qp_ctx *qp_ctx = req->qp_ctx;
1446 	struct skcipher_request *backlog_sk_req;
1447 	struct sec_req *backlog_req;
1448 
1449 	sec_free_req_id(req);
1450 
1451 	/* IV output at encrypto of CBC/CTR mode */
1452 	if (!err && (ctx->c_ctx.c_mode == SEC_CMODE_CBC ||
1453 	    ctx->c_ctx.c_mode == SEC_CMODE_CTR) && req->c_req.encrypt)
1454 		sec_update_iv(req, SEC_SKCIPHER);
1455 
1456 	while (1) {
1457 		backlog_req = sec_back_req_clear(ctx, qp_ctx);
1458 		if (!backlog_req)
1459 			break;
1460 
1461 		backlog_sk_req = backlog_req->c_req.sk_req;
1462 		skcipher_request_complete(backlog_sk_req, -EINPROGRESS);
1463 		atomic64_inc(&ctx->sec->debug.dfx.recv_busy_cnt);
1464 	}
1465 
1466 	skcipher_request_complete(sk_req, err);
1467 }
1468 
1469 static void set_aead_auth_iv(struct sec_ctx *ctx, struct sec_req *req)
1470 {
1471 	struct aead_request *aead_req = req->aead_req.aead_req;
1472 	struct sec_cipher_req *c_req = &req->c_req;
1473 	struct sec_aead_req *a_req = &req->aead_req;
1474 	size_t authsize = ctx->a_ctx.mac_len;
1475 	u32 data_size = aead_req->cryptlen;
1476 	u8 flage = 0;
1477 	u8 cm, cl;
1478 
1479 	/* the specification has been checked in aead_iv_demension_check() */
1480 	cl = c_req->c_ivin[0] + 1;
1481 	c_req->c_ivin[ctx->c_ctx.ivsize - cl] = 0x00;
1482 	memset(&c_req->c_ivin[ctx->c_ctx.ivsize - cl], 0, cl);
1483 	c_req->c_ivin[ctx->c_ctx.ivsize - IV_LAST_BYTE1] = IV_CTR_INIT;
1484 
1485 	/* the last 3bit is L' */
1486 	flage |= c_req->c_ivin[0] & IV_CL_MASK;
1487 
1488 	/* the M' is bit3~bit5, the Flags is bit6 */
1489 	cm = (authsize - IV_CM_CAL_NUM) / IV_CM_CAL_NUM;
1490 	flage |= cm << IV_CM_OFFSET;
1491 	if (aead_req->assoclen)
1492 		flage |= 0x01 << IV_FLAGS_OFFSET;
1493 
1494 	memcpy(a_req->a_ivin, c_req->c_ivin, ctx->c_ctx.ivsize);
1495 	a_req->a_ivin[0] = flage;
1496 
1497 	/*
1498 	 * the last 32bit is counter's initial number,
1499 	 * but the nonce uses the first 16bit
1500 	 * the tail 16bit fill with the cipher length
1501 	 */
1502 	if (!c_req->encrypt)
1503 		data_size = aead_req->cryptlen - authsize;
1504 
1505 	a_req->a_ivin[ctx->c_ctx.ivsize - IV_LAST_BYTE1] =
1506 			data_size & IV_LAST_BYTE_MASK;
1507 	data_size >>= IV_BYTE_OFFSET;
1508 	a_req->a_ivin[ctx->c_ctx.ivsize - IV_LAST_BYTE2] =
1509 			data_size & IV_LAST_BYTE_MASK;
1510 }
1511 
1512 static void sec_aead_set_iv(struct sec_ctx *ctx, struct sec_req *req)
1513 {
1514 	struct aead_request *aead_req = req->aead_req.aead_req;
1515 	struct crypto_aead *tfm = crypto_aead_reqtfm(aead_req);
1516 	size_t authsize = crypto_aead_authsize(tfm);
1517 	struct sec_cipher_req *c_req = &req->c_req;
1518 	struct sec_aead_req *a_req = &req->aead_req;
1519 
1520 	memcpy(c_req->c_ivin, aead_req->iv, ctx->c_ctx.ivsize);
1521 
1522 	if (ctx->c_ctx.c_mode == SEC_CMODE_CCM) {
1523 		/*
1524 		 * CCM 16Byte Cipher_IV: {1B_Flage,13B_IV,2B_counter},
1525 		 * the  counter must set to 0x01
1526 		 */
1527 		ctx->a_ctx.mac_len = authsize;
1528 		/* CCM 16Byte Auth_IV: {1B_AFlage,13B_IV,2B_Ptext_length} */
1529 		set_aead_auth_iv(ctx, req);
1530 	}
1531 
1532 	/* GCM 12Byte Cipher_IV == Auth_IV */
1533 	if (ctx->c_ctx.c_mode == SEC_CMODE_GCM) {
1534 		ctx->a_ctx.mac_len = authsize;
1535 		memcpy(a_req->a_ivin, c_req->c_ivin, SEC_AIV_SIZE);
1536 	}
1537 }
1538 
1539 static void sec_auth_bd_fill_xcm(struct sec_auth_ctx *ctx, int dir,
1540 				 struct sec_req *req, struct sec_sqe *sec_sqe)
1541 {
1542 	struct sec_aead_req *a_req = &req->aead_req;
1543 	struct aead_request *aq = a_req->aead_req;
1544 
1545 	/* C_ICV_Len is MAC size, 0x4 ~ 0x10 */
1546 	sec_sqe->type2.icvw_kmode |= cpu_to_le16((u16)ctx->mac_len);
1547 
1548 	/* mode set to CCM/GCM, don't set {A_Alg, AKey_Len, MAC_Len} */
1549 	sec_sqe->type2.a_key_addr = sec_sqe->type2.c_key_addr;
1550 	sec_sqe->type2.a_ivin_addr = cpu_to_le64(a_req->a_ivin_dma);
1551 	sec_sqe->type_cipher_auth |= SEC_NO_AUTH << SEC_AUTH_OFFSET;
1552 
1553 	if (dir)
1554 		sec_sqe->sds_sa_type &= SEC_CIPHER_AUTH;
1555 	else
1556 		sec_sqe->sds_sa_type |= SEC_AUTH_CIPHER;
1557 
1558 	sec_sqe->type2.alen_ivllen = cpu_to_le32(aq->assoclen);
1559 	sec_sqe->type2.auth_src_offset = cpu_to_le16(0x0);
1560 	sec_sqe->type2.cipher_src_offset = cpu_to_le16((u16)aq->assoclen);
1561 
1562 	sec_sqe->type2.mac_addr = cpu_to_le64(a_req->out_mac_dma);
1563 }
1564 
1565 static void sec_auth_bd_fill_xcm_v3(struct sec_auth_ctx *ctx, int dir,
1566 				    struct sec_req *req, struct sec_sqe3 *sqe3)
1567 {
1568 	struct sec_aead_req *a_req = &req->aead_req;
1569 	struct aead_request *aq = a_req->aead_req;
1570 
1571 	/* C_ICV_Len is MAC size, 0x4 ~ 0x10 */
1572 	sqe3->c_icv_key |= cpu_to_le16((u16)ctx->mac_len << SEC_MAC_OFFSET_V3);
1573 
1574 	/* mode set to CCM/GCM, don't set {A_Alg, AKey_Len, MAC_Len} */
1575 	sqe3->a_key_addr = sqe3->c_key_addr;
1576 	sqe3->auth_ivin.a_ivin_addr = cpu_to_le64(a_req->a_ivin_dma);
1577 	sqe3->auth_mac_key |= SEC_NO_AUTH;
1578 
1579 	if (dir)
1580 		sqe3->huk_iv_seq &= SEC_CIPHER_AUTH_V3;
1581 	else
1582 		sqe3->huk_iv_seq |= SEC_AUTH_CIPHER_V3;
1583 
1584 	sqe3->a_len_key = cpu_to_le32(aq->assoclen);
1585 	sqe3->auth_src_offset = cpu_to_le16(0x0);
1586 	sqe3->cipher_src_offset = cpu_to_le16((u16)aq->assoclen);
1587 	sqe3->mac_addr = cpu_to_le64(a_req->out_mac_dma);
1588 }
1589 
1590 static void sec_auth_bd_fill_ex(struct sec_auth_ctx *ctx, int dir,
1591 			       struct sec_req *req, struct sec_sqe *sec_sqe)
1592 {
1593 	struct sec_aead_req *a_req = &req->aead_req;
1594 	struct sec_cipher_req *c_req = &req->c_req;
1595 	struct aead_request *aq = a_req->aead_req;
1596 
1597 	sec_sqe->type2.a_key_addr = cpu_to_le64(ctx->a_key_dma);
1598 
1599 	sec_sqe->type2.mac_key_alg =
1600 			cpu_to_le32(ctx->mac_len / SEC_SQE_LEN_RATE);
1601 
1602 	sec_sqe->type2.mac_key_alg |=
1603 			cpu_to_le32((u32)((ctx->a_key_len) /
1604 			SEC_SQE_LEN_RATE) << SEC_AKEY_OFFSET);
1605 
1606 	sec_sqe->type2.mac_key_alg |=
1607 			cpu_to_le32((u32)(ctx->a_alg) << SEC_AEAD_ALG_OFFSET);
1608 
1609 	if (dir) {
1610 		sec_sqe->type_cipher_auth |= SEC_AUTH_TYPE1 << SEC_AUTH_OFFSET;
1611 		sec_sqe->sds_sa_type &= SEC_CIPHER_AUTH;
1612 	} else {
1613 		sec_sqe->type_cipher_auth |= SEC_AUTH_TYPE2 << SEC_AUTH_OFFSET;
1614 		sec_sqe->sds_sa_type |= SEC_AUTH_CIPHER;
1615 	}
1616 	sec_sqe->type2.alen_ivllen = cpu_to_le32(c_req->c_len + aq->assoclen);
1617 
1618 	sec_sqe->type2.cipher_src_offset = cpu_to_le16((u16)aq->assoclen);
1619 
1620 	sec_sqe->type2.mac_addr = cpu_to_le64(a_req->out_mac_dma);
1621 }
1622 
1623 static int sec_aead_bd_fill(struct sec_ctx *ctx, struct sec_req *req)
1624 {
1625 	struct sec_auth_ctx *auth_ctx = &ctx->a_ctx;
1626 	struct sec_sqe *sec_sqe = &req->sec_sqe;
1627 	int ret;
1628 
1629 	ret = sec_skcipher_bd_fill(ctx, req);
1630 	if (unlikely(ret)) {
1631 		dev_err(ctx->dev, "skcipher bd fill is error!\n");
1632 		return ret;
1633 	}
1634 
1635 	if (ctx->c_ctx.c_mode == SEC_CMODE_CCM ||
1636 	    ctx->c_ctx.c_mode == SEC_CMODE_GCM)
1637 		sec_auth_bd_fill_xcm(auth_ctx, req->c_req.encrypt, req, sec_sqe);
1638 	else
1639 		sec_auth_bd_fill_ex(auth_ctx, req->c_req.encrypt, req, sec_sqe);
1640 
1641 	return 0;
1642 }
1643 
1644 static void sec_auth_bd_fill_ex_v3(struct sec_auth_ctx *ctx, int dir,
1645 				   struct sec_req *req, struct sec_sqe3 *sqe3)
1646 {
1647 	struct sec_aead_req *a_req = &req->aead_req;
1648 	struct sec_cipher_req *c_req = &req->c_req;
1649 	struct aead_request *aq = a_req->aead_req;
1650 
1651 	sqe3->a_key_addr = cpu_to_le64(ctx->a_key_dma);
1652 
1653 	sqe3->auth_mac_key |=
1654 			cpu_to_le32((u32)(ctx->mac_len /
1655 			SEC_SQE_LEN_RATE) << SEC_MAC_OFFSET_V3);
1656 
1657 	sqe3->auth_mac_key |=
1658 			cpu_to_le32((u32)(ctx->a_key_len /
1659 			SEC_SQE_LEN_RATE) << SEC_AKEY_OFFSET_V3);
1660 
1661 	sqe3->auth_mac_key |=
1662 			cpu_to_le32((u32)(ctx->a_alg) << SEC_AUTH_ALG_OFFSET_V3);
1663 
1664 	if (dir) {
1665 		sqe3->auth_mac_key |= cpu_to_le32((u32)SEC_AUTH_TYPE1);
1666 		sqe3->huk_iv_seq &= SEC_CIPHER_AUTH_V3;
1667 	} else {
1668 		sqe3->auth_mac_key |= cpu_to_le32((u32)SEC_AUTH_TYPE2);
1669 		sqe3->huk_iv_seq |= SEC_AUTH_CIPHER_V3;
1670 	}
1671 	sqe3->a_len_key = cpu_to_le32(c_req->c_len + aq->assoclen);
1672 
1673 	sqe3->cipher_src_offset = cpu_to_le16((u16)aq->assoclen);
1674 
1675 	sqe3->mac_addr = cpu_to_le64(a_req->out_mac_dma);
1676 }
1677 
1678 static int sec_aead_bd_fill_v3(struct sec_ctx *ctx, struct sec_req *req)
1679 {
1680 	struct sec_auth_ctx *auth_ctx = &ctx->a_ctx;
1681 	struct sec_sqe3 *sec_sqe3 = &req->sec_sqe3;
1682 	int ret;
1683 
1684 	ret = sec_skcipher_bd_fill_v3(ctx, req);
1685 	if (unlikely(ret)) {
1686 		dev_err(ctx->dev, "skcipher bd3 fill is error!\n");
1687 		return ret;
1688 	}
1689 
1690 	if (ctx->c_ctx.c_mode == SEC_CMODE_CCM ||
1691 	    ctx->c_ctx.c_mode == SEC_CMODE_GCM)
1692 		sec_auth_bd_fill_xcm_v3(auth_ctx, req->c_req.encrypt,
1693 					req, sec_sqe3);
1694 	else
1695 		sec_auth_bd_fill_ex_v3(auth_ctx, req->c_req.encrypt,
1696 				       req, sec_sqe3);
1697 
1698 	return 0;
1699 }
1700 
1701 static void sec_aead_callback(struct sec_ctx *c, struct sec_req *req, int err)
1702 {
1703 	struct aead_request *a_req = req->aead_req.aead_req;
1704 	struct crypto_aead *tfm = crypto_aead_reqtfm(a_req);
1705 	struct sec_aead_req *aead_req = &req->aead_req;
1706 	struct sec_cipher_req *c_req = &req->c_req;
1707 	size_t authsize = crypto_aead_authsize(tfm);
1708 	struct sec_qp_ctx *qp_ctx = req->qp_ctx;
1709 	struct aead_request *backlog_aead_req;
1710 	struct sec_req *backlog_req;
1711 	size_t sz;
1712 
1713 	if (!err && c->c_ctx.c_mode == SEC_CMODE_CBC && c_req->encrypt)
1714 		sec_update_iv(req, SEC_AEAD);
1715 
1716 	/* Copy output mac */
1717 	if (!err && c_req->encrypt) {
1718 		struct scatterlist *sgl = a_req->dst;
1719 
1720 		sz = sg_pcopy_from_buffer(sgl, sg_nents(sgl),
1721 					  aead_req->out_mac,
1722 					  authsize, a_req->cryptlen +
1723 					  a_req->assoclen);
1724 		if (unlikely(sz != authsize)) {
1725 			dev_err(c->dev, "copy out mac err!\n");
1726 			err = -EINVAL;
1727 		}
1728 	}
1729 
1730 	sec_free_req_id(req);
1731 
1732 	while (1) {
1733 		backlog_req = sec_back_req_clear(c, qp_ctx);
1734 		if (!backlog_req)
1735 			break;
1736 
1737 		backlog_aead_req = backlog_req->aead_req.aead_req;
1738 		aead_request_complete(backlog_aead_req, -EINPROGRESS);
1739 		atomic64_inc(&c->sec->debug.dfx.recv_busy_cnt);
1740 	}
1741 
1742 	aead_request_complete(a_req, err);
1743 }
1744 
1745 static void sec_request_uninit(struct sec_ctx *ctx, struct sec_req *req)
1746 {
1747 	sec_free_req_id(req);
1748 	sec_free_queue_id(ctx, req);
1749 }
1750 
1751 static int sec_request_init(struct sec_ctx *ctx, struct sec_req *req)
1752 {
1753 	struct sec_qp_ctx *qp_ctx;
1754 	int queue_id;
1755 
1756 	/* To load balance */
1757 	queue_id = sec_alloc_queue_id(ctx, req);
1758 	qp_ctx = &ctx->qp_ctx[queue_id];
1759 
1760 	req->req_id = sec_alloc_req_id(req, qp_ctx);
1761 	if (unlikely(req->req_id < 0)) {
1762 		sec_free_queue_id(ctx, req);
1763 		return req->req_id;
1764 	}
1765 
1766 	return 0;
1767 }
1768 
1769 static int sec_process(struct sec_ctx *ctx, struct sec_req *req)
1770 {
1771 	struct sec_cipher_req *c_req = &req->c_req;
1772 	int ret;
1773 
1774 	ret = sec_request_init(ctx, req);
1775 	if (unlikely(ret))
1776 		return ret;
1777 
1778 	ret = sec_request_transfer(ctx, req);
1779 	if (unlikely(ret))
1780 		goto err_uninit_req;
1781 
1782 	/* Output IV as decrypto */
1783 	if (!req->c_req.encrypt && (ctx->c_ctx.c_mode == SEC_CMODE_CBC ||
1784 	    ctx->c_ctx.c_mode == SEC_CMODE_CTR))
1785 		sec_update_iv(req, ctx->alg_type);
1786 
1787 	ret = ctx->req_op->bd_send(ctx, req);
1788 	if (unlikely((ret != -EBUSY && ret != -EINPROGRESS) ||
1789 		(ret == -EBUSY && !(req->flag & CRYPTO_TFM_REQ_MAY_BACKLOG)))) {
1790 		dev_err_ratelimited(ctx->dev, "send sec request failed!\n");
1791 		goto err_send_req;
1792 	}
1793 
1794 	return ret;
1795 
1796 err_send_req:
1797 	/* As failing, restore the IV from user */
1798 	if (ctx->c_ctx.c_mode == SEC_CMODE_CBC && !req->c_req.encrypt) {
1799 		if (ctx->alg_type == SEC_SKCIPHER)
1800 			memcpy(req->c_req.sk_req->iv, c_req->c_ivin,
1801 			       ctx->c_ctx.ivsize);
1802 		else
1803 			memcpy(req->aead_req.aead_req->iv, c_req->c_ivin,
1804 			       ctx->c_ctx.ivsize);
1805 	}
1806 
1807 	sec_request_untransfer(ctx, req);
1808 err_uninit_req:
1809 	sec_request_uninit(ctx, req);
1810 	return ret;
1811 }
1812 
1813 static const struct sec_req_op sec_skcipher_req_ops = {
1814 	.buf_map	= sec_skcipher_sgl_map,
1815 	.buf_unmap	= sec_skcipher_sgl_unmap,
1816 	.do_transfer	= sec_skcipher_copy_iv,
1817 	.bd_fill	= sec_skcipher_bd_fill,
1818 	.bd_send	= sec_bd_send,
1819 	.callback	= sec_skcipher_callback,
1820 	.process	= sec_process,
1821 };
1822 
1823 static const struct sec_req_op sec_aead_req_ops = {
1824 	.buf_map	= sec_aead_sgl_map,
1825 	.buf_unmap	= sec_aead_sgl_unmap,
1826 	.do_transfer	= sec_aead_set_iv,
1827 	.bd_fill	= sec_aead_bd_fill,
1828 	.bd_send	= sec_bd_send,
1829 	.callback	= sec_aead_callback,
1830 	.process	= sec_process,
1831 };
1832 
1833 static const struct sec_req_op sec_skcipher_req_ops_v3 = {
1834 	.buf_map	= sec_skcipher_sgl_map,
1835 	.buf_unmap	= sec_skcipher_sgl_unmap,
1836 	.do_transfer	= sec_skcipher_copy_iv,
1837 	.bd_fill	= sec_skcipher_bd_fill_v3,
1838 	.bd_send	= sec_bd_send,
1839 	.callback	= sec_skcipher_callback,
1840 	.process	= sec_process,
1841 };
1842 
1843 static const struct sec_req_op sec_aead_req_ops_v3 = {
1844 	.buf_map	= sec_aead_sgl_map,
1845 	.buf_unmap	= sec_aead_sgl_unmap,
1846 	.do_transfer	= sec_aead_set_iv,
1847 	.bd_fill	= sec_aead_bd_fill_v3,
1848 	.bd_send	= sec_bd_send,
1849 	.callback	= sec_aead_callback,
1850 	.process	= sec_process,
1851 };
1852 
1853 static int sec_skcipher_ctx_init(struct crypto_skcipher *tfm)
1854 {
1855 	struct sec_ctx *ctx = crypto_skcipher_ctx(tfm);
1856 	int ret;
1857 
1858 	ret = sec_skcipher_init(tfm);
1859 	if (ret)
1860 		return ret;
1861 
1862 	if (ctx->sec->qm.ver < QM_HW_V3) {
1863 		ctx->type_supported = SEC_BD_TYPE2;
1864 		ctx->req_op = &sec_skcipher_req_ops;
1865 	} else {
1866 		ctx->type_supported = SEC_BD_TYPE3;
1867 		ctx->req_op = &sec_skcipher_req_ops_v3;
1868 	}
1869 
1870 	return ret;
1871 }
1872 
1873 static void sec_skcipher_ctx_exit(struct crypto_skcipher *tfm)
1874 {
1875 	sec_skcipher_uninit(tfm);
1876 }
1877 
1878 static int sec_aead_init(struct crypto_aead *tfm)
1879 {
1880 	struct sec_ctx *ctx = crypto_aead_ctx(tfm);
1881 	int ret;
1882 
1883 	crypto_aead_set_reqsize(tfm, sizeof(struct sec_req));
1884 	ctx->alg_type = SEC_AEAD;
1885 	ctx->c_ctx.ivsize = crypto_aead_ivsize(tfm);
1886 	if (ctx->c_ctx.ivsize < SEC_AIV_SIZE ||
1887 	    ctx->c_ctx.ivsize > SEC_IV_SIZE) {
1888 		pr_err("get error aead iv size!\n");
1889 		return -EINVAL;
1890 	}
1891 
1892 	ret = sec_ctx_base_init(ctx);
1893 	if (ret)
1894 		return ret;
1895 	if (ctx->sec->qm.ver < QM_HW_V3) {
1896 		ctx->type_supported = SEC_BD_TYPE2;
1897 		ctx->req_op = &sec_aead_req_ops;
1898 	} else {
1899 		ctx->type_supported = SEC_BD_TYPE3;
1900 		ctx->req_op = &sec_aead_req_ops_v3;
1901 	}
1902 
1903 	ret = sec_auth_init(ctx);
1904 	if (ret)
1905 		goto err_auth_init;
1906 
1907 	ret = sec_cipher_init(ctx);
1908 	if (ret)
1909 		goto err_cipher_init;
1910 
1911 	return ret;
1912 
1913 err_cipher_init:
1914 	sec_auth_uninit(ctx);
1915 err_auth_init:
1916 	sec_ctx_base_uninit(ctx);
1917 	return ret;
1918 }
1919 
1920 static void sec_aead_exit(struct crypto_aead *tfm)
1921 {
1922 	struct sec_ctx *ctx = crypto_aead_ctx(tfm);
1923 
1924 	sec_cipher_uninit(ctx);
1925 	sec_auth_uninit(ctx);
1926 	sec_ctx_base_uninit(ctx);
1927 }
1928 
1929 static int sec_aead_ctx_init(struct crypto_aead *tfm, const char *hash_name)
1930 {
1931 	struct sec_ctx *ctx = crypto_aead_ctx(tfm);
1932 	struct sec_auth_ctx *auth_ctx = &ctx->a_ctx;
1933 	int ret;
1934 
1935 	ret = sec_aead_init(tfm);
1936 	if (ret) {
1937 		pr_err("hisi_sec2: aead init error!\n");
1938 		return ret;
1939 	}
1940 
1941 	auth_ctx->hash_tfm = crypto_alloc_shash(hash_name, 0, 0);
1942 	if (IS_ERR(auth_ctx->hash_tfm)) {
1943 		dev_err(ctx->dev, "aead alloc shash error!\n");
1944 		sec_aead_exit(tfm);
1945 		return PTR_ERR(auth_ctx->hash_tfm);
1946 	}
1947 
1948 	return 0;
1949 }
1950 
1951 static void sec_aead_ctx_exit(struct crypto_aead *tfm)
1952 {
1953 	struct sec_ctx *ctx = crypto_aead_ctx(tfm);
1954 
1955 	crypto_free_shash(ctx->a_ctx.hash_tfm);
1956 	sec_aead_exit(tfm);
1957 }
1958 
1959 static int sec_aead_xcm_ctx_init(struct crypto_aead *tfm)
1960 {
1961 	struct aead_alg *alg = crypto_aead_alg(tfm);
1962 	struct sec_ctx *ctx = crypto_aead_ctx(tfm);
1963 	struct sec_auth_ctx *a_ctx = &ctx->a_ctx;
1964 	const char *aead_name = alg->base.cra_name;
1965 	int ret;
1966 
1967 	ret = sec_aead_init(tfm);
1968 	if (ret) {
1969 		dev_err(ctx->dev, "hisi_sec2: aead xcm init error!\n");
1970 		return ret;
1971 	}
1972 
1973 	a_ctx->fallback_aead_tfm = crypto_alloc_aead(aead_name, 0,
1974 						     CRYPTO_ALG_NEED_FALLBACK |
1975 						     CRYPTO_ALG_ASYNC);
1976 	if (IS_ERR(a_ctx->fallback_aead_tfm)) {
1977 		dev_err(ctx->dev, "aead driver alloc fallback tfm error!\n");
1978 		sec_aead_exit(tfm);
1979 		return PTR_ERR(a_ctx->fallback_aead_tfm);
1980 	}
1981 	a_ctx->fallback = false;
1982 
1983 	return 0;
1984 }
1985 
1986 static void sec_aead_xcm_ctx_exit(struct crypto_aead *tfm)
1987 {
1988 	struct sec_ctx *ctx = crypto_aead_ctx(tfm);
1989 
1990 	crypto_free_aead(ctx->a_ctx.fallback_aead_tfm);
1991 	sec_aead_exit(tfm);
1992 }
1993 
1994 static int sec_aead_sha1_ctx_init(struct crypto_aead *tfm)
1995 {
1996 	return sec_aead_ctx_init(tfm, "sha1");
1997 }
1998 
1999 static int sec_aead_sha256_ctx_init(struct crypto_aead *tfm)
2000 {
2001 	return sec_aead_ctx_init(tfm, "sha256");
2002 }
2003 
2004 static int sec_aead_sha512_ctx_init(struct crypto_aead *tfm)
2005 {
2006 	return sec_aead_ctx_init(tfm, "sha512");
2007 }
2008 
2009 static int sec_skcipher_cryptlen_check(struct sec_ctx *ctx,
2010 	struct sec_req *sreq)
2011 {
2012 	u32 cryptlen = sreq->c_req.sk_req->cryptlen;
2013 	struct device *dev = ctx->dev;
2014 	u8 c_mode = ctx->c_ctx.c_mode;
2015 	int ret = 0;
2016 
2017 	switch (c_mode) {
2018 	case SEC_CMODE_XTS:
2019 		if (unlikely(cryptlen < AES_BLOCK_SIZE)) {
2020 			dev_err(dev, "skcipher XTS mode input length error!\n");
2021 			ret = -EINVAL;
2022 		}
2023 		break;
2024 	case SEC_CMODE_ECB:
2025 	case SEC_CMODE_CBC:
2026 		if (unlikely(cryptlen & (AES_BLOCK_SIZE - 1))) {
2027 			dev_err(dev, "skcipher AES input length error!\n");
2028 			ret = -EINVAL;
2029 		}
2030 		break;
2031 	case SEC_CMODE_CFB:
2032 	case SEC_CMODE_OFB:
2033 	case SEC_CMODE_CTR:
2034 		if (unlikely(ctx->sec->qm.ver < QM_HW_V3)) {
2035 			dev_err(dev, "skcipher HW version error!\n");
2036 			ret = -EINVAL;
2037 		}
2038 		break;
2039 	default:
2040 		ret = -EINVAL;
2041 	}
2042 
2043 	return ret;
2044 }
2045 
2046 static int sec_skcipher_param_check(struct sec_ctx *ctx, struct sec_req *sreq)
2047 {
2048 	struct skcipher_request *sk_req = sreq->c_req.sk_req;
2049 	struct device *dev = ctx->dev;
2050 	u8 c_alg = ctx->c_ctx.c_alg;
2051 
2052 	if (unlikely(!sk_req->src || !sk_req->dst ||
2053 		     sk_req->cryptlen > MAX_INPUT_DATA_LEN)) {
2054 		dev_err(dev, "skcipher input param error!\n");
2055 		return -EINVAL;
2056 	}
2057 	sreq->c_req.c_len = sk_req->cryptlen;
2058 
2059 	if (ctx->pbuf_supported && sk_req->cryptlen <= SEC_PBUF_SZ)
2060 		sreq->use_pbuf = true;
2061 	else
2062 		sreq->use_pbuf = false;
2063 
2064 	if (c_alg == SEC_CALG_3DES) {
2065 		if (unlikely(sk_req->cryptlen & (DES3_EDE_BLOCK_SIZE - 1))) {
2066 			dev_err(dev, "skcipher 3des input length error!\n");
2067 			return -EINVAL;
2068 		}
2069 		return 0;
2070 	} else if (c_alg == SEC_CALG_AES || c_alg == SEC_CALG_SM4) {
2071 		return sec_skcipher_cryptlen_check(ctx, sreq);
2072 	}
2073 
2074 	dev_err(dev, "skcipher algorithm error!\n");
2075 
2076 	return -EINVAL;
2077 }
2078 
2079 static int sec_skcipher_soft_crypto(struct sec_ctx *ctx,
2080 				    struct skcipher_request *sreq, bool encrypt)
2081 {
2082 	struct sec_cipher_ctx *c_ctx = &ctx->c_ctx;
2083 	SYNC_SKCIPHER_REQUEST_ON_STACK(subreq, c_ctx->fbtfm);
2084 	struct device *dev = ctx->dev;
2085 	int ret;
2086 
2087 	if (!c_ctx->fbtfm) {
2088 		dev_err_ratelimited(dev, "the soft tfm isn't supported in the current system.\n");
2089 		return -EINVAL;
2090 	}
2091 
2092 	skcipher_request_set_sync_tfm(subreq, c_ctx->fbtfm);
2093 
2094 	/* software need sync mode to do crypto */
2095 	skcipher_request_set_callback(subreq, sreq->base.flags,
2096 				      NULL, NULL);
2097 	skcipher_request_set_crypt(subreq, sreq->src, sreq->dst,
2098 				   sreq->cryptlen, sreq->iv);
2099 	if (encrypt)
2100 		ret = crypto_skcipher_encrypt(subreq);
2101 	else
2102 		ret = crypto_skcipher_decrypt(subreq);
2103 
2104 	skcipher_request_zero(subreq);
2105 
2106 	return ret;
2107 }
2108 
2109 static int sec_skcipher_crypto(struct skcipher_request *sk_req, bool encrypt)
2110 {
2111 	struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(sk_req);
2112 	struct sec_req *req = skcipher_request_ctx(sk_req);
2113 	struct sec_ctx *ctx = crypto_skcipher_ctx(tfm);
2114 	int ret;
2115 
2116 	if (!sk_req->cryptlen) {
2117 		if (ctx->c_ctx.c_mode == SEC_CMODE_XTS)
2118 			return -EINVAL;
2119 		return 0;
2120 	}
2121 
2122 	req->flag = sk_req->base.flags;
2123 	req->c_req.sk_req = sk_req;
2124 	req->c_req.encrypt = encrypt;
2125 	req->ctx = ctx;
2126 
2127 	ret = sec_skcipher_param_check(ctx, req);
2128 	if (unlikely(ret))
2129 		return -EINVAL;
2130 
2131 	if (unlikely(ctx->c_ctx.fallback))
2132 		return sec_skcipher_soft_crypto(ctx, sk_req, encrypt);
2133 
2134 	return ctx->req_op->process(ctx, req);
2135 }
2136 
2137 static int sec_skcipher_encrypt(struct skcipher_request *sk_req)
2138 {
2139 	return sec_skcipher_crypto(sk_req, true);
2140 }
2141 
2142 static int sec_skcipher_decrypt(struct skcipher_request *sk_req)
2143 {
2144 	return sec_skcipher_crypto(sk_req, false);
2145 }
2146 
2147 #define SEC_SKCIPHER_GEN_ALG(sec_cra_name, sec_set_key, sec_min_key_size, \
2148 	sec_max_key_size, ctx_init, ctx_exit, blk_size, iv_size)\
2149 {\
2150 	.base = {\
2151 		.cra_name = sec_cra_name,\
2152 		.cra_driver_name = "hisi_sec_"sec_cra_name,\
2153 		.cra_priority = SEC_PRIORITY,\
2154 		.cra_flags = CRYPTO_ALG_ASYNC |\
2155 		 CRYPTO_ALG_NEED_FALLBACK,\
2156 		.cra_blocksize = blk_size,\
2157 		.cra_ctxsize = sizeof(struct sec_ctx),\
2158 		.cra_module = THIS_MODULE,\
2159 	},\
2160 	.init = ctx_init,\
2161 	.exit = ctx_exit,\
2162 	.setkey = sec_set_key,\
2163 	.decrypt = sec_skcipher_decrypt,\
2164 	.encrypt = sec_skcipher_encrypt,\
2165 	.min_keysize = sec_min_key_size,\
2166 	.max_keysize = sec_max_key_size,\
2167 	.ivsize = iv_size,\
2168 }
2169 
2170 #define SEC_SKCIPHER_ALG(name, key_func, min_key_size, \
2171 	max_key_size, blk_size, iv_size) \
2172 	SEC_SKCIPHER_GEN_ALG(name, key_func, min_key_size, max_key_size, \
2173 	sec_skcipher_ctx_init, sec_skcipher_ctx_exit, blk_size, iv_size)
2174 
2175 static struct sec_skcipher sec_skciphers[] = {
2176 	{
2177 		.alg_msk = BIT(0),
2178 		.alg = SEC_SKCIPHER_ALG("ecb(aes)", sec_setkey_aes_ecb, AES_MIN_KEY_SIZE,
2179 					AES_MAX_KEY_SIZE, AES_BLOCK_SIZE, 0),
2180 	},
2181 	{
2182 		.alg_msk = BIT(1),
2183 		.alg = SEC_SKCIPHER_ALG("cbc(aes)", sec_setkey_aes_cbc, AES_MIN_KEY_SIZE,
2184 					AES_MAX_KEY_SIZE, AES_BLOCK_SIZE, AES_BLOCK_SIZE),
2185 	},
2186 	{
2187 		.alg_msk = BIT(2),
2188 		.alg = SEC_SKCIPHER_ALG("ctr(aes)", sec_setkey_aes_ctr,	AES_MIN_KEY_SIZE,
2189 					AES_MAX_KEY_SIZE, SEC_MIN_BLOCK_SZ, AES_BLOCK_SIZE),
2190 	},
2191 	{
2192 		.alg_msk = BIT(3),
2193 		.alg = SEC_SKCIPHER_ALG("xts(aes)", sec_setkey_aes_xts,	SEC_XTS_MIN_KEY_SIZE,
2194 					SEC_XTS_MAX_KEY_SIZE, AES_BLOCK_SIZE, AES_BLOCK_SIZE),
2195 	},
2196 	{
2197 		.alg_msk = BIT(4),
2198 		.alg = SEC_SKCIPHER_ALG("ofb(aes)", sec_setkey_aes_ofb,	AES_MIN_KEY_SIZE,
2199 					AES_MAX_KEY_SIZE, SEC_MIN_BLOCK_SZ, AES_BLOCK_SIZE),
2200 	},
2201 	{
2202 		.alg_msk = BIT(5),
2203 		.alg = SEC_SKCIPHER_ALG("cfb(aes)", sec_setkey_aes_cfb,	AES_MIN_KEY_SIZE,
2204 					AES_MAX_KEY_SIZE, SEC_MIN_BLOCK_SZ, AES_BLOCK_SIZE),
2205 	},
2206 	{
2207 		.alg_msk = BIT(12),
2208 		.alg = SEC_SKCIPHER_ALG("cbc(sm4)", sec_setkey_sm4_cbc,	AES_MIN_KEY_SIZE,
2209 					AES_MIN_KEY_SIZE, AES_BLOCK_SIZE, AES_BLOCK_SIZE),
2210 	},
2211 	{
2212 		.alg_msk = BIT(13),
2213 		.alg = SEC_SKCIPHER_ALG("ctr(sm4)", sec_setkey_sm4_ctr, AES_MIN_KEY_SIZE,
2214 					AES_MIN_KEY_SIZE, SEC_MIN_BLOCK_SZ, AES_BLOCK_SIZE),
2215 	},
2216 	{
2217 		.alg_msk = BIT(14),
2218 		.alg = SEC_SKCIPHER_ALG("xts(sm4)", sec_setkey_sm4_xts,	SEC_XTS_MIN_KEY_SIZE,
2219 					SEC_XTS_MIN_KEY_SIZE, AES_BLOCK_SIZE, AES_BLOCK_SIZE),
2220 	},
2221 	{
2222 		.alg_msk = BIT(15),
2223 		.alg = SEC_SKCIPHER_ALG("ofb(sm4)", sec_setkey_sm4_ofb,	AES_MIN_KEY_SIZE,
2224 					AES_MIN_KEY_SIZE, SEC_MIN_BLOCK_SZ, AES_BLOCK_SIZE),
2225 	},
2226 	{
2227 		.alg_msk = BIT(16),
2228 		.alg = SEC_SKCIPHER_ALG("cfb(sm4)", sec_setkey_sm4_cfb,	AES_MIN_KEY_SIZE,
2229 					AES_MIN_KEY_SIZE, SEC_MIN_BLOCK_SZ, AES_BLOCK_SIZE),
2230 	},
2231 	{
2232 		.alg_msk = BIT(23),
2233 		.alg = SEC_SKCIPHER_ALG("ecb(des3_ede)", sec_setkey_3des_ecb, SEC_DES3_3KEY_SIZE,
2234 					SEC_DES3_3KEY_SIZE, DES3_EDE_BLOCK_SIZE, 0),
2235 	},
2236 	{
2237 		.alg_msk = BIT(24),
2238 		.alg = SEC_SKCIPHER_ALG("cbc(des3_ede)", sec_setkey_3des_cbc, SEC_DES3_3KEY_SIZE,
2239 					SEC_DES3_3KEY_SIZE, DES3_EDE_BLOCK_SIZE,
2240 					DES3_EDE_BLOCK_SIZE),
2241 	},
2242 };
2243 
2244 static int aead_iv_demension_check(struct aead_request *aead_req)
2245 {
2246 	u8 cl;
2247 
2248 	cl = aead_req->iv[0] + 1;
2249 	if (cl < IV_CL_MIN || cl > IV_CL_MAX)
2250 		return -EINVAL;
2251 
2252 	if (cl < IV_CL_MID && aead_req->cryptlen >> (BYTE_BITS * cl))
2253 		return -EOVERFLOW;
2254 
2255 	return 0;
2256 }
2257 
2258 static int sec_aead_spec_check(struct sec_ctx *ctx, struct sec_req *sreq)
2259 {
2260 	struct aead_request *req = sreq->aead_req.aead_req;
2261 	struct crypto_aead *tfm = crypto_aead_reqtfm(req);
2262 	size_t authsize = crypto_aead_authsize(tfm);
2263 	u8 c_mode = ctx->c_ctx.c_mode;
2264 	struct device *dev = ctx->dev;
2265 	int ret;
2266 
2267 	if (unlikely(req->cryptlen + req->assoclen > MAX_INPUT_DATA_LEN ||
2268 	    req->assoclen > SEC_MAX_AAD_LEN)) {
2269 		dev_err(dev, "aead input spec error!\n");
2270 		return -EINVAL;
2271 	}
2272 
2273 	if (unlikely((c_mode == SEC_CMODE_GCM && authsize < DES_BLOCK_SIZE) ||
2274 	   (c_mode == SEC_CMODE_CCM && (authsize < MIN_MAC_LEN ||
2275 		authsize & MAC_LEN_MASK)))) {
2276 		dev_err(dev, "aead input mac length error!\n");
2277 		return -EINVAL;
2278 	}
2279 
2280 	if (c_mode == SEC_CMODE_CCM) {
2281 		if (unlikely(req->assoclen > SEC_MAX_CCM_AAD_LEN)) {
2282 			dev_err_ratelimited(dev, "CCM input aad parameter is too long!\n");
2283 			return -EINVAL;
2284 		}
2285 		ret = aead_iv_demension_check(req);
2286 		if (ret) {
2287 			dev_err(dev, "aead input iv param error!\n");
2288 			return ret;
2289 		}
2290 	}
2291 
2292 	if (sreq->c_req.encrypt)
2293 		sreq->c_req.c_len = req->cryptlen;
2294 	else
2295 		sreq->c_req.c_len = req->cryptlen - authsize;
2296 	if (c_mode == SEC_CMODE_CBC) {
2297 		if (unlikely(sreq->c_req.c_len & (AES_BLOCK_SIZE - 1))) {
2298 			dev_err(dev, "aead crypto length error!\n");
2299 			return -EINVAL;
2300 		}
2301 	}
2302 
2303 	return 0;
2304 }
2305 
2306 static int sec_aead_param_check(struct sec_ctx *ctx, struct sec_req *sreq)
2307 {
2308 	struct aead_request *req = sreq->aead_req.aead_req;
2309 	struct crypto_aead *tfm = crypto_aead_reqtfm(req);
2310 	size_t authsize = crypto_aead_authsize(tfm);
2311 	struct device *dev = ctx->dev;
2312 	u8 c_alg = ctx->c_ctx.c_alg;
2313 
2314 	if (unlikely(!req->src || !req->dst)) {
2315 		dev_err(dev, "aead input param error!\n");
2316 		return -EINVAL;
2317 	}
2318 
2319 	if (ctx->sec->qm.ver == QM_HW_V2) {
2320 		if (unlikely(!req->cryptlen || (!sreq->c_req.encrypt &&
2321 		    req->cryptlen <= authsize))) {
2322 			ctx->a_ctx.fallback = true;
2323 			return -EINVAL;
2324 		}
2325 	}
2326 
2327 	/* Support AES or SM4 */
2328 	if (unlikely(c_alg != SEC_CALG_AES && c_alg != SEC_CALG_SM4)) {
2329 		dev_err(dev, "aead crypto alg error!\n");
2330 		return -EINVAL;
2331 	}
2332 
2333 	if (unlikely(sec_aead_spec_check(ctx, sreq)))
2334 		return -EINVAL;
2335 
2336 	if (ctx->pbuf_supported && (req->cryptlen + req->assoclen) <=
2337 		SEC_PBUF_SZ)
2338 		sreq->use_pbuf = true;
2339 	else
2340 		sreq->use_pbuf = false;
2341 
2342 	return 0;
2343 }
2344 
2345 static int sec_aead_soft_crypto(struct sec_ctx *ctx,
2346 				struct aead_request *aead_req,
2347 				bool encrypt)
2348 {
2349 	struct sec_auth_ctx *a_ctx = &ctx->a_ctx;
2350 	struct device *dev = ctx->dev;
2351 	struct aead_request *subreq;
2352 	int ret;
2353 
2354 	/* Kunpeng920 aead mode not support input 0 size */
2355 	if (!a_ctx->fallback_aead_tfm) {
2356 		dev_err(dev, "aead fallback tfm is NULL!\n");
2357 		return -EINVAL;
2358 	}
2359 
2360 	subreq = aead_request_alloc(a_ctx->fallback_aead_tfm, GFP_KERNEL);
2361 	if (!subreq)
2362 		return -ENOMEM;
2363 
2364 	aead_request_set_tfm(subreq, a_ctx->fallback_aead_tfm);
2365 	aead_request_set_callback(subreq, aead_req->base.flags,
2366 				  aead_req->base.complete, aead_req->base.data);
2367 	aead_request_set_crypt(subreq, aead_req->src, aead_req->dst,
2368 			       aead_req->cryptlen, aead_req->iv);
2369 	aead_request_set_ad(subreq, aead_req->assoclen);
2370 
2371 	if (encrypt)
2372 		ret = crypto_aead_encrypt(subreq);
2373 	else
2374 		ret = crypto_aead_decrypt(subreq);
2375 	aead_request_free(subreq);
2376 
2377 	return ret;
2378 }
2379 
2380 static int sec_aead_crypto(struct aead_request *a_req, bool encrypt)
2381 {
2382 	struct crypto_aead *tfm = crypto_aead_reqtfm(a_req);
2383 	struct sec_req *req = aead_request_ctx(a_req);
2384 	struct sec_ctx *ctx = crypto_aead_ctx(tfm);
2385 	int ret;
2386 
2387 	req->flag = a_req->base.flags;
2388 	req->aead_req.aead_req = a_req;
2389 	req->c_req.encrypt = encrypt;
2390 	req->ctx = ctx;
2391 
2392 	ret = sec_aead_param_check(ctx, req);
2393 	if (unlikely(ret)) {
2394 		if (ctx->a_ctx.fallback)
2395 			return sec_aead_soft_crypto(ctx, a_req, encrypt);
2396 		return -EINVAL;
2397 	}
2398 
2399 	return ctx->req_op->process(ctx, req);
2400 }
2401 
2402 static int sec_aead_encrypt(struct aead_request *a_req)
2403 {
2404 	return sec_aead_crypto(a_req, true);
2405 }
2406 
2407 static int sec_aead_decrypt(struct aead_request *a_req)
2408 {
2409 	return sec_aead_crypto(a_req, false);
2410 }
2411 
2412 #define SEC_AEAD_ALG(sec_cra_name, sec_set_key, ctx_init,\
2413 			 ctx_exit, blk_size, iv_size, max_authsize)\
2414 {\
2415 	.base = {\
2416 		.cra_name = sec_cra_name,\
2417 		.cra_driver_name = "hisi_sec_"sec_cra_name,\
2418 		.cra_priority = SEC_PRIORITY,\
2419 		.cra_flags = CRYPTO_ALG_ASYNC |\
2420 		 CRYPTO_ALG_NEED_FALLBACK,\
2421 		.cra_blocksize = blk_size,\
2422 		.cra_ctxsize = sizeof(struct sec_ctx),\
2423 		.cra_module = THIS_MODULE,\
2424 	},\
2425 	.init = ctx_init,\
2426 	.exit = ctx_exit,\
2427 	.setkey = sec_set_key,\
2428 	.setauthsize = sec_aead_setauthsize,\
2429 	.decrypt = sec_aead_decrypt,\
2430 	.encrypt = sec_aead_encrypt,\
2431 	.ivsize = iv_size,\
2432 	.maxauthsize = max_authsize,\
2433 }
2434 
2435 static struct sec_aead sec_aeads[] = {
2436 	{
2437 		.alg_msk = BIT(6),
2438 		.alg = SEC_AEAD_ALG("ccm(aes)", sec_setkey_aes_ccm, sec_aead_xcm_ctx_init,
2439 				    sec_aead_xcm_ctx_exit, SEC_MIN_BLOCK_SZ, AES_BLOCK_SIZE,
2440 				    AES_BLOCK_SIZE),
2441 	},
2442 	{
2443 		.alg_msk = BIT(7),
2444 		.alg = SEC_AEAD_ALG("gcm(aes)", sec_setkey_aes_gcm, sec_aead_xcm_ctx_init,
2445 				    sec_aead_xcm_ctx_exit, SEC_MIN_BLOCK_SZ, SEC_AIV_SIZE,
2446 				    AES_BLOCK_SIZE),
2447 	},
2448 	{
2449 		.alg_msk = BIT(17),
2450 		.alg = SEC_AEAD_ALG("ccm(sm4)", sec_setkey_sm4_ccm, sec_aead_xcm_ctx_init,
2451 				    sec_aead_xcm_ctx_exit, SEC_MIN_BLOCK_SZ, AES_BLOCK_SIZE,
2452 				    AES_BLOCK_SIZE),
2453 	},
2454 	{
2455 		.alg_msk = BIT(18),
2456 		.alg = SEC_AEAD_ALG("gcm(sm4)", sec_setkey_sm4_gcm, sec_aead_xcm_ctx_init,
2457 				    sec_aead_xcm_ctx_exit, SEC_MIN_BLOCK_SZ, SEC_AIV_SIZE,
2458 				    AES_BLOCK_SIZE),
2459 	},
2460 	{
2461 		.alg_msk = BIT(43),
2462 		.alg = SEC_AEAD_ALG("authenc(hmac(sha1),cbc(aes))", sec_setkey_aes_cbc_sha1,
2463 				    sec_aead_sha1_ctx_init, sec_aead_ctx_exit, AES_BLOCK_SIZE,
2464 				    AES_BLOCK_SIZE, SHA1_DIGEST_SIZE),
2465 	},
2466 	{
2467 		.alg_msk = BIT(44),
2468 		.alg = SEC_AEAD_ALG("authenc(hmac(sha256),cbc(aes))", sec_setkey_aes_cbc_sha256,
2469 				    sec_aead_sha256_ctx_init, sec_aead_ctx_exit, AES_BLOCK_SIZE,
2470 				    AES_BLOCK_SIZE, SHA256_DIGEST_SIZE),
2471 	},
2472 	{
2473 		.alg_msk = BIT(45),
2474 		.alg = SEC_AEAD_ALG("authenc(hmac(sha512),cbc(aes))", sec_setkey_aes_cbc_sha512,
2475 				    sec_aead_sha512_ctx_init, sec_aead_ctx_exit, AES_BLOCK_SIZE,
2476 				    AES_BLOCK_SIZE, SHA512_DIGEST_SIZE),
2477 	},
2478 };
2479 
2480 static void sec_unregister_skcipher(u64 alg_mask, int end)
2481 {
2482 	int i;
2483 
2484 	for (i = 0; i < end; i++)
2485 		if (sec_skciphers[i].alg_msk & alg_mask)
2486 			crypto_unregister_skcipher(&sec_skciphers[i].alg);
2487 }
2488 
2489 static int sec_register_skcipher(u64 alg_mask)
2490 {
2491 	int i, ret, count;
2492 
2493 	count = ARRAY_SIZE(sec_skciphers);
2494 
2495 	for (i = 0; i < count; i++) {
2496 		if (!(sec_skciphers[i].alg_msk & alg_mask))
2497 			continue;
2498 
2499 		ret = crypto_register_skcipher(&sec_skciphers[i].alg);
2500 		if (ret)
2501 			goto err;
2502 	}
2503 
2504 	return 0;
2505 
2506 err:
2507 	sec_unregister_skcipher(alg_mask, i);
2508 
2509 	return ret;
2510 }
2511 
2512 static void sec_unregister_aead(u64 alg_mask, int end)
2513 {
2514 	int i;
2515 
2516 	for (i = 0; i < end; i++)
2517 		if (sec_aeads[i].alg_msk & alg_mask)
2518 			crypto_unregister_aead(&sec_aeads[i].alg);
2519 }
2520 
2521 static int sec_register_aead(u64 alg_mask)
2522 {
2523 	int i, ret, count;
2524 
2525 	count = ARRAY_SIZE(sec_aeads);
2526 
2527 	for (i = 0; i < count; i++) {
2528 		if (!(sec_aeads[i].alg_msk & alg_mask))
2529 			continue;
2530 
2531 		ret = crypto_register_aead(&sec_aeads[i].alg);
2532 		if (ret)
2533 			goto err;
2534 	}
2535 
2536 	return 0;
2537 
2538 err:
2539 	sec_unregister_aead(alg_mask, i);
2540 
2541 	return ret;
2542 }
2543 
2544 int sec_register_to_crypto(struct hisi_qm *qm)
2545 {
2546 	u64 alg_mask;
2547 	int ret = 0;
2548 
2549 	alg_mask = sec_get_alg_bitmap(qm, SEC_DRV_ALG_BITMAP_HIGH_IDX,
2550 				      SEC_DRV_ALG_BITMAP_LOW_IDX);
2551 
2552 
2553 	ret = sec_register_skcipher(alg_mask);
2554 	if (ret)
2555 		return ret;
2556 
2557 	ret = sec_register_aead(alg_mask);
2558 	if (ret)
2559 		sec_unregister_skcipher(alg_mask, ARRAY_SIZE(sec_skciphers));
2560 
2561 	return ret;
2562 }
2563 
2564 void sec_unregister_from_crypto(struct hisi_qm *qm)
2565 {
2566 	u64 alg_mask;
2567 
2568 	alg_mask = sec_get_alg_bitmap(qm, SEC_DRV_ALG_BITMAP_HIGH_IDX,
2569 				      SEC_DRV_ALG_BITMAP_LOW_IDX);
2570 
2571 	sec_unregister_aead(alg_mask, ARRAY_SIZE(sec_aeads));
2572 	sec_unregister_skcipher(alg_mask, ARRAY_SIZE(sec_skciphers));
2573 }
2574