1 // SPDX-License-Identifier: GPL-2.0
2 /* Copyright (c) 2019 HiSilicon Limited. */
3 
4 #include <crypto/aes.h>
5 #include <crypto/aead.h>
6 #include <crypto/algapi.h>
7 #include <crypto/authenc.h>
8 #include <crypto/des.h>
9 #include <crypto/hash.h>
10 #include <crypto/internal/aead.h>
11 #include <crypto/internal/des.h>
12 #include <crypto/sha1.h>
13 #include <crypto/sha2.h>
14 #include <crypto/skcipher.h>
15 #include <crypto/xts.h>
16 #include <linux/crypto.h>
17 #include <linux/dma-mapping.h>
18 #include <linux/idr.h>
19 
20 #include "sec.h"
21 #include "sec_crypto.h"
22 
23 #define SEC_PRIORITY		4001
24 #define SEC_XTS_MIN_KEY_SIZE	(2 * AES_MIN_KEY_SIZE)
25 #define SEC_XTS_MID_KEY_SIZE	(3 * AES_MIN_KEY_SIZE)
26 #define SEC_XTS_MAX_KEY_SIZE	(2 * AES_MAX_KEY_SIZE)
27 #define SEC_DES3_2KEY_SIZE	(2 * DES_KEY_SIZE)
28 #define SEC_DES3_3KEY_SIZE	(3 * DES_KEY_SIZE)
29 
30 /* SEC sqe(bd) bit operational relative MACRO */
31 #define SEC_DE_OFFSET		1
32 #define SEC_CIPHER_OFFSET	4
33 #define SEC_SCENE_OFFSET	3
34 #define SEC_DST_SGL_OFFSET	2
35 #define SEC_SRC_SGL_OFFSET	7
36 #define SEC_CKEY_OFFSET		9
37 #define SEC_CMODE_OFFSET	12
38 #define SEC_AKEY_OFFSET         5
39 #define SEC_AEAD_ALG_OFFSET     11
40 #define SEC_AUTH_OFFSET		6
41 
42 #define SEC_DE_OFFSET_V3		9
43 #define SEC_SCENE_OFFSET_V3	5
44 #define SEC_CKEY_OFFSET_V3	13
45 #define SEC_SRC_SGL_OFFSET_V3	11
46 #define SEC_DST_SGL_OFFSET_V3	14
47 #define SEC_CALG_OFFSET_V3	4
48 #define SEC_AKEY_OFFSET_V3	9
49 #define SEC_MAC_OFFSET_V3	4
50 #define SEC_AUTH_ALG_OFFSET_V3	15
51 #define SEC_CIPHER_AUTH_V3	0xbf
52 #define SEC_AUTH_CIPHER_V3	0x40
53 #define SEC_FLAG_OFFSET		7
54 #define SEC_FLAG_MASK		0x0780
55 #define SEC_TYPE_MASK		0x0F
56 #define SEC_DONE_MASK		0x0001
57 #define SEC_ICV_MASK		0x000E
58 #define SEC_SQE_LEN_RATE_MASK	0x3
59 
60 #define SEC_TOTAL_IV_SZ		(SEC_IV_SIZE * QM_Q_DEPTH)
61 #define SEC_SGL_SGE_NR		128
62 #define SEC_CIPHER_AUTH		0xfe
63 #define SEC_AUTH_CIPHER		0x1
64 #define SEC_MAX_MAC_LEN		64
65 #define SEC_MAX_AAD_LEN		65535
66 #define SEC_TOTAL_MAC_SZ	(SEC_MAX_MAC_LEN * QM_Q_DEPTH)
67 
68 #define SEC_PBUF_SZ			512
69 #define SEC_PBUF_IV_OFFSET		SEC_PBUF_SZ
70 #define SEC_PBUF_MAC_OFFSET		(SEC_PBUF_SZ + SEC_IV_SIZE)
71 #define SEC_PBUF_PKG		(SEC_PBUF_SZ + SEC_IV_SIZE +	\
72 			SEC_MAX_MAC_LEN * 2)
73 #define SEC_PBUF_NUM		(PAGE_SIZE / SEC_PBUF_PKG)
74 #define SEC_PBUF_PAGE_NUM	(QM_Q_DEPTH / SEC_PBUF_NUM)
75 #define SEC_PBUF_LEFT_SZ	(SEC_PBUF_PKG * (QM_Q_DEPTH -	\
76 			SEC_PBUF_PAGE_NUM * SEC_PBUF_NUM))
77 #define SEC_TOTAL_PBUF_SZ	(PAGE_SIZE * SEC_PBUF_PAGE_NUM +	\
78 			SEC_PBUF_LEFT_SZ)
79 
80 #define SEC_SQE_LEN_RATE	4
81 #define SEC_SQE_CFLAG		2
82 #define SEC_SQE_AEAD_FLAG	3
83 #define SEC_SQE_DONE		0x1
84 #define SEC_ICV_ERR		0x2
85 #define MIN_MAC_LEN		4
86 #define MAC_LEN_MASK		0x1U
87 #define MAX_INPUT_DATA_LEN	0xFFFE00
88 #define BITS_MASK		0xFF
89 #define BYTE_BITS		0x8
90 #define SEC_XTS_NAME_SZ		0x3
91 #define IV_CM_CAL_NUM		2
92 #define IV_CL_MASK		0x7
93 #define IV_CL_MIN		2
94 #define IV_CL_MID		4
95 #define IV_CL_MAX		8
96 #define IV_FLAGS_OFFSET	0x6
97 #define IV_CM_OFFSET		0x3
98 #define IV_LAST_BYTE1		1
99 #define IV_LAST_BYTE2		2
100 #define IV_LAST_BYTE_MASK	0xFF
101 #define IV_CTR_INIT		0x1
102 #define IV_BYTE_OFFSET		0x8
103 
104 /* Get an en/de-cipher queue cyclically to balance load over queues of TFM */
105 static inline int sec_alloc_queue_id(struct sec_ctx *ctx, struct sec_req *req)
106 {
107 	if (req->c_req.encrypt)
108 		return (u32)atomic_inc_return(&ctx->enc_qcyclic) %
109 				 ctx->hlf_q_num;
110 
111 	return (u32)atomic_inc_return(&ctx->dec_qcyclic) % ctx->hlf_q_num +
112 				 ctx->hlf_q_num;
113 }
114 
115 static inline void sec_free_queue_id(struct sec_ctx *ctx, struct sec_req *req)
116 {
117 	if (req->c_req.encrypt)
118 		atomic_dec(&ctx->enc_qcyclic);
119 	else
120 		atomic_dec(&ctx->dec_qcyclic);
121 }
122 
123 static int sec_alloc_req_id(struct sec_req *req, struct sec_qp_ctx *qp_ctx)
124 {
125 	int req_id;
126 
127 	mutex_lock(&qp_ctx->req_lock);
128 
129 	req_id = idr_alloc_cyclic(&qp_ctx->req_idr, NULL,
130 				  0, QM_Q_DEPTH, GFP_ATOMIC);
131 	mutex_unlock(&qp_ctx->req_lock);
132 	if (unlikely(req_id < 0)) {
133 		dev_err(req->ctx->dev, "alloc req id fail!\n");
134 		return req_id;
135 	}
136 
137 	req->qp_ctx = qp_ctx;
138 	qp_ctx->req_list[req_id] = req;
139 
140 	return req_id;
141 }
142 
143 static void sec_free_req_id(struct sec_req *req)
144 {
145 	struct sec_qp_ctx *qp_ctx = req->qp_ctx;
146 	int req_id = req->req_id;
147 
148 	if (unlikely(req_id < 0 || req_id >= QM_Q_DEPTH)) {
149 		dev_err(req->ctx->dev, "free request id invalid!\n");
150 		return;
151 	}
152 
153 	qp_ctx->req_list[req_id] = NULL;
154 	req->qp_ctx = NULL;
155 
156 	mutex_lock(&qp_ctx->req_lock);
157 	idr_remove(&qp_ctx->req_idr, req_id);
158 	mutex_unlock(&qp_ctx->req_lock);
159 }
160 
161 static u8 pre_parse_finished_bd(struct bd_status *status, void *resp)
162 {
163 	struct sec_sqe *bd = resp;
164 
165 	status->done = le16_to_cpu(bd->type2.done_flag) & SEC_DONE_MASK;
166 	status->icv = (le16_to_cpu(bd->type2.done_flag) & SEC_ICV_MASK) >> 1;
167 	status->flag = (le16_to_cpu(bd->type2.done_flag) &
168 					SEC_FLAG_MASK) >> SEC_FLAG_OFFSET;
169 	status->tag = le16_to_cpu(bd->type2.tag);
170 	status->err_type = bd->type2.error_type;
171 
172 	return bd->type_cipher_auth & SEC_TYPE_MASK;
173 }
174 
175 static u8 pre_parse_finished_bd3(struct bd_status *status, void *resp)
176 {
177 	struct sec_sqe3 *bd3 = resp;
178 
179 	status->done = le16_to_cpu(bd3->done_flag) & SEC_DONE_MASK;
180 	status->icv = (le16_to_cpu(bd3->done_flag) & SEC_ICV_MASK) >> 1;
181 	status->flag = (le16_to_cpu(bd3->done_flag) &
182 					SEC_FLAG_MASK) >> SEC_FLAG_OFFSET;
183 	status->tag = le64_to_cpu(bd3->tag);
184 	status->err_type = bd3->error_type;
185 
186 	return le32_to_cpu(bd3->bd_param) & SEC_TYPE_MASK;
187 }
188 
189 static int sec_cb_status_check(struct sec_req *req,
190 			       struct bd_status *status)
191 {
192 	struct sec_ctx *ctx = req->ctx;
193 
194 	if (unlikely(req->err_type || status->done != SEC_SQE_DONE)) {
195 		dev_err_ratelimited(ctx->dev, "err_type[%d], done[%u]\n",
196 				    req->err_type, status->done);
197 		return -EIO;
198 	}
199 
200 	if (unlikely(ctx->alg_type == SEC_SKCIPHER)) {
201 		if (unlikely(status->flag != SEC_SQE_CFLAG)) {
202 			dev_err_ratelimited(ctx->dev, "flag[%u]\n",
203 					    status->flag);
204 			return -EIO;
205 		}
206 	} else if (unlikely(ctx->alg_type == SEC_AEAD)) {
207 		if (unlikely(status->flag != SEC_SQE_AEAD_FLAG ||
208 			     status->icv == SEC_ICV_ERR)) {
209 			dev_err_ratelimited(ctx->dev,
210 					    "flag[%u], icv[%u]\n",
211 					    status->flag, status->icv);
212 			return -EBADMSG;
213 		}
214 	}
215 
216 	return 0;
217 }
218 
219 static void sec_req_cb(struct hisi_qp *qp, void *resp)
220 {
221 	struct sec_qp_ctx *qp_ctx = qp->qp_ctx;
222 	struct sec_dfx *dfx = &qp_ctx->ctx->sec->debug.dfx;
223 	u8 type_supported = qp_ctx->ctx->type_supported;
224 	struct bd_status status;
225 	struct sec_ctx *ctx;
226 	struct sec_req *req;
227 	int err;
228 	u8 type;
229 
230 	if (type_supported == SEC_BD_TYPE2) {
231 		type = pre_parse_finished_bd(&status, resp);
232 		req = qp_ctx->req_list[status.tag];
233 	} else {
234 		type = pre_parse_finished_bd3(&status, resp);
235 		req = (void *)(uintptr_t)status.tag;
236 	}
237 
238 	if (unlikely(type != type_supported)) {
239 		atomic64_inc(&dfx->err_bd_cnt);
240 		pr_err("err bd type [%d]\n", type);
241 		return;
242 	}
243 
244 	if (unlikely(!req)) {
245 		atomic64_inc(&dfx->invalid_req_cnt);
246 		atomic_inc(&qp->qp_status.used);
247 		return;
248 	}
249 
250 	req->err_type = status.err_type;
251 	ctx = req->ctx;
252 	err = sec_cb_status_check(req, &status);
253 	if (err)
254 		atomic64_inc(&dfx->done_flag_cnt);
255 
256 	atomic64_inc(&dfx->recv_cnt);
257 
258 	ctx->req_op->buf_unmap(ctx, req);
259 
260 	ctx->req_op->callback(ctx, req, err);
261 }
262 
263 static int sec_bd_send(struct sec_ctx *ctx, struct sec_req *req)
264 {
265 	struct sec_qp_ctx *qp_ctx = req->qp_ctx;
266 	int ret;
267 
268 	if (ctx->fake_req_limit <=
269 	    atomic_read(&qp_ctx->qp->qp_status.used) &&
270 	    !(req->flag & CRYPTO_TFM_REQ_MAY_BACKLOG))
271 		return -EBUSY;
272 
273 	mutex_lock(&qp_ctx->req_lock);
274 	ret = hisi_qp_send(qp_ctx->qp, &req->sec_sqe);
275 
276 	if (ctx->fake_req_limit <=
277 	    atomic_read(&qp_ctx->qp->qp_status.used) && !ret) {
278 		list_add_tail(&req->backlog_head, &qp_ctx->backlog);
279 		atomic64_inc(&ctx->sec->debug.dfx.send_cnt);
280 		atomic64_inc(&ctx->sec->debug.dfx.send_busy_cnt);
281 		mutex_unlock(&qp_ctx->req_lock);
282 		return -EBUSY;
283 	}
284 	mutex_unlock(&qp_ctx->req_lock);
285 
286 	if (unlikely(ret == -EBUSY))
287 		return -ENOBUFS;
288 
289 	if (likely(!ret)) {
290 		ret = -EINPROGRESS;
291 		atomic64_inc(&ctx->sec->debug.dfx.send_cnt);
292 	}
293 
294 	return ret;
295 }
296 
297 /* Get DMA memory resources */
298 static int sec_alloc_civ_resource(struct device *dev, struct sec_alg_res *res)
299 {
300 	int i;
301 
302 	res->c_ivin = dma_alloc_coherent(dev, SEC_TOTAL_IV_SZ,
303 					 &res->c_ivin_dma, GFP_KERNEL);
304 	if (!res->c_ivin)
305 		return -ENOMEM;
306 
307 	for (i = 1; i < QM_Q_DEPTH; i++) {
308 		res[i].c_ivin_dma = res->c_ivin_dma + i * SEC_IV_SIZE;
309 		res[i].c_ivin = res->c_ivin + i * SEC_IV_SIZE;
310 	}
311 
312 	return 0;
313 }
314 
315 static void sec_free_civ_resource(struct device *dev, struct sec_alg_res *res)
316 {
317 	if (res->c_ivin)
318 		dma_free_coherent(dev, SEC_TOTAL_IV_SZ,
319 				  res->c_ivin, res->c_ivin_dma);
320 }
321 
322 static int sec_alloc_aiv_resource(struct device *dev, struct sec_alg_res *res)
323 {
324 	int i;
325 
326 	res->a_ivin = dma_alloc_coherent(dev, SEC_TOTAL_IV_SZ,
327 					 &res->a_ivin_dma, GFP_KERNEL);
328 	if (!res->a_ivin)
329 		return -ENOMEM;
330 
331 	for (i = 1; i < QM_Q_DEPTH; i++) {
332 		res[i].a_ivin_dma = res->a_ivin_dma + i * SEC_IV_SIZE;
333 		res[i].a_ivin = res->a_ivin + i * SEC_IV_SIZE;
334 	}
335 
336 	return 0;
337 }
338 
339 static void sec_free_aiv_resource(struct device *dev, struct sec_alg_res *res)
340 {
341 	if (res->a_ivin)
342 		dma_free_coherent(dev, SEC_TOTAL_IV_SZ,
343 				  res->a_ivin, res->a_ivin_dma);
344 }
345 
346 static int sec_alloc_mac_resource(struct device *dev, struct sec_alg_res *res)
347 {
348 	int i;
349 
350 	res->out_mac = dma_alloc_coherent(dev, SEC_TOTAL_MAC_SZ << 1,
351 					  &res->out_mac_dma, GFP_KERNEL);
352 	if (!res->out_mac)
353 		return -ENOMEM;
354 
355 	for (i = 1; i < QM_Q_DEPTH; i++) {
356 		res[i].out_mac_dma = res->out_mac_dma +
357 				     i * (SEC_MAX_MAC_LEN << 1);
358 		res[i].out_mac = res->out_mac + i * (SEC_MAX_MAC_LEN << 1);
359 	}
360 
361 	return 0;
362 }
363 
364 static void sec_free_mac_resource(struct device *dev, struct sec_alg_res *res)
365 {
366 	if (res->out_mac)
367 		dma_free_coherent(dev, SEC_TOTAL_MAC_SZ << 1,
368 				  res->out_mac, res->out_mac_dma);
369 }
370 
371 static void sec_free_pbuf_resource(struct device *dev, struct sec_alg_res *res)
372 {
373 	if (res->pbuf)
374 		dma_free_coherent(dev, SEC_TOTAL_PBUF_SZ,
375 				  res->pbuf, res->pbuf_dma);
376 }
377 
378 /*
379  * To improve performance, pbuffer is used for
380  * small packets (< 512Bytes) as IOMMU translation using.
381  */
382 static int sec_alloc_pbuf_resource(struct device *dev, struct sec_alg_res *res)
383 {
384 	int pbuf_page_offset;
385 	int i, j, k;
386 
387 	res->pbuf = dma_alloc_coherent(dev, SEC_TOTAL_PBUF_SZ,
388 				&res->pbuf_dma, GFP_KERNEL);
389 	if (!res->pbuf)
390 		return -ENOMEM;
391 
392 	/*
393 	 * SEC_PBUF_PKG contains data pbuf, iv and
394 	 * out_mac : <SEC_PBUF|SEC_IV|SEC_MAC>
395 	 * Every PAGE contains six SEC_PBUF_PKG
396 	 * The sec_qp_ctx contains QM_Q_DEPTH numbers of SEC_PBUF_PKG
397 	 * So we need SEC_PBUF_PAGE_NUM numbers of PAGE
398 	 * for the SEC_TOTAL_PBUF_SZ
399 	 */
400 	for (i = 0; i <= SEC_PBUF_PAGE_NUM; i++) {
401 		pbuf_page_offset = PAGE_SIZE * i;
402 		for (j = 0; j < SEC_PBUF_NUM; j++) {
403 			k = i * SEC_PBUF_NUM + j;
404 			if (k == QM_Q_DEPTH)
405 				break;
406 			res[k].pbuf = res->pbuf +
407 				j * SEC_PBUF_PKG + pbuf_page_offset;
408 			res[k].pbuf_dma = res->pbuf_dma +
409 				j * SEC_PBUF_PKG + pbuf_page_offset;
410 		}
411 	}
412 
413 	return 0;
414 }
415 
416 static int sec_alg_resource_alloc(struct sec_ctx *ctx,
417 				  struct sec_qp_ctx *qp_ctx)
418 {
419 	struct sec_alg_res *res = qp_ctx->res;
420 	struct device *dev = ctx->dev;
421 	int ret;
422 
423 	ret = sec_alloc_civ_resource(dev, res);
424 	if (ret)
425 		return ret;
426 
427 	if (ctx->alg_type == SEC_AEAD) {
428 		ret = sec_alloc_aiv_resource(dev, res);
429 		if (ret)
430 			goto alloc_aiv_fail;
431 
432 		ret = sec_alloc_mac_resource(dev, res);
433 		if (ret)
434 			goto alloc_mac_fail;
435 	}
436 	if (ctx->pbuf_supported) {
437 		ret = sec_alloc_pbuf_resource(dev, res);
438 		if (ret) {
439 			dev_err(dev, "fail to alloc pbuf dma resource!\n");
440 			goto alloc_pbuf_fail;
441 		}
442 	}
443 
444 	return 0;
445 
446 alloc_pbuf_fail:
447 	if (ctx->alg_type == SEC_AEAD)
448 		sec_free_mac_resource(dev, qp_ctx->res);
449 alloc_mac_fail:
450 	if (ctx->alg_type == SEC_AEAD)
451 		sec_free_aiv_resource(dev, res);
452 alloc_aiv_fail:
453 	sec_free_civ_resource(dev, res);
454 	return ret;
455 }
456 
457 static void sec_alg_resource_free(struct sec_ctx *ctx,
458 				  struct sec_qp_ctx *qp_ctx)
459 {
460 	struct device *dev = ctx->dev;
461 
462 	sec_free_civ_resource(dev, qp_ctx->res);
463 
464 	if (ctx->pbuf_supported)
465 		sec_free_pbuf_resource(dev, qp_ctx->res);
466 	if (ctx->alg_type == SEC_AEAD)
467 		sec_free_mac_resource(dev, qp_ctx->res);
468 }
469 
470 static int sec_create_qp_ctx(struct hisi_qm *qm, struct sec_ctx *ctx,
471 			     int qp_ctx_id, int alg_type)
472 {
473 	struct device *dev = ctx->dev;
474 	struct sec_qp_ctx *qp_ctx;
475 	struct hisi_qp *qp;
476 	int ret = -ENOMEM;
477 
478 	qp_ctx = &ctx->qp_ctx[qp_ctx_id];
479 	qp = ctx->qps[qp_ctx_id];
480 	qp->req_type = 0;
481 	qp->qp_ctx = qp_ctx;
482 	qp_ctx->qp = qp;
483 	qp_ctx->ctx = ctx;
484 
485 	qp->req_cb = sec_req_cb;
486 
487 	mutex_init(&qp_ctx->req_lock);
488 	idr_init(&qp_ctx->req_idr);
489 	INIT_LIST_HEAD(&qp_ctx->backlog);
490 
491 	qp_ctx->c_in_pool = hisi_acc_create_sgl_pool(dev, QM_Q_DEPTH,
492 						     SEC_SGL_SGE_NR);
493 	if (IS_ERR(qp_ctx->c_in_pool)) {
494 		dev_err(dev, "fail to create sgl pool for input!\n");
495 		goto err_destroy_idr;
496 	}
497 
498 	qp_ctx->c_out_pool = hisi_acc_create_sgl_pool(dev, QM_Q_DEPTH,
499 						      SEC_SGL_SGE_NR);
500 	if (IS_ERR(qp_ctx->c_out_pool)) {
501 		dev_err(dev, "fail to create sgl pool for output!\n");
502 		goto err_free_c_in_pool;
503 	}
504 
505 	ret = sec_alg_resource_alloc(ctx, qp_ctx);
506 	if (ret)
507 		goto err_free_c_out_pool;
508 
509 	ret = hisi_qm_start_qp(qp, 0);
510 	if (ret < 0)
511 		goto err_queue_free;
512 
513 	return 0;
514 
515 err_queue_free:
516 	sec_alg_resource_free(ctx, qp_ctx);
517 err_free_c_out_pool:
518 	hisi_acc_free_sgl_pool(dev, qp_ctx->c_out_pool);
519 err_free_c_in_pool:
520 	hisi_acc_free_sgl_pool(dev, qp_ctx->c_in_pool);
521 err_destroy_idr:
522 	idr_destroy(&qp_ctx->req_idr);
523 	return ret;
524 }
525 
526 static void sec_release_qp_ctx(struct sec_ctx *ctx,
527 			       struct sec_qp_ctx *qp_ctx)
528 {
529 	struct device *dev = ctx->dev;
530 
531 	hisi_qm_stop_qp(qp_ctx->qp);
532 	sec_alg_resource_free(ctx, qp_ctx);
533 
534 	hisi_acc_free_sgl_pool(dev, qp_ctx->c_out_pool);
535 	hisi_acc_free_sgl_pool(dev, qp_ctx->c_in_pool);
536 
537 	idr_destroy(&qp_ctx->req_idr);
538 }
539 
540 static int sec_ctx_base_init(struct sec_ctx *ctx)
541 {
542 	struct sec_dev *sec;
543 	int i, ret;
544 
545 	ctx->qps = sec_create_qps();
546 	if (!ctx->qps) {
547 		pr_err("Can not create sec qps!\n");
548 		return -ENODEV;
549 	}
550 
551 	sec = container_of(ctx->qps[0]->qm, struct sec_dev, qm);
552 	ctx->sec = sec;
553 	ctx->dev = &sec->qm.pdev->dev;
554 	ctx->hlf_q_num = sec->ctx_q_num >> 1;
555 
556 	ctx->pbuf_supported = ctx->sec->iommu_used;
557 
558 	/* Half of queue depth is taken as fake requests limit in the queue. */
559 	ctx->fake_req_limit = QM_Q_DEPTH >> 1;
560 	ctx->qp_ctx = kcalloc(sec->ctx_q_num, sizeof(struct sec_qp_ctx),
561 			      GFP_KERNEL);
562 	if (!ctx->qp_ctx) {
563 		ret = -ENOMEM;
564 		goto err_destroy_qps;
565 	}
566 
567 	for (i = 0; i < sec->ctx_q_num; i++) {
568 		ret = sec_create_qp_ctx(&sec->qm, ctx, i, 0);
569 		if (ret)
570 			goto err_sec_release_qp_ctx;
571 	}
572 
573 	return 0;
574 
575 err_sec_release_qp_ctx:
576 	for (i = i - 1; i >= 0; i--)
577 		sec_release_qp_ctx(ctx, &ctx->qp_ctx[i]);
578 	kfree(ctx->qp_ctx);
579 err_destroy_qps:
580 	sec_destroy_qps(ctx->qps, sec->ctx_q_num);
581 	return ret;
582 }
583 
584 static void sec_ctx_base_uninit(struct sec_ctx *ctx)
585 {
586 	int i;
587 
588 	for (i = 0; i < ctx->sec->ctx_q_num; i++)
589 		sec_release_qp_ctx(ctx, &ctx->qp_ctx[i]);
590 
591 	sec_destroy_qps(ctx->qps, ctx->sec->ctx_q_num);
592 	kfree(ctx->qp_ctx);
593 }
594 
595 static int sec_cipher_init(struct sec_ctx *ctx)
596 {
597 	struct sec_cipher_ctx *c_ctx = &ctx->c_ctx;
598 
599 	c_ctx->c_key = dma_alloc_coherent(ctx->dev, SEC_MAX_KEY_SIZE,
600 					  &c_ctx->c_key_dma, GFP_KERNEL);
601 	if (!c_ctx->c_key)
602 		return -ENOMEM;
603 
604 	return 0;
605 }
606 
607 static void sec_cipher_uninit(struct sec_ctx *ctx)
608 {
609 	struct sec_cipher_ctx *c_ctx = &ctx->c_ctx;
610 
611 	memzero_explicit(c_ctx->c_key, SEC_MAX_KEY_SIZE);
612 	dma_free_coherent(ctx->dev, SEC_MAX_KEY_SIZE,
613 			  c_ctx->c_key, c_ctx->c_key_dma);
614 }
615 
616 static int sec_auth_init(struct sec_ctx *ctx)
617 {
618 	struct sec_auth_ctx *a_ctx = &ctx->a_ctx;
619 
620 	a_ctx->a_key = dma_alloc_coherent(ctx->dev, SEC_MAX_KEY_SIZE,
621 					  &a_ctx->a_key_dma, GFP_KERNEL);
622 	if (!a_ctx->a_key)
623 		return -ENOMEM;
624 
625 	return 0;
626 }
627 
628 static void sec_auth_uninit(struct sec_ctx *ctx)
629 {
630 	struct sec_auth_ctx *a_ctx = &ctx->a_ctx;
631 
632 	memzero_explicit(a_ctx->a_key, SEC_MAX_KEY_SIZE);
633 	dma_free_coherent(ctx->dev, SEC_MAX_KEY_SIZE,
634 			  a_ctx->a_key, a_ctx->a_key_dma);
635 }
636 
637 static int sec_skcipher_fbtfm_init(struct crypto_skcipher *tfm)
638 {
639 	const char *alg = crypto_tfm_alg_name(&tfm->base);
640 	struct sec_ctx *ctx = crypto_skcipher_ctx(tfm);
641 	struct sec_cipher_ctx *c_ctx = &ctx->c_ctx;
642 
643 	c_ctx->fallback = false;
644 	if (likely(strncmp(alg, "xts", SEC_XTS_NAME_SZ)))
645 		return 0;
646 
647 	c_ctx->fbtfm = crypto_alloc_sync_skcipher(alg, 0,
648 						  CRYPTO_ALG_NEED_FALLBACK);
649 	if (IS_ERR(c_ctx->fbtfm)) {
650 		pr_err("failed to alloc fallback tfm!\n");
651 		return PTR_ERR(c_ctx->fbtfm);
652 	}
653 
654 	return 0;
655 }
656 
657 static int sec_skcipher_init(struct crypto_skcipher *tfm)
658 {
659 	struct sec_ctx *ctx = crypto_skcipher_ctx(tfm);
660 	int ret;
661 
662 	ctx->alg_type = SEC_SKCIPHER;
663 	crypto_skcipher_set_reqsize(tfm, sizeof(struct sec_req));
664 	ctx->c_ctx.ivsize = crypto_skcipher_ivsize(tfm);
665 	if (ctx->c_ctx.ivsize > SEC_IV_SIZE) {
666 		pr_err("get error skcipher iv size!\n");
667 		return -EINVAL;
668 	}
669 
670 	ret = sec_ctx_base_init(ctx);
671 	if (ret)
672 		return ret;
673 
674 	ret = sec_cipher_init(ctx);
675 	if (ret)
676 		goto err_cipher_init;
677 
678 	ret = sec_skcipher_fbtfm_init(tfm);
679 	if (ret)
680 		goto err_fbtfm_init;
681 
682 	return 0;
683 
684 err_fbtfm_init:
685 	sec_cipher_uninit(ctx);
686 err_cipher_init:
687 	sec_ctx_base_uninit(ctx);
688 	return ret;
689 }
690 
691 static void sec_skcipher_uninit(struct crypto_skcipher *tfm)
692 {
693 	struct sec_ctx *ctx = crypto_skcipher_ctx(tfm);
694 
695 	if (ctx->c_ctx.fbtfm)
696 		crypto_free_sync_skcipher(ctx->c_ctx.fbtfm);
697 
698 	sec_cipher_uninit(ctx);
699 	sec_ctx_base_uninit(ctx);
700 }
701 
702 static int sec_skcipher_3des_setkey(struct crypto_skcipher *tfm, const u8 *key,
703 				    const u32 keylen,
704 				    const enum sec_cmode c_mode)
705 {
706 	struct sec_ctx *ctx = crypto_skcipher_ctx(tfm);
707 	struct sec_cipher_ctx *c_ctx = &ctx->c_ctx;
708 	int ret;
709 
710 	ret = verify_skcipher_des3_key(tfm, key);
711 	if (ret)
712 		return ret;
713 
714 	switch (keylen) {
715 	case SEC_DES3_2KEY_SIZE:
716 		c_ctx->c_key_len = SEC_CKEY_3DES_2KEY;
717 		break;
718 	case SEC_DES3_3KEY_SIZE:
719 		c_ctx->c_key_len = SEC_CKEY_3DES_3KEY;
720 		break;
721 	default:
722 		return -EINVAL;
723 	}
724 
725 	return 0;
726 }
727 
728 static int sec_skcipher_aes_sm4_setkey(struct sec_cipher_ctx *c_ctx,
729 				       const u32 keylen,
730 				       const enum sec_cmode c_mode)
731 {
732 	if (c_mode == SEC_CMODE_XTS) {
733 		switch (keylen) {
734 		case SEC_XTS_MIN_KEY_SIZE:
735 			c_ctx->c_key_len = SEC_CKEY_128BIT;
736 			break;
737 		case SEC_XTS_MID_KEY_SIZE:
738 			c_ctx->fallback = true;
739 			break;
740 		case SEC_XTS_MAX_KEY_SIZE:
741 			c_ctx->c_key_len = SEC_CKEY_256BIT;
742 			break;
743 		default:
744 			pr_err("hisi_sec2: xts mode key error!\n");
745 			return -EINVAL;
746 		}
747 	} else {
748 		if (c_ctx->c_alg == SEC_CALG_SM4 &&
749 		    keylen != AES_KEYSIZE_128) {
750 			pr_err("hisi_sec2: sm4 key error!\n");
751 			return -EINVAL;
752 		} else {
753 			switch (keylen) {
754 			case AES_KEYSIZE_128:
755 				c_ctx->c_key_len = SEC_CKEY_128BIT;
756 				break;
757 			case AES_KEYSIZE_192:
758 				c_ctx->c_key_len = SEC_CKEY_192BIT;
759 				break;
760 			case AES_KEYSIZE_256:
761 				c_ctx->c_key_len = SEC_CKEY_256BIT;
762 				break;
763 			default:
764 				pr_err("hisi_sec2: aes key error!\n");
765 				return -EINVAL;
766 			}
767 		}
768 	}
769 
770 	return 0;
771 }
772 
773 static int sec_skcipher_setkey(struct crypto_skcipher *tfm, const u8 *key,
774 			       const u32 keylen, const enum sec_calg c_alg,
775 			       const enum sec_cmode c_mode)
776 {
777 	struct sec_ctx *ctx = crypto_skcipher_ctx(tfm);
778 	struct sec_cipher_ctx *c_ctx = &ctx->c_ctx;
779 	struct device *dev = ctx->dev;
780 	int ret;
781 
782 	if (c_mode == SEC_CMODE_XTS) {
783 		ret = xts_verify_key(tfm, key, keylen);
784 		if (ret) {
785 			dev_err(dev, "xts mode key err!\n");
786 			return ret;
787 		}
788 	}
789 
790 	c_ctx->c_alg  = c_alg;
791 	c_ctx->c_mode = c_mode;
792 
793 	switch (c_alg) {
794 	case SEC_CALG_3DES:
795 		ret = sec_skcipher_3des_setkey(tfm, key, keylen, c_mode);
796 		break;
797 	case SEC_CALG_AES:
798 	case SEC_CALG_SM4:
799 		ret = sec_skcipher_aes_sm4_setkey(c_ctx, keylen, c_mode);
800 		break;
801 	default:
802 		return -EINVAL;
803 	}
804 
805 	if (ret) {
806 		dev_err(dev, "set sec key err!\n");
807 		return ret;
808 	}
809 
810 	memcpy(c_ctx->c_key, key, keylen);
811 	if (c_ctx->fallback) {
812 		ret = crypto_sync_skcipher_setkey(c_ctx->fbtfm, key, keylen);
813 		if (ret) {
814 			dev_err(dev, "failed to set fallback skcipher key!\n");
815 			return ret;
816 		}
817 	}
818 	return 0;
819 }
820 
821 #define GEN_SEC_SETKEY_FUNC(name, c_alg, c_mode)			\
822 static int sec_setkey_##name(struct crypto_skcipher *tfm, const u8 *key,\
823 	u32 keylen)							\
824 {									\
825 	return sec_skcipher_setkey(tfm, key, keylen, c_alg, c_mode);	\
826 }
827 
828 GEN_SEC_SETKEY_FUNC(aes_ecb, SEC_CALG_AES, SEC_CMODE_ECB)
829 GEN_SEC_SETKEY_FUNC(aes_cbc, SEC_CALG_AES, SEC_CMODE_CBC)
830 GEN_SEC_SETKEY_FUNC(aes_xts, SEC_CALG_AES, SEC_CMODE_XTS)
831 GEN_SEC_SETKEY_FUNC(aes_ofb, SEC_CALG_AES, SEC_CMODE_OFB)
832 GEN_SEC_SETKEY_FUNC(aes_cfb, SEC_CALG_AES, SEC_CMODE_CFB)
833 GEN_SEC_SETKEY_FUNC(aes_ctr, SEC_CALG_AES, SEC_CMODE_CTR)
834 GEN_SEC_SETKEY_FUNC(3des_ecb, SEC_CALG_3DES, SEC_CMODE_ECB)
835 GEN_SEC_SETKEY_FUNC(3des_cbc, SEC_CALG_3DES, SEC_CMODE_CBC)
836 GEN_SEC_SETKEY_FUNC(sm4_xts, SEC_CALG_SM4, SEC_CMODE_XTS)
837 GEN_SEC_SETKEY_FUNC(sm4_cbc, SEC_CALG_SM4, SEC_CMODE_CBC)
838 GEN_SEC_SETKEY_FUNC(sm4_ofb, SEC_CALG_SM4, SEC_CMODE_OFB)
839 GEN_SEC_SETKEY_FUNC(sm4_cfb, SEC_CALG_SM4, SEC_CMODE_CFB)
840 GEN_SEC_SETKEY_FUNC(sm4_ctr, SEC_CALG_SM4, SEC_CMODE_CTR)
841 
842 static int sec_cipher_pbuf_map(struct sec_ctx *ctx, struct sec_req *req,
843 			struct scatterlist *src)
844 {
845 	struct sec_aead_req *a_req = &req->aead_req;
846 	struct aead_request *aead_req = a_req->aead_req;
847 	struct sec_cipher_req *c_req = &req->c_req;
848 	struct sec_qp_ctx *qp_ctx = req->qp_ctx;
849 	struct device *dev = ctx->dev;
850 	int copy_size, pbuf_length;
851 	int req_id = req->req_id;
852 	struct crypto_aead *tfm;
853 	size_t authsize;
854 	u8 *mac_offset;
855 
856 	if (ctx->alg_type == SEC_AEAD)
857 		copy_size = aead_req->cryptlen + aead_req->assoclen;
858 	else
859 		copy_size = c_req->c_len;
860 
861 	pbuf_length = sg_copy_to_buffer(src, sg_nents(src),
862 			qp_ctx->res[req_id].pbuf, copy_size);
863 	if (unlikely(pbuf_length != copy_size)) {
864 		dev_err(dev, "copy src data to pbuf error!\n");
865 		return -EINVAL;
866 	}
867 	if (!c_req->encrypt && ctx->alg_type == SEC_AEAD) {
868 		tfm = crypto_aead_reqtfm(aead_req);
869 		authsize = crypto_aead_authsize(tfm);
870 		mac_offset = qp_ctx->res[req_id].pbuf + copy_size - authsize;
871 		memcpy(a_req->out_mac, mac_offset, authsize);
872 	}
873 
874 	req->in_dma = qp_ctx->res[req_id].pbuf_dma;
875 	c_req->c_out_dma = req->in_dma;
876 
877 	return 0;
878 }
879 
880 static void sec_cipher_pbuf_unmap(struct sec_ctx *ctx, struct sec_req *req,
881 			struct scatterlist *dst)
882 {
883 	struct aead_request *aead_req = req->aead_req.aead_req;
884 	struct sec_cipher_req *c_req = &req->c_req;
885 	struct sec_qp_ctx *qp_ctx = req->qp_ctx;
886 	int copy_size, pbuf_length;
887 	int req_id = req->req_id;
888 
889 	if (ctx->alg_type == SEC_AEAD)
890 		copy_size = c_req->c_len + aead_req->assoclen;
891 	else
892 		copy_size = c_req->c_len;
893 
894 	pbuf_length = sg_copy_from_buffer(dst, sg_nents(dst),
895 			qp_ctx->res[req_id].pbuf, copy_size);
896 	if (unlikely(pbuf_length != copy_size))
897 		dev_err(ctx->dev, "copy pbuf data to dst error!\n");
898 }
899 
900 static int sec_aead_mac_init(struct sec_aead_req *req)
901 {
902 	struct aead_request *aead_req = req->aead_req;
903 	struct crypto_aead *tfm = crypto_aead_reqtfm(aead_req);
904 	size_t authsize = crypto_aead_authsize(tfm);
905 	u8 *mac_out = req->out_mac;
906 	struct scatterlist *sgl = aead_req->src;
907 	size_t copy_size;
908 	off_t skip_size;
909 
910 	/* Copy input mac */
911 	skip_size = aead_req->assoclen + aead_req->cryptlen - authsize;
912 	copy_size = sg_pcopy_to_buffer(sgl, sg_nents(sgl), mac_out,
913 				       authsize, skip_size);
914 	if (unlikely(copy_size != authsize))
915 		return -EINVAL;
916 
917 	return 0;
918 }
919 
920 static int sec_cipher_map(struct sec_ctx *ctx, struct sec_req *req,
921 			  struct scatterlist *src, struct scatterlist *dst)
922 {
923 	struct sec_cipher_req *c_req = &req->c_req;
924 	struct sec_aead_req *a_req = &req->aead_req;
925 	struct sec_qp_ctx *qp_ctx = req->qp_ctx;
926 	struct sec_alg_res *res = &qp_ctx->res[req->req_id];
927 	struct device *dev = ctx->dev;
928 	int ret;
929 
930 	if (req->use_pbuf) {
931 		c_req->c_ivin = res->pbuf + SEC_PBUF_IV_OFFSET;
932 		c_req->c_ivin_dma = res->pbuf_dma + SEC_PBUF_IV_OFFSET;
933 		if (ctx->alg_type == SEC_AEAD) {
934 			a_req->a_ivin = res->a_ivin;
935 			a_req->a_ivin_dma = res->a_ivin_dma;
936 			a_req->out_mac = res->pbuf + SEC_PBUF_MAC_OFFSET;
937 			a_req->out_mac_dma = res->pbuf_dma +
938 					SEC_PBUF_MAC_OFFSET;
939 		}
940 		ret = sec_cipher_pbuf_map(ctx, req, src);
941 
942 		return ret;
943 	}
944 	c_req->c_ivin = res->c_ivin;
945 	c_req->c_ivin_dma = res->c_ivin_dma;
946 	if (ctx->alg_type == SEC_AEAD) {
947 		a_req->a_ivin = res->a_ivin;
948 		a_req->a_ivin_dma = res->a_ivin_dma;
949 		a_req->out_mac = res->out_mac;
950 		a_req->out_mac_dma = res->out_mac_dma;
951 	}
952 
953 	req->in = hisi_acc_sg_buf_map_to_hw_sgl(dev, src,
954 						qp_ctx->c_in_pool,
955 						req->req_id,
956 						&req->in_dma);
957 	if (IS_ERR(req->in)) {
958 		dev_err(dev, "fail to dma map input sgl buffers!\n");
959 		return PTR_ERR(req->in);
960 	}
961 
962 	if (!c_req->encrypt && ctx->alg_type == SEC_AEAD) {
963 		ret = sec_aead_mac_init(a_req);
964 		if (unlikely(ret)) {
965 			dev_err(dev, "fail to init mac data for ICV!\n");
966 			return ret;
967 		}
968 	}
969 
970 	if (dst == src) {
971 		c_req->c_out = req->in;
972 		c_req->c_out_dma = req->in_dma;
973 	} else {
974 		c_req->c_out = hisi_acc_sg_buf_map_to_hw_sgl(dev, dst,
975 							     qp_ctx->c_out_pool,
976 							     req->req_id,
977 							     &c_req->c_out_dma);
978 
979 		if (IS_ERR(c_req->c_out)) {
980 			dev_err(dev, "fail to dma map output sgl buffers!\n");
981 			hisi_acc_sg_buf_unmap(dev, src, req->in);
982 			return PTR_ERR(c_req->c_out);
983 		}
984 	}
985 
986 	return 0;
987 }
988 
989 static void sec_cipher_unmap(struct sec_ctx *ctx, struct sec_req *req,
990 			     struct scatterlist *src, struct scatterlist *dst)
991 {
992 	struct sec_cipher_req *c_req = &req->c_req;
993 	struct device *dev = ctx->dev;
994 
995 	if (req->use_pbuf) {
996 		sec_cipher_pbuf_unmap(ctx, req, dst);
997 	} else {
998 		if (dst != src)
999 			hisi_acc_sg_buf_unmap(dev, src, req->in);
1000 
1001 		hisi_acc_sg_buf_unmap(dev, dst, c_req->c_out);
1002 	}
1003 }
1004 
1005 static int sec_skcipher_sgl_map(struct sec_ctx *ctx, struct sec_req *req)
1006 {
1007 	struct skcipher_request *sq = req->c_req.sk_req;
1008 
1009 	return sec_cipher_map(ctx, req, sq->src, sq->dst);
1010 }
1011 
1012 static void sec_skcipher_sgl_unmap(struct sec_ctx *ctx, struct sec_req *req)
1013 {
1014 	struct skcipher_request *sq = req->c_req.sk_req;
1015 
1016 	sec_cipher_unmap(ctx, req, sq->src, sq->dst);
1017 }
1018 
1019 static int sec_aead_aes_set_key(struct sec_cipher_ctx *c_ctx,
1020 				struct crypto_authenc_keys *keys)
1021 {
1022 	switch (keys->enckeylen) {
1023 	case AES_KEYSIZE_128:
1024 		c_ctx->c_key_len = SEC_CKEY_128BIT;
1025 		break;
1026 	case AES_KEYSIZE_192:
1027 		c_ctx->c_key_len = SEC_CKEY_192BIT;
1028 		break;
1029 	case AES_KEYSIZE_256:
1030 		c_ctx->c_key_len = SEC_CKEY_256BIT;
1031 		break;
1032 	default:
1033 		pr_err("hisi_sec2: aead aes key error!\n");
1034 		return -EINVAL;
1035 	}
1036 	memcpy(c_ctx->c_key, keys->enckey, keys->enckeylen);
1037 
1038 	return 0;
1039 }
1040 
1041 static int sec_aead_auth_set_key(struct sec_auth_ctx *ctx,
1042 				 struct crypto_authenc_keys *keys)
1043 {
1044 	struct crypto_shash *hash_tfm = ctx->hash_tfm;
1045 	int blocksize, digestsize, ret;
1046 
1047 	if (!keys->authkeylen) {
1048 		pr_err("hisi_sec2: aead auth key error!\n");
1049 		return -EINVAL;
1050 	}
1051 
1052 	blocksize = crypto_shash_blocksize(hash_tfm);
1053 	digestsize = crypto_shash_digestsize(hash_tfm);
1054 	if (keys->authkeylen > blocksize) {
1055 		ret = crypto_shash_tfm_digest(hash_tfm, keys->authkey,
1056 					      keys->authkeylen, ctx->a_key);
1057 		if (ret) {
1058 			pr_err("hisi_sec2: aead auth digest error!\n");
1059 			return -EINVAL;
1060 		}
1061 		ctx->a_key_len = digestsize;
1062 	} else {
1063 		memcpy(ctx->a_key, keys->authkey, keys->authkeylen);
1064 		ctx->a_key_len = keys->authkeylen;
1065 	}
1066 
1067 	return 0;
1068 }
1069 
1070 static int sec_aead_setauthsize(struct crypto_aead *aead, unsigned int authsize)
1071 {
1072 	struct crypto_tfm *tfm = crypto_aead_tfm(aead);
1073 	struct sec_ctx *ctx = crypto_tfm_ctx(tfm);
1074 	struct sec_auth_ctx *a_ctx = &ctx->a_ctx;
1075 
1076 	if (unlikely(a_ctx->fallback_aead_tfm))
1077 		return crypto_aead_setauthsize(a_ctx->fallback_aead_tfm, authsize);
1078 
1079 	return 0;
1080 }
1081 
1082 static int sec_aead_fallback_setkey(struct sec_auth_ctx *a_ctx,
1083 				    struct crypto_aead *tfm, const u8 *key,
1084 				    unsigned int keylen)
1085 {
1086 	crypto_aead_clear_flags(a_ctx->fallback_aead_tfm, CRYPTO_TFM_REQ_MASK);
1087 	crypto_aead_set_flags(a_ctx->fallback_aead_tfm,
1088 			      crypto_aead_get_flags(tfm) & CRYPTO_TFM_REQ_MASK);
1089 	return crypto_aead_setkey(a_ctx->fallback_aead_tfm, key, keylen);
1090 }
1091 
1092 static int sec_aead_setkey(struct crypto_aead *tfm, const u8 *key,
1093 			   const u32 keylen, const enum sec_hash_alg a_alg,
1094 			   const enum sec_calg c_alg,
1095 			   const enum sec_mac_len mac_len,
1096 			   const enum sec_cmode c_mode)
1097 {
1098 	struct sec_ctx *ctx = crypto_aead_ctx(tfm);
1099 	struct sec_cipher_ctx *c_ctx = &ctx->c_ctx;
1100 	struct sec_auth_ctx *a_ctx = &ctx->a_ctx;
1101 	struct device *dev = ctx->dev;
1102 	struct crypto_authenc_keys keys;
1103 	int ret;
1104 
1105 	ctx->a_ctx.a_alg = a_alg;
1106 	ctx->c_ctx.c_alg = c_alg;
1107 	ctx->a_ctx.mac_len = mac_len;
1108 	c_ctx->c_mode = c_mode;
1109 
1110 	if (c_mode == SEC_CMODE_CCM || c_mode == SEC_CMODE_GCM) {
1111 		ret = sec_skcipher_aes_sm4_setkey(c_ctx, keylen, c_mode);
1112 		if (ret) {
1113 			dev_err(dev, "set sec aes ccm cipher key err!\n");
1114 			return ret;
1115 		}
1116 		memcpy(c_ctx->c_key, key, keylen);
1117 
1118 		if (unlikely(a_ctx->fallback_aead_tfm)) {
1119 			ret = sec_aead_fallback_setkey(a_ctx, tfm, key, keylen);
1120 			if (ret)
1121 				return ret;
1122 		}
1123 
1124 		return 0;
1125 	}
1126 
1127 	if (crypto_authenc_extractkeys(&keys, key, keylen))
1128 		goto bad_key;
1129 
1130 	ret = sec_aead_aes_set_key(c_ctx, &keys);
1131 	if (ret) {
1132 		dev_err(dev, "set sec cipher key err!\n");
1133 		goto bad_key;
1134 	}
1135 
1136 	ret = sec_aead_auth_set_key(&ctx->a_ctx, &keys);
1137 	if (ret) {
1138 		dev_err(dev, "set sec auth key err!\n");
1139 		goto bad_key;
1140 	}
1141 
1142 	if ((ctx->a_ctx.mac_len & SEC_SQE_LEN_RATE_MASK)  ||
1143 	    (ctx->a_ctx.a_key_len & SEC_SQE_LEN_RATE_MASK)) {
1144 		dev_err(dev, "MAC or AUTH key length error!\n");
1145 		goto bad_key;
1146 	}
1147 
1148 	return 0;
1149 
1150 bad_key:
1151 	memzero_explicit(&keys, sizeof(struct crypto_authenc_keys));
1152 	return -EINVAL;
1153 }
1154 
1155 
1156 #define GEN_SEC_AEAD_SETKEY_FUNC(name, aalg, calg, maclen, cmode)	\
1157 static int sec_setkey_##name(struct crypto_aead *tfm, const u8 *key,	\
1158 	u32 keylen)							\
1159 {									\
1160 	return sec_aead_setkey(tfm, key, keylen, aalg, calg, maclen, cmode);\
1161 }
1162 
1163 GEN_SEC_AEAD_SETKEY_FUNC(aes_cbc_sha1, SEC_A_HMAC_SHA1,
1164 			 SEC_CALG_AES, SEC_HMAC_SHA1_MAC, SEC_CMODE_CBC)
1165 GEN_SEC_AEAD_SETKEY_FUNC(aes_cbc_sha256, SEC_A_HMAC_SHA256,
1166 			 SEC_CALG_AES, SEC_HMAC_SHA256_MAC, SEC_CMODE_CBC)
1167 GEN_SEC_AEAD_SETKEY_FUNC(aes_cbc_sha512, SEC_A_HMAC_SHA512,
1168 			 SEC_CALG_AES, SEC_HMAC_SHA512_MAC, SEC_CMODE_CBC)
1169 GEN_SEC_AEAD_SETKEY_FUNC(aes_ccm, 0, SEC_CALG_AES,
1170 			 SEC_HMAC_CCM_MAC, SEC_CMODE_CCM)
1171 GEN_SEC_AEAD_SETKEY_FUNC(aes_gcm, 0, SEC_CALG_AES,
1172 			 SEC_HMAC_GCM_MAC, SEC_CMODE_GCM)
1173 GEN_SEC_AEAD_SETKEY_FUNC(sm4_ccm, 0, SEC_CALG_SM4,
1174 			 SEC_HMAC_CCM_MAC, SEC_CMODE_CCM)
1175 GEN_SEC_AEAD_SETKEY_FUNC(sm4_gcm, 0, SEC_CALG_SM4,
1176 			 SEC_HMAC_GCM_MAC, SEC_CMODE_GCM)
1177 
1178 static int sec_aead_sgl_map(struct sec_ctx *ctx, struct sec_req *req)
1179 {
1180 	struct aead_request *aq = req->aead_req.aead_req;
1181 
1182 	return sec_cipher_map(ctx, req, aq->src, aq->dst);
1183 }
1184 
1185 static void sec_aead_sgl_unmap(struct sec_ctx *ctx, struct sec_req *req)
1186 {
1187 	struct aead_request *aq = req->aead_req.aead_req;
1188 
1189 	sec_cipher_unmap(ctx, req, aq->src, aq->dst);
1190 }
1191 
1192 static int sec_request_transfer(struct sec_ctx *ctx, struct sec_req *req)
1193 {
1194 	int ret;
1195 
1196 	ret = ctx->req_op->buf_map(ctx, req);
1197 	if (unlikely(ret))
1198 		return ret;
1199 
1200 	ctx->req_op->do_transfer(ctx, req);
1201 
1202 	ret = ctx->req_op->bd_fill(ctx, req);
1203 	if (unlikely(ret))
1204 		goto unmap_req_buf;
1205 
1206 	return ret;
1207 
1208 unmap_req_buf:
1209 	ctx->req_op->buf_unmap(ctx, req);
1210 	return ret;
1211 }
1212 
1213 static void sec_request_untransfer(struct sec_ctx *ctx, struct sec_req *req)
1214 {
1215 	ctx->req_op->buf_unmap(ctx, req);
1216 }
1217 
1218 static void sec_skcipher_copy_iv(struct sec_ctx *ctx, struct sec_req *req)
1219 {
1220 	struct skcipher_request *sk_req = req->c_req.sk_req;
1221 	struct sec_cipher_req *c_req = &req->c_req;
1222 
1223 	memcpy(c_req->c_ivin, sk_req->iv, ctx->c_ctx.ivsize);
1224 }
1225 
1226 static int sec_skcipher_bd_fill(struct sec_ctx *ctx, struct sec_req *req)
1227 {
1228 	struct sec_cipher_ctx *c_ctx = &ctx->c_ctx;
1229 	struct sec_cipher_req *c_req = &req->c_req;
1230 	struct sec_sqe *sec_sqe = &req->sec_sqe;
1231 	u8 scene, sa_type, da_type;
1232 	u8 bd_type, cipher;
1233 	u8 de = 0;
1234 
1235 	memset(sec_sqe, 0, sizeof(struct sec_sqe));
1236 
1237 	sec_sqe->type2.c_key_addr = cpu_to_le64(c_ctx->c_key_dma);
1238 	sec_sqe->type2.c_ivin_addr = cpu_to_le64(c_req->c_ivin_dma);
1239 	sec_sqe->type2.data_src_addr = cpu_to_le64(req->in_dma);
1240 	sec_sqe->type2.data_dst_addr = cpu_to_le64(c_req->c_out_dma);
1241 
1242 	sec_sqe->type2.icvw_kmode |= cpu_to_le16(((u16)c_ctx->c_mode) <<
1243 						SEC_CMODE_OFFSET);
1244 	sec_sqe->type2.c_alg = c_ctx->c_alg;
1245 	sec_sqe->type2.icvw_kmode |= cpu_to_le16(((u16)c_ctx->c_key_len) <<
1246 						SEC_CKEY_OFFSET);
1247 
1248 	bd_type = SEC_BD_TYPE2;
1249 	if (c_req->encrypt)
1250 		cipher = SEC_CIPHER_ENC << SEC_CIPHER_OFFSET;
1251 	else
1252 		cipher = SEC_CIPHER_DEC << SEC_CIPHER_OFFSET;
1253 	sec_sqe->type_cipher_auth = bd_type | cipher;
1254 
1255 	/* Set destination and source address type */
1256 	if (req->use_pbuf) {
1257 		sa_type = SEC_PBUF << SEC_SRC_SGL_OFFSET;
1258 		da_type = SEC_PBUF << SEC_DST_SGL_OFFSET;
1259 	} else {
1260 		sa_type = SEC_SGL << SEC_SRC_SGL_OFFSET;
1261 		da_type = SEC_SGL << SEC_DST_SGL_OFFSET;
1262 	}
1263 
1264 	sec_sqe->sdm_addr_type |= da_type;
1265 	scene = SEC_COMM_SCENE << SEC_SCENE_OFFSET;
1266 	if (req->in_dma != c_req->c_out_dma)
1267 		de = 0x1 << SEC_DE_OFFSET;
1268 
1269 	sec_sqe->sds_sa_type = (de | scene | sa_type);
1270 
1271 	sec_sqe->type2.clen_ivhlen |= cpu_to_le32(c_req->c_len);
1272 	sec_sqe->type2.tag = cpu_to_le16((u16)req->req_id);
1273 
1274 	return 0;
1275 }
1276 
1277 static int sec_skcipher_bd_fill_v3(struct sec_ctx *ctx, struct sec_req *req)
1278 {
1279 	struct sec_sqe3 *sec_sqe3 = &req->sec_sqe3;
1280 	struct sec_cipher_ctx *c_ctx = &ctx->c_ctx;
1281 	struct sec_cipher_req *c_req = &req->c_req;
1282 	u32 bd_param = 0;
1283 	u16 cipher;
1284 
1285 	memset(sec_sqe3, 0, sizeof(struct sec_sqe3));
1286 
1287 	sec_sqe3->c_key_addr = cpu_to_le64(c_ctx->c_key_dma);
1288 	sec_sqe3->no_scene.c_ivin_addr = cpu_to_le64(c_req->c_ivin_dma);
1289 	sec_sqe3->data_src_addr = cpu_to_le64(req->in_dma);
1290 	sec_sqe3->data_dst_addr = cpu_to_le64(c_req->c_out_dma);
1291 
1292 	sec_sqe3->c_mode_alg = ((u8)c_ctx->c_alg << SEC_CALG_OFFSET_V3) |
1293 						c_ctx->c_mode;
1294 	sec_sqe3->c_icv_key |= cpu_to_le16(((u16)c_ctx->c_key_len) <<
1295 						SEC_CKEY_OFFSET_V3);
1296 
1297 	if (c_req->encrypt)
1298 		cipher = SEC_CIPHER_ENC;
1299 	else
1300 		cipher = SEC_CIPHER_DEC;
1301 	sec_sqe3->c_icv_key |= cpu_to_le16(cipher);
1302 
1303 	if (req->use_pbuf) {
1304 		bd_param |= SEC_PBUF << SEC_SRC_SGL_OFFSET_V3;
1305 		bd_param |= SEC_PBUF << SEC_DST_SGL_OFFSET_V3;
1306 	} else {
1307 		bd_param |= SEC_SGL << SEC_SRC_SGL_OFFSET_V3;
1308 		bd_param |= SEC_SGL << SEC_DST_SGL_OFFSET_V3;
1309 	}
1310 
1311 	bd_param |= SEC_COMM_SCENE << SEC_SCENE_OFFSET_V3;
1312 	if (req->in_dma != c_req->c_out_dma)
1313 		bd_param |= 0x1 << SEC_DE_OFFSET_V3;
1314 
1315 	bd_param |= SEC_BD_TYPE3;
1316 	sec_sqe3->bd_param = cpu_to_le32(bd_param);
1317 
1318 	sec_sqe3->c_len_ivin |= cpu_to_le32(c_req->c_len);
1319 	sec_sqe3->tag = cpu_to_le64(req);
1320 
1321 	return 0;
1322 }
1323 
1324 /* increment counter (128-bit int) */
1325 static void ctr_iv_inc(__u8 *counter, __u8 bits, __u32 nums)
1326 {
1327 	do {
1328 		--bits;
1329 		nums += counter[bits];
1330 		counter[bits] = nums & BITS_MASK;
1331 		nums >>= BYTE_BITS;
1332 	} while (bits && nums);
1333 }
1334 
1335 static void sec_update_iv(struct sec_req *req, enum sec_alg_type alg_type)
1336 {
1337 	struct aead_request *aead_req = req->aead_req.aead_req;
1338 	struct skcipher_request *sk_req = req->c_req.sk_req;
1339 	u32 iv_size = req->ctx->c_ctx.ivsize;
1340 	struct scatterlist *sgl;
1341 	unsigned int cryptlen;
1342 	size_t sz;
1343 	u8 *iv;
1344 
1345 	if (req->c_req.encrypt)
1346 		sgl = alg_type == SEC_SKCIPHER ? sk_req->dst : aead_req->dst;
1347 	else
1348 		sgl = alg_type == SEC_SKCIPHER ? sk_req->src : aead_req->src;
1349 
1350 	if (alg_type == SEC_SKCIPHER) {
1351 		iv = sk_req->iv;
1352 		cryptlen = sk_req->cryptlen;
1353 	} else {
1354 		iv = aead_req->iv;
1355 		cryptlen = aead_req->cryptlen;
1356 	}
1357 
1358 	if (req->ctx->c_ctx.c_mode == SEC_CMODE_CBC) {
1359 		sz = sg_pcopy_to_buffer(sgl, sg_nents(sgl), iv, iv_size,
1360 					cryptlen - iv_size);
1361 		if (unlikely(sz != iv_size))
1362 			dev_err(req->ctx->dev, "copy output iv error!\n");
1363 	} else {
1364 		sz = cryptlen / iv_size;
1365 		if (cryptlen % iv_size)
1366 			sz += 1;
1367 		ctr_iv_inc(iv, iv_size, sz);
1368 	}
1369 }
1370 
1371 static struct sec_req *sec_back_req_clear(struct sec_ctx *ctx,
1372 				struct sec_qp_ctx *qp_ctx)
1373 {
1374 	struct sec_req *backlog_req = NULL;
1375 
1376 	mutex_lock(&qp_ctx->req_lock);
1377 	if (ctx->fake_req_limit >=
1378 	    atomic_read(&qp_ctx->qp->qp_status.used) &&
1379 	    !list_empty(&qp_ctx->backlog)) {
1380 		backlog_req = list_first_entry(&qp_ctx->backlog,
1381 				typeof(*backlog_req), backlog_head);
1382 		list_del(&backlog_req->backlog_head);
1383 	}
1384 	mutex_unlock(&qp_ctx->req_lock);
1385 
1386 	return backlog_req;
1387 }
1388 
1389 static void sec_skcipher_callback(struct sec_ctx *ctx, struct sec_req *req,
1390 				  int err)
1391 {
1392 	struct skcipher_request *sk_req = req->c_req.sk_req;
1393 	struct sec_qp_ctx *qp_ctx = req->qp_ctx;
1394 	struct skcipher_request *backlog_sk_req;
1395 	struct sec_req *backlog_req;
1396 
1397 	sec_free_req_id(req);
1398 
1399 	/* IV output at encrypto of CBC/CTR mode */
1400 	if (!err && (ctx->c_ctx.c_mode == SEC_CMODE_CBC ||
1401 	    ctx->c_ctx.c_mode == SEC_CMODE_CTR) && req->c_req.encrypt)
1402 		sec_update_iv(req, SEC_SKCIPHER);
1403 
1404 	while (1) {
1405 		backlog_req = sec_back_req_clear(ctx, qp_ctx);
1406 		if (!backlog_req)
1407 			break;
1408 
1409 		backlog_sk_req = backlog_req->c_req.sk_req;
1410 		backlog_sk_req->base.complete(&backlog_sk_req->base,
1411 						-EINPROGRESS);
1412 		atomic64_inc(&ctx->sec->debug.dfx.recv_busy_cnt);
1413 	}
1414 
1415 	sk_req->base.complete(&sk_req->base, err);
1416 }
1417 
1418 static void set_aead_auth_iv(struct sec_ctx *ctx, struct sec_req *req)
1419 {
1420 	struct aead_request *aead_req = req->aead_req.aead_req;
1421 	struct sec_cipher_req *c_req = &req->c_req;
1422 	struct sec_aead_req *a_req = &req->aead_req;
1423 	size_t authsize = ctx->a_ctx.mac_len;
1424 	u32 data_size = aead_req->cryptlen;
1425 	u8 flage = 0;
1426 	u8 cm, cl;
1427 
1428 	/* the specification has been checked in aead_iv_demension_check() */
1429 	cl = c_req->c_ivin[0] + 1;
1430 	c_req->c_ivin[ctx->c_ctx.ivsize - cl] = 0x00;
1431 	memset(&c_req->c_ivin[ctx->c_ctx.ivsize - cl], 0, cl);
1432 	c_req->c_ivin[ctx->c_ctx.ivsize - IV_LAST_BYTE1] = IV_CTR_INIT;
1433 
1434 	/* the last 3bit is L' */
1435 	flage |= c_req->c_ivin[0] & IV_CL_MASK;
1436 
1437 	/* the M' is bit3~bit5, the Flags is bit6 */
1438 	cm = (authsize - IV_CM_CAL_NUM) / IV_CM_CAL_NUM;
1439 	flage |= cm << IV_CM_OFFSET;
1440 	if (aead_req->assoclen)
1441 		flage |= 0x01 << IV_FLAGS_OFFSET;
1442 
1443 	memcpy(a_req->a_ivin, c_req->c_ivin, ctx->c_ctx.ivsize);
1444 	a_req->a_ivin[0] = flage;
1445 
1446 	/*
1447 	 * the last 32bit is counter's initial number,
1448 	 * but the nonce uses the first 16bit
1449 	 * the tail 16bit fill with the cipher length
1450 	 */
1451 	if (!c_req->encrypt)
1452 		data_size = aead_req->cryptlen - authsize;
1453 
1454 	a_req->a_ivin[ctx->c_ctx.ivsize - IV_LAST_BYTE1] =
1455 			data_size & IV_LAST_BYTE_MASK;
1456 	data_size >>= IV_BYTE_OFFSET;
1457 	a_req->a_ivin[ctx->c_ctx.ivsize - IV_LAST_BYTE2] =
1458 			data_size & IV_LAST_BYTE_MASK;
1459 }
1460 
1461 static void sec_aead_set_iv(struct sec_ctx *ctx, struct sec_req *req)
1462 {
1463 	struct aead_request *aead_req = req->aead_req.aead_req;
1464 	struct crypto_aead *tfm = crypto_aead_reqtfm(aead_req);
1465 	size_t authsize = crypto_aead_authsize(tfm);
1466 	struct sec_cipher_req *c_req = &req->c_req;
1467 	struct sec_aead_req *a_req = &req->aead_req;
1468 
1469 	memcpy(c_req->c_ivin, aead_req->iv, ctx->c_ctx.ivsize);
1470 
1471 	if (ctx->c_ctx.c_mode == SEC_CMODE_CCM) {
1472 		/*
1473 		 * CCM 16Byte Cipher_IV: {1B_Flage,13B_IV,2B_counter},
1474 		 * the  counter must set to 0x01
1475 		 */
1476 		ctx->a_ctx.mac_len = authsize;
1477 		/* CCM 16Byte Auth_IV: {1B_AFlage,13B_IV,2B_Ptext_length} */
1478 		set_aead_auth_iv(ctx, req);
1479 	}
1480 
1481 	/* GCM 12Byte Cipher_IV == Auth_IV */
1482 	if (ctx->c_ctx.c_mode == SEC_CMODE_GCM) {
1483 		ctx->a_ctx.mac_len = authsize;
1484 		memcpy(a_req->a_ivin, c_req->c_ivin, SEC_AIV_SIZE);
1485 	}
1486 }
1487 
1488 static void sec_auth_bd_fill_xcm(struct sec_auth_ctx *ctx, int dir,
1489 				 struct sec_req *req, struct sec_sqe *sec_sqe)
1490 {
1491 	struct sec_aead_req *a_req = &req->aead_req;
1492 	struct aead_request *aq = a_req->aead_req;
1493 
1494 	/* C_ICV_Len is MAC size, 0x4 ~ 0x10 */
1495 	sec_sqe->type2.icvw_kmode |= cpu_to_le16((u16)ctx->mac_len);
1496 
1497 	/* mode set to CCM/GCM, don't set {A_Alg, AKey_Len, MAC_Len} */
1498 	sec_sqe->type2.a_key_addr = sec_sqe->type2.c_key_addr;
1499 	sec_sqe->type2.a_ivin_addr = cpu_to_le64(a_req->a_ivin_dma);
1500 	sec_sqe->type_cipher_auth |= SEC_NO_AUTH << SEC_AUTH_OFFSET;
1501 
1502 	if (dir)
1503 		sec_sqe->sds_sa_type &= SEC_CIPHER_AUTH;
1504 	else
1505 		sec_sqe->sds_sa_type |= SEC_AUTH_CIPHER;
1506 
1507 	sec_sqe->type2.alen_ivllen = cpu_to_le32(aq->assoclen);
1508 	sec_sqe->type2.auth_src_offset = cpu_to_le16(0x0);
1509 	sec_sqe->type2.cipher_src_offset = cpu_to_le16((u16)aq->assoclen);
1510 
1511 	sec_sqe->type2.mac_addr = cpu_to_le64(a_req->out_mac_dma);
1512 }
1513 
1514 static void sec_auth_bd_fill_xcm_v3(struct sec_auth_ctx *ctx, int dir,
1515 				    struct sec_req *req, struct sec_sqe3 *sqe3)
1516 {
1517 	struct sec_aead_req *a_req = &req->aead_req;
1518 	struct aead_request *aq = a_req->aead_req;
1519 
1520 	/* C_ICV_Len is MAC size, 0x4 ~ 0x10 */
1521 	sqe3->c_icv_key |= cpu_to_le16((u16)ctx->mac_len << SEC_MAC_OFFSET_V3);
1522 
1523 	/* mode set to CCM/GCM, don't set {A_Alg, AKey_Len, MAC_Len} */
1524 	sqe3->a_key_addr = sqe3->c_key_addr;
1525 	sqe3->auth_ivin.a_ivin_addr = cpu_to_le64(a_req->a_ivin_dma);
1526 	sqe3->auth_mac_key |= SEC_NO_AUTH;
1527 
1528 	if (dir)
1529 		sqe3->huk_iv_seq &= SEC_CIPHER_AUTH_V3;
1530 	else
1531 		sqe3->huk_iv_seq |= SEC_AUTH_CIPHER_V3;
1532 
1533 	sqe3->a_len_key = cpu_to_le32(aq->assoclen);
1534 	sqe3->auth_src_offset = cpu_to_le16(0x0);
1535 	sqe3->cipher_src_offset = cpu_to_le16((u16)aq->assoclen);
1536 	sqe3->mac_addr = cpu_to_le64(a_req->out_mac_dma);
1537 }
1538 
1539 static void sec_auth_bd_fill_ex(struct sec_auth_ctx *ctx, int dir,
1540 			       struct sec_req *req, struct sec_sqe *sec_sqe)
1541 {
1542 	struct sec_aead_req *a_req = &req->aead_req;
1543 	struct sec_cipher_req *c_req = &req->c_req;
1544 	struct aead_request *aq = a_req->aead_req;
1545 
1546 	sec_sqe->type2.a_key_addr = cpu_to_le64(ctx->a_key_dma);
1547 
1548 	sec_sqe->type2.mac_key_alg =
1549 			cpu_to_le32(ctx->mac_len / SEC_SQE_LEN_RATE);
1550 
1551 	sec_sqe->type2.mac_key_alg |=
1552 			cpu_to_le32((u32)((ctx->a_key_len) /
1553 			SEC_SQE_LEN_RATE) << SEC_AKEY_OFFSET);
1554 
1555 	sec_sqe->type2.mac_key_alg |=
1556 			cpu_to_le32((u32)(ctx->a_alg) << SEC_AEAD_ALG_OFFSET);
1557 
1558 	if (dir) {
1559 		sec_sqe->type_cipher_auth |= SEC_AUTH_TYPE1 << SEC_AUTH_OFFSET;
1560 		sec_sqe->sds_sa_type &= SEC_CIPHER_AUTH;
1561 	} else {
1562 		sec_sqe->type_cipher_auth |= SEC_AUTH_TYPE2 << SEC_AUTH_OFFSET;
1563 		sec_sqe->sds_sa_type |= SEC_AUTH_CIPHER;
1564 	}
1565 	sec_sqe->type2.alen_ivllen = cpu_to_le32(c_req->c_len + aq->assoclen);
1566 
1567 	sec_sqe->type2.cipher_src_offset = cpu_to_le16((u16)aq->assoclen);
1568 
1569 	sec_sqe->type2.mac_addr = cpu_to_le64(a_req->out_mac_dma);
1570 }
1571 
1572 static int sec_aead_bd_fill(struct sec_ctx *ctx, struct sec_req *req)
1573 {
1574 	struct sec_auth_ctx *auth_ctx = &ctx->a_ctx;
1575 	struct sec_sqe *sec_sqe = &req->sec_sqe;
1576 	int ret;
1577 
1578 	ret = sec_skcipher_bd_fill(ctx, req);
1579 	if (unlikely(ret)) {
1580 		dev_err(ctx->dev, "skcipher bd fill is error!\n");
1581 		return ret;
1582 	}
1583 
1584 	if (ctx->c_ctx.c_mode == SEC_CMODE_CCM ||
1585 	    ctx->c_ctx.c_mode == SEC_CMODE_GCM)
1586 		sec_auth_bd_fill_xcm(auth_ctx, req->c_req.encrypt, req, sec_sqe);
1587 	else
1588 		sec_auth_bd_fill_ex(auth_ctx, req->c_req.encrypt, req, sec_sqe);
1589 
1590 	return 0;
1591 }
1592 
1593 static void sec_auth_bd_fill_ex_v3(struct sec_auth_ctx *ctx, int dir,
1594 				   struct sec_req *req, struct sec_sqe3 *sqe3)
1595 {
1596 	struct sec_aead_req *a_req = &req->aead_req;
1597 	struct sec_cipher_req *c_req = &req->c_req;
1598 	struct aead_request *aq = a_req->aead_req;
1599 
1600 	sqe3->a_key_addr = cpu_to_le64(ctx->a_key_dma);
1601 
1602 	sqe3->auth_mac_key |=
1603 			cpu_to_le32((u32)(ctx->mac_len /
1604 			SEC_SQE_LEN_RATE) << SEC_MAC_OFFSET_V3);
1605 
1606 	sqe3->auth_mac_key |=
1607 			cpu_to_le32((u32)(ctx->a_key_len /
1608 			SEC_SQE_LEN_RATE) << SEC_AKEY_OFFSET_V3);
1609 
1610 	sqe3->auth_mac_key |=
1611 			cpu_to_le32((u32)(ctx->a_alg) << SEC_AUTH_ALG_OFFSET_V3);
1612 
1613 	if (dir) {
1614 		sqe3->auth_mac_key |= cpu_to_le32((u32)SEC_AUTH_TYPE1);
1615 		sqe3->huk_iv_seq &= SEC_CIPHER_AUTH_V3;
1616 	} else {
1617 		sqe3->auth_mac_key |= cpu_to_le32((u32)SEC_AUTH_TYPE1);
1618 		sqe3->huk_iv_seq |= SEC_AUTH_CIPHER_V3;
1619 	}
1620 	sqe3->a_len_key = cpu_to_le32(c_req->c_len + aq->assoclen);
1621 
1622 	sqe3->cipher_src_offset = cpu_to_le16((u16)aq->assoclen);
1623 
1624 	sqe3->mac_addr = cpu_to_le64(a_req->out_mac_dma);
1625 }
1626 
1627 static int sec_aead_bd_fill_v3(struct sec_ctx *ctx, struct sec_req *req)
1628 {
1629 	struct sec_auth_ctx *auth_ctx = &ctx->a_ctx;
1630 	struct sec_sqe3 *sec_sqe3 = &req->sec_sqe3;
1631 	int ret;
1632 
1633 	ret = sec_skcipher_bd_fill_v3(ctx, req);
1634 	if (unlikely(ret)) {
1635 		dev_err(ctx->dev, "skcipher bd3 fill is error!\n");
1636 		return ret;
1637 	}
1638 
1639 	if (ctx->c_ctx.c_mode == SEC_CMODE_CCM ||
1640 	    ctx->c_ctx.c_mode == SEC_CMODE_GCM)
1641 		sec_auth_bd_fill_xcm_v3(auth_ctx, req->c_req.encrypt,
1642 					req, sec_sqe3);
1643 	else
1644 		sec_auth_bd_fill_ex_v3(auth_ctx, req->c_req.encrypt,
1645 				       req, sec_sqe3);
1646 
1647 	return 0;
1648 }
1649 
1650 static void sec_aead_callback(struct sec_ctx *c, struct sec_req *req, int err)
1651 {
1652 	struct aead_request *a_req = req->aead_req.aead_req;
1653 	struct crypto_aead *tfm = crypto_aead_reqtfm(a_req);
1654 	struct sec_aead_req *aead_req = &req->aead_req;
1655 	struct sec_cipher_req *c_req = &req->c_req;
1656 	size_t authsize = crypto_aead_authsize(tfm);
1657 	struct sec_qp_ctx *qp_ctx = req->qp_ctx;
1658 	struct aead_request *backlog_aead_req;
1659 	struct sec_req *backlog_req;
1660 	size_t sz;
1661 
1662 	if (!err && c->c_ctx.c_mode == SEC_CMODE_CBC && c_req->encrypt)
1663 		sec_update_iv(req, SEC_AEAD);
1664 
1665 	/* Copy output mac */
1666 	if (!err && c_req->encrypt) {
1667 		struct scatterlist *sgl = a_req->dst;
1668 
1669 		sz = sg_pcopy_from_buffer(sgl, sg_nents(sgl),
1670 					  aead_req->out_mac,
1671 					  authsize, a_req->cryptlen +
1672 					  a_req->assoclen);
1673 
1674 		if (unlikely(sz != authsize)) {
1675 			dev_err(c->dev, "copy out mac err!\n");
1676 			err = -EINVAL;
1677 		}
1678 	}
1679 
1680 	sec_free_req_id(req);
1681 
1682 	while (1) {
1683 		backlog_req = sec_back_req_clear(c, qp_ctx);
1684 		if (!backlog_req)
1685 			break;
1686 
1687 		backlog_aead_req = backlog_req->aead_req.aead_req;
1688 		backlog_aead_req->base.complete(&backlog_aead_req->base,
1689 						-EINPROGRESS);
1690 		atomic64_inc(&c->sec->debug.dfx.recv_busy_cnt);
1691 	}
1692 
1693 	a_req->base.complete(&a_req->base, err);
1694 }
1695 
1696 static void sec_request_uninit(struct sec_ctx *ctx, struct sec_req *req)
1697 {
1698 	sec_free_req_id(req);
1699 	sec_free_queue_id(ctx, req);
1700 }
1701 
1702 static int sec_request_init(struct sec_ctx *ctx, struct sec_req *req)
1703 {
1704 	struct sec_qp_ctx *qp_ctx;
1705 	int queue_id;
1706 
1707 	/* To load balance */
1708 	queue_id = sec_alloc_queue_id(ctx, req);
1709 	qp_ctx = &ctx->qp_ctx[queue_id];
1710 
1711 	req->req_id = sec_alloc_req_id(req, qp_ctx);
1712 	if (unlikely(req->req_id < 0)) {
1713 		sec_free_queue_id(ctx, req);
1714 		return req->req_id;
1715 	}
1716 
1717 	return 0;
1718 }
1719 
1720 static int sec_process(struct sec_ctx *ctx, struct sec_req *req)
1721 {
1722 	struct sec_cipher_req *c_req = &req->c_req;
1723 	int ret;
1724 
1725 	ret = sec_request_init(ctx, req);
1726 	if (unlikely(ret))
1727 		return ret;
1728 
1729 	ret = sec_request_transfer(ctx, req);
1730 	if (unlikely(ret))
1731 		goto err_uninit_req;
1732 
1733 	/* Output IV as decrypto */
1734 	if (!req->c_req.encrypt && (ctx->c_ctx.c_mode == SEC_CMODE_CBC ||
1735 	    ctx->c_ctx.c_mode == SEC_CMODE_CTR))
1736 		sec_update_iv(req, ctx->alg_type);
1737 
1738 	ret = ctx->req_op->bd_send(ctx, req);
1739 	if (unlikely((ret != -EBUSY && ret != -EINPROGRESS) ||
1740 		(ret == -EBUSY && !(req->flag & CRYPTO_TFM_REQ_MAY_BACKLOG)))) {
1741 		dev_err_ratelimited(ctx->dev, "send sec request failed!\n");
1742 		goto err_send_req;
1743 	}
1744 
1745 	return ret;
1746 
1747 err_send_req:
1748 	/* As failing, restore the IV from user */
1749 	if (ctx->c_ctx.c_mode == SEC_CMODE_CBC && !req->c_req.encrypt) {
1750 		if (ctx->alg_type == SEC_SKCIPHER)
1751 			memcpy(req->c_req.sk_req->iv, c_req->c_ivin,
1752 			       ctx->c_ctx.ivsize);
1753 		else
1754 			memcpy(req->aead_req.aead_req->iv, c_req->c_ivin,
1755 			       ctx->c_ctx.ivsize);
1756 	}
1757 
1758 	sec_request_untransfer(ctx, req);
1759 err_uninit_req:
1760 	sec_request_uninit(ctx, req);
1761 	return ret;
1762 }
1763 
1764 static const struct sec_req_op sec_skcipher_req_ops = {
1765 	.buf_map	= sec_skcipher_sgl_map,
1766 	.buf_unmap	= sec_skcipher_sgl_unmap,
1767 	.do_transfer	= sec_skcipher_copy_iv,
1768 	.bd_fill	= sec_skcipher_bd_fill,
1769 	.bd_send	= sec_bd_send,
1770 	.callback	= sec_skcipher_callback,
1771 	.process	= sec_process,
1772 };
1773 
1774 static const struct sec_req_op sec_aead_req_ops = {
1775 	.buf_map	= sec_aead_sgl_map,
1776 	.buf_unmap	= sec_aead_sgl_unmap,
1777 	.do_transfer	= sec_aead_set_iv,
1778 	.bd_fill	= sec_aead_bd_fill,
1779 	.bd_send	= sec_bd_send,
1780 	.callback	= sec_aead_callback,
1781 	.process	= sec_process,
1782 };
1783 
1784 static const struct sec_req_op sec_skcipher_req_ops_v3 = {
1785 	.buf_map	= sec_skcipher_sgl_map,
1786 	.buf_unmap	= sec_skcipher_sgl_unmap,
1787 	.do_transfer	= sec_skcipher_copy_iv,
1788 	.bd_fill	= sec_skcipher_bd_fill_v3,
1789 	.bd_send	= sec_bd_send,
1790 	.callback	= sec_skcipher_callback,
1791 	.process	= sec_process,
1792 };
1793 
1794 static const struct sec_req_op sec_aead_req_ops_v3 = {
1795 	.buf_map	= sec_aead_sgl_map,
1796 	.buf_unmap	= sec_aead_sgl_unmap,
1797 	.do_transfer	= sec_aead_set_iv,
1798 	.bd_fill	= sec_aead_bd_fill_v3,
1799 	.bd_send	= sec_bd_send,
1800 	.callback	= sec_aead_callback,
1801 	.process	= sec_process,
1802 };
1803 
1804 static int sec_skcipher_ctx_init(struct crypto_skcipher *tfm)
1805 {
1806 	struct sec_ctx *ctx = crypto_skcipher_ctx(tfm);
1807 	int ret;
1808 
1809 	ret = sec_skcipher_init(tfm);
1810 	if (ret)
1811 		return ret;
1812 
1813 	if (ctx->sec->qm.ver < QM_HW_V3) {
1814 		ctx->type_supported = SEC_BD_TYPE2;
1815 		ctx->req_op = &sec_skcipher_req_ops;
1816 	} else {
1817 		ctx->type_supported = SEC_BD_TYPE3;
1818 		ctx->req_op = &sec_skcipher_req_ops_v3;
1819 	}
1820 
1821 	return ret;
1822 }
1823 
1824 static void sec_skcipher_ctx_exit(struct crypto_skcipher *tfm)
1825 {
1826 	sec_skcipher_uninit(tfm);
1827 }
1828 
1829 static int sec_aead_init(struct crypto_aead *tfm)
1830 {
1831 	struct sec_ctx *ctx = crypto_aead_ctx(tfm);
1832 	int ret;
1833 
1834 	crypto_aead_set_reqsize(tfm, sizeof(struct sec_req));
1835 	ctx->alg_type = SEC_AEAD;
1836 	ctx->c_ctx.ivsize = crypto_aead_ivsize(tfm);
1837 	if (ctx->c_ctx.ivsize < SEC_AIV_SIZE ||
1838 	    ctx->c_ctx.ivsize > SEC_IV_SIZE) {
1839 		pr_err("get error aead iv size!\n");
1840 		return -EINVAL;
1841 	}
1842 
1843 	ret = sec_ctx_base_init(ctx);
1844 	if (ret)
1845 		return ret;
1846 	if (ctx->sec->qm.ver < QM_HW_V3) {
1847 		ctx->type_supported = SEC_BD_TYPE2;
1848 		ctx->req_op = &sec_aead_req_ops;
1849 	} else {
1850 		ctx->type_supported = SEC_BD_TYPE3;
1851 		ctx->req_op = &sec_aead_req_ops_v3;
1852 	}
1853 
1854 	ret = sec_auth_init(ctx);
1855 	if (ret)
1856 		goto err_auth_init;
1857 
1858 	ret = sec_cipher_init(ctx);
1859 	if (ret)
1860 		goto err_cipher_init;
1861 
1862 	return ret;
1863 
1864 err_cipher_init:
1865 	sec_auth_uninit(ctx);
1866 err_auth_init:
1867 	sec_ctx_base_uninit(ctx);
1868 	return ret;
1869 }
1870 
1871 static void sec_aead_exit(struct crypto_aead *tfm)
1872 {
1873 	struct sec_ctx *ctx = crypto_aead_ctx(tfm);
1874 
1875 	sec_cipher_uninit(ctx);
1876 	sec_auth_uninit(ctx);
1877 	sec_ctx_base_uninit(ctx);
1878 }
1879 
1880 static int sec_aead_ctx_init(struct crypto_aead *tfm, const char *hash_name)
1881 {
1882 	struct sec_ctx *ctx = crypto_aead_ctx(tfm);
1883 	struct sec_auth_ctx *auth_ctx = &ctx->a_ctx;
1884 	int ret;
1885 
1886 	ret = sec_aead_init(tfm);
1887 	if (ret) {
1888 		pr_err("hisi_sec2: aead init error!\n");
1889 		return ret;
1890 	}
1891 
1892 	auth_ctx->hash_tfm = crypto_alloc_shash(hash_name, 0, 0);
1893 	if (IS_ERR(auth_ctx->hash_tfm)) {
1894 		dev_err(ctx->dev, "aead alloc shash error!\n");
1895 		sec_aead_exit(tfm);
1896 		return PTR_ERR(auth_ctx->hash_tfm);
1897 	}
1898 
1899 	return 0;
1900 }
1901 
1902 static void sec_aead_ctx_exit(struct crypto_aead *tfm)
1903 {
1904 	struct sec_ctx *ctx = crypto_aead_ctx(tfm);
1905 
1906 	crypto_free_shash(ctx->a_ctx.hash_tfm);
1907 	sec_aead_exit(tfm);
1908 }
1909 
1910 static int sec_aead_xcm_ctx_init(struct crypto_aead *tfm)
1911 {
1912 	struct aead_alg *alg = crypto_aead_alg(tfm);
1913 	struct sec_ctx *ctx = crypto_aead_ctx(tfm);
1914 	struct sec_auth_ctx *a_ctx = &ctx->a_ctx;
1915 	const char *aead_name = alg->base.cra_name;
1916 	int ret;
1917 
1918 	ret = sec_aead_init(tfm);
1919 	if (ret) {
1920 		dev_err(ctx->dev, "hisi_sec2: aead xcm init error!\n");
1921 		return ret;
1922 	}
1923 
1924 	a_ctx->fallback_aead_tfm = crypto_alloc_aead(aead_name, 0,
1925 						     CRYPTO_ALG_NEED_FALLBACK |
1926 						     CRYPTO_ALG_ASYNC);
1927 	if (IS_ERR(a_ctx->fallback_aead_tfm)) {
1928 		dev_err(ctx->dev, "aead driver alloc fallback tfm error!\n");
1929 		sec_aead_exit(tfm);
1930 		return PTR_ERR(a_ctx->fallback_aead_tfm);
1931 	}
1932 	a_ctx->fallback = false;
1933 
1934 	return 0;
1935 }
1936 
1937 static void sec_aead_xcm_ctx_exit(struct crypto_aead *tfm)
1938 {
1939 	struct sec_ctx *ctx = crypto_aead_ctx(tfm);
1940 
1941 	crypto_free_aead(ctx->a_ctx.fallback_aead_tfm);
1942 	sec_aead_exit(tfm);
1943 }
1944 
1945 static int sec_aead_sha1_ctx_init(struct crypto_aead *tfm)
1946 {
1947 	return sec_aead_ctx_init(tfm, "sha1");
1948 }
1949 
1950 static int sec_aead_sha256_ctx_init(struct crypto_aead *tfm)
1951 {
1952 	return sec_aead_ctx_init(tfm, "sha256");
1953 }
1954 
1955 static int sec_aead_sha512_ctx_init(struct crypto_aead *tfm)
1956 {
1957 	return sec_aead_ctx_init(tfm, "sha512");
1958 }
1959 
1960 
1961 static int sec_skcipher_cryptlen_ckeck(struct sec_ctx *ctx,
1962 	struct sec_req *sreq)
1963 {
1964 	u32 cryptlen = sreq->c_req.sk_req->cryptlen;
1965 	struct device *dev = ctx->dev;
1966 	u8 c_mode = ctx->c_ctx.c_mode;
1967 	int ret = 0;
1968 
1969 	switch (c_mode) {
1970 	case SEC_CMODE_XTS:
1971 		if (unlikely(cryptlen < AES_BLOCK_SIZE)) {
1972 			dev_err(dev, "skcipher XTS mode input length error!\n");
1973 			ret = -EINVAL;
1974 		}
1975 		break;
1976 	case SEC_CMODE_ECB:
1977 	case SEC_CMODE_CBC:
1978 		if (unlikely(cryptlen & (AES_BLOCK_SIZE - 1))) {
1979 			dev_err(dev, "skcipher AES input length error!\n");
1980 			ret = -EINVAL;
1981 		}
1982 		break;
1983 	case SEC_CMODE_CFB:
1984 	case SEC_CMODE_OFB:
1985 	case SEC_CMODE_CTR:
1986 		if (unlikely(ctx->sec->qm.ver < QM_HW_V3)) {
1987 			dev_err(dev, "skcipher HW version error!\n");
1988 			ret = -EINVAL;
1989 		}
1990 		break;
1991 	default:
1992 		ret = -EINVAL;
1993 	}
1994 
1995 	return ret;
1996 }
1997 
1998 static int sec_skcipher_param_check(struct sec_ctx *ctx, struct sec_req *sreq)
1999 {
2000 	struct skcipher_request *sk_req = sreq->c_req.sk_req;
2001 	struct device *dev = ctx->dev;
2002 	u8 c_alg = ctx->c_ctx.c_alg;
2003 
2004 	if (unlikely(!sk_req->src || !sk_req->dst ||
2005 		     sk_req->cryptlen > MAX_INPUT_DATA_LEN)) {
2006 		dev_err(dev, "skcipher input param error!\n");
2007 		return -EINVAL;
2008 	}
2009 	sreq->c_req.c_len = sk_req->cryptlen;
2010 
2011 	if (ctx->pbuf_supported && sk_req->cryptlen <= SEC_PBUF_SZ)
2012 		sreq->use_pbuf = true;
2013 	else
2014 		sreq->use_pbuf = false;
2015 
2016 	if (c_alg == SEC_CALG_3DES) {
2017 		if (unlikely(sk_req->cryptlen & (DES3_EDE_BLOCK_SIZE - 1))) {
2018 			dev_err(dev, "skcipher 3des input length error!\n");
2019 			return -EINVAL;
2020 		}
2021 		return 0;
2022 	} else if (c_alg == SEC_CALG_AES || c_alg == SEC_CALG_SM4) {
2023 		return sec_skcipher_cryptlen_ckeck(ctx, sreq);
2024 	}
2025 
2026 	dev_err(dev, "skcipher algorithm error!\n");
2027 
2028 	return -EINVAL;
2029 }
2030 
2031 static int sec_skcipher_soft_crypto(struct sec_ctx *ctx,
2032 				    struct skcipher_request *sreq, bool encrypt)
2033 {
2034 	struct sec_cipher_ctx *c_ctx = &ctx->c_ctx;
2035 	struct device *dev = ctx->dev;
2036 	int ret;
2037 
2038 	SYNC_SKCIPHER_REQUEST_ON_STACK(subreq, c_ctx->fbtfm);
2039 
2040 	if (!c_ctx->fbtfm) {
2041 		dev_err(dev, "failed to check fallback tfm\n");
2042 		return -EINVAL;
2043 	}
2044 
2045 	skcipher_request_set_sync_tfm(subreq, c_ctx->fbtfm);
2046 
2047 	/* software need sync mode to do crypto */
2048 	skcipher_request_set_callback(subreq, sreq->base.flags,
2049 				      NULL, NULL);
2050 	skcipher_request_set_crypt(subreq, sreq->src, sreq->dst,
2051 				   sreq->cryptlen, sreq->iv);
2052 	if (encrypt)
2053 		ret = crypto_skcipher_encrypt(subreq);
2054 	else
2055 		ret = crypto_skcipher_decrypt(subreq);
2056 
2057 	skcipher_request_zero(subreq);
2058 
2059 	return ret;
2060 }
2061 
2062 static int sec_skcipher_crypto(struct skcipher_request *sk_req, bool encrypt)
2063 {
2064 	struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(sk_req);
2065 	struct sec_req *req = skcipher_request_ctx(sk_req);
2066 	struct sec_ctx *ctx = crypto_skcipher_ctx(tfm);
2067 	int ret;
2068 
2069 	if (!sk_req->cryptlen) {
2070 		if (ctx->c_ctx.c_mode == SEC_CMODE_XTS)
2071 			return -EINVAL;
2072 		return 0;
2073 	}
2074 
2075 	req->flag = sk_req->base.flags;
2076 	req->c_req.sk_req = sk_req;
2077 	req->c_req.encrypt = encrypt;
2078 	req->ctx = ctx;
2079 
2080 	ret = sec_skcipher_param_check(ctx, req);
2081 	if (unlikely(ret))
2082 		return -EINVAL;
2083 
2084 	if (unlikely(ctx->c_ctx.fallback))
2085 		return sec_skcipher_soft_crypto(ctx, sk_req, encrypt);
2086 
2087 	return ctx->req_op->process(ctx, req);
2088 }
2089 
2090 static int sec_skcipher_encrypt(struct skcipher_request *sk_req)
2091 {
2092 	return sec_skcipher_crypto(sk_req, true);
2093 }
2094 
2095 static int sec_skcipher_decrypt(struct skcipher_request *sk_req)
2096 {
2097 	return sec_skcipher_crypto(sk_req, false);
2098 }
2099 
2100 #define SEC_SKCIPHER_GEN_ALG(sec_cra_name, sec_set_key, sec_min_key_size, \
2101 	sec_max_key_size, ctx_init, ctx_exit, blk_size, iv_size)\
2102 {\
2103 	.base = {\
2104 		.cra_name = sec_cra_name,\
2105 		.cra_driver_name = "hisi_sec_"sec_cra_name,\
2106 		.cra_priority = SEC_PRIORITY,\
2107 		.cra_flags = CRYPTO_ALG_ASYNC |\
2108 		 CRYPTO_ALG_ALLOCATES_MEMORY |\
2109 		 CRYPTO_ALG_NEED_FALLBACK,\
2110 		.cra_blocksize = blk_size,\
2111 		.cra_ctxsize = sizeof(struct sec_ctx),\
2112 		.cra_module = THIS_MODULE,\
2113 	},\
2114 	.init = ctx_init,\
2115 	.exit = ctx_exit,\
2116 	.setkey = sec_set_key,\
2117 	.decrypt = sec_skcipher_decrypt,\
2118 	.encrypt = sec_skcipher_encrypt,\
2119 	.min_keysize = sec_min_key_size,\
2120 	.max_keysize = sec_max_key_size,\
2121 	.ivsize = iv_size,\
2122 },
2123 
2124 #define SEC_SKCIPHER_ALG(name, key_func, min_key_size, \
2125 	max_key_size, blk_size, iv_size) \
2126 	SEC_SKCIPHER_GEN_ALG(name, key_func, min_key_size, max_key_size, \
2127 	sec_skcipher_ctx_init, sec_skcipher_ctx_exit, blk_size, iv_size)
2128 
2129 static struct skcipher_alg sec_skciphers[] = {
2130 	SEC_SKCIPHER_ALG("ecb(aes)", sec_setkey_aes_ecb,
2131 			 AES_MIN_KEY_SIZE, AES_MAX_KEY_SIZE,
2132 			 AES_BLOCK_SIZE, 0)
2133 
2134 	SEC_SKCIPHER_ALG("cbc(aes)", sec_setkey_aes_cbc,
2135 			 AES_MIN_KEY_SIZE, AES_MAX_KEY_SIZE,
2136 			 AES_BLOCK_SIZE, AES_BLOCK_SIZE)
2137 
2138 	SEC_SKCIPHER_ALG("xts(aes)", sec_setkey_aes_xts,
2139 			 SEC_XTS_MIN_KEY_SIZE, SEC_XTS_MAX_KEY_SIZE,
2140 			 AES_BLOCK_SIZE, AES_BLOCK_SIZE)
2141 
2142 	SEC_SKCIPHER_ALG("ecb(des3_ede)", sec_setkey_3des_ecb,
2143 			 SEC_DES3_3KEY_SIZE, SEC_DES3_3KEY_SIZE,
2144 			 DES3_EDE_BLOCK_SIZE, 0)
2145 
2146 	SEC_SKCIPHER_ALG("cbc(des3_ede)", sec_setkey_3des_cbc,
2147 			 SEC_DES3_3KEY_SIZE, SEC_DES3_3KEY_SIZE,
2148 			 DES3_EDE_BLOCK_SIZE, DES3_EDE_BLOCK_SIZE)
2149 
2150 	SEC_SKCIPHER_ALG("xts(sm4)", sec_setkey_sm4_xts,
2151 			 SEC_XTS_MIN_KEY_SIZE, SEC_XTS_MIN_KEY_SIZE,
2152 			 AES_BLOCK_SIZE, AES_BLOCK_SIZE)
2153 
2154 	SEC_SKCIPHER_ALG("cbc(sm4)", sec_setkey_sm4_cbc,
2155 			 AES_MIN_KEY_SIZE, AES_MIN_KEY_SIZE,
2156 			 AES_BLOCK_SIZE, AES_BLOCK_SIZE)
2157 };
2158 
2159 static struct skcipher_alg sec_skciphers_v3[] = {
2160 	SEC_SKCIPHER_ALG("ofb(aes)", sec_setkey_aes_ofb,
2161 			 AES_MIN_KEY_SIZE, AES_MAX_KEY_SIZE,
2162 			 SEC_MIN_BLOCK_SZ, AES_BLOCK_SIZE)
2163 
2164 	SEC_SKCIPHER_ALG("cfb(aes)", sec_setkey_aes_cfb,
2165 			 AES_MIN_KEY_SIZE, AES_MAX_KEY_SIZE,
2166 			 SEC_MIN_BLOCK_SZ, AES_BLOCK_SIZE)
2167 
2168 	SEC_SKCIPHER_ALG("ctr(aes)", sec_setkey_aes_ctr,
2169 			 AES_MIN_KEY_SIZE, AES_MAX_KEY_SIZE,
2170 			 SEC_MIN_BLOCK_SZ, AES_BLOCK_SIZE)
2171 
2172 	SEC_SKCIPHER_ALG("ofb(sm4)", sec_setkey_sm4_ofb,
2173 			 AES_MIN_KEY_SIZE, AES_MIN_KEY_SIZE,
2174 			 SEC_MIN_BLOCK_SZ, AES_BLOCK_SIZE)
2175 
2176 	SEC_SKCIPHER_ALG("cfb(sm4)", sec_setkey_sm4_cfb,
2177 			 AES_MIN_KEY_SIZE, AES_MIN_KEY_SIZE,
2178 			 SEC_MIN_BLOCK_SZ, AES_BLOCK_SIZE)
2179 
2180 	SEC_SKCIPHER_ALG("ctr(sm4)", sec_setkey_sm4_ctr,
2181 			 AES_MIN_KEY_SIZE, AES_MIN_KEY_SIZE,
2182 			 SEC_MIN_BLOCK_SZ, AES_BLOCK_SIZE)
2183 };
2184 
2185 static int aead_iv_demension_check(struct aead_request *aead_req)
2186 {
2187 	u8 cl;
2188 
2189 	cl = aead_req->iv[0] + 1;
2190 	if (cl < IV_CL_MIN || cl > IV_CL_MAX)
2191 		return -EINVAL;
2192 
2193 	if (cl < IV_CL_MID && aead_req->cryptlen >> (BYTE_BITS * cl))
2194 		return -EOVERFLOW;
2195 
2196 	return 0;
2197 }
2198 
2199 static int sec_aead_spec_check(struct sec_ctx *ctx, struct sec_req *sreq)
2200 {
2201 	struct aead_request *req = sreq->aead_req.aead_req;
2202 	struct crypto_aead *tfm = crypto_aead_reqtfm(req);
2203 	size_t authsize = crypto_aead_authsize(tfm);
2204 	u8 c_mode = ctx->c_ctx.c_mode;
2205 	struct device *dev = ctx->dev;
2206 	int ret;
2207 
2208 	if (unlikely(req->cryptlen + req->assoclen > MAX_INPUT_DATA_LEN ||
2209 	    req->assoclen > SEC_MAX_AAD_LEN)) {
2210 		dev_err(dev, "aead input spec error!\n");
2211 		return -EINVAL;
2212 	}
2213 
2214 	if (unlikely((c_mode == SEC_CMODE_GCM && authsize < DES_BLOCK_SIZE) ||
2215 	   (c_mode == SEC_CMODE_CCM && (authsize < MIN_MAC_LEN ||
2216 		authsize & MAC_LEN_MASK)))) {
2217 		dev_err(dev, "aead input mac length error!\n");
2218 		return -EINVAL;
2219 	}
2220 
2221 	if (c_mode == SEC_CMODE_CCM) {
2222 		ret = aead_iv_demension_check(req);
2223 		if (ret) {
2224 			dev_err(dev, "aead input iv param error!\n");
2225 			return ret;
2226 		}
2227 	}
2228 
2229 	if (sreq->c_req.encrypt)
2230 		sreq->c_req.c_len = req->cryptlen;
2231 	else
2232 		sreq->c_req.c_len = req->cryptlen - authsize;
2233 	if (c_mode == SEC_CMODE_CBC) {
2234 		if (unlikely(sreq->c_req.c_len & (AES_BLOCK_SIZE - 1))) {
2235 			dev_err(dev, "aead crypto length error!\n");
2236 			return -EINVAL;
2237 		}
2238 	}
2239 
2240 	return 0;
2241 }
2242 
2243 static int sec_aead_param_check(struct sec_ctx *ctx, struct sec_req *sreq)
2244 {
2245 	struct aead_request *req = sreq->aead_req.aead_req;
2246 	struct crypto_aead *tfm = crypto_aead_reqtfm(req);
2247 	size_t authsize = crypto_aead_authsize(tfm);
2248 	struct device *dev = ctx->dev;
2249 	u8 c_alg = ctx->c_ctx.c_alg;
2250 
2251 	if (unlikely(!req->src || !req->dst)) {
2252 		dev_err(dev, "aead input param error!\n");
2253 		return -EINVAL;
2254 	}
2255 
2256 	if (ctx->sec->qm.ver == QM_HW_V2) {
2257 		if (unlikely(!req->cryptlen || (!sreq->c_req.encrypt &&
2258 		    req->cryptlen <= authsize))) {
2259 			dev_err(dev, "Kunpeng920 not support 0 length!\n");
2260 			ctx->a_ctx.fallback = true;
2261 			return -EINVAL;
2262 		}
2263 	}
2264 
2265 	/* Support AES or SM4 */
2266 	if (unlikely(c_alg != SEC_CALG_AES && c_alg != SEC_CALG_SM4)) {
2267 		dev_err(dev, "aead crypto alg error!\n");
2268 		return -EINVAL;
2269 	}
2270 
2271 	if (unlikely(sec_aead_spec_check(ctx, sreq)))
2272 		return -EINVAL;
2273 
2274 	if (ctx->pbuf_supported && (req->cryptlen + req->assoclen) <=
2275 		SEC_PBUF_SZ)
2276 		sreq->use_pbuf = true;
2277 	else
2278 		sreq->use_pbuf = false;
2279 
2280 	return 0;
2281 }
2282 
2283 static int sec_aead_soft_crypto(struct sec_ctx *ctx,
2284 				struct aead_request *aead_req,
2285 				bool encrypt)
2286 {
2287 	struct aead_request *subreq = aead_request_ctx(aead_req);
2288 	struct sec_auth_ctx *a_ctx = &ctx->a_ctx;
2289 	struct device *dev = ctx->dev;
2290 
2291 	/* Kunpeng920 aead mode not support input 0 size */
2292 	if (!a_ctx->fallback_aead_tfm) {
2293 		dev_err(dev, "aead fallback tfm is NULL!\n");
2294 		return -EINVAL;
2295 	}
2296 
2297 	aead_request_set_tfm(subreq, a_ctx->fallback_aead_tfm);
2298 	aead_request_set_callback(subreq, aead_req->base.flags,
2299 				  aead_req->base.complete, aead_req->base.data);
2300 	aead_request_set_crypt(subreq, aead_req->src, aead_req->dst,
2301 			       aead_req->cryptlen, aead_req->iv);
2302 	aead_request_set_ad(subreq, aead_req->assoclen);
2303 
2304 	return encrypt ? crypto_aead_encrypt(subreq) :
2305 		   crypto_aead_decrypt(subreq);
2306 }
2307 
2308 static int sec_aead_crypto(struct aead_request *a_req, bool encrypt)
2309 {
2310 	struct crypto_aead *tfm = crypto_aead_reqtfm(a_req);
2311 	struct sec_req *req = aead_request_ctx(a_req);
2312 	struct sec_ctx *ctx = crypto_aead_ctx(tfm);
2313 	int ret;
2314 
2315 	req->flag = a_req->base.flags;
2316 	req->aead_req.aead_req = a_req;
2317 	req->c_req.encrypt = encrypt;
2318 	req->ctx = ctx;
2319 
2320 	ret = sec_aead_param_check(ctx, req);
2321 	if (unlikely(ret)) {
2322 		if (ctx->a_ctx.fallback)
2323 			return sec_aead_soft_crypto(ctx, a_req, encrypt);
2324 		return -EINVAL;
2325 	}
2326 
2327 	return ctx->req_op->process(ctx, req);
2328 }
2329 
2330 static int sec_aead_encrypt(struct aead_request *a_req)
2331 {
2332 	return sec_aead_crypto(a_req, true);
2333 }
2334 
2335 static int sec_aead_decrypt(struct aead_request *a_req)
2336 {
2337 	return sec_aead_crypto(a_req, false);
2338 }
2339 
2340 #define SEC_AEAD_ALG(sec_cra_name, sec_set_key, ctx_init,\
2341 			 ctx_exit, blk_size, iv_size, max_authsize)\
2342 {\
2343 	.base = {\
2344 		.cra_name = sec_cra_name,\
2345 		.cra_driver_name = "hisi_sec_"sec_cra_name,\
2346 		.cra_priority = SEC_PRIORITY,\
2347 		.cra_flags = CRYPTO_ALG_ASYNC |\
2348 		 CRYPTO_ALG_ALLOCATES_MEMORY |\
2349 		 CRYPTO_ALG_NEED_FALLBACK,\
2350 		.cra_blocksize = blk_size,\
2351 		.cra_ctxsize = sizeof(struct sec_ctx),\
2352 		.cra_module = THIS_MODULE,\
2353 	},\
2354 	.init = ctx_init,\
2355 	.exit = ctx_exit,\
2356 	.setkey = sec_set_key,\
2357 	.setauthsize = sec_aead_setauthsize,\
2358 	.decrypt = sec_aead_decrypt,\
2359 	.encrypt = sec_aead_encrypt,\
2360 	.ivsize = iv_size,\
2361 	.maxauthsize = max_authsize,\
2362 }
2363 
2364 static struct aead_alg sec_aeads[] = {
2365 	SEC_AEAD_ALG("authenc(hmac(sha1),cbc(aes))",
2366 		     sec_setkey_aes_cbc_sha1, sec_aead_sha1_ctx_init,
2367 		     sec_aead_ctx_exit, AES_BLOCK_SIZE,
2368 		     AES_BLOCK_SIZE, SHA1_DIGEST_SIZE),
2369 
2370 	SEC_AEAD_ALG("authenc(hmac(sha256),cbc(aes))",
2371 		     sec_setkey_aes_cbc_sha256, sec_aead_sha256_ctx_init,
2372 		     sec_aead_ctx_exit, AES_BLOCK_SIZE,
2373 		     AES_BLOCK_SIZE, SHA256_DIGEST_SIZE),
2374 
2375 	SEC_AEAD_ALG("authenc(hmac(sha512),cbc(aes))",
2376 		     sec_setkey_aes_cbc_sha512, sec_aead_sha512_ctx_init,
2377 		     sec_aead_ctx_exit, AES_BLOCK_SIZE,
2378 		     AES_BLOCK_SIZE, SHA512_DIGEST_SIZE),
2379 
2380 	SEC_AEAD_ALG("ccm(aes)", sec_setkey_aes_ccm, sec_aead_xcm_ctx_init,
2381 		     sec_aead_xcm_ctx_exit, SEC_MIN_BLOCK_SZ,
2382 		     AES_BLOCK_SIZE, AES_BLOCK_SIZE),
2383 
2384 	SEC_AEAD_ALG("gcm(aes)", sec_setkey_aes_gcm, sec_aead_xcm_ctx_init,
2385 		     sec_aead_xcm_ctx_exit, SEC_MIN_BLOCK_SZ,
2386 		     SEC_AIV_SIZE, AES_BLOCK_SIZE)
2387 };
2388 
2389 static struct aead_alg sec_aeads_v3[] = {
2390 	SEC_AEAD_ALG("ccm(sm4)", sec_setkey_sm4_ccm, sec_aead_xcm_ctx_init,
2391 		     sec_aead_xcm_ctx_exit, SEC_MIN_BLOCK_SZ,
2392 		     AES_BLOCK_SIZE, AES_BLOCK_SIZE),
2393 
2394 	SEC_AEAD_ALG("gcm(sm4)", sec_setkey_sm4_gcm, sec_aead_xcm_ctx_init,
2395 		     sec_aead_xcm_ctx_exit, SEC_MIN_BLOCK_SZ,
2396 		     SEC_AIV_SIZE, AES_BLOCK_SIZE)
2397 };
2398 
2399 int sec_register_to_crypto(struct hisi_qm *qm)
2400 {
2401 	int ret;
2402 
2403 	/* To avoid repeat register */
2404 	ret = crypto_register_skciphers(sec_skciphers,
2405 					ARRAY_SIZE(sec_skciphers));
2406 	if (ret)
2407 		return ret;
2408 
2409 	if (qm->ver > QM_HW_V2) {
2410 		ret = crypto_register_skciphers(sec_skciphers_v3,
2411 						ARRAY_SIZE(sec_skciphers_v3));
2412 		if (ret)
2413 			goto reg_skcipher_fail;
2414 	}
2415 
2416 	ret = crypto_register_aeads(sec_aeads, ARRAY_SIZE(sec_aeads));
2417 	if (ret)
2418 		goto reg_aead_fail;
2419 	if (qm->ver > QM_HW_V2) {
2420 		ret = crypto_register_aeads(sec_aeads_v3, ARRAY_SIZE(sec_aeads_v3));
2421 		if (ret)
2422 			goto reg_aead_v3_fail;
2423 	}
2424 	return ret;
2425 
2426 reg_aead_v3_fail:
2427 	crypto_unregister_aeads(sec_aeads, ARRAY_SIZE(sec_aeads));
2428 reg_aead_fail:
2429 	if (qm->ver > QM_HW_V2)
2430 		crypto_unregister_skciphers(sec_skciphers_v3,
2431 					    ARRAY_SIZE(sec_skciphers_v3));
2432 reg_skcipher_fail:
2433 	crypto_unregister_skciphers(sec_skciphers,
2434 				    ARRAY_SIZE(sec_skciphers));
2435 	return ret;
2436 }
2437 
2438 void sec_unregister_from_crypto(struct hisi_qm *qm)
2439 {
2440 	if (qm->ver > QM_HW_V2)
2441 		crypto_unregister_aeads(sec_aeads_v3,
2442 					ARRAY_SIZE(sec_aeads_v3));
2443 	crypto_unregister_aeads(sec_aeads, ARRAY_SIZE(sec_aeads));
2444 
2445 	if (qm->ver > QM_HW_V2)
2446 		crypto_unregister_skciphers(sec_skciphers_v3,
2447 					    ARRAY_SIZE(sec_skciphers_v3));
2448 	crypto_unregister_skciphers(sec_skciphers,
2449 				    ARRAY_SIZE(sec_skciphers));
2450 }
2451