1 // SPDX-License-Identifier: GPL-2.0-only
2 /* Copyright (C) 2020 Marvell. */
3 
4 #include <crypto/aes.h>
5 #include <crypto/authenc.h>
6 #include <crypto/cryptd.h>
7 #include <crypto/des.h>
8 #include <crypto/internal/aead.h>
9 #include <crypto/sha1.h>
10 #include <crypto/sha2.h>
11 #include <crypto/xts.h>
12 #include <crypto/gcm.h>
13 #include <crypto/scatterwalk.h>
14 #include <linux/rtnetlink.h>
15 #include <linux/sort.h>
16 #include <linux/module.h>
17 #include "otx2_cptvf.h"
18 #include "otx2_cptvf_algs.h"
19 #include "otx2_cpt_reqmgr.h"
20 
21 /* Size of salt in AES GCM mode */
22 #define AES_GCM_SALT_SIZE 4
23 /* Size of IV in AES GCM mode */
24 #define AES_GCM_IV_SIZE 8
25 /* Size of ICV (Integrity Check Value) in AES GCM mode */
26 #define AES_GCM_ICV_SIZE 16
27 /* Offset of IV in AES GCM mode */
28 #define AES_GCM_IV_OFFSET 8
29 #define CONTROL_WORD_LEN 8
30 #define KEY2_OFFSET 48
31 #define DMA_MODE_FLAG(dma_mode) \
32 	(((dma_mode) == OTX2_CPT_DMA_MODE_SG) ? (1 << 7) : 0)
33 
34 /* Truncated SHA digest size */
35 #define SHA1_TRUNC_DIGEST_SIZE 12
36 #define SHA256_TRUNC_DIGEST_SIZE 16
37 #define SHA384_TRUNC_DIGEST_SIZE 24
38 #define SHA512_TRUNC_DIGEST_SIZE 32
39 
40 static DEFINE_MUTEX(mutex);
41 static int is_crypto_registered;
42 
43 struct cpt_device_desc {
44 	struct pci_dev *dev;
45 	int num_queues;
46 };
47 
48 struct cpt_device_table {
49 	atomic_t count;
50 	struct cpt_device_desc desc[OTX2_CPT_MAX_LFS_NUM];
51 };
52 
53 static struct cpt_device_table se_devices = {
54 	.count = ATOMIC_INIT(0)
55 };
56 
57 static inline int get_se_device(struct pci_dev **pdev, int *cpu_num)
58 {
59 	int count;
60 
61 	count = atomic_read(&se_devices.count);
62 	if (count < 1)
63 		return -ENODEV;
64 
65 	*cpu_num = get_cpu();
66 	/*
67 	 * On OcteonTX2 platform CPT instruction queue is bound to each
68 	 * local function LF, in turn LFs can be attached to PF
69 	 * or VF therefore we always use first device. We get maximum
70 	 * performance if one CPT queue is available for each cpu
71 	 * otherwise CPT queues need to be shared between cpus.
72 	 */
73 	if (*cpu_num >= se_devices.desc[0].num_queues)
74 		*cpu_num %= se_devices.desc[0].num_queues;
75 	*pdev = se_devices.desc[0].dev;
76 
77 	put_cpu();
78 
79 	return 0;
80 }
81 
82 static inline int validate_hmac_cipher_null(struct otx2_cpt_req_info *cpt_req)
83 {
84 	struct otx2_cpt_req_ctx *rctx;
85 	struct aead_request *req;
86 	struct crypto_aead *tfm;
87 
88 	req = container_of(cpt_req->areq, struct aead_request, base);
89 	tfm = crypto_aead_reqtfm(req);
90 	rctx = aead_request_ctx(req);
91 	if (memcmp(rctx->fctx.hmac.s.hmac_calc,
92 		   rctx->fctx.hmac.s.hmac_recv,
93 		   crypto_aead_authsize(tfm)) != 0)
94 		return -EBADMSG;
95 
96 	return 0;
97 }
98 
99 static void otx2_cpt_aead_callback(int status, void *arg1, void *arg2)
100 {
101 	struct otx2_cpt_inst_info *inst_info = arg2;
102 	struct crypto_async_request *areq = arg1;
103 	struct otx2_cpt_req_info *cpt_req;
104 	struct pci_dev *pdev;
105 
106 	if (inst_info) {
107 		cpt_req = inst_info->req;
108 		if (!status) {
109 			/*
110 			 * When selected cipher is NULL we need to manually
111 			 * verify whether calculated hmac value matches
112 			 * received hmac value
113 			 */
114 			if (cpt_req->req_type ==
115 			    OTX2_CPT_AEAD_ENC_DEC_NULL_REQ &&
116 			    !cpt_req->is_enc)
117 				status = validate_hmac_cipher_null(cpt_req);
118 		}
119 		pdev = inst_info->pdev;
120 		otx2_cpt_info_destroy(pdev, inst_info);
121 	}
122 	if (areq)
123 		areq->complete(areq, status);
124 }
125 
126 static void output_iv_copyback(struct crypto_async_request *areq)
127 {
128 	struct otx2_cpt_req_info *req_info;
129 	struct otx2_cpt_req_ctx *rctx;
130 	struct skcipher_request *sreq;
131 	struct crypto_skcipher *stfm;
132 	struct otx2_cpt_enc_ctx *ctx;
133 	u32 start, ivsize;
134 
135 	sreq = container_of(areq, struct skcipher_request, base);
136 	stfm = crypto_skcipher_reqtfm(sreq);
137 	ctx = crypto_skcipher_ctx(stfm);
138 	if (ctx->cipher_type == OTX2_CPT_AES_CBC ||
139 	    ctx->cipher_type == OTX2_CPT_DES3_CBC) {
140 		rctx = skcipher_request_ctx(sreq);
141 		req_info = &rctx->cpt_req;
142 		ivsize = crypto_skcipher_ivsize(stfm);
143 		start = sreq->cryptlen - ivsize;
144 
145 		if (req_info->is_enc) {
146 			scatterwalk_map_and_copy(sreq->iv, sreq->dst, start,
147 						 ivsize, 0);
148 		} else {
149 			if (sreq->src != sreq->dst) {
150 				scatterwalk_map_and_copy(sreq->iv, sreq->src,
151 							 start, ivsize, 0);
152 			} else {
153 				memcpy(sreq->iv, req_info->iv_out, ivsize);
154 				kfree(req_info->iv_out);
155 			}
156 		}
157 	}
158 }
159 
160 static void otx2_cpt_skcipher_callback(int status, void *arg1, void *arg2)
161 {
162 	struct otx2_cpt_inst_info *inst_info = arg2;
163 	struct crypto_async_request *areq = arg1;
164 	struct pci_dev *pdev;
165 
166 	if (areq) {
167 		if (!status)
168 			output_iv_copyback(areq);
169 		if (inst_info) {
170 			pdev = inst_info->pdev;
171 			otx2_cpt_info_destroy(pdev, inst_info);
172 		}
173 		areq->complete(areq, status);
174 	}
175 }
176 
177 static inline void update_input_data(struct otx2_cpt_req_info *req_info,
178 				     struct scatterlist *inp_sg,
179 				     u32 nbytes, u32 *argcnt)
180 {
181 	req_info->req.dlen += nbytes;
182 
183 	while (nbytes) {
184 		u32 len = (nbytes < inp_sg->length) ? nbytes : inp_sg->length;
185 		u8 *ptr = sg_virt(inp_sg);
186 
187 		req_info->in[*argcnt].vptr = (void *)ptr;
188 		req_info->in[*argcnt].size = len;
189 		nbytes -= len;
190 		++(*argcnt);
191 		inp_sg = sg_next(inp_sg);
192 	}
193 }
194 
195 static inline void update_output_data(struct otx2_cpt_req_info *req_info,
196 				      struct scatterlist *outp_sg,
197 				      u32 offset, u32 nbytes, u32 *argcnt)
198 {
199 	u32 len, sg_len;
200 	u8 *ptr;
201 
202 	req_info->rlen += nbytes;
203 
204 	while (nbytes) {
205 		sg_len = outp_sg->length - offset;
206 		len = (nbytes < sg_len) ? nbytes : sg_len;
207 		ptr = sg_virt(outp_sg);
208 
209 		req_info->out[*argcnt].vptr = (void *) (ptr + offset);
210 		req_info->out[*argcnt].size = len;
211 		nbytes -= len;
212 		++(*argcnt);
213 		offset = 0;
214 		outp_sg = sg_next(outp_sg);
215 	}
216 }
217 
218 static inline int create_ctx_hdr(struct skcipher_request *req, u32 enc,
219 				 u32 *argcnt)
220 {
221 	struct crypto_skcipher *stfm = crypto_skcipher_reqtfm(req);
222 	struct otx2_cpt_req_ctx *rctx = skcipher_request_ctx(req);
223 	struct otx2_cpt_enc_ctx *ctx = crypto_skcipher_ctx(stfm);
224 	struct otx2_cpt_req_info *req_info = &rctx->cpt_req;
225 	struct otx2_cpt_fc_ctx *fctx = &rctx->fctx;
226 	int ivsize = crypto_skcipher_ivsize(stfm);
227 	u32 start = req->cryptlen - ivsize;
228 	gfp_t flags;
229 
230 	flags = (req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP) ?
231 			GFP_KERNEL : GFP_ATOMIC;
232 	req_info->ctrl.s.dma_mode = OTX2_CPT_DMA_MODE_SG;
233 	req_info->ctrl.s.se_req = 1;
234 
235 	req_info->req.opcode.s.major = OTX2_CPT_MAJOR_OP_FC |
236 				DMA_MODE_FLAG(OTX2_CPT_DMA_MODE_SG);
237 	if (enc) {
238 		req_info->req.opcode.s.minor = 2;
239 	} else {
240 		req_info->req.opcode.s.minor = 3;
241 		if ((ctx->cipher_type == OTX2_CPT_AES_CBC ||
242 		    ctx->cipher_type == OTX2_CPT_DES3_CBC) &&
243 		    req->src == req->dst) {
244 			req_info->iv_out = kmalloc(ivsize, flags);
245 			if (!req_info->iv_out)
246 				return -ENOMEM;
247 
248 			scatterwalk_map_and_copy(req_info->iv_out, req->src,
249 						 start, ivsize, 0);
250 		}
251 	}
252 	/* Encryption data length */
253 	req_info->req.param1 = req->cryptlen;
254 	/* Authentication data length */
255 	req_info->req.param2 = 0;
256 
257 	fctx->enc.enc_ctrl.e.enc_cipher = ctx->cipher_type;
258 	fctx->enc.enc_ctrl.e.aes_key = ctx->key_type;
259 	fctx->enc.enc_ctrl.e.iv_source = OTX2_CPT_FROM_CPTR;
260 
261 	if (ctx->cipher_type == OTX2_CPT_AES_XTS)
262 		memcpy(fctx->enc.encr_key, ctx->enc_key, ctx->key_len * 2);
263 	else
264 		memcpy(fctx->enc.encr_key, ctx->enc_key, ctx->key_len);
265 
266 	memcpy(fctx->enc.encr_iv, req->iv, crypto_skcipher_ivsize(stfm));
267 
268 	cpu_to_be64s(&fctx->enc.enc_ctrl.u);
269 
270 	/*
271 	 * Storing  Packet Data Information in offset
272 	 * Control Word First 8 bytes
273 	 */
274 	req_info->in[*argcnt].vptr = (u8 *)&rctx->ctrl_word;
275 	req_info->in[*argcnt].size = CONTROL_WORD_LEN;
276 	req_info->req.dlen += CONTROL_WORD_LEN;
277 	++(*argcnt);
278 
279 	req_info->in[*argcnt].vptr = (u8 *)fctx;
280 	req_info->in[*argcnt].size = sizeof(struct otx2_cpt_fc_ctx);
281 	req_info->req.dlen += sizeof(struct otx2_cpt_fc_ctx);
282 
283 	++(*argcnt);
284 
285 	return 0;
286 }
287 
288 static inline int create_input_list(struct skcipher_request *req, u32 enc,
289 				    u32 enc_iv_len)
290 {
291 	struct otx2_cpt_req_ctx *rctx = skcipher_request_ctx(req);
292 	struct otx2_cpt_req_info *req_info = &rctx->cpt_req;
293 	u32 argcnt =  0;
294 	int ret;
295 
296 	ret = create_ctx_hdr(req, enc, &argcnt);
297 	if (ret)
298 		return ret;
299 
300 	update_input_data(req_info, req->src, req->cryptlen, &argcnt);
301 	req_info->in_cnt = argcnt;
302 
303 	return 0;
304 }
305 
306 static inline void create_output_list(struct skcipher_request *req,
307 				      u32 enc_iv_len)
308 {
309 	struct otx2_cpt_req_ctx *rctx = skcipher_request_ctx(req);
310 	struct otx2_cpt_req_info *req_info = &rctx->cpt_req;
311 	u32 argcnt = 0;
312 
313 	/*
314 	 * OUTPUT Buffer Processing
315 	 * AES encryption/decryption output would be
316 	 * received in the following format
317 	 *
318 	 * ------IV--------|------ENCRYPTED/DECRYPTED DATA-----|
319 	 * [ 16 Bytes/     [   Request Enc/Dec/ DATA Len AES CBC ]
320 	 */
321 	update_output_data(req_info, req->dst, 0, req->cryptlen, &argcnt);
322 	req_info->out_cnt = argcnt;
323 }
324 
325 static int skcipher_do_fallback(struct skcipher_request *req, bool is_enc)
326 {
327 	struct crypto_skcipher *stfm = crypto_skcipher_reqtfm(req);
328 	struct otx2_cpt_req_ctx *rctx = skcipher_request_ctx(req);
329 	struct otx2_cpt_enc_ctx *ctx = crypto_skcipher_ctx(stfm);
330 	int ret;
331 
332 	if (ctx->fbk_cipher) {
333 		skcipher_request_set_tfm(&rctx->sk_fbk_req, ctx->fbk_cipher);
334 		skcipher_request_set_callback(&rctx->sk_fbk_req,
335 					      req->base.flags,
336 					      req->base.complete,
337 					      req->base.data);
338 		skcipher_request_set_crypt(&rctx->sk_fbk_req, req->src,
339 					   req->dst, req->cryptlen, req->iv);
340 		ret = is_enc ? crypto_skcipher_encrypt(&rctx->sk_fbk_req) :
341 			       crypto_skcipher_decrypt(&rctx->sk_fbk_req);
342 	} else {
343 		ret = -EINVAL;
344 	}
345 	return ret;
346 }
347 
348 static inline int cpt_enc_dec(struct skcipher_request *req, u32 enc)
349 {
350 	struct crypto_skcipher *stfm = crypto_skcipher_reqtfm(req);
351 	struct otx2_cpt_req_ctx *rctx = skcipher_request_ctx(req);
352 	struct otx2_cpt_enc_ctx *ctx = crypto_skcipher_ctx(stfm);
353 	struct otx2_cpt_req_info *req_info = &rctx->cpt_req;
354 	u32 enc_iv_len = crypto_skcipher_ivsize(stfm);
355 	struct pci_dev *pdev;
356 	int status, cpu_num;
357 
358 	if (req->cryptlen == 0)
359 		return 0;
360 
361 	if (!IS_ALIGNED(req->cryptlen, ctx->enc_align_len))
362 		return -EINVAL;
363 
364 	if (req->cryptlen > OTX2_CPT_MAX_REQ_SIZE)
365 		return skcipher_do_fallback(req, enc);
366 
367 	/* Clear control words */
368 	rctx->ctrl_word.flags = 0;
369 	rctx->fctx.enc.enc_ctrl.u = 0;
370 
371 	status = create_input_list(req, enc, enc_iv_len);
372 	if (status)
373 		return status;
374 	create_output_list(req, enc_iv_len);
375 
376 	status = get_se_device(&pdev, &cpu_num);
377 	if (status)
378 		return status;
379 
380 	req_info->callback = otx2_cpt_skcipher_callback;
381 	req_info->areq = &req->base;
382 	req_info->req_type = OTX2_CPT_ENC_DEC_REQ;
383 	req_info->is_enc = enc;
384 	req_info->is_trunc_hmac = false;
385 	req_info->ctrl.s.grp = otx2_cpt_get_kcrypto_eng_grp_num(pdev);
386 
387 	/*
388 	 * We perform an asynchronous send and once
389 	 * the request is completed the driver would
390 	 * intimate through registered call back functions
391 	 */
392 	status = otx2_cpt_do_request(pdev, req_info, cpu_num);
393 
394 	return status;
395 }
396 
397 static int otx2_cpt_skcipher_encrypt(struct skcipher_request *req)
398 {
399 	return cpt_enc_dec(req, true);
400 }
401 
402 static int otx2_cpt_skcipher_decrypt(struct skcipher_request *req)
403 {
404 	return cpt_enc_dec(req, false);
405 }
406 
407 static int otx2_cpt_skcipher_xts_setkey(struct crypto_skcipher *tfm,
408 				       const u8 *key, u32 keylen)
409 {
410 	struct otx2_cpt_enc_ctx *ctx = crypto_skcipher_ctx(tfm);
411 	const u8 *key2 = key + (keylen / 2);
412 	const u8 *key1 = key;
413 	int ret;
414 
415 	ret = xts_check_key(crypto_skcipher_tfm(tfm), key, keylen);
416 	if (ret)
417 		return ret;
418 	ctx->key_len = keylen;
419 	ctx->enc_align_len = 1;
420 	memcpy(ctx->enc_key, key1, keylen / 2);
421 	memcpy(ctx->enc_key + KEY2_OFFSET, key2, keylen / 2);
422 	ctx->cipher_type = OTX2_CPT_AES_XTS;
423 	switch (ctx->key_len) {
424 	case 2 * AES_KEYSIZE_128:
425 		ctx->key_type = OTX2_CPT_AES_128_BIT;
426 		break;
427 	case 2 * AES_KEYSIZE_192:
428 		ctx->key_type = OTX2_CPT_AES_192_BIT;
429 		break;
430 	case 2 * AES_KEYSIZE_256:
431 		ctx->key_type = OTX2_CPT_AES_256_BIT;
432 		break;
433 	default:
434 		return -EINVAL;
435 	}
436 	return crypto_skcipher_setkey(ctx->fbk_cipher, key, keylen);
437 }
438 
439 static int cpt_des_setkey(struct crypto_skcipher *tfm, const u8 *key,
440 			  u32 keylen, u8 cipher_type)
441 {
442 	struct otx2_cpt_enc_ctx *ctx = crypto_skcipher_ctx(tfm);
443 
444 	if (keylen != DES3_EDE_KEY_SIZE)
445 		return -EINVAL;
446 
447 	ctx->key_len = keylen;
448 	ctx->cipher_type = cipher_type;
449 	ctx->enc_align_len = 8;
450 
451 	memcpy(ctx->enc_key, key, keylen);
452 
453 	return crypto_skcipher_setkey(ctx->fbk_cipher, key, keylen);
454 }
455 
456 static int cpt_aes_setkey(struct crypto_skcipher *tfm, const u8 *key,
457 			  u32 keylen, u8 cipher_type)
458 {
459 	struct otx2_cpt_enc_ctx *ctx = crypto_skcipher_ctx(tfm);
460 
461 	switch (keylen) {
462 	case AES_KEYSIZE_128:
463 		ctx->key_type = OTX2_CPT_AES_128_BIT;
464 		break;
465 	case AES_KEYSIZE_192:
466 		ctx->key_type = OTX2_CPT_AES_192_BIT;
467 		break;
468 	case AES_KEYSIZE_256:
469 		ctx->key_type = OTX2_CPT_AES_256_BIT;
470 		break;
471 	default:
472 		return -EINVAL;
473 	}
474 	if (cipher_type == OTX2_CPT_AES_CBC || cipher_type == OTX2_CPT_AES_ECB)
475 		ctx->enc_align_len = 16;
476 	else
477 		ctx->enc_align_len = 1;
478 
479 	ctx->key_len = keylen;
480 	ctx->cipher_type = cipher_type;
481 
482 	memcpy(ctx->enc_key, key, keylen);
483 
484 	return crypto_skcipher_setkey(ctx->fbk_cipher, key, keylen);
485 }
486 
487 static int otx2_cpt_skcipher_cbc_aes_setkey(struct crypto_skcipher *tfm,
488 					    const u8 *key, u32 keylen)
489 {
490 	return cpt_aes_setkey(tfm, key, keylen, OTX2_CPT_AES_CBC);
491 }
492 
493 static int otx2_cpt_skcipher_ecb_aes_setkey(struct crypto_skcipher *tfm,
494 					    const u8 *key, u32 keylen)
495 {
496 	return cpt_aes_setkey(tfm, key, keylen, OTX2_CPT_AES_ECB);
497 }
498 
499 static int otx2_cpt_skcipher_cbc_des3_setkey(struct crypto_skcipher *tfm,
500 					     const u8 *key, u32 keylen)
501 {
502 	return cpt_des_setkey(tfm, key, keylen, OTX2_CPT_DES3_CBC);
503 }
504 
505 static int otx2_cpt_skcipher_ecb_des3_setkey(struct crypto_skcipher *tfm,
506 					     const u8 *key, u32 keylen)
507 {
508 	return cpt_des_setkey(tfm, key, keylen, OTX2_CPT_DES3_ECB);
509 }
510 
511 static int cpt_skcipher_fallback_init(struct otx2_cpt_enc_ctx *ctx,
512 				      struct crypto_alg *alg)
513 {
514 	if (alg->cra_flags & CRYPTO_ALG_NEED_FALLBACK) {
515 		ctx->fbk_cipher =
516 				crypto_alloc_skcipher(alg->cra_name, 0,
517 						      CRYPTO_ALG_ASYNC |
518 						      CRYPTO_ALG_NEED_FALLBACK);
519 		if (IS_ERR(ctx->fbk_cipher)) {
520 			pr_err("%s() failed to allocate fallback for %s\n",
521 				__func__, alg->cra_name);
522 			return PTR_ERR(ctx->fbk_cipher);
523 		}
524 	}
525 	return 0;
526 }
527 
528 static int otx2_cpt_enc_dec_init(struct crypto_skcipher *stfm)
529 {
530 	struct otx2_cpt_enc_ctx *ctx = crypto_skcipher_ctx(stfm);
531 	struct crypto_tfm *tfm = crypto_skcipher_tfm(stfm);
532 	struct crypto_alg *alg = tfm->__crt_alg;
533 
534 	memset(ctx, 0, sizeof(*ctx));
535 	/*
536 	 * Additional memory for skcipher_request is
537 	 * allocated since the cryptd daemon uses
538 	 * this memory for request_ctx information
539 	 */
540 	crypto_skcipher_set_reqsize(stfm, sizeof(struct otx2_cpt_req_ctx) +
541 					sizeof(struct skcipher_request));
542 
543 	return cpt_skcipher_fallback_init(ctx, alg);
544 }
545 
546 static void otx2_cpt_skcipher_exit(struct crypto_skcipher *tfm)
547 {
548 	struct otx2_cpt_enc_ctx *ctx = crypto_skcipher_ctx(tfm);
549 
550 	if (ctx->fbk_cipher) {
551 		crypto_free_skcipher(ctx->fbk_cipher);
552 		ctx->fbk_cipher = NULL;
553 	}
554 }
555 
556 static int cpt_aead_fallback_init(struct otx2_cpt_aead_ctx *ctx,
557 				  struct crypto_alg *alg)
558 {
559 	if (alg->cra_flags & CRYPTO_ALG_NEED_FALLBACK) {
560 		ctx->fbk_cipher =
561 			    crypto_alloc_aead(alg->cra_name, 0,
562 					      CRYPTO_ALG_ASYNC |
563 					      CRYPTO_ALG_NEED_FALLBACK);
564 		if (IS_ERR(ctx->fbk_cipher)) {
565 			pr_err("%s() failed to allocate fallback for %s\n",
566 				__func__, alg->cra_name);
567 			return PTR_ERR(ctx->fbk_cipher);
568 		}
569 	}
570 	return 0;
571 }
572 
573 static int cpt_aead_init(struct crypto_aead *atfm, u8 cipher_type, u8 mac_type)
574 {
575 	struct otx2_cpt_aead_ctx *ctx = crypto_aead_ctx(atfm);
576 	struct crypto_tfm *tfm = crypto_aead_tfm(atfm);
577 	struct crypto_alg *alg = tfm->__crt_alg;
578 
579 	ctx->cipher_type = cipher_type;
580 	ctx->mac_type = mac_type;
581 
582 	/*
583 	 * When selected cipher is NULL we use HMAC opcode instead of
584 	 * FLEXICRYPTO opcode therefore we don't need to use HASH algorithms
585 	 * for calculating ipad and opad
586 	 */
587 	if (ctx->cipher_type != OTX2_CPT_CIPHER_NULL) {
588 		switch (ctx->mac_type) {
589 		case OTX2_CPT_SHA1:
590 			ctx->hashalg = crypto_alloc_shash("sha1", 0,
591 							  CRYPTO_ALG_ASYNC);
592 			if (IS_ERR(ctx->hashalg))
593 				return PTR_ERR(ctx->hashalg);
594 			break;
595 
596 		case OTX2_CPT_SHA256:
597 			ctx->hashalg = crypto_alloc_shash("sha256", 0,
598 							  CRYPTO_ALG_ASYNC);
599 			if (IS_ERR(ctx->hashalg))
600 				return PTR_ERR(ctx->hashalg);
601 			break;
602 
603 		case OTX2_CPT_SHA384:
604 			ctx->hashalg = crypto_alloc_shash("sha384", 0,
605 							  CRYPTO_ALG_ASYNC);
606 			if (IS_ERR(ctx->hashalg))
607 				return PTR_ERR(ctx->hashalg);
608 			break;
609 
610 		case OTX2_CPT_SHA512:
611 			ctx->hashalg = crypto_alloc_shash("sha512", 0,
612 							  CRYPTO_ALG_ASYNC);
613 			if (IS_ERR(ctx->hashalg))
614 				return PTR_ERR(ctx->hashalg);
615 			break;
616 		}
617 	}
618 	switch (ctx->cipher_type) {
619 	case OTX2_CPT_AES_CBC:
620 	case OTX2_CPT_AES_ECB:
621 		ctx->enc_align_len = 16;
622 		break;
623 	case OTX2_CPT_DES3_CBC:
624 	case OTX2_CPT_DES3_ECB:
625 		ctx->enc_align_len = 8;
626 		break;
627 	case OTX2_CPT_AES_GCM:
628 	case OTX2_CPT_CIPHER_NULL:
629 		ctx->enc_align_len = 1;
630 		break;
631 	}
632 	crypto_aead_set_reqsize(atfm, sizeof(struct otx2_cpt_req_ctx));
633 
634 	return cpt_aead_fallback_init(ctx, alg);
635 }
636 
637 static int otx2_cpt_aead_cbc_aes_sha1_init(struct crypto_aead *tfm)
638 {
639 	return cpt_aead_init(tfm, OTX2_CPT_AES_CBC, OTX2_CPT_SHA1);
640 }
641 
642 static int otx2_cpt_aead_cbc_aes_sha256_init(struct crypto_aead *tfm)
643 {
644 	return cpt_aead_init(tfm, OTX2_CPT_AES_CBC, OTX2_CPT_SHA256);
645 }
646 
647 static int otx2_cpt_aead_cbc_aes_sha384_init(struct crypto_aead *tfm)
648 {
649 	return cpt_aead_init(tfm, OTX2_CPT_AES_CBC, OTX2_CPT_SHA384);
650 }
651 
652 static int otx2_cpt_aead_cbc_aes_sha512_init(struct crypto_aead *tfm)
653 {
654 	return cpt_aead_init(tfm, OTX2_CPT_AES_CBC, OTX2_CPT_SHA512);
655 }
656 
657 static int otx2_cpt_aead_ecb_null_sha1_init(struct crypto_aead *tfm)
658 {
659 	return cpt_aead_init(tfm, OTX2_CPT_CIPHER_NULL, OTX2_CPT_SHA1);
660 }
661 
662 static int otx2_cpt_aead_ecb_null_sha256_init(struct crypto_aead *tfm)
663 {
664 	return cpt_aead_init(tfm, OTX2_CPT_CIPHER_NULL, OTX2_CPT_SHA256);
665 }
666 
667 static int otx2_cpt_aead_ecb_null_sha384_init(struct crypto_aead *tfm)
668 {
669 	return cpt_aead_init(tfm, OTX2_CPT_CIPHER_NULL, OTX2_CPT_SHA384);
670 }
671 
672 static int otx2_cpt_aead_ecb_null_sha512_init(struct crypto_aead *tfm)
673 {
674 	return cpt_aead_init(tfm, OTX2_CPT_CIPHER_NULL, OTX2_CPT_SHA512);
675 }
676 
677 static int otx2_cpt_aead_gcm_aes_init(struct crypto_aead *tfm)
678 {
679 	return cpt_aead_init(tfm, OTX2_CPT_AES_GCM, OTX2_CPT_MAC_NULL);
680 }
681 
682 static void otx2_cpt_aead_exit(struct crypto_aead *tfm)
683 {
684 	struct otx2_cpt_aead_ctx *ctx = crypto_aead_ctx(tfm);
685 
686 	kfree(ctx->ipad);
687 	kfree(ctx->opad);
688 	if (ctx->hashalg)
689 		crypto_free_shash(ctx->hashalg);
690 	kfree(ctx->sdesc);
691 
692 	if (ctx->fbk_cipher) {
693 		crypto_free_aead(ctx->fbk_cipher);
694 		ctx->fbk_cipher = NULL;
695 	}
696 }
697 
698 static int otx2_cpt_aead_gcm_set_authsize(struct crypto_aead *tfm,
699 					  unsigned int authsize)
700 {
701 	struct otx2_cpt_aead_ctx *ctx = crypto_aead_ctx(tfm);
702 
703 	if (crypto_rfc4106_check_authsize(authsize))
704 		return -EINVAL;
705 
706 	tfm->authsize = authsize;
707 	/* Set authsize for fallback case */
708 	if (ctx->fbk_cipher)
709 		ctx->fbk_cipher->authsize = authsize;
710 
711 	return 0;
712 }
713 
714 static int otx2_cpt_aead_set_authsize(struct crypto_aead *tfm,
715 				      unsigned int authsize)
716 {
717 	tfm->authsize = authsize;
718 
719 	return 0;
720 }
721 
722 static int otx2_cpt_aead_null_set_authsize(struct crypto_aead *tfm,
723 					   unsigned int authsize)
724 {
725 	struct otx2_cpt_aead_ctx *ctx = crypto_aead_ctx(tfm);
726 
727 	ctx->is_trunc_hmac = true;
728 	tfm->authsize = authsize;
729 
730 	return 0;
731 }
732 
733 static struct otx2_cpt_sdesc *alloc_sdesc(struct crypto_shash *alg)
734 {
735 	struct otx2_cpt_sdesc *sdesc;
736 	int size;
737 
738 	size = sizeof(struct shash_desc) + crypto_shash_descsize(alg);
739 	sdesc = kmalloc(size, GFP_KERNEL);
740 	if (!sdesc)
741 		return NULL;
742 
743 	sdesc->shash.tfm = alg;
744 
745 	return sdesc;
746 }
747 
748 static inline void swap_data32(void *buf, u32 len)
749 {
750 	cpu_to_be32_array(buf, buf, len / 4);
751 }
752 
753 static inline void swap_data64(void *buf, u32 len)
754 {
755 	u64 *src = buf;
756 	int i = 0;
757 
758 	for (i = 0 ; i < len / 8; i++, src++)
759 		cpu_to_be64s(src);
760 }
761 
762 static int copy_pad(u8 mac_type, u8 *out_pad, u8 *in_pad)
763 {
764 	struct sha512_state *sha512;
765 	struct sha256_state *sha256;
766 	struct sha1_state *sha1;
767 
768 	switch (mac_type) {
769 	case OTX2_CPT_SHA1:
770 		sha1 = (struct sha1_state *) in_pad;
771 		swap_data32(sha1->state, SHA1_DIGEST_SIZE);
772 		memcpy(out_pad, &sha1->state, SHA1_DIGEST_SIZE);
773 		break;
774 
775 	case OTX2_CPT_SHA256:
776 		sha256 = (struct sha256_state *) in_pad;
777 		swap_data32(sha256->state, SHA256_DIGEST_SIZE);
778 		memcpy(out_pad, &sha256->state, SHA256_DIGEST_SIZE);
779 		break;
780 
781 	case OTX2_CPT_SHA384:
782 	case OTX2_CPT_SHA512:
783 		sha512 = (struct sha512_state *) in_pad;
784 		swap_data64(sha512->state, SHA512_DIGEST_SIZE);
785 		memcpy(out_pad, &sha512->state, SHA512_DIGEST_SIZE);
786 		break;
787 
788 	default:
789 		return -EINVAL;
790 	}
791 
792 	return 0;
793 }
794 
795 static int aead_hmac_init(struct crypto_aead *cipher)
796 {
797 	struct otx2_cpt_aead_ctx *ctx = crypto_aead_ctx(cipher);
798 	int state_size = crypto_shash_statesize(ctx->hashalg);
799 	int ds = crypto_shash_digestsize(ctx->hashalg);
800 	int bs = crypto_shash_blocksize(ctx->hashalg);
801 	int authkeylen = ctx->auth_key_len;
802 	u8 *ipad = NULL, *opad = NULL;
803 	int ret = 0, icount = 0;
804 
805 	ctx->sdesc = alloc_sdesc(ctx->hashalg);
806 	if (!ctx->sdesc)
807 		return -ENOMEM;
808 
809 	ctx->ipad = kzalloc(bs, GFP_KERNEL);
810 	if (!ctx->ipad) {
811 		ret = -ENOMEM;
812 		goto calc_fail;
813 	}
814 
815 	ctx->opad = kzalloc(bs, GFP_KERNEL);
816 	if (!ctx->opad) {
817 		ret = -ENOMEM;
818 		goto calc_fail;
819 	}
820 
821 	ipad = kzalloc(state_size, GFP_KERNEL);
822 	if (!ipad) {
823 		ret = -ENOMEM;
824 		goto calc_fail;
825 	}
826 
827 	opad = kzalloc(state_size, GFP_KERNEL);
828 	if (!opad) {
829 		ret = -ENOMEM;
830 		goto calc_fail;
831 	}
832 
833 	if (authkeylen > bs) {
834 		ret = crypto_shash_digest(&ctx->sdesc->shash, ctx->key,
835 					  authkeylen, ipad);
836 		if (ret)
837 			goto calc_fail;
838 
839 		authkeylen = ds;
840 	} else {
841 		memcpy(ipad, ctx->key, authkeylen);
842 	}
843 
844 	memset(ipad + authkeylen, 0, bs - authkeylen);
845 	memcpy(opad, ipad, bs);
846 
847 	for (icount = 0; icount < bs; icount++) {
848 		ipad[icount] ^= 0x36;
849 		opad[icount] ^= 0x5c;
850 	}
851 
852 	/*
853 	 * Partial Hash calculated from the software
854 	 * algorithm is retrieved for IPAD & OPAD
855 	 */
856 
857 	/* IPAD Calculation */
858 	crypto_shash_init(&ctx->sdesc->shash);
859 	crypto_shash_update(&ctx->sdesc->shash, ipad, bs);
860 	crypto_shash_export(&ctx->sdesc->shash, ipad);
861 	ret = copy_pad(ctx->mac_type, ctx->ipad, ipad);
862 	if (ret)
863 		goto calc_fail;
864 
865 	/* OPAD Calculation */
866 	crypto_shash_init(&ctx->sdesc->shash);
867 	crypto_shash_update(&ctx->sdesc->shash, opad, bs);
868 	crypto_shash_export(&ctx->sdesc->shash, opad);
869 	ret = copy_pad(ctx->mac_type, ctx->opad, opad);
870 	if (ret)
871 		goto calc_fail;
872 
873 	kfree(ipad);
874 	kfree(opad);
875 
876 	return 0;
877 
878 calc_fail:
879 	kfree(ctx->ipad);
880 	ctx->ipad = NULL;
881 	kfree(ctx->opad);
882 	ctx->opad = NULL;
883 	kfree(ipad);
884 	kfree(opad);
885 	kfree(ctx->sdesc);
886 	ctx->sdesc = NULL;
887 
888 	return ret;
889 }
890 
891 static int otx2_cpt_aead_cbc_aes_sha_setkey(struct crypto_aead *cipher,
892 					    const unsigned char *key,
893 					    unsigned int keylen)
894 {
895 	struct otx2_cpt_aead_ctx *ctx = crypto_aead_ctx(cipher);
896 	struct crypto_authenc_key_param *param;
897 	int enckeylen = 0, authkeylen = 0;
898 	struct rtattr *rta = (void *)key;
899 	int status;
900 
901 	if (!RTA_OK(rta, keylen))
902 		return -EINVAL;
903 
904 	if (rta->rta_type != CRYPTO_AUTHENC_KEYA_PARAM)
905 		return -EINVAL;
906 
907 	if (RTA_PAYLOAD(rta) < sizeof(*param))
908 		return -EINVAL;
909 
910 	param = RTA_DATA(rta);
911 	enckeylen = be32_to_cpu(param->enckeylen);
912 	key += RTA_ALIGN(rta->rta_len);
913 	keylen -= RTA_ALIGN(rta->rta_len);
914 	if (keylen < enckeylen)
915 		return -EINVAL;
916 
917 	if (keylen > OTX2_CPT_MAX_KEY_SIZE)
918 		return -EINVAL;
919 
920 	authkeylen = keylen - enckeylen;
921 	memcpy(ctx->key, key, keylen);
922 
923 	switch (enckeylen) {
924 	case AES_KEYSIZE_128:
925 		ctx->key_type = OTX2_CPT_AES_128_BIT;
926 		break;
927 	case AES_KEYSIZE_192:
928 		ctx->key_type = OTX2_CPT_AES_192_BIT;
929 		break;
930 	case AES_KEYSIZE_256:
931 		ctx->key_type = OTX2_CPT_AES_256_BIT;
932 		break;
933 	default:
934 		/* Invalid key length */
935 		return -EINVAL;
936 	}
937 
938 	ctx->enc_key_len = enckeylen;
939 	ctx->auth_key_len = authkeylen;
940 
941 	status = aead_hmac_init(cipher);
942 	if (status)
943 		return status;
944 
945 	return 0;
946 }
947 
948 static int otx2_cpt_aead_ecb_null_sha_setkey(struct crypto_aead *cipher,
949 					     const unsigned char *key,
950 					     unsigned int keylen)
951 {
952 	struct otx2_cpt_aead_ctx *ctx = crypto_aead_ctx(cipher);
953 	struct crypto_authenc_key_param *param;
954 	struct rtattr *rta = (void *)key;
955 	int enckeylen = 0;
956 
957 	if (!RTA_OK(rta, keylen))
958 		return -EINVAL;
959 
960 	if (rta->rta_type != CRYPTO_AUTHENC_KEYA_PARAM)
961 		return -EINVAL;
962 
963 	if (RTA_PAYLOAD(rta) < sizeof(*param))
964 		return -EINVAL;
965 
966 	param = RTA_DATA(rta);
967 	enckeylen = be32_to_cpu(param->enckeylen);
968 	key += RTA_ALIGN(rta->rta_len);
969 	keylen -= RTA_ALIGN(rta->rta_len);
970 	if (enckeylen != 0)
971 		return -EINVAL;
972 
973 	if (keylen > OTX2_CPT_MAX_KEY_SIZE)
974 		return -EINVAL;
975 
976 	memcpy(ctx->key, key, keylen);
977 	ctx->enc_key_len = enckeylen;
978 	ctx->auth_key_len = keylen;
979 
980 	return 0;
981 }
982 
983 static int otx2_cpt_aead_gcm_aes_setkey(struct crypto_aead *cipher,
984 					const unsigned char *key,
985 					unsigned int keylen)
986 {
987 	struct otx2_cpt_aead_ctx *ctx = crypto_aead_ctx(cipher);
988 
989 	/*
990 	 * For aes gcm we expect to get encryption key (16, 24, 32 bytes)
991 	 * and salt (4 bytes)
992 	 */
993 	switch (keylen) {
994 	case AES_KEYSIZE_128 + AES_GCM_SALT_SIZE:
995 		ctx->key_type = OTX2_CPT_AES_128_BIT;
996 		ctx->enc_key_len = AES_KEYSIZE_128;
997 		break;
998 	case AES_KEYSIZE_192 + AES_GCM_SALT_SIZE:
999 		ctx->key_type = OTX2_CPT_AES_192_BIT;
1000 		ctx->enc_key_len = AES_KEYSIZE_192;
1001 		break;
1002 	case AES_KEYSIZE_256 + AES_GCM_SALT_SIZE:
1003 		ctx->key_type = OTX2_CPT_AES_256_BIT;
1004 		ctx->enc_key_len = AES_KEYSIZE_256;
1005 		break;
1006 	default:
1007 		/* Invalid key and salt length */
1008 		return -EINVAL;
1009 	}
1010 
1011 	/* Store encryption key and salt */
1012 	memcpy(ctx->key, key, keylen);
1013 
1014 	return crypto_aead_setkey(ctx->fbk_cipher, key, keylen);
1015 }
1016 
1017 static inline int create_aead_ctx_hdr(struct aead_request *req, u32 enc,
1018 				      u32 *argcnt)
1019 {
1020 	struct otx2_cpt_req_ctx *rctx = aead_request_ctx(req);
1021 	struct crypto_aead *tfm = crypto_aead_reqtfm(req);
1022 	struct otx2_cpt_aead_ctx *ctx = crypto_aead_ctx(tfm);
1023 	struct otx2_cpt_req_info *req_info = &rctx->cpt_req;
1024 	struct otx2_cpt_fc_ctx *fctx = &rctx->fctx;
1025 	int mac_len = crypto_aead_authsize(tfm);
1026 	int ds;
1027 
1028 	rctx->ctrl_word.e.enc_data_offset = req->assoclen;
1029 
1030 	switch (ctx->cipher_type) {
1031 	case OTX2_CPT_AES_CBC:
1032 		if (req->assoclen > 248 || !IS_ALIGNED(req->assoclen, 8))
1033 			return -EINVAL;
1034 
1035 		fctx->enc.enc_ctrl.e.iv_source = OTX2_CPT_FROM_CPTR;
1036 		/* Copy encryption key to context */
1037 		memcpy(fctx->enc.encr_key, ctx->key + ctx->auth_key_len,
1038 		       ctx->enc_key_len);
1039 		/* Copy IV to context */
1040 		memcpy(fctx->enc.encr_iv, req->iv, crypto_aead_ivsize(tfm));
1041 
1042 		ds = crypto_shash_digestsize(ctx->hashalg);
1043 		if (ctx->mac_type == OTX2_CPT_SHA384)
1044 			ds = SHA512_DIGEST_SIZE;
1045 		if (ctx->ipad)
1046 			memcpy(fctx->hmac.e.ipad, ctx->ipad, ds);
1047 		if (ctx->opad)
1048 			memcpy(fctx->hmac.e.opad, ctx->opad, ds);
1049 		break;
1050 
1051 	case OTX2_CPT_AES_GCM:
1052 		if (crypto_ipsec_check_assoclen(req->assoclen))
1053 			return -EINVAL;
1054 
1055 		fctx->enc.enc_ctrl.e.iv_source = OTX2_CPT_FROM_DPTR;
1056 		/* Copy encryption key to context */
1057 		memcpy(fctx->enc.encr_key, ctx->key, ctx->enc_key_len);
1058 		/* Copy salt to context */
1059 		memcpy(fctx->enc.encr_iv, ctx->key + ctx->enc_key_len,
1060 		       AES_GCM_SALT_SIZE);
1061 
1062 		rctx->ctrl_word.e.iv_offset = req->assoclen - AES_GCM_IV_OFFSET;
1063 		break;
1064 
1065 	default:
1066 		/* Unknown cipher type */
1067 		return -EINVAL;
1068 	}
1069 	cpu_to_be64s(&rctx->ctrl_word.flags);
1070 
1071 	req_info->ctrl.s.dma_mode = OTX2_CPT_DMA_MODE_SG;
1072 	req_info->ctrl.s.se_req = 1;
1073 	req_info->req.opcode.s.major = OTX2_CPT_MAJOR_OP_FC |
1074 				 DMA_MODE_FLAG(OTX2_CPT_DMA_MODE_SG);
1075 	if (enc) {
1076 		req_info->req.opcode.s.minor = 2;
1077 		req_info->req.param1 = req->cryptlen;
1078 		req_info->req.param2 = req->cryptlen + req->assoclen;
1079 	} else {
1080 		req_info->req.opcode.s.minor = 3;
1081 		req_info->req.param1 = req->cryptlen - mac_len;
1082 		req_info->req.param2 = req->cryptlen + req->assoclen - mac_len;
1083 	}
1084 
1085 	fctx->enc.enc_ctrl.e.enc_cipher = ctx->cipher_type;
1086 	fctx->enc.enc_ctrl.e.aes_key = ctx->key_type;
1087 	fctx->enc.enc_ctrl.e.mac_type = ctx->mac_type;
1088 	fctx->enc.enc_ctrl.e.mac_len = mac_len;
1089 	cpu_to_be64s(&fctx->enc.enc_ctrl.u);
1090 
1091 	/*
1092 	 * Storing Packet Data Information in offset
1093 	 * Control Word First 8 bytes
1094 	 */
1095 	req_info->in[*argcnt].vptr = (u8 *)&rctx->ctrl_word;
1096 	req_info->in[*argcnt].size = CONTROL_WORD_LEN;
1097 	req_info->req.dlen += CONTROL_WORD_LEN;
1098 	++(*argcnt);
1099 
1100 	req_info->in[*argcnt].vptr = (u8 *)fctx;
1101 	req_info->in[*argcnt].size = sizeof(struct otx2_cpt_fc_ctx);
1102 	req_info->req.dlen += sizeof(struct otx2_cpt_fc_ctx);
1103 	++(*argcnt);
1104 
1105 	return 0;
1106 }
1107 
1108 static inline void create_hmac_ctx_hdr(struct aead_request *req, u32 *argcnt,
1109 				      u32 enc)
1110 {
1111 	struct otx2_cpt_req_ctx *rctx = aead_request_ctx(req);
1112 	struct crypto_aead *tfm = crypto_aead_reqtfm(req);
1113 	struct otx2_cpt_aead_ctx *ctx = crypto_aead_ctx(tfm);
1114 	struct otx2_cpt_req_info *req_info = &rctx->cpt_req;
1115 
1116 	req_info->ctrl.s.dma_mode = OTX2_CPT_DMA_MODE_SG;
1117 	req_info->ctrl.s.se_req = 1;
1118 	req_info->req.opcode.s.major = OTX2_CPT_MAJOR_OP_HMAC |
1119 				 DMA_MODE_FLAG(OTX2_CPT_DMA_MODE_SG);
1120 	req_info->is_trunc_hmac = ctx->is_trunc_hmac;
1121 
1122 	req_info->req.opcode.s.minor = 0;
1123 	req_info->req.param1 = ctx->auth_key_len;
1124 	req_info->req.param2 = ctx->mac_type << 8;
1125 
1126 	/* Add authentication key */
1127 	req_info->in[*argcnt].vptr = ctx->key;
1128 	req_info->in[*argcnt].size = round_up(ctx->auth_key_len, 8);
1129 	req_info->req.dlen += round_up(ctx->auth_key_len, 8);
1130 	++(*argcnt);
1131 }
1132 
1133 static inline int create_aead_input_list(struct aead_request *req, u32 enc)
1134 {
1135 	struct otx2_cpt_req_ctx *rctx = aead_request_ctx(req);
1136 	struct otx2_cpt_req_info *req_info = &rctx->cpt_req;
1137 	u32 inputlen =  req->cryptlen + req->assoclen;
1138 	u32 status, argcnt = 0;
1139 
1140 	status = create_aead_ctx_hdr(req, enc, &argcnt);
1141 	if (status)
1142 		return status;
1143 	update_input_data(req_info, req->src, inputlen, &argcnt);
1144 	req_info->in_cnt = argcnt;
1145 
1146 	return 0;
1147 }
1148 
1149 static inline void create_aead_output_list(struct aead_request *req, u32 enc,
1150 					   u32 mac_len)
1151 {
1152 	struct otx2_cpt_req_ctx *rctx = aead_request_ctx(req);
1153 	struct otx2_cpt_req_info *req_info =  &rctx->cpt_req;
1154 	u32 argcnt = 0, outputlen = 0;
1155 
1156 	if (enc)
1157 		outputlen = req->cryptlen +  req->assoclen + mac_len;
1158 	else
1159 		outputlen = req->cryptlen + req->assoclen - mac_len;
1160 
1161 	update_output_data(req_info, req->dst, 0, outputlen, &argcnt);
1162 	req_info->out_cnt = argcnt;
1163 }
1164 
1165 static inline void create_aead_null_input_list(struct aead_request *req,
1166 					       u32 enc, u32 mac_len)
1167 {
1168 	struct otx2_cpt_req_ctx *rctx = aead_request_ctx(req);
1169 	struct otx2_cpt_req_info *req_info = &rctx->cpt_req;
1170 	u32 inputlen, argcnt = 0;
1171 
1172 	if (enc)
1173 		inputlen =  req->cryptlen + req->assoclen;
1174 	else
1175 		inputlen =  req->cryptlen + req->assoclen - mac_len;
1176 
1177 	create_hmac_ctx_hdr(req, &argcnt, enc);
1178 	update_input_data(req_info, req->src, inputlen, &argcnt);
1179 	req_info->in_cnt = argcnt;
1180 }
1181 
1182 static inline int create_aead_null_output_list(struct aead_request *req,
1183 					       u32 enc, u32 mac_len)
1184 {
1185 	struct otx2_cpt_req_ctx *rctx = aead_request_ctx(req);
1186 	struct otx2_cpt_req_info *req_info =  &rctx->cpt_req;
1187 	struct scatterlist *dst;
1188 	u8 *ptr = NULL;
1189 	int argcnt = 0, status, offset;
1190 	u32 inputlen;
1191 
1192 	if (enc)
1193 		inputlen =  req->cryptlen + req->assoclen;
1194 	else
1195 		inputlen =  req->cryptlen + req->assoclen - mac_len;
1196 
1197 	/*
1198 	 * If source and destination are different
1199 	 * then copy payload to destination
1200 	 */
1201 	if (req->src != req->dst) {
1202 
1203 		ptr = kmalloc(inputlen, (req_info->areq->flags &
1204 					 CRYPTO_TFM_REQ_MAY_SLEEP) ?
1205 					 GFP_KERNEL : GFP_ATOMIC);
1206 		if (!ptr)
1207 			return -ENOMEM;
1208 
1209 		status = sg_copy_to_buffer(req->src, sg_nents(req->src), ptr,
1210 					   inputlen);
1211 		if (status != inputlen) {
1212 			status = -EINVAL;
1213 			goto error_free;
1214 		}
1215 		status = sg_copy_from_buffer(req->dst, sg_nents(req->dst), ptr,
1216 					     inputlen);
1217 		if (status != inputlen) {
1218 			status = -EINVAL;
1219 			goto error_free;
1220 		}
1221 		kfree(ptr);
1222 	}
1223 
1224 	if (enc) {
1225 		/*
1226 		 * In an encryption scenario hmac needs
1227 		 * to be appended after payload
1228 		 */
1229 		dst = req->dst;
1230 		offset = inputlen;
1231 		while (offset >= dst->length) {
1232 			offset -= dst->length;
1233 			dst = sg_next(dst);
1234 			if (!dst)
1235 				return -ENOENT;
1236 		}
1237 
1238 		update_output_data(req_info, dst, offset, mac_len, &argcnt);
1239 	} else {
1240 		/*
1241 		 * In a decryption scenario calculated hmac for received
1242 		 * payload needs to be compare with hmac received
1243 		 */
1244 		status = sg_copy_buffer(req->src, sg_nents(req->src),
1245 					rctx->fctx.hmac.s.hmac_recv, mac_len,
1246 					inputlen, true);
1247 		if (status != mac_len)
1248 			return -EINVAL;
1249 
1250 		req_info->out[argcnt].vptr = rctx->fctx.hmac.s.hmac_calc;
1251 		req_info->out[argcnt].size = mac_len;
1252 		argcnt++;
1253 	}
1254 
1255 	req_info->out_cnt = argcnt;
1256 	return 0;
1257 
1258 error_free:
1259 	kfree(ptr);
1260 	return status;
1261 }
1262 
1263 static int aead_do_fallback(struct aead_request *req, bool is_enc)
1264 {
1265 	struct otx2_cpt_req_ctx *rctx = aead_request_ctx(req);
1266 	struct crypto_aead *aead = crypto_aead_reqtfm(req);
1267 	struct otx2_cpt_aead_ctx *ctx = crypto_aead_ctx(aead);
1268 	int ret;
1269 
1270 	if (ctx->fbk_cipher) {
1271 		/* Store the cipher tfm and then use the fallback tfm */
1272 		aead_request_set_tfm(&rctx->fbk_req, ctx->fbk_cipher);
1273 		aead_request_set_callback(&rctx->fbk_req, req->base.flags,
1274 					  req->base.complete, req->base.data);
1275 		aead_request_set_crypt(&rctx->fbk_req, req->src,
1276 				       req->dst, req->cryptlen, req->iv);
1277 		ret = is_enc ? crypto_aead_encrypt(&rctx->fbk_req) :
1278 			       crypto_aead_decrypt(&rctx->fbk_req);
1279 	} else {
1280 		ret = -EINVAL;
1281 	}
1282 
1283 	return ret;
1284 }
1285 
1286 static int cpt_aead_enc_dec(struct aead_request *req, u8 reg_type, u8 enc)
1287 {
1288 	struct otx2_cpt_req_ctx *rctx = aead_request_ctx(req);
1289 	struct otx2_cpt_req_info *req_info = &rctx->cpt_req;
1290 	struct crypto_aead *tfm = crypto_aead_reqtfm(req);
1291 	struct otx2_cpt_aead_ctx *ctx = crypto_aead_ctx(tfm);
1292 	struct pci_dev *pdev;
1293 	int status, cpu_num;
1294 
1295 	/* Clear control words */
1296 	rctx->ctrl_word.flags = 0;
1297 	rctx->fctx.enc.enc_ctrl.u = 0;
1298 
1299 	req_info->callback = otx2_cpt_aead_callback;
1300 	req_info->areq = &req->base;
1301 	req_info->req_type = reg_type;
1302 	req_info->is_enc = enc;
1303 	req_info->is_trunc_hmac = false;
1304 
1305 	switch (reg_type) {
1306 	case OTX2_CPT_AEAD_ENC_DEC_REQ:
1307 		status = create_aead_input_list(req, enc);
1308 		if (status)
1309 			return status;
1310 		create_aead_output_list(req, enc, crypto_aead_authsize(tfm));
1311 		break;
1312 
1313 	case OTX2_CPT_AEAD_ENC_DEC_NULL_REQ:
1314 		create_aead_null_input_list(req, enc,
1315 					    crypto_aead_authsize(tfm));
1316 		status = create_aead_null_output_list(req, enc,
1317 						crypto_aead_authsize(tfm));
1318 		if (status)
1319 			return status;
1320 		break;
1321 
1322 	default:
1323 		return -EINVAL;
1324 	}
1325 	if (!IS_ALIGNED(req_info->req.param1, ctx->enc_align_len))
1326 		return -EINVAL;
1327 
1328 	if (!req_info->req.param2 ||
1329 	    (req_info->req.param1 > OTX2_CPT_MAX_REQ_SIZE) ||
1330 	    (req_info->req.param2 > OTX2_CPT_MAX_REQ_SIZE))
1331 		return aead_do_fallback(req, enc);
1332 
1333 	status = get_se_device(&pdev, &cpu_num);
1334 	if (status)
1335 		return status;
1336 
1337 	req_info->ctrl.s.grp = otx2_cpt_get_kcrypto_eng_grp_num(pdev);
1338 
1339 	/*
1340 	 * We perform an asynchronous send and once
1341 	 * the request is completed the driver would
1342 	 * intimate through registered call back functions
1343 	 */
1344 	return otx2_cpt_do_request(pdev, req_info, cpu_num);
1345 }
1346 
1347 static int otx2_cpt_aead_encrypt(struct aead_request *req)
1348 {
1349 	return cpt_aead_enc_dec(req, OTX2_CPT_AEAD_ENC_DEC_REQ, true);
1350 }
1351 
1352 static int otx2_cpt_aead_decrypt(struct aead_request *req)
1353 {
1354 	return cpt_aead_enc_dec(req, OTX2_CPT_AEAD_ENC_DEC_REQ, false);
1355 }
1356 
1357 static int otx2_cpt_aead_null_encrypt(struct aead_request *req)
1358 {
1359 	return cpt_aead_enc_dec(req, OTX2_CPT_AEAD_ENC_DEC_NULL_REQ, true);
1360 }
1361 
1362 static int otx2_cpt_aead_null_decrypt(struct aead_request *req)
1363 {
1364 	return cpt_aead_enc_dec(req, OTX2_CPT_AEAD_ENC_DEC_NULL_REQ, false);
1365 }
1366 
1367 static struct skcipher_alg otx2_cpt_skciphers[] = { {
1368 	.base.cra_name = "xts(aes)",
1369 	.base.cra_driver_name = "cpt_xts_aes",
1370 	.base.cra_flags = CRYPTO_ALG_ASYNC | CRYPTO_ALG_NEED_FALLBACK,
1371 	.base.cra_blocksize = AES_BLOCK_SIZE,
1372 	.base.cra_ctxsize = sizeof(struct otx2_cpt_enc_ctx),
1373 	.base.cra_alignmask = 7,
1374 	.base.cra_priority = 4001,
1375 	.base.cra_module = THIS_MODULE,
1376 
1377 	.init = otx2_cpt_enc_dec_init,
1378 	.exit = otx2_cpt_skcipher_exit,
1379 	.ivsize = AES_BLOCK_SIZE,
1380 	.min_keysize = 2 * AES_MIN_KEY_SIZE,
1381 	.max_keysize = 2 * AES_MAX_KEY_SIZE,
1382 	.setkey = otx2_cpt_skcipher_xts_setkey,
1383 	.encrypt = otx2_cpt_skcipher_encrypt,
1384 	.decrypt = otx2_cpt_skcipher_decrypt,
1385 }, {
1386 	.base.cra_name = "cbc(aes)",
1387 	.base.cra_driver_name = "cpt_cbc_aes",
1388 	.base.cra_flags = CRYPTO_ALG_ASYNC | CRYPTO_ALG_NEED_FALLBACK,
1389 	.base.cra_blocksize = AES_BLOCK_SIZE,
1390 	.base.cra_ctxsize = sizeof(struct otx2_cpt_enc_ctx),
1391 	.base.cra_alignmask = 7,
1392 	.base.cra_priority = 4001,
1393 	.base.cra_module = THIS_MODULE,
1394 
1395 	.init = otx2_cpt_enc_dec_init,
1396 	.exit = otx2_cpt_skcipher_exit,
1397 	.ivsize = AES_BLOCK_SIZE,
1398 	.min_keysize = AES_MIN_KEY_SIZE,
1399 	.max_keysize = AES_MAX_KEY_SIZE,
1400 	.setkey = otx2_cpt_skcipher_cbc_aes_setkey,
1401 	.encrypt = otx2_cpt_skcipher_encrypt,
1402 	.decrypt = otx2_cpt_skcipher_decrypt,
1403 }, {
1404 	.base.cra_name = "ecb(aes)",
1405 	.base.cra_driver_name = "cpt_ecb_aes",
1406 	.base.cra_flags = CRYPTO_ALG_ASYNC | CRYPTO_ALG_NEED_FALLBACK,
1407 	.base.cra_blocksize = AES_BLOCK_SIZE,
1408 	.base.cra_ctxsize = sizeof(struct otx2_cpt_enc_ctx),
1409 	.base.cra_alignmask = 7,
1410 	.base.cra_priority = 4001,
1411 	.base.cra_module = THIS_MODULE,
1412 
1413 	.init = otx2_cpt_enc_dec_init,
1414 	.exit = otx2_cpt_skcipher_exit,
1415 	.ivsize = 0,
1416 	.min_keysize = AES_MIN_KEY_SIZE,
1417 	.max_keysize = AES_MAX_KEY_SIZE,
1418 	.setkey = otx2_cpt_skcipher_ecb_aes_setkey,
1419 	.encrypt = otx2_cpt_skcipher_encrypt,
1420 	.decrypt = otx2_cpt_skcipher_decrypt,
1421 }, {
1422 	.base.cra_name = "cbc(des3_ede)",
1423 	.base.cra_driver_name = "cpt_cbc_des3_ede",
1424 	.base.cra_flags = CRYPTO_ALG_ASYNC | CRYPTO_ALG_NEED_FALLBACK,
1425 	.base.cra_blocksize = DES3_EDE_BLOCK_SIZE,
1426 	.base.cra_ctxsize = sizeof(struct otx2_cpt_enc_ctx),
1427 	.base.cra_alignmask = 7,
1428 	.base.cra_priority = 4001,
1429 	.base.cra_module = THIS_MODULE,
1430 
1431 	.init = otx2_cpt_enc_dec_init,
1432 	.exit = otx2_cpt_skcipher_exit,
1433 	.min_keysize = DES3_EDE_KEY_SIZE,
1434 	.max_keysize = DES3_EDE_KEY_SIZE,
1435 	.ivsize = DES_BLOCK_SIZE,
1436 	.setkey = otx2_cpt_skcipher_cbc_des3_setkey,
1437 	.encrypt = otx2_cpt_skcipher_encrypt,
1438 	.decrypt = otx2_cpt_skcipher_decrypt,
1439 }, {
1440 	.base.cra_name = "ecb(des3_ede)",
1441 	.base.cra_driver_name = "cpt_ecb_des3_ede",
1442 	.base.cra_flags = CRYPTO_ALG_ASYNC | CRYPTO_ALG_NEED_FALLBACK,
1443 	.base.cra_blocksize = DES3_EDE_BLOCK_SIZE,
1444 	.base.cra_ctxsize = sizeof(struct otx2_cpt_enc_ctx),
1445 	.base.cra_alignmask = 7,
1446 	.base.cra_priority = 4001,
1447 	.base.cra_module = THIS_MODULE,
1448 
1449 	.init = otx2_cpt_enc_dec_init,
1450 	.exit = otx2_cpt_skcipher_exit,
1451 	.min_keysize = DES3_EDE_KEY_SIZE,
1452 	.max_keysize = DES3_EDE_KEY_SIZE,
1453 	.ivsize = 0,
1454 	.setkey = otx2_cpt_skcipher_ecb_des3_setkey,
1455 	.encrypt = otx2_cpt_skcipher_encrypt,
1456 	.decrypt = otx2_cpt_skcipher_decrypt,
1457 } };
1458 
1459 static struct aead_alg otx2_cpt_aeads[] = { {
1460 	.base = {
1461 		.cra_name = "authenc(hmac(sha1),cbc(aes))",
1462 		.cra_driver_name = "cpt_hmac_sha1_cbc_aes",
1463 		.cra_blocksize = AES_BLOCK_SIZE,
1464 		.cra_flags = CRYPTO_ALG_ASYNC | CRYPTO_ALG_NEED_FALLBACK,
1465 		.cra_ctxsize = sizeof(struct otx2_cpt_aead_ctx),
1466 		.cra_priority = 4001,
1467 		.cra_alignmask = 0,
1468 		.cra_module = THIS_MODULE,
1469 	},
1470 	.init = otx2_cpt_aead_cbc_aes_sha1_init,
1471 	.exit = otx2_cpt_aead_exit,
1472 	.setkey = otx2_cpt_aead_cbc_aes_sha_setkey,
1473 	.setauthsize = otx2_cpt_aead_set_authsize,
1474 	.encrypt = otx2_cpt_aead_encrypt,
1475 	.decrypt = otx2_cpt_aead_decrypt,
1476 	.ivsize = AES_BLOCK_SIZE,
1477 	.maxauthsize = SHA1_DIGEST_SIZE,
1478 }, {
1479 	.base = {
1480 		.cra_name = "authenc(hmac(sha256),cbc(aes))",
1481 		.cra_driver_name = "cpt_hmac_sha256_cbc_aes",
1482 		.cra_blocksize = AES_BLOCK_SIZE,
1483 		.cra_flags = CRYPTO_ALG_ASYNC | CRYPTO_ALG_NEED_FALLBACK,
1484 		.cra_ctxsize = sizeof(struct otx2_cpt_aead_ctx),
1485 		.cra_priority = 4001,
1486 		.cra_alignmask = 0,
1487 		.cra_module = THIS_MODULE,
1488 	},
1489 	.init = otx2_cpt_aead_cbc_aes_sha256_init,
1490 	.exit = otx2_cpt_aead_exit,
1491 	.setkey = otx2_cpt_aead_cbc_aes_sha_setkey,
1492 	.setauthsize = otx2_cpt_aead_set_authsize,
1493 	.encrypt = otx2_cpt_aead_encrypt,
1494 	.decrypt = otx2_cpt_aead_decrypt,
1495 	.ivsize = AES_BLOCK_SIZE,
1496 	.maxauthsize = SHA256_DIGEST_SIZE,
1497 }, {
1498 	.base = {
1499 		.cra_name = "authenc(hmac(sha384),cbc(aes))",
1500 		.cra_driver_name = "cpt_hmac_sha384_cbc_aes",
1501 		.cra_blocksize = AES_BLOCK_SIZE,
1502 		.cra_flags = CRYPTO_ALG_ASYNC | CRYPTO_ALG_NEED_FALLBACK,
1503 		.cra_ctxsize = sizeof(struct otx2_cpt_aead_ctx),
1504 		.cra_priority = 4001,
1505 		.cra_alignmask = 0,
1506 		.cra_module = THIS_MODULE,
1507 	},
1508 	.init = otx2_cpt_aead_cbc_aes_sha384_init,
1509 	.exit = otx2_cpt_aead_exit,
1510 	.setkey = otx2_cpt_aead_cbc_aes_sha_setkey,
1511 	.setauthsize = otx2_cpt_aead_set_authsize,
1512 	.encrypt = otx2_cpt_aead_encrypt,
1513 	.decrypt = otx2_cpt_aead_decrypt,
1514 	.ivsize = AES_BLOCK_SIZE,
1515 	.maxauthsize = SHA384_DIGEST_SIZE,
1516 }, {
1517 	.base = {
1518 		.cra_name = "authenc(hmac(sha512),cbc(aes))",
1519 		.cra_driver_name = "cpt_hmac_sha512_cbc_aes",
1520 		.cra_blocksize = AES_BLOCK_SIZE,
1521 		.cra_flags = CRYPTO_ALG_ASYNC | CRYPTO_ALG_NEED_FALLBACK,
1522 		.cra_ctxsize = sizeof(struct otx2_cpt_aead_ctx),
1523 		.cra_priority = 4001,
1524 		.cra_alignmask = 0,
1525 		.cra_module = THIS_MODULE,
1526 	},
1527 	.init = otx2_cpt_aead_cbc_aes_sha512_init,
1528 	.exit = otx2_cpt_aead_exit,
1529 	.setkey = otx2_cpt_aead_cbc_aes_sha_setkey,
1530 	.setauthsize = otx2_cpt_aead_set_authsize,
1531 	.encrypt = otx2_cpt_aead_encrypt,
1532 	.decrypt = otx2_cpt_aead_decrypt,
1533 	.ivsize = AES_BLOCK_SIZE,
1534 	.maxauthsize = SHA512_DIGEST_SIZE,
1535 }, {
1536 	.base = {
1537 		.cra_name = "authenc(hmac(sha1),ecb(cipher_null))",
1538 		.cra_driver_name = "cpt_hmac_sha1_ecb_null",
1539 		.cra_blocksize = 1,
1540 		.cra_flags = CRYPTO_ALG_ASYNC,
1541 		.cra_ctxsize = sizeof(struct otx2_cpt_aead_ctx),
1542 		.cra_priority = 4001,
1543 		.cra_alignmask = 0,
1544 		.cra_module = THIS_MODULE,
1545 	},
1546 	.init = otx2_cpt_aead_ecb_null_sha1_init,
1547 	.exit = otx2_cpt_aead_exit,
1548 	.setkey = otx2_cpt_aead_ecb_null_sha_setkey,
1549 	.setauthsize = otx2_cpt_aead_null_set_authsize,
1550 	.encrypt = otx2_cpt_aead_null_encrypt,
1551 	.decrypt = otx2_cpt_aead_null_decrypt,
1552 	.ivsize = 0,
1553 	.maxauthsize = SHA1_DIGEST_SIZE,
1554 }, {
1555 	.base = {
1556 		.cra_name = "authenc(hmac(sha256),ecb(cipher_null))",
1557 		.cra_driver_name = "cpt_hmac_sha256_ecb_null",
1558 		.cra_blocksize = 1,
1559 		.cra_flags = CRYPTO_ALG_ASYNC,
1560 		.cra_ctxsize = sizeof(struct otx2_cpt_aead_ctx),
1561 		.cra_priority = 4001,
1562 		.cra_alignmask = 0,
1563 		.cra_module = THIS_MODULE,
1564 	},
1565 	.init = otx2_cpt_aead_ecb_null_sha256_init,
1566 	.exit = otx2_cpt_aead_exit,
1567 	.setkey = otx2_cpt_aead_ecb_null_sha_setkey,
1568 	.setauthsize = otx2_cpt_aead_null_set_authsize,
1569 	.encrypt = otx2_cpt_aead_null_encrypt,
1570 	.decrypt = otx2_cpt_aead_null_decrypt,
1571 	.ivsize = 0,
1572 	.maxauthsize = SHA256_DIGEST_SIZE,
1573 }, {
1574 	.base = {
1575 		.cra_name = "authenc(hmac(sha384),ecb(cipher_null))",
1576 		.cra_driver_name = "cpt_hmac_sha384_ecb_null",
1577 		.cra_blocksize = 1,
1578 		.cra_flags = CRYPTO_ALG_ASYNC,
1579 		.cra_ctxsize = sizeof(struct otx2_cpt_aead_ctx),
1580 		.cra_priority = 4001,
1581 		.cra_alignmask = 0,
1582 		.cra_module = THIS_MODULE,
1583 	},
1584 	.init = otx2_cpt_aead_ecb_null_sha384_init,
1585 	.exit = otx2_cpt_aead_exit,
1586 	.setkey = otx2_cpt_aead_ecb_null_sha_setkey,
1587 	.setauthsize = otx2_cpt_aead_null_set_authsize,
1588 	.encrypt = otx2_cpt_aead_null_encrypt,
1589 	.decrypt = otx2_cpt_aead_null_decrypt,
1590 	.ivsize = 0,
1591 	.maxauthsize = SHA384_DIGEST_SIZE,
1592 }, {
1593 	.base = {
1594 		.cra_name = "authenc(hmac(sha512),ecb(cipher_null))",
1595 		.cra_driver_name = "cpt_hmac_sha512_ecb_null",
1596 		.cra_blocksize = 1,
1597 		.cra_flags = CRYPTO_ALG_ASYNC,
1598 		.cra_ctxsize = sizeof(struct otx2_cpt_aead_ctx),
1599 		.cra_priority = 4001,
1600 		.cra_alignmask = 0,
1601 		.cra_module = THIS_MODULE,
1602 	},
1603 	.init = otx2_cpt_aead_ecb_null_sha512_init,
1604 	.exit = otx2_cpt_aead_exit,
1605 	.setkey = otx2_cpt_aead_ecb_null_sha_setkey,
1606 	.setauthsize = otx2_cpt_aead_null_set_authsize,
1607 	.encrypt = otx2_cpt_aead_null_encrypt,
1608 	.decrypt = otx2_cpt_aead_null_decrypt,
1609 	.ivsize = 0,
1610 	.maxauthsize = SHA512_DIGEST_SIZE,
1611 }, {
1612 	.base = {
1613 		.cra_name = "rfc4106(gcm(aes))",
1614 		.cra_driver_name = "cpt_rfc4106_gcm_aes",
1615 		.cra_blocksize = 1,
1616 		.cra_flags = CRYPTO_ALG_ASYNC | CRYPTO_ALG_NEED_FALLBACK,
1617 		.cra_ctxsize = sizeof(struct otx2_cpt_aead_ctx),
1618 		.cra_priority = 4001,
1619 		.cra_alignmask = 0,
1620 		.cra_module = THIS_MODULE,
1621 	},
1622 	.init = otx2_cpt_aead_gcm_aes_init,
1623 	.exit = otx2_cpt_aead_exit,
1624 	.setkey = otx2_cpt_aead_gcm_aes_setkey,
1625 	.setauthsize = otx2_cpt_aead_gcm_set_authsize,
1626 	.encrypt = otx2_cpt_aead_encrypt,
1627 	.decrypt = otx2_cpt_aead_decrypt,
1628 	.ivsize = AES_GCM_IV_SIZE,
1629 	.maxauthsize = AES_GCM_ICV_SIZE,
1630 } };
1631 
1632 static inline int cpt_register_algs(void)
1633 {
1634 	int i, err = 0;
1635 
1636 	if (!IS_ENABLED(CONFIG_DM_CRYPT)) {
1637 		for (i = 0; i < ARRAY_SIZE(otx2_cpt_skciphers); i++)
1638 			otx2_cpt_skciphers[i].base.cra_flags &=
1639 							~CRYPTO_ALG_DEAD;
1640 
1641 		err = crypto_register_skciphers(otx2_cpt_skciphers,
1642 						ARRAY_SIZE(otx2_cpt_skciphers));
1643 		if (err)
1644 			return err;
1645 	}
1646 
1647 	for (i = 0; i < ARRAY_SIZE(otx2_cpt_aeads); i++)
1648 		otx2_cpt_aeads[i].base.cra_flags &= ~CRYPTO_ALG_DEAD;
1649 
1650 	err = crypto_register_aeads(otx2_cpt_aeads,
1651 				    ARRAY_SIZE(otx2_cpt_aeads));
1652 	if (err) {
1653 		crypto_unregister_skciphers(otx2_cpt_skciphers,
1654 					    ARRAY_SIZE(otx2_cpt_skciphers));
1655 		return err;
1656 	}
1657 
1658 	return 0;
1659 }
1660 
1661 static inline void cpt_unregister_algs(void)
1662 {
1663 	crypto_unregister_skciphers(otx2_cpt_skciphers,
1664 				    ARRAY_SIZE(otx2_cpt_skciphers));
1665 	crypto_unregister_aeads(otx2_cpt_aeads, ARRAY_SIZE(otx2_cpt_aeads));
1666 }
1667 
1668 static int compare_func(const void *lptr, const void *rptr)
1669 {
1670 	const struct cpt_device_desc *ldesc = (struct cpt_device_desc *) lptr;
1671 	const struct cpt_device_desc *rdesc = (struct cpt_device_desc *) rptr;
1672 
1673 	if (ldesc->dev->devfn < rdesc->dev->devfn)
1674 		return -1;
1675 	if (ldesc->dev->devfn > rdesc->dev->devfn)
1676 		return 1;
1677 	return 0;
1678 }
1679 
1680 static void swap_func(void *lptr, void *rptr, int size)
1681 {
1682 	struct cpt_device_desc *ldesc = lptr;
1683 	struct cpt_device_desc *rdesc = rptr;
1684 	struct cpt_device_desc desc;
1685 
1686 	desc = *ldesc;
1687 	*ldesc = *rdesc;
1688 	*rdesc = desc;
1689 }
1690 
1691 int otx2_cpt_crypto_init(struct pci_dev *pdev, struct module *mod,
1692 			 int num_queues, int num_devices)
1693 {
1694 	int ret = 0;
1695 	int count;
1696 
1697 	mutex_lock(&mutex);
1698 	count = atomic_read(&se_devices.count);
1699 	if (count >= OTX2_CPT_MAX_LFS_NUM) {
1700 		dev_err(&pdev->dev, "No space to add a new device\n");
1701 		ret = -ENOSPC;
1702 		goto unlock;
1703 	}
1704 	se_devices.desc[count].num_queues = num_queues;
1705 	se_devices.desc[count++].dev = pdev;
1706 	atomic_inc(&se_devices.count);
1707 
1708 	if (atomic_read(&se_devices.count) == num_devices &&
1709 	    is_crypto_registered == false) {
1710 		if (cpt_register_algs()) {
1711 			dev_err(&pdev->dev,
1712 				"Error in registering crypto algorithms\n");
1713 			ret =  -EINVAL;
1714 			goto unlock;
1715 		}
1716 		try_module_get(mod);
1717 		is_crypto_registered = true;
1718 	}
1719 	sort(se_devices.desc, count, sizeof(struct cpt_device_desc),
1720 	     compare_func, swap_func);
1721 
1722 unlock:
1723 	mutex_unlock(&mutex);
1724 	return ret;
1725 }
1726 
1727 void otx2_cpt_crypto_exit(struct pci_dev *pdev, struct module *mod)
1728 {
1729 	struct cpt_device_table *dev_tbl;
1730 	bool dev_found = false;
1731 	int i, j, count;
1732 
1733 	mutex_lock(&mutex);
1734 
1735 	dev_tbl = &se_devices;
1736 	count = atomic_read(&dev_tbl->count);
1737 	for (i = 0; i < count; i++) {
1738 		if (pdev == dev_tbl->desc[i].dev) {
1739 			for (j = i; j < count-1; j++)
1740 				dev_tbl->desc[j] = dev_tbl->desc[j+1];
1741 			dev_found = true;
1742 			break;
1743 		}
1744 	}
1745 
1746 	if (!dev_found) {
1747 		dev_err(&pdev->dev, "%s device not found\n", __func__);
1748 		goto unlock;
1749 	}
1750 	if (atomic_dec_and_test(&se_devices.count)) {
1751 		cpt_unregister_algs();
1752 		module_put(mod);
1753 		is_crypto_registered = false;
1754 	}
1755 
1756 unlock:
1757 	mutex_unlock(&mutex);
1758 }
1759