1 // SPDX-License-Identifier: GPL-2.0-only
2 /* Copyright (C) 2020 Marvell. */
3 
4 #include <crypto/aes.h>
5 #include <crypto/authenc.h>
6 #include <crypto/cryptd.h>
7 #include <crypto/des.h>
8 #include <crypto/internal/aead.h>
9 #include <crypto/sha1.h>
10 #include <crypto/sha2.h>
11 #include <crypto/xts.h>
12 #include <crypto/gcm.h>
13 #include <crypto/scatterwalk.h>
14 #include <linux/rtnetlink.h>
15 #include <linux/sort.h>
16 #include <linux/module.h>
17 #include "otx2_cptvf.h"
18 #include "otx2_cptvf_algs.h"
19 #include "otx2_cpt_reqmgr.h"
20 
21 /* Size of salt in AES GCM mode */
22 #define AES_GCM_SALT_SIZE 4
23 /* Size of IV in AES GCM mode */
24 #define AES_GCM_IV_SIZE 8
25 /* Size of ICV (Integrity Check Value) in AES GCM mode */
26 #define AES_GCM_ICV_SIZE 16
27 /* Offset of IV in AES GCM mode */
28 #define AES_GCM_IV_OFFSET 8
29 #define CONTROL_WORD_LEN 8
30 #define KEY2_OFFSET 48
31 #define DMA_MODE_FLAG(dma_mode) \
32 	(((dma_mode) == OTX2_CPT_DMA_MODE_SG) ? (1 << 7) : 0)
33 
34 /* Truncated SHA digest size */
35 #define SHA1_TRUNC_DIGEST_SIZE 12
36 #define SHA256_TRUNC_DIGEST_SIZE 16
37 #define SHA384_TRUNC_DIGEST_SIZE 24
38 #define SHA512_TRUNC_DIGEST_SIZE 32
39 
40 static DEFINE_MUTEX(mutex);
41 static int is_crypto_registered;
42 
43 struct cpt_device_desc {
44 	struct pci_dev *dev;
45 	int num_queues;
46 };
47 
48 struct cpt_device_table {
49 	atomic_t count;
50 	struct cpt_device_desc desc[OTX2_CPT_MAX_LFS_NUM];
51 };
52 
53 static struct cpt_device_table se_devices = {
54 	.count = ATOMIC_INIT(0)
55 };
56 
57 static inline int get_se_device(struct pci_dev **pdev, int *cpu_num)
58 {
59 	int count;
60 
61 	count = atomic_read(&se_devices.count);
62 	if (count < 1)
63 		return -ENODEV;
64 
65 	*cpu_num = get_cpu();
66 	/*
67 	 * On OcteonTX2 platform CPT instruction queue is bound to each
68 	 * local function LF, in turn LFs can be attached to PF
69 	 * or VF therefore we always use first device. We get maximum
70 	 * performance if one CPT queue is available for each cpu
71 	 * otherwise CPT queues need to be shared between cpus.
72 	 */
73 	if (*cpu_num >= se_devices.desc[0].num_queues)
74 		*cpu_num %= se_devices.desc[0].num_queues;
75 	*pdev = se_devices.desc[0].dev;
76 
77 	put_cpu();
78 
79 	return 0;
80 }
81 
82 static inline int validate_hmac_cipher_null(struct otx2_cpt_req_info *cpt_req)
83 {
84 	struct otx2_cpt_req_ctx *rctx;
85 	struct aead_request *req;
86 	struct crypto_aead *tfm;
87 
88 	req = container_of(cpt_req->areq, struct aead_request, base);
89 	tfm = crypto_aead_reqtfm(req);
90 	rctx = aead_request_ctx(req);
91 	if (memcmp(rctx->fctx.hmac.s.hmac_calc,
92 		   rctx->fctx.hmac.s.hmac_recv,
93 		   crypto_aead_authsize(tfm)) != 0)
94 		return -EBADMSG;
95 
96 	return 0;
97 }
98 
99 static void otx2_cpt_aead_callback(int status, void *arg1, void *arg2)
100 {
101 	struct otx2_cpt_inst_info *inst_info = arg2;
102 	struct crypto_async_request *areq = arg1;
103 	struct otx2_cpt_req_info *cpt_req;
104 	struct pci_dev *pdev;
105 
106 	if (inst_info) {
107 		cpt_req = inst_info->req;
108 		if (!status) {
109 			/*
110 			 * When selected cipher is NULL we need to manually
111 			 * verify whether calculated hmac value matches
112 			 * received hmac value
113 			 */
114 			if (cpt_req->req_type ==
115 			    OTX2_CPT_AEAD_ENC_DEC_NULL_REQ &&
116 			    !cpt_req->is_enc)
117 				status = validate_hmac_cipher_null(cpt_req);
118 		}
119 		pdev = inst_info->pdev;
120 		otx2_cpt_info_destroy(pdev, inst_info);
121 	}
122 	if (areq)
123 		areq->complete(areq, status);
124 }
125 
126 static void output_iv_copyback(struct crypto_async_request *areq)
127 {
128 	struct otx2_cpt_req_info *req_info;
129 	struct otx2_cpt_req_ctx *rctx;
130 	struct skcipher_request *sreq;
131 	struct crypto_skcipher *stfm;
132 	struct otx2_cpt_enc_ctx *ctx;
133 	u32 start, ivsize;
134 
135 	sreq = container_of(areq, struct skcipher_request, base);
136 	stfm = crypto_skcipher_reqtfm(sreq);
137 	ctx = crypto_skcipher_ctx(stfm);
138 	if (ctx->cipher_type == OTX2_CPT_AES_CBC ||
139 	    ctx->cipher_type == OTX2_CPT_DES3_CBC) {
140 		rctx = skcipher_request_ctx(sreq);
141 		req_info = &rctx->cpt_req;
142 		ivsize = crypto_skcipher_ivsize(stfm);
143 		start = sreq->cryptlen - ivsize;
144 
145 		if (req_info->is_enc) {
146 			scatterwalk_map_and_copy(sreq->iv, sreq->dst, start,
147 						 ivsize, 0);
148 		} else {
149 			if (sreq->src != sreq->dst) {
150 				scatterwalk_map_and_copy(sreq->iv, sreq->src,
151 							 start, ivsize, 0);
152 			} else {
153 				memcpy(sreq->iv, req_info->iv_out, ivsize);
154 				kfree(req_info->iv_out);
155 			}
156 		}
157 	}
158 }
159 
160 static void otx2_cpt_skcipher_callback(int status, void *arg1, void *arg2)
161 {
162 	struct otx2_cpt_inst_info *inst_info = arg2;
163 	struct crypto_async_request *areq = arg1;
164 	struct pci_dev *pdev;
165 
166 	if (areq) {
167 		if (!status)
168 			output_iv_copyback(areq);
169 		if (inst_info) {
170 			pdev = inst_info->pdev;
171 			otx2_cpt_info_destroy(pdev, inst_info);
172 		}
173 		areq->complete(areq, status);
174 	}
175 }
176 
177 static inline void update_input_data(struct otx2_cpt_req_info *req_info,
178 				     struct scatterlist *inp_sg,
179 				     u32 nbytes, u32 *argcnt)
180 {
181 	req_info->req.dlen += nbytes;
182 
183 	while (nbytes) {
184 		u32 len = (nbytes < inp_sg->length) ? nbytes : inp_sg->length;
185 		u8 *ptr = sg_virt(inp_sg);
186 
187 		req_info->in[*argcnt].vptr = (void *)ptr;
188 		req_info->in[*argcnt].size = len;
189 		nbytes -= len;
190 		++(*argcnt);
191 		inp_sg = sg_next(inp_sg);
192 	}
193 }
194 
195 static inline void update_output_data(struct otx2_cpt_req_info *req_info,
196 				      struct scatterlist *outp_sg,
197 				      u32 offset, u32 nbytes, u32 *argcnt)
198 {
199 	u32 len, sg_len;
200 	u8 *ptr;
201 
202 	req_info->rlen += nbytes;
203 
204 	while (nbytes) {
205 		sg_len = outp_sg->length - offset;
206 		len = (nbytes < sg_len) ? nbytes : sg_len;
207 		ptr = sg_virt(outp_sg);
208 
209 		req_info->out[*argcnt].vptr = (void *) (ptr + offset);
210 		req_info->out[*argcnt].size = len;
211 		nbytes -= len;
212 		++(*argcnt);
213 		offset = 0;
214 		outp_sg = sg_next(outp_sg);
215 	}
216 }
217 
218 static inline int create_ctx_hdr(struct skcipher_request *req, u32 enc,
219 				 u32 *argcnt)
220 {
221 	struct crypto_skcipher *stfm = crypto_skcipher_reqtfm(req);
222 	struct otx2_cpt_req_ctx *rctx = skcipher_request_ctx(req);
223 	struct otx2_cpt_enc_ctx *ctx = crypto_skcipher_ctx(stfm);
224 	struct otx2_cpt_req_info *req_info = &rctx->cpt_req;
225 	struct otx2_cpt_fc_ctx *fctx = &rctx->fctx;
226 	int ivsize = crypto_skcipher_ivsize(stfm);
227 	u32 start = req->cryptlen - ivsize;
228 	gfp_t flags;
229 
230 	flags = (req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP) ?
231 			GFP_KERNEL : GFP_ATOMIC;
232 	req_info->ctrl.s.dma_mode = OTX2_CPT_DMA_MODE_SG;
233 	req_info->ctrl.s.se_req = 1;
234 
235 	req_info->req.opcode.s.major = OTX2_CPT_MAJOR_OP_FC |
236 				DMA_MODE_FLAG(OTX2_CPT_DMA_MODE_SG);
237 	if (enc) {
238 		req_info->req.opcode.s.minor = 2;
239 	} else {
240 		req_info->req.opcode.s.minor = 3;
241 		if ((ctx->cipher_type == OTX2_CPT_AES_CBC ||
242 		    ctx->cipher_type == OTX2_CPT_DES3_CBC) &&
243 		    req->src == req->dst) {
244 			req_info->iv_out = kmalloc(ivsize, flags);
245 			if (!req_info->iv_out)
246 				return -ENOMEM;
247 
248 			scatterwalk_map_and_copy(req_info->iv_out, req->src,
249 						 start, ivsize, 0);
250 		}
251 	}
252 	/* Encryption data length */
253 	req_info->req.param1 = req->cryptlen;
254 	/* Authentication data length */
255 	req_info->req.param2 = 0;
256 
257 	fctx->enc.enc_ctrl.e.enc_cipher = ctx->cipher_type;
258 	fctx->enc.enc_ctrl.e.aes_key = ctx->key_type;
259 	fctx->enc.enc_ctrl.e.iv_source = OTX2_CPT_FROM_CPTR;
260 
261 	if (ctx->cipher_type == OTX2_CPT_AES_XTS)
262 		memcpy(fctx->enc.encr_key, ctx->enc_key, ctx->key_len * 2);
263 	else
264 		memcpy(fctx->enc.encr_key, ctx->enc_key, ctx->key_len);
265 
266 	memcpy(fctx->enc.encr_iv, req->iv, crypto_skcipher_ivsize(stfm));
267 
268 	cpu_to_be64s(&fctx->enc.enc_ctrl.u);
269 
270 	/*
271 	 * Storing  Packet Data Information in offset
272 	 * Control Word First 8 bytes
273 	 */
274 	req_info->in[*argcnt].vptr = (u8 *)&rctx->ctrl_word;
275 	req_info->in[*argcnt].size = CONTROL_WORD_LEN;
276 	req_info->req.dlen += CONTROL_WORD_LEN;
277 	++(*argcnt);
278 
279 	req_info->in[*argcnt].vptr = (u8 *)fctx;
280 	req_info->in[*argcnt].size = sizeof(struct otx2_cpt_fc_ctx);
281 	req_info->req.dlen += sizeof(struct otx2_cpt_fc_ctx);
282 
283 	++(*argcnt);
284 
285 	return 0;
286 }
287 
288 static inline int create_input_list(struct skcipher_request *req, u32 enc,
289 				    u32 enc_iv_len)
290 {
291 	struct otx2_cpt_req_ctx *rctx = skcipher_request_ctx(req);
292 	struct otx2_cpt_req_info *req_info = &rctx->cpt_req;
293 	u32 argcnt =  0;
294 	int ret;
295 
296 	ret = create_ctx_hdr(req, enc, &argcnt);
297 	if (ret)
298 		return ret;
299 
300 	update_input_data(req_info, req->src, req->cryptlen, &argcnt);
301 	req_info->in_cnt = argcnt;
302 
303 	return 0;
304 }
305 
306 static inline void create_output_list(struct skcipher_request *req,
307 				      u32 enc_iv_len)
308 {
309 	struct otx2_cpt_req_ctx *rctx = skcipher_request_ctx(req);
310 	struct otx2_cpt_req_info *req_info = &rctx->cpt_req;
311 	u32 argcnt = 0;
312 
313 	/*
314 	 * OUTPUT Buffer Processing
315 	 * AES encryption/decryption output would be
316 	 * received in the following format
317 	 *
318 	 * ------IV--------|------ENCRYPTED/DECRYPTED DATA-----|
319 	 * [ 16 Bytes/     [   Request Enc/Dec/ DATA Len AES CBC ]
320 	 */
321 	update_output_data(req_info, req->dst, 0, req->cryptlen, &argcnt);
322 	req_info->out_cnt = argcnt;
323 }
324 
325 static int skcipher_do_fallback(struct skcipher_request *req, bool is_enc)
326 {
327 	struct crypto_skcipher *stfm = crypto_skcipher_reqtfm(req);
328 	struct otx2_cpt_req_ctx *rctx = skcipher_request_ctx(req);
329 	struct otx2_cpt_enc_ctx *ctx = crypto_skcipher_ctx(stfm);
330 	int ret;
331 
332 	if (ctx->fbk_cipher) {
333 		skcipher_request_set_tfm(&rctx->sk_fbk_req, ctx->fbk_cipher);
334 		skcipher_request_set_callback(&rctx->sk_fbk_req,
335 					      req->base.flags,
336 					      req->base.complete,
337 					      req->base.data);
338 		skcipher_request_set_crypt(&rctx->sk_fbk_req, req->src,
339 					   req->dst, req->cryptlen, req->iv);
340 		ret = is_enc ? crypto_skcipher_encrypt(&rctx->sk_fbk_req) :
341 			       crypto_skcipher_decrypt(&rctx->sk_fbk_req);
342 	} else {
343 		ret = -EINVAL;
344 	}
345 	return ret;
346 }
347 
348 static inline int cpt_enc_dec(struct skcipher_request *req, u32 enc)
349 {
350 	struct crypto_skcipher *stfm = crypto_skcipher_reqtfm(req);
351 	struct otx2_cpt_req_ctx *rctx = skcipher_request_ctx(req);
352 	struct otx2_cpt_enc_ctx *ctx = crypto_skcipher_ctx(stfm);
353 	struct otx2_cpt_req_info *req_info = &rctx->cpt_req;
354 	u32 enc_iv_len = crypto_skcipher_ivsize(stfm);
355 	struct pci_dev *pdev;
356 	int status, cpu_num;
357 
358 	if (req->cryptlen == 0)
359 		return 0;
360 
361 	if (!IS_ALIGNED(req->cryptlen, ctx->enc_align_len))
362 		return -EINVAL;
363 
364 	if (req->cryptlen > OTX2_CPT_MAX_REQ_SIZE)
365 		return skcipher_do_fallback(req, enc);
366 
367 	/* Clear control words */
368 	rctx->ctrl_word.flags = 0;
369 	rctx->fctx.enc.enc_ctrl.u = 0;
370 
371 	status = create_input_list(req, enc, enc_iv_len);
372 	if (status)
373 		return status;
374 	create_output_list(req, enc_iv_len);
375 
376 	status = get_se_device(&pdev, &cpu_num);
377 	if (status)
378 		return status;
379 
380 	req_info->callback = otx2_cpt_skcipher_callback;
381 	req_info->areq = &req->base;
382 	req_info->req_type = OTX2_CPT_ENC_DEC_REQ;
383 	req_info->is_enc = enc;
384 	req_info->is_trunc_hmac = false;
385 	req_info->ctrl.s.grp = otx2_cpt_get_kcrypto_eng_grp_num(pdev);
386 
387 	/*
388 	 * We perform an asynchronous send and once
389 	 * the request is completed the driver would
390 	 * intimate through registered call back functions
391 	 */
392 	status = otx2_cpt_do_request(pdev, req_info, cpu_num);
393 
394 	return status;
395 }
396 
397 static int otx2_cpt_skcipher_encrypt(struct skcipher_request *req)
398 {
399 	return cpt_enc_dec(req, true);
400 }
401 
402 static int otx2_cpt_skcipher_decrypt(struct skcipher_request *req)
403 {
404 	return cpt_enc_dec(req, false);
405 }
406 
407 static int otx2_cpt_skcipher_xts_setkey(struct crypto_skcipher *tfm,
408 				       const u8 *key, u32 keylen)
409 {
410 	struct otx2_cpt_enc_ctx *ctx = crypto_skcipher_ctx(tfm);
411 	const u8 *key2 = key + (keylen / 2);
412 	const u8 *key1 = key;
413 	int ret;
414 
415 	ret = xts_check_key(crypto_skcipher_tfm(tfm), key, keylen);
416 	if (ret)
417 		return ret;
418 	ctx->key_len = keylen;
419 	ctx->enc_align_len = 1;
420 	memcpy(ctx->enc_key, key1, keylen / 2);
421 	memcpy(ctx->enc_key + KEY2_OFFSET, key2, keylen / 2);
422 	ctx->cipher_type = OTX2_CPT_AES_XTS;
423 	switch (ctx->key_len) {
424 	case 2 * AES_KEYSIZE_128:
425 		ctx->key_type = OTX2_CPT_AES_128_BIT;
426 		break;
427 	case 2 * AES_KEYSIZE_192:
428 		ctx->key_type = OTX2_CPT_AES_192_BIT;
429 		break;
430 	case 2 * AES_KEYSIZE_256:
431 		ctx->key_type = OTX2_CPT_AES_256_BIT;
432 		break;
433 	default:
434 		return -EINVAL;
435 	}
436 	return crypto_skcipher_setkey(ctx->fbk_cipher, key, keylen);
437 }
438 
439 static int cpt_des_setkey(struct crypto_skcipher *tfm, const u8 *key,
440 			  u32 keylen, u8 cipher_type)
441 {
442 	struct otx2_cpt_enc_ctx *ctx = crypto_skcipher_ctx(tfm);
443 
444 	if (keylen != DES3_EDE_KEY_SIZE)
445 		return -EINVAL;
446 
447 	ctx->key_len = keylen;
448 	ctx->cipher_type = cipher_type;
449 	ctx->enc_align_len = 8;
450 
451 	memcpy(ctx->enc_key, key, keylen);
452 
453 	return crypto_skcipher_setkey(ctx->fbk_cipher, key, keylen);
454 }
455 
456 static int cpt_aes_setkey(struct crypto_skcipher *tfm, const u8 *key,
457 			  u32 keylen, u8 cipher_type)
458 {
459 	struct otx2_cpt_enc_ctx *ctx = crypto_skcipher_ctx(tfm);
460 
461 	switch (keylen) {
462 	case AES_KEYSIZE_128:
463 		ctx->key_type = OTX2_CPT_AES_128_BIT;
464 		break;
465 	case AES_KEYSIZE_192:
466 		ctx->key_type = OTX2_CPT_AES_192_BIT;
467 		break;
468 	case AES_KEYSIZE_256:
469 		ctx->key_type = OTX2_CPT_AES_256_BIT;
470 		break;
471 	default:
472 		return -EINVAL;
473 	}
474 	if (cipher_type == OTX2_CPT_AES_CBC || cipher_type == OTX2_CPT_AES_ECB)
475 		ctx->enc_align_len = 16;
476 	else
477 		ctx->enc_align_len = 1;
478 
479 	ctx->key_len = keylen;
480 	ctx->cipher_type = cipher_type;
481 
482 	memcpy(ctx->enc_key, key, keylen);
483 
484 	return crypto_skcipher_setkey(ctx->fbk_cipher, key, keylen);
485 }
486 
487 static int otx2_cpt_skcipher_cbc_aes_setkey(struct crypto_skcipher *tfm,
488 					    const u8 *key, u32 keylen)
489 {
490 	return cpt_aes_setkey(tfm, key, keylen, OTX2_CPT_AES_CBC);
491 }
492 
493 static int otx2_cpt_skcipher_ecb_aes_setkey(struct crypto_skcipher *tfm,
494 					    const u8 *key, u32 keylen)
495 {
496 	return cpt_aes_setkey(tfm, key, keylen, OTX2_CPT_AES_ECB);
497 }
498 
499 static int otx2_cpt_skcipher_cbc_des3_setkey(struct crypto_skcipher *tfm,
500 					     const u8 *key, u32 keylen)
501 {
502 	return cpt_des_setkey(tfm, key, keylen, OTX2_CPT_DES3_CBC);
503 }
504 
505 static int otx2_cpt_skcipher_ecb_des3_setkey(struct crypto_skcipher *tfm,
506 					     const u8 *key, u32 keylen)
507 {
508 	return cpt_des_setkey(tfm, key, keylen, OTX2_CPT_DES3_ECB);
509 }
510 
511 static int cpt_skcipher_fallback_init(struct otx2_cpt_enc_ctx *ctx,
512 				      struct crypto_alg *alg)
513 {
514 	if (alg->cra_flags & CRYPTO_ALG_NEED_FALLBACK) {
515 		ctx->fbk_cipher =
516 				crypto_alloc_skcipher(alg->cra_name, 0,
517 						      CRYPTO_ALG_ASYNC |
518 						      CRYPTO_ALG_NEED_FALLBACK);
519 		if (IS_ERR(ctx->fbk_cipher)) {
520 			pr_err("%s() failed to allocate fallback for %s\n",
521 				__func__, alg->cra_name);
522 			return PTR_ERR(ctx->fbk_cipher);
523 		}
524 	}
525 	return 0;
526 }
527 
528 static int otx2_cpt_enc_dec_init(struct crypto_skcipher *stfm)
529 {
530 	struct otx2_cpt_enc_ctx *ctx = crypto_skcipher_ctx(stfm);
531 	struct crypto_tfm *tfm = crypto_skcipher_tfm(stfm);
532 	struct crypto_alg *alg = tfm->__crt_alg;
533 
534 	memset(ctx, 0, sizeof(*ctx));
535 	/*
536 	 * Additional memory for skcipher_request is
537 	 * allocated since the cryptd daemon uses
538 	 * this memory for request_ctx information
539 	 */
540 	crypto_skcipher_set_reqsize(stfm, sizeof(struct otx2_cpt_req_ctx) +
541 					sizeof(struct skcipher_request));
542 
543 	return cpt_skcipher_fallback_init(ctx, alg);
544 }
545 
546 static void otx2_cpt_skcipher_exit(struct crypto_skcipher *tfm)
547 {
548 	struct otx2_cpt_enc_ctx *ctx = crypto_skcipher_ctx(tfm);
549 
550 	if (ctx->fbk_cipher) {
551 		crypto_free_skcipher(ctx->fbk_cipher);
552 		ctx->fbk_cipher = NULL;
553 	}
554 }
555 
556 static int cpt_aead_fallback_init(struct otx2_cpt_aead_ctx *ctx,
557 				  struct crypto_alg *alg)
558 {
559 	if (alg->cra_flags & CRYPTO_ALG_NEED_FALLBACK) {
560 		ctx->fbk_cipher =
561 			    crypto_alloc_aead(alg->cra_name, 0,
562 					      CRYPTO_ALG_ASYNC |
563 					      CRYPTO_ALG_NEED_FALLBACK);
564 		if (IS_ERR(ctx->fbk_cipher)) {
565 			pr_err("%s() failed to allocate fallback for %s\n",
566 				__func__, alg->cra_name);
567 			return PTR_ERR(ctx->fbk_cipher);
568 		}
569 	}
570 	return 0;
571 }
572 
573 static int cpt_aead_init(struct crypto_aead *atfm, u8 cipher_type, u8 mac_type)
574 {
575 	struct otx2_cpt_aead_ctx *ctx = crypto_aead_ctx(atfm);
576 	struct crypto_tfm *tfm = crypto_aead_tfm(atfm);
577 	struct crypto_alg *alg = tfm->__crt_alg;
578 
579 	ctx->cipher_type = cipher_type;
580 	ctx->mac_type = mac_type;
581 
582 	/*
583 	 * When selected cipher is NULL we use HMAC opcode instead of
584 	 * FLEXICRYPTO opcode therefore we don't need to use HASH algorithms
585 	 * for calculating ipad and opad
586 	 */
587 	if (ctx->cipher_type != OTX2_CPT_CIPHER_NULL) {
588 		switch (ctx->mac_type) {
589 		case OTX2_CPT_SHA1:
590 			ctx->hashalg = crypto_alloc_shash("sha1", 0,
591 							  CRYPTO_ALG_ASYNC);
592 			if (IS_ERR(ctx->hashalg))
593 				return PTR_ERR(ctx->hashalg);
594 			break;
595 
596 		case OTX2_CPT_SHA256:
597 			ctx->hashalg = crypto_alloc_shash("sha256", 0,
598 							  CRYPTO_ALG_ASYNC);
599 			if (IS_ERR(ctx->hashalg))
600 				return PTR_ERR(ctx->hashalg);
601 			break;
602 
603 		case OTX2_CPT_SHA384:
604 			ctx->hashalg = crypto_alloc_shash("sha384", 0,
605 							  CRYPTO_ALG_ASYNC);
606 			if (IS_ERR(ctx->hashalg))
607 				return PTR_ERR(ctx->hashalg);
608 			break;
609 
610 		case OTX2_CPT_SHA512:
611 			ctx->hashalg = crypto_alloc_shash("sha512", 0,
612 							  CRYPTO_ALG_ASYNC);
613 			if (IS_ERR(ctx->hashalg))
614 				return PTR_ERR(ctx->hashalg);
615 			break;
616 		}
617 	}
618 	switch (ctx->cipher_type) {
619 	case OTX2_CPT_AES_CBC:
620 	case OTX2_CPT_AES_ECB:
621 		ctx->enc_align_len = 16;
622 		break;
623 	case OTX2_CPT_DES3_CBC:
624 	case OTX2_CPT_DES3_ECB:
625 		ctx->enc_align_len = 8;
626 		break;
627 	case OTX2_CPT_AES_GCM:
628 	case OTX2_CPT_CIPHER_NULL:
629 		ctx->enc_align_len = 1;
630 		break;
631 	}
632 	crypto_aead_set_reqsize(atfm, sizeof(struct otx2_cpt_req_ctx));
633 
634 	return cpt_aead_fallback_init(ctx, alg);
635 }
636 
637 static int otx2_cpt_aead_cbc_aes_sha1_init(struct crypto_aead *tfm)
638 {
639 	return cpt_aead_init(tfm, OTX2_CPT_AES_CBC, OTX2_CPT_SHA1);
640 }
641 
642 static int otx2_cpt_aead_cbc_aes_sha256_init(struct crypto_aead *tfm)
643 {
644 	return cpt_aead_init(tfm, OTX2_CPT_AES_CBC, OTX2_CPT_SHA256);
645 }
646 
647 static int otx2_cpt_aead_cbc_aes_sha384_init(struct crypto_aead *tfm)
648 {
649 	return cpt_aead_init(tfm, OTX2_CPT_AES_CBC, OTX2_CPT_SHA384);
650 }
651 
652 static int otx2_cpt_aead_cbc_aes_sha512_init(struct crypto_aead *tfm)
653 {
654 	return cpt_aead_init(tfm, OTX2_CPT_AES_CBC, OTX2_CPT_SHA512);
655 }
656 
657 static int otx2_cpt_aead_ecb_null_sha1_init(struct crypto_aead *tfm)
658 {
659 	return cpt_aead_init(tfm, OTX2_CPT_CIPHER_NULL, OTX2_CPT_SHA1);
660 }
661 
662 static int otx2_cpt_aead_ecb_null_sha256_init(struct crypto_aead *tfm)
663 {
664 	return cpt_aead_init(tfm, OTX2_CPT_CIPHER_NULL, OTX2_CPT_SHA256);
665 }
666 
667 static int otx2_cpt_aead_ecb_null_sha384_init(struct crypto_aead *tfm)
668 {
669 	return cpt_aead_init(tfm, OTX2_CPT_CIPHER_NULL, OTX2_CPT_SHA384);
670 }
671 
672 static int otx2_cpt_aead_ecb_null_sha512_init(struct crypto_aead *tfm)
673 {
674 	return cpt_aead_init(tfm, OTX2_CPT_CIPHER_NULL, OTX2_CPT_SHA512);
675 }
676 
677 static int otx2_cpt_aead_gcm_aes_init(struct crypto_aead *tfm)
678 {
679 	return cpt_aead_init(tfm, OTX2_CPT_AES_GCM, OTX2_CPT_MAC_NULL);
680 }
681 
682 static void otx2_cpt_aead_exit(struct crypto_aead *tfm)
683 {
684 	struct otx2_cpt_aead_ctx *ctx = crypto_aead_ctx(tfm);
685 
686 	kfree(ctx->ipad);
687 	kfree(ctx->opad);
688 	if (ctx->hashalg)
689 		crypto_free_shash(ctx->hashalg);
690 	kfree(ctx->sdesc);
691 
692 	if (ctx->fbk_cipher) {
693 		crypto_free_aead(ctx->fbk_cipher);
694 		ctx->fbk_cipher = NULL;
695 	}
696 }
697 
698 static int otx2_cpt_aead_gcm_set_authsize(struct crypto_aead *tfm,
699 					  unsigned int authsize)
700 {
701 	struct otx2_cpt_aead_ctx *ctx = crypto_aead_ctx(tfm);
702 
703 	if (crypto_rfc4106_check_authsize(authsize))
704 		return -EINVAL;
705 
706 	tfm->authsize = authsize;
707 	/* Set authsize for fallback case */
708 	if (ctx->fbk_cipher)
709 		ctx->fbk_cipher->authsize = authsize;
710 
711 	return 0;
712 }
713 
714 static int otx2_cpt_aead_set_authsize(struct crypto_aead *tfm,
715 				      unsigned int authsize)
716 {
717 	tfm->authsize = authsize;
718 
719 	return 0;
720 }
721 
722 static int otx2_cpt_aead_null_set_authsize(struct crypto_aead *tfm,
723 					   unsigned int authsize)
724 {
725 	struct otx2_cpt_aead_ctx *ctx = crypto_aead_ctx(tfm);
726 
727 	ctx->is_trunc_hmac = true;
728 	tfm->authsize = authsize;
729 
730 	return 0;
731 }
732 
733 static struct otx2_cpt_sdesc *alloc_sdesc(struct crypto_shash *alg)
734 {
735 	struct otx2_cpt_sdesc *sdesc;
736 	int size;
737 
738 	size = sizeof(struct shash_desc) + crypto_shash_descsize(alg);
739 	sdesc = kmalloc(size, GFP_KERNEL);
740 	if (!sdesc)
741 		return NULL;
742 
743 	sdesc->shash.tfm = alg;
744 
745 	return sdesc;
746 }
747 
748 static inline void swap_data32(void *buf, u32 len)
749 {
750 	cpu_to_be32_array(buf, buf, len / 4);
751 }
752 
753 static inline void swap_data64(void *buf, u32 len)
754 {
755 	u64 *src = buf;
756 	int i = 0;
757 
758 	for (i = 0 ; i < len / 8; i++, src++)
759 		cpu_to_be64s(src);
760 }
761 
762 static int copy_pad(u8 mac_type, u8 *out_pad, u8 *in_pad)
763 {
764 	struct sha512_state *sha512;
765 	struct sha256_state *sha256;
766 	struct sha1_state *sha1;
767 
768 	switch (mac_type) {
769 	case OTX2_CPT_SHA1:
770 		sha1 = (struct sha1_state *) in_pad;
771 		swap_data32(sha1->state, SHA1_DIGEST_SIZE);
772 		memcpy(out_pad, &sha1->state, SHA1_DIGEST_SIZE);
773 		break;
774 
775 	case OTX2_CPT_SHA256:
776 		sha256 = (struct sha256_state *) in_pad;
777 		swap_data32(sha256->state, SHA256_DIGEST_SIZE);
778 		memcpy(out_pad, &sha256->state, SHA256_DIGEST_SIZE);
779 		break;
780 
781 	case OTX2_CPT_SHA384:
782 	case OTX2_CPT_SHA512:
783 		sha512 = (struct sha512_state *) in_pad;
784 		swap_data64(sha512->state, SHA512_DIGEST_SIZE);
785 		memcpy(out_pad, &sha512->state, SHA512_DIGEST_SIZE);
786 		break;
787 
788 	default:
789 		return -EINVAL;
790 	}
791 
792 	return 0;
793 }
794 
795 static int aead_hmac_init(struct crypto_aead *cipher)
796 {
797 	struct otx2_cpt_aead_ctx *ctx = crypto_aead_ctx(cipher);
798 	int state_size = crypto_shash_statesize(ctx->hashalg);
799 	int ds = crypto_shash_digestsize(ctx->hashalg);
800 	int bs = crypto_shash_blocksize(ctx->hashalg);
801 	int authkeylen = ctx->auth_key_len;
802 	u8 *ipad = NULL, *opad = NULL;
803 	int ret = 0, icount = 0;
804 
805 	ctx->sdesc = alloc_sdesc(ctx->hashalg);
806 	if (!ctx->sdesc)
807 		return -ENOMEM;
808 
809 	ctx->ipad = kzalloc(bs, GFP_KERNEL);
810 	if (!ctx->ipad) {
811 		ret = -ENOMEM;
812 		goto calc_fail;
813 	}
814 
815 	ctx->opad = kzalloc(bs, GFP_KERNEL);
816 	if (!ctx->opad) {
817 		ret = -ENOMEM;
818 		goto calc_fail;
819 	}
820 
821 	ipad = kzalloc(state_size, GFP_KERNEL);
822 	if (!ipad) {
823 		ret = -ENOMEM;
824 		goto calc_fail;
825 	}
826 
827 	opad = kzalloc(state_size, GFP_KERNEL);
828 	if (!opad) {
829 		ret = -ENOMEM;
830 		goto calc_fail;
831 	}
832 
833 	if (authkeylen > bs) {
834 		ret = crypto_shash_digest(&ctx->sdesc->shash, ctx->key,
835 					  authkeylen, ipad);
836 		if (ret)
837 			goto calc_fail;
838 
839 		authkeylen = ds;
840 	} else {
841 		memcpy(ipad, ctx->key, authkeylen);
842 	}
843 
844 	memset(ipad + authkeylen, 0, bs - authkeylen);
845 	memcpy(opad, ipad, bs);
846 
847 	for (icount = 0; icount < bs; icount++) {
848 		ipad[icount] ^= 0x36;
849 		opad[icount] ^= 0x5c;
850 	}
851 
852 	/*
853 	 * Partial Hash calculated from the software
854 	 * algorithm is retrieved for IPAD & OPAD
855 	 */
856 
857 	/* IPAD Calculation */
858 	crypto_shash_init(&ctx->sdesc->shash);
859 	crypto_shash_update(&ctx->sdesc->shash, ipad, bs);
860 	crypto_shash_export(&ctx->sdesc->shash, ipad);
861 	ret = copy_pad(ctx->mac_type, ctx->ipad, ipad);
862 	if (ret)
863 		goto calc_fail;
864 
865 	/* OPAD Calculation */
866 	crypto_shash_init(&ctx->sdesc->shash);
867 	crypto_shash_update(&ctx->sdesc->shash, opad, bs);
868 	crypto_shash_export(&ctx->sdesc->shash, opad);
869 	ret = copy_pad(ctx->mac_type, ctx->opad, opad);
870 	if (ret)
871 		goto calc_fail;
872 
873 	kfree(ipad);
874 	kfree(opad);
875 
876 	return 0;
877 
878 calc_fail:
879 	kfree(ctx->ipad);
880 	ctx->ipad = NULL;
881 	kfree(ctx->opad);
882 	ctx->opad = NULL;
883 	kfree(ipad);
884 	kfree(opad);
885 	kfree(ctx->sdesc);
886 	ctx->sdesc = NULL;
887 
888 	return ret;
889 }
890 
891 static int otx2_cpt_aead_cbc_aes_sha_setkey(struct crypto_aead *cipher,
892 					    const unsigned char *key,
893 					    unsigned int keylen)
894 {
895 	struct otx2_cpt_aead_ctx *ctx = crypto_aead_ctx(cipher);
896 	struct crypto_authenc_key_param *param;
897 	int enckeylen = 0, authkeylen = 0;
898 	struct rtattr *rta = (void *)key;
899 
900 	if (!RTA_OK(rta, keylen))
901 		return -EINVAL;
902 
903 	if (rta->rta_type != CRYPTO_AUTHENC_KEYA_PARAM)
904 		return -EINVAL;
905 
906 	if (RTA_PAYLOAD(rta) < sizeof(*param))
907 		return -EINVAL;
908 
909 	param = RTA_DATA(rta);
910 	enckeylen = be32_to_cpu(param->enckeylen);
911 	key += RTA_ALIGN(rta->rta_len);
912 	keylen -= RTA_ALIGN(rta->rta_len);
913 	if (keylen < enckeylen)
914 		return -EINVAL;
915 
916 	if (keylen > OTX2_CPT_MAX_KEY_SIZE)
917 		return -EINVAL;
918 
919 	authkeylen = keylen - enckeylen;
920 	memcpy(ctx->key, key, keylen);
921 
922 	switch (enckeylen) {
923 	case AES_KEYSIZE_128:
924 		ctx->key_type = OTX2_CPT_AES_128_BIT;
925 		break;
926 	case AES_KEYSIZE_192:
927 		ctx->key_type = OTX2_CPT_AES_192_BIT;
928 		break;
929 	case AES_KEYSIZE_256:
930 		ctx->key_type = OTX2_CPT_AES_256_BIT;
931 		break;
932 	default:
933 		/* Invalid key length */
934 		return -EINVAL;
935 	}
936 
937 	ctx->enc_key_len = enckeylen;
938 	ctx->auth_key_len = authkeylen;
939 
940 	return aead_hmac_init(cipher);
941 }
942 
943 static int otx2_cpt_aead_ecb_null_sha_setkey(struct crypto_aead *cipher,
944 					     const unsigned char *key,
945 					     unsigned int keylen)
946 {
947 	struct otx2_cpt_aead_ctx *ctx = crypto_aead_ctx(cipher);
948 	struct crypto_authenc_key_param *param;
949 	struct rtattr *rta = (void *)key;
950 	int enckeylen = 0;
951 
952 	if (!RTA_OK(rta, keylen))
953 		return -EINVAL;
954 
955 	if (rta->rta_type != CRYPTO_AUTHENC_KEYA_PARAM)
956 		return -EINVAL;
957 
958 	if (RTA_PAYLOAD(rta) < sizeof(*param))
959 		return -EINVAL;
960 
961 	param = RTA_DATA(rta);
962 	enckeylen = be32_to_cpu(param->enckeylen);
963 	key += RTA_ALIGN(rta->rta_len);
964 	keylen -= RTA_ALIGN(rta->rta_len);
965 	if (enckeylen != 0)
966 		return -EINVAL;
967 
968 	if (keylen > OTX2_CPT_MAX_KEY_SIZE)
969 		return -EINVAL;
970 
971 	memcpy(ctx->key, key, keylen);
972 	ctx->enc_key_len = enckeylen;
973 	ctx->auth_key_len = keylen;
974 
975 	return 0;
976 }
977 
978 static int otx2_cpt_aead_gcm_aes_setkey(struct crypto_aead *cipher,
979 					const unsigned char *key,
980 					unsigned int keylen)
981 {
982 	struct otx2_cpt_aead_ctx *ctx = crypto_aead_ctx(cipher);
983 
984 	/*
985 	 * For aes gcm we expect to get encryption key (16, 24, 32 bytes)
986 	 * and salt (4 bytes)
987 	 */
988 	switch (keylen) {
989 	case AES_KEYSIZE_128 + AES_GCM_SALT_SIZE:
990 		ctx->key_type = OTX2_CPT_AES_128_BIT;
991 		ctx->enc_key_len = AES_KEYSIZE_128;
992 		break;
993 	case AES_KEYSIZE_192 + AES_GCM_SALT_SIZE:
994 		ctx->key_type = OTX2_CPT_AES_192_BIT;
995 		ctx->enc_key_len = AES_KEYSIZE_192;
996 		break;
997 	case AES_KEYSIZE_256 + AES_GCM_SALT_SIZE:
998 		ctx->key_type = OTX2_CPT_AES_256_BIT;
999 		ctx->enc_key_len = AES_KEYSIZE_256;
1000 		break;
1001 	default:
1002 		/* Invalid key and salt length */
1003 		return -EINVAL;
1004 	}
1005 
1006 	/* Store encryption key and salt */
1007 	memcpy(ctx->key, key, keylen);
1008 
1009 	return crypto_aead_setkey(ctx->fbk_cipher, key, keylen);
1010 }
1011 
1012 static inline int create_aead_ctx_hdr(struct aead_request *req, u32 enc,
1013 				      u32 *argcnt)
1014 {
1015 	struct otx2_cpt_req_ctx *rctx = aead_request_ctx(req);
1016 	struct crypto_aead *tfm = crypto_aead_reqtfm(req);
1017 	struct otx2_cpt_aead_ctx *ctx = crypto_aead_ctx(tfm);
1018 	struct otx2_cpt_req_info *req_info = &rctx->cpt_req;
1019 	struct otx2_cpt_fc_ctx *fctx = &rctx->fctx;
1020 	int mac_len = crypto_aead_authsize(tfm);
1021 	int ds;
1022 
1023 	rctx->ctrl_word.e.enc_data_offset = req->assoclen;
1024 
1025 	switch (ctx->cipher_type) {
1026 	case OTX2_CPT_AES_CBC:
1027 		if (req->assoclen > 248 || !IS_ALIGNED(req->assoclen, 8))
1028 			return -EINVAL;
1029 
1030 		fctx->enc.enc_ctrl.e.iv_source = OTX2_CPT_FROM_CPTR;
1031 		/* Copy encryption key to context */
1032 		memcpy(fctx->enc.encr_key, ctx->key + ctx->auth_key_len,
1033 		       ctx->enc_key_len);
1034 		/* Copy IV to context */
1035 		memcpy(fctx->enc.encr_iv, req->iv, crypto_aead_ivsize(tfm));
1036 
1037 		ds = crypto_shash_digestsize(ctx->hashalg);
1038 		if (ctx->mac_type == OTX2_CPT_SHA384)
1039 			ds = SHA512_DIGEST_SIZE;
1040 		if (ctx->ipad)
1041 			memcpy(fctx->hmac.e.ipad, ctx->ipad, ds);
1042 		if (ctx->opad)
1043 			memcpy(fctx->hmac.e.opad, ctx->opad, ds);
1044 		break;
1045 
1046 	case OTX2_CPT_AES_GCM:
1047 		if (crypto_ipsec_check_assoclen(req->assoclen))
1048 			return -EINVAL;
1049 
1050 		fctx->enc.enc_ctrl.e.iv_source = OTX2_CPT_FROM_DPTR;
1051 		/* Copy encryption key to context */
1052 		memcpy(fctx->enc.encr_key, ctx->key, ctx->enc_key_len);
1053 		/* Copy salt to context */
1054 		memcpy(fctx->enc.encr_iv, ctx->key + ctx->enc_key_len,
1055 		       AES_GCM_SALT_SIZE);
1056 
1057 		rctx->ctrl_word.e.iv_offset = req->assoclen - AES_GCM_IV_OFFSET;
1058 		break;
1059 
1060 	default:
1061 		/* Unknown cipher type */
1062 		return -EINVAL;
1063 	}
1064 	cpu_to_be64s(&rctx->ctrl_word.flags);
1065 
1066 	req_info->ctrl.s.dma_mode = OTX2_CPT_DMA_MODE_SG;
1067 	req_info->ctrl.s.se_req = 1;
1068 	req_info->req.opcode.s.major = OTX2_CPT_MAJOR_OP_FC |
1069 				 DMA_MODE_FLAG(OTX2_CPT_DMA_MODE_SG);
1070 	if (enc) {
1071 		req_info->req.opcode.s.minor = 2;
1072 		req_info->req.param1 = req->cryptlen;
1073 		req_info->req.param2 = req->cryptlen + req->assoclen;
1074 	} else {
1075 		req_info->req.opcode.s.minor = 3;
1076 		req_info->req.param1 = req->cryptlen - mac_len;
1077 		req_info->req.param2 = req->cryptlen + req->assoclen - mac_len;
1078 	}
1079 
1080 	fctx->enc.enc_ctrl.e.enc_cipher = ctx->cipher_type;
1081 	fctx->enc.enc_ctrl.e.aes_key = ctx->key_type;
1082 	fctx->enc.enc_ctrl.e.mac_type = ctx->mac_type;
1083 	fctx->enc.enc_ctrl.e.mac_len = mac_len;
1084 	cpu_to_be64s(&fctx->enc.enc_ctrl.u);
1085 
1086 	/*
1087 	 * Storing Packet Data Information in offset
1088 	 * Control Word First 8 bytes
1089 	 */
1090 	req_info->in[*argcnt].vptr = (u8 *)&rctx->ctrl_word;
1091 	req_info->in[*argcnt].size = CONTROL_WORD_LEN;
1092 	req_info->req.dlen += CONTROL_WORD_LEN;
1093 	++(*argcnt);
1094 
1095 	req_info->in[*argcnt].vptr = (u8 *)fctx;
1096 	req_info->in[*argcnt].size = sizeof(struct otx2_cpt_fc_ctx);
1097 	req_info->req.dlen += sizeof(struct otx2_cpt_fc_ctx);
1098 	++(*argcnt);
1099 
1100 	return 0;
1101 }
1102 
1103 static inline void create_hmac_ctx_hdr(struct aead_request *req, u32 *argcnt,
1104 				      u32 enc)
1105 {
1106 	struct otx2_cpt_req_ctx *rctx = aead_request_ctx(req);
1107 	struct crypto_aead *tfm = crypto_aead_reqtfm(req);
1108 	struct otx2_cpt_aead_ctx *ctx = crypto_aead_ctx(tfm);
1109 	struct otx2_cpt_req_info *req_info = &rctx->cpt_req;
1110 
1111 	req_info->ctrl.s.dma_mode = OTX2_CPT_DMA_MODE_SG;
1112 	req_info->ctrl.s.se_req = 1;
1113 	req_info->req.opcode.s.major = OTX2_CPT_MAJOR_OP_HMAC |
1114 				 DMA_MODE_FLAG(OTX2_CPT_DMA_MODE_SG);
1115 	req_info->is_trunc_hmac = ctx->is_trunc_hmac;
1116 
1117 	req_info->req.opcode.s.minor = 0;
1118 	req_info->req.param1 = ctx->auth_key_len;
1119 	req_info->req.param2 = ctx->mac_type << 8;
1120 
1121 	/* Add authentication key */
1122 	req_info->in[*argcnt].vptr = ctx->key;
1123 	req_info->in[*argcnt].size = round_up(ctx->auth_key_len, 8);
1124 	req_info->req.dlen += round_up(ctx->auth_key_len, 8);
1125 	++(*argcnt);
1126 }
1127 
1128 static inline int create_aead_input_list(struct aead_request *req, u32 enc)
1129 {
1130 	struct otx2_cpt_req_ctx *rctx = aead_request_ctx(req);
1131 	struct otx2_cpt_req_info *req_info = &rctx->cpt_req;
1132 	u32 inputlen =  req->cryptlen + req->assoclen;
1133 	u32 status, argcnt = 0;
1134 
1135 	status = create_aead_ctx_hdr(req, enc, &argcnt);
1136 	if (status)
1137 		return status;
1138 	update_input_data(req_info, req->src, inputlen, &argcnt);
1139 	req_info->in_cnt = argcnt;
1140 
1141 	return 0;
1142 }
1143 
1144 static inline void create_aead_output_list(struct aead_request *req, u32 enc,
1145 					   u32 mac_len)
1146 {
1147 	struct otx2_cpt_req_ctx *rctx = aead_request_ctx(req);
1148 	struct otx2_cpt_req_info *req_info =  &rctx->cpt_req;
1149 	u32 argcnt = 0, outputlen = 0;
1150 
1151 	if (enc)
1152 		outputlen = req->cryptlen +  req->assoclen + mac_len;
1153 	else
1154 		outputlen = req->cryptlen + req->assoclen - mac_len;
1155 
1156 	update_output_data(req_info, req->dst, 0, outputlen, &argcnt);
1157 	req_info->out_cnt = argcnt;
1158 }
1159 
1160 static inline void create_aead_null_input_list(struct aead_request *req,
1161 					       u32 enc, u32 mac_len)
1162 {
1163 	struct otx2_cpt_req_ctx *rctx = aead_request_ctx(req);
1164 	struct otx2_cpt_req_info *req_info = &rctx->cpt_req;
1165 	u32 inputlen, argcnt = 0;
1166 
1167 	if (enc)
1168 		inputlen =  req->cryptlen + req->assoclen;
1169 	else
1170 		inputlen =  req->cryptlen + req->assoclen - mac_len;
1171 
1172 	create_hmac_ctx_hdr(req, &argcnt, enc);
1173 	update_input_data(req_info, req->src, inputlen, &argcnt);
1174 	req_info->in_cnt = argcnt;
1175 }
1176 
1177 static inline int create_aead_null_output_list(struct aead_request *req,
1178 					       u32 enc, u32 mac_len)
1179 {
1180 	struct otx2_cpt_req_ctx *rctx = aead_request_ctx(req);
1181 	struct otx2_cpt_req_info *req_info =  &rctx->cpt_req;
1182 	struct scatterlist *dst;
1183 	u8 *ptr = NULL;
1184 	int argcnt = 0, status, offset;
1185 	u32 inputlen;
1186 
1187 	if (enc)
1188 		inputlen =  req->cryptlen + req->assoclen;
1189 	else
1190 		inputlen =  req->cryptlen + req->assoclen - mac_len;
1191 
1192 	/*
1193 	 * If source and destination are different
1194 	 * then copy payload to destination
1195 	 */
1196 	if (req->src != req->dst) {
1197 
1198 		ptr = kmalloc(inputlen, (req_info->areq->flags &
1199 					 CRYPTO_TFM_REQ_MAY_SLEEP) ?
1200 					 GFP_KERNEL : GFP_ATOMIC);
1201 		if (!ptr)
1202 			return -ENOMEM;
1203 
1204 		status = sg_copy_to_buffer(req->src, sg_nents(req->src), ptr,
1205 					   inputlen);
1206 		if (status != inputlen) {
1207 			status = -EINVAL;
1208 			goto error_free;
1209 		}
1210 		status = sg_copy_from_buffer(req->dst, sg_nents(req->dst), ptr,
1211 					     inputlen);
1212 		if (status != inputlen) {
1213 			status = -EINVAL;
1214 			goto error_free;
1215 		}
1216 		kfree(ptr);
1217 	}
1218 
1219 	if (enc) {
1220 		/*
1221 		 * In an encryption scenario hmac needs
1222 		 * to be appended after payload
1223 		 */
1224 		dst = req->dst;
1225 		offset = inputlen;
1226 		while (offset >= dst->length) {
1227 			offset -= dst->length;
1228 			dst = sg_next(dst);
1229 			if (!dst)
1230 				return -ENOENT;
1231 		}
1232 
1233 		update_output_data(req_info, dst, offset, mac_len, &argcnt);
1234 	} else {
1235 		/*
1236 		 * In a decryption scenario calculated hmac for received
1237 		 * payload needs to be compare with hmac received
1238 		 */
1239 		status = sg_copy_buffer(req->src, sg_nents(req->src),
1240 					rctx->fctx.hmac.s.hmac_recv, mac_len,
1241 					inputlen, true);
1242 		if (status != mac_len)
1243 			return -EINVAL;
1244 
1245 		req_info->out[argcnt].vptr = rctx->fctx.hmac.s.hmac_calc;
1246 		req_info->out[argcnt].size = mac_len;
1247 		argcnt++;
1248 	}
1249 
1250 	req_info->out_cnt = argcnt;
1251 	return 0;
1252 
1253 error_free:
1254 	kfree(ptr);
1255 	return status;
1256 }
1257 
1258 static int aead_do_fallback(struct aead_request *req, bool is_enc)
1259 {
1260 	struct otx2_cpt_req_ctx *rctx = aead_request_ctx(req);
1261 	struct crypto_aead *aead = crypto_aead_reqtfm(req);
1262 	struct otx2_cpt_aead_ctx *ctx = crypto_aead_ctx(aead);
1263 	int ret;
1264 
1265 	if (ctx->fbk_cipher) {
1266 		/* Store the cipher tfm and then use the fallback tfm */
1267 		aead_request_set_tfm(&rctx->fbk_req, ctx->fbk_cipher);
1268 		aead_request_set_callback(&rctx->fbk_req, req->base.flags,
1269 					  req->base.complete, req->base.data);
1270 		aead_request_set_crypt(&rctx->fbk_req, req->src,
1271 				       req->dst, req->cryptlen, req->iv);
1272 		aead_request_set_ad(&rctx->fbk_req, req->assoclen);
1273 		ret = is_enc ? crypto_aead_encrypt(&rctx->fbk_req) :
1274 			       crypto_aead_decrypt(&rctx->fbk_req);
1275 	} else {
1276 		ret = -EINVAL;
1277 	}
1278 
1279 	return ret;
1280 }
1281 
1282 static int cpt_aead_enc_dec(struct aead_request *req, u8 reg_type, u8 enc)
1283 {
1284 	struct otx2_cpt_req_ctx *rctx = aead_request_ctx(req);
1285 	struct otx2_cpt_req_info *req_info = &rctx->cpt_req;
1286 	struct crypto_aead *tfm = crypto_aead_reqtfm(req);
1287 	struct otx2_cpt_aead_ctx *ctx = crypto_aead_ctx(tfm);
1288 	struct pci_dev *pdev;
1289 	int status, cpu_num;
1290 
1291 	/* Clear control words */
1292 	rctx->ctrl_word.flags = 0;
1293 	rctx->fctx.enc.enc_ctrl.u = 0;
1294 
1295 	req_info->callback = otx2_cpt_aead_callback;
1296 	req_info->areq = &req->base;
1297 	req_info->req_type = reg_type;
1298 	req_info->is_enc = enc;
1299 	req_info->is_trunc_hmac = false;
1300 
1301 	switch (reg_type) {
1302 	case OTX2_CPT_AEAD_ENC_DEC_REQ:
1303 		status = create_aead_input_list(req, enc);
1304 		if (status)
1305 			return status;
1306 		create_aead_output_list(req, enc, crypto_aead_authsize(tfm));
1307 		break;
1308 
1309 	case OTX2_CPT_AEAD_ENC_DEC_NULL_REQ:
1310 		create_aead_null_input_list(req, enc,
1311 					    crypto_aead_authsize(tfm));
1312 		status = create_aead_null_output_list(req, enc,
1313 						crypto_aead_authsize(tfm));
1314 		if (status)
1315 			return status;
1316 		break;
1317 
1318 	default:
1319 		return -EINVAL;
1320 	}
1321 	if (!IS_ALIGNED(req_info->req.param1, ctx->enc_align_len))
1322 		return -EINVAL;
1323 
1324 	if (!req_info->req.param2 ||
1325 	    (req_info->req.param1 > OTX2_CPT_MAX_REQ_SIZE) ||
1326 	    (req_info->req.param2 > OTX2_CPT_MAX_REQ_SIZE))
1327 		return aead_do_fallback(req, enc);
1328 
1329 	status = get_se_device(&pdev, &cpu_num);
1330 	if (status)
1331 		return status;
1332 
1333 	req_info->ctrl.s.grp = otx2_cpt_get_kcrypto_eng_grp_num(pdev);
1334 
1335 	/*
1336 	 * We perform an asynchronous send and once
1337 	 * the request is completed the driver would
1338 	 * intimate through registered call back functions
1339 	 */
1340 	return otx2_cpt_do_request(pdev, req_info, cpu_num);
1341 }
1342 
1343 static int otx2_cpt_aead_encrypt(struct aead_request *req)
1344 {
1345 	return cpt_aead_enc_dec(req, OTX2_CPT_AEAD_ENC_DEC_REQ, true);
1346 }
1347 
1348 static int otx2_cpt_aead_decrypt(struct aead_request *req)
1349 {
1350 	return cpt_aead_enc_dec(req, OTX2_CPT_AEAD_ENC_DEC_REQ, false);
1351 }
1352 
1353 static int otx2_cpt_aead_null_encrypt(struct aead_request *req)
1354 {
1355 	return cpt_aead_enc_dec(req, OTX2_CPT_AEAD_ENC_DEC_NULL_REQ, true);
1356 }
1357 
1358 static int otx2_cpt_aead_null_decrypt(struct aead_request *req)
1359 {
1360 	return cpt_aead_enc_dec(req, OTX2_CPT_AEAD_ENC_DEC_NULL_REQ, false);
1361 }
1362 
1363 static struct skcipher_alg otx2_cpt_skciphers[] = { {
1364 	.base.cra_name = "xts(aes)",
1365 	.base.cra_driver_name = "cpt_xts_aes",
1366 	.base.cra_flags = CRYPTO_ALG_ASYNC | CRYPTO_ALG_NEED_FALLBACK,
1367 	.base.cra_blocksize = AES_BLOCK_SIZE,
1368 	.base.cra_ctxsize = sizeof(struct otx2_cpt_enc_ctx),
1369 	.base.cra_alignmask = 7,
1370 	.base.cra_priority = 4001,
1371 	.base.cra_module = THIS_MODULE,
1372 
1373 	.init = otx2_cpt_enc_dec_init,
1374 	.exit = otx2_cpt_skcipher_exit,
1375 	.ivsize = AES_BLOCK_SIZE,
1376 	.min_keysize = 2 * AES_MIN_KEY_SIZE,
1377 	.max_keysize = 2 * AES_MAX_KEY_SIZE,
1378 	.setkey = otx2_cpt_skcipher_xts_setkey,
1379 	.encrypt = otx2_cpt_skcipher_encrypt,
1380 	.decrypt = otx2_cpt_skcipher_decrypt,
1381 }, {
1382 	.base.cra_name = "cbc(aes)",
1383 	.base.cra_driver_name = "cpt_cbc_aes",
1384 	.base.cra_flags = CRYPTO_ALG_ASYNC | CRYPTO_ALG_NEED_FALLBACK,
1385 	.base.cra_blocksize = AES_BLOCK_SIZE,
1386 	.base.cra_ctxsize = sizeof(struct otx2_cpt_enc_ctx),
1387 	.base.cra_alignmask = 7,
1388 	.base.cra_priority = 4001,
1389 	.base.cra_module = THIS_MODULE,
1390 
1391 	.init = otx2_cpt_enc_dec_init,
1392 	.exit = otx2_cpt_skcipher_exit,
1393 	.ivsize = AES_BLOCK_SIZE,
1394 	.min_keysize = AES_MIN_KEY_SIZE,
1395 	.max_keysize = AES_MAX_KEY_SIZE,
1396 	.setkey = otx2_cpt_skcipher_cbc_aes_setkey,
1397 	.encrypt = otx2_cpt_skcipher_encrypt,
1398 	.decrypt = otx2_cpt_skcipher_decrypt,
1399 }, {
1400 	.base.cra_name = "ecb(aes)",
1401 	.base.cra_driver_name = "cpt_ecb_aes",
1402 	.base.cra_flags = CRYPTO_ALG_ASYNC | CRYPTO_ALG_NEED_FALLBACK,
1403 	.base.cra_blocksize = AES_BLOCK_SIZE,
1404 	.base.cra_ctxsize = sizeof(struct otx2_cpt_enc_ctx),
1405 	.base.cra_alignmask = 7,
1406 	.base.cra_priority = 4001,
1407 	.base.cra_module = THIS_MODULE,
1408 
1409 	.init = otx2_cpt_enc_dec_init,
1410 	.exit = otx2_cpt_skcipher_exit,
1411 	.ivsize = 0,
1412 	.min_keysize = AES_MIN_KEY_SIZE,
1413 	.max_keysize = AES_MAX_KEY_SIZE,
1414 	.setkey = otx2_cpt_skcipher_ecb_aes_setkey,
1415 	.encrypt = otx2_cpt_skcipher_encrypt,
1416 	.decrypt = otx2_cpt_skcipher_decrypt,
1417 }, {
1418 	.base.cra_name = "cbc(des3_ede)",
1419 	.base.cra_driver_name = "cpt_cbc_des3_ede",
1420 	.base.cra_flags = CRYPTO_ALG_ASYNC | CRYPTO_ALG_NEED_FALLBACK,
1421 	.base.cra_blocksize = DES3_EDE_BLOCK_SIZE,
1422 	.base.cra_ctxsize = sizeof(struct otx2_cpt_enc_ctx),
1423 	.base.cra_alignmask = 7,
1424 	.base.cra_priority = 4001,
1425 	.base.cra_module = THIS_MODULE,
1426 
1427 	.init = otx2_cpt_enc_dec_init,
1428 	.exit = otx2_cpt_skcipher_exit,
1429 	.min_keysize = DES3_EDE_KEY_SIZE,
1430 	.max_keysize = DES3_EDE_KEY_SIZE,
1431 	.ivsize = DES_BLOCK_SIZE,
1432 	.setkey = otx2_cpt_skcipher_cbc_des3_setkey,
1433 	.encrypt = otx2_cpt_skcipher_encrypt,
1434 	.decrypt = otx2_cpt_skcipher_decrypt,
1435 }, {
1436 	.base.cra_name = "ecb(des3_ede)",
1437 	.base.cra_driver_name = "cpt_ecb_des3_ede",
1438 	.base.cra_flags = CRYPTO_ALG_ASYNC | CRYPTO_ALG_NEED_FALLBACK,
1439 	.base.cra_blocksize = DES3_EDE_BLOCK_SIZE,
1440 	.base.cra_ctxsize = sizeof(struct otx2_cpt_enc_ctx),
1441 	.base.cra_alignmask = 7,
1442 	.base.cra_priority = 4001,
1443 	.base.cra_module = THIS_MODULE,
1444 
1445 	.init = otx2_cpt_enc_dec_init,
1446 	.exit = otx2_cpt_skcipher_exit,
1447 	.min_keysize = DES3_EDE_KEY_SIZE,
1448 	.max_keysize = DES3_EDE_KEY_SIZE,
1449 	.ivsize = 0,
1450 	.setkey = otx2_cpt_skcipher_ecb_des3_setkey,
1451 	.encrypt = otx2_cpt_skcipher_encrypt,
1452 	.decrypt = otx2_cpt_skcipher_decrypt,
1453 } };
1454 
1455 static struct aead_alg otx2_cpt_aeads[] = { {
1456 	.base = {
1457 		.cra_name = "authenc(hmac(sha1),cbc(aes))",
1458 		.cra_driver_name = "cpt_hmac_sha1_cbc_aes",
1459 		.cra_blocksize = AES_BLOCK_SIZE,
1460 		.cra_flags = CRYPTO_ALG_ASYNC | CRYPTO_ALG_NEED_FALLBACK,
1461 		.cra_ctxsize = sizeof(struct otx2_cpt_aead_ctx),
1462 		.cra_priority = 4001,
1463 		.cra_alignmask = 0,
1464 		.cra_module = THIS_MODULE,
1465 	},
1466 	.init = otx2_cpt_aead_cbc_aes_sha1_init,
1467 	.exit = otx2_cpt_aead_exit,
1468 	.setkey = otx2_cpt_aead_cbc_aes_sha_setkey,
1469 	.setauthsize = otx2_cpt_aead_set_authsize,
1470 	.encrypt = otx2_cpt_aead_encrypt,
1471 	.decrypt = otx2_cpt_aead_decrypt,
1472 	.ivsize = AES_BLOCK_SIZE,
1473 	.maxauthsize = SHA1_DIGEST_SIZE,
1474 }, {
1475 	.base = {
1476 		.cra_name = "authenc(hmac(sha256),cbc(aes))",
1477 		.cra_driver_name = "cpt_hmac_sha256_cbc_aes",
1478 		.cra_blocksize = AES_BLOCK_SIZE,
1479 		.cra_flags = CRYPTO_ALG_ASYNC | CRYPTO_ALG_NEED_FALLBACK,
1480 		.cra_ctxsize = sizeof(struct otx2_cpt_aead_ctx),
1481 		.cra_priority = 4001,
1482 		.cra_alignmask = 0,
1483 		.cra_module = THIS_MODULE,
1484 	},
1485 	.init = otx2_cpt_aead_cbc_aes_sha256_init,
1486 	.exit = otx2_cpt_aead_exit,
1487 	.setkey = otx2_cpt_aead_cbc_aes_sha_setkey,
1488 	.setauthsize = otx2_cpt_aead_set_authsize,
1489 	.encrypt = otx2_cpt_aead_encrypt,
1490 	.decrypt = otx2_cpt_aead_decrypt,
1491 	.ivsize = AES_BLOCK_SIZE,
1492 	.maxauthsize = SHA256_DIGEST_SIZE,
1493 }, {
1494 	.base = {
1495 		.cra_name = "authenc(hmac(sha384),cbc(aes))",
1496 		.cra_driver_name = "cpt_hmac_sha384_cbc_aes",
1497 		.cra_blocksize = AES_BLOCK_SIZE,
1498 		.cra_flags = CRYPTO_ALG_ASYNC | CRYPTO_ALG_NEED_FALLBACK,
1499 		.cra_ctxsize = sizeof(struct otx2_cpt_aead_ctx),
1500 		.cra_priority = 4001,
1501 		.cra_alignmask = 0,
1502 		.cra_module = THIS_MODULE,
1503 	},
1504 	.init = otx2_cpt_aead_cbc_aes_sha384_init,
1505 	.exit = otx2_cpt_aead_exit,
1506 	.setkey = otx2_cpt_aead_cbc_aes_sha_setkey,
1507 	.setauthsize = otx2_cpt_aead_set_authsize,
1508 	.encrypt = otx2_cpt_aead_encrypt,
1509 	.decrypt = otx2_cpt_aead_decrypt,
1510 	.ivsize = AES_BLOCK_SIZE,
1511 	.maxauthsize = SHA384_DIGEST_SIZE,
1512 }, {
1513 	.base = {
1514 		.cra_name = "authenc(hmac(sha512),cbc(aes))",
1515 		.cra_driver_name = "cpt_hmac_sha512_cbc_aes",
1516 		.cra_blocksize = AES_BLOCK_SIZE,
1517 		.cra_flags = CRYPTO_ALG_ASYNC | CRYPTO_ALG_NEED_FALLBACK,
1518 		.cra_ctxsize = sizeof(struct otx2_cpt_aead_ctx),
1519 		.cra_priority = 4001,
1520 		.cra_alignmask = 0,
1521 		.cra_module = THIS_MODULE,
1522 	},
1523 	.init = otx2_cpt_aead_cbc_aes_sha512_init,
1524 	.exit = otx2_cpt_aead_exit,
1525 	.setkey = otx2_cpt_aead_cbc_aes_sha_setkey,
1526 	.setauthsize = otx2_cpt_aead_set_authsize,
1527 	.encrypt = otx2_cpt_aead_encrypt,
1528 	.decrypt = otx2_cpt_aead_decrypt,
1529 	.ivsize = AES_BLOCK_SIZE,
1530 	.maxauthsize = SHA512_DIGEST_SIZE,
1531 }, {
1532 	.base = {
1533 		.cra_name = "authenc(hmac(sha1),ecb(cipher_null))",
1534 		.cra_driver_name = "cpt_hmac_sha1_ecb_null",
1535 		.cra_blocksize = 1,
1536 		.cra_flags = CRYPTO_ALG_ASYNC,
1537 		.cra_ctxsize = sizeof(struct otx2_cpt_aead_ctx),
1538 		.cra_priority = 4001,
1539 		.cra_alignmask = 0,
1540 		.cra_module = THIS_MODULE,
1541 	},
1542 	.init = otx2_cpt_aead_ecb_null_sha1_init,
1543 	.exit = otx2_cpt_aead_exit,
1544 	.setkey = otx2_cpt_aead_ecb_null_sha_setkey,
1545 	.setauthsize = otx2_cpt_aead_null_set_authsize,
1546 	.encrypt = otx2_cpt_aead_null_encrypt,
1547 	.decrypt = otx2_cpt_aead_null_decrypt,
1548 	.ivsize = 0,
1549 	.maxauthsize = SHA1_DIGEST_SIZE,
1550 }, {
1551 	.base = {
1552 		.cra_name = "authenc(hmac(sha256),ecb(cipher_null))",
1553 		.cra_driver_name = "cpt_hmac_sha256_ecb_null",
1554 		.cra_blocksize = 1,
1555 		.cra_flags = CRYPTO_ALG_ASYNC,
1556 		.cra_ctxsize = sizeof(struct otx2_cpt_aead_ctx),
1557 		.cra_priority = 4001,
1558 		.cra_alignmask = 0,
1559 		.cra_module = THIS_MODULE,
1560 	},
1561 	.init = otx2_cpt_aead_ecb_null_sha256_init,
1562 	.exit = otx2_cpt_aead_exit,
1563 	.setkey = otx2_cpt_aead_ecb_null_sha_setkey,
1564 	.setauthsize = otx2_cpt_aead_null_set_authsize,
1565 	.encrypt = otx2_cpt_aead_null_encrypt,
1566 	.decrypt = otx2_cpt_aead_null_decrypt,
1567 	.ivsize = 0,
1568 	.maxauthsize = SHA256_DIGEST_SIZE,
1569 }, {
1570 	.base = {
1571 		.cra_name = "authenc(hmac(sha384),ecb(cipher_null))",
1572 		.cra_driver_name = "cpt_hmac_sha384_ecb_null",
1573 		.cra_blocksize = 1,
1574 		.cra_flags = CRYPTO_ALG_ASYNC,
1575 		.cra_ctxsize = sizeof(struct otx2_cpt_aead_ctx),
1576 		.cra_priority = 4001,
1577 		.cra_alignmask = 0,
1578 		.cra_module = THIS_MODULE,
1579 	},
1580 	.init = otx2_cpt_aead_ecb_null_sha384_init,
1581 	.exit = otx2_cpt_aead_exit,
1582 	.setkey = otx2_cpt_aead_ecb_null_sha_setkey,
1583 	.setauthsize = otx2_cpt_aead_null_set_authsize,
1584 	.encrypt = otx2_cpt_aead_null_encrypt,
1585 	.decrypt = otx2_cpt_aead_null_decrypt,
1586 	.ivsize = 0,
1587 	.maxauthsize = SHA384_DIGEST_SIZE,
1588 }, {
1589 	.base = {
1590 		.cra_name = "authenc(hmac(sha512),ecb(cipher_null))",
1591 		.cra_driver_name = "cpt_hmac_sha512_ecb_null",
1592 		.cra_blocksize = 1,
1593 		.cra_flags = CRYPTO_ALG_ASYNC,
1594 		.cra_ctxsize = sizeof(struct otx2_cpt_aead_ctx),
1595 		.cra_priority = 4001,
1596 		.cra_alignmask = 0,
1597 		.cra_module = THIS_MODULE,
1598 	},
1599 	.init = otx2_cpt_aead_ecb_null_sha512_init,
1600 	.exit = otx2_cpt_aead_exit,
1601 	.setkey = otx2_cpt_aead_ecb_null_sha_setkey,
1602 	.setauthsize = otx2_cpt_aead_null_set_authsize,
1603 	.encrypt = otx2_cpt_aead_null_encrypt,
1604 	.decrypt = otx2_cpt_aead_null_decrypt,
1605 	.ivsize = 0,
1606 	.maxauthsize = SHA512_DIGEST_SIZE,
1607 }, {
1608 	.base = {
1609 		.cra_name = "rfc4106(gcm(aes))",
1610 		.cra_driver_name = "cpt_rfc4106_gcm_aes",
1611 		.cra_blocksize = 1,
1612 		.cra_flags = CRYPTO_ALG_ASYNC | CRYPTO_ALG_NEED_FALLBACK,
1613 		.cra_ctxsize = sizeof(struct otx2_cpt_aead_ctx),
1614 		.cra_priority = 4001,
1615 		.cra_alignmask = 0,
1616 		.cra_module = THIS_MODULE,
1617 	},
1618 	.init = otx2_cpt_aead_gcm_aes_init,
1619 	.exit = otx2_cpt_aead_exit,
1620 	.setkey = otx2_cpt_aead_gcm_aes_setkey,
1621 	.setauthsize = otx2_cpt_aead_gcm_set_authsize,
1622 	.encrypt = otx2_cpt_aead_encrypt,
1623 	.decrypt = otx2_cpt_aead_decrypt,
1624 	.ivsize = AES_GCM_IV_SIZE,
1625 	.maxauthsize = AES_GCM_ICV_SIZE,
1626 } };
1627 
1628 static inline int cpt_register_algs(void)
1629 {
1630 	int i, err = 0;
1631 
1632 	for (i = 0; i < ARRAY_SIZE(otx2_cpt_skciphers); i++)
1633 		otx2_cpt_skciphers[i].base.cra_flags &= ~CRYPTO_ALG_DEAD;
1634 
1635 	err = crypto_register_skciphers(otx2_cpt_skciphers,
1636 					ARRAY_SIZE(otx2_cpt_skciphers));
1637 	if (err)
1638 		return err;
1639 
1640 	for (i = 0; i < ARRAY_SIZE(otx2_cpt_aeads); i++)
1641 		otx2_cpt_aeads[i].base.cra_flags &= ~CRYPTO_ALG_DEAD;
1642 
1643 	err = crypto_register_aeads(otx2_cpt_aeads,
1644 				    ARRAY_SIZE(otx2_cpt_aeads));
1645 	if (err) {
1646 		crypto_unregister_skciphers(otx2_cpt_skciphers,
1647 					    ARRAY_SIZE(otx2_cpt_skciphers));
1648 		return err;
1649 	}
1650 
1651 	return 0;
1652 }
1653 
1654 static inline void cpt_unregister_algs(void)
1655 {
1656 	crypto_unregister_skciphers(otx2_cpt_skciphers,
1657 				    ARRAY_SIZE(otx2_cpt_skciphers));
1658 	crypto_unregister_aeads(otx2_cpt_aeads, ARRAY_SIZE(otx2_cpt_aeads));
1659 }
1660 
1661 static int compare_func(const void *lptr, const void *rptr)
1662 {
1663 	const struct cpt_device_desc *ldesc = (struct cpt_device_desc *) lptr;
1664 	const struct cpt_device_desc *rdesc = (struct cpt_device_desc *) rptr;
1665 
1666 	if (ldesc->dev->devfn < rdesc->dev->devfn)
1667 		return -1;
1668 	if (ldesc->dev->devfn > rdesc->dev->devfn)
1669 		return 1;
1670 	return 0;
1671 }
1672 
1673 static void swap_func(void *lptr, void *rptr, int size)
1674 {
1675 	struct cpt_device_desc *ldesc = lptr;
1676 	struct cpt_device_desc *rdesc = rptr;
1677 
1678 	swap(*ldesc, *rdesc);
1679 }
1680 
1681 int otx2_cpt_crypto_init(struct pci_dev *pdev, struct module *mod,
1682 			 int num_queues, int num_devices)
1683 {
1684 	int ret = 0;
1685 	int count;
1686 
1687 	mutex_lock(&mutex);
1688 	count = atomic_read(&se_devices.count);
1689 	if (count >= OTX2_CPT_MAX_LFS_NUM) {
1690 		dev_err(&pdev->dev, "No space to add a new device\n");
1691 		ret = -ENOSPC;
1692 		goto unlock;
1693 	}
1694 	se_devices.desc[count].num_queues = num_queues;
1695 	se_devices.desc[count++].dev = pdev;
1696 	atomic_inc(&se_devices.count);
1697 
1698 	if (atomic_read(&se_devices.count) == num_devices &&
1699 	    is_crypto_registered == false) {
1700 		if (cpt_register_algs()) {
1701 			dev_err(&pdev->dev,
1702 				"Error in registering crypto algorithms\n");
1703 			ret =  -EINVAL;
1704 			goto unlock;
1705 		}
1706 		try_module_get(mod);
1707 		is_crypto_registered = true;
1708 	}
1709 	sort(se_devices.desc, count, sizeof(struct cpt_device_desc),
1710 	     compare_func, swap_func);
1711 
1712 unlock:
1713 	mutex_unlock(&mutex);
1714 	return ret;
1715 }
1716 
1717 void otx2_cpt_crypto_exit(struct pci_dev *pdev, struct module *mod)
1718 {
1719 	struct cpt_device_table *dev_tbl;
1720 	bool dev_found = false;
1721 	int i, j, count;
1722 
1723 	mutex_lock(&mutex);
1724 
1725 	dev_tbl = &se_devices;
1726 	count = atomic_read(&dev_tbl->count);
1727 	for (i = 0; i < count; i++) {
1728 		if (pdev == dev_tbl->desc[i].dev) {
1729 			for (j = i; j < count-1; j++)
1730 				dev_tbl->desc[j] = dev_tbl->desc[j+1];
1731 			dev_found = true;
1732 			break;
1733 		}
1734 	}
1735 
1736 	if (!dev_found) {
1737 		dev_err(&pdev->dev, "%s device not found\n", __func__);
1738 		goto unlock;
1739 	}
1740 	if (atomic_dec_and_test(&se_devices.count)) {
1741 		cpt_unregister_algs();
1742 		module_put(mod);
1743 		is_crypto_registered = false;
1744 	}
1745 
1746 unlock:
1747 	mutex_unlock(&mutex);
1748 }
1749