1 // SPDX-License-Identifier: (GPL-2.0+ OR BSD-3-Clause)
2 /*
3  * Copyright 2015-2016 Freescale Semiconductor Inc.
4  * Copyright 2017-2019 NXP
5  */
6 
7 #include "compat.h"
8 #include "regs.h"
9 #include "caamalg_qi2.h"
10 #include "dpseci_cmd.h"
11 #include "desc_constr.h"
12 #include "error.h"
13 #include "sg_sw_sec4.h"
14 #include "sg_sw_qm2.h"
15 #include "key_gen.h"
16 #include "caamalg_desc.h"
17 #include "caamhash_desc.h"
18 #include "dpseci-debugfs.h"
19 #include <linux/fsl/mc.h>
20 #include <soc/fsl/dpaa2-io.h>
21 #include <soc/fsl/dpaa2-fd.h>
22 
23 #define CAAM_CRA_PRIORITY	2000
24 
25 /* max key is sum of AES_MAX_KEY_SIZE, max split key size */
26 #define CAAM_MAX_KEY_SIZE	(AES_MAX_KEY_SIZE + CTR_RFC3686_NONCE_SIZE + \
27 				 SHA512_DIGEST_SIZE * 2)
28 
29 /*
30  * This is a a cache of buffers, from which the users of CAAM QI driver
31  * can allocate short buffers. It's speedier than doing kmalloc on the hotpath.
32  * NOTE: A more elegant solution would be to have some headroom in the frames
33  *       being processed. This can be added by the dpaa2-eth driver. This would
34  *       pose a problem for userspace application processing which cannot
35  *       know of this limitation. So for now, this will work.
36  * NOTE: The memcache is SMP-safe. No need to handle spinlocks in-here
37  */
38 static struct kmem_cache *qi_cache;
39 
40 struct caam_alg_entry {
41 	struct device *dev;
42 	int class1_alg_type;
43 	int class2_alg_type;
44 	bool rfc3686;
45 	bool geniv;
46 	bool nodkp;
47 };
48 
49 struct caam_aead_alg {
50 	struct aead_alg aead;
51 	struct caam_alg_entry caam;
52 	bool registered;
53 };
54 
55 struct caam_skcipher_alg {
56 	struct skcipher_alg skcipher;
57 	struct caam_alg_entry caam;
58 	bool registered;
59 };
60 
61 /**
62  * caam_ctx - per-session context
63  * @flc: Flow Contexts array
64  * @key:  [authentication key], encryption key
65  * @flc_dma: I/O virtual addresses of the Flow Contexts
66  * @key_dma: I/O virtual address of the key
67  * @dir: DMA direction for mapping key and Flow Contexts
68  * @dev: dpseci device
69  * @adata: authentication algorithm details
70  * @cdata: encryption algorithm details
71  * @authsize: authentication tag (a.k.a. ICV / MAC) size
72  */
73 struct caam_ctx {
74 	struct caam_flc flc[NUM_OP];
75 	u8 key[CAAM_MAX_KEY_SIZE];
76 	dma_addr_t flc_dma[NUM_OP];
77 	dma_addr_t key_dma;
78 	enum dma_data_direction dir;
79 	struct device *dev;
80 	struct alginfo adata;
81 	struct alginfo cdata;
82 	unsigned int authsize;
83 };
84 
85 static void *dpaa2_caam_iova_to_virt(struct dpaa2_caam_priv *priv,
86 				     dma_addr_t iova_addr)
87 {
88 	phys_addr_t phys_addr;
89 
90 	phys_addr = priv->domain ? iommu_iova_to_phys(priv->domain, iova_addr) :
91 				   iova_addr;
92 
93 	return phys_to_virt(phys_addr);
94 }
95 
96 /*
97  * qi_cache_zalloc - Allocate buffers from CAAM-QI cache
98  *
99  * Allocate data on the hotpath. Instead of using kzalloc, one can use the
100  * services of the CAAM QI memory cache (backed by kmem_cache). The buffers
101  * will have a size of CAAM_QI_MEMCACHE_SIZE, which should be sufficient for
102  * hosting 16 SG entries.
103  *
104  * @flags - flags that would be used for the equivalent kmalloc(..) call
105  *
106  * Returns a pointer to a retrieved buffer on success or NULL on failure.
107  */
108 static inline void *qi_cache_zalloc(gfp_t flags)
109 {
110 	return kmem_cache_zalloc(qi_cache, flags);
111 }
112 
113 /*
114  * qi_cache_free - Frees buffers allocated from CAAM-QI cache
115  *
116  * @obj - buffer previously allocated by qi_cache_zalloc
117  *
118  * No checking is being done, the call is a passthrough call to
119  * kmem_cache_free(...)
120  */
121 static inline void qi_cache_free(void *obj)
122 {
123 	kmem_cache_free(qi_cache, obj);
124 }
125 
126 static struct caam_request *to_caam_req(struct crypto_async_request *areq)
127 {
128 	switch (crypto_tfm_alg_type(areq->tfm)) {
129 	case CRYPTO_ALG_TYPE_SKCIPHER:
130 		return skcipher_request_ctx(skcipher_request_cast(areq));
131 	case CRYPTO_ALG_TYPE_AEAD:
132 		return aead_request_ctx(container_of(areq, struct aead_request,
133 						     base));
134 	case CRYPTO_ALG_TYPE_AHASH:
135 		return ahash_request_ctx(ahash_request_cast(areq));
136 	default:
137 		return ERR_PTR(-EINVAL);
138 	}
139 }
140 
141 static void caam_unmap(struct device *dev, struct scatterlist *src,
142 		       struct scatterlist *dst, int src_nents,
143 		       int dst_nents, dma_addr_t iv_dma, int ivsize,
144 		       enum dma_data_direction iv_dir, dma_addr_t qm_sg_dma,
145 		       int qm_sg_bytes)
146 {
147 	if (dst != src) {
148 		if (src_nents)
149 			dma_unmap_sg(dev, src, src_nents, DMA_TO_DEVICE);
150 		if (dst_nents)
151 			dma_unmap_sg(dev, dst, dst_nents, DMA_FROM_DEVICE);
152 	} else {
153 		dma_unmap_sg(dev, src, src_nents, DMA_BIDIRECTIONAL);
154 	}
155 
156 	if (iv_dma)
157 		dma_unmap_single(dev, iv_dma, ivsize, iv_dir);
158 
159 	if (qm_sg_bytes)
160 		dma_unmap_single(dev, qm_sg_dma, qm_sg_bytes, DMA_TO_DEVICE);
161 }
162 
163 static int aead_set_sh_desc(struct crypto_aead *aead)
164 {
165 	struct caam_aead_alg *alg = container_of(crypto_aead_alg(aead),
166 						 typeof(*alg), aead);
167 	struct caam_ctx *ctx = crypto_aead_ctx(aead);
168 	unsigned int ivsize = crypto_aead_ivsize(aead);
169 	struct device *dev = ctx->dev;
170 	struct dpaa2_caam_priv *priv = dev_get_drvdata(dev);
171 	struct caam_flc *flc;
172 	u32 *desc;
173 	u32 ctx1_iv_off = 0;
174 	u32 *nonce = NULL;
175 	unsigned int data_len[2];
176 	u32 inl_mask;
177 	const bool ctr_mode = ((ctx->cdata.algtype & OP_ALG_AAI_MASK) ==
178 			       OP_ALG_AAI_CTR_MOD128);
179 	const bool is_rfc3686 = alg->caam.rfc3686;
180 
181 	if (!ctx->cdata.keylen || !ctx->authsize)
182 		return 0;
183 
184 	/*
185 	 * AES-CTR needs to load IV in CONTEXT1 reg
186 	 * at an offset of 128bits (16bytes)
187 	 * CONTEXT1[255:128] = IV
188 	 */
189 	if (ctr_mode)
190 		ctx1_iv_off = 16;
191 
192 	/*
193 	 * RFC3686 specific:
194 	 *	CONTEXT1[255:128] = {NONCE, IV, COUNTER}
195 	 */
196 	if (is_rfc3686) {
197 		ctx1_iv_off = 16 + CTR_RFC3686_NONCE_SIZE;
198 		nonce = (u32 *)((void *)ctx->key + ctx->adata.keylen_pad +
199 				ctx->cdata.keylen - CTR_RFC3686_NONCE_SIZE);
200 	}
201 
202 	/*
203 	 * In case |user key| > |derived key|, using DKP<imm,imm> would result
204 	 * in invalid opcodes (last bytes of user key) in the resulting
205 	 * descriptor. Use DKP<ptr,imm> instead => both virtual and dma key
206 	 * addresses are needed.
207 	 */
208 	ctx->adata.key_virt = ctx->key;
209 	ctx->adata.key_dma = ctx->key_dma;
210 
211 	ctx->cdata.key_virt = ctx->key + ctx->adata.keylen_pad;
212 	ctx->cdata.key_dma = ctx->key_dma + ctx->adata.keylen_pad;
213 
214 	data_len[0] = ctx->adata.keylen_pad;
215 	data_len[1] = ctx->cdata.keylen;
216 
217 	/* aead_encrypt shared descriptor */
218 	if (desc_inline_query((alg->caam.geniv ? DESC_QI_AEAD_GIVENC_LEN :
219 						 DESC_QI_AEAD_ENC_LEN) +
220 			      (is_rfc3686 ? DESC_AEAD_CTR_RFC3686_LEN : 0),
221 			      DESC_JOB_IO_LEN, data_len, &inl_mask,
222 			      ARRAY_SIZE(data_len)) < 0)
223 		return -EINVAL;
224 
225 	ctx->adata.key_inline = !!(inl_mask & 1);
226 	ctx->cdata.key_inline = !!(inl_mask & 2);
227 
228 	flc = &ctx->flc[ENCRYPT];
229 	desc = flc->sh_desc;
230 
231 	if (alg->caam.geniv)
232 		cnstr_shdsc_aead_givencap(desc, &ctx->cdata, &ctx->adata,
233 					  ivsize, ctx->authsize, is_rfc3686,
234 					  nonce, ctx1_iv_off, true,
235 					  priv->sec_attr.era);
236 	else
237 		cnstr_shdsc_aead_encap(desc, &ctx->cdata, &ctx->adata,
238 				       ivsize, ctx->authsize, is_rfc3686, nonce,
239 				       ctx1_iv_off, true, priv->sec_attr.era);
240 
241 	flc->flc[1] = cpu_to_caam32(desc_len(desc)); /* SDL */
242 	dma_sync_single_for_device(dev, ctx->flc_dma[ENCRYPT],
243 				   sizeof(flc->flc) + desc_bytes(desc),
244 				   ctx->dir);
245 
246 	/* aead_decrypt shared descriptor */
247 	if (desc_inline_query(DESC_QI_AEAD_DEC_LEN +
248 			      (is_rfc3686 ? DESC_AEAD_CTR_RFC3686_LEN : 0),
249 			      DESC_JOB_IO_LEN, data_len, &inl_mask,
250 			      ARRAY_SIZE(data_len)) < 0)
251 		return -EINVAL;
252 
253 	ctx->adata.key_inline = !!(inl_mask & 1);
254 	ctx->cdata.key_inline = !!(inl_mask & 2);
255 
256 	flc = &ctx->flc[DECRYPT];
257 	desc = flc->sh_desc;
258 	cnstr_shdsc_aead_decap(desc, &ctx->cdata, &ctx->adata,
259 			       ivsize, ctx->authsize, alg->caam.geniv,
260 			       is_rfc3686, nonce, ctx1_iv_off, true,
261 			       priv->sec_attr.era);
262 	flc->flc[1] = cpu_to_caam32(desc_len(desc)); /* SDL */
263 	dma_sync_single_for_device(dev, ctx->flc_dma[DECRYPT],
264 				   sizeof(flc->flc) + desc_bytes(desc),
265 				   ctx->dir);
266 
267 	return 0;
268 }
269 
270 static int aead_setauthsize(struct crypto_aead *authenc, unsigned int authsize)
271 {
272 	struct caam_ctx *ctx = crypto_aead_ctx(authenc);
273 
274 	ctx->authsize = authsize;
275 	aead_set_sh_desc(authenc);
276 
277 	return 0;
278 }
279 
280 static int aead_setkey(struct crypto_aead *aead, const u8 *key,
281 		       unsigned int keylen)
282 {
283 	struct caam_ctx *ctx = crypto_aead_ctx(aead);
284 	struct device *dev = ctx->dev;
285 	struct crypto_authenc_keys keys;
286 
287 	if (crypto_authenc_extractkeys(&keys, key, keylen) != 0)
288 		goto badkey;
289 
290 	dev_dbg(dev, "keylen %d enckeylen %d authkeylen %d\n",
291 		keys.authkeylen + keys.enckeylen, keys.enckeylen,
292 		keys.authkeylen);
293 	print_hex_dump_debug("key in @" __stringify(__LINE__)": ",
294 			     DUMP_PREFIX_ADDRESS, 16, 4, key, keylen, 1);
295 
296 	ctx->adata.keylen = keys.authkeylen;
297 	ctx->adata.keylen_pad = split_key_len(ctx->adata.algtype &
298 					      OP_ALG_ALGSEL_MASK);
299 
300 	if (ctx->adata.keylen_pad + keys.enckeylen > CAAM_MAX_KEY_SIZE)
301 		goto badkey;
302 
303 	memcpy(ctx->key, keys.authkey, keys.authkeylen);
304 	memcpy(ctx->key + ctx->adata.keylen_pad, keys.enckey, keys.enckeylen);
305 	dma_sync_single_for_device(dev, ctx->key_dma, ctx->adata.keylen_pad +
306 				   keys.enckeylen, ctx->dir);
307 	print_hex_dump_debug("ctx.key@" __stringify(__LINE__)": ",
308 			     DUMP_PREFIX_ADDRESS, 16, 4, ctx->key,
309 			     ctx->adata.keylen_pad + keys.enckeylen, 1);
310 
311 	ctx->cdata.keylen = keys.enckeylen;
312 
313 	memzero_explicit(&keys, sizeof(keys));
314 	return aead_set_sh_desc(aead);
315 badkey:
316 	memzero_explicit(&keys, sizeof(keys));
317 	return -EINVAL;
318 }
319 
320 static int des3_aead_setkey(struct crypto_aead *aead, const u8 *key,
321 			    unsigned int keylen)
322 {
323 	struct crypto_authenc_keys keys;
324 	int err;
325 
326 	err = crypto_authenc_extractkeys(&keys, key, keylen);
327 	if (unlikely(err))
328 		goto out;
329 
330 	err = -EINVAL;
331 	if (keys.enckeylen != DES3_EDE_KEY_SIZE)
332 		goto out;
333 
334 	err = crypto_des3_ede_verify_key(crypto_aead_tfm(aead), keys.enckey) ?:
335 	      aead_setkey(aead, key, keylen);
336 
337 out:
338 	memzero_explicit(&keys, sizeof(keys));
339 	return err;
340 }
341 
342 static struct aead_edesc *aead_edesc_alloc(struct aead_request *req,
343 					   bool encrypt)
344 {
345 	struct crypto_aead *aead = crypto_aead_reqtfm(req);
346 	struct caam_request *req_ctx = aead_request_ctx(req);
347 	struct dpaa2_fl_entry *in_fle = &req_ctx->fd_flt[1];
348 	struct dpaa2_fl_entry *out_fle = &req_ctx->fd_flt[0];
349 	struct caam_ctx *ctx = crypto_aead_ctx(aead);
350 	struct caam_aead_alg *alg = container_of(crypto_aead_alg(aead),
351 						 typeof(*alg), aead);
352 	struct device *dev = ctx->dev;
353 	gfp_t flags = (req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP) ?
354 		      GFP_KERNEL : GFP_ATOMIC;
355 	int src_nents, mapped_src_nents, dst_nents = 0, mapped_dst_nents = 0;
356 	int src_len, dst_len = 0;
357 	struct aead_edesc *edesc;
358 	dma_addr_t qm_sg_dma, iv_dma = 0;
359 	int ivsize = 0;
360 	unsigned int authsize = ctx->authsize;
361 	int qm_sg_index = 0, qm_sg_nents = 0, qm_sg_bytes;
362 	int in_len, out_len;
363 	struct dpaa2_sg_entry *sg_table;
364 
365 	/* allocate space for base edesc, link tables and IV */
366 	edesc = qi_cache_zalloc(GFP_DMA | flags);
367 	if (unlikely(!edesc)) {
368 		dev_err(dev, "could not allocate extended descriptor\n");
369 		return ERR_PTR(-ENOMEM);
370 	}
371 
372 	if (unlikely(req->dst != req->src)) {
373 		src_len = req->assoclen + req->cryptlen;
374 		dst_len = src_len + (encrypt ? authsize : (-authsize));
375 
376 		src_nents = sg_nents_for_len(req->src, src_len);
377 		if (unlikely(src_nents < 0)) {
378 			dev_err(dev, "Insufficient bytes (%d) in src S/G\n",
379 				src_len);
380 			qi_cache_free(edesc);
381 			return ERR_PTR(src_nents);
382 		}
383 
384 		dst_nents = sg_nents_for_len(req->dst, dst_len);
385 		if (unlikely(dst_nents < 0)) {
386 			dev_err(dev, "Insufficient bytes (%d) in dst S/G\n",
387 				dst_len);
388 			qi_cache_free(edesc);
389 			return ERR_PTR(dst_nents);
390 		}
391 
392 		if (src_nents) {
393 			mapped_src_nents = dma_map_sg(dev, req->src, src_nents,
394 						      DMA_TO_DEVICE);
395 			if (unlikely(!mapped_src_nents)) {
396 				dev_err(dev, "unable to map source\n");
397 				qi_cache_free(edesc);
398 				return ERR_PTR(-ENOMEM);
399 			}
400 		} else {
401 			mapped_src_nents = 0;
402 		}
403 
404 		if (dst_nents) {
405 			mapped_dst_nents = dma_map_sg(dev, req->dst, dst_nents,
406 						      DMA_FROM_DEVICE);
407 			if (unlikely(!mapped_dst_nents)) {
408 				dev_err(dev, "unable to map destination\n");
409 				dma_unmap_sg(dev, req->src, src_nents,
410 					     DMA_TO_DEVICE);
411 				qi_cache_free(edesc);
412 				return ERR_PTR(-ENOMEM);
413 			}
414 		} else {
415 			mapped_dst_nents = 0;
416 		}
417 	} else {
418 		src_len = req->assoclen + req->cryptlen +
419 			  (encrypt ? authsize : 0);
420 
421 		src_nents = sg_nents_for_len(req->src, src_len);
422 		if (unlikely(src_nents < 0)) {
423 			dev_err(dev, "Insufficient bytes (%d) in src S/G\n",
424 				src_len);
425 			qi_cache_free(edesc);
426 			return ERR_PTR(src_nents);
427 		}
428 
429 		mapped_src_nents = dma_map_sg(dev, req->src, src_nents,
430 					      DMA_BIDIRECTIONAL);
431 		if (unlikely(!mapped_src_nents)) {
432 			dev_err(dev, "unable to map source\n");
433 			qi_cache_free(edesc);
434 			return ERR_PTR(-ENOMEM);
435 		}
436 	}
437 
438 	if ((alg->caam.rfc3686 && encrypt) || !alg->caam.geniv)
439 		ivsize = crypto_aead_ivsize(aead);
440 
441 	/*
442 	 * Create S/G table: req->assoclen, [IV,] req->src [, req->dst].
443 	 * Input is not contiguous.
444 	 * HW reads 4 S/G entries at a time; make sure the reads don't go beyond
445 	 * the end of the table by allocating more S/G entries. Logic:
446 	 * if (src != dst && output S/G)
447 	 *      pad output S/G, if needed
448 	 * else if (src == dst && S/G)
449 	 *      overlapping S/Gs; pad one of them
450 	 * else if (input S/G) ...
451 	 *      pad input S/G, if needed
452 	 */
453 	qm_sg_nents = 1 + !!ivsize + mapped_src_nents;
454 	if (mapped_dst_nents > 1)
455 		qm_sg_nents += pad_sg_nents(mapped_dst_nents);
456 	else if ((req->src == req->dst) && (mapped_src_nents > 1))
457 		qm_sg_nents = max(pad_sg_nents(qm_sg_nents),
458 				  1 + !!ivsize +
459 				  pad_sg_nents(mapped_src_nents));
460 	else
461 		qm_sg_nents = pad_sg_nents(qm_sg_nents);
462 
463 	sg_table = &edesc->sgt[0];
464 	qm_sg_bytes = qm_sg_nents * sizeof(*sg_table);
465 	if (unlikely(offsetof(struct aead_edesc, sgt) + qm_sg_bytes + ivsize >
466 		     CAAM_QI_MEMCACHE_SIZE)) {
467 		dev_err(dev, "No space for %d S/G entries and/or %dB IV\n",
468 			qm_sg_nents, ivsize);
469 		caam_unmap(dev, req->src, req->dst, src_nents, dst_nents, 0,
470 			   0, DMA_NONE, 0, 0);
471 		qi_cache_free(edesc);
472 		return ERR_PTR(-ENOMEM);
473 	}
474 
475 	if (ivsize) {
476 		u8 *iv = (u8 *)(sg_table + qm_sg_nents);
477 
478 		/* Make sure IV is located in a DMAable area */
479 		memcpy(iv, req->iv, ivsize);
480 
481 		iv_dma = dma_map_single(dev, iv, ivsize, DMA_TO_DEVICE);
482 		if (dma_mapping_error(dev, iv_dma)) {
483 			dev_err(dev, "unable to map IV\n");
484 			caam_unmap(dev, req->src, req->dst, src_nents,
485 				   dst_nents, 0, 0, DMA_NONE, 0, 0);
486 			qi_cache_free(edesc);
487 			return ERR_PTR(-ENOMEM);
488 		}
489 	}
490 
491 	edesc->src_nents = src_nents;
492 	edesc->dst_nents = dst_nents;
493 	edesc->iv_dma = iv_dma;
494 
495 	if ((alg->caam.class1_alg_type & OP_ALG_ALGSEL_MASK) ==
496 	    OP_ALG_ALGSEL_CHACHA20 && ivsize != CHACHAPOLY_IV_SIZE)
497 		/*
498 		 * The associated data comes already with the IV but we need
499 		 * to skip it when we authenticate or encrypt...
500 		 */
501 		edesc->assoclen = cpu_to_caam32(req->assoclen - ivsize);
502 	else
503 		edesc->assoclen = cpu_to_caam32(req->assoclen);
504 	edesc->assoclen_dma = dma_map_single(dev, &edesc->assoclen, 4,
505 					     DMA_TO_DEVICE);
506 	if (dma_mapping_error(dev, edesc->assoclen_dma)) {
507 		dev_err(dev, "unable to map assoclen\n");
508 		caam_unmap(dev, req->src, req->dst, src_nents, dst_nents,
509 			   iv_dma, ivsize, DMA_TO_DEVICE, 0, 0);
510 		qi_cache_free(edesc);
511 		return ERR_PTR(-ENOMEM);
512 	}
513 
514 	dma_to_qm_sg_one(sg_table, edesc->assoclen_dma, 4, 0);
515 	qm_sg_index++;
516 	if (ivsize) {
517 		dma_to_qm_sg_one(sg_table + qm_sg_index, iv_dma, ivsize, 0);
518 		qm_sg_index++;
519 	}
520 	sg_to_qm_sg_last(req->src, src_len, sg_table + qm_sg_index, 0);
521 	qm_sg_index += mapped_src_nents;
522 
523 	if (mapped_dst_nents > 1)
524 		sg_to_qm_sg_last(req->dst, dst_len, sg_table + qm_sg_index, 0);
525 
526 	qm_sg_dma = dma_map_single(dev, sg_table, qm_sg_bytes, DMA_TO_DEVICE);
527 	if (dma_mapping_error(dev, qm_sg_dma)) {
528 		dev_err(dev, "unable to map S/G table\n");
529 		dma_unmap_single(dev, edesc->assoclen_dma, 4, DMA_TO_DEVICE);
530 		caam_unmap(dev, req->src, req->dst, src_nents, dst_nents,
531 			   iv_dma, ivsize, DMA_TO_DEVICE, 0, 0);
532 		qi_cache_free(edesc);
533 		return ERR_PTR(-ENOMEM);
534 	}
535 
536 	edesc->qm_sg_dma = qm_sg_dma;
537 	edesc->qm_sg_bytes = qm_sg_bytes;
538 
539 	out_len = req->assoclen + req->cryptlen +
540 		  (encrypt ? ctx->authsize : (-ctx->authsize));
541 	in_len = 4 + ivsize + req->assoclen + req->cryptlen;
542 
543 	memset(&req_ctx->fd_flt, 0, sizeof(req_ctx->fd_flt));
544 	dpaa2_fl_set_final(in_fle, true);
545 	dpaa2_fl_set_format(in_fle, dpaa2_fl_sg);
546 	dpaa2_fl_set_addr(in_fle, qm_sg_dma);
547 	dpaa2_fl_set_len(in_fle, in_len);
548 
549 	if (req->dst == req->src) {
550 		if (mapped_src_nents == 1) {
551 			dpaa2_fl_set_format(out_fle, dpaa2_fl_single);
552 			dpaa2_fl_set_addr(out_fle, sg_dma_address(req->src));
553 		} else {
554 			dpaa2_fl_set_format(out_fle, dpaa2_fl_sg);
555 			dpaa2_fl_set_addr(out_fle, qm_sg_dma +
556 					  (1 + !!ivsize) * sizeof(*sg_table));
557 		}
558 	} else if (!mapped_dst_nents) {
559 		/*
560 		 * crypto engine requires the output entry to be present when
561 		 * "frame list" FD is used.
562 		 * Since engine does not support FMT=2'b11 (unused entry type),
563 		 * leaving out_fle zeroized is the best option.
564 		 */
565 		goto skip_out_fle;
566 	} else if (mapped_dst_nents == 1) {
567 		dpaa2_fl_set_format(out_fle, dpaa2_fl_single);
568 		dpaa2_fl_set_addr(out_fle, sg_dma_address(req->dst));
569 	} else {
570 		dpaa2_fl_set_format(out_fle, dpaa2_fl_sg);
571 		dpaa2_fl_set_addr(out_fle, qm_sg_dma + qm_sg_index *
572 				  sizeof(*sg_table));
573 	}
574 
575 	dpaa2_fl_set_len(out_fle, out_len);
576 
577 skip_out_fle:
578 	return edesc;
579 }
580 
581 static int chachapoly_set_sh_desc(struct crypto_aead *aead)
582 {
583 	struct caam_ctx *ctx = crypto_aead_ctx(aead);
584 	unsigned int ivsize = crypto_aead_ivsize(aead);
585 	struct device *dev = ctx->dev;
586 	struct caam_flc *flc;
587 	u32 *desc;
588 
589 	if (!ctx->cdata.keylen || !ctx->authsize)
590 		return 0;
591 
592 	flc = &ctx->flc[ENCRYPT];
593 	desc = flc->sh_desc;
594 	cnstr_shdsc_chachapoly(desc, &ctx->cdata, &ctx->adata, ivsize,
595 			       ctx->authsize, true, true);
596 	flc->flc[1] = cpu_to_caam32(desc_len(desc)); /* SDL */
597 	dma_sync_single_for_device(dev, ctx->flc_dma[ENCRYPT],
598 				   sizeof(flc->flc) + desc_bytes(desc),
599 				   ctx->dir);
600 
601 	flc = &ctx->flc[DECRYPT];
602 	desc = flc->sh_desc;
603 	cnstr_shdsc_chachapoly(desc, &ctx->cdata, &ctx->adata, ivsize,
604 			       ctx->authsize, false, true);
605 	flc->flc[1] = cpu_to_caam32(desc_len(desc)); /* SDL */
606 	dma_sync_single_for_device(dev, ctx->flc_dma[DECRYPT],
607 				   sizeof(flc->flc) + desc_bytes(desc),
608 				   ctx->dir);
609 
610 	return 0;
611 }
612 
613 static int chachapoly_setauthsize(struct crypto_aead *aead,
614 				  unsigned int authsize)
615 {
616 	struct caam_ctx *ctx = crypto_aead_ctx(aead);
617 
618 	if (authsize != POLY1305_DIGEST_SIZE)
619 		return -EINVAL;
620 
621 	ctx->authsize = authsize;
622 	return chachapoly_set_sh_desc(aead);
623 }
624 
625 static int chachapoly_setkey(struct crypto_aead *aead, const u8 *key,
626 			     unsigned int keylen)
627 {
628 	struct caam_ctx *ctx = crypto_aead_ctx(aead);
629 	unsigned int ivsize = crypto_aead_ivsize(aead);
630 	unsigned int saltlen = CHACHAPOLY_IV_SIZE - ivsize;
631 
632 	if (keylen != CHACHA_KEY_SIZE + saltlen)
633 		return -EINVAL;
634 
635 	ctx->cdata.key_virt = key;
636 	ctx->cdata.keylen = keylen - saltlen;
637 
638 	return chachapoly_set_sh_desc(aead);
639 }
640 
641 static int gcm_set_sh_desc(struct crypto_aead *aead)
642 {
643 	struct caam_ctx *ctx = crypto_aead_ctx(aead);
644 	struct device *dev = ctx->dev;
645 	unsigned int ivsize = crypto_aead_ivsize(aead);
646 	struct caam_flc *flc;
647 	u32 *desc;
648 	int rem_bytes = CAAM_DESC_BYTES_MAX - DESC_JOB_IO_LEN -
649 			ctx->cdata.keylen;
650 
651 	if (!ctx->cdata.keylen || !ctx->authsize)
652 		return 0;
653 
654 	/*
655 	 * AES GCM encrypt shared descriptor
656 	 * Job Descriptor and Shared Descriptor
657 	 * must fit into the 64-word Descriptor h/w Buffer
658 	 */
659 	if (rem_bytes >= DESC_QI_GCM_ENC_LEN) {
660 		ctx->cdata.key_inline = true;
661 		ctx->cdata.key_virt = ctx->key;
662 	} else {
663 		ctx->cdata.key_inline = false;
664 		ctx->cdata.key_dma = ctx->key_dma;
665 	}
666 
667 	flc = &ctx->flc[ENCRYPT];
668 	desc = flc->sh_desc;
669 	cnstr_shdsc_gcm_encap(desc, &ctx->cdata, ivsize, ctx->authsize, true);
670 	flc->flc[1] = cpu_to_caam32(desc_len(desc)); /* SDL */
671 	dma_sync_single_for_device(dev, ctx->flc_dma[ENCRYPT],
672 				   sizeof(flc->flc) + desc_bytes(desc),
673 				   ctx->dir);
674 
675 	/*
676 	 * Job Descriptor and Shared Descriptors
677 	 * must all fit into the 64-word Descriptor h/w Buffer
678 	 */
679 	if (rem_bytes >= DESC_QI_GCM_DEC_LEN) {
680 		ctx->cdata.key_inline = true;
681 		ctx->cdata.key_virt = ctx->key;
682 	} else {
683 		ctx->cdata.key_inline = false;
684 		ctx->cdata.key_dma = ctx->key_dma;
685 	}
686 
687 	flc = &ctx->flc[DECRYPT];
688 	desc = flc->sh_desc;
689 	cnstr_shdsc_gcm_decap(desc, &ctx->cdata, ivsize, ctx->authsize, true);
690 	flc->flc[1] = cpu_to_caam32(desc_len(desc)); /* SDL */
691 	dma_sync_single_for_device(dev, ctx->flc_dma[DECRYPT],
692 				   sizeof(flc->flc) + desc_bytes(desc),
693 				   ctx->dir);
694 
695 	return 0;
696 }
697 
698 static int gcm_setauthsize(struct crypto_aead *authenc, unsigned int authsize)
699 {
700 	struct caam_ctx *ctx = crypto_aead_ctx(authenc);
701 	int err;
702 
703 	err = crypto_gcm_check_authsize(authsize);
704 	if (err)
705 		return err;
706 
707 	ctx->authsize = authsize;
708 	gcm_set_sh_desc(authenc);
709 
710 	return 0;
711 }
712 
713 static int gcm_setkey(struct crypto_aead *aead,
714 		      const u8 *key, unsigned int keylen)
715 {
716 	struct caam_ctx *ctx = crypto_aead_ctx(aead);
717 	struct device *dev = ctx->dev;
718 	int ret;
719 
720 	ret = aes_check_keylen(keylen);
721 	if (ret)
722 		return ret;
723 	print_hex_dump_debug("key in @" __stringify(__LINE__)": ",
724 			     DUMP_PREFIX_ADDRESS, 16, 4, key, keylen, 1);
725 
726 	memcpy(ctx->key, key, keylen);
727 	dma_sync_single_for_device(dev, ctx->key_dma, keylen, ctx->dir);
728 	ctx->cdata.keylen = keylen;
729 
730 	return gcm_set_sh_desc(aead);
731 }
732 
733 static int rfc4106_set_sh_desc(struct crypto_aead *aead)
734 {
735 	struct caam_ctx *ctx = crypto_aead_ctx(aead);
736 	struct device *dev = ctx->dev;
737 	unsigned int ivsize = crypto_aead_ivsize(aead);
738 	struct caam_flc *flc;
739 	u32 *desc;
740 	int rem_bytes = CAAM_DESC_BYTES_MAX - DESC_JOB_IO_LEN -
741 			ctx->cdata.keylen;
742 
743 	if (!ctx->cdata.keylen || !ctx->authsize)
744 		return 0;
745 
746 	ctx->cdata.key_virt = ctx->key;
747 
748 	/*
749 	 * RFC4106 encrypt shared descriptor
750 	 * Job Descriptor and Shared Descriptor
751 	 * must fit into the 64-word Descriptor h/w Buffer
752 	 */
753 	if (rem_bytes >= DESC_QI_RFC4106_ENC_LEN) {
754 		ctx->cdata.key_inline = true;
755 	} else {
756 		ctx->cdata.key_inline = false;
757 		ctx->cdata.key_dma = ctx->key_dma;
758 	}
759 
760 	flc = &ctx->flc[ENCRYPT];
761 	desc = flc->sh_desc;
762 	cnstr_shdsc_rfc4106_encap(desc, &ctx->cdata, ivsize, ctx->authsize,
763 				  true);
764 	flc->flc[1] = cpu_to_caam32(desc_len(desc)); /* SDL */
765 	dma_sync_single_for_device(dev, ctx->flc_dma[ENCRYPT],
766 				   sizeof(flc->flc) + desc_bytes(desc),
767 				   ctx->dir);
768 
769 	/*
770 	 * Job Descriptor and Shared Descriptors
771 	 * must all fit into the 64-word Descriptor h/w Buffer
772 	 */
773 	if (rem_bytes >= DESC_QI_RFC4106_DEC_LEN) {
774 		ctx->cdata.key_inline = true;
775 	} else {
776 		ctx->cdata.key_inline = false;
777 		ctx->cdata.key_dma = ctx->key_dma;
778 	}
779 
780 	flc = &ctx->flc[DECRYPT];
781 	desc = flc->sh_desc;
782 	cnstr_shdsc_rfc4106_decap(desc, &ctx->cdata, ivsize, ctx->authsize,
783 				  true);
784 	flc->flc[1] = cpu_to_caam32(desc_len(desc)); /* SDL */
785 	dma_sync_single_for_device(dev, ctx->flc_dma[DECRYPT],
786 				   sizeof(flc->flc) + desc_bytes(desc),
787 				   ctx->dir);
788 
789 	return 0;
790 }
791 
792 static int rfc4106_setauthsize(struct crypto_aead *authenc,
793 			       unsigned int authsize)
794 {
795 	struct caam_ctx *ctx = crypto_aead_ctx(authenc);
796 	int err;
797 
798 	err = crypto_rfc4106_check_authsize(authsize);
799 	if (err)
800 		return err;
801 
802 	ctx->authsize = authsize;
803 	rfc4106_set_sh_desc(authenc);
804 
805 	return 0;
806 }
807 
808 static int rfc4106_setkey(struct crypto_aead *aead,
809 			  const u8 *key, unsigned int keylen)
810 {
811 	struct caam_ctx *ctx = crypto_aead_ctx(aead);
812 	struct device *dev = ctx->dev;
813 	int ret;
814 
815 	ret = aes_check_keylen(keylen - 4);
816 	if (ret)
817 		return ret;
818 
819 	print_hex_dump_debug("key in @" __stringify(__LINE__)": ",
820 			     DUMP_PREFIX_ADDRESS, 16, 4, key, keylen, 1);
821 
822 	memcpy(ctx->key, key, keylen);
823 	/*
824 	 * The last four bytes of the key material are used as the salt value
825 	 * in the nonce. Update the AES key length.
826 	 */
827 	ctx->cdata.keylen = keylen - 4;
828 	dma_sync_single_for_device(dev, ctx->key_dma, ctx->cdata.keylen,
829 				   ctx->dir);
830 
831 	return rfc4106_set_sh_desc(aead);
832 }
833 
834 static int rfc4543_set_sh_desc(struct crypto_aead *aead)
835 {
836 	struct caam_ctx *ctx = crypto_aead_ctx(aead);
837 	struct device *dev = ctx->dev;
838 	unsigned int ivsize = crypto_aead_ivsize(aead);
839 	struct caam_flc *flc;
840 	u32 *desc;
841 	int rem_bytes = CAAM_DESC_BYTES_MAX - DESC_JOB_IO_LEN -
842 			ctx->cdata.keylen;
843 
844 	if (!ctx->cdata.keylen || !ctx->authsize)
845 		return 0;
846 
847 	ctx->cdata.key_virt = ctx->key;
848 
849 	/*
850 	 * RFC4543 encrypt shared descriptor
851 	 * Job Descriptor and Shared Descriptor
852 	 * must fit into the 64-word Descriptor h/w Buffer
853 	 */
854 	if (rem_bytes >= DESC_QI_RFC4543_ENC_LEN) {
855 		ctx->cdata.key_inline = true;
856 	} else {
857 		ctx->cdata.key_inline = false;
858 		ctx->cdata.key_dma = ctx->key_dma;
859 	}
860 
861 	flc = &ctx->flc[ENCRYPT];
862 	desc = flc->sh_desc;
863 	cnstr_shdsc_rfc4543_encap(desc, &ctx->cdata, ivsize, ctx->authsize,
864 				  true);
865 	flc->flc[1] = cpu_to_caam32(desc_len(desc)); /* SDL */
866 	dma_sync_single_for_device(dev, ctx->flc_dma[ENCRYPT],
867 				   sizeof(flc->flc) + desc_bytes(desc),
868 				   ctx->dir);
869 
870 	/*
871 	 * Job Descriptor and Shared Descriptors
872 	 * must all fit into the 64-word Descriptor h/w Buffer
873 	 */
874 	if (rem_bytes >= DESC_QI_RFC4543_DEC_LEN) {
875 		ctx->cdata.key_inline = true;
876 	} else {
877 		ctx->cdata.key_inline = false;
878 		ctx->cdata.key_dma = ctx->key_dma;
879 	}
880 
881 	flc = &ctx->flc[DECRYPT];
882 	desc = flc->sh_desc;
883 	cnstr_shdsc_rfc4543_decap(desc, &ctx->cdata, ivsize, ctx->authsize,
884 				  true);
885 	flc->flc[1] = cpu_to_caam32(desc_len(desc)); /* SDL */
886 	dma_sync_single_for_device(dev, ctx->flc_dma[DECRYPT],
887 				   sizeof(flc->flc) + desc_bytes(desc),
888 				   ctx->dir);
889 
890 	return 0;
891 }
892 
893 static int rfc4543_setauthsize(struct crypto_aead *authenc,
894 			       unsigned int authsize)
895 {
896 	struct caam_ctx *ctx = crypto_aead_ctx(authenc);
897 
898 	if (authsize != 16)
899 		return -EINVAL;
900 
901 	ctx->authsize = authsize;
902 	rfc4543_set_sh_desc(authenc);
903 
904 	return 0;
905 }
906 
907 static int rfc4543_setkey(struct crypto_aead *aead,
908 			  const u8 *key, unsigned int keylen)
909 {
910 	struct caam_ctx *ctx = crypto_aead_ctx(aead);
911 	struct device *dev = ctx->dev;
912 	int ret;
913 
914 	ret = aes_check_keylen(keylen - 4);
915 	if (ret)
916 		return ret;
917 
918 	print_hex_dump_debug("key in @" __stringify(__LINE__)": ",
919 			     DUMP_PREFIX_ADDRESS, 16, 4, key, keylen, 1);
920 
921 	memcpy(ctx->key, key, keylen);
922 	/*
923 	 * The last four bytes of the key material are used as the salt value
924 	 * in the nonce. Update the AES key length.
925 	 */
926 	ctx->cdata.keylen = keylen - 4;
927 	dma_sync_single_for_device(dev, ctx->key_dma, ctx->cdata.keylen,
928 				   ctx->dir);
929 
930 	return rfc4543_set_sh_desc(aead);
931 }
932 
933 static int skcipher_setkey(struct crypto_skcipher *skcipher, const u8 *key,
934 			   unsigned int keylen, const u32 ctx1_iv_off)
935 {
936 	struct caam_ctx *ctx = crypto_skcipher_ctx(skcipher);
937 	struct caam_skcipher_alg *alg =
938 		container_of(crypto_skcipher_alg(skcipher),
939 			     struct caam_skcipher_alg, skcipher);
940 	struct device *dev = ctx->dev;
941 	struct caam_flc *flc;
942 	unsigned int ivsize = crypto_skcipher_ivsize(skcipher);
943 	u32 *desc;
944 	const bool is_rfc3686 = alg->caam.rfc3686;
945 
946 	print_hex_dump_debug("key in @" __stringify(__LINE__)": ",
947 			     DUMP_PREFIX_ADDRESS, 16, 4, key, keylen, 1);
948 
949 	ctx->cdata.keylen = keylen;
950 	ctx->cdata.key_virt = key;
951 	ctx->cdata.key_inline = true;
952 
953 	/* skcipher_encrypt shared descriptor */
954 	flc = &ctx->flc[ENCRYPT];
955 	desc = flc->sh_desc;
956 	cnstr_shdsc_skcipher_encap(desc, &ctx->cdata, ivsize, is_rfc3686,
957 				   ctx1_iv_off);
958 	flc->flc[1] = cpu_to_caam32(desc_len(desc)); /* SDL */
959 	dma_sync_single_for_device(dev, ctx->flc_dma[ENCRYPT],
960 				   sizeof(flc->flc) + desc_bytes(desc),
961 				   ctx->dir);
962 
963 	/* skcipher_decrypt shared descriptor */
964 	flc = &ctx->flc[DECRYPT];
965 	desc = flc->sh_desc;
966 	cnstr_shdsc_skcipher_decap(desc, &ctx->cdata, ivsize, is_rfc3686,
967 				   ctx1_iv_off);
968 	flc->flc[1] = cpu_to_caam32(desc_len(desc)); /* SDL */
969 	dma_sync_single_for_device(dev, ctx->flc_dma[DECRYPT],
970 				   sizeof(flc->flc) + desc_bytes(desc),
971 				   ctx->dir);
972 
973 	return 0;
974 }
975 
976 static int aes_skcipher_setkey(struct crypto_skcipher *skcipher,
977 			       const u8 *key, unsigned int keylen)
978 {
979 	int err;
980 
981 	err = aes_check_keylen(keylen);
982 	if (err)
983 		return err;
984 
985 	return skcipher_setkey(skcipher, key, keylen, 0);
986 }
987 
988 static int rfc3686_skcipher_setkey(struct crypto_skcipher *skcipher,
989 				   const u8 *key, unsigned int keylen)
990 {
991 	u32 ctx1_iv_off;
992 	int err;
993 
994 	/*
995 	 * RFC3686 specific:
996 	 *	| CONTEXT1[255:128] = {NONCE, IV, COUNTER}
997 	 *	| *key = {KEY, NONCE}
998 	 */
999 	ctx1_iv_off = 16 + CTR_RFC3686_NONCE_SIZE;
1000 	keylen -= CTR_RFC3686_NONCE_SIZE;
1001 
1002 	err = aes_check_keylen(keylen);
1003 	if (err)
1004 		return err;
1005 
1006 	return skcipher_setkey(skcipher, key, keylen, ctx1_iv_off);
1007 }
1008 
1009 static int ctr_skcipher_setkey(struct crypto_skcipher *skcipher,
1010 			       const u8 *key, unsigned int keylen)
1011 {
1012 	u32 ctx1_iv_off;
1013 	int err;
1014 
1015 	/*
1016 	 * AES-CTR needs to load IV in CONTEXT1 reg
1017 	 * at an offset of 128bits (16bytes)
1018 	 * CONTEXT1[255:128] = IV
1019 	 */
1020 	ctx1_iv_off = 16;
1021 
1022 	err = aes_check_keylen(keylen);
1023 	if (err)
1024 		return err;
1025 
1026 	return skcipher_setkey(skcipher, key, keylen, ctx1_iv_off);
1027 }
1028 
1029 static int chacha20_skcipher_setkey(struct crypto_skcipher *skcipher,
1030 				    const u8 *key, unsigned int keylen)
1031 {
1032 	if (keylen != CHACHA_KEY_SIZE)
1033 		return -EINVAL;
1034 
1035 	return skcipher_setkey(skcipher, key, keylen, 0);
1036 }
1037 
1038 static int des_skcipher_setkey(struct crypto_skcipher *skcipher,
1039 			       const u8 *key, unsigned int keylen)
1040 {
1041 	return verify_skcipher_des_key(skcipher, key) ?:
1042 	       skcipher_setkey(skcipher, key, keylen, 0);
1043 }
1044 
1045 static int des3_skcipher_setkey(struct crypto_skcipher *skcipher,
1046 			        const u8 *key, unsigned int keylen)
1047 {
1048 	return verify_skcipher_des3_key(skcipher, key) ?:
1049 	       skcipher_setkey(skcipher, key, keylen, 0);
1050 }
1051 
1052 static int xts_skcipher_setkey(struct crypto_skcipher *skcipher, const u8 *key,
1053 			       unsigned int keylen)
1054 {
1055 	struct caam_ctx *ctx = crypto_skcipher_ctx(skcipher);
1056 	struct device *dev = ctx->dev;
1057 	struct caam_flc *flc;
1058 	u32 *desc;
1059 
1060 	if (keylen != 2 * AES_MIN_KEY_SIZE  && keylen != 2 * AES_MAX_KEY_SIZE) {
1061 		dev_err(dev, "key size mismatch\n");
1062 		return -EINVAL;
1063 	}
1064 
1065 	ctx->cdata.keylen = keylen;
1066 	ctx->cdata.key_virt = key;
1067 	ctx->cdata.key_inline = true;
1068 
1069 	/* xts_skcipher_encrypt shared descriptor */
1070 	flc = &ctx->flc[ENCRYPT];
1071 	desc = flc->sh_desc;
1072 	cnstr_shdsc_xts_skcipher_encap(desc, &ctx->cdata);
1073 	flc->flc[1] = cpu_to_caam32(desc_len(desc)); /* SDL */
1074 	dma_sync_single_for_device(dev, ctx->flc_dma[ENCRYPT],
1075 				   sizeof(flc->flc) + desc_bytes(desc),
1076 				   ctx->dir);
1077 
1078 	/* xts_skcipher_decrypt shared descriptor */
1079 	flc = &ctx->flc[DECRYPT];
1080 	desc = flc->sh_desc;
1081 	cnstr_shdsc_xts_skcipher_decap(desc, &ctx->cdata);
1082 	flc->flc[1] = cpu_to_caam32(desc_len(desc)); /* SDL */
1083 	dma_sync_single_for_device(dev, ctx->flc_dma[DECRYPT],
1084 				   sizeof(flc->flc) + desc_bytes(desc),
1085 				   ctx->dir);
1086 
1087 	return 0;
1088 }
1089 
1090 static struct skcipher_edesc *skcipher_edesc_alloc(struct skcipher_request *req)
1091 {
1092 	struct crypto_skcipher *skcipher = crypto_skcipher_reqtfm(req);
1093 	struct caam_request *req_ctx = skcipher_request_ctx(req);
1094 	struct dpaa2_fl_entry *in_fle = &req_ctx->fd_flt[1];
1095 	struct dpaa2_fl_entry *out_fle = &req_ctx->fd_flt[0];
1096 	struct caam_ctx *ctx = crypto_skcipher_ctx(skcipher);
1097 	struct device *dev = ctx->dev;
1098 	gfp_t flags = (req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP) ?
1099 		       GFP_KERNEL : GFP_ATOMIC;
1100 	int src_nents, mapped_src_nents, dst_nents = 0, mapped_dst_nents = 0;
1101 	struct skcipher_edesc *edesc;
1102 	dma_addr_t iv_dma;
1103 	u8 *iv;
1104 	int ivsize = crypto_skcipher_ivsize(skcipher);
1105 	int dst_sg_idx, qm_sg_ents, qm_sg_bytes;
1106 	struct dpaa2_sg_entry *sg_table;
1107 
1108 	src_nents = sg_nents_for_len(req->src, req->cryptlen);
1109 	if (unlikely(src_nents < 0)) {
1110 		dev_err(dev, "Insufficient bytes (%d) in src S/G\n",
1111 			req->cryptlen);
1112 		return ERR_PTR(src_nents);
1113 	}
1114 
1115 	if (unlikely(req->dst != req->src)) {
1116 		dst_nents = sg_nents_for_len(req->dst, req->cryptlen);
1117 		if (unlikely(dst_nents < 0)) {
1118 			dev_err(dev, "Insufficient bytes (%d) in dst S/G\n",
1119 				req->cryptlen);
1120 			return ERR_PTR(dst_nents);
1121 		}
1122 
1123 		mapped_src_nents = dma_map_sg(dev, req->src, src_nents,
1124 					      DMA_TO_DEVICE);
1125 		if (unlikely(!mapped_src_nents)) {
1126 			dev_err(dev, "unable to map source\n");
1127 			return ERR_PTR(-ENOMEM);
1128 		}
1129 
1130 		mapped_dst_nents = dma_map_sg(dev, req->dst, dst_nents,
1131 					      DMA_FROM_DEVICE);
1132 		if (unlikely(!mapped_dst_nents)) {
1133 			dev_err(dev, "unable to map destination\n");
1134 			dma_unmap_sg(dev, req->src, src_nents, DMA_TO_DEVICE);
1135 			return ERR_PTR(-ENOMEM);
1136 		}
1137 	} else {
1138 		mapped_src_nents = dma_map_sg(dev, req->src, src_nents,
1139 					      DMA_BIDIRECTIONAL);
1140 		if (unlikely(!mapped_src_nents)) {
1141 			dev_err(dev, "unable to map source\n");
1142 			return ERR_PTR(-ENOMEM);
1143 		}
1144 	}
1145 
1146 	qm_sg_ents = 1 + mapped_src_nents;
1147 	dst_sg_idx = qm_sg_ents;
1148 
1149 	/*
1150 	 * Input, output HW S/G tables: [IV, src][dst, IV]
1151 	 * IV entries point to the same buffer
1152 	 * If src == dst, S/G entries are reused (S/G tables overlap)
1153 	 *
1154 	 * HW reads 4 S/G entries at a time; make sure the reads don't go beyond
1155 	 * the end of the table by allocating more S/G entries.
1156 	 */
1157 	if (req->src != req->dst)
1158 		qm_sg_ents += pad_sg_nents(mapped_dst_nents + 1);
1159 	else
1160 		qm_sg_ents = 1 + pad_sg_nents(qm_sg_ents);
1161 
1162 	qm_sg_bytes = qm_sg_ents * sizeof(struct dpaa2_sg_entry);
1163 	if (unlikely(offsetof(struct skcipher_edesc, sgt) + qm_sg_bytes +
1164 		     ivsize > CAAM_QI_MEMCACHE_SIZE)) {
1165 		dev_err(dev, "No space for %d S/G entries and/or %dB IV\n",
1166 			qm_sg_ents, ivsize);
1167 		caam_unmap(dev, req->src, req->dst, src_nents, dst_nents, 0,
1168 			   0, DMA_NONE, 0, 0);
1169 		return ERR_PTR(-ENOMEM);
1170 	}
1171 
1172 	/* allocate space for base edesc, link tables and IV */
1173 	edesc = qi_cache_zalloc(GFP_DMA | flags);
1174 	if (unlikely(!edesc)) {
1175 		dev_err(dev, "could not allocate extended descriptor\n");
1176 		caam_unmap(dev, req->src, req->dst, src_nents, dst_nents, 0,
1177 			   0, DMA_NONE, 0, 0);
1178 		return ERR_PTR(-ENOMEM);
1179 	}
1180 
1181 	/* Make sure IV is located in a DMAable area */
1182 	sg_table = &edesc->sgt[0];
1183 	iv = (u8 *)(sg_table + qm_sg_ents);
1184 	memcpy(iv, req->iv, ivsize);
1185 
1186 	iv_dma = dma_map_single(dev, iv, ivsize, DMA_BIDIRECTIONAL);
1187 	if (dma_mapping_error(dev, iv_dma)) {
1188 		dev_err(dev, "unable to map IV\n");
1189 		caam_unmap(dev, req->src, req->dst, src_nents, dst_nents, 0,
1190 			   0, DMA_NONE, 0, 0);
1191 		qi_cache_free(edesc);
1192 		return ERR_PTR(-ENOMEM);
1193 	}
1194 
1195 	edesc->src_nents = src_nents;
1196 	edesc->dst_nents = dst_nents;
1197 	edesc->iv_dma = iv_dma;
1198 	edesc->qm_sg_bytes = qm_sg_bytes;
1199 
1200 	dma_to_qm_sg_one(sg_table, iv_dma, ivsize, 0);
1201 	sg_to_qm_sg(req->src, req->cryptlen, sg_table + 1, 0);
1202 
1203 	if (req->src != req->dst)
1204 		sg_to_qm_sg(req->dst, req->cryptlen, sg_table + dst_sg_idx, 0);
1205 
1206 	dma_to_qm_sg_one(sg_table + dst_sg_idx + mapped_dst_nents, iv_dma,
1207 			 ivsize, 0);
1208 
1209 	edesc->qm_sg_dma = dma_map_single(dev, sg_table, edesc->qm_sg_bytes,
1210 					  DMA_TO_DEVICE);
1211 	if (dma_mapping_error(dev, edesc->qm_sg_dma)) {
1212 		dev_err(dev, "unable to map S/G table\n");
1213 		caam_unmap(dev, req->src, req->dst, src_nents, dst_nents,
1214 			   iv_dma, ivsize, DMA_BIDIRECTIONAL, 0, 0);
1215 		qi_cache_free(edesc);
1216 		return ERR_PTR(-ENOMEM);
1217 	}
1218 
1219 	memset(&req_ctx->fd_flt, 0, sizeof(req_ctx->fd_flt));
1220 	dpaa2_fl_set_final(in_fle, true);
1221 	dpaa2_fl_set_len(in_fle, req->cryptlen + ivsize);
1222 	dpaa2_fl_set_len(out_fle, req->cryptlen + ivsize);
1223 
1224 	dpaa2_fl_set_format(in_fle, dpaa2_fl_sg);
1225 	dpaa2_fl_set_addr(in_fle, edesc->qm_sg_dma);
1226 
1227 	dpaa2_fl_set_format(out_fle, dpaa2_fl_sg);
1228 
1229 	if (req->src == req->dst)
1230 		dpaa2_fl_set_addr(out_fle, edesc->qm_sg_dma +
1231 				  sizeof(*sg_table));
1232 	else
1233 		dpaa2_fl_set_addr(out_fle, edesc->qm_sg_dma + dst_sg_idx *
1234 				  sizeof(*sg_table));
1235 
1236 	return edesc;
1237 }
1238 
1239 static void aead_unmap(struct device *dev, struct aead_edesc *edesc,
1240 		       struct aead_request *req)
1241 {
1242 	struct crypto_aead *aead = crypto_aead_reqtfm(req);
1243 	int ivsize = crypto_aead_ivsize(aead);
1244 
1245 	caam_unmap(dev, req->src, req->dst, edesc->src_nents, edesc->dst_nents,
1246 		   edesc->iv_dma, ivsize, DMA_TO_DEVICE, edesc->qm_sg_dma,
1247 		   edesc->qm_sg_bytes);
1248 	dma_unmap_single(dev, edesc->assoclen_dma, 4, DMA_TO_DEVICE);
1249 }
1250 
1251 static void skcipher_unmap(struct device *dev, struct skcipher_edesc *edesc,
1252 			   struct skcipher_request *req)
1253 {
1254 	struct crypto_skcipher *skcipher = crypto_skcipher_reqtfm(req);
1255 	int ivsize = crypto_skcipher_ivsize(skcipher);
1256 
1257 	caam_unmap(dev, req->src, req->dst, edesc->src_nents, edesc->dst_nents,
1258 		   edesc->iv_dma, ivsize, DMA_BIDIRECTIONAL, edesc->qm_sg_dma,
1259 		   edesc->qm_sg_bytes);
1260 }
1261 
1262 static void aead_encrypt_done(void *cbk_ctx, u32 status)
1263 {
1264 	struct crypto_async_request *areq = cbk_ctx;
1265 	struct aead_request *req = container_of(areq, struct aead_request,
1266 						base);
1267 	struct caam_request *req_ctx = to_caam_req(areq);
1268 	struct aead_edesc *edesc = req_ctx->edesc;
1269 	struct crypto_aead *aead = crypto_aead_reqtfm(req);
1270 	struct caam_ctx *ctx = crypto_aead_ctx(aead);
1271 	int ecode = 0;
1272 
1273 	dev_dbg(ctx->dev, "%s %d: err 0x%x\n", __func__, __LINE__, status);
1274 
1275 	if (unlikely(status))
1276 		ecode = caam_qi2_strstatus(ctx->dev, status);
1277 
1278 	aead_unmap(ctx->dev, edesc, req);
1279 	qi_cache_free(edesc);
1280 	aead_request_complete(req, ecode);
1281 }
1282 
1283 static void aead_decrypt_done(void *cbk_ctx, u32 status)
1284 {
1285 	struct crypto_async_request *areq = cbk_ctx;
1286 	struct aead_request *req = container_of(areq, struct aead_request,
1287 						base);
1288 	struct caam_request *req_ctx = to_caam_req(areq);
1289 	struct aead_edesc *edesc = req_ctx->edesc;
1290 	struct crypto_aead *aead = crypto_aead_reqtfm(req);
1291 	struct caam_ctx *ctx = crypto_aead_ctx(aead);
1292 	int ecode = 0;
1293 
1294 	dev_dbg(ctx->dev, "%s %d: err 0x%x\n", __func__, __LINE__, status);
1295 
1296 	if (unlikely(status))
1297 		ecode = caam_qi2_strstatus(ctx->dev, status);
1298 
1299 	aead_unmap(ctx->dev, edesc, req);
1300 	qi_cache_free(edesc);
1301 	aead_request_complete(req, ecode);
1302 }
1303 
1304 static int aead_encrypt(struct aead_request *req)
1305 {
1306 	struct aead_edesc *edesc;
1307 	struct crypto_aead *aead = crypto_aead_reqtfm(req);
1308 	struct caam_ctx *ctx = crypto_aead_ctx(aead);
1309 	struct caam_request *caam_req = aead_request_ctx(req);
1310 	int ret;
1311 
1312 	/* allocate extended descriptor */
1313 	edesc = aead_edesc_alloc(req, true);
1314 	if (IS_ERR(edesc))
1315 		return PTR_ERR(edesc);
1316 
1317 	caam_req->flc = &ctx->flc[ENCRYPT];
1318 	caam_req->flc_dma = ctx->flc_dma[ENCRYPT];
1319 	caam_req->cbk = aead_encrypt_done;
1320 	caam_req->ctx = &req->base;
1321 	caam_req->edesc = edesc;
1322 	ret = dpaa2_caam_enqueue(ctx->dev, caam_req);
1323 	if (ret != -EINPROGRESS &&
1324 	    !(ret == -EBUSY && req->base.flags & CRYPTO_TFM_REQ_MAY_BACKLOG)) {
1325 		aead_unmap(ctx->dev, edesc, req);
1326 		qi_cache_free(edesc);
1327 	}
1328 
1329 	return ret;
1330 }
1331 
1332 static int aead_decrypt(struct aead_request *req)
1333 {
1334 	struct aead_edesc *edesc;
1335 	struct crypto_aead *aead = crypto_aead_reqtfm(req);
1336 	struct caam_ctx *ctx = crypto_aead_ctx(aead);
1337 	struct caam_request *caam_req = aead_request_ctx(req);
1338 	int ret;
1339 
1340 	/* allocate extended descriptor */
1341 	edesc = aead_edesc_alloc(req, false);
1342 	if (IS_ERR(edesc))
1343 		return PTR_ERR(edesc);
1344 
1345 	caam_req->flc = &ctx->flc[DECRYPT];
1346 	caam_req->flc_dma = ctx->flc_dma[DECRYPT];
1347 	caam_req->cbk = aead_decrypt_done;
1348 	caam_req->ctx = &req->base;
1349 	caam_req->edesc = edesc;
1350 	ret = dpaa2_caam_enqueue(ctx->dev, caam_req);
1351 	if (ret != -EINPROGRESS &&
1352 	    !(ret == -EBUSY && req->base.flags & CRYPTO_TFM_REQ_MAY_BACKLOG)) {
1353 		aead_unmap(ctx->dev, edesc, req);
1354 		qi_cache_free(edesc);
1355 	}
1356 
1357 	return ret;
1358 }
1359 
1360 static int ipsec_gcm_encrypt(struct aead_request *req)
1361 {
1362 	return crypto_ipsec_check_assoclen(req->assoclen) ? : aead_encrypt(req);
1363 }
1364 
1365 static int ipsec_gcm_decrypt(struct aead_request *req)
1366 {
1367 	return crypto_ipsec_check_assoclen(req->assoclen) ? : aead_decrypt(req);
1368 }
1369 
1370 static void skcipher_encrypt_done(void *cbk_ctx, u32 status)
1371 {
1372 	struct crypto_async_request *areq = cbk_ctx;
1373 	struct skcipher_request *req = skcipher_request_cast(areq);
1374 	struct caam_request *req_ctx = to_caam_req(areq);
1375 	struct crypto_skcipher *skcipher = crypto_skcipher_reqtfm(req);
1376 	struct caam_ctx *ctx = crypto_skcipher_ctx(skcipher);
1377 	struct skcipher_edesc *edesc = req_ctx->edesc;
1378 	int ecode = 0;
1379 	int ivsize = crypto_skcipher_ivsize(skcipher);
1380 
1381 	dev_dbg(ctx->dev, "%s %d: err 0x%x\n", __func__, __LINE__, status);
1382 
1383 	if (unlikely(status))
1384 		ecode = caam_qi2_strstatus(ctx->dev, status);
1385 
1386 	print_hex_dump_debug("dstiv  @" __stringify(__LINE__)": ",
1387 			     DUMP_PREFIX_ADDRESS, 16, 4, req->iv,
1388 			     edesc->src_nents > 1 ? 100 : ivsize, 1);
1389 	caam_dump_sg("dst    @" __stringify(__LINE__)": ",
1390 		     DUMP_PREFIX_ADDRESS, 16, 4, req->dst,
1391 		     edesc->dst_nents > 1 ? 100 : req->cryptlen, 1);
1392 
1393 	skcipher_unmap(ctx->dev, edesc, req);
1394 
1395 	/*
1396 	 * The crypto API expects us to set the IV (req->iv) to the last
1397 	 * ciphertext block (CBC mode) or last counter (CTR mode).
1398 	 * This is used e.g. by the CTS mode.
1399 	 */
1400 	if (!ecode)
1401 		memcpy(req->iv, (u8 *)&edesc->sgt[0] + edesc->qm_sg_bytes,
1402 		       ivsize);
1403 
1404 	qi_cache_free(edesc);
1405 	skcipher_request_complete(req, ecode);
1406 }
1407 
1408 static void skcipher_decrypt_done(void *cbk_ctx, u32 status)
1409 {
1410 	struct crypto_async_request *areq = cbk_ctx;
1411 	struct skcipher_request *req = skcipher_request_cast(areq);
1412 	struct caam_request *req_ctx = to_caam_req(areq);
1413 	struct crypto_skcipher *skcipher = crypto_skcipher_reqtfm(req);
1414 	struct caam_ctx *ctx = crypto_skcipher_ctx(skcipher);
1415 	struct skcipher_edesc *edesc = req_ctx->edesc;
1416 	int ecode = 0;
1417 	int ivsize = crypto_skcipher_ivsize(skcipher);
1418 
1419 	dev_dbg(ctx->dev, "%s %d: err 0x%x\n", __func__, __LINE__, status);
1420 
1421 	if (unlikely(status))
1422 		ecode = caam_qi2_strstatus(ctx->dev, status);
1423 
1424 	print_hex_dump_debug("dstiv  @" __stringify(__LINE__)": ",
1425 			     DUMP_PREFIX_ADDRESS, 16, 4, req->iv,
1426 			     edesc->src_nents > 1 ? 100 : ivsize, 1);
1427 	caam_dump_sg("dst    @" __stringify(__LINE__)": ",
1428 		     DUMP_PREFIX_ADDRESS, 16, 4, req->dst,
1429 		     edesc->dst_nents > 1 ? 100 : req->cryptlen, 1);
1430 
1431 	skcipher_unmap(ctx->dev, edesc, req);
1432 
1433 	/*
1434 	 * The crypto API expects us to set the IV (req->iv) to the last
1435 	 * ciphertext block (CBC mode) or last counter (CTR mode).
1436 	 * This is used e.g. by the CTS mode.
1437 	 */
1438 	if (!ecode)
1439 		memcpy(req->iv, (u8 *)&edesc->sgt[0] + edesc->qm_sg_bytes,
1440 		       ivsize);
1441 
1442 	qi_cache_free(edesc);
1443 	skcipher_request_complete(req, ecode);
1444 }
1445 
1446 static int skcipher_encrypt(struct skcipher_request *req)
1447 {
1448 	struct skcipher_edesc *edesc;
1449 	struct crypto_skcipher *skcipher = crypto_skcipher_reqtfm(req);
1450 	struct caam_ctx *ctx = crypto_skcipher_ctx(skcipher);
1451 	struct caam_request *caam_req = skcipher_request_ctx(req);
1452 	int ret;
1453 
1454 	if (!req->cryptlen)
1455 		return 0;
1456 
1457 	/* allocate extended descriptor */
1458 	edesc = skcipher_edesc_alloc(req);
1459 	if (IS_ERR(edesc))
1460 		return PTR_ERR(edesc);
1461 
1462 	caam_req->flc = &ctx->flc[ENCRYPT];
1463 	caam_req->flc_dma = ctx->flc_dma[ENCRYPT];
1464 	caam_req->cbk = skcipher_encrypt_done;
1465 	caam_req->ctx = &req->base;
1466 	caam_req->edesc = edesc;
1467 	ret = dpaa2_caam_enqueue(ctx->dev, caam_req);
1468 	if (ret != -EINPROGRESS &&
1469 	    !(ret == -EBUSY && req->base.flags & CRYPTO_TFM_REQ_MAY_BACKLOG)) {
1470 		skcipher_unmap(ctx->dev, edesc, req);
1471 		qi_cache_free(edesc);
1472 	}
1473 
1474 	return ret;
1475 }
1476 
1477 static int skcipher_decrypt(struct skcipher_request *req)
1478 {
1479 	struct skcipher_edesc *edesc;
1480 	struct crypto_skcipher *skcipher = crypto_skcipher_reqtfm(req);
1481 	struct caam_ctx *ctx = crypto_skcipher_ctx(skcipher);
1482 	struct caam_request *caam_req = skcipher_request_ctx(req);
1483 	int ret;
1484 
1485 	if (!req->cryptlen)
1486 		return 0;
1487 	/* allocate extended descriptor */
1488 	edesc = skcipher_edesc_alloc(req);
1489 	if (IS_ERR(edesc))
1490 		return PTR_ERR(edesc);
1491 
1492 	caam_req->flc = &ctx->flc[DECRYPT];
1493 	caam_req->flc_dma = ctx->flc_dma[DECRYPT];
1494 	caam_req->cbk = skcipher_decrypt_done;
1495 	caam_req->ctx = &req->base;
1496 	caam_req->edesc = edesc;
1497 	ret = dpaa2_caam_enqueue(ctx->dev, caam_req);
1498 	if (ret != -EINPROGRESS &&
1499 	    !(ret == -EBUSY && req->base.flags & CRYPTO_TFM_REQ_MAY_BACKLOG)) {
1500 		skcipher_unmap(ctx->dev, edesc, req);
1501 		qi_cache_free(edesc);
1502 	}
1503 
1504 	return ret;
1505 }
1506 
1507 static int caam_cra_init(struct caam_ctx *ctx, struct caam_alg_entry *caam,
1508 			 bool uses_dkp)
1509 {
1510 	dma_addr_t dma_addr;
1511 	int i;
1512 
1513 	/* copy descriptor header template value */
1514 	ctx->cdata.algtype = OP_TYPE_CLASS1_ALG | caam->class1_alg_type;
1515 	ctx->adata.algtype = OP_TYPE_CLASS2_ALG | caam->class2_alg_type;
1516 
1517 	ctx->dev = caam->dev;
1518 	ctx->dir = uses_dkp ? DMA_BIDIRECTIONAL : DMA_TO_DEVICE;
1519 
1520 	dma_addr = dma_map_single_attrs(ctx->dev, ctx->flc,
1521 					offsetof(struct caam_ctx, flc_dma),
1522 					ctx->dir, DMA_ATTR_SKIP_CPU_SYNC);
1523 	if (dma_mapping_error(ctx->dev, dma_addr)) {
1524 		dev_err(ctx->dev, "unable to map key, shared descriptors\n");
1525 		return -ENOMEM;
1526 	}
1527 
1528 	for (i = 0; i < NUM_OP; i++)
1529 		ctx->flc_dma[i] = dma_addr + i * sizeof(ctx->flc[i]);
1530 	ctx->key_dma = dma_addr + NUM_OP * sizeof(ctx->flc[0]);
1531 
1532 	return 0;
1533 }
1534 
1535 static int caam_cra_init_skcipher(struct crypto_skcipher *tfm)
1536 {
1537 	struct skcipher_alg *alg = crypto_skcipher_alg(tfm);
1538 	struct caam_skcipher_alg *caam_alg =
1539 		container_of(alg, typeof(*caam_alg), skcipher);
1540 
1541 	crypto_skcipher_set_reqsize(tfm, sizeof(struct caam_request));
1542 	return caam_cra_init(crypto_skcipher_ctx(tfm), &caam_alg->caam, false);
1543 }
1544 
1545 static int caam_cra_init_aead(struct crypto_aead *tfm)
1546 {
1547 	struct aead_alg *alg = crypto_aead_alg(tfm);
1548 	struct caam_aead_alg *caam_alg = container_of(alg, typeof(*caam_alg),
1549 						      aead);
1550 
1551 	crypto_aead_set_reqsize(tfm, sizeof(struct caam_request));
1552 	return caam_cra_init(crypto_aead_ctx(tfm), &caam_alg->caam,
1553 			     !caam_alg->caam.nodkp);
1554 }
1555 
1556 static void caam_exit_common(struct caam_ctx *ctx)
1557 {
1558 	dma_unmap_single_attrs(ctx->dev, ctx->flc_dma[0],
1559 			       offsetof(struct caam_ctx, flc_dma), ctx->dir,
1560 			       DMA_ATTR_SKIP_CPU_SYNC);
1561 }
1562 
1563 static void caam_cra_exit(struct crypto_skcipher *tfm)
1564 {
1565 	caam_exit_common(crypto_skcipher_ctx(tfm));
1566 }
1567 
1568 static void caam_cra_exit_aead(struct crypto_aead *tfm)
1569 {
1570 	caam_exit_common(crypto_aead_ctx(tfm));
1571 }
1572 
1573 static struct caam_skcipher_alg driver_algs[] = {
1574 	{
1575 		.skcipher = {
1576 			.base = {
1577 				.cra_name = "cbc(aes)",
1578 				.cra_driver_name = "cbc-aes-caam-qi2",
1579 				.cra_blocksize = AES_BLOCK_SIZE,
1580 			},
1581 			.setkey = aes_skcipher_setkey,
1582 			.encrypt = skcipher_encrypt,
1583 			.decrypt = skcipher_decrypt,
1584 			.min_keysize = AES_MIN_KEY_SIZE,
1585 			.max_keysize = AES_MAX_KEY_SIZE,
1586 			.ivsize = AES_BLOCK_SIZE,
1587 		},
1588 		.caam.class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CBC,
1589 	},
1590 	{
1591 		.skcipher = {
1592 			.base = {
1593 				.cra_name = "cbc(des3_ede)",
1594 				.cra_driver_name = "cbc-3des-caam-qi2",
1595 				.cra_blocksize = DES3_EDE_BLOCK_SIZE,
1596 			},
1597 			.setkey = des3_skcipher_setkey,
1598 			.encrypt = skcipher_encrypt,
1599 			.decrypt = skcipher_decrypt,
1600 			.min_keysize = DES3_EDE_KEY_SIZE,
1601 			.max_keysize = DES3_EDE_KEY_SIZE,
1602 			.ivsize = DES3_EDE_BLOCK_SIZE,
1603 		},
1604 		.caam.class1_alg_type = OP_ALG_ALGSEL_3DES | OP_ALG_AAI_CBC,
1605 	},
1606 	{
1607 		.skcipher = {
1608 			.base = {
1609 				.cra_name = "cbc(des)",
1610 				.cra_driver_name = "cbc-des-caam-qi2",
1611 				.cra_blocksize = DES_BLOCK_SIZE,
1612 			},
1613 			.setkey = des_skcipher_setkey,
1614 			.encrypt = skcipher_encrypt,
1615 			.decrypt = skcipher_decrypt,
1616 			.min_keysize = DES_KEY_SIZE,
1617 			.max_keysize = DES_KEY_SIZE,
1618 			.ivsize = DES_BLOCK_SIZE,
1619 		},
1620 		.caam.class1_alg_type = OP_ALG_ALGSEL_DES | OP_ALG_AAI_CBC,
1621 	},
1622 	{
1623 		.skcipher = {
1624 			.base = {
1625 				.cra_name = "ctr(aes)",
1626 				.cra_driver_name = "ctr-aes-caam-qi2",
1627 				.cra_blocksize = 1,
1628 			},
1629 			.setkey = ctr_skcipher_setkey,
1630 			.encrypt = skcipher_encrypt,
1631 			.decrypt = skcipher_decrypt,
1632 			.min_keysize = AES_MIN_KEY_SIZE,
1633 			.max_keysize = AES_MAX_KEY_SIZE,
1634 			.ivsize = AES_BLOCK_SIZE,
1635 			.chunksize = AES_BLOCK_SIZE,
1636 		},
1637 		.caam.class1_alg_type = OP_ALG_ALGSEL_AES |
1638 					OP_ALG_AAI_CTR_MOD128,
1639 	},
1640 	{
1641 		.skcipher = {
1642 			.base = {
1643 				.cra_name = "rfc3686(ctr(aes))",
1644 				.cra_driver_name = "rfc3686-ctr-aes-caam-qi2",
1645 				.cra_blocksize = 1,
1646 			},
1647 			.setkey = rfc3686_skcipher_setkey,
1648 			.encrypt = skcipher_encrypt,
1649 			.decrypt = skcipher_decrypt,
1650 			.min_keysize = AES_MIN_KEY_SIZE +
1651 				       CTR_RFC3686_NONCE_SIZE,
1652 			.max_keysize = AES_MAX_KEY_SIZE +
1653 				       CTR_RFC3686_NONCE_SIZE,
1654 			.ivsize = CTR_RFC3686_IV_SIZE,
1655 			.chunksize = AES_BLOCK_SIZE,
1656 		},
1657 		.caam = {
1658 			.class1_alg_type = OP_ALG_ALGSEL_AES |
1659 					   OP_ALG_AAI_CTR_MOD128,
1660 			.rfc3686 = true,
1661 		},
1662 	},
1663 	{
1664 		.skcipher = {
1665 			.base = {
1666 				.cra_name = "xts(aes)",
1667 				.cra_driver_name = "xts-aes-caam-qi2",
1668 				.cra_blocksize = AES_BLOCK_SIZE,
1669 			},
1670 			.setkey = xts_skcipher_setkey,
1671 			.encrypt = skcipher_encrypt,
1672 			.decrypt = skcipher_decrypt,
1673 			.min_keysize = 2 * AES_MIN_KEY_SIZE,
1674 			.max_keysize = 2 * AES_MAX_KEY_SIZE,
1675 			.ivsize = AES_BLOCK_SIZE,
1676 		},
1677 		.caam.class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_XTS,
1678 	},
1679 	{
1680 		.skcipher = {
1681 			.base = {
1682 				.cra_name = "chacha20",
1683 				.cra_driver_name = "chacha20-caam-qi2",
1684 				.cra_blocksize = 1,
1685 			},
1686 			.setkey = chacha20_skcipher_setkey,
1687 			.encrypt = skcipher_encrypt,
1688 			.decrypt = skcipher_decrypt,
1689 			.min_keysize = CHACHA_KEY_SIZE,
1690 			.max_keysize = CHACHA_KEY_SIZE,
1691 			.ivsize = CHACHA_IV_SIZE,
1692 		},
1693 		.caam.class1_alg_type = OP_ALG_ALGSEL_CHACHA20,
1694 	},
1695 };
1696 
1697 static struct caam_aead_alg driver_aeads[] = {
1698 	{
1699 		.aead = {
1700 			.base = {
1701 				.cra_name = "rfc4106(gcm(aes))",
1702 				.cra_driver_name = "rfc4106-gcm-aes-caam-qi2",
1703 				.cra_blocksize = 1,
1704 			},
1705 			.setkey = rfc4106_setkey,
1706 			.setauthsize = rfc4106_setauthsize,
1707 			.encrypt = ipsec_gcm_encrypt,
1708 			.decrypt = ipsec_gcm_decrypt,
1709 			.ivsize = 8,
1710 			.maxauthsize = AES_BLOCK_SIZE,
1711 		},
1712 		.caam = {
1713 			.class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_GCM,
1714 			.nodkp = true,
1715 		},
1716 	},
1717 	{
1718 		.aead = {
1719 			.base = {
1720 				.cra_name = "rfc4543(gcm(aes))",
1721 				.cra_driver_name = "rfc4543-gcm-aes-caam-qi2",
1722 				.cra_blocksize = 1,
1723 			},
1724 			.setkey = rfc4543_setkey,
1725 			.setauthsize = rfc4543_setauthsize,
1726 			.encrypt = ipsec_gcm_encrypt,
1727 			.decrypt = ipsec_gcm_decrypt,
1728 			.ivsize = 8,
1729 			.maxauthsize = AES_BLOCK_SIZE,
1730 		},
1731 		.caam = {
1732 			.class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_GCM,
1733 			.nodkp = true,
1734 		},
1735 	},
1736 	/* Galois Counter Mode */
1737 	{
1738 		.aead = {
1739 			.base = {
1740 				.cra_name = "gcm(aes)",
1741 				.cra_driver_name = "gcm-aes-caam-qi2",
1742 				.cra_blocksize = 1,
1743 			},
1744 			.setkey = gcm_setkey,
1745 			.setauthsize = gcm_setauthsize,
1746 			.encrypt = aead_encrypt,
1747 			.decrypt = aead_decrypt,
1748 			.ivsize = 12,
1749 			.maxauthsize = AES_BLOCK_SIZE,
1750 		},
1751 		.caam = {
1752 			.class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_GCM,
1753 			.nodkp = true,
1754 		}
1755 	},
1756 	/* single-pass ipsec_esp descriptor */
1757 	{
1758 		.aead = {
1759 			.base = {
1760 				.cra_name = "authenc(hmac(md5),cbc(aes))",
1761 				.cra_driver_name = "authenc-hmac-md5-"
1762 						   "cbc-aes-caam-qi2",
1763 				.cra_blocksize = AES_BLOCK_SIZE,
1764 			},
1765 			.setkey = aead_setkey,
1766 			.setauthsize = aead_setauthsize,
1767 			.encrypt = aead_encrypt,
1768 			.decrypt = aead_decrypt,
1769 			.ivsize = AES_BLOCK_SIZE,
1770 			.maxauthsize = MD5_DIGEST_SIZE,
1771 		},
1772 		.caam = {
1773 			.class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CBC,
1774 			.class2_alg_type = OP_ALG_ALGSEL_MD5 |
1775 					   OP_ALG_AAI_HMAC_PRECOMP,
1776 		}
1777 	},
1778 	{
1779 		.aead = {
1780 			.base = {
1781 				.cra_name = "echainiv(authenc(hmac(md5),"
1782 					    "cbc(aes)))",
1783 				.cra_driver_name = "echainiv-authenc-hmac-md5-"
1784 						   "cbc-aes-caam-qi2",
1785 				.cra_blocksize = AES_BLOCK_SIZE,
1786 			},
1787 			.setkey = aead_setkey,
1788 			.setauthsize = aead_setauthsize,
1789 			.encrypt = aead_encrypt,
1790 			.decrypt = aead_decrypt,
1791 			.ivsize = AES_BLOCK_SIZE,
1792 			.maxauthsize = MD5_DIGEST_SIZE,
1793 		},
1794 		.caam = {
1795 			.class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CBC,
1796 			.class2_alg_type = OP_ALG_ALGSEL_MD5 |
1797 					   OP_ALG_AAI_HMAC_PRECOMP,
1798 			.geniv = true,
1799 		}
1800 	},
1801 	{
1802 		.aead = {
1803 			.base = {
1804 				.cra_name = "authenc(hmac(sha1),cbc(aes))",
1805 				.cra_driver_name = "authenc-hmac-sha1-"
1806 						   "cbc-aes-caam-qi2",
1807 				.cra_blocksize = AES_BLOCK_SIZE,
1808 			},
1809 			.setkey = aead_setkey,
1810 			.setauthsize = aead_setauthsize,
1811 			.encrypt = aead_encrypt,
1812 			.decrypt = aead_decrypt,
1813 			.ivsize = AES_BLOCK_SIZE,
1814 			.maxauthsize = SHA1_DIGEST_SIZE,
1815 		},
1816 		.caam = {
1817 			.class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CBC,
1818 			.class2_alg_type = OP_ALG_ALGSEL_SHA1 |
1819 					   OP_ALG_AAI_HMAC_PRECOMP,
1820 		}
1821 	},
1822 	{
1823 		.aead = {
1824 			.base = {
1825 				.cra_name = "echainiv(authenc(hmac(sha1),"
1826 					    "cbc(aes)))",
1827 				.cra_driver_name = "echainiv-authenc-"
1828 						   "hmac-sha1-cbc-aes-caam-qi2",
1829 				.cra_blocksize = AES_BLOCK_SIZE,
1830 			},
1831 			.setkey = aead_setkey,
1832 			.setauthsize = aead_setauthsize,
1833 			.encrypt = aead_encrypt,
1834 			.decrypt = aead_decrypt,
1835 			.ivsize = AES_BLOCK_SIZE,
1836 			.maxauthsize = SHA1_DIGEST_SIZE,
1837 		},
1838 		.caam = {
1839 			.class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CBC,
1840 			.class2_alg_type = OP_ALG_ALGSEL_SHA1 |
1841 					   OP_ALG_AAI_HMAC_PRECOMP,
1842 			.geniv = true,
1843 		},
1844 	},
1845 	{
1846 		.aead = {
1847 			.base = {
1848 				.cra_name = "authenc(hmac(sha224),cbc(aes))",
1849 				.cra_driver_name = "authenc-hmac-sha224-"
1850 						   "cbc-aes-caam-qi2",
1851 				.cra_blocksize = AES_BLOCK_SIZE,
1852 			},
1853 			.setkey = aead_setkey,
1854 			.setauthsize = aead_setauthsize,
1855 			.encrypt = aead_encrypt,
1856 			.decrypt = aead_decrypt,
1857 			.ivsize = AES_BLOCK_SIZE,
1858 			.maxauthsize = SHA224_DIGEST_SIZE,
1859 		},
1860 		.caam = {
1861 			.class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CBC,
1862 			.class2_alg_type = OP_ALG_ALGSEL_SHA224 |
1863 					   OP_ALG_AAI_HMAC_PRECOMP,
1864 		}
1865 	},
1866 	{
1867 		.aead = {
1868 			.base = {
1869 				.cra_name = "echainiv(authenc(hmac(sha224),"
1870 					    "cbc(aes)))",
1871 				.cra_driver_name = "echainiv-authenc-"
1872 						   "hmac-sha224-cbc-aes-caam-qi2",
1873 				.cra_blocksize = AES_BLOCK_SIZE,
1874 			},
1875 			.setkey = aead_setkey,
1876 			.setauthsize = aead_setauthsize,
1877 			.encrypt = aead_encrypt,
1878 			.decrypt = aead_decrypt,
1879 			.ivsize = AES_BLOCK_SIZE,
1880 			.maxauthsize = SHA224_DIGEST_SIZE,
1881 		},
1882 		.caam = {
1883 			.class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CBC,
1884 			.class2_alg_type = OP_ALG_ALGSEL_SHA224 |
1885 					   OP_ALG_AAI_HMAC_PRECOMP,
1886 			.geniv = true,
1887 		}
1888 	},
1889 	{
1890 		.aead = {
1891 			.base = {
1892 				.cra_name = "authenc(hmac(sha256),cbc(aes))",
1893 				.cra_driver_name = "authenc-hmac-sha256-"
1894 						   "cbc-aes-caam-qi2",
1895 				.cra_blocksize = AES_BLOCK_SIZE,
1896 			},
1897 			.setkey = aead_setkey,
1898 			.setauthsize = aead_setauthsize,
1899 			.encrypt = aead_encrypt,
1900 			.decrypt = aead_decrypt,
1901 			.ivsize = AES_BLOCK_SIZE,
1902 			.maxauthsize = SHA256_DIGEST_SIZE,
1903 		},
1904 		.caam = {
1905 			.class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CBC,
1906 			.class2_alg_type = OP_ALG_ALGSEL_SHA256 |
1907 					   OP_ALG_AAI_HMAC_PRECOMP,
1908 		}
1909 	},
1910 	{
1911 		.aead = {
1912 			.base = {
1913 				.cra_name = "echainiv(authenc(hmac(sha256),"
1914 					    "cbc(aes)))",
1915 				.cra_driver_name = "echainiv-authenc-"
1916 						   "hmac-sha256-cbc-aes-"
1917 						   "caam-qi2",
1918 				.cra_blocksize = AES_BLOCK_SIZE,
1919 			},
1920 			.setkey = aead_setkey,
1921 			.setauthsize = aead_setauthsize,
1922 			.encrypt = aead_encrypt,
1923 			.decrypt = aead_decrypt,
1924 			.ivsize = AES_BLOCK_SIZE,
1925 			.maxauthsize = SHA256_DIGEST_SIZE,
1926 		},
1927 		.caam = {
1928 			.class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CBC,
1929 			.class2_alg_type = OP_ALG_ALGSEL_SHA256 |
1930 					   OP_ALG_AAI_HMAC_PRECOMP,
1931 			.geniv = true,
1932 		}
1933 	},
1934 	{
1935 		.aead = {
1936 			.base = {
1937 				.cra_name = "authenc(hmac(sha384),cbc(aes))",
1938 				.cra_driver_name = "authenc-hmac-sha384-"
1939 						   "cbc-aes-caam-qi2",
1940 				.cra_blocksize = AES_BLOCK_SIZE,
1941 			},
1942 			.setkey = aead_setkey,
1943 			.setauthsize = aead_setauthsize,
1944 			.encrypt = aead_encrypt,
1945 			.decrypt = aead_decrypt,
1946 			.ivsize = AES_BLOCK_SIZE,
1947 			.maxauthsize = SHA384_DIGEST_SIZE,
1948 		},
1949 		.caam = {
1950 			.class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CBC,
1951 			.class2_alg_type = OP_ALG_ALGSEL_SHA384 |
1952 					   OP_ALG_AAI_HMAC_PRECOMP,
1953 		}
1954 	},
1955 	{
1956 		.aead = {
1957 			.base = {
1958 				.cra_name = "echainiv(authenc(hmac(sha384),"
1959 					    "cbc(aes)))",
1960 				.cra_driver_name = "echainiv-authenc-"
1961 						   "hmac-sha384-cbc-aes-"
1962 						   "caam-qi2",
1963 				.cra_blocksize = AES_BLOCK_SIZE,
1964 			},
1965 			.setkey = aead_setkey,
1966 			.setauthsize = aead_setauthsize,
1967 			.encrypt = aead_encrypt,
1968 			.decrypt = aead_decrypt,
1969 			.ivsize = AES_BLOCK_SIZE,
1970 			.maxauthsize = SHA384_DIGEST_SIZE,
1971 		},
1972 		.caam = {
1973 			.class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CBC,
1974 			.class2_alg_type = OP_ALG_ALGSEL_SHA384 |
1975 					   OP_ALG_AAI_HMAC_PRECOMP,
1976 			.geniv = true,
1977 		}
1978 	},
1979 	{
1980 		.aead = {
1981 			.base = {
1982 				.cra_name = "authenc(hmac(sha512),cbc(aes))",
1983 				.cra_driver_name = "authenc-hmac-sha512-"
1984 						   "cbc-aes-caam-qi2",
1985 				.cra_blocksize = AES_BLOCK_SIZE,
1986 			},
1987 			.setkey = aead_setkey,
1988 			.setauthsize = aead_setauthsize,
1989 			.encrypt = aead_encrypt,
1990 			.decrypt = aead_decrypt,
1991 			.ivsize = AES_BLOCK_SIZE,
1992 			.maxauthsize = SHA512_DIGEST_SIZE,
1993 		},
1994 		.caam = {
1995 			.class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CBC,
1996 			.class2_alg_type = OP_ALG_ALGSEL_SHA512 |
1997 					   OP_ALG_AAI_HMAC_PRECOMP,
1998 		}
1999 	},
2000 	{
2001 		.aead = {
2002 			.base = {
2003 				.cra_name = "echainiv(authenc(hmac(sha512),"
2004 					    "cbc(aes)))",
2005 				.cra_driver_name = "echainiv-authenc-"
2006 						   "hmac-sha512-cbc-aes-"
2007 						   "caam-qi2",
2008 				.cra_blocksize = AES_BLOCK_SIZE,
2009 			},
2010 			.setkey = aead_setkey,
2011 			.setauthsize = aead_setauthsize,
2012 			.encrypt = aead_encrypt,
2013 			.decrypt = aead_decrypt,
2014 			.ivsize = AES_BLOCK_SIZE,
2015 			.maxauthsize = SHA512_DIGEST_SIZE,
2016 		},
2017 		.caam = {
2018 			.class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CBC,
2019 			.class2_alg_type = OP_ALG_ALGSEL_SHA512 |
2020 					   OP_ALG_AAI_HMAC_PRECOMP,
2021 			.geniv = true,
2022 		}
2023 	},
2024 	{
2025 		.aead = {
2026 			.base = {
2027 				.cra_name = "authenc(hmac(md5),cbc(des3_ede))",
2028 				.cra_driver_name = "authenc-hmac-md5-"
2029 						   "cbc-des3_ede-caam-qi2",
2030 				.cra_blocksize = DES3_EDE_BLOCK_SIZE,
2031 			},
2032 			.setkey = des3_aead_setkey,
2033 			.setauthsize = aead_setauthsize,
2034 			.encrypt = aead_encrypt,
2035 			.decrypt = aead_decrypt,
2036 			.ivsize = DES3_EDE_BLOCK_SIZE,
2037 			.maxauthsize = MD5_DIGEST_SIZE,
2038 		},
2039 		.caam = {
2040 			.class1_alg_type = OP_ALG_ALGSEL_3DES | OP_ALG_AAI_CBC,
2041 			.class2_alg_type = OP_ALG_ALGSEL_MD5 |
2042 					   OP_ALG_AAI_HMAC_PRECOMP,
2043 		}
2044 	},
2045 	{
2046 		.aead = {
2047 			.base = {
2048 				.cra_name = "echainiv(authenc(hmac(md5),"
2049 					    "cbc(des3_ede)))",
2050 				.cra_driver_name = "echainiv-authenc-hmac-md5-"
2051 						   "cbc-des3_ede-caam-qi2",
2052 				.cra_blocksize = DES3_EDE_BLOCK_SIZE,
2053 			},
2054 			.setkey = des3_aead_setkey,
2055 			.setauthsize = aead_setauthsize,
2056 			.encrypt = aead_encrypt,
2057 			.decrypt = aead_decrypt,
2058 			.ivsize = DES3_EDE_BLOCK_SIZE,
2059 			.maxauthsize = MD5_DIGEST_SIZE,
2060 		},
2061 		.caam = {
2062 			.class1_alg_type = OP_ALG_ALGSEL_3DES | OP_ALG_AAI_CBC,
2063 			.class2_alg_type = OP_ALG_ALGSEL_MD5 |
2064 					   OP_ALG_AAI_HMAC_PRECOMP,
2065 			.geniv = true,
2066 		}
2067 	},
2068 	{
2069 		.aead = {
2070 			.base = {
2071 				.cra_name = "authenc(hmac(sha1),"
2072 					    "cbc(des3_ede))",
2073 				.cra_driver_name = "authenc-hmac-sha1-"
2074 						   "cbc-des3_ede-caam-qi2",
2075 				.cra_blocksize = DES3_EDE_BLOCK_SIZE,
2076 			},
2077 			.setkey = des3_aead_setkey,
2078 			.setauthsize = aead_setauthsize,
2079 			.encrypt = aead_encrypt,
2080 			.decrypt = aead_decrypt,
2081 			.ivsize = DES3_EDE_BLOCK_SIZE,
2082 			.maxauthsize = SHA1_DIGEST_SIZE,
2083 		},
2084 		.caam = {
2085 			.class1_alg_type = OP_ALG_ALGSEL_3DES | OP_ALG_AAI_CBC,
2086 			.class2_alg_type = OP_ALG_ALGSEL_SHA1 |
2087 					   OP_ALG_AAI_HMAC_PRECOMP,
2088 		},
2089 	},
2090 	{
2091 		.aead = {
2092 			.base = {
2093 				.cra_name = "echainiv(authenc(hmac(sha1),"
2094 					    "cbc(des3_ede)))",
2095 				.cra_driver_name = "echainiv-authenc-"
2096 						   "hmac-sha1-"
2097 						   "cbc-des3_ede-caam-qi2",
2098 				.cra_blocksize = DES3_EDE_BLOCK_SIZE,
2099 			},
2100 			.setkey = des3_aead_setkey,
2101 			.setauthsize = aead_setauthsize,
2102 			.encrypt = aead_encrypt,
2103 			.decrypt = aead_decrypt,
2104 			.ivsize = DES3_EDE_BLOCK_SIZE,
2105 			.maxauthsize = SHA1_DIGEST_SIZE,
2106 		},
2107 		.caam = {
2108 			.class1_alg_type = OP_ALG_ALGSEL_3DES | OP_ALG_AAI_CBC,
2109 			.class2_alg_type = OP_ALG_ALGSEL_SHA1 |
2110 					   OP_ALG_AAI_HMAC_PRECOMP,
2111 			.geniv = true,
2112 		}
2113 	},
2114 	{
2115 		.aead = {
2116 			.base = {
2117 				.cra_name = "authenc(hmac(sha224),"
2118 					    "cbc(des3_ede))",
2119 				.cra_driver_name = "authenc-hmac-sha224-"
2120 						   "cbc-des3_ede-caam-qi2",
2121 				.cra_blocksize = DES3_EDE_BLOCK_SIZE,
2122 			},
2123 			.setkey = des3_aead_setkey,
2124 			.setauthsize = aead_setauthsize,
2125 			.encrypt = aead_encrypt,
2126 			.decrypt = aead_decrypt,
2127 			.ivsize = DES3_EDE_BLOCK_SIZE,
2128 			.maxauthsize = SHA224_DIGEST_SIZE,
2129 		},
2130 		.caam = {
2131 			.class1_alg_type = OP_ALG_ALGSEL_3DES | OP_ALG_AAI_CBC,
2132 			.class2_alg_type = OP_ALG_ALGSEL_SHA224 |
2133 					   OP_ALG_AAI_HMAC_PRECOMP,
2134 		},
2135 	},
2136 	{
2137 		.aead = {
2138 			.base = {
2139 				.cra_name = "echainiv(authenc(hmac(sha224),"
2140 					    "cbc(des3_ede)))",
2141 				.cra_driver_name = "echainiv-authenc-"
2142 						   "hmac-sha224-"
2143 						   "cbc-des3_ede-caam-qi2",
2144 				.cra_blocksize = DES3_EDE_BLOCK_SIZE,
2145 			},
2146 			.setkey = des3_aead_setkey,
2147 			.setauthsize = aead_setauthsize,
2148 			.encrypt = aead_encrypt,
2149 			.decrypt = aead_decrypt,
2150 			.ivsize = DES3_EDE_BLOCK_SIZE,
2151 			.maxauthsize = SHA224_DIGEST_SIZE,
2152 		},
2153 		.caam = {
2154 			.class1_alg_type = OP_ALG_ALGSEL_3DES | OP_ALG_AAI_CBC,
2155 			.class2_alg_type = OP_ALG_ALGSEL_SHA224 |
2156 					   OP_ALG_AAI_HMAC_PRECOMP,
2157 			.geniv = true,
2158 		}
2159 	},
2160 	{
2161 		.aead = {
2162 			.base = {
2163 				.cra_name = "authenc(hmac(sha256),"
2164 					    "cbc(des3_ede))",
2165 				.cra_driver_name = "authenc-hmac-sha256-"
2166 						   "cbc-des3_ede-caam-qi2",
2167 				.cra_blocksize = DES3_EDE_BLOCK_SIZE,
2168 			},
2169 			.setkey = des3_aead_setkey,
2170 			.setauthsize = aead_setauthsize,
2171 			.encrypt = aead_encrypt,
2172 			.decrypt = aead_decrypt,
2173 			.ivsize = DES3_EDE_BLOCK_SIZE,
2174 			.maxauthsize = SHA256_DIGEST_SIZE,
2175 		},
2176 		.caam = {
2177 			.class1_alg_type = OP_ALG_ALGSEL_3DES | OP_ALG_AAI_CBC,
2178 			.class2_alg_type = OP_ALG_ALGSEL_SHA256 |
2179 					   OP_ALG_AAI_HMAC_PRECOMP,
2180 		},
2181 	},
2182 	{
2183 		.aead = {
2184 			.base = {
2185 				.cra_name = "echainiv(authenc(hmac(sha256),"
2186 					    "cbc(des3_ede)))",
2187 				.cra_driver_name = "echainiv-authenc-"
2188 						   "hmac-sha256-"
2189 						   "cbc-des3_ede-caam-qi2",
2190 				.cra_blocksize = DES3_EDE_BLOCK_SIZE,
2191 			},
2192 			.setkey = des3_aead_setkey,
2193 			.setauthsize = aead_setauthsize,
2194 			.encrypt = aead_encrypt,
2195 			.decrypt = aead_decrypt,
2196 			.ivsize = DES3_EDE_BLOCK_SIZE,
2197 			.maxauthsize = SHA256_DIGEST_SIZE,
2198 		},
2199 		.caam = {
2200 			.class1_alg_type = OP_ALG_ALGSEL_3DES | OP_ALG_AAI_CBC,
2201 			.class2_alg_type = OP_ALG_ALGSEL_SHA256 |
2202 					   OP_ALG_AAI_HMAC_PRECOMP,
2203 			.geniv = true,
2204 		}
2205 	},
2206 	{
2207 		.aead = {
2208 			.base = {
2209 				.cra_name = "authenc(hmac(sha384),"
2210 					    "cbc(des3_ede))",
2211 				.cra_driver_name = "authenc-hmac-sha384-"
2212 						   "cbc-des3_ede-caam-qi2",
2213 				.cra_blocksize = DES3_EDE_BLOCK_SIZE,
2214 			},
2215 			.setkey = des3_aead_setkey,
2216 			.setauthsize = aead_setauthsize,
2217 			.encrypt = aead_encrypt,
2218 			.decrypt = aead_decrypt,
2219 			.ivsize = DES3_EDE_BLOCK_SIZE,
2220 			.maxauthsize = SHA384_DIGEST_SIZE,
2221 		},
2222 		.caam = {
2223 			.class1_alg_type = OP_ALG_ALGSEL_3DES | OP_ALG_AAI_CBC,
2224 			.class2_alg_type = OP_ALG_ALGSEL_SHA384 |
2225 					   OP_ALG_AAI_HMAC_PRECOMP,
2226 		},
2227 	},
2228 	{
2229 		.aead = {
2230 			.base = {
2231 				.cra_name = "echainiv(authenc(hmac(sha384),"
2232 					    "cbc(des3_ede)))",
2233 				.cra_driver_name = "echainiv-authenc-"
2234 						   "hmac-sha384-"
2235 						   "cbc-des3_ede-caam-qi2",
2236 				.cra_blocksize = DES3_EDE_BLOCK_SIZE,
2237 			},
2238 			.setkey = des3_aead_setkey,
2239 			.setauthsize = aead_setauthsize,
2240 			.encrypt = aead_encrypt,
2241 			.decrypt = aead_decrypt,
2242 			.ivsize = DES3_EDE_BLOCK_SIZE,
2243 			.maxauthsize = SHA384_DIGEST_SIZE,
2244 		},
2245 		.caam = {
2246 			.class1_alg_type = OP_ALG_ALGSEL_3DES | OP_ALG_AAI_CBC,
2247 			.class2_alg_type = OP_ALG_ALGSEL_SHA384 |
2248 					   OP_ALG_AAI_HMAC_PRECOMP,
2249 			.geniv = true,
2250 		}
2251 	},
2252 	{
2253 		.aead = {
2254 			.base = {
2255 				.cra_name = "authenc(hmac(sha512),"
2256 					    "cbc(des3_ede))",
2257 				.cra_driver_name = "authenc-hmac-sha512-"
2258 						   "cbc-des3_ede-caam-qi2",
2259 				.cra_blocksize = DES3_EDE_BLOCK_SIZE,
2260 			},
2261 			.setkey = des3_aead_setkey,
2262 			.setauthsize = aead_setauthsize,
2263 			.encrypt = aead_encrypt,
2264 			.decrypt = aead_decrypt,
2265 			.ivsize = DES3_EDE_BLOCK_SIZE,
2266 			.maxauthsize = SHA512_DIGEST_SIZE,
2267 		},
2268 		.caam = {
2269 			.class1_alg_type = OP_ALG_ALGSEL_3DES | OP_ALG_AAI_CBC,
2270 			.class2_alg_type = OP_ALG_ALGSEL_SHA512 |
2271 					   OP_ALG_AAI_HMAC_PRECOMP,
2272 		},
2273 	},
2274 	{
2275 		.aead = {
2276 			.base = {
2277 				.cra_name = "echainiv(authenc(hmac(sha512),"
2278 					    "cbc(des3_ede)))",
2279 				.cra_driver_name = "echainiv-authenc-"
2280 						   "hmac-sha512-"
2281 						   "cbc-des3_ede-caam-qi2",
2282 				.cra_blocksize = DES3_EDE_BLOCK_SIZE,
2283 			},
2284 			.setkey = des3_aead_setkey,
2285 			.setauthsize = aead_setauthsize,
2286 			.encrypt = aead_encrypt,
2287 			.decrypt = aead_decrypt,
2288 			.ivsize = DES3_EDE_BLOCK_SIZE,
2289 			.maxauthsize = SHA512_DIGEST_SIZE,
2290 		},
2291 		.caam = {
2292 			.class1_alg_type = OP_ALG_ALGSEL_3DES | OP_ALG_AAI_CBC,
2293 			.class2_alg_type = OP_ALG_ALGSEL_SHA512 |
2294 					   OP_ALG_AAI_HMAC_PRECOMP,
2295 			.geniv = true,
2296 		}
2297 	},
2298 	{
2299 		.aead = {
2300 			.base = {
2301 				.cra_name = "authenc(hmac(md5),cbc(des))",
2302 				.cra_driver_name = "authenc-hmac-md5-"
2303 						   "cbc-des-caam-qi2",
2304 				.cra_blocksize = DES_BLOCK_SIZE,
2305 			},
2306 			.setkey = aead_setkey,
2307 			.setauthsize = aead_setauthsize,
2308 			.encrypt = aead_encrypt,
2309 			.decrypt = aead_decrypt,
2310 			.ivsize = DES_BLOCK_SIZE,
2311 			.maxauthsize = MD5_DIGEST_SIZE,
2312 		},
2313 		.caam = {
2314 			.class1_alg_type = OP_ALG_ALGSEL_DES | OP_ALG_AAI_CBC,
2315 			.class2_alg_type = OP_ALG_ALGSEL_MD5 |
2316 					   OP_ALG_AAI_HMAC_PRECOMP,
2317 		},
2318 	},
2319 	{
2320 		.aead = {
2321 			.base = {
2322 				.cra_name = "echainiv(authenc(hmac(md5),"
2323 					    "cbc(des)))",
2324 				.cra_driver_name = "echainiv-authenc-hmac-md5-"
2325 						   "cbc-des-caam-qi2",
2326 				.cra_blocksize = DES_BLOCK_SIZE,
2327 			},
2328 			.setkey = aead_setkey,
2329 			.setauthsize = aead_setauthsize,
2330 			.encrypt = aead_encrypt,
2331 			.decrypt = aead_decrypt,
2332 			.ivsize = DES_BLOCK_SIZE,
2333 			.maxauthsize = MD5_DIGEST_SIZE,
2334 		},
2335 		.caam = {
2336 			.class1_alg_type = OP_ALG_ALGSEL_DES | OP_ALG_AAI_CBC,
2337 			.class2_alg_type = OP_ALG_ALGSEL_MD5 |
2338 					   OP_ALG_AAI_HMAC_PRECOMP,
2339 			.geniv = true,
2340 		}
2341 	},
2342 	{
2343 		.aead = {
2344 			.base = {
2345 				.cra_name = "authenc(hmac(sha1),cbc(des))",
2346 				.cra_driver_name = "authenc-hmac-sha1-"
2347 						   "cbc-des-caam-qi2",
2348 				.cra_blocksize = DES_BLOCK_SIZE,
2349 			},
2350 			.setkey = aead_setkey,
2351 			.setauthsize = aead_setauthsize,
2352 			.encrypt = aead_encrypt,
2353 			.decrypt = aead_decrypt,
2354 			.ivsize = DES_BLOCK_SIZE,
2355 			.maxauthsize = SHA1_DIGEST_SIZE,
2356 		},
2357 		.caam = {
2358 			.class1_alg_type = OP_ALG_ALGSEL_DES | OP_ALG_AAI_CBC,
2359 			.class2_alg_type = OP_ALG_ALGSEL_SHA1 |
2360 					   OP_ALG_AAI_HMAC_PRECOMP,
2361 		},
2362 	},
2363 	{
2364 		.aead = {
2365 			.base = {
2366 				.cra_name = "echainiv(authenc(hmac(sha1),"
2367 					    "cbc(des)))",
2368 				.cra_driver_name = "echainiv-authenc-"
2369 						   "hmac-sha1-cbc-des-caam-qi2",
2370 				.cra_blocksize = DES_BLOCK_SIZE,
2371 			},
2372 			.setkey = aead_setkey,
2373 			.setauthsize = aead_setauthsize,
2374 			.encrypt = aead_encrypt,
2375 			.decrypt = aead_decrypt,
2376 			.ivsize = DES_BLOCK_SIZE,
2377 			.maxauthsize = SHA1_DIGEST_SIZE,
2378 		},
2379 		.caam = {
2380 			.class1_alg_type = OP_ALG_ALGSEL_DES | OP_ALG_AAI_CBC,
2381 			.class2_alg_type = OP_ALG_ALGSEL_SHA1 |
2382 					   OP_ALG_AAI_HMAC_PRECOMP,
2383 			.geniv = true,
2384 		}
2385 	},
2386 	{
2387 		.aead = {
2388 			.base = {
2389 				.cra_name = "authenc(hmac(sha224),cbc(des))",
2390 				.cra_driver_name = "authenc-hmac-sha224-"
2391 						   "cbc-des-caam-qi2",
2392 				.cra_blocksize = DES_BLOCK_SIZE,
2393 			},
2394 			.setkey = aead_setkey,
2395 			.setauthsize = aead_setauthsize,
2396 			.encrypt = aead_encrypt,
2397 			.decrypt = aead_decrypt,
2398 			.ivsize = DES_BLOCK_SIZE,
2399 			.maxauthsize = SHA224_DIGEST_SIZE,
2400 		},
2401 		.caam = {
2402 			.class1_alg_type = OP_ALG_ALGSEL_DES | OP_ALG_AAI_CBC,
2403 			.class2_alg_type = OP_ALG_ALGSEL_SHA224 |
2404 					   OP_ALG_AAI_HMAC_PRECOMP,
2405 		},
2406 	},
2407 	{
2408 		.aead = {
2409 			.base = {
2410 				.cra_name = "echainiv(authenc(hmac(sha224),"
2411 					    "cbc(des)))",
2412 				.cra_driver_name = "echainiv-authenc-"
2413 						   "hmac-sha224-cbc-des-"
2414 						   "caam-qi2",
2415 				.cra_blocksize = DES_BLOCK_SIZE,
2416 			},
2417 			.setkey = aead_setkey,
2418 			.setauthsize = aead_setauthsize,
2419 			.encrypt = aead_encrypt,
2420 			.decrypt = aead_decrypt,
2421 			.ivsize = DES_BLOCK_SIZE,
2422 			.maxauthsize = SHA224_DIGEST_SIZE,
2423 		},
2424 		.caam = {
2425 			.class1_alg_type = OP_ALG_ALGSEL_DES | OP_ALG_AAI_CBC,
2426 			.class2_alg_type = OP_ALG_ALGSEL_SHA224 |
2427 					   OP_ALG_AAI_HMAC_PRECOMP,
2428 			.geniv = true,
2429 		}
2430 	},
2431 	{
2432 		.aead = {
2433 			.base = {
2434 				.cra_name = "authenc(hmac(sha256),cbc(des))",
2435 				.cra_driver_name = "authenc-hmac-sha256-"
2436 						   "cbc-des-caam-qi2",
2437 				.cra_blocksize = DES_BLOCK_SIZE,
2438 			},
2439 			.setkey = aead_setkey,
2440 			.setauthsize = aead_setauthsize,
2441 			.encrypt = aead_encrypt,
2442 			.decrypt = aead_decrypt,
2443 			.ivsize = DES_BLOCK_SIZE,
2444 			.maxauthsize = SHA256_DIGEST_SIZE,
2445 		},
2446 		.caam = {
2447 			.class1_alg_type = OP_ALG_ALGSEL_DES | OP_ALG_AAI_CBC,
2448 			.class2_alg_type = OP_ALG_ALGSEL_SHA256 |
2449 					   OP_ALG_AAI_HMAC_PRECOMP,
2450 		},
2451 	},
2452 	{
2453 		.aead = {
2454 			.base = {
2455 				.cra_name = "echainiv(authenc(hmac(sha256),"
2456 					    "cbc(des)))",
2457 				.cra_driver_name = "echainiv-authenc-"
2458 						   "hmac-sha256-cbc-des-"
2459 						   "caam-qi2",
2460 				.cra_blocksize = DES_BLOCK_SIZE,
2461 			},
2462 			.setkey = aead_setkey,
2463 			.setauthsize = aead_setauthsize,
2464 			.encrypt = aead_encrypt,
2465 			.decrypt = aead_decrypt,
2466 			.ivsize = DES_BLOCK_SIZE,
2467 			.maxauthsize = SHA256_DIGEST_SIZE,
2468 		},
2469 		.caam = {
2470 			.class1_alg_type = OP_ALG_ALGSEL_DES | OP_ALG_AAI_CBC,
2471 			.class2_alg_type = OP_ALG_ALGSEL_SHA256 |
2472 					   OP_ALG_AAI_HMAC_PRECOMP,
2473 			.geniv = true,
2474 		},
2475 	},
2476 	{
2477 		.aead = {
2478 			.base = {
2479 				.cra_name = "authenc(hmac(sha384),cbc(des))",
2480 				.cra_driver_name = "authenc-hmac-sha384-"
2481 						   "cbc-des-caam-qi2",
2482 				.cra_blocksize = DES_BLOCK_SIZE,
2483 			},
2484 			.setkey = aead_setkey,
2485 			.setauthsize = aead_setauthsize,
2486 			.encrypt = aead_encrypt,
2487 			.decrypt = aead_decrypt,
2488 			.ivsize = DES_BLOCK_SIZE,
2489 			.maxauthsize = SHA384_DIGEST_SIZE,
2490 		},
2491 		.caam = {
2492 			.class1_alg_type = OP_ALG_ALGSEL_DES | OP_ALG_AAI_CBC,
2493 			.class2_alg_type = OP_ALG_ALGSEL_SHA384 |
2494 					   OP_ALG_AAI_HMAC_PRECOMP,
2495 		},
2496 	},
2497 	{
2498 		.aead = {
2499 			.base = {
2500 				.cra_name = "echainiv(authenc(hmac(sha384),"
2501 					    "cbc(des)))",
2502 				.cra_driver_name = "echainiv-authenc-"
2503 						   "hmac-sha384-cbc-des-"
2504 						   "caam-qi2",
2505 				.cra_blocksize = DES_BLOCK_SIZE,
2506 			},
2507 			.setkey = aead_setkey,
2508 			.setauthsize = aead_setauthsize,
2509 			.encrypt = aead_encrypt,
2510 			.decrypt = aead_decrypt,
2511 			.ivsize = DES_BLOCK_SIZE,
2512 			.maxauthsize = SHA384_DIGEST_SIZE,
2513 		},
2514 		.caam = {
2515 			.class1_alg_type = OP_ALG_ALGSEL_DES | OP_ALG_AAI_CBC,
2516 			.class2_alg_type = OP_ALG_ALGSEL_SHA384 |
2517 					   OP_ALG_AAI_HMAC_PRECOMP,
2518 			.geniv = true,
2519 		}
2520 	},
2521 	{
2522 		.aead = {
2523 			.base = {
2524 				.cra_name = "authenc(hmac(sha512),cbc(des))",
2525 				.cra_driver_name = "authenc-hmac-sha512-"
2526 						   "cbc-des-caam-qi2",
2527 				.cra_blocksize = DES_BLOCK_SIZE,
2528 			},
2529 			.setkey = aead_setkey,
2530 			.setauthsize = aead_setauthsize,
2531 			.encrypt = aead_encrypt,
2532 			.decrypt = aead_decrypt,
2533 			.ivsize = DES_BLOCK_SIZE,
2534 			.maxauthsize = SHA512_DIGEST_SIZE,
2535 		},
2536 		.caam = {
2537 			.class1_alg_type = OP_ALG_ALGSEL_DES | OP_ALG_AAI_CBC,
2538 			.class2_alg_type = OP_ALG_ALGSEL_SHA512 |
2539 					   OP_ALG_AAI_HMAC_PRECOMP,
2540 		}
2541 	},
2542 	{
2543 		.aead = {
2544 			.base = {
2545 				.cra_name = "echainiv(authenc(hmac(sha512),"
2546 					    "cbc(des)))",
2547 				.cra_driver_name = "echainiv-authenc-"
2548 						   "hmac-sha512-cbc-des-"
2549 						   "caam-qi2",
2550 				.cra_blocksize = DES_BLOCK_SIZE,
2551 			},
2552 			.setkey = aead_setkey,
2553 			.setauthsize = aead_setauthsize,
2554 			.encrypt = aead_encrypt,
2555 			.decrypt = aead_decrypt,
2556 			.ivsize = DES_BLOCK_SIZE,
2557 			.maxauthsize = SHA512_DIGEST_SIZE,
2558 		},
2559 		.caam = {
2560 			.class1_alg_type = OP_ALG_ALGSEL_DES | OP_ALG_AAI_CBC,
2561 			.class2_alg_type = OP_ALG_ALGSEL_SHA512 |
2562 					   OP_ALG_AAI_HMAC_PRECOMP,
2563 			.geniv = true,
2564 		}
2565 	},
2566 	{
2567 		.aead = {
2568 			.base = {
2569 				.cra_name = "authenc(hmac(md5),"
2570 					    "rfc3686(ctr(aes)))",
2571 				.cra_driver_name = "authenc-hmac-md5-"
2572 						   "rfc3686-ctr-aes-caam-qi2",
2573 				.cra_blocksize = 1,
2574 			},
2575 			.setkey = aead_setkey,
2576 			.setauthsize = aead_setauthsize,
2577 			.encrypt = aead_encrypt,
2578 			.decrypt = aead_decrypt,
2579 			.ivsize = CTR_RFC3686_IV_SIZE,
2580 			.maxauthsize = MD5_DIGEST_SIZE,
2581 		},
2582 		.caam = {
2583 			.class1_alg_type = OP_ALG_ALGSEL_AES |
2584 					   OP_ALG_AAI_CTR_MOD128,
2585 			.class2_alg_type = OP_ALG_ALGSEL_MD5 |
2586 					   OP_ALG_AAI_HMAC_PRECOMP,
2587 			.rfc3686 = true,
2588 		},
2589 	},
2590 	{
2591 		.aead = {
2592 			.base = {
2593 				.cra_name = "seqiv(authenc("
2594 					    "hmac(md5),rfc3686(ctr(aes))))",
2595 				.cra_driver_name = "seqiv-authenc-hmac-md5-"
2596 						   "rfc3686-ctr-aes-caam-qi2",
2597 				.cra_blocksize = 1,
2598 			},
2599 			.setkey = aead_setkey,
2600 			.setauthsize = aead_setauthsize,
2601 			.encrypt = aead_encrypt,
2602 			.decrypt = aead_decrypt,
2603 			.ivsize = CTR_RFC3686_IV_SIZE,
2604 			.maxauthsize = MD5_DIGEST_SIZE,
2605 		},
2606 		.caam = {
2607 			.class1_alg_type = OP_ALG_ALGSEL_AES |
2608 					   OP_ALG_AAI_CTR_MOD128,
2609 			.class2_alg_type = OP_ALG_ALGSEL_MD5 |
2610 					   OP_ALG_AAI_HMAC_PRECOMP,
2611 			.rfc3686 = true,
2612 			.geniv = true,
2613 		},
2614 	},
2615 	{
2616 		.aead = {
2617 			.base = {
2618 				.cra_name = "authenc(hmac(sha1),"
2619 					    "rfc3686(ctr(aes)))",
2620 				.cra_driver_name = "authenc-hmac-sha1-"
2621 						   "rfc3686-ctr-aes-caam-qi2",
2622 				.cra_blocksize = 1,
2623 			},
2624 			.setkey = aead_setkey,
2625 			.setauthsize = aead_setauthsize,
2626 			.encrypt = aead_encrypt,
2627 			.decrypt = aead_decrypt,
2628 			.ivsize = CTR_RFC3686_IV_SIZE,
2629 			.maxauthsize = SHA1_DIGEST_SIZE,
2630 		},
2631 		.caam = {
2632 			.class1_alg_type = OP_ALG_ALGSEL_AES |
2633 					   OP_ALG_AAI_CTR_MOD128,
2634 			.class2_alg_type = OP_ALG_ALGSEL_SHA1 |
2635 					   OP_ALG_AAI_HMAC_PRECOMP,
2636 			.rfc3686 = true,
2637 		},
2638 	},
2639 	{
2640 		.aead = {
2641 			.base = {
2642 				.cra_name = "seqiv(authenc("
2643 					    "hmac(sha1),rfc3686(ctr(aes))))",
2644 				.cra_driver_name = "seqiv-authenc-hmac-sha1-"
2645 						   "rfc3686-ctr-aes-caam-qi2",
2646 				.cra_blocksize = 1,
2647 			},
2648 			.setkey = aead_setkey,
2649 			.setauthsize = aead_setauthsize,
2650 			.encrypt = aead_encrypt,
2651 			.decrypt = aead_decrypt,
2652 			.ivsize = CTR_RFC3686_IV_SIZE,
2653 			.maxauthsize = SHA1_DIGEST_SIZE,
2654 		},
2655 		.caam = {
2656 			.class1_alg_type = OP_ALG_ALGSEL_AES |
2657 					   OP_ALG_AAI_CTR_MOD128,
2658 			.class2_alg_type = OP_ALG_ALGSEL_SHA1 |
2659 					   OP_ALG_AAI_HMAC_PRECOMP,
2660 			.rfc3686 = true,
2661 			.geniv = true,
2662 		},
2663 	},
2664 	{
2665 		.aead = {
2666 			.base = {
2667 				.cra_name = "authenc(hmac(sha224),"
2668 					    "rfc3686(ctr(aes)))",
2669 				.cra_driver_name = "authenc-hmac-sha224-"
2670 						   "rfc3686-ctr-aes-caam-qi2",
2671 				.cra_blocksize = 1,
2672 			},
2673 			.setkey = aead_setkey,
2674 			.setauthsize = aead_setauthsize,
2675 			.encrypt = aead_encrypt,
2676 			.decrypt = aead_decrypt,
2677 			.ivsize = CTR_RFC3686_IV_SIZE,
2678 			.maxauthsize = SHA224_DIGEST_SIZE,
2679 		},
2680 		.caam = {
2681 			.class1_alg_type = OP_ALG_ALGSEL_AES |
2682 					   OP_ALG_AAI_CTR_MOD128,
2683 			.class2_alg_type = OP_ALG_ALGSEL_SHA224 |
2684 					   OP_ALG_AAI_HMAC_PRECOMP,
2685 			.rfc3686 = true,
2686 		},
2687 	},
2688 	{
2689 		.aead = {
2690 			.base = {
2691 				.cra_name = "seqiv(authenc("
2692 					    "hmac(sha224),rfc3686(ctr(aes))))",
2693 				.cra_driver_name = "seqiv-authenc-hmac-sha224-"
2694 						   "rfc3686-ctr-aes-caam-qi2",
2695 				.cra_blocksize = 1,
2696 			},
2697 			.setkey = aead_setkey,
2698 			.setauthsize = aead_setauthsize,
2699 			.encrypt = aead_encrypt,
2700 			.decrypt = aead_decrypt,
2701 			.ivsize = CTR_RFC3686_IV_SIZE,
2702 			.maxauthsize = SHA224_DIGEST_SIZE,
2703 		},
2704 		.caam = {
2705 			.class1_alg_type = OP_ALG_ALGSEL_AES |
2706 					   OP_ALG_AAI_CTR_MOD128,
2707 			.class2_alg_type = OP_ALG_ALGSEL_SHA224 |
2708 					   OP_ALG_AAI_HMAC_PRECOMP,
2709 			.rfc3686 = true,
2710 			.geniv = true,
2711 		},
2712 	},
2713 	{
2714 		.aead = {
2715 			.base = {
2716 				.cra_name = "authenc(hmac(sha256),"
2717 					    "rfc3686(ctr(aes)))",
2718 				.cra_driver_name = "authenc-hmac-sha256-"
2719 						   "rfc3686-ctr-aes-caam-qi2",
2720 				.cra_blocksize = 1,
2721 			},
2722 			.setkey = aead_setkey,
2723 			.setauthsize = aead_setauthsize,
2724 			.encrypt = aead_encrypt,
2725 			.decrypt = aead_decrypt,
2726 			.ivsize = CTR_RFC3686_IV_SIZE,
2727 			.maxauthsize = SHA256_DIGEST_SIZE,
2728 		},
2729 		.caam = {
2730 			.class1_alg_type = OP_ALG_ALGSEL_AES |
2731 					   OP_ALG_AAI_CTR_MOD128,
2732 			.class2_alg_type = OP_ALG_ALGSEL_SHA256 |
2733 					   OP_ALG_AAI_HMAC_PRECOMP,
2734 			.rfc3686 = true,
2735 		},
2736 	},
2737 	{
2738 		.aead = {
2739 			.base = {
2740 				.cra_name = "seqiv(authenc(hmac(sha256),"
2741 					    "rfc3686(ctr(aes))))",
2742 				.cra_driver_name = "seqiv-authenc-hmac-sha256-"
2743 						   "rfc3686-ctr-aes-caam-qi2",
2744 				.cra_blocksize = 1,
2745 			},
2746 			.setkey = aead_setkey,
2747 			.setauthsize = aead_setauthsize,
2748 			.encrypt = aead_encrypt,
2749 			.decrypt = aead_decrypt,
2750 			.ivsize = CTR_RFC3686_IV_SIZE,
2751 			.maxauthsize = SHA256_DIGEST_SIZE,
2752 		},
2753 		.caam = {
2754 			.class1_alg_type = OP_ALG_ALGSEL_AES |
2755 					   OP_ALG_AAI_CTR_MOD128,
2756 			.class2_alg_type = OP_ALG_ALGSEL_SHA256 |
2757 					   OP_ALG_AAI_HMAC_PRECOMP,
2758 			.rfc3686 = true,
2759 			.geniv = true,
2760 		},
2761 	},
2762 	{
2763 		.aead = {
2764 			.base = {
2765 				.cra_name = "authenc(hmac(sha384),"
2766 					    "rfc3686(ctr(aes)))",
2767 				.cra_driver_name = "authenc-hmac-sha384-"
2768 						   "rfc3686-ctr-aes-caam-qi2",
2769 				.cra_blocksize = 1,
2770 			},
2771 			.setkey = aead_setkey,
2772 			.setauthsize = aead_setauthsize,
2773 			.encrypt = aead_encrypt,
2774 			.decrypt = aead_decrypt,
2775 			.ivsize = CTR_RFC3686_IV_SIZE,
2776 			.maxauthsize = SHA384_DIGEST_SIZE,
2777 		},
2778 		.caam = {
2779 			.class1_alg_type = OP_ALG_ALGSEL_AES |
2780 					   OP_ALG_AAI_CTR_MOD128,
2781 			.class2_alg_type = OP_ALG_ALGSEL_SHA384 |
2782 					   OP_ALG_AAI_HMAC_PRECOMP,
2783 			.rfc3686 = true,
2784 		},
2785 	},
2786 	{
2787 		.aead = {
2788 			.base = {
2789 				.cra_name = "seqiv(authenc(hmac(sha384),"
2790 					    "rfc3686(ctr(aes))))",
2791 				.cra_driver_name = "seqiv-authenc-hmac-sha384-"
2792 						   "rfc3686-ctr-aes-caam-qi2",
2793 				.cra_blocksize = 1,
2794 			},
2795 			.setkey = aead_setkey,
2796 			.setauthsize = aead_setauthsize,
2797 			.encrypt = aead_encrypt,
2798 			.decrypt = aead_decrypt,
2799 			.ivsize = CTR_RFC3686_IV_SIZE,
2800 			.maxauthsize = SHA384_DIGEST_SIZE,
2801 		},
2802 		.caam = {
2803 			.class1_alg_type = OP_ALG_ALGSEL_AES |
2804 					   OP_ALG_AAI_CTR_MOD128,
2805 			.class2_alg_type = OP_ALG_ALGSEL_SHA384 |
2806 					   OP_ALG_AAI_HMAC_PRECOMP,
2807 			.rfc3686 = true,
2808 			.geniv = true,
2809 		},
2810 	},
2811 	{
2812 		.aead = {
2813 			.base = {
2814 				.cra_name = "rfc7539(chacha20,poly1305)",
2815 				.cra_driver_name = "rfc7539-chacha20-poly1305-"
2816 						   "caam-qi2",
2817 				.cra_blocksize = 1,
2818 			},
2819 			.setkey = chachapoly_setkey,
2820 			.setauthsize = chachapoly_setauthsize,
2821 			.encrypt = aead_encrypt,
2822 			.decrypt = aead_decrypt,
2823 			.ivsize = CHACHAPOLY_IV_SIZE,
2824 			.maxauthsize = POLY1305_DIGEST_SIZE,
2825 		},
2826 		.caam = {
2827 			.class1_alg_type = OP_ALG_ALGSEL_CHACHA20 |
2828 					   OP_ALG_AAI_AEAD,
2829 			.class2_alg_type = OP_ALG_ALGSEL_POLY1305 |
2830 					   OP_ALG_AAI_AEAD,
2831 			.nodkp = true,
2832 		},
2833 	},
2834 	{
2835 		.aead = {
2836 			.base = {
2837 				.cra_name = "rfc7539esp(chacha20,poly1305)",
2838 				.cra_driver_name = "rfc7539esp-chacha20-"
2839 						   "poly1305-caam-qi2",
2840 				.cra_blocksize = 1,
2841 			},
2842 			.setkey = chachapoly_setkey,
2843 			.setauthsize = chachapoly_setauthsize,
2844 			.encrypt = aead_encrypt,
2845 			.decrypt = aead_decrypt,
2846 			.ivsize = 8,
2847 			.maxauthsize = POLY1305_DIGEST_SIZE,
2848 		},
2849 		.caam = {
2850 			.class1_alg_type = OP_ALG_ALGSEL_CHACHA20 |
2851 					   OP_ALG_AAI_AEAD,
2852 			.class2_alg_type = OP_ALG_ALGSEL_POLY1305 |
2853 					   OP_ALG_AAI_AEAD,
2854 			.nodkp = true,
2855 		},
2856 	},
2857 	{
2858 		.aead = {
2859 			.base = {
2860 				.cra_name = "authenc(hmac(sha512),"
2861 					    "rfc3686(ctr(aes)))",
2862 				.cra_driver_name = "authenc-hmac-sha512-"
2863 						   "rfc3686-ctr-aes-caam-qi2",
2864 				.cra_blocksize = 1,
2865 			},
2866 			.setkey = aead_setkey,
2867 			.setauthsize = aead_setauthsize,
2868 			.encrypt = aead_encrypt,
2869 			.decrypt = aead_decrypt,
2870 			.ivsize = CTR_RFC3686_IV_SIZE,
2871 			.maxauthsize = SHA512_DIGEST_SIZE,
2872 		},
2873 		.caam = {
2874 			.class1_alg_type = OP_ALG_ALGSEL_AES |
2875 					   OP_ALG_AAI_CTR_MOD128,
2876 			.class2_alg_type = OP_ALG_ALGSEL_SHA512 |
2877 					   OP_ALG_AAI_HMAC_PRECOMP,
2878 			.rfc3686 = true,
2879 		},
2880 	},
2881 	{
2882 		.aead = {
2883 			.base = {
2884 				.cra_name = "seqiv(authenc(hmac(sha512),"
2885 					    "rfc3686(ctr(aes))))",
2886 				.cra_driver_name = "seqiv-authenc-hmac-sha512-"
2887 						   "rfc3686-ctr-aes-caam-qi2",
2888 				.cra_blocksize = 1,
2889 			},
2890 			.setkey = aead_setkey,
2891 			.setauthsize = aead_setauthsize,
2892 			.encrypt = aead_encrypt,
2893 			.decrypt = aead_decrypt,
2894 			.ivsize = CTR_RFC3686_IV_SIZE,
2895 			.maxauthsize = SHA512_DIGEST_SIZE,
2896 		},
2897 		.caam = {
2898 			.class1_alg_type = OP_ALG_ALGSEL_AES |
2899 					   OP_ALG_AAI_CTR_MOD128,
2900 			.class2_alg_type = OP_ALG_ALGSEL_SHA512 |
2901 					   OP_ALG_AAI_HMAC_PRECOMP,
2902 			.rfc3686 = true,
2903 			.geniv = true,
2904 		},
2905 	},
2906 };
2907 
2908 static void caam_skcipher_alg_init(struct caam_skcipher_alg *t_alg)
2909 {
2910 	struct skcipher_alg *alg = &t_alg->skcipher;
2911 
2912 	alg->base.cra_module = THIS_MODULE;
2913 	alg->base.cra_priority = CAAM_CRA_PRIORITY;
2914 	alg->base.cra_ctxsize = sizeof(struct caam_ctx);
2915 	alg->base.cra_flags = CRYPTO_ALG_ASYNC | CRYPTO_ALG_KERN_DRIVER_ONLY;
2916 
2917 	alg->init = caam_cra_init_skcipher;
2918 	alg->exit = caam_cra_exit;
2919 }
2920 
2921 static void caam_aead_alg_init(struct caam_aead_alg *t_alg)
2922 {
2923 	struct aead_alg *alg = &t_alg->aead;
2924 
2925 	alg->base.cra_module = THIS_MODULE;
2926 	alg->base.cra_priority = CAAM_CRA_PRIORITY;
2927 	alg->base.cra_ctxsize = sizeof(struct caam_ctx);
2928 	alg->base.cra_flags = CRYPTO_ALG_ASYNC | CRYPTO_ALG_KERN_DRIVER_ONLY;
2929 
2930 	alg->init = caam_cra_init_aead;
2931 	alg->exit = caam_cra_exit_aead;
2932 }
2933 
2934 /* max hash key is max split key size */
2935 #define CAAM_MAX_HASH_KEY_SIZE		(SHA512_DIGEST_SIZE * 2)
2936 
2937 #define CAAM_MAX_HASH_BLOCK_SIZE	SHA512_BLOCK_SIZE
2938 
2939 /* caam context sizes for hashes: running digest + 8 */
2940 #define HASH_MSG_LEN			8
2941 #define MAX_CTX_LEN			(HASH_MSG_LEN + SHA512_DIGEST_SIZE)
2942 
2943 enum hash_optype {
2944 	UPDATE = 0,
2945 	UPDATE_FIRST,
2946 	FINALIZE,
2947 	DIGEST,
2948 	HASH_NUM_OP
2949 };
2950 
2951 /**
2952  * caam_hash_ctx - ahash per-session context
2953  * @flc: Flow Contexts array
2954  * @key: authentication key
2955  * @flc_dma: I/O virtual addresses of the Flow Contexts
2956  * @dev: dpseci device
2957  * @ctx_len: size of Context Register
2958  * @adata: hashing algorithm details
2959  */
2960 struct caam_hash_ctx {
2961 	struct caam_flc flc[HASH_NUM_OP];
2962 	u8 key[CAAM_MAX_HASH_BLOCK_SIZE] ____cacheline_aligned;
2963 	dma_addr_t flc_dma[HASH_NUM_OP];
2964 	struct device *dev;
2965 	int ctx_len;
2966 	struct alginfo adata;
2967 };
2968 
2969 /* ahash state */
2970 struct caam_hash_state {
2971 	struct caam_request caam_req;
2972 	dma_addr_t buf_dma;
2973 	dma_addr_t ctx_dma;
2974 	int ctx_dma_len;
2975 	u8 buf[CAAM_MAX_HASH_BLOCK_SIZE] ____cacheline_aligned;
2976 	int buflen;
2977 	int next_buflen;
2978 	u8 caam_ctx[MAX_CTX_LEN] ____cacheline_aligned;
2979 	int (*update)(struct ahash_request *req);
2980 	int (*final)(struct ahash_request *req);
2981 	int (*finup)(struct ahash_request *req);
2982 };
2983 
2984 struct caam_export_state {
2985 	u8 buf[CAAM_MAX_HASH_BLOCK_SIZE];
2986 	u8 caam_ctx[MAX_CTX_LEN];
2987 	int buflen;
2988 	int (*update)(struct ahash_request *req);
2989 	int (*final)(struct ahash_request *req);
2990 	int (*finup)(struct ahash_request *req);
2991 };
2992 
2993 /* Map current buffer in state (if length > 0) and put it in link table */
2994 static inline int buf_map_to_qm_sg(struct device *dev,
2995 				   struct dpaa2_sg_entry *qm_sg,
2996 				   struct caam_hash_state *state)
2997 {
2998 	int buflen = state->buflen;
2999 
3000 	if (!buflen)
3001 		return 0;
3002 
3003 	state->buf_dma = dma_map_single(dev, state->buf, buflen,
3004 					DMA_TO_DEVICE);
3005 	if (dma_mapping_error(dev, state->buf_dma)) {
3006 		dev_err(dev, "unable to map buf\n");
3007 		state->buf_dma = 0;
3008 		return -ENOMEM;
3009 	}
3010 
3011 	dma_to_qm_sg_one(qm_sg, state->buf_dma, buflen, 0);
3012 
3013 	return 0;
3014 }
3015 
3016 /* Map state->caam_ctx, and add it to link table */
3017 static inline int ctx_map_to_qm_sg(struct device *dev,
3018 				   struct caam_hash_state *state, int ctx_len,
3019 				   struct dpaa2_sg_entry *qm_sg, u32 flag)
3020 {
3021 	state->ctx_dma_len = ctx_len;
3022 	state->ctx_dma = dma_map_single(dev, state->caam_ctx, ctx_len, flag);
3023 	if (dma_mapping_error(dev, state->ctx_dma)) {
3024 		dev_err(dev, "unable to map ctx\n");
3025 		state->ctx_dma = 0;
3026 		return -ENOMEM;
3027 	}
3028 
3029 	dma_to_qm_sg_one(qm_sg, state->ctx_dma, ctx_len, 0);
3030 
3031 	return 0;
3032 }
3033 
3034 static int ahash_set_sh_desc(struct crypto_ahash *ahash)
3035 {
3036 	struct caam_hash_ctx *ctx = crypto_ahash_ctx(ahash);
3037 	int digestsize = crypto_ahash_digestsize(ahash);
3038 	struct dpaa2_caam_priv *priv = dev_get_drvdata(ctx->dev);
3039 	struct caam_flc *flc;
3040 	u32 *desc;
3041 
3042 	/* ahash_update shared descriptor */
3043 	flc = &ctx->flc[UPDATE];
3044 	desc = flc->sh_desc;
3045 	cnstr_shdsc_ahash(desc, &ctx->adata, OP_ALG_AS_UPDATE, ctx->ctx_len,
3046 			  ctx->ctx_len, true, priv->sec_attr.era);
3047 	flc->flc[1] = cpu_to_caam32(desc_len(desc)); /* SDL */
3048 	dma_sync_single_for_device(ctx->dev, ctx->flc_dma[UPDATE],
3049 				   desc_bytes(desc), DMA_BIDIRECTIONAL);
3050 	print_hex_dump_debug("ahash update shdesc@" __stringify(__LINE__)": ",
3051 			     DUMP_PREFIX_ADDRESS, 16, 4, desc, desc_bytes(desc),
3052 			     1);
3053 
3054 	/* ahash_update_first shared descriptor */
3055 	flc = &ctx->flc[UPDATE_FIRST];
3056 	desc = flc->sh_desc;
3057 	cnstr_shdsc_ahash(desc, &ctx->adata, OP_ALG_AS_INIT, ctx->ctx_len,
3058 			  ctx->ctx_len, false, priv->sec_attr.era);
3059 	flc->flc[1] = cpu_to_caam32(desc_len(desc)); /* SDL */
3060 	dma_sync_single_for_device(ctx->dev, ctx->flc_dma[UPDATE_FIRST],
3061 				   desc_bytes(desc), DMA_BIDIRECTIONAL);
3062 	print_hex_dump_debug("ahash update first shdesc@" __stringify(__LINE__)": ",
3063 			     DUMP_PREFIX_ADDRESS, 16, 4, desc, desc_bytes(desc),
3064 			     1);
3065 
3066 	/* ahash_final shared descriptor */
3067 	flc = &ctx->flc[FINALIZE];
3068 	desc = flc->sh_desc;
3069 	cnstr_shdsc_ahash(desc, &ctx->adata, OP_ALG_AS_FINALIZE, digestsize,
3070 			  ctx->ctx_len, true, priv->sec_attr.era);
3071 	flc->flc[1] = cpu_to_caam32(desc_len(desc)); /* SDL */
3072 	dma_sync_single_for_device(ctx->dev, ctx->flc_dma[FINALIZE],
3073 				   desc_bytes(desc), DMA_BIDIRECTIONAL);
3074 	print_hex_dump_debug("ahash final shdesc@" __stringify(__LINE__)": ",
3075 			     DUMP_PREFIX_ADDRESS, 16, 4, desc, desc_bytes(desc),
3076 			     1);
3077 
3078 	/* ahash_digest shared descriptor */
3079 	flc = &ctx->flc[DIGEST];
3080 	desc = flc->sh_desc;
3081 	cnstr_shdsc_ahash(desc, &ctx->adata, OP_ALG_AS_INITFINAL, digestsize,
3082 			  ctx->ctx_len, false, priv->sec_attr.era);
3083 	flc->flc[1] = cpu_to_caam32(desc_len(desc)); /* SDL */
3084 	dma_sync_single_for_device(ctx->dev, ctx->flc_dma[DIGEST],
3085 				   desc_bytes(desc), DMA_BIDIRECTIONAL);
3086 	print_hex_dump_debug("ahash digest shdesc@" __stringify(__LINE__)": ",
3087 			     DUMP_PREFIX_ADDRESS, 16, 4, desc, desc_bytes(desc),
3088 			     1);
3089 
3090 	return 0;
3091 }
3092 
3093 struct split_key_sh_result {
3094 	struct completion completion;
3095 	int err;
3096 	struct device *dev;
3097 };
3098 
3099 static void split_key_sh_done(void *cbk_ctx, u32 err)
3100 {
3101 	struct split_key_sh_result *res = cbk_ctx;
3102 
3103 	dev_dbg(res->dev, "%s %d: err 0x%x\n", __func__, __LINE__, err);
3104 
3105 	res->err = err ? caam_qi2_strstatus(res->dev, err) : 0;
3106 	complete(&res->completion);
3107 }
3108 
3109 /* Digest hash size if it is too large */
3110 static int hash_digest_key(struct caam_hash_ctx *ctx, u32 *keylen, u8 *key,
3111 			   u32 digestsize)
3112 {
3113 	struct caam_request *req_ctx;
3114 	u32 *desc;
3115 	struct split_key_sh_result result;
3116 	dma_addr_t key_dma;
3117 	struct caam_flc *flc;
3118 	dma_addr_t flc_dma;
3119 	int ret = -ENOMEM;
3120 	struct dpaa2_fl_entry *in_fle, *out_fle;
3121 
3122 	req_ctx = kzalloc(sizeof(*req_ctx), GFP_KERNEL | GFP_DMA);
3123 	if (!req_ctx)
3124 		return -ENOMEM;
3125 
3126 	in_fle = &req_ctx->fd_flt[1];
3127 	out_fle = &req_ctx->fd_flt[0];
3128 
3129 	flc = kzalloc(sizeof(*flc), GFP_KERNEL | GFP_DMA);
3130 	if (!flc)
3131 		goto err_flc;
3132 
3133 	key_dma = dma_map_single(ctx->dev, key, *keylen, DMA_BIDIRECTIONAL);
3134 	if (dma_mapping_error(ctx->dev, key_dma)) {
3135 		dev_err(ctx->dev, "unable to map key memory\n");
3136 		goto err_key_dma;
3137 	}
3138 
3139 	desc = flc->sh_desc;
3140 
3141 	init_sh_desc(desc, 0);
3142 
3143 	/* descriptor to perform unkeyed hash on key_in */
3144 	append_operation(desc, ctx->adata.algtype | OP_ALG_ENCRYPT |
3145 			 OP_ALG_AS_INITFINAL);
3146 	append_seq_fifo_load(desc, *keylen, FIFOLD_CLASS_CLASS2 |
3147 			     FIFOLD_TYPE_LAST2 | FIFOLD_TYPE_MSG);
3148 	append_seq_store(desc, digestsize, LDST_CLASS_2_CCB |
3149 			 LDST_SRCDST_BYTE_CONTEXT);
3150 
3151 	flc->flc[1] = cpu_to_caam32(desc_len(desc)); /* SDL */
3152 	flc_dma = dma_map_single(ctx->dev, flc, sizeof(flc->flc) +
3153 				 desc_bytes(desc), DMA_TO_DEVICE);
3154 	if (dma_mapping_error(ctx->dev, flc_dma)) {
3155 		dev_err(ctx->dev, "unable to map shared descriptor\n");
3156 		goto err_flc_dma;
3157 	}
3158 
3159 	dpaa2_fl_set_final(in_fle, true);
3160 	dpaa2_fl_set_format(in_fle, dpaa2_fl_single);
3161 	dpaa2_fl_set_addr(in_fle, key_dma);
3162 	dpaa2_fl_set_len(in_fle, *keylen);
3163 	dpaa2_fl_set_format(out_fle, dpaa2_fl_single);
3164 	dpaa2_fl_set_addr(out_fle, key_dma);
3165 	dpaa2_fl_set_len(out_fle, digestsize);
3166 
3167 	print_hex_dump_debug("key_in@" __stringify(__LINE__)": ",
3168 			     DUMP_PREFIX_ADDRESS, 16, 4, key, *keylen, 1);
3169 	print_hex_dump_debug("shdesc@" __stringify(__LINE__)": ",
3170 			     DUMP_PREFIX_ADDRESS, 16, 4, desc, desc_bytes(desc),
3171 			     1);
3172 
3173 	result.err = 0;
3174 	init_completion(&result.completion);
3175 	result.dev = ctx->dev;
3176 
3177 	req_ctx->flc = flc;
3178 	req_ctx->flc_dma = flc_dma;
3179 	req_ctx->cbk = split_key_sh_done;
3180 	req_ctx->ctx = &result;
3181 
3182 	ret = dpaa2_caam_enqueue(ctx->dev, req_ctx);
3183 	if (ret == -EINPROGRESS) {
3184 		/* in progress */
3185 		wait_for_completion(&result.completion);
3186 		ret = result.err;
3187 		print_hex_dump_debug("digested key@" __stringify(__LINE__)": ",
3188 				     DUMP_PREFIX_ADDRESS, 16, 4, key,
3189 				     digestsize, 1);
3190 	}
3191 
3192 	dma_unmap_single(ctx->dev, flc_dma, sizeof(flc->flc) + desc_bytes(desc),
3193 			 DMA_TO_DEVICE);
3194 err_flc_dma:
3195 	dma_unmap_single(ctx->dev, key_dma, *keylen, DMA_BIDIRECTIONAL);
3196 err_key_dma:
3197 	kfree(flc);
3198 err_flc:
3199 	kfree(req_ctx);
3200 
3201 	*keylen = digestsize;
3202 
3203 	return ret;
3204 }
3205 
3206 static int ahash_setkey(struct crypto_ahash *ahash, const u8 *key,
3207 			unsigned int keylen)
3208 {
3209 	struct caam_hash_ctx *ctx = crypto_ahash_ctx(ahash);
3210 	unsigned int blocksize = crypto_tfm_alg_blocksize(&ahash->base);
3211 	unsigned int digestsize = crypto_ahash_digestsize(ahash);
3212 	int ret;
3213 	u8 *hashed_key = NULL;
3214 
3215 	dev_dbg(ctx->dev, "keylen %d blocksize %d\n", keylen, blocksize);
3216 
3217 	if (keylen > blocksize) {
3218 		hashed_key = kmemdup(key, keylen, GFP_KERNEL | GFP_DMA);
3219 		if (!hashed_key)
3220 			return -ENOMEM;
3221 		ret = hash_digest_key(ctx, &keylen, hashed_key, digestsize);
3222 		if (ret)
3223 			goto bad_free_key;
3224 		key = hashed_key;
3225 	}
3226 
3227 	ctx->adata.keylen = keylen;
3228 	ctx->adata.keylen_pad = split_key_len(ctx->adata.algtype &
3229 					      OP_ALG_ALGSEL_MASK);
3230 	if (ctx->adata.keylen_pad > CAAM_MAX_HASH_KEY_SIZE)
3231 		goto bad_free_key;
3232 
3233 	ctx->adata.key_virt = key;
3234 	ctx->adata.key_inline = true;
3235 
3236 	/*
3237 	 * In case |user key| > |derived key|, using DKP<imm,imm> would result
3238 	 * in invalid opcodes (last bytes of user key) in the resulting
3239 	 * descriptor. Use DKP<ptr,imm> instead => both virtual and dma key
3240 	 * addresses are needed.
3241 	 */
3242 	if (keylen > ctx->adata.keylen_pad) {
3243 		memcpy(ctx->key, key, keylen);
3244 		dma_sync_single_for_device(ctx->dev, ctx->adata.key_dma,
3245 					   ctx->adata.keylen_pad,
3246 					   DMA_TO_DEVICE);
3247 	}
3248 
3249 	ret = ahash_set_sh_desc(ahash);
3250 	kfree(hashed_key);
3251 	return ret;
3252 bad_free_key:
3253 	kfree(hashed_key);
3254 	return -EINVAL;
3255 }
3256 
3257 static inline void ahash_unmap(struct device *dev, struct ahash_edesc *edesc,
3258 			       struct ahash_request *req)
3259 {
3260 	struct caam_hash_state *state = ahash_request_ctx(req);
3261 
3262 	if (edesc->src_nents)
3263 		dma_unmap_sg(dev, req->src, edesc->src_nents, DMA_TO_DEVICE);
3264 
3265 	if (edesc->qm_sg_bytes)
3266 		dma_unmap_single(dev, edesc->qm_sg_dma, edesc->qm_sg_bytes,
3267 				 DMA_TO_DEVICE);
3268 
3269 	if (state->buf_dma) {
3270 		dma_unmap_single(dev, state->buf_dma, state->buflen,
3271 				 DMA_TO_DEVICE);
3272 		state->buf_dma = 0;
3273 	}
3274 }
3275 
3276 static inline void ahash_unmap_ctx(struct device *dev,
3277 				   struct ahash_edesc *edesc,
3278 				   struct ahash_request *req, u32 flag)
3279 {
3280 	struct caam_hash_state *state = ahash_request_ctx(req);
3281 
3282 	if (state->ctx_dma) {
3283 		dma_unmap_single(dev, state->ctx_dma, state->ctx_dma_len, flag);
3284 		state->ctx_dma = 0;
3285 	}
3286 	ahash_unmap(dev, edesc, req);
3287 }
3288 
3289 static void ahash_done(void *cbk_ctx, u32 status)
3290 {
3291 	struct crypto_async_request *areq = cbk_ctx;
3292 	struct ahash_request *req = ahash_request_cast(areq);
3293 	struct crypto_ahash *ahash = crypto_ahash_reqtfm(req);
3294 	struct caam_hash_state *state = ahash_request_ctx(req);
3295 	struct ahash_edesc *edesc = state->caam_req.edesc;
3296 	struct caam_hash_ctx *ctx = crypto_ahash_ctx(ahash);
3297 	int digestsize = crypto_ahash_digestsize(ahash);
3298 	int ecode = 0;
3299 
3300 	dev_dbg(ctx->dev, "%s %d: err 0x%x\n", __func__, __LINE__, status);
3301 
3302 	if (unlikely(status))
3303 		ecode = caam_qi2_strstatus(ctx->dev, status);
3304 
3305 	ahash_unmap_ctx(ctx->dev, edesc, req, DMA_FROM_DEVICE);
3306 	memcpy(req->result, state->caam_ctx, digestsize);
3307 	qi_cache_free(edesc);
3308 
3309 	print_hex_dump_debug("ctx@" __stringify(__LINE__)": ",
3310 			     DUMP_PREFIX_ADDRESS, 16, 4, state->caam_ctx,
3311 			     ctx->ctx_len, 1);
3312 
3313 	req->base.complete(&req->base, ecode);
3314 }
3315 
3316 static void ahash_done_bi(void *cbk_ctx, u32 status)
3317 {
3318 	struct crypto_async_request *areq = cbk_ctx;
3319 	struct ahash_request *req = ahash_request_cast(areq);
3320 	struct crypto_ahash *ahash = crypto_ahash_reqtfm(req);
3321 	struct caam_hash_state *state = ahash_request_ctx(req);
3322 	struct ahash_edesc *edesc = state->caam_req.edesc;
3323 	struct caam_hash_ctx *ctx = crypto_ahash_ctx(ahash);
3324 	int ecode = 0;
3325 
3326 	dev_dbg(ctx->dev, "%s %d: err 0x%x\n", __func__, __LINE__, status);
3327 
3328 	if (unlikely(status))
3329 		ecode = caam_qi2_strstatus(ctx->dev, status);
3330 
3331 	ahash_unmap_ctx(ctx->dev, edesc, req, DMA_BIDIRECTIONAL);
3332 	qi_cache_free(edesc);
3333 
3334 	scatterwalk_map_and_copy(state->buf, req->src,
3335 				 req->nbytes - state->next_buflen,
3336 				 state->next_buflen, 0);
3337 	state->buflen = state->next_buflen;
3338 
3339 	print_hex_dump_debug("buf@" __stringify(__LINE__)": ",
3340 			     DUMP_PREFIX_ADDRESS, 16, 4, state->buf,
3341 			     state->buflen, 1);
3342 
3343 	print_hex_dump_debug("ctx@" __stringify(__LINE__)": ",
3344 			     DUMP_PREFIX_ADDRESS, 16, 4, state->caam_ctx,
3345 			     ctx->ctx_len, 1);
3346 	if (req->result)
3347 		print_hex_dump_debug("result@" __stringify(__LINE__)": ",
3348 				     DUMP_PREFIX_ADDRESS, 16, 4, req->result,
3349 				     crypto_ahash_digestsize(ahash), 1);
3350 
3351 	req->base.complete(&req->base, ecode);
3352 }
3353 
3354 static void ahash_done_ctx_src(void *cbk_ctx, u32 status)
3355 {
3356 	struct crypto_async_request *areq = cbk_ctx;
3357 	struct ahash_request *req = ahash_request_cast(areq);
3358 	struct crypto_ahash *ahash = crypto_ahash_reqtfm(req);
3359 	struct caam_hash_state *state = ahash_request_ctx(req);
3360 	struct ahash_edesc *edesc = state->caam_req.edesc;
3361 	struct caam_hash_ctx *ctx = crypto_ahash_ctx(ahash);
3362 	int digestsize = crypto_ahash_digestsize(ahash);
3363 	int ecode = 0;
3364 
3365 	dev_dbg(ctx->dev, "%s %d: err 0x%x\n", __func__, __LINE__, status);
3366 
3367 	if (unlikely(status))
3368 		ecode = caam_qi2_strstatus(ctx->dev, status);
3369 
3370 	ahash_unmap_ctx(ctx->dev, edesc, req, DMA_BIDIRECTIONAL);
3371 	memcpy(req->result, state->caam_ctx, digestsize);
3372 	qi_cache_free(edesc);
3373 
3374 	print_hex_dump_debug("ctx@" __stringify(__LINE__)": ",
3375 			     DUMP_PREFIX_ADDRESS, 16, 4, state->caam_ctx,
3376 			     ctx->ctx_len, 1);
3377 
3378 	req->base.complete(&req->base, ecode);
3379 }
3380 
3381 static void ahash_done_ctx_dst(void *cbk_ctx, u32 status)
3382 {
3383 	struct crypto_async_request *areq = cbk_ctx;
3384 	struct ahash_request *req = ahash_request_cast(areq);
3385 	struct crypto_ahash *ahash = crypto_ahash_reqtfm(req);
3386 	struct caam_hash_state *state = ahash_request_ctx(req);
3387 	struct ahash_edesc *edesc = state->caam_req.edesc;
3388 	struct caam_hash_ctx *ctx = crypto_ahash_ctx(ahash);
3389 	int ecode = 0;
3390 
3391 	dev_dbg(ctx->dev, "%s %d: err 0x%x\n", __func__, __LINE__, status);
3392 
3393 	if (unlikely(status))
3394 		ecode = caam_qi2_strstatus(ctx->dev, status);
3395 
3396 	ahash_unmap_ctx(ctx->dev, edesc, req, DMA_FROM_DEVICE);
3397 	qi_cache_free(edesc);
3398 
3399 	scatterwalk_map_and_copy(state->buf, req->src,
3400 				 req->nbytes - state->next_buflen,
3401 				 state->next_buflen, 0);
3402 	state->buflen = state->next_buflen;
3403 
3404 	print_hex_dump_debug("buf@" __stringify(__LINE__)": ",
3405 			     DUMP_PREFIX_ADDRESS, 16, 4, state->buf,
3406 			     state->buflen, 1);
3407 
3408 	print_hex_dump_debug("ctx@" __stringify(__LINE__)": ",
3409 			     DUMP_PREFIX_ADDRESS, 16, 4, state->caam_ctx,
3410 			     ctx->ctx_len, 1);
3411 	if (req->result)
3412 		print_hex_dump_debug("result@" __stringify(__LINE__)": ",
3413 				     DUMP_PREFIX_ADDRESS, 16, 4, req->result,
3414 				     crypto_ahash_digestsize(ahash), 1);
3415 
3416 	req->base.complete(&req->base, ecode);
3417 }
3418 
3419 static int ahash_update_ctx(struct ahash_request *req)
3420 {
3421 	struct crypto_ahash *ahash = crypto_ahash_reqtfm(req);
3422 	struct caam_hash_ctx *ctx = crypto_ahash_ctx(ahash);
3423 	struct caam_hash_state *state = ahash_request_ctx(req);
3424 	struct caam_request *req_ctx = &state->caam_req;
3425 	struct dpaa2_fl_entry *in_fle = &req_ctx->fd_flt[1];
3426 	struct dpaa2_fl_entry *out_fle = &req_ctx->fd_flt[0];
3427 	gfp_t flags = (req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP) ?
3428 		      GFP_KERNEL : GFP_ATOMIC;
3429 	u8 *buf = state->buf;
3430 	int *buflen = &state->buflen;
3431 	int *next_buflen = &state->next_buflen;
3432 	int in_len = *buflen + req->nbytes, to_hash;
3433 	int src_nents, mapped_nents, qm_sg_bytes, qm_sg_src_index;
3434 	struct ahash_edesc *edesc;
3435 	int ret = 0;
3436 
3437 	*next_buflen = in_len & (crypto_tfm_alg_blocksize(&ahash->base) - 1);
3438 	to_hash = in_len - *next_buflen;
3439 
3440 	if (to_hash) {
3441 		struct dpaa2_sg_entry *sg_table;
3442 		int src_len = req->nbytes - *next_buflen;
3443 
3444 		src_nents = sg_nents_for_len(req->src, src_len);
3445 		if (src_nents < 0) {
3446 			dev_err(ctx->dev, "Invalid number of src SG.\n");
3447 			return src_nents;
3448 		}
3449 
3450 		if (src_nents) {
3451 			mapped_nents = dma_map_sg(ctx->dev, req->src, src_nents,
3452 						  DMA_TO_DEVICE);
3453 			if (!mapped_nents) {
3454 				dev_err(ctx->dev, "unable to DMA map source\n");
3455 				return -ENOMEM;
3456 			}
3457 		} else {
3458 			mapped_nents = 0;
3459 		}
3460 
3461 		/* allocate space for base edesc and link tables */
3462 		edesc = qi_cache_zalloc(GFP_DMA | flags);
3463 		if (!edesc) {
3464 			dma_unmap_sg(ctx->dev, req->src, src_nents,
3465 				     DMA_TO_DEVICE);
3466 			return -ENOMEM;
3467 		}
3468 
3469 		edesc->src_nents = src_nents;
3470 		qm_sg_src_index = 1 + (*buflen ? 1 : 0);
3471 		qm_sg_bytes = pad_sg_nents(qm_sg_src_index + mapped_nents) *
3472 			      sizeof(*sg_table);
3473 		sg_table = &edesc->sgt[0];
3474 
3475 		ret = ctx_map_to_qm_sg(ctx->dev, state, ctx->ctx_len, sg_table,
3476 				       DMA_BIDIRECTIONAL);
3477 		if (ret)
3478 			goto unmap_ctx;
3479 
3480 		ret = buf_map_to_qm_sg(ctx->dev, sg_table + 1, state);
3481 		if (ret)
3482 			goto unmap_ctx;
3483 
3484 		if (mapped_nents) {
3485 			sg_to_qm_sg_last(req->src, src_len,
3486 					 sg_table + qm_sg_src_index, 0);
3487 		} else {
3488 			dpaa2_sg_set_final(sg_table + qm_sg_src_index - 1,
3489 					   true);
3490 		}
3491 
3492 		edesc->qm_sg_dma = dma_map_single(ctx->dev, sg_table,
3493 						  qm_sg_bytes, DMA_TO_DEVICE);
3494 		if (dma_mapping_error(ctx->dev, edesc->qm_sg_dma)) {
3495 			dev_err(ctx->dev, "unable to map S/G table\n");
3496 			ret = -ENOMEM;
3497 			goto unmap_ctx;
3498 		}
3499 		edesc->qm_sg_bytes = qm_sg_bytes;
3500 
3501 		memset(&req_ctx->fd_flt, 0, sizeof(req_ctx->fd_flt));
3502 		dpaa2_fl_set_final(in_fle, true);
3503 		dpaa2_fl_set_format(in_fle, dpaa2_fl_sg);
3504 		dpaa2_fl_set_addr(in_fle, edesc->qm_sg_dma);
3505 		dpaa2_fl_set_len(in_fle, ctx->ctx_len + to_hash);
3506 		dpaa2_fl_set_format(out_fle, dpaa2_fl_single);
3507 		dpaa2_fl_set_addr(out_fle, state->ctx_dma);
3508 		dpaa2_fl_set_len(out_fle, ctx->ctx_len);
3509 
3510 		req_ctx->flc = &ctx->flc[UPDATE];
3511 		req_ctx->flc_dma = ctx->flc_dma[UPDATE];
3512 		req_ctx->cbk = ahash_done_bi;
3513 		req_ctx->ctx = &req->base;
3514 		req_ctx->edesc = edesc;
3515 
3516 		ret = dpaa2_caam_enqueue(ctx->dev, req_ctx);
3517 		if (ret != -EINPROGRESS &&
3518 		    !(ret == -EBUSY &&
3519 		      req->base.flags & CRYPTO_TFM_REQ_MAY_BACKLOG))
3520 			goto unmap_ctx;
3521 	} else if (*next_buflen) {
3522 		scatterwalk_map_and_copy(buf + *buflen, req->src, 0,
3523 					 req->nbytes, 0);
3524 		*buflen = *next_buflen;
3525 
3526 		print_hex_dump_debug("buf@" __stringify(__LINE__)": ",
3527 				     DUMP_PREFIX_ADDRESS, 16, 4, buf,
3528 				     *buflen, 1);
3529 	}
3530 
3531 	return ret;
3532 unmap_ctx:
3533 	ahash_unmap_ctx(ctx->dev, edesc, req, DMA_BIDIRECTIONAL);
3534 	qi_cache_free(edesc);
3535 	return ret;
3536 }
3537 
3538 static int ahash_final_ctx(struct ahash_request *req)
3539 {
3540 	struct crypto_ahash *ahash = crypto_ahash_reqtfm(req);
3541 	struct caam_hash_ctx *ctx = crypto_ahash_ctx(ahash);
3542 	struct caam_hash_state *state = ahash_request_ctx(req);
3543 	struct caam_request *req_ctx = &state->caam_req;
3544 	struct dpaa2_fl_entry *in_fle = &req_ctx->fd_flt[1];
3545 	struct dpaa2_fl_entry *out_fle = &req_ctx->fd_flt[0];
3546 	gfp_t flags = (req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP) ?
3547 		      GFP_KERNEL : GFP_ATOMIC;
3548 	int buflen = state->buflen;
3549 	int qm_sg_bytes;
3550 	int digestsize = crypto_ahash_digestsize(ahash);
3551 	struct ahash_edesc *edesc;
3552 	struct dpaa2_sg_entry *sg_table;
3553 	int ret;
3554 
3555 	/* allocate space for base edesc and link tables */
3556 	edesc = qi_cache_zalloc(GFP_DMA | flags);
3557 	if (!edesc)
3558 		return -ENOMEM;
3559 
3560 	qm_sg_bytes = pad_sg_nents(1 + (buflen ? 1 : 0)) * sizeof(*sg_table);
3561 	sg_table = &edesc->sgt[0];
3562 
3563 	ret = ctx_map_to_qm_sg(ctx->dev, state, ctx->ctx_len, sg_table,
3564 			       DMA_BIDIRECTIONAL);
3565 	if (ret)
3566 		goto unmap_ctx;
3567 
3568 	ret = buf_map_to_qm_sg(ctx->dev, sg_table + 1, state);
3569 	if (ret)
3570 		goto unmap_ctx;
3571 
3572 	dpaa2_sg_set_final(sg_table + (buflen ? 1 : 0), true);
3573 
3574 	edesc->qm_sg_dma = dma_map_single(ctx->dev, sg_table, qm_sg_bytes,
3575 					  DMA_TO_DEVICE);
3576 	if (dma_mapping_error(ctx->dev, edesc->qm_sg_dma)) {
3577 		dev_err(ctx->dev, "unable to map S/G table\n");
3578 		ret = -ENOMEM;
3579 		goto unmap_ctx;
3580 	}
3581 	edesc->qm_sg_bytes = qm_sg_bytes;
3582 
3583 	memset(&req_ctx->fd_flt, 0, sizeof(req_ctx->fd_flt));
3584 	dpaa2_fl_set_final(in_fle, true);
3585 	dpaa2_fl_set_format(in_fle, dpaa2_fl_sg);
3586 	dpaa2_fl_set_addr(in_fle, edesc->qm_sg_dma);
3587 	dpaa2_fl_set_len(in_fle, ctx->ctx_len + buflen);
3588 	dpaa2_fl_set_format(out_fle, dpaa2_fl_single);
3589 	dpaa2_fl_set_addr(out_fle, state->ctx_dma);
3590 	dpaa2_fl_set_len(out_fle, digestsize);
3591 
3592 	req_ctx->flc = &ctx->flc[FINALIZE];
3593 	req_ctx->flc_dma = ctx->flc_dma[FINALIZE];
3594 	req_ctx->cbk = ahash_done_ctx_src;
3595 	req_ctx->ctx = &req->base;
3596 	req_ctx->edesc = edesc;
3597 
3598 	ret = dpaa2_caam_enqueue(ctx->dev, req_ctx);
3599 	if (ret == -EINPROGRESS ||
3600 	    (ret == -EBUSY && req->base.flags & CRYPTO_TFM_REQ_MAY_BACKLOG))
3601 		return ret;
3602 
3603 unmap_ctx:
3604 	ahash_unmap_ctx(ctx->dev, edesc, req, DMA_BIDIRECTIONAL);
3605 	qi_cache_free(edesc);
3606 	return ret;
3607 }
3608 
3609 static int ahash_finup_ctx(struct ahash_request *req)
3610 {
3611 	struct crypto_ahash *ahash = crypto_ahash_reqtfm(req);
3612 	struct caam_hash_ctx *ctx = crypto_ahash_ctx(ahash);
3613 	struct caam_hash_state *state = ahash_request_ctx(req);
3614 	struct caam_request *req_ctx = &state->caam_req;
3615 	struct dpaa2_fl_entry *in_fle = &req_ctx->fd_flt[1];
3616 	struct dpaa2_fl_entry *out_fle = &req_ctx->fd_flt[0];
3617 	gfp_t flags = (req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP) ?
3618 		      GFP_KERNEL : GFP_ATOMIC;
3619 	int buflen = state->buflen;
3620 	int qm_sg_bytes, qm_sg_src_index;
3621 	int src_nents, mapped_nents;
3622 	int digestsize = crypto_ahash_digestsize(ahash);
3623 	struct ahash_edesc *edesc;
3624 	struct dpaa2_sg_entry *sg_table;
3625 	int ret;
3626 
3627 	src_nents = sg_nents_for_len(req->src, req->nbytes);
3628 	if (src_nents < 0) {
3629 		dev_err(ctx->dev, "Invalid number of src SG.\n");
3630 		return src_nents;
3631 	}
3632 
3633 	if (src_nents) {
3634 		mapped_nents = dma_map_sg(ctx->dev, req->src, src_nents,
3635 					  DMA_TO_DEVICE);
3636 		if (!mapped_nents) {
3637 			dev_err(ctx->dev, "unable to DMA map source\n");
3638 			return -ENOMEM;
3639 		}
3640 	} else {
3641 		mapped_nents = 0;
3642 	}
3643 
3644 	/* allocate space for base edesc and link tables */
3645 	edesc = qi_cache_zalloc(GFP_DMA | flags);
3646 	if (!edesc) {
3647 		dma_unmap_sg(ctx->dev, req->src, src_nents, DMA_TO_DEVICE);
3648 		return -ENOMEM;
3649 	}
3650 
3651 	edesc->src_nents = src_nents;
3652 	qm_sg_src_index = 1 + (buflen ? 1 : 0);
3653 	qm_sg_bytes = pad_sg_nents(qm_sg_src_index + mapped_nents) *
3654 		      sizeof(*sg_table);
3655 	sg_table = &edesc->sgt[0];
3656 
3657 	ret = ctx_map_to_qm_sg(ctx->dev, state, ctx->ctx_len, sg_table,
3658 			       DMA_BIDIRECTIONAL);
3659 	if (ret)
3660 		goto unmap_ctx;
3661 
3662 	ret = buf_map_to_qm_sg(ctx->dev, sg_table + 1, state);
3663 	if (ret)
3664 		goto unmap_ctx;
3665 
3666 	sg_to_qm_sg_last(req->src, req->nbytes, sg_table + qm_sg_src_index, 0);
3667 
3668 	edesc->qm_sg_dma = dma_map_single(ctx->dev, sg_table, qm_sg_bytes,
3669 					  DMA_TO_DEVICE);
3670 	if (dma_mapping_error(ctx->dev, edesc->qm_sg_dma)) {
3671 		dev_err(ctx->dev, "unable to map S/G table\n");
3672 		ret = -ENOMEM;
3673 		goto unmap_ctx;
3674 	}
3675 	edesc->qm_sg_bytes = qm_sg_bytes;
3676 
3677 	memset(&req_ctx->fd_flt, 0, sizeof(req_ctx->fd_flt));
3678 	dpaa2_fl_set_final(in_fle, true);
3679 	dpaa2_fl_set_format(in_fle, dpaa2_fl_sg);
3680 	dpaa2_fl_set_addr(in_fle, edesc->qm_sg_dma);
3681 	dpaa2_fl_set_len(in_fle, ctx->ctx_len + buflen + req->nbytes);
3682 	dpaa2_fl_set_format(out_fle, dpaa2_fl_single);
3683 	dpaa2_fl_set_addr(out_fle, state->ctx_dma);
3684 	dpaa2_fl_set_len(out_fle, digestsize);
3685 
3686 	req_ctx->flc = &ctx->flc[FINALIZE];
3687 	req_ctx->flc_dma = ctx->flc_dma[FINALIZE];
3688 	req_ctx->cbk = ahash_done_ctx_src;
3689 	req_ctx->ctx = &req->base;
3690 	req_ctx->edesc = edesc;
3691 
3692 	ret = dpaa2_caam_enqueue(ctx->dev, req_ctx);
3693 	if (ret == -EINPROGRESS ||
3694 	    (ret == -EBUSY && req->base.flags & CRYPTO_TFM_REQ_MAY_BACKLOG))
3695 		return ret;
3696 
3697 unmap_ctx:
3698 	ahash_unmap_ctx(ctx->dev, edesc, req, DMA_BIDIRECTIONAL);
3699 	qi_cache_free(edesc);
3700 	return ret;
3701 }
3702 
3703 static int ahash_digest(struct ahash_request *req)
3704 {
3705 	struct crypto_ahash *ahash = crypto_ahash_reqtfm(req);
3706 	struct caam_hash_ctx *ctx = crypto_ahash_ctx(ahash);
3707 	struct caam_hash_state *state = ahash_request_ctx(req);
3708 	struct caam_request *req_ctx = &state->caam_req;
3709 	struct dpaa2_fl_entry *in_fle = &req_ctx->fd_flt[1];
3710 	struct dpaa2_fl_entry *out_fle = &req_ctx->fd_flt[0];
3711 	gfp_t flags = (req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP) ?
3712 		      GFP_KERNEL : GFP_ATOMIC;
3713 	int digestsize = crypto_ahash_digestsize(ahash);
3714 	int src_nents, mapped_nents;
3715 	struct ahash_edesc *edesc;
3716 	int ret = -ENOMEM;
3717 
3718 	state->buf_dma = 0;
3719 
3720 	src_nents = sg_nents_for_len(req->src, req->nbytes);
3721 	if (src_nents < 0) {
3722 		dev_err(ctx->dev, "Invalid number of src SG.\n");
3723 		return src_nents;
3724 	}
3725 
3726 	if (src_nents) {
3727 		mapped_nents = dma_map_sg(ctx->dev, req->src, src_nents,
3728 					  DMA_TO_DEVICE);
3729 		if (!mapped_nents) {
3730 			dev_err(ctx->dev, "unable to map source for DMA\n");
3731 			return ret;
3732 		}
3733 	} else {
3734 		mapped_nents = 0;
3735 	}
3736 
3737 	/* allocate space for base edesc and link tables */
3738 	edesc = qi_cache_zalloc(GFP_DMA | flags);
3739 	if (!edesc) {
3740 		dma_unmap_sg(ctx->dev, req->src, src_nents, DMA_TO_DEVICE);
3741 		return ret;
3742 	}
3743 
3744 	edesc->src_nents = src_nents;
3745 	memset(&req_ctx->fd_flt, 0, sizeof(req_ctx->fd_flt));
3746 
3747 	if (mapped_nents > 1) {
3748 		int qm_sg_bytes;
3749 		struct dpaa2_sg_entry *sg_table = &edesc->sgt[0];
3750 
3751 		qm_sg_bytes = pad_sg_nents(mapped_nents) * sizeof(*sg_table);
3752 		sg_to_qm_sg_last(req->src, req->nbytes, sg_table, 0);
3753 		edesc->qm_sg_dma = dma_map_single(ctx->dev, sg_table,
3754 						  qm_sg_bytes, DMA_TO_DEVICE);
3755 		if (dma_mapping_error(ctx->dev, edesc->qm_sg_dma)) {
3756 			dev_err(ctx->dev, "unable to map S/G table\n");
3757 			goto unmap;
3758 		}
3759 		edesc->qm_sg_bytes = qm_sg_bytes;
3760 		dpaa2_fl_set_format(in_fle, dpaa2_fl_sg);
3761 		dpaa2_fl_set_addr(in_fle, edesc->qm_sg_dma);
3762 	} else {
3763 		dpaa2_fl_set_format(in_fle, dpaa2_fl_single);
3764 		dpaa2_fl_set_addr(in_fle, sg_dma_address(req->src));
3765 	}
3766 
3767 	state->ctx_dma_len = digestsize;
3768 	state->ctx_dma = dma_map_single(ctx->dev, state->caam_ctx, digestsize,
3769 					DMA_FROM_DEVICE);
3770 	if (dma_mapping_error(ctx->dev, state->ctx_dma)) {
3771 		dev_err(ctx->dev, "unable to map ctx\n");
3772 		state->ctx_dma = 0;
3773 		goto unmap;
3774 	}
3775 
3776 	dpaa2_fl_set_final(in_fle, true);
3777 	dpaa2_fl_set_len(in_fle, req->nbytes);
3778 	dpaa2_fl_set_format(out_fle, dpaa2_fl_single);
3779 	dpaa2_fl_set_addr(out_fle, state->ctx_dma);
3780 	dpaa2_fl_set_len(out_fle, digestsize);
3781 
3782 	req_ctx->flc = &ctx->flc[DIGEST];
3783 	req_ctx->flc_dma = ctx->flc_dma[DIGEST];
3784 	req_ctx->cbk = ahash_done;
3785 	req_ctx->ctx = &req->base;
3786 	req_ctx->edesc = edesc;
3787 	ret = dpaa2_caam_enqueue(ctx->dev, req_ctx);
3788 	if (ret == -EINPROGRESS ||
3789 	    (ret == -EBUSY && req->base.flags & CRYPTO_TFM_REQ_MAY_BACKLOG))
3790 		return ret;
3791 
3792 unmap:
3793 	ahash_unmap_ctx(ctx->dev, edesc, req, DMA_FROM_DEVICE);
3794 	qi_cache_free(edesc);
3795 	return ret;
3796 }
3797 
3798 static int ahash_final_no_ctx(struct ahash_request *req)
3799 {
3800 	struct crypto_ahash *ahash = crypto_ahash_reqtfm(req);
3801 	struct caam_hash_ctx *ctx = crypto_ahash_ctx(ahash);
3802 	struct caam_hash_state *state = ahash_request_ctx(req);
3803 	struct caam_request *req_ctx = &state->caam_req;
3804 	struct dpaa2_fl_entry *in_fle = &req_ctx->fd_flt[1];
3805 	struct dpaa2_fl_entry *out_fle = &req_ctx->fd_flt[0];
3806 	gfp_t flags = (req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP) ?
3807 		      GFP_KERNEL : GFP_ATOMIC;
3808 	u8 *buf = state->buf;
3809 	int buflen = state->buflen;
3810 	int digestsize = crypto_ahash_digestsize(ahash);
3811 	struct ahash_edesc *edesc;
3812 	int ret = -ENOMEM;
3813 
3814 	/* allocate space for base edesc and link tables */
3815 	edesc = qi_cache_zalloc(GFP_DMA | flags);
3816 	if (!edesc)
3817 		return ret;
3818 
3819 	if (buflen) {
3820 		state->buf_dma = dma_map_single(ctx->dev, buf, buflen,
3821 						DMA_TO_DEVICE);
3822 		if (dma_mapping_error(ctx->dev, state->buf_dma)) {
3823 			dev_err(ctx->dev, "unable to map src\n");
3824 			goto unmap;
3825 		}
3826 	}
3827 
3828 	state->ctx_dma_len = digestsize;
3829 	state->ctx_dma = dma_map_single(ctx->dev, state->caam_ctx, digestsize,
3830 					DMA_FROM_DEVICE);
3831 	if (dma_mapping_error(ctx->dev, state->ctx_dma)) {
3832 		dev_err(ctx->dev, "unable to map ctx\n");
3833 		state->ctx_dma = 0;
3834 		goto unmap;
3835 	}
3836 
3837 	memset(&req_ctx->fd_flt, 0, sizeof(req_ctx->fd_flt));
3838 	dpaa2_fl_set_final(in_fle, true);
3839 	/*
3840 	 * crypto engine requires the input entry to be present when
3841 	 * "frame list" FD is used.
3842 	 * Since engine does not support FMT=2'b11 (unused entry type), leaving
3843 	 * in_fle zeroized (except for "Final" flag) is the best option.
3844 	 */
3845 	if (buflen) {
3846 		dpaa2_fl_set_format(in_fle, dpaa2_fl_single);
3847 		dpaa2_fl_set_addr(in_fle, state->buf_dma);
3848 		dpaa2_fl_set_len(in_fle, buflen);
3849 	}
3850 	dpaa2_fl_set_format(out_fle, dpaa2_fl_single);
3851 	dpaa2_fl_set_addr(out_fle, state->ctx_dma);
3852 	dpaa2_fl_set_len(out_fle, digestsize);
3853 
3854 	req_ctx->flc = &ctx->flc[DIGEST];
3855 	req_ctx->flc_dma = ctx->flc_dma[DIGEST];
3856 	req_ctx->cbk = ahash_done;
3857 	req_ctx->ctx = &req->base;
3858 	req_ctx->edesc = edesc;
3859 
3860 	ret = dpaa2_caam_enqueue(ctx->dev, req_ctx);
3861 	if (ret == -EINPROGRESS ||
3862 	    (ret == -EBUSY && req->base.flags & CRYPTO_TFM_REQ_MAY_BACKLOG))
3863 		return ret;
3864 
3865 unmap:
3866 	ahash_unmap_ctx(ctx->dev, edesc, req, DMA_FROM_DEVICE);
3867 	qi_cache_free(edesc);
3868 	return ret;
3869 }
3870 
3871 static int ahash_update_no_ctx(struct ahash_request *req)
3872 {
3873 	struct crypto_ahash *ahash = crypto_ahash_reqtfm(req);
3874 	struct caam_hash_ctx *ctx = crypto_ahash_ctx(ahash);
3875 	struct caam_hash_state *state = ahash_request_ctx(req);
3876 	struct caam_request *req_ctx = &state->caam_req;
3877 	struct dpaa2_fl_entry *in_fle = &req_ctx->fd_flt[1];
3878 	struct dpaa2_fl_entry *out_fle = &req_ctx->fd_flt[0];
3879 	gfp_t flags = (req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP) ?
3880 		      GFP_KERNEL : GFP_ATOMIC;
3881 	u8 *buf = state->buf;
3882 	int *buflen = &state->buflen;
3883 	int *next_buflen = &state->next_buflen;
3884 	int in_len = *buflen + req->nbytes, to_hash;
3885 	int qm_sg_bytes, src_nents, mapped_nents;
3886 	struct ahash_edesc *edesc;
3887 	int ret = 0;
3888 
3889 	*next_buflen = in_len & (crypto_tfm_alg_blocksize(&ahash->base) - 1);
3890 	to_hash = in_len - *next_buflen;
3891 
3892 	if (to_hash) {
3893 		struct dpaa2_sg_entry *sg_table;
3894 		int src_len = req->nbytes - *next_buflen;
3895 
3896 		src_nents = sg_nents_for_len(req->src, src_len);
3897 		if (src_nents < 0) {
3898 			dev_err(ctx->dev, "Invalid number of src SG.\n");
3899 			return src_nents;
3900 		}
3901 
3902 		if (src_nents) {
3903 			mapped_nents = dma_map_sg(ctx->dev, req->src, src_nents,
3904 						  DMA_TO_DEVICE);
3905 			if (!mapped_nents) {
3906 				dev_err(ctx->dev, "unable to DMA map source\n");
3907 				return -ENOMEM;
3908 			}
3909 		} else {
3910 			mapped_nents = 0;
3911 		}
3912 
3913 		/* allocate space for base edesc and link tables */
3914 		edesc = qi_cache_zalloc(GFP_DMA | flags);
3915 		if (!edesc) {
3916 			dma_unmap_sg(ctx->dev, req->src, src_nents,
3917 				     DMA_TO_DEVICE);
3918 			return -ENOMEM;
3919 		}
3920 
3921 		edesc->src_nents = src_nents;
3922 		qm_sg_bytes = pad_sg_nents(1 + mapped_nents) *
3923 			      sizeof(*sg_table);
3924 		sg_table = &edesc->sgt[0];
3925 
3926 		ret = buf_map_to_qm_sg(ctx->dev, sg_table, state);
3927 		if (ret)
3928 			goto unmap_ctx;
3929 
3930 		sg_to_qm_sg_last(req->src, src_len, sg_table + 1, 0);
3931 
3932 		edesc->qm_sg_dma = dma_map_single(ctx->dev, sg_table,
3933 						  qm_sg_bytes, DMA_TO_DEVICE);
3934 		if (dma_mapping_error(ctx->dev, edesc->qm_sg_dma)) {
3935 			dev_err(ctx->dev, "unable to map S/G table\n");
3936 			ret = -ENOMEM;
3937 			goto unmap_ctx;
3938 		}
3939 		edesc->qm_sg_bytes = qm_sg_bytes;
3940 
3941 		state->ctx_dma_len = ctx->ctx_len;
3942 		state->ctx_dma = dma_map_single(ctx->dev, state->caam_ctx,
3943 						ctx->ctx_len, DMA_FROM_DEVICE);
3944 		if (dma_mapping_error(ctx->dev, state->ctx_dma)) {
3945 			dev_err(ctx->dev, "unable to map ctx\n");
3946 			state->ctx_dma = 0;
3947 			ret = -ENOMEM;
3948 			goto unmap_ctx;
3949 		}
3950 
3951 		memset(&req_ctx->fd_flt, 0, sizeof(req_ctx->fd_flt));
3952 		dpaa2_fl_set_final(in_fle, true);
3953 		dpaa2_fl_set_format(in_fle, dpaa2_fl_sg);
3954 		dpaa2_fl_set_addr(in_fle, edesc->qm_sg_dma);
3955 		dpaa2_fl_set_len(in_fle, to_hash);
3956 		dpaa2_fl_set_format(out_fle, dpaa2_fl_single);
3957 		dpaa2_fl_set_addr(out_fle, state->ctx_dma);
3958 		dpaa2_fl_set_len(out_fle, ctx->ctx_len);
3959 
3960 		req_ctx->flc = &ctx->flc[UPDATE_FIRST];
3961 		req_ctx->flc_dma = ctx->flc_dma[UPDATE_FIRST];
3962 		req_ctx->cbk = ahash_done_ctx_dst;
3963 		req_ctx->ctx = &req->base;
3964 		req_ctx->edesc = edesc;
3965 
3966 		ret = dpaa2_caam_enqueue(ctx->dev, req_ctx);
3967 		if (ret != -EINPROGRESS &&
3968 		    !(ret == -EBUSY &&
3969 		      req->base.flags & CRYPTO_TFM_REQ_MAY_BACKLOG))
3970 			goto unmap_ctx;
3971 
3972 		state->update = ahash_update_ctx;
3973 		state->finup = ahash_finup_ctx;
3974 		state->final = ahash_final_ctx;
3975 	} else if (*next_buflen) {
3976 		scatterwalk_map_and_copy(buf + *buflen, req->src, 0,
3977 					 req->nbytes, 0);
3978 		*buflen = *next_buflen;
3979 
3980 		print_hex_dump_debug("buf@" __stringify(__LINE__)": ",
3981 				     DUMP_PREFIX_ADDRESS, 16, 4, buf,
3982 				     *buflen, 1);
3983 	}
3984 
3985 	return ret;
3986 unmap_ctx:
3987 	ahash_unmap_ctx(ctx->dev, edesc, req, DMA_TO_DEVICE);
3988 	qi_cache_free(edesc);
3989 	return ret;
3990 }
3991 
3992 static int ahash_finup_no_ctx(struct ahash_request *req)
3993 {
3994 	struct crypto_ahash *ahash = crypto_ahash_reqtfm(req);
3995 	struct caam_hash_ctx *ctx = crypto_ahash_ctx(ahash);
3996 	struct caam_hash_state *state = ahash_request_ctx(req);
3997 	struct caam_request *req_ctx = &state->caam_req;
3998 	struct dpaa2_fl_entry *in_fle = &req_ctx->fd_flt[1];
3999 	struct dpaa2_fl_entry *out_fle = &req_ctx->fd_flt[0];
4000 	gfp_t flags = (req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP) ?
4001 		      GFP_KERNEL : GFP_ATOMIC;
4002 	int buflen = state->buflen;
4003 	int qm_sg_bytes, src_nents, mapped_nents;
4004 	int digestsize = crypto_ahash_digestsize(ahash);
4005 	struct ahash_edesc *edesc;
4006 	struct dpaa2_sg_entry *sg_table;
4007 	int ret;
4008 
4009 	src_nents = sg_nents_for_len(req->src, req->nbytes);
4010 	if (src_nents < 0) {
4011 		dev_err(ctx->dev, "Invalid number of src SG.\n");
4012 		return src_nents;
4013 	}
4014 
4015 	if (src_nents) {
4016 		mapped_nents = dma_map_sg(ctx->dev, req->src, src_nents,
4017 					  DMA_TO_DEVICE);
4018 		if (!mapped_nents) {
4019 			dev_err(ctx->dev, "unable to DMA map source\n");
4020 			return -ENOMEM;
4021 		}
4022 	} else {
4023 		mapped_nents = 0;
4024 	}
4025 
4026 	/* allocate space for base edesc and link tables */
4027 	edesc = qi_cache_zalloc(GFP_DMA | flags);
4028 	if (!edesc) {
4029 		dma_unmap_sg(ctx->dev, req->src, src_nents, DMA_TO_DEVICE);
4030 		return -ENOMEM;
4031 	}
4032 
4033 	edesc->src_nents = src_nents;
4034 	qm_sg_bytes = pad_sg_nents(2 + mapped_nents) * sizeof(*sg_table);
4035 	sg_table = &edesc->sgt[0];
4036 
4037 	ret = buf_map_to_qm_sg(ctx->dev, sg_table, state);
4038 	if (ret)
4039 		goto unmap;
4040 
4041 	sg_to_qm_sg_last(req->src, req->nbytes, sg_table + 1, 0);
4042 
4043 	edesc->qm_sg_dma = dma_map_single(ctx->dev, sg_table, qm_sg_bytes,
4044 					  DMA_TO_DEVICE);
4045 	if (dma_mapping_error(ctx->dev, edesc->qm_sg_dma)) {
4046 		dev_err(ctx->dev, "unable to map S/G table\n");
4047 		ret = -ENOMEM;
4048 		goto unmap;
4049 	}
4050 	edesc->qm_sg_bytes = qm_sg_bytes;
4051 
4052 	state->ctx_dma_len = digestsize;
4053 	state->ctx_dma = dma_map_single(ctx->dev, state->caam_ctx, digestsize,
4054 					DMA_FROM_DEVICE);
4055 	if (dma_mapping_error(ctx->dev, state->ctx_dma)) {
4056 		dev_err(ctx->dev, "unable to map ctx\n");
4057 		state->ctx_dma = 0;
4058 		ret = -ENOMEM;
4059 		goto unmap;
4060 	}
4061 
4062 	memset(&req_ctx->fd_flt, 0, sizeof(req_ctx->fd_flt));
4063 	dpaa2_fl_set_final(in_fle, true);
4064 	dpaa2_fl_set_format(in_fle, dpaa2_fl_sg);
4065 	dpaa2_fl_set_addr(in_fle, edesc->qm_sg_dma);
4066 	dpaa2_fl_set_len(in_fle, buflen + req->nbytes);
4067 	dpaa2_fl_set_format(out_fle, dpaa2_fl_single);
4068 	dpaa2_fl_set_addr(out_fle, state->ctx_dma);
4069 	dpaa2_fl_set_len(out_fle, digestsize);
4070 
4071 	req_ctx->flc = &ctx->flc[DIGEST];
4072 	req_ctx->flc_dma = ctx->flc_dma[DIGEST];
4073 	req_ctx->cbk = ahash_done;
4074 	req_ctx->ctx = &req->base;
4075 	req_ctx->edesc = edesc;
4076 	ret = dpaa2_caam_enqueue(ctx->dev, req_ctx);
4077 	if (ret != -EINPROGRESS &&
4078 	    !(ret == -EBUSY && req->base.flags & CRYPTO_TFM_REQ_MAY_BACKLOG))
4079 		goto unmap;
4080 
4081 	return ret;
4082 unmap:
4083 	ahash_unmap_ctx(ctx->dev, edesc, req, DMA_FROM_DEVICE);
4084 	qi_cache_free(edesc);
4085 	return -ENOMEM;
4086 }
4087 
4088 static int ahash_update_first(struct ahash_request *req)
4089 {
4090 	struct crypto_ahash *ahash = crypto_ahash_reqtfm(req);
4091 	struct caam_hash_ctx *ctx = crypto_ahash_ctx(ahash);
4092 	struct caam_hash_state *state = ahash_request_ctx(req);
4093 	struct caam_request *req_ctx = &state->caam_req;
4094 	struct dpaa2_fl_entry *in_fle = &req_ctx->fd_flt[1];
4095 	struct dpaa2_fl_entry *out_fle = &req_ctx->fd_flt[0];
4096 	gfp_t flags = (req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP) ?
4097 		      GFP_KERNEL : GFP_ATOMIC;
4098 	u8 *buf = state->buf;
4099 	int *buflen = &state->buflen;
4100 	int *next_buflen = &state->next_buflen;
4101 	int to_hash;
4102 	int src_nents, mapped_nents;
4103 	struct ahash_edesc *edesc;
4104 	int ret = 0;
4105 
4106 	*next_buflen = req->nbytes & (crypto_tfm_alg_blocksize(&ahash->base) -
4107 				      1);
4108 	to_hash = req->nbytes - *next_buflen;
4109 
4110 	if (to_hash) {
4111 		struct dpaa2_sg_entry *sg_table;
4112 		int src_len = req->nbytes - *next_buflen;
4113 
4114 		src_nents = sg_nents_for_len(req->src, src_len);
4115 		if (src_nents < 0) {
4116 			dev_err(ctx->dev, "Invalid number of src SG.\n");
4117 			return src_nents;
4118 		}
4119 
4120 		if (src_nents) {
4121 			mapped_nents = dma_map_sg(ctx->dev, req->src, src_nents,
4122 						  DMA_TO_DEVICE);
4123 			if (!mapped_nents) {
4124 				dev_err(ctx->dev, "unable to map source for DMA\n");
4125 				return -ENOMEM;
4126 			}
4127 		} else {
4128 			mapped_nents = 0;
4129 		}
4130 
4131 		/* allocate space for base edesc and link tables */
4132 		edesc = qi_cache_zalloc(GFP_DMA | flags);
4133 		if (!edesc) {
4134 			dma_unmap_sg(ctx->dev, req->src, src_nents,
4135 				     DMA_TO_DEVICE);
4136 			return -ENOMEM;
4137 		}
4138 
4139 		edesc->src_nents = src_nents;
4140 		sg_table = &edesc->sgt[0];
4141 
4142 		memset(&req_ctx->fd_flt, 0, sizeof(req_ctx->fd_flt));
4143 		dpaa2_fl_set_final(in_fle, true);
4144 		dpaa2_fl_set_len(in_fle, to_hash);
4145 
4146 		if (mapped_nents > 1) {
4147 			int qm_sg_bytes;
4148 
4149 			sg_to_qm_sg_last(req->src, src_len, sg_table, 0);
4150 			qm_sg_bytes = pad_sg_nents(mapped_nents) *
4151 				      sizeof(*sg_table);
4152 			edesc->qm_sg_dma = dma_map_single(ctx->dev, sg_table,
4153 							  qm_sg_bytes,
4154 							  DMA_TO_DEVICE);
4155 			if (dma_mapping_error(ctx->dev, edesc->qm_sg_dma)) {
4156 				dev_err(ctx->dev, "unable to map S/G table\n");
4157 				ret = -ENOMEM;
4158 				goto unmap_ctx;
4159 			}
4160 			edesc->qm_sg_bytes = qm_sg_bytes;
4161 			dpaa2_fl_set_format(in_fle, dpaa2_fl_sg);
4162 			dpaa2_fl_set_addr(in_fle, edesc->qm_sg_dma);
4163 		} else {
4164 			dpaa2_fl_set_format(in_fle, dpaa2_fl_single);
4165 			dpaa2_fl_set_addr(in_fle, sg_dma_address(req->src));
4166 		}
4167 
4168 		state->ctx_dma_len = ctx->ctx_len;
4169 		state->ctx_dma = dma_map_single(ctx->dev, state->caam_ctx,
4170 						ctx->ctx_len, DMA_FROM_DEVICE);
4171 		if (dma_mapping_error(ctx->dev, state->ctx_dma)) {
4172 			dev_err(ctx->dev, "unable to map ctx\n");
4173 			state->ctx_dma = 0;
4174 			ret = -ENOMEM;
4175 			goto unmap_ctx;
4176 		}
4177 
4178 		dpaa2_fl_set_format(out_fle, dpaa2_fl_single);
4179 		dpaa2_fl_set_addr(out_fle, state->ctx_dma);
4180 		dpaa2_fl_set_len(out_fle, ctx->ctx_len);
4181 
4182 		req_ctx->flc = &ctx->flc[UPDATE_FIRST];
4183 		req_ctx->flc_dma = ctx->flc_dma[UPDATE_FIRST];
4184 		req_ctx->cbk = ahash_done_ctx_dst;
4185 		req_ctx->ctx = &req->base;
4186 		req_ctx->edesc = edesc;
4187 
4188 		ret = dpaa2_caam_enqueue(ctx->dev, req_ctx);
4189 		if (ret != -EINPROGRESS &&
4190 		    !(ret == -EBUSY && req->base.flags &
4191 		      CRYPTO_TFM_REQ_MAY_BACKLOG))
4192 			goto unmap_ctx;
4193 
4194 		state->update = ahash_update_ctx;
4195 		state->finup = ahash_finup_ctx;
4196 		state->final = ahash_final_ctx;
4197 	} else if (*next_buflen) {
4198 		state->update = ahash_update_no_ctx;
4199 		state->finup = ahash_finup_no_ctx;
4200 		state->final = ahash_final_no_ctx;
4201 		scatterwalk_map_and_copy(buf, req->src, 0,
4202 					 req->nbytes, 0);
4203 		*buflen = *next_buflen;
4204 
4205 		print_hex_dump_debug("buf@" __stringify(__LINE__)": ",
4206 				     DUMP_PREFIX_ADDRESS, 16, 4, buf,
4207 				     *buflen, 1);
4208 	}
4209 
4210 	return ret;
4211 unmap_ctx:
4212 	ahash_unmap_ctx(ctx->dev, edesc, req, DMA_TO_DEVICE);
4213 	qi_cache_free(edesc);
4214 	return ret;
4215 }
4216 
4217 static int ahash_finup_first(struct ahash_request *req)
4218 {
4219 	return ahash_digest(req);
4220 }
4221 
4222 static int ahash_init(struct ahash_request *req)
4223 {
4224 	struct caam_hash_state *state = ahash_request_ctx(req);
4225 
4226 	state->update = ahash_update_first;
4227 	state->finup = ahash_finup_first;
4228 	state->final = ahash_final_no_ctx;
4229 
4230 	state->ctx_dma = 0;
4231 	state->ctx_dma_len = 0;
4232 	state->buf_dma = 0;
4233 	state->buflen = 0;
4234 	state->next_buflen = 0;
4235 
4236 	return 0;
4237 }
4238 
4239 static int ahash_update(struct ahash_request *req)
4240 {
4241 	struct caam_hash_state *state = ahash_request_ctx(req);
4242 
4243 	return state->update(req);
4244 }
4245 
4246 static int ahash_finup(struct ahash_request *req)
4247 {
4248 	struct caam_hash_state *state = ahash_request_ctx(req);
4249 
4250 	return state->finup(req);
4251 }
4252 
4253 static int ahash_final(struct ahash_request *req)
4254 {
4255 	struct caam_hash_state *state = ahash_request_ctx(req);
4256 
4257 	return state->final(req);
4258 }
4259 
4260 static int ahash_export(struct ahash_request *req, void *out)
4261 {
4262 	struct caam_hash_state *state = ahash_request_ctx(req);
4263 	struct caam_export_state *export = out;
4264 	u8 *buf = state->buf;
4265 	int len = state->buflen;
4266 
4267 	memcpy(export->buf, buf, len);
4268 	memcpy(export->caam_ctx, state->caam_ctx, sizeof(export->caam_ctx));
4269 	export->buflen = len;
4270 	export->update = state->update;
4271 	export->final = state->final;
4272 	export->finup = state->finup;
4273 
4274 	return 0;
4275 }
4276 
4277 static int ahash_import(struct ahash_request *req, const void *in)
4278 {
4279 	struct caam_hash_state *state = ahash_request_ctx(req);
4280 	const struct caam_export_state *export = in;
4281 
4282 	memset(state, 0, sizeof(*state));
4283 	memcpy(state->buf, export->buf, export->buflen);
4284 	memcpy(state->caam_ctx, export->caam_ctx, sizeof(state->caam_ctx));
4285 	state->buflen = export->buflen;
4286 	state->update = export->update;
4287 	state->final = export->final;
4288 	state->finup = export->finup;
4289 
4290 	return 0;
4291 }
4292 
4293 struct caam_hash_template {
4294 	char name[CRYPTO_MAX_ALG_NAME];
4295 	char driver_name[CRYPTO_MAX_ALG_NAME];
4296 	char hmac_name[CRYPTO_MAX_ALG_NAME];
4297 	char hmac_driver_name[CRYPTO_MAX_ALG_NAME];
4298 	unsigned int blocksize;
4299 	struct ahash_alg template_ahash;
4300 	u32 alg_type;
4301 };
4302 
4303 /* ahash descriptors */
4304 static struct caam_hash_template driver_hash[] = {
4305 	{
4306 		.name = "sha1",
4307 		.driver_name = "sha1-caam-qi2",
4308 		.hmac_name = "hmac(sha1)",
4309 		.hmac_driver_name = "hmac-sha1-caam-qi2",
4310 		.blocksize = SHA1_BLOCK_SIZE,
4311 		.template_ahash = {
4312 			.init = ahash_init,
4313 			.update = ahash_update,
4314 			.final = ahash_final,
4315 			.finup = ahash_finup,
4316 			.digest = ahash_digest,
4317 			.export = ahash_export,
4318 			.import = ahash_import,
4319 			.setkey = ahash_setkey,
4320 			.halg = {
4321 				.digestsize = SHA1_DIGEST_SIZE,
4322 				.statesize = sizeof(struct caam_export_state),
4323 			},
4324 		},
4325 		.alg_type = OP_ALG_ALGSEL_SHA1,
4326 	}, {
4327 		.name = "sha224",
4328 		.driver_name = "sha224-caam-qi2",
4329 		.hmac_name = "hmac(sha224)",
4330 		.hmac_driver_name = "hmac-sha224-caam-qi2",
4331 		.blocksize = SHA224_BLOCK_SIZE,
4332 		.template_ahash = {
4333 			.init = ahash_init,
4334 			.update = ahash_update,
4335 			.final = ahash_final,
4336 			.finup = ahash_finup,
4337 			.digest = ahash_digest,
4338 			.export = ahash_export,
4339 			.import = ahash_import,
4340 			.setkey = ahash_setkey,
4341 			.halg = {
4342 				.digestsize = SHA224_DIGEST_SIZE,
4343 				.statesize = sizeof(struct caam_export_state),
4344 			},
4345 		},
4346 		.alg_type = OP_ALG_ALGSEL_SHA224,
4347 	}, {
4348 		.name = "sha256",
4349 		.driver_name = "sha256-caam-qi2",
4350 		.hmac_name = "hmac(sha256)",
4351 		.hmac_driver_name = "hmac-sha256-caam-qi2",
4352 		.blocksize = SHA256_BLOCK_SIZE,
4353 		.template_ahash = {
4354 			.init = ahash_init,
4355 			.update = ahash_update,
4356 			.final = ahash_final,
4357 			.finup = ahash_finup,
4358 			.digest = ahash_digest,
4359 			.export = ahash_export,
4360 			.import = ahash_import,
4361 			.setkey = ahash_setkey,
4362 			.halg = {
4363 				.digestsize = SHA256_DIGEST_SIZE,
4364 				.statesize = sizeof(struct caam_export_state),
4365 			},
4366 		},
4367 		.alg_type = OP_ALG_ALGSEL_SHA256,
4368 	}, {
4369 		.name = "sha384",
4370 		.driver_name = "sha384-caam-qi2",
4371 		.hmac_name = "hmac(sha384)",
4372 		.hmac_driver_name = "hmac-sha384-caam-qi2",
4373 		.blocksize = SHA384_BLOCK_SIZE,
4374 		.template_ahash = {
4375 			.init = ahash_init,
4376 			.update = ahash_update,
4377 			.final = ahash_final,
4378 			.finup = ahash_finup,
4379 			.digest = ahash_digest,
4380 			.export = ahash_export,
4381 			.import = ahash_import,
4382 			.setkey = ahash_setkey,
4383 			.halg = {
4384 				.digestsize = SHA384_DIGEST_SIZE,
4385 				.statesize = sizeof(struct caam_export_state),
4386 			},
4387 		},
4388 		.alg_type = OP_ALG_ALGSEL_SHA384,
4389 	}, {
4390 		.name = "sha512",
4391 		.driver_name = "sha512-caam-qi2",
4392 		.hmac_name = "hmac(sha512)",
4393 		.hmac_driver_name = "hmac-sha512-caam-qi2",
4394 		.blocksize = SHA512_BLOCK_SIZE,
4395 		.template_ahash = {
4396 			.init = ahash_init,
4397 			.update = ahash_update,
4398 			.final = ahash_final,
4399 			.finup = ahash_finup,
4400 			.digest = ahash_digest,
4401 			.export = ahash_export,
4402 			.import = ahash_import,
4403 			.setkey = ahash_setkey,
4404 			.halg = {
4405 				.digestsize = SHA512_DIGEST_SIZE,
4406 				.statesize = sizeof(struct caam_export_state),
4407 			},
4408 		},
4409 		.alg_type = OP_ALG_ALGSEL_SHA512,
4410 	}, {
4411 		.name = "md5",
4412 		.driver_name = "md5-caam-qi2",
4413 		.hmac_name = "hmac(md5)",
4414 		.hmac_driver_name = "hmac-md5-caam-qi2",
4415 		.blocksize = MD5_BLOCK_WORDS * 4,
4416 		.template_ahash = {
4417 			.init = ahash_init,
4418 			.update = ahash_update,
4419 			.final = ahash_final,
4420 			.finup = ahash_finup,
4421 			.digest = ahash_digest,
4422 			.export = ahash_export,
4423 			.import = ahash_import,
4424 			.setkey = ahash_setkey,
4425 			.halg = {
4426 				.digestsize = MD5_DIGEST_SIZE,
4427 				.statesize = sizeof(struct caam_export_state),
4428 			},
4429 		},
4430 		.alg_type = OP_ALG_ALGSEL_MD5,
4431 	}
4432 };
4433 
4434 struct caam_hash_alg {
4435 	struct list_head entry;
4436 	struct device *dev;
4437 	int alg_type;
4438 	struct ahash_alg ahash_alg;
4439 };
4440 
4441 static int caam_hash_cra_init(struct crypto_tfm *tfm)
4442 {
4443 	struct crypto_ahash *ahash = __crypto_ahash_cast(tfm);
4444 	struct crypto_alg *base = tfm->__crt_alg;
4445 	struct hash_alg_common *halg =
4446 		 container_of(base, struct hash_alg_common, base);
4447 	struct ahash_alg *alg =
4448 		 container_of(halg, struct ahash_alg, halg);
4449 	struct caam_hash_alg *caam_hash =
4450 		 container_of(alg, struct caam_hash_alg, ahash_alg);
4451 	struct caam_hash_ctx *ctx = crypto_tfm_ctx(tfm);
4452 	/* Sizes for MDHA running digests: MD5, SHA1, 224, 256, 384, 512 */
4453 	static const u8 runninglen[] = { HASH_MSG_LEN + MD5_DIGEST_SIZE,
4454 					 HASH_MSG_LEN + SHA1_DIGEST_SIZE,
4455 					 HASH_MSG_LEN + 32,
4456 					 HASH_MSG_LEN + SHA256_DIGEST_SIZE,
4457 					 HASH_MSG_LEN + 64,
4458 					 HASH_MSG_LEN + SHA512_DIGEST_SIZE };
4459 	dma_addr_t dma_addr;
4460 	int i;
4461 
4462 	ctx->dev = caam_hash->dev;
4463 
4464 	if (alg->setkey) {
4465 		ctx->adata.key_dma = dma_map_single_attrs(ctx->dev, ctx->key,
4466 							  ARRAY_SIZE(ctx->key),
4467 							  DMA_TO_DEVICE,
4468 							  DMA_ATTR_SKIP_CPU_SYNC);
4469 		if (dma_mapping_error(ctx->dev, ctx->adata.key_dma)) {
4470 			dev_err(ctx->dev, "unable to map key\n");
4471 			return -ENOMEM;
4472 		}
4473 	}
4474 
4475 	dma_addr = dma_map_single_attrs(ctx->dev, ctx->flc, sizeof(ctx->flc),
4476 					DMA_BIDIRECTIONAL,
4477 					DMA_ATTR_SKIP_CPU_SYNC);
4478 	if (dma_mapping_error(ctx->dev, dma_addr)) {
4479 		dev_err(ctx->dev, "unable to map shared descriptors\n");
4480 		if (ctx->adata.key_dma)
4481 			dma_unmap_single_attrs(ctx->dev, ctx->adata.key_dma,
4482 					       ARRAY_SIZE(ctx->key),
4483 					       DMA_TO_DEVICE,
4484 					       DMA_ATTR_SKIP_CPU_SYNC);
4485 		return -ENOMEM;
4486 	}
4487 
4488 	for (i = 0; i < HASH_NUM_OP; i++)
4489 		ctx->flc_dma[i] = dma_addr + i * sizeof(ctx->flc[i]);
4490 
4491 	/* copy descriptor header template value */
4492 	ctx->adata.algtype = OP_TYPE_CLASS2_ALG | caam_hash->alg_type;
4493 
4494 	ctx->ctx_len = runninglen[(ctx->adata.algtype &
4495 				   OP_ALG_ALGSEL_SUBMASK) >>
4496 				  OP_ALG_ALGSEL_SHIFT];
4497 
4498 	crypto_ahash_set_reqsize(__crypto_ahash_cast(tfm),
4499 				 sizeof(struct caam_hash_state));
4500 
4501 	return ahash_set_sh_desc(ahash);
4502 }
4503 
4504 static void caam_hash_cra_exit(struct crypto_tfm *tfm)
4505 {
4506 	struct caam_hash_ctx *ctx = crypto_tfm_ctx(tfm);
4507 
4508 	dma_unmap_single_attrs(ctx->dev, ctx->flc_dma[0], sizeof(ctx->flc),
4509 			       DMA_BIDIRECTIONAL, DMA_ATTR_SKIP_CPU_SYNC);
4510 	if (ctx->adata.key_dma)
4511 		dma_unmap_single_attrs(ctx->dev, ctx->adata.key_dma,
4512 				       ARRAY_SIZE(ctx->key), DMA_TO_DEVICE,
4513 				       DMA_ATTR_SKIP_CPU_SYNC);
4514 }
4515 
4516 static struct caam_hash_alg *caam_hash_alloc(struct device *dev,
4517 	struct caam_hash_template *template, bool keyed)
4518 {
4519 	struct caam_hash_alg *t_alg;
4520 	struct ahash_alg *halg;
4521 	struct crypto_alg *alg;
4522 
4523 	t_alg = kzalloc(sizeof(*t_alg), GFP_KERNEL);
4524 	if (!t_alg)
4525 		return ERR_PTR(-ENOMEM);
4526 
4527 	t_alg->ahash_alg = template->template_ahash;
4528 	halg = &t_alg->ahash_alg;
4529 	alg = &halg->halg.base;
4530 
4531 	if (keyed) {
4532 		snprintf(alg->cra_name, CRYPTO_MAX_ALG_NAME, "%s",
4533 			 template->hmac_name);
4534 		snprintf(alg->cra_driver_name, CRYPTO_MAX_ALG_NAME, "%s",
4535 			 template->hmac_driver_name);
4536 	} else {
4537 		snprintf(alg->cra_name, CRYPTO_MAX_ALG_NAME, "%s",
4538 			 template->name);
4539 		snprintf(alg->cra_driver_name, CRYPTO_MAX_ALG_NAME, "%s",
4540 			 template->driver_name);
4541 		t_alg->ahash_alg.setkey = NULL;
4542 	}
4543 	alg->cra_module = THIS_MODULE;
4544 	alg->cra_init = caam_hash_cra_init;
4545 	alg->cra_exit = caam_hash_cra_exit;
4546 	alg->cra_ctxsize = sizeof(struct caam_hash_ctx);
4547 	alg->cra_priority = CAAM_CRA_PRIORITY;
4548 	alg->cra_blocksize = template->blocksize;
4549 	alg->cra_alignmask = 0;
4550 	alg->cra_flags = CRYPTO_ALG_ASYNC;
4551 
4552 	t_alg->alg_type = template->alg_type;
4553 	t_alg->dev = dev;
4554 
4555 	return t_alg;
4556 }
4557 
4558 static void dpaa2_caam_fqdan_cb(struct dpaa2_io_notification_ctx *nctx)
4559 {
4560 	struct dpaa2_caam_priv_per_cpu *ppriv;
4561 
4562 	ppriv = container_of(nctx, struct dpaa2_caam_priv_per_cpu, nctx);
4563 	napi_schedule_irqoff(&ppriv->napi);
4564 }
4565 
4566 static int __cold dpaa2_dpseci_dpio_setup(struct dpaa2_caam_priv *priv)
4567 {
4568 	struct device *dev = priv->dev;
4569 	struct dpaa2_io_notification_ctx *nctx;
4570 	struct dpaa2_caam_priv_per_cpu *ppriv;
4571 	int err, i = 0, cpu;
4572 
4573 	for_each_online_cpu(cpu) {
4574 		ppriv = per_cpu_ptr(priv->ppriv, cpu);
4575 		ppriv->priv = priv;
4576 		nctx = &ppriv->nctx;
4577 		nctx->is_cdan = 0;
4578 		nctx->id = ppriv->rsp_fqid;
4579 		nctx->desired_cpu = cpu;
4580 		nctx->cb = dpaa2_caam_fqdan_cb;
4581 
4582 		/* Register notification callbacks */
4583 		ppriv->dpio = dpaa2_io_service_select(cpu);
4584 		err = dpaa2_io_service_register(ppriv->dpio, nctx, dev);
4585 		if (unlikely(err)) {
4586 			dev_dbg(dev, "No affine DPIO for cpu %d\n", cpu);
4587 			nctx->cb = NULL;
4588 			/*
4589 			 * If no affine DPIO for this core, there's probably
4590 			 * none available for next cores either. Signal we want
4591 			 * to retry later, in case the DPIO devices weren't
4592 			 * probed yet.
4593 			 */
4594 			err = -EPROBE_DEFER;
4595 			goto err;
4596 		}
4597 
4598 		ppriv->store = dpaa2_io_store_create(DPAA2_CAAM_STORE_SIZE,
4599 						     dev);
4600 		if (unlikely(!ppriv->store)) {
4601 			dev_err(dev, "dpaa2_io_store_create() failed\n");
4602 			err = -ENOMEM;
4603 			goto err;
4604 		}
4605 
4606 		if (++i == priv->num_pairs)
4607 			break;
4608 	}
4609 
4610 	return 0;
4611 
4612 err:
4613 	for_each_online_cpu(cpu) {
4614 		ppriv = per_cpu_ptr(priv->ppriv, cpu);
4615 		if (!ppriv->nctx.cb)
4616 			break;
4617 		dpaa2_io_service_deregister(ppriv->dpio, &ppriv->nctx, dev);
4618 	}
4619 
4620 	for_each_online_cpu(cpu) {
4621 		ppriv = per_cpu_ptr(priv->ppriv, cpu);
4622 		if (!ppriv->store)
4623 			break;
4624 		dpaa2_io_store_destroy(ppriv->store);
4625 	}
4626 
4627 	return err;
4628 }
4629 
4630 static void __cold dpaa2_dpseci_dpio_free(struct dpaa2_caam_priv *priv)
4631 {
4632 	struct dpaa2_caam_priv_per_cpu *ppriv;
4633 	int i = 0, cpu;
4634 
4635 	for_each_online_cpu(cpu) {
4636 		ppriv = per_cpu_ptr(priv->ppriv, cpu);
4637 		dpaa2_io_service_deregister(ppriv->dpio, &ppriv->nctx,
4638 					    priv->dev);
4639 		dpaa2_io_store_destroy(ppriv->store);
4640 
4641 		if (++i == priv->num_pairs)
4642 			return;
4643 	}
4644 }
4645 
4646 static int dpaa2_dpseci_bind(struct dpaa2_caam_priv *priv)
4647 {
4648 	struct dpseci_rx_queue_cfg rx_queue_cfg;
4649 	struct device *dev = priv->dev;
4650 	struct fsl_mc_device *ls_dev = to_fsl_mc_device(dev);
4651 	struct dpaa2_caam_priv_per_cpu *ppriv;
4652 	int err = 0, i = 0, cpu;
4653 
4654 	/* Configure Rx queues */
4655 	for_each_online_cpu(cpu) {
4656 		ppriv = per_cpu_ptr(priv->ppriv, cpu);
4657 
4658 		rx_queue_cfg.options = DPSECI_QUEUE_OPT_DEST |
4659 				       DPSECI_QUEUE_OPT_USER_CTX;
4660 		rx_queue_cfg.order_preservation_en = 0;
4661 		rx_queue_cfg.dest_cfg.dest_type = DPSECI_DEST_DPIO;
4662 		rx_queue_cfg.dest_cfg.dest_id = ppriv->nctx.dpio_id;
4663 		/*
4664 		 * Rx priority (WQ) doesn't really matter, since we use
4665 		 * pull mode, i.e. volatile dequeues from specific FQs
4666 		 */
4667 		rx_queue_cfg.dest_cfg.priority = 0;
4668 		rx_queue_cfg.user_ctx = ppriv->nctx.qman64;
4669 
4670 		err = dpseci_set_rx_queue(priv->mc_io, 0, ls_dev->mc_handle, i,
4671 					  &rx_queue_cfg);
4672 		if (err) {
4673 			dev_err(dev, "dpseci_set_rx_queue() failed with err %d\n",
4674 				err);
4675 			return err;
4676 		}
4677 
4678 		if (++i == priv->num_pairs)
4679 			break;
4680 	}
4681 
4682 	return err;
4683 }
4684 
4685 static void dpaa2_dpseci_congestion_free(struct dpaa2_caam_priv *priv)
4686 {
4687 	struct device *dev = priv->dev;
4688 
4689 	if (!priv->cscn_mem)
4690 		return;
4691 
4692 	dma_unmap_single(dev, priv->cscn_dma, DPAA2_CSCN_SIZE, DMA_FROM_DEVICE);
4693 	kfree(priv->cscn_mem);
4694 }
4695 
4696 static void dpaa2_dpseci_free(struct dpaa2_caam_priv *priv)
4697 {
4698 	struct device *dev = priv->dev;
4699 	struct fsl_mc_device *ls_dev = to_fsl_mc_device(dev);
4700 
4701 	dpaa2_dpseci_congestion_free(priv);
4702 	dpseci_close(priv->mc_io, 0, ls_dev->mc_handle);
4703 }
4704 
4705 static void dpaa2_caam_process_fd(struct dpaa2_caam_priv *priv,
4706 				  const struct dpaa2_fd *fd)
4707 {
4708 	struct caam_request *req;
4709 	u32 fd_err;
4710 
4711 	if (dpaa2_fd_get_format(fd) != dpaa2_fd_list) {
4712 		dev_err(priv->dev, "Only Frame List FD format is supported!\n");
4713 		return;
4714 	}
4715 
4716 	fd_err = dpaa2_fd_get_ctrl(fd) & FD_CTRL_ERR_MASK;
4717 	if (unlikely(fd_err))
4718 		dev_err_ratelimited(priv->dev, "FD error: %08x\n", fd_err);
4719 
4720 	/*
4721 	 * FD[ADDR] is guaranteed to be valid, irrespective of errors reported
4722 	 * in FD[ERR] or FD[FRC].
4723 	 */
4724 	req = dpaa2_caam_iova_to_virt(priv, dpaa2_fd_get_addr(fd));
4725 	dma_unmap_single(priv->dev, req->fd_flt_dma, sizeof(req->fd_flt),
4726 			 DMA_BIDIRECTIONAL);
4727 	req->cbk(req->ctx, dpaa2_fd_get_frc(fd));
4728 }
4729 
4730 static int dpaa2_caam_pull_fq(struct dpaa2_caam_priv_per_cpu *ppriv)
4731 {
4732 	int err;
4733 
4734 	/* Retry while portal is busy */
4735 	do {
4736 		err = dpaa2_io_service_pull_fq(ppriv->dpio, ppriv->rsp_fqid,
4737 					       ppriv->store);
4738 	} while (err == -EBUSY);
4739 
4740 	if (unlikely(err))
4741 		dev_err(ppriv->priv->dev, "dpaa2_io_service_pull err %d", err);
4742 
4743 	return err;
4744 }
4745 
4746 static int dpaa2_caam_store_consume(struct dpaa2_caam_priv_per_cpu *ppriv)
4747 {
4748 	struct dpaa2_dq *dq;
4749 	int cleaned = 0, is_last;
4750 
4751 	do {
4752 		dq = dpaa2_io_store_next(ppriv->store, &is_last);
4753 		if (unlikely(!dq)) {
4754 			if (unlikely(!is_last)) {
4755 				dev_dbg(ppriv->priv->dev,
4756 					"FQ %d returned no valid frames\n",
4757 					ppriv->rsp_fqid);
4758 				/*
4759 				 * MUST retry until we get some sort of
4760 				 * valid response token (be it "empty dequeue"
4761 				 * or a valid frame).
4762 				 */
4763 				continue;
4764 			}
4765 			break;
4766 		}
4767 
4768 		/* Process FD */
4769 		dpaa2_caam_process_fd(ppriv->priv, dpaa2_dq_fd(dq));
4770 		cleaned++;
4771 	} while (!is_last);
4772 
4773 	return cleaned;
4774 }
4775 
4776 static int dpaa2_dpseci_poll(struct napi_struct *napi, int budget)
4777 {
4778 	struct dpaa2_caam_priv_per_cpu *ppriv;
4779 	struct dpaa2_caam_priv *priv;
4780 	int err, cleaned = 0, store_cleaned;
4781 
4782 	ppriv = container_of(napi, struct dpaa2_caam_priv_per_cpu, napi);
4783 	priv = ppriv->priv;
4784 
4785 	if (unlikely(dpaa2_caam_pull_fq(ppriv)))
4786 		return 0;
4787 
4788 	do {
4789 		store_cleaned = dpaa2_caam_store_consume(ppriv);
4790 		cleaned += store_cleaned;
4791 
4792 		if (store_cleaned == 0 ||
4793 		    cleaned > budget - DPAA2_CAAM_STORE_SIZE)
4794 			break;
4795 
4796 		/* Try to dequeue some more */
4797 		err = dpaa2_caam_pull_fq(ppriv);
4798 		if (unlikely(err))
4799 			break;
4800 	} while (1);
4801 
4802 	if (cleaned < budget) {
4803 		napi_complete_done(napi, cleaned);
4804 		err = dpaa2_io_service_rearm(ppriv->dpio, &ppriv->nctx);
4805 		if (unlikely(err))
4806 			dev_err(priv->dev, "Notification rearm failed: %d\n",
4807 				err);
4808 	}
4809 
4810 	return cleaned;
4811 }
4812 
4813 static int dpaa2_dpseci_congestion_setup(struct dpaa2_caam_priv *priv,
4814 					 u16 token)
4815 {
4816 	struct dpseci_congestion_notification_cfg cong_notif_cfg = { 0 };
4817 	struct device *dev = priv->dev;
4818 	int err;
4819 
4820 	/*
4821 	 * Congestion group feature supported starting with DPSECI API v5.1
4822 	 * and only when object has been created with this capability.
4823 	 */
4824 	if ((DPSECI_VER(priv->major_ver, priv->minor_ver) < DPSECI_VER(5, 1)) ||
4825 	    !(priv->dpseci_attr.options & DPSECI_OPT_HAS_CG))
4826 		return 0;
4827 
4828 	priv->cscn_mem = kzalloc(DPAA2_CSCN_SIZE + DPAA2_CSCN_ALIGN,
4829 				 GFP_KERNEL | GFP_DMA);
4830 	if (!priv->cscn_mem)
4831 		return -ENOMEM;
4832 
4833 	priv->cscn_mem_aligned = PTR_ALIGN(priv->cscn_mem, DPAA2_CSCN_ALIGN);
4834 	priv->cscn_dma = dma_map_single(dev, priv->cscn_mem_aligned,
4835 					DPAA2_CSCN_SIZE, DMA_FROM_DEVICE);
4836 	if (dma_mapping_error(dev, priv->cscn_dma)) {
4837 		dev_err(dev, "Error mapping CSCN memory area\n");
4838 		err = -ENOMEM;
4839 		goto err_dma_map;
4840 	}
4841 
4842 	cong_notif_cfg.units = DPSECI_CONGESTION_UNIT_BYTES;
4843 	cong_notif_cfg.threshold_entry = DPAA2_SEC_CONG_ENTRY_THRESH;
4844 	cong_notif_cfg.threshold_exit = DPAA2_SEC_CONG_EXIT_THRESH;
4845 	cong_notif_cfg.message_ctx = (uintptr_t)priv;
4846 	cong_notif_cfg.message_iova = priv->cscn_dma;
4847 	cong_notif_cfg.notification_mode = DPSECI_CGN_MODE_WRITE_MEM_ON_ENTER |
4848 					DPSECI_CGN_MODE_WRITE_MEM_ON_EXIT |
4849 					DPSECI_CGN_MODE_COHERENT_WRITE;
4850 
4851 	err = dpseci_set_congestion_notification(priv->mc_io, 0, token,
4852 						 &cong_notif_cfg);
4853 	if (err) {
4854 		dev_err(dev, "dpseci_set_congestion_notification failed\n");
4855 		goto err_set_cong;
4856 	}
4857 
4858 	return 0;
4859 
4860 err_set_cong:
4861 	dma_unmap_single(dev, priv->cscn_dma, DPAA2_CSCN_SIZE, DMA_FROM_DEVICE);
4862 err_dma_map:
4863 	kfree(priv->cscn_mem);
4864 
4865 	return err;
4866 }
4867 
4868 static int __cold dpaa2_dpseci_setup(struct fsl_mc_device *ls_dev)
4869 {
4870 	struct device *dev = &ls_dev->dev;
4871 	struct dpaa2_caam_priv *priv;
4872 	struct dpaa2_caam_priv_per_cpu *ppriv;
4873 	int err, cpu;
4874 	u8 i;
4875 
4876 	priv = dev_get_drvdata(dev);
4877 
4878 	priv->dev = dev;
4879 	priv->dpsec_id = ls_dev->obj_desc.id;
4880 
4881 	/* Get a handle for the DPSECI this interface is associate with */
4882 	err = dpseci_open(priv->mc_io, 0, priv->dpsec_id, &ls_dev->mc_handle);
4883 	if (err) {
4884 		dev_err(dev, "dpseci_open() failed: %d\n", err);
4885 		goto err_open;
4886 	}
4887 
4888 	err = dpseci_get_api_version(priv->mc_io, 0, &priv->major_ver,
4889 				     &priv->minor_ver);
4890 	if (err) {
4891 		dev_err(dev, "dpseci_get_api_version() failed\n");
4892 		goto err_get_vers;
4893 	}
4894 
4895 	dev_info(dev, "dpseci v%d.%d\n", priv->major_ver, priv->minor_ver);
4896 
4897 	err = dpseci_get_attributes(priv->mc_io, 0, ls_dev->mc_handle,
4898 				    &priv->dpseci_attr);
4899 	if (err) {
4900 		dev_err(dev, "dpseci_get_attributes() failed\n");
4901 		goto err_get_vers;
4902 	}
4903 
4904 	err = dpseci_get_sec_attr(priv->mc_io, 0, ls_dev->mc_handle,
4905 				  &priv->sec_attr);
4906 	if (err) {
4907 		dev_err(dev, "dpseci_get_sec_attr() failed\n");
4908 		goto err_get_vers;
4909 	}
4910 
4911 	err = dpaa2_dpseci_congestion_setup(priv, ls_dev->mc_handle);
4912 	if (err) {
4913 		dev_err(dev, "setup_congestion() failed\n");
4914 		goto err_get_vers;
4915 	}
4916 
4917 	priv->num_pairs = min(priv->dpseci_attr.num_rx_queues,
4918 			      priv->dpseci_attr.num_tx_queues);
4919 	if (priv->num_pairs > num_online_cpus()) {
4920 		dev_warn(dev, "%d queues won't be used\n",
4921 			 priv->num_pairs - num_online_cpus());
4922 		priv->num_pairs = num_online_cpus();
4923 	}
4924 
4925 	for (i = 0; i < priv->dpseci_attr.num_rx_queues; i++) {
4926 		err = dpseci_get_rx_queue(priv->mc_io, 0, ls_dev->mc_handle, i,
4927 					  &priv->rx_queue_attr[i]);
4928 		if (err) {
4929 			dev_err(dev, "dpseci_get_rx_queue() failed\n");
4930 			goto err_get_rx_queue;
4931 		}
4932 	}
4933 
4934 	for (i = 0; i < priv->dpseci_attr.num_tx_queues; i++) {
4935 		err = dpseci_get_tx_queue(priv->mc_io, 0, ls_dev->mc_handle, i,
4936 					  &priv->tx_queue_attr[i]);
4937 		if (err) {
4938 			dev_err(dev, "dpseci_get_tx_queue() failed\n");
4939 			goto err_get_rx_queue;
4940 		}
4941 	}
4942 
4943 	i = 0;
4944 	for_each_online_cpu(cpu) {
4945 		u8 j;
4946 
4947 		j = i % priv->num_pairs;
4948 
4949 		ppriv = per_cpu_ptr(priv->ppriv, cpu);
4950 		ppriv->req_fqid = priv->tx_queue_attr[j].fqid;
4951 
4952 		/*
4953 		 * Allow all cores to enqueue, while only some of them
4954 		 * will take part in dequeuing.
4955 		 */
4956 		if (++i > priv->num_pairs)
4957 			continue;
4958 
4959 		ppriv->rsp_fqid = priv->rx_queue_attr[j].fqid;
4960 		ppriv->prio = j;
4961 
4962 		dev_dbg(dev, "pair %d: rx queue %d, tx queue %d\n", j,
4963 			priv->rx_queue_attr[j].fqid,
4964 			priv->tx_queue_attr[j].fqid);
4965 
4966 		ppriv->net_dev.dev = *dev;
4967 		INIT_LIST_HEAD(&ppriv->net_dev.napi_list);
4968 		netif_napi_add(&ppriv->net_dev, &ppriv->napi, dpaa2_dpseci_poll,
4969 			       DPAA2_CAAM_NAPI_WEIGHT);
4970 	}
4971 
4972 	return 0;
4973 
4974 err_get_rx_queue:
4975 	dpaa2_dpseci_congestion_free(priv);
4976 err_get_vers:
4977 	dpseci_close(priv->mc_io, 0, ls_dev->mc_handle);
4978 err_open:
4979 	return err;
4980 }
4981 
4982 static int dpaa2_dpseci_enable(struct dpaa2_caam_priv *priv)
4983 {
4984 	struct device *dev = priv->dev;
4985 	struct fsl_mc_device *ls_dev = to_fsl_mc_device(dev);
4986 	struct dpaa2_caam_priv_per_cpu *ppriv;
4987 	int i;
4988 
4989 	for (i = 0; i < priv->num_pairs; i++) {
4990 		ppriv = per_cpu_ptr(priv->ppriv, i);
4991 		napi_enable(&ppriv->napi);
4992 	}
4993 
4994 	return dpseci_enable(priv->mc_io, 0, ls_dev->mc_handle);
4995 }
4996 
4997 static int __cold dpaa2_dpseci_disable(struct dpaa2_caam_priv *priv)
4998 {
4999 	struct device *dev = priv->dev;
5000 	struct dpaa2_caam_priv_per_cpu *ppriv;
5001 	struct fsl_mc_device *ls_dev = to_fsl_mc_device(dev);
5002 	int i, err = 0, enabled;
5003 
5004 	err = dpseci_disable(priv->mc_io, 0, ls_dev->mc_handle);
5005 	if (err) {
5006 		dev_err(dev, "dpseci_disable() failed\n");
5007 		return err;
5008 	}
5009 
5010 	err = dpseci_is_enabled(priv->mc_io, 0, ls_dev->mc_handle, &enabled);
5011 	if (err) {
5012 		dev_err(dev, "dpseci_is_enabled() failed\n");
5013 		return err;
5014 	}
5015 
5016 	dev_dbg(dev, "disable: %s\n", enabled ? "false" : "true");
5017 
5018 	for (i = 0; i < priv->num_pairs; i++) {
5019 		ppriv = per_cpu_ptr(priv->ppriv, i);
5020 		napi_disable(&ppriv->napi);
5021 		netif_napi_del(&ppriv->napi);
5022 	}
5023 
5024 	return 0;
5025 }
5026 
5027 static struct list_head hash_list;
5028 
5029 static int dpaa2_caam_probe(struct fsl_mc_device *dpseci_dev)
5030 {
5031 	struct device *dev;
5032 	struct dpaa2_caam_priv *priv;
5033 	int i, err = 0;
5034 	bool registered = false;
5035 
5036 	/*
5037 	 * There is no way to get CAAM endianness - there is no direct register
5038 	 * space access and MC f/w does not provide this attribute.
5039 	 * All DPAA2-based SoCs have little endian CAAM, thus hard-code this
5040 	 * property.
5041 	 */
5042 	caam_little_end = true;
5043 
5044 	caam_imx = false;
5045 
5046 	dev = &dpseci_dev->dev;
5047 
5048 	priv = devm_kzalloc(dev, sizeof(*priv), GFP_KERNEL);
5049 	if (!priv)
5050 		return -ENOMEM;
5051 
5052 	dev_set_drvdata(dev, priv);
5053 
5054 	priv->domain = iommu_get_domain_for_dev(dev);
5055 
5056 	qi_cache = kmem_cache_create("dpaa2_caamqicache", CAAM_QI_MEMCACHE_SIZE,
5057 				     0, SLAB_CACHE_DMA, NULL);
5058 	if (!qi_cache) {
5059 		dev_err(dev, "Can't allocate SEC cache\n");
5060 		return -ENOMEM;
5061 	}
5062 
5063 	err = dma_set_mask_and_coherent(dev, DMA_BIT_MASK(49));
5064 	if (err) {
5065 		dev_err(dev, "dma_set_mask_and_coherent() failed\n");
5066 		goto err_dma_mask;
5067 	}
5068 
5069 	/* Obtain a MC portal */
5070 	err = fsl_mc_portal_allocate(dpseci_dev, 0, &priv->mc_io);
5071 	if (err) {
5072 		if (err == -ENXIO)
5073 			err = -EPROBE_DEFER;
5074 		else
5075 			dev_err(dev, "MC portal allocation failed\n");
5076 
5077 		goto err_dma_mask;
5078 	}
5079 
5080 	priv->ppriv = alloc_percpu(*priv->ppriv);
5081 	if (!priv->ppriv) {
5082 		dev_err(dev, "alloc_percpu() failed\n");
5083 		err = -ENOMEM;
5084 		goto err_alloc_ppriv;
5085 	}
5086 
5087 	/* DPSECI initialization */
5088 	err = dpaa2_dpseci_setup(dpseci_dev);
5089 	if (err) {
5090 		dev_err(dev, "dpaa2_dpseci_setup() failed\n");
5091 		goto err_dpseci_setup;
5092 	}
5093 
5094 	/* DPIO */
5095 	err = dpaa2_dpseci_dpio_setup(priv);
5096 	if (err) {
5097 		if (err != -EPROBE_DEFER)
5098 			dev_err(dev, "dpaa2_dpseci_dpio_setup() failed\n");
5099 		goto err_dpio_setup;
5100 	}
5101 
5102 	/* DPSECI binding to DPIO */
5103 	err = dpaa2_dpseci_bind(priv);
5104 	if (err) {
5105 		dev_err(dev, "dpaa2_dpseci_bind() failed\n");
5106 		goto err_bind;
5107 	}
5108 
5109 	/* DPSECI enable */
5110 	err = dpaa2_dpseci_enable(priv);
5111 	if (err) {
5112 		dev_err(dev, "dpaa2_dpseci_enable() failed\n");
5113 		goto err_bind;
5114 	}
5115 
5116 	dpaa2_dpseci_debugfs_init(priv);
5117 
5118 	/* register crypto algorithms the device supports */
5119 	for (i = 0; i < ARRAY_SIZE(driver_algs); i++) {
5120 		struct caam_skcipher_alg *t_alg = driver_algs + i;
5121 		u32 alg_sel = t_alg->caam.class1_alg_type & OP_ALG_ALGSEL_MASK;
5122 
5123 		/* Skip DES algorithms if not supported by device */
5124 		if (!priv->sec_attr.des_acc_num &&
5125 		    (alg_sel == OP_ALG_ALGSEL_3DES ||
5126 		     alg_sel == OP_ALG_ALGSEL_DES))
5127 			continue;
5128 
5129 		/* Skip AES algorithms if not supported by device */
5130 		if (!priv->sec_attr.aes_acc_num &&
5131 		    alg_sel == OP_ALG_ALGSEL_AES)
5132 			continue;
5133 
5134 		/* Skip CHACHA20 algorithms if not supported by device */
5135 		if (alg_sel == OP_ALG_ALGSEL_CHACHA20 &&
5136 		    !priv->sec_attr.ccha_acc_num)
5137 			continue;
5138 
5139 		t_alg->caam.dev = dev;
5140 		caam_skcipher_alg_init(t_alg);
5141 
5142 		err = crypto_register_skcipher(&t_alg->skcipher);
5143 		if (err) {
5144 			dev_warn(dev, "%s alg registration failed: %d\n",
5145 				 t_alg->skcipher.base.cra_driver_name, err);
5146 			continue;
5147 		}
5148 
5149 		t_alg->registered = true;
5150 		registered = true;
5151 	}
5152 
5153 	for (i = 0; i < ARRAY_SIZE(driver_aeads); i++) {
5154 		struct caam_aead_alg *t_alg = driver_aeads + i;
5155 		u32 c1_alg_sel = t_alg->caam.class1_alg_type &
5156 				 OP_ALG_ALGSEL_MASK;
5157 		u32 c2_alg_sel = t_alg->caam.class2_alg_type &
5158 				 OP_ALG_ALGSEL_MASK;
5159 
5160 		/* Skip DES algorithms if not supported by device */
5161 		if (!priv->sec_attr.des_acc_num &&
5162 		    (c1_alg_sel == OP_ALG_ALGSEL_3DES ||
5163 		     c1_alg_sel == OP_ALG_ALGSEL_DES))
5164 			continue;
5165 
5166 		/* Skip AES algorithms if not supported by device */
5167 		if (!priv->sec_attr.aes_acc_num &&
5168 		    c1_alg_sel == OP_ALG_ALGSEL_AES)
5169 			continue;
5170 
5171 		/* Skip CHACHA20 algorithms if not supported by device */
5172 		if (c1_alg_sel == OP_ALG_ALGSEL_CHACHA20 &&
5173 		    !priv->sec_attr.ccha_acc_num)
5174 			continue;
5175 
5176 		/* Skip POLY1305 algorithms if not supported by device */
5177 		if (c2_alg_sel == OP_ALG_ALGSEL_POLY1305 &&
5178 		    !priv->sec_attr.ptha_acc_num)
5179 			continue;
5180 
5181 		/*
5182 		 * Skip algorithms requiring message digests
5183 		 * if MD not supported by device.
5184 		 */
5185 		if ((c2_alg_sel & ~OP_ALG_ALGSEL_SUBMASK) == 0x40 &&
5186 		    !priv->sec_attr.md_acc_num)
5187 			continue;
5188 
5189 		t_alg->caam.dev = dev;
5190 		caam_aead_alg_init(t_alg);
5191 
5192 		err = crypto_register_aead(&t_alg->aead);
5193 		if (err) {
5194 			dev_warn(dev, "%s alg registration failed: %d\n",
5195 				 t_alg->aead.base.cra_driver_name, err);
5196 			continue;
5197 		}
5198 
5199 		t_alg->registered = true;
5200 		registered = true;
5201 	}
5202 	if (registered)
5203 		dev_info(dev, "algorithms registered in /proc/crypto\n");
5204 
5205 	/* register hash algorithms the device supports */
5206 	INIT_LIST_HEAD(&hash_list);
5207 
5208 	/*
5209 	 * Skip registration of any hashing algorithms if MD block
5210 	 * is not present.
5211 	 */
5212 	if (!priv->sec_attr.md_acc_num)
5213 		return 0;
5214 
5215 	for (i = 0; i < ARRAY_SIZE(driver_hash); i++) {
5216 		struct caam_hash_alg *t_alg;
5217 		struct caam_hash_template *alg = driver_hash + i;
5218 
5219 		/* register hmac version */
5220 		t_alg = caam_hash_alloc(dev, alg, true);
5221 		if (IS_ERR(t_alg)) {
5222 			err = PTR_ERR(t_alg);
5223 			dev_warn(dev, "%s hash alg allocation failed: %d\n",
5224 				 alg->driver_name, err);
5225 			continue;
5226 		}
5227 
5228 		err = crypto_register_ahash(&t_alg->ahash_alg);
5229 		if (err) {
5230 			dev_warn(dev, "%s alg registration failed: %d\n",
5231 				 t_alg->ahash_alg.halg.base.cra_driver_name,
5232 				 err);
5233 			kfree(t_alg);
5234 		} else {
5235 			list_add_tail(&t_alg->entry, &hash_list);
5236 		}
5237 
5238 		/* register unkeyed version */
5239 		t_alg = caam_hash_alloc(dev, alg, false);
5240 		if (IS_ERR(t_alg)) {
5241 			err = PTR_ERR(t_alg);
5242 			dev_warn(dev, "%s alg allocation failed: %d\n",
5243 				 alg->driver_name, err);
5244 			continue;
5245 		}
5246 
5247 		err = crypto_register_ahash(&t_alg->ahash_alg);
5248 		if (err) {
5249 			dev_warn(dev, "%s alg registration failed: %d\n",
5250 				 t_alg->ahash_alg.halg.base.cra_driver_name,
5251 				 err);
5252 			kfree(t_alg);
5253 		} else {
5254 			list_add_tail(&t_alg->entry, &hash_list);
5255 		}
5256 	}
5257 	if (!list_empty(&hash_list))
5258 		dev_info(dev, "hash algorithms registered in /proc/crypto\n");
5259 
5260 	return err;
5261 
5262 err_bind:
5263 	dpaa2_dpseci_dpio_free(priv);
5264 err_dpio_setup:
5265 	dpaa2_dpseci_free(priv);
5266 err_dpseci_setup:
5267 	free_percpu(priv->ppriv);
5268 err_alloc_ppriv:
5269 	fsl_mc_portal_free(priv->mc_io);
5270 err_dma_mask:
5271 	kmem_cache_destroy(qi_cache);
5272 
5273 	return err;
5274 }
5275 
5276 static int __cold dpaa2_caam_remove(struct fsl_mc_device *ls_dev)
5277 {
5278 	struct device *dev;
5279 	struct dpaa2_caam_priv *priv;
5280 	int i;
5281 
5282 	dev = &ls_dev->dev;
5283 	priv = dev_get_drvdata(dev);
5284 
5285 	dpaa2_dpseci_debugfs_exit(priv);
5286 
5287 	for (i = 0; i < ARRAY_SIZE(driver_aeads); i++) {
5288 		struct caam_aead_alg *t_alg = driver_aeads + i;
5289 
5290 		if (t_alg->registered)
5291 			crypto_unregister_aead(&t_alg->aead);
5292 	}
5293 
5294 	for (i = 0; i < ARRAY_SIZE(driver_algs); i++) {
5295 		struct caam_skcipher_alg *t_alg = driver_algs + i;
5296 
5297 		if (t_alg->registered)
5298 			crypto_unregister_skcipher(&t_alg->skcipher);
5299 	}
5300 
5301 	if (hash_list.next) {
5302 		struct caam_hash_alg *t_hash_alg, *p;
5303 
5304 		list_for_each_entry_safe(t_hash_alg, p, &hash_list, entry) {
5305 			crypto_unregister_ahash(&t_hash_alg->ahash_alg);
5306 			list_del(&t_hash_alg->entry);
5307 			kfree(t_hash_alg);
5308 		}
5309 	}
5310 
5311 	dpaa2_dpseci_disable(priv);
5312 	dpaa2_dpseci_dpio_free(priv);
5313 	dpaa2_dpseci_free(priv);
5314 	free_percpu(priv->ppriv);
5315 	fsl_mc_portal_free(priv->mc_io);
5316 	kmem_cache_destroy(qi_cache);
5317 
5318 	return 0;
5319 }
5320 
5321 int dpaa2_caam_enqueue(struct device *dev, struct caam_request *req)
5322 {
5323 	struct dpaa2_fd fd;
5324 	struct dpaa2_caam_priv *priv = dev_get_drvdata(dev);
5325 	struct dpaa2_caam_priv_per_cpu *ppriv;
5326 	int err = 0, i;
5327 
5328 	if (IS_ERR(req))
5329 		return PTR_ERR(req);
5330 
5331 	if (priv->cscn_mem) {
5332 		dma_sync_single_for_cpu(priv->dev, priv->cscn_dma,
5333 					DPAA2_CSCN_SIZE,
5334 					DMA_FROM_DEVICE);
5335 		if (unlikely(dpaa2_cscn_state_congested(priv->cscn_mem_aligned))) {
5336 			dev_dbg_ratelimited(dev, "Dropping request\n");
5337 			return -EBUSY;
5338 		}
5339 	}
5340 
5341 	dpaa2_fl_set_flc(&req->fd_flt[1], req->flc_dma);
5342 
5343 	req->fd_flt_dma = dma_map_single(dev, req->fd_flt, sizeof(req->fd_flt),
5344 					 DMA_BIDIRECTIONAL);
5345 	if (dma_mapping_error(dev, req->fd_flt_dma)) {
5346 		dev_err(dev, "DMA mapping error for QI enqueue request\n");
5347 		goto err_out;
5348 	}
5349 
5350 	memset(&fd, 0, sizeof(fd));
5351 	dpaa2_fd_set_format(&fd, dpaa2_fd_list);
5352 	dpaa2_fd_set_addr(&fd, req->fd_flt_dma);
5353 	dpaa2_fd_set_len(&fd, dpaa2_fl_get_len(&req->fd_flt[1]));
5354 	dpaa2_fd_set_flc(&fd, req->flc_dma);
5355 
5356 	ppriv = this_cpu_ptr(priv->ppriv);
5357 	for (i = 0; i < (priv->dpseci_attr.num_tx_queues << 1); i++) {
5358 		err = dpaa2_io_service_enqueue_fq(ppriv->dpio, ppriv->req_fqid,
5359 						  &fd);
5360 		if (err != -EBUSY)
5361 			break;
5362 
5363 		cpu_relax();
5364 	}
5365 
5366 	if (unlikely(err)) {
5367 		dev_err_ratelimited(dev, "Error enqueuing frame: %d\n", err);
5368 		goto err_out;
5369 	}
5370 
5371 	return -EINPROGRESS;
5372 
5373 err_out:
5374 	dma_unmap_single(dev, req->fd_flt_dma, sizeof(req->fd_flt),
5375 			 DMA_BIDIRECTIONAL);
5376 	return -EIO;
5377 }
5378 EXPORT_SYMBOL(dpaa2_caam_enqueue);
5379 
5380 static const struct fsl_mc_device_id dpaa2_caam_match_id_table[] = {
5381 	{
5382 		.vendor = FSL_MC_VENDOR_FREESCALE,
5383 		.obj_type = "dpseci",
5384 	},
5385 	{ .vendor = 0x0 }
5386 };
5387 
5388 static struct fsl_mc_driver dpaa2_caam_driver = {
5389 	.driver = {
5390 		.name		= KBUILD_MODNAME,
5391 		.owner		= THIS_MODULE,
5392 	},
5393 	.probe		= dpaa2_caam_probe,
5394 	.remove		= dpaa2_caam_remove,
5395 	.match_id_table = dpaa2_caam_match_id_table
5396 };
5397 
5398 MODULE_LICENSE("Dual BSD/GPL");
5399 MODULE_AUTHOR("Freescale Semiconductor, Inc");
5400 MODULE_DESCRIPTION("Freescale DPAA2 CAAM Driver");
5401 
5402 module_fsl_mc_driver(dpaa2_caam_driver);
5403