1 // SPDX-License-Identifier: (GPL-2.0+ OR BSD-3-Clause)
2 /*
3  * Copyright 2015-2016 Freescale Semiconductor Inc.
4  * Copyright 2017-2019 NXP
5  */
6 
7 #include "compat.h"
8 #include "regs.h"
9 #include "caamalg_qi2.h"
10 #include "dpseci_cmd.h"
11 #include "desc_constr.h"
12 #include "error.h"
13 #include "sg_sw_sec4.h"
14 #include "sg_sw_qm2.h"
15 #include "key_gen.h"
16 #include "caamalg_desc.h"
17 #include "caamhash_desc.h"
18 #include "dpseci-debugfs.h"
19 #include <linux/dma-mapping.h>
20 #include <linux/fsl/mc.h>
21 #include <linux/kernel.h>
22 #include <soc/fsl/dpaa2-io.h>
23 #include <soc/fsl/dpaa2-fd.h>
24 #include <crypto/xts.h>
25 #include <asm/unaligned.h>
26 
27 #define CAAM_CRA_PRIORITY	2000
28 
29 /* max key is sum of AES_MAX_KEY_SIZE, max split key size */
30 #define CAAM_MAX_KEY_SIZE	(AES_MAX_KEY_SIZE + CTR_RFC3686_NONCE_SIZE + \
31 				 SHA512_DIGEST_SIZE * 2)
32 
33 /*
34  * This is a cache of buffers, from which the users of CAAM QI driver
35  * can allocate short buffers. It's speedier than doing kmalloc on the hotpath.
36  * NOTE: A more elegant solution would be to have some headroom in the frames
37  *       being processed. This can be added by the dpaa2-eth driver. This would
38  *       pose a problem for userspace application processing which cannot
39  *       know of this limitation. So for now, this will work.
40  * NOTE: The memcache is SMP-safe. No need to handle spinlocks in-here
41  */
42 static struct kmem_cache *qi_cache;
43 
44 struct caam_alg_entry {
45 	struct device *dev;
46 	int class1_alg_type;
47 	int class2_alg_type;
48 	bool rfc3686;
49 	bool geniv;
50 	bool nodkp;
51 };
52 
53 struct caam_aead_alg {
54 	struct aead_alg aead;
55 	struct caam_alg_entry caam;
56 	bool registered;
57 };
58 
59 struct caam_skcipher_alg {
60 	struct skcipher_alg skcipher;
61 	struct caam_alg_entry caam;
62 	bool registered;
63 };
64 
65 /**
66  * struct caam_ctx - per-session context
67  * @flc: Flow Contexts array
68  * @key:  [authentication key], encryption key
69  * @flc_dma: I/O virtual addresses of the Flow Contexts
70  * @key_dma: I/O virtual address of the key
71  * @dir: DMA direction for mapping key and Flow Contexts
72  * @dev: dpseci device
73  * @adata: authentication algorithm details
74  * @cdata: encryption algorithm details
75  * @authsize: authentication tag (a.k.a. ICV / MAC) size
76  * @xts_key_fallback: true if fallback tfm needs to be used due
77  *		      to unsupported xts key lengths
78  * @fallback: xts fallback tfm
79  */
80 struct caam_ctx {
81 	struct caam_flc flc[NUM_OP];
82 	u8 key[CAAM_MAX_KEY_SIZE];
83 	dma_addr_t flc_dma[NUM_OP];
84 	dma_addr_t key_dma;
85 	enum dma_data_direction dir;
86 	struct device *dev;
87 	struct alginfo adata;
88 	struct alginfo cdata;
89 	unsigned int authsize;
90 	bool xts_key_fallback;
91 	struct crypto_skcipher *fallback;
92 };
93 
94 static void *dpaa2_caam_iova_to_virt(struct dpaa2_caam_priv *priv,
95 				     dma_addr_t iova_addr)
96 {
97 	phys_addr_t phys_addr;
98 
99 	phys_addr = priv->domain ? iommu_iova_to_phys(priv->domain, iova_addr) :
100 				   iova_addr;
101 
102 	return phys_to_virt(phys_addr);
103 }
104 
105 /*
106  * qi_cache_zalloc - Allocate buffers from CAAM-QI cache
107  *
108  * Allocate data on the hotpath. Instead of using kzalloc, one can use the
109  * services of the CAAM QI memory cache (backed by kmem_cache). The buffers
110  * will have a size of CAAM_QI_MEMCACHE_SIZE, which should be sufficient for
111  * hosting 16 SG entries.
112  *
113  * @flags - flags that would be used for the equivalent kmalloc(..) call
114  *
115  * Returns a pointer to a retrieved buffer on success or NULL on failure.
116  */
117 static inline void *qi_cache_zalloc(gfp_t flags)
118 {
119 	return kmem_cache_zalloc(qi_cache, flags);
120 }
121 
122 /*
123  * qi_cache_free - Frees buffers allocated from CAAM-QI cache
124  *
125  * @obj - buffer previously allocated by qi_cache_zalloc
126  *
127  * No checking is being done, the call is a passthrough call to
128  * kmem_cache_free(...)
129  */
130 static inline void qi_cache_free(void *obj)
131 {
132 	kmem_cache_free(qi_cache, obj);
133 }
134 
135 static struct caam_request *to_caam_req(struct crypto_async_request *areq)
136 {
137 	switch (crypto_tfm_alg_type(areq->tfm)) {
138 	case CRYPTO_ALG_TYPE_SKCIPHER:
139 		return skcipher_request_ctx_dma(skcipher_request_cast(areq));
140 	case CRYPTO_ALG_TYPE_AEAD:
141 		return aead_request_ctx_dma(
142 			container_of(areq, struct aead_request, base));
143 	case CRYPTO_ALG_TYPE_AHASH:
144 		return ahash_request_ctx_dma(ahash_request_cast(areq));
145 	default:
146 		return ERR_PTR(-EINVAL);
147 	}
148 }
149 
150 static void caam_unmap(struct device *dev, struct scatterlist *src,
151 		       struct scatterlist *dst, int src_nents,
152 		       int dst_nents, dma_addr_t iv_dma, int ivsize,
153 		       enum dma_data_direction iv_dir, dma_addr_t qm_sg_dma,
154 		       int qm_sg_bytes)
155 {
156 	if (dst != src) {
157 		if (src_nents)
158 			dma_unmap_sg(dev, src, src_nents, DMA_TO_DEVICE);
159 		if (dst_nents)
160 			dma_unmap_sg(dev, dst, dst_nents, DMA_FROM_DEVICE);
161 	} else {
162 		dma_unmap_sg(dev, src, src_nents, DMA_BIDIRECTIONAL);
163 	}
164 
165 	if (iv_dma)
166 		dma_unmap_single(dev, iv_dma, ivsize, iv_dir);
167 
168 	if (qm_sg_bytes)
169 		dma_unmap_single(dev, qm_sg_dma, qm_sg_bytes, DMA_TO_DEVICE);
170 }
171 
172 static int aead_set_sh_desc(struct crypto_aead *aead)
173 {
174 	struct caam_aead_alg *alg = container_of(crypto_aead_alg(aead),
175 						 typeof(*alg), aead);
176 	struct caam_ctx *ctx = crypto_aead_ctx_dma(aead);
177 	unsigned int ivsize = crypto_aead_ivsize(aead);
178 	struct device *dev = ctx->dev;
179 	struct dpaa2_caam_priv *priv = dev_get_drvdata(dev);
180 	struct caam_flc *flc;
181 	u32 *desc;
182 	u32 ctx1_iv_off = 0;
183 	u32 *nonce = NULL;
184 	unsigned int data_len[2];
185 	u32 inl_mask;
186 	const bool ctr_mode = ((ctx->cdata.algtype & OP_ALG_AAI_MASK) ==
187 			       OP_ALG_AAI_CTR_MOD128);
188 	const bool is_rfc3686 = alg->caam.rfc3686;
189 
190 	if (!ctx->cdata.keylen || !ctx->authsize)
191 		return 0;
192 
193 	/*
194 	 * AES-CTR needs to load IV in CONTEXT1 reg
195 	 * at an offset of 128bits (16bytes)
196 	 * CONTEXT1[255:128] = IV
197 	 */
198 	if (ctr_mode)
199 		ctx1_iv_off = 16;
200 
201 	/*
202 	 * RFC3686 specific:
203 	 *	CONTEXT1[255:128] = {NONCE, IV, COUNTER}
204 	 */
205 	if (is_rfc3686) {
206 		ctx1_iv_off = 16 + CTR_RFC3686_NONCE_SIZE;
207 		nonce = (u32 *)((void *)ctx->key + ctx->adata.keylen_pad +
208 				ctx->cdata.keylen - CTR_RFC3686_NONCE_SIZE);
209 	}
210 
211 	/*
212 	 * In case |user key| > |derived key|, using DKP<imm,imm> would result
213 	 * in invalid opcodes (last bytes of user key) in the resulting
214 	 * descriptor. Use DKP<ptr,imm> instead => both virtual and dma key
215 	 * addresses are needed.
216 	 */
217 	ctx->adata.key_virt = ctx->key;
218 	ctx->adata.key_dma = ctx->key_dma;
219 
220 	ctx->cdata.key_virt = ctx->key + ctx->adata.keylen_pad;
221 	ctx->cdata.key_dma = ctx->key_dma + ctx->adata.keylen_pad;
222 
223 	data_len[0] = ctx->adata.keylen_pad;
224 	data_len[1] = ctx->cdata.keylen;
225 
226 	/* aead_encrypt shared descriptor */
227 	if (desc_inline_query((alg->caam.geniv ? DESC_QI_AEAD_GIVENC_LEN :
228 						 DESC_QI_AEAD_ENC_LEN) +
229 			      (is_rfc3686 ? DESC_AEAD_CTR_RFC3686_LEN : 0),
230 			      DESC_JOB_IO_LEN, data_len, &inl_mask,
231 			      ARRAY_SIZE(data_len)) < 0)
232 		return -EINVAL;
233 
234 	ctx->adata.key_inline = !!(inl_mask & 1);
235 	ctx->cdata.key_inline = !!(inl_mask & 2);
236 
237 	flc = &ctx->flc[ENCRYPT];
238 	desc = flc->sh_desc;
239 
240 	if (alg->caam.geniv)
241 		cnstr_shdsc_aead_givencap(desc, &ctx->cdata, &ctx->adata,
242 					  ivsize, ctx->authsize, is_rfc3686,
243 					  nonce, ctx1_iv_off, true,
244 					  priv->sec_attr.era);
245 	else
246 		cnstr_shdsc_aead_encap(desc, &ctx->cdata, &ctx->adata,
247 				       ivsize, ctx->authsize, is_rfc3686, nonce,
248 				       ctx1_iv_off, true, priv->sec_attr.era);
249 
250 	flc->flc[1] = cpu_to_caam32(desc_len(desc)); /* SDL */
251 	dma_sync_single_for_device(dev, ctx->flc_dma[ENCRYPT],
252 				   sizeof(flc->flc) + desc_bytes(desc),
253 				   ctx->dir);
254 
255 	/* aead_decrypt shared descriptor */
256 	if (desc_inline_query(DESC_QI_AEAD_DEC_LEN +
257 			      (is_rfc3686 ? DESC_AEAD_CTR_RFC3686_LEN : 0),
258 			      DESC_JOB_IO_LEN, data_len, &inl_mask,
259 			      ARRAY_SIZE(data_len)) < 0)
260 		return -EINVAL;
261 
262 	ctx->adata.key_inline = !!(inl_mask & 1);
263 	ctx->cdata.key_inline = !!(inl_mask & 2);
264 
265 	flc = &ctx->flc[DECRYPT];
266 	desc = flc->sh_desc;
267 	cnstr_shdsc_aead_decap(desc, &ctx->cdata, &ctx->adata,
268 			       ivsize, ctx->authsize, alg->caam.geniv,
269 			       is_rfc3686, nonce, ctx1_iv_off, true,
270 			       priv->sec_attr.era);
271 	flc->flc[1] = cpu_to_caam32(desc_len(desc)); /* SDL */
272 	dma_sync_single_for_device(dev, ctx->flc_dma[DECRYPT],
273 				   sizeof(flc->flc) + desc_bytes(desc),
274 				   ctx->dir);
275 
276 	return 0;
277 }
278 
279 static int aead_setauthsize(struct crypto_aead *authenc, unsigned int authsize)
280 {
281 	struct caam_ctx *ctx = crypto_aead_ctx_dma(authenc);
282 
283 	ctx->authsize = authsize;
284 	aead_set_sh_desc(authenc);
285 
286 	return 0;
287 }
288 
289 static int aead_setkey(struct crypto_aead *aead, const u8 *key,
290 		       unsigned int keylen)
291 {
292 	struct caam_ctx *ctx = crypto_aead_ctx_dma(aead);
293 	struct device *dev = ctx->dev;
294 	struct crypto_authenc_keys keys;
295 
296 	if (crypto_authenc_extractkeys(&keys, key, keylen) != 0)
297 		goto badkey;
298 
299 	dev_dbg(dev, "keylen %d enckeylen %d authkeylen %d\n",
300 		keys.authkeylen + keys.enckeylen, keys.enckeylen,
301 		keys.authkeylen);
302 	print_hex_dump_debug("key in @" __stringify(__LINE__)": ",
303 			     DUMP_PREFIX_ADDRESS, 16, 4, key, keylen, 1);
304 
305 	ctx->adata.keylen = keys.authkeylen;
306 	ctx->adata.keylen_pad = split_key_len(ctx->adata.algtype &
307 					      OP_ALG_ALGSEL_MASK);
308 
309 	if (ctx->adata.keylen_pad + keys.enckeylen > CAAM_MAX_KEY_SIZE)
310 		goto badkey;
311 
312 	memcpy(ctx->key, keys.authkey, keys.authkeylen);
313 	memcpy(ctx->key + ctx->adata.keylen_pad, keys.enckey, keys.enckeylen);
314 	dma_sync_single_for_device(dev, ctx->key_dma, ctx->adata.keylen_pad +
315 				   keys.enckeylen, ctx->dir);
316 	print_hex_dump_debug("ctx.key@" __stringify(__LINE__)": ",
317 			     DUMP_PREFIX_ADDRESS, 16, 4, ctx->key,
318 			     ctx->adata.keylen_pad + keys.enckeylen, 1);
319 
320 	ctx->cdata.keylen = keys.enckeylen;
321 
322 	memzero_explicit(&keys, sizeof(keys));
323 	return aead_set_sh_desc(aead);
324 badkey:
325 	memzero_explicit(&keys, sizeof(keys));
326 	return -EINVAL;
327 }
328 
329 static int des3_aead_setkey(struct crypto_aead *aead, const u8 *key,
330 			    unsigned int keylen)
331 {
332 	struct crypto_authenc_keys keys;
333 	int err;
334 
335 	err = crypto_authenc_extractkeys(&keys, key, keylen);
336 	if (unlikely(err))
337 		goto out;
338 
339 	err = -EINVAL;
340 	if (keys.enckeylen != DES3_EDE_KEY_SIZE)
341 		goto out;
342 
343 	err = crypto_des3_ede_verify_key(crypto_aead_tfm(aead), keys.enckey) ?:
344 	      aead_setkey(aead, key, keylen);
345 
346 out:
347 	memzero_explicit(&keys, sizeof(keys));
348 	return err;
349 }
350 
351 static struct aead_edesc *aead_edesc_alloc(struct aead_request *req,
352 					   bool encrypt)
353 {
354 	struct crypto_aead *aead = crypto_aead_reqtfm(req);
355 	struct caam_request *req_ctx = aead_request_ctx_dma(req);
356 	struct dpaa2_fl_entry *in_fle = &req_ctx->fd_flt[1];
357 	struct dpaa2_fl_entry *out_fle = &req_ctx->fd_flt[0];
358 	struct caam_ctx *ctx = crypto_aead_ctx_dma(aead);
359 	struct caam_aead_alg *alg = container_of(crypto_aead_alg(aead),
360 						 typeof(*alg), aead);
361 	struct device *dev = ctx->dev;
362 	gfp_t flags = (req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP) ?
363 		      GFP_KERNEL : GFP_ATOMIC;
364 	int src_nents, mapped_src_nents, dst_nents = 0, mapped_dst_nents = 0;
365 	int src_len, dst_len = 0;
366 	struct aead_edesc *edesc;
367 	dma_addr_t qm_sg_dma, iv_dma = 0;
368 	int ivsize = 0;
369 	unsigned int authsize = ctx->authsize;
370 	int qm_sg_index = 0, qm_sg_nents = 0, qm_sg_bytes;
371 	int in_len, out_len;
372 	struct dpaa2_sg_entry *sg_table;
373 
374 	/* allocate space for base edesc, link tables and IV */
375 	edesc = qi_cache_zalloc(flags);
376 	if (unlikely(!edesc)) {
377 		dev_err(dev, "could not allocate extended descriptor\n");
378 		return ERR_PTR(-ENOMEM);
379 	}
380 
381 	if (unlikely(req->dst != req->src)) {
382 		src_len = req->assoclen + req->cryptlen;
383 		dst_len = src_len + (encrypt ? authsize : (-authsize));
384 
385 		src_nents = sg_nents_for_len(req->src, src_len);
386 		if (unlikely(src_nents < 0)) {
387 			dev_err(dev, "Insufficient bytes (%d) in src S/G\n",
388 				src_len);
389 			qi_cache_free(edesc);
390 			return ERR_PTR(src_nents);
391 		}
392 
393 		dst_nents = sg_nents_for_len(req->dst, dst_len);
394 		if (unlikely(dst_nents < 0)) {
395 			dev_err(dev, "Insufficient bytes (%d) in dst S/G\n",
396 				dst_len);
397 			qi_cache_free(edesc);
398 			return ERR_PTR(dst_nents);
399 		}
400 
401 		if (src_nents) {
402 			mapped_src_nents = dma_map_sg(dev, req->src, src_nents,
403 						      DMA_TO_DEVICE);
404 			if (unlikely(!mapped_src_nents)) {
405 				dev_err(dev, "unable to map source\n");
406 				qi_cache_free(edesc);
407 				return ERR_PTR(-ENOMEM);
408 			}
409 		} else {
410 			mapped_src_nents = 0;
411 		}
412 
413 		if (dst_nents) {
414 			mapped_dst_nents = dma_map_sg(dev, req->dst, dst_nents,
415 						      DMA_FROM_DEVICE);
416 			if (unlikely(!mapped_dst_nents)) {
417 				dev_err(dev, "unable to map destination\n");
418 				dma_unmap_sg(dev, req->src, src_nents,
419 					     DMA_TO_DEVICE);
420 				qi_cache_free(edesc);
421 				return ERR_PTR(-ENOMEM);
422 			}
423 		} else {
424 			mapped_dst_nents = 0;
425 		}
426 	} else {
427 		src_len = req->assoclen + req->cryptlen +
428 			  (encrypt ? authsize : 0);
429 
430 		src_nents = sg_nents_for_len(req->src, src_len);
431 		if (unlikely(src_nents < 0)) {
432 			dev_err(dev, "Insufficient bytes (%d) in src S/G\n",
433 				src_len);
434 			qi_cache_free(edesc);
435 			return ERR_PTR(src_nents);
436 		}
437 
438 		mapped_src_nents = dma_map_sg(dev, req->src, src_nents,
439 					      DMA_BIDIRECTIONAL);
440 		if (unlikely(!mapped_src_nents)) {
441 			dev_err(dev, "unable to map source\n");
442 			qi_cache_free(edesc);
443 			return ERR_PTR(-ENOMEM);
444 		}
445 	}
446 
447 	if ((alg->caam.rfc3686 && encrypt) || !alg->caam.geniv)
448 		ivsize = crypto_aead_ivsize(aead);
449 
450 	/*
451 	 * Create S/G table: req->assoclen, [IV,] req->src [, req->dst].
452 	 * Input is not contiguous.
453 	 * HW reads 4 S/G entries at a time; make sure the reads don't go beyond
454 	 * the end of the table by allocating more S/G entries. Logic:
455 	 * if (src != dst && output S/G)
456 	 *      pad output S/G, if needed
457 	 * else if (src == dst && S/G)
458 	 *      overlapping S/Gs; pad one of them
459 	 * else if (input S/G) ...
460 	 *      pad input S/G, if needed
461 	 */
462 	qm_sg_nents = 1 + !!ivsize + mapped_src_nents;
463 	if (mapped_dst_nents > 1)
464 		qm_sg_nents += pad_sg_nents(mapped_dst_nents);
465 	else if ((req->src == req->dst) && (mapped_src_nents > 1))
466 		qm_sg_nents = max(pad_sg_nents(qm_sg_nents),
467 				  1 + !!ivsize +
468 				  pad_sg_nents(mapped_src_nents));
469 	else
470 		qm_sg_nents = pad_sg_nents(qm_sg_nents);
471 
472 	sg_table = &edesc->sgt[0];
473 	qm_sg_bytes = qm_sg_nents * sizeof(*sg_table);
474 	if (unlikely(offsetof(struct aead_edesc, sgt) + qm_sg_bytes + ivsize >
475 		     CAAM_QI_MEMCACHE_SIZE)) {
476 		dev_err(dev, "No space for %d S/G entries and/or %dB IV\n",
477 			qm_sg_nents, ivsize);
478 		caam_unmap(dev, req->src, req->dst, src_nents, dst_nents, 0,
479 			   0, DMA_NONE, 0, 0);
480 		qi_cache_free(edesc);
481 		return ERR_PTR(-ENOMEM);
482 	}
483 
484 	if (ivsize) {
485 		u8 *iv = (u8 *)(sg_table + qm_sg_nents);
486 
487 		/* Make sure IV is located in a DMAable area */
488 		memcpy(iv, req->iv, ivsize);
489 
490 		iv_dma = dma_map_single(dev, iv, ivsize, DMA_TO_DEVICE);
491 		if (dma_mapping_error(dev, iv_dma)) {
492 			dev_err(dev, "unable to map IV\n");
493 			caam_unmap(dev, req->src, req->dst, src_nents,
494 				   dst_nents, 0, 0, DMA_NONE, 0, 0);
495 			qi_cache_free(edesc);
496 			return ERR_PTR(-ENOMEM);
497 		}
498 	}
499 
500 	edesc->src_nents = src_nents;
501 	edesc->dst_nents = dst_nents;
502 	edesc->iv_dma = iv_dma;
503 
504 	if ((alg->caam.class1_alg_type & OP_ALG_ALGSEL_MASK) ==
505 	    OP_ALG_ALGSEL_CHACHA20 && ivsize != CHACHAPOLY_IV_SIZE)
506 		/*
507 		 * The associated data comes already with the IV but we need
508 		 * to skip it when we authenticate or encrypt...
509 		 */
510 		edesc->assoclen = cpu_to_caam32(req->assoclen - ivsize);
511 	else
512 		edesc->assoclen = cpu_to_caam32(req->assoclen);
513 	edesc->assoclen_dma = dma_map_single(dev, &edesc->assoclen, 4,
514 					     DMA_TO_DEVICE);
515 	if (dma_mapping_error(dev, edesc->assoclen_dma)) {
516 		dev_err(dev, "unable to map assoclen\n");
517 		caam_unmap(dev, req->src, req->dst, src_nents, dst_nents,
518 			   iv_dma, ivsize, DMA_TO_DEVICE, 0, 0);
519 		qi_cache_free(edesc);
520 		return ERR_PTR(-ENOMEM);
521 	}
522 
523 	dma_to_qm_sg_one(sg_table, edesc->assoclen_dma, 4, 0);
524 	qm_sg_index++;
525 	if (ivsize) {
526 		dma_to_qm_sg_one(sg_table + qm_sg_index, iv_dma, ivsize, 0);
527 		qm_sg_index++;
528 	}
529 	sg_to_qm_sg_last(req->src, src_len, sg_table + qm_sg_index, 0);
530 	qm_sg_index += mapped_src_nents;
531 
532 	if (mapped_dst_nents > 1)
533 		sg_to_qm_sg_last(req->dst, dst_len, sg_table + qm_sg_index, 0);
534 
535 	qm_sg_dma = dma_map_single(dev, sg_table, qm_sg_bytes, DMA_TO_DEVICE);
536 	if (dma_mapping_error(dev, qm_sg_dma)) {
537 		dev_err(dev, "unable to map S/G table\n");
538 		dma_unmap_single(dev, edesc->assoclen_dma, 4, DMA_TO_DEVICE);
539 		caam_unmap(dev, req->src, req->dst, src_nents, dst_nents,
540 			   iv_dma, ivsize, DMA_TO_DEVICE, 0, 0);
541 		qi_cache_free(edesc);
542 		return ERR_PTR(-ENOMEM);
543 	}
544 
545 	edesc->qm_sg_dma = qm_sg_dma;
546 	edesc->qm_sg_bytes = qm_sg_bytes;
547 
548 	out_len = req->assoclen + req->cryptlen +
549 		  (encrypt ? ctx->authsize : (-ctx->authsize));
550 	in_len = 4 + ivsize + req->assoclen + req->cryptlen;
551 
552 	memset(&req_ctx->fd_flt, 0, sizeof(req_ctx->fd_flt));
553 	dpaa2_fl_set_final(in_fle, true);
554 	dpaa2_fl_set_format(in_fle, dpaa2_fl_sg);
555 	dpaa2_fl_set_addr(in_fle, qm_sg_dma);
556 	dpaa2_fl_set_len(in_fle, in_len);
557 
558 	if (req->dst == req->src) {
559 		if (mapped_src_nents == 1) {
560 			dpaa2_fl_set_format(out_fle, dpaa2_fl_single);
561 			dpaa2_fl_set_addr(out_fle, sg_dma_address(req->src));
562 		} else {
563 			dpaa2_fl_set_format(out_fle, dpaa2_fl_sg);
564 			dpaa2_fl_set_addr(out_fle, qm_sg_dma +
565 					  (1 + !!ivsize) * sizeof(*sg_table));
566 		}
567 	} else if (!mapped_dst_nents) {
568 		/*
569 		 * crypto engine requires the output entry to be present when
570 		 * "frame list" FD is used.
571 		 * Since engine does not support FMT=2'b11 (unused entry type),
572 		 * leaving out_fle zeroized is the best option.
573 		 */
574 		goto skip_out_fle;
575 	} else if (mapped_dst_nents == 1) {
576 		dpaa2_fl_set_format(out_fle, dpaa2_fl_single);
577 		dpaa2_fl_set_addr(out_fle, sg_dma_address(req->dst));
578 	} else {
579 		dpaa2_fl_set_format(out_fle, dpaa2_fl_sg);
580 		dpaa2_fl_set_addr(out_fle, qm_sg_dma + qm_sg_index *
581 				  sizeof(*sg_table));
582 	}
583 
584 	dpaa2_fl_set_len(out_fle, out_len);
585 
586 skip_out_fle:
587 	return edesc;
588 }
589 
590 static int chachapoly_set_sh_desc(struct crypto_aead *aead)
591 {
592 	struct caam_ctx *ctx = crypto_aead_ctx_dma(aead);
593 	unsigned int ivsize = crypto_aead_ivsize(aead);
594 	struct device *dev = ctx->dev;
595 	struct caam_flc *flc;
596 	u32 *desc;
597 
598 	if (!ctx->cdata.keylen || !ctx->authsize)
599 		return 0;
600 
601 	flc = &ctx->flc[ENCRYPT];
602 	desc = flc->sh_desc;
603 	cnstr_shdsc_chachapoly(desc, &ctx->cdata, &ctx->adata, ivsize,
604 			       ctx->authsize, true, true);
605 	flc->flc[1] = cpu_to_caam32(desc_len(desc)); /* SDL */
606 	dma_sync_single_for_device(dev, ctx->flc_dma[ENCRYPT],
607 				   sizeof(flc->flc) + desc_bytes(desc),
608 				   ctx->dir);
609 
610 	flc = &ctx->flc[DECRYPT];
611 	desc = flc->sh_desc;
612 	cnstr_shdsc_chachapoly(desc, &ctx->cdata, &ctx->adata, ivsize,
613 			       ctx->authsize, false, true);
614 	flc->flc[1] = cpu_to_caam32(desc_len(desc)); /* SDL */
615 	dma_sync_single_for_device(dev, ctx->flc_dma[DECRYPT],
616 				   sizeof(flc->flc) + desc_bytes(desc),
617 				   ctx->dir);
618 
619 	return 0;
620 }
621 
622 static int chachapoly_setauthsize(struct crypto_aead *aead,
623 				  unsigned int authsize)
624 {
625 	struct caam_ctx *ctx = crypto_aead_ctx_dma(aead);
626 
627 	if (authsize != POLY1305_DIGEST_SIZE)
628 		return -EINVAL;
629 
630 	ctx->authsize = authsize;
631 	return chachapoly_set_sh_desc(aead);
632 }
633 
634 static int chachapoly_setkey(struct crypto_aead *aead, const u8 *key,
635 			     unsigned int keylen)
636 {
637 	struct caam_ctx *ctx = crypto_aead_ctx_dma(aead);
638 	unsigned int ivsize = crypto_aead_ivsize(aead);
639 	unsigned int saltlen = CHACHAPOLY_IV_SIZE - ivsize;
640 
641 	if (keylen != CHACHA_KEY_SIZE + saltlen)
642 		return -EINVAL;
643 
644 	memcpy(ctx->key, key, keylen);
645 	ctx->cdata.key_virt = ctx->key;
646 	ctx->cdata.keylen = keylen - saltlen;
647 
648 	return chachapoly_set_sh_desc(aead);
649 }
650 
651 static int gcm_set_sh_desc(struct crypto_aead *aead)
652 {
653 	struct caam_ctx *ctx = crypto_aead_ctx_dma(aead);
654 	struct device *dev = ctx->dev;
655 	unsigned int ivsize = crypto_aead_ivsize(aead);
656 	struct caam_flc *flc;
657 	u32 *desc;
658 	int rem_bytes = CAAM_DESC_BYTES_MAX - DESC_JOB_IO_LEN -
659 			ctx->cdata.keylen;
660 
661 	if (!ctx->cdata.keylen || !ctx->authsize)
662 		return 0;
663 
664 	/*
665 	 * AES GCM encrypt shared descriptor
666 	 * Job Descriptor and Shared Descriptor
667 	 * must fit into the 64-word Descriptor h/w Buffer
668 	 */
669 	if (rem_bytes >= DESC_QI_GCM_ENC_LEN) {
670 		ctx->cdata.key_inline = true;
671 		ctx->cdata.key_virt = ctx->key;
672 	} else {
673 		ctx->cdata.key_inline = false;
674 		ctx->cdata.key_dma = ctx->key_dma;
675 	}
676 
677 	flc = &ctx->flc[ENCRYPT];
678 	desc = flc->sh_desc;
679 	cnstr_shdsc_gcm_encap(desc, &ctx->cdata, ivsize, ctx->authsize, true);
680 	flc->flc[1] = cpu_to_caam32(desc_len(desc)); /* SDL */
681 	dma_sync_single_for_device(dev, ctx->flc_dma[ENCRYPT],
682 				   sizeof(flc->flc) + desc_bytes(desc),
683 				   ctx->dir);
684 
685 	/*
686 	 * Job Descriptor and Shared Descriptors
687 	 * must all fit into the 64-word Descriptor h/w Buffer
688 	 */
689 	if (rem_bytes >= DESC_QI_GCM_DEC_LEN) {
690 		ctx->cdata.key_inline = true;
691 		ctx->cdata.key_virt = ctx->key;
692 	} else {
693 		ctx->cdata.key_inline = false;
694 		ctx->cdata.key_dma = ctx->key_dma;
695 	}
696 
697 	flc = &ctx->flc[DECRYPT];
698 	desc = flc->sh_desc;
699 	cnstr_shdsc_gcm_decap(desc, &ctx->cdata, ivsize, ctx->authsize, true);
700 	flc->flc[1] = cpu_to_caam32(desc_len(desc)); /* SDL */
701 	dma_sync_single_for_device(dev, ctx->flc_dma[DECRYPT],
702 				   sizeof(flc->flc) + desc_bytes(desc),
703 				   ctx->dir);
704 
705 	return 0;
706 }
707 
708 static int gcm_setauthsize(struct crypto_aead *authenc, unsigned int authsize)
709 {
710 	struct caam_ctx *ctx = crypto_aead_ctx_dma(authenc);
711 	int err;
712 
713 	err = crypto_gcm_check_authsize(authsize);
714 	if (err)
715 		return err;
716 
717 	ctx->authsize = authsize;
718 	gcm_set_sh_desc(authenc);
719 
720 	return 0;
721 }
722 
723 static int gcm_setkey(struct crypto_aead *aead,
724 		      const u8 *key, unsigned int keylen)
725 {
726 	struct caam_ctx *ctx = crypto_aead_ctx_dma(aead);
727 	struct device *dev = ctx->dev;
728 	int ret;
729 
730 	ret = aes_check_keylen(keylen);
731 	if (ret)
732 		return ret;
733 	print_hex_dump_debug("key in @" __stringify(__LINE__)": ",
734 			     DUMP_PREFIX_ADDRESS, 16, 4, key, keylen, 1);
735 
736 	memcpy(ctx->key, key, keylen);
737 	dma_sync_single_for_device(dev, ctx->key_dma, keylen, ctx->dir);
738 	ctx->cdata.keylen = keylen;
739 
740 	return gcm_set_sh_desc(aead);
741 }
742 
743 static int rfc4106_set_sh_desc(struct crypto_aead *aead)
744 {
745 	struct caam_ctx *ctx = crypto_aead_ctx_dma(aead);
746 	struct device *dev = ctx->dev;
747 	unsigned int ivsize = crypto_aead_ivsize(aead);
748 	struct caam_flc *flc;
749 	u32 *desc;
750 	int rem_bytes = CAAM_DESC_BYTES_MAX - DESC_JOB_IO_LEN -
751 			ctx->cdata.keylen;
752 
753 	if (!ctx->cdata.keylen || !ctx->authsize)
754 		return 0;
755 
756 	ctx->cdata.key_virt = ctx->key;
757 
758 	/*
759 	 * RFC4106 encrypt shared descriptor
760 	 * Job Descriptor and Shared Descriptor
761 	 * must fit into the 64-word Descriptor h/w Buffer
762 	 */
763 	if (rem_bytes >= DESC_QI_RFC4106_ENC_LEN) {
764 		ctx->cdata.key_inline = true;
765 	} else {
766 		ctx->cdata.key_inline = false;
767 		ctx->cdata.key_dma = ctx->key_dma;
768 	}
769 
770 	flc = &ctx->flc[ENCRYPT];
771 	desc = flc->sh_desc;
772 	cnstr_shdsc_rfc4106_encap(desc, &ctx->cdata, ivsize, ctx->authsize,
773 				  true);
774 	flc->flc[1] = cpu_to_caam32(desc_len(desc)); /* SDL */
775 	dma_sync_single_for_device(dev, ctx->flc_dma[ENCRYPT],
776 				   sizeof(flc->flc) + desc_bytes(desc),
777 				   ctx->dir);
778 
779 	/*
780 	 * Job Descriptor and Shared Descriptors
781 	 * must all fit into the 64-word Descriptor h/w Buffer
782 	 */
783 	if (rem_bytes >= DESC_QI_RFC4106_DEC_LEN) {
784 		ctx->cdata.key_inline = true;
785 	} else {
786 		ctx->cdata.key_inline = false;
787 		ctx->cdata.key_dma = ctx->key_dma;
788 	}
789 
790 	flc = &ctx->flc[DECRYPT];
791 	desc = flc->sh_desc;
792 	cnstr_shdsc_rfc4106_decap(desc, &ctx->cdata, ivsize, ctx->authsize,
793 				  true);
794 	flc->flc[1] = cpu_to_caam32(desc_len(desc)); /* SDL */
795 	dma_sync_single_for_device(dev, ctx->flc_dma[DECRYPT],
796 				   sizeof(flc->flc) + desc_bytes(desc),
797 				   ctx->dir);
798 
799 	return 0;
800 }
801 
802 static int rfc4106_setauthsize(struct crypto_aead *authenc,
803 			       unsigned int authsize)
804 {
805 	struct caam_ctx *ctx = crypto_aead_ctx_dma(authenc);
806 	int err;
807 
808 	err = crypto_rfc4106_check_authsize(authsize);
809 	if (err)
810 		return err;
811 
812 	ctx->authsize = authsize;
813 	rfc4106_set_sh_desc(authenc);
814 
815 	return 0;
816 }
817 
818 static int rfc4106_setkey(struct crypto_aead *aead,
819 			  const u8 *key, unsigned int keylen)
820 {
821 	struct caam_ctx *ctx = crypto_aead_ctx_dma(aead);
822 	struct device *dev = ctx->dev;
823 	int ret;
824 
825 	ret = aes_check_keylen(keylen - 4);
826 	if (ret)
827 		return ret;
828 
829 	print_hex_dump_debug("key in @" __stringify(__LINE__)": ",
830 			     DUMP_PREFIX_ADDRESS, 16, 4, key, keylen, 1);
831 
832 	memcpy(ctx->key, key, keylen);
833 	/*
834 	 * The last four bytes of the key material are used as the salt value
835 	 * in the nonce. Update the AES key length.
836 	 */
837 	ctx->cdata.keylen = keylen - 4;
838 	dma_sync_single_for_device(dev, ctx->key_dma, ctx->cdata.keylen,
839 				   ctx->dir);
840 
841 	return rfc4106_set_sh_desc(aead);
842 }
843 
844 static int rfc4543_set_sh_desc(struct crypto_aead *aead)
845 {
846 	struct caam_ctx *ctx = crypto_aead_ctx_dma(aead);
847 	struct device *dev = ctx->dev;
848 	unsigned int ivsize = crypto_aead_ivsize(aead);
849 	struct caam_flc *flc;
850 	u32 *desc;
851 	int rem_bytes = CAAM_DESC_BYTES_MAX - DESC_JOB_IO_LEN -
852 			ctx->cdata.keylen;
853 
854 	if (!ctx->cdata.keylen || !ctx->authsize)
855 		return 0;
856 
857 	ctx->cdata.key_virt = ctx->key;
858 
859 	/*
860 	 * RFC4543 encrypt shared descriptor
861 	 * Job Descriptor and Shared Descriptor
862 	 * must fit into the 64-word Descriptor h/w Buffer
863 	 */
864 	if (rem_bytes >= DESC_QI_RFC4543_ENC_LEN) {
865 		ctx->cdata.key_inline = true;
866 	} else {
867 		ctx->cdata.key_inline = false;
868 		ctx->cdata.key_dma = ctx->key_dma;
869 	}
870 
871 	flc = &ctx->flc[ENCRYPT];
872 	desc = flc->sh_desc;
873 	cnstr_shdsc_rfc4543_encap(desc, &ctx->cdata, ivsize, ctx->authsize,
874 				  true);
875 	flc->flc[1] = cpu_to_caam32(desc_len(desc)); /* SDL */
876 	dma_sync_single_for_device(dev, ctx->flc_dma[ENCRYPT],
877 				   sizeof(flc->flc) + desc_bytes(desc),
878 				   ctx->dir);
879 
880 	/*
881 	 * Job Descriptor and Shared Descriptors
882 	 * must all fit into the 64-word Descriptor h/w Buffer
883 	 */
884 	if (rem_bytes >= DESC_QI_RFC4543_DEC_LEN) {
885 		ctx->cdata.key_inline = true;
886 	} else {
887 		ctx->cdata.key_inline = false;
888 		ctx->cdata.key_dma = ctx->key_dma;
889 	}
890 
891 	flc = &ctx->flc[DECRYPT];
892 	desc = flc->sh_desc;
893 	cnstr_shdsc_rfc4543_decap(desc, &ctx->cdata, ivsize, ctx->authsize,
894 				  true);
895 	flc->flc[1] = cpu_to_caam32(desc_len(desc)); /* SDL */
896 	dma_sync_single_for_device(dev, ctx->flc_dma[DECRYPT],
897 				   sizeof(flc->flc) + desc_bytes(desc),
898 				   ctx->dir);
899 
900 	return 0;
901 }
902 
903 static int rfc4543_setauthsize(struct crypto_aead *authenc,
904 			       unsigned int authsize)
905 {
906 	struct caam_ctx *ctx = crypto_aead_ctx_dma(authenc);
907 
908 	if (authsize != 16)
909 		return -EINVAL;
910 
911 	ctx->authsize = authsize;
912 	rfc4543_set_sh_desc(authenc);
913 
914 	return 0;
915 }
916 
917 static int rfc4543_setkey(struct crypto_aead *aead,
918 			  const u8 *key, unsigned int keylen)
919 {
920 	struct caam_ctx *ctx = crypto_aead_ctx_dma(aead);
921 	struct device *dev = ctx->dev;
922 	int ret;
923 
924 	ret = aes_check_keylen(keylen - 4);
925 	if (ret)
926 		return ret;
927 
928 	print_hex_dump_debug("key in @" __stringify(__LINE__)": ",
929 			     DUMP_PREFIX_ADDRESS, 16, 4, key, keylen, 1);
930 
931 	memcpy(ctx->key, key, keylen);
932 	/*
933 	 * The last four bytes of the key material are used as the salt value
934 	 * in the nonce. Update the AES key length.
935 	 */
936 	ctx->cdata.keylen = keylen - 4;
937 	dma_sync_single_for_device(dev, ctx->key_dma, ctx->cdata.keylen,
938 				   ctx->dir);
939 
940 	return rfc4543_set_sh_desc(aead);
941 }
942 
943 static int skcipher_setkey(struct crypto_skcipher *skcipher, const u8 *key,
944 			   unsigned int keylen, const u32 ctx1_iv_off)
945 {
946 	struct caam_ctx *ctx = crypto_skcipher_ctx_dma(skcipher);
947 	struct caam_skcipher_alg *alg =
948 		container_of(crypto_skcipher_alg(skcipher),
949 			     struct caam_skcipher_alg, skcipher);
950 	struct device *dev = ctx->dev;
951 	struct caam_flc *flc;
952 	unsigned int ivsize = crypto_skcipher_ivsize(skcipher);
953 	u32 *desc;
954 	const bool is_rfc3686 = alg->caam.rfc3686;
955 
956 	print_hex_dump_debug("key in @" __stringify(__LINE__)": ",
957 			     DUMP_PREFIX_ADDRESS, 16, 4, key, keylen, 1);
958 
959 	ctx->cdata.keylen = keylen;
960 	ctx->cdata.key_virt = key;
961 	ctx->cdata.key_inline = true;
962 
963 	/* skcipher_encrypt shared descriptor */
964 	flc = &ctx->flc[ENCRYPT];
965 	desc = flc->sh_desc;
966 	cnstr_shdsc_skcipher_encap(desc, &ctx->cdata, ivsize, is_rfc3686,
967 				   ctx1_iv_off);
968 	flc->flc[1] = cpu_to_caam32(desc_len(desc)); /* SDL */
969 	dma_sync_single_for_device(dev, ctx->flc_dma[ENCRYPT],
970 				   sizeof(flc->flc) + desc_bytes(desc),
971 				   ctx->dir);
972 
973 	/* skcipher_decrypt shared descriptor */
974 	flc = &ctx->flc[DECRYPT];
975 	desc = flc->sh_desc;
976 	cnstr_shdsc_skcipher_decap(desc, &ctx->cdata, ivsize, is_rfc3686,
977 				   ctx1_iv_off);
978 	flc->flc[1] = cpu_to_caam32(desc_len(desc)); /* SDL */
979 	dma_sync_single_for_device(dev, ctx->flc_dma[DECRYPT],
980 				   sizeof(flc->flc) + desc_bytes(desc),
981 				   ctx->dir);
982 
983 	return 0;
984 }
985 
986 static int aes_skcipher_setkey(struct crypto_skcipher *skcipher,
987 			       const u8 *key, unsigned int keylen)
988 {
989 	int err;
990 
991 	err = aes_check_keylen(keylen);
992 	if (err)
993 		return err;
994 
995 	return skcipher_setkey(skcipher, key, keylen, 0);
996 }
997 
998 static int rfc3686_skcipher_setkey(struct crypto_skcipher *skcipher,
999 				   const u8 *key, unsigned int keylen)
1000 {
1001 	u32 ctx1_iv_off;
1002 	int err;
1003 
1004 	/*
1005 	 * RFC3686 specific:
1006 	 *	| CONTEXT1[255:128] = {NONCE, IV, COUNTER}
1007 	 *	| *key = {KEY, NONCE}
1008 	 */
1009 	ctx1_iv_off = 16 + CTR_RFC3686_NONCE_SIZE;
1010 	keylen -= CTR_RFC3686_NONCE_SIZE;
1011 
1012 	err = aes_check_keylen(keylen);
1013 	if (err)
1014 		return err;
1015 
1016 	return skcipher_setkey(skcipher, key, keylen, ctx1_iv_off);
1017 }
1018 
1019 static int ctr_skcipher_setkey(struct crypto_skcipher *skcipher,
1020 			       const u8 *key, unsigned int keylen)
1021 {
1022 	u32 ctx1_iv_off;
1023 	int err;
1024 
1025 	/*
1026 	 * AES-CTR needs to load IV in CONTEXT1 reg
1027 	 * at an offset of 128bits (16bytes)
1028 	 * CONTEXT1[255:128] = IV
1029 	 */
1030 	ctx1_iv_off = 16;
1031 
1032 	err = aes_check_keylen(keylen);
1033 	if (err)
1034 		return err;
1035 
1036 	return skcipher_setkey(skcipher, key, keylen, ctx1_iv_off);
1037 }
1038 
1039 static int chacha20_skcipher_setkey(struct crypto_skcipher *skcipher,
1040 				    const u8 *key, unsigned int keylen)
1041 {
1042 	if (keylen != CHACHA_KEY_SIZE)
1043 		return -EINVAL;
1044 
1045 	return skcipher_setkey(skcipher, key, keylen, 0);
1046 }
1047 
1048 static int des_skcipher_setkey(struct crypto_skcipher *skcipher,
1049 			       const u8 *key, unsigned int keylen)
1050 {
1051 	return verify_skcipher_des_key(skcipher, key) ?:
1052 	       skcipher_setkey(skcipher, key, keylen, 0);
1053 }
1054 
1055 static int des3_skcipher_setkey(struct crypto_skcipher *skcipher,
1056 			        const u8 *key, unsigned int keylen)
1057 {
1058 	return verify_skcipher_des3_key(skcipher, key) ?:
1059 	       skcipher_setkey(skcipher, key, keylen, 0);
1060 }
1061 
1062 static int xts_skcipher_setkey(struct crypto_skcipher *skcipher, const u8 *key,
1063 			       unsigned int keylen)
1064 {
1065 	struct caam_ctx *ctx = crypto_skcipher_ctx_dma(skcipher);
1066 	struct device *dev = ctx->dev;
1067 	struct dpaa2_caam_priv *priv = dev_get_drvdata(dev);
1068 	struct caam_flc *flc;
1069 	u32 *desc;
1070 	int err;
1071 
1072 	err = xts_verify_key(skcipher, key, keylen);
1073 	if (err) {
1074 		dev_dbg(dev, "key size mismatch\n");
1075 		return err;
1076 	}
1077 
1078 	if (keylen != 2 * AES_KEYSIZE_128 && keylen != 2 * AES_KEYSIZE_256)
1079 		ctx->xts_key_fallback = true;
1080 
1081 	if (priv->sec_attr.era <= 8 || ctx->xts_key_fallback) {
1082 		err = crypto_skcipher_setkey(ctx->fallback, key, keylen);
1083 		if (err)
1084 			return err;
1085 	}
1086 
1087 	ctx->cdata.keylen = keylen;
1088 	ctx->cdata.key_virt = key;
1089 	ctx->cdata.key_inline = true;
1090 
1091 	/* xts_skcipher_encrypt shared descriptor */
1092 	flc = &ctx->flc[ENCRYPT];
1093 	desc = flc->sh_desc;
1094 	cnstr_shdsc_xts_skcipher_encap(desc, &ctx->cdata);
1095 	flc->flc[1] = cpu_to_caam32(desc_len(desc)); /* SDL */
1096 	dma_sync_single_for_device(dev, ctx->flc_dma[ENCRYPT],
1097 				   sizeof(flc->flc) + desc_bytes(desc),
1098 				   ctx->dir);
1099 
1100 	/* xts_skcipher_decrypt shared descriptor */
1101 	flc = &ctx->flc[DECRYPT];
1102 	desc = flc->sh_desc;
1103 	cnstr_shdsc_xts_skcipher_decap(desc, &ctx->cdata);
1104 	flc->flc[1] = cpu_to_caam32(desc_len(desc)); /* SDL */
1105 	dma_sync_single_for_device(dev, ctx->flc_dma[DECRYPT],
1106 				   sizeof(flc->flc) + desc_bytes(desc),
1107 				   ctx->dir);
1108 
1109 	return 0;
1110 }
1111 
1112 static struct skcipher_edesc *skcipher_edesc_alloc(struct skcipher_request *req)
1113 {
1114 	struct crypto_skcipher *skcipher = crypto_skcipher_reqtfm(req);
1115 	struct caam_request *req_ctx = skcipher_request_ctx_dma(req);
1116 	struct dpaa2_fl_entry *in_fle = &req_ctx->fd_flt[1];
1117 	struct dpaa2_fl_entry *out_fle = &req_ctx->fd_flt[0];
1118 	struct caam_ctx *ctx = crypto_skcipher_ctx_dma(skcipher);
1119 	struct device *dev = ctx->dev;
1120 	gfp_t flags = (req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP) ?
1121 		       GFP_KERNEL : GFP_ATOMIC;
1122 	int src_nents, mapped_src_nents, dst_nents = 0, mapped_dst_nents = 0;
1123 	struct skcipher_edesc *edesc;
1124 	dma_addr_t iv_dma;
1125 	u8 *iv;
1126 	int ivsize = crypto_skcipher_ivsize(skcipher);
1127 	int dst_sg_idx, qm_sg_ents, qm_sg_bytes;
1128 	struct dpaa2_sg_entry *sg_table;
1129 
1130 	src_nents = sg_nents_for_len(req->src, req->cryptlen);
1131 	if (unlikely(src_nents < 0)) {
1132 		dev_err(dev, "Insufficient bytes (%d) in src S/G\n",
1133 			req->cryptlen);
1134 		return ERR_PTR(src_nents);
1135 	}
1136 
1137 	if (unlikely(req->dst != req->src)) {
1138 		dst_nents = sg_nents_for_len(req->dst, req->cryptlen);
1139 		if (unlikely(dst_nents < 0)) {
1140 			dev_err(dev, "Insufficient bytes (%d) in dst S/G\n",
1141 				req->cryptlen);
1142 			return ERR_PTR(dst_nents);
1143 		}
1144 
1145 		mapped_src_nents = dma_map_sg(dev, req->src, src_nents,
1146 					      DMA_TO_DEVICE);
1147 		if (unlikely(!mapped_src_nents)) {
1148 			dev_err(dev, "unable to map source\n");
1149 			return ERR_PTR(-ENOMEM);
1150 		}
1151 
1152 		mapped_dst_nents = dma_map_sg(dev, req->dst, dst_nents,
1153 					      DMA_FROM_DEVICE);
1154 		if (unlikely(!mapped_dst_nents)) {
1155 			dev_err(dev, "unable to map destination\n");
1156 			dma_unmap_sg(dev, req->src, src_nents, DMA_TO_DEVICE);
1157 			return ERR_PTR(-ENOMEM);
1158 		}
1159 	} else {
1160 		mapped_src_nents = dma_map_sg(dev, req->src, src_nents,
1161 					      DMA_BIDIRECTIONAL);
1162 		if (unlikely(!mapped_src_nents)) {
1163 			dev_err(dev, "unable to map source\n");
1164 			return ERR_PTR(-ENOMEM);
1165 		}
1166 	}
1167 
1168 	qm_sg_ents = 1 + mapped_src_nents;
1169 	dst_sg_idx = qm_sg_ents;
1170 
1171 	/*
1172 	 * Input, output HW S/G tables: [IV, src][dst, IV]
1173 	 * IV entries point to the same buffer
1174 	 * If src == dst, S/G entries are reused (S/G tables overlap)
1175 	 *
1176 	 * HW reads 4 S/G entries at a time; make sure the reads don't go beyond
1177 	 * the end of the table by allocating more S/G entries.
1178 	 */
1179 	if (req->src != req->dst)
1180 		qm_sg_ents += pad_sg_nents(mapped_dst_nents + 1);
1181 	else
1182 		qm_sg_ents = 1 + pad_sg_nents(qm_sg_ents);
1183 
1184 	qm_sg_bytes = qm_sg_ents * sizeof(struct dpaa2_sg_entry);
1185 	if (unlikely(offsetof(struct skcipher_edesc, sgt) + qm_sg_bytes +
1186 		     ivsize > CAAM_QI_MEMCACHE_SIZE)) {
1187 		dev_err(dev, "No space for %d S/G entries and/or %dB IV\n",
1188 			qm_sg_ents, ivsize);
1189 		caam_unmap(dev, req->src, req->dst, src_nents, dst_nents, 0,
1190 			   0, DMA_NONE, 0, 0);
1191 		return ERR_PTR(-ENOMEM);
1192 	}
1193 
1194 	/* allocate space for base edesc, link tables and IV */
1195 	edesc = qi_cache_zalloc(flags);
1196 	if (unlikely(!edesc)) {
1197 		dev_err(dev, "could not allocate extended descriptor\n");
1198 		caam_unmap(dev, req->src, req->dst, src_nents, dst_nents, 0,
1199 			   0, DMA_NONE, 0, 0);
1200 		return ERR_PTR(-ENOMEM);
1201 	}
1202 
1203 	/* Make sure IV is located in a DMAable area */
1204 	sg_table = &edesc->sgt[0];
1205 	iv = (u8 *)(sg_table + qm_sg_ents);
1206 	memcpy(iv, req->iv, ivsize);
1207 
1208 	iv_dma = dma_map_single(dev, iv, ivsize, DMA_BIDIRECTIONAL);
1209 	if (dma_mapping_error(dev, iv_dma)) {
1210 		dev_err(dev, "unable to map IV\n");
1211 		caam_unmap(dev, req->src, req->dst, src_nents, dst_nents, 0,
1212 			   0, DMA_NONE, 0, 0);
1213 		qi_cache_free(edesc);
1214 		return ERR_PTR(-ENOMEM);
1215 	}
1216 
1217 	edesc->src_nents = src_nents;
1218 	edesc->dst_nents = dst_nents;
1219 	edesc->iv_dma = iv_dma;
1220 	edesc->qm_sg_bytes = qm_sg_bytes;
1221 
1222 	dma_to_qm_sg_one(sg_table, iv_dma, ivsize, 0);
1223 	sg_to_qm_sg(req->src, req->cryptlen, sg_table + 1, 0);
1224 
1225 	if (req->src != req->dst)
1226 		sg_to_qm_sg(req->dst, req->cryptlen, sg_table + dst_sg_idx, 0);
1227 
1228 	dma_to_qm_sg_one(sg_table + dst_sg_idx + mapped_dst_nents, iv_dma,
1229 			 ivsize, 0);
1230 
1231 	edesc->qm_sg_dma = dma_map_single(dev, sg_table, edesc->qm_sg_bytes,
1232 					  DMA_TO_DEVICE);
1233 	if (dma_mapping_error(dev, edesc->qm_sg_dma)) {
1234 		dev_err(dev, "unable to map S/G table\n");
1235 		caam_unmap(dev, req->src, req->dst, src_nents, dst_nents,
1236 			   iv_dma, ivsize, DMA_BIDIRECTIONAL, 0, 0);
1237 		qi_cache_free(edesc);
1238 		return ERR_PTR(-ENOMEM);
1239 	}
1240 
1241 	memset(&req_ctx->fd_flt, 0, sizeof(req_ctx->fd_flt));
1242 	dpaa2_fl_set_final(in_fle, true);
1243 	dpaa2_fl_set_len(in_fle, req->cryptlen + ivsize);
1244 	dpaa2_fl_set_len(out_fle, req->cryptlen + ivsize);
1245 
1246 	dpaa2_fl_set_format(in_fle, dpaa2_fl_sg);
1247 	dpaa2_fl_set_addr(in_fle, edesc->qm_sg_dma);
1248 
1249 	dpaa2_fl_set_format(out_fle, dpaa2_fl_sg);
1250 
1251 	if (req->src == req->dst)
1252 		dpaa2_fl_set_addr(out_fle, edesc->qm_sg_dma +
1253 				  sizeof(*sg_table));
1254 	else
1255 		dpaa2_fl_set_addr(out_fle, edesc->qm_sg_dma + dst_sg_idx *
1256 				  sizeof(*sg_table));
1257 
1258 	return edesc;
1259 }
1260 
1261 static void aead_unmap(struct device *dev, struct aead_edesc *edesc,
1262 		       struct aead_request *req)
1263 {
1264 	struct crypto_aead *aead = crypto_aead_reqtfm(req);
1265 	int ivsize = crypto_aead_ivsize(aead);
1266 
1267 	caam_unmap(dev, req->src, req->dst, edesc->src_nents, edesc->dst_nents,
1268 		   edesc->iv_dma, ivsize, DMA_TO_DEVICE, edesc->qm_sg_dma,
1269 		   edesc->qm_sg_bytes);
1270 	dma_unmap_single(dev, edesc->assoclen_dma, 4, DMA_TO_DEVICE);
1271 }
1272 
1273 static void skcipher_unmap(struct device *dev, struct skcipher_edesc *edesc,
1274 			   struct skcipher_request *req)
1275 {
1276 	struct crypto_skcipher *skcipher = crypto_skcipher_reqtfm(req);
1277 	int ivsize = crypto_skcipher_ivsize(skcipher);
1278 
1279 	caam_unmap(dev, req->src, req->dst, edesc->src_nents, edesc->dst_nents,
1280 		   edesc->iv_dma, ivsize, DMA_BIDIRECTIONAL, edesc->qm_sg_dma,
1281 		   edesc->qm_sg_bytes);
1282 }
1283 
1284 static void aead_encrypt_done(void *cbk_ctx, u32 status)
1285 {
1286 	struct crypto_async_request *areq = cbk_ctx;
1287 	struct aead_request *req = container_of(areq, struct aead_request,
1288 						base);
1289 	struct caam_request *req_ctx = to_caam_req(areq);
1290 	struct aead_edesc *edesc = req_ctx->edesc;
1291 	struct crypto_aead *aead = crypto_aead_reqtfm(req);
1292 	struct caam_ctx *ctx = crypto_aead_ctx_dma(aead);
1293 	int ecode = 0;
1294 
1295 	dev_dbg(ctx->dev, "%s %d: err 0x%x\n", __func__, __LINE__, status);
1296 
1297 	if (unlikely(status))
1298 		ecode = caam_qi2_strstatus(ctx->dev, status);
1299 
1300 	aead_unmap(ctx->dev, edesc, req);
1301 	qi_cache_free(edesc);
1302 	aead_request_complete(req, ecode);
1303 }
1304 
1305 static void aead_decrypt_done(void *cbk_ctx, u32 status)
1306 {
1307 	struct crypto_async_request *areq = cbk_ctx;
1308 	struct aead_request *req = container_of(areq, struct aead_request,
1309 						base);
1310 	struct caam_request *req_ctx = to_caam_req(areq);
1311 	struct aead_edesc *edesc = req_ctx->edesc;
1312 	struct crypto_aead *aead = crypto_aead_reqtfm(req);
1313 	struct caam_ctx *ctx = crypto_aead_ctx_dma(aead);
1314 	int ecode = 0;
1315 
1316 	dev_dbg(ctx->dev, "%s %d: err 0x%x\n", __func__, __LINE__, status);
1317 
1318 	if (unlikely(status))
1319 		ecode = caam_qi2_strstatus(ctx->dev, status);
1320 
1321 	aead_unmap(ctx->dev, edesc, req);
1322 	qi_cache_free(edesc);
1323 	aead_request_complete(req, ecode);
1324 }
1325 
1326 static int aead_encrypt(struct aead_request *req)
1327 {
1328 	struct aead_edesc *edesc;
1329 	struct crypto_aead *aead = crypto_aead_reqtfm(req);
1330 	struct caam_ctx *ctx = crypto_aead_ctx_dma(aead);
1331 	struct caam_request *caam_req = aead_request_ctx_dma(req);
1332 	int ret;
1333 
1334 	/* allocate extended descriptor */
1335 	edesc = aead_edesc_alloc(req, true);
1336 	if (IS_ERR(edesc))
1337 		return PTR_ERR(edesc);
1338 
1339 	caam_req->flc = &ctx->flc[ENCRYPT];
1340 	caam_req->flc_dma = ctx->flc_dma[ENCRYPT];
1341 	caam_req->cbk = aead_encrypt_done;
1342 	caam_req->ctx = &req->base;
1343 	caam_req->edesc = edesc;
1344 	ret = dpaa2_caam_enqueue(ctx->dev, caam_req);
1345 	if (ret != -EINPROGRESS &&
1346 	    !(ret == -EBUSY && req->base.flags & CRYPTO_TFM_REQ_MAY_BACKLOG)) {
1347 		aead_unmap(ctx->dev, edesc, req);
1348 		qi_cache_free(edesc);
1349 	}
1350 
1351 	return ret;
1352 }
1353 
1354 static int aead_decrypt(struct aead_request *req)
1355 {
1356 	struct aead_edesc *edesc;
1357 	struct crypto_aead *aead = crypto_aead_reqtfm(req);
1358 	struct caam_ctx *ctx = crypto_aead_ctx_dma(aead);
1359 	struct caam_request *caam_req = aead_request_ctx_dma(req);
1360 	int ret;
1361 
1362 	/* allocate extended descriptor */
1363 	edesc = aead_edesc_alloc(req, false);
1364 	if (IS_ERR(edesc))
1365 		return PTR_ERR(edesc);
1366 
1367 	caam_req->flc = &ctx->flc[DECRYPT];
1368 	caam_req->flc_dma = ctx->flc_dma[DECRYPT];
1369 	caam_req->cbk = aead_decrypt_done;
1370 	caam_req->ctx = &req->base;
1371 	caam_req->edesc = edesc;
1372 	ret = dpaa2_caam_enqueue(ctx->dev, caam_req);
1373 	if (ret != -EINPROGRESS &&
1374 	    !(ret == -EBUSY && req->base.flags & CRYPTO_TFM_REQ_MAY_BACKLOG)) {
1375 		aead_unmap(ctx->dev, edesc, req);
1376 		qi_cache_free(edesc);
1377 	}
1378 
1379 	return ret;
1380 }
1381 
1382 static int ipsec_gcm_encrypt(struct aead_request *req)
1383 {
1384 	return crypto_ipsec_check_assoclen(req->assoclen) ? : aead_encrypt(req);
1385 }
1386 
1387 static int ipsec_gcm_decrypt(struct aead_request *req)
1388 {
1389 	return crypto_ipsec_check_assoclen(req->assoclen) ? : aead_decrypt(req);
1390 }
1391 
1392 static void skcipher_encrypt_done(void *cbk_ctx, u32 status)
1393 {
1394 	struct crypto_async_request *areq = cbk_ctx;
1395 	struct skcipher_request *req = skcipher_request_cast(areq);
1396 	struct caam_request *req_ctx = to_caam_req(areq);
1397 	struct crypto_skcipher *skcipher = crypto_skcipher_reqtfm(req);
1398 	struct caam_ctx *ctx = crypto_skcipher_ctx_dma(skcipher);
1399 	struct skcipher_edesc *edesc = req_ctx->edesc;
1400 	int ecode = 0;
1401 	int ivsize = crypto_skcipher_ivsize(skcipher);
1402 
1403 	dev_dbg(ctx->dev, "%s %d: err 0x%x\n", __func__, __LINE__, status);
1404 
1405 	if (unlikely(status))
1406 		ecode = caam_qi2_strstatus(ctx->dev, status);
1407 
1408 	print_hex_dump_debug("dstiv  @" __stringify(__LINE__)": ",
1409 			     DUMP_PREFIX_ADDRESS, 16, 4, req->iv,
1410 			     edesc->src_nents > 1 ? 100 : ivsize, 1);
1411 	caam_dump_sg("dst    @" __stringify(__LINE__)": ",
1412 		     DUMP_PREFIX_ADDRESS, 16, 4, req->dst,
1413 		     edesc->dst_nents > 1 ? 100 : req->cryptlen, 1);
1414 
1415 	skcipher_unmap(ctx->dev, edesc, req);
1416 
1417 	/*
1418 	 * The crypto API expects us to set the IV (req->iv) to the last
1419 	 * ciphertext block (CBC mode) or last counter (CTR mode).
1420 	 * This is used e.g. by the CTS mode.
1421 	 */
1422 	if (!ecode)
1423 		memcpy(req->iv, (u8 *)&edesc->sgt[0] + edesc->qm_sg_bytes,
1424 		       ivsize);
1425 
1426 	qi_cache_free(edesc);
1427 	skcipher_request_complete(req, ecode);
1428 }
1429 
1430 static void skcipher_decrypt_done(void *cbk_ctx, u32 status)
1431 {
1432 	struct crypto_async_request *areq = cbk_ctx;
1433 	struct skcipher_request *req = skcipher_request_cast(areq);
1434 	struct caam_request *req_ctx = to_caam_req(areq);
1435 	struct crypto_skcipher *skcipher = crypto_skcipher_reqtfm(req);
1436 	struct caam_ctx *ctx = crypto_skcipher_ctx_dma(skcipher);
1437 	struct skcipher_edesc *edesc = req_ctx->edesc;
1438 	int ecode = 0;
1439 	int ivsize = crypto_skcipher_ivsize(skcipher);
1440 
1441 	dev_dbg(ctx->dev, "%s %d: err 0x%x\n", __func__, __LINE__, status);
1442 
1443 	if (unlikely(status))
1444 		ecode = caam_qi2_strstatus(ctx->dev, status);
1445 
1446 	print_hex_dump_debug("dstiv  @" __stringify(__LINE__)": ",
1447 			     DUMP_PREFIX_ADDRESS, 16, 4, req->iv,
1448 			     edesc->src_nents > 1 ? 100 : ivsize, 1);
1449 	caam_dump_sg("dst    @" __stringify(__LINE__)": ",
1450 		     DUMP_PREFIX_ADDRESS, 16, 4, req->dst,
1451 		     edesc->dst_nents > 1 ? 100 : req->cryptlen, 1);
1452 
1453 	skcipher_unmap(ctx->dev, edesc, req);
1454 
1455 	/*
1456 	 * The crypto API expects us to set the IV (req->iv) to the last
1457 	 * ciphertext block (CBC mode) or last counter (CTR mode).
1458 	 * This is used e.g. by the CTS mode.
1459 	 */
1460 	if (!ecode)
1461 		memcpy(req->iv, (u8 *)&edesc->sgt[0] + edesc->qm_sg_bytes,
1462 		       ivsize);
1463 
1464 	qi_cache_free(edesc);
1465 	skcipher_request_complete(req, ecode);
1466 }
1467 
1468 static inline bool xts_skcipher_ivsize(struct skcipher_request *req)
1469 {
1470 	struct crypto_skcipher *skcipher = crypto_skcipher_reqtfm(req);
1471 	unsigned int ivsize = crypto_skcipher_ivsize(skcipher);
1472 
1473 	return !!get_unaligned((u64 *)(req->iv + (ivsize / 2)));
1474 }
1475 
1476 static int skcipher_encrypt(struct skcipher_request *req)
1477 {
1478 	struct skcipher_edesc *edesc;
1479 	struct crypto_skcipher *skcipher = crypto_skcipher_reqtfm(req);
1480 	struct caam_ctx *ctx = crypto_skcipher_ctx_dma(skcipher);
1481 	struct caam_request *caam_req = skcipher_request_ctx_dma(req);
1482 	struct dpaa2_caam_priv *priv = dev_get_drvdata(ctx->dev);
1483 	int ret;
1484 
1485 	/*
1486 	 * XTS is expected to return an error even for input length = 0
1487 	 * Note that the case input length < block size will be caught during
1488 	 * HW offloading and return an error.
1489 	 */
1490 	if (!req->cryptlen && !ctx->fallback)
1491 		return 0;
1492 
1493 	if (ctx->fallback && ((priv->sec_attr.era <= 8 && xts_skcipher_ivsize(req)) ||
1494 			      ctx->xts_key_fallback)) {
1495 		skcipher_request_set_tfm(&caam_req->fallback_req, ctx->fallback);
1496 		skcipher_request_set_callback(&caam_req->fallback_req,
1497 					      req->base.flags,
1498 					      req->base.complete,
1499 					      req->base.data);
1500 		skcipher_request_set_crypt(&caam_req->fallback_req, req->src,
1501 					   req->dst, req->cryptlen, req->iv);
1502 
1503 		return crypto_skcipher_encrypt(&caam_req->fallback_req);
1504 	}
1505 
1506 	/* allocate extended descriptor */
1507 	edesc = skcipher_edesc_alloc(req);
1508 	if (IS_ERR(edesc))
1509 		return PTR_ERR(edesc);
1510 
1511 	caam_req->flc = &ctx->flc[ENCRYPT];
1512 	caam_req->flc_dma = ctx->flc_dma[ENCRYPT];
1513 	caam_req->cbk = skcipher_encrypt_done;
1514 	caam_req->ctx = &req->base;
1515 	caam_req->edesc = edesc;
1516 	ret = dpaa2_caam_enqueue(ctx->dev, caam_req);
1517 	if (ret != -EINPROGRESS &&
1518 	    !(ret == -EBUSY && req->base.flags & CRYPTO_TFM_REQ_MAY_BACKLOG)) {
1519 		skcipher_unmap(ctx->dev, edesc, req);
1520 		qi_cache_free(edesc);
1521 	}
1522 
1523 	return ret;
1524 }
1525 
1526 static int skcipher_decrypt(struct skcipher_request *req)
1527 {
1528 	struct skcipher_edesc *edesc;
1529 	struct crypto_skcipher *skcipher = crypto_skcipher_reqtfm(req);
1530 	struct caam_ctx *ctx = crypto_skcipher_ctx_dma(skcipher);
1531 	struct caam_request *caam_req = skcipher_request_ctx_dma(req);
1532 	struct dpaa2_caam_priv *priv = dev_get_drvdata(ctx->dev);
1533 	int ret;
1534 
1535 	/*
1536 	 * XTS is expected to return an error even for input length = 0
1537 	 * Note that the case input length < block size will be caught during
1538 	 * HW offloading and return an error.
1539 	 */
1540 	if (!req->cryptlen && !ctx->fallback)
1541 		return 0;
1542 
1543 	if (ctx->fallback && ((priv->sec_attr.era <= 8 && xts_skcipher_ivsize(req)) ||
1544 			      ctx->xts_key_fallback)) {
1545 		skcipher_request_set_tfm(&caam_req->fallback_req, ctx->fallback);
1546 		skcipher_request_set_callback(&caam_req->fallback_req,
1547 					      req->base.flags,
1548 					      req->base.complete,
1549 					      req->base.data);
1550 		skcipher_request_set_crypt(&caam_req->fallback_req, req->src,
1551 					   req->dst, req->cryptlen, req->iv);
1552 
1553 		return crypto_skcipher_decrypt(&caam_req->fallback_req);
1554 	}
1555 
1556 	/* allocate extended descriptor */
1557 	edesc = skcipher_edesc_alloc(req);
1558 	if (IS_ERR(edesc))
1559 		return PTR_ERR(edesc);
1560 
1561 	caam_req->flc = &ctx->flc[DECRYPT];
1562 	caam_req->flc_dma = ctx->flc_dma[DECRYPT];
1563 	caam_req->cbk = skcipher_decrypt_done;
1564 	caam_req->ctx = &req->base;
1565 	caam_req->edesc = edesc;
1566 	ret = dpaa2_caam_enqueue(ctx->dev, caam_req);
1567 	if (ret != -EINPROGRESS &&
1568 	    !(ret == -EBUSY && req->base.flags & CRYPTO_TFM_REQ_MAY_BACKLOG)) {
1569 		skcipher_unmap(ctx->dev, edesc, req);
1570 		qi_cache_free(edesc);
1571 	}
1572 
1573 	return ret;
1574 }
1575 
1576 static int caam_cra_init(struct caam_ctx *ctx, struct caam_alg_entry *caam,
1577 			 bool uses_dkp)
1578 {
1579 	dma_addr_t dma_addr;
1580 	int i;
1581 
1582 	/* copy descriptor header template value */
1583 	ctx->cdata.algtype = OP_TYPE_CLASS1_ALG | caam->class1_alg_type;
1584 	ctx->adata.algtype = OP_TYPE_CLASS2_ALG | caam->class2_alg_type;
1585 
1586 	ctx->dev = caam->dev;
1587 	ctx->dir = uses_dkp ? DMA_BIDIRECTIONAL : DMA_TO_DEVICE;
1588 
1589 	dma_addr = dma_map_single_attrs(ctx->dev, ctx->flc,
1590 					offsetof(struct caam_ctx, flc_dma),
1591 					ctx->dir, DMA_ATTR_SKIP_CPU_SYNC);
1592 	if (dma_mapping_error(ctx->dev, dma_addr)) {
1593 		dev_err(ctx->dev, "unable to map key, shared descriptors\n");
1594 		return -ENOMEM;
1595 	}
1596 
1597 	for (i = 0; i < NUM_OP; i++)
1598 		ctx->flc_dma[i] = dma_addr + i * sizeof(ctx->flc[i]);
1599 	ctx->key_dma = dma_addr + NUM_OP * sizeof(ctx->flc[0]);
1600 
1601 	return 0;
1602 }
1603 
1604 static int caam_cra_init_skcipher(struct crypto_skcipher *tfm)
1605 {
1606 	struct skcipher_alg *alg = crypto_skcipher_alg(tfm);
1607 	struct caam_skcipher_alg *caam_alg =
1608 		container_of(alg, typeof(*caam_alg), skcipher);
1609 	struct caam_ctx *ctx = crypto_skcipher_ctx_dma(tfm);
1610 	u32 alg_aai = caam_alg->caam.class1_alg_type & OP_ALG_AAI_MASK;
1611 	int ret = 0;
1612 
1613 	if (alg_aai == OP_ALG_AAI_XTS) {
1614 		const char *tfm_name = crypto_tfm_alg_name(&tfm->base);
1615 		struct crypto_skcipher *fallback;
1616 
1617 		fallback = crypto_alloc_skcipher(tfm_name, 0,
1618 						 CRYPTO_ALG_NEED_FALLBACK);
1619 		if (IS_ERR(fallback)) {
1620 			dev_err(caam_alg->caam.dev,
1621 				"Failed to allocate %s fallback: %ld\n",
1622 				tfm_name, PTR_ERR(fallback));
1623 			return PTR_ERR(fallback);
1624 		}
1625 
1626 		ctx->fallback = fallback;
1627 		crypto_skcipher_set_reqsize_dma(
1628 			tfm, sizeof(struct caam_request) +
1629 			     crypto_skcipher_reqsize(fallback));
1630 	} else {
1631 		crypto_skcipher_set_reqsize_dma(tfm,
1632 						sizeof(struct caam_request));
1633 	}
1634 
1635 	ret = caam_cra_init(ctx, &caam_alg->caam, false);
1636 	if (ret && ctx->fallback)
1637 		crypto_free_skcipher(ctx->fallback);
1638 
1639 	return ret;
1640 }
1641 
1642 static int caam_cra_init_aead(struct crypto_aead *tfm)
1643 {
1644 	struct aead_alg *alg = crypto_aead_alg(tfm);
1645 	struct caam_aead_alg *caam_alg = container_of(alg, typeof(*caam_alg),
1646 						      aead);
1647 
1648 	crypto_aead_set_reqsize_dma(tfm, sizeof(struct caam_request));
1649 	return caam_cra_init(crypto_aead_ctx_dma(tfm), &caam_alg->caam,
1650 			     !caam_alg->caam.nodkp);
1651 }
1652 
1653 static void caam_exit_common(struct caam_ctx *ctx)
1654 {
1655 	dma_unmap_single_attrs(ctx->dev, ctx->flc_dma[0],
1656 			       offsetof(struct caam_ctx, flc_dma), ctx->dir,
1657 			       DMA_ATTR_SKIP_CPU_SYNC);
1658 }
1659 
1660 static void caam_cra_exit(struct crypto_skcipher *tfm)
1661 {
1662 	struct caam_ctx *ctx = crypto_skcipher_ctx_dma(tfm);
1663 
1664 	if (ctx->fallback)
1665 		crypto_free_skcipher(ctx->fallback);
1666 	caam_exit_common(ctx);
1667 }
1668 
1669 static void caam_cra_exit_aead(struct crypto_aead *tfm)
1670 {
1671 	caam_exit_common(crypto_aead_ctx_dma(tfm));
1672 }
1673 
1674 static struct caam_skcipher_alg driver_algs[] = {
1675 	{
1676 		.skcipher = {
1677 			.base = {
1678 				.cra_name = "cbc(aes)",
1679 				.cra_driver_name = "cbc-aes-caam-qi2",
1680 				.cra_blocksize = AES_BLOCK_SIZE,
1681 			},
1682 			.setkey = aes_skcipher_setkey,
1683 			.encrypt = skcipher_encrypt,
1684 			.decrypt = skcipher_decrypt,
1685 			.min_keysize = AES_MIN_KEY_SIZE,
1686 			.max_keysize = AES_MAX_KEY_SIZE,
1687 			.ivsize = AES_BLOCK_SIZE,
1688 		},
1689 		.caam.class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CBC,
1690 	},
1691 	{
1692 		.skcipher = {
1693 			.base = {
1694 				.cra_name = "cbc(des3_ede)",
1695 				.cra_driver_name = "cbc-3des-caam-qi2",
1696 				.cra_blocksize = DES3_EDE_BLOCK_SIZE,
1697 			},
1698 			.setkey = des3_skcipher_setkey,
1699 			.encrypt = skcipher_encrypt,
1700 			.decrypt = skcipher_decrypt,
1701 			.min_keysize = DES3_EDE_KEY_SIZE,
1702 			.max_keysize = DES3_EDE_KEY_SIZE,
1703 			.ivsize = DES3_EDE_BLOCK_SIZE,
1704 		},
1705 		.caam.class1_alg_type = OP_ALG_ALGSEL_3DES | OP_ALG_AAI_CBC,
1706 	},
1707 	{
1708 		.skcipher = {
1709 			.base = {
1710 				.cra_name = "cbc(des)",
1711 				.cra_driver_name = "cbc-des-caam-qi2",
1712 				.cra_blocksize = DES_BLOCK_SIZE,
1713 			},
1714 			.setkey = des_skcipher_setkey,
1715 			.encrypt = skcipher_encrypt,
1716 			.decrypt = skcipher_decrypt,
1717 			.min_keysize = DES_KEY_SIZE,
1718 			.max_keysize = DES_KEY_SIZE,
1719 			.ivsize = DES_BLOCK_SIZE,
1720 		},
1721 		.caam.class1_alg_type = OP_ALG_ALGSEL_DES | OP_ALG_AAI_CBC,
1722 	},
1723 	{
1724 		.skcipher = {
1725 			.base = {
1726 				.cra_name = "ctr(aes)",
1727 				.cra_driver_name = "ctr-aes-caam-qi2",
1728 				.cra_blocksize = 1,
1729 			},
1730 			.setkey = ctr_skcipher_setkey,
1731 			.encrypt = skcipher_encrypt,
1732 			.decrypt = skcipher_decrypt,
1733 			.min_keysize = AES_MIN_KEY_SIZE,
1734 			.max_keysize = AES_MAX_KEY_SIZE,
1735 			.ivsize = AES_BLOCK_SIZE,
1736 			.chunksize = AES_BLOCK_SIZE,
1737 		},
1738 		.caam.class1_alg_type = OP_ALG_ALGSEL_AES |
1739 					OP_ALG_AAI_CTR_MOD128,
1740 	},
1741 	{
1742 		.skcipher = {
1743 			.base = {
1744 				.cra_name = "rfc3686(ctr(aes))",
1745 				.cra_driver_name = "rfc3686-ctr-aes-caam-qi2",
1746 				.cra_blocksize = 1,
1747 			},
1748 			.setkey = rfc3686_skcipher_setkey,
1749 			.encrypt = skcipher_encrypt,
1750 			.decrypt = skcipher_decrypt,
1751 			.min_keysize = AES_MIN_KEY_SIZE +
1752 				       CTR_RFC3686_NONCE_SIZE,
1753 			.max_keysize = AES_MAX_KEY_SIZE +
1754 				       CTR_RFC3686_NONCE_SIZE,
1755 			.ivsize = CTR_RFC3686_IV_SIZE,
1756 			.chunksize = AES_BLOCK_SIZE,
1757 		},
1758 		.caam = {
1759 			.class1_alg_type = OP_ALG_ALGSEL_AES |
1760 					   OP_ALG_AAI_CTR_MOD128,
1761 			.rfc3686 = true,
1762 		},
1763 	},
1764 	{
1765 		.skcipher = {
1766 			.base = {
1767 				.cra_name = "xts(aes)",
1768 				.cra_driver_name = "xts-aes-caam-qi2",
1769 				.cra_flags = CRYPTO_ALG_NEED_FALLBACK,
1770 				.cra_blocksize = AES_BLOCK_SIZE,
1771 			},
1772 			.setkey = xts_skcipher_setkey,
1773 			.encrypt = skcipher_encrypt,
1774 			.decrypt = skcipher_decrypt,
1775 			.min_keysize = 2 * AES_MIN_KEY_SIZE,
1776 			.max_keysize = 2 * AES_MAX_KEY_SIZE,
1777 			.ivsize = AES_BLOCK_SIZE,
1778 		},
1779 		.caam.class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_XTS,
1780 	},
1781 	{
1782 		.skcipher = {
1783 			.base = {
1784 				.cra_name = "chacha20",
1785 				.cra_driver_name = "chacha20-caam-qi2",
1786 				.cra_blocksize = 1,
1787 			},
1788 			.setkey = chacha20_skcipher_setkey,
1789 			.encrypt = skcipher_encrypt,
1790 			.decrypt = skcipher_decrypt,
1791 			.min_keysize = CHACHA_KEY_SIZE,
1792 			.max_keysize = CHACHA_KEY_SIZE,
1793 			.ivsize = CHACHA_IV_SIZE,
1794 		},
1795 		.caam.class1_alg_type = OP_ALG_ALGSEL_CHACHA20,
1796 	},
1797 };
1798 
1799 static struct caam_aead_alg driver_aeads[] = {
1800 	{
1801 		.aead = {
1802 			.base = {
1803 				.cra_name = "rfc4106(gcm(aes))",
1804 				.cra_driver_name = "rfc4106-gcm-aes-caam-qi2",
1805 				.cra_blocksize = 1,
1806 			},
1807 			.setkey = rfc4106_setkey,
1808 			.setauthsize = rfc4106_setauthsize,
1809 			.encrypt = ipsec_gcm_encrypt,
1810 			.decrypt = ipsec_gcm_decrypt,
1811 			.ivsize = 8,
1812 			.maxauthsize = AES_BLOCK_SIZE,
1813 		},
1814 		.caam = {
1815 			.class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_GCM,
1816 			.nodkp = true,
1817 		},
1818 	},
1819 	{
1820 		.aead = {
1821 			.base = {
1822 				.cra_name = "rfc4543(gcm(aes))",
1823 				.cra_driver_name = "rfc4543-gcm-aes-caam-qi2",
1824 				.cra_blocksize = 1,
1825 			},
1826 			.setkey = rfc4543_setkey,
1827 			.setauthsize = rfc4543_setauthsize,
1828 			.encrypt = ipsec_gcm_encrypt,
1829 			.decrypt = ipsec_gcm_decrypt,
1830 			.ivsize = 8,
1831 			.maxauthsize = AES_BLOCK_SIZE,
1832 		},
1833 		.caam = {
1834 			.class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_GCM,
1835 			.nodkp = true,
1836 		},
1837 	},
1838 	/* Galois Counter Mode */
1839 	{
1840 		.aead = {
1841 			.base = {
1842 				.cra_name = "gcm(aes)",
1843 				.cra_driver_name = "gcm-aes-caam-qi2",
1844 				.cra_blocksize = 1,
1845 			},
1846 			.setkey = gcm_setkey,
1847 			.setauthsize = gcm_setauthsize,
1848 			.encrypt = aead_encrypt,
1849 			.decrypt = aead_decrypt,
1850 			.ivsize = 12,
1851 			.maxauthsize = AES_BLOCK_SIZE,
1852 		},
1853 		.caam = {
1854 			.class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_GCM,
1855 			.nodkp = true,
1856 		}
1857 	},
1858 	/* single-pass ipsec_esp descriptor */
1859 	{
1860 		.aead = {
1861 			.base = {
1862 				.cra_name = "authenc(hmac(md5),cbc(aes))",
1863 				.cra_driver_name = "authenc-hmac-md5-"
1864 						   "cbc-aes-caam-qi2",
1865 				.cra_blocksize = AES_BLOCK_SIZE,
1866 			},
1867 			.setkey = aead_setkey,
1868 			.setauthsize = aead_setauthsize,
1869 			.encrypt = aead_encrypt,
1870 			.decrypt = aead_decrypt,
1871 			.ivsize = AES_BLOCK_SIZE,
1872 			.maxauthsize = MD5_DIGEST_SIZE,
1873 		},
1874 		.caam = {
1875 			.class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CBC,
1876 			.class2_alg_type = OP_ALG_ALGSEL_MD5 |
1877 					   OP_ALG_AAI_HMAC_PRECOMP,
1878 		}
1879 	},
1880 	{
1881 		.aead = {
1882 			.base = {
1883 				.cra_name = "echainiv(authenc(hmac(md5),"
1884 					    "cbc(aes)))",
1885 				.cra_driver_name = "echainiv-authenc-hmac-md5-"
1886 						   "cbc-aes-caam-qi2",
1887 				.cra_blocksize = AES_BLOCK_SIZE,
1888 			},
1889 			.setkey = aead_setkey,
1890 			.setauthsize = aead_setauthsize,
1891 			.encrypt = aead_encrypt,
1892 			.decrypt = aead_decrypt,
1893 			.ivsize = AES_BLOCK_SIZE,
1894 			.maxauthsize = MD5_DIGEST_SIZE,
1895 		},
1896 		.caam = {
1897 			.class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CBC,
1898 			.class2_alg_type = OP_ALG_ALGSEL_MD5 |
1899 					   OP_ALG_AAI_HMAC_PRECOMP,
1900 			.geniv = true,
1901 		}
1902 	},
1903 	{
1904 		.aead = {
1905 			.base = {
1906 				.cra_name = "authenc(hmac(sha1),cbc(aes))",
1907 				.cra_driver_name = "authenc-hmac-sha1-"
1908 						   "cbc-aes-caam-qi2",
1909 				.cra_blocksize = AES_BLOCK_SIZE,
1910 			},
1911 			.setkey = aead_setkey,
1912 			.setauthsize = aead_setauthsize,
1913 			.encrypt = aead_encrypt,
1914 			.decrypt = aead_decrypt,
1915 			.ivsize = AES_BLOCK_SIZE,
1916 			.maxauthsize = SHA1_DIGEST_SIZE,
1917 		},
1918 		.caam = {
1919 			.class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CBC,
1920 			.class2_alg_type = OP_ALG_ALGSEL_SHA1 |
1921 					   OP_ALG_AAI_HMAC_PRECOMP,
1922 		}
1923 	},
1924 	{
1925 		.aead = {
1926 			.base = {
1927 				.cra_name = "echainiv(authenc(hmac(sha1),"
1928 					    "cbc(aes)))",
1929 				.cra_driver_name = "echainiv-authenc-"
1930 						   "hmac-sha1-cbc-aes-caam-qi2",
1931 				.cra_blocksize = AES_BLOCK_SIZE,
1932 			},
1933 			.setkey = aead_setkey,
1934 			.setauthsize = aead_setauthsize,
1935 			.encrypt = aead_encrypt,
1936 			.decrypt = aead_decrypt,
1937 			.ivsize = AES_BLOCK_SIZE,
1938 			.maxauthsize = SHA1_DIGEST_SIZE,
1939 		},
1940 		.caam = {
1941 			.class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CBC,
1942 			.class2_alg_type = OP_ALG_ALGSEL_SHA1 |
1943 					   OP_ALG_AAI_HMAC_PRECOMP,
1944 			.geniv = true,
1945 		},
1946 	},
1947 	{
1948 		.aead = {
1949 			.base = {
1950 				.cra_name = "authenc(hmac(sha224),cbc(aes))",
1951 				.cra_driver_name = "authenc-hmac-sha224-"
1952 						   "cbc-aes-caam-qi2",
1953 				.cra_blocksize = AES_BLOCK_SIZE,
1954 			},
1955 			.setkey = aead_setkey,
1956 			.setauthsize = aead_setauthsize,
1957 			.encrypt = aead_encrypt,
1958 			.decrypt = aead_decrypt,
1959 			.ivsize = AES_BLOCK_SIZE,
1960 			.maxauthsize = SHA224_DIGEST_SIZE,
1961 		},
1962 		.caam = {
1963 			.class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CBC,
1964 			.class2_alg_type = OP_ALG_ALGSEL_SHA224 |
1965 					   OP_ALG_AAI_HMAC_PRECOMP,
1966 		}
1967 	},
1968 	{
1969 		.aead = {
1970 			.base = {
1971 				.cra_name = "echainiv(authenc(hmac(sha224),"
1972 					    "cbc(aes)))",
1973 				.cra_driver_name = "echainiv-authenc-"
1974 						   "hmac-sha224-cbc-aes-caam-qi2",
1975 				.cra_blocksize = AES_BLOCK_SIZE,
1976 			},
1977 			.setkey = aead_setkey,
1978 			.setauthsize = aead_setauthsize,
1979 			.encrypt = aead_encrypt,
1980 			.decrypt = aead_decrypt,
1981 			.ivsize = AES_BLOCK_SIZE,
1982 			.maxauthsize = SHA224_DIGEST_SIZE,
1983 		},
1984 		.caam = {
1985 			.class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CBC,
1986 			.class2_alg_type = OP_ALG_ALGSEL_SHA224 |
1987 					   OP_ALG_AAI_HMAC_PRECOMP,
1988 			.geniv = true,
1989 		}
1990 	},
1991 	{
1992 		.aead = {
1993 			.base = {
1994 				.cra_name = "authenc(hmac(sha256),cbc(aes))",
1995 				.cra_driver_name = "authenc-hmac-sha256-"
1996 						   "cbc-aes-caam-qi2",
1997 				.cra_blocksize = AES_BLOCK_SIZE,
1998 			},
1999 			.setkey = aead_setkey,
2000 			.setauthsize = aead_setauthsize,
2001 			.encrypt = aead_encrypt,
2002 			.decrypt = aead_decrypt,
2003 			.ivsize = AES_BLOCK_SIZE,
2004 			.maxauthsize = SHA256_DIGEST_SIZE,
2005 		},
2006 		.caam = {
2007 			.class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CBC,
2008 			.class2_alg_type = OP_ALG_ALGSEL_SHA256 |
2009 					   OP_ALG_AAI_HMAC_PRECOMP,
2010 		}
2011 	},
2012 	{
2013 		.aead = {
2014 			.base = {
2015 				.cra_name = "echainiv(authenc(hmac(sha256),"
2016 					    "cbc(aes)))",
2017 				.cra_driver_name = "echainiv-authenc-"
2018 						   "hmac-sha256-cbc-aes-"
2019 						   "caam-qi2",
2020 				.cra_blocksize = AES_BLOCK_SIZE,
2021 			},
2022 			.setkey = aead_setkey,
2023 			.setauthsize = aead_setauthsize,
2024 			.encrypt = aead_encrypt,
2025 			.decrypt = aead_decrypt,
2026 			.ivsize = AES_BLOCK_SIZE,
2027 			.maxauthsize = SHA256_DIGEST_SIZE,
2028 		},
2029 		.caam = {
2030 			.class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CBC,
2031 			.class2_alg_type = OP_ALG_ALGSEL_SHA256 |
2032 					   OP_ALG_AAI_HMAC_PRECOMP,
2033 			.geniv = true,
2034 		}
2035 	},
2036 	{
2037 		.aead = {
2038 			.base = {
2039 				.cra_name = "authenc(hmac(sha384),cbc(aes))",
2040 				.cra_driver_name = "authenc-hmac-sha384-"
2041 						   "cbc-aes-caam-qi2",
2042 				.cra_blocksize = AES_BLOCK_SIZE,
2043 			},
2044 			.setkey = aead_setkey,
2045 			.setauthsize = aead_setauthsize,
2046 			.encrypt = aead_encrypt,
2047 			.decrypt = aead_decrypt,
2048 			.ivsize = AES_BLOCK_SIZE,
2049 			.maxauthsize = SHA384_DIGEST_SIZE,
2050 		},
2051 		.caam = {
2052 			.class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CBC,
2053 			.class2_alg_type = OP_ALG_ALGSEL_SHA384 |
2054 					   OP_ALG_AAI_HMAC_PRECOMP,
2055 		}
2056 	},
2057 	{
2058 		.aead = {
2059 			.base = {
2060 				.cra_name = "echainiv(authenc(hmac(sha384),"
2061 					    "cbc(aes)))",
2062 				.cra_driver_name = "echainiv-authenc-"
2063 						   "hmac-sha384-cbc-aes-"
2064 						   "caam-qi2",
2065 				.cra_blocksize = AES_BLOCK_SIZE,
2066 			},
2067 			.setkey = aead_setkey,
2068 			.setauthsize = aead_setauthsize,
2069 			.encrypt = aead_encrypt,
2070 			.decrypt = aead_decrypt,
2071 			.ivsize = AES_BLOCK_SIZE,
2072 			.maxauthsize = SHA384_DIGEST_SIZE,
2073 		},
2074 		.caam = {
2075 			.class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CBC,
2076 			.class2_alg_type = OP_ALG_ALGSEL_SHA384 |
2077 					   OP_ALG_AAI_HMAC_PRECOMP,
2078 			.geniv = true,
2079 		}
2080 	},
2081 	{
2082 		.aead = {
2083 			.base = {
2084 				.cra_name = "authenc(hmac(sha512),cbc(aes))",
2085 				.cra_driver_name = "authenc-hmac-sha512-"
2086 						   "cbc-aes-caam-qi2",
2087 				.cra_blocksize = AES_BLOCK_SIZE,
2088 			},
2089 			.setkey = aead_setkey,
2090 			.setauthsize = aead_setauthsize,
2091 			.encrypt = aead_encrypt,
2092 			.decrypt = aead_decrypt,
2093 			.ivsize = AES_BLOCK_SIZE,
2094 			.maxauthsize = SHA512_DIGEST_SIZE,
2095 		},
2096 		.caam = {
2097 			.class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CBC,
2098 			.class2_alg_type = OP_ALG_ALGSEL_SHA512 |
2099 					   OP_ALG_AAI_HMAC_PRECOMP,
2100 		}
2101 	},
2102 	{
2103 		.aead = {
2104 			.base = {
2105 				.cra_name = "echainiv(authenc(hmac(sha512),"
2106 					    "cbc(aes)))",
2107 				.cra_driver_name = "echainiv-authenc-"
2108 						   "hmac-sha512-cbc-aes-"
2109 						   "caam-qi2",
2110 				.cra_blocksize = AES_BLOCK_SIZE,
2111 			},
2112 			.setkey = aead_setkey,
2113 			.setauthsize = aead_setauthsize,
2114 			.encrypt = aead_encrypt,
2115 			.decrypt = aead_decrypt,
2116 			.ivsize = AES_BLOCK_SIZE,
2117 			.maxauthsize = SHA512_DIGEST_SIZE,
2118 		},
2119 		.caam = {
2120 			.class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CBC,
2121 			.class2_alg_type = OP_ALG_ALGSEL_SHA512 |
2122 					   OP_ALG_AAI_HMAC_PRECOMP,
2123 			.geniv = true,
2124 		}
2125 	},
2126 	{
2127 		.aead = {
2128 			.base = {
2129 				.cra_name = "authenc(hmac(md5),cbc(des3_ede))",
2130 				.cra_driver_name = "authenc-hmac-md5-"
2131 						   "cbc-des3_ede-caam-qi2",
2132 				.cra_blocksize = DES3_EDE_BLOCK_SIZE,
2133 			},
2134 			.setkey = des3_aead_setkey,
2135 			.setauthsize = aead_setauthsize,
2136 			.encrypt = aead_encrypt,
2137 			.decrypt = aead_decrypt,
2138 			.ivsize = DES3_EDE_BLOCK_SIZE,
2139 			.maxauthsize = MD5_DIGEST_SIZE,
2140 		},
2141 		.caam = {
2142 			.class1_alg_type = OP_ALG_ALGSEL_3DES | OP_ALG_AAI_CBC,
2143 			.class2_alg_type = OP_ALG_ALGSEL_MD5 |
2144 					   OP_ALG_AAI_HMAC_PRECOMP,
2145 		}
2146 	},
2147 	{
2148 		.aead = {
2149 			.base = {
2150 				.cra_name = "echainiv(authenc(hmac(md5),"
2151 					    "cbc(des3_ede)))",
2152 				.cra_driver_name = "echainiv-authenc-hmac-md5-"
2153 						   "cbc-des3_ede-caam-qi2",
2154 				.cra_blocksize = DES3_EDE_BLOCK_SIZE,
2155 			},
2156 			.setkey = des3_aead_setkey,
2157 			.setauthsize = aead_setauthsize,
2158 			.encrypt = aead_encrypt,
2159 			.decrypt = aead_decrypt,
2160 			.ivsize = DES3_EDE_BLOCK_SIZE,
2161 			.maxauthsize = MD5_DIGEST_SIZE,
2162 		},
2163 		.caam = {
2164 			.class1_alg_type = OP_ALG_ALGSEL_3DES | OP_ALG_AAI_CBC,
2165 			.class2_alg_type = OP_ALG_ALGSEL_MD5 |
2166 					   OP_ALG_AAI_HMAC_PRECOMP,
2167 			.geniv = true,
2168 		}
2169 	},
2170 	{
2171 		.aead = {
2172 			.base = {
2173 				.cra_name = "authenc(hmac(sha1),"
2174 					    "cbc(des3_ede))",
2175 				.cra_driver_name = "authenc-hmac-sha1-"
2176 						   "cbc-des3_ede-caam-qi2",
2177 				.cra_blocksize = DES3_EDE_BLOCK_SIZE,
2178 			},
2179 			.setkey = des3_aead_setkey,
2180 			.setauthsize = aead_setauthsize,
2181 			.encrypt = aead_encrypt,
2182 			.decrypt = aead_decrypt,
2183 			.ivsize = DES3_EDE_BLOCK_SIZE,
2184 			.maxauthsize = SHA1_DIGEST_SIZE,
2185 		},
2186 		.caam = {
2187 			.class1_alg_type = OP_ALG_ALGSEL_3DES | OP_ALG_AAI_CBC,
2188 			.class2_alg_type = OP_ALG_ALGSEL_SHA1 |
2189 					   OP_ALG_AAI_HMAC_PRECOMP,
2190 		},
2191 	},
2192 	{
2193 		.aead = {
2194 			.base = {
2195 				.cra_name = "echainiv(authenc(hmac(sha1),"
2196 					    "cbc(des3_ede)))",
2197 				.cra_driver_name = "echainiv-authenc-"
2198 						   "hmac-sha1-"
2199 						   "cbc-des3_ede-caam-qi2",
2200 				.cra_blocksize = DES3_EDE_BLOCK_SIZE,
2201 			},
2202 			.setkey = des3_aead_setkey,
2203 			.setauthsize = aead_setauthsize,
2204 			.encrypt = aead_encrypt,
2205 			.decrypt = aead_decrypt,
2206 			.ivsize = DES3_EDE_BLOCK_SIZE,
2207 			.maxauthsize = SHA1_DIGEST_SIZE,
2208 		},
2209 		.caam = {
2210 			.class1_alg_type = OP_ALG_ALGSEL_3DES | OP_ALG_AAI_CBC,
2211 			.class2_alg_type = OP_ALG_ALGSEL_SHA1 |
2212 					   OP_ALG_AAI_HMAC_PRECOMP,
2213 			.geniv = true,
2214 		}
2215 	},
2216 	{
2217 		.aead = {
2218 			.base = {
2219 				.cra_name = "authenc(hmac(sha224),"
2220 					    "cbc(des3_ede))",
2221 				.cra_driver_name = "authenc-hmac-sha224-"
2222 						   "cbc-des3_ede-caam-qi2",
2223 				.cra_blocksize = DES3_EDE_BLOCK_SIZE,
2224 			},
2225 			.setkey = des3_aead_setkey,
2226 			.setauthsize = aead_setauthsize,
2227 			.encrypt = aead_encrypt,
2228 			.decrypt = aead_decrypt,
2229 			.ivsize = DES3_EDE_BLOCK_SIZE,
2230 			.maxauthsize = SHA224_DIGEST_SIZE,
2231 		},
2232 		.caam = {
2233 			.class1_alg_type = OP_ALG_ALGSEL_3DES | OP_ALG_AAI_CBC,
2234 			.class2_alg_type = OP_ALG_ALGSEL_SHA224 |
2235 					   OP_ALG_AAI_HMAC_PRECOMP,
2236 		},
2237 	},
2238 	{
2239 		.aead = {
2240 			.base = {
2241 				.cra_name = "echainiv(authenc(hmac(sha224),"
2242 					    "cbc(des3_ede)))",
2243 				.cra_driver_name = "echainiv-authenc-"
2244 						   "hmac-sha224-"
2245 						   "cbc-des3_ede-caam-qi2",
2246 				.cra_blocksize = DES3_EDE_BLOCK_SIZE,
2247 			},
2248 			.setkey = des3_aead_setkey,
2249 			.setauthsize = aead_setauthsize,
2250 			.encrypt = aead_encrypt,
2251 			.decrypt = aead_decrypt,
2252 			.ivsize = DES3_EDE_BLOCK_SIZE,
2253 			.maxauthsize = SHA224_DIGEST_SIZE,
2254 		},
2255 		.caam = {
2256 			.class1_alg_type = OP_ALG_ALGSEL_3DES | OP_ALG_AAI_CBC,
2257 			.class2_alg_type = OP_ALG_ALGSEL_SHA224 |
2258 					   OP_ALG_AAI_HMAC_PRECOMP,
2259 			.geniv = true,
2260 		}
2261 	},
2262 	{
2263 		.aead = {
2264 			.base = {
2265 				.cra_name = "authenc(hmac(sha256),"
2266 					    "cbc(des3_ede))",
2267 				.cra_driver_name = "authenc-hmac-sha256-"
2268 						   "cbc-des3_ede-caam-qi2",
2269 				.cra_blocksize = DES3_EDE_BLOCK_SIZE,
2270 			},
2271 			.setkey = des3_aead_setkey,
2272 			.setauthsize = aead_setauthsize,
2273 			.encrypt = aead_encrypt,
2274 			.decrypt = aead_decrypt,
2275 			.ivsize = DES3_EDE_BLOCK_SIZE,
2276 			.maxauthsize = SHA256_DIGEST_SIZE,
2277 		},
2278 		.caam = {
2279 			.class1_alg_type = OP_ALG_ALGSEL_3DES | OP_ALG_AAI_CBC,
2280 			.class2_alg_type = OP_ALG_ALGSEL_SHA256 |
2281 					   OP_ALG_AAI_HMAC_PRECOMP,
2282 		},
2283 	},
2284 	{
2285 		.aead = {
2286 			.base = {
2287 				.cra_name = "echainiv(authenc(hmac(sha256),"
2288 					    "cbc(des3_ede)))",
2289 				.cra_driver_name = "echainiv-authenc-"
2290 						   "hmac-sha256-"
2291 						   "cbc-des3_ede-caam-qi2",
2292 				.cra_blocksize = DES3_EDE_BLOCK_SIZE,
2293 			},
2294 			.setkey = des3_aead_setkey,
2295 			.setauthsize = aead_setauthsize,
2296 			.encrypt = aead_encrypt,
2297 			.decrypt = aead_decrypt,
2298 			.ivsize = DES3_EDE_BLOCK_SIZE,
2299 			.maxauthsize = SHA256_DIGEST_SIZE,
2300 		},
2301 		.caam = {
2302 			.class1_alg_type = OP_ALG_ALGSEL_3DES | OP_ALG_AAI_CBC,
2303 			.class2_alg_type = OP_ALG_ALGSEL_SHA256 |
2304 					   OP_ALG_AAI_HMAC_PRECOMP,
2305 			.geniv = true,
2306 		}
2307 	},
2308 	{
2309 		.aead = {
2310 			.base = {
2311 				.cra_name = "authenc(hmac(sha384),"
2312 					    "cbc(des3_ede))",
2313 				.cra_driver_name = "authenc-hmac-sha384-"
2314 						   "cbc-des3_ede-caam-qi2",
2315 				.cra_blocksize = DES3_EDE_BLOCK_SIZE,
2316 			},
2317 			.setkey = des3_aead_setkey,
2318 			.setauthsize = aead_setauthsize,
2319 			.encrypt = aead_encrypt,
2320 			.decrypt = aead_decrypt,
2321 			.ivsize = DES3_EDE_BLOCK_SIZE,
2322 			.maxauthsize = SHA384_DIGEST_SIZE,
2323 		},
2324 		.caam = {
2325 			.class1_alg_type = OP_ALG_ALGSEL_3DES | OP_ALG_AAI_CBC,
2326 			.class2_alg_type = OP_ALG_ALGSEL_SHA384 |
2327 					   OP_ALG_AAI_HMAC_PRECOMP,
2328 		},
2329 	},
2330 	{
2331 		.aead = {
2332 			.base = {
2333 				.cra_name = "echainiv(authenc(hmac(sha384),"
2334 					    "cbc(des3_ede)))",
2335 				.cra_driver_name = "echainiv-authenc-"
2336 						   "hmac-sha384-"
2337 						   "cbc-des3_ede-caam-qi2",
2338 				.cra_blocksize = DES3_EDE_BLOCK_SIZE,
2339 			},
2340 			.setkey = des3_aead_setkey,
2341 			.setauthsize = aead_setauthsize,
2342 			.encrypt = aead_encrypt,
2343 			.decrypt = aead_decrypt,
2344 			.ivsize = DES3_EDE_BLOCK_SIZE,
2345 			.maxauthsize = SHA384_DIGEST_SIZE,
2346 		},
2347 		.caam = {
2348 			.class1_alg_type = OP_ALG_ALGSEL_3DES | OP_ALG_AAI_CBC,
2349 			.class2_alg_type = OP_ALG_ALGSEL_SHA384 |
2350 					   OP_ALG_AAI_HMAC_PRECOMP,
2351 			.geniv = true,
2352 		}
2353 	},
2354 	{
2355 		.aead = {
2356 			.base = {
2357 				.cra_name = "authenc(hmac(sha512),"
2358 					    "cbc(des3_ede))",
2359 				.cra_driver_name = "authenc-hmac-sha512-"
2360 						   "cbc-des3_ede-caam-qi2",
2361 				.cra_blocksize = DES3_EDE_BLOCK_SIZE,
2362 			},
2363 			.setkey = des3_aead_setkey,
2364 			.setauthsize = aead_setauthsize,
2365 			.encrypt = aead_encrypt,
2366 			.decrypt = aead_decrypt,
2367 			.ivsize = DES3_EDE_BLOCK_SIZE,
2368 			.maxauthsize = SHA512_DIGEST_SIZE,
2369 		},
2370 		.caam = {
2371 			.class1_alg_type = OP_ALG_ALGSEL_3DES | OP_ALG_AAI_CBC,
2372 			.class2_alg_type = OP_ALG_ALGSEL_SHA512 |
2373 					   OP_ALG_AAI_HMAC_PRECOMP,
2374 		},
2375 	},
2376 	{
2377 		.aead = {
2378 			.base = {
2379 				.cra_name = "echainiv(authenc(hmac(sha512),"
2380 					    "cbc(des3_ede)))",
2381 				.cra_driver_name = "echainiv-authenc-"
2382 						   "hmac-sha512-"
2383 						   "cbc-des3_ede-caam-qi2",
2384 				.cra_blocksize = DES3_EDE_BLOCK_SIZE,
2385 			},
2386 			.setkey = des3_aead_setkey,
2387 			.setauthsize = aead_setauthsize,
2388 			.encrypt = aead_encrypt,
2389 			.decrypt = aead_decrypt,
2390 			.ivsize = DES3_EDE_BLOCK_SIZE,
2391 			.maxauthsize = SHA512_DIGEST_SIZE,
2392 		},
2393 		.caam = {
2394 			.class1_alg_type = OP_ALG_ALGSEL_3DES | OP_ALG_AAI_CBC,
2395 			.class2_alg_type = OP_ALG_ALGSEL_SHA512 |
2396 					   OP_ALG_AAI_HMAC_PRECOMP,
2397 			.geniv = true,
2398 		}
2399 	},
2400 	{
2401 		.aead = {
2402 			.base = {
2403 				.cra_name = "authenc(hmac(md5),cbc(des))",
2404 				.cra_driver_name = "authenc-hmac-md5-"
2405 						   "cbc-des-caam-qi2",
2406 				.cra_blocksize = DES_BLOCK_SIZE,
2407 			},
2408 			.setkey = aead_setkey,
2409 			.setauthsize = aead_setauthsize,
2410 			.encrypt = aead_encrypt,
2411 			.decrypt = aead_decrypt,
2412 			.ivsize = DES_BLOCK_SIZE,
2413 			.maxauthsize = MD5_DIGEST_SIZE,
2414 		},
2415 		.caam = {
2416 			.class1_alg_type = OP_ALG_ALGSEL_DES | OP_ALG_AAI_CBC,
2417 			.class2_alg_type = OP_ALG_ALGSEL_MD5 |
2418 					   OP_ALG_AAI_HMAC_PRECOMP,
2419 		},
2420 	},
2421 	{
2422 		.aead = {
2423 			.base = {
2424 				.cra_name = "echainiv(authenc(hmac(md5),"
2425 					    "cbc(des)))",
2426 				.cra_driver_name = "echainiv-authenc-hmac-md5-"
2427 						   "cbc-des-caam-qi2",
2428 				.cra_blocksize = DES_BLOCK_SIZE,
2429 			},
2430 			.setkey = aead_setkey,
2431 			.setauthsize = aead_setauthsize,
2432 			.encrypt = aead_encrypt,
2433 			.decrypt = aead_decrypt,
2434 			.ivsize = DES_BLOCK_SIZE,
2435 			.maxauthsize = MD5_DIGEST_SIZE,
2436 		},
2437 		.caam = {
2438 			.class1_alg_type = OP_ALG_ALGSEL_DES | OP_ALG_AAI_CBC,
2439 			.class2_alg_type = OP_ALG_ALGSEL_MD5 |
2440 					   OP_ALG_AAI_HMAC_PRECOMP,
2441 			.geniv = true,
2442 		}
2443 	},
2444 	{
2445 		.aead = {
2446 			.base = {
2447 				.cra_name = "authenc(hmac(sha1),cbc(des))",
2448 				.cra_driver_name = "authenc-hmac-sha1-"
2449 						   "cbc-des-caam-qi2",
2450 				.cra_blocksize = DES_BLOCK_SIZE,
2451 			},
2452 			.setkey = aead_setkey,
2453 			.setauthsize = aead_setauthsize,
2454 			.encrypt = aead_encrypt,
2455 			.decrypt = aead_decrypt,
2456 			.ivsize = DES_BLOCK_SIZE,
2457 			.maxauthsize = SHA1_DIGEST_SIZE,
2458 		},
2459 		.caam = {
2460 			.class1_alg_type = OP_ALG_ALGSEL_DES | OP_ALG_AAI_CBC,
2461 			.class2_alg_type = OP_ALG_ALGSEL_SHA1 |
2462 					   OP_ALG_AAI_HMAC_PRECOMP,
2463 		},
2464 	},
2465 	{
2466 		.aead = {
2467 			.base = {
2468 				.cra_name = "echainiv(authenc(hmac(sha1),"
2469 					    "cbc(des)))",
2470 				.cra_driver_name = "echainiv-authenc-"
2471 						   "hmac-sha1-cbc-des-caam-qi2",
2472 				.cra_blocksize = DES_BLOCK_SIZE,
2473 			},
2474 			.setkey = aead_setkey,
2475 			.setauthsize = aead_setauthsize,
2476 			.encrypt = aead_encrypt,
2477 			.decrypt = aead_decrypt,
2478 			.ivsize = DES_BLOCK_SIZE,
2479 			.maxauthsize = SHA1_DIGEST_SIZE,
2480 		},
2481 		.caam = {
2482 			.class1_alg_type = OP_ALG_ALGSEL_DES | OP_ALG_AAI_CBC,
2483 			.class2_alg_type = OP_ALG_ALGSEL_SHA1 |
2484 					   OP_ALG_AAI_HMAC_PRECOMP,
2485 			.geniv = true,
2486 		}
2487 	},
2488 	{
2489 		.aead = {
2490 			.base = {
2491 				.cra_name = "authenc(hmac(sha224),cbc(des))",
2492 				.cra_driver_name = "authenc-hmac-sha224-"
2493 						   "cbc-des-caam-qi2",
2494 				.cra_blocksize = DES_BLOCK_SIZE,
2495 			},
2496 			.setkey = aead_setkey,
2497 			.setauthsize = aead_setauthsize,
2498 			.encrypt = aead_encrypt,
2499 			.decrypt = aead_decrypt,
2500 			.ivsize = DES_BLOCK_SIZE,
2501 			.maxauthsize = SHA224_DIGEST_SIZE,
2502 		},
2503 		.caam = {
2504 			.class1_alg_type = OP_ALG_ALGSEL_DES | OP_ALG_AAI_CBC,
2505 			.class2_alg_type = OP_ALG_ALGSEL_SHA224 |
2506 					   OP_ALG_AAI_HMAC_PRECOMP,
2507 		},
2508 	},
2509 	{
2510 		.aead = {
2511 			.base = {
2512 				.cra_name = "echainiv(authenc(hmac(sha224),"
2513 					    "cbc(des)))",
2514 				.cra_driver_name = "echainiv-authenc-"
2515 						   "hmac-sha224-cbc-des-"
2516 						   "caam-qi2",
2517 				.cra_blocksize = DES_BLOCK_SIZE,
2518 			},
2519 			.setkey = aead_setkey,
2520 			.setauthsize = aead_setauthsize,
2521 			.encrypt = aead_encrypt,
2522 			.decrypt = aead_decrypt,
2523 			.ivsize = DES_BLOCK_SIZE,
2524 			.maxauthsize = SHA224_DIGEST_SIZE,
2525 		},
2526 		.caam = {
2527 			.class1_alg_type = OP_ALG_ALGSEL_DES | OP_ALG_AAI_CBC,
2528 			.class2_alg_type = OP_ALG_ALGSEL_SHA224 |
2529 					   OP_ALG_AAI_HMAC_PRECOMP,
2530 			.geniv = true,
2531 		}
2532 	},
2533 	{
2534 		.aead = {
2535 			.base = {
2536 				.cra_name = "authenc(hmac(sha256),cbc(des))",
2537 				.cra_driver_name = "authenc-hmac-sha256-"
2538 						   "cbc-des-caam-qi2",
2539 				.cra_blocksize = DES_BLOCK_SIZE,
2540 			},
2541 			.setkey = aead_setkey,
2542 			.setauthsize = aead_setauthsize,
2543 			.encrypt = aead_encrypt,
2544 			.decrypt = aead_decrypt,
2545 			.ivsize = DES_BLOCK_SIZE,
2546 			.maxauthsize = SHA256_DIGEST_SIZE,
2547 		},
2548 		.caam = {
2549 			.class1_alg_type = OP_ALG_ALGSEL_DES | OP_ALG_AAI_CBC,
2550 			.class2_alg_type = OP_ALG_ALGSEL_SHA256 |
2551 					   OP_ALG_AAI_HMAC_PRECOMP,
2552 		},
2553 	},
2554 	{
2555 		.aead = {
2556 			.base = {
2557 				.cra_name = "echainiv(authenc(hmac(sha256),"
2558 					    "cbc(des)))",
2559 				.cra_driver_name = "echainiv-authenc-"
2560 						   "hmac-sha256-cbc-des-"
2561 						   "caam-qi2",
2562 				.cra_blocksize = DES_BLOCK_SIZE,
2563 			},
2564 			.setkey = aead_setkey,
2565 			.setauthsize = aead_setauthsize,
2566 			.encrypt = aead_encrypt,
2567 			.decrypt = aead_decrypt,
2568 			.ivsize = DES_BLOCK_SIZE,
2569 			.maxauthsize = SHA256_DIGEST_SIZE,
2570 		},
2571 		.caam = {
2572 			.class1_alg_type = OP_ALG_ALGSEL_DES | OP_ALG_AAI_CBC,
2573 			.class2_alg_type = OP_ALG_ALGSEL_SHA256 |
2574 					   OP_ALG_AAI_HMAC_PRECOMP,
2575 			.geniv = true,
2576 		},
2577 	},
2578 	{
2579 		.aead = {
2580 			.base = {
2581 				.cra_name = "authenc(hmac(sha384),cbc(des))",
2582 				.cra_driver_name = "authenc-hmac-sha384-"
2583 						   "cbc-des-caam-qi2",
2584 				.cra_blocksize = DES_BLOCK_SIZE,
2585 			},
2586 			.setkey = aead_setkey,
2587 			.setauthsize = aead_setauthsize,
2588 			.encrypt = aead_encrypt,
2589 			.decrypt = aead_decrypt,
2590 			.ivsize = DES_BLOCK_SIZE,
2591 			.maxauthsize = SHA384_DIGEST_SIZE,
2592 		},
2593 		.caam = {
2594 			.class1_alg_type = OP_ALG_ALGSEL_DES | OP_ALG_AAI_CBC,
2595 			.class2_alg_type = OP_ALG_ALGSEL_SHA384 |
2596 					   OP_ALG_AAI_HMAC_PRECOMP,
2597 		},
2598 	},
2599 	{
2600 		.aead = {
2601 			.base = {
2602 				.cra_name = "echainiv(authenc(hmac(sha384),"
2603 					    "cbc(des)))",
2604 				.cra_driver_name = "echainiv-authenc-"
2605 						   "hmac-sha384-cbc-des-"
2606 						   "caam-qi2",
2607 				.cra_blocksize = DES_BLOCK_SIZE,
2608 			},
2609 			.setkey = aead_setkey,
2610 			.setauthsize = aead_setauthsize,
2611 			.encrypt = aead_encrypt,
2612 			.decrypt = aead_decrypt,
2613 			.ivsize = DES_BLOCK_SIZE,
2614 			.maxauthsize = SHA384_DIGEST_SIZE,
2615 		},
2616 		.caam = {
2617 			.class1_alg_type = OP_ALG_ALGSEL_DES | OP_ALG_AAI_CBC,
2618 			.class2_alg_type = OP_ALG_ALGSEL_SHA384 |
2619 					   OP_ALG_AAI_HMAC_PRECOMP,
2620 			.geniv = true,
2621 		}
2622 	},
2623 	{
2624 		.aead = {
2625 			.base = {
2626 				.cra_name = "authenc(hmac(sha512),cbc(des))",
2627 				.cra_driver_name = "authenc-hmac-sha512-"
2628 						   "cbc-des-caam-qi2",
2629 				.cra_blocksize = DES_BLOCK_SIZE,
2630 			},
2631 			.setkey = aead_setkey,
2632 			.setauthsize = aead_setauthsize,
2633 			.encrypt = aead_encrypt,
2634 			.decrypt = aead_decrypt,
2635 			.ivsize = DES_BLOCK_SIZE,
2636 			.maxauthsize = SHA512_DIGEST_SIZE,
2637 		},
2638 		.caam = {
2639 			.class1_alg_type = OP_ALG_ALGSEL_DES | OP_ALG_AAI_CBC,
2640 			.class2_alg_type = OP_ALG_ALGSEL_SHA512 |
2641 					   OP_ALG_AAI_HMAC_PRECOMP,
2642 		}
2643 	},
2644 	{
2645 		.aead = {
2646 			.base = {
2647 				.cra_name = "echainiv(authenc(hmac(sha512),"
2648 					    "cbc(des)))",
2649 				.cra_driver_name = "echainiv-authenc-"
2650 						   "hmac-sha512-cbc-des-"
2651 						   "caam-qi2",
2652 				.cra_blocksize = DES_BLOCK_SIZE,
2653 			},
2654 			.setkey = aead_setkey,
2655 			.setauthsize = aead_setauthsize,
2656 			.encrypt = aead_encrypt,
2657 			.decrypt = aead_decrypt,
2658 			.ivsize = DES_BLOCK_SIZE,
2659 			.maxauthsize = SHA512_DIGEST_SIZE,
2660 		},
2661 		.caam = {
2662 			.class1_alg_type = OP_ALG_ALGSEL_DES | OP_ALG_AAI_CBC,
2663 			.class2_alg_type = OP_ALG_ALGSEL_SHA512 |
2664 					   OP_ALG_AAI_HMAC_PRECOMP,
2665 			.geniv = true,
2666 		}
2667 	},
2668 	{
2669 		.aead = {
2670 			.base = {
2671 				.cra_name = "authenc(hmac(md5),"
2672 					    "rfc3686(ctr(aes)))",
2673 				.cra_driver_name = "authenc-hmac-md5-"
2674 						   "rfc3686-ctr-aes-caam-qi2",
2675 				.cra_blocksize = 1,
2676 			},
2677 			.setkey = aead_setkey,
2678 			.setauthsize = aead_setauthsize,
2679 			.encrypt = aead_encrypt,
2680 			.decrypt = aead_decrypt,
2681 			.ivsize = CTR_RFC3686_IV_SIZE,
2682 			.maxauthsize = MD5_DIGEST_SIZE,
2683 		},
2684 		.caam = {
2685 			.class1_alg_type = OP_ALG_ALGSEL_AES |
2686 					   OP_ALG_AAI_CTR_MOD128,
2687 			.class2_alg_type = OP_ALG_ALGSEL_MD5 |
2688 					   OP_ALG_AAI_HMAC_PRECOMP,
2689 			.rfc3686 = true,
2690 		},
2691 	},
2692 	{
2693 		.aead = {
2694 			.base = {
2695 				.cra_name = "seqiv(authenc("
2696 					    "hmac(md5),rfc3686(ctr(aes))))",
2697 				.cra_driver_name = "seqiv-authenc-hmac-md5-"
2698 						   "rfc3686-ctr-aes-caam-qi2",
2699 				.cra_blocksize = 1,
2700 			},
2701 			.setkey = aead_setkey,
2702 			.setauthsize = aead_setauthsize,
2703 			.encrypt = aead_encrypt,
2704 			.decrypt = aead_decrypt,
2705 			.ivsize = CTR_RFC3686_IV_SIZE,
2706 			.maxauthsize = MD5_DIGEST_SIZE,
2707 		},
2708 		.caam = {
2709 			.class1_alg_type = OP_ALG_ALGSEL_AES |
2710 					   OP_ALG_AAI_CTR_MOD128,
2711 			.class2_alg_type = OP_ALG_ALGSEL_MD5 |
2712 					   OP_ALG_AAI_HMAC_PRECOMP,
2713 			.rfc3686 = true,
2714 			.geniv = true,
2715 		},
2716 	},
2717 	{
2718 		.aead = {
2719 			.base = {
2720 				.cra_name = "authenc(hmac(sha1),"
2721 					    "rfc3686(ctr(aes)))",
2722 				.cra_driver_name = "authenc-hmac-sha1-"
2723 						   "rfc3686-ctr-aes-caam-qi2",
2724 				.cra_blocksize = 1,
2725 			},
2726 			.setkey = aead_setkey,
2727 			.setauthsize = aead_setauthsize,
2728 			.encrypt = aead_encrypt,
2729 			.decrypt = aead_decrypt,
2730 			.ivsize = CTR_RFC3686_IV_SIZE,
2731 			.maxauthsize = SHA1_DIGEST_SIZE,
2732 		},
2733 		.caam = {
2734 			.class1_alg_type = OP_ALG_ALGSEL_AES |
2735 					   OP_ALG_AAI_CTR_MOD128,
2736 			.class2_alg_type = OP_ALG_ALGSEL_SHA1 |
2737 					   OP_ALG_AAI_HMAC_PRECOMP,
2738 			.rfc3686 = true,
2739 		},
2740 	},
2741 	{
2742 		.aead = {
2743 			.base = {
2744 				.cra_name = "seqiv(authenc("
2745 					    "hmac(sha1),rfc3686(ctr(aes))))",
2746 				.cra_driver_name = "seqiv-authenc-hmac-sha1-"
2747 						   "rfc3686-ctr-aes-caam-qi2",
2748 				.cra_blocksize = 1,
2749 			},
2750 			.setkey = aead_setkey,
2751 			.setauthsize = aead_setauthsize,
2752 			.encrypt = aead_encrypt,
2753 			.decrypt = aead_decrypt,
2754 			.ivsize = CTR_RFC3686_IV_SIZE,
2755 			.maxauthsize = SHA1_DIGEST_SIZE,
2756 		},
2757 		.caam = {
2758 			.class1_alg_type = OP_ALG_ALGSEL_AES |
2759 					   OP_ALG_AAI_CTR_MOD128,
2760 			.class2_alg_type = OP_ALG_ALGSEL_SHA1 |
2761 					   OP_ALG_AAI_HMAC_PRECOMP,
2762 			.rfc3686 = true,
2763 			.geniv = true,
2764 		},
2765 	},
2766 	{
2767 		.aead = {
2768 			.base = {
2769 				.cra_name = "authenc(hmac(sha224),"
2770 					    "rfc3686(ctr(aes)))",
2771 				.cra_driver_name = "authenc-hmac-sha224-"
2772 						   "rfc3686-ctr-aes-caam-qi2",
2773 				.cra_blocksize = 1,
2774 			},
2775 			.setkey = aead_setkey,
2776 			.setauthsize = aead_setauthsize,
2777 			.encrypt = aead_encrypt,
2778 			.decrypt = aead_decrypt,
2779 			.ivsize = CTR_RFC3686_IV_SIZE,
2780 			.maxauthsize = SHA224_DIGEST_SIZE,
2781 		},
2782 		.caam = {
2783 			.class1_alg_type = OP_ALG_ALGSEL_AES |
2784 					   OP_ALG_AAI_CTR_MOD128,
2785 			.class2_alg_type = OP_ALG_ALGSEL_SHA224 |
2786 					   OP_ALG_AAI_HMAC_PRECOMP,
2787 			.rfc3686 = true,
2788 		},
2789 	},
2790 	{
2791 		.aead = {
2792 			.base = {
2793 				.cra_name = "seqiv(authenc("
2794 					    "hmac(sha224),rfc3686(ctr(aes))))",
2795 				.cra_driver_name = "seqiv-authenc-hmac-sha224-"
2796 						   "rfc3686-ctr-aes-caam-qi2",
2797 				.cra_blocksize = 1,
2798 			},
2799 			.setkey = aead_setkey,
2800 			.setauthsize = aead_setauthsize,
2801 			.encrypt = aead_encrypt,
2802 			.decrypt = aead_decrypt,
2803 			.ivsize = CTR_RFC3686_IV_SIZE,
2804 			.maxauthsize = SHA224_DIGEST_SIZE,
2805 		},
2806 		.caam = {
2807 			.class1_alg_type = OP_ALG_ALGSEL_AES |
2808 					   OP_ALG_AAI_CTR_MOD128,
2809 			.class2_alg_type = OP_ALG_ALGSEL_SHA224 |
2810 					   OP_ALG_AAI_HMAC_PRECOMP,
2811 			.rfc3686 = true,
2812 			.geniv = true,
2813 		},
2814 	},
2815 	{
2816 		.aead = {
2817 			.base = {
2818 				.cra_name = "authenc(hmac(sha256),"
2819 					    "rfc3686(ctr(aes)))",
2820 				.cra_driver_name = "authenc-hmac-sha256-"
2821 						   "rfc3686-ctr-aes-caam-qi2",
2822 				.cra_blocksize = 1,
2823 			},
2824 			.setkey = aead_setkey,
2825 			.setauthsize = aead_setauthsize,
2826 			.encrypt = aead_encrypt,
2827 			.decrypt = aead_decrypt,
2828 			.ivsize = CTR_RFC3686_IV_SIZE,
2829 			.maxauthsize = SHA256_DIGEST_SIZE,
2830 		},
2831 		.caam = {
2832 			.class1_alg_type = OP_ALG_ALGSEL_AES |
2833 					   OP_ALG_AAI_CTR_MOD128,
2834 			.class2_alg_type = OP_ALG_ALGSEL_SHA256 |
2835 					   OP_ALG_AAI_HMAC_PRECOMP,
2836 			.rfc3686 = true,
2837 		},
2838 	},
2839 	{
2840 		.aead = {
2841 			.base = {
2842 				.cra_name = "seqiv(authenc(hmac(sha256),"
2843 					    "rfc3686(ctr(aes))))",
2844 				.cra_driver_name = "seqiv-authenc-hmac-sha256-"
2845 						   "rfc3686-ctr-aes-caam-qi2",
2846 				.cra_blocksize = 1,
2847 			},
2848 			.setkey = aead_setkey,
2849 			.setauthsize = aead_setauthsize,
2850 			.encrypt = aead_encrypt,
2851 			.decrypt = aead_decrypt,
2852 			.ivsize = CTR_RFC3686_IV_SIZE,
2853 			.maxauthsize = SHA256_DIGEST_SIZE,
2854 		},
2855 		.caam = {
2856 			.class1_alg_type = OP_ALG_ALGSEL_AES |
2857 					   OP_ALG_AAI_CTR_MOD128,
2858 			.class2_alg_type = OP_ALG_ALGSEL_SHA256 |
2859 					   OP_ALG_AAI_HMAC_PRECOMP,
2860 			.rfc3686 = true,
2861 			.geniv = true,
2862 		},
2863 	},
2864 	{
2865 		.aead = {
2866 			.base = {
2867 				.cra_name = "authenc(hmac(sha384),"
2868 					    "rfc3686(ctr(aes)))",
2869 				.cra_driver_name = "authenc-hmac-sha384-"
2870 						   "rfc3686-ctr-aes-caam-qi2",
2871 				.cra_blocksize = 1,
2872 			},
2873 			.setkey = aead_setkey,
2874 			.setauthsize = aead_setauthsize,
2875 			.encrypt = aead_encrypt,
2876 			.decrypt = aead_decrypt,
2877 			.ivsize = CTR_RFC3686_IV_SIZE,
2878 			.maxauthsize = SHA384_DIGEST_SIZE,
2879 		},
2880 		.caam = {
2881 			.class1_alg_type = OP_ALG_ALGSEL_AES |
2882 					   OP_ALG_AAI_CTR_MOD128,
2883 			.class2_alg_type = OP_ALG_ALGSEL_SHA384 |
2884 					   OP_ALG_AAI_HMAC_PRECOMP,
2885 			.rfc3686 = true,
2886 		},
2887 	},
2888 	{
2889 		.aead = {
2890 			.base = {
2891 				.cra_name = "seqiv(authenc(hmac(sha384),"
2892 					    "rfc3686(ctr(aes))))",
2893 				.cra_driver_name = "seqiv-authenc-hmac-sha384-"
2894 						   "rfc3686-ctr-aes-caam-qi2",
2895 				.cra_blocksize = 1,
2896 			},
2897 			.setkey = aead_setkey,
2898 			.setauthsize = aead_setauthsize,
2899 			.encrypt = aead_encrypt,
2900 			.decrypt = aead_decrypt,
2901 			.ivsize = CTR_RFC3686_IV_SIZE,
2902 			.maxauthsize = SHA384_DIGEST_SIZE,
2903 		},
2904 		.caam = {
2905 			.class1_alg_type = OP_ALG_ALGSEL_AES |
2906 					   OP_ALG_AAI_CTR_MOD128,
2907 			.class2_alg_type = OP_ALG_ALGSEL_SHA384 |
2908 					   OP_ALG_AAI_HMAC_PRECOMP,
2909 			.rfc3686 = true,
2910 			.geniv = true,
2911 		},
2912 	},
2913 	{
2914 		.aead = {
2915 			.base = {
2916 				.cra_name = "rfc7539(chacha20,poly1305)",
2917 				.cra_driver_name = "rfc7539-chacha20-poly1305-"
2918 						   "caam-qi2",
2919 				.cra_blocksize = 1,
2920 			},
2921 			.setkey = chachapoly_setkey,
2922 			.setauthsize = chachapoly_setauthsize,
2923 			.encrypt = aead_encrypt,
2924 			.decrypt = aead_decrypt,
2925 			.ivsize = CHACHAPOLY_IV_SIZE,
2926 			.maxauthsize = POLY1305_DIGEST_SIZE,
2927 		},
2928 		.caam = {
2929 			.class1_alg_type = OP_ALG_ALGSEL_CHACHA20 |
2930 					   OP_ALG_AAI_AEAD,
2931 			.class2_alg_type = OP_ALG_ALGSEL_POLY1305 |
2932 					   OP_ALG_AAI_AEAD,
2933 			.nodkp = true,
2934 		},
2935 	},
2936 	{
2937 		.aead = {
2938 			.base = {
2939 				.cra_name = "rfc7539esp(chacha20,poly1305)",
2940 				.cra_driver_name = "rfc7539esp-chacha20-"
2941 						   "poly1305-caam-qi2",
2942 				.cra_blocksize = 1,
2943 			},
2944 			.setkey = chachapoly_setkey,
2945 			.setauthsize = chachapoly_setauthsize,
2946 			.encrypt = aead_encrypt,
2947 			.decrypt = aead_decrypt,
2948 			.ivsize = 8,
2949 			.maxauthsize = POLY1305_DIGEST_SIZE,
2950 		},
2951 		.caam = {
2952 			.class1_alg_type = OP_ALG_ALGSEL_CHACHA20 |
2953 					   OP_ALG_AAI_AEAD,
2954 			.class2_alg_type = OP_ALG_ALGSEL_POLY1305 |
2955 					   OP_ALG_AAI_AEAD,
2956 			.nodkp = true,
2957 		},
2958 	},
2959 	{
2960 		.aead = {
2961 			.base = {
2962 				.cra_name = "authenc(hmac(sha512),"
2963 					    "rfc3686(ctr(aes)))",
2964 				.cra_driver_name = "authenc-hmac-sha512-"
2965 						   "rfc3686-ctr-aes-caam-qi2",
2966 				.cra_blocksize = 1,
2967 			},
2968 			.setkey = aead_setkey,
2969 			.setauthsize = aead_setauthsize,
2970 			.encrypt = aead_encrypt,
2971 			.decrypt = aead_decrypt,
2972 			.ivsize = CTR_RFC3686_IV_SIZE,
2973 			.maxauthsize = SHA512_DIGEST_SIZE,
2974 		},
2975 		.caam = {
2976 			.class1_alg_type = OP_ALG_ALGSEL_AES |
2977 					   OP_ALG_AAI_CTR_MOD128,
2978 			.class2_alg_type = OP_ALG_ALGSEL_SHA512 |
2979 					   OP_ALG_AAI_HMAC_PRECOMP,
2980 			.rfc3686 = true,
2981 		},
2982 	},
2983 	{
2984 		.aead = {
2985 			.base = {
2986 				.cra_name = "seqiv(authenc(hmac(sha512),"
2987 					    "rfc3686(ctr(aes))))",
2988 				.cra_driver_name = "seqiv-authenc-hmac-sha512-"
2989 						   "rfc3686-ctr-aes-caam-qi2",
2990 				.cra_blocksize = 1,
2991 			},
2992 			.setkey = aead_setkey,
2993 			.setauthsize = aead_setauthsize,
2994 			.encrypt = aead_encrypt,
2995 			.decrypt = aead_decrypt,
2996 			.ivsize = CTR_RFC3686_IV_SIZE,
2997 			.maxauthsize = SHA512_DIGEST_SIZE,
2998 		},
2999 		.caam = {
3000 			.class1_alg_type = OP_ALG_ALGSEL_AES |
3001 					   OP_ALG_AAI_CTR_MOD128,
3002 			.class2_alg_type = OP_ALG_ALGSEL_SHA512 |
3003 					   OP_ALG_AAI_HMAC_PRECOMP,
3004 			.rfc3686 = true,
3005 			.geniv = true,
3006 		},
3007 	},
3008 };
3009 
3010 static void caam_skcipher_alg_init(struct caam_skcipher_alg *t_alg)
3011 {
3012 	struct skcipher_alg *alg = &t_alg->skcipher;
3013 
3014 	alg->base.cra_module = THIS_MODULE;
3015 	alg->base.cra_priority = CAAM_CRA_PRIORITY;
3016 	alg->base.cra_ctxsize = sizeof(struct caam_ctx) + crypto_dma_padding();
3017 	alg->base.cra_flags |= (CRYPTO_ALG_ASYNC | CRYPTO_ALG_ALLOCATES_MEMORY |
3018 			      CRYPTO_ALG_KERN_DRIVER_ONLY);
3019 
3020 	alg->init = caam_cra_init_skcipher;
3021 	alg->exit = caam_cra_exit;
3022 }
3023 
3024 static void caam_aead_alg_init(struct caam_aead_alg *t_alg)
3025 {
3026 	struct aead_alg *alg = &t_alg->aead;
3027 
3028 	alg->base.cra_module = THIS_MODULE;
3029 	alg->base.cra_priority = CAAM_CRA_PRIORITY;
3030 	alg->base.cra_ctxsize = sizeof(struct caam_ctx) + crypto_dma_padding();
3031 	alg->base.cra_flags = CRYPTO_ALG_ASYNC | CRYPTO_ALG_ALLOCATES_MEMORY |
3032 			      CRYPTO_ALG_KERN_DRIVER_ONLY;
3033 
3034 	alg->init = caam_cra_init_aead;
3035 	alg->exit = caam_cra_exit_aead;
3036 }
3037 
3038 /* max hash key is max split key size */
3039 #define CAAM_MAX_HASH_KEY_SIZE		(SHA512_DIGEST_SIZE * 2)
3040 
3041 #define CAAM_MAX_HASH_BLOCK_SIZE	SHA512_BLOCK_SIZE
3042 
3043 /* caam context sizes for hashes: running digest + 8 */
3044 #define HASH_MSG_LEN			8
3045 #define MAX_CTX_LEN			(HASH_MSG_LEN + SHA512_DIGEST_SIZE)
3046 
3047 enum hash_optype {
3048 	UPDATE = 0,
3049 	UPDATE_FIRST,
3050 	FINALIZE,
3051 	DIGEST,
3052 	HASH_NUM_OP
3053 };
3054 
3055 /**
3056  * struct caam_hash_ctx - ahash per-session context
3057  * @flc: Flow Contexts array
3058  * @key: authentication key
3059  * @flc_dma: I/O virtual addresses of the Flow Contexts
3060  * @dev: dpseci device
3061  * @ctx_len: size of Context Register
3062  * @adata: hashing algorithm details
3063  */
3064 struct caam_hash_ctx {
3065 	struct caam_flc flc[HASH_NUM_OP];
3066 	u8 key[CAAM_MAX_HASH_BLOCK_SIZE] ____cacheline_aligned;
3067 	dma_addr_t flc_dma[HASH_NUM_OP];
3068 	struct device *dev;
3069 	int ctx_len;
3070 	struct alginfo adata;
3071 };
3072 
3073 /* ahash state */
3074 struct caam_hash_state {
3075 	struct caam_request caam_req;
3076 	dma_addr_t buf_dma;
3077 	dma_addr_t ctx_dma;
3078 	int ctx_dma_len;
3079 	u8 buf[CAAM_MAX_HASH_BLOCK_SIZE] ____cacheline_aligned;
3080 	int buflen;
3081 	int next_buflen;
3082 	u8 caam_ctx[MAX_CTX_LEN] ____cacheline_aligned;
3083 	int (*update)(struct ahash_request *req);
3084 	int (*final)(struct ahash_request *req);
3085 	int (*finup)(struct ahash_request *req);
3086 };
3087 
3088 struct caam_export_state {
3089 	u8 buf[CAAM_MAX_HASH_BLOCK_SIZE];
3090 	u8 caam_ctx[MAX_CTX_LEN];
3091 	int buflen;
3092 	int (*update)(struct ahash_request *req);
3093 	int (*final)(struct ahash_request *req);
3094 	int (*finup)(struct ahash_request *req);
3095 };
3096 
3097 /* Map current buffer in state (if length > 0) and put it in link table */
3098 static inline int buf_map_to_qm_sg(struct device *dev,
3099 				   struct dpaa2_sg_entry *qm_sg,
3100 				   struct caam_hash_state *state)
3101 {
3102 	int buflen = state->buflen;
3103 
3104 	if (!buflen)
3105 		return 0;
3106 
3107 	state->buf_dma = dma_map_single(dev, state->buf, buflen,
3108 					DMA_TO_DEVICE);
3109 	if (dma_mapping_error(dev, state->buf_dma)) {
3110 		dev_err(dev, "unable to map buf\n");
3111 		state->buf_dma = 0;
3112 		return -ENOMEM;
3113 	}
3114 
3115 	dma_to_qm_sg_one(qm_sg, state->buf_dma, buflen, 0);
3116 
3117 	return 0;
3118 }
3119 
3120 /* Map state->caam_ctx, and add it to link table */
3121 static inline int ctx_map_to_qm_sg(struct device *dev,
3122 				   struct caam_hash_state *state, int ctx_len,
3123 				   struct dpaa2_sg_entry *qm_sg, u32 flag)
3124 {
3125 	state->ctx_dma_len = ctx_len;
3126 	state->ctx_dma = dma_map_single(dev, state->caam_ctx, ctx_len, flag);
3127 	if (dma_mapping_error(dev, state->ctx_dma)) {
3128 		dev_err(dev, "unable to map ctx\n");
3129 		state->ctx_dma = 0;
3130 		return -ENOMEM;
3131 	}
3132 
3133 	dma_to_qm_sg_one(qm_sg, state->ctx_dma, ctx_len, 0);
3134 
3135 	return 0;
3136 }
3137 
3138 static int ahash_set_sh_desc(struct crypto_ahash *ahash)
3139 {
3140 	struct caam_hash_ctx *ctx = crypto_ahash_ctx_dma(ahash);
3141 	int digestsize = crypto_ahash_digestsize(ahash);
3142 	struct dpaa2_caam_priv *priv = dev_get_drvdata(ctx->dev);
3143 	struct caam_flc *flc;
3144 	u32 *desc;
3145 
3146 	/* ahash_update shared descriptor */
3147 	flc = &ctx->flc[UPDATE];
3148 	desc = flc->sh_desc;
3149 	cnstr_shdsc_ahash(desc, &ctx->adata, OP_ALG_AS_UPDATE, ctx->ctx_len,
3150 			  ctx->ctx_len, true, priv->sec_attr.era);
3151 	flc->flc[1] = cpu_to_caam32(desc_len(desc)); /* SDL */
3152 	dma_sync_single_for_device(ctx->dev, ctx->flc_dma[UPDATE],
3153 				   desc_bytes(desc), DMA_BIDIRECTIONAL);
3154 	print_hex_dump_debug("ahash update shdesc@" __stringify(__LINE__)": ",
3155 			     DUMP_PREFIX_ADDRESS, 16, 4, desc, desc_bytes(desc),
3156 			     1);
3157 
3158 	/* ahash_update_first shared descriptor */
3159 	flc = &ctx->flc[UPDATE_FIRST];
3160 	desc = flc->sh_desc;
3161 	cnstr_shdsc_ahash(desc, &ctx->adata, OP_ALG_AS_INIT, ctx->ctx_len,
3162 			  ctx->ctx_len, false, priv->sec_attr.era);
3163 	flc->flc[1] = cpu_to_caam32(desc_len(desc)); /* SDL */
3164 	dma_sync_single_for_device(ctx->dev, ctx->flc_dma[UPDATE_FIRST],
3165 				   desc_bytes(desc), DMA_BIDIRECTIONAL);
3166 	print_hex_dump_debug("ahash update first shdesc@" __stringify(__LINE__)": ",
3167 			     DUMP_PREFIX_ADDRESS, 16, 4, desc, desc_bytes(desc),
3168 			     1);
3169 
3170 	/* ahash_final shared descriptor */
3171 	flc = &ctx->flc[FINALIZE];
3172 	desc = flc->sh_desc;
3173 	cnstr_shdsc_ahash(desc, &ctx->adata, OP_ALG_AS_FINALIZE, digestsize,
3174 			  ctx->ctx_len, true, priv->sec_attr.era);
3175 	flc->flc[1] = cpu_to_caam32(desc_len(desc)); /* SDL */
3176 	dma_sync_single_for_device(ctx->dev, ctx->flc_dma[FINALIZE],
3177 				   desc_bytes(desc), DMA_BIDIRECTIONAL);
3178 	print_hex_dump_debug("ahash final shdesc@" __stringify(__LINE__)": ",
3179 			     DUMP_PREFIX_ADDRESS, 16, 4, desc, desc_bytes(desc),
3180 			     1);
3181 
3182 	/* ahash_digest shared descriptor */
3183 	flc = &ctx->flc[DIGEST];
3184 	desc = flc->sh_desc;
3185 	cnstr_shdsc_ahash(desc, &ctx->adata, OP_ALG_AS_INITFINAL, digestsize,
3186 			  ctx->ctx_len, false, priv->sec_attr.era);
3187 	flc->flc[1] = cpu_to_caam32(desc_len(desc)); /* SDL */
3188 	dma_sync_single_for_device(ctx->dev, ctx->flc_dma[DIGEST],
3189 				   desc_bytes(desc), DMA_BIDIRECTIONAL);
3190 	print_hex_dump_debug("ahash digest shdesc@" __stringify(__LINE__)": ",
3191 			     DUMP_PREFIX_ADDRESS, 16, 4, desc, desc_bytes(desc),
3192 			     1);
3193 
3194 	return 0;
3195 }
3196 
3197 struct split_key_sh_result {
3198 	struct completion completion;
3199 	int err;
3200 	struct device *dev;
3201 };
3202 
3203 static void split_key_sh_done(void *cbk_ctx, u32 err)
3204 {
3205 	struct split_key_sh_result *res = cbk_ctx;
3206 
3207 	dev_dbg(res->dev, "%s %d: err 0x%x\n", __func__, __LINE__, err);
3208 
3209 	res->err = err ? caam_qi2_strstatus(res->dev, err) : 0;
3210 	complete(&res->completion);
3211 }
3212 
3213 /* Digest hash size if it is too large */
3214 static int hash_digest_key(struct caam_hash_ctx *ctx, u32 *keylen, u8 *key,
3215 			   u32 digestsize)
3216 {
3217 	struct caam_request *req_ctx;
3218 	u32 *desc;
3219 	struct split_key_sh_result result;
3220 	dma_addr_t key_dma;
3221 	struct caam_flc *flc;
3222 	dma_addr_t flc_dma;
3223 	int ret = -ENOMEM;
3224 	struct dpaa2_fl_entry *in_fle, *out_fle;
3225 
3226 	req_ctx = kzalloc(sizeof(*req_ctx), GFP_KERNEL);
3227 	if (!req_ctx)
3228 		return -ENOMEM;
3229 
3230 	in_fle = &req_ctx->fd_flt[1];
3231 	out_fle = &req_ctx->fd_flt[0];
3232 
3233 	flc = kzalloc(sizeof(*flc), GFP_KERNEL);
3234 	if (!flc)
3235 		goto err_flc;
3236 
3237 	key_dma = dma_map_single(ctx->dev, key, *keylen, DMA_BIDIRECTIONAL);
3238 	if (dma_mapping_error(ctx->dev, key_dma)) {
3239 		dev_err(ctx->dev, "unable to map key memory\n");
3240 		goto err_key_dma;
3241 	}
3242 
3243 	desc = flc->sh_desc;
3244 
3245 	init_sh_desc(desc, 0);
3246 
3247 	/* descriptor to perform unkeyed hash on key_in */
3248 	append_operation(desc, ctx->adata.algtype | OP_ALG_ENCRYPT |
3249 			 OP_ALG_AS_INITFINAL);
3250 	append_seq_fifo_load(desc, *keylen, FIFOLD_CLASS_CLASS2 |
3251 			     FIFOLD_TYPE_LAST2 | FIFOLD_TYPE_MSG);
3252 	append_seq_store(desc, digestsize, LDST_CLASS_2_CCB |
3253 			 LDST_SRCDST_BYTE_CONTEXT);
3254 
3255 	flc->flc[1] = cpu_to_caam32(desc_len(desc)); /* SDL */
3256 	flc_dma = dma_map_single(ctx->dev, flc, sizeof(flc->flc) +
3257 				 desc_bytes(desc), DMA_TO_DEVICE);
3258 	if (dma_mapping_error(ctx->dev, flc_dma)) {
3259 		dev_err(ctx->dev, "unable to map shared descriptor\n");
3260 		goto err_flc_dma;
3261 	}
3262 
3263 	dpaa2_fl_set_final(in_fle, true);
3264 	dpaa2_fl_set_format(in_fle, dpaa2_fl_single);
3265 	dpaa2_fl_set_addr(in_fle, key_dma);
3266 	dpaa2_fl_set_len(in_fle, *keylen);
3267 	dpaa2_fl_set_format(out_fle, dpaa2_fl_single);
3268 	dpaa2_fl_set_addr(out_fle, key_dma);
3269 	dpaa2_fl_set_len(out_fle, digestsize);
3270 
3271 	print_hex_dump_debug("key_in@" __stringify(__LINE__)": ",
3272 			     DUMP_PREFIX_ADDRESS, 16, 4, key, *keylen, 1);
3273 	print_hex_dump_debug("shdesc@" __stringify(__LINE__)": ",
3274 			     DUMP_PREFIX_ADDRESS, 16, 4, desc, desc_bytes(desc),
3275 			     1);
3276 
3277 	result.err = 0;
3278 	init_completion(&result.completion);
3279 	result.dev = ctx->dev;
3280 
3281 	req_ctx->flc = flc;
3282 	req_ctx->flc_dma = flc_dma;
3283 	req_ctx->cbk = split_key_sh_done;
3284 	req_ctx->ctx = &result;
3285 
3286 	ret = dpaa2_caam_enqueue(ctx->dev, req_ctx);
3287 	if (ret == -EINPROGRESS) {
3288 		/* in progress */
3289 		wait_for_completion(&result.completion);
3290 		ret = result.err;
3291 		print_hex_dump_debug("digested key@" __stringify(__LINE__)": ",
3292 				     DUMP_PREFIX_ADDRESS, 16, 4, key,
3293 				     digestsize, 1);
3294 	}
3295 
3296 	dma_unmap_single(ctx->dev, flc_dma, sizeof(flc->flc) + desc_bytes(desc),
3297 			 DMA_TO_DEVICE);
3298 err_flc_dma:
3299 	dma_unmap_single(ctx->dev, key_dma, *keylen, DMA_BIDIRECTIONAL);
3300 err_key_dma:
3301 	kfree(flc);
3302 err_flc:
3303 	kfree(req_ctx);
3304 
3305 	*keylen = digestsize;
3306 
3307 	return ret;
3308 }
3309 
3310 static int ahash_setkey(struct crypto_ahash *ahash, const u8 *key,
3311 			unsigned int keylen)
3312 {
3313 	struct caam_hash_ctx *ctx = crypto_ahash_ctx_dma(ahash);
3314 	unsigned int blocksize = crypto_tfm_alg_blocksize(&ahash->base);
3315 	unsigned int digestsize = crypto_ahash_digestsize(ahash);
3316 	int ret;
3317 	u8 *hashed_key = NULL;
3318 
3319 	dev_dbg(ctx->dev, "keylen %d blocksize %d\n", keylen, blocksize);
3320 
3321 	if (keylen > blocksize) {
3322 		unsigned int aligned_len =
3323 			ALIGN(keylen, dma_get_cache_alignment());
3324 
3325 		if (aligned_len < keylen)
3326 			return -EOVERFLOW;
3327 
3328 		hashed_key = kmemdup(key, aligned_len, GFP_KERNEL);
3329 		if (!hashed_key)
3330 			return -ENOMEM;
3331 		ret = hash_digest_key(ctx, &keylen, hashed_key, digestsize);
3332 		if (ret)
3333 			goto bad_free_key;
3334 		key = hashed_key;
3335 	}
3336 
3337 	ctx->adata.keylen = keylen;
3338 	ctx->adata.keylen_pad = split_key_len(ctx->adata.algtype &
3339 					      OP_ALG_ALGSEL_MASK);
3340 	if (ctx->adata.keylen_pad > CAAM_MAX_HASH_KEY_SIZE)
3341 		goto bad_free_key;
3342 
3343 	ctx->adata.key_virt = key;
3344 	ctx->adata.key_inline = true;
3345 
3346 	/*
3347 	 * In case |user key| > |derived key|, using DKP<imm,imm> would result
3348 	 * in invalid opcodes (last bytes of user key) in the resulting
3349 	 * descriptor. Use DKP<ptr,imm> instead => both virtual and dma key
3350 	 * addresses are needed.
3351 	 */
3352 	if (keylen > ctx->adata.keylen_pad) {
3353 		memcpy(ctx->key, key, keylen);
3354 		dma_sync_single_for_device(ctx->dev, ctx->adata.key_dma,
3355 					   ctx->adata.keylen_pad,
3356 					   DMA_TO_DEVICE);
3357 	}
3358 
3359 	ret = ahash_set_sh_desc(ahash);
3360 	kfree(hashed_key);
3361 	return ret;
3362 bad_free_key:
3363 	kfree(hashed_key);
3364 	return -EINVAL;
3365 }
3366 
3367 static inline void ahash_unmap(struct device *dev, struct ahash_edesc *edesc,
3368 			       struct ahash_request *req)
3369 {
3370 	struct caam_hash_state *state = ahash_request_ctx_dma(req);
3371 
3372 	if (edesc->src_nents)
3373 		dma_unmap_sg(dev, req->src, edesc->src_nents, DMA_TO_DEVICE);
3374 
3375 	if (edesc->qm_sg_bytes)
3376 		dma_unmap_single(dev, edesc->qm_sg_dma, edesc->qm_sg_bytes,
3377 				 DMA_TO_DEVICE);
3378 
3379 	if (state->buf_dma) {
3380 		dma_unmap_single(dev, state->buf_dma, state->buflen,
3381 				 DMA_TO_DEVICE);
3382 		state->buf_dma = 0;
3383 	}
3384 }
3385 
3386 static inline void ahash_unmap_ctx(struct device *dev,
3387 				   struct ahash_edesc *edesc,
3388 				   struct ahash_request *req, u32 flag)
3389 {
3390 	struct caam_hash_state *state = ahash_request_ctx_dma(req);
3391 
3392 	if (state->ctx_dma) {
3393 		dma_unmap_single(dev, state->ctx_dma, state->ctx_dma_len, flag);
3394 		state->ctx_dma = 0;
3395 	}
3396 	ahash_unmap(dev, edesc, req);
3397 }
3398 
3399 static void ahash_done(void *cbk_ctx, u32 status)
3400 {
3401 	struct crypto_async_request *areq = cbk_ctx;
3402 	struct ahash_request *req = ahash_request_cast(areq);
3403 	struct crypto_ahash *ahash = crypto_ahash_reqtfm(req);
3404 	struct caam_hash_state *state = ahash_request_ctx_dma(req);
3405 	struct ahash_edesc *edesc = state->caam_req.edesc;
3406 	struct caam_hash_ctx *ctx = crypto_ahash_ctx_dma(ahash);
3407 	int digestsize = crypto_ahash_digestsize(ahash);
3408 	int ecode = 0;
3409 
3410 	dev_dbg(ctx->dev, "%s %d: err 0x%x\n", __func__, __LINE__, status);
3411 
3412 	if (unlikely(status))
3413 		ecode = caam_qi2_strstatus(ctx->dev, status);
3414 
3415 	ahash_unmap_ctx(ctx->dev, edesc, req, DMA_FROM_DEVICE);
3416 	memcpy(req->result, state->caam_ctx, digestsize);
3417 	qi_cache_free(edesc);
3418 
3419 	print_hex_dump_debug("ctx@" __stringify(__LINE__)": ",
3420 			     DUMP_PREFIX_ADDRESS, 16, 4, state->caam_ctx,
3421 			     ctx->ctx_len, 1);
3422 
3423 	ahash_request_complete(req, ecode);
3424 }
3425 
3426 static void ahash_done_bi(void *cbk_ctx, u32 status)
3427 {
3428 	struct crypto_async_request *areq = cbk_ctx;
3429 	struct ahash_request *req = ahash_request_cast(areq);
3430 	struct crypto_ahash *ahash = crypto_ahash_reqtfm(req);
3431 	struct caam_hash_state *state = ahash_request_ctx_dma(req);
3432 	struct ahash_edesc *edesc = state->caam_req.edesc;
3433 	struct caam_hash_ctx *ctx = crypto_ahash_ctx_dma(ahash);
3434 	int ecode = 0;
3435 
3436 	dev_dbg(ctx->dev, "%s %d: err 0x%x\n", __func__, __LINE__, status);
3437 
3438 	if (unlikely(status))
3439 		ecode = caam_qi2_strstatus(ctx->dev, status);
3440 
3441 	ahash_unmap_ctx(ctx->dev, edesc, req, DMA_BIDIRECTIONAL);
3442 	qi_cache_free(edesc);
3443 
3444 	scatterwalk_map_and_copy(state->buf, req->src,
3445 				 req->nbytes - state->next_buflen,
3446 				 state->next_buflen, 0);
3447 	state->buflen = state->next_buflen;
3448 
3449 	print_hex_dump_debug("buf@" __stringify(__LINE__)": ",
3450 			     DUMP_PREFIX_ADDRESS, 16, 4, state->buf,
3451 			     state->buflen, 1);
3452 
3453 	print_hex_dump_debug("ctx@" __stringify(__LINE__)": ",
3454 			     DUMP_PREFIX_ADDRESS, 16, 4, state->caam_ctx,
3455 			     ctx->ctx_len, 1);
3456 	if (req->result)
3457 		print_hex_dump_debug("result@" __stringify(__LINE__)": ",
3458 				     DUMP_PREFIX_ADDRESS, 16, 4, req->result,
3459 				     crypto_ahash_digestsize(ahash), 1);
3460 
3461 	ahash_request_complete(req, ecode);
3462 }
3463 
3464 static void ahash_done_ctx_src(void *cbk_ctx, u32 status)
3465 {
3466 	struct crypto_async_request *areq = cbk_ctx;
3467 	struct ahash_request *req = ahash_request_cast(areq);
3468 	struct crypto_ahash *ahash = crypto_ahash_reqtfm(req);
3469 	struct caam_hash_state *state = ahash_request_ctx_dma(req);
3470 	struct ahash_edesc *edesc = state->caam_req.edesc;
3471 	struct caam_hash_ctx *ctx = crypto_ahash_ctx_dma(ahash);
3472 	int digestsize = crypto_ahash_digestsize(ahash);
3473 	int ecode = 0;
3474 
3475 	dev_dbg(ctx->dev, "%s %d: err 0x%x\n", __func__, __LINE__, status);
3476 
3477 	if (unlikely(status))
3478 		ecode = caam_qi2_strstatus(ctx->dev, status);
3479 
3480 	ahash_unmap_ctx(ctx->dev, edesc, req, DMA_BIDIRECTIONAL);
3481 	memcpy(req->result, state->caam_ctx, digestsize);
3482 	qi_cache_free(edesc);
3483 
3484 	print_hex_dump_debug("ctx@" __stringify(__LINE__)": ",
3485 			     DUMP_PREFIX_ADDRESS, 16, 4, state->caam_ctx,
3486 			     ctx->ctx_len, 1);
3487 
3488 	ahash_request_complete(req, ecode);
3489 }
3490 
3491 static void ahash_done_ctx_dst(void *cbk_ctx, u32 status)
3492 {
3493 	struct crypto_async_request *areq = cbk_ctx;
3494 	struct ahash_request *req = ahash_request_cast(areq);
3495 	struct crypto_ahash *ahash = crypto_ahash_reqtfm(req);
3496 	struct caam_hash_state *state = ahash_request_ctx_dma(req);
3497 	struct ahash_edesc *edesc = state->caam_req.edesc;
3498 	struct caam_hash_ctx *ctx = crypto_ahash_ctx_dma(ahash);
3499 	int ecode = 0;
3500 
3501 	dev_dbg(ctx->dev, "%s %d: err 0x%x\n", __func__, __LINE__, status);
3502 
3503 	if (unlikely(status))
3504 		ecode = caam_qi2_strstatus(ctx->dev, status);
3505 
3506 	ahash_unmap_ctx(ctx->dev, edesc, req, DMA_FROM_DEVICE);
3507 	qi_cache_free(edesc);
3508 
3509 	scatterwalk_map_and_copy(state->buf, req->src,
3510 				 req->nbytes - state->next_buflen,
3511 				 state->next_buflen, 0);
3512 	state->buflen = state->next_buflen;
3513 
3514 	print_hex_dump_debug("buf@" __stringify(__LINE__)": ",
3515 			     DUMP_PREFIX_ADDRESS, 16, 4, state->buf,
3516 			     state->buflen, 1);
3517 
3518 	print_hex_dump_debug("ctx@" __stringify(__LINE__)": ",
3519 			     DUMP_PREFIX_ADDRESS, 16, 4, state->caam_ctx,
3520 			     ctx->ctx_len, 1);
3521 	if (req->result)
3522 		print_hex_dump_debug("result@" __stringify(__LINE__)": ",
3523 				     DUMP_PREFIX_ADDRESS, 16, 4, req->result,
3524 				     crypto_ahash_digestsize(ahash), 1);
3525 
3526 	ahash_request_complete(req, ecode);
3527 }
3528 
3529 static int ahash_update_ctx(struct ahash_request *req)
3530 {
3531 	struct crypto_ahash *ahash = crypto_ahash_reqtfm(req);
3532 	struct caam_hash_ctx *ctx = crypto_ahash_ctx_dma(ahash);
3533 	struct caam_hash_state *state = ahash_request_ctx_dma(req);
3534 	struct caam_request *req_ctx = &state->caam_req;
3535 	struct dpaa2_fl_entry *in_fle = &req_ctx->fd_flt[1];
3536 	struct dpaa2_fl_entry *out_fle = &req_ctx->fd_flt[0];
3537 	gfp_t flags = (req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP) ?
3538 		      GFP_KERNEL : GFP_ATOMIC;
3539 	u8 *buf = state->buf;
3540 	int *buflen = &state->buflen;
3541 	int *next_buflen = &state->next_buflen;
3542 	int in_len = *buflen + req->nbytes, to_hash;
3543 	int src_nents, mapped_nents, qm_sg_bytes, qm_sg_src_index;
3544 	struct ahash_edesc *edesc;
3545 	int ret = 0;
3546 
3547 	*next_buflen = in_len & (crypto_tfm_alg_blocksize(&ahash->base) - 1);
3548 	to_hash = in_len - *next_buflen;
3549 
3550 	if (to_hash) {
3551 		struct dpaa2_sg_entry *sg_table;
3552 		int src_len = req->nbytes - *next_buflen;
3553 
3554 		src_nents = sg_nents_for_len(req->src, src_len);
3555 		if (src_nents < 0) {
3556 			dev_err(ctx->dev, "Invalid number of src SG.\n");
3557 			return src_nents;
3558 		}
3559 
3560 		if (src_nents) {
3561 			mapped_nents = dma_map_sg(ctx->dev, req->src, src_nents,
3562 						  DMA_TO_DEVICE);
3563 			if (!mapped_nents) {
3564 				dev_err(ctx->dev, "unable to DMA map source\n");
3565 				return -ENOMEM;
3566 			}
3567 		} else {
3568 			mapped_nents = 0;
3569 		}
3570 
3571 		/* allocate space for base edesc and link tables */
3572 		edesc = qi_cache_zalloc(flags);
3573 		if (!edesc) {
3574 			dma_unmap_sg(ctx->dev, req->src, src_nents,
3575 				     DMA_TO_DEVICE);
3576 			return -ENOMEM;
3577 		}
3578 
3579 		edesc->src_nents = src_nents;
3580 		qm_sg_src_index = 1 + (*buflen ? 1 : 0);
3581 		qm_sg_bytes = pad_sg_nents(qm_sg_src_index + mapped_nents) *
3582 			      sizeof(*sg_table);
3583 		sg_table = &edesc->sgt[0];
3584 
3585 		ret = ctx_map_to_qm_sg(ctx->dev, state, ctx->ctx_len, sg_table,
3586 				       DMA_BIDIRECTIONAL);
3587 		if (ret)
3588 			goto unmap_ctx;
3589 
3590 		ret = buf_map_to_qm_sg(ctx->dev, sg_table + 1, state);
3591 		if (ret)
3592 			goto unmap_ctx;
3593 
3594 		if (mapped_nents) {
3595 			sg_to_qm_sg_last(req->src, src_len,
3596 					 sg_table + qm_sg_src_index, 0);
3597 		} else {
3598 			dpaa2_sg_set_final(sg_table + qm_sg_src_index - 1,
3599 					   true);
3600 		}
3601 
3602 		edesc->qm_sg_dma = dma_map_single(ctx->dev, sg_table,
3603 						  qm_sg_bytes, DMA_TO_DEVICE);
3604 		if (dma_mapping_error(ctx->dev, edesc->qm_sg_dma)) {
3605 			dev_err(ctx->dev, "unable to map S/G table\n");
3606 			ret = -ENOMEM;
3607 			goto unmap_ctx;
3608 		}
3609 		edesc->qm_sg_bytes = qm_sg_bytes;
3610 
3611 		memset(&req_ctx->fd_flt, 0, sizeof(req_ctx->fd_flt));
3612 		dpaa2_fl_set_final(in_fle, true);
3613 		dpaa2_fl_set_format(in_fle, dpaa2_fl_sg);
3614 		dpaa2_fl_set_addr(in_fle, edesc->qm_sg_dma);
3615 		dpaa2_fl_set_len(in_fle, ctx->ctx_len + to_hash);
3616 		dpaa2_fl_set_format(out_fle, dpaa2_fl_single);
3617 		dpaa2_fl_set_addr(out_fle, state->ctx_dma);
3618 		dpaa2_fl_set_len(out_fle, ctx->ctx_len);
3619 
3620 		req_ctx->flc = &ctx->flc[UPDATE];
3621 		req_ctx->flc_dma = ctx->flc_dma[UPDATE];
3622 		req_ctx->cbk = ahash_done_bi;
3623 		req_ctx->ctx = &req->base;
3624 		req_ctx->edesc = edesc;
3625 
3626 		ret = dpaa2_caam_enqueue(ctx->dev, req_ctx);
3627 		if (ret != -EINPROGRESS &&
3628 		    !(ret == -EBUSY &&
3629 		      req->base.flags & CRYPTO_TFM_REQ_MAY_BACKLOG))
3630 			goto unmap_ctx;
3631 	} else if (*next_buflen) {
3632 		scatterwalk_map_and_copy(buf + *buflen, req->src, 0,
3633 					 req->nbytes, 0);
3634 		*buflen = *next_buflen;
3635 
3636 		print_hex_dump_debug("buf@" __stringify(__LINE__)": ",
3637 				     DUMP_PREFIX_ADDRESS, 16, 4, buf,
3638 				     *buflen, 1);
3639 	}
3640 
3641 	return ret;
3642 unmap_ctx:
3643 	ahash_unmap_ctx(ctx->dev, edesc, req, DMA_BIDIRECTIONAL);
3644 	qi_cache_free(edesc);
3645 	return ret;
3646 }
3647 
3648 static int ahash_final_ctx(struct ahash_request *req)
3649 {
3650 	struct crypto_ahash *ahash = crypto_ahash_reqtfm(req);
3651 	struct caam_hash_ctx *ctx = crypto_ahash_ctx_dma(ahash);
3652 	struct caam_hash_state *state = ahash_request_ctx_dma(req);
3653 	struct caam_request *req_ctx = &state->caam_req;
3654 	struct dpaa2_fl_entry *in_fle = &req_ctx->fd_flt[1];
3655 	struct dpaa2_fl_entry *out_fle = &req_ctx->fd_flt[0];
3656 	gfp_t flags = (req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP) ?
3657 		      GFP_KERNEL : GFP_ATOMIC;
3658 	int buflen = state->buflen;
3659 	int qm_sg_bytes;
3660 	int digestsize = crypto_ahash_digestsize(ahash);
3661 	struct ahash_edesc *edesc;
3662 	struct dpaa2_sg_entry *sg_table;
3663 	int ret;
3664 
3665 	/* allocate space for base edesc and link tables */
3666 	edesc = qi_cache_zalloc(flags);
3667 	if (!edesc)
3668 		return -ENOMEM;
3669 
3670 	qm_sg_bytes = pad_sg_nents(1 + (buflen ? 1 : 0)) * sizeof(*sg_table);
3671 	sg_table = &edesc->sgt[0];
3672 
3673 	ret = ctx_map_to_qm_sg(ctx->dev, state, ctx->ctx_len, sg_table,
3674 			       DMA_BIDIRECTIONAL);
3675 	if (ret)
3676 		goto unmap_ctx;
3677 
3678 	ret = buf_map_to_qm_sg(ctx->dev, sg_table + 1, state);
3679 	if (ret)
3680 		goto unmap_ctx;
3681 
3682 	dpaa2_sg_set_final(sg_table + (buflen ? 1 : 0), true);
3683 
3684 	edesc->qm_sg_dma = dma_map_single(ctx->dev, sg_table, qm_sg_bytes,
3685 					  DMA_TO_DEVICE);
3686 	if (dma_mapping_error(ctx->dev, edesc->qm_sg_dma)) {
3687 		dev_err(ctx->dev, "unable to map S/G table\n");
3688 		ret = -ENOMEM;
3689 		goto unmap_ctx;
3690 	}
3691 	edesc->qm_sg_bytes = qm_sg_bytes;
3692 
3693 	memset(&req_ctx->fd_flt, 0, sizeof(req_ctx->fd_flt));
3694 	dpaa2_fl_set_final(in_fle, true);
3695 	dpaa2_fl_set_format(in_fle, dpaa2_fl_sg);
3696 	dpaa2_fl_set_addr(in_fle, edesc->qm_sg_dma);
3697 	dpaa2_fl_set_len(in_fle, ctx->ctx_len + buflen);
3698 	dpaa2_fl_set_format(out_fle, dpaa2_fl_single);
3699 	dpaa2_fl_set_addr(out_fle, state->ctx_dma);
3700 	dpaa2_fl_set_len(out_fle, digestsize);
3701 
3702 	req_ctx->flc = &ctx->flc[FINALIZE];
3703 	req_ctx->flc_dma = ctx->flc_dma[FINALIZE];
3704 	req_ctx->cbk = ahash_done_ctx_src;
3705 	req_ctx->ctx = &req->base;
3706 	req_ctx->edesc = edesc;
3707 
3708 	ret = dpaa2_caam_enqueue(ctx->dev, req_ctx);
3709 	if (ret == -EINPROGRESS ||
3710 	    (ret == -EBUSY && req->base.flags & CRYPTO_TFM_REQ_MAY_BACKLOG))
3711 		return ret;
3712 
3713 unmap_ctx:
3714 	ahash_unmap_ctx(ctx->dev, edesc, req, DMA_BIDIRECTIONAL);
3715 	qi_cache_free(edesc);
3716 	return ret;
3717 }
3718 
3719 static int ahash_finup_ctx(struct ahash_request *req)
3720 {
3721 	struct crypto_ahash *ahash = crypto_ahash_reqtfm(req);
3722 	struct caam_hash_ctx *ctx = crypto_ahash_ctx_dma(ahash);
3723 	struct caam_hash_state *state = ahash_request_ctx_dma(req);
3724 	struct caam_request *req_ctx = &state->caam_req;
3725 	struct dpaa2_fl_entry *in_fle = &req_ctx->fd_flt[1];
3726 	struct dpaa2_fl_entry *out_fle = &req_ctx->fd_flt[0];
3727 	gfp_t flags = (req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP) ?
3728 		      GFP_KERNEL : GFP_ATOMIC;
3729 	int buflen = state->buflen;
3730 	int qm_sg_bytes, qm_sg_src_index;
3731 	int src_nents, mapped_nents;
3732 	int digestsize = crypto_ahash_digestsize(ahash);
3733 	struct ahash_edesc *edesc;
3734 	struct dpaa2_sg_entry *sg_table;
3735 	int ret;
3736 
3737 	src_nents = sg_nents_for_len(req->src, req->nbytes);
3738 	if (src_nents < 0) {
3739 		dev_err(ctx->dev, "Invalid number of src SG.\n");
3740 		return src_nents;
3741 	}
3742 
3743 	if (src_nents) {
3744 		mapped_nents = dma_map_sg(ctx->dev, req->src, src_nents,
3745 					  DMA_TO_DEVICE);
3746 		if (!mapped_nents) {
3747 			dev_err(ctx->dev, "unable to DMA map source\n");
3748 			return -ENOMEM;
3749 		}
3750 	} else {
3751 		mapped_nents = 0;
3752 	}
3753 
3754 	/* allocate space for base edesc and link tables */
3755 	edesc = qi_cache_zalloc(flags);
3756 	if (!edesc) {
3757 		dma_unmap_sg(ctx->dev, req->src, src_nents, DMA_TO_DEVICE);
3758 		return -ENOMEM;
3759 	}
3760 
3761 	edesc->src_nents = src_nents;
3762 	qm_sg_src_index = 1 + (buflen ? 1 : 0);
3763 	qm_sg_bytes = pad_sg_nents(qm_sg_src_index + mapped_nents) *
3764 		      sizeof(*sg_table);
3765 	sg_table = &edesc->sgt[0];
3766 
3767 	ret = ctx_map_to_qm_sg(ctx->dev, state, ctx->ctx_len, sg_table,
3768 			       DMA_BIDIRECTIONAL);
3769 	if (ret)
3770 		goto unmap_ctx;
3771 
3772 	ret = buf_map_to_qm_sg(ctx->dev, sg_table + 1, state);
3773 	if (ret)
3774 		goto unmap_ctx;
3775 
3776 	sg_to_qm_sg_last(req->src, req->nbytes, sg_table + qm_sg_src_index, 0);
3777 
3778 	edesc->qm_sg_dma = dma_map_single(ctx->dev, sg_table, qm_sg_bytes,
3779 					  DMA_TO_DEVICE);
3780 	if (dma_mapping_error(ctx->dev, edesc->qm_sg_dma)) {
3781 		dev_err(ctx->dev, "unable to map S/G table\n");
3782 		ret = -ENOMEM;
3783 		goto unmap_ctx;
3784 	}
3785 	edesc->qm_sg_bytes = qm_sg_bytes;
3786 
3787 	memset(&req_ctx->fd_flt, 0, sizeof(req_ctx->fd_flt));
3788 	dpaa2_fl_set_final(in_fle, true);
3789 	dpaa2_fl_set_format(in_fle, dpaa2_fl_sg);
3790 	dpaa2_fl_set_addr(in_fle, edesc->qm_sg_dma);
3791 	dpaa2_fl_set_len(in_fle, ctx->ctx_len + buflen + req->nbytes);
3792 	dpaa2_fl_set_format(out_fle, dpaa2_fl_single);
3793 	dpaa2_fl_set_addr(out_fle, state->ctx_dma);
3794 	dpaa2_fl_set_len(out_fle, digestsize);
3795 
3796 	req_ctx->flc = &ctx->flc[FINALIZE];
3797 	req_ctx->flc_dma = ctx->flc_dma[FINALIZE];
3798 	req_ctx->cbk = ahash_done_ctx_src;
3799 	req_ctx->ctx = &req->base;
3800 	req_ctx->edesc = edesc;
3801 
3802 	ret = dpaa2_caam_enqueue(ctx->dev, req_ctx);
3803 	if (ret == -EINPROGRESS ||
3804 	    (ret == -EBUSY && req->base.flags & CRYPTO_TFM_REQ_MAY_BACKLOG))
3805 		return ret;
3806 
3807 unmap_ctx:
3808 	ahash_unmap_ctx(ctx->dev, edesc, req, DMA_BIDIRECTIONAL);
3809 	qi_cache_free(edesc);
3810 	return ret;
3811 }
3812 
3813 static int ahash_digest(struct ahash_request *req)
3814 {
3815 	struct crypto_ahash *ahash = crypto_ahash_reqtfm(req);
3816 	struct caam_hash_ctx *ctx = crypto_ahash_ctx_dma(ahash);
3817 	struct caam_hash_state *state = ahash_request_ctx_dma(req);
3818 	struct caam_request *req_ctx = &state->caam_req;
3819 	struct dpaa2_fl_entry *in_fle = &req_ctx->fd_flt[1];
3820 	struct dpaa2_fl_entry *out_fle = &req_ctx->fd_flt[0];
3821 	gfp_t flags = (req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP) ?
3822 		      GFP_KERNEL : GFP_ATOMIC;
3823 	int digestsize = crypto_ahash_digestsize(ahash);
3824 	int src_nents, mapped_nents;
3825 	struct ahash_edesc *edesc;
3826 	int ret = -ENOMEM;
3827 
3828 	state->buf_dma = 0;
3829 
3830 	src_nents = sg_nents_for_len(req->src, req->nbytes);
3831 	if (src_nents < 0) {
3832 		dev_err(ctx->dev, "Invalid number of src SG.\n");
3833 		return src_nents;
3834 	}
3835 
3836 	if (src_nents) {
3837 		mapped_nents = dma_map_sg(ctx->dev, req->src, src_nents,
3838 					  DMA_TO_DEVICE);
3839 		if (!mapped_nents) {
3840 			dev_err(ctx->dev, "unable to map source for DMA\n");
3841 			return ret;
3842 		}
3843 	} else {
3844 		mapped_nents = 0;
3845 	}
3846 
3847 	/* allocate space for base edesc and link tables */
3848 	edesc = qi_cache_zalloc(flags);
3849 	if (!edesc) {
3850 		dma_unmap_sg(ctx->dev, req->src, src_nents, DMA_TO_DEVICE);
3851 		return ret;
3852 	}
3853 
3854 	edesc->src_nents = src_nents;
3855 	memset(&req_ctx->fd_flt, 0, sizeof(req_ctx->fd_flt));
3856 
3857 	if (mapped_nents > 1) {
3858 		int qm_sg_bytes;
3859 		struct dpaa2_sg_entry *sg_table = &edesc->sgt[0];
3860 
3861 		qm_sg_bytes = pad_sg_nents(mapped_nents) * sizeof(*sg_table);
3862 		sg_to_qm_sg_last(req->src, req->nbytes, sg_table, 0);
3863 		edesc->qm_sg_dma = dma_map_single(ctx->dev, sg_table,
3864 						  qm_sg_bytes, DMA_TO_DEVICE);
3865 		if (dma_mapping_error(ctx->dev, edesc->qm_sg_dma)) {
3866 			dev_err(ctx->dev, "unable to map S/G table\n");
3867 			goto unmap;
3868 		}
3869 		edesc->qm_sg_bytes = qm_sg_bytes;
3870 		dpaa2_fl_set_format(in_fle, dpaa2_fl_sg);
3871 		dpaa2_fl_set_addr(in_fle, edesc->qm_sg_dma);
3872 	} else {
3873 		dpaa2_fl_set_format(in_fle, dpaa2_fl_single);
3874 		dpaa2_fl_set_addr(in_fle, sg_dma_address(req->src));
3875 	}
3876 
3877 	state->ctx_dma_len = digestsize;
3878 	state->ctx_dma = dma_map_single(ctx->dev, state->caam_ctx, digestsize,
3879 					DMA_FROM_DEVICE);
3880 	if (dma_mapping_error(ctx->dev, state->ctx_dma)) {
3881 		dev_err(ctx->dev, "unable to map ctx\n");
3882 		state->ctx_dma = 0;
3883 		goto unmap;
3884 	}
3885 
3886 	dpaa2_fl_set_final(in_fle, true);
3887 	dpaa2_fl_set_len(in_fle, req->nbytes);
3888 	dpaa2_fl_set_format(out_fle, dpaa2_fl_single);
3889 	dpaa2_fl_set_addr(out_fle, state->ctx_dma);
3890 	dpaa2_fl_set_len(out_fle, digestsize);
3891 
3892 	req_ctx->flc = &ctx->flc[DIGEST];
3893 	req_ctx->flc_dma = ctx->flc_dma[DIGEST];
3894 	req_ctx->cbk = ahash_done;
3895 	req_ctx->ctx = &req->base;
3896 	req_ctx->edesc = edesc;
3897 	ret = dpaa2_caam_enqueue(ctx->dev, req_ctx);
3898 	if (ret == -EINPROGRESS ||
3899 	    (ret == -EBUSY && req->base.flags & CRYPTO_TFM_REQ_MAY_BACKLOG))
3900 		return ret;
3901 
3902 unmap:
3903 	ahash_unmap_ctx(ctx->dev, edesc, req, DMA_FROM_DEVICE);
3904 	qi_cache_free(edesc);
3905 	return ret;
3906 }
3907 
3908 static int ahash_final_no_ctx(struct ahash_request *req)
3909 {
3910 	struct crypto_ahash *ahash = crypto_ahash_reqtfm(req);
3911 	struct caam_hash_ctx *ctx = crypto_ahash_ctx_dma(ahash);
3912 	struct caam_hash_state *state = ahash_request_ctx_dma(req);
3913 	struct caam_request *req_ctx = &state->caam_req;
3914 	struct dpaa2_fl_entry *in_fle = &req_ctx->fd_flt[1];
3915 	struct dpaa2_fl_entry *out_fle = &req_ctx->fd_flt[0];
3916 	gfp_t flags = (req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP) ?
3917 		      GFP_KERNEL : GFP_ATOMIC;
3918 	u8 *buf = state->buf;
3919 	int buflen = state->buflen;
3920 	int digestsize = crypto_ahash_digestsize(ahash);
3921 	struct ahash_edesc *edesc;
3922 	int ret = -ENOMEM;
3923 
3924 	/* allocate space for base edesc and link tables */
3925 	edesc = qi_cache_zalloc(flags);
3926 	if (!edesc)
3927 		return ret;
3928 
3929 	if (buflen) {
3930 		state->buf_dma = dma_map_single(ctx->dev, buf, buflen,
3931 						DMA_TO_DEVICE);
3932 		if (dma_mapping_error(ctx->dev, state->buf_dma)) {
3933 			dev_err(ctx->dev, "unable to map src\n");
3934 			goto unmap;
3935 		}
3936 	}
3937 
3938 	state->ctx_dma_len = digestsize;
3939 	state->ctx_dma = dma_map_single(ctx->dev, state->caam_ctx, digestsize,
3940 					DMA_FROM_DEVICE);
3941 	if (dma_mapping_error(ctx->dev, state->ctx_dma)) {
3942 		dev_err(ctx->dev, "unable to map ctx\n");
3943 		state->ctx_dma = 0;
3944 		goto unmap;
3945 	}
3946 
3947 	memset(&req_ctx->fd_flt, 0, sizeof(req_ctx->fd_flt));
3948 	dpaa2_fl_set_final(in_fle, true);
3949 	/*
3950 	 * crypto engine requires the input entry to be present when
3951 	 * "frame list" FD is used.
3952 	 * Since engine does not support FMT=2'b11 (unused entry type), leaving
3953 	 * in_fle zeroized (except for "Final" flag) is the best option.
3954 	 */
3955 	if (buflen) {
3956 		dpaa2_fl_set_format(in_fle, dpaa2_fl_single);
3957 		dpaa2_fl_set_addr(in_fle, state->buf_dma);
3958 		dpaa2_fl_set_len(in_fle, buflen);
3959 	}
3960 	dpaa2_fl_set_format(out_fle, dpaa2_fl_single);
3961 	dpaa2_fl_set_addr(out_fle, state->ctx_dma);
3962 	dpaa2_fl_set_len(out_fle, digestsize);
3963 
3964 	req_ctx->flc = &ctx->flc[DIGEST];
3965 	req_ctx->flc_dma = ctx->flc_dma[DIGEST];
3966 	req_ctx->cbk = ahash_done;
3967 	req_ctx->ctx = &req->base;
3968 	req_ctx->edesc = edesc;
3969 
3970 	ret = dpaa2_caam_enqueue(ctx->dev, req_ctx);
3971 	if (ret == -EINPROGRESS ||
3972 	    (ret == -EBUSY && req->base.flags & CRYPTO_TFM_REQ_MAY_BACKLOG))
3973 		return ret;
3974 
3975 unmap:
3976 	ahash_unmap_ctx(ctx->dev, edesc, req, DMA_FROM_DEVICE);
3977 	qi_cache_free(edesc);
3978 	return ret;
3979 }
3980 
3981 static int ahash_update_no_ctx(struct ahash_request *req)
3982 {
3983 	struct crypto_ahash *ahash = crypto_ahash_reqtfm(req);
3984 	struct caam_hash_ctx *ctx = crypto_ahash_ctx_dma(ahash);
3985 	struct caam_hash_state *state = ahash_request_ctx_dma(req);
3986 	struct caam_request *req_ctx = &state->caam_req;
3987 	struct dpaa2_fl_entry *in_fle = &req_ctx->fd_flt[1];
3988 	struct dpaa2_fl_entry *out_fle = &req_ctx->fd_flt[0];
3989 	gfp_t flags = (req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP) ?
3990 		      GFP_KERNEL : GFP_ATOMIC;
3991 	u8 *buf = state->buf;
3992 	int *buflen = &state->buflen;
3993 	int *next_buflen = &state->next_buflen;
3994 	int in_len = *buflen + req->nbytes, to_hash;
3995 	int qm_sg_bytes, src_nents, mapped_nents;
3996 	struct ahash_edesc *edesc;
3997 	int ret = 0;
3998 
3999 	*next_buflen = in_len & (crypto_tfm_alg_blocksize(&ahash->base) - 1);
4000 	to_hash = in_len - *next_buflen;
4001 
4002 	if (to_hash) {
4003 		struct dpaa2_sg_entry *sg_table;
4004 		int src_len = req->nbytes - *next_buflen;
4005 
4006 		src_nents = sg_nents_for_len(req->src, src_len);
4007 		if (src_nents < 0) {
4008 			dev_err(ctx->dev, "Invalid number of src SG.\n");
4009 			return src_nents;
4010 		}
4011 
4012 		if (src_nents) {
4013 			mapped_nents = dma_map_sg(ctx->dev, req->src, src_nents,
4014 						  DMA_TO_DEVICE);
4015 			if (!mapped_nents) {
4016 				dev_err(ctx->dev, "unable to DMA map source\n");
4017 				return -ENOMEM;
4018 			}
4019 		} else {
4020 			mapped_nents = 0;
4021 		}
4022 
4023 		/* allocate space for base edesc and link tables */
4024 		edesc = qi_cache_zalloc(flags);
4025 		if (!edesc) {
4026 			dma_unmap_sg(ctx->dev, req->src, src_nents,
4027 				     DMA_TO_DEVICE);
4028 			return -ENOMEM;
4029 		}
4030 
4031 		edesc->src_nents = src_nents;
4032 		qm_sg_bytes = pad_sg_nents(1 + mapped_nents) *
4033 			      sizeof(*sg_table);
4034 		sg_table = &edesc->sgt[0];
4035 
4036 		ret = buf_map_to_qm_sg(ctx->dev, sg_table, state);
4037 		if (ret)
4038 			goto unmap_ctx;
4039 
4040 		sg_to_qm_sg_last(req->src, src_len, sg_table + 1, 0);
4041 
4042 		edesc->qm_sg_dma = dma_map_single(ctx->dev, sg_table,
4043 						  qm_sg_bytes, DMA_TO_DEVICE);
4044 		if (dma_mapping_error(ctx->dev, edesc->qm_sg_dma)) {
4045 			dev_err(ctx->dev, "unable to map S/G table\n");
4046 			ret = -ENOMEM;
4047 			goto unmap_ctx;
4048 		}
4049 		edesc->qm_sg_bytes = qm_sg_bytes;
4050 
4051 		state->ctx_dma_len = ctx->ctx_len;
4052 		state->ctx_dma = dma_map_single(ctx->dev, state->caam_ctx,
4053 						ctx->ctx_len, DMA_FROM_DEVICE);
4054 		if (dma_mapping_error(ctx->dev, state->ctx_dma)) {
4055 			dev_err(ctx->dev, "unable to map ctx\n");
4056 			state->ctx_dma = 0;
4057 			ret = -ENOMEM;
4058 			goto unmap_ctx;
4059 		}
4060 
4061 		memset(&req_ctx->fd_flt, 0, sizeof(req_ctx->fd_flt));
4062 		dpaa2_fl_set_final(in_fle, true);
4063 		dpaa2_fl_set_format(in_fle, dpaa2_fl_sg);
4064 		dpaa2_fl_set_addr(in_fle, edesc->qm_sg_dma);
4065 		dpaa2_fl_set_len(in_fle, to_hash);
4066 		dpaa2_fl_set_format(out_fle, dpaa2_fl_single);
4067 		dpaa2_fl_set_addr(out_fle, state->ctx_dma);
4068 		dpaa2_fl_set_len(out_fle, ctx->ctx_len);
4069 
4070 		req_ctx->flc = &ctx->flc[UPDATE_FIRST];
4071 		req_ctx->flc_dma = ctx->flc_dma[UPDATE_FIRST];
4072 		req_ctx->cbk = ahash_done_ctx_dst;
4073 		req_ctx->ctx = &req->base;
4074 		req_ctx->edesc = edesc;
4075 
4076 		ret = dpaa2_caam_enqueue(ctx->dev, req_ctx);
4077 		if (ret != -EINPROGRESS &&
4078 		    !(ret == -EBUSY &&
4079 		      req->base.flags & CRYPTO_TFM_REQ_MAY_BACKLOG))
4080 			goto unmap_ctx;
4081 
4082 		state->update = ahash_update_ctx;
4083 		state->finup = ahash_finup_ctx;
4084 		state->final = ahash_final_ctx;
4085 	} else if (*next_buflen) {
4086 		scatterwalk_map_and_copy(buf + *buflen, req->src, 0,
4087 					 req->nbytes, 0);
4088 		*buflen = *next_buflen;
4089 
4090 		print_hex_dump_debug("buf@" __stringify(__LINE__)": ",
4091 				     DUMP_PREFIX_ADDRESS, 16, 4, buf,
4092 				     *buflen, 1);
4093 	}
4094 
4095 	return ret;
4096 unmap_ctx:
4097 	ahash_unmap_ctx(ctx->dev, edesc, req, DMA_TO_DEVICE);
4098 	qi_cache_free(edesc);
4099 	return ret;
4100 }
4101 
4102 static int ahash_finup_no_ctx(struct ahash_request *req)
4103 {
4104 	struct crypto_ahash *ahash = crypto_ahash_reqtfm(req);
4105 	struct caam_hash_ctx *ctx = crypto_ahash_ctx_dma(ahash);
4106 	struct caam_hash_state *state = ahash_request_ctx_dma(req);
4107 	struct caam_request *req_ctx = &state->caam_req;
4108 	struct dpaa2_fl_entry *in_fle = &req_ctx->fd_flt[1];
4109 	struct dpaa2_fl_entry *out_fle = &req_ctx->fd_flt[0];
4110 	gfp_t flags = (req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP) ?
4111 		      GFP_KERNEL : GFP_ATOMIC;
4112 	int buflen = state->buflen;
4113 	int qm_sg_bytes, src_nents, mapped_nents;
4114 	int digestsize = crypto_ahash_digestsize(ahash);
4115 	struct ahash_edesc *edesc;
4116 	struct dpaa2_sg_entry *sg_table;
4117 	int ret = -ENOMEM;
4118 
4119 	src_nents = sg_nents_for_len(req->src, req->nbytes);
4120 	if (src_nents < 0) {
4121 		dev_err(ctx->dev, "Invalid number of src SG.\n");
4122 		return src_nents;
4123 	}
4124 
4125 	if (src_nents) {
4126 		mapped_nents = dma_map_sg(ctx->dev, req->src, src_nents,
4127 					  DMA_TO_DEVICE);
4128 		if (!mapped_nents) {
4129 			dev_err(ctx->dev, "unable to DMA map source\n");
4130 			return ret;
4131 		}
4132 	} else {
4133 		mapped_nents = 0;
4134 	}
4135 
4136 	/* allocate space for base edesc and link tables */
4137 	edesc = qi_cache_zalloc(flags);
4138 	if (!edesc) {
4139 		dma_unmap_sg(ctx->dev, req->src, src_nents, DMA_TO_DEVICE);
4140 		return ret;
4141 	}
4142 
4143 	edesc->src_nents = src_nents;
4144 	qm_sg_bytes = pad_sg_nents(2 + mapped_nents) * sizeof(*sg_table);
4145 	sg_table = &edesc->sgt[0];
4146 
4147 	ret = buf_map_to_qm_sg(ctx->dev, sg_table, state);
4148 	if (ret)
4149 		goto unmap;
4150 
4151 	sg_to_qm_sg_last(req->src, req->nbytes, sg_table + 1, 0);
4152 
4153 	edesc->qm_sg_dma = dma_map_single(ctx->dev, sg_table, qm_sg_bytes,
4154 					  DMA_TO_DEVICE);
4155 	if (dma_mapping_error(ctx->dev, edesc->qm_sg_dma)) {
4156 		dev_err(ctx->dev, "unable to map S/G table\n");
4157 		ret = -ENOMEM;
4158 		goto unmap;
4159 	}
4160 	edesc->qm_sg_bytes = qm_sg_bytes;
4161 
4162 	state->ctx_dma_len = digestsize;
4163 	state->ctx_dma = dma_map_single(ctx->dev, state->caam_ctx, digestsize,
4164 					DMA_FROM_DEVICE);
4165 	if (dma_mapping_error(ctx->dev, state->ctx_dma)) {
4166 		dev_err(ctx->dev, "unable to map ctx\n");
4167 		state->ctx_dma = 0;
4168 		ret = -ENOMEM;
4169 		goto unmap;
4170 	}
4171 
4172 	memset(&req_ctx->fd_flt, 0, sizeof(req_ctx->fd_flt));
4173 	dpaa2_fl_set_final(in_fle, true);
4174 	dpaa2_fl_set_format(in_fle, dpaa2_fl_sg);
4175 	dpaa2_fl_set_addr(in_fle, edesc->qm_sg_dma);
4176 	dpaa2_fl_set_len(in_fle, buflen + req->nbytes);
4177 	dpaa2_fl_set_format(out_fle, dpaa2_fl_single);
4178 	dpaa2_fl_set_addr(out_fle, state->ctx_dma);
4179 	dpaa2_fl_set_len(out_fle, digestsize);
4180 
4181 	req_ctx->flc = &ctx->flc[DIGEST];
4182 	req_ctx->flc_dma = ctx->flc_dma[DIGEST];
4183 	req_ctx->cbk = ahash_done;
4184 	req_ctx->ctx = &req->base;
4185 	req_ctx->edesc = edesc;
4186 	ret = dpaa2_caam_enqueue(ctx->dev, req_ctx);
4187 	if (ret != -EINPROGRESS &&
4188 	    !(ret == -EBUSY && req->base.flags & CRYPTO_TFM_REQ_MAY_BACKLOG))
4189 		goto unmap;
4190 
4191 	return ret;
4192 unmap:
4193 	ahash_unmap_ctx(ctx->dev, edesc, req, DMA_FROM_DEVICE);
4194 	qi_cache_free(edesc);
4195 	return ret;
4196 }
4197 
4198 static int ahash_update_first(struct ahash_request *req)
4199 {
4200 	struct crypto_ahash *ahash = crypto_ahash_reqtfm(req);
4201 	struct caam_hash_ctx *ctx = crypto_ahash_ctx_dma(ahash);
4202 	struct caam_hash_state *state = ahash_request_ctx_dma(req);
4203 	struct caam_request *req_ctx = &state->caam_req;
4204 	struct dpaa2_fl_entry *in_fle = &req_ctx->fd_flt[1];
4205 	struct dpaa2_fl_entry *out_fle = &req_ctx->fd_flt[0];
4206 	gfp_t flags = (req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP) ?
4207 		      GFP_KERNEL : GFP_ATOMIC;
4208 	u8 *buf = state->buf;
4209 	int *buflen = &state->buflen;
4210 	int *next_buflen = &state->next_buflen;
4211 	int to_hash;
4212 	int src_nents, mapped_nents;
4213 	struct ahash_edesc *edesc;
4214 	int ret = 0;
4215 
4216 	*next_buflen = req->nbytes & (crypto_tfm_alg_blocksize(&ahash->base) -
4217 				      1);
4218 	to_hash = req->nbytes - *next_buflen;
4219 
4220 	if (to_hash) {
4221 		struct dpaa2_sg_entry *sg_table;
4222 		int src_len = req->nbytes - *next_buflen;
4223 
4224 		src_nents = sg_nents_for_len(req->src, src_len);
4225 		if (src_nents < 0) {
4226 			dev_err(ctx->dev, "Invalid number of src SG.\n");
4227 			return src_nents;
4228 		}
4229 
4230 		if (src_nents) {
4231 			mapped_nents = dma_map_sg(ctx->dev, req->src, src_nents,
4232 						  DMA_TO_DEVICE);
4233 			if (!mapped_nents) {
4234 				dev_err(ctx->dev, "unable to map source for DMA\n");
4235 				return -ENOMEM;
4236 			}
4237 		} else {
4238 			mapped_nents = 0;
4239 		}
4240 
4241 		/* allocate space for base edesc and link tables */
4242 		edesc = qi_cache_zalloc(flags);
4243 		if (!edesc) {
4244 			dma_unmap_sg(ctx->dev, req->src, src_nents,
4245 				     DMA_TO_DEVICE);
4246 			return -ENOMEM;
4247 		}
4248 
4249 		edesc->src_nents = src_nents;
4250 		sg_table = &edesc->sgt[0];
4251 
4252 		memset(&req_ctx->fd_flt, 0, sizeof(req_ctx->fd_flt));
4253 		dpaa2_fl_set_final(in_fle, true);
4254 		dpaa2_fl_set_len(in_fle, to_hash);
4255 
4256 		if (mapped_nents > 1) {
4257 			int qm_sg_bytes;
4258 
4259 			sg_to_qm_sg_last(req->src, src_len, sg_table, 0);
4260 			qm_sg_bytes = pad_sg_nents(mapped_nents) *
4261 				      sizeof(*sg_table);
4262 			edesc->qm_sg_dma = dma_map_single(ctx->dev, sg_table,
4263 							  qm_sg_bytes,
4264 							  DMA_TO_DEVICE);
4265 			if (dma_mapping_error(ctx->dev, edesc->qm_sg_dma)) {
4266 				dev_err(ctx->dev, "unable to map S/G table\n");
4267 				ret = -ENOMEM;
4268 				goto unmap_ctx;
4269 			}
4270 			edesc->qm_sg_bytes = qm_sg_bytes;
4271 			dpaa2_fl_set_format(in_fle, dpaa2_fl_sg);
4272 			dpaa2_fl_set_addr(in_fle, edesc->qm_sg_dma);
4273 		} else {
4274 			dpaa2_fl_set_format(in_fle, dpaa2_fl_single);
4275 			dpaa2_fl_set_addr(in_fle, sg_dma_address(req->src));
4276 		}
4277 
4278 		state->ctx_dma_len = ctx->ctx_len;
4279 		state->ctx_dma = dma_map_single(ctx->dev, state->caam_ctx,
4280 						ctx->ctx_len, DMA_FROM_DEVICE);
4281 		if (dma_mapping_error(ctx->dev, state->ctx_dma)) {
4282 			dev_err(ctx->dev, "unable to map ctx\n");
4283 			state->ctx_dma = 0;
4284 			ret = -ENOMEM;
4285 			goto unmap_ctx;
4286 		}
4287 
4288 		dpaa2_fl_set_format(out_fle, dpaa2_fl_single);
4289 		dpaa2_fl_set_addr(out_fle, state->ctx_dma);
4290 		dpaa2_fl_set_len(out_fle, ctx->ctx_len);
4291 
4292 		req_ctx->flc = &ctx->flc[UPDATE_FIRST];
4293 		req_ctx->flc_dma = ctx->flc_dma[UPDATE_FIRST];
4294 		req_ctx->cbk = ahash_done_ctx_dst;
4295 		req_ctx->ctx = &req->base;
4296 		req_ctx->edesc = edesc;
4297 
4298 		ret = dpaa2_caam_enqueue(ctx->dev, req_ctx);
4299 		if (ret != -EINPROGRESS &&
4300 		    !(ret == -EBUSY && req->base.flags &
4301 		      CRYPTO_TFM_REQ_MAY_BACKLOG))
4302 			goto unmap_ctx;
4303 
4304 		state->update = ahash_update_ctx;
4305 		state->finup = ahash_finup_ctx;
4306 		state->final = ahash_final_ctx;
4307 	} else if (*next_buflen) {
4308 		state->update = ahash_update_no_ctx;
4309 		state->finup = ahash_finup_no_ctx;
4310 		state->final = ahash_final_no_ctx;
4311 		scatterwalk_map_and_copy(buf, req->src, 0,
4312 					 req->nbytes, 0);
4313 		*buflen = *next_buflen;
4314 
4315 		print_hex_dump_debug("buf@" __stringify(__LINE__)": ",
4316 				     DUMP_PREFIX_ADDRESS, 16, 4, buf,
4317 				     *buflen, 1);
4318 	}
4319 
4320 	return ret;
4321 unmap_ctx:
4322 	ahash_unmap_ctx(ctx->dev, edesc, req, DMA_TO_DEVICE);
4323 	qi_cache_free(edesc);
4324 	return ret;
4325 }
4326 
4327 static int ahash_finup_first(struct ahash_request *req)
4328 {
4329 	return ahash_digest(req);
4330 }
4331 
4332 static int ahash_init(struct ahash_request *req)
4333 {
4334 	struct caam_hash_state *state = ahash_request_ctx_dma(req);
4335 
4336 	state->update = ahash_update_first;
4337 	state->finup = ahash_finup_first;
4338 	state->final = ahash_final_no_ctx;
4339 
4340 	state->ctx_dma = 0;
4341 	state->ctx_dma_len = 0;
4342 	state->buf_dma = 0;
4343 	state->buflen = 0;
4344 	state->next_buflen = 0;
4345 
4346 	return 0;
4347 }
4348 
4349 static int ahash_update(struct ahash_request *req)
4350 {
4351 	struct caam_hash_state *state = ahash_request_ctx_dma(req);
4352 
4353 	return state->update(req);
4354 }
4355 
4356 static int ahash_finup(struct ahash_request *req)
4357 {
4358 	struct caam_hash_state *state = ahash_request_ctx_dma(req);
4359 
4360 	return state->finup(req);
4361 }
4362 
4363 static int ahash_final(struct ahash_request *req)
4364 {
4365 	struct caam_hash_state *state = ahash_request_ctx_dma(req);
4366 
4367 	return state->final(req);
4368 }
4369 
4370 static int ahash_export(struct ahash_request *req, void *out)
4371 {
4372 	struct caam_hash_state *state = ahash_request_ctx_dma(req);
4373 	struct caam_export_state *export = out;
4374 	u8 *buf = state->buf;
4375 	int len = state->buflen;
4376 
4377 	memcpy(export->buf, buf, len);
4378 	memcpy(export->caam_ctx, state->caam_ctx, sizeof(export->caam_ctx));
4379 	export->buflen = len;
4380 	export->update = state->update;
4381 	export->final = state->final;
4382 	export->finup = state->finup;
4383 
4384 	return 0;
4385 }
4386 
4387 static int ahash_import(struct ahash_request *req, const void *in)
4388 {
4389 	struct caam_hash_state *state = ahash_request_ctx_dma(req);
4390 	const struct caam_export_state *export = in;
4391 
4392 	memset(state, 0, sizeof(*state));
4393 	memcpy(state->buf, export->buf, export->buflen);
4394 	memcpy(state->caam_ctx, export->caam_ctx, sizeof(state->caam_ctx));
4395 	state->buflen = export->buflen;
4396 	state->update = export->update;
4397 	state->final = export->final;
4398 	state->finup = export->finup;
4399 
4400 	return 0;
4401 }
4402 
4403 struct caam_hash_template {
4404 	char name[CRYPTO_MAX_ALG_NAME];
4405 	char driver_name[CRYPTO_MAX_ALG_NAME];
4406 	char hmac_name[CRYPTO_MAX_ALG_NAME];
4407 	char hmac_driver_name[CRYPTO_MAX_ALG_NAME];
4408 	unsigned int blocksize;
4409 	struct ahash_alg template_ahash;
4410 	u32 alg_type;
4411 };
4412 
4413 /* ahash descriptors */
4414 static struct caam_hash_template driver_hash[] = {
4415 	{
4416 		.name = "sha1",
4417 		.driver_name = "sha1-caam-qi2",
4418 		.hmac_name = "hmac(sha1)",
4419 		.hmac_driver_name = "hmac-sha1-caam-qi2",
4420 		.blocksize = SHA1_BLOCK_SIZE,
4421 		.template_ahash = {
4422 			.init = ahash_init,
4423 			.update = ahash_update,
4424 			.final = ahash_final,
4425 			.finup = ahash_finup,
4426 			.digest = ahash_digest,
4427 			.export = ahash_export,
4428 			.import = ahash_import,
4429 			.setkey = ahash_setkey,
4430 			.halg = {
4431 				.digestsize = SHA1_DIGEST_SIZE,
4432 				.statesize = sizeof(struct caam_export_state),
4433 			},
4434 		},
4435 		.alg_type = OP_ALG_ALGSEL_SHA1,
4436 	}, {
4437 		.name = "sha224",
4438 		.driver_name = "sha224-caam-qi2",
4439 		.hmac_name = "hmac(sha224)",
4440 		.hmac_driver_name = "hmac-sha224-caam-qi2",
4441 		.blocksize = SHA224_BLOCK_SIZE,
4442 		.template_ahash = {
4443 			.init = ahash_init,
4444 			.update = ahash_update,
4445 			.final = ahash_final,
4446 			.finup = ahash_finup,
4447 			.digest = ahash_digest,
4448 			.export = ahash_export,
4449 			.import = ahash_import,
4450 			.setkey = ahash_setkey,
4451 			.halg = {
4452 				.digestsize = SHA224_DIGEST_SIZE,
4453 				.statesize = sizeof(struct caam_export_state),
4454 			},
4455 		},
4456 		.alg_type = OP_ALG_ALGSEL_SHA224,
4457 	}, {
4458 		.name = "sha256",
4459 		.driver_name = "sha256-caam-qi2",
4460 		.hmac_name = "hmac(sha256)",
4461 		.hmac_driver_name = "hmac-sha256-caam-qi2",
4462 		.blocksize = SHA256_BLOCK_SIZE,
4463 		.template_ahash = {
4464 			.init = ahash_init,
4465 			.update = ahash_update,
4466 			.final = ahash_final,
4467 			.finup = ahash_finup,
4468 			.digest = ahash_digest,
4469 			.export = ahash_export,
4470 			.import = ahash_import,
4471 			.setkey = ahash_setkey,
4472 			.halg = {
4473 				.digestsize = SHA256_DIGEST_SIZE,
4474 				.statesize = sizeof(struct caam_export_state),
4475 			},
4476 		},
4477 		.alg_type = OP_ALG_ALGSEL_SHA256,
4478 	}, {
4479 		.name = "sha384",
4480 		.driver_name = "sha384-caam-qi2",
4481 		.hmac_name = "hmac(sha384)",
4482 		.hmac_driver_name = "hmac-sha384-caam-qi2",
4483 		.blocksize = SHA384_BLOCK_SIZE,
4484 		.template_ahash = {
4485 			.init = ahash_init,
4486 			.update = ahash_update,
4487 			.final = ahash_final,
4488 			.finup = ahash_finup,
4489 			.digest = ahash_digest,
4490 			.export = ahash_export,
4491 			.import = ahash_import,
4492 			.setkey = ahash_setkey,
4493 			.halg = {
4494 				.digestsize = SHA384_DIGEST_SIZE,
4495 				.statesize = sizeof(struct caam_export_state),
4496 			},
4497 		},
4498 		.alg_type = OP_ALG_ALGSEL_SHA384,
4499 	}, {
4500 		.name = "sha512",
4501 		.driver_name = "sha512-caam-qi2",
4502 		.hmac_name = "hmac(sha512)",
4503 		.hmac_driver_name = "hmac-sha512-caam-qi2",
4504 		.blocksize = SHA512_BLOCK_SIZE,
4505 		.template_ahash = {
4506 			.init = ahash_init,
4507 			.update = ahash_update,
4508 			.final = ahash_final,
4509 			.finup = ahash_finup,
4510 			.digest = ahash_digest,
4511 			.export = ahash_export,
4512 			.import = ahash_import,
4513 			.setkey = ahash_setkey,
4514 			.halg = {
4515 				.digestsize = SHA512_DIGEST_SIZE,
4516 				.statesize = sizeof(struct caam_export_state),
4517 			},
4518 		},
4519 		.alg_type = OP_ALG_ALGSEL_SHA512,
4520 	}, {
4521 		.name = "md5",
4522 		.driver_name = "md5-caam-qi2",
4523 		.hmac_name = "hmac(md5)",
4524 		.hmac_driver_name = "hmac-md5-caam-qi2",
4525 		.blocksize = MD5_BLOCK_WORDS * 4,
4526 		.template_ahash = {
4527 			.init = ahash_init,
4528 			.update = ahash_update,
4529 			.final = ahash_final,
4530 			.finup = ahash_finup,
4531 			.digest = ahash_digest,
4532 			.export = ahash_export,
4533 			.import = ahash_import,
4534 			.setkey = ahash_setkey,
4535 			.halg = {
4536 				.digestsize = MD5_DIGEST_SIZE,
4537 				.statesize = sizeof(struct caam_export_state),
4538 			},
4539 		},
4540 		.alg_type = OP_ALG_ALGSEL_MD5,
4541 	}
4542 };
4543 
4544 struct caam_hash_alg {
4545 	struct list_head entry;
4546 	struct device *dev;
4547 	int alg_type;
4548 	struct ahash_alg ahash_alg;
4549 };
4550 
4551 static int caam_hash_cra_init(struct crypto_tfm *tfm)
4552 {
4553 	struct crypto_ahash *ahash = __crypto_ahash_cast(tfm);
4554 	struct crypto_alg *base = tfm->__crt_alg;
4555 	struct hash_alg_common *halg =
4556 		 container_of(base, struct hash_alg_common, base);
4557 	struct ahash_alg *alg =
4558 		 container_of(halg, struct ahash_alg, halg);
4559 	struct caam_hash_alg *caam_hash =
4560 		 container_of(alg, struct caam_hash_alg, ahash_alg);
4561 	struct caam_hash_ctx *ctx = crypto_tfm_ctx_dma(tfm);
4562 	/* Sizes for MDHA running digests: MD5, SHA1, 224, 256, 384, 512 */
4563 	static const u8 runninglen[] = { HASH_MSG_LEN + MD5_DIGEST_SIZE,
4564 					 HASH_MSG_LEN + SHA1_DIGEST_SIZE,
4565 					 HASH_MSG_LEN + 32,
4566 					 HASH_MSG_LEN + SHA256_DIGEST_SIZE,
4567 					 HASH_MSG_LEN + 64,
4568 					 HASH_MSG_LEN + SHA512_DIGEST_SIZE };
4569 	dma_addr_t dma_addr;
4570 	int i;
4571 
4572 	ctx->dev = caam_hash->dev;
4573 
4574 	if (alg->setkey) {
4575 		ctx->adata.key_dma = dma_map_single_attrs(ctx->dev, ctx->key,
4576 							  ARRAY_SIZE(ctx->key),
4577 							  DMA_TO_DEVICE,
4578 							  DMA_ATTR_SKIP_CPU_SYNC);
4579 		if (dma_mapping_error(ctx->dev, ctx->adata.key_dma)) {
4580 			dev_err(ctx->dev, "unable to map key\n");
4581 			return -ENOMEM;
4582 		}
4583 	}
4584 
4585 	dma_addr = dma_map_single_attrs(ctx->dev, ctx->flc, sizeof(ctx->flc),
4586 					DMA_BIDIRECTIONAL,
4587 					DMA_ATTR_SKIP_CPU_SYNC);
4588 	if (dma_mapping_error(ctx->dev, dma_addr)) {
4589 		dev_err(ctx->dev, "unable to map shared descriptors\n");
4590 		if (ctx->adata.key_dma)
4591 			dma_unmap_single_attrs(ctx->dev, ctx->adata.key_dma,
4592 					       ARRAY_SIZE(ctx->key),
4593 					       DMA_TO_DEVICE,
4594 					       DMA_ATTR_SKIP_CPU_SYNC);
4595 		return -ENOMEM;
4596 	}
4597 
4598 	for (i = 0; i < HASH_NUM_OP; i++)
4599 		ctx->flc_dma[i] = dma_addr + i * sizeof(ctx->flc[i]);
4600 
4601 	/* copy descriptor header template value */
4602 	ctx->adata.algtype = OP_TYPE_CLASS2_ALG | caam_hash->alg_type;
4603 
4604 	ctx->ctx_len = runninglen[(ctx->adata.algtype &
4605 				   OP_ALG_ALGSEL_SUBMASK) >>
4606 				  OP_ALG_ALGSEL_SHIFT];
4607 
4608 	crypto_ahash_set_reqsize_dma(ahash, sizeof(struct caam_hash_state));
4609 
4610 	/*
4611 	 * For keyed hash algorithms shared descriptors
4612 	 * will be created later in setkey() callback
4613 	 */
4614 	return alg->setkey ? 0 : ahash_set_sh_desc(ahash);
4615 }
4616 
4617 static void caam_hash_cra_exit(struct crypto_tfm *tfm)
4618 {
4619 	struct caam_hash_ctx *ctx = crypto_tfm_ctx_dma(tfm);
4620 
4621 	dma_unmap_single_attrs(ctx->dev, ctx->flc_dma[0], sizeof(ctx->flc),
4622 			       DMA_BIDIRECTIONAL, DMA_ATTR_SKIP_CPU_SYNC);
4623 	if (ctx->adata.key_dma)
4624 		dma_unmap_single_attrs(ctx->dev, ctx->adata.key_dma,
4625 				       ARRAY_SIZE(ctx->key), DMA_TO_DEVICE,
4626 				       DMA_ATTR_SKIP_CPU_SYNC);
4627 }
4628 
4629 static struct caam_hash_alg *caam_hash_alloc(struct device *dev,
4630 	struct caam_hash_template *template, bool keyed)
4631 {
4632 	struct caam_hash_alg *t_alg;
4633 	struct ahash_alg *halg;
4634 	struct crypto_alg *alg;
4635 
4636 	t_alg = kzalloc(sizeof(*t_alg), GFP_KERNEL);
4637 	if (!t_alg)
4638 		return ERR_PTR(-ENOMEM);
4639 
4640 	t_alg->ahash_alg = template->template_ahash;
4641 	halg = &t_alg->ahash_alg;
4642 	alg = &halg->halg.base;
4643 
4644 	if (keyed) {
4645 		snprintf(alg->cra_name, CRYPTO_MAX_ALG_NAME, "%s",
4646 			 template->hmac_name);
4647 		snprintf(alg->cra_driver_name, CRYPTO_MAX_ALG_NAME, "%s",
4648 			 template->hmac_driver_name);
4649 	} else {
4650 		snprintf(alg->cra_name, CRYPTO_MAX_ALG_NAME, "%s",
4651 			 template->name);
4652 		snprintf(alg->cra_driver_name, CRYPTO_MAX_ALG_NAME, "%s",
4653 			 template->driver_name);
4654 		t_alg->ahash_alg.setkey = NULL;
4655 	}
4656 	alg->cra_module = THIS_MODULE;
4657 	alg->cra_init = caam_hash_cra_init;
4658 	alg->cra_exit = caam_hash_cra_exit;
4659 	alg->cra_ctxsize = sizeof(struct caam_hash_ctx) + crypto_dma_padding();
4660 	alg->cra_priority = CAAM_CRA_PRIORITY;
4661 	alg->cra_blocksize = template->blocksize;
4662 	alg->cra_alignmask = 0;
4663 	alg->cra_flags = CRYPTO_ALG_ASYNC | CRYPTO_ALG_ALLOCATES_MEMORY;
4664 
4665 	t_alg->alg_type = template->alg_type;
4666 	t_alg->dev = dev;
4667 
4668 	return t_alg;
4669 }
4670 
4671 static void dpaa2_caam_fqdan_cb(struct dpaa2_io_notification_ctx *nctx)
4672 {
4673 	struct dpaa2_caam_priv_per_cpu *ppriv;
4674 
4675 	ppriv = container_of(nctx, struct dpaa2_caam_priv_per_cpu, nctx);
4676 	napi_schedule_irqoff(&ppriv->napi);
4677 }
4678 
4679 static int __cold dpaa2_dpseci_dpio_setup(struct dpaa2_caam_priv *priv)
4680 {
4681 	struct device *dev = priv->dev;
4682 	struct dpaa2_io_notification_ctx *nctx;
4683 	struct dpaa2_caam_priv_per_cpu *ppriv;
4684 	int err, i = 0, cpu;
4685 
4686 	for_each_online_cpu(cpu) {
4687 		ppriv = per_cpu_ptr(priv->ppriv, cpu);
4688 		ppriv->priv = priv;
4689 		nctx = &ppriv->nctx;
4690 		nctx->is_cdan = 0;
4691 		nctx->id = ppriv->rsp_fqid;
4692 		nctx->desired_cpu = cpu;
4693 		nctx->cb = dpaa2_caam_fqdan_cb;
4694 
4695 		/* Register notification callbacks */
4696 		ppriv->dpio = dpaa2_io_service_select(cpu);
4697 		err = dpaa2_io_service_register(ppriv->dpio, nctx, dev);
4698 		if (unlikely(err)) {
4699 			dev_dbg(dev, "No affine DPIO for cpu %d\n", cpu);
4700 			nctx->cb = NULL;
4701 			/*
4702 			 * If no affine DPIO for this core, there's probably
4703 			 * none available for next cores either. Signal we want
4704 			 * to retry later, in case the DPIO devices weren't
4705 			 * probed yet.
4706 			 */
4707 			err = -EPROBE_DEFER;
4708 			goto err;
4709 		}
4710 
4711 		ppriv->store = dpaa2_io_store_create(DPAA2_CAAM_STORE_SIZE,
4712 						     dev);
4713 		if (unlikely(!ppriv->store)) {
4714 			dev_err(dev, "dpaa2_io_store_create() failed\n");
4715 			err = -ENOMEM;
4716 			goto err;
4717 		}
4718 
4719 		if (++i == priv->num_pairs)
4720 			break;
4721 	}
4722 
4723 	return 0;
4724 
4725 err:
4726 	for_each_online_cpu(cpu) {
4727 		ppriv = per_cpu_ptr(priv->ppriv, cpu);
4728 		if (!ppriv->nctx.cb)
4729 			break;
4730 		dpaa2_io_service_deregister(ppriv->dpio, &ppriv->nctx, dev);
4731 	}
4732 
4733 	for_each_online_cpu(cpu) {
4734 		ppriv = per_cpu_ptr(priv->ppriv, cpu);
4735 		if (!ppriv->store)
4736 			break;
4737 		dpaa2_io_store_destroy(ppriv->store);
4738 	}
4739 
4740 	return err;
4741 }
4742 
4743 static void __cold dpaa2_dpseci_dpio_free(struct dpaa2_caam_priv *priv)
4744 {
4745 	struct dpaa2_caam_priv_per_cpu *ppriv;
4746 	int i = 0, cpu;
4747 
4748 	for_each_online_cpu(cpu) {
4749 		ppriv = per_cpu_ptr(priv->ppriv, cpu);
4750 		dpaa2_io_service_deregister(ppriv->dpio, &ppriv->nctx,
4751 					    priv->dev);
4752 		dpaa2_io_store_destroy(ppriv->store);
4753 
4754 		if (++i == priv->num_pairs)
4755 			return;
4756 	}
4757 }
4758 
4759 static int dpaa2_dpseci_bind(struct dpaa2_caam_priv *priv)
4760 {
4761 	struct dpseci_rx_queue_cfg rx_queue_cfg;
4762 	struct device *dev = priv->dev;
4763 	struct fsl_mc_device *ls_dev = to_fsl_mc_device(dev);
4764 	struct dpaa2_caam_priv_per_cpu *ppriv;
4765 	int err = 0, i = 0, cpu;
4766 
4767 	/* Configure Rx queues */
4768 	for_each_online_cpu(cpu) {
4769 		ppriv = per_cpu_ptr(priv->ppriv, cpu);
4770 
4771 		rx_queue_cfg.options = DPSECI_QUEUE_OPT_DEST |
4772 				       DPSECI_QUEUE_OPT_USER_CTX;
4773 		rx_queue_cfg.order_preservation_en = 0;
4774 		rx_queue_cfg.dest_cfg.dest_type = DPSECI_DEST_DPIO;
4775 		rx_queue_cfg.dest_cfg.dest_id = ppriv->nctx.dpio_id;
4776 		/*
4777 		 * Rx priority (WQ) doesn't really matter, since we use
4778 		 * pull mode, i.e. volatile dequeues from specific FQs
4779 		 */
4780 		rx_queue_cfg.dest_cfg.priority = 0;
4781 		rx_queue_cfg.user_ctx = ppriv->nctx.qman64;
4782 
4783 		err = dpseci_set_rx_queue(priv->mc_io, 0, ls_dev->mc_handle, i,
4784 					  &rx_queue_cfg);
4785 		if (err) {
4786 			dev_err(dev, "dpseci_set_rx_queue() failed with err %d\n",
4787 				err);
4788 			return err;
4789 		}
4790 
4791 		if (++i == priv->num_pairs)
4792 			break;
4793 	}
4794 
4795 	return err;
4796 }
4797 
4798 static void dpaa2_dpseci_congestion_free(struct dpaa2_caam_priv *priv)
4799 {
4800 	struct device *dev = priv->dev;
4801 
4802 	if (!priv->cscn_mem)
4803 		return;
4804 
4805 	dma_unmap_single(dev, priv->cscn_dma, DPAA2_CSCN_SIZE, DMA_FROM_DEVICE);
4806 	kfree(priv->cscn_mem);
4807 }
4808 
4809 static void dpaa2_dpseci_free(struct dpaa2_caam_priv *priv)
4810 {
4811 	struct device *dev = priv->dev;
4812 	struct fsl_mc_device *ls_dev = to_fsl_mc_device(dev);
4813 	int err;
4814 
4815 	if (DPSECI_VER(priv->major_ver, priv->minor_ver) > DPSECI_VER(5, 3)) {
4816 		err = dpseci_reset(priv->mc_io, 0, ls_dev->mc_handle);
4817 		if (err)
4818 			dev_err(dev, "dpseci_reset() failed\n");
4819 	}
4820 
4821 	dpaa2_dpseci_congestion_free(priv);
4822 	dpseci_close(priv->mc_io, 0, ls_dev->mc_handle);
4823 }
4824 
4825 static void dpaa2_caam_process_fd(struct dpaa2_caam_priv *priv,
4826 				  const struct dpaa2_fd *fd)
4827 {
4828 	struct caam_request *req;
4829 	u32 fd_err;
4830 
4831 	if (dpaa2_fd_get_format(fd) != dpaa2_fd_list) {
4832 		dev_err(priv->dev, "Only Frame List FD format is supported!\n");
4833 		return;
4834 	}
4835 
4836 	fd_err = dpaa2_fd_get_ctrl(fd) & FD_CTRL_ERR_MASK;
4837 	if (unlikely(fd_err))
4838 		dev_err_ratelimited(priv->dev, "FD error: %08x\n", fd_err);
4839 
4840 	/*
4841 	 * FD[ADDR] is guaranteed to be valid, irrespective of errors reported
4842 	 * in FD[ERR] or FD[FRC].
4843 	 */
4844 	req = dpaa2_caam_iova_to_virt(priv, dpaa2_fd_get_addr(fd));
4845 	dma_unmap_single(priv->dev, req->fd_flt_dma, sizeof(req->fd_flt),
4846 			 DMA_BIDIRECTIONAL);
4847 	req->cbk(req->ctx, dpaa2_fd_get_frc(fd));
4848 }
4849 
4850 static int dpaa2_caam_pull_fq(struct dpaa2_caam_priv_per_cpu *ppriv)
4851 {
4852 	int err;
4853 
4854 	/* Retry while portal is busy */
4855 	do {
4856 		err = dpaa2_io_service_pull_fq(ppriv->dpio, ppriv->rsp_fqid,
4857 					       ppriv->store);
4858 	} while (err == -EBUSY);
4859 
4860 	if (unlikely(err))
4861 		dev_err(ppriv->priv->dev, "dpaa2_io_service_pull err %d", err);
4862 
4863 	return err;
4864 }
4865 
4866 static int dpaa2_caam_store_consume(struct dpaa2_caam_priv_per_cpu *ppriv)
4867 {
4868 	struct dpaa2_dq *dq;
4869 	int cleaned = 0, is_last;
4870 
4871 	do {
4872 		dq = dpaa2_io_store_next(ppriv->store, &is_last);
4873 		if (unlikely(!dq)) {
4874 			if (unlikely(!is_last)) {
4875 				dev_dbg(ppriv->priv->dev,
4876 					"FQ %d returned no valid frames\n",
4877 					ppriv->rsp_fqid);
4878 				/*
4879 				 * MUST retry until we get some sort of
4880 				 * valid response token (be it "empty dequeue"
4881 				 * or a valid frame).
4882 				 */
4883 				continue;
4884 			}
4885 			break;
4886 		}
4887 
4888 		/* Process FD */
4889 		dpaa2_caam_process_fd(ppriv->priv, dpaa2_dq_fd(dq));
4890 		cleaned++;
4891 	} while (!is_last);
4892 
4893 	return cleaned;
4894 }
4895 
4896 static int dpaa2_dpseci_poll(struct napi_struct *napi, int budget)
4897 {
4898 	struct dpaa2_caam_priv_per_cpu *ppriv;
4899 	struct dpaa2_caam_priv *priv;
4900 	int err, cleaned = 0, store_cleaned;
4901 
4902 	ppriv = container_of(napi, struct dpaa2_caam_priv_per_cpu, napi);
4903 	priv = ppriv->priv;
4904 
4905 	if (unlikely(dpaa2_caam_pull_fq(ppriv)))
4906 		return 0;
4907 
4908 	do {
4909 		store_cleaned = dpaa2_caam_store_consume(ppriv);
4910 		cleaned += store_cleaned;
4911 
4912 		if (store_cleaned == 0 ||
4913 		    cleaned > budget - DPAA2_CAAM_STORE_SIZE)
4914 			break;
4915 
4916 		/* Try to dequeue some more */
4917 		err = dpaa2_caam_pull_fq(ppriv);
4918 		if (unlikely(err))
4919 			break;
4920 	} while (1);
4921 
4922 	if (cleaned < budget) {
4923 		napi_complete_done(napi, cleaned);
4924 		err = dpaa2_io_service_rearm(ppriv->dpio, &ppriv->nctx);
4925 		if (unlikely(err))
4926 			dev_err(priv->dev, "Notification rearm failed: %d\n",
4927 				err);
4928 	}
4929 
4930 	return cleaned;
4931 }
4932 
4933 static int dpaa2_dpseci_congestion_setup(struct dpaa2_caam_priv *priv,
4934 					 u16 token)
4935 {
4936 	struct dpseci_congestion_notification_cfg cong_notif_cfg = { 0 };
4937 	struct device *dev = priv->dev;
4938 	unsigned int alignmask;
4939 	int err;
4940 
4941 	/*
4942 	 * Congestion group feature supported starting with DPSECI API v5.1
4943 	 * and only when object has been created with this capability.
4944 	 */
4945 	if ((DPSECI_VER(priv->major_ver, priv->minor_ver) < DPSECI_VER(5, 1)) ||
4946 	    !(priv->dpseci_attr.options & DPSECI_OPT_HAS_CG))
4947 		return 0;
4948 
4949 	alignmask = DPAA2_CSCN_ALIGN - 1;
4950 	alignmask |= dma_get_cache_alignment() - 1;
4951 	priv->cscn_mem = kzalloc(ALIGN(DPAA2_CSCN_SIZE, alignmask + 1),
4952 				 GFP_KERNEL);
4953 	if (!priv->cscn_mem)
4954 		return -ENOMEM;
4955 
4956 	priv->cscn_dma = dma_map_single(dev, priv->cscn_mem,
4957 					DPAA2_CSCN_SIZE, DMA_FROM_DEVICE);
4958 	if (dma_mapping_error(dev, priv->cscn_dma)) {
4959 		dev_err(dev, "Error mapping CSCN memory area\n");
4960 		err = -ENOMEM;
4961 		goto err_dma_map;
4962 	}
4963 
4964 	cong_notif_cfg.units = DPSECI_CONGESTION_UNIT_BYTES;
4965 	cong_notif_cfg.threshold_entry = DPAA2_SEC_CONG_ENTRY_THRESH;
4966 	cong_notif_cfg.threshold_exit = DPAA2_SEC_CONG_EXIT_THRESH;
4967 	cong_notif_cfg.message_ctx = (uintptr_t)priv;
4968 	cong_notif_cfg.message_iova = priv->cscn_dma;
4969 	cong_notif_cfg.notification_mode = DPSECI_CGN_MODE_WRITE_MEM_ON_ENTER |
4970 					DPSECI_CGN_MODE_WRITE_MEM_ON_EXIT |
4971 					DPSECI_CGN_MODE_COHERENT_WRITE;
4972 
4973 	err = dpseci_set_congestion_notification(priv->mc_io, 0, token,
4974 						 &cong_notif_cfg);
4975 	if (err) {
4976 		dev_err(dev, "dpseci_set_congestion_notification failed\n");
4977 		goto err_set_cong;
4978 	}
4979 
4980 	return 0;
4981 
4982 err_set_cong:
4983 	dma_unmap_single(dev, priv->cscn_dma, DPAA2_CSCN_SIZE, DMA_FROM_DEVICE);
4984 err_dma_map:
4985 	kfree(priv->cscn_mem);
4986 
4987 	return err;
4988 }
4989 
4990 static int __cold dpaa2_dpseci_setup(struct fsl_mc_device *ls_dev)
4991 {
4992 	struct device *dev = &ls_dev->dev;
4993 	struct dpaa2_caam_priv *priv;
4994 	struct dpaa2_caam_priv_per_cpu *ppriv;
4995 	int err, cpu;
4996 	u8 i;
4997 
4998 	priv = dev_get_drvdata(dev);
4999 
5000 	priv->dev = dev;
5001 	priv->dpsec_id = ls_dev->obj_desc.id;
5002 
5003 	/* Get a handle for the DPSECI this interface is associate with */
5004 	err = dpseci_open(priv->mc_io, 0, priv->dpsec_id, &ls_dev->mc_handle);
5005 	if (err) {
5006 		dev_err(dev, "dpseci_open() failed: %d\n", err);
5007 		goto err_open;
5008 	}
5009 
5010 	err = dpseci_get_api_version(priv->mc_io, 0, &priv->major_ver,
5011 				     &priv->minor_ver);
5012 	if (err) {
5013 		dev_err(dev, "dpseci_get_api_version() failed\n");
5014 		goto err_get_vers;
5015 	}
5016 
5017 	dev_info(dev, "dpseci v%d.%d\n", priv->major_ver, priv->minor_ver);
5018 
5019 	if (DPSECI_VER(priv->major_ver, priv->minor_ver) > DPSECI_VER(5, 3)) {
5020 		err = dpseci_reset(priv->mc_io, 0, ls_dev->mc_handle);
5021 		if (err) {
5022 			dev_err(dev, "dpseci_reset() failed\n");
5023 			goto err_get_vers;
5024 		}
5025 	}
5026 
5027 	err = dpseci_get_attributes(priv->mc_io, 0, ls_dev->mc_handle,
5028 				    &priv->dpseci_attr);
5029 	if (err) {
5030 		dev_err(dev, "dpseci_get_attributes() failed\n");
5031 		goto err_get_vers;
5032 	}
5033 
5034 	err = dpseci_get_sec_attr(priv->mc_io, 0, ls_dev->mc_handle,
5035 				  &priv->sec_attr);
5036 	if (err) {
5037 		dev_err(dev, "dpseci_get_sec_attr() failed\n");
5038 		goto err_get_vers;
5039 	}
5040 
5041 	err = dpaa2_dpseci_congestion_setup(priv, ls_dev->mc_handle);
5042 	if (err) {
5043 		dev_err(dev, "setup_congestion() failed\n");
5044 		goto err_get_vers;
5045 	}
5046 
5047 	priv->num_pairs = min(priv->dpseci_attr.num_rx_queues,
5048 			      priv->dpseci_attr.num_tx_queues);
5049 	if (priv->num_pairs > num_online_cpus()) {
5050 		dev_warn(dev, "%d queues won't be used\n",
5051 			 priv->num_pairs - num_online_cpus());
5052 		priv->num_pairs = num_online_cpus();
5053 	}
5054 
5055 	for (i = 0; i < priv->dpseci_attr.num_rx_queues; i++) {
5056 		err = dpseci_get_rx_queue(priv->mc_io, 0, ls_dev->mc_handle, i,
5057 					  &priv->rx_queue_attr[i]);
5058 		if (err) {
5059 			dev_err(dev, "dpseci_get_rx_queue() failed\n");
5060 			goto err_get_rx_queue;
5061 		}
5062 	}
5063 
5064 	for (i = 0; i < priv->dpseci_attr.num_tx_queues; i++) {
5065 		err = dpseci_get_tx_queue(priv->mc_io, 0, ls_dev->mc_handle, i,
5066 					  &priv->tx_queue_attr[i]);
5067 		if (err) {
5068 			dev_err(dev, "dpseci_get_tx_queue() failed\n");
5069 			goto err_get_rx_queue;
5070 		}
5071 	}
5072 
5073 	i = 0;
5074 	for_each_online_cpu(cpu) {
5075 		u8 j;
5076 
5077 		j = i % priv->num_pairs;
5078 
5079 		ppriv = per_cpu_ptr(priv->ppriv, cpu);
5080 		ppriv->req_fqid = priv->tx_queue_attr[j].fqid;
5081 
5082 		/*
5083 		 * Allow all cores to enqueue, while only some of them
5084 		 * will take part in dequeuing.
5085 		 */
5086 		if (++i > priv->num_pairs)
5087 			continue;
5088 
5089 		ppriv->rsp_fqid = priv->rx_queue_attr[j].fqid;
5090 		ppriv->prio = j;
5091 
5092 		dev_dbg(dev, "pair %d: rx queue %d, tx queue %d\n", j,
5093 			priv->rx_queue_attr[j].fqid,
5094 			priv->tx_queue_attr[j].fqid);
5095 
5096 		ppriv->net_dev.dev = *dev;
5097 		INIT_LIST_HEAD(&ppriv->net_dev.napi_list);
5098 		netif_napi_add_tx_weight(&ppriv->net_dev, &ppriv->napi,
5099 					 dpaa2_dpseci_poll,
5100 					 DPAA2_CAAM_NAPI_WEIGHT);
5101 	}
5102 
5103 	return 0;
5104 
5105 err_get_rx_queue:
5106 	dpaa2_dpseci_congestion_free(priv);
5107 err_get_vers:
5108 	dpseci_close(priv->mc_io, 0, ls_dev->mc_handle);
5109 err_open:
5110 	return err;
5111 }
5112 
5113 static int dpaa2_dpseci_enable(struct dpaa2_caam_priv *priv)
5114 {
5115 	struct device *dev = priv->dev;
5116 	struct fsl_mc_device *ls_dev = to_fsl_mc_device(dev);
5117 	struct dpaa2_caam_priv_per_cpu *ppriv;
5118 	int i;
5119 
5120 	for (i = 0; i < priv->num_pairs; i++) {
5121 		ppriv = per_cpu_ptr(priv->ppriv, i);
5122 		napi_enable(&ppriv->napi);
5123 	}
5124 
5125 	return dpseci_enable(priv->mc_io, 0, ls_dev->mc_handle);
5126 }
5127 
5128 static int __cold dpaa2_dpseci_disable(struct dpaa2_caam_priv *priv)
5129 {
5130 	struct device *dev = priv->dev;
5131 	struct dpaa2_caam_priv_per_cpu *ppriv;
5132 	struct fsl_mc_device *ls_dev = to_fsl_mc_device(dev);
5133 	int i, err = 0, enabled;
5134 
5135 	err = dpseci_disable(priv->mc_io, 0, ls_dev->mc_handle);
5136 	if (err) {
5137 		dev_err(dev, "dpseci_disable() failed\n");
5138 		return err;
5139 	}
5140 
5141 	err = dpseci_is_enabled(priv->mc_io, 0, ls_dev->mc_handle, &enabled);
5142 	if (err) {
5143 		dev_err(dev, "dpseci_is_enabled() failed\n");
5144 		return err;
5145 	}
5146 
5147 	dev_dbg(dev, "disable: %s\n", enabled ? "false" : "true");
5148 
5149 	for (i = 0; i < priv->num_pairs; i++) {
5150 		ppriv = per_cpu_ptr(priv->ppriv, i);
5151 		napi_disable(&ppriv->napi);
5152 		netif_napi_del(&ppriv->napi);
5153 	}
5154 
5155 	return 0;
5156 }
5157 
5158 static struct list_head hash_list;
5159 
5160 static int dpaa2_caam_probe(struct fsl_mc_device *dpseci_dev)
5161 {
5162 	struct device *dev;
5163 	struct dpaa2_caam_priv *priv;
5164 	int i, err = 0;
5165 	bool registered = false;
5166 
5167 	/*
5168 	 * There is no way to get CAAM endianness - there is no direct register
5169 	 * space access and MC f/w does not provide this attribute.
5170 	 * All DPAA2-based SoCs have little endian CAAM, thus hard-code this
5171 	 * property.
5172 	 */
5173 	caam_little_end = true;
5174 
5175 	caam_imx = false;
5176 
5177 	dev = &dpseci_dev->dev;
5178 
5179 	priv = devm_kzalloc(dev, sizeof(*priv), GFP_KERNEL);
5180 	if (!priv)
5181 		return -ENOMEM;
5182 
5183 	dev_set_drvdata(dev, priv);
5184 
5185 	priv->domain = iommu_get_domain_for_dev(dev);
5186 
5187 	qi_cache = kmem_cache_create("dpaa2_caamqicache", CAAM_QI_MEMCACHE_SIZE,
5188 				     0, 0, NULL);
5189 	if (!qi_cache) {
5190 		dev_err(dev, "Can't allocate SEC cache\n");
5191 		return -ENOMEM;
5192 	}
5193 
5194 	err = dma_set_mask_and_coherent(dev, DMA_BIT_MASK(49));
5195 	if (err) {
5196 		dev_err(dev, "dma_set_mask_and_coherent() failed\n");
5197 		goto err_dma_mask;
5198 	}
5199 
5200 	/* Obtain a MC portal */
5201 	err = fsl_mc_portal_allocate(dpseci_dev, 0, &priv->mc_io);
5202 	if (err) {
5203 		if (err == -ENXIO)
5204 			err = -EPROBE_DEFER;
5205 		else
5206 			dev_err(dev, "MC portal allocation failed\n");
5207 
5208 		goto err_dma_mask;
5209 	}
5210 
5211 	priv->ppriv = alloc_percpu(*priv->ppriv);
5212 	if (!priv->ppriv) {
5213 		dev_err(dev, "alloc_percpu() failed\n");
5214 		err = -ENOMEM;
5215 		goto err_alloc_ppriv;
5216 	}
5217 
5218 	/* DPSECI initialization */
5219 	err = dpaa2_dpseci_setup(dpseci_dev);
5220 	if (err) {
5221 		dev_err(dev, "dpaa2_dpseci_setup() failed\n");
5222 		goto err_dpseci_setup;
5223 	}
5224 
5225 	/* DPIO */
5226 	err = dpaa2_dpseci_dpio_setup(priv);
5227 	if (err) {
5228 		dev_err_probe(dev, err, "dpaa2_dpseci_dpio_setup() failed\n");
5229 		goto err_dpio_setup;
5230 	}
5231 
5232 	/* DPSECI binding to DPIO */
5233 	err = dpaa2_dpseci_bind(priv);
5234 	if (err) {
5235 		dev_err(dev, "dpaa2_dpseci_bind() failed\n");
5236 		goto err_bind;
5237 	}
5238 
5239 	/* DPSECI enable */
5240 	err = dpaa2_dpseci_enable(priv);
5241 	if (err) {
5242 		dev_err(dev, "dpaa2_dpseci_enable() failed\n");
5243 		goto err_bind;
5244 	}
5245 
5246 	dpaa2_dpseci_debugfs_init(priv);
5247 
5248 	/* register crypto algorithms the device supports */
5249 	for (i = 0; i < ARRAY_SIZE(driver_algs); i++) {
5250 		struct caam_skcipher_alg *t_alg = driver_algs + i;
5251 		u32 alg_sel = t_alg->caam.class1_alg_type & OP_ALG_ALGSEL_MASK;
5252 
5253 		/* Skip DES algorithms if not supported by device */
5254 		if (!priv->sec_attr.des_acc_num &&
5255 		    (alg_sel == OP_ALG_ALGSEL_3DES ||
5256 		     alg_sel == OP_ALG_ALGSEL_DES))
5257 			continue;
5258 
5259 		/* Skip AES algorithms if not supported by device */
5260 		if (!priv->sec_attr.aes_acc_num &&
5261 		    alg_sel == OP_ALG_ALGSEL_AES)
5262 			continue;
5263 
5264 		/* Skip CHACHA20 algorithms if not supported by device */
5265 		if (alg_sel == OP_ALG_ALGSEL_CHACHA20 &&
5266 		    !priv->sec_attr.ccha_acc_num)
5267 			continue;
5268 
5269 		t_alg->caam.dev = dev;
5270 		caam_skcipher_alg_init(t_alg);
5271 
5272 		err = crypto_register_skcipher(&t_alg->skcipher);
5273 		if (err) {
5274 			dev_warn(dev, "%s alg registration failed: %d\n",
5275 				 t_alg->skcipher.base.cra_driver_name, err);
5276 			continue;
5277 		}
5278 
5279 		t_alg->registered = true;
5280 		registered = true;
5281 	}
5282 
5283 	for (i = 0; i < ARRAY_SIZE(driver_aeads); i++) {
5284 		struct caam_aead_alg *t_alg = driver_aeads + i;
5285 		u32 c1_alg_sel = t_alg->caam.class1_alg_type &
5286 				 OP_ALG_ALGSEL_MASK;
5287 		u32 c2_alg_sel = t_alg->caam.class2_alg_type &
5288 				 OP_ALG_ALGSEL_MASK;
5289 
5290 		/* Skip DES algorithms if not supported by device */
5291 		if (!priv->sec_attr.des_acc_num &&
5292 		    (c1_alg_sel == OP_ALG_ALGSEL_3DES ||
5293 		     c1_alg_sel == OP_ALG_ALGSEL_DES))
5294 			continue;
5295 
5296 		/* Skip AES algorithms if not supported by device */
5297 		if (!priv->sec_attr.aes_acc_num &&
5298 		    c1_alg_sel == OP_ALG_ALGSEL_AES)
5299 			continue;
5300 
5301 		/* Skip CHACHA20 algorithms if not supported by device */
5302 		if (c1_alg_sel == OP_ALG_ALGSEL_CHACHA20 &&
5303 		    !priv->sec_attr.ccha_acc_num)
5304 			continue;
5305 
5306 		/* Skip POLY1305 algorithms if not supported by device */
5307 		if (c2_alg_sel == OP_ALG_ALGSEL_POLY1305 &&
5308 		    !priv->sec_attr.ptha_acc_num)
5309 			continue;
5310 
5311 		/*
5312 		 * Skip algorithms requiring message digests
5313 		 * if MD not supported by device.
5314 		 */
5315 		if ((c2_alg_sel & ~OP_ALG_ALGSEL_SUBMASK) == 0x40 &&
5316 		    !priv->sec_attr.md_acc_num)
5317 			continue;
5318 
5319 		t_alg->caam.dev = dev;
5320 		caam_aead_alg_init(t_alg);
5321 
5322 		err = crypto_register_aead(&t_alg->aead);
5323 		if (err) {
5324 			dev_warn(dev, "%s alg registration failed: %d\n",
5325 				 t_alg->aead.base.cra_driver_name, err);
5326 			continue;
5327 		}
5328 
5329 		t_alg->registered = true;
5330 		registered = true;
5331 	}
5332 	if (registered)
5333 		dev_info(dev, "algorithms registered in /proc/crypto\n");
5334 
5335 	/* register hash algorithms the device supports */
5336 	INIT_LIST_HEAD(&hash_list);
5337 
5338 	/*
5339 	 * Skip registration of any hashing algorithms if MD block
5340 	 * is not present.
5341 	 */
5342 	if (!priv->sec_attr.md_acc_num)
5343 		return 0;
5344 
5345 	for (i = 0; i < ARRAY_SIZE(driver_hash); i++) {
5346 		struct caam_hash_alg *t_alg;
5347 		struct caam_hash_template *alg = driver_hash + i;
5348 
5349 		/* register hmac version */
5350 		t_alg = caam_hash_alloc(dev, alg, true);
5351 		if (IS_ERR(t_alg)) {
5352 			err = PTR_ERR(t_alg);
5353 			dev_warn(dev, "%s hash alg allocation failed: %d\n",
5354 				 alg->hmac_driver_name, err);
5355 			continue;
5356 		}
5357 
5358 		err = crypto_register_ahash(&t_alg->ahash_alg);
5359 		if (err) {
5360 			dev_warn(dev, "%s alg registration failed: %d\n",
5361 				 t_alg->ahash_alg.halg.base.cra_driver_name,
5362 				 err);
5363 			kfree(t_alg);
5364 		} else {
5365 			list_add_tail(&t_alg->entry, &hash_list);
5366 		}
5367 
5368 		/* register unkeyed version */
5369 		t_alg = caam_hash_alloc(dev, alg, false);
5370 		if (IS_ERR(t_alg)) {
5371 			err = PTR_ERR(t_alg);
5372 			dev_warn(dev, "%s alg allocation failed: %d\n",
5373 				 alg->driver_name, err);
5374 			continue;
5375 		}
5376 
5377 		err = crypto_register_ahash(&t_alg->ahash_alg);
5378 		if (err) {
5379 			dev_warn(dev, "%s alg registration failed: %d\n",
5380 				 t_alg->ahash_alg.halg.base.cra_driver_name,
5381 				 err);
5382 			kfree(t_alg);
5383 		} else {
5384 			list_add_tail(&t_alg->entry, &hash_list);
5385 		}
5386 	}
5387 	if (!list_empty(&hash_list))
5388 		dev_info(dev, "hash algorithms registered in /proc/crypto\n");
5389 
5390 	return err;
5391 
5392 err_bind:
5393 	dpaa2_dpseci_dpio_free(priv);
5394 err_dpio_setup:
5395 	dpaa2_dpseci_free(priv);
5396 err_dpseci_setup:
5397 	free_percpu(priv->ppriv);
5398 err_alloc_ppriv:
5399 	fsl_mc_portal_free(priv->mc_io);
5400 err_dma_mask:
5401 	kmem_cache_destroy(qi_cache);
5402 
5403 	return err;
5404 }
5405 
5406 static void __cold dpaa2_caam_remove(struct fsl_mc_device *ls_dev)
5407 {
5408 	struct device *dev;
5409 	struct dpaa2_caam_priv *priv;
5410 	int i;
5411 
5412 	dev = &ls_dev->dev;
5413 	priv = dev_get_drvdata(dev);
5414 
5415 	dpaa2_dpseci_debugfs_exit(priv);
5416 
5417 	for (i = 0; i < ARRAY_SIZE(driver_aeads); i++) {
5418 		struct caam_aead_alg *t_alg = driver_aeads + i;
5419 
5420 		if (t_alg->registered)
5421 			crypto_unregister_aead(&t_alg->aead);
5422 	}
5423 
5424 	for (i = 0; i < ARRAY_SIZE(driver_algs); i++) {
5425 		struct caam_skcipher_alg *t_alg = driver_algs + i;
5426 
5427 		if (t_alg->registered)
5428 			crypto_unregister_skcipher(&t_alg->skcipher);
5429 	}
5430 
5431 	if (hash_list.next) {
5432 		struct caam_hash_alg *t_hash_alg, *p;
5433 
5434 		list_for_each_entry_safe(t_hash_alg, p, &hash_list, entry) {
5435 			crypto_unregister_ahash(&t_hash_alg->ahash_alg);
5436 			list_del(&t_hash_alg->entry);
5437 			kfree(t_hash_alg);
5438 		}
5439 	}
5440 
5441 	dpaa2_dpseci_disable(priv);
5442 	dpaa2_dpseci_dpio_free(priv);
5443 	dpaa2_dpseci_free(priv);
5444 	free_percpu(priv->ppriv);
5445 	fsl_mc_portal_free(priv->mc_io);
5446 	kmem_cache_destroy(qi_cache);
5447 }
5448 
5449 int dpaa2_caam_enqueue(struct device *dev, struct caam_request *req)
5450 {
5451 	struct dpaa2_fd fd;
5452 	struct dpaa2_caam_priv *priv = dev_get_drvdata(dev);
5453 	struct dpaa2_caam_priv_per_cpu *ppriv;
5454 	int err = 0, i;
5455 
5456 	if (IS_ERR(req))
5457 		return PTR_ERR(req);
5458 
5459 	if (priv->cscn_mem) {
5460 		dma_sync_single_for_cpu(priv->dev, priv->cscn_dma,
5461 					DPAA2_CSCN_SIZE,
5462 					DMA_FROM_DEVICE);
5463 		if (unlikely(dpaa2_cscn_state_congested(priv->cscn_mem))) {
5464 			dev_dbg_ratelimited(dev, "Dropping request\n");
5465 			return -EBUSY;
5466 		}
5467 	}
5468 
5469 	dpaa2_fl_set_flc(&req->fd_flt[1], req->flc_dma);
5470 
5471 	req->fd_flt_dma = dma_map_single(dev, req->fd_flt, sizeof(req->fd_flt),
5472 					 DMA_BIDIRECTIONAL);
5473 	if (dma_mapping_error(dev, req->fd_flt_dma)) {
5474 		dev_err(dev, "DMA mapping error for QI enqueue request\n");
5475 		goto err_out;
5476 	}
5477 
5478 	memset(&fd, 0, sizeof(fd));
5479 	dpaa2_fd_set_format(&fd, dpaa2_fd_list);
5480 	dpaa2_fd_set_addr(&fd, req->fd_flt_dma);
5481 	dpaa2_fd_set_len(&fd, dpaa2_fl_get_len(&req->fd_flt[1]));
5482 	dpaa2_fd_set_flc(&fd, req->flc_dma);
5483 
5484 	ppriv = raw_cpu_ptr(priv->ppriv);
5485 	for (i = 0; i < (priv->dpseci_attr.num_tx_queues << 1); i++) {
5486 		err = dpaa2_io_service_enqueue_fq(ppriv->dpio, ppriv->req_fqid,
5487 						  &fd);
5488 		if (err != -EBUSY)
5489 			break;
5490 
5491 		cpu_relax();
5492 	}
5493 
5494 	if (unlikely(err)) {
5495 		dev_err_ratelimited(dev, "Error enqueuing frame: %d\n", err);
5496 		goto err_out;
5497 	}
5498 
5499 	return -EINPROGRESS;
5500 
5501 err_out:
5502 	dma_unmap_single(dev, req->fd_flt_dma, sizeof(req->fd_flt),
5503 			 DMA_BIDIRECTIONAL);
5504 	return -EIO;
5505 }
5506 EXPORT_SYMBOL(dpaa2_caam_enqueue);
5507 
5508 static const struct fsl_mc_device_id dpaa2_caam_match_id_table[] = {
5509 	{
5510 		.vendor = FSL_MC_VENDOR_FREESCALE,
5511 		.obj_type = "dpseci",
5512 	},
5513 	{ .vendor = 0x0 }
5514 };
5515 MODULE_DEVICE_TABLE(fslmc, dpaa2_caam_match_id_table);
5516 
5517 static struct fsl_mc_driver dpaa2_caam_driver = {
5518 	.driver = {
5519 		.name		= KBUILD_MODNAME,
5520 		.owner		= THIS_MODULE,
5521 	},
5522 	.probe		= dpaa2_caam_probe,
5523 	.remove		= dpaa2_caam_remove,
5524 	.match_id_table = dpaa2_caam_match_id_table
5525 };
5526 
5527 MODULE_LICENSE("Dual BSD/GPL");
5528 MODULE_AUTHOR("Freescale Semiconductor, Inc");
5529 MODULE_DESCRIPTION("Freescale DPAA2 CAAM Driver");
5530 
5531 module_fsl_mc_driver(dpaa2_caam_driver);
5532