1 // SPDX-License-Identifier: (GPL-2.0+ OR BSD-3-Clause)
2 /*
3  * Copyright 2015-2016 Freescale Semiconductor Inc.
4  * Copyright 2017-2019 NXP
5  */
6 
7 #include "compat.h"
8 #include "regs.h"
9 #include "caamalg_qi2.h"
10 #include "dpseci_cmd.h"
11 #include "desc_constr.h"
12 #include "error.h"
13 #include "sg_sw_sec4.h"
14 #include "sg_sw_qm2.h"
15 #include "key_gen.h"
16 #include "caamalg_desc.h"
17 #include "caamhash_desc.h"
18 #include "dpseci-debugfs.h"
19 #include <linux/fsl/mc.h>
20 #include <soc/fsl/dpaa2-io.h>
21 #include <soc/fsl/dpaa2-fd.h>
22 #include <crypto/xts.h>
23 #include <asm/unaligned.h>
24 
25 #define CAAM_CRA_PRIORITY	2000
26 
27 /* max key is sum of AES_MAX_KEY_SIZE, max split key size */
28 #define CAAM_MAX_KEY_SIZE	(AES_MAX_KEY_SIZE + CTR_RFC3686_NONCE_SIZE + \
29 				 SHA512_DIGEST_SIZE * 2)
30 
31 /*
32  * This is a cache of buffers, from which the users of CAAM QI driver
33  * can allocate short buffers. It's speedier than doing kmalloc on the hotpath.
34  * NOTE: A more elegant solution would be to have some headroom in the frames
35  *       being processed. This can be added by the dpaa2-eth driver. This would
36  *       pose a problem for userspace application processing which cannot
37  *       know of this limitation. So for now, this will work.
38  * NOTE: The memcache is SMP-safe. No need to handle spinlocks in-here
39  */
40 static struct kmem_cache *qi_cache;
41 
42 struct caam_alg_entry {
43 	struct device *dev;
44 	int class1_alg_type;
45 	int class2_alg_type;
46 	bool rfc3686;
47 	bool geniv;
48 	bool nodkp;
49 };
50 
51 struct caam_aead_alg {
52 	struct aead_alg aead;
53 	struct caam_alg_entry caam;
54 	bool registered;
55 };
56 
57 struct caam_skcipher_alg {
58 	struct skcipher_alg skcipher;
59 	struct caam_alg_entry caam;
60 	bool registered;
61 };
62 
63 /**
64  * struct caam_ctx - per-session context
65  * @flc: Flow Contexts array
66  * @key:  [authentication key], encryption key
67  * @flc_dma: I/O virtual addresses of the Flow Contexts
68  * @key_dma: I/O virtual address of the key
69  * @dir: DMA direction for mapping key and Flow Contexts
70  * @dev: dpseci device
71  * @adata: authentication algorithm details
72  * @cdata: encryption algorithm details
73  * @authsize: authentication tag (a.k.a. ICV / MAC) size
74  * @xts_key_fallback: true if fallback tfm needs to be used due
75  *		      to unsupported xts key lengths
76  * @fallback: xts fallback tfm
77  */
78 struct caam_ctx {
79 	struct caam_flc flc[NUM_OP];
80 	u8 key[CAAM_MAX_KEY_SIZE];
81 	dma_addr_t flc_dma[NUM_OP];
82 	dma_addr_t key_dma;
83 	enum dma_data_direction dir;
84 	struct device *dev;
85 	struct alginfo adata;
86 	struct alginfo cdata;
87 	unsigned int authsize;
88 	bool xts_key_fallback;
89 	struct crypto_skcipher *fallback;
90 };
91 
92 static void *dpaa2_caam_iova_to_virt(struct dpaa2_caam_priv *priv,
93 				     dma_addr_t iova_addr)
94 {
95 	phys_addr_t phys_addr;
96 
97 	phys_addr = priv->domain ? iommu_iova_to_phys(priv->domain, iova_addr) :
98 				   iova_addr;
99 
100 	return phys_to_virt(phys_addr);
101 }
102 
103 /*
104  * qi_cache_zalloc - Allocate buffers from CAAM-QI cache
105  *
106  * Allocate data on the hotpath. Instead of using kzalloc, one can use the
107  * services of the CAAM QI memory cache (backed by kmem_cache). The buffers
108  * will have a size of CAAM_QI_MEMCACHE_SIZE, which should be sufficient for
109  * hosting 16 SG entries.
110  *
111  * @flags - flags that would be used for the equivalent kmalloc(..) call
112  *
113  * Returns a pointer to a retrieved buffer on success or NULL on failure.
114  */
115 static inline void *qi_cache_zalloc(gfp_t flags)
116 {
117 	return kmem_cache_zalloc(qi_cache, flags);
118 }
119 
120 /*
121  * qi_cache_free - Frees buffers allocated from CAAM-QI cache
122  *
123  * @obj - buffer previously allocated by qi_cache_zalloc
124  *
125  * No checking is being done, the call is a passthrough call to
126  * kmem_cache_free(...)
127  */
128 static inline void qi_cache_free(void *obj)
129 {
130 	kmem_cache_free(qi_cache, obj);
131 }
132 
133 static struct caam_request *to_caam_req(struct crypto_async_request *areq)
134 {
135 	switch (crypto_tfm_alg_type(areq->tfm)) {
136 	case CRYPTO_ALG_TYPE_SKCIPHER:
137 		return skcipher_request_ctx_dma(skcipher_request_cast(areq));
138 	case CRYPTO_ALG_TYPE_AEAD:
139 		return aead_request_ctx_dma(
140 			container_of(areq, struct aead_request, base));
141 	case CRYPTO_ALG_TYPE_AHASH:
142 		return ahash_request_ctx_dma(ahash_request_cast(areq));
143 	default:
144 		return ERR_PTR(-EINVAL);
145 	}
146 }
147 
148 static void caam_unmap(struct device *dev, struct scatterlist *src,
149 		       struct scatterlist *dst, int src_nents,
150 		       int dst_nents, dma_addr_t iv_dma, int ivsize,
151 		       enum dma_data_direction iv_dir, dma_addr_t qm_sg_dma,
152 		       int qm_sg_bytes)
153 {
154 	if (dst != src) {
155 		if (src_nents)
156 			dma_unmap_sg(dev, src, src_nents, DMA_TO_DEVICE);
157 		if (dst_nents)
158 			dma_unmap_sg(dev, dst, dst_nents, DMA_FROM_DEVICE);
159 	} else {
160 		dma_unmap_sg(dev, src, src_nents, DMA_BIDIRECTIONAL);
161 	}
162 
163 	if (iv_dma)
164 		dma_unmap_single(dev, iv_dma, ivsize, iv_dir);
165 
166 	if (qm_sg_bytes)
167 		dma_unmap_single(dev, qm_sg_dma, qm_sg_bytes, DMA_TO_DEVICE);
168 }
169 
170 static int aead_set_sh_desc(struct crypto_aead *aead)
171 {
172 	struct caam_aead_alg *alg = container_of(crypto_aead_alg(aead),
173 						 typeof(*alg), aead);
174 	struct caam_ctx *ctx = crypto_aead_ctx_dma(aead);
175 	unsigned int ivsize = crypto_aead_ivsize(aead);
176 	struct device *dev = ctx->dev;
177 	struct dpaa2_caam_priv *priv = dev_get_drvdata(dev);
178 	struct caam_flc *flc;
179 	u32 *desc;
180 	u32 ctx1_iv_off = 0;
181 	u32 *nonce = NULL;
182 	unsigned int data_len[2];
183 	u32 inl_mask;
184 	const bool ctr_mode = ((ctx->cdata.algtype & OP_ALG_AAI_MASK) ==
185 			       OP_ALG_AAI_CTR_MOD128);
186 	const bool is_rfc3686 = alg->caam.rfc3686;
187 
188 	if (!ctx->cdata.keylen || !ctx->authsize)
189 		return 0;
190 
191 	/*
192 	 * AES-CTR needs to load IV in CONTEXT1 reg
193 	 * at an offset of 128bits (16bytes)
194 	 * CONTEXT1[255:128] = IV
195 	 */
196 	if (ctr_mode)
197 		ctx1_iv_off = 16;
198 
199 	/*
200 	 * RFC3686 specific:
201 	 *	CONTEXT1[255:128] = {NONCE, IV, COUNTER}
202 	 */
203 	if (is_rfc3686) {
204 		ctx1_iv_off = 16 + CTR_RFC3686_NONCE_SIZE;
205 		nonce = (u32 *)((void *)ctx->key + ctx->adata.keylen_pad +
206 				ctx->cdata.keylen - CTR_RFC3686_NONCE_SIZE);
207 	}
208 
209 	/*
210 	 * In case |user key| > |derived key|, using DKP<imm,imm> would result
211 	 * in invalid opcodes (last bytes of user key) in the resulting
212 	 * descriptor. Use DKP<ptr,imm> instead => both virtual and dma key
213 	 * addresses are needed.
214 	 */
215 	ctx->adata.key_virt = ctx->key;
216 	ctx->adata.key_dma = ctx->key_dma;
217 
218 	ctx->cdata.key_virt = ctx->key + ctx->adata.keylen_pad;
219 	ctx->cdata.key_dma = ctx->key_dma + ctx->adata.keylen_pad;
220 
221 	data_len[0] = ctx->adata.keylen_pad;
222 	data_len[1] = ctx->cdata.keylen;
223 
224 	/* aead_encrypt shared descriptor */
225 	if (desc_inline_query((alg->caam.geniv ? DESC_QI_AEAD_GIVENC_LEN :
226 						 DESC_QI_AEAD_ENC_LEN) +
227 			      (is_rfc3686 ? DESC_AEAD_CTR_RFC3686_LEN : 0),
228 			      DESC_JOB_IO_LEN, data_len, &inl_mask,
229 			      ARRAY_SIZE(data_len)) < 0)
230 		return -EINVAL;
231 
232 	ctx->adata.key_inline = !!(inl_mask & 1);
233 	ctx->cdata.key_inline = !!(inl_mask & 2);
234 
235 	flc = &ctx->flc[ENCRYPT];
236 	desc = flc->sh_desc;
237 
238 	if (alg->caam.geniv)
239 		cnstr_shdsc_aead_givencap(desc, &ctx->cdata, &ctx->adata,
240 					  ivsize, ctx->authsize, is_rfc3686,
241 					  nonce, ctx1_iv_off, true,
242 					  priv->sec_attr.era);
243 	else
244 		cnstr_shdsc_aead_encap(desc, &ctx->cdata, &ctx->adata,
245 				       ivsize, ctx->authsize, is_rfc3686, nonce,
246 				       ctx1_iv_off, true, priv->sec_attr.era);
247 
248 	flc->flc[1] = cpu_to_caam32(desc_len(desc)); /* SDL */
249 	dma_sync_single_for_device(dev, ctx->flc_dma[ENCRYPT],
250 				   sizeof(flc->flc) + desc_bytes(desc),
251 				   ctx->dir);
252 
253 	/* aead_decrypt shared descriptor */
254 	if (desc_inline_query(DESC_QI_AEAD_DEC_LEN +
255 			      (is_rfc3686 ? DESC_AEAD_CTR_RFC3686_LEN : 0),
256 			      DESC_JOB_IO_LEN, data_len, &inl_mask,
257 			      ARRAY_SIZE(data_len)) < 0)
258 		return -EINVAL;
259 
260 	ctx->adata.key_inline = !!(inl_mask & 1);
261 	ctx->cdata.key_inline = !!(inl_mask & 2);
262 
263 	flc = &ctx->flc[DECRYPT];
264 	desc = flc->sh_desc;
265 	cnstr_shdsc_aead_decap(desc, &ctx->cdata, &ctx->adata,
266 			       ivsize, ctx->authsize, alg->caam.geniv,
267 			       is_rfc3686, nonce, ctx1_iv_off, true,
268 			       priv->sec_attr.era);
269 	flc->flc[1] = cpu_to_caam32(desc_len(desc)); /* SDL */
270 	dma_sync_single_for_device(dev, ctx->flc_dma[DECRYPT],
271 				   sizeof(flc->flc) + desc_bytes(desc),
272 				   ctx->dir);
273 
274 	return 0;
275 }
276 
277 static int aead_setauthsize(struct crypto_aead *authenc, unsigned int authsize)
278 {
279 	struct caam_ctx *ctx = crypto_aead_ctx_dma(authenc);
280 
281 	ctx->authsize = authsize;
282 	aead_set_sh_desc(authenc);
283 
284 	return 0;
285 }
286 
287 static int aead_setkey(struct crypto_aead *aead, const u8 *key,
288 		       unsigned int keylen)
289 {
290 	struct caam_ctx *ctx = crypto_aead_ctx_dma(aead);
291 	struct device *dev = ctx->dev;
292 	struct crypto_authenc_keys keys;
293 
294 	if (crypto_authenc_extractkeys(&keys, key, keylen) != 0)
295 		goto badkey;
296 
297 	dev_dbg(dev, "keylen %d enckeylen %d authkeylen %d\n",
298 		keys.authkeylen + keys.enckeylen, keys.enckeylen,
299 		keys.authkeylen);
300 	print_hex_dump_debug("key in @" __stringify(__LINE__)": ",
301 			     DUMP_PREFIX_ADDRESS, 16, 4, key, keylen, 1);
302 
303 	ctx->adata.keylen = keys.authkeylen;
304 	ctx->adata.keylen_pad = split_key_len(ctx->adata.algtype &
305 					      OP_ALG_ALGSEL_MASK);
306 
307 	if (ctx->adata.keylen_pad + keys.enckeylen > CAAM_MAX_KEY_SIZE)
308 		goto badkey;
309 
310 	memcpy(ctx->key, keys.authkey, keys.authkeylen);
311 	memcpy(ctx->key + ctx->adata.keylen_pad, keys.enckey, keys.enckeylen);
312 	dma_sync_single_for_device(dev, ctx->key_dma, ctx->adata.keylen_pad +
313 				   keys.enckeylen, ctx->dir);
314 	print_hex_dump_debug("ctx.key@" __stringify(__LINE__)": ",
315 			     DUMP_PREFIX_ADDRESS, 16, 4, ctx->key,
316 			     ctx->adata.keylen_pad + keys.enckeylen, 1);
317 
318 	ctx->cdata.keylen = keys.enckeylen;
319 
320 	memzero_explicit(&keys, sizeof(keys));
321 	return aead_set_sh_desc(aead);
322 badkey:
323 	memzero_explicit(&keys, sizeof(keys));
324 	return -EINVAL;
325 }
326 
327 static int des3_aead_setkey(struct crypto_aead *aead, const u8 *key,
328 			    unsigned int keylen)
329 {
330 	struct crypto_authenc_keys keys;
331 	int err;
332 
333 	err = crypto_authenc_extractkeys(&keys, key, keylen);
334 	if (unlikely(err))
335 		goto out;
336 
337 	err = -EINVAL;
338 	if (keys.enckeylen != DES3_EDE_KEY_SIZE)
339 		goto out;
340 
341 	err = crypto_des3_ede_verify_key(crypto_aead_tfm(aead), keys.enckey) ?:
342 	      aead_setkey(aead, key, keylen);
343 
344 out:
345 	memzero_explicit(&keys, sizeof(keys));
346 	return err;
347 }
348 
349 static struct aead_edesc *aead_edesc_alloc(struct aead_request *req,
350 					   bool encrypt)
351 {
352 	struct crypto_aead *aead = crypto_aead_reqtfm(req);
353 	struct caam_request *req_ctx = aead_request_ctx_dma(req);
354 	struct dpaa2_fl_entry *in_fle = &req_ctx->fd_flt[1];
355 	struct dpaa2_fl_entry *out_fle = &req_ctx->fd_flt[0];
356 	struct caam_ctx *ctx = crypto_aead_ctx_dma(aead);
357 	struct caam_aead_alg *alg = container_of(crypto_aead_alg(aead),
358 						 typeof(*alg), aead);
359 	struct device *dev = ctx->dev;
360 	gfp_t flags = (req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP) ?
361 		      GFP_KERNEL : GFP_ATOMIC;
362 	int src_nents, mapped_src_nents, dst_nents = 0, mapped_dst_nents = 0;
363 	int src_len, dst_len = 0;
364 	struct aead_edesc *edesc;
365 	dma_addr_t qm_sg_dma, iv_dma = 0;
366 	int ivsize = 0;
367 	unsigned int authsize = ctx->authsize;
368 	int qm_sg_index = 0, qm_sg_nents = 0, qm_sg_bytes;
369 	int in_len, out_len;
370 	struct dpaa2_sg_entry *sg_table;
371 
372 	/* allocate space for base edesc, link tables and IV */
373 	edesc = qi_cache_zalloc(GFP_DMA | flags);
374 	if (unlikely(!edesc)) {
375 		dev_err(dev, "could not allocate extended descriptor\n");
376 		return ERR_PTR(-ENOMEM);
377 	}
378 
379 	if (unlikely(req->dst != req->src)) {
380 		src_len = req->assoclen + req->cryptlen;
381 		dst_len = src_len + (encrypt ? authsize : (-authsize));
382 
383 		src_nents = sg_nents_for_len(req->src, src_len);
384 		if (unlikely(src_nents < 0)) {
385 			dev_err(dev, "Insufficient bytes (%d) in src S/G\n",
386 				src_len);
387 			qi_cache_free(edesc);
388 			return ERR_PTR(src_nents);
389 		}
390 
391 		dst_nents = sg_nents_for_len(req->dst, dst_len);
392 		if (unlikely(dst_nents < 0)) {
393 			dev_err(dev, "Insufficient bytes (%d) in dst S/G\n",
394 				dst_len);
395 			qi_cache_free(edesc);
396 			return ERR_PTR(dst_nents);
397 		}
398 
399 		if (src_nents) {
400 			mapped_src_nents = dma_map_sg(dev, req->src, src_nents,
401 						      DMA_TO_DEVICE);
402 			if (unlikely(!mapped_src_nents)) {
403 				dev_err(dev, "unable to map source\n");
404 				qi_cache_free(edesc);
405 				return ERR_PTR(-ENOMEM);
406 			}
407 		} else {
408 			mapped_src_nents = 0;
409 		}
410 
411 		if (dst_nents) {
412 			mapped_dst_nents = dma_map_sg(dev, req->dst, dst_nents,
413 						      DMA_FROM_DEVICE);
414 			if (unlikely(!mapped_dst_nents)) {
415 				dev_err(dev, "unable to map destination\n");
416 				dma_unmap_sg(dev, req->src, src_nents,
417 					     DMA_TO_DEVICE);
418 				qi_cache_free(edesc);
419 				return ERR_PTR(-ENOMEM);
420 			}
421 		} else {
422 			mapped_dst_nents = 0;
423 		}
424 	} else {
425 		src_len = req->assoclen + req->cryptlen +
426 			  (encrypt ? authsize : 0);
427 
428 		src_nents = sg_nents_for_len(req->src, src_len);
429 		if (unlikely(src_nents < 0)) {
430 			dev_err(dev, "Insufficient bytes (%d) in src S/G\n",
431 				src_len);
432 			qi_cache_free(edesc);
433 			return ERR_PTR(src_nents);
434 		}
435 
436 		mapped_src_nents = dma_map_sg(dev, req->src, src_nents,
437 					      DMA_BIDIRECTIONAL);
438 		if (unlikely(!mapped_src_nents)) {
439 			dev_err(dev, "unable to map source\n");
440 			qi_cache_free(edesc);
441 			return ERR_PTR(-ENOMEM);
442 		}
443 	}
444 
445 	if ((alg->caam.rfc3686 && encrypt) || !alg->caam.geniv)
446 		ivsize = crypto_aead_ivsize(aead);
447 
448 	/*
449 	 * Create S/G table: req->assoclen, [IV,] req->src [, req->dst].
450 	 * Input is not contiguous.
451 	 * HW reads 4 S/G entries at a time; make sure the reads don't go beyond
452 	 * the end of the table by allocating more S/G entries. Logic:
453 	 * if (src != dst && output S/G)
454 	 *      pad output S/G, if needed
455 	 * else if (src == dst && S/G)
456 	 *      overlapping S/Gs; pad one of them
457 	 * else if (input S/G) ...
458 	 *      pad input S/G, if needed
459 	 */
460 	qm_sg_nents = 1 + !!ivsize + mapped_src_nents;
461 	if (mapped_dst_nents > 1)
462 		qm_sg_nents += pad_sg_nents(mapped_dst_nents);
463 	else if ((req->src == req->dst) && (mapped_src_nents > 1))
464 		qm_sg_nents = max(pad_sg_nents(qm_sg_nents),
465 				  1 + !!ivsize +
466 				  pad_sg_nents(mapped_src_nents));
467 	else
468 		qm_sg_nents = pad_sg_nents(qm_sg_nents);
469 
470 	sg_table = &edesc->sgt[0];
471 	qm_sg_bytes = qm_sg_nents * sizeof(*sg_table);
472 	if (unlikely(offsetof(struct aead_edesc, sgt) + qm_sg_bytes + ivsize >
473 		     CAAM_QI_MEMCACHE_SIZE)) {
474 		dev_err(dev, "No space for %d S/G entries and/or %dB IV\n",
475 			qm_sg_nents, ivsize);
476 		caam_unmap(dev, req->src, req->dst, src_nents, dst_nents, 0,
477 			   0, DMA_NONE, 0, 0);
478 		qi_cache_free(edesc);
479 		return ERR_PTR(-ENOMEM);
480 	}
481 
482 	if (ivsize) {
483 		u8 *iv = (u8 *)(sg_table + qm_sg_nents);
484 
485 		/* Make sure IV is located in a DMAable area */
486 		memcpy(iv, req->iv, ivsize);
487 
488 		iv_dma = dma_map_single(dev, iv, ivsize, DMA_TO_DEVICE);
489 		if (dma_mapping_error(dev, iv_dma)) {
490 			dev_err(dev, "unable to map IV\n");
491 			caam_unmap(dev, req->src, req->dst, src_nents,
492 				   dst_nents, 0, 0, DMA_NONE, 0, 0);
493 			qi_cache_free(edesc);
494 			return ERR_PTR(-ENOMEM);
495 		}
496 	}
497 
498 	edesc->src_nents = src_nents;
499 	edesc->dst_nents = dst_nents;
500 	edesc->iv_dma = iv_dma;
501 
502 	if ((alg->caam.class1_alg_type & OP_ALG_ALGSEL_MASK) ==
503 	    OP_ALG_ALGSEL_CHACHA20 && ivsize != CHACHAPOLY_IV_SIZE)
504 		/*
505 		 * The associated data comes already with the IV but we need
506 		 * to skip it when we authenticate or encrypt...
507 		 */
508 		edesc->assoclen = cpu_to_caam32(req->assoclen - ivsize);
509 	else
510 		edesc->assoclen = cpu_to_caam32(req->assoclen);
511 	edesc->assoclen_dma = dma_map_single(dev, &edesc->assoclen, 4,
512 					     DMA_TO_DEVICE);
513 	if (dma_mapping_error(dev, edesc->assoclen_dma)) {
514 		dev_err(dev, "unable to map assoclen\n");
515 		caam_unmap(dev, req->src, req->dst, src_nents, dst_nents,
516 			   iv_dma, ivsize, DMA_TO_DEVICE, 0, 0);
517 		qi_cache_free(edesc);
518 		return ERR_PTR(-ENOMEM);
519 	}
520 
521 	dma_to_qm_sg_one(sg_table, edesc->assoclen_dma, 4, 0);
522 	qm_sg_index++;
523 	if (ivsize) {
524 		dma_to_qm_sg_one(sg_table + qm_sg_index, iv_dma, ivsize, 0);
525 		qm_sg_index++;
526 	}
527 	sg_to_qm_sg_last(req->src, src_len, sg_table + qm_sg_index, 0);
528 	qm_sg_index += mapped_src_nents;
529 
530 	if (mapped_dst_nents > 1)
531 		sg_to_qm_sg_last(req->dst, dst_len, sg_table + qm_sg_index, 0);
532 
533 	qm_sg_dma = dma_map_single(dev, sg_table, qm_sg_bytes, DMA_TO_DEVICE);
534 	if (dma_mapping_error(dev, qm_sg_dma)) {
535 		dev_err(dev, "unable to map S/G table\n");
536 		dma_unmap_single(dev, edesc->assoclen_dma, 4, DMA_TO_DEVICE);
537 		caam_unmap(dev, req->src, req->dst, src_nents, dst_nents,
538 			   iv_dma, ivsize, DMA_TO_DEVICE, 0, 0);
539 		qi_cache_free(edesc);
540 		return ERR_PTR(-ENOMEM);
541 	}
542 
543 	edesc->qm_sg_dma = qm_sg_dma;
544 	edesc->qm_sg_bytes = qm_sg_bytes;
545 
546 	out_len = req->assoclen + req->cryptlen +
547 		  (encrypt ? ctx->authsize : (-ctx->authsize));
548 	in_len = 4 + ivsize + req->assoclen + req->cryptlen;
549 
550 	memset(&req_ctx->fd_flt, 0, sizeof(req_ctx->fd_flt));
551 	dpaa2_fl_set_final(in_fle, true);
552 	dpaa2_fl_set_format(in_fle, dpaa2_fl_sg);
553 	dpaa2_fl_set_addr(in_fle, qm_sg_dma);
554 	dpaa2_fl_set_len(in_fle, in_len);
555 
556 	if (req->dst == req->src) {
557 		if (mapped_src_nents == 1) {
558 			dpaa2_fl_set_format(out_fle, dpaa2_fl_single);
559 			dpaa2_fl_set_addr(out_fle, sg_dma_address(req->src));
560 		} else {
561 			dpaa2_fl_set_format(out_fle, dpaa2_fl_sg);
562 			dpaa2_fl_set_addr(out_fle, qm_sg_dma +
563 					  (1 + !!ivsize) * sizeof(*sg_table));
564 		}
565 	} else if (!mapped_dst_nents) {
566 		/*
567 		 * crypto engine requires the output entry to be present when
568 		 * "frame list" FD is used.
569 		 * Since engine does not support FMT=2'b11 (unused entry type),
570 		 * leaving out_fle zeroized is the best option.
571 		 */
572 		goto skip_out_fle;
573 	} else if (mapped_dst_nents == 1) {
574 		dpaa2_fl_set_format(out_fle, dpaa2_fl_single);
575 		dpaa2_fl_set_addr(out_fle, sg_dma_address(req->dst));
576 	} else {
577 		dpaa2_fl_set_format(out_fle, dpaa2_fl_sg);
578 		dpaa2_fl_set_addr(out_fle, qm_sg_dma + qm_sg_index *
579 				  sizeof(*sg_table));
580 	}
581 
582 	dpaa2_fl_set_len(out_fle, out_len);
583 
584 skip_out_fle:
585 	return edesc;
586 }
587 
588 static int chachapoly_set_sh_desc(struct crypto_aead *aead)
589 {
590 	struct caam_ctx *ctx = crypto_aead_ctx_dma(aead);
591 	unsigned int ivsize = crypto_aead_ivsize(aead);
592 	struct device *dev = ctx->dev;
593 	struct caam_flc *flc;
594 	u32 *desc;
595 
596 	if (!ctx->cdata.keylen || !ctx->authsize)
597 		return 0;
598 
599 	flc = &ctx->flc[ENCRYPT];
600 	desc = flc->sh_desc;
601 	cnstr_shdsc_chachapoly(desc, &ctx->cdata, &ctx->adata, ivsize,
602 			       ctx->authsize, true, true);
603 	flc->flc[1] = cpu_to_caam32(desc_len(desc)); /* SDL */
604 	dma_sync_single_for_device(dev, ctx->flc_dma[ENCRYPT],
605 				   sizeof(flc->flc) + desc_bytes(desc),
606 				   ctx->dir);
607 
608 	flc = &ctx->flc[DECRYPT];
609 	desc = flc->sh_desc;
610 	cnstr_shdsc_chachapoly(desc, &ctx->cdata, &ctx->adata, ivsize,
611 			       ctx->authsize, false, true);
612 	flc->flc[1] = cpu_to_caam32(desc_len(desc)); /* SDL */
613 	dma_sync_single_for_device(dev, ctx->flc_dma[DECRYPT],
614 				   sizeof(flc->flc) + desc_bytes(desc),
615 				   ctx->dir);
616 
617 	return 0;
618 }
619 
620 static int chachapoly_setauthsize(struct crypto_aead *aead,
621 				  unsigned int authsize)
622 {
623 	struct caam_ctx *ctx = crypto_aead_ctx_dma(aead);
624 
625 	if (authsize != POLY1305_DIGEST_SIZE)
626 		return -EINVAL;
627 
628 	ctx->authsize = authsize;
629 	return chachapoly_set_sh_desc(aead);
630 }
631 
632 static int chachapoly_setkey(struct crypto_aead *aead, const u8 *key,
633 			     unsigned int keylen)
634 {
635 	struct caam_ctx *ctx = crypto_aead_ctx_dma(aead);
636 	unsigned int ivsize = crypto_aead_ivsize(aead);
637 	unsigned int saltlen = CHACHAPOLY_IV_SIZE - ivsize;
638 
639 	if (keylen != CHACHA_KEY_SIZE + saltlen)
640 		return -EINVAL;
641 
642 	ctx->cdata.key_virt = key;
643 	ctx->cdata.keylen = keylen - saltlen;
644 
645 	return chachapoly_set_sh_desc(aead);
646 }
647 
648 static int gcm_set_sh_desc(struct crypto_aead *aead)
649 {
650 	struct caam_ctx *ctx = crypto_aead_ctx_dma(aead);
651 	struct device *dev = ctx->dev;
652 	unsigned int ivsize = crypto_aead_ivsize(aead);
653 	struct caam_flc *flc;
654 	u32 *desc;
655 	int rem_bytes = CAAM_DESC_BYTES_MAX - DESC_JOB_IO_LEN -
656 			ctx->cdata.keylen;
657 
658 	if (!ctx->cdata.keylen || !ctx->authsize)
659 		return 0;
660 
661 	/*
662 	 * AES GCM encrypt shared descriptor
663 	 * Job Descriptor and Shared Descriptor
664 	 * must fit into the 64-word Descriptor h/w Buffer
665 	 */
666 	if (rem_bytes >= DESC_QI_GCM_ENC_LEN) {
667 		ctx->cdata.key_inline = true;
668 		ctx->cdata.key_virt = ctx->key;
669 	} else {
670 		ctx->cdata.key_inline = false;
671 		ctx->cdata.key_dma = ctx->key_dma;
672 	}
673 
674 	flc = &ctx->flc[ENCRYPT];
675 	desc = flc->sh_desc;
676 	cnstr_shdsc_gcm_encap(desc, &ctx->cdata, ivsize, ctx->authsize, true);
677 	flc->flc[1] = cpu_to_caam32(desc_len(desc)); /* SDL */
678 	dma_sync_single_for_device(dev, ctx->flc_dma[ENCRYPT],
679 				   sizeof(flc->flc) + desc_bytes(desc),
680 				   ctx->dir);
681 
682 	/*
683 	 * Job Descriptor and Shared Descriptors
684 	 * must all fit into the 64-word Descriptor h/w Buffer
685 	 */
686 	if (rem_bytes >= DESC_QI_GCM_DEC_LEN) {
687 		ctx->cdata.key_inline = true;
688 		ctx->cdata.key_virt = ctx->key;
689 	} else {
690 		ctx->cdata.key_inline = false;
691 		ctx->cdata.key_dma = ctx->key_dma;
692 	}
693 
694 	flc = &ctx->flc[DECRYPT];
695 	desc = flc->sh_desc;
696 	cnstr_shdsc_gcm_decap(desc, &ctx->cdata, ivsize, ctx->authsize, true);
697 	flc->flc[1] = cpu_to_caam32(desc_len(desc)); /* SDL */
698 	dma_sync_single_for_device(dev, ctx->flc_dma[DECRYPT],
699 				   sizeof(flc->flc) + desc_bytes(desc),
700 				   ctx->dir);
701 
702 	return 0;
703 }
704 
705 static int gcm_setauthsize(struct crypto_aead *authenc, unsigned int authsize)
706 {
707 	struct caam_ctx *ctx = crypto_aead_ctx_dma(authenc);
708 	int err;
709 
710 	err = crypto_gcm_check_authsize(authsize);
711 	if (err)
712 		return err;
713 
714 	ctx->authsize = authsize;
715 	gcm_set_sh_desc(authenc);
716 
717 	return 0;
718 }
719 
720 static int gcm_setkey(struct crypto_aead *aead,
721 		      const u8 *key, unsigned int keylen)
722 {
723 	struct caam_ctx *ctx = crypto_aead_ctx_dma(aead);
724 	struct device *dev = ctx->dev;
725 	int ret;
726 
727 	ret = aes_check_keylen(keylen);
728 	if (ret)
729 		return ret;
730 	print_hex_dump_debug("key in @" __stringify(__LINE__)": ",
731 			     DUMP_PREFIX_ADDRESS, 16, 4, key, keylen, 1);
732 
733 	memcpy(ctx->key, key, keylen);
734 	dma_sync_single_for_device(dev, ctx->key_dma, keylen, ctx->dir);
735 	ctx->cdata.keylen = keylen;
736 
737 	return gcm_set_sh_desc(aead);
738 }
739 
740 static int rfc4106_set_sh_desc(struct crypto_aead *aead)
741 {
742 	struct caam_ctx *ctx = crypto_aead_ctx_dma(aead);
743 	struct device *dev = ctx->dev;
744 	unsigned int ivsize = crypto_aead_ivsize(aead);
745 	struct caam_flc *flc;
746 	u32 *desc;
747 	int rem_bytes = CAAM_DESC_BYTES_MAX - DESC_JOB_IO_LEN -
748 			ctx->cdata.keylen;
749 
750 	if (!ctx->cdata.keylen || !ctx->authsize)
751 		return 0;
752 
753 	ctx->cdata.key_virt = ctx->key;
754 
755 	/*
756 	 * RFC4106 encrypt shared descriptor
757 	 * Job Descriptor and Shared Descriptor
758 	 * must fit into the 64-word Descriptor h/w Buffer
759 	 */
760 	if (rem_bytes >= DESC_QI_RFC4106_ENC_LEN) {
761 		ctx->cdata.key_inline = true;
762 	} else {
763 		ctx->cdata.key_inline = false;
764 		ctx->cdata.key_dma = ctx->key_dma;
765 	}
766 
767 	flc = &ctx->flc[ENCRYPT];
768 	desc = flc->sh_desc;
769 	cnstr_shdsc_rfc4106_encap(desc, &ctx->cdata, ivsize, ctx->authsize,
770 				  true);
771 	flc->flc[1] = cpu_to_caam32(desc_len(desc)); /* SDL */
772 	dma_sync_single_for_device(dev, ctx->flc_dma[ENCRYPT],
773 				   sizeof(flc->flc) + desc_bytes(desc),
774 				   ctx->dir);
775 
776 	/*
777 	 * Job Descriptor and Shared Descriptors
778 	 * must all fit into the 64-word Descriptor h/w Buffer
779 	 */
780 	if (rem_bytes >= DESC_QI_RFC4106_DEC_LEN) {
781 		ctx->cdata.key_inline = true;
782 	} else {
783 		ctx->cdata.key_inline = false;
784 		ctx->cdata.key_dma = ctx->key_dma;
785 	}
786 
787 	flc = &ctx->flc[DECRYPT];
788 	desc = flc->sh_desc;
789 	cnstr_shdsc_rfc4106_decap(desc, &ctx->cdata, ivsize, ctx->authsize,
790 				  true);
791 	flc->flc[1] = cpu_to_caam32(desc_len(desc)); /* SDL */
792 	dma_sync_single_for_device(dev, ctx->flc_dma[DECRYPT],
793 				   sizeof(flc->flc) + desc_bytes(desc),
794 				   ctx->dir);
795 
796 	return 0;
797 }
798 
799 static int rfc4106_setauthsize(struct crypto_aead *authenc,
800 			       unsigned int authsize)
801 {
802 	struct caam_ctx *ctx = crypto_aead_ctx_dma(authenc);
803 	int err;
804 
805 	err = crypto_rfc4106_check_authsize(authsize);
806 	if (err)
807 		return err;
808 
809 	ctx->authsize = authsize;
810 	rfc4106_set_sh_desc(authenc);
811 
812 	return 0;
813 }
814 
815 static int rfc4106_setkey(struct crypto_aead *aead,
816 			  const u8 *key, unsigned int keylen)
817 {
818 	struct caam_ctx *ctx = crypto_aead_ctx_dma(aead);
819 	struct device *dev = ctx->dev;
820 	int ret;
821 
822 	ret = aes_check_keylen(keylen - 4);
823 	if (ret)
824 		return ret;
825 
826 	print_hex_dump_debug("key in @" __stringify(__LINE__)": ",
827 			     DUMP_PREFIX_ADDRESS, 16, 4, key, keylen, 1);
828 
829 	memcpy(ctx->key, key, keylen);
830 	/*
831 	 * The last four bytes of the key material are used as the salt value
832 	 * in the nonce. Update the AES key length.
833 	 */
834 	ctx->cdata.keylen = keylen - 4;
835 	dma_sync_single_for_device(dev, ctx->key_dma, ctx->cdata.keylen,
836 				   ctx->dir);
837 
838 	return rfc4106_set_sh_desc(aead);
839 }
840 
841 static int rfc4543_set_sh_desc(struct crypto_aead *aead)
842 {
843 	struct caam_ctx *ctx = crypto_aead_ctx_dma(aead);
844 	struct device *dev = ctx->dev;
845 	unsigned int ivsize = crypto_aead_ivsize(aead);
846 	struct caam_flc *flc;
847 	u32 *desc;
848 	int rem_bytes = CAAM_DESC_BYTES_MAX - DESC_JOB_IO_LEN -
849 			ctx->cdata.keylen;
850 
851 	if (!ctx->cdata.keylen || !ctx->authsize)
852 		return 0;
853 
854 	ctx->cdata.key_virt = ctx->key;
855 
856 	/*
857 	 * RFC4543 encrypt shared descriptor
858 	 * Job Descriptor and Shared Descriptor
859 	 * must fit into the 64-word Descriptor h/w Buffer
860 	 */
861 	if (rem_bytes >= DESC_QI_RFC4543_ENC_LEN) {
862 		ctx->cdata.key_inline = true;
863 	} else {
864 		ctx->cdata.key_inline = false;
865 		ctx->cdata.key_dma = ctx->key_dma;
866 	}
867 
868 	flc = &ctx->flc[ENCRYPT];
869 	desc = flc->sh_desc;
870 	cnstr_shdsc_rfc4543_encap(desc, &ctx->cdata, ivsize, ctx->authsize,
871 				  true);
872 	flc->flc[1] = cpu_to_caam32(desc_len(desc)); /* SDL */
873 	dma_sync_single_for_device(dev, ctx->flc_dma[ENCRYPT],
874 				   sizeof(flc->flc) + desc_bytes(desc),
875 				   ctx->dir);
876 
877 	/*
878 	 * Job Descriptor and Shared Descriptors
879 	 * must all fit into the 64-word Descriptor h/w Buffer
880 	 */
881 	if (rem_bytes >= DESC_QI_RFC4543_DEC_LEN) {
882 		ctx->cdata.key_inline = true;
883 	} else {
884 		ctx->cdata.key_inline = false;
885 		ctx->cdata.key_dma = ctx->key_dma;
886 	}
887 
888 	flc = &ctx->flc[DECRYPT];
889 	desc = flc->sh_desc;
890 	cnstr_shdsc_rfc4543_decap(desc, &ctx->cdata, ivsize, ctx->authsize,
891 				  true);
892 	flc->flc[1] = cpu_to_caam32(desc_len(desc)); /* SDL */
893 	dma_sync_single_for_device(dev, ctx->flc_dma[DECRYPT],
894 				   sizeof(flc->flc) + desc_bytes(desc),
895 				   ctx->dir);
896 
897 	return 0;
898 }
899 
900 static int rfc4543_setauthsize(struct crypto_aead *authenc,
901 			       unsigned int authsize)
902 {
903 	struct caam_ctx *ctx = crypto_aead_ctx_dma(authenc);
904 
905 	if (authsize != 16)
906 		return -EINVAL;
907 
908 	ctx->authsize = authsize;
909 	rfc4543_set_sh_desc(authenc);
910 
911 	return 0;
912 }
913 
914 static int rfc4543_setkey(struct crypto_aead *aead,
915 			  const u8 *key, unsigned int keylen)
916 {
917 	struct caam_ctx *ctx = crypto_aead_ctx_dma(aead);
918 	struct device *dev = ctx->dev;
919 	int ret;
920 
921 	ret = aes_check_keylen(keylen - 4);
922 	if (ret)
923 		return ret;
924 
925 	print_hex_dump_debug("key in @" __stringify(__LINE__)": ",
926 			     DUMP_PREFIX_ADDRESS, 16, 4, key, keylen, 1);
927 
928 	memcpy(ctx->key, key, keylen);
929 	/*
930 	 * The last four bytes of the key material are used as the salt value
931 	 * in the nonce. Update the AES key length.
932 	 */
933 	ctx->cdata.keylen = keylen - 4;
934 	dma_sync_single_for_device(dev, ctx->key_dma, ctx->cdata.keylen,
935 				   ctx->dir);
936 
937 	return rfc4543_set_sh_desc(aead);
938 }
939 
940 static int skcipher_setkey(struct crypto_skcipher *skcipher, const u8 *key,
941 			   unsigned int keylen, const u32 ctx1_iv_off)
942 {
943 	struct caam_ctx *ctx = crypto_skcipher_ctx_dma(skcipher);
944 	struct caam_skcipher_alg *alg =
945 		container_of(crypto_skcipher_alg(skcipher),
946 			     struct caam_skcipher_alg, skcipher);
947 	struct device *dev = ctx->dev;
948 	struct caam_flc *flc;
949 	unsigned int ivsize = crypto_skcipher_ivsize(skcipher);
950 	u32 *desc;
951 	const bool is_rfc3686 = alg->caam.rfc3686;
952 
953 	print_hex_dump_debug("key in @" __stringify(__LINE__)": ",
954 			     DUMP_PREFIX_ADDRESS, 16, 4, key, keylen, 1);
955 
956 	ctx->cdata.keylen = keylen;
957 	ctx->cdata.key_virt = key;
958 	ctx->cdata.key_inline = true;
959 
960 	/* skcipher_encrypt shared descriptor */
961 	flc = &ctx->flc[ENCRYPT];
962 	desc = flc->sh_desc;
963 	cnstr_shdsc_skcipher_encap(desc, &ctx->cdata, ivsize, is_rfc3686,
964 				   ctx1_iv_off);
965 	flc->flc[1] = cpu_to_caam32(desc_len(desc)); /* SDL */
966 	dma_sync_single_for_device(dev, ctx->flc_dma[ENCRYPT],
967 				   sizeof(flc->flc) + desc_bytes(desc),
968 				   ctx->dir);
969 
970 	/* skcipher_decrypt shared descriptor */
971 	flc = &ctx->flc[DECRYPT];
972 	desc = flc->sh_desc;
973 	cnstr_shdsc_skcipher_decap(desc, &ctx->cdata, ivsize, is_rfc3686,
974 				   ctx1_iv_off);
975 	flc->flc[1] = cpu_to_caam32(desc_len(desc)); /* SDL */
976 	dma_sync_single_for_device(dev, ctx->flc_dma[DECRYPT],
977 				   sizeof(flc->flc) + desc_bytes(desc),
978 				   ctx->dir);
979 
980 	return 0;
981 }
982 
983 static int aes_skcipher_setkey(struct crypto_skcipher *skcipher,
984 			       const u8 *key, unsigned int keylen)
985 {
986 	int err;
987 
988 	err = aes_check_keylen(keylen);
989 	if (err)
990 		return err;
991 
992 	return skcipher_setkey(skcipher, key, keylen, 0);
993 }
994 
995 static int rfc3686_skcipher_setkey(struct crypto_skcipher *skcipher,
996 				   const u8 *key, unsigned int keylen)
997 {
998 	u32 ctx1_iv_off;
999 	int err;
1000 
1001 	/*
1002 	 * RFC3686 specific:
1003 	 *	| CONTEXT1[255:128] = {NONCE, IV, COUNTER}
1004 	 *	| *key = {KEY, NONCE}
1005 	 */
1006 	ctx1_iv_off = 16 + CTR_RFC3686_NONCE_SIZE;
1007 	keylen -= CTR_RFC3686_NONCE_SIZE;
1008 
1009 	err = aes_check_keylen(keylen);
1010 	if (err)
1011 		return err;
1012 
1013 	return skcipher_setkey(skcipher, key, keylen, ctx1_iv_off);
1014 }
1015 
1016 static int ctr_skcipher_setkey(struct crypto_skcipher *skcipher,
1017 			       const u8 *key, unsigned int keylen)
1018 {
1019 	u32 ctx1_iv_off;
1020 	int err;
1021 
1022 	/*
1023 	 * AES-CTR needs to load IV in CONTEXT1 reg
1024 	 * at an offset of 128bits (16bytes)
1025 	 * CONTEXT1[255:128] = IV
1026 	 */
1027 	ctx1_iv_off = 16;
1028 
1029 	err = aes_check_keylen(keylen);
1030 	if (err)
1031 		return err;
1032 
1033 	return skcipher_setkey(skcipher, key, keylen, ctx1_iv_off);
1034 }
1035 
1036 static int chacha20_skcipher_setkey(struct crypto_skcipher *skcipher,
1037 				    const u8 *key, unsigned int keylen)
1038 {
1039 	if (keylen != CHACHA_KEY_SIZE)
1040 		return -EINVAL;
1041 
1042 	return skcipher_setkey(skcipher, key, keylen, 0);
1043 }
1044 
1045 static int des_skcipher_setkey(struct crypto_skcipher *skcipher,
1046 			       const u8 *key, unsigned int keylen)
1047 {
1048 	return verify_skcipher_des_key(skcipher, key) ?:
1049 	       skcipher_setkey(skcipher, key, keylen, 0);
1050 }
1051 
1052 static int des3_skcipher_setkey(struct crypto_skcipher *skcipher,
1053 			        const u8 *key, unsigned int keylen)
1054 {
1055 	return verify_skcipher_des3_key(skcipher, key) ?:
1056 	       skcipher_setkey(skcipher, key, keylen, 0);
1057 }
1058 
1059 static int xts_skcipher_setkey(struct crypto_skcipher *skcipher, const u8 *key,
1060 			       unsigned int keylen)
1061 {
1062 	struct caam_ctx *ctx = crypto_skcipher_ctx_dma(skcipher);
1063 	struct device *dev = ctx->dev;
1064 	struct dpaa2_caam_priv *priv = dev_get_drvdata(dev);
1065 	struct caam_flc *flc;
1066 	u32 *desc;
1067 	int err;
1068 
1069 	err = xts_verify_key(skcipher, key, keylen);
1070 	if (err) {
1071 		dev_dbg(dev, "key size mismatch\n");
1072 		return err;
1073 	}
1074 
1075 	if (keylen != 2 * AES_KEYSIZE_128 && keylen != 2 * AES_KEYSIZE_256)
1076 		ctx->xts_key_fallback = true;
1077 
1078 	if (priv->sec_attr.era <= 8 || ctx->xts_key_fallback) {
1079 		err = crypto_skcipher_setkey(ctx->fallback, key, keylen);
1080 		if (err)
1081 			return err;
1082 	}
1083 
1084 	ctx->cdata.keylen = keylen;
1085 	ctx->cdata.key_virt = key;
1086 	ctx->cdata.key_inline = true;
1087 
1088 	/* xts_skcipher_encrypt shared descriptor */
1089 	flc = &ctx->flc[ENCRYPT];
1090 	desc = flc->sh_desc;
1091 	cnstr_shdsc_xts_skcipher_encap(desc, &ctx->cdata);
1092 	flc->flc[1] = cpu_to_caam32(desc_len(desc)); /* SDL */
1093 	dma_sync_single_for_device(dev, ctx->flc_dma[ENCRYPT],
1094 				   sizeof(flc->flc) + desc_bytes(desc),
1095 				   ctx->dir);
1096 
1097 	/* xts_skcipher_decrypt shared descriptor */
1098 	flc = &ctx->flc[DECRYPT];
1099 	desc = flc->sh_desc;
1100 	cnstr_shdsc_xts_skcipher_decap(desc, &ctx->cdata);
1101 	flc->flc[1] = cpu_to_caam32(desc_len(desc)); /* SDL */
1102 	dma_sync_single_for_device(dev, ctx->flc_dma[DECRYPT],
1103 				   sizeof(flc->flc) + desc_bytes(desc),
1104 				   ctx->dir);
1105 
1106 	return 0;
1107 }
1108 
1109 static struct skcipher_edesc *skcipher_edesc_alloc(struct skcipher_request *req)
1110 {
1111 	struct crypto_skcipher *skcipher = crypto_skcipher_reqtfm(req);
1112 	struct caam_request *req_ctx = skcipher_request_ctx_dma(req);
1113 	struct dpaa2_fl_entry *in_fle = &req_ctx->fd_flt[1];
1114 	struct dpaa2_fl_entry *out_fle = &req_ctx->fd_flt[0];
1115 	struct caam_ctx *ctx = crypto_skcipher_ctx_dma(skcipher);
1116 	struct device *dev = ctx->dev;
1117 	gfp_t flags = (req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP) ?
1118 		       GFP_KERNEL : GFP_ATOMIC;
1119 	int src_nents, mapped_src_nents, dst_nents = 0, mapped_dst_nents = 0;
1120 	struct skcipher_edesc *edesc;
1121 	dma_addr_t iv_dma;
1122 	u8 *iv;
1123 	int ivsize = crypto_skcipher_ivsize(skcipher);
1124 	int dst_sg_idx, qm_sg_ents, qm_sg_bytes;
1125 	struct dpaa2_sg_entry *sg_table;
1126 
1127 	src_nents = sg_nents_for_len(req->src, req->cryptlen);
1128 	if (unlikely(src_nents < 0)) {
1129 		dev_err(dev, "Insufficient bytes (%d) in src S/G\n",
1130 			req->cryptlen);
1131 		return ERR_PTR(src_nents);
1132 	}
1133 
1134 	if (unlikely(req->dst != req->src)) {
1135 		dst_nents = sg_nents_for_len(req->dst, req->cryptlen);
1136 		if (unlikely(dst_nents < 0)) {
1137 			dev_err(dev, "Insufficient bytes (%d) in dst S/G\n",
1138 				req->cryptlen);
1139 			return ERR_PTR(dst_nents);
1140 		}
1141 
1142 		mapped_src_nents = dma_map_sg(dev, req->src, src_nents,
1143 					      DMA_TO_DEVICE);
1144 		if (unlikely(!mapped_src_nents)) {
1145 			dev_err(dev, "unable to map source\n");
1146 			return ERR_PTR(-ENOMEM);
1147 		}
1148 
1149 		mapped_dst_nents = dma_map_sg(dev, req->dst, dst_nents,
1150 					      DMA_FROM_DEVICE);
1151 		if (unlikely(!mapped_dst_nents)) {
1152 			dev_err(dev, "unable to map destination\n");
1153 			dma_unmap_sg(dev, req->src, src_nents, DMA_TO_DEVICE);
1154 			return ERR_PTR(-ENOMEM);
1155 		}
1156 	} else {
1157 		mapped_src_nents = dma_map_sg(dev, req->src, src_nents,
1158 					      DMA_BIDIRECTIONAL);
1159 		if (unlikely(!mapped_src_nents)) {
1160 			dev_err(dev, "unable to map source\n");
1161 			return ERR_PTR(-ENOMEM);
1162 		}
1163 	}
1164 
1165 	qm_sg_ents = 1 + mapped_src_nents;
1166 	dst_sg_idx = qm_sg_ents;
1167 
1168 	/*
1169 	 * Input, output HW S/G tables: [IV, src][dst, IV]
1170 	 * IV entries point to the same buffer
1171 	 * If src == dst, S/G entries are reused (S/G tables overlap)
1172 	 *
1173 	 * HW reads 4 S/G entries at a time; make sure the reads don't go beyond
1174 	 * the end of the table by allocating more S/G entries.
1175 	 */
1176 	if (req->src != req->dst)
1177 		qm_sg_ents += pad_sg_nents(mapped_dst_nents + 1);
1178 	else
1179 		qm_sg_ents = 1 + pad_sg_nents(qm_sg_ents);
1180 
1181 	qm_sg_bytes = qm_sg_ents * sizeof(struct dpaa2_sg_entry);
1182 	if (unlikely(offsetof(struct skcipher_edesc, sgt) + qm_sg_bytes +
1183 		     ivsize > CAAM_QI_MEMCACHE_SIZE)) {
1184 		dev_err(dev, "No space for %d S/G entries and/or %dB IV\n",
1185 			qm_sg_ents, ivsize);
1186 		caam_unmap(dev, req->src, req->dst, src_nents, dst_nents, 0,
1187 			   0, DMA_NONE, 0, 0);
1188 		return ERR_PTR(-ENOMEM);
1189 	}
1190 
1191 	/* allocate space for base edesc, link tables and IV */
1192 	edesc = qi_cache_zalloc(GFP_DMA | flags);
1193 	if (unlikely(!edesc)) {
1194 		dev_err(dev, "could not allocate extended descriptor\n");
1195 		caam_unmap(dev, req->src, req->dst, src_nents, dst_nents, 0,
1196 			   0, DMA_NONE, 0, 0);
1197 		return ERR_PTR(-ENOMEM);
1198 	}
1199 
1200 	/* Make sure IV is located in a DMAable area */
1201 	sg_table = &edesc->sgt[0];
1202 	iv = (u8 *)(sg_table + qm_sg_ents);
1203 	memcpy(iv, req->iv, ivsize);
1204 
1205 	iv_dma = dma_map_single(dev, iv, ivsize, DMA_BIDIRECTIONAL);
1206 	if (dma_mapping_error(dev, iv_dma)) {
1207 		dev_err(dev, "unable to map IV\n");
1208 		caam_unmap(dev, req->src, req->dst, src_nents, dst_nents, 0,
1209 			   0, DMA_NONE, 0, 0);
1210 		qi_cache_free(edesc);
1211 		return ERR_PTR(-ENOMEM);
1212 	}
1213 
1214 	edesc->src_nents = src_nents;
1215 	edesc->dst_nents = dst_nents;
1216 	edesc->iv_dma = iv_dma;
1217 	edesc->qm_sg_bytes = qm_sg_bytes;
1218 
1219 	dma_to_qm_sg_one(sg_table, iv_dma, ivsize, 0);
1220 	sg_to_qm_sg(req->src, req->cryptlen, sg_table + 1, 0);
1221 
1222 	if (req->src != req->dst)
1223 		sg_to_qm_sg(req->dst, req->cryptlen, sg_table + dst_sg_idx, 0);
1224 
1225 	dma_to_qm_sg_one(sg_table + dst_sg_idx + mapped_dst_nents, iv_dma,
1226 			 ivsize, 0);
1227 
1228 	edesc->qm_sg_dma = dma_map_single(dev, sg_table, edesc->qm_sg_bytes,
1229 					  DMA_TO_DEVICE);
1230 	if (dma_mapping_error(dev, edesc->qm_sg_dma)) {
1231 		dev_err(dev, "unable to map S/G table\n");
1232 		caam_unmap(dev, req->src, req->dst, src_nents, dst_nents,
1233 			   iv_dma, ivsize, DMA_BIDIRECTIONAL, 0, 0);
1234 		qi_cache_free(edesc);
1235 		return ERR_PTR(-ENOMEM);
1236 	}
1237 
1238 	memset(&req_ctx->fd_flt, 0, sizeof(req_ctx->fd_flt));
1239 	dpaa2_fl_set_final(in_fle, true);
1240 	dpaa2_fl_set_len(in_fle, req->cryptlen + ivsize);
1241 	dpaa2_fl_set_len(out_fle, req->cryptlen + ivsize);
1242 
1243 	dpaa2_fl_set_format(in_fle, dpaa2_fl_sg);
1244 	dpaa2_fl_set_addr(in_fle, edesc->qm_sg_dma);
1245 
1246 	dpaa2_fl_set_format(out_fle, dpaa2_fl_sg);
1247 
1248 	if (req->src == req->dst)
1249 		dpaa2_fl_set_addr(out_fle, edesc->qm_sg_dma +
1250 				  sizeof(*sg_table));
1251 	else
1252 		dpaa2_fl_set_addr(out_fle, edesc->qm_sg_dma + dst_sg_idx *
1253 				  sizeof(*sg_table));
1254 
1255 	return edesc;
1256 }
1257 
1258 static void aead_unmap(struct device *dev, struct aead_edesc *edesc,
1259 		       struct aead_request *req)
1260 {
1261 	struct crypto_aead *aead = crypto_aead_reqtfm(req);
1262 	int ivsize = crypto_aead_ivsize(aead);
1263 
1264 	caam_unmap(dev, req->src, req->dst, edesc->src_nents, edesc->dst_nents,
1265 		   edesc->iv_dma, ivsize, DMA_TO_DEVICE, edesc->qm_sg_dma,
1266 		   edesc->qm_sg_bytes);
1267 	dma_unmap_single(dev, edesc->assoclen_dma, 4, DMA_TO_DEVICE);
1268 }
1269 
1270 static void skcipher_unmap(struct device *dev, struct skcipher_edesc *edesc,
1271 			   struct skcipher_request *req)
1272 {
1273 	struct crypto_skcipher *skcipher = crypto_skcipher_reqtfm(req);
1274 	int ivsize = crypto_skcipher_ivsize(skcipher);
1275 
1276 	caam_unmap(dev, req->src, req->dst, edesc->src_nents, edesc->dst_nents,
1277 		   edesc->iv_dma, ivsize, DMA_BIDIRECTIONAL, edesc->qm_sg_dma,
1278 		   edesc->qm_sg_bytes);
1279 }
1280 
1281 static void aead_encrypt_done(void *cbk_ctx, u32 status)
1282 {
1283 	struct crypto_async_request *areq = cbk_ctx;
1284 	struct aead_request *req = container_of(areq, struct aead_request,
1285 						base);
1286 	struct caam_request *req_ctx = to_caam_req(areq);
1287 	struct aead_edesc *edesc = req_ctx->edesc;
1288 	struct crypto_aead *aead = crypto_aead_reqtfm(req);
1289 	struct caam_ctx *ctx = crypto_aead_ctx_dma(aead);
1290 	int ecode = 0;
1291 
1292 	dev_dbg(ctx->dev, "%s %d: err 0x%x\n", __func__, __LINE__, status);
1293 
1294 	if (unlikely(status))
1295 		ecode = caam_qi2_strstatus(ctx->dev, status);
1296 
1297 	aead_unmap(ctx->dev, edesc, req);
1298 	qi_cache_free(edesc);
1299 	aead_request_complete(req, ecode);
1300 }
1301 
1302 static void aead_decrypt_done(void *cbk_ctx, u32 status)
1303 {
1304 	struct crypto_async_request *areq = cbk_ctx;
1305 	struct aead_request *req = container_of(areq, struct aead_request,
1306 						base);
1307 	struct caam_request *req_ctx = to_caam_req(areq);
1308 	struct aead_edesc *edesc = req_ctx->edesc;
1309 	struct crypto_aead *aead = crypto_aead_reqtfm(req);
1310 	struct caam_ctx *ctx = crypto_aead_ctx_dma(aead);
1311 	int ecode = 0;
1312 
1313 	dev_dbg(ctx->dev, "%s %d: err 0x%x\n", __func__, __LINE__, status);
1314 
1315 	if (unlikely(status))
1316 		ecode = caam_qi2_strstatus(ctx->dev, status);
1317 
1318 	aead_unmap(ctx->dev, edesc, req);
1319 	qi_cache_free(edesc);
1320 	aead_request_complete(req, ecode);
1321 }
1322 
1323 static int aead_encrypt(struct aead_request *req)
1324 {
1325 	struct aead_edesc *edesc;
1326 	struct crypto_aead *aead = crypto_aead_reqtfm(req);
1327 	struct caam_ctx *ctx = crypto_aead_ctx_dma(aead);
1328 	struct caam_request *caam_req = aead_request_ctx_dma(req);
1329 	int ret;
1330 
1331 	/* allocate extended descriptor */
1332 	edesc = aead_edesc_alloc(req, true);
1333 	if (IS_ERR(edesc))
1334 		return PTR_ERR(edesc);
1335 
1336 	caam_req->flc = &ctx->flc[ENCRYPT];
1337 	caam_req->flc_dma = ctx->flc_dma[ENCRYPT];
1338 	caam_req->cbk = aead_encrypt_done;
1339 	caam_req->ctx = &req->base;
1340 	caam_req->edesc = edesc;
1341 	ret = dpaa2_caam_enqueue(ctx->dev, caam_req);
1342 	if (ret != -EINPROGRESS &&
1343 	    !(ret == -EBUSY && req->base.flags & CRYPTO_TFM_REQ_MAY_BACKLOG)) {
1344 		aead_unmap(ctx->dev, edesc, req);
1345 		qi_cache_free(edesc);
1346 	}
1347 
1348 	return ret;
1349 }
1350 
1351 static int aead_decrypt(struct aead_request *req)
1352 {
1353 	struct aead_edesc *edesc;
1354 	struct crypto_aead *aead = crypto_aead_reqtfm(req);
1355 	struct caam_ctx *ctx = crypto_aead_ctx_dma(aead);
1356 	struct caam_request *caam_req = aead_request_ctx_dma(req);
1357 	int ret;
1358 
1359 	/* allocate extended descriptor */
1360 	edesc = aead_edesc_alloc(req, false);
1361 	if (IS_ERR(edesc))
1362 		return PTR_ERR(edesc);
1363 
1364 	caam_req->flc = &ctx->flc[DECRYPT];
1365 	caam_req->flc_dma = ctx->flc_dma[DECRYPT];
1366 	caam_req->cbk = aead_decrypt_done;
1367 	caam_req->ctx = &req->base;
1368 	caam_req->edesc = edesc;
1369 	ret = dpaa2_caam_enqueue(ctx->dev, caam_req);
1370 	if (ret != -EINPROGRESS &&
1371 	    !(ret == -EBUSY && req->base.flags & CRYPTO_TFM_REQ_MAY_BACKLOG)) {
1372 		aead_unmap(ctx->dev, edesc, req);
1373 		qi_cache_free(edesc);
1374 	}
1375 
1376 	return ret;
1377 }
1378 
1379 static int ipsec_gcm_encrypt(struct aead_request *req)
1380 {
1381 	return crypto_ipsec_check_assoclen(req->assoclen) ? : aead_encrypt(req);
1382 }
1383 
1384 static int ipsec_gcm_decrypt(struct aead_request *req)
1385 {
1386 	return crypto_ipsec_check_assoclen(req->assoclen) ? : aead_decrypt(req);
1387 }
1388 
1389 static void skcipher_encrypt_done(void *cbk_ctx, u32 status)
1390 {
1391 	struct crypto_async_request *areq = cbk_ctx;
1392 	struct skcipher_request *req = skcipher_request_cast(areq);
1393 	struct caam_request *req_ctx = to_caam_req(areq);
1394 	struct crypto_skcipher *skcipher = crypto_skcipher_reqtfm(req);
1395 	struct caam_ctx *ctx = crypto_skcipher_ctx_dma(skcipher);
1396 	struct skcipher_edesc *edesc = req_ctx->edesc;
1397 	int ecode = 0;
1398 	int ivsize = crypto_skcipher_ivsize(skcipher);
1399 
1400 	dev_dbg(ctx->dev, "%s %d: err 0x%x\n", __func__, __LINE__, status);
1401 
1402 	if (unlikely(status))
1403 		ecode = caam_qi2_strstatus(ctx->dev, status);
1404 
1405 	print_hex_dump_debug("dstiv  @" __stringify(__LINE__)": ",
1406 			     DUMP_PREFIX_ADDRESS, 16, 4, req->iv,
1407 			     edesc->src_nents > 1 ? 100 : ivsize, 1);
1408 	caam_dump_sg("dst    @" __stringify(__LINE__)": ",
1409 		     DUMP_PREFIX_ADDRESS, 16, 4, req->dst,
1410 		     edesc->dst_nents > 1 ? 100 : req->cryptlen, 1);
1411 
1412 	skcipher_unmap(ctx->dev, edesc, req);
1413 
1414 	/*
1415 	 * The crypto API expects us to set the IV (req->iv) to the last
1416 	 * ciphertext block (CBC mode) or last counter (CTR mode).
1417 	 * This is used e.g. by the CTS mode.
1418 	 */
1419 	if (!ecode)
1420 		memcpy(req->iv, (u8 *)&edesc->sgt[0] + edesc->qm_sg_bytes,
1421 		       ivsize);
1422 
1423 	qi_cache_free(edesc);
1424 	skcipher_request_complete(req, ecode);
1425 }
1426 
1427 static void skcipher_decrypt_done(void *cbk_ctx, u32 status)
1428 {
1429 	struct crypto_async_request *areq = cbk_ctx;
1430 	struct skcipher_request *req = skcipher_request_cast(areq);
1431 	struct caam_request *req_ctx = to_caam_req(areq);
1432 	struct crypto_skcipher *skcipher = crypto_skcipher_reqtfm(req);
1433 	struct caam_ctx *ctx = crypto_skcipher_ctx_dma(skcipher);
1434 	struct skcipher_edesc *edesc = req_ctx->edesc;
1435 	int ecode = 0;
1436 	int ivsize = crypto_skcipher_ivsize(skcipher);
1437 
1438 	dev_dbg(ctx->dev, "%s %d: err 0x%x\n", __func__, __LINE__, status);
1439 
1440 	if (unlikely(status))
1441 		ecode = caam_qi2_strstatus(ctx->dev, status);
1442 
1443 	print_hex_dump_debug("dstiv  @" __stringify(__LINE__)": ",
1444 			     DUMP_PREFIX_ADDRESS, 16, 4, req->iv,
1445 			     edesc->src_nents > 1 ? 100 : ivsize, 1);
1446 	caam_dump_sg("dst    @" __stringify(__LINE__)": ",
1447 		     DUMP_PREFIX_ADDRESS, 16, 4, req->dst,
1448 		     edesc->dst_nents > 1 ? 100 : req->cryptlen, 1);
1449 
1450 	skcipher_unmap(ctx->dev, edesc, req);
1451 
1452 	/*
1453 	 * The crypto API expects us to set the IV (req->iv) to the last
1454 	 * ciphertext block (CBC mode) or last counter (CTR mode).
1455 	 * This is used e.g. by the CTS mode.
1456 	 */
1457 	if (!ecode)
1458 		memcpy(req->iv, (u8 *)&edesc->sgt[0] + edesc->qm_sg_bytes,
1459 		       ivsize);
1460 
1461 	qi_cache_free(edesc);
1462 	skcipher_request_complete(req, ecode);
1463 }
1464 
1465 static inline bool xts_skcipher_ivsize(struct skcipher_request *req)
1466 {
1467 	struct crypto_skcipher *skcipher = crypto_skcipher_reqtfm(req);
1468 	unsigned int ivsize = crypto_skcipher_ivsize(skcipher);
1469 
1470 	return !!get_unaligned((u64 *)(req->iv + (ivsize / 2)));
1471 }
1472 
1473 static int skcipher_encrypt(struct skcipher_request *req)
1474 {
1475 	struct skcipher_edesc *edesc;
1476 	struct crypto_skcipher *skcipher = crypto_skcipher_reqtfm(req);
1477 	struct caam_ctx *ctx = crypto_skcipher_ctx_dma(skcipher);
1478 	struct caam_request *caam_req = skcipher_request_ctx_dma(req);
1479 	struct dpaa2_caam_priv *priv = dev_get_drvdata(ctx->dev);
1480 	int ret;
1481 
1482 	/*
1483 	 * XTS is expected to return an error even for input length = 0
1484 	 * Note that the case input length < block size will be caught during
1485 	 * HW offloading and return an error.
1486 	 */
1487 	if (!req->cryptlen && !ctx->fallback)
1488 		return 0;
1489 
1490 	if (ctx->fallback && ((priv->sec_attr.era <= 8 && xts_skcipher_ivsize(req)) ||
1491 			      ctx->xts_key_fallback)) {
1492 		skcipher_request_set_tfm(&caam_req->fallback_req, ctx->fallback);
1493 		skcipher_request_set_callback(&caam_req->fallback_req,
1494 					      req->base.flags,
1495 					      req->base.complete,
1496 					      req->base.data);
1497 		skcipher_request_set_crypt(&caam_req->fallback_req, req->src,
1498 					   req->dst, req->cryptlen, req->iv);
1499 
1500 		return crypto_skcipher_encrypt(&caam_req->fallback_req);
1501 	}
1502 
1503 	/* allocate extended descriptor */
1504 	edesc = skcipher_edesc_alloc(req);
1505 	if (IS_ERR(edesc))
1506 		return PTR_ERR(edesc);
1507 
1508 	caam_req->flc = &ctx->flc[ENCRYPT];
1509 	caam_req->flc_dma = ctx->flc_dma[ENCRYPT];
1510 	caam_req->cbk = skcipher_encrypt_done;
1511 	caam_req->ctx = &req->base;
1512 	caam_req->edesc = edesc;
1513 	ret = dpaa2_caam_enqueue(ctx->dev, caam_req);
1514 	if (ret != -EINPROGRESS &&
1515 	    !(ret == -EBUSY && req->base.flags & CRYPTO_TFM_REQ_MAY_BACKLOG)) {
1516 		skcipher_unmap(ctx->dev, edesc, req);
1517 		qi_cache_free(edesc);
1518 	}
1519 
1520 	return ret;
1521 }
1522 
1523 static int skcipher_decrypt(struct skcipher_request *req)
1524 {
1525 	struct skcipher_edesc *edesc;
1526 	struct crypto_skcipher *skcipher = crypto_skcipher_reqtfm(req);
1527 	struct caam_ctx *ctx = crypto_skcipher_ctx_dma(skcipher);
1528 	struct caam_request *caam_req = skcipher_request_ctx_dma(req);
1529 	struct dpaa2_caam_priv *priv = dev_get_drvdata(ctx->dev);
1530 	int ret;
1531 
1532 	/*
1533 	 * XTS is expected to return an error even for input length = 0
1534 	 * Note that the case input length < block size will be caught during
1535 	 * HW offloading and return an error.
1536 	 */
1537 	if (!req->cryptlen && !ctx->fallback)
1538 		return 0;
1539 
1540 	if (ctx->fallback && ((priv->sec_attr.era <= 8 && xts_skcipher_ivsize(req)) ||
1541 			      ctx->xts_key_fallback)) {
1542 		skcipher_request_set_tfm(&caam_req->fallback_req, ctx->fallback);
1543 		skcipher_request_set_callback(&caam_req->fallback_req,
1544 					      req->base.flags,
1545 					      req->base.complete,
1546 					      req->base.data);
1547 		skcipher_request_set_crypt(&caam_req->fallback_req, req->src,
1548 					   req->dst, req->cryptlen, req->iv);
1549 
1550 		return crypto_skcipher_decrypt(&caam_req->fallback_req);
1551 	}
1552 
1553 	/* allocate extended descriptor */
1554 	edesc = skcipher_edesc_alloc(req);
1555 	if (IS_ERR(edesc))
1556 		return PTR_ERR(edesc);
1557 
1558 	caam_req->flc = &ctx->flc[DECRYPT];
1559 	caam_req->flc_dma = ctx->flc_dma[DECRYPT];
1560 	caam_req->cbk = skcipher_decrypt_done;
1561 	caam_req->ctx = &req->base;
1562 	caam_req->edesc = edesc;
1563 	ret = dpaa2_caam_enqueue(ctx->dev, caam_req);
1564 	if (ret != -EINPROGRESS &&
1565 	    !(ret == -EBUSY && req->base.flags & CRYPTO_TFM_REQ_MAY_BACKLOG)) {
1566 		skcipher_unmap(ctx->dev, edesc, req);
1567 		qi_cache_free(edesc);
1568 	}
1569 
1570 	return ret;
1571 }
1572 
1573 static int caam_cra_init(struct caam_ctx *ctx, struct caam_alg_entry *caam,
1574 			 bool uses_dkp)
1575 {
1576 	dma_addr_t dma_addr;
1577 	int i;
1578 
1579 	/* copy descriptor header template value */
1580 	ctx->cdata.algtype = OP_TYPE_CLASS1_ALG | caam->class1_alg_type;
1581 	ctx->adata.algtype = OP_TYPE_CLASS2_ALG | caam->class2_alg_type;
1582 
1583 	ctx->dev = caam->dev;
1584 	ctx->dir = uses_dkp ? DMA_BIDIRECTIONAL : DMA_TO_DEVICE;
1585 
1586 	dma_addr = dma_map_single_attrs(ctx->dev, ctx->flc,
1587 					offsetof(struct caam_ctx, flc_dma),
1588 					ctx->dir, DMA_ATTR_SKIP_CPU_SYNC);
1589 	if (dma_mapping_error(ctx->dev, dma_addr)) {
1590 		dev_err(ctx->dev, "unable to map key, shared descriptors\n");
1591 		return -ENOMEM;
1592 	}
1593 
1594 	for (i = 0; i < NUM_OP; i++)
1595 		ctx->flc_dma[i] = dma_addr + i * sizeof(ctx->flc[i]);
1596 	ctx->key_dma = dma_addr + NUM_OP * sizeof(ctx->flc[0]);
1597 
1598 	return 0;
1599 }
1600 
1601 static int caam_cra_init_skcipher(struct crypto_skcipher *tfm)
1602 {
1603 	struct skcipher_alg *alg = crypto_skcipher_alg(tfm);
1604 	struct caam_skcipher_alg *caam_alg =
1605 		container_of(alg, typeof(*caam_alg), skcipher);
1606 	struct caam_ctx *ctx = crypto_skcipher_ctx_dma(tfm);
1607 	u32 alg_aai = caam_alg->caam.class1_alg_type & OP_ALG_AAI_MASK;
1608 	int ret = 0;
1609 
1610 	if (alg_aai == OP_ALG_AAI_XTS) {
1611 		const char *tfm_name = crypto_tfm_alg_name(&tfm->base);
1612 		struct crypto_skcipher *fallback;
1613 
1614 		fallback = crypto_alloc_skcipher(tfm_name, 0,
1615 						 CRYPTO_ALG_NEED_FALLBACK);
1616 		if (IS_ERR(fallback)) {
1617 			dev_err(caam_alg->caam.dev,
1618 				"Failed to allocate %s fallback: %ld\n",
1619 				tfm_name, PTR_ERR(fallback));
1620 			return PTR_ERR(fallback);
1621 		}
1622 
1623 		ctx->fallback = fallback;
1624 		crypto_skcipher_set_reqsize_dma(
1625 			tfm, sizeof(struct caam_request) +
1626 			     crypto_skcipher_reqsize(fallback));
1627 	} else {
1628 		crypto_skcipher_set_reqsize_dma(tfm,
1629 						sizeof(struct caam_request));
1630 	}
1631 
1632 	ret = caam_cra_init(ctx, &caam_alg->caam, false);
1633 	if (ret && ctx->fallback)
1634 		crypto_free_skcipher(ctx->fallback);
1635 
1636 	return ret;
1637 }
1638 
1639 static int caam_cra_init_aead(struct crypto_aead *tfm)
1640 {
1641 	struct aead_alg *alg = crypto_aead_alg(tfm);
1642 	struct caam_aead_alg *caam_alg = container_of(alg, typeof(*caam_alg),
1643 						      aead);
1644 
1645 	crypto_aead_set_reqsize_dma(tfm, sizeof(struct caam_request));
1646 	return caam_cra_init(crypto_aead_ctx_dma(tfm), &caam_alg->caam,
1647 			     !caam_alg->caam.nodkp);
1648 }
1649 
1650 static void caam_exit_common(struct caam_ctx *ctx)
1651 {
1652 	dma_unmap_single_attrs(ctx->dev, ctx->flc_dma[0],
1653 			       offsetof(struct caam_ctx, flc_dma), ctx->dir,
1654 			       DMA_ATTR_SKIP_CPU_SYNC);
1655 }
1656 
1657 static void caam_cra_exit(struct crypto_skcipher *tfm)
1658 {
1659 	struct caam_ctx *ctx = crypto_skcipher_ctx_dma(tfm);
1660 
1661 	if (ctx->fallback)
1662 		crypto_free_skcipher(ctx->fallback);
1663 	caam_exit_common(ctx);
1664 }
1665 
1666 static void caam_cra_exit_aead(struct crypto_aead *tfm)
1667 {
1668 	caam_exit_common(crypto_aead_ctx_dma(tfm));
1669 }
1670 
1671 static struct caam_skcipher_alg driver_algs[] = {
1672 	{
1673 		.skcipher = {
1674 			.base = {
1675 				.cra_name = "cbc(aes)",
1676 				.cra_driver_name = "cbc-aes-caam-qi2",
1677 				.cra_blocksize = AES_BLOCK_SIZE,
1678 			},
1679 			.setkey = aes_skcipher_setkey,
1680 			.encrypt = skcipher_encrypt,
1681 			.decrypt = skcipher_decrypt,
1682 			.min_keysize = AES_MIN_KEY_SIZE,
1683 			.max_keysize = AES_MAX_KEY_SIZE,
1684 			.ivsize = AES_BLOCK_SIZE,
1685 		},
1686 		.caam.class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CBC,
1687 	},
1688 	{
1689 		.skcipher = {
1690 			.base = {
1691 				.cra_name = "cbc(des3_ede)",
1692 				.cra_driver_name = "cbc-3des-caam-qi2",
1693 				.cra_blocksize = DES3_EDE_BLOCK_SIZE,
1694 			},
1695 			.setkey = des3_skcipher_setkey,
1696 			.encrypt = skcipher_encrypt,
1697 			.decrypt = skcipher_decrypt,
1698 			.min_keysize = DES3_EDE_KEY_SIZE,
1699 			.max_keysize = DES3_EDE_KEY_SIZE,
1700 			.ivsize = DES3_EDE_BLOCK_SIZE,
1701 		},
1702 		.caam.class1_alg_type = OP_ALG_ALGSEL_3DES | OP_ALG_AAI_CBC,
1703 	},
1704 	{
1705 		.skcipher = {
1706 			.base = {
1707 				.cra_name = "cbc(des)",
1708 				.cra_driver_name = "cbc-des-caam-qi2",
1709 				.cra_blocksize = DES_BLOCK_SIZE,
1710 			},
1711 			.setkey = des_skcipher_setkey,
1712 			.encrypt = skcipher_encrypt,
1713 			.decrypt = skcipher_decrypt,
1714 			.min_keysize = DES_KEY_SIZE,
1715 			.max_keysize = DES_KEY_SIZE,
1716 			.ivsize = DES_BLOCK_SIZE,
1717 		},
1718 		.caam.class1_alg_type = OP_ALG_ALGSEL_DES | OP_ALG_AAI_CBC,
1719 	},
1720 	{
1721 		.skcipher = {
1722 			.base = {
1723 				.cra_name = "ctr(aes)",
1724 				.cra_driver_name = "ctr-aes-caam-qi2",
1725 				.cra_blocksize = 1,
1726 			},
1727 			.setkey = ctr_skcipher_setkey,
1728 			.encrypt = skcipher_encrypt,
1729 			.decrypt = skcipher_decrypt,
1730 			.min_keysize = AES_MIN_KEY_SIZE,
1731 			.max_keysize = AES_MAX_KEY_SIZE,
1732 			.ivsize = AES_BLOCK_SIZE,
1733 			.chunksize = AES_BLOCK_SIZE,
1734 		},
1735 		.caam.class1_alg_type = OP_ALG_ALGSEL_AES |
1736 					OP_ALG_AAI_CTR_MOD128,
1737 	},
1738 	{
1739 		.skcipher = {
1740 			.base = {
1741 				.cra_name = "rfc3686(ctr(aes))",
1742 				.cra_driver_name = "rfc3686-ctr-aes-caam-qi2",
1743 				.cra_blocksize = 1,
1744 			},
1745 			.setkey = rfc3686_skcipher_setkey,
1746 			.encrypt = skcipher_encrypt,
1747 			.decrypt = skcipher_decrypt,
1748 			.min_keysize = AES_MIN_KEY_SIZE +
1749 				       CTR_RFC3686_NONCE_SIZE,
1750 			.max_keysize = AES_MAX_KEY_SIZE +
1751 				       CTR_RFC3686_NONCE_SIZE,
1752 			.ivsize = CTR_RFC3686_IV_SIZE,
1753 			.chunksize = AES_BLOCK_SIZE,
1754 		},
1755 		.caam = {
1756 			.class1_alg_type = OP_ALG_ALGSEL_AES |
1757 					   OP_ALG_AAI_CTR_MOD128,
1758 			.rfc3686 = true,
1759 		},
1760 	},
1761 	{
1762 		.skcipher = {
1763 			.base = {
1764 				.cra_name = "xts(aes)",
1765 				.cra_driver_name = "xts-aes-caam-qi2",
1766 				.cra_flags = CRYPTO_ALG_NEED_FALLBACK,
1767 				.cra_blocksize = AES_BLOCK_SIZE,
1768 			},
1769 			.setkey = xts_skcipher_setkey,
1770 			.encrypt = skcipher_encrypt,
1771 			.decrypt = skcipher_decrypt,
1772 			.min_keysize = 2 * AES_MIN_KEY_SIZE,
1773 			.max_keysize = 2 * AES_MAX_KEY_SIZE,
1774 			.ivsize = AES_BLOCK_SIZE,
1775 		},
1776 		.caam.class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_XTS,
1777 	},
1778 	{
1779 		.skcipher = {
1780 			.base = {
1781 				.cra_name = "chacha20",
1782 				.cra_driver_name = "chacha20-caam-qi2",
1783 				.cra_blocksize = 1,
1784 			},
1785 			.setkey = chacha20_skcipher_setkey,
1786 			.encrypt = skcipher_encrypt,
1787 			.decrypt = skcipher_decrypt,
1788 			.min_keysize = CHACHA_KEY_SIZE,
1789 			.max_keysize = CHACHA_KEY_SIZE,
1790 			.ivsize = CHACHA_IV_SIZE,
1791 		},
1792 		.caam.class1_alg_type = OP_ALG_ALGSEL_CHACHA20,
1793 	},
1794 };
1795 
1796 static struct caam_aead_alg driver_aeads[] = {
1797 	{
1798 		.aead = {
1799 			.base = {
1800 				.cra_name = "rfc4106(gcm(aes))",
1801 				.cra_driver_name = "rfc4106-gcm-aes-caam-qi2",
1802 				.cra_blocksize = 1,
1803 			},
1804 			.setkey = rfc4106_setkey,
1805 			.setauthsize = rfc4106_setauthsize,
1806 			.encrypt = ipsec_gcm_encrypt,
1807 			.decrypt = ipsec_gcm_decrypt,
1808 			.ivsize = 8,
1809 			.maxauthsize = AES_BLOCK_SIZE,
1810 		},
1811 		.caam = {
1812 			.class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_GCM,
1813 			.nodkp = true,
1814 		},
1815 	},
1816 	{
1817 		.aead = {
1818 			.base = {
1819 				.cra_name = "rfc4543(gcm(aes))",
1820 				.cra_driver_name = "rfc4543-gcm-aes-caam-qi2",
1821 				.cra_blocksize = 1,
1822 			},
1823 			.setkey = rfc4543_setkey,
1824 			.setauthsize = rfc4543_setauthsize,
1825 			.encrypt = ipsec_gcm_encrypt,
1826 			.decrypt = ipsec_gcm_decrypt,
1827 			.ivsize = 8,
1828 			.maxauthsize = AES_BLOCK_SIZE,
1829 		},
1830 		.caam = {
1831 			.class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_GCM,
1832 			.nodkp = true,
1833 		},
1834 	},
1835 	/* Galois Counter Mode */
1836 	{
1837 		.aead = {
1838 			.base = {
1839 				.cra_name = "gcm(aes)",
1840 				.cra_driver_name = "gcm-aes-caam-qi2",
1841 				.cra_blocksize = 1,
1842 			},
1843 			.setkey = gcm_setkey,
1844 			.setauthsize = gcm_setauthsize,
1845 			.encrypt = aead_encrypt,
1846 			.decrypt = aead_decrypt,
1847 			.ivsize = 12,
1848 			.maxauthsize = AES_BLOCK_SIZE,
1849 		},
1850 		.caam = {
1851 			.class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_GCM,
1852 			.nodkp = true,
1853 		}
1854 	},
1855 	/* single-pass ipsec_esp descriptor */
1856 	{
1857 		.aead = {
1858 			.base = {
1859 				.cra_name = "authenc(hmac(md5),cbc(aes))",
1860 				.cra_driver_name = "authenc-hmac-md5-"
1861 						   "cbc-aes-caam-qi2",
1862 				.cra_blocksize = AES_BLOCK_SIZE,
1863 			},
1864 			.setkey = aead_setkey,
1865 			.setauthsize = aead_setauthsize,
1866 			.encrypt = aead_encrypt,
1867 			.decrypt = aead_decrypt,
1868 			.ivsize = AES_BLOCK_SIZE,
1869 			.maxauthsize = MD5_DIGEST_SIZE,
1870 		},
1871 		.caam = {
1872 			.class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CBC,
1873 			.class2_alg_type = OP_ALG_ALGSEL_MD5 |
1874 					   OP_ALG_AAI_HMAC_PRECOMP,
1875 		}
1876 	},
1877 	{
1878 		.aead = {
1879 			.base = {
1880 				.cra_name = "echainiv(authenc(hmac(md5),"
1881 					    "cbc(aes)))",
1882 				.cra_driver_name = "echainiv-authenc-hmac-md5-"
1883 						   "cbc-aes-caam-qi2",
1884 				.cra_blocksize = AES_BLOCK_SIZE,
1885 			},
1886 			.setkey = aead_setkey,
1887 			.setauthsize = aead_setauthsize,
1888 			.encrypt = aead_encrypt,
1889 			.decrypt = aead_decrypt,
1890 			.ivsize = AES_BLOCK_SIZE,
1891 			.maxauthsize = MD5_DIGEST_SIZE,
1892 		},
1893 		.caam = {
1894 			.class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CBC,
1895 			.class2_alg_type = OP_ALG_ALGSEL_MD5 |
1896 					   OP_ALG_AAI_HMAC_PRECOMP,
1897 			.geniv = true,
1898 		}
1899 	},
1900 	{
1901 		.aead = {
1902 			.base = {
1903 				.cra_name = "authenc(hmac(sha1),cbc(aes))",
1904 				.cra_driver_name = "authenc-hmac-sha1-"
1905 						   "cbc-aes-caam-qi2",
1906 				.cra_blocksize = AES_BLOCK_SIZE,
1907 			},
1908 			.setkey = aead_setkey,
1909 			.setauthsize = aead_setauthsize,
1910 			.encrypt = aead_encrypt,
1911 			.decrypt = aead_decrypt,
1912 			.ivsize = AES_BLOCK_SIZE,
1913 			.maxauthsize = SHA1_DIGEST_SIZE,
1914 		},
1915 		.caam = {
1916 			.class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CBC,
1917 			.class2_alg_type = OP_ALG_ALGSEL_SHA1 |
1918 					   OP_ALG_AAI_HMAC_PRECOMP,
1919 		}
1920 	},
1921 	{
1922 		.aead = {
1923 			.base = {
1924 				.cra_name = "echainiv(authenc(hmac(sha1),"
1925 					    "cbc(aes)))",
1926 				.cra_driver_name = "echainiv-authenc-"
1927 						   "hmac-sha1-cbc-aes-caam-qi2",
1928 				.cra_blocksize = AES_BLOCK_SIZE,
1929 			},
1930 			.setkey = aead_setkey,
1931 			.setauthsize = aead_setauthsize,
1932 			.encrypt = aead_encrypt,
1933 			.decrypt = aead_decrypt,
1934 			.ivsize = AES_BLOCK_SIZE,
1935 			.maxauthsize = SHA1_DIGEST_SIZE,
1936 		},
1937 		.caam = {
1938 			.class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CBC,
1939 			.class2_alg_type = OP_ALG_ALGSEL_SHA1 |
1940 					   OP_ALG_AAI_HMAC_PRECOMP,
1941 			.geniv = true,
1942 		},
1943 	},
1944 	{
1945 		.aead = {
1946 			.base = {
1947 				.cra_name = "authenc(hmac(sha224),cbc(aes))",
1948 				.cra_driver_name = "authenc-hmac-sha224-"
1949 						   "cbc-aes-caam-qi2",
1950 				.cra_blocksize = AES_BLOCK_SIZE,
1951 			},
1952 			.setkey = aead_setkey,
1953 			.setauthsize = aead_setauthsize,
1954 			.encrypt = aead_encrypt,
1955 			.decrypt = aead_decrypt,
1956 			.ivsize = AES_BLOCK_SIZE,
1957 			.maxauthsize = SHA224_DIGEST_SIZE,
1958 		},
1959 		.caam = {
1960 			.class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CBC,
1961 			.class2_alg_type = OP_ALG_ALGSEL_SHA224 |
1962 					   OP_ALG_AAI_HMAC_PRECOMP,
1963 		}
1964 	},
1965 	{
1966 		.aead = {
1967 			.base = {
1968 				.cra_name = "echainiv(authenc(hmac(sha224),"
1969 					    "cbc(aes)))",
1970 				.cra_driver_name = "echainiv-authenc-"
1971 						   "hmac-sha224-cbc-aes-caam-qi2",
1972 				.cra_blocksize = AES_BLOCK_SIZE,
1973 			},
1974 			.setkey = aead_setkey,
1975 			.setauthsize = aead_setauthsize,
1976 			.encrypt = aead_encrypt,
1977 			.decrypt = aead_decrypt,
1978 			.ivsize = AES_BLOCK_SIZE,
1979 			.maxauthsize = SHA224_DIGEST_SIZE,
1980 		},
1981 		.caam = {
1982 			.class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CBC,
1983 			.class2_alg_type = OP_ALG_ALGSEL_SHA224 |
1984 					   OP_ALG_AAI_HMAC_PRECOMP,
1985 			.geniv = true,
1986 		}
1987 	},
1988 	{
1989 		.aead = {
1990 			.base = {
1991 				.cra_name = "authenc(hmac(sha256),cbc(aes))",
1992 				.cra_driver_name = "authenc-hmac-sha256-"
1993 						   "cbc-aes-caam-qi2",
1994 				.cra_blocksize = AES_BLOCK_SIZE,
1995 			},
1996 			.setkey = aead_setkey,
1997 			.setauthsize = aead_setauthsize,
1998 			.encrypt = aead_encrypt,
1999 			.decrypt = aead_decrypt,
2000 			.ivsize = AES_BLOCK_SIZE,
2001 			.maxauthsize = SHA256_DIGEST_SIZE,
2002 		},
2003 		.caam = {
2004 			.class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CBC,
2005 			.class2_alg_type = OP_ALG_ALGSEL_SHA256 |
2006 					   OP_ALG_AAI_HMAC_PRECOMP,
2007 		}
2008 	},
2009 	{
2010 		.aead = {
2011 			.base = {
2012 				.cra_name = "echainiv(authenc(hmac(sha256),"
2013 					    "cbc(aes)))",
2014 				.cra_driver_name = "echainiv-authenc-"
2015 						   "hmac-sha256-cbc-aes-"
2016 						   "caam-qi2",
2017 				.cra_blocksize = AES_BLOCK_SIZE,
2018 			},
2019 			.setkey = aead_setkey,
2020 			.setauthsize = aead_setauthsize,
2021 			.encrypt = aead_encrypt,
2022 			.decrypt = aead_decrypt,
2023 			.ivsize = AES_BLOCK_SIZE,
2024 			.maxauthsize = SHA256_DIGEST_SIZE,
2025 		},
2026 		.caam = {
2027 			.class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CBC,
2028 			.class2_alg_type = OP_ALG_ALGSEL_SHA256 |
2029 					   OP_ALG_AAI_HMAC_PRECOMP,
2030 			.geniv = true,
2031 		}
2032 	},
2033 	{
2034 		.aead = {
2035 			.base = {
2036 				.cra_name = "authenc(hmac(sha384),cbc(aes))",
2037 				.cra_driver_name = "authenc-hmac-sha384-"
2038 						   "cbc-aes-caam-qi2",
2039 				.cra_blocksize = AES_BLOCK_SIZE,
2040 			},
2041 			.setkey = aead_setkey,
2042 			.setauthsize = aead_setauthsize,
2043 			.encrypt = aead_encrypt,
2044 			.decrypt = aead_decrypt,
2045 			.ivsize = AES_BLOCK_SIZE,
2046 			.maxauthsize = SHA384_DIGEST_SIZE,
2047 		},
2048 		.caam = {
2049 			.class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CBC,
2050 			.class2_alg_type = OP_ALG_ALGSEL_SHA384 |
2051 					   OP_ALG_AAI_HMAC_PRECOMP,
2052 		}
2053 	},
2054 	{
2055 		.aead = {
2056 			.base = {
2057 				.cra_name = "echainiv(authenc(hmac(sha384),"
2058 					    "cbc(aes)))",
2059 				.cra_driver_name = "echainiv-authenc-"
2060 						   "hmac-sha384-cbc-aes-"
2061 						   "caam-qi2",
2062 				.cra_blocksize = AES_BLOCK_SIZE,
2063 			},
2064 			.setkey = aead_setkey,
2065 			.setauthsize = aead_setauthsize,
2066 			.encrypt = aead_encrypt,
2067 			.decrypt = aead_decrypt,
2068 			.ivsize = AES_BLOCK_SIZE,
2069 			.maxauthsize = SHA384_DIGEST_SIZE,
2070 		},
2071 		.caam = {
2072 			.class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CBC,
2073 			.class2_alg_type = OP_ALG_ALGSEL_SHA384 |
2074 					   OP_ALG_AAI_HMAC_PRECOMP,
2075 			.geniv = true,
2076 		}
2077 	},
2078 	{
2079 		.aead = {
2080 			.base = {
2081 				.cra_name = "authenc(hmac(sha512),cbc(aes))",
2082 				.cra_driver_name = "authenc-hmac-sha512-"
2083 						   "cbc-aes-caam-qi2",
2084 				.cra_blocksize = AES_BLOCK_SIZE,
2085 			},
2086 			.setkey = aead_setkey,
2087 			.setauthsize = aead_setauthsize,
2088 			.encrypt = aead_encrypt,
2089 			.decrypt = aead_decrypt,
2090 			.ivsize = AES_BLOCK_SIZE,
2091 			.maxauthsize = SHA512_DIGEST_SIZE,
2092 		},
2093 		.caam = {
2094 			.class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CBC,
2095 			.class2_alg_type = OP_ALG_ALGSEL_SHA512 |
2096 					   OP_ALG_AAI_HMAC_PRECOMP,
2097 		}
2098 	},
2099 	{
2100 		.aead = {
2101 			.base = {
2102 				.cra_name = "echainiv(authenc(hmac(sha512),"
2103 					    "cbc(aes)))",
2104 				.cra_driver_name = "echainiv-authenc-"
2105 						   "hmac-sha512-cbc-aes-"
2106 						   "caam-qi2",
2107 				.cra_blocksize = AES_BLOCK_SIZE,
2108 			},
2109 			.setkey = aead_setkey,
2110 			.setauthsize = aead_setauthsize,
2111 			.encrypt = aead_encrypt,
2112 			.decrypt = aead_decrypt,
2113 			.ivsize = AES_BLOCK_SIZE,
2114 			.maxauthsize = SHA512_DIGEST_SIZE,
2115 		},
2116 		.caam = {
2117 			.class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CBC,
2118 			.class2_alg_type = OP_ALG_ALGSEL_SHA512 |
2119 					   OP_ALG_AAI_HMAC_PRECOMP,
2120 			.geniv = true,
2121 		}
2122 	},
2123 	{
2124 		.aead = {
2125 			.base = {
2126 				.cra_name = "authenc(hmac(md5),cbc(des3_ede))",
2127 				.cra_driver_name = "authenc-hmac-md5-"
2128 						   "cbc-des3_ede-caam-qi2",
2129 				.cra_blocksize = DES3_EDE_BLOCK_SIZE,
2130 			},
2131 			.setkey = des3_aead_setkey,
2132 			.setauthsize = aead_setauthsize,
2133 			.encrypt = aead_encrypt,
2134 			.decrypt = aead_decrypt,
2135 			.ivsize = DES3_EDE_BLOCK_SIZE,
2136 			.maxauthsize = MD5_DIGEST_SIZE,
2137 		},
2138 		.caam = {
2139 			.class1_alg_type = OP_ALG_ALGSEL_3DES | OP_ALG_AAI_CBC,
2140 			.class2_alg_type = OP_ALG_ALGSEL_MD5 |
2141 					   OP_ALG_AAI_HMAC_PRECOMP,
2142 		}
2143 	},
2144 	{
2145 		.aead = {
2146 			.base = {
2147 				.cra_name = "echainiv(authenc(hmac(md5),"
2148 					    "cbc(des3_ede)))",
2149 				.cra_driver_name = "echainiv-authenc-hmac-md5-"
2150 						   "cbc-des3_ede-caam-qi2",
2151 				.cra_blocksize = DES3_EDE_BLOCK_SIZE,
2152 			},
2153 			.setkey = des3_aead_setkey,
2154 			.setauthsize = aead_setauthsize,
2155 			.encrypt = aead_encrypt,
2156 			.decrypt = aead_decrypt,
2157 			.ivsize = DES3_EDE_BLOCK_SIZE,
2158 			.maxauthsize = MD5_DIGEST_SIZE,
2159 		},
2160 		.caam = {
2161 			.class1_alg_type = OP_ALG_ALGSEL_3DES | OP_ALG_AAI_CBC,
2162 			.class2_alg_type = OP_ALG_ALGSEL_MD5 |
2163 					   OP_ALG_AAI_HMAC_PRECOMP,
2164 			.geniv = true,
2165 		}
2166 	},
2167 	{
2168 		.aead = {
2169 			.base = {
2170 				.cra_name = "authenc(hmac(sha1),"
2171 					    "cbc(des3_ede))",
2172 				.cra_driver_name = "authenc-hmac-sha1-"
2173 						   "cbc-des3_ede-caam-qi2",
2174 				.cra_blocksize = DES3_EDE_BLOCK_SIZE,
2175 			},
2176 			.setkey = des3_aead_setkey,
2177 			.setauthsize = aead_setauthsize,
2178 			.encrypt = aead_encrypt,
2179 			.decrypt = aead_decrypt,
2180 			.ivsize = DES3_EDE_BLOCK_SIZE,
2181 			.maxauthsize = SHA1_DIGEST_SIZE,
2182 		},
2183 		.caam = {
2184 			.class1_alg_type = OP_ALG_ALGSEL_3DES | OP_ALG_AAI_CBC,
2185 			.class2_alg_type = OP_ALG_ALGSEL_SHA1 |
2186 					   OP_ALG_AAI_HMAC_PRECOMP,
2187 		},
2188 	},
2189 	{
2190 		.aead = {
2191 			.base = {
2192 				.cra_name = "echainiv(authenc(hmac(sha1),"
2193 					    "cbc(des3_ede)))",
2194 				.cra_driver_name = "echainiv-authenc-"
2195 						   "hmac-sha1-"
2196 						   "cbc-des3_ede-caam-qi2",
2197 				.cra_blocksize = DES3_EDE_BLOCK_SIZE,
2198 			},
2199 			.setkey = des3_aead_setkey,
2200 			.setauthsize = aead_setauthsize,
2201 			.encrypt = aead_encrypt,
2202 			.decrypt = aead_decrypt,
2203 			.ivsize = DES3_EDE_BLOCK_SIZE,
2204 			.maxauthsize = SHA1_DIGEST_SIZE,
2205 		},
2206 		.caam = {
2207 			.class1_alg_type = OP_ALG_ALGSEL_3DES | OP_ALG_AAI_CBC,
2208 			.class2_alg_type = OP_ALG_ALGSEL_SHA1 |
2209 					   OP_ALG_AAI_HMAC_PRECOMP,
2210 			.geniv = true,
2211 		}
2212 	},
2213 	{
2214 		.aead = {
2215 			.base = {
2216 				.cra_name = "authenc(hmac(sha224),"
2217 					    "cbc(des3_ede))",
2218 				.cra_driver_name = "authenc-hmac-sha224-"
2219 						   "cbc-des3_ede-caam-qi2",
2220 				.cra_blocksize = DES3_EDE_BLOCK_SIZE,
2221 			},
2222 			.setkey = des3_aead_setkey,
2223 			.setauthsize = aead_setauthsize,
2224 			.encrypt = aead_encrypt,
2225 			.decrypt = aead_decrypt,
2226 			.ivsize = DES3_EDE_BLOCK_SIZE,
2227 			.maxauthsize = SHA224_DIGEST_SIZE,
2228 		},
2229 		.caam = {
2230 			.class1_alg_type = OP_ALG_ALGSEL_3DES | OP_ALG_AAI_CBC,
2231 			.class2_alg_type = OP_ALG_ALGSEL_SHA224 |
2232 					   OP_ALG_AAI_HMAC_PRECOMP,
2233 		},
2234 	},
2235 	{
2236 		.aead = {
2237 			.base = {
2238 				.cra_name = "echainiv(authenc(hmac(sha224),"
2239 					    "cbc(des3_ede)))",
2240 				.cra_driver_name = "echainiv-authenc-"
2241 						   "hmac-sha224-"
2242 						   "cbc-des3_ede-caam-qi2",
2243 				.cra_blocksize = DES3_EDE_BLOCK_SIZE,
2244 			},
2245 			.setkey = des3_aead_setkey,
2246 			.setauthsize = aead_setauthsize,
2247 			.encrypt = aead_encrypt,
2248 			.decrypt = aead_decrypt,
2249 			.ivsize = DES3_EDE_BLOCK_SIZE,
2250 			.maxauthsize = SHA224_DIGEST_SIZE,
2251 		},
2252 		.caam = {
2253 			.class1_alg_type = OP_ALG_ALGSEL_3DES | OP_ALG_AAI_CBC,
2254 			.class2_alg_type = OP_ALG_ALGSEL_SHA224 |
2255 					   OP_ALG_AAI_HMAC_PRECOMP,
2256 			.geniv = true,
2257 		}
2258 	},
2259 	{
2260 		.aead = {
2261 			.base = {
2262 				.cra_name = "authenc(hmac(sha256),"
2263 					    "cbc(des3_ede))",
2264 				.cra_driver_name = "authenc-hmac-sha256-"
2265 						   "cbc-des3_ede-caam-qi2",
2266 				.cra_blocksize = DES3_EDE_BLOCK_SIZE,
2267 			},
2268 			.setkey = des3_aead_setkey,
2269 			.setauthsize = aead_setauthsize,
2270 			.encrypt = aead_encrypt,
2271 			.decrypt = aead_decrypt,
2272 			.ivsize = DES3_EDE_BLOCK_SIZE,
2273 			.maxauthsize = SHA256_DIGEST_SIZE,
2274 		},
2275 		.caam = {
2276 			.class1_alg_type = OP_ALG_ALGSEL_3DES | OP_ALG_AAI_CBC,
2277 			.class2_alg_type = OP_ALG_ALGSEL_SHA256 |
2278 					   OP_ALG_AAI_HMAC_PRECOMP,
2279 		},
2280 	},
2281 	{
2282 		.aead = {
2283 			.base = {
2284 				.cra_name = "echainiv(authenc(hmac(sha256),"
2285 					    "cbc(des3_ede)))",
2286 				.cra_driver_name = "echainiv-authenc-"
2287 						   "hmac-sha256-"
2288 						   "cbc-des3_ede-caam-qi2",
2289 				.cra_blocksize = DES3_EDE_BLOCK_SIZE,
2290 			},
2291 			.setkey = des3_aead_setkey,
2292 			.setauthsize = aead_setauthsize,
2293 			.encrypt = aead_encrypt,
2294 			.decrypt = aead_decrypt,
2295 			.ivsize = DES3_EDE_BLOCK_SIZE,
2296 			.maxauthsize = SHA256_DIGEST_SIZE,
2297 		},
2298 		.caam = {
2299 			.class1_alg_type = OP_ALG_ALGSEL_3DES | OP_ALG_AAI_CBC,
2300 			.class2_alg_type = OP_ALG_ALGSEL_SHA256 |
2301 					   OP_ALG_AAI_HMAC_PRECOMP,
2302 			.geniv = true,
2303 		}
2304 	},
2305 	{
2306 		.aead = {
2307 			.base = {
2308 				.cra_name = "authenc(hmac(sha384),"
2309 					    "cbc(des3_ede))",
2310 				.cra_driver_name = "authenc-hmac-sha384-"
2311 						   "cbc-des3_ede-caam-qi2",
2312 				.cra_blocksize = DES3_EDE_BLOCK_SIZE,
2313 			},
2314 			.setkey = des3_aead_setkey,
2315 			.setauthsize = aead_setauthsize,
2316 			.encrypt = aead_encrypt,
2317 			.decrypt = aead_decrypt,
2318 			.ivsize = DES3_EDE_BLOCK_SIZE,
2319 			.maxauthsize = SHA384_DIGEST_SIZE,
2320 		},
2321 		.caam = {
2322 			.class1_alg_type = OP_ALG_ALGSEL_3DES | OP_ALG_AAI_CBC,
2323 			.class2_alg_type = OP_ALG_ALGSEL_SHA384 |
2324 					   OP_ALG_AAI_HMAC_PRECOMP,
2325 		},
2326 	},
2327 	{
2328 		.aead = {
2329 			.base = {
2330 				.cra_name = "echainiv(authenc(hmac(sha384),"
2331 					    "cbc(des3_ede)))",
2332 				.cra_driver_name = "echainiv-authenc-"
2333 						   "hmac-sha384-"
2334 						   "cbc-des3_ede-caam-qi2",
2335 				.cra_blocksize = DES3_EDE_BLOCK_SIZE,
2336 			},
2337 			.setkey = des3_aead_setkey,
2338 			.setauthsize = aead_setauthsize,
2339 			.encrypt = aead_encrypt,
2340 			.decrypt = aead_decrypt,
2341 			.ivsize = DES3_EDE_BLOCK_SIZE,
2342 			.maxauthsize = SHA384_DIGEST_SIZE,
2343 		},
2344 		.caam = {
2345 			.class1_alg_type = OP_ALG_ALGSEL_3DES | OP_ALG_AAI_CBC,
2346 			.class2_alg_type = OP_ALG_ALGSEL_SHA384 |
2347 					   OP_ALG_AAI_HMAC_PRECOMP,
2348 			.geniv = true,
2349 		}
2350 	},
2351 	{
2352 		.aead = {
2353 			.base = {
2354 				.cra_name = "authenc(hmac(sha512),"
2355 					    "cbc(des3_ede))",
2356 				.cra_driver_name = "authenc-hmac-sha512-"
2357 						   "cbc-des3_ede-caam-qi2",
2358 				.cra_blocksize = DES3_EDE_BLOCK_SIZE,
2359 			},
2360 			.setkey = des3_aead_setkey,
2361 			.setauthsize = aead_setauthsize,
2362 			.encrypt = aead_encrypt,
2363 			.decrypt = aead_decrypt,
2364 			.ivsize = DES3_EDE_BLOCK_SIZE,
2365 			.maxauthsize = SHA512_DIGEST_SIZE,
2366 		},
2367 		.caam = {
2368 			.class1_alg_type = OP_ALG_ALGSEL_3DES | OP_ALG_AAI_CBC,
2369 			.class2_alg_type = OP_ALG_ALGSEL_SHA512 |
2370 					   OP_ALG_AAI_HMAC_PRECOMP,
2371 		},
2372 	},
2373 	{
2374 		.aead = {
2375 			.base = {
2376 				.cra_name = "echainiv(authenc(hmac(sha512),"
2377 					    "cbc(des3_ede)))",
2378 				.cra_driver_name = "echainiv-authenc-"
2379 						   "hmac-sha512-"
2380 						   "cbc-des3_ede-caam-qi2",
2381 				.cra_blocksize = DES3_EDE_BLOCK_SIZE,
2382 			},
2383 			.setkey = des3_aead_setkey,
2384 			.setauthsize = aead_setauthsize,
2385 			.encrypt = aead_encrypt,
2386 			.decrypt = aead_decrypt,
2387 			.ivsize = DES3_EDE_BLOCK_SIZE,
2388 			.maxauthsize = SHA512_DIGEST_SIZE,
2389 		},
2390 		.caam = {
2391 			.class1_alg_type = OP_ALG_ALGSEL_3DES | OP_ALG_AAI_CBC,
2392 			.class2_alg_type = OP_ALG_ALGSEL_SHA512 |
2393 					   OP_ALG_AAI_HMAC_PRECOMP,
2394 			.geniv = true,
2395 		}
2396 	},
2397 	{
2398 		.aead = {
2399 			.base = {
2400 				.cra_name = "authenc(hmac(md5),cbc(des))",
2401 				.cra_driver_name = "authenc-hmac-md5-"
2402 						   "cbc-des-caam-qi2",
2403 				.cra_blocksize = DES_BLOCK_SIZE,
2404 			},
2405 			.setkey = aead_setkey,
2406 			.setauthsize = aead_setauthsize,
2407 			.encrypt = aead_encrypt,
2408 			.decrypt = aead_decrypt,
2409 			.ivsize = DES_BLOCK_SIZE,
2410 			.maxauthsize = MD5_DIGEST_SIZE,
2411 		},
2412 		.caam = {
2413 			.class1_alg_type = OP_ALG_ALGSEL_DES | OP_ALG_AAI_CBC,
2414 			.class2_alg_type = OP_ALG_ALGSEL_MD5 |
2415 					   OP_ALG_AAI_HMAC_PRECOMP,
2416 		},
2417 	},
2418 	{
2419 		.aead = {
2420 			.base = {
2421 				.cra_name = "echainiv(authenc(hmac(md5),"
2422 					    "cbc(des)))",
2423 				.cra_driver_name = "echainiv-authenc-hmac-md5-"
2424 						   "cbc-des-caam-qi2",
2425 				.cra_blocksize = DES_BLOCK_SIZE,
2426 			},
2427 			.setkey = aead_setkey,
2428 			.setauthsize = aead_setauthsize,
2429 			.encrypt = aead_encrypt,
2430 			.decrypt = aead_decrypt,
2431 			.ivsize = DES_BLOCK_SIZE,
2432 			.maxauthsize = MD5_DIGEST_SIZE,
2433 		},
2434 		.caam = {
2435 			.class1_alg_type = OP_ALG_ALGSEL_DES | OP_ALG_AAI_CBC,
2436 			.class2_alg_type = OP_ALG_ALGSEL_MD5 |
2437 					   OP_ALG_AAI_HMAC_PRECOMP,
2438 			.geniv = true,
2439 		}
2440 	},
2441 	{
2442 		.aead = {
2443 			.base = {
2444 				.cra_name = "authenc(hmac(sha1),cbc(des))",
2445 				.cra_driver_name = "authenc-hmac-sha1-"
2446 						   "cbc-des-caam-qi2",
2447 				.cra_blocksize = DES_BLOCK_SIZE,
2448 			},
2449 			.setkey = aead_setkey,
2450 			.setauthsize = aead_setauthsize,
2451 			.encrypt = aead_encrypt,
2452 			.decrypt = aead_decrypt,
2453 			.ivsize = DES_BLOCK_SIZE,
2454 			.maxauthsize = SHA1_DIGEST_SIZE,
2455 		},
2456 		.caam = {
2457 			.class1_alg_type = OP_ALG_ALGSEL_DES | OP_ALG_AAI_CBC,
2458 			.class2_alg_type = OP_ALG_ALGSEL_SHA1 |
2459 					   OP_ALG_AAI_HMAC_PRECOMP,
2460 		},
2461 	},
2462 	{
2463 		.aead = {
2464 			.base = {
2465 				.cra_name = "echainiv(authenc(hmac(sha1),"
2466 					    "cbc(des)))",
2467 				.cra_driver_name = "echainiv-authenc-"
2468 						   "hmac-sha1-cbc-des-caam-qi2",
2469 				.cra_blocksize = DES_BLOCK_SIZE,
2470 			},
2471 			.setkey = aead_setkey,
2472 			.setauthsize = aead_setauthsize,
2473 			.encrypt = aead_encrypt,
2474 			.decrypt = aead_decrypt,
2475 			.ivsize = DES_BLOCK_SIZE,
2476 			.maxauthsize = SHA1_DIGEST_SIZE,
2477 		},
2478 		.caam = {
2479 			.class1_alg_type = OP_ALG_ALGSEL_DES | OP_ALG_AAI_CBC,
2480 			.class2_alg_type = OP_ALG_ALGSEL_SHA1 |
2481 					   OP_ALG_AAI_HMAC_PRECOMP,
2482 			.geniv = true,
2483 		}
2484 	},
2485 	{
2486 		.aead = {
2487 			.base = {
2488 				.cra_name = "authenc(hmac(sha224),cbc(des))",
2489 				.cra_driver_name = "authenc-hmac-sha224-"
2490 						   "cbc-des-caam-qi2",
2491 				.cra_blocksize = DES_BLOCK_SIZE,
2492 			},
2493 			.setkey = aead_setkey,
2494 			.setauthsize = aead_setauthsize,
2495 			.encrypt = aead_encrypt,
2496 			.decrypt = aead_decrypt,
2497 			.ivsize = DES_BLOCK_SIZE,
2498 			.maxauthsize = SHA224_DIGEST_SIZE,
2499 		},
2500 		.caam = {
2501 			.class1_alg_type = OP_ALG_ALGSEL_DES | OP_ALG_AAI_CBC,
2502 			.class2_alg_type = OP_ALG_ALGSEL_SHA224 |
2503 					   OP_ALG_AAI_HMAC_PRECOMP,
2504 		},
2505 	},
2506 	{
2507 		.aead = {
2508 			.base = {
2509 				.cra_name = "echainiv(authenc(hmac(sha224),"
2510 					    "cbc(des)))",
2511 				.cra_driver_name = "echainiv-authenc-"
2512 						   "hmac-sha224-cbc-des-"
2513 						   "caam-qi2",
2514 				.cra_blocksize = DES_BLOCK_SIZE,
2515 			},
2516 			.setkey = aead_setkey,
2517 			.setauthsize = aead_setauthsize,
2518 			.encrypt = aead_encrypt,
2519 			.decrypt = aead_decrypt,
2520 			.ivsize = DES_BLOCK_SIZE,
2521 			.maxauthsize = SHA224_DIGEST_SIZE,
2522 		},
2523 		.caam = {
2524 			.class1_alg_type = OP_ALG_ALGSEL_DES | OP_ALG_AAI_CBC,
2525 			.class2_alg_type = OP_ALG_ALGSEL_SHA224 |
2526 					   OP_ALG_AAI_HMAC_PRECOMP,
2527 			.geniv = true,
2528 		}
2529 	},
2530 	{
2531 		.aead = {
2532 			.base = {
2533 				.cra_name = "authenc(hmac(sha256),cbc(des))",
2534 				.cra_driver_name = "authenc-hmac-sha256-"
2535 						   "cbc-des-caam-qi2",
2536 				.cra_blocksize = DES_BLOCK_SIZE,
2537 			},
2538 			.setkey = aead_setkey,
2539 			.setauthsize = aead_setauthsize,
2540 			.encrypt = aead_encrypt,
2541 			.decrypt = aead_decrypt,
2542 			.ivsize = DES_BLOCK_SIZE,
2543 			.maxauthsize = SHA256_DIGEST_SIZE,
2544 		},
2545 		.caam = {
2546 			.class1_alg_type = OP_ALG_ALGSEL_DES | OP_ALG_AAI_CBC,
2547 			.class2_alg_type = OP_ALG_ALGSEL_SHA256 |
2548 					   OP_ALG_AAI_HMAC_PRECOMP,
2549 		},
2550 	},
2551 	{
2552 		.aead = {
2553 			.base = {
2554 				.cra_name = "echainiv(authenc(hmac(sha256),"
2555 					    "cbc(des)))",
2556 				.cra_driver_name = "echainiv-authenc-"
2557 						   "hmac-sha256-cbc-des-"
2558 						   "caam-qi2",
2559 				.cra_blocksize = DES_BLOCK_SIZE,
2560 			},
2561 			.setkey = aead_setkey,
2562 			.setauthsize = aead_setauthsize,
2563 			.encrypt = aead_encrypt,
2564 			.decrypt = aead_decrypt,
2565 			.ivsize = DES_BLOCK_SIZE,
2566 			.maxauthsize = SHA256_DIGEST_SIZE,
2567 		},
2568 		.caam = {
2569 			.class1_alg_type = OP_ALG_ALGSEL_DES | OP_ALG_AAI_CBC,
2570 			.class2_alg_type = OP_ALG_ALGSEL_SHA256 |
2571 					   OP_ALG_AAI_HMAC_PRECOMP,
2572 			.geniv = true,
2573 		},
2574 	},
2575 	{
2576 		.aead = {
2577 			.base = {
2578 				.cra_name = "authenc(hmac(sha384),cbc(des))",
2579 				.cra_driver_name = "authenc-hmac-sha384-"
2580 						   "cbc-des-caam-qi2",
2581 				.cra_blocksize = DES_BLOCK_SIZE,
2582 			},
2583 			.setkey = aead_setkey,
2584 			.setauthsize = aead_setauthsize,
2585 			.encrypt = aead_encrypt,
2586 			.decrypt = aead_decrypt,
2587 			.ivsize = DES_BLOCK_SIZE,
2588 			.maxauthsize = SHA384_DIGEST_SIZE,
2589 		},
2590 		.caam = {
2591 			.class1_alg_type = OP_ALG_ALGSEL_DES | OP_ALG_AAI_CBC,
2592 			.class2_alg_type = OP_ALG_ALGSEL_SHA384 |
2593 					   OP_ALG_AAI_HMAC_PRECOMP,
2594 		},
2595 	},
2596 	{
2597 		.aead = {
2598 			.base = {
2599 				.cra_name = "echainiv(authenc(hmac(sha384),"
2600 					    "cbc(des)))",
2601 				.cra_driver_name = "echainiv-authenc-"
2602 						   "hmac-sha384-cbc-des-"
2603 						   "caam-qi2",
2604 				.cra_blocksize = DES_BLOCK_SIZE,
2605 			},
2606 			.setkey = aead_setkey,
2607 			.setauthsize = aead_setauthsize,
2608 			.encrypt = aead_encrypt,
2609 			.decrypt = aead_decrypt,
2610 			.ivsize = DES_BLOCK_SIZE,
2611 			.maxauthsize = SHA384_DIGEST_SIZE,
2612 		},
2613 		.caam = {
2614 			.class1_alg_type = OP_ALG_ALGSEL_DES | OP_ALG_AAI_CBC,
2615 			.class2_alg_type = OP_ALG_ALGSEL_SHA384 |
2616 					   OP_ALG_AAI_HMAC_PRECOMP,
2617 			.geniv = true,
2618 		}
2619 	},
2620 	{
2621 		.aead = {
2622 			.base = {
2623 				.cra_name = "authenc(hmac(sha512),cbc(des))",
2624 				.cra_driver_name = "authenc-hmac-sha512-"
2625 						   "cbc-des-caam-qi2",
2626 				.cra_blocksize = DES_BLOCK_SIZE,
2627 			},
2628 			.setkey = aead_setkey,
2629 			.setauthsize = aead_setauthsize,
2630 			.encrypt = aead_encrypt,
2631 			.decrypt = aead_decrypt,
2632 			.ivsize = DES_BLOCK_SIZE,
2633 			.maxauthsize = SHA512_DIGEST_SIZE,
2634 		},
2635 		.caam = {
2636 			.class1_alg_type = OP_ALG_ALGSEL_DES | OP_ALG_AAI_CBC,
2637 			.class2_alg_type = OP_ALG_ALGSEL_SHA512 |
2638 					   OP_ALG_AAI_HMAC_PRECOMP,
2639 		}
2640 	},
2641 	{
2642 		.aead = {
2643 			.base = {
2644 				.cra_name = "echainiv(authenc(hmac(sha512),"
2645 					    "cbc(des)))",
2646 				.cra_driver_name = "echainiv-authenc-"
2647 						   "hmac-sha512-cbc-des-"
2648 						   "caam-qi2",
2649 				.cra_blocksize = DES_BLOCK_SIZE,
2650 			},
2651 			.setkey = aead_setkey,
2652 			.setauthsize = aead_setauthsize,
2653 			.encrypt = aead_encrypt,
2654 			.decrypt = aead_decrypt,
2655 			.ivsize = DES_BLOCK_SIZE,
2656 			.maxauthsize = SHA512_DIGEST_SIZE,
2657 		},
2658 		.caam = {
2659 			.class1_alg_type = OP_ALG_ALGSEL_DES | OP_ALG_AAI_CBC,
2660 			.class2_alg_type = OP_ALG_ALGSEL_SHA512 |
2661 					   OP_ALG_AAI_HMAC_PRECOMP,
2662 			.geniv = true,
2663 		}
2664 	},
2665 	{
2666 		.aead = {
2667 			.base = {
2668 				.cra_name = "authenc(hmac(md5),"
2669 					    "rfc3686(ctr(aes)))",
2670 				.cra_driver_name = "authenc-hmac-md5-"
2671 						   "rfc3686-ctr-aes-caam-qi2",
2672 				.cra_blocksize = 1,
2673 			},
2674 			.setkey = aead_setkey,
2675 			.setauthsize = aead_setauthsize,
2676 			.encrypt = aead_encrypt,
2677 			.decrypt = aead_decrypt,
2678 			.ivsize = CTR_RFC3686_IV_SIZE,
2679 			.maxauthsize = MD5_DIGEST_SIZE,
2680 		},
2681 		.caam = {
2682 			.class1_alg_type = OP_ALG_ALGSEL_AES |
2683 					   OP_ALG_AAI_CTR_MOD128,
2684 			.class2_alg_type = OP_ALG_ALGSEL_MD5 |
2685 					   OP_ALG_AAI_HMAC_PRECOMP,
2686 			.rfc3686 = true,
2687 		},
2688 	},
2689 	{
2690 		.aead = {
2691 			.base = {
2692 				.cra_name = "seqiv(authenc("
2693 					    "hmac(md5),rfc3686(ctr(aes))))",
2694 				.cra_driver_name = "seqiv-authenc-hmac-md5-"
2695 						   "rfc3686-ctr-aes-caam-qi2",
2696 				.cra_blocksize = 1,
2697 			},
2698 			.setkey = aead_setkey,
2699 			.setauthsize = aead_setauthsize,
2700 			.encrypt = aead_encrypt,
2701 			.decrypt = aead_decrypt,
2702 			.ivsize = CTR_RFC3686_IV_SIZE,
2703 			.maxauthsize = MD5_DIGEST_SIZE,
2704 		},
2705 		.caam = {
2706 			.class1_alg_type = OP_ALG_ALGSEL_AES |
2707 					   OP_ALG_AAI_CTR_MOD128,
2708 			.class2_alg_type = OP_ALG_ALGSEL_MD5 |
2709 					   OP_ALG_AAI_HMAC_PRECOMP,
2710 			.rfc3686 = true,
2711 			.geniv = true,
2712 		},
2713 	},
2714 	{
2715 		.aead = {
2716 			.base = {
2717 				.cra_name = "authenc(hmac(sha1),"
2718 					    "rfc3686(ctr(aes)))",
2719 				.cra_driver_name = "authenc-hmac-sha1-"
2720 						   "rfc3686-ctr-aes-caam-qi2",
2721 				.cra_blocksize = 1,
2722 			},
2723 			.setkey = aead_setkey,
2724 			.setauthsize = aead_setauthsize,
2725 			.encrypt = aead_encrypt,
2726 			.decrypt = aead_decrypt,
2727 			.ivsize = CTR_RFC3686_IV_SIZE,
2728 			.maxauthsize = SHA1_DIGEST_SIZE,
2729 		},
2730 		.caam = {
2731 			.class1_alg_type = OP_ALG_ALGSEL_AES |
2732 					   OP_ALG_AAI_CTR_MOD128,
2733 			.class2_alg_type = OP_ALG_ALGSEL_SHA1 |
2734 					   OP_ALG_AAI_HMAC_PRECOMP,
2735 			.rfc3686 = true,
2736 		},
2737 	},
2738 	{
2739 		.aead = {
2740 			.base = {
2741 				.cra_name = "seqiv(authenc("
2742 					    "hmac(sha1),rfc3686(ctr(aes))))",
2743 				.cra_driver_name = "seqiv-authenc-hmac-sha1-"
2744 						   "rfc3686-ctr-aes-caam-qi2",
2745 				.cra_blocksize = 1,
2746 			},
2747 			.setkey = aead_setkey,
2748 			.setauthsize = aead_setauthsize,
2749 			.encrypt = aead_encrypt,
2750 			.decrypt = aead_decrypt,
2751 			.ivsize = CTR_RFC3686_IV_SIZE,
2752 			.maxauthsize = SHA1_DIGEST_SIZE,
2753 		},
2754 		.caam = {
2755 			.class1_alg_type = OP_ALG_ALGSEL_AES |
2756 					   OP_ALG_AAI_CTR_MOD128,
2757 			.class2_alg_type = OP_ALG_ALGSEL_SHA1 |
2758 					   OP_ALG_AAI_HMAC_PRECOMP,
2759 			.rfc3686 = true,
2760 			.geniv = true,
2761 		},
2762 	},
2763 	{
2764 		.aead = {
2765 			.base = {
2766 				.cra_name = "authenc(hmac(sha224),"
2767 					    "rfc3686(ctr(aes)))",
2768 				.cra_driver_name = "authenc-hmac-sha224-"
2769 						   "rfc3686-ctr-aes-caam-qi2",
2770 				.cra_blocksize = 1,
2771 			},
2772 			.setkey = aead_setkey,
2773 			.setauthsize = aead_setauthsize,
2774 			.encrypt = aead_encrypt,
2775 			.decrypt = aead_decrypt,
2776 			.ivsize = CTR_RFC3686_IV_SIZE,
2777 			.maxauthsize = SHA224_DIGEST_SIZE,
2778 		},
2779 		.caam = {
2780 			.class1_alg_type = OP_ALG_ALGSEL_AES |
2781 					   OP_ALG_AAI_CTR_MOD128,
2782 			.class2_alg_type = OP_ALG_ALGSEL_SHA224 |
2783 					   OP_ALG_AAI_HMAC_PRECOMP,
2784 			.rfc3686 = true,
2785 		},
2786 	},
2787 	{
2788 		.aead = {
2789 			.base = {
2790 				.cra_name = "seqiv(authenc("
2791 					    "hmac(sha224),rfc3686(ctr(aes))))",
2792 				.cra_driver_name = "seqiv-authenc-hmac-sha224-"
2793 						   "rfc3686-ctr-aes-caam-qi2",
2794 				.cra_blocksize = 1,
2795 			},
2796 			.setkey = aead_setkey,
2797 			.setauthsize = aead_setauthsize,
2798 			.encrypt = aead_encrypt,
2799 			.decrypt = aead_decrypt,
2800 			.ivsize = CTR_RFC3686_IV_SIZE,
2801 			.maxauthsize = SHA224_DIGEST_SIZE,
2802 		},
2803 		.caam = {
2804 			.class1_alg_type = OP_ALG_ALGSEL_AES |
2805 					   OP_ALG_AAI_CTR_MOD128,
2806 			.class2_alg_type = OP_ALG_ALGSEL_SHA224 |
2807 					   OP_ALG_AAI_HMAC_PRECOMP,
2808 			.rfc3686 = true,
2809 			.geniv = true,
2810 		},
2811 	},
2812 	{
2813 		.aead = {
2814 			.base = {
2815 				.cra_name = "authenc(hmac(sha256),"
2816 					    "rfc3686(ctr(aes)))",
2817 				.cra_driver_name = "authenc-hmac-sha256-"
2818 						   "rfc3686-ctr-aes-caam-qi2",
2819 				.cra_blocksize = 1,
2820 			},
2821 			.setkey = aead_setkey,
2822 			.setauthsize = aead_setauthsize,
2823 			.encrypt = aead_encrypt,
2824 			.decrypt = aead_decrypt,
2825 			.ivsize = CTR_RFC3686_IV_SIZE,
2826 			.maxauthsize = SHA256_DIGEST_SIZE,
2827 		},
2828 		.caam = {
2829 			.class1_alg_type = OP_ALG_ALGSEL_AES |
2830 					   OP_ALG_AAI_CTR_MOD128,
2831 			.class2_alg_type = OP_ALG_ALGSEL_SHA256 |
2832 					   OP_ALG_AAI_HMAC_PRECOMP,
2833 			.rfc3686 = true,
2834 		},
2835 	},
2836 	{
2837 		.aead = {
2838 			.base = {
2839 				.cra_name = "seqiv(authenc(hmac(sha256),"
2840 					    "rfc3686(ctr(aes))))",
2841 				.cra_driver_name = "seqiv-authenc-hmac-sha256-"
2842 						   "rfc3686-ctr-aes-caam-qi2",
2843 				.cra_blocksize = 1,
2844 			},
2845 			.setkey = aead_setkey,
2846 			.setauthsize = aead_setauthsize,
2847 			.encrypt = aead_encrypt,
2848 			.decrypt = aead_decrypt,
2849 			.ivsize = CTR_RFC3686_IV_SIZE,
2850 			.maxauthsize = SHA256_DIGEST_SIZE,
2851 		},
2852 		.caam = {
2853 			.class1_alg_type = OP_ALG_ALGSEL_AES |
2854 					   OP_ALG_AAI_CTR_MOD128,
2855 			.class2_alg_type = OP_ALG_ALGSEL_SHA256 |
2856 					   OP_ALG_AAI_HMAC_PRECOMP,
2857 			.rfc3686 = true,
2858 			.geniv = true,
2859 		},
2860 	},
2861 	{
2862 		.aead = {
2863 			.base = {
2864 				.cra_name = "authenc(hmac(sha384),"
2865 					    "rfc3686(ctr(aes)))",
2866 				.cra_driver_name = "authenc-hmac-sha384-"
2867 						   "rfc3686-ctr-aes-caam-qi2",
2868 				.cra_blocksize = 1,
2869 			},
2870 			.setkey = aead_setkey,
2871 			.setauthsize = aead_setauthsize,
2872 			.encrypt = aead_encrypt,
2873 			.decrypt = aead_decrypt,
2874 			.ivsize = CTR_RFC3686_IV_SIZE,
2875 			.maxauthsize = SHA384_DIGEST_SIZE,
2876 		},
2877 		.caam = {
2878 			.class1_alg_type = OP_ALG_ALGSEL_AES |
2879 					   OP_ALG_AAI_CTR_MOD128,
2880 			.class2_alg_type = OP_ALG_ALGSEL_SHA384 |
2881 					   OP_ALG_AAI_HMAC_PRECOMP,
2882 			.rfc3686 = true,
2883 		},
2884 	},
2885 	{
2886 		.aead = {
2887 			.base = {
2888 				.cra_name = "seqiv(authenc(hmac(sha384),"
2889 					    "rfc3686(ctr(aes))))",
2890 				.cra_driver_name = "seqiv-authenc-hmac-sha384-"
2891 						   "rfc3686-ctr-aes-caam-qi2",
2892 				.cra_blocksize = 1,
2893 			},
2894 			.setkey = aead_setkey,
2895 			.setauthsize = aead_setauthsize,
2896 			.encrypt = aead_encrypt,
2897 			.decrypt = aead_decrypt,
2898 			.ivsize = CTR_RFC3686_IV_SIZE,
2899 			.maxauthsize = SHA384_DIGEST_SIZE,
2900 		},
2901 		.caam = {
2902 			.class1_alg_type = OP_ALG_ALGSEL_AES |
2903 					   OP_ALG_AAI_CTR_MOD128,
2904 			.class2_alg_type = OP_ALG_ALGSEL_SHA384 |
2905 					   OP_ALG_AAI_HMAC_PRECOMP,
2906 			.rfc3686 = true,
2907 			.geniv = true,
2908 		},
2909 	},
2910 	{
2911 		.aead = {
2912 			.base = {
2913 				.cra_name = "rfc7539(chacha20,poly1305)",
2914 				.cra_driver_name = "rfc7539-chacha20-poly1305-"
2915 						   "caam-qi2",
2916 				.cra_blocksize = 1,
2917 			},
2918 			.setkey = chachapoly_setkey,
2919 			.setauthsize = chachapoly_setauthsize,
2920 			.encrypt = aead_encrypt,
2921 			.decrypt = aead_decrypt,
2922 			.ivsize = CHACHAPOLY_IV_SIZE,
2923 			.maxauthsize = POLY1305_DIGEST_SIZE,
2924 		},
2925 		.caam = {
2926 			.class1_alg_type = OP_ALG_ALGSEL_CHACHA20 |
2927 					   OP_ALG_AAI_AEAD,
2928 			.class2_alg_type = OP_ALG_ALGSEL_POLY1305 |
2929 					   OP_ALG_AAI_AEAD,
2930 			.nodkp = true,
2931 		},
2932 	},
2933 	{
2934 		.aead = {
2935 			.base = {
2936 				.cra_name = "rfc7539esp(chacha20,poly1305)",
2937 				.cra_driver_name = "rfc7539esp-chacha20-"
2938 						   "poly1305-caam-qi2",
2939 				.cra_blocksize = 1,
2940 			},
2941 			.setkey = chachapoly_setkey,
2942 			.setauthsize = chachapoly_setauthsize,
2943 			.encrypt = aead_encrypt,
2944 			.decrypt = aead_decrypt,
2945 			.ivsize = 8,
2946 			.maxauthsize = POLY1305_DIGEST_SIZE,
2947 		},
2948 		.caam = {
2949 			.class1_alg_type = OP_ALG_ALGSEL_CHACHA20 |
2950 					   OP_ALG_AAI_AEAD,
2951 			.class2_alg_type = OP_ALG_ALGSEL_POLY1305 |
2952 					   OP_ALG_AAI_AEAD,
2953 			.nodkp = true,
2954 		},
2955 	},
2956 	{
2957 		.aead = {
2958 			.base = {
2959 				.cra_name = "authenc(hmac(sha512),"
2960 					    "rfc3686(ctr(aes)))",
2961 				.cra_driver_name = "authenc-hmac-sha512-"
2962 						   "rfc3686-ctr-aes-caam-qi2",
2963 				.cra_blocksize = 1,
2964 			},
2965 			.setkey = aead_setkey,
2966 			.setauthsize = aead_setauthsize,
2967 			.encrypt = aead_encrypt,
2968 			.decrypt = aead_decrypt,
2969 			.ivsize = CTR_RFC3686_IV_SIZE,
2970 			.maxauthsize = SHA512_DIGEST_SIZE,
2971 		},
2972 		.caam = {
2973 			.class1_alg_type = OP_ALG_ALGSEL_AES |
2974 					   OP_ALG_AAI_CTR_MOD128,
2975 			.class2_alg_type = OP_ALG_ALGSEL_SHA512 |
2976 					   OP_ALG_AAI_HMAC_PRECOMP,
2977 			.rfc3686 = true,
2978 		},
2979 	},
2980 	{
2981 		.aead = {
2982 			.base = {
2983 				.cra_name = "seqiv(authenc(hmac(sha512),"
2984 					    "rfc3686(ctr(aes))))",
2985 				.cra_driver_name = "seqiv-authenc-hmac-sha512-"
2986 						   "rfc3686-ctr-aes-caam-qi2",
2987 				.cra_blocksize = 1,
2988 			},
2989 			.setkey = aead_setkey,
2990 			.setauthsize = aead_setauthsize,
2991 			.encrypt = aead_encrypt,
2992 			.decrypt = aead_decrypt,
2993 			.ivsize = CTR_RFC3686_IV_SIZE,
2994 			.maxauthsize = SHA512_DIGEST_SIZE,
2995 		},
2996 		.caam = {
2997 			.class1_alg_type = OP_ALG_ALGSEL_AES |
2998 					   OP_ALG_AAI_CTR_MOD128,
2999 			.class2_alg_type = OP_ALG_ALGSEL_SHA512 |
3000 					   OP_ALG_AAI_HMAC_PRECOMP,
3001 			.rfc3686 = true,
3002 			.geniv = true,
3003 		},
3004 	},
3005 };
3006 
3007 static void caam_skcipher_alg_init(struct caam_skcipher_alg *t_alg)
3008 {
3009 	struct skcipher_alg *alg = &t_alg->skcipher;
3010 
3011 	alg->base.cra_module = THIS_MODULE;
3012 	alg->base.cra_priority = CAAM_CRA_PRIORITY;
3013 	alg->base.cra_ctxsize = sizeof(struct caam_ctx) + crypto_dma_padding();
3014 	alg->base.cra_flags |= (CRYPTO_ALG_ASYNC | CRYPTO_ALG_ALLOCATES_MEMORY |
3015 			      CRYPTO_ALG_KERN_DRIVER_ONLY);
3016 
3017 	alg->init = caam_cra_init_skcipher;
3018 	alg->exit = caam_cra_exit;
3019 }
3020 
3021 static void caam_aead_alg_init(struct caam_aead_alg *t_alg)
3022 {
3023 	struct aead_alg *alg = &t_alg->aead;
3024 
3025 	alg->base.cra_module = THIS_MODULE;
3026 	alg->base.cra_priority = CAAM_CRA_PRIORITY;
3027 	alg->base.cra_ctxsize = sizeof(struct caam_ctx) + crypto_dma_padding();
3028 	alg->base.cra_flags = CRYPTO_ALG_ASYNC | CRYPTO_ALG_ALLOCATES_MEMORY |
3029 			      CRYPTO_ALG_KERN_DRIVER_ONLY;
3030 
3031 	alg->init = caam_cra_init_aead;
3032 	alg->exit = caam_cra_exit_aead;
3033 }
3034 
3035 /* max hash key is max split key size */
3036 #define CAAM_MAX_HASH_KEY_SIZE		(SHA512_DIGEST_SIZE * 2)
3037 
3038 #define CAAM_MAX_HASH_BLOCK_SIZE	SHA512_BLOCK_SIZE
3039 
3040 /* caam context sizes for hashes: running digest + 8 */
3041 #define HASH_MSG_LEN			8
3042 #define MAX_CTX_LEN			(HASH_MSG_LEN + SHA512_DIGEST_SIZE)
3043 
3044 enum hash_optype {
3045 	UPDATE = 0,
3046 	UPDATE_FIRST,
3047 	FINALIZE,
3048 	DIGEST,
3049 	HASH_NUM_OP
3050 };
3051 
3052 /**
3053  * struct caam_hash_ctx - ahash per-session context
3054  * @flc: Flow Contexts array
3055  * @key: authentication key
3056  * @flc_dma: I/O virtual addresses of the Flow Contexts
3057  * @dev: dpseci device
3058  * @ctx_len: size of Context Register
3059  * @adata: hashing algorithm details
3060  */
3061 struct caam_hash_ctx {
3062 	struct caam_flc flc[HASH_NUM_OP];
3063 	u8 key[CAAM_MAX_HASH_BLOCK_SIZE] ____cacheline_aligned;
3064 	dma_addr_t flc_dma[HASH_NUM_OP];
3065 	struct device *dev;
3066 	int ctx_len;
3067 	struct alginfo adata;
3068 };
3069 
3070 /* ahash state */
3071 struct caam_hash_state {
3072 	struct caam_request caam_req;
3073 	dma_addr_t buf_dma;
3074 	dma_addr_t ctx_dma;
3075 	int ctx_dma_len;
3076 	u8 buf[CAAM_MAX_HASH_BLOCK_SIZE] ____cacheline_aligned;
3077 	int buflen;
3078 	int next_buflen;
3079 	u8 caam_ctx[MAX_CTX_LEN] ____cacheline_aligned;
3080 	int (*update)(struct ahash_request *req);
3081 	int (*final)(struct ahash_request *req);
3082 	int (*finup)(struct ahash_request *req);
3083 };
3084 
3085 struct caam_export_state {
3086 	u8 buf[CAAM_MAX_HASH_BLOCK_SIZE];
3087 	u8 caam_ctx[MAX_CTX_LEN];
3088 	int buflen;
3089 	int (*update)(struct ahash_request *req);
3090 	int (*final)(struct ahash_request *req);
3091 	int (*finup)(struct ahash_request *req);
3092 };
3093 
3094 /* Map current buffer in state (if length > 0) and put it in link table */
3095 static inline int buf_map_to_qm_sg(struct device *dev,
3096 				   struct dpaa2_sg_entry *qm_sg,
3097 				   struct caam_hash_state *state)
3098 {
3099 	int buflen = state->buflen;
3100 
3101 	if (!buflen)
3102 		return 0;
3103 
3104 	state->buf_dma = dma_map_single(dev, state->buf, buflen,
3105 					DMA_TO_DEVICE);
3106 	if (dma_mapping_error(dev, state->buf_dma)) {
3107 		dev_err(dev, "unable to map buf\n");
3108 		state->buf_dma = 0;
3109 		return -ENOMEM;
3110 	}
3111 
3112 	dma_to_qm_sg_one(qm_sg, state->buf_dma, buflen, 0);
3113 
3114 	return 0;
3115 }
3116 
3117 /* Map state->caam_ctx, and add it to link table */
3118 static inline int ctx_map_to_qm_sg(struct device *dev,
3119 				   struct caam_hash_state *state, int ctx_len,
3120 				   struct dpaa2_sg_entry *qm_sg, u32 flag)
3121 {
3122 	state->ctx_dma_len = ctx_len;
3123 	state->ctx_dma = dma_map_single(dev, state->caam_ctx, ctx_len, flag);
3124 	if (dma_mapping_error(dev, state->ctx_dma)) {
3125 		dev_err(dev, "unable to map ctx\n");
3126 		state->ctx_dma = 0;
3127 		return -ENOMEM;
3128 	}
3129 
3130 	dma_to_qm_sg_one(qm_sg, state->ctx_dma, ctx_len, 0);
3131 
3132 	return 0;
3133 }
3134 
3135 static int ahash_set_sh_desc(struct crypto_ahash *ahash)
3136 {
3137 	struct caam_hash_ctx *ctx = crypto_ahash_ctx_dma(ahash);
3138 	int digestsize = crypto_ahash_digestsize(ahash);
3139 	struct dpaa2_caam_priv *priv = dev_get_drvdata(ctx->dev);
3140 	struct caam_flc *flc;
3141 	u32 *desc;
3142 
3143 	/* ahash_update shared descriptor */
3144 	flc = &ctx->flc[UPDATE];
3145 	desc = flc->sh_desc;
3146 	cnstr_shdsc_ahash(desc, &ctx->adata, OP_ALG_AS_UPDATE, ctx->ctx_len,
3147 			  ctx->ctx_len, true, priv->sec_attr.era);
3148 	flc->flc[1] = cpu_to_caam32(desc_len(desc)); /* SDL */
3149 	dma_sync_single_for_device(ctx->dev, ctx->flc_dma[UPDATE],
3150 				   desc_bytes(desc), DMA_BIDIRECTIONAL);
3151 	print_hex_dump_debug("ahash update shdesc@" __stringify(__LINE__)": ",
3152 			     DUMP_PREFIX_ADDRESS, 16, 4, desc, desc_bytes(desc),
3153 			     1);
3154 
3155 	/* ahash_update_first shared descriptor */
3156 	flc = &ctx->flc[UPDATE_FIRST];
3157 	desc = flc->sh_desc;
3158 	cnstr_shdsc_ahash(desc, &ctx->adata, OP_ALG_AS_INIT, ctx->ctx_len,
3159 			  ctx->ctx_len, false, priv->sec_attr.era);
3160 	flc->flc[1] = cpu_to_caam32(desc_len(desc)); /* SDL */
3161 	dma_sync_single_for_device(ctx->dev, ctx->flc_dma[UPDATE_FIRST],
3162 				   desc_bytes(desc), DMA_BIDIRECTIONAL);
3163 	print_hex_dump_debug("ahash update first shdesc@" __stringify(__LINE__)": ",
3164 			     DUMP_PREFIX_ADDRESS, 16, 4, desc, desc_bytes(desc),
3165 			     1);
3166 
3167 	/* ahash_final shared descriptor */
3168 	flc = &ctx->flc[FINALIZE];
3169 	desc = flc->sh_desc;
3170 	cnstr_shdsc_ahash(desc, &ctx->adata, OP_ALG_AS_FINALIZE, digestsize,
3171 			  ctx->ctx_len, true, priv->sec_attr.era);
3172 	flc->flc[1] = cpu_to_caam32(desc_len(desc)); /* SDL */
3173 	dma_sync_single_for_device(ctx->dev, ctx->flc_dma[FINALIZE],
3174 				   desc_bytes(desc), DMA_BIDIRECTIONAL);
3175 	print_hex_dump_debug("ahash final shdesc@" __stringify(__LINE__)": ",
3176 			     DUMP_PREFIX_ADDRESS, 16, 4, desc, desc_bytes(desc),
3177 			     1);
3178 
3179 	/* ahash_digest shared descriptor */
3180 	flc = &ctx->flc[DIGEST];
3181 	desc = flc->sh_desc;
3182 	cnstr_shdsc_ahash(desc, &ctx->adata, OP_ALG_AS_INITFINAL, digestsize,
3183 			  ctx->ctx_len, false, priv->sec_attr.era);
3184 	flc->flc[1] = cpu_to_caam32(desc_len(desc)); /* SDL */
3185 	dma_sync_single_for_device(ctx->dev, ctx->flc_dma[DIGEST],
3186 				   desc_bytes(desc), DMA_BIDIRECTIONAL);
3187 	print_hex_dump_debug("ahash digest shdesc@" __stringify(__LINE__)": ",
3188 			     DUMP_PREFIX_ADDRESS, 16, 4, desc, desc_bytes(desc),
3189 			     1);
3190 
3191 	return 0;
3192 }
3193 
3194 struct split_key_sh_result {
3195 	struct completion completion;
3196 	int err;
3197 	struct device *dev;
3198 };
3199 
3200 static void split_key_sh_done(void *cbk_ctx, u32 err)
3201 {
3202 	struct split_key_sh_result *res = cbk_ctx;
3203 
3204 	dev_dbg(res->dev, "%s %d: err 0x%x\n", __func__, __LINE__, err);
3205 
3206 	res->err = err ? caam_qi2_strstatus(res->dev, err) : 0;
3207 	complete(&res->completion);
3208 }
3209 
3210 /* Digest hash size if it is too large */
3211 static int hash_digest_key(struct caam_hash_ctx *ctx, u32 *keylen, u8 *key,
3212 			   u32 digestsize)
3213 {
3214 	struct caam_request *req_ctx;
3215 	u32 *desc;
3216 	struct split_key_sh_result result;
3217 	dma_addr_t key_dma;
3218 	struct caam_flc *flc;
3219 	dma_addr_t flc_dma;
3220 	int ret = -ENOMEM;
3221 	struct dpaa2_fl_entry *in_fle, *out_fle;
3222 
3223 	req_ctx = kzalloc(sizeof(*req_ctx), GFP_KERNEL | GFP_DMA);
3224 	if (!req_ctx)
3225 		return -ENOMEM;
3226 
3227 	in_fle = &req_ctx->fd_flt[1];
3228 	out_fle = &req_ctx->fd_flt[0];
3229 
3230 	flc = kzalloc(sizeof(*flc), GFP_KERNEL | GFP_DMA);
3231 	if (!flc)
3232 		goto err_flc;
3233 
3234 	key_dma = dma_map_single(ctx->dev, key, *keylen, DMA_BIDIRECTIONAL);
3235 	if (dma_mapping_error(ctx->dev, key_dma)) {
3236 		dev_err(ctx->dev, "unable to map key memory\n");
3237 		goto err_key_dma;
3238 	}
3239 
3240 	desc = flc->sh_desc;
3241 
3242 	init_sh_desc(desc, 0);
3243 
3244 	/* descriptor to perform unkeyed hash on key_in */
3245 	append_operation(desc, ctx->adata.algtype | OP_ALG_ENCRYPT |
3246 			 OP_ALG_AS_INITFINAL);
3247 	append_seq_fifo_load(desc, *keylen, FIFOLD_CLASS_CLASS2 |
3248 			     FIFOLD_TYPE_LAST2 | FIFOLD_TYPE_MSG);
3249 	append_seq_store(desc, digestsize, LDST_CLASS_2_CCB |
3250 			 LDST_SRCDST_BYTE_CONTEXT);
3251 
3252 	flc->flc[1] = cpu_to_caam32(desc_len(desc)); /* SDL */
3253 	flc_dma = dma_map_single(ctx->dev, flc, sizeof(flc->flc) +
3254 				 desc_bytes(desc), DMA_TO_DEVICE);
3255 	if (dma_mapping_error(ctx->dev, flc_dma)) {
3256 		dev_err(ctx->dev, "unable to map shared descriptor\n");
3257 		goto err_flc_dma;
3258 	}
3259 
3260 	dpaa2_fl_set_final(in_fle, true);
3261 	dpaa2_fl_set_format(in_fle, dpaa2_fl_single);
3262 	dpaa2_fl_set_addr(in_fle, key_dma);
3263 	dpaa2_fl_set_len(in_fle, *keylen);
3264 	dpaa2_fl_set_format(out_fle, dpaa2_fl_single);
3265 	dpaa2_fl_set_addr(out_fle, key_dma);
3266 	dpaa2_fl_set_len(out_fle, digestsize);
3267 
3268 	print_hex_dump_debug("key_in@" __stringify(__LINE__)": ",
3269 			     DUMP_PREFIX_ADDRESS, 16, 4, key, *keylen, 1);
3270 	print_hex_dump_debug("shdesc@" __stringify(__LINE__)": ",
3271 			     DUMP_PREFIX_ADDRESS, 16, 4, desc, desc_bytes(desc),
3272 			     1);
3273 
3274 	result.err = 0;
3275 	init_completion(&result.completion);
3276 	result.dev = ctx->dev;
3277 
3278 	req_ctx->flc = flc;
3279 	req_ctx->flc_dma = flc_dma;
3280 	req_ctx->cbk = split_key_sh_done;
3281 	req_ctx->ctx = &result;
3282 
3283 	ret = dpaa2_caam_enqueue(ctx->dev, req_ctx);
3284 	if (ret == -EINPROGRESS) {
3285 		/* in progress */
3286 		wait_for_completion(&result.completion);
3287 		ret = result.err;
3288 		print_hex_dump_debug("digested key@" __stringify(__LINE__)": ",
3289 				     DUMP_PREFIX_ADDRESS, 16, 4, key,
3290 				     digestsize, 1);
3291 	}
3292 
3293 	dma_unmap_single(ctx->dev, flc_dma, sizeof(flc->flc) + desc_bytes(desc),
3294 			 DMA_TO_DEVICE);
3295 err_flc_dma:
3296 	dma_unmap_single(ctx->dev, key_dma, *keylen, DMA_BIDIRECTIONAL);
3297 err_key_dma:
3298 	kfree(flc);
3299 err_flc:
3300 	kfree(req_ctx);
3301 
3302 	*keylen = digestsize;
3303 
3304 	return ret;
3305 }
3306 
3307 static int ahash_setkey(struct crypto_ahash *ahash, const u8 *key,
3308 			unsigned int keylen)
3309 {
3310 	struct caam_hash_ctx *ctx = crypto_ahash_ctx_dma(ahash);
3311 	unsigned int blocksize = crypto_tfm_alg_blocksize(&ahash->base);
3312 	unsigned int digestsize = crypto_ahash_digestsize(ahash);
3313 	int ret;
3314 	u8 *hashed_key = NULL;
3315 
3316 	dev_dbg(ctx->dev, "keylen %d blocksize %d\n", keylen, blocksize);
3317 
3318 	if (keylen > blocksize) {
3319 		hashed_key = kmemdup(key, keylen, GFP_KERNEL | GFP_DMA);
3320 		if (!hashed_key)
3321 			return -ENOMEM;
3322 		ret = hash_digest_key(ctx, &keylen, hashed_key, digestsize);
3323 		if (ret)
3324 			goto bad_free_key;
3325 		key = hashed_key;
3326 	}
3327 
3328 	ctx->adata.keylen = keylen;
3329 	ctx->adata.keylen_pad = split_key_len(ctx->adata.algtype &
3330 					      OP_ALG_ALGSEL_MASK);
3331 	if (ctx->adata.keylen_pad > CAAM_MAX_HASH_KEY_SIZE)
3332 		goto bad_free_key;
3333 
3334 	ctx->adata.key_virt = key;
3335 	ctx->adata.key_inline = true;
3336 
3337 	/*
3338 	 * In case |user key| > |derived key|, using DKP<imm,imm> would result
3339 	 * in invalid opcodes (last bytes of user key) in the resulting
3340 	 * descriptor. Use DKP<ptr,imm> instead => both virtual and dma key
3341 	 * addresses are needed.
3342 	 */
3343 	if (keylen > ctx->adata.keylen_pad) {
3344 		memcpy(ctx->key, key, keylen);
3345 		dma_sync_single_for_device(ctx->dev, ctx->adata.key_dma,
3346 					   ctx->adata.keylen_pad,
3347 					   DMA_TO_DEVICE);
3348 	}
3349 
3350 	ret = ahash_set_sh_desc(ahash);
3351 	kfree(hashed_key);
3352 	return ret;
3353 bad_free_key:
3354 	kfree(hashed_key);
3355 	return -EINVAL;
3356 }
3357 
3358 static inline void ahash_unmap(struct device *dev, struct ahash_edesc *edesc,
3359 			       struct ahash_request *req)
3360 {
3361 	struct caam_hash_state *state = ahash_request_ctx_dma(req);
3362 
3363 	if (edesc->src_nents)
3364 		dma_unmap_sg(dev, req->src, edesc->src_nents, DMA_TO_DEVICE);
3365 
3366 	if (edesc->qm_sg_bytes)
3367 		dma_unmap_single(dev, edesc->qm_sg_dma, edesc->qm_sg_bytes,
3368 				 DMA_TO_DEVICE);
3369 
3370 	if (state->buf_dma) {
3371 		dma_unmap_single(dev, state->buf_dma, state->buflen,
3372 				 DMA_TO_DEVICE);
3373 		state->buf_dma = 0;
3374 	}
3375 }
3376 
3377 static inline void ahash_unmap_ctx(struct device *dev,
3378 				   struct ahash_edesc *edesc,
3379 				   struct ahash_request *req, u32 flag)
3380 {
3381 	struct caam_hash_state *state = ahash_request_ctx_dma(req);
3382 
3383 	if (state->ctx_dma) {
3384 		dma_unmap_single(dev, state->ctx_dma, state->ctx_dma_len, flag);
3385 		state->ctx_dma = 0;
3386 	}
3387 	ahash_unmap(dev, edesc, req);
3388 }
3389 
3390 static void ahash_done(void *cbk_ctx, u32 status)
3391 {
3392 	struct crypto_async_request *areq = cbk_ctx;
3393 	struct ahash_request *req = ahash_request_cast(areq);
3394 	struct crypto_ahash *ahash = crypto_ahash_reqtfm(req);
3395 	struct caam_hash_state *state = ahash_request_ctx_dma(req);
3396 	struct ahash_edesc *edesc = state->caam_req.edesc;
3397 	struct caam_hash_ctx *ctx = crypto_ahash_ctx_dma(ahash);
3398 	int digestsize = crypto_ahash_digestsize(ahash);
3399 	int ecode = 0;
3400 
3401 	dev_dbg(ctx->dev, "%s %d: err 0x%x\n", __func__, __LINE__, status);
3402 
3403 	if (unlikely(status))
3404 		ecode = caam_qi2_strstatus(ctx->dev, status);
3405 
3406 	ahash_unmap_ctx(ctx->dev, edesc, req, DMA_FROM_DEVICE);
3407 	memcpy(req->result, state->caam_ctx, digestsize);
3408 	qi_cache_free(edesc);
3409 
3410 	print_hex_dump_debug("ctx@" __stringify(__LINE__)": ",
3411 			     DUMP_PREFIX_ADDRESS, 16, 4, state->caam_ctx,
3412 			     ctx->ctx_len, 1);
3413 
3414 	req->base.complete(&req->base, ecode);
3415 }
3416 
3417 static void ahash_done_bi(void *cbk_ctx, u32 status)
3418 {
3419 	struct crypto_async_request *areq = cbk_ctx;
3420 	struct ahash_request *req = ahash_request_cast(areq);
3421 	struct crypto_ahash *ahash = crypto_ahash_reqtfm(req);
3422 	struct caam_hash_state *state = ahash_request_ctx_dma(req);
3423 	struct ahash_edesc *edesc = state->caam_req.edesc;
3424 	struct caam_hash_ctx *ctx = crypto_ahash_ctx_dma(ahash);
3425 	int ecode = 0;
3426 
3427 	dev_dbg(ctx->dev, "%s %d: err 0x%x\n", __func__, __LINE__, status);
3428 
3429 	if (unlikely(status))
3430 		ecode = caam_qi2_strstatus(ctx->dev, status);
3431 
3432 	ahash_unmap_ctx(ctx->dev, edesc, req, DMA_BIDIRECTIONAL);
3433 	qi_cache_free(edesc);
3434 
3435 	scatterwalk_map_and_copy(state->buf, req->src,
3436 				 req->nbytes - state->next_buflen,
3437 				 state->next_buflen, 0);
3438 	state->buflen = state->next_buflen;
3439 
3440 	print_hex_dump_debug("buf@" __stringify(__LINE__)": ",
3441 			     DUMP_PREFIX_ADDRESS, 16, 4, state->buf,
3442 			     state->buflen, 1);
3443 
3444 	print_hex_dump_debug("ctx@" __stringify(__LINE__)": ",
3445 			     DUMP_PREFIX_ADDRESS, 16, 4, state->caam_ctx,
3446 			     ctx->ctx_len, 1);
3447 	if (req->result)
3448 		print_hex_dump_debug("result@" __stringify(__LINE__)": ",
3449 				     DUMP_PREFIX_ADDRESS, 16, 4, req->result,
3450 				     crypto_ahash_digestsize(ahash), 1);
3451 
3452 	req->base.complete(&req->base, ecode);
3453 }
3454 
3455 static void ahash_done_ctx_src(void *cbk_ctx, u32 status)
3456 {
3457 	struct crypto_async_request *areq = cbk_ctx;
3458 	struct ahash_request *req = ahash_request_cast(areq);
3459 	struct crypto_ahash *ahash = crypto_ahash_reqtfm(req);
3460 	struct caam_hash_state *state = ahash_request_ctx_dma(req);
3461 	struct ahash_edesc *edesc = state->caam_req.edesc;
3462 	struct caam_hash_ctx *ctx = crypto_ahash_ctx_dma(ahash);
3463 	int digestsize = crypto_ahash_digestsize(ahash);
3464 	int ecode = 0;
3465 
3466 	dev_dbg(ctx->dev, "%s %d: err 0x%x\n", __func__, __LINE__, status);
3467 
3468 	if (unlikely(status))
3469 		ecode = caam_qi2_strstatus(ctx->dev, status);
3470 
3471 	ahash_unmap_ctx(ctx->dev, edesc, req, DMA_BIDIRECTIONAL);
3472 	memcpy(req->result, state->caam_ctx, digestsize);
3473 	qi_cache_free(edesc);
3474 
3475 	print_hex_dump_debug("ctx@" __stringify(__LINE__)": ",
3476 			     DUMP_PREFIX_ADDRESS, 16, 4, state->caam_ctx,
3477 			     ctx->ctx_len, 1);
3478 
3479 	req->base.complete(&req->base, ecode);
3480 }
3481 
3482 static void ahash_done_ctx_dst(void *cbk_ctx, u32 status)
3483 {
3484 	struct crypto_async_request *areq = cbk_ctx;
3485 	struct ahash_request *req = ahash_request_cast(areq);
3486 	struct crypto_ahash *ahash = crypto_ahash_reqtfm(req);
3487 	struct caam_hash_state *state = ahash_request_ctx_dma(req);
3488 	struct ahash_edesc *edesc = state->caam_req.edesc;
3489 	struct caam_hash_ctx *ctx = crypto_ahash_ctx_dma(ahash);
3490 	int ecode = 0;
3491 
3492 	dev_dbg(ctx->dev, "%s %d: err 0x%x\n", __func__, __LINE__, status);
3493 
3494 	if (unlikely(status))
3495 		ecode = caam_qi2_strstatus(ctx->dev, status);
3496 
3497 	ahash_unmap_ctx(ctx->dev, edesc, req, DMA_FROM_DEVICE);
3498 	qi_cache_free(edesc);
3499 
3500 	scatterwalk_map_and_copy(state->buf, req->src,
3501 				 req->nbytes - state->next_buflen,
3502 				 state->next_buflen, 0);
3503 	state->buflen = state->next_buflen;
3504 
3505 	print_hex_dump_debug("buf@" __stringify(__LINE__)": ",
3506 			     DUMP_PREFIX_ADDRESS, 16, 4, state->buf,
3507 			     state->buflen, 1);
3508 
3509 	print_hex_dump_debug("ctx@" __stringify(__LINE__)": ",
3510 			     DUMP_PREFIX_ADDRESS, 16, 4, state->caam_ctx,
3511 			     ctx->ctx_len, 1);
3512 	if (req->result)
3513 		print_hex_dump_debug("result@" __stringify(__LINE__)": ",
3514 				     DUMP_PREFIX_ADDRESS, 16, 4, req->result,
3515 				     crypto_ahash_digestsize(ahash), 1);
3516 
3517 	req->base.complete(&req->base, ecode);
3518 }
3519 
3520 static int ahash_update_ctx(struct ahash_request *req)
3521 {
3522 	struct crypto_ahash *ahash = crypto_ahash_reqtfm(req);
3523 	struct caam_hash_ctx *ctx = crypto_ahash_ctx_dma(ahash);
3524 	struct caam_hash_state *state = ahash_request_ctx_dma(req);
3525 	struct caam_request *req_ctx = &state->caam_req;
3526 	struct dpaa2_fl_entry *in_fle = &req_ctx->fd_flt[1];
3527 	struct dpaa2_fl_entry *out_fle = &req_ctx->fd_flt[0];
3528 	gfp_t flags = (req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP) ?
3529 		      GFP_KERNEL : GFP_ATOMIC;
3530 	u8 *buf = state->buf;
3531 	int *buflen = &state->buflen;
3532 	int *next_buflen = &state->next_buflen;
3533 	int in_len = *buflen + req->nbytes, to_hash;
3534 	int src_nents, mapped_nents, qm_sg_bytes, qm_sg_src_index;
3535 	struct ahash_edesc *edesc;
3536 	int ret = 0;
3537 
3538 	*next_buflen = in_len & (crypto_tfm_alg_blocksize(&ahash->base) - 1);
3539 	to_hash = in_len - *next_buflen;
3540 
3541 	if (to_hash) {
3542 		struct dpaa2_sg_entry *sg_table;
3543 		int src_len = req->nbytes - *next_buflen;
3544 
3545 		src_nents = sg_nents_for_len(req->src, src_len);
3546 		if (src_nents < 0) {
3547 			dev_err(ctx->dev, "Invalid number of src SG.\n");
3548 			return src_nents;
3549 		}
3550 
3551 		if (src_nents) {
3552 			mapped_nents = dma_map_sg(ctx->dev, req->src, src_nents,
3553 						  DMA_TO_DEVICE);
3554 			if (!mapped_nents) {
3555 				dev_err(ctx->dev, "unable to DMA map source\n");
3556 				return -ENOMEM;
3557 			}
3558 		} else {
3559 			mapped_nents = 0;
3560 		}
3561 
3562 		/* allocate space for base edesc and link tables */
3563 		edesc = qi_cache_zalloc(GFP_DMA | flags);
3564 		if (!edesc) {
3565 			dma_unmap_sg(ctx->dev, req->src, src_nents,
3566 				     DMA_TO_DEVICE);
3567 			return -ENOMEM;
3568 		}
3569 
3570 		edesc->src_nents = src_nents;
3571 		qm_sg_src_index = 1 + (*buflen ? 1 : 0);
3572 		qm_sg_bytes = pad_sg_nents(qm_sg_src_index + mapped_nents) *
3573 			      sizeof(*sg_table);
3574 		sg_table = &edesc->sgt[0];
3575 
3576 		ret = ctx_map_to_qm_sg(ctx->dev, state, ctx->ctx_len, sg_table,
3577 				       DMA_BIDIRECTIONAL);
3578 		if (ret)
3579 			goto unmap_ctx;
3580 
3581 		ret = buf_map_to_qm_sg(ctx->dev, sg_table + 1, state);
3582 		if (ret)
3583 			goto unmap_ctx;
3584 
3585 		if (mapped_nents) {
3586 			sg_to_qm_sg_last(req->src, src_len,
3587 					 sg_table + qm_sg_src_index, 0);
3588 		} else {
3589 			dpaa2_sg_set_final(sg_table + qm_sg_src_index - 1,
3590 					   true);
3591 		}
3592 
3593 		edesc->qm_sg_dma = dma_map_single(ctx->dev, sg_table,
3594 						  qm_sg_bytes, DMA_TO_DEVICE);
3595 		if (dma_mapping_error(ctx->dev, edesc->qm_sg_dma)) {
3596 			dev_err(ctx->dev, "unable to map S/G table\n");
3597 			ret = -ENOMEM;
3598 			goto unmap_ctx;
3599 		}
3600 		edesc->qm_sg_bytes = qm_sg_bytes;
3601 
3602 		memset(&req_ctx->fd_flt, 0, sizeof(req_ctx->fd_flt));
3603 		dpaa2_fl_set_final(in_fle, true);
3604 		dpaa2_fl_set_format(in_fle, dpaa2_fl_sg);
3605 		dpaa2_fl_set_addr(in_fle, edesc->qm_sg_dma);
3606 		dpaa2_fl_set_len(in_fle, ctx->ctx_len + to_hash);
3607 		dpaa2_fl_set_format(out_fle, dpaa2_fl_single);
3608 		dpaa2_fl_set_addr(out_fle, state->ctx_dma);
3609 		dpaa2_fl_set_len(out_fle, ctx->ctx_len);
3610 
3611 		req_ctx->flc = &ctx->flc[UPDATE];
3612 		req_ctx->flc_dma = ctx->flc_dma[UPDATE];
3613 		req_ctx->cbk = ahash_done_bi;
3614 		req_ctx->ctx = &req->base;
3615 		req_ctx->edesc = edesc;
3616 
3617 		ret = dpaa2_caam_enqueue(ctx->dev, req_ctx);
3618 		if (ret != -EINPROGRESS &&
3619 		    !(ret == -EBUSY &&
3620 		      req->base.flags & CRYPTO_TFM_REQ_MAY_BACKLOG))
3621 			goto unmap_ctx;
3622 	} else if (*next_buflen) {
3623 		scatterwalk_map_and_copy(buf + *buflen, req->src, 0,
3624 					 req->nbytes, 0);
3625 		*buflen = *next_buflen;
3626 
3627 		print_hex_dump_debug("buf@" __stringify(__LINE__)": ",
3628 				     DUMP_PREFIX_ADDRESS, 16, 4, buf,
3629 				     *buflen, 1);
3630 	}
3631 
3632 	return ret;
3633 unmap_ctx:
3634 	ahash_unmap_ctx(ctx->dev, edesc, req, DMA_BIDIRECTIONAL);
3635 	qi_cache_free(edesc);
3636 	return ret;
3637 }
3638 
3639 static int ahash_final_ctx(struct ahash_request *req)
3640 {
3641 	struct crypto_ahash *ahash = crypto_ahash_reqtfm(req);
3642 	struct caam_hash_ctx *ctx = crypto_ahash_ctx_dma(ahash);
3643 	struct caam_hash_state *state = ahash_request_ctx_dma(req);
3644 	struct caam_request *req_ctx = &state->caam_req;
3645 	struct dpaa2_fl_entry *in_fle = &req_ctx->fd_flt[1];
3646 	struct dpaa2_fl_entry *out_fle = &req_ctx->fd_flt[0];
3647 	gfp_t flags = (req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP) ?
3648 		      GFP_KERNEL : GFP_ATOMIC;
3649 	int buflen = state->buflen;
3650 	int qm_sg_bytes;
3651 	int digestsize = crypto_ahash_digestsize(ahash);
3652 	struct ahash_edesc *edesc;
3653 	struct dpaa2_sg_entry *sg_table;
3654 	int ret;
3655 
3656 	/* allocate space for base edesc and link tables */
3657 	edesc = qi_cache_zalloc(GFP_DMA | flags);
3658 	if (!edesc)
3659 		return -ENOMEM;
3660 
3661 	qm_sg_bytes = pad_sg_nents(1 + (buflen ? 1 : 0)) * sizeof(*sg_table);
3662 	sg_table = &edesc->sgt[0];
3663 
3664 	ret = ctx_map_to_qm_sg(ctx->dev, state, ctx->ctx_len, sg_table,
3665 			       DMA_BIDIRECTIONAL);
3666 	if (ret)
3667 		goto unmap_ctx;
3668 
3669 	ret = buf_map_to_qm_sg(ctx->dev, sg_table + 1, state);
3670 	if (ret)
3671 		goto unmap_ctx;
3672 
3673 	dpaa2_sg_set_final(sg_table + (buflen ? 1 : 0), true);
3674 
3675 	edesc->qm_sg_dma = dma_map_single(ctx->dev, sg_table, qm_sg_bytes,
3676 					  DMA_TO_DEVICE);
3677 	if (dma_mapping_error(ctx->dev, edesc->qm_sg_dma)) {
3678 		dev_err(ctx->dev, "unable to map S/G table\n");
3679 		ret = -ENOMEM;
3680 		goto unmap_ctx;
3681 	}
3682 	edesc->qm_sg_bytes = qm_sg_bytes;
3683 
3684 	memset(&req_ctx->fd_flt, 0, sizeof(req_ctx->fd_flt));
3685 	dpaa2_fl_set_final(in_fle, true);
3686 	dpaa2_fl_set_format(in_fle, dpaa2_fl_sg);
3687 	dpaa2_fl_set_addr(in_fle, edesc->qm_sg_dma);
3688 	dpaa2_fl_set_len(in_fle, ctx->ctx_len + buflen);
3689 	dpaa2_fl_set_format(out_fle, dpaa2_fl_single);
3690 	dpaa2_fl_set_addr(out_fle, state->ctx_dma);
3691 	dpaa2_fl_set_len(out_fle, digestsize);
3692 
3693 	req_ctx->flc = &ctx->flc[FINALIZE];
3694 	req_ctx->flc_dma = ctx->flc_dma[FINALIZE];
3695 	req_ctx->cbk = ahash_done_ctx_src;
3696 	req_ctx->ctx = &req->base;
3697 	req_ctx->edesc = edesc;
3698 
3699 	ret = dpaa2_caam_enqueue(ctx->dev, req_ctx);
3700 	if (ret == -EINPROGRESS ||
3701 	    (ret == -EBUSY && req->base.flags & CRYPTO_TFM_REQ_MAY_BACKLOG))
3702 		return ret;
3703 
3704 unmap_ctx:
3705 	ahash_unmap_ctx(ctx->dev, edesc, req, DMA_BIDIRECTIONAL);
3706 	qi_cache_free(edesc);
3707 	return ret;
3708 }
3709 
3710 static int ahash_finup_ctx(struct ahash_request *req)
3711 {
3712 	struct crypto_ahash *ahash = crypto_ahash_reqtfm(req);
3713 	struct caam_hash_ctx *ctx = crypto_ahash_ctx_dma(ahash);
3714 	struct caam_hash_state *state = ahash_request_ctx_dma(req);
3715 	struct caam_request *req_ctx = &state->caam_req;
3716 	struct dpaa2_fl_entry *in_fle = &req_ctx->fd_flt[1];
3717 	struct dpaa2_fl_entry *out_fle = &req_ctx->fd_flt[0];
3718 	gfp_t flags = (req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP) ?
3719 		      GFP_KERNEL : GFP_ATOMIC;
3720 	int buflen = state->buflen;
3721 	int qm_sg_bytes, qm_sg_src_index;
3722 	int src_nents, mapped_nents;
3723 	int digestsize = crypto_ahash_digestsize(ahash);
3724 	struct ahash_edesc *edesc;
3725 	struct dpaa2_sg_entry *sg_table;
3726 	int ret;
3727 
3728 	src_nents = sg_nents_for_len(req->src, req->nbytes);
3729 	if (src_nents < 0) {
3730 		dev_err(ctx->dev, "Invalid number of src SG.\n");
3731 		return src_nents;
3732 	}
3733 
3734 	if (src_nents) {
3735 		mapped_nents = dma_map_sg(ctx->dev, req->src, src_nents,
3736 					  DMA_TO_DEVICE);
3737 		if (!mapped_nents) {
3738 			dev_err(ctx->dev, "unable to DMA map source\n");
3739 			return -ENOMEM;
3740 		}
3741 	} else {
3742 		mapped_nents = 0;
3743 	}
3744 
3745 	/* allocate space for base edesc and link tables */
3746 	edesc = qi_cache_zalloc(GFP_DMA | flags);
3747 	if (!edesc) {
3748 		dma_unmap_sg(ctx->dev, req->src, src_nents, DMA_TO_DEVICE);
3749 		return -ENOMEM;
3750 	}
3751 
3752 	edesc->src_nents = src_nents;
3753 	qm_sg_src_index = 1 + (buflen ? 1 : 0);
3754 	qm_sg_bytes = pad_sg_nents(qm_sg_src_index + mapped_nents) *
3755 		      sizeof(*sg_table);
3756 	sg_table = &edesc->sgt[0];
3757 
3758 	ret = ctx_map_to_qm_sg(ctx->dev, state, ctx->ctx_len, sg_table,
3759 			       DMA_BIDIRECTIONAL);
3760 	if (ret)
3761 		goto unmap_ctx;
3762 
3763 	ret = buf_map_to_qm_sg(ctx->dev, sg_table + 1, state);
3764 	if (ret)
3765 		goto unmap_ctx;
3766 
3767 	sg_to_qm_sg_last(req->src, req->nbytes, sg_table + qm_sg_src_index, 0);
3768 
3769 	edesc->qm_sg_dma = dma_map_single(ctx->dev, sg_table, qm_sg_bytes,
3770 					  DMA_TO_DEVICE);
3771 	if (dma_mapping_error(ctx->dev, edesc->qm_sg_dma)) {
3772 		dev_err(ctx->dev, "unable to map S/G table\n");
3773 		ret = -ENOMEM;
3774 		goto unmap_ctx;
3775 	}
3776 	edesc->qm_sg_bytes = qm_sg_bytes;
3777 
3778 	memset(&req_ctx->fd_flt, 0, sizeof(req_ctx->fd_flt));
3779 	dpaa2_fl_set_final(in_fle, true);
3780 	dpaa2_fl_set_format(in_fle, dpaa2_fl_sg);
3781 	dpaa2_fl_set_addr(in_fle, edesc->qm_sg_dma);
3782 	dpaa2_fl_set_len(in_fle, ctx->ctx_len + buflen + req->nbytes);
3783 	dpaa2_fl_set_format(out_fle, dpaa2_fl_single);
3784 	dpaa2_fl_set_addr(out_fle, state->ctx_dma);
3785 	dpaa2_fl_set_len(out_fle, digestsize);
3786 
3787 	req_ctx->flc = &ctx->flc[FINALIZE];
3788 	req_ctx->flc_dma = ctx->flc_dma[FINALIZE];
3789 	req_ctx->cbk = ahash_done_ctx_src;
3790 	req_ctx->ctx = &req->base;
3791 	req_ctx->edesc = edesc;
3792 
3793 	ret = dpaa2_caam_enqueue(ctx->dev, req_ctx);
3794 	if (ret == -EINPROGRESS ||
3795 	    (ret == -EBUSY && req->base.flags & CRYPTO_TFM_REQ_MAY_BACKLOG))
3796 		return ret;
3797 
3798 unmap_ctx:
3799 	ahash_unmap_ctx(ctx->dev, edesc, req, DMA_BIDIRECTIONAL);
3800 	qi_cache_free(edesc);
3801 	return ret;
3802 }
3803 
3804 static int ahash_digest(struct ahash_request *req)
3805 {
3806 	struct crypto_ahash *ahash = crypto_ahash_reqtfm(req);
3807 	struct caam_hash_ctx *ctx = crypto_ahash_ctx_dma(ahash);
3808 	struct caam_hash_state *state = ahash_request_ctx_dma(req);
3809 	struct caam_request *req_ctx = &state->caam_req;
3810 	struct dpaa2_fl_entry *in_fle = &req_ctx->fd_flt[1];
3811 	struct dpaa2_fl_entry *out_fle = &req_ctx->fd_flt[0];
3812 	gfp_t flags = (req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP) ?
3813 		      GFP_KERNEL : GFP_ATOMIC;
3814 	int digestsize = crypto_ahash_digestsize(ahash);
3815 	int src_nents, mapped_nents;
3816 	struct ahash_edesc *edesc;
3817 	int ret = -ENOMEM;
3818 
3819 	state->buf_dma = 0;
3820 
3821 	src_nents = sg_nents_for_len(req->src, req->nbytes);
3822 	if (src_nents < 0) {
3823 		dev_err(ctx->dev, "Invalid number of src SG.\n");
3824 		return src_nents;
3825 	}
3826 
3827 	if (src_nents) {
3828 		mapped_nents = dma_map_sg(ctx->dev, req->src, src_nents,
3829 					  DMA_TO_DEVICE);
3830 		if (!mapped_nents) {
3831 			dev_err(ctx->dev, "unable to map source for DMA\n");
3832 			return ret;
3833 		}
3834 	} else {
3835 		mapped_nents = 0;
3836 	}
3837 
3838 	/* allocate space for base edesc and link tables */
3839 	edesc = qi_cache_zalloc(GFP_DMA | flags);
3840 	if (!edesc) {
3841 		dma_unmap_sg(ctx->dev, req->src, src_nents, DMA_TO_DEVICE);
3842 		return ret;
3843 	}
3844 
3845 	edesc->src_nents = src_nents;
3846 	memset(&req_ctx->fd_flt, 0, sizeof(req_ctx->fd_flt));
3847 
3848 	if (mapped_nents > 1) {
3849 		int qm_sg_bytes;
3850 		struct dpaa2_sg_entry *sg_table = &edesc->sgt[0];
3851 
3852 		qm_sg_bytes = pad_sg_nents(mapped_nents) * sizeof(*sg_table);
3853 		sg_to_qm_sg_last(req->src, req->nbytes, sg_table, 0);
3854 		edesc->qm_sg_dma = dma_map_single(ctx->dev, sg_table,
3855 						  qm_sg_bytes, DMA_TO_DEVICE);
3856 		if (dma_mapping_error(ctx->dev, edesc->qm_sg_dma)) {
3857 			dev_err(ctx->dev, "unable to map S/G table\n");
3858 			goto unmap;
3859 		}
3860 		edesc->qm_sg_bytes = qm_sg_bytes;
3861 		dpaa2_fl_set_format(in_fle, dpaa2_fl_sg);
3862 		dpaa2_fl_set_addr(in_fle, edesc->qm_sg_dma);
3863 	} else {
3864 		dpaa2_fl_set_format(in_fle, dpaa2_fl_single);
3865 		dpaa2_fl_set_addr(in_fle, sg_dma_address(req->src));
3866 	}
3867 
3868 	state->ctx_dma_len = digestsize;
3869 	state->ctx_dma = dma_map_single(ctx->dev, state->caam_ctx, digestsize,
3870 					DMA_FROM_DEVICE);
3871 	if (dma_mapping_error(ctx->dev, state->ctx_dma)) {
3872 		dev_err(ctx->dev, "unable to map ctx\n");
3873 		state->ctx_dma = 0;
3874 		goto unmap;
3875 	}
3876 
3877 	dpaa2_fl_set_final(in_fle, true);
3878 	dpaa2_fl_set_len(in_fle, req->nbytes);
3879 	dpaa2_fl_set_format(out_fle, dpaa2_fl_single);
3880 	dpaa2_fl_set_addr(out_fle, state->ctx_dma);
3881 	dpaa2_fl_set_len(out_fle, digestsize);
3882 
3883 	req_ctx->flc = &ctx->flc[DIGEST];
3884 	req_ctx->flc_dma = ctx->flc_dma[DIGEST];
3885 	req_ctx->cbk = ahash_done;
3886 	req_ctx->ctx = &req->base;
3887 	req_ctx->edesc = edesc;
3888 	ret = dpaa2_caam_enqueue(ctx->dev, req_ctx);
3889 	if (ret == -EINPROGRESS ||
3890 	    (ret == -EBUSY && req->base.flags & CRYPTO_TFM_REQ_MAY_BACKLOG))
3891 		return ret;
3892 
3893 unmap:
3894 	ahash_unmap_ctx(ctx->dev, edesc, req, DMA_FROM_DEVICE);
3895 	qi_cache_free(edesc);
3896 	return ret;
3897 }
3898 
3899 static int ahash_final_no_ctx(struct ahash_request *req)
3900 {
3901 	struct crypto_ahash *ahash = crypto_ahash_reqtfm(req);
3902 	struct caam_hash_ctx *ctx = crypto_ahash_ctx_dma(ahash);
3903 	struct caam_hash_state *state = ahash_request_ctx_dma(req);
3904 	struct caam_request *req_ctx = &state->caam_req;
3905 	struct dpaa2_fl_entry *in_fle = &req_ctx->fd_flt[1];
3906 	struct dpaa2_fl_entry *out_fle = &req_ctx->fd_flt[0];
3907 	gfp_t flags = (req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP) ?
3908 		      GFP_KERNEL : GFP_ATOMIC;
3909 	u8 *buf = state->buf;
3910 	int buflen = state->buflen;
3911 	int digestsize = crypto_ahash_digestsize(ahash);
3912 	struct ahash_edesc *edesc;
3913 	int ret = -ENOMEM;
3914 
3915 	/* allocate space for base edesc and link tables */
3916 	edesc = qi_cache_zalloc(GFP_DMA | flags);
3917 	if (!edesc)
3918 		return ret;
3919 
3920 	if (buflen) {
3921 		state->buf_dma = dma_map_single(ctx->dev, buf, buflen,
3922 						DMA_TO_DEVICE);
3923 		if (dma_mapping_error(ctx->dev, state->buf_dma)) {
3924 			dev_err(ctx->dev, "unable to map src\n");
3925 			goto unmap;
3926 		}
3927 	}
3928 
3929 	state->ctx_dma_len = digestsize;
3930 	state->ctx_dma = dma_map_single(ctx->dev, state->caam_ctx, digestsize,
3931 					DMA_FROM_DEVICE);
3932 	if (dma_mapping_error(ctx->dev, state->ctx_dma)) {
3933 		dev_err(ctx->dev, "unable to map ctx\n");
3934 		state->ctx_dma = 0;
3935 		goto unmap;
3936 	}
3937 
3938 	memset(&req_ctx->fd_flt, 0, sizeof(req_ctx->fd_flt));
3939 	dpaa2_fl_set_final(in_fle, true);
3940 	/*
3941 	 * crypto engine requires the input entry to be present when
3942 	 * "frame list" FD is used.
3943 	 * Since engine does not support FMT=2'b11 (unused entry type), leaving
3944 	 * in_fle zeroized (except for "Final" flag) is the best option.
3945 	 */
3946 	if (buflen) {
3947 		dpaa2_fl_set_format(in_fle, dpaa2_fl_single);
3948 		dpaa2_fl_set_addr(in_fle, state->buf_dma);
3949 		dpaa2_fl_set_len(in_fle, buflen);
3950 	}
3951 	dpaa2_fl_set_format(out_fle, dpaa2_fl_single);
3952 	dpaa2_fl_set_addr(out_fle, state->ctx_dma);
3953 	dpaa2_fl_set_len(out_fle, digestsize);
3954 
3955 	req_ctx->flc = &ctx->flc[DIGEST];
3956 	req_ctx->flc_dma = ctx->flc_dma[DIGEST];
3957 	req_ctx->cbk = ahash_done;
3958 	req_ctx->ctx = &req->base;
3959 	req_ctx->edesc = edesc;
3960 
3961 	ret = dpaa2_caam_enqueue(ctx->dev, req_ctx);
3962 	if (ret == -EINPROGRESS ||
3963 	    (ret == -EBUSY && req->base.flags & CRYPTO_TFM_REQ_MAY_BACKLOG))
3964 		return ret;
3965 
3966 unmap:
3967 	ahash_unmap_ctx(ctx->dev, edesc, req, DMA_FROM_DEVICE);
3968 	qi_cache_free(edesc);
3969 	return ret;
3970 }
3971 
3972 static int ahash_update_no_ctx(struct ahash_request *req)
3973 {
3974 	struct crypto_ahash *ahash = crypto_ahash_reqtfm(req);
3975 	struct caam_hash_ctx *ctx = crypto_ahash_ctx_dma(ahash);
3976 	struct caam_hash_state *state = ahash_request_ctx_dma(req);
3977 	struct caam_request *req_ctx = &state->caam_req;
3978 	struct dpaa2_fl_entry *in_fle = &req_ctx->fd_flt[1];
3979 	struct dpaa2_fl_entry *out_fle = &req_ctx->fd_flt[0];
3980 	gfp_t flags = (req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP) ?
3981 		      GFP_KERNEL : GFP_ATOMIC;
3982 	u8 *buf = state->buf;
3983 	int *buflen = &state->buflen;
3984 	int *next_buflen = &state->next_buflen;
3985 	int in_len = *buflen + req->nbytes, to_hash;
3986 	int qm_sg_bytes, src_nents, mapped_nents;
3987 	struct ahash_edesc *edesc;
3988 	int ret = 0;
3989 
3990 	*next_buflen = in_len & (crypto_tfm_alg_blocksize(&ahash->base) - 1);
3991 	to_hash = in_len - *next_buflen;
3992 
3993 	if (to_hash) {
3994 		struct dpaa2_sg_entry *sg_table;
3995 		int src_len = req->nbytes - *next_buflen;
3996 
3997 		src_nents = sg_nents_for_len(req->src, src_len);
3998 		if (src_nents < 0) {
3999 			dev_err(ctx->dev, "Invalid number of src SG.\n");
4000 			return src_nents;
4001 		}
4002 
4003 		if (src_nents) {
4004 			mapped_nents = dma_map_sg(ctx->dev, req->src, src_nents,
4005 						  DMA_TO_DEVICE);
4006 			if (!mapped_nents) {
4007 				dev_err(ctx->dev, "unable to DMA map source\n");
4008 				return -ENOMEM;
4009 			}
4010 		} else {
4011 			mapped_nents = 0;
4012 		}
4013 
4014 		/* allocate space for base edesc and link tables */
4015 		edesc = qi_cache_zalloc(GFP_DMA | flags);
4016 		if (!edesc) {
4017 			dma_unmap_sg(ctx->dev, req->src, src_nents,
4018 				     DMA_TO_DEVICE);
4019 			return -ENOMEM;
4020 		}
4021 
4022 		edesc->src_nents = src_nents;
4023 		qm_sg_bytes = pad_sg_nents(1 + mapped_nents) *
4024 			      sizeof(*sg_table);
4025 		sg_table = &edesc->sgt[0];
4026 
4027 		ret = buf_map_to_qm_sg(ctx->dev, sg_table, state);
4028 		if (ret)
4029 			goto unmap_ctx;
4030 
4031 		sg_to_qm_sg_last(req->src, src_len, sg_table + 1, 0);
4032 
4033 		edesc->qm_sg_dma = dma_map_single(ctx->dev, sg_table,
4034 						  qm_sg_bytes, DMA_TO_DEVICE);
4035 		if (dma_mapping_error(ctx->dev, edesc->qm_sg_dma)) {
4036 			dev_err(ctx->dev, "unable to map S/G table\n");
4037 			ret = -ENOMEM;
4038 			goto unmap_ctx;
4039 		}
4040 		edesc->qm_sg_bytes = qm_sg_bytes;
4041 
4042 		state->ctx_dma_len = ctx->ctx_len;
4043 		state->ctx_dma = dma_map_single(ctx->dev, state->caam_ctx,
4044 						ctx->ctx_len, DMA_FROM_DEVICE);
4045 		if (dma_mapping_error(ctx->dev, state->ctx_dma)) {
4046 			dev_err(ctx->dev, "unable to map ctx\n");
4047 			state->ctx_dma = 0;
4048 			ret = -ENOMEM;
4049 			goto unmap_ctx;
4050 		}
4051 
4052 		memset(&req_ctx->fd_flt, 0, sizeof(req_ctx->fd_flt));
4053 		dpaa2_fl_set_final(in_fle, true);
4054 		dpaa2_fl_set_format(in_fle, dpaa2_fl_sg);
4055 		dpaa2_fl_set_addr(in_fle, edesc->qm_sg_dma);
4056 		dpaa2_fl_set_len(in_fle, to_hash);
4057 		dpaa2_fl_set_format(out_fle, dpaa2_fl_single);
4058 		dpaa2_fl_set_addr(out_fle, state->ctx_dma);
4059 		dpaa2_fl_set_len(out_fle, ctx->ctx_len);
4060 
4061 		req_ctx->flc = &ctx->flc[UPDATE_FIRST];
4062 		req_ctx->flc_dma = ctx->flc_dma[UPDATE_FIRST];
4063 		req_ctx->cbk = ahash_done_ctx_dst;
4064 		req_ctx->ctx = &req->base;
4065 		req_ctx->edesc = edesc;
4066 
4067 		ret = dpaa2_caam_enqueue(ctx->dev, req_ctx);
4068 		if (ret != -EINPROGRESS &&
4069 		    !(ret == -EBUSY &&
4070 		      req->base.flags & CRYPTO_TFM_REQ_MAY_BACKLOG))
4071 			goto unmap_ctx;
4072 
4073 		state->update = ahash_update_ctx;
4074 		state->finup = ahash_finup_ctx;
4075 		state->final = ahash_final_ctx;
4076 	} else if (*next_buflen) {
4077 		scatterwalk_map_and_copy(buf + *buflen, req->src, 0,
4078 					 req->nbytes, 0);
4079 		*buflen = *next_buflen;
4080 
4081 		print_hex_dump_debug("buf@" __stringify(__LINE__)": ",
4082 				     DUMP_PREFIX_ADDRESS, 16, 4, buf,
4083 				     *buflen, 1);
4084 	}
4085 
4086 	return ret;
4087 unmap_ctx:
4088 	ahash_unmap_ctx(ctx->dev, edesc, req, DMA_TO_DEVICE);
4089 	qi_cache_free(edesc);
4090 	return ret;
4091 }
4092 
4093 static int ahash_finup_no_ctx(struct ahash_request *req)
4094 {
4095 	struct crypto_ahash *ahash = crypto_ahash_reqtfm(req);
4096 	struct caam_hash_ctx *ctx = crypto_ahash_ctx_dma(ahash);
4097 	struct caam_hash_state *state = ahash_request_ctx_dma(req);
4098 	struct caam_request *req_ctx = &state->caam_req;
4099 	struct dpaa2_fl_entry *in_fle = &req_ctx->fd_flt[1];
4100 	struct dpaa2_fl_entry *out_fle = &req_ctx->fd_flt[0];
4101 	gfp_t flags = (req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP) ?
4102 		      GFP_KERNEL : GFP_ATOMIC;
4103 	int buflen = state->buflen;
4104 	int qm_sg_bytes, src_nents, mapped_nents;
4105 	int digestsize = crypto_ahash_digestsize(ahash);
4106 	struct ahash_edesc *edesc;
4107 	struct dpaa2_sg_entry *sg_table;
4108 	int ret = -ENOMEM;
4109 
4110 	src_nents = sg_nents_for_len(req->src, req->nbytes);
4111 	if (src_nents < 0) {
4112 		dev_err(ctx->dev, "Invalid number of src SG.\n");
4113 		return src_nents;
4114 	}
4115 
4116 	if (src_nents) {
4117 		mapped_nents = dma_map_sg(ctx->dev, req->src, src_nents,
4118 					  DMA_TO_DEVICE);
4119 		if (!mapped_nents) {
4120 			dev_err(ctx->dev, "unable to DMA map source\n");
4121 			return ret;
4122 		}
4123 	} else {
4124 		mapped_nents = 0;
4125 	}
4126 
4127 	/* allocate space for base edesc and link tables */
4128 	edesc = qi_cache_zalloc(GFP_DMA | flags);
4129 	if (!edesc) {
4130 		dma_unmap_sg(ctx->dev, req->src, src_nents, DMA_TO_DEVICE);
4131 		return ret;
4132 	}
4133 
4134 	edesc->src_nents = src_nents;
4135 	qm_sg_bytes = pad_sg_nents(2 + mapped_nents) * sizeof(*sg_table);
4136 	sg_table = &edesc->sgt[0];
4137 
4138 	ret = buf_map_to_qm_sg(ctx->dev, sg_table, state);
4139 	if (ret)
4140 		goto unmap;
4141 
4142 	sg_to_qm_sg_last(req->src, req->nbytes, sg_table + 1, 0);
4143 
4144 	edesc->qm_sg_dma = dma_map_single(ctx->dev, sg_table, qm_sg_bytes,
4145 					  DMA_TO_DEVICE);
4146 	if (dma_mapping_error(ctx->dev, edesc->qm_sg_dma)) {
4147 		dev_err(ctx->dev, "unable to map S/G table\n");
4148 		ret = -ENOMEM;
4149 		goto unmap;
4150 	}
4151 	edesc->qm_sg_bytes = qm_sg_bytes;
4152 
4153 	state->ctx_dma_len = digestsize;
4154 	state->ctx_dma = dma_map_single(ctx->dev, state->caam_ctx, digestsize,
4155 					DMA_FROM_DEVICE);
4156 	if (dma_mapping_error(ctx->dev, state->ctx_dma)) {
4157 		dev_err(ctx->dev, "unable to map ctx\n");
4158 		state->ctx_dma = 0;
4159 		ret = -ENOMEM;
4160 		goto unmap;
4161 	}
4162 
4163 	memset(&req_ctx->fd_flt, 0, sizeof(req_ctx->fd_flt));
4164 	dpaa2_fl_set_final(in_fle, true);
4165 	dpaa2_fl_set_format(in_fle, dpaa2_fl_sg);
4166 	dpaa2_fl_set_addr(in_fle, edesc->qm_sg_dma);
4167 	dpaa2_fl_set_len(in_fle, buflen + req->nbytes);
4168 	dpaa2_fl_set_format(out_fle, dpaa2_fl_single);
4169 	dpaa2_fl_set_addr(out_fle, state->ctx_dma);
4170 	dpaa2_fl_set_len(out_fle, digestsize);
4171 
4172 	req_ctx->flc = &ctx->flc[DIGEST];
4173 	req_ctx->flc_dma = ctx->flc_dma[DIGEST];
4174 	req_ctx->cbk = ahash_done;
4175 	req_ctx->ctx = &req->base;
4176 	req_ctx->edesc = edesc;
4177 	ret = dpaa2_caam_enqueue(ctx->dev, req_ctx);
4178 	if (ret != -EINPROGRESS &&
4179 	    !(ret == -EBUSY && req->base.flags & CRYPTO_TFM_REQ_MAY_BACKLOG))
4180 		goto unmap;
4181 
4182 	return ret;
4183 unmap:
4184 	ahash_unmap_ctx(ctx->dev, edesc, req, DMA_FROM_DEVICE);
4185 	qi_cache_free(edesc);
4186 	return ret;
4187 }
4188 
4189 static int ahash_update_first(struct ahash_request *req)
4190 {
4191 	struct crypto_ahash *ahash = crypto_ahash_reqtfm(req);
4192 	struct caam_hash_ctx *ctx = crypto_ahash_ctx_dma(ahash);
4193 	struct caam_hash_state *state = ahash_request_ctx_dma(req);
4194 	struct caam_request *req_ctx = &state->caam_req;
4195 	struct dpaa2_fl_entry *in_fle = &req_ctx->fd_flt[1];
4196 	struct dpaa2_fl_entry *out_fle = &req_ctx->fd_flt[0];
4197 	gfp_t flags = (req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP) ?
4198 		      GFP_KERNEL : GFP_ATOMIC;
4199 	u8 *buf = state->buf;
4200 	int *buflen = &state->buflen;
4201 	int *next_buflen = &state->next_buflen;
4202 	int to_hash;
4203 	int src_nents, mapped_nents;
4204 	struct ahash_edesc *edesc;
4205 	int ret = 0;
4206 
4207 	*next_buflen = req->nbytes & (crypto_tfm_alg_blocksize(&ahash->base) -
4208 				      1);
4209 	to_hash = req->nbytes - *next_buflen;
4210 
4211 	if (to_hash) {
4212 		struct dpaa2_sg_entry *sg_table;
4213 		int src_len = req->nbytes - *next_buflen;
4214 
4215 		src_nents = sg_nents_for_len(req->src, src_len);
4216 		if (src_nents < 0) {
4217 			dev_err(ctx->dev, "Invalid number of src SG.\n");
4218 			return src_nents;
4219 		}
4220 
4221 		if (src_nents) {
4222 			mapped_nents = dma_map_sg(ctx->dev, req->src, src_nents,
4223 						  DMA_TO_DEVICE);
4224 			if (!mapped_nents) {
4225 				dev_err(ctx->dev, "unable to map source for DMA\n");
4226 				return -ENOMEM;
4227 			}
4228 		} else {
4229 			mapped_nents = 0;
4230 		}
4231 
4232 		/* allocate space for base edesc and link tables */
4233 		edesc = qi_cache_zalloc(GFP_DMA | flags);
4234 		if (!edesc) {
4235 			dma_unmap_sg(ctx->dev, req->src, src_nents,
4236 				     DMA_TO_DEVICE);
4237 			return -ENOMEM;
4238 		}
4239 
4240 		edesc->src_nents = src_nents;
4241 		sg_table = &edesc->sgt[0];
4242 
4243 		memset(&req_ctx->fd_flt, 0, sizeof(req_ctx->fd_flt));
4244 		dpaa2_fl_set_final(in_fle, true);
4245 		dpaa2_fl_set_len(in_fle, to_hash);
4246 
4247 		if (mapped_nents > 1) {
4248 			int qm_sg_bytes;
4249 
4250 			sg_to_qm_sg_last(req->src, src_len, sg_table, 0);
4251 			qm_sg_bytes = pad_sg_nents(mapped_nents) *
4252 				      sizeof(*sg_table);
4253 			edesc->qm_sg_dma = dma_map_single(ctx->dev, sg_table,
4254 							  qm_sg_bytes,
4255 							  DMA_TO_DEVICE);
4256 			if (dma_mapping_error(ctx->dev, edesc->qm_sg_dma)) {
4257 				dev_err(ctx->dev, "unable to map S/G table\n");
4258 				ret = -ENOMEM;
4259 				goto unmap_ctx;
4260 			}
4261 			edesc->qm_sg_bytes = qm_sg_bytes;
4262 			dpaa2_fl_set_format(in_fle, dpaa2_fl_sg);
4263 			dpaa2_fl_set_addr(in_fle, edesc->qm_sg_dma);
4264 		} else {
4265 			dpaa2_fl_set_format(in_fle, dpaa2_fl_single);
4266 			dpaa2_fl_set_addr(in_fle, sg_dma_address(req->src));
4267 		}
4268 
4269 		state->ctx_dma_len = ctx->ctx_len;
4270 		state->ctx_dma = dma_map_single(ctx->dev, state->caam_ctx,
4271 						ctx->ctx_len, DMA_FROM_DEVICE);
4272 		if (dma_mapping_error(ctx->dev, state->ctx_dma)) {
4273 			dev_err(ctx->dev, "unable to map ctx\n");
4274 			state->ctx_dma = 0;
4275 			ret = -ENOMEM;
4276 			goto unmap_ctx;
4277 		}
4278 
4279 		dpaa2_fl_set_format(out_fle, dpaa2_fl_single);
4280 		dpaa2_fl_set_addr(out_fle, state->ctx_dma);
4281 		dpaa2_fl_set_len(out_fle, ctx->ctx_len);
4282 
4283 		req_ctx->flc = &ctx->flc[UPDATE_FIRST];
4284 		req_ctx->flc_dma = ctx->flc_dma[UPDATE_FIRST];
4285 		req_ctx->cbk = ahash_done_ctx_dst;
4286 		req_ctx->ctx = &req->base;
4287 		req_ctx->edesc = edesc;
4288 
4289 		ret = dpaa2_caam_enqueue(ctx->dev, req_ctx);
4290 		if (ret != -EINPROGRESS &&
4291 		    !(ret == -EBUSY && req->base.flags &
4292 		      CRYPTO_TFM_REQ_MAY_BACKLOG))
4293 			goto unmap_ctx;
4294 
4295 		state->update = ahash_update_ctx;
4296 		state->finup = ahash_finup_ctx;
4297 		state->final = ahash_final_ctx;
4298 	} else if (*next_buflen) {
4299 		state->update = ahash_update_no_ctx;
4300 		state->finup = ahash_finup_no_ctx;
4301 		state->final = ahash_final_no_ctx;
4302 		scatterwalk_map_and_copy(buf, req->src, 0,
4303 					 req->nbytes, 0);
4304 		*buflen = *next_buflen;
4305 
4306 		print_hex_dump_debug("buf@" __stringify(__LINE__)": ",
4307 				     DUMP_PREFIX_ADDRESS, 16, 4, buf,
4308 				     *buflen, 1);
4309 	}
4310 
4311 	return ret;
4312 unmap_ctx:
4313 	ahash_unmap_ctx(ctx->dev, edesc, req, DMA_TO_DEVICE);
4314 	qi_cache_free(edesc);
4315 	return ret;
4316 }
4317 
4318 static int ahash_finup_first(struct ahash_request *req)
4319 {
4320 	return ahash_digest(req);
4321 }
4322 
4323 static int ahash_init(struct ahash_request *req)
4324 {
4325 	struct caam_hash_state *state = ahash_request_ctx_dma(req);
4326 
4327 	state->update = ahash_update_first;
4328 	state->finup = ahash_finup_first;
4329 	state->final = ahash_final_no_ctx;
4330 
4331 	state->ctx_dma = 0;
4332 	state->ctx_dma_len = 0;
4333 	state->buf_dma = 0;
4334 	state->buflen = 0;
4335 	state->next_buflen = 0;
4336 
4337 	return 0;
4338 }
4339 
4340 static int ahash_update(struct ahash_request *req)
4341 {
4342 	struct caam_hash_state *state = ahash_request_ctx_dma(req);
4343 
4344 	return state->update(req);
4345 }
4346 
4347 static int ahash_finup(struct ahash_request *req)
4348 {
4349 	struct caam_hash_state *state = ahash_request_ctx_dma(req);
4350 
4351 	return state->finup(req);
4352 }
4353 
4354 static int ahash_final(struct ahash_request *req)
4355 {
4356 	struct caam_hash_state *state = ahash_request_ctx_dma(req);
4357 
4358 	return state->final(req);
4359 }
4360 
4361 static int ahash_export(struct ahash_request *req, void *out)
4362 {
4363 	struct caam_hash_state *state = ahash_request_ctx_dma(req);
4364 	struct caam_export_state *export = out;
4365 	u8 *buf = state->buf;
4366 	int len = state->buflen;
4367 
4368 	memcpy(export->buf, buf, len);
4369 	memcpy(export->caam_ctx, state->caam_ctx, sizeof(export->caam_ctx));
4370 	export->buflen = len;
4371 	export->update = state->update;
4372 	export->final = state->final;
4373 	export->finup = state->finup;
4374 
4375 	return 0;
4376 }
4377 
4378 static int ahash_import(struct ahash_request *req, const void *in)
4379 {
4380 	struct caam_hash_state *state = ahash_request_ctx_dma(req);
4381 	const struct caam_export_state *export = in;
4382 
4383 	memset(state, 0, sizeof(*state));
4384 	memcpy(state->buf, export->buf, export->buflen);
4385 	memcpy(state->caam_ctx, export->caam_ctx, sizeof(state->caam_ctx));
4386 	state->buflen = export->buflen;
4387 	state->update = export->update;
4388 	state->final = export->final;
4389 	state->finup = export->finup;
4390 
4391 	return 0;
4392 }
4393 
4394 struct caam_hash_template {
4395 	char name[CRYPTO_MAX_ALG_NAME];
4396 	char driver_name[CRYPTO_MAX_ALG_NAME];
4397 	char hmac_name[CRYPTO_MAX_ALG_NAME];
4398 	char hmac_driver_name[CRYPTO_MAX_ALG_NAME];
4399 	unsigned int blocksize;
4400 	struct ahash_alg template_ahash;
4401 	u32 alg_type;
4402 };
4403 
4404 /* ahash descriptors */
4405 static struct caam_hash_template driver_hash[] = {
4406 	{
4407 		.name = "sha1",
4408 		.driver_name = "sha1-caam-qi2",
4409 		.hmac_name = "hmac(sha1)",
4410 		.hmac_driver_name = "hmac-sha1-caam-qi2",
4411 		.blocksize = SHA1_BLOCK_SIZE,
4412 		.template_ahash = {
4413 			.init = ahash_init,
4414 			.update = ahash_update,
4415 			.final = ahash_final,
4416 			.finup = ahash_finup,
4417 			.digest = ahash_digest,
4418 			.export = ahash_export,
4419 			.import = ahash_import,
4420 			.setkey = ahash_setkey,
4421 			.halg = {
4422 				.digestsize = SHA1_DIGEST_SIZE,
4423 				.statesize = sizeof(struct caam_export_state),
4424 			},
4425 		},
4426 		.alg_type = OP_ALG_ALGSEL_SHA1,
4427 	}, {
4428 		.name = "sha224",
4429 		.driver_name = "sha224-caam-qi2",
4430 		.hmac_name = "hmac(sha224)",
4431 		.hmac_driver_name = "hmac-sha224-caam-qi2",
4432 		.blocksize = SHA224_BLOCK_SIZE,
4433 		.template_ahash = {
4434 			.init = ahash_init,
4435 			.update = ahash_update,
4436 			.final = ahash_final,
4437 			.finup = ahash_finup,
4438 			.digest = ahash_digest,
4439 			.export = ahash_export,
4440 			.import = ahash_import,
4441 			.setkey = ahash_setkey,
4442 			.halg = {
4443 				.digestsize = SHA224_DIGEST_SIZE,
4444 				.statesize = sizeof(struct caam_export_state),
4445 			},
4446 		},
4447 		.alg_type = OP_ALG_ALGSEL_SHA224,
4448 	}, {
4449 		.name = "sha256",
4450 		.driver_name = "sha256-caam-qi2",
4451 		.hmac_name = "hmac(sha256)",
4452 		.hmac_driver_name = "hmac-sha256-caam-qi2",
4453 		.blocksize = SHA256_BLOCK_SIZE,
4454 		.template_ahash = {
4455 			.init = ahash_init,
4456 			.update = ahash_update,
4457 			.final = ahash_final,
4458 			.finup = ahash_finup,
4459 			.digest = ahash_digest,
4460 			.export = ahash_export,
4461 			.import = ahash_import,
4462 			.setkey = ahash_setkey,
4463 			.halg = {
4464 				.digestsize = SHA256_DIGEST_SIZE,
4465 				.statesize = sizeof(struct caam_export_state),
4466 			},
4467 		},
4468 		.alg_type = OP_ALG_ALGSEL_SHA256,
4469 	}, {
4470 		.name = "sha384",
4471 		.driver_name = "sha384-caam-qi2",
4472 		.hmac_name = "hmac(sha384)",
4473 		.hmac_driver_name = "hmac-sha384-caam-qi2",
4474 		.blocksize = SHA384_BLOCK_SIZE,
4475 		.template_ahash = {
4476 			.init = ahash_init,
4477 			.update = ahash_update,
4478 			.final = ahash_final,
4479 			.finup = ahash_finup,
4480 			.digest = ahash_digest,
4481 			.export = ahash_export,
4482 			.import = ahash_import,
4483 			.setkey = ahash_setkey,
4484 			.halg = {
4485 				.digestsize = SHA384_DIGEST_SIZE,
4486 				.statesize = sizeof(struct caam_export_state),
4487 			},
4488 		},
4489 		.alg_type = OP_ALG_ALGSEL_SHA384,
4490 	}, {
4491 		.name = "sha512",
4492 		.driver_name = "sha512-caam-qi2",
4493 		.hmac_name = "hmac(sha512)",
4494 		.hmac_driver_name = "hmac-sha512-caam-qi2",
4495 		.blocksize = SHA512_BLOCK_SIZE,
4496 		.template_ahash = {
4497 			.init = ahash_init,
4498 			.update = ahash_update,
4499 			.final = ahash_final,
4500 			.finup = ahash_finup,
4501 			.digest = ahash_digest,
4502 			.export = ahash_export,
4503 			.import = ahash_import,
4504 			.setkey = ahash_setkey,
4505 			.halg = {
4506 				.digestsize = SHA512_DIGEST_SIZE,
4507 				.statesize = sizeof(struct caam_export_state),
4508 			},
4509 		},
4510 		.alg_type = OP_ALG_ALGSEL_SHA512,
4511 	}, {
4512 		.name = "md5",
4513 		.driver_name = "md5-caam-qi2",
4514 		.hmac_name = "hmac(md5)",
4515 		.hmac_driver_name = "hmac-md5-caam-qi2",
4516 		.blocksize = MD5_BLOCK_WORDS * 4,
4517 		.template_ahash = {
4518 			.init = ahash_init,
4519 			.update = ahash_update,
4520 			.final = ahash_final,
4521 			.finup = ahash_finup,
4522 			.digest = ahash_digest,
4523 			.export = ahash_export,
4524 			.import = ahash_import,
4525 			.setkey = ahash_setkey,
4526 			.halg = {
4527 				.digestsize = MD5_DIGEST_SIZE,
4528 				.statesize = sizeof(struct caam_export_state),
4529 			},
4530 		},
4531 		.alg_type = OP_ALG_ALGSEL_MD5,
4532 	}
4533 };
4534 
4535 struct caam_hash_alg {
4536 	struct list_head entry;
4537 	struct device *dev;
4538 	int alg_type;
4539 	struct ahash_alg ahash_alg;
4540 };
4541 
4542 static int caam_hash_cra_init(struct crypto_tfm *tfm)
4543 {
4544 	struct crypto_ahash *ahash = __crypto_ahash_cast(tfm);
4545 	struct crypto_alg *base = tfm->__crt_alg;
4546 	struct hash_alg_common *halg =
4547 		 container_of(base, struct hash_alg_common, base);
4548 	struct ahash_alg *alg =
4549 		 container_of(halg, struct ahash_alg, halg);
4550 	struct caam_hash_alg *caam_hash =
4551 		 container_of(alg, struct caam_hash_alg, ahash_alg);
4552 	struct caam_hash_ctx *ctx = crypto_tfm_ctx_dma(tfm);
4553 	/* Sizes for MDHA running digests: MD5, SHA1, 224, 256, 384, 512 */
4554 	static const u8 runninglen[] = { HASH_MSG_LEN + MD5_DIGEST_SIZE,
4555 					 HASH_MSG_LEN + SHA1_DIGEST_SIZE,
4556 					 HASH_MSG_LEN + 32,
4557 					 HASH_MSG_LEN + SHA256_DIGEST_SIZE,
4558 					 HASH_MSG_LEN + 64,
4559 					 HASH_MSG_LEN + SHA512_DIGEST_SIZE };
4560 	dma_addr_t dma_addr;
4561 	int i;
4562 
4563 	ctx->dev = caam_hash->dev;
4564 
4565 	if (alg->setkey) {
4566 		ctx->adata.key_dma = dma_map_single_attrs(ctx->dev, ctx->key,
4567 							  ARRAY_SIZE(ctx->key),
4568 							  DMA_TO_DEVICE,
4569 							  DMA_ATTR_SKIP_CPU_SYNC);
4570 		if (dma_mapping_error(ctx->dev, ctx->adata.key_dma)) {
4571 			dev_err(ctx->dev, "unable to map key\n");
4572 			return -ENOMEM;
4573 		}
4574 	}
4575 
4576 	dma_addr = dma_map_single_attrs(ctx->dev, ctx->flc, sizeof(ctx->flc),
4577 					DMA_BIDIRECTIONAL,
4578 					DMA_ATTR_SKIP_CPU_SYNC);
4579 	if (dma_mapping_error(ctx->dev, dma_addr)) {
4580 		dev_err(ctx->dev, "unable to map shared descriptors\n");
4581 		if (ctx->adata.key_dma)
4582 			dma_unmap_single_attrs(ctx->dev, ctx->adata.key_dma,
4583 					       ARRAY_SIZE(ctx->key),
4584 					       DMA_TO_DEVICE,
4585 					       DMA_ATTR_SKIP_CPU_SYNC);
4586 		return -ENOMEM;
4587 	}
4588 
4589 	for (i = 0; i < HASH_NUM_OP; i++)
4590 		ctx->flc_dma[i] = dma_addr + i * sizeof(ctx->flc[i]);
4591 
4592 	/* copy descriptor header template value */
4593 	ctx->adata.algtype = OP_TYPE_CLASS2_ALG | caam_hash->alg_type;
4594 
4595 	ctx->ctx_len = runninglen[(ctx->adata.algtype &
4596 				   OP_ALG_ALGSEL_SUBMASK) >>
4597 				  OP_ALG_ALGSEL_SHIFT];
4598 
4599 	crypto_ahash_set_reqsize_dma(ahash, sizeof(struct caam_hash_state));
4600 
4601 	/*
4602 	 * For keyed hash algorithms shared descriptors
4603 	 * will be created later in setkey() callback
4604 	 */
4605 	return alg->setkey ? 0 : ahash_set_sh_desc(ahash);
4606 }
4607 
4608 static void caam_hash_cra_exit(struct crypto_tfm *tfm)
4609 {
4610 	struct caam_hash_ctx *ctx = crypto_tfm_ctx_dma(tfm);
4611 
4612 	dma_unmap_single_attrs(ctx->dev, ctx->flc_dma[0], sizeof(ctx->flc),
4613 			       DMA_BIDIRECTIONAL, DMA_ATTR_SKIP_CPU_SYNC);
4614 	if (ctx->adata.key_dma)
4615 		dma_unmap_single_attrs(ctx->dev, ctx->adata.key_dma,
4616 				       ARRAY_SIZE(ctx->key), DMA_TO_DEVICE,
4617 				       DMA_ATTR_SKIP_CPU_SYNC);
4618 }
4619 
4620 static struct caam_hash_alg *caam_hash_alloc(struct device *dev,
4621 	struct caam_hash_template *template, bool keyed)
4622 {
4623 	struct caam_hash_alg *t_alg;
4624 	struct ahash_alg *halg;
4625 	struct crypto_alg *alg;
4626 
4627 	t_alg = kzalloc(sizeof(*t_alg), GFP_KERNEL);
4628 	if (!t_alg)
4629 		return ERR_PTR(-ENOMEM);
4630 
4631 	t_alg->ahash_alg = template->template_ahash;
4632 	halg = &t_alg->ahash_alg;
4633 	alg = &halg->halg.base;
4634 
4635 	if (keyed) {
4636 		snprintf(alg->cra_name, CRYPTO_MAX_ALG_NAME, "%s",
4637 			 template->hmac_name);
4638 		snprintf(alg->cra_driver_name, CRYPTO_MAX_ALG_NAME, "%s",
4639 			 template->hmac_driver_name);
4640 	} else {
4641 		snprintf(alg->cra_name, CRYPTO_MAX_ALG_NAME, "%s",
4642 			 template->name);
4643 		snprintf(alg->cra_driver_name, CRYPTO_MAX_ALG_NAME, "%s",
4644 			 template->driver_name);
4645 		t_alg->ahash_alg.setkey = NULL;
4646 	}
4647 	alg->cra_module = THIS_MODULE;
4648 	alg->cra_init = caam_hash_cra_init;
4649 	alg->cra_exit = caam_hash_cra_exit;
4650 	alg->cra_ctxsize = sizeof(struct caam_hash_ctx) + crypto_dma_padding();
4651 	alg->cra_priority = CAAM_CRA_PRIORITY;
4652 	alg->cra_blocksize = template->blocksize;
4653 	alg->cra_alignmask = 0;
4654 	alg->cra_flags = CRYPTO_ALG_ASYNC | CRYPTO_ALG_ALLOCATES_MEMORY;
4655 
4656 	t_alg->alg_type = template->alg_type;
4657 	t_alg->dev = dev;
4658 
4659 	return t_alg;
4660 }
4661 
4662 static void dpaa2_caam_fqdan_cb(struct dpaa2_io_notification_ctx *nctx)
4663 {
4664 	struct dpaa2_caam_priv_per_cpu *ppriv;
4665 
4666 	ppriv = container_of(nctx, struct dpaa2_caam_priv_per_cpu, nctx);
4667 	napi_schedule_irqoff(&ppriv->napi);
4668 }
4669 
4670 static int __cold dpaa2_dpseci_dpio_setup(struct dpaa2_caam_priv *priv)
4671 {
4672 	struct device *dev = priv->dev;
4673 	struct dpaa2_io_notification_ctx *nctx;
4674 	struct dpaa2_caam_priv_per_cpu *ppriv;
4675 	int err, i = 0, cpu;
4676 
4677 	for_each_online_cpu(cpu) {
4678 		ppriv = per_cpu_ptr(priv->ppriv, cpu);
4679 		ppriv->priv = priv;
4680 		nctx = &ppriv->nctx;
4681 		nctx->is_cdan = 0;
4682 		nctx->id = ppriv->rsp_fqid;
4683 		nctx->desired_cpu = cpu;
4684 		nctx->cb = dpaa2_caam_fqdan_cb;
4685 
4686 		/* Register notification callbacks */
4687 		ppriv->dpio = dpaa2_io_service_select(cpu);
4688 		err = dpaa2_io_service_register(ppriv->dpio, nctx, dev);
4689 		if (unlikely(err)) {
4690 			dev_dbg(dev, "No affine DPIO for cpu %d\n", cpu);
4691 			nctx->cb = NULL;
4692 			/*
4693 			 * If no affine DPIO for this core, there's probably
4694 			 * none available for next cores either. Signal we want
4695 			 * to retry later, in case the DPIO devices weren't
4696 			 * probed yet.
4697 			 */
4698 			err = -EPROBE_DEFER;
4699 			goto err;
4700 		}
4701 
4702 		ppriv->store = dpaa2_io_store_create(DPAA2_CAAM_STORE_SIZE,
4703 						     dev);
4704 		if (unlikely(!ppriv->store)) {
4705 			dev_err(dev, "dpaa2_io_store_create() failed\n");
4706 			err = -ENOMEM;
4707 			goto err;
4708 		}
4709 
4710 		if (++i == priv->num_pairs)
4711 			break;
4712 	}
4713 
4714 	return 0;
4715 
4716 err:
4717 	for_each_online_cpu(cpu) {
4718 		ppriv = per_cpu_ptr(priv->ppriv, cpu);
4719 		if (!ppriv->nctx.cb)
4720 			break;
4721 		dpaa2_io_service_deregister(ppriv->dpio, &ppriv->nctx, dev);
4722 	}
4723 
4724 	for_each_online_cpu(cpu) {
4725 		ppriv = per_cpu_ptr(priv->ppriv, cpu);
4726 		if (!ppriv->store)
4727 			break;
4728 		dpaa2_io_store_destroy(ppriv->store);
4729 	}
4730 
4731 	return err;
4732 }
4733 
4734 static void __cold dpaa2_dpseci_dpio_free(struct dpaa2_caam_priv *priv)
4735 {
4736 	struct dpaa2_caam_priv_per_cpu *ppriv;
4737 	int i = 0, cpu;
4738 
4739 	for_each_online_cpu(cpu) {
4740 		ppriv = per_cpu_ptr(priv->ppriv, cpu);
4741 		dpaa2_io_service_deregister(ppriv->dpio, &ppriv->nctx,
4742 					    priv->dev);
4743 		dpaa2_io_store_destroy(ppriv->store);
4744 
4745 		if (++i == priv->num_pairs)
4746 			return;
4747 	}
4748 }
4749 
4750 static int dpaa2_dpseci_bind(struct dpaa2_caam_priv *priv)
4751 {
4752 	struct dpseci_rx_queue_cfg rx_queue_cfg;
4753 	struct device *dev = priv->dev;
4754 	struct fsl_mc_device *ls_dev = to_fsl_mc_device(dev);
4755 	struct dpaa2_caam_priv_per_cpu *ppriv;
4756 	int err = 0, i = 0, cpu;
4757 
4758 	/* Configure Rx queues */
4759 	for_each_online_cpu(cpu) {
4760 		ppriv = per_cpu_ptr(priv->ppriv, cpu);
4761 
4762 		rx_queue_cfg.options = DPSECI_QUEUE_OPT_DEST |
4763 				       DPSECI_QUEUE_OPT_USER_CTX;
4764 		rx_queue_cfg.order_preservation_en = 0;
4765 		rx_queue_cfg.dest_cfg.dest_type = DPSECI_DEST_DPIO;
4766 		rx_queue_cfg.dest_cfg.dest_id = ppriv->nctx.dpio_id;
4767 		/*
4768 		 * Rx priority (WQ) doesn't really matter, since we use
4769 		 * pull mode, i.e. volatile dequeues from specific FQs
4770 		 */
4771 		rx_queue_cfg.dest_cfg.priority = 0;
4772 		rx_queue_cfg.user_ctx = ppriv->nctx.qman64;
4773 
4774 		err = dpseci_set_rx_queue(priv->mc_io, 0, ls_dev->mc_handle, i,
4775 					  &rx_queue_cfg);
4776 		if (err) {
4777 			dev_err(dev, "dpseci_set_rx_queue() failed with err %d\n",
4778 				err);
4779 			return err;
4780 		}
4781 
4782 		if (++i == priv->num_pairs)
4783 			break;
4784 	}
4785 
4786 	return err;
4787 }
4788 
4789 static void dpaa2_dpseci_congestion_free(struct dpaa2_caam_priv *priv)
4790 {
4791 	struct device *dev = priv->dev;
4792 
4793 	if (!priv->cscn_mem)
4794 		return;
4795 
4796 	dma_unmap_single(dev, priv->cscn_dma, DPAA2_CSCN_SIZE, DMA_FROM_DEVICE);
4797 	kfree(priv->cscn_mem);
4798 }
4799 
4800 static void dpaa2_dpseci_free(struct dpaa2_caam_priv *priv)
4801 {
4802 	struct device *dev = priv->dev;
4803 	struct fsl_mc_device *ls_dev = to_fsl_mc_device(dev);
4804 	int err;
4805 
4806 	if (DPSECI_VER(priv->major_ver, priv->minor_ver) > DPSECI_VER(5, 3)) {
4807 		err = dpseci_reset(priv->mc_io, 0, ls_dev->mc_handle);
4808 		if (err)
4809 			dev_err(dev, "dpseci_reset() failed\n");
4810 	}
4811 
4812 	dpaa2_dpseci_congestion_free(priv);
4813 	dpseci_close(priv->mc_io, 0, ls_dev->mc_handle);
4814 }
4815 
4816 static void dpaa2_caam_process_fd(struct dpaa2_caam_priv *priv,
4817 				  const struct dpaa2_fd *fd)
4818 {
4819 	struct caam_request *req;
4820 	u32 fd_err;
4821 
4822 	if (dpaa2_fd_get_format(fd) != dpaa2_fd_list) {
4823 		dev_err(priv->dev, "Only Frame List FD format is supported!\n");
4824 		return;
4825 	}
4826 
4827 	fd_err = dpaa2_fd_get_ctrl(fd) & FD_CTRL_ERR_MASK;
4828 	if (unlikely(fd_err))
4829 		dev_err_ratelimited(priv->dev, "FD error: %08x\n", fd_err);
4830 
4831 	/*
4832 	 * FD[ADDR] is guaranteed to be valid, irrespective of errors reported
4833 	 * in FD[ERR] or FD[FRC].
4834 	 */
4835 	req = dpaa2_caam_iova_to_virt(priv, dpaa2_fd_get_addr(fd));
4836 	dma_unmap_single(priv->dev, req->fd_flt_dma, sizeof(req->fd_flt),
4837 			 DMA_BIDIRECTIONAL);
4838 	req->cbk(req->ctx, dpaa2_fd_get_frc(fd));
4839 }
4840 
4841 static int dpaa2_caam_pull_fq(struct dpaa2_caam_priv_per_cpu *ppriv)
4842 {
4843 	int err;
4844 
4845 	/* Retry while portal is busy */
4846 	do {
4847 		err = dpaa2_io_service_pull_fq(ppriv->dpio, ppriv->rsp_fqid,
4848 					       ppriv->store);
4849 	} while (err == -EBUSY);
4850 
4851 	if (unlikely(err))
4852 		dev_err(ppriv->priv->dev, "dpaa2_io_service_pull err %d", err);
4853 
4854 	return err;
4855 }
4856 
4857 static int dpaa2_caam_store_consume(struct dpaa2_caam_priv_per_cpu *ppriv)
4858 {
4859 	struct dpaa2_dq *dq;
4860 	int cleaned = 0, is_last;
4861 
4862 	do {
4863 		dq = dpaa2_io_store_next(ppriv->store, &is_last);
4864 		if (unlikely(!dq)) {
4865 			if (unlikely(!is_last)) {
4866 				dev_dbg(ppriv->priv->dev,
4867 					"FQ %d returned no valid frames\n",
4868 					ppriv->rsp_fqid);
4869 				/*
4870 				 * MUST retry until we get some sort of
4871 				 * valid response token (be it "empty dequeue"
4872 				 * or a valid frame).
4873 				 */
4874 				continue;
4875 			}
4876 			break;
4877 		}
4878 
4879 		/* Process FD */
4880 		dpaa2_caam_process_fd(ppriv->priv, dpaa2_dq_fd(dq));
4881 		cleaned++;
4882 	} while (!is_last);
4883 
4884 	return cleaned;
4885 }
4886 
4887 static int dpaa2_dpseci_poll(struct napi_struct *napi, int budget)
4888 {
4889 	struct dpaa2_caam_priv_per_cpu *ppriv;
4890 	struct dpaa2_caam_priv *priv;
4891 	int err, cleaned = 0, store_cleaned;
4892 
4893 	ppriv = container_of(napi, struct dpaa2_caam_priv_per_cpu, napi);
4894 	priv = ppriv->priv;
4895 
4896 	if (unlikely(dpaa2_caam_pull_fq(ppriv)))
4897 		return 0;
4898 
4899 	do {
4900 		store_cleaned = dpaa2_caam_store_consume(ppriv);
4901 		cleaned += store_cleaned;
4902 
4903 		if (store_cleaned == 0 ||
4904 		    cleaned > budget - DPAA2_CAAM_STORE_SIZE)
4905 			break;
4906 
4907 		/* Try to dequeue some more */
4908 		err = dpaa2_caam_pull_fq(ppriv);
4909 		if (unlikely(err))
4910 			break;
4911 	} while (1);
4912 
4913 	if (cleaned < budget) {
4914 		napi_complete_done(napi, cleaned);
4915 		err = dpaa2_io_service_rearm(ppriv->dpio, &ppriv->nctx);
4916 		if (unlikely(err))
4917 			dev_err(priv->dev, "Notification rearm failed: %d\n",
4918 				err);
4919 	}
4920 
4921 	return cleaned;
4922 }
4923 
4924 static int dpaa2_dpseci_congestion_setup(struct dpaa2_caam_priv *priv,
4925 					 u16 token)
4926 {
4927 	struct dpseci_congestion_notification_cfg cong_notif_cfg = { 0 };
4928 	struct device *dev = priv->dev;
4929 	int err;
4930 
4931 	/*
4932 	 * Congestion group feature supported starting with DPSECI API v5.1
4933 	 * and only when object has been created with this capability.
4934 	 */
4935 	if ((DPSECI_VER(priv->major_ver, priv->minor_ver) < DPSECI_VER(5, 1)) ||
4936 	    !(priv->dpseci_attr.options & DPSECI_OPT_HAS_CG))
4937 		return 0;
4938 
4939 	priv->cscn_mem = kzalloc(DPAA2_CSCN_SIZE + DPAA2_CSCN_ALIGN,
4940 				 GFP_KERNEL | GFP_DMA);
4941 	if (!priv->cscn_mem)
4942 		return -ENOMEM;
4943 
4944 	priv->cscn_mem_aligned = PTR_ALIGN(priv->cscn_mem, DPAA2_CSCN_ALIGN);
4945 	priv->cscn_dma = dma_map_single(dev, priv->cscn_mem_aligned,
4946 					DPAA2_CSCN_SIZE, DMA_FROM_DEVICE);
4947 	if (dma_mapping_error(dev, priv->cscn_dma)) {
4948 		dev_err(dev, "Error mapping CSCN memory area\n");
4949 		err = -ENOMEM;
4950 		goto err_dma_map;
4951 	}
4952 
4953 	cong_notif_cfg.units = DPSECI_CONGESTION_UNIT_BYTES;
4954 	cong_notif_cfg.threshold_entry = DPAA2_SEC_CONG_ENTRY_THRESH;
4955 	cong_notif_cfg.threshold_exit = DPAA2_SEC_CONG_EXIT_THRESH;
4956 	cong_notif_cfg.message_ctx = (uintptr_t)priv;
4957 	cong_notif_cfg.message_iova = priv->cscn_dma;
4958 	cong_notif_cfg.notification_mode = DPSECI_CGN_MODE_WRITE_MEM_ON_ENTER |
4959 					DPSECI_CGN_MODE_WRITE_MEM_ON_EXIT |
4960 					DPSECI_CGN_MODE_COHERENT_WRITE;
4961 
4962 	err = dpseci_set_congestion_notification(priv->mc_io, 0, token,
4963 						 &cong_notif_cfg);
4964 	if (err) {
4965 		dev_err(dev, "dpseci_set_congestion_notification failed\n");
4966 		goto err_set_cong;
4967 	}
4968 
4969 	return 0;
4970 
4971 err_set_cong:
4972 	dma_unmap_single(dev, priv->cscn_dma, DPAA2_CSCN_SIZE, DMA_FROM_DEVICE);
4973 err_dma_map:
4974 	kfree(priv->cscn_mem);
4975 
4976 	return err;
4977 }
4978 
4979 static int __cold dpaa2_dpseci_setup(struct fsl_mc_device *ls_dev)
4980 {
4981 	struct device *dev = &ls_dev->dev;
4982 	struct dpaa2_caam_priv *priv;
4983 	struct dpaa2_caam_priv_per_cpu *ppriv;
4984 	int err, cpu;
4985 	u8 i;
4986 
4987 	priv = dev_get_drvdata(dev);
4988 
4989 	priv->dev = dev;
4990 	priv->dpsec_id = ls_dev->obj_desc.id;
4991 
4992 	/* Get a handle for the DPSECI this interface is associate with */
4993 	err = dpseci_open(priv->mc_io, 0, priv->dpsec_id, &ls_dev->mc_handle);
4994 	if (err) {
4995 		dev_err(dev, "dpseci_open() failed: %d\n", err);
4996 		goto err_open;
4997 	}
4998 
4999 	err = dpseci_get_api_version(priv->mc_io, 0, &priv->major_ver,
5000 				     &priv->minor_ver);
5001 	if (err) {
5002 		dev_err(dev, "dpseci_get_api_version() failed\n");
5003 		goto err_get_vers;
5004 	}
5005 
5006 	dev_info(dev, "dpseci v%d.%d\n", priv->major_ver, priv->minor_ver);
5007 
5008 	if (DPSECI_VER(priv->major_ver, priv->minor_ver) > DPSECI_VER(5, 3)) {
5009 		err = dpseci_reset(priv->mc_io, 0, ls_dev->mc_handle);
5010 		if (err) {
5011 			dev_err(dev, "dpseci_reset() failed\n");
5012 			goto err_get_vers;
5013 		}
5014 	}
5015 
5016 	err = dpseci_get_attributes(priv->mc_io, 0, ls_dev->mc_handle,
5017 				    &priv->dpseci_attr);
5018 	if (err) {
5019 		dev_err(dev, "dpseci_get_attributes() failed\n");
5020 		goto err_get_vers;
5021 	}
5022 
5023 	err = dpseci_get_sec_attr(priv->mc_io, 0, ls_dev->mc_handle,
5024 				  &priv->sec_attr);
5025 	if (err) {
5026 		dev_err(dev, "dpseci_get_sec_attr() failed\n");
5027 		goto err_get_vers;
5028 	}
5029 
5030 	err = dpaa2_dpseci_congestion_setup(priv, ls_dev->mc_handle);
5031 	if (err) {
5032 		dev_err(dev, "setup_congestion() failed\n");
5033 		goto err_get_vers;
5034 	}
5035 
5036 	priv->num_pairs = min(priv->dpseci_attr.num_rx_queues,
5037 			      priv->dpseci_attr.num_tx_queues);
5038 	if (priv->num_pairs > num_online_cpus()) {
5039 		dev_warn(dev, "%d queues won't be used\n",
5040 			 priv->num_pairs - num_online_cpus());
5041 		priv->num_pairs = num_online_cpus();
5042 	}
5043 
5044 	for (i = 0; i < priv->dpseci_attr.num_rx_queues; i++) {
5045 		err = dpseci_get_rx_queue(priv->mc_io, 0, ls_dev->mc_handle, i,
5046 					  &priv->rx_queue_attr[i]);
5047 		if (err) {
5048 			dev_err(dev, "dpseci_get_rx_queue() failed\n");
5049 			goto err_get_rx_queue;
5050 		}
5051 	}
5052 
5053 	for (i = 0; i < priv->dpseci_attr.num_tx_queues; i++) {
5054 		err = dpseci_get_tx_queue(priv->mc_io, 0, ls_dev->mc_handle, i,
5055 					  &priv->tx_queue_attr[i]);
5056 		if (err) {
5057 			dev_err(dev, "dpseci_get_tx_queue() failed\n");
5058 			goto err_get_rx_queue;
5059 		}
5060 	}
5061 
5062 	i = 0;
5063 	for_each_online_cpu(cpu) {
5064 		u8 j;
5065 
5066 		j = i % priv->num_pairs;
5067 
5068 		ppriv = per_cpu_ptr(priv->ppriv, cpu);
5069 		ppriv->req_fqid = priv->tx_queue_attr[j].fqid;
5070 
5071 		/*
5072 		 * Allow all cores to enqueue, while only some of them
5073 		 * will take part in dequeuing.
5074 		 */
5075 		if (++i > priv->num_pairs)
5076 			continue;
5077 
5078 		ppriv->rsp_fqid = priv->rx_queue_attr[j].fqid;
5079 		ppriv->prio = j;
5080 
5081 		dev_dbg(dev, "pair %d: rx queue %d, tx queue %d\n", j,
5082 			priv->rx_queue_attr[j].fqid,
5083 			priv->tx_queue_attr[j].fqid);
5084 
5085 		ppriv->net_dev.dev = *dev;
5086 		INIT_LIST_HEAD(&ppriv->net_dev.napi_list);
5087 		netif_napi_add_tx_weight(&ppriv->net_dev, &ppriv->napi,
5088 					 dpaa2_dpseci_poll,
5089 					 DPAA2_CAAM_NAPI_WEIGHT);
5090 	}
5091 
5092 	return 0;
5093 
5094 err_get_rx_queue:
5095 	dpaa2_dpseci_congestion_free(priv);
5096 err_get_vers:
5097 	dpseci_close(priv->mc_io, 0, ls_dev->mc_handle);
5098 err_open:
5099 	return err;
5100 }
5101 
5102 static int dpaa2_dpseci_enable(struct dpaa2_caam_priv *priv)
5103 {
5104 	struct device *dev = priv->dev;
5105 	struct fsl_mc_device *ls_dev = to_fsl_mc_device(dev);
5106 	struct dpaa2_caam_priv_per_cpu *ppriv;
5107 	int i;
5108 
5109 	for (i = 0; i < priv->num_pairs; i++) {
5110 		ppriv = per_cpu_ptr(priv->ppriv, i);
5111 		napi_enable(&ppriv->napi);
5112 	}
5113 
5114 	return dpseci_enable(priv->mc_io, 0, ls_dev->mc_handle);
5115 }
5116 
5117 static int __cold dpaa2_dpseci_disable(struct dpaa2_caam_priv *priv)
5118 {
5119 	struct device *dev = priv->dev;
5120 	struct dpaa2_caam_priv_per_cpu *ppriv;
5121 	struct fsl_mc_device *ls_dev = to_fsl_mc_device(dev);
5122 	int i, err = 0, enabled;
5123 
5124 	err = dpseci_disable(priv->mc_io, 0, ls_dev->mc_handle);
5125 	if (err) {
5126 		dev_err(dev, "dpseci_disable() failed\n");
5127 		return err;
5128 	}
5129 
5130 	err = dpseci_is_enabled(priv->mc_io, 0, ls_dev->mc_handle, &enabled);
5131 	if (err) {
5132 		dev_err(dev, "dpseci_is_enabled() failed\n");
5133 		return err;
5134 	}
5135 
5136 	dev_dbg(dev, "disable: %s\n", enabled ? "false" : "true");
5137 
5138 	for (i = 0; i < priv->num_pairs; i++) {
5139 		ppriv = per_cpu_ptr(priv->ppriv, i);
5140 		napi_disable(&ppriv->napi);
5141 		netif_napi_del(&ppriv->napi);
5142 	}
5143 
5144 	return 0;
5145 }
5146 
5147 static struct list_head hash_list;
5148 
5149 static int dpaa2_caam_probe(struct fsl_mc_device *dpseci_dev)
5150 {
5151 	struct device *dev;
5152 	struct dpaa2_caam_priv *priv;
5153 	int i, err = 0;
5154 	bool registered = false;
5155 
5156 	/*
5157 	 * There is no way to get CAAM endianness - there is no direct register
5158 	 * space access and MC f/w does not provide this attribute.
5159 	 * All DPAA2-based SoCs have little endian CAAM, thus hard-code this
5160 	 * property.
5161 	 */
5162 	caam_little_end = true;
5163 
5164 	caam_imx = false;
5165 
5166 	dev = &dpseci_dev->dev;
5167 
5168 	priv = devm_kzalloc(dev, sizeof(*priv), GFP_KERNEL);
5169 	if (!priv)
5170 		return -ENOMEM;
5171 
5172 	dev_set_drvdata(dev, priv);
5173 
5174 	priv->domain = iommu_get_domain_for_dev(dev);
5175 
5176 	qi_cache = kmem_cache_create("dpaa2_caamqicache", CAAM_QI_MEMCACHE_SIZE,
5177 				     0, SLAB_CACHE_DMA, NULL);
5178 	if (!qi_cache) {
5179 		dev_err(dev, "Can't allocate SEC cache\n");
5180 		return -ENOMEM;
5181 	}
5182 
5183 	err = dma_set_mask_and_coherent(dev, DMA_BIT_MASK(49));
5184 	if (err) {
5185 		dev_err(dev, "dma_set_mask_and_coherent() failed\n");
5186 		goto err_dma_mask;
5187 	}
5188 
5189 	/* Obtain a MC portal */
5190 	err = fsl_mc_portal_allocate(dpseci_dev, 0, &priv->mc_io);
5191 	if (err) {
5192 		if (err == -ENXIO)
5193 			err = -EPROBE_DEFER;
5194 		else
5195 			dev_err(dev, "MC portal allocation failed\n");
5196 
5197 		goto err_dma_mask;
5198 	}
5199 
5200 	priv->ppriv = alloc_percpu(*priv->ppriv);
5201 	if (!priv->ppriv) {
5202 		dev_err(dev, "alloc_percpu() failed\n");
5203 		err = -ENOMEM;
5204 		goto err_alloc_ppriv;
5205 	}
5206 
5207 	/* DPSECI initialization */
5208 	err = dpaa2_dpseci_setup(dpseci_dev);
5209 	if (err) {
5210 		dev_err(dev, "dpaa2_dpseci_setup() failed\n");
5211 		goto err_dpseci_setup;
5212 	}
5213 
5214 	/* DPIO */
5215 	err = dpaa2_dpseci_dpio_setup(priv);
5216 	if (err) {
5217 		dev_err_probe(dev, err, "dpaa2_dpseci_dpio_setup() failed\n");
5218 		goto err_dpio_setup;
5219 	}
5220 
5221 	/* DPSECI binding to DPIO */
5222 	err = dpaa2_dpseci_bind(priv);
5223 	if (err) {
5224 		dev_err(dev, "dpaa2_dpseci_bind() failed\n");
5225 		goto err_bind;
5226 	}
5227 
5228 	/* DPSECI enable */
5229 	err = dpaa2_dpseci_enable(priv);
5230 	if (err) {
5231 		dev_err(dev, "dpaa2_dpseci_enable() failed\n");
5232 		goto err_bind;
5233 	}
5234 
5235 	dpaa2_dpseci_debugfs_init(priv);
5236 
5237 	/* register crypto algorithms the device supports */
5238 	for (i = 0; i < ARRAY_SIZE(driver_algs); i++) {
5239 		struct caam_skcipher_alg *t_alg = driver_algs + i;
5240 		u32 alg_sel = t_alg->caam.class1_alg_type & OP_ALG_ALGSEL_MASK;
5241 
5242 		/* Skip DES algorithms if not supported by device */
5243 		if (!priv->sec_attr.des_acc_num &&
5244 		    (alg_sel == OP_ALG_ALGSEL_3DES ||
5245 		     alg_sel == OP_ALG_ALGSEL_DES))
5246 			continue;
5247 
5248 		/* Skip AES algorithms if not supported by device */
5249 		if (!priv->sec_attr.aes_acc_num &&
5250 		    alg_sel == OP_ALG_ALGSEL_AES)
5251 			continue;
5252 
5253 		/* Skip CHACHA20 algorithms if not supported by device */
5254 		if (alg_sel == OP_ALG_ALGSEL_CHACHA20 &&
5255 		    !priv->sec_attr.ccha_acc_num)
5256 			continue;
5257 
5258 		t_alg->caam.dev = dev;
5259 		caam_skcipher_alg_init(t_alg);
5260 
5261 		err = crypto_register_skcipher(&t_alg->skcipher);
5262 		if (err) {
5263 			dev_warn(dev, "%s alg registration failed: %d\n",
5264 				 t_alg->skcipher.base.cra_driver_name, err);
5265 			continue;
5266 		}
5267 
5268 		t_alg->registered = true;
5269 		registered = true;
5270 	}
5271 
5272 	for (i = 0; i < ARRAY_SIZE(driver_aeads); i++) {
5273 		struct caam_aead_alg *t_alg = driver_aeads + i;
5274 		u32 c1_alg_sel = t_alg->caam.class1_alg_type &
5275 				 OP_ALG_ALGSEL_MASK;
5276 		u32 c2_alg_sel = t_alg->caam.class2_alg_type &
5277 				 OP_ALG_ALGSEL_MASK;
5278 
5279 		/* Skip DES algorithms if not supported by device */
5280 		if (!priv->sec_attr.des_acc_num &&
5281 		    (c1_alg_sel == OP_ALG_ALGSEL_3DES ||
5282 		     c1_alg_sel == OP_ALG_ALGSEL_DES))
5283 			continue;
5284 
5285 		/* Skip AES algorithms if not supported by device */
5286 		if (!priv->sec_attr.aes_acc_num &&
5287 		    c1_alg_sel == OP_ALG_ALGSEL_AES)
5288 			continue;
5289 
5290 		/* Skip CHACHA20 algorithms if not supported by device */
5291 		if (c1_alg_sel == OP_ALG_ALGSEL_CHACHA20 &&
5292 		    !priv->sec_attr.ccha_acc_num)
5293 			continue;
5294 
5295 		/* Skip POLY1305 algorithms if not supported by device */
5296 		if (c2_alg_sel == OP_ALG_ALGSEL_POLY1305 &&
5297 		    !priv->sec_attr.ptha_acc_num)
5298 			continue;
5299 
5300 		/*
5301 		 * Skip algorithms requiring message digests
5302 		 * if MD not supported by device.
5303 		 */
5304 		if ((c2_alg_sel & ~OP_ALG_ALGSEL_SUBMASK) == 0x40 &&
5305 		    !priv->sec_attr.md_acc_num)
5306 			continue;
5307 
5308 		t_alg->caam.dev = dev;
5309 		caam_aead_alg_init(t_alg);
5310 
5311 		err = crypto_register_aead(&t_alg->aead);
5312 		if (err) {
5313 			dev_warn(dev, "%s alg registration failed: %d\n",
5314 				 t_alg->aead.base.cra_driver_name, err);
5315 			continue;
5316 		}
5317 
5318 		t_alg->registered = true;
5319 		registered = true;
5320 	}
5321 	if (registered)
5322 		dev_info(dev, "algorithms registered in /proc/crypto\n");
5323 
5324 	/* register hash algorithms the device supports */
5325 	INIT_LIST_HEAD(&hash_list);
5326 
5327 	/*
5328 	 * Skip registration of any hashing algorithms if MD block
5329 	 * is not present.
5330 	 */
5331 	if (!priv->sec_attr.md_acc_num)
5332 		return 0;
5333 
5334 	for (i = 0; i < ARRAY_SIZE(driver_hash); i++) {
5335 		struct caam_hash_alg *t_alg;
5336 		struct caam_hash_template *alg = driver_hash + i;
5337 
5338 		/* register hmac version */
5339 		t_alg = caam_hash_alloc(dev, alg, true);
5340 		if (IS_ERR(t_alg)) {
5341 			err = PTR_ERR(t_alg);
5342 			dev_warn(dev, "%s hash alg allocation failed: %d\n",
5343 				 alg->hmac_driver_name, err);
5344 			continue;
5345 		}
5346 
5347 		err = crypto_register_ahash(&t_alg->ahash_alg);
5348 		if (err) {
5349 			dev_warn(dev, "%s alg registration failed: %d\n",
5350 				 t_alg->ahash_alg.halg.base.cra_driver_name,
5351 				 err);
5352 			kfree(t_alg);
5353 		} else {
5354 			list_add_tail(&t_alg->entry, &hash_list);
5355 		}
5356 
5357 		/* register unkeyed version */
5358 		t_alg = caam_hash_alloc(dev, alg, false);
5359 		if (IS_ERR(t_alg)) {
5360 			err = PTR_ERR(t_alg);
5361 			dev_warn(dev, "%s alg allocation failed: %d\n",
5362 				 alg->driver_name, err);
5363 			continue;
5364 		}
5365 
5366 		err = crypto_register_ahash(&t_alg->ahash_alg);
5367 		if (err) {
5368 			dev_warn(dev, "%s alg registration failed: %d\n",
5369 				 t_alg->ahash_alg.halg.base.cra_driver_name,
5370 				 err);
5371 			kfree(t_alg);
5372 		} else {
5373 			list_add_tail(&t_alg->entry, &hash_list);
5374 		}
5375 	}
5376 	if (!list_empty(&hash_list))
5377 		dev_info(dev, "hash algorithms registered in /proc/crypto\n");
5378 
5379 	return err;
5380 
5381 err_bind:
5382 	dpaa2_dpseci_dpio_free(priv);
5383 err_dpio_setup:
5384 	dpaa2_dpseci_free(priv);
5385 err_dpseci_setup:
5386 	free_percpu(priv->ppriv);
5387 err_alloc_ppriv:
5388 	fsl_mc_portal_free(priv->mc_io);
5389 err_dma_mask:
5390 	kmem_cache_destroy(qi_cache);
5391 
5392 	return err;
5393 }
5394 
5395 static int __cold dpaa2_caam_remove(struct fsl_mc_device *ls_dev)
5396 {
5397 	struct device *dev;
5398 	struct dpaa2_caam_priv *priv;
5399 	int i;
5400 
5401 	dev = &ls_dev->dev;
5402 	priv = dev_get_drvdata(dev);
5403 
5404 	dpaa2_dpseci_debugfs_exit(priv);
5405 
5406 	for (i = 0; i < ARRAY_SIZE(driver_aeads); i++) {
5407 		struct caam_aead_alg *t_alg = driver_aeads + i;
5408 
5409 		if (t_alg->registered)
5410 			crypto_unregister_aead(&t_alg->aead);
5411 	}
5412 
5413 	for (i = 0; i < ARRAY_SIZE(driver_algs); i++) {
5414 		struct caam_skcipher_alg *t_alg = driver_algs + i;
5415 
5416 		if (t_alg->registered)
5417 			crypto_unregister_skcipher(&t_alg->skcipher);
5418 	}
5419 
5420 	if (hash_list.next) {
5421 		struct caam_hash_alg *t_hash_alg, *p;
5422 
5423 		list_for_each_entry_safe(t_hash_alg, p, &hash_list, entry) {
5424 			crypto_unregister_ahash(&t_hash_alg->ahash_alg);
5425 			list_del(&t_hash_alg->entry);
5426 			kfree(t_hash_alg);
5427 		}
5428 	}
5429 
5430 	dpaa2_dpseci_disable(priv);
5431 	dpaa2_dpseci_dpio_free(priv);
5432 	dpaa2_dpseci_free(priv);
5433 	free_percpu(priv->ppriv);
5434 	fsl_mc_portal_free(priv->mc_io);
5435 	kmem_cache_destroy(qi_cache);
5436 
5437 	return 0;
5438 }
5439 
5440 int dpaa2_caam_enqueue(struct device *dev, struct caam_request *req)
5441 {
5442 	struct dpaa2_fd fd;
5443 	struct dpaa2_caam_priv *priv = dev_get_drvdata(dev);
5444 	struct dpaa2_caam_priv_per_cpu *ppriv;
5445 	int err = 0, i;
5446 
5447 	if (IS_ERR(req))
5448 		return PTR_ERR(req);
5449 
5450 	if (priv->cscn_mem) {
5451 		dma_sync_single_for_cpu(priv->dev, priv->cscn_dma,
5452 					DPAA2_CSCN_SIZE,
5453 					DMA_FROM_DEVICE);
5454 		if (unlikely(dpaa2_cscn_state_congested(priv->cscn_mem_aligned))) {
5455 			dev_dbg_ratelimited(dev, "Dropping request\n");
5456 			return -EBUSY;
5457 		}
5458 	}
5459 
5460 	dpaa2_fl_set_flc(&req->fd_flt[1], req->flc_dma);
5461 
5462 	req->fd_flt_dma = dma_map_single(dev, req->fd_flt, sizeof(req->fd_flt),
5463 					 DMA_BIDIRECTIONAL);
5464 	if (dma_mapping_error(dev, req->fd_flt_dma)) {
5465 		dev_err(dev, "DMA mapping error for QI enqueue request\n");
5466 		goto err_out;
5467 	}
5468 
5469 	memset(&fd, 0, sizeof(fd));
5470 	dpaa2_fd_set_format(&fd, dpaa2_fd_list);
5471 	dpaa2_fd_set_addr(&fd, req->fd_flt_dma);
5472 	dpaa2_fd_set_len(&fd, dpaa2_fl_get_len(&req->fd_flt[1]));
5473 	dpaa2_fd_set_flc(&fd, req->flc_dma);
5474 
5475 	ppriv = raw_cpu_ptr(priv->ppriv);
5476 	for (i = 0; i < (priv->dpseci_attr.num_tx_queues << 1); i++) {
5477 		err = dpaa2_io_service_enqueue_fq(ppriv->dpio, ppriv->req_fqid,
5478 						  &fd);
5479 		if (err != -EBUSY)
5480 			break;
5481 
5482 		cpu_relax();
5483 	}
5484 
5485 	if (unlikely(err)) {
5486 		dev_err_ratelimited(dev, "Error enqueuing frame: %d\n", err);
5487 		goto err_out;
5488 	}
5489 
5490 	return -EINPROGRESS;
5491 
5492 err_out:
5493 	dma_unmap_single(dev, req->fd_flt_dma, sizeof(req->fd_flt),
5494 			 DMA_BIDIRECTIONAL);
5495 	return -EIO;
5496 }
5497 EXPORT_SYMBOL(dpaa2_caam_enqueue);
5498 
5499 static const struct fsl_mc_device_id dpaa2_caam_match_id_table[] = {
5500 	{
5501 		.vendor = FSL_MC_VENDOR_FREESCALE,
5502 		.obj_type = "dpseci",
5503 	},
5504 	{ .vendor = 0x0 }
5505 };
5506 MODULE_DEVICE_TABLE(fslmc, dpaa2_caam_match_id_table);
5507 
5508 static struct fsl_mc_driver dpaa2_caam_driver = {
5509 	.driver = {
5510 		.name		= KBUILD_MODNAME,
5511 		.owner		= THIS_MODULE,
5512 	},
5513 	.probe		= dpaa2_caam_probe,
5514 	.remove		= dpaa2_caam_remove,
5515 	.match_id_table = dpaa2_caam_match_id_table
5516 };
5517 
5518 MODULE_LICENSE("Dual BSD/GPL");
5519 MODULE_AUTHOR("Freescale Semiconductor, Inc");
5520 MODULE_DESCRIPTION("Freescale DPAA2 CAAM Driver");
5521 
5522 module_fsl_mc_driver(dpaa2_caam_driver);
5523