1 // SPDX-License-Identifier: (GPL-2.0+ OR BSD-3-Clause)
2 /*
3  * Copyright 2015-2016 Freescale Semiconductor Inc.
4  * Copyright 2017-2019 NXP
5  */
6 
7 #include "compat.h"
8 #include "regs.h"
9 #include "caamalg_qi2.h"
10 #include "dpseci_cmd.h"
11 #include "desc_constr.h"
12 #include "error.h"
13 #include "sg_sw_sec4.h"
14 #include "sg_sw_qm2.h"
15 #include "key_gen.h"
16 #include "caamalg_desc.h"
17 #include "caamhash_desc.h"
18 #include "dpseci-debugfs.h"
19 #include <linux/fsl/mc.h>
20 #include <soc/fsl/dpaa2-io.h>
21 #include <soc/fsl/dpaa2-fd.h>
22 #include <crypto/xts.h>
23 #include <asm/unaligned.h>
24 
25 #define CAAM_CRA_PRIORITY	2000
26 
27 /* max key is sum of AES_MAX_KEY_SIZE, max split key size */
28 #define CAAM_MAX_KEY_SIZE	(AES_MAX_KEY_SIZE + CTR_RFC3686_NONCE_SIZE + \
29 				 SHA512_DIGEST_SIZE * 2)
30 
31 /*
32  * This is a a cache of buffers, from which the users of CAAM QI driver
33  * can allocate short buffers. It's speedier than doing kmalloc on the hotpath.
34  * NOTE: A more elegant solution would be to have some headroom in the frames
35  *       being processed. This can be added by the dpaa2-eth driver. This would
36  *       pose a problem for userspace application processing which cannot
37  *       know of this limitation. So for now, this will work.
38  * NOTE: The memcache is SMP-safe. No need to handle spinlocks in-here
39  */
40 static struct kmem_cache *qi_cache;
41 
42 struct caam_alg_entry {
43 	struct device *dev;
44 	int class1_alg_type;
45 	int class2_alg_type;
46 	bool rfc3686;
47 	bool geniv;
48 	bool nodkp;
49 };
50 
51 struct caam_aead_alg {
52 	struct aead_alg aead;
53 	struct caam_alg_entry caam;
54 	bool registered;
55 };
56 
57 struct caam_skcipher_alg {
58 	struct skcipher_alg skcipher;
59 	struct caam_alg_entry caam;
60 	bool registered;
61 };
62 
63 /**
64  * struct caam_ctx - per-session context
65  * @flc: Flow Contexts array
66  * @key:  [authentication key], encryption key
67  * @flc_dma: I/O virtual addresses of the Flow Contexts
68  * @key_dma: I/O virtual address of the key
69  * @dir: DMA direction for mapping key and Flow Contexts
70  * @dev: dpseci device
71  * @adata: authentication algorithm details
72  * @cdata: encryption algorithm details
73  * @authsize: authentication tag (a.k.a. ICV / MAC) size
74  * @xts_key_fallback: true if fallback tfm needs to be used due
75  *		      to unsupported xts key lengths
76  * @fallback: xts fallback tfm
77  */
78 struct caam_ctx {
79 	struct caam_flc flc[NUM_OP];
80 	u8 key[CAAM_MAX_KEY_SIZE];
81 	dma_addr_t flc_dma[NUM_OP];
82 	dma_addr_t key_dma;
83 	enum dma_data_direction dir;
84 	struct device *dev;
85 	struct alginfo adata;
86 	struct alginfo cdata;
87 	unsigned int authsize;
88 	bool xts_key_fallback;
89 	struct crypto_skcipher *fallback;
90 };
91 
92 static void *dpaa2_caam_iova_to_virt(struct dpaa2_caam_priv *priv,
93 				     dma_addr_t iova_addr)
94 {
95 	phys_addr_t phys_addr;
96 
97 	phys_addr = priv->domain ? iommu_iova_to_phys(priv->domain, iova_addr) :
98 				   iova_addr;
99 
100 	return phys_to_virt(phys_addr);
101 }
102 
103 /*
104  * qi_cache_zalloc - Allocate buffers from CAAM-QI cache
105  *
106  * Allocate data on the hotpath. Instead of using kzalloc, one can use the
107  * services of the CAAM QI memory cache (backed by kmem_cache). The buffers
108  * will have a size of CAAM_QI_MEMCACHE_SIZE, which should be sufficient for
109  * hosting 16 SG entries.
110  *
111  * @flags - flags that would be used for the equivalent kmalloc(..) call
112  *
113  * Returns a pointer to a retrieved buffer on success or NULL on failure.
114  */
115 static inline void *qi_cache_zalloc(gfp_t flags)
116 {
117 	return kmem_cache_zalloc(qi_cache, flags);
118 }
119 
120 /*
121  * qi_cache_free - Frees buffers allocated from CAAM-QI cache
122  *
123  * @obj - buffer previously allocated by qi_cache_zalloc
124  *
125  * No checking is being done, the call is a passthrough call to
126  * kmem_cache_free(...)
127  */
128 static inline void qi_cache_free(void *obj)
129 {
130 	kmem_cache_free(qi_cache, obj);
131 }
132 
133 static struct caam_request *to_caam_req(struct crypto_async_request *areq)
134 {
135 	switch (crypto_tfm_alg_type(areq->tfm)) {
136 	case CRYPTO_ALG_TYPE_SKCIPHER:
137 		return skcipher_request_ctx(skcipher_request_cast(areq));
138 	case CRYPTO_ALG_TYPE_AEAD:
139 		return aead_request_ctx(container_of(areq, struct aead_request,
140 						     base));
141 	case CRYPTO_ALG_TYPE_AHASH:
142 		return ahash_request_ctx(ahash_request_cast(areq));
143 	default:
144 		return ERR_PTR(-EINVAL);
145 	}
146 }
147 
148 static void caam_unmap(struct device *dev, struct scatterlist *src,
149 		       struct scatterlist *dst, int src_nents,
150 		       int dst_nents, dma_addr_t iv_dma, int ivsize,
151 		       enum dma_data_direction iv_dir, dma_addr_t qm_sg_dma,
152 		       int qm_sg_bytes)
153 {
154 	if (dst != src) {
155 		if (src_nents)
156 			dma_unmap_sg(dev, src, src_nents, DMA_TO_DEVICE);
157 		if (dst_nents)
158 			dma_unmap_sg(dev, dst, dst_nents, DMA_FROM_DEVICE);
159 	} else {
160 		dma_unmap_sg(dev, src, src_nents, DMA_BIDIRECTIONAL);
161 	}
162 
163 	if (iv_dma)
164 		dma_unmap_single(dev, iv_dma, ivsize, iv_dir);
165 
166 	if (qm_sg_bytes)
167 		dma_unmap_single(dev, qm_sg_dma, qm_sg_bytes, DMA_TO_DEVICE);
168 }
169 
170 static int aead_set_sh_desc(struct crypto_aead *aead)
171 {
172 	struct caam_aead_alg *alg = container_of(crypto_aead_alg(aead),
173 						 typeof(*alg), aead);
174 	struct caam_ctx *ctx = crypto_aead_ctx(aead);
175 	unsigned int ivsize = crypto_aead_ivsize(aead);
176 	struct device *dev = ctx->dev;
177 	struct dpaa2_caam_priv *priv = dev_get_drvdata(dev);
178 	struct caam_flc *flc;
179 	u32 *desc;
180 	u32 ctx1_iv_off = 0;
181 	u32 *nonce = NULL;
182 	unsigned int data_len[2];
183 	u32 inl_mask;
184 	const bool ctr_mode = ((ctx->cdata.algtype & OP_ALG_AAI_MASK) ==
185 			       OP_ALG_AAI_CTR_MOD128);
186 	const bool is_rfc3686 = alg->caam.rfc3686;
187 
188 	if (!ctx->cdata.keylen || !ctx->authsize)
189 		return 0;
190 
191 	/*
192 	 * AES-CTR needs to load IV in CONTEXT1 reg
193 	 * at an offset of 128bits (16bytes)
194 	 * CONTEXT1[255:128] = IV
195 	 */
196 	if (ctr_mode)
197 		ctx1_iv_off = 16;
198 
199 	/*
200 	 * RFC3686 specific:
201 	 *	CONTEXT1[255:128] = {NONCE, IV, COUNTER}
202 	 */
203 	if (is_rfc3686) {
204 		ctx1_iv_off = 16 + CTR_RFC3686_NONCE_SIZE;
205 		nonce = (u32 *)((void *)ctx->key + ctx->adata.keylen_pad +
206 				ctx->cdata.keylen - CTR_RFC3686_NONCE_SIZE);
207 	}
208 
209 	/*
210 	 * In case |user key| > |derived key|, using DKP<imm,imm> would result
211 	 * in invalid opcodes (last bytes of user key) in the resulting
212 	 * descriptor. Use DKP<ptr,imm> instead => both virtual and dma key
213 	 * addresses are needed.
214 	 */
215 	ctx->adata.key_virt = ctx->key;
216 	ctx->adata.key_dma = ctx->key_dma;
217 
218 	ctx->cdata.key_virt = ctx->key + ctx->adata.keylen_pad;
219 	ctx->cdata.key_dma = ctx->key_dma + ctx->adata.keylen_pad;
220 
221 	data_len[0] = ctx->adata.keylen_pad;
222 	data_len[1] = ctx->cdata.keylen;
223 
224 	/* aead_encrypt shared descriptor */
225 	if (desc_inline_query((alg->caam.geniv ? DESC_QI_AEAD_GIVENC_LEN :
226 						 DESC_QI_AEAD_ENC_LEN) +
227 			      (is_rfc3686 ? DESC_AEAD_CTR_RFC3686_LEN : 0),
228 			      DESC_JOB_IO_LEN, data_len, &inl_mask,
229 			      ARRAY_SIZE(data_len)) < 0)
230 		return -EINVAL;
231 
232 	ctx->adata.key_inline = !!(inl_mask & 1);
233 	ctx->cdata.key_inline = !!(inl_mask & 2);
234 
235 	flc = &ctx->flc[ENCRYPT];
236 	desc = flc->sh_desc;
237 
238 	if (alg->caam.geniv)
239 		cnstr_shdsc_aead_givencap(desc, &ctx->cdata, &ctx->adata,
240 					  ivsize, ctx->authsize, is_rfc3686,
241 					  nonce, ctx1_iv_off, true,
242 					  priv->sec_attr.era);
243 	else
244 		cnstr_shdsc_aead_encap(desc, &ctx->cdata, &ctx->adata,
245 				       ivsize, ctx->authsize, is_rfc3686, nonce,
246 				       ctx1_iv_off, true, priv->sec_attr.era);
247 
248 	flc->flc[1] = cpu_to_caam32(desc_len(desc)); /* SDL */
249 	dma_sync_single_for_device(dev, ctx->flc_dma[ENCRYPT],
250 				   sizeof(flc->flc) + desc_bytes(desc),
251 				   ctx->dir);
252 
253 	/* aead_decrypt shared descriptor */
254 	if (desc_inline_query(DESC_QI_AEAD_DEC_LEN +
255 			      (is_rfc3686 ? DESC_AEAD_CTR_RFC3686_LEN : 0),
256 			      DESC_JOB_IO_LEN, data_len, &inl_mask,
257 			      ARRAY_SIZE(data_len)) < 0)
258 		return -EINVAL;
259 
260 	ctx->adata.key_inline = !!(inl_mask & 1);
261 	ctx->cdata.key_inline = !!(inl_mask & 2);
262 
263 	flc = &ctx->flc[DECRYPT];
264 	desc = flc->sh_desc;
265 	cnstr_shdsc_aead_decap(desc, &ctx->cdata, &ctx->adata,
266 			       ivsize, ctx->authsize, alg->caam.geniv,
267 			       is_rfc3686, nonce, ctx1_iv_off, true,
268 			       priv->sec_attr.era);
269 	flc->flc[1] = cpu_to_caam32(desc_len(desc)); /* SDL */
270 	dma_sync_single_for_device(dev, ctx->flc_dma[DECRYPT],
271 				   sizeof(flc->flc) + desc_bytes(desc),
272 				   ctx->dir);
273 
274 	return 0;
275 }
276 
277 static int aead_setauthsize(struct crypto_aead *authenc, unsigned int authsize)
278 {
279 	struct caam_ctx *ctx = crypto_aead_ctx(authenc);
280 
281 	ctx->authsize = authsize;
282 	aead_set_sh_desc(authenc);
283 
284 	return 0;
285 }
286 
287 static int aead_setkey(struct crypto_aead *aead, const u8 *key,
288 		       unsigned int keylen)
289 {
290 	struct caam_ctx *ctx = crypto_aead_ctx(aead);
291 	struct device *dev = ctx->dev;
292 	struct crypto_authenc_keys keys;
293 
294 	if (crypto_authenc_extractkeys(&keys, key, keylen) != 0)
295 		goto badkey;
296 
297 	dev_dbg(dev, "keylen %d enckeylen %d authkeylen %d\n",
298 		keys.authkeylen + keys.enckeylen, keys.enckeylen,
299 		keys.authkeylen);
300 	print_hex_dump_debug("key in @" __stringify(__LINE__)": ",
301 			     DUMP_PREFIX_ADDRESS, 16, 4, key, keylen, 1);
302 
303 	ctx->adata.keylen = keys.authkeylen;
304 	ctx->adata.keylen_pad = split_key_len(ctx->adata.algtype &
305 					      OP_ALG_ALGSEL_MASK);
306 
307 	if (ctx->adata.keylen_pad + keys.enckeylen > CAAM_MAX_KEY_SIZE)
308 		goto badkey;
309 
310 	memcpy(ctx->key, keys.authkey, keys.authkeylen);
311 	memcpy(ctx->key + ctx->adata.keylen_pad, keys.enckey, keys.enckeylen);
312 	dma_sync_single_for_device(dev, ctx->key_dma, ctx->adata.keylen_pad +
313 				   keys.enckeylen, ctx->dir);
314 	print_hex_dump_debug("ctx.key@" __stringify(__LINE__)": ",
315 			     DUMP_PREFIX_ADDRESS, 16, 4, ctx->key,
316 			     ctx->adata.keylen_pad + keys.enckeylen, 1);
317 
318 	ctx->cdata.keylen = keys.enckeylen;
319 
320 	memzero_explicit(&keys, sizeof(keys));
321 	return aead_set_sh_desc(aead);
322 badkey:
323 	memzero_explicit(&keys, sizeof(keys));
324 	return -EINVAL;
325 }
326 
327 static int des3_aead_setkey(struct crypto_aead *aead, const u8 *key,
328 			    unsigned int keylen)
329 {
330 	struct crypto_authenc_keys keys;
331 	int err;
332 
333 	err = crypto_authenc_extractkeys(&keys, key, keylen);
334 	if (unlikely(err))
335 		goto out;
336 
337 	err = -EINVAL;
338 	if (keys.enckeylen != DES3_EDE_KEY_SIZE)
339 		goto out;
340 
341 	err = crypto_des3_ede_verify_key(crypto_aead_tfm(aead), keys.enckey) ?:
342 	      aead_setkey(aead, key, keylen);
343 
344 out:
345 	memzero_explicit(&keys, sizeof(keys));
346 	return err;
347 }
348 
349 static struct aead_edesc *aead_edesc_alloc(struct aead_request *req,
350 					   bool encrypt)
351 {
352 	struct crypto_aead *aead = crypto_aead_reqtfm(req);
353 	struct caam_request *req_ctx = aead_request_ctx(req);
354 	struct dpaa2_fl_entry *in_fle = &req_ctx->fd_flt[1];
355 	struct dpaa2_fl_entry *out_fle = &req_ctx->fd_flt[0];
356 	struct caam_ctx *ctx = crypto_aead_ctx(aead);
357 	struct caam_aead_alg *alg = container_of(crypto_aead_alg(aead),
358 						 typeof(*alg), aead);
359 	struct device *dev = ctx->dev;
360 	gfp_t flags = (req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP) ?
361 		      GFP_KERNEL : GFP_ATOMIC;
362 	int src_nents, mapped_src_nents, dst_nents = 0, mapped_dst_nents = 0;
363 	int src_len, dst_len = 0;
364 	struct aead_edesc *edesc;
365 	dma_addr_t qm_sg_dma, iv_dma = 0;
366 	int ivsize = 0;
367 	unsigned int authsize = ctx->authsize;
368 	int qm_sg_index = 0, qm_sg_nents = 0, qm_sg_bytes;
369 	int in_len, out_len;
370 	struct dpaa2_sg_entry *sg_table;
371 
372 	/* allocate space for base edesc, link tables and IV */
373 	edesc = qi_cache_zalloc(GFP_DMA | flags);
374 	if (unlikely(!edesc)) {
375 		dev_err(dev, "could not allocate extended descriptor\n");
376 		return ERR_PTR(-ENOMEM);
377 	}
378 
379 	if (unlikely(req->dst != req->src)) {
380 		src_len = req->assoclen + req->cryptlen;
381 		dst_len = src_len + (encrypt ? authsize : (-authsize));
382 
383 		src_nents = sg_nents_for_len(req->src, src_len);
384 		if (unlikely(src_nents < 0)) {
385 			dev_err(dev, "Insufficient bytes (%d) in src S/G\n",
386 				src_len);
387 			qi_cache_free(edesc);
388 			return ERR_PTR(src_nents);
389 		}
390 
391 		dst_nents = sg_nents_for_len(req->dst, dst_len);
392 		if (unlikely(dst_nents < 0)) {
393 			dev_err(dev, "Insufficient bytes (%d) in dst S/G\n",
394 				dst_len);
395 			qi_cache_free(edesc);
396 			return ERR_PTR(dst_nents);
397 		}
398 
399 		if (src_nents) {
400 			mapped_src_nents = dma_map_sg(dev, req->src, src_nents,
401 						      DMA_TO_DEVICE);
402 			if (unlikely(!mapped_src_nents)) {
403 				dev_err(dev, "unable to map source\n");
404 				qi_cache_free(edesc);
405 				return ERR_PTR(-ENOMEM);
406 			}
407 		} else {
408 			mapped_src_nents = 0;
409 		}
410 
411 		if (dst_nents) {
412 			mapped_dst_nents = dma_map_sg(dev, req->dst, dst_nents,
413 						      DMA_FROM_DEVICE);
414 			if (unlikely(!mapped_dst_nents)) {
415 				dev_err(dev, "unable to map destination\n");
416 				dma_unmap_sg(dev, req->src, src_nents,
417 					     DMA_TO_DEVICE);
418 				qi_cache_free(edesc);
419 				return ERR_PTR(-ENOMEM);
420 			}
421 		} else {
422 			mapped_dst_nents = 0;
423 		}
424 	} else {
425 		src_len = req->assoclen + req->cryptlen +
426 			  (encrypt ? authsize : 0);
427 
428 		src_nents = sg_nents_for_len(req->src, src_len);
429 		if (unlikely(src_nents < 0)) {
430 			dev_err(dev, "Insufficient bytes (%d) in src S/G\n",
431 				src_len);
432 			qi_cache_free(edesc);
433 			return ERR_PTR(src_nents);
434 		}
435 
436 		mapped_src_nents = dma_map_sg(dev, req->src, src_nents,
437 					      DMA_BIDIRECTIONAL);
438 		if (unlikely(!mapped_src_nents)) {
439 			dev_err(dev, "unable to map source\n");
440 			qi_cache_free(edesc);
441 			return ERR_PTR(-ENOMEM);
442 		}
443 	}
444 
445 	if ((alg->caam.rfc3686 && encrypt) || !alg->caam.geniv)
446 		ivsize = crypto_aead_ivsize(aead);
447 
448 	/*
449 	 * Create S/G table: req->assoclen, [IV,] req->src [, req->dst].
450 	 * Input is not contiguous.
451 	 * HW reads 4 S/G entries at a time; make sure the reads don't go beyond
452 	 * the end of the table by allocating more S/G entries. Logic:
453 	 * if (src != dst && output S/G)
454 	 *      pad output S/G, if needed
455 	 * else if (src == dst && S/G)
456 	 *      overlapping S/Gs; pad one of them
457 	 * else if (input S/G) ...
458 	 *      pad input S/G, if needed
459 	 */
460 	qm_sg_nents = 1 + !!ivsize + mapped_src_nents;
461 	if (mapped_dst_nents > 1)
462 		qm_sg_nents += pad_sg_nents(mapped_dst_nents);
463 	else if ((req->src == req->dst) && (mapped_src_nents > 1))
464 		qm_sg_nents = max(pad_sg_nents(qm_sg_nents),
465 				  1 + !!ivsize +
466 				  pad_sg_nents(mapped_src_nents));
467 	else
468 		qm_sg_nents = pad_sg_nents(qm_sg_nents);
469 
470 	sg_table = &edesc->sgt[0];
471 	qm_sg_bytes = qm_sg_nents * sizeof(*sg_table);
472 	if (unlikely(offsetof(struct aead_edesc, sgt) + qm_sg_bytes + ivsize >
473 		     CAAM_QI_MEMCACHE_SIZE)) {
474 		dev_err(dev, "No space for %d S/G entries and/or %dB IV\n",
475 			qm_sg_nents, ivsize);
476 		caam_unmap(dev, req->src, req->dst, src_nents, dst_nents, 0,
477 			   0, DMA_NONE, 0, 0);
478 		qi_cache_free(edesc);
479 		return ERR_PTR(-ENOMEM);
480 	}
481 
482 	if (ivsize) {
483 		u8 *iv = (u8 *)(sg_table + qm_sg_nents);
484 
485 		/* Make sure IV is located in a DMAable area */
486 		memcpy(iv, req->iv, ivsize);
487 
488 		iv_dma = dma_map_single(dev, iv, ivsize, DMA_TO_DEVICE);
489 		if (dma_mapping_error(dev, iv_dma)) {
490 			dev_err(dev, "unable to map IV\n");
491 			caam_unmap(dev, req->src, req->dst, src_nents,
492 				   dst_nents, 0, 0, DMA_NONE, 0, 0);
493 			qi_cache_free(edesc);
494 			return ERR_PTR(-ENOMEM);
495 		}
496 	}
497 
498 	edesc->src_nents = src_nents;
499 	edesc->dst_nents = dst_nents;
500 	edesc->iv_dma = iv_dma;
501 
502 	if ((alg->caam.class1_alg_type & OP_ALG_ALGSEL_MASK) ==
503 	    OP_ALG_ALGSEL_CHACHA20 && ivsize != CHACHAPOLY_IV_SIZE)
504 		/*
505 		 * The associated data comes already with the IV but we need
506 		 * to skip it when we authenticate or encrypt...
507 		 */
508 		edesc->assoclen = cpu_to_caam32(req->assoclen - ivsize);
509 	else
510 		edesc->assoclen = cpu_to_caam32(req->assoclen);
511 	edesc->assoclen_dma = dma_map_single(dev, &edesc->assoclen, 4,
512 					     DMA_TO_DEVICE);
513 	if (dma_mapping_error(dev, edesc->assoclen_dma)) {
514 		dev_err(dev, "unable to map assoclen\n");
515 		caam_unmap(dev, req->src, req->dst, src_nents, dst_nents,
516 			   iv_dma, ivsize, DMA_TO_DEVICE, 0, 0);
517 		qi_cache_free(edesc);
518 		return ERR_PTR(-ENOMEM);
519 	}
520 
521 	dma_to_qm_sg_one(sg_table, edesc->assoclen_dma, 4, 0);
522 	qm_sg_index++;
523 	if (ivsize) {
524 		dma_to_qm_sg_one(sg_table + qm_sg_index, iv_dma, ivsize, 0);
525 		qm_sg_index++;
526 	}
527 	sg_to_qm_sg_last(req->src, src_len, sg_table + qm_sg_index, 0);
528 	qm_sg_index += mapped_src_nents;
529 
530 	if (mapped_dst_nents > 1)
531 		sg_to_qm_sg_last(req->dst, dst_len, sg_table + qm_sg_index, 0);
532 
533 	qm_sg_dma = dma_map_single(dev, sg_table, qm_sg_bytes, DMA_TO_DEVICE);
534 	if (dma_mapping_error(dev, qm_sg_dma)) {
535 		dev_err(dev, "unable to map S/G table\n");
536 		dma_unmap_single(dev, edesc->assoclen_dma, 4, DMA_TO_DEVICE);
537 		caam_unmap(dev, req->src, req->dst, src_nents, dst_nents,
538 			   iv_dma, ivsize, DMA_TO_DEVICE, 0, 0);
539 		qi_cache_free(edesc);
540 		return ERR_PTR(-ENOMEM);
541 	}
542 
543 	edesc->qm_sg_dma = qm_sg_dma;
544 	edesc->qm_sg_bytes = qm_sg_bytes;
545 
546 	out_len = req->assoclen + req->cryptlen +
547 		  (encrypt ? ctx->authsize : (-ctx->authsize));
548 	in_len = 4 + ivsize + req->assoclen + req->cryptlen;
549 
550 	memset(&req_ctx->fd_flt, 0, sizeof(req_ctx->fd_flt));
551 	dpaa2_fl_set_final(in_fle, true);
552 	dpaa2_fl_set_format(in_fle, dpaa2_fl_sg);
553 	dpaa2_fl_set_addr(in_fle, qm_sg_dma);
554 	dpaa2_fl_set_len(in_fle, in_len);
555 
556 	if (req->dst == req->src) {
557 		if (mapped_src_nents == 1) {
558 			dpaa2_fl_set_format(out_fle, dpaa2_fl_single);
559 			dpaa2_fl_set_addr(out_fle, sg_dma_address(req->src));
560 		} else {
561 			dpaa2_fl_set_format(out_fle, dpaa2_fl_sg);
562 			dpaa2_fl_set_addr(out_fle, qm_sg_dma +
563 					  (1 + !!ivsize) * sizeof(*sg_table));
564 		}
565 	} else if (!mapped_dst_nents) {
566 		/*
567 		 * crypto engine requires the output entry to be present when
568 		 * "frame list" FD is used.
569 		 * Since engine does not support FMT=2'b11 (unused entry type),
570 		 * leaving out_fle zeroized is the best option.
571 		 */
572 		goto skip_out_fle;
573 	} else if (mapped_dst_nents == 1) {
574 		dpaa2_fl_set_format(out_fle, dpaa2_fl_single);
575 		dpaa2_fl_set_addr(out_fle, sg_dma_address(req->dst));
576 	} else {
577 		dpaa2_fl_set_format(out_fle, dpaa2_fl_sg);
578 		dpaa2_fl_set_addr(out_fle, qm_sg_dma + qm_sg_index *
579 				  sizeof(*sg_table));
580 	}
581 
582 	dpaa2_fl_set_len(out_fle, out_len);
583 
584 skip_out_fle:
585 	return edesc;
586 }
587 
588 static int chachapoly_set_sh_desc(struct crypto_aead *aead)
589 {
590 	struct caam_ctx *ctx = crypto_aead_ctx(aead);
591 	unsigned int ivsize = crypto_aead_ivsize(aead);
592 	struct device *dev = ctx->dev;
593 	struct caam_flc *flc;
594 	u32 *desc;
595 
596 	if (!ctx->cdata.keylen || !ctx->authsize)
597 		return 0;
598 
599 	flc = &ctx->flc[ENCRYPT];
600 	desc = flc->sh_desc;
601 	cnstr_shdsc_chachapoly(desc, &ctx->cdata, &ctx->adata, ivsize,
602 			       ctx->authsize, true, true);
603 	flc->flc[1] = cpu_to_caam32(desc_len(desc)); /* SDL */
604 	dma_sync_single_for_device(dev, ctx->flc_dma[ENCRYPT],
605 				   sizeof(flc->flc) + desc_bytes(desc),
606 				   ctx->dir);
607 
608 	flc = &ctx->flc[DECRYPT];
609 	desc = flc->sh_desc;
610 	cnstr_shdsc_chachapoly(desc, &ctx->cdata, &ctx->adata, ivsize,
611 			       ctx->authsize, false, true);
612 	flc->flc[1] = cpu_to_caam32(desc_len(desc)); /* SDL */
613 	dma_sync_single_for_device(dev, ctx->flc_dma[DECRYPT],
614 				   sizeof(flc->flc) + desc_bytes(desc),
615 				   ctx->dir);
616 
617 	return 0;
618 }
619 
620 static int chachapoly_setauthsize(struct crypto_aead *aead,
621 				  unsigned int authsize)
622 {
623 	struct caam_ctx *ctx = crypto_aead_ctx(aead);
624 
625 	if (authsize != POLY1305_DIGEST_SIZE)
626 		return -EINVAL;
627 
628 	ctx->authsize = authsize;
629 	return chachapoly_set_sh_desc(aead);
630 }
631 
632 static int chachapoly_setkey(struct crypto_aead *aead, const u8 *key,
633 			     unsigned int keylen)
634 {
635 	struct caam_ctx *ctx = crypto_aead_ctx(aead);
636 	unsigned int ivsize = crypto_aead_ivsize(aead);
637 	unsigned int saltlen = CHACHAPOLY_IV_SIZE - ivsize;
638 
639 	if (keylen != CHACHA_KEY_SIZE + saltlen)
640 		return -EINVAL;
641 
642 	ctx->cdata.key_virt = key;
643 	ctx->cdata.keylen = keylen - saltlen;
644 
645 	return chachapoly_set_sh_desc(aead);
646 }
647 
648 static int gcm_set_sh_desc(struct crypto_aead *aead)
649 {
650 	struct caam_ctx *ctx = crypto_aead_ctx(aead);
651 	struct device *dev = ctx->dev;
652 	unsigned int ivsize = crypto_aead_ivsize(aead);
653 	struct caam_flc *flc;
654 	u32 *desc;
655 	int rem_bytes = CAAM_DESC_BYTES_MAX - DESC_JOB_IO_LEN -
656 			ctx->cdata.keylen;
657 
658 	if (!ctx->cdata.keylen || !ctx->authsize)
659 		return 0;
660 
661 	/*
662 	 * AES GCM encrypt shared descriptor
663 	 * Job Descriptor and Shared Descriptor
664 	 * must fit into the 64-word Descriptor h/w Buffer
665 	 */
666 	if (rem_bytes >= DESC_QI_GCM_ENC_LEN) {
667 		ctx->cdata.key_inline = true;
668 		ctx->cdata.key_virt = ctx->key;
669 	} else {
670 		ctx->cdata.key_inline = false;
671 		ctx->cdata.key_dma = ctx->key_dma;
672 	}
673 
674 	flc = &ctx->flc[ENCRYPT];
675 	desc = flc->sh_desc;
676 	cnstr_shdsc_gcm_encap(desc, &ctx->cdata, ivsize, ctx->authsize, true);
677 	flc->flc[1] = cpu_to_caam32(desc_len(desc)); /* SDL */
678 	dma_sync_single_for_device(dev, ctx->flc_dma[ENCRYPT],
679 				   sizeof(flc->flc) + desc_bytes(desc),
680 				   ctx->dir);
681 
682 	/*
683 	 * Job Descriptor and Shared Descriptors
684 	 * must all fit into the 64-word Descriptor h/w Buffer
685 	 */
686 	if (rem_bytes >= DESC_QI_GCM_DEC_LEN) {
687 		ctx->cdata.key_inline = true;
688 		ctx->cdata.key_virt = ctx->key;
689 	} else {
690 		ctx->cdata.key_inline = false;
691 		ctx->cdata.key_dma = ctx->key_dma;
692 	}
693 
694 	flc = &ctx->flc[DECRYPT];
695 	desc = flc->sh_desc;
696 	cnstr_shdsc_gcm_decap(desc, &ctx->cdata, ivsize, ctx->authsize, true);
697 	flc->flc[1] = cpu_to_caam32(desc_len(desc)); /* SDL */
698 	dma_sync_single_for_device(dev, ctx->flc_dma[DECRYPT],
699 				   sizeof(flc->flc) + desc_bytes(desc),
700 				   ctx->dir);
701 
702 	return 0;
703 }
704 
705 static int gcm_setauthsize(struct crypto_aead *authenc, unsigned int authsize)
706 {
707 	struct caam_ctx *ctx = crypto_aead_ctx(authenc);
708 	int err;
709 
710 	err = crypto_gcm_check_authsize(authsize);
711 	if (err)
712 		return err;
713 
714 	ctx->authsize = authsize;
715 	gcm_set_sh_desc(authenc);
716 
717 	return 0;
718 }
719 
720 static int gcm_setkey(struct crypto_aead *aead,
721 		      const u8 *key, unsigned int keylen)
722 {
723 	struct caam_ctx *ctx = crypto_aead_ctx(aead);
724 	struct device *dev = ctx->dev;
725 	int ret;
726 
727 	ret = aes_check_keylen(keylen);
728 	if (ret)
729 		return ret;
730 	print_hex_dump_debug("key in @" __stringify(__LINE__)": ",
731 			     DUMP_PREFIX_ADDRESS, 16, 4, key, keylen, 1);
732 
733 	memcpy(ctx->key, key, keylen);
734 	dma_sync_single_for_device(dev, ctx->key_dma, keylen, ctx->dir);
735 	ctx->cdata.keylen = keylen;
736 
737 	return gcm_set_sh_desc(aead);
738 }
739 
740 static int rfc4106_set_sh_desc(struct crypto_aead *aead)
741 {
742 	struct caam_ctx *ctx = crypto_aead_ctx(aead);
743 	struct device *dev = ctx->dev;
744 	unsigned int ivsize = crypto_aead_ivsize(aead);
745 	struct caam_flc *flc;
746 	u32 *desc;
747 	int rem_bytes = CAAM_DESC_BYTES_MAX - DESC_JOB_IO_LEN -
748 			ctx->cdata.keylen;
749 
750 	if (!ctx->cdata.keylen || !ctx->authsize)
751 		return 0;
752 
753 	ctx->cdata.key_virt = ctx->key;
754 
755 	/*
756 	 * RFC4106 encrypt shared descriptor
757 	 * Job Descriptor and Shared Descriptor
758 	 * must fit into the 64-word Descriptor h/w Buffer
759 	 */
760 	if (rem_bytes >= DESC_QI_RFC4106_ENC_LEN) {
761 		ctx->cdata.key_inline = true;
762 	} else {
763 		ctx->cdata.key_inline = false;
764 		ctx->cdata.key_dma = ctx->key_dma;
765 	}
766 
767 	flc = &ctx->flc[ENCRYPT];
768 	desc = flc->sh_desc;
769 	cnstr_shdsc_rfc4106_encap(desc, &ctx->cdata, ivsize, ctx->authsize,
770 				  true);
771 	flc->flc[1] = cpu_to_caam32(desc_len(desc)); /* SDL */
772 	dma_sync_single_for_device(dev, ctx->flc_dma[ENCRYPT],
773 				   sizeof(flc->flc) + desc_bytes(desc),
774 				   ctx->dir);
775 
776 	/*
777 	 * Job Descriptor and Shared Descriptors
778 	 * must all fit into the 64-word Descriptor h/w Buffer
779 	 */
780 	if (rem_bytes >= DESC_QI_RFC4106_DEC_LEN) {
781 		ctx->cdata.key_inline = true;
782 	} else {
783 		ctx->cdata.key_inline = false;
784 		ctx->cdata.key_dma = ctx->key_dma;
785 	}
786 
787 	flc = &ctx->flc[DECRYPT];
788 	desc = flc->sh_desc;
789 	cnstr_shdsc_rfc4106_decap(desc, &ctx->cdata, ivsize, ctx->authsize,
790 				  true);
791 	flc->flc[1] = cpu_to_caam32(desc_len(desc)); /* SDL */
792 	dma_sync_single_for_device(dev, ctx->flc_dma[DECRYPT],
793 				   sizeof(flc->flc) + desc_bytes(desc),
794 				   ctx->dir);
795 
796 	return 0;
797 }
798 
799 static int rfc4106_setauthsize(struct crypto_aead *authenc,
800 			       unsigned int authsize)
801 {
802 	struct caam_ctx *ctx = crypto_aead_ctx(authenc);
803 	int err;
804 
805 	err = crypto_rfc4106_check_authsize(authsize);
806 	if (err)
807 		return err;
808 
809 	ctx->authsize = authsize;
810 	rfc4106_set_sh_desc(authenc);
811 
812 	return 0;
813 }
814 
815 static int rfc4106_setkey(struct crypto_aead *aead,
816 			  const u8 *key, unsigned int keylen)
817 {
818 	struct caam_ctx *ctx = crypto_aead_ctx(aead);
819 	struct device *dev = ctx->dev;
820 	int ret;
821 
822 	ret = aes_check_keylen(keylen - 4);
823 	if (ret)
824 		return ret;
825 
826 	print_hex_dump_debug("key in @" __stringify(__LINE__)": ",
827 			     DUMP_PREFIX_ADDRESS, 16, 4, key, keylen, 1);
828 
829 	memcpy(ctx->key, key, keylen);
830 	/*
831 	 * The last four bytes of the key material are used as the salt value
832 	 * in the nonce. Update the AES key length.
833 	 */
834 	ctx->cdata.keylen = keylen - 4;
835 	dma_sync_single_for_device(dev, ctx->key_dma, ctx->cdata.keylen,
836 				   ctx->dir);
837 
838 	return rfc4106_set_sh_desc(aead);
839 }
840 
841 static int rfc4543_set_sh_desc(struct crypto_aead *aead)
842 {
843 	struct caam_ctx *ctx = crypto_aead_ctx(aead);
844 	struct device *dev = ctx->dev;
845 	unsigned int ivsize = crypto_aead_ivsize(aead);
846 	struct caam_flc *flc;
847 	u32 *desc;
848 	int rem_bytes = CAAM_DESC_BYTES_MAX - DESC_JOB_IO_LEN -
849 			ctx->cdata.keylen;
850 
851 	if (!ctx->cdata.keylen || !ctx->authsize)
852 		return 0;
853 
854 	ctx->cdata.key_virt = ctx->key;
855 
856 	/*
857 	 * RFC4543 encrypt shared descriptor
858 	 * Job Descriptor and Shared Descriptor
859 	 * must fit into the 64-word Descriptor h/w Buffer
860 	 */
861 	if (rem_bytes >= DESC_QI_RFC4543_ENC_LEN) {
862 		ctx->cdata.key_inline = true;
863 	} else {
864 		ctx->cdata.key_inline = false;
865 		ctx->cdata.key_dma = ctx->key_dma;
866 	}
867 
868 	flc = &ctx->flc[ENCRYPT];
869 	desc = flc->sh_desc;
870 	cnstr_shdsc_rfc4543_encap(desc, &ctx->cdata, ivsize, ctx->authsize,
871 				  true);
872 	flc->flc[1] = cpu_to_caam32(desc_len(desc)); /* SDL */
873 	dma_sync_single_for_device(dev, ctx->flc_dma[ENCRYPT],
874 				   sizeof(flc->flc) + desc_bytes(desc),
875 				   ctx->dir);
876 
877 	/*
878 	 * Job Descriptor and Shared Descriptors
879 	 * must all fit into the 64-word Descriptor h/w Buffer
880 	 */
881 	if (rem_bytes >= DESC_QI_RFC4543_DEC_LEN) {
882 		ctx->cdata.key_inline = true;
883 	} else {
884 		ctx->cdata.key_inline = false;
885 		ctx->cdata.key_dma = ctx->key_dma;
886 	}
887 
888 	flc = &ctx->flc[DECRYPT];
889 	desc = flc->sh_desc;
890 	cnstr_shdsc_rfc4543_decap(desc, &ctx->cdata, ivsize, ctx->authsize,
891 				  true);
892 	flc->flc[1] = cpu_to_caam32(desc_len(desc)); /* SDL */
893 	dma_sync_single_for_device(dev, ctx->flc_dma[DECRYPT],
894 				   sizeof(flc->flc) + desc_bytes(desc),
895 				   ctx->dir);
896 
897 	return 0;
898 }
899 
900 static int rfc4543_setauthsize(struct crypto_aead *authenc,
901 			       unsigned int authsize)
902 {
903 	struct caam_ctx *ctx = crypto_aead_ctx(authenc);
904 
905 	if (authsize != 16)
906 		return -EINVAL;
907 
908 	ctx->authsize = authsize;
909 	rfc4543_set_sh_desc(authenc);
910 
911 	return 0;
912 }
913 
914 static int rfc4543_setkey(struct crypto_aead *aead,
915 			  const u8 *key, unsigned int keylen)
916 {
917 	struct caam_ctx *ctx = crypto_aead_ctx(aead);
918 	struct device *dev = ctx->dev;
919 	int ret;
920 
921 	ret = aes_check_keylen(keylen - 4);
922 	if (ret)
923 		return ret;
924 
925 	print_hex_dump_debug("key in @" __stringify(__LINE__)": ",
926 			     DUMP_PREFIX_ADDRESS, 16, 4, key, keylen, 1);
927 
928 	memcpy(ctx->key, key, keylen);
929 	/*
930 	 * The last four bytes of the key material are used as the salt value
931 	 * in the nonce. Update the AES key length.
932 	 */
933 	ctx->cdata.keylen = keylen - 4;
934 	dma_sync_single_for_device(dev, ctx->key_dma, ctx->cdata.keylen,
935 				   ctx->dir);
936 
937 	return rfc4543_set_sh_desc(aead);
938 }
939 
940 static int skcipher_setkey(struct crypto_skcipher *skcipher, const u8 *key,
941 			   unsigned int keylen, const u32 ctx1_iv_off)
942 {
943 	struct caam_ctx *ctx = crypto_skcipher_ctx(skcipher);
944 	struct caam_skcipher_alg *alg =
945 		container_of(crypto_skcipher_alg(skcipher),
946 			     struct caam_skcipher_alg, skcipher);
947 	struct device *dev = ctx->dev;
948 	struct caam_flc *flc;
949 	unsigned int ivsize = crypto_skcipher_ivsize(skcipher);
950 	u32 *desc;
951 	const bool is_rfc3686 = alg->caam.rfc3686;
952 
953 	print_hex_dump_debug("key in @" __stringify(__LINE__)": ",
954 			     DUMP_PREFIX_ADDRESS, 16, 4, key, keylen, 1);
955 
956 	ctx->cdata.keylen = keylen;
957 	ctx->cdata.key_virt = key;
958 	ctx->cdata.key_inline = true;
959 
960 	/* skcipher_encrypt shared descriptor */
961 	flc = &ctx->flc[ENCRYPT];
962 	desc = flc->sh_desc;
963 	cnstr_shdsc_skcipher_encap(desc, &ctx->cdata, ivsize, is_rfc3686,
964 				   ctx1_iv_off);
965 	flc->flc[1] = cpu_to_caam32(desc_len(desc)); /* SDL */
966 	dma_sync_single_for_device(dev, ctx->flc_dma[ENCRYPT],
967 				   sizeof(flc->flc) + desc_bytes(desc),
968 				   ctx->dir);
969 
970 	/* skcipher_decrypt shared descriptor */
971 	flc = &ctx->flc[DECRYPT];
972 	desc = flc->sh_desc;
973 	cnstr_shdsc_skcipher_decap(desc, &ctx->cdata, ivsize, is_rfc3686,
974 				   ctx1_iv_off);
975 	flc->flc[1] = cpu_to_caam32(desc_len(desc)); /* SDL */
976 	dma_sync_single_for_device(dev, ctx->flc_dma[DECRYPT],
977 				   sizeof(flc->flc) + desc_bytes(desc),
978 				   ctx->dir);
979 
980 	return 0;
981 }
982 
983 static int aes_skcipher_setkey(struct crypto_skcipher *skcipher,
984 			       const u8 *key, unsigned int keylen)
985 {
986 	int err;
987 
988 	err = aes_check_keylen(keylen);
989 	if (err)
990 		return err;
991 
992 	return skcipher_setkey(skcipher, key, keylen, 0);
993 }
994 
995 static int rfc3686_skcipher_setkey(struct crypto_skcipher *skcipher,
996 				   const u8 *key, unsigned int keylen)
997 {
998 	u32 ctx1_iv_off;
999 	int err;
1000 
1001 	/*
1002 	 * RFC3686 specific:
1003 	 *	| CONTEXT1[255:128] = {NONCE, IV, COUNTER}
1004 	 *	| *key = {KEY, NONCE}
1005 	 */
1006 	ctx1_iv_off = 16 + CTR_RFC3686_NONCE_SIZE;
1007 	keylen -= CTR_RFC3686_NONCE_SIZE;
1008 
1009 	err = aes_check_keylen(keylen);
1010 	if (err)
1011 		return err;
1012 
1013 	return skcipher_setkey(skcipher, key, keylen, ctx1_iv_off);
1014 }
1015 
1016 static int ctr_skcipher_setkey(struct crypto_skcipher *skcipher,
1017 			       const u8 *key, unsigned int keylen)
1018 {
1019 	u32 ctx1_iv_off;
1020 	int err;
1021 
1022 	/*
1023 	 * AES-CTR needs to load IV in CONTEXT1 reg
1024 	 * at an offset of 128bits (16bytes)
1025 	 * CONTEXT1[255:128] = IV
1026 	 */
1027 	ctx1_iv_off = 16;
1028 
1029 	err = aes_check_keylen(keylen);
1030 	if (err)
1031 		return err;
1032 
1033 	return skcipher_setkey(skcipher, key, keylen, ctx1_iv_off);
1034 }
1035 
1036 static int chacha20_skcipher_setkey(struct crypto_skcipher *skcipher,
1037 				    const u8 *key, unsigned int keylen)
1038 {
1039 	if (keylen != CHACHA_KEY_SIZE)
1040 		return -EINVAL;
1041 
1042 	return skcipher_setkey(skcipher, key, keylen, 0);
1043 }
1044 
1045 static int des_skcipher_setkey(struct crypto_skcipher *skcipher,
1046 			       const u8 *key, unsigned int keylen)
1047 {
1048 	return verify_skcipher_des_key(skcipher, key) ?:
1049 	       skcipher_setkey(skcipher, key, keylen, 0);
1050 }
1051 
1052 static int des3_skcipher_setkey(struct crypto_skcipher *skcipher,
1053 			        const u8 *key, unsigned int keylen)
1054 {
1055 	return verify_skcipher_des3_key(skcipher, key) ?:
1056 	       skcipher_setkey(skcipher, key, keylen, 0);
1057 }
1058 
1059 static int xts_skcipher_setkey(struct crypto_skcipher *skcipher, const u8 *key,
1060 			       unsigned int keylen)
1061 {
1062 	struct caam_ctx *ctx = crypto_skcipher_ctx(skcipher);
1063 	struct device *dev = ctx->dev;
1064 	struct dpaa2_caam_priv *priv = dev_get_drvdata(dev);
1065 	struct caam_flc *flc;
1066 	u32 *desc;
1067 	int err;
1068 
1069 	err = xts_verify_key(skcipher, key, keylen);
1070 	if (err) {
1071 		dev_dbg(dev, "key size mismatch\n");
1072 		return err;
1073 	}
1074 
1075 	if (keylen != 2 * AES_KEYSIZE_128 && keylen != 2 * AES_KEYSIZE_256)
1076 		ctx->xts_key_fallback = true;
1077 
1078 	if (priv->sec_attr.era <= 8 || ctx->xts_key_fallback) {
1079 		err = crypto_skcipher_setkey(ctx->fallback, key, keylen);
1080 		if (err)
1081 			return err;
1082 	}
1083 
1084 	ctx->cdata.keylen = keylen;
1085 	ctx->cdata.key_virt = key;
1086 	ctx->cdata.key_inline = true;
1087 
1088 	/* xts_skcipher_encrypt shared descriptor */
1089 	flc = &ctx->flc[ENCRYPT];
1090 	desc = flc->sh_desc;
1091 	cnstr_shdsc_xts_skcipher_encap(desc, &ctx->cdata);
1092 	flc->flc[1] = cpu_to_caam32(desc_len(desc)); /* SDL */
1093 	dma_sync_single_for_device(dev, ctx->flc_dma[ENCRYPT],
1094 				   sizeof(flc->flc) + desc_bytes(desc),
1095 				   ctx->dir);
1096 
1097 	/* xts_skcipher_decrypt shared descriptor */
1098 	flc = &ctx->flc[DECRYPT];
1099 	desc = flc->sh_desc;
1100 	cnstr_shdsc_xts_skcipher_decap(desc, &ctx->cdata);
1101 	flc->flc[1] = cpu_to_caam32(desc_len(desc)); /* SDL */
1102 	dma_sync_single_for_device(dev, ctx->flc_dma[DECRYPT],
1103 				   sizeof(flc->flc) + desc_bytes(desc),
1104 				   ctx->dir);
1105 
1106 	return 0;
1107 }
1108 
1109 static struct skcipher_edesc *skcipher_edesc_alloc(struct skcipher_request *req)
1110 {
1111 	struct crypto_skcipher *skcipher = crypto_skcipher_reqtfm(req);
1112 	struct caam_request *req_ctx = skcipher_request_ctx(req);
1113 	struct dpaa2_fl_entry *in_fle = &req_ctx->fd_flt[1];
1114 	struct dpaa2_fl_entry *out_fle = &req_ctx->fd_flt[0];
1115 	struct caam_ctx *ctx = crypto_skcipher_ctx(skcipher);
1116 	struct device *dev = ctx->dev;
1117 	gfp_t flags = (req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP) ?
1118 		       GFP_KERNEL : GFP_ATOMIC;
1119 	int src_nents, mapped_src_nents, dst_nents = 0, mapped_dst_nents = 0;
1120 	struct skcipher_edesc *edesc;
1121 	dma_addr_t iv_dma;
1122 	u8 *iv;
1123 	int ivsize = crypto_skcipher_ivsize(skcipher);
1124 	int dst_sg_idx, qm_sg_ents, qm_sg_bytes;
1125 	struct dpaa2_sg_entry *sg_table;
1126 
1127 	src_nents = sg_nents_for_len(req->src, req->cryptlen);
1128 	if (unlikely(src_nents < 0)) {
1129 		dev_err(dev, "Insufficient bytes (%d) in src S/G\n",
1130 			req->cryptlen);
1131 		return ERR_PTR(src_nents);
1132 	}
1133 
1134 	if (unlikely(req->dst != req->src)) {
1135 		dst_nents = sg_nents_for_len(req->dst, req->cryptlen);
1136 		if (unlikely(dst_nents < 0)) {
1137 			dev_err(dev, "Insufficient bytes (%d) in dst S/G\n",
1138 				req->cryptlen);
1139 			return ERR_PTR(dst_nents);
1140 		}
1141 
1142 		mapped_src_nents = dma_map_sg(dev, req->src, src_nents,
1143 					      DMA_TO_DEVICE);
1144 		if (unlikely(!mapped_src_nents)) {
1145 			dev_err(dev, "unable to map source\n");
1146 			return ERR_PTR(-ENOMEM);
1147 		}
1148 
1149 		mapped_dst_nents = dma_map_sg(dev, req->dst, dst_nents,
1150 					      DMA_FROM_DEVICE);
1151 		if (unlikely(!mapped_dst_nents)) {
1152 			dev_err(dev, "unable to map destination\n");
1153 			dma_unmap_sg(dev, req->src, src_nents, DMA_TO_DEVICE);
1154 			return ERR_PTR(-ENOMEM);
1155 		}
1156 	} else {
1157 		mapped_src_nents = dma_map_sg(dev, req->src, src_nents,
1158 					      DMA_BIDIRECTIONAL);
1159 		if (unlikely(!mapped_src_nents)) {
1160 			dev_err(dev, "unable to map source\n");
1161 			return ERR_PTR(-ENOMEM);
1162 		}
1163 	}
1164 
1165 	qm_sg_ents = 1 + mapped_src_nents;
1166 	dst_sg_idx = qm_sg_ents;
1167 
1168 	/*
1169 	 * Input, output HW S/G tables: [IV, src][dst, IV]
1170 	 * IV entries point to the same buffer
1171 	 * If src == dst, S/G entries are reused (S/G tables overlap)
1172 	 *
1173 	 * HW reads 4 S/G entries at a time; make sure the reads don't go beyond
1174 	 * the end of the table by allocating more S/G entries.
1175 	 */
1176 	if (req->src != req->dst)
1177 		qm_sg_ents += pad_sg_nents(mapped_dst_nents + 1);
1178 	else
1179 		qm_sg_ents = 1 + pad_sg_nents(qm_sg_ents);
1180 
1181 	qm_sg_bytes = qm_sg_ents * sizeof(struct dpaa2_sg_entry);
1182 	if (unlikely(offsetof(struct skcipher_edesc, sgt) + qm_sg_bytes +
1183 		     ivsize > CAAM_QI_MEMCACHE_SIZE)) {
1184 		dev_err(dev, "No space for %d S/G entries and/or %dB IV\n",
1185 			qm_sg_ents, ivsize);
1186 		caam_unmap(dev, req->src, req->dst, src_nents, dst_nents, 0,
1187 			   0, DMA_NONE, 0, 0);
1188 		return ERR_PTR(-ENOMEM);
1189 	}
1190 
1191 	/* allocate space for base edesc, link tables and IV */
1192 	edesc = qi_cache_zalloc(GFP_DMA | flags);
1193 	if (unlikely(!edesc)) {
1194 		dev_err(dev, "could not allocate extended descriptor\n");
1195 		caam_unmap(dev, req->src, req->dst, src_nents, dst_nents, 0,
1196 			   0, DMA_NONE, 0, 0);
1197 		return ERR_PTR(-ENOMEM);
1198 	}
1199 
1200 	/* Make sure IV is located in a DMAable area */
1201 	sg_table = &edesc->sgt[0];
1202 	iv = (u8 *)(sg_table + qm_sg_ents);
1203 	memcpy(iv, req->iv, ivsize);
1204 
1205 	iv_dma = dma_map_single(dev, iv, ivsize, DMA_BIDIRECTIONAL);
1206 	if (dma_mapping_error(dev, iv_dma)) {
1207 		dev_err(dev, "unable to map IV\n");
1208 		caam_unmap(dev, req->src, req->dst, src_nents, dst_nents, 0,
1209 			   0, DMA_NONE, 0, 0);
1210 		qi_cache_free(edesc);
1211 		return ERR_PTR(-ENOMEM);
1212 	}
1213 
1214 	edesc->src_nents = src_nents;
1215 	edesc->dst_nents = dst_nents;
1216 	edesc->iv_dma = iv_dma;
1217 	edesc->qm_sg_bytes = qm_sg_bytes;
1218 
1219 	dma_to_qm_sg_one(sg_table, iv_dma, ivsize, 0);
1220 	sg_to_qm_sg(req->src, req->cryptlen, sg_table + 1, 0);
1221 
1222 	if (req->src != req->dst)
1223 		sg_to_qm_sg(req->dst, req->cryptlen, sg_table + dst_sg_idx, 0);
1224 
1225 	dma_to_qm_sg_one(sg_table + dst_sg_idx + mapped_dst_nents, iv_dma,
1226 			 ivsize, 0);
1227 
1228 	edesc->qm_sg_dma = dma_map_single(dev, sg_table, edesc->qm_sg_bytes,
1229 					  DMA_TO_DEVICE);
1230 	if (dma_mapping_error(dev, edesc->qm_sg_dma)) {
1231 		dev_err(dev, "unable to map S/G table\n");
1232 		caam_unmap(dev, req->src, req->dst, src_nents, dst_nents,
1233 			   iv_dma, ivsize, DMA_BIDIRECTIONAL, 0, 0);
1234 		qi_cache_free(edesc);
1235 		return ERR_PTR(-ENOMEM);
1236 	}
1237 
1238 	memset(&req_ctx->fd_flt, 0, sizeof(req_ctx->fd_flt));
1239 	dpaa2_fl_set_final(in_fle, true);
1240 	dpaa2_fl_set_len(in_fle, req->cryptlen + ivsize);
1241 	dpaa2_fl_set_len(out_fle, req->cryptlen + ivsize);
1242 
1243 	dpaa2_fl_set_format(in_fle, dpaa2_fl_sg);
1244 	dpaa2_fl_set_addr(in_fle, edesc->qm_sg_dma);
1245 
1246 	dpaa2_fl_set_format(out_fle, dpaa2_fl_sg);
1247 
1248 	if (req->src == req->dst)
1249 		dpaa2_fl_set_addr(out_fle, edesc->qm_sg_dma +
1250 				  sizeof(*sg_table));
1251 	else
1252 		dpaa2_fl_set_addr(out_fle, edesc->qm_sg_dma + dst_sg_idx *
1253 				  sizeof(*sg_table));
1254 
1255 	return edesc;
1256 }
1257 
1258 static void aead_unmap(struct device *dev, struct aead_edesc *edesc,
1259 		       struct aead_request *req)
1260 {
1261 	struct crypto_aead *aead = crypto_aead_reqtfm(req);
1262 	int ivsize = crypto_aead_ivsize(aead);
1263 
1264 	caam_unmap(dev, req->src, req->dst, edesc->src_nents, edesc->dst_nents,
1265 		   edesc->iv_dma, ivsize, DMA_TO_DEVICE, edesc->qm_sg_dma,
1266 		   edesc->qm_sg_bytes);
1267 	dma_unmap_single(dev, edesc->assoclen_dma, 4, DMA_TO_DEVICE);
1268 }
1269 
1270 static void skcipher_unmap(struct device *dev, struct skcipher_edesc *edesc,
1271 			   struct skcipher_request *req)
1272 {
1273 	struct crypto_skcipher *skcipher = crypto_skcipher_reqtfm(req);
1274 	int ivsize = crypto_skcipher_ivsize(skcipher);
1275 
1276 	caam_unmap(dev, req->src, req->dst, edesc->src_nents, edesc->dst_nents,
1277 		   edesc->iv_dma, ivsize, DMA_BIDIRECTIONAL, edesc->qm_sg_dma,
1278 		   edesc->qm_sg_bytes);
1279 }
1280 
1281 static void aead_encrypt_done(void *cbk_ctx, u32 status)
1282 {
1283 	struct crypto_async_request *areq = cbk_ctx;
1284 	struct aead_request *req = container_of(areq, struct aead_request,
1285 						base);
1286 	struct caam_request *req_ctx = to_caam_req(areq);
1287 	struct aead_edesc *edesc = req_ctx->edesc;
1288 	struct crypto_aead *aead = crypto_aead_reqtfm(req);
1289 	struct caam_ctx *ctx = crypto_aead_ctx(aead);
1290 	int ecode = 0;
1291 
1292 	dev_dbg(ctx->dev, "%s %d: err 0x%x\n", __func__, __LINE__, status);
1293 
1294 	if (unlikely(status))
1295 		ecode = caam_qi2_strstatus(ctx->dev, status);
1296 
1297 	aead_unmap(ctx->dev, edesc, req);
1298 	qi_cache_free(edesc);
1299 	aead_request_complete(req, ecode);
1300 }
1301 
1302 static void aead_decrypt_done(void *cbk_ctx, u32 status)
1303 {
1304 	struct crypto_async_request *areq = cbk_ctx;
1305 	struct aead_request *req = container_of(areq, struct aead_request,
1306 						base);
1307 	struct caam_request *req_ctx = to_caam_req(areq);
1308 	struct aead_edesc *edesc = req_ctx->edesc;
1309 	struct crypto_aead *aead = crypto_aead_reqtfm(req);
1310 	struct caam_ctx *ctx = crypto_aead_ctx(aead);
1311 	int ecode = 0;
1312 
1313 	dev_dbg(ctx->dev, "%s %d: err 0x%x\n", __func__, __LINE__, status);
1314 
1315 	if (unlikely(status))
1316 		ecode = caam_qi2_strstatus(ctx->dev, status);
1317 
1318 	aead_unmap(ctx->dev, edesc, req);
1319 	qi_cache_free(edesc);
1320 	aead_request_complete(req, ecode);
1321 }
1322 
1323 static int aead_encrypt(struct aead_request *req)
1324 {
1325 	struct aead_edesc *edesc;
1326 	struct crypto_aead *aead = crypto_aead_reqtfm(req);
1327 	struct caam_ctx *ctx = crypto_aead_ctx(aead);
1328 	struct caam_request *caam_req = aead_request_ctx(req);
1329 	int ret;
1330 
1331 	/* allocate extended descriptor */
1332 	edesc = aead_edesc_alloc(req, true);
1333 	if (IS_ERR(edesc))
1334 		return PTR_ERR(edesc);
1335 
1336 	caam_req->flc = &ctx->flc[ENCRYPT];
1337 	caam_req->flc_dma = ctx->flc_dma[ENCRYPT];
1338 	caam_req->cbk = aead_encrypt_done;
1339 	caam_req->ctx = &req->base;
1340 	caam_req->edesc = edesc;
1341 	ret = dpaa2_caam_enqueue(ctx->dev, caam_req);
1342 	if (ret != -EINPROGRESS &&
1343 	    !(ret == -EBUSY && req->base.flags & CRYPTO_TFM_REQ_MAY_BACKLOG)) {
1344 		aead_unmap(ctx->dev, edesc, req);
1345 		qi_cache_free(edesc);
1346 	}
1347 
1348 	return ret;
1349 }
1350 
1351 static int aead_decrypt(struct aead_request *req)
1352 {
1353 	struct aead_edesc *edesc;
1354 	struct crypto_aead *aead = crypto_aead_reqtfm(req);
1355 	struct caam_ctx *ctx = crypto_aead_ctx(aead);
1356 	struct caam_request *caam_req = aead_request_ctx(req);
1357 	int ret;
1358 
1359 	/* allocate extended descriptor */
1360 	edesc = aead_edesc_alloc(req, false);
1361 	if (IS_ERR(edesc))
1362 		return PTR_ERR(edesc);
1363 
1364 	caam_req->flc = &ctx->flc[DECRYPT];
1365 	caam_req->flc_dma = ctx->flc_dma[DECRYPT];
1366 	caam_req->cbk = aead_decrypt_done;
1367 	caam_req->ctx = &req->base;
1368 	caam_req->edesc = edesc;
1369 	ret = dpaa2_caam_enqueue(ctx->dev, caam_req);
1370 	if (ret != -EINPROGRESS &&
1371 	    !(ret == -EBUSY && req->base.flags & CRYPTO_TFM_REQ_MAY_BACKLOG)) {
1372 		aead_unmap(ctx->dev, edesc, req);
1373 		qi_cache_free(edesc);
1374 	}
1375 
1376 	return ret;
1377 }
1378 
1379 static int ipsec_gcm_encrypt(struct aead_request *req)
1380 {
1381 	return crypto_ipsec_check_assoclen(req->assoclen) ? : aead_encrypt(req);
1382 }
1383 
1384 static int ipsec_gcm_decrypt(struct aead_request *req)
1385 {
1386 	return crypto_ipsec_check_assoclen(req->assoclen) ? : aead_decrypt(req);
1387 }
1388 
1389 static void skcipher_encrypt_done(void *cbk_ctx, u32 status)
1390 {
1391 	struct crypto_async_request *areq = cbk_ctx;
1392 	struct skcipher_request *req = skcipher_request_cast(areq);
1393 	struct caam_request *req_ctx = to_caam_req(areq);
1394 	struct crypto_skcipher *skcipher = crypto_skcipher_reqtfm(req);
1395 	struct caam_ctx *ctx = crypto_skcipher_ctx(skcipher);
1396 	struct skcipher_edesc *edesc = req_ctx->edesc;
1397 	int ecode = 0;
1398 	int ivsize = crypto_skcipher_ivsize(skcipher);
1399 
1400 	dev_dbg(ctx->dev, "%s %d: err 0x%x\n", __func__, __LINE__, status);
1401 
1402 	if (unlikely(status))
1403 		ecode = caam_qi2_strstatus(ctx->dev, status);
1404 
1405 	print_hex_dump_debug("dstiv  @" __stringify(__LINE__)": ",
1406 			     DUMP_PREFIX_ADDRESS, 16, 4, req->iv,
1407 			     edesc->src_nents > 1 ? 100 : ivsize, 1);
1408 	caam_dump_sg("dst    @" __stringify(__LINE__)": ",
1409 		     DUMP_PREFIX_ADDRESS, 16, 4, req->dst,
1410 		     edesc->dst_nents > 1 ? 100 : req->cryptlen, 1);
1411 
1412 	skcipher_unmap(ctx->dev, edesc, req);
1413 
1414 	/*
1415 	 * The crypto API expects us to set the IV (req->iv) to the last
1416 	 * ciphertext block (CBC mode) or last counter (CTR mode).
1417 	 * This is used e.g. by the CTS mode.
1418 	 */
1419 	if (!ecode)
1420 		memcpy(req->iv, (u8 *)&edesc->sgt[0] + edesc->qm_sg_bytes,
1421 		       ivsize);
1422 
1423 	qi_cache_free(edesc);
1424 	skcipher_request_complete(req, ecode);
1425 }
1426 
1427 static void skcipher_decrypt_done(void *cbk_ctx, u32 status)
1428 {
1429 	struct crypto_async_request *areq = cbk_ctx;
1430 	struct skcipher_request *req = skcipher_request_cast(areq);
1431 	struct caam_request *req_ctx = to_caam_req(areq);
1432 	struct crypto_skcipher *skcipher = crypto_skcipher_reqtfm(req);
1433 	struct caam_ctx *ctx = crypto_skcipher_ctx(skcipher);
1434 	struct skcipher_edesc *edesc = req_ctx->edesc;
1435 	int ecode = 0;
1436 	int ivsize = crypto_skcipher_ivsize(skcipher);
1437 
1438 	dev_dbg(ctx->dev, "%s %d: err 0x%x\n", __func__, __LINE__, status);
1439 
1440 	if (unlikely(status))
1441 		ecode = caam_qi2_strstatus(ctx->dev, status);
1442 
1443 	print_hex_dump_debug("dstiv  @" __stringify(__LINE__)": ",
1444 			     DUMP_PREFIX_ADDRESS, 16, 4, req->iv,
1445 			     edesc->src_nents > 1 ? 100 : ivsize, 1);
1446 	caam_dump_sg("dst    @" __stringify(__LINE__)": ",
1447 		     DUMP_PREFIX_ADDRESS, 16, 4, req->dst,
1448 		     edesc->dst_nents > 1 ? 100 : req->cryptlen, 1);
1449 
1450 	skcipher_unmap(ctx->dev, edesc, req);
1451 
1452 	/*
1453 	 * The crypto API expects us to set the IV (req->iv) to the last
1454 	 * ciphertext block (CBC mode) or last counter (CTR mode).
1455 	 * This is used e.g. by the CTS mode.
1456 	 */
1457 	if (!ecode)
1458 		memcpy(req->iv, (u8 *)&edesc->sgt[0] + edesc->qm_sg_bytes,
1459 		       ivsize);
1460 
1461 	qi_cache_free(edesc);
1462 	skcipher_request_complete(req, ecode);
1463 }
1464 
1465 static inline bool xts_skcipher_ivsize(struct skcipher_request *req)
1466 {
1467 	struct crypto_skcipher *skcipher = crypto_skcipher_reqtfm(req);
1468 	unsigned int ivsize = crypto_skcipher_ivsize(skcipher);
1469 
1470 	return !!get_unaligned((u64 *)(req->iv + (ivsize / 2)));
1471 }
1472 
1473 static int skcipher_encrypt(struct skcipher_request *req)
1474 {
1475 	struct skcipher_edesc *edesc;
1476 	struct crypto_skcipher *skcipher = crypto_skcipher_reqtfm(req);
1477 	struct caam_ctx *ctx = crypto_skcipher_ctx(skcipher);
1478 	struct caam_request *caam_req = skcipher_request_ctx(req);
1479 	struct dpaa2_caam_priv *priv = dev_get_drvdata(ctx->dev);
1480 	int ret;
1481 
1482 	/*
1483 	 * XTS is expected to return an error even for input length = 0
1484 	 * Note that the case input length < block size will be caught during
1485 	 * HW offloading and return an error.
1486 	 */
1487 	if (!req->cryptlen && !ctx->fallback)
1488 		return 0;
1489 
1490 	if (ctx->fallback && ((priv->sec_attr.era <= 8 && xts_skcipher_ivsize(req)) ||
1491 			      ctx->xts_key_fallback)) {
1492 		skcipher_request_set_tfm(&caam_req->fallback_req, ctx->fallback);
1493 		skcipher_request_set_callback(&caam_req->fallback_req,
1494 					      req->base.flags,
1495 					      req->base.complete,
1496 					      req->base.data);
1497 		skcipher_request_set_crypt(&caam_req->fallback_req, req->src,
1498 					   req->dst, req->cryptlen, req->iv);
1499 
1500 		return crypto_skcipher_encrypt(&caam_req->fallback_req);
1501 	}
1502 
1503 	/* allocate extended descriptor */
1504 	edesc = skcipher_edesc_alloc(req);
1505 	if (IS_ERR(edesc))
1506 		return PTR_ERR(edesc);
1507 
1508 	caam_req->flc = &ctx->flc[ENCRYPT];
1509 	caam_req->flc_dma = ctx->flc_dma[ENCRYPT];
1510 	caam_req->cbk = skcipher_encrypt_done;
1511 	caam_req->ctx = &req->base;
1512 	caam_req->edesc = edesc;
1513 	ret = dpaa2_caam_enqueue(ctx->dev, caam_req);
1514 	if (ret != -EINPROGRESS &&
1515 	    !(ret == -EBUSY && req->base.flags & CRYPTO_TFM_REQ_MAY_BACKLOG)) {
1516 		skcipher_unmap(ctx->dev, edesc, req);
1517 		qi_cache_free(edesc);
1518 	}
1519 
1520 	return ret;
1521 }
1522 
1523 static int skcipher_decrypt(struct skcipher_request *req)
1524 {
1525 	struct skcipher_edesc *edesc;
1526 	struct crypto_skcipher *skcipher = crypto_skcipher_reqtfm(req);
1527 	struct caam_ctx *ctx = crypto_skcipher_ctx(skcipher);
1528 	struct caam_request *caam_req = skcipher_request_ctx(req);
1529 	struct dpaa2_caam_priv *priv = dev_get_drvdata(ctx->dev);
1530 	int ret;
1531 
1532 	/*
1533 	 * XTS is expected to return an error even for input length = 0
1534 	 * Note that the case input length < block size will be caught during
1535 	 * HW offloading and return an error.
1536 	 */
1537 	if (!req->cryptlen && !ctx->fallback)
1538 		return 0;
1539 
1540 	if (ctx->fallback && ((priv->sec_attr.era <= 8 && xts_skcipher_ivsize(req)) ||
1541 			      ctx->xts_key_fallback)) {
1542 		skcipher_request_set_tfm(&caam_req->fallback_req, ctx->fallback);
1543 		skcipher_request_set_callback(&caam_req->fallback_req,
1544 					      req->base.flags,
1545 					      req->base.complete,
1546 					      req->base.data);
1547 		skcipher_request_set_crypt(&caam_req->fallback_req, req->src,
1548 					   req->dst, req->cryptlen, req->iv);
1549 
1550 		return crypto_skcipher_decrypt(&caam_req->fallback_req);
1551 	}
1552 
1553 	/* allocate extended descriptor */
1554 	edesc = skcipher_edesc_alloc(req);
1555 	if (IS_ERR(edesc))
1556 		return PTR_ERR(edesc);
1557 
1558 	caam_req->flc = &ctx->flc[DECRYPT];
1559 	caam_req->flc_dma = ctx->flc_dma[DECRYPT];
1560 	caam_req->cbk = skcipher_decrypt_done;
1561 	caam_req->ctx = &req->base;
1562 	caam_req->edesc = edesc;
1563 	ret = dpaa2_caam_enqueue(ctx->dev, caam_req);
1564 	if (ret != -EINPROGRESS &&
1565 	    !(ret == -EBUSY && req->base.flags & CRYPTO_TFM_REQ_MAY_BACKLOG)) {
1566 		skcipher_unmap(ctx->dev, edesc, req);
1567 		qi_cache_free(edesc);
1568 	}
1569 
1570 	return ret;
1571 }
1572 
1573 static int caam_cra_init(struct caam_ctx *ctx, struct caam_alg_entry *caam,
1574 			 bool uses_dkp)
1575 {
1576 	dma_addr_t dma_addr;
1577 	int i;
1578 
1579 	/* copy descriptor header template value */
1580 	ctx->cdata.algtype = OP_TYPE_CLASS1_ALG | caam->class1_alg_type;
1581 	ctx->adata.algtype = OP_TYPE_CLASS2_ALG | caam->class2_alg_type;
1582 
1583 	ctx->dev = caam->dev;
1584 	ctx->dir = uses_dkp ? DMA_BIDIRECTIONAL : DMA_TO_DEVICE;
1585 
1586 	dma_addr = dma_map_single_attrs(ctx->dev, ctx->flc,
1587 					offsetof(struct caam_ctx, flc_dma),
1588 					ctx->dir, DMA_ATTR_SKIP_CPU_SYNC);
1589 	if (dma_mapping_error(ctx->dev, dma_addr)) {
1590 		dev_err(ctx->dev, "unable to map key, shared descriptors\n");
1591 		return -ENOMEM;
1592 	}
1593 
1594 	for (i = 0; i < NUM_OP; i++)
1595 		ctx->flc_dma[i] = dma_addr + i * sizeof(ctx->flc[i]);
1596 	ctx->key_dma = dma_addr + NUM_OP * sizeof(ctx->flc[0]);
1597 
1598 	return 0;
1599 }
1600 
1601 static int caam_cra_init_skcipher(struct crypto_skcipher *tfm)
1602 {
1603 	struct skcipher_alg *alg = crypto_skcipher_alg(tfm);
1604 	struct caam_skcipher_alg *caam_alg =
1605 		container_of(alg, typeof(*caam_alg), skcipher);
1606 	struct caam_ctx *ctx = crypto_skcipher_ctx(tfm);
1607 	u32 alg_aai = caam_alg->caam.class1_alg_type & OP_ALG_AAI_MASK;
1608 	int ret = 0;
1609 
1610 	if (alg_aai == OP_ALG_AAI_XTS) {
1611 		const char *tfm_name = crypto_tfm_alg_name(&tfm->base);
1612 		struct crypto_skcipher *fallback;
1613 
1614 		fallback = crypto_alloc_skcipher(tfm_name, 0,
1615 						 CRYPTO_ALG_NEED_FALLBACK);
1616 		if (IS_ERR(fallback)) {
1617 			dev_err(caam_alg->caam.dev,
1618 				"Failed to allocate %s fallback: %ld\n",
1619 				tfm_name, PTR_ERR(fallback));
1620 			return PTR_ERR(fallback);
1621 		}
1622 
1623 		ctx->fallback = fallback;
1624 		crypto_skcipher_set_reqsize(tfm, sizeof(struct caam_request) +
1625 					    crypto_skcipher_reqsize(fallback));
1626 	} else {
1627 		crypto_skcipher_set_reqsize(tfm, sizeof(struct caam_request));
1628 	}
1629 
1630 	ret = caam_cra_init(ctx, &caam_alg->caam, false);
1631 	if (ret && ctx->fallback)
1632 		crypto_free_skcipher(ctx->fallback);
1633 
1634 	return ret;
1635 }
1636 
1637 static int caam_cra_init_aead(struct crypto_aead *tfm)
1638 {
1639 	struct aead_alg *alg = crypto_aead_alg(tfm);
1640 	struct caam_aead_alg *caam_alg = container_of(alg, typeof(*caam_alg),
1641 						      aead);
1642 
1643 	crypto_aead_set_reqsize(tfm, sizeof(struct caam_request));
1644 	return caam_cra_init(crypto_aead_ctx(tfm), &caam_alg->caam,
1645 			     !caam_alg->caam.nodkp);
1646 }
1647 
1648 static void caam_exit_common(struct caam_ctx *ctx)
1649 {
1650 	dma_unmap_single_attrs(ctx->dev, ctx->flc_dma[0],
1651 			       offsetof(struct caam_ctx, flc_dma), ctx->dir,
1652 			       DMA_ATTR_SKIP_CPU_SYNC);
1653 }
1654 
1655 static void caam_cra_exit(struct crypto_skcipher *tfm)
1656 {
1657 	struct caam_ctx *ctx = crypto_skcipher_ctx(tfm);
1658 
1659 	if (ctx->fallback)
1660 		crypto_free_skcipher(ctx->fallback);
1661 	caam_exit_common(ctx);
1662 }
1663 
1664 static void caam_cra_exit_aead(struct crypto_aead *tfm)
1665 {
1666 	caam_exit_common(crypto_aead_ctx(tfm));
1667 }
1668 
1669 static struct caam_skcipher_alg driver_algs[] = {
1670 	{
1671 		.skcipher = {
1672 			.base = {
1673 				.cra_name = "cbc(aes)",
1674 				.cra_driver_name = "cbc-aes-caam-qi2",
1675 				.cra_blocksize = AES_BLOCK_SIZE,
1676 			},
1677 			.setkey = aes_skcipher_setkey,
1678 			.encrypt = skcipher_encrypt,
1679 			.decrypt = skcipher_decrypt,
1680 			.min_keysize = AES_MIN_KEY_SIZE,
1681 			.max_keysize = AES_MAX_KEY_SIZE,
1682 			.ivsize = AES_BLOCK_SIZE,
1683 		},
1684 		.caam.class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CBC,
1685 	},
1686 	{
1687 		.skcipher = {
1688 			.base = {
1689 				.cra_name = "cbc(des3_ede)",
1690 				.cra_driver_name = "cbc-3des-caam-qi2",
1691 				.cra_blocksize = DES3_EDE_BLOCK_SIZE,
1692 			},
1693 			.setkey = des3_skcipher_setkey,
1694 			.encrypt = skcipher_encrypt,
1695 			.decrypt = skcipher_decrypt,
1696 			.min_keysize = DES3_EDE_KEY_SIZE,
1697 			.max_keysize = DES3_EDE_KEY_SIZE,
1698 			.ivsize = DES3_EDE_BLOCK_SIZE,
1699 		},
1700 		.caam.class1_alg_type = OP_ALG_ALGSEL_3DES | OP_ALG_AAI_CBC,
1701 	},
1702 	{
1703 		.skcipher = {
1704 			.base = {
1705 				.cra_name = "cbc(des)",
1706 				.cra_driver_name = "cbc-des-caam-qi2",
1707 				.cra_blocksize = DES_BLOCK_SIZE,
1708 			},
1709 			.setkey = des_skcipher_setkey,
1710 			.encrypt = skcipher_encrypt,
1711 			.decrypt = skcipher_decrypt,
1712 			.min_keysize = DES_KEY_SIZE,
1713 			.max_keysize = DES_KEY_SIZE,
1714 			.ivsize = DES_BLOCK_SIZE,
1715 		},
1716 		.caam.class1_alg_type = OP_ALG_ALGSEL_DES | OP_ALG_AAI_CBC,
1717 	},
1718 	{
1719 		.skcipher = {
1720 			.base = {
1721 				.cra_name = "ctr(aes)",
1722 				.cra_driver_name = "ctr-aes-caam-qi2",
1723 				.cra_blocksize = 1,
1724 			},
1725 			.setkey = ctr_skcipher_setkey,
1726 			.encrypt = skcipher_encrypt,
1727 			.decrypt = skcipher_decrypt,
1728 			.min_keysize = AES_MIN_KEY_SIZE,
1729 			.max_keysize = AES_MAX_KEY_SIZE,
1730 			.ivsize = AES_BLOCK_SIZE,
1731 			.chunksize = AES_BLOCK_SIZE,
1732 		},
1733 		.caam.class1_alg_type = OP_ALG_ALGSEL_AES |
1734 					OP_ALG_AAI_CTR_MOD128,
1735 	},
1736 	{
1737 		.skcipher = {
1738 			.base = {
1739 				.cra_name = "rfc3686(ctr(aes))",
1740 				.cra_driver_name = "rfc3686-ctr-aes-caam-qi2",
1741 				.cra_blocksize = 1,
1742 			},
1743 			.setkey = rfc3686_skcipher_setkey,
1744 			.encrypt = skcipher_encrypt,
1745 			.decrypt = skcipher_decrypt,
1746 			.min_keysize = AES_MIN_KEY_SIZE +
1747 				       CTR_RFC3686_NONCE_SIZE,
1748 			.max_keysize = AES_MAX_KEY_SIZE +
1749 				       CTR_RFC3686_NONCE_SIZE,
1750 			.ivsize = CTR_RFC3686_IV_SIZE,
1751 			.chunksize = AES_BLOCK_SIZE,
1752 		},
1753 		.caam = {
1754 			.class1_alg_type = OP_ALG_ALGSEL_AES |
1755 					   OP_ALG_AAI_CTR_MOD128,
1756 			.rfc3686 = true,
1757 		},
1758 	},
1759 	{
1760 		.skcipher = {
1761 			.base = {
1762 				.cra_name = "xts(aes)",
1763 				.cra_driver_name = "xts-aes-caam-qi2",
1764 				.cra_flags = CRYPTO_ALG_NEED_FALLBACK,
1765 				.cra_blocksize = AES_BLOCK_SIZE,
1766 			},
1767 			.setkey = xts_skcipher_setkey,
1768 			.encrypt = skcipher_encrypt,
1769 			.decrypt = skcipher_decrypt,
1770 			.min_keysize = 2 * AES_MIN_KEY_SIZE,
1771 			.max_keysize = 2 * AES_MAX_KEY_SIZE,
1772 			.ivsize = AES_BLOCK_SIZE,
1773 		},
1774 		.caam.class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_XTS,
1775 	},
1776 	{
1777 		.skcipher = {
1778 			.base = {
1779 				.cra_name = "chacha20",
1780 				.cra_driver_name = "chacha20-caam-qi2",
1781 				.cra_blocksize = 1,
1782 			},
1783 			.setkey = chacha20_skcipher_setkey,
1784 			.encrypt = skcipher_encrypt,
1785 			.decrypt = skcipher_decrypt,
1786 			.min_keysize = CHACHA_KEY_SIZE,
1787 			.max_keysize = CHACHA_KEY_SIZE,
1788 			.ivsize = CHACHA_IV_SIZE,
1789 		},
1790 		.caam.class1_alg_type = OP_ALG_ALGSEL_CHACHA20,
1791 	},
1792 };
1793 
1794 static struct caam_aead_alg driver_aeads[] = {
1795 	{
1796 		.aead = {
1797 			.base = {
1798 				.cra_name = "rfc4106(gcm(aes))",
1799 				.cra_driver_name = "rfc4106-gcm-aes-caam-qi2",
1800 				.cra_blocksize = 1,
1801 			},
1802 			.setkey = rfc4106_setkey,
1803 			.setauthsize = rfc4106_setauthsize,
1804 			.encrypt = ipsec_gcm_encrypt,
1805 			.decrypt = ipsec_gcm_decrypt,
1806 			.ivsize = 8,
1807 			.maxauthsize = AES_BLOCK_SIZE,
1808 		},
1809 		.caam = {
1810 			.class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_GCM,
1811 			.nodkp = true,
1812 		},
1813 	},
1814 	{
1815 		.aead = {
1816 			.base = {
1817 				.cra_name = "rfc4543(gcm(aes))",
1818 				.cra_driver_name = "rfc4543-gcm-aes-caam-qi2",
1819 				.cra_blocksize = 1,
1820 			},
1821 			.setkey = rfc4543_setkey,
1822 			.setauthsize = rfc4543_setauthsize,
1823 			.encrypt = ipsec_gcm_encrypt,
1824 			.decrypt = ipsec_gcm_decrypt,
1825 			.ivsize = 8,
1826 			.maxauthsize = AES_BLOCK_SIZE,
1827 		},
1828 		.caam = {
1829 			.class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_GCM,
1830 			.nodkp = true,
1831 		},
1832 	},
1833 	/* Galois Counter Mode */
1834 	{
1835 		.aead = {
1836 			.base = {
1837 				.cra_name = "gcm(aes)",
1838 				.cra_driver_name = "gcm-aes-caam-qi2",
1839 				.cra_blocksize = 1,
1840 			},
1841 			.setkey = gcm_setkey,
1842 			.setauthsize = gcm_setauthsize,
1843 			.encrypt = aead_encrypt,
1844 			.decrypt = aead_decrypt,
1845 			.ivsize = 12,
1846 			.maxauthsize = AES_BLOCK_SIZE,
1847 		},
1848 		.caam = {
1849 			.class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_GCM,
1850 			.nodkp = true,
1851 		}
1852 	},
1853 	/* single-pass ipsec_esp descriptor */
1854 	{
1855 		.aead = {
1856 			.base = {
1857 				.cra_name = "authenc(hmac(md5),cbc(aes))",
1858 				.cra_driver_name = "authenc-hmac-md5-"
1859 						   "cbc-aes-caam-qi2",
1860 				.cra_blocksize = AES_BLOCK_SIZE,
1861 			},
1862 			.setkey = aead_setkey,
1863 			.setauthsize = aead_setauthsize,
1864 			.encrypt = aead_encrypt,
1865 			.decrypt = aead_decrypt,
1866 			.ivsize = AES_BLOCK_SIZE,
1867 			.maxauthsize = MD5_DIGEST_SIZE,
1868 		},
1869 		.caam = {
1870 			.class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CBC,
1871 			.class2_alg_type = OP_ALG_ALGSEL_MD5 |
1872 					   OP_ALG_AAI_HMAC_PRECOMP,
1873 		}
1874 	},
1875 	{
1876 		.aead = {
1877 			.base = {
1878 				.cra_name = "echainiv(authenc(hmac(md5),"
1879 					    "cbc(aes)))",
1880 				.cra_driver_name = "echainiv-authenc-hmac-md5-"
1881 						   "cbc-aes-caam-qi2",
1882 				.cra_blocksize = AES_BLOCK_SIZE,
1883 			},
1884 			.setkey = aead_setkey,
1885 			.setauthsize = aead_setauthsize,
1886 			.encrypt = aead_encrypt,
1887 			.decrypt = aead_decrypt,
1888 			.ivsize = AES_BLOCK_SIZE,
1889 			.maxauthsize = MD5_DIGEST_SIZE,
1890 		},
1891 		.caam = {
1892 			.class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CBC,
1893 			.class2_alg_type = OP_ALG_ALGSEL_MD5 |
1894 					   OP_ALG_AAI_HMAC_PRECOMP,
1895 			.geniv = true,
1896 		}
1897 	},
1898 	{
1899 		.aead = {
1900 			.base = {
1901 				.cra_name = "authenc(hmac(sha1),cbc(aes))",
1902 				.cra_driver_name = "authenc-hmac-sha1-"
1903 						   "cbc-aes-caam-qi2",
1904 				.cra_blocksize = AES_BLOCK_SIZE,
1905 			},
1906 			.setkey = aead_setkey,
1907 			.setauthsize = aead_setauthsize,
1908 			.encrypt = aead_encrypt,
1909 			.decrypt = aead_decrypt,
1910 			.ivsize = AES_BLOCK_SIZE,
1911 			.maxauthsize = SHA1_DIGEST_SIZE,
1912 		},
1913 		.caam = {
1914 			.class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CBC,
1915 			.class2_alg_type = OP_ALG_ALGSEL_SHA1 |
1916 					   OP_ALG_AAI_HMAC_PRECOMP,
1917 		}
1918 	},
1919 	{
1920 		.aead = {
1921 			.base = {
1922 				.cra_name = "echainiv(authenc(hmac(sha1),"
1923 					    "cbc(aes)))",
1924 				.cra_driver_name = "echainiv-authenc-"
1925 						   "hmac-sha1-cbc-aes-caam-qi2",
1926 				.cra_blocksize = AES_BLOCK_SIZE,
1927 			},
1928 			.setkey = aead_setkey,
1929 			.setauthsize = aead_setauthsize,
1930 			.encrypt = aead_encrypt,
1931 			.decrypt = aead_decrypt,
1932 			.ivsize = AES_BLOCK_SIZE,
1933 			.maxauthsize = SHA1_DIGEST_SIZE,
1934 		},
1935 		.caam = {
1936 			.class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CBC,
1937 			.class2_alg_type = OP_ALG_ALGSEL_SHA1 |
1938 					   OP_ALG_AAI_HMAC_PRECOMP,
1939 			.geniv = true,
1940 		},
1941 	},
1942 	{
1943 		.aead = {
1944 			.base = {
1945 				.cra_name = "authenc(hmac(sha224),cbc(aes))",
1946 				.cra_driver_name = "authenc-hmac-sha224-"
1947 						   "cbc-aes-caam-qi2",
1948 				.cra_blocksize = AES_BLOCK_SIZE,
1949 			},
1950 			.setkey = aead_setkey,
1951 			.setauthsize = aead_setauthsize,
1952 			.encrypt = aead_encrypt,
1953 			.decrypt = aead_decrypt,
1954 			.ivsize = AES_BLOCK_SIZE,
1955 			.maxauthsize = SHA224_DIGEST_SIZE,
1956 		},
1957 		.caam = {
1958 			.class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CBC,
1959 			.class2_alg_type = OP_ALG_ALGSEL_SHA224 |
1960 					   OP_ALG_AAI_HMAC_PRECOMP,
1961 		}
1962 	},
1963 	{
1964 		.aead = {
1965 			.base = {
1966 				.cra_name = "echainiv(authenc(hmac(sha224),"
1967 					    "cbc(aes)))",
1968 				.cra_driver_name = "echainiv-authenc-"
1969 						   "hmac-sha224-cbc-aes-caam-qi2",
1970 				.cra_blocksize = AES_BLOCK_SIZE,
1971 			},
1972 			.setkey = aead_setkey,
1973 			.setauthsize = aead_setauthsize,
1974 			.encrypt = aead_encrypt,
1975 			.decrypt = aead_decrypt,
1976 			.ivsize = AES_BLOCK_SIZE,
1977 			.maxauthsize = SHA224_DIGEST_SIZE,
1978 		},
1979 		.caam = {
1980 			.class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CBC,
1981 			.class2_alg_type = OP_ALG_ALGSEL_SHA224 |
1982 					   OP_ALG_AAI_HMAC_PRECOMP,
1983 			.geniv = true,
1984 		}
1985 	},
1986 	{
1987 		.aead = {
1988 			.base = {
1989 				.cra_name = "authenc(hmac(sha256),cbc(aes))",
1990 				.cra_driver_name = "authenc-hmac-sha256-"
1991 						   "cbc-aes-caam-qi2",
1992 				.cra_blocksize = AES_BLOCK_SIZE,
1993 			},
1994 			.setkey = aead_setkey,
1995 			.setauthsize = aead_setauthsize,
1996 			.encrypt = aead_encrypt,
1997 			.decrypt = aead_decrypt,
1998 			.ivsize = AES_BLOCK_SIZE,
1999 			.maxauthsize = SHA256_DIGEST_SIZE,
2000 		},
2001 		.caam = {
2002 			.class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CBC,
2003 			.class2_alg_type = OP_ALG_ALGSEL_SHA256 |
2004 					   OP_ALG_AAI_HMAC_PRECOMP,
2005 		}
2006 	},
2007 	{
2008 		.aead = {
2009 			.base = {
2010 				.cra_name = "echainiv(authenc(hmac(sha256),"
2011 					    "cbc(aes)))",
2012 				.cra_driver_name = "echainiv-authenc-"
2013 						   "hmac-sha256-cbc-aes-"
2014 						   "caam-qi2",
2015 				.cra_blocksize = AES_BLOCK_SIZE,
2016 			},
2017 			.setkey = aead_setkey,
2018 			.setauthsize = aead_setauthsize,
2019 			.encrypt = aead_encrypt,
2020 			.decrypt = aead_decrypt,
2021 			.ivsize = AES_BLOCK_SIZE,
2022 			.maxauthsize = SHA256_DIGEST_SIZE,
2023 		},
2024 		.caam = {
2025 			.class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CBC,
2026 			.class2_alg_type = OP_ALG_ALGSEL_SHA256 |
2027 					   OP_ALG_AAI_HMAC_PRECOMP,
2028 			.geniv = true,
2029 		}
2030 	},
2031 	{
2032 		.aead = {
2033 			.base = {
2034 				.cra_name = "authenc(hmac(sha384),cbc(aes))",
2035 				.cra_driver_name = "authenc-hmac-sha384-"
2036 						   "cbc-aes-caam-qi2",
2037 				.cra_blocksize = AES_BLOCK_SIZE,
2038 			},
2039 			.setkey = aead_setkey,
2040 			.setauthsize = aead_setauthsize,
2041 			.encrypt = aead_encrypt,
2042 			.decrypt = aead_decrypt,
2043 			.ivsize = AES_BLOCK_SIZE,
2044 			.maxauthsize = SHA384_DIGEST_SIZE,
2045 		},
2046 		.caam = {
2047 			.class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CBC,
2048 			.class2_alg_type = OP_ALG_ALGSEL_SHA384 |
2049 					   OP_ALG_AAI_HMAC_PRECOMP,
2050 		}
2051 	},
2052 	{
2053 		.aead = {
2054 			.base = {
2055 				.cra_name = "echainiv(authenc(hmac(sha384),"
2056 					    "cbc(aes)))",
2057 				.cra_driver_name = "echainiv-authenc-"
2058 						   "hmac-sha384-cbc-aes-"
2059 						   "caam-qi2",
2060 				.cra_blocksize = AES_BLOCK_SIZE,
2061 			},
2062 			.setkey = aead_setkey,
2063 			.setauthsize = aead_setauthsize,
2064 			.encrypt = aead_encrypt,
2065 			.decrypt = aead_decrypt,
2066 			.ivsize = AES_BLOCK_SIZE,
2067 			.maxauthsize = SHA384_DIGEST_SIZE,
2068 		},
2069 		.caam = {
2070 			.class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CBC,
2071 			.class2_alg_type = OP_ALG_ALGSEL_SHA384 |
2072 					   OP_ALG_AAI_HMAC_PRECOMP,
2073 			.geniv = true,
2074 		}
2075 	},
2076 	{
2077 		.aead = {
2078 			.base = {
2079 				.cra_name = "authenc(hmac(sha512),cbc(aes))",
2080 				.cra_driver_name = "authenc-hmac-sha512-"
2081 						   "cbc-aes-caam-qi2",
2082 				.cra_blocksize = AES_BLOCK_SIZE,
2083 			},
2084 			.setkey = aead_setkey,
2085 			.setauthsize = aead_setauthsize,
2086 			.encrypt = aead_encrypt,
2087 			.decrypt = aead_decrypt,
2088 			.ivsize = AES_BLOCK_SIZE,
2089 			.maxauthsize = SHA512_DIGEST_SIZE,
2090 		},
2091 		.caam = {
2092 			.class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CBC,
2093 			.class2_alg_type = OP_ALG_ALGSEL_SHA512 |
2094 					   OP_ALG_AAI_HMAC_PRECOMP,
2095 		}
2096 	},
2097 	{
2098 		.aead = {
2099 			.base = {
2100 				.cra_name = "echainiv(authenc(hmac(sha512),"
2101 					    "cbc(aes)))",
2102 				.cra_driver_name = "echainiv-authenc-"
2103 						   "hmac-sha512-cbc-aes-"
2104 						   "caam-qi2",
2105 				.cra_blocksize = AES_BLOCK_SIZE,
2106 			},
2107 			.setkey = aead_setkey,
2108 			.setauthsize = aead_setauthsize,
2109 			.encrypt = aead_encrypt,
2110 			.decrypt = aead_decrypt,
2111 			.ivsize = AES_BLOCK_SIZE,
2112 			.maxauthsize = SHA512_DIGEST_SIZE,
2113 		},
2114 		.caam = {
2115 			.class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CBC,
2116 			.class2_alg_type = OP_ALG_ALGSEL_SHA512 |
2117 					   OP_ALG_AAI_HMAC_PRECOMP,
2118 			.geniv = true,
2119 		}
2120 	},
2121 	{
2122 		.aead = {
2123 			.base = {
2124 				.cra_name = "authenc(hmac(md5),cbc(des3_ede))",
2125 				.cra_driver_name = "authenc-hmac-md5-"
2126 						   "cbc-des3_ede-caam-qi2",
2127 				.cra_blocksize = DES3_EDE_BLOCK_SIZE,
2128 			},
2129 			.setkey = des3_aead_setkey,
2130 			.setauthsize = aead_setauthsize,
2131 			.encrypt = aead_encrypt,
2132 			.decrypt = aead_decrypt,
2133 			.ivsize = DES3_EDE_BLOCK_SIZE,
2134 			.maxauthsize = MD5_DIGEST_SIZE,
2135 		},
2136 		.caam = {
2137 			.class1_alg_type = OP_ALG_ALGSEL_3DES | OP_ALG_AAI_CBC,
2138 			.class2_alg_type = OP_ALG_ALGSEL_MD5 |
2139 					   OP_ALG_AAI_HMAC_PRECOMP,
2140 		}
2141 	},
2142 	{
2143 		.aead = {
2144 			.base = {
2145 				.cra_name = "echainiv(authenc(hmac(md5),"
2146 					    "cbc(des3_ede)))",
2147 				.cra_driver_name = "echainiv-authenc-hmac-md5-"
2148 						   "cbc-des3_ede-caam-qi2",
2149 				.cra_blocksize = DES3_EDE_BLOCK_SIZE,
2150 			},
2151 			.setkey = des3_aead_setkey,
2152 			.setauthsize = aead_setauthsize,
2153 			.encrypt = aead_encrypt,
2154 			.decrypt = aead_decrypt,
2155 			.ivsize = DES3_EDE_BLOCK_SIZE,
2156 			.maxauthsize = MD5_DIGEST_SIZE,
2157 		},
2158 		.caam = {
2159 			.class1_alg_type = OP_ALG_ALGSEL_3DES | OP_ALG_AAI_CBC,
2160 			.class2_alg_type = OP_ALG_ALGSEL_MD5 |
2161 					   OP_ALG_AAI_HMAC_PRECOMP,
2162 			.geniv = true,
2163 		}
2164 	},
2165 	{
2166 		.aead = {
2167 			.base = {
2168 				.cra_name = "authenc(hmac(sha1),"
2169 					    "cbc(des3_ede))",
2170 				.cra_driver_name = "authenc-hmac-sha1-"
2171 						   "cbc-des3_ede-caam-qi2",
2172 				.cra_blocksize = DES3_EDE_BLOCK_SIZE,
2173 			},
2174 			.setkey = des3_aead_setkey,
2175 			.setauthsize = aead_setauthsize,
2176 			.encrypt = aead_encrypt,
2177 			.decrypt = aead_decrypt,
2178 			.ivsize = DES3_EDE_BLOCK_SIZE,
2179 			.maxauthsize = SHA1_DIGEST_SIZE,
2180 		},
2181 		.caam = {
2182 			.class1_alg_type = OP_ALG_ALGSEL_3DES | OP_ALG_AAI_CBC,
2183 			.class2_alg_type = OP_ALG_ALGSEL_SHA1 |
2184 					   OP_ALG_AAI_HMAC_PRECOMP,
2185 		},
2186 	},
2187 	{
2188 		.aead = {
2189 			.base = {
2190 				.cra_name = "echainiv(authenc(hmac(sha1),"
2191 					    "cbc(des3_ede)))",
2192 				.cra_driver_name = "echainiv-authenc-"
2193 						   "hmac-sha1-"
2194 						   "cbc-des3_ede-caam-qi2",
2195 				.cra_blocksize = DES3_EDE_BLOCK_SIZE,
2196 			},
2197 			.setkey = des3_aead_setkey,
2198 			.setauthsize = aead_setauthsize,
2199 			.encrypt = aead_encrypt,
2200 			.decrypt = aead_decrypt,
2201 			.ivsize = DES3_EDE_BLOCK_SIZE,
2202 			.maxauthsize = SHA1_DIGEST_SIZE,
2203 		},
2204 		.caam = {
2205 			.class1_alg_type = OP_ALG_ALGSEL_3DES | OP_ALG_AAI_CBC,
2206 			.class2_alg_type = OP_ALG_ALGSEL_SHA1 |
2207 					   OP_ALG_AAI_HMAC_PRECOMP,
2208 			.geniv = true,
2209 		}
2210 	},
2211 	{
2212 		.aead = {
2213 			.base = {
2214 				.cra_name = "authenc(hmac(sha224),"
2215 					    "cbc(des3_ede))",
2216 				.cra_driver_name = "authenc-hmac-sha224-"
2217 						   "cbc-des3_ede-caam-qi2",
2218 				.cra_blocksize = DES3_EDE_BLOCK_SIZE,
2219 			},
2220 			.setkey = des3_aead_setkey,
2221 			.setauthsize = aead_setauthsize,
2222 			.encrypt = aead_encrypt,
2223 			.decrypt = aead_decrypt,
2224 			.ivsize = DES3_EDE_BLOCK_SIZE,
2225 			.maxauthsize = SHA224_DIGEST_SIZE,
2226 		},
2227 		.caam = {
2228 			.class1_alg_type = OP_ALG_ALGSEL_3DES | OP_ALG_AAI_CBC,
2229 			.class2_alg_type = OP_ALG_ALGSEL_SHA224 |
2230 					   OP_ALG_AAI_HMAC_PRECOMP,
2231 		},
2232 	},
2233 	{
2234 		.aead = {
2235 			.base = {
2236 				.cra_name = "echainiv(authenc(hmac(sha224),"
2237 					    "cbc(des3_ede)))",
2238 				.cra_driver_name = "echainiv-authenc-"
2239 						   "hmac-sha224-"
2240 						   "cbc-des3_ede-caam-qi2",
2241 				.cra_blocksize = DES3_EDE_BLOCK_SIZE,
2242 			},
2243 			.setkey = des3_aead_setkey,
2244 			.setauthsize = aead_setauthsize,
2245 			.encrypt = aead_encrypt,
2246 			.decrypt = aead_decrypt,
2247 			.ivsize = DES3_EDE_BLOCK_SIZE,
2248 			.maxauthsize = SHA224_DIGEST_SIZE,
2249 		},
2250 		.caam = {
2251 			.class1_alg_type = OP_ALG_ALGSEL_3DES | OP_ALG_AAI_CBC,
2252 			.class2_alg_type = OP_ALG_ALGSEL_SHA224 |
2253 					   OP_ALG_AAI_HMAC_PRECOMP,
2254 			.geniv = true,
2255 		}
2256 	},
2257 	{
2258 		.aead = {
2259 			.base = {
2260 				.cra_name = "authenc(hmac(sha256),"
2261 					    "cbc(des3_ede))",
2262 				.cra_driver_name = "authenc-hmac-sha256-"
2263 						   "cbc-des3_ede-caam-qi2",
2264 				.cra_blocksize = DES3_EDE_BLOCK_SIZE,
2265 			},
2266 			.setkey = des3_aead_setkey,
2267 			.setauthsize = aead_setauthsize,
2268 			.encrypt = aead_encrypt,
2269 			.decrypt = aead_decrypt,
2270 			.ivsize = DES3_EDE_BLOCK_SIZE,
2271 			.maxauthsize = SHA256_DIGEST_SIZE,
2272 		},
2273 		.caam = {
2274 			.class1_alg_type = OP_ALG_ALGSEL_3DES | OP_ALG_AAI_CBC,
2275 			.class2_alg_type = OP_ALG_ALGSEL_SHA256 |
2276 					   OP_ALG_AAI_HMAC_PRECOMP,
2277 		},
2278 	},
2279 	{
2280 		.aead = {
2281 			.base = {
2282 				.cra_name = "echainiv(authenc(hmac(sha256),"
2283 					    "cbc(des3_ede)))",
2284 				.cra_driver_name = "echainiv-authenc-"
2285 						   "hmac-sha256-"
2286 						   "cbc-des3_ede-caam-qi2",
2287 				.cra_blocksize = DES3_EDE_BLOCK_SIZE,
2288 			},
2289 			.setkey = des3_aead_setkey,
2290 			.setauthsize = aead_setauthsize,
2291 			.encrypt = aead_encrypt,
2292 			.decrypt = aead_decrypt,
2293 			.ivsize = DES3_EDE_BLOCK_SIZE,
2294 			.maxauthsize = SHA256_DIGEST_SIZE,
2295 		},
2296 		.caam = {
2297 			.class1_alg_type = OP_ALG_ALGSEL_3DES | OP_ALG_AAI_CBC,
2298 			.class2_alg_type = OP_ALG_ALGSEL_SHA256 |
2299 					   OP_ALG_AAI_HMAC_PRECOMP,
2300 			.geniv = true,
2301 		}
2302 	},
2303 	{
2304 		.aead = {
2305 			.base = {
2306 				.cra_name = "authenc(hmac(sha384),"
2307 					    "cbc(des3_ede))",
2308 				.cra_driver_name = "authenc-hmac-sha384-"
2309 						   "cbc-des3_ede-caam-qi2",
2310 				.cra_blocksize = DES3_EDE_BLOCK_SIZE,
2311 			},
2312 			.setkey = des3_aead_setkey,
2313 			.setauthsize = aead_setauthsize,
2314 			.encrypt = aead_encrypt,
2315 			.decrypt = aead_decrypt,
2316 			.ivsize = DES3_EDE_BLOCK_SIZE,
2317 			.maxauthsize = SHA384_DIGEST_SIZE,
2318 		},
2319 		.caam = {
2320 			.class1_alg_type = OP_ALG_ALGSEL_3DES | OP_ALG_AAI_CBC,
2321 			.class2_alg_type = OP_ALG_ALGSEL_SHA384 |
2322 					   OP_ALG_AAI_HMAC_PRECOMP,
2323 		},
2324 	},
2325 	{
2326 		.aead = {
2327 			.base = {
2328 				.cra_name = "echainiv(authenc(hmac(sha384),"
2329 					    "cbc(des3_ede)))",
2330 				.cra_driver_name = "echainiv-authenc-"
2331 						   "hmac-sha384-"
2332 						   "cbc-des3_ede-caam-qi2",
2333 				.cra_blocksize = DES3_EDE_BLOCK_SIZE,
2334 			},
2335 			.setkey = des3_aead_setkey,
2336 			.setauthsize = aead_setauthsize,
2337 			.encrypt = aead_encrypt,
2338 			.decrypt = aead_decrypt,
2339 			.ivsize = DES3_EDE_BLOCK_SIZE,
2340 			.maxauthsize = SHA384_DIGEST_SIZE,
2341 		},
2342 		.caam = {
2343 			.class1_alg_type = OP_ALG_ALGSEL_3DES | OP_ALG_AAI_CBC,
2344 			.class2_alg_type = OP_ALG_ALGSEL_SHA384 |
2345 					   OP_ALG_AAI_HMAC_PRECOMP,
2346 			.geniv = true,
2347 		}
2348 	},
2349 	{
2350 		.aead = {
2351 			.base = {
2352 				.cra_name = "authenc(hmac(sha512),"
2353 					    "cbc(des3_ede))",
2354 				.cra_driver_name = "authenc-hmac-sha512-"
2355 						   "cbc-des3_ede-caam-qi2",
2356 				.cra_blocksize = DES3_EDE_BLOCK_SIZE,
2357 			},
2358 			.setkey = des3_aead_setkey,
2359 			.setauthsize = aead_setauthsize,
2360 			.encrypt = aead_encrypt,
2361 			.decrypt = aead_decrypt,
2362 			.ivsize = DES3_EDE_BLOCK_SIZE,
2363 			.maxauthsize = SHA512_DIGEST_SIZE,
2364 		},
2365 		.caam = {
2366 			.class1_alg_type = OP_ALG_ALGSEL_3DES | OP_ALG_AAI_CBC,
2367 			.class2_alg_type = OP_ALG_ALGSEL_SHA512 |
2368 					   OP_ALG_AAI_HMAC_PRECOMP,
2369 		},
2370 	},
2371 	{
2372 		.aead = {
2373 			.base = {
2374 				.cra_name = "echainiv(authenc(hmac(sha512),"
2375 					    "cbc(des3_ede)))",
2376 				.cra_driver_name = "echainiv-authenc-"
2377 						   "hmac-sha512-"
2378 						   "cbc-des3_ede-caam-qi2",
2379 				.cra_blocksize = DES3_EDE_BLOCK_SIZE,
2380 			},
2381 			.setkey = des3_aead_setkey,
2382 			.setauthsize = aead_setauthsize,
2383 			.encrypt = aead_encrypt,
2384 			.decrypt = aead_decrypt,
2385 			.ivsize = DES3_EDE_BLOCK_SIZE,
2386 			.maxauthsize = SHA512_DIGEST_SIZE,
2387 		},
2388 		.caam = {
2389 			.class1_alg_type = OP_ALG_ALGSEL_3DES | OP_ALG_AAI_CBC,
2390 			.class2_alg_type = OP_ALG_ALGSEL_SHA512 |
2391 					   OP_ALG_AAI_HMAC_PRECOMP,
2392 			.geniv = true,
2393 		}
2394 	},
2395 	{
2396 		.aead = {
2397 			.base = {
2398 				.cra_name = "authenc(hmac(md5),cbc(des))",
2399 				.cra_driver_name = "authenc-hmac-md5-"
2400 						   "cbc-des-caam-qi2",
2401 				.cra_blocksize = DES_BLOCK_SIZE,
2402 			},
2403 			.setkey = aead_setkey,
2404 			.setauthsize = aead_setauthsize,
2405 			.encrypt = aead_encrypt,
2406 			.decrypt = aead_decrypt,
2407 			.ivsize = DES_BLOCK_SIZE,
2408 			.maxauthsize = MD5_DIGEST_SIZE,
2409 		},
2410 		.caam = {
2411 			.class1_alg_type = OP_ALG_ALGSEL_DES | OP_ALG_AAI_CBC,
2412 			.class2_alg_type = OP_ALG_ALGSEL_MD5 |
2413 					   OP_ALG_AAI_HMAC_PRECOMP,
2414 		},
2415 	},
2416 	{
2417 		.aead = {
2418 			.base = {
2419 				.cra_name = "echainiv(authenc(hmac(md5),"
2420 					    "cbc(des)))",
2421 				.cra_driver_name = "echainiv-authenc-hmac-md5-"
2422 						   "cbc-des-caam-qi2",
2423 				.cra_blocksize = DES_BLOCK_SIZE,
2424 			},
2425 			.setkey = aead_setkey,
2426 			.setauthsize = aead_setauthsize,
2427 			.encrypt = aead_encrypt,
2428 			.decrypt = aead_decrypt,
2429 			.ivsize = DES_BLOCK_SIZE,
2430 			.maxauthsize = MD5_DIGEST_SIZE,
2431 		},
2432 		.caam = {
2433 			.class1_alg_type = OP_ALG_ALGSEL_DES | OP_ALG_AAI_CBC,
2434 			.class2_alg_type = OP_ALG_ALGSEL_MD5 |
2435 					   OP_ALG_AAI_HMAC_PRECOMP,
2436 			.geniv = true,
2437 		}
2438 	},
2439 	{
2440 		.aead = {
2441 			.base = {
2442 				.cra_name = "authenc(hmac(sha1),cbc(des))",
2443 				.cra_driver_name = "authenc-hmac-sha1-"
2444 						   "cbc-des-caam-qi2",
2445 				.cra_blocksize = DES_BLOCK_SIZE,
2446 			},
2447 			.setkey = aead_setkey,
2448 			.setauthsize = aead_setauthsize,
2449 			.encrypt = aead_encrypt,
2450 			.decrypt = aead_decrypt,
2451 			.ivsize = DES_BLOCK_SIZE,
2452 			.maxauthsize = SHA1_DIGEST_SIZE,
2453 		},
2454 		.caam = {
2455 			.class1_alg_type = OP_ALG_ALGSEL_DES | OP_ALG_AAI_CBC,
2456 			.class2_alg_type = OP_ALG_ALGSEL_SHA1 |
2457 					   OP_ALG_AAI_HMAC_PRECOMP,
2458 		},
2459 	},
2460 	{
2461 		.aead = {
2462 			.base = {
2463 				.cra_name = "echainiv(authenc(hmac(sha1),"
2464 					    "cbc(des)))",
2465 				.cra_driver_name = "echainiv-authenc-"
2466 						   "hmac-sha1-cbc-des-caam-qi2",
2467 				.cra_blocksize = DES_BLOCK_SIZE,
2468 			},
2469 			.setkey = aead_setkey,
2470 			.setauthsize = aead_setauthsize,
2471 			.encrypt = aead_encrypt,
2472 			.decrypt = aead_decrypt,
2473 			.ivsize = DES_BLOCK_SIZE,
2474 			.maxauthsize = SHA1_DIGEST_SIZE,
2475 		},
2476 		.caam = {
2477 			.class1_alg_type = OP_ALG_ALGSEL_DES | OP_ALG_AAI_CBC,
2478 			.class2_alg_type = OP_ALG_ALGSEL_SHA1 |
2479 					   OP_ALG_AAI_HMAC_PRECOMP,
2480 			.geniv = true,
2481 		}
2482 	},
2483 	{
2484 		.aead = {
2485 			.base = {
2486 				.cra_name = "authenc(hmac(sha224),cbc(des))",
2487 				.cra_driver_name = "authenc-hmac-sha224-"
2488 						   "cbc-des-caam-qi2",
2489 				.cra_blocksize = DES_BLOCK_SIZE,
2490 			},
2491 			.setkey = aead_setkey,
2492 			.setauthsize = aead_setauthsize,
2493 			.encrypt = aead_encrypt,
2494 			.decrypt = aead_decrypt,
2495 			.ivsize = DES_BLOCK_SIZE,
2496 			.maxauthsize = SHA224_DIGEST_SIZE,
2497 		},
2498 		.caam = {
2499 			.class1_alg_type = OP_ALG_ALGSEL_DES | OP_ALG_AAI_CBC,
2500 			.class2_alg_type = OP_ALG_ALGSEL_SHA224 |
2501 					   OP_ALG_AAI_HMAC_PRECOMP,
2502 		},
2503 	},
2504 	{
2505 		.aead = {
2506 			.base = {
2507 				.cra_name = "echainiv(authenc(hmac(sha224),"
2508 					    "cbc(des)))",
2509 				.cra_driver_name = "echainiv-authenc-"
2510 						   "hmac-sha224-cbc-des-"
2511 						   "caam-qi2",
2512 				.cra_blocksize = DES_BLOCK_SIZE,
2513 			},
2514 			.setkey = aead_setkey,
2515 			.setauthsize = aead_setauthsize,
2516 			.encrypt = aead_encrypt,
2517 			.decrypt = aead_decrypt,
2518 			.ivsize = DES_BLOCK_SIZE,
2519 			.maxauthsize = SHA224_DIGEST_SIZE,
2520 		},
2521 		.caam = {
2522 			.class1_alg_type = OP_ALG_ALGSEL_DES | OP_ALG_AAI_CBC,
2523 			.class2_alg_type = OP_ALG_ALGSEL_SHA224 |
2524 					   OP_ALG_AAI_HMAC_PRECOMP,
2525 			.geniv = true,
2526 		}
2527 	},
2528 	{
2529 		.aead = {
2530 			.base = {
2531 				.cra_name = "authenc(hmac(sha256),cbc(des))",
2532 				.cra_driver_name = "authenc-hmac-sha256-"
2533 						   "cbc-des-caam-qi2",
2534 				.cra_blocksize = DES_BLOCK_SIZE,
2535 			},
2536 			.setkey = aead_setkey,
2537 			.setauthsize = aead_setauthsize,
2538 			.encrypt = aead_encrypt,
2539 			.decrypt = aead_decrypt,
2540 			.ivsize = DES_BLOCK_SIZE,
2541 			.maxauthsize = SHA256_DIGEST_SIZE,
2542 		},
2543 		.caam = {
2544 			.class1_alg_type = OP_ALG_ALGSEL_DES | OP_ALG_AAI_CBC,
2545 			.class2_alg_type = OP_ALG_ALGSEL_SHA256 |
2546 					   OP_ALG_AAI_HMAC_PRECOMP,
2547 		},
2548 	},
2549 	{
2550 		.aead = {
2551 			.base = {
2552 				.cra_name = "echainiv(authenc(hmac(sha256),"
2553 					    "cbc(des)))",
2554 				.cra_driver_name = "echainiv-authenc-"
2555 						   "hmac-sha256-cbc-des-"
2556 						   "caam-qi2",
2557 				.cra_blocksize = DES_BLOCK_SIZE,
2558 			},
2559 			.setkey = aead_setkey,
2560 			.setauthsize = aead_setauthsize,
2561 			.encrypt = aead_encrypt,
2562 			.decrypt = aead_decrypt,
2563 			.ivsize = DES_BLOCK_SIZE,
2564 			.maxauthsize = SHA256_DIGEST_SIZE,
2565 		},
2566 		.caam = {
2567 			.class1_alg_type = OP_ALG_ALGSEL_DES | OP_ALG_AAI_CBC,
2568 			.class2_alg_type = OP_ALG_ALGSEL_SHA256 |
2569 					   OP_ALG_AAI_HMAC_PRECOMP,
2570 			.geniv = true,
2571 		},
2572 	},
2573 	{
2574 		.aead = {
2575 			.base = {
2576 				.cra_name = "authenc(hmac(sha384),cbc(des))",
2577 				.cra_driver_name = "authenc-hmac-sha384-"
2578 						   "cbc-des-caam-qi2",
2579 				.cra_blocksize = DES_BLOCK_SIZE,
2580 			},
2581 			.setkey = aead_setkey,
2582 			.setauthsize = aead_setauthsize,
2583 			.encrypt = aead_encrypt,
2584 			.decrypt = aead_decrypt,
2585 			.ivsize = DES_BLOCK_SIZE,
2586 			.maxauthsize = SHA384_DIGEST_SIZE,
2587 		},
2588 		.caam = {
2589 			.class1_alg_type = OP_ALG_ALGSEL_DES | OP_ALG_AAI_CBC,
2590 			.class2_alg_type = OP_ALG_ALGSEL_SHA384 |
2591 					   OP_ALG_AAI_HMAC_PRECOMP,
2592 		},
2593 	},
2594 	{
2595 		.aead = {
2596 			.base = {
2597 				.cra_name = "echainiv(authenc(hmac(sha384),"
2598 					    "cbc(des)))",
2599 				.cra_driver_name = "echainiv-authenc-"
2600 						   "hmac-sha384-cbc-des-"
2601 						   "caam-qi2",
2602 				.cra_blocksize = DES_BLOCK_SIZE,
2603 			},
2604 			.setkey = aead_setkey,
2605 			.setauthsize = aead_setauthsize,
2606 			.encrypt = aead_encrypt,
2607 			.decrypt = aead_decrypt,
2608 			.ivsize = DES_BLOCK_SIZE,
2609 			.maxauthsize = SHA384_DIGEST_SIZE,
2610 		},
2611 		.caam = {
2612 			.class1_alg_type = OP_ALG_ALGSEL_DES | OP_ALG_AAI_CBC,
2613 			.class2_alg_type = OP_ALG_ALGSEL_SHA384 |
2614 					   OP_ALG_AAI_HMAC_PRECOMP,
2615 			.geniv = true,
2616 		}
2617 	},
2618 	{
2619 		.aead = {
2620 			.base = {
2621 				.cra_name = "authenc(hmac(sha512),cbc(des))",
2622 				.cra_driver_name = "authenc-hmac-sha512-"
2623 						   "cbc-des-caam-qi2",
2624 				.cra_blocksize = DES_BLOCK_SIZE,
2625 			},
2626 			.setkey = aead_setkey,
2627 			.setauthsize = aead_setauthsize,
2628 			.encrypt = aead_encrypt,
2629 			.decrypt = aead_decrypt,
2630 			.ivsize = DES_BLOCK_SIZE,
2631 			.maxauthsize = SHA512_DIGEST_SIZE,
2632 		},
2633 		.caam = {
2634 			.class1_alg_type = OP_ALG_ALGSEL_DES | OP_ALG_AAI_CBC,
2635 			.class2_alg_type = OP_ALG_ALGSEL_SHA512 |
2636 					   OP_ALG_AAI_HMAC_PRECOMP,
2637 		}
2638 	},
2639 	{
2640 		.aead = {
2641 			.base = {
2642 				.cra_name = "echainiv(authenc(hmac(sha512),"
2643 					    "cbc(des)))",
2644 				.cra_driver_name = "echainiv-authenc-"
2645 						   "hmac-sha512-cbc-des-"
2646 						   "caam-qi2",
2647 				.cra_blocksize = DES_BLOCK_SIZE,
2648 			},
2649 			.setkey = aead_setkey,
2650 			.setauthsize = aead_setauthsize,
2651 			.encrypt = aead_encrypt,
2652 			.decrypt = aead_decrypt,
2653 			.ivsize = DES_BLOCK_SIZE,
2654 			.maxauthsize = SHA512_DIGEST_SIZE,
2655 		},
2656 		.caam = {
2657 			.class1_alg_type = OP_ALG_ALGSEL_DES | OP_ALG_AAI_CBC,
2658 			.class2_alg_type = OP_ALG_ALGSEL_SHA512 |
2659 					   OP_ALG_AAI_HMAC_PRECOMP,
2660 			.geniv = true,
2661 		}
2662 	},
2663 	{
2664 		.aead = {
2665 			.base = {
2666 				.cra_name = "authenc(hmac(md5),"
2667 					    "rfc3686(ctr(aes)))",
2668 				.cra_driver_name = "authenc-hmac-md5-"
2669 						   "rfc3686-ctr-aes-caam-qi2",
2670 				.cra_blocksize = 1,
2671 			},
2672 			.setkey = aead_setkey,
2673 			.setauthsize = aead_setauthsize,
2674 			.encrypt = aead_encrypt,
2675 			.decrypt = aead_decrypt,
2676 			.ivsize = CTR_RFC3686_IV_SIZE,
2677 			.maxauthsize = MD5_DIGEST_SIZE,
2678 		},
2679 		.caam = {
2680 			.class1_alg_type = OP_ALG_ALGSEL_AES |
2681 					   OP_ALG_AAI_CTR_MOD128,
2682 			.class2_alg_type = OP_ALG_ALGSEL_MD5 |
2683 					   OP_ALG_AAI_HMAC_PRECOMP,
2684 			.rfc3686 = true,
2685 		},
2686 	},
2687 	{
2688 		.aead = {
2689 			.base = {
2690 				.cra_name = "seqiv(authenc("
2691 					    "hmac(md5),rfc3686(ctr(aes))))",
2692 				.cra_driver_name = "seqiv-authenc-hmac-md5-"
2693 						   "rfc3686-ctr-aes-caam-qi2",
2694 				.cra_blocksize = 1,
2695 			},
2696 			.setkey = aead_setkey,
2697 			.setauthsize = aead_setauthsize,
2698 			.encrypt = aead_encrypt,
2699 			.decrypt = aead_decrypt,
2700 			.ivsize = CTR_RFC3686_IV_SIZE,
2701 			.maxauthsize = MD5_DIGEST_SIZE,
2702 		},
2703 		.caam = {
2704 			.class1_alg_type = OP_ALG_ALGSEL_AES |
2705 					   OP_ALG_AAI_CTR_MOD128,
2706 			.class2_alg_type = OP_ALG_ALGSEL_MD5 |
2707 					   OP_ALG_AAI_HMAC_PRECOMP,
2708 			.rfc3686 = true,
2709 			.geniv = true,
2710 		},
2711 	},
2712 	{
2713 		.aead = {
2714 			.base = {
2715 				.cra_name = "authenc(hmac(sha1),"
2716 					    "rfc3686(ctr(aes)))",
2717 				.cra_driver_name = "authenc-hmac-sha1-"
2718 						   "rfc3686-ctr-aes-caam-qi2",
2719 				.cra_blocksize = 1,
2720 			},
2721 			.setkey = aead_setkey,
2722 			.setauthsize = aead_setauthsize,
2723 			.encrypt = aead_encrypt,
2724 			.decrypt = aead_decrypt,
2725 			.ivsize = CTR_RFC3686_IV_SIZE,
2726 			.maxauthsize = SHA1_DIGEST_SIZE,
2727 		},
2728 		.caam = {
2729 			.class1_alg_type = OP_ALG_ALGSEL_AES |
2730 					   OP_ALG_AAI_CTR_MOD128,
2731 			.class2_alg_type = OP_ALG_ALGSEL_SHA1 |
2732 					   OP_ALG_AAI_HMAC_PRECOMP,
2733 			.rfc3686 = true,
2734 		},
2735 	},
2736 	{
2737 		.aead = {
2738 			.base = {
2739 				.cra_name = "seqiv(authenc("
2740 					    "hmac(sha1),rfc3686(ctr(aes))))",
2741 				.cra_driver_name = "seqiv-authenc-hmac-sha1-"
2742 						   "rfc3686-ctr-aes-caam-qi2",
2743 				.cra_blocksize = 1,
2744 			},
2745 			.setkey = aead_setkey,
2746 			.setauthsize = aead_setauthsize,
2747 			.encrypt = aead_encrypt,
2748 			.decrypt = aead_decrypt,
2749 			.ivsize = CTR_RFC3686_IV_SIZE,
2750 			.maxauthsize = SHA1_DIGEST_SIZE,
2751 		},
2752 		.caam = {
2753 			.class1_alg_type = OP_ALG_ALGSEL_AES |
2754 					   OP_ALG_AAI_CTR_MOD128,
2755 			.class2_alg_type = OP_ALG_ALGSEL_SHA1 |
2756 					   OP_ALG_AAI_HMAC_PRECOMP,
2757 			.rfc3686 = true,
2758 			.geniv = true,
2759 		},
2760 	},
2761 	{
2762 		.aead = {
2763 			.base = {
2764 				.cra_name = "authenc(hmac(sha224),"
2765 					    "rfc3686(ctr(aes)))",
2766 				.cra_driver_name = "authenc-hmac-sha224-"
2767 						   "rfc3686-ctr-aes-caam-qi2",
2768 				.cra_blocksize = 1,
2769 			},
2770 			.setkey = aead_setkey,
2771 			.setauthsize = aead_setauthsize,
2772 			.encrypt = aead_encrypt,
2773 			.decrypt = aead_decrypt,
2774 			.ivsize = CTR_RFC3686_IV_SIZE,
2775 			.maxauthsize = SHA224_DIGEST_SIZE,
2776 		},
2777 		.caam = {
2778 			.class1_alg_type = OP_ALG_ALGSEL_AES |
2779 					   OP_ALG_AAI_CTR_MOD128,
2780 			.class2_alg_type = OP_ALG_ALGSEL_SHA224 |
2781 					   OP_ALG_AAI_HMAC_PRECOMP,
2782 			.rfc3686 = true,
2783 		},
2784 	},
2785 	{
2786 		.aead = {
2787 			.base = {
2788 				.cra_name = "seqiv(authenc("
2789 					    "hmac(sha224),rfc3686(ctr(aes))))",
2790 				.cra_driver_name = "seqiv-authenc-hmac-sha224-"
2791 						   "rfc3686-ctr-aes-caam-qi2",
2792 				.cra_blocksize = 1,
2793 			},
2794 			.setkey = aead_setkey,
2795 			.setauthsize = aead_setauthsize,
2796 			.encrypt = aead_encrypt,
2797 			.decrypt = aead_decrypt,
2798 			.ivsize = CTR_RFC3686_IV_SIZE,
2799 			.maxauthsize = SHA224_DIGEST_SIZE,
2800 		},
2801 		.caam = {
2802 			.class1_alg_type = OP_ALG_ALGSEL_AES |
2803 					   OP_ALG_AAI_CTR_MOD128,
2804 			.class2_alg_type = OP_ALG_ALGSEL_SHA224 |
2805 					   OP_ALG_AAI_HMAC_PRECOMP,
2806 			.rfc3686 = true,
2807 			.geniv = true,
2808 		},
2809 	},
2810 	{
2811 		.aead = {
2812 			.base = {
2813 				.cra_name = "authenc(hmac(sha256),"
2814 					    "rfc3686(ctr(aes)))",
2815 				.cra_driver_name = "authenc-hmac-sha256-"
2816 						   "rfc3686-ctr-aes-caam-qi2",
2817 				.cra_blocksize = 1,
2818 			},
2819 			.setkey = aead_setkey,
2820 			.setauthsize = aead_setauthsize,
2821 			.encrypt = aead_encrypt,
2822 			.decrypt = aead_decrypt,
2823 			.ivsize = CTR_RFC3686_IV_SIZE,
2824 			.maxauthsize = SHA256_DIGEST_SIZE,
2825 		},
2826 		.caam = {
2827 			.class1_alg_type = OP_ALG_ALGSEL_AES |
2828 					   OP_ALG_AAI_CTR_MOD128,
2829 			.class2_alg_type = OP_ALG_ALGSEL_SHA256 |
2830 					   OP_ALG_AAI_HMAC_PRECOMP,
2831 			.rfc3686 = true,
2832 		},
2833 	},
2834 	{
2835 		.aead = {
2836 			.base = {
2837 				.cra_name = "seqiv(authenc(hmac(sha256),"
2838 					    "rfc3686(ctr(aes))))",
2839 				.cra_driver_name = "seqiv-authenc-hmac-sha256-"
2840 						   "rfc3686-ctr-aes-caam-qi2",
2841 				.cra_blocksize = 1,
2842 			},
2843 			.setkey = aead_setkey,
2844 			.setauthsize = aead_setauthsize,
2845 			.encrypt = aead_encrypt,
2846 			.decrypt = aead_decrypt,
2847 			.ivsize = CTR_RFC3686_IV_SIZE,
2848 			.maxauthsize = SHA256_DIGEST_SIZE,
2849 		},
2850 		.caam = {
2851 			.class1_alg_type = OP_ALG_ALGSEL_AES |
2852 					   OP_ALG_AAI_CTR_MOD128,
2853 			.class2_alg_type = OP_ALG_ALGSEL_SHA256 |
2854 					   OP_ALG_AAI_HMAC_PRECOMP,
2855 			.rfc3686 = true,
2856 			.geniv = true,
2857 		},
2858 	},
2859 	{
2860 		.aead = {
2861 			.base = {
2862 				.cra_name = "authenc(hmac(sha384),"
2863 					    "rfc3686(ctr(aes)))",
2864 				.cra_driver_name = "authenc-hmac-sha384-"
2865 						   "rfc3686-ctr-aes-caam-qi2",
2866 				.cra_blocksize = 1,
2867 			},
2868 			.setkey = aead_setkey,
2869 			.setauthsize = aead_setauthsize,
2870 			.encrypt = aead_encrypt,
2871 			.decrypt = aead_decrypt,
2872 			.ivsize = CTR_RFC3686_IV_SIZE,
2873 			.maxauthsize = SHA384_DIGEST_SIZE,
2874 		},
2875 		.caam = {
2876 			.class1_alg_type = OP_ALG_ALGSEL_AES |
2877 					   OP_ALG_AAI_CTR_MOD128,
2878 			.class2_alg_type = OP_ALG_ALGSEL_SHA384 |
2879 					   OP_ALG_AAI_HMAC_PRECOMP,
2880 			.rfc3686 = true,
2881 		},
2882 	},
2883 	{
2884 		.aead = {
2885 			.base = {
2886 				.cra_name = "seqiv(authenc(hmac(sha384),"
2887 					    "rfc3686(ctr(aes))))",
2888 				.cra_driver_name = "seqiv-authenc-hmac-sha384-"
2889 						   "rfc3686-ctr-aes-caam-qi2",
2890 				.cra_blocksize = 1,
2891 			},
2892 			.setkey = aead_setkey,
2893 			.setauthsize = aead_setauthsize,
2894 			.encrypt = aead_encrypt,
2895 			.decrypt = aead_decrypt,
2896 			.ivsize = CTR_RFC3686_IV_SIZE,
2897 			.maxauthsize = SHA384_DIGEST_SIZE,
2898 		},
2899 		.caam = {
2900 			.class1_alg_type = OP_ALG_ALGSEL_AES |
2901 					   OP_ALG_AAI_CTR_MOD128,
2902 			.class2_alg_type = OP_ALG_ALGSEL_SHA384 |
2903 					   OP_ALG_AAI_HMAC_PRECOMP,
2904 			.rfc3686 = true,
2905 			.geniv = true,
2906 		},
2907 	},
2908 	{
2909 		.aead = {
2910 			.base = {
2911 				.cra_name = "rfc7539(chacha20,poly1305)",
2912 				.cra_driver_name = "rfc7539-chacha20-poly1305-"
2913 						   "caam-qi2",
2914 				.cra_blocksize = 1,
2915 			},
2916 			.setkey = chachapoly_setkey,
2917 			.setauthsize = chachapoly_setauthsize,
2918 			.encrypt = aead_encrypt,
2919 			.decrypt = aead_decrypt,
2920 			.ivsize = CHACHAPOLY_IV_SIZE,
2921 			.maxauthsize = POLY1305_DIGEST_SIZE,
2922 		},
2923 		.caam = {
2924 			.class1_alg_type = OP_ALG_ALGSEL_CHACHA20 |
2925 					   OP_ALG_AAI_AEAD,
2926 			.class2_alg_type = OP_ALG_ALGSEL_POLY1305 |
2927 					   OP_ALG_AAI_AEAD,
2928 			.nodkp = true,
2929 		},
2930 	},
2931 	{
2932 		.aead = {
2933 			.base = {
2934 				.cra_name = "rfc7539esp(chacha20,poly1305)",
2935 				.cra_driver_name = "rfc7539esp-chacha20-"
2936 						   "poly1305-caam-qi2",
2937 				.cra_blocksize = 1,
2938 			},
2939 			.setkey = chachapoly_setkey,
2940 			.setauthsize = chachapoly_setauthsize,
2941 			.encrypt = aead_encrypt,
2942 			.decrypt = aead_decrypt,
2943 			.ivsize = 8,
2944 			.maxauthsize = POLY1305_DIGEST_SIZE,
2945 		},
2946 		.caam = {
2947 			.class1_alg_type = OP_ALG_ALGSEL_CHACHA20 |
2948 					   OP_ALG_AAI_AEAD,
2949 			.class2_alg_type = OP_ALG_ALGSEL_POLY1305 |
2950 					   OP_ALG_AAI_AEAD,
2951 			.nodkp = true,
2952 		},
2953 	},
2954 	{
2955 		.aead = {
2956 			.base = {
2957 				.cra_name = "authenc(hmac(sha512),"
2958 					    "rfc3686(ctr(aes)))",
2959 				.cra_driver_name = "authenc-hmac-sha512-"
2960 						   "rfc3686-ctr-aes-caam-qi2",
2961 				.cra_blocksize = 1,
2962 			},
2963 			.setkey = aead_setkey,
2964 			.setauthsize = aead_setauthsize,
2965 			.encrypt = aead_encrypt,
2966 			.decrypt = aead_decrypt,
2967 			.ivsize = CTR_RFC3686_IV_SIZE,
2968 			.maxauthsize = SHA512_DIGEST_SIZE,
2969 		},
2970 		.caam = {
2971 			.class1_alg_type = OP_ALG_ALGSEL_AES |
2972 					   OP_ALG_AAI_CTR_MOD128,
2973 			.class2_alg_type = OP_ALG_ALGSEL_SHA512 |
2974 					   OP_ALG_AAI_HMAC_PRECOMP,
2975 			.rfc3686 = true,
2976 		},
2977 	},
2978 	{
2979 		.aead = {
2980 			.base = {
2981 				.cra_name = "seqiv(authenc(hmac(sha512),"
2982 					    "rfc3686(ctr(aes))))",
2983 				.cra_driver_name = "seqiv-authenc-hmac-sha512-"
2984 						   "rfc3686-ctr-aes-caam-qi2",
2985 				.cra_blocksize = 1,
2986 			},
2987 			.setkey = aead_setkey,
2988 			.setauthsize = aead_setauthsize,
2989 			.encrypt = aead_encrypt,
2990 			.decrypt = aead_decrypt,
2991 			.ivsize = CTR_RFC3686_IV_SIZE,
2992 			.maxauthsize = SHA512_DIGEST_SIZE,
2993 		},
2994 		.caam = {
2995 			.class1_alg_type = OP_ALG_ALGSEL_AES |
2996 					   OP_ALG_AAI_CTR_MOD128,
2997 			.class2_alg_type = OP_ALG_ALGSEL_SHA512 |
2998 					   OP_ALG_AAI_HMAC_PRECOMP,
2999 			.rfc3686 = true,
3000 			.geniv = true,
3001 		},
3002 	},
3003 };
3004 
3005 static void caam_skcipher_alg_init(struct caam_skcipher_alg *t_alg)
3006 {
3007 	struct skcipher_alg *alg = &t_alg->skcipher;
3008 
3009 	alg->base.cra_module = THIS_MODULE;
3010 	alg->base.cra_priority = CAAM_CRA_PRIORITY;
3011 	alg->base.cra_ctxsize = sizeof(struct caam_ctx);
3012 	alg->base.cra_flags |= (CRYPTO_ALG_ASYNC | CRYPTO_ALG_ALLOCATES_MEMORY |
3013 			      CRYPTO_ALG_KERN_DRIVER_ONLY);
3014 
3015 	alg->init = caam_cra_init_skcipher;
3016 	alg->exit = caam_cra_exit;
3017 }
3018 
3019 static void caam_aead_alg_init(struct caam_aead_alg *t_alg)
3020 {
3021 	struct aead_alg *alg = &t_alg->aead;
3022 
3023 	alg->base.cra_module = THIS_MODULE;
3024 	alg->base.cra_priority = CAAM_CRA_PRIORITY;
3025 	alg->base.cra_ctxsize = sizeof(struct caam_ctx);
3026 	alg->base.cra_flags = CRYPTO_ALG_ASYNC | CRYPTO_ALG_ALLOCATES_MEMORY |
3027 			      CRYPTO_ALG_KERN_DRIVER_ONLY;
3028 
3029 	alg->init = caam_cra_init_aead;
3030 	alg->exit = caam_cra_exit_aead;
3031 }
3032 
3033 /* max hash key is max split key size */
3034 #define CAAM_MAX_HASH_KEY_SIZE		(SHA512_DIGEST_SIZE * 2)
3035 
3036 #define CAAM_MAX_HASH_BLOCK_SIZE	SHA512_BLOCK_SIZE
3037 
3038 /* caam context sizes for hashes: running digest + 8 */
3039 #define HASH_MSG_LEN			8
3040 #define MAX_CTX_LEN			(HASH_MSG_LEN + SHA512_DIGEST_SIZE)
3041 
3042 enum hash_optype {
3043 	UPDATE = 0,
3044 	UPDATE_FIRST,
3045 	FINALIZE,
3046 	DIGEST,
3047 	HASH_NUM_OP
3048 };
3049 
3050 /**
3051  * struct caam_hash_ctx - ahash per-session context
3052  * @flc: Flow Contexts array
3053  * @key: authentication key
3054  * @flc_dma: I/O virtual addresses of the Flow Contexts
3055  * @dev: dpseci device
3056  * @ctx_len: size of Context Register
3057  * @adata: hashing algorithm details
3058  */
3059 struct caam_hash_ctx {
3060 	struct caam_flc flc[HASH_NUM_OP];
3061 	u8 key[CAAM_MAX_HASH_BLOCK_SIZE] ____cacheline_aligned;
3062 	dma_addr_t flc_dma[HASH_NUM_OP];
3063 	struct device *dev;
3064 	int ctx_len;
3065 	struct alginfo adata;
3066 };
3067 
3068 /* ahash state */
3069 struct caam_hash_state {
3070 	struct caam_request caam_req;
3071 	dma_addr_t buf_dma;
3072 	dma_addr_t ctx_dma;
3073 	int ctx_dma_len;
3074 	u8 buf[CAAM_MAX_HASH_BLOCK_SIZE] ____cacheline_aligned;
3075 	int buflen;
3076 	int next_buflen;
3077 	u8 caam_ctx[MAX_CTX_LEN] ____cacheline_aligned;
3078 	int (*update)(struct ahash_request *req);
3079 	int (*final)(struct ahash_request *req);
3080 	int (*finup)(struct ahash_request *req);
3081 };
3082 
3083 struct caam_export_state {
3084 	u8 buf[CAAM_MAX_HASH_BLOCK_SIZE];
3085 	u8 caam_ctx[MAX_CTX_LEN];
3086 	int buflen;
3087 	int (*update)(struct ahash_request *req);
3088 	int (*final)(struct ahash_request *req);
3089 	int (*finup)(struct ahash_request *req);
3090 };
3091 
3092 /* Map current buffer in state (if length > 0) and put it in link table */
3093 static inline int buf_map_to_qm_sg(struct device *dev,
3094 				   struct dpaa2_sg_entry *qm_sg,
3095 				   struct caam_hash_state *state)
3096 {
3097 	int buflen = state->buflen;
3098 
3099 	if (!buflen)
3100 		return 0;
3101 
3102 	state->buf_dma = dma_map_single(dev, state->buf, buflen,
3103 					DMA_TO_DEVICE);
3104 	if (dma_mapping_error(dev, state->buf_dma)) {
3105 		dev_err(dev, "unable to map buf\n");
3106 		state->buf_dma = 0;
3107 		return -ENOMEM;
3108 	}
3109 
3110 	dma_to_qm_sg_one(qm_sg, state->buf_dma, buflen, 0);
3111 
3112 	return 0;
3113 }
3114 
3115 /* Map state->caam_ctx, and add it to link table */
3116 static inline int ctx_map_to_qm_sg(struct device *dev,
3117 				   struct caam_hash_state *state, int ctx_len,
3118 				   struct dpaa2_sg_entry *qm_sg, u32 flag)
3119 {
3120 	state->ctx_dma_len = ctx_len;
3121 	state->ctx_dma = dma_map_single(dev, state->caam_ctx, ctx_len, flag);
3122 	if (dma_mapping_error(dev, state->ctx_dma)) {
3123 		dev_err(dev, "unable to map ctx\n");
3124 		state->ctx_dma = 0;
3125 		return -ENOMEM;
3126 	}
3127 
3128 	dma_to_qm_sg_one(qm_sg, state->ctx_dma, ctx_len, 0);
3129 
3130 	return 0;
3131 }
3132 
3133 static int ahash_set_sh_desc(struct crypto_ahash *ahash)
3134 {
3135 	struct caam_hash_ctx *ctx = crypto_ahash_ctx(ahash);
3136 	int digestsize = crypto_ahash_digestsize(ahash);
3137 	struct dpaa2_caam_priv *priv = dev_get_drvdata(ctx->dev);
3138 	struct caam_flc *flc;
3139 	u32 *desc;
3140 
3141 	/* ahash_update shared descriptor */
3142 	flc = &ctx->flc[UPDATE];
3143 	desc = flc->sh_desc;
3144 	cnstr_shdsc_ahash(desc, &ctx->adata, OP_ALG_AS_UPDATE, ctx->ctx_len,
3145 			  ctx->ctx_len, true, priv->sec_attr.era);
3146 	flc->flc[1] = cpu_to_caam32(desc_len(desc)); /* SDL */
3147 	dma_sync_single_for_device(ctx->dev, ctx->flc_dma[UPDATE],
3148 				   desc_bytes(desc), DMA_BIDIRECTIONAL);
3149 	print_hex_dump_debug("ahash update shdesc@" __stringify(__LINE__)": ",
3150 			     DUMP_PREFIX_ADDRESS, 16, 4, desc, desc_bytes(desc),
3151 			     1);
3152 
3153 	/* ahash_update_first shared descriptor */
3154 	flc = &ctx->flc[UPDATE_FIRST];
3155 	desc = flc->sh_desc;
3156 	cnstr_shdsc_ahash(desc, &ctx->adata, OP_ALG_AS_INIT, ctx->ctx_len,
3157 			  ctx->ctx_len, false, priv->sec_attr.era);
3158 	flc->flc[1] = cpu_to_caam32(desc_len(desc)); /* SDL */
3159 	dma_sync_single_for_device(ctx->dev, ctx->flc_dma[UPDATE_FIRST],
3160 				   desc_bytes(desc), DMA_BIDIRECTIONAL);
3161 	print_hex_dump_debug("ahash update first shdesc@" __stringify(__LINE__)": ",
3162 			     DUMP_PREFIX_ADDRESS, 16, 4, desc, desc_bytes(desc),
3163 			     1);
3164 
3165 	/* ahash_final shared descriptor */
3166 	flc = &ctx->flc[FINALIZE];
3167 	desc = flc->sh_desc;
3168 	cnstr_shdsc_ahash(desc, &ctx->adata, OP_ALG_AS_FINALIZE, digestsize,
3169 			  ctx->ctx_len, true, priv->sec_attr.era);
3170 	flc->flc[1] = cpu_to_caam32(desc_len(desc)); /* SDL */
3171 	dma_sync_single_for_device(ctx->dev, ctx->flc_dma[FINALIZE],
3172 				   desc_bytes(desc), DMA_BIDIRECTIONAL);
3173 	print_hex_dump_debug("ahash final shdesc@" __stringify(__LINE__)": ",
3174 			     DUMP_PREFIX_ADDRESS, 16, 4, desc, desc_bytes(desc),
3175 			     1);
3176 
3177 	/* ahash_digest shared descriptor */
3178 	flc = &ctx->flc[DIGEST];
3179 	desc = flc->sh_desc;
3180 	cnstr_shdsc_ahash(desc, &ctx->adata, OP_ALG_AS_INITFINAL, digestsize,
3181 			  ctx->ctx_len, false, priv->sec_attr.era);
3182 	flc->flc[1] = cpu_to_caam32(desc_len(desc)); /* SDL */
3183 	dma_sync_single_for_device(ctx->dev, ctx->flc_dma[DIGEST],
3184 				   desc_bytes(desc), DMA_BIDIRECTIONAL);
3185 	print_hex_dump_debug("ahash digest shdesc@" __stringify(__LINE__)": ",
3186 			     DUMP_PREFIX_ADDRESS, 16, 4, desc, desc_bytes(desc),
3187 			     1);
3188 
3189 	return 0;
3190 }
3191 
3192 struct split_key_sh_result {
3193 	struct completion completion;
3194 	int err;
3195 	struct device *dev;
3196 };
3197 
3198 static void split_key_sh_done(void *cbk_ctx, u32 err)
3199 {
3200 	struct split_key_sh_result *res = cbk_ctx;
3201 
3202 	dev_dbg(res->dev, "%s %d: err 0x%x\n", __func__, __LINE__, err);
3203 
3204 	res->err = err ? caam_qi2_strstatus(res->dev, err) : 0;
3205 	complete(&res->completion);
3206 }
3207 
3208 /* Digest hash size if it is too large */
3209 static int hash_digest_key(struct caam_hash_ctx *ctx, u32 *keylen, u8 *key,
3210 			   u32 digestsize)
3211 {
3212 	struct caam_request *req_ctx;
3213 	u32 *desc;
3214 	struct split_key_sh_result result;
3215 	dma_addr_t key_dma;
3216 	struct caam_flc *flc;
3217 	dma_addr_t flc_dma;
3218 	int ret = -ENOMEM;
3219 	struct dpaa2_fl_entry *in_fle, *out_fle;
3220 
3221 	req_ctx = kzalloc(sizeof(*req_ctx), GFP_KERNEL | GFP_DMA);
3222 	if (!req_ctx)
3223 		return -ENOMEM;
3224 
3225 	in_fle = &req_ctx->fd_flt[1];
3226 	out_fle = &req_ctx->fd_flt[0];
3227 
3228 	flc = kzalloc(sizeof(*flc), GFP_KERNEL | GFP_DMA);
3229 	if (!flc)
3230 		goto err_flc;
3231 
3232 	key_dma = dma_map_single(ctx->dev, key, *keylen, DMA_BIDIRECTIONAL);
3233 	if (dma_mapping_error(ctx->dev, key_dma)) {
3234 		dev_err(ctx->dev, "unable to map key memory\n");
3235 		goto err_key_dma;
3236 	}
3237 
3238 	desc = flc->sh_desc;
3239 
3240 	init_sh_desc(desc, 0);
3241 
3242 	/* descriptor to perform unkeyed hash on key_in */
3243 	append_operation(desc, ctx->adata.algtype | OP_ALG_ENCRYPT |
3244 			 OP_ALG_AS_INITFINAL);
3245 	append_seq_fifo_load(desc, *keylen, FIFOLD_CLASS_CLASS2 |
3246 			     FIFOLD_TYPE_LAST2 | FIFOLD_TYPE_MSG);
3247 	append_seq_store(desc, digestsize, LDST_CLASS_2_CCB |
3248 			 LDST_SRCDST_BYTE_CONTEXT);
3249 
3250 	flc->flc[1] = cpu_to_caam32(desc_len(desc)); /* SDL */
3251 	flc_dma = dma_map_single(ctx->dev, flc, sizeof(flc->flc) +
3252 				 desc_bytes(desc), DMA_TO_DEVICE);
3253 	if (dma_mapping_error(ctx->dev, flc_dma)) {
3254 		dev_err(ctx->dev, "unable to map shared descriptor\n");
3255 		goto err_flc_dma;
3256 	}
3257 
3258 	dpaa2_fl_set_final(in_fle, true);
3259 	dpaa2_fl_set_format(in_fle, dpaa2_fl_single);
3260 	dpaa2_fl_set_addr(in_fle, key_dma);
3261 	dpaa2_fl_set_len(in_fle, *keylen);
3262 	dpaa2_fl_set_format(out_fle, dpaa2_fl_single);
3263 	dpaa2_fl_set_addr(out_fle, key_dma);
3264 	dpaa2_fl_set_len(out_fle, digestsize);
3265 
3266 	print_hex_dump_debug("key_in@" __stringify(__LINE__)": ",
3267 			     DUMP_PREFIX_ADDRESS, 16, 4, key, *keylen, 1);
3268 	print_hex_dump_debug("shdesc@" __stringify(__LINE__)": ",
3269 			     DUMP_PREFIX_ADDRESS, 16, 4, desc, desc_bytes(desc),
3270 			     1);
3271 
3272 	result.err = 0;
3273 	init_completion(&result.completion);
3274 	result.dev = ctx->dev;
3275 
3276 	req_ctx->flc = flc;
3277 	req_ctx->flc_dma = flc_dma;
3278 	req_ctx->cbk = split_key_sh_done;
3279 	req_ctx->ctx = &result;
3280 
3281 	ret = dpaa2_caam_enqueue(ctx->dev, req_ctx);
3282 	if (ret == -EINPROGRESS) {
3283 		/* in progress */
3284 		wait_for_completion(&result.completion);
3285 		ret = result.err;
3286 		print_hex_dump_debug("digested key@" __stringify(__LINE__)": ",
3287 				     DUMP_PREFIX_ADDRESS, 16, 4, key,
3288 				     digestsize, 1);
3289 	}
3290 
3291 	dma_unmap_single(ctx->dev, flc_dma, sizeof(flc->flc) + desc_bytes(desc),
3292 			 DMA_TO_DEVICE);
3293 err_flc_dma:
3294 	dma_unmap_single(ctx->dev, key_dma, *keylen, DMA_BIDIRECTIONAL);
3295 err_key_dma:
3296 	kfree(flc);
3297 err_flc:
3298 	kfree(req_ctx);
3299 
3300 	*keylen = digestsize;
3301 
3302 	return ret;
3303 }
3304 
3305 static int ahash_setkey(struct crypto_ahash *ahash, const u8 *key,
3306 			unsigned int keylen)
3307 {
3308 	struct caam_hash_ctx *ctx = crypto_ahash_ctx(ahash);
3309 	unsigned int blocksize = crypto_tfm_alg_blocksize(&ahash->base);
3310 	unsigned int digestsize = crypto_ahash_digestsize(ahash);
3311 	int ret;
3312 	u8 *hashed_key = NULL;
3313 
3314 	dev_dbg(ctx->dev, "keylen %d blocksize %d\n", keylen, blocksize);
3315 
3316 	if (keylen > blocksize) {
3317 		hashed_key = kmemdup(key, keylen, GFP_KERNEL | GFP_DMA);
3318 		if (!hashed_key)
3319 			return -ENOMEM;
3320 		ret = hash_digest_key(ctx, &keylen, hashed_key, digestsize);
3321 		if (ret)
3322 			goto bad_free_key;
3323 		key = hashed_key;
3324 	}
3325 
3326 	ctx->adata.keylen = keylen;
3327 	ctx->adata.keylen_pad = split_key_len(ctx->adata.algtype &
3328 					      OP_ALG_ALGSEL_MASK);
3329 	if (ctx->adata.keylen_pad > CAAM_MAX_HASH_KEY_SIZE)
3330 		goto bad_free_key;
3331 
3332 	ctx->adata.key_virt = key;
3333 	ctx->adata.key_inline = true;
3334 
3335 	/*
3336 	 * In case |user key| > |derived key|, using DKP<imm,imm> would result
3337 	 * in invalid opcodes (last bytes of user key) in the resulting
3338 	 * descriptor. Use DKP<ptr,imm> instead => both virtual and dma key
3339 	 * addresses are needed.
3340 	 */
3341 	if (keylen > ctx->adata.keylen_pad) {
3342 		memcpy(ctx->key, key, keylen);
3343 		dma_sync_single_for_device(ctx->dev, ctx->adata.key_dma,
3344 					   ctx->adata.keylen_pad,
3345 					   DMA_TO_DEVICE);
3346 	}
3347 
3348 	ret = ahash_set_sh_desc(ahash);
3349 	kfree(hashed_key);
3350 	return ret;
3351 bad_free_key:
3352 	kfree(hashed_key);
3353 	return -EINVAL;
3354 }
3355 
3356 static inline void ahash_unmap(struct device *dev, struct ahash_edesc *edesc,
3357 			       struct ahash_request *req)
3358 {
3359 	struct caam_hash_state *state = ahash_request_ctx(req);
3360 
3361 	if (edesc->src_nents)
3362 		dma_unmap_sg(dev, req->src, edesc->src_nents, DMA_TO_DEVICE);
3363 
3364 	if (edesc->qm_sg_bytes)
3365 		dma_unmap_single(dev, edesc->qm_sg_dma, edesc->qm_sg_bytes,
3366 				 DMA_TO_DEVICE);
3367 
3368 	if (state->buf_dma) {
3369 		dma_unmap_single(dev, state->buf_dma, state->buflen,
3370 				 DMA_TO_DEVICE);
3371 		state->buf_dma = 0;
3372 	}
3373 }
3374 
3375 static inline void ahash_unmap_ctx(struct device *dev,
3376 				   struct ahash_edesc *edesc,
3377 				   struct ahash_request *req, u32 flag)
3378 {
3379 	struct caam_hash_state *state = ahash_request_ctx(req);
3380 
3381 	if (state->ctx_dma) {
3382 		dma_unmap_single(dev, state->ctx_dma, state->ctx_dma_len, flag);
3383 		state->ctx_dma = 0;
3384 	}
3385 	ahash_unmap(dev, edesc, req);
3386 }
3387 
3388 static void ahash_done(void *cbk_ctx, u32 status)
3389 {
3390 	struct crypto_async_request *areq = cbk_ctx;
3391 	struct ahash_request *req = ahash_request_cast(areq);
3392 	struct crypto_ahash *ahash = crypto_ahash_reqtfm(req);
3393 	struct caam_hash_state *state = ahash_request_ctx(req);
3394 	struct ahash_edesc *edesc = state->caam_req.edesc;
3395 	struct caam_hash_ctx *ctx = crypto_ahash_ctx(ahash);
3396 	int digestsize = crypto_ahash_digestsize(ahash);
3397 	int ecode = 0;
3398 
3399 	dev_dbg(ctx->dev, "%s %d: err 0x%x\n", __func__, __LINE__, status);
3400 
3401 	if (unlikely(status))
3402 		ecode = caam_qi2_strstatus(ctx->dev, status);
3403 
3404 	ahash_unmap_ctx(ctx->dev, edesc, req, DMA_FROM_DEVICE);
3405 	memcpy(req->result, state->caam_ctx, digestsize);
3406 	qi_cache_free(edesc);
3407 
3408 	print_hex_dump_debug("ctx@" __stringify(__LINE__)": ",
3409 			     DUMP_PREFIX_ADDRESS, 16, 4, state->caam_ctx,
3410 			     ctx->ctx_len, 1);
3411 
3412 	req->base.complete(&req->base, ecode);
3413 }
3414 
3415 static void ahash_done_bi(void *cbk_ctx, u32 status)
3416 {
3417 	struct crypto_async_request *areq = cbk_ctx;
3418 	struct ahash_request *req = ahash_request_cast(areq);
3419 	struct crypto_ahash *ahash = crypto_ahash_reqtfm(req);
3420 	struct caam_hash_state *state = ahash_request_ctx(req);
3421 	struct ahash_edesc *edesc = state->caam_req.edesc;
3422 	struct caam_hash_ctx *ctx = crypto_ahash_ctx(ahash);
3423 	int ecode = 0;
3424 
3425 	dev_dbg(ctx->dev, "%s %d: err 0x%x\n", __func__, __LINE__, status);
3426 
3427 	if (unlikely(status))
3428 		ecode = caam_qi2_strstatus(ctx->dev, status);
3429 
3430 	ahash_unmap_ctx(ctx->dev, edesc, req, DMA_BIDIRECTIONAL);
3431 	qi_cache_free(edesc);
3432 
3433 	scatterwalk_map_and_copy(state->buf, req->src,
3434 				 req->nbytes - state->next_buflen,
3435 				 state->next_buflen, 0);
3436 	state->buflen = state->next_buflen;
3437 
3438 	print_hex_dump_debug("buf@" __stringify(__LINE__)": ",
3439 			     DUMP_PREFIX_ADDRESS, 16, 4, state->buf,
3440 			     state->buflen, 1);
3441 
3442 	print_hex_dump_debug("ctx@" __stringify(__LINE__)": ",
3443 			     DUMP_PREFIX_ADDRESS, 16, 4, state->caam_ctx,
3444 			     ctx->ctx_len, 1);
3445 	if (req->result)
3446 		print_hex_dump_debug("result@" __stringify(__LINE__)": ",
3447 				     DUMP_PREFIX_ADDRESS, 16, 4, req->result,
3448 				     crypto_ahash_digestsize(ahash), 1);
3449 
3450 	req->base.complete(&req->base, ecode);
3451 }
3452 
3453 static void ahash_done_ctx_src(void *cbk_ctx, u32 status)
3454 {
3455 	struct crypto_async_request *areq = cbk_ctx;
3456 	struct ahash_request *req = ahash_request_cast(areq);
3457 	struct crypto_ahash *ahash = crypto_ahash_reqtfm(req);
3458 	struct caam_hash_state *state = ahash_request_ctx(req);
3459 	struct ahash_edesc *edesc = state->caam_req.edesc;
3460 	struct caam_hash_ctx *ctx = crypto_ahash_ctx(ahash);
3461 	int digestsize = crypto_ahash_digestsize(ahash);
3462 	int ecode = 0;
3463 
3464 	dev_dbg(ctx->dev, "%s %d: err 0x%x\n", __func__, __LINE__, status);
3465 
3466 	if (unlikely(status))
3467 		ecode = caam_qi2_strstatus(ctx->dev, status);
3468 
3469 	ahash_unmap_ctx(ctx->dev, edesc, req, DMA_BIDIRECTIONAL);
3470 	memcpy(req->result, state->caam_ctx, digestsize);
3471 	qi_cache_free(edesc);
3472 
3473 	print_hex_dump_debug("ctx@" __stringify(__LINE__)": ",
3474 			     DUMP_PREFIX_ADDRESS, 16, 4, state->caam_ctx,
3475 			     ctx->ctx_len, 1);
3476 
3477 	req->base.complete(&req->base, ecode);
3478 }
3479 
3480 static void ahash_done_ctx_dst(void *cbk_ctx, u32 status)
3481 {
3482 	struct crypto_async_request *areq = cbk_ctx;
3483 	struct ahash_request *req = ahash_request_cast(areq);
3484 	struct crypto_ahash *ahash = crypto_ahash_reqtfm(req);
3485 	struct caam_hash_state *state = ahash_request_ctx(req);
3486 	struct ahash_edesc *edesc = state->caam_req.edesc;
3487 	struct caam_hash_ctx *ctx = crypto_ahash_ctx(ahash);
3488 	int ecode = 0;
3489 
3490 	dev_dbg(ctx->dev, "%s %d: err 0x%x\n", __func__, __LINE__, status);
3491 
3492 	if (unlikely(status))
3493 		ecode = caam_qi2_strstatus(ctx->dev, status);
3494 
3495 	ahash_unmap_ctx(ctx->dev, edesc, req, DMA_FROM_DEVICE);
3496 	qi_cache_free(edesc);
3497 
3498 	scatterwalk_map_and_copy(state->buf, req->src,
3499 				 req->nbytes - state->next_buflen,
3500 				 state->next_buflen, 0);
3501 	state->buflen = state->next_buflen;
3502 
3503 	print_hex_dump_debug("buf@" __stringify(__LINE__)": ",
3504 			     DUMP_PREFIX_ADDRESS, 16, 4, state->buf,
3505 			     state->buflen, 1);
3506 
3507 	print_hex_dump_debug("ctx@" __stringify(__LINE__)": ",
3508 			     DUMP_PREFIX_ADDRESS, 16, 4, state->caam_ctx,
3509 			     ctx->ctx_len, 1);
3510 	if (req->result)
3511 		print_hex_dump_debug("result@" __stringify(__LINE__)": ",
3512 				     DUMP_PREFIX_ADDRESS, 16, 4, req->result,
3513 				     crypto_ahash_digestsize(ahash), 1);
3514 
3515 	req->base.complete(&req->base, ecode);
3516 }
3517 
3518 static int ahash_update_ctx(struct ahash_request *req)
3519 {
3520 	struct crypto_ahash *ahash = crypto_ahash_reqtfm(req);
3521 	struct caam_hash_ctx *ctx = crypto_ahash_ctx(ahash);
3522 	struct caam_hash_state *state = ahash_request_ctx(req);
3523 	struct caam_request *req_ctx = &state->caam_req;
3524 	struct dpaa2_fl_entry *in_fle = &req_ctx->fd_flt[1];
3525 	struct dpaa2_fl_entry *out_fle = &req_ctx->fd_flt[0];
3526 	gfp_t flags = (req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP) ?
3527 		      GFP_KERNEL : GFP_ATOMIC;
3528 	u8 *buf = state->buf;
3529 	int *buflen = &state->buflen;
3530 	int *next_buflen = &state->next_buflen;
3531 	int in_len = *buflen + req->nbytes, to_hash;
3532 	int src_nents, mapped_nents, qm_sg_bytes, qm_sg_src_index;
3533 	struct ahash_edesc *edesc;
3534 	int ret = 0;
3535 
3536 	*next_buflen = in_len & (crypto_tfm_alg_blocksize(&ahash->base) - 1);
3537 	to_hash = in_len - *next_buflen;
3538 
3539 	if (to_hash) {
3540 		struct dpaa2_sg_entry *sg_table;
3541 		int src_len = req->nbytes - *next_buflen;
3542 
3543 		src_nents = sg_nents_for_len(req->src, src_len);
3544 		if (src_nents < 0) {
3545 			dev_err(ctx->dev, "Invalid number of src SG.\n");
3546 			return src_nents;
3547 		}
3548 
3549 		if (src_nents) {
3550 			mapped_nents = dma_map_sg(ctx->dev, req->src, src_nents,
3551 						  DMA_TO_DEVICE);
3552 			if (!mapped_nents) {
3553 				dev_err(ctx->dev, "unable to DMA map source\n");
3554 				return -ENOMEM;
3555 			}
3556 		} else {
3557 			mapped_nents = 0;
3558 		}
3559 
3560 		/* allocate space for base edesc and link tables */
3561 		edesc = qi_cache_zalloc(GFP_DMA | flags);
3562 		if (!edesc) {
3563 			dma_unmap_sg(ctx->dev, req->src, src_nents,
3564 				     DMA_TO_DEVICE);
3565 			return -ENOMEM;
3566 		}
3567 
3568 		edesc->src_nents = src_nents;
3569 		qm_sg_src_index = 1 + (*buflen ? 1 : 0);
3570 		qm_sg_bytes = pad_sg_nents(qm_sg_src_index + mapped_nents) *
3571 			      sizeof(*sg_table);
3572 		sg_table = &edesc->sgt[0];
3573 
3574 		ret = ctx_map_to_qm_sg(ctx->dev, state, ctx->ctx_len, sg_table,
3575 				       DMA_BIDIRECTIONAL);
3576 		if (ret)
3577 			goto unmap_ctx;
3578 
3579 		ret = buf_map_to_qm_sg(ctx->dev, sg_table + 1, state);
3580 		if (ret)
3581 			goto unmap_ctx;
3582 
3583 		if (mapped_nents) {
3584 			sg_to_qm_sg_last(req->src, src_len,
3585 					 sg_table + qm_sg_src_index, 0);
3586 		} else {
3587 			dpaa2_sg_set_final(sg_table + qm_sg_src_index - 1,
3588 					   true);
3589 		}
3590 
3591 		edesc->qm_sg_dma = dma_map_single(ctx->dev, sg_table,
3592 						  qm_sg_bytes, DMA_TO_DEVICE);
3593 		if (dma_mapping_error(ctx->dev, edesc->qm_sg_dma)) {
3594 			dev_err(ctx->dev, "unable to map S/G table\n");
3595 			ret = -ENOMEM;
3596 			goto unmap_ctx;
3597 		}
3598 		edesc->qm_sg_bytes = qm_sg_bytes;
3599 
3600 		memset(&req_ctx->fd_flt, 0, sizeof(req_ctx->fd_flt));
3601 		dpaa2_fl_set_final(in_fle, true);
3602 		dpaa2_fl_set_format(in_fle, dpaa2_fl_sg);
3603 		dpaa2_fl_set_addr(in_fle, edesc->qm_sg_dma);
3604 		dpaa2_fl_set_len(in_fle, ctx->ctx_len + to_hash);
3605 		dpaa2_fl_set_format(out_fle, dpaa2_fl_single);
3606 		dpaa2_fl_set_addr(out_fle, state->ctx_dma);
3607 		dpaa2_fl_set_len(out_fle, ctx->ctx_len);
3608 
3609 		req_ctx->flc = &ctx->flc[UPDATE];
3610 		req_ctx->flc_dma = ctx->flc_dma[UPDATE];
3611 		req_ctx->cbk = ahash_done_bi;
3612 		req_ctx->ctx = &req->base;
3613 		req_ctx->edesc = edesc;
3614 
3615 		ret = dpaa2_caam_enqueue(ctx->dev, req_ctx);
3616 		if (ret != -EINPROGRESS &&
3617 		    !(ret == -EBUSY &&
3618 		      req->base.flags & CRYPTO_TFM_REQ_MAY_BACKLOG))
3619 			goto unmap_ctx;
3620 	} else if (*next_buflen) {
3621 		scatterwalk_map_and_copy(buf + *buflen, req->src, 0,
3622 					 req->nbytes, 0);
3623 		*buflen = *next_buflen;
3624 
3625 		print_hex_dump_debug("buf@" __stringify(__LINE__)": ",
3626 				     DUMP_PREFIX_ADDRESS, 16, 4, buf,
3627 				     *buflen, 1);
3628 	}
3629 
3630 	return ret;
3631 unmap_ctx:
3632 	ahash_unmap_ctx(ctx->dev, edesc, req, DMA_BIDIRECTIONAL);
3633 	qi_cache_free(edesc);
3634 	return ret;
3635 }
3636 
3637 static int ahash_final_ctx(struct ahash_request *req)
3638 {
3639 	struct crypto_ahash *ahash = crypto_ahash_reqtfm(req);
3640 	struct caam_hash_ctx *ctx = crypto_ahash_ctx(ahash);
3641 	struct caam_hash_state *state = ahash_request_ctx(req);
3642 	struct caam_request *req_ctx = &state->caam_req;
3643 	struct dpaa2_fl_entry *in_fle = &req_ctx->fd_flt[1];
3644 	struct dpaa2_fl_entry *out_fle = &req_ctx->fd_flt[0];
3645 	gfp_t flags = (req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP) ?
3646 		      GFP_KERNEL : GFP_ATOMIC;
3647 	int buflen = state->buflen;
3648 	int qm_sg_bytes;
3649 	int digestsize = crypto_ahash_digestsize(ahash);
3650 	struct ahash_edesc *edesc;
3651 	struct dpaa2_sg_entry *sg_table;
3652 	int ret;
3653 
3654 	/* allocate space for base edesc and link tables */
3655 	edesc = qi_cache_zalloc(GFP_DMA | flags);
3656 	if (!edesc)
3657 		return -ENOMEM;
3658 
3659 	qm_sg_bytes = pad_sg_nents(1 + (buflen ? 1 : 0)) * sizeof(*sg_table);
3660 	sg_table = &edesc->sgt[0];
3661 
3662 	ret = ctx_map_to_qm_sg(ctx->dev, state, ctx->ctx_len, sg_table,
3663 			       DMA_BIDIRECTIONAL);
3664 	if (ret)
3665 		goto unmap_ctx;
3666 
3667 	ret = buf_map_to_qm_sg(ctx->dev, sg_table + 1, state);
3668 	if (ret)
3669 		goto unmap_ctx;
3670 
3671 	dpaa2_sg_set_final(sg_table + (buflen ? 1 : 0), true);
3672 
3673 	edesc->qm_sg_dma = dma_map_single(ctx->dev, sg_table, qm_sg_bytes,
3674 					  DMA_TO_DEVICE);
3675 	if (dma_mapping_error(ctx->dev, edesc->qm_sg_dma)) {
3676 		dev_err(ctx->dev, "unable to map S/G table\n");
3677 		ret = -ENOMEM;
3678 		goto unmap_ctx;
3679 	}
3680 	edesc->qm_sg_bytes = qm_sg_bytes;
3681 
3682 	memset(&req_ctx->fd_flt, 0, sizeof(req_ctx->fd_flt));
3683 	dpaa2_fl_set_final(in_fle, true);
3684 	dpaa2_fl_set_format(in_fle, dpaa2_fl_sg);
3685 	dpaa2_fl_set_addr(in_fle, edesc->qm_sg_dma);
3686 	dpaa2_fl_set_len(in_fle, ctx->ctx_len + buflen);
3687 	dpaa2_fl_set_format(out_fle, dpaa2_fl_single);
3688 	dpaa2_fl_set_addr(out_fle, state->ctx_dma);
3689 	dpaa2_fl_set_len(out_fle, digestsize);
3690 
3691 	req_ctx->flc = &ctx->flc[FINALIZE];
3692 	req_ctx->flc_dma = ctx->flc_dma[FINALIZE];
3693 	req_ctx->cbk = ahash_done_ctx_src;
3694 	req_ctx->ctx = &req->base;
3695 	req_ctx->edesc = edesc;
3696 
3697 	ret = dpaa2_caam_enqueue(ctx->dev, req_ctx);
3698 	if (ret == -EINPROGRESS ||
3699 	    (ret == -EBUSY && req->base.flags & CRYPTO_TFM_REQ_MAY_BACKLOG))
3700 		return ret;
3701 
3702 unmap_ctx:
3703 	ahash_unmap_ctx(ctx->dev, edesc, req, DMA_BIDIRECTIONAL);
3704 	qi_cache_free(edesc);
3705 	return ret;
3706 }
3707 
3708 static int ahash_finup_ctx(struct ahash_request *req)
3709 {
3710 	struct crypto_ahash *ahash = crypto_ahash_reqtfm(req);
3711 	struct caam_hash_ctx *ctx = crypto_ahash_ctx(ahash);
3712 	struct caam_hash_state *state = ahash_request_ctx(req);
3713 	struct caam_request *req_ctx = &state->caam_req;
3714 	struct dpaa2_fl_entry *in_fle = &req_ctx->fd_flt[1];
3715 	struct dpaa2_fl_entry *out_fle = &req_ctx->fd_flt[0];
3716 	gfp_t flags = (req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP) ?
3717 		      GFP_KERNEL : GFP_ATOMIC;
3718 	int buflen = state->buflen;
3719 	int qm_sg_bytes, qm_sg_src_index;
3720 	int src_nents, mapped_nents;
3721 	int digestsize = crypto_ahash_digestsize(ahash);
3722 	struct ahash_edesc *edesc;
3723 	struct dpaa2_sg_entry *sg_table;
3724 	int ret;
3725 
3726 	src_nents = sg_nents_for_len(req->src, req->nbytes);
3727 	if (src_nents < 0) {
3728 		dev_err(ctx->dev, "Invalid number of src SG.\n");
3729 		return src_nents;
3730 	}
3731 
3732 	if (src_nents) {
3733 		mapped_nents = dma_map_sg(ctx->dev, req->src, src_nents,
3734 					  DMA_TO_DEVICE);
3735 		if (!mapped_nents) {
3736 			dev_err(ctx->dev, "unable to DMA map source\n");
3737 			return -ENOMEM;
3738 		}
3739 	} else {
3740 		mapped_nents = 0;
3741 	}
3742 
3743 	/* allocate space for base edesc and link tables */
3744 	edesc = qi_cache_zalloc(GFP_DMA | flags);
3745 	if (!edesc) {
3746 		dma_unmap_sg(ctx->dev, req->src, src_nents, DMA_TO_DEVICE);
3747 		return -ENOMEM;
3748 	}
3749 
3750 	edesc->src_nents = src_nents;
3751 	qm_sg_src_index = 1 + (buflen ? 1 : 0);
3752 	qm_sg_bytes = pad_sg_nents(qm_sg_src_index + mapped_nents) *
3753 		      sizeof(*sg_table);
3754 	sg_table = &edesc->sgt[0];
3755 
3756 	ret = ctx_map_to_qm_sg(ctx->dev, state, ctx->ctx_len, sg_table,
3757 			       DMA_BIDIRECTIONAL);
3758 	if (ret)
3759 		goto unmap_ctx;
3760 
3761 	ret = buf_map_to_qm_sg(ctx->dev, sg_table + 1, state);
3762 	if (ret)
3763 		goto unmap_ctx;
3764 
3765 	sg_to_qm_sg_last(req->src, req->nbytes, sg_table + qm_sg_src_index, 0);
3766 
3767 	edesc->qm_sg_dma = dma_map_single(ctx->dev, sg_table, qm_sg_bytes,
3768 					  DMA_TO_DEVICE);
3769 	if (dma_mapping_error(ctx->dev, edesc->qm_sg_dma)) {
3770 		dev_err(ctx->dev, "unable to map S/G table\n");
3771 		ret = -ENOMEM;
3772 		goto unmap_ctx;
3773 	}
3774 	edesc->qm_sg_bytes = qm_sg_bytes;
3775 
3776 	memset(&req_ctx->fd_flt, 0, sizeof(req_ctx->fd_flt));
3777 	dpaa2_fl_set_final(in_fle, true);
3778 	dpaa2_fl_set_format(in_fle, dpaa2_fl_sg);
3779 	dpaa2_fl_set_addr(in_fle, edesc->qm_sg_dma);
3780 	dpaa2_fl_set_len(in_fle, ctx->ctx_len + buflen + req->nbytes);
3781 	dpaa2_fl_set_format(out_fle, dpaa2_fl_single);
3782 	dpaa2_fl_set_addr(out_fle, state->ctx_dma);
3783 	dpaa2_fl_set_len(out_fle, digestsize);
3784 
3785 	req_ctx->flc = &ctx->flc[FINALIZE];
3786 	req_ctx->flc_dma = ctx->flc_dma[FINALIZE];
3787 	req_ctx->cbk = ahash_done_ctx_src;
3788 	req_ctx->ctx = &req->base;
3789 	req_ctx->edesc = edesc;
3790 
3791 	ret = dpaa2_caam_enqueue(ctx->dev, req_ctx);
3792 	if (ret == -EINPROGRESS ||
3793 	    (ret == -EBUSY && req->base.flags & CRYPTO_TFM_REQ_MAY_BACKLOG))
3794 		return ret;
3795 
3796 unmap_ctx:
3797 	ahash_unmap_ctx(ctx->dev, edesc, req, DMA_BIDIRECTIONAL);
3798 	qi_cache_free(edesc);
3799 	return ret;
3800 }
3801 
3802 static int ahash_digest(struct ahash_request *req)
3803 {
3804 	struct crypto_ahash *ahash = crypto_ahash_reqtfm(req);
3805 	struct caam_hash_ctx *ctx = crypto_ahash_ctx(ahash);
3806 	struct caam_hash_state *state = ahash_request_ctx(req);
3807 	struct caam_request *req_ctx = &state->caam_req;
3808 	struct dpaa2_fl_entry *in_fle = &req_ctx->fd_flt[1];
3809 	struct dpaa2_fl_entry *out_fle = &req_ctx->fd_flt[0];
3810 	gfp_t flags = (req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP) ?
3811 		      GFP_KERNEL : GFP_ATOMIC;
3812 	int digestsize = crypto_ahash_digestsize(ahash);
3813 	int src_nents, mapped_nents;
3814 	struct ahash_edesc *edesc;
3815 	int ret = -ENOMEM;
3816 
3817 	state->buf_dma = 0;
3818 
3819 	src_nents = sg_nents_for_len(req->src, req->nbytes);
3820 	if (src_nents < 0) {
3821 		dev_err(ctx->dev, "Invalid number of src SG.\n");
3822 		return src_nents;
3823 	}
3824 
3825 	if (src_nents) {
3826 		mapped_nents = dma_map_sg(ctx->dev, req->src, src_nents,
3827 					  DMA_TO_DEVICE);
3828 		if (!mapped_nents) {
3829 			dev_err(ctx->dev, "unable to map source for DMA\n");
3830 			return ret;
3831 		}
3832 	} else {
3833 		mapped_nents = 0;
3834 	}
3835 
3836 	/* allocate space for base edesc and link tables */
3837 	edesc = qi_cache_zalloc(GFP_DMA | flags);
3838 	if (!edesc) {
3839 		dma_unmap_sg(ctx->dev, req->src, src_nents, DMA_TO_DEVICE);
3840 		return ret;
3841 	}
3842 
3843 	edesc->src_nents = src_nents;
3844 	memset(&req_ctx->fd_flt, 0, sizeof(req_ctx->fd_flt));
3845 
3846 	if (mapped_nents > 1) {
3847 		int qm_sg_bytes;
3848 		struct dpaa2_sg_entry *sg_table = &edesc->sgt[0];
3849 
3850 		qm_sg_bytes = pad_sg_nents(mapped_nents) * sizeof(*sg_table);
3851 		sg_to_qm_sg_last(req->src, req->nbytes, sg_table, 0);
3852 		edesc->qm_sg_dma = dma_map_single(ctx->dev, sg_table,
3853 						  qm_sg_bytes, DMA_TO_DEVICE);
3854 		if (dma_mapping_error(ctx->dev, edesc->qm_sg_dma)) {
3855 			dev_err(ctx->dev, "unable to map S/G table\n");
3856 			goto unmap;
3857 		}
3858 		edesc->qm_sg_bytes = qm_sg_bytes;
3859 		dpaa2_fl_set_format(in_fle, dpaa2_fl_sg);
3860 		dpaa2_fl_set_addr(in_fle, edesc->qm_sg_dma);
3861 	} else {
3862 		dpaa2_fl_set_format(in_fle, dpaa2_fl_single);
3863 		dpaa2_fl_set_addr(in_fle, sg_dma_address(req->src));
3864 	}
3865 
3866 	state->ctx_dma_len = digestsize;
3867 	state->ctx_dma = dma_map_single(ctx->dev, state->caam_ctx, digestsize,
3868 					DMA_FROM_DEVICE);
3869 	if (dma_mapping_error(ctx->dev, state->ctx_dma)) {
3870 		dev_err(ctx->dev, "unable to map ctx\n");
3871 		state->ctx_dma = 0;
3872 		goto unmap;
3873 	}
3874 
3875 	dpaa2_fl_set_final(in_fle, true);
3876 	dpaa2_fl_set_len(in_fle, req->nbytes);
3877 	dpaa2_fl_set_format(out_fle, dpaa2_fl_single);
3878 	dpaa2_fl_set_addr(out_fle, state->ctx_dma);
3879 	dpaa2_fl_set_len(out_fle, digestsize);
3880 
3881 	req_ctx->flc = &ctx->flc[DIGEST];
3882 	req_ctx->flc_dma = ctx->flc_dma[DIGEST];
3883 	req_ctx->cbk = ahash_done;
3884 	req_ctx->ctx = &req->base;
3885 	req_ctx->edesc = edesc;
3886 	ret = dpaa2_caam_enqueue(ctx->dev, req_ctx);
3887 	if (ret == -EINPROGRESS ||
3888 	    (ret == -EBUSY && req->base.flags & CRYPTO_TFM_REQ_MAY_BACKLOG))
3889 		return ret;
3890 
3891 unmap:
3892 	ahash_unmap_ctx(ctx->dev, edesc, req, DMA_FROM_DEVICE);
3893 	qi_cache_free(edesc);
3894 	return ret;
3895 }
3896 
3897 static int ahash_final_no_ctx(struct ahash_request *req)
3898 {
3899 	struct crypto_ahash *ahash = crypto_ahash_reqtfm(req);
3900 	struct caam_hash_ctx *ctx = crypto_ahash_ctx(ahash);
3901 	struct caam_hash_state *state = ahash_request_ctx(req);
3902 	struct caam_request *req_ctx = &state->caam_req;
3903 	struct dpaa2_fl_entry *in_fle = &req_ctx->fd_flt[1];
3904 	struct dpaa2_fl_entry *out_fle = &req_ctx->fd_flt[0];
3905 	gfp_t flags = (req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP) ?
3906 		      GFP_KERNEL : GFP_ATOMIC;
3907 	u8 *buf = state->buf;
3908 	int buflen = state->buflen;
3909 	int digestsize = crypto_ahash_digestsize(ahash);
3910 	struct ahash_edesc *edesc;
3911 	int ret = -ENOMEM;
3912 
3913 	/* allocate space for base edesc and link tables */
3914 	edesc = qi_cache_zalloc(GFP_DMA | flags);
3915 	if (!edesc)
3916 		return ret;
3917 
3918 	if (buflen) {
3919 		state->buf_dma = dma_map_single(ctx->dev, buf, buflen,
3920 						DMA_TO_DEVICE);
3921 		if (dma_mapping_error(ctx->dev, state->buf_dma)) {
3922 			dev_err(ctx->dev, "unable to map src\n");
3923 			goto unmap;
3924 		}
3925 	}
3926 
3927 	state->ctx_dma_len = digestsize;
3928 	state->ctx_dma = dma_map_single(ctx->dev, state->caam_ctx, digestsize,
3929 					DMA_FROM_DEVICE);
3930 	if (dma_mapping_error(ctx->dev, state->ctx_dma)) {
3931 		dev_err(ctx->dev, "unable to map ctx\n");
3932 		state->ctx_dma = 0;
3933 		goto unmap;
3934 	}
3935 
3936 	memset(&req_ctx->fd_flt, 0, sizeof(req_ctx->fd_flt));
3937 	dpaa2_fl_set_final(in_fle, true);
3938 	/*
3939 	 * crypto engine requires the input entry to be present when
3940 	 * "frame list" FD is used.
3941 	 * Since engine does not support FMT=2'b11 (unused entry type), leaving
3942 	 * in_fle zeroized (except for "Final" flag) is the best option.
3943 	 */
3944 	if (buflen) {
3945 		dpaa2_fl_set_format(in_fle, dpaa2_fl_single);
3946 		dpaa2_fl_set_addr(in_fle, state->buf_dma);
3947 		dpaa2_fl_set_len(in_fle, buflen);
3948 	}
3949 	dpaa2_fl_set_format(out_fle, dpaa2_fl_single);
3950 	dpaa2_fl_set_addr(out_fle, state->ctx_dma);
3951 	dpaa2_fl_set_len(out_fle, digestsize);
3952 
3953 	req_ctx->flc = &ctx->flc[DIGEST];
3954 	req_ctx->flc_dma = ctx->flc_dma[DIGEST];
3955 	req_ctx->cbk = ahash_done;
3956 	req_ctx->ctx = &req->base;
3957 	req_ctx->edesc = edesc;
3958 
3959 	ret = dpaa2_caam_enqueue(ctx->dev, req_ctx);
3960 	if (ret == -EINPROGRESS ||
3961 	    (ret == -EBUSY && req->base.flags & CRYPTO_TFM_REQ_MAY_BACKLOG))
3962 		return ret;
3963 
3964 unmap:
3965 	ahash_unmap_ctx(ctx->dev, edesc, req, DMA_FROM_DEVICE);
3966 	qi_cache_free(edesc);
3967 	return ret;
3968 }
3969 
3970 static int ahash_update_no_ctx(struct ahash_request *req)
3971 {
3972 	struct crypto_ahash *ahash = crypto_ahash_reqtfm(req);
3973 	struct caam_hash_ctx *ctx = crypto_ahash_ctx(ahash);
3974 	struct caam_hash_state *state = ahash_request_ctx(req);
3975 	struct caam_request *req_ctx = &state->caam_req;
3976 	struct dpaa2_fl_entry *in_fle = &req_ctx->fd_flt[1];
3977 	struct dpaa2_fl_entry *out_fle = &req_ctx->fd_flt[0];
3978 	gfp_t flags = (req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP) ?
3979 		      GFP_KERNEL : GFP_ATOMIC;
3980 	u8 *buf = state->buf;
3981 	int *buflen = &state->buflen;
3982 	int *next_buflen = &state->next_buflen;
3983 	int in_len = *buflen + req->nbytes, to_hash;
3984 	int qm_sg_bytes, src_nents, mapped_nents;
3985 	struct ahash_edesc *edesc;
3986 	int ret = 0;
3987 
3988 	*next_buflen = in_len & (crypto_tfm_alg_blocksize(&ahash->base) - 1);
3989 	to_hash = in_len - *next_buflen;
3990 
3991 	if (to_hash) {
3992 		struct dpaa2_sg_entry *sg_table;
3993 		int src_len = req->nbytes - *next_buflen;
3994 
3995 		src_nents = sg_nents_for_len(req->src, src_len);
3996 		if (src_nents < 0) {
3997 			dev_err(ctx->dev, "Invalid number of src SG.\n");
3998 			return src_nents;
3999 		}
4000 
4001 		if (src_nents) {
4002 			mapped_nents = dma_map_sg(ctx->dev, req->src, src_nents,
4003 						  DMA_TO_DEVICE);
4004 			if (!mapped_nents) {
4005 				dev_err(ctx->dev, "unable to DMA map source\n");
4006 				return -ENOMEM;
4007 			}
4008 		} else {
4009 			mapped_nents = 0;
4010 		}
4011 
4012 		/* allocate space for base edesc and link tables */
4013 		edesc = qi_cache_zalloc(GFP_DMA | flags);
4014 		if (!edesc) {
4015 			dma_unmap_sg(ctx->dev, req->src, src_nents,
4016 				     DMA_TO_DEVICE);
4017 			return -ENOMEM;
4018 		}
4019 
4020 		edesc->src_nents = src_nents;
4021 		qm_sg_bytes = pad_sg_nents(1 + mapped_nents) *
4022 			      sizeof(*sg_table);
4023 		sg_table = &edesc->sgt[0];
4024 
4025 		ret = buf_map_to_qm_sg(ctx->dev, sg_table, state);
4026 		if (ret)
4027 			goto unmap_ctx;
4028 
4029 		sg_to_qm_sg_last(req->src, src_len, sg_table + 1, 0);
4030 
4031 		edesc->qm_sg_dma = dma_map_single(ctx->dev, sg_table,
4032 						  qm_sg_bytes, DMA_TO_DEVICE);
4033 		if (dma_mapping_error(ctx->dev, edesc->qm_sg_dma)) {
4034 			dev_err(ctx->dev, "unable to map S/G table\n");
4035 			ret = -ENOMEM;
4036 			goto unmap_ctx;
4037 		}
4038 		edesc->qm_sg_bytes = qm_sg_bytes;
4039 
4040 		state->ctx_dma_len = ctx->ctx_len;
4041 		state->ctx_dma = dma_map_single(ctx->dev, state->caam_ctx,
4042 						ctx->ctx_len, DMA_FROM_DEVICE);
4043 		if (dma_mapping_error(ctx->dev, state->ctx_dma)) {
4044 			dev_err(ctx->dev, "unable to map ctx\n");
4045 			state->ctx_dma = 0;
4046 			ret = -ENOMEM;
4047 			goto unmap_ctx;
4048 		}
4049 
4050 		memset(&req_ctx->fd_flt, 0, sizeof(req_ctx->fd_flt));
4051 		dpaa2_fl_set_final(in_fle, true);
4052 		dpaa2_fl_set_format(in_fle, dpaa2_fl_sg);
4053 		dpaa2_fl_set_addr(in_fle, edesc->qm_sg_dma);
4054 		dpaa2_fl_set_len(in_fle, to_hash);
4055 		dpaa2_fl_set_format(out_fle, dpaa2_fl_single);
4056 		dpaa2_fl_set_addr(out_fle, state->ctx_dma);
4057 		dpaa2_fl_set_len(out_fle, ctx->ctx_len);
4058 
4059 		req_ctx->flc = &ctx->flc[UPDATE_FIRST];
4060 		req_ctx->flc_dma = ctx->flc_dma[UPDATE_FIRST];
4061 		req_ctx->cbk = ahash_done_ctx_dst;
4062 		req_ctx->ctx = &req->base;
4063 		req_ctx->edesc = edesc;
4064 
4065 		ret = dpaa2_caam_enqueue(ctx->dev, req_ctx);
4066 		if (ret != -EINPROGRESS &&
4067 		    !(ret == -EBUSY &&
4068 		      req->base.flags & CRYPTO_TFM_REQ_MAY_BACKLOG))
4069 			goto unmap_ctx;
4070 
4071 		state->update = ahash_update_ctx;
4072 		state->finup = ahash_finup_ctx;
4073 		state->final = ahash_final_ctx;
4074 	} else if (*next_buflen) {
4075 		scatterwalk_map_and_copy(buf + *buflen, req->src, 0,
4076 					 req->nbytes, 0);
4077 		*buflen = *next_buflen;
4078 
4079 		print_hex_dump_debug("buf@" __stringify(__LINE__)": ",
4080 				     DUMP_PREFIX_ADDRESS, 16, 4, buf,
4081 				     *buflen, 1);
4082 	}
4083 
4084 	return ret;
4085 unmap_ctx:
4086 	ahash_unmap_ctx(ctx->dev, edesc, req, DMA_TO_DEVICE);
4087 	qi_cache_free(edesc);
4088 	return ret;
4089 }
4090 
4091 static int ahash_finup_no_ctx(struct ahash_request *req)
4092 {
4093 	struct crypto_ahash *ahash = crypto_ahash_reqtfm(req);
4094 	struct caam_hash_ctx *ctx = crypto_ahash_ctx(ahash);
4095 	struct caam_hash_state *state = ahash_request_ctx(req);
4096 	struct caam_request *req_ctx = &state->caam_req;
4097 	struct dpaa2_fl_entry *in_fle = &req_ctx->fd_flt[1];
4098 	struct dpaa2_fl_entry *out_fle = &req_ctx->fd_flt[0];
4099 	gfp_t flags = (req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP) ?
4100 		      GFP_KERNEL : GFP_ATOMIC;
4101 	int buflen = state->buflen;
4102 	int qm_sg_bytes, src_nents, mapped_nents;
4103 	int digestsize = crypto_ahash_digestsize(ahash);
4104 	struct ahash_edesc *edesc;
4105 	struct dpaa2_sg_entry *sg_table;
4106 	int ret = -ENOMEM;
4107 
4108 	src_nents = sg_nents_for_len(req->src, req->nbytes);
4109 	if (src_nents < 0) {
4110 		dev_err(ctx->dev, "Invalid number of src SG.\n");
4111 		return src_nents;
4112 	}
4113 
4114 	if (src_nents) {
4115 		mapped_nents = dma_map_sg(ctx->dev, req->src, src_nents,
4116 					  DMA_TO_DEVICE);
4117 		if (!mapped_nents) {
4118 			dev_err(ctx->dev, "unable to DMA map source\n");
4119 			return ret;
4120 		}
4121 	} else {
4122 		mapped_nents = 0;
4123 	}
4124 
4125 	/* allocate space for base edesc and link tables */
4126 	edesc = qi_cache_zalloc(GFP_DMA | flags);
4127 	if (!edesc) {
4128 		dma_unmap_sg(ctx->dev, req->src, src_nents, DMA_TO_DEVICE);
4129 		return ret;
4130 	}
4131 
4132 	edesc->src_nents = src_nents;
4133 	qm_sg_bytes = pad_sg_nents(2 + mapped_nents) * sizeof(*sg_table);
4134 	sg_table = &edesc->sgt[0];
4135 
4136 	ret = buf_map_to_qm_sg(ctx->dev, sg_table, state);
4137 	if (ret)
4138 		goto unmap;
4139 
4140 	sg_to_qm_sg_last(req->src, req->nbytes, sg_table + 1, 0);
4141 
4142 	edesc->qm_sg_dma = dma_map_single(ctx->dev, sg_table, qm_sg_bytes,
4143 					  DMA_TO_DEVICE);
4144 	if (dma_mapping_error(ctx->dev, edesc->qm_sg_dma)) {
4145 		dev_err(ctx->dev, "unable to map S/G table\n");
4146 		ret = -ENOMEM;
4147 		goto unmap;
4148 	}
4149 	edesc->qm_sg_bytes = qm_sg_bytes;
4150 
4151 	state->ctx_dma_len = digestsize;
4152 	state->ctx_dma = dma_map_single(ctx->dev, state->caam_ctx, digestsize,
4153 					DMA_FROM_DEVICE);
4154 	if (dma_mapping_error(ctx->dev, state->ctx_dma)) {
4155 		dev_err(ctx->dev, "unable to map ctx\n");
4156 		state->ctx_dma = 0;
4157 		ret = -ENOMEM;
4158 		goto unmap;
4159 	}
4160 
4161 	memset(&req_ctx->fd_flt, 0, sizeof(req_ctx->fd_flt));
4162 	dpaa2_fl_set_final(in_fle, true);
4163 	dpaa2_fl_set_format(in_fle, dpaa2_fl_sg);
4164 	dpaa2_fl_set_addr(in_fle, edesc->qm_sg_dma);
4165 	dpaa2_fl_set_len(in_fle, buflen + req->nbytes);
4166 	dpaa2_fl_set_format(out_fle, dpaa2_fl_single);
4167 	dpaa2_fl_set_addr(out_fle, state->ctx_dma);
4168 	dpaa2_fl_set_len(out_fle, digestsize);
4169 
4170 	req_ctx->flc = &ctx->flc[DIGEST];
4171 	req_ctx->flc_dma = ctx->flc_dma[DIGEST];
4172 	req_ctx->cbk = ahash_done;
4173 	req_ctx->ctx = &req->base;
4174 	req_ctx->edesc = edesc;
4175 	ret = dpaa2_caam_enqueue(ctx->dev, req_ctx);
4176 	if (ret != -EINPROGRESS &&
4177 	    !(ret == -EBUSY && req->base.flags & CRYPTO_TFM_REQ_MAY_BACKLOG))
4178 		goto unmap;
4179 
4180 	return ret;
4181 unmap:
4182 	ahash_unmap_ctx(ctx->dev, edesc, req, DMA_FROM_DEVICE);
4183 	qi_cache_free(edesc);
4184 	return ret;
4185 }
4186 
4187 static int ahash_update_first(struct ahash_request *req)
4188 {
4189 	struct crypto_ahash *ahash = crypto_ahash_reqtfm(req);
4190 	struct caam_hash_ctx *ctx = crypto_ahash_ctx(ahash);
4191 	struct caam_hash_state *state = ahash_request_ctx(req);
4192 	struct caam_request *req_ctx = &state->caam_req;
4193 	struct dpaa2_fl_entry *in_fle = &req_ctx->fd_flt[1];
4194 	struct dpaa2_fl_entry *out_fle = &req_ctx->fd_flt[0];
4195 	gfp_t flags = (req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP) ?
4196 		      GFP_KERNEL : GFP_ATOMIC;
4197 	u8 *buf = state->buf;
4198 	int *buflen = &state->buflen;
4199 	int *next_buflen = &state->next_buflen;
4200 	int to_hash;
4201 	int src_nents, mapped_nents;
4202 	struct ahash_edesc *edesc;
4203 	int ret = 0;
4204 
4205 	*next_buflen = req->nbytes & (crypto_tfm_alg_blocksize(&ahash->base) -
4206 				      1);
4207 	to_hash = req->nbytes - *next_buflen;
4208 
4209 	if (to_hash) {
4210 		struct dpaa2_sg_entry *sg_table;
4211 		int src_len = req->nbytes - *next_buflen;
4212 
4213 		src_nents = sg_nents_for_len(req->src, src_len);
4214 		if (src_nents < 0) {
4215 			dev_err(ctx->dev, "Invalid number of src SG.\n");
4216 			return src_nents;
4217 		}
4218 
4219 		if (src_nents) {
4220 			mapped_nents = dma_map_sg(ctx->dev, req->src, src_nents,
4221 						  DMA_TO_DEVICE);
4222 			if (!mapped_nents) {
4223 				dev_err(ctx->dev, "unable to map source for DMA\n");
4224 				return -ENOMEM;
4225 			}
4226 		} else {
4227 			mapped_nents = 0;
4228 		}
4229 
4230 		/* allocate space for base edesc and link tables */
4231 		edesc = qi_cache_zalloc(GFP_DMA | flags);
4232 		if (!edesc) {
4233 			dma_unmap_sg(ctx->dev, req->src, src_nents,
4234 				     DMA_TO_DEVICE);
4235 			return -ENOMEM;
4236 		}
4237 
4238 		edesc->src_nents = src_nents;
4239 		sg_table = &edesc->sgt[0];
4240 
4241 		memset(&req_ctx->fd_flt, 0, sizeof(req_ctx->fd_flt));
4242 		dpaa2_fl_set_final(in_fle, true);
4243 		dpaa2_fl_set_len(in_fle, to_hash);
4244 
4245 		if (mapped_nents > 1) {
4246 			int qm_sg_bytes;
4247 
4248 			sg_to_qm_sg_last(req->src, src_len, sg_table, 0);
4249 			qm_sg_bytes = pad_sg_nents(mapped_nents) *
4250 				      sizeof(*sg_table);
4251 			edesc->qm_sg_dma = dma_map_single(ctx->dev, sg_table,
4252 							  qm_sg_bytes,
4253 							  DMA_TO_DEVICE);
4254 			if (dma_mapping_error(ctx->dev, edesc->qm_sg_dma)) {
4255 				dev_err(ctx->dev, "unable to map S/G table\n");
4256 				ret = -ENOMEM;
4257 				goto unmap_ctx;
4258 			}
4259 			edesc->qm_sg_bytes = qm_sg_bytes;
4260 			dpaa2_fl_set_format(in_fle, dpaa2_fl_sg);
4261 			dpaa2_fl_set_addr(in_fle, edesc->qm_sg_dma);
4262 		} else {
4263 			dpaa2_fl_set_format(in_fle, dpaa2_fl_single);
4264 			dpaa2_fl_set_addr(in_fle, sg_dma_address(req->src));
4265 		}
4266 
4267 		state->ctx_dma_len = ctx->ctx_len;
4268 		state->ctx_dma = dma_map_single(ctx->dev, state->caam_ctx,
4269 						ctx->ctx_len, DMA_FROM_DEVICE);
4270 		if (dma_mapping_error(ctx->dev, state->ctx_dma)) {
4271 			dev_err(ctx->dev, "unable to map ctx\n");
4272 			state->ctx_dma = 0;
4273 			ret = -ENOMEM;
4274 			goto unmap_ctx;
4275 		}
4276 
4277 		dpaa2_fl_set_format(out_fle, dpaa2_fl_single);
4278 		dpaa2_fl_set_addr(out_fle, state->ctx_dma);
4279 		dpaa2_fl_set_len(out_fle, ctx->ctx_len);
4280 
4281 		req_ctx->flc = &ctx->flc[UPDATE_FIRST];
4282 		req_ctx->flc_dma = ctx->flc_dma[UPDATE_FIRST];
4283 		req_ctx->cbk = ahash_done_ctx_dst;
4284 		req_ctx->ctx = &req->base;
4285 		req_ctx->edesc = edesc;
4286 
4287 		ret = dpaa2_caam_enqueue(ctx->dev, req_ctx);
4288 		if (ret != -EINPROGRESS &&
4289 		    !(ret == -EBUSY && req->base.flags &
4290 		      CRYPTO_TFM_REQ_MAY_BACKLOG))
4291 			goto unmap_ctx;
4292 
4293 		state->update = ahash_update_ctx;
4294 		state->finup = ahash_finup_ctx;
4295 		state->final = ahash_final_ctx;
4296 	} else if (*next_buflen) {
4297 		state->update = ahash_update_no_ctx;
4298 		state->finup = ahash_finup_no_ctx;
4299 		state->final = ahash_final_no_ctx;
4300 		scatterwalk_map_and_copy(buf, req->src, 0,
4301 					 req->nbytes, 0);
4302 		*buflen = *next_buflen;
4303 
4304 		print_hex_dump_debug("buf@" __stringify(__LINE__)": ",
4305 				     DUMP_PREFIX_ADDRESS, 16, 4, buf,
4306 				     *buflen, 1);
4307 	}
4308 
4309 	return ret;
4310 unmap_ctx:
4311 	ahash_unmap_ctx(ctx->dev, edesc, req, DMA_TO_DEVICE);
4312 	qi_cache_free(edesc);
4313 	return ret;
4314 }
4315 
4316 static int ahash_finup_first(struct ahash_request *req)
4317 {
4318 	return ahash_digest(req);
4319 }
4320 
4321 static int ahash_init(struct ahash_request *req)
4322 {
4323 	struct caam_hash_state *state = ahash_request_ctx(req);
4324 
4325 	state->update = ahash_update_first;
4326 	state->finup = ahash_finup_first;
4327 	state->final = ahash_final_no_ctx;
4328 
4329 	state->ctx_dma = 0;
4330 	state->ctx_dma_len = 0;
4331 	state->buf_dma = 0;
4332 	state->buflen = 0;
4333 	state->next_buflen = 0;
4334 
4335 	return 0;
4336 }
4337 
4338 static int ahash_update(struct ahash_request *req)
4339 {
4340 	struct caam_hash_state *state = ahash_request_ctx(req);
4341 
4342 	return state->update(req);
4343 }
4344 
4345 static int ahash_finup(struct ahash_request *req)
4346 {
4347 	struct caam_hash_state *state = ahash_request_ctx(req);
4348 
4349 	return state->finup(req);
4350 }
4351 
4352 static int ahash_final(struct ahash_request *req)
4353 {
4354 	struct caam_hash_state *state = ahash_request_ctx(req);
4355 
4356 	return state->final(req);
4357 }
4358 
4359 static int ahash_export(struct ahash_request *req, void *out)
4360 {
4361 	struct caam_hash_state *state = ahash_request_ctx(req);
4362 	struct caam_export_state *export = out;
4363 	u8 *buf = state->buf;
4364 	int len = state->buflen;
4365 
4366 	memcpy(export->buf, buf, len);
4367 	memcpy(export->caam_ctx, state->caam_ctx, sizeof(export->caam_ctx));
4368 	export->buflen = len;
4369 	export->update = state->update;
4370 	export->final = state->final;
4371 	export->finup = state->finup;
4372 
4373 	return 0;
4374 }
4375 
4376 static int ahash_import(struct ahash_request *req, const void *in)
4377 {
4378 	struct caam_hash_state *state = ahash_request_ctx(req);
4379 	const struct caam_export_state *export = in;
4380 
4381 	memset(state, 0, sizeof(*state));
4382 	memcpy(state->buf, export->buf, export->buflen);
4383 	memcpy(state->caam_ctx, export->caam_ctx, sizeof(state->caam_ctx));
4384 	state->buflen = export->buflen;
4385 	state->update = export->update;
4386 	state->final = export->final;
4387 	state->finup = export->finup;
4388 
4389 	return 0;
4390 }
4391 
4392 struct caam_hash_template {
4393 	char name[CRYPTO_MAX_ALG_NAME];
4394 	char driver_name[CRYPTO_MAX_ALG_NAME];
4395 	char hmac_name[CRYPTO_MAX_ALG_NAME];
4396 	char hmac_driver_name[CRYPTO_MAX_ALG_NAME];
4397 	unsigned int blocksize;
4398 	struct ahash_alg template_ahash;
4399 	u32 alg_type;
4400 };
4401 
4402 /* ahash descriptors */
4403 static struct caam_hash_template driver_hash[] = {
4404 	{
4405 		.name = "sha1",
4406 		.driver_name = "sha1-caam-qi2",
4407 		.hmac_name = "hmac(sha1)",
4408 		.hmac_driver_name = "hmac-sha1-caam-qi2",
4409 		.blocksize = SHA1_BLOCK_SIZE,
4410 		.template_ahash = {
4411 			.init = ahash_init,
4412 			.update = ahash_update,
4413 			.final = ahash_final,
4414 			.finup = ahash_finup,
4415 			.digest = ahash_digest,
4416 			.export = ahash_export,
4417 			.import = ahash_import,
4418 			.setkey = ahash_setkey,
4419 			.halg = {
4420 				.digestsize = SHA1_DIGEST_SIZE,
4421 				.statesize = sizeof(struct caam_export_state),
4422 			},
4423 		},
4424 		.alg_type = OP_ALG_ALGSEL_SHA1,
4425 	}, {
4426 		.name = "sha224",
4427 		.driver_name = "sha224-caam-qi2",
4428 		.hmac_name = "hmac(sha224)",
4429 		.hmac_driver_name = "hmac-sha224-caam-qi2",
4430 		.blocksize = SHA224_BLOCK_SIZE,
4431 		.template_ahash = {
4432 			.init = ahash_init,
4433 			.update = ahash_update,
4434 			.final = ahash_final,
4435 			.finup = ahash_finup,
4436 			.digest = ahash_digest,
4437 			.export = ahash_export,
4438 			.import = ahash_import,
4439 			.setkey = ahash_setkey,
4440 			.halg = {
4441 				.digestsize = SHA224_DIGEST_SIZE,
4442 				.statesize = sizeof(struct caam_export_state),
4443 			},
4444 		},
4445 		.alg_type = OP_ALG_ALGSEL_SHA224,
4446 	}, {
4447 		.name = "sha256",
4448 		.driver_name = "sha256-caam-qi2",
4449 		.hmac_name = "hmac(sha256)",
4450 		.hmac_driver_name = "hmac-sha256-caam-qi2",
4451 		.blocksize = SHA256_BLOCK_SIZE,
4452 		.template_ahash = {
4453 			.init = ahash_init,
4454 			.update = ahash_update,
4455 			.final = ahash_final,
4456 			.finup = ahash_finup,
4457 			.digest = ahash_digest,
4458 			.export = ahash_export,
4459 			.import = ahash_import,
4460 			.setkey = ahash_setkey,
4461 			.halg = {
4462 				.digestsize = SHA256_DIGEST_SIZE,
4463 				.statesize = sizeof(struct caam_export_state),
4464 			},
4465 		},
4466 		.alg_type = OP_ALG_ALGSEL_SHA256,
4467 	}, {
4468 		.name = "sha384",
4469 		.driver_name = "sha384-caam-qi2",
4470 		.hmac_name = "hmac(sha384)",
4471 		.hmac_driver_name = "hmac-sha384-caam-qi2",
4472 		.blocksize = SHA384_BLOCK_SIZE,
4473 		.template_ahash = {
4474 			.init = ahash_init,
4475 			.update = ahash_update,
4476 			.final = ahash_final,
4477 			.finup = ahash_finup,
4478 			.digest = ahash_digest,
4479 			.export = ahash_export,
4480 			.import = ahash_import,
4481 			.setkey = ahash_setkey,
4482 			.halg = {
4483 				.digestsize = SHA384_DIGEST_SIZE,
4484 				.statesize = sizeof(struct caam_export_state),
4485 			},
4486 		},
4487 		.alg_type = OP_ALG_ALGSEL_SHA384,
4488 	}, {
4489 		.name = "sha512",
4490 		.driver_name = "sha512-caam-qi2",
4491 		.hmac_name = "hmac(sha512)",
4492 		.hmac_driver_name = "hmac-sha512-caam-qi2",
4493 		.blocksize = SHA512_BLOCK_SIZE,
4494 		.template_ahash = {
4495 			.init = ahash_init,
4496 			.update = ahash_update,
4497 			.final = ahash_final,
4498 			.finup = ahash_finup,
4499 			.digest = ahash_digest,
4500 			.export = ahash_export,
4501 			.import = ahash_import,
4502 			.setkey = ahash_setkey,
4503 			.halg = {
4504 				.digestsize = SHA512_DIGEST_SIZE,
4505 				.statesize = sizeof(struct caam_export_state),
4506 			},
4507 		},
4508 		.alg_type = OP_ALG_ALGSEL_SHA512,
4509 	}, {
4510 		.name = "md5",
4511 		.driver_name = "md5-caam-qi2",
4512 		.hmac_name = "hmac(md5)",
4513 		.hmac_driver_name = "hmac-md5-caam-qi2",
4514 		.blocksize = MD5_BLOCK_WORDS * 4,
4515 		.template_ahash = {
4516 			.init = ahash_init,
4517 			.update = ahash_update,
4518 			.final = ahash_final,
4519 			.finup = ahash_finup,
4520 			.digest = ahash_digest,
4521 			.export = ahash_export,
4522 			.import = ahash_import,
4523 			.setkey = ahash_setkey,
4524 			.halg = {
4525 				.digestsize = MD5_DIGEST_SIZE,
4526 				.statesize = sizeof(struct caam_export_state),
4527 			},
4528 		},
4529 		.alg_type = OP_ALG_ALGSEL_MD5,
4530 	}
4531 };
4532 
4533 struct caam_hash_alg {
4534 	struct list_head entry;
4535 	struct device *dev;
4536 	int alg_type;
4537 	struct ahash_alg ahash_alg;
4538 };
4539 
4540 static int caam_hash_cra_init(struct crypto_tfm *tfm)
4541 {
4542 	struct crypto_ahash *ahash = __crypto_ahash_cast(tfm);
4543 	struct crypto_alg *base = tfm->__crt_alg;
4544 	struct hash_alg_common *halg =
4545 		 container_of(base, struct hash_alg_common, base);
4546 	struct ahash_alg *alg =
4547 		 container_of(halg, struct ahash_alg, halg);
4548 	struct caam_hash_alg *caam_hash =
4549 		 container_of(alg, struct caam_hash_alg, ahash_alg);
4550 	struct caam_hash_ctx *ctx = crypto_tfm_ctx(tfm);
4551 	/* Sizes for MDHA running digests: MD5, SHA1, 224, 256, 384, 512 */
4552 	static const u8 runninglen[] = { HASH_MSG_LEN + MD5_DIGEST_SIZE,
4553 					 HASH_MSG_LEN + SHA1_DIGEST_SIZE,
4554 					 HASH_MSG_LEN + 32,
4555 					 HASH_MSG_LEN + SHA256_DIGEST_SIZE,
4556 					 HASH_MSG_LEN + 64,
4557 					 HASH_MSG_LEN + SHA512_DIGEST_SIZE };
4558 	dma_addr_t dma_addr;
4559 	int i;
4560 
4561 	ctx->dev = caam_hash->dev;
4562 
4563 	if (alg->setkey) {
4564 		ctx->adata.key_dma = dma_map_single_attrs(ctx->dev, ctx->key,
4565 							  ARRAY_SIZE(ctx->key),
4566 							  DMA_TO_DEVICE,
4567 							  DMA_ATTR_SKIP_CPU_SYNC);
4568 		if (dma_mapping_error(ctx->dev, ctx->adata.key_dma)) {
4569 			dev_err(ctx->dev, "unable to map key\n");
4570 			return -ENOMEM;
4571 		}
4572 	}
4573 
4574 	dma_addr = dma_map_single_attrs(ctx->dev, ctx->flc, sizeof(ctx->flc),
4575 					DMA_BIDIRECTIONAL,
4576 					DMA_ATTR_SKIP_CPU_SYNC);
4577 	if (dma_mapping_error(ctx->dev, dma_addr)) {
4578 		dev_err(ctx->dev, "unable to map shared descriptors\n");
4579 		if (ctx->adata.key_dma)
4580 			dma_unmap_single_attrs(ctx->dev, ctx->adata.key_dma,
4581 					       ARRAY_SIZE(ctx->key),
4582 					       DMA_TO_DEVICE,
4583 					       DMA_ATTR_SKIP_CPU_SYNC);
4584 		return -ENOMEM;
4585 	}
4586 
4587 	for (i = 0; i < HASH_NUM_OP; i++)
4588 		ctx->flc_dma[i] = dma_addr + i * sizeof(ctx->flc[i]);
4589 
4590 	/* copy descriptor header template value */
4591 	ctx->adata.algtype = OP_TYPE_CLASS2_ALG | caam_hash->alg_type;
4592 
4593 	ctx->ctx_len = runninglen[(ctx->adata.algtype &
4594 				   OP_ALG_ALGSEL_SUBMASK) >>
4595 				  OP_ALG_ALGSEL_SHIFT];
4596 
4597 	crypto_ahash_set_reqsize(__crypto_ahash_cast(tfm),
4598 				 sizeof(struct caam_hash_state));
4599 
4600 	/*
4601 	 * For keyed hash algorithms shared descriptors
4602 	 * will be created later in setkey() callback
4603 	 */
4604 	return alg->setkey ? 0 : ahash_set_sh_desc(ahash);
4605 }
4606 
4607 static void caam_hash_cra_exit(struct crypto_tfm *tfm)
4608 {
4609 	struct caam_hash_ctx *ctx = crypto_tfm_ctx(tfm);
4610 
4611 	dma_unmap_single_attrs(ctx->dev, ctx->flc_dma[0], sizeof(ctx->flc),
4612 			       DMA_BIDIRECTIONAL, DMA_ATTR_SKIP_CPU_SYNC);
4613 	if (ctx->adata.key_dma)
4614 		dma_unmap_single_attrs(ctx->dev, ctx->adata.key_dma,
4615 				       ARRAY_SIZE(ctx->key), DMA_TO_DEVICE,
4616 				       DMA_ATTR_SKIP_CPU_SYNC);
4617 }
4618 
4619 static struct caam_hash_alg *caam_hash_alloc(struct device *dev,
4620 	struct caam_hash_template *template, bool keyed)
4621 {
4622 	struct caam_hash_alg *t_alg;
4623 	struct ahash_alg *halg;
4624 	struct crypto_alg *alg;
4625 
4626 	t_alg = kzalloc(sizeof(*t_alg), GFP_KERNEL);
4627 	if (!t_alg)
4628 		return ERR_PTR(-ENOMEM);
4629 
4630 	t_alg->ahash_alg = template->template_ahash;
4631 	halg = &t_alg->ahash_alg;
4632 	alg = &halg->halg.base;
4633 
4634 	if (keyed) {
4635 		snprintf(alg->cra_name, CRYPTO_MAX_ALG_NAME, "%s",
4636 			 template->hmac_name);
4637 		snprintf(alg->cra_driver_name, CRYPTO_MAX_ALG_NAME, "%s",
4638 			 template->hmac_driver_name);
4639 	} else {
4640 		snprintf(alg->cra_name, CRYPTO_MAX_ALG_NAME, "%s",
4641 			 template->name);
4642 		snprintf(alg->cra_driver_name, CRYPTO_MAX_ALG_NAME, "%s",
4643 			 template->driver_name);
4644 		t_alg->ahash_alg.setkey = NULL;
4645 	}
4646 	alg->cra_module = THIS_MODULE;
4647 	alg->cra_init = caam_hash_cra_init;
4648 	alg->cra_exit = caam_hash_cra_exit;
4649 	alg->cra_ctxsize = sizeof(struct caam_hash_ctx);
4650 	alg->cra_priority = CAAM_CRA_PRIORITY;
4651 	alg->cra_blocksize = template->blocksize;
4652 	alg->cra_alignmask = 0;
4653 	alg->cra_flags = CRYPTO_ALG_ASYNC | CRYPTO_ALG_ALLOCATES_MEMORY;
4654 
4655 	t_alg->alg_type = template->alg_type;
4656 	t_alg->dev = dev;
4657 
4658 	return t_alg;
4659 }
4660 
4661 static void dpaa2_caam_fqdan_cb(struct dpaa2_io_notification_ctx *nctx)
4662 {
4663 	struct dpaa2_caam_priv_per_cpu *ppriv;
4664 
4665 	ppriv = container_of(nctx, struct dpaa2_caam_priv_per_cpu, nctx);
4666 	napi_schedule_irqoff(&ppriv->napi);
4667 }
4668 
4669 static int __cold dpaa2_dpseci_dpio_setup(struct dpaa2_caam_priv *priv)
4670 {
4671 	struct device *dev = priv->dev;
4672 	struct dpaa2_io_notification_ctx *nctx;
4673 	struct dpaa2_caam_priv_per_cpu *ppriv;
4674 	int err, i = 0, cpu;
4675 
4676 	for_each_online_cpu(cpu) {
4677 		ppriv = per_cpu_ptr(priv->ppriv, cpu);
4678 		ppriv->priv = priv;
4679 		nctx = &ppriv->nctx;
4680 		nctx->is_cdan = 0;
4681 		nctx->id = ppriv->rsp_fqid;
4682 		nctx->desired_cpu = cpu;
4683 		nctx->cb = dpaa2_caam_fqdan_cb;
4684 
4685 		/* Register notification callbacks */
4686 		ppriv->dpio = dpaa2_io_service_select(cpu);
4687 		err = dpaa2_io_service_register(ppriv->dpio, nctx, dev);
4688 		if (unlikely(err)) {
4689 			dev_dbg(dev, "No affine DPIO for cpu %d\n", cpu);
4690 			nctx->cb = NULL;
4691 			/*
4692 			 * If no affine DPIO for this core, there's probably
4693 			 * none available for next cores either. Signal we want
4694 			 * to retry later, in case the DPIO devices weren't
4695 			 * probed yet.
4696 			 */
4697 			err = -EPROBE_DEFER;
4698 			goto err;
4699 		}
4700 
4701 		ppriv->store = dpaa2_io_store_create(DPAA2_CAAM_STORE_SIZE,
4702 						     dev);
4703 		if (unlikely(!ppriv->store)) {
4704 			dev_err(dev, "dpaa2_io_store_create() failed\n");
4705 			err = -ENOMEM;
4706 			goto err;
4707 		}
4708 
4709 		if (++i == priv->num_pairs)
4710 			break;
4711 	}
4712 
4713 	return 0;
4714 
4715 err:
4716 	for_each_online_cpu(cpu) {
4717 		ppriv = per_cpu_ptr(priv->ppriv, cpu);
4718 		if (!ppriv->nctx.cb)
4719 			break;
4720 		dpaa2_io_service_deregister(ppriv->dpio, &ppriv->nctx, dev);
4721 	}
4722 
4723 	for_each_online_cpu(cpu) {
4724 		ppriv = per_cpu_ptr(priv->ppriv, cpu);
4725 		if (!ppriv->store)
4726 			break;
4727 		dpaa2_io_store_destroy(ppriv->store);
4728 	}
4729 
4730 	return err;
4731 }
4732 
4733 static void __cold dpaa2_dpseci_dpio_free(struct dpaa2_caam_priv *priv)
4734 {
4735 	struct dpaa2_caam_priv_per_cpu *ppriv;
4736 	int i = 0, cpu;
4737 
4738 	for_each_online_cpu(cpu) {
4739 		ppriv = per_cpu_ptr(priv->ppriv, cpu);
4740 		dpaa2_io_service_deregister(ppriv->dpio, &ppriv->nctx,
4741 					    priv->dev);
4742 		dpaa2_io_store_destroy(ppriv->store);
4743 
4744 		if (++i == priv->num_pairs)
4745 			return;
4746 	}
4747 }
4748 
4749 static int dpaa2_dpseci_bind(struct dpaa2_caam_priv *priv)
4750 {
4751 	struct dpseci_rx_queue_cfg rx_queue_cfg;
4752 	struct device *dev = priv->dev;
4753 	struct fsl_mc_device *ls_dev = to_fsl_mc_device(dev);
4754 	struct dpaa2_caam_priv_per_cpu *ppriv;
4755 	int err = 0, i = 0, cpu;
4756 
4757 	/* Configure Rx queues */
4758 	for_each_online_cpu(cpu) {
4759 		ppriv = per_cpu_ptr(priv->ppriv, cpu);
4760 
4761 		rx_queue_cfg.options = DPSECI_QUEUE_OPT_DEST |
4762 				       DPSECI_QUEUE_OPT_USER_CTX;
4763 		rx_queue_cfg.order_preservation_en = 0;
4764 		rx_queue_cfg.dest_cfg.dest_type = DPSECI_DEST_DPIO;
4765 		rx_queue_cfg.dest_cfg.dest_id = ppriv->nctx.dpio_id;
4766 		/*
4767 		 * Rx priority (WQ) doesn't really matter, since we use
4768 		 * pull mode, i.e. volatile dequeues from specific FQs
4769 		 */
4770 		rx_queue_cfg.dest_cfg.priority = 0;
4771 		rx_queue_cfg.user_ctx = ppriv->nctx.qman64;
4772 
4773 		err = dpseci_set_rx_queue(priv->mc_io, 0, ls_dev->mc_handle, i,
4774 					  &rx_queue_cfg);
4775 		if (err) {
4776 			dev_err(dev, "dpseci_set_rx_queue() failed with err %d\n",
4777 				err);
4778 			return err;
4779 		}
4780 
4781 		if (++i == priv->num_pairs)
4782 			break;
4783 	}
4784 
4785 	return err;
4786 }
4787 
4788 static void dpaa2_dpseci_congestion_free(struct dpaa2_caam_priv *priv)
4789 {
4790 	struct device *dev = priv->dev;
4791 
4792 	if (!priv->cscn_mem)
4793 		return;
4794 
4795 	dma_unmap_single(dev, priv->cscn_dma, DPAA2_CSCN_SIZE, DMA_FROM_DEVICE);
4796 	kfree(priv->cscn_mem);
4797 }
4798 
4799 static void dpaa2_dpseci_free(struct dpaa2_caam_priv *priv)
4800 {
4801 	struct device *dev = priv->dev;
4802 	struct fsl_mc_device *ls_dev = to_fsl_mc_device(dev);
4803 	int err;
4804 
4805 	if (DPSECI_VER(priv->major_ver, priv->minor_ver) > DPSECI_VER(5, 3)) {
4806 		err = dpseci_reset(priv->mc_io, 0, ls_dev->mc_handle);
4807 		if (err)
4808 			dev_err(dev, "dpseci_reset() failed\n");
4809 	}
4810 
4811 	dpaa2_dpseci_congestion_free(priv);
4812 	dpseci_close(priv->mc_io, 0, ls_dev->mc_handle);
4813 }
4814 
4815 static void dpaa2_caam_process_fd(struct dpaa2_caam_priv *priv,
4816 				  const struct dpaa2_fd *fd)
4817 {
4818 	struct caam_request *req;
4819 	u32 fd_err;
4820 
4821 	if (dpaa2_fd_get_format(fd) != dpaa2_fd_list) {
4822 		dev_err(priv->dev, "Only Frame List FD format is supported!\n");
4823 		return;
4824 	}
4825 
4826 	fd_err = dpaa2_fd_get_ctrl(fd) & FD_CTRL_ERR_MASK;
4827 	if (unlikely(fd_err))
4828 		dev_err_ratelimited(priv->dev, "FD error: %08x\n", fd_err);
4829 
4830 	/*
4831 	 * FD[ADDR] is guaranteed to be valid, irrespective of errors reported
4832 	 * in FD[ERR] or FD[FRC].
4833 	 */
4834 	req = dpaa2_caam_iova_to_virt(priv, dpaa2_fd_get_addr(fd));
4835 	dma_unmap_single(priv->dev, req->fd_flt_dma, sizeof(req->fd_flt),
4836 			 DMA_BIDIRECTIONAL);
4837 	req->cbk(req->ctx, dpaa2_fd_get_frc(fd));
4838 }
4839 
4840 static int dpaa2_caam_pull_fq(struct dpaa2_caam_priv_per_cpu *ppriv)
4841 {
4842 	int err;
4843 
4844 	/* Retry while portal is busy */
4845 	do {
4846 		err = dpaa2_io_service_pull_fq(ppriv->dpio, ppriv->rsp_fqid,
4847 					       ppriv->store);
4848 	} while (err == -EBUSY);
4849 
4850 	if (unlikely(err))
4851 		dev_err(ppriv->priv->dev, "dpaa2_io_service_pull err %d", err);
4852 
4853 	return err;
4854 }
4855 
4856 static int dpaa2_caam_store_consume(struct dpaa2_caam_priv_per_cpu *ppriv)
4857 {
4858 	struct dpaa2_dq *dq;
4859 	int cleaned = 0, is_last;
4860 
4861 	do {
4862 		dq = dpaa2_io_store_next(ppriv->store, &is_last);
4863 		if (unlikely(!dq)) {
4864 			if (unlikely(!is_last)) {
4865 				dev_dbg(ppriv->priv->dev,
4866 					"FQ %d returned no valid frames\n",
4867 					ppriv->rsp_fqid);
4868 				/*
4869 				 * MUST retry until we get some sort of
4870 				 * valid response token (be it "empty dequeue"
4871 				 * or a valid frame).
4872 				 */
4873 				continue;
4874 			}
4875 			break;
4876 		}
4877 
4878 		/* Process FD */
4879 		dpaa2_caam_process_fd(ppriv->priv, dpaa2_dq_fd(dq));
4880 		cleaned++;
4881 	} while (!is_last);
4882 
4883 	return cleaned;
4884 }
4885 
4886 static int dpaa2_dpseci_poll(struct napi_struct *napi, int budget)
4887 {
4888 	struct dpaa2_caam_priv_per_cpu *ppriv;
4889 	struct dpaa2_caam_priv *priv;
4890 	int err, cleaned = 0, store_cleaned;
4891 
4892 	ppriv = container_of(napi, struct dpaa2_caam_priv_per_cpu, napi);
4893 	priv = ppriv->priv;
4894 
4895 	if (unlikely(dpaa2_caam_pull_fq(ppriv)))
4896 		return 0;
4897 
4898 	do {
4899 		store_cleaned = dpaa2_caam_store_consume(ppriv);
4900 		cleaned += store_cleaned;
4901 
4902 		if (store_cleaned == 0 ||
4903 		    cleaned > budget - DPAA2_CAAM_STORE_SIZE)
4904 			break;
4905 
4906 		/* Try to dequeue some more */
4907 		err = dpaa2_caam_pull_fq(ppriv);
4908 		if (unlikely(err))
4909 			break;
4910 	} while (1);
4911 
4912 	if (cleaned < budget) {
4913 		napi_complete_done(napi, cleaned);
4914 		err = dpaa2_io_service_rearm(ppriv->dpio, &ppriv->nctx);
4915 		if (unlikely(err))
4916 			dev_err(priv->dev, "Notification rearm failed: %d\n",
4917 				err);
4918 	}
4919 
4920 	return cleaned;
4921 }
4922 
4923 static int dpaa2_dpseci_congestion_setup(struct dpaa2_caam_priv *priv,
4924 					 u16 token)
4925 {
4926 	struct dpseci_congestion_notification_cfg cong_notif_cfg = { 0 };
4927 	struct device *dev = priv->dev;
4928 	int err;
4929 
4930 	/*
4931 	 * Congestion group feature supported starting with DPSECI API v5.1
4932 	 * and only when object has been created with this capability.
4933 	 */
4934 	if ((DPSECI_VER(priv->major_ver, priv->minor_ver) < DPSECI_VER(5, 1)) ||
4935 	    !(priv->dpseci_attr.options & DPSECI_OPT_HAS_CG))
4936 		return 0;
4937 
4938 	priv->cscn_mem = kzalloc(DPAA2_CSCN_SIZE + DPAA2_CSCN_ALIGN,
4939 				 GFP_KERNEL | GFP_DMA);
4940 	if (!priv->cscn_mem)
4941 		return -ENOMEM;
4942 
4943 	priv->cscn_mem_aligned = PTR_ALIGN(priv->cscn_mem, DPAA2_CSCN_ALIGN);
4944 	priv->cscn_dma = dma_map_single(dev, priv->cscn_mem_aligned,
4945 					DPAA2_CSCN_SIZE, DMA_FROM_DEVICE);
4946 	if (dma_mapping_error(dev, priv->cscn_dma)) {
4947 		dev_err(dev, "Error mapping CSCN memory area\n");
4948 		err = -ENOMEM;
4949 		goto err_dma_map;
4950 	}
4951 
4952 	cong_notif_cfg.units = DPSECI_CONGESTION_UNIT_BYTES;
4953 	cong_notif_cfg.threshold_entry = DPAA2_SEC_CONG_ENTRY_THRESH;
4954 	cong_notif_cfg.threshold_exit = DPAA2_SEC_CONG_EXIT_THRESH;
4955 	cong_notif_cfg.message_ctx = (uintptr_t)priv;
4956 	cong_notif_cfg.message_iova = priv->cscn_dma;
4957 	cong_notif_cfg.notification_mode = DPSECI_CGN_MODE_WRITE_MEM_ON_ENTER |
4958 					DPSECI_CGN_MODE_WRITE_MEM_ON_EXIT |
4959 					DPSECI_CGN_MODE_COHERENT_WRITE;
4960 
4961 	err = dpseci_set_congestion_notification(priv->mc_io, 0, token,
4962 						 &cong_notif_cfg);
4963 	if (err) {
4964 		dev_err(dev, "dpseci_set_congestion_notification failed\n");
4965 		goto err_set_cong;
4966 	}
4967 
4968 	return 0;
4969 
4970 err_set_cong:
4971 	dma_unmap_single(dev, priv->cscn_dma, DPAA2_CSCN_SIZE, DMA_FROM_DEVICE);
4972 err_dma_map:
4973 	kfree(priv->cscn_mem);
4974 
4975 	return err;
4976 }
4977 
4978 static int __cold dpaa2_dpseci_setup(struct fsl_mc_device *ls_dev)
4979 {
4980 	struct device *dev = &ls_dev->dev;
4981 	struct dpaa2_caam_priv *priv;
4982 	struct dpaa2_caam_priv_per_cpu *ppriv;
4983 	int err, cpu;
4984 	u8 i;
4985 
4986 	priv = dev_get_drvdata(dev);
4987 
4988 	priv->dev = dev;
4989 	priv->dpsec_id = ls_dev->obj_desc.id;
4990 
4991 	/* Get a handle for the DPSECI this interface is associate with */
4992 	err = dpseci_open(priv->mc_io, 0, priv->dpsec_id, &ls_dev->mc_handle);
4993 	if (err) {
4994 		dev_err(dev, "dpseci_open() failed: %d\n", err);
4995 		goto err_open;
4996 	}
4997 
4998 	err = dpseci_get_api_version(priv->mc_io, 0, &priv->major_ver,
4999 				     &priv->minor_ver);
5000 	if (err) {
5001 		dev_err(dev, "dpseci_get_api_version() failed\n");
5002 		goto err_get_vers;
5003 	}
5004 
5005 	dev_info(dev, "dpseci v%d.%d\n", priv->major_ver, priv->minor_ver);
5006 
5007 	if (DPSECI_VER(priv->major_ver, priv->minor_ver) > DPSECI_VER(5, 3)) {
5008 		err = dpseci_reset(priv->mc_io, 0, ls_dev->mc_handle);
5009 		if (err) {
5010 			dev_err(dev, "dpseci_reset() failed\n");
5011 			goto err_get_vers;
5012 		}
5013 	}
5014 
5015 	err = dpseci_get_attributes(priv->mc_io, 0, ls_dev->mc_handle,
5016 				    &priv->dpseci_attr);
5017 	if (err) {
5018 		dev_err(dev, "dpseci_get_attributes() failed\n");
5019 		goto err_get_vers;
5020 	}
5021 
5022 	err = dpseci_get_sec_attr(priv->mc_io, 0, ls_dev->mc_handle,
5023 				  &priv->sec_attr);
5024 	if (err) {
5025 		dev_err(dev, "dpseci_get_sec_attr() failed\n");
5026 		goto err_get_vers;
5027 	}
5028 
5029 	err = dpaa2_dpseci_congestion_setup(priv, ls_dev->mc_handle);
5030 	if (err) {
5031 		dev_err(dev, "setup_congestion() failed\n");
5032 		goto err_get_vers;
5033 	}
5034 
5035 	priv->num_pairs = min(priv->dpseci_attr.num_rx_queues,
5036 			      priv->dpseci_attr.num_tx_queues);
5037 	if (priv->num_pairs > num_online_cpus()) {
5038 		dev_warn(dev, "%d queues won't be used\n",
5039 			 priv->num_pairs - num_online_cpus());
5040 		priv->num_pairs = num_online_cpus();
5041 	}
5042 
5043 	for (i = 0; i < priv->dpseci_attr.num_rx_queues; i++) {
5044 		err = dpseci_get_rx_queue(priv->mc_io, 0, ls_dev->mc_handle, i,
5045 					  &priv->rx_queue_attr[i]);
5046 		if (err) {
5047 			dev_err(dev, "dpseci_get_rx_queue() failed\n");
5048 			goto err_get_rx_queue;
5049 		}
5050 	}
5051 
5052 	for (i = 0; i < priv->dpseci_attr.num_tx_queues; i++) {
5053 		err = dpseci_get_tx_queue(priv->mc_io, 0, ls_dev->mc_handle, i,
5054 					  &priv->tx_queue_attr[i]);
5055 		if (err) {
5056 			dev_err(dev, "dpseci_get_tx_queue() failed\n");
5057 			goto err_get_rx_queue;
5058 		}
5059 	}
5060 
5061 	i = 0;
5062 	for_each_online_cpu(cpu) {
5063 		u8 j;
5064 
5065 		j = i % priv->num_pairs;
5066 
5067 		ppriv = per_cpu_ptr(priv->ppriv, cpu);
5068 		ppriv->req_fqid = priv->tx_queue_attr[j].fqid;
5069 
5070 		/*
5071 		 * Allow all cores to enqueue, while only some of them
5072 		 * will take part in dequeuing.
5073 		 */
5074 		if (++i > priv->num_pairs)
5075 			continue;
5076 
5077 		ppriv->rsp_fqid = priv->rx_queue_attr[j].fqid;
5078 		ppriv->prio = j;
5079 
5080 		dev_dbg(dev, "pair %d: rx queue %d, tx queue %d\n", j,
5081 			priv->rx_queue_attr[j].fqid,
5082 			priv->tx_queue_attr[j].fqid);
5083 
5084 		ppriv->net_dev.dev = *dev;
5085 		INIT_LIST_HEAD(&ppriv->net_dev.napi_list);
5086 		netif_napi_add(&ppriv->net_dev, &ppriv->napi, dpaa2_dpseci_poll,
5087 			       DPAA2_CAAM_NAPI_WEIGHT);
5088 	}
5089 
5090 	return 0;
5091 
5092 err_get_rx_queue:
5093 	dpaa2_dpseci_congestion_free(priv);
5094 err_get_vers:
5095 	dpseci_close(priv->mc_io, 0, ls_dev->mc_handle);
5096 err_open:
5097 	return err;
5098 }
5099 
5100 static int dpaa2_dpseci_enable(struct dpaa2_caam_priv *priv)
5101 {
5102 	struct device *dev = priv->dev;
5103 	struct fsl_mc_device *ls_dev = to_fsl_mc_device(dev);
5104 	struct dpaa2_caam_priv_per_cpu *ppriv;
5105 	int i;
5106 
5107 	for (i = 0; i < priv->num_pairs; i++) {
5108 		ppriv = per_cpu_ptr(priv->ppriv, i);
5109 		napi_enable(&ppriv->napi);
5110 	}
5111 
5112 	return dpseci_enable(priv->mc_io, 0, ls_dev->mc_handle);
5113 }
5114 
5115 static int __cold dpaa2_dpseci_disable(struct dpaa2_caam_priv *priv)
5116 {
5117 	struct device *dev = priv->dev;
5118 	struct dpaa2_caam_priv_per_cpu *ppriv;
5119 	struct fsl_mc_device *ls_dev = to_fsl_mc_device(dev);
5120 	int i, err = 0, enabled;
5121 
5122 	err = dpseci_disable(priv->mc_io, 0, ls_dev->mc_handle);
5123 	if (err) {
5124 		dev_err(dev, "dpseci_disable() failed\n");
5125 		return err;
5126 	}
5127 
5128 	err = dpseci_is_enabled(priv->mc_io, 0, ls_dev->mc_handle, &enabled);
5129 	if (err) {
5130 		dev_err(dev, "dpseci_is_enabled() failed\n");
5131 		return err;
5132 	}
5133 
5134 	dev_dbg(dev, "disable: %s\n", enabled ? "false" : "true");
5135 
5136 	for (i = 0; i < priv->num_pairs; i++) {
5137 		ppriv = per_cpu_ptr(priv->ppriv, i);
5138 		napi_disable(&ppriv->napi);
5139 		netif_napi_del(&ppriv->napi);
5140 	}
5141 
5142 	return 0;
5143 }
5144 
5145 static struct list_head hash_list;
5146 
5147 static int dpaa2_caam_probe(struct fsl_mc_device *dpseci_dev)
5148 {
5149 	struct device *dev;
5150 	struct dpaa2_caam_priv *priv;
5151 	int i, err = 0;
5152 	bool registered = false;
5153 
5154 	/*
5155 	 * There is no way to get CAAM endianness - there is no direct register
5156 	 * space access and MC f/w does not provide this attribute.
5157 	 * All DPAA2-based SoCs have little endian CAAM, thus hard-code this
5158 	 * property.
5159 	 */
5160 	caam_little_end = true;
5161 
5162 	caam_imx = false;
5163 
5164 	dev = &dpseci_dev->dev;
5165 
5166 	priv = devm_kzalloc(dev, sizeof(*priv), GFP_KERNEL);
5167 	if (!priv)
5168 		return -ENOMEM;
5169 
5170 	dev_set_drvdata(dev, priv);
5171 
5172 	priv->domain = iommu_get_domain_for_dev(dev);
5173 
5174 	qi_cache = kmem_cache_create("dpaa2_caamqicache", CAAM_QI_MEMCACHE_SIZE,
5175 				     0, SLAB_CACHE_DMA, NULL);
5176 	if (!qi_cache) {
5177 		dev_err(dev, "Can't allocate SEC cache\n");
5178 		return -ENOMEM;
5179 	}
5180 
5181 	err = dma_set_mask_and_coherent(dev, DMA_BIT_MASK(49));
5182 	if (err) {
5183 		dev_err(dev, "dma_set_mask_and_coherent() failed\n");
5184 		goto err_dma_mask;
5185 	}
5186 
5187 	/* Obtain a MC portal */
5188 	err = fsl_mc_portal_allocate(dpseci_dev, 0, &priv->mc_io);
5189 	if (err) {
5190 		if (err == -ENXIO)
5191 			err = -EPROBE_DEFER;
5192 		else
5193 			dev_err(dev, "MC portal allocation failed\n");
5194 
5195 		goto err_dma_mask;
5196 	}
5197 
5198 	priv->ppriv = alloc_percpu(*priv->ppriv);
5199 	if (!priv->ppriv) {
5200 		dev_err(dev, "alloc_percpu() failed\n");
5201 		err = -ENOMEM;
5202 		goto err_alloc_ppriv;
5203 	}
5204 
5205 	/* DPSECI initialization */
5206 	err = dpaa2_dpseci_setup(dpseci_dev);
5207 	if (err) {
5208 		dev_err(dev, "dpaa2_dpseci_setup() failed\n");
5209 		goto err_dpseci_setup;
5210 	}
5211 
5212 	/* DPIO */
5213 	err = dpaa2_dpseci_dpio_setup(priv);
5214 	if (err) {
5215 		dev_err_probe(dev, err, "dpaa2_dpseci_dpio_setup() failed\n");
5216 		goto err_dpio_setup;
5217 	}
5218 
5219 	/* DPSECI binding to DPIO */
5220 	err = dpaa2_dpseci_bind(priv);
5221 	if (err) {
5222 		dev_err(dev, "dpaa2_dpseci_bind() failed\n");
5223 		goto err_bind;
5224 	}
5225 
5226 	/* DPSECI enable */
5227 	err = dpaa2_dpseci_enable(priv);
5228 	if (err) {
5229 		dev_err(dev, "dpaa2_dpseci_enable() failed\n");
5230 		goto err_bind;
5231 	}
5232 
5233 	dpaa2_dpseci_debugfs_init(priv);
5234 
5235 	/* register crypto algorithms the device supports */
5236 	for (i = 0; i < ARRAY_SIZE(driver_algs); i++) {
5237 		struct caam_skcipher_alg *t_alg = driver_algs + i;
5238 		u32 alg_sel = t_alg->caam.class1_alg_type & OP_ALG_ALGSEL_MASK;
5239 
5240 		/* Skip DES algorithms if not supported by device */
5241 		if (!priv->sec_attr.des_acc_num &&
5242 		    (alg_sel == OP_ALG_ALGSEL_3DES ||
5243 		     alg_sel == OP_ALG_ALGSEL_DES))
5244 			continue;
5245 
5246 		/* Skip AES algorithms if not supported by device */
5247 		if (!priv->sec_attr.aes_acc_num &&
5248 		    alg_sel == OP_ALG_ALGSEL_AES)
5249 			continue;
5250 
5251 		/* Skip CHACHA20 algorithms if not supported by device */
5252 		if (alg_sel == OP_ALG_ALGSEL_CHACHA20 &&
5253 		    !priv->sec_attr.ccha_acc_num)
5254 			continue;
5255 
5256 		t_alg->caam.dev = dev;
5257 		caam_skcipher_alg_init(t_alg);
5258 
5259 		err = crypto_register_skcipher(&t_alg->skcipher);
5260 		if (err) {
5261 			dev_warn(dev, "%s alg registration failed: %d\n",
5262 				 t_alg->skcipher.base.cra_driver_name, err);
5263 			continue;
5264 		}
5265 
5266 		t_alg->registered = true;
5267 		registered = true;
5268 	}
5269 
5270 	for (i = 0; i < ARRAY_SIZE(driver_aeads); i++) {
5271 		struct caam_aead_alg *t_alg = driver_aeads + i;
5272 		u32 c1_alg_sel = t_alg->caam.class1_alg_type &
5273 				 OP_ALG_ALGSEL_MASK;
5274 		u32 c2_alg_sel = t_alg->caam.class2_alg_type &
5275 				 OP_ALG_ALGSEL_MASK;
5276 
5277 		/* Skip DES algorithms if not supported by device */
5278 		if (!priv->sec_attr.des_acc_num &&
5279 		    (c1_alg_sel == OP_ALG_ALGSEL_3DES ||
5280 		     c1_alg_sel == OP_ALG_ALGSEL_DES))
5281 			continue;
5282 
5283 		/* Skip AES algorithms if not supported by device */
5284 		if (!priv->sec_attr.aes_acc_num &&
5285 		    c1_alg_sel == OP_ALG_ALGSEL_AES)
5286 			continue;
5287 
5288 		/* Skip CHACHA20 algorithms if not supported by device */
5289 		if (c1_alg_sel == OP_ALG_ALGSEL_CHACHA20 &&
5290 		    !priv->sec_attr.ccha_acc_num)
5291 			continue;
5292 
5293 		/* Skip POLY1305 algorithms if not supported by device */
5294 		if (c2_alg_sel == OP_ALG_ALGSEL_POLY1305 &&
5295 		    !priv->sec_attr.ptha_acc_num)
5296 			continue;
5297 
5298 		/*
5299 		 * Skip algorithms requiring message digests
5300 		 * if MD not supported by device.
5301 		 */
5302 		if ((c2_alg_sel & ~OP_ALG_ALGSEL_SUBMASK) == 0x40 &&
5303 		    !priv->sec_attr.md_acc_num)
5304 			continue;
5305 
5306 		t_alg->caam.dev = dev;
5307 		caam_aead_alg_init(t_alg);
5308 
5309 		err = crypto_register_aead(&t_alg->aead);
5310 		if (err) {
5311 			dev_warn(dev, "%s alg registration failed: %d\n",
5312 				 t_alg->aead.base.cra_driver_name, err);
5313 			continue;
5314 		}
5315 
5316 		t_alg->registered = true;
5317 		registered = true;
5318 	}
5319 	if (registered)
5320 		dev_info(dev, "algorithms registered in /proc/crypto\n");
5321 
5322 	/* register hash algorithms the device supports */
5323 	INIT_LIST_HEAD(&hash_list);
5324 
5325 	/*
5326 	 * Skip registration of any hashing algorithms if MD block
5327 	 * is not present.
5328 	 */
5329 	if (!priv->sec_attr.md_acc_num)
5330 		return 0;
5331 
5332 	for (i = 0; i < ARRAY_SIZE(driver_hash); i++) {
5333 		struct caam_hash_alg *t_alg;
5334 		struct caam_hash_template *alg = driver_hash + i;
5335 
5336 		/* register hmac version */
5337 		t_alg = caam_hash_alloc(dev, alg, true);
5338 		if (IS_ERR(t_alg)) {
5339 			err = PTR_ERR(t_alg);
5340 			dev_warn(dev, "%s hash alg allocation failed: %d\n",
5341 				 alg->hmac_driver_name, err);
5342 			continue;
5343 		}
5344 
5345 		err = crypto_register_ahash(&t_alg->ahash_alg);
5346 		if (err) {
5347 			dev_warn(dev, "%s alg registration failed: %d\n",
5348 				 t_alg->ahash_alg.halg.base.cra_driver_name,
5349 				 err);
5350 			kfree(t_alg);
5351 		} else {
5352 			list_add_tail(&t_alg->entry, &hash_list);
5353 		}
5354 
5355 		/* register unkeyed version */
5356 		t_alg = caam_hash_alloc(dev, alg, false);
5357 		if (IS_ERR(t_alg)) {
5358 			err = PTR_ERR(t_alg);
5359 			dev_warn(dev, "%s alg allocation failed: %d\n",
5360 				 alg->driver_name, err);
5361 			continue;
5362 		}
5363 
5364 		err = crypto_register_ahash(&t_alg->ahash_alg);
5365 		if (err) {
5366 			dev_warn(dev, "%s alg registration failed: %d\n",
5367 				 t_alg->ahash_alg.halg.base.cra_driver_name,
5368 				 err);
5369 			kfree(t_alg);
5370 		} else {
5371 			list_add_tail(&t_alg->entry, &hash_list);
5372 		}
5373 	}
5374 	if (!list_empty(&hash_list))
5375 		dev_info(dev, "hash algorithms registered in /proc/crypto\n");
5376 
5377 	return err;
5378 
5379 err_bind:
5380 	dpaa2_dpseci_dpio_free(priv);
5381 err_dpio_setup:
5382 	dpaa2_dpseci_free(priv);
5383 err_dpseci_setup:
5384 	free_percpu(priv->ppriv);
5385 err_alloc_ppriv:
5386 	fsl_mc_portal_free(priv->mc_io);
5387 err_dma_mask:
5388 	kmem_cache_destroy(qi_cache);
5389 
5390 	return err;
5391 }
5392 
5393 static int __cold dpaa2_caam_remove(struct fsl_mc_device *ls_dev)
5394 {
5395 	struct device *dev;
5396 	struct dpaa2_caam_priv *priv;
5397 	int i;
5398 
5399 	dev = &ls_dev->dev;
5400 	priv = dev_get_drvdata(dev);
5401 
5402 	dpaa2_dpseci_debugfs_exit(priv);
5403 
5404 	for (i = 0; i < ARRAY_SIZE(driver_aeads); i++) {
5405 		struct caam_aead_alg *t_alg = driver_aeads + i;
5406 
5407 		if (t_alg->registered)
5408 			crypto_unregister_aead(&t_alg->aead);
5409 	}
5410 
5411 	for (i = 0; i < ARRAY_SIZE(driver_algs); i++) {
5412 		struct caam_skcipher_alg *t_alg = driver_algs + i;
5413 
5414 		if (t_alg->registered)
5415 			crypto_unregister_skcipher(&t_alg->skcipher);
5416 	}
5417 
5418 	if (hash_list.next) {
5419 		struct caam_hash_alg *t_hash_alg, *p;
5420 
5421 		list_for_each_entry_safe(t_hash_alg, p, &hash_list, entry) {
5422 			crypto_unregister_ahash(&t_hash_alg->ahash_alg);
5423 			list_del(&t_hash_alg->entry);
5424 			kfree(t_hash_alg);
5425 		}
5426 	}
5427 
5428 	dpaa2_dpseci_disable(priv);
5429 	dpaa2_dpseci_dpio_free(priv);
5430 	dpaa2_dpseci_free(priv);
5431 	free_percpu(priv->ppriv);
5432 	fsl_mc_portal_free(priv->mc_io);
5433 	kmem_cache_destroy(qi_cache);
5434 
5435 	return 0;
5436 }
5437 
5438 int dpaa2_caam_enqueue(struct device *dev, struct caam_request *req)
5439 {
5440 	struct dpaa2_fd fd;
5441 	struct dpaa2_caam_priv *priv = dev_get_drvdata(dev);
5442 	struct dpaa2_caam_priv_per_cpu *ppriv;
5443 	int err = 0, i;
5444 
5445 	if (IS_ERR(req))
5446 		return PTR_ERR(req);
5447 
5448 	if (priv->cscn_mem) {
5449 		dma_sync_single_for_cpu(priv->dev, priv->cscn_dma,
5450 					DPAA2_CSCN_SIZE,
5451 					DMA_FROM_DEVICE);
5452 		if (unlikely(dpaa2_cscn_state_congested(priv->cscn_mem_aligned))) {
5453 			dev_dbg_ratelimited(dev, "Dropping request\n");
5454 			return -EBUSY;
5455 		}
5456 	}
5457 
5458 	dpaa2_fl_set_flc(&req->fd_flt[1], req->flc_dma);
5459 
5460 	req->fd_flt_dma = dma_map_single(dev, req->fd_flt, sizeof(req->fd_flt),
5461 					 DMA_BIDIRECTIONAL);
5462 	if (dma_mapping_error(dev, req->fd_flt_dma)) {
5463 		dev_err(dev, "DMA mapping error for QI enqueue request\n");
5464 		goto err_out;
5465 	}
5466 
5467 	memset(&fd, 0, sizeof(fd));
5468 	dpaa2_fd_set_format(&fd, dpaa2_fd_list);
5469 	dpaa2_fd_set_addr(&fd, req->fd_flt_dma);
5470 	dpaa2_fd_set_len(&fd, dpaa2_fl_get_len(&req->fd_flt[1]));
5471 	dpaa2_fd_set_flc(&fd, req->flc_dma);
5472 
5473 	ppriv = this_cpu_ptr(priv->ppriv);
5474 	for (i = 0; i < (priv->dpseci_attr.num_tx_queues << 1); i++) {
5475 		err = dpaa2_io_service_enqueue_fq(ppriv->dpio, ppriv->req_fqid,
5476 						  &fd);
5477 		if (err != -EBUSY)
5478 			break;
5479 
5480 		cpu_relax();
5481 	}
5482 
5483 	if (unlikely(err)) {
5484 		dev_err_ratelimited(dev, "Error enqueuing frame: %d\n", err);
5485 		goto err_out;
5486 	}
5487 
5488 	return -EINPROGRESS;
5489 
5490 err_out:
5491 	dma_unmap_single(dev, req->fd_flt_dma, sizeof(req->fd_flt),
5492 			 DMA_BIDIRECTIONAL);
5493 	return -EIO;
5494 }
5495 EXPORT_SYMBOL(dpaa2_caam_enqueue);
5496 
5497 static const struct fsl_mc_device_id dpaa2_caam_match_id_table[] = {
5498 	{
5499 		.vendor = FSL_MC_VENDOR_FREESCALE,
5500 		.obj_type = "dpseci",
5501 	},
5502 	{ .vendor = 0x0 }
5503 };
5504 MODULE_DEVICE_TABLE(fslmc, dpaa2_caam_match_id_table);
5505 
5506 static struct fsl_mc_driver dpaa2_caam_driver = {
5507 	.driver = {
5508 		.name		= KBUILD_MODNAME,
5509 		.owner		= THIS_MODULE,
5510 	},
5511 	.probe		= dpaa2_caam_probe,
5512 	.remove		= dpaa2_caam_remove,
5513 	.match_id_table = dpaa2_caam_match_id_table
5514 };
5515 
5516 MODULE_LICENSE("Dual BSD/GPL");
5517 MODULE_AUTHOR("Freescale Semiconductor, Inc");
5518 MODULE_DESCRIPTION("Freescale DPAA2 CAAM Driver");
5519 
5520 module_fsl_mc_driver(dpaa2_caam_driver);
5521