xref: /openbmc/linux/drivers/crypto/caam/caamalg.c (revision c819e2cf)
1 /*
2  * caam - Freescale FSL CAAM support for crypto API
3  *
4  * Copyright 2008-2011 Freescale Semiconductor, Inc.
5  *
6  * Based on talitos crypto API driver.
7  *
8  * relationship of job descriptors to shared descriptors (SteveC Dec 10 2008):
9  *
10  * ---------------                     ---------------
11  * | JobDesc #1  |-------------------->|  ShareDesc  |
12  * | *(packet 1) |                     |   (PDB)     |
13  * ---------------      |------------->|  (hashKey)  |
14  *       .              |              | (cipherKey) |
15  *       .              |    |-------->| (operation) |
16  * ---------------      |    |         ---------------
17  * | JobDesc #2  |------|    |
18  * | *(packet 2) |           |
19  * ---------------           |
20  *       .                   |
21  *       .                   |
22  * ---------------           |
23  * | JobDesc #3  |------------
24  * | *(packet 3) |
25  * ---------------
26  *
27  * The SharedDesc never changes for a connection unless rekeyed, but
28  * each packet will likely be in a different place. So all we need
29  * to know to process the packet is where the input is, where the
30  * output goes, and what context we want to process with. Context is
31  * in the SharedDesc, packet references in the JobDesc.
32  *
33  * So, a job desc looks like:
34  *
35  * ---------------------
36  * | Header            |
37  * | ShareDesc Pointer |
38  * | SEQ_OUT_PTR       |
39  * | (output buffer)   |
40  * | (output length)   |
41  * | SEQ_IN_PTR        |
42  * | (input buffer)    |
43  * | (input length)    |
44  * ---------------------
45  */
46 
47 #include "compat.h"
48 
49 #include "regs.h"
50 #include "intern.h"
51 #include "desc_constr.h"
52 #include "jr.h"
53 #include "error.h"
54 #include "sg_sw_sec4.h"
55 #include "key_gen.h"
56 
57 /*
58  * crypto alg
59  */
60 #define CAAM_CRA_PRIORITY		3000
61 /* max key is sum of AES_MAX_KEY_SIZE, max split key size */
62 #define CAAM_MAX_KEY_SIZE		(AES_MAX_KEY_SIZE + \
63 					 CTR_RFC3686_NONCE_SIZE + \
64 					 SHA512_DIGEST_SIZE * 2)
65 /* max IV is max of AES_BLOCK_SIZE, DES3_EDE_BLOCK_SIZE */
66 #define CAAM_MAX_IV_LENGTH		16
67 
68 /* length of descriptors text */
69 #define DESC_AEAD_BASE			(4 * CAAM_CMD_SZ)
70 #define DESC_AEAD_ENC_LEN		(DESC_AEAD_BASE + 15 * CAAM_CMD_SZ)
71 #define DESC_AEAD_DEC_LEN		(DESC_AEAD_BASE + 18 * CAAM_CMD_SZ)
72 #define DESC_AEAD_GIVENC_LEN		(DESC_AEAD_ENC_LEN + 7 * CAAM_CMD_SZ)
73 
74 /* Note: Nonce is counted in enckeylen */
75 #define DESC_AEAD_CTR_RFC3686_LEN	(6 * CAAM_CMD_SZ)
76 
77 #define DESC_AEAD_NULL_BASE		(3 * CAAM_CMD_SZ)
78 #define DESC_AEAD_NULL_ENC_LEN		(DESC_AEAD_NULL_BASE + 14 * CAAM_CMD_SZ)
79 #define DESC_AEAD_NULL_DEC_LEN		(DESC_AEAD_NULL_BASE + 17 * CAAM_CMD_SZ)
80 
81 #define DESC_GCM_BASE			(3 * CAAM_CMD_SZ)
82 #define DESC_GCM_ENC_LEN		(DESC_GCM_BASE + 23 * CAAM_CMD_SZ)
83 #define DESC_GCM_DEC_LEN		(DESC_GCM_BASE + 19 * CAAM_CMD_SZ)
84 
85 #define DESC_RFC4106_BASE		(3 * CAAM_CMD_SZ)
86 #define DESC_RFC4106_ENC_LEN		(DESC_RFC4106_BASE + 15 * CAAM_CMD_SZ)
87 #define DESC_RFC4106_DEC_LEN		(DESC_RFC4106_BASE + 14 * CAAM_CMD_SZ)
88 #define DESC_RFC4106_GIVENC_LEN		(DESC_RFC4106_BASE + 21 * CAAM_CMD_SZ)
89 
90 #define DESC_RFC4543_BASE		(3 * CAAM_CMD_SZ)
91 #define DESC_RFC4543_ENC_LEN		(DESC_RFC4543_BASE + 25 * CAAM_CMD_SZ)
92 #define DESC_RFC4543_DEC_LEN		(DESC_RFC4543_BASE + 27 * CAAM_CMD_SZ)
93 #define DESC_RFC4543_GIVENC_LEN		(DESC_RFC4543_BASE + 30 * CAAM_CMD_SZ)
94 
95 #define DESC_ABLKCIPHER_BASE		(3 * CAAM_CMD_SZ)
96 #define DESC_ABLKCIPHER_ENC_LEN		(DESC_ABLKCIPHER_BASE + \
97 					 20 * CAAM_CMD_SZ)
98 #define DESC_ABLKCIPHER_DEC_LEN		(DESC_ABLKCIPHER_BASE + \
99 					 15 * CAAM_CMD_SZ)
100 
101 #define DESC_MAX_USED_BYTES		(DESC_RFC4543_GIVENC_LEN + \
102 					 CAAM_MAX_KEY_SIZE)
103 #define DESC_MAX_USED_LEN		(DESC_MAX_USED_BYTES / CAAM_CMD_SZ)
104 
105 #ifdef DEBUG
106 /* for print_hex_dumps with line references */
107 #define debug(format, arg...) printk(format, arg)
108 #else
109 #define debug(format, arg...)
110 #endif
111 static struct list_head alg_list;
112 
113 /* Set DK bit in class 1 operation if shared */
114 static inline void append_dec_op1(u32 *desc, u32 type)
115 {
116 	u32 *jump_cmd, *uncond_jump_cmd;
117 
118 	/* DK bit is valid only for AES */
119 	if ((type & OP_ALG_ALGSEL_MASK) != OP_ALG_ALGSEL_AES) {
120 		append_operation(desc, type | OP_ALG_AS_INITFINAL |
121 				 OP_ALG_DECRYPT);
122 		return;
123 	}
124 
125 	jump_cmd = append_jump(desc, JUMP_TEST_ALL | JUMP_COND_SHRD);
126 	append_operation(desc, type | OP_ALG_AS_INITFINAL |
127 			 OP_ALG_DECRYPT);
128 	uncond_jump_cmd = append_jump(desc, JUMP_TEST_ALL);
129 	set_jump_tgt_here(desc, jump_cmd);
130 	append_operation(desc, type | OP_ALG_AS_INITFINAL |
131 			 OP_ALG_DECRYPT | OP_ALG_AAI_DK);
132 	set_jump_tgt_here(desc, uncond_jump_cmd);
133 }
134 
135 /*
136  * For aead functions, read payload and write payload,
137  * both of which are specified in req->src and req->dst
138  */
139 static inline void aead_append_src_dst(u32 *desc, u32 msg_type)
140 {
141 	append_seq_fifo_store(desc, 0, FIFOST_TYPE_MESSAGE_DATA | KEY_VLF);
142 	append_seq_fifo_load(desc, 0, FIFOLD_CLASS_BOTH |
143 			     KEY_VLF | msg_type | FIFOLD_TYPE_LASTBOTH);
144 }
145 
146 /*
147  * For aead encrypt and decrypt, read iv for both classes
148  */
149 static inline void aead_append_ld_iv(u32 *desc, int ivsize, int ivoffset)
150 {
151 	append_seq_load(desc, ivsize, LDST_CLASS_1_CCB |
152 			LDST_SRCDST_BYTE_CONTEXT |
153 			(ivoffset << LDST_OFFSET_SHIFT));
154 	append_move(desc, MOVE_SRC_CLASS1CTX | MOVE_DEST_CLASS2INFIFO |
155 		    (ivoffset << MOVE_OFFSET_SHIFT) | ivsize);
156 }
157 
158 /*
159  * For ablkcipher encrypt and decrypt, read from req->src and
160  * write to req->dst
161  */
162 static inline void ablkcipher_append_src_dst(u32 *desc)
163 {
164 	append_math_add(desc, VARSEQOUTLEN, SEQINLEN, REG0, CAAM_CMD_SZ);
165 	append_math_add(desc, VARSEQINLEN, SEQINLEN, REG0, CAAM_CMD_SZ);
166 	append_seq_fifo_load(desc, 0, FIFOLD_CLASS_CLASS1 |
167 			     KEY_VLF | FIFOLD_TYPE_MSG | FIFOLD_TYPE_LAST1);
168 	append_seq_fifo_store(desc, 0, FIFOST_TYPE_MESSAGE_DATA | KEY_VLF);
169 }
170 
171 /*
172  * If all data, including src (with assoc and iv) or dst (with iv only) are
173  * contiguous
174  */
175 #define GIV_SRC_CONTIG		1
176 #define GIV_DST_CONTIG		(1 << 1)
177 
178 /*
179  * per-session context
180  */
181 struct caam_ctx {
182 	struct device *jrdev;
183 	u32 sh_desc_enc[DESC_MAX_USED_LEN];
184 	u32 sh_desc_dec[DESC_MAX_USED_LEN];
185 	u32 sh_desc_givenc[DESC_MAX_USED_LEN];
186 	dma_addr_t sh_desc_enc_dma;
187 	dma_addr_t sh_desc_dec_dma;
188 	dma_addr_t sh_desc_givenc_dma;
189 	u32 class1_alg_type;
190 	u32 class2_alg_type;
191 	u32 alg_op;
192 	u8 key[CAAM_MAX_KEY_SIZE];
193 	dma_addr_t key_dma;
194 	unsigned int enckeylen;
195 	unsigned int split_key_len;
196 	unsigned int split_key_pad_len;
197 	unsigned int authsize;
198 };
199 
200 static void append_key_aead(u32 *desc, struct caam_ctx *ctx,
201 			    int keys_fit_inline, bool is_rfc3686)
202 {
203 	u32 *nonce;
204 	unsigned int enckeylen = ctx->enckeylen;
205 
206 	/*
207 	 * RFC3686 specific:
208 	 *	| ctx->key = {AUTH_KEY, ENC_KEY, NONCE}
209 	 *	| enckeylen = encryption key size + nonce size
210 	 */
211 	if (is_rfc3686)
212 		enckeylen -= CTR_RFC3686_NONCE_SIZE;
213 
214 	if (keys_fit_inline) {
215 		append_key_as_imm(desc, ctx->key, ctx->split_key_pad_len,
216 				  ctx->split_key_len, CLASS_2 |
217 				  KEY_DEST_MDHA_SPLIT | KEY_ENC);
218 		append_key_as_imm(desc, (void *)ctx->key +
219 				  ctx->split_key_pad_len, enckeylen,
220 				  enckeylen, CLASS_1 | KEY_DEST_CLASS_REG);
221 	} else {
222 		append_key(desc, ctx->key_dma, ctx->split_key_len, CLASS_2 |
223 			   KEY_DEST_MDHA_SPLIT | KEY_ENC);
224 		append_key(desc, ctx->key_dma + ctx->split_key_pad_len,
225 			   enckeylen, CLASS_1 | KEY_DEST_CLASS_REG);
226 	}
227 
228 	/* Load Counter into CONTEXT1 reg */
229 	if (is_rfc3686) {
230 		nonce = (u32 *)((void *)ctx->key + ctx->split_key_pad_len +
231 			       enckeylen);
232 		append_load_imm_u32(desc, *nonce, LDST_CLASS_IND_CCB |
233 				    LDST_SRCDST_BYTE_OUTFIFO | LDST_IMM);
234 		append_move(desc,
235 			    MOVE_SRC_OUTFIFO |
236 			    MOVE_DEST_CLASS1CTX |
237 			    (16 << MOVE_OFFSET_SHIFT) |
238 			    (CTR_RFC3686_NONCE_SIZE << MOVE_LEN_SHIFT));
239 	}
240 }
241 
242 static void init_sh_desc_key_aead(u32 *desc, struct caam_ctx *ctx,
243 				  int keys_fit_inline, bool is_rfc3686)
244 {
245 	u32 *key_jump_cmd;
246 
247 	/* Note: Context registers are saved. */
248 	init_sh_desc(desc, HDR_SHARE_SERIAL | HDR_SAVECTX);
249 
250 	/* Skip if already shared */
251 	key_jump_cmd = append_jump(desc, JUMP_JSL | JUMP_TEST_ALL |
252 				   JUMP_COND_SHRD);
253 
254 	append_key_aead(desc, ctx, keys_fit_inline, is_rfc3686);
255 
256 	set_jump_tgt_here(desc, key_jump_cmd);
257 }
258 
259 static int aead_null_set_sh_desc(struct crypto_aead *aead)
260 {
261 	struct aead_tfm *tfm = &aead->base.crt_aead;
262 	struct caam_ctx *ctx = crypto_aead_ctx(aead);
263 	struct device *jrdev = ctx->jrdev;
264 	bool keys_fit_inline = false;
265 	u32 *key_jump_cmd, *jump_cmd, *read_move_cmd, *write_move_cmd;
266 	u32 *desc;
267 
268 	/*
269 	 * Job Descriptor and Shared Descriptors
270 	 * must all fit into the 64-word Descriptor h/w Buffer
271 	 */
272 	if (DESC_AEAD_NULL_ENC_LEN + DESC_JOB_IO_LEN +
273 	    ctx->split_key_pad_len <= CAAM_DESC_BYTES_MAX)
274 		keys_fit_inline = true;
275 
276 	/* aead_encrypt shared descriptor */
277 	desc = ctx->sh_desc_enc;
278 
279 	init_sh_desc(desc, HDR_SHARE_SERIAL);
280 
281 	/* Skip if already shared */
282 	key_jump_cmd = append_jump(desc, JUMP_JSL | JUMP_TEST_ALL |
283 				   JUMP_COND_SHRD);
284 	if (keys_fit_inline)
285 		append_key_as_imm(desc, ctx->key, ctx->split_key_pad_len,
286 				  ctx->split_key_len, CLASS_2 |
287 				  KEY_DEST_MDHA_SPLIT | KEY_ENC);
288 	else
289 		append_key(desc, ctx->key_dma, ctx->split_key_len, CLASS_2 |
290 			   KEY_DEST_MDHA_SPLIT | KEY_ENC);
291 	set_jump_tgt_here(desc, key_jump_cmd);
292 
293 	/* cryptlen = seqoutlen - authsize */
294 	append_math_sub_imm_u32(desc, REG3, SEQOUTLEN, IMM, ctx->authsize);
295 
296 	/*
297 	 * NULL encryption; IV is zero
298 	 * assoclen = (assoclen + cryptlen) - cryptlen
299 	 */
300 	append_math_sub(desc, VARSEQINLEN, SEQINLEN, REG3, CAAM_CMD_SZ);
301 
302 	/* read assoc before reading payload */
303 	append_seq_fifo_load(desc, 0, FIFOLD_CLASS_CLASS2 | FIFOLD_TYPE_MSG |
304 			     KEY_VLF);
305 
306 	/* Prepare to read and write cryptlen bytes */
307 	append_math_add(desc, VARSEQINLEN, ZERO, REG3, CAAM_CMD_SZ);
308 	append_math_add(desc, VARSEQOUTLEN, ZERO, REG3, CAAM_CMD_SZ);
309 
310 	/*
311 	 * MOVE_LEN opcode is not available in all SEC HW revisions,
312 	 * thus need to do some magic, i.e. self-patch the descriptor
313 	 * buffer.
314 	 */
315 	read_move_cmd = append_move(desc, MOVE_SRC_DESCBUF |
316 				    MOVE_DEST_MATH3 |
317 				    (0x6 << MOVE_LEN_SHIFT));
318 	write_move_cmd = append_move(desc, MOVE_SRC_MATH3 |
319 				     MOVE_DEST_DESCBUF |
320 				     MOVE_WAITCOMP |
321 				     (0x8 << MOVE_LEN_SHIFT));
322 
323 	/* Class 2 operation */
324 	append_operation(desc, ctx->class2_alg_type |
325 			 OP_ALG_AS_INITFINAL | OP_ALG_ENCRYPT);
326 
327 	/* Read and write cryptlen bytes */
328 	aead_append_src_dst(desc, FIFOLD_TYPE_MSG | FIFOLD_TYPE_FLUSH1);
329 
330 	set_move_tgt_here(desc, read_move_cmd);
331 	set_move_tgt_here(desc, write_move_cmd);
332 	append_cmd(desc, CMD_LOAD | DISABLE_AUTO_INFO_FIFO);
333 	append_move(desc, MOVE_SRC_INFIFO_CL | MOVE_DEST_OUTFIFO |
334 		    MOVE_AUX_LS);
335 
336 	/* Write ICV */
337 	append_seq_store(desc, ctx->authsize, LDST_CLASS_2_CCB |
338 			 LDST_SRCDST_BYTE_CONTEXT);
339 
340 	ctx->sh_desc_enc_dma = dma_map_single(jrdev, desc,
341 					      desc_bytes(desc),
342 					      DMA_TO_DEVICE);
343 	if (dma_mapping_error(jrdev, ctx->sh_desc_enc_dma)) {
344 		dev_err(jrdev, "unable to map shared descriptor\n");
345 		return -ENOMEM;
346 	}
347 #ifdef DEBUG
348 	print_hex_dump(KERN_ERR,
349 		       "aead null enc shdesc@"__stringify(__LINE__)": ",
350 		       DUMP_PREFIX_ADDRESS, 16, 4, desc,
351 		       desc_bytes(desc), 1);
352 #endif
353 
354 	/*
355 	 * Job Descriptor and Shared Descriptors
356 	 * must all fit into the 64-word Descriptor h/w Buffer
357 	 */
358 	keys_fit_inline = false;
359 	if (DESC_AEAD_NULL_DEC_LEN + DESC_JOB_IO_LEN +
360 	    ctx->split_key_pad_len <= CAAM_DESC_BYTES_MAX)
361 		keys_fit_inline = true;
362 
363 	desc = ctx->sh_desc_dec;
364 
365 	/* aead_decrypt shared descriptor */
366 	init_sh_desc(desc, HDR_SHARE_SERIAL);
367 
368 	/* Skip if already shared */
369 	key_jump_cmd = append_jump(desc, JUMP_JSL | JUMP_TEST_ALL |
370 				   JUMP_COND_SHRD);
371 	if (keys_fit_inline)
372 		append_key_as_imm(desc, ctx->key, ctx->split_key_pad_len,
373 				  ctx->split_key_len, CLASS_2 |
374 				  KEY_DEST_MDHA_SPLIT | KEY_ENC);
375 	else
376 		append_key(desc, ctx->key_dma, ctx->split_key_len, CLASS_2 |
377 			   KEY_DEST_MDHA_SPLIT | KEY_ENC);
378 	set_jump_tgt_here(desc, key_jump_cmd);
379 
380 	/* Class 2 operation */
381 	append_operation(desc, ctx->class2_alg_type |
382 			 OP_ALG_AS_INITFINAL | OP_ALG_DECRYPT | OP_ALG_ICV_ON);
383 
384 	/* assoclen + cryptlen = seqinlen - ivsize - authsize */
385 	append_math_sub_imm_u32(desc, REG3, SEQINLEN, IMM,
386 				ctx->authsize + tfm->ivsize);
387 	/* assoclen = (assoclen + cryptlen) - cryptlen */
388 	append_math_sub(desc, REG2, SEQOUTLEN, REG0, CAAM_CMD_SZ);
389 	append_math_sub(desc, VARSEQINLEN, REG3, REG2, CAAM_CMD_SZ);
390 
391 	/* read assoc before reading payload */
392 	append_seq_fifo_load(desc, 0, FIFOLD_CLASS_CLASS2 | FIFOLD_TYPE_MSG |
393 			     KEY_VLF);
394 
395 	/* Prepare to read and write cryptlen bytes */
396 	append_math_add(desc, VARSEQINLEN, ZERO, REG2, CAAM_CMD_SZ);
397 	append_math_add(desc, VARSEQOUTLEN, ZERO, REG2, CAAM_CMD_SZ);
398 
399 	/*
400 	 * MOVE_LEN opcode is not available in all SEC HW revisions,
401 	 * thus need to do some magic, i.e. self-patch the descriptor
402 	 * buffer.
403 	 */
404 	read_move_cmd = append_move(desc, MOVE_SRC_DESCBUF |
405 				    MOVE_DEST_MATH2 |
406 				    (0x6 << MOVE_LEN_SHIFT));
407 	write_move_cmd = append_move(desc, MOVE_SRC_MATH2 |
408 				     MOVE_DEST_DESCBUF |
409 				     MOVE_WAITCOMP |
410 				     (0x8 << MOVE_LEN_SHIFT));
411 
412 	/* Read and write cryptlen bytes */
413 	aead_append_src_dst(desc, FIFOLD_TYPE_MSG | FIFOLD_TYPE_FLUSH1);
414 
415 	/*
416 	 * Insert a NOP here, since we need at least 4 instructions between
417 	 * code patching the descriptor buffer and the location being patched.
418 	 */
419 	jump_cmd = append_jump(desc, JUMP_TEST_ALL);
420 	set_jump_tgt_here(desc, jump_cmd);
421 
422 	set_move_tgt_here(desc, read_move_cmd);
423 	set_move_tgt_here(desc, write_move_cmd);
424 	append_cmd(desc, CMD_LOAD | DISABLE_AUTO_INFO_FIFO);
425 	append_move(desc, MOVE_SRC_INFIFO_CL | MOVE_DEST_OUTFIFO |
426 		    MOVE_AUX_LS);
427 	append_cmd(desc, CMD_LOAD | ENABLE_AUTO_INFO_FIFO);
428 
429 	/* Load ICV */
430 	append_seq_fifo_load(desc, ctx->authsize, FIFOLD_CLASS_CLASS2 |
431 			     FIFOLD_TYPE_LAST2 | FIFOLD_TYPE_ICV);
432 
433 	ctx->sh_desc_dec_dma = dma_map_single(jrdev, desc,
434 					      desc_bytes(desc),
435 					      DMA_TO_DEVICE);
436 	if (dma_mapping_error(jrdev, ctx->sh_desc_dec_dma)) {
437 		dev_err(jrdev, "unable to map shared descriptor\n");
438 		return -ENOMEM;
439 	}
440 #ifdef DEBUG
441 	print_hex_dump(KERN_ERR,
442 		       "aead null dec shdesc@"__stringify(__LINE__)": ",
443 		       DUMP_PREFIX_ADDRESS, 16, 4, desc,
444 		       desc_bytes(desc), 1);
445 #endif
446 
447 	return 0;
448 }
449 
450 static int aead_set_sh_desc(struct crypto_aead *aead)
451 {
452 	struct aead_tfm *tfm = &aead->base.crt_aead;
453 	struct caam_ctx *ctx = crypto_aead_ctx(aead);
454 	struct crypto_tfm *ctfm = crypto_aead_tfm(aead);
455 	const char *alg_name = crypto_tfm_alg_name(ctfm);
456 	struct device *jrdev = ctx->jrdev;
457 	bool keys_fit_inline;
458 	u32 geniv, moveiv;
459 	u32 ctx1_iv_off = 0;
460 	u32 *desc;
461 	const bool ctr_mode = ((ctx->class1_alg_type & OP_ALG_AAI_MASK) ==
462 			       OP_ALG_AAI_CTR_MOD128);
463 	const bool is_rfc3686 = (ctr_mode &&
464 				 (strstr(alg_name, "rfc3686") != NULL));
465 
466 	if (!ctx->authsize)
467 		return 0;
468 
469 	/* NULL encryption / decryption */
470 	if (!ctx->enckeylen)
471 		return aead_null_set_sh_desc(aead);
472 
473 	/*
474 	 * AES-CTR needs to load IV in CONTEXT1 reg
475 	 * at an offset of 128bits (16bytes)
476 	 * CONTEXT1[255:128] = IV
477 	 */
478 	if (ctr_mode)
479 		ctx1_iv_off = 16;
480 
481 	/*
482 	 * RFC3686 specific:
483 	 *	CONTEXT1[255:128] = {NONCE, IV, COUNTER}
484 	 */
485 	if (is_rfc3686)
486 		ctx1_iv_off = 16 + CTR_RFC3686_NONCE_SIZE;
487 
488 	/*
489 	 * Job Descriptor and Shared Descriptors
490 	 * must all fit into the 64-word Descriptor h/w Buffer
491 	 */
492 	keys_fit_inline = false;
493 	if (DESC_AEAD_ENC_LEN + DESC_JOB_IO_LEN +
494 	    ctx->split_key_pad_len + ctx->enckeylen +
495 	    (is_rfc3686 ? DESC_AEAD_CTR_RFC3686_LEN : 0) <=
496 	    CAAM_DESC_BYTES_MAX)
497 		keys_fit_inline = true;
498 
499 	/* aead_encrypt shared descriptor */
500 	desc = ctx->sh_desc_enc;
501 
502 	/* Note: Context registers are saved. */
503 	init_sh_desc_key_aead(desc, ctx, keys_fit_inline, is_rfc3686);
504 
505 	/* Class 2 operation */
506 	append_operation(desc, ctx->class2_alg_type |
507 			 OP_ALG_AS_INITFINAL | OP_ALG_ENCRYPT);
508 
509 	/* cryptlen = seqoutlen - authsize */
510 	append_math_sub_imm_u32(desc, REG3, SEQOUTLEN, IMM, ctx->authsize);
511 
512 	/* assoclen + cryptlen = seqinlen - ivsize */
513 	append_math_sub_imm_u32(desc, REG2, SEQINLEN, IMM, tfm->ivsize);
514 
515 	/* assoclen = (assoclen + cryptlen) - cryptlen */
516 	append_math_sub(desc, VARSEQINLEN, REG2, REG3, CAAM_CMD_SZ);
517 
518 	/* read assoc before reading payload */
519 	append_seq_fifo_load(desc, 0, FIFOLD_CLASS_CLASS2 | FIFOLD_TYPE_MSG |
520 			     KEY_VLF);
521 	aead_append_ld_iv(desc, tfm->ivsize, ctx1_iv_off);
522 
523 	/* Load Counter into CONTEXT1 reg */
524 	if (is_rfc3686)
525 		append_load_imm_u32(desc, be32_to_cpu(1), LDST_IMM |
526 				    LDST_CLASS_1_CCB |
527 				    LDST_SRCDST_BYTE_CONTEXT |
528 				    ((ctx1_iv_off + CTR_RFC3686_IV_SIZE) <<
529 				     LDST_OFFSET_SHIFT));
530 
531 	/* Class 1 operation */
532 	append_operation(desc, ctx->class1_alg_type |
533 			 OP_ALG_AS_INITFINAL | OP_ALG_ENCRYPT);
534 
535 	/* Read and write cryptlen bytes */
536 	append_math_add(desc, VARSEQINLEN, ZERO, REG3, CAAM_CMD_SZ);
537 	append_math_add(desc, VARSEQOUTLEN, ZERO, REG3, CAAM_CMD_SZ);
538 	aead_append_src_dst(desc, FIFOLD_TYPE_MSG1OUT2);
539 
540 	/* Write ICV */
541 	append_seq_store(desc, ctx->authsize, LDST_CLASS_2_CCB |
542 			 LDST_SRCDST_BYTE_CONTEXT);
543 
544 	ctx->sh_desc_enc_dma = dma_map_single(jrdev, desc,
545 					      desc_bytes(desc),
546 					      DMA_TO_DEVICE);
547 	if (dma_mapping_error(jrdev, ctx->sh_desc_enc_dma)) {
548 		dev_err(jrdev, "unable to map shared descriptor\n");
549 		return -ENOMEM;
550 	}
551 #ifdef DEBUG
552 	print_hex_dump(KERN_ERR, "aead enc shdesc@"__stringify(__LINE__)": ",
553 		       DUMP_PREFIX_ADDRESS, 16, 4, desc,
554 		       desc_bytes(desc), 1);
555 #endif
556 
557 	/*
558 	 * Job Descriptor and Shared Descriptors
559 	 * must all fit into the 64-word Descriptor h/w Buffer
560 	 */
561 	keys_fit_inline = false;
562 	if (DESC_AEAD_DEC_LEN + DESC_JOB_IO_LEN +
563 	    ctx->split_key_pad_len + ctx->enckeylen +
564 	    (is_rfc3686 ? DESC_AEAD_CTR_RFC3686_LEN : 0) <=
565 	    CAAM_DESC_BYTES_MAX)
566 		keys_fit_inline = true;
567 
568 	/* aead_decrypt shared descriptor */
569 	desc = ctx->sh_desc_dec;
570 
571 	/* Note: Context registers are saved. */
572 	init_sh_desc_key_aead(desc, ctx, keys_fit_inline, is_rfc3686);
573 
574 	/* Class 2 operation */
575 	append_operation(desc, ctx->class2_alg_type |
576 			 OP_ALG_AS_INITFINAL | OP_ALG_DECRYPT | OP_ALG_ICV_ON);
577 
578 	/* assoclen + cryptlen = seqinlen - ivsize - authsize */
579 	append_math_sub_imm_u32(desc, REG3, SEQINLEN, IMM,
580 				ctx->authsize + tfm->ivsize);
581 	/* assoclen = (assoclen + cryptlen) - cryptlen */
582 	append_math_sub(desc, REG2, SEQOUTLEN, REG0, CAAM_CMD_SZ);
583 	append_math_sub(desc, VARSEQINLEN, REG3, REG2, CAAM_CMD_SZ);
584 
585 	/* read assoc before reading payload */
586 	append_seq_fifo_load(desc, 0, FIFOLD_CLASS_CLASS2 | FIFOLD_TYPE_MSG |
587 			     KEY_VLF);
588 
589 	aead_append_ld_iv(desc, tfm->ivsize, ctx1_iv_off);
590 
591 	/* Load Counter into CONTEXT1 reg */
592 	if (is_rfc3686)
593 		append_load_imm_u32(desc, be32_to_cpu(1), LDST_IMM |
594 				    LDST_CLASS_1_CCB |
595 				    LDST_SRCDST_BYTE_CONTEXT |
596 				    ((ctx1_iv_off + CTR_RFC3686_IV_SIZE) <<
597 				     LDST_OFFSET_SHIFT));
598 
599 	/* Choose operation */
600 	if (ctr_mode)
601 		append_operation(desc, ctx->class1_alg_type |
602 				 OP_ALG_AS_INITFINAL | OP_ALG_DECRYPT);
603 	else
604 		append_dec_op1(desc, ctx->class1_alg_type);
605 
606 	/* Read and write cryptlen bytes */
607 	append_math_add(desc, VARSEQINLEN, ZERO, REG2, CAAM_CMD_SZ);
608 	append_math_add(desc, VARSEQOUTLEN, ZERO, REG2, CAAM_CMD_SZ);
609 	aead_append_src_dst(desc, FIFOLD_TYPE_MSG);
610 
611 	/* Load ICV */
612 	append_seq_fifo_load(desc, ctx->authsize, FIFOLD_CLASS_CLASS2 |
613 			     FIFOLD_TYPE_LAST2 | FIFOLD_TYPE_ICV);
614 
615 	ctx->sh_desc_dec_dma = dma_map_single(jrdev, desc,
616 					      desc_bytes(desc),
617 					      DMA_TO_DEVICE);
618 	if (dma_mapping_error(jrdev, ctx->sh_desc_dec_dma)) {
619 		dev_err(jrdev, "unable to map shared descriptor\n");
620 		return -ENOMEM;
621 	}
622 #ifdef DEBUG
623 	print_hex_dump(KERN_ERR, "aead dec shdesc@"__stringify(__LINE__)": ",
624 		       DUMP_PREFIX_ADDRESS, 16, 4, desc,
625 		       desc_bytes(desc), 1);
626 #endif
627 
628 	/*
629 	 * Job Descriptor and Shared Descriptors
630 	 * must all fit into the 64-word Descriptor h/w Buffer
631 	 */
632 	keys_fit_inline = false;
633 	if (DESC_AEAD_GIVENC_LEN + DESC_JOB_IO_LEN +
634 	    ctx->split_key_pad_len + ctx->enckeylen +
635 	    (is_rfc3686 ? DESC_AEAD_CTR_RFC3686_LEN : 0) <=
636 	    CAAM_DESC_BYTES_MAX)
637 		keys_fit_inline = true;
638 
639 	/* aead_givencrypt shared descriptor */
640 	desc = ctx->sh_desc_givenc;
641 
642 	/* Note: Context registers are saved. */
643 	init_sh_desc_key_aead(desc, ctx, keys_fit_inline, is_rfc3686);
644 
645 	/* Generate IV */
646 	geniv = NFIFOENTRY_STYPE_PAD | NFIFOENTRY_DEST_DECO |
647 		NFIFOENTRY_DTYPE_MSG | NFIFOENTRY_LC1 |
648 		NFIFOENTRY_PTYPE_RND | (tfm->ivsize << NFIFOENTRY_DLEN_SHIFT);
649 	append_load_imm_u32(desc, geniv, LDST_CLASS_IND_CCB |
650 			    LDST_SRCDST_WORD_INFO_FIFO | LDST_IMM);
651 	append_cmd(desc, CMD_LOAD | DISABLE_AUTO_INFO_FIFO);
652 	append_move(desc, MOVE_WAITCOMP |
653 		    MOVE_SRC_INFIFO | MOVE_DEST_CLASS1CTX |
654 		    (ctx1_iv_off << MOVE_OFFSET_SHIFT) |
655 		    (tfm->ivsize << MOVE_LEN_SHIFT));
656 	append_cmd(desc, CMD_LOAD | ENABLE_AUTO_INFO_FIFO);
657 
658 	/* Copy IV to class 1 context */
659 	append_move(desc, MOVE_SRC_CLASS1CTX | MOVE_DEST_OUTFIFO |
660 		    (ctx1_iv_off << MOVE_OFFSET_SHIFT) |
661 		    (tfm->ivsize << MOVE_LEN_SHIFT));
662 
663 	/* Return to encryption */
664 	append_operation(desc, ctx->class2_alg_type |
665 			 OP_ALG_AS_INITFINAL | OP_ALG_ENCRYPT);
666 
667 	/* ivsize + cryptlen = seqoutlen - authsize */
668 	append_math_sub_imm_u32(desc, REG3, SEQOUTLEN, IMM, ctx->authsize);
669 
670 	/* assoclen = seqinlen - (ivsize + cryptlen) */
671 	append_math_sub(desc, VARSEQINLEN, SEQINLEN, REG3, CAAM_CMD_SZ);
672 
673 	/* read assoc before reading payload */
674 	append_seq_fifo_load(desc, 0, FIFOLD_CLASS_CLASS2 | FIFOLD_TYPE_MSG |
675 			     KEY_VLF);
676 
677 	/* Copy iv from outfifo to class 2 fifo */
678 	moveiv = NFIFOENTRY_STYPE_OFIFO | NFIFOENTRY_DEST_CLASS2 |
679 		 NFIFOENTRY_DTYPE_MSG | (tfm->ivsize << NFIFOENTRY_DLEN_SHIFT);
680 	append_load_imm_u32(desc, moveiv, LDST_CLASS_IND_CCB |
681 			    LDST_SRCDST_WORD_INFO_FIFO | LDST_IMM);
682 	append_load_imm_u32(desc, tfm->ivsize, LDST_CLASS_2_CCB |
683 			    LDST_SRCDST_WORD_DATASZ_REG | LDST_IMM);
684 
685 	/* Load Counter into CONTEXT1 reg */
686 	if (is_rfc3686)
687 		append_load_imm_u32(desc, be32_to_cpu(1), LDST_IMM |
688 				    LDST_CLASS_1_CCB |
689 				    LDST_SRCDST_BYTE_CONTEXT |
690 				    ((ctx1_iv_off + CTR_RFC3686_IV_SIZE) <<
691 				     LDST_OFFSET_SHIFT));
692 
693 	/* Class 1 operation */
694 	append_operation(desc, ctx->class1_alg_type |
695 			 OP_ALG_AS_INITFINAL | OP_ALG_ENCRYPT);
696 
697 	/* Will write ivsize + cryptlen */
698 	append_math_add(desc, VARSEQOUTLEN, SEQINLEN, REG0, CAAM_CMD_SZ);
699 
700 	/* Not need to reload iv */
701 	append_seq_fifo_load(desc, tfm->ivsize,
702 			     FIFOLD_CLASS_SKIP);
703 
704 	/* Will read cryptlen */
705 	append_math_add(desc, VARSEQINLEN, SEQINLEN, REG0, CAAM_CMD_SZ);
706 	aead_append_src_dst(desc, FIFOLD_TYPE_MSG1OUT2);
707 
708 	/* Write ICV */
709 	append_seq_store(desc, ctx->authsize, LDST_CLASS_2_CCB |
710 			 LDST_SRCDST_BYTE_CONTEXT);
711 
712 	ctx->sh_desc_givenc_dma = dma_map_single(jrdev, desc,
713 						 desc_bytes(desc),
714 						 DMA_TO_DEVICE);
715 	if (dma_mapping_error(jrdev, ctx->sh_desc_givenc_dma)) {
716 		dev_err(jrdev, "unable to map shared descriptor\n");
717 		return -ENOMEM;
718 	}
719 #ifdef DEBUG
720 	print_hex_dump(KERN_ERR, "aead givenc shdesc@"__stringify(__LINE__)": ",
721 		       DUMP_PREFIX_ADDRESS, 16, 4, desc,
722 		       desc_bytes(desc), 1);
723 #endif
724 
725 	return 0;
726 }
727 
728 static int aead_setauthsize(struct crypto_aead *authenc,
729 				    unsigned int authsize)
730 {
731 	struct caam_ctx *ctx = crypto_aead_ctx(authenc);
732 
733 	ctx->authsize = authsize;
734 	aead_set_sh_desc(authenc);
735 
736 	return 0;
737 }
738 
739 static int gcm_set_sh_desc(struct crypto_aead *aead)
740 {
741 	struct aead_tfm *tfm = &aead->base.crt_aead;
742 	struct caam_ctx *ctx = crypto_aead_ctx(aead);
743 	struct device *jrdev = ctx->jrdev;
744 	bool keys_fit_inline = false;
745 	u32 *key_jump_cmd, *zero_payload_jump_cmd,
746 	    *zero_assoc_jump_cmd1, *zero_assoc_jump_cmd2;
747 	u32 *desc;
748 
749 	if (!ctx->enckeylen || !ctx->authsize)
750 		return 0;
751 
752 	/*
753 	 * AES GCM encrypt shared descriptor
754 	 * Job Descriptor and Shared Descriptor
755 	 * must fit into the 64-word Descriptor h/w Buffer
756 	 */
757 	if (DESC_GCM_ENC_LEN + DESC_JOB_IO_LEN +
758 	    ctx->enckeylen <= CAAM_DESC_BYTES_MAX)
759 		keys_fit_inline = true;
760 
761 	desc = ctx->sh_desc_enc;
762 
763 	init_sh_desc(desc, HDR_SHARE_SERIAL);
764 
765 	/* skip key loading if they are loaded due to sharing */
766 	key_jump_cmd = append_jump(desc, JUMP_JSL | JUMP_TEST_ALL |
767 				   JUMP_COND_SHRD | JUMP_COND_SELF);
768 	if (keys_fit_inline)
769 		append_key_as_imm(desc, (void *)ctx->key, ctx->enckeylen,
770 				  ctx->enckeylen, CLASS_1 | KEY_DEST_CLASS_REG);
771 	else
772 		append_key(desc, ctx->key_dma, ctx->enckeylen,
773 			   CLASS_1 | KEY_DEST_CLASS_REG);
774 	set_jump_tgt_here(desc, key_jump_cmd);
775 
776 	/* class 1 operation */
777 	append_operation(desc, ctx->class1_alg_type |
778 			 OP_ALG_AS_INITFINAL | OP_ALG_ENCRYPT);
779 
780 	/* cryptlen = seqoutlen - authsize */
781 	append_math_sub_imm_u32(desc, REG3, SEQOUTLEN, IMM, ctx->authsize);
782 
783 	/* assoclen + cryptlen = seqinlen - ivsize */
784 	append_math_sub_imm_u32(desc, REG2, SEQINLEN, IMM, tfm->ivsize);
785 
786 	/* assoclen = (assoclen + cryptlen) - cryptlen */
787 	append_math_sub(desc, REG1, REG2, REG3, CAAM_CMD_SZ);
788 
789 	/* if cryptlen is ZERO jump to zero-payload commands */
790 	append_math_add(desc, VARSEQOUTLEN, ZERO, REG3, CAAM_CMD_SZ);
791 	zero_payload_jump_cmd = append_jump(desc, JUMP_TEST_ALL |
792 					    JUMP_COND_MATH_Z);
793 	/* read IV */
794 	append_seq_fifo_load(desc, tfm->ivsize, FIFOLD_CLASS_CLASS1 |
795 			     FIFOLD_TYPE_IV | FIFOLD_TYPE_FLUSH1);
796 
797 	/* if assoclen is ZERO, skip reading the assoc data */
798 	append_math_add(desc, VARSEQINLEN, ZERO, REG1, CAAM_CMD_SZ);
799 	zero_assoc_jump_cmd1 = append_jump(desc, JUMP_TEST_ALL |
800 					   JUMP_COND_MATH_Z);
801 
802 	/* read assoc data */
803 	append_seq_fifo_load(desc, 0, FIFOLD_CLASS_CLASS1 | FIFOLDST_VLF |
804 			     FIFOLD_TYPE_AAD | FIFOLD_TYPE_FLUSH1);
805 	set_jump_tgt_here(desc, zero_assoc_jump_cmd1);
806 
807 	append_math_add(desc, VARSEQINLEN, ZERO, REG3, CAAM_CMD_SZ);
808 
809 	/* write encrypted data */
810 	append_seq_fifo_store(desc, 0, FIFOST_TYPE_MESSAGE_DATA | FIFOLDST_VLF);
811 
812 	/* read payload data */
813 	append_seq_fifo_load(desc, 0, FIFOLD_CLASS_CLASS1 | FIFOLDST_VLF |
814 			     FIFOLD_TYPE_MSG | FIFOLD_TYPE_LAST1);
815 
816 	/* jump the zero-payload commands */
817 	append_jump(desc, JUMP_TEST_ALL | 7);
818 
819 	/* zero-payload commands */
820 	set_jump_tgt_here(desc, zero_payload_jump_cmd);
821 
822 	/* if assoclen is ZERO, jump to IV reading - is the only input data */
823 	append_math_add(desc, VARSEQINLEN, ZERO, REG1, CAAM_CMD_SZ);
824 	zero_assoc_jump_cmd2 = append_jump(desc, JUMP_TEST_ALL |
825 					   JUMP_COND_MATH_Z);
826 	/* read IV */
827 	append_seq_fifo_load(desc, tfm->ivsize, FIFOLD_CLASS_CLASS1 |
828 			     FIFOLD_TYPE_IV | FIFOLD_TYPE_FLUSH1);
829 
830 	/* read assoc data */
831 	append_seq_fifo_load(desc, 0, FIFOLD_CLASS_CLASS1 | FIFOLDST_VLF |
832 			     FIFOLD_TYPE_AAD | FIFOLD_TYPE_LAST1);
833 
834 	/* jump to ICV writing */
835 	append_jump(desc, JUMP_TEST_ALL | 2);
836 
837 	/* read IV - is the only input data */
838 	set_jump_tgt_here(desc, zero_assoc_jump_cmd2);
839 	append_seq_fifo_load(desc, tfm->ivsize, FIFOLD_CLASS_CLASS1 |
840 			     FIFOLD_TYPE_IV | FIFOLD_TYPE_FLUSH1 |
841 			     FIFOLD_TYPE_LAST1);
842 
843 	/* write ICV */
844 	append_seq_store(desc, ctx->authsize, LDST_CLASS_1_CCB |
845 			 LDST_SRCDST_BYTE_CONTEXT);
846 
847 	ctx->sh_desc_enc_dma = dma_map_single(jrdev, desc,
848 					      desc_bytes(desc),
849 					      DMA_TO_DEVICE);
850 	if (dma_mapping_error(jrdev, ctx->sh_desc_enc_dma)) {
851 		dev_err(jrdev, "unable to map shared descriptor\n");
852 		return -ENOMEM;
853 	}
854 #ifdef DEBUG
855 	print_hex_dump(KERN_ERR, "gcm enc shdesc@"__stringify(__LINE__)": ",
856 		       DUMP_PREFIX_ADDRESS, 16, 4, desc,
857 		       desc_bytes(desc), 1);
858 #endif
859 
860 	/*
861 	 * Job Descriptor and Shared Descriptors
862 	 * must all fit into the 64-word Descriptor h/w Buffer
863 	 */
864 	keys_fit_inline = false;
865 	if (DESC_GCM_DEC_LEN + DESC_JOB_IO_LEN +
866 	    ctx->enckeylen <= CAAM_DESC_BYTES_MAX)
867 		keys_fit_inline = true;
868 
869 	desc = ctx->sh_desc_dec;
870 
871 	init_sh_desc(desc, HDR_SHARE_SERIAL);
872 
873 	/* skip key loading if they are loaded due to sharing */
874 	key_jump_cmd = append_jump(desc, JUMP_JSL |
875 				   JUMP_TEST_ALL | JUMP_COND_SHRD |
876 				   JUMP_COND_SELF);
877 	if (keys_fit_inline)
878 		append_key_as_imm(desc, (void *)ctx->key, ctx->enckeylen,
879 				  ctx->enckeylen, CLASS_1 | KEY_DEST_CLASS_REG);
880 	else
881 		append_key(desc, ctx->key_dma, ctx->enckeylen,
882 			   CLASS_1 | KEY_DEST_CLASS_REG);
883 	set_jump_tgt_here(desc, key_jump_cmd);
884 
885 	/* class 1 operation */
886 	append_operation(desc, ctx->class1_alg_type |
887 			 OP_ALG_AS_INITFINAL | OP_ALG_DECRYPT | OP_ALG_ICV_ON);
888 
889 	/* assoclen + cryptlen = seqinlen - ivsize - icvsize */
890 	append_math_sub_imm_u32(desc, REG3, SEQINLEN, IMM,
891 				ctx->authsize + tfm->ivsize);
892 
893 	/* assoclen = (assoclen + cryptlen) - cryptlen */
894 	append_math_sub(desc, REG2, SEQOUTLEN, REG0, CAAM_CMD_SZ);
895 	append_math_sub(desc, REG1, REG3, REG2, CAAM_CMD_SZ);
896 
897 	/* read IV */
898 	append_seq_fifo_load(desc, tfm->ivsize, FIFOLD_CLASS_CLASS1 |
899 			     FIFOLD_TYPE_IV | FIFOLD_TYPE_FLUSH1);
900 
901 	/* jump to zero-payload command if cryptlen is zero */
902 	append_math_add(desc, VARSEQOUTLEN, ZERO, REG2, CAAM_CMD_SZ);
903 	zero_payload_jump_cmd = append_jump(desc, JUMP_TEST_ALL |
904 					    JUMP_COND_MATH_Z);
905 
906 	append_math_add(desc, VARSEQINLEN, ZERO, REG1, CAAM_CMD_SZ);
907 	/* if asoclen is ZERO, skip reading assoc data */
908 	zero_assoc_jump_cmd1 = append_jump(desc, JUMP_TEST_ALL |
909 					   JUMP_COND_MATH_Z);
910 	/* read assoc data */
911 	append_seq_fifo_load(desc, 0, FIFOLD_CLASS_CLASS1 | FIFOLDST_VLF |
912 			     FIFOLD_TYPE_AAD | FIFOLD_TYPE_FLUSH1);
913 	set_jump_tgt_here(desc, zero_assoc_jump_cmd1);
914 
915 	append_math_add(desc, VARSEQINLEN, ZERO, REG2, CAAM_CMD_SZ);
916 
917 	/* store encrypted data */
918 	append_seq_fifo_store(desc, 0, FIFOST_TYPE_MESSAGE_DATA | FIFOLDST_VLF);
919 
920 	/* read payload data */
921 	append_seq_fifo_load(desc, 0, FIFOLD_CLASS_CLASS1 | FIFOLDST_VLF |
922 			     FIFOLD_TYPE_MSG | FIFOLD_TYPE_FLUSH1);
923 
924 	/* jump the zero-payload commands */
925 	append_jump(desc, JUMP_TEST_ALL | 4);
926 
927 	/* zero-payload command */
928 	set_jump_tgt_here(desc, zero_payload_jump_cmd);
929 
930 	/* if assoclen is ZERO, jump to ICV reading */
931 	append_math_add(desc, VARSEQINLEN, ZERO, REG1, CAAM_CMD_SZ);
932 	zero_assoc_jump_cmd2 = append_jump(desc, JUMP_TEST_ALL |
933 					   JUMP_COND_MATH_Z);
934 	/* read assoc data */
935 	append_seq_fifo_load(desc, 0, FIFOLD_CLASS_CLASS1 | FIFOLDST_VLF |
936 			     FIFOLD_TYPE_AAD | FIFOLD_TYPE_FLUSH1);
937 	set_jump_tgt_here(desc, zero_assoc_jump_cmd2);
938 
939 	/* read ICV */
940 	append_seq_fifo_load(desc, ctx->authsize, FIFOLD_CLASS_CLASS1 |
941 			     FIFOLD_TYPE_ICV | FIFOLD_TYPE_LAST1);
942 
943 	ctx->sh_desc_dec_dma = dma_map_single(jrdev, desc,
944 					      desc_bytes(desc),
945 					      DMA_TO_DEVICE);
946 	if (dma_mapping_error(jrdev, ctx->sh_desc_dec_dma)) {
947 		dev_err(jrdev, "unable to map shared descriptor\n");
948 		return -ENOMEM;
949 	}
950 #ifdef DEBUG
951 	print_hex_dump(KERN_ERR, "gcm dec shdesc@"__stringify(__LINE__)": ",
952 		       DUMP_PREFIX_ADDRESS, 16, 4, desc,
953 		       desc_bytes(desc), 1);
954 #endif
955 
956 	return 0;
957 }
958 
959 static int gcm_setauthsize(struct crypto_aead *authenc, unsigned int authsize)
960 {
961 	struct caam_ctx *ctx = crypto_aead_ctx(authenc);
962 
963 	ctx->authsize = authsize;
964 	gcm_set_sh_desc(authenc);
965 
966 	return 0;
967 }
968 
969 static int rfc4106_set_sh_desc(struct crypto_aead *aead)
970 {
971 	struct aead_tfm *tfm = &aead->base.crt_aead;
972 	struct caam_ctx *ctx = crypto_aead_ctx(aead);
973 	struct device *jrdev = ctx->jrdev;
974 	bool keys_fit_inline = false;
975 	u32 *key_jump_cmd, *move_cmd, *write_iv_cmd;
976 	u32 *desc;
977 	u32 geniv;
978 
979 	if (!ctx->enckeylen || !ctx->authsize)
980 		return 0;
981 
982 	/*
983 	 * RFC4106 encrypt shared descriptor
984 	 * Job Descriptor and Shared Descriptor
985 	 * must fit into the 64-word Descriptor h/w Buffer
986 	 */
987 	if (DESC_RFC4106_ENC_LEN + DESC_JOB_IO_LEN +
988 	    ctx->enckeylen <= CAAM_DESC_BYTES_MAX)
989 		keys_fit_inline = true;
990 
991 	desc = ctx->sh_desc_enc;
992 
993 	init_sh_desc(desc, HDR_SHARE_SERIAL);
994 
995 	/* Skip key loading if it is loaded due to sharing */
996 	key_jump_cmd = append_jump(desc, JUMP_JSL | JUMP_TEST_ALL |
997 				   JUMP_COND_SHRD);
998 	if (keys_fit_inline)
999 		append_key_as_imm(desc, (void *)ctx->key, ctx->enckeylen,
1000 				  ctx->enckeylen, CLASS_1 | KEY_DEST_CLASS_REG);
1001 	else
1002 		append_key(desc, ctx->key_dma, ctx->enckeylen,
1003 			   CLASS_1 | KEY_DEST_CLASS_REG);
1004 	set_jump_tgt_here(desc, key_jump_cmd);
1005 
1006 	/* Class 1 operation */
1007 	append_operation(desc, ctx->class1_alg_type |
1008 			 OP_ALG_AS_INITFINAL | OP_ALG_ENCRYPT);
1009 
1010 	/* cryptlen = seqoutlen - authsize */
1011 	append_math_sub_imm_u32(desc, REG3, SEQOUTLEN, IMM, ctx->authsize);
1012 	append_math_add(desc, VARSEQOUTLEN, ZERO, REG3, CAAM_CMD_SZ);
1013 
1014 	/* assoclen + cryptlen = seqinlen - ivsize */
1015 	append_math_sub_imm_u32(desc, REG2, SEQINLEN, IMM, tfm->ivsize);
1016 
1017 	/* assoclen = (assoclen + cryptlen) - cryptlen */
1018 	append_math_sub(desc, VARSEQINLEN, REG2, REG3, CAAM_CMD_SZ);
1019 
1020 	/* Read Salt */
1021 	append_fifo_load_as_imm(desc, (void *)(ctx->key + ctx->enckeylen),
1022 				4, FIFOLD_CLASS_CLASS1 | FIFOLD_TYPE_IV);
1023 	/* Read AES-GCM-ESP IV */
1024 	append_seq_fifo_load(desc, tfm->ivsize, FIFOLD_CLASS_CLASS1 |
1025 			     FIFOLD_TYPE_IV | FIFOLD_TYPE_FLUSH1);
1026 
1027 	/* Read assoc data */
1028 	append_seq_fifo_load(desc, 0, FIFOLD_CLASS_CLASS1 | FIFOLDST_VLF |
1029 			     FIFOLD_TYPE_AAD | FIFOLD_TYPE_FLUSH1);
1030 
1031 	/* Will read cryptlen bytes */
1032 	append_math_add(desc, VARSEQINLEN, ZERO, REG3, CAAM_CMD_SZ);
1033 
1034 	/* Write encrypted data */
1035 	append_seq_fifo_store(desc, 0, FIFOST_TYPE_MESSAGE_DATA | FIFOLDST_VLF);
1036 
1037 	/* Read payload data */
1038 	append_seq_fifo_load(desc, 0, FIFOLD_CLASS_CLASS1 | FIFOLDST_VLF |
1039 			     FIFOLD_TYPE_MSG | FIFOLD_TYPE_LAST1);
1040 
1041 	/* Write ICV */
1042 	append_seq_store(desc, ctx->authsize, LDST_CLASS_1_CCB |
1043 			 LDST_SRCDST_BYTE_CONTEXT);
1044 
1045 	ctx->sh_desc_enc_dma = dma_map_single(jrdev, desc,
1046 					      desc_bytes(desc),
1047 					      DMA_TO_DEVICE);
1048 	if (dma_mapping_error(jrdev, ctx->sh_desc_enc_dma)) {
1049 		dev_err(jrdev, "unable to map shared descriptor\n");
1050 		return -ENOMEM;
1051 	}
1052 #ifdef DEBUG
1053 	print_hex_dump(KERN_ERR, "rfc4106 enc shdesc@"__stringify(__LINE__)": ",
1054 		       DUMP_PREFIX_ADDRESS, 16, 4, desc,
1055 		       desc_bytes(desc), 1);
1056 #endif
1057 
1058 	/*
1059 	 * Job Descriptor and Shared Descriptors
1060 	 * must all fit into the 64-word Descriptor h/w Buffer
1061 	 */
1062 	keys_fit_inline = false;
1063 	if (DESC_RFC4106_DEC_LEN + DESC_JOB_IO_LEN +
1064 	    ctx->enckeylen <= CAAM_DESC_BYTES_MAX)
1065 		keys_fit_inline = true;
1066 
1067 	desc = ctx->sh_desc_dec;
1068 
1069 	init_sh_desc(desc, HDR_SHARE_SERIAL);
1070 
1071 	/* Skip key loading if it is loaded due to sharing */
1072 	key_jump_cmd = append_jump(desc, JUMP_JSL |
1073 				   JUMP_TEST_ALL | JUMP_COND_SHRD);
1074 	if (keys_fit_inline)
1075 		append_key_as_imm(desc, (void *)ctx->key, ctx->enckeylen,
1076 				  ctx->enckeylen, CLASS_1 | KEY_DEST_CLASS_REG);
1077 	else
1078 		append_key(desc, ctx->key_dma, ctx->enckeylen,
1079 			   CLASS_1 | KEY_DEST_CLASS_REG);
1080 	set_jump_tgt_here(desc, key_jump_cmd);
1081 
1082 	/* Class 1 operation */
1083 	append_operation(desc, ctx->class1_alg_type |
1084 			 OP_ALG_AS_INITFINAL | OP_ALG_DECRYPT | OP_ALG_ICV_ON);
1085 
1086 	/* assoclen + cryptlen = seqinlen - ivsize - icvsize */
1087 	append_math_sub_imm_u32(desc, REG3, SEQINLEN, IMM,
1088 				ctx->authsize + tfm->ivsize);
1089 
1090 	/* assoclen = (assoclen + cryptlen) - cryptlen */
1091 	append_math_sub(desc, REG2, SEQOUTLEN, REG0, CAAM_CMD_SZ);
1092 	append_math_sub(desc, VARSEQINLEN, REG3, REG2, CAAM_CMD_SZ);
1093 
1094 	/* Will write cryptlen bytes */
1095 	append_math_sub(desc, VARSEQOUTLEN, SEQOUTLEN, REG0, CAAM_CMD_SZ);
1096 
1097 	/* Read Salt */
1098 	append_fifo_load_as_imm(desc, (void *)(ctx->key + ctx->enckeylen),
1099 				4, FIFOLD_CLASS_CLASS1 | FIFOLD_TYPE_IV);
1100 	/* Read AES-GCM-ESP IV */
1101 	append_seq_fifo_load(desc, tfm->ivsize, FIFOLD_CLASS_CLASS1 |
1102 			     FIFOLD_TYPE_IV | FIFOLD_TYPE_FLUSH1);
1103 
1104 	/* Read assoc data */
1105 	append_seq_fifo_load(desc, 0, FIFOLD_CLASS_CLASS1 | FIFOLDST_VLF |
1106 			     FIFOLD_TYPE_AAD | FIFOLD_TYPE_FLUSH1);
1107 
1108 	/* Will read cryptlen bytes */
1109 	append_math_add(desc, VARSEQINLEN, ZERO, REG2, CAAM_CMD_SZ);
1110 
1111 	/* Store payload data */
1112 	append_seq_fifo_store(desc, 0, FIFOST_TYPE_MESSAGE_DATA | FIFOLDST_VLF);
1113 
1114 	/* Read encrypted data */
1115 	append_seq_fifo_load(desc, 0, FIFOLD_CLASS_CLASS1 | FIFOLDST_VLF |
1116 			     FIFOLD_TYPE_MSG | FIFOLD_TYPE_FLUSH1);
1117 
1118 	/* Read ICV */
1119 	append_seq_fifo_load(desc, ctx->authsize, FIFOLD_CLASS_CLASS1 |
1120 			     FIFOLD_TYPE_ICV | FIFOLD_TYPE_LAST1);
1121 
1122 	ctx->sh_desc_dec_dma = dma_map_single(jrdev, desc,
1123 					      desc_bytes(desc),
1124 					      DMA_TO_DEVICE);
1125 	if (dma_mapping_error(jrdev, ctx->sh_desc_dec_dma)) {
1126 		dev_err(jrdev, "unable to map shared descriptor\n");
1127 		return -ENOMEM;
1128 	}
1129 #ifdef DEBUG
1130 	print_hex_dump(KERN_ERR, "rfc4106 dec shdesc@"__stringify(__LINE__)": ",
1131 		       DUMP_PREFIX_ADDRESS, 16, 4, desc,
1132 		       desc_bytes(desc), 1);
1133 #endif
1134 
1135 	/*
1136 	 * Job Descriptor and Shared Descriptors
1137 	 * must all fit into the 64-word Descriptor h/w Buffer
1138 	 */
1139 	keys_fit_inline = false;
1140 	if (DESC_RFC4106_GIVENC_LEN + DESC_JOB_IO_LEN +
1141 	    ctx->split_key_pad_len + ctx->enckeylen <=
1142 	    CAAM_DESC_BYTES_MAX)
1143 		keys_fit_inline = true;
1144 
1145 	/* rfc4106_givencrypt shared descriptor */
1146 	desc = ctx->sh_desc_givenc;
1147 
1148 	init_sh_desc(desc, HDR_SHARE_SERIAL);
1149 
1150 	/* Skip key loading if it is loaded due to sharing */
1151 	key_jump_cmd = append_jump(desc, JUMP_JSL | JUMP_TEST_ALL |
1152 				   JUMP_COND_SHRD);
1153 	if (keys_fit_inline)
1154 		append_key_as_imm(desc, (void *)ctx->key, ctx->enckeylen,
1155 				  ctx->enckeylen, CLASS_1 | KEY_DEST_CLASS_REG);
1156 	else
1157 		append_key(desc, ctx->key_dma, ctx->enckeylen,
1158 			   CLASS_1 | KEY_DEST_CLASS_REG);
1159 	set_jump_tgt_here(desc, key_jump_cmd);
1160 
1161 	/* Generate IV */
1162 	geniv = NFIFOENTRY_STYPE_PAD | NFIFOENTRY_DEST_DECO |
1163 		NFIFOENTRY_DTYPE_MSG | NFIFOENTRY_LC1 |
1164 		NFIFOENTRY_PTYPE_RND | (tfm->ivsize << NFIFOENTRY_DLEN_SHIFT);
1165 	append_load_imm_u32(desc, geniv, LDST_CLASS_IND_CCB |
1166 			    LDST_SRCDST_WORD_INFO_FIFO | LDST_IMM);
1167 	append_cmd(desc, CMD_LOAD | DISABLE_AUTO_INFO_FIFO);
1168 	move_cmd = append_move(desc, MOVE_SRC_INFIFO | MOVE_DEST_DESCBUF |
1169 			       (tfm->ivsize << MOVE_LEN_SHIFT));
1170 	append_cmd(desc, CMD_LOAD | ENABLE_AUTO_INFO_FIFO);
1171 
1172 	/* Copy generated IV to OFIFO */
1173 	write_iv_cmd = append_move(desc, MOVE_SRC_DESCBUF | MOVE_DEST_OUTFIFO |
1174 				   (tfm->ivsize << MOVE_LEN_SHIFT));
1175 
1176 	/* Class 1 operation */
1177 	append_operation(desc, ctx->class1_alg_type |
1178 			 OP_ALG_AS_INITFINAL | OP_ALG_ENCRYPT);
1179 
1180 	/* ivsize + cryptlen = seqoutlen - authsize */
1181 	append_math_sub_imm_u32(desc, REG3, SEQOUTLEN, IMM, ctx->authsize);
1182 
1183 	/* assoclen = seqinlen - (ivsize + cryptlen) */
1184 	append_math_sub(desc, VARSEQINLEN, SEQINLEN, REG3, CAAM_CMD_SZ);
1185 
1186 	/* Will write ivsize + cryptlen */
1187 	append_math_add(desc, VARSEQOUTLEN, REG3, REG0, CAAM_CMD_SZ);
1188 
1189 	/* Read Salt and generated IV */
1190 	append_cmd(desc, CMD_FIFO_LOAD | FIFOLD_CLASS_CLASS1 | FIFOLD_TYPE_IV |
1191 		   FIFOLD_TYPE_FLUSH1 | IMMEDIATE | 12);
1192 	/* Append Salt */
1193 	append_data(desc, (void *)(ctx->key + ctx->enckeylen), 4);
1194 	set_move_tgt_here(desc, move_cmd);
1195 	set_move_tgt_here(desc, write_iv_cmd);
1196 	/* Blank commands. Will be overwritten by generated IV. */
1197 	append_cmd(desc, 0x00000000);
1198 	append_cmd(desc, 0x00000000);
1199 	/* End of blank commands */
1200 
1201 	/* No need to reload iv */
1202 	append_seq_fifo_load(desc, tfm->ivsize, FIFOLD_CLASS_SKIP);
1203 
1204 	/* Read assoc data */
1205 	append_seq_fifo_load(desc, 0, FIFOLD_CLASS_CLASS1 | FIFOLDST_VLF |
1206 			     FIFOLD_TYPE_AAD | FIFOLD_TYPE_FLUSH1);
1207 
1208 	/* Will read cryptlen */
1209 	append_math_add(desc, VARSEQINLEN, SEQINLEN, REG0, CAAM_CMD_SZ);
1210 
1211 	/* Store generated IV and encrypted data */
1212 	append_seq_fifo_store(desc, 0, FIFOST_TYPE_MESSAGE_DATA | FIFOLDST_VLF);
1213 
1214 	/* Read payload data */
1215 	append_seq_fifo_load(desc, 0, FIFOLD_CLASS_CLASS1 | FIFOLDST_VLF |
1216 			     FIFOLD_TYPE_MSG | FIFOLD_TYPE_LAST1);
1217 
1218 	/* Write ICV */
1219 	append_seq_store(desc, ctx->authsize, LDST_CLASS_1_CCB |
1220 			 LDST_SRCDST_BYTE_CONTEXT);
1221 
1222 	ctx->sh_desc_givenc_dma = dma_map_single(jrdev, desc,
1223 						 desc_bytes(desc),
1224 						 DMA_TO_DEVICE);
1225 	if (dma_mapping_error(jrdev, ctx->sh_desc_givenc_dma)) {
1226 		dev_err(jrdev, "unable to map shared descriptor\n");
1227 		return -ENOMEM;
1228 	}
1229 #ifdef DEBUG
1230 	print_hex_dump(KERN_ERR,
1231 		       "rfc4106 givenc shdesc@"__stringify(__LINE__)": ",
1232 		       DUMP_PREFIX_ADDRESS, 16, 4, desc,
1233 		       desc_bytes(desc), 1);
1234 #endif
1235 
1236 	return 0;
1237 }
1238 
1239 static int rfc4106_setauthsize(struct crypto_aead *authenc,
1240 			       unsigned int authsize)
1241 {
1242 	struct caam_ctx *ctx = crypto_aead_ctx(authenc);
1243 
1244 	ctx->authsize = authsize;
1245 	rfc4106_set_sh_desc(authenc);
1246 
1247 	return 0;
1248 }
1249 
1250 static int rfc4543_set_sh_desc(struct crypto_aead *aead)
1251 {
1252 	struct aead_tfm *tfm = &aead->base.crt_aead;
1253 	struct caam_ctx *ctx = crypto_aead_ctx(aead);
1254 	struct device *jrdev = ctx->jrdev;
1255 	bool keys_fit_inline = false;
1256 	u32 *key_jump_cmd, *write_iv_cmd, *write_aad_cmd;
1257 	u32 *read_move_cmd, *write_move_cmd;
1258 	u32 *desc;
1259 	u32 geniv;
1260 
1261 	if (!ctx->enckeylen || !ctx->authsize)
1262 		return 0;
1263 
1264 	/*
1265 	 * RFC4543 encrypt shared descriptor
1266 	 * Job Descriptor and Shared Descriptor
1267 	 * must fit into the 64-word Descriptor h/w Buffer
1268 	 */
1269 	if (DESC_RFC4543_ENC_LEN + DESC_JOB_IO_LEN +
1270 	    ctx->enckeylen <= CAAM_DESC_BYTES_MAX)
1271 		keys_fit_inline = true;
1272 
1273 	desc = ctx->sh_desc_enc;
1274 
1275 	init_sh_desc(desc, HDR_SHARE_SERIAL);
1276 
1277 	/* Skip key loading if it is loaded due to sharing */
1278 	key_jump_cmd = append_jump(desc, JUMP_JSL | JUMP_TEST_ALL |
1279 				   JUMP_COND_SHRD);
1280 	if (keys_fit_inline)
1281 		append_key_as_imm(desc, (void *)ctx->key, ctx->enckeylen,
1282 				  ctx->enckeylen, CLASS_1 | KEY_DEST_CLASS_REG);
1283 	else
1284 		append_key(desc, ctx->key_dma, ctx->enckeylen,
1285 			   CLASS_1 | KEY_DEST_CLASS_REG);
1286 	set_jump_tgt_here(desc, key_jump_cmd);
1287 
1288 	/* Class 1 operation */
1289 	append_operation(desc, ctx->class1_alg_type |
1290 			 OP_ALG_AS_INITFINAL | OP_ALG_ENCRYPT);
1291 
1292 	/* Load AES-GMAC ESP IV into Math1 register */
1293 	append_cmd(desc, CMD_SEQ_LOAD | LDST_SRCDST_WORD_DECO_MATH1 |
1294 		   LDST_CLASS_DECO | tfm->ivsize);
1295 
1296 	/* Wait the DMA transaction to finish */
1297 	append_jump(desc, JUMP_TEST_ALL | JUMP_COND_CALM |
1298 		    (1 << JUMP_OFFSET_SHIFT));
1299 
1300 	/* Overwrite blank immediate AES-GMAC ESP IV data */
1301 	write_iv_cmd = append_move(desc, MOVE_SRC_MATH1 | MOVE_DEST_DESCBUF |
1302 				   (tfm->ivsize << MOVE_LEN_SHIFT));
1303 
1304 	/* Overwrite blank immediate AAD data */
1305 	write_aad_cmd = append_move(desc, MOVE_SRC_MATH1 | MOVE_DEST_DESCBUF |
1306 				    (tfm->ivsize << MOVE_LEN_SHIFT));
1307 
1308 	/* cryptlen = seqoutlen - authsize */
1309 	append_math_sub_imm_u32(desc, REG3, SEQOUTLEN, IMM, ctx->authsize);
1310 
1311 	/* assoclen = (seqinlen - ivsize) - cryptlen */
1312 	append_math_sub(desc, VARSEQINLEN, SEQINLEN, REG3, CAAM_CMD_SZ);
1313 
1314 	/* Read Salt and AES-GMAC ESP IV */
1315 	append_cmd(desc, CMD_FIFO_LOAD | FIFOLD_CLASS_CLASS1 | IMMEDIATE |
1316 		   FIFOLD_TYPE_IV | FIFOLD_TYPE_FLUSH1 | (4 + tfm->ivsize));
1317 	/* Append Salt */
1318 	append_data(desc, (void *)(ctx->key + ctx->enckeylen), 4);
1319 	set_move_tgt_here(desc, write_iv_cmd);
1320 	/* Blank commands. Will be overwritten by AES-GMAC ESP IV. */
1321 	append_cmd(desc, 0x00000000);
1322 	append_cmd(desc, 0x00000000);
1323 	/* End of blank commands */
1324 
1325 	/* Read assoc data */
1326 	append_seq_fifo_load(desc, 0, FIFOLD_CLASS_CLASS1 | FIFOLDST_VLF |
1327 			     FIFOLD_TYPE_AAD);
1328 
1329 	/* Will read cryptlen bytes */
1330 	append_math_add(desc, VARSEQINLEN, ZERO, REG3, CAAM_CMD_SZ);
1331 
1332 	/* Will write cryptlen bytes */
1333 	append_math_add(desc, VARSEQOUTLEN, ZERO, REG3, CAAM_CMD_SZ);
1334 
1335 	/*
1336 	 * MOVE_LEN opcode is not available in all SEC HW revisions,
1337 	 * thus need to do some magic, i.e. self-patch the descriptor
1338 	 * buffer.
1339 	 */
1340 	read_move_cmd = append_move(desc, MOVE_SRC_DESCBUF | MOVE_DEST_MATH3 |
1341 				    (0x6 << MOVE_LEN_SHIFT));
1342 	write_move_cmd = append_move(desc, MOVE_SRC_MATH3 | MOVE_DEST_DESCBUF |
1343 				     (0x8 << MOVE_LEN_SHIFT));
1344 
1345 	/* Authenticate AES-GMAC ESP IV  */
1346 	append_cmd(desc, CMD_FIFO_LOAD | FIFOLD_CLASS_CLASS1 | IMMEDIATE |
1347 		   FIFOLD_TYPE_AAD | tfm->ivsize);
1348 	set_move_tgt_here(desc, write_aad_cmd);
1349 	/* Blank commands. Will be overwritten by AES-GMAC ESP IV. */
1350 	append_cmd(desc, 0x00000000);
1351 	append_cmd(desc, 0x00000000);
1352 	/* End of blank commands */
1353 
1354 	/* Read and write cryptlen bytes */
1355 	aead_append_src_dst(desc, FIFOLD_TYPE_AAD);
1356 
1357 	set_move_tgt_here(desc, read_move_cmd);
1358 	set_move_tgt_here(desc, write_move_cmd);
1359 	append_cmd(desc, CMD_LOAD | DISABLE_AUTO_INFO_FIFO);
1360 	/* Move payload data to OFIFO */
1361 	append_move(desc, MOVE_SRC_INFIFO_CL | MOVE_DEST_OUTFIFO);
1362 
1363 	/* Write ICV */
1364 	append_seq_store(desc, ctx->authsize, LDST_CLASS_1_CCB |
1365 			 LDST_SRCDST_BYTE_CONTEXT);
1366 
1367 	ctx->sh_desc_enc_dma = dma_map_single(jrdev, desc,
1368 					      desc_bytes(desc),
1369 					      DMA_TO_DEVICE);
1370 	if (dma_mapping_error(jrdev, ctx->sh_desc_enc_dma)) {
1371 		dev_err(jrdev, "unable to map shared descriptor\n");
1372 		return -ENOMEM;
1373 	}
1374 #ifdef DEBUG
1375 	print_hex_dump(KERN_ERR, "rfc4543 enc shdesc@"__stringify(__LINE__)": ",
1376 		       DUMP_PREFIX_ADDRESS, 16, 4, desc,
1377 		       desc_bytes(desc), 1);
1378 #endif
1379 
1380 	/*
1381 	 * Job Descriptor and Shared Descriptors
1382 	 * must all fit into the 64-word Descriptor h/w Buffer
1383 	 */
1384 	keys_fit_inline = false;
1385 	if (DESC_RFC4543_DEC_LEN + DESC_JOB_IO_LEN +
1386 	    ctx->enckeylen <= CAAM_DESC_BYTES_MAX)
1387 		keys_fit_inline = true;
1388 
1389 	desc = ctx->sh_desc_dec;
1390 
1391 	init_sh_desc(desc, HDR_SHARE_SERIAL);
1392 
1393 	/* Skip key loading if it is loaded due to sharing */
1394 	key_jump_cmd = append_jump(desc, JUMP_JSL |
1395 				   JUMP_TEST_ALL | JUMP_COND_SHRD);
1396 	if (keys_fit_inline)
1397 		append_key_as_imm(desc, (void *)ctx->key, ctx->enckeylen,
1398 				  ctx->enckeylen, CLASS_1 | KEY_DEST_CLASS_REG);
1399 	else
1400 		append_key(desc, ctx->key_dma, ctx->enckeylen,
1401 			   CLASS_1 | KEY_DEST_CLASS_REG);
1402 	set_jump_tgt_here(desc, key_jump_cmd);
1403 
1404 	/* Class 1 operation */
1405 	append_operation(desc, ctx->class1_alg_type |
1406 			 OP_ALG_AS_INITFINAL | OP_ALG_DECRYPT | OP_ALG_ICV_ON);
1407 
1408 	/* Load AES-GMAC ESP IV into Math1 register */
1409 	append_cmd(desc, CMD_SEQ_LOAD | LDST_SRCDST_WORD_DECO_MATH1 |
1410 		   LDST_CLASS_DECO | tfm->ivsize);
1411 
1412 	/* Wait the DMA transaction to finish */
1413 	append_jump(desc, JUMP_TEST_ALL | JUMP_COND_CALM |
1414 		    (1 << JUMP_OFFSET_SHIFT));
1415 
1416 	/* assoclen + cryptlen = (seqinlen - ivsize) - icvsize */
1417 	append_math_sub_imm_u32(desc, REG3, SEQINLEN, IMM, ctx->authsize);
1418 
1419 	/* Overwrite blank immediate AES-GMAC ESP IV data */
1420 	write_iv_cmd = append_move(desc, MOVE_SRC_MATH1 | MOVE_DEST_DESCBUF |
1421 				   (tfm->ivsize << MOVE_LEN_SHIFT));
1422 
1423 	/* Overwrite blank immediate AAD data */
1424 	write_aad_cmd = append_move(desc, MOVE_SRC_MATH1 | MOVE_DEST_DESCBUF |
1425 				    (tfm->ivsize << MOVE_LEN_SHIFT));
1426 
1427 	/* assoclen = (assoclen + cryptlen) - cryptlen */
1428 	append_math_sub(desc, REG2, SEQOUTLEN, REG0, CAAM_CMD_SZ);
1429 	append_math_sub(desc, VARSEQINLEN, REG3, REG2, CAAM_CMD_SZ);
1430 
1431 	/*
1432 	 * MOVE_LEN opcode is not available in all SEC HW revisions,
1433 	 * thus need to do some magic, i.e. self-patch the descriptor
1434 	 * buffer.
1435 	 */
1436 	read_move_cmd = append_move(desc, MOVE_SRC_DESCBUF | MOVE_DEST_MATH3 |
1437 				    (0x6 << MOVE_LEN_SHIFT));
1438 	write_move_cmd = append_move(desc, MOVE_SRC_MATH3 | MOVE_DEST_DESCBUF |
1439 				     (0x8 << MOVE_LEN_SHIFT));
1440 
1441 	/* Read Salt and AES-GMAC ESP IV */
1442 	append_cmd(desc, CMD_FIFO_LOAD | FIFOLD_CLASS_CLASS1 | IMMEDIATE |
1443 		   FIFOLD_TYPE_IV | FIFOLD_TYPE_FLUSH1 | (4 + tfm->ivsize));
1444 	/* Append Salt */
1445 	append_data(desc, (void *)(ctx->key + ctx->enckeylen), 4);
1446 	set_move_tgt_here(desc, write_iv_cmd);
1447 	/* Blank commands. Will be overwritten by AES-GMAC ESP IV. */
1448 	append_cmd(desc, 0x00000000);
1449 	append_cmd(desc, 0x00000000);
1450 	/* End of blank commands */
1451 
1452 	/* Read assoc data */
1453 	append_seq_fifo_load(desc, 0, FIFOLD_CLASS_CLASS1 | FIFOLDST_VLF |
1454 			     FIFOLD_TYPE_AAD);
1455 
1456 	/* Will read cryptlen bytes */
1457 	append_math_add(desc, VARSEQINLEN, ZERO, REG2, CAAM_CMD_SZ);
1458 
1459 	/* Will write cryptlen bytes */
1460 	append_math_add(desc, VARSEQOUTLEN, ZERO, REG2, CAAM_CMD_SZ);
1461 
1462 	/* Authenticate AES-GMAC ESP IV  */
1463 	append_cmd(desc, CMD_FIFO_LOAD | FIFOLD_CLASS_CLASS1 | IMMEDIATE |
1464 		   FIFOLD_TYPE_AAD | tfm->ivsize);
1465 	set_move_tgt_here(desc, write_aad_cmd);
1466 	/* Blank commands. Will be overwritten by AES-GMAC ESP IV. */
1467 	append_cmd(desc, 0x00000000);
1468 	append_cmd(desc, 0x00000000);
1469 	/* End of blank commands */
1470 
1471 	/* Store payload data */
1472 	append_seq_fifo_store(desc, 0, FIFOST_TYPE_MESSAGE_DATA | FIFOLDST_VLF);
1473 
1474 	/* In-snoop cryptlen data */
1475 	append_seq_fifo_load(desc, 0, FIFOLD_CLASS_BOTH | FIFOLDST_VLF |
1476 			     FIFOLD_TYPE_AAD | FIFOLD_TYPE_LAST2FLUSH1);
1477 
1478 	set_move_tgt_here(desc, read_move_cmd);
1479 	set_move_tgt_here(desc, write_move_cmd);
1480 	append_cmd(desc, CMD_LOAD | DISABLE_AUTO_INFO_FIFO);
1481 	/* Move payload data to OFIFO */
1482 	append_move(desc, MOVE_SRC_INFIFO_CL | MOVE_DEST_OUTFIFO);
1483 	append_cmd(desc, CMD_LOAD | ENABLE_AUTO_INFO_FIFO);
1484 
1485 	/* Read ICV */
1486 	append_seq_fifo_load(desc, ctx->authsize, FIFOLD_CLASS_CLASS1 |
1487 			     FIFOLD_TYPE_ICV | FIFOLD_TYPE_LAST1);
1488 
1489 	ctx->sh_desc_dec_dma = dma_map_single(jrdev, desc,
1490 					      desc_bytes(desc),
1491 					      DMA_TO_DEVICE);
1492 	if (dma_mapping_error(jrdev, ctx->sh_desc_dec_dma)) {
1493 		dev_err(jrdev, "unable to map shared descriptor\n");
1494 		return -ENOMEM;
1495 	}
1496 #ifdef DEBUG
1497 	print_hex_dump(KERN_ERR, "rfc4543 dec shdesc@"__stringify(__LINE__)": ",
1498 		       DUMP_PREFIX_ADDRESS, 16, 4, desc,
1499 		       desc_bytes(desc), 1);
1500 #endif
1501 
1502 	/*
1503 	 * Job Descriptor and Shared Descriptors
1504 	 * must all fit into the 64-word Descriptor h/w Buffer
1505 	 */
1506 	keys_fit_inline = false;
1507 	if (DESC_RFC4543_GIVENC_LEN + DESC_JOB_IO_LEN +
1508 	    ctx->enckeylen <= CAAM_DESC_BYTES_MAX)
1509 		keys_fit_inline = true;
1510 
1511 	/* rfc4543_givencrypt shared descriptor */
1512 	desc = ctx->sh_desc_givenc;
1513 
1514 	init_sh_desc(desc, HDR_SHARE_SERIAL);
1515 
1516 	/* Skip key loading if it is loaded due to sharing */
1517 	key_jump_cmd = append_jump(desc, JUMP_JSL | JUMP_TEST_ALL |
1518 				   JUMP_COND_SHRD);
1519 	if (keys_fit_inline)
1520 		append_key_as_imm(desc, (void *)ctx->key, ctx->enckeylen,
1521 				  ctx->enckeylen, CLASS_1 | KEY_DEST_CLASS_REG);
1522 	else
1523 		append_key(desc, ctx->key_dma, ctx->enckeylen,
1524 			   CLASS_1 | KEY_DEST_CLASS_REG);
1525 	set_jump_tgt_here(desc, key_jump_cmd);
1526 
1527 	/* Generate IV */
1528 	geniv = NFIFOENTRY_STYPE_PAD | NFIFOENTRY_DEST_DECO |
1529 		NFIFOENTRY_DTYPE_MSG | NFIFOENTRY_LC1 |
1530 		NFIFOENTRY_PTYPE_RND | (tfm->ivsize << NFIFOENTRY_DLEN_SHIFT);
1531 	append_load_imm_u32(desc, geniv, LDST_CLASS_IND_CCB |
1532 			    LDST_SRCDST_WORD_INFO_FIFO | LDST_IMM);
1533 	append_cmd(desc, CMD_LOAD | DISABLE_AUTO_INFO_FIFO);
1534 	/* Move generated IV to Math1 register */
1535 	append_move(desc, MOVE_SRC_INFIFO | MOVE_DEST_MATH1 |
1536 		    (tfm->ivsize << MOVE_LEN_SHIFT));
1537 	append_cmd(desc, CMD_LOAD | ENABLE_AUTO_INFO_FIFO);
1538 
1539 	/* Overwrite blank immediate AES-GMAC IV data */
1540 	write_iv_cmd = append_move(desc, MOVE_SRC_MATH1 | MOVE_DEST_DESCBUF |
1541 				   (tfm->ivsize << MOVE_LEN_SHIFT));
1542 
1543 	/* Overwrite blank immediate AAD data */
1544 	write_aad_cmd = append_move(desc, MOVE_SRC_MATH1 | MOVE_DEST_DESCBUF |
1545 				    (tfm->ivsize << MOVE_LEN_SHIFT));
1546 
1547 	/* Copy generated IV to OFIFO */
1548 	append_move(desc, MOVE_SRC_MATH1 | MOVE_DEST_OUTFIFO |
1549 		    (tfm->ivsize << MOVE_LEN_SHIFT));
1550 
1551 	/* Class 1 operation */
1552 	append_operation(desc, ctx->class1_alg_type |
1553 			 OP_ALG_AS_INITFINAL | OP_ALG_ENCRYPT);
1554 
1555 	/* ivsize + cryptlen = seqoutlen - authsize */
1556 	append_math_sub_imm_u32(desc, REG3, SEQOUTLEN, IMM, ctx->authsize);
1557 
1558 	/* assoclen = seqinlen - (ivsize + cryptlen) */
1559 	append_math_sub(desc, VARSEQINLEN, SEQINLEN, REG3, CAAM_CMD_SZ);
1560 
1561 	/* Will write ivsize + cryptlen */
1562 	append_math_add(desc, VARSEQOUTLEN, REG3, REG0, CAAM_CMD_SZ);
1563 
1564 	/*
1565 	 * MOVE_LEN opcode is not available in all SEC HW revisions,
1566 	 * thus need to do some magic, i.e. self-patch the descriptor
1567 	 * buffer.
1568 	 */
1569 	read_move_cmd = append_move(desc, MOVE_SRC_DESCBUF | MOVE_DEST_MATH3 |
1570 				    (0x6 << MOVE_LEN_SHIFT));
1571 	write_move_cmd = append_move(desc, MOVE_SRC_MATH3 | MOVE_DEST_DESCBUF |
1572 				     (0x8 << MOVE_LEN_SHIFT));
1573 
1574 	/* Read Salt and AES-GMAC generated IV */
1575 	append_cmd(desc, CMD_FIFO_LOAD | FIFOLD_CLASS_CLASS1 | IMMEDIATE |
1576 		   FIFOLD_TYPE_IV | FIFOLD_TYPE_FLUSH1 | (4 + tfm->ivsize));
1577 	/* Append Salt */
1578 	append_data(desc, (void *)(ctx->key + ctx->enckeylen), 4);
1579 	set_move_tgt_here(desc, write_iv_cmd);
1580 	/* Blank commands. Will be overwritten by AES-GMAC generated IV. */
1581 	append_cmd(desc, 0x00000000);
1582 	append_cmd(desc, 0x00000000);
1583 	/* End of blank commands */
1584 
1585 	/* No need to reload iv */
1586 	append_seq_fifo_load(desc, tfm->ivsize, FIFOLD_CLASS_SKIP);
1587 
1588 	/* Read assoc data */
1589 	append_seq_fifo_load(desc, 0, FIFOLD_CLASS_CLASS1 | FIFOLDST_VLF |
1590 			     FIFOLD_TYPE_AAD);
1591 
1592 	/* Will read cryptlen */
1593 	append_math_add(desc, VARSEQINLEN, SEQINLEN, REG0, CAAM_CMD_SZ);
1594 
1595 	/* Authenticate AES-GMAC IV  */
1596 	append_cmd(desc, CMD_FIFO_LOAD | FIFOLD_CLASS_CLASS1 | IMMEDIATE |
1597 		   FIFOLD_TYPE_AAD | tfm->ivsize);
1598 	set_move_tgt_here(desc, write_aad_cmd);
1599 	/* Blank commands. Will be overwritten by AES-GMAC IV. */
1600 	append_cmd(desc, 0x00000000);
1601 	append_cmd(desc, 0x00000000);
1602 	/* End of blank commands */
1603 
1604 	/* Read and write cryptlen bytes */
1605 	aead_append_src_dst(desc, FIFOLD_TYPE_AAD);
1606 
1607 	set_move_tgt_here(desc, read_move_cmd);
1608 	set_move_tgt_here(desc, write_move_cmd);
1609 	append_cmd(desc, CMD_LOAD | DISABLE_AUTO_INFO_FIFO);
1610 	/* Move payload data to OFIFO */
1611 	append_move(desc, MOVE_SRC_INFIFO_CL | MOVE_DEST_OUTFIFO);
1612 
1613 	/* Write ICV */
1614 	append_seq_store(desc, ctx->authsize, LDST_CLASS_1_CCB |
1615 			 LDST_SRCDST_BYTE_CONTEXT);
1616 
1617 	ctx->sh_desc_givenc_dma = dma_map_single(jrdev, desc,
1618 						 desc_bytes(desc),
1619 						 DMA_TO_DEVICE);
1620 	if (dma_mapping_error(jrdev, ctx->sh_desc_givenc_dma)) {
1621 		dev_err(jrdev, "unable to map shared descriptor\n");
1622 		return -ENOMEM;
1623 	}
1624 #ifdef DEBUG
1625 	print_hex_dump(KERN_ERR,
1626 		       "rfc4543 givenc shdesc@"__stringify(__LINE__)": ",
1627 		       DUMP_PREFIX_ADDRESS, 16, 4, desc,
1628 		       desc_bytes(desc), 1);
1629 #endif
1630 
1631 	return 0;
1632 }
1633 
1634 static int rfc4543_setauthsize(struct crypto_aead *authenc,
1635 			       unsigned int authsize)
1636 {
1637 	struct caam_ctx *ctx = crypto_aead_ctx(authenc);
1638 
1639 	ctx->authsize = authsize;
1640 	rfc4543_set_sh_desc(authenc);
1641 
1642 	return 0;
1643 }
1644 
1645 static u32 gen_split_aead_key(struct caam_ctx *ctx, const u8 *key_in,
1646 			      u32 authkeylen)
1647 {
1648 	return gen_split_key(ctx->jrdev, ctx->key, ctx->split_key_len,
1649 			       ctx->split_key_pad_len, key_in, authkeylen,
1650 			       ctx->alg_op);
1651 }
1652 
1653 static int aead_setkey(struct crypto_aead *aead,
1654 			       const u8 *key, unsigned int keylen)
1655 {
1656 	/* Sizes for MDHA pads (*not* keys): MD5, SHA1, 224, 256, 384, 512 */
1657 	static const u8 mdpadlen[] = { 16, 20, 32, 32, 64, 64 };
1658 	struct caam_ctx *ctx = crypto_aead_ctx(aead);
1659 	struct device *jrdev = ctx->jrdev;
1660 	struct crypto_authenc_keys keys;
1661 	int ret = 0;
1662 
1663 	if (crypto_authenc_extractkeys(&keys, key, keylen) != 0)
1664 		goto badkey;
1665 
1666 	/* Pick class 2 key length from algorithm submask */
1667 	ctx->split_key_len = mdpadlen[(ctx->alg_op & OP_ALG_ALGSEL_SUBMASK) >>
1668 				      OP_ALG_ALGSEL_SHIFT] * 2;
1669 	ctx->split_key_pad_len = ALIGN(ctx->split_key_len, 16);
1670 
1671 	if (ctx->split_key_pad_len + keys.enckeylen > CAAM_MAX_KEY_SIZE)
1672 		goto badkey;
1673 
1674 #ifdef DEBUG
1675 	printk(KERN_ERR "keylen %d enckeylen %d authkeylen %d\n",
1676 	       keys.authkeylen + keys.enckeylen, keys.enckeylen,
1677 	       keys.authkeylen);
1678 	printk(KERN_ERR "split_key_len %d split_key_pad_len %d\n",
1679 	       ctx->split_key_len, ctx->split_key_pad_len);
1680 	print_hex_dump(KERN_ERR, "key in @"__stringify(__LINE__)": ",
1681 		       DUMP_PREFIX_ADDRESS, 16, 4, key, keylen, 1);
1682 #endif
1683 
1684 	ret = gen_split_aead_key(ctx, keys.authkey, keys.authkeylen);
1685 	if (ret) {
1686 		goto badkey;
1687 	}
1688 
1689 	/* postpend encryption key to auth split key */
1690 	memcpy(ctx->key + ctx->split_key_pad_len, keys.enckey, keys.enckeylen);
1691 
1692 	ctx->key_dma = dma_map_single(jrdev, ctx->key, ctx->split_key_pad_len +
1693 				      keys.enckeylen, DMA_TO_DEVICE);
1694 	if (dma_mapping_error(jrdev, ctx->key_dma)) {
1695 		dev_err(jrdev, "unable to map key i/o memory\n");
1696 		return -ENOMEM;
1697 	}
1698 #ifdef DEBUG
1699 	print_hex_dump(KERN_ERR, "ctx.key@"__stringify(__LINE__)": ",
1700 		       DUMP_PREFIX_ADDRESS, 16, 4, ctx->key,
1701 		       ctx->split_key_pad_len + keys.enckeylen, 1);
1702 #endif
1703 
1704 	ctx->enckeylen = keys.enckeylen;
1705 
1706 	ret = aead_set_sh_desc(aead);
1707 	if (ret) {
1708 		dma_unmap_single(jrdev, ctx->key_dma, ctx->split_key_pad_len +
1709 				 keys.enckeylen, DMA_TO_DEVICE);
1710 	}
1711 
1712 	return ret;
1713 badkey:
1714 	crypto_aead_set_flags(aead, CRYPTO_TFM_RES_BAD_KEY_LEN);
1715 	return -EINVAL;
1716 }
1717 
1718 static int gcm_setkey(struct crypto_aead *aead,
1719 		      const u8 *key, unsigned int keylen)
1720 {
1721 	struct caam_ctx *ctx = crypto_aead_ctx(aead);
1722 	struct device *jrdev = ctx->jrdev;
1723 	int ret = 0;
1724 
1725 #ifdef DEBUG
1726 	print_hex_dump(KERN_ERR, "key in @"__stringify(__LINE__)": ",
1727 		       DUMP_PREFIX_ADDRESS, 16, 4, key, keylen, 1);
1728 #endif
1729 
1730 	memcpy(ctx->key, key, keylen);
1731 	ctx->key_dma = dma_map_single(jrdev, ctx->key, keylen,
1732 				      DMA_TO_DEVICE);
1733 	if (dma_mapping_error(jrdev, ctx->key_dma)) {
1734 		dev_err(jrdev, "unable to map key i/o memory\n");
1735 		return -ENOMEM;
1736 	}
1737 	ctx->enckeylen = keylen;
1738 
1739 	ret = gcm_set_sh_desc(aead);
1740 	if (ret) {
1741 		dma_unmap_single(jrdev, ctx->key_dma, ctx->enckeylen,
1742 				 DMA_TO_DEVICE);
1743 	}
1744 
1745 	return ret;
1746 }
1747 
1748 static int rfc4106_setkey(struct crypto_aead *aead,
1749 			  const u8 *key, unsigned int keylen)
1750 {
1751 	struct caam_ctx *ctx = crypto_aead_ctx(aead);
1752 	struct device *jrdev = ctx->jrdev;
1753 	int ret = 0;
1754 
1755 	if (keylen < 4)
1756 		return -EINVAL;
1757 
1758 #ifdef DEBUG
1759 	print_hex_dump(KERN_ERR, "key in @"__stringify(__LINE__)": ",
1760 		       DUMP_PREFIX_ADDRESS, 16, 4, key, keylen, 1);
1761 #endif
1762 
1763 	memcpy(ctx->key, key, keylen);
1764 
1765 	/*
1766 	 * The last four bytes of the key material are used as the salt value
1767 	 * in the nonce. Update the AES key length.
1768 	 */
1769 	ctx->enckeylen = keylen - 4;
1770 
1771 	ctx->key_dma = dma_map_single(jrdev, ctx->key, ctx->enckeylen,
1772 				      DMA_TO_DEVICE);
1773 	if (dma_mapping_error(jrdev, ctx->key_dma)) {
1774 		dev_err(jrdev, "unable to map key i/o memory\n");
1775 		return -ENOMEM;
1776 	}
1777 
1778 	ret = rfc4106_set_sh_desc(aead);
1779 	if (ret) {
1780 		dma_unmap_single(jrdev, ctx->key_dma, ctx->enckeylen,
1781 				 DMA_TO_DEVICE);
1782 	}
1783 
1784 	return ret;
1785 }
1786 
1787 static int rfc4543_setkey(struct crypto_aead *aead,
1788 			  const u8 *key, unsigned int keylen)
1789 {
1790 	struct caam_ctx *ctx = crypto_aead_ctx(aead);
1791 	struct device *jrdev = ctx->jrdev;
1792 	int ret = 0;
1793 
1794 	if (keylen < 4)
1795 		return -EINVAL;
1796 
1797 #ifdef DEBUG
1798 	print_hex_dump(KERN_ERR, "key in @"__stringify(__LINE__)": ",
1799 		       DUMP_PREFIX_ADDRESS, 16, 4, key, keylen, 1);
1800 #endif
1801 
1802 	memcpy(ctx->key, key, keylen);
1803 
1804 	/*
1805 	 * The last four bytes of the key material are used as the salt value
1806 	 * in the nonce. Update the AES key length.
1807 	 */
1808 	ctx->enckeylen = keylen - 4;
1809 
1810 	ctx->key_dma = dma_map_single(jrdev, ctx->key, ctx->enckeylen,
1811 				      DMA_TO_DEVICE);
1812 	if (dma_mapping_error(jrdev, ctx->key_dma)) {
1813 		dev_err(jrdev, "unable to map key i/o memory\n");
1814 		return -ENOMEM;
1815 	}
1816 
1817 	ret = rfc4543_set_sh_desc(aead);
1818 	if (ret) {
1819 		dma_unmap_single(jrdev, ctx->key_dma, ctx->enckeylen,
1820 				 DMA_TO_DEVICE);
1821 	}
1822 
1823 	return ret;
1824 }
1825 
1826 static int ablkcipher_setkey(struct crypto_ablkcipher *ablkcipher,
1827 			     const u8 *key, unsigned int keylen)
1828 {
1829 	struct caam_ctx *ctx = crypto_ablkcipher_ctx(ablkcipher);
1830 	struct ablkcipher_tfm *crt = &ablkcipher->base.crt_ablkcipher;
1831 	struct crypto_tfm *tfm = crypto_ablkcipher_tfm(ablkcipher);
1832 	const char *alg_name = crypto_tfm_alg_name(tfm);
1833 	struct device *jrdev = ctx->jrdev;
1834 	int ret = 0;
1835 	u32 *key_jump_cmd;
1836 	u32 *desc;
1837 	u32 *nonce;
1838 	u32 geniv;
1839 	u32 ctx1_iv_off = 0;
1840 	const bool ctr_mode = ((ctx->class1_alg_type & OP_ALG_AAI_MASK) ==
1841 			       OP_ALG_AAI_CTR_MOD128);
1842 	const bool is_rfc3686 = (ctr_mode &&
1843 				 (strstr(alg_name, "rfc3686") != NULL));
1844 
1845 #ifdef DEBUG
1846 	print_hex_dump(KERN_ERR, "key in @"__stringify(__LINE__)": ",
1847 		       DUMP_PREFIX_ADDRESS, 16, 4, key, keylen, 1);
1848 #endif
1849 	/*
1850 	 * AES-CTR needs to load IV in CONTEXT1 reg
1851 	 * at an offset of 128bits (16bytes)
1852 	 * CONTEXT1[255:128] = IV
1853 	 */
1854 	if (ctr_mode)
1855 		ctx1_iv_off = 16;
1856 
1857 	/*
1858 	 * RFC3686 specific:
1859 	 *	| CONTEXT1[255:128] = {NONCE, IV, COUNTER}
1860 	 *	| *key = {KEY, NONCE}
1861 	 */
1862 	if (is_rfc3686) {
1863 		ctx1_iv_off = 16 + CTR_RFC3686_NONCE_SIZE;
1864 		keylen -= CTR_RFC3686_NONCE_SIZE;
1865 	}
1866 
1867 	memcpy(ctx->key, key, keylen);
1868 	ctx->key_dma = dma_map_single(jrdev, ctx->key, keylen,
1869 				      DMA_TO_DEVICE);
1870 	if (dma_mapping_error(jrdev, ctx->key_dma)) {
1871 		dev_err(jrdev, "unable to map key i/o memory\n");
1872 		return -ENOMEM;
1873 	}
1874 	ctx->enckeylen = keylen;
1875 
1876 	/* ablkcipher_encrypt shared descriptor */
1877 	desc = ctx->sh_desc_enc;
1878 	init_sh_desc(desc, HDR_SHARE_SERIAL | HDR_SAVECTX);
1879 	/* Skip if already shared */
1880 	key_jump_cmd = append_jump(desc, JUMP_JSL | JUMP_TEST_ALL |
1881 				   JUMP_COND_SHRD);
1882 
1883 	/* Load class1 key only */
1884 	append_key_as_imm(desc, (void *)ctx->key, ctx->enckeylen,
1885 			  ctx->enckeylen, CLASS_1 |
1886 			  KEY_DEST_CLASS_REG);
1887 
1888 	/* Load nonce into CONTEXT1 reg */
1889 	if (is_rfc3686) {
1890 		nonce = (u32 *)(key + keylen);
1891 		append_load_imm_u32(desc, *nonce, LDST_CLASS_IND_CCB |
1892 				    LDST_SRCDST_BYTE_OUTFIFO | LDST_IMM);
1893 		append_move(desc, MOVE_WAITCOMP |
1894 			    MOVE_SRC_OUTFIFO |
1895 			    MOVE_DEST_CLASS1CTX |
1896 			    (16 << MOVE_OFFSET_SHIFT) |
1897 			    (CTR_RFC3686_NONCE_SIZE << MOVE_LEN_SHIFT));
1898 	}
1899 
1900 	set_jump_tgt_here(desc, key_jump_cmd);
1901 
1902 	/* Load iv */
1903 	append_seq_load(desc, crt->ivsize, LDST_SRCDST_BYTE_CONTEXT |
1904 			LDST_CLASS_1_CCB | (ctx1_iv_off << LDST_OFFSET_SHIFT));
1905 
1906 	/* Load counter into CONTEXT1 reg */
1907 	if (is_rfc3686)
1908 		append_load_imm_u32(desc, be32_to_cpu(1), LDST_IMM |
1909 				    LDST_CLASS_1_CCB |
1910 				    LDST_SRCDST_BYTE_CONTEXT |
1911 				    ((ctx1_iv_off + CTR_RFC3686_IV_SIZE) <<
1912 				     LDST_OFFSET_SHIFT));
1913 
1914 	/* Load operation */
1915 	append_operation(desc, ctx->class1_alg_type |
1916 			 OP_ALG_AS_INITFINAL | OP_ALG_ENCRYPT);
1917 
1918 	/* Perform operation */
1919 	ablkcipher_append_src_dst(desc);
1920 
1921 	ctx->sh_desc_enc_dma = dma_map_single(jrdev, desc,
1922 					      desc_bytes(desc),
1923 					      DMA_TO_DEVICE);
1924 	if (dma_mapping_error(jrdev, ctx->sh_desc_enc_dma)) {
1925 		dev_err(jrdev, "unable to map shared descriptor\n");
1926 		return -ENOMEM;
1927 	}
1928 #ifdef DEBUG
1929 	print_hex_dump(KERN_ERR,
1930 		       "ablkcipher enc shdesc@"__stringify(__LINE__)": ",
1931 		       DUMP_PREFIX_ADDRESS, 16, 4, desc,
1932 		       desc_bytes(desc), 1);
1933 #endif
1934 	/* ablkcipher_decrypt shared descriptor */
1935 	desc = ctx->sh_desc_dec;
1936 
1937 	init_sh_desc(desc, HDR_SHARE_SERIAL | HDR_SAVECTX);
1938 	/* Skip if already shared */
1939 	key_jump_cmd = append_jump(desc, JUMP_JSL | JUMP_TEST_ALL |
1940 				   JUMP_COND_SHRD);
1941 
1942 	/* Load class1 key only */
1943 	append_key_as_imm(desc, (void *)ctx->key, ctx->enckeylen,
1944 			  ctx->enckeylen, CLASS_1 |
1945 			  KEY_DEST_CLASS_REG);
1946 
1947 	/* Load nonce into CONTEXT1 reg */
1948 	if (is_rfc3686) {
1949 		nonce = (u32 *)(key + keylen);
1950 		append_load_imm_u32(desc, *nonce, LDST_CLASS_IND_CCB |
1951 				    LDST_SRCDST_BYTE_OUTFIFO | LDST_IMM);
1952 		append_move(desc, MOVE_WAITCOMP |
1953 			    MOVE_SRC_OUTFIFO |
1954 			    MOVE_DEST_CLASS1CTX |
1955 			    (16 << MOVE_OFFSET_SHIFT) |
1956 			    (CTR_RFC3686_NONCE_SIZE << MOVE_LEN_SHIFT));
1957 	}
1958 
1959 	set_jump_tgt_here(desc, key_jump_cmd);
1960 
1961 	/* load IV */
1962 	append_seq_load(desc, crt->ivsize, LDST_SRCDST_BYTE_CONTEXT |
1963 			LDST_CLASS_1_CCB | (ctx1_iv_off << LDST_OFFSET_SHIFT));
1964 
1965 	/* Load counter into CONTEXT1 reg */
1966 	if (is_rfc3686)
1967 		append_load_imm_u32(desc, be32_to_cpu(1), LDST_IMM |
1968 				    LDST_CLASS_1_CCB |
1969 				    LDST_SRCDST_BYTE_CONTEXT |
1970 				    ((ctx1_iv_off + CTR_RFC3686_IV_SIZE) <<
1971 				     LDST_OFFSET_SHIFT));
1972 
1973 	/* Choose operation */
1974 	if (ctr_mode)
1975 		append_operation(desc, ctx->class1_alg_type |
1976 				 OP_ALG_AS_INITFINAL | OP_ALG_DECRYPT);
1977 	else
1978 		append_dec_op1(desc, ctx->class1_alg_type);
1979 
1980 	/* Perform operation */
1981 	ablkcipher_append_src_dst(desc);
1982 
1983 	ctx->sh_desc_dec_dma = dma_map_single(jrdev, desc,
1984 					      desc_bytes(desc),
1985 					      DMA_TO_DEVICE);
1986 	if (dma_mapping_error(jrdev, ctx->sh_desc_dec_dma)) {
1987 		dev_err(jrdev, "unable to map shared descriptor\n");
1988 		return -ENOMEM;
1989 	}
1990 
1991 #ifdef DEBUG
1992 	print_hex_dump(KERN_ERR,
1993 		       "ablkcipher dec shdesc@"__stringify(__LINE__)": ",
1994 		       DUMP_PREFIX_ADDRESS, 16, 4, desc,
1995 		       desc_bytes(desc), 1);
1996 #endif
1997 	/* ablkcipher_givencrypt shared descriptor */
1998 	desc = ctx->sh_desc_givenc;
1999 
2000 	init_sh_desc(desc, HDR_SHARE_SERIAL | HDR_SAVECTX);
2001 	/* Skip if already shared */
2002 	key_jump_cmd = append_jump(desc, JUMP_JSL | JUMP_TEST_ALL |
2003 				   JUMP_COND_SHRD);
2004 
2005 	/* Load class1 key only */
2006 	append_key_as_imm(desc, (void *)ctx->key, ctx->enckeylen,
2007 			  ctx->enckeylen, CLASS_1 |
2008 			  KEY_DEST_CLASS_REG);
2009 
2010 	/* Load Nonce into CONTEXT1 reg */
2011 	if (is_rfc3686) {
2012 		nonce = (u32 *)(key + keylen);
2013 		append_load_imm_u32(desc, *nonce, LDST_CLASS_IND_CCB |
2014 				    LDST_SRCDST_BYTE_OUTFIFO | LDST_IMM);
2015 		append_move(desc, MOVE_WAITCOMP |
2016 			    MOVE_SRC_OUTFIFO |
2017 			    MOVE_DEST_CLASS1CTX |
2018 			    (16 << MOVE_OFFSET_SHIFT) |
2019 			    (CTR_RFC3686_NONCE_SIZE << MOVE_LEN_SHIFT));
2020 	}
2021 	set_jump_tgt_here(desc, key_jump_cmd);
2022 
2023 	/* Generate IV */
2024 	geniv = NFIFOENTRY_STYPE_PAD | NFIFOENTRY_DEST_DECO |
2025 		NFIFOENTRY_DTYPE_MSG | NFIFOENTRY_LC1 |
2026 		NFIFOENTRY_PTYPE_RND | (crt->ivsize << NFIFOENTRY_DLEN_SHIFT);
2027 	append_load_imm_u32(desc, geniv, LDST_CLASS_IND_CCB |
2028 			    LDST_SRCDST_WORD_INFO_FIFO | LDST_IMM);
2029 	append_cmd(desc, CMD_LOAD | DISABLE_AUTO_INFO_FIFO);
2030 	append_move(desc, MOVE_WAITCOMP |
2031 		    MOVE_SRC_INFIFO |
2032 		    MOVE_DEST_CLASS1CTX |
2033 		    (crt->ivsize << MOVE_LEN_SHIFT) |
2034 		    (ctx1_iv_off << MOVE_OFFSET_SHIFT));
2035 	append_cmd(desc, CMD_LOAD | ENABLE_AUTO_INFO_FIFO);
2036 
2037 	/* Copy generated IV to memory */
2038 	append_seq_store(desc, crt->ivsize,
2039 			 LDST_SRCDST_BYTE_CONTEXT | LDST_CLASS_1_CCB |
2040 			 (ctx1_iv_off << LDST_OFFSET_SHIFT));
2041 
2042 	/* Load Counter into CONTEXT1 reg */
2043 	if (is_rfc3686)
2044 		append_load_imm_u32(desc, (u32)1, LDST_IMM |
2045 				    LDST_CLASS_1_CCB |
2046 				    LDST_SRCDST_BYTE_CONTEXT |
2047 				    ((ctx1_iv_off + CTR_RFC3686_IV_SIZE) <<
2048 				     LDST_OFFSET_SHIFT));
2049 
2050 	if (ctx1_iv_off)
2051 		append_jump(desc, JUMP_JSL | JUMP_TEST_ALL | JUMP_COND_NCP |
2052 			    (1 << JUMP_OFFSET_SHIFT));
2053 
2054 	/* Load operation */
2055 	append_operation(desc, ctx->class1_alg_type |
2056 			 OP_ALG_AS_INITFINAL | OP_ALG_ENCRYPT);
2057 
2058 	/* Perform operation */
2059 	ablkcipher_append_src_dst(desc);
2060 
2061 	ctx->sh_desc_givenc_dma = dma_map_single(jrdev, desc,
2062 						 desc_bytes(desc),
2063 						 DMA_TO_DEVICE);
2064 	if (dma_mapping_error(jrdev, ctx->sh_desc_givenc_dma)) {
2065 		dev_err(jrdev, "unable to map shared descriptor\n");
2066 		return -ENOMEM;
2067 	}
2068 #ifdef DEBUG
2069 	print_hex_dump(KERN_ERR,
2070 		       "ablkcipher givenc shdesc@" __stringify(__LINE__) ": ",
2071 		       DUMP_PREFIX_ADDRESS, 16, 4, desc,
2072 		       desc_bytes(desc), 1);
2073 #endif
2074 
2075 	return ret;
2076 }
2077 
2078 /*
2079  * aead_edesc - s/w-extended aead descriptor
2080  * @assoc_nents: number of segments in associated data (SPI+Seq) scatterlist
2081  * @assoc_chained: if source is chained
2082  * @src_nents: number of segments in input scatterlist
2083  * @src_chained: if source is chained
2084  * @dst_nents: number of segments in output scatterlist
2085  * @dst_chained: if destination is chained
2086  * @iv_dma: dma address of iv for checking continuity and link table
2087  * @desc: h/w descriptor (variable length; must not exceed MAX_CAAM_DESCSIZE)
2088  * @sec4_sg_bytes: length of dma mapped sec4_sg space
2089  * @sec4_sg_dma: bus physical mapped address of h/w link table
2090  * @hw_desc: the h/w job descriptor followed by any referenced link tables
2091  */
2092 struct aead_edesc {
2093 	int assoc_nents;
2094 	bool assoc_chained;
2095 	int src_nents;
2096 	bool src_chained;
2097 	int dst_nents;
2098 	bool dst_chained;
2099 	dma_addr_t iv_dma;
2100 	int sec4_sg_bytes;
2101 	dma_addr_t sec4_sg_dma;
2102 	struct sec4_sg_entry *sec4_sg;
2103 	u32 hw_desc[0];
2104 };
2105 
2106 /*
2107  * ablkcipher_edesc - s/w-extended ablkcipher descriptor
2108  * @src_nents: number of segments in input scatterlist
2109  * @src_chained: if source is chained
2110  * @dst_nents: number of segments in output scatterlist
2111  * @dst_chained: if destination is chained
2112  * @iv_dma: dma address of iv for checking continuity and link table
2113  * @desc: h/w descriptor (variable length; must not exceed MAX_CAAM_DESCSIZE)
2114  * @sec4_sg_bytes: length of dma mapped sec4_sg space
2115  * @sec4_sg_dma: bus physical mapped address of h/w link table
2116  * @hw_desc: the h/w job descriptor followed by any referenced link tables
2117  */
2118 struct ablkcipher_edesc {
2119 	int src_nents;
2120 	bool src_chained;
2121 	int dst_nents;
2122 	bool dst_chained;
2123 	dma_addr_t iv_dma;
2124 	int sec4_sg_bytes;
2125 	dma_addr_t sec4_sg_dma;
2126 	struct sec4_sg_entry *sec4_sg;
2127 	u32 hw_desc[0];
2128 };
2129 
2130 static void caam_unmap(struct device *dev, struct scatterlist *src,
2131 		       struct scatterlist *dst, int src_nents,
2132 		       bool src_chained, int dst_nents, bool dst_chained,
2133 		       dma_addr_t iv_dma, int ivsize, dma_addr_t sec4_sg_dma,
2134 		       int sec4_sg_bytes)
2135 {
2136 	if (dst != src) {
2137 		dma_unmap_sg_chained(dev, src, src_nents ? : 1, DMA_TO_DEVICE,
2138 				     src_chained);
2139 		dma_unmap_sg_chained(dev, dst, dst_nents ? : 1, DMA_FROM_DEVICE,
2140 				     dst_chained);
2141 	} else {
2142 		dma_unmap_sg_chained(dev, src, src_nents ? : 1,
2143 				     DMA_BIDIRECTIONAL, src_chained);
2144 	}
2145 
2146 	if (iv_dma)
2147 		dma_unmap_single(dev, iv_dma, ivsize, DMA_TO_DEVICE);
2148 	if (sec4_sg_bytes)
2149 		dma_unmap_single(dev, sec4_sg_dma, sec4_sg_bytes,
2150 				 DMA_TO_DEVICE);
2151 }
2152 
2153 static void aead_unmap(struct device *dev,
2154 		       struct aead_edesc *edesc,
2155 		       struct aead_request *req)
2156 {
2157 	struct crypto_aead *aead = crypto_aead_reqtfm(req);
2158 	int ivsize = crypto_aead_ivsize(aead);
2159 
2160 	dma_unmap_sg_chained(dev, req->assoc, edesc->assoc_nents,
2161 			     DMA_TO_DEVICE, edesc->assoc_chained);
2162 
2163 	caam_unmap(dev, req->src, req->dst,
2164 		   edesc->src_nents, edesc->src_chained, edesc->dst_nents,
2165 		   edesc->dst_chained, edesc->iv_dma, ivsize,
2166 		   edesc->sec4_sg_dma, edesc->sec4_sg_bytes);
2167 }
2168 
2169 static void ablkcipher_unmap(struct device *dev,
2170 			     struct ablkcipher_edesc *edesc,
2171 			     struct ablkcipher_request *req)
2172 {
2173 	struct crypto_ablkcipher *ablkcipher = crypto_ablkcipher_reqtfm(req);
2174 	int ivsize = crypto_ablkcipher_ivsize(ablkcipher);
2175 
2176 	caam_unmap(dev, req->src, req->dst,
2177 		   edesc->src_nents, edesc->src_chained, edesc->dst_nents,
2178 		   edesc->dst_chained, edesc->iv_dma, ivsize,
2179 		   edesc->sec4_sg_dma, edesc->sec4_sg_bytes);
2180 }
2181 
2182 static void aead_encrypt_done(struct device *jrdev, u32 *desc, u32 err,
2183 				   void *context)
2184 {
2185 	struct aead_request *req = context;
2186 	struct aead_edesc *edesc;
2187 #ifdef DEBUG
2188 	struct crypto_aead *aead = crypto_aead_reqtfm(req);
2189 	struct caam_ctx *ctx = crypto_aead_ctx(aead);
2190 	int ivsize = crypto_aead_ivsize(aead);
2191 
2192 	dev_err(jrdev, "%s %d: err 0x%x\n", __func__, __LINE__, err);
2193 #endif
2194 
2195 	edesc = (struct aead_edesc *)((char *)desc -
2196 		 offsetof(struct aead_edesc, hw_desc));
2197 
2198 	if (err)
2199 		caam_jr_strstatus(jrdev, err);
2200 
2201 	aead_unmap(jrdev, edesc, req);
2202 
2203 #ifdef DEBUG
2204 	print_hex_dump(KERN_ERR, "assoc  @"__stringify(__LINE__)": ",
2205 		       DUMP_PREFIX_ADDRESS, 16, 4, sg_virt(req->assoc),
2206 		       req->assoclen , 1);
2207 	print_hex_dump(KERN_ERR, "dstiv  @"__stringify(__LINE__)": ",
2208 		       DUMP_PREFIX_ADDRESS, 16, 4, sg_virt(req->src) - ivsize,
2209 		       edesc->src_nents ? 100 : ivsize, 1);
2210 	print_hex_dump(KERN_ERR, "dst    @"__stringify(__LINE__)": ",
2211 		       DUMP_PREFIX_ADDRESS, 16, 4, sg_virt(req->src),
2212 		       edesc->src_nents ? 100 : req->cryptlen +
2213 		       ctx->authsize + 4, 1);
2214 #endif
2215 
2216 	kfree(edesc);
2217 
2218 	aead_request_complete(req, err);
2219 }
2220 
2221 static void aead_decrypt_done(struct device *jrdev, u32 *desc, u32 err,
2222 				   void *context)
2223 {
2224 	struct aead_request *req = context;
2225 	struct aead_edesc *edesc;
2226 #ifdef DEBUG
2227 	struct crypto_aead *aead = crypto_aead_reqtfm(req);
2228 	struct caam_ctx *ctx = crypto_aead_ctx(aead);
2229 	int ivsize = crypto_aead_ivsize(aead);
2230 
2231 	dev_err(jrdev, "%s %d: err 0x%x\n", __func__, __LINE__, err);
2232 #endif
2233 
2234 	edesc = (struct aead_edesc *)((char *)desc -
2235 		 offsetof(struct aead_edesc, hw_desc));
2236 
2237 #ifdef DEBUG
2238 	print_hex_dump(KERN_ERR, "dstiv  @"__stringify(__LINE__)": ",
2239 		       DUMP_PREFIX_ADDRESS, 16, 4, req->iv,
2240 		       ivsize, 1);
2241 	print_hex_dump(KERN_ERR, "dst    @"__stringify(__LINE__)": ",
2242 		       DUMP_PREFIX_ADDRESS, 16, 4, sg_virt(req->dst),
2243 		       req->cryptlen - ctx->authsize, 1);
2244 #endif
2245 
2246 	if (err)
2247 		caam_jr_strstatus(jrdev, err);
2248 
2249 	aead_unmap(jrdev, edesc, req);
2250 
2251 	/*
2252 	 * verify hw auth check passed else return -EBADMSG
2253 	 */
2254 	if ((err & JRSTA_CCBERR_ERRID_MASK) == JRSTA_CCBERR_ERRID_ICVCHK)
2255 		err = -EBADMSG;
2256 
2257 #ifdef DEBUG
2258 	print_hex_dump(KERN_ERR, "iphdrout@"__stringify(__LINE__)": ",
2259 		       DUMP_PREFIX_ADDRESS, 16, 4,
2260 		       ((char *)sg_virt(req->assoc) - sizeof(struct iphdr)),
2261 		       sizeof(struct iphdr) + req->assoclen +
2262 		       ((req->cryptlen > 1500) ? 1500 : req->cryptlen) +
2263 		       ctx->authsize + 36, 1);
2264 	if (!err && edesc->sec4_sg_bytes) {
2265 		struct scatterlist *sg = sg_last(req->src, edesc->src_nents);
2266 		print_hex_dump(KERN_ERR, "sglastout@"__stringify(__LINE__)": ",
2267 			       DUMP_PREFIX_ADDRESS, 16, 4, sg_virt(sg),
2268 			sg->length + ctx->authsize + 16, 1);
2269 	}
2270 #endif
2271 
2272 	kfree(edesc);
2273 
2274 	aead_request_complete(req, err);
2275 }
2276 
2277 static void ablkcipher_encrypt_done(struct device *jrdev, u32 *desc, u32 err,
2278 				   void *context)
2279 {
2280 	struct ablkcipher_request *req = context;
2281 	struct ablkcipher_edesc *edesc;
2282 #ifdef DEBUG
2283 	struct crypto_ablkcipher *ablkcipher = crypto_ablkcipher_reqtfm(req);
2284 	int ivsize = crypto_ablkcipher_ivsize(ablkcipher);
2285 
2286 	dev_err(jrdev, "%s %d: err 0x%x\n", __func__, __LINE__, err);
2287 #endif
2288 
2289 	edesc = (struct ablkcipher_edesc *)((char *)desc -
2290 		 offsetof(struct ablkcipher_edesc, hw_desc));
2291 
2292 	if (err)
2293 		caam_jr_strstatus(jrdev, err);
2294 
2295 #ifdef DEBUG
2296 	print_hex_dump(KERN_ERR, "dstiv  @"__stringify(__LINE__)": ",
2297 		       DUMP_PREFIX_ADDRESS, 16, 4, req->info,
2298 		       edesc->src_nents > 1 ? 100 : ivsize, 1);
2299 	print_hex_dump(KERN_ERR, "dst    @"__stringify(__LINE__)": ",
2300 		       DUMP_PREFIX_ADDRESS, 16, 4, sg_virt(req->src),
2301 		       edesc->dst_nents > 1 ? 100 : req->nbytes, 1);
2302 #endif
2303 
2304 	ablkcipher_unmap(jrdev, edesc, req);
2305 	kfree(edesc);
2306 
2307 	ablkcipher_request_complete(req, err);
2308 }
2309 
2310 static void ablkcipher_decrypt_done(struct device *jrdev, u32 *desc, u32 err,
2311 				    void *context)
2312 {
2313 	struct ablkcipher_request *req = context;
2314 	struct ablkcipher_edesc *edesc;
2315 #ifdef DEBUG
2316 	struct crypto_ablkcipher *ablkcipher = crypto_ablkcipher_reqtfm(req);
2317 	int ivsize = crypto_ablkcipher_ivsize(ablkcipher);
2318 
2319 	dev_err(jrdev, "%s %d: err 0x%x\n", __func__, __LINE__, err);
2320 #endif
2321 
2322 	edesc = (struct ablkcipher_edesc *)((char *)desc -
2323 		 offsetof(struct ablkcipher_edesc, hw_desc));
2324 	if (err)
2325 		caam_jr_strstatus(jrdev, err);
2326 
2327 #ifdef DEBUG
2328 	print_hex_dump(KERN_ERR, "dstiv  @"__stringify(__LINE__)": ",
2329 		       DUMP_PREFIX_ADDRESS, 16, 4, req->info,
2330 		       ivsize, 1);
2331 	print_hex_dump(KERN_ERR, "dst    @"__stringify(__LINE__)": ",
2332 		       DUMP_PREFIX_ADDRESS, 16, 4, sg_virt(req->src),
2333 		       edesc->dst_nents > 1 ? 100 : req->nbytes, 1);
2334 #endif
2335 
2336 	ablkcipher_unmap(jrdev, edesc, req);
2337 	kfree(edesc);
2338 
2339 	ablkcipher_request_complete(req, err);
2340 }
2341 
2342 /*
2343  * Fill in aead job descriptor
2344  */
2345 static void init_aead_job(u32 *sh_desc, dma_addr_t ptr,
2346 			  struct aead_edesc *edesc,
2347 			  struct aead_request *req,
2348 			  bool all_contig, bool encrypt)
2349 {
2350 	struct crypto_aead *aead = crypto_aead_reqtfm(req);
2351 	struct caam_ctx *ctx = crypto_aead_ctx(aead);
2352 	int ivsize = crypto_aead_ivsize(aead);
2353 	int authsize = ctx->authsize;
2354 	u32 *desc = edesc->hw_desc;
2355 	u32 out_options = 0, in_options;
2356 	dma_addr_t dst_dma, src_dma;
2357 	int len, sec4_sg_index = 0;
2358 	bool is_gcm = false;
2359 
2360 #ifdef DEBUG
2361 	debug("assoclen %d cryptlen %d authsize %d\n",
2362 	      req->assoclen, req->cryptlen, authsize);
2363 	print_hex_dump(KERN_ERR, "assoc  @"__stringify(__LINE__)": ",
2364 		       DUMP_PREFIX_ADDRESS, 16, 4, sg_virt(req->assoc),
2365 		       req->assoclen , 1);
2366 	print_hex_dump(KERN_ERR, "presciv@"__stringify(__LINE__)": ",
2367 		       DUMP_PREFIX_ADDRESS, 16, 4, req->iv,
2368 		       edesc->src_nents ? 100 : ivsize, 1);
2369 	print_hex_dump(KERN_ERR, "src    @"__stringify(__LINE__)": ",
2370 		       DUMP_PREFIX_ADDRESS, 16, 4, sg_virt(req->src),
2371 			edesc->src_nents ? 100 : req->cryptlen, 1);
2372 	print_hex_dump(KERN_ERR, "shrdesc@"__stringify(__LINE__)": ",
2373 		       DUMP_PREFIX_ADDRESS, 16, 4, sh_desc,
2374 		       desc_bytes(sh_desc), 1);
2375 #endif
2376 
2377 	if (((ctx->class1_alg_type & OP_ALG_ALGSEL_MASK) ==
2378 	      OP_ALG_ALGSEL_AES) &&
2379 	    ((ctx->class1_alg_type & OP_ALG_AAI_MASK) == OP_ALG_AAI_GCM))
2380 		is_gcm = true;
2381 
2382 	len = desc_len(sh_desc);
2383 	init_job_desc_shared(desc, ptr, len, HDR_SHARE_DEFER | HDR_REVERSE);
2384 
2385 	if (all_contig) {
2386 		if (is_gcm)
2387 			src_dma = edesc->iv_dma;
2388 		else
2389 			src_dma = sg_dma_address(req->assoc);
2390 		in_options = 0;
2391 	} else {
2392 		src_dma = edesc->sec4_sg_dma;
2393 		sec4_sg_index += (edesc->assoc_nents ? : 1) + 1 +
2394 				 (edesc->src_nents ? : 1);
2395 		in_options = LDST_SGF;
2396 	}
2397 
2398 	append_seq_in_ptr(desc, src_dma, req->assoclen + ivsize + req->cryptlen,
2399 			  in_options);
2400 
2401 	if (likely(req->src == req->dst)) {
2402 		if (all_contig) {
2403 			dst_dma = sg_dma_address(req->src);
2404 		} else {
2405 			dst_dma = src_dma + sizeof(struct sec4_sg_entry) *
2406 				  ((edesc->assoc_nents ? : 1) + 1);
2407 			out_options = LDST_SGF;
2408 		}
2409 	} else {
2410 		if (!edesc->dst_nents) {
2411 			dst_dma = sg_dma_address(req->dst);
2412 		} else {
2413 			dst_dma = edesc->sec4_sg_dma +
2414 				  sec4_sg_index *
2415 				  sizeof(struct sec4_sg_entry);
2416 			out_options = LDST_SGF;
2417 		}
2418 	}
2419 	if (encrypt)
2420 		append_seq_out_ptr(desc, dst_dma, req->cryptlen + authsize,
2421 				   out_options);
2422 	else
2423 		append_seq_out_ptr(desc, dst_dma, req->cryptlen - authsize,
2424 				   out_options);
2425 }
2426 
2427 /*
2428  * Fill in aead givencrypt job descriptor
2429  */
2430 static void init_aead_giv_job(u32 *sh_desc, dma_addr_t ptr,
2431 			      struct aead_edesc *edesc,
2432 			      struct aead_request *req,
2433 			      int contig)
2434 {
2435 	struct crypto_aead *aead = crypto_aead_reqtfm(req);
2436 	struct caam_ctx *ctx = crypto_aead_ctx(aead);
2437 	int ivsize = crypto_aead_ivsize(aead);
2438 	int authsize = ctx->authsize;
2439 	u32 *desc = edesc->hw_desc;
2440 	u32 out_options = 0, in_options;
2441 	dma_addr_t dst_dma, src_dma;
2442 	int len, sec4_sg_index = 0;
2443 	bool is_gcm = false;
2444 
2445 #ifdef DEBUG
2446 	debug("assoclen %d cryptlen %d authsize %d\n",
2447 	      req->assoclen, req->cryptlen, authsize);
2448 	print_hex_dump(KERN_ERR, "assoc  @"__stringify(__LINE__)": ",
2449 		       DUMP_PREFIX_ADDRESS, 16, 4, sg_virt(req->assoc),
2450 		       req->assoclen , 1);
2451 	print_hex_dump(KERN_ERR, "presciv@"__stringify(__LINE__)": ",
2452 		       DUMP_PREFIX_ADDRESS, 16, 4, req->iv, ivsize, 1);
2453 	print_hex_dump(KERN_ERR, "src    @"__stringify(__LINE__)": ",
2454 		       DUMP_PREFIX_ADDRESS, 16, 4, sg_virt(req->src),
2455 			edesc->src_nents > 1 ? 100 : req->cryptlen, 1);
2456 	print_hex_dump(KERN_ERR, "shrdesc@"__stringify(__LINE__)": ",
2457 		       DUMP_PREFIX_ADDRESS, 16, 4, sh_desc,
2458 		       desc_bytes(sh_desc), 1);
2459 #endif
2460 
2461 	if (((ctx->class1_alg_type & OP_ALG_ALGSEL_MASK) ==
2462 	      OP_ALG_ALGSEL_AES) &&
2463 	    ((ctx->class1_alg_type & OP_ALG_AAI_MASK) == OP_ALG_AAI_GCM))
2464 		is_gcm = true;
2465 
2466 	len = desc_len(sh_desc);
2467 	init_job_desc_shared(desc, ptr, len, HDR_SHARE_DEFER | HDR_REVERSE);
2468 
2469 	if (contig & GIV_SRC_CONTIG) {
2470 		if (is_gcm)
2471 			src_dma = edesc->iv_dma;
2472 		else
2473 			src_dma = sg_dma_address(req->assoc);
2474 		in_options = 0;
2475 	} else {
2476 		src_dma = edesc->sec4_sg_dma;
2477 		sec4_sg_index += edesc->assoc_nents + 1 + edesc->src_nents;
2478 		in_options = LDST_SGF;
2479 	}
2480 	append_seq_in_ptr(desc, src_dma, req->assoclen + ivsize + req->cryptlen,
2481 			  in_options);
2482 
2483 	if (contig & GIV_DST_CONTIG) {
2484 		dst_dma = edesc->iv_dma;
2485 	} else {
2486 		if (likely(req->src == req->dst)) {
2487 			dst_dma = src_dma + sizeof(struct sec4_sg_entry) *
2488 				  (edesc->assoc_nents +
2489 				   (is_gcm ? 1 + edesc->src_nents : 0));
2490 			out_options = LDST_SGF;
2491 		} else {
2492 			dst_dma = edesc->sec4_sg_dma +
2493 				  sec4_sg_index *
2494 				  sizeof(struct sec4_sg_entry);
2495 			out_options = LDST_SGF;
2496 		}
2497 	}
2498 
2499 	append_seq_out_ptr(desc, dst_dma, ivsize + req->cryptlen + authsize,
2500 			   out_options);
2501 }
2502 
2503 /*
2504  * Fill in ablkcipher job descriptor
2505  */
2506 static void init_ablkcipher_job(u32 *sh_desc, dma_addr_t ptr,
2507 				struct ablkcipher_edesc *edesc,
2508 				struct ablkcipher_request *req,
2509 				bool iv_contig)
2510 {
2511 	struct crypto_ablkcipher *ablkcipher = crypto_ablkcipher_reqtfm(req);
2512 	int ivsize = crypto_ablkcipher_ivsize(ablkcipher);
2513 	u32 *desc = edesc->hw_desc;
2514 	u32 out_options = 0, in_options;
2515 	dma_addr_t dst_dma, src_dma;
2516 	int len, sec4_sg_index = 0;
2517 
2518 #ifdef DEBUG
2519 	print_hex_dump(KERN_ERR, "presciv@"__stringify(__LINE__)": ",
2520 		       DUMP_PREFIX_ADDRESS, 16, 4, req->info,
2521 		       ivsize, 1);
2522 	print_hex_dump(KERN_ERR, "src    @"__stringify(__LINE__)": ",
2523 		       DUMP_PREFIX_ADDRESS, 16, 4, sg_virt(req->src),
2524 		       edesc->src_nents ? 100 : req->nbytes, 1);
2525 #endif
2526 
2527 	len = desc_len(sh_desc);
2528 	init_job_desc_shared(desc, ptr, len, HDR_SHARE_DEFER | HDR_REVERSE);
2529 
2530 	if (iv_contig) {
2531 		src_dma = edesc->iv_dma;
2532 		in_options = 0;
2533 	} else {
2534 		src_dma = edesc->sec4_sg_dma;
2535 		sec4_sg_index += (iv_contig ? 0 : 1) + edesc->src_nents;
2536 		in_options = LDST_SGF;
2537 	}
2538 	append_seq_in_ptr(desc, src_dma, req->nbytes + ivsize, in_options);
2539 
2540 	if (likely(req->src == req->dst)) {
2541 		if (!edesc->src_nents && iv_contig) {
2542 			dst_dma = sg_dma_address(req->src);
2543 		} else {
2544 			dst_dma = edesc->sec4_sg_dma +
2545 				sizeof(struct sec4_sg_entry);
2546 			out_options = LDST_SGF;
2547 		}
2548 	} else {
2549 		if (!edesc->dst_nents) {
2550 			dst_dma = sg_dma_address(req->dst);
2551 		} else {
2552 			dst_dma = edesc->sec4_sg_dma +
2553 				sec4_sg_index * sizeof(struct sec4_sg_entry);
2554 			out_options = LDST_SGF;
2555 		}
2556 	}
2557 	append_seq_out_ptr(desc, dst_dma, req->nbytes, out_options);
2558 }
2559 
2560 /*
2561  * Fill in ablkcipher givencrypt job descriptor
2562  */
2563 static void init_ablkcipher_giv_job(u32 *sh_desc, dma_addr_t ptr,
2564 				    struct ablkcipher_edesc *edesc,
2565 				    struct ablkcipher_request *req,
2566 				    bool iv_contig)
2567 {
2568 	struct crypto_ablkcipher *ablkcipher = crypto_ablkcipher_reqtfm(req);
2569 	int ivsize = crypto_ablkcipher_ivsize(ablkcipher);
2570 	u32 *desc = edesc->hw_desc;
2571 	u32 out_options, in_options;
2572 	dma_addr_t dst_dma, src_dma;
2573 	int len, sec4_sg_index = 0;
2574 
2575 #ifdef DEBUG
2576 	print_hex_dump(KERN_ERR, "presciv@" __stringify(__LINE__) ": ",
2577 		       DUMP_PREFIX_ADDRESS, 16, 4, req->info,
2578 		       ivsize, 1);
2579 	print_hex_dump(KERN_ERR, "src    @" __stringify(__LINE__) ": ",
2580 		       DUMP_PREFIX_ADDRESS, 16, 4, sg_virt(req->src),
2581 		       edesc->src_nents ? 100 : req->nbytes, 1);
2582 #endif
2583 
2584 	len = desc_len(sh_desc);
2585 	init_job_desc_shared(desc, ptr, len, HDR_SHARE_DEFER | HDR_REVERSE);
2586 
2587 	if (!edesc->src_nents) {
2588 		src_dma = sg_dma_address(req->src);
2589 		in_options = 0;
2590 	} else {
2591 		src_dma = edesc->sec4_sg_dma;
2592 		sec4_sg_index += edesc->src_nents;
2593 		in_options = LDST_SGF;
2594 	}
2595 	append_seq_in_ptr(desc, src_dma, req->nbytes, in_options);
2596 
2597 	if (iv_contig) {
2598 		dst_dma = edesc->iv_dma;
2599 		out_options = 0;
2600 	} else {
2601 		dst_dma = edesc->sec4_sg_dma +
2602 			  sec4_sg_index * sizeof(struct sec4_sg_entry);
2603 		out_options = LDST_SGF;
2604 	}
2605 	append_seq_out_ptr(desc, dst_dma, req->nbytes + ivsize, out_options);
2606 }
2607 
2608 /*
2609  * allocate and map the aead extended descriptor
2610  */
2611 static struct aead_edesc *aead_edesc_alloc(struct aead_request *req,
2612 					   int desc_bytes, bool *all_contig_ptr,
2613 					   bool encrypt)
2614 {
2615 	struct crypto_aead *aead = crypto_aead_reqtfm(req);
2616 	struct caam_ctx *ctx = crypto_aead_ctx(aead);
2617 	struct device *jrdev = ctx->jrdev;
2618 	gfp_t flags = (req->base.flags & (CRYPTO_TFM_REQ_MAY_BACKLOG |
2619 		       CRYPTO_TFM_REQ_MAY_SLEEP)) ? GFP_KERNEL : GFP_ATOMIC;
2620 	int assoc_nents, src_nents, dst_nents = 0;
2621 	struct aead_edesc *edesc;
2622 	dma_addr_t iv_dma = 0;
2623 	int sgc;
2624 	bool all_contig = true;
2625 	bool assoc_chained = false, src_chained = false, dst_chained = false;
2626 	int ivsize = crypto_aead_ivsize(aead);
2627 	int sec4_sg_index, sec4_sg_len = 0, sec4_sg_bytes;
2628 	unsigned int authsize = ctx->authsize;
2629 	bool is_gcm = false;
2630 
2631 	assoc_nents = sg_count(req->assoc, req->assoclen, &assoc_chained);
2632 
2633 	if (unlikely(req->dst != req->src)) {
2634 		src_nents = sg_count(req->src, req->cryptlen, &src_chained);
2635 		dst_nents = sg_count(req->dst,
2636 				     req->cryptlen +
2637 					(encrypt ? authsize : (-authsize)),
2638 				     &dst_chained);
2639 	} else {
2640 		src_nents = sg_count(req->src,
2641 				     req->cryptlen +
2642 					(encrypt ? authsize : 0),
2643 				     &src_chained);
2644 	}
2645 
2646 	sgc = dma_map_sg_chained(jrdev, req->assoc, assoc_nents ? : 1,
2647 				 DMA_TO_DEVICE, assoc_chained);
2648 	if (likely(req->src == req->dst)) {
2649 		sgc = dma_map_sg_chained(jrdev, req->src, src_nents ? : 1,
2650 					 DMA_BIDIRECTIONAL, src_chained);
2651 	} else {
2652 		sgc = dma_map_sg_chained(jrdev, req->src, src_nents ? : 1,
2653 					 DMA_TO_DEVICE, src_chained);
2654 		sgc = dma_map_sg_chained(jrdev, req->dst, dst_nents ? : 1,
2655 					 DMA_FROM_DEVICE, dst_chained);
2656 	}
2657 
2658 	iv_dma = dma_map_single(jrdev, req->iv, ivsize, DMA_TO_DEVICE);
2659 	if (dma_mapping_error(jrdev, iv_dma)) {
2660 		dev_err(jrdev, "unable to map IV\n");
2661 		return ERR_PTR(-ENOMEM);
2662 	}
2663 
2664 	if (((ctx->class1_alg_type & OP_ALG_ALGSEL_MASK) ==
2665 	      OP_ALG_ALGSEL_AES) &&
2666 	    ((ctx->class1_alg_type & OP_ALG_AAI_MASK) == OP_ALG_AAI_GCM))
2667 		is_gcm = true;
2668 
2669 	/*
2670 	 * Check if data are contiguous.
2671 	 * GCM expected input sequence: IV, AAD, text
2672 	 * All other - expected input sequence: AAD, IV, text
2673 	 */
2674 	if (is_gcm)
2675 		all_contig = (!assoc_nents &&
2676 			      iv_dma + ivsize == sg_dma_address(req->assoc) &&
2677 			      !src_nents && sg_dma_address(req->assoc) +
2678 			      req->assoclen == sg_dma_address(req->src));
2679 	else
2680 		all_contig = (!assoc_nents && sg_dma_address(req->assoc) +
2681 			      req->assoclen == iv_dma && !src_nents &&
2682 			      iv_dma + ivsize == sg_dma_address(req->src));
2683 	if (!all_contig) {
2684 		assoc_nents = assoc_nents ? : 1;
2685 		src_nents = src_nents ? : 1;
2686 		sec4_sg_len = assoc_nents + 1 + src_nents;
2687 	}
2688 
2689 	sec4_sg_len += dst_nents;
2690 
2691 	sec4_sg_bytes = sec4_sg_len * sizeof(struct sec4_sg_entry);
2692 
2693 	/* allocate space for base edesc and hw desc commands, link tables */
2694 	edesc = kmalloc(sizeof(struct aead_edesc) + desc_bytes +
2695 			sec4_sg_bytes, GFP_DMA | flags);
2696 	if (!edesc) {
2697 		dev_err(jrdev, "could not allocate extended descriptor\n");
2698 		return ERR_PTR(-ENOMEM);
2699 	}
2700 
2701 	edesc->assoc_nents = assoc_nents;
2702 	edesc->assoc_chained = assoc_chained;
2703 	edesc->src_nents = src_nents;
2704 	edesc->src_chained = src_chained;
2705 	edesc->dst_nents = dst_nents;
2706 	edesc->dst_chained = dst_chained;
2707 	edesc->iv_dma = iv_dma;
2708 	edesc->sec4_sg_bytes = sec4_sg_bytes;
2709 	edesc->sec4_sg = (void *)edesc + sizeof(struct aead_edesc) +
2710 			 desc_bytes;
2711 	*all_contig_ptr = all_contig;
2712 
2713 	sec4_sg_index = 0;
2714 	if (!all_contig) {
2715 		if (!is_gcm) {
2716 			sg_to_sec4_sg(req->assoc,
2717 				      (assoc_nents ? : 1),
2718 				      edesc->sec4_sg +
2719 				      sec4_sg_index, 0);
2720 			sec4_sg_index += assoc_nents ? : 1;
2721 		}
2722 
2723 		dma_to_sec4_sg_one(edesc->sec4_sg + sec4_sg_index,
2724 				   iv_dma, ivsize, 0);
2725 		sec4_sg_index += 1;
2726 
2727 		if (is_gcm) {
2728 			sg_to_sec4_sg(req->assoc,
2729 				      (assoc_nents ? : 1),
2730 				      edesc->sec4_sg +
2731 				      sec4_sg_index, 0);
2732 			sec4_sg_index += assoc_nents ? : 1;
2733 		}
2734 
2735 		sg_to_sec4_sg_last(req->src,
2736 				   (src_nents ? : 1),
2737 				   edesc->sec4_sg +
2738 				   sec4_sg_index, 0);
2739 		sec4_sg_index += src_nents ? : 1;
2740 	}
2741 	if (dst_nents) {
2742 		sg_to_sec4_sg_last(req->dst, dst_nents,
2743 				   edesc->sec4_sg + sec4_sg_index, 0);
2744 	}
2745 	edesc->sec4_sg_dma = dma_map_single(jrdev, edesc->sec4_sg,
2746 					    sec4_sg_bytes, DMA_TO_DEVICE);
2747 	if (dma_mapping_error(jrdev, edesc->sec4_sg_dma)) {
2748 		dev_err(jrdev, "unable to map S/G table\n");
2749 		return ERR_PTR(-ENOMEM);
2750 	}
2751 
2752 	return edesc;
2753 }
2754 
2755 static int aead_encrypt(struct aead_request *req)
2756 {
2757 	struct aead_edesc *edesc;
2758 	struct crypto_aead *aead = crypto_aead_reqtfm(req);
2759 	struct caam_ctx *ctx = crypto_aead_ctx(aead);
2760 	struct device *jrdev = ctx->jrdev;
2761 	bool all_contig;
2762 	u32 *desc;
2763 	int ret = 0;
2764 
2765 	/* allocate extended descriptor */
2766 	edesc = aead_edesc_alloc(req, DESC_JOB_IO_LEN *
2767 				 CAAM_CMD_SZ, &all_contig, true);
2768 	if (IS_ERR(edesc))
2769 		return PTR_ERR(edesc);
2770 
2771 	/* Create and submit job descriptor */
2772 	init_aead_job(ctx->sh_desc_enc, ctx->sh_desc_enc_dma, edesc, req,
2773 		      all_contig, true);
2774 #ifdef DEBUG
2775 	print_hex_dump(KERN_ERR, "aead jobdesc@"__stringify(__LINE__)": ",
2776 		       DUMP_PREFIX_ADDRESS, 16, 4, edesc->hw_desc,
2777 		       desc_bytes(edesc->hw_desc), 1);
2778 #endif
2779 
2780 	desc = edesc->hw_desc;
2781 	ret = caam_jr_enqueue(jrdev, desc, aead_encrypt_done, req);
2782 	if (!ret) {
2783 		ret = -EINPROGRESS;
2784 	} else {
2785 		aead_unmap(jrdev, edesc, req);
2786 		kfree(edesc);
2787 	}
2788 
2789 	return ret;
2790 }
2791 
2792 static int aead_decrypt(struct aead_request *req)
2793 {
2794 	struct aead_edesc *edesc;
2795 	struct crypto_aead *aead = crypto_aead_reqtfm(req);
2796 	struct caam_ctx *ctx = crypto_aead_ctx(aead);
2797 	struct device *jrdev = ctx->jrdev;
2798 	bool all_contig;
2799 	u32 *desc;
2800 	int ret = 0;
2801 
2802 	/* allocate extended descriptor */
2803 	edesc = aead_edesc_alloc(req, DESC_JOB_IO_LEN *
2804 				 CAAM_CMD_SZ, &all_contig, false);
2805 	if (IS_ERR(edesc))
2806 		return PTR_ERR(edesc);
2807 
2808 #ifdef DEBUG
2809 	print_hex_dump(KERN_ERR, "dec src@"__stringify(__LINE__)": ",
2810 		       DUMP_PREFIX_ADDRESS, 16, 4, sg_virt(req->src),
2811 		       req->cryptlen, 1);
2812 #endif
2813 
2814 	/* Create and submit job descriptor*/
2815 	init_aead_job(ctx->sh_desc_dec,
2816 		      ctx->sh_desc_dec_dma, edesc, req, all_contig, false);
2817 #ifdef DEBUG
2818 	print_hex_dump(KERN_ERR, "aead jobdesc@"__stringify(__LINE__)": ",
2819 		       DUMP_PREFIX_ADDRESS, 16, 4, edesc->hw_desc,
2820 		       desc_bytes(edesc->hw_desc), 1);
2821 #endif
2822 
2823 	desc = edesc->hw_desc;
2824 	ret = caam_jr_enqueue(jrdev, desc, aead_decrypt_done, req);
2825 	if (!ret) {
2826 		ret = -EINPROGRESS;
2827 	} else {
2828 		aead_unmap(jrdev, edesc, req);
2829 		kfree(edesc);
2830 	}
2831 
2832 	return ret;
2833 }
2834 
2835 /*
2836  * allocate and map the aead extended descriptor for aead givencrypt
2837  */
2838 static struct aead_edesc *aead_giv_edesc_alloc(struct aead_givcrypt_request
2839 					       *greq, int desc_bytes,
2840 					       u32 *contig_ptr)
2841 {
2842 	struct aead_request *req = &greq->areq;
2843 	struct crypto_aead *aead = crypto_aead_reqtfm(req);
2844 	struct caam_ctx *ctx = crypto_aead_ctx(aead);
2845 	struct device *jrdev = ctx->jrdev;
2846 	gfp_t flags = (req->base.flags & (CRYPTO_TFM_REQ_MAY_BACKLOG |
2847 		       CRYPTO_TFM_REQ_MAY_SLEEP)) ? GFP_KERNEL : GFP_ATOMIC;
2848 	int assoc_nents, src_nents, dst_nents = 0;
2849 	struct aead_edesc *edesc;
2850 	dma_addr_t iv_dma = 0;
2851 	int sgc;
2852 	u32 contig = GIV_SRC_CONTIG | GIV_DST_CONTIG;
2853 	int ivsize = crypto_aead_ivsize(aead);
2854 	bool assoc_chained = false, src_chained = false, dst_chained = false;
2855 	int sec4_sg_index, sec4_sg_len = 0, sec4_sg_bytes;
2856 	bool is_gcm = false;
2857 
2858 	assoc_nents = sg_count(req->assoc, req->assoclen, &assoc_chained);
2859 	src_nents = sg_count(req->src, req->cryptlen, &src_chained);
2860 
2861 	if (unlikely(req->dst != req->src))
2862 		dst_nents = sg_count(req->dst, req->cryptlen + ctx->authsize,
2863 				     &dst_chained);
2864 
2865 	sgc = dma_map_sg_chained(jrdev, req->assoc, assoc_nents ? : 1,
2866 				 DMA_TO_DEVICE, assoc_chained);
2867 	if (likely(req->src == req->dst)) {
2868 		sgc = dma_map_sg_chained(jrdev, req->src, src_nents ? : 1,
2869 					 DMA_BIDIRECTIONAL, src_chained);
2870 	} else {
2871 		sgc = dma_map_sg_chained(jrdev, req->src, src_nents ? : 1,
2872 					 DMA_TO_DEVICE, src_chained);
2873 		sgc = dma_map_sg_chained(jrdev, req->dst, dst_nents ? : 1,
2874 					 DMA_FROM_DEVICE, dst_chained);
2875 	}
2876 
2877 	iv_dma = dma_map_single(jrdev, greq->giv, ivsize, DMA_TO_DEVICE);
2878 	if (dma_mapping_error(jrdev, iv_dma)) {
2879 		dev_err(jrdev, "unable to map IV\n");
2880 		return ERR_PTR(-ENOMEM);
2881 	}
2882 
2883 	if (((ctx->class1_alg_type & OP_ALG_ALGSEL_MASK) ==
2884 	      OP_ALG_ALGSEL_AES) &&
2885 	    ((ctx->class1_alg_type & OP_ALG_AAI_MASK) == OP_ALG_AAI_GCM))
2886 		is_gcm = true;
2887 
2888 	/*
2889 	 * Check if data are contiguous.
2890 	 * GCM expected input sequence: IV, AAD, text
2891 	 * All other - expected input sequence: AAD, IV, text
2892 	 */
2893 
2894 	if (is_gcm) {
2895 		if (assoc_nents || iv_dma + ivsize !=
2896 		    sg_dma_address(req->assoc) || src_nents ||
2897 		    sg_dma_address(req->assoc) + req->assoclen !=
2898 		    sg_dma_address(req->src))
2899 			contig &= ~GIV_SRC_CONTIG;
2900 	} else {
2901 		if (assoc_nents ||
2902 		    sg_dma_address(req->assoc) + req->assoclen != iv_dma ||
2903 		    src_nents || iv_dma + ivsize != sg_dma_address(req->src))
2904 			contig &= ~GIV_SRC_CONTIG;
2905 	}
2906 
2907 	if (dst_nents || iv_dma + ivsize != sg_dma_address(req->dst))
2908 		contig &= ~GIV_DST_CONTIG;
2909 
2910 	if (!(contig & GIV_SRC_CONTIG)) {
2911 		assoc_nents = assoc_nents ? : 1;
2912 		src_nents = src_nents ? : 1;
2913 		sec4_sg_len += assoc_nents + 1 + src_nents;
2914 		if (req->src == req->dst &&
2915 		    (src_nents || iv_dma + ivsize != sg_dma_address(req->src)))
2916 			contig &= ~GIV_DST_CONTIG;
2917 	}
2918 
2919 	/*
2920 	 * Add new sg entries for GCM output sequence.
2921 	 * Expected output sequence: IV, encrypted text.
2922 	 */
2923 	if (is_gcm && req->src == req->dst && !(contig & GIV_DST_CONTIG))
2924 		sec4_sg_len += 1 + src_nents;
2925 
2926 	if (unlikely(req->src != req->dst)) {
2927 		dst_nents = dst_nents ? : 1;
2928 		sec4_sg_len += 1 + dst_nents;
2929 	}
2930 
2931 	sec4_sg_bytes = sec4_sg_len * sizeof(struct sec4_sg_entry);
2932 
2933 	/* allocate space for base edesc and hw desc commands, link tables */
2934 	edesc = kmalloc(sizeof(struct aead_edesc) + desc_bytes +
2935 			sec4_sg_bytes, GFP_DMA | flags);
2936 	if (!edesc) {
2937 		dev_err(jrdev, "could not allocate extended descriptor\n");
2938 		return ERR_PTR(-ENOMEM);
2939 	}
2940 
2941 	edesc->assoc_nents = assoc_nents;
2942 	edesc->assoc_chained = assoc_chained;
2943 	edesc->src_nents = src_nents;
2944 	edesc->src_chained = src_chained;
2945 	edesc->dst_nents = dst_nents;
2946 	edesc->dst_chained = dst_chained;
2947 	edesc->iv_dma = iv_dma;
2948 	edesc->sec4_sg_bytes = sec4_sg_bytes;
2949 	edesc->sec4_sg = (void *)edesc + sizeof(struct aead_edesc) +
2950 			 desc_bytes;
2951 	*contig_ptr = contig;
2952 
2953 	sec4_sg_index = 0;
2954 	if (!(contig & GIV_SRC_CONTIG)) {
2955 		if (!is_gcm) {
2956 			sg_to_sec4_sg(req->assoc, assoc_nents,
2957 				      edesc->sec4_sg + sec4_sg_index, 0);
2958 			sec4_sg_index += assoc_nents;
2959 		}
2960 
2961 		dma_to_sec4_sg_one(edesc->sec4_sg + sec4_sg_index,
2962 				   iv_dma, ivsize, 0);
2963 		sec4_sg_index += 1;
2964 
2965 		if (is_gcm) {
2966 			sg_to_sec4_sg(req->assoc, assoc_nents,
2967 				      edesc->sec4_sg + sec4_sg_index, 0);
2968 			sec4_sg_index += assoc_nents;
2969 		}
2970 
2971 		sg_to_sec4_sg_last(req->src, src_nents,
2972 				   edesc->sec4_sg +
2973 				   sec4_sg_index, 0);
2974 		sec4_sg_index += src_nents;
2975 	}
2976 
2977 	if (is_gcm && req->src == req->dst && !(contig & GIV_DST_CONTIG)) {
2978 		dma_to_sec4_sg_one(edesc->sec4_sg + sec4_sg_index,
2979 				   iv_dma, ivsize, 0);
2980 		sec4_sg_index += 1;
2981 		sg_to_sec4_sg_last(req->src, src_nents,
2982 				   edesc->sec4_sg + sec4_sg_index, 0);
2983 	}
2984 
2985 	if (unlikely(req->src != req->dst && !(contig & GIV_DST_CONTIG))) {
2986 		dma_to_sec4_sg_one(edesc->sec4_sg + sec4_sg_index,
2987 				   iv_dma, ivsize, 0);
2988 		sec4_sg_index += 1;
2989 		sg_to_sec4_sg_last(req->dst, dst_nents,
2990 				   edesc->sec4_sg + sec4_sg_index, 0);
2991 	}
2992 	edesc->sec4_sg_dma = dma_map_single(jrdev, edesc->sec4_sg,
2993 					    sec4_sg_bytes, DMA_TO_DEVICE);
2994 	if (dma_mapping_error(jrdev, edesc->sec4_sg_dma)) {
2995 		dev_err(jrdev, "unable to map S/G table\n");
2996 		return ERR_PTR(-ENOMEM);
2997 	}
2998 
2999 	return edesc;
3000 }
3001 
3002 static int aead_givencrypt(struct aead_givcrypt_request *areq)
3003 {
3004 	struct aead_request *req = &areq->areq;
3005 	struct aead_edesc *edesc;
3006 	struct crypto_aead *aead = crypto_aead_reqtfm(req);
3007 	struct caam_ctx *ctx = crypto_aead_ctx(aead);
3008 	struct device *jrdev = ctx->jrdev;
3009 	u32 contig;
3010 	u32 *desc;
3011 	int ret = 0;
3012 
3013 	/* allocate extended descriptor */
3014 	edesc = aead_giv_edesc_alloc(areq, DESC_JOB_IO_LEN *
3015 				     CAAM_CMD_SZ, &contig);
3016 
3017 	if (IS_ERR(edesc))
3018 		return PTR_ERR(edesc);
3019 
3020 #ifdef DEBUG
3021 	print_hex_dump(KERN_ERR, "giv src@"__stringify(__LINE__)": ",
3022 		       DUMP_PREFIX_ADDRESS, 16, 4, sg_virt(req->src),
3023 		       req->cryptlen, 1);
3024 #endif
3025 
3026 	/* Create and submit job descriptor*/
3027 	init_aead_giv_job(ctx->sh_desc_givenc,
3028 			  ctx->sh_desc_givenc_dma, edesc, req, contig);
3029 #ifdef DEBUG
3030 	print_hex_dump(KERN_ERR, "aead jobdesc@"__stringify(__LINE__)": ",
3031 		       DUMP_PREFIX_ADDRESS, 16, 4, edesc->hw_desc,
3032 		       desc_bytes(edesc->hw_desc), 1);
3033 #endif
3034 
3035 	desc = edesc->hw_desc;
3036 	ret = caam_jr_enqueue(jrdev, desc, aead_encrypt_done, req);
3037 	if (!ret) {
3038 		ret = -EINPROGRESS;
3039 	} else {
3040 		aead_unmap(jrdev, edesc, req);
3041 		kfree(edesc);
3042 	}
3043 
3044 	return ret;
3045 }
3046 
3047 static int aead_null_givencrypt(struct aead_givcrypt_request *areq)
3048 {
3049 	return aead_encrypt(&areq->areq);
3050 }
3051 
3052 /*
3053  * allocate and map the ablkcipher extended descriptor for ablkcipher
3054  */
3055 static struct ablkcipher_edesc *ablkcipher_edesc_alloc(struct ablkcipher_request
3056 						       *req, int desc_bytes,
3057 						       bool *iv_contig_out)
3058 {
3059 	struct crypto_ablkcipher *ablkcipher = crypto_ablkcipher_reqtfm(req);
3060 	struct caam_ctx *ctx = crypto_ablkcipher_ctx(ablkcipher);
3061 	struct device *jrdev = ctx->jrdev;
3062 	gfp_t flags = (req->base.flags & (CRYPTO_TFM_REQ_MAY_BACKLOG |
3063 					  CRYPTO_TFM_REQ_MAY_SLEEP)) ?
3064 		       GFP_KERNEL : GFP_ATOMIC;
3065 	int src_nents, dst_nents = 0, sec4_sg_bytes;
3066 	struct ablkcipher_edesc *edesc;
3067 	dma_addr_t iv_dma = 0;
3068 	bool iv_contig = false;
3069 	int sgc;
3070 	int ivsize = crypto_ablkcipher_ivsize(ablkcipher);
3071 	bool src_chained = false, dst_chained = false;
3072 	int sec4_sg_index;
3073 
3074 	src_nents = sg_count(req->src, req->nbytes, &src_chained);
3075 
3076 	if (req->dst != req->src)
3077 		dst_nents = sg_count(req->dst, req->nbytes, &dst_chained);
3078 
3079 	if (likely(req->src == req->dst)) {
3080 		sgc = dma_map_sg_chained(jrdev, req->src, src_nents ? : 1,
3081 					 DMA_BIDIRECTIONAL, src_chained);
3082 	} else {
3083 		sgc = dma_map_sg_chained(jrdev, req->src, src_nents ? : 1,
3084 					 DMA_TO_DEVICE, src_chained);
3085 		sgc = dma_map_sg_chained(jrdev, req->dst, dst_nents ? : 1,
3086 					 DMA_FROM_DEVICE, dst_chained);
3087 	}
3088 
3089 	iv_dma = dma_map_single(jrdev, req->info, ivsize, DMA_TO_DEVICE);
3090 	if (dma_mapping_error(jrdev, iv_dma)) {
3091 		dev_err(jrdev, "unable to map IV\n");
3092 		return ERR_PTR(-ENOMEM);
3093 	}
3094 
3095 	/*
3096 	 * Check if iv can be contiguous with source and destination.
3097 	 * If so, include it. If not, create scatterlist.
3098 	 */
3099 	if (!src_nents && iv_dma + ivsize == sg_dma_address(req->src))
3100 		iv_contig = true;
3101 	else
3102 		src_nents = src_nents ? : 1;
3103 	sec4_sg_bytes = ((iv_contig ? 0 : 1) + src_nents + dst_nents) *
3104 			sizeof(struct sec4_sg_entry);
3105 
3106 	/* allocate space for base edesc and hw desc commands, link tables */
3107 	edesc = kmalloc(sizeof(struct ablkcipher_edesc) + desc_bytes +
3108 			sec4_sg_bytes, GFP_DMA | flags);
3109 	if (!edesc) {
3110 		dev_err(jrdev, "could not allocate extended descriptor\n");
3111 		return ERR_PTR(-ENOMEM);
3112 	}
3113 
3114 	edesc->src_nents = src_nents;
3115 	edesc->src_chained = src_chained;
3116 	edesc->dst_nents = dst_nents;
3117 	edesc->dst_chained = dst_chained;
3118 	edesc->sec4_sg_bytes = sec4_sg_bytes;
3119 	edesc->sec4_sg = (void *)edesc + sizeof(struct ablkcipher_edesc) +
3120 			 desc_bytes;
3121 
3122 	sec4_sg_index = 0;
3123 	if (!iv_contig) {
3124 		dma_to_sec4_sg_one(edesc->sec4_sg, iv_dma, ivsize, 0);
3125 		sg_to_sec4_sg_last(req->src, src_nents,
3126 				   edesc->sec4_sg + 1, 0);
3127 		sec4_sg_index += 1 + src_nents;
3128 	}
3129 
3130 	if (dst_nents) {
3131 		sg_to_sec4_sg_last(req->dst, dst_nents,
3132 			edesc->sec4_sg + sec4_sg_index, 0);
3133 	}
3134 
3135 	edesc->sec4_sg_dma = dma_map_single(jrdev, edesc->sec4_sg,
3136 					    sec4_sg_bytes, DMA_TO_DEVICE);
3137 	if (dma_mapping_error(jrdev, edesc->sec4_sg_dma)) {
3138 		dev_err(jrdev, "unable to map S/G table\n");
3139 		return ERR_PTR(-ENOMEM);
3140 	}
3141 
3142 	edesc->iv_dma = iv_dma;
3143 
3144 #ifdef DEBUG
3145 	print_hex_dump(KERN_ERR, "ablkcipher sec4_sg@"__stringify(__LINE__)": ",
3146 		       DUMP_PREFIX_ADDRESS, 16, 4, edesc->sec4_sg,
3147 		       sec4_sg_bytes, 1);
3148 #endif
3149 
3150 	*iv_contig_out = iv_contig;
3151 	return edesc;
3152 }
3153 
3154 static int ablkcipher_encrypt(struct ablkcipher_request *req)
3155 {
3156 	struct ablkcipher_edesc *edesc;
3157 	struct crypto_ablkcipher *ablkcipher = crypto_ablkcipher_reqtfm(req);
3158 	struct caam_ctx *ctx = crypto_ablkcipher_ctx(ablkcipher);
3159 	struct device *jrdev = ctx->jrdev;
3160 	bool iv_contig;
3161 	u32 *desc;
3162 	int ret = 0;
3163 
3164 	/* allocate extended descriptor */
3165 	edesc = ablkcipher_edesc_alloc(req, DESC_JOB_IO_LEN *
3166 				       CAAM_CMD_SZ, &iv_contig);
3167 	if (IS_ERR(edesc))
3168 		return PTR_ERR(edesc);
3169 
3170 	/* Create and submit job descriptor*/
3171 	init_ablkcipher_job(ctx->sh_desc_enc,
3172 		ctx->sh_desc_enc_dma, edesc, req, iv_contig);
3173 #ifdef DEBUG
3174 	print_hex_dump(KERN_ERR, "ablkcipher jobdesc@"__stringify(__LINE__)": ",
3175 		       DUMP_PREFIX_ADDRESS, 16, 4, edesc->hw_desc,
3176 		       desc_bytes(edesc->hw_desc), 1);
3177 #endif
3178 	desc = edesc->hw_desc;
3179 	ret = caam_jr_enqueue(jrdev, desc, ablkcipher_encrypt_done, req);
3180 
3181 	if (!ret) {
3182 		ret = -EINPROGRESS;
3183 	} else {
3184 		ablkcipher_unmap(jrdev, edesc, req);
3185 		kfree(edesc);
3186 	}
3187 
3188 	return ret;
3189 }
3190 
3191 static int ablkcipher_decrypt(struct ablkcipher_request *req)
3192 {
3193 	struct ablkcipher_edesc *edesc;
3194 	struct crypto_ablkcipher *ablkcipher = crypto_ablkcipher_reqtfm(req);
3195 	struct caam_ctx *ctx = crypto_ablkcipher_ctx(ablkcipher);
3196 	struct device *jrdev = ctx->jrdev;
3197 	bool iv_contig;
3198 	u32 *desc;
3199 	int ret = 0;
3200 
3201 	/* allocate extended descriptor */
3202 	edesc = ablkcipher_edesc_alloc(req, DESC_JOB_IO_LEN *
3203 				       CAAM_CMD_SZ, &iv_contig);
3204 	if (IS_ERR(edesc))
3205 		return PTR_ERR(edesc);
3206 
3207 	/* Create and submit job descriptor*/
3208 	init_ablkcipher_job(ctx->sh_desc_dec,
3209 		ctx->sh_desc_dec_dma, edesc, req, iv_contig);
3210 	desc = edesc->hw_desc;
3211 #ifdef DEBUG
3212 	print_hex_dump(KERN_ERR, "ablkcipher jobdesc@"__stringify(__LINE__)": ",
3213 		       DUMP_PREFIX_ADDRESS, 16, 4, edesc->hw_desc,
3214 		       desc_bytes(edesc->hw_desc), 1);
3215 #endif
3216 
3217 	ret = caam_jr_enqueue(jrdev, desc, ablkcipher_decrypt_done, req);
3218 	if (!ret) {
3219 		ret = -EINPROGRESS;
3220 	} else {
3221 		ablkcipher_unmap(jrdev, edesc, req);
3222 		kfree(edesc);
3223 	}
3224 
3225 	return ret;
3226 }
3227 
3228 /*
3229  * allocate and map the ablkcipher extended descriptor
3230  * for ablkcipher givencrypt
3231  */
3232 static struct ablkcipher_edesc *ablkcipher_giv_edesc_alloc(
3233 				struct skcipher_givcrypt_request *greq,
3234 				int desc_bytes,
3235 				bool *iv_contig_out)
3236 {
3237 	struct ablkcipher_request *req = &greq->creq;
3238 	struct crypto_ablkcipher *ablkcipher = crypto_ablkcipher_reqtfm(req);
3239 	struct caam_ctx *ctx = crypto_ablkcipher_ctx(ablkcipher);
3240 	struct device *jrdev = ctx->jrdev;
3241 	gfp_t flags = (req->base.flags & (CRYPTO_TFM_REQ_MAY_BACKLOG |
3242 					  CRYPTO_TFM_REQ_MAY_SLEEP)) ?
3243 		       GFP_KERNEL : GFP_ATOMIC;
3244 	int src_nents, dst_nents = 0, sec4_sg_bytes;
3245 	struct ablkcipher_edesc *edesc;
3246 	dma_addr_t iv_dma = 0;
3247 	bool iv_contig = false;
3248 	int sgc;
3249 	int ivsize = crypto_ablkcipher_ivsize(ablkcipher);
3250 	bool src_chained = false, dst_chained = false;
3251 	int sec4_sg_index;
3252 
3253 	src_nents = sg_count(req->src, req->nbytes, &src_chained);
3254 
3255 	if (unlikely(req->dst != req->src))
3256 		dst_nents = sg_count(req->dst, req->nbytes, &dst_chained);
3257 
3258 	if (likely(req->src == req->dst)) {
3259 		sgc = dma_map_sg_chained(jrdev, req->src, src_nents ? : 1,
3260 					 DMA_BIDIRECTIONAL, src_chained);
3261 	} else {
3262 		sgc = dma_map_sg_chained(jrdev, req->src, src_nents ? : 1,
3263 					 DMA_TO_DEVICE, src_chained);
3264 		sgc = dma_map_sg_chained(jrdev, req->dst, dst_nents ? : 1,
3265 					 DMA_FROM_DEVICE, dst_chained);
3266 	}
3267 
3268 	/*
3269 	 * Check if iv can be contiguous with source and destination.
3270 	 * If so, include it. If not, create scatterlist.
3271 	 */
3272 	iv_dma = dma_map_single(jrdev, greq->giv, ivsize, DMA_TO_DEVICE);
3273 	if (dma_mapping_error(jrdev, iv_dma)) {
3274 		dev_err(jrdev, "unable to map IV\n");
3275 		return ERR_PTR(-ENOMEM);
3276 	}
3277 
3278 	if (!dst_nents && iv_dma + ivsize == sg_dma_address(req->dst))
3279 		iv_contig = true;
3280 	else
3281 		dst_nents = dst_nents ? : 1;
3282 	sec4_sg_bytes = ((iv_contig ? 0 : 1) + src_nents + dst_nents) *
3283 			sizeof(struct sec4_sg_entry);
3284 
3285 	/* allocate space for base edesc and hw desc commands, link tables */
3286 	edesc = kmalloc(sizeof(*edesc) + desc_bytes +
3287 			sec4_sg_bytes, GFP_DMA | flags);
3288 	if (!edesc) {
3289 		dev_err(jrdev, "could not allocate extended descriptor\n");
3290 		return ERR_PTR(-ENOMEM);
3291 	}
3292 
3293 	edesc->src_nents = src_nents;
3294 	edesc->src_chained = src_chained;
3295 	edesc->dst_nents = dst_nents;
3296 	edesc->dst_chained = dst_chained;
3297 	edesc->sec4_sg_bytes = sec4_sg_bytes;
3298 	edesc->sec4_sg = (void *)edesc + sizeof(struct ablkcipher_edesc) +
3299 			 desc_bytes;
3300 
3301 	sec4_sg_index = 0;
3302 	if (src_nents) {
3303 		sg_to_sec4_sg_last(req->src, src_nents, edesc->sec4_sg, 0);
3304 		sec4_sg_index += src_nents;
3305 	}
3306 
3307 	if (!iv_contig) {
3308 		dma_to_sec4_sg_one(edesc->sec4_sg + sec4_sg_index,
3309 				   iv_dma, ivsize, 0);
3310 		sec4_sg_index += 1;
3311 		sg_to_sec4_sg_last(req->dst, dst_nents,
3312 				   edesc->sec4_sg + sec4_sg_index, 0);
3313 	}
3314 
3315 	edesc->sec4_sg_dma = dma_map_single(jrdev, edesc->sec4_sg,
3316 					    sec4_sg_bytes, DMA_TO_DEVICE);
3317 	if (dma_mapping_error(jrdev, edesc->sec4_sg_dma)) {
3318 		dev_err(jrdev, "unable to map S/G table\n");
3319 		return ERR_PTR(-ENOMEM);
3320 	}
3321 	edesc->iv_dma = iv_dma;
3322 
3323 #ifdef DEBUG
3324 	print_hex_dump(KERN_ERR,
3325 		       "ablkcipher sec4_sg@" __stringify(__LINE__) ": ",
3326 		       DUMP_PREFIX_ADDRESS, 16, 4, edesc->sec4_sg,
3327 		       sec4_sg_bytes, 1);
3328 #endif
3329 
3330 	*iv_contig_out = iv_contig;
3331 	return edesc;
3332 }
3333 
3334 static int ablkcipher_givencrypt(struct skcipher_givcrypt_request *creq)
3335 {
3336 	struct ablkcipher_request *req = &creq->creq;
3337 	struct ablkcipher_edesc *edesc;
3338 	struct crypto_ablkcipher *ablkcipher = crypto_ablkcipher_reqtfm(req);
3339 	struct caam_ctx *ctx = crypto_ablkcipher_ctx(ablkcipher);
3340 	struct device *jrdev = ctx->jrdev;
3341 	bool iv_contig;
3342 	u32 *desc;
3343 	int ret = 0;
3344 
3345 	/* allocate extended descriptor */
3346 	edesc = ablkcipher_giv_edesc_alloc(creq, DESC_JOB_IO_LEN *
3347 				       CAAM_CMD_SZ, &iv_contig);
3348 	if (IS_ERR(edesc))
3349 		return PTR_ERR(edesc);
3350 
3351 	/* Create and submit job descriptor*/
3352 	init_ablkcipher_giv_job(ctx->sh_desc_givenc, ctx->sh_desc_givenc_dma,
3353 				edesc, req, iv_contig);
3354 #ifdef DEBUG
3355 	print_hex_dump(KERN_ERR,
3356 		       "ablkcipher jobdesc@" __stringify(__LINE__) ": ",
3357 		       DUMP_PREFIX_ADDRESS, 16, 4, edesc->hw_desc,
3358 		       desc_bytes(edesc->hw_desc), 1);
3359 #endif
3360 	desc = edesc->hw_desc;
3361 	ret = caam_jr_enqueue(jrdev, desc, ablkcipher_encrypt_done, req);
3362 
3363 	if (!ret) {
3364 		ret = -EINPROGRESS;
3365 	} else {
3366 		ablkcipher_unmap(jrdev, edesc, req);
3367 		kfree(edesc);
3368 	}
3369 
3370 	return ret;
3371 }
3372 
3373 #define template_aead		template_u.aead
3374 #define template_ablkcipher	template_u.ablkcipher
3375 struct caam_alg_template {
3376 	char name[CRYPTO_MAX_ALG_NAME];
3377 	char driver_name[CRYPTO_MAX_ALG_NAME];
3378 	unsigned int blocksize;
3379 	u32 type;
3380 	union {
3381 		struct ablkcipher_alg ablkcipher;
3382 		struct aead_alg aead;
3383 		struct blkcipher_alg blkcipher;
3384 		struct cipher_alg cipher;
3385 		struct compress_alg compress;
3386 		struct rng_alg rng;
3387 	} template_u;
3388 	u32 class1_alg_type;
3389 	u32 class2_alg_type;
3390 	u32 alg_op;
3391 };
3392 
3393 static struct caam_alg_template driver_algs[] = {
3394 	/* single-pass ipsec_esp descriptor */
3395 	{
3396 		.name = "authenc(hmac(md5),ecb(cipher_null))",
3397 		.driver_name = "authenc-hmac-md5-ecb-cipher_null-caam",
3398 		.blocksize = NULL_BLOCK_SIZE,
3399 		.type = CRYPTO_ALG_TYPE_AEAD,
3400 		.template_aead = {
3401 			.setkey = aead_setkey,
3402 			.setauthsize = aead_setauthsize,
3403 			.encrypt = aead_encrypt,
3404 			.decrypt = aead_decrypt,
3405 			.givencrypt = aead_null_givencrypt,
3406 			.geniv = "<built-in>",
3407 			.ivsize = NULL_IV_SIZE,
3408 			.maxauthsize = MD5_DIGEST_SIZE,
3409 			},
3410 		.class1_alg_type = 0,
3411 		.class2_alg_type = OP_ALG_ALGSEL_MD5 | OP_ALG_AAI_HMAC_PRECOMP,
3412 		.alg_op = OP_ALG_ALGSEL_MD5 | OP_ALG_AAI_HMAC,
3413 	},
3414 	{
3415 		.name = "authenc(hmac(sha1),ecb(cipher_null))",
3416 		.driver_name = "authenc-hmac-sha1-ecb-cipher_null-caam",
3417 		.blocksize = NULL_BLOCK_SIZE,
3418 		.type = CRYPTO_ALG_TYPE_AEAD,
3419 		.template_aead = {
3420 			.setkey = aead_setkey,
3421 			.setauthsize = aead_setauthsize,
3422 			.encrypt = aead_encrypt,
3423 			.decrypt = aead_decrypt,
3424 			.givencrypt = aead_null_givencrypt,
3425 			.geniv = "<built-in>",
3426 			.ivsize = NULL_IV_SIZE,
3427 			.maxauthsize = SHA1_DIGEST_SIZE,
3428 			},
3429 		.class1_alg_type = 0,
3430 		.class2_alg_type = OP_ALG_ALGSEL_SHA1 | OP_ALG_AAI_HMAC_PRECOMP,
3431 		.alg_op = OP_ALG_ALGSEL_SHA1 | OP_ALG_AAI_HMAC,
3432 	},
3433 	{
3434 		.name = "authenc(hmac(sha224),ecb(cipher_null))",
3435 		.driver_name = "authenc-hmac-sha224-ecb-cipher_null-caam",
3436 		.blocksize = NULL_BLOCK_SIZE,
3437 		.type = CRYPTO_ALG_TYPE_AEAD,
3438 		.template_aead = {
3439 			.setkey = aead_setkey,
3440 			.setauthsize = aead_setauthsize,
3441 			.encrypt = aead_encrypt,
3442 			.decrypt = aead_decrypt,
3443 			.givencrypt = aead_null_givencrypt,
3444 			.geniv = "<built-in>",
3445 			.ivsize = NULL_IV_SIZE,
3446 			.maxauthsize = SHA224_DIGEST_SIZE,
3447 			},
3448 		.class1_alg_type = 0,
3449 		.class2_alg_type = OP_ALG_ALGSEL_SHA224 |
3450 				   OP_ALG_AAI_HMAC_PRECOMP,
3451 		.alg_op = OP_ALG_ALGSEL_SHA224 | OP_ALG_AAI_HMAC,
3452 	},
3453 	{
3454 		.name = "authenc(hmac(sha256),ecb(cipher_null))",
3455 		.driver_name = "authenc-hmac-sha256-ecb-cipher_null-caam",
3456 		.blocksize = NULL_BLOCK_SIZE,
3457 		.type = CRYPTO_ALG_TYPE_AEAD,
3458 		.template_aead = {
3459 			.setkey = aead_setkey,
3460 			.setauthsize = aead_setauthsize,
3461 			.encrypt = aead_encrypt,
3462 			.decrypt = aead_decrypt,
3463 			.givencrypt = aead_null_givencrypt,
3464 			.geniv = "<built-in>",
3465 			.ivsize = NULL_IV_SIZE,
3466 			.maxauthsize = SHA256_DIGEST_SIZE,
3467 			},
3468 		.class1_alg_type = 0,
3469 		.class2_alg_type = OP_ALG_ALGSEL_SHA256 |
3470 				   OP_ALG_AAI_HMAC_PRECOMP,
3471 		.alg_op = OP_ALG_ALGSEL_SHA256 | OP_ALG_AAI_HMAC,
3472 	},
3473 	{
3474 		.name = "authenc(hmac(sha384),ecb(cipher_null))",
3475 		.driver_name = "authenc-hmac-sha384-ecb-cipher_null-caam",
3476 		.blocksize = NULL_BLOCK_SIZE,
3477 		.type = CRYPTO_ALG_TYPE_AEAD,
3478 		.template_aead = {
3479 			.setkey = aead_setkey,
3480 			.setauthsize = aead_setauthsize,
3481 			.encrypt = aead_encrypt,
3482 			.decrypt = aead_decrypt,
3483 			.givencrypt = aead_null_givencrypt,
3484 			.geniv = "<built-in>",
3485 			.ivsize = NULL_IV_SIZE,
3486 			.maxauthsize = SHA384_DIGEST_SIZE,
3487 			},
3488 		.class1_alg_type = 0,
3489 		.class2_alg_type = OP_ALG_ALGSEL_SHA384 |
3490 				   OP_ALG_AAI_HMAC_PRECOMP,
3491 		.alg_op = OP_ALG_ALGSEL_SHA384 | OP_ALG_AAI_HMAC,
3492 	},
3493 	{
3494 		.name = "authenc(hmac(sha512),ecb(cipher_null))",
3495 		.driver_name = "authenc-hmac-sha512-ecb-cipher_null-caam",
3496 		.blocksize = NULL_BLOCK_SIZE,
3497 		.type = CRYPTO_ALG_TYPE_AEAD,
3498 		.template_aead = {
3499 			.setkey = aead_setkey,
3500 			.setauthsize = aead_setauthsize,
3501 			.encrypt = aead_encrypt,
3502 			.decrypt = aead_decrypt,
3503 			.givencrypt = aead_null_givencrypt,
3504 			.geniv = "<built-in>",
3505 			.ivsize = NULL_IV_SIZE,
3506 			.maxauthsize = SHA512_DIGEST_SIZE,
3507 			},
3508 		.class1_alg_type = 0,
3509 		.class2_alg_type = OP_ALG_ALGSEL_SHA512 |
3510 				   OP_ALG_AAI_HMAC_PRECOMP,
3511 		.alg_op = OP_ALG_ALGSEL_SHA512 | OP_ALG_AAI_HMAC,
3512 	},
3513 	{
3514 		.name = "authenc(hmac(md5),cbc(aes))",
3515 		.driver_name = "authenc-hmac-md5-cbc-aes-caam",
3516 		.blocksize = AES_BLOCK_SIZE,
3517 		.type = CRYPTO_ALG_TYPE_AEAD,
3518 		.template_aead = {
3519 			.setkey = aead_setkey,
3520 			.setauthsize = aead_setauthsize,
3521 			.encrypt = aead_encrypt,
3522 			.decrypt = aead_decrypt,
3523 			.givencrypt = aead_givencrypt,
3524 			.geniv = "<built-in>",
3525 			.ivsize = AES_BLOCK_SIZE,
3526 			.maxauthsize = MD5_DIGEST_SIZE,
3527 			},
3528 		.class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CBC,
3529 		.class2_alg_type = OP_ALG_ALGSEL_MD5 | OP_ALG_AAI_HMAC_PRECOMP,
3530 		.alg_op = OP_ALG_ALGSEL_MD5 | OP_ALG_AAI_HMAC,
3531 	},
3532 	{
3533 		.name = "authenc(hmac(sha1),cbc(aes))",
3534 		.driver_name = "authenc-hmac-sha1-cbc-aes-caam",
3535 		.blocksize = AES_BLOCK_SIZE,
3536 		.type = CRYPTO_ALG_TYPE_AEAD,
3537 		.template_aead = {
3538 			.setkey = aead_setkey,
3539 			.setauthsize = aead_setauthsize,
3540 			.encrypt = aead_encrypt,
3541 			.decrypt = aead_decrypt,
3542 			.givencrypt = aead_givencrypt,
3543 			.geniv = "<built-in>",
3544 			.ivsize = AES_BLOCK_SIZE,
3545 			.maxauthsize = SHA1_DIGEST_SIZE,
3546 			},
3547 		.class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CBC,
3548 		.class2_alg_type = OP_ALG_ALGSEL_SHA1 | OP_ALG_AAI_HMAC_PRECOMP,
3549 		.alg_op = OP_ALG_ALGSEL_SHA1 | OP_ALG_AAI_HMAC,
3550 	},
3551 	{
3552 		.name = "authenc(hmac(sha224),cbc(aes))",
3553 		.driver_name = "authenc-hmac-sha224-cbc-aes-caam",
3554 		.blocksize = AES_BLOCK_SIZE,
3555 		.type = CRYPTO_ALG_TYPE_AEAD,
3556 		.template_aead = {
3557 			.setkey = aead_setkey,
3558 			.setauthsize = aead_setauthsize,
3559 			.encrypt = aead_encrypt,
3560 			.decrypt = aead_decrypt,
3561 			.givencrypt = aead_givencrypt,
3562 			.geniv = "<built-in>",
3563 			.ivsize = AES_BLOCK_SIZE,
3564 			.maxauthsize = SHA224_DIGEST_SIZE,
3565 			},
3566 		.class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CBC,
3567 		.class2_alg_type = OP_ALG_ALGSEL_SHA224 |
3568 				   OP_ALG_AAI_HMAC_PRECOMP,
3569 		.alg_op = OP_ALG_ALGSEL_SHA224 | OP_ALG_AAI_HMAC,
3570 	},
3571 	{
3572 		.name = "authenc(hmac(sha256),cbc(aes))",
3573 		.driver_name = "authenc-hmac-sha256-cbc-aes-caam",
3574 		.blocksize = AES_BLOCK_SIZE,
3575 		.type = CRYPTO_ALG_TYPE_AEAD,
3576 		.template_aead = {
3577 			.setkey = aead_setkey,
3578 			.setauthsize = aead_setauthsize,
3579 			.encrypt = aead_encrypt,
3580 			.decrypt = aead_decrypt,
3581 			.givencrypt = aead_givencrypt,
3582 			.geniv = "<built-in>",
3583 			.ivsize = AES_BLOCK_SIZE,
3584 			.maxauthsize = SHA256_DIGEST_SIZE,
3585 			},
3586 		.class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CBC,
3587 		.class2_alg_type = OP_ALG_ALGSEL_SHA256 |
3588 				   OP_ALG_AAI_HMAC_PRECOMP,
3589 		.alg_op = OP_ALG_ALGSEL_SHA256 | OP_ALG_AAI_HMAC,
3590 	},
3591 	{
3592 		.name = "authenc(hmac(sha384),cbc(aes))",
3593 		.driver_name = "authenc-hmac-sha384-cbc-aes-caam",
3594 		.blocksize = AES_BLOCK_SIZE,
3595 		.type = CRYPTO_ALG_TYPE_AEAD,
3596 		.template_aead = {
3597 			.setkey = aead_setkey,
3598 			.setauthsize = aead_setauthsize,
3599 			.encrypt = aead_encrypt,
3600 			.decrypt = aead_decrypt,
3601 			.givencrypt = aead_givencrypt,
3602 			.geniv = "<built-in>",
3603 			.ivsize = AES_BLOCK_SIZE,
3604 			.maxauthsize = SHA384_DIGEST_SIZE,
3605 			},
3606 		.class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CBC,
3607 		.class2_alg_type = OP_ALG_ALGSEL_SHA384 |
3608 				   OP_ALG_AAI_HMAC_PRECOMP,
3609 		.alg_op = OP_ALG_ALGSEL_SHA384 | OP_ALG_AAI_HMAC,
3610 	},
3611 
3612 	{
3613 		.name = "authenc(hmac(sha512),cbc(aes))",
3614 		.driver_name = "authenc-hmac-sha512-cbc-aes-caam",
3615 		.blocksize = AES_BLOCK_SIZE,
3616 		.type = CRYPTO_ALG_TYPE_AEAD,
3617 		.template_aead = {
3618 			.setkey = aead_setkey,
3619 			.setauthsize = aead_setauthsize,
3620 			.encrypt = aead_encrypt,
3621 			.decrypt = aead_decrypt,
3622 			.givencrypt = aead_givencrypt,
3623 			.geniv = "<built-in>",
3624 			.ivsize = AES_BLOCK_SIZE,
3625 			.maxauthsize = SHA512_DIGEST_SIZE,
3626 			},
3627 		.class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CBC,
3628 		.class2_alg_type = OP_ALG_ALGSEL_SHA512 |
3629 				   OP_ALG_AAI_HMAC_PRECOMP,
3630 		.alg_op = OP_ALG_ALGSEL_SHA512 | OP_ALG_AAI_HMAC,
3631 	},
3632 	{
3633 		.name = "authenc(hmac(md5),cbc(des3_ede))",
3634 		.driver_name = "authenc-hmac-md5-cbc-des3_ede-caam",
3635 		.blocksize = DES3_EDE_BLOCK_SIZE,
3636 		.type = CRYPTO_ALG_TYPE_AEAD,
3637 		.template_aead = {
3638 			.setkey = aead_setkey,
3639 			.setauthsize = aead_setauthsize,
3640 			.encrypt = aead_encrypt,
3641 			.decrypt = aead_decrypt,
3642 			.givencrypt = aead_givencrypt,
3643 			.geniv = "<built-in>",
3644 			.ivsize = DES3_EDE_BLOCK_SIZE,
3645 			.maxauthsize = MD5_DIGEST_SIZE,
3646 			},
3647 		.class1_alg_type = OP_ALG_ALGSEL_3DES | OP_ALG_AAI_CBC,
3648 		.class2_alg_type = OP_ALG_ALGSEL_MD5 | OP_ALG_AAI_HMAC_PRECOMP,
3649 		.alg_op = OP_ALG_ALGSEL_MD5 | OP_ALG_AAI_HMAC,
3650 	},
3651 	{
3652 		.name = "authenc(hmac(sha1),cbc(des3_ede))",
3653 		.driver_name = "authenc-hmac-sha1-cbc-des3_ede-caam",
3654 		.blocksize = DES3_EDE_BLOCK_SIZE,
3655 		.type = CRYPTO_ALG_TYPE_AEAD,
3656 		.template_aead = {
3657 			.setkey = aead_setkey,
3658 			.setauthsize = aead_setauthsize,
3659 			.encrypt = aead_encrypt,
3660 			.decrypt = aead_decrypt,
3661 			.givencrypt = aead_givencrypt,
3662 			.geniv = "<built-in>",
3663 			.ivsize = DES3_EDE_BLOCK_SIZE,
3664 			.maxauthsize = SHA1_DIGEST_SIZE,
3665 			},
3666 		.class1_alg_type = OP_ALG_ALGSEL_3DES | OP_ALG_AAI_CBC,
3667 		.class2_alg_type = OP_ALG_ALGSEL_SHA1 | OP_ALG_AAI_HMAC_PRECOMP,
3668 		.alg_op = OP_ALG_ALGSEL_SHA1 | OP_ALG_AAI_HMAC,
3669 	},
3670 	{
3671 		.name = "authenc(hmac(sha224),cbc(des3_ede))",
3672 		.driver_name = "authenc-hmac-sha224-cbc-des3_ede-caam",
3673 		.blocksize = DES3_EDE_BLOCK_SIZE,
3674 		.type = CRYPTO_ALG_TYPE_AEAD,
3675 		.template_aead = {
3676 			.setkey = aead_setkey,
3677 			.setauthsize = aead_setauthsize,
3678 			.encrypt = aead_encrypt,
3679 			.decrypt = aead_decrypt,
3680 			.givencrypt = aead_givencrypt,
3681 			.geniv = "<built-in>",
3682 			.ivsize = DES3_EDE_BLOCK_SIZE,
3683 			.maxauthsize = SHA224_DIGEST_SIZE,
3684 			},
3685 		.class1_alg_type = OP_ALG_ALGSEL_3DES | OP_ALG_AAI_CBC,
3686 		.class2_alg_type = OP_ALG_ALGSEL_SHA224 |
3687 				   OP_ALG_AAI_HMAC_PRECOMP,
3688 		.alg_op = OP_ALG_ALGSEL_SHA224 | OP_ALG_AAI_HMAC,
3689 	},
3690 	{
3691 		.name = "authenc(hmac(sha256),cbc(des3_ede))",
3692 		.driver_name = "authenc-hmac-sha256-cbc-des3_ede-caam",
3693 		.blocksize = DES3_EDE_BLOCK_SIZE,
3694 		.type = CRYPTO_ALG_TYPE_AEAD,
3695 		.template_aead = {
3696 			.setkey = aead_setkey,
3697 			.setauthsize = aead_setauthsize,
3698 			.encrypt = aead_encrypt,
3699 			.decrypt = aead_decrypt,
3700 			.givencrypt = aead_givencrypt,
3701 			.geniv = "<built-in>",
3702 			.ivsize = DES3_EDE_BLOCK_SIZE,
3703 			.maxauthsize = SHA256_DIGEST_SIZE,
3704 			},
3705 		.class1_alg_type = OP_ALG_ALGSEL_3DES | OP_ALG_AAI_CBC,
3706 		.class2_alg_type = OP_ALG_ALGSEL_SHA256 |
3707 				   OP_ALG_AAI_HMAC_PRECOMP,
3708 		.alg_op = OP_ALG_ALGSEL_SHA256 | OP_ALG_AAI_HMAC,
3709 	},
3710 	{
3711 		.name = "authenc(hmac(sha384),cbc(des3_ede))",
3712 		.driver_name = "authenc-hmac-sha384-cbc-des3_ede-caam",
3713 		.blocksize = DES3_EDE_BLOCK_SIZE,
3714 		.type = CRYPTO_ALG_TYPE_AEAD,
3715 		.template_aead = {
3716 			.setkey = aead_setkey,
3717 			.setauthsize = aead_setauthsize,
3718 			.encrypt = aead_encrypt,
3719 			.decrypt = aead_decrypt,
3720 			.givencrypt = aead_givencrypt,
3721 			.geniv = "<built-in>",
3722 			.ivsize = DES3_EDE_BLOCK_SIZE,
3723 			.maxauthsize = SHA384_DIGEST_SIZE,
3724 			},
3725 		.class1_alg_type = OP_ALG_ALGSEL_3DES | OP_ALG_AAI_CBC,
3726 		.class2_alg_type = OP_ALG_ALGSEL_SHA384 |
3727 				   OP_ALG_AAI_HMAC_PRECOMP,
3728 		.alg_op = OP_ALG_ALGSEL_SHA384 | OP_ALG_AAI_HMAC,
3729 	},
3730 	{
3731 		.name = "authenc(hmac(sha512),cbc(des3_ede))",
3732 		.driver_name = "authenc-hmac-sha512-cbc-des3_ede-caam",
3733 		.blocksize = DES3_EDE_BLOCK_SIZE,
3734 		.type = CRYPTO_ALG_TYPE_AEAD,
3735 		.template_aead = {
3736 			.setkey = aead_setkey,
3737 			.setauthsize = aead_setauthsize,
3738 			.encrypt = aead_encrypt,
3739 			.decrypt = aead_decrypt,
3740 			.givencrypt = aead_givencrypt,
3741 			.geniv = "<built-in>",
3742 			.ivsize = DES3_EDE_BLOCK_SIZE,
3743 			.maxauthsize = SHA512_DIGEST_SIZE,
3744 			},
3745 		.class1_alg_type = OP_ALG_ALGSEL_3DES | OP_ALG_AAI_CBC,
3746 		.class2_alg_type = OP_ALG_ALGSEL_SHA512 |
3747 				   OP_ALG_AAI_HMAC_PRECOMP,
3748 		.alg_op = OP_ALG_ALGSEL_SHA512 | OP_ALG_AAI_HMAC,
3749 	},
3750 	{
3751 		.name = "authenc(hmac(md5),cbc(des))",
3752 		.driver_name = "authenc-hmac-md5-cbc-des-caam",
3753 		.blocksize = DES_BLOCK_SIZE,
3754 		.type = CRYPTO_ALG_TYPE_AEAD,
3755 		.template_aead = {
3756 			.setkey = aead_setkey,
3757 			.setauthsize = aead_setauthsize,
3758 			.encrypt = aead_encrypt,
3759 			.decrypt = aead_decrypt,
3760 			.givencrypt = aead_givencrypt,
3761 			.geniv = "<built-in>",
3762 			.ivsize = DES_BLOCK_SIZE,
3763 			.maxauthsize = MD5_DIGEST_SIZE,
3764 			},
3765 		.class1_alg_type = OP_ALG_ALGSEL_DES | OP_ALG_AAI_CBC,
3766 		.class2_alg_type = OP_ALG_ALGSEL_MD5 | OP_ALG_AAI_HMAC_PRECOMP,
3767 		.alg_op = OP_ALG_ALGSEL_MD5 | OP_ALG_AAI_HMAC,
3768 	},
3769 	{
3770 		.name = "authenc(hmac(sha1),cbc(des))",
3771 		.driver_name = "authenc-hmac-sha1-cbc-des-caam",
3772 		.blocksize = DES_BLOCK_SIZE,
3773 		.type = CRYPTO_ALG_TYPE_AEAD,
3774 		.template_aead = {
3775 			.setkey = aead_setkey,
3776 			.setauthsize = aead_setauthsize,
3777 			.encrypt = aead_encrypt,
3778 			.decrypt = aead_decrypt,
3779 			.givencrypt = aead_givencrypt,
3780 			.geniv = "<built-in>",
3781 			.ivsize = DES_BLOCK_SIZE,
3782 			.maxauthsize = SHA1_DIGEST_SIZE,
3783 			},
3784 		.class1_alg_type = OP_ALG_ALGSEL_DES | OP_ALG_AAI_CBC,
3785 		.class2_alg_type = OP_ALG_ALGSEL_SHA1 | OP_ALG_AAI_HMAC_PRECOMP,
3786 		.alg_op = OP_ALG_ALGSEL_SHA1 | OP_ALG_AAI_HMAC,
3787 	},
3788 	{
3789 		.name = "authenc(hmac(sha224),cbc(des))",
3790 		.driver_name = "authenc-hmac-sha224-cbc-des-caam",
3791 		.blocksize = DES_BLOCK_SIZE,
3792 		.type = CRYPTO_ALG_TYPE_AEAD,
3793 		.template_aead = {
3794 			.setkey = aead_setkey,
3795 			.setauthsize = aead_setauthsize,
3796 			.encrypt = aead_encrypt,
3797 			.decrypt = aead_decrypt,
3798 			.givencrypt = aead_givencrypt,
3799 			.geniv = "<built-in>",
3800 			.ivsize = DES_BLOCK_SIZE,
3801 			.maxauthsize = SHA224_DIGEST_SIZE,
3802 			},
3803 		.class1_alg_type = OP_ALG_ALGSEL_DES | OP_ALG_AAI_CBC,
3804 		.class2_alg_type = OP_ALG_ALGSEL_SHA224 |
3805 				   OP_ALG_AAI_HMAC_PRECOMP,
3806 		.alg_op = OP_ALG_ALGSEL_SHA224 | OP_ALG_AAI_HMAC,
3807 	},
3808 	{
3809 		.name = "authenc(hmac(sha256),cbc(des))",
3810 		.driver_name = "authenc-hmac-sha256-cbc-des-caam",
3811 		.blocksize = DES_BLOCK_SIZE,
3812 		.type = CRYPTO_ALG_TYPE_AEAD,
3813 		.template_aead = {
3814 			.setkey = aead_setkey,
3815 			.setauthsize = aead_setauthsize,
3816 			.encrypt = aead_encrypt,
3817 			.decrypt = aead_decrypt,
3818 			.givencrypt = aead_givencrypt,
3819 			.geniv = "<built-in>",
3820 			.ivsize = DES_BLOCK_SIZE,
3821 			.maxauthsize = SHA256_DIGEST_SIZE,
3822 			},
3823 		.class1_alg_type = OP_ALG_ALGSEL_DES | OP_ALG_AAI_CBC,
3824 		.class2_alg_type = OP_ALG_ALGSEL_SHA256 |
3825 				   OP_ALG_AAI_HMAC_PRECOMP,
3826 		.alg_op = OP_ALG_ALGSEL_SHA256 | OP_ALG_AAI_HMAC,
3827 	},
3828 	{
3829 		.name = "authenc(hmac(sha384),cbc(des))",
3830 		.driver_name = "authenc-hmac-sha384-cbc-des-caam",
3831 		.blocksize = DES_BLOCK_SIZE,
3832 		.type = CRYPTO_ALG_TYPE_AEAD,
3833 		.template_aead = {
3834 			.setkey = aead_setkey,
3835 			.setauthsize = aead_setauthsize,
3836 			.encrypt = aead_encrypt,
3837 			.decrypt = aead_decrypt,
3838 			.givencrypt = aead_givencrypt,
3839 			.geniv = "<built-in>",
3840 			.ivsize = DES_BLOCK_SIZE,
3841 			.maxauthsize = SHA384_DIGEST_SIZE,
3842 			},
3843 		.class1_alg_type = OP_ALG_ALGSEL_DES | OP_ALG_AAI_CBC,
3844 		.class2_alg_type = OP_ALG_ALGSEL_SHA384 |
3845 				   OP_ALG_AAI_HMAC_PRECOMP,
3846 		.alg_op = OP_ALG_ALGSEL_SHA384 | OP_ALG_AAI_HMAC,
3847 	},
3848 	{
3849 		.name = "authenc(hmac(sha512),cbc(des))",
3850 		.driver_name = "authenc-hmac-sha512-cbc-des-caam",
3851 		.blocksize = DES_BLOCK_SIZE,
3852 		.type = CRYPTO_ALG_TYPE_AEAD,
3853 		.template_aead = {
3854 			.setkey = aead_setkey,
3855 			.setauthsize = aead_setauthsize,
3856 			.encrypt = aead_encrypt,
3857 			.decrypt = aead_decrypt,
3858 			.givencrypt = aead_givencrypt,
3859 			.geniv = "<built-in>",
3860 			.ivsize = DES_BLOCK_SIZE,
3861 			.maxauthsize = SHA512_DIGEST_SIZE,
3862 			},
3863 		.class1_alg_type = OP_ALG_ALGSEL_DES | OP_ALG_AAI_CBC,
3864 		.class2_alg_type = OP_ALG_ALGSEL_SHA512 |
3865 				   OP_ALG_AAI_HMAC_PRECOMP,
3866 		.alg_op = OP_ALG_ALGSEL_SHA512 | OP_ALG_AAI_HMAC,
3867 	},
3868 	{
3869 		.name = "authenc(hmac(md5),rfc3686(ctr(aes)))",
3870 		.driver_name = "authenc-hmac-md5-rfc3686-ctr-aes-caam",
3871 		.blocksize = 1,
3872 		.type = CRYPTO_ALG_TYPE_AEAD,
3873 		.template_aead = {
3874 			.setkey = aead_setkey,
3875 			.setauthsize = aead_setauthsize,
3876 			.encrypt = aead_encrypt,
3877 			.decrypt = aead_decrypt,
3878 			.givencrypt = aead_givencrypt,
3879 			.geniv = "<built-in>",
3880 			.ivsize = CTR_RFC3686_IV_SIZE,
3881 			.maxauthsize = MD5_DIGEST_SIZE,
3882 			},
3883 		.class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CTR_MOD128,
3884 		.class2_alg_type = OP_ALG_ALGSEL_MD5 | OP_ALG_AAI_HMAC_PRECOMP,
3885 		.alg_op = OP_ALG_ALGSEL_MD5 | OP_ALG_AAI_HMAC,
3886 	},
3887 	{
3888 		.name = "authenc(hmac(sha1),rfc3686(ctr(aes)))",
3889 		.driver_name = "authenc-hmac-sha1-rfc3686-ctr-aes-caam",
3890 		.blocksize = 1,
3891 		.type = CRYPTO_ALG_TYPE_AEAD,
3892 		.template_aead = {
3893 			.setkey = aead_setkey,
3894 			.setauthsize = aead_setauthsize,
3895 			.encrypt = aead_encrypt,
3896 			.decrypt = aead_decrypt,
3897 			.givencrypt = aead_givencrypt,
3898 			.geniv = "<built-in>",
3899 			.ivsize = CTR_RFC3686_IV_SIZE,
3900 			.maxauthsize = SHA1_DIGEST_SIZE,
3901 			},
3902 		.class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CTR_MOD128,
3903 		.class2_alg_type = OP_ALG_ALGSEL_SHA1 | OP_ALG_AAI_HMAC_PRECOMP,
3904 		.alg_op = OP_ALG_ALGSEL_SHA1 | OP_ALG_AAI_HMAC,
3905 	},
3906 	{
3907 		.name = "authenc(hmac(sha224),rfc3686(ctr(aes)))",
3908 		.driver_name = "authenc-hmac-sha224-rfc3686-ctr-aes-caam",
3909 		.blocksize = 1,
3910 		.type = CRYPTO_ALG_TYPE_AEAD,
3911 		.template_aead = {
3912 			.setkey = aead_setkey,
3913 			.setauthsize = aead_setauthsize,
3914 			.encrypt = aead_encrypt,
3915 			.decrypt = aead_decrypt,
3916 			.givencrypt = aead_givencrypt,
3917 			.geniv = "<built-in>",
3918 			.ivsize = CTR_RFC3686_IV_SIZE,
3919 			.maxauthsize = SHA224_DIGEST_SIZE,
3920 			},
3921 		.class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CTR_MOD128,
3922 		.class2_alg_type = OP_ALG_ALGSEL_SHA224 |
3923 				   OP_ALG_AAI_HMAC_PRECOMP,
3924 		.alg_op = OP_ALG_ALGSEL_SHA224 | OP_ALG_AAI_HMAC,
3925 	},
3926 	{
3927 		.name = "authenc(hmac(sha256),rfc3686(ctr(aes)))",
3928 		.driver_name = "authenc-hmac-sha256-rfc3686-ctr-aes-caam",
3929 		.blocksize = 1,
3930 		.type = CRYPTO_ALG_TYPE_AEAD,
3931 		.template_aead = {
3932 			.setkey = aead_setkey,
3933 			.setauthsize = aead_setauthsize,
3934 			.encrypt = aead_encrypt,
3935 			.decrypt = aead_decrypt,
3936 			.givencrypt = aead_givencrypt,
3937 			.geniv = "<built-in>",
3938 			.ivsize = CTR_RFC3686_IV_SIZE,
3939 			.maxauthsize = SHA256_DIGEST_SIZE,
3940 			},
3941 		.class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CTR_MOD128,
3942 		.class2_alg_type = OP_ALG_ALGSEL_SHA256 |
3943 				   OP_ALG_AAI_HMAC_PRECOMP,
3944 		.alg_op = OP_ALG_ALGSEL_SHA256 | OP_ALG_AAI_HMAC,
3945 	},
3946 	{
3947 		.name = "authenc(hmac(sha384),rfc3686(ctr(aes)))",
3948 		.driver_name = "authenc-hmac-sha384-rfc3686-ctr-aes-caam",
3949 		.blocksize = 1,
3950 		.type = CRYPTO_ALG_TYPE_AEAD,
3951 		.template_aead = {
3952 			.setkey = aead_setkey,
3953 			.setauthsize = aead_setauthsize,
3954 			.encrypt = aead_encrypt,
3955 			.decrypt = aead_decrypt,
3956 			.givencrypt = aead_givencrypt,
3957 			.geniv = "<built-in>",
3958 			.ivsize = CTR_RFC3686_IV_SIZE,
3959 			.maxauthsize = SHA384_DIGEST_SIZE,
3960 			},
3961 		.class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CTR_MOD128,
3962 		.class2_alg_type = OP_ALG_ALGSEL_SHA384 |
3963 				   OP_ALG_AAI_HMAC_PRECOMP,
3964 		.alg_op = OP_ALG_ALGSEL_SHA384 | OP_ALG_AAI_HMAC,
3965 	},
3966 	{
3967 		.name = "authenc(hmac(sha512),rfc3686(ctr(aes)))",
3968 		.driver_name = "authenc-hmac-sha512-rfc3686-ctr-aes-caam",
3969 		.blocksize = 1,
3970 		.type = CRYPTO_ALG_TYPE_AEAD,
3971 		.template_aead = {
3972 			.setkey = aead_setkey,
3973 			.setauthsize = aead_setauthsize,
3974 			.encrypt = aead_encrypt,
3975 			.decrypt = aead_decrypt,
3976 			.givencrypt = aead_givencrypt,
3977 			.geniv = "<built-in>",
3978 			.ivsize = CTR_RFC3686_IV_SIZE,
3979 			.maxauthsize = SHA512_DIGEST_SIZE,
3980 			},
3981 		.class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CTR_MOD128,
3982 		.class2_alg_type = OP_ALG_ALGSEL_SHA512 |
3983 				   OP_ALG_AAI_HMAC_PRECOMP,
3984 		.alg_op = OP_ALG_ALGSEL_SHA512 | OP_ALG_AAI_HMAC,
3985 	},
3986 	{
3987 		.name = "rfc4106(gcm(aes))",
3988 		.driver_name = "rfc4106-gcm-aes-caam",
3989 		.blocksize = 1,
3990 		.type = CRYPTO_ALG_TYPE_AEAD,
3991 		.template_aead = {
3992 			.setkey = rfc4106_setkey,
3993 			.setauthsize = rfc4106_setauthsize,
3994 			.encrypt = aead_encrypt,
3995 			.decrypt = aead_decrypt,
3996 			.givencrypt = aead_givencrypt,
3997 			.geniv = "<built-in>",
3998 			.ivsize = 8,
3999 			.maxauthsize = AES_BLOCK_SIZE,
4000 			},
4001 		.class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_GCM,
4002 	},
4003 	{
4004 		.name = "rfc4543(gcm(aes))",
4005 		.driver_name = "rfc4543-gcm-aes-caam",
4006 		.blocksize = 1,
4007 		.type = CRYPTO_ALG_TYPE_AEAD,
4008 		.template_aead = {
4009 			.setkey = rfc4543_setkey,
4010 			.setauthsize = rfc4543_setauthsize,
4011 			.encrypt = aead_encrypt,
4012 			.decrypt = aead_decrypt,
4013 			.givencrypt = aead_givencrypt,
4014 			.geniv = "<built-in>",
4015 			.ivsize = 8,
4016 			.maxauthsize = AES_BLOCK_SIZE,
4017 			},
4018 		.class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_GCM,
4019 	},
4020 	/* Galois Counter Mode */
4021 	{
4022 		.name = "gcm(aes)",
4023 		.driver_name = "gcm-aes-caam",
4024 		.blocksize = 1,
4025 		.type = CRYPTO_ALG_TYPE_AEAD,
4026 		.template_aead = {
4027 			.setkey = gcm_setkey,
4028 			.setauthsize = gcm_setauthsize,
4029 			.encrypt = aead_encrypt,
4030 			.decrypt = aead_decrypt,
4031 			.givencrypt = NULL,
4032 			.geniv = "<built-in>",
4033 			.ivsize = 12,
4034 			.maxauthsize = AES_BLOCK_SIZE,
4035 			},
4036 		.class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_GCM,
4037 	},
4038 	/* ablkcipher descriptor */
4039 	{
4040 		.name = "cbc(aes)",
4041 		.driver_name = "cbc-aes-caam",
4042 		.blocksize = AES_BLOCK_SIZE,
4043 		.type = CRYPTO_ALG_TYPE_GIVCIPHER,
4044 		.template_ablkcipher = {
4045 			.setkey = ablkcipher_setkey,
4046 			.encrypt = ablkcipher_encrypt,
4047 			.decrypt = ablkcipher_decrypt,
4048 			.givencrypt = ablkcipher_givencrypt,
4049 			.geniv = "<built-in>",
4050 			.min_keysize = AES_MIN_KEY_SIZE,
4051 			.max_keysize = AES_MAX_KEY_SIZE,
4052 			.ivsize = AES_BLOCK_SIZE,
4053 			},
4054 		.class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CBC,
4055 	},
4056 	{
4057 		.name = "cbc(des3_ede)",
4058 		.driver_name = "cbc-3des-caam",
4059 		.blocksize = DES3_EDE_BLOCK_SIZE,
4060 		.type = CRYPTO_ALG_TYPE_GIVCIPHER,
4061 		.template_ablkcipher = {
4062 			.setkey = ablkcipher_setkey,
4063 			.encrypt = ablkcipher_encrypt,
4064 			.decrypt = ablkcipher_decrypt,
4065 			.givencrypt = ablkcipher_givencrypt,
4066 			.geniv = "<built-in>",
4067 			.min_keysize = DES3_EDE_KEY_SIZE,
4068 			.max_keysize = DES3_EDE_KEY_SIZE,
4069 			.ivsize = DES3_EDE_BLOCK_SIZE,
4070 			},
4071 		.class1_alg_type = OP_ALG_ALGSEL_3DES | OP_ALG_AAI_CBC,
4072 	},
4073 	{
4074 		.name = "cbc(des)",
4075 		.driver_name = "cbc-des-caam",
4076 		.blocksize = DES_BLOCK_SIZE,
4077 		.type = CRYPTO_ALG_TYPE_GIVCIPHER,
4078 		.template_ablkcipher = {
4079 			.setkey = ablkcipher_setkey,
4080 			.encrypt = ablkcipher_encrypt,
4081 			.decrypt = ablkcipher_decrypt,
4082 			.givencrypt = ablkcipher_givencrypt,
4083 			.geniv = "<built-in>",
4084 			.min_keysize = DES_KEY_SIZE,
4085 			.max_keysize = DES_KEY_SIZE,
4086 			.ivsize = DES_BLOCK_SIZE,
4087 			},
4088 		.class1_alg_type = OP_ALG_ALGSEL_DES | OP_ALG_AAI_CBC,
4089 	},
4090 	{
4091 		.name = "ctr(aes)",
4092 		.driver_name = "ctr-aes-caam",
4093 		.blocksize = 1,
4094 		.type = CRYPTO_ALG_TYPE_ABLKCIPHER,
4095 		.template_ablkcipher = {
4096 			.setkey = ablkcipher_setkey,
4097 			.encrypt = ablkcipher_encrypt,
4098 			.decrypt = ablkcipher_decrypt,
4099 			.geniv = "chainiv",
4100 			.min_keysize = AES_MIN_KEY_SIZE,
4101 			.max_keysize = AES_MAX_KEY_SIZE,
4102 			.ivsize = AES_BLOCK_SIZE,
4103 			},
4104 		.class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CTR_MOD128,
4105 	},
4106 	{
4107 		.name = "rfc3686(ctr(aes))",
4108 		.driver_name = "rfc3686-ctr-aes-caam",
4109 		.blocksize = 1,
4110 		.type = CRYPTO_ALG_TYPE_GIVCIPHER,
4111 		.template_ablkcipher = {
4112 			.setkey = ablkcipher_setkey,
4113 			.encrypt = ablkcipher_encrypt,
4114 			.decrypt = ablkcipher_decrypt,
4115 			.givencrypt = ablkcipher_givencrypt,
4116 			.geniv = "<built-in>",
4117 			.min_keysize = AES_MIN_KEY_SIZE +
4118 				       CTR_RFC3686_NONCE_SIZE,
4119 			.max_keysize = AES_MAX_KEY_SIZE +
4120 				       CTR_RFC3686_NONCE_SIZE,
4121 			.ivsize = CTR_RFC3686_IV_SIZE,
4122 			},
4123 		.class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CTR_MOD128,
4124 	}
4125 };
4126 
4127 struct caam_crypto_alg {
4128 	struct list_head entry;
4129 	int class1_alg_type;
4130 	int class2_alg_type;
4131 	int alg_op;
4132 	struct crypto_alg crypto_alg;
4133 };
4134 
4135 static int caam_cra_init(struct crypto_tfm *tfm)
4136 {
4137 	struct crypto_alg *alg = tfm->__crt_alg;
4138 	struct caam_crypto_alg *caam_alg =
4139 		 container_of(alg, struct caam_crypto_alg, crypto_alg);
4140 	struct caam_ctx *ctx = crypto_tfm_ctx(tfm);
4141 
4142 	ctx->jrdev = caam_jr_alloc();
4143 	if (IS_ERR(ctx->jrdev)) {
4144 		pr_err("Job Ring Device allocation for transform failed\n");
4145 		return PTR_ERR(ctx->jrdev);
4146 	}
4147 
4148 	/* copy descriptor header template value */
4149 	ctx->class1_alg_type = OP_TYPE_CLASS1_ALG | caam_alg->class1_alg_type;
4150 	ctx->class2_alg_type = OP_TYPE_CLASS2_ALG | caam_alg->class2_alg_type;
4151 	ctx->alg_op = OP_TYPE_CLASS2_ALG | caam_alg->alg_op;
4152 
4153 	return 0;
4154 }
4155 
4156 static void caam_cra_exit(struct crypto_tfm *tfm)
4157 {
4158 	struct caam_ctx *ctx = crypto_tfm_ctx(tfm);
4159 
4160 	if (ctx->sh_desc_enc_dma &&
4161 	    !dma_mapping_error(ctx->jrdev, ctx->sh_desc_enc_dma))
4162 		dma_unmap_single(ctx->jrdev, ctx->sh_desc_enc_dma,
4163 				 desc_bytes(ctx->sh_desc_enc), DMA_TO_DEVICE);
4164 	if (ctx->sh_desc_dec_dma &&
4165 	    !dma_mapping_error(ctx->jrdev, ctx->sh_desc_dec_dma))
4166 		dma_unmap_single(ctx->jrdev, ctx->sh_desc_dec_dma,
4167 				 desc_bytes(ctx->sh_desc_dec), DMA_TO_DEVICE);
4168 	if (ctx->sh_desc_givenc_dma &&
4169 	    !dma_mapping_error(ctx->jrdev, ctx->sh_desc_givenc_dma))
4170 		dma_unmap_single(ctx->jrdev, ctx->sh_desc_givenc_dma,
4171 				 desc_bytes(ctx->sh_desc_givenc),
4172 				 DMA_TO_DEVICE);
4173 	if (ctx->key_dma &&
4174 	    !dma_mapping_error(ctx->jrdev, ctx->key_dma))
4175 		dma_unmap_single(ctx->jrdev, ctx->key_dma,
4176 				 ctx->enckeylen + ctx->split_key_pad_len,
4177 				 DMA_TO_DEVICE);
4178 
4179 	caam_jr_free(ctx->jrdev);
4180 }
4181 
4182 static void __exit caam_algapi_exit(void)
4183 {
4184 
4185 	struct caam_crypto_alg *t_alg, *n;
4186 
4187 	if (!alg_list.next)
4188 		return;
4189 
4190 	list_for_each_entry_safe(t_alg, n, &alg_list, entry) {
4191 		crypto_unregister_alg(&t_alg->crypto_alg);
4192 		list_del(&t_alg->entry);
4193 		kfree(t_alg);
4194 	}
4195 }
4196 
4197 static struct caam_crypto_alg *caam_alg_alloc(struct caam_alg_template
4198 					      *template)
4199 {
4200 	struct caam_crypto_alg *t_alg;
4201 	struct crypto_alg *alg;
4202 
4203 	t_alg = kzalloc(sizeof(struct caam_crypto_alg), GFP_KERNEL);
4204 	if (!t_alg) {
4205 		pr_err("failed to allocate t_alg\n");
4206 		return ERR_PTR(-ENOMEM);
4207 	}
4208 
4209 	alg = &t_alg->crypto_alg;
4210 
4211 	snprintf(alg->cra_name, CRYPTO_MAX_ALG_NAME, "%s", template->name);
4212 	snprintf(alg->cra_driver_name, CRYPTO_MAX_ALG_NAME, "%s",
4213 		 template->driver_name);
4214 	alg->cra_module = THIS_MODULE;
4215 	alg->cra_init = caam_cra_init;
4216 	alg->cra_exit = caam_cra_exit;
4217 	alg->cra_priority = CAAM_CRA_PRIORITY;
4218 	alg->cra_blocksize = template->blocksize;
4219 	alg->cra_alignmask = 0;
4220 	alg->cra_ctxsize = sizeof(struct caam_ctx);
4221 	alg->cra_flags = CRYPTO_ALG_ASYNC | CRYPTO_ALG_KERN_DRIVER_ONLY |
4222 			 template->type;
4223 	switch (template->type) {
4224 	case CRYPTO_ALG_TYPE_GIVCIPHER:
4225 		alg->cra_type = &crypto_givcipher_type;
4226 		alg->cra_ablkcipher = template->template_ablkcipher;
4227 		break;
4228 	case CRYPTO_ALG_TYPE_ABLKCIPHER:
4229 		alg->cra_type = &crypto_ablkcipher_type;
4230 		alg->cra_ablkcipher = template->template_ablkcipher;
4231 		break;
4232 	case CRYPTO_ALG_TYPE_AEAD:
4233 		alg->cra_type = &crypto_aead_type;
4234 		alg->cra_aead = template->template_aead;
4235 		break;
4236 	}
4237 
4238 	t_alg->class1_alg_type = template->class1_alg_type;
4239 	t_alg->class2_alg_type = template->class2_alg_type;
4240 	t_alg->alg_op = template->alg_op;
4241 
4242 	return t_alg;
4243 }
4244 
4245 static int __init caam_algapi_init(void)
4246 {
4247 	struct device_node *dev_node;
4248 	struct platform_device *pdev;
4249 	struct device *ctrldev;
4250 	void *priv;
4251 	int i = 0, err = 0;
4252 
4253 	dev_node = of_find_compatible_node(NULL, NULL, "fsl,sec-v4.0");
4254 	if (!dev_node) {
4255 		dev_node = of_find_compatible_node(NULL, NULL, "fsl,sec4.0");
4256 		if (!dev_node)
4257 			return -ENODEV;
4258 	}
4259 
4260 	pdev = of_find_device_by_node(dev_node);
4261 	if (!pdev) {
4262 		of_node_put(dev_node);
4263 		return -ENODEV;
4264 	}
4265 
4266 	ctrldev = &pdev->dev;
4267 	priv = dev_get_drvdata(ctrldev);
4268 	of_node_put(dev_node);
4269 
4270 	/*
4271 	 * If priv is NULL, it's probably because the caam driver wasn't
4272 	 * properly initialized (e.g. RNG4 init failed). Thus, bail out here.
4273 	 */
4274 	if (!priv)
4275 		return -ENODEV;
4276 
4277 
4278 	INIT_LIST_HEAD(&alg_list);
4279 
4280 	/* register crypto algorithms the device supports */
4281 	for (i = 0; i < ARRAY_SIZE(driver_algs); i++) {
4282 		/* TODO: check if h/w supports alg */
4283 		struct caam_crypto_alg *t_alg;
4284 
4285 		t_alg = caam_alg_alloc(&driver_algs[i]);
4286 		if (IS_ERR(t_alg)) {
4287 			err = PTR_ERR(t_alg);
4288 			pr_warn("%s alg allocation failed\n",
4289 				driver_algs[i].driver_name);
4290 			continue;
4291 		}
4292 
4293 		err = crypto_register_alg(&t_alg->crypto_alg);
4294 		if (err) {
4295 			pr_warn("%s alg registration failed\n",
4296 				t_alg->crypto_alg.cra_driver_name);
4297 			kfree(t_alg);
4298 		} else
4299 			list_add_tail(&t_alg->entry, &alg_list);
4300 	}
4301 	if (!list_empty(&alg_list))
4302 		pr_info("caam algorithms registered in /proc/crypto\n");
4303 
4304 	return err;
4305 }
4306 
4307 module_init(caam_algapi_init);
4308 module_exit(caam_algapi_exit);
4309 
4310 MODULE_LICENSE("GPL");
4311 MODULE_DESCRIPTION("FSL CAAM support for crypto API");
4312 MODULE_AUTHOR("Freescale Semiconductor - NMG/STC");
4313