1 // SPDX-License-Identifier: GPL-2.0+
2 /*
3  * Shared descriptors for aead, skcipher algorithms
4  *
5  * Copyright 2016-2019 NXP
6  */
7 
8 #include "compat.h"
9 #include "desc_constr.h"
10 #include "caamalg_desc.h"
11 
12 /*
13  * For aead functions, read payload and write payload,
14  * both of which are specified in req->src and req->dst
15  */
16 static inline void aead_append_src_dst(u32 *desc, u32 msg_type)
17 {
18 	append_seq_fifo_store(desc, 0, FIFOST_TYPE_MESSAGE_DATA | KEY_VLF);
19 	append_seq_fifo_load(desc, 0, FIFOLD_CLASS_BOTH |
20 			     KEY_VLF | msg_type | FIFOLD_TYPE_LASTBOTH);
21 }
22 
23 /* Set DK bit in class 1 operation if shared */
24 static inline void append_dec_op1(u32 *desc, u32 type)
25 {
26 	u32 *jump_cmd, *uncond_jump_cmd;
27 
28 	/* DK bit is valid only for AES */
29 	if ((type & OP_ALG_ALGSEL_MASK) != OP_ALG_ALGSEL_AES) {
30 		append_operation(desc, type | OP_ALG_AS_INITFINAL |
31 				 OP_ALG_DECRYPT);
32 		return;
33 	}
34 
35 	jump_cmd = append_jump(desc, JUMP_TEST_ALL | JUMP_COND_SHRD);
36 	append_operation(desc, type | OP_ALG_AS_INIT | OP_ALG_DECRYPT);
37 	uncond_jump_cmd = append_jump(desc, JUMP_TEST_ALL);
38 	set_jump_tgt_here(desc, jump_cmd);
39 	append_operation(desc, type | OP_ALG_AS_INIT | OP_ALG_DECRYPT |
40 			 OP_ALG_AAI_DK);
41 	set_jump_tgt_here(desc, uncond_jump_cmd);
42 }
43 
44 /**
45  * cnstr_shdsc_aead_null_encap - IPSec ESP encapsulation shared descriptor
46  *                               (non-protocol) with no (null) encryption.
47  * @desc: pointer to buffer used for descriptor construction
48  * @adata: pointer to authentication transform definitions.
49  *         A split key is required for SEC Era < 6; the size of the split key
50  *         is specified in this case. Valid algorithm values - one of
51  *         OP_ALG_ALGSEL_{MD5, SHA1, SHA224, SHA256, SHA384, SHA512} ANDed
52  *         with OP_ALG_AAI_HMAC_PRECOMP.
53  * @icvsize: integrity check value (ICV) size (truncated or full)
54  * @era: SEC Era
55  */
56 void cnstr_shdsc_aead_null_encap(u32 * const desc, struct alginfo *adata,
57 				 unsigned int icvsize, int era)
58 {
59 	u32 *key_jump_cmd, *read_move_cmd, *write_move_cmd;
60 
61 	init_sh_desc(desc, HDR_SHARE_SERIAL);
62 
63 	/* Skip if already shared */
64 	key_jump_cmd = append_jump(desc, JUMP_JSL | JUMP_TEST_ALL |
65 				   JUMP_COND_SHRD);
66 	if (era < 6) {
67 		if (adata->key_inline)
68 			append_key_as_imm(desc, adata->key_virt,
69 					  adata->keylen_pad, adata->keylen,
70 					  CLASS_2 | KEY_DEST_MDHA_SPLIT |
71 					  KEY_ENC);
72 		else
73 			append_key(desc, adata->key_dma, adata->keylen,
74 				   CLASS_2 | KEY_DEST_MDHA_SPLIT | KEY_ENC);
75 	} else {
76 		append_proto_dkp(desc, adata);
77 	}
78 	set_jump_tgt_here(desc, key_jump_cmd);
79 
80 	/* assoclen + cryptlen = seqinlen */
81 	append_math_sub(desc, REG3, SEQINLEN, REG0, CAAM_CMD_SZ);
82 
83 	/* Prepare to read and write cryptlen + assoclen bytes */
84 	append_math_add(desc, VARSEQINLEN, ZERO, REG3, CAAM_CMD_SZ);
85 	append_math_add(desc, VARSEQOUTLEN, ZERO, REG3, CAAM_CMD_SZ);
86 
87 	/*
88 	 * MOVE_LEN opcode is not available in all SEC HW revisions,
89 	 * thus need to do some magic, i.e. self-patch the descriptor
90 	 * buffer.
91 	 */
92 	read_move_cmd = append_move(desc, MOVE_SRC_DESCBUF |
93 				    MOVE_DEST_MATH3 |
94 				    (0x6 << MOVE_LEN_SHIFT));
95 	write_move_cmd = append_move(desc, MOVE_SRC_MATH3 |
96 				     MOVE_DEST_DESCBUF |
97 				     MOVE_WAITCOMP |
98 				     (0x8 << MOVE_LEN_SHIFT));
99 
100 	/* Class 2 operation */
101 	append_operation(desc, adata->algtype | OP_ALG_AS_INITFINAL |
102 			 OP_ALG_ENCRYPT);
103 
104 	/* Read and write cryptlen bytes */
105 	aead_append_src_dst(desc, FIFOLD_TYPE_MSG | FIFOLD_TYPE_FLUSH1);
106 
107 	set_move_tgt_here(desc, read_move_cmd);
108 	set_move_tgt_here(desc, write_move_cmd);
109 	append_cmd(desc, CMD_LOAD | DISABLE_AUTO_INFO_FIFO);
110 	append_move(desc, MOVE_SRC_INFIFO_CL | MOVE_DEST_OUTFIFO |
111 		    MOVE_AUX_LS);
112 
113 	/* Write ICV */
114 	append_seq_store(desc, icvsize, LDST_CLASS_2_CCB |
115 			 LDST_SRCDST_BYTE_CONTEXT);
116 
117 	print_hex_dump_debug("aead null enc shdesc@" __stringify(__LINE__)": ",
118 			     DUMP_PREFIX_ADDRESS, 16, 4, desc, desc_bytes(desc),
119 			     1);
120 }
121 EXPORT_SYMBOL(cnstr_shdsc_aead_null_encap);
122 
123 /**
124  * cnstr_shdsc_aead_null_decap - IPSec ESP decapsulation shared descriptor
125  *                               (non-protocol) with no (null) decryption.
126  * @desc: pointer to buffer used for descriptor construction
127  * @adata: pointer to authentication transform definitions.
128  *         A split key is required for SEC Era < 6; the size of the split key
129  *         is specified in this case. Valid algorithm values - one of
130  *         OP_ALG_ALGSEL_{MD5, SHA1, SHA224, SHA256, SHA384, SHA512} ANDed
131  *         with OP_ALG_AAI_HMAC_PRECOMP.
132  * @icvsize: integrity check value (ICV) size (truncated or full)
133  * @era: SEC Era
134  */
135 void cnstr_shdsc_aead_null_decap(u32 * const desc, struct alginfo *adata,
136 				 unsigned int icvsize, int era)
137 {
138 	u32 *key_jump_cmd, *read_move_cmd, *write_move_cmd, *jump_cmd;
139 
140 	init_sh_desc(desc, HDR_SHARE_SERIAL);
141 
142 	/* Skip if already shared */
143 	key_jump_cmd = append_jump(desc, JUMP_JSL | JUMP_TEST_ALL |
144 				   JUMP_COND_SHRD);
145 	if (era < 6) {
146 		if (adata->key_inline)
147 			append_key_as_imm(desc, adata->key_virt,
148 					  adata->keylen_pad, adata->keylen,
149 					  CLASS_2 | KEY_DEST_MDHA_SPLIT |
150 					  KEY_ENC);
151 		else
152 			append_key(desc, adata->key_dma, adata->keylen,
153 				   CLASS_2 | KEY_DEST_MDHA_SPLIT | KEY_ENC);
154 	} else {
155 		append_proto_dkp(desc, adata);
156 	}
157 	set_jump_tgt_here(desc, key_jump_cmd);
158 
159 	/* Class 2 operation */
160 	append_operation(desc, adata->algtype | OP_ALG_AS_INITFINAL |
161 			 OP_ALG_DECRYPT | OP_ALG_ICV_ON);
162 
163 	/* assoclen + cryptlen = seqoutlen */
164 	append_math_sub(desc, REG2, SEQOUTLEN, REG0, CAAM_CMD_SZ);
165 
166 	/* Prepare to read and write cryptlen + assoclen bytes */
167 	append_math_add(desc, VARSEQINLEN, ZERO, REG2, CAAM_CMD_SZ);
168 	append_math_add(desc, VARSEQOUTLEN, ZERO, REG2, CAAM_CMD_SZ);
169 
170 	/*
171 	 * MOVE_LEN opcode is not available in all SEC HW revisions,
172 	 * thus need to do some magic, i.e. self-patch the descriptor
173 	 * buffer.
174 	 */
175 	read_move_cmd = append_move(desc, MOVE_SRC_DESCBUF |
176 				    MOVE_DEST_MATH2 |
177 				    (0x6 << MOVE_LEN_SHIFT));
178 	write_move_cmd = append_move(desc, MOVE_SRC_MATH2 |
179 				     MOVE_DEST_DESCBUF |
180 				     MOVE_WAITCOMP |
181 				     (0x8 << MOVE_LEN_SHIFT));
182 
183 	/* Read and write cryptlen bytes */
184 	aead_append_src_dst(desc, FIFOLD_TYPE_MSG | FIFOLD_TYPE_FLUSH1);
185 
186 	/*
187 	 * Insert a NOP here, since we need at least 4 instructions between
188 	 * code patching the descriptor buffer and the location being patched.
189 	 */
190 	jump_cmd = append_jump(desc, JUMP_TEST_ALL);
191 	set_jump_tgt_here(desc, jump_cmd);
192 
193 	set_move_tgt_here(desc, read_move_cmd);
194 	set_move_tgt_here(desc, write_move_cmd);
195 	append_cmd(desc, CMD_LOAD | DISABLE_AUTO_INFO_FIFO);
196 	append_move(desc, MOVE_SRC_INFIFO_CL | MOVE_DEST_OUTFIFO |
197 		    MOVE_AUX_LS);
198 	append_cmd(desc, CMD_LOAD | ENABLE_AUTO_INFO_FIFO);
199 
200 	/* Load ICV */
201 	append_seq_fifo_load(desc, icvsize, FIFOLD_CLASS_CLASS2 |
202 			     FIFOLD_TYPE_LAST2 | FIFOLD_TYPE_ICV);
203 
204 	print_hex_dump_debug("aead null dec shdesc@" __stringify(__LINE__)": ",
205 			     DUMP_PREFIX_ADDRESS, 16, 4, desc, desc_bytes(desc),
206 			     1);
207 }
208 EXPORT_SYMBOL(cnstr_shdsc_aead_null_decap);
209 
210 static void init_sh_desc_key_aead(u32 * const desc,
211 				  struct alginfo * const cdata,
212 				  struct alginfo * const adata,
213 				  const bool is_rfc3686, u32 *nonce, int era)
214 {
215 	u32 *key_jump_cmd;
216 	unsigned int enckeylen = cdata->keylen;
217 
218 	/* Note: Context registers are saved. */
219 	init_sh_desc(desc, HDR_SHARE_SERIAL | HDR_SAVECTX);
220 
221 	/* Skip if already shared */
222 	key_jump_cmd = append_jump(desc, JUMP_JSL | JUMP_TEST_ALL |
223 				   JUMP_COND_SHRD);
224 
225 	/*
226 	 * RFC3686 specific:
227 	 *	| key = {AUTH_KEY, ENC_KEY, NONCE}
228 	 *	| enckeylen = encryption key size + nonce size
229 	 */
230 	if (is_rfc3686)
231 		enckeylen -= CTR_RFC3686_NONCE_SIZE;
232 
233 	if (era < 6) {
234 		if (adata->key_inline)
235 			append_key_as_imm(desc, adata->key_virt,
236 					  adata->keylen_pad, adata->keylen,
237 					  CLASS_2 | KEY_DEST_MDHA_SPLIT |
238 					  KEY_ENC);
239 		else
240 			append_key(desc, adata->key_dma, adata->keylen,
241 				   CLASS_2 | KEY_DEST_MDHA_SPLIT | KEY_ENC);
242 	} else {
243 		append_proto_dkp(desc, adata);
244 	}
245 
246 	if (cdata->key_inline)
247 		append_key_as_imm(desc, cdata->key_virt, enckeylen,
248 				  enckeylen, CLASS_1 | KEY_DEST_CLASS_REG);
249 	else
250 		append_key(desc, cdata->key_dma, enckeylen, CLASS_1 |
251 			   KEY_DEST_CLASS_REG);
252 
253 	/* Load Counter into CONTEXT1 reg */
254 	if (is_rfc3686) {
255 		append_load_as_imm(desc, nonce, CTR_RFC3686_NONCE_SIZE,
256 				   LDST_CLASS_IND_CCB |
257 				   LDST_SRCDST_BYTE_OUTFIFO | LDST_IMM);
258 		append_move(desc,
259 			    MOVE_SRC_OUTFIFO |
260 			    MOVE_DEST_CLASS1CTX |
261 			    (16 << MOVE_OFFSET_SHIFT) |
262 			    (CTR_RFC3686_NONCE_SIZE << MOVE_LEN_SHIFT));
263 	}
264 
265 	set_jump_tgt_here(desc, key_jump_cmd);
266 }
267 
268 /**
269  * cnstr_shdsc_aead_encap - IPSec ESP encapsulation shared descriptor
270  *                          (non-protocol).
271  * @desc: pointer to buffer used for descriptor construction
272  * @cdata: pointer to block cipher transform definitions
273  *         Valid algorithm values - one of OP_ALG_ALGSEL_{AES, DES, 3DES} ANDed
274  *         with OP_ALG_AAI_CBC or OP_ALG_AAI_CTR_MOD128.
275  * @adata: pointer to authentication transform definitions.
276  *         A split key is required for SEC Era < 6; the size of the split key
277  *         is specified in this case. Valid algorithm values - one of
278  *         OP_ALG_ALGSEL_{MD5, SHA1, SHA224, SHA256, SHA384, SHA512} ANDed
279  *         with OP_ALG_AAI_HMAC_PRECOMP.
280  * @ivsize: initialization vector size
281  * @icvsize: integrity check value (ICV) size (truncated or full)
282  * @is_rfc3686: true when ctr(aes) is wrapped by rfc3686 template
283  * @nonce: pointer to rfc3686 nonce
284  * @ctx1_iv_off: IV offset in CONTEXT1 register
285  * @is_qi: true when called from caam/qi
286  * @era: SEC Era
287  */
288 void cnstr_shdsc_aead_encap(u32 * const desc, struct alginfo *cdata,
289 			    struct alginfo *adata, unsigned int ivsize,
290 			    unsigned int icvsize, const bool is_rfc3686,
291 			    u32 *nonce, const u32 ctx1_iv_off, const bool is_qi,
292 			    int era)
293 {
294 	/* Note: Context registers are saved. */
295 	init_sh_desc_key_aead(desc, cdata, adata, is_rfc3686, nonce, era);
296 
297 	/* Class 2 operation */
298 	append_operation(desc, adata->algtype | OP_ALG_AS_INITFINAL |
299 			 OP_ALG_ENCRYPT);
300 
301 	if (is_qi) {
302 		u32 *wait_load_cmd;
303 
304 		/* REG3 = assoclen */
305 		append_seq_load(desc, 4, LDST_CLASS_DECO |
306 				LDST_SRCDST_WORD_DECO_MATH3 |
307 				(4 << LDST_OFFSET_SHIFT));
308 
309 		wait_load_cmd = append_jump(desc, JUMP_JSL | JUMP_TEST_ALL |
310 					    JUMP_COND_CALM | JUMP_COND_NCP |
311 					    JUMP_COND_NOP | JUMP_COND_NIP |
312 					    JUMP_COND_NIFP);
313 		set_jump_tgt_here(desc, wait_load_cmd);
314 
315 		append_seq_load(desc, ivsize, LDST_CLASS_1_CCB |
316 				LDST_SRCDST_BYTE_CONTEXT |
317 				(ctx1_iv_off << LDST_OFFSET_SHIFT));
318 	}
319 
320 	/* Read and write assoclen bytes */
321 	if (is_qi || era < 3) {
322 		append_math_add(desc, VARSEQINLEN, ZERO, REG3, CAAM_CMD_SZ);
323 		append_math_add(desc, VARSEQOUTLEN, ZERO, REG3, CAAM_CMD_SZ);
324 	} else {
325 		append_math_add(desc, VARSEQINLEN, ZERO, DPOVRD, CAAM_CMD_SZ);
326 		append_math_add(desc, VARSEQOUTLEN, ZERO, DPOVRD, CAAM_CMD_SZ);
327 	}
328 
329 	/* Skip assoc data */
330 	append_seq_fifo_store(desc, 0, FIFOST_TYPE_SKIP | FIFOLDST_VLF);
331 
332 	/* read assoc before reading payload */
333 	append_seq_fifo_load(desc, 0, FIFOLD_CLASS_CLASS2 | FIFOLD_TYPE_MSG |
334 				      FIFOLDST_VLF);
335 
336 	/* Load Counter into CONTEXT1 reg */
337 	if (is_rfc3686)
338 		append_load_imm_be32(desc, 1, LDST_IMM | LDST_CLASS_1_CCB |
339 				     LDST_SRCDST_BYTE_CONTEXT |
340 				     ((ctx1_iv_off + CTR_RFC3686_IV_SIZE) <<
341 				      LDST_OFFSET_SHIFT));
342 
343 	/* Class 1 operation */
344 	append_operation(desc, cdata->algtype | OP_ALG_AS_INITFINAL |
345 			 OP_ALG_ENCRYPT);
346 
347 	/* Read and write cryptlen bytes */
348 	append_math_add(desc, VARSEQINLEN, SEQINLEN, REG0, CAAM_CMD_SZ);
349 	append_math_add(desc, VARSEQOUTLEN, SEQINLEN, REG0, CAAM_CMD_SZ);
350 	aead_append_src_dst(desc, FIFOLD_TYPE_MSG1OUT2);
351 
352 	/* Write ICV */
353 	append_seq_store(desc, icvsize, LDST_CLASS_2_CCB |
354 			 LDST_SRCDST_BYTE_CONTEXT);
355 
356 	print_hex_dump_debug("aead enc shdesc@" __stringify(__LINE__)": ",
357 			     DUMP_PREFIX_ADDRESS, 16, 4, desc, desc_bytes(desc),
358 			     1);
359 }
360 EXPORT_SYMBOL(cnstr_shdsc_aead_encap);
361 
362 /**
363  * cnstr_shdsc_aead_decap - IPSec ESP decapsulation shared descriptor
364  *                          (non-protocol).
365  * @desc: pointer to buffer used for descriptor construction
366  * @cdata: pointer to block cipher transform definitions
367  *         Valid algorithm values - one of OP_ALG_ALGSEL_{AES, DES, 3DES} ANDed
368  *         with OP_ALG_AAI_CBC or OP_ALG_AAI_CTR_MOD128.
369  * @adata: pointer to authentication transform definitions.
370  *         A split key is required for SEC Era < 6; the size of the split key
371  *         is specified in this case. Valid algorithm values - one of
372  *         OP_ALG_ALGSEL_{MD5, SHA1, SHA224, SHA256, SHA384, SHA512} ANDed
373  *         with OP_ALG_AAI_HMAC_PRECOMP.
374  * @ivsize: initialization vector size
375  * @icvsize: integrity check value (ICV) size (truncated or full)
376  * @geniv: whether to generate Encrypted Chain IV
377  * @is_rfc3686: true when ctr(aes) is wrapped by rfc3686 template
378  * @nonce: pointer to rfc3686 nonce
379  * @ctx1_iv_off: IV offset in CONTEXT1 register
380  * @is_qi: true when called from caam/qi
381  * @era: SEC Era
382  */
383 void cnstr_shdsc_aead_decap(u32 * const desc, struct alginfo *cdata,
384 			    struct alginfo *adata, unsigned int ivsize,
385 			    unsigned int icvsize, const bool geniv,
386 			    const bool is_rfc3686, u32 *nonce,
387 			    const u32 ctx1_iv_off, const bool is_qi, int era)
388 {
389 	/* Note: Context registers are saved. */
390 	init_sh_desc_key_aead(desc, cdata, adata, is_rfc3686, nonce, era);
391 
392 	/* Class 2 operation */
393 	append_operation(desc, adata->algtype | OP_ALG_AS_INITFINAL |
394 			 OP_ALG_DECRYPT | OP_ALG_ICV_ON);
395 
396 	if (is_qi) {
397 		u32 *wait_load_cmd;
398 
399 		/* REG3 = assoclen */
400 		append_seq_load(desc, 4, LDST_CLASS_DECO |
401 				LDST_SRCDST_WORD_DECO_MATH3 |
402 				(4 << LDST_OFFSET_SHIFT));
403 
404 		wait_load_cmd = append_jump(desc, JUMP_JSL | JUMP_TEST_ALL |
405 					    JUMP_COND_CALM | JUMP_COND_NCP |
406 					    JUMP_COND_NOP | JUMP_COND_NIP |
407 					    JUMP_COND_NIFP);
408 		set_jump_tgt_here(desc, wait_load_cmd);
409 
410 		if (!geniv)
411 			append_seq_load(desc, ivsize, LDST_CLASS_1_CCB |
412 					LDST_SRCDST_BYTE_CONTEXT |
413 					(ctx1_iv_off << LDST_OFFSET_SHIFT));
414 	}
415 
416 	/* Read and write assoclen bytes */
417 	if (is_qi || era < 3) {
418 		append_math_add(desc, VARSEQINLEN, ZERO, REG3, CAAM_CMD_SZ);
419 		if (geniv)
420 			append_math_add_imm_u32(desc, VARSEQOUTLEN, REG3, IMM,
421 						ivsize);
422 		else
423 			append_math_add(desc, VARSEQOUTLEN, ZERO, REG3,
424 					CAAM_CMD_SZ);
425 	} else {
426 		append_math_add(desc, VARSEQINLEN, ZERO, DPOVRD, CAAM_CMD_SZ);
427 		if (geniv)
428 			append_math_add_imm_u32(desc, VARSEQOUTLEN, DPOVRD, IMM,
429 						ivsize);
430 		else
431 			append_math_add(desc, VARSEQOUTLEN, ZERO, DPOVRD,
432 					CAAM_CMD_SZ);
433 	}
434 
435 	/* Skip assoc data */
436 	append_seq_fifo_store(desc, 0, FIFOST_TYPE_SKIP | FIFOLDST_VLF);
437 
438 	/* read assoc before reading payload */
439 	append_seq_fifo_load(desc, 0, FIFOLD_CLASS_CLASS2 | FIFOLD_TYPE_MSG |
440 			     KEY_VLF);
441 
442 	if (geniv) {
443 		append_seq_load(desc, ivsize, LDST_CLASS_1_CCB |
444 				LDST_SRCDST_BYTE_CONTEXT |
445 				(ctx1_iv_off << LDST_OFFSET_SHIFT));
446 		append_move(desc, MOVE_SRC_CLASS1CTX | MOVE_DEST_CLASS2INFIFO |
447 			    (ctx1_iv_off << MOVE_OFFSET_SHIFT) | ivsize);
448 	}
449 
450 	/* Load Counter into CONTEXT1 reg */
451 	if (is_rfc3686)
452 		append_load_imm_be32(desc, 1, LDST_IMM | LDST_CLASS_1_CCB |
453 				     LDST_SRCDST_BYTE_CONTEXT |
454 				     ((ctx1_iv_off + CTR_RFC3686_IV_SIZE) <<
455 				      LDST_OFFSET_SHIFT));
456 
457 	/* Choose operation */
458 	if (ctx1_iv_off)
459 		append_operation(desc, cdata->algtype | OP_ALG_AS_INITFINAL |
460 				 OP_ALG_DECRYPT);
461 	else
462 		append_dec_op1(desc, cdata->algtype);
463 
464 	/* Read and write cryptlen bytes */
465 	append_math_add(desc, VARSEQINLEN, SEQOUTLEN, REG0, CAAM_CMD_SZ);
466 	append_math_add(desc, VARSEQOUTLEN, SEQOUTLEN, REG0, CAAM_CMD_SZ);
467 	aead_append_src_dst(desc, FIFOLD_TYPE_MSG);
468 
469 	/* Load ICV */
470 	append_seq_fifo_load(desc, icvsize, FIFOLD_CLASS_CLASS2 |
471 			     FIFOLD_TYPE_LAST2 | FIFOLD_TYPE_ICV);
472 
473 	print_hex_dump_debug("aead dec shdesc@" __stringify(__LINE__)": ",
474 			     DUMP_PREFIX_ADDRESS, 16, 4, desc, desc_bytes(desc),
475 			     1);
476 }
477 EXPORT_SYMBOL(cnstr_shdsc_aead_decap);
478 
479 /**
480  * cnstr_shdsc_aead_givencap - IPSec ESP encapsulation shared descriptor
481  *                             (non-protocol) with HW-generated initialization
482  *                             vector.
483  * @desc: pointer to buffer used for descriptor construction
484  * @cdata: pointer to block cipher transform definitions
485  *         Valid algorithm values - one of OP_ALG_ALGSEL_{AES, DES, 3DES} ANDed
486  *         with OP_ALG_AAI_CBC or OP_ALG_AAI_CTR_MOD128.
487  * @adata: pointer to authentication transform definitions.
488  *         A split key is required for SEC Era < 6; the size of the split key
489  *         is specified in this case. Valid algorithm values - one of
490  *         OP_ALG_ALGSEL_{MD5, SHA1, SHA224, SHA256, SHA384, SHA512} ANDed
491  *         with OP_ALG_AAI_HMAC_PRECOMP.
492  * @ivsize: initialization vector size
493  * @icvsize: integrity check value (ICV) size (truncated or full)
494  * @is_rfc3686: true when ctr(aes) is wrapped by rfc3686 template
495  * @nonce: pointer to rfc3686 nonce
496  * @ctx1_iv_off: IV offset in CONTEXT1 register
497  * @is_qi: true when called from caam/qi
498  * @era: SEC Era
499  */
500 void cnstr_shdsc_aead_givencap(u32 * const desc, struct alginfo *cdata,
501 			       struct alginfo *adata, unsigned int ivsize,
502 			       unsigned int icvsize, const bool is_rfc3686,
503 			       u32 *nonce, const u32 ctx1_iv_off,
504 			       const bool is_qi, int era)
505 {
506 	u32 geniv, moveiv;
507 	u32 *wait_cmd;
508 
509 	/* Note: Context registers are saved. */
510 	init_sh_desc_key_aead(desc, cdata, adata, is_rfc3686, nonce, era);
511 
512 	if (is_qi) {
513 		u32 *wait_load_cmd;
514 
515 		/* REG3 = assoclen */
516 		append_seq_load(desc, 4, LDST_CLASS_DECO |
517 				LDST_SRCDST_WORD_DECO_MATH3 |
518 				(4 << LDST_OFFSET_SHIFT));
519 
520 		wait_load_cmd = append_jump(desc, JUMP_JSL | JUMP_TEST_ALL |
521 					    JUMP_COND_CALM | JUMP_COND_NCP |
522 					    JUMP_COND_NOP | JUMP_COND_NIP |
523 					    JUMP_COND_NIFP);
524 		set_jump_tgt_here(desc, wait_load_cmd);
525 	}
526 
527 	if (is_rfc3686) {
528 		if (is_qi)
529 			append_seq_load(desc, ivsize, LDST_CLASS_1_CCB |
530 					LDST_SRCDST_BYTE_CONTEXT |
531 					(ctx1_iv_off << LDST_OFFSET_SHIFT));
532 
533 		goto copy_iv;
534 	}
535 
536 	/* Generate IV */
537 	geniv = NFIFOENTRY_STYPE_PAD | NFIFOENTRY_DEST_DECO |
538 		NFIFOENTRY_DTYPE_MSG | NFIFOENTRY_LC1 |
539 		NFIFOENTRY_PTYPE_RND | (ivsize << NFIFOENTRY_DLEN_SHIFT);
540 	append_load_imm_u32(desc, geniv, LDST_CLASS_IND_CCB |
541 			    LDST_SRCDST_WORD_INFO_FIFO | LDST_IMM);
542 	append_cmd(desc, CMD_LOAD | DISABLE_AUTO_INFO_FIFO);
543 	append_move(desc, MOVE_WAITCOMP |
544 		    MOVE_SRC_INFIFO | MOVE_DEST_CLASS1CTX |
545 		    (ctx1_iv_off << MOVE_OFFSET_SHIFT) |
546 		    (ivsize << MOVE_LEN_SHIFT));
547 	append_cmd(desc, CMD_LOAD | ENABLE_AUTO_INFO_FIFO);
548 
549 copy_iv:
550 	/* Copy IV to class 1 context */
551 	append_move(desc, MOVE_SRC_CLASS1CTX | MOVE_DEST_OUTFIFO |
552 		    (ctx1_iv_off << MOVE_OFFSET_SHIFT) |
553 		    (ivsize << MOVE_LEN_SHIFT));
554 
555 	/* Return to encryption */
556 	append_operation(desc, adata->algtype | OP_ALG_AS_INITFINAL |
557 			 OP_ALG_ENCRYPT);
558 
559 	/* Read and write assoclen bytes */
560 	if (is_qi || era < 3) {
561 		append_math_add(desc, VARSEQINLEN, ZERO, REG3, CAAM_CMD_SZ);
562 		append_math_add(desc, VARSEQOUTLEN, ZERO, REG3, CAAM_CMD_SZ);
563 	} else {
564 		append_math_add(desc, VARSEQINLEN, ZERO, DPOVRD, CAAM_CMD_SZ);
565 		append_math_add(desc, VARSEQOUTLEN, ZERO, DPOVRD, CAAM_CMD_SZ);
566 	}
567 
568 	/* Skip assoc data */
569 	append_seq_fifo_store(desc, 0, FIFOST_TYPE_SKIP | FIFOLDST_VLF);
570 
571 	/* read assoc before reading payload */
572 	append_seq_fifo_load(desc, 0, FIFOLD_CLASS_CLASS2 | FIFOLD_TYPE_MSG |
573 			     KEY_VLF);
574 
575 	/* Copy iv from outfifo to class 2 fifo */
576 	moveiv = NFIFOENTRY_STYPE_OFIFO | NFIFOENTRY_DEST_CLASS2 |
577 		 NFIFOENTRY_DTYPE_MSG | (ivsize << NFIFOENTRY_DLEN_SHIFT);
578 	append_load_imm_u32(desc, moveiv, LDST_CLASS_IND_CCB |
579 			    LDST_SRCDST_WORD_INFO_FIFO | LDST_IMM);
580 	append_load_imm_u32(desc, ivsize, LDST_CLASS_2_CCB |
581 			    LDST_SRCDST_WORD_DATASZ_REG | LDST_IMM);
582 
583 	/* Load Counter into CONTEXT1 reg */
584 	if (is_rfc3686)
585 		append_load_imm_be32(desc, 1, LDST_IMM | LDST_CLASS_1_CCB |
586 				     LDST_SRCDST_BYTE_CONTEXT |
587 				     ((ctx1_iv_off + CTR_RFC3686_IV_SIZE) <<
588 				      LDST_OFFSET_SHIFT));
589 
590 	/* Class 1 operation */
591 	append_operation(desc, cdata->algtype | OP_ALG_AS_INITFINAL |
592 			 OP_ALG_ENCRYPT);
593 
594 	/* Will write ivsize + cryptlen */
595 	append_math_add(desc, VARSEQOUTLEN, SEQINLEN, REG0, CAAM_CMD_SZ);
596 
597 	/* Not need to reload iv */
598 	append_seq_fifo_load(desc, ivsize,
599 			     FIFOLD_CLASS_SKIP);
600 
601 	/* Will read cryptlen */
602 	append_math_add(desc, VARSEQINLEN, SEQINLEN, REG0, CAAM_CMD_SZ);
603 
604 	/*
605 	 * Wait for IV transfer (ofifo -> class2) to finish before starting
606 	 * ciphertext transfer (ofifo -> external memory).
607 	 */
608 	wait_cmd = append_jump(desc, JUMP_JSL | JUMP_TEST_ALL | JUMP_COND_NIFP);
609 	set_jump_tgt_here(desc, wait_cmd);
610 
611 	append_seq_fifo_load(desc, 0, FIFOLD_CLASS_BOTH | KEY_VLF |
612 			     FIFOLD_TYPE_MSG1OUT2 | FIFOLD_TYPE_LASTBOTH);
613 	append_seq_fifo_store(desc, 0, FIFOST_TYPE_MESSAGE_DATA | KEY_VLF);
614 
615 	/* Write ICV */
616 	append_seq_store(desc, icvsize, LDST_CLASS_2_CCB |
617 			 LDST_SRCDST_BYTE_CONTEXT);
618 
619 	print_hex_dump_debug("aead givenc shdesc@" __stringify(__LINE__)": ",
620 			     DUMP_PREFIX_ADDRESS, 16, 4, desc, desc_bytes(desc),
621 			     1);
622 }
623 EXPORT_SYMBOL(cnstr_shdsc_aead_givencap);
624 
625 /**
626  * cnstr_shdsc_gcm_encap - gcm encapsulation shared descriptor
627  * @desc: pointer to buffer used for descriptor construction
628  * @cdata: pointer to block cipher transform definitions
629  *         Valid algorithm values - OP_ALG_ALGSEL_AES ANDed with OP_ALG_AAI_GCM.
630  * @ivsize: initialization vector size
631  * @icvsize: integrity check value (ICV) size (truncated or full)
632  * @is_qi: true when called from caam/qi
633  */
634 void cnstr_shdsc_gcm_encap(u32 * const desc, struct alginfo *cdata,
635 			   unsigned int ivsize, unsigned int icvsize,
636 			   const bool is_qi)
637 {
638 	u32 *key_jump_cmd, *zero_payload_jump_cmd, *zero_assoc_jump_cmd1,
639 	    *zero_assoc_jump_cmd2;
640 
641 	init_sh_desc(desc, HDR_SHARE_SERIAL);
642 
643 	/* skip key loading if they are loaded due to sharing */
644 	key_jump_cmd = append_jump(desc, JUMP_JSL | JUMP_TEST_ALL |
645 				   JUMP_COND_SHRD);
646 	if (cdata->key_inline)
647 		append_key_as_imm(desc, cdata->key_virt, cdata->keylen,
648 				  cdata->keylen, CLASS_1 | KEY_DEST_CLASS_REG);
649 	else
650 		append_key(desc, cdata->key_dma, cdata->keylen, CLASS_1 |
651 			   KEY_DEST_CLASS_REG);
652 	set_jump_tgt_here(desc, key_jump_cmd);
653 
654 	/* class 1 operation */
655 	append_operation(desc, cdata->algtype | OP_ALG_AS_INITFINAL |
656 			 OP_ALG_ENCRYPT);
657 
658 	if (is_qi) {
659 		u32 *wait_load_cmd;
660 
661 		/* REG3 = assoclen */
662 		append_seq_load(desc, 4, LDST_CLASS_DECO |
663 				LDST_SRCDST_WORD_DECO_MATH3 |
664 				(4 << LDST_OFFSET_SHIFT));
665 
666 		wait_load_cmd = append_jump(desc, JUMP_JSL | JUMP_TEST_ALL |
667 					    JUMP_COND_CALM | JUMP_COND_NCP |
668 					    JUMP_COND_NOP | JUMP_COND_NIP |
669 					    JUMP_COND_NIFP);
670 		set_jump_tgt_here(desc, wait_load_cmd);
671 
672 		append_math_sub_imm_u32(desc, VARSEQOUTLEN, SEQINLEN, IMM,
673 					ivsize);
674 	} else {
675 		append_math_sub(desc, VARSEQOUTLEN, SEQINLEN, REG0,
676 				CAAM_CMD_SZ);
677 	}
678 
679 	/* if assoclen + cryptlen is ZERO, skip to ICV write */
680 	zero_assoc_jump_cmd2 = append_jump(desc, JUMP_TEST_ALL |
681 						 JUMP_COND_MATH_Z);
682 
683 	if (is_qi)
684 		append_seq_fifo_load(desc, ivsize, FIFOLD_CLASS_CLASS1 |
685 				     FIFOLD_TYPE_IV | FIFOLD_TYPE_FLUSH1);
686 
687 	/* if assoclen is ZERO, skip reading the assoc data */
688 	append_math_add(desc, VARSEQINLEN, ZERO, REG3, CAAM_CMD_SZ);
689 	zero_assoc_jump_cmd1 = append_jump(desc, JUMP_TEST_ALL |
690 					   JUMP_COND_MATH_Z);
691 
692 	append_math_add(desc, VARSEQOUTLEN, ZERO, REG3, CAAM_CMD_SZ);
693 
694 	/* skip assoc data */
695 	append_seq_fifo_store(desc, 0, FIFOST_TYPE_SKIP | FIFOLDST_VLF);
696 
697 	/* cryptlen = seqinlen - assoclen */
698 	append_math_sub(desc, VARSEQOUTLEN, SEQINLEN, REG3, CAAM_CMD_SZ);
699 
700 	/* if cryptlen is ZERO jump to zero-payload commands */
701 	zero_payload_jump_cmd = append_jump(desc, JUMP_TEST_ALL |
702 					    JUMP_COND_MATH_Z);
703 
704 	/* read assoc data */
705 	append_seq_fifo_load(desc, 0, FIFOLD_CLASS_CLASS1 | FIFOLDST_VLF |
706 			     FIFOLD_TYPE_AAD | FIFOLD_TYPE_FLUSH1);
707 	set_jump_tgt_here(desc, zero_assoc_jump_cmd1);
708 
709 	append_math_sub(desc, VARSEQINLEN, SEQINLEN, REG0, CAAM_CMD_SZ);
710 
711 	/* write encrypted data */
712 	append_seq_fifo_store(desc, 0, FIFOST_TYPE_MESSAGE_DATA | FIFOLDST_VLF);
713 
714 	/* read payload data */
715 	append_seq_fifo_load(desc, 0, FIFOLD_CLASS_CLASS1 | FIFOLDST_VLF |
716 			     FIFOLD_TYPE_MSG | FIFOLD_TYPE_LAST1);
717 
718 	/* jump to ICV writing */
719 	if (is_qi)
720 		append_jump(desc, JUMP_TEST_ALL | 4);
721 	else
722 		append_jump(desc, JUMP_TEST_ALL | 2);
723 
724 	/* zero-payload commands */
725 	set_jump_tgt_here(desc, zero_payload_jump_cmd);
726 
727 	/* read assoc data */
728 	append_seq_fifo_load(desc, 0, FIFOLD_CLASS_CLASS1 | FIFOLDST_VLF |
729 			     FIFOLD_TYPE_AAD | FIFOLD_TYPE_LAST1);
730 	if (is_qi)
731 		/* jump to ICV writing */
732 		append_jump(desc, JUMP_TEST_ALL | 2);
733 
734 	/* There is no input data */
735 	set_jump_tgt_here(desc, zero_assoc_jump_cmd2);
736 
737 	if (is_qi)
738 		append_seq_fifo_load(desc, ivsize, FIFOLD_CLASS_CLASS1 |
739 				     FIFOLD_TYPE_IV | FIFOLD_TYPE_FLUSH1 |
740 				     FIFOLD_TYPE_LAST1);
741 
742 	/* write ICV */
743 	append_seq_store(desc, icvsize, LDST_CLASS_1_CCB |
744 			 LDST_SRCDST_BYTE_CONTEXT);
745 
746 	print_hex_dump_debug("gcm enc shdesc@" __stringify(__LINE__)": ",
747 			     DUMP_PREFIX_ADDRESS, 16, 4, desc, desc_bytes(desc),
748 			     1);
749 }
750 EXPORT_SYMBOL(cnstr_shdsc_gcm_encap);
751 
752 /**
753  * cnstr_shdsc_gcm_decap - gcm decapsulation shared descriptor
754  * @desc: pointer to buffer used for descriptor construction
755  * @cdata: pointer to block cipher transform definitions
756  *         Valid algorithm values - OP_ALG_ALGSEL_AES ANDed with OP_ALG_AAI_GCM.
757  * @ivsize: initialization vector size
758  * @icvsize: integrity check value (ICV) size (truncated or full)
759  * @is_qi: true when called from caam/qi
760  */
761 void cnstr_shdsc_gcm_decap(u32 * const desc, struct alginfo *cdata,
762 			   unsigned int ivsize, unsigned int icvsize,
763 			   const bool is_qi)
764 {
765 	u32 *key_jump_cmd, *zero_payload_jump_cmd, *zero_assoc_jump_cmd1;
766 
767 	init_sh_desc(desc, HDR_SHARE_SERIAL);
768 
769 	/* skip key loading if they are loaded due to sharing */
770 	key_jump_cmd = append_jump(desc, JUMP_JSL |
771 				   JUMP_TEST_ALL | JUMP_COND_SHRD);
772 	if (cdata->key_inline)
773 		append_key_as_imm(desc, cdata->key_virt, cdata->keylen,
774 				  cdata->keylen, CLASS_1 | KEY_DEST_CLASS_REG);
775 	else
776 		append_key(desc, cdata->key_dma, cdata->keylen, CLASS_1 |
777 			   KEY_DEST_CLASS_REG);
778 	set_jump_tgt_here(desc, key_jump_cmd);
779 
780 	/* class 1 operation */
781 	append_operation(desc, cdata->algtype | OP_ALG_AS_INITFINAL |
782 			 OP_ALG_DECRYPT | OP_ALG_ICV_ON);
783 
784 	if (is_qi) {
785 		u32 *wait_load_cmd;
786 
787 		/* REG3 = assoclen */
788 		append_seq_load(desc, 4, LDST_CLASS_DECO |
789 				LDST_SRCDST_WORD_DECO_MATH3 |
790 				(4 << LDST_OFFSET_SHIFT));
791 
792 		wait_load_cmd = append_jump(desc, JUMP_JSL | JUMP_TEST_ALL |
793 					    JUMP_COND_CALM | JUMP_COND_NCP |
794 					    JUMP_COND_NOP | JUMP_COND_NIP |
795 					    JUMP_COND_NIFP);
796 		set_jump_tgt_here(desc, wait_load_cmd);
797 
798 		append_seq_fifo_load(desc, ivsize, FIFOLD_CLASS_CLASS1 |
799 				     FIFOLD_TYPE_IV | FIFOLD_TYPE_FLUSH1);
800 	}
801 
802 	/* if assoclen is ZERO, skip reading the assoc data */
803 	append_math_add(desc, VARSEQINLEN, ZERO, REG3, CAAM_CMD_SZ);
804 	zero_assoc_jump_cmd1 = append_jump(desc, JUMP_TEST_ALL |
805 						 JUMP_COND_MATH_Z);
806 
807 	append_math_add(desc, VARSEQOUTLEN, ZERO, REG3, CAAM_CMD_SZ);
808 
809 	/* skip assoc data */
810 	append_seq_fifo_store(desc, 0, FIFOST_TYPE_SKIP | FIFOLDST_VLF);
811 
812 	/* read assoc data */
813 	append_seq_fifo_load(desc, 0, FIFOLD_CLASS_CLASS1 | FIFOLDST_VLF |
814 			     FIFOLD_TYPE_AAD | FIFOLD_TYPE_FLUSH1);
815 
816 	set_jump_tgt_here(desc, zero_assoc_jump_cmd1);
817 
818 	/* cryptlen = seqoutlen - assoclen */
819 	append_math_sub(desc, VARSEQINLEN, SEQOUTLEN, REG0, CAAM_CMD_SZ);
820 
821 	/* jump to zero-payload command if cryptlen is zero */
822 	zero_payload_jump_cmd = append_jump(desc, JUMP_TEST_ALL |
823 					    JUMP_COND_MATH_Z);
824 
825 	append_math_sub(desc, VARSEQOUTLEN, SEQOUTLEN, REG0, CAAM_CMD_SZ);
826 
827 	/* store encrypted data */
828 	append_seq_fifo_store(desc, 0, FIFOST_TYPE_MESSAGE_DATA | FIFOLDST_VLF);
829 
830 	/* read payload data */
831 	append_seq_fifo_load(desc, 0, FIFOLD_CLASS_CLASS1 | FIFOLDST_VLF |
832 			     FIFOLD_TYPE_MSG | FIFOLD_TYPE_FLUSH1);
833 
834 	/* zero-payload command */
835 	set_jump_tgt_here(desc, zero_payload_jump_cmd);
836 
837 	/* read ICV */
838 	append_seq_fifo_load(desc, icvsize, FIFOLD_CLASS_CLASS1 |
839 			     FIFOLD_TYPE_ICV | FIFOLD_TYPE_LAST1);
840 
841 	print_hex_dump_debug("gcm dec shdesc@" __stringify(__LINE__)": ",
842 			     DUMP_PREFIX_ADDRESS, 16, 4, desc, desc_bytes(desc),
843 			     1);
844 }
845 EXPORT_SYMBOL(cnstr_shdsc_gcm_decap);
846 
847 /**
848  * cnstr_shdsc_rfc4106_encap - IPSec ESP gcm encapsulation shared descriptor
849  *                             (non-protocol).
850  * @desc: pointer to buffer used for descriptor construction
851  * @cdata: pointer to block cipher transform definitions
852  *         Valid algorithm values - OP_ALG_ALGSEL_AES ANDed with OP_ALG_AAI_GCM.
853  * @ivsize: initialization vector size
854  * @icvsize: integrity check value (ICV) size (truncated or full)
855  * @is_qi: true when called from caam/qi
856  *
857  * Input sequence: AAD | PTXT
858  * Output sequence: AAD | CTXT | ICV
859  * AAD length (assoclen), which includes the IV length, is available in Math3.
860  */
861 void cnstr_shdsc_rfc4106_encap(u32 * const desc, struct alginfo *cdata,
862 			       unsigned int ivsize, unsigned int icvsize,
863 			       const bool is_qi)
864 {
865 	u32 *key_jump_cmd, *zero_cryptlen_jump_cmd, *skip_instructions;
866 	init_sh_desc(desc, HDR_SHARE_SERIAL);
867 
868 	/* Skip key loading if it is loaded due to sharing */
869 	key_jump_cmd = append_jump(desc, JUMP_JSL | JUMP_TEST_ALL |
870 				   JUMP_COND_SHRD);
871 	if (cdata->key_inline)
872 		append_key_as_imm(desc, cdata->key_virt, cdata->keylen,
873 				  cdata->keylen, CLASS_1 | KEY_DEST_CLASS_REG);
874 	else
875 		append_key(desc, cdata->key_dma, cdata->keylen, CLASS_1 |
876 			   KEY_DEST_CLASS_REG);
877 	set_jump_tgt_here(desc, key_jump_cmd);
878 
879 	/* Class 1 operation */
880 	append_operation(desc, cdata->algtype | OP_ALG_AS_INITFINAL |
881 			 OP_ALG_ENCRYPT);
882 
883 	if (is_qi) {
884 		u32 *wait_load_cmd;
885 
886 		/* REG3 = assoclen */
887 		append_seq_load(desc, 4, LDST_CLASS_DECO |
888 				LDST_SRCDST_WORD_DECO_MATH3 |
889 				(4 << LDST_OFFSET_SHIFT));
890 
891 		wait_load_cmd = append_jump(desc, JUMP_JSL | JUMP_TEST_ALL |
892 					    JUMP_COND_CALM | JUMP_COND_NCP |
893 					    JUMP_COND_NOP | JUMP_COND_NIP |
894 					    JUMP_COND_NIFP);
895 		set_jump_tgt_here(desc, wait_load_cmd);
896 
897 		/* Read salt and IV */
898 		append_fifo_load_as_imm(desc, (void *)(cdata->key_virt +
899 					cdata->keylen), 4, FIFOLD_CLASS_CLASS1 |
900 					FIFOLD_TYPE_IV);
901 		append_seq_fifo_load(desc, ivsize, FIFOLD_CLASS_CLASS1 |
902 				     FIFOLD_TYPE_IV | FIFOLD_TYPE_FLUSH1);
903 	}
904 
905 	append_math_sub_imm_u32(desc, VARSEQINLEN, REG3, IMM, ivsize);
906 	append_math_add(desc, VARSEQOUTLEN, ZERO, REG3, CAAM_CMD_SZ);
907 
908 	/* Skip AAD */
909 	append_seq_fifo_store(desc, 0, FIFOST_TYPE_SKIP | FIFOLDST_VLF);
910 
911 	/* Read cryptlen and set this value into VARSEQOUTLEN */
912 	append_math_sub(desc, VARSEQOUTLEN, SEQINLEN, REG3, CAAM_CMD_SZ);
913 
914 	/* If cryptlen is ZERO jump to AAD command */
915 	zero_cryptlen_jump_cmd = append_jump(desc, JUMP_TEST_ALL |
916 					    JUMP_COND_MATH_Z);
917 
918 	/* Read AAD data */
919 	append_seq_fifo_load(desc, 0, FIFOLD_CLASS_CLASS1 | FIFOLDST_VLF |
920 			     FIFOLD_TYPE_AAD | FIFOLD_TYPE_FLUSH1);
921 
922 	/* Workaround for erratum A-005473 (simultaneous SEQ FIFO skips) */
923 	append_seq_fifo_store(desc, 0, FIFOST_TYPE_MESSAGE_DATA);
924 
925 	/* Skip IV */
926 	append_seq_fifo_load(desc, ivsize, FIFOLD_CLASS_SKIP);
927 	append_math_add(desc, VARSEQINLEN, VARSEQOUTLEN, REG0, CAAM_CMD_SZ);
928 
929 	/* Write encrypted data */
930 	append_seq_fifo_store(desc, 0, FIFOST_TYPE_MESSAGE_DATA | FIFOLDST_VLF);
931 
932 	/* Read payload data */
933 	append_seq_fifo_load(desc, 0, FIFOLD_CLASS_CLASS1 | FIFOLDST_VLF |
934 			     FIFOLD_TYPE_MSG | FIFOLD_TYPE_LAST1);
935 
936 	/* Jump instructions to avoid double reading of AAD */
937 	skip_instructions = append_jump(desc, JUMP_TEST_ALL);
938 
939 	/* There is no input data, cryptlen = 0 */
940 	set_jump_tgt_here(desc, zero_cryptlen_jump_cmd);
941 
942 	/* Read AAD */
943 	append_seq_fifo_load(desc, 0, FIFOLD_CLASS_CLASS1 | FIFOLDST_VLF |
944 			     FIFOLD_TYPE_AAD | FIFOLD_TYPE_LAST1);
945 
946 	set_jump_tgt_here(desc, skip_instructions);
947 
948 	/* Write ICV */
949 	append_seq_store(desc, icvsize, LDST_CLASS_1_CCB |
950 			 LDST_SRCDST_BYTE_CONTEXT);
951 
952 	print_hex_dump_debug("rfc4106 enc shdesc@" __stringify(__LINE__)": ",
953 			     DUMP_PREFIX_ADDRESS, 16, 4, desc, desc_bytes(desc),
954 			     1);
955 }
956 EXPORT_SYMBOL(cnstr_shdsc_rfc4106_encap);
957 
958 /**
959  * cnstr_shdsc_rfc4106_decap - IPSec ESP gcm decapsulation shared descriptor
960  *                             (non-protocol).
961  * @desc: pointer to buffer used for descriptor construction
962  * @cdata: pointer to block cipher transform definitions
963  *         Valid algorithm values - OP_ALG_ALGSEL_AES ANDed with OP_ALG_AAI_GCM.
964  * @ivsize: initialization vector size
965  * @icvsize: integrity check value (ICV) size (truncated or full)
966  * @is_qi: true when called from caam/qi
967  */
968 void cnstr_shdsc_rfc4106_decap(u32 * const desc, struct alginfo *cdata,
969 			       unsigned int ivsize, unsigned int icvsize,
970 			       const bool is_qi)
971 {
972 	u32 *key_jump_cmd;
973 
974 	init_sh_desc(desc, HDR_SHARE_SERIAL);
975 
976 	/* Skip key loading if it is loaded due to sharing */
977 	key_jump_cmd = append_jump(desc, JUMP_JSL | JUMP_TEST_ALL |
978 				   JUMP_COND_SHRD);
979 	if (cdata->key_inline)
980 		append_key_as_imm(desc, cdata->key_virt, cdata->keylen,
981 				  cdata->keylen, CLASS_1 |
982 				  KEY_DEST_CLASS_REG);
983 	else
984 		append_key(desc, cdata->key_dma, cdata->keylen, CLASS_1 |
985 			   KEY_DEST_CLASS_REG);
986 	set_jump_tgt_here(desc, key_jump_cmd);
987 
988 	/* Class 1 operation */
989 	append_operation(desc, cdata->algtype | OP_ALG_AS_INITFINAL |
990 			 OP_ALG_DECRYPT | OP_ALG_ICV_ON);
991 
992 	if (is_qi) {
993 		u32 *wait_load_cmd;
994 
995 		/* REG3 = assoclen */
996 		append_seq_load(desc, 4, LDST_CLASS_DECO |
997 				LDST_SRCDST_WORD_DECO_MATH3 |
998 				(4 << LDST_OFFSET_SHIFT));
999 
1000 		wait_load_cmd = append_jump(desc, JUMP_JSL | JUMP_TEST_ALL |
1001 					    JUMP_COND_CALM | JUMP_COND_NCP |
1002 					    JUMP_COND_NOP | JUMP_COND_NIP |
1003 					    JUMP_COND_NIFP);
1004 		set_jump_tgt_here(desc, wait_load_cmd);
1005 
1006 		/* Read salt and IV */
1007 		append_fifo_load_as_imm(desc, (void *)(cdata->key_virt +
1008 					cdata->keylen), 4, FIFOLD_CLASS_CLASS1 |
1009 					FIFOLD_TYPE_IV);
1010 		append_seq_fifo_load(desc, ivsize, FIFOLD_CLASS_CLASS1 |
1011 				     FIFOLD_TYPE_IV | FIFOLD_TYPE_FLUSH1);
1012 	}
1013 
1014 	append_math_sub_imm_u32(desc, VARSEQINLEN, REG3, IMM, ivsize);
1015 	append_math_add(desc, VARSEQOUTLEN, ZERO, REG3, CAAM_CMD_SZ);
1016 
1017 	/* Read assoc data */
1018 	append_seq_fifo_load(desc, 0, FIFOLD_CLASS_CLASS1 | FIFOLDST_VLF |
1019 			     FIFOLD_TYPE_AAD | FIFOLD_TYPE_FLUSH1);
1020 
1021 	/* Skip IV */
1022 	append_seq_fifo_load(desc, ivsize, FIFOLD_CLASS_SKIP);
1023 
1024 	/* Will read cryptlen bytes */
1025 	append_math_sub(desc, VARSEQINLEN, SEQOUTLEN, REG3, CAAM_CMD_SZ);
1026 
1027 	/* Workaround for erratum A-005473 (simultaneous SEQ FIFO skips) */
1028 	append_seq_fifo_load(desc, 0, FIFOLD_CLASS_CLASS1 | FIFOLD_TYPE_MSG);
1029 
1030 	/* Skip assoc data */
1031 	append_seq_fifo_store(desc, 0, FIFOST_TYPE_SKIP | FIFOLDST_VLF);
1032 
1033 	/* Will write cryptlen bytes */
1034 	append_math_sub(desc, VARSEQOUTLEN, SEQOUTLEN, REG0, CAAM_CMD_SZ);
1035 
1036 	/* Store payload data */
1037 	append_seq_fifo_store(desc, 0, FIFOST_TYPE_MESSAGE_DATA | FIFOLDST_VLF);
1038 
1039 	/* Read encrypted data */
1040 	append_seq_fifo_load(desc, 0, FIFOLD_CLASS_CLASS1 | FIFOLDST_VLF |
1041 			     FIFOLD_TYPE_MSG | FIFOLD_TYPE_FLUSH1);
1042 
1043 	/* Read ICV */
1044 	append_seq_fifo_load(desc, icvsize, FIFOLD_CLASS_CLASS1 |
1045 			     FIFOLD_TYPE_ICV | FIFOLD_TYPE_LAST1);
1046 
1047 	print_hex_dump_debug("rfc4106 dec shdesc@" __stringify(__LINE__)": ",
1048 			     DUMP_PREFIX_ADDRESS, 16, 4, desc, desc_bytes(desc),
1049 			     1);
1050 }
1051 EXPORT_SYMBOL(cnstr_shdsc_rfc4106_decap);
1052 
1053 /**
1054  * cnstr_shdsc_rfc4543_encap - IPSec ESP gmac encapsulation shared descriptor
1055  *                             (non-protocol).
1056  * @desc: pointer to buffer used for descriptor construction
1057  * @cdata: pointer to block cipher transform definitions
1058  *         Valid algorithm values - OP_ALG_ALGSEL_AES ANDed with OP_ALG_AAI_GCM.
1059  * @ivsize: initialization vector size
1060  * @icvsize: integrity check value (ICV) size (truncated or full)
1061  * @is_qi: true when called from caam/qi
1062  */
1063 void cnstr_shdsc_rfc4543_encap(u32 * const desc, struct alginfo *cdata,
1064 			       unsigned int ivsize, unsigned int icvsize,
1065 			       const bool is_qi)
1066 {
1067 	u32 *key_jump_cmd, *read_move_cmd, *write_move_cmd;
1068 
1069 	init_sh_desc(desc, HDR_SHARE_SERIAL);
1070 
1071 	/* Skip key loading if it is loaded due to sharing */
1072 	key_jump_cmd = append_jump(desc, JUMP_JSL | JUMP_TEST_ALL |
1073 				   JUMP_COND_SHRD);
1074 	if (cdata->key_inline)
1075 		append_key_as_imm(desc, cdata->key_virt, cdata->keylen,
1076 				  cdata->keylen, CLASS_1 | KEY_DEST_CLASS_REG);
1077 	else
1078 		append_key(desc, cdata->key_dma, cdata->keylen, CLASS_1 |
1079 			   KEY_DEST_CLASS_REG);
1080 	set_jump_tgt_here(desc, key_jump_cmd);
1081 
1082 	/* Class 1 operation */
1083 	append_operation(desc, cdata->algtype | OP_ALG_AS_INITFINAL |
1084 			 OP_ALG_ENCRYPT);
1085 
1086 	if (is_qi) {
1087 		/* assoclen is not needed, skip it */
1088 		append_seq_fifo_load(desc, 4, FIFOLD_CLASS_SKIP);
1089 
1090 		/* Read salt and IV */
1091 		append_fifo_load_as_imm(desc, (void *)(cdata->key_virt +
1092 					cdata->keylen), 4, FIFOLD_CLASS_CLASS1 |
1093 					FIFOLD_TYPE_IV);
1094 		append_seq_fifo_load(desc, ivsize, FIFOLD_CLASS_CLASS1 |
1095 				     FIFOLD_TYPE_IV | FIFOLD_TYPE_FLUSH1);
1096 	}
1097 
1098 	/* assoclen + cryptlen = seqinlen */
1099 	append_math_sub(desc, REG3, SEQINLEN, REG0, CAAM_CMD_SZ);
1100 
1101 	/*
1102 	 * MOVE_LEN opcode is not available in all SEC HW revisions,
1103 	 * thus need to do some magic, i.e. self-patch the descriptor
1104 	 * buffer.
1105 	 */
1106 	read_move_cmd = append_move(desc, MOVE_SRC_DESCBUF | MOVE_DEST_MATH3 |
1107 				    (0x6 << MOVE_LEN_SHIFT));
1108 	write_move_cmd = append_move(desc, MOVE_SRC_MATH3 | MOVE_DEST_DESCBUF |
1109 				     (0x8 << MOVE_LEN_SHIFT) | MOVE_WAITCOMP);
1110 
1111 	/* Will read assoclen + cryptlen bytes */
1112 	append_math_sub(desc, VARSEQINLEN, SEQINLEN, REG0, CAAM_CMD_SZ);
1113 
1114 	/* Will write assoclen + cryptlen bytes */
1115 	append_math_sub(desc, VARSEQOUTLEN, SEQINLEN, REG0, CAAM_CMD_SZ);
1116 
1117 	/* Read and write assoclen + cryptlen bytes */
1118 	aead_append_src_dst(desc, FIFOLD_TYPE_AAD);
1119 
1120 	set_move_tgt_here(desc, read_move_cmd);
1121 	set_move_tgt_here(desc, write_move_cmd);
1122 	append_cmd(desc, CMD_LOAD | DISABLE_AUTO_INFO_FIFO);
1123 	/* Move payload data to OFIFO */
1124 	append_move(desc, MOVE_SRC_INFIFO_CL | MOVE_DEST_OUTFIFO);
1125 
1126 	/* Write ICV */
1127 	append_seq_store(desc, icvsize, LDST_CLASS_1_CCB |
1128 			 LDST_SRCDST_BYTE_CONTEXT);
1129 
1130 	print_hex_dump_debug("rfc4543 enc shdesc@" __stringify(__LINE__)": ",
1131 			     DUMP_PREFIX_ADDRESS, 16, 4, desc, desc_bytes(desc),
1132 			     1);
1133 }
1134 EXPORT_SYMBOL(cnstr_shdsc_rfc4543_encap);
1135 
1136 /**
1137  * cnstr_shdsc_rfc4543_decap - IPSec ESP gmac decapsulation shared descriptor
1138  *                             (non-protocol).
1139  * @desc: pointer to buffer used for descriptor construction
1140  * @cdata: pointer to block cipher transform definitions
1141  *         Valid algorithm values - OP_ALG_ALGSEL_AES ANDed with OP_ALG_AAI_GCM.
1142  * @ivsize: initialization vector size
1143  * @icvsize: integrity check value (ICV) size (truncated or full)
1144  * @is_qi: true when called from caam/qi
1145  */
1146 void cnstr_shdsc_rfc4543_decap(u32 * const desc, struct alginfo *cdata,
1147 			       unsigned int ivsize, unsigned int icvsize,
1148 			       const bool is_qi)
1149 {
1150 	u32 *key_jump_cmd, *read_move_cmd, *write_move_cmd;
1151 
1152 	init_sh_desc(desc, HDR_SHARE_SERIAL);
1153 
1154 	/* Skip key loading if it is loaded due to sharing */
1155 	key_jump_cmd = append_jump(desc, JUMP_JSL | JUMP_TEST_ALL |
1156 				   JUMP_COND_SHRD);
1157 	if (cdata->key_inline)
1158 		append_key_as_imm(desc, cdata->key_virt, cdata->keylen,
1159 				  cdata->keylen, CLASS_1 | KEY_DEST_CLASS_REG);
1160 	else
1161 		append_key(desc, cdata->key_dma, cdata->keylen, CLASS_1 |
1162 			   KEY_DEST_CLASS_REG);
1163 	set_jump_tgt_here(desc, key_jump_cmd);
1164 
1165 	/* Class 1 operation */
1166 	append_operation(desc, cdata->algtype | OP_ALG_AS_INITFINAL |
1167 			 OP_ALG_DECRYPT | OP_ALG_ICV_ON);
1168 
1169 	if (is_qi) {
1170 		/* assoclen is not needed, skip it */
1171 		append_seq_fifo_load(desc, 4, FIFOLD_CLASS_SKIP);
1172 
1173 		/* Read salt and IV */
1174 		append_fifo_load_as_imm(desc, (void *)(cdata->key_virt +
1175 					cdata->keylen), 4, FIFOLD_CLASS_CLASS1 |
1176 					FIFOLD_TYPE_IV);
1177 		append_seq_fifo_load(desc, ivsize, FIFOLD_CLASS_CLASS1 |
1178 				     FIFOLD_TYPE_IV | FIFOLD_TYPE_FLUSH1);
1179 	}
1180 
1181 	/* assoclen + cryptlen = seqoutlen */
1182 	append_math_sub(desc, REG3, SEQOUTLEN, REG0, CAAM_CMD_SZ);
1183 
1184 	/*
1185 	 * MOVE_LEN opcode is not available in all SEC HW revisions,
1186 	 * thus need to do some magic, i.e. self-patch the descriptor
1187 	 * buffer.
1188 	 */
1189 	read_move_cmd = append_move(desc, MOVE_SRC_DESCBUF | MOVE_DEST_MATH3 |
1190 				    (0x6 << MOVE_LEN_SHIFT));
1191 	write_move_cmd = append_move(desc, MOVE_SRC_MATH3 | MOVE_DEST_DESCBUF |
1192 				     (0x8 << MOVE_LEN_SHIFT) | MOVE_WAITCOMP);
1193 
1194 	/* Will read assoclen + cryptlen bytes */
1195 	append_math_sub(desc, VARSEQINLEN, SEQOUTLEN, REG0, CAAM_CMD_SZ);
1196 
1197 	/* Will write assoclen + cryptlen bytes */
1198 	append_math_sub(desc, VARSEQOUTLEN, SEQOUTLEN, REG0, CAAM_CMD_SZ);
1199 
1200 	/* Store payload data */
1201 	append_seq_fifo_store(desc, 0, FIFOST_TYPE_MESSAGE_DATA | FIFOLDST_VLF);
1202 
1203 	/* In-snoop assoclen + cryptlen data */
1204 	append_seq_fifo_load(desc, 0, FIFOLD_CLASS_BOTH | FIFOLDST_VLF |
1205 			     FIFOLD_TYPE_AAD | FIFOLD_TYPE_LAST2FLUSH1);
1206 
1207 	set_move_tgt_here(desc, read_move_cmd);
1208 	set_move_tgt_here(desc, write_move_cmd);
1209 	append_cmd(desc, CMD_LOAD | DISABLE_AUTO_INFO_FIFO);
1210 	/* Move payload data to OFIFO */
1211 	append_move(desc, MOVE_SRC_INFIFO_CL | MOVE_DEST_OUTFIFO);
1212 	append_cmd(desc, CMD_LOAD | ENABLE_AUTO_INFO_FIFO);
1213 
1214 	/* Read ICV */
1215 	append_seq_fifo_load(desc, icvsize, FIFOLD_CLASS_CLASS1 |
1216 			     FIFOLD_TYPE_ICV | FIFOLD_TYPE_LAST1);
1217 
1218 	print_hex_dump_debug("rfc4543 dec shdesc@" __stringify(__LINE__)": ",
1219 			     DUMP_PREFIX_ADDRESS, 16, 4, desc, desc_bytes(desc),
1220 			     1);
1221 }
1222 EXPORT_SYMBOL(cnstr_shdsc_rfc4543_decap);
1223 
1224 /**
1225  * cnstr_shdsc_chachapoly - Chacha20 + Poly1305 generic AEAD (rfc7539) and
1226  *                          IPsec ESP (rfc7634, a.k.a. rfc7539esp) shared
1227  *                          descriptor (non-protocol).
1228  * @desc: pointer to buffer used for descriptor construction
1229  * @cdata: pointer to block cipher transform definitions
1230  *         Valid algorithm values - OP_ALG_ALGSEL_CHACHA20 ANDed with
1231  *         OP_ALG_AAI_AEAD.
1232  * @adata: pointer to authentication transform definitions
1233  *         Valid algorithm values - OP_ALG_ALGSEL_POLY1305 ANDed with
1234  *         OP_ALG_AAI_AEAD.
1235  * @ivsize: initialization vector size
1236  * @icvsize: integrity check value (ICV) size (truncated or full)
1237  * @encap: true if encapsulation, false if decapsulation
1238  * @is_qi: true when called from caam/qi
1239  */
1240 void cnstr_shdsc_chachapoly(u32 * const desc, struct alginfo *cdata,
1241 			    struct alginfo *adata, unsigned int ivsize,
1242 			    unsigned int icvsize, const bool encap,
1243 			    const bool is_qi)
1244 {
1245 	u32 *key_jump_cmd, *wait_cmd;
1246 	u32 nfifo;
1247 	const bool is_ipsec = (ivsize != CHACHAPOLY_IV_SIZE);
1248 
1249 	/* Note: Context registers are saved. */
1250 	init_sh_desc(desc, HDR_SHARE_SERIAL | HDR_SAVECTX);
1251 
1252 	/* skip key loading if they are loaded due to sharing */
1253 	key_jump_cmd = append_jump(desc, JUMP_JSL | JUMP_TEST_ALL |
1254 				   JUMP_COND_SHRD);
1255 
1256 	append_key_as_imm(desc, cdata->key_virt, cdata->keylen, cdata->keylen,
1257 			  CLASS_1 | KEY_DEST_CLASS_REG);
1258 
1259 	/* For IPsec load the salt from keymat in the context register */
1260 	if (is_ipsec)
1261 		append_load_as_imm(desc, cdata->key_virt + cdata->keylen, 4,
1262 				   LDST_CLASS_1_CCB | LDST_SRCDST_BYTE_CONTEXT |
1263 				   4 << LDST_OFFSET_SHIFT);
1264 
1265 	set_jump_tgt_here(desc, key_jump_cmd);
1266 
1267 	/* Class 2 and 1 operations: Poly & ChaCha */
1268 	if (encap) {
1269 		append_operation(desc, adata->algtype | OP_ALG_AS_INITFINAL |
1270 				 OP_ALG_ENCRYPT);
1271 		append_operation(desc, cdata->algtype | OP_ALG_AS_INITFINAL |
1272 				 OP_ALG_ENCRYPT);
1273 	} else {
1274 		append_operation(desc, adata->algtype | OP_ALG_AS_INITFINAL |
1275 				 OP_ALG_DECRYPT | OP_ALG_ICV_ON);
1276 		append_operation(desc, cdata->algtype | OP_ALG_AS_INITFINAL |
1277 				 OP_ALG_DECRYPT);
1278 	}
1279 
1280 	if (is_qi) {
1281 		u32 *wait_load_cmd;
1282 		u32 ctx1_iv_off = is_ipsec ? 8 : 4;
1283 
1284 		/* REG3 = assoclen */
1285 		append_seq_load(desc, 4, LDST_CLASS_DECO |
1286 				LDST_SRCDST_WORD_DECO_MATH3 |
1287 				4 << LDST_OFFSET_SHIFT);
1288 
1289 		wait_load_cmd = append_jump(desc, JUMP_JSL | JUMP_TEST_ALL |
1290 					    JUMP_COND_CALM | JUMP_COND_NCP |
1291 					    JUMP_COND_NOP | JUMP_COND_NIP |
1292 					    JUMP_COND_NIFP);
1293 		set_jump_tgt_here(desc, wait_load_cmd);
1294 
1295 		append_seq_load(desc, ivsize, LDST_CLASS_1_CCB |
1296 				LDST_SRCDST_BYTE_CONTEXT |
1297 				ctx1_iv_off << LDST_OFFSET_SHIFT);
1298 	}
1299 
1300 	/*
1301 	 * MAGIC with NFIFO
1302 	 * Read associated data from the input and send them to class1 and
1303 	 * class2 alignment blocks. From class1 send data to output fifo and
1304 	 * then write it to memory since we don't need to encrypt AD.
1305 	 */
1306 	nfifo = NFIFOENTRY_DEST_BOTH | NFIFOENTRY_FC1 | NFIFOENTRY_FC2 |
1307 		NFIFOENTRY_DTYPE_POLY | NFIFOENTRY_BND;
1308 	append_load_imm_u32(desc, nfifo, LDST_CLASS_IND_CCB |
1309 			    LDST_SRCDST_WORD_INFO_FIFO_SM | LDLEN_MATH3);
1310 
1311 	append_math_add(desc, VARSEQINLEN, ZERO, REG3, CAAM_CMD_SZ);
1312 	append_math_add(desc, VARSEQOUTLEN, ZERO, REG3, CAAM_CMD_SZ);
1313 	append_seq_fifo_load(desc, 0, FIFOLD_TYPE_NOINFOFIFO |
1314 			     FIFOLD_CLASS_CLASS1 | LDST_VLF);
1315 	append_move_len(desc, MOVE_AUX_LS | MOVE_SRC_AUX_ABLK |
1316 			MOVE_DEST_OUTFIFO | MOVELEN_MRSEL_MATH3);
1317 	append_seq_fifo_store(desc, 0, FIFOST_TYPE_MESSAGE_DATA | LDST_VLF);
1318 
1319 	/* IPsec - copy IV at the output */
1320 	if (is_ipsec)
1321 		append_seq_fifo_store(desc, ivsize, FIFOST_TYPE_METADATA |
1322 				      0x2 << 25);
1323 
1324 	wait_cmd = append_jump(desc, JUMP_JSL | JUMP_TYPE_LOCAL |
1325 			       JUMP_COND_NOP | JUMP_TEST_ALL);
1326 	set_jump_tgt_here(desc, wait_cmd);
1327 
1328 	if (encap) {
1329 		/* Read and write cryptlen bytes */
1330 		append_math_add(desc, VARSEQINLEN, SEQINLEN, REG0, CAAM_CMD_SZ);
1331 		append_math_add(desc, VARSEQOUTLEN, SEQINLEN, REG0,
1332 				CAAM_CMD_SZ);
1333 		aead_append_src_dst(desc, FIFOLD_TYPE_MSG1OUT2);
1334 
1335 		/* Write ICV */
1336 		append_seq_store(desc, icvsize, LDST_CLASS_2_CCB |
1337 				 LDST_SRCDST_BYTE_CONTEXT);
1338 	} else {
1339 		/* Read and write cryptlen bytes */
1340 		append_math_add(desc, VARSEQINLEN, SEQOUTLEN, REG0,
1341 				CAAM_CMD_SZ);
1342 		append_math_add(desc, VARSEQOUTLEN, SEQOUTLEN, REG0,
1343 				CAAM_CMD_SZ);
1344 		aead_append_src_dst(desc, FIFOLD_TYPE_MSG);
1345 
1346 		/* Load ICV for verification */
1347 		append_seq_fifo_load(desc, icvsize, FIFOLD_CLASS_CLASS2 |
1348 				     FIFOLD_TYPE_LAST2 | FIFOLD_TYPE_ICV);
1349 	}
1350 
1351 	print_hex_dump_debug("chachapoly shdesc@" __stringify(__LINE__)": ",
1352 			     DUMP_PREFIX_ADDRESS, 16, 4, desc, desc_bytes(desc),
1353 			     1);
1354 }
1355 EXPORT_SYMBOL(cnstr_shdsc_chachapoly);
1356 
1357 /* For skcipher encrypt and decrypt, read from req->src and write to req->dst */
1358 static inline void skcipher_append_src_dst(u32 *desc)
1359 {
1360 	append_math_add(desc, VARSEQOUTLEN, SEQINLEN, REG0, CAAM_CMD_SZ);
1361 	append_math_add(desc, VARSEQINLEN, SEQINLEN, REG0, CAAM_CMD_SZ);
1362 	append_seq_fifo_load(desc, 0, FIFOLD_CLASS_CLASS1 |
1363 			     KEY_VLF | FIFOLD_TYPE_MSG | FIFOLD_TYPE_LAST1);
1364 	append_seq_fifo_store(desc, 0, FIFOST_TYPE_MESSAGE_DATA | KEY_VLF);
1365 }
1366 
1367 /**
1368  * cnstr_shdsc_skcipher_encap - skcipher encapsulation shared descriptor
1369  * @desc: pointer to buffer used for descriptor construction
1370  * @cdata: pointer to block cipher transform definitions
1371  *         Valid algorithm values - one of OP_ALG_ALGSEL_{AES, DES, 3DES} ANDed
1372  *         with OP_ALG_AAI_CBC or OP_ALG_AAI_CTR_MOD128
1373  *                                - OP_ALG_ALGSEL_CHACHA20
1374  * @ivsize: initialization vector size
1375  * @is_rfc3686: true when ctr(aes) is wrapped by rfc3686 template
1376  * @ctx1_iv_off: IV offset in CONTEXT1 register
1377  */
1378 void cnstr_shdsc_skcipher_encap(u32 * const desc, struct alginfo *cdata,
1379 				unsigned int ivsize, const bool is_rfc3686,
1380 				const u32 ctx1_iv_off)
1381 {
1382 	u32 *key_jump_cmd;
1383 	u32 options = cdata->algtype | OP_ALG_AS_INIT | OP_ALG_ENCRYPT;
1384 	bool is_chacha20 = ((cdata->algtype & OP_ALG_ALGSEL_MASK) ==
1385 			    OP_ALG_ALGSEL_CHACHA20);
1386 
1387 	init_sh_desc(desc, HDR_SHARE_SERIAL | HDR_SAVECTX);
1388 	/* Skip if already shared */
1389 	key_jump_cmd = append_jump(desc, JUMP_JSL | JUMP_TEST_ALL |
1390 				   JUMP_COND_SHRD);
1391 
1392 	/* Load class1 key only */
1393 	append_key_as_imm(desc, cdata->key_virt, cdata->keylen,
1394 			  cdata->keylen, CLASS_1 | KEY_DEST_CLASS_REG);
1395 
1396 	/* Load nonce into CONTEXT1 reg */
1397 	if (is_rfc3686) {
1398 		const u8 *nonce = cdata->key_virt + cdata->keylen;
1399 
1400 		append_load_as_imm(desc, nonce, CTR_RFC3686_NONCE_SIZE,
1401 				   LDST_CLASS_IND_CCB |
1402 				   LDST_SRCDST_BYTE_OUTFIFO | LDST_IMM);
1403 		append_move(desc, MOVE_WAITCOMP | MOVE_SRC_OUTFIFO |
1404 			    MOVE_DEST_CLASS1CTX | (16 << MOVE_OFFSET_SHIFT) |
1405 			    (CTR_RFC3686_NONCE_SIZE << MOVE_LEN_SHIFT));
1406 	}
1407 
1408 	set_jump_tgt_here(desc, key_jump_cmd);
1409 
1410 	/* Load IV, if there is one */
1411 	if (ivsize)
1412 		append_seq_load(desc, ivsize, LDST_SRCDST_BYTE_CONTEXT |
1413 				LDST_CLASS_1_CCB | (ctx1_iv_off <<
1414 				LDST_OFFSET_SHIFT));
1415 
1416 	/* Load counter into CONTEXT1 reg */
1417 	if (is_rfc3686)
1418 		append_load_imm_be32(desc, 1, LDST_IMM | LDST_CLASS_1_CCB |
1419 				     LDST_SRCDST_BYTE_CONTEXT |
1420 				     ((ctx1_iv_off + CTR_RFC3686_IV_SIZE) <<
1421 				      LDST_OFFSET_SHIFT));
1422 
1423 	/* Load operation */
1424 	if (is_chacha20)
1425 		options |= OP_ALG_AS_FINALIZE;
1426 	append_operation(desc, options);
1427 
1428 	/* Perform operation */
1429 	skcipher_append_src_dst(desc);
1430 
1431 	/* Store IV */
1432 	if (!is_chacha20 && ivsize)
1433 		append_seq_store(desc, ivsize, LDST_SRCDST_BYTE_CONTEXT |
1434 				 LDST_CLASS_1_CCB | (ctx1_iv_off <<
1435 				 LDST_OFFSET_SHIFT));
1436 
1437 	print_hex_dump_debug("skcipher enc shdesc@" __stringify(__LINE__)": ",
1438 			     DUMP_PREFIX_ADDRESS, 16, 4, desc, desc_bytes(desc),
1439 			     1);
1440 }
1441 EXPORT_SYMBOL(cnstr_shdsc_skcipher_encap);
1442 
1443 /**
1444  * cnstr_shdsc_skcipher_decap - skcipher decapsulation shared descriptor
1445  * @desc: pointer to buffer used for descriptor construction
1446  * @cdata: pointer to block cipher transform definitions
1447  *         Valid algorithm values - one of OP_ALG_ALGSEL_{AES, DES, 3DES} ANDed
1448  *         with OP_ALG_AAI_CBC or OP_ALG_AAI_CTR_MOD128
1449  *                                - OP_ALG_ALGSEL_CHACHA20
1450  * @ivsize: initialization vector size
1451  * @is_rfc3686: true when ctr(aes) is wrapped by rfc3686 template
1452  * @ctx1_iv_off: IV offset in CONTEXT1 register
1453  */
1454 void cnstr_shdsc_skcipher_decap(u32 * const desc, struct alginfo *cdata,
1455 				unsigned int ivsize, const bool is_rfc3686,
1456 				const u32 ctx1_iv_off)
1457 {
1458 	u32 *key_jump_cmd;
1459 	bool is_chacha20 = ((cdata->algtype & OP_ALG_ALGSEL_MASK) ==
1460 			    OP_ALG_ALGSEL_CHACHA20);
1461 
1462 	init_sh_desc(desc, HDR_SHARE_SERIAL | HDR_SAVECTX);
1463 	/* Skip if already shared */
1464 	key_jump_cmd = append_jump(desc, JUMP_JSL | JUMP_TEST_ALL |
1465 				   JUMP_COND_SHRD);
1466 
1467 	/* Load class1 key only */
1468 	append_key_as_imm(desc, cdata->key_virt, cdata->keylen,
1469 			  cdata->keylen, CLASS_1 | KEY_DEST_CLASS_REG);
1470 
1471 	/* Load nonce into CONTEXT1 reg */
1472 	if (is_rfc3686) {
1473 		const u8 *nonce = cdata->key_virt + cdata->keylen;
1474 
1475 		append_load_as_imm(desc, nonce, CTR_RFC3686_NONCE_SIZE,
1476 				   LDST_CLASS_IND_CCB |
1477 				   LDST_SRCDST_BYTE_OUTFIFO | LDST_IMM);
1478 		append_move(desc, MOVE_WAITCOMP | MOVE_SRC_OUTFIFO |
1479 			    MOVE_DEST_CLASS1CTX | (16 << MOVE_OFFSET_SHIFT) |
1480 			    (CTR_RFC3686_NONCE_SIZE << MOVE_LEN_SHIFT));
1481 	}
1482 
1483 	set_jump_tgt_here(desc, key_jump_cmd);
1484 
1485 	/* Load IV, if there is one */
1486 	if (ivsize)
1487 		append_seq_load(desc, ivsize, LDST_SRCDST_BYTE_CONTEXT |
1488 				LDST_CLASS_1_CCB | (ctx1_iv_off <<
1489 				LDST_OFFSET_SHIFT));
1490 
1491 	/* Load counter into CONTEXT1 reg */
1492 	if (is_rfc3686)
1493 		append_load_imm_be32(desc, 1, LDST_IMM | LDST_CLASS_1_CCB |
1494 				     LDST_SRCDST_BYTE_CONTEXT |
1495 				     ((ctx1_iv_off + CTR_RFC3686_IV_SIZE) <<
1496 				      LDST_OFFSET_SHIFT));
1497 
1498 	/* Choose operation */
1499 	if (ctx1_iv_off)
1500 		append_operation(desc, cdata->algtype | OP_ALG_AS_INIT |
1501 				 OP_ALG_DECRYPT);
1502 	else
1503 		append_dec_op1(desc, cdata->algtype);
1504 
1505 	/* Perform operation */
1506 	skcipher_append_src_dst(desc);
1507 
1508 	/* Store IV */
1509 	if (!is_chacha20 && ivsize)
1510 		append_seq_store(desc, ivsize, LDST_SRCDST_BYTE_CONTEXT |
1511 				 LDST_CLASS_1_CCB | (ctx1_iv_off <<
1512 				 LDST_OFFSET_SHIFT));
1513 
1514 	print_hex_dump_debug("skcipher dec shdesc@" __stringify(__LINE__)": ",
1515 			     DUMP_PREFIX_ADDRESS, 16, 4, desc, desc_bytes(desc),
1516 			     1);
1517 }
1518 EXPORT_SYMBOL(cnstr_shdsc_skcipher_decap);
1519 
1520 /**
1521  * cnstr_shdsc_xts_skcipher_encap - xts skcipher encapsulation shared descriptor
1522  * @desc: pointer to buffer used for descriptor construction
1523  * @cdata: pointer to block cipher transform definitions
1524  *         Valid algorithm values - OP_ALG_ALGSEL_AES ANDed with OP_ALG_AAI_XTS.
1525  */
1526 void cnstr_shdsc_xts_skcipher_encap(u32 * const desc, struct alginfo *cdata)
1527 {
1528 	/*
1529 	 * Set sector size to a big value, practically disabling
1530 	 * sector size segmentation in xts implementation. We cannot
1531 	 * take full advantage of this HW feature with existing
1532 	 * crypto API / dm-crypt SW architecture.
1533 	 */
1534 	__be64 sector_size = cpu_to_be64(BIT(15));
1535 	u32 *key_jump_cmd;
1536 
1537 	init_sh_desc(desc, HDR_SHARE_SERIAL | HDR_SAVECTX);
1538 	/* Skip if already shared */
1539 	key_jump_cmd = append_jump(desc, JUMP_JSL | JUMP_TEST_ALL |
1540 				   JUMP_COND_SHRD);
1541 
1542 	/* Load class1 keys only */
1543 	append_key_as_imm(desc, cdata->key_virt, cdata->keylen,
1544 			  cdata->keylen, CLASS_1 | KEY_DEST_CLASS_REG);
1545 
1546 	/* Load sector size with index 40 bytes (0x28) */
1547 	append_load_as_imm(desc, (void *)&sector_size, 8, LDST_CLASS_1_CCB |
1548 			   LDST_SRCDST_BYTE_CONTEXT |
1549 			   (0x28 << LDST_OFFSET_SHIFT));
1550 
1551 	set_jump_tgt_here(desc, key_jump_cmd);
1552 
1553 	/*
1554 	 * create sequence for loading the sector index / 16B tweak value
1555 	 * Lower 8B of IV - sector index / tweak lower half
1556 	 * Upper 8B of IV - upper half of 16B tweak
1557 	 */
1558 	append_seq_load(desc, 8, LDST_SRCDST_BYTE_CONTEXT | LDST_CLASS_1_CCB |
1559 			(0x20 << LDST_OFFSET_SHIFT));
1560 	append_seq_load(desc, 8, LDST_SRCDST_BYTE_CONTEXT | LDST_CLASS_1_CCB |
1561 			(0x30 << LDST_OFFSET_SHIFT));
1562 
1563 	/* Load operation */
1564 	append_operation(desc, cdata->algtype | OP_ALG_AS_INITFINAL |
1565 			 OP_ALG_ENCRYPT);
1566 
1567 	/* Perform operation */
1568 	skcipher_append_src_dst(desc);
1569 
1570 	/* Store lower 8B and upper 8B of IV */
1571 	append_seq_store(desc, 8, LDST_SRCDST_BYTE_CONTEXT | LDST_CLASS_1_CCB |
1572 			 (0x20 << LDST_OFFSET_SHIFT));
1573 	append_seq_store(desc, 8, LDST_SRCDST_BYTE_CONTEXT | LDST_CLASS_1_CCB |
1574 			 (0x30 << LDST_OFFSET_SHIFT));
1575 
1576 	print_hex_dump_debug("xts skcipher enc shdesc@" __stringify(__LINE__)
1577 			     ": ", DUMP_PREFIX_ADDRESS, 16, 4,
1578 			     desc, desc_bytes(desc), 1);
1579 }
1580 EXPORT_SYMBOL(cnstr_shdsc_xts_skcipher_encap);
1581 
1582 /**
1583  * cnstr_shdsc_xts_skcipher_decap - xts skcipher decapsulation shared descriptor
1584  * @desc: pointer to buffer used for descriptor construction
1585  * @cdata: pointer to block cipher transform definitions
1586  *         Valid algorithm values - OP_ALG_ALGSEL_AES ANDed with OP_ALG_AAI_XTS.
1587  */
1588 void cnstr_shdsc_xts_skcipher_decap(u32 * const desc, struct alginfo *cdata)
1589 {
1590 	/*
1591 	 * Set sector size to a big value, practically disabling
1592 	 * sector size segmentation in xts implementation. We cannot
1593 	 * take full advantage of this HW feature with existing
1594 	 * crypto API / dm-crypt SW architecture.
1595 	 */
1596 	__be64 sector_size = cpu_to_be64(BIT(15));
1597 	u32 *key_jump_cmd;
1598 
1599 	init_sh_desc(desc, HDR_SHARE_SERIAL | HDR_SAVECTX);
1600 	/* Skip if already shared */
1601 	key_jump_cmd = append_jump(desc, JUMP_JSL | JUMP_TEST_ALL |
1602 				   JUMP_COND_SHRD);
1603 
1604 	/* Load class1 key only */
1605 	append_key_as_imm(desc, cdata->key_virt, cdata->keylen,
1606 			  cdata->keylen, CLASS_1 | KEY_DEST_CLASS_REG);
1607 
1608 	/* Load sector size with index 40 bytes (0x28) */
1609 	append_load_as_imm(desc, (void *)&sector_size, 8, LDST_CLASS_1_CCB |
1610 			   LDST_SRCDST_BYTE_CONTEXT |
1611 			   (0x28 << LDST_OFFSET_SHIFT));
1612 
1613 	set_jump_tgt_here(desc, key_jump_cmd);
1614 
1615 	/*
1616 	 * create sequence for loading the sector index / 16B tweak value
1617 	 * Lower 8B of IV - sector index / tweak lower half
1618 	 * Upper 8B of IV - upper half of 16B tweak
1619 	 */
1620 	append_seq_load(desc, 8, LDST_SRCDST_BYTE_CONTEXT | LDST_CLASS_1_CCB |
1621 			(0x20 << LDST_OFFSET_SHIFT));
1622 	append_seq_load(desc, 8, LDST_SRCDST_BYTE_CONTEXT | LDST_CLASS_1_CCB |
1623 			(0x30 << LDST_OFFSET_SHIFT));
1624 	/* Load operation */
1625 	append_dec_op1(desc, cdata->algtype);
1626 
1627 	/* Perform operation */
1628 	skcipher_append_src_dst(desc);
1629 
1630 	/* Store lower 8B and upper 8B of IV */
1631 	append_seq_store(desc, 8, LDST_SRCDST_BYTE_CONTEXT | LDST_CLASS_1_CCB |
1632 			 (0x20 << LDST_OFFSET_SHIFT));
1633 	append_seq_store(desc, 8, LDST_SRCDST_BYTE_CONTEXT | LDST_CLASS_1_CCB |
1634 			 (0x30 << LDST_OFFSET_SHIFT));
1635 
1636 	print_hex_dump_debug("xts skcipher dec shdesc@" __stringify(__LINE__)
1637 			     ": ", DUMP_PREFIX_ADDRESS, 16, 4, desc,
1638 			     desc_bytes(desc), 1);
1639 }
1640 EXPORT_SYMBOL(cnstr_shdsc_xts_skcipher_decap);
1641 
1642 MODULE_LICENSE("GPL");
1643 MODULE_DESCRIPTION("FSL CAAM descriptor support");
1644 MODULE_AUTHOR("Freescale Semiconductor - NMG/STC");
1645