1 // SPDX-License-Identifier: GPL-2.0+
2 /*
3  * Shared descriptors for aead, skcipher algorithms
4  *
5  * Copyright 2016-2019 NXP
6  */
7 
8 #include "compat.h"
9 #include "desc_constr.h"
10 #include "caamalg_desc.h"
11 
12 /*
13  * For aead functions, read payload and write payload,
14  * both of which are specified in req->src and req->dst
15  */
16 static inline void aead_append_src_dst(u32 *desc, u32 msg_type)
17 {
18 	append_seq_fifo_store(desc, 0, FIFOST_TYPE_MESSAGE_DATA | KEY_VLF);
19 	append_seq_fifo_load(desc, 0, FIFOLD_CLASS_BOTH |
20 			     KEY_VLF | msg_type | FIFOLD_TYPE_LASTBOTH);
21 }
22 
23 /* Set DK bit in class 1 operation if shared */
24 static inline void append_dec_op1(u32 *desc, u32 type)
25 {
26 	u32 *jump_cmd, *uncond_jump_cmd;
27 
28 	/* DK bit is valid only for AES */
29 	if ((type & OP_ALG_ALGSEL_MASK) != OP_ALG_ALGSEL_AES) {
30 		append_operation(desc, type | OP_ALG_AS_INITFINAL |
31 				 OP_ALG_DECRYPT);
32 		return;
33 	}
34 
35 	jump_cmd = append_jump(desc, JUMP_TEST_ALL | JUMP_COND_SHRD);
36 	append_operation(desc, type | OP_ALG_AS_INITFINAL |
37 			 OP_ALG_DECRYPT);
38 	uncond_jump_cmd = append_jump(desc, JUMP_TEST_ALL);
39 	set_jump_tgt_here(desc, jump_cmd);
40 	append_operation(desc, type | OP_ALG_AS_INITFINAL |
41 			 OP_ALG_DECRYPT | OP_ALG_AAI_DK);
42 	set_jump_tgt_here(desc, uncond_jump_cmd);
43 }
44 
45 /**
46  * cnstr_shdsc_aead_null_encap - IPSec ESP encapsulation shared descriptor
47  *                               (non-protocol) with no (null) encryption.
48  * @desc: pointer to buffer used for descriptor construction
49  * @adata: pointer to authentication transform definitions.
50  *         A split key is required for SEC Era < 6; the size of the split key
51  *         is specified in this case. Valid algorithm values - one of
52  *         OP_ALG_ALGSEL_{MD5, SHA1, SHA224, SHA256, SHA384, SHA512} ANDed
53  *         with OP_ALG_AAI_HMAC_PRECOMP.
54  * @icvsize: integrity check value (ICV) size (truncated or full)
55  * @era: SEC Era
56  */
57 void cnstr_shdsc_aead_null_encap(u32 * const desc, struct alginfo *adata,
58 				 unsigned int icvsize, int era)
59 {
60 	u32 *key_jump_cmd, *read_move_cmd, *write_move_cmd;
61 
62 	init_sh_desc(desc, HDR_SHARE_SERIAL);
63 
64 	/* Skip if already shared */
65 	key_jump_cmd = append_jump(desc, JUMP_JSL | JUMP_TEST_ALL |
66 				   JUMP_COND_SHRD);
67 	if (era < 6) {
68 		if (adata->key_inline)
69 			append_key_as_imm(desc, adata->key_virt,
70 					  adata->keylen_pad, adata->keylen,
71 					  CLASS_2 | KEY_DEST_MDHA_SPLIT |
72 					  KEY_ENC);
73 		else
74 			append_key(desc, adata->key_dma, adata->keylen,
75 				   CLASS_2 | KEY_DEST_MDHA_SPLIT | KEY_ENC);
76 	} else {
77 		append_proto_dkp(desc, adata);
78 	}
79 	set_jump_tgt_here(desc, key_jump_cmd);
80 
81 	/* assoclen + cryptlen = seqinlen */
82 	append_math_sub(desc, REG3, SEQINLEN, REG0, CAAM_CMD_SZ);
83 
84 	/* Prepare to read and write cryptlen + assoclen bytes */
85 	append_math_add(desc, VARSEQINLEN, ZERO, REG3, CAAM_CMD_SZ);
86 	append_math_add(desc, VARSEQOUTLEN, ZERO, REG3, CAAM_CMD_SZ);
87 
88 	/*
89 	 * MOVE_LEN opcode is not available in all SEC HW revisions,
90 	 * thus need to do some magic, i.e. self-patch the descriptor
91 	 * buffer.
92 	 */
93 	read_move_cmd = append_move(desc, MOVE_SRC_DESCBUF |
94 				    MOVE_DEST_MATH3 |
95 				    (0x6 << MOVE_LEN_SHIFT));
96 	write_move_cmd = append_move(desc, MOVE_SRC_MATH3 |
97 				     MOVE_DEST_DESCBUF |
98 				     MOVE_WAITCOMP |
99 				     (0x8 << MOVE_LEN_SHIFT));
100 
101 	/* Class 2 operation */
102 	append_operation(desc, adata->algtype | OP_ALG_AS_INITFINAL |
103 			 OP_ALG_ENCRYPT);
104 
105 	/* Read and write cryptlen bytes */
106 	aead_append_src_dst(desc, FIFOLD_TYPE_MSG | FIFOLD_TYPE_FLUSH1);
107 
108 	set_move_tgt_here(desc, read_move_cmd);
109 	set_move_tgt_here(desc, write_move_cmd);
110 	append_cmd(desc, CMD_LOAD | DISABLE_AUTO_INFO_FIFO);
111 	append_move(desc, MOVE_SRC_INFIFO_CL | MOVE_DEST_OUTFIFO |
112 		    MOVE_AUX_LS);
113 
114 	/* Write ICV */
115 	append_seq_store(desc, icvsize, LDST_CLASS_2_CCB |
116 			 LDST_SRCDST_BYTE_CONTEXT);
117 
118 #ifdef DEBUG
119 	print_hex_dump(KERN_ERR,
120 		       "aead null enc shdesc@" __stringify(__LINE__)": ",
121 		       DUMP_PREFIX_ADDRESS, 16, 4, desc, desc_bytes(desc), 1);
122 #endif
123 }
124 EXPORT_SYMBOL(cnstr_shdsc_aead_null_encap);
125 
126 /**
127  * cnstr_shdsc_aead_null_decap - IPSec ESP decapsulation shared descriptor
128  *                               (non-protocol) with no (null) decryption.
129  * @desc: pointer to buffer used for descriptor construction
130  * @adata: pointer to authentication transform definitions.
131  *         A split key is required for SEC Era < 6; the size of the split key
132  *         is specified in this case. Valid algorithm values - one of
133  *         OP_ALG_ALGSEL_{MD5, SHA1, SHA224, SHA256, SHA384, SHA512} ANDed
134  *         with OP_ALG_AAI_HMAC_PRECOMP.
135  * @icvsize: integrity check value (ICV) size (truncated or full)
136  * @era: SEC Era
137  */
138 void cnstr_shdsc_aead_null_decap(u32 * const desc, struct alginfo *adata,
139 				 unsigned int icvsize, int era)
140 {
141 	u32 *key_jump_cmd, *read_move_cmd, *write_move_cmd, *jump_cmd;
142 
143 	init_sh_desc(desc, HDR_SHARE_SERIAL);
144 
145 	/* Skip if already shared */
146 	key_jump_cmd = append_jump(desc, JUMP_JSL | JUMP_TEST_ALL |
147 				   JUMP_COND_SHRD);
148 	if (era < 6) {
149 		if (adata->key_inline)
150 			append_key_as_imm(desc, adata->key_virt,
151 					  adata->keylen_pad, adata->keylen,
152 					  CLASS_2 | KEY_DEST_MDHA_SPLIT |
153 					  KEY_ENC);
154 		else
155 			append_key(desc, adata->key_dma, adata->keylen,
156 				   CLASS_2 | KEY_DEST_MDHA_SPLIT | KEY_ENC);
157 	} else {
158 		append_proto_dkp(desc, adata);
159 	}
160 	set_jump_tgt_here(desc, key_jump_cmd);
161 
162 	/* Class 2 operation */
163 	append_operation(desc, adata->algtype | OP_ALG_AS_INITFINAL |
164 			 OP_ALG_DECRYPT | OP_ALG_ICV_ON);
165 
166 	/* assoclen + cryptlen = seqoutlen */
167 	append_math_sub(desc, REG2, SEQOUTLEN, REG0, CAAM_CMD_SZ);
168 
169 	/* Prepare to read and write cryptlen + assoclen bytes */
170 	append_math_add(desc, VARSEQINLEN, ZERO, REG2, CAAM_CMD_SZ);
171 	append_math_add(desc, VARSEQOUTLEN, ZERO, REG2, CAAM_CMD_SZ);
172 
173 	/*
174 	 * MOVE_LEN opcode is not available in all SEC HW revisions,
175 	 * thus need to do some magic, i.e. self-patch the descriptor
176 	 * buffer.
177 	 */
178 	read_move_cmd = append_move(desc, MOVE_SRC_DESCBUF |
179 				    MOVE_DEST_MATH2 |
180 				    (0x6 << MOVE_LEN_SHIFT));
181 	write_move_cmd = append_move(desc, MOVE_SRC_MATH2 |
182 				     MOVE_DEST_DESCBUF |
183 				     MOVE_WAITCOMP |
184 				     (0x8 << MOVE_LEN_SHIFT));
185 
186 	/* Read and write cryptlen bytes */
187 	aead_append_src_dst(desc, FIFOLD_TYPE_MSG | FIFOLD_TYPE_FLUSH1);
188 
189 	/*
190 	 * Insert a NOP here, since we need at least 4 instructions between
191 	 * code patching the descriptor buffer and the location being patched.
192 	 */
193 	jump_cmd = append_jump(desc, JUMP_TEST_ALL);
194 	set_jump_tgt_here(desc, jump_cmd);
195 
196 	set_move_tgt_here(desc, read_move_cmd);
197 	set_move_tgt_here(desc, write_move_cmd);
198 	append_cmd(desc, CMD_LOAD | DISABLE_AUTO_INFO_FIFO);
199 	append_move(desc, MOVE_SRC_INFIFO_CL | MOVE_DEST_OUTFIFO |
200 		    MOVE_AUX_LS);
201 	append_cmd(desc, CMD_LOAD | ENABLE_AUTO_INFO_FIFO);
202 
203 	/* Load ICV */
204 	append_seq_fifo_load(desc, icvsize, FIFOLD_CLASS_CLASS2 |
205 			     FIFOLD_TYPE_LAST2 | FIFOLD_TYPE_ICV);
206 
207 #ifdef DEBUG
208 	print_hex_dump(KERN_ERR,
209 		       "aead null dec shdesc@" __stringify(__LINE__)": ",
210 		       DUMP_PREFIX_ADDRESS, 16, 4, desc, desc_bytes(desc), 1);
211 #endif
212 }
213 EXPORT_SYMBOL(cnstr_shdsc_aead_null_decap);
214 
215 static void init_sh_desc_key_aead(u32 * const desc,
216 				  struct alginfo * const cdata,
217 				  struct alginfo * const adata,
218 				  const bool is_rfc3686, u32 *nonce, int era)
219 {
220 	u32 *key_jump_cmd;
221 	unsigned int enckeylen = cdata->keylen;
222 
223 	/* Note: Context registers are saved. */
224 	init_sh_desc(desc, HDR_SHARE_SERIAL | HDR_SAVECTX);
225 
226 	/* Skip if already shared */
227 	key_jump_cmd = append_jump(desc, JUMP_JSL | JUMP_TEST_ALL |
228 				   JUMP_COND_SHRD);
229 
230 	/*
231 	 * RFC3686 specific:
232 	 *	| key = {AUTH_KEY, ENC_KEY, NONCE}
233 	 *	| enckeylen = encryption key size + nonce size
234 	 */
235 	if (is_rfc3686)
236 		enckeylen -= CTR_RFC3686_NONCE_SIZE;
237 
238 	if (era < 6) {
239 		if (adata->key_inline)
240 			append_key_as_imm(desc, adata->key_virt,
241 					  adata->keylen_pad, adata->keylen,
242 					  CLASS_2 | KEY_DEST_MDHA_SPLIT |
243 					  KEY_ENC);
244 		else
245 			append_key(desc, adata->key_dma, adata->keylen,
246 				   CLASS_2 | KEY_DEST_MDHA_SPLIT | KEY_ENC);
247 	} else {
248 		append_proto_dkp(desc, adata);
249 	}
250 
251 	if (cdata->key_inline)
252 		append_key_as_imm(desc, cdata->key_virt, enckeylen,
253 				  enckeylen, CLASS_1 | KEY_DEST_CLASS_REG);
254 	else
255 		append_key(desc, cdata->key_dma, enckeylen, CLASS_1 |
256 			   KEY_DEST_CLASS_REG);
257 
258 	/* Load Counter into CONTEXT1 reg */
259 	if (is_rfc3686) {
260 		append_load_as_imm(desc, nonce, CTR_RFC3686_NONCE_SIZE,
261 				   LDST_CLASS_IND_CCB |
262 				   LDST_SRCDST_BYTE_OUTFIFO | LDST_IMM);
263 		append_move(desc,
264 			    MOVE_SRC_OUTFIFO |
265 			    MOVE_DEST_CLASS1CTX |
266 			    (16 << MOVE_OFFSET_SHIFT) |
267 			    (CTR_RFC3686_NONCE_SIZE << MOVE_LEN_SHIFT));
268 	}
269 
270 	set_jump_tgt_here(desc, key_jump_cmd);
271 }
272 
273 /**
274  * cnstr_shdsc_aead_encap - IPSec ESP encapsulation shared descriptor
275  *                          (non-protocol).
276  * @desc: pointer to buffer used for descriptor construction
277  * @cdata: pointer to block cipher transform definitions
278  *         Valid algorithm values - one of OP_ALG_ALGSEL_{AES, DES, 3DES} ANDed
279  *         with OP_ALG_AAI_CBC or OP_ALG_AAI_CTR_MOD128.
280  * @adata: pointer to authentication transform definitions.
281  *         A split key is required for SEC Era < 6; the size of the split key
282  *         is specified in this case. Valid algorithm values - one of
283  *         OP_ALG_ALGSEL_{MD5, SHA1, SHA224, SHA256, SHA384, SHA512} ANDed
284  *         with OP_ALG_AAI_HMAC_PRECOMP.
285  * @ivsize: initialization vector size
286  * @icvsize: integrity check value (ICV) size (truncated or full)
287  * @is_rfc3686: true when ctr(aes) is wrapped by rfc3686 template
288  * @nonce: pointer to rfc3686 nonce
289  * @ctx1_iv_off: IV offset in CONTEXT1 register
290  * @is_qi: true when called from caam/qi
291  * @era: SEC Era
292  */
293 void cnstr_shdsc_aead_encap(u32 * const desc, struct alginfo *cdata,
294 			    struct alginfo *adata, unsigned int ivsize,
295 			    unsigned int icvsize, const bool is_rfc3686,
296 			    u32 *nonce, const u32 ctx1_iv_off, const bool is_qi,
297 			    int era)
298 {
299 	/* Note: Context registers are saved. */
300 	init_sh_desc_key_aead(desc, cdata, adata, is_rfc3686, nonce, era);
301 
302 	/* Class 2 operation */
303 	append_operation(desc, adata->algtype | OP_ALG_AS_INITFINAL |
304 			 OP_ALG_ENCRYPT);
305 
306 	if (is_qi) {
307 		u32 *wait_load_cmd;
308 
309 		/* REG3 = assoclen */
310 		append_seq_load(desc, 4, LDST_CLASS_DECO |
311 				LDST_SRCDST_WORD_DECO_MATH3 |
312 				(4 << LDST_OFFSET_SHIFT));
313 
314 		wait_load_cmd = append_jump(desc, JUMP_JSL | JUMP_TEST_ALL |
315 					    JUMP_COND_CALM | JUMP_COND_NCP |
316 					    JUMP_COND_NOP | JUMP_COND_NIP |
317 					    JUMP_COND_NIFP);
318 		set_jump_tgt_here(desc, wait_load_cmd);
319 
320 		append_seq_load(desc, ivsize, LDST_CLASS_1_CCB |
321 				LDST_SRCDST_BYTE_CONTEXT |
322 				(ctx1_iv_off << LDST_OFFSET_SHIFT));
323 	}
324 
325 	/* Read and write assoclen bytes */
326 	if (is_qi || era < 3) {
327 		append_math_add(desc, VARSEQINLEN, ZERO, REG3, CAAM_CMD_SZ);
328 		append_math_add(desc, VARSEQOUTLEN, ZERO, REG3, CAAM_CMD_SZ);
329 	} else {
330 		append_math_add(desc, VARSEQINLEN, ZERO, DPOVRD, CAAM_CMD_SZ);
331 		append_math_add(desc, VARSEQOUTLEN, ZERO, DPOVRD, CAAM_CMD_SZ);
332 	}
333 
334 	/* Skip assoc data */
335 	append_seq_fifo_store(desc, 0, FIFOST_TYPE_SKIP | FIFOLDST_VLF);
336 
337 	/* read assoc before reading payload */
338 	append_seq_fifo_load(desc, 0, FIFOLD_CLASS_CLASS2 | FIFOLD_TYPE_MSG |
339 				      FIFOLDST_VLF);
340 
341 	/* Load Counter into CONTEXT1 reg */
342 	if (is_rfc3686)
343 		append_load_imm_be32(desc, 1, LDST_IMM | LDST_CLASS_1_CCB |
344 				     LDST_SRCDST_BYTE_CONTEXT |
345 				     ((ctx1_iv_off + CTR_RFC3686_IV_SIZE) <<
346 				      LDST_OFFSET_SHIFT));
347 
348 	/* Class 1 operation */
349 	append_operation(desc, cdata->algtype | OP_ALG_AS_INITFINAL |
350 			 OP_ALG_ENCRYPT);
351 
352 	/* Read and write cryptlen bytes */
353 	append_math_add(desc, VARSEQINLEN, SEQINLEN, REG0, CAAM_CMD_SZ);
354 	append_math_add(desc, VARSEQOUTLEN, SEQINLEN, REG0, CAAM_CMD_SZ);
355 	aead_append_src_dst(desc, FIFOLD_TYPE_MSG1OUT2);
356 
357 	/* Write ICV */
358 	append_seq_store(desc, icvsize, LDST_CLASS_2_CCB |
359 			 LDST_SRCDST_BYTE_CONTEXT);
360 
361 #ifdef DEBUG
362 	print_hex_dump(KERN_ERR, "aead enc shdesc@" __stringify(__LINE__)": ",
363 		       DUMP_PREFIX_ADDRESS, 16, 4, desc, desc_bytes(desc), 1);
364 #endif
365 }
366 EXPORT_SYMBOL(cnstr_shdsc_aead_encap);
367 
368 /**
369  * cnstr_shdsc_aead_decap - IPSec ESP decapsulation shared descriptor
370  *                          (non-protocol).
371  * @desc: pointer to buffer used for descriptor construction
372  * @cdata: pointer to block cipher transform definitions
373  *         Valid algorithm values - one of OP_ALG_ALGSEL_{AES, DES, 3DES} ANDed
374  *         with OP_ALG_AAI_CBC or OP_ALG_AAI_CTR_MOD128.
375  * @adata: pointer to authentication transform definitions.
376  *         A split key is required for SEC Era < 6; the size of the split key
377  *         is specified in this case. Valid algorithm values - one of
378  *         OP_ALG_ALGSEL_{MD5, SHA1, SHA224, SHA256, SHA384, SHA512} ANDed
379  *         with OP_ALG_AAI_HMAC_PRECOMP.
380  * @ivsize: initialization vector size
381  * @icvsize: integrity check value (ICV) size (truncated or full)
382  * @is_rfc3686: true when ctr(aes) is wrapped by rfc3686 template
383  * @nonce: pointer to rfc3686 nonce
384  * @ctx1_iv_off: IV offset in CONTEXT1 register
385  * @is_qi: true when called from caam/qi
386  * @era: SEC Era
387  */
388 void cnstr_shdsc_aead_decap(u32 * const desc, struct alginfo *cdata,
389 			    struct alginfo *adata, unsigned int ivsize,
390 			    unsigned int icvsize, const bool geniv,
391 			    const bool is_rfc3686, u32 *nonce,
392 			    const u32 ctx1_iv_off, const bool is_qi, int era)
393 {
394 	/* Note: Context registers are saved. */
395 	init_sh_desc_key_aead(desc, cdata, adata, is_rfc3686, nonce, era);
396 
397 	/* Class 2 operation */
398 	append_operation(desc, adata->algtype | OP_ALG_AS_INITFINAL |
399 			 OP_ALG_DECRYPT | OP_ALG_ICV_ON);
400 
401 	if (is_qi) {
402 		u32 *wait_load_cmd;
403 
404 		/* REG3 = assoclen */
405 		append_seq_load(desc, 4, LDST_CLASS_DECO |
406 				LDST_SRCDST_WORD_DECO_MATH3 |
407 				(4 << LDST_OFFSET_SHIFT));
408 
409 		wait_load_cmd = append_jump(desc, JUMP_JSL | JUMP_TEST_ALL |
410 					    JUMP_COND_CALM | JUMP_COND_NCP |
411 					    JUMP_COND_NOP | JUMP_COND_NIP |
412 					    JUMP_COND_NIFP);
413 		set_jump_tgt_here(desc, wait_load_cmd);
414 
415 		if (!geniv)
416 			append_seq_load(desc, ivsize, LDST_CLASS_1_CCB |
417 					LDST_SRCDST_BYTE_CONTEXT |
418 					(ctx1_iv_off << LDST_OFFSET_SHIFT));
419 	}
420 
421 	/* Read and write assoclen bytes */
422 	if (is_qi || era < 3) {
423 		append_math_add(desc, VARSEQINLEN, ZERO, REG3, CAAM_CMD_SZ);
424 		if (geniv)
425 			append_math_add_imm_u32(desc, VARSEQOUTLEN, REG3, IMM,
426 						ivsize);
427 		else
428 			append_math_add(desc, VARSEQOUTLEN, ZERO, REG3,
429 					CAAM_CMD_SZ);
430 	} else {
431 		append_math_add(desc, VARSEQINLEN, ZERO, DPOVRD, CAAM_CMD_SZ);
432 		if (geniv)
433 			append_math_add_imm_u32(desc, VARSEQOUTLEN, DPOVRD, IMM,
434 						ivsize);
435 		else
436 			append_math_add(desc, VARSEQOUTLEN, ZERO, DPOVRD,
437 					CAAM_CMD_SZ);
438 	}
439 
440 	/* Skip assoc data */
441 	append_seq_fifo_store(desc, 0, FIFOST_TYPE_SKIP | FIFOLDST_VLF);
442 
443 	/* read assoc before reading payload */
444 	append_seq_fifo_load(desc, 0, FIFOLD_CLASS_CLASS2 | FIFOLD_TYPE_MSG |
445 			     KEY_VLF);
446 
447 	if (geniv) {
448 		append_seq_load(desc, ivsize, LDST_CLASS_1_CCB |
449 				LDST_SRCDST_BYTE_CONTEXT |
450 				(ctx1_iv_off << LDST_OFFSET_SHIFT));
451 		append_move(desc, MOVE_SRC_CLASS1CTX | MOVE_DEST_CLASS2INFIFO |
452 			    (ctx1_iv_off << MOVE_OFFSET_SHIFT) | ivsize);
453 	}
454 
455 	/* Load Counter into CONTEXT1 reg */
456 	if (is_rfc3686)
457 		append_load_imm_be32(desc, 1, LDST_IMM | LDST_CLASS_1_CCB |
458 				     LDST_SRCDST_BYTE_CONTEXT |
459 				     ((ctx1_iv_off + CTR_RFC3686_IV_SIZE) <<
460 				      LDST_OFFSET_SHIFT));
461 
462 	/* Choose operation */
463 	if (ctx1_iv_off)
464 		append_operation(desc, cdata->algtype | OP_ALG_AS_INITFINAL |
465 				 OP_ALG_DECRYPT);
466 	else
467 		append_dec_op1(desc, cdata->algtype);
468 
469 	/* Read and write cryptlen bytes */
470 	append_math_add(desc, VARSEQINLEN, SEQOUTLEN, REG0, CAAM_CMD_SZ);
471 	append_math_add(desc, VARSEQOUTLEN, SEQOUTLEN, REG0, CAAM_CMD_SZ);
472 	aead_append_src_dst(desc, FIFOLD_TYPE_MSG);
473 
474 	/* Load ICV */
475 	append_seq_fifo_load(desc, icvsize, FIFOLD_CLASS_CLASS2 |
476 			     FIFOLD_TYPE_LAST2 | FIFOLD_TYPE_ICV);
477 
478 #ifdef DEBUG
479 	print_hex_dump(KERN_ERR, "aead dec shdesc@" __stringify(__LINE__)": ",
480 		       DUMP_PREFIX_ADDRESS, 16, 4, desc, desc_bytes(desc), 1);
481 #endif
482 }
483 EXPORT_SYMBOL(cnstr_shdsc_aead_decap);
484 
485 /**
486  * cnstr_shdsc_aead_givencap - IPSec ESP encapsulation shared descriptor
487  *                             (non-protocol) with HW-generated initialization
488  *                             vector.
489  * @desc: pointer to buffer used for descriptor construction
490  * @cdata: pointer to block cipher transform definitions
491  *         Valid algorithm values - one of OP_ALG_ALGSEL_{AES, DES, 3DES} ANDed
492  *         with OP_ALG_AAI_CBC or OP_ALG_AAI_CTR_MOD128.
493  * @adata: pointer to authentication transform definitions.
494  *         A split key is required for SEC Era < 6; the size of the split key
495  *         is specified in this case. Valid algorithm values - one of
496  *         OP_ALG_ALGSEL_{MD5, SHA1, SHA224, SHA256, SHA384, SHA512} ANDed
497  *         with OP_ALG_AAI_HMAC_PRECOMP.
498  * @ivsize: initialization vector size
499  * @icvsize: integrity check value (ICV) size (truncated or full)
500  * @is_rfc3686: true when ctr(aes) is wrapped by rfc3686 template
501  * @nonce: pointer to rfc3686 nonce
502  * @ctx1_iv_off: IV offset in CONTEXT1 register
503  * @is_qi: true when called from caam/qi
504  * @era: SEC Era
505  */
506 void cnstr_shdsc_aead_givencap(u32 * const desc, struct alginfo *cdata,
507 			       struct alginfo *adata, unsigned int ivsize,
508 			       unsigned int icvsize, const bool is_rfc3686,
509 			       u32 *nonce, const u32 ctx1_iv_off,
510 			       const bool is_qi, int era)
511 {
512 	u32 geniv, moveiv;
513 
514 	/* Note: Context registers are saved. */
515 	init_sh_desc_key_aead(desc, cdata, adata, is_rfc3686, nonce, era);
516 
517 	if (is_qi) {
518 		u32 *wait_load_cmd;
519 
520 		/* REG3 = assoclen */
521 		append_seq_load(desc, 4, LDST_CLASS_DECO |
522 				LDST_SRCDST_WORD_DECO_MATH3 |
523 				(4 << LDST_OFFSET_SHIFT));
524 
525 		wait_load_cmd = append_jump(desc, JUMP_JSL | JUMP_TEST_ALL |
526 					    JUMP_COND_CALM | JUMP_COND_NCP |
527 					    JUMP_COND_NOP | JUMP_COND_NIP |
528 					    JUMP_COND_NIFP);
529 		set_jump_tgt_here(desc, wait_load_cmd);
530 	}
531 
532 	if (is_rfc3686) {
533 		if (is_qi)
534 			append_seq_load(desc, ivsize, LDST_CLASS_1_CCB |
535 					LDST_SRCDST_BYTE_CONTEXT |
536 					(ctx1_iv_off << LDST_OFFSET_SHIFT));
537 
538 		goto copy_iv;
539 	}
540 
541 	/* Generate IV */
542 	geniv = NFIFOENTRY_STYPE_PAD | NFIFOENTRY_DEST_DECO |
543 		NFIFOENTRY_DTYPE_MSG | NFIFOENTRY_LC1 |
544 		NFIFOENTRY_PTYPE_RND | (ivsize << NFIFOENTRY_DLEN_SHIFT);
545 	append_load_imm_u32(desc, geniv, LDST_CLASS_IND_CCB |
546 			    LDST_SRCDST_WORD_INFO_FIFO | LDST_IMM);
547 	append_cmd(desc, CMD_LOAD | DISABLE_AUTO_INFO_FIFO);
548 	append_move(desc, MOVE_WAITCOMP |
549 		    MOVE_SRC_INFIFO | MOVE_DEST_CLASS1CTX |
550 		    (ctx1_iv_off << MOVE_OFFSET_SHIFT) |
551 		    (ivsize << MOVE_LEN_SHIFT));
552 	append_cmd(desc, CMD_LOAD | ENABLE_AUTO_INFO_FIFO);
553 
554 copy_iv:
555 	/* Copy IV to class 1 context */
556 	append_move(desc, MOVE_SRC_CLASS1CTX | MOVE_DEST_OUTFIFO |
557 		    (ctx1_iv_off << MOVE_OFFSET_SHIFT) |
558 		    (ivsize << MOVE_LEN_SHIFT));
559 
560 	/* Return to encryption */
561 	append_operation(desc, adata->algtype | OP_ALG_AS_INITFINAL |
562 			 OP_ALG_ENCRYPT);
563 
564 	/* Read and write assoclen bytes */
565 	if (is_qi || era < 3) {
566 		append_math_add(desc, VARSEQINLEN, ZERO, REG3, CAAM_CMD_SZ);
567 		append_math_add(desc, VARSEQOUTLEN, ZERO, REG3, CAAM_CMD_SZ);
568 	} else {
569 		append_math_add(desc, VARSEQINLEN, ZERO, DPOVRD, CAAM_CMD_SZ);
570 		append_math_add(desc, VARSEQOUTLEN, ZERO, DPOVRD, CAAM_CMD_SZ);
571 	}
572 
573 	/* Skip assoc data */
574 	append_seq_fifo_store(desc, 0, FIFOST_TYPE_SKIP | FIFOLDST_VLF);
575 
576 	/* read assoc before reading payload */
577 	append_seq_fifo_load(desc, 0, FIFOLD_CLASS_CLASS2 | FIFOLD_TYPE_MSG |
578 			     KEY_VLF);
579 
580 	/* Copy iv from outfifo to class 2 fifo */
581 	moveiv = NFIFOENTRY_STYPE_OFIFO | NFIFOENTRY_DEST_CLASS2 |
582 		 NFIFOENTRY_DTYPE_MSG | (ivsize << NFIFOENTRY_DLEN_SHIFT);
583 	append_load_imm_u32(desc, moveiv, LDST_CLASS_IND_CCB |
584 			    LDST_SRCDST_WORD_INFO_FIFO | LDST_IMM);
585 	append_load_imm_u32(desc, ivsize, LDST_CLASS_2_CCB |
586 			    LDST_SRCDST_WORD_DATASZ_REG | LDST_IMM);
587 
588 	/* Load Counter into CONTEXT1 reg */
589 	if (is_rfc3686)
590 		append_load_imm_be32(desc, 1, LDST_IMM | LDST_CLASS_1_CCB |
591 				     LDST_SRCDST_BYTE_CONTEXT |
592 				     ((ctx1_iv_off + CTR_RFC3686_IV_SIZE) <<
593 				      LDST_OFFSET_SHIFT));
594 
595 	/* Class 1 operation */
596 	append_operation(desc, cdata->algtype | OP_ALG_AS_INITFINAL |
597 			 OP_ALG_ENCRYPT);
598 
599 	/* Will write ivsize + cryptlen */
600 	append_math_add(desc, VARSEQOUTLEN, SEQINLEN, REG0, CAAM_CMD_SZ);
601 
602 	/* Not need to reload iv */
603 	append_seq_fifo_load(desc, ivsize,
604 			     FIFOLD_CLASS_SKIP);
605 
606 	/* Will read cryptlen */
607 	append_math_add(desc, VARSEQINLEN, SEQINLEN, REG0, CAAM_CMD_SZ);
608 	append_seq_fifo_load(desc, 0, FIFOLD_CLASS_BOTH | KEY_VLF |
609 			     FIFOLD_TYPE_MSG1OUT2 | FIFOLD_TYPE_LASTBOTH);
610 	append_seq_fifo_store(desc, 0, FIFOST_TYPE_MESSAGE_DATA | KEY_VLF);
611 
612 	/* Write ICV */
613 	append_seq_store(desc, icvsize, LDST_CLASS_2_CCB |
614 			 LDST_SRCDST_BYTE_CONTEXT);
615 
616 #ifdef DEBUG
617 	print_hex_dump(KERN_ERR,
618 		       "aead givenc shdesc@" __stringify(__LINE__)": ",
619 		       DUMP_PREFIX_ADDRESS, 16, 4, desc, desc_bytes(desc), 1);
620 #endif
621 }
622 EXPORT_SYMBOL(cnstr_shdsc_aead_givencap);
623 
624 /**
625  * cnstr_shdsc_gcm_encap - gcm encapsulation shared descriptor
626  * @desc: pointer to buffer used for descriptor construction
627  * @cdata: pointer to block cipher transform definitions
628  *         Valid algorithm values - OP_ALG_ALGSEL_AES ANDed with OP_ALG_AAI_GCM.
629  * @ivsize: initialization vector size
630  * @icvsize: integrity check value (ICV) size (truncated or full)
631  * @is_qi: true when called from caam/qi
632  */
633 void cnstr_shdsc_gcm_encap(u32 * const desc, struct alginfo *cdata,
634 			   unsigned int ivsize, unsigned int icvsize,
635 			   const bool is_qi)
636 {
637 	u32 *key_jump_cmd, *zero_payload_jump_cmd, *zero_assoc_jump_cmd1,
638 	    *zero_assoc_jump_cmd2;
639 
640 	init_sh_desc(desc, HDR_SHARE_SERIAL);
641 
642 	/* skip key loading if they are loaded due to sharing */
643 	key_jump_cmd = append_jump(desc, JUMP_JSL | JUMP_TEST_ALL |
644 				   JUMP_COND_SHRD);
645 	if (cdata->key_inline)
646 		append_key_as_imm(desc, cdata->key_virt, cdata->keylen,
647 				  cdata->keylen, CLASS_1 | KEY_DEST_CLASS_REG);
648 	else
649 		append_key(desc, cdata->key_dma, cdata->keylen, CLASS_1 |
650 			   KEY_DEST_CLASS_REG);
651 	set_jump_tgt_here(desc, key_jump_cmd);
652 
653 	/* class 1 operation */
654 	append_operation(desc, cdata->algtype | OP_ALG_AS_INITFINAL |
655 			 OP_ALG_ENCRYPT);
656 
657 	if (is_qi) {
658 		u32 *wait_load_cmd;
659 
660 		/* REG3 = assoclen */
661 		append_seq_load(desc, 4, LDST_CLASS_DECO |
662 				LDST_SRCDST_WORD_DECO_MATH3 |
663 				(4 << LDST_OFFSET_SHIFT));
664 
665 		wait_load_cmd = append_jump(desc, JUMP_JSL | JUMP_TEST_ALL |
666 					    JUMP_COND_CALM | JUMP_COND_NCP |
667 					    JUMP_COND_NOP | JUMP_COND_NIP |
668 					    JUMP_COND_NIFP);
669 		set_jump_tgt_here(desc, wait_load_cmd);
670 
671 		append_math_sub_imm_u32(desc, VARSEQOUTLEN, SEQINLEN, IMM,
672 					ivsize);
673 	} else {
674 		append_math_sub(desc, VARSEQOUTLEN, SEQINLEN, REG0,
675 				CAAM_CMD_SZ);
676 	}
677 
678 	/* if assoclen + cryptlen is ZERO, skip to ICV write */
679 	zero_assoc_jump_cmd2 = append_jump(desc, JUMP_TEST_ALL |
680 						 JUMP_COND_MATH_Z);
681 
682 	if (is_qi)
683 		append_seq_fifo_load(desc, ivsize, FIFOLD_CLASS_CLASS1 |
684 				     FIFOLD_TYPE_IV | FIFOLD_TYPE_FLUSH1);
685 
686 	/* if assoclen is ZERO, skip reading the assoc data */
687 	append_math_add(desc, VARSEQINLEN, ZERO, REG3, CAAM_CMD_SZ);
688 	zero_assoc_jump_cmd1 = append_jump(desc, JUMP_TEST_ALL |
689 					   JUMP_COND_MATH_Z);
690 
691 	append_math_add(desc, VARSEQOUTLEN, ZERO, REG3, CAAM_CMD_SZ);
692 
693 	/* skip assoc data */
694 	append_seq_fifo_store(desc, 0, FIFOST_TYPE_SKIP | FIFOLDST_VLF);
695 
696 	/* cryptlen = seqinlen - assoclen */
697 	append_math_sub(desc, VARSEQOUTLEN, SEQINLEN, REG3, CAAM_CMD_SZ);
698 
699 	/* if cryptlen is ZERO jump to zero-payload commands */
700 	zero_payload_jump_cmd = append_jump(desc, JUMP_TEST_ALL |
701 					    JUMP_COND_MATH_Z);
702 
703 	/* read assoc data */
704 	append_seq_fifo_load(desc, 0, FIFOLD_CLASS_CLASS1 | FIFOLDST_VLF |
705 			     FIFOLD_TYPE_AAD | FIFOLD_TYPE_FLUSH1);
706 	set_jump_tgt_here(desc, zero_assoc_jump_cmd1);
707 
708 	append_math_sub(desc, VARSEQINLEN, SEQINLEN, REG0, CAAM_CMD_SZ);
709 
710 	/* write encrypted data */
711 	append_seq_fifo_store(desc, 0, FIFOST_TYPE_MESSAGE_DATA | FIFOLDST_VLF);
712 
713 	/* read payload data */
714 	append_seq_fifo_load(desc, 0, FIFOLD_CLASS_CLASS1 | FIFOLDST_VLF |
715 			     FIFOLD_TYPE_MSG | FIFOLD_TYPE_LAST1);
716 
717 	/* jump to ICV writing */
718 	if (is_qi)
719 		append_jump(desc, JUMP_TEST_ALL | 4);
720 	else
721 		append_jump(desc, JUMP_TEST_ALL | 2);
722 
723 	/* zero-payload commands */
724 	set_jump_tgt_here(desc, zero_payload_jump_cmd);
725 
726 	/* read assoc data */
727 	append_seq_fifo_load(desc, 0, FIFOLD_CLASS_CLASS1 | FIFOLDST_VLF |
728 			     FIFOLD_TYPE_AAD | FIFOLD_TYPE_LAST1);
729 	if (is_qi)
730 		/* jump to ICV writing */
731 		append_jump(desc, JUMP_TEST_ALL | 2);
732 
733 	/* There is no input data */
734 	set_jump_tgt_here(desc, zero_assoc_jump_cmd2);
735 
736 	if (is_qi)
737 		append_seq_fifo_load(desc, ivsize, FIFOLD_CLASS_CLASS1 |
738 				     FIFOLD_TYPE_IV | FIFOLD_TYPE_FLUSH1 |
739 				     FIFOLD_TYPE_LAST1);
740 
741 	/* write ICV */
742 	append_seq_store(desc, icvsize, LDST_CLASS_1_CCB |
743 			 LDST_SRCDST_BYTE_CONTEXT);
744 
745 #ifdef DEBUG
746 	print_hex_dump(KERN_ERR, "gcm enc shdesc@" __stringify(__LINE__)": ",
747 		       DUMP_PREFIX_ADDRESS, 16, 4, desc, desc_bytes(desc), 1);
748 #endif
749 }
750 EXPORT_SYMBOL(cnstr_shdsc_gcm_encap);
751 
752 /**
753  * cnstr_shdsc_gcm_decap - gcm decapsulation shared descriptor
754  * @desc: pointer to buffer used for descriptor construction
755  * @cdata: pointer to block cipher transform definitions
756  *         Valid algorithm values - OP_ALG_ALGSEL_AES ANDed with OP_ALG_AAI_GCM.
757  * @ivsize: initialization vector size
758  * @icvsize: integrity check value (ICV) size (truncated or full)
759  * @is_qi: true when called from caam/qi
760  */
761 void cnstr_shdsc_gcm_decap(u32 * const desc, struct alginfo *cdata,
762 			   unsigned int ivsize, unsigned int icvsize,
763 			   const bool is_qi)
764 {
765 	u32 *key_jump_cmd, *zero_payload_jump_cmd, *zero_assoc_jump_cmd1;
766 
767 	init_sh_desc(desc, HDR_SHARE_SERIAL);
768 
769 	/* skip key loading if they are loaded due to sharing */
770 	key_jump_cmd = append_jump(desc, JUMP_JSL |
771 				   JUMP_TEST_ALL | JUMP_COND_SHRD);
772 	if (cdata->key_inline)
773 		append_key_as_imm(desc, cdata->key_virt, cdata->keylen,
774 				  cdata->keylen, CLASS_1 | KEY_DEST_CLASS_REG);
775 	else
776 		append_key(desc, cdata->key_dma, cdata->keylen, CLASS_1 |
777 			   KEY_DEST_CLASS_REG);
778 	set_jump_tgt_here(desc, key_jump_cmd);
779 
780 	/* class 1 operation */
781 	append_operation(desc, cdata->algtype | OP_ALG_AS_INITFINAL |
782 			 OP_ALG_DECRYPT | OP_ALG_ICV_ON);
783 
784 	if (is_qi) {
785 		u32 *wait_load_cmd;
786 
787 		/* REG3 = assoclen */
788 		append_seq_load(desc, 4, LDST_CLASS_DECO |
789 				LDST_SRCDST_WORD_DECO_MATH3 |
790 				(4 << LDST_OFFSET_SHIFT));
791 
792 		wait_load_cmd = append_jump(desc, JUMP_JSL | JUMP_TEST_ALL |
793 					    JUMP_COND_CALM | JUMP_COND_NCP |
794 					    JUMP_COND_NOP | JUMP_COND_NIP |
795 					    JUMP_COND_NIFP);
796 		set_jump_tgt_here(desc, wait_load_cmd);
797 
798 		append_seq_fifo_load(desc, ivsize, FIFOLD_CLASS_CLASS1 |
799 				     FIFOLD_TYPE_IV | FIFOLD_TYPE_FLUSH1);
800 	}
801 
802 	/* if assoclen is ZERO, skip reading the assoc data */
803 	append_math_add(desc, VARSEQINLEN, ZERO, REG3, CAAM_CMD_SZ);
804 	zero_assoc_jump_cmd1 = append_jump(desc, JUMP_TEST_ALL |
805 						 JUMP_COND_MATH_Z);
806 
807 	append_math_add(desc, VARSEQOUTLEN, ZERO, REG3, CAAM_CMD_SZ);
808 
809 	/* skip assoc data */
810 	append_seq_fifo_store(desc, 0, FIFOST_TYPE_SKIP | FIFOLDST_VLF);
811 
812 	/* read assoc data */
813 	append_seq_fifo_load(desc, 0, FIFOLD_CLASS_CLASS1 | FIFOLDST_VLF |
814 			     FIFOLD_TYPE_AAD | FIFOLD_TYPE_FLUSH1);
815 
816 	set_jump_tgt_here(desc, zero_assoc_jump_cmd1);
817 
818 	/* cryptlen = seqoutlen - assoclen */
819 	append_math_sub(desc, VARSEQINLEN, SEQOUTLEN, REG0, CAAM_CMD_SZ);
820 
821 	/* jump to zero-payload command if cryptlen is zero */
822 	zero_payload_jump_cmd = append_jump(desc, JUMP_TEST_ALL |
823 					    JUMP_COND_MATH_Z);
824 
825 	append_math_sub(desc, VARSEQOUTLEN, SEQOUTLEN, REG0, CAAM_CMD_SZ);
826 
827 	/* store encrypted data */
828 	append_seq_fifo_store(desc, 0, FIFOST_TYPE_MESSAGE_DATA | FIFOLDST_VLF);
829 
830 	/* read payload data */
831 	append_seq_fifo_load(desc, 0, FIFOLD_CLASS_CLASS1 | FIFOLDST_VLF |
832 			     FIFOLD_TYPE_MSG | FIFOLD_TYPE_FLUSH1);
833 
834 	/* zero-payload command */
835 	set_jump_tgt_here(desc, zero_payload_jump_cmd);
836 
837 	/* read ICV */
838 	append_seq_fifo_load(desc, icvsize, FIFOLD_CLASS_CLASS1 |
839 			     FIFOLD_TYPE_ICV | FIFOLD_TYPE_LAST1);
840 
841 #ifdef DEBUG
842 	print_hex_dump(KERN_ERR, "gcm dec shdesc@" __stringify(__LINE__)": ",
843 		       DUMP_PREFIX_ADDRESS, 16, 4, desc, desc_bytes(desc), 1);
844 #endif
845 }
846 EXPORT_SYMBOL(cnstr_shdsc_gcm_decap);
847 
848 /**
849  * cnstr_shdsc_rfc4106_encap - IPSec ESP gcm encapsulation shared descriptor
850  *                             (non-protocol).
851  * @desc: pointer to buffer used for descriptor construction
852  * @cdata: pointer to block cipher transform definitions
853  *         Valid algorithm values - OP_ALG_ALGSEL_AES ANDed with OP_ALG_AAI_GCM.
854  * @ivsize: initialization vector size
855  * @icvsize: integrity check value (ICV) size (truncated or full)
856  * @is_qi: true when called from caam/qi
857  */
858 void cnstr_shdsc_rfc4106_encap(u32 * const desc, struct alginfo *cdata,
859 			       unsigned int ivsize, unsigned int icvsize,
860 			       const bool is_qi)
861 {
862 	u32 *key_jump_cmd;
863 
864 	init_sh_desc(desc, HDR_SHARE_SERIAL);
865 
866 	/* Skip key loading if it is loaded due to sharing */
867 	key_jump_cmd = append_jump(desc, JUMP_JSL | JUMP_TEST_ALL |
868 				   JUMP_COND_SHRD);
869 	if (cdata->key_inline)
870 		append_key_as_imm(desc, cdata->key_virt, cdata->keylen,
871 				  cdata->keylen, CLASS_1 | KEY_DEST_CLASS_REG);
872 	else
873 		append_key(desc, cdata->key_dma, cdata->keylen, CLASS_1 |
874 			   KEY_DEST_CLASS_REG);
875 	set_jump_tgt_here(desc, key_jump_cmd);
876 
877 	/* Class 1 operation */
878 	append_operation(desc, cdata->algtype | OP_ALG_AS_INITFINAL |
879 			 OP_ALG_ENCRYPT);
880 
881 	if (is_qi) {
882 		u32 *wait_load_cmd;
883 
884 		/* REG3 = assoclen */
885 		append_seq_load(desc, 4, LDST_CLASS_DECO |
886 				LDST_SRCDST_WORD_DECO_MATH3 |
887 				(4 << LDST_OFFSET_SHIFT));
888 
889 		wait_load_cmd = append_jump(desc, JUMP_JSL | JUMP_TEST_ALL |
890 					    JUMP_COND_CALM | JUMP_COND_NCP |
891 					    JUMP_COND_NOP | JUMP_COND_NIP |
892 					    JUMP_COND_NIFP);
893 		set_jump_tgt_here(desc, wait_load_cmd);
894 
895 		/* Read salt and IV */
896 		append_fifo_load_as_imm(desc, (void *)(cdata->key_virt +
897 					cdata->keylen), 4, FIFOLD_CLASS_CLASS1 |
898 					FIFOLD_TYPE_IV);
899 		append_seq_fifo_load(desc, ivsize, FIFOLD_CLASS_CLASS1 |
900 				     FIFOLD_TYPE_IV | FIFOLD_TYPE_FLUSH1);
901 	}
902 
903 	append_math_sub_imm_u32(desc, VARSEQINLEN, REG3, IMM, ivsize);
904 	append_math_add(desc, VARSEQOUTLEN, ZERO, REG3, CAAM_CMD_SZ);
905 
906 	/* Read assoc data */
907 	append_seq_fifo_load(desc, 0, FIFOLD_CLASS_CLASS1 | FIFOLDST_VLF |
908 			     FIFOLD_TYPE_AAD | FIFOLD_TYPE_FLUSH1);
909 
910 	/* Skip IV */
911 	append_seq_fifo_load(desc, ivsize, FIFOLD_CLASS_SKIP);
912 
913 	/* Will read cryptlen bytes */
914 	append_math_sub(desc, VARSEQINLEN, SEQINLEN, REG0, CAAM_CMD_SZ);
915 
916 	/* Workaround for erratum A-005473 (simultaneous SEQ FIFO skips) */
917 	append_seq_fifo_load(desc, 0, FIFOLD_CLASS_CLASS1 | FIFOLD_TYPE_MSG);
918 
919 	/* Skip assoc data */
920 	append_seq_fifo_store(desc, 0, FIFOST_TYPE_SKIP | FIFOLDST_VLF);
921 
922 	/* cryptlen = seqoutlen - assoclen */
923 	append_math_sub(desc, VARSEQOUTLEN, VARSEQINLEN, REG0, CAAM_CMD_SZ);
924 
925 	/* Write encrypted data */
926 	append_seq_fifo_store(desc, 0, FIFOST_TYPE_MESSAGE_DATA | FIFOLDST_VLF);
927 
928 	/* Read payload data */
929 	append_seq_fifo_load(desc, 0, FIFOLD_CLASS_CLASS1 | FIFOLDST_VLF |
930 			     FIFOLD_TYPE_MSG | FIFOLD_TYPE_LAST1);
931 
932 	/* Write ICV */
933 	append_seq_store(desc, icvsize, LDST_CLASS_1_CCB |
934 			 LDST_SRCDST_BYTE_CONTEXT);
935 
936 #ifdef DEBUG
937 	print_hex_dump(KERN_ERR,
938 		       "rfc4106 enc shdesc@" __stringify(__LINE__)": ",
939 		       DUMP_PREFIX_ADDRESS, 16, 4, desc, desc_bytes(desc), 1);
940 #endif
941 }
942 EXPORT_SYMBOL(cnstr_shdsc_rfc4106_encap);
943 
944 /**
945  * cnstr_shdsc_rfc4106_decap - IPSec ESP gcm decapsulation shared descriptor
946  *                             (non-protocol).
947  * @desc: pointer to buffer used for descriptor construction
948  * @cdata: pointer to block cipher transform definitions
949  *         Valid algorithm values - OP_ALG_ALGSEL_AES ANDed with OP_ALG_AAI_GCM.
950  * @ivsize: initialization vector size
951  * @icvsize: integrity check value (ICV) size (truncated or full)
952  * @is_qi: true when called from caam/qi
953  */
954 void cnstr_shdsc_rfc4106_decap(u32 * const desc, struct alginfo *cdata,
955 			       unsigned int ivsize, unsigned int icvsize,
956 			       const bool is_qi)
957 {
958 	u32 *key_jump_cmd;
959 
960 	init_sh_desc(desc, HDR_SHARE_SERIAL);
961 
962 	/* Skip key loading if it is loaded due to sharing */
963 	key_jump_cmd = append_jump(desc, JUMP_JSL | JUMP_TEST_ALL |
964 				   JUMP_COND_SHRD);
965 	if (cdata->key_inline)
966 		append_key_as_imm(desc, cdata->key_virt, cdata->keylen,
967 				  cdata->keylen, CLASS_1 |
968 				  KEY_DEST_CLASS_REG);
969 	else
970 		append_key(desc, cdata->key_dma, cdata->keylen, CLASS_1 |
971 			   KEY_DEST_CLASS_REG);
972 	set_jump_tgt_here(desc, key_jump_cmd);
973 
974 	/* Class 1 operation */
975 	append_operation(desc, cdata->algtype | OP_ALG_AS_INITFINAL |
976 			 OP_ALG_DECRYPT | OP_ALG_ICV_ON);
977 
978 	if (is_qi) {
979 		u32 *wait_load_cmd;
980 
981 		/* REG3 = assoclen */
982 		append_seq_load(desc, 4, LDST_CLASS_DECO |
983 				LDST_SRCDST_WORD_DECO_MATH3 |
984 				(4 << LDST_OFFSET_SHIFT));
985 
986 		wait_load_cmd = append_jump(desc, JUMP_JSL | JUMP_TEST_ALL |
987 					    JUMP_COND_CALM | JUMP_COND_NCP |
988 					    JUMP_COND_NOP | JUMP_COND_NIP |
989 					    JUMP_COND_NIFP);
990 		set_jump_tgt_here(desc, wait_load_cmd);
991 
992 		/* Read salt and IV */
993 		append_fifo_load_as_imm(desc, (void *)(cdata->key_virt +
994 					cdata->keylen), 4, FIFOLD_CLASS_CLASS1 |
995 					FIFOLD_TYPE_IV);
996 		append_seq_fifo_load(desc, ivsize, FIFOLD_CLASS_CLASS1 |
997 				     FIFOLD_TYPE_IV | FIFOLD_TYPE_FLUSH1);
998 	}
999 
1000 	append_math_sub_imm_u32(desc, VARSEQINLEN, REG3, IMM, ivsize);
1001 	append_math_add(desc, VARSEQOUTLEN, ZERO, REG3, CAAM_CMD_SZ);
1002 
1003 	/* Read assoc data */
1004 	append_seq_fifo_load(desc, 0, FIFOLD_CLASS_CLASS1 | FIFOLDST_VLF |
1005 			     FIFOLD_TYPE_AAD | FIFOLD_TYPE_FLUSH1);
1006 
1007 	/* Skip IV */
1008 	append_seq_fifo_load(desc, ivsize, FIFOLD_CLASS_SKIP);
1009 
1010 	/* Will read cryptlen bytes */
1011 	append_math_sub(desc, VARSEQINLEN, SEQOUTLEN, REG3, CAAM_CMD_SZ);
1012 
1013 	/* Workaround for erratum A-005473 (simultaneous SEQ FIFO skips) */
1014 	append_seq_fifo_load(desc, 0, FIFOLD_CLASS_CLASS1 | FIFOLD_TYPE_MSG);
1015 
1016 	/* Skip assoc data */
1017 	append_seq_fifo_store(desc, 0, FIFOST_TYPE_SKIP | FIFOLDST_VLF);
1018 
1019 	/* Will write cryptlen bytes */
1020 	append_math_sub(desc, VARSEQOUTLEN, SEQOUTLEN, REG0, CAAM_CMD_SZ);
1021 
1022 	/* Store payload data */
1023 	append_seq_fifo_store(desc, 0, FIFOST_TYPE_MESSAGE_DATA | FIFOLDST_VLF);
1024 
1025 	/* Read encrypted data */
1026 	append_seq_fifo_load(desc, 0, FIFOLD_CLASS_CLASS1 | FIFOLDST_VLF |
1027 			     FIFOLD_TYPE_MSG | FIFOLD_TYPE_FLUSH1);
1028 
1029 	/* Read ICV */
1030 	append_seq_fifo_load(desc, icvsize, FIFOLD_CLASS_CLASS1 |
1031 			     FIFOLD_TYPE_ICV | FIFOLD_TYPE_LAST1);
1032 
1033 #ifdef DEBUG
1034 	print_hex_dump(KERN_ERR,
1035 		       "rfc4106 dec shdesc@" __stringify(__LINE__)": ",
1036 		       DUMP_PREFIX_ADDRESS, 16, 4, desc, desc_bytes(desc), 1);
1037 #endif
1038 }
1039 EXPORT_SYMBOL(cnstr_shdsc_rfc4106_decap);
1040 
1041 /**
1042  * cnstr_shdsc_rfc4543_encap - IPSec ESP gmac encapsulation shared descriptor
1043  *                             (non-protocol).
1044  * @desc: pointer to buffer used for descriptor construction
1045  * @cdata: pointer to block cipher transform definitions
1046  *         Valid algorithm values - OP_ALG_ALGSEL_AES ANDed with OP_ALG_AAI_GCM.
1047  * @ivsize: initialization vector size
1048  * @icvsize: integrity check value (ICV) size (truncated or full)
1049  * @is_qi: true when called from caam/qi
1050  */
1051 void cnstr_shdsc_rfc4543_encap(u32 * const desc, struct alginfo *cdata,
1052 			       unsigned int ivsize, unsigned int icvsize,
1053 			       const bool is_qi)
1054 {
1055 	u32 *key_jump_cmd, *read_move_cmd, *write_move_cmd;
1056 
1057 	init_sh_desc(desc, HDR_SHARE_SERIAL);
1058 
1059 	/* Skip key loading if it is loaded due to sharing */
1060 	key_jump_cmd = append_jump(desc, JUMP_JSL | JUMP_TEST_ALL |
1061 				   JUMP_COND_SHRD);
1062 	if (cdata->key_inline)
1063 		append_key_as_imm(desc, cdata->key_virt, cdata->keylen,
1064 				  cdata->keylen, CLASS_1 | KEY_DEST_CLASS_REG);
1065 	else
1066 		append_key(desc, cdata->key_dma, cdata->keylen, CLASS_1 |
1067 			   KEY_DEST_CLASS_REG);
1068 	set_jump_tgt_here(desc, key_jump_cmd);
1069 
1070 	/* Class 1 operation */
1071 	append_operation(desc, cdata->algtype | OP_ALG_AS_INITFINAL |
1072 			 OP_ALG_ENCRYPT);
1073 
1074 	if (is_qi) {
1075 		/* assoclen is not needed, skip it */
1076 		append_seq_fifo_load(desc, 4, FIFOLD_CLASS_SKIP);
1077 
1078 		/* Read salt and IV */
1079 		append_fifo_load_as_imm(desc, (void *)(cdata->key_virt +
1080 					cdata->keylen), 4, FIFOLD_CLASS_CLASS1 |
1081 					FIFOLD_TYPE_IV);
1082 		append_seq_fifo_load(desc, ivsize, FIFOLD_CLASS_CLASS1 |
1083 				     FIFOLD_TYPE_IV | FIFOLD_TYPE_FLUSH1);
1084 	}
1085 
1086 	/* assoclen + cryptlen = seqinlen */
1087 	append_math_sub(desc, REG3, SEQINLEN, REG0, CAAM_CMD_SZ);
1088 
1089 	/*
1090 	 * MOVE_LEN opcode is not available in all SEC HW revisions,
1091 	 * thus need to do some magic, i.e. self-patch the descriptor
1092 	 * buffer.
1093 	 */
1094 	read_move_cmd = append_move(desc, MOVE_SRC_DESCBUF | MOVE_DEST_MATH3 |
1095 				    (0x6 << MOVE_LEN_SHIFT));
1096 	write_move_cmd = append_move(desc, MOVE_SRC_MATH3 | MOVE_DEST_DESCBUF |
1097 				     (0x8 << MOVE_LEN_SHIFT) | MOVE_WAITCOMP);
1098 
1099 	/* Will read assoclen + cryptlen bytes */
1100 	append_math_sub(desc, VARSEQINLEN, SEQINLEN, REG0, CAAM_CMD_SZ);
1101 
1102 	/* Will write assoclen + cryptlen bytes */
1103 	append_math_sub(desc, VARSEQOUTLEN, SEQINLEN, REG0, CAAM_CMD_SZ);
1104 
1105 	/* Read and write assoclen + cryptlen bytes */
1106 	aead_append_src_dst(desc, FIFOLD_TYPE_AAD);
1107 
1108 	set_move_tgt_here(desc, read_move_cmd);
1109 	set_move_tgt_here(desc, write_move_cmd);
1110 	append_cmd(desc, CMD_LOAD | DISABLE_AUTO_INFO_FIFO);
1111 	/* Move payload data to OFIFO */
1112 	append_move(desc, MOVE_SRC_INFIFO_CL | MOVE_DEST_OUTFIFO);
1113 
1114 	/* Write ICV */
1115 	append_seq_store(desc, icvsize, LDST_CLASS_1_CCB |
1116 			 LDST_SRCDST_BYTE_CONTEXT);
1117 
1118 #ifdef DEBUG
1119 	print_hex_dump(KERN_ERR,
1120 		       "rfc4543 enc shdesc@" __stringify(__LINE__)": ",
1121 		       DUMP_PREFIX_ADDRESS, 16, 4, desc, desc_bytes(desc), 1);
1122 #endif
1123 }
1124 EXPORT_SYMBOL(cnstr_shdsc_rfc4543_encap);
1125 
1126 /**
1127  * cnstr_shdsc_rfc4543_decap - IPSec ESP gmac decapsulation shared descriptor
1128  *                             (non-protocol).
1129  * @desc: pointer to buffer used for descriptor construction
1130  * @cdata: pointer to block cipher transform definitions
1131  *         Valid algorithm values - OP_ALG_ALGSEL_AES ANDed with OP_ALG_AAI_GCM.
1132  * @ivsize: initialization vector size
1133  * @icvsize: integrity check value (ICV) size (truncated or full)
1134  * @is_qi: true when called from caam/qi
1135  */
1136 void cnstr_shdsc_rfc4543_decap(u32 * const desc, struct alginfo *cdata,
1137 			       unsigned int ivsize, unsigned int icvsize,
1138 			       const bool is_qi)
1139 {
1140 	u32 *key_jump_cmd, *read_move_cmd, *write_move_cmd;
1141 
1142 	init_sh_desc(desc, HDR_SHARE_SERIAL);
1143 
1144 	/* Skip key loading if it is loaded due to sharing */
1145 	key_jump_cmd = append_jump(desc, JUMP_JSL | JUMP_TEST_ALL |
1146 				   JUMP_COND_SHRD);
1147 	if (cdata->key_inline)
1148 		append_key_as_imm(desc, cdata->key_virt, cdata->keylen,
1149 				  cdata->keylen, CLASS_1 | KEY_DEST_CLASS_REG);
1150 	else
1151 		append_key(desc, cdata->key_dma, cdata->keylen, CLASS_1 |
1152 			   KEY_DEST_CLASS_REG);
1153 	set_jump_tgt_here(desc, key_jump_cmd);
1154 
1155 	/* Class 1 operation */
1156 	append_operation(desc, cdata->algtype | OP_ALG_AS_INITFINAL |
1157 			 OP_ALG_DECRYPT | OP_ALG_ICV_ON);
1158 
1159 	if (is_qi) {
1160 		/* assoclen is not needed, skip it */
1161 		append_seq_fifo_load(desc, 4, FIFOLD_CLASS_SKIP);
1162 
1163 		/* Read salt and IV */
1164 		append_fifo_load_as_imm(desc, (void *)(cdata->key_virt +
1165 					cdata->keylen), 4, FIFOLD_CLASS_CLASS1 |
1166 					FIFOLD_TYPE_IV);
1167 		append_seq_fifo_load(desc, ivsize, FIFOLD_CLASS_CLASS1 |
1168 				     FIFOLD_TYPE_IV | FIFOLD_TYPE_FLUSH1);
1169 	}
1170 
1171 	/* assoclen + cryptlen = seqoutlen */
1172 	append_math_sub(desc, REG3, SEQOUTLEN, REG0, CAAM_CMD_SZ);
1173 
1174 	/*
1175 	 * MOVE_LEN opcode is not available in all SEC HW revisions,
1176 	 * thus need to do some magic, i.e. self-patch the descriptor
1177 	 * buffer.
1178 	 */
1179 	read_move_cmd = append_move(desc, MOVE_SRC_DESCBUF | MOVE_DEST_MATH3 |
1180 				    (0x6 << MOVE_LEN_SHIFT));
1181 	write_move_cmd = append_move(desc, MOVE_SRC_MATH3 | MOVE_DEST_DESCBUF |
1182 				     (0x8 << MOVE_LEN_SHIFT) | MOVE_WAITCOMP);
1183 
1184 	/* Will read assoclen + cryptlen bytes */
1185 	append_math_sub(desc, VARSEQINLEN, SEQOUTLEN, REG0, CAAM_CMD_SZ);
1186 
1187 	/* Will write assoclen + cryptlen bytes */
1188 	append_math_sub(desc, VARSEQOUTLEN, SEQOUTLEN, REG0, CAAM_CMD_SZ);
1189 
1190 	/* Store payload data */
1191 	append_seq_fifo_store(desc, 0, FIFOST_TYPE_MESSAGE_DATA | FIFOLDST_VLF);
1192 
1193 	/* In-snoop assoclen + cryptlen data */
1194 	append_seq_fifo_load(desc, 0, FIFOLD_CLASS_BOTH | FIFOLDST_VLF |
1195 			     FIFOLD_TYPE_AAD | FIFOLD_TYPE_LAST2FLUSH1);
1196 
1197 	set_move_tgt_here(desc, read_move_cmd);
1198 	set_move_tgt_here(desc, write_move_cmd);
1199 	append_cmd(desc, CMD_LOAD | DISABLE_AUTO_INFO_FIFO);
1200 	/* Move payload data to OFIFO */
1201 	append_move(desc, MOVE_SRC_INFIFO_CL | MOVE_DEST_OUTFIFO);
1202 	append_cmd(desc, CMD_LOAD | ENABLE_AUTO_INFO_FIFO);
1203 
1204 	/* Read ICV */
1205 	append_seq_fifo_load(desc, icvsize, FIFOLD_CLASS_CLASS1 |
1206 			     FIFOLD_TYPE_ICV | FIFOLD_TYPE_LAST1);
1207 
1208 #ifdef DEBUG
1209 	print_hex_dump(KERN_ERR,
1210 		       "rfc4543 dec shdesc@" __stringify(__LINE__)": ",
1211 		       DUMP_PREFIX_ADDRESS, 16, 4, desc, desc_bytes(desc), 1);
1212 #endif
1213 }
1214 EXPORT_SYMBOL(cnstr_shdsc_rfc4543_decap);
1215 
1216 /**
1217  * cnstr_shdsc_chachapoly - Chacha20 + Poly1305 generic AEAD (rfc7539) and
1218  *                          IPsec ESP (rfc7634, a.k.a. rfc7539esp) shared
1219  *                          descriptor (non-protocol).
1220  * @desc: pointer to buffer used for descriptor construction
1221  * @cdata: pointer to block cipher transform definitions
1222  *         Valid algorithm values - OP_ALG_ALGSEL_CHACHA20 ANDed with
1223  *         OP_ALG_AAI_AEAD.
1224  * @adata: pointer to authentication transform definitions
1225  *         Valid algorithm values - OP_ALG_ALGSEL_POLY1305 ANDed with
1226  *         OP_ALG_AAI_AEAD.
1227  * @ivsize: initialization vector size
1228  * @icvsize: integrity check value (ICV) size (truncated or full)
1229  * @encap: true if encapsulation, false if decapsulation
1230  * @is_qi: true when called from caam/qi
1231  */
1232 void cnstr_shdsc_chachapoly(u32 * const desc, struct alginfo *cdata,
1233 			    struct alginfo *adata, unsigned int ivsize,
1234 			    unsigned int icvsize, const bool encap,
1235 			    const bool is_qi)
1236 {
1237 	u32 *key_jump_cmd, *wait_cmd;
1238 	u32 nfifo;
1239 	const bool is_ipsec = (ivsize != CHACHAPOLY_IV_SIZE);
1240 
1241 	/* Note: Context registers are saved. */
1242 	init_sh_desc(desc, HDR_SHARE_SERIAL | HDR_SAVECTX);
1243 
1244 	/* skip key loading if they are loaded due to sharing */
1245 	key_jump_cmd = append_jump(desc, JUMP_JSL | JUMP_TEST_ALL |
1246 				   JUMP_COND_SHRD);
1247 
1248 	append_key_as_imm(desc, cdata->key_virt, cdata->keylen, cdata->keylen,
1249 			  CLASS_1 | KEY_DEST_CLASS_REG);
1250 
1251 	/* For IPsec load the salt from keymat in the context register */
1252 	if (is_ipsec)
1253 		append_load_as_imm(desc, cdata->key_virt + cdata->keylen, 4,
1254 				   LDST_CLASS_1_CCB | LDST_SRCDST_BYTE_CONTEXT |
1255 				   4 << LDST_OFFSET_SHIFT);
1256 
1257 	set_jump_tgt_here(desc, key_jump_cmd);
1258 
1259 	/* Class 2 and 1 operations: Poly & ChaCha */
1260 	if (encap) {
1261 		append_operation(desc, adata->algtype | OP_ALG_AS_INITFINAL |
1262 				 OP_ALG_ENCRYPT);
1263 		append_operation(desc, cdata->algtype | OP_ALG_AS_INITFINAL |
1264 				 OP_ALG_ENCRYPT);
1265 	} else {
1266 		append_operation(desc, adata->algtype | OP_ALG_AS_INITFINAL |
1267 				 OP_ALG_DECRYPT | OP_ALG_ICV_ON);
1268 		append_operation(desc, cdata->algtype | OP_ALG_AS_INITFINAL |
1269 				 OP_ALG_DECRYPT);
1270 	}
1271 
1272 	if (is_qi) {
1273 		u32 *wait_load_cmd;
1274 		u32 ctx1_iv_off = is_ipsec ? 8 : 4;
1275 
1276 		/* REG3 = assoclen */
1277 		append_seq_load(desc, 4, LDST_CLASS_DECO |
1278 				LDST_SRCDST_WORD_DECO_MATH3 |
1279 				4 << LDST_OFFSET_SHIFT);
1280 
1281 		wait_load_cmd = append_jump(desc, JUMP_JSL | JUMP_TEST_ALL |
1282 					    JUMP_COND_CALM | JUMP_COND_NCP |
1283 					    JUMP_COND_NOP | JUMP_COND_NIP |
1284 					    JUMP_COND_NIFP);
1285 		set_jump_tgt_here(desc, wait_load_cmd);
1286 
1287 		append_seq_load(desc, ivsize, LDST_CLASS_1_CCB |
1288 				LDST_SRCDST_BYTE_CONTEXT |
1289 				ctx1_iv_off << LDST_OFFSET_SHIFT);
1290 	}
1291 
1292 	/*
1293 	 * MAGIC with NFIFO
1294 	 * Read associated data from the input and send them to class1 and
1295 	 * class2 alignment blocks. From class1 send data to output fifo and
1296 	 * then write it to memory since we don't need to encrypt AD.
1297 	 */
1298 	nfifo = NFIFOENTRY_DEST_BOTH | NFIFOENTRY_FC1 | NFIFOENTRY_FC2 |
1299 		NFIFOENTRY_DTYPE_POLY | NFIFOENTRY_BND;
1300 	append_load_imm_u32(desc, nfifo, LDST_CLASS_IND_CCB |
1301 			    LDST_SRCDST_WORD_INFO_FIFO_SM | LDLEN_MATH3);
1302 
1303 	append_math_add(desc, VARSEQINLEN, ZERO, REG3, CAAM_CMD_SZ);
1304 	append_math_add(desc, VARSEQOUTLEN, ZERO, REG3, CAAM_CMD_SZ);
1305 	append_seq_fifo_load(desc, 0, FIFOLD_TYPE_NOINFOFIFO |
1306 			     FIFOLD_CLASS_CLASS1 | LDST_VLF);
1307 	append_move_len(desc, MOVE_AUX_LS | MOVE_SRC_AUX_ABLK |
1308 			MOVE_DEST_OUTFIFO | MOVELEN_MRSEL_MATH3);
1309 	append_seq_fifo_store(desc, 0, FIFOST_TYPE_MESSAGE_DATA | LDST_VLF);
1310 
1311 	/* IPsec - copy IV at the output */
1312 	if (is_ipsec)
1313 		append_seq_fifo_store(desc, ivsize, FIFOST_TYPE_METADATA |
1314 				      0x2 << 25);
1315 
1316 	wait_cmd = append_jump(desc, JUMP_JSL | JUMP_TYPE_LOCAL |
1317 			       JUMP_COND_NOP | JUMP_TEST_ALL);
1318 	set_jump_tgt_here(desc, wait_cmd);
1319 
1320 	if (encap) {
1321 		/* Read and write cryptlen bytes */
1322 		append_math_add(desc, VARSEQINLEN, SEQINLEN, REG0, CAAM_CMD_SZ);
1323 		append_math_add(desc, VARSEQOUTLEN, SEQINLEN, REG0,
1324 				CAAM_CMD_SZ);
1325 		aead_append_src_dst(desc, FIFOLD_TYPE_MSG1OUT2);
1326 
1327 		/* Write ICV */
1328 		append_seq_store(desc, icvsize, LDST_CLASS_2_CCB |
1329 				 LDST_SRCDST_BYTE_CONTEXT);
1330 	} else {
1331 		/* Read and write cryptlen bytes */
1332 		append_math_add(desc, VARSEQINLEN, SEQOUTLEN, REG0,
1333 				CAAM_CMD_SZ);
1334 		append_math_add(desc, VARSEQOUTLEN, SEQOUTLEN, REG0,
1335 				CAAM_CMD_SZ);
1336 		aead_append_src_dst(desc, FIFOLD_TYPE_MSG);
1337 
1338 		/* Load ICV for verification */
1339 		append_seq_fifo_load(desc, icvsize, FIFOLD_CLASS_CLASS2 |
1340 				     FIFOLD_TYPE_LAST2 | FIFOLD_TYPE_ICV);
1341 	}
1342 
1343 	print_hex_dump_debug("chachapoly shdesc@" __stringify(__LINE__)": ",
1344 			     DUMP_PREFIX_ADDRESS, 16, 4, desc, desc_bytes(desc),
1345 			     1);
1346 }
1347 EXPORT_SYMBOL(cnstr_shdsc_chachapoly);
1348 
1349 /* For skcipher encrypt and decrypt, read from req->src and write to req->dst */
1350 static inline void skcipher_append_src_dst(u32 *desc)
1351 {
1352 	append_math_add(desc, VARSEQOUTLEN, SEQINLEN, REG0, CAAM_CMD_SZ);
1353 	append_math_add(desc, VARSEQINLEN, SEQINLEN, REG0, CAAM_CMD_SZ);
1354 	append_seq_fifo_load(desc, 0, FIFOLD_CLASS_CLASS1 |
1355 			     KEY_VLF | FIFOLD_TYPE_MSG | FIFOLD_TYPE_LAST1);
1356 	append_seq_fifo_store(desc, 0, FIFOST_TYPE_MESSAGE_DATA | KEY_VLF);
1357 }
1358 
1359 /**
1360  * cnstr_shdsc_skcipher_encap - skcipher encapsulation shared descriptor
1361  * @desc: pointer to buffer used for descriptor construction
1362  * @cdata: pointer to block cipher transform definitions
1363  *         Valid algorithm values - one of OP_ALG_ALGSEL_{AES, DES, 3DES} ANDed
1364  *         with OP_ALG_AAI_CBC or OP_ALG_AAI_CTR_MOD128
1365  *                                - OP_ALG_ALGSEL_CHACHA20
1366  * @ivsize: initialization vector size
1367  * @is_rfc3686: true when ctr(aes) is wrapped by rfc3686 template
1368  * @ctx1_iv_off: IV offset in CONTEXT1 register
1369  */
1370 void cnstr_shdsc_skcipher_encap(u32 * const desc, struct alginfo *cdata,
1371 				unsigned int ivsize, const bool is_rfc3686,
1372 				const u32 ctx1_iv_off)
1373 {
1374 	u32 *key_jump_cmd;
1375 
1376 	init_sh_desc(desc, HDR_SHARE_SERIAL | HDR_SAVECTX);
1377 	/* Skip if already shared */
1378 	key_jump_cmd = append_jump(desc, JUMP_JSL | JUMP_TEST_ALL |
1379 				   JUMP_COND_SHRD);
1380 
1381 	/* Load class1 key only */
1382 	append_key_as_imm(desc, cdata->key_virt, cdata->keylen,
1383 			  cdata->keylen, CLASS_1 | KEY_DEST_CLASS_REG);
1384 
1385 	/* Load nonce into CONTEXT1 reg */
1386 	if (is_rfc3686) {
1387 		const u8 *nonce = cdata->key_virt + cdata->keylen;
1388 
1389 		append_load_as_imm(desc, nonce, CTR_RFC3686_NONCE_SIZE,
1390 				   LDST_CLASS_IND_CCB |
1391 				   LDST_SRCDST_BYTE_OUTFIFO | LDST_IMM);
1392 		append_move(desc, MOVE_WAITCOMP | MOVE_SRC_OUTFIFO |
1393 			    MOVE_DEST_CLASS1CTX | (16 << MOVE_OFFSET_SHIFT) |
1394 			    (CTR_RFC3686_NONCE_SIZE << MOVE_LEN_SHIFT));
1395 	}
1396 
1397 	set_jump_tgt_here(desc, key_jump_cmd);
1398 
1399 	/* Load IV, if there is one */
1400 	if (ivsize)
1401 		append_seq_load(desc, ivsize, LDST_SRCDST_BYTE_CONTEXT |
1402 				LDST_CLASS_1_CCB | (ctx1_iv_off <<
1403 				LDST_OFFSET_SHIFT));
1404 
1405 	/* Load counter into CONTEXT1 reg */
1406 	if (is_rfc3686)
1407 		append_load_imm_be32(desc, 1, LDST_IMM | LDST_CLASS_1_CCB |
1408 				     LDST_SRCDST_BYTE_CONTEXT |
1409 				     ((ctx1_iv_off + CTR_RFC3686_IV_SIZE) <<
1410 				      LDST_OFFSET_SHIFT));
1411 
1412 	/* Load operation */
1413 	append_operation(desc, cdata->algtype | OP_ALG_AS_INITFINAL |
1414 			 OP_ALG_ENCRYPT);
1415 
1416 	/* Perform operation */
1417 	skcipher_append_src_dst(desc);
1418 
1419 #ifdef DEBUG
1420 	print_hex_dump(KERN_ERR,
1421 		       "skcipher enc shdesc@" __stringify(__LINE__)": ",
1422 		       DUMP_PREFIX_ADDRESS, 16, 4, desc, desc_bytes(desc), 1);
1423 #endif
1424 }
1425 EXPORT_SYMBOL(cnstr_shdsc_skcipher_encap);
1426 
1427 /**
1428  * cnstr_shdsc_skcipher_decap - skcipher decapsulation shared descriptor
1429  * @desc: pointer to buffer used for descriptor construction
1430  * @cdata: pointer to block cipher transform definitions
1431  *         Valid algorithm values - one of OP_ALG_ALGSEL_{AES, DES, 3DES} ANDed
1432  *         with OP_ALG_AAI_CBC or OP_ALG_AAI_CTR_MOD128
1433  *                                - OP_ALG_ALGSEL_CHACHA20
1434  * @ivsize: initialization vector size
1435  * @is_rfc3686: true when ctr(aes) is wrapped by rfc3686 template
1436  * @ctx1_iv_off: IV offset in CONTEXT1 register
1437  */
1438 void cnstr_shdsc_skcipher_decap(u32 * const desc, struct alginfo *cdata,
1439 				unsigned int ivsize, const bool is_rfc3686,
1440 				const u32 ctx1_iv_off)
1441 {
1442 	u32 *key_jump_cmd;
1443 
1444 	init_sh_desc(desc, HDR_SHARE_SERIAL | HDR_SAVECTX);
1445 	/* Skip if already shared */
1446 	key_jump_cmd = append_jump(desc, JUMP_JSL | JUMP_TEST_ALL |
1447 				   JUMP_COND_SHRD);
1448 
1449 	/* Load class1 key only */
1450 	append_key_as_imm(desc, cdata->key_virt, cdata->keylen,
1451 			  cdata->keylen, CLASS_1 | KEY_DEST_CLASS_REG);
1452 
1453 	/* Load nonce into CONTEXT1 reg */
1454 	if (is_rfc3686) {
1455 		const u8 *nonce = cdata->key_virt + cdata->keylen;
1456 
1457 		append_load_as_imm(desc, nonce, CTR_RFC3686_NONCE_SIZE,
1458 				   LDST_CLASS_IND_CCB |
1459 				   LDST_SRCDST_BYTE_OUTFIFO | LDST_IMM);
1460 		append_move(desc, MOVE_WAITCOMP | MOVE_SRC_OUTFIFO |
1461 			    MOVE_DEST_CLASS1CTX | (16 << MOVE_OFFSET_SHIFT) |
1462 			    (CTR_RFC3686_NONCE_SIZE << MOVE_LEN_SHIFT));
1463 	}
1464 
1465 	set_jump_tgt_here(desc, key_jump_cmd);
1466 
1467 	/* Load IV, if there is one */
1468 	if (ivsize)
1469 		append_seq_load(desc, ivsize, LDST_SRCDST_BYTE_CONTEXT |
1470 				LDST_CLASS_1_CCB | (ctx1_iv_off <<
1471 				LDST_OFFSET_SHIFT));
1472 
1473 	/* Load counter into CONTEXT1 reg */
1474 	if (is_rfc3686)
1475 		append_load_imm_be32(desc, 1, LDST_IMM | LDST_CLASS_1_CCB |
1476 				     LDST_SRCDST_BYTE_CONTEXT |
1477 				     ((ctx1_iv_off + CTR_RFC3686_IV_SIZE) <<
1478 				      LDST_OFFSET_SHIFT));
1479 
1480 	/* Choose operation */
1481 	if (ctx1_iv_off)
1482 		append_operation(desc, cdata->algtype | OP_ALG_AS_INITFINAL |
1483 				 OP_ALG_DECRYPT);
1484 	else
1485 		append_dec_op1(desc, cdata->algtype);
1486 
1487 	/* Perform operation */
1488 	skcipher_append_src_dst(desc);
1489 
1490 #ifdef DEBUG
1491 	print_hex_dump(KERN_ERR,
1492 		       "skcipher dec shdesc@" __stringify(__LINE__)": ",
1493 		       DUMP_PREFIX_ADDRESS, 16, 4, desc, desc_bytes(desc), 1);
1494 #endif
1495 }
1496 EXPORT_SYMBOL(cnstr_shdsc_skcipher_decap);
1497 
1498 /**
1499  * cnstr_shdsc_xts_skcipher_encap - xts skcipher encapsulation shared descriptor
1500  * @desc: pointer to buffer used for descriptor construction
1501  * @cdata: pointer to block cipher transform definitions
1502  *         Valid algorithm values - OP_ALG_ALGSEL_AES ANDed with OP_ALG_AAI_XTS.
1503  */
1504 void cnstr_shdsc_xts_skcipher_encap(u32 * const desc, struct alginfo *cdata)
1505 {
1506 	__be64 sector_size = cpu_to_be64(512);
1507 	u32 *key_jump_cmd;
1508 
1509 	init_sh_desc(desc, HDR_SHARE_SERIAL | HDR_SAVECTX);
1510 	/* Skip if already shared */
1511 	key_jump_cmd = append_jump(desc, JUMP_JSL | JUMP_TEST_ALL |
1512 				   JUMP_COND_SHRD);
1513 
1514 	/* Load class1 keys only */
1515 	append_key_as_imm(desc, cdata->key_virt, cdata->keylen,
1516 			  cdata->keylen, CLASS_1 | KEY_DEST_CLASS_REG);
1517 
1518 	/* Load sector size with index 40 bytes (0x28) */
1519 	append_load_as_imm(desc, (void *)&sector_size, 8, LDST_CLASS_1_CCB |
1520 			   LDST_SRCDST_BYTE_CONTEXT |
1521 			   (0x28 << LDST_OFFSET_SHIFT));
1522 
1523 	set_jump_tgt_here(desc, key_jump_cmd);
1524 
1525 	/*
1526 	 * create sequence for loading the sector index
1527 	 * Upper 8B of IV - will be used as sector index
1528 	 * Lower 8B of IV - will be discarded
1529 	 */
1530 	append_seq_load(desc, 8, LDST_SRCDST_BYTE_CONTEXT | LDST_CLASS_1_CCB |
1531 			(0x20 << LDST_OFFSET_SHIFT));
1532 	append_seq_fifo_load(desc, 8, FIFOLD_CLASS_SKIP);
1533 
1534 	/* Load operation */
1535 	append_operation(desc, cdata->algtype | OP_ALG_AS_INITFINAL |
1536 			 OP_ALG_ENCRYPT);
1537 
1538 	/* Perform operation */
1539 	skcipher_append_src_dst(desc);
1540 
1541 #ifdef DEBUG
1542 	print_hex_dump(KERN_ERR,
1543 		       "xts skcipher enc shdesc@" __stringify(__LINE__) ": ",
1544 		       DUMP_PREFIX_ADDRESS, 16, 4, desc, desc_bytes(desc), 1);
1545 #endif
1546 }
1547 EXPORT_SYMBOL(cnstr_shdsc_xts_skcipher_encap);
1548 
1549 /**
1550  * cnstr_shdsc_xts_skcipher_decap - xts skcipher decapsulation shared descriptor
1551  * @desc: pointer to buffer used for descriptor construction
1552  * @cdata: pointer to block cipher transform definitions
1553  *         Valid algorithm values - OP_ALG_ALGSEL_AES ANDed with OP_ALG_AAI_XTS.
1554  */
1555 void cnstr_shdsc_xts_skcipher_decap(u32 * const desc, struct alginfo *cdata)
1556 {
1557 	__be64 sector_size = cpu_to_be64(512);
1558 	u32 *key_jump_cmd;
1559 
1560 	init_sh_desc(desc, HDR_SHARE_SERIAL | HDR_SAVECTX);
1561 	/* Skip if already shared */
1562 	key_jump_cmd = append_jump(desc, JUMP_JSL | JUMP_TEST_ALL |
1563 				   JUMP_COND_SHRD);
1564 
1565 	/* Load class1 key only */
1566 	append_key_as_imm(desc, cdata->key_virt, cdata->keylen,
1567 			  cdata->keylen, CLASS_1 | KEY_DEST_CLASS_REG);
1568 
1569 	/* Load sector size with index 40 bytes (0x28) */
1570 	append_load_as_imm(desc, (void *)&sector_size, 8, LDST_CLASS_1_CCB |
1571 			   LDST_SRCDST_BYTE_CONTEXT |
1572 			   (0x28 << LDST_OFFSET_SHIFT));
1573 
1574 	set_jump_tgt_here(desc, key_jump_cmd);
1575 
1576 	/*
1577 	 * create sequence for loading the sector index
1578 	 * Upper 8B of IV - will be used as sector index
1579 	 * Lower 8B of IV - will be discarded
1580 	 */
1581 	append_seq_load(desc, 8, LDST_SRCDST_BYTE_CONTEXT | LDST_CLASS_1_CCB |
1582 			(0x20 << LDST_OFFSET_SHIFT));
1583 	append_seq_fifo_load(desc, 8, FIFOLD_CLASS_SKIP);
1584 
1585 	/* Load operation */
1586 	append_dec_op1(desc, cdata->algtype);
1587 
1588 	/* Perform operation */
1589 	skcipher_append_src_dst(desc);
1590 
1591 #ifdef DEBUG
1592 	print_hex_dump(KERN_ERR,
1593 		       "xts skcipher dec shdesc@" __stringify(__LINE__) ": ",
1594 		       DUMP_PREFIX_ADDRESS, 16, 4, desc, desc_bytes(desc), 1);
1595 #endif
1596 }
1597 EXPORT_SYMBOL(cnstr_shdsc_xts_skcipher_decap);
1598 
1599 MODULE_LICENSE("GPL");
1600 MODULE_DESCRIPTION("FSL CAAM descriptor support");
1601 MODULE_AUTHOR("Freescale Semiconductor - NMG/STC");
1602