1 /* 2 * Shared descriptors for aead, ablkcipher algorithms 3 * 4 * Copyright 2016 NXP 5 */ 6 7 #include "compat.h" 8 #include "desc_constr.h" 9 #include "caamalg_desc.h" 10 11 /* 12 * For aead functions, read payload and write payload, 13 * both of which are specified in req->src and req->dst 14 */ 15 static inline void aead_append_src_dst(u32 *desc, u32 msg_type) 16 { 17 append_seq_fifo_store(desc, 0, FIFOST_TYPE_MESSAGE_DATA | KEY_VLF); 18 append_seq_fifo_load(desc, 0, FIFOLD_CLASS_BOTH | 19 KEY_VLF | msg_type | FIFOLD_TYPE_LASTBOTH); 20 } 21 22 /* Set DK bit in class 1 operation if shared */ 23 static inline void append_dec_op1(u32 *desc, u32 type) 24 { 25 u32 *jump_cmd, *uncond_jump_cmd; 26 27 /* DK bit is valid only for AES */ 28 if ((type & OP_ALG_ALGSEL_MASK) != OP_ALG_ALGSEL_AES) { 29 append_operation(desc, type | OP_ALG_AS_INITFINAL | 30 OP_ALG_DECRYPT); 31 return; 32 } 33 34 jump_cmd = append_jump(desc, JUMP_TEST_ALL | JUMP_COND_SHRD); 35 append_operation(desc, type | OP_ALG_AS_INITFINAL | 36 OP_ALG_DECRYPT); 37 uncond_jump_cmd = append_jump(desc, JUMP_TEST_ALL); 38 set_jump_tgt_here(desc, jump_cmd); 39 append_operation(desc, type | OP_ALG_AS_INITFINAL | 40 OP_ALG_DECRYPT | OP_ALG_AAI_DK); 41 set_jump_tgt_here(desc, uncond_jump_cmd); 42 } 43 44 /** 45 * cnstr_shdsc_aead_null_encap - IPSec ESP encapsulation shared descriptor 46 * (non-protocol) with no (null) encryption. 47 * @desc: pointer to buffer used for descriptor construction 48 * @adata: pointer to authentication transform definitions. Note that since a 49 * split key is to be used, the size of the split key itself is 50 * specified. Valid algorithm values - one of OP_ALG_ALGSEL_{MD5, SHA1, 51 * SHA224, SHA256, SHA384, SHA512} ANDed with OP_ALG_AAI_HMAC_PRECOMP. 52 * @icvsize: integrity check value (ICV) size (truncated or full) 53 * 54 * Note: Requires an MDHA split key. 55 */ 56 void cnstr_shdsc_aead_null_encap(u32 * const desc, struct alginfo *adata, 57 unsigned int icvsize) 58 { 59 u32 *key_jump_cmd, *read_move_cmd, *write_move_cmd; 60 61 init_sh_desc(desc, HDR_SHARE_SERIAL); 62 63 /* Skip if already shared */ 64 key_jump_cmd = append_jump(desc, JUMP_JSL | JUMP_TEST_ALL | 65 JUMP_COND_SHRD); 66 if (adata->key_inline) 67 append_key_as_imm(desc, adata->key_virt, adata->keylen_pad, 68 adata->keylen, CLASS_2 | KEY_DEST_MDHA_SPLIT | 69 KEY_ENC); 70 else 71 append_key(desc, adata->key_dma, adata->keylen, CLASS_2 | 72 KEY_DEST_MDHA_SPLIT | KEY_ENC); 73 set_jump_tgt_here(desc, key_jump_cmd); 74 75 /* assoclen + cryptlen = seqinlen */ 76 append_math_sub(desc, REG3, SEQINLEN, REG0, CAAM_CMD_SZ); 77 78 /* Prepare to read and write cryptlen + assoclen bytes */ 79 append_math_add(desc, VARSEQINLEN, ZERO, REG3, CAAM_CMD_SZ); 80 append_math_add(desc, VARSEQOUTLEN, ZERO, REG3, CAAM_CMD_SZ); 81 82 /* 83 * MOVE_LEN opcode is not available in all SEC HW revisions, 84 * thus need to do some magic, i.e. self-patch the descriptor 85 * buffer. 86 */ 87 read_move_cmd = append_move(desc, MOVE_SRC_DESCBUF | 88 MOVE_DEST_MATH3 | 89 (0x6 << MOVE_LEN_SHIFT)); 90 write_move_cmd = append_move(desc, MOVE_SRC_MATH3 | 91 MOVE_DEST_DESCBUF | 92 MOVE_WAITCOMP | 93 (0x8 << MOVE_LEN_SHIFT)); 94 95 /* Class 2 operation */ 96 append_operation(desc, adata->algtype | OP_ALG_AS_INITFINAL | 97 OP_ALG_ENCRYPT); 98 99 /* Read and write cryptlen bytes */ 100 aead_append_src_dst(desc, FIFOLD_TYPE_MSG | FIFOLD_TYPE_FLUSH1); 101 102 set_move_tgt_here(desc, read_move_cmd); 103 set_move_tgt_here(desc, write_move_cmd); 104 append_cmd(desc, CMD_LOAD | DISABLE_AUTO_INFO_FIFO); 105 append_move(desc, MOVE_SRC_INFIFO_CL | MOVE_DEST_OUTFIFO | 106 MOVE_AUX_LS); 107 108 /* Write ICV */ 109 append_seq_store(desc, icvsize, LDST_CLASS_2_CCB | 110 LDST_SRCDST_BYTE_CONTEXT); 111 112 #ifdef DEBUG 113 print_hex_dump(KERN_ERR, 114 "aead null enc shdesc@" __stringify(__LINE__)": ", 115 DUMP_PREFIX_ADDRESS, 16, 4, desc, desc_bytes(desc), 1); 116 #endif 117 } 118 EXPORT_SYMBOL(cnstr_shdsc_aead_null_encap); 119 120 /** 121 * cnstr_shdsc_aead_null_decap - IPSec ESP decapsulation shared descriptor 122 * (non-protocol) with no (null) decryption. 123 * @desc: pointer to buffer used for descriptor construction 124 * @adata: pointer to authentication transform definitions. Note that since a 125 * split key is to be used, the size of the split key itself is 126 * specified. Valid algorithm values - one of OP_ALG_ALGSEL_{MD5, SHA1, 127 * SHA224, SHA256, SHA384, SHA512} ANDed with OP_ALG_AAI_HMAC_PRECOMP. 128 * @icvsize: integrity check value (ICV) size (truncated or full) 129 * 130 * Note: Requires an MDHA split key. 131 */ 132 void cnstr_shdsc_aead_null_decap(u32 * const desc, struct alginfo *adata, 133 unsigned int icvsize) 134 { 135 u32 *key_jump_cmd, *read_move_cmd, *write_move_cmd, *jump_cmd; 136 137 init_sh_desc(desc, HDR_SHARE_SERIAL); 138 139 /* Skip if already shared */ 140 key_jump_cmd = append_jump(desc, JUMP_JSL | JUMP_TEST_ALL | 141 JUMP_COND_SHRD); 142 if (adata->key_inline) 143 append_key_as_imm(desc, adata->key_virt, adata->keylen_pad, 144 adata->keylen, CLASS_2 | 145 KEY_DEST_MDHA_SPLIT | KEY_ENC); 146 else 147 append_key(desc, adata->key_dma, adata->keylen, CLASS_2 | 148 KEY_DEST_MDHA_SPLIT | KEY_ENC); 149 set_jump_tgt_here(desc, key_jump_cmd); 150 151 /* Class 2 operation */ 152 append_operation(desc, adata->algtype | OP_ALG_AS_INITFINAL | 153 OP_ALG_DECRYPT | OP_ALG_ICV_ON); 154 155 /* assoclen + cryptlen = seqoutlen */ 156 append_math_sub(desc, REG2, SEQOUTLEN, REG0, CAAM_CMD_SZ); 157 158 /* Prepare to read and write cryptlen + assoclen bytes */ 159 append_math_add(desc, VARSEQINLEN, ZERO, REG2, CAAM_CMD_SZ); 160 append_math_add(desc, VARSEQOUTLEN, ZERO, REG2, CAAM_CMD_SZ); 161 162 /* 163 * MOVE_LEN opcode is not available in all SEC HW revisions, 164 * thus need to do some magic, i.e. self-patch the descriptor 165 * buffer. 166 */ 167 read_move_cmd = append_move(desc, MOVE_SRC_DESCBUF | 168 MOVE_DEST_MATH2 | 169 (0x6 << MOVE_LEN_SHIFT)); 170 write_move_cmd = append_move(desc, MOVE_SRC_MATH2 | 171 MOVE_DEST_DESCBUF | 172 MOVE_WAITCOMP | 173 (0x8 << MOVE_LEN_SHIFT)); 174 175 /* Read and write cryptlen bytes */ 176 aead_append_src_dst(desc, FIFOLD_TYPE_MSG | FIFOLD_TYPE_FLUSH1); 177 178 /* 179 * Insert a NOP here, since we need at least 4 instructions between 180 * code patching the descriptor buffer and the location being patched. 181 */ 182 jump_cmd = append_jump(desc, JUMP_TEST_ALL); 183 set_jump_tgt_here(desc, jump_cmd); 184 185 set_move_tgt_here(desc, read_move_cmd); 186 set_move_tgt_here(desc, write_move_cmd); 187 append_cmd(desc, CMD_LOAD | DISABLE_AUTO_INFO_FIFO); 188 append_move(desc, MOVE_SRC_INFIFO_CL | MOVE_DEST_OUTFIFO | 189 MOVE_AUX_LS); 190 append_cmd(desc, CMD_LOAD | ENABLE_AUTO_INFO_FIFO); 191 192 /* Load ICV */ 193 append_seq_fifo_load(desc, icvsize, FIFOLD_CLASS_CLASS2 | 194 FIFOLD_TYPE_LAST2 | FIFOLD_TYPE_ICV); 195 196 #ifdef DEBUG 197 print_hex_dump(KERN_ERR, 198 "aead null dec shdesc@" __stringify(__LINE__)": ", 199 DUMP_PREFIX_ADDRESS, 16, 4, desc, desc_bytes(desc), 1); 200 #endif 201 } 202 EXPORT_SYMBOL(cnstr_shdsc_aead_null_decap); 203 204 static void init_sh_desc_key_aead(u32 * const desc, 205 struct alginfo * const cdata, 206 struct alginfo * const adata, 207 const bool is_rfc3686, u32 *nonce) 208 { 209 u32 *key_jump_cmd; 210 unsigned int enckeylen = cdata->keylen; 211 212 /* Note: Context registers are saved. */ 213 init_sh_desc(desc, HDR_SHARE_SERIAL | HDR_SAVECTX); 214 215 /* Skip if already shared */ 216 key_jump_cmd = append_jump(desc, JUMP_JSL | JUMP_TEST_ALL | 217 JUMP_COND_SHRD); 218 219 /* 220 * RFC3686 specific: 221 * | key = {AUTH_KEY, ENC_KEY, NONCE} 222 * | enckeylen = encryption key size + nonce size 223 */ 224 if (is_rfc3686) 225 enckeylen -= CTR_RFC3686_NONCE_SIZE; 226 227 if (adata->key_inline) 228 append_key_as_imm(desc, adata->key_virt, adata->keylen_pad, 229 adata->keylen, CLASS_2 | 230 KEY_DEST_MDHA_SPLIT | KEY_ENC); 231 else 232 append_key(desc, adata->key_dma, adata->keylen, CLASS_2 | 233 KEY_DEST_MDHA_SPLIT | KEY_ENC); 234 235 if (cdata->key_inline) 236 append_key_as_imm(desc, cdata->key_virt, enckeylen, 237 enckeylen, CLASS_1 | KEY_DEST_CLASS_REG); 238 else 239 append_key(desc, cdata->key_dma, enckeylen, CLASS_1 | 240 KEY_DEST_CLASS_REG); 241 242 /* Load Counter into CONTEXT1 reg */ 243 if (is_rfc3686) { 244 append_load_as_imm(desc, nonce, CTR_RFC3686_NONCE_SIZE, 245 LDST_CLASS_IND_CCB | 246 LDST_SRCDST_BYTE_OUTFIFO | LDST_IMM); 247 append_move(desc, 248 MOVE_SRC_OUTFIFO | 249 MOVE_DEST_CLASS1CTX | 250 (16 << MOVE_OFFSET_SHIFT) | 251 (CTR_RFC3686_NONCE_SIZE << MOVE_LEN_SHIFT)); 252 } 253 254 set_jump_tgt_here(desc, key_jump_cmd); 255 } 256 257 /** 258 * cnstr_shdsc_aead_encap - IPSec ESP encapsulation shared descriptor 259 * (non-protocol). 260 * @desc: pointer to buffer used for descriptor construction 261 * @cdata: pointer to block cipher transform definitions 262 * Valid algorithm values - one of OP_ALG_ALGSEL_{AES, DES, 3DES} ANDed 263 * with OP_ALG_AAI_CBC or OP_ALG_AAI_CTR_MOD128. 264 * @adata: pointer to authentication transform definitions. Note that since a 265 * split key is to be used, the size of the split key itself is 266 * specified. Valid algorithm values - one of OP_ALG_ALGSEL_{MD5, SHA1, 267 * SHA224, SHA256, SHA384, SHA512} ANDed with OP_ALG_AAI_HMAC_PRECOMP. 268 * @ivsize: initialization vector size 269 * @icvsize: integrity check value (ICV) size (truncated or full) 270 * @is_rfc3686: true when ctr(aes) is wrapped by rfc3686 template 271 * @nonce: pointer to rfc3686 nonce 272 * @ctx1_iv_off: IV offset in CONTEXT1 register 273 * @is_qi: true when called from caam/qi 274 * 275 * Note: Requires an MDHA split key. 276 */ 277 void cnstr_shdsc_aead_encap(u32 * const desc, struct alginfo *cdata, 278 struct alginfo *adata, unsigned int ivsize, 279 unsigned int icvsize, const bool is_rfc3686, 280 u32 *nonce, const u32 ctx1_iv_off, const bool is_qi) 281 { 282 /* Note: Context registers are saved. */ 283 init_sh_desc_key_aead(desc, cdata, adata, is_rfc3686, nonce); 284 285 /* Class 2 operation */ 286 append_operation(desc, adata->algtype | OP_ALG_AS_INITFINAL | 287 OP_ALG_ENCRYPT); 288 289 if (is_qi) { 290 u32 *wait_load_cmd; 291 292 /* REG3 = assoclen */ 293 append_seq_load(desc, 4, LDST_CLASS_DECO | 294 LDST_SRCDST_WORD_DECO_MATH3 | 295 (4 << LDST_OFFSET_SHIFT)); 296 297 wait_load_cmd = append_jump(desc, JUMP_JSL | JUMP_TEST_ALL | 298 JUMP_COND_CALM | JUMP_COND_NCP | 299 JUMP_COND_NOP | JUMP_COND_NIP | 300 JUMP_COND_NIFP); 301 set_jump_tgt_here(desc, wait_load_cmd); 302 303 append_seq_load(desc, ivsize, LDST_CLASS_1_CCB | 304 LDST_SRCDST_BYTE_CONTEXT | 305 (ctx1_iv_off << LDST_OFFSET_SHIFT)); 306 } 307 308 /* Read and write assoclen bytes */ 309 append_math_add(desc, VARSEQINLEN, ZERO, REG3, CAAM_CMD_SZ); 310 append_math_add(desc, VARSEQOUTLEN, ZERO, REG3, CAAM_CMD_SZ); 311 312 /* Skip assoc data */ 313 append_seq_fifo_store(desc, 0, FIFOST_TYPE_SKIP | FIFOLDST_VLF); 314 315 /* read assoc before reading payload */ 316 append_seq_fifo_load(desc, 0, FIFOLD_CLASS_CLASS2 | FIFOLD_TYPE_MSG | 317 FIFOLDST_VLF); 318 319 /* Load Counter into CONTEXT1 reg */ 320 if (is_rfc3686) 321 append_load_imm_be32(desc, 1, LDST_IMM | LDST_CLASS_1_CCB | 322 LDST_SRCDST_BYTE_CONTEXT | 323 ((ctx1_iv_off + CTR_RFC3686_IV_SIZE) << 324 LDST_OFFSET_SHIFT)); 325 326 /* Class 1 operation */ 327 append_operation(desc, cdata->algtype | OP_ALG_AS_INITFINAL | 328 OP_ALG_ENCRYPT); 329 330 /* Read and write cryptlen bytes */ 331 append_math_add(desc, VARSEQINLEN, SEQINLEN, REG0, CAAM_CMD_SZ); 332 append_math_add(desc, VARSEQOUTLEN, SEQINLEN, REG0, CAAM_CMD_SZ); 333 aead_append_src_dst(desc, FIFOLD_TYPE_MSG1OUT2); 334 335 /* Write ICV */ 336 append_seq_store(desc, icvsize, LDST_CLASS_2_CCB | 337 LDST_SRCDST_BYTE_CONTEXT); 338 339 #ifdef DEBUG 340 print_hex_dump(KERN_ERR, "aead enc shdesc@" __stringify(__LINE__)": ", 341 DUMP_PREFIX_ADDRESS, 16, 4, desc, desc_bytes(desc), 1); 342 #endif 343 } 344 EXPORT_SYMBOL(cnstr_shdsc_aead_encap); 345 346 /** 347 * cnstr_shdsc_aead_decap - IPSec ESP decapsulation shared descriptor 348 * (non-protocol). 349 * @desc: pointer to buffer used for descriptor construction 350 * @cdata: pointer to block cipher transform definitions 351 * Valid algorithm values - one of OP_ALG_ALGSEL_{AES, DES, 3DES} ANDed 352 * with OP_ALG_AAI_CBC or OP_ALG_AAI_CTR_MOD128. 353 * @adata: pointer to authentication transform definitions. Note that since a 354 * split key is to be used, the size of the split key itself is 355 * specified. Valid algorithm values - one of OP_ALG_ALGSEL_{MD5, SHA1, 356 * SHA224, SHA256, SHA384, SHA512} ANDed with OP_ALG_AAI_HMAC_PRECOMP. 357 * @ivsize: initialization vector size 358 * @icvsize: integrity check value (ICV) size (truncated or full) 359 * @is_rfc3686: true when ctr(aes) is wrapped by rfc3686 template 360 * @nonce: pointer to rfc3686 nonce 361 * @ctx1_iv_off: IV offset in CONTEXT1 register 362 * @is_qi: true when called from caam/qi 363 * 364 * Note: Requires an MDHA split key. 365 */ 366 void cnstr_shdsc_aead_decap(u32 * const desc, struct alginfo *cdata, 367 struct alginfo *adata, unsigned int ivsize, 368 unsigned int icvsize, const bool geniv, 369 const bool is_rfc3686, u32 *nonce, 370 const u32 ctx1_iv_off, const bool is_qi) 371 { 372 /* Note: Context registers are saved. */ 373 init_sh_desc_key_aead(desc, cdata, adata, is_rfc3686, nonce); 374 375 /* Class 2 operation */ 376 append_operation(desc, adata->algtype | OP_ALG_AS_INITFINAL | 377 OP_ALG_DECRYPT | OP_ALG_ICV_ON); 378 379 if (is_qi) { 380 u32 *wait_load_cmd; 381 382 /* REG3 = assoclen */ 383 append_seq_load(desc, 4, LDST_CLASS_DECO | 384 LDST_SRCDST_WORD_DECO_MATH3 | 385 (4 << LDST_OFFSET_SHIFT)); 386 387 wait_load_cmd = append_jump(desc, JUMP_JSL | JUMP_TEST_ALL | 388 JUMP_COND_CALM | JUMP_COND_NCP | 389 JUMP_COND_NOP | JUMP_COND_NIP | 390 JUMP_COND_NIFP); 391 set_jump_tgt_here(desc, wait_load_cmd); 392 393 if (!geniv) 394 append_seq_load(desc, ivsize, LDST_CLASS_1_CCB | 395 LDST_SRCDST_BYTE_CONTEXT | 396 (ctx1_iv_off << LDST_OFFSET_SHIFT)); 397 } 398 399 /* Read and write assoclen bytes */ 400 append_math_add(desc, VARSEQINLEN, ZERO, REG3, CAAM_CMD_SZ); 401 if (geniv) 402 append_math_add_imm_u32(desc, VARSEQOUTLEN, REG3, IMM, ivsize); 403 else 404 append_math_add(desc, VARSEQOUTLEN, ZERO, REG3, CAAM_CMD_SZ); 405 406 /* Skip assoc data */ 407 append_seq_fifo_store(desc, 0, FIFOST_TYPE_SKIP | FIFOLDST_VLF); 408 409 /* read assoc before reading payload */ 410 append_seq_fifo_load(desc, 0, FIFOLD_CLASS_CLASS2 | FIFOLD_TYPE_MSG | 411 KEY_VLF); 412 413 if (geniv) { 414 append_seq_load(desc, ivsize, LDST_CLASS_1_CCB | 415 LDST_SRCDST_BYTE_CONTEXT | 416 (ctx1_iv_off << LDST_OFFSET_SHIFT)); 417 append_move(desc, MOVE_SRC_CLASS1CTX | MOVE_DEST_CLASS2INFIFO | 418 (ctx1_iv_off << MOVE_OFFSET_SHIFT) | ivsize); 419 } 420 421 /* Load Counter into CONTEXT1 reg */ 422 if (is_rfc3686) 423 append_load_imm_be32(desc, 1, LDST_IMM | LDST_CLASS_1_CCB | 424 LDST_SRCDST_BYTE_CONTEXT | 425 ((ctx1_iv_off + CTR_RFC3686_IV_SIZE) << 426 LDST_OFFSET_SHIFT)); 427 428 /* Choose operation */ 429 if (ctx1_iv_off) 430 append_operation(desc, cdata->algtype | OP_ALG_AS_INITFINAL | 431 OP_ALG_DECRYPT); 432 else 433 append_dec_op1(desc, cdata->algtype); 434 435 /* Read and write cryptlen bytes */ 436 append_math_add(desc, VARSEQINLEN, SEQOUTLEN, REG0, CAAM_CMD_SZ); 437 append_math_add(desc, VARSEQOUTLEN, SEQOUTLEN, REG0, CAAM_CMD_SZ); 438 aead_append_src_dst(desc, FIFOLD_TYPE_MSG); 439 440 /* Load ICV */ 441 append_seq_fifo_load(desc, icvsize, FIFOLD_CLASS_CLASS2 | 442 FIFOLD_TYPE_LAST2 | FIFOLD_TYPE_ICV); 443 444 #ifdef DEBUG 445 print_hex_dump(KERN_ERR, "aead dec shdesc@" __stringify(__LINE__)": ", 446 DUMP_PREFIX_ADDRESS, 16, 4, desc, desc_bytes(desc), 1); 447 #endif 448 } 449 EXPORT_SYMBOL(cnstr_shdsc_aead_decap); 450 451 /** 452 * cnstr_shdsc_aead_givencap - IPSec ESP encapsulation shared descriptor 453 * (non-protocol) with HW-generated initialization 454 * vector. 455 * @desc: pointer to buffer used for descriptor construction 456 * @cdata: pointer to block cipher transform definitions 457 * Valid algorithm values - one of OP_ALG_ALGSEL_{AES, DES, 3DES} ANDed 458 * with OP_ALG_AAI_CBC or OP_ALG_AAI_CTR_MOD128. 459 * @adata: pointer to authentication transform definitions. Note that since a 460 * split key is to be used, the size of the split key itself is 461 * specified. Valid algorithm values - one of OP_ALG_ALGSEL_{MD5, SHA1, 462 * SHA224, SHA256, SHA384, SHA512} ANDed with OP_ALG_AAI_HMAC_PRECOMP. 463 * @ivsize: initialization vector size 464 * @icvsize: integrity check value (ICV) size (truncated or full) 465 * @is_rfc3686: true when ctr(aes) is wrapped by rfc3686 template 466 * @nonce: pointer to rfc3686 nonce 467 * @ctx1_iv_off: IV offset in CONTEXT1 register 468 * @is_qi: true when called from caam/qi 469 * 470 * Note: Requires an MDHA split key. 471 */ 472 void cnstr_shdsc_aead_givencap(u32 * const desc, struct alginfo *cdata, 473 struct alginfo *adata, unsigned int ivsize, 474 unsigned int icvsize, const bool is_rfc3686, 475 u32 *nonce, const u32 ctx1_iv_off, 476 const bool is_qi) 477 { 478 u32 geniv, moveiv; 479 480 /* Note: Context registers are saved. */ 481 init_sh_desc_key_aead(desc, cdata, adata, is_rfc3686, nonce); 482 483 if (is_qi) { 484 u32 *wait_load_cmd; 485 486 /* REG3 = assoclen */ 487 append_seq_load(desc, 4, LDST_CLASS_DECO | 488 LDST_SRCDST_WORD_DECO_MATH3 | 489 (4 << LDST_OFFSET_SHIFT)); 490 491 wait_load_cmd = append_jump(desc, JUMP_JSL | JUMP_TEST_ALL | 492 JUMP_COND_CALM | JUMP_COND_NCP | 493 JUMP_COND_NOP | JUMP_COND_NIP | 494 JUMP_COND_NIFP); 495 set_jump_tgt_here(desc, wait_load_cmd); 496 } 497 498 if (is_rfc3686) { 499 if (is_qi) 500 append_seq_load(desc, ivsize, LDST_CLASS_1_CCB | 501 LDST_SRCDST_BYTE_CONTEXT | 502 (ctx1_iv_off << LDST_OFFSET_SHIFT)); 503 504 goto copy_iv; 505 } 506 507 /* Generate IV */ 508 geniv = NFIFOENTRY_STYPE_PAD | NFIFOENTRY_DEST_DECO | 509 NFIFOENTRY_DTYPE_MSG | NFIFOENTRY_LC1 | 510 NFIFOENTRY_PTYPE_RND | (ivsize << NFIFOENTRY_DLEN_SHIFT); 511 append_load_imm_u32(desc, geniv, LDST_CLASS_IND_CCB | 512 LDST_SRCDST_WORD_INFO_FIFO | LDST_IMM); 513 append_cmd(desc, CMD_LOAD | DISABLE_AUTO_INFO_FIFO); 514 append_move(desc, MOVE_WAITCOMP | 515 MOVE_SRC_INFIFO | MOVE_DEST_CLASS1CTX | 516 (ctx1_iv_off << MOVE_OFFSET_SHIFT) | 517 (ivsize << MOVE_LEN_SHIFT)); 518 append_cmd(desc, CMD_LOAD | ENABLE_AUTO_INFO_FIFO); 519 520 copy_iv: 521 /* Copy IV to class 1 context */ 522 append_move(desc, MOVE_SRC_CLASS1CTX | MOVE_DEST_OUTFIFO | 523 (ctx1_iv_off << MOVE_OFFSET_SHIFT) | 524 (ivsize << MOVE_LEN_SHIFT)); 525 526 /* Return to encryption */ 527 append_operation(desc, adata->algtype | OP_ALG_AS_INITFINAL | 528 OP_ALG_ENCRYPT); 529 530 /* Read and write assoclen bytes */ 531 append_math_add(desc, VARSEQINLEN, ZERO, REG3, CAAM_CMD_SZ); 532 append_math_add(desc, VARSEQOUTLEN, ZERO, REG3, CAAM_CMD_SZ); 533 534 /* Skip assoc data */ 535 append_seq_fifo_store(desc, 0, FIFOST_TYPE_SKIP | FIFOLDST_VLF); 536 537 /* read assoc before reading payload */ 538 append_seq_fifo_load(desc, 0, FIFOLD_CLASS_CLASS2 | FIFOLD_TYPE_MSG | 539 KEY_VLF); 540 541 /* Copy iv from outfifo to class 2 fifo */ 542 moveiv = NFIFOENTRY_STYPE_OFIFO | NFIFOENTRY_DEST_CLASS2 | 543 NFIFOENTRY_DTYPE_MSG | (ivsize << NFIFOENTRY_DLEN_SHIFT); 544 append_load_imm_u32(desc, moveiv, LDST_CLASS_IND_CCB | 545 LDST_SRCDST_WORD_INFO_FIFO | LDST_IMM); 546 append_load_imm_u32(desc, ivsize, LDST_CLASS_2_CCB | 547 LDST_SRCDST_WORD_DATASZ_REG | LDST_IMM); 548 549 /* Load Counter into CONTEXT1 reg */ 550 if (is_rfc3686) 551 append_load_imm_be32(desc, 1, LDST_IMM | LDST_CLASS_1_CCB | 552 LDST_SRCDST_BYTE_CONTEXT | 553 ((ctx1_iv_off + CTR_RFC3686_IV_SIZE) << 554 LDST_OFFSET_SHIFT)); 555 556 /* Class 1 operation */ 557 append_operation(desc, cdata->algtype | OP_ALG_AS_INITFINAL | 558 OP_ALG_ENCRYPT); 559 560 /* Will write ivsize + cryptlen */ 561 append_math_add(desc, VARSEQOUTLEN, SEQINLEN, REG0, CAAM_CMD_SZ); 562 563 /* Not need to reload iv */ 564 append_seq_fifo_load(desc, ivsize, 565 FIFOLD_CLASS_SKIP); 566 567 /* Will read cryptlen */ 568 append_math_add(desc, VARSEQINLEN, SEQINLEN, REG0, CAAM_CMD_SZ); 569 append_seq_fifo_load(desc, 0, FIFOLD_CLASS_BOTH | KEY_VLF | 570 FIFOLD_TYPE_MSG1OUT2 | FIFOLD_TYPE_LASTBOTH); 571 append_seq_fifo_store(desc, 0, FIFOST_TYPE_MESSAGE_DATA | KEY_VLF); 572 573 /* Write ICV */ 574 append_seq_store(desc, icvsize, LDST_CLASS_2_CCB | 575 LDST_SRCDST_BYTE_CONTEXT); 576 577 #ifdef DEBUG 578 print_hex_dump(KERN_ERR, 579 "aead givenc shdesc@" __stringify(__LINE__)": ", 580 DUMP_PREFIX_ADDRESS, 16, 4, desc, desc_bytes(desc), 1); 581 #endif 582 } 583 EXPORT_SYMBOL(cnstr_shdsc_aead_givencap); 584 585 /** 586 * cnstr_shdsc_gcm_encap - gcm encapsulation shared descriptor 587 * @desc: pointer to buffer used for descriptor construction 588 * @cdata: pointer to block cipher transform definitions 589 * Valid algorithm values - OP_ALG_ALGSEL_AES ANDed with OP_ALG_AAI_GCM. 590 * @icvsize: integrity check value (ICV) size (truncated or full) 591 */ 592 void cnstr_shdsc_gcm_encap(u32 * const desc, struct alginfo *cdata, 593 unsigned int icvsize) 594 { 595 u32 *key_jump_cmd, *zero_payload_jump_cmd, *zero_assoc_jump_cmd1, 596 *zero_assoc_jump_cmd2; 597 598 init_sh_desc(desc, HDR_SHARE_SERIAL); 599 600 /* skip key loading if they are loaded due to sharing */ 601 key_jump_cmd = append_jump(desc, JUMP_JSL | JUMP_TEST_ALL | 602 JUMP_COND_SHRD | JUMP_COND_SELF); 603 if (cdata->key_inline) 604 append_key_as_imm(desc, cdata->key_virt, cdata->keylen, 605 cdata->keylen, CLASS_1 | KEY_DEST_CLASS_REG); 606 else 607 append_key(desc, cdata->key_dma, cdata->keylen, CLASS_1 | 608 KEY_DEST_CLASS_REG); 609 set_jump_tgt_here(desc, key_jump_cmd); 610 611 /* class 1 operation */ 612 append_operation(desc, cdata->algtype | OP_ALG_AS_INITFINAL | 613 OP_ALG_ENCRYPT); 614 615 /* if assoclen + cryptlen is ZERO, skip to ICV write */ 616 append_math_sub(desc, VARSEQOUTLEN, SEQINLEN, REG0, CAAM_CMD_SZ); 617 zero_assoc_jump_cmd2 = append_jump(desc, JUMP_TEST_ALL | 618 JUMP_COND_MATH_Z); 619 620 /* if assoclen is ZERO, skip reading the assoc data */ 621 append_math_add(desc, VARSEQINLEN, ZERO, REG3, CAAM_CMD_SZ); 622 zero_assoc_jump_cmd1 = append_jump(desc, JUMP_TEST_ALL | 623 JUMP_COND_MATH_Z); 624 625 append_math_add(desc, VARSEQOUTLEN, ZERO, REG3, CAAM_CMD_SZ); 626 627 /* skip assoc data */ 628 append_seq_fifo_store(desc, 0, FIFOST_TYPE_SKIP | FIFOLDST_VLF); 629 630 /* cryptlen = seqinlen - assoclen */ 631 append_math_sub(desc, VARSEQOUTLEN, SEQINLEN, REG3, CAAM_CMD_SZ); 632 633 /* if cryptlen is ZERO jump to zero-payload commands */ 634 zero_payload_jump_cmd = append_jump(desc, JUMP_TEST_ALL | 635 JUMP_COND_MATH_Z); 636 637 /* read assoc data */ 638 append_seq_fifo_load(desc, 0, FIFOLD_CLASS_CLASS1 | FIFOLDST_VLF | 639 FIFOLD_TYPE_AAD | FIFOLD_TYPE_FLUSH1); 640 set_jump_tgt_here(desc, zero_assoc_jump_cmd1); 641 642 append_math_sub(desc, VARSEQINLEN, SEQINLEN, REG0, CAAM_CMD_SZ); 643 644 /* write encrypted data */ 645 append_seq_fifo_store(desc, 0, FIFOST_TYPE_MESSAGE_DATA | FIFOLDST_VLF); 646 647 /* read payload data */ 648 append_seq_fifo_load(desc, 0, FIFOLD_CLASS_CLASS1 | FIFOLDST_VLF | 649 FIFOLD_TYPE_MSG | FIFOLD_TYPE_LAST1); 650 651 /* jump the zero-payload commands */ 652 append_jump(desc, JUMP_TEST_ALL | 2); 653 654 /* zero-payload commands */ 655 set_jump_tgt_here(desc, zero_payload_jump_cmd); 656 657 /* read assoc data */ 658 append_seq_fifo_load(desc, 0, FIFOLD_CLASS_CLASS1 | FIFOLDST_VLF | 659 FIFOLD_TYPE_AAD | FIFOLD_TYPE_LAST1); 660 661 /* There is no input data */ 662 set_jump_tgt_here(desc, zero_assoc_jump_cmd2); 663 664 /* write ICV */ 665 append_seq_store(desc, icvsize, LDST_CLASS_1_CCB | 666 LDST_SRCDST_BYTE_CONTEXT); 667 668 #ifdef DEBUG 669 print_hex_dump(KERN_ERR, "gcm enc shdesc@" __stringify(__LINE__)": ", 670 DUMP_PREFIX_ADDRESS, 16, 4, desc, desc_bytes(desc), 1); 671 #endif 672 } 673 EXPORT_SYMBOL(cnstr_shdsc_gcm_encap); 674 675 /** 676 * cnstr_shdsc_gcm_decap - gcm decapsulation shared descriptor 677 * @desc: pointer to buffer used for descriptor construction 678 * @cdata: pointer to block cipher transform definitions 679 * Valid algorithm values - OP_ALG_ALGSEL_AES ANDed with OP_ALG_AAI_GCM. 680 * @icvsize: integrity check value (ICV) size (truncated or full) 681 */ 682 void cnstr_shdsc_gcm_decap(u32 * const desc, struct alginfo *cdata, 683 unsigned int icvsize) 684 { 685 u32 *key_jump_cmd, *zero_payload_jump_cmd, *zero_assoc_jump_cmd1; 686 687 init_sh_desc(desc, HDR_SHARE_SERIAL); 688 689 /* skip key loading if they are loaded due to sharing */ 690 key_jump_cmd = append_jump(desc, JUMP_JSL | 691 JUMP_TEST_ALL | JUMP_COND_SHRD | 692 JUMP_COND_SELF); 693 if (cdata->key_inline) 694 append_key_as_imm(desc, cdata->key_virt, cdata->keylen, 695 cdata->keylen, CLASS_1 | KEY_DEST_CLASS_REG); 696 else 697 append_key(desc, cdata->key_dma, cdata->keylen, CLASS_1 | 698 KEY_DEST_CLASS_REG); 699 set_jump_tgt_here(desc, key_jump_cmd); 700 701 /* class 1 operation */ 702 append_operation(desc, cdata->algtype | OP_ALG_AS_INITFINAL | 703 OP_ALG_DECRYPT | OP_ALG_ICV_ON); 704 705 /* if assoclen is ZERO, skip reading the assoc data */ 706 append_math_add(desc, VARSEQINLEN, ZERO, REG3, CAAM_CMD_SZ); 707 zero_assoc_jump_cmd1 = append_jump(desc, JUMP_TEST_ALL | 708 JUMP_COND_MATH_Z); 709 710 append_math_add(desc, VARSEQOUTLEN, ZERO, REG3, CAAM_CMD_SZ); 711 712 /* skip assoc data */ 713 append_seq_fifo_store(desc, 0, FIFOST_TYPE_SKIP | FIFOLDST_VLF); 714 715 /* read assoc data */ 716 append_seq_fifo_load(desc, 0, FIFOLD_CLASS_CLASS1 | FIFOLDST_VLF | 717 FIFOLD_TYPE_AAD | FIFOLD_TYPE_FLUSH1); 718 719 set_jump_tgt_here(desc, zero_assoc_jump_cmd1); 720 721 /* cryptlen = seqoutlen - assoclen */ 722 append_math_sub(desc, VARSEQINLEN, SEQOUTLEN, REG0, CAAM_CMD_SZ); 723 724 /* jump to zero-payload command if cryptlen is zero */ 725 zero_payload_jump_cmd = append_jump(desc, JUMP_TEST_ALL | 726 JUMP_COND_MATH_Z); 727 728 append_math_sub(desc, VARSEQOUTLEN, SEQOUTLEN, REG0, CAAM_CMD_SZ); 729 730 /* store encrypted data */ 731 append_seq_fifo_store(desc, 0, FIFOST_TYPE_MESSAGE_DATA | FIFOLDST_VLF); 732 733 /* read payload data */ 734 append_seq_fifo_load(desc, 0, FIFOLD_CLASS_CLASS1 | FIFOLDST_VLF | 735 FIFOLD_TYPE_MSG | FIFOLD_TYPE_FLUSH1); 736 737 /* zero-payload command */ 738 set_jump_tgt_here(desc, zero_payload_jump_cmd); 739 740 /* read ICV */ 741 append_seq_fifo_load(desc, icvsize, FIFOLD_CLASS_CLASS1 | 742 FIFOLD_TYPE_ICV | FIFOLD_TYPE_LAST1); 743 744 #ifdef DEBUG 745 print_hex_dump(KERN_ERR, "gcm dec shdesc@" __stringify(__LINE__)": ", 746 DUMP_PREFIX_ADDRESS, 16, 4, desc, desc_bytes(desc), 1); 747 #endif 748 } 749 EXPORT_SYMBOL(cnstr_shdsc_gcm_decap); 750 751 /** 752 * cnstr_shdsc_rfc4106_encap - IPSec ESP gcm encapsulation shared descriptor 753 * (non-protocol). 754 * @desc: pointer to buffer used for descriptor construction 755 * @cdata: pointer to block cipher transform definitions 756 * Valid algorithm values - OP_ALG_ALGSEL_AES ANDed with OP_ALG_AAI_GCM. 757 * @icvsize: integrity check value (ICV) size (truncated or full) 758 */ 759 void cnstr_shdsc_rfc4106_encap(u32 * const desc, struct alginfo *cdata, 760 unsigned int icvsize) 761 { 762 u32 *key_jump_cmd; 763 764 init_sh_desc(desc, HDR_SHARE_SERIAL); 765 766 /* Skip key loading if it is loaded due to sharing */ 767 key_jump_cmd = append_jump(desc, JUMP_JSL | JUMP_TEST_ALL | 768 JUMP_COND_SHRD); 769 if (cdata->key_inline) 770 append_key_as_imm(desc, cdata->key_virt, cdata->keylen, 771 cdata->keylen, CLASS_1 | KEY_DEST_CLASS_REG); 772 else 773 append_key(desc, cdata->key_dma, cdata->keylen, CLASS_1 | 774 KEY_DEST_CLASS_REG); 775 set_jump_tgt_here(desc, key_jump_cmd); 776 777 /* Class 1 operation */ 778 append_operation(desc, cdata->algtype | OP_ALG_AS_INITFINAL | 779 OP_ALG_ENCRYPT); 780 781 append_math_sub_imm_u32(desc, VARSEQINLEN, REG3, IMM, 8); 782 append_math_add(desc, VARSEQOUTLEN, ZERO, REG3, CAAM_CMD_SZ); 783 784 /* Read assoc data */ 785 append_seq_fifo_load(desc, 0, FIFOLD_CLASS_CLASS1 | FIFOLDST_VLF | 786 FIFOLD_TYPE_AAD | FIFOLD_TYPE_FLUSH1); 787 788 /* Skip IV */ 789 append_seq_fifo_load(desc, 8, FIFOLD_CLASS_SKIP); 790 791 /* Will read cryptlen bytes */ 792 append_math_sub(desc, VARSEQINLEN, SEQINLEN, REG0, CAAM_CMD_SZ); 793 794 /* Workaround for erratum A-005473 (simultaneous SEQ FIFO skips) */ 795 append_seq_fifo_load(desc, 0, FIFOLD_CLASS_CLASS1 | FIFOLD_TYPE_MSG); 796 797 /* Skip assoc data */ 798 append_seq_fifo_store(desc, 0, FIFOST_TYPE_SKIP | FIFOLDST_VLF); 799 800 /* cryptlen = seqoutlen - assoclen */ 801 append_math_sub(desc, VARSEQOUTLEN, VARSEQINLEN, REG0, CAAM_CMD_SZ); 802 803 /* Write encrypted data */ 804 append_seq_fifo_store(desc, 0, FIFOST_TYPE_MESSAGE_DATA | FIFOLDST_VLF); 805 806 /* Read payload data */ 807 append_seq_fifo_load(desc, 0, FIFOLD_CLASS_CLASS1 | FIFOLDST_VLF | 808 FIFOLD_TYPE_MSG | FIFOLD_TYPE_LAST1); 809 810 /* Write ICV */ 811 append_seq_store(desc, icvsize, LDST_CLASS_1_CCB | 812 LDST_SRCDST_BYTE_CONTEXT); 813 814 #ifdef DEBUG 815 print_hex_dump(KERN_ERR, 816 "rfc4106 enc shdesc@" __stringify(__LINE__)": ", 817 DUMP_PREFIX_ADDRESS, 16, 4, desc, desc_bytes(desc), 1); 818 #endif 819 } 820 EXPORT_SYMBOL(cnstr_shdsc_rfc4106_encap); 821 822 /** 823 * cnstr_shdsc_rfc4106_decap - IPSec ESP gcm decapsulation shared descriptor 824 * (non-protocol). 825 * @desc: pointer to buffer used for descriptor construction 826 * @cdata: pointer to block cipher transform definitions 827 * Valid algorithm values - OP_ALG_ALGSEL_AES ANDed with OP_ALG_AAI_GCM. 828 * @icvsize: integrity check value (ICV) size (truncated or full) 829 */ 830 void cnstr_shdsc_rfc4106_decap(u32 * const desc, struct alginfo *cdata, 831 unsigned int icvsize) 832 { 833 u32 *key_jump_cmd; 834 835 init_sh_desc(desc, HDR_SHARE_SERIAL); 836 837 /* Skip key loading if it is loaded due to sharing */ 838 key_jump_cmd = append_jump(desc, JUMP_JSL | JUMP_TEST_ALL | 839 JUMP_COND_SHRD); 840 if (cdata->key_inline) 841 append_key_as_imm(desc, cdata->key_virt, cdata->keylen, 842 cdata->keylen, CLASS_1 | 843 KEY_DEST_CLASS_REG); 844 else 845 append_key(desc, cdata->key_dma, cdata->keylen, CLASS_1 | 846 KEY_DEST_CLASS_REG); 847 set_jump_tgt_here(desc, key_jump_cmd); 848 849 /* Class 1 operation */ 850 append_operation(desc, cdata->algtype | OP_ALG_AS_INITFINAL | 851 OP_ALG_DECRYPT | OP_ALG_ICV_ON); 852 853 append_math_sub_imm_u32(desc, VARSEQINLEN, REG3, IMM, 8); 854 append_math_add(desc, VARSEQOUTLEN, ZERO, REG3, CAAM_CMD_SZ); 855 856 /* Read assoc data */ 857 append_seq_fifo_load(desc, 0, FIFOLD_CLASS_CLASS1 | FIFOLDST_VLF | 858 FIFOLD_TYPE_AAD | FIFOLD_TYPE_FLUSH1); 859 860 /* Skip IV */ 861 append_seq_fifo_load(desc, 8, FIFOLD_CLASS_SKIP); 862 863 /* Will read cryptlen bytes */ 864 append_math_sub(desc, VARSEQINLEN, SEQOUTLEN, REG3, CAAM_CMD_SZ); 865 866 /* Workaround for erratum A-005473 (simultaneous SEQ FIFO skips) */ 867 append_seq_fifo_load(desc, 0, FIFOLD_CLASS_CLASS1 | FIFOLD_TYPE_MSG); 868 869 /* Skip assoc data */ 870 append_seq_fifo_store(desc, 0, FIFOST_TYPE_SKIP | FIFOLDST_VLF); 871 872 /* Will write cryptlen bytes */ 873 append_math_sub(desc, VARSEQOUTLEN, SEQOUTLEN, REG0, CAAM_CMD_SZ); 874 875 /* Store payload data */ 876 append_seq_fifo_store(desc, 0, FIFOST_TYPE_MESSAGE_DATA | FIFOLDST_VLF); 877 878 /* Read encrypted data */ 879 append_seq_fifo_load(desc, 0, FIFOLD_CLASS_CLASS1 | FIFOLDST_VLF | 880 FIFOLD_TYPE_MSG | FIFOLD_TYPE_FLUSH1); 881 882 /* Read ICV */ 883 append_seq_fifo_load(desc, icvsize, FIFOLD_CLASS_CLASS1 | 884 FIFOLD_TYPE_ICV | FIFOLD_TYPE_LAST1); 885 886 #ifdef DEBUG 887 print_hex_dump(KERN_ERR, 888 "rfc4106 dec shdesc@" __stringify(__LINE__)": ", 889 DUMP_PREFIX_ADDRESS, 16, 4, desc, desc_bytes(desc), 1); 890 #endif 891 } 892 EXPORT_SYMBOL(cnstr_shdsc_rfc4106_decap); 893 894 /** 895 * cnstr_shdsc_rfc4543_encap - IPSec ESP gmac encapsulation shared descriptor 896 * (non-protocol). 897 * @desc: pointer to buffer used for descriptor construction 898 * @cdata: pointer to block cipher transform definitions 899 * Valid algorithm values - OP_ALG_ALGSEL_AES ANDed with OP_ALG_AAI_GCM. 900 * @icvsize: integrity check value (ICV) size (truncated or full) 901 */ 902 void cnstr_shdsc_rfc4543_encap(u32 * const desc, struct alginfo *cdata, 903 unsigned int icvsize) 904 { 905 u32 *key_jump_cmd, *read_move_cmd, *write_move_cmd; 906 907 init_sh_desc(desc, HDR_SHARE_SERIAL); 908 909 /* Skip key loading if it is loaded due to sharing */ 910 key_jump_cmd = append_jump(desc, JUMP_JSL | JUMP_TEST_ALL | 911 JUMP_COND_SHRD); 912 if (cdata->key_inline) 913 append_key_as_imm(desc, cdata->key_virt, cdata->keylen, 914 cdata->keylen, CLASS_1 | KEY_DEST_CLASS_REG); 915 else 916 append_key(desc, cdata->key_dma, cdata->keylen, CLASS_1 | 917 KEY_DEST_CLASS_REG); 918 set_jump_tgt_here(desc, key_jump_cmd); 919 920 /* Class 1 operation */ 921 append_operation(desc, cdata->algtype | OP_ALG_AS_INITFINAL | 922 OP_ALG_ENCRYPT); 923 924 /* assoclen + cryptlen = seqinlen */ 925 append_math_sub(desc, REG3, SEQINLEN, REG0, CAAM_CMD_SZ); 926 927 /* 928 * MOVE_LEN opcode is not available in all SEC HW revisions, 929 * thus need to do some magic, i.e. self-patch the descriptor 930 * buffer. 931 */ 932 read_move_cmd = append_move(desc, MOVE_SRC_DESCBUF | MOVE_DEST_MATH3 | 933 (0x6 << MOVE_LEN_SHIFT)); 934 write_move_cmd = append_move(desc, MOVE_SRC_MATH3 | MOVE_DEST_DESCBUF | 935 (0x8 << MOVE_LEN_SHIFT)); 936 937 /* Will read assoclen + cryptlen bytes */ 938 append_math_sub(desc, VARSEQINLEN, SEQINLEN, REG0, CAAM_CMD_SZ); 939 940 /* Will write assoclen + cryptlen bytes */ 941 append_math_sub(desc, VARSEQOUTLEN, SEQINLEN, REG0, CAAM_CMD_SZ); 942 943 /* Read and write assoclen + cryptlen bytes */ 944 aead_append_src_dst(desc, FIFOLD_TYPE_AAD); 945 946 set_move_tgt_here(desc, read_move_cmd); 947 set_move_tgt_here(desc, write_move_cmd); 948 append_cmd(desc, CMD_LOAD | DISABLE_AUTO_INFO_FIFO); 949 /* Move payload data to OFIFO */ 950 append_move(desc, MOVE_SRC_INFIFO_CL | MOVE_DEST_OUTFIFO); 951 952 /* Write ICV */ 953 append_seq_store(desc, icvsize, LDST_CLASS_1_CCB | 954 LDST_SRCDST_BYTE_CONTEXT); 955 956 #ifdef DEBUG 957 print_hex_dump(KERN_ERR, 958 "rfc4543 enc shdesc@" __stringify(__LINE__)": ", 959 DUMP_PREFIX_ADDRESS, 16, 4, desc, desc_bytes(desc), 1); 960 #endif 961 } 962 EXPORT_SYMBOL(cnstr_shdsc_rfc4543_encap); 963 964 /** 965 * cnstr_shdsc_rfc4543_decap - IPSec ESP gmac decapsulation shared descriptor 966 * (non-protocol). 967 * @desc: pointer to buffer used for descriptor construction 968 * @cdata: pointer to block cipher transform definitions 969 * Valid algorithm values - OP_ALG_ALGSEL_AES ANDed with OP_ALG_AAI_GCM. 970 * @icvsize: integrity check value (ICV) size (truncated or full) 971 */ 972 void cnstr_shdsc_rfc4543_decap(u32 * const desc, struct alginfo *cdata, 973 unsigned int icvsize) 974 { 975 u32 *key_jump_cmd, *read_move_cmd, *write_move_cmd; 976 977 init_sh_desc(desc, HDR_SHARE_SERIAL); 978 979 /* Skip key loading if it is loaded due to sharing */ 980 key_jump_cmd = append_jump(desc, JUMP_JSL | JUMP_TEST_ALL | 981 JUMP_COND_SHRD); 982 if (cdata->key_inline) 983 append_key_as_imm(desc, cdata->key_virt, cdata->keylen, 984 cdata->keylen, CLASS_1 | KEY_DEST_CLASS_REG); 985 else 986 append_key(desc, cdata->key_dma, cdata->keylen, CLASS_1 | 987 KEY_DEST_CLASS_REG); 988 set_jump_tgt_here(desc, key_jump_cmd); 989 990 /* Class 1 operation */ 991 append_operation(desc, cdata->algtype | OP_ALG_AS_INITFINAL | 992 OP_ALG_DECRYPT | OP_ALG_ICV_ON); 993 994 /* assoclen + cryptlen = seqoutlen */ 995 append_math_sub(desc, REG3, SEQOUTLEN, REG0, CAAM_CMD_SZ); 996 997 /* 998 * MOVE_LEN opcode is not available in all SEC HW revisions, 999 * thus need to do some magic, i.e. self-patch the descriptor 1000 * buffer. 1001 */ 1002 read_move_cmd = append_move(desc, MOVE_SRC_DESCBUF | MOVE_DEST_MATH3 | 1003 (0x6 << MOVE_LEN_SHIFT)); 1004 write_move_cmd = append_move(desc, MOVE_SRC_MATH3 | MOVE_DEST_DESCBUF | 1005 (0x8 << MOVE_LEN_SHIFT)); 1006 1007 /* Will read assoclen + cryptlen bytes */ 1008 append_math_sub(desc, VARSEQINLEN, SEQOUTLEN, REG0, CAAM_CMD_SZ); 1009 1010 /* Will write assoclen + cryptlen bytes */ 1011 append_math_sub(desc, VARSEQOUTLEN, SEQOUTLEN, REG0, CAAM_CMD_SZ); 1012 1013 /* Store payload data */ 1014 append_seq_fifo_store(desc, 0, FIFOST_TYPE_MESSAGE_DATA | FIFOLDST_VLF); 1015 1016 /* In-snoop assoclen + cryptlen data */ 1017 append_seq_fifo_load(desc, 0, FIFOLD_CLASS_BOTH | FIFOLDST_VLF | 1018 FIFOLD_TYPE_AAD | FIFOLD_TYPE_LAST2FLUSH1); 1019 1020 set_move_tgt_here(desc, read_move_cmd); 1021 set_move_tgt_here(desc, write_move_cmd); 1022 append_cmd(desc, CMD_LOAD | DISABLE_AUTO_INFO_FIFO); 1023 /* Move payload data to OFIFO */ 1024 append_move(desc, MOVE_SRC_INFIFO_CL | MOVE_DEST_OUTFIFO); 1025 append_cmd(desc, CMD_LOAD | ENABLE_AUTO_INFO_FIFO); 1026 1027 /* Read ICV */ 1028 append_seq_fifo_load(desc, icvsize, FIFOLD_CLASS_CLASS1 | 1029 FIFOLD_TYPE_ICV | FIFOLD_TYPE_LAST1); 1030 1031 #ifdef DEBUG 1032 print_hex_dump(KERN_ERR, 1033 "rfc4543 dec shdesc@" __stringify(__LINE__)": ", 1034 DUMP_PREFIX_ADDRESS, 16, 4, desc, desc_bytes(desc), 1); 1035 #endif 1036 } 1037 EXPORT_SYMBOL(cnstr_shdsc_rfc4543_decap); 1038 1039 /* 1040 * For ablkcipher encrypt and decrypt, read from req->src and 1041 * write to req->dst 1042 */ 1043 static inline void ablkcipher_append_src_dst(u32 *desc) 1044 { 1045 append_math_add(desc, VARSEQOUTLEN, SEQINLEN, REG0, CAAM_CMD_SZ); 1046 append_math_add(desc, VARSEQINLEN, SEQINLEN, REG0, CAAM_CMD_SZ); 1047 append_seq_fifo_load(desc, 0, FIFOLD_CLASS_CLASS1 | 1048 KEY_VLF | FIFOLD_TYPE_MSG | FIFOLD_TYPE_LAST1); 1049 append_seq_fifo_store(desc, 0, FIFOST_TYPE_MESSAGE_DATA | KEY_VLF); 1050 } 1051 1052 /** 1053 * cnstr_shdsc_ablkcipher_encap - ablkcipher encapsulation shared descriptor 1054 * @desc: pointer to buffer used for descriptor construction 1055 * @cdata: pointer to block cipher transform definitions 1056 * Valid algorithm values - one of OP_ALG_ALGSEL_{AES, DES, 3DES} ANDed 1057 * with OP_ALG_AAI_CBC or OP_ALG_AAI_CTR_MOD128. 1058 * @ivsize: initialization vector size 1059 * @is_rfc3686: true when ctr(aes) is wrapped by rfc3686 template 1060 * @ctx1_iv_off: IV offset in CONTEXT1 register 1061 */ 1062 void cnstr_shdsc_ablkcipher_encap(u32 * const desc, struct alginfo *cdata, 1063 unsigned int ivsize, const bool is_rfc3686, 1064 const u32 ctx1_iv_off) 1065 { 1066 u32 *key_jump_cmd; 1067 1068 init_sh_desc(desc, HDR_SHARE_SERIAL | HDR_SAVECTX); 1069 /* Skip if already shared */ 1070 key_jump_cmd = append_jump(desc, JUMP_JSL | JUMP_TEST_ALL | 1071 JUMP_COND_SHRD); 1072 1073 /* Load class1 key only */ 1074 append_key_as_imm(desc, cdata->key_virt, cdata->keylen, 1075 cdata->keylen, CLASS_1 | KEY_DEST_CLASS_REG); 1076 1077 /* Load nonce into CONTEXT1 reg */ 1078 if (is_rfc3686) { 1079 u8 *nonce = cdata->key_virt + cdata->keylen; 1080 1081 append_load_as_imm(desc, nonce, CTR_RFC3686_NONCE_SIZE, 1082 LDST_CLASS_IND_CCB | 1083 LDST_SRCDST_BYTE_OUTFIFO | LDST_IMM); 1084 append_move(desc, MOVE_WAITCOMP | MOVE_SRC_OUTFIFO | 1085 MOVE_DEST_CLASS1CTX | (16 << MOVE_OFFSET_SHIFT) | 1086 (CTR_RFC3686_NONCE_SIZE << MOVE_LEN_SHIFT)); 1087 } 1088 1089 set_jump_tgt_here(desc, key_jump_cmd); 1090 1091 /* Load iv */ 1092 append_seq_load(desc, ivsize, LDST_SRCDST_BYTE_CONTEXT | 1093 LDST_CLASS_1_CCB | (ctx1_iv_off << LDST_OFFSET_SHIFT)); 1094 1095 /* Load counter into CONTEXT1 reg */ 1096 if (is_rfc3686) 1097 append_load_imm_be32(desc, 1, LDST_IMM | LDST_CLASS_1_CCB | 1098 LDST_SRCDST_BYTE_CONTEXT | 1099 ((ctx1_iv_off + CTR_RFC3686_IV_SIZE) << 1100 LDST_OFFSET_SHIFT)); 1101 1102 /* Load operation */ 1103 append_operation(desc, cdata->algtype | OP_ALG_AS_INITFINAL | 1104 OP_ALG_ENCRYPT); 1105 1106 /* Perform operation */ 1107 ablkcipher_append_src_dst(desc); 1108 1109 #ifdef DEBUG 1110 print_hex_dump(KERN_ERR, 1111 "ablkcipher enc shdesc@" __stringify(__LINE__)": ", 1112 DUMP_PREFIX_ADDRESS, 16, 4, desc, desc_bytes(desc), 1); 1113 #endif 1114 } 1115 EXPORT_SYMBOL(cnstr_shdsc_ablkcipher_encap); 1116 1117 /** 1118 * cnstr_shdsc_ablkcipher_decap - ablkcipher decapsulation shared descriptor 1119 * @desc: pointer to buffer used for descriptor construction 1120 * @cdata: pointer to block cipher transform definitions 1121 * Valid algorithm values - one of OP_ALG_ALGSEL_{AES, DES, 3DES} ANDed 1122 * with OP_ALG_AAI_CBC or OP_ALG_AAI_CTR_MOD128. 1123 * @ivsize: initialization vector size 1124 * @is_rfc3686: true when ctr(aes) is wrapped by rfc3686 template 1125 * @ctx1_iv_off: IV offset in CONTEXT1 register 1126 */ 1127 void cnstr_shdsc_ablkcipher_decap(u32 * const desc, struct alginfo *cdata, 1128 unsigned int ivsize, const bool is_rfc3686, 1129 const u32 ctx1_iv_off) 1130 { 1131 u32 *key_jump_cmd; 1132 1133 init_sh_desc(desc, HDR_SHARE_SERIAL | HDR_SAVECTX); 1134 /* Skip if already shared */ 1135 key_jump_cmd = append_jump(desc, JUMP_JSL | JUMP_TEST_ALL | 1136 JUMP_COND_SHRD); 1137 1138 /* Load class1 key only */ 1139 append_key_as_imm(desc, cdata->key_virt, cdata->keylen, 1140 cdata->keylen, CLASS_1 | KEY_DEST_CLASS_REG); 1141 1142 /* Load nonce into CONTEXT1 reg */ 1143 if (is_rfc3686) { 1144 u8 *nonce = cdata->key_virt + cdata->keylen; 1145 1146 append_load_as_imm(desc, nonce, CTR_RFC3686_NONCE_SIZE, 1147 LDST_CLASS_IND_CCB | 1148 LDST_SRCDST_BYTE_OUTFIFO | LDST_IMM); 1149 append_move(desc, MOVE_WAITCOMP | MOVE_SRC_OUTFIFO | 1150 MOVE_DEST_CLASS1CTX | (16 << MOVE_OFFSET_SHIFT) | 1151 (CTR_RFC3686_NONCE_SIZE << MOVE_LEN_SHIFT)); 1152 } 1153 1154 set_jump_tgt_here(desc, key_jump_cmd); 1155 1156 /* load IV */ 1157 append_seq_load(desc, ivsize, LDST_SRCDST_BYTE_CONTEXT | 1158 LDST_CLASS_1_CCB | (ctx1_iv_off << LDST_OFFSET_SHIFT)); 1159 1160 /* Load counter into CONTEXT1 reg */ 1161 if (is_rfc3686) 1162 append_load_imm_be32(desc, 1, LDST_IMM | LDST_CLASS_1_CCB | 1163 LDST_SRCDST_BYTE_CONTEXT | 1164 ((ctx1_iv_off + CTR_RFC3686_IV_SIZE) << 1165 LDST_OFFSET_SHIFT)); 1166 1167 /* Choose operation */ 1168 if (ctx1_iv_off) 1169 append_operation(desc, cdata->algtype | OP_ALG_AS_INITFINAL | 1170 OP_ALG_DECRYPT); 1171 else 1172 append_dec_op1(desc, cdata->algtype); 1173 1174 /* Perform operation */ 1175 ablkcipher_append_src_dst(desc); 1176 1177 #ifdef DEBUG 1178 print_hex_dump(KERN_ERR, 1179 "ablkcipher dec shdesc@" __stringify(__LINE__)": ", 1180 DUMP_PREFIX_ADDRESS, 16, 4, desc, desc_bytes(desc), 1); 1181 #endif 1182 } 1183 EXPORT_SYMBOL(cnstr_shdsc_ablkcipher_decap); 1184 1185 /** 1186 * cnstr_shdsc_ablkcipher_givencap - ablkcipher encapsulation shared descriptor 1187 * with HW-generated initialization vector. 1188 * @desc: pointer to buffer used for descriptor construction 1189 * @cdata: pointer to block cipher transform definitions 1190 * Valid algorithm values - one of OP_ALG_ALGSEL_{AES, DES, 3DES} ANDed 1191 * with OP_ALG_AAI_CBC. 1192 * @ivsize: initialization vector size 1193 * @is_rfc3686: true when ctr(aes) is wrapped by rfc3686 template 1194 * @ctx1_iv_off: IV offset in CONTEXT1 register 1195 */ 1196 void cnstr_shdsc_ablkcipher_givencap(u32 * const desc, struct alginfo *cdata, 1197 unsigned int ivsize, const bool is_rfc3686, 1198 const u32 ctx1_iv_off) 1199 { 1200 u32 *key_jump_cmd, geniv; 1201 1202 init_sh_desc(desc, HDR_SHARE_SERIAL | HDR_SAVECTX); 1203 /* Skip if already shared */ 1204 key_jump_cmd = append_jump(desc, JUMP_JSL | JUMP_TEST_ALL | 1205 JUMP_COND_SHRD); 1206 1207 /* Load class1 key only */ 1208 append_key_as_imm(desc, cdata->key_virt, cdata->keylen, 1209 cdata->keylen, CLASS_1 | KEY_DEST_CLASS_REG); 1210 1211 /* Load Nonce into CONTEXT1 reg */ 1212 if (is_rfc3686) { 1213 u8 *nonce = cdata->key_virt + cdata->keylen; 1214 1215 append_load_as_imm(desc, nonce, CTR_RFC3686_NONCE_SIZE, 1216 LDST_CLASS_IND_CCB | 1217 LDST_SRCDST_BYTE_OUTFIFO | LDST_IMM); 1218 append_move(desc, MOVE_WAITCOMP | MOVE_SRC_OUTFIFO | 1219 MOVE_DEST_CLASS1CTX | (16 << MOVE_OFFSET_SHIFT) | 1220 (CTR_RFC3686_NONCE_SIZE << MOVE_LEN_SHIFT)); 1221 } 1222 set_jump_tgt_here(desc, key_jump_cmd); 1223 1224 /* Generate IV */ 1225 geniv = NFIFOENTRY_STYPE_PAD | NFIFOENTRY_DEST_DECO | 1226 NFIFOENTRY_DTYPE_MSG | NFIFOENTRY_LC1 | NFIFOENTRY_PTYPE_RND | 1227 (ivsize << NFIFOENTRY_DLEN_SHIFT); 1228 append_load_imm_u32(desc, geniv, LDST_CLASS_IND_CCB | 1229 LDST_SRCDST_WORD_INFO_FIFO | LDST_IMM); 1230 append_cmd(desc, CMD_LOAD | DISABLE_AUTO_INFO_FIFO); 1231 append_move(desc, MOVE_WAITCOMP | MOVE_SRC_INFIFO | 1232 MOVE_DEST_CLASS1CTX | (ivsize << MOVE_LEN_SHIFT) | 1233 (ctx1_iv_off << MOVE_OFFSET_SHIFT)); 1234 append_cmd(desc, CMD_LOAD | ENABLE_AUTO_INFO_FIFO); 1235 1236 /* Copy generated IV to memory */ 1237 append_seq_store(desc, ivsize, LDST_SRCDST_BYTE_CONTEXT | 1238 LDST_CLASS_1_CCB | (ctx1_iv_off << LDST_OFFSET_SHIFT)); 1239 1240 /* Load Counter into CONTEXT1 reg */ 1241 if (is_rfc3686) 1242 append_load_imm_be32(desc, 1, LDST_IMM | LDST_CLASS_1_CCB | 1243 LDST_SRCDST_BYTE_CONTEXT | 1244 ((ctx1_iv_off + CTR_RFC3686_IV_SIZE) << 1245 LDST_OFFSET_SHIFT)); 1246 1247 if (ctx1_iv_off) 1248 append_jump(desc, JUMP_JSL | JUMP_TEST_ALL | JUMP_COND_NCP | 1249 (1 << JUMP_OFFSET_SHIFT)); 1250 1251 /* Load operation */ 1252 append_operation(desc, cdata->algtype | OP_ALG_AS_INITFINAL | 1253 OP_ALG_ENCRYPT); 1254 1255 /* Perform operation */ 1256 ablkcipher_append_src_dst(desc); 1257 1258 #ifdef DEBUG 1259 print_hex_dump(KERN_ERR, 1260 "ablkcipher givenc shdesc@" __stringify(__LINE__) ": ", 1261 DUMP_PREFIX_ADDRESS, 16, 4, desc, desc_bytes(desc), 1); 1262 #endif 1263 } 1264 EXPORT_SYMBOL(cnstr_shdsc_ablkcipher_givencap); 1265 1266 /** 1267 * cnstr_shdsc_xts_ablkcipher_encap - xts ablkcipher encapsulation shared 1268 * descriptor 1269 * @desc: pointer to buffer used for descriptor construction 1270 * @cdata: pointer to block cipher transform definitions 1271 * Valid algorithm values - OP_ALG_ALGSEL_AES ANDed with OP_ALG_AAI_XTS. 1272 */ 1273 void cnstr_shdsc_xts_ablkcipher_encap(u32 * const desc, struct alginfo *cdata) 1274 { 1275 __be64 sector_size = cpu_to_be64(512); 1276 u32 *key_jump_cmd; 1277 1278 init_sh_desc(desc, HDR_SHARE_SERIAL | HDR_SAVECTX); 1279 /* Skip if already shared */ 1280 key_jump_cmd = append_jump(desc, JUMP_JSL | JUMP_TEST_ALL | 1281 JUMP_COND_SHRD); 1282 1283 /* Load class1 keys only */ 1284 append_key_as_imm(desc, cdata->key_virt, cdata->keylen, 1285 cdata->keylen, CLASS_1 | KEY_DEST_CLASS_REG); 1286 1287 /* Load sector size with index 40 bytes (0x28) */ 1288 append_load_as_imm(desc, (void *)§or_size, 8, LDST_CLASS_1_CCB | 1289 LDST_SRCDST_BYTE_CONTEXT | 1290 (0x28 << LDST_OFFSET_SHIFT)); 1291 1292 set_jump_tgt_here(desc, key_jump_cmd); 1293 1294 /* 1295 * create sequence for loading the sector index 1296 * Upper 8B of IV - will be used as sector index 1297 * Lower 8B of IV - will be discarded 1298 */ 1299 append_seq_load(desc, 8, LDST_SRCDST_BYTE_CONTEXT | LDST_CLASS_1_CCB | 1300 (0x20 << LDST_OFFSET_SHIFT)); 1301 append_seq_fifo_load(desc, 8, FIFOLD_CLASS_SKIP); 1302 1303 /* Load operation */ 1304 append_operation(desc, cdata->algtype | OP_ALG_AS_INITFINAL | 1305 OP_ALG_ENCRYPT); 1306 1307 /* Perform operation */ 1308 ablkcipher_append_src_dst(desc); 1309 1310 #ifdef DEBUG 1311 print_hex_dump(KERN_ERR, 1312 "xts ablkcipher enc shdesc@" __stringify(__LINE__) ": ", 1313 DUMP_PREFIX_ADDRESS, 16, 4, desc, desc_bytes(desc), 1); 1314 #endif 1315 } 1316 EXPORT_SYMBOL(cnstr_shdsc_xts_ablkcipher_encap); 1317 1318 /** 1319 * cnstr_shdsc_xts_ablkcipher_decap - xts ablkcipher decapsulation shared 1320 * descriptor 1321 * @desc: pointer to buffer used for descriptor construction 1322 * @cdata: pointer to block cipher transform definitions 1323 * Valid algorithm values - OP_ALG_ALGSEL_AES ANDed with OP_ALG_AAI_XTS. 1324 */ 1325 void cnstr_shdsc_xts_ablkcipher_decap(u32 * const desc, struct alginfo *cdata) 1326 { 1327 __be64 sector_size = cpu_to_be64(512); 1328 u32 *key_jump_cmd; 1329 1330 init_sh_desc(desc, HDR_SHARE_SERIAL | HDR_SAVECTX); 1331 /* Skip if already shared */ 1332 key_jump_cmd = append_jump(desc, JUMP_JSL | JUMP_TEST_ALL | 1333 JUMP_COND_SHRD); 1334 1335 /* Load class1 key only */ 1336 append_key_as_imm(desc, cdata->key_virt, cdata->keylen, 1337 cdata->keylen, CLASS_1 | KEY_DEST_CLASS_REG); 1338 1339 /* Load sector size with index 40 bytes (0x28) */ 1340 append_load_as_imm(desc, (void *)§or_size, 8, LDST_CLASS_1_CCB | 1341 LDST_SRCDST_BYTE_CONTEXT | 1342 (0x28 << LDST_OFFSET_SHIFT)); 1343 1344 set_jump_tgt_here(desc, key_jump_cmd); 1345 1346 /* 1347 * create sequence for loading the sector index 1348 * Upper 8B of IV - will be used as sector index 1349 * Lower 8B of IV - will be discarded 1350 */ 1351 append_seq_load(desc, 8, LDST_SRCDST_BYTE_CONTEXT | LDST_CLASS_1_CCB | 1352 (0x20 << LDST_OFFSET_SHIFT)); 1353 append_seq_fifo_load(desc, 8, FIFOLD_CLASS_SKIP); 1354 1355 /* Load operation */ 1356 append_dec_op1(desc, cdata->algtype); 1357 1358 /* Perform operation */ 1359 ablkcipher_append_src_dst(desc); 1360 1361 #ifdef DEBUG 1362 print_hex_dump(KERN_ERR, 1363 "xts ablkcipher dec shdesc@" __stringify(__LINE__) ": ", 1364 DUMP_PREFIX_ADDRESS, 16, 4, desc, desc_bytes(desc), 1); 1365 #endif 1366 } 1367 EXPORT_SYMBOL(cnstr_shdsc_xts_ablkcipher_decap); 1368 1369 MODULE_LICENSE("GPL"); 1370 MODULE_DESCRIPTION("FSL CAAM descriptor support"); 1371 MODULE_AUTHOR("Freescale Semiconductor - NMG/STC"); 1372