1 /* 2 * Shared descriptors for aead, ablkcipher algorithms 3 * 4 * Copyright 2016 NXP 5 */ 6 7 #include "compat.h" 8 #include "desc_constr.h" 9 #include "caamalg_desc.h" 10 11 /* 12 * For aead functions, read payload and write payload, 13 * both of which are specified in req->src and req->dst 14 */ 15 static inline void aead_append_src_dst(u32 *desc, u32 msg_type) 16 { 17 append_seq_fifo_store(desc, 0, FIFOST_TYPE_MESSAGE_DATA | KEY_VLF); 18 append_seq_fifo_load(desc, 0, FIFOLD_CLASS_BOTH | 19 KEY_VLF | msg_type | FIFOLD_TYPE_LASTBOTH); 20 } 21 22 /* Set DK bit in class 1 operation if shared */ 23 static inline void append_dec_op1(u32 *desc, u32 type) 24 { 25 u32 *jump_cmd, *uncond_jump_cmd; 26 27 /* DK bit is valid only for AES */ 28 if ((type & OP_ALG_ALGSEL_MASK) != OP_ALG_ALGSEL_AES) { 29 append_operation(desc, type | OP_ALG_AS_INITFINAL | 30 OP_ALG_DECRYPT); 31 return; 32 } 33 34 jump_cmd = append_jump(desc, JUMP_TEST_ALL | JUMP_COND_SHRD); 35 append_operation(desc, type | OP_ALG_AS_INITFINAL | 36 OP_ALG_DECRYPT); 37 uncond_jump_cmd = append_jump(desc, JUMP_TEST_ALL); 38 set_jump_tgt_here(desc, jump_cmd); 39 append_operation(desc, type | OP_ALG_AS_INITFINAL | 40 OP_ALG_DECRYPT | OP_ALG_AAI_DK); 41 set_jump_tgt_here(desc, uncond_jump_cmd); 42 } 43 44 /** 45 * cnstr_shdsc_aead_null_encap - IPSec ESP encapsulation shared descriptor 46 * (non-protocol) with no (null) encryption. 47 * @desc: pointer to buffer used for descriptor construction 48 * @adata: pointer to authentication transform definitions. Note that since a 49 * split key is to be used, the size of the split key itself is 50 * specified. Valid algorithm values - one of OP_ALG_ALGSEL_{MD5, SHA1, 51 * SHA224, SHA256, SHA384, SHA512} ANDed with OP_ALG_AAI_HMAC_PRECOMP. 52 * @icvsize: integrity check value (ICV) size (truncated or full) 53 * 54 * Note: Requires an MDHA split key. 55 */ 56 void cnstr_shdsc_aead_null_encap(u32 * const desc, struct alginfo *adata, 57 unsigned int icvsize) 58 { 59 u32 *key_jump_cmd, *read_move_cmd, *write_move_cmd; 60 61 init_sh_desc(desc, HDR_SHARE_SERIAL); 62 63 /* Skip if already shared */ 64 key_jump_cmd = append_jump(desc, JUMP_JSL | JUMP_TEST_ALL | 65 JUMP_COND_SHRD); 66 if (adata->key_inline) 67 append_key_as_imm(desc, adata->key_virt, adata->keylen_pad, 68 adata->keylen, CLASS_2 | KEY_DEST_MDHA_SPLIT | 69 KEY_ENC); 70 else 71 append_key(desc, adata->key_dma, adata->keylen, CLASS_2 | 72 KEY_DEST_MDHA_SPLIT | KEY_ENC); 73 set_jump_tgt_here(desc, key_jump_cmd); 74 75 /* assoclen + cryptlen = seqinlen */ 76 append_math_sub(desc, REG3, SEQINLEN, REG0, CAAM_CMD_SZ); 77 78 /* Prepare to read and write cryptlen + assoclen bytes */ 79 append_math_add(desc, VARSEQINLEN, ZERO, REG3, CAAM_CMD_SZ); 80 append_math_add(desc, VARSEQOUTLEN, ZERO, REG3, CAAM_CMD_SZ); 81 82 /* 83 * MOVE_LEN opcode is not available in all SEC HW revisions, 84 * thus need to do some magic, i.e. self-patch the descriptor 85 * buffer. 86 */ 87 read_move_cmd = append_move(desc, MOVE_SRC_DESCBUF | 88 MOVE_DEST_MATH3 | 89 (0x6 << MOVE_LEN_SHIFT)); 90 write_move_cmd = append_move(desc, MOVE_SRC_MATH3 | 91 MOVE_DEST_DESCBUF | 92 MOVE_WAITCOMP | 93 (0x8 << MOVE_LEN_SHIFT)); 94 95 /* Class 2 operation */ 96 append_operation(desc, adata->algtype | OP_ALG_AS_INITFINAL | 97 OP_ALG_ENCRYPT); 98 99 /* Read and write cryptlen bytes */ 100 aead_append_src_dst(desc, FIFOLD_TYPE_MSG | FIFOLD_TYPE_FLUSH1); 101 102 set_move_tgt_here(desc, read_move_cmd); 103 set_move_tgt_here(desc, write_move_cmd); 104 append_cmd(desc, CMD_LOAD | DISABLE_AUTO_INFO_FIFO); 105 append_move(desc, MOVE_SRC_INFIFO_CL | MOVE_DEST_OUTFIFO | 106 MOVE_AUX_LS); 107 108 /* Write ICV */ 109 append_seq_store(desc, icvsize, LDST_CLASS_2_CCB | 110 LDST_SRCDST_BYTE_CONTEXT); 111 112 #ifdef DEBUG 113 print_hex_dump(KERN_ERR, 114 "aead null enc shdesc@" __stringify(__LINE__)": ", 115 DUMP_PREFIX_ADDRESS, 16, 4, desc, desc_bytes(desc), 1); 116 #endif 117 } 118 EXPORT_SYMBOL(cnstr_shdsc_aead_null_encap); 119 120 /** 121 * cnstr_shdsc_aead_null_decap - IPSec ESP decapsulation shared descriptor 122 * (non-protocol) with no (null) decryption. 123 * @desc: pointer to buffer used for descriptor construction 124 * @adata: pointer to authentication transform definitions. Note that since a 125 * split key is to be used, the size of the split key itself is 126 * specified. Valid algorithm values - one of OP_ALG_ALGSEL_{MD5, SHA1, 127 * SHA224, SHA256, SHA384, SHA512} ANDed with OP_ALG_AAI_HMAC_PRECOMP. 128 * @icvsize: integrity check value (ICV) size (truncated or full) 129 * 130 * Note: Requires an MDHA split key. 131 */ 132 void cnstr_shdsc_aead_null_decap(u32 * const desc, struct alginfo *adata, 133 unsigned int icvsize) 134 { 135 u32 *key_jump_cmd, *read_move_cmd, *write_move_cmd, *jump_cmd; 136 137 init_sh_desc(desc, HDR_SHARE_SERIAL); 138 139 /* Skip if already shared */ 140 key_jump_cmd = append_jump(desc, JUMP_JSL | JUMP_TEST_ALL | 141 JUMP_COND_SHRD); 142 if (adata->key_inline) 143 append_key_as_imm(desc, adata->key_virt, adata->keylen_pad, 144 adata->keylen, CLASS_2 | 145 KEY_DEST_MDHA_SPLIT | KEY_ENC); 146 else 147 append_key(desc, adata->key_dma, adata->keylen, CLASS_2 | 148 KEY_DEST_MDHA_SPLIT | KEY_ENC); 149 set_jump_tgt_here(desc, key_jump_cmd); 150 151 /* Class 2 operation */ 152 append_operation(desc, adata->algtype | OP_ALG_AS_INITFINAL | 153 OP_ALG_DECRYPT | OP_ALG_ICV_ON); 154 155 /* assoclen + cryptlen = seqoutlen */ 156 append_math_sub(desc, REG2, SEQOUTLEN, REG0, CAAM_CMD_SZ); 157 158 /* Prepare to read and write cryptlen + assoclen bytes */ 159 append_math_add(desc, VARSEQINLEN, ZERO, REG2, CAAM_CMD_SZ); 160 append_math_add(desc, VARSEQOUTLEN, ZERO, REG2, CAAM_CMD_SZ); 161 162 /* 163 * MOVE_LEN opcode is not available in all SEC HW revisions, 164 * thus need to do some magic, i.e. self-patch the descriptor 165 * buffer. 166 */ 167 read_move_cmd = append_move(desc, MOVE_SRC_DESCBUF | 168 MOVE_DEST_MATH2 | 169 (0x6 << MOVE_LEN_SHIFT)); 170 write_move_cmd = append_move(desc, MOVE_SRC_MATH2 | 171 MOVE_DEST_DESCBUF | 172 MOVE_WAITCOMP | 173 (0x8 << MOVE_LEN_SHIFT)); 174 175 /* Read and write cryptlen bytes */ 176 aead_append_src_dst(desc, FIFOLD_TYPE_MSG | FIFOLD_TYPE_FLUSH1); 177 178 /* 179 * Insert a NOP here, since we need at least 4 instructions between 180 * code patching the descriptor buffer and the location being patched. 181 */ 182 jump_cmd = append_jump(desc, JUMP_TEST_ALL); 183 set_jump_tgt_here(desc, jump_cmd); 184 185 set_move_tgt_here(desc, read_move_cmd); 186 set_move_tgt_here(desc, write_move_cmd); 187 append_cmd(desc, CMD_LOAD | DISABLE_AUTO_INFO_FIFO); 188 append_move(desc, MOVE_SRC_INFIFO_CL | MOVE_DEST_OUTFIFO | 189 MOVE_AUX_LS); 190 append_cmd(desc, CMD_LOAD | ENABLE_AUTO_INFO_FIFO); 191 192 /* Load ICV */ 193 append_seq_fifo_load(desc, icvsize, FIFOLD_CLASS_CLASS2 | 194 FIFOLD_TYPE_LAST2 | FIFOLD_TYPE_ICV); 195 196 #ifdef DEBUG 197 print_hex_dump(KERN_ERR, 198 "aead null dec shdesc@" __stringify(__LINE__)": ", 199 DUMP_PREFIX_ADDRESS, 16, 4, desc, desc_bytes(desc), 1); 200 #endif 201 } 202 EXPORT_SYMBOL(cnstr_shdsc_aead_null_decap); 203 204 static void init_sh_desc_key_aead(u32 * const desc, 205 struct alginfo * const cdata, 206 struct alginfo * const adata, 207 const bool is_rfc3686, u32 *nonce) 208 { 209 u32 *key_jump_cmd; 210 unsigned int enckeylen = cdata->keylen; 211 212 /* Note: Context registers are saved. */ 213 init_sh_desc(desc, HDR_SHARE_SERIAL | HDR_SAVECTX); 214 215 /* Skip if already shared */ 216 key_jump_cmd = append_jump(desc, JUMP_JSL | JUMP_TEST_ALL | 217 JUMP_COND_SHRD); 218 219 /* 220 * RFC3686 specific: 221 * | key = {AUTH_KEY, ENC_KEY, NONCE} 222 * | enckeylen = encryption key size + nonce size 223 */ 224 if (is_rfc3686) 225 enckeylen -= CTR_RFC3686_NONCE_SIZE; 226 227 if (adata->key_inline) 228 append_key_as_imm(desc, adata->key_virt, adata->keylen_pad, 229 adata->keylen, CLASS_2 | 230 KEY_DEST_MDHA_SPLIT | KEY_ENC); 231 else 232 append_key(desc, adata->key_dma, adata->keylen, CLASS_2 | 233 KEY_DEST_MDHA_SPLIT | KEY_ENC); 234 235 if (cdata->key_inline) 236 append_key_as_imm(desc, cdata->key_virt, enckeylen, 237 enckeylen, CLASS_1 | KEY_DEST_CLASS_REG); 238 else 239 append_key(desc, cdata->key_dma, enckeylen, CLASS_1 | 240 KEY_DEST_CLASS_REG); 241 242 /* Load Counter into CONTEXT1 reg */ 243 if (is_rfc3686) { 244 append_load_as_imm(desc, nonce, CTR_RFC3686_NONCE_SIZE, 245 LDST_CLASS_IND_CCB | 246 LDST_SRCDST_BYTE_OUTFIFO | LDST_IMM); 247 append_move(desc, 248 MOVE_SRC_OUTFIFO | 249 MOVE_DEST_CLASS1CTX | 250 (16 << MOVE_OFFSET_SHIFT) | 251 (CTR_RFC3686_NONCE_SIZE << MOVE_LEN_SHIFT)); 252 } 253 254 set_jump_tgt_here(desc, key_jump_cmd); 255 } 256 257 /** 258 * cnstr_shdsc_aead_encap - IPSec ESP encapsulation shared descriptor 259 * (non-protocol). 260 * @desc: pointer to buffer used for descriptor construction 261 * @cdata: pointer to block cipher transform definitions 262 * Valid algorithm values - one of OP_ALG_ALGSEL_{AES, DES, 3DES} ANDed 263 * with OP_ALG_AAI_CBC or OP_ALG_AAI_CTR_MOD128. 264 * @adata: pointer to authentication transform definitions. Note that since a 265 * split key is to be used, the size of the split key itself is 266 * specified. Valid algorithm values - one of OP_ALG_ALGSEL_{MD5, SHA1, 267 * SHA224, SHA256, SHA384, SHA512} ANDed with OP_ALG_AAI_HMAC_PRECOMP. 268 * @ivsize: initialization vector size 269 * @icvsize: integrity check value (ICV) size (truncated or full) 270 * @is_rfc3686: true when ctr(aes) is wrapped by rfc3686 template 271 * @nonce: pointer to rfc3686 nonce 272 * @ctx1_iv_off: IV offset in CONTEXT1 register 273 * @is_qi: true when called from caam/qi 274 * 275 * Note: Requires an MDHA split key. 276 */ 277 void cnstr_shdsc_aead_encap(u32 * const desc, struct alginfo *cdata, 278 struct alginfo *adata, unsigned int ivsize, 279 unsigned int icvsize, const bool is_rfc3686, 280 u32 *nonce, const u32 ctx1_iv_off, const bool is_qi) 281 { 282 /* Note: Context registers are saved. */ 283 init_sh_desc_key_aead(desc, cdata, adata, is_rfc3686, nonce); 284 285 /* Class 2 operation */ 286 append_operation(desc, adata->algtype | OP_ALG_AS_INITFINAL | 287 OP_ALG_ENCRYPT); 288 289 if (is_qi) { 290 u32 *wait_load_cmd; 291 292 /* REG3 = assoclen */ 293 append_seq_load(desc, 4, LDST_CLASS_DECO | 294 LDST_SRCDST_WORD_DECO_MATH3 | 295 (4 << LDST_OFFSET_SHIFT)); 296 297 wait_load_cmd = append_jump(desc, JUMP_JSL | JUMP_TEST_ALL | 298 JUMP_COND_CALM | JUMP_COND_NCP | 299 JUMP_COND_NOP | JUMP_COND_NIP | 300 JUMP_COND_NIFP); 301 set_jump_tgt_here(desc, wait_load_cmd); 302 303 append_seq_load(desc, ivsize, LDST_CLASS_1_CCB | 304 LDST_SRCDST_BYTE_CONTEXT | 305 (ctx1_iv_off << LDST_OFFSET_SHIFT)); 306 } 307 308 /* Read and write assoclen bytes */ 309 append_math_add(desc, VARSEQINLEN, ZERO, REG3, CAAM_CMD_SZ); 310 append_math_add(desc, VARSEQOUTLEN, ZERO, REG3, CAAM_CMD_SZ); 311 312 /* Skip assoc data */ 313 append_seq_fifo_store(desc, 0, FIFOST_TYPE_SKIP | FIFOLDST_VLF); 314 315 /* read assoc before reading payload */ 316 append_seq_fifo_load(desc, 0, FIFOLD_CLASS_CLASS2 | FIFOLD_TYPE_MSG | 317 FIFOLDST_VLF); 318 319 /* Load Counter into CONTEXT1 reg */ 320 if (is_rfc3686) 321 append_load_imm_be32(desc, 1, LDST_IMM | LDST_CLASS_1_CCB | 322 LDST_SRCDST_BYTE_CONTEXT | 323 ((ctx1_iv_off + CTR_RFC3686_IV_SIZE) << 324 LDST_OFFSET_SHIFT)); 325 326 /* Class 1 operation */ 327 append_operation(desc, cdata->algtype | OP_ALG_AS_INITFINAL | 328 OP_ALG_ENCRYPT); 329 330 /* Read and write cryptlen bytes */ 331 append_math_add(desc, VARSEQINLEN, SEQINLEN, REG0, CAAM_CMD_SZ); 332 append_math_add(desc, VARSEQOUTLEN, SEQINLEN, REG0, CAAM_CMD_SZ); 333 aead_append_src_dst(desc, FIFOLD_TYPE_MSG1OUT2); 334 335 /* Write ICV */ 336 append_seq_store(desc, icvsize, LDST_CLASS_2_CCB | 337 LDST_SRCDST_BYTE_CONTEXT); 338 339 #ifdef DEBUG 340 print_hex_dump(KERN_ERR, "aead enc shdesc@" __stringify(__LINE__)": ", 341 DUMP_PREFIX_ADDRESS, 16, 4, desc, desc_bytes(desc), 1); 342 #endif 343 } 344 EXPORT_SYMBOL(cnstr_shdsc_aead_encap); 345 346 /** 347 * cnstr_shdsc_aead_decap - IPSec ESP decapsulation shared descriptor 348 * (non-protocol). 349 * @desc: pointer to buffer used for descriptor construction 350 * @cdata: pointer to block cipher transform definitions 351 * Valid algorithm values - one of OP_ALG_ALGSEL_{AES, DES, 3DES} ANDed 352 * with OP_ALG_AAI_CBC or OP_ALG_AAI_CTR_MOD128. 353 * @adata: pointer to authentication transform definitions. Note that since a 354 * split key is to be used, the size of the split key itself is 355 * specified. Valid algorithm values - one of OP_ALG_ALGSEL_{MD5, SHA1, 356 * SHA224, SHA256, SHA384, SHA512} ANDed with OP_ALG_AAI_HMAC_PRECOMP. 357 * @ivsize: initialization vector size 358 * @icvsize: integrity check value (ICV) size (truncated or full) 359 * @is_rfc3686: true when ctr(aes) is wrapped by rfc3686 template 360 * @nonce: pointer to rfc3686 nonce 361 * @ctx1_iv_off: IV offset in CONTEXT1 register 362 * @is_qi: true when called from caam/qi 363 * 364 * Note: Requires an MDHA split key. 365 */ 366 void cnstr_shdsc_aead_decap(u32 * const desc, struct alginfo *cdata, 367 struct alginfo *adata, unsigned int ivsize, 368 unsigned int icvsize, const bool geniv, 369 const bool is_rfc3686, u32 *nonce, 370 const u32 ctx1_iv_off, const bool is_qi) 371 { 372 /* Note: Context registers are saved. */ 373 init_sh_desc_key_aead(desc, cdata, adata, is_rfc3686, nonce); 374 375 /* Class 2 operation */ 376 append_operation(desc, adata->algtype | OP_ALG_AS_INITFINAL | 377 OP_ALG_DECRYPT | OP_ALG_ICV_ON); 378 379 if (is_qi) { 380 u32 *wait_load_cmd; 381 382 /* REG3 = assoclen */ 383 append_seq_load(desc, 4, LDST_CLASS_DECO | 384 LDST_SRCDST_WORD_DECO_MATH3 | 385 (4 << LDST_OFFSET_SHIFT)); 386 387 wait_load_cmd = append_jump(desc, JUMP_JSL | JUMP_TEST_ALL | 388 JUMP_COND_CALM | JUMP_COND_NCP | 389 JUMP_COND_NOP | JUMP_COND_NIP | 390 JUMP_COND_NIFP); 391 set_jump_tgt_here(desc, wait_load_cmd); 392 393 if (!geniv) 394 append_seq_load(desc, ivsize, LDST_CLASS_1_CCB | 395 LDST_SRCDST_BYTE_CONTEXT | 396 (ctx1_iv_off << LDST_OFFSET_SHIFT)); 397 } 398 399 /* Read and write assoclen bytes */ 400 append_math_add(desc, VARSEQINLEN, ZERO, REG3, CAAM_CMD_SZ); 401 if (geniv) 402 append_math_add_imm_u32(desc, VARSEQOUTLEN, REG3, IMM, ivsize); 403 else 404 append_math_add(desc, VARSEQOUTLEN, ZERO, REG3, CAAM_CMD_SZ); 405 406 /* Skip assoc data */ 407 append_seq_fifo_store(desc, 0, FIFOST_TYPE_SKIP | FIFOLDST_VLF); 408 409 /* read assoc before reading payload */ 410 append_seq_fifo_load(desc, 0, FIFOLD_CLASS_CLASS2 | FIFOLD_TYPE_MSG | 411 KEY_VLF); 412 413 if (geniv) { 414 append_seq_load(desc, ivsize, LDST_CLASS_1_CCB | 415 LDST_SRCDST_BYTE_CONTEXT | 416 (ctx1_iv_off << LDST_OFFSET_SHIFT)); 417 append_move(desc, MOVE_SRC_CLASS1CTX | MOVE_DEST_CLASS2INFIFO | 418 (ctx1_iv_off << MOVE_OFFSET_SHIFT) | ivsize); 419 } 420 421 /* Load Counter into CONTEXT1 reg */ 422 if (is_rfc3686) 423 append_load_imm_be32(desc, 1, LDST_IMM | LDST_CLASS_1_CCB | 424 LDST_SRCDST_BYTE_CONTEXT | 425 ((ctx1_iv_off + CTR_RFC3686_IV_SIZE) << 426 LDST_OFFSET_SHIFT)); 427 428 /* Choose operation */ 429 if (ctx1_iv_off) 430 append_operation(desc, cdata->algtype | OP_ALG_AS_INITFINAL | 431 OP_ALG_DECRYPT); 432 else 433 append_dec_op1(desc, cdata->algtype); 434 435 /* Read and write cryptlen bytes */ 436 append_math_add(desc, VARSEQINLEN, SEQOUTLEN, REG0, CAAM_CMD_SZ); 437 append_math_add(desc, VARSEQOUTLEN, SEQOUTLEN, REG0, CAAM_CMD_SZ); 438 aead_append_src_dst(desc, FIFOLD_TYPE_MSG); 439 440 /* Load ICV */ 441 append_seq_fifo_load(desc, icvsize, FIFOLD_CLASS_CLASS2 | 442 FIFOLD_TYPE_LAST2 | FIFOLD_TYPE_ICV); 443 444 #ifdef DEBUG 445 print_hex_dump(KERN_ERR, "aead dec shdesc@" __stringify(__LINE__)": ", 446 DUMP_PREFIX_ADDRESS, 16, 4, desc, desc_bytes(desc), 1); 447 #endif 448 } 449 EXPORT_SYMBOL(cnstr_shdsc_aead_decap); 450 451 /** 452 * cnstr_shdsc_aead_givencap - IPSec ESP encapsulation shared descriptor 453 * (non-protocol) with HW-generated initialization 454 * vector. 455 * @desc: pointer to buffer used for descriptor construction 456 * @cdata: pointer to block cipher transform definitions 457 * Valid algorithm values - one of OP_ALG_ALGSEL_{AES, DES, 3DES} ANDed 458 * with OP_ALG_AAI_CBC or OP_ALG_AAI_CTR_MOD128. 459 * @adata: pointer to authentication transform definitions. Note that since a 460 * split key is to be used, the size of the split key itself is 461 * specified. Valid algorithm values - one of OP_ALG_ALGSEL_{MD5, SHA1, 462 * SHA224, SHA256, SHA384, SHA512} ANDed with OP_ALG_AAI_HMAC_PRECOMP. 463 * @ivsize: initialization vector size 464 * @icvsize: integrity check value (ICV) size (truncated or full) 465 * @is_rfc3686: true when ctr(aes) is wrapped by rfc3686 template 466 * @nonce: pointer to rfc3686 nonce 467 * @ctx1_iv_off: IV offset in CONTEXT1 register 468 * @is_qi: true when called from caam/qi 469 * 470 * Note: Requires an MDHA split key. 471 */ 472 void cnstr_shdsc_aead_givencap(u32 * const desc, struct alginfo *cdata, 473 struct alginfo *adata, unsigned int ivsize, 474 unsigned int icvsize, const bool is_rfc3686, 475 u32 *nonce, const u32 ctx1_iv_off, 476 const bool is_qi) 477 { 478 u32 geniv, moveiv; 479 480 /* Note: Context registers are saved. */ 481 init_sh_desc_key_aead(desc, cdata, adata, is_rfc3686, nonce); 482 483 if (is_qi) { 484 u32 *wait_load_cmd; 485 486 /* REG3 = assoclen */ 487 append_seq_load(desc, 4, LDST_CLASS_DECO | 488 LDST_SRCDST_WORD_DECO_MATH3 | 489 (4 << LDST_OFFSET_SHIFT)); 490 491 wait_load_cmd = append_jump(desc, JUMP_JSL | JUMP_TEST_ALL | 492 JUMP_COND_CALM | JUMP_COND_NCP | 493 JUMP_COND_NOP | JUMP_COND_NIP | 494 JUMP_COND_NIFP); 495 set_jump_tgt_here(desc, wait_load_cmd); 496 } 497 498 if (is_rfc3686) { 499 if (is_qi) 500 append_seq_load(desc, ivsize, LDST_CLASS_1_CCB | 501 LDST_SRCDST_BYTE_CONTEXT | 502 (ctx1_iv_off << LDST_OFFSET_SHIFT)); 503 504 goto copy_iv; 505 } 506 507 /* Generate IV */ 508 geniv = NFIFOENTRY_STYPE_PAD | NFIFOENTRY_DEST_DECO | 509 NFIFOENTRY_DTYPE_MSG | NFIFOENTRY_LC1 | 510 NFIFOENTRY_PTYPE_RND | (ivsize << NFIFOENTRY_DLEN_SHIFT); 511 append_load_imm_u32(desc, geniv, LDST_CLASS_IND_CCB | 512 LDST_SRCDST_WORD_INFO_FIFO | LDST_IMM); 513 append_cmd(desc, CMD_LOAD | DISABLE_AUTO_INFO_FIFO); 514 append_move(desc, MOVE_WAITCOMP | 515 MOVE_SRC_INFIFO | MOVE_DEST_CLASS1CTX | 516 (ctx1_iv_off << MOVE_OFFSET_SHIFT) | 517 (ivsize << MOVE_LEN_SHIFT)); 518 append_cmd(desc, CMD_LOAD | ENABLE_AUTO_INFO_FIFO); 519 520 copy_iv: 521 /* Copy IV to class 1 context */ 522 append_move(desc, MOVE_SRC_CLASS1CTX | MOVE_DEST_OUTFIFO | 523 (ctx1_iv_off << MOVE_OFFSET_SHIFT) | 524 (ivsize << MOVE_LEN_SHIFT)); 525 526 /* Return to encryption */ 527 append_operation(desc, adata->algtype | OP_ALG_AS_INITFINAL | 528 OP_ALG_ENCRYPT); 529 530 /* Read and write assoclen bytes */ 531 append_math_add(desc, VARSEQINLEN, ZERO, REG3, CAAM_CMD_SZ); 532 append_math_add(desc, VARSEQOUTLEN, ZERO, REG3, CAAM_CMD_SZ); 533 534 /* Skip assoc data */ 535 append_seq_fifo_store(desc, 0, FIFOST_TYPE_SKIP | FIFOLDST_VLF); 536 537 /* read assoc before reading payload */ 538 append_seq_fifo_load(desc, 0, FIFOLD_CLASS_CLASS2 | FIFOLD_TYPE_MSG | 539 KEY_VLF); 540 541 /* Copy iv from outfifo to class 2 fifo */ 542 moveiv = NFIFOENTRY_STYPE_OFIFO | NFIFOENTRY_DEST_CLASS2 | 543 NFIFOENTRY_DTYPE_MSG | (ivsize << NFIFOENTRY_DLEN_SHIFT); 544 append_load_imm_u32(desc, moveiv, LDST_CLASS_IND_CCB | 545 LDST_SRCDST_WORD_INFO_FIFO | LDST_IMM); 546 append_load_imm_u32(desc, ivsize, LDST_CLASS_2_CCB | 547 LDST_SRCDST_WORD_DATASZ_REG | LDST_IMM); 548 549 /* Load Counter into CONTEXT1 reg */ 550 if (is_rfc3686) 551 append_load_imm_be32(desc, 1, LDST_IMM | LDST_CLASS_1_CCB | 552 LDST_SRCDST_BYTE_CONTEXT | 553 ((ctx1_iv_off + CTR_RFC3686_IV_SIZE) << 554 LDST_OFFSET_SHIFT)); 555 556 /* Class 1 operation */ 557 append_operation(desc, cdata->algtype | OP_ALG_AS_INITFINAL | 558 OP_ALG_ENCRYPT); 559 560 /* Will write ivsize + cryptlen */ 561 append_math_add(desc, VARSEQOUTLEN, SEQINLEN, REG0, CAAM_CMD_SZ); 562 563 /* Not need to reload iv */ 564 append_seq_fifo_load(desc, ivsize, 565 FIFOLD_CLASS_SKIP); 566 567 /* Will read cryptlen */ 568 append_math_add(desc, VARSEQINLEN, SEQINLEN, REG0, CAAM_CMD_SZ); 569 append_seq_fifo_load(desc, 0, FIFOLD_CLASS_BOTH | KEY_VLF | 570 FIFOLD_TYPE_MSG1OUT2 | FIFOLD_TYPE_LASTBOTH); 571 append_seq_fifo_store(desc, 0, FIFOST_TYPE_MESSAGE_DATA | KEY_VLF); 572 573 /* Write ICV */ 574 append_seq_store(desc, icvsize, LDST_CLASS_2_CCB | 575 LDST_SRCDST_BYTE_CONTEXT); 576 577 #ifdef DEBUG 578 print_hex_dump(KERN_ERR, 579 "aead givenc shdesc@" __stringify(__LINE__)": ", 580 DUMP_PREFIX_ADDRESS, 16, 4, desc, desc_bytes(desc), 1); 581 #endif 582 } 583 EXPORT_SYMBOL(cnstr_shdsc_aead_givencap); 584 585 /** 586 * cnstr_shdsc_gcm_encap - gcm encapsulation shared descriptor 587 * @desc: pointer to buffer used for descriptor construction 588 * @cdata: pointer to block cipher transform definitions 589 * Valid algorithm values - OP_ALG_ALGSEL_AES ANDed with OP_ALG_AAI_GCM. 590 * @icvsize: integrity check value (ICV) size (truncated or full) 591 */ 592 void cnstr_shdsc_gcm_encap(u32 * const desc, struct alginfo *cdata, 593 unsigned int icvsize) 594 { 595 u32 *key_jump_cmd, *zero_payload_jump_cmd, *zero_assoc_jump_cmd1, 596 *zero_assoc_jump_cmd2; 597 598 init_sh_desc(desc, HDR_SHARE_SERIAL); 599 600 /* skip key loading if they are loaded due to sharing */ 601 key_jump_cmd = append_jump(desc, JUMP_JSL | JUMP_TEST_ALL | 602 JUMP_COND_SHRD); 603 if (cdata->key_inline) 604 append_key_as_imm(desc, cdata->key_virt, cdata->keylen, 605 cdata->keylen, CLASS_1 | KEY_DEST_CLASS_REG); 606 else 607 append_key(desc, cdata->key_dma, cdata->keylen, CLASS_1 | 608 KEY_DEST_CLASS_REG); 609 set_jump_tgt_here(desc, key_jump_cmd); 610 611 /* class 1 operation */ 612 append_operation(desc, cdata->algtype | OP_ALG_AS_INITFINAL | 613 OP_ALG_ENCRYPT); 614 615 /* if assoclen + cryptlen is ZERO, skip to ICV write */ 616 append_math_sub(desc, VARSEQOUTLEN, SEQINLEN, REG0, CAAM_CMD_SZ); 617 zero_assoc_jump_cmd2 = append_jump(desc, JUMP_TEST_ALL | 618 JUMP_COND_MATH_Z); 619 620 /* if assoclen is ZERO, skip reading the assoc data */ 621 append_math_add(desc, VARSEQINLEN, ZERO, REG3, CAAM_CMD_SZ); 622 zero_assoc_jump_cmd1 = append_jump(desc, JUMP_TEST_ALL | 623 JUMP_COND_MATH_Z); 624 625 append_math_add(desc, VARSEQOUTLEN, ZERO, REG3, CAAM_CMD_SZ); 626 627 /* skip assoc data */ 628 append_seq_fifo_store(desc, 0, FIFOST_TYPE_SKIP | FIFOLDST_VLF); 629 630 /* cryptlen = seqinlen - assoclen */ 631 append_math_sub(desc, VARSEQOUTLEN, SEQINLEN, REG3, CAAM_CMD_SZ); 632 633 /* if cryptlen is ZERO jump to zero-payload commands */ 634 zero_payload_jump_cmd = append_jump(desc, JUMP_TEST_ALL | 635 JUMP_COND_MATH_Z); 636 637 /* read assoc data */ 638 append_seq_fifo_load(desc, 0, FIFOLD_CLASS_CLASS1 | FIFOLDST_VLF | 639 FIFOLD_TYPE_AAD | FIFOLD_TYPE_FLUSH1); 640 set_jump_tgt_here(desc, zero_assoc_jump_cmd1); 641 642 append_math_sub(desc, VARSEQINLEN, SEQINLEN, REG0, CAAM_CMD_SZ); 643 644 /* write encrypted data */ 645 append_seq_fifo_store(desc, 0, FIFOST_TYPE_MESSAGE_DATA | FIFOLDST_VLF); 646 647 /* read payload data */ 648 append_seq_fifo_load(desc, 0, FIFOLD_CLASS_CLASS1 | FIFOLDST_VLF | 649 FIFOLD_TYPE_MSG | FIFOLD_TYPE_LAST1); 650 651 /* jump the zero-payload commands */ 652 append_jump(desc, JUMP_TEST_ALL | 2); 653 654 /* zero-payload commands */ 655 set_jump_tgt_here(desc, zero_payload_jump_cmd); 656 657 /* read assoc data */ 658 append_seq_fifo_load(desc, 0, FIFOLD_CLASS_CLASS1 | FIFOLDST_VLF | 659 FIFOLD_TYPE_AAD | FIFOLD_TYPE_LAST1); 660 661 /* There is no input data */ 662 set_jump_tgt_here(desc, zero_assoc_jump_cmd2); 663 664 /* write ICV */ 665 append_seq_store(desc, icvsize, LDST_CLASS_1_CCB | 666 LDST_SRCDST_BYTE_CONTEXT); 667 668 #ifdef DEBUG 669 print_hex_dump(KERN_ERR, "gcm enc shdesc@" __stringify(__LINE__)": ", 670 DUMP_PREFIX_ADDRESS, 16, 4, desc, desc_bytes(desc), 1); 671 #endif 672 } 673 EXPORT_SYMBOL(cnstr_shdsc_gcm_encap); 674 675 /** 676 * cnstr_shdsc_gcm_decap - gcm decapsulation shared descriptor 677 * @desc: pointer to buffer used for descriptor construction 678 * @cdata: pointer to block cipher transform definitions 679 * Valid algorithm values - OP_ALG_ALGSEL_AES ANDed with OP_ALG_AAI_GCM. 680 * @icvsize: integrity check value (ICV) size (truncated or full) 681 */ 682 void cnstr_shdsc_gcm_decap(u32 * const desc, struct alginfo *cdata, 683 unsigned int icvsize) 684 { 685 u32 *key_jump_cmd, *zero_payload_jump_cmd, *zero_assoc_jump_cmd1; 686 687 init_sh_desc(desc, HDR_SHARE_SERIAL); 688 689 /* skip key loading if they are loaded due to sharing */ 690 key_jump_cmd = append_jump(desc, JUMP_JSL | 691 JUMP_TEST_ALL | JUMP_COND_SHRD); 692 if (cdata->key_inline) 693 append_key_as_imm(desc, cdata->key_virt, cdata->keylen, 694 cdata->keylen, CLASS_1 | KEY_DEST_CLASS_REG); 695 else 696 append_key(desc, cdata->key_dma, cdata->keylen, CLASS_1 | 697 KEY_DEST_CLASS_REG); 698 set_jump_tgt_here(desc, key_jump_cmd); 699 700 /* class 1 operation */ 701 append_operation(desc, cdata->algtype | OP_ALG_AS_INITFINAL | 702 OP_ALG_DECRYPT | OP_ALG_ICV_ON); 703 704 /* if assoclen is ZERO, skip reading the assoc data */ 705 append_math_add(desc, VARSEQINLEN, ZERO, REG3, CAAM_CMD_SZ); 706 zero_assoc_jump_cmd1 = append_jump(desc, JUMP_TEST_ALL | 707 JUMP_COND_MATH_Z); 708 709 append_math_add(desc, VARSEQOUTLEN, ZERO, REG3, CAAM_CMD_SZ); 710 711 /* skip assoc data */ 712 append_seq_fifo_store(desc, 0, FIFOST_TYPE_SKIP | FIFOLDST_VLF); 713 714 /* read assoc data */ 715 append_seq_fifo_load(desc, 0, FIFOLD_CLASS_CLASS1 | FIFOLDST_VLF | 716 FIFOLD_TYPE_AAD | FIFOLD_TYPE_FLUSH1); 717 718 set_jump_tgt_here(desc, zero_assoc_jump_cmd1); 719 720 /* cryptlen = seqoutlen - assoclen */ 721 append_math_sub(desc, VARSEQINLEN, SEQOUTLEN, REG0, CAAM_CMD_SZ); 722 723 /* jump to zero-payload command if cryptlen is zero */ 724 zero_payload_jump_cmd = append_jump(desc, JUMP_TEST_ALL | 725 JUMP_COND_MATH_Z); 726 727 append_math_sub(desc, VARSEQOUTLEN, SEQOUTLEN, REG0, CAAM_CMD_SZ); 728 729 /* store encrypted data */ 730 append_seq_fifo_store(desc, 0, FIFOST_TYPE_MESSAGE_DATA | FIFOLDST_VLF); 731 732 /* read payload data */ 733 append_seq_fifo_load(desc, 0, FIFOLD_CLASS_CLASS1 | FIFOLDST_VLF | 734 FIFOLD_TYPE_MSG | FIFOLD_TYPE_FLUSH1); 735 736 /* zero-payload command */ 737 set_jump_tgt_here(desc, zero_payload_jump_cmd); 738 739 /* read ICV */ 740 append_seq_fifo_load(desc, icvsize, FIFOLD_CLASS_CLASS1 | 741 FIFOLD_TYPE_ICV | FIFOLD_TYPE_LAST1); 742 743 #ifdef DEBUG 744 print_hex_dump(KERN_ERR, "gcm dec shdesc@" __stringify(__LINE__)": ", 745 DUMP_PREFIX_ADDRESS, 16, 4, desc, desc_bytes(desc), 1); 746 #endif 747 } 748 EXPORT_SYMBOL(cnstr_shdsc_gcm_decap); 749 750 /** 751 * cnstr_shdsc_rfc4106_encap - IPSec ESP gcm encapsulation shared descriptor 752 * (non-protocol). 753 * @desc: pointer to buffer used for descriptor construction 754 * @cdata: pointer to block cipher transform definitions 755 * Valid algorithm values - OP_ALG_ALGSEL_AES ANDed with OP_ALG_AAI_GCM. 756 * @icvsize: integrity check value (ICV) size (truncated or full) 757 */ 758 void cnstr_shdsc_rfc4106_encap(u32 * const desc, struct alginfo *cdata, 759 unsigned int icvsize) 760 { 761 u32 *key_jump_cmd; 762 763 init_sh_desc(desc, HDR_SHARE_SERIAL); 764 765 /* Skip key loading if it is loaded due to sharing */ 766 key_jump_cmd = append_jump(desc, JUMP_JSL | JUMP_TEST_ALL | 767 JUMP_COND_SHRD); 768 if (cdata->key_inline) 769 append_key_as_imm(desc, cdata->key_virt, cdata->keylen, 770 cdata->keylen, CLASS_1 | KEY_DEST_CLASS_REG); 771 else 772 append_key(desc, cdata->key_dma, cdata->keylen, CLASS_1 | 773 KEY_DEST_CLASS_REG); 774 set_jump_tgt_here(desc, key_jump_cmd); 775 776 /* Class 1 operation */ 777 append_operation(desc, cdata->algtype | OP_ALG_AS_INITFINAL | 778 OP_ALG_ENCRYPT); 779 780 append_math_sub_imm_u32(desc, VARSEQINLEN, REG3, IMM, 8); 781 append_math_add(desc, VARSEQOUTLEN, ZERO, REG3, CAAM_CMD_SZ); 782 783 /* Read assoc data */ 784 append_seq_fifo_load(desc, 0, FIFOLD_CLASS_CLASS1 | FIFOLDST_VLF | 785 FIFOLD_TYPE_AAD | FIFOLD_TYPE_FLUSH1); 786 787 /* Skip IV */ 788 append_seq_fifo_load(desc, 8, FIFOLD_CLASS_SKIP); 789 790 /* Will read cryptlen bytes */ 791 append_math_sub(desc, VARSEQINLEN, SEQINLEN, REG0, CAAM_CMD_SZ); 792 793 /* Workaround for erratum A-005473 (simultaneous SEQ FIFO skips) */ 794 append_seq_fifo_load(desc, 0, FIFOLD_CLASS_CLASS1 | FIFOLD_TYPE_MSG); 795 796 /* Skip assoc data */ 797 append_seq_fifo_store(desc, 0, FIFOST_TYPE_SKIP | FIFOLDST_VLF); 798 799 /* cryptlen = seqoutlen - assoclen */ 800 append_math_sub(desc, VARSEQOUTLEN, VARSEQINLEN, REG0, CAAM_CMD_SZ); 801 802 /* Write encrypted data */ 803 append_seq_fifo_store(desc, 0, FIFOST_TYPE_MESSAGE_DATA | FIFOLDST_VLF); 804 805 /* Read payload data */ 806 append_seq_fifo_load(desc, 0, FIFOLD_CLASS_CLASS1 | FIFOLDST_VLF | 807 FIFOLD_TYPE_MSG | FIFOLD_TYPE_LAST1); 808 809 /* Write ICV */ 810 append_seq_store(desc, icvsize, LDST_CLASS_1_CCB | 811 LDST_SRCDST_BYTE_CONTEXT); 812 813 #ifdef DEBUG 814 print_hex_dump(KERN_ERR, 815 "rfc4106 enc shdesc@" __stringify(__LINE__)": ", 816 DUMP_PREFIX_ADDRESS, 16, 4, desc, desc_bytes(desc), 1); 817 #endif 818 } 819 EXPORT_SYMBOL(cnstr_shdsc_rfc4106_encap); 820 821 /** 822 * cnstr_shdsc_rfc4106_decap - IPSec ESP gcm decapsulation shared descriptor 823 * (non-protocol). 824 * @desc: pointer to buffer used for descriptor construction 825 * @cdata: pointer to block cipher transform definitions 826 * Valid algorithm values - OP_ALG_ALGSEL_AES ANDed with OP_ALG_AAI_GCM. 827 * @icvsize: integrity check value (ICV) size (truncated or full) 828 */ 829 void cnstr_shdsc_rfc4106_decap(u32 * const desc, struct alginfo *cdata, 830 unsigned int icvsize) 831 { 832 u32 *key_jump_cmd; 833 834 init_sh_desc(desc, HDR_SHARE_SERIAL); 835 836 /* Skip key loading if it is loaded due to sharing */ 837 key_jump_cmd = append_jump(desc, JUMP_JSL | JUMP_TEST_ALL | 838 JUMP_COND_SHRD); 839 if (cdata->key_inline) 840 append_key_as_imm(desc, cdata->key_virt, cdata->keylen, 841 cdata->keylen, CLASS_1 | 842 KEY_DEST_CLASS_REG); 843 else 844 append_key(desc, cdata->key_dma, cdata->keylen, CLASS_1 | 845 KEY_DEST_CLASS_REG); 846 set_jump_tgt_here(desc, key_jump_cmd); 847 848 /* Class 1 operation */ 849 append_operation(desc, cdata->algtype | OP_ALG_AS_INITFINAL | 850 OP_ALG_DECRYPT | OP_ALG_ICV_ON); 851 852 append_math_sub_imm_u32(desc, VARSEQINLEN, REG3, IMM, 8); 853 append_math_add(desc, VARSEQOUTLEN, ZERO, REG3, CAAM_CMD_SZ); 854 855 /* Read assoc data */ 856 append_seq_fifo_load(desc, 0, FIFOLD_CLASS_CLASS1 | FIFOLDST_VLF | 857 FIFOLD_TYPE_AAD | FIFOLD_TYPE_FLUSH1); 858 859 /* Skip IV */ 860 append_seq_fifo_load(desc, 8, FIFOLD_CLASS_SKIP); 861 862 /* Will read cryptlen bytes */ 863 append_math_sub(desc, VARSEQINLEN, SEQOUTLEN, REG3, CAAM_CMD_SZ); 864 865 /* Workaround for erratum A-005473 (simultaneous SEQ FIFO skips) */ 866 append_seq_fifo_load(desc, 0, FIFOLD_CLASS_CLASS1 | FIFOLD_TYPE_MSG); 867 868 /* Skip assoc data */ 869 append_seq_fifo_store(desc, 0, FIFOST_TYPE_SKIP | FIFOLDST_VLF); 870 871 /* Will write cryptlen bytes */ 872 append_math_sub(desc, VARSEQOUTLEN, SEQOUTLEN, REG0, CAAM_CMD_SZ); 873 874 /* Store payload data */ 875 append_seq_fifo_store(desc, 0, FIFOST_TYPE_MESSAGE_DATA | FIFOLDST_VLF); 876 877 /* Read encrypted data */ 878 append_seq_fifo_load(desc, 0, FIFOLD_CLASS_CLASS1 | FIFOLDST_VLF | 879 FIFOLD_TYPE_MSG | FIFOLD_TYPE_FLUSH1); 880 881 /* Read ICV */ 882 append_seq_fifo_load(desc, icvsize, FIFOLD_CLASS_CLASS1 | 883 FIFOLD_TYPE_ICV | FIFOLD_TYPE_LAST1); 884 885 #ifdef DEBUG 886 print_hex_dump(KERN_ERR, 887 "rfc4106 dec shdesc@" __stringify(__LINE__)": ", 888 DUMP_PREFIX_ADDRESS, 16, 4, desc, desc_bytes(desc), 1); 889 #endif 890 } 891 EXPORT_SYMBOL(cnstr_shdsc_rfc4106_decap); 892 893 /** 894 * cnstr_shdsc_rfc4543_encap - IPSec ESP gmac encapsulation shared descriptor 895 * (non-protocol). 896 * @desc: pointer to buffer used for descriptor construction 897 * @cdata: pointer to block cipher transform definitions 898 * Valid algorithm values - OP_ALG_ALGSEL_AES ANDed with OP_ALG_AAI_GCM. 899 * @icvsize: integrity check value (ICV) size (truncated or full) 900 */ 901 void cnstr_shdsc_rfc4543_encap(u32 * const desc, struct alginfo *cdata, 902 unsigned int icvsize) 903 { 904 u32 *key_jump_cmd, *read_move_cmd, *write_move_cmd; 905 906 init_sh_desc(desc, HDR_SHARE_SERIAL); 907 908 /* Skip key loading if it is loaded due to sharing */ 909 key_jump_cmd = append_jump(desc, JUMP_JSL | JUMP_TEST_ALL | 910 JUMP_COND_SHRD); 911 if (cdata->key_inline) 912 append_key_as_imm(desc, cdata->key_virt, cdata->keylen, 913 cdata->keylen, CLASS_1 | KEY_DEST_CLASS_REG); 914 else 915 append_key(desc, cdata->key_dma, cdata->keylen, CLASS_1 | 916 KEY_DEST_CLASS_REG); 917 set_jump_tgt_here(desc, key_jump_cmd); 918 919 /* Class 1 operation */ 920 append_operation(desc, cdata->algtype | OP_ALG_AS_INITFINAL | 921 OP_ALG_ENCRYPT); 922 923 /* assoclen + cryptlen = seqinlen */ 924 append_math_sub(desc, REG3, SEQINLEN, REG0, CAAM_CMD_SZ); 925 926 /* 927 * MOVE_LEN opcode is not available in all SEC HW revisions, 928 * thus need to do some magic, i.e. self-patch the descriptor 929 * buffer. 930 */ 931 read_move_cmd = append_move(desc, MOVE_SRC_DESCBUF | MOVE_DEST_MATH3 | 932 (0x6 << MOVE_LEN_SHIFT)); 933 write_move_cmd = append_move(desc, MOVE_SRC_MATH3 | MOVE_DEST_DESCBUF | 934 (0x8 << MOVE_LEN_SHIFT)); 935 936 /* Will read assoclen + cryptlen bytes */ 937 append_math_sub(desc, VARSEQINLEN, SEQINLEN, REG0, CAAM_CMD_SZ); 938 939 /* Will write assoclen + cryptlen bytes */ 940 append_math_sub(desc, VARSEQOUTLEN, SEQINLEN, REG0, CAAM_CMD_SZ); 941 942 /* Read and write assoclen + cryptlen bytes */ 943 aead_append_src_dst(desc, FIFOLD_TYPE_AAD); 944 945 set_move_tgt_here(desc, read_move_cmd); 946 set_move_tgt_here(desc, write_move_cmd); 947 append_cmd(desc, CMD_LOAD | DISABLE_AUTO_INFO_FIFO); 948 /* Move payload data to OFIFO */ 949 append_move(desc, MOVE_SRC_INFIFO_CL | MOVE_DEST_OUTFIFO); 950 951 /* Write ICV */ 952 append_seq_store(desc, icvsize, LDST_CLASS_1_CCB | 953 LDST_SRCDST_BYTE_CONTEXT); 954 955 #ifdef DEBUG 956 print_hex_dump(KERN_ERR, 957 "rfc4543 enc shdesc@" __stringify(__LINE__)": ", 958 DUMP_PREFIX_ADDRESS, 16, 4, desc, desc_bytes(desc), 1); 959 #endif 960 } 961 EXPORT_SYMBOL(cnstr_shdsc_rfc4543_encap); 962 963 /** 964 * cnstr_shdsc_rfc4543_decap - IPSec ESP gmac decapsulation shared descriptor 965 * (non-protocol). 966 * @desc: pointer to buffer used for descriptor construction 967 * @cdata: pointer to block cipher transform definitions 968 * Valid algorithm values - OP_ALG_ALGSEL_AES ANDed with OP_ALG_AAI_GCM. 969 * @icvsize: integrity check value (ICV) size (truncated or full) 970 */ 971 void cnstr_shdsc_rfc4543_decap(u32 * const desc, struct alginfo *cdata, 972 unsigned int icvsize) 973 { 974 u32 *key_jump_cmd, *read_move_cmd, *write_move_cmd; 975 976 init_sh_desc(desc, HDR_SHARE_SERIAL); 977 978 /* Skip key loading if it is loaded due to sharing */ 979 key_jump_cmd = append_jump(desc, JUMP_JSL | JUMP_TEST_ALL | 980 JUMP_COND_SHRD); 981 if (cdata->key_inline) 982 append_key_as_imm(desc, cdata->key_virt, cdata->keylen, 983 cdata->keylen, CLASS_1 | KEY_DEST_CLASS_REG); 984 else 985 append_key(desc, cdata->key_dma, cdata->keylen, CLASS_1 | 986 KEY_DEST_CLASS_REG); 987 set_jump_tgt_here(desc, key_jump_cmd); 988 989 /* Class 1 operation */ 990 append_operation(desc, cdata->algtype | OP_ALG_AS_INITFINAL | 991 OP_ALG_DECRYPT | OP_ALG_ICV_ON); 992 993 /* assoclen + cryptlen = seqoutlen */ 994 append_math_sub(desc, REG3, SEQOUTLEN, REG0, CAAM_CMD_SZ); 995 996 /* 997 * MOVE_LEN opcode is not available in all SEC HW revisions, 998 * thus need to do some magic, i.e. self-patch the descriptor 999 * buffer. 1000 */ 1001 read_move_cmd = append_move(desc, MOVE_SRC_DESCBUF | MOVE_DEST_MATH3 | 1002 (0x6 << MOVE_LEN_SHIFT)); 1003 write_move_cmd = append_move(desc, MOVE_SRC_MATH3 | MOVE_DEST_DESCBUF | 1004 (0x8 << MOVE_LEN_SHIFT)); 1005 1006 /* Will read assoclen + cryptlen bytes */ 1007 append_math_sub(desc, VARSEQINLEN, SEQOUTLEN, REG0, CAAM_CMD_SZ); 1008 1009 /* Will write assoclen + cryptlen bytes */ 1010 append_math_sub(desc, VARSEQOUTLEN, SEQOUTLEN, REG0, CAAM_CMD_SZ); 1011 1012 /* Store payload data */ 1013 append_seq_fifo_store(desc, 0, FIFOST_TYPE_MESSAGE_DATA | FIFOLDST_VLF); 1014 1015 /* In-snoop assoclen + cryptlen data */ 1016 append_seq_fifo_load(desc, 0, FIFOLD_CLASS_BOTH | FIFOLDST_VLF | 1017 FIFOLD_TYPE_AAD | FIFOLD_TYPE_LAST2FLUSH1); 1018 1019 set_move_tgt_here(desc, read_move_cmd); 1020 set_move_tgt_here(desc, write_move_cmd); 1021 append_cmd(desc, CMD_LOAD | DISABLE_AUTO_INFO_FIFO); 1022 /* Move payload data to OFIFO */ 1023 append_move(desc, MOVE_SRC_INFIFO_CL | MOVE_DEST_OUTFIFO); 1024 append_cmd(desc, CMD_LOAD | ENABLE_AUTO_INFO_FIFO); 1025 1026 /* Read ICV */ 1027 append_seq_fifo_load(desc, icvsize, FIFOLD_CLASS_CLASS1 | 1028 FIFOLD_TYPE_ICV | FIFOLD_TYPE_LAST1); 1029 1030 #ifdef DEBUG 1031 print_hex_dump(KERN_ERR, 1032 "rfc4543 dec shdesc@" __stringify(__LINE__)": ", 1033 DUMP_PREFIX_ADDRESS, 16, 4, desc, desc_bytes(desc), 1); 1034 #endif 1035 } 1036 EXPORT_SYMBOL(cnstr_shdsc_rfc4543_decap); 1037 1038 /* 1039 * For ablkcipher encrypt and decrypt, read from req->src and 1040 * write to req->dst 1041 */ 1042 static inline void ablkcipher_append_src_dst(u32 *desc) 1043 { 1044 append_math_add(desc, VARSEQOUTLEN, SEQINLEN, REG0, CAAM_CMD_SZ); 1045 append_math_add(desc, VARSEQINLEN, SEQINLEN, REG0, CAAM_CMD_SZ); 1046 append_seq_fifo_load(desc, 0, FIFOLD_CLASS_CLASS1 | 1047 KEY_VLF | FIFOLD_TYPE_MSG | FIFOLD_TYPE_LAST1); 1048 append_seq_fifo_store(desc, 0, FIFOST_TYPE_MESSAGE_DATA | KEY_VLF); 1049 } 1050 1051 /** 1052 * cnstr_shdsc_ablkcipher_encap - ablkcipher encapsulation shared descriptor 1053 * @desc: pointer to buffer used for descriptor construction 1054 * @cdata: pointer to block cipher transform definitions 1055 * Valid algorithm values - one of OP_ALG_ALGSEL_{AES, DES, 3DES} ANDed 1056 * with OP_ALG_AAI_CBC or OP_ALG_AAI_CTR_MOD128. 1057 * @ivsize: initialization vector size 1058 * @is_rfc3686: true when ctr(aes) is wrapped by rfc3686 template 1059 * @ctx1_iv_off: IV offset in CONTEXT1 register 1060 */ 1061 void cnstr_shdsc_ablkcipher_encap(u32 * const desc, struct alginfo *cdata, 1062 unsigned int ivsize, const bool is_rfc3686, 1063 const u32 ctx1_iv_off) 1064 { 1065 u32 *key_jump_cmd; 1066 1067 init_sh_desc(desc, HDR_SHARE_SERIAL | HDR_SAVECTX); 1068 /* Skip if already shared */ 1069 key_jump_cmd = append_jump(desc, JUMP_JSL | JUMP_TEST_ALL | 1070 JUMP_COND_SHRD); 1071 1072 /* Load class1 key only */ 1073 append_key_as_imm(desc, cdata->key_virt, cdata->keylen, 1074 cdata->keylen, CLASS_1 | KEY_DEST_CLASS_REG); 1075 1076 /* Load nonce into CONTEXT1 reg */ 1077 if (is_rfc3686) { 1078 u8 *nonce = cdata->key_virt + cdata->keylen; 1079 1080 append_load_as_imm(desc, nonce, CTR_RFC3686_NONCE_SIZE, 1081 LDST_CLASS_IND_CCB | 1082 LDST_SRCDST_BYTE_OUTFIFO | LDST_IMM); 1083 append_move(desc, MOVE_WAITCOMP | MOVE_SRC_OUTFIFO | 1084 MOVE_DEST_CLASS1CTX | (16 << MOVE_OFFSET_SHIFT) | 1085 (CTR_RFC3686_NONCE_SIZE << MOVE_LEN_SHIFT)); 1086 } 1087 1088 set_jump_tgt_here(desc, key_jump_cmd); 1089 1090 /* Load iv */ 1091 append_seq_load(desc, ivsize, LDST_SRCDST_BYTE_CONTEXT | 1092 LDST_CLASS_1_CCB | (ctx1_iv_off << LDST_OFFSET_SHIFT)); 1093 1094 /* Load counter into CONTEXT1 reg */ 1095 if (is_rfc3686) 1096 append_load_imm_be32(desc, 1, LDST_IMM | LDST_CLASS_1_CCB | 1097 LDST_SRCDST_BYTE_CONTEXT | 1098 ((ctx1_iv_off + CTR_RFC3686_IV_SIZE) << 1099 LDST_OFFSET_SHIFT)); 1100 1101 /* Load operation */ 1102 append_operation(desc, cdata->algtype | OP_ALG_AS_INITFINAL | 1103 OP_ALG_ENCRYPT); 1104 1105 /* Perform operation */ 1106 ablkcipher_append_src_dst(desc); 1107 1108 #ifdef DEBUG 1109 print_hex_dump(KERN_ERR, 1110 "ablkcipher enc shdesc@" __stringify(__LINE__)": ", 1111 DUMP_PREFIX_ADDRESS, 16, 4, desc, desc_bytes(desc), 1); 1112 #endif 1113 } 1114 EXPORT_SYMBOL(cnstr_shdsc_ablkcipher_encap); 1115 1116 /** 1117 * cnstr_shdsc_ablkcipher_decap - ablkcipher decapsulation shared descriptor 1118 * @desc: pointer to buffer used for descriptor construction 1119 * @cdata: pointer to block cipher transform definitions 1120 * Valid algorithm values - one of OP_ALG_ALGSEL_{AES, DES, 3DES} ANDed 1121 * with OP_ALG_AAI_CBC or OP_ALG_AAI_CTR_MOD128. 1122 * @ivsize: initialization vector size 1123 * @is_rfc3686: true when ctr(aes) is wrapped by rfc3686 template 1124 * @ctx1_iv_off: IV offset in CONTEXT1 register 1125 */ 1126 void cnstr_shdsc_ablkcipher_decap(u32 * const desc, struct alginfo *cdata, 1127 unsigned int ivsize, const bool is_rfc3686, 1128 const u32 ctx1_iv_off) 1129 { 1130 u32 *key_jump_cmd; 1131 1132 init_sh_desc(desc, HDR_SHARE_SERIAL | HDR_SAVECTX); 1133 /* Skip if already shared */ 1134 key_jump_cmd = append_jump(desc, JUMP_JSL | JUMP_TEST_ALL | 1135 JUMP_COND_SHRD); 1136 1137 /* Load class1 key only */ 1138 append_key_as_imm(desc, cdata->key_virt, cdata->keylen, 1139 cdata->keylen, CLASS_1 | KEY_DEST_CLASS_REG); 1140 1141 /* Load nonce into CONTEXT1 reg */ 1142 if (is_rfc3686) { 1143 u8 *nonce = cdata->key_virt + cdata->keylen; 1144 1145 append_load_as_imm(desc, nonce, CTR_RFC3686_NONCE_SIZE, 1146 LDST_CLASS_IND_CCB | 1147 LDST_SRCDST_BYTE_OUTFIFO | LDST_IMM); 1148 append_move(desc, MOVE_WAITCOMP | MOVE_SRC_OUTFIFO | 1149 MOVE_DEST_CLASS1CTX | (16 << MOVE_OFFSET_SHIFT) | 1150 (CTR_RFC3686_NONCE_SIZE << MOVE_LEN_SHIFT)); 1151 } 1152 1153 set_jump_tgt_here(desc, key_jump_cmd); 1154 1155 /* load IV */ 1156 append_seq_load(desc, ivsize, LDST_SRCDST_BYTE_CONTEXT | 1157 LDST_CLASS_1_CCB | (ctx1_iv_off << LDST_OFFSET_SHIFT)); 1158 1159 /* Load counter into CONTEXT1 reg */ 1160 if (is_rfc3686) 1161 append_load_imm_be32(desc, 1, LDST_IMM | LDST_CLASS_1_CCB | 1162 LDST_SRCDST_BYTE_CONTEXT | 1163 ((ctx1_iv_off + CTR_RFC3686_IV_SIZE) << 1164 LDST_OFFSET_SHIFT)); 1165 1166 /* Choose operation */ 1167 if (ctx1_iv_off) 1168 append_operation(desc, cdata->algtype | OP_ALG_AS_INITFINAL | 1169 OP_ALG_DECRYPT); 1170 else 1171 append_dec_op1(desc, cdata->algtype); 1172 1173 /* Perform operation */ 1174 ablkcipher_append_src_dst(desc); 1175 1176 #ifdef DEBUG 1177 print_hex_dump(KERN_ERR, 1178 "ablkcipher dec shdesc@" __stringify(__LINE__)": ", 1179 DUMP_PREFIX_ADDRESS, 16, 4, desc, desc_bytes(desc), 1); 1180 #endif 1181 } 1182 EXPORT_SYMBOL(cnstr_shdsc_ablkcipher_decap); 1183 1184 /** 1185 * cnstr_shdsc_ablkcipher_givencap - ablkcipher encapsulation shared descriptor 1186 * with HW-generated initialization vector. 1187 * @desc: pointer to buffer used for descriptor construction 1188 * @cdata: pointer to block cipher transform definitions 1189 * Valid algorithm values - one of OP_ALG_ALGSEL_{AES, DES, 3DES} ANDed 1190 * with OP_ALG_AAI_CBC. 1191 * @ivsize: initialization vector size 1192 * @is_rfc3686: true when ctr(aes) is wrapped by rfc3686 template 1193 * @ctx1_iv_off: IV offset in CONTEXT1 register 1194 */ 1195 void cnstr_shdsc_ablkcipher_givencap(u32 * const desc, struct alginfo *cdata, 1196 unsigned int ivsize, const bool is_rfc3686, 1197 const u32 ctx1_iv_off) 1198 { 1199 u32 *key_jump_cmd, geniv; 1200 1201 init_sh_desc(desc, HDR_SHARE_SERIAL | HDR_SAVECTX); 1202 /* Skip if already shared */ 1203 key_jump_cmd = append_jump(desc, JUMP_JSL | JUMP_TEST_ALL | 1204 JUMP_COND_SHRD); 1205 1206 /* Load class1 key only */ 1207 append_key_as_imm(desc, cdata->key_virt, cdata->keylen, 1208 cdata->keylen, CLASS_1 | KEY_DEST_CLASS_REG); 1209 1210 /* Load Nonce into CONTEXT1 reg */ 1211 if (is_rfc3686) { 1212 u8 *nonce = cdata->key_virt + cdata->keylen; 1213 1214 append_load_as_imm(desc, nonce, CTR_RFC3686_NONCE_SIZE, 1215 LDST_CLASS_IND_CCB | 1216 LDST_SRCDST_BYTE_OUTFIFO | LDST_IMM); 1217 append_move(desc, MOVE_WAITCOMP | MOVE_SRC_OUTFIFO | 1218 MOVE_DEST_CLASS1CTX | (16 << MOVE_OFFSET_SHIFT) | 1219 (CTR_RFC3686_NONCE_SIZE << MOVE_LEN_SHIFT)); 1220 } 1221 set_jump_tgt_here(desc, key_jump_cmd); 1222 1223 /* Generate IV */ 1224 geniv = NFIFOENTRY_STYPE_PAD | NFIFOENTRY_DEST_DECO | 1225 NFIFOENTRY_DTYPE_MSG | NFIFOENTRY_LC1 | NFIFOENTRY_PTYPE_RND | 1226 (ivsize << NFIFOENTRY_DLEN_SHIFT); 1227 append_load_imm_u32(desc, geniv, LDST_CLASS_IND_CCB | 1228 LDST_SRCDST_WORD_INFO_FIFO | LDST_IMM); 1229 append_cmd(desc, CMD_LOAD | DISABLE_AUTO_INFO_FIFO); 1230 append_move(desc, MOVE_WAITCOMP | MOVE_SRC_INFIFO | 1231 MOVE_DEST_CLASS1CTX | (ivsize << MOVE_LEN_SHIFT) | 1232 (ctx1_iv_off << MOVE_OFFSET_SHIFT)); 1233 append_cmd(desc, CMD_LOAD | ENABLE_AUTO_INFO_FIFO); 1234 1235 /* Copy generated IV to memory */ 1236 append_seq_store(desc, ivsize, LDST_SRCDST_BYTE_CONTEXT | 1237 LDST_CLASS_1_CCB | (ctx1_iv_off << LDST_OFFSET_SHIFT)); 1238 1239 /* Load Counter into CONTEXT1 reg */ 1240 if (is_rfc3686) 1241 append_load_imm_be32(desc, 1, LDST_IMM | LDST_CLASS_1_CCB | 1242 LDST_SRCDST_BYTE_CONTEXT | 1243 ((ctx1_iv_off + CTR_RFC3686_IV_SIZE) << 1244 LDST_OFFSET_SHIFT)); 1245 1246 if (ctx1_iv_off) 1247 append_jump(desc, JUMP_JSL | JUMP_TEST_ALL | JUMP_COND_NCP | 1248 (1 << JUMP_OFFSET_SHIFT)); 1249 1250 /* Load operation */ 1251 append_operation(desc, cdata->algtype | OP_ALG_AS_INITFINAL | 1252 OP_ALG_ENCRYPT); 1253 1254 /* Perform operation */ 1255 ablkcipher_append_src_dst(desc); 1256 1257 #ifdef DEBUG 1258 print_hex_dump(KERN_ERR, 1259 "ablkcipher givenc shdesc@" __stringify(__LINE__) ": ", 1260 DUMP_PREFIX_ADDRESS, 16, 4, desc, desc_bytes(desc), 1); 1261 #endif 1262 } 1263 EXPORT_SYMBOL(cnstr_shdsc_ablkcipher_givencap); 1264 1265 /** 1266 * cnstr_shdsc_xts_ablkcipher_encap - xts ablkcipher encapsulation shared 1267 * descriptor 1268 * @desc: pointer to buffer used for descriptor construction 1269 * @cdata: pointer to block cipher transform definitions 1270 * Valid algorithm values - OP_ALG_ALGSEL_AES ANDed with OP_ALG_AAI_XTS. 1271 */ 1272 void cnstr_shdsc_xts_ablkcipher_encap(u32 * const desc, struct alginfo *cdata) 1273 { 1274 __be64 sector_size = cpu_to_be64(512); 1275 u32 *key_jump_cmd; 1276 1277 init_sh_desc(desc, HDR_SHARE_SERIAL | HDR_SAVECTX); 1278 /* Skip if already shared */ 1279 key_jump_cmd = append_jump(desc, JUMP_JSL | JUMP_TEST_ALL | 1280 JUMP_COND_SHRD); 1281 1282 /* Load class1 keys only */ 1283 append_key_as_imm(desc, cdata->key_virt, cdata->keylen, 1284 cdata->keylen, CLASS_1 | KEY_DEST_CLASS_REG); 1285 1286 /* Load sector size with index 40 bytes (0x28) */ 1287 append_load_as_imm(desc, (void *)§or_size, 8, LDST_CLASS_1_CCB | 1288 LDST_SRCDST_BYTE_CONTEXT | 1289 (0x28 << LDST_OFFSET_SHIFT)); 1290 1291 set_jump_tgt_here(desc, key_jump_cmd); 1292 1293 /* 1294 * create sequence for loading the sector index 1295 * Upper 8B of IV - will be used as sector index 1296 * Lower 8B of IV - will be discarded 1297 */ 1298 append_seq_load(desc, 8, LDST_SRCDST_BYTE_CONTEXT | LDST_CLASS_1_CCB | 1299 (0x20 << LDST_OFFSET_SHIFT)); 1300 append_seq_fifo_load(desc, 8, FIFOLD_CLASS_SKIP); 1301 1302 /* Load operation */ 1303 append_operation(desc, cdata->algtype | OP_ALG_AS_INITFINAL | 1304 OP_ALG_ENCRYPT); 1305 1306 /* Perform operation */ 1307 ablkcipher_append_src_dst(desc); 1308 1309 #ifdef DEBUG 1310 print_hex_dump(KERN_ERR, 1311 "xts ablkcipher enc shdesc@" __stringify(__LINE__) ": ", 1312 DUMP_PREFIX_ADDRESS, 16, 4, desc, desc_bytes(desc), 1); 1313 #endif 1314 } 1315 EXPORT_SYMBOL(cnstr_shdsc_xts_ablkcipher_encap); 1316 1317 /** 1318 * cnstr_shdsc_xts_ablkcipher_decap - xts ablkcipher decapsulation shared 1319 * descriptor 1320 * @desc: pointer to buffer used for descriptor construction 1321 * @cdata: pointer to block cipher transform definitions 1322 * Valid algorithm values - OP_ALG_ALGSEL_AES ANDed with OP_ALG_AAI_XTS. 1323 */ 1324 void cnstr_shdsc_xts_ablkcipher_decap(u32 * const desc, struct alginfo *cdata) 1325 { 1326 __be64 sector_size = cpu_to_be64(512); 1327 u32 *key_jump_cmd; 1328 1329 init_sh_desc(desc, HDR_SHARE_SERIAL | HDR_SAVECTX); 1330 /* Skip if already shared */ 1331 key_jump_cmd = append_jump(desc, JUMP_JSL | JUMP_TEST_ALL | 1332 JUMP_COND_SHRD); 1333 1334 /* Load class1 key only */ 1335 append_key_as_imm(desc, cdata->key_virt, cdata->keylen, 1336 cdata->keylen, CLASS_1 | KEY_DEST_CLASS_REG); 1337 1338 /* Load sector size with index 40 bytes (0x28) */ 1339 append_load_as_imm(desc, (void *)§or_size, 8, LDST_CLASS_1_CCB | 1340 LDST_SRCDST_BYTE_CONTEXT | 1341 (0x28 << LDST_OFFSET_SHIFT)); 1342 1343 set_jump_tgt_here(desc, key_jump_cmd); 1344 1345 /* 1346 * create sequence for loading the sector index 1347 * Upper 8B of IV - will be used as sector index 1348 * Lower 8B of IV - will be discarded 1349 */ 1350 append_seq_load(desc, 8, LDST_SRCDST_BYTE_CONTEXT | LDST_CLASS_1_CCB | 1351 (0x20 << LDST_OFFSET_SHIFT)); 1352 append_seq_fifo_load(desc, 8, FIFOLD_CLASS_SKIP); 1353 1354 /* Load operation */ 1355 append_dec_op1(desc, cdata->algtype); 1356 1357 /* Perform operation */ 1358 ablkcipher_append_src_dst(desc); 1359 1360 #ifdef DEBUG 1361 print_hex_dump(KERN_ERR, 1362 "xts ablkcipher dec shdesc@" __stringify(__LINE__) ": ", 1363 DUMP_PREFIX_ADDRESS, 16, 4, desc, desc_bytes(desc), 1); 1364 #endif 1365 } 1366 EXPORT_SYMBOL(cnstr_shdsc_xts_ablkcipher_decap); 1367 1368 MODULE_LICENSE("GPL"); 1369 MODULE_DESCRIPTION("FSL CAAM descriptor support"); 1370 MODULE_AUTHOR("Freescale Semiconductor - NMG/STC"); 1371