1 /* 2 * Shared descriptors for aead, ablkcipher algorithms 3 * 4 * Copyright 2016 NXP 5 */ 6 7 #include "compat.h" 8 #include "desc_constr.h" 9 #include "caamalg_desc.h" 10 11 /* 12 * For aead functions, read payload and write payload, 13 * both of which are specified in req->src and req->dst 14 */ 15 static inline void aead_append_src_dst(u32 *desc, u32 msg_type) 16 { 17 append_seq_fifo_store(desc, 0, FIFOST_TYPE_MESSAGE_DATA | KEY_VLF); 18 append_seq_fifo_load(desc, 0, FIFOLD_CLASS_BOTH | 19 KEY_VLF | msg_type | FIFOLD_TYPE_LASTBOTH); 20 } 21 22 /* Set DK bit in class 1 operation if shared */ 23 static inline void append_dec_op1(u32 *desc, u32 type) 24 { 25 u32 *jump_cmd, *uncond_jump_cmd; 26 27 /* DK bit is valid only for AES */ 28 if ((type & OP_ALG_ALGSEL_MASK) != OP_ALG_ALGSEL_AES) { 29 append_operation(desc, type | OP_ALG_AS_INITFINAL | 30 OP_ALG_DECRYPT); 31 return; 32 } 33 34 jump_cmd = append_jump(desc, JUMP_TEST_ALL | JUMP_COND_SHRD); 35 append_operation(desc, type | OP_ALG_AS_INITFINAL | 36 OP_ALG_DECRYPT); 37 uncond_jump_cmd = append_jump(desc, JUMP_TEST_ALL); 38 set_jump_tgt_here(desc, jump_cmd); 39 append_operation(desc, type | OP_ALG_AS_INITFINAL | 40 OP_ALG_DECRYPT | OP_ALG_AAI_DK); 41 set_jump_tgt_here(desc, uncond_jump_cmd); 42 } 43 44 /** 45 * cnstr_shdsc_aead_null_encap - IPSec ESP encapsulation shared descriptor 46 * (non-protocol) with no (null) encryption. 47 * @desc: pointer to buffer used for descriptor construction 48 * @adata: pointer to authentication transform definitions. 49 * A split key is required for SEC Era < 6; the size of the split key 50 * is specified in this case. Valid algorithm values - one of 51 * OP_ALG_ALGSEL_{MD5, SHA1, SHA224, SHA256, SHA384, SHA512} ANDed 52 * with OP_ALG_AAI_HMAC_PRECOMP. 53 * @icvsize: integrity check value (ICV) size (truncated or full) 54 * @era: SEC Era 55 */ 56 void cnstr_shdsc_aead_null_encap(u32 * const desc, struct alginfo *adata, 57 unsigned int icvsize, int era) 58 { 59 u32 *key_jump_cmd, *read_move_cmd, *write_move_cmd; 60 61 init_sh_desc(desc, HDR_SHARE_SERIAL); 62 63 /* Skip if already shared */ 64 key_jump_cmd = append_jump(desc, JUMP_JSL | JUMP_TEST_ALL | 65 JUMP_COND_SHRD); 66 if (era < 6) { 67 if (adata->key_inline) 68 append_key_as_imm(desc, adata->key_virt, 69 adata->keylen_pad, adata->keylen, 70 CLASS_2 | KEY_DEST_MDHA_SPLIT | 71 KEY_ENC); 72 else 73 append_key(desc, adata->key_dma, adata->keylen, 74 CLASS_2 | KEY_DEST_MDHA_SPLIT | KEY_ENC); 75 } else { 76 append_proto_dkp(desc, adata); 77 } 78 set_jump_tgt_here(desc, key_jump_cmd); 79 80 /* assoclen + cryptlen = seqinlen */ 81 append_math_sub(desc, REG3, SEQINLEN, REG0, CAAM_CMD_SZ); 82 83 /* Prepare to read and write cryptlen + assoclen bytes */ 84 append_math_add(desc, VARSEQINLEN, ZERO, REG3, CAAM_CMD_SZ); 85 append_math_add(desc, VARSEQOUTLEN, ZERO, REG3, CAAM_CMD_SZ); 86 87 /* 88 * MOVE_LEN opcode is not available in all SEC HW revisions, 89 * thus need to do some magic, i.e. self-patch the descriptor 90 * buffer. 91 */ 92 read_move_cmd = append_move(desc, MOVE_SRC_DESCBUF | 93 MOVE_DEST_MATH3 | 94 (0x6 << MOVE_LEN_SHIFT)); 95 write_move_cmd = append_move(desc, MOVE_SRC_MATH3 | 96 MOVE_DEST_DESCBUF | 97 MOVE_WAITCOMP | 98 (0x8 << MOVE_LEN_SHIFT)); 99 100 /* Class 2 operation */ 101 append_operation(desc, adata->algtype | OP_ALG_AS_INITFINAL | 102 OP_ALG_ENCRYPT); 103 104 /* Read and write cryptlen bytes */ 105 aead_append_src_dst(desc, FIFOLD_TYPE_MSG | FIFOLD_TYPE_FLUSH1); 106 107 set_move_tgt_here(desc, read_move_cmd); 108 set_move_tgt_here(desc, write_move_cmd); 109 append_cmd(desc, CMD_LOAD | DISABLE_AUTO_INFO_FIFO); 110 append_move(desc, MOVE_SRC_INFIFO_CL | MOVE_DEST_OUTFIFO | 111 MOVE_AUX_LS); 112 113 /* Write ICV */ 114 append_seq_store(desc, icvsize, LDST_CLASS_2_CCB | 115 LDST_SRCDST_BYTE_CONTEXT); 116 117 #ifdef DEBUG 118 print_hex_dump(KERN_ERR, 119 "aead null enc shdesc@" __stringify(__LINE__)": ", 120 DUMP_PREFIX_ADDRESS, 16, 4, desc, desc_bytes(desc), 1); 121 #endif 122 } 123 EXPORT_SYMBOL(cnstr_shdsc_aead_null_encap); 124 125 /** 126 * cnstr_shdsc_aead_null_decap - IPSec ESP decapsulation shared descriptor 127 * (non-protocol) with no (null) decryption. 128 * @desc: pointer to buffer used for descriptor construction 129 * @adata: pointer to authentication transform definitions. 130 * A split key is required for SEC Era < 6; the size of the split key 131 * is specified in this case. Valid algorithm values - one of 132 * OP_ALG_ALGSEL_{MD5, SHA1, SHA224, SHA256, SHA384, SHA512} ANDed 133 * with OP_ALG_AAI_HMAC_PRECOMP. 134 * @icvsize: integrity check value (ICV) size (truncated or full) 135 * @era: SEC Era 136 */ 137 void cnstr_shdsc_aead_null_decap(u32 * const desc, struct alginfo *adata, 138 unsigned int icvsize, int era) 139 { 140 u32 *key_jump_cmd, *read_move_cmd, *write_move_cmd, *jump_cmd; 141 142 init_sh_desc(desc, HDR_SHARE_SERIAL); 143 144 /* Skip if already shared */ 145 key_jump_cmd = append_jump(desc, JUMP_JSL | JUMP_TEST_ALL | 146 JUMP_COND_SHRD); 147 if (era < 6) { 148 if (adata->key_inline) 149 append_key_as_imm(desc, adata->key_virt, 150 adata->keylen_pad, adata->keylen, 151 CLASS_2 | KEY_DEST_MDHA_SPLIT | 152 KEY_ENC); 153 else 154 append_key(desc, adata->key_dma, adata->keylen, 155 CLASS_2 | KEY_DEST_MDHA_SPLIT | KEY_ENC); 156 } else { 157 append_proto_dkp(desc, adata); 158 } 159 set_jump_tgt_here(desc, key_jump_cmd); 160 161 /* Class 2 operation */ 162 append_operation(desc, adata->algtype | OP_ALG_AS_INITFINAL | 163 OP_ALG_DECRYPT | OP_ALG_ICV_ON); 164 165 /* assoclen + cryptlen = seqoutlen */ 166 append_math_sub(desc, REG2, SEQOUTLEN, REG0, CAAM_CMD_SZ); 167 168 /* Prepare to read and write cryptlen + assoclen bytes */ 169 append_math_add(desc, VARSEQINLEN, ZERO, REG2, CAAM_CMD_SZ); 170 append_math_add(desc, VARSEQOUTLEN, ZERO, REG2, CAAM_CMD_SZ); 171 172 /* 173 * MOVE_LEN opcode is not available in all SEC HW revisions, 174 * thus need to do some magic, i.e. self-patch the descriptor 175 * buffer. 176 */ 177 read_move_cmd = append_move(desc, MOVE_SRC_DESCBUF | 178 MOVE_DEST_MATH2 | 179 (0x6 << MOVE_LEN_SHIFT)); 180 write_move_cmd = append_move(desc, MOVE_SRC_MATH2 | 181 MOVE_DEST_DESCBUF | 182 MOVE_WAITCOMP | 183 (0x8 << MOVE_LEN_SHIFT)); 184 185 /* Read and write cryptlen bytes */ 186 aead_append_src_dst(desc, FIFOLD_TYPE_MSG | FIFOLD_TYPE_FLUSH1); 187 188 /* 189 * Insert a NOP here, since we need at least 4 instructions between 190 * code patching the descriptor buffer and the location being patched. 191 */ 192 jump_cmd = append_jump(desc, JUMP_TEST_ALL); 193 set_jump_tgt_here(desc, jump_cmd); 194 195 set_move_tgt_here(desc, read_move_cmd); 196 set_move_tgt_here(desc, write_move_cmd); 197 append_cmd(desc, CMD_LOAD | DISABLE_AUTO_INFO_FIFO); 198 append_move(desc, MOVE_SRC_INFIFO_CL | MOVE_DEST_OUTFIFO | 199 MOVE_AUX_LS); 200 append_cmd(desc, CMD_LOAD | ENABLE_AUTO_INFO_FIFO); 201 202 /* Load ICV */ 203 append_seq_fifo_load(desc, icvsize, FIFOLD_CLASS_CLASS2 | 204 FIFOLD_TYPE_LAST2 | FIFOLD_TYPE_ICV); 205 206 #ifdef DEBUG 207 print_hex_dump(KERN_ERR, 208 "aead null dec shdesc@" __stringify(__LINE__)": ", 209 DUMP_PREFIX_ADDRESS, 16, 4, desc, desc_bytes(desc), 1); 210 #endif 211 } 212 EXPORT_SYMBOL(cnstr_shdsc_aead_null_decap); 213 214 static void init_sh_desc_key_aead(u32 * const desc, 215 struct alginfo * const cdata, 216 struct alginfo * const adata, 217 const bool is_rfc3686, u32 *nonce, int era) 218 { 219 u32 *key_jump_cmd; 220 unsigned int enckeylen = cdata->keylen; 221 222 /* Note: Context registers are saved. */ 223 init_sh_desc(desc, HDR_SHARE_SERIAL | HDR_SAVECTX); 224 225 /* Skip if already shared */ 226 key_jump_cmd = append_jump(desc, JUMP_JSL | JUMP_TEST_ALL | 227 JUMP_COND_SHRD); 228 229 /* 230 * RFC3686 specific: 231 * | key = {AUTH_KEY, ENC_KEY, NONCE} 232 * | enckeylen = encryption key size + nonce size 233 */ 234 if (is_rfc3686) 235 enckeylen -= CTR_RFC3686_NONCE_SIZE; 236 237 if (era < 6) { 238 if (adata->key_inline) 239 append_key_as_imm(desc, adata->key_virt, 240 adata->keylen_pad, adata->keylen, 241 CLASS_2 | KEY_DEST_MDHA_SPLIT | 242 KEY_ENC); 243 else 244 append_key(desc, adata->key_dma, adata->keylen, 245 CLASS_2 | KEY_DEST_MDHA_SPLIT | KEY_ENC); 246 } else { 247 append_proto_dkp(desc, adata); 248 } 249 250 if (cdata->key_inline) 251 append_key_as_imm(desc, cdata->key_virt, enckeylen, 252 enckeylen, CLASS_1 | KEY_DEST_CLASS_REG); 253 else 254 append_key(desc, cdata->key_dma, enckeylen, CLASS_1 | 255 KEY_DEST_CLASS_REG); 256 257 /* Load Counter into CONTEXT1 reg */ 258 if (is_rfc3686) { 259 append_load_as_imm(desc, nonce, CTR_RFC3686_NONCE_SIZE, 260 LDST_CLASS_IND_CCB | 261 LDST_SRCDST_BYTE_OUTFIFO | LDST_IMM); 262 append_move(desc, 263 MOVE_SRC_OUTFIFO | 264 MOVE_DEST_CLASS1CTX | 265 (16 << MOVE_OFFSET_SHIFT) | 266 (CTR_RFC3686_NONCE_SIZE << MOVE_LEN_SHIFT)); 267 } 268 269 set_jump_tgt_here(desc, key_jump_cmd); 270 } 271 272 /** 273 * cnstr_shdsc_aead_encap - IPSec ESP encapsulation shared descriptor 274 * (non-protocol). 275 * @desc: pointer to buffer used for descriptor construction 276 * @cdata: pointer to block cipher transform definitions 277 * Valid algorithm values - one of OP_ALG_ALGSEL_{AES, DES, 3DES} ANDed 278 * with OP_ALG_AAI_CBC or OP_ALG_AAI_CTR_MOD128. 279 * @adata: pointer to authentication transform definitions. 280 * A split key is required for SEC Era < 6; the size of the split key 281 * is specified in this case. Valid algorithm values - one of 282 * OP_ALG_ALGSEL_{MD5, SHA1, SHA224, SHA256, SHA384, SHA512} ANDed 283 * with OP_ALG_AAI_HMAC_PRECOMP. 284 * @ivsize: initialization vector size 285 * @icvsize: integrity check value (ICV) size (truncated or full) 286 * @is_rfc3686: true when ctr(aes) is wrapped by rfc3686 template 287 * @nonce: pointer to rfc3686 nonce 288 * @ctx1_iv_off: IV offset in CONTEXT1 register 289 * @is_qi: true when called from caam/qi 290 * @era: SEC Era 291 */ 292 void cnstr_shdsc_aead_encap(u32 * const desc, struct alginfo *cdata, 293 struct alginfo *adata, unsigned int ivsize, 294 unsigned int icvsize, const bool is_rfc3686, 295 u32 *nonce, const u32 ctx1_iv_off, const bool is_qi, 296 int era) 297 { 298 /* Note: Context registers are saved. */ 299 init_sh_desc_key_aead(desc, cdata, adata, is_rfc3686, nonce, era); 300 301 /* Class 2 operation */ 302 append_operation(desc, adata->algtype | OP_ALG_AS_INITFINAL | 303 OP_ALG_ENCRYPT); 304 305 if (is_qi) { 306 u32 *wait_load_cmd; 307 308 /* REG3 = assoclen */ 309 append_seq_load(desc, 4, LDST_CLASS_DECO | 310 LDST_SRCDST_WORD_DECO_MATH3 | 311 (4 << LDST_OFFSET_SHIFT)); 312 313 wait_load_cmd = append_jump(desc, JUMP_JSL | JUMP_TEST_ALL | 314 JUMP_COND_CALM | JUMP_COND_NCP | 315 JUMP_COND_NOP | JUMP_COND_NIP | 316 JUMP_COND_NIFP); 317 set_jump_tgt_here(desc, wait_load_cmd); 318 319 append_seq_load(desc, ivsize, LDST_CLASS_1_CCB | 320 LDST_SRCDST_BYTE_CONTEXT | 321 (ctx1_iv_off << LDST_OFFSET_SHIFT)); 322 } 323 324 /* Read and write assoclen bytes */ 325 if (is_qi || era < 3) { 326 append_math_add(desc, VARSEQINLEN, ZERO, REG3, CAAM_CMD_SZ); 327 append_math_add(desc, VARSEQOUTLEN, ZERO, REG3, CAAM_CMD_SZ); 328 } else { 329 append_math_add(desc, VARSEQINLEN, ZERO, DPOVRD, CAAM_CMD_SZ); 330 append_math_add(desc, VARSEQOUTLEN, ZERO, DPOVRD, CAAM_CMD_SZ); 331 } 332 333 /* Skip assoc data */ 334 append_seq_fifo_store(desc, 0, FIFOST_TYPE_SKIP | FIFOLDST_VLF); 335 336 /* read assoc before reading payload */ 337 append_seq_fifo_load(desc, 0, FIFOLD_CLASS_CLASS2 | FIFOLD_TYPE_MSG | 338 FIFOLDST_VLF); 339 340 /* Load Counter into CONTEXT1 reg */ 341 if (is_rfc3686) 342 append_load_imm_be32(desc, 1, LDST_IMM | LDST_CLASS_1_CCB | 343 LDST_SRCDST_BYTE_CONTEXT | 344 ((ctx1_iv_off + CTR_RFC3686_IV_SIZE) << 345 LDST_OFFSET_SHIFT)); 346 347 /* Class 1 operation */ 348 append_operation(desc, cdata->algtype | OP_ALG_AS_INITFINAL | 349 OP_ALG_ENCRYPT); 350 351 /* Read and write cryptlen bytes */ 352 append_math_add(desc, VARSEQINLEN, SEQINLEN, REG0, CAAM_CMD_SZ); 353 append_math_add(desc, VARSEQOUTLEN, SEQINLEN, REG0, CAAM_CMD_SZ); 354 aead_append_src_dst(desc, FIFOLD_TYPE_MSG1OUT2); 355 356 /* Write ICV */ 357 append_seq_store(desc, icvsize, LDST_CLASS_2_CCB | 358 LDST_SRCDST_BYTE_CONTEXT); 359 360 #ifdef DEBUG 361 print_hex_dump(KERN_ERR, "aead enc shdesc@" __stringify(__LINE__)": ", 362 DUMP_PREFIX_ADDRESS, 16, 4, desc, desc_bytes(desc), 1); 363 #endif 364 } 365 EXPORT_SYMBOL(cnstr_shdsc_aead_encap); 366 367 /** 368 * cnstr_shdsc_aead_decap - IPSec ESP decapsulation shared descriptor 369 * (non-protocol). 370 * @desc: pointer to buffer used for descriptor construction 371 * @cdata: pointer to block cipher transform definitions 372 * Valid algorithm values - one of OP_ALG_ALGSEL_{AES, DES, 3DES} ANDed 373 * with OP_ALG_AAI_CBC or OP_ALG_AAI_CTR_MOD128. 374 * @adata: pointer to authentication transform definitions. 375 * A split key is required for SEC Era < 6; the size of the split key 376 * is specified in this case. Valid algorithm values - one of 377 * OP_ALG_ALGSEL_{MD5, SHA1, SHA224, SHA256, SHA384, SHA512} ANDed 378 * with OP_ALG_AAI_HMAC_PRECOMP. 379 * @ivsize: initialization vector size 380 * @icvsize: integrity check value (ICV) size (truncated or full) 381 * @is_rfc3686: true when ctr(aes) is wrapped by rfc3686 template 382 * @nonce: pointer to rfc3686 nonce 383 * @ctx1_iv_off: IV offset in CONTEXT1 register 384 * @is_qi: true when called from caam/qi 385 * @era: SEC Era 386 */ 387 void cnstr_shdsc_aead_decap(u32 * const desc, struct alginfo *cdata, 388 struct alginfo *adata, unsigned int ivsize, 389 unsigned int icvsize, const bool geniv, 390 const bool is_rfc3686, u32 *nonce, 391 const u32 ctx1_iv_off, const bool is_qi, int era) 392 { 393 /* Note: Context registers are saved. */ 394 init_sh_desc_key_aead(desc, cdata, adata, is_rfc3686, nonce, era); 395 396 /* Class 2 operation */ 397 append_operation(desc, adata->algtype | OP_ALG_AS_INITFINAL | 398 OP_ALG_DECRYPT | OP_ALG_ICV_ON); 399 400 if (is_qi) { 401 u32 *wait_load_cmd; 402 403 /* REG3 = assoclen */ 404 append_seq_load(desc, 4, LDST_CLASS_DECO | 405 LDST_SRCDST_WORD_DECO_MATH3 | 406 (4 << LDST_OFFSET_SHIFT)); 407 408 wait_load_cmd = append_jump(desc, JUMP_JSL | JUMP_TEST_ALL | 409 JUMP_COND_CALM | JUMP_COND_NCP | 410 JUMP_COND_NOP | JUMP_COND_NIP | 411 JUMP_COND_NIFP); 412 set_jump_tgt_here(desc, wait_load_cmd); 413 414 if (!geniv) 415 append_seq_load(desc, ivsize, LDST_CLASS_1_CCB | 416 LDST_SRCDST_BYTE_CONTEXT | 417 (ctx1_iv_off << LDST_OFFSET_SHIFT)); 418 } 419 420 /* Read and write assoclen bytes */ 421 if (is_qi || era < 3) { 422 append_math_add(desc, VARSEQINLEN, ZERO, REG3, CAAM_CMD_SZ); 423 if (geniv) 424 append_math_add_imm_u32(desc, VARSEQOUTLEN, REG3, IMM, 425 ivsize); 426 else 427 append_math_add(desc, VARSEQOUTLEN, ZERO, REG3, 428 CAAM_CMD_SZ); 429 } else { 430 append_math_add(desc, VARSEQINLEN, ZERO, DPOVRD, CAAM_CMD_SZ); 431 if (geniv) 432 append_math_add_imm_u32(desc, VARSEQOUTLEN, DPOVRD, IMM, 433 ivsize); 434 else 435 append_math_add(desc, VARSEQOUTLEN, ZERO, DPOVRD, 436 CAAM_CMD_SZ); 437 } 438 439 /* Skip assoc data */ 440 append_seq_fifo_store(desc, 0, FIFOST_TYPE_SKIP | FIFOLDST_VLF); 441 442 /* read assoc before reading payload */ 443 append_seq_fifo_load(desc, 0, FIFOLD_CLASS_CLASS2 | FIFOLD_TYPE_MSG | 444 KEY_VLF); 445 446 if (geniv) { 447 append_seq_load(desc, ivsize, LDST_CLASS_1_CCB | 448 LDST_SRCDST_BYTE_CONTEXT | 449 (ctx1_iv_off << LDST_OFFSET_SHIFT)); 450 append_move(desc, MOVE_SRC_CLASS1CTX | MOVE_DEST_CLASS2INFIFO | 451 (ctx1_iv_off << MOVE_OFFSET_SHIFT) | ivsize); 452 } 453 454 /* Load Counter into CONTEXT1 reg */ 455 if (is_rfc3686) 456 append_load_imm_be32(desc, 1, LDST_IMM | LDST_CLASS_1_CCB | 457 LDST_SRCDST_BYTE_CONTEXT | 458 ((ctx1_iv_off + CTR_RFC3686_IV_SIZE) << 459 LDST_OFFSET_SHIFT)); 460 461 /* Choose operation */ 462 if (ctx1_iv_off) 463 append_operation(desc, cdata->algtype | OP_ALG_AS_INITFINAL | 464 OP_ALG_DECRYPT); 465 else 466 append_dec_op1(desc, cdata->algtype); 467 468 /* Read and write cryptlen bytes */ 469 append_math_add(desc, VARSEQINLEN, SEQOUTLEN, REG0, CAAM_CMD_SZ); 470 append_math_add(desc, VARSEQOUTLEN, SEQOUTLEN, REG0, CAAM_CMD_SZ); 471 aead_append_src_dst(desc, FIFOLD_TYPE_MSG); 472 473 /* Load ICV */ 474 append_seq_fifo_load(desc, icvsize, FIFOLD_CLASS_CLASS2 | 475 FIFOLD_TYPE_LAST2 | FIFOLD_TYPE_ICV); 476 477 #ifdef DEBUG 478 print_hex_dump(KERN_ERR, "aead dec shdesc@" __stringify(__LINE__)": ", 479 DUMP_PREFIX_ADDRESS, 16, 4, desc, desc_bytes(desc), 1); 480 #endif 481 } 482 EXPORT_SYMBOL(cnstr_shdsc_aead_decap); 483 484 /** 485 * cnstr_shdsc_aead_givencap - IPSec ESP encapsulation shared descriptor 486 * (non-protocol) with HW-generated initialization 487 * vector. 488 * @desc: pointer to buffer used for descriptor construction 489 * @cdata: pointer to block cipher transform definitions 490 * Valid algorithm values - one of OP_ALG_ALGSEL_{AES, DES, 3DES} ANDed 491 * with OP_ALG_AAI_CBC or OP_ALG_AAI_CTR_MOD128. 492 * @adata: pointer to authentication transform definitions. 493 * A split key is required for SEC Era < 6; the size of the split key 494 * is specified in this case. Valid algorithm values - one of 495 * OP_ALG_ALGSEL_{MD5, SHA1, SHA224, SHA256, SHA384, SHA512} ANDed 496 * with OP_ALG_AAI_HMAC_PRECOMP. 497 * @ivsize: initialization vector size 498 * @icvsize: integrity check value (ICV) size (truncated or full) 499 * @is_rfc3686: true when ctr(aes) is wrapped by rfc3686 template 500 * @nonce: pointer to rfc3686 nonce 501 * @ctx1_iv_off: IV offset in CONTEXT1 register 502 * @is_qi: true when called from caam/qi 503 * @era: SEC Era 504 */ 505 void cnstr_shdsc_aead_givencap(u32 * const desc, struct alginfo *cdata, 506 struct alginfo *adata, unsigned int ivsize, 507 unsigned int icvsize, const bool is_rfc3686, 508 u32 *nonce, const u32 ctx1_iv_off, 509 const bool is_qi, int era) 510 { 511 u32 geniv, moveiv; 512 513 /* Note: Context registers are saved. */ 514 init_sh_desc_key_aead(desc, cdata, adata, is_rfc3686, nonce, era); 515 516 if (is_qi) { 517 u32 *wait_load_cmd; 518 519 /* REG3 = assoclen */ 520 append_seq_load(desc, 4, LDST_CLASS_DECO | 521 LDST_SRCDST_WORD_DECO_MATH3 | 522 (4 << LDST_OFFSET_SHIFT)); 523 524 wait_load_cmd = append_jump(desc, JUMP_JSL | JUMP_TEST_ALL | 525 JUMP_COND_CALM | JUMP_COND_NCP | 526 JUMP_COND_NOP | JUMP_COND_NIP | 527 JUMP_COND_NIFP); 528 set_jump_tgt_here(desc, wait_load_cmd); 529 } 530 531 if (is_rfc3686) { 532 if (is_qi) 533 append_seq_load(desc, ivsize, LDST_CLASS_1_CCB | 534 LDST_SRCDST_BYTE_CONTEXT | 535 (ctx1_iv_off << LDST_OFFSET_SHIFT)); 536 537 goto copy_iv; 538 } 539 540 /* Generate IV */ 541 geniv = NFIFOENTRY_STYPE_PAD | NFIFOENTRY_DEST_DECO | 542 NFIFOENTRY_DTYPE_MSG | NFIFOENTRY_LC1 | 543 NFIFOENTRY_PTYPE_RND | (ivsize << NFIFOENTRY_DLEN_SHIFT); 544 append_load_imm_u32(desc, geniv, LDST_CLASS_IND_CCB | 545 LDST_SRCDST_WORD_INFO_FIFO | LDST_IMM); 546 append_cmd(desc, CMD_LOAD | DISABLE_AUTO_INFO_FIFO); 547 append_move(desc, MOVE_WAITCOMP | 548 MOVE_SRC_INFIFO | MOVE_DEST_CLASS1CTX | 549 (ctx1_iv_off << MOVE_OFFSET_SHIFT) | 550 (ivsize << MOVE_LEN_SHIFT)); 551 append_cmd(desc, CMD_LOAD | ENABLE_AUTO_INFO_FIFO); 552 553 copy_iv: 554 /* Copy IV to class 1 context */ 555 append_move(desc, MOVE_SRC_CLASS1CTX | MOVE_DEST_OUTFIFO | 556 (ctx1_iv_off << MOVE_OFFSET_SHIFT) | 557 (ivsize << MOVE_LEN_SHIFT)); 558 559 /* Return to encryption */ 560 append_operation(desc, adata->algtype | OP_ALG_AS_INITFINAL | 561 OP_ALG_ENCRYPT); 562 563 /* Read and write assoclen bytes */ 564 if (is_qi || era < 3) { 565 append_math_add(desc, VARSEQINLEN, ZERO, REG3, CAAM_CMD_SZ); 566 append_math_add(desc, VARSEQOUTLEN, ZERO, REG3, CAAM_CMD_SZ); 567 } else { 568 append_math_add(desc, VARSEQINLEN, ZERO, DPOVRD, CAAM_CMD_SZ); 569 append_math_add(desc, VARSEQOUTLEN, ZERO, DPOVRD, CAAM_CMD_SZ); 570 } 571 572 /* Skip assoc data */ 573 append_seq_fifo_store(desc, 0, FIFOST_TYPE_SKIP | FIFOLDST_VLF); 574 575 /* read assoc before reading payload */ 576 append_seq_fifo_load(desc, 0, FIFOLD_CLASS_CLASS2 | FIFOLD_TYPE_MSG | 577 KEY_VLF); 578 579 /* Copy iv from outfifo to class 2 fifo */ 580 moveiv = NFIFOENTRY_STYPE_OFIFO | NFIFOENTRY_DEST_CLASS2 | 581 NFIFOENTRY_DTYPE_MSG | (ivsize << NFIFOENTRY_DLEN_SHIFT); 582 append_load_imm_u32(desc, moveiv, LDST_CLASS_IND_CCB | 583 LDST_SRCDST_WORD_INFO_FIFO | LDST_IMM); 584 append_load_imm_u32(desc, ivsize, LDST_CLASS_2_CCB | 585 LDST_SRCDST_WORD_DATASZ_REG | LDST_IMM); 586 587 /* Load Counter into CONTEXT1 reg */ 588 if (is_rfc3686) 589 append_load_imm_be32(desc, 1, LDST_IMM | LDST_CLASS_1_CCB | 590 LDST_SRCDST_BYTE_CONTEXT | 591 ((ctx1_iv_off + CTR_RFC3686_IV_SIZE) << 592 LDST_OFFSET_SHIFT)); 593 594 /* Class 1 operation */ 595 append_operation(desc, cdata->algtype | OP_ALG_AS_INITFINAL | 596 OP_ALG_ENCRYPT); 597 598 /* Will write ivsize + cryptlen */ 599 append_math_add(desc, VARSEQOUTLEN, SEQINLEN, REG0, CAAM_CMD_SZ); 600 601 /* Not need to reload iv */ 602 append_seq_fifo_load(desc, ivsize, 603 FIFOLD_CLASS_SKIP); 604 605 /* Will read cryptlen */ 606 append_math_add(desc, VARSEQINLEN, SEQINLEN, REG0, CAAM_CMD_SZ); 607 append_seq_fifo_load(desc, 0, FIFOLD_CLASS_BOTH | KEY_VLF | 608 FIFOLD_TYPE_MSG1OUT2 | FIFOLD_TYPE_LASTBOTH); 609 append_seq_fifo_store(desc, 0, FIFOST_TYPE_MESSAGE_DATA | KEY_VLF); 610 611 /* Write ICV */ 612 append_seq_store(desc, icvsize, LDST_CLASS_2_CCB | 613 LDST_SRCDST_BYTE_CONTEXT); 614 615 #ifdef DEBUG 616 print_hex_dump(KERN_ERR, 617 "aead givenc shdesc@" __stringify(__LINE__)": ", 618 DUMP_PREFIX_ADDRESS, 16, 4, desc, desc_bytes(desc), 1); 619 #endif 620 } 621 EXPORT_SYMBOL(cnstr_shdsc_aead_givencap); 622 623 /** 624 * cnstr_shdsc_gcm_encap - gcm encapsulation shared descriptor 625 * @desc: pointer to buffer used for descriptor construction 626 * @cdata: pointer to block cipher transform definitions 627 * Valid algorithm values - OP_ALG_ALGSEL_AES ANDed with OP_ALG_AAI_GCM. 628 * @ivsize: initialization vector size 629 * @icvsize: integrity check value (ICV) size (truncated or full) 630 * @is_qi: true when called from caam/qi 631 */ 632 void cnstr_shdsc_gcm_encap(u32 * const desc, struct alginfo *cdata, 633 unsigned int ivsize, unsigned int icvsize, 634 const bool is_qi) 635 { 636 u32 *key_jump_cmd, *zero_payload_jump_cmd, *zero_assoc_jump_cmd1, 637 *zero_assoc_jump_cmd2; 638 639 init_sh_desc(desc, HDR_SHARE_SERIAL); 640 641 /* skip key loading if they are loaded due to sharing */ 642 key_jump_cmd = append_jump(desc, JUMP_JSL | JUMP_TEST_ALL | 643 JUMP_COND_SHRD); 644 if (cdata->key_inline) 645 append_key_as_imm(desc, cdata->key_virt, cdata->keylen, 646 cdata->keylen, CLASS_1 | KEY_DEST_CLASS_REG); 647 else 648 append_key(desc, cdata->key_dma, cdata->keylen, CLASS_1 | 649 KEY_DEST_CLASS_REG); 650 set_jump_tgt_here(desc, key_jump_cmd); 651 652 /* class 1 operation */ 653 append_operation(desc, cdata->algtype | OP_ALG_AS_INITFINAL | 654 OP_ALG_ENCRYPT); 655 656 if (is_qi) { 657 u32 *wait_load_cmd; 658 659 /* REG3 = assoclen */ 660 append_seq_load(desc, 4, LDST_CLASS_DECO | 661 LDST_SRCDST_WORD_DECO_MATH3 | 662 (4 << LDST_OFFSET_SHIFT)); 663 664 wait_load_cmd = append_jump(desc, JUMP_JSL | JUMP_TEST_ALL | 665 JUMP_COND_CALM | JUMP_COND_NCP | 666 JUMP_COND_NOP | JUMP_COND_NIP | 667 JUMP_COND_NIFP); 668 set_jump_tgt_here(desc, wait_load_cmd); 669 670 append_math_sub_imm_u32(desc, VARSEQOUTLEN, SEQINLEN, IMM, 671 ivsize); 672 } else { 673 append_math_sub(desc, VARSEQOUTLEN, SEQINLEN, REG0, 674 CAAM_CMD_SZ); 675 } 676 677 /* if assoclen + cryptlen is ZERO, skip to ICV write */ 678 zero_assoc_jump_cmd2 = append_jump(desc, JUMP_TEST_ALL | 679 JUMP_COND_MATH_Z); 680 681 if (is_qi) 682 append_seq_fifo_load(desc, ivsize, FIFOLD_CLASS_CLASS1 | 683 FIFOLD_TYPE_IV | FIFOLD_TYPE_FLUSH1); 684 685 /* if assoclen is ZERO, skip reading the assoc data */ 686 append_math_add(desc, VARSEQINLEN, ZERO, REG3, CAAM_CMD_SZ); 687 zero_assoc_jump_cmd1 = append_jump(desc, JUMP_TEST_ALL | 688 JUMP_COND_MATH_Z); 689 690 append_math_add(desc, VARSEQOUTLEN, ZERO, REG3, CAAM_CMD_SZ); 691 692 /* skip assoc data */ 693 append_seq_fifo_store(desc, 0, FIFOST_TYPE_SKIP | FIFOLDST_VLF); 694 695 /* cryptlen = seqinlen - assoclen */ 696 append_math_sub(desc, VARSEQOUTLEN, SEQINLEN, REG3, CAAM_CMD_SZ); 697 698 /* if cryptlen is ZERO jump to zero-payload commands */ 699 zero_payload_jump_cmd = append_jump(desc, JUMP_TEST_ALL | 700 JUMP_COND_MATH_Z); 701 702 /* read assoc data */ 703 append_seq_fifo_load(desc, 0, FIFOLD_CLASS_CLASS1 | FIFOLDST_VLF | 704 FIFOLD_TYPE_AAD | FIFOLD_TYPE_FLUSH1); 705 set_jump_tgt_here(desc, zero_assoc_jump_cmd1); 706 707 append_math_sub(desc, VARSEQINLEN, SEQINLEN, REG0, CAAM_CMD_SZ); 708 709 /* write encrypted data */ 710 append_seq_fifo_store(desc, 0, FIFOST_TYPE_MESSAGE_DATA | FIFOLDST_VLF); 711 712 /* read payload data */ 713 append_seq_fifo_load(desc, 0, FIFOLD_CLASS_CLASS1 | FIFOLDST_VLF | 714 FIFOLD_TYPE_MSG | FIFOLD_TYPE_LAST1); 715 716 /* jump to ICV writing */ 717 if (is_qi) 718 append_jump(desc, JUMP_TEST_ALL | 4); 719 else 720 append_jump(desc, JUMP_TEST_ALL | 2); 721 722 /* zero-payload commands */ 723 set_jump_tgt_here(desc, zero_payload_jump_cmd); 724 725 /* read assoc data */ 726 append_seq_fifo_load(desc, 0, FIFOLD_CLASS_CLASS1 | FIFOLDST_VLF | 727 FIFOLD_TYPE_AAD | FIFOLD_TYPE_LAST1); 728 if (is_qi) 729 /* jump to ICV writing */ 730 append_jump(desc, JUMP_TEST_ALL | 2); 731 732 /* There is no input data */ 733 set_jump_tgt_here(desc, zero_assoc_jump_cmd2); 734 735 if (is_qi) 736 append_seq_fifo_load(desc, ivsize, FIFOLD_CLASS_CLASS1 | 737 FIFOLD_TYPE_IV | FIFOLD_TYPE_FLUSH1 | 738 FIFOLD_TYPE_LAST1); 739 740 /* write ICV */ 741 append_seq_store(desc, icvsize, LDST_CLASS_1_CCB | 742 LDST_SRCDST_BYTE_CONTEXT); 743 744 #ifdef DEBUG 745 print_hex_dump(KERN_ERR, "gcm enc shdesc@" __stringify(__LINE__)": ", 746 DUMP_PREFIX_ADDRESS, 16, 4, desc, desc_bytes(desc), 1); 747 #endif 748 } 749 EXPORT_SYMBOL(cnstr_shdsc_gcm_encap); 750 751 /** 752 * cnstr_shdsc_gcm_decap - gcm decapsulation shared descriptor 753 * @desc: pointer to buffer used for descriptor construction 754 * @cdata: pointer to block cipher transform definitions 755 * Valid algorithm values - OP_ALG_ALGSEL_AES ANDed with OP_ALG_AAI_GCM. 756 * @ivsize: initialization vector size 757 * @icvsize: integrity check value (ICV) size (truncated or full) 758 * @is_qi: true when called from caam/qi 759 */ 760 void cnstr_shdsc_gcm_decap(u32 * const desc, struct alginfo *cdata, 761 unsigned int ivsize, unsigned int icvsize, 762 const bool is_qi) 763 { 764 u32 *key_jump_cmd, *zero_payload_jump_cmd, *zero_assoc_jump_cmd1; 765 766 init_sh_desc(desc, HDR_SHARE_SERIAL); 767 768 /* skip key loading if they are loaded due to sharing */ 769 key_jump_cmd = append_jump(desc, JUMP_JSL | 770 JUMP_TEST_ALL | JUMP_COND_SHRD); 771 if (cdata->key_inline) 772 append_key_as_imm(desc, cdata->key_virt, cdata->keylen, 773 cdata->keylen, CLASS_1 | KEY_DEST_CLASS_REG); 774 else 775 append_key(desc, cdata->key_dma, cdata->keylen, CLASS_1 | 776 KEY_DEST_CLASS_REG); 777 set_jump_tgt_here(desc, key_jump_cmd); 778 779 /* class 1 operation */ 780 append_operation(desc, cdata->algtype | OP_ALG_AS_INITFINAL | 781 OP_ALG_DECRYPT | OP_ALG_ICV_ON); 782 783 if (is_qi) { 784 u32 *wait_load_cmd; 785 786 /* REG3 = assoclen */ 787 append_seq_load(desc, 4, LDST_CLASS_DECO | 788 LDST_SRCDST_WORD_DECO_MATH3 | 789 (4 << LDST_OFFSET_SHIFT)); 790 791 wait_load_cmd = append_jump(desc, JUMP_JSL | JUMP_TEST_ALL | 792 JUMP_COND_CALM | JUMP_COND_NCP | 793 JUMP_COND_NOP | JUMP_COND_NIP | 794 JUMP_COND_NIFP); 795 set_jump_tgt_here(desc, wait_load_cmd); 796 797 append_seq_fifo_load(desc, ivsize, FIFOLD_CLASS_CLASS1 | 798 FIFOLD_TYPE_IV | FIFOLD_TYPE_FLUSH1); 799 } 800 801 /* if assoclen is ZERO, skip reading the assoc data */ 802 append_math_add(desc, VARSEQINLEN, ZERO, REG3, CAAM_CMD_SZ); 803 zero_assoc_jump_cmd1 = append_jump(desc, JUMP_TEST_ALL | 804 JUMP_COND_MATH_Z); 805 806 append_math_add(desc, VARSEQOUTLEN, ZERO, REG3, CAAM_CMD_SZ); 807 808 /* skip assoc data */ 809 append_seq_fifo_store(desc, 0, FIFOST_TYPE_SKIP | FIFOLDST_VLF); 810 811 /* read assoc data */ 812 append_seq_fifo_load(desc, 0, FIFOLD_CLASS_CLASS1 | FIFOLDST_VLF | 813 FIFOLD_TYPE_AAD | FIFOLD_TYPE_FLUSH1); 814 815 set_jump_tgt_here(desc, zero_assoc_jump_cmd1); 816 817 /* cryptlen = seqoutlen - assoclen */ 818 append_math_sub(desc, VARSEQINLEN, SEQOUTLEN, REG0, CAAM_CMD_SZ); 819 820 /* jump to zero-payload command if cryptlen is zero */ 821 zero_payload_jump_cmd = append_jump(desc, JUMP_TEST_ALL | 822 JUMP_COND_MATH_Z); 823 824 append_math_sub(desc, VARSEQOUTLEN, SEQOUTLEN, REG0, CAAM_CMD_SZ); 825 826 /* store encrypted data */ 827 append_seq_fifo_store(desc, 0, FIFOST_TYPE_MESSAGE_DATA | FIFOLDST_VLF); 828 829 /* read payload data */ 830 append_seq_fifo_load(desc, 0, FIFOLD_CLASS_CLASS1 | FIFOLDST_VLF | 831 FIFOLD_TYPE_MSG | FIFOLD_TYPE_FLUSH1); 832 833 /* zero-payload command */ 834 set_jump_tgt_here(desc, zero_payload_jump_cmd); 835 836 /* read ICV */ 837 append_seq_fifo_load(desc, icvsize, FIFOLD_CLASS_CLASS1 | 838 FIFOLD_TYPE_ICV | FIFOLD_TYPE_LAST1); 839 840 #ifdef DEBUG 841 print_hex_dump(KERN_ERR, "gcm dec shdesc@" __stringify(__LINE__)": ", 842 DUMP_PREFIX_ADDRESS, 16, 4, desc, desc_bytes(desc), 1); 843 #endif 844 } 845 EXPORT_SYMBOL(cnstr_shdsc_gcm_decap); 846 847 /** 848 * cnstr_shdsc_rfc4106_encap - IPSec ESP gcm encapsulation shared descriptor 849 * (non-protocol). 850 * @desc: pointer to buffer used for descriptor construction 851 * @cdata: pointer to block cipher transform definitions 852 * Valid algorithm values - OP_ALG_ALGSEL_AES ANDed with OP_ALG_AAI_GCM. 853 * @ivsize: initialization vector size 854 * @icvsize: integrity check value (ICV) size (truncated or full) 855 * @is_qi: true when called from caam/qi 856 */ 857 void cnstr_shdsc_rfc4106_encap(u32 * const desc, struct alginfo *cdata, 858 unsigned int ivsize, unsigned int icvsize, 859 const bool is_qi) 860 { 861 u32 *key_jump_cmd; 862 863 init_sh_desc(desc, HDR_SHARE_SERIAL); 864 865 /* Skip key loading if it is loaded due to sharing */ 866 key_jump_cmd = append_jump(desc, JUMP_JSL | JUMP_TEST_ALL | 867 JUMP_COND_SHRD); 868 if (cdata->key_inline) 869 append_key_as_imm(desc, cdata->key_virt, cdata->keylen, 870 cdata->keylen, CLASS_1 | KEY_DEST_CLASS_REG); 871 else 872 append_key(desc, cdata->key_dma, cdata->keylen, CLASS_1 | 873 KEY_DEST_CLASS_REG); 874 set_jump_tgt_here(desc, key_jump_cmd); 875 876 /* Class 1 operation */ 877 append_operation(desc, cdata->algtype | OP_ALG_AS_INITFINAL | 878 OP_ALG_ENCRYPT); 879 880 if (is_qi) { 881 u32 *wait_load_cmd; 882 883 /* REG3 = assoclen */ 884 append_seq_load(desc, 4, LDST_CLASS_DECO | 885 LDST_SRCDST_WORD_DECO_MATH3 | 886 (4 << LDST_OFFSET_SHIFT)); 887 888 wait_load_cmd = append_jump(desc, JUMP_JSL | JUMP_TEST_ALL | 889 JUMP_COND_CALM | JUMP_COND_NCP | 890 JUMP_COND_NOP | JUMP_COND_NIP | 891 JUMP_COND_NIFP); 892 set_jump_tgt_here(desc, wait_load_cmd); 893 894 /* Read salt and IV */ 895 append_fifo_load_as_imm(desc, (void *)(cdata->key_virt + 896 cdata->keylen), 4, FIFOLD_CLASS_CLASS1 | 897 FIFOLD_TYPE_IV); 898 append_seq_fifo_load(desc, ivsize, FIFOLD_CLASS_CLASS1 | 899 FIFOLD_TYPE_IV | FIFOLD_TYPE_FLUSH1); 900 } 901 902 append_math_sub_imm_u32(desc, VARSEQINLEN, REG3, IMM, ivsize); 903 append_math_add(desc, VARSEQOUTLEN, ZERO, REG3, CAAM_CMD_SZ); 904 905 /* Read assoc data */ 906 append_seq_fifo_load(desc, 0, FIFOLD_CLASS_CLASS1 | FIFOLDST_VLF | 907 FIFOLD_TYPE_AAD | FIFOLD_TYPE_FLUSH1); 908 909 /* Skip IV */ 910 append_seq_fifo_load(desc, ivsize, FIFOLD_CLASS_SKIP); 911 912 /* Will read cryptlen bytes */ 913 append_math_sub(desc, VARSEQINLEN, SEQINLEN, REG0, CAAM_CMD_SZ); 914 915 /* Workaround for erratum A-005473 (simultaneous SEQ FIFO skips) */ 916 append_seq_fifo_load(desc, 0, FIFOLD_CLASS_CLASS1 | FIFOLD_TYPE_MSG); 917 918 /* Skip assoc data */ 919 append_seq_fifo_store(desc, 0, FIFOST_TYPE_SKIP | FIFOLDST_VLF); 920 921 /* cryptlen = seqoutlen - assoclen */ 922 append_math_sub(desc, VARSEQOUTLEN, VARSEQINLEN, REG0, CAAM_CMD_SZ); 923 924 /* Write encrypted data */ 925 append_seq_fifo_store(desc, 0, FIFOST_TYPE_MESSAGE_DATA | FIFOLDST_VLF); 926 927 /* Read payload data */ 928 append_seq_fifo_load(desc, 0, FIFOLD_CLASS_CLASS1 | FIFOLDST_VLF | 929 FIFOLD_TYPE_MSG | FIFOLD_TYPE_LAST1); 930 931 /* Write ICV */ 932 append_seq_store(desc, icvsize, LDST_CLASS_1_CCB | 933 LDST_SRCDST_BYTE_CONTEXT); 934 935 #ifdef DEBUG 936 print_hex_dump(KERN_ERR, 937 "rfc4106 enc shdesc@" __stringify(__LINE__)": ", 938 DUMP_PREFIX_ADDRESS, 16, 4, desc, desc_bytes(desc), 1); 939 #endif 940 } 941 EXPORT_SYMBOL(cnstr_shdsc_rfc4106_encap); 942 943 /** 944 * cnstr_shdsc_rfc4106_decap - IPSec ESP gcm decapsulation shared descriptor 945 * (non-protocol). 946 * @desc: pointer to buffer used for descriptor construction 947 * @cdata: pointer to block cipher transform definitions 948 * Valid algorithm values - OP_ALG_ALGSEL_AES ANDed with OP_ALG_AAI_GCM. 949 * @ivsize: initialization vector size 950 * @icvsize: integrity check value (ICV) size (truncated or full) 951 * @is_qi: true when called from caam/qi 952 */ 953 void cnstr_shdsc_rfc4106_decap(u32 * const desc, struct alginfo *cdata, 954 unsigned int ivsize, unsigned int icvsize, 955 const bool is_qi) 956 { 957 u32 *key_jump_cmd; 958 959 init_sh_desc(desc, HDR_SHARE_SERIAL); 960 961 /* Skip key loading if it is loaded due to sharing */ 962 key_jump_cmd = append_jump(desc, JUMP_JSL | JUMP_TEST_ALL | 963 JUMP_COND_SHRD); 964 if (cdata->key_inline) 965 append_key_as_imm(desc, cdata->key_virt, cdata->keylen, 966 cdata->keylen, CLASS_1 | 967 KEY_DEST_CLASS_REG); 968 else 969 append_key(desc, cdata->key_dma, cdata->keylen, CLASS_1 | 970 KEY_DEST_CLASS_REG); 971 set_jump_tgt_here(desc, key_jump_cmd); 972 973 /* Class 1 operation */ 974 append_operation(desc, cdata->algtype | OP_ALG_AS_INITFINAL | 975 OP_ALG_DECRYPT | OP_ALG_ICV_ON); 976 977 if (is_qi) { 978 u32 *wait_load_cmd; 979 980 /* REG3 = assoclen */ 981 append_seq_load(desc, 4, LDST_CLASS_DECO | 982 LDST_SRCDST_WORD_DECO_MATH3 | 983 (4 << LDST_OFFSET_SHIFT)); 984 985 wait_load_cmd = append_jump(desc, JUMP_JSL | JUMP_TEST_ALL | 986 JUMP_COND_CALM | JUMP_COND_NCP | 987 JUMP_COND_NOP | JUMP_COND_NIP | 988 JUMP_COND_NIFP); 989 set_jump_tgt_here(desc, wait_load_cmd); 990 991 /* Read salt and IV */ 992 append_fifo_load_as_imm(desc, (void *)(cdata->key_virt + 993 cdata->keylen), 4, FIFOLD_CLASS_CLASS1 | 994 FIFOLD_TYPE_IV); 995 append_seq_fifo_load(desc, ivsize, FIFOLD_CLASS_CLASS1 | 996 FIFOLD_TYPE_IV | FIFOLD_TYPE_FLUSH1); 997 } 998 999 append_math_sub_imm_u32(desc, VARSEQINLEN, REG3, IMM, ivsize); 1000 append_math_add(desc, VARSEQOUTLEN, ZERO, REG3, CAAM_CMD_SZ); 1001 1002 /* Read assoc data */ 1003 append_seq_fifo_load(desc, 0, FIFOLD_CLASS_CLASS1 | FIFOLDST_VLF | 1004 FIFOLD_TYPE_AAD | FIFOLD_TYPE_FLUSH1); 1005 1006 /* Skip IV */ 1007 append_seq_fifo_load(desc, ivsize, FIFOLD_CLASS_SKIP); 1008 1009 /* Will read cryptlen bytes */ 1010 append_math_sub(desc, VARSEQINLEN, SEQOUTLEN, REG3, CAAM_CMD_SZ); 1011 1012 /* Workaround for erratum A-005473 (simultaneous SEQ FIFO skips) */ 1013 append_seq_fifo_load(desc, 0, FIFOLD_CLASS_CLASS1 | FIFOLD_TYPE_MSG); 1014 1015 /* Skip assoc data */ 1016 append_seq_fifo_store(desc, 0, FIFOST_TYPE_SKIP | FIFOLDST_VLF); 1017 1018 /* Will write cryptlen bytes */ 1019 append_math_sub(desc, VARSEQOUTLEN, SEQOUTLEN, REG0, CAAM_CMD_SZ); 1020 1021 /* Store payload data */ 1022 append_seq_fifo_store(desc, 0, FIFOST_TYPE_MESSAGE_DATA | FIFOLDST_VLF); 1023 1024 /* Read encrypted data */ 1025 append_seq_fifo_load(desc, 0, FIFOLD_CLASS_CLASS1 | FIFOLDST_VLF | 1026 FIFOLD_TYPE_MSG | FIFOLD_TYPE_FLUSH1); 1027 1028 /* Read ICV */ 1029 append_seq_fifo_load(desc, icvsize, FIFOLD_CLASS_CLASS1 | 1030 FIFOLD_TYPE_ICV | FIFOLD_TYPE_LAST1); 1031 1032 #ifdef DEBUG 1033 print_hex_dump(KERN_ERR, 1034 "rfc4106 dec shdesc@" __stringify(__LINE__)": ", 1035 DUMP_PREFIX_ADDRESS, 16, 4, desc, desc_bytes(desc), 1); 1036 #endif 1037 } 1038 EXPORT_SYMBOL(cnstr_shdsc_rfc4106_decap); 1039 1040 /** 1041 * cnstr_shdsc_rfc4543_encap - IPSec ESP gmac encapsulation shared descriptor 1042 * (non-protocol). 1043 * @desc: pointer to buffer used for descriptor construction 1044 * @cdata: pointer to block cipher transform definitions 1045 * Valid algorithm values - OP_ALG_ALGSEL_AES ANDed with OP_ALG_AAI_GCM. 1046 * @ivsize: initialization vector size 1047 * @icvsize: integrity check value (ICV) size (truncated or full) 1048 * @is_qi: true when called from caam/qi 1049 */ 1050 void cnstr_shdsc_rfc4543_encap(u32 * const desc, struct alginfo *cdata, 1051 unsigned int ivsize, unsigned int icvsize, 1052 const bool is_qi) 1053 { 1054 u32 *key_jump_cmd, *read_move_cmd, *write_move_cmd; 1055 1056 init_sh_desc(desc, HDR_SHARE_SERIAL); 1057 1058 /* Skip key loading if it is loaded due to sharing */ 1059 key_jump_cmd = append_jump(desc, JUMP_JSL | JUMP_TEST_ALL | 1060 JUMP_COND_SHRD); 1061 if (cdata->key_inline) 1062 append_key_as_imm(desc, cdata->key_virt, cdata->keylen, 1063 cdata->keylen, CLASS_1 | KEY_DEST_CLASS_REG); 1064 else 1065 append_key(desc, cdata->key_dma, cdata->keylen, CLASS_1 | 1066 KEY_DEST_CLASS_REG); 1067 set_jump_tgt_here(desc, key_jump_cmd); 1068 1069 /* Class 1 operation */ 1070 append_operation(desc, cdata->algtype | OP_ALG_AS_INITFINAL | 1071 OP_ALG_ENCRYPT); 1072 1073 if (is_qi) { 1074 /* assoclen is not needed, skip it */ 1075 append_seq_fifo_load(desc, 4, FIFOLD_CLASS_SKIP); 1076 1077 /* Read salt and IV */ 1078 append_fifo_load_as_imm(desc, (void *)(cdata->key_virt + 1079 cdata->keylen), 4, FIFOLD_CLASS_CLASS1 | 1080 FIFOLD_TYPE_IV); 1081 append_seq_fifo_load(desc, ivsize, FIFOLD_CLASS_CLASS1 | 1082 FIFOLD_TYPE_IV | FIFOLD_TYPE_FLUSH1); 1083 } 1084 1085 /* assoclen + cryptlen = seqinlen */ 1086 append_math_sub(desc, REG3, SEQINLEN, REG0, CAAM_CMD_SZ); 1087 1088 /* 1089 * MOVE_LEN opcode is not available in all SEC HW revisions, 1090 * thus need to do some magic, i.e. self-patch the descriptor 1091 * buffer. 1092 */ 1093 read_move_cmd = append_move(desc, MOVE_SRC_DESCBUF | MOVE_DEST_MATH3 | 1094 (0x6 << MOVE_LEN_SHIFT)); 1095 write_move_cmd = append_move(desc, MOVE_SRC_MATH3 | MOVE_DEST_DESCBUF | 1096 (0x8 << MOVE_LEN_SHIFT) | MOVE_WAITCOMP); 1097 1098 /* Will read assoclen + cryptlen bytes */ 1099 append_math_sub(desc, VARSEQINLEN, SEQINLEN, REG0, CAAM_CMD_SZ); 1100 1101 /* Will write assoclen + cryptlen bytes */ 1102 append_math_sub(desc, VARSEQOUTLEN, SEQINLEN, REG0, CAAM_CMD_SZ); 1103 1104 /* Read and write assoclen + cryptlen bytes */ 1105 aead_append_src_dst(desc, FIFOLD_TYPE_AAD); 1106 1107 set_move_tgt_here(desc, read_move_cmd); 1108 set_move_tgt_here(desc, write_move_cmd); 1109 append_cmd(desc, CMD_LOAD | DISABLE_AUTO_INFO_FIFO); 1110 /* Move payload data to OFIFO */ 1111 append_move(desc, MOVE_SRC_INFIFO_CL | MOVE_DEST_OUTFIFO); 1112 1113 /* Write ICV */ 1114 append_seq_store(desc, icvsize, LDST_CLASS_1_CCB | 1115 LDST_SRCDST_BYTE_CONTEXT); 1116 1117 #ifdef DEBUG 1118 print_hex_dump(KERN_ERR, 1119 "rfc4543 enc shdesc@" __stringify(__LINE__)": ", 1120 DUMP_PREFIX_ADDRESS, 16, 4, desc, desc_bytes(desc), 1); 1121 #endif 1122 } 1123 EXPORT_SYMBOL(cnstr_shdsc_rfc4543_encap); 1124 1125 /** 1126 * cnstr_shdsc_rfc4543_decap - IPSec ESP gmac decapsulation shared descriptor 1127 * (non-protocol). 1128 * @desc: pointer to buffer used for descriptor construction 1129 * @cdata: pointer to block cipher transform definitions 1130 * Valid algorithm values - OP_ALG_ALGSEL_AES ANDed with OP_ALG_AAI_GCM. 1131 * @ivsize: initialization vector size 1132 * @icvsize: integrity check value (ICV) size (truncated or full) 1133 * @is_qi: true when called from caam/qi 1134 */ 1135 void cnstr_shdsc_rfc4543_decap(u32 * const desc, struct alginfo *cdata, 1136 unsigned int ivsize, unsigned int icvsize, 1137 const bool is_qi) 1138 { 1139 u32 *key_jump_cmd, *read_move_cmd, *write_move_cmd; 1140 1141 init_sh_desc(desc, HDR_SHARE_SERIAL); 1142 1143 /* Skip key loading if it is loaded due to sharing */ 1144 key_jump_cmd = append_jump(desc, JUMP_JSL | JUMP_TEST_ALL | 1145 JUMP_COND_SHRD); 1146 if (cdata->key_inline) 1147 append_key_as_imm(desc, cdata->key_virt, cdata->keylen, 1148 cdata->keylen, CLASS_1 | KEY_DEST_CLASS_REG); 1149 else 1150 append_key(desc, cdata->key_dma, cdata->keylen, CLASS_1 | 1151 KEY_DEST_CLASS_REG); 1152 set_jump_tgt_here(desc, key_jump_cmd); 1153 1154 /* Class 1 operation */ 1155 append_operation(desc, cdata->algtype | OP_ALG_AS_INITFINAL | 1156 OP_ALG_DECRYPT | OP_ALG_ICV_ON); 1157 1158 if (is_qi) { 1159 /* assoclen is not needed, skip it */ 1160 append_seq_fifo_load(desc, 4, FIFOLD_CLASS_SKIP); 1161 1162 /* Read salt and IV */ 1163 append_fifo_load_as_imm(desc, (void *)(cdata->key_virt + 1164 cdata->keylen), 4, FIFOLD_CLASS_CLASS1 | 1165 FIFOLD_TYPE_IV); 1166 append_seq_fifo_load(desc, ivsize, FIFOLD_CLASS_CLASS1 | 1167 FIFOLD_TYPE_IV | FIFOLD_TYPE_FLUSH1); 1168 } 1169 1170 /* assoclen + cryptlen = seqoutlen */ 1171 append_math_sub(desc, REG3, SEQOUTLEN, REG0, CAAM_CMD_SZ); 1172 1173 /* 1174 * MOVE_LEN opcode is not available in all SEC HW revisions, 1175 * thus need to do some magic, i.e. self-patch the descriptor 1176 * buffer. 1177 */ 1178 read_move_cmd = append_move(desc, MOVE_SRC_DESCBUF | MOVE_DEST_MATH3 | 1179 (0x6 << MOVE_LEN_SHIFT)); 1180 write_move_cmd = append_move(desc, MOVE_SRC_MATH3 | MOVE_DEST_DESCBUF | 1181 (0x8 << MOVE_LEN_SHIFT) | MOVE_WAITCOMP); 1182 1183 /* Will read assoclen + cryptlen bytes */ 1184 append_math_sub(desc, VARSEQINLEN, SEQOUTLEN, REG0, CAAM_CMD_SZ); 1185 1186 /* Will write assoclen + cryptlen bytes */ 1187 append_math_sub(desc, VARSEQOUTLEN, SEQOUTLEN, REG0, CAAM_CMD_SZ); 1188 1189 /* Store payload data */ 1190 append_seq_fifo_store(desc, 0, FIFOST_TYPE_MESSAGE_DATA | FIFOLDST_VLF); 1191 1192 /* In-snoop assoclen + cryptlen data */ 1193 append_seq_fifo_load(desc, 0, FIFOLD_CLASS_BOTH | FIFOLDST_VLF | 1194 FIFOLD_TYPE_AAD | FIFOLD_TYPE_LAST2FLUSH1); 1195 1196 set_move_tgt_here(desc, read_move_cmd); 1197 set_move_tgt_here(desc, write_move_cmd); 1198 append_cmd(desc, CMD_LOAD | DISABLE_AUTO_INFO_FIFO); 1199 /* Move payload data to OFIFO */ 1200 append_move(desc, MOVE_SRC_INFIFO_CL | MOVE_DEST_OUTFIFO); 1201 append_cmd(desc, CMD_LOAD | ENABLE_AUTO_INFO_FIFO); 1202 1203 /* Read ICV */ 1204 append_seq_fifo_load(desc, icvsize, FIFOLD_CLASS_CLASS1 | 1205 FIFOLD_TYPE_ICV | FIFOLD_TYPE_LAST1); 1206 1207 #ifdef DEBUG 1208 print_hex_dump(KERN_ERR, 1209 "rfc4543 dec shdesc@" __stringify(__LINE__)": ", 1210 DUMP_PREFIX_ADDRESS, 16, 4, desc, desc_bytes(desc), 1); 1211 #endif 1212 } 1213 EXPORT_SYMBOL(cnstr_shdsc_rfc4543_decap); 1214 1215 /* 1216 * For ablkcipher encrypt and decrypt, read from req->src and 1217 * write to req->dst 1218 */ 1219 static inline void ablkcipher_append_src_dst(u32 *desc) 1220 { 1221 append_math_add(desc, VARSEQOUTLEN, SEQINLEN, REG0, CAAM_CMD_SZ); 1222 append_math_add(desc, VARSEQINLEN, SEQINLEN, REG0, CAAM_CMD_SZ); 1223 append_seq_fifo_load(desc, 0, FIFOLD_CLASS_CLASS1 | 1224 KEY_VLF | FIFOLD_TYPE_MSG | FIFOLD_TYPE_LAST1); 1225 append_seq_fifo_store(desc, 0, FIFOST_TYPE_MESSAGE_DATA | KEY_VLF); 1226 } 1227 1228 /** 1229 * cnstr_shdsc_ablkcipher_encap - ablkcipher encapsulation shared descriptor 1230 * @desc: pointer to buffer used for descriptor construction 1231 * @cdata: pointer to block cipher transform definitions 1232 * Valid algorithm values - one of OP_ALG_ALGSEL_{AES, DES, 3DES} ANDed 1233 * with OP_ALG_AAI_CBC or OP_ALG_AAI_CTR_MOD128. 1234 * @ivsize: initialization vector size 1235 * @is_rfc3686: true when ctr(aes) is wrapped by rfc3686 template 1236 * @ctx1_iv_off: IV offset in CONTEXT1 register 1237 */ 1238 void cnstr_shdsc_ablkcipher_encap(u32 * const desc, struct alginfo *cdata, 1239 unsigned int ivsize, const bool is_rfc3686, 1240 const u32 ctx1_iv_off) 1241 { 1242 u32 *key_jump_cmd; 1243 1244 init_sh_desc(desc, HDR_SHARE_SERIAL | HDR_SAVECTX); 1245 /* Skip if already shared */ 1246 key_jump_cmd = append_jump(desc, JUMP_JSL | JUMP_TEST_ALL | 1247 JUMP_COND_SHRD); 1248 1249 /* Load class1 key only */ 1250 append_key_as_imm(desc, cdata->key_virt, cdata->keylen, 1251 cdata->keylen, CLASS_1 | KEY_DEST_CLASS_REG); 1252 1253 /* Load nonce into CONTEXT1 reg */ 1254 if (is_rfc3686) { 1255 const u8 *nonce = cdata->key_virt + cdata->keylen; 1256 1257 append_load_as_imm(desc, nonce, CTR_RFC3686_NONCE_SIZE, 1258 LDST_CLASS_IND_CCB | 1259 LDST_SRCDST_BYTE_OUTFIFO | LDST_IMM); 1260 append_move(desc, MOVE_WAITCOMP | MOVE_SRC_OUTFIFO | 1261 MOVE_DEST_CLASS1CTX | (16 << MOVE_OFFSET_SHIFT) | 1262 (CTR_RFC3686_NONCE_SIZE << MOVE_LEN_SHIFT)); 1263 } 1264 1265 set_jump_tgt_here(desc, key_jump_cmd); 1266 1267 /* Load iv */ 1268 append_seq_load(desc, ivsize, LDST_SRCDST_BYTE_CONTEXT | 1269 LDST_CLASS_1_CCB | (ctx1_iv_off << LDST_OFFSET_SHIFT)); 1270 1271 /* Load counter into CONTEXT1 reg */ 1272 if (is_rfc3686) 1273 append_load_imm_be32(desc, 1, LDST_IMM | LDST_CLASS_1_CCB | 1274 LDST_SRCDST_BYTE_CONTEXT | 1275 ((ctx1_iv_off + CTR_RFC3686_IV_SIZE) << 1276 LDST_OFFSET_SHIFT)); 1277 1278 /* Load operation */ 1279 append_operation(desc, cdata->algtype | OP_ALG_AS_INITFINAL | 1280 OP_ALG_ENCRYPT); 1281 1282 /* Perform operation */ 1283 ablkcipher_append_src_dst(desc); 1284 1285 #ifdef DEBUG 1286 print_hex_dump(KERN_ERR, 1287 "ablkcipher enc shdesc@" __stringify(__LINE__)": ", 1288 DUMP_PREFIX_ADDRESS, 16, 4, desc, desc_bytes(desc), 1); 1289 #endif 1290 } 1291 EXPORT_SYMBOL(cnstr_shdsc_ablkcipher_encap); 1292 1293 /** 1294 * cnstr_shdsc_ablkcipher_decap - ablkcipher decapsulation shared descriptor 1295 * @desc: pointer to buffer used for descriptor construction 1296 * @cdata: pointer to block cipher transform definitions 1297 * Valid algorithm values - one of OP_ALG_ALGSEL_{AES, DES, 3DES} ANDed 1298 * with OP_ALG_AAI_CBC or OP_ALG_AAI_CTR_MOD128. 1299 * @ivsize: initialization vector size 1300 * @is_rfc3686: true when ctr(aes) is wrapped by rfc3686 template 1301 * @ctx1_iv_off: IV offset in CONTEXT1 register 1302 */ 1303 void cnstr_shdsc_ablkcipher_decap(u32 * const desc, struct alginfo *cdata, 1304 unsigned int ivsize, const bool is_rfc3686, 1305 const u32 ctx1_iv_off) 1306 { 1307 u32 *key_jump_cmd; 1308 1309 init_sh_desc(desc, HDR_SHARE_SERIAL | HDR_SAVECTX); 1310 /* Skip if already shared */ 1311 key_jump_cmd = append_jump(desc, JUMP_JSL | JUMP_TEST_ALL | 1312 JUMP_COND_SHRD); 1313 1314 /* Load class1 key only */ 1315 append_key_as_imm(desc, cdata->key_virt, cdata->keylen, 1316 cdata->keylen, CLASS_1 | KEY_DEST_CLASS_REG); 1317 1318 /* Load nonce into CONTEXT1 reg */ 1319 if (is_rfc3686) { 1320 const u8 *nonce = cdata->key_virt + cdata->keylen; 1321 1322 append_load_as_imm(desc, nonce, CTR_RFC3686_NONCE_SIZE, 1323 LDST_CLASS_IND_CCB | 1324 LDST_SRCDST_BYTE_OUTFIFO | LDST_IMM); 1325 append_move(desc, MOVE_WAITCOMP | MOVE_SRC_OUTFIFO | 1326 MOVE_DEST_CLASS1CTX | (16 << MOVE_OFFSET_SHIFT) | 1327 (CTR_RFC3686_NONCE_SIZE << MOVE_LEN_SHIFT)); 1328 } 1329 1330 set_jump_tgt_here(desc, key_jump_cmd); 1331 1332 /* load IV */ 1333 append_seq_load(desc, ivsize, LDST_SRCDST_BYTE_CONTEXT | 1334 LDST_CLASS_1_CCB | (ctx1_iv_off << LDST_OFFSET_SHIFT)); 1335 1336 /* Load counter into CONTEXT1 reg */ 1337 if (is_rfc3686) 1338 append_load_imm_be32(desc, 1, LDST_IMM | LDST_CLASS_1_CCB | 1339 LDST_SRCDST_BYTE_CONTEXT | 1340 ((ctx1_iv_off + CTR_RFC3686_IV_SIZE) << 1341 LDST_OFFSET_SHIFT)); 1342 1343 /* Choose operation */ 1344 if (ctx1_iv_off) 1345 append_operation(desc, cdata->algtype | OP_ALG_AS_INITFINAL | 1346 OP_ALG_DECRYPT); 1347 else 1348 append_dec_op1(desc, cdata->algtype); 1349 1350 /* Perform operation */ 1351 ablkcipher_append_src_dst(desc); 1352 1353 #ifdef DEBUG 1354 print_hex_dump(KERN_ERR, 1355 "ablkcipher dec shdesc@" __stringify(__LINE__)": ", 1356 DUMP_PREFIX_ADDRESS, 16, 4, desc, desc_bytes(desc), 1); 1357 #endif 1358 } 1359 EXPORT_SYMBOL(cnstr_shdsc_ablkcipher_decap); 1360 1361 /** 1362 * cnstr_shdsc_ablkcipher_givencap - ablkcipher encapsulation shared descriptor 1363 * with HW-generated initialization vector. 1364 * @desc: pointer to buffer used for descriptor construction 1365 * @cdata: pointer to block cipher transform definitions 1366 * Valid algorithm values - one of OP_ALG_ALGSEL_{AES, DES, 3DES} ANDed 1367 * with OP_ALG_AAI_CBC. 1368 * @ivsize: initialization vector size 1369 * @is_rfc3686: true when ctr(aes) is wrapped by rfc3686 template 1370 * @ctx1_iv_off: IV offset in CONTEXT1 register 1371 */ 1372 void cnstr_shdsc_ablkcipher_givencap(u32 * const desc, struct alginfo *cdata, 1373 unsigned int ivsize, const bool is_rfc3686, 1374 const u32 ctx1_iv_off) 1375 { 1376 u32 *key_jump_cmd, geniv; 1377 1378 init_sh_desc(desc, HDR_SHARE_SERIAL | HDR_SAVECTX); 1379 /* Skip if already shared */ 1380 key_jump_cmd = append_jump(desc, JUMP_JSL | JUMP_TEST_ALL | 1381 JUMP_COND_SHRD); 1382 1383 /* Load class1 key only */ 1384 append_key_as_imm(desc, cdata->key_virt, cdata->keylen, 1385 cdata->keylen, CLASS_1 | KEY_DEST_CLASS_REG); 1386 1387 /* Load Nonce into CONTEXT1 reg */ 1388 if (is_rfc3686) { 1389 const u8 *nonce = cdata->key_virt + cdata->keylen; 1390 1391 append_load_as_imm(desc, nonce, CTR_RFC3686_NONCE_SIZE, 1392 LDST_CLASS_IND_CCB | 1393 LDST_SRCDST_BYTE_OUTFIFO | LDST_IMM); 1394 append_move(desc, MOVE_WAITCOMP | MOVE_SRC_OUTFIFO | 1395 MOVE_DEST_CLASS1CTX | (16 << MOVE_OFFSET_SHIFT) | 1396 (CTR_RFC3686_NONCE_SIZE << MOVE_LEN_SHIFT)); 1397 } 1398 set_jump_tgt_here(desc, key_jump_cmd); 1399 1400 /* Generate IV */ 1401 geniv = NFIFOENTRY_STYPE_PAD | NFIFOENTRY_DEST_DECO | 1402 NFIFOENTRY_DTYPE_MSG | NFIFOENTRY_LC1 | NFIFOENTRY_PTYPE_RND | 1403 (ivsize << NFIFOENTRY_DLEN_SHIFT); 1404 append_load_imm_u32(desc, geniv, LDST_CLASS_IND_CCB | 1405 LDST_SRCDST_WORD_INFO_FIFO | LDST_IMM); 1406 append_cmd(desc, CMD_LOAD | DISABLE_AUTO_INFO_FIFO); 1407 append_move(desc, MOVE_WAITCOMP | MOVE_SRC_INFIFO | 1408 MOVE_DEST_CLASS1CTX | (ivsize << MOVE_LEN_SHIFT) | 1409 (ctx1_iv_off << MOVE_OFFSET_SHIFT)); 1410 append_cmd(desc, CMD_LOAD | ENABLE_AUTO_INFO_FIFO); 1411 1412 /* Copy generated IV to memory */ 1413 append_seq_store(desc, ivsize, LDST_SRCDST_BYTE_CONTEXT | 1414 LDST_CLASS_1_CCB | (ctx1_iv_off << LDST_OFFSET_SHIFT)); 1415 1416 /* Load Counter into CONTEXT1 reg */ 1417 if (is_rfc3686) 1418 append_load_imm_be32(desc, 1, LDST_IMM | LDST_CLASS_1_CCB | 1419 LDST_SRCDST_BYTE_CONTEXT | 1420 ((ctx1_iv_off + CTR_RFC3686_IV_SIZE) << 1421 LDST_OFFSET_SHIFT)); 1422 1423 if (ctx1_iv_off) 1424 append_jump(desc, JUMP_JSL | JUMP_TEST_ALL | JUMP_COND_NCP | 1425 (1 << JUMP_OFFSET_SHIFT)); 1426 1427 /* Load operation */ 1428 append_operation(desc, cdata->algtype | OP_ALG_AS_INITFINAL | 1429 OP_ALG_ENCRYPT); 1430 1431 /* Perform operation */ 1432 ablkcipher_append_src_dst(desc); 1433 1434 #ifdef DEBUG 1435 print_hex_dump(KERN_ERR, 1436 "ablkcipher givenc shdesc@" __stringify(__LINE__) ": ", 1437 DUMP_PREFIX_ADDRESS, 16, 4, desc, desc_bytes(desc), 1); 1438 #endif 1439 } 1440 EXPORT_SYMBOL(cnstr_shdsc_ablkcipher_givencap); 1441 1442 /** 1443 * cnstr_shdsc_xts_ablkcipher_encap - xts ablkcipher encapsulation shared 1444 * descriptor 1445 * @desc: pointer to buffer used for descriptor construction 1446 * @cdata: pointer to block cipher transform definitions 1447 * Valid algorithm values - OP_ALG_ALGSEL_AES ANDed with OP_ALG_AAI_XTS. 1448 */ 1449 void cnstr_shdsc_xts_ablkcipher_encap(u32 * const desc, struct alginfo *cdata) 1450 { 1451 __be64 sector_size = cpu_to_be64(512); 1452 u32 *key_jump_cmd; 1453 1454 init_sh_desc(desc, HDR_SHARE_SERIAL | HDR_SAVECTX); 1455 /* Skip if already shared */ 1456 key_jump_cmd = append_jump(desc, JUMP_JSL | JUMP_TEST_ALL | 1457 JUMP_COND_SHRD); 1458 1459 /* Load class1 keys only */ 1460 append_key_as_imm(desc, cdata->key_virt, cdata->keylen, 1461 cdata->keylen, CLASS_1 | KEY_DEST_CLASS_REG); 1462 1463 /* Load sector size with index 40 bytes (0x28) */ 1464 append_load_as_imm(desc, (void *)§or_size, 8, LDST_CLASS_1_CCB | 1465 LDST_SRCDST_BYTE_CONTEXT | 1466 (0x28 << LDST_OFFSET_SHIFT)); 1467 1468 set_jump_tgt_here(desc, key_jump_cmd); 1469 1470 /* 1471 * create sequence for loading the sector index 1472 * Upper 8B of IV - will be used as sector index 1473 * Lower 8B of IV - will be discarded 1474 */ 1475 append_seq_load(desc, 8, LDST_SRCDST_BYTE_CONTEXT | LDST_CLASS_1_CCB | 1476 (0x20 << LDST_OFFSET_SHIFT)); 1477 append_seq_fifo_load(desc, 8, FIFOLD_CLASS_SKIP); 1478 1479 /* Load operation */ 1480 append_operation(desc, cdata->algtype | OP_ALG_AS_INITFINAL | 1481 OP_ALG_ENCRYPT); 1482 1483 /* Perform operation */ 1484 ablkcipher_append_src_dst(desc); 1485 1486 #ifdef DEBUG 1487 print_hex_dump(KERN_ERR, 1488 "xts ablkcipher enc shdesc@" __stringify(__LINE__) ": ", 1489 DUMP_PREFIX_ADDRESS, 16, 4, desc, desc_bytes(desc), 1); 1490 #endif 1491 } 1492 EXPORT_SYMBOL(cnstr_shdsc_xts_ablkcipher_encap); 1493 1494 /** 1495 * cnstr_shdsc_xts_ablkcipher_decap - xts ablkcipher decapsulation shared 1496 * descriptor 1497 * @desc: pointer to buffer used for descriptor construction 1498 * @cdata: pointer to block cipher transform definitions 1499 * Valid algorithm values - OP_ALG_ALGSEL_AES ANDed with OP_ALG_AAI_XTS. 1500 */ 1501 void cnstr_shdsc_xts_ablkcipher_decap(u32 * const desc, struct alginfo *cdata) 1502 { 1503 __be64 sector_size = cpu_to_be64(512); 1504 u32 *key_jump_cmd; 1505 1506 init_sh_desc(desc, HDR_SHARE_SERIAL | HDR_SAVECTX); 1507 /* Skip if already shared */ 1508 key_jump_cmd = append_jump(desc, JUMP_JSL | JUMP_TEST_ALL | 1509 JUMP_COND_SHRD); 1510 1511 /* Load class1 key only */ 1512 append_key_as_imm(desc, cdata->key_virt, cdata->keylen, 1513 cdata->keylen, CLASS_1 | KEY_DEST_CLASS_REG); 1514 1515 /* Load sector size with index 40 bytes (0x28) */ 1516 append_load_as_imm(desc, (void *)§or_size, 8, LDST_CLASS_1_CCB | 1517 LDST_SRCDST_BYTE_CONTEXT | 1518 (0x28 << LDST_OFFSET_SHIFT)); 1519 1520 set_jump_tgt_here(desc, key_jump_cmd); 1521 1522 /* 1523 * create sequence for loading the sector index 1524 * Upper 8B of IV - will be used as sector index 1525 * Lower 8B of IV - will be discarded 1526 */ 1527 append_seq_load(desc, 8, LDST_SRCDST_BYTE_CONTEXT | LDST_CLASS_1_CCB | 1528 (0x20 << LDST_OFFSET_SHIFT)); 1529 append_seq_fifo_load(desc, 8, FIFOLD_CLASS_SKIP); 1530 1531 /* Load operation */ 1532 append_dec_op1(desc, cdata->algtype); 1533 1534 /* Perform operation */ 1535 ablkcipher_append_src_dst(desc); 1536 1537 #ifdef DEBUG 1538 print_hex_dump(KERN_ERR, 1539 "xts ablkcipher dec shdesc@" __stringify(__LINE__) ": ", 1540 DUMP_PREFIX_ADDRESS, 16, 4, desc, desc_bytes(desc), 1); 1541 #endif 1542 } 1543 EXPORT_SYMBOL(cnstr_shdsc_xts_ablkcipher_decap); 1544 1545 MODULE_LICENSE("GPL"); 1546 MODULE_DESCRIPTION("FSL CAAM descriptor support"); 1547 MODULE_AUTHOR("Freescale Semiconductor - NMG/STC"); 1548