1 /* 2 * caam - Freescale FSL CAAM support for crypto API 3 * 4 * Copyright 2008-2011 Freescale Semiconductor, Inc. 5 * 6 * Based on talitos crypto API driver. 7 * 8 * relationship of job descriptors to shared descriptors (SteveC Dec 10 2008): 9 * 10 * --------------- --------------- 11 * | JobDesc #1 |-------------------->| ShareDesc | 12 * | *(packet 1) | | (PDB) | 13 * --------------- |------------->| (hashKey) | 14 * . | | (cipherKey) | 15 * . | |-------->| (operation) | 16 * --------------- | | --------------- 17 * | JobDesc #2 |------| | 18 * | *(packet 2) | | 19 * --------------- | 20 * . | 21 * . | 22 * --------------- | 23 * | JobDesc #3 |------------ 24 * | *(packet 3) | 25 * --------------- 26 * 27 * The SharedDesc never changes for a connection unless rekeyed, but 28 * each packet will likely be in a different place. So all we need 29 * to know to process the packet is where the input is, where the 30 * output goes, and what context we want to process with. Context is 31 * in the SharedDesc, packet references in the JobDesc. 32 * 33 * So, a job desc looks like: 34 * 35 * --------------------- 36 * | Header | 37 * | ShareDesc Pointer | 38 * | SEQ_OUT_PTR | 39 * | (output buffer) | 40 * | (output length) | 41 * | SEQ_IN_PTR | 42 * | (input buffer) | 43 * | (input length) | 44 * --------------------- 45 */ 46 47 #include "compat.h" 48 49 #include "regs.h" 50 #include "intern.h" 51 #include "desc_constr.h" 52 #include "jr.h" 53 #include "error.h" 54 #include "sg_sw_sec4.h" 55 #include "key_gen.h" 56 57 /* 58 * crypto alg 59 */ 60 #define CAAM_CRA_PRIORITY 3000 61 /* max key is sum of AES_MAX_KEY_SIZE, max split key size */ 62 #define CAAM_MAX_KEY_SIZE (AES_MAX_KEY_SIZE + \ 63 CTR_RFC3686_NONCE_SIZE + \ 64 SHA512_DIGEST_SIZE * 2) 65 /* max IV is max of AES_BLOCK_SIZE, DES3_EDE_BLOCK_SIZE */ 66 #define CAAM_MAX_IV_LENGTH 16 67 68 #define AEAD_DESC_JOB_IO_LEN (DESC_JOB_IO_LEN + CAAM_CMD_SZ * 2) 69 #define GCM_DESC_JOB_IO_LEN (AEAD_DESC_JOB_IO_LEN + \ 70 CAAM_CMD_SZ * 4) 71 #define AUTHENC_DESC_JOB_IO_LEN (AEAD_DESC_JOB_IO_LEN + \ 72 CAAM_CMD_SZ * 5) 73 74 /* length of descriptors text */ 75 #define DESC_AEAD_BASE (4 * CAAM_CMD_SZ) 76 #define DESC_AEAD_ENC_LEN (DESC_AEAD_BASE + 11 * CAAM_CMD_SZ) 77 #define DESC_AEAD_DEC_LEN (DESC_AEAD_BASE + 15 * CAAM_CMD_SZ) 78 #define DESC_AEAD_GIVENC_LEN (DESC_AEAD_ENC_LEN + 9 * CAAM_CMD_SZ) 79 80 /* Note: Nonce is counted in enckeylen */ 81 #define DESC_AEAD_CTR_RFC3686_LEN (4 * CAAM_CMD_SZ) 82 83 #define DESC_AEAD_NULL_BASE (3 * CAAM_CMD_SZ) 84 #define DESC_AEAD_NULL_ENC_LEN (DESC_AEAD_NULL_BASE + 11 * CAAM_CMD_SZ) 85 #define DESC_AEAD_NULL_DEC_LEN (DESC_AEAD_NULL_BASE + 13 * CAAM_CMD_SZ) 86 87 #define DESC_GCM_BASE (3 * CAAM_CMD_SZ) 88 #define DESC_GCM_ENC_LEN (DESC_GCM_BASE + 16 * CAAM_CMD_SZ) 89 #define DESC_GCM_DEC_LEN (DESC_GCM_BASE + 12 * CAAM_CMD_SZ) 90 91 #define DESC_RFC4106_BASE (3 * CAAM_CMD_SZ) 92 #define DESC_RFC4106_ENC_LEN (DESC_RFC4106_BASE + 13 * CAAM_CMD_SZ) 93 #define DESC_RFC4106_DEC_LEN (DESC_RFC4106_BASE + 13 * CAAM_CMD_SZ) 94 95 #define DESC_RFC4543_BASE (3 * CAAM_CMD_SZ) 96 #define DESC_RFC4543_ENC_LEN (DESC_RFC4543_BASE + 11 * CAAM_CMD_SZ) 97 #define DESC_RFC4543_DEC_LEN (DESC_RFC4543_BASE + 12 * CAAM_CMD_SZ) 98 99 #define DESC_ABLKCIPHER_BASE (3 * CAAM_CMD_SZ) 100 #define DESC_ABLKCIPHER_ENC_LEN (DESC_ABLKCIPHER_BASE + \ 101 20 * CAAM_CMD_SZ) 102 #define DESC_ABLKCIPHER_DEC_LEN (DESC_ABLKCIPHER_BASE + \ 103 15 * CAAM_CMD_SZ) 104 105 #define DESC_MAX_USED_BYTES (CAAM_DESC_BYTES_MAX - DESC_JOB_IO_LEN) 106 #define DESC_MAX_USED_LEN (DESC_MAX_USED_BYTES / CAAM_CMD_SZ) 107 108 #ifdef DEBUG 109 /* for print_hex_dumps with line references */ 110 #define debug(format, arg...) printk(format, arg) 111 #else 112 #define debug(format, arg...) 113 #endif 114 static struct list_head alg_list; 115 116 struct caam_alg_entry { 117 int class1_alg_type; 118 int class2_alg_type; 119 int alg_op; 120 bool rfc3686; 121 bool geniv; 122 }; 123 124 struct caam_aead_alg { 125 struct aead_alg aead; 126 struct caam_alg_entry caam; 127 bool registered; 128 }; 129 130 /* Set DK bit in class 1 operation if shared */ 131 static inline void append_dec_op1(u32 *desc, u32 type) 132 { 133 u32 *jump_cmd, *uncond_jump_cmd; 134 135 /* DK bit is valid only for AES */ 136 if ((type & OP_ALG_ALGSEL_MASK) != OP_ALG_ALGSEL_AES) { 137 append_operation(desc, type | OP_ALG_AS_INITFINAL | 138 OP_ALG_DECRYPT); 139 return; 140 } 141 142 jump_cmd = append_jump(desc, JUMP_TEST_ALL | JUMP_COND_SHRD); 143 append_operation(desc, type | OP_ALG_AS_INITFINAL | 144 OP_ALG_DECRYPT); 145 uncond_jump_cmd = append_jump(desc, JUMP_TEST_ALL); 146 set_jump_tgt_here(desc, jump_cmd); 147 append_operation(desc, type | OP_ALG_AS_INITFINAL | 148 OP_ALG_DECRYPT | OP_ALG_AAI_DK); 149 set_jump_tgt_here(desc, uncond_jump_cmd); 150 } 151 152 /* 153 * For aead functions, read payload and write payload, 154 * both of which are specified in req->src and req->dst 155 */ 156 static inline void aead_append_src_dst(u32 *desc, u32 msg_type) 157 { 158 append_seq_fifo_store(desc, 0, FIFOST_TYPE_MESSAGE_DATA | KEY_VLF); 159 append_seq_fifo_load(desc, 0, FIFOLD_CLASS_BOTH | 160 KEY_VLF | msg_type | FIFOLD_TYPE_LASTBOTH); 161 } 162 163 /* 164 * For ablkcipher encrypt and decrypt, read from req->src and 165 * write to req->dst 166 */ 167 static inline void ablkcipher_append_src_dst(u32 *desc) 168 { 169 append_math_add(desc, VARSEQOUTLEN, SEQINLEN, REG0, CAAM_CMD_SZ); 170 append_math_add(desc, VARSEQINLEN, SEQINLEN, REG0, CAAM_CMD_SZ); 171 append_seq_fifo_load(desc, 0, FIFOLD_CLASS_CLASS1 | 172 KEY_VLF | FIFOLD_TYPE_MSG | FIFOLD_TYPE_LAST1); 173 append_seq_fifo_store(desc, 0, FIFOST_TYPE_MESSAGE_DATA | KEY_VLF); 174 } 175 176 /* 177 * per-session context 178 */ 179 struct caam_ctx { 180 struct device *jrdev; 181 u32 sh_desc_enc[DESC_MAX_USED_LEN]; 182 u32 sh_desc_dec[DESC_MAX_USED_LEN]; 183 u32 sh_desc_givenc[DESC_MAX_USED_LEN]; 184 dma_addr_t sh_desc_enc_dma; 185 dma_addr_t sh_desc_dec_dma; 186 dma_addr_t sh_desc_givenc_dma; 187 u32 class1_alg_type; 188 u32 class2_alg_type; 189 u32 alg_op; 190 u8 key[CAAM_MAX_KEY_SIZE]; 191 dma_addr_t key_dma; 192 unsigned int enckeylen; 193 unsigned int split_key_len; 194 unsigned int split_key_pad_len; 195 unsigned int authsize; 196 }; 197 198 static void append_key_aead(u32 *desc, struct caam_ctx *ctx, 199 int keys_fit_inline, bool is_rfc3686) 200 { 201 u32 *nonce; 202 unsigned int enckeylen = ctx->enckeylen; 203 204 /* 205 * RFC3686 specific: 206 * | ctx->key = {AUTH_KEY, ENC_KEY, NONCE} 207 * | enckeylen = encryption key size + nonce size 208 */ 209 if (is_rfc3686) 210 enckeylen -= CTR_RFC3686_NONCE_SIZE; 211 212 if (keys_fit_inline) { 213 append_key_as_imm(desc, ctx->key, ctx->split_key_pad_len, 214 ctx->split_key_len, CLASS_2 | 215 KEY_DEST_MDHA_SPLIT | KEY_ENC); 216 append_key_as_imm(desc, (void *)ctx->key + 217 ctx->split_key_pad_len, enckeylen, 218 enckeylen, CLASS_1 | KEY_DEST_CLASS_REG); 219 } else { 220 append_key(desc, ctx->key_dma, ctx->split_key_len, CLASS_2 | 221 KEY_DEST_MDHA_SPLIT | KEY_ENC); 222 append_key(desc, ctx->key_dma + ctx->split_key_pad_len, 223 enckeylen, CLASS_1 | KEY_DEST_CLASS_REG); 224 } 225 226 /* Load Counter into CONTEXT1 reg */ 227 if (is_rfc3686) { 228 nonce = (u32 *)((void *)ctx->key + ctx->split_key_pad_len + 229 enckeylen); 230 append_load_imm_u32(desc, *nonce, LDST_CLASS_IND_CCB | 231 LDST_SRCDST_BYTE_OUTFIFO | LDST_IMM); 232 append_move(desc, 233 MOVE_SRC_OUTFIFO | 234 MOVE_DEST_CLASS1CTX | 235 (16 << MOVE_OFFSET_SHIFT) | 236 (CTR_RFC3686_NONCE_SIZE << MOVE_LEN_SHIFT)); 237 } 238 } 239 240 static void init_sh_desc_key_aead(u32 *desc, struct caam_ctx *ctx, 241 int keys_fit_inline, bool is_rfc3686) 242 { 243 u32 *key_jump_cmd; 244 245 /* Note: Context registers are saved. */ 246 init_sh_desc(desc, HDR_SHARE_SERIAL | HDR_SAVECTX); 247 248 /* Skip if already shared */ 249 key_jump_cmd = append_jump(desc, JUMP_JSL | JUMP_TEST_ALL | 250 JUMP_COND_SHRD); 251 252 append_key_aead(desc, ctx, keys_fit_inline, is_rfc3686); 253 254 set_jump_tgt_here(desc, key_jump_cmd); 255 } 256 257 static int aead_null_set_sh_desc(struct crypto_aead *aead) 258 { 259 struct caam_ctx *ctx = crypto_aead_ctx(aead); 260 struct device *jrdev = ctx->jrdev; 261 bool keys_fit_inline = false; 262 u32 *key_jump_cmd, *jump_cmd, *read_move_cmd, *write_move_cmd; 263 u32 *desc; 264 265 /* 266 * Job Descriptor and Shared Descriptors 267 * must all fit into the 64-word Descriptor h/w Buffer 268 */ 269 if (DESC_AEAD_NULL_ENC_LEN + AEAD_DESC_JOB_IO_LEN + 270 ctx->split_key_pad_len <= CAAM_DESC_BYTES_MAX) 271 keys_fit_inline = true; 272 273 /* aead_encrypt shared descriptor */ 274 desc = ctx->sh_desc_enc; 275 276 init_sh_desc(desc, HDR_SHARE_SERIAL); 277 278 /* Skip if already shared */ 279 key_jump_cmd = append_jump(desc, JUMP_JSL | JUMP_TEST_ALL | 280 JUMP_COND_SHRD); 281 if (keys_fit_inline) 282 append_key_as_imm(desc, ctx->key, ctx->split_key_pad_len, 283 ctx->split_key_len, CLASS_2 | 284 KEY_DEST_MDHA_SPLIT | KEY_ENC); 285 else 286 append_key(desc, ctx->key_dma, ctx->split_key_len, CLASS_2 | 287 KEY_DEST_MDHA_SPLIT | KEY_ENC); 288 set_jump_tgt_here(desc, key_jump_cmd); 289 290 /* assoclen + cryptlen = seqinlen */ 291 append_math_sub(desc, REG3, SEQINLEN, REG0, CAAM_CMD_SZ); 292 293 /* Prepare to read and write cryptlen + assoclen bytes */ 294 append_math_add(desc, VARSEQINLEN, ZERO, REG3, CAAM_CMD_SZ); 295 append_math_add(desc, VARSEQOUTLEN, ZERO, REG3, CAAM_CMD_SZ); 296 297 /* 298 * MOVE_LEN opcode is not available in all SEC HW revisions, 299 * thus need to do some magic, i.e. self-patch the descriptor 300 * buffer. 301 */ 302 read_move_cmd = append_move(desc, MOVE_SRC_DESCBUF | 303 MOVE_DEST_MATH3 | 304 (0x6 << MOVE_LEN_SHIFT)); 305 write_move_cmd = append_move(desc, MOVE_SRC_MATH3 | 306 MOVE_DEST_DESCBUF | 307 MOVE_WAITCOMP | 308 (0x8 << MOVE_LEN_SHIFT)); 309 310 /* Class 2 operation */ 311 append_operation(desc, ctx->class2_alg_type | 312 OP_ALG_AS_INITFINAL | OP_ALG_ENCRYPT); 313 314 /* Read and write cryptlen bytes */ 315 aead_append_src_dst(desc, FIFOLD_TYPE_MSG | FIFOLD_TYPE_FLUSH1); 316 317 set_move_tgt_here(desc, read_move_cmd); 318 set_move_tgt_here(desc, write_move_cmd); 319 append_cmd(desc, CMD_LOAD | DISABLE_AUTO_INFO_FIFO); 320 append_move(desc, MOVE_SRC_INFIFO_CL | MOVE_DEST_OUTFIFO | 321 MOVE_AUX_LS); 322 323 /* Write ICV */ 324 append_seq_store(desc, ctx->authsize, LDST_CLASS_2_CCB | 325 LDST_SRCDST_BYTE_CONTEXT); 326 327 ctx->sh_desc_enc_dma = dma_map_single(jrdev, desc, 328 desc_bytes(desc), 329 DMA_TO_DEVICE); 330 if (dma_mapping_error(jrdev, ctx->sh_desc_enc_dma)) { 331 dev_err(jrdev, "unable to map shared descriptor\n"); 332 return -ENOMEM; 333 } 334 #ifdef DEBUG 335 print_hex_dump(KERN_ERR, 336 "aead null enc shdesc@"__stringify(__LINE__)": ", 337 DUMP_PREFIX_ADDRESS, 16, 4, desc, 338 desc_bytes(desc), 1); 339 #endif 340 341 /* 342 * Job Descriptor and Shared Descriptors 343 * must all fit into the 64-word Descriptor h/w Buffer 344 */ 345 keys_fit_inline = false; 346 if (DESC_AEAD_NULL_DEC_LEN + DESC_JOB_IO_LEN + 347 ctx->split_key_pad_len <= CAAM_DESC_BYTES_MAX) 348 keys_fit_inline = true; 349 350 desc = ctx->sh_desc_dec; 351 352 /* aead_decrypt shared descriptor */ 353 init_sh_desc(desc, HDR_SHARE_SERIAL); 354 355 /* Skip if already shared */ 356 key_jump_cmd = append_jump(desc, JUMP_JSL | JUMP_TEST_ALL | 357 JUMP_COND_SHRD); 358 if (keys_fit_inline) 359 append_key_as_imm(desc, ctx->key, ctx->split_key_pad_len, 360 ctx->split_key_len, CLASS_2 | 361 KEY_DEST_MDHA_SPLIT | KEY_ENC); 362 else 363 append_key(desc, ctx->key_dma, ctx->split_key_len, CLASS_2 | 364 KEY_DEST_MDHA_SPLIT | KEY_ENC); 365 set_jump_tgt_here(desc, key_jump_cmd); 366 367 /* Class 2 operation */ 368 append_operation(desc, ctx->class2_alg_type | 369 OP_ALG_AS_INITFINAL | OP_ALG_DECRYPT | OP_ALG_ICV_ON); 370 371 /* assoclen + cryptlen = seqoutlen */ 372 append_math_sub(desc, REG2, SEQOUTLEN, REG0, CAAM_CMD_SZ); 373 374 /* Prepare to read and write cryptlen + assoclen bytes */ 375 append_math_add(desc, VARSEQINLEN, ZERO, REG2, CAAM_CMD_SZ); 376 append_math_add(desc, VARSEQOUTLEN, ZERO, REG2, CAAM_CMD_SZ); 377 378 /* 379 * MOVE_LEN opcode is not available in all SEC HW revisions, 380 * thus need to do some magic, i.e. self-patch the descriptor 381 * buffer. 382 */ 383 read_move_cmd = append_move(desc, MOVE_SRC_DESCBUF | 384 MOVE_DEST_MATH2 | 385 (0x6 << MOVE_LEN_SHIFT)); 386 write_move_cmd = append_move(desc, MOVE_SRC_MATH2 | 387 MOVE_DEST_DESCBUF | 388 MOVE_WAITCOMP | 389 (0x8 << MOVE_LEN_SHIFT)); 390 391 /* Read and write cryptlen bytes */ 392 aead_append_src_dst(desc, FIFOLD_TYPE_MSG | FIFOLD_TYPE_FLUSH1); 393 394 /* 395 * Insert a NOP here, since we need at least 4 instructions between 396 * code patching the descriptor buffer and the location being patched. 397 */ 398 jump_cmd = append_jump(desc, JUMP_TEST_ALL); 399 set_jump_tgt_here(desc, jump_cmd); 400 401 set_move_tgt_here(desc, read_move_cmd); 402 set_move_tgt_here(desc, write_move_cmd); 403 append_cmd(desc, CMD_LOAD | DISABLE_AUTO_INFO_FIFO); 404 append_move(desc, MOVE_SRC_INFIFO_CL | MOVE_DEST_OUTFIFO | 405 MOVE_AUX_LS); 406 append_cmd(desc, CMD_LOAD | ENABLE_AUTO_INFO_FIFO); 407 408 /* Load ICV */ 409 append_seq_fifo_load(desc, ctx->authsize, FIFOLD_CLASS_CLASS2 | 410 FIFOLD_TYPE_LAST2 | FIFOLD_TYPE_ICV); 411 412 ctx->sh_desc_dec_dma = dma_map_single(jrdev, desc, 413 desc_bytes(desc), 414 DMA_TO_DEVICE); 415 if (dma_mapping_error(jrdev, ctx->sh_desc_dec_dma)) { 416 dev_err(jrdev, "unable to map shared descriptor\n"); 417 return -ENOMEM; 418 } 419 #ifdef DEBUG 420 print_hex_dump(KERN_ERR, 421 "aead null dec shdesc@"__stringify(__LINE__)": ", 422 DUMP_PREFIX_ADDRESS, 16, 4, desc, 423 desc_bytes(desc), 1); 424 #endif 425 426 return 0; 427 } 428 429 static int aead_set_sh_desc(struct crypto_aead *aead) 430 { 431 struct caam_aead_alg *alg = container_of(crypto_aead_alg(aead), 432 struct caam_aead_alg, aead); 433 unsigned int ivsize = crypto_aead_ivsize(aead); 434 struct caam_ctx *ctx = crypto_aead_ctx(aead); 435 struct device *jrdev = ctx->jrdev; 436 bool keys_fit_inline; 437 u32 geniv, moveiv; 438 u32 ctx1_iv_off = 0; 439 u32 *desc; 440 const bool ctr_mode = ((ctx->class1_alg_type & OP_ALG_AAI_MASK) == 441 OP_ALG_AAI_CTR_MOD128); 442 const bool is_rfc3686 = alg->caam.rfc3686; 443 444 if (!ctx->authsize) 445 return 0; 446 447 /* NULL encryption / decryption */ 448 if (!ctx->enckeylen) 449 return aead_null_set_sh_desc(aead); 450 451 /* 452 * AES-CTR needs to load IV in CONTEXT1 reg 453 * at an offset of 128bits (16bytes) 454 * CONTEXT1[255:128] = IV 455 */ 456 if (ctr_mode) 457 ctx1_iv_off = 16; 458 459 /* 460 * RFC3686 specific: 461 * CONTEXT1[255:128] = {NONCE, IV, COUNTER} 462 */ 463 if (is_rfc3686) 464 ctx1_iv_off = 16 + CTR_RFC3686_NONCE_SIZE; 465 466 if (alg->caam.geniv) 467 goto skip_enc; 468 469 /* 470 * Job Descriptor and Shared Descriptors 471 * must all fit into the 64-word Descriptor h/w Buffer 472 */ 473 keys_fit_inline = false; 474 if (DESC_AEAD_ENC_LEN + AUTHENC_DESC_JOB_IO_LEN + 475 ctx->split_key_pad_len + ctx->enckeylen + 476 (is_rfc3686 ? DESC_AEAD_CTR_RFC3686_LEN : 0) <= 477 CAAM_DESC_BYTES_MAX) 478 keys_fit_inline = true; 479 480 /* aead_encrypt shared descriptor */ 481 desc = ctx->sh_desc_enc; 482 483 /* Note: Context registers are saved. */ 484 init_sh_desc_key_aead(desc, ctx, keys_fit_inline, is_rfc3686); 485 486 /* Class 2 operation */ 487 append_operation(desc, ctx->class2_alg_type | 488 OP_ALG_AS_INITFINAL | OP_ALG_ENCRYPT); 489 490 /* Read and write assoclen bytes */ 491 append_math_add(desc, VARSEQINLEN, ZERO, REG3, CAAM_CMD_SZ); 492 append_math_add(desc, VARSEQOUTLEN, ZERO, REG3, CAAM_CMD_SZ); 493 494 /* Skip assoc data */ 495 append_seq_fifo_store(desc, 0, FIFOST_TYPE_SKIP | FIFOLDST_VLF); 496 497 /* read assoc before reading payload */ 498 append_seq_fifo_load(desc, 0, FIFOLD_CLASS_CLASS2 | FIFOLD_TYPE_MSG | 499 FIFOLDST_VLF); 500 501 /* Load Counter into CONTEXT1 reg */ 502 if (is_rfc3686) 503 append_load_imm_u32(desc, be32_to_cpu(1), LDST_IMM | 504 LDST_CLASS_1_CCB | 505 LDST_SRCDST_BYTE_CONTEXT | 506 ((ctx1_iv_off + CTR_RFC3686_IV_SIZE) << 507 LDST_OFFSET_SHIFT)); 508 509 /* Class 1 operation */ 510 append_operation(desc, ctx->class1_alg_type | 511 OP_ALG_AS_INITFINAL | OP_ALG_ENCRYPT); 512 513 /* Read and write cryptlen bytes */ 514 append_math_add(desc, VARSEQINLEN, SEQINLEN, REG0, CAAM_CMD_SZ); 515 append_math_add(desc, VARSEQOUTLEN, SEQINLEN, REG0, CAAM_CMD_SZ); 516 aead_append_src_dst(desc, FIFOLD_TYPE_MSG1OUT2); 517 518 /* Write ICV */ 519 append_seq_store(desc, ctx->authsize, LDST_CLASS_2_CCB | 520 LDST_SRCDST_BYTE_CONTEXT); 521 522 ctx->sh_desc_enc_dma = dma_map_single(jrdev, desc, 523 desc_bytes(desc), 524 DMA_TO_DEVICE); 525 if (dma_mapping_error(jrdev, ctx->sh_desc_enc_dma)) { 526 dev_err(jrdev, "unable to map shared descriptor\n"); 527 return -ENOMEM; 528 } 529 #ifdef DEBUG 530 print_hex_dump(KERN_ERR, "aead enc shdesc@"__stringify(__LINE__)": ", 531 DUMP_PREFIX_ADDRESS, 16, 4, desc, 532 desc_bytes(desc), 1); 533 #endif 534 535 skip_enc: 536 /* 537 * Job Descriptor and Shared Descriptors 538 * must all fit into the 64-word Descriptor h/w Buffer 539 */ 540 keys_fit_inline = false; 541 if (DESC_AEAD_DEC_LEN + AUTHENC_DESC_JOB_IO_LEN + 542 ctx->split_key_pad_len + ctx->enckeylen + 543 (is_rfc3686 ? DESC_AEAD_CTR_RFC3686_LEN : 0) <= 544 CAAM_DESC_BYTES_MAX) 545 keys_fit_inline = true; 546 547 /* aead_decrypt shared descriptor */ 548 desc = ctx->sh_desc_dec; 549 550 /* Note: Context registers are saved. */ 551 init_sh_desc_key_aead(desc, ctx, keys_fit_inline, is_rfc3686); 552 553 /* Class 2 operation */ 554 append_operation(desc, ctx->class2_alg_type | 555 OP_ALG_AS_INITFINAL | OP_ALG_DECRYPT | OP_ALG_ICV_ON); 556 557 /* Read and write assoclen bytes */ 558 append_math_add(desc, VARSEQINLEN, ZERO, REG3, CAAM_CMD_SZ); 559 append_math_add(desc, VARSEQOUTLEN, ZERO, REG3, CAAM_CMD_SZ); 560 561 /* Skip assoc data */ 562 append_seq_fifo_store(desc, 0, FIFOST_TYPE_SKIP | FIFOLDST_VLF); 563 564 /* read assoc before reading payload */ 565 append_seq_fifo_load(desc, 0, FIFOLD_CLASS_CLASS2 | FIFOLD_TYPE_MSG | 566 KEY_VLF); 567 568 /* Load Counter into CONTEXT1 reg */ 569 if (is_rfc3686) 570 append_load_imm_u32(desc, be32_to_cpu(1), LDST_IMM | 571 LDST_CLASS_1_CCB | 572 LDST_SRCDST_BYTE_CONTEXT | 573 ((ctx1_iv_off + CTR_RFC3686_IV_SIZE) << 574 LDST_OFFSET_SHIFT)); 575 576 /* Choose operation */ 577 if (ctr_mode) 578 append_operation(desc, ctx->class1_alg_type | 579 OP_ALG_AS_INITFINAL | OP_ALG_DECRYPT); 580 else 581 append_dec_op1(desc, ctx->class1_alg_type); 582 583 /* Read and write cryptlen bytes */ 584 append_math_add(desc, VARSEQINLEN, SEQOUTLEN, REG0, CAAM_CMD_SZ); 585 append_math_add(desc, VARSEQOUTLEN, SEQOUTLEN, REG0, CAAM_CMD_SZ); 586 aead_append_src_dst(desc, FIFOLD_TYPE_MSG); 587 588 /* Load ICV */ 589 append_seq_fifo_load(desc, ctx->authsize, FIFOLD_CLASS_CLASS2 | 590 FIFOLD_TYPE_LAST2 | FIFOLD_TYPE_ICV); 591 592 ctx->sh_desc_dec_dma = dma_map_single(jrdev, desc, 593 desc_bytes(desc), 594 DMA_TO_DEVICE); 595 if (dma_mapping_error(jrdev, ctx->sh_desc_dec_dma)) { 596 dev_err(jrdev, "unable to map shared descriptor\n"); 597 return -ENOMEM; 598 } 599 #ifdef DEBUG 600 print_hex_dump(KERN_ERR, "aead dec shdesc@"__stringify(__LINE__)": ", 601 DUMP_PREFIX_ADDRESS, 16, 4, desc, 602 desc_bytes(desc), 1); 603 #endif 604 605 if (!alg->caam.geniv) 606 goto skip_givenc; 607 608 /* 609 * Job Descriptor and Shared Descriptors 610 * must all fit into the 64-word Descriptor h/w Buffer 611 */ 612 keys_fit_inline = false; 613 if (DESC_AEAD_GIVENC_LEN + AUTHENC_DESC_JOB_IO_LEN + 614 ctx->split_key_pad_len + ctx->enckeylen + 615 (is_rfc3686 ? DESC_AEAD_CTR_RFC3686_LEN : 0) <= 616 CAAM_DESC_BYTES_MAX) 617 keys_fit_inline = true; 618 619 /* aead_givencrypt shared descriptor */ 620 desc = ctx->sh_desc_enc; 621 622 /* Note: Context registers are saved. */ 623 init_sh_desc_key_aead(desc, ctx, keys_fit_inline, is_rfc3686); 624 625 if (is_rfc3686) 626 goto copy_iv; 627 628 /* Generate IV */ 629 geniv = NFIFOENTRY_STYPE_PAD | NFIFOENTRY_DEST_DECO | 630 NFIFOENTRY_DTYPE_MSG | NFIFOENTRY_LC1 | 631 NFIFOENTRY_PTYPE_RND | (ivsize << NFIFOENTRY_DLEN_SHIFT); 632 append_load_imm_u32(desc, geniv, LDST_CLASS_IND_CCB | 633 LDST_SRCDST_WORD_INFO_FIFO | LDST_IMM); 634 append_cmd(desc, CMD_LOAD | DISABLE_AUTO_INFO_FIFO); 635 append_move(desc, MOVE_WAITCOMP | 636 MOVE_SRC_INFIFO | MOVE_DEST_CLASS1CTX | 637 (ctx1_iv_off << MOVE_OFFSET_SHIFT) | 638 (ivsize << MOVE_LEN_SHIFT)); 639 append_cmd(desc, CMD_LOAD | ENABLE_AUTO_INFO_FIFO); 640 641 copy_iv: 642 /* Copy IV to class 1 context */ 643 append_move(desc, MOVE_SRC_CLASS1CTX | MOVE_DEST_OUTFIFO | 644 (ctx1_iv_off << MOVE_OFFSET_SHIFT) | 645 (ivsize << MOVE_LEN_SHIFT)); 646 647 /* Return to encryption */ 648 append_operation(desc, ctx->class2_alg_type | 649 OP_ALG_AS_INITFINAL | OP_ALG_ENCRYPT); 650 651 /* Read and write assoclen bytes */ 652 append_math_add(desc, VARSEQINLEN, ZERO, REG3, CAAM_CMD_SZ); 653 append_math_add(desc, VARSEQOUTLEN, ZERO, REG3, CAAM_CMD_SZ); 654 655 /* ivsize + cryptlen = seqoutlen - authsize */ 656 append_math_sub_imm_u32(desc, REG3, SEQOUTLEN, IMM, ctx->authsize); 657 658 /* Skip assoc data */ 659 append_seq_fifo_store(desc, 0, FIFOST_TYPE_SKIP | FIFOLDST_VLF); 660 661 /* read assoc before reading payload */ 662 append_seq_fifo_load(desc, 0, FIFOLD_CLASS_CLASS2 | FIFOLD_TYPE_MSG | 663 KEY_VLF); 664 665 /* Copy iv from outfifo to class 2 fifo */ 666 moveiv = NFIFOENTRY_STYPE_OFIFO | NFIFOENTRY_DEST_CLASS2 | 667 NFIFOENTRY_DTYPE_MSG | (ivsize << NFIFOENTRY_DLEN_SHIFT); 668 append_load_imm_u32(desc, moveiv, LDST_CLASS_IND_CCB | 669 LDST_SRCDST_WORD_INFO_FIFO | LDST_IMM); 670 append_load_imm_u32(desc, ivsize, LDST_CLASS_2_CCB | 671 LDST_SRCDST_WORD_DATASZ_REG | LDST_IMM); 672 673 /* Load Counter into CONTEXT1 reg */ 674 if (is_rfc3686) 675 append_load_imm_u32(desc, be32_to_cpu(1), LDST_IMM | 676 LDST_CLASS_1_CCB | 677 LDST_SRCDST_BYTE_CONTEXT | 678 ((ctx1_iv_off + CTR_RFC3686_IV_SIZE) << 679 LDST_OFFSET_SHIFT)); 680 681 /* Class 1 operation */ 682 append_operation(desc, ctx->class1_alg_type | 683 OP_ALG_AS_INITFINAL | OP_ALG_ENCRYPT); 684 685 /* Will write ivsize + cryptlen */ 686 append_math_add(desc, VARSEQOUTLEN, SEQINLEN, REG0, CAAM_CMD_SZ); 687 688 /* Not need to reload iv */ 689 append_seq_fifo_load(desc, ivsize, 690 FIFOLD_CLASS_SKIP); 691 692 /* Will read cryptlen */ 693 append_math_add(desc, VARSEQINLEN, SEQINLEN, REG0, CAAM_CMD_SZ); 694 aead_append_src_dst(desc, FIFOLD_TYPE_MSG1OUT2); 695 696 /* Write ICV */ 697 append_seq_store(desc, ctx->authsize, LDST_CLASS_2_CCB | 698 LDST_SRCDST_BYTE_CONTEXT); 699 700 ctx->sh_desc_enc_dma = dma_map_single(jrdev, desc, 701 desc_bytes(desc), 702 DMA_TO_DEVICE); 703 if (dma_mapping_error(jrdev, ctx->sh_desc_enc_dma)) { 704 dev_err(jrdev, "unable to map shared descriptor\n"); 705 return -ENOMEM; 706 } 707 #ifdef DEBUG 708 print_hex_dump(KERN_ERR, "aead givenc shdesc@"__stringify(__LINE__)": ", 709 DUMP_PREFIX_ADDRESS, 16, 4, desc, 710 desc_bytes(desc), 1); 711 #endif 712 713 skip_givenc: 714 return 0; 715 } 716 717 static int aead_setauthsize(struct crypto_aead *authenc, 718 unsigned int authsize) 719 { 720 struct caam_ctx *ctx = crypto_aead_ctx(authenc); 721 722 ctx->authsize = authsize; 723 aead_set_sh_desc(authenc); 724 725 return 0; 726 } 727 728 static int gcm_set_sh_desc(struct crypto_aead *aead) 729 { 730 struct caam_ctx *ctx = crypto_aead_ctx(aead); 731 struct device *jrdev = ctx->jrdev; 732 bool keys_fit_inline = false; 733 u32 *key_jump_cmd, *zero_payload_jump_cmd, 734 *zero_assoc_jump_cmd1, *zero_assoc_jump_cmd2; 735 u32 *desc; 736 737 if (!ctx->enckeylen || !ctx->authsize) 738 return 0; 739 740 /* 741 * AES GCM encrypt shared descriptor 742 * Job Descriptor and Shared Descriptor 743 * must fit into the 64-word Descriptor h/w Buffer 744 */ 745 if (DESC_GCM_ENC_LEN + GCM_DESC_JOB_IO_LEN + 746 ctx->enckeylen <= CAAM_DESC_BYTES_MAX) 747 keys_fit_inline = true; 748 749 desc = ctx->sh_desc_enc; 750 751 init_sh_desc(desc, HDR_SHARE_SERIAL); 752 753 /* skip key loading if they are loaded due to sharing */ 754 key_jump_cmd = append_jump(desc, JUMP_JSL | JUMP_TEST_ALL | 755 JUMP_COND_SHRD | JUMP_COND_SELF); 756 if (keys_fit_inline) 757 append_key_as_imm(desc, (void *)ctx->key, ctx->enckeylen, 758 ctx->enckeylen, CLASS_1 | KEY_DEST_CLASS_REG); 759 else 760 append_key(desc, ctx->key_dma, ctx->enckeylen, 761 CLASS_1 | KEY_DEST_CLASS_REG); 762 set_jump_tgt_here(desc, key_jump_cmd); 763 764 /* class 1 operation */ 765 append_operation(desc, ctx->class1_alg_type | 766 OP_ALG_AS_INITFINAL | OP_ALG_ENCRYPT); 767 768 /* if assoclen + cryptlen is ZERO, skip to ICV write */ 769 append_math_sub(desc, VARSEQOUTLEN, SEQINLEN, REG0, CAAM_CMD_SZ); 770 zero_assoc_jump_cmd2 = append_jump(desc, JUMP_TEST_ALL | 771 JUMP_COND_MATH_Z); 772 773 /* if assoclen is ZERO, skip reading the assoc data */ 774 append_math_add(desc, VARSEQINLEN, ZERO, REG3, CAAM_CMD_SZ); 775 zero_assoc_jump_cmd1 = append_jump(desc, JUMP_TEST_ALL | 776 JUMP_COND_MATH_Z); 777 778 append_math_add(desc, VARSEQOUTLEN, ZERO, REG3, CAAM_CMD_SZ); 779 780 /* skip assoc data */ 781 append_seq_fifo_store(desc, 0, FIFOST_TYPE_SKIP | FIFOLDST_VLF); 782 783 /* cryptlen = seqinlen - assoclen */ 784 append_math_sub(desc, VARSEQOUTLEN, SEQINLEN, REG3, CAAM_CMD_SZ); 785 786 /* if cryptlen is ZERO jump to zero-payload commands */ 787 zero_payload_jump_cmd = append_jump(desc, JUMP_TEST_ALL | 788 JUMP_COND_MATH_Z); 789 790 /* read assoc data */ 791 append_seq_fifo_load(desc, 0, FIFOLD_CLASS_CLASS1 | FIFOLDST_VLF | 792 FIFOLD_TYPE_AAD | FIFOLD_TYPE_FLUSH1); 793 set_jump_tgt_here(desc, zero_assoc_jump_cmd1); 794 795 append_math_sub(desc, VARSEQINLEN, SEQINLEN, REG0, CAAM_CMD_SZ); 796 797 /* write encrypted data */ 798 append_seq_fifo_store(desc, 0, FIFOST_TYPE_MESSAGE_DATA | FIFOLDST_VLF); 799 800 /* read payload data */ 801 append_seq_fifo_load(desc, 0, FIFOLD_CLASS_CLASS1 | FIFOLDST_VLF | 802 FIFOLD_TYPE_MSG | FIFOLD_TYPE_LAST1); 803 804 /* jump the zero-payload commands */ 805 append_jump(desc, JUMP_TEST_ALL | 2); 806 807 /* zero-payload commands */ 808 set_jump_tgt_here(desc, zero_payload_jump_cmd); 809 810 /* read assoc data */ 811 append_seq_fifo_load(desc, 0, FIFOLD_CLASS_CLASS1 | FIFOLDST_VLF | 812 FIFOLD_TYPE_AAD | FIFOLD_TYPE_LAST1); 813 814 /* There is no input data */ 815 set_jump_tgt_here(desc, zero_assoc_jump_cmd2); 816 817 /* write ICV */ 818 append_seq_store(desc, ctx->authsize, LDST_CLASS_1_CCB | 819 LDST_SRCDST_BYTE_CONTEXT); 820 821 ctx->sh_desc_enc_dma = dma_map_single(jrdev, desc, 822 desc_bytes(desc), 823 DMA_TO_DEVICE); 824 if (dma_mapping_error(jrdev, ctx->sh_desc_enc_dma)) { 825 dev_err(jrdev, "unable to map shared descriptor\n"); 826 return -ENOMEM; 827 } 828 #ifdef DEBUG 829 print_hex_dump(KERN_ERR, "gcm enc shdesc@"__stringify(__LINE__)": ", 830 DUMP_PREFIX_ADDRESS, 16, 4, desc, 831 desc_bytes(desc), 1); 832 #endif 833 834 /* 835 * Job Descriptor and Shared Descriptors 836 * must all fit into the 64-word Descriptor h/w Buffer 837 */ 838 keys_fit_inline = false; 839 if (DESC_GCM_DEC_LEN + GCM_DESC_JOB_IO_LEN + 840 ctx->enckeylen <= CAAM_DESC_BYTES_MAX) 841 keys_fit_inline = true; 842 843 desc = ctx->sh_desc_dec; 844 845 init_sh_desc(desc, HDR_SHARE_SERIAL); 846 847 /* skip key loading if they are loaded due to sharing */ 848 key_jump_cmd = append_jump(desc, JUMP_JSL | 849 JUMP_TEST_ALL | JUMP_COND_SHRD | 850 JUMP_COND_SELF); 851 if (keys_fit_inline) 852 append_key_as_imm(desc, (void *)ctx->key, ctx->enckeylen, 853 ctx->enckeylen, CLASS_1 | KEY_DEST_CLASS_REG); 854 else 855 append_key(desc, ctx->key_dma, ctx->enckeylen, 856 CLASS_1 | KEY_DEST_CLASS_REG); 857 set_jump_tgt_here(desc, key_jump_cmd); 858 859 /* class 1 operation */ 860 append_operation(desc, ctx->class1_alg_type | 861 OP_ALG_AS_INITFINAL | OP_ALG_DECRYPT | OP_ALG_ICV_ON); 862 863 /* if assoclen is ZERO, skip reading the assoc data */ 864 append_math_add(desc, VARSEQINLEN, ZERO, REG3, CAAM_CMD_SZ); 865 zero_assoc_jump_cmd1 = append_jump(desc, JUMP_TEST_ALL | 866 JUMP_COND_MATH_Z); 867 868 append_math_add(desc, VARSEQOUTLEN, ZERO, REG3, CAAM_CMD_SZ); 869 870 /* skip assoc data */ 871 append_seq_fifo_store(desc, 0, FIFOST_TYPE_SKIP | FIFOLDST_VLF); 872 873 /* read assoc data */ 874 append_seq_fifo_load(desc, 0, FIFOLD_CLASS_CLASS1 | FIFOLDST_VLF | 875 FIFOLD_TYPE_AAD | FIFOLD_TYPE_FLUSH1); 876 877 set_jump_tgt_here(desc, zero_assoc_jump_cmd1); 878 879 /* cryptlen = seqoutlen - assoclen */ 880 append_math_sub(desc, VARSEQINLEN, SEQOUTLEN, REG0, CAAM_CMD_SZ); 881 882 /* jump to zero-payload command if cryptlen is zero */ 883 zero_payload_jump_cmd = append_jump(desc, JUMP_TEST_ALL | 884 JUMP_COND_MATH_Z); 885 886 append_math_sub(desc, VARSEQOUTLEN, SEQOUTLEN, REG0, CAAM_CMD_SZ); 887 888 /* store encrypted data */ 889 append_seq_fifo_store(desc, 0, FIFOST_TYPE_MESSAGE_DATA | FIFOLDST_VLF); 890 891 /* read payload data */ 892 append_seq_fifo_load(desc, 0, FIFOLD_CLASS_CLASS1 | FIFOLDST_VLF | 893 FIFOLD_TYPE_MSG | FIFOLD_TYPE_FLUSH1); 894 895 /* zero-payload command */ 896 set_jump_tgt_here(desc, zero_payload_jump_cmd); 897 898 /* read ICV */ 899 append_seq_fifo_load(desc, ctx->authsize, FIFOLD_CLASS_CLASS1 | 900 FIFOLD_TYPE_ICV | FIFOLD_TYPE_LAST1); 901 902 ctx->sh_desc_dec_dma = dma_map_single(jrdev, desc, 903 desc_bytes(desc), 904 DMA_TO_DEVICE); 905 if (dma_mapping_error(jrdev, ctx->sh_desc_dec_dma)) { 906 dev_err(jrdev, "unable to map shared descriptor\n"); 907 return -ENOMEM; 908 } 909 #ifdef DEBUG 910 print_hex_dump(KERN_ERR, "gcm dec shdesc@"__stringify(__LINE__)": ", 911 DUMP_PREFIX_ADDRESS, 16, 4, desc, 912 desc_bytes(desc), 1); 913 #endif 914 915 return 0; 916 } 917 918 static int gcm_setauthsize(struct crypto_aead *authenc, unsigned int authsize) 919 { 920 struct caam_ctx *ctx = crypto_aead_ctx(authenc); 921 922 ctx->authsize = authsize; 923 gcm_set_sh_desc(authenc); 924 925 return 0; 926 } 927 928 static int rfc4106_set_sh_desc(struct crypto_aead *aead) 929 { 930 struct caam_ctx *ctx = crypto_aead_ctx(aead); 931 struct device *jrdev = ctx->jrdev; 932 bool keys_fit_inline = false; 933 u32 *key_jump_cmd; 934 u32 *desc; 935 936 if (!ctx->enckeylen || !ctx->authsize) 937 return 0; 938 939 /* 940 * RFC4106 encrypt shared descriptor 941 * Job Descriptor and Shared Descriptor 942 * must fit into the 64-word Descriptor h/w Buffer 943 */ 944 if (DESC_RFC4106_ENC_LEN + GCM_DESC_JOB_IO_LEN + 945 ctx->enckeylen <= CAAM_DESC_BYTES_MAX) 946 keys_fit_inline = true; 947 948 desc = ctx->sh_desc_enc; 949 950 init_sh_desc(desc, HDR_SHARE_SERIAL); 951 952 /* Skip key loading if it is loaded due to sharing */ 953 key_jump_cmd = append_jump(desc, JUMP_JSL | JUMP_TEST_ALL | 954 JUMP_COND_SHRD); 955 if (keys_fit_inline) 956 append_key_as_imm(desc, (void *)ctx->key, ctx->enckeylen, 957 ctx->enckeylen, CLASS_1 | KEY_DEST_CLASS_REG); 958 else 959 append_key(desc, ctx->key_dma, ctx->enckeylen, 960 CLASS_1 | KEY_DEST_CLASS_REG); 961 set_jump_tgt_here(desc, key_jump_cmd); 962 963 /* Class 1 operation */ 964 append_operation(desc, ctx->class1_alg_type | 965 OP_ALG_AS_INITFINAL | OP_ALG_ENCRYPT); 966 967 append_math_sub_imm_u32(desc, VARSEQINLEN, REG3, IMM, 8); 968 append_math_add(desc, VARSEQOUTLEN, ZERO, REG3, CAAM_CMD_SZ); 969 970 /* Read assoc data */ 971 append_seq_fifo_load(desc, 0, FIFOLD_CLASS_CLASS1 | FIFOLDST_VLF | 972 FIFOLD_TYPE_AAD | FIFOLD_TYPE_FLUSH1); 973 974 /* Skip IV */ 975 append_seq_fifo_load(desc, 8, FIFOLD_CLASS_SKIP); 976 977 /* Will read cryptlen bytes */ 978 append_math_sub(desc, VARSEQINLEN, SEQINLEN, REG0, CAAM_CMD_SZ); 979 980 /* Workaround for erratum A-005473 (simultaneous SEQ FIFO skips) */ 981 append_seq_fifo_load(desc, 0, FIFOLD_CLASS_CLASS1 | FIFOLD_TYPE_MSG); 982 983 /* Skip assoc data */ 984 append_seq_fifo_store(desc, 0, FIFOST_TYPE_SKIP | FIFOLDST_VLF); 985 986 /* cryptlen = seqoutlen - assoclen */ 987 append_math_sub(desc, VARSEQOUTLEN, VARSEQINLEN, REG0, CAAM_CMD_SZ); 988 989 /* Write encrypted data */ 990 append_seq_fifo_store(desc, 0, FIFOST_TYPE_MESSAGE_DATA | FIFOLDST_VLF); 991 992 /* Read payload data */ 993 append_seq_fifo_load(desc, 0, FIFOLD_CLASS_CLASS1 | FIFOLDST_VLF | 994 FIFOLD_TYPE_MSG | FIFOLD_TYPE_LAST1); 995 996 /* Write ICV */ 997 append_seq_store(desc, ctx->authsize, LDST_CLASS_1_CCB | 998 LDST_SRCDST_BYTE_CONTEXT); 999 1000 ctx->sh_desc_enc_dma = dma_map_single(jrdev, desc, 1001 desc_bytes(desc), 1002 DMA_TO_DEVICE); 1003 if (dma_mapping_error(jrdev, ctx->sh_desc_enc_dma)) { 1004 dev_err(jrdev, "unable to map shared descriptor\n"); 1005 return -ENOMEM; 1006 } 1007 #ifdef DEBUG 1008 print_hex_dump(KERN_ERR, "rfc4106 enc shdesc@"__stringify(__LINE__)": ", 1009 DUMP_PREFIX_ADDRESS, 16, 4, desc, 1010 desc_bytes(desc), 1); 1011 #endif 1012 1013 /* 1014 * Job Descriptor and Shared Descriptors 1015 * must all fit into the 64-word Descriptor h/w Buffer 1016 */ 1017 keys_fit_inline = false; 1018 if (DESC_RFC4106_DEC_LEN + DESC_JOB_IO_LEN + 1019 ctx->enckeylen <= CAAM_DESC_BYTES_MAX) 1020 keys_fit_inline = true; 1021 1022 desc = ctx->sh_desc_dec; 1023 1024 init_sh_desc(desc, HDR_SHARE_SERIAL); 1025 1026 /* Skip key loading if it is loaded due to sharing */ 1027 key_jump_cmd = append_jump(desc, JUMP_JSL | 1028 JUMP_TEST_ALL | JUMP_COND_SHRD); 1029 if (keys_fit_inline) 1030 append_key_as_imm(desc, (void *)ctx->key, ctx->enckeylen, 1031 ctx->enckeylen, CLASS_1 | KEY_DEST_CLASS_REG); 1032 else 1033 append_key(desc, ctx->key_dma, ctx->enckeylen, 1034 CLASS_1 | KEY_DEST_CLASS_REG); 1035 set_jump_tgt_here(desc, key_jump_cmd); 1036 1037 /* Class 1 operation */ 1038 append_operation(desc, ctx->class1_alg_type | 1039 OP_ALG_AS_INITFINAL | OP_ALG_DECRYPT | OP_ALG_ICV_ON); 1040 1041 append_math_sub_imm_u32(desc, VARSEQINLEN, REG3, IMM, 8); 1042 append_math_add(desc, VARSEQOUTLEN, ZERO, REG3, CAAM_CMD_SZ); 1043 1044 /* Read assoc data */ 1045 append_seq_fifo_load(desc, 0, FIFOLD_CLASS_CLASS1 | FIFOLDST_VLF | 1046 FIFOLD_TYPE_AAD | FIFOLD_TYPE_FLUSH1); 1047 1048 /* Skip IV */ 1049 append_seq_fifo_load(desc, 8, FIFOLD_CLASS_SKIP); 1050 1051 /* Will read cryptlen bytes */ 1052 append_math_sub(desc, VARSEQINLEN, SEQOUTLEN, REG3, CAAM_CMD_SZ); 1053 1054 /* Workaround for erratum A-005473 (simultaneous SEQ FIFO skips) */ 1055 append_seq_fifo_load(desc, 0, FIFOLD_CLASS_CLASS1 | FIFOLD_TYPE_MSG); 1056 1057 /* Skip assoc data */ 1058 append_seq_fifo_store(desc, 0, FIFOST_TYPE_SKIP | FIFOLDST_VLF); 1059 1060 /* Will write cryptlen bytes */ 1061 append_math_sub(desc, VARSEQOUTLEN, SEQOUTLEN, REG0, CAAM_CMD_SZ); 1062 1063 /* Store payload data */ 1064 append_seq_fifo_store(desc, 0, FIFOST_TYPE_MESSAGE_DATA | FIFOLDST_VLF); 1065 1066 /* Read encrypted data */ 1067 append_seq_fifo_load(desc, 0, FIFOLD_CLASS_CLASS1 | FIFOLDST_VLF | 1068 FIFOLD_TYPE_MSG | FIFOLD_TYPE_FLUSH1); 1069 1070 /* Read ICV */ 1071 append_seq_fifo_load(desc, ctx->authsize, FIFOLD_CLASS_CLASS1 | 1072 FIFOLD_TYPE_ICV | FIFOLD_TYPE_LAST1); 1073 1074 ctx->sh_desc_dec_dma = dma_map_single(jrdev, desc, 1075 desc_bytes(desc), 1076 DMA_TO_DEVICE); 1077 if (dma_mapping_error(jrdev, ctx->sh_desc_dec_dma)) { 1078 dev_err(jrdev, "unable to map shared descriptor\n"); 1079 return -ENOMEM; 1080 } 1081 #ifdef DEBUG 1082 print_hex_dump(KERN_ERR, "rfc4106 dec shdesc@"__stringify(__LINE__)": ", 1083 DUMP_PREFIX_ADDRESS, 16, 4, desc, 1084 desc_bytes(desc), 1); 1085 #endif 1086 1087 return 0; 1088 } 1089 1090 static int rfc4106_setauthsize(struct crypto_aead *authenc, 1091 unsigned int authsize) 1092 { 1093 struct caam_ctx *ctx = crypto_aead_ctx(authenc); 1094 1095 ctx->authsize = authsize; 1096 rfc4106_set_sh_desc(authenc); 1097 1098 return 0; 1099 } 1100 1101 static int rfc4543_set_sh_desc(struct crypto_aead *aead) 1102 { 1103 struct caam_ctx *ctx = crypto_aead_ctx(aead); 1104 struct device *jrdev = ctx->jrdev; 1105 bool keys_fit_inline = false; 1106 u32 *key_jump_cmd; 1107 u32 *read_move_cmd, *write_move_cmd; 1108 u32 *desc; 1109 1110 if (!ctx->enckeylen || !ctx->authsize) 1111 return 0; 1112 1113 /* 1114 * RFC4543 encrypt shared descriptor 1115 * Job Descriptor and Shared Descriptor 1116 * must fit into the 64-word Descriptor h/w Buffer 1117 */ 1118 if (DESC_RFC4543_ENC_LEN + GCM_DESC_JOB_IO_LEN + 1119 ctx->enckeylen <= CAAM_DESC_BYTES_MAX) 1120 keys_fit_inline = true; 1121 1122 desc = ctx->sh_desc_enc; 1123 1124 init_sh_desc(desc, HDR_SHARE_SERIAL); 1125 1126 /* Skip key loading if it is loaded due to sharing */ 1127 key_jump_cmd = append_jump(desc, JUMP_JSL | JUMP_TEST_ALL | 1128 JUMP_COND_SHRD); 1129 if (keys_fit_inline) 1130 append_key_as_imm(desc, (void *)ctx->key, ctx->enckeylen, 1131 ctx->enckeylen, CLASS_1 | KEY_DEST_CLASS_REG); 1132 else 1133 append_key(desc, ctx->key_dma, ctx->enckeylen, 1134 CLASS_1 | KEY_DEST_CLASS_REG); 1135 set_jump_tgt_here(desc, key_jump_cmd); 1136 1137 /* Class 1 operation */ 1138 append_operation(desc, ctx->class1_alg_type | 1139 OP_ALG_AS_INITFINAL | OP_ALG_ENCRYPT); 1140 1141 /* assoclen + cryptlen = seqinlen */ 1142 append_math_sub(desc, REG3, SEQINLEN, REG0, CAAM_CMD_SZ); 1143 1144 /* 1145 * MOVE_LEN opcode is not available in all SEC HW revisions, 1146 * thus need to do some magic, i.e. self-patch the descriptor 1147 * buffer. 1148 */ 1149 read_move_cmd = append_move(desc, MOVE_SRC_DESCBUF | MOVE_DEST_MATH3 | 1150 (0x6 << MOVE_LEN_SHIFT)); 1151 write_move_cmd = append_move(desc, MOVE_SRC_MATH3 | MOVE_DEST_DESCBUF | 1152 (0x8 << MOVE_LEN_SHIFT)); 1153 1154 /* Will read assoclen + cryptlen bytes */ 1155 append_math_sub(desc, VARSEQINLEN, SEQINLEN, REG0, CAAM_CMD_SZ); 1156 1157 /* Will write assoclen + cryptlen bytes */ 1158 append_math_sub(desc, VARSEQOUTLEN, SEQINLEN, REG0, CAAM_CMD_SZ); 1159 1160 /* Read and write assoclen + cryptlen bytes */ 1161 aead_append_src_dst(desc, FIFOLD_TYPE_AAD); 1162 1163 set_move_tgt_here(desc, read_move_cmd); 1164 set_move_tgt_here(desc, write_move_cmd); 1165 append_cmd(desc, CMD_LOAD | DISABLE_AUTO_INFO_FIFO); 1166 /* Move payload data to OFIFO */ 1167 append_move(desc, MOVE_SRC_INFIFO_CL | MOVE_DEST_OUTFIFO); 1168 1169 /* Write ICV */ 1170 append_seq_store(desc, ctx->authsize, LDST_CLASS_1_CCB | 1171 LDST_SRCDST_BYTE_CONTEXT); 1172 1173 ctx->sh_desc_enc_dma = dma_map_single(jrdev, desc, 1174 desc_bytes(desc), 1175 DMA_TO_DEVICE); 1176 if (dma_mapping_error(jrdev, ctx->sh_desc_enc_dma)) { 1177 dev_err(jrdev, "unable to map shared descriptor\n"); 1178 return -ENOMEM; 1179 } 1180 #ifdef DEBUG 1181 print_hex_dump(KERN_ERR, "rfc4543 enc shdesc@"__stringify(__LINE__)": ", 1182 DUMP_PREFIX_ADDRESS, 16, 4, desc, 1183 desc_bytes(desc), 1); 1184 #endif 1185 1186 /* 1187 * Job Descriptor and Shared Descriptors 1188 * must all fit into the 64-word Descriptor h/w Buffer 1189 */ 1190 keys_fit_inline = false; 1191 if (DESC_RFC4543_DEC_LEN + GCM_DESC_JOB_IO_LEN + 1192 ctx->enckeylen <= CAAM_DESC_BYTES_MAX) 1193 keys_fit_inline = true; 1194 1195 desc = ctx->sh_desc_dec; 1196 1197 init_sh_desc(desc, HDR_SHARE_SERIAL); 1198 1199 /* Skip key loading if it is loaded due to sharing */ 1200 key_jump_cmd = append_jump(desc, JUMP_JSL | 1201 JUMP_TEST_ALL | JUMP_COND_SHRD); 1202 if (keys_fit_inline) 1203 append_key_as_imm(desc, (void *)ctx->key, ctx->enckeylen, 1204 ctx->enckeylen, CLASS_1 | KEY_DEST_CLASS_REG); 1205 else 1206 append_key(desc, ctx->key_dma, ctx->enckeylen, 1207 CLASS_1 | KEY_DEST_CLASS_REG); 1208 set_jump_tgt_here(desc, key_jump_cmd); 1209 1210 /* Class 1 operation */ 1211 append_operation(desc, ctx->class1_alg_type | 1212 OP_ALG_AS_INITFINAL | OP_ALG_DECRYPT | OP_ALG_ICV_ON); 1213 1214 /* assoclen + cryptlen = seqoutlen */ 1215 append_math_sub(desc, REG3, SEQOUTLEN, REG0, CAAM_CMD_SZ); 1216 1217 /* 1218 * MOVE_LEN opcode is not available in all SEC HW revisions, 1219 * thus need to do some magic, i.e. self-patch the descriptor 1220 * buffer. 1221 */ 1222 read_move_cmd = append_move(desc, MOVE_SRC_DESCBUF | MOVE_DEST_MATH3 | 1223 (0x6 << MOVE_LEN_SHIFT)); 1224 write_move_cmd = append_move(desc, MOVE_SRC_MATH3 | MOVE_DEST_DESCBUF | 1225 (0x8 << MOVE_LEN_SHIFT)); 1226 1227 /* Will read assoclen + cryptlen bytes */ 1228 append_math_sub(desc, VARSEQINLEN, SEQOUTLEN, REG0, CAAM_CMD_SZ); 1229 1230 /* Will write assoclen + cryptlen bytes */ 1231 append_math_sub(desc, VARSEQOUTLEN, SEQOUTLEN, REG0, CAAM_CMD_SZ); 1232 1233 /* Store payload data */ 1234 append_seq_fifo_store(desc, 0, FIFOST_TYPE_MESSAGE_DATA | FIFOLDST_VLF); 1235 1236 /* In-snoop assoclen + cryptlen data */ 1237 append_seq_fifo_load(desc, 0, FIFOLD_CLASS_BOTH | FIFOLDST_VLF | 1238 FIFOLD_TYPE_AAD | FIFOLD_TYPE_LAST2FLUSH1); 1239 1240 set_move_tgt_here(desc, read_move_cmd); 1241 set_move_tgt_here(desc, write_move_cmd); 1242 append_cmd(desc, CMD_LOAD | DISABLE_AUTO_INFO_FIFO); 1243 /* Move payload data to OFIFO */ 1244 append_move(desc, MOVE_SRC_INFIFO_CL | MOVE_DEST_OUTFIFO); 1245 append_cmd(desc, CMD_LOAD | ENABLE_AUTO_INFO_FIFO); 1246 1247 /* Read ICV */ 1248 append_seq_fifo_load(desc, ctx->authsize, FIFOLD_CLASS_CLASS1 | 1249 FIFOLD_TYPE_ICV | FIFOLD_TYPE_LAST1); 1250 1251 ctx->sh_desc_dec_dma = dma_map_single(jrdev, desc, 1252 desc_bytes(desc), 1253 DMA_TO_DEVICE); 1254 if (dma_mapping_error(jrdev, ctx->sh_desc_dec_dma)) { 1255 dev_err(jrdev, "unable to map shared descriptor\n"); 1256 return -ENOMEM; 1257 } 1258 #ifdef DEBUG 1259 print_hex_dump(KERN_ERR, "rfc4543 dec shdesc@"__stringify(__LINE__)": ", 1260 DUMP_PREFIX_ADDRESS, 16, 4, desc, 1261 desc_bytes(desc), 1); 1262 #endif 1263 1264 return 0; 1265 } 1266 1267 static int rfc4543_setauthsize(struct crypto_aead *authenc, 1268 unsigned int authsize) 1269 { 1270 struct caam_ctx *ctx = crypto_aead_ctx(authenc); 1271 1272 ctx->authsize = authsize; 1273 rfc4543_set_sh_desc(authenc); 1274 1275 return 0; 1276 } 1277 1278 static u32 gen_split_aead_key(struct caam_ctx *ctx, const u8 *key_in, 1279 u32 authkeylen) 1280 { 1281 return gen_split_key(ctx->jrdev, ctx->key, ctx->split_key_len, 1282 ctx->split_key_pad_len, key_in, authkeylen, 1283 ctx->alg_op); 1284 } 1285 1286 static int aead_setkey(struct crypto_aead *aead, 1287 const u8 *key, unsigned int keylen) 1288 { 1289 /* Sizes for MDHA pads (*not* keys): MD5, SHA1, 224, 256, 384, 512 */ 1290 static const u8 mdpadlen[] = { 16, 20, 32, 32, 64, 64 }; 1291 struct caam_ctx *ctx = crypto_aead_ctx(aead); 1292 struct device *jrdev = ctx->jrdev; 1293 struct crypto_authenc_keys keys; 1294 int ret = 0; 1295 1296 if (crypto_authenc_extractkeys(&keys, key, keylen) != 0) 1297 goto badkey; 1298 1299 /* Pick class 2 key length from algorithm submask */ 1300 ctx->split_key_len = mdpadlen[(ctx->alg_op & OP_ALG_ALGSEL_SUBMASK) >> 1301 OP_ALG_ALGSEL_SHIFT] * 2; 1302 ctx->split_key_pad_len = ALIGN(ctx->split_key_len, 16); 1303 1304 if (ctx->split_key_pad_len + keys.enckeylen > CAAM_MAX_KEY_SIZE) 1305 goto badkey; 1306 1307 #ifdef DEBUG 1308 printk(KERN_ERR "keylen %d enckeylen %d authkeylen %d\n", 1309 keys.authkeylen + keys.enckeylen, keys.enckeylen, 1310 keys.authkeylen); 1311 printk(KERN_ERR "split_key_len %d split_key_pad_len %d\n", 1312 ctx->split_key_len, ctx->split_key_pad_len); 1313 print_hex_dump(KERN_ERR, "key in @"__stringify(__LINE__)": ", 1314 DUMP_PREFIX_ADDRESS, 16, 4, key, keylen, 1); 1315 #endif 1316 1317 ret = gen_split_aead_key(ctx, keys.authkey, keys.authkeylen); 1318 if (ret) { 1319 goto badkey; 1320 } 1321 1322 /* postpend encryption key to auth split key */ 1323 memcpy(ctx->key + ctx->split_key_pad_len, keys.enckey, keys.enckeylen); 1324 1325 ctx->key_dma = dma_map_single(jrdev, ctx->key, ctx->split_key_pad_len + 1326 keys.enckeylen, DMA_TO_DEVICE); 1327 if (dma_mapping_error(jrdev, ctx->key_dma)) { 1328 dev_err(jrdev, "unable to map key i/o memory\n"); 1329 return -ENOMEM; 1330 } 1331 #ifdef DEBUG 1332 print_hex_dump(KERN_ERR, "ctx.key@"__stringify(__LINE__)": ", 1333 DUMP_PREFIX_ADDRESS, 16, 4, ctx->key, 1334 ctx->split_key_pad_len + keys.enckeylen, 1); 1335 #endif 1336 1337 ctx->enckeylen = keys.enckeylen; 1338 1339 ret = aead_set_sh_desc(aead); 1340 if (ret) { 1341 dma_unmap_single(jrdev, ctx->key_dma, ctx->split_key_pad_len + 1342 keys.enckeylen, DMA_TO_DEVICE); 1343 } 1344 1345 return ret; 1346 badkey: 1347 crypto_aead_set_flags(aead, CRYPTO_TFM_RES_BAD_KEY_LEN); 1348 return -EINVAL; 1349 } 1350 1351 static int gcm_setkey(struct crypto_aead *aead, 1352 const u8 *key, unsigned int keylen) 1353 { 1354 struct caam_ctx *ctx = crypto_aead_ctx(aead); 1355 struct device *jrdev = ctx->jrdev; 1356 int ret = 0; 1357 1358 #ifdef DEBUG 1359 print_hex_dump(KERN_ERR, "key in @"__stringify(__LINE__)": ", 1360 DUMP_PREFIX_ADDRESS, 16, 4, key, keylen, 1); 1361 #endif 1362 1363 memcpy(ctx->key, key, keylen); 1364 ctx->key_dma = dma_map_single(jrdev, ctx->key, keylen, 1365 DMA_TO_DEVICE); 1366 if (dma_mapping_error(jrdev, ctx->key_dma)) { 1367 dev_err(jrdev, "unable to map key i/o memory\n"); 1368 return -ENOMEM; 1369 } 1370 ctx->enckeylen = keylen; 1371 1372 ret = gcm_set_sh_desc(aead); 1373 if (ret) { 1374 dma_unmap_single(jrdev, ctx->key_dma, ctx->enckeylen, 1375 DMA_TO_DEVICE); 1376 } 1377 1378 return ret; 1379 } 1380 1381 static int rfc4106_setkey(struct crypto_aead *aead, 1382 const u8 *key, unsigned int keylen) 1383 { 1384 struct caam_ctx *ctx = crypto_aead_ctx(aead); 1385 struct device *jrdev = ctx->jrdev; 1386 int ret = 0; 1387 1388 if (keylen < 4) 1389 return -EINVAL; 1390 1391 #ifdef DEBUG 1392 print_hex_dump(KERN_ERR, "key in @"__stringify(__LINE__)": ", 1393 DUMP_PREFIX_ADDRESS, 16, 4, key, keylen, 1); 1394 #endif 1395 1396 memcpy(ctx->key, key, keylen); 1397 1398 /* 1399 * The last four bytes of the key material are used as the salt value 1400 * in the nonce. Update the AES key length. 1401 */ 1402 ctx->enckeylen = keylen - 4; 1403 1404 ctx->key_dma = dma_map_single(jrdev, ctx->key, ctx->enckeylen, 1405 DMA_TO_DEVICE); 1406 if (dma_mapping_error(jrdev, ctx->key_dma)) { 1407 dev_err(jrdev, "unable to map key i/o memory\n"); 1408 return -ENOMEM; 1409 } 1410 1411 ret = rfc4106_set_sh_desc(aead); 1412 if (ret) { 1413 dma_unmap_single(jrdev, ctx->key_dma, ctx->enckeylen, 1414 DMA_TO_DEVICE); 1415 } 1416 1417 return ret; 1418 } 1419 1420 static int rfc4543_setkey(struct crypto_aead *aead, 1421 const u8 *key, unsigned int keylen) 1422 { 1423 struct caam_ctx *ctx = crypto_aead_ctx(aead); 1424 struct device *jrdev = ctx->jrdev; 1425 int ret = 0; 1426 1427 if (keylen < 4) 1428 return -EINVAL; 1429 1430 #ifdef DEBUG 1431 print_hex_dump(KERN_ERR, "key in @"__stringify(__LINE__)": ", 1432 DUMP_PREFIX_ADDRESS, 16, 4, key, keylen, 1); 1433 #endif 1434 1435 memcpy(ctx->key, key, keylen); 1436 1437 /* 1438 * The last four bytes of the key material are used as the salt value 1439 * in the nonce. Update the AES key length. 1440 */ 1441 ctx->enckeylen = keylen - 4; 1442 1443 ctx->key_dma = dma_map_single(jrdev, ctx->key, ctx->enckeylen, 1444 DMA_TO_DEVICE); 1445 if (dma_mapping_error(jrdev, ctx->key_dma)) { 1446 dev_err(jrdev, "unable to map key i/o memory\n"); 1447 return -ENOMEM; 1448 } 1449 1450 ret = rfc4543_set_sh_desc(aead); 1451 if (ret) { 1452 dma_unmap_single(jrdev, ctx->key_dma, ctx->enckeylen, 1453 DMA_TO_DEVICE); 1454 } 1455 1456 return ret; 1457 } 1458 1459 static int ablkcipher_setkey(struct crypto_ablkcipher *ablkcipher, 1460 const u8 *key, unsigned int keylen) 1461 { 1462 struct caam_ctx *ctx = crypto_ablkcipher_ctx(ablkcipher); 1463 struct ablkcipher_tfm *crt = &ablkcipher->base.crt_ablkcipher; 1464 struct crypto_tfm *tfm = crypto_ablkcipher_tfm(ablkcipher); 1465 const char *alg_name = crypto_tfm_alg_name(tfm); 1466 struct device *jrdev = ctx->jrdev; 1467 int ret = 0; 1468 u32 *key_jump_cmd; 1469 u32 *desc; 1470 u32 *nonce; 1471 u32 geniv; 1472 u32 ctx1_iv_off = 0; 1473 const bool ctr_mode = ((ctx->class1_alg_type & OP_ALG_AAI_MASK) == 1474 OP_ALG_AAI_CTR_MOD128); 1475 const bool is_rfc3686 = (ctr_mode && 1476 (strstr(alg_name, "rfc3686") != NULL)); 1477 1478 #ifdef DEBUG 1479 print_hex_dump(KERN_ERR, "key in @"__stringify(__LINE__)": ", 1480 DUMP_PREFIX_ADDRESS, 16, 4, key, keylen, 1); 1481 #endif 1482 /* 1483 * AES-CTR needs to load IV in CONTEXT1 reg 1484 * at an offset of 128bits (16bytes) 1485 * CONTEXT1[255:128] = IV 1486 */ 1487 if (ctr_mode) 1488 ctx1_iv_off = 16; 1489 1490 /* 1491 * RFC3686 specific: 1492 * | CONTEXT1[255:128] = {NONCE, IV, COUNTER} 1493 * | *key = {KEY, NONCE} 1494 */ 1495 if (is_rfc3686) { 1496 ctx1_iv_off = 16 + CTR_RFC3686_NONCE_SIZE; 1497 keylen -= CTR_RFC3686_NONCE_SIZE; 1498 } 1499 1500 memcpy(ctx->key, key, keylen); 1501 ctx->key_dma = dma_map_single(jrdev, ctx->key, keylen, 1502 DMA_TO_DEVICE); 1503 if (dma_mapping_error(jrdev, ctx->key_dma)) { 1504 dev_err(jrdev, "unable to map key i/o memory\n"); 1505 return -ENOMEM; 1506 } 1507 ctx->enckeylen = keylen; 1508 1509 /* ablkcipher_encrypt shared descriptor */ 1510 desc = ctx->sh_desc_enc; 1511 init_sh_desc(desc, HDR_SHARE_SERIAL | HDR_SAVECTX); 1512 /* Skip if already shared */ 1513 key_jump_cmd = append_jump(desc, JUMP_JSL | JUMP_TEST_ALL | 1514 JUMP_COND_SHRD); 1515 1516 /* Load class1 key only */ 1517 append_key_as_imm(desc, (void *)ctx->key, ctx->enckeylen, 1518 ctx->enckeylen, CLASS_1 | 1519 KEY_DEST_CLASS_REG); 1520 1521 /* Load nonce into CONTEXT1 reg */ 1522 if (is_rfc3686) { 1523 nonce = (u32 *)(key + keylen); 1524 append_load_imm_u32(desc, *nonce, LDST_CLASS_IND_CCB | 1525 LDST_SRCDST_BYTE_OUTFIFO | LDST_IMM); 1526 append_move(desc, MOVE_WAITCOMP | 1527 MOVE_SRC_OUTFIFO | 1528 MOVE_DEST_CLASS1CTX | 1529 (16 << MOVE_OFFSET_SHIFT) | 1530 (CTR_RFC3686_NONCE_SIZE << MOVE_LEN_SHIFT)); 1531 } 1532 1533 set_jump_tgt_here(desc, key_jump_cmd); 1534 1535 /* Load iv */ 1536 append_seq_load(desc, crt->ivsize, LDST_SRCDST_BYTE_CONTEXT | 1537 LDST_CLASS_1_CCB | (ctx1_iv_off << LDST_OFFSET_SHIFT)); 1538 1539 /* Load counter into CONTEXT1 reg */ 1540 if (is_rfc3686) 1541 append_load_imm_u32(desc, be32_to_cpu(1), LDST_IMM | 1542 LDST_CLASS_1_CCB | 1543 LDST_SRCDST_BYTE_CONTEXT | 1544 ((ctx1_iv_off + CTR_RFC3686_IV_SIZE) << 1545 LDST_OFFSET_SHIFT)); 1546 1547 /* Load operation */ 1548 append_operation(desc, ctx->class1_alg_type | 1549 OP_ALG_AS_INITFINAL | OP_ALG_ENCRYPT); 1550 1551 /* Perform operation */ 1552 ablkcipher_append_src_dst(desc); 1553 1554 ctx->sh_desc_enc_dma = dma_map_single(jrdev, desc, 1555 desc_bytes(desc), 1556 DMA_TO_DEVICE); 1557 if (dma_mapping_error(jrdev, ctx->sh_desc_enc_dma)) { 1558 dev_err(jrdev, "unable to map shared descriptor\n"); 1559 return -ENOMEM; 1560 } 1561 #ifdef DEBUG 1562 print_hex_dump(KERN_ERR, 1563 "ablkcipher enc shdesc@"__stringify(__LINE__)": ", 1564 DUMP_PREFIX_ADDRESS, 16, 4, desc, 1565 desc_bytes(desc), 1); 1566 #endif 1567 /* ablkcipher_decrypt shared descriptor */ 1568 desc = ctx->sh_desc_dec; 1569 1570 init_sh_desc(desc, HDR_SHARE_SERIAL | HDR_SAVECTX); 1571 /* Skip if already shared */ 1572 key_jump_cmd = append_jump(desc, JUMP_JSL | JUMP_TEST_ALL | 1573 JUMP_COND_SHRD); 1574 1575 /* Load class1 key only */ 1576 append_key_as_imm(desc, (void *)ctx->key, ctx->enckeylen, 1577 ctx->enckeylen, CLASS_1 | 1578 KEY_DEST_CLASS_REG); 1579 1580 /* Load nonce into CONTEXT1 reg */ 1581 if (is_rfc3686) { 1582 nonce = (u32 *)(key + keylen); 1583 append_load_imm_u32(desc, *nonce, LDST_CLASS_IND_CCB | 1584 LDST_SRCDST_BYTE_OUTFIFO | LDST_IMM); 1585 append_move(desc, MOVE_WAITCOMP | 1586 MOVE_SRC_OUTFIFO | 1587 MOVE_DEST_CLASS1CTX | 1588 (16 << MOVE_OFFSET_SHIFT) | 1589 (CTR_RFC3686_NONCE_SIZE << MOVE_LEN_SHIFT)); 1590 } 1591 1592 set_jump_tgt_here(desc, key_jump_cmd); 1593 1594 /* load IV */ 1595 append_seq_load(desc, crt->ivsize, LDST_SRCDST_BYTE_CONTEXT | 1596 LDST_CLASS_1_CCB | (ctx1_iv_off << LDST_OFFSET_SHIFT)); 1597 1598 /* Load counter into CONTEXT1 reg */ 1599 if (is_rfc3686) 1600 append_load_imm_u32(desc, be32_to_cpu(1), LDST_IMM | 1601 LDST_CLASS_1_CCB | 1602 LDST_SRCDST_BYTE_CONTEXT | 1603 ((ctx1_iv_off + CTR_RFC3686_IV_SIZE) << 1604 LDST_OFFSET_SHIFT)); 1605 1606 /* Choose operation */ 1607 if (ctr_mode) 1608 append_operation(desc, ctx->class1_alg_type | 1609 OP_ALG_AS_INITFINAL | OP_ALG_DECRYPT); 1610 else 1611 append_dec_op1(desc, ctx->class1_alg_type); 1612 1613 /* Perform operation */ 1614 ablkcipher_append_src_dst(desc); 1615 1616 ctx->sh_desc_dec_dma = dma_map_single(jrdev, desc, 1617 desc_bytes(desc), 1618 DMA_TO_DEVICE); 1619 if (dma_mapping_error(jrdev, ctx->sh_desc_dec_dma)) { 1620 dev_err(jrdev, "unable to map shared descriptor\n"); 1621 return -ENOMEM; 1622 } 1623 1624 #ifdef DEBUG 1625 print_hex_dump(KERN_ERR, 1626 "ablkcipher dec shdesc@"__stringify(__LINE__)": ", 1627 DUMP_PREFIX_ADDRESS, 16, 4, desc, 1628 desc_bytes(desc), 1); 1629 #endif 1630 /* ablkcipher_givencrypt shared descriptor */ 1631 desc = ctx->sh_desc_givenc; 1632 1633 init_sh_desc(desc, HDR_SHARE_SERIAL | HDR_SAVECTX); 1634 /* Skip if already shared */ 1635 key_jump_cmd = append_jump(desc, JUMP_JSL | JUMP_TEST_ALL | 1636 JUMP_COND_SHRD); 1637 1638 /* Load class1 key only */ 1639 append_key_as_imm(desc, (void *)ctx->key, ctx->enckeylen, 1640 ctx->enckeylen, CLASS_1 | 1641 KEY_DEST_CLASS_REG); 1642 1643 /* Load Nonce into CONTEXT1 reg */ 1644 if (is_rfc3686) { 1645 nonce = (u32 *)(key + keylen); 1646 append_load_imm_u32(desc, *nonce, LDST_CLASS_IND_CCB | 1647 LDST_SRCDST_BYTE_OUTFIFO | LDST_IMM); 1648 append_move(desc, MOVE_WAITCOMP | 1649 MOVE_SRC_OUTFIFO | 1650 MOVE_DEST_CLASS1CTX | 1651 (16 << MOVE_OFFSET_SHIFT) | 1652 (CTR_RFC3686_NONCE_SIZE << MOVE_LEN_SHIFT)); 1653 } 1654 set_jump_tgt_here(desc, key_jump_cmd); 1655 1656 /* Generate IV */ 1657 geniv = NFIFOENTRY_STYPE_PAD | NFIFOENTRY_DEST_DECO | 1658 NFIFOENTRY_DTYPE_MSG | NFIFOENTRY_LC1 | 1659 NFIFOENTRY_PTYPE_RND | (crt->ivsize << NFIFOENTRY_DLEN_SHIFT); 1660 append_load_imm_u32(desc, geniv, LDST_CLASS_IND_CCB | 1661 LDST_SRCDST_WORD_INFO_FIFO | LDST_IMM); 1662 append_cmd(desc, CMD_LOAD | DISABLE_AUTO_INFO_FIFO); 1663 append_move(desc, MOVE_WAITCOMP | 1664 MOVE_SRC_INFIFO | 1665 MOVE_DEST_CLASS1CTX | 1666 (crt->ivsize << MOVE_LEN_SHIFT) | 1667 (ctx1_iv_off << MOVE_OFFSET_SHIFT)); 1668 append_cmd(desc, CMD_LOAD | ENABLE_AUTO_INFO_FIFO); 1669 1670 /* Copy generated IV to memory */ 1671 append_seq_store(desc, crt->ivsize, 1672 LDST_SRCDST_BYTE_CONTEXT | LDST_CLASS_1_CCB | 1673 (ctx1_iv_off << LDST_OFFSET_SHIFT)); 1674 1675 /* Load Counter into CONTEXT1 reg */ 1676 if (is_rfc3686) 1677 append_load_imm_u32(desc, (u32)1, LDST_IMM | 1678 LDST_CLASS_1_CCB | 1679 LDST_SRCDST_BYTE_CONTEXT | 1680 ((ctx1_iv_off + CTR_RFC3686_IV_SIZE) << 1681 LDST_OFFSET_SHIFT)); 1682 1683 if (ctx1_iv_off) 1684 append_jump(desc, JUMP_JSL | JUMP_TEST_ALL | JUMP_COND_NCP | 1685 (1 << JUMP_OFFSET_SHIFT)); 1686 1687 /* Load operation */ 1688 append_operation(desc, ctx->class1_alg_type | 1689 OP_ALG_AS_INITFINAL | OP_ALG_ENCRYPT); 1690 1691 /* Perform operation */ 1692 ablkcipher_append_src_dst(desc); 1693 1694 ctx->sh_desc_givenc_dma = dma_map_single(jrdev, desc, 1695 desc_bytes(desc), 1696 DMA_TO_DEVICE); 1697 if (dma_mapping_error(jrdev, ctx->sh_desc_givenc_dma)) { 1698 dev_err(jrdev, "unable to map shared descriptor\n"); 1699 return -ENOMEM; 1700 } 1701 #ifdef DEBUG 1702 print_hex_dump(KERN_ERR, 1703 "ablkcipher givenc shdesc@" __stringify(__LINE__) ": ", 1704 DUMP_PREFIX_ADDRESS, 16, 4, desc, 1705 desc_bytes(desc), 1); 1706 #endif 1707 1708 return ret; 1709 } 1710 1711 static int xts_ablkcipher_setkey(struct crypto_ablkcipher *ablkcipher, 1712 const u8 *key, unsigned int keylen) 1713 { 1714 struct caam_ctx *ctx = crypto_ablkcipher_ctx(ablkcipher); 1715 struct device *jrdev = ctx->jrdev; 1716 u32 *key_jump_cmd, *desc; 1717 __be64 sector_size = cpu_to_be64(512); 1718 1719 if (keylen != 2 * AES_MIN_KEY_SIZE && keylen != 2 * AES_MAX_KEY_SIZE) { 1720 crypto_ablkcipher_set_flags(ablkcipher, 1721 CRYPTO_TFM_RES_BAD_KEY_LEN); 1722 dev_err(jrdev, "key size mismatch\n"); 1723 return -EINVAL; 1724 } 1725 1726 memcpy(ctx->key, key, keylen); 1727 ctx->key_dma = dma_map_single(jrdev, ctx->key, keylen, DMA_TO_DEVICE); 1728 if (dma_mapping_error(jrdev, ctx->key_dma)) { 1729 dev_err(jrdev, "unable to map key i/o memory\n"); 1730 return -ENOMEM; 1731 } 1732 ctx->enckeylen = keylen; 1733 1734 /* xts_ablkcipher_encrypt shared descriptor */ 1735 desc = ctx->sh_desc_enc; 1736 init_sh_desc(desc, HDR_SHARE_SERIAL | HDR_SAVECTX); 1737 /* Skip if already shared */ 1738 key_jump_cmd = append_jump(desc, JUMP_JSL | JUMP_TEST_ALL | 1739 JUMP_COND_SHRD); 1740 1741 /* Load class1 keys only */ 1742 append_key_as_imm(desc, (void *)ctx->key, ctx->enckeylen, 1743 ctx->enckeylen, CLASS_1 | KEY_DEST_CLASS_REG); 1744 1745 /* Load sector size with index 40 bytes (0x28) */ 1746 append_cmd(desc, CMD_LOAD | IMMEDIATE | LDST_SRCDST_BYTE_CONTEXT | 1747 LDST_CLASS_1_CCB | (0x28 << LDST_OFFSET_SHIFT) | 8); 1748 append_data(desc, (void *)§or_size, 8); 1749 1750 set_jump_tgt_here(desc, key_jump_cmd); 1751 1752 /* 1753 * create sequence for loading the sector index 1754 * Upper 8B of IV - will be used as sector index 1755 * Lower 8B of IV - will be discarded 1756 */ 1757 append_cmd(desc, CMD_SEQ_LOAD | LDST_SRCDST_BYTE_CONTEXT | 1758 LDST_CLASS_1_CCB | (0x20 << LDST_OFFSET_SHIFT) | 8); 1759 append_seq_fifo_load(desc, 8, FIFOLD_CLASS_SKIP); 1760 1761 /* Load operation */ 1762 append_operation(desc, ctx->class1_alg_type | OP_ALG_AS_INITFINAL | 1763 OP_ALG_ENCRYPT); 1764 1765 /* Perform operation */ 1766 ablkcipher_append_src_dst(desc); 1767 1768 ctx->sh_desc_enc_dma = dma_map_single(jrdev, desc, desc_bytes(desc), 1769 DMA_TO_DEVICE); 1770 if (dma_mapping_error(jrdev, ctx->sh_desc_enc_dma)) { 1771 dev_err(jrdev, "unable to map shared descriptor\n"); 1772 return -ENOMEM; 1773 } 1774 #ifdef DEBUG 1775 print_hex_dump(KERN_ERR, 1776 "xts ablkcipher enc shdesc@" __stringify(__LINE__) ": ", 1777 DUMP_PREFIX_ADDRESS, 16, 4, desc, desc_bytes(desc), 1); 1778 #endif 1779 1780 /* xts_ablkcipher_decrypt shared descriptor */ 1781 desc = ctx->sh_desc_dec; 1782 1783 init_sh_desc(desc, HDR_SHARE_SERIAL | HDR_SAVECTX); 1784 /* Skip if already shared */ 1785 key_jump_cmd = append_jump(desc, JUMP_JSL | JUMP_TEST_ALL | 1786 JUMP_COND_SHRD); 1787 1788 /* Load class1 key only */ 1789 append_key_as_imm(desc, (void *)ctx->key, ctx->enckeylen, 1790 ctx->enckeylen, CLASS_1 | KEY_DEST_CLASS_REG); 1791 1792 /* Load sector size with index 40 bytes (0x28) */ 1793 append_cmd(desc, CMD_LOAD | IMMEDIATE | LDST_SRCDST_BYTE_CONTEXT | 1794 LDST_CLASS_1_CCB | (0x28 << LDST_OFFSET_SHIFT) | 8); 1795 append_data(desc, (void *)§or_size, 8); 1796 1797 set_jump_tgt_here(desc, key_jump_cmd); 1798 1799 /* 1800 * create sequence for loading the sector index 1801 * Upper 8B of IV - will be used as sector index 1802 * Lower 8B of IV - will be discarded 1803 */ 1804 append_cmd(desc, CMD_SEQ_LOAD | LDST_SRCDST_BYTE_CONTEXT | 1805 LDST_CLASS_1_CCB | (0x20 << LDST_OFFSET_SHIFT) | 8); 1806 append_seq_fifo_load(desc, 8, FIFOLD_CLASS_SKIP); 1807 1808 /* Load operation */ 1809 append_dec_op1(desc, ctx->class1_alg_type); 1810 1811 /* Perform operation */ 1812 ablkcipher_append_src_dst(desc); 1813 1814 ctx->sh_desc_dec_dma = dma_map_single(jrdev, desc, desc_bytes(desc), 1815 DMA_TO_DEVICE); 1816 if (dma_mapping_error(jrdev, ctx->sh_desc_dec_dma)) { 1817 dma_unmap_single(jrdev, ctx->sh_desc_enc_dma, 1818 desc_bytes(ctx->sh_desc_enc), DMA_TO_DEVICE); 1819 dev_err(jrdev, "unable to map shared descriptor\n"); 1820 return -ENOMEM; 1821 } 1822 #ifdef DEBUG 1823 print_hex_dump(KERN_ERR, 1824 "xts ablkcipher dec shdesc@" __stringify(__LINE__) ": ", 1825 DUMP_PREFIX_ADDRESS, 16, 4, desc, desc_bytes(desc), 1); 1826 #endif 1827 1828 return 0; 1829 } 1830 1831 /* 1832 * aead_edesc - s/w-extended aead descriptor 1833 * @assoc_nents: number of segments in associated data (SPI+Seq) scatterlist 1834 * @src_nents: number of segments in input scatterlist 1835 * @dst_nents: number of segments in output scatterlist 1836 * @iv_dma: dma address of iv for checking continuity and link table 1837 * @desc: h/w descriptor (variable length; must not exceed MAX_CAAM_DESCSIZE) 1838 * @sec4_sg_bytes: length of dma mapped sec4_sg space 1839 * @sec4_sg_dma: bus physical mapped address of h/w link table 1840 * @hw_desc: the h/w job descriptor followed by any referenced link tables 1841 */ 1842 struct aead_edesc { 1843 int assoc_nents; 1844 int src_nents; 1845 int dst_nents; 1846 dma_addr_t iv_dma; 1847 int sec4_sg_bytes; 1848 dma_addr_t sec4_sg_dma; 1849 struct sec4_sg_entry *sec4_sg; 1850 u32 hw_desc[]; 1851 }; 1852 1853 /* 1854 * ablkcipher_edesc - s/w-extended ablkcipher descriptor 1855 * @src_nents: number of segments in input scatterlist 1856 * @dst_nents: number of segments in output scatterlist 1857 * @iv_dma: dma address of iv for checking continuity and link table 1858 * @desc: h/w descriptor (variable length; must not exceed MAX_CAAM_DESCSIZE) 1859 * @sec4_sg_bytes: length of dma mapped sec4_sg space 1860 * @sec4_sg_dma: bus physical mapped address of h/w link table 1861 * @hw_desc: the h/w job descriptor followed by any referenced link tables 1862 */ 1863 struct ablkcipher_edesc { 1864 int src_nents; 1865 int dst_nents; 1866 dma_addr_t iv_dma; 1867 int sec4_sg_bytes; 1868 dma_addr_t sec4_sg_dma; 1869 struct sec4_sg_entry *sec4_sg; 1870 u32 hw_desc[0]; 1871 }; 1872 1873 static void caam_unmap(struct device *dev, struct scatterlist *src, 1874 struct scatterlist *dst, int src_nents, 1875 int dst_nents, 1876 dma_addr_t iv_dma, int ivsize, dma_addr_t sec4_sg_dma, 1877 int sec4_sg_bytes) 1878 { 1879 if (dst != src) { 1880 dma_unmap_sg(dev, src, src_nents ? : 1, DMA_TO_DEVICE); 1881 dma_unmap_sg(dev, dst, dst_nents ? : 1, DMA_FROM_DEVICE); 1882 } else { 1883 dma_unmap_sg(dev, src, src_nents ? : 1, DMA_BIDIRECTIONAL); 1884 } 1885 1886 if (iv_dma) 1887 dma_unmap_single(dev, iv_dma, ivsize, DMA_TO_DEVICE); 1888 if (sec4_sg_bytes) 1889 dma_unmap_single(dev, sec4_sg_dma, sec4_sg_bytes, 1890 DMA_TO_DEVICE); 1891 } 1892 1893 static void aead_unmap(struct device *dev, 1894 struct aead_edesc *edesc, 1895 struct aead_request *req) 1896 { 1897 caam_unmap(dev, req->src, req->dst, 1898 edesc->src_nents, edesc->dst_nents, 0, 0, 1899 edesc->sec4_sg_dma, edesc->sec4_sg_bytes); 1900 } 1901 1902 static void ablkcipher_unmap(struct device *dev, 1903 struct ablkcipher_edesc *edesc, 1904 struct ablkcipher_request *req) 1905 { 1906 struct crypto_ablkcipher *ablkcipher = crypto_ablkcipher_reqtfm(req); 1907 int ivsize = crypto_ablkcipher_ivsize(ablkcipher); 1908 1909 caam_unmap(dev, req->src, req->dst, 1910 edesc->src_nents, edesc->dst_nents, 1911 edesc->iv_dma, ivsize, 1912 edesc->sec4_sg_dma, edesc->sec4_sg_bytes); 1913 } 1914 1915 static void aead_encrypt_done(struct device *jrdev, u32 *desc, u32 err, 1916 void *context) 1917 { 1918 struct aead_request *req = context; 1919 struct aead_edesc *edesc; 1920 1921 #ifdef DEBUG 1922 dev_err(jrdev, "%s %d: err 0x%x\n", __func__, __LINE__, err); 1923 #endif 1924 1925 edesc = container_of(desc, struct aead_edesc, hw_desc[0]); 1926 1927 if (err) 1928 caam_jr_strstatus(jrdev, err); 1929 1930 aead_unmap(jrdev, edesc, req); 1931 1932 kfree(edesc); 1933 1934 aead_request_complete(req, err); 1935 } 1936 1937 static void aead_decrypt_done(struct device *jrdev, u32 *desc, u32 err, 1938 void *context) 1939 { 1940 struct aead_request *req = context; 1941 struct aead_edesc *edesc; 1942 1943 #ifdef DEBUG 1944 dev_err(jrdev, "%s %d: err 0x%x\n", __func__, __LINE__, err); 1945 #endif 1946 1947 edesc = container_of(desc, struct aead_edesc, hw_desc[0]); 1948 1949 if (err) 1950 caam_jr_strstatus(jrdev, err); 1951 1952 aead_unmap(jrdev, edesc, req); 1953 1954 /* 1955 * verify hw auth check passed else return -EBADMSG 1956 */ 1957 if ((err & JRSTA_CCBERR_ERRID_MASK) == JRSTA_CCBERR_ERRID_ICVCHK) 1958 err = -EBADMSG; 1959 1960 kfree(edesc); 1961 1962 aead_request_complete(req, err); 1963 } 1964 1965 static void ablkcipher_encrypt_done(struct device *jrdev, u32 *desc, u32 err, 1966 void *context) 1967 { 1968 struct ablkcipher_request *req = context; 1969 struct ablkcipher_edesc *edesc; 1970 #ifdef DEBUG 1971 struct crypto_ablkcipher *ablkcipher = crypto_ablkcipher_reqtfm(req); 1972 int ivsize = crypto_ablkcipher_ivsize(ablkcipher); 1973 1974 dev_err(jrdev, "%s %d: err 0x%x\n", __func__, __LINE__, err); 1975 #endif 1976 1977 edesc = (struct ablkcipher_edesc *)((char *)desc - 1978 offsetof(struct ablkcipher_edesc, hw_desc)); 1979 1980 if (err) 1981 caam_jr_strstatus(jrdev, err); 1982 1983 #ifdef DEBUG 1984 print_hex_dump(KERN_ERR, "dstiv @"__stringify(__LINE__)": ", 1985 DUMP_PREFIX_ADDRESS, 16, 4, req->info, 1986 edesc->src_nents > 1 ? 100 : ivsize, 1); 1987 print_hex_dump(KERN_ERR, "dst @"__stringify(__LINE__)": ", 1988 DUMP_PREFIX_ADDRESS, 16, 4, sg_virt(req->src), 1989 edesc->dst_nents > 1 ? 100 : req->nbytes, 1); 1990 #endif 1991 1992 ablkcipher_unmap(jrdev, edesc, req); 1993 kfree(edesc); 1994 1995 ablkcipher_request_complete(req, err); 1996 } 1997 1998 static void ablkcipher_decrypt_done(struct device *jrdev, u32 *desc, u32 err, 1999 void *context) 2000 { 2001 struct ablkcipher_request *req = context; 2002 struct ablkcipher_edesc *edesc; 2003 #ifdef DEBUG 2004 struct crypto_ablkcipher *ablkcipher = crypto_ablkcipher_reqtfm(req); 2005 int ivsize = crypto_ablkcipher_ivsize(ablkcipher); 2006 2007 dev_err(jrdev, "%s %d: err 0x%x\n", __func__, __LINE__, err); 2008 #endif 2009 2010 edesc = (struct ablkcipher_edesc *)((char *)desc - 2011 offsetof(struct ablkcipher_edesc, hw_desc)); 2012 if (err) 2013 caam_jr_strstatus(jrdev, err); 2014 2015 #ifdef DEBUG 2016 print_hex_dump(KERN_ERR, "dstiv @"__stringify(__LINE__)": ", 2017 DUMP_PREFIX_ADDRESS, 16, 4, req->info, 2018 ivsize, 1); 2019 print_hex_dump(KERN_ERR, "dst @"__stringify(__LINE__)": ", 2020 DUMP_PREFIX_ADDRESS, 16, 4, sg_virt(req->src), 2021 edesc->dst_nents > 1 ? 100 : req->nbytes, 1); 2022 #endif 2023 2024 ablkcipher_unmap(jrdev, edesc, req); 2025 kfree(edesc); 2026 2027 ablkcipher_request_complete(req, err); 2028 } 2029 2030 /* 2031 * Fill in aead job descriptor 2032 */ 2033 static void init_aead_job(struct aead_request *req, 2034 struct aead_edesc *edesc, 2035 bool all_contig, bool encrypt) 2036 { 2037 struct crypto_aead *aead = crypto_aead_reqtfm(req); 2038 struct caam_ctx *ctx = crypto_aead_ctx(aead); 2039 int authsize = ctx->authsize; 2040 u32 *desc = edesc->hw_desc; 2041 u32 out_options, in_options; 2042 dma_addr_t dst_dma, src_dma; 2043 int len, sec4_sg_index = 0; 2044 dma_addr_t ptr; 2045 u32 *sh_desc; 2046 2047 sh_desc = encrypt ? ctx->sh_desc_enc : ctx->sh_desc_dec; 2048 ptr = encrypt ? ctx->sh_desc_enc_dma : ctx->sh_desc_dec_dma; 2049 2050 len = desc_len(sh_desc); 2051 init_job_desc_shared(desc, ptr, len, HDR_SHARE_DEFER | HDR_REVERSE); 2052 2053 if (all_contig) { 2054 src_dma = sg_dma_address(req->src); 2055 in_options = 0; 2056 } else { 2057 src_dma = edesc->sec4_sg_dma; 2058 sec4_sg_index += edesc->src_nents; 2059 in_options = LDST_SGF; 2060 } 2061 2062 append_seq_in_ptr(desc, src_dma, req->assoclen + req->cryptlen, 2063 in_options); 2064 2065 dst_dma = src_dma; 2066 out_options = in_options; 2067 2068 if (unlikely(req->src != req->dst)) { 2069 if (!edesc->dst_nents) { 2070 dst_dma = sg_dma_address(req->dst); 2071 } else { 2072 dst_dma = edesc->sec4_sg_dma + 2073 sec4_sg_index * 2074 sizeof(struct sec4_sg_entry); 2075 out_options = LDST_SGF; 2076 } 2077 } 2078 2079 if (encrypt) 2080 append_seq_out_ptr(desc, dst_dma, 2081 req->assoclen + req->cryptlen + authsize, 2082 out_options); 2083 else 2084 append_seq_out_ptr(desc, dst_dma, 2085 req->assoclen + req->cryptlen - authsize, 2086 out_options); 2087 2088 /* REG3 = assoclen */ 2089 append_math_add_imm_u32(desc, REG3, ZERO, IMM, req->assoclen); 2090 } 2091 2092 static void init_gcm_job(struct aead_request *req, 2093 struct aead_edesc *edesc, 2094 bool all_contig, bool encrypt) 2095 { 2096 struct crypto_aead *aead = crypto_aead_reqtfm(req); 2097 struct caam_ctx *ctx = crypto_aead_ctx(aead); 2098 unsigned int ivsize = crypto_aead_ivsize(aead); 2099 u32 *desc = edesc->hw_desc; 2100 bool generic_gcm = (ivsize == 12); 2101 unsigned int last; 2102 2103 init_aead_job(req, edesc, all_contig, encrypt); 2104 2105 /* BUG This should not be specific to generic GCM. */ 2106 last = 0; 2107 if (encrypt && generic_gcm && !(req->assoclen + req->cryptlen)) 2108 last = FIFOLD_TYPE_LAST1; 2109 2110 /* Read GCM IV */ 2111 append_cmd(desc, CMD_FIFO_LOAD | FIFOLD_CLASS_CLASS1 | IMMEDIATE | 2112 FIFOLD_TYPE_IV | FIFOLD_TYPE_FLUSH1 | 12 | last); 2113 /* Append Salt */ 2114 if (!generic_gcm) 2115 append_data(desc, ctx->key + ctx->enckeylen, 4); 2116 /* Append IV */ 2117 append_data(desc, req->iv, ivsize); 2118 /* End of blank commands */ 2119 } 2120 2121 static void init_authenc_job(struct aead_request *req, 2122 struct aead_edesc *edesc, 2123 bool all_contig, bool encrypt) 2124 { 2125 struct crypto_aead *aead = crypto_aead_reqtfm(req); 2126 struct caam_aead_alg *alg = container_of(crypto_aead_alg(aead), 2127 struct caam_aead_alg, aead); 2128 unsigned int ivsize = crypto_aead_ivsize(aead); 2129 struct caam_ctx *ctx = crypto_aead_ctx(aead); 2130 const bool ctr_mode = ((ctx->class1_alg_type & OP_ALG_AAI_MASK) == 2131 OP_ALG_AAI_CTR_MOD128); 2132 const bool is_rfc3686 = alg->caam.rfc3686; 2133 u32 *desc = edesc->hw_desc; 2134 u32 ivoffset = 0; 2135 2136 /* 2137 * AES-CTR needs to load IV in CONTEXT1 reg 2138 * at an offset of 128bits (16bytes) 2139 * CONTEXT1[255:128] = IV 2140 */ 2141 if (ctr_mode) 2142 ivoffset = 16; 2143 2144 /* 2145 * RFC3686 specific: 2146 * CONTEXT1[255:128] = {NONCE, IV, COUNTER} 2147 */ 2148 if (is_rfc3686) 2149 ivoffset = 16 + CTR_RFC3686_NONCE_SIZE; 2150 2151 init_aead_job(req, edesc, all_contig, encrypt); 2152 2153 if (ivsize && (is_rfc3686 || !(alg->caam.geniv && encrypt))) 2154 append_load_as_imm(desc, req->iv, ivsize, 2155 LDST_CLASS_1_CCB | 2156 LDST_SRCDST_BYTE_CONTEXT | 2157 (ivoffset << LDST_OFFSET_SHIFT)); 2158 } 2159 2160 /* 2161 * Fill in ablkcipher job descriptor 2162 */ 2163 static void init_ablkcipher_job(u32 *sh_desc, dma_addr_t ptr, 2164 struct ablkcipher_edesc *edesc, 2165 struct ablkcipher_request *req, 2166 bool iv_contig) 2167 { 2168 struct crypto_ablkcipher *ablkcipher = crypto_ablkcipher_reqtfm(req); 2169 int ivsize = crypto_ablkcipher_ivsize(ablkcipher); 2170 u32 *desc = edesc->hw_desc; 2171 u32 out_options = 0, in_options; 2172 dma_addr_t dst_dma, src_dma; 2173 int len, sec4_sg_index = 0; 2174 2175 #ifdef DEBUG 2176 print_hex_dump(KERN_ERR, "presciv@"__stringify(__LINE__)": ", 2177 DUMP_PREFIX_ADDRESS, 16, 4, req->info, 2178 ivsize, 1); 2179 print_hex_dump(KERN_ERR, "src @"__stringify(__LINE__)": ", 2180 DUMP_PREFIX_ADDRESS, 16, 4, sg_virt(req->src), 2181 edesc->src_nents ? 100 : req->nbytes, 1); 2182 #endif 2183 2184 len = desc_len(sh_desc); 2185 init_job_desc_shared(desc, ptr, len, HDR_SHARE_DEFER | HDR_REVERSE); 2186 2187 if (iv_contig) { 2188 src_dma = edesc->iv_dma; 2189 in_options = 0; 2190 } else { 2191 src_dma = edesc->sec4_sg_dma; 2192 sec4_sg_index += edesc->src_nents + 1; 2193 in_options = LDST_SGF; 2194 } 2195 append_seq_in_ptr(desc, src_dma, req->nbytes + ivsize, in_options); 2196 2197 if (likely(req->src == req->dst)) { 2198 if (!edesc->src_nents && iv_contig) { 2199 dst_dma = sg_dma_address(req->src); 2200 } else { 2201 dst_dma = edesc->sec4_sg_dma + 2202 sizeof(struct sec4_sg_entry); 2203 out_options = LDST_SGF; 2204 } 2205 } else { 2206 if (!edesc->dst_nents) { 2207 dst_dma = sg_dma_address(req->dst); 2208 } else { 2209 dst_dma = edesc->sec4_sg_dma + 2210 sec4_sg_index * sizeof(struct sec4_sg_entry); 2211 out_options = LDST_SGF; 2212 } 2213 } 2214 append_seq_out_ptr(desc, dst_dma, req->nbytes, out_options); 2215 } 2216 2217 /* 2218 * Fill in ablkcipher givencrypt job descriptor 2219 */ 2220 static void init_ablkcipher_giv_job(u32 *sh_desc, dma_addr_t ptr, 2221 struct ablkcipher_edesc *edesc, 2222 struct ablkcipher_request *req, 2223 bool iv_contig) 2224 { 2225 struct crypto_ablkcipher *ablkcipher = crypto_ablkcipher_reqtfm(req); 2226 int ivsize = crypto_ablkcipher_ivsize(ablkcipher); 2227 u32 *desc = edesc->hw_desc; 2228 u32 out_options, in_options; 2229 dma_addr_t dst_dma, src_dma; 2230 int len, sec4_sg_index = 0; 2231 2232 #ifdef DEBUG 2233 print_hex_dump(KERN_ERR, "presciv@" __stringify(__LINE__) ": ", 2234 DUMP_PREFIX_ADDRESS, 16, 4, req->info, 2235 ivsize, 1); 2236 print_hex_dump(KERN_ERR, "src @" __stringify(__LINE__) ": ", 2237 DUMP_PREFIX_ADDRESS, 16, 4, sg_virt(req->src), 2238 edesc->src_nents ? 100 : req->nbytes, 1); 2239 #endif 2240 2241 len = desc_len(sh_desc); 2242 init_job_desc_shared(desc, ptr, len, HDR_SHARE_DEFER | HDR_REVERSE); 2243 2244 if (!edesc->src_nents) { 2245 src_dma = sg_dma_address(req->src); 2246 in_options = 0; 2247 } else { 2248 src_dma = edesc->sec4_sg_dma; 2249 sec4_sg_index += edesc->src_nents; 2250 in_options = LDST_SGF; 2251 } 2252 append_seq_in_ptr(desc, src_dma, req->nbytes, in_options); 2253 2254 if (iv_contig) { 2255 dst_dma = edesc->iv_dma; 2256 out_options = 0; 2257 } else { 2258 dst_dma = edesc->sec4_sg_dma + 2259 sec4_sg_index * sizeof(struct sec4_sg_entry); 2260 out_options = LDST_SGF; 2261 } 2262 append_seq_out_ptr(desc, dst_dma, req->nbytes + ivsize, out_options); 2263 } 2264 2265 /* 2266 * allocate and map the aead extended descriptor 2267 */ 2268 static struct aead_edesc *aead_edesc_alloc(struct aead_request *req, 2269 int desc_bytes, bool *all_contig_ptr, 2270 bool encrypt) 2271 { 2272 struct crypto_aead *aead = crypto_aead_reqtfm(req); 2273 struct caam_ctx *ctx = crypto_aead_ctx(aead); 2274 struct device *jrdev = ctx->jrdev; 2275 gfp_t flags = (req->base.flags & (CRYPTO_TFM_REQ_MAY_BACKLOG | 2276 CRYPTO_TFM_REQ_MAY_SLEEP)) ? GFP_KERNEL : GFP_ATOMIC; 2277 int src_nents, dst_nents = 0; 2278 struct aead_edesc *edesc; 2279 int sgc; 2280 bool all_contig = true; 2281 int sec4_sg_index, sec4_sg_len = 0, sec4_sg_bytes; 2282 unsigned int authsize = ctx->authsize; 2283 2284 if (unlikely(req->dst != req->src)) { 2285 src_nents = sg_count(req->src, req->assoclen + req->cryptlen); 2286 dst_nents = sg_count(req->dst, 2287 req->assoclen + req->cryptlen + 2288 (encrypt ? authsize : (-authsize))); 2289 } else { 2290 src_nents = sg_count(req->src, 2291 req->assoclen + req->cryptlen + 2292 (encrypt ? authsize : 0)); 2293 } 2294 2295 /* Check if data are contiguous. */ 2296 all_contig = !src_nents; 2297 if (!all_contig) { 2298 src_nents = src_nents ? : 1; 2299 sec4_sg_len = src_nents; 2300 } 2301 2302 sec4_sg_len += dst_nents; 2303 2304 sec4_sg_bytes = sec4_sg_len * sizeof(struct sec4_sg_entry); 2305 2306 /* allocate space for base edesc and hw desc commands, link tables */ 2307 edesc = kzalloc(sizeof(*edesc) + desc_bytes + sec4_sg_bytes, 2308 GFP_DMA | flags); 2309 if (!edesc) { 2310 dev_err(jrdev, "could not allocate extended descriptor\n"); 2311 return ERR_PTR(-ENOMEM); 2312 } 2313 2314 if (likely(req->src == req->dst)) { 2315 sgc = dma_map_sg(jrdev, req->src, src_nents ? : 1, 2316 DMA_BIDIRECTIONAL); 2317 if (unlikely(!sgc)) { 2318 dev_err(jrdev, "unable to map source\n"); 2319 kfree(edesc); 2320 return ERR_PTR(-ENOMEM); 2321 } 2322 } else { 2323 sgc = dma_map_sg(jrdev, req->src, src_nents ? : 1, 2324 DMA_TO_DEVICE); 2325 if (unlikely(!sgc)) { 2326 dev_err(jrdev, "unable to map source\n"); 2327 kfree(edesc); 2328 return ERR_PTR(-ENOMEM); 2329 } 2330 2331 sgc = dma_map_sg(jrdev, req->dst, dst_nents ? : 1, 2332 DMA_FROM_DEVICE); 2333 if (unlikely(!sgc)) { 2334 dev_err(jrdev, "unable to map destination\n"); 2335 dma_unmap_sg(jrdev, req->src, src_nents ? : 1, 2336 DMA_TO_DEVICE); 2337 kfree(edesc); 2338 return ERR_PTR(-ENOMEM); 2339 } 2340 } 2341 2342 edesc->src_nents = src_nents; 2343 edesc->dst_nents = dst_nents; 2344 edesc->sec4_sg = (void *)edesc + sizeof(struct aead_edesc) + 2345 desc_bytes; 2346 *all_contig_ptr = all_contig; 2347 2348 sec4_sg_index = 0; 2349 if (!all_contig) { 2350 sg_to_sec4_sg_last(req->src, src_nents, 2351 edesc->sec4_sg + sec4_sg_index, 0); 2352 sec4_sg_index += src_nents; 2353 } 2354 if (dst_nents) { 2355 sg_to_sec4_sg_last(req->dst, dst_nents, 2356 edesc->sec4_sg + sec4_sg_index, 0); 2357 } 2358 2359 if (!sec4_sg_bytes) 2360 return edesc; 2361 2362 edesc->sec4_sg_dma = dma_map_single(jrdev, edesc->sec4_sg, 2363 sec4_sg_bytes, DMA_TO_DEVICE); 2364 if (dma_mapping_error(jrdev, edesc->sec4_sg_dma)) { 2365 dev_err(jrdev, "unable to map S/G table\n"); 2366 aead_unmap(jrdev, edesc, req); 2367 kfree(edesc); 2368 return ERR_PTR(-ENOMEM); 2369 } 2370 2371 edesc->sec4_sg_bytes = sec4_sg_bytes; 2372 2373 return edesc; 2374 } 2375 2376 static int gcm_encrypt(struct aead_request *req) 2377 { 2378 struct aead_edesc *edesc; 2379 struct crypto_aead *aead = crypto_aead_reqtfm(req); 2380 struct caam_ctx *ctx = crypto_aead_ctx(aead); 2381 struct device *jrdev = ctx->jrdev; 2382 bool all_contig; 2383 u32 *desc; 2384 int ret = 0; 2385 2386 /* allocate extended descriptor */ 2387 edesc = aead_edesc_alloc(req, GCM_DESC_JOB_IO_LEN, &all_contig, true); 2388 if (IS_ERR(edesc)) 2389 return PTR_ERR(edesc); 2390 2391 /* Create and submit job descriptor */ 2392 init_gcm_job(req, edesc, all_contig, true); 2393 #ifdef DEBUG 2394 print_hex_dump(KERN_ERR, "aead jobdesc@"__stringify(__LINE__)": ", 2395 DUMP_PREFIX_ADDRESS, 16, 4, edesc->hw_desc, 2396 desc_bytes(edesc->hw_desc), 1); 2397 #endif 2398 2399 desc = edesc->hw_desc; 2400 ret = caam_jr_enqueue(jrdev, desc, aead_encrypt_done, req); 2401 if (!ret) { 2402 ret = -EINPROGRESS; 2403 } else { 2404 aead_unmap(jrdev, edesc, req); 2405 kfree(edesc); 2406 } 2407 2408 return ret; 2409 } 2410 2411 static int ipsec_gcm_encrypt(struct aead_request *req) 2412 { 2413 if (req->assoclen < 8) 2414 return -EINVAL; 2415 2416 return gcm_encrypt(req); 2417 } 2418 2419 static int aead_encrypt(struct aead_request *req) 2420 { 2421 struct aead_edesc *edesc; 2422 struct crypto_aead *aead = crypto_aead_reqtfm(req); 2423 struct caam_ctx *ctx = crypto_aead_ctx(aead); 2424 struct device *jrdev = ctx->jrdev; 2425 bool all_contig; 2426 u32 *desc; 2427 int ret = 0; 2428 2429 /* allocate extended descriptor */ 2430 edesc = aead_edesc_alloc(req, AUTHENC_DESC_JOB_IO_LEN, 2431 &all_contig, true); 2432 if (IS_ERR(edesc)) 2433 return PTR_ERR(edesc); 2434 2435 /* Create and submit job descriptor */ 2436 init_authenc_job(req, edesc, all_contig, true); 2437 #ifdef DEBUG 2438 print_hex_dump(KERN_ERR, "aead jobdesc@"__stringify(__LINE__)": ", 2439 DUMP_PREFIX_ADDRESS, 16, 4, edesc->hw_desc, 2440 desc_bytes(edesc->hw_desc), 1); 2441 #endif 2442 2443 desc = edesc->hw_desc; 2444 ret = caam_jr_enqueue(jrdev, desc, aead_encrypt_done, req); 2445 if (!ret) { 2446 ret = -EINPROGRESS; 2447 } else { 2448 aead_unmap(jrdev, edesc, req); 2449 kfree(edesc); 2450 } 2451 2452 return ret; 2453 } 2454 2455 static int gcm_decrypt(struct aead_request *req) 2456 { 2457 struct aead_edesc *edesc; 2458 struct crypto_aead *aead = crypto_aead_reqtfm(req); 2459 struct caam_ctx *ctx = crypto_aead_ctx(aead); 2460 struct device *jrdev = ctx->jrdev; 2461 bool all_contig; 2462 u32 *desc; 2463 int ret = 0; 2464 2465 /* allocate extended descriptor */ 2466 edesc = aead_edesc_alloc(req, GCM_DESC_JOB_IO_LEN, &all_contig, false); 2467 if (IS_ERR(edesc)) 2468 return PTR_ERR(edesc); 2469 2470 /* Create and submit job descriptor*/ 2471 init_gcm_job(req, edesc, all_contig, false); 2472 #ifdef DEBUG 2473 print_hex_dump(KERN_ERR, "aead jobdesc@"__stringify(__LINE__)": ", 2474 DUMP_PREFIX_ADDRESS, 16, 4, edesc->hw_desc, 2475 desc_bytes(edesc->hw_desc), 1); 2476 #endif 2477 2478 desc = edesc->hw_desc; 2479 ret = caam_jr_enqueue(jrdev, desc, aead_decrypt_done, req); 2480 if (!ret) { 2481 ret = -EINPROGRESS; 2482 } else { 2483 aead_unmap(jrdev, edesc, req); 2484 kfree(edesc); 2485 } 2486 2487 return ret; 2488 } 2489 2490 static int ipsec_gcm_decrypt(struct aead_request *req) 2491 { 2492 if (req->assoclen < 8) 2493 return -EINVAL; 2494 2495 return gcm_decrypt(req); 2496 } 2497 2498 static int aead_decrypt(struct aead_request *req) 2499 { 2500 struct aead_edesc *edesc; 2501 struct crypto_aead *aead = crypto_aead_reqtfm(req); 2502 struct caam_ctx *ctx = crypto_aead_ctx(aead); 2503 struct device *jrdev = ctx->jrdev; 2504 bool all_contig; 2505 u32 *desc; 2506 int ret = 0; 2507 2508 /* allocate extended descriptor */ 2509 edesc = aead_edesc_alloc(req, AUTHENC_DESC_JOB_IO_LEN, 2510 &all_contig, false); 2511 if (IS_ERR(edesc)) 2512 return PTR_ERR(edesc); 2513 2514 #ifdef DEBUG 2515 print_hex_dump(KERN_ERR, "dec src@"__stringify(__LINE__)": ", 2516 DUMP_PREFIX_ADDRESS, 16, 4, sg_virt(req->src), 2517 req->assoclen + req->cryptlen, 1); 2518 #endif 2519 2520 /* Create and submit job descriptor*/ 2521 init_authenc_job(req, edesc, all_contig, false); 2522 #ifdef DEBUG 2523 print_hex_dump(KERN_ERR, "aead jobdesc@"__stringify(__LINE__)": ", 2524 DUMP_PREFIX_ADDRESS, 16, 4, edesc->hw_desc, 2525 desc_bytes(edesc->hw_desc), 1); 2526 #endif 2527 2528 desc = edesc->hw_desc; 2529 ret = caam_jr_enqueue(jrdev, desc, aead_decrypt_done, req); 2530 if (!ret) { 2531 ret = -EINPROGRESS; 2532 } else { 2533 aead_unmap(jrdev, edesc, req); 2534 kfree(edesc); 2535 } 2536 2537 return ret; 2538 } 2539 2540 static int aead_givdecrypt(struct aead_request *req) 2541 { 2542 struct crypto_aead *aead = crypto_aead_reqtfm(req); 2543 unsigned int ivsize = crypto_aead_ivsize(aead); 2544 2545 if (req->cryptlen < ivsize) 2546 return -EINVAL; 2547 2548 req->cryptlen -= ivsize; 2549 req->assoclen += ivsize; 2550 2551 return aead_decrypt(req); 2552 } 2553 2554 /* 2555 * allocate and map the ablkcipher extended descriptor for ablkcipher 2556 */ 2557 static struct ablkcipher_edesc *ablkcipher_edesc_alloc(struct ablkcipher_request 2558 *req, int desc_bytes, 2559 bool *iv_contig_out) 2560 { 2561 struct crypto_ablkcipher *ablkcipher = crypto_ablkcipher_reqtfm(req); 2562 struct caam_ctx *ctx = crypto_ablkcipher_ctx(ablkcipher); 2563 struct device *jrdev = ctx->jrdev; 2564 gfp_t flags = (req->base.flags & (CRYPTO_TFM_REQ_MAY_BACKLOG | 2565 CRYPTO_TFM_REQ_MAY_SLEEP)) ? 2566 GFP_KERNEL : GFP_ATOMIC; 2567 int src_nents, dst_nents = 0, sec4_sg_bytes; 2568 struct ablkcipher_edesc *edesc; 2569 dma_addr_t iv_dma = 0; 2570 bool iv_contig = false; 2571 int sgc; 2572 int ivsize = crypto_ablkcipher_ivsize(ablkcipher); 2573 int sec4_sg_index; 2574 2575 src_nents = sg_count(req->src, req->nbytes); 2576 2577 if (req->dst != req->src) 2578 dst_nents = sg_count(req->dst, req->nbytes); 2579 2580 if (likely(req->src == req->dst)) { 2581 sgc = dma_map_sg(jrdev, req->src, src_nents ? : 1, 2582 DMA_BIDIRECTIONAL); 2583 } else { 2584 sgc = dma_map_sg(jrdev, req->src, src_nents ? : 1, 2585 DMA_TO_DEVICE); 2586 sgc = dma_map_sg(jrdev, req->dst, dst_nents ? : 1, 2587 DMA_FROM_DEVICE); 2588 } 2589 2590 iv_dma = dma_map_single(jrdev, req->info, ivsize, DMA_TO_DEVICE); 2591 if (dma_mapping_error(jrdev, iv_dma)) { 2592 dev_err(jrdev, "unable to map IV\n"); 2593 return ERR_PTR(-ENOMEM); 2594 } 2595 2596 /* 2597 * Check if iv can be contiguous with source and destination. 2598 * If so, include it. If not, create scatterlist. 2599 */ 2600 if (!src_nents && iv_dma + ivsize == sg_dma_address(req->src)) 2601 iv_contig = true; 2602 else 2603 src_nents = src_nents ? : 1; 2604 sec4_sg_bytes = ((iv_contig ? 0 : 1) + src_nents + dst_nents) * 2605 sizeof(struct sec4_sg_entry); 2606 2607 /* allocate space for base edesc and hw desc commands, link tables */ 2608 edesc = kzalloc(sizeof(*edesc) + desc_bytes + sec4_sg_bytes, 2609 GFP_DMA | flags); 2610 if (!edesc) { 2611 dev_err(jrdev, "could not allocate extended descriptor\n"); 2612 return ERR_PTR(-ENOMEM); 2613 } 2614 2615 edesc->src_nents = src_nents; 2616 edesc->dst_nents = dst_nents; 2617 edesc->sec4_sg_bytes = sec4_sg_bytes; 2618 edesc->sec4_sg = (void *)edesc + sizeof(struct ablkcipher_edesc) + 2619 desc_bytes; 2620 2621 sec4_sg_index = 0; 2622 if (!iv_contig) { 2623 dma_to_sec4_sg_one(edesc->sec4_sg, iv_dma, ivsize, 0); 2624 sg_to_sec4_sg_last(req->src, src_nents, 2625 edesc->sec4_sg + 1, 0); 2626 sec4_sg_index += 1 + src_nents; 2627 } 2628 2629 if (dst_nents) { 2630 sg_to_sec4_sg_last(req->dst, dst_nents, 2631 edesc->sec4_sg + sec4_sg_index, 0); 2632 } 2633 2634 edesc->sec4_sg_dma = dma_map_single(jrdev, edesc->sec4_sg, 2635 sec4_sg_bytes, DMA_TO_DEVICE); 2636 if (dma_mapping_error(jrdev, edesc->sec4_sg_dma)) { 2637 dev_err(jrdev, "unable to map S/G table\n"); 2638 return ERR_PTR(-ENOMEM); 2639 } 2640 2641 edesc->iv_dma = iv_dma; 2642 2643 #ifdef DEBUG 2644 print_hex_dump(KERN_ERR, "ablkcipher sec4_sg@"__stringify(__LINE__)": ", 2645 DUMP_PREFIX_ADDRESS, 16, 4, edesc->sec4_sg, 2646 sec4_sg_bytes, 1); 2647 #endif 2648 2649 *iv_contig_out = iv_contig; 2650 return edesc; 2651 } 2652 2653 static int ablkcipher_encrypt(struct ablkcipher_request *req) 2654 { 2655 struct ablkcipher_edesc *edesc; 2656 struct crypto_ablkcipher *ablkcipher = crypto_ablkcipher_reqtfm(req); 2657 struct caam_ctx *ctx = crypto_ablkcipher_ctx(ablkcipher); 2658 struct device *jrdev = ctx->jrdev; 2659 bool iv_contig; 2660 u32 *desc; 2661 int ret = 0; 2662 2663 /* allocate extended descriptor */ 2664 edesc = ablkcipher_edesc_alloc(req, DESC_JOB_IO_LEN * 2665 CAAM_CMD_SZ, &iv_contig); 2666 if (IS_ERR(edesc)) 2667 return PTR_ERR(edesc); 2668 2669 /* Create and submit job descriptor*/ 2670 init_ablkcipher_job(ctx->sh_desc_enc, 2671 ctx->sh_desc_enc_dma, edesc, req, iv_contig); 2672 #ifdef DEBUG 2673 print_hex_dump(KERN_ERR, "ablkcipher jobdesc@"__stringify(__LINE__)": ", 2674 DUMP_PREFIX_ADDRESS, 16, 4, edesc->hw_desc, 2675 desc_bytes(edesc->hw_desc), 1); 2676 #endif 2677 desc = edesc->hw_desc; 2678 ret = caam_jr_enqueue(jrdev, desc, ablkcipher_encrypt_done, req); 2679 2680 if (!ret) { 2681 ret = -EINPROGRESS; 2682 } else { 2683 ablkcipher_unmap(jrdev, edesc, req); 2684 kfree(edesc); 2685 } 2686 2687 return ret; 2688 } 2689 2690 static int ablkcipher_decrypt(struct ablkcipher_request *req) 2691 { 2692 struct ablkcipher_edesc *edesc; 2693 struct crypto_ablkcipher *ablkcipher = crypto_ablkcipher_reqtfm(req); 2694 struct caam_ctx *ctx = crypto_ablkcipher_ctx(ablkcipher); 2695 struct device *jrdev = ctx->jrdev; 2696 bool iv_contig; 2697 u32 *desc; 2698 int ret = 0; 2699 2700 /* allocate extended descriptor */ 2701 edesc = ablkcipher_edesc_alloc(req, DESC_JOB_IO_LEN * 2702 CAAM_CMD_SZ, &iv_contig); 2703 if (IS_ERR(edesc)) 2704 return PTR_ERR(edesc); 2705 2706 /* Create and submit job descriptor*/ 2707 init_ablkcipher_job(ctx->sh_desc_dec, 2708 ctx->sh_desc_dec_dma, edesc, req, iv_contig); 2709 desc = edesc->hw_desc; 2710 #ifdef DEBUG 2711 print_hex_dump(KERN_ERR, "ablkcipher jobdesc@"__stringify(__LINE__)": ", 2712 DUMP_PREFIX_ADDRESS, 16, 4, edesc->hw_desc, 2713 desc_bytes(edesc->hw_desc), 1); 2714 #endif 2715 2716 ret = caam_jr_enqueue(jrdev, desc, ablkcipher_decrypt_done, req); 2717 if (!ret) { 2718 ret = -EINPROGRESS; 2719 } else { 2720 ablkcipher_unmap(jrdev, edesc, req); 2721 kfree(edesc); 2722 } 2723 2724 return ret; 2725 } 2726 2727 /* 2728 * allocate and map the ablkcipher extended descriptor 2729 * for ablkcipher givencrypt 2730 */ 2731 static struct ablkcipher_edesc *ablkcipher_giv_edesc_alloc( 2732 struct skcipher_givcrypt_request *greq, 2733 int desc_bytes, 2734 bool *iv_contig_out) 2735 { 2736 struct ablkcipher_request *req = &greq->creq; 2737 struct crypto_ablkcipher *ablkcipher = crypto_ablkcipher_reqtfm(req); 2738 struct caam_ctx *ctx = crypto_ablkcipher_ctx(ablkcipher); 2739 struct device *jrdev = ctx->jrdev; 2740 gfp_t flags = (req->base.flags & (CRYPTO_TFM_REQ_MAY_BACKLOG | 2741 CRYPTO_TFM_REQ_MAY_SLEEP)) ? 2742 GFP_KERNEL : GFP_ATOMIC; 2743 int src_nents, dst_nents = 0, sec4_sg_bytes; 2744 struct ablkcipher_edesc *edesc; 2745 dma_addr_t iv_dma = 0; 2746 bool iv_contig = false; 2747 int sgc; 2748 int ivsize = crypto_ablkcipher_ivsize(ablkcipher); 2749 int sec4_sg_index; 2750 2751 src_nents = sg_count(req->src, req->nbytes); 2752 2753 if (unlikely(req->dst != req->src)) 2754 dst_nents = sg_count(req->dst, req->nbytes); 2755 2756 if (likely(req->src == req->dst)) { 2757 sgc = dma_map_sg(jrdev, req->src, src_nents ? : 1, 2758 DMA_BIDIRECTIONAL); 2759 } else { 2760 sgc = dma_map_sg(jrdev, req->src, src_nents ? : 1, 2761 DMA_TO_DEVICE); 2762 sgc = dma_map_sg(jrdev, req->dst, dst_nents ? : 1, 2763 DMA_FROM_DEVICE); 2764 } 2765 2766 /* 2767 * Check if iv can be contiguous with source and destination. 2768 * If so, include it. If not, create scatterlist. 2769 */ 2770 iv_dma = dma_map_single(jrdev, greq->giv, ivsize, DMA_TO_DEVICE); 2771 if (dma_mapping_error(jrdev, iv_dma)) { 2772 dev_err(jrdev, "unable to map IV\n"); 2773 return ERR_PTR(-ENOMEM); 2774 } 2775 2776 if (!dst_nents && iv_dma + ivsize == sg_dma_address(req->dst)) 2777 iv_contig = true; 2778 else 2779 dst_nents = dst_nents ? : 1; 2780 sec4_sg_bytes = ((iv_contig ? 0 : 1) + src_nents + dst_nents) * 2781 sizeof(struct sec4_sg_entry); 2782 2783 /* allocate space for base edesc and hw desc commands, link tables */ 2784 edesc = kzalloc(sizeof(*edesc) + desc_bytes + sec4_sg_bytes, 2785 GFP_DMA | flags); 2786 if (!edesc) { 2787 dev_err(jrdev, "could not allocate extended descriptor\n"); 2788 return ERR_PTR(-ENOMEM); 2789 } 2790 2791 edesc->src_nents = src_nents; 2792 edesc->dst_nents = dst_nents; 2793 edesc->sec4_sg_bytes = sec4_sg_bytes; 2794 edesc->sec4_sg = (void *)edesc + sizeof(struct ablkcipher_edesc) + 2795 desc_bytes; 2796 2797 sec4_sg_index = 0; 2798 if (src_nents) { 2799 sg_to_sec4_sg_last(req->src, src_nents, edesc->sec4_sg, 0); 2800 sec4_sg_index += src_nents; 2801 } 2802 2803 if (!iv_contig) { 2804 dma_to_sec4_sg_one(edesc->sec4_sg + sec4_sg_index, 2805 iv_dma, ivsize, 0); 2806 sec4_sg_index += 1; 2807 sg_to_sec4_sg_last(req->dst, dst_nents, 2808 edesc->sec4_sg + sec4_sg_index, 0); 2809 } 2810 2811 edesc->sec4_sg_dma = dma_map_single(jrdev, edesc->sec4_sg, 2812 sec4_sg_bytes, DMA_TO_DEVICE); 2813 if (dma_mapping_error(jrdev, edesc->sec4_sg_dma)) { 2814 dev_err(jrdev, "unable to map S/G table\n"); 2815 return ERR_PTR(-ENOMEM); 2816 } 2817 edesc->iv_dma = iv_dma; 2818 2819 #ifdef DEBUG 2820 print_hex_dump(KERN_ERR, 2821 "ablkcipher sec4_sg@" __stringify(__LINE__) ": ", 2822 DUMP_PREFIX_ADDRESS, 16, 4, edesc->sec4_sg, 2823 sec4_sg_bytes, 1); 2824 #endif 2825 2826 *iv_contig_out = iv_contig; 2827 return edesc; 2828 } 2829 2830 static int ablkcipher_givencrypt(struct skcipher_givcrypt_request *creq) 2831 { 2832 struct ablkcipher_request *req = &creq->creq; 2833 struct ablkcipher_edesc *edesc; 2834 struct crypto_ablkcipher *ablkcipher = crypto_ablkcipher_reqtfm(req); 2835 struct caam_ctx *ctx = crypto_ablkcipher_ctx(ablkcipher); 2836 struct device *jrdev = ctx->jrdev; 2837 bool iv_contig; 2838 u32 *desc; 2839 int ret = 0; 2840 2841 /* allocate extended descriptor */ 2842 edesc = ablkcipher_giv_edesc_alloc(creq, DESC_JOB_IO_LEN * 2843 CAAM_CMD_SZ, &iv_contig); 2844 if (IS_ERR(edesc)) 2845 return PTR_ERR(edesc); 2846 2847 /* Create and submit job descriptor*/ 2848 init_ablkcipher_giv_job(ctx->sh_desc_givenc, ctx->sh_desc_givenc_dma, 2849 edesc, req, iv_contig); 2850 #ifdef DEBUG 2851 print_hex_dump(KERN_ERR, 2852 "ablkcipher jobdesc@" __stringify(__LINE__) ": ", 2853 DUMP_PREFIX_ADDRESS, 16, 4, edesc->hw_desc, 2854 desc_bytes(edesc->hw_desc), 1); 2855 #endif 2856 desc = edesc->hw_desc; 2857 ret = caam_jr_enqueue(jrdev, desc, ablkcipher_encrypt_done, req); 2858 2859 if (!ret) { 2860 ret = -EINPROGRESS; 2861 } else { 2862 ablkcipher_unmap(jrdev, edesc, req); 2863 kfree(edesc); 2864 } 2865 2866 return ret; 2867 } 2868 2869 #define template_aead template_u.aead 2870 #define template_ablkcipher template_u.ablkcipher 2871 struct caam_alg_template { 2872 char name[CRYPTO_MAX_ALG_NAME]; 2873 char driver_name[CRYPTO_MAX_ALG_NAME]; 2874 unsigned int blocksize; 2875 u32 type; 2876 union { 2877 struct ablkcipher_alg ablkcipher; 2878 } template_u; 2879 u32 class1_alg_type; 2880 u32 class2_alg_type; 2881 u32 alg_op; 2882 }; 2883 2884 static struct caam_alg_template driver_algs[] = { 2885 /* ablkcipher descriptor */ 2886 { 2887 .name = "cbc(aes)", 2888 .driver_name = "cbc-aes-caam", 2889 .blocksize = AES_BLOCK_SIZE, 2890 .type = CRYPTO_ALG_TYPE_GIVCIPHER, 2891 .template_ablkcipher = { 2892 .setkey = ablkcipher_setkey, 2893 .encrypt = ablkcipher_encrypt, 2894 .decrypt = ablkcipher_decrypt, 2895 .givencrypt = ablkcipher_givencrypt, 2896 .geniv = "<built-in>", 2897 .min_keysize = AES_MIN_KEY_SIZE, 2898 .max_keysize = AES_MAX_KEY_SIZE, 2899 .ivsize = AES_BLOCK_SIZE, 2900 }, 2901 .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CBC, 2902 }, 2903 { 2904 .name = "cbc(des3_ede)", 2905 .driver_name = "cbc-3des-caam", 2906 .blocksize = DES3_EDE_BLOCK_SIZE, 2907 .type = CRYPTO_ALG_TYPE_GIVCIPHER, 2908 .template_ablkcipher = { 2909 .setkey = ablkcipher_setkey, 2910 .encrypt = ablkcipher_encrypt, 2911 .decrypt = ablkcipher_decrypt, 2912 .givencrypt = ablkcipher_givencrypt, 2913 .geniv = "<built-in>", 2914 .min_keysize = DES3_EDE_KEY_SIZE, 2915 .max_keysize = DES3_EDE_KEY_SIZE, 2916 .ivsize = DES3_EDE_BLOCK_SIZE, 2917 }, 2918 .class1_alg_type = OP_ALG_ALGSEL_3DES | OP_ALG_AAI_CBC, 2919 }, 2920 { 2921 .name = "cbc(des)", 2922 .driver_name = "cbc-des-caam", 2923 .blocksize = DES_BLOCK_SIZE, 2924 .type = CRYPTO_ALG_TYPE_GIVCIPHER, 2925 .template_ablkcipher = { 2926 .setkey = ablkcipher_setkey, 2927 .encrypt = ablkcipher_encrypt, 2928 .decrypt = ablkcipher_decrypt, 2929 .givencrypt = ablkcipher_givencrypt, 2930 .geniv = "<built-in>", 2931 .min_keysize = DES_KEY_SIZE, 2932 .max_keysize = DES_KEY_SIZE, 2933 .ivsize = DES_BLOCK_SIZE, 2934 }, 2935 .class1_alg_type = OP_ALG_ALGSEL_DES | OP_ALG_AAI_CBC, 2936 }, 2937 { 2938 .name = "ctr(aes)", 2939 .driver_name = "ctr-aes-caam", 2940 .blocksize = 1, 2941 .type = CRYPTO_ALG_TYPE_ABLKCIPHER, 2942 .template_ablkcipher = { 2943 .setkey = ablkcipher_setkey, 2944 .encrypt = ablkcipher_encrypt, 2945 .decrypt = ablkcipher_decrypt, 2946 .geniv = "chainiv", 2947 .min_keysize = AES_MIN_KEY_SIZE, 2948 .max_keysize = AES_MAX_KEY_SIZE, 2949 .ivsize = AES_BLOCK_SIZE, 2950 }, 2951 .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CTR_MOD128, 2952 }, 2953 { 2954 .name = "rfc3686(ctr(aes))", 2955 .driver_name = "rfc3686-ctr-aes-caam", 2956 .blocksize = 1, 2957 .type = CRYPTO_ALG_TYPE_GIVCIPHER, 2958 .template_ablkcipher = { 2959 .setkey = ablkcipher_setkey, 2960 .encrypt = ablkcipher_encrypt, 2961 .decrypt = ablkcipher_decrypt, 2962 .givencrypt = ablkcipher_givencrypt, 2963 .geniv = "<built-in>", 2964 .min_keysize = AES_MIN_KEY_SIZE + 2965 CTR_RFC3686_NONCE_SIZE, 2966 .max_keysize = AES_MAX_KEY_SIZE + 2967 CTR_RFC3686_NONCE_SIZE, 2968 .ivsize = CTR_RFC3686_IV_SIZE, 2969 }, 2970 .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CTR_MOD128, 2971 }, 2972 { 2973 .name = "xts(aes)", 2974 .driver_name = "xts-aes-caam", 2975 .blocksize = AES_BLOCK_SIZE, 2976 .type = CRYPTO_ALG_TYPE_ABLKCIPHER, 2977 .template_ablkcipher = { 2978 .setkey = xts_ablkcipher_setkey, 2979 .encrypt = ablkcipher_encrypt, 2980 .decrypt = ablkcipher_decrypt, 2981 .geniv = "eseqiv", 2982 .min_keysize = 2 * AES_MIN_KEY_SIZE, 2983 .max_keysize = 2 * AES_MAX_KEY_SIZE, 2984 .ivsize = AES_BLOCK_SIZE, 2985 }, 2986 .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_XTS, 2987 }, 2988 }; 2989 2990 static struct caam_aead_alg driver_aeads[] = { 2991 { 2992 .aead = { 2993 .base = { 2994 .cra_name = "rfc4106(gcm(aes))", 2995 .cra_driver_name = "rfc4106-gcm-aes-caam", 2996 .cra_blocksize = 1, 2997 }, 2998 .setkey = rfc4106_setkey, 2999 .setauthsize = rfc4106_setauthsize, 3000 .encrypt = ipsec_gcm_encrypt, 3001 .decrypt = ipsec_gcm_decrypt, 3002 .ivsize = 8, 3003 .maxauthsize = AES_BLOCK_SIZE, 3004 }, 3005 .caam = { 3006 .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_GCM, 3007 }, 3008 }, 3009 { 3010 .aead = { 3011 .base = { 3012 .cra_name = "rfc4543(gcm(aes))", 3013 .cra_driver_name = "rfc4543-gcm-aes-caam", 3014 .cra_blocksize = 1, 3015 }, 3016 .setkey = rfc4543_setkey, 3017 .setauthsize = rfc4543_setauthsize, 3018 .encrypt = ipsec_gcm_encrypt, 3019 .decrypt = ipsec_gcm_decrypt, 3020 .ivsize = 8, 3021 .maxauthsize = AES_BLOCK_SIZE, 3022 }, 3023 .caam = { 3024 .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_GCM, 3025 }, 3026 }, 3027 /* Galois Counter Mode */ 3028 { 3029 .aead = { 3030 .base = { 3031 .cra_name = "gcm(aes)", 3032 .cra_driver_name = "gcm-aes-caam", 3033 .cra_blocksize = 1, 3034 }, 3035 .setkey = gcm_setkey, 3036 .setauthsize = gcm_setauthsize, 3037 .encrypt = gcm_encrypt, 3038 .decrypt = gcm_decrypt, 3039 .ivsize = 12, 3040 .maxauthsize = AES_BLOCK_SIZE, 3041 }, 3042 .caam = { 3043 .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_GCM, 3044 }, 3045 }, 3046 /* single-pass ipsec_esp descriptor */ 3047 { 3048 .aead = { 3049 .base = { 3050 .cra_name = "authenc(hmac(md5)," 3051 "ecb(cipher_null))", 3052 .cra_driver_name = "authenc-hmac-md5-" 3053 "ecb-cipher_null-caam", 3054 .cra_blocksize = NULL_BLOCK_SIZE, 3055 }, 3056 .setkey = aead_setkey, 3057 .setauthsize = aead_setauthsize, 3058 .encrypt = aead_encrypt, 3059 .decrypt = aead_decrypt, 3060 .ivsize = NULL_IV_SIZE, 3061 .maxauthsize = MD5_DIGEST_SIZE, 3062 }, 3063 .caam = { 3064 .class2_alg_type = OP_ALG_ALGSEL_MD5 | 3065 OP_ALG_AAI_HMAC_PRECOMP, 3066 .alg_op = OP_ALG_ALGSEL_MD5 | OP_ALG_AAI_HMAC, 3067 }, 3068 }, 3069 { 3070 .aead = { 3071 .base = { 3072 .cra_name = "authenc(hmac(sha1)," 3073 "ecb(cipher_null))", 3074 .cra_driver_name = "authenc-hmac-sha1-" 3075 "ecb-cipher_null-caam", 3076 .cra_blocksize = NULL_BLOCK_SIZE, 3077 }, 3078 .setkey = aead_setkey, 3079 .setauthsize = aead_setauthsize, 3080 .encrypt = aead_encrypt, 3081 .decrypt = aead_decrypt, 3082 .ivsize = NULL_IV_SIZE, 3083 .maxauthsize = SHA1_DIGEST_SIZE, 3084 }, 3085 .caam = { 3086 .class2_alg_type = OP_ALG_ALGSEL_SHA1 | 3087 OP_ALG_AAI_HMAC_PRECOMP, 3088 .alg_op = OP_ALG_ALGSEL_SHA1 | OP_ALG_AAI_HMAC, 3089 }, 3090 }, 3091 { 3092 .aead = { 3093 .base = { 3094 .cra_name = "authenc(hmac(sha224)," 3095 "ecb(cipher_null))", 3096 .cra_driver_name = "authenc-hmac-sha224-" 3097 "ecb-cipher_null-caam", 3098 .cra_blocksize = NULL_BLOCK_SIZE, 3099 }, 3100 .setkey = aead_setkey, 3101 .setauthsize = aead_setauthsize, 3102 .encrypt = aead_encrypt, 3103 .decrypt = aead_decrypt, 3104 .ivsize = NULL_IV_SIZE, 3105 .maxauthsize = SHA224_DIGEST_SIZE, 3106 }, 3107 .caam = { 3108 .class2_alg_type = OP_ALG_ALGSEL_SHA224 | 3109 OP_ALG_AAI_HMAC_PRECOMP, 3110 .alg_op = OP_ALG_ALGSEL_SHA224 | OP_ALG_AAI_HMAC, 3111 }, 3112 }, 3113 { 3114 .aead = { 3115 .base = { 3116 .cra_name = "authenc(hmac(sha256)," 3117 "ecb(cipher_null))", 3118 .cra_driver_name = "authenc-hmac-sha256-" 3119 "ecb-cipher_null-caam", 3120 .cra_blocksize = NULL_BLOCK_SIZE, 3121 }, 3122 .setkey = aead_setkey, 3123 .setauthsize = aead_setauthsize, 3124 .encrypt = aead_encrypt, 3125 .decrypt = aead_decrypt, 3126 .ivsize = NULL_IV_SIZE, 3127 .maxauthsize = SHA256_DIGEST_SIZE, 3128 }, 3129 .caam = { 3130 .class2_alg_type = OP_ALG_ALGSEL_SHA256 | 3131 OP_ALG_AAI_HMAC_PRECOMP, 3132 .alg_op = OP_ALG_ALGSEL_SHA256 | OP_ALG_AAI_HMAC, 3133 }, 3134 }, 3135 { 3136 .aead = { 3137 .base = { 3138 .cra_name = "authenc(hmac(sha384)," 3139 "ecb(cipher_null))", 3140 .cra_driver_name = "authenc-hmac-sha384-" 3141 "ecb-cipher_null-caam", 3142 .cra_blocksize = NULL_BLOCK_SIZE, 3143 }, 3144 .setkey = aead_setkey, 3145 .setauthsize = aead_setauthsize, 3146 .encrypt = aead_encrypt, 3147 .decrypt = aead_decrypt, 3148 .ivsize = NULL_IV_SIZE, 3149 .maxauthsize = SHA384_DIGEST_SIZE, 3150 }, 3151 .caam = { 3152 .class2_alg_type = OP_ALG_ALGSEL_SHA384 | 3153 OP_ALG_AAI_HMAC_PRECOMP, 3154 .alg_op = OP_ALG_ALGSEL_SHA384 | OP_ALG_AAI_HMAC, 3155 }, 3156 }, 3157 { 3158 .aead = { 3159 .base = { 3160 .cra_name = "authenc(hmac(sha512)," 3161 "ecb(cipher_null))", 3162 .cra_driver_name = "authenc-hmac-sha512-" 3163 "ecb-cipher_null-caam", 3164 .cra_blocksize = NULL_BLOCK_SIZE, 3165 }, 3166 .setkey = aead_setkey, 3167 .setauthsize = aead_setauthsize, 3168 .encrypt = aead_encrypt, 3169 .decrypt = aead_decrypt, 3170 .ivsize = NULL_IV_SIZE, 3171 .maxauthsize = SHA512_DIGEST_SIZE, 3172 }, 3173 .caam = { 3174 .class2_alg_type = OP_ALG_ALGSEL_SHA512 | 3175 OP_ALG_AAI_HMAC_PRECOMP, 3176 .alg_op = OP_ALG_ALGSEL_SHA512 | OP_ALG_AAI_HMAC, 3177 }, 3178 }, 3179 { 3180 .aead = { 3181 .base = { 3182 .cra_name = "authenc(hmac(md5),cbc(aes))", 3183 .cra_driver_name = "authenc-hmac-md5-" 3184 "cbc-aes-caam", 3185 .cra_blocksize = AES_BLOCK_SIZE, 3186 }, 3187 .setkey = aead_setkey, 3188 .setauthsize = aead_setauthsize, 3189 .encrypt = aead_encrypt, 3190 .decrypt = aead_decrypt, 3191 .ivsize = AES_BLOCK_SIZE, 3192 .maxauthsize = MD5_DIGEST_SIZE, 3193 }, 3194 .caam = { 3195 .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CBC, 3196 .class2_alg_type = OP_ALG_ALGSEL_MD5 | 3197 OP_ALG_AAI_HMAC_PRECOMP, 3198 .alg_op = OP_ALG_ALGSEL_MD5 | OP_ALG_AAI_HMAC, 3199 }, 3200 }, 3201 { 3202 .aead = { 3203 .base = { 3204 .cra_name = "echainiv(authenc(hmac(md5)," 3205 "cbc(aes)))", 3206 .cra_driver_name = "echainiv-authenc-hmac-md5-" 3207 "cbc-aes-caam", 3208 .cra_blocksize = AES_BLOCK_SIZE, 3209 }, 3210 .setkey = aead_setkey, 3211 .setauthsize = aead_setauthsize, 3212 .encrypt = aead_encrypt, 3213 .decrypt = aead_givdecrypt, 3214 .ivsize = AES_BLOCK_SIZE, 3215 .maxauthsize = MD5_DIGEST_SIZE, 3216 }, 3217 .caam = { 3218 .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CBC, 3219 .class2_alg_type = OP_ALG_ALGSEL_MD5 | 3220 OP_ALG_AAI_HMAC_PRECOMP, 3221 .alg_op = OP_ALG_ALGSEL_MD5 | OP_ALG_AAI_HMAC, 3222 .geniv = true, 3223 }, 3224 }, 3225 { 3226 .aead = { 3227 .base = { 3228 .cra_name = "authenc(hmac(sha1),cbc(aes))", 3229 .cra_driver_name = "authenc-hmac-sha1-" 3230 "cbc-aes-caam", 3231 .cra_blocksize = AES_BLOCK_SIZE, 3232 }, 3233 .setkey = aead_setkey, 3234 .setauthsize = aead_setauthsize, 3235 .encrypt = aead_encrypt, 3236 .decrypt = aead_decrypt, 3237 .ivsize = AES_BLOCK_SIZE, 3238 .maxauthsize = SHA1_DIGEST_SIZE, 3239 }, 3240 .caam = { 3241 .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CBC, 3242 .class2_alg_type = OP_ALG_ALGSEL_SHA1 | 3243 OP_ALG_AAI_HMAC_PRECOMP, 3244 .alg_op = OP_ALG_ALGSEL_SHA1 | OP_ALG_AAI_HMAC, 3245 }, 3246 }, 3247 { 3248 .aead = { 3249 .base = { 3250 .cra_name = "echainiv(authenc(hmac(sha1)," 3251 "cbc(aes)))", 3252 .cra_driver_name = "echainiv-authenc-" 3253 "hmac-sha1-cbc-aes-caam", 3254 .cra_blocksize = AES_BLOCK_SIZE, 3255 }, 3256 .setkey = aead_setkey, 3257 .setauthsize = aead_setauthsize, 3258 .encrypt = aead_encrypt, 3259 .decrypt = aead_givdecrypt, 3260 .ivsize = AES_BLOCK_SIZE, 3261 .maxauthsize = SHA1_DIGEST_SIZE, 3262 }, 3263 .caam = { 3264 .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CBC, 3265 .class2_alg_type = OP_ALG_ALGSEL_SHA1 | 3266 OP_ALG_AAI_HMAC_PRECOMP, 3267 .alg_op = OP_ALG_ALGSEL_SHA1 | OP_ALG_AAI_HMAC, 3268 .geniv = true, 3269 }, 3270 }, 3271 { 3272 .aead = { 3273 .base = { 3274 .cra_name = "authenc(hmac(sha224),cbc(aes))", 3275 .cra_driver_name = "authenc-hmac-sha224-" 3276 "cbc-aes-caam", 3277 .cra_blocksize = AES_BLOCK_SIZE, 3278 }, 3279 .setkey = aead_setkey, 3280 .setauthsize = aead_setauthsize, 3281 .encrypt = aead_encrypt, 3282 .decrypt = aead_decrypt, 3283 .ivsize = AES_BLOCK_SIZE, 3284 .maxauthsize = SHA224_DIGEST_SIZE, 3285 }, 3286 .caam = { 3287 .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CBC, 3288 .class2_alg_type = OP_ALG_ALGSEL_SHA224 | 3289 OP_ALG_AAI_HMAC_PRECOMP, 3290 .alg_op = OP_ALG_ALGSEL_SHA224 | OP_ALG_AAI_HMAC, 3291 }, 3292 }, 3293 { 3294 .aead = { 3295 .base = { 3296 .cra_name = "echainiv(authenc(hmac(sha224)," 3297 "cbc(aes)))", 3298 .cra_driver_name = "echainiv-authenc-" 3299 "hmac-sha224-cbc-aes-caam", 3300 .cra_blocksize = AES_BLOCK_SIZE, 3301 }, 3302 .setkey = aead_setkey, 3303 .setauthsize = aead_setauthsize, 3304 .encrypt = aead_encrypt, 3305 .decrypt = aead_givdecrypt, 3306 .ivsize = AES_BLOCK_SIZE, 3307 .maxauthsize = SHA224_DIGEST_SIZE, 3308 }, 3309 .caam = { 3310 .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CBC, 3311 .class2_alg_type = OP_ALG_ALGSEL_SHA224 | 3312 OP_ALG_AAI_HMAC_PRECOMP, 3313 .alg_op = OP_ALG_ALGSEL_SHA224 | OP_ALG_AAI_HMAC, 3314 .geniv = true, 3315 }, 3316 }, 3317 { 3318 .aead = { 3319 .base = { 3320 .cra_name = "authenc(hmac(sha256),cbc(aes))", 3321 .cra_driver_name = "authenc-hmac-sha256-" 3322 "cbc-aes-caam", 3323 .cra_blocksize = AES_BLOCK_SIZE, 3324 }, 3325 .setkey = aead_setkey, 3326 .setauthsize = aead_setauthsize, 3327 .encrypt = aead_encrypt, 3328 .decrypt = aead_decrypt, 3329 .ivsize = AES_BLOCK_SIZE, 3330 .maxauthsize = SHA256_DIGEST_SIZE, 3331 }, 3332 .caam = { 3333 .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CBC, 3334 .class2_alg_type = OP_ALG_ALGSEL_SHA256 | 3335 OP_ALG_AAI_HMAC_PRECOMP, 3336 .alg_op = OP_ALG_ALGSEL_SHA256 | OP_ALG_AAI_HMAC, 3337 }, 3338 }, 3339 { 3340 .aead = { 3341 .base = { 3342 .cra_name = "echainiv(authenc(hmac(sha256)," 3343 "cbc(aes)))", 3344 .cra_driver_name = "echainiv-authenc-" 3345 "hmac-sha256-cbc-aes-caam", 3346 .cra_blocksize = AES_BLOCK_SIZE, 3347 }, 3348 .setkey = aead_setkey, 3349 .setauthsize = aead_setauthsize, 3350 .encrypt = aead_encrypt, 3351 .decrypt = aead_givdecrypt, 3352 .ivsize = AES_BLOCK_SIZE, 3353 .maxauthsize = SHA256_DIGEST_SIZE, 3354 }, 3355 .caam = { 3356 .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CBC, 3357 .class2_alg_type = OP_ALG_ALGSEL_SHA256 | 3358 OP_ALG_AAI_HMAC_PRECOMP, 3359 .alg_op = OP_ALG_ALGSEL_SHA256 | OP_ALG_AAI_HMAC, 3360 .geniv = true, 3361 }, 3362 }, 3363 { 3364 .aead = { 3365 .base = { 3366 .cra_name = "authenc(hmac(sha384),cbc(aes))", 3367 .cra_driver_name = "authenc-hmac-sha384-" 3368 "cbc-aes-caam", 3369 .cra_blocksize = AES_BLOCK_SIZE, 3370 }, 3371 .setkey = aead_setkey, 3372 .setauthsize = aead_setauthsize, 3373 .encrypt = aead_encrypt, 3374 .decrypt = aead_decrypt, 3375 .ivsize = AES_BLOCK_SIZE, 3376 .maxauthsize = SHA384_DIGEST_SIZE, 3377 }, 3378 .caam = { 3379 .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CBC, 3380 .class2_alg_type = OP_ALG_ALGSEL_SHA384 | 3381 OP_ALG_AAI_HMAC_PRECOMP, 3382 .alg_op = OP_ALG_ALGSEL_SHA384 | OP_ALG_AAI_HMAC, 3383 }, 3384 }, 3385 { 3386 .aead = { 3387 .base = { 3388 .cra_name = "echainiv(authenc(hmac(sha384)," 3389 "cbc(aes)))", 3390 .cra_driver_name = "echainiv-authenc-" 3391 "hmac-sha384-cbc-aes-caam", 3392 .cra_blocksize = AES_BLOCK_SIZE, 3393 }, 3394 .setkey = aead_setkey, 3395 .setauthsize = aead_setauthsize, 3396 .encrypt = aead_encrypt, 3397 .decrypt = aead_givdecrypt, 3398 .ivsize = AES_BLOCK_SIZE, 3399 .maxauthsize = SHA384_DIGEST_SIZE, 3400 }, 3401 .caam = { 3402 .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CBC, 3403 .class2_alg_type = OP_ALG_ALGSEL_SHA384 | 3404 OP_ALG_AAI_HMAC_PRECOMP, 3405 .alg_op = OP_ALG_ALGSEL_SHA384 | OP_ALG_AAI_HMAC, 3406 .geniv = true, 3407 }, 3408 }, 3409 { 3410 .aead = { 3411 .base = { 3412 .cra_name = "authenc(hmac(sha512),cbc(aes))", 3413 .cra_driver_name = "authenc-hmac-sha512-" 3414 "cbc-aes-caam", 3415 .cra_blocksize = AES_BLOCK_SIZE, 3416 }, 3417 .setkey = aead_setkey, 3418 .setauthsize = aead_setauthsize, 3419 .encrypt = aead_encrypt, 3420 .decrypt = aead_decrypt, 3421 .ivsize = AES_BLOCK_SIZE, 3422 .maxauthsize = SHA512_DIGEST_SIZE, 3423 }, 3424 .caam = { 3425 .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CBC, 3426 .class2_alg_type = OP_ALG_ALGSEL_SHA512 | 3427 OP_ALG_AAI_HMAC_PRECOMP, 3428 .alg_op = OP_ALG_ALGSEL_SHA512 | OP_ALG_AAI_HMAC, 3429 }, 3430 }, 3431 { 3432 .aead = { 3433 .base = { 3434 .cra_name = "echainiv(authenc(hmac(sha512)," 3435 "cbc(aes)))", 3436 .cra_driver_name = "echainiv-authenc-" 3437 "hmac-sha512-cbc-aes-caam", 3438 .cra_blocksize = AES_BLOCK_SIZE, 3439 }, 3440 .setkey = aead_setkey, 3441 .setauthsize = aead_setauthsize, 3442 .encrypt = aead_encrypt, 3443 .decrypt = aead_givdecrypt, 3444 .ivsize = AES_BLOCK_SIZE, 3445 .maxauthsize = SHA512_DIGEST_SIZE, 3446 }, 3447 .caam = { 3448 .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CBC, 3449 .class2_alg_type = OP_ALG_ALGSEL_SHA512 | 3450 OP_ALG_AAI_HMAC_PRECOMP, 3451 .alg_op = OP_ALG_ALGSEL_SHA512 | OP_ALG_AAI_HMAC, 3452 .geniv = true, 3453 }, 3454 }, 3455 { 3456 .aead = { 3457 .base = { 3458 .cra_name = "authenc(hmac(md5),cbc(des3_ede))", 3459 .cra_driver_name = "authenc-hmac-md5-" 3460 "cbc-des3_ede-caam", 3461 .cra_blocksize = DES3_EDE_BLOCK_SIZE, 3462 }, 3463 .setkey = aead_setkey, 3464 .setauthsize = aead_setauthsize, 3465 .encrypt = aead_encrypt, 3466 .decrypt = aead_decrypt, 3467 .ivsize = DES3_EDE_BLOCK_SIZE, 3468 .maxauthsize = MD5_DIGEST_SIZE, 3469 }, 3470 .caam = { 3471 .class1_alg_type = OP_ALG_ALGSEL_3DES | OP_ALG_AAI_CBC, 3472 .class2_alg_type = OP_ALG_ALGSEL_MD5 | 3473 OP_ALG_AAI_HMAC_PRECOMP, 3474 .alg_op = OP_ALG_ALGSEL_MD5 | OP_ALG_AAI_HMAC, 3475 } 3476 }, 3477 { 3478 .aead = { 3479 .base = { 3480 .cra_name = "echainiv(authenc(hmac(md5)," 3481 "cbc(des3_ede)))", 3482 .cra_driver_name = "echainiv-authenc-hmac-md5-" 3483 "cbc-des3_ede-caam", 3484 .cra_blocksize = DES3_EDE_BLOCK_SIZE, 3485 }, 3486 .setkey = aead_setkey, 3487 .setauthsize = aead_setauthsize, 3488 .encrypt = aead_encrypt, 3489 .decrypt = aead_givdecrypt, 3490 .ivsize = DES3_EDE_BLOCK_SIZE, 3491 .maxauthsize = MD5_DIGEST_SIZE, 3492 }, 3493 .caam = { 3494 .class1_alg_type = OP_ALG_ALGSEL_3DES | OP_ALG_AAI_CBC, 3495 .class2_alg_type = OP_ALG_ALGSEL_MD5 | 3496 OP_ALG_AAI_HMAC_PRECOMP, 3497 .alg_op = OP_ALG_ALGSEL_MD5 | OP_ALG_AAI_HMAC, 3498 .geniv = true, 3499 } 3500 }, 3501 { 3502 .aead = { 3503 .base = { 3504 .cra_name = "authenc(hmac(sha1)," 3505 "cbc(des3_ede))", 3506 .cra_driver_name = "authenc-hmac-sha1-" 3507 "cbc-des3_ede-caam", 3508 .cra_blocksize = DES3_EDE_BLOCK_SIZE, 3509 }, 3510 .setkey = aead_setkey, 3511 .setauthsize = aead_setauthsize, 3512 .encrypt = aead_encrypt, 3513 .decrypt = aead_decrypt, 3514 .ivsize = DES3_EDE_BLOCK_SIZE, 3515 .maxauthsize = SHA1_DIGEST_SIZE, 3516 }, 3517 .caam = { 3518 .class1_alg_type = OP_ALG_ALGSEL_3DES | OP_ALG_AAI_CBC, 3519 .class2_alg_type = OP_ALG_ALGSEL_SHA1 | 3520 OP_ALG_AAI_HMAC_PRECOMP, 3521 .alg_op = OP_ALG_ALGSEL_SHA1 | OP_ALG_AAI_HMAC, 3522 }, 3523 }, 3524 { 3525 .aead = { 3526 .base = { 3527 .cra_name = "echainiv(authenc(hmac(sha1)," 3528 "cbc(des3_ede)))", 3529 .cra_driver_name = "echainiv-authenc-" 3530 "hmac-sha1-" 3531 "cbc-des3_ede-caam", 3532 .cra_blocksize = DES3_EDE_BLOCK_SIZE, 3533 }, 3534 .setkey = aead_setkey, 3535 .setauthsize = aead_setauthsize, 3536 .encrypt = aead_encrypt, 3537 .decrypt = aead_givdecrypt, 3538 .ivsize = DES3_EDE_BLOCK_SIZE, 3539 .maxauthsize = SHA1_DIGEST_SIZE, 3540 }, 3541 .caam = { 3542 .class1_alg_type = OP_ALG_ALGSEL_3DES | OP_ALG_AAI_CBC, 3543 .class2_alg_type = OP_ALG_ALGSEL_SHA1 | 3544 OP_ALG_AAI_HMAC_PRECOMP, 3545 .alg_op = OP_ALG_ALGSEL_SHA1 | OP_ALG_AAI_HMAC, 3546 .geniv = true, 3547 }, 3548 }, 3549 { 3550 .aead = { 3551 .base = { 3552 .cra_name = "authenc(hmac(sha224)," 3553 "cbc(des3_ede))", 3554 .cra_driver_name = "authenc-hmac-sha224-" 3555 "cbc-des3_ede-caam", 3556 .cra_blocksize = DES3_EDE_BLOCK_SIZE, 3557 }, 3558 .setkey = aead_setkey, 3559 .setauthsize = aead_setauthsize, 3560 .encrypt = aead_encrypt, 3561 .decrypt = aead_decrypt, 3562 .ivsize = DES3_EDE_BLOCK_SIZE, 3563 .maxauthsize = SHA224_DIGEST_SIZE, 3564 }, 3565 .caam = { 3566 .class1_alg_type = OP_ALG_ALGSEL_3DES | OP_ALG_AAI_CBC, 3567 .class2_alg_type = OP_ALG_ALGSEL_SHA224 | 3568 OP_ALG_AAI_HMAC_PRECOMP, 3569 .alg_op = OP_ALG_ALGSEL_SHA224 | OP_ALG_AAI_HMAC, 3570 }, 3571 }, 3572 { 3573 .aead = { 3574 .base = { 3575 .cra_name = "echainiv(authenc(hmac(sha224)," 3576 "cbc(des3_ede)))", 3577 .cra_driver_name = "echainiv-authenc-" 3578 "hmac-sha224-" 3579 "cbc-des3_ede-caam", 3580 .cra_blocksize = DES3_EDE_BLOCK_SIZE, 3581 }, 3582 .setkey = aead_setkey, 3583 .setauthsize = aead_setauthsize, 3584 .encrypt = aead_encrypt, 3585 .decrypt = aead_givdecrypt, 3586 .ivsize = DES3_EDE_BLOCK_SIZE, 3587 .maxauthsize = SHA224_DIGEST_SIZE, 3588 }, 3589 .caam = { 3590 .class1_alg_type = OP_ALG_ALGSEL_3DES | OP_ALG_AAI_CBC, 3591 .class2_alg_type = OP_ALG_ALGSEL_SHA224 | 3592 OP_ALG_AAI_HMAC_PRECOMP, 3593 .alg_op = OP_ALG_ALGSEL_SHA224 | OP_ALG_AAI_HMAC, 3594 .geniv = true, 3595 }, 3596 }, 3597 { 3598 .aead = { 3599 .base = { 3600 .cra_name = "authenc(hmac(sha256)," 3601 "cbc(des3_ede))", 3602 .cra_driver_name = "authenc-hmac-sha256-" 3603 "cbc-des3_ede-caam", 3604 .cra_blocksize = DES3_EDE_BLOCK_SIZE, 3605 }, 3606 .setkey = aead_setkey, 3607 .setauthsize = aead_setauthsize, 3608 .encrypt = aead_encrypt, 3609 .decrypt = aead_decrypt, 3610 .ivsize = DES3_EDE_BLOCK_SIZE, 3611 .maxauthsize = SHA256_DIGEST_SIZE, 3612 }, 3613 .caam = { 3614 .class1_alg_type = OP_ALG_ALGSEL_3DES | OP_ALG_AAI_CBC, 3615 .class2_alg_type = OP_ALG_ALGSEL_SHA256 | 3616 OP_ALG_AAI_HMAC_PRECOMP, 3617 .alg_op = OP_ALG_ALGSEL_SHA256 | OP_ALG_AAI_HMAC, 3618 }, 3619 }, 3620 { 3621 .aead = { 3622 .base = { 3623 .cra_name = "echainiv(authenc(hmac(sha256)," 3624 "cbc(des3_ede)))", 3625 .cra_driver_name = "echainiv-authenc-" 3626 "hmac-sha256-" 3627 "cbc-des3_ede-caam", 3628 .cra_blocksize = DES3_EDE_BLOCK_SIZE, 3629 }, 3630 .setkey = aead_setkey, 3631 .setauthsize = aead_setauthsize, 3632 .encrypt = aead_encrypt, 3633 .decrypt = aead_givdecrypt, 3634 .ivsize = DES3_EDE_BLOCK_SIZE, 3635 .maxauthsize = SHA256_DIGEST_SIZE, 3636 }, 3637 .caam = { 3638 .class1_alg_type = OP_ALG_ALGSEL_3DES | OP_ALG_AAI_CBC, 3639 .class2_alg_type = OP_ALG_ALGSEL_SHA256 | 3640 OP_ALG_AAI_HMAC_PRECOMP, 3641 .alg_op = OP_ALG_ALGSEL_SHA256 | OP_ALG_AAI_HMAC, 3642 .geniv = true, 3643 }, 3644 }, 3645 { 3646 .aead = { 3647 .base = { 3648 .cra_name = "authenc(hmac(sha384)," 3649 "cbc(des3_ede))", 3650 .cra_driver_name = "authenc-hmac-sha384-" 3651 "cbc-des3_ede-caam", 3652 .cra_blocksize = DES3_EDE_BLOCK_SIZE, 3653 }, 3654 .setkey = aead_setkey, 3655 .setauthsize = aead_setauthsize, 3656 .encrypt = aead_encrypt, 3657 .decrypt = aead_decrypt, 3658 .ivsize = DES3_EDE_BLOCK_SIZE, 3659 .maxauthsize = SHA384_DIGEST_SIZE, 3660 }, 3661 .caam = { 3662 .class1_alg_type = OP_ALG_ALGSEL_3DES | OP_ALG_AAI_CBC, 3663 .class2_alg_type = OP_ALG_ALGSEL_SHA384 | 3664 OP_ALG_AAI_HMAC_PRECOMP, 3665 .alg_op = OP_ALG_ALGSEL_SHA384 | OP_ALG_AAI_HMAC, 3666 }, 3667 }, 3668 { 3669 .aead = { 3670 .base = { 3671 .cra_name = "echainiv(authenc(hmac(sha384)," 3672 "cbc(des3_ede)))", 3673 .cra_driver_name = "echainiv-authenc-" 3674 "hmac-sha384-" 3675 "cbc-des3_ede-caam", 3676 .cra_blocksize = DES3_EDE_BLOCK_SIZE, 3677 }, 3678 .setkey = aead_setkey, 3679 .setauthsize = aead_setauthsize, 3680 .encrypt = aead_encrypt, 3681 .decrypt = aead_givdecrypt, 3682 .ivsize = DES3_EDE_BLOCK_SIZE, 3683 .maxauthsize = SHA384_DIGEST_SIZE, 3684 }, 3685 .caam = { 3686 .class1_alg_type = OP_ALG_ALGSEL_3DES | OP_ALG_AAI_CBC, 3687 .class2_alg_type = OP_ALG_ALGSEL_SHA384 | 3688 OP_ALG_AAI_HMAC_PRECOMP, 3689 .alg_op = OP_ALG_ALGSEL_SHA384 | OP_ALG_AAI_HMAC, 3690 .geniv = true, 3691 }, 3692 }, 3693 { 3694 .aead = { 3695 .base = { 3696 .cra_name = "authenc(hmac(sha512)," 3697 "cbc(des3_ede))", 3698 .cra_driver_name = "authenc-hmac-sha512-" 3699 "cbc-des3_ede-caam", 3700 .cra_blocksize = DES3_EDE_BLOCK_SIZE, 3701 }, 3702 .setkey = aead_setkey, 3703 .setauthsize = aead_setauthsize, 3704 .encrypt = aead_encrypt, 3705 .decrypt = aead_decrypt, 3706 .ivsize = DES3_EDE_BLOCK_SIZE, 3707 .maxauthsize = SHA512_DIGEST_SIZE, 3708 }, 3709 .caam = { 3710 .class1_alg_type = OP_ALG_ALGSEL_3DES | OP_ALG_AAI_CBC, 3711 .class2_alg_type = OP_ALG_ALGSEL_SHA512 | 3712 OP_ALG_AAI_HMAC_PRECOMP, 3713 .alg_op = OP_ALG_ALGSEL_SHA512 | OP_ALG_AAI_HMAC, 3714 }, 3715 }, 3716 { 3717 .aead = { 3718 .base = { 3719 .cra_name = "echainiv(authenc(hmac(sha512)," 3720 "cbc(des3_ede)))", 3721 .cra_driver_name = "echainiv-authenc-" 3722 "hmac-sha512-" 3723 "cbc-des3_ede-caam", 3724 .cra_blocksize = DES3_EDE_BLOCK_SIZE, 3725 }, 3726 .setkey = aead_setkey, 3727 .setauthsize = aead_setauthsize, 3728 .encrypt = aead_encrypt, 3729 .decrypt = aead_givdecrypt, 3730 .ivsize = DES3_EDE_BLOCK_SIZE, 3731 .maxauthsize = SHA512_DIGEST_SIZE, 3732 }, 3733 .caam = { 3734 .class1_alg_type = OP_ALG_ALGSEL_3DES | OP_ALG_AAI_CBC, 3735 .class2_alg_type = OP_ALG_ALGSEL_SHA512 | 3736 OP_ALG_AAI_HMAC_PRECOMP, 3737 .alg_op = OP_ALG_ALGSEL_SHA512 | OP_ALG_AAI_HMAC, 3738 .geniv = true, 3739 }, 3740 }, 3741 { 3742 .aead = { 3743 .base = { 3744 .cra_name = "authenc(hmac(md5),cbc(des))", 3745 .cra_driver_name = "authenc-hmac-md5-" 3746 "cbc-des-caam", 3747 .cra_blocksize = DES_BLOCK_SIZE, 3748 }, 3749 .setkey = aead_setkey, 3750 .setauthsize = aead_setauthsize, 3751 .encrypt = aead_encrypt, 3752 .decrypt = aead_decrypt, 3753 .ivsize = DES_BLOCK_SIZE, 3754 .maxauthsize = MD5_DIGEST_SIZE, 3755 }, 3756 .caam = { 3757 .class1_alg_type = OP_ALG_ALGSEL_DES | OP_ALG_AAI_CBC, 3758 .class2_alg_type = OP_ALG_ALGSEL_MD5 | 3759 OP_ALG_AAI_HMAC_PRECOMP, 3760 .alg_op = OP_ALG_ALGSEL_MD5 | OP_ALG_AAI_HMAC, 3761 }, 3762 }, 3763 { 3764 .aead = { 3765 .base = { 3766 .cra_name = "echainiv(authenc(hmac(md5)," 3767 "cbc(des)))", 3768 .cra_driver_name = "echainiv-authenc-hmac-md5-" 3769 "cbc-des-caam", 3770 .cra_blocksize = DES_BLOCK_SIZE, 3771 }, 3772 .setkey = aead_setkey, 3773 .setauthsize = aead_setauthsize, 3774 .encrypt = aead_encrypt, 3775 .decrypt = aead_givdecrypt, 3776 .ivsize = DES_BLOCK_SIZE, 3777 .maxauthsize = MD5_DIGEST_SIZE, 3778 }, 3779 .caam = { 3780 .class1_alg_type = OP_ALG_ALGSEL_DES | OP_ALG_AAI_CBC, 3781 .class2_alg_type = OP_ALG_ALGSEL_MD5 | 3782 OP_ALG_AAI_HMAC_PRECOMP, 3783 .alg_op = OP_ALG_ALGSEL_MD5 | OP_ALG_AAI_HMAC, 3784 .geniv = true, 3785 }, 3786 }, 3787 { 3788 .aead = { 3789 .base = { 3790 .cra_name = "authenc(hmac(sha1),cbc(des))", 3791 .cra_driver_name = "authenc-hmac-sha1-" 3792 "cbc-des-caam", 3793 .cra_blocksize = DES_BLOCK_SIZE, 3794 }, 3795 .setkey = aead_setkey, 3796 .setauthsize = aead_setauthsize, 3797 .encrypt = aead_encrypt, 3798 .decrypt = aead_decrypt, 3799 .ivsize = DES_BLOCK_SIZE, 3800 .maxauthsize = SHA1_DIGEST_SIZE, 3801 }, 3802 .caam = { 3803 .class1_alg_type = OP_ALG_ALGSEL_DES | OP_ALG_AAI_CBC, 3804 .class2_alg_type = OP_ALG_ALGSEL_SHA1 | 3805 OP_ALG_AAI_HMAC_PRECOMP, 3806 .alg_op = OP_ALG_ALGSEL_SHA1 | OP_ALG_AAI_HMAC, 3807 }, 3808 }, 3809 { 3810 .aead = { 3811 .base = { 3812 .cra_name = "echainiv(authenc(hmac(sha1)," 3813 "cbc(des)))", 3814 .cra_driver_name = "echainiv-authenc-" 3815 "hmac-sha1-cbc-des-caam", 3816 .cra_blocksize = DES_BLOCK_SIZE, 3817 }, 3818 .setkey = aead_setkey, 3819 .setauthsize = aead_setauthsize, 3820 .encrypt = aead_encrypt, 3821 .decrypt = aead_givdecrypt, 3822 .ivsize = DES_BLOCK_SIZE, 3823 .maxauthsize = SHA1_DIGEST_SIZE, 3824 }, 3825 .caam = { 3826 .class1_alg_type = OP_ALG_ALGSEL_DES | OP_ALG_AAI_CBC, 3827 .class2_alg_type = OP_ALG_ALGSEL_SHA1 | 3828 OP_ALG_AAI_HMAC_PRECOMP, 3829 .alg_op = OP_ALG_ALGSEL_SHA1 | OP_ALG_AAI_HMAC, 3830 .geniv = true, 3831 }, 3832 }, 3833 { 3834 .aead = { 3835 .base = { 3836 .cra_name = "authenc(hmac(sha224),cbc(des))", 3837 .cra_driver_name = "authenc-hmac-sha224-" 3838 "cbc-des-caam", 3839 .cra_blocksize = DES_BLOCK_SIZE, 3840 }, 3841 .setkey = aead_setkey, 3842 .setauthsize = aead_setauthsize, 3843 .encrypt = aead_encrypt, 3844 .decrypt = aead_decrypt, 3845 .ivsize = DES_BLOCK_SIZE, 3846 .maxauthsize = SHA224_DIGEST_SIZE, 3847 }, 3848 .caam = { 3849 .class1_alg_type = OP_ALG_ALGSEL_DES | OP_ALG_AAI_CBC, 3850 .class2_alg_type = OP_ALG_ALGSEL_SHA224 | 3851 OP_ALG_AAI_HMAC_PRECOMP, 3852 .alg_op = OP_ALG_ALGSEL_SHA224 | OP_ALG_AAI_HMAC, 3853 }, 3854 }, 3855 { 3856 .aead = { 3857 .base = { 3858 .cra_name = "echainiv(authenc(hmac(sha224)," 3859 "cbc(des)))", 3860 .cra_driver_name = "echainiv-authenc-" 3861 "hmac-sha224-cbc-des-caam", 3862 .cra_blocksize = DES_BLOCK_SIZE, 3863 }, 3864 .setkey = aead_setkey, 3865 .setauthsize = aead_setauthsize, 3866 .encrypt = aead_encrypt, 3867 .decrypt = aead_givdecrypt, 3868 .ivsize = DES_BLOCK_SIZE, 3869 .maxauthsize = SHA224_DIGEST_SIZE, 3870 }, 3871 .caam = { 3872 .class1_alg_type = OP_ALG_ALGSEL_DES | OP_ALG_AAI_CBC, 3873 .class2_alg_type = OP_ALG_ALGSEL_SHA224 | 3874 OP_ALG_AAI_HMAC_PRECOMP, 3875 .alg_op = OP_ALG_ALGSEL_SHA224 | OP_ALG_AAI_HMAC, 3876 .geniv = true, 3877 }, 3878 }, 3879 { 3880 .aead = { 3881 .base = { 3882 .cra_name = "authenc(hmac(sha256),cbc(des))", 3883 .cra_driver_name = "authenc-hmac-sha256-" 3884 "cbc-des-caam", 3885 .cra_blocksize = DES_BLOCK_SIZE, 3886 }, 3887 .setkey = aead_setkey, 3888 .setauthsize = aead_setauthsize, 3889 .encrypt = aead_encrypt, 3890 .decrypt = aead_decrypt, 3891 .ivsize = DES_BLOCK_SIZE, 3892 .maxauthsize = SHA256_DIGEST_SIZE, 3893 }, 3894 .caam = { 3895 .class1_alg_type = OP_ALG_ALGSEL_DES | OP_ALG_AAI_CBC, 3896 .class2_alg_type = OP_ALG_ALGSEL_SHA256 | 3897 OP_ALG_AAI_HMAC_PRECOMP, 3898 .alg_op = OP_ALG_ALGSEL_SHA256 | OP_ALG_AAI_HMAC, 3899 }, 3900 }, 3901 { 3902 .aead = { 3903 .base = { 3904 .cra_name = "echainiv(authenc(hmac(sha256)," 3905 "cbc(des)))", 3906 .cra_driver_name = "echainiv-authenc-" 3907 "hmac-sha256-cbc-des-caam", 3908 .cra_blocksize = DES_BLOCK_SIZE, 3909 }, 3910 .setkey = aead_setkey, 3911 .setauthsize = aead_setauthsize, 3912 .encrypt = aead_encrypt, 3913 .decrypt = aead_givdecrypt, 3914 .ivsize = DES_BLOCK_SIZE, 3915 .maxauthsize = SHA256_DIGEST_SIZE, 3916 }, 3917 .caam = { 3918 .class1_alg_type = OP_ALG_ALGSEL_DES | OP_ALG_AAI_CBC, 3919 .class2_alg_type = OP_ALG_ALGSEL_SHA256 | 3920 OP_ALG_AAI_HMAC_PRECOMP, 3921 .alg_op = OP_ALG_ALGSEL_SHA256 | OP_ALG_AAI_HMAC, 3922 .geniv = true, 3923 }, 3924 }, 3925 { 3926 .aead = { 3927 .base = { 3928 .cra_name = "authenc(hmac(sha384),cbc(des))", 3929 .cra_driver_name = "authenc-hmac-sha384-" 3930 "cbc-des-caam", 3931 .cra_blocksize = DES_BLOCK_SIZE, 3932 }, 3933 .setkey = aead_setkey, 3934 .setauthsize = aead_setauthsize, 3935 .encrypt = aead_encrypt, 3936 .decrypt = aead_decrypt, 3937 .ivsize = DES_BLOCK_SIZE, 3938 .maxauthsize = SHA384_DIGEST_SIZE, 3939 }, 3940 .caam = { 3941 .class1_alg_type = OP_ALG_ALGSEL_DES | OP_ALG_AAI_CBC, 3942 .class2_alg_type = OP_ALG_ALGSEL_SHA384 | 3943 OP_ALG_AAI_HMAC_PRECOMP, 3944 .alg_op = OP_ALG_ALGSEL_SHA384 | OP_ALG_AAI_HMAC, 3945 }, 3946 }, 3947 { 3948 .aead = { 3949 .base = { 3950 .cra_name = "echainiv(authenc(hmac(sha384)," 3951 "cbc(des)))", 3952 .cra_driver_name = "echainiv-authenc-" 3953 "hmac-sha384-cbc-des-caam", 3954 .cra_blocksize = DES_BLOCK_SIZE, 3955 }, 3956 .setkey = aead_setkey, 3957 .setauthsize = aead_setauthsize, 3958 .encrypt = aead_encrypt, 3959 .decrypt = aead_givdecrypt, 3960 .ivsize = DES_BLOCK_SIZE, 3961 .maxauthsize = SHA384_DIGEST_SIZE, 3962 }, 3963 .caam = { 3964 .class1_alg_type = OP_ALG_ALGSEL_DES | OP_ALG_AAI_CBC, 3965 .class2_alg_type = OP_ALG_ALGSEL_SHA384 | 3966 OP_ALG_AAI_HMAC_PRECOMP, 3967 .alg_op = OP_ALG_ALGSEL_SHA384 | OP_ALG_AAI_HMAC, 3968 .geniv = true, 3969 }, 3970 }, 3971 { 3972 .aead = { 3973 .base = { 3974 .cra_name = "authenc(hmac(sha512),cbc(des))", 3975 .cra_driver_name = "authenc-hmac-sha512-" 3976 "cbc-des-caam", 3977 .cra_blocksize = DES_BLOCK_SIZE, 3978 }, 3979 .setkey = aead_setkey, 3980 .setauthsize = aead_setauthsize, 3981 .encrypt = aead_encrypt, 3982 .decrypt = aead_decrypt, 3983 .ivsize = DES_BLOCK_SIZE, 3984 .maxauthsize = SHA512_DIGEST_SIZE, 3985 }, 3986 .caam = { 3987 .class1_alg_type = OP_ALG_ALGSEL_DES | OP_ALG_AAI_CBC, 3988 .class2_alg_type = OP_ALG_ALGSEL_SHA512 | 3989 OP_ALG_AAI_HMAC_PRECOMP, 3990 .alg_op = OP_ALG_ALGSEL_SHA512 | OP_ALG_AAI_HMAC, 3991 }, 3992 }, 3993 { 3994 .aead = { 3995 .base = { 3996 .cra_name = "echainiv(authenc(hmac(sha512)," 3997 "cbc(des)))", 3998 .cra_driver_name = "echainiv-authenc-" 3999 "hmac-sha512-cbc-des-caam", 4000 .cra_blocksize = DES_BLOCK_SIZE, 4001 }, 4002 .setkey = aead_setkey, 4003 .setauthsize = aead_setauthsize, 4004 .encrypt = aead_encrypt, 4005 .decrypt = aead_givdecrypt, 4006 .ivsize = DES_BLOCK_SIZE, 4007 .maxauthsize = SHA512_DIGEST_SIZE, 4008 }, 4009 .caam = { 4010 .class1_alg_type = OP_ALG_ALGSEL_DES | OP_ALG_AAI_CBC, 4011 .class2_alg_type = OP_ALG_ALGSEL_SHA512 | 4012 OP_ALG_AAI_HMAC_PRECOMP, 4013 .alg_op = OP_ALG_ALGSEL_SHA512 | OP_ALG_AAI_HMAC, 4014 .geniv = true, 4015 }, 4016 }, 4017 { 4018 .aead = { 4019 .base = { 4020 .cra_name = "authenc(hmac(md5)," 4021 "rfc3686(ctr(aes)))", 4022 .cra_driver_name = "authenc-hmac-md5-" 4023 "rfc3686-ctr-aes-caam", 4024 .cra_blocksize = 1, 4025 }, 4026 .setkey = aead_setkey, 4027 .setauthsize = aead_setauthsize, 4028 .encrypt = aead_encrypt, 4029 .decrypt = aead_decrypt, 4030 .ivsize = CTR_RFC3686_IV_SIZE, 4031 .maxauthsize = MD5_DIGEST_SIZE, 4032 }, 4033 .caam = { 4034 .class1_alg_type = OP_ALG_ALGSEL_AES | 4035 OP_ALG_AAI_CTR_MOD128, 4036 .class2_alg_type = OP_ALG_ALGSEL_MD5 | 4037 OP_ALG_AAI_HMAC_PRECOMP, 4038 .alg_op = OP_ALG_ALGSEL_MD5 | OP_ALG_AAI_HMAC, 4039 .rfc3686 = true, 4040 }, 4041 }, 4042 { 4043 .aead = { 4044 .base = { 4045 .cra_name = "seqiv(authenc(" 4046 "hmac(md5),rfc3686(ctr(aes))))", 4047 .cra_driver_name = "seqiv-authenc-hmac-md5-" 4048 "rfc3686-ctr-aes-caam", 4049 .cra_blocksize = 1, 4050 }, 4051 .setkey = aead_setkey, 4052 .setauthsize = aead_setauthsize, 4053 .encrypt = aead_encrypt, 4054 .decrypt = aead_givdecrypt, 4055 .ivsize = CTR_RFC3686_IV_SIZE, 4056 .maxauthsize = MD5_DIGEST_SIZE, 4057 }, 4058 .caam = { 4059 .class1_alg_type = OP_ALG_ALGSEL_AES | 4060 OP_ALG_AAI_CTR_MOD128, 4061 .class2_alg_type = OP_ALG_ALGSEL_MD5 | 4062 OP_ALG_AAI_HMAC_PRECOMP, 4063 .alg_op = OP_ALG_ALGSEL_MD5 | OP_ALG_AAI_HMAC, 4064 .rfc3686 = true, 4065 .geniv = true, 4066 }, 4067 }, 4068 { 4069 .aead = { 4070 .base = { 4071 .cra_name = "authenc(hmac(sha1)," 4072 "rfc3686(ctr(aes)))", 4073 .cra_driver_name = "authenc-hmac-sha1-" 4074 "rfc3686-ctr-aes-caam", 4075 .cra_blocksize = 1, 4076 }, 4077 .setkey = aead_setkey, 4078 .setauthsize = aead_setauthsize, 4079 .encrypt = aead_encrypt, 4080 .decrypt = aead_decrypt, 4081 .ivsize = CTR_RFC3686_IV_SIZE, 4082 .maxauthsize = SHA1_DIGEST_SIZE, 4083 }, 4084 .caam = { 4085 .class1_alg_type = OP_ALG_ALGSEL_AES | 4086 OP_ALG_AAI_CTR_MOD128, 4087 .class2_alg_type = OP_ALG_ALGSEL_SHA1 | 4088 OP_ALG_AAI_HMAC_PRECOMP, 4089 .alg_op = OP_ALG_ALGSEL_SHA1 | OP_ALG_AAI_HMAC, 4090 .rfc3686 = true, 4091 }, 4092 }, 4093 { 4094 .aead = { 4095 .base = { 4096 .cra_name = "seqiv(authenc(" 4097 "hmac(sha1),rfc3686(ctr(aes))))", 4098 .cra_driver_name = "seqiv-authenc-hmac-sha1-" 4099 "rfc3686-ctr-aes-caam", 4100 .cra_blocksize = 1, 4101 }, 4102 .setkey = aead_setkey, 4103 .setauthsize = aead_setauthsize, 4104 .encrypt = aead_encrypt, 4105 .decrypt = aead_givdecrypt, 4106 .ivsize = CTR_RFC3686_IV_SIZE, 4107 .maxauthsize = SHA1_DIGEST_SIZE, 4108 }, 4109 .caam = { 4110 .class1_alg_type = OP_ALG_ALGSEL_AES | 4111 OP_ALG_AAI_CTR_MOD128, 4112 .class2_alg_type = OP_ALG_ALGSEL_SHA1 | 4113 OP_ALG_AAI_HMAC_PRECOMP, 4114 .alg_op = OP_ALG_ALGSEL_SHA1 | OP_ALG_AAI_HMAC, 4115 .rfc3686 = true, 4116 .geniv = true, 4117 }, 4118 }, 4119 { 4120 .aead = { 4121 .base = { 4122 .cra_name = "authenc(hmac(sha224)," 4123 "rfc3686(ctr(aes)))", 4124 .cra_driver_name = "authenc-hmac-sha224-" 4125 "rfc3686-ctr-aes-caam", 4126 .cra_blocksize = 1, 4127 }, 4128 .setkey = aead_setkey, 4129 .setauthsize = aead_setauthsize, 4130 .encrypt = aead_encrypt, 4131 .decrypt = aead_decrypt, 4132 .ivsize = CTR_RFC3686_IV_SIZE, 4133 .maxauthsize = SHA224_DIGEST_SIZE, 4134 }, 4135 .caam = { 4136 .class1_alg_type = OP_ALG_ALGSEL_AES | 4137 OP_ALG_AAI_CTR_MOD128, 4138 .class2_alg_type = OP_ALG_ALGSEL_SHA224 | 4139 OP_ALG_AAI_HMAC_PRECOMP, 4140 .alg_op = OP_ALG_ALGSEL_SHA224 | OP_ALG_AAI_HMAC, 4141 .rfc3686 = true, 4142 }, 4143 }, 4144 { 4145 .aead = { 4146 .base = { 4147 .cra_name = "seqiv(authenc(" 4148 "hmac(sha224),rfc3686(ctr(aes))))", 4149 .cra_driver_name = "seqiv-authenc-hmac-sha224-" 4150 "rfc3686-ctr-aes-caam", 4151 .cra_blocksize = 1, 4152 }, 4153 .setkey = aead_setkey, 4154 .setauthsize = aead_setauthsize, 4155 .encrypt = aead_encrypt, 4156 .decrypt = aead_givdecrypt, 4157 .ivsize = CTR_RFC3686_IV_SIZE, 4158 .maxauthsize = SHA224_DIGEST_SIZE, 4159 }, 4160 .caam = { 4161 .class1_alg_type = OP_ALG_ALGSEL_AES | 4162 OP_ALG_AAI_CTR_MOD128, 4163 .class2_alg_type = OP_ALG_ALGSEL_SHA224 | 4164 OP_ALG_AAI_HMAC_PRECOMP, 4165 .alg_op = OP_ALG_ALGSEL_SHA224 | OP_ALG_AAI_HMAC, 4166 .rfc3686 = true, 4167 .geniv = true, 4168 }, 4169 }, 4170 { 4171 .aead = { 4172 .base = { 4173 .cra_name = "authenc(hmac(sha256)," 4174 "rfc3686(ctr(aes)))", 4175 .cra_driver_name = "authenc-hmac-sha256-" 4176 "rfc3686-ctr-aes-caam", 4177 .cra_blocksize = 1, 4178 }, 4179 .setkey = aead_setkey, 4180 .setauthsize = aead_setauthsize, 4181 .encrypt = aead_encrypt, 4182 .decrypt = aead_decrypt, 4183 .ivsize = CTR_RFC3686_IV_SIZE, 4184 .maxauthsize = SHA256_DIGEST_SIZE, 4185 }, 4186 .caam = { 4187 .class1_alg_type = OP_ALG_ALGSEL_AES | 4188 OP_ALG_AAI_CTR_MOD128, 4189 .class2_alg_type = OP_ALG_ALGSEL_SHA256 | 4190 OP_ALG_AAI_HMAC_PRECOMP, 4191 .alg_op = OP_ALG_ALGSEL_SHA256 | OP_ALG_AAI_HMAC, 4192 .rfc3686 = true, 4193 }, 4194 }, 4195 { 4196 .aead = { 4197 .base = { 4198 .cra_name = "seqiv(authenc(hmac(sha256)," 4199 "rfc3686(ctr(aes))))", 4200 .cra_driver_name = "seqiv-authenc-hmac-sha256-" 4201 "rfc3686-ctr-aes-caam", 4202 .cra_blocksize = 1, 4203 }, 4204 .setkey = aead_setkey, 4205 .setauthsize = aead_setauthsize, 4206 .encrypt = aead_encrypt, 4207 .decrypt = aead_givdecrypt, 4208 .ivsize = CTR_RFC3686_IV_SIZE, 4209 .maxauthsize = SHA256_DIGEST_SIZE, 4210 }, 4211 .caam = { 4212 .class1_alg_type = OP_ALG_ALGSEL_AES | 4213 OP_ALG_AAI_CTR_MOD128, 4214 .class2_alg_type = OP_ALG_ALGSEL_SHA256 | 4215 OP_ALG_AAI_HMAC_PRECOMP, 4216 .alg_op = OP_ALG_ALGSEL_SHA256 | OP_ALG_AAI_HMAC, 4217 .rfc3686 = true, 4218 .geniv = true, 4219 }, 4220 }, 4221 { 4222 .aead = { 4223 .base = { 4224 .cra_name = "authenc(hmac(sha384)," 4225 "rfc3686(ctr(aes)))", 4226 .cra_driver_name = "authenc-hmac-sha384-" 4227 "rfc3686-ctr-aes-caam", 4228 .cra_blocksize = 1, 4229 }, 4230 .setkey = aead_setkey, 4231 .setauthsize = aead_setauthsize, 4232 .encrypt = aead_encrypt, 4233 .decrypt = aead_decrypt, 4234 .ivsize = CTR_RFC3686_IV_SIZE, 4235 .maxauthsize = SHA384_DIGEST_SIZE, 4236 }, 4237 .caam = { 4238 .class1_alg_type = OP_ALG_ALGSEL_AES | 4239 OP_ALG_AAI_CTR_MOD128, 4240 .class2_alg_type = OP_ALG_ALGSEL_SHA384 | 4241 OP_ALG_AAI_HMAC_PRECOMP, 4242 .alg_op = OP_ALG_ALGSEL_SHA384 | OP_ALG_AAI_HMAC, 4243 .rfc3686 = true, 4244 }, 4245 }, 4246 { 4247 .aead = { 4248 .base = { 4249 .cra_name = "seqiv(authenc(hmac(sha384)," 4250 "rfc3686(ctr(aes))))", 4251 .cra_driver_name = "seqiv-authenc-hmac-sha384-" 4252 "rfc3686-ctr-aes-caam", 4253 .cra_blocksize = 1, 4254 }, 4255 .setkey = aead_setkey, 4256 .setauthsize = aead_setauthsize, 4257 .encrypt = aead_encrypt, 4258 .decrypt = aead_givdecrypt, 4259 .ivsize = CTR_RFC3686_IV_SIZE, 4260 .maxauthsize = SHA384_DIGEST_SIZE, 4261 }, 4262 .caam = { 4263 .class1_alg_type = OP_ALG_ALGSEL_AES | 4264 OP_ALG_AAI_CTR_MOD128, 4265 .class2_alg_type = OP_ALG_ALGSEL_SHA384 | 4266 OP_ALG_AAI_HMAC_PRECOMP, 4267 .alg_op = OP_ALG_ALGSEL_SHA384 | OP_ALG_AAI_HMAC, 4268 .rfc3686 = true, 4269 .geniv = true, 4270 }, 4271 }, 4272 { 4273 .aead = { 4274 .base = { 4275 .cra_name = "authenc(hmac(sha512)," 4276 "rfc3686(ctr(aes)))", 4277 .cra_driver_name = "authenc-hmac-sha512-" 4278 "rfc3686-ctr-aes-caam", 4279 .cra_blocksize = 1, 4280 }, 4281 .setkey = aead_setkey, 4282 .setauthsize = aead_setauthsize, 4283 .encrypt = aead_encrypt, 4284 .decrypt = aead_decrypt, 4285 .ivsize = CTR_RFC3686_IV_SIZE, 4286 .maxauthsize = SHA512_DIGEST_SIZE, 4287 }, 4288 .caam = { 4289 .class1_alg_type = OP_ALG_ALGSEL_AES | 4290 OP_ALG_AAI_CTR_MOD128, 4291 .class2_alg_type = OP_ALG_ALGSEL_SHA512 | 4292 OP_ALG_AAI_HMAC_PRECOMP, 4293 .alg_op = OP_ALG_ALGSEL_SHA512 | OP_ALG_AAI_HMAC, 4294 .rfc3686 = true, 4295 }, 4296 }, 4297 { 4298 .aead = { 4299 .base = { 4300 .cra_name = "seqiv(authenc(hmac(sha512)," 4301 "rfc3686(ctr(aes))))", 4302 .cra_driver_name = "seqiv-authenc-hmac-sha512-" 4303 "rfc3686-ctr-aes-caam", 4304 .cra_blocksize = 1, 4305 }, 4306 .setkey = aead_setkey, 4307 .setauthsize = aead_setauthsize, 4308 .encrypt = aead_encrypt, 4309 .decrypt = aead_givdecrypt, 4310 .ivsize = CTR_RFC3686_IV_SIZE, 4311 .maxauthsize = SHA512_DIGEST_SIZE, 4312 }, 4313 .caam = { 4314 .class1_alg_type = OP_ALG_ALGSEL_AES | 4315 OP_ALG_AAI_CTR_MOD128, 4316 .class2_alg_type = OP_ALG_ALGSEL_SHA512 | 4317 OP_ALG_AAI_HMAC_PRECOMP, 4318 .alg_op = OP_ALG_ALGSEL_SHA512 | OP_ALG_AAI_HMAC, 4319 .rfc3686 = true, 4320 .geniv = true, 4321 }, 4322 }, 4323 }; 4324 4325 struct caam_crypto_alg { 4326 struct crypto_alg crypto_alg; 4327 struct list_head entry; 4328 struct caam_alg_entry caam; 4329 }; 4330 4331 static int caam_init_common(struct caam_ctx *ctx, struct caam_alg_entry *caam) 4332 { 4333 ctx->jrdev = caam_jr_alloc(); 4334 if (IS_ERR(ctx->jrdev)) { 4335 pr_err("Job Ring Device allocation for transform failed\n"); 4336 return PTR_ERR(ctx->jrdev); 4337 } 4338 4339 /* copy descriptor header template value */ 4340 ctx->class1_alg_type = OP_TYPE_CLASS1_ALG | caam->class1_alg_type; 4341 ctx->class2_alg_type = OP_TYPE_CLASS2_ALG | caam->class2_alg_type; 4342 ctx->alg_op = OP_TYPE_CLASS2_ALG | caam->alg_op; 4343 4344 return 0; 4345 } 4346 4347 static int caam_cra_init(struct crypto_tfm *tfm) 4348 { 4349 struct crypto_alg *alg = tfm->__crt_alg; 4350 struct caam_crypto_alg *caam_alg = 4351 container_of(alg, struct caam_crypto_alg, crypto_alg); 4352 struct caam_ctx *ctx = crypto_tfm_ctx(tfm); 4353 4354 return caam_init_common(ctx, &caam_alg->caam); 4355 } 4356 4357 static int caam_aead_init(struct crypto_aead *tfm) 4358 { 4359 struct aead_alg *alg = crypto_aead_alg(tfm); 4360 struct caam_aead_alg *caam_alg = 4361 container_of(alg, struct caam_aead_alg, aead); 4362 struct caam_ctx *ctx = crypto_aead_ctx(tfm); 4363 4364 return caam_init_common(ctx, &caam_alg->caam); 4365 } 4366 4367 static void caam_exit_common(struct caam_ctx *ctx) 4368 { 4369 if (ctx->sh_desc_enc_dma && 4370 !dma_mapping_error(ctx->jrdev, ctx->sh_desc_enc_dma)) 4371 dma_unmap_single(ctx->jrdev, ctx->sh_desc_enc_dma, 4372 desc_bytes(ctx->sh_desc_enc), DMA_TO_DEVICE); 4373 if (ctx->sh_desc_dec_dma && 4374 !dma_mapping_error(ctx->jrdev, ctx->sh_desc_dec_dma)) 4375 dma_unmap_single(ctx->jrdev, ctx->sh_desc_dec_dma, 4376 desc_bytes(ctx->sh_desc_dec), DMA_TO_DEVICE); 4377 if (ctx->sh_desc_givenc_dma && 4378 !dma_mapping_error(ctx->jrdev, ctx->sh_desc_givenc_dma)) 4379 dma_unmap_single(ctx->jrdev, ctx->sh_desc_givenc_dma, 4380 desc_bytes(ctx->sh_desc_givenc), 4381 DMA_TO_DEVICE); 4382 if (ctx->key_dma && 4383 !dma_mapping_error(ctx->jrdev, ctx->key_dma)) 4384 dma_unmap_single(ctx->jrdev, ctx->key_dma, 4385 ctx->enckeylen + ctx->split_key_pad_len, 4386 DMA_TO_DEVICE); 4387 4388 caam_jr_free(ctx->jrdev); 4389 } 4390 4391 static void caam_cra_exit(struct crypto_tfm *tfm) 4392 { 4393 caam_exit_common(crypto_tfm_ctx(tfm)); 4394 } 4395 4396 static void caam_aead_exit(struct crypto_aead *tfm) 4397 { 4398 caam_exit_common(crypto_aead_ctx(tfm)); 4399 } 4400 4401 static void __exit caam_algapi_exit(void) 4402 { 4403 4404 struct caam_crypto_alg *t_alg, *n; 4405 int i; 4406 4407 for (i = 0; i < ARRAY_SIZE(driver_aeads); i++) { 4408 struct caam_aead_alg *t_alg = driver_aeads + i; 4409 4410 if (t_alg->registered) 4411 crypto_unregister_aead(&t_alg->aead); 4412 } 4413 4414 if (!alg_list.next) 4415 return; 4416 4417 list_for_each_entry_safe(t_alg, n, &alg_list, entry) { 4418 crypto_unregister_alg(&t_alg->crypto_alg); 4419 list_del(&t_alg->entry); 4420 kfree(t_alg); 4421 } 4422 } 4423 4424 static struct caam_crypto_alg *caam_alg_alloc(struct caam_alg_template 4425 *template) 4426 { 4427 struct caam_crypto_alg *t_alg; 4428 struct crypto_alg *alg; 4429 4430 t_alg = kzalloc(sizeof(*t_alg), GFP_KERNEL); 4431 if (!t_alg) { 4432 pr_err("failed to allocate t_alg\n"); 4433 return ERR_PTR(-ENOMEM); 4434 } 4435 4436 alg = &t_alg->crypto_alg; 4437 4438 snprintf(alg->cra_name, CRYPTO_MAX_ALG_NAME, "%s", template->name); 4439 snprintf(alg->cra_driver_name, CRYPTO_MAX_ALG_NAME, "%s", 4440 template->driver_name); 4441 alg->cra_module = THIS_MODULE; 4442 alg->cra_init = caam_cra_init; 4443 alg->cra_exit = caam_cra_exit; 4444 alg->cra_priority = CAAM_CRA_PRIORITY; 4445 alg->cra_blocksize = template->blocksize; 4446 alg->cra_alignmask = 0; 4447 alg->cra_ctxsize = sizeof(struct caam_ctx); 4448 alg->cra_flags = CRYPTO_ALG_ASYNC | CRYPTO_ALG_KERN_DRIVER_ONLY | 4449 template->type; 4450 switch (template->type) { 4451 case CRYPTO_ALG_TYPE_GIVCIPHER: 4452 alg->cra_type = &crypto_givcipher_type; 4453 alg->cra_ablkcipher = template->template_ablkcipher; 4454 break; 4455 case CRYPTO_ALG_TYPE_ABLKCIPHER: 4456 alg->cra_type = &crypto_ablkcipher_type; 4457 alg->cra_ablkcipher = template->template_ablkcipher; 4458 break; 4459 } 4460 4461 t_alg->caam.class1_alg_type = template->class1_alg_type; 4462 t_alg->caam.class2_alg_type = template->class2_alg_type; 4463 t_alg->caam.alg_op = template->alg_op; 4464 4465 return t_alg; 4466 } 4467 4468 static void caam_aead_alg_init(struct caam_aead_alg *t_alg) 4469 { 4470 struct aead_alg *alg = &t_alg->aead; 4471 4472 alg->base.cra_module = THIS_MODULE; 4473 alg->base.cra_priority = CAAM_CRA_PRIORITY; 4474 alg->base.cra_ctxsize = sizeof(struct caam_ctx); 4475 alg->base.cra_flags = CRYPTO_ALG_ASYNC | CRYPTO_ALG_KERN_DRIVER_ONLY; 4476 4477 alg->init = caam_aead_init; 4478 alg->exit = caam_aead_exit; 4479 } 4480 4481 static int __init caam_algapi_init(void) 4482 { 4483 struct device_node *dev_node; 4484 struct platform_device *pdev; 4485 struct device *ctrldev; 4486 struct caam_drv_private *priv; 4487 int i = 0, err = 0; 4488 u32 cha_vid, cha_inst, des_inst, aes_inst, md_inst; 4489 unsigned int md_limit = SHA512_DIGEST_SIZE; 4490 bool registered = false; 4491 4492 dev_node = of_find_compatible_node(NULL, NULL, "fsl,sec-v4.0"); 4493 if (!dev_node) { 4494 dev_node = of_find_compatible_node(NULL, NULL, "fsl,sec4.0"); 4495 if (!dev_node) 4496 return -ENODEV; 4497 } 4498 4499 pdev = of_find_device_by_node(dev_node); 4500 if (!pdev) { 4501 of_node_put(dev_node); 4502 return -ENODEV; 4503 } 4504 4505 ctrldev = &pdev->dev; 4506 priv = dev_get_drvdata(ctrldev); 4507 of_node_put(dev_node); 4508 4509 /* 4510 * If priv is NULL, it's probably because the caam driver wasn't 4511 * properly initialized (e.g. RNG4 init failed). Thus, bail out here. 4512 */ 4513 if (!priv) 4514 return -ENODEV; 4515 4516 4517 INIT_LIST_HEAD(&alg_list); 4518 4519 /* 4520 * Register crypto algorithms the device supports. 4521 * First, detect presence and attributes of DES, AES, and MD blocks. 4522 */ 4523 cha_vid = rd_reg32(&priv->ctrl->perfmon.cha_id_ls); 4524 cha_inst = rd_reg32(&priv->ctrl->perfmon.cha_num_ls); 4525 des_inst = (cha_inst & CHA_ID_LS_DES_MASK) >> CHA_ID_LS_DES_SHIFT; 4526 aes_inst = (cha_inst & CHA_ID_LS_AES_MASK) >> CHA_ID_LS_AES_SHIFT; 4527 md_inst = (cha_inst & CHA_ID_LS_MD_MASK) >> CHA_ID_LS_MD_SHIFT; 4528 4529 /* If MD is present, limit digest size based on LP256 */ 4530 if (md_inst && ((cha_vid & CHA_ID_LS_MD_MASK) == CHA_ID_LS_MD_LP256)) 4531 md_limit = SHA256_DIGEST_SIZE; 4532 4533 for (i = 0; i < ARRAY_SIZE(driver_algs); i++) { 4534 struct caam_crypto_alg *t_alg; 4535 struct caam_alg_template *alg = driver_algs + i; 4536 u32 alg_sel = alg->class1_alg_type & OP_ALG_ALGSEL_MASK; 4537 4538 /* Skip DES algorithms if not supported by device */ 4539 if (!des_inst && 4540 ((alg_sel == OP_ALG_ALGSEL_3DES) || 4541 (alg_sel == OP_ALG_ALGSEL_DES))) 4542 continue; 4543 4544 /* Skip AES algorithms if not supported by device */ 4545 if (!aes_inst && (alg_sel == OP_ALG_ALGSEL_AES)) 4546 continue; 4547 4548 t_alg = caam_alg_alloc(alg); 4549 if (IS_ERR(t_alg)) { 4550 err = PTR_ERR(t_alg); 4551 pr_warn("%s alg allocation failed\n", alg->driver_name); 4552 continue; 4553 } 4554 4555 err = crypto_register_alg(&t_alg->crypto_alg); 4556 if (err) { 4557 pr_warn("%s alg registration failed\n", 4558 t_alg->crypto_alg.cra_driver_name); 4559 kfree(t_alg); 4560 continue; 4561 } 4562 4563 list_add_tail(&t_alg->entry, &alg_list); 4564 registered = true; 4565 } 4566 4567 for (i = 0; i < ARRAY_SIZE(driver_aeads); i++) { 4568 struct caam_aead_alg *t_alg = driver_aeads + i; 4569 u32 c1_alg_sel = t_alg->caam.class1_alg_type & 4570 OP_ALG_ALGSEL_MASK; 4571 u32 c2_alg_sel = t_alg->caam.class2_alg_type & 4572 OP_ALG_ALGSEL_MASK; 4573 u32 alg_aai = t_alg->caam.class1_alg_type & OP_ALG_AAI_MASK; 4574 4575 /* Skip DES algorithms if not supported by device */ 4576 if (!des_inst && 4577 ((c1_alg_sel == OP_ALG_ALGSEL_3DES) || 4578 (c1_alg_sel == OP_ALG_ALGSEL_DES))) 4579 continue; 4580 4581 /* Skip AES algorithms if not supported by device */ 4582 if (!aes_inst && (c1_alg_sel == OP_ALG_ALGSEL_AES)) 4583 continue; 4584 4585 /* 4586 * Check support for AES algorithms not available 4587 * on LP devices. 4588 */ 4589 if ((cha_vid & CHA_ID_LS_AES_MASK) == CHA_ID_LS_AES_LP) 4590 if (alg_aai == OP_ALG_AAI_GCM) 4591 continue; 4592 4593 /* 4594 * Skip algorithms requiring message digests 4595 * if MD or MD size is not supported by device. 4596 */ 4597 if (c2_alg_sel && 4598 (!md_inst || (t_alg->aead.maxauthsize > md_limit))) 4599 continue; 4600 4601 caam_aead_alg_init(t_alg); 4602 4603 err = crypto_register_aead(&t_alg->aead); 4604 if (err) { 4605 pr_warn("%s alg registration failed\n", 4606 t_alg->aead.base.cra_driver_name); 4607 continue; 4608 } 4609 4610 t_alg->registered = true; 4611 registered = true; 4612 } 4613 4614 if (registered) 4615 pr_info("caam algorithms registered in /proc/crypto\n"); 4616 4617 return err; 4618 } 4619 4620 module_init(caam_algapi_init); 4621 module_exit(caam_algapi_exit); 4622 4623 MODULE_LICENSE("GPL"); 4624 MODULE_DESCRIPTION("FSL CAAM support for crypto API"); 4625 MODULE_AUTHOR("Freescale Semiconductor - NMG/STC"); 4626