1 /* 2 * caam - Freescale FSL CAAM support for crypto API 3 * 4 * Copyright 2008-2011 Freescale Semiconductor, Inc. 5 * 6 * Based on talitos crypto API driver. 7 * 8 * relationship of job descriptors to shared descriptors (SteveC Dec 10 2008): 9 * 10 * --------------- --------------- 11 * | JobDesc #1 |-------------------->| ShareDesc | 12 * | *(packet 1) | | (PDB) | 13 * --------------- |------------->| (hashKey) | 14 * . | | (cipherKey) | 15 * . | |-------->| (operation) | 16 * --------------- | | --------------- 17 * | JobDesc #2 |------| | 18 * | *(packet 2) | | 19 * --------------- | 20 * . | 21 * . | 22 * --------------- | 23 * | JobDesc #3 |------------ 24 * | *(packet 3) | 25 * --------------- 26 * 27 * The SharedDesc never changes for a connection unless rekeyed, but 28 * each packet will likely be in a different place. So all we need 29 * to know to process the packet is where the input is, where the 30 * output goes, and what context we want to process with. Context is 31 * in the SharedDesc, packet references in the JobDesc. 32 * 33 * So, a job desc looks like: 34 * 35 * --------------------- 36 * | Header | 37 * | ShareDesc Pointer | 38 * | SEQ_OUT_PTR | 39 * | (output buffer) | 40 * | (output length) | 41 * | SEQ_IN_PTR | 42 * | (input buffer) | 43 * | (input length) | 44 * --------------------- 45 */ 46 47 #include "compat.h" 48 49 #include "regs.h" 50 #include "intern.h" 51 #include "desc_constr.h" 52 #include "jr.h" 53 #include "error.h" 54 #include "sg_sw_sec4.h" 55 #include "key_gen.h" 56 57 /* 58 * crypto alg 59 */ 60 #define CAAM_CRA_PRIORITY 3000 61 /* max key is sum of AES_MAX_KEY_SIZE, max split key size */ 62 #define CAAM_MAX_KEY_SIZE (AES_MAX_KEY_SIZE + \ 63 CTR_RFC3686_NONCE_SIZE + \ 64 SHA512_DIGEST_SIZE * 2) 65 /* max IV is max of AES_BLOCK_SIZE, DES3_EDE_BLOCK_SIZE */ 66 #define CAAM_MAX_IV_LENGTH 16 67 68 #define AEAD_DESC_JOB_IO_LEN (DESC_JOB_IO_LEN + CAAM_CMD_SZ * 2) 69 #define GCM_DESC_JOB_IO_LEN (AEAD_DESC_JOB_IO_LEN + \ 70 CAAM_CMD_SZ * 4) 71 #define AUTHENC_DESC_JOB_IO_LEN (AEAD_DESC_JOB_IO_LEN + \ 72 CAAM_CMD_SZ * 5) 73 74 /* length of descriptors text */ 75 #define DESC_AEAD_BASE (4 * CAAM_CMD_SZ) 76 #define DESC_AEAD_ENC_LEN (DESC_AEAD_BASE + 11 * CAAM_CMD_SZ) 77 #define DESC_AEAD_DEC_LEN (DESC_AEAD_BASE + 15 * CAAM_CMD_SZ) 78 #define DESC_AEAD_GIVENC_LEN (DESC_AEAD_ENC_LEN + 9 * CAAM_CMD_SZ) 79 80 /* Note: Nonce is counted in enckeylen */ 81 #define DESC_AEAD_CTR_RFC3686_LEN (4 * CAAM_CMD_SZ) 82 83 #define DESC_AEAD_NULL_BASE (3 * CAAM_CMD_SZ) 84 #define DESC_AEAD_NULL_ENC_LEN (DESC_AEAD_NULL_BASE + 11 * CAAM_CMD_SZ) 85 #define DESC_AEAD_NULL_DEC_LEN (DESC_AEAD_NULL_BASE + 13 * CAAM_CMD_SZ) 86 87 #define DESC_GCM_BASE (3 * CAAM_CMD_SZ) 88 #define DESC_GCM_ENC_LEN (DESC_GCM_BASE + 16 * CAAM_CMD_SZ) 89 #define DESC_GCM_DEC_LEN (DESC_GCM_BASE + 12 * CAAM_CMD_SZ) 90 91 #define DESC_RFC4106_BASE (3 * CAAM_CMD_SZ) 92 #define DESC_RFC4106_ENC_LEN (DESC_RFC4106_BASE + 13 * CAAM_CMD_SZ) 93 #define DESC_RFC4106_DEC_LEN (DESC_RFC4106_BASE + 13 * CAAM_CMD_SZ) 94 95 #define DESC_RFC4543_BASE (3 * CAAM_CMD_SZ) 96 #define DESC_RFC4543_ENC_LEN (DESC_RFC4543_BASE + 11 * CAAM_CMD_SZ) 97 #define DESC_RFC4543_DEC_LEN (DESC_RFC4543_BASE + 12 * CAAM_CMD_SZ) 98 99 #define DESC_ABLKCIPHER_BASE (3 * CAAM_CMD_SZ) 100 #define DESC_ABLKCIPHER_ENC_LEN (DESC_ABLKCIPHER_BASE + \ 101 20 * CAAM_CMD_SZ) 102 #define DESC_ABLKCIPHER_DEC_LEN (DESC_ABLKCIPHER_BASE + \ 103 15 * CAAM_CMD_SZ) 104 105 #define DESC_MAX_USED_BYTES (CAAM_DESC_BYTES_MAX - DESC_JOB_IO_LEN) 106 #define DESC_MAX_USED_LEN (DESC_MAX_USED_BYTES / CAAM_CMD_SZ) 107 108 #ifdef DEBUG 109 /* for print_hex_dumps with line references */ 110 #define debug(format, arg...) printk(format, arg) 111 #else 112 #define debug(format, arg...) 113 #endif 114 static struct list_head alg_list; 115 116 struct caam_alg_entry { 117 int class1_alg_type; 118 int class2_alg_type; 119 int alg_op; 120 bool rfc3686; 121 bool geniv; 122 }; 123 124 struct caam_aead_alg { 125 struct aead_alg aead; 126 struct caam_alg_entry caam; 127 bool registered; 128 }; 129 130 /* Set DK bit in class 1 operation if shared */ 131 static inline void append_dec_op1(u32 *desc, u32 type) 132 { 133 u32 *jump_cmd, *uncond_jump_cmd; 134 135 /* DK bit is valid only for AES */ 136 if ((type & OP_ALG_ALGSEL_MASK) != OP_ALG_ALGSEL_AES) { 137 append_operation(desc, type | OP_ALG_AS_INITFINAL | 138 OP_ALG_DECRYPT); 139 return; 140 } 141 142 jump_cmd = append_jump(desc, JUMP_TEST_ALL | JUMP_COND_SHRD); 143 append_operation(desc, type | OP_ALG_AS_INITFINAL | 144 OP_ALG_DECRYPT); 145 uncond_jump_cmd = append_jump(desc, JUMP_TEST_ALL); 146 set_jump_tgt_here(desc, jump_cmd); 147 append_operation(desc, type | OP_ALG_AS_INITFINAL | 148 OP_ALG_DECRYPT | OP_ALG_AAI_DK); 149 set_jump_tgt_here(desc, uncond_jump_cmd); 150 } 151 152 /* 153 * For aead functions, read payload and write payload, 154 * both of which are specified in req->src and req->dst 155 */ 156 static inline void aead_append_src_dst(u32 *desc, u32 msg_type) 157 { 158 append_seq_fifo_store(desc, 0, FIFOST_TYPE_MESSAGE_DATA | KEY_VLF); 159 append_seq_fifo_load(desc, 0, FIFOLD_CLASS_BOTH | 160 KEY_VLF | msg_type | FIFOLD_TYPE_LASTBOTH); 161 } 162 163 /* 164 * For ablkcipher encrypt and decrypt, read from req->src and 165 * write to req->dst 166 */ 167 static inline void ablkcipher_append_src_dst(u32 *desc) 168 { 169 append_math_add(desc, VARSEQOUTLEN, SEQINLEN, REG0, CAAM_CMD_SZ); 170 append_math_add(desc, VARSEQINLEN, SEQINLEN, REG0, CAAM_CMD_SZ); 171 append_seq_fifo_load(desc, 0, FIFOLD_CLASS_CLASS1 | 172 KEY_VLF | FIFOLD_TYPE_MSG | FIFOLD_TYPE_LAST1); 173 append_seq_fifo_store(desc, 0, FIFOST_TYPE_MESSAGE_DATA | KEY_VLF); 174 } 175 176 /* 177 * per-session context 178 */ 179 struct caam_ctx { 180 struct device *jrdev; 181 u32 sh_desc_enc[DESC_MAX_USED_LEN]; 182 u32 sh_desc_dec[DESC_MAX_USED_LEN]; 183 u32 sh_desc_givenc[DESC_MAX_USED_LEN]; 184 dma_addr_t sh_desc_enc_dma; 185 dma_addr_t sh_desc_dec_dma; 186 dma_addr_t sh_desc_givenc_dma; 187 u32 class1_alg_type; 188 u32 class2_alg_type; 189 u32 alg_op; 190 u8 key[CAAM_MAX_KEY_SIZE]; 191 dma_addr_t key_dma; 192 unsigned int enckeylen; 193 unsigned int split_key_len; 194 unsigned int split_key_pad_len; 195 unsigned int authsize; 196 }; 197 198 static void append_key_aead(u32 *desc, struct caam_ctx *ctx, 199 int keys_fit_inline, bool is_rfc3686) 200 { 201 u32 *nonce; 202 unsigned int enckeylen = ctx->enckeylen; 203 204 /* 205 * RFC3686 specific: 206 * | ctx->key = {AUTH_KEY, ENC_KEY, NONCE} 207 * | enckeylen = encryption key size + nonce size 208 */ 209 if (is_rfc3686) 210 enckeylen -= CTR_RFC3686_NONCE_SIZE; 211 212 if (keys_fit_inline) { 213 append_key_as_imm(desc, ctx->key, ctx->split_key_pad_len, 214 ctx->split_key_len, CLASS_2 | 215 KEY_DEST_MDHA_SPLIT | KEY_ENC); 216 append_key_as_imm(desc, (void *)ctx->key + 217 ctx->split_key_pad_len, enckeylen, 218 enckeylen, CLASS_1 | KEY_DEST_CLASS_REG); 219 } else { 220 append_key(desc, ctx->key_dma, ctx->split_key_len, CLASS_2 | 221 KEY_DEST_MDHA_SPLIT | KEY_ENC); 222 append_key(desc, ctx->key_dma + ctx->split_key_pad_len, 223 enckeylen, CLASS_1 | KEY_DEST_CLASS_REG); 224 } 225 226 /* Load Counter into CONTEXT1 reg */ 227 if (is_rfc3686) { 228 nonce = (u32 *)((void *)ctx->key + ctx->split_key_pad_len + 229 enckeylen); 230 append_load_imm_u32(desc, *nonce, LDST_CLASS_IND_CCB | 231 LDST_SRCDST_BYTE_OUTFIFO | LDST_IMM); 232 append_move(desc, 233 MOVE_SRC_OUTFIFO | 234 MOVE_DEST_CLASS1CTX | 235 (16 << MOVE_OFFSET_SHIFT) | 236 (CTR_RFC3686_NONCE_SIZE << MOVE_LEN_SHIFT)); 237 } 238 } 239 240 static void init_sh_desc_key_aead(u32 *desc, struct caam_ctx *ctx, 241 int keys_fit_inline, bool is_rfc3686) 242 { 243 u32 *key_jump_cmd; 244 245 /* Note: Context registers are saved. */ 246 init_sh_desc(desc, HDR_SHARE_SERIAL | HDR_SAVECTX); 247 248 /* Skip if already shared */ 249 key_jump_cmd = append_jump(desc, JUMP_JSL | JUMP_TEST_ALL | 250 JUMP_COND_SHRD); 251 252 append_key_aead(desc, ctx, keys_fit_inline, is_rfc3686); 253 254 set_jump_tgt_here(desc, key_jump_cmd); 255 } 256 257 static int aead_null_set_sh_desc(struct crypto_aead *aead) 258 { 259 struct caam_ctx *ctx = crypto_aead_ctx(aead); 260 struct device *jrdev = ctx->jrdev; 261 bool keys_fit_inline = false; 262 u32 *key_jump_cmd, *jump_cmd, *read_move_cmd, *write_move_cmd; 263 u32 *desc; 264 265 /* 266 * Job Descriptor and Shared Descriptors 267 * must all fit into the 64-word Descriptor h/w Buffer 268 */ 269 if (DESC_AEAD_NULL_ENC_LEN + AEAD_DESC_JOB_IO_LEN + 270 ctx->split_key_pad_len <= CAAM_DESC_BYTES_MAX) 271 keys_fit_inline = true; 272 273 /* aead_encrypt shared descriptor */ 274 desc = ctx->sh_desc_enc; 275 276 init_sh_desc(desc, HDR_SHARE_SERIAL); 277 278 /* Skip if already shared */ 279 key_jump_cmd = append_jump(desc, JUMP_JSL | JUMP_TEST_ALL | 280 JUMP_COND_SHRD); 281 if (keys_fit_inline) 282 append_key_as_imm(desc, ctx->key, ctx->split_key_pad_len, 283 ctx->split_key_len, CLASS_2 | 284 KEY_DEST_MDHA_SPLIT | KEY_ENC); 285 else 286 append_key(desc, ctx->key_dma, ctx->split_key_len, CLASS_2 | 287 KEY_DEST_MDHA_SPLIT | KEY_ENC); 288 set_jump_tgt_here(desc, key_jump_cmd); 289 290 /* assoclen + cryptlen = seqinlen */ 291 append_math_sub(desc, REG3, SEQINLEN, REG0, CAAM_CMD_SZ); 292 293 /* Prepare to read and write cryptlen + assoclen bytes */ 294 append_math_add(desc, VARSEQINLEN, ZERO, REG3, CAAM_CMD_SZ); 295 append_math_add(desc, VARSEQOUTLEN, ZERO, REG3, CAAM_CMD_SZ); 296 297 /* 298 * MOVE_LEN opcode is not available in all SEC HW revisions, 299 * thus need to do some magic, i.e. self-patch the descriptor 300 * buffer. 301 */ 302 read_move_cmd = append_move(desc, MOVE_SRC_DESCBUF | 303 MOVE_DEST_MATH3 | 304 (0x6 << MOVE_LEN_SHIFT)); 305 write_move_cmd = append_move(desc, MOVE_SRC_MATH3 | 306 MOVE_DEST_DESCBUF | 307 MOVE_WAITCOMP | 308 (0x8 << MOVE_LEN_SHIFT)); 309 310 /* Class 2 operation */ 311 append_operation(desc, ctx->class2_alg_type | 312 OP_ALG_AS_INITFINAL | OP_ALG_ENCRYPT); 313 314 /* Read and write cryptlen bytes */ 315 aead_append_src_dst(desc, FIFOLD_TYPE_MSG | FIFOLD_TYPE_FLUSH1); 316 317 set_move_tgt_here(desc, read_move_cmd); 318 set_move_tgt_here(desc, write_move_cmd); 319 append_cmd(desc, CMD_LOAD | DISABLE_AUTO_INFO_FIFO); 320 append_move(desc, MOVE_SRC_INFIFO_CL | MOVE_DEST_OUTFIFO | 321 MOVE_AUX_LS); 322 323 /* Write ICV */ 324 append_seq_store(desc, ctx->authsize, LDST_CLASS_2_CCB | 325 LDST_SRCDST_BYTE_CONTEXT); 326 327 ctx->sh_desc_enc_dma = dma_map_single(jrdev, desc, 328 desc_bytes(desc), 329 DMA_TO_DEVICE); 330 if (dma_mapping_error(jrdev, ctx->sh_desc_enc_dma)) { 331 dev_err(jrdev, "unable to map shared descriptor\n"); 332 return -ENOMEM; 333 } 334 #ifdef DEBUG 335 print_hex_dump(KERN_ERR, 336 "aead null enc shdesc@"__stringify(__LINE__)": ", 337 DUMP_PREFIX_ADDRESS, 16, 4, desc, 338 desc_bytes(desc), 1); 339 #endif 340 341 /* 342 * Job Descriptor and Shared Descriptors 343 * must all fit into the 64-word Descriptor h/w Buffer 344 */ 345 keys_fit_inline = false; 346 if (DESC_AEAD_NULL_DEC_LEN + DESC_JOB_IO_LEN + 347 ctx->split_key_pad_len <= CAAM_DESC_BYTES_MAX) 348 keys_fit_inline = true; 349 350 desc = ctx->sh_desc_dec; 351 352 /* aead_decrypt shared descriptor */ 353 init_sh_desc(desc, HDR_SHARE_SERIAL); 354 355 /* Skip if already shared */ 356 key_jump_cmd = append_jump(desc, JUMP_JSL | JUMP_TEST_ALL | 357 JUMP_COND_SHRD); 358 if (keys_fit_inline) 359 append_key_as_imm(desc, ctx->key, ctx->split_key_pad_len, 360 ctx->split_key_len, CLASS_2 | 361 KEY_DEST_MDHA_SPLIT | KEY_ENC); 362 else 363 append_key(desc, ctx->key_dma, ctx->split_key_len, CLASS_2 | 364 KEY_DEST_MDHA_SPLIT | KEY_ENC); 365 set_jump_tgt_here(desc, key_jump_cmd); 366 367 /* Class 2 operation */ 368 append_operation(desc, ctx->class2_alg_type | 369 OP_ALG_AS_INITFINAL | OP_ALG_DECRYPT | OP_ALG_ICV_ON); 370 371 /* assoclen + cryptlen = seqoutlen */ 372 append_math_sub(desc, REG2, SEQOUTLEN, REG0, CAAM_CMD_SZ); 373 374 /* Prepare to read and write cryptlen + assoclen bytes */ 375 append_math_add(desc, VARSEQINLEN, ZERO, REG2, CAAM_CMD_SZ); 376 append_math_add(desc, VARSEQOUTLEN, ZERO, REG2, CAAM_CMD_SZ); 377 378 /* 379 * MOVE_LEN opcode is not available in all SEC HW revisions, 380 * thus need to do some magic, i.e. self-patch the descriptor 381 * buffer. 382 */ 383 read_move_cmd = append_move(desc, MOVE_SRC_DESCBUF | 384 MOVE_DEST_MATH2 | 385 (0x6 << MOVE_LEN_SHIFT)); 386 write_move_cmd = append_move(desc, MOVE_SRC_MATH2 | 387 MOVE_DEST_DESCBUF | 388 MOVE_WAITCOMP | 389 (0x8 << MOVE_LEN_SHIFT)); 390 391 /* Read and write cryptlen bytes */ 392 aead_append_src_dst(desc, FIFOLD_TYPE_MSG | FIFOLD_TYPE_FLUSH1); 393 394 /* 395 * Insert a NOP here, since we need at least 4 instructions between 396 * code patching the descriptor buffer and the location being patched. 397 */ 398 jump_cmd = append_jump(desc, JUMP_TEST_ALL); 399 set_jump_tgt_here(desc, jump_cmd); 400 401 set_move_tgt_here(desc, read_move_cmd); 402 set_move_tgt_here(desc, write_move_cmd); 403 append_cmd(desc, CMD_LOAD | DISABLE_AUTO_INFO_FIFO); 404 append_move(desc, MOVE_SRC_INFIFO_CL | MOVE_DEST_OUTFIFO | 405 MOVE_AUX_LS); 406 append_cmd(desc, CMD_LOAD | ENABLE_AUTO_INFO_FIFO); 407 408 /* Load ICV */ 409 append_seq_fifo_load(desc, ctx->authsize, FIFOLD_CLASS_CLASS2 | 410 FIFOLD_TYPE_LAST2 | FIFOLD_TYPE_ICV); 411 412 ctx->sh_desc_dec_dma = dma_map_single(jrdev, desc, 413 desc_bytes(desc), 414 DMA_TO_DEVICE); 415 if (dma_mapping_error(jrdev, ctx->sh_desc_dec_dma)) { 416 dev_err(jrdev, "unable to map shared descriptor\n"); 417 return -ENOMEM; 418 } 419 #ifdef DEBUG 420 print_hex_dump(KERN_ERR, 421 "aead null dec shdesc@"__stringify(__LINE__)": ", 422 DUMP_PREFIX_ADDRESS, 16, 4, desc, 423 desc_bytes(desc), 1); 424 #endif 425 426 return 0; 427 } 428 429 static int aead_set_sh_desc(struct crypto_aead *aead) 430 { 431 struct caam_aead_alg *alg = container_of(crypto_aead_alg(aead), 432 struct caam_aead_alg, aead); 433 unsigned int ivsize = crypto_aead_ivsize(aead); 434 struct caam_ctx *ctx = crypto_aead_ctx(aead); 435 struct device *jrdev = ctx->jrdev; 436 bool keys_fit_inline; 437 u32 geniv, moveiv; 438 u32 ctx1_iv_off = 0; 439 u32 *desc; 440 const bool ctr_mode = ((ctx->class1_alg_type & OP_ALG_AAI_MASK) == 441 OP_ALG_AAI_CTR_MOD128); 442 const bool is_rfc3686 = alg->caam.rfc3686; 443 444 if (!ctx->authsize) 445 return 0; 446 447 /* NULL encryption / decryption */ 448 if (!ctx->enckeylen) 449 return aead_null_set_sh_desc(aead); 450 451 /* 452 * AES-CTR needs to load IV in CONTEXT1 reg 453 * at an offset of 128bits (16bytes) 454 * CONTEXT1[255:128] = IV 455 */ 456 if (ctr_mode) 457 ctx1_iv_off = 16; 458 459 /* 460 * RFC3686 specific: 461 * CONTEXT1[255:128] = {NONCE, IV, COUNTER} 462 */ 463 if (is_rfc3686) 464 ctx1_iv_off = 16 + CTR_RFC3686_NONCE_SIZE; 465 466 if (alg->caam.geniv) 467 goto skip_enc; 468 469 /* 470 * Job Descriptor and Shared Descriptors 471 * must all fit into the 64-word Descriptor h/w Buffer 472 */ 473 keys_fit_inline = false; 474 if (DESC_AEAD_ENC_LEN + AUTHENC_DESC_JOB_IO_LEN + 475 ctx->split_key_pad_len + ctx->enckeylen + 476 (is_rfc3686 ? DESC_AEAD_CTR_RFC3686_LEN : 0) <= 477 CAAM_DESC_BYTES_MAX) 478 keys_fit_inline = true; 479 480 /* aead_encrypt shared descriptor */ 481 desc = ctx->sh_desc_enc; 482 483 /* Note: Context registers are saved. */ 484 init_sh_desc_key_aead(desc, ctx, keys_fit_inline, is_rfc3686); 485 486 /* Class 2 operation */ 487 append_operation(desc, ctx->class2_alg_type | 488 OP_ALG_AS_INITFINAL | OP_ALG_ENCRYPT); 489 490 /* Read and write assoclen bytes */ 491 append_math_add(desc, VARSEQINLEN, ZERO, REG3, CAAM_CMD_SZ); 492 append_math_add(desc, VARSEQOUTLEN, ZERO, REG3, CAAM_CMD_SZ); 493 494 /* Skip assoc data */ 495 append_seq_fifo_store(desc, 0, FIFOST_TYPE_SKIP | FIFOLDST_VLF); 496 497 /* read assoc before reading payload */ 498 append_seq_fifo_load(desc, 0, FIFOLD_CLASS_CLASS2 | FIFOLD_TYPE_MSG | 499 FIFOLDST_VLF); 500 501 /* Load Counter into CONTEXT1 reg */ 502 if (is_rfc3686) 503 append_load_imm_u32(desc, be32_to_cpu(1), LDST_IMM | 504 LDST_CLASS_1_CCB | 505 LDST_SRCDST_BYTE_CONTEXT | 506 ((ctx1_iv_off + CTR_RFC3686_IV_SIZE) << 507 LDST_OFFSET_SHIFT)); 508 509 /* Class 1 operation */ 510 append_operation(desc, ctx->class1_alg_type | 511 OP_ALG_AS_INITFINAL | OP_ALG_ENCRYPT); 512 513 /* Read and write cryptlen bytes */ 514 append_math_add(desc, VARSEQINLEN, SEQINLEN, REG0, CAAM_CMD_SZ); 515 append_math_add(desc, VARSEQOUTLEN, SEQINLEN, REG0, CAAM_CMD_SZ); 516 aead_append_src_dst(desc, FIFOLD_TYPE_MSG1OUT2); 517 518 /* Write ICV */ 519 append_seq_store(desc, ctx->authsize, LDST_CLASS_2_CCB | 520 LDST_SRCDST_BYTE_CONTEXT); 521 522 ctx->sh_desc_enc_dma = dma_map_single(jrdev, desc, 523 desc_bytes(desc), 524 DMA_TO_DEVICE); 525 if (dma_mapping_error(jrdev, ctx->sh_desc_enc_dma)) { 526 dev_err(jrdev, "unable to map shared descriptor\n"); 527 return -ENOMEM; 528 } 529 #ifdef DEBUG 530 print_hex_dump(KERN_ERR, "aead enc shdesc@"__stringify(__LINE__)": ", 531 DUMP_PREFIX_ADDRESS, 16, 4, desc, 532 desc_bytes(desc), 1); 533 #endif 534 535 skip_enc: 536 /* 537 * Job Descriptor and Shared Descriptors 538 * must all fit into the 64-word Descriptor h/w Buffer 539 */ 540 keys_fit_inline = false; 541 if (DESC_AEAD_DEC_LEN + AUTHENC_DESC_JOB_IO_LEN + 542 ctx->split_key_pad_len + ctx->enckeylen + 543 (is_rfc3686 ? DESC_AEAD_CTR_RFC3686_LEN : 0) <= 544 CAAM_DESC_BYTES_MAX) 545 keys_fit_inline = true; 546 547 /* aead_decrypt shared descriptor */ 548 desc = ctx->sh_desc_dec; 549 550 /* Note: Context registers are saved. */ 551 init_sh_desc_key_aead(desc, ctx, keys_fit_inline, is_rfc3686); 552 553 /* Class 2 operation */ 554 append_operation(desc, ctx->class2_alg_type | 555 OP_ALG_AS_INITFINAL | OP_ALG_DECRYPT | OP_ALG_ICV_ON); 556 557 /* Read and write assoclen bytes */ 558 append_math_add(desc, VARSEQINLEN, ZERO, REG3, CAAM_CMD_SZ); 559 if (alg->caam.geniv) 560 append_math_add_imm_u32(desc, VARSEQOUTLEN, REG3, IMM, ivsize); 561 else 562 append_math_add(desc, VARSEQOUTLEN, ZERO, REG3, CAAM_CMD_SZ); 563 564 /* Skip assoc data */ 565 append_seq_fifo_store(desc, 0, FIFOST_TYPE_SKIP | FIFOLDST_VLF); 566 567 /* read assoc before reading payload */ 568 append_seq_fifo_load(desc, 0, FIFOLD_CLASS_CLASS2 | FIFOLD_TYPE_MSG | 569 KEY_VLF); 570 571 if (alg->caam.geniv) { 572 append_seq_load(desc, ivsize, LDST_CLASS_1_CCB | 573 LDST_SRCDST_BYTE_CONTEXT | 574 (ctx1_iv_off << LDST_OFFSET_SHIFT)); 575 append_move(desc, MOVE_SRC_CLASS1CTX | MOVE_DEST_CLASS2INFIFO | 576 (ctx1_iv_off << MOVE_OFFSET_SHIFT) | ivsize); 577 } 578 579 /* Load Counter into CONTEXT1 reg */ 580 if (is_rfc3686) 581 append_load_imm_u32(desc, be32_to_cpu(1), LDST_IMM | 582 LDST_CLASS_1_CCB | 583 LDST_SRCDST_BYTE_CONTEXT | 584 ((ctx1_iv_off + CTR_RFC3686_IV_SIZE) << 585 LDST_OFFSET_SHIFT)); 586 587 /* Choose operation */ 588 if (ctr_mode) 589 append_operation(desc, ctx->class1_alg_type | 590 OP_ALG_AS_INITFINAL | OP_ALG_DECRYPT); 591 else 592 append_dec_op1(desc, ctx->class1_alg_type); 593 594 /* Read and write cryptlen bytes */ 595 append_math_add(desc, VARSEQINLEN, SEQOUTLEN, REG0, CAAM_CMD_SZ); 596 append_math_add(desc, VARSEQOUTLEN, SEQOUTLEN, REG0, CAAM_CMD_SZ); 597 aead_append_src_dst(desc, FIFOLD_TYPE_MSG); 598 599 /* Load ICV */ 600 append_seq_fifo_load(desc, ctx->authsize, FIFOLD_CLASS_CLASS2 | 601 FIFOLD_TYPE_LAST2 | FIFOLD_TYPE_ICV); 602 603 ctx->sh_desc_dec_dma = dma_map_single(jrdev, desc, 604 desc_bytes(desc), 605 DMA_TO_DEVICE); 606 if (dma_mapping_error(jrdev, ctx->sh_desc_dec_dma)) { 607 dev_err(jrdev, "unable to map shared descriptor\n"); 608 return -ENOMEM; 609 } 610 #ifdef DEBUG 611 print_hex_dump(KERN_ERR, "aead dec shdesc@"__stringify(__LINE__)": ", 612 DUMP_PREFIX_ADDRESS, 16, 4, desc, 613 desc_bytes(desc), 1); 614 #endif 615 616 if (!alg->caam.geniv) 617 goto skip_givenc; 618 619 /* 620 * Job Descriptor and Shared Descriptors 621 * must all fit into the 64-word Descriptor h/w Buffer 622 */ 623 keys_fit_inline = false; 624 if (DESC_AEAD_GIVENC_LEN + AUTHENC_DESC_JOB_IO_LEN + 625 ctx->split_key_pad_len + ctx->enckeylen + 626 (is_rfc3686 ? DESC_AEAD_CTR_RFC3686_LEN : 0) <= 627 CAAM_DESC_BYTES_MAX) 628 keys_fit_inline = true; 629 630 /* aead_givencrypt shared descriptor */ 631 desc = ctx->sh_desc_enc; 632 633 /* Note: Context registers are saved. */ 634 init_sh_desc_key_aead(desc, ctx, keys_fit_inline, is_rfc3686); 635 636 if (is_rfc3686) 637 goto copy_iv; 638 639 /* Generate IV */ 640 geniv = NFIFOENTRY_STYPE_PAD | NFIFOENTRY_DEST_DECO | 641 NFIFOENTRY_DTYPE_MSG | NFIFOENTRY_LC1 | 642 NFIFOENTRY_PTYPE_RND | (ivsize << NFIFOENTRY_DLEN_SHIFT); 643 append_load_imm_u32(desc, geniv, LDST_CLASS_IND_CCB | 644 LDST_SRCDST_WORD_INFO_FIFO | LDST_IMM); 645 append_cmd(desc, CMD_LOAD | DISABLE_AUTO_INFO_FIFO); 646 append_move(desc, MOVE_WAITCOMP | 647 MOVE_SRC_INFIFO | MOVE_DEST_CLASS1CTX | 648 (ctx1_iv_off << MOVE_OFFSET_SHIFT) | 649 (ivsize << MOVE_LEN_SHIFT)); 650 append_cmd(desc, CMD_LOAD | ENABLE_AUTO_INFO_FIFO); 651 652 copy_iv: 653 /* Copy IV to class 1 context */ 654 append_move(desc, MOVE_SRC_CLASS1CTX | MOVE_DEST_OUTFIFO | 655 (ctx1_iv_off << MOVE_OFFSET_SHIFT) | 656 (ivsize << MOVE_LEN_SHIFT)); 657 658 /* Return to encryption */ 659 append_operation(desc, ctx->class2_alg_type | 660 OP_ALG_AS_INITFINAL | OP_ALG_ENCRYPT); 661 662 /* Read and write assoclen bytes */ 663 append_math_add(desc, VARSEQINLEN, ZERO, REG3, CAAM_CMD_SZ); 664 append_math_add(desc, VARSEQOUTLEN, ZERO, REG3, CAAM_CMD_SZ); 665 666 /* ivsize + cryptlen = seqoutlen - authsize */ 667 append_math_sub_imm_u32(desc, REG3, SEQOUTLEN, IMM, ctx->authsize); 668 669 /* Skip assoc data */ 670 append_seq_fifo_store(desc, 0, FIFOST_TYPE_SKIP | FIFOLDST_VLF); 671 672 /* read assoc before reading payload */ 673 append_seq_fifo_load(desc, 0, FIFOLD_CLASS_CLASS2 | FIFOLD_TYPE_MSG | 674 KEY_VLF); 675 676 /* Copy iv from outfifo to class 2 fifo */ 677 moveiv = NFIFOENTRY_STYPE_OFIFO | NFIFOENTRY_DEST_CLASS2 | 678 NFIFOENTRY_DTYPE_MSG | (ivsize << NFIFOENTRY_DLEN_SHIFT); 679 append_load_imm_u32(desc, moveiv, LDST_CLASS_IND_CCB | 680 LDST_SRCDST_WORD_INFO_FIFO | LDST_IMM); 681 append_load_imm_u32(desc, ivsize, LDST_CLASS_2_CCB | 682 LDST_SRCDST_WORD_DATASZ_REG | LDST_IMM); 683 684 /* Load Counter into CONTEXT1 reg */ 685 if (is_rfc3686) 686 append_load_imm_u32(desc, be32_to_cpu(1), LDST_IMM | 687 LDST_CLASS_1_CCB | 688 LDST_SRCDST_BYTE_CONTEXT | 689 ((ctx1_iv_off + CTR_RFC3686_IV_SIZE) << 690 LDST_OFFSET_SHIFT)); 691 692 /* Class 1 operation */ 693 append_operation(desc, ctx->class1_alg_type | 694 OP_ALG_AS_INITFINAL | OP_ALG_ENCRYPT); 695 696 /* Will write ivsize + cryptlen */ 697 append_math_add(desc, VARSEQOUTLEN, SEQINLEN, REG0, CAAM_CMD_SZ); 698 699 /* Not need to reload iv */ 700 append_seq_fifo_load(desc, ivsize, 701 FIFOLD_CLASS_SKIP); 702 703 /* Will read cryptlen */ 704 append_math_add(desc, VARSEQINLEN, SEQINLEN, REG0, CAAM_CMD_SZ); 705 aead_append_src_dst(desc, FIFOLD_TYPE_MSG1OUT2); 706 707 /* Write ICV */ 708 append_seq_store(desc, ctx->authsize, LDST_CLASS_2_CCB | 709 LDST_SRCDST_BYTE_CONTEXT); 710 711 ctx->sh_desc_enc_dma = dma_map_single(jrdev, desc, 712 desc_bytes(desc), 713 DMA_TO_DEVICE); 714 if (dma_mapping_error(jrdev, ctx->sh_desc_enc_dma)) { 715 dev_err(jrdev, "unable to map shared descriptor\n"); 716 return -ENOMEM; 717 } 718 #ifdef DEBUG 719 print_hex_dump(KERN_ERR, "aead givenc shdesc@"__stringify(__LINE__)": ", 720 DUMP_PREFIX_ADDRESS, 16, 4, desc, 721 desc_bytes(desc), 1); 722 #endif 723 724 skip_givenc: 725 return 0; 726 } 727 728 static int aead_setauthsize(struct crypto_aead *authenc, 729 unsigned int authsize) 730 { 731 struct caam_ctx *ctx = crypto_aead_ctx(authenc); 732 733 ctx->authsize = authsize; 734 aead_set_sh_desc(authenc); 735 736 return 0; 737 } 738 739 static int gcm_set_sh_desc(struct crypto_aead *aead) 740 { 741 struct caam_ctx *ctx = crypto_aead_ctx(aead); 742 struct device *jrdev = ctx->jrdev; 743 bool keys_fit_inline = false; 744 u32 *key_jump_cmd, *zero_payload_jump_cmd, 745 *zero_assoc_jump_cmd1, *zero_assoc_jump_cmd2; 746 u32 *desc; 747 748 if (!ctx->enckeylen || !ctx->authsize) 749 return 0; 750 751 /* 752 * AES GCM encrypt shared descriptor 753 * Job Descriptor and Shared Descriptor 754 * must fit into the 64-word Descriptor h/w Buffer 755 */ 756 if (DESC_GCM_ENC_LEN + GCM_DESC_JOB_IO_LEN + 757 ctx->enckeylen <= CAAM_DESC_BYTES_MAX) 758 keys_fit_inline = true; 759 760 desc = ctx->sh_desc_enc; 761 762 init_sh_desc(desc, HDR_SHARE_SERIAL); 763 764 /* skip key loading if they are loaded due to sharing */ 765 key_jump_cmd = append_jump(desc, JUMP_JSL | JUMP_TEST_ALL | 766 JUMP_COND_SHRD | JUMP_COND_SELF); 767 if (keys_fit_inline) 768 append_key_as_imm(desc, (void *)ctx->key, ctx->enckeylen, 769 ctx->enckeylen, CLASS_1 | KEY_DEST_CLASS_REG); 770 else 771 append_key(desc, ctx->key_dma, ctx->enckeylen, 772 CLASS_1 | KEY_DEST_CLASS_REG); 773 set_jump_tgt_here(desc, key_jump_cmd); 774 775 /* class 1 operation */ 776 append_operation(desc, ctx->class1_alg_type | 777 OP_ALG_AS_INITFINAL | OP_ALG_ENCRYPT); 778 779 /* if assoclen + cryptlen is ZERO, skip to ICV write */ 780 append_math_sub(desc, VARSEQOUTLEN, SEQINLEN, REG0, CAAM_CMD_SZ); 781 zero_assoc_jump_cmd2 = append_jump(desc, JUMP_TEST_ALL | 782 JUMP_COND_MATH_Z); 783 784 /* if assoclen is ZERO, skip reading the assoc data */ 785 append_math_add(desc, VARSEQINLEN, ZERO, REG3, CAAM_CMD_SZ); 786 zero_assoc_jump_cmd1 = append_jump(desc, JUMP_TEST_ALL | 787 JUMP_COND_MATH_Z); 788 789 append_math_add(desc, VARSEQOUTLEN, ZERO, REG3, CAAM_CMD_SZ); 790 791 /* skip assoc data */ 792 append_seq_fifo_store(desc, 0, FIFOST_TYPE_SKIP | FIFOLDST_VLF); 793 794 /* cryptlen = seqinlen - assoclen */ 795 append_math_sub(desc, VARSEQOUTLEN, SEQINLEN, REG3, CAAM_CMD_SZ); 796 797 /* if cryptlen is ZERO jump to zero-payload commands */ 798 zero_payload_jump_cmd = append_jump(desc, JUMP_TEST_ALL | 799 JUMP_COND_MATH_Z); 800 801 /* read assoc data */ 802 append_seq_fifo_load(desc, 0, FIFOLD_CLASS_CLASS1 | FIFOLDST_VLF | 803 FIFOLD_TYPE_AAD | FIFOLD_TYPE_FLUSH1); 804 set_jump_tgt_here(desc, zero_assoc_jump_cmd1); 805 806 append_math_sub(desc, VARSEQINLEN, SEQINLEN, REG0, CAAM_CMD_SZ); 807 808 /* write encrypted data */ 809 append_seq_fifo_store(desc, 0, FIFOST_TYPE_MESSAGE_DATA | FIFOLDST_VLF); 810 811 /* read payload data */ 812 append_seq_fifo_load(desc, 0, FIFOLD_CLASS_CLASS1 | FIFOLDST_VLF | 813 FIFOLD_TYPE_MSG | FIFOLD_TYPE_LAST1); 814 815 /* jump the zero-payload commands */ 816 append_jump(desc, JUMP_TEST_ALL | 2); 817 818 /* zero-payload commands */ 819 set_jump_tgt_here(desc, zero_payload_jump_cmd); 820 821 /* read assoc data */ 822 append_seq_fifo_load(desc, 0, FIFOLD_CLASS_CLASS1 | FIFOLDST_VLF | 823 FIFOLD_TYPE_AAD | FIFOLD_TYPE_LAST1); 824 825 /* There is no input data */ 826 set_jump_tgt_here(desc, zero_assoc_jump_cmd2); 827 828 /* write ICV */ 829 append_seq_store(desc, ctx->authsize, LDST_CLASS_1_CCB | 830 LDST_SRCDST_BYTE_CONTEXT); 831 832 ctx->sh_desc_enc_dma = dma_map_single(jrdev, desc, 833 desc_bytes(desc), 834 DMA_TO_DEVICE); 835 if (dma_mapping_error(jrdev, ctx->sh_desc_enc_dma)) { 836 dev_err(jrdev, "unable to map shared descriptor\n"); 837 return -ENOMEM; 838 } 839 #ifdef DEBUG 840 print_hex_dump(KERN_ERR, "gcm enc shdesc@"__stringify(__LINE__)": ", 841 DUMP_PREFIX_ADDRESS, 16, 4, desc, 842 desc_bytes(desc), 1); 843 #endif 844 845 /* 846 * Job Descriptor and Shared Descriptors 847 * must all fit into the 64-word Descriptor h/w Buffer 848 */ 849 keys_fit_inline = false; 850 if (DESC_GCM_DEC_LEN + GCM_DESC_JOB_IO_LEN + 851 ctx->enckeylen <= CAAM_DESC_BYTES_MAX) 852 keys_fit_inline = true; 853 854 desc = ctx->sh_desc_dec; 855 856 init_sh_desc(desc, HDR_SHARE_SERIAL); 857 858 /* skip key loading if they are loaded due to sharing */ 859 key_jump_cmd = append_jump(desc, JUMP_JSL | 860 JUMP_TEST_ALL | JUMP_COND_SHRD | 861 JUMP_COND_SELF); 862 if (keys_fit_inline) 863 append_key_as_imm(desc, (void *)ctx->key, ctx->enckeylen, 864 ctx->enckeylen, CLASS_1 | KEY_DEST_CLASS_REG); 865 else 866 append_key(desc, ctx->key_dma, ctx->enckeylen, 867 CLASS_1 | KEY_DEST_CLASS_REG); 868 set_jump_tgt_here(desc, key_jump_cmd); 869 870 /* class 1 operation */ 871 append_operation(desc, ctx->class1_alg_type | 872 OP_ALG_AS_INITFINAL | OP_ALG_DECRYPT | OP_ALG_ICV_ON); 873 874 /* if assoclen is ZERO, skip reading the assoc data */ 875 append_math_add(desc, VARSEQINLEN, ZERO, REG3, CAAM_CMD_SZ); 876 zero_assoc_jump_cmd1 = append_jump(desc, JUMP_TEST_ALL | 877 JUMP_COND_MATH_Z); 878 879 append_math_add(desc, VARSEQOUTLEN, ZERO, REG3, CAAM_CMD_SZ); 880 881 /* skip assoc data */ 882 append_seq_fifo_store(desc, 0, FIFOST_TYPE_SKIP | FIFOLDST_VLF); 883 884 /* read assoc data */ 885 append_seq_fifo_load(desc, 0, FIFOLD_CLASS_CLASS1 | FIFOLDST_VLF | 886 FIFOLD_TYPE_AAD | FIFOLD_TYPE_FLUSH1); 887 888 set_jump_tgt_here(desc, zero_assoc_jump_cmd1); 889 890 /* cryptlen = seqoutlen - assoclen */ 891 append_math_sub(desc, VARSEQINLEN, SEQOUTLEN, REG0, CAAM_CMD_SZ); 892 893 /* jump to zero-payload command if cryptlen is zero */ 894 zero_payload_jump_cmd = append_jump(desc, JUMP_TEST_ALL | 895 JUMP_COND_MATH_Z); 896 897 append_math_sub(desc, VARSEQOUTLEN, SEQOUTLEN, REG0, CAAM_CMD_SZ); 898 899 /* store encrypted data */ 900 append_seq_fifo_store(desc, 0, FIFOST_TYPE_MESSAGE_DATA | FIFOLDST_VLF); 901 902 /* read payload data */ 903 append_seq_fifo_load(desc, 0, FIFOLD_CLASS_CLASS1 | FIFOLDST_VLF | 904 FIFOLD_TYPE_MSG | FIFOLD_TYPE_FLUSH1); 905 906 /* zero-payload command */ 907 set_jump_tgt_here(desc, zero_payload_jump_cmd); 908 909 /* read ICV */ 910 append_seq_fifo_load(desc, ctx->authsize, FIFOLD_CLASS_CLASS1 | 911 FIFOLD_TYPE_ICV | FIFOLD_TYPE_LAST1); 912 913 ctx->sh_desc_dec_dma = dma_map_single(jrdev, desc, 914 desc_bytes(desc), 915 DMA_TO_DEVICE); 916 if (dma_mapping_error(jrdev, ctx->sh_desc_dec_dma)) { 917 dev_err(jrdev, "unable to map shared descriptor\n"); 918 return -ENOMEM; 919 } 920 #ifdef DEBUG 921 print_hex_dump(KERN_ERR, "gcm dec shdesc@"__stringify(__LINE__)": ", 922 DUMP_PREFIX_ADDRESS, 16, 4, desc, 923 desc_bytes(desc), 1); 924 #endif 925 926 return 0; 927 } 928 929 static int gcm_setauthsize(struct crypto_aead *authenc, unsigned int authsize) 930 { 931 struct caam_ctx *ctx = crypto_aead_ctx(authenc); 932 933 ctx->authsize = authsize; 934 gcm_set_sh_desc(authenc); 935 936 return 0; 937 } 938 939 static int rfc4106_set_sh_desc(struct crypto_aead *aead) 940 { 941 struct caam_ctx *ctx = crypto_aead_ctx(aead); 942 struct device *jrdev = ctx->jrdev; 943 bool keys_fit_inline = false; 944 u32 *key_jump_cmd; 945 u32 *desc; 946 947 if (!ctx->enckeylen || !ctx->authsize) 948 return 0; 949 950 /* 951 * RFC4106 encrypt shared descriptor 952 * Job Descriptor and Shared Descriptor 953 * must fit into the 64-word Descriptor h/w Buffer 954 */ 955 if (DESC_RFC4106_ENC_LEN + GCM_DESC_JOB_IO_LEN + 956 ctx->enckeylen <= CAAM_DESC_BYTES_MAX) 957 keys_fit_inline = true; 958 959 desc = ctx->sh_desc_enc; 960 961 init_sh_desc(desc, HDR_SHARE_SERIAL); 962 963 /* Skip key loading if it is loaded due to sharing */ 964 key_jump_cmd = append_jump(desc, JUMP_JSL | JUMP_TEST_ALL | 965 JUMP_COND_SHRD); 966 if (keys_fit_inline) 967 append_key_as_imm(desc, (void *)ctx->key, ctx->enckeylen, 968 ctx->enckeylen, CLASS_1 | KEY_DEST_CLASS_REG); 969 else 970 append_key(desc, ctx->key_dma, ctx->enckeylen, 971 CLASS_1 | KEY_DEST_CLASS_REG); 972 set_jump_tgt_here(desc, key_jump_cmd); 973 974 /* Class 1 operation */ 975 append_operation(desc, ctx->class1_alg_type | 976 OP_ALG_AS_INITFINAL | OP_ALG_ENCRYPT); 977 978 append_math_sub_imm_u32(desc, VARSEQINLEN, REG3, IMM, 8); 979 append_math_add(desc, VARSEQOUTLEN, ZERO, REG3, CAAM_CMD_SZ); 980 981 /* Read assoc data */ 982 append_seq_fifo_load(desc, 0, FIFOLD_CLASS_CLASS1 | FIFOLDST_VLF | 983 FIFOLD_TYPE_AAD | FIFOLD_TYPE_FLUSH1); 984 985 /* Skip IV */ 986 append_seq_fifo_load(desc, 8, FIFOLD_CLASS_SKIP); 987 988 /* Will read cryptlen bytes */ 989 append_math_sub(desc, VARSEQINLEN, SEQINLEN, REG0, CAAM_CMD_SZ); 990 991 /* Workaround for erratum A-005473 (simultaneous SEQ FIFO skips) */ 992 append_seq_fifo_load(desc, 0, FIFOLD_CLASS_CLASS1 | FIFOLD_TYPE_MSG); 993 994 /* Skip assoc data */ 995 append_seq_fifo_store(desc, 0, FIFOST_TYPE_SKIP | FIFOLDST_VLF); 996 997 /* cryptlen = seqoutlen - assoclen */ 998 append_math_sub(desc, VARSEQOUTLEN, VARSEQINLEN, REG0, CAAM_CMD_SZ); 999 1000 /* Write encrypted data */ 1001 append_seq_fifo_store(desc, 0, FIFOST_TYPE_MESSAGE_DATA | FIFOLDST_VLF); 1002 1003 /* Read payload data */ 1004 append_seq_fifo_load(desc, 0, FIFOLD_CLASS_CLASS1 | FIFOLDST_VLF | 1005 FIFOLD_TYPE_MSG | FIFOLD_TYPE_LAST1); 1006 1007 /* Write ICV */ 1008 append_seq_store(desc, ctx->authsize, LDST_CLASS_1_CCB | 1009 LDST_SRCDST_BYTE_CONTEXT); 1010 1011 ctx->sh_desc_enc_dma = dma_map_single(jrdev, desc, 1012 desc_bytes(desc), 1013 DMA_TO_DEVICE); 1014 if (dma_mapping_error(jrdev, ctx->sh_desc_enc_dma)) { 1015 dev_err(jrdev, "unable to map shared descriptor\n"); 1016 return -ENOMEM; 1017 } 1018 #ifdef DEBUG 1019 print_hex_dump(KERN_ERR, "rfc4106 enc shdesc@"__stringify(__LINE__)": ", 1020 DUMP_PREFIX_ADDRESS, 16, 4, desc, 1021 desc_bytes(desc), 1); 1022 #endif 1023 1024 /* 1025 * Job Descriptor and Shared Descriptors 1026 * must all fit into the 64-word Descriptor h/w Buffer 1027 */ 1028 keys_fit_inline = false; 1029 if (DESC_RFC4106_DEC_LEN + DESC_JOB_IO_LEN + 1030 ctx->enckeylen <= CAAM_DESC_BYTES_MAX) 1031 keys_fit_inline = true; 1032 1033 desc = ctx->sh_desc_dec; 1034 1035 init_sh_desc(desc, HDR_SHARE_SERIAL); 1036 1037 /* Skip key loading if it is loaded due to sharing */ 1038 key_jump_cmd = append_jump(desc, JUMP_JSL | 1039 JUMP_TEST_ALL | JUMP_COND_SHRD); 1040 if (keys_fit_inline) 1041 append_key_as_imm(desc, (void *)ctx->key, ctx->enckeylen, 1042 ctx->enckeylen, CLASS_1 | KEY_DEST_CLASS_REG); 1043 else 1044 append_key(desc, ctx->key_dma, ctx->enckeylen, 1045 CLASS_1 | KEY_DEST_CLASS_REG); 1046 set_jump_tgt_here(desc, key_jump_cmd); 1047 1048 /* Class 1 operation */ 1049 append_operation(desc, ctx->class1_alg_type | 1050 OP_ALG_AS_INITFINAL | OP_ALG_DECRYPT | OP_ALG_ICV_ON); 1051 1052 append_math_sub_imm_u32(desc, VARSEQINLEN, REG3, IMM, 8); 1053 append_math_add(desc, VARSEQOUTLEN, ZERO, REG3, CAAM_CMD_SZ); 1054 1055 /* Read assoc data */ 1056 append_seq_fifo_load(desc, 0, FIFOLD_CLASS_CLASS1 | FIFOLDST_VLF | 1057 FIFOLD_TYPE_AAD | FIFOLD_TYPE_FLUSH1); 1058 1059 /* Skip IV */ 1060 append_seq_fifo_load(desc, 8, FIFOLD_CLASS_SKIP); 1061 1062 /* Will read cryptlen bytes */ 1063 append_math_sub(desc, VARSEQINLEN, SEQOUTLEN, REG3, CAAM_CMD_SZ); 1064 1065 /* Workaround for erratum A-005473 (simultaneous SEQ FIFO skips) */ 1066 append_seq_fifo_load(desc, 0, FIFOLD_CLASS_CLASS1 | FIFOLD_TYPE_MSG); 1067 1068 /* Skip assoc data */ 1069 append_seq_fifo_store(desc, 0, FIFOST_TYPE_SKIP | FIFOLDST_VLF); 1070 1071 /* Will write cryptlen bytes */ 1072 append_math_sub(desc, VARSEQOUTLEN, SEQOUTLEN, REG0, CAAM_CMD_SZ); 1073 1074 /* Store payload data */ 1075 append_seq_fifo_store(desc, 0, FIFOST_TYPE_MESSAGE_DATA | FIFOLDST_VLF); 1076 1077 /* Read encrypted data */ 1078 append_seq_fifo_load(desc, 0, FIFOLD_CLASS_CLASS1 | FIFOLDST_VLF | 1079 FIFOLD_TYPE_MSG | FIFOLD_TYPE_FLUSH1); 1080 1081 /* Read ICV */ 1082 append_seq_fifo_load(desc, ctx->authsize, FIFOLD_CLASS_CLASS1 | 1083 FIFOLD_TYPE_ICV | FIFOLD_TYPE_LAST1); 1084 1085 ctx->sh_desc_dec_dma = dma_map_single(jrdev, desc, 1086 desc_bytes(desc), 1087 DMA_TO_DEVICE); 1088 if (dma_mapping_error(jrdev, ctx->sh_desc_dec_dma)) { 1089 dev_err(jrdev, "unable to map shared descriptor\n"); 1090 return -ENOMEM; 1091 } 1092 #ifdef DEBUG 1093 print_hex_dump(KERN_ERR, "rfc4106 dec shdesc@"__stringify(__LINE__)": ", 1094 DUMP_PREFIX_ADDRESS, 16, 4, desc, 1095 desc_bytes(desc), 1); 1096 #endif 1097 1098 return 0; 1099 } 1100 1101 static int rfc4106_setauthsize(struct crypto_aead *authenc, 1102 unsigned int authsize) 1103 { 1104 struct caam_ctx *ctx = crypto_aead_ctx(authenc); 1105 1106 ctx->authsize = authsize; 1107 rfc4106_set_sh_desc(authenc); 1108 1109 return 0; 1110 } 1111 1112 static int rfc4543_set_sh_desc(struct crypto_aead *aead) 1113 { 1114 struct caam_ctx *ctx = crypto_aead_ctx(aead); 1115 struct device *jrdev = ctx->jrdev; 1116 bool keys_fit_inline = false; 1117 u32 *key_jump_cmd; 1118 u32 *read_move_cmd, *write_move_cmd; 1119 u32 *desc; 1120 1121 if (!ctx->enckeylen || !ctx->authsize) 1122 return 0; 1123 1124 /* 1125 * RFC4543 encrypt shared descriptor 1126 * Job Descriptor and Shared Descriptor 1127 * must fit into the 64-word Descriptor h/w Buffer 1128 */ 1129 if (DESC_RFC4543_ENC_LEN + GCM_DESC_JOB_IO_LEN + 1130 ctx->enckeylen <= CAAM_DESC_BYTES_MAX) 1131 keys_fit_inline = true; 1132 1133 desc = ctx->sh_desc_enc; 1134 1135 init_sh_desc(desc, HDR_SHARE_SERIAL); 1136 1137 /* Skip key loading if it is loaded due to sharing */ 1138 key_jump_cmd = append_jump(desc, JUMP_JSL | JUMP_TEST_ALL | 1139 JUMP_COND_SHRD); 1140 if (keys_fit_inline) 1141 append_key_as_imm(desc, (void *)ctx->key, ctx->enckeylen, 1142 ctx->enckeylen, CLASS_1 | KEY_DEST_CLASS_REG); 1143 else 1144 append_key(desc, ctx->key_dma, ctx->enckeylen, 1145 CLASS_1 | KEY_DEST_CLASS_REG); 1146 set_jump_tgt_here(desc, key_jump_cmd); 1147 1148 /* Class 1 operation */ 1149 append_operation(desc, ctx->class1_alg_type | 1150 OP_ALG_AS_INITFINAL | OP_ALG_ENCRYPT); 1151 1152 /* assoclen + cryptlen = seqinlen */ 1153 append_math_sub(desc, REG3, SEQINLEN, REG0, CAAM_CMD_SZ); 1154 1155 /* 1156 * MOVE_LEN opcode is not available in all SEC HW revisions, 1157 * thus need to do some magic, i.e. self-patch the descriptor 1158 * buffer. 1159 */ 1160 read_move_cmd = append_move(desc, MOVE_SRC_DESCBUF | MOVE_DEST_MATH3 | 1161 (0x6 << MOVE_LEN_SHIFT)); 1162 write_move_cmd = append_move(desc, MOVE_SRC_MATH3 | MOVE_DEST_DESCBUF | 1163 (0x8 << MOVE_LEN_SHIFT)); 1164 1165 /* Will read assoclen + cryptlen bytes */ 1166 append_math_sub(desc, VARSEQINLEN, SEQINLEN, REG0, CAAM_CMD_SZ); 1167 1168 /* Will write assoclen + cryptlen bytes */ 1169 append_math_sub(desc, VARSEQOUTLEN, SEQINLEN, REG0, CAAM_CMD_SZ); 1170 1171 /* Read and write assoclen + cryptlen bytes */ 1172 aead_append_src_dst(desc, FIFOLD_TYPE_AAD); 1173 1174 set_move_tgt_here(desc, read_move_cmd); 1175 set_move_tgt_here(desc, write_move_cmd); 1176 append_cmd(desc, CMD_LOAD | DISABLE_AUTO_INFO_FIFO); 1177 /* Move payload data to OFIFO */ 1178 append_move(desc, MOVE_SRC_INFIFO_CL | MOVE_DEST_OUTFIFO); 1179 1180 /* Write ICV */ 1181 append_seq_store(desc, ctx->authsize, LDST_CLASS_1_CCB | 1182 LDST_SRCDST_BYTE_CONTEXT); 1183 1184 ctx->sh_desc_enc_dma = dma_map_single(jrdev, desc, 1185 desc_bytes(desc), 1186 DMA_TO_DEVICE); 1187 if (dma_mapping_error(jrdev, ctx->sh_desc_enc_dma)) { 1188 dev_err(jrdev, "unable to map shared descriptor\n"); 1189 return -ENOMEM; 1190 } 1191 #ifdef DEBUG 1192 print_hex_dump(KERN_ERR, "rfc4543 enc shdesc@"__stringify(__LINE__)": ", 1193 DUMP_PREFIX_ADDRESS, 16, 4, desc, 1194 desc_bytes(desc), 1); 1195 #endif 1196 1197 /* 1198 * Job Descriptor and Shared Descriptors 1199 * must all fit into the 64-word Descriptor h/w Buffer 1200 */ 1201 keys_fit_inline = false; 1202 if (DESC_RFC4543_DEC_LEN + GCM_DESC_JOB_IO_LEN + 1203 ctx->enckeylen <= CAAM_DESC_BYTES_MAX) 1204 keys_fit_inline = true; 1205 1206 desc = ctx->sh_desc_dec; 1207 1208 init_sh_desc(desc, HDR_SHARE_SERIAL); 1209 1210 /* Skip key loading if it is loaded due to sharing */ 1211 key_jump_cmd = append_jump(desc, JUMP_JSL | 1212 JUMP_TEST_ALL | JUMP_COND_SHRD); 1213 if (keys_fit_inline) 1214 append_key_as_imm(desc, (void *)ctx->key, ctx->enckeylen, 1215 ctx->enckeylen, CLASS_1 | KEY_DEST_CLASS_REG); 1216 else 1217 append_key(desc, ctx->key_dma, ctx->enckeylen, 1218 CLASS_1 | KEY_DEST_CLASS_REG); 1219 set_jump_tgt_here(desc, key_jump_cmd); 1220 1221 /* Class 1 operation */ 1222 append_operation(desc, ctx->class1_alg_type | 1223 OP_ALG_AS_INITFINAL | OP_ALG_DECRYPT | OP_ALG_ICV_ON); 1224 1225 /* assoclen + cryptlen = seqoutlen */ 1226 append_math_sub(desc, REG3, SEQOUTLEN, REG0, CAAM_CMD_SZ); 1227 1228 /* 1229 * MOVE_LEN opcode is not available in all SEC HW revisions, 1230 * thus need to do some magic, i.e. self-patch the descriptor 1231 * buffer. 1232 */ 1233 read_move_cmd = append_move(desc, MOVE_SRC_DESCBUF | MOVE_DEST_MATH3 | 1234 (0x6 << MOVE_LEN_SHIFT)); 1235 write_move_cmd = append_move(desc, MOVE_SRC_MATH3 | MOVE_DEST_DESCBUF | 1236 (0x8 << MOVE_LEN_SHIFT)); 1237 1238 /* Will read assoclen + cryptlen bytes */ 1239 append_math_sub(desc, VARSEQINLEN, SEQOUTLEN, REG0, CAAM_CMD_SZ); 1240 1241 /* Will write assoclen + cryptlen bytes */ 1242 append_math_sub(desc, VARSEQOUTLEN, SEQOUTLEN, REG0, CAAM_CMD_SZ); 1243 1244 /* Store payload data */ 1245 append_seq_fifo_store(desc, 0, FIFOST_TYPE_MESSAGE_DATA | FIFOLDST_VLF); 1246 1247 /* In-snoop assoclen + cryptlen data */ 1248 append_seq_fifo_load(desc, 0, FIFOLD_CLASS_BOTH | FIFOLDST_VLF | 1249 FIFOLD_TYPE_AAD | FIFOLD_TYPE_LAST2FLUSH1); 1250 1251 set_move_tgt_here(desc, read_move_cmd); 1252 set_move_tgt_here(desc, write_move_cmd); 1253 append_cmd(desc, CMD_LOAD | DISABLE_AUTO_INFO_FIFO); 1254 /* Move payload data to OFIFO */ 1255 append_move(desc, MOVE_SRC_INFIFO_CL | MOVE_DEST_OUTFIFO); 1256 append_cmd(desc, CMD_LOAD | ENABLE_AUTO_INFO_FIFO); 1257 1258 /* Read ICV */ 1259 append_seq_fifo_load(desc, ctx->authsize, FIFOLD_CLASS_CLASS1 | 1260 FIFOLD_TYPE_ICV | FIFOLD_TYPE_LAST1); 1261 1262 ctx->sh_desc_dec_dma = dma_map_single(jrdev, desc, 1263 desc_bytes(desc), 1264 DMA_TO_DEVICE); 1265 if (dma_mapping_error(jrdev, ctx->sh_desc_dec_dma)) { 1266 dev_err(jrdev, "unable to map shared descriptor\n"); 1267 return -ENOMEM; 1268 } 1269 #ifdef DEBUG 1270 print_hex_dump(KERN_ERR, "rfc4543 dec shdesc@"__stringify(__LINE__)": ", 1271 DUMP_PREFIX_ADDRESS, 16, 4, desc, 1272 desc_bytes(desc), 1); 1273 #endif 1274 1275 return 0; 1276 } 1277 1278 static int rfc4543_setauthsize(struct crypto_aead *authenc, 1279 unsigned int authsize) 1280 { 1281 struct caam_ctx *ctx = crypto_aead_ctx(authenc); 1282 1283 ctx->authsize = authsize; 1284 rfc4543_set_sh_desc(authenc); 1285 1286 return 0; 1287 } 1288 1289 static u32 gen_split_aead_key(struct caam_ctx *ctx, const u8 *key_in, 1290 u32 authkeylen) 1291 { 1292 return gen_split_key(ctx->jrdev, ctx->key, ctx->split_key_len, 1293 ctx->split_key_pad_len, key_in, authkeylen, 1294 ctx->alg_op); 1295 } 1296 1297 static int aead_setkey(struct crypto_aead *aead, 1298 const u8 *key, unsigned int keylen) 1299 { 1300 /* Sizes for MDHA pads (*not* keys): MD5, SHA1, 224, 256, 384, 512 */ 1301 static const u8 mdpadlen[] = { 16, 20, 32, 32, 64, 64 }; 1302 struct caam_ctx *ctx = crypto_aead_ctx(aead); 1303 struct device *jrdev = ctx->jrdev; 1304 struct crypto_authenc_keys keys; 1305 int ret = 0; 1306 1307 if (crypto_authenc_extractkeys(&keys, key, keylen) != 0) 1308 goto badkey; 1309 1310 /* Pick class 2 key length from algorithm submask */ 1311 ctx->split_key_len = mdpadlen[(ctx->alg_op & OP_ALG_ALGSEL_SUBMASK) >> 1312 OP_ALG_ALGSEL_SHIFT] * 2; 1313 ctx->split_key_pad_len = ALIGN(ctx->split_key_len, 16); 1314 1315 if (ctx->split_key_pad_len + keys.enckeylen > CAAM_MAX_KEY_SIZE) 1316 goto badkey; 1317 1318 #ifdef DEBUG 1319 printk(KERN_ERR "keylen %d enckeylen %d authkeylen %d\n", 1320 keys.authkeylen + keys.enckeylen, keys.enckeylen, 1321 keys.authkeylen); 1322 printk(KERN_ERR "split_key_len %d split_key_pad_len %d\n", 1323 ctx->split_key_len, ctx->split_key_pad_len); 1324 print_hex_dump(KERN_ERR, "key in @"__stringify(__LINE__)": ", 1325 DUMP_PREFIX_ADDRESS, 16, 4, key, keylen, 1); 1326 #endif 1327 1328 ret = gen_split_aead_key(ctx, keys.authkey, keys.authkeylen); 1329 if (ret) { 1330 goto badkey; 1331 } 1332 1333 /* postpend encryption key to auth split key */ 1334 memcpy(ctx->key + ctx->split_key_pad_len, keys.enckey, keys.enckeylen); 1335 1336 ctx->key_dma = dma_map_single(jrdev, ctx->key, ctx->split_key_pad_len + 1337 keys.enckeylen, DMA_TO_DEVICE); 1338 if (dma_mapping_error(jrdev, ctx->key_dma)) { 1339 dev_err(jrdev, "unable to map key i/o memory\n"); 1340 return -ENOMEM; 1341 } 1342 #ifdef DEBUG 1343 print_hex_dump(KERN_ERR, "ctx.key@"__stringify(__LINE__)": ", 1344 DUMP_PREFIX_ADDRESS, 16, 4, ctx->key, 1345 ctx->split_key_pad_len + keys.enckeylen, 1); 1346 #endif 1347 1348 ctx->enckeylen = keys.enckeylen; 1349 1350 ret = aead_set_sh_desc(aead); 1351 if (ret) { 1352 dma_unmap_single(jrdev, ctx->key_dma, ctx->split_key_pad_len + 1353 keys.enckeylen, DMA_TO_DEVICE); 1354 } 1355 1356 return ret; 1357 badkey: 1358 crypto_aead_set_flags(aead, CRYPTO_TFM_RES_BAD_KEY_LEN); 1359 return -EINVAL; 1360 } 1361 1362 static int gcm_setkey(struct crypto_aead *aead, 1363 const u8 *key, unsigned int keylen) 1364 { 1365 struct caam_ctx *ctx = crypto_aead_ctx(aead); 1366 struct device *jrdev = ctx->jrdev; 1367 int ret = 0; 1368 1369 #ifdef DEBUG 1370 print_hex_dump(KERN_ERR, "key in @"__stringify(__LINE__)": ", 1371 DUMP_PREFIX_ADDRESS, 16, 4, key, keylen, 1); 1372 #endif 1373 1374 memcpy(ctx->key, key, keylen); 1375 ctx->key_dma = dma_map_single(jrdev, ctx->key, keylen, 1376 DMA_TO_DEVICE); 1377 if (dma_mapping_error(jrdev, ctx->key_dma)) { 1378 dev_err(jrdev, "unable to map key i/o memory\n"); 1379 return -ENOMEM; 1380 } 1381 ctx->enckeylen = keylen; 1382 1383 ret = gcm_set_sh_desc(aead); 1384 if (ret) { 1385 dma_unmap_single(jrdev, ctx->key_dma, ctx->enckeylen, 1386 DMA_TO_DEVICE); 1387 } 1388 1389 return ret; 1390 } 1391 1392 static int rfc4106_setkey(struct crypto_aead *aead, 1393 const u8 *key, unsigned int keylen) 1394 { 1395 struct caam_ctx *ctx = crypto_aead_ctx(aead); 1396 struct device *jrdev = ctx->jrdev; 1397 int ret = 0; 1398 1399 if (keylen < 4) 1400 return -EINVAL; 1401 1402 #ifdef DEBUG 1403 print_hex_dump(KERN_ERR, "key in @"__stringify(__LINE__)": ", 1404 DUMP_PREFIX_ADDRESS, 16, 4, key, keylen, 1); 1405 #endif 1406 1407 memcpy(ctx->key, key, keylen); 1408 1409 /* 1410 * The last four bytes of the key material are used as the salt value 1411 * in the nonce. Update the AES key length. 1412 */ 1413 ctx->enckeylen = keylen - 4; 1414 1415 ctx->key_dma = dma_map_single(jrdev, ctx->key, ctx->enckeylen, 1416 DMA_TO_DEVICE); 1417 if (dma_mapping_error(jrdev, ctx->key_dma)) { 1418 dev_err(jrdev, "unable to map key i/o memory\n"); 1419 return -ENOMEM; 1420 } 1421 1422 ret = rfc4106_set_sh_desc(aead); 1423 if (ret) { 1424 dma_unmap_single(jrdev, ctx->key_dma, ctx->enckeylen, 1425 DMA_TO_DEVICE); 1426 } 1427 1428 return ret; 1429 } 1430 1431 static int rfc4543_setkey(struct crypto_aead *aead, 1432 const u8 *key, unsigned int keylen) 1433 { 1434 struct caam_ctx *ctx = crypto_aead_ctx(aead); 1435 struct device *jrdev = ctx->jrdev; 1436 int ret = 0; 1437 1438 if (keylen < 4) 1439 return -EINVAL; 1440 1441 #ifdef DEBUG 1442 print_hex_dump(KERN_ERR, "key in @"__stringify(__LINE__)": ", 1443 DUMP_PREFIX_ADDRESS, 16, 4, key, keylen, 1); 1444 #endif 1445 1446 memcpy(ctx->key, key, keylen); 1447 1448 /* 1449 * The last four bytes of the key material are used as the salt value 1450 * in the nonce. Update the AES key length. 1451 */ 1452 ctx->enckeylen = keylen - 4; 1453 1454 ctx->key_dma = dma_map_single(jrdev, ctx->key, ctx->enckeylen, 1455 DMA_TO_DEVICE); 1456 if (dma_mapping_error(jrdev, ctx->key_dma)) { 1457 dev_err(jrdev, "unable to map key i/o memory\n"); 1458 return -ENOMEM; 1459 } 1460 1461 ret = rfc4543_set_sh_desc(aead); 1462 if (ret) { 1463 dma_unmap_single(jrdev, ctx->key_dma, ctx->enckeylen, 1464 DMA_TO_DEVICE); 1465 } 1466 1467 return ret; 1468 } 1469 1470 static int ablkcipher_setkey(struct crypto_ablkcipher *ablkcipher, 1471 const u8 *key, unsigned int keylen) 1472 { 1473 struct caam_ctx *ctx = crypto_ablkcipher_ctx(ablkcipher); 1474 struct ablkcipher_tfm *crt = &ablkcipher->base.crt_ablkcipher; 1475 struct crypto_tfm *tfm = crypto_ablkcipher_tfm(ablkcipher); 1476 const char *alg_name = crypto_tfm_alg_name(tfm); 1477 struct device *jrdev = ctx->jrdev; 1478 int ret = 0; 1479 u32 *key_jump_cmd; 1480 u32 *desc; 1481 u32 *nonce; 1482 u32 geniv; 1483 u32 ctx1_iv_off = 0; 1484 const bool ctr_mode = ((ctx->class1_alg_type & OP_ALG_AAI_MASK) == 1485 OP_ALG_AAI_CTR_MOD128); 1486 const bool is_rfc3686 = (ctr_mode && 1487 (strstr(alg_name, "rfc3686") != NULL)); 1488 1489 #ifdef DEBUG 1490 print_hex_dump(KERN_ERR, "key in @"__stringify(__LINE__)": ", 1491 DUMP_PREFIX_ADDRESS, 16, 4, key, keylen, 1); 1492 #endif 1493 /* 1494 * AES-CTR needs to load IV in CONTEXT1 reg 1495 * at an offset of 128bits (16bytes) 1496 * CONTEXT1[255:128] = IV 1497 */ 1498 if (ctr_mode) 1499 ctx1_iv_off = 16; 1500 1501 /* 1502 * RFC3686 specific: 1503 * | CONTEXT1[255:128] = {NONCE, IV, COUNTER} 1504 * | *key = {KEY, NONCE} 1505 */ 1506 if (is_rfc3686) { 1507 ctx1_iv_off = 16 + CTR_RFC3686_NONCE_SIZE; 1508 keylen -= CTR_RFC3686_NONCE_SIZE; 1509 } 1510 1511 memcpy(ctx->key, key, keylen); 1512 ctx->key_dma = dma_map_single(jrdev, ctx->key, keylen, 1513 DMA_TO_DEVICE); 1514 if (dma_mapping_error(jrdev, ctx->key_dma)) { 1515 dev_err(jrdev, "unable to map key i/o memory\n"); 1516 return -ENOMEM; 1517 } 1518 ctx->enckeylen = keylen; 1519 1520 /* ablkcipher_encrypt shared descriptor */ 1521 desc = ctx->sh_desc_enc; 1522 init_sh_desc(desc, HDR_SHARE_SERIAL | HDR_SAVECTX); 1523 /* Skip if already shared */ 1524 key_jump_cmd = append_jump(desc, JUMP_JSL | JUMP_TEST_ALL | 1525 JUMP_COND_SHRD); 1526 1527 /* Load class1 key only */ 1528 append_key_as_imm(desc, (void *)ctx->key, ctx->enckeylen, 1529 ctx->enckeylen, CLASS_1 | 1530 KEY_DEST_CLASS_REG); 1531 1532 /* Load nonce into CONTEXT1 reg */ 1533 if (is_rfc3686) { 1534 nonce = (u32 *)(key + keylen); 1535 append_load_imm_u32(desc, *nonce, LDST_CLASS_IND_CCB | 1536 LDST_SRCDST_BYTE_OUTFIFO | LDST_IMM); 1537 append_move(desc, MOVE_WAITCOMP | 1538 MOVE_SRC_OUTFIFO | 1539 MOVE_DEST_CLASS1CTX | 1540 (16 << MOVE_OFFSET_SHIFT) | 1541 (CTR_RFC3686_NONCE_SIZE << MOVE_LEN_SHIFT)); 1542 } 1543 1544 set_jump_tgt_here(desc, key_jump_cmd); 1545 1546 /* Load iv */ 1547 append_seq_load(desc, crt->ivsize, LDST_SRCDST_BYTE_CONTEXT | 1548 LDST_CLASS_1_CCB | (ctx1_iv_off << LDST_OFFSET_SHIFT)); 1549 1550 /* Load counter into CONTEXT1 reg */ 1551 if (is_rfc3686) 1552 append_load_imm_u32(desc, be32_to_cpu(1), LDST_IMM | 1553 LDST_CLASS_1_CCB | 1554 LDST_SRCDST_BYTE_CONTEXT | 1555 ((ctx1_iv_off + CTR_RFC3686_IV_SIZE) << 1556 LDST_OFFSET_SHIFT)); 1557 1558 /* Load operation */ 1559 append_operation(desc, ctx->class1_alg_type | 1560 OP_ALG_AS_INITFINAL | OP_ALG_ENCRYPT); 1561 1562 /* Perform operation */ 1563 ablkcipher_append_src_dst(desc); 1564 1565 ctx->sh_desc_enc_dma = dma_map_single(jrdev, desc, 1566 desc_bytes(desc), 1567 DMA_TO_DEVICE); 1568 if (dma_mapping_error(jrdev, ctx->sh_desc_enc_dma)) { 1569 dev_err(jrdev, "unable to map shared descriptor\n"); 1570 return -ENOMEM; 1571 } 1572 #ifdef DEBUG 1573 print_hex_dump(KERN_ERR, 1574 "ablkcipher enc shdesc@"__stringify(__LINE__)": ", 1575 DUMP_PREFIX_ADDRESS, 16, 4, desc, 1576 desc_bytes(desc), 1); 1577 #endif 1578 /* ablkcipher_decrypt shared descriptor */ 1579 desc = ctx->sh_desc_dec; 1580 1581 init_sh_desc(desc, HDR_SHARE_SERIAL | HDR_SAVECTX); 1582 /* Skip if already shared */ 1583 key_jump_cmd = append_jump(desc, JUMP_JSL | JUMP_TEST_ALL | 1584 JUMP_COND_SHRD); 1585 1586 /* Load class1 key only */ 1587 append_key_as_imm(desc, (void *)ctx->key, ctx->enckeylen, 1588 ctx->enckeylen, CLASS_1 | 1589 KEY_DEST_CLASS_REG); 1590 1591 /* Load nonce into CONTEXT1 reg */ 1592 if (is_rfc3686) { 1593 nonce = (u32 *)(key + keylen); 1594 append_load_imm_u32(desc, *nonce, LDST_CLASS_IND_CCB | 1595 LDST_SRCDST_BYTE_OUTFIFO | LDST_IMM); 1596 append_move(desc, MOVE_WAITCOMP | 1597 MOVE_SRC_OUTFIFO | 1598 MOVE_DEST_CLASS1CTX | 1599 (16 << MOVE_OFFSET_SHIFT) | 1600 (CTR_RFC3686_NONCE_SIZE << MOVE_LEN_SHIFT)); 1601 } 1602 1603 set_jump_tgt_here(desc, key_jump_cmd); 1604 1605 /* load IV */ 1606 append_seq_load(desc, crt->ivsize, LDST_SRCDST_BYTE_CONTEXT | 1607 LDST_CLASS_1_CCB | (ctx1_iv_off << LDST_OFFSET_SHIFT)); 1608 1609 /* Load counter into CONTEXT1 reg */ 1610 if (is_rfc3686) 1611 append_load_imm_u32(desc, be32_to_cpu(1), LDST_IMM | 1612 LDST_CLASS_1_CCB | 1613 LDST_SRCDST_BYTE_CONTEXT | 1614 ((ctx1_iv_off + CTR_RFC3686_IV_SIZE) << 1615 LDST_OFFSET_SHIFT)); 1616 1617 /* Choose operation */ 1618 if (ctr_mode) 1619 append_operation(desc, ctx->class1_alg_type | 1620 OP_ALG_AS_INITFINAL | OP_ALG_DECRYPT); 1621 else 1622 append_dec_op1(desc, ctx->class1_alg_type); 1623 1624 /* Perform operation */ 1625 ablkcipher_append_src_dst(desc); 1626 1627 ctx->sh_desc_dec_dma = dma_map_single(jrdev, desc, 1628 desc_bytes(desc), 1629 DMA_TO_DEVICE); 1630 if (dma_mapping_error(jrdev, ctx->sh_desc_dec_dma)) { 1631 dev_err(jrdev, "unable to map shared descriptor\n"); 1632 return -ENOMEM; 1633 } 1634 1635 #ifdef DEBUG 1636 print_hex_dump(KERN_ERR, 1637 "ablkcipher dec shdesc@"__stringify(__LINE__)": ", 1638 DUMP_PREFIX_ADDRESS, 16, 4, desc, 1639 desc_bytes(desc), 1); 1640 #endif 1641 /* ablkcipher_givencrypt shared descriptor */ 1642 desc = ctx->sh_desc_givenc; 1643 1644 init_sh_desc(desc, HDR_SHARE_SERIAL | HDR_SAVECTX); 1645 /* Skip if already shared */ 1646 key_jump_cmd = append_jump(desc, JUMP_JSL | JUMP_TEST_ALL | 1647 JUMP_COND_SHRD); 1648 1649 /* Load class1 key only */ 1650 append_key_as_imm(desc, (void *)ctx->key, ctx->enckeylen, 1651 ctx->enckeylen, CLASS_1 | 1652 KEY_DEST_CLASS_REG); 1653 1654 /* Load Nonce into CONTEXT1 reg */ 1655 if (is_rfc3686) { 1656 nonce = (u32 *)(key + keylen); 1657 append_load_imm_u32(desc, *nonce, LDST_CLASS_IND_CCB | 1658 LDST_SRCDST_BYTE_OUTFIFO | LDST_IMM); 1659 append_move(desc, MOVE_WAITCOMP | 1660 MOVE_SRC_OUTFIFO | 1661 MOVE_DEST_CLASS1CTX | 1662 (16 << MOVE_OFFSET_SHIFT) | 1663 (CTR_RFC3686_NONCE_SIZE << MOVE_LEN_SHIFT)); 1664 } 1665 set_jump_tgt_here(desc, key_jump_cmd); 1666 1667 /* Generate IV */ 1668 geniv = NFIFOENTRY_STYPE_PAD | NFIFOENTRY_DEST_DECO | 1669 NFIFOENTRY_DTYPE_MSG | NFIFOENTRY_LC1 | 1670 NFIFOENTRY_PTYPE_RND | (crt->ivsize << NFIFOENTRY_DLEN_SHIFT); 1671 append_load_imm_u32(desc, geniv, LDST_CLASS_IND_CCB | 1672 LDST_SRCDST_WORD_INFO_FIFO | LDST_IMM); 1673 append_cmd(desc, CMD_LOAD | DISABLE_AUTO_INFO_FIFO); 1674 append_move(desc, MOVE_WAITCOMP | 1675 MOVE_SRC_INFIFO | 1676 MOVE_DEST_CLASS1CTX | 1677 (crt->ivsize << MOVE_LEN_SHIFT) | 1678 (ctx1_iv_off << MOVE_OFFSET_SHIFT)); 1679 append_cmd(desc, CMD_LOAD | ENABLE_AUTO_INFO_FIFO); 1680 1681 /* Copy generated IV to memory */ 1682 append_seq_store(desc, crt->ivsize, 1683 LDST_SRCDST_BYTE_CONTEXT | LDST_CLASS_1_CCB | 1684 (ctx1_iv_off << LDST_OFFSET_SHIFT)); 1685 1686 /* Load Counter into CONTEXT1 reg */ 1687 if (is_rfc3686) 1688 append_load_imm_u32(desc, (u32)1, LDST_IMM | 1689 LDST_CLASS_1_CCB | 1690 LDST_SRCDST_BYTE_CONTEXT | 1691 ((ctx1_iv_off + CTR_RFC3686_IV_SIZE) << 1692 LDST_OFFSET_SHIFT)); 1693 1694 if (ctx1_iv_off) 1695 append_jump(desc, JUMP_JSL | JUMP_TEST_ALL | JUMP_COND_NCP | 1696 (1 << JUMP_OFFSET_SHIFT)); 1697 1698 /* Load operation */ 1699 append_operation(desc, ctx->class1_alg_type | 1700 OP_ALG_AS_INITFINAL | OP_ALG_ENCRYPT); 1701 1702 /* Perform operation */ 1703 ablkcipher_append_src_dst(desc); 1704 1705 ctx->sh_desc_givenc_dma = dma_map_single(jrdev, desc, 1706 desc_bytes(desc), 1707 DMA_TO_DEVICE); 1708 if (dma_mapping_error(jrdev, ctx->sh_desc_givenc_dma)) { 1709 dev_err(jrdev, "unable to map shared descriptor\n"); 1710 return -ENOMEM; 1711 } 1712 #ifdef DEBUG 1713 print_hex_dump(KERN_ERR, 1714 "ablkcipher givenc shdesc@" __stringify(__LINE__) ": ", 1715 DUMP_PREFIX_ADDRESS, 16, 4, desc, 1716 desc_bytes(desc), 1); 1717 #endif 1718 1719 return ret; 1720 } 1721 1722 static int xts_ablkcipher_setkey(struct crypto_ablkcipher *ablkcipher, 1723 const u8 *key, unsigned int keylen) 1724 { 1725 struct caam_ctx *ctx = crypto_ablkcipher_ctx(ablkcipher); 1726 struct device *jrdev = ctx->jrdev; 1727 u32 *key_jump_cmd, *desc; 1728 __be64 sector_size = cpu_to_be64(512); 1729 1730 if (keylen != 2 * AES_MIN_KEY_SIZE && keylen != 2 * AES_MAX_KEY_SIZE) { 1731 crypto_ablkcipher_set_flags(ablkcipher, 1732 CRYPTO_TFM_RES_BAD_KEY_LEN); 1733 dev_err(jrdev, "key size mismatch\n"); 1734 return -EINVAL; 1735 } 1736 1737 memcpy(ctx->key, key, keylen); 1738 ctx->key_dma = dma_map_single(jrdev, ctx->key, keylen, DMA_TO_DEVICE); 1739 if (dma_mapping_error(jrdev, ctx->key_dma)) { 1740 dev_err(jrdev, "unable to map key i/o memory\n"); 1741 return -ENOMEM; 1742 } 1743 ctx->enckeylen = keylen; 1744 1745 /* xts_ablkcipher_encrypt shared descriptor */ 1746 desc = ctx->sh_desc_enc; 1747 init_sh_desc(desc, HDR_SHARE_SERIAL | HDR_SAVECTX); 1748 /* Skip if already shared */ 1749 key_jump_cmd = append_jump(desc, JUMP_JSL | JUMP_TEST_ALL | 1750 JUMP_COND_SHRD); 1751 1752 /* Load class1 keys only */ 1753 append_key_as_imm(desc, (void *)ctx->key, ctx->enckeylen, 1754 ctx->enckeylen, CLASS_1 | KEY_DEST_CLASS_REG); 1755 1756 /* Load sector size with index 40 bytes (0x28) */ 1757 append_cmd(desc, CMD_LOAD | IMMEDIATE | LDST_SRCDST_BYTE_CONTEXT | 1758 LDST_CLASS_1_CCB | (0x28 << LDST_OFFSET_SHIFT) | 8); 1759 append_data(desc, (void *)§or_size, 8); 1760 1761 set_jump_tgt_here(desc, key_jump_cmd); 1762 1763 /* 1764 * create sequence for loading the sector index 1765 * Upper 8B of IV - will be used as sector index 1766 * Lower 8B of IV - will be discarded 1767 */ 1768 append_cmd(desc, CMD_SEQ_LOAD | LDST_SRCDST_BYTE_CONTEXT | 1769 LDST_CLASS_1_CCB | (0x20 << LDST_OFFSET_SHIFT) | 8); 1770 append_seq_fifo_load(desc, 8, FIFOLD_CLASS_SKIP); 1771 1772 /* Load operation */ 1773 append_operation(desc, ctx->class1_alg_type | OP_ALG_AS_INITFINAL | 1774 OP_ALG_ENCRYPT); 1775 1776 /* Perform operation */ 1777 ablkcipher_append_src_dst(desc); 1778 1779 ctx->sh_desc_enc_dma = dma_map_single(jrdev, desc, desc_bytes(desc), 1780 DMA_TO_DEVICE); 1781 if (dma_mapping_error(jrdev, ctx->sh_desc_enc_dma)) { 1782 dev_err(jrdev, "unable to map shared descriptor\n"); 1783 return -ENOMEM; 1784 } 1785 #ifdef DEBUG 1786 print_hex_dump(KERN_ERR, 1787 "xts ablkcipher enc shdesc@" __stringify(__LINE__) ": ", 1788 DUMP_PREFIX_ADDRESS, 16, 4, desc, desc_bytes(desc), 1); 1789 #endif 1790 1791 /* xts_ablkcipher_decrypt shared descriptor */ 1792 desc = ctx->sh_desc_dec; 1793 1794 init_sh_desc(desc, HDR_SHARE_SERIAL | HDR_SAVECTX); 1795 /* Skip if already shared */ 1796 key_jump_cmd = append_jump(desc, JUMP_JSL | JUMP_TEST_ALL | 1797 JUMP_COND_SHRD); 1798 1799 /* Load class1 key only */ 1800 append_key_as_imm(desc, (void *)ctx->key, ctx->enckeylen, 1801 ctx->enckeylen, CLASS_1 | KEY_DEST_CLASS_REG); 1802 1803 /* Load sector size with index 40 bytes (0x28) */ 1804 append_cmd(desc, CMD_LOAD | IMMEDIATE | LDST_SRCDST_BYTE_CONTEXT | 1805 LDST_CLASS_1_CCB | (0x28 << LDST_OFFSET_SHIFT) | 8); 1806 append_data(desc, (void *)§or_size, 8); 1807 1808 set_jump_tgt_here(desc, key_jump_cmd); 1809 1810 /* 1811 * create sequence for loading the sector index 1812 * Upper 8B of IV - will be used as sector index 1813 * Lower 8B of IV - will be discarded 1814 */ 1815 append_cmd(desc, CMD_SEQ_LOAD | LDST_SRCDST_BYTE_CONTEXT | 1816 LDST_CLASS_1_CCB | (0x20 << LDST_OFFSET_SHIFT) | 8); 1817 append_seq_fifo_load(desc, 8, FIFOLD_CLASS_SKIP); 1818 1819 /* Load operation */ 1820 append_dec_op1(desc, ctx->class1_alg_type); 1821 1822 /* Perform operation */ 1823 ablkcipher_append_src_dst(desc); 1824 1825 ctx->sh_desc_dec_dma = dma_map_single(jrdev, desc, desc_bytes(desc), 1826 DMA_TO_DEVICE); 1827 if (dma_mapping_error(jrdev, ctx->sh_desc_dec_dma)) { 1828 dma_unmap_single(jrdev, ctx->sh_desc_enc_dma, 1829 desc_bytes(ctx->sh_desc_enc), DMA_TO_DEVICE); 1830 dev_err(jrdev, "unable to map shared descriptor\n"); 1831 return -ENOMEM; 1832 } 1833 #ifdef DEBUG 1834 print_hex_dump(KERN_ERR, 1835 "xts ablkcipher dec shdesc@" __stringify(__LINE__) ": ", 1836 DUMP_PREFIX_ADDRESS, 16, 4, desc, desc_bytes(desc), 1); 1837 #endif 1838 1839 return 0; 1840 } 1841 1842 /* 1843 * aead_edesc - s/w-extended aead descriptor 1844 * @assoc_nents: number of segments in associated data (SPI+Seq) scatterlist 1845 * @src_nents: number of segments in input scatterlist 1846 * @dst_nents: number of segments in output scatterlist 1847 * @iv_dma: dma address of iv for checking continuity and link table 1848 * @desc: h/w descriptor (variable length; must not exceed MAX_CAAM_DESCSIZE) 1849 * @sec4_sg_bytes: length of dma mapped sec4_sg space 1850 * @sec4_sg_dma: bus physical mapped address of h/w link table 1851 * @hw_desc: the h/w job descriptor followed by any referenced link tables 1852 */ 1853 struct aead_edesc { 1854 int assoc_nents; 1855 int src_nents; 1856 int dst_nents; 1857 dma_addr_t iv_dma; 1858 int sec4_sg_bytes; 1859 dma_addr_t sec4_sg_dma; 1860 struct sec4_sg_entry *sec4_sg; 1861 u32 hw_desc[]; 1862 }; 1863 1864 /* 1865 * ablkcipher_edesc - s/w-extended ablkcipher descriptor 1866 * @src_nents: number of segments in input scatterlist 1867 * @dst_nents: number of segments in output scatterlist 1868 * @iv_dma: dma address of iv for checking continuity and link table 1869 * @desc: h/w descriptor (variable length; must not exceed MAX_CAAM_DESCSIZE) 1870 * @sec4_sg_bytes: length of dma mapped sec4_sg space 1871 * @sec4_sg_dma: bus physical mapped address of h/w link table 1872 * @hw_desc: the h/w job descriptor followed by any referenced link tables 1873 */ 1874 struct ablkcipher_edesc { 1875 int src_nents; 1876 int dst_nents; 1877 dma_addr_t iv_dma; 1878 int sec4_sg_bytes; 1879 dma_addr_t sec4_sg_dma; 1880 struct sec4_sg_entry *sec4_sg; 1881 u32 hw_desc[0]; 1882 }; 1883 1884 static void caam_unmap(struct device *dev, struct scatterlist *src, 1885 struct scatterlist *dst, int src_nents, 1886 int dst_nents, 1887 dma_addr_t iv_dma, int ivsize, dma_addr_t sec4_sg_dma, 1888 int sec4_sg_bytes) 1889 { 1890 if (dst != src) { 1891 dma_unmap_sg(dev, src, src_nents ? : 1, DMA_TO_DEVICE); 1892 dma_unmap_sg(dev, dst, dst_nents ? : 1, DMA_FROM_DEVICE); 1893 } else { 1894 dma_unmap_sg(dev, src, src_nents ? : 1, DMA_BIDIRECTIONAL); 1895 } 1896 1897 if (iv_dma) 1898 dma_unmap_single(dev, iv_dma, ivsize, DMA_TO_DEVICE); 1899 if (sec4_sg_bytes) 1900 dma_unmap_single(dev, sec4_sg_dma, sec4_sg_bytes, 1901 DMA_TO_DEVICE); 1902 } 1903 1904 static void aead_unmap(struct device *dev, 1905 struct aead_edesc *edesc, 1906 struct aead_request *req) 1907 { 1908 caam_unmap(dev, req->src, req->dst, 1909 edesc->src_nents, edesc->dst_nents, 0, 0, 1910 edesc->sec4_sg_dma, edesc->sec4_sg_bytes); 1911 } 1912 1913 static void ablkcipher_unmap(struct device *dev, 1914 struct ablkcipher_edesc *edesc, 1915 struct ablkcipher_request *req) 1916 { 1917 struct crypto_ablkcipher *ablkcipher = crypto_ablkcipher_reqtfm(req); 1918 int ivsize = crypto_ablkcipher_ivsize(ablkcipher); 1919 1920 caam_unmap(dev, req->src, req->dst, 1921 edesc->src_nents, edesc->dst_nents, 1922 edesc->iv_dma, ivsize, 1923 edesc->sec4_sg_dma, edesc->sec4_sg_bytes); 1924 } 1925 1926 static void aead_encrypt_done(struct device *jrdev, u32 *desc, u32 err, 1927 void *context) 1928 { 1929 struct aead_request *req = context; 1930 struct aead_edesc *edesc; 1931 1932 #ifdef DEBUG 1933 dev_err(jrdev, "%s %d: err 0x%x\n", __func__, __LINE__, err); 1934 #endif 1935 1936 edesc = container_of(desc, struct aead_edesc, hw_desc[0]); 1937 1938 if (err) 1939 caam_jr_strstatus(jrdev, err); 1940 1941 aead_unmap(jrdev, edesc, req); 1942 1943 kfree(edesc); 1944 1945 aead_request_complete(req, err); 1946 } 1947 1948 static void aead_decrypt_done(struct device *jrdev, u32 *desc, u32 err, 1949 void *context) 1950 { 1951 struct aead_request *req = context; 1952 struct aead_edesc *edesc; 1953 1954 #ifdef DEBUG 1955 dev_err(jrdev, "%s %d: err 0x%x\n", __func__, __LINE__, err); 1956 #endif 1957 1958 edesc = container_of(desc, struct aead_edesc, hw_desc[0]); 1959 1960 if (err) 1961 caam_jr_strstatus(jrdev, err); 1962 1963 aead_unmap(jrdev, edesc, req); 1964 1965 /* 1966 * verify hw auth check passed else return -EBADMSG 1967 */ 1968 if ((err & JRSTA_CCBERR_ERRID_MASK) == JRSTA_CCBERR_ERRID_ICVCHK) 1969 err = -EBADMSG; 1970 1971 kfree(edesc); 1972 1973 aead_request_complete(req, err); 1974 } 1975 1976 static void ablkcipher_encrypt_done(struct device *jrdev, u32 *desc, u32 err, 1977 void *context) 1978 { 1979 struct ablkcipher_request *req = context; 1980 struct ablkcipher_edesc *edesc; 1981 #ifdef DEBUG 1982 struct crypto_ablkcipher *ablkcipher = crypto_ablkcipher_reqtfm(req); 1983 int ivsize = crypto_ablkcipher_ivsize(ablkcipher); 1984 1985 dev_err(jrdev, "%s %d: err 0x%x\n", __func__, __LINE__, err); 1986 #endif 1987 1988 edesc = (struct ablkcipher_edesc *)((char *)desc - 1989 offsetof(struct ablkcipher_edesc, hw_desc)); 1990 1991 if (err) 1992 caam_jr_strstatus(jrdev, err); 1993 1994 #ifdef DEBUG 1995 print_hex_dump(KERN_ERR, "dstiv @"__stringify(__LINE__)": ", 1996 DUMP_PREFIX_ADDRESS, 16, 4, req->info, 1997 edesc->src_nents > 1 ? 100 : ivsize, 1); 1998 print_hex_dump(KERN_ERR, "dst @"__stringify(__LINE__)": ", 1999 DUMP_PREFIX_ADDRESS, 16, 4, sg_virt(req->src), 2000 edesc->dst_nents > 1 ? 100 : req->nbytes, 1); 2001 #endif 2002 2003 ablkcipher_unmap(jrdev, edesc, req); 2004 kfree(edesc); 2005 2006 ablkcipher_request_complete(req, err); 2007 } 2008 2009 static void ablkcipher_decrypt_done(struct device *jrdev, u32 *desc, u32 err, 2010 void *context) 2011 { 2012 struct ablkcipher_request *req = context; 2013 struct ablkcipher_edesc *edesc; 2014 #ifdef DEBUG 2015 struct crypto_ablkcipher *ablkcipher = crypto_ablkcipher_reqtfm(req); 2016 int ivsize = crypto_ablkcipher_ivsize(ablkcipher); 2017 2018 dev_err(jrdev, "%s %d: err 0x%x\n", __func__, __LINE__, err); 2019 #endif 2020 2021 edesc = (struct ablkcipher_edesc *)((char *)desc - 2022 offsetof(struct ablkcipher_edesc, hw_desc)); 2023 if (err) 2024 caam_jr_strstatus(jrdev, err); 2025 2026 #ifdef DEBUG 2027 print_hex_dump(KERN_ERR, "dstiv @"__stringify(__LINE__)": ", 2028 DUMP_PREFIX_ADDRESS, 16, 4, req->info, 2029 ivsize, 1); 2030 print_hex_dump(KERN_ERR, "dst @"__stringify(__LINE__)": ", 2031 DUMP_PREFIX_ADDRESS, 16, 4, sg_virt(req->src), 2032 edesc->dst_nents > 1 ? 100 : req->nbytes, 1); 2033 #endif 2034 2035 ablkcipher_unmap(jrdev, edesc, req); 2036 kfree(edesc); 2037 2038 ablkcipher_request_complete(req, err); 2039 } 2040 2041 /* 2042 * Fill in aead job descriptor 2043 */ 2044 static void init_aead_job(struct aead_request *req, 2045 struct aead_edesc *edesc, 2046 bool all_contig, bool encrypt) 2047 { 2048 struct crypto_aead *aead = crypto_aead_reqtfm(req); 2049 struct caam_ctx *ctx = crypto_aead_ctx(aead); 2050 int authsize = ctx->authsize; 2051 u32 *desc = edesc->hw_desc; 2052 u32 out_options, in_options; 2053 dma_addr_t dst_dma, src_dma; 2054 int len, sec4_sg_index = 0; 2055 dma_addr_t ptr; 2056 u32 *sh_desc; 2057 2058 sh_desc = encrypt ? ctx->sh_desc_enc : ctx->sh_desc_dec; 2059 ptr = encrypt ? ctx->sh_desc_enc_dma : ctx->sh_desc_dec_dma; 2060 2061 len = desc_len(sh_desc); 2062 init_job_desc_shared(desc, ptr, len, HDR_SHARE_DEFER | HDR_REVERSE); 2063 2064 if (all_contig) { 2065 src_dma = sg_dma_address(req->src); 2066 in_options = 0; 2067 } else { 2068 src_dma = edesc->sec4_sg_dma; 2069 sec4_sg_index += edesc->src_nents; 2070 in_options = LDST_SGF; 2071 } 2072 2073 append_seq_in_ptr(desc, src_dma, req->assoclen + req->cryptlen, 2074 in_options); 2075 2076 dst_dma = src_dma; 2077 out_options = in_options; 2078 2079 if (unlikely(req->src != req->dst)) { 2080 if (!edesc->dst_nents) { 2081 dst_dma = sg_dma_address(req->dst); 2082 } else { 2083 dst_dma = edesc->sec4_sg_dma + 2084 sec4_sg_index * 2085 sizeof(struct sec4_sg_entry); 2086 out_options = LDST_SGF; 2087 } 2088 } 2089 2090 if (encrypt) 2091 append_seq_out_ptr(desc, dst_dma, 2092 req->assoclen + req->cryptlen + authsize, 2093 out_options); 2094 else 2095 append_seq_out_ptr(desc, dst_dma, 2096 req->assoclen + req->cryptlen - authsize, 2097 out_options); 2098 2099 /* REG3 = assoclen */ 2100 append_math_add_imm_u32(desc, REG3, ZERO, IMM, req->assoclen); 2101 } 2102 2103 static void init_gcm_job(struct aead_request *req, 2104 struct aead_edesc *edesc, 2105 bool all_contig, bool encrypt) 2106 { 2107 struct crypto_aead *aead = crypto_aead_reqtfm(req); 2108 struct caam_ctx *ctx = crypto_aead_ctx(aead); 2109 unsigned int ivsize = crypto_aead_ivsize(aead); 2110 u32 *desc = edesc->hw_desc; 2111 bool generic_gcm = (ivsize == 12); 2112 unsigned int last; 2113 2114 init_aead_job(req, edesc, all_contig, encrypt); 2115 2116 /* BUG This should not be specific to generic GCM. */ 2117 last = 0; 2118 if (encrypt && generic_gcm && !(req->assoclen + req->cryptlen)) 2119 last = FIFOLD_TYPE_LAST1; 2120 2121 /* Read GCM IV */ 2122 append_cmd(desc, CMD_FIFO_LOAD | FIFOLD_CLASS_CLASS1 | IMMEDIATE | 2123 FIFOLD_TYPE_IV | FIFOLD_TYPE_FLUSH1 | 12 | last); 2124 /* Append Salt */ 2125 if (!generic_gcm) 2126 append_data(desc, ctx->key + ctx->enckeylen, 4); 2127 /* Append IV */ 2128 append_data(desc, req->iv, ivsize); 2129 /* End of blank commands */ 2130 } 2131 2132 static void init_authenc_job(struct aead_request *req, 2133 struct aead_edesc *edesc, 2134 bool all_contig, bool encrypt) 2135 { 2136 struct crypto_aead *aead = crypto_aead_reqtfm(req); 2137 struct caam_aead_alg *alg = container_of(crypto_aead_alg(aead), 2138 struct caam_aead_alg, aead); 2139 unsigned int ivsize = crypto_aead_ivsize(aead); 2140 struct caam_ctx *ctx = crypto_aead_ctx(aead); 2141 const bool ctr_mode = ((ctx->class1_alg_type & OP_ALG_AAI_MASK) == 2142 OP_ALG_AAI_CTR_MOD128); 2143 const bool is_rfc3686 = alg->caam.rfc3686; 2144 u32 *desc = edesc->hw_desc; 2145 u32 ivoffset = 0; 2146 2147 /* 2148 * AES-CTR needs to load IV in CONTEXT1 reg 2149 * at an offset of 128bits (16bytes) 2150 * CONTEXT1[255:128] = IV 2151 */ 2152 if (ctr_mode) 2153 ivoffset = 16; 2154 2155 /* 2156 * RFC3686 specific: 2157 * CONTEXT1[255:128] = {NONCE, IV, COUNTER} 2158 */ 2159 if (is_rfc3686) 2160 ivoffset = 16 + CTR_RFC3686_NONCE_SIZE; 2161 2162 init_aead_job(req, edesc, all_contig, encrypt); 2163 2164 if (ivsize && ((is_rfc3686 && encrypt) || !alg->caam.geniv)) 2165 append_load_as_imm(desc, req->iv, ivsize, 2166 LDST_CLASS_1_CCB | 2167 LDST_SRCDST_BYTE_CONTEXT | 2168 (ivoffset << LDST_OFFSET_SHIFT)); 2169 } 2170 2171 /* 2172 * Fill in ablkcipher job descriptor 2173 */ 2174 static void init_ablkcipher_job(u32 *sh_desc, dma_addr_t ptr, 2175 struct ablkcipher_edesc *edesc, 2176 struct ablkcipher_request *req, 2177 bool iv_contig) 2178 { 2179 struct crypto_ablkcipher *ablkcipher = crypto_ablkcipher_reqtfm(req); 2180 int ivsize = crypto_ablkcipher_ivsize(ablkcipher); 2181 u32 *desc = edesc->hw_desc; 2182 u32 out_options = 0, in_options; 2183 dma_addr_t dst_dma, src_dma; 2184 int len, sec4_sg_index = 0; 2185 2186 #ifdef DEBUG 2187 print_hex_dump(KERN_ERR, "presciv@"__stringify(__LINE__)": ", 2188 DUMP_PREFIX_ADDRESS, 16, 4, req->info, 2189 ivsize, 1); 2190 print_hex_dump(KERN_ERR, "src @"__stringify(__LINE__)": ", 2191 DUMP_PREFIX_ADDRESS, 16, 4, sg_virt(req->src), 2192 edesc->src_nents ? 100 : req->nbytes, 1); 2193 #endif 2194 2195 len = desc_len(sh_desc); 2196 init_job_desc_shared(desc, ptr, len, HDR_SHARE_DEFER | HDR_REVERSE); 2197 2198 if (iv_contig) { 2199 src_dma = edesc->iv_dma; 2200 in_options = 0; 2201 } else { 2202 src_dma = edesc->sec4_sg_dma; 2203 sec4_sg_index += edesc->src_nents + 1; 2204 in_options = LDST_SGF; 2205 } 2206 append_seq_in_ptr(desc, src_dma, req->nbytes + ivsize, in_options); 2207 2208 if (likely(req->src == req->dst)) { 2209 if (!edesc->src_nents && iv_contig) { 2210 dst_dma = sg_dma_address(req->src); 2211 } else { 2212 dst_dma = edesc->sec4_sg_dma + 2213 sizeof(struct sec4_sg_entry); 2214 out_options = LDST_SGF; 2215 } 2216 } else { 2217 if (!edesc->dst_nents) { 2218 dst_dma = sg_dma_address(req->dst); 2219 } else { 2220 dst_dma = edesc->sec4_sg_dma + 2221 sec4_sg_index * sizeof(struct sec4_sg_entry); 2222 out_options = LDST_SGF; 2223 } 2224 } 2225 append_seq_out_ptr(desc, dst_dma, req->nbytes, out_options); 2226 } 2227 2228 /* 2229 * Fill in ablkcipher givencrypt job descriptor 2230 */ 2231 static void init_ablkcipher_giv_job(u32 *sh_desc, dma_addr_t ptr, 2232 struct ablkcipher_edesc *edesc, 2233 struct ablkcipher_request *req, 2234 bool iv_contig) 2235 { 2236 struct crypto_ablkcipher *ablkcipher = crypto_ablkcipher_reqtfm(req); 2237 int ivsize = crypto_ablkcipher_ivsize(ablkcipher); 2238 u32 *desc = edesc->hw_desc; 2239 u32 out_options, in_options; 2240 dma_addr_t dst_dma, src_dma; 2241 int len, sec4_sg_index = 0; 2242 2243 #ifdef DEBUG 2244 print_hex_dump(KERN_ERR, "presciv@" __stringify(__LINE__) ": ", 2245 DUMP_PREFIX_ADDRESS, 16, 4, req->info, 2246 ivsize, 1); 2247 print_hex_dump(KERN_ERR, "src @" __stringify(__LINE__) ": ", 2248 DUMP_PREFIX_ADDRESS, 16, 4, sg_virt(req->src), 2249 edesc->src_nents ? 100 : req->nbytes, 1); 2250 #endif 2251 2252 len = desc_len(sh_desc); 2253 init_job_desc_shared(desc, ptr, len, HDR_SHARE_DEFER | HDR_REVERSE); 2254 2255 if (!edesc->src_nents) { 2256 src_dma = sg_dma_address(req->src); 2257 in_options = 0; 2258 } else { 2259 src_dma = edesc->sec4_sg_dma; 2260 sec4_sg_index += edesc->src_nents; 2261 in_options = LDST_SGF; 2262 } 2263 append_seq_in_ptr(desc, src_dma, req->nbytes, in_options); 2264 2265 if (iv_contig) { 2266 dst_dma = edesc->iv_dma; 2267 out_options = 0; 2268 } else { 2269 dst_dma = edesc->sec4_sg_dma + 2270 sec4_sg_index * sizeof(struct sec4_sg_entry); 2271 out_options = LDST_SGF; 2272 } 2273 append_seq_out_ptr(desc, dst_dma, req->nbytes + ivsize, out_options); 2274 } 2275 2276 /* 2277 * allocate and map the aead extended descriptor 2278 */ 2279 static struct aead_edesc *aead_edesc_alloc(struct aead_request *req, 2280 int desc_bytes, bool *all_contig_ptr, 2281 bool encrypt) 2282 { 2283 struct crypto_aead *aead = crypto_aead_reqtfm(req); 2284 struct caam_ctx *ctx = crypto_aead_ctx(aead); 2285 struct device *jrdev = ctx->jrdev; 2286 gfp_t flags = (req->base.flags & (CRYPTO_TFM_REQ_MAY_BACKLOG | 2287 CRYPTO_TFM_REQ_MAY_SLEEP)) ? GFP_KERNEL : GFP_ATOMIC; 2288 int src_nents, dst_nents = 0; 2289 struct aead_edesc *edesc; 2290 int sgc; 2291 bool all_contig = true; 2292 int sec4_sg_index, sec4_sg_len = 0, sec4_sg_bytes; 2293 unsigned int authsize = ctx->authsize; 2294 2295 if (unlikely(req->dst != req->src)) { 2296 src_nents = sg_count(req->src, req->assoclen + req->cryptlen); 2297 dst_nents = sg_count(req->dst, 2298 req->assoclen + req->cryptlen + 2299 (encrypt ? authsize : (-authsize))); 2300 } else { 2301 src_nents = sg_count(req->src, 2302 req->assoclen + req->cryptlen + 2303 (encrypt ? authsize : 0)); 2304 } 2305 2306 /* Check if data are contiguous. */ 2307 all_contig = !src_nents; 2308 if (!all_contig) { 2309 src_nents = src_nents ? : 1; 2310 sec4_sg_len = src_nents; 2311 } 2312 2313 sec4_sg_len += dst_nents; 2314 2315 sec4_sg_bytes = sec4_sg_len * sizeof(struct sec4_sg_entry); 2316 2317 /* allocate space for base edesc and hw desc commands, link tables */ 2318 edesc = kzalloc(sizeof(*edesc) + desc_bytes + sec4_sg_bytes, 2319 GFP_DMA | flags); 2320 if (!edesc) { 2321 dev_err(jrdev, "could not allocate extended descriptor\n"); 2322 return ERR_PTR(-ENOMEM); 2323 } 2324 2325 if (likely(req->src == req->dst)) { 2326 sgc = dma_map_sg(jrdev, req->src, src_nents ? : 1, 2327 DMA_BIDIRECTIONAL); 2328 if (unlikely(!sgc)) { 2329 dev_err(jrdev, "unable to map source\n"); 2330 kfree(edesc); 2331 return ERR_PTR(-ENOMEM); 2332 } 2333 } else { 2334 sgc = dma_map_sg(jrdev, req->src, src_nents ? : 1, 2335 DMA_TO_DEVICE); 2336 if (unlikely(!sgc)) { 2337 dev_err(jrdev, "unable to map source\n"); 2338 kfree(edesc); 2339 return ERR_PTR(-ENOMEM); 2340 } 2341 2342 sgc = dma_map_sg(jrdev, req->dst, dst_nents ? : 1, 2343 DMA_FROM_DEVICE); 2344 if (unlikely(!sgc)) { 2345 dev_err(jrdev, "unable to map destination\n"); 2346 dma_unmap_sg(jrdev, req->src, src_nents ? : 1, 2347 DMA_TO_DEVICE); 2348 kfree(edesc); 2349 return ERR_PTR(-ENOMEM); 2350 } 2351 } 2352 2353 edesc->src_nents = src_nents; 2354 edesc->dst_nents = dst_nents; 2355 edesc->sec4_sg = (void *)edesc + sizeof(struct aead_edesc) + 2356 desc_bytes; 2357 *all_contig_ptr = all_contig; 2358 2359 sec4_sg_index = 0; 2360 if (!all_contig) { 2361 sg_to_sec4_sg_last(req->src, src_nents, 2362 edesc->sec4_sg + sec4_sg_index, 0); 2363 sec4_sg_index += src_nents; 2364 } 2365 if (dst_nents) { 2366 sg_to_sec4_sg_last(req->dst, dst_nents, 2367 edesc->sec4_sg + sec4_sg_index, 0); 2368 } 2369 2370 if (!sec4_sg_bytes) 2371 return edesc; 2372 2373 edesc->sec4_sg_dma = dma_map_single(jrdev, edesc->sec4_sg, 2374 sec4_sg_bytes, DMA_TO_DEVICE); 2375 if (dma_mapping_error(jrdev, edesc->sec4_sg_dma)) { 2376 dev_err(jrdev, "unable to map S/G table\n"); 2377 aead_unmap(jrdev, edesc, req); 2378 kfree(edesc); 2379 return ERR_PTR(-ENOMEM); 2380 } 2381 2382 edesc->sec4_sg_bytes = sec4_sg_bytes; 2383 2384 return edesc; 2385 } 2386 2387 static int gcm_encrypt(struct aead_request *req) 2388 { 2389 struct aead_edesc *edesc; 2390 struct crypto_aead *aead = crypto_aead_reqtfm(req); 2391 struct caam_ctx *ctx = crypto_aead_ctx(aead); 2392 struct device *jrdev = ctx->jrdev; 2393 bool all_contig; 2394 u32 *desc; 2395 int ret = 0; 2396 2397 /* allocate extended descriptor */ 2398 edesc = aead_edesc_alloc(req, GCM_DESC_JOB_IO_LEN, &all_contig, true); 2399 if (IS_ERR(edesc)) 2400 return PTR_ERR(edesc); 2401 2402 /* Create and submit job descriptor */ 2403 init_gcm_job(req, edesc, all_contig, true); 2404 #ifdef DEBUG 2405 print_hex_dump(KERN_ERR, "aead jobdesc@"__stringify(__LINE__)": ", 2406 DUMP_PREFIX_ADDRESS, 16, 4, edesc->hw_desc, 2407 desc_bytes(edesc->hw_desc), 1); 2408 #endif 2409 2410 desc = edesc->hw_desc; 2411 ret = caam_jr_enqueue(jrdev, desc, aead_encrypt_done, req); 2412 if (!ret) { 2413 ret = -EINPROGRESS; 2414 } else { 2415 aead_unmap(jrdev, edesc, req); 2416 kfree(edesc); 2417 } 2418 2419 return ret; 2420 } 2421 2422 static int ipsec_gcm_encrypt(struct aead_request *req) 2423 { 2424 if (req->assoclen < 8) 2425 return -EINVAL; 2426 2427 return gcm_encrypt(req); 2428 } 2429 2430 static int aead_encrypt(struct aead_request *req) 2431 { 2432 struct aead_edesc *edesc; 2433 struct crypto_aead *aead = crypto_aead_reqtfm(req); 2434 struct caam_ctx *ctx = crypto_aead_ctx(aead); 2435 struct device *jrdev = ctx->jrdev; 2436 bool all_contig; 2437 u32 *desc; 2438 int ret = 0; 2439 2440 /* allocate extended descriptor */ 2441 edesc = aead_edesc_alloc(req, AUTHENC_DESC_JOB_IO_LEN, 2442 &all_contig, true); 2443 if (IS_ERR(edesc)) 2444 return PTR_ERR(edesc); 2445 2446 /* Create and submit job descriptor */ 2447 init_authenc_job(req, edesc, all_contig, true); 2448 #ifdef DEBUG 2449 print_hex_dump(KERN_ERR, "aead jobdesc@"__stringify(__LINE__)": ", 2450 DUMP_PREFIX_ADDRESS, 16, 4, edesc->hw_desc, 2451 desc_bytes(edesc->hw_desc), 1); 2452 #endif 2453 2454 desc = edesc->hw_desc; 2455 ret = caam_jr_enqueue(jrdev, desc, aead_encrypt_done, req); 2456 if (!ret) { 2457 ret = -EINPROGRESS; 2458 } else { 2459 aead_unmap(jrdev, edesc, req); 2460 kfree(edesc); 2461 } 2462 2463 return ret; 2464 } 2465 2466 static int gcm_decrypt(struct aead_request *req) 2467 { 2468 struct aead_edesc *edesc; 2469 struct crypto_aead *aead = crypto_aead_reqtfm(req); 2470 struct caam_ctx *ctx = crypto_aead_ctx(aead); 2471 struct device *jrdev = ctx->jrdev; 2472 bool all_contig; 2473 u32 *desc; 2474 int ret = 0; 2475 2476 /* allocate extended descriptor */ 2477 edesc = aead_edesc_alloc(req, GCM_DESC_JOB_IO_LEN, &all_contig, false); 2478 if (IS_ERR(edesc)) 2479 return PTR_ERR(edesc); 2480 2481 /* Create and submit job descriptor*/ 2482 init_gcm_job(req, edesc, all_contig, false); 2483 #ifdef DEBUG 2484 print_hex_dump(KERN_ERR, "aead jobdesc@"__stringify(__LINE__)": ", 2485 DUMP_PREFIX_ADDRESS, 16, 4, edesc->hw_desc, 2486 desc_bytes(edesc->hw_desc), 1); 2487 #endif 2488 2489 desc = edesc->hw_desc; 2490 ret = caam_jr_enqueue(jrdev, desc, aead_decrypt_done, req); 2491 if (!ret) { 2492 ret = -EINPROGRESS; 2493 } else { 2494 aead_unmap(jrdev, edesc, req); 2495 kfree(edesc); 2496 } 2497 2498 return ret; 2499 } 2500 2501 static int ipsec_gcm_decrypt(struct aead_request *req) 2502 { 2503 if (req->assoclen < 8) 2504 return -EINVAL; 2505 2506 return gcm_decrypt(req); 2507 } 2508 2509 static int aead_decrypt(struct aead_request *req) 2510 { 2511 struct aead_edesc *edesc; 2512 struct crypto_aead *aead = crypto_aead_reqtfm(req); 2513 struct caam_ctx *ctx = crypto_aead_ctx(aead); 2514 struct device *jrdev = ctx->jrdev; 2515 bool all_contig; 2516 u32 *desc; 2517 int ret = 0; 2518 2519 /* allocate extended descriptor */ 2520 edesc = aead_edesc_alloc(req, AUTHENC_DESC_JOB_IO_LEN, 2521 &all_contig, false); 2522 if (IS_ERR(edesc)) 2523 return PTR_ERR(edesc); 2524 2525 #ifdef DEBUG 2526 print_hex_dump(KERN_ERR, "dec src@"__stringify(__LINE__)": ", 2527 DUMP_PREFIX_ADDRESS, 16, 4, sg_virt(req->src), 2528 req->assoclen + req->cryptlen, 1); 2529 #endif 2530 2531 /* Create and submit job descriptor*/ 2532 init_authenc_job(req, edesc, all_contig, false); 2533 #ifdef DEBUG 2534 print_hex_dump(KERN_ERR, "aead jobdesc@"__stringify(__LINE__)": ", 2535 DUMP_PREFIX_ADDRESS, 16, 4, edesc->hw_desc, 2536 desc_bytes(edesc->hw_desc), 1); 2537 #endif 2538 2539 desc = edesc->hw_desc; 2540 ret = caam_jr_enqueue(jrdev, desc, aead_decrypt_done, req); 2541 if (!ret) { 2542 ret = -EINPROGRESS; 2543 } else { 2544 aead_unmap(jrdev, edesc, req); 2545 kfree(edesc); 2546 } 2547 2548 return ret; 2549 } 2550 2551 /* 2552 * allocate and map the ablkcipher extended descriptor for ablkcipher 2553 */ 2554 static struct ablkcipher_edesc *ablkcipher_edesc_alloc(struct ablkcipher_request 2555 *req, int desc_bytes, 2556 bool *iv_contig_out) 2557 { 2558 struct crypto_ablkcipher *ablkcipher = crypto_ablkcipher_reqtfm(req); 2559 struct caam_ctx *ctx = crypto_ablkcipher_ctx(ablkcipher); 2560 struct device *jrdev = ctx->jrdev; 2561 gfp_t flags = (req->base.flags & (CRYPTO_TFM_REQ_MAY_BACKLOG | 2562 CRYPTO_TFM_REQ_MAY_SLEEP)) ? 2563 GFP_KERNEL : GFP_ATOMIC; 2564 int src_nents, dst_nents = 0, sec4_sg_bytes; 2565 struct ablkcipher_edesc *edesc; 2566 dma_addr_t iv_dma = 0; 2567 bool iv_contig = false; 2568 int sgc; 2569 int ivsize = crypto_ablkcipher_ivsize(ablkcipher); 2570 int sec4_sg_index; 2571 2572 src_nents = sg_count(req->src, req->nbytes); 2573 2574 if (req->dst != req->src) 2575 dst_nents = sg_count(req->dst, req->nbytes); 2576 2577 if (likely(req->src == req->dst)) { 2578 sgc = dma_map_sg(jrdev, req->src, src_nents ? : 1, 2579 DMA_BIDIRECTIONAL); 2580 } else { 2581 sgc = dma_map_sg(jrdev, req->src, src_nents ? : 1, 2582 DMA_TO_DEVICE); 2583 sgc = dma_map_sg(jrdev, req->dst, dst_nents ? : 1, 2584 DMA_FROM_DEVICE); 2585 } 2586 2587 iv_dma = dma_map_single(jrdev, req->info, ivsize, DMA_TO_DEVICE); 2588 if (dma_mapping_error(jrdev, iv_dma)) { 2589 dev_err(jrdev, "unable to map IV\n"); 2590 return ERR_PTR(-ENOMEM); 2591 } 2592 2593 /* 2594 * Check if iv can be contiguous with source and destination. 2595 * If so, include it. If not, create scatterlist. 2596 */ 2597 if (!src_nents && iv_dma + ivsize == sg_dma_address(req->src)) 2598 iv_contig = true; 2599 else 2600 src_nents = src_nents ? : 1; 2601 sec4_sg_bytes = ((iv_contig ? 0 : 1) + src_nents + dst_nents) * 2602 sizeof(struct sec4_sg_entry); 2603 2604 /* allocate space for base edesc and hw desc commands, link tables */ 2605 edesc = kzalloc(sizeof(*edesc) + desc_bytes + sec4_sg_bytes, 2606 GFP_DMA | flags); 2607 if (!edesc) { 2608 dev_err(jrdev, "could not allocate extended descriptor\n"); 2609 return ERR_PTR(-ENOMEM); 2610 } 2611 2612 edesc->src_nents = src_nents; 2613 edesc->dst_nents = dst_nents; 2614 edesc->sec4_sg_bytes = sec4_sg_bytes; 2615 edesc->sec4_sg = (void *)edesc + sizeof(struct ablkcipher_edesc) + 2616 desc_bytes; 2617 2618 sec4_sg_index = 0; 2619 if (!iv_contig) { 2620 dma_to_sec4_sg_one(edesc->sec4_sg, iv_dma, ivsize, 0); 2621 sg_to_sec4_sg_last(req->src, src_nents, 2622 edesc->sec4_sg + 1, 0); 2623 sec4_sg_index += 1 + src_nents; 2624 } 2625 2626 if (dst_nents) { 2627 sg_to_sec4_sg_last(req->dst, dst_nents, 2628 edesc->sec4_sg + sec4_sg_index, 0); 2629 } 2630 2631 edesc->sec4_sg_dma = dma_map_single(jrdev, edesc->sec4_sg, 2632 sec4_sg_bytes, DMA_TO_DEVICE); 2633 if (dma_mapping_error(jrdev, edesc->sec4_sg_dma)) { 2634 dev_err(jrdev, "unable to map S/G table\n"); 2635 return ERR_PTR(-ENOMEM); 2636 } 2637 2638 edesc->iv_dma = iv_dma; 2639 2640 #ifdef DEBUG 2641 print_hex_dump(KERN_ERR, "ablkcipher sec4_sg@"__stringify(__LINE__)": ", 2642 DUMP_PREFIX_ADDRESS, 16, 4, edesc->sec4_sg, 2643 sec4_sg_bytes, 1); 2644 #endif 2645 2646 *iv_contig_out = iv_contig; 2647 return edesc; 2648 } 2649 2650 static int ablkcipher_encrypt(struct ablkcipher_request *req) 2651 { 2652 struct ablkcipher_edesc *edesc; 2653 struct crypto_ablkcipher *ablkcipher = crypto_ablkcipher_reqtfm(req); 2654 struct caam_ctx *ctx = crypto_ablkcipher_ctx(ablkcipher); 2655 struct device *jrdev = ctx->jrdev; 2656 bool iv_contig; 2657 u32 *desc; 2658 int ret = 0; 2659 2660 /* allocate extended descriptor */ 2661 edesc = ablkcipher_edesc_alloc(req, DESC_JOB_IO_LEN * 2662 CAAM_CMD_SZ, &iv_contig); 2663 if (IS_ERR(edesc)) 2664 return PTR_ERR(edesc); 2665 2666 /* Create and submit job descriptor*/ 2667 init_ablkcipher_job(ctx->sh_desc_enc, 2668 ctx->sh_desc_enc_dma, edesc, req, iv_contig); 2669 #ifdef DEBUG 2670 print_hex_dump(KERN_ERR, "ablkcipher jobdesc@"__stringify(__LINE__)": ", 2671 DUMP_PREFIX_ADDRESS, 16, 4, edesc->hw_desc, 2672 desc_bytes(edesc->hw_desc), 1); 2673 #endif 2674 desc = edesc->hw_desc; 2675 ret = caam_jr_enqueue(jrdev, desc, ablkcipher_encrypt_done, req); 2676 2677 if (!ret) { 2678 ret = -EINPROGRESS; 2679 } else { 2680 ablkcipher_unmap(jrdev, edesc, req); 2681 kfree(edesc); 2682 } 2683 2684 return ret; 2685 } 2686 2687 static int ablkcipher_decrypt(struct ablkcipher_request *req) 2688 { 2689 struct ablkcipher_edesc *edesc; 2690 struct crypto_ablkcipher *ablkcipher = crypto_ablkcipher_reqtfm(req); 2691 struct caam_ctx *ctx = crypto_ablkcipher_ctx(ablkcipher); 2692 struct device *jrdev = ctx->jrdev; 2693 bool iv_contig; 2694 u32 *desc; 2695 int ret = 0; 2696 2697 /* allocate extended descriptor */ 2698 edesc = ablkcipher_edesc_alloc(req, DESC_JOB_IO_LEN * 2699 CAAM_CMD_SZ, &iv_contig); 2700 if (IS_ERR(edesc)) 2701 return PTR_ERR(edesc); 2702 2703 /* Create and submit job descriptor*/ 2704 init_ablkcipher_job(ctx->sh_desc_dec, 2705 ctx->sh_desc_dec_dma, edesc, req, iv_contig); 2706 desc = edesc->hw_desc; 2707 #ifdef DEBUG 2708 print_hex_dump(KERN_ERR, "ablkcipher jobdesc@"__stringify(__LINE__)": ", 2709 DUMP_PREFIX_ADDRESS, 16, 4, edesc->hw_desc, 2710 desc_bytes(edesc->hw_desc), 1); 2711 #endif 2712 2713 ret = caam_jr_enqueue(jrdev, desc, ablkcipher_decrypt_done, req); 2714 if (!ret) { 2715 ret = -EINPROGRESS; 2716 } else { 2717 ablkcipher_unmap(jrdev, edesc, req); 2718 kfree(edesc); 2719 } 2720 2721 return ret; 2722 } 2723 2724 /* 2725 * allocate and map the ablkcipher extended descriptor 2726 * for ablkcipher givencrypt 2727 */ 2728 static struct ablkcipher_edesc *ablkcipher_giv_edesc_alloc( 2729 struct skcipher_givcrypt_request *greq, 2730 int desc_bytes, 2731 bool *iv_contig_out) 2732 { 2733 struct ablkcipher_request *req = &greq->creq; 2734 struct crypto_ablkcipher *ablkcipher = crypto_ablkcipher_reqtfm(req); 2735 struct caam_ctx *ctx = crypto_ablkcipher_ctx(ablkcipher); 2736 struct device *jrdev = ctx->jrdev; 2737 gfp_t flags = (req->base.flags & (CRYPTO_TFM_REQ_MAY_BACKLOG | 2738 CRYPTO_TFM_REQ_MAY_SLEEP)) ? 2739 GFP_KERNEL : GFP_ATOMIC; 2740 int src_nents, dst_nents = 0, sec4_sg_bytes; 2741 struct ablkcipher_edesc *edesc; 2742 dma_addr_t iv_dma = 0; 2743 bool iv_contig = false; 2744 int sgc; 2745 int ivsize = crypto_ablkcipher_ivsize(ablkcipher); 2746 int sec4_sg_index; 2747 2748 src_nents = sg_count(req->src, req->nbytes); 2749 2750 if (unlikely(req->dst != req->src)) 2751 dst_nents = sg_count(req->dst, req->nbytes); 2752 2753 if (likely(req->src == req->dst)) { 2754 sgc = dma_map_sg(jrdev, req->src, src_nents ? : 1, 2755 DMA_BIDIRECTIONAL); 2756 } else { 2757 sgc = dma_map_sg(jrdev, req->src, src_nents ? : 1, 2758 DMA_TO_DEVICE); 2759 sgc = dma_map_sg(jrdev, req->dst, dst_nents ? : 1, 2760 DMA_FROM_DEVICE); 2761 } 2762 2763 /* 2764 * Check if iv can be contiguous with source and destination. 2765 * If so, include it. If not, create scatterlist. 2766 */ 2767 iv_dma = dma_map_single(jrdev, greq->giv, ivsize, DMA_TO_DEVICE); 2768 if (dma_mapping_error(jrdev, iv_dma)) { 2769 dev_err(jrdev, "unable to map IV\n"); 2770 return ERR_PTR(-ENOMEM); 2771 } 2772 2773 if (!dst_nents && iv_dma + ivsize == sg_dma_address(req->dst)) 2774 iv_contig = true; 2775 else 2776 dst_nents = dst_nents ? : 1; 2777 sec4_sg_bytes = ((iv_contig ? 0 : 1) + src_nents + dst_nents) * 2778 sizeof(struct sec4_sg_entry); 2779 2780 /* allocate space for base edesc and hw desc commands, link tables */ 2781 edesc = kzalloc(sizeof(*edesc) + desc_bytes + sec4_sg_bytes, 2782 GFP_DMA | flags); 2783 if (!edesc) { 2784 dev_err(jrdev, "could not allocate extended descriptor\n"); 2785 return ERR_PTR(-ENOMEM); 2786 } 2787 2788 edesc->src_nents = src_nents; 2789 edesc->dst_nents = dst_nents; 2790 edesc->sec4_sg_bytes = sec4_sg_bytes; 2791 edesc->sec4_sg = (void *)edesc + sizeof(struct ablkcipher_edesc) + 2792 desc_bytes; 2793 2794 sec4_sg_index = 0; 2795 if (src_nents) { 2796 sg_to_sec4_sg_last(req->src, src_nents, edesc->sec4_sg, 0); 2797 sec4_sg_index += src_nents; 2798 } 2799 2800 if (!iv_contig) { 2801 dma_to_sec4_sg_one(edesc->sec4_sg + sec4_sg_index, 2802 iv_dma, ivsize, 0); 2803 sec4_sg_index += 1; 2804 sg_to_sec4_sg_last(req->dst, dst_nents, 2805 edesc->sec4_sg + sec4_sg_index, 0); 2806 } 2807 2808 edesc->sec4_sg_dma = dma_map_single(jrdev, edesc->sec4_sg, 2809 sec4_sg_bytes, DMA_TO_DEVICE); 2810 if (dma_mapping_error(jrdev, edesc->sec4_sg_dma)) { 2811 dev_err(jrdev, "unable to map S/G table\n"); 2812 return ERR_PTR(-ENOMEM); 2813 } 2814 edesc->iv_dma = iv_dma; 2815 2816 #ifdef DEBUG 2817 print_hex_dump(KERN_ERR, 2818 "ablkcipher sec4_sg@" __stringify(__LINE__) ": ", 2819 DUMP_PREFIX_ADDRESS, 16, 4, edesc->sec4_sg, 2820 sec4_sg_bytes, 1); 2821 #endif 2822 2823 *iv_contig_out = iv_contig; 2824 return edesc; 2825 } 2826 2827 static int ablkcipher_givencrypt(struct skcipher_givcrypt_request *creq) 2828 { 2829 struct ablkcipher_request *req = &creq->creq; 2830 struct ablkcipher_edesc *edesc; 2831 struct crypto_ablkcipher *ablkcipher = crypto_ablkcipher_reqtfm(req); 2832 struct caam_ctx *ctx = crypto_ablkcipher_ctx(ablkcipher); 2833 struct device *jrdev = ctx->jrdev; 2834 bool iv_contig; 2835 u32 *desc; 2836 int ret = 0; 2837 2838 /* allocate extended descriptor */ 2839 edesc = ablkcipher_giv_edesc_alloc(creq, DESC_JOB_IO_LEN * 2840 CAAM_CMD_SZ, &iv_contig); 2841 if (IS_ERR(edesc)) 2842 return PTR_ERR(edesc); 2843 2844 /* Create and submit job descriptor*/ 2845 init_ablkcipher_giv_job(ctx->sh_desc_givenc, ctx->sh_desc_givenc_dma, 2846 edesc, req, iv_contig); 2847 #ifdef DEBUG 2848 print_hex_dump(KERN_ERR, 2849 "ablkcipher jobdesc@" __stringify(__LINE__) ": ", 2850 DUMP_PREFIX_ADDRESS, 16, 4, edesc->hw_desc, 2851 desc_bytes(edesc->hw_desc), 1); 2852 #endif 2853 desc = edesc->hw_desc; 2854 ret = caam_jr_enqueue(jrdev, desc, ablkcipher_encrypt_done, req); 2855 2856 if (!ret) { 2857 ret = -EINPROGRESS; 2858 } else { 2859 ablkcipher_unmap(jrdev, edesc, req); 2860 kfree(edesc); 2861 } 2862 2863 return ret; 2864 } 2865 2866 #define template_aead template_u.aead 2867 #define template_ablkcipher template_u.ablkcipher 2868 struct caam_alg_template { 2869 char name[CRYPTO_MAX_ALG_NAME]; 2870 char driver_name[CRYPTO_MAX_ALG_NAME]; 2871 unsigned int blocksize; 2872 u32 type; 2873 union { 2874 struct ablkcipher_alg ablkcipher; 2875 } template_u; 2876 u32 class1_alg_type; 2877 u32 class2_alg_type; 2878 u32 alg_op; 2879 }; 2880 2881 static struct caam_alg_template driver_algs[] = { 2882 /* ablkcipher descriptor */ 2883 { 2884 .name = "cbc(aes)", 2885 .driver_name = "cbc-aes-caam", 2886 .blocksize = AES_BLOCK_SIZE, 2887 .type = CRYPTO_ALG_TYPE_GIVCIPHER, 2888 .template_ablkcipher = { 2889 .setkey = ablkcipher_setkey, 2890 .encrypt = ablkcipher_encrypt, 2891 .decrypt = ablkcipher_decrypt, 2892 .givencrypt = ablkcipher_givencrypt, 2893 .geniv = "<built-in>", 2894 .min_keysize = AES_MIN_KEY_SIZE, 2895 .max_keysize = AES_MAX_KEY_SIZE, 2896 .ivsize = AES_BLOCK_SIZE, 2897 }, 2898 .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CBC, 2899 }, 2900 { 2901 .name = "cbc(des3_ede)", 2902 .driver_name = "cbc-3des-caam", 2903 .blocksize = DES3_EDE_BLOCK_SIZE, 2904 .type = CRYPTO_ALG_TYPE_GIVCIPHER, 2905 .template_ablkcipher = { 2906 .setkey = ablkcipher_setkey, 2907 .encrypt = ablkcipher_encrypt, 2908 .decrypt = ablkcipher_decrypt, 2909 .givencrypt = ablkcipher_givencrypt, 2910 .geniv = "<built-in>", 2911 .min_keysize = DES3_EDE_KEY_SIZE, 2912 .max_keysize = DES3_EDE_KEY_SIZE, 2913 .ivsize = DES3_EDE_BLOCK_SIZE, 2914 }, 2915 .class1_alg_type = OP_ALG_ALGSEL_3DES | OP_ALG_AAI_CBC, 2916 }, 2917 { 2918 .name = "cbc(des)", 2919 .driver_name = "cbc-des-caam", 2920 .blocksize = DES_BLOCK_SIZE, 2921 .type = CRYPTO_ALG_TYPE_GIVCIPHER, 2922 .template_ablkcipher = { 2923 .setkey = ablkcipher_setkey, 2924 .encrypt = ablkcipher_encrypt, 2925 .decrypt = ablkcipher_decrypt, 2926 .givencrypt = ablkcipher_givencrypt, 2927 .geniv = "<built-in>", 2928 .min_keysize = DES_KEY_SIZE, 2929 .max_keysize = DES_KEY_SIZE, 2930 .ivsize = DES_BLOCK_SIZE, 2931 }, 2932 .class1_alg_type = OP_ALG_ALGSEL_DES | OP_ALG_AAI_CBC, 2933 }, 2934 { 2935 .name = "ctr(aes)", 2936 .driver_name = "ctr-aes-caam", 2937 .blocksize = 1, 2938 .type = CRYPTO_ALG_TYPE_ABLKCIPHER, 2939 .template_ablkcipher = { 2940 .setkey = ablkcipher_setkey, 2941 .encrypt = ablkcipher_encrypt, 2942 .decrypt = ablkcipher_decrypt, 2943 .geniv = "chainiv", 2944 .min_keysize = AES_MIN_KEY_SIZE, 2945 .max_keysize = AES_MAX_KEY_SIZE, 2946 .ivsize = AES_BLOCK_SIZE, 2947 }, 2948 .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CTR_MOD128, 2949 }, 2950 { 2951 .name = "rfc3686(ctr(aes))", 2952 .driver_name = "rfc3686-ctr-aes-caam", 2953 .blocksize = 1, 2954 .type = CRYPTO_ALG_TYPE_GIVCIPHER, 2955 .template_ablkcipher = { 2956 .setkey = ablkcipher_setkey, 2957 .encrypt = ablkcipher_encrypt, 2958 .decrypt = ablkcipher_decrypt, 2959 .givencrypt = ablkcipher_givencrypt, 2960 .geniv = "<built-in>", 2961 .min_keysize = AES_MIN_KEY_SIZE + 2962 CTR_RFC3686_NONCE_SIZE, 2963 .max_keysize = AES_MAX_KEY_SIZE + 2964 CTR_RFC3686_NONCE_SIZE, 2965 .ivsize = CTR_RFC3686_IV_SIZE, 2966 }, 2967 .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CTR_MOD128, 2968 }, 2969 { 2970 .name = "xts(aes)", 2971 .driver_name = "xts-aes-caam", 2972 .blocksize = AES_BLOCK_SIZE, 2973 .type = CRYPTO_ALG_TYPE_ABLKCIPHER, 2974 .template_ablkcipher = { 2975 .setkey = xts_ablkcipher_setkey, 2976 .encrypt = ablkcipher_encrypt, 2977 .decrypt = ablkcipher_decrypt, 2978 .geniv = "eseqiv", 2979 .min_keysize = 2 * AES_MIN_KEY_SIZE, 2980 .max_keysize = 2 * AES_MAX_KEY_SIZE, 2981 .ivsize = AES_BLOCK_SIZE, 2982 }, 2983 .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_XTS, 2984 }, 2985 }; 2986 2987 static struct caam_aead_alg driver_aeads[] = { 2988 { 2989 .aead = { 2990 .base = { 2991 .cra_name = "rfc4106(gcm(aes))", 2992 .cra_driver_name = "rfc4106-gcm-aes-caam", 2993 .cra_blocksize = 1, 2994 }, 2995 .setkey = rfc4106_setkey, 2996 .setauthsize = rfc4106_setauthsize, 2997 .encrypt = ipsec_gcm_encrypt, 2998 .decrypt = ipsec_gcm_decrypt, 2999 .ivsize = 8, 3000 .maxauthsize = AES_BLOCK_SIZE, 3001 }, 3002 .caam = { 3003 .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_GCM, 3004 }, 3005 }, 3006 { 3007 .aead = { 3008 .base = { 3009 .cra_name = "rfc4543(gcm(aes))", 3010 .cra_driver_name = "rfc4543-gcm-aes-caam", 3011 .cra_blocksize = 1, 3012 }, 3013 .setkey = rfc4543_setkey, 3014 .setauthsize = rfc4543_setauthsize, 3015 .encrypt = ipsec_gcm_encrypt, 3016 .decrypt = ipsec_gcm_decrypt, 3017 .ivsize = 8, 3018 .maxauthsize = AES_BLOCK_SIZE, 3019 }, 3020 .caam = { 3021 .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_GCM, 3022 }, 3023 }, 3024 /* Galois Counter Mode */ 3025 { 3026 .aead = { 3027 .base = { 3028 .cra_name = "gcm(aes)", 3029 .cra_driver_name = "gcm-aes-caam", 3030 .cra_blocksize = 1, 3031 }, 3032 .setkey = gcm_setkey, 3033 .setauthsize = gcm_setauthsize, 3034 .encrypt = gcm_encrypt, 3035 .decrypt = gcm_decrypt, 3036 .ivsize = 12, 3037 .maxauthsize = AES_BLOCK_SIZE, 3038 }, 3039 .caam = { 3040 .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_GCM, 3041 }, 3042 }, 3043 /* single-pass ipsec_esp descriptor */ 3044 { 3045 .aead = { 3046 .base = { 3047 .cra_name = "authenc(hmac(md5)," 3048 "ecb(cipher_null))", 3049 .cra_driver_name = "authenc-hmac-md5-" 3050 "ecb-cipher_null-caam", 3051 .cra_blocksize = NULL_BLOCK_SIZE, 3052 }, 3053 .setkey = aead_setkey, 3054 .setauthsize = aead_setauthsize, 3055 .encrypt = aead_encrypt, 3056 .decrypt = aead_decrypt, 3057 .ivsize = NULL_IV_SIZE, 3058 .maxauthsize = MD5_DIGEST_SIZE, 3059 }, 3060 .caam = { 3061 .class2_alg_type = OP_ALG_ALGSEL_MD5 | 3062 OP_ALG_AAI_HMAC_PRECOMP, 3063 .alg_op = OP_ALG_ALGSEL_MD5 | OP_ALG_AAI_HMAC, 3064 }, 3065 }, 3066 { 3067 .aead = { 3068 .base = { 3069 .cra_name = "authenc(hmac(sha1)," 3070 "ecb(cipher_null))", 3071 .cra_driver_name = "authenc-hmac-sha1-" 3072 "ecb-cipher_null-caam", 3073 .cra_blocksize = NULL_BLOCK_SIZE, 3074 }, 3075 .setkey = aead_setkey, 3076 .setauthsize = aead_setauthsize, 3077 .encrypt = aead_encrypt, 3078 .decrypt = aead_decrypt, 3079 .ivsize = NULL_IV_SIZE, 3080 .maxauthsize = SHA1_DIGEST_SIZE, 3081 }, 3082 .caam = { 3083 .class2_alg_type = OP_ALG_ALGSEL_SHA1 | 3084 OP_ALG_AAI_HMAC_PRECOMP, 3085 .alg_op = OP_ALG_ALGSEL_SHA1 | OP_ALG_AAI_HMAC, 3086 }, 3087 }, 3088 { 3089 .aead = { 3090 .base = { 3091 .cra_name = "authenc(hmac(sha224)," 3092 "ecb(cipher_null))", 3093 .cra_driver_name = "authenc-hmac-sha224-" 3094 "ecb-cipher_null-caam", 3095 .cra_blocksize = NULL_BLOCK_SIZE, 3096 }, 3097 .setkey = aead_setkey, 3098 .setauthsize = aead_setauthsize, 3099 .encrypt = aead_encrypt, 3100 .decrypt = aead_decrypt, 3101 .ivsize = NULL_IV_SIZE, 3102 .maxauthsize = SHA224_DIGEST_SIZE, 3103 }, 3104 .caam = { 3105 .class2_alg_type = OP_ALG_ALGSEL_SHA224 | 3106 OP_ALG_AAI_HMAC_PRECOMP, 3107 .alg_op = OP_ALG_ALGSEL_SHA224 | OP_ALG_AAI_HMAC, 3108 }, 3109 }, 3110 { 3111 .aead = { 3112 .base = { 3113 .cra_name = "authenc(hmac(sha256)," 3114 "ecb(cipher_null))", 3115 .cra_driver_name = "authenc-hmac-sha256-" 3116 "ecb-cipher_null-caam", 3117 .cra_blocksize = NULL_BLOCK_SIZE, 3118 }, 3119 .setkey = aead_setkey, 3120 .setauthsize = aead_setauthsize, 3121 .encrypt = aead_encrypt, 3122 .decrypt = aead_decrypt, 3123 .ivsize = NULL_IV_SIZE, 3124 .maxauthsize = SHA256_DIGEST_SIZE, 3125 }, 3126 .caam = { 3127 .class2_alg_type = OP_ALG_ALGSEL_SHA256 | 3128 OP_ALG_AAI_HMAC_PRECOMP, 3129 .alg_op = OP_ALG_ALGSEL_SHA256 | OP_ALG_AAI_HMAC, 3130 }, 3131 }, 3132 { 3133 .aead = { 3134 .base = { 3135 .cra_name = "authenc(hmac(sha384)," 3136 "ecb(cipher_null))", 3137 .cra_driver_name = "authenc-hmac-sha384-" 3138 "ecb-cipher_null-caam", 3139 .cra_blocksize = NULL_BLOCK_SIZE, 3140 }, 3141 .setkey = aead_setkey, 3142 .setauthsize = aead_setauthsize, 3143 .encrypt = aead_encrypt, 3144 .decrypt = aead_decrypt, 3145 .ivsize = NULL_IV_SIZE, 3146 .maxauthsize = SHA384_DIGEST_SIZE, 3147 }, 3148 .caam = { 3149 .class2_alg_type = OP_ALG_ALGSEL_SHA384 | 3150 OP_ALG_AAI_HMAC_PRECOMP, 3151 .alg_op = OP_ALG_ALGSEL_SHA384 | OP_ALG_AAI_HMAC, 3152 }, 3153 }, 3154 { 3155 .aead = { 3156 .base = { 3157 .cra_name = "authenc(hmac(sha512)," 3158 "ecb(cipher_null))", 3159 .cra_driver_name = "authenc-hmac-sha512-" 3160 "ecb-cipher_null-caam", 3161 .cra_blocksize = NULL_BLOCK_SIZE, 3162 }, 3163 .setkey = aead_setkey, 3164 .setauthsize = aead_setauthsize, 3165 .encrypt = aead_encrypt, 3166 .decrypt = aead_decrypt, 3167 .ivsize = NULL_IV_SIZE, 3168 .maxauthsize = SHA512_DIGEST_SIZE, 3169 }, 3170 .caam = { 3171 .class2_alg_type = OP_ALG_ALGSEL_SHA512 | 3172 OP_ALG_AAI_HMAC_PRECOMP, 3173 .alg_op = OP_ALG_ALGSEL_SHA512 | OP_ALG_AAI_HMAC, 3174 }, 3175 }, 3176 { 3177 .aead = { 3178 .base = { 3179 .cra_name = "authenc(hmac(md5),cbc(aes))", 3180 .cra_driver_name = "authenc-hmac-md5-" 3181 "cbc-aes-caam", 3182 .cra_blocksize = AES_BLOCK_SIZE, 3183 }, 3184 .setkey = aead_setkey, 3185 .setauthsize = aead_setauthsize, 3186 .encrypt = aead_encrypt, 3187 .decrypt = aead_decrypt, 3188 .ivsize = AES_BLOCK_SIZE, 3189 .maxauthsize = MD5_DIGEST_SIZE, 3190 }, 3191 .caam = { 3192 .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CBC, 3193 .class2_alg_type = OP_ALG_ALGSEL_MD5 | 3194 OP_ALG_AAI_HMAC_PRECOMP, 3195 .alg_op = OP_ALG_ALGSEL_MD5 | OP_ALG_AAI_HMAC, 3196 }, 3197 }, 3198 { 3199 .aead = { 3200 .base = { 3201 .cra_name = "echainiv(authenc(hmac(md5)," 3202 "cbc(aes)))", 3203 .cra_driver_name = "echainiv-authenc-hmac-md5-" 3204 "cbc-aes-caam", 3205 .cra_blocksize = AES_BLOCK_SIZE, 3206 }, 3207 .setkey = aead_setkey, 3208 .setauthsize = aead_setauthsize, 3209 .encrypt = aead_encrypt, 3210 .decrypt = aead_decrypt, 3211 .ivsize = AES_BLOCK_SIZE, 3212 .maxauthsize = MD5_DIGEST_SIZE, 3213 }, 3214 .caam = { 3215 .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CBC, 3216 .class2_alg_type = OP_ALG_ALGSEL_MD5 | 3217 OP_ALG_AAI_HMAC_PRECOMP, 3218 .alg_op = OP_ALG_ALGSEL_MD5 | OP_ALG_AAI_HMAC, 3219 .geniv = true, 3220 }, 3221 }, 3222 { 3223 .aead = { 3224 .base = { 3225 .cra_name = "authenc(hmac(sha1),cbc(aes))", 3226 .cra_driver_name = "authenc-hmac-sha1-" 3227 "cbc-aes-caam", 3228 .cra_blocksize = AES_BLOCK_SIZE, 3229 }, 3230 .setkey = aead_setkey, 3231 .setauthsize = aead_setauthsize, 3232 .encrypt = aead_encrypt, 3233 .decrypt = aead_decrypt, 3234 .ivsize = AES_BLOCK_SIZE, 3235 .maxauthsize = SHA1_DIGEST_SIZE, 3236 }, 3237 .caam = { 3238 .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CBC, 3239 .class2_alg_type = OP_ALG_ALGSEL_SHA1 | 3240 OP_ALG_AAI_HMAC_PRECOMP, 3241 .alg_op = OP_ALG_ALGSEL_SHA1 | OP_ALG_AAI_HMAC, 3242 }, 3243 }, 3244 { 3245 .aead = { 3246 .base = { 3247 .cra_name = "echainiv(authenc(hmac(sha1)," 3248 "cbc(aes)))", 3249 .cra_driver_name = "echainiv-authenc-" 3250 "hmac-sha1-cbc-aes-caam", 3251 .cra_blocksize = AES_BLOCK_SIZE, 3252 }, 3253 .setkey = aead_setkey, 3254 .setauthsize = aead_setauthsize, 3255 .encrypt = aead_encrypt, 3256 .decrypt = aead_decrypt, 3257 .ivsize = AES_BLOCK_SIZE, 3258 .maxauthsize = SHA1_DIGEST_SIZE, 3259 }, 3260 .caam = { 3261 .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CBC, 3262 .class2_alg_type = OP_ALG_ALGSEL_SHA1 | 3263 OP_ALG_AAI_HMAC_PRECOMP, 3264 .alg_op = OP_ALG_ALGSEL_SHA1 | OP_ALG_AAI_HMAC, 3265 .geniv = true, 3266 }, 3267 }, 3268 { 3269 .aead = { 3270 .base = { 3271 .cra_name = "authenc(hmac(sha224),cbc(aes))", 3272 .cra_driver_name = "authenc-hmac-sha224-" 3273 "cbc-aes-caam", 3274 .cra_blocksize = AES_BLOCK_SIZE, 3275 }, 3276 .setkey = aead_setkey, 3277 .setauthsize = aead_setauthsize, 3278 .encrypt = aead_encrypt, 3279 .decrypt = aead_decrypt, 3280 .ivsize = AES_BLOCK_SIZE, 3281 .maxauthsize = SHA224_DIGEST_SIZE, 3282 }, 3283 .caam = { 3284 .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CBC, 3285 .class2_alg_type = OP_ALG_ALGSEL_SHA224 | 3286 OP_ALG_AAI_HMAC_PRECOMP, 3287 .alg_op = OP_ALG_ALGSEL_SHA224 | OP_ALG_AAI_HMAC, 3288 }, 3289 }, 3290 { 3291 .aead = { 3292 .base = { 3293 .cra_name = "echainiv(authenc(hmac(sha224)," 3294 "cbc(aes)))", 3295 .cra_driver_name = "echainiv-authenc-" 3296 "hmac-sha224-cbc-aes-caam", 3297 .cra_blocksize = AES_BLOCK_SIZE, 3298 }, 3299 .setkey = aead_setkey, 3300 .setauthsize = aead_setauthsize, 3301 .encrypt = aead_encrypt, 3302 .decrypt = aead_decrypt, 3303 .ivsize = AES_BLOCK_SIZE, 3304 .maxauthsize = SHA224_DIGEST_SIZE, 3305 }, 3306 .caam = { 3307 .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CBC, 3308 .class2_alg_type = OP_ALG_ALGSEL_SHA224 | 3309 OP_ALG_AAI_HMAC_PRECOMP, 3310 .alg_op = OP_ALG_ALGSEL_SHA224 | OP_ALG_AAI_HMAC, 3311 .geniv = true, 3312 }, 3313 }, 3314 { 3315 .aead = { 3316 .base = { 3317 .cra_name = "authenc(hmac(sha256),cbc(aes))", 3318 .cra_driver_name = "authenc-hmac-sha256-" 3319 "cbc-aes-caam", 3320 .cra_blocksize = AES_BLOCK_SIZE, 3321 }, 3322 .setkey = aead_setkey, 3323 .setauthsize = aead_setauthsize, 3324 .encrypt = aead_encrypt, 3325 .decrypt = aead_decrypt, 3326 .ivsize = AES_BLOCK_SIZE, 3327 .maxauthsize = SHA256_DIGEST_SIZE, 3328 }, 3329 .caam = { 3330 .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CBC, 3331 .class2_alg_type = OP_ALG_ALGSEL_SHA256 | 3332 OP_ALG_AAI_HMAC_PRECOMP, 3333 .alg_op = OP_ALG_ALGSEL_SHA256 | OP_ALG_AAI_HMAC, 3334 }, 3335 }, 3336 { 3337 .aead = { 3338 .base = { 3339 .cra_name = "echainiv(authenc(hmac(sha256)," 3340 "cbc(aes)))", 3341 .cra_driver_name = "echainiv-authenc-" 3342 "hmac-sha256-cbc-aes-caam", 3343 .cra_blocksize = AES_BLOCK_SIZE, 3344 }, 3345 .setkey = aead_setkey, 3346 .setauthsize = aead_setauthsize, 3347 .encrypt = aead_encrypt, 3348 .decrypt = aead_decrypt, 3349 .ivsize = AES_BLOCK_SIZE, 3350 .maxauthsize = SHA256_DIGEST_SIZE, 3351 }, 3352 .caam = { 3353 .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CBC, 3354 .class2_alg_type = OP_ALG_ALGSEL_SHA256 | 3355 OP_ALG_AAI_HMAC_PRECOMP, 3356 .alg_op = OP_ALG_ALGSEL_SHA256 | OP_ALG_AAI_HMAC, 3357 .geniv = true, 3358 }, 3359 }, 3360 { 3361 .aead = { 3362 .base = { 3363 .cra_name = "authenc(hmac(sha384),cbc(aes))", 3364 .cra_driver_name = "authenc-hmac-sha384-" 3365 "cbc-aes-caam", 3366 .cra_blocksize = AES_BLOCK_SIZE, 3367 }, 3368 .setkey = aead_setkey, 3369 .setauthsize = aead_setauthsize, 3370 .encrypt = aead_encrypt, 3371 .decrypt = aead_decrypt, 3372 .ivsize = AES_BLOCK_SIZE, 3373 .maxauthsize = SHA384_DIGEST_SIZE, 3374 }, 3375 .caam = { 3376 .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CBC, 3377 .class2_alg_type = OP_ALG_ALGSEL_SHA384 | 3378 OP_ALG_AAI_HMAC_PRECOMP, 3379 .alg_op = OP_ALG_ALGSEL_SHA384 | OP_ALG_AAI_HMAC, 3380 }, 3381 }, 3382 { 3383 .aead = { 3384 .base = { 3385 .cra_name = "echainiv(authenc(hmac(sha384)," 3386 "cbc(aes)))", 3387 .cra_driver_name = "echainiv-authenc-" 3388 "hmac-sha384-cbc-aes-caam", 3389 .cra_blocksize = AES_BLOCK_SIZE, 3390 }, 3391 .setkey = aead_setkey, 3392 .setauthsize = aead_setauthsize, 3393 .encrypt = aead_encrypt, 3394 .decrypt = aead_decrypt, 3395 .ivsize = AES_BLOCK_SIZE, 3396 .maxauthsize = SHA384_DIGEST_SIZE, 3397 }, 3398 .caam = { 3399 .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CBC, 3400 .class2_alg_type = OP_ALG_ALGSEL_SHA384 | 3401 OP_ALG_AAI_HMAC_PRECOMP, 3402 .alg_op = OP_ALG_ALGSEL_SHA384 | OP_ALG_AAI_HMAC, 3403 .geniv = true, 3404 }, 3405 }, 3406 { 3407 .aead = { 3408 .base = { 3409 .cra_name = "authenc(hmac(sha512),cbc(aes))", 3410 .cra_driver_name = "authenc-hmac-sha512-" 3411 "cbc-aes-caam", 3412 .cra_blocksize = AES_BLOCK_SIZE, 3413 }, 3414 .setkey = aead_setkey, 3415 .setauthsize = aead_setauthsize, 3416 .encrypt = aead_encrypt, 3417 .decrypt = aead_decrypt, 3418 .ivsize = AES_BLOCK_SIZE, 3419 .maxauthsize = SHA512_DIGEST_SIZE, 3420 }, 3421 .caam = { 3422 .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CBC, 3423 .class2_alg_type = OP_ALG_ALGSEL_SHA512 | 3424 OP_ALG_AAI_HMAC_PRECOMP, 3425 .alg_op = OP_ALG_ALGSEL_SHA512 | OP_ALG_AAI_HMAC, 3426 }, 3427 }, 3428 { 3429 .aead = { 3430 .base = { 3431 .cra_name = "echainiv(authenc(hmac(sha512)," 3432 "cbc(aes)))", 3433 .cra_driver_name = "echainiv-authenc-" 3434 "hmac-sha512-cbc-aes-caam", 3435 .cra_blocksize = AES_BLOCK_SIZE, 3436 }, 3437 .setkey = aead_setkey, 3438 .setauthsize = aead_setauthsize, 3439 .encrypt = aead_encrypt, 3440 .decrypt = aead_decrypt, 3441 .ivsize = AES_BLOCK_SIZE, 3442 .maxauthsize = SHA512_DIGEST_SIZE, 3443 }, 3444 .caam = { 3445 .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CBC, 3446 .class2_alg_type = OP_ALG_ALGSEL_SHA512 | 3447 OP_ALG_AAI_HMAC_PRECOMP, 3448 .alg_op = OP_ALG_ALGSEL_SHA512 | OP_ALG_AAI_HMAC, 3449 .geniv = true, 3450 }, 3451 }, 3452 { 3453 .aead = { 3454 .base = { 3455 .cra_name = "authenc(hmac(md5),cbc(des3_ede))", 3456 .cra_driver_name = "authenc-hmac-md5-" 3457 "cbc-des3_ede-caam", 3458 .cra_blocksize = DES3_EDE_BLOCK_SIZE, 3459 }, 3460 .setkey = aead_setkey, 3461 .setauthsize = aead_setauthsize, 3462 .encrypt = aead_encrypt, 3463 .decrypt = aead_decrypt, 3464 .ivsize = DES3_EDE_BLOCK_SIZE, 3465 .maxauthsize = MD5_DIGEST_SIZE, 3466 }, 3467 .caam = { 3468 .class1_alg_type = OP_ALG_ALGSEL_3DES | OP_ALG_AAI_CBC, 3469 .class2_alg_type = OP_ALG_ALGSEL_MD5 | 3470 OP_ALG_AAI_HMAC_PRECOMP, 3471 .alg_op = OP_ALG_ALGSEL_MD5 | OP_ALG_AAI_HMAC, 3472 } 3473 }, 3474 { 3475 .aead = { 3476 .base = { 3477 .cra_name = "echainiv(authenc(hmac(md5)," 3478 "cbc(des3_ede)))", 3479 .cra_driver_name = "echainiv-authenc-hmac-md5-" 3480 "cbc-des3_ede-caam", 3481 .cra_blocksize = DES3_EDE_BLOCK_SIZE, 3482 }, 3483 .setkey = aead_setkey, 3484 .setauthsize = aead_setauthsize, 3485 .encrypt = aead_encrypt, 3486 .decrypt = aead_decrypt, 3487 .ivsize = DES3_EDE_BLOCK_SIZE, 3488 .maxauthsize = MD5_DIGEST_SIZE, 3489 }, 3490 .caam = { 3491 .class1_alg_type = OP_ALG_ALGSEL_3DES | OP_ALG_AAI_CBC, 3492 .class2_alg_type = OP_ALG_ALGSEL_MD5 | 3493 OP_ALG_AAI_HMAC_PRECOMP, 3494 .alg_op = OP_ALG_ALGSEL_MD5 | OP_ALG_AAI_HMAC, 3495 .geniv = true, 3496 } 3497 }, 3498 { 3499 .aead = { 3500 .base = { 3501 .cra_name = "authenc(hmac(sha1)," 3502 "cbc(des3_ede))", 3503 .cra_driver_name = "authenc-hmac-sha1-" 3504 "cbc-des3_ede-caam", 3505 .cra_blocksize = DES3_EDE_BLOCK_SIZE, 3506 }, 3507 .setkey = aead_setkey, 3508 .setauthsize = aead_setauthsize, 3509 .encrypt = aead_encrypt, 3510 .decrypt = aead_decrypt, 3511 .ivsize = DES3_EDE_BLOCK_SIZE, 3512 .maxauthsize = SHA1_DIGEST_SIZE, 3513 }, 3514 .caam = { 3515 .class1_alg_type = OP_ALG_ALGSEL_3DES | OP_ALG_AAI_CBC, 3516 .class2_alg_type = OP_ALG_ALGSEL_SHA1 | 3517 OP_ALG_AAI_HMAC_PRECOMP, 3518 .alg_op = OP_ALG_ALGSEL_SHA1 | OP_ALG_AAI_HMAC, 3519 }, 3520 }, 3521 { 3522 .aead = { 3523 .base = { 3524 .cra_name = "echainiv(authenc(hmac(sha1)," 3525 "cbc(des3_ede)))", 3526 .cra_driver_name = "echainiv-authenc-" 3527 "hmac-sha1-" 3528 "cbc-des3_ede-caam", 3529 .cra_blocksize = DES3_EDE_BLOCK_SIZE, 3530 }, 3531 .setkey = aead_setkey, 3532 .setauthsize = aead_setauthsize, 3533 .encrypt = aead_encrypt, 3534 .decrypt = aead_decrypt, 3535 .ivsize = DES3_EDE_BLOCK_SIZE, 3536 .maxauthsize = SHA1_DIGEST_SIZE, 3537 }, 3538 .caam = { 3539 .class1_alg_type = OP_ALG_ALGSEL_3DES | OP_ALG_AAI_CBC, 3540 .class2_alg_type = OP_ALG_ALGSEL_SHA1 | 3541 OP_ALG_AAI_HMAC_PRECOMP, 3542 .alg_op = OP_ALG_ALGSEL_SHA1 | OP_ALG_AAI_HMAC, 3543 .geniv = true, 3544 }, 3545 }, 3546 { 3547 .aead = { 3548 .base = { 3549 .cra_name = "authenc(hmac(sha224)," 3550 "cbc(des3_ede))", 3551 .cra_driver_name = "authenc-hmac-sha224-" 3552 "cbc-des3_ede-caam", 3553 .cra_blocksize = DES3_EDE_BLOCK_SIZE, 3554 }, 3555 .setkey = aead_setkey, 3556 .setauthsize = aead_setauthsize, 3557 .encrypt = aead_encrypt, 3558 .decrypt = aead_decrypt, 3559 .ivsize = DES3_EDE_BLOCK_SIZE, 3560 .maxauthsize = SHA224_DIGEST_SIZE, 3561 }, 3562 .caam = { 3563 .class1_alg_type = OP_ALG_ALGSEL_3DES | OP_ALG_AAI_CBC, 3564 .class2_alg_type = OP_ALG_ALGSEL_SHA224 | 3565 OP_ALG_AAI_HMAC_PRECOMP, 3566 .alg_op = OP_ALG_ALGSEL_SHA224 | OP_ALG_AAI_HMAC, 3567 }, 3568 }, 3569 { 3570 .aead = { 3571 .base = { 3572 .cra_name = "echainiv(authenc(hmac(sha224)," 3573 "cbc(des3_ede)))", 3574 .cra_driver_name = "echainiv-authenc-" 3575 "hmac-sha224-" 3576 "cbc-des3_ede-caam", 3577 .cra_blocksize = DES3_EDE_BLOCK_SIZE, 3578 }, 3579 .setkey = aead_setkey, 3580 .setauthsize = aead_setauthsize, 3581 .encrypt = aead_encrypt, 3582 .decrypt = aead_decrypt, 3583 .ivsize = DES3_EDE_BLOCK_SIZE, 3584 .maxauthsize = SHA224_DIGEST_SIZE, 3585 }, 3586 .caam = { 3587 .class1_alg_type = OP_ALG_ALGSEL_3DES | OP_ALG_AAI_CBC, 3588 .class2_alg_type = OP_ALG_ALGSEL_SHA224 | 3589 OP_ALG_AAI_HMAC_PRECOMP, 3590 .alg_op = OP_ALG_ALGSEL_SHA224 | OP_ALG_AAI_HMAC, 3591 .geniv = true, 3592 }, 3593 }, 3594 { 3595 .aead = { 3596 .base = { 3597 .cra_name = "authenc(hmac(sha256)," 3598 "cbc(des3_ede))", 3599 .cra_driver_name = "authenc-hmac-sha256-" 3600 "cbc-des3_ede-caam", 3601 .cra_blocksize = DES3_EDE_BLOCK_SIZE, 3602 }, 3603 .setkey = aead_setkey, 3604 .setauthsize = aead_setauthsize, 3605 .encrypt = aead_encrypt, 3606 .decrypt = aead_decrypt, 3607 .ivsize = DES3_EDE_BLOCK_SIZE, 3608 .maxauthsize = SHA256_DIGEST_SIZE, 3609 }, 3610 .caam = { 3611 .class1_alg_type = OP_ALG_ALGSEL_3DES | OP_ALG_AAI_CBC, 3612 .class2_alg_type = OP_ALG_ALGSEL_SHA256 | 3613 OP_ALG_AAI_HMAC_PRECOMP, 3614 .alg_op = OP_ALG_ALGSEL_SHA256 | OP_ALG_AAI_HMAC, 3615 }, 3616 }, 3617 { 3618 .aead = { 3619 .base = { 3620 .cra_name = "echainiv(authenc(hmac(sha256)," 3621 "cbc(des3_ede)))", 3622 .cra_driver_name = "echainiv-authenc-" 3623 "hmac-sha256-" 3624 "cbc-des3_ede-caam", 3625 .cra_blocksize = DES3_EDE_BLOCK_SIZE, 3626 }, 3627 .setkey = aead_setkey, 3628 .setauthsize = aead_setauthsize, 3629 .encrypt = aead_encrypt, 3630 .decrypt = aead_decrypt, 3631 .ivsize = DES3_EDE_BLOCK_SIZE, 3632 .maxauthsize = SHA256_DIGEST_SIZE, 3633 }, 3634 .caam = { 3635 .class1_alg_type = OP_ALG_ALGSEL_3DES | OP_ALG_AAI_CBC, 3636 .class2_alg_type = OP_ALG_ALGSEL_SHA256 | 3637 OP_ALG_AAI_HMAC_PRECOMP, 3638 .alg_op = OP_ALG_ALGSEL_SHA256 | OP_ALG_AAI_HMAC, 3639 .geniv = true, 3640 }, 3641 }, 3642 { 3643 .aead = { 3644 .base = { 3645 .cra_name = "authenc(hmac(sha384)," 3646 "cbc(des3_ede))", 3647 .cra_driver_name = "authenc-hmac-sha384-" 3648 "cbc-des3_ede-caam", 3649 .cra_blocksize = DES3_EDE_BLOCK_SIZE, 3650 }, 3651 .setkey = aead_setkey, 3652 .setauthsize = aead_setauthsize, 3653 .encrypt = aead_encrypt, 3654 .decrypt = aead_decrypt, 3655 .ivsize = DES3_EDE_BLOCK_SIZE, 3656 .maxauthsize = SHA384_DIGEST_SIZE, 3657 }, 3658 .caam = { 3659 .class1_alg_type = OP_ALG_ALGSEL_3DES | OP_ALG_AAI_CBC, 3660 .class2_alg_type = OP_ALG_ALGSEL_SHA384 | 3661 OP_ALG_AAI_HMAC_PRECOMP, 3662 .alg_op = OP_ALG_ALGSEL_SHA384 | OP_ALG_AAI_HMAC, 3663 }, 3664 }, 3665 { 3666 .aead = { 3667 .base = { 3668 .cra_name = "echainiv(authenc(hmac(sha384)," 3669 "cbc(des3_ede)))", 3670 .cra_driver_name = "echainiv-authenc-" 3671 "hmac-sha384-" 3672 "cbc-des3_ede-caam", 3673 .cra_blocksize = DES3_EDE_BLOCK_SIZE, 3674 }, 3675 .setkey = aead_setkey, 3676 .setauthsize = aead_setauthsize, 3677 .encrypt = aead_encrypt, 3678 .decrypt = aead_decrypt, 3679 .ivsize = DES3_EDE_BLOCK_SIZE, 3680 .maxauthsize = SHA384_DIGEST_SIZE, 3681 }, 3682 .caam = { 3683 .class1_alg_type = OP_ALG_ALGSEL_3DES | OP_ALG_AAI_CBC, 3684 .class2_alg_type = OP_ALG_ALGSEL_SHA384 | 3685 OP_ALG_AAI_HMAC_PRECOMP, 3686 .alg_op = OP_ALG_ALGSEL_SHA384 | OP_ALG_AAI_HMAC, 3687 .geniv = true, 3688 }, 3689 }, 3690 { 3691 .aead = { 3692 .base = { 3693 .cra_name = "authenc(hmac(sha512)," 3694 "cbc(des3_ede))", 3695 .cra_driver_name = "authenc-hmac-sha512-" 3696 "cbc-des3_ede-caam", 3697 .cra_blocksize = DES3_EDE_BLOCK_SIZE, 3698 }, 3699 .setkey = aead_setkey, 3700 .setauthsize = aead_setauthsize, 3701 .encrypt = aead_encrypt, 3702 .decrypt = aead_decrypt, 3703 .ivsize = DES3_EDE_BLOCK_SIZE, 3704 .maxauthsize = SHA512_DIGEST_SIZE, 3705 }, 3706 .caam = { 3707 .class1_alg_type = OP_ALG_ALGSEL_3DES | OP_ALG_AAI_CBC, 3708 .class2_alg_type = OP_ALG_ALGSEL_SHA512 | 3709 OP_ALG_AAI_HMAC_PRECOMP, 3710 .alg_op = OP_ALG_ALGSEL_SHA512 | OP_ALG_AAI_HMAC, 3711 }, 3712 }, 3713 { 3714 .aead = { 3715 .base = { 3716 .cra_name = "echainiv(authenc(hmac(sha512)," 3717 "cbc(des3_ede)))", 3718 .cra_driver_name = "echainiv-authenc-" 3719 "hmac-sha512-" 3720 "cbc-des3_ede-caam", 3721 .cra_blocksize = DES3_EDE_BLOCK_SIZE, 3722 }, 3723 .setkey = aead_setkey, 3724 .setauthsize = aead_setauthsize, 3725 .encrypt = aead_encrypt, 3726 .decrypt = aead_decrypt, 3727 .ivsize = DES3_EDE_BLOCK_SIZE, 3728 .maxauthsize = SHA512_DIGEST_SIZE, 3729 }, 3730 .caam = { 3731 .class1_alg_type = OP_ALG_ALGSEL_3DES | OP_ALG_AAI_CBC, 3732 .class2_alg_type = OP_ALG_ALGSEL_SHA512 | 3733 OP_ALG_AAI_HMAC_PRECOMP, 3734 .alg_op = OP_ALG_ALGSEL_SHA512 | OP_ALG_AAI_HMAC, 3735 .geniv = true, 3736 }, 3737 }, 3738 { 3739 .aead = { 3740 .base = { 3741 .cra_name = "authenc(hmac(md5),cbc(des))", 3742 .cra_driver_name = "authenc-hmac-md5-" 3743 "cbc-des-caam", 3744 .cra_blocksize = DES_BLOCK_SIZE, 3745 }, 3746 .setkey = aead_setkey, 3747 .setauthsize = aead_setauthsize, 3748 .encrypt = aead_encrypt, 3749 .decrypt = aead_decrypt, 3750 .ivsize = DES_BLOCK_SIZE, 3751 .maxauthsize = MD5_DIGEST_SIZE, 3752 }, 3753 .caam = { 3754 .class1_alg_type = OP_ALG_ALGSEL_DES | OP_ALG_AAI_CBC, 3755 .class2_alg_type = OP_ALG_ALGSEL_MD5 | 3756 OP_ALG_AAI_HMAC_PRECOMP, 3757 .alg_op = OP_ALG_ALGSEL_MD5 | OP_ALG_AAI_HMAC, 3758 }, 3759 }, 3760 { 3761 .aead = { 3762 .base = { 3763 .cra_name = "echainiv(authenc(hmac(md5)," 3764 "cbc(des)))", 3765 .cra_driver_name = "echainiv-authenc-hmac-md5-" 3766 "cbc-des-caam", 3767 .cra_blocksize = DES_BLOCK_SIZE, 3768 }, 3769 .setkey = aead_setkey, 3770 .setauthsize = aead_setauthsize, 3771 .encrypt = aead_encrypt, 3772 .decrypt = aead_decrypt, 3773 .ivsize = DES_BLOCK_SIZE, 3774 .maxauthsize = MD5_DIGEST_SIZE, 3775 }, 3776 .caam = { 3777 .class1_alg_type = OP_ALG_ALGSEL_DES | OP_ALG_AAI_CBC, 3778 .class2_alg_type = OP_ALG_ALGSEL_MD5 | 3779 OP_ALG_AAI_HMAC_PRECOMP, 3780 .alg_op = OP_ALG_ALGSEL_MD5 | OP_ALG_AAI_HMAC, 3781 .geniv = true, 3782 }, 3783 }, 3784 { 3785 .aead = { 3786 .base = { 3787 .cra_name = "authenc(hmac(sha1),cbc(des))", 3788 .cra_driver_name = "authenc-hmac-sha1-" 3789 "cbc-des-caam", 3790 .cra_blocksize = DES_BLOCK_SIZE, 3791 }, 3792 .setkey = aead_setkey, 3793 .setauthsize = aead_setauthsize, 3794 .encrypt = aead_encrypt, 3795 .decrypt = aead_decrypt, 3796 .ivsize = DES_BLOCK_SIZE, 3797 .maxauthsize = SHA1_DIGEST_SIZE, 3798 }, 3799 .caam = { 3800 .class1_alg_type = OP_ALG_ALGSEL_DES | OP_ALG_AAI_CBC, 3801 .class2_alg_type = OP_ALG_ALGSEL_SHA1 | 3802 OP_ALG_AAI_HMAC_PRECOMP, 3803 .alg_op = OP_ALG_ALGSEL_SHA1 | OP_ALG_AAI_HMAC, 3804 }, 3805 }, 3806 { 3807 .aead = { 3808 .base = { 3809 .cra_name = "echainiv(authenc(hmac(sha1)," 3810 "cbc(des)))", 3811 .cra_driver_name = "echainiv-authenc-" 3812 "hmac-sha1-cbc-des-caam", 3813 .cra_blocksize = DES_BLOCK_SIZE, 3814 }, 3815 .setkey = aead_setkey, 3816 .setauthsize = aead_setauthsize, 3817 .encrypt = aead_encrypt, 3818 .decrypt = aead_decrypt, 3819 .ivsize = DES_BLOCK_SIZE, 3820 .maxauthsize = SHA1_DIGEST_SIZE, 3821 }, 3822 .caam = { 3823 .class1_alg_type = OP_ALG_ALGSEL_DES | OP_ALG_AAI_CBC, 3824 .class2_alg_type = OP_ALG_ALGSEL_SHA1 | 3825 OP_ALG_AAI_HMAC_PRECOMP, 3826 .alg_op = OP_ALG_ALGSEL_SHA1 | OP_ALG_AAI_HMAC, 3827 .geniv = true, 3828 }, 3829 }, 3830 { 3831 .aead = { 3832 .base = { 3833 .cra_name = "authenc(hmac(sha224),cbc(des))", 3834 .cra_driver_name = "authenc-hmac-sha224-" 3835 "cbc-des-caam", 3836 .cra_blocksize = DES_BLOCK_SIZE, 3837 }, 3838 .setkey = aead_setkey, 3839 .setauthsize = aead_setauthsize, 3840 .encrypt = aead_encrypt, 3841 .decrypt = aead_decrypt, 3842 .ivsize = DES_BLOCK_SIZE, 3843 .maxauthsize = SHA224_DIGEST_SIZE, 3844 }, 3845 .caam = { 3846 .class1_alg_type = OP_ALG_ALGSEL_DES | OP_ALG_AAI_CBC, 3847 .class2_alg_type = OP_ALG_ALGSEL_SHA224 | 3848 OP_ALG_AAI_HMAC_PRECOMP, 3849 .alg_op = OP_ALG_ALGSEL_SHA224 | OP_ALG_AAI_HMAC, 3850 }, 3851 }, 3852 { 3853 .aead = { 3854 .base = { 3855 .cra_name = "echainiv(authenc(hmac(sha224)," 3856 "cbc(des)))", 3857 .cra_driver_name = "echainiv-authenc-" 3858 "hmac-sha224-cbc-des-caam", 3859 .cra_blocksize = DES_BLOCK_SIZE, 3860 }, 3861 .setkey = aead_setkey, 3862 .setauthsize = aead_setauthsize, 3863 .encrypt = aead_encrypt, 3864 .decrypt = aead_decrypt, 3865 .ivsize = DES_BLOCK_SIZE, 3866 .maxauthsize = SHA224_DIGEST_SIZE, 3867 }, 3868 .caam = { 3869 .class1_alg_type = OP_ALG_ALGSEL_DES | OP_ALG_AAI_CBC, 3870 .class2_alg_type = OP_ALG_ALGSEL_SHA224 | 3871 OP_ALG_AAI_HMAC_PRECOMP, 3872 .alg_op = OP_ALG_ALGSEL_SHA224 | OP_ALG_AAI_HMAC, 3873 .geniv = true, 3874 }, 3875 }, 3876 { 3877 .aead = { 3878 .base = { 3879 .cra_name = "authenc(hmac(sha256),cbc(des))", 3880 .cra_driver_name = "authenc-hmac-sha256-" 3881 "cbc-des-caam", 3882 .cra_blocksize = DES_BLOCK_SIZE, 3883 }, 3884 .setkey = aead_setkey, 3885 .setauthsize = aead_setauthsize, 3886 .encrypt = aead_encrypt, 3887 .decrypt = aead_decrypt, 3888 .ivsize = DES_BLOCK_SIZE, 3889 .maxauthsize = SHA256_DIGEST_SIZE, 3890 }, 3891 .caam = { 3892 .class1_alg_type = OP_ALG_ALGSEL_DES | OP_ALG_AAI_CBC, 3893 .class2_alg_type = OP_ALG_ALGSEL_SHA256 | 3894 OP_ALG_AAI_HMAC_PRECOMP, 3895 .alg_op = OP_ALG_ALGSEL_SHA256 | OP_ALG_AAI_HMAC, 3896 }, 3897 }, 3898 { 3899 .aead = { 3900 .base = { 3901 .cra_name = "echainiv(authenc(hmac(sha256)," 3902 "cbc(des)))", 3903 .cra_driver_name = "echainiv-authenc-" 3904 "hmac-sha256-cbc-des-caam", 3905 .cra_blocksize = DES_BLOCK_SIZE, 3906 }, 3907 .setkey = aead_setkey, 3908 .setauthsize = aead_setauthsize, 3909 .encrypt = aead_encrypt, 3910 .decrypt = aead_decrypt, 3911 .ivsize = DES_BLOCK_SIZE, 3912 .maxauthsize = SHA256_DIGEST_SIZE, 3913 }, 3914 .caam = { 3915 .class1_alg_type = OP_ALG_ALGSEL_DES | OP_ALG_AAI_CBC, 3916 .class2_alg_type = OP_ALG_ALGSEL_SHA256 | 3917 OP_ALG_AAI_HMAC_PRECOMP, 3918 .alg_op = OP_ALG_ALGSEL_SHA256 | OP_ALG_AAI_HMAC, 3919 .geniv = true, 3920 }, 3921 }, 3922 { 3923 .aead = { 3924 .base = { 3925 .cra_name = "authenc(hmac(sha384),cbc(des))", 3926 .cra_driver_name = "authenc-hmac-sha384-" 3927 "cbc-des-caam", 3928 .cra_blocksize = DES_BLOCK_SIZE, 3929 }, 3930 .setkey = aead_setkey, 3931 .setauthsize = aead_setauthsize, 3932 .encrypt = aead_encrypt, 3933 .decrypt = aead_decrypt, 3934 .ivsize = DES_BLOCK_SIZE, 3935 .maxauthsize = SHA384_DIGEST_SIZE, 3936 }, 3937 .caam = { 3938 .class1_alg_type = OP_ALG_ALGSEL_DES | OP_ALG_AAI_CBC, 3939 .class2_alg_type = OP_ALG_ALGSEL_SHA384 | 3940 OP_ALG_AAI_HMAC_PRECOMP, 3941 .alg_op = OP_ALG_ALGSEL_SHA384 | OP_ALG_AAI_HMAC, 3942 }, 3943 }, 3944 { 3945 .aead = { 3946 .base = { 3947 .cra_name = "echainiv(authenc(hmac(sha384)," 3948 "cbc(des)))", 3949 .cra_driver_name = "echainiv-authenc-" 3950 "hmac-sha384-cbc-des-caam", 3951 .cra_blocksize = DES_BLOCK_SIZE, 3952 }, 3953 .setkey = aead_setkey, 3954 .setauthsize = aead_setauthsize, 3955 .encrypt = aead_encrypt, 3956 .decrypt = aead_decrypt, 3957 .ivsize = DES_BLOCK_SIZE, 3958 .maxauthsize = SHA384_DIGEST_SIZE, 3959 }, 3960 .caam = { 3961 .class1_alg_type = OP_ALG_ALGSEL_DES | OP_ALG_AAI_CBC, 3962 .class2_alg_type = OP_ALG_ALGSEL_SHA384 | 3963 OP_ALG_AAI_HMAC_PRECOMP, 3964 .alg_op = OP_ALG_ALGSEL_SHA384 | OP_ALG_AAI_HMAC, 3965 .geniv = true, 3966 }, 3967 }, 3968 { 3969 .aead = { 3970 .base = { 3971 .cra_name = "authenc(hmac(sha512),cbc(des))", 3972 .cra_driver_name = "authenc-hmac-sha512-" 3973 "cbc-des-caam", 3974 .cra_blocksize = DES_BLOCK_SIZE, 3975 }, 3976 .setkey = aead_setkey, 3977 .setauthsize = aead_setauthsize, 3978 .encrypt = aead_encrypt, 3979 .decrypt = aead_decrypt, 3980 .ivsize = DES_BLOCK_SIZE, 3981 .maxauthsize = SHA512_DIGEST_SIZE, 3982 }, 3983 .caam = { 3984 .class1_alg_type = OP_ALG_ALGSEL_DES | OP_ALG_AAI_CBC, 3985 .class2_alg_type = OP_ALG_ALGSEL_SHA512 | 3986 OP_ALG_AAI_HMAC_PRECOMP, 3987 .alg_op = OP_ALG_ALGSEL_SHA512 | OP_ALG_AAI_HMAC, 3988 }, 3989 }, 3990 { 3991 .aead = { 3992 .base = { 3993 .cra_name = "echainiv(authenc(hmac(sha512)," 3994 "cbc(des)))", 3995 .cra_driver_name = "echainiv-authenc-" 3996 "hmac-sha512-cbc-des-caam", 3997 .cra_blocksize = DES_BLOCK_SIZE, 3998 }, 3999 .setkey = aead_setkey, 4000 .setauthsize = aead_setauthsize, 4001 .encrypt = aead_encrypt, 4002 .decrypt = aead_decrypt, 4003 .ivsize = DES_BLOCK_SIZE, 4004 .maxauthsize = SHA512_DIGEST_SIZE, 4005 }, 4006 .caam = { 4007 .class1_alg_type = OP_ALG_ALGSEL_DES | OP_ALG_AAI_CBC, 4008 .class2_alg_type = OP_ALG_ALGSEL_SHA512 | 4009 OP_ALG_AAI_HMAC_PRECOMP, 4010 .alg_op = OP_ALG_ALGSEL_SHA512 | OP_ALG_AAI_HMAC, 4011 .geniv = true, 4012 }, 4013 }, 4014 { 4015 .aead = { 4016 .base = { 4017 .cra_name = "authenc(hmac(md5)," 4018 "rfc3686(ctr(aes)))", 4019 .cra_driver_name = "authenc-hmac-md5-" 4020 "rfc3686-ctr-aes-caam", 4021 .cra_blocksize = 1, 4022 }, 4023 .setkey = aead_setkey, 4024 .setauthsize = aead_setauthsize, 4025 .encrypt = aead_encrypt, 4026 .decrypt = aead_decrypt, 4027 .ivsize = CTR_RFC3686_IV_SIZE, 4028 .maxauthsize = MD5_DIGEST_SIZE, 4029 }, 4030 .caam = { 4031 .class1_alg_type = OP_ALG_ALGSEL_AES | 4032 OP_ALG_AAI_CTR_MOD128, 4033 .class2_alg_type = OP_ALG_ALGSEL_MD5 | 4034 OP_ALG_AAI_HMAC_PRECOMP, 4035 .alg_op = OP_ALG_ALGSEL_MD5 | OP_ALG_AAI_HMAC, 4036 .rfc3686 = true, 4037 }, 4038 }, 4039 { 4040 .aead = { 4041 .base = { 4042 .cra_name = "seqiv(authenc(" 4043 "hmac(md5),rfc3686(ctr(aes))))", 4044 .cra_driver_name = "seqiv-authenc-hmac-md5-" 4045 "rfc3686-ctr-aes-caam", 4046 .cra_blocksize = 1, 4047 }, 4048 .setkey = aead_setkey, 4049 .setauthsize = aead_setauthsize, 4050 .encrypt = aead_encrypt, 4051 .decrypt = aead_decrypt, 4052 .ivsize = CTR_RFC3686_IV_SIZE, 4053 .maxauthsize = MD5_DIGEST_SIZE, 4054 }, 4055 .caam = { 4056 .class1_alg_type = OP_ALG_ALGSEL_AES | 4057 OP_ALG_AAI_CTR_MOD128, 4058 .class2_alg_type = OP_ALG_ALGSEL_MD5 | 4059 OP_ALG_AAI_HMAC_PRECOMP, 4060 .alg_op = OP_ALG_ALGSEL_MD5 | OP_ALG_AAI_HMAC, 4061 .rfc3686 = true, 4062 .geniv = true, 4063 }, 4064 }, 4065 { 4066 .aead = { 4067 .base = { 4068 .cra_name = "authenc(hmac(sha1)," 4069 "rfc3686(ctr(aes)))", 4070 .cra_driver_name = "authenc-hmac-sha1-" 4071 "rfc3686-ctr-aes-caam", 4072 .cra_blocksize = 1, 4073 }, 4074 .setkey = aead_setkey, 4075 .setauthsize = aead_setauthsize, 4076 .encrypt = aead_encrypt, 4077 .decrypt = aead_decrypt, 4078 .ivsize = CTR_RFC3686_IV_SIZE, 4079 .maxauthsize = SHA1_DIGEST_SIZE, 4080 }, 4081 .caam = { 4082 .class1_alg_type = OP_ALG_ALGSEL_AES | 4083 OP_ALG_AAI_CTR_MOD128, 4084 .class2_alg_type = OP_ALG_ALGSEL_SHA1 | 4085 OP_ALG_AAI_HMAC_PRECOMP, 4086 .alg_op = OP_ALG_ALGSEL_SHA1 | OP_ALG_AAI_HMAC, 4087 .rfc3686 = true, 4088 }, 4089 }, 4090 { 4091 .aead = { 4092 .base = { 4093 .cra_name = "seqiv(authenc(" 4094 "hmac(sha1),rfc3686(ctr(aes))))", 4095 .cra_driver_name = "seqiv-authenc-hmac-sha1-" 4096 "rfc3686-ctr-aes-caam", 4097 .cra_blocksize = 1, 4098 }, 4099 .setkey = aead_setkey, 4100 .setauthsize = aead_setauthsize, 4101 .encrypt = aead_encrypt, 4102 .decrypt = aead_decrypt, 4103 .ivsize = CTR_RFC3686_IV_SIZE, 4104 .maxauthsize = SHA1_DIGEST_SIZE, 4105 }, 4106 .caam = { 4107 .class1_alg_type = OP_ALG_ALGSEL_AES | 4108 OP_ALG_AAI_CTR_MOD128, 4109 .class2_alg_type = OP_ALG_ALGSEL_SHA1 | 4110 OP_ALG_AAI_HMAC_PRECOMP, 4111 .alg_op = OP_ALG_ALGSEL_SHA1 | OP_ALG_AAI_HMAC, 4112 .rfc3686 = true, 4113 .geniv = true, 4114 }, 4115 }, 4116 { 4117 .aead = { 4118 .base = { 4119 .cra_name = "authenc(hmac(sha224)," 4120 "rfc3686(ctr(aes)))", 4121 .cra_driver_name = "authenc-hmac-sha224-" 4122 "rfc3686-ctr-aes-caam", 4123 .cra_blocksize = 1, 4124 }, 4125 .setkey = aead_setkey, 4126 .setauthsize = aead_setauthsize, 4127 .encrypt = aead_encrypt, 4128 .decrypt = aead_decrypt, 4129 .ivsize = CTR_RFC3686_IV_SIZE, 4130 .maxauthsize = SHA224_DIGEST_SIZE, 4131 }, 4132 .caam = { 4133 .class1_alg_type = OP_ALG_ALGSEL_AES | 4134 OP_ALG_AAI_CTR_MOD128, 4135 .class2_alg_type = OP_ALG_ALGSEL_SHA224 | 4136 OP_ALG_AAI_HMAC_PRECOMP, 4137 .alg_op = OP_ALG_ALGSEL_SHA224 | OP_ALG_AAI_HMAC, 4138 .rfc3686 = true, 4139 }, 4140 }, 4141 { 4142 .aead = { 4143 .base = { 4144 .cra_name = "seqiv(authenc(" 4145 "hmac(sha224),rfc3686(ctr(aes))))", 4146 .cra_driver_name = "seqiv-authenc-hmac-sha224-" 4147 "rfc3686-ctr-aes-caam", 4148 .cra_blocksize = 1, 4149 }, 4150 .setkey = aead_setkey, 4151 .setauthsize = aead_setauthsize, 4152 .encrypt = aead_encrypt, 4153 .decrypt = aead_decrypt, 4154 .ivsize = CTR_RFC3686_IV_SIZE, 4155 .maxauthsize = SHA224_DIGEST_SIZE, 4156 }, 4157 .caam = { 4158 .class1_alg_type = OP_ALG_ALGSEL_AES | 4159 OP_ALG_AAI_CTR_MOD128, 4160 .class2_alg_type = OP_ALG_ALGSEL_SHA224 | 4161 OP_ALG_AAI_HMAC_PRECOMP, 4162 .alg_op = OP_ALG_ALGSEL_SHA224 | OP_ALG_AAI_HMAC, 4163 .rfc3686 = true, 4164 .geniv = true, 4165 }, 4166 }, 4167 { 4168 .aead = { 4169 .base = { 4170 .cra_name = "authenc(hmac(sha256)," 4171 "rfc3686(ctr(aes)))", 4172 .cra_driver_name = "authenc-hmac-sha256-" 4173 "rfc3686-ctr-aes-caam", 4174 .cra_blocksize = 1, 4175 }, 4176 .setkey = aead_setkey, 4177 .setauthsize = aead_setauthsize, 4178 .encrypt = aead_encrypt, 4179 .decrypt = aead_decrypt, 4180 .ivsize = CTR_RFC3686_IV_SIZE, 4181 .maxauthsize = SHA256_DIGEST_SIZE, 4182 }, 4183 .caam = { 4184 .class1_alg_type = OP_ALG_ALGSEL_AES | 4185 OP_ALG_AAI_CTR_MOD128, 4186 .class2_alg_type = OP_ALG_ALGSEL_SHA256 | 4187 OP_ALG_AAI_HMAC_PRECOMP, 4188 .alg_op = OP_ALG_ALGSEL_SHA256 | OP_ALG_AAI_HMAC, 4189 .rfc3686 = true, 4190 }, 4191 }, 4192 { 4193 .aead = { 4194 .base = { 4195 .cra_name = "seqiv(authenc(hmac(sha256)," 4196 "rfc3686(ctr(aes))))", 4197 .cra_driver_name = "seqiv-authenc-hmac-sha256-" 4198 "rfc3686-ctr-aes-caam", 4199 .cra_blocksize = 1, 4200 }, 4201 .setkey = aead_setkey, 4202 .setauthsize = aead_setauthsize, 4203 .encrypt = aead_encrypt, 4204 .decrypt = aead_decrypt, 4205 .ivsize = CTR_RFC3686_IV_SIZE, 4206 .maxauthsize = SHA256_DIGEST_SIZE, 4207 }, 4208 .caam = { 4209 .class1_alg_type = OP_ALG_ALGSEL_AES | 4210 OP_ALG_AAI_CTR_MOD128, 4211 .class2_alg_type = OP_ALG_ALGSEL_SHA256 | 4212 OP_ALG_AAI_HMAC_PRECOMP, 4213 .alg_op = OP_ALG_ALGSEL_SHA256 | OP_ALG_AAI_HMAC, 4214 .rfc3686 = true, 4215 .geniv = true, 4216 }, 4217 }, 4218 { 4219 .aead = { 4220 .base = { 4221 .cra_name = "authenc(hmac(sha384)," 4222 "rfc3686(ctr(aes)))", 4223 .cra_driver_name = "authenc-hmac-sha384-" 4224 "rfc3686-ctr-aes-caam", 4225 .cra_blocksize = 1, 4226 }, 4227 .setkey = aead_setkey, 4228 .setauthsize = aead_setauthsize, 4229 .encrypt = aead_encrypt, 4230 .decrypt = aead_decrypt, 4231 .ivsize = CTR_RFC3686_IV_SIZE, 4232 .maxauthsize = SHA384_DIGEST_SIZE, 4233 }, 4234 .caam = { 4235 .class1_alg_type = OP_ALG_ALGSEL_AES | 4236 OP_ALG_AAI_CTR_MOD128, 4237 .class2_alg_type = OP_ALG_ALGSEL_SHA384 | 4238 OP_ALG_AAI_HMAC_PRECOMP, 4239 .alg_op = OP_ALG_ALGSEL_SHA384 | OP_ALG_AAI_HMAC, 4240 .rfc3686 = true, 4241 }, 4242 }, 4243 { 4244 .aead = { 4245 .base = { 4246 .cra_name = "seqiv(authenc(hmac(sha384)," 4247 "rfc3686(ctr(aes))))", 4248 .cra_driver_name = "seqiv-authenc-hmac-sha384-" 4249 "rfc3686-ctr-aes-caam", 4250 .cra_blocksize = 1, 4251 }, 4252 .setkey = aead_setkey, 4253 .setauthsize = aead_setauthsize, 4254 .encrypt = aead_encrypt, 4255 .decrypt = aead_decrypt, 4256 .ivsize = CTR_RFC3686_IV_SIZE, 4257 .maxauthsize = SHA384_DIGEST_SIZE, 4258 }, 4259 .caam = { 4260 .class1_alg_type = OP_ALG_ALGSEL_AES | 4261 OP_ALG_AAI_CTR_MOD128, 4262 .class2_alg_type = OP_ALG_ALGSEL_SHA384 | 4263 OP_ALG_AAI_HMAC_PRECOMP, 4264 .alg_op = OP_ALG_ALGSEL_SHA384 | OP_ALG_AAI_HMAC, 4265 .rfc3686 = true, 4266 .geniv = true, 4267 }, 4268 }, 4269 { 4270 .aead = { 4271 .base = { 4272 .cra_name = "authenc(hmac(sha512)," 4273 "rfc3686(ctr(aes)))", 4274 .cra_driver_name = "authenc-hmac-sha512-" 4275 "rfc3686-ctr-aes-caam", 4276 .cra_blocksize = 1, 4277 }, 4278 .setkey = aead_setkey, 4279 .setauthsize = aead_setauthsize, 4280 .encrypt = aead_encrypt, 4281 .decrypt = aead_decrypt, 4282 .ivsize = CTR_RFC3686_IV_SIZE, 4283 .maxauthsize = SHA512_DIGEST_SIZE, 4284 }, 4285 .caam = { 4286 .class1_alg_type = OP_ALG_ALGSEL_AES | 4287 OP_ALG_AAI_CTR_MOD128, 4288 .class2_alg_type = OP_ALG_ALGSEL_SHA512 | 4289 OP_ALG_AAI_HMAC_PRECOMP, 4290 .alg_op = OP_ALG_ALGSEL_SHA512 | OP_ALG_AAI_HMAC, 4291 .rfc3686 = true, 4292 }, 4293 }, 4294 { 4295 .aead = { 4296 .base = { 4297 .cra_name = "seqiv(authenc(hmac(sha512)," 4298 "rfc3686(ctr(aes))))", 4299 .cra_driver_name = "seqiv-authenc-hmac-sha512-" 4300 "rfc3686-ctr-aes-caam", 4301 .cra_blocksize = 1, 4302 }, 4303 .setkey = aead_setkey, 4304 .setauthsize = aead_setauthsize, 4305 .encrypt = aead_encrypt, 4306 .decrypt = aead_decrypt, 4307 .ivsize = CTR_RFC3686_IV_SIZE, 4308 .maxauthsize = SHA512_DIGEST_SIZE, 4309 }, 4310 .caam = { 4311 .class1_alg_type = OP_ALG_ALGSEL_AES | 4312 OP_ALG_AAI_CTR_MOD128, 4313 .class2_alg_type = OP_ALG_ALGSEL_SHA512 | 4314 OP_ALG_AAI_HMAC_PRECOMP, 4315 .alg_op = OP_ALG_ALGSEL_SHA512 | OP_ALG_AAI_HMAC, 4316 .rfc3686 = true, 4317 .geniv = true, 4318 }, 4319 }, 4320 }; 4321 4322 struct caam_crypto_alg { 4323 struct crypto_alg crypto_alg; 4324 struct list_head entry; 4325 struct caam_alg_entry caam; 4326 }; 4327 4328 static int caam_init_common(struct caam_ctx *ctx, struct caam_alg_entry *caam) 4329 { 4330 ctx->jrdev = caam_jr_alloc(); 4331 if (IS_ERR(ctx->jrdev)) { 4332 pr_err("Job Ring Device allocation for transform failed\n"); 4333 return PTR_ERR(ctx->jrdev); 4334 } 4335 4336 /* copy descriptor header template value */ 4337 ctx->class1_alg_type = OP_TYPE_CLASS1_ALG | caam->class1_alg_type; 4338 ctx->class2_alg_type = OP_TYPE_CLASS2_ALG | caam->class2_alg_type; 4339 ctx->alg_op = OP_TYPE_CLASS2_ALG | caam->alg_op; 4340 4341 return 0; 4342 } 4343 4344 static int caam_cra_init(struct crypto_tfm *tfm) 4345 { 4346 struct crypto_alg *alg = tfm->__crt_alg; 4347 struct caam_crypto_alg *caam_alg = 4348 container_of(alg, struct caam_crypto_alg, crypto_alg); 4349 struct caam_ctx *ctx = crypto_tfm_ctx(tfm); 4350 4351 return caam_init_common(ctx, &caam_alg->caam); 4352 } 4353 4354 static int caam_aead_init(struct crypto_aead *tfm) 4355 { 4356 struct aead_alg *alg = crypto_aead_alg(tfm); 4357 struct caam_aead_alg *caam_alg = 4358 container_of(alg, struct caam_aead_alg, aead); 4359 struct caam_ctx *ctx = crypto_aead_ctx(tfm); 4360 4361 return caam_init_common(ctx, &caam_alg->caam); 4362 } 4363 4364 static void caam_exit_common(struct caam_ctx *ctx) 4365 { 4366 if (ctx->sh_desc_enc_dma && 4367 !dma_mapping_error(ctx->jrdev, ctx->sh_desc_enc_dma)) 4368 dma_unmap_single(ctx->jrdev, ctx->sh_desc_enc_dma, 4369 desc_bytes(ctx->sh_desc_enc), DMA_TO_DEVICE); 4370 if (ctx->sh_desc_dec_dma && 4371 !dma_mapping_error(ctx->jrdev, ctx->sh_desc_dec_dma)) 4372 dma_unmap_single(ctx->jrdev, ctx->sh_desc_dec_dma, 4373 desc_bytes(ctx->sh_desc_dec), DMA_TO_DEVICE); 4374 if (ctx->sh_desc_givenc_dma && 4375 !dma_mapping_error(ctx->jrdev, ctx->sh_desc_givenc_dma)) 4376 dma_unmap_single(ctx->jrdev, ctx->sh_desc_givenc_dma, 4377 desc_bytes(ctx->sh_desc_givenc), 4378 DMA_TO_DEVICE); 4379 if (ctx->key_dma && 4380 !dma_mapping_error(ctx->jrdev, ctx->key_dma)) 4381 dma_unmap_single(ctx->jrdev, ctx->key_dma, 4382 ctx->enckeylen + ctx->split_key_pad_len, 4383 DMA_TO_DEVICE); 4384 4385 caam_jr_free(ctx->jrdev); 4386 } 4387 4388 static void caam_cra_exit(struct crypto_tfm *tfm) 4389 { 4390 caam_exit_common(crypto_tfm_ctx(tfm)); 4391 } 4392 4393 static void caam_aead_exit(struct crypto_aead *tfm) 4394 { 4395 caam_exit_common(crypto_aead_ctx(tfm)); 4396 } 4397 4398 static void __exit caam_algapi_exit(void) 4399 { 4400 4401 struct caam_crypto_alg *t_alg, *n; 4402 int i; 4403 4404 for (i = 0; i < ARRAY_SIZE(driver_aeads); i++) { 4405 struct caam_aead_alg *t_alg = driver_aeads + i; 4406 4407 if (t_alg->registered) 4408 crypto_unregister_aead(&t_alg->aead); 4409 } 4410 4411 if (!alg_list.next) 4412 return; 4413 4414 list_for_each_entry_safe(t_alg, n, &alg_list, entry) { 4415 crypto_unregister_alg(&t_alg->crypto_alg); 4416 list_del(&t_alg->entry); 4417 kfree(t_alg); 4418 } 4419 } 4420 4421 static struct caam_crypto_alg *caam_alg_alloc(struct caam_alg_template 4422 *template) 4423 { 4424 struct caam_crypto_alg *t_alg; 4425 struct crypto_alg *alg; 4426 4427 t_alg = kzalloc(sizeof(*t_alg), GFP_KERNEL); 4428 if (!t_alg) { 4429 pr_err("failed to allocate t_alg\n"); 4430 return ERR_PTR(-ENOMEM); 4431 } 4432 4433 alg = &t_alg->crypto_alg; 4434 4435 snprintf(alg->cra_name, CRYPTO_MAX_ALG_NAME, "%s", template->name); 4436 snprintf(alg->cra_driver_name, CRYPTO_MAX_ALG_NAME, "%s", 4437 template->driver_name); 4438 alg->cra_module = THIS_MODULE; 4439 alg->cra_init = caam_cra_init; 4440 alg->cra_exit = caam_cra_exit; 4441 alg->cra_priority = CAAM_CRA_PRIORITY; 4442 alg->cra_blocksize = template->blocksize; 4443 alg->cra_alignmask = 0; 4444 alg->cra_ctxsize = sizeof(struct caam_ctx); 4445 alg->cra_flags = CRYPTO_ALG_ASYNC | CRYPTO_ALG_KERN_DRIVER_ONLY | 4446 template->type; 4447 switch (template->type) { 4448 case CRYPTO_ALG_TYPE_GIVCIPHER: 4449 alg->cra_type = &crypto_givcipher_type; 4450 alg->cra_ablkcipher = template->template_ablkcipher; 4451 break; 4452 case CRYPTO_ALG_TYPE_ABLKCIPHER: 4453 alg->cra_type = &crypto_ablkcipher_type; 4454 alg->cra_ablkcipher = template->template_ablkcipher; 4455 break; 4456 } 4457 4458 t_alg->caam.class1_alg_type = template->class1_alg_type; 4459 t_alg->caam.class2_alg_type = template->class2_alg_type; 4460 t_alg->caam.alg_op = template->alg_op; 4461 4462 return t_alg; 4463 } 4464 4465 static void caam_aead_alg_init(struct caam_aead_alg *t_alg) 4466 { 4467 struct aead_alg *alg = &t_alg->aead; 4468 4469 alg->base.cra_module = THIS_MODULE; 4470 alg->base.cra_priority = CAAM_CRA_PRIORITY; 4471 alg->base.cra_ctxsize = sizeof(struct caam_ctx); 4472 alg->base.cra_flags = CRYPTO_ALG_ASYNC | CRYPTO_ALG_KERN_DRIVER_ONLY; 4473 4474 alg->init = caam_aead_init; 4475 alg->exit = caam_aead_exit; 4476 } 4477 4478 static int __init caam_algapi_init(void) 4479 { 4480 struct device_node *dev_node; 4481 struct platform_device *pdev; 4482 struct device *ctrldev; 4483 struct caam_drv_private *priv; 4484 int i = 0, err = 0; 4485 u32 cha_vid, cha_inst, des_inst, aes_inst, md_inst; 4486 unsigned int md_limit = SHA512_DIGEST_SIZE; 4487 bool registered = false; 4488 4489 dev_node = of_find_compatible_node(NULL, NULL, "fsl,sec-v4.0"); 4490 if (!dev_node) { 4491 dev_node = of_find_compatible_node(NULL, NULL, "fsl,sec4.0"); 4492 if (!dev_node) 4493 return -ENODEV; 4494 } 4495 4496 pdev = of_find_device_by_node(dev_node); 4497 if (!pdev) { 4498 of_node_put(dev_node); 4499 return -ENODEV; 4500 } 4501 4502 ctrldev = &pdev->dev; 4503 priv = dev_get_drvdata(ctrldev); 4504 of_node_put(dev_node); 4505 4506 /* 4507 * If priv is NULL, it's probably because the caam driver wasn't 4508 * properly initialized (e.g. RNG4 init failed). Thus, bail out here. 4509 */ 4510 if (!priv) 4511 return -ENODEV; 4512 4513 4514 INIT_LIST_HEAD(&alg_list); 4515 4516 /* 4517 * Register crypto algorithms the device supports. 4518 * First, detect presence and attributes of DES, AES, and MD blocks. 4519 */ 4520 cha_vid = rd_reg32(&priv->ctrl->perfmon.cha_id_ls); 4521 cha_inst = rd_reg32(&priv->ctrl->perfmon.cha_num_ls); 4522 des_inst = (cha_inst & CHA_ID_LS_DES_MASK) >> CHA_ID_LS_DES_SHIFT; 4523 aes_inst = (cha_inst & CHA_ID_LS_AES_MASK) >> CHA_ID_LS_AES_SHIFT; 4524 md_inst = (cha_inst & CHA_ID_LS_MD_MASK) >> CHA_ID_LS_MD_SHIFT; 4525 4526 /* If MD is present, limit digest size based on LP256 */ 4527 if (md_inst && ((cha_vid & CHA_ID_LS_MD_MASK) == CHA_ID_LS_MD_LP256)) 4528 md_limit = SHA256_DIGEST_SIZE; 4529 4530 for (i = 0; i < ARRAY_SIZE(driver_algs); i++) { 4531 struct caam_crypto_alg *t_alg; 4532 struct caam_alg_template *alg = driver_algs + i; 4533 u32 alg_sel = alg->class1_alg_type & OP_ALG_ALGSEL_MASK; 4534 4535 /* Skip DES algorithms if not supported by device */ 4536 if (!des_inst && 4537 ((alg_sel == OP_ALG_ALGSEL_3DES) || 4538 (alg_sel == OP_ALG_ALGSEL_DES))) 4539 continue; 4540 4541 /* Skip AES algorithms if not supported by device */ 4542 if (!aes_inst && (alg_sel == OP_ALG_ALGSEL_AES)) 4543 continue; 4544 4545 t_alg = caam_alg_alloc(alg); 4546 if (IS_ERR(t_alg)) { 4547 err = PTR_ERR(t_alg); 4548 pr_warn("%s alg allocation failed\n", alg->driver_name); 4549 continue; 4550 } 4551 4552 err = crypto_register_alg(&t_alg->crypto_alg); 4553 if (err) { 4554 pr_warn("%s alg registration failed\n", 4555 t_alg->crypto_alg.cra_driver_name); 4556 kfree(t_alg); 4557 continue; 4558 } 4559 4560 list_add_tail(&t_alg->entry, &alg_list); 4561 registered = true; 4562 } 4563 4564 for (i = 0; i < ARRAY_SIZE(driver_aeads); i++) { 4565 struct caam_aead_alg *t_alg = driver_aeads + i; 4566 u32 c1_alg_sel = t_alg->caam.class1_alg_type & 4567 OP_ALG_ALGSEL_MASK; 4568 u32 c2_alg_sel = t_alg->caam.class2_alg_type & 4569 OP_ALG_ALGSEL_MASK; 4570 u32 alg_aai = t_alg->caam.class1_alg_type & OP_ALG_AAI_MASK; 4571 4572 /* Skip DES algorithms if not supported by device */ 4573 if (!des_inst && 4574 ((c1_alg_sel == OP_ALG_ALGSEL_3DES) || 4575 (c1_alg_sel == OP_ALG_ALGSEL_DES))) 4576 continue; 4577 4578 /* Skip AES algorithms if not supported by device */ 4579 if (!aes_inst && (c1_alg_sel == OP_ALG_ALGSEL_AES)) 4580 continue; 4581 4582 /* 4583 * Check support for AES algorithms not available 4584 * on LP devices. 4585 */ 4586 if ((cha_vid & CHA_ID_LS_AES_MASK) == CHA_ID_LS_AES_LP) 4587 if (alg_aai == OP_ALG_AAI_GCM) 4588 continue; 4589 4590 /* 4591 * Skip algorithms requiring message digests 4592 * if MD or MD size is not supported by device. 4593 */ 4594 if (c2_alg_sel && 4595 (!md_inst || (t_alg->aead.maxauthsize > md_limit))) 4596 continue; 4597 4598 caam_aead_alg_init(t_alg); 4599 4600 err = crypto_register_aead(&t_alg->aead); 4601 if (err) { 4602 pr_warn("%s alg registration failed\n", 4603 t_alg->aead.base.cra_driver_name); 4604 continue; 4605 } 4606 4607 t_alg->registered = true; 4608 registered = true; 4609 } 4610 4611 if (registered) 4612 pr_info("caam algorithms registered in /proc/crypto\n"); 4613 4614 return err; 4615 } 4616 4617 module_init(caam_algapi_init); 4618 module_exit(caam_algapi_exit); 4619 4620 MODULE_LICENSE("GPL"); 4621 MODULE_DESCRIPTION("FSL CAAM support for crypto API"); 4622 MODULE_AUTHOR("Freescale Semiconductor - NMG/STC"); 4623