1 /* 2 * caam - Freescale FSL CAAM support for crypto API 3 * 4 * Copyright 2008-2011 Freescale Semiconductor, Inc. 5 * 6 * Based on talitos crypto API driver. 7 * 8 * relationship of job descriptors to shared descriptors (SteveC Dec 10 2008): 9 * 10 * --------------- --------------- 11 * | JobDesc #1 |-------------------->| ShareDesc | 12 * | *(packet 1) | | (PDB) | 13 * --------------- |------------->| (hashKey) | 14 * . | | (cipherKey) | 15 * . | |-------->| (operation) | 16 * --------------- | | --------------- 17 * | JobDesc #2 |------| | 18 * | *(packet 2) | | 19 * --------------- | 20 * . | 21 * . | 22 * --------------- | 23 * | JobDesc #3 |------------ 24 * | *(packet 3) | 25 * --------------- 26 * 27 * The SharedDesc never changes for a connection unless rekeyed, but 28 * each packet will likely be in a different place. So all we need 29 * to know to process the packet is where the input is, where the 30 * output goes, and what context we want to process with. Context is 31 * in the SharedDesc, packet references in the JobDesc. 32 * 33 * So, a job desc looks like: 34 * 35 * --------------------- 36 * | Header | 37 * | ShareDesc Pointer | 38 * | SEQ_OUT_PTR | 39 * | (output buffer) | 40 * | (output length) | 41 * | SEQ_IN_PTR | 42 * | (input buffer) | 43 * | (input length) | 44 * --------------------- 45 */ 46 47 #include "compat.h" 48 49 #include "regs.h" 50 #include "intern.h" 51 #include "desc_constr.h" 52 #include "jr.h" 53 #include "error.h" 54 #include "sg_sw_sec4.h" 55 #include "key_gen.h" 56 57 /* 58 * crypto alg 59 */ 60 #define CAAM_CRA_PRIORITY 3000 61 /* max key is sum of AES_MAX_KEY_SIZE, max split key size */ 62 #define CAAM_MAX_KEY_SIZE (AES_MAX_KEY_SIZE + \ 63 CTR_RFC3686_NONCE_SIZE + \ 64 SHA512_DIGEST_SIZE * 2) 65 /* max IV is max of AES_BLOCK_SIZE, DES3_EDE_BLOCK_SIZE */ 66 #define CAAM_MAX_IV_LENGTH 16 67 68 #define AEAD_DESC_JOB_IO_LEN (DESC_JOB_IO_LEN + CAAM_CMD_SZ * 2) 69 #define GCM_DESC_JOB_IO_LEN (AEAD_DESC_JOB_IO_LEN + \ 70 CAAM_CMD_SZ * 4) 71 #define AUTHENC_DESC_JOB_IO_LEN (AEAD_DESC_JOB_IO_LEN + \ 72 CAAM_CMD_SZ * 5) 73 74 /* length of descriptors text */ 75 #define DESC_AEAD_BASE (4 * CAAM_CMD_SZ) 76 #define DESC_AEAD_ENC_LEN (DESC_AEAD_BASE + 11 * CAAM_CMD_SZ) 77 #define DESC_AEAD_DEC_LEN (DESC_AEAD_BASE + 15 * CAAM_CMD_SZ) 78 #define DESC_AEAD_GIVENC_LEN (DESC_AEAD_ENC_LEN + 9 * CAAM_CMD_SZ) 79 80 /* Note: Nonce is counted in enckeylen */ 81 #define DESC_AEAD_CTR_RFC3686_LEN (4 * CAAM_CMD_SZ) 82 83 #define DESC_AEAD_NULL_BASE (3 * CAAM_CMD_SZ) 84 #define DESC_AEAD_NULL_ENC_LEN (DESC_AEAD_NULL_BASE + 11 * CAAM_CMD_SZ) 85 #define DESC_AEAD_NULL_DEC_LEN (DESC_AEAD_NULL_BASE + 13 * CAAM_CMD_SZ) 86 87 #define DESC_GCM_BASE (3 * CAAM_CMD_SZ) 88 #define DESC_GCM_ENC_LEN (DESC_GCM_BASE + 16 * CAAM_CMD_SZ) 89 #define DESC_GCM_DEC_LEN (DESC_GCM_BASE + 12 * CAAM_CMD_SZ) 90 91 #define DESC_RFC4106_BASE (3 * CAAM_CMD_SZ) 92 #define DESC_RFC4106_ENC_LEN (DESC_RFC4106_BASE + 13 * CAAM_CMD_SZ) 93 #define DESC_RFC4106_DEC_LEN (DESC_RFC4106_BASE + 13 * CAAM_CMD_SZ) 94 95 #define DESC_RFC4543_BASE (3 * CAAM_CMD_SZ) 96 #define DESC_RFC4543_ENC_LEN (DESC_RFC4543_BASE + 11 * CAAM_CMD_SZ) 97 #define DESC_RFC4543_DEC_LEN (DESC_RFC4543_BASE + 12 * CAAM_CMD_SZ) 98 99 #define DESC_ABLKCIPHER_BASE (3 * CAAM_CMD_SZ) 100 #define DESC_ABLKCIPHER_ENC_LEN (DESC_ABLKCIPHER_BASE + \ 101 20 * CAAM_CMD_SZ) 102 #define DESC_ABLKCIPHER_DEC_LEN (DESC_ABLKCIPHER_BASE + \ 103 15 * CAAM_CMD_SZ) 104 105 #define DESC_MAX_USED_BYTES (CAAM_DESC_BYTES_MAX - DESC_JOB_IO_LEN) 106 #define DESC_MAX_USED_LEN (DESC_MAX_USED_BYTES / CAAM_CMD_SZ) 107 108 #ifdef DEBUG 109 /* for print_hex_dumps with line references */ 110 #define debug(format, arg...) printk(format, arg) 111 #else 112 #define debug(format, arg...) 113 #endif 114 static struct list_head alg_list; 115 116 struct caam_alg_entry { 117 int class1_alg_type; 118 int class2_alg_type; 119 int alg_op; 120 bool rfc3686; 121 bool geniv; 122 }; 123 124 struct caam_aead_alg { 125 struct aead_alg aead; 126 struct caam_alg_entry caam; 127 bool registered; 128 }; 129 130 /* Set DK bit in class 1 operation if shared */ 131 static inline void append_dec_op1(u32 *desc, u32 type) 132 { 133 u32 *jump_cmd, *uncond_jump_cmd; 134 135 /* DK bit is valid only for AES */ 136 if ((type & OP_ALG_ALGSEL_MASK) != OP_ALG_ALGSEL_AES) { 137 append_operation(desc, type | OP_ALG_AS_INITFINAL | 138 OP_ALG_DECRYPT); 139 return; 140 } 141 142 jump_cmd = append_jump(desc, JUMP_TEST_ALL | JUMP_COND_SHRD); 143 append_operation(desc, type | OP_ALG_AS_INITFINAL | 144 OP_ALG_DECRYPT); 145 uncond_jump_cmd = append_jump(desc, JUMP_TEST_ALL); 146 set_jump_tgt_here(desc, jump_cmd); 147 append_operation(desc, type | OP_ALG_AS_INITFINAL | 148 OP_ALG_DECRYPT | OP_ALG_AAI_DK); 149 set_jump_tgt_here(desc, uncond_jump_cmd); 150 } 151 152 /* 153 * For aead functions, read payload and write payload, 154 * both of which are specified in req->src and req->dst 155 */ 156 static inline void aead_append_src_dst(u32 *desc, u32 msg_type) 157 { 158 append_seq_fifo_store(desc, 0, FIFOST_TYPE_MESSAGE_DATA | KEY_VLF); 159 append_seq_fifo_load(desc, 0, FIFOLD_CLASS_BOTH | 160 KEY_VLF | msg_type | FIFOLD_TYPE_LASTBOTH); 161 } 162 163 /* 164 * For ablkcipher encrypt and decrypt, read from req->src and 165 * write to req->dst 166 */ 167 static inline void ablkcipher_append_src_dst(u32 *desc) 168 { 169 append_math_add(desc, VARSEQOUTLEN, SEQINLEN, REG0, CAAM_CMD_SZ); 170 append_math_add(desc, VARSEQINLEN, SEQINLEN, REG0, CAAM_CMD_SZ); 171 append_seq_fifo_load(desc, 0, FIFOLD_CLASS_CLASS1 | 172 KEY_VLF | FIFOLD_TYPE_MSG | FIFOLD_TYPE_LAST1); 173 append_seq_fifo_store(desc, 0, FIFOST_TYPE_MESSAGE_DATA | KEY_VLF); 174 } 175 176 /* 177 * per-session context 178 */ 179 struct caam_ctx { 180 struct device *jrdev; 181 u32 sh_desc_enc[DESC_MAX_USED_LEN]; 182 u32 sh_desc_dec[DESC_MAX_USED_LEN]; 183 u32 sh_desc_givenc[DESC_MAX_USED_LEN]; 184 dma_addr_t sh_desc_enc_dma; 185 dma_addr_t sh_desc_dec_dma; 186 dma_addr_t sh_desc_givenc_dma; 187 u32 class1_alg_type; 188 u32 class2_alg_type; 189 u32 alg_op; 190 u8 key[CAAM_MAX_KEY_SIZE]; 191 dma_addr_t key_dma; 192 unsigned int enckeylen; 193 unsigned int split_key_len; 194 unsigned int split_key_pad_len; 195 unsigned int authsize; 196 }; 197 198 static void append_key_aead(u32 *desc, struct caam_ctx *ctx, 199 int keys_fit_inline, bool is_rfc3686) 200 { 201 u32 *nonce; 202 unsigned int enckeylen = ctx->enckeylen; 203 204 /* 205 * RFC3686 specific: 206 * | ctx->key = {AUTH_KEY, ENC_KEY, NONCE} 207 * | enckeylen = encryption key size + nonce size 208 */ 209 if (is_rfc3686) 210 enckeylen -= CTR_RFC3686_NONCE_SIZE; 211 212 if (keys_fit_inline) { 213 append_key_as_imm(desc, ctx->key, ctx->split_key_pad_len, 214 ctx->split_key_len, CLASS_2 | 215 KEY_DEST_MDHA_SPLIT | KEY_ENC); 216 append_key_as_imm(desc, (void *)ctx->key + 217 ctx->split_key_pad_len, enckeylen, 218 enckeylen, CLASS_1 | KEY_DEST_CLASS_REG); 219 } else { 220 append_key(desc, ctx->key_dma, ctx->split_key_len, CLASS_2 | 221 KEY_DEST_MDHA_SPLIT | KEY_ENC); 222 append_key(desc, ctx->key_dma + ctx->split_key_pad_len, 223 enckeylen, CLASS_1 | KEY_DEST_CLASS_REG); 224 } 225 226 /* Load Counter into CONTEXT1 reg */ 227 if (is_rfc3686) { 228 nonce = (u32 *)((void *)ctx->key + ctx->split_key_pad_len + 229 enckeylen); 230 append_load_imm_u32(desc, *nonce, LDST_CLASS_IND_CCB | 231 LDST_SRCDST_BYTE_OUTFIFO | LDST_IMM); 232 append_move(desc, 233 MOVE_SRC_OUTFIFO | 234 MOVE_DEST_CLASS1CTX | 235 (16 << MOVE_OFFSET_SHIFT) | 236 (CTR_RFC3686_NONCE_SIZE << MOVE_LEN_SHIFT)); 237 } 238 } 239 240 static void init_sh_desc_key_aead(u32 *desc, struct caam_ctx *ctx, 241 int keys_fit_inline, bool is_rfc3686) 242 { 243 u32 *key_jump_cmd; 244 245 /* Note: Context registers are saved. */ 246 init_sh_desc(desc, HDR_SHARE_SERIAL | HDR_SAVECTX); 247 248 /* Skip if already shared */ 249 key_jump_cmd = append_jump(desc, JUMP_JSL | JUMP_TEST_ALL | 250 JUMP_COND_SHRD); 251 252 append_key_aead(desc, ctx, keys_fit_inline, is_rfc3686); 253 254 set_jump_tgt_here(desc, key_jump_cmd); 255 } 256 257 static int aead_null_set_sh_desc(struct crypto_aead *aead) 258 { 259 struct caam_ctx *ctx = crypto_aead_ctx(aead); 260 struct device *jrdev = ctx->jrdev; 261 bool keys_fit_inline = false; 262 u32 *key_jump_cmd, *jump_cmd, *read_move_cmd, *write_move_cmd; 263 u32 *desc; 264 265 /* 266 * Job Descriptor and Shared Descriptors 267 * must all fit into the 64-word Descriptor h/w Buffer 268 */ 269 if (DESC_AEAD_NULL_ENC_LEN + AEAD_DESC_JOB_IO_LEN + 270 ctx->split_key_pad_len <= CAAM_DESC_BYTES_MAX) 271 keys_fit_inline = true; 272 273 /* aead_encrypt shared descriptor */ 274 desc = ctx->sh_desc_enc; 275 276 init_sh_desc(desc, HDR_SHARE_SERIAL); 277 278 /* Skip if already shared */ 279 key_jump_cmd = append_jump(desc, JUMP_JSL | JUMP_TEST_ALL | 280 JUMP_COND_SHRD); 281 if (keys_fit_inline) 282 append_key_as_imm(desc, ctx->key, ctx->split_key_pad_len, 283 ctx->split_key_len, CLASS_2 | 284 KEY_DEST_MDHA_SPLIT | KEY_ENC); 285 else 286 append_key(desc, ctx->key_dma, ctx->split_key_len, CLASS_2 | 287 KEY_DEST_MDHA_SPLIT | KEY_ENC); 288 set_jump_tgt_here(desc, key_jump_cmd); 289 290 /* assoclen + cryptlen = seqinlen */ 291 append_math_sub(desc, REG3, SEQINLEN, REG0, CAAM_CMD_SZ); 292 293 /* Prepare to read and write cryptlen + assoclen bytes */ 294 append_math_add(desc, VARSEQINLEN, ZERO, REG3, CAAM_CMD_SZ); 295 append_math_add(desc, VARSEQOUTLEN, ZERO, REG3, CAAM_CMD_SZ); 296 297 /* 298 * MOVE_LEN opcode is not available in all SEC HW revisions, 299 * thus need to do some magic, i.e. self-patch the descriptor 300 * buffer. 301 */ 302 read_move_cmd = append_move(desc, MOVE_SRC_DESCBUF | 303 MOVE_DEST_MATH3 | 304 (0x6 << MOVE_LEN_SHIFT)); 305 write_move_cmd = append_move(desc, MOVE_SRC_MATH3 | 306 MOVE_DEST_DESCBUF | 307 MOVE_WAITCOMP | 308 (0x8 << MOVE_LEN_SHIFT)); 309 310 /* Class 2 operation */ 311 append_operation(desc, ctx->class2_alg_type | 312 OP_ALG_AS_INITFINAL | OP_ALG_ENCRYPT); 313 314 /* Read and write cryptlen bytes */ 315 aead_append_src_dst(desc, FIFOLD_TYPE_MSG | FIFOLD_TYPE_FLUSH1); 316 317 set_move_tgt_here(desc, read_move_cmd); 318 set_move_tgt_here(desc, write_move_cmd); 319 append_cmd(desc, CMD_LOAD | DISABLE_AUTO_INFO_FIFO); 320 append_move(desc, MOVE_SRC_INFIFO_CL | MOVE_DEST_OUTFIFO | 321 MOVE_AUX_LS); 322 323 /* Write ICV */ 324 append_seq_store(desc, ctx->authsize, LDST_CLASS_2_CCB | 325 LDST_SRCDST_BYTE_CONTEXT); 326 327 ctx->sh_desc_enc_dma = dma_map_single(jrdev, desc, 328 desc_bytes(desc), 329 DMA_TO_DEVICE); 330 if (dma_mapping_error(jrdev, ctx->sh_desc_enc_dma)) { 331 dev_err(jrdev, "unable to map shared descriptor\n"); 332 return -ENOMEM; 333 } 334 #ifdef DEBUG 335 print_hex_dump(KERN_ERR, 336 "aead null enc shdesc@"__stringify(__LINE__)": ", 337 DUMP_PREFIX_ADDRESS, 16, 4, desc, 338 desc_bytes(desc), 1); 339 #endif 340 341 /* 342 * Job Descriptor and Shared Descriptors 343 * must all fit into the 64-word Descriptor h/w Buffer 344 */ 345 keys_fit_inline = false; 346 if (DESC_AEAD_NULL_DEC_LEN + DESC_JOB_IO_LEN + 347 ctx->split_key_pad_len <= CAAM_DESC_BYTES_MAX) 348 keys_fit_inline = true; 349 350 desc = ctx->sh_desc_dec; 351 352 /* aead_decrypt shared descriptor */ 353 init_sh_desc(desc, HDR_SHARE_SERIAL); 354 355 /* Skip if already shared */ 356 key_jump_cmd = append_jump(desc, JUMP_JSL | JUMP_TEST_ALL | 357 JUMP_COND_SHRD); 358 if (keys_fit_inline) 359 append_key_as_imm(desc, ctx->key, ctx->split_key_pad_len, 360 ctx->split_key_len, CLASS_2 | 361 KEY_DEST_MDHA_SPLIT | KEY_ENC); 362 else 363 append_key(desc, ctx->key_dma, ctx->split_key_len, CLASS_2 | 364 KEY_DEST_MDHA_SPLIT | KEY_ENC); 365 set_jump_tgt_here(desc, key_jump_cmd); 366 367 /* Class 2 operation */ 368 append_operation(desc, ctx->class2_alg_type | 369 OP_ALG_AS_INITFINAL | OP_ALG_DECRYPT | OP_ALG_ICV_ON); 370 371 /* assoclen + cryptlen = seqoutlen */ 372 append_math_sub(desc, REG2, SEQOUTLEN, REG0, CAAM_CMD_SZ); 373 374 /* Prepare to read and write cryptlen + assoclen bytes */ 375 append_math_add(desc, VARSEQINLEN, ZERO, REG2, CAAM_CMD_SZ); 376 append_math_add(desc, VARSEQOUTLEN, ZERO, REG2, CAAM_CMD_SZ); 377 378 /* 379 * MOVE_LEN opcode is not available in all SEC HW revisions, 380 * thus need to do some magic, i.e. self-patch the descriptor 381 * buffer. 382 */ 383 read_move_cmd = append_move(desc, MOVE_SRC_DESCBUF | 384 MOVE_DEST_MATH2 | 385 (0x6 << MOVE_LEN_SHIFT)); 386 write_move_cmd = append_move(desc, MOVE_SRC_MATH2 | 387 MOVE_DEST_DESCBUF | 388 MOVE_WAITCOMP | 389 (0x8 << MOVE_LEN_SHIFT)); 390 391 /* Read and write cryptlen bytes */ 392 aead_append_src_dst(desc, FIFOLD_TYPE_MSG | FIFOLD_TYPE_FLUSH1); 393 394 /* 395 * Insert a NOP here, since we need at least 4 instructions between 396 * code patching the descriptor buffer and the location being patched. 397 */ 398 jump_cmd = append_jump(desc, JUMP_TEST_ALL); 399 set_jump_tgt_here(desc, jump_cmd); 400 401 set_move_tgt_here(desc, read_move_cmd); 402 set_move_tgt_here(desc, write_move_cmd); 403 append_cmd(desc, CMD_LOAD | DISABLE_AUTO_INFO_FIFO); 404 append_move(desc, MOVE_SRC_INFIFO_CL | MOVE_DEST_OUTFIFO | 405 MOVE_AUX_LS); 406 append_cmd(desc, CMD_LOAD | ENABLE_AUTO_INFO_FIFO); 407 408 /* Load ICV */ 409 append_seq_fifo_load(desc, ctx->authsize, FIFOLD_CLASS_CLASS2 | 410 FIFOLD_TYPE_LAST2 | FIFOLD_TYPE_ICV); 411 412 ctx->sh_desc_dec_dma = dma_map_single(jrdev, desc, 413 desc_bytes(desc), 414 DMA_TO_DEVICE); 415 if (dma_mapping_error(jrdev, ctx->sh_desc_dec_dma)) { 416 dev_err(jrdev, "unable to map shared descriptor\n"); 417 return -ENOMEM; 418 } 419 #ifdef DEBUG 420 print_hex_dump(KERN_ERR, 421 "aead null dec shdesc@"__stringify(__LINE__)": ", 422 DUMP_PREFIX_ADDRESS, 16, 4, desc, 423 desc_bytes(desc), 1); 424 #endif 425 426 return 0; 427 } 428 429 static int aead_set_sh_desc(struct crypto_aead *aead) 430 { 431 struct caam_aead_alg *alg = container_of(crypto_aead_alg(aead), 432 struct caam_aead_alg, aead); 433 unsigned int ivsize = crypto_aead_ivsize(aead); 434 struct caam_ctx *ctx = crypto_aead_ctx(aead); 435 struct device *jrdev = ctx->jrdev; 436 bool keys_fit_inline; 437 u32 geniv, moveiv; 438 u32 ctx1_iv_off = 0; 439 u32 *desc; 440 const bool ctr_mode = ((ctx->class1_alg_type & OP_ALG_AAI_MASK) == 441 OP_ALG_AAI_CTR_MOD128); 442 const bool is_rfc3686 = alg->caam.rfc3686; 443 444 /* NULL encryption / decryption */ 445 if (!ctx->enckeylen) 446 return aead_null_set_sh_desc(aead); 447 448 /* 449 * AES-CTR needs to load IV in CONTEXT1 reg 450 * at an offset of 128bits (16bytes) 451 * CONTEXT1[255:128] = IV 452 */ 453 if (ctr_mode) 454 ctx1_iv_off = 16; 455 456 /* 457 * RFC3686 specific: 458 * CONTEXT1[255:128] = {NONCE, IV, COUNTER} 459 */ 460 if (is_rfc3686) 461 ctx1_iv_off = 16 + CTR_RFC3686_NONCE_SIZE; 462 463 if (alg->caam.geniv) 464 goto skip_enc; 465 466 /* 467 * Job Descriptor and Shared Descriptors 468 * must all fit into the 64-word Descriptor h/w Buffer 469 */ 470 keys_fit_inline = false; 471 if (DESC_AEAD_ENC_LEN + AUTHENC_DESC_JOB_IO_LEN + 472 ctx->split_key_pad_len + ctx->enckeylen + 473 (is_rfc3686 ? DESC_AEAD_CTR_RFC3686_LEN : 0) <= 474 CAAM_DESC_BYTES_MAX) 475 keys_fit_inline = true; 476 477 /* aead_encrypt shared descriptor */ 478 desc = ctx->sh_desc_enc; 479 480 /* Note: Context registers are saved. */ 481 init_sh_desc_key_aead(desc, ctx, keys_fit_inline, is_rfc3686); 482 483 /* Class 2 operation */ 484 append_operation(desc, ctx->class2_alg_type | 485 OP_ALG_AS_INITFINAL | OP_ALG_ENCRYPT); 486 487 /* Read and write assoclen bytes */ 488 append_math_add(desc, VARSEQINLEN, ZERO, REG3, CAAM_CMD_SZ); 489 append_math_add(desc, VARSEQOUTLEN, ZERO, REG3, CAAM_CMD_SZ); 490 491 /* Skip assoc data */ 492 append_seq_fifo_store(desc, 0, FIFOST_TYPE_SKIP | FIFOLDST_VLF); 493 494 /* read assoc before reading payload */ 495 append_seq_fifo_load(desc, 0, FIFOLD_CLASS_CLASS2 | FIFOLD_TYPE_MSG | 496 FIFOLDST_VLF); 497 498 /* Load Counter into CONTEXT1 reg */ 499 if (is_rfc3686) 500 append_load_imm_u32(desc, be32_to_cpu(1), LDST_IMM | 501 LDST_CLASS_1_CCB | 502 LDST_SRCDST_BYTE_CONTEXT | 503 ((ctx1_iv_off + CTR_RFC3686_IV_SIZE) << 504 LDST_OFFSET_SHIFT)); 505 506 /* Class 1 operation */ 507 append_operation(desc, ctx->class1_alg_type | 508 OP_ALG_AS_INITFINAL | OP_ALG_ENCRYPT); 509 510 /* Read and write cryptlen bytes */ 511 append_math_add(desc, VARSEQINLEN, SEQINLEN, REG0, CAAM_CMD_SZ); 512 append_math_add(desc, VARSEQOUTLEN, SEQINLEN, REG0, CAAM_CMD_SZ); 513 aead_append_src_dst(desc, FIFOLD_TYPE_MSG1OUT2); 514 515 /* Write ICV */ 516 append_seq_store(desc, ctx->authsize, LDST_CLASS_2_CCB | 517 LDST_SRCDST_BYTE_CONTEXT); 518 519 ctx->sh_desc_enc_dma = dma_map_single(jrdev, desc, 520 desc_bytes(desc), 521 DMA_TO_DEVICE); 522 if (dma_mapping_error(jrdev, ctx->sh_desc_enc_dma)) { 523 dev_err(jrdev, "unable to map shared descriptor\n"); 524 return -ENOMEM; 525 } 526 #ifdef DEBUG 527 print_hex_dump(KERN_ERR, "aead enc shdesc@"__stringify(__LINE__)": ", 528 DUMP_PREFIX_ADDRESS, 16, 4, desc, 529 desc_bytes(desc), 1); 530 #endif 531 532 skip_enc: 533 /* 534 * Job Descriptor and Shared Descriptors 535 * must all fit into the 64-word Descriptor h/w Buffer 536 */ 537 keys_fit_inline = false; 538 if (DESC_AEAD_DEC_LEN + AUTHENC_DESC_JOB_IO_LEN + 539 ctx->split_key_pad_len + ctx->enckeylen + 540 (is_rfc3686 ? DESC_AEAD_CTR_RFC3686_LEN : 0) <= 541 CAAM_DESC_BYTES_MAX) 542 keys_fit_inline = true; 543 544 /* aead_decrypt shared descriptor */ 545 desc = ctx->sh_desc_dec; 546 547 /* Note: Context registers are saved. */ 548 init_sh_desc_key_aead(desc, ctx, keys_fit_inline, is_rfc3686); 549 550 /* Class 2 operation */ 551 append_operation(desc, ctx->class2_alg_type | 552 OP_ALG_AS_INITFINAL | OP_ALG_DECRYPT | OP_ALG_ICV_ON); 553 554 /* Read and write assoclen bytes */ 555 append_math_add(desc, VARSEQINLEN, ZERO, REG3, CAAM_CMD_SZ); 556 append_math_add(desc, VARSEQOUTLEN, ZERO, REG3, CAAM_CMD_SZ); 557 558 /* Skip assoc data */ 559 append_seq_fifo_store(desc, 0, FIFOST_TYPE_SKIP | FIFOLDST_VLF); 560 561 /* read assoc before reading payload */ 562 append_seq_fifo_load(desc, 0, FIFOLD_CLASS_CLASS2 | FIFOLD_TYPE_MSG | 563 KEY_VLF); 564 565 /* Load Counter into CONTEXT1 reg */ 566 if (is_rfc3686) 567 append_load_imm_u32(desc, be32_to_cpu(1), LDST_IMM | 568 LDST_CLASS_1_CCB | 569 LDST_SRCDST_BYTE_CONTEXT | 570 ((ctx1_iv_off + CTR_RFC3686_IV_SIZE) << 571 LDST_OFFSET_SHIFT)); 572 573 /* Choose operation */ 574 if (ctr_mode) 575 append_operation(desc, ctx->class1_alg_type | 576 OP_ALG_AS_INITFINAL | OP_ALG_DECRYPT); 577 else 578 append_dec_op1(desc, ctx->class1_alg_type); 579 580 /* Read and write cryptlen bytes */ 581 append_math_add(desc, VARSEQINLEN, SEQOUTLEN, REG0, CAAM_CMD_SZ); 582 append_math_add(desc, VARSEQOUTLEN, SEQOUTLEN, REG0, CAAM_CMD_SZ); 583 aead_append_src_dst(desc, FIFOLD_TYPE_MSG); 584 585 /* Load ICV */ 586 append_seq_fifo_load(desc, ctx->authsize, FIFOLD_CLASS_CLASS2 | 587 FIFOLD_TYPE_LAST2 | FIFOLD_TYPE_ICV); 588 589 ctx->sh_desc_dec_dma = dma_map_single(jrdev, desc, 590 desc_bytes(desc), 591 DMA_TO_DEVICE); 592 if (dma_mapping_error(jrdev, ctx->sh_desc_dec_dma)) { 593 dev_err(jrdev, "unable to map shared descriptor\n"); 594 return -ENOMEM; 595 } 596 #ifdef DEBUG 597 print_hex_dump(KERN_ERR, "aead dec shdesc@"__stringify(__LINE__)": ", 598 DUMP_PREFIX_ADDRESS, 16, 4, desc, 599 desc_bytes(desc), 1); 600 #endif 601 602 if (!alg->caam.geniv) 603 goto skip_givenc; 604 605 /* 606 * Job Descriptor and Shared Descriptors 607 * must all fit into the 64-word Descriptor h/w Buffer 608 */ 609 keys_fit_inline = false; 610 if (DESC_AEAD_GIVENC_LEN + AUTHENC_DESC_JOB_IO_LEN + 611 ctx->split_key_pad_len + ctx->enckeylen + 612 (is_rfc3686 ? DESC_AEAD_CTR_RFC3686_LEN : 0) <= 613 CAAM_DESC_BYTES_MAX) 614 keys_fit_inline = true; 615 616 /* aead_givencrypt shared descriptor */ 617 desc = ctx->sh_desc_givenc; 618 619 /* Note: Context registers are saved. */ 620 init_sh_desc_key_aead(desc, ctx, keys_fit_inline, is_rfc3686); 621 622 if (is_rfc3686) 623 goto copy_iv; 624 625 /* Generate IV */ 626 geniv = NFIFOENTRY_STYPE_PAD | NFIFOENTRY_DEST_DECO | 627 NFIFOENTRY_DTYPE_MSG | NFIFOENTRY_LC1 | 628 NFIFOENTRY_PTYPE_RND | (ivsize << NFIFOENTRY_DLEN_SHIFT); 629 append_load_imm_u32(desc, geniv, LDST_CLASS_IND_CCB | 630 LDST_SRCDST_WORD_INFO_FIFO | LDST_IMM); 631 append_cmd(desc, CMD_LOAD | DISABLE_AUTO_INFO_FIFO); 632 append_move(desc, MOVE_WAITCOMP | 633 MOVE_SRC_INFIFO | MOVE_DEST_CLASS1CTX | 634 (ctx1_iv_off << MOVE_OFFSET_SHIFT) | 635 (ivsize << MOVE_LEN_SHIFT)); 636 append_cmd(desc, CMD_LOAD | ENABLE_AUTO_INFO_FIFO); 637 638 copy_iv: 639 /* Copy IV to class 1 context */ 640 append_move(desc, MOVE_SRC_CLASS1CTX | MOVE_DEST_OUTFIFO | 641 (ctx1_iv_off << MOVE_OFFSET_SHIFT) | 642 (ivsize << MOVE_LEN_SHIFT)); 643 644 /* Return to encryption */ 645 append_operation(desc, ctx->class2_alg_type | 646 OP_ALG_AS_INITFINAL | OP_ALG_ENCRYPT); 647 648 /* ivsize + cryptlen = seqoutlen - authsize */ 649 append_math_sub_imm_u32(desc, REG3, SEQOUTLEN, IMM, ctx->authsize); 650 651 /* Read and write assoclen bytes */ 652 append_math_add(desc, VARSEQINLEN, ZERO, REG3, CAAM_CMD_SZ); 653 append_math_add(desc, VARSEQOUTLEN, ZERO, REG3, CAAM_CMD_SZ); 654 655 /* Skip assoc data */ 656 append_seq_fifo_store(desc, 0, FIFOST_TYPE_SKIP | FIFOLDST_VLF); 657 658 /* read assoc before reading payload */ 659 append_seq_fifo_load(desc, 0, FIFOLD_CLASS_CLASS2 | FIFOLD_TYPE_MSG | 660 KEY_VLF); 661 662 /* Copy iv from outfifo to class 2 fifo */ 663 moveiv = NFIFOENTRY_STYPE_OFIFO | NFIFOENTRY_DEST_CLASS2 | 664 NFIFOENTRY_DTYPE_MSG | (ivsize << NFIFOENTRY_DLEN_SHIFT); 665 append_load_imm_u32(desc, moveiv, LDST_CLASS_IND_CCB | 666 LDST_SRCDST_WORD_INFO_FIFO | LDST_IMM); 667 append_load_imm_u32(desc, ivsize, LDST_CLASS_2_CCB | 668 LDST_SRCDST_WORD_DATASZ_REG | LDST_IMM); 669 670 /* Load Counter into CONTEXT1 reg */ 671 if (is_rfc3686) 672 append_load_imm_u32(desc, be32_to_cpu(1), LDST_IMM | 673 LDST_CLASS_1_CCB | 674 LDST_SRCDST_BYTE_CONTEXT | 675 ((ctx1_iv_off + CTR_RFC3686_IV_SIZE) << 676 LDST_OFFSET_SHIFT)); 677 678 /* Class 1 operation */ 679 append_operation(desc, ctx->class1_alg_type | 680 OP_ALG_AS_INITFINAL | OP_ALG_ENCRYPT); 681 682 /* Will write ivsize + cryptlen */ 683 append_math_add(desc, VARSEQOUTLEN, SEQINLEN, REG0, CAAM_CMD_SZ); 684 685 /* Not need to reload iv */ 686 append_seq_fifo_load(desc, ivsize, 687 FIFOLD_CLASS_SKIP); 688 689 /* Will read cryptlen */ 690 append_math_add(desc, VARSEQINLEN, SEQINLEN, REG0, CAAM_CMD_SZ); 691 aead_append_src_dst(desc, FIFOLD_TYPE_MSG1OUT2); 692 693 /* Write ICV */ 694 append_seq_store(desc, ctx->authsize, LDST_CLASS_2_CCB | 695 LDST_SRCDST_BYTE_CONTEXT); 696 697 ctx->sh_desc_enc_dma = dma_map_single(jrdev, desc, 698 desc_bytes(desc), 699 DMA_TO_DEVICE); 700 if (dma_mapping_error(jrdev, ctx->sh_desc_givenc_dma)) { 701 dev_err(jrdev, "unable to map shared descriptor\n"); 702 return -ENOMEM; 703 } 704 #ifdef DEBUG 705 print_hex_dump(KERN_ERR, "aead givenc shdesc@"__stringify(__LINE__)": ", 706 DUMP_PREFIX_ADDRESS, 16, 4, desc, 707 desc_bytes(desc), 1); 708 #endif 709 710 skip_givenc: 711 return 0; 712 } 713 714 static int aead_setauthsize(struct crypto_aead *authenc, 715 unsigned int authsize) 716 { 717 struct caam_ctx *ctx = crypto_aead_ctx(authenc); 718 719 ctx->authsize = authsize; 720 aead_set_sh_desc(authenc); 721 722 return 0; 723 } 724 725 static int gcm_set_sh_desc(struct crypto_aead *aead) 726 { 727 struct caam_ctx *ctx = crypto_aead_ctx(aead); 728 struct device *jrdev = ctx->jrdev; 729 bool keys_fit_inline = false; 730 u32 *key_jump_cmd, *zero_payload_jump_cmd, 731 *zero_assoc_jump_cmd1, *zero_assoc_jump_cmd2; 732 u32 *desc; 733 734 if (!ctx->enckeylen || !ctx->authsize) 735 return 0; 736 737 /* 738 * AES GCM encrypt shared descriptor 739 * Job Descriptor and Shared Descriptor 740 * must fit into the 64-word Descriptor h/w Buffer 741 */ 742 if (DESC_GCM_ENC_LEN + GCM_DESC_JOB_IO_LEN + 743 ctx->enckeylen <= CAAM_DESC_BYTES_MAX) 744 keys_fit_inline = true; 745 746 desc = ctx->sh_desc_enc; 747 748 init_sh_desc(desc, HDR_SHARE_SERIAL); 749 750 /* skip key loading if they are loaded due to sharing */ 751 key_jump_cmd = append_jump(desc, JUMP_JSL | JUMP_TEST_ALL | 752 JUMP_COND_SHRD | JUMP_COND_SELF); 753 if (keys_fit_inline) 754 append_key_as_imm(desc, (void *)ctx->key, ctx->enckeylen, 755 ctx->enckeylen, CLASS_1 | KEY_DEST_CLASS_REG); 756 else 757 append_key(desc, ctx->key_dma, ctx->enckeylen, 758 CLASS_1 | KEY_DEST_CLASS_REG); 759 set_jump_tgt_here(desc, key_jump_cmd); 760 761 /* class 1 operation */ 762 append_operation(desc, ctx->class1_alg_type | 763 OP_ALG_AS_INITFINAL | OP_ALG_ENCRYPT); 764 765 /* if assoclen + cryptlen is ZERO, skip to ICV write */ 766 append_math_sub(desc, VARSEQOUTLEN, SEQINLEN, REG0, CAAM_CMD_SZ); 767 zero_assoc_jump_cmd2 = append_jump(desc, JUMP_TEST_ALL | 768 JUMP_COND_MATH_Z); 769 770 /* if assoclen is ZERO, skip reading the assoc data */ 771 append_math_add(desc, VARSEQINLEN, ZERO, REG3, CAAM_CMD_SZ); 772 zero_assoc_jump_cmd1 = append_jump(desc, JUMP_TEST_ALL | 773 JUMP_COND_MATH_Z); 774 775 append_math_add(desc, VARSEQOUTLEN, ZERO, REG3, CAAM_CMD_SZ); 776 777 /* skip assoc data */ 778 append_seq_fifo_store(desc, 0, FIFOST_TYPE_SKIP | FIFOLDST_VLF); 779 780 /* cryptlen = seqinlen - assoclen */ 781 append_math_sub(desc, VARSEQOUTLEN, SEQINLEN, REG3, CAAM_CMD_SZ); 782 783 /* if cryptlen is ZERO jump to zero-payload commands */ 784 zero_payload_jump_cmd = append_jump(desc, JUMP_TEST_ALL | 785 JUMP_COND_MATH_Z); 786 787 /* read assoc data */ 788 append_seq_fifo_load(desc, 0, FIFOLD_CLASS_CLASS1 | FIFOLDST_VLF | 789 FIFOLD_TYPE_AAD | FIFOLD_TYPE_FLUSH1); 790 set_jump_tgt_here(desc, zero_assoc_jump_cmd1); 791 792 append_math_sub(desc, VARSEQINLEN, SEQINLEN, REG0, CAAM_CMD_SZ); 793 794 /* write encrypted data */ 795 append_seq_fifo_store(desc, 0, FIFOST_TYPE_MESSAGE_DATA | FIFOLDST_VLF); 796 797 /* read payload data */ 798 append_seq_fifo_load(desc, 0, FIFOLD_CLASS_CLASS1 | FIFOLDST_VLF | 799 FIFOLD_TYPE_MSG | FIFOLD_TYPE_LAST1); 800 801 /* jump the zero-payload commands */ 802 append_jump(desc, JUMP_TEST_ALL | 2); 803 804 /* zero-payload commands */ 805 set_jump_tgt_here(desc, zero_payload_jump_cmd); 806 807 /* read assoc data */ 808 append_seq_fifo_load(desc, 0, FIFOLD_CLASS_CLASS1 | FIFOLDST_VLF | 809 FIFOLD_TYPE_AAD | FIFOLD_TYPE_LAST1); 810 811 /* There is no input data */ 812 set_jump_tgt_here(desc, zero_assoc_jump_cmd2); 813 814 /* write ICV */ 815 append_seq_store(desc, ctx->authsize, LDST_CLASS_1_CCB | 816 LDST_SRCDST_BYTE_CONTEXT); 817 818 ctx->sh_desc_enc_dma = dma_map_single(jrdev, desc, 819 desc_bytes(desc), 820 DMA_TO_DEVICE); 821 if (dma_mapping_error(jrdev, ctx->sh_desc_enc_dma)) { 822 dev_err(jrdev, "unable to map shared descriptor\n"); 823 return -ENOMEM; 824 } 825 #ifdef DEBUG 826 print_hex_dump(KERN_ERR, "gcm enc shdesc@"__stringify(__LINE__)": ", 827 DUMP_PREFIX_ADDRESS, 16, 4, desc, 828 desc_bytes(desc), 1); 829 #endif 830 831 /* 832 * Job Descriptor and Shared Descriptors 833 * must all fit into the 64-word Descriptor h/w Buffer 834 */ 835 keys_fit_inline = false; 836 if (DESC_GCM_DEC_LEN + GCM_DESC_JOB_IO_LEN + 837 ctx->enckeylen <= CAAM_DESC_BYTES_MAX) 838 keys_fit_inline = true; 839 840 desc = ctx->sh_desc_dec; 841 842 init_sh_desc(desc, HDR_SHARE_SERIAL); 843 844 /* skip key loading if they are loaded due to sharing */ 845 key_jump_cmd = append_jump(desc, JUMP_JSL | 846 JUMP_TEST_ALL | JUMP_COND_SHRD | 847 JUMP_COND_SELF); 848 if (keys_fit_inline) 849 append_key_as_imm(desc, (void *)ctx->key, ctx->enckeylen, 850 ctx->enckeylen, CLASS_1 | KEY_DEST_CLASS_REG); 851 else 852 append_key(desc, ctx->key_dma, ctx->enckeylen, 853 CLASS_1 | KEY_DEST_CLASS_REG); 854 set_jump_tgt_here(desc, key_jump_cmd); 855 856 /* class 1 operation */ 857 append_operation(desc, ctx->class1_alg_type | 858 OP_ALG_AS_INITFINAL | OP_ALG_DECRYPT | OP_ALG_ICV_ON); 859 860 /* if assoclen is ZERO, skip reading the assoc data */ 861 append_math_add(desc, VARSEQINLEN, ZERO, REG3, CAAM_CMD_SZ); 862 zero_assoc_jump_cmd1 = append_jump(desc, JUMP_TEST_ALL | 863 JUMP_COND_MATH_Z); 864 865 append_math_add(desc, VARSEQOUTLEN, ZERO, REG3, CAAM_CMD_SZ); 866 867 /* skip assoc data */ 868 append_seq_fifo_store(desc, 0, FIFOST_TYPE_SKIP | FIFOLDST_VLF); 869 870 /* read assoc data */ 871 append_seq_fifo_load(desc, 0, FIFOLD_CLASS_CLASS1 | FIFOLDST_VLF | 872 FIFOLD_TYPE_AAD | FIFOLD_TYPE_FLUSH1); 873 874 set_jump_tgt_here(desc, zero_assoc_jump_cmd1); 875 876 /* cryptlen = seqoutlen - assoclen */ 877 append_math_sub(desc, VARSEQINLEN, SEQOUTLEN, REG0, CAAM_CMD_SZ); 878 879 /* jump to zero-payload command if cryptlen is zero */ 880 zero_payload_jump_cmd = append_jump(desc, JUMP_TEST_ALL | 881 JUMP_COND_MATH_Z); 882 883 append_math_sub(desc, VARSEQOUTLEN, SEQOUTLEN, REG0, CAAM_CMD_SZ); 884 885 /* store encrypted data */ 886 append_seq_fifo_store(desc, 0, FIFOST_TYPE_MESSAGE_DATA | FIFOLDST_VLF); 887 888 /* read payload data */ 889 append_seq_fifo_load(desc, 0, FIFOLD_CLASS_CLASS1 | FIFOLDST_VLF | 890 FIFOLD_TYPE_MSG | FIFOLD_TYPE_FLUSH1); 891 892 /* zero-payload command */ 893 set_jump_tgt_here(desc, zero_payload_jump_cmd); 894 895 /* read ICV */ 896 append_seq_fifo_load(desc, ctx->authsize, FIFOLD_CLASS_CLASS1 | 897 FIFOLD_TYPE_ICV | FIFOLD_TYPE_LAST1); 898 899 ctx->sh_desc_dec_dma = dma_map_single(jrdev, desc, 900 desc_bytes(desc), 901 DMA_TO_DEVICE); 902 if (dma_mapping_error(jrdev, ctx->sh_desc_dec_dma)) { 903 dev_err(jrdev, "unable to map shared descriptor\n"); 904 return -ENOMEM; 905 } 906 #ifdef DEBUG 907 print_hex_dump(KERN_ERR, "gcm dec shdesc@"__stringify(__LINE__)": ", 908 DUMP_PREFIX_ADDRESS, 16, 4, desc, 909 desc_bytes(desc), 1); 910 #endif 911 912 return 0; 913 } 914 915 static int gcm_setauthsize(struct crypto_aead *authenc, unsigned int authsize) 916 { 917 struct caam_ctx *ctx = crypto_aead_ctx(authenc); 918 919 ctx->authsize = authsize; 920 gcm_set_sh_desc(authenc); 921 922 return 0; 923 } 924 925 static int rfc4106_set_sh_desc(struct crypto_aead *aead) 926 { 927 struct caam_ctx *ctx = crypto_aead_ctx(aead); 928 struct device *jrdev = ctx->jrdev; 929 bool keys_fit_inline = false; 930 u32 *key_jump_cmd; 931 u32 *desc; 932 933 if (!ctx->enckeylen || !ctx->authsize) 934 return 0; 935 936 /* 937 * RFC4106 encrypt shared descriptor 938 * Job Descriptor and Shared Descriptor 939 * must fit into the 64-word Descriptor h/w Buffer 940 */ 941 if (DESC_RFC4106_ENC_LEN + GCM_DESC_JOB_IO_LEN + 942 ctx->enckeylen <= CAAM_DESC_BYTES_MAX) 943 keys_fit_inline = true; 944 945 desc = ctx->sh_desc_enc; 946 947 init_sh_desc(desc, HDR_SHARE_SERIAL); 948 949 /* Skip key loading if it is loaded due to sharing */ 950 key_jump_cmd = append_jump(desc, JUMP_JSL | JUMP_TEST_ALL | 951 JUMP_COND_SHRD); 952 if (keys_fit_inline) 953 append_key_as_imm(desc, (void *)ctx->key, ctx->enckeylen, 954 ctx->enckeylen, CLASS_1 | KEY_DEST_CLASS_REG); 955 else 956 append_key(desc, ctx->key_dma, ctx->enckeylen, 957 CLASS_1 | KEY_DEST_CLASS_REG); 958 set_jump_tgt_here(desc, key_jump_cmd); 959 960 /* Class 1 operation */ 961 append_operation(desc, ctx->class1_alg_type | 962 OP_ALG_AS_INITFINAL | OP_ALG_ENCRYPT); 963 964 append_math_sub_imm_u32(desc, VARSEQINLEN, REG3, IMM, 8); 965 append_math_add(desc, VARSEQOUTLEN, ZERO, REG3, CAAM_CMD_SZ); 966 967 /* Read assoc data */ 968 append_seq_fifo_load(desc, 0, FIFOLD_CLASS_CLASS1 | FIFOLDST_VLF | 969 FIFOLD_TYPE_AAD | FIFOLD_TYPE_FLUSH1); 970 971 /* Skip IV */ 972 append_seq_fifo_load(desc, 8, FIFOLD_CLASS_SKIP); 973 974 /* Will read cryptlen bytes */ 975 append_math_sub(desc, VARSEQINLEN, SEQINLEN, REG0, CAAM_CMD_SZ); 976 977 /* Workaround for erratum A-005473 (simultaneous SEQ FIFO skips) */ 978 append_seq_fifo_load(desc, 0, FIFOLD_CLASS_CLASS1 | FIFOLD_TYPE_MSG); 979 980 /* Skip assoc data */ 981 append_seq_fifo_store(desc, 0, FIFOST_TYPE_SKIP | FIFOLDST_VLF); 982 983 /* cryptlen = seqoutlen - assoclen */ 984 append_math_sub(desc, VARSEQOUTLEN, VARSEQINLEN, REG0, CAAM_CMD_SZ); 985 986 /* Write encrypted data */ 987 append_seq_fifo_store(desc, 0, FIFOST_TYPE_MESSAGE_DATA | FIFOLDST_VLF); 988 989 /* Read payload data */ 990 append_seq_fifo_load(desc, 0, FIFOLD_CLASS_CLASS1 | FIFOLDST_VLF | 991 FIFOLD_TYPE_MSG | FIFOLD_TYPE_LAST1); 992 993 /* Write ICV */ 994 append_seq_store(desc, ctx->authsize, LDST_CLASS_1_CCB | 995 LDST_SRCDST_BYTE_CONTEXT); 996 997 ctx->sh_desc_enc_dma = dma_map_single(jrdev, desc, 998 desc_bytes(desc), 999 DMA_TO_DEVICE); 1000 if (dma_mapping_error(jrdev, ctx->sh_desc_enc_dma)) { 1001 dev_err(jrdev, "unable to map shared descriptor\n"); 1002 return -ENOMEM; 1003 } 1004 #ifdef DEBUG 1005 print_hex_dump(KERN_ERR, "rfc4106 enc shdesc@"__stringify(__LINE__)": ", 1006 DUMP_PREFIX_ADDRESS, 16, 4, desc, 1007 desc_bytes(desc), 1); 1008 #endif 1009 1010 /* 1011 * Job Descriptor and Shared Descriptors 1012 * must all fit into the 64-word Descriptor h/w Buffer 1013 */ 1014 keys_fit_inline = false; 1015 if (DESC_RFC4106_DEC_LEN + DESC_JOB_IO_LEN + 1016 ctx->enckeylen <= CAAM_DESC_BYTES_MAX) 1017 keys_fit_inline = true; 1018 1019 desc = ctx->sh_desc_dec; 1020 1021 init_sh_desc(desc, HDR_SHARE_SERIAL); 1022 1023 /* Skip key loading if it is loaded due to sharing */ 1024 key_jump_cmd = append_jump(desc, JUMP_JSL | 1025 JUMP_TEST_ALL | JUMP_COND_SHRD); 1026 if (keys_fit_inline) 1027 append_key_as_imm(desc, (void *)ctx->key, ctx->enckeylen, 1028 ctx->enckeylen, CLASS_1 | KEY_DEST_CLASS_REG); 1029 else 1030 append_key(desc, ctx->key_dma, ctx->enckeylen, 1031 CLASS_1 | KEY_DEST_CLASS_REG); 1032 set_jump_tgt_here(desc, key_jump_cmd); 1033 1034 /* Class 1 operation */ 1035 append_operation(desc, ctx->class1_alg_type | 1036 OP_ALG_AS_INITFINAL | OP_ALG_DECRYPT | OP_ALG_ICV_ON); 1037 1038 append_math_sub_imm_u32(desc, VARSEQINLEN, REG3, IMM, 8); 1039 append_math_add(desc, VARSEQOUTLEN, ZERO, REG3, CAAM_CMD_SZ); 1040 1041 /* Read assoc data */ 1042 append_seq_fifo_load(desc, 0, FIFOLD_CLASS_CLASS1 | FIFOLDST_VLF | 1043 FIFOLD_TYPE_AAD | FIFOLD_TYPE_FLUSH1); 1044 1045 /* Skip IV */ 1046 append_seq_fifo_load(desc, 8, FIFOLD_CLASS_SKIP); 1047 1048 /* Will read cryptlen bytes */ 1049 append_math_sub(desc, VARSEQINLEN, SEQOUTLEN, REG3, CAAM_CMD_SZ); 1050 1051 /* Workaround for erratum A-005473 (simultaneous SEQ FIFO skips) */ 1052 append_seq_fifo_load(desc, 0, FIFOLD_CLASS_CLASS1 | FIFOLD_TYPE_MSG); 1053 1054 /* Skip assoc data */ 1055 append_seq_fifo_store(desc, 0, FIFOST_TYPE_SKIP | FIFOLDST_VLF); 1056 1057 /* Will write cryptlen bytes */ 1058 append_math_sub(desc, VARSEQOUTLEN, SEQOUTLEN, REG0, CAAM_CMD_SZ); 1059 1060 /* Store payload data */ 1061 append_seq_fifo_store(desc, 0, FIFOST_TYPE_MESSAGE_DATA | FIFOLDST_VLF); 1062 1063 /* Read encrypted data */ 1064 append_seq_fifo_load(desc, 0, FIFOLD_CLASS_CLASS1 | FIFOLDST_VLF | 1065 FIFOLD_TYPE_MSG | FIFOLD_TYPE_FLUSH1); 1066 1067 /* Read ICV */ 1068 append_seq_fifo_load(desc, ctx->authsize, FIFOLD_CLASS_CLASS1 | 1069 FIFOLD_TYPE_ICV | FIFOLD_TYPE_LAST1); 1070 1071 ctx->sh_desc_dec_dma = dma_map_single(jrdev, desc, 1072 desc_bytes(desc), 1073 DMA_TO_DEVICE); 1074 if (dma_mapping_error(jrdev, ctx->sh_desc_dec_dma)) { 1075 dev_err(jrdev, "unable to map shared descriptor\n"); 1076 return -ENOMEM; 1077 } 1078 #ifdef DEBUG 1079 print_hex_dump(KERN_ERR, "rfc4106 dec shdesc@"__stringify(__LINE__)": ", 1080 DUMP_PREFIX_ADDRESS, 16, 4, desc, 1081 desc_bytes(desc), 1); 1082 #endif 1083 1084 return 0; 1085 } 1086 1087 static int rfc4106_setauthsize(struct crypto_aead *authenc, 1088 unsigned int authsize) 1089 { 1090 struct caam_ctx *ctx = crypto_aead_ctx(authenc); 1091 1092 ctx->authsize = authsize; 1093 rfc4106_set_sh_desc(authenc); 1094 1095 return 0; 1096 } 1097 1098 static int rfc4543_set_sh_desc(struct crypto_aead *aead) 1099 { 1100 struct caam_ctx *ctx = crypto_aead_ctx(aead); 1101 struct device *jrdev = ctx->jrdev; 1102 bool keys_fit_inline = false; 1103 u32 *key_jump_cmd; 1104 u32 *read_move_cmd, *write_move_cmd; 1105 u32 *desc; 1106 1107 if (!ctx->enckeylen || !ctx->authsize) 1108 return 0; 1109 1110 /* 1111 * RFC4543 encrypt shared descriptor 1112 * Job Descriptor and Shared Descriptor 1113 * must fit into the 64-word Descriptor h/w Buffer 1114 */ 1115 if (DESC_RFC4543_ENC_LEN + GCM_DESC_JOB_IO_LEN + 1116 ctx->enckeylen <= CAAM_DESC_BYTES_MAX) 1117 keys_fit_inline = true; 1118 1119 desc = ctx->sh_desc_enc; 1120 1121 init_sh_desc(desc, HDR_SHARE_SERIAL); 1122 1123 /* Skip key loading if it is loaded due to sharing */ 1124 key_jump_cmd = append_jump(desc, JUMP_JSL | JUMP_TEST_ALL | 1125 JUMP_COND_SHRD); 1126 if (keys_fit_inline) 1127 append_key_as_imm(desc, (void *)ctx->key, ctx->enckeylen, 1128 ctx->enckeylen, CLASS_1 | KEY_DEST_CLASS_REG); 1129 else 1130 append_key(desc, ctx->key_dma, ctx->enckeylen, 1131 CLASS_1 | KEY_DEST_CLASS_REG); 1132 set_jump_tgt_here(desc, key_jump_cmd); 1133 1134 /* Class 1 operation */ 1135 append_operation(desc, ctx->class1_alg_type | 1136 OP_ALG_AS_INITFINAL | OP_ALG_ENCRYPT); 1137 1138 /* assoclen + cryptlen = seqinlen */ 1139 append_math_sub(desc, REG3, SEQINLEN, REG0, CAAM_CMD_SZ); 1140 1141 /* 1142 * MOVE_LEN opcode is not available in all SEC HW revisions, 1143 * thus need to do some magic, i.e. self-patch the descriptor 1144 * buffer. 1145 */ 1146 read_move_cmd = append_move(desc, MOVE_SRC_DESCBUF | MOVE_DEST_MATH3 | 1147 (0x6 << MOVE_LEN_SHIFT)); 1148 write_move_cmd = append_move(desc, MOVE_SRC_MATH3 | MOVE_DEST_DESCBUF | 1149 (0x8 << MOVE_LEN_SHIFT)); 1150 1151 /* Will read assoclen + cryptlen bytes */ 1152 append_math_sub(desc, VARSEQINLEN, SEQINLEN, REG0, CAAM_CMD_SZ); 1153 1154 /* Will write assoclen + cryptlen bytes */ 1155 append_math_sub(desc, VARSEQOUTLEN, SEQINLEN, REG0, CAAM_CMD_SZ); 1156 1157 /* Read and write assoclen + cryptlen bytes */ 1158 aead_append_src_dst(desc, FIFOLD_TYPE_AAD); 1159 1160 set_move_tgt_here(desc, read_move_cmd); 1161 set_move_tgt_here(desc, write_move_cmd); 1162 append_cmd(desc, CMD_LOAD | DISABLE_AUTO_INFO_FIFO); 1163 /* Move payload data to OFIFO */ 1164 append_move(desc, MOVE_SRC_INFIFO_CL | MOVE_DEST_OUTFIFO); 1165 1166 /* Write ICV */ 1167 append_seq_store(desc, ctx->authsize, LDST_CLASS_1_CCB | 1168 LDST_SRCDST_BYTE_CONTEXT); 1169 1170 ctx->sh_desc_enc_dma = dma_map_single(jrdev, desc, 1171 desc_bytes(desc), 1172 DMA_TO_DEVICE); 1173 if (dma_mapping_error(jrdev, ctx->sh_desc_enc_dma)) { 1174 dev_err(jrdev, "unable to map shared descriptor\n"); 1175 return -ENOMEM; 1176 } 1177 #ifdef DEBUG 1178 print_hex_dump(KERN_ERR, "rfc4543 enc shdesc@"__stringify(__LINE__)": ", 1179 DUMP_PREFIX_ADDRESS, 16, 4, desc, 1180 desc_bytes(desc), 1); 1181 #endif 1182 1183 /* 1184 * Job Descriptor and Shared Descriptors 1185 * must all fit into the 64-word Descriptor h/w Buffer 1186 */ 1187 keys_fit_inline = false; 1188 if (DESC_RFC4543_DEC_LEN + GCM_DESC_JOB_IO_LEN + 1189 ctx->enckeylen <= CAAM_DESC_BYTES_MAX) 1190 keys_fit_inline = true; 1191 1192 desc = ctx->sh_desc_dec; 1193 1194 init_sh_desc(desc, HDR_SHARE_SERIAL); 1195 1196 /* Skip key loading if it is loaded due to sharing */ 1197 key_jump_cmd = append_jump(desc, JUMP_JSL | 1198 JUMP_TEST_ALL | JUMP_COND_SHRD); 1199 if (keys_fit_inline) 1200 append_key_as_imm(desc, (void *)ctx->key, ctx->enckeylen, 1201 ctx->enckeylen, CLASS_1 | KEY_DEST_CLASS_REG); 1202 else 1203 append_key(desc, ctx->key_dma, ctx->enckeylen, 1204 CLASS_1 | KEY_DEST_CLASS_REG); 1205 set_jump_tgt_here(desc, key_jump_cmd); 1206 1207 /* Class 1 operation */ 1208 append_operation(desc, ctx->class1_alg_type | 1209 OP_ALG_AS_INITFINAL | OP_ALG_DECRYPT | OP_ALG_ICV_ON); 1210 1211 /* assoclen + cryptlen = seqoutlen */ 1212 append_math_sub(desc, REG3, SEQOUTLEN, REG0, CAAM_CMD_SZ); 1213 1214 /* 1215 * MOVE_LEN opcode is not available in all SEC HW revisions, 1216 * thus need to do some magic, i.e. self-patch the descriptor 1217 * buffer. 1218 */ 1219 read_move_cmd = append_move(desc, MOVE_SRC_DESCBUF | MOVE_DEST_MATH3 | 1220 (0x6 << MOVE_LEN_SHIFT)); 1221 write_move_cmd = append_move(desc, MOVE_SRC_MATH3 | MOVE_DEST_DESCBUF | 1222 (0x8 << MOVE_LEN_SHIFT)); 1223 1224 /* Will read assoclen + cryptlen bytes */ 1225 append_math_sub(desc, VARSEQINLEN, SEQOUTLEN, REG0, CAAM_CMD_SZ); 1226 1227 /* Will write assoclen + cryptlen bytes */ 1228 append_math_sub(desc, VARSEQOUTLEN, SEQOUTLEN, REG0, CAAM_CMD_SZ); 1229 1230 /* Store payload data */ 1231 append_seq_fifo_store(desc, 0, FIFOST_TYPE_MESSAGE_DATA | FIFOLDST_VLF); 1232 1233 /* In-snoop assoclen + cryptlen data */ 1234 append_seq_fifo_load(desc, 0, FIFOLD_CLASS_BOTH | FIFOLDST_VLF | 1235 FIFOLD_TYPE_AAD | FIFOLD_TYPE_LAST2FLUSH1); 1236 1237 set_move_tgt_here(desc, read_move_cmd); 1238 set_move_tgt_here(desc, write_move_cmd); 1239 append_cmd(desc, CMD_LOAD | DISABLE_AUTO_INFO_FIFO); 1240 /* Move payload data to OFIFO */ 1241 append_move(desc, MOVE_SRC_INFIFO_CL | MOVE_DEST_OUTFIFO); 1242 append_cmd(desc, CMD_LOAD | ENABLE_AUTO_INFO_FIFO); 1243 1244 /* Read ICV */ 1245 append_seq_fifo_load(desc, ctx->authsize, FIFOLD_CLASS_CLASS1 | 1246 FIFOLD_TYPE_ICV | FIFOLD_TYPE_LAST1); 1247 1248 ctx->sh_desc_dec_dma = dma_map_single(jrdev, desc, 1249 desc_bytes(desc), 1250 DMA_TO_DEVICE); 1251 if (dma_mapping_error(jrdev, ctx->sh_desc_dec_dma)) { 1252 dev_err(jrdev, "unable to map shared descriptor\n"); 1253 return -ENOMEM; 1254 } 1255 #ifdef DEBUG 1256 print_hex_dump(KERN_ERR, "rfc4543 dec shdesc@"__stringify(__LINE__)": ", 1257 DUMP_PREFIX_ADDRESS, 16, 4, desc, 1258 desc_bytes(desc), 1); 1259 #endif 1260 1261 return 0; 1262 } 1263 1264 static int rfc4543_setauthsize(struct crypto_aead *authenc, 1265 unsigned int authsize) 1266 { 1267 struct caam_ctx *ctx = crypto_aead_ctx(authenc); 1268 1269 ctx->authsize = authsize; 1270 rfc4543_set_sh_desc(authenc); 1271 1272 return 0; 1273 } 1274 1275 static u32 gen_split_aead_key(struct caam_ctx *ctx, const u8 *key_in, 1276 u32 authkeylen) 1277 { 1278 return gen_split_key(ctx->jrdev, ctx->key, ctx->split_key_len, 1279 ctx->split_key_pad_len, key_in, authkeylen, 1280 ctx->alg_op); 1281 } 1282 1283 static int aead_setkey(struct crypto_aead *aead, 1284 const u8 *key, unsigned int keylen) 1285 { 1286 /* Sizes for MDHA pads (*not* keys): MD5, SHA1, 224, 256, 384, 512 */ 1287 static const u8 mdpadlen[] = { 16, 20, 32, 32, 64, 64 }; 1288 struct caam_ctx *ctx = crypto_aead_ctx(aead); 1289 struct device *jrdev = ctx->jrdev; 1290 struct crypto_authenc_keys keys; 1291 int ret = 0; 1292 1293 if (crypto_authenc_extractkeys(&keys, key, keylen) != 0) 1294 goto badkey; 1295 1296 /* Pick class 2 key length from algorithm submask */ 1297 ctx->split_key_len = mdpadlen[(ctx->alg_op & OP_ALG_ALGSEL_SUBMASK) >> 1298 OP_ALG_ALGSEL_SHIFT] * 2; 1299 ctx->split_key_pad_len = ALIGN(ctx->split_key_len, 16); 1300 1301 if (ctx->split_key_pad_len + keys.enckeylen > CAAM_MAX_KEY_SIZE) 1302 goto badkey; 1303 1304 #ifdef DEBUG 1305 printk(KERN_ERR "keylen %d enckeylen %d authkeylen %d\n", 1306 keys.authkeylen + keys.enckeylen, keys.enckeylen, 1307 keys.authkeylen); 1308 printk(KERN_ERR "split_key_len %d split_key_pad_len %d\n", 1309 ctx->split_key_len, ctx->split_key_pad_len); 1310 print_hex_dump(KERN_ERR, "key in @"__stringify(__LINE__)": ", 1311 DUMP_PREFIX_ADDRESS, 16, 4, key, keylen, 1); 1312 #endif 1313 1314 ret = gen_split_aead_key(ctx, keys.authkey, keys.authkeylen); 1315 if (ret) { 1316 goto badkey; 1317 } 1318 1319 /* postpend encryption key to auth split key */ 1320 memcpy(ctx->key + ctx->split_key_pad_len, keys.enckey, keys.enckeylen); 1321 1322 ctx->key_dma = dma_map_single(jrdev, ctx->key, ctx->split_key_pad_len + 1323 keys.enckeylen, DMA_TO_DEVICE); 1324 if (dma_mapping_error(jrdev, ctx->key_dma)) { 1325 dev_err(jrdev, "unable to map key i/o memory\n"); 1326 return -ENOMEM; 1327 } 1328 #ifdef DEBUG 1329 print_hex_dump(KERN_ERR, "ctx.key@"__stringify(__LINE__)": ", 1330 DUMP_PREFIX_ADDRESS, 16, 4, ctx->key, 1331 ctx->split_key_pad_len + keys.enckeylen, 1); 1332 #endif 1333 1334 ctx->enckeylen = keys.enckeylen; 1335 1336 ret = aead_set_sh_desc(aead); 1337 if (ret) { 1338 dma_unmap_single(jrdev, ctx->key_dma, ctx->split_key_pad_len + 1339 keys.enckeylen, DMA_TO_DEVICE); 1340 } 1341 1342 return ret; 1343 badkey: 1344 crypto_aead_set_flags(aead, CRYPTO_TFM_RES_BAD_KEY_LEN); 1345 return -EINVAL; 1346 } 1347 1348 static int gcm_setkey(struct crypto_aead *aead, 1349 const u8 *key, unsigned int keylen) 1350 { 1351 struct caam_ctx *ctx = crypto_aead_ctx(aead); 1352 struct device *jrdev = ctx->jrdev; 1353 int ret = 0; 1354 1355 #ifdef DEBUG 1356 print_hex_dump(KERN_ERR, "key in @"__stringify(__LINE__)": ", 1357 DUMP_PREFIX_ADDRESS, 16, 4, key, keylen, 1); 1358 #endif 1359 1360 memcpy(ctx->key, key, keylen); 1361 ctx->key_dma = dma_map_single(jrdev, ctx->key, keylen, 1362 DMA_TO_DEVICE); 1363 if (dma_mapping_error(jrdev, ctx->key_dma)) { 1364 dev_err(jrdev, "unable to map key i/o memory\n"); 1365 return -ENOMEM; 1366 } 1367 ctx->enckeylen = keylen; 1368 1369 ret = gcm_set_sh_desc(aead); 1370 if (ret) { 1371 dma_unmap_single(jrdev, ctx->key_dma, ctx->enckeylen, 1372 DMA_TO_DEVICE); 1373 } 1374 1375 return ret; 1376 } 1377 1378 static int rfc4106_setkey(struct crypto_aead *aead, 1379 const u8 *key, unsigned int keylen) 1380 { 1381 struct caam_ctx *ctx = crypto_aead_ctx(aead); 1382 struct device *jrdev = ctx->jrdev; 1383 int ret = 0; 1384 1385 if (keylen < 4) 1386 return -EINVAL; 1387 1388 #ifdef DEBUG 1389 print_hex_dump(KERN_ERR, "key in @"__stringify(__LINE__)": ", 1390 DUMP_PREFIX_ADDRESS, 16, 4, key, keylen, 1); 1391 #endif 1392 1393 memcpy(ctx->key, key, keylen); 1394 1395 /* 1396 * The last four bytes of the key material are used as the salt value 1397 * in the nonce. Update the AES key length. 1398 */ 1399 ctx->enckeylen = keylen - 4; 1400 1401 ctx->key_dma = dma_map_single(jrdev, ctx->key, ctx->enckeylen, 1402 DMA_TO_DEVICE); 1403 if (dma_mapping_error(jrdev, ctx->key_dma)) { 1404 dev_err(jrdev, "unable to map key i/o memory\n"); 1405 return -ENOMEM; 1406 } 1407 1408 ret = rfc4106_set_sh_desc(aead); 1409 if (ret) { 1410 dma_unmap_single(jrdev, ctx->key_dma, ctx->enckeylen, 1411 DMA_TO_DEVICE); 1412 } 1413 1414 return ret; 1415 } 1416 1417 static int rfc4543_setkey(struct crypto_aead *aead, 1418 const u8 *key, unsigned int keylen) 1419 { 1420 struct caam_ctx *ctx = crypto_aead_ctx(aead); 1421 struct device *jrdev = ctx->jrdev; 1422 int ret = 0; 1423 1424 if (keylen < 4) 1425 return -EINVAL; 1426 1427 #ifdef DEBUG 1428 print_hex_dump(KERN_ERR, "key in @"__stringify(__LINE__)": ", 1429 DUMP_PREFIX_ADDRESS, 16, 4, key, keylen, 1); 1430 #endif 1431 1432 memcpy(ctx->key, key, keylen); 1433 1434 /* 1435 * The last four bytes of the key material are used as the salt value 1436 * in the nonce. Update the AES key length. 1437 */ 1438 ctx->enckeylen = keylen - 4; 1439 1440 ctx->key_dma = dma_map_single(jrdev, ctx->key, ctx->enckeylen, 1441 DMA_TO_DEVICE); 1442 if (dma_mapping_error(jrdev, ctx->key_dma)) { 1443 dev_err(jrdev, "unable to map key i/o memory\n"); 1444 return -ENOMEM; 1445 } 1446 1447 ret = rfc4543_set_sh_desc(aead); 1448 if (ret) { 1449 dma_unmap_single(jrdev, ctx->key_dma, ctx->enckeylen, 1450 DMA_TO_DEVICE); 1451 } 1452 1453 return ret; 1454 } 1455 1456 static int ablkcipher_setkey(struct crypto_ablkcipher *ablkcipher, 1457 const u8 *key, unsigned int keylen) 1458 { 1459 struct caam_ctx *ctx = crypto_ablkcipher_ctx(ablkcipher); 1460 struct ablkcipher_tfm *crt = &ablkcipher->base.crt_ablkcipher; 1461 struct crypto_tfm *tfm = crypto_ablkcipher_tfm(ablkcipher); 1462 const char *alg_name = crypto_tfm_alg_name(tfm); 1463 struct device *jrdev = ctx->jrdev; 1464 int ret = 0; 1465 u32 *key_jump_cmd; 1466 u32 *desc; 1467 u32 *nonce; 1468 u32 geniv; 1469 u32 ctx1_iv_off = 0; 1470 const bool ctr_mode = ((ctx->class1_alg_type & OP_ALG_AAI_MASK) == 1471 OP_ALG_AAI_CTR_MOD128); 1472 const bool is_rfc3686 = (ctr_mode && 1473 (strstr(alg_name, "rfc3686") != NULL)); 1474 1475 #ifdef DEBUG 1476 print_hex_dump(KERN_ERR, "key in @"__stringify(__LINE__)": ", 1477 DUMP_PREFIX_ADDRESS, 16, 4, key, keylen, 1); 1478 #endif 1479 /* 1480 * AES-CTR needs to load IV in CONTEXT1 reg 1481 * at an offset of 128bits (16bytes) 1482 * CONTEXT1[255:128] = IV 1483 */ 1484 if (ctr_mode) 1485 ctx1_iv_off = 16; 1486 1487 /* 1488 * RFC3686 specific: 1489 * | CONTEXT1[255:128] = {NONCE, IV, COUNTER} 1490 * | *key = {KEY, NONCE} 1491 */ 1492 if (is_rfc3686) { 1493 ctx1_iv_off = 16 + CTR_RFC3686_NONCE_SIZE; 1494 keylen -= CTR_RFC3686_NONCE_SIZE; 1495 } 1496 1497 memcpy(ctx->key, key, keylen); 1498 ctx->key_dma = dma_map_single(jrdev, ctx->key, keylen, 1499 DMA_TO_DEVICE); 1500 if (dma_mapping_error(jrdev, ctx->key_dma)) { 1501 dev_err(jrdev, "unable to map key i/o memory\n"); 1502 return -ENOMEM; 1503 } 1504 ctx->enckeylen = keylen; 1505 1506 /* ablkcipher_encrypt shared descriptor */ 1507 desc = ctx->sh_desc_enc; 1508 init_sh_desc(desc, HDR_SHARE_SERIAL | HDR_SAVECTX); 1509 /* Skip if already shared */ 1510 key_jump_cmd = append_jump(desc, JUMP_JSL | JUMP_TEST_ALL | 1511 JUMP_COND_SHRD); 1512 1513 /* Load class1 key only */ 1514 append_key_as_imm(desc, (void *)ctx->key, ctx->enckeylen, 1515 ctx->enckeylen, CLASS_1 | 1516 KEY_DEST_CLASS_REG); 1517 1518 /* Load nonce into CONTEXT1 reg */ 1519 if (is_rfc3686) { 1520 nonce = (u32 *)(key + keylen); 1521 append_load_imm_u32(desc, *nonce, LDST_CLASS_IND_CCB | 1522 LDST_SRCDST_BYTE_OUTFIFO | LDST_IMM); 1523 append_move(desc, MOVE_WAITCOMP | 1524 MOVE_SRC_OUTFIFO | 1525 MOVE_DEST_CLASS1CTX | 1526 (16 << MOVE_OFFSET_SHIFT) | 1527 (CTR_RFC3686_NONCE_SIZE << MOVE_LEN_SHIFT)); 1528 } 1529 1530 set_jump_tgt_here(desc, key_jump_cmd); 1531 1532 /* Load iv */ 1533 append_seq_load(desc, crt->ivsize, LDST_SRCDST_BYTE_CONTEXT | 1534 LDST_CLASS_1_CCB | (ctx1_iv_off << LDST_OFFSET_SHIFT)); 1535 1536 /* Load counter into CONTEXT1 reg */ 1537 if (is_rfc3686) 1538 append_load_imm_u32(desc, be32_to_cpu(1), LDST_IMM | 1539 LDST_CLASS_1_CCB | 1540 LDST_SRCDST_BYTE_CONTEXT | 1541 ((ctx1_iv_off + CTR_RFC3686_IV_SIZE) << 1542 LDST_OFFSET_SHIFT)); 1543 1544 /* Load operation */ 1545 append_operation(desc, ctx->class1_alg_type | 1546 OP_ALG_AS_INITFINAL | OP_ALG_ENCRYPT); 1547 1548 /* Perform operation */ 1549 ablkcipher_append_src_dst(desc); 1550 1551 ctx->sh_desc_enc_dma = dma_map_single(jrdev, desc, 1552 desc_bytes(desc), 1553 DMA_TO_DEVICE); 1554 if (dma_mapping_error(jrdev, ctx->sh_desc_enc_dma)) { 1555 dev_err(jrdev, "unable to map shared descriptor\n"); 1556 return -ENOMEM; 1557 } 1558 #ifdef DEBUG 1559 print_hex_dump(KERN_ERR, 1560 "ablkcipher enc shdesc@"__stringify(__LINE__)": ", 1561 DUMP_PREFIX_ADDRESS, 16, 4, desc, 1562 desc_bytes(desc), 1); 1563 #endif 1564 /* ablkcipher_decrypt shared descriptor */ 1565 desc = ctx->sh_desc_dec; 1566 1567 init_sh_desc(desc, HDR_SHARE_SERIAL | HDR_SAVECTX); 1568 /* Skip if already shared */ 1569 key_jump_cmd = append_jump(desc, JUMP_JSL | JUMP_TEST_ALL | 1570 JUMP_COND_SHRD); 1571 1572 /* Load class1 key only */ 1573 append_key_as_imm(desc, (void *)ctx->key, ctx->enckeylen, 1574 ctx->enckeylen, CLASS_1 | 1575 KEY_DEST_CLASS_REG); 1576 1577 /* Load nonce into CONTEXT1 reg */ 1578 if (is_rfc3686) { 1579 nonce = (u32 *)(key + keylen); 1580 append_load_imm_u32(desc, *nonce, LDST_CLASS_IND_CCB | 1581 LDST_SRCDST_BYTE_OUTFIFO | LDST_IMM); 1582 append_move(desc, MOVE_WAITCOMP | 1583 MOVE_SRC_OUTFIFO | 1584 MOVE_DEST_CLASS1CTX | 1585 (16 << MOVE_OFFSET_SHIFT) | 1586 (CTR_RFC3686_NONCE_SIZE << MOVE_LEN_SHIFT)); 1587 } 1588 1589 set_jump_tgt_here(desc, key_jump_cmd); 1590 1591 /* load IV */ 1592 append_seq_load(desc, crt->ivsize, LDST_SRCDST_BYTE_CONTEXT | 1593 LDST_CLASS_1_CCB | (ctx1_iv_off << LDST_OFFSET_SHIFT)); 1594 1595 /* Load counter into CONTEXT1 reg */ 1596 if (is_rfc3686) 1597 append_load_imm_u32(desc, be32_to_cpu(1), LDST_IMM | 1598 LDST_CLASS_1_CCB | 1599 LDST_SRCDST_BYTE_CONTEXT | 1600 ((ctx1_iv_off + CTR_RFC3686_IV_SIZE) << 1601 LDST_OFFSET_SHIFT)); 1602 1603 /* Choose operation */ 1604 if (ctr_mode) 1605 append_operation(desc, ctx->class1_alg_type | 1606 OP_ALG_AS_INITFINAL | OP_ALG_DECRYPT); 1607 else 1608 append_dec_op1(desc, ctx->class1_alg_type); 1609 1610 /* Perform operation */ 1611 ablkcipher_append_src_dst(desc); 1612 1613 ctx->sh_desc_dec_dma = dma_map_single(jrdev, desc, 1614 desc_bytes(desc), 1615 DMA_TO_DEVICE); 1616 if (dma_mapping_error(jrdev, ctx->sh_desc_dec_dma)) { 1617 dev_err(jrdev, "unable to map shared descriptor\n"); 1618 return -ENOMEM; 1619 } 1620 1621 #ifdef DEBUG 1622 print_hex_dump(KERN_ERR, 1623 "ablkcipher dec shdesc@"__stringify(__LINE__)": ", 1624 DUMP_PREFIX_ADDRESS, 16, 4, desc, 1625 desc_bytes(desc), 1); 1626 #endif 1627 /* ablkcipher_givencrypt shared descriptor */ 1628 desc = ctx->sh_desc_givenc; 1629 1630 init_sh_desc(desc, HDR_SHARE_SERIAL | HDR_SAVECTX); 1631 /* Skip if already shared */ 1632 key_jump_cmd = append_jump(desc, JUMP_JSL | JUMP_TEST_ALL | 1633 JUMP_COND_SHRD); 1634 1635 /* Load class1 key only */ 1636 append_key_as_imm(desc, (void *)ctx->key, ctx->enckeylen, 1637 ctx->enckeylen, CLASS_1 | 1638 KEY_DEST_CLASS_REG); 1639 1640 /* Load Nonce into CONTEXT1 reg */ 1641 if (is_rfc3686) { 1642 nonce = (u32 *)(key + keylen); 1643 append_load_imm_u32(desc, *nonce, LDST_CLASS_IND_CCB | 1644 LDST_SRCDST_BYTE_OUTFIFO | LDST_IMM); 1645 append_move(desc, MOVE_WAITCOMP | 1646 MOVE_SRC_OUTFIFO | 1647 MOVE_DEST_CLASS1CTX | 1648 (16 << MOVE_OFFSET_SHIFT) | 1649 (CTR_RFC3686_NONCE_SIZE << MOVE_LEN_SHIFT)); 1650 } 1651 set_jump_tgt_here(desc, key_jump_cmd); 1652 1653 /* Generate IV */ 1654 geniv = NFIFOENTRY_STYPE_PAD | NFIFOENTRY_DEST_DECO | 1655 NFIFOENTRY_DTYPE_MSG | NFIFOENTRY_LC1 | 1656 NFIFOENTRY_PTYPE_RND | (crt->ivsize << NFIFOENTRY_DLEN_SHIFT); 1657 append_load_imm_u32(desc, geniv, LDST_CLASS_IND_CCB | 1658 LDST_SRCDST_WORD_INFO_FIFO | LDST_IMM); 1659 append_cmd(desc, CMD_LOAD | DISABLE_AUTO_INFO_FIFO); 1660 append_move(desc, MOVE_WAITCOMP | 1661 MOVE_SRC_INFIFO | 1662 MOVE_DEST_CLASS1CTX | 1663 (crt->ivsize << MOVE_LEN_SHIFT) | 1664 (ctx1_iv_off << MOVE_OFFSET_SHIFT)); 1665 append_cmd(desc, CMD_LOAD | ENABLE_AUTO_INFO_FIFO); 1666 1667 /* Copy generated IV to memory */ 1668 append_seq_store(desc, crt->ivsize, 1669 LDST_SRCDST_BYTE_CONTEXT | LDST_CLASS_1_CCB | 1670 (ctx1_iv_off << LDST_OFFSET_SHIFT)); 1671 1672 /* Load Counter into CONTEXT1 reg */ 1673 if (is_rfc3686) 1674 append_load_imm_u32(desc, (u32)1, LDST_IMM | 1675 LDST_CLASS_1_CCB | 1676 LDST_SRCDST_BYTE_CONTEXT | 1677 ((ctx1_iv_off + CTR_RFC3686_IV_SIZE) << 1678 LDST_OFFSET_SHIFT)); 1679 1680 if (ctx1_iv_off) 1681 append_jump(desc, JUMP_JSL | JUMP_TEST_ALL | JUMP_COND_NCP | 1682 (1 << JUMP_OFFSET_SHIFT)); 1683 1684 /* Load operation */ 1685 append_operation(desc, ctx->class1_alg_type | 1686 OP_ALG_AS_INITFINAL | OP_ALG_ENCRYPT); 1687 1688 /* Perform operation */ 1689 ablkcipher_append_src_dst(desc); 1690 1691 ctx->sh_desc_givenc_dma = dma_map_single(jrdev, desc, 1692 desc_bytes(desc), 1693 DMA_TO_DEVICE); 1694 if (dma_mapping_error(jrdev, ctx->sh_desc_givenc_dma)) { 1695 dev_err(jrdev, "unable to map shared descriptor\n"); 1696 return -ENOMEM; 1697 } 1698 #ifdef DEBUG 1699 print_hex_dump(KERN_ERR, 1700 "ablkcipher givenc shdesc@" __stringify(__LINE__) ": ", 1701 DUMP_PREFIX_ADDRESS, 16, 4, desc, 1702 desc_bytes(desc), 1); 1703 #endif 1704 1705 return ret; 1706 } 1707 1708 /* 1709 * aead_edesc - s/w-extended aead descriptor 1710 * @assoc_nents: number of segments in associated data (SPI+Seq) scatterlist 1711 * @assoc_chained: if source is chained 1712 * @src_nents: number of segments in input scatterlist 1713 * @src_chained: if source is chained 1714 * @dst_nents: number of segments in output scatterlist 1715 * @dst_chained: if destination is chained 1716 * @iv_dma: dma address of iv for checking continuity and link table 1717 * @desc: h/w descriptor (variable length; must not exceed MAX_CAAM_DESCSIZE) 1718 * @sec4_sg_bytes: length of dma mapped sec4_sg space 1719 * @sec4_sg_dma: bus physical mapped address of h/w link table 1720 * @hw_desc: the h/w job descriptor followed by any referenced link tables 1721 */ 1722 struct aead_edesc { 1723 int assoc_nents; 1724 bool assoc_chained; 1725 int src_nents; 1726 bool src_chained; 1727 int dst_nents; 1728 bool dst_chained; 1729 dma_addr_t iv_dma; 1730 int sec4_sg_bytes; 1731 dma_addr_t sec4_sg_dma; 1732 struct sec4_sg_entry *sec4_sg; 1733 u32 hw_desc[]; 1734 }; 1735 1736 /* 1737 * ablkcipher_edesc - s/w-extended ablkcipher descriptor 1738 * @src_nents: number of segments in input scatterlist 1739 * @src_chained: if source is chained 1740 * @dst_nents: number of segments in output scatterlist 1741 * @dst_chained: if destination is chained 1742 * @iv_dma: dma address of iv for checking continuity and link table 1743 * @desc: h/w descriptor (variable length; must not exceed MAX_CAAM_DESCSIZE) 1744 * @sec4_sg_bytes: length of dma mapped sec4_sg space 1745 * @sec4_sg_dma: bus physical mapped address of h/w link table 1746 * @hw_desc: the h/w job descriptor followed by any referenced link tables 1747 */ 1748 struct ablkcipher_edesc { 1749 int src_nents; 1750 bool src_chained; 1751 int dst_nents; 1752 bool dst_chained; 1753 dma_addr_t iv_dma; 1754 int sec4_sg_bytes; 1755 dma_addr_t sec4_sg_dma; 1756 struct sec4_sg_entry *sec4_sg; 1757 u32 hw_desc[0]; 1758 }; 1759 1760 static void caam_unmap(struct device *dev, struct scatterlist *src, 1761 struct scatterlist *dst, int src_nents, 1762 bool src_chained, int dst_nents, bool dst_chained, 1763 dma_addr_t iv_dma, int ivsize, dma_addr_t sec4_sg_dma, 1764 int sec4_sg_bytes) 1765 { 1766 if (dst != src) { 1767 dma_unmap_sg_chained(dev, src, src_nents ? : 1, DMA_TO_DEVICE, 1768 src_chained); 1769 dma_unmap_sg_chained(dev, dst, dst_nents ? : 1, DMA_FROM_DEVICE, 1770 dst_chained); 1771 } else { 1772 dma_unmap_sg_chained(dev, src, src_nents ? : 1, 1773 DMA_BIDIRECTIONAL, src_chained); 1774 } 1775 1776 if (iv_dma) 1777 dma_unmap_single(dev, iv_dma, ivsize, DMA_TO_DEVICE); 1778 if (sec4_sg_bytes) 1779 dma_unmap_single(dev, sec4_sg_dma, sec4_sg_bytes, 1780 DMA_TO_DEVICE); 1781 } 1782 1783 static void aead_unmap(struct device *dev, 1784 struct aead_edesc *edesc, 1785 struct aead_request *req) 1786 { 1787 caam_unmap(dev, req->src, req->dst, 1788 edesc->src_nents, edesc->src_chained, edesc->dst_nents, 1789 edesc->dst_chained, 0, 0, 1790 edesc->sec4_sg_dma, edesc->sec4_sg_bytes); 1791 } 1792 1793 static void ablkcipher_unmap(struct device *dev, 1794 struct ablkcipher_edesc *edesc, 1795 struct ablkcipher_request *req) 1796 { 1797 struct crypto_ablkcipher *ablkcipher = crypto_ablkcipher_reqtfm(req); 1798 int ivsize = crypto_ablkcipher_ivsize(ablkcipher); 1799 1800 caam_unmap(dev, req->src, req->dst, 1801 edesc->src_nents, edesc->src_chained, edesc->dst_nents, 1802 edesc->dst_chained, edesc->iv_dma, ivsize, 1803 edesc->sec4_sg_dma, edesc->sec4_sg_bytes); 1804 } 1805 1806 static void aead_encrypt_done(struct device *jrdev, u32 *desc, u32 err, 1807 void *context) 1808 { 1809 struct aead_request *req = context; 1810 struct aead_edesc *edesc; 1811 1812 #ifdef DEBUG 1813 dev_err(jrdev, "%s %d: err 0x%x\n", __func__, __LINE__, err); 1814 #endif 1815 1816 edesc = container_of(desc, struct aead_edesc, hw_desc[0]); 1817 1818 if (err) 1819 caam_jr_strstatus(jrdev, err); 1820 1821 aead_unmap(jrdev, edesc, req); 1822 1823 kfree(edesc); 1824 1825 aead_request_complete(req, err); 1826 } 1827 1828 static void aead_decrypt_done(struct device *jrdev, u32 *desc, u32 err, 1829 void *context) 1830 { 1831 struct aead_request *req = context; 1832 struct aead_edesc *edesc; 1833 1834 #ifdef DEBUG 1835 dev_err(jrdev, "%s %d: err 0x%x\n", __func__, __LINE__, err); 1836 #endif 1837 1838 edesc = container_of(desc, struct aead_edesc, hw_desc[0]); 1839 1840 if (err) 1841 caam_jr_strstatus(jrdev, err); 1842 1843 aead_unmap(jrdev, edesc, req); 1844 1845 /* 1846 * verify hw auth check passed else return -EBADMSG 1847 */ 1848 if ((err & JRSTA_CCBERR_ERRID_MASK) == JRSTA_CCBERR_ERRID_ICVCHK) 1849 err = -EBADMSG; 1850 1851 kfree(edesc); 1852 1853 aead_request_complete(req, err); 1854 } 1855 1856 static void ablkcipher_encrypt_done(struct device *jrdev, u32 *desc, u32 err, 1857 void *context) 1858 { 1859 struct ablkcipher_request *req = context; 1860 struct ablkcipher_edesc *edesc; 1861 #ifdef DEBUG 1862 struct crypto_ablkcipher *ablkcipher = crypto_ablkcipher_reqtfm(req); 1863 int ivsize = crypto_ablkcipher_ivsize(ablkcipher); 1864 1865 dev_err(jrdev, "%s %d: err 0x%x\n", __func__, __LINE__, err); 1866 #endif 1867 1868 edesc = (struct ablkcipher_edesc *)((char *)desc - 1869 offsetof(struct ablkcipher_edesc, hw_desc)); 1870 1871 if (err) 1872 caam_jr_strstatus(jrdev, err); 1873 1874 #ifdef DEBUG 1875 print_hex_dump(KERN_ERR, "dstiv @"__stringify(__LINE__)": ", 1876 DUMP_PREFIX_ADDRESS, 16, 4, req->info, 1877 edesc->src_nents > 1 ? 100 : ivsize, 1); 1878 print_hex_dump(KERN_ERR, "dst @"__stringify(__LINE__)": ", 1879 DUMP_PREFIX_ADDRESS, 16, 4, sg_virt(req->src), 1880 edesc->dst_nents > 1 ? 100 : req->nbytes, 1); 1881 #endif 1882 1883 ablkcipher_unmap(jrdev, edesc, req); 1884 kfree(edesc); 1885 1886 ablkcipher_request_complete(req, err); 1887 } 1888 1889 static void ablkcipher_decrypt_done(struct device *jrdev, u32 *desc, u32 err, 1890 void *context) 1891 { 1892 struct ablkcipher_request *req = context; 1893 struct ablkcipher_edesc *edesc; 1894 #ifdef DEBUG 1895 struct crypto_ablkcipher *ablkcipher = crypto_ablkcipher_reqtfm(req); 1896 int ivsize = crypto_ablkcipher_ivsize(ablkcipher); 1897 1898 dev_err(jrdev, "%s %d: err 0x%x\n", __func__, __LINE__, err); 1899 #endif 1900 1901 edesc = (struct ablkcipher_edesc *)((char *)desc - 1902 offsetof(struct ablkcipher_edesc, hw_desc)); 1903 if (err) 1904 caam_jr_strstatus(jrdev, err); 1905 1906 #ifdef DEBUG 1907 print_hex_dump(KERN_ERR, "dstiv @"__stringify(__LINE__)": ", 1908 DUMP_PREFIX_ADDRESS, 16, 4, req->info, 1909 ivsize, 1); 1910 print_hex_dump(KERN_ERR, "dst @"__stringify(__LINE__)": ", 1911 DUMP_PREFIX_ADDRESS, 16, 4, sg_virt(req->src), 1912 edesc->dst_nents > 1 ? 100 : req->nbytes, 1); 1913 #endif 1914 1915 ablkcipher_unmap(jrdev, edesc, req); 1916 kfree(edesc); 1917 1918 ablkcipher_request_complete(req, err); 1919 } 1920 1921 /* 1922 * Fill in aead job descriptor 1923 */ 1924 static void init_aead_job(struct aead_request *req, 1925 struct aead_edesc *edesc, 1926 bool all_contig, bool encrypt) 1927 { 1928 struct crypto_aead *aead = crypto_aead_reqtfm(req); 1929 struct caam_ctx *ctx = crypto_aead_ctx(aead); 1930 int authsize = ctx->authsize; 1931 u32 *desc = edesc->hw_desc; 1932 u32 out_options, in_options; 1933 dma_addr_t dst_dma, src_dma; 1934 int len, sec4_sg_index = 0; 1935 dma_addr_t ptr; 1936 u32 *sh_desc; 1937 1938 sh_desc = encrypt ? ctx->sh_desc_enc : ctx->sh_desc_dec; 1939 ptr = encrypt ? ctx->sh_desc_enc_dma : ctx->sh_desc_dec_dma; 1940 1941 len = desc_len(sh_desc); 1942 init_job_desc_shared(desc, ptr, len, HDR_SHARE_DEFER | HDR_REVERSE); 1943 1944 if (all_contig) { 1945 src_dma = sg_dma_address(req->src); 1946 in_options = 0; 1947 } else { 1948 src_dma = edesc->sec4_sg_dma; 1949 sec4_sg_index += edesc->src_nents; 1950 in_options = LDST_SGF; 1951 } 1952 1953 append_seq_in_ptr(desc, src_dma, req->assoclen + req->cryptlen, 1954 in_options); 1955 1956 dst_dma = src_dma; 1957 out_options = in_options; 1958 1959 if (unlikely(req->src != req->dst)) { 1960 if (!edesc->dst_nents) { 1961 dst_dma = sg_dma_address(req->dst); 1962 } else { 1963 dst_dma = edesc->sec4_sg_dma + 1964 sec4_sg_index * 1965 sizeof(struct sec4_sg_entry); 1966 out_options = LDST_SGF; 1967 } 1968 } 1969 1970 if (encrypt) 1971 append_seq_out_ptr(desc, dst_dma, 1972 req->assoclen + req->cryptlen + authsize, 1973 out_options); 1974 else 1975 append_seq_out_ptr(desc, dst_dma, 1976 req->assoclen + req->cryptlen - authsize, 1977 out_options); 1978 1979 /* REG3 = assoclen */ 1980 append_math_add_imm_u32(desc, REG3, ZERO, IMM, req->assoclen); 1981 } 1982 1983 static void init_gcm_job(struct aead_request *req, 1984 struct aead_edesc *edesc, 1985 bool all_contig, bool encrypt) 1986 { 1987 struct crypto_aead *aead = crypto_aead_reqtfm(req); 1988 struct caam_ctx *ctx = crypto_aead_ctx(aead); 1989 unsigned int ivsize = crypto_aead_ivsize(aead); 1990 u32 *desc = edesc->hw_desc; 1991 bool generic_gcm = (ivsize == 12); 1992 unsigned int last; 1993 1994 init_aead_job(req, edesc, all_contig, encrypt); 1995 1996 /* BUG This should not be specific to generic GCM. */ 1997 last = 0; 1998 if (encrypt && generic_gcm && !(req->assoclen + req->cryptlen)) 1999 last = FIFOLD_TYPE_LAST1; 2000 2001 /* Read GCM IV */ 2002 append_cmd(desc, CMD_FIFO_LOAD | FIFOLD_CLASS_CLASS1 | IMMEDIATE | 2003 FIFOLD_TYPE_IV | FIFOLD_TYPE_FLUSH1 | 12 | last); 2004 /* Append Salt */ 2005 if (!generic_gcm) 2006 append_data(desc, ctx->key + ctx->enckeylen, 4); 2007 /* Append IV */ 2008 append_data(desc, req->iv, ivsize); 2009 /* End of blank commands */ 2010 } 2011 2012 static void init_authenc_job(struct aead_request *req, 2013 struct aead_edesc *edesc, 2014 bool all_contig, bool encrypt) 2015 { 2016 struct crypto_aead *aead = crypto_aead_reqtfm(req); 2017 struct caam_aead_alg *alg = container_of(crypto_aead_alg(aead), 2018 struct caam_aead_alg, aead); 2019 unsigned int ivsize = crypto_aead_ivsize(aead); 2020 struct caam_ctx *ctx = crypto_aead_ctx(aead); 2021 const bool ctr_mode = ((ctx->class1_alg_type & OP_ALG_AAI_MASK) == 2022 OP_ALG_AAI_CTR_MOD128); 2023 const bool is_rfc3686 = alg->caam.rfc3686; 2024 u32 *desc = edesc->hw_desc; 2025 u32 ivoffset = 0; 2026 2027 /* 2028 * AES-CTR needs to load IV in CONTEXT1 reg 2029 * at an offset of 128bits (16bytes) 2030 * CONTEXT1[255:128] = IV 2031 */ 2032 if (ctr_mode) 2033 ivoffset = 16; 2034 2035 /* 2036 * RFC3686 specific: 2037 * CONTEXT1[255:128] = {NONCE, IV, COUNTER} 2038 */ 2039 if (is_rfc3686) 2040 ivoffset = 16 + CTR_RFC3686_NONCE_SIZE; 2041 2042 init_aead_job(req, edesc, all_contig, encrypt); 2043 2044 if (ivsize && (is_rfc3686 || !(alg->caam.geniv && encrypt))) 2045 append_load_as_imm(desc, req->iv, ivsize, 2046 LDST_CLASS_1_CCB | 2047 LDST_SRCDST_BYTE_CONTEXT | 2048 (ivoffset << LDST_OFFSET_SHIFT)); 2049 } 2050 2051 /* 2052 * Fill in ablkcipher job descriptor 2053 */ 2054 static void init_ablkcipher_job(u32 *sh_desc, dma_addr_t ptr, 2055 struct ablkcipher_edesc *edesc, 2056 struct ablkcipher_request *req, 2057 bool iv_contig) 2058 { 2059 struct crypto_ablkcipher *ablkcipher = crypto_ablkcipher_reqtfm(req); 2060 int ivsize = crypto_ablkcipher_ivsize(ablkcipher); 2061 u32 *desc = edesc->hw_desc; 2062 u32 out_options = 0, in_options; 2063 dma_addr_t dst_dma, src_dma; 2064 int len, sec4_sg_index = 0; 2065 2066 #ifdef DEBUG 2067 print_hex_dump(KERN_ERR, "presciv@"__stringify(__LINE__)": ", 2068 DUMP_PREFIX_ADDRESS, 16, 4, req->info, 2069 ivsize, 1); 2070 print_hex_dump(KERN_ERR, "src @"__stringify(__LINE__)": ", 2071 DUMP_PREFIX_ADDRESS, 16, 4, sg_virt(req->src), 2072 edesc->src_nents ? 100 : req->nbytes, 1); 2073 #endif 2074 2075 len = desc_len(sh_desc); 2076 init_job_desc_shared(desc, ptr, len, HDR_SHARE_DEFER | HDR_REVERSE); 2077 2078 if (iv_contig) { 2079 src_dma = edesc->iv_dma; 2080 in_options = 0; 2081 } else { 2082 src_dma = edesc->sec4_sg_dma; 2083 sec4_sg_index += edesc->src_nents + 1; 2084 in_options = LDST_SGF; 2085 } 2086 append_seq_in_ptr(desc, src_dma, req->nbytes + ivsize, in_options); 2087 2088 if (likely(req->src == req->dst)) { 2089 if (!edesc->src_nents && iv_contig) { 2090 dst_dma = sg_dma_address(req->src); 2091 } else { 2092 dst_dma = edesc->sec4_sg_dma + 2093 sizeof(struct sec4_sg_entry); 2094 out_options = LDST_SGF; 2095 } 2096 } else { 2097 if (!edesc->dst_nents) { 2098 dst_dma = sg_dma_address(req->dst); 2099 } else { 2100 dst_dma = edesc->sec4_sg_dma + 2101 sec4_sg_index * sizeof(struct sec4_sg_entry); 2102 out_options = LDST_SGF; 2103 } 2104 } 2105 append_seq_out_ptr(desc, dst_dma, req->nbytes, out_options); 2106 } 2107 2108 /* 2109 * Fill in ablkcipher givencrypt job descriptor 2110 */ 2111 static void init_ablkcipher_giv_job(u32 *sh_desc, dma_addr_t ptr, 2112 struct ablkcipher_edesc *edesc, 2113 struct ablkcipher_request *req, 2114 bool iv_contig) 2115 { 2116 struct crypto_ablkcipher *ablkcipher = crypto_ablkcipher_reqtfm(req); 2117 int ivsize = crypto_ablkcipher_ivsize(ablkcipher); 2118 u32 *desc = edesc->hw_desc; 2119 u32 out_options, in_options; 2120 dma_addr_t dst_dma, src_dma; 2121 int len, sec4_sg_index = 0; 2122 2123 #ifdef DEBUG 2124 print_hex_dump(KERN_ERR, "presciv@" __stringify(__LINE__) ": ", 2125 DUMP_PREFIX_ADDRESS, 16, 4, req->info, 2126 ivsize, 1); 2127 print_hex_dump(KERN_ERR, "src @" __stringify(__LINE__) ": ", 2128 DUMP_PREFIX_ADDRESS, 16, 4, sg_virt(req->src), 2129 edesc->src_nents ? 100 : req->nbytes, 1); 2130 #endif 2131 2132 len = desc_len(sh_desc); 2133 init_job_desc_shared(desc, ptr, len, HDR_SHARE_DEFER | HDR_REVERSE); 2134 2135 if (!edesc->src_nents) { 2136 src_dma = sg_dma_address(req->src); 2137 in_options = 0; 2138 } else { 2139 src_dma = edesc->sec4_sg_dma; 2140 sec4_sg_index += edesc->src_nents; 2141 in_options = LDST_SGF; 2142 } 2143 append_seq_in_ptr(desc, src_dma, req->nbytes, in_options); 2144 2145 if (iv_contig) { 2146 dst_dma = edesc->iv_dma; 2147 out_options = 0; 2148 } else { 2149 dst_dma = edesc->sec4_sg_dma + 2150 sec4_sg_index * sizeof(struct sec4_sg_entry); 2151 out_options = LDST_SGF; 2152 } 2153 append_seq_out_ptr(desc, dst_dma, req->nbytes + ivsize, out_options); 2154 } 2155 2156 /* 2157 * allocate and map the aead extended descriptor 2158 */ 2159 static struct aead_edesc *aead_edesc_alloc(struct aead_request *req, 2160 int desc_bytes, bool *all_contig_ptr, 2161 bool encrypt) 2162 { 2163 struct crypto_aead *aead = crypto_aead_reqtfm(req); 2164 struct caam_ctx *ctx = crypto_aead_ctx(aead); 2165 struct device *jrdev = ctx->jrdev; 2166 gfp_t flags = (req->base.flags & (CRYPTO_TFM_REQ_MAY_BACKLOG | 2167 CRYPTO_TFM_REQ_MAY_SLEEP)) ? GFP_KERNEL : GFP_ATOMIC; 2168 int src_nents, dst_nents = 0; 2169 struct aead_edesc *edesc; 2170 int sgc; 2171 bool all_contig = true; 2172 bool src_chained = false, dst_chained = false; 2173 int sec4_sg_index, sec4_sg_len = 0, sec4_sg_bytes; 2174 unsigned int authsize = ctx->authsize; 2175 2176 if (unlikely(req->dst != req->src)) { 2177 src_nents = sg_count(req->src, req->assoclen + req->cryptlen, 2178 &src_chained); 2179 dst_nents = sg_count(req->dst, 2180 req->assoclen + req->cryptlen + 2181 (encrypt ? authsize : (-authsize)), 2182 &dst_chained); 2183 } else { 2184 src_nents = sg_count(req->src, 2185 req->assoclen + req->cryptlen + 2186 (encrypt ? authsize : 0), 2187 &src_chained); 2188 } 2189 2190 /* Check if data are contiguous. */ 2191 all_contig = !src_nents; 2192 if (!all_contig) { 2193 src_nents = src_nents ? : 1; 2194 sec4_sg_len = src_nents; 2195 } 2196 2197 sec4_sg_len += dst_nents; 2198 2199 sec4_sg_bytes = sec4_sg_len * sizeof(struct sec4_sg_entry); 2200 2201 /* allocate space for base edesc and hw desc commands, link tables */ 2202 edesc = kzalloc(sizeof(*edesc) + desc_bytes + sec4_sg_bytes, 2203 GFP_DMA | flags); 2204 if (!edesc) { 2205 dev_err(jrdev, "could not allocate extended descriptor\n"); 2206 return ERR_PTR(-ENOMEM); 2207 } 2208 2209 if (likely(req->src == req->dst)) { 2210 sgc = dma_map_sg_chained(jrdev, req->src, src_nents ? : 1, 2211 DMA_BIDIRECTIONAL, src_chained); 2212 if (unlikely(!sgc)) { 2213 dev_err(jrdev, "unable to map source\n"); 2214 kfree(edesc); 2215 return ERR_PTR(-ENOMEM); 2216 } 2217 } else { 2218 sgc = dma_map_sg_chained(jrdev, req->src, src_nents ? : 1, 2219 DMA_TO_DEVICE, src_chained); 2220 if (unlikely(!sgc)) { 2221 dev_err(jrdev, "unable to map source\n"); 2222 kfree(edesc); 2223 return ERR_PTR(-ENOMEM); 2224 } 2225 2226 sgc = dma_map_sg_chained(jrdev, req->dst, dst_nents ? : 1, 2227 DMA_FROM_DEVICE, dst_chained); 2228 if (unlikely(!sgc)) { 2229 dev_err(jrdev, "unable to map destination\n"); 2230 dma_unmap_sg_chained(jrdev, req->src, src_nents ? : 1, 2231 DMA_TO_DEVICE, src_chained); 2232 kfree(edesc); 2233 return ERR_PTR(-ENOMEM); 2234 } 2235 } 2236 2237 edesc->src_nents = src_nents; 2238 edesc->src_chained = src_chained; 2239 edesc->dst_nents = dst_nents; 2240 edesc->dst_chained = dst_chained; 2241 edesc->sec4_sg = (void *)edesc + sizeof(struct aead_edesc) + 2242 desc_bytes; 2243 *all_contig_ptr = all_contig; 2244 2245 sec4_sg_index = 0; 2246 if (!all_contig) { 2247 sg_to_sec4_sg_last(req->src, src_nents, 2248 edesc->sec4_sg + sec4_sg_index, 0); 2249 sec4_sg_index += src_nents; 2250 } 2251 if (dst_nents) { 2252 sg_to_sec4_sg_last(req->dst, dst_nents, 2253 edesc->sec4_sg + sec4_sg_index, 0); 2254 } 2255 2256 if (!sec4_sg_bytes) 2257 return edesc; 2258 2259 edesc->sec4_sg_dma = dma_map_single(jrdev, edesc->sec4_sg, 2260 sec4_sg_bytes, DMA_TO_DEVICE); 2261 if (dma_mapping_error(jrdev, edesc->sec4_sg_dma)) { 2262 dev_err(jrdev, "unable to map S/G table\n"); 2263 aead_unmap(jrdev, edesc, req); 2264 kfree(edesc); 2265 return ERR_PTR(-ENOMEM); 2266 } 2267 2268 edesc->sec4_sg_bytes = sec4_sg_bytes; 2269 2270 return edesc; 2271 } 2272 2273 static int gcm_encrypt(struct aead_request *req) 2274 { 2275 struct aead_edesc *edesc; 2276 struct crypto_aead *aead = crypto_aead_reqtfm(req); 2277 struct caam_ctx *ctx = crypto_aead_ctx(aead); 2278 struct device *jrdev = ctx->jrdev; 2279 bool all_contig; 2280 u32 *desc; 2281 int ret = 0; 2282 2283 /* allocate extended descriptor */ 2284 edesc = aead_edesc_alloc(req, GCM_DESC_JOB_IO_LEN, &all_contig, true); 2285 if (IS_ERR(edesc)) 2286 return PTR_ERR(edesc); 2287 2288 /* Create and submit job descriptor */ 2289 init_gcm_job(req, edesc, all_contig, true); 2290 #ifdef DEBUG 2291 print_hex_dump(KERN_ERR, "aead jobdesc@"__stringify(__LINE__)": ", 2292 DUMP_PREFIX_ADDRESS, 16, 4, edesc->hw_desc, 2293 desc_bytes(edesc->hw_desc), 1); 2294 #endif 2295 2296 desc = edesc->hw_desc; 2297 ret = caam_jr_enqueue(jrdev, desc, aead_encrypt_done, req); 2298 if (!ret) { 2299 ret = -EINPROGRESS; 2300 } else { 2301 aead_unmap(jrdev, edesc, req); 2302 kfree(edesc); 2303 } 2304 2305 return ret; 2306 } 2307 2308 static int ipsec_gcm_encrypt(struct aead_request *req) 2309 { 2310 if (req->assoclen < 8) 2311 return -EINVAL; 2312 2313 return gcm_encrypt(req); 2314 } 2315 2316 static int aead_encrypt(struct aead_request *req) 2317 { 2318 struct aead_edesc *edesc; 2319 struct crypto_aead *aead = crypto_aead_reqtfm(req); 2320 struct caam_ctx *ctx = crypto_aead_ctx(aead); 2321 struct device *jrdev = ctx->jrdev; 2322 bool all_contig; 2323 u32 *desc; 2324 int ret = 0; 2325 2326 /* allocate extended descriptor */ 2327 edesc = aead_edesc_alloc(req, AUTHENC_DESC_JOB_IO_LEN, 2328 &all_contig, true); 2329 if (IS_ERR(edesc)) 2330 return PTR_ERR(edesc); 2331 2332 /* Create and submit job descriptor */ 2333 init_authenc_job(req, edesc, all_contig, true); 2334 #ifdef DEBUG 2335 print_hex_dump(KERN_ERR, "aead jobdesc@"__stringify(__LINE__)": ", 2336 DUMP_PREFIX_ADDRESS, 16, 4, edesc->hw_desc, 2337 desc_bytes(edesc->hw_desc), 1); 2338 #endif 2339 2340 desc = edesc->hw_desc; 2341 ret = caam_jr_enqueue(jrdev, desc, aead_encrypt_done, req); 2342 if (!ret) { 2343 ret = -EINPROGRESS; 2344 } else { 2345 aead_unmap(jrdev, edesc, req); 2346 kfree(edesc); 2347 } 2348 2349 return ret; 2350 } 2351 2352 static int gcm_decrypt(struct aead_request *req) 2353 { 2354 struct aead_edesc *edesc; 2355 struct crypto_aead *aead = crypto_aead_reqtfm(req); 2356 struct caam_ctx *ctx = crypto_aead_ctx(aead); 2357 struct device *jrdev = ctx->jrdev; 2358 bool all_contig; 2359 u32 *desc; 2360 int ret = 0; 2361 2362 /* allocate extended descriptor */ 2363 edesc = aead_edesc_alloc(req, GCM_DESC_JOB_IO_LEN, &all_contig, false); 2364 if (IS_ERR(edesc)) 2365 return PTR_ERR(edesc); 2366 2367 /* Create and submit job descriptor*/ 2368 init_gcm_job(req, edesc, all_contig, false); 2369 #ifdef DEBUG 2370 print_hex_dump(KERN_ERR, "aead jobdesc@"__stringify(__LINE__)": ", 2371 DUMP_PREFIX_ADDRESS, 16, 4, edesc->hw_desc, 2372 desc_bytes(edesc->hw_desc), 1); 2373 #endif 2374 2375 desc = edesc->hw_desc; 2376 ret = caam_jr_enqueue(jrdev, desc, aead_decrypt_done, req); 2377 if (!ret) { 2378 ret = -EINPROGRESS; 2379 } else { 2380 aead_unmap(jrdev, edesc, req); 2381 kfree(edesc); 2382 } 2383 2384 return ret; 2385 } 2386 2387 static int ipsec_gcm_decrypt(struct aead_request *req) 2388 { 2389 if (req->assoclen < 8) 2390 return -EINVAL; 2391 2392 return gcm_decrypt(req); 2393 } 2394 2395 static int aead_decrypt(struct aead_request *req) 2396 { 2397 struct aead_edesc *edesc; 2398 struct crypto_aead *aead = crypto_aead_reqtfm(req); 2399 struct caam_ctx *ctx = crypto_aead_ctx(aead); 2400 struct device *jrdev = ctx->jrdev; 2401 bool all_contig; 2402 u32 *desc; 2403 int ret = 0; 2404 2405 /* allocate extended descriptor */ 2406 edesc = aead_edesc_alloc(req, AUTHENC_DESC_JOB_IO_LEN, 2407 &all_contig, false); 2408 if (IS_ERR(edesc)) 2409 return PTR_ERR(edesc); 2410 2411 #ifdef DEBUG 2412 print_hex_dump(KERN_ERR, "dec src@"__stringify(__LINE__)": ", 2413 DUMP_PREFIX_ADDRESS, 16, 4, sg_virt(req->src), 2414 req->assoclen + req->cryptlen, 1); 2415 #endif 2416 2417 /* Create and submit job descriptor*/ 2418 init_authenc_job(req, edesc, all_contig, false); 2419 #ifdef DEBUG 2420 print_hex_dump(KERN_ERR, "aead jobdesc@"__stringify(__LINE__)": ", 2421 DUMP_PREFIX_ADDRESS, 16, 4, edesc->hw_desc, 2422 desc_bytes(edesc->hw_desc), 1); 2423 #endif 2424 2425 desc = edesc->hw_desc; 2426 ret = caam_jr_enqueue(jrdev, desc, aead_decrypt_done, req); 2427 if (!ret) { 2428 ret = -EINPROGRESS; 2429 } else { 2430 aead_unmap(jrdev, edesc, req); 2431 kfree(edesc); 2432 } 2433 2434 return ret; 2435 } 2436 2437 static int aead_givdecrypt(struct aead_request *req) 2438 { 2439 struct crypto_aead *aead = crypto_aead_reqtfm(req); 2440 unsigned int ivsize = crypto_aead_ivsize(aead); 2441 2442 if (req->cryptlen < ivsize) 2443 return -EINVAL; 2444 2445 req->cryptlen -= ivsize; 2446 req->assoclen += ivsize; 2447 2448 return aead_decrypt(req); 2449 } 2450 2451 /* 2452 * allocate and map the ablkcipher extended descriptor for ablkcipher 2453 */ 2454 static struct ablkcipher_edesc *ablkcipher_edesc_alloc(struct ablkcipher_request 2455 *req, int desc_bytes, 2456 bool *iv_contig_out) 2457 { 2458 struct crypto_ablkcipher *ablkcipher = crypto_ablkcipher_reqtfm(req); 2459 struct caam_ctx *ctx = crypto_ablkcipher_ctx(ablkcipher); 2460 struct device *jrdev = ctx->jrdev; 2461 gfp_t flags = (req->base.flags & (CRYPTO_TFM_REQ_MAY_BACKLOG | 2462 CRYPTO_TFM_REQ_MAY_SLEEP)) ? 2463 GFP_KERNEL : GFP_ATOMIC; 2464 int src_nents, dst_nents = 0, sec4_sg_bytes; 2465 struct ablkcipher_edesc *edesc; 2466 dma_addr_t iv_dma = 0; 2467 bool iv_contig = false; 2468 int sgc; 2469 int ivsize = crypto_ablkcipher_ivsize(ablkcipher); 2470 bool src_chained = false, dst_chained = false; 2471 int sec4_sg_index; 2472 2473 src_nents = sg_count(req->src, req->nbytes, &src_chained); 2474 2475 if (req->dst != req->src) 2476 dst_nents = sg_count(req->dst, req->nbytes, &dst_chained); 2477 2478 if (likely(req->src == req->dst)) { 2479 sgc = dma_map_sg_chained(jrdev, req->src, src_nents ? : 1, 2480 DMA_BIDIRECTIONAL, src_chained); 2481 } else { 2482 sgc = dma_map_sg_chained(jrdev, req->src, src_nents ? : 1, 2483 DMA_TO_DEVICE, src_chained); 2484 sgc = dma_map_sg_chained(jrdev, req->dst, dst_nents ? : 1, 2485 DMA_FROM_DEVICE, dst_chained); 2486 } 2487 2488 iv_dma = dma_map_single(jrdev, req->info, ivsize, DMA_TO_DEVICE); 2489 if (dma_mapping_error(jrdev, iv_dma)) { 2490 dev_err(jrdev, "unable to map IV\n"); 2491 return ERR_PTR(-ENOMEM); 2492 } 2493 2494 /* 2495 * Check if iv can be contiguous with source and destination. 2496 * If so, include it. If not, create scatterlist. 2497 */ 2498 if (!src_nents && iv_dma + ivsize == sg_dma_address(req->src)) 2499 iv_contig = true; 2500 else 2501 src_nents = src_nents ? : 1; 2502 sec4_sg_bytes = ((iv_contig ? 0 : 1) + src_nents + dst_nents) * 2503 sizeof(struct sec4_sg_entry); 2504 2505 /* allocate space for base edesc and hw desc commands, link tables */ 2506 edesc = kzalloc(sizeof(*edesc) + desc_bytes + sec4_sg_bytes, 2507 GFP_DMA | flags); 2508 if (!edesc) { 2509 dev_err(jrdev, "could not allocate extended descriptor\n"); 2510 return ERR_PTR(-ENOMEM); 2511 } 2512 2513 edesc->src_nents = src_nents; 2514 edesc->src_chained = src_chained; 2515 edesc->dst_nents = dst_nents; 2516 edesc->dst_chained = dst_chained; 2517 edesc->sec4_sg_bytes = sec4_sg_bytes; 2518 edesc->sec4_sg = (void *)edesc + sizeof(struct ablkcipher_edesc) + 2519 desc_bytes; 2520 2521 sec4_sg_index = 0; 2522 if (!iv_contig) { 2523 dma_to_sec4_sg_one(edesc->sec4_sg, iv_dma, ivsize, 0); 2524 sg_to_sec4_sg_last(req->src, src_nents, 2525 edesc->sec4_sg + 1, 0); 2526 sec4_sg_index += 1 + src_nents; 2527 } 2528 2529 if (dst_nents) { 2530 sg_to_sec4_sg_last(req->dst, dst_nents, 2531 edesc->sec4_sg + sec4_sg_index, 0); 2532 } 2533 2534 edesc->sec4_sg_dma = dma_map_single(jrdev, edesc->sec4_sg, 2535 sec4_sg_bytes, DMA_TO_DEVICE); 2536 if (dma_mapping_error(jrdev, edesc->sec4_sg_dma)) { 2537 dev_err(jrdev, "unable to map S/G table\n"); 2538 return ERR_PTR(-ENOMEM); 2539 } 2540 2541 edesc->iv_dma = iv_dma; 2542 2543 #ifdef DEBUG 2544 print_hex_dump(KERN_ERR, "ablkcipher sec4_sg@"__stringify(__LINE__)": ", 2545 DUMP_PREFIX_ADDRESS, 16, 4, edesc->sec4_sg, 2546 sec4_sg_bytes, 1); 2547 #endif 2548 2549 *iv_contig_out = iv_contig; 2550 return edesc; 2551 } 2552 2553 static int ablkcipher_encrypt(struct ablkcipher_request *req) 2554 { 2555 struct ablkcipher_edesc *edesc; 2556 struct crypto_ablkcipher *ablkcipher = crypto_ablkcipher_reqtfm(req); 2557 struct caam_ctx *ctx = crypto_ablkcipher_ctx(ablkcipher); 2558 struct device *jrdev = ctx->jrdev; 2559 bool iv_contig; 2560 u32 *desc; 2561 int ret = 0; 2562 2563 /* allocate extended descriptor */ 2564 edesc = ablkcipher_edesc_alloc(req, DESC_JOB_IO_LEN * 2565 CAAM_CMD_SZ, &iv_contig); 2566 if (IS_ERR(edesc)) 2567 return PTR_ERR(edesc); 2568 2569 /* Create and submit job descriptor*/ 2570 init_ablkcipher_job(ctx->sh_desc_enc, 2571 ctx->sh_desc_enc_dma, edesc, req, iv_contig); 2572 #ifdef DEBUG 2573 print_hex_dump(KERN_ERR, "ablkcipher jobdesc@"__stringify(__LINE__)": ", 2574 DUMP_PREFIX_ADDRESS, 16, 4, edesc->hw_desc, 2575 desc_bytes(edesc->hw_desc), 1); 2576 #endif 2577 desc = edesc->hw_desc; 2578 ret = caam_jr_enqueue(jrdev, desc, ablkcipher_encrypt_done, req); 2579 2580 if (!ret) { 2581 ret = -EINPROGRESS; 2582 } else { 2583 ablkcipher_unmap(jrdev, edesc, req); 2584 kfree(edesc); 2585 } 2586 2587 return ret; 2588 } 2589 2590 static int ablkcipher_decrypt(struct ablkcipher_request *req) 2591 { 2592 struct ablkcipher_edesc *edesc; 2593 struct crypto_ablkcipher *ablkcipher = crypto_ablkcipher_reqtfm(req); 2594 struct caam_ctx *ctx = crypto_ablkcipher_ctx(ablkcipher); 2595 struct device *jrdev = ctx->jrdev; 2596 bool iv_contig; 2597 u32 *desc; 2598 int ret = 0; 2599 2600 /* allocate extended descriptor */ 2601 edesc = ablkcipher_edesc_alloc(req, DESC_JOB_IO_LEN * 2602 CAAM_CMD_SZ, &iv_contig); 2603 if (IS_ERR(edesc)) 2604 return PTR_ERR(edesc); 2605 2606 /* Create and submit job descriptor*/ 2607 init_ablkcipher_job(ctx->sh_desc_dec, 2608 ctx->sh_desc_dec_dma, edesc, req, iv_contig); 2609 desc = edesc->hw_desc; 2610 #ifdef DEBUG 2611 print_hex_dump(KERN_ERR, "ablkcipher jobdesc@"__stringify(__LINE__)": ", 2612 DUMP_PREFIX_ADDRESS, 16, 4, edesc->hw_desc, 2613 desc_bytes(edesc->hw_desc), 1); 2614 #endif 2615 2616 ret = caam_jr_enqueue(jrdev, desc, ablkcipher_decrypt_done, req); 2617 if (!ret) { 2618 ret = -EINPROGRESS; 2619 } else { 2620 ablkcipher_unmap(jrdev, edesc, req); 2621 kfree(edesc); 2622 } 2623 2624 return ret; 2625 } 2626 2627 /* 2628 * allocate and map the ablkcipher extended descriptor 2629 * for ablkcipher givencrypt 2630 */ 2631 static struct ablkcipher_edesc *ablkcipher_giv_edesc_alloc( 2632 struct skcipher_givcrypt_request *greq, 2633 int desc_bytes, 2634 bool *iv_contig_out) 2635 { 2636 struct ablkcipher_request *req = &greq->creq; 2637 struct crypto_ablkcipher *ablkcipher = crypto_ablkcipher_reqtfm(req); 2638 struct caam_ctx *ctx = crypto_ablkcipher_ctx(ablkcipher); 2639 struct device *jrdev = ctx->jrdev; 2640 gfp_t flags = (req->base.flags & (CRYPTO_TFM_REQ_MAY_BACKLOG | 2641 CRYPTO_TFM_REQ_MAY_SLEEP)) ? 2642 GFP_KERNEL : GFP_ATOMIC; 2643 int src_nents, dst_nents = 0, sec4_sg_bytes; 2644 struct ablkcipher_edesc *edesc; 2645 dma_addr_t iv_dma = 0; 2646 bool iv_contig = false; 2647 int sgc; 2648 int ivsize = crypto_ablkcipher_ivsize(ablkcipher); 2649 bool src_chained = false, dst_chained = false; 2650 int sec4_sg_index; 2651 2652 src_nents = sg_count(req->src, req->nbytes, &src_chained); 2653 2654 if (unlikely(req->dst != req->src)) 2655 dst_nents = sg_count(req->dst, req->nbytes, &dst_chained); 2656 2657 if (likely(req->src == req->dst)) { 2658 sgc = dma_map_sg_chained(jrdev, req->src, src_nents ? : 1, 2659 DMA_BIDIRECTIONAL, src_chained); 2660 } else { 2661 sgc = dma_map_sg_chained(jrdev, req->src, src_nents ? : 1, 2662 DMA_TO_DEVICE, src_chained); 2663 sgc = dma_map_sg_chained(jrdev, req->dst, dst_nents ? : 1, 2664 DMA_FROM_DEVICE, dst_chained); 2665 } 2666 2667 /* 2668 * Check if iv can be contiguous with source and destination. 2669 * If so, include it. If not, create scatterlist. 2670 */ 2671 iv_dma = dma_map_single(jrdev, greq->giv, ivsize, DMA_TO_DEVICE); 2672 if (dma_mapping_error(jrdev, iv_dma)) { 2673 dev_err(jrdev, "unable to map IV\n"); 2674 return ERR_PTR(-ENOMEM); 2675 } 2676 2677 if (!dst_nents && iv_dma + ivsize == sg_dma_address(req->dst)) 2678 iv_contig = true; 2679 else 2680 dst_nents = dst_nents ? : 1; 2681 sec4_sg_bytes = ((iv_contig ? 0 : 1) + src_nents + dst_nents) * 2682 sizeof(struct sec4_sg_entry); 2683 2684 /* allocate space for base edesc and hw desc commands, link tables */ 2685 edesc = kzalloc(sizeof(*edesc) + desc_bytes + sec4_sg_bytes, 2686 GFP_DMA | flags); 2687 if (!edesc) { 2688 dev_err(jrdev, "could not allocate extended descriptor\n"); 2689 return ERR_PTR(-ENOMEM); 2690 } 2691 2692 edesc->src_nents = src_nents; 2693 edesc->src_chained = src_chained; 2694 edesc->dst_nents = dst_nents; 2695 edesc->dst_chained = dst_chained; 2696 edesc->sec4_sg_bytes = sec4_sg_bytes; 2697 edesc->sec4_sg = (void *)edesc + sizeof(struct ablkcipher_edesc) + 2698 desc_bytes; 2699 2700 sec4_sg_index = 0; 2701 if (src_nents) { 2702 sg_to_sec4_sg_last(req->src, src_nents, edesc->sec4_sg, 0); 2703 sec4_sg_index += src_nents; 2704 } 2705 2706 if (!iv_contig) { 2707 dma_to_sec4_sg_one(edesc->sec4_sg + sec4_sg_index, 2708 iv_dma, ivsize, 0); 2709 sec4_sg_index += 1; 2710 sg_to_sec4_sg_last(req->dst, dst_nents, 2711 edesc->sec4_sg + sec4_sg_index, 0); 2712 } 2713 2714 edesc->sec4_sg_dma = dma_map_single(jrdev, edesc->sec4_sg, 2715 sec4_sg_bytes, DMA_TO_DEVICE); 2716 if (dma_mapping_error(jrdev, edesc->sec4_sg_dma)) { 2717 dev_err(jrdev, "unable to map S/G table\n"); 2718 return ERR_PTR(-ENOMEM); 2719 } 2720 edesc->iv_dma = iv_dma; 2721 2722 #ifdef DEBUG 2723 print_hex_dump(KERN_ERR, 2724 "ablkcipher sec4_sg@" __stringify(__LINE__) ": ", 2725 DUMP_PREFIX_ADDRESS, 16, 4, edesc->sec4_sg, 2726 sec4_sg_bytes, 1); 2727 #endif 2728 2729 *iv_contig_out = iv_contig; 2730 return edesc; 2731 } 2732 2733 static int ablkcipher_givencrypt(struct skcipher_givcrypt_request *creq) 2734 { 2735 struct ablkcipher_request *req = &creq->creq; 2736 struct ablkcipher_edesc *edesc; 2737 struct crypto_ablkcipher *ablkcipher = crypto_ablkcipher_reqtfm(req); 2738 struct caam_ctx *ctx = crypto_ablkcipher_ctx(ablkcipher); 2739 struct device *jrdev = ctx->jrdev; 2740 bool iv_contig; 2741 u32 *desc; 2742 int ret = 0; 2743 2744 /* allocate extended descriptor */ 2745 edesc = ablkcipher_giv_edesc_alloc(creq, DESC_JOB_IO_LEN * 2746 CAAM_CMD_SZ, &iv_contig); 2747 if (IS_ERR(edesc)) 2748 return PTR_ERR(edesc); 2749 2750 /* Create and submit job descriptor*/ 2751 init_ablkcipher_giv_job(ctx->sh_desc_givenc, ctx->sh_desc_givenc_dma, 2752 edesc, req, iv_contig); 2753 #ifdef DEBUG 2754 print_hex_dump(KERN_ERR, 2755 "ablkcipher jobdesc@" __stringify(__LINE__) ": ", 2756 DUMP_PREFIX_ADDRESS, 16, 4, edesc->hw_desc, 2757 desc_bytes(edesc->hw_desc), 1); 2758 #endif 2759 desc = edesc->hw_desc; 2760 ret = caam_jr_enqueue(jrdev, desc, ablkcipher_encrypt_done, req); 2761 2762 if (!ret) { 2763 ret = -EINPROGRESS; 2764 } else { 2765 ablkcipher_unmap(jrdev, edesc, req); 2766 kfree(edesc); 2767 } 2768 2769 return ret; 2770 } 2771 2772 #define template_aead template_u.aead 2773 #define template_ablkcipher template_u.ablkcipher 2774 struct caam_alg_template { 2775 char name[CRYPTO_MAX_ALG_NAME]; 2776 char driver_name[CRYPTO_MAX_ALG_NAME]; 2777 unsigned int blocksize; 2778 u32 type; 2779 union { 2780 struct ablkcipher_alg ablkcipher; 2781 } template_u; 2782 u32 class1_alg_type; 2783 u32 class2_alg_type; 2784 u32 alg_op; 2785 }; 2786 2787 static struct caam_alg_template driver_algs[] = { 2788 /* ablkcipher descriptor */ 2789 { 2790 .name = "cbc(aes)", 2791 .driver_name = "cbc-aes-caam", 2792 .blocksize = AES_BLOCK_SIZE, 2793 .type = CRYPTO_ALG_TYPE_GIVCIPHER, 2794 .template_ablkcipher = { 2795 .setkey = ablkcipher_setkey, 2796 .encrypt = ablkcipher_encrypt, 2797 .decrypt = ablkcipher_decrypt, 2798 .givencrypt = ablkcipher_givencrypt, 2799 .geniv = "<built-in>", 2800 .min_keysize = AES_MIN_KEY_SIZE, 2801 .max_keysize = AES_MAX_KEY_SIZE, 2802 .ivsize = AES_BLOCK_SIZE, 2803 }, 2804 .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CBC, 2805 }, 2806 { 2807 .name = "cbc(des3_ede)", 2808 .driver_name = "cbc-3des-caam", 2809 .blocksize = DES3_EDE_BLOCK_SIZE, 2810 .type = CRYPTO_ALG_TYPE_GIVCIPHER, 2811 .template_ablkcipher = { 2812 .setkey = ablkcipher_setkey, 2813 .encrypt = ablkcipher_encrypt, 2814 .decrypt = ablkcipher_decrypt, 2815 .givencrypt = ablkcipher_givencrypt, 2816 .geniv = "<built-in>", 2817 .min_keysize = DES3_EDE_KEY_SIZE, 2818 .max_keysize = DES3_EDE_KEY_SIZE, 2819 .ivsize = DES3_EDE_BLOCK_SIZE, 2820 }, 2821 .class1_alg_type = OP_ALG_ALGSEL_3DES | OP_ALG_AAI_CBC, 2822 }, 2823 { 2824 .name = "cbc(des)", 2825 .driver_name = "cbc-des-caam", 2826 .blocksize = DES_BLOCK_SIZE, 2827 .type = CRYPTO_ALG_TYPE_GIVCIPHER, 2828 .template_ablkcipher = { 2829 .setkey = ablkcipher_setkey, 2830 .encrypt = ablkcipher_encrypt, 2831 .decrypt = ablkcipher_decrypt, 2832 .givencrypt = ablkcipher_givencrypt, 2833 .geniv = "<built-in>", 2834 .min_keysize = DES_KEY_SIZE, 2835 .max_keysize = DES_KEY_SIZE, 2836 .ivsize = DES_BLOCK_SIZE, 2837 }, 2838 .class1_alg_type = OP_ALG_ALGSEL_DES | OP_ALG_AAI_CBC, 2839 }, 2840 { 2841 .name = "ctr(aes)", 2842 .driver_name = "ctr-aes-caam", 2843 .blocksize = 1, 2844 .type = CRYPTO_ALG_TYPE_ABLKCIPHER, 2845 .template_ablkcipher = { 2846 .setkey = ablkcipher_setkey, 2847 .encrypt = ablkcipher_encrypt, 2848 .decrypt = ablkcipher_decrypt, 2849 .geniv = "chainiv", 2850 .min_keysize = AES_MIN_KEY_SIZE, 2851 .max_keysize = AES_MAX_KEY_SIZE, 2852 .ivsize = AES_BLOCK_SIZE, 2853 }, 2854 .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CTR_MOD128, 2855 }, 2856 { 2857 .name = "rfc3686(ctr(aes))", 2858 .driver_name = "rfc3686-ctr-aes-caam", 2859 .blocksize = 1, 2860 .type = CRYPTO_ALG_TYPE_GIVCIPHER, 2861 .template_ablkcipher = { 2862 .setkey = ablkcipher_setkey, 2863 .encrypt = ablkcipher_encrypt, 2864 .decrypt = ablkcipher_decrypt, 2865 .givencrypt = ablkcipher_givencrypt, 2866 .geniv = "<built-in>", 2867 .min_keysize = AES_MIN_KEY_SIZE + 2868 CTR_RFC3686_NONCE_SIZE, 2869 .max_keysize = AES_MAX_KEY_SIZE + 2870 CTR_RFC3686_NONCE_SIZE, 2871 .ivsize = CTR_RFC3686_IV_SIZE, 2872 }, 2873 .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CTR_MOD128, 2874 } 2875 }; 2876 2877 static struct caam_aead_alg driver_aeads[] = { 2878 { 2879 .aead = { 2880 .base = { 2881 .cra_name = "rfc4106(gcm(aes))", 2882 .cra_driver_name = "rfc4106-gcm-aes-caam", 2883 .cra_blocksize = 1, 2884 }, 2885 .setkey = rfc4106_setkey, 2886 .setauthsize = rfc4106_setauthsize, 2887 .encrypt = ipsec_gcm_encrypt, 2888 .decrypt = ipsec_gcm_decrypt, 2889 .ivsize = 8, 2890 .maxauthsize = AES_BLOCK_SIZE, 2891 }, 2892 .caam = { 2893 .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_GCM, 2894 }, 2895 }, 2896 { 2897 .aead = { 2898 .base = { 2899 .cra_name = "rfc4543(gcm(aes))", 2900 .cra_driver_name = "rfc4543-gcm-aes-caam", 2901 .cra_blocksize = 1, 2902 }, 2903 .setkey = rfc4543_setkey, 2904 .setauthsize = rfc4543_setauthsize, 2905 .encrypt = ipsec_gcm_encrypt, 2906 .decrypt = ipsec_gcm_decrypt, 2907 .ivsize = 8, 2908 .maxauthsize = AES_BLOCK_SIZE, 2909 }, 2910 .caam = { 2911 .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_GCM, 2912 }, 2913 }, 2914 /* Galois Counter Mode */ 2915 { 2916 .aead = { 2917 .base = { 2918 .cra_name = "gcm(aes)", 2919 .cra_driver_name = "gcm-aes-caam", 2920 .cra_blocksize = 1, 2921 }, 2922 .setkey = gcm_setkey, 2923 .setauthsize = gcm_setauthsize, 2924 .encrypt = gcm_encrypt, 2925 .decrypt = gcm_decrypt, 2926 .ivsize = 12, 2927 .maxauthsize = AES_BLOCK_SIZE, 2928 }, 2929 .caam = { 2930 .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_GCM, 2931 }, 2932 }, 2933 /* single-pass ipsec_esp descriptor */ 2934 { 2935 .aead = { 2936 .base = { 2937 .cra_name = "authenc(hmac(md5)," 2938 "ecb(cipher_null))", 2939 .cra_driver_name = "authenc-hmac-md5-" 2940 "ecb-cipher_null-caam", 2941 .cra_blocksize = NULL_BLOCK_SIZE, 2942 }, 2943 .setkey = aead_setkey, 2944 .setauthsize = aead_setauthsize, 2945 .encrypt = aead_encrypt, 2946 .decrypt = aead_decrypt, 2947 .ivsize = NULL_IV_SIZE, 2948 .maxauthsize = MD5_DIGEST_SIZE, 2949 }, 2950 .caam = { 2951 .class2_alg_type = OP_ALG_ALGSEL_MD5 | 2952 OP_ALG_AAI_HMAC_PRECOMP, 2953 .alg_op = OP_ALG_ALGSEL_MD5 | OP_ALG_AAI_HMAC, 2954 }, 2955 }, 2956 { 2957 .aead = { 2958 .base = { 2959 .cra_name = "authenc(hmac(sha1)," 2960 "ecb(cipher_null))", 2961 .cra_driver_name = "authenc-hmac-sha1-" 2962 "ecb-cipher_null-caam", 2963 .cra_blocksize = NULL_BLOCK_SIZE, 2964 }, 2965 .setkey = aead_setkey, 2966 .setauthsize = aead_setauthsize, 2967 .encrypt = aead_encrypt, 2968 .decrypt = aead_decrypt, 2969 .ivsize = NULL_IV_SIZE, 2970 .maxauthsize = SHA1_DIGEST_SIZE, 2971 }, 2972 .caam = { 2973 .class2_alg_type = OP_ALG_ALGSEL_SHA1 | 2974 OP_ALG_AAI_HMAC_PRECOMP, 2975 .alg_op = OP_ALG_ALGSEL_SHA1 | OP_ALG_AAI_HMAC, 2976 }, 2977 }, 2978 { 2979 .aead = { 2980 .base = { 2981 .cra_name = "authenc(hmac(sha224)," 2982 "ecb(cipher_null))", 2983 .cra_driver_name = "authenc-hmac-sha224-" 2984 "ecb-cipher_null-caam", 2985 .cra_blocksize = NULL_BLOCK_SIZE, 2986 }, 2987 .setkey = aead_setkey, 2988 .setauthsize = aead_setauthsize, 2989 .encrypt = aead_encrypt, 2990 .decrypt = aead_decrypt, 2991 .ivsize = NULL_IV_SIZE, 2992 .maxauthsize = SHA224_DIGEST_SIZE, 2993 }, 2994 .caam = { 2995 .class2_alg_type = OP_ALG_ALGSEL_SHA224 | 2996 OP_ALG_AAI_HMAC_PRECOMP, 2997 .alg_op = OP_ALG_ALGSEL_SHA224 | OP_ALG_AAI_HMAC, 2998 }, 2999 }, 3000 { 3001 .aead = { 3002 .base = { 3003 .cra_name = "authenc(hmac(sha256)," 3004 "ecb(cipher_null))", 3005 .cra_driver_name = "authenc-hmac-sha256-" 3006 "ecb-cipher_null-caam", 3007 .cra_blocksize = NULL_BLOCK_SIZE, 3008 }, 3009 .setkey = aead_setkey, 3010 .setauthsize = aead_setauthsize, 3011 .encrypt = aead_encrypt, 3012 .decrypt = aead_decrypt, 3013 .ivsize = NULL_IV_SIZE, 3014 .maxauthsize = SHA256_DIGEST_SIZE, 3015 }, 3016 .caam = { 3017 .class2_alg_type = OP_ALG_ALGSEL_SHA256 | 3018 OP_ALG_AAI_HMAC_PRECOMP, 3019 .alg_op = OP_ALG_ALGSEL_SHA256 | OP_ALG_AAI_HMAC, 3020 }, 3021 }, 3022 { 3023 .aead = { 3024 .base = { 3025 .cra_name = "authenc(hmac(sha384)," 3026 "ecb(cipher_null))", 3027 .cra_driver_name = "authenc-hmac-sha384-" 3028 "ecb-cipher_null-caam", 3029 .cra_blocksize = NULL_BLOCK_SIZE, 3030 }, 3031 .setkey = aead_setkey, 3032 .setauthsize = aead_setauthsize, 3033 .encrypt = aead_encrypt, 3034 .decrypt = aead_decrypt, 3035 .ivsize = NULL_IV_SIZE, 3036 .maxauthsize = SHA384_DIGEST_SIZE, 3037 }, 3038 .caam = { 3039 .class2_alg_type = OP_ALG_ALGSEL_SHA384 | 3040 OP_ALG_AAI_HMAC_PRECOMP, 3041 .alg_op = OP_ALG_ALGSEL_SHA384 | OP_ALG_AAI_HMAC, 3042 }, 3043 }, 3044 { 3045 .aead = { 3046 .base = { 3047 .cra_name = "authenc(hmac(sha512)," 3048 "ecb(cipher_null))", 3049 .cra_driver_name = "authenc-hmac-sha512-" 3050 "ecb-cipher_null-caam", 3051 .cra_blocksize = NULL_BLOCK_SIZE, 3052 }, 3053 .setkey = aead_setkey, 3054 .setauthsize = aead_setauthsize, 3055 .encrypt = aead_encrypt, 3056 .decrypt = aead_decrypt, 3057 .ivsize = NULL_IV_SIZE, 3058 .maxauthsize = SHA512_DIGEST_SIZE, 3059 }, 3060 .caam = { 3061 .class2_alg_type = OP_ALG_ALGSEL_SHA512 | 3062 OP_ALG_AAI_HMAC_PRECOMP, 3063 .alg_op = OP_ALG_ALGSEL_SHA512 | OP_ALG_AAI_HMAC, 3064 }, 3065 }, 3066 { 3067 .aead = { 3068 .base = { 3069 .cra_name = "authenc(hmac(md5),cbc(aes))", 3070 .cra_driver_name = "authenc-hmac-md5-" 3071 "cbc-aes-caam", 3072 .cra_blocksize = AES_BLOCK_SIZE, 3073 }, 3074 .setkey = aead_setkey, 3075 .setauthsize = aead_setauthsize, 3076 .encrypt = aead_encrypt, 3077 .decrypt = aead_decrypt, 3078 .ivsize = AES_BLOCK_SIZE, 3079 .maxauthsize = MD5_DIGEST_SIZE, 3080 }, 3081 .caam = { 3082 .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CBC, 3083 .class2_alg_type = OP_ALG_ALGSEL_MD5 | 3084 OP_ALG_AAI_HMAC_PRECOMP, 3085 .alg_op = OP_ALG_ALGSEL_MD5 | OP_ALG_AAI_HMAC, 3086 }, 3087 }, 3088 { 3089 .aead = { 3090 .base = { 3091 .cra_name = "echainiv(authenc(hmac(md5)," 3092 "cbc(aes)))", 3093 .cra_driver_name = "echainiv-authenc-hmac-md5-" 3094 "cbc-aes-caam", 3095 .cra_blocksize = AES_BLOCK_SIZE, 3096 }, 3097 .setkey = aead_setkey, 3098 .setauthsize = aead_setauthsize, 3099 .encrypt = aead_encrypt, 3100 .decrypt = aead_givdecrypt, 3101 .ivsize = AES_BLOCK_SIZE, 3102 .maxauthsize = MD5_DIGEST_SIZE, 3103 }, 3104 .caam = { 3105 .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CBC, 3106 .class2_alg_type = OP_ALG_ALGSEL_MD5 | 3107 OP_ALG_AAI_HMAC_PRECOMP, 3108 .alg_op = OP_ALG_ALGSEL_MD5 | OP_ALG_AAI_HMAC, 3109 .geniv = true, 3110 }, 3111 }, 3112 { 3113 .aead = { 3114 .base = { 3115 .cra_name = "authenc(hmac(sha1),cbc(aes))", 3116 .cra_driver_name = "authenc-hmac-sha1-" 3117 "cbc-aes-caam", 3118 .cra_blocksize = AES_BLOCK_SIZE, 3119 }, 3120 .setkey = aead_setkey, 3121 .setauthsize = aead_setauthsize, 3122 .encrypt = aead_encrypt, 3123 .decrypt = aead_decrypt, 3124 .ivsize = AES_BLOCK_SIZE, 3125 .maxauthsize = SHA1_DIGEST_SIZE, 3126 }, 3127 .caam = { 3128 .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CBC, 3129 .class2_alg_type = OP_ALG_ALGSEL_SHA1 | 3130 OP_ALG_AAI_HMAC_PRECOMP, 3131 .alg_op = OP_ALG_ALGSEL_SHA1 | OP_ALG_AAI_HMAC, 3132 }, 3133 }, 3134 { 3135 .aead = { 3136 .base = { 3137 .cra_name = "echainiv(authenc(hmac(sha1)," 3138 "cbc(aes)))", 3139 .cra_driver_name = "echainiv-authenc-" 3140 "hmac-sha1-cbc-aes-caam", 3141 .cra_blocksize = AES_BLOCK_SIZE, 3142 }, 3143 .setkey = aead_setkey, 3144 .setauthsize = aead_setauthsize, 3145 .encrypt = aead_encrypt, 3146 .decrypt = aead_givdecrypt, 3147 .ivsize = AES_BLOCK_SIZE, 3148 .maxauthsize = SHA1_DIGEST_SIZE, 3149 }, 3150 .caam = { 3151 .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CBC, 3152 .class2_alg_type = OP_ALG_ALGSEL_SHA1 | 3153 OP_ALG_AAI_HMAC_PRECOMP, 3154 .alg_op = OP_ALG_ALGSEL_SHA1 | OP_ALG_AAI_HMAC, 3155 .geniv = true, 3156 }, 3157 }, 3158 { 3159 .aead = { 3160 .base = { 3161 .cra_name = "authenc(hmac(sha224),cbc(aes))", 3162 .cra_driver_name = "authenc-hmac-sha224-" 3163 "cbc-aes-caam", 3164 .cra_blocksize = AES_BLOCK_SIZE, 3165 }, 3166 .setkey = aead_setkey, 3167 .setauthsize = aead_setauthsize, 3168 .encrypt = aead_encrypt, 3169 .decrypt = aead_decrypt, 3170 .ivsize = AES_BLOCK_SIZE, 3171 .maxauthsize = SHA224_DIGEST_SIZE, 3172 }, 3173 .caam = { 3174 .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CBC, 3175 .class2_alg_type = OP_ALG_ALGSEL_SHA224 | 3176 OP_ALG_AAI_HMAC_PRECOMP, 3177 .alg_op = OP_ALG_ALGSEL_SHA224 | OP_ALG_AAI_HMAC, 3178 }, 3179 }, 3180 { 3181 .aead = { 3182 .base = { 3183 .cra_name = "echainiv(authenc(hmac(sha224)," 3184 "cbc(aes)))", 3185 .cra_driver_name = "echainiv-authenc-" 3186 "hmac-sha224-cbc-aes-caam", 3187 .cra_blocksize = AES_BLOCK_SIZE, 3188 }, 3189 .setkey = aead_setkey, 3190 .setauthsize = aead_setauthsize, 3191 .encrypt = aead_encrypt, 3192 .decrypt = aead_givdecrypt, 3193 .ivsize = AES_BLOCK_SIZE, 3194 .maxauthsize = SHA224_DIGEST_SIZE, 3195 }, 3196 .caam = { 3197 .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CBC, 3198 .class2_alg_type = OP_ALG_ALGSEL_SHA224 | 3199 OP_ALG_AAI_HMAC_PRECOMP, 3200 .alg_op = OP_ALG_ALGSEL_SHA224 | OP_ALG_AAI_HMAC, 3201 .geniv = true, 3202 }, 3203 }, 3204 { 3205 .aead = { 3206 .base = { 3207 .cra_name = "authenc(hmac(sha256),cbc(aes))", 3208 .cra_driver_name = "authenc-hmac-sha256-" 3209 "cbc-aes-caam", 3210 .cra_blocksize = AES_BLOCK_SIZE, 3211 }, 3212 .setkey = aead_setkey, 3213 .setauthsize = aead_setauthsize, 3214 .encrypt = aead_encrypt, 3215 .decrypt = aead_decrypt, 3216 .ivsize = AES_BLOCK_SIZE, 3217 .maxauthsize = SHA256_DIGEST_SIZE, 3218 }, 3219 .caam = { 3220 .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CBC, 3221 .class2_alg_type = OP_ALG_ALGSEL_SHA256 | 3222 OP_ALG_AAI_HMAC_PRECOMP, 3223 .alg_op = OP_ALG_ALGSEL_SHA256 | OP_ALG_AAI_HMAC, 3224 }, 3225 }, 3226 { 3227 .aead = { 3228 .base = { 3229 .cra_name = "echainiv(authenc(hmac(sha256)," 3230 "cbc(aes)))", 3231 .cra_driver_name = "echainiv-authenc-" 3232 "hmac-sha256-cbc-aes-caam", 3233 .cra_blocksize = AES_BLOCK_SIZE, 3234 }, 3235 .setkey = aead_setkey, 3236 .setauthsize = aead_setauthsize, 3237 .encrypt = aead_encrypt, 3238 .decrypt = aead_givdecrypt, 3239 .ivsize = AES_BLOCK_SIZE, 3240 .maxauthsize = SHA256_DIGEST_SIZE, 3241 }, 3242 .caam = { 3243 .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CBC, 3244 .class2_alg_type = OP_ALG_ALGSEL_SHA256 | 3245 OP_ALG_AAI_HMAC_PRECOMP, 3246 .alg_op = OP_ALG_ALGSEL_SHA256 | OP_ALG_AAI_HMAC, 3247 .geniv = true, 3248 }, 3249 }, 3250 { 3251 .aead = { 3252 .base = { 3253 .cra_name = "authenc(hmac(sha384),cbc(aes))", 3254 .cra_driver_name = "authenc-hmac-sha384-" 3255 "cbc-aes-caam", 3256 .cra_blocksize = AES_BLOCK_SIZE, 3257 }, 3258 .setkey = aead_setkey, 3259 .setauthsize = aead_setauthsize, 3260 .encrypt = aead_encrypt, 3261 .decrypt = aead_decrypt, 3262 .ivsize = AES_BLOCK_SIZE, 3263 .maxauthsize = SHA384_DIGEST_SIZE, 3264 }, 3265 .caam = { 3266 .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CBC, 3267 .class2_alg_type = OP_ALG_ALGSEL_SHA384 | 3268 OP_ALG_AAI_HMAC_PRECOMP, 3269 .alg_op = OP_ALG_ALGSEL_SHA384 | OP_ALG_AAI_HMAC, 3270 }, 3271 }, 3272 { 3273 .aead = { 3274 .base = { 3275 .cra_name = "echainiv(authenc(hmac(sha384)," 3276 "cbc(aes)))", 3277 .cra_driver_name = "echainiv-authenc-" 3278 "hmac-sha384-cbc-aes-caam", 3279 .cra_blocksize = AES_BLOCK_SIZE, 3280 }, 3281 .setkey = aead_setkey, 3282 .setauthsize = aead_setauthsize, 3283 .encrypt = aead_encrypt, 3284 .decrypt = aead_givdecrypt, 3285 .ivsize = AES_BLOCK_SIZE, 3286 .maxauthsize = SHA384_DIGEST_SIZE, 3287 }, 3288 .caam = { 3289 .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CBC, 3290 .class2_alg_type = OP_ALG_ALGSEL_SHA384 | 3291 OP_ALG_AAI_HMAC_PRECOMP, 3292 .alg_op = OP_ALG_ALGSEL_SHA384 | OP_ALG_AAI_HMAC, 3293 .geniv = true, 3294 }, 3295 }, 3296 { 3297 .aead = { 3298 .base = { 3299 .cra_name = "authenc(hmac(sha512),cbc(aes))", 3300 .cra_driver_name = "authenc-hmac-sha512-" 3301 "cbc-aes-caam", 3302 .cra_blocksize = AES_BLOCK_SIZE, 3303 }, 3304 .setkey = aead_setkey, 3305 .setauthsize = aead_setauthsize, 3306 .encrypt = aead_encrypt, 3307 .decrypt = aead_decrypt, 3308 .ivsize = AES_BLOCK_SIZE, 3309 .maxauthsize = SHA512_DIGEST_SIZE, 3310 }, 3311 .caam = { 3312 .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CBC, 3313 .class2_alg_type = OP_ALG_ALGSEL_SHA512 | 3314 OP_ALG_AAI_HMAC_PRECOMP, 3315 .alg_op = OP_ALG_ALGSEL_SHA512 | OP_ALG_AAI_HMAC, 3316 }, 3317 }, 3318 { 3319 .aead = { 3320 .base = { 3321 .cra_name = "echainiv(authenc(hmac(sha512)," 3322 "cbc(aes)))", 3323 .cra_driver_name = "echainiv-authenc-" 3324 "hmac-sha512-cbc-aes-caam", 3325 .cra_blocksize = AES_BLOCK_SIZE, 3326 }, 3327 .setkey = aead_setkey, 3328 .setauthsize = aead_setauthsize, 3329 .encrypt = aead_encrypt, 3330 .decrypt = aead_givdecrypt, 3331 .ivsize = AES_BLOCK_SIZE, 3332 .maxauthsize = SHA512_DIGEST_SIZE, 3333 }, 3334 .caam = { 3335 .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CBC, 3336 .class2_alg_type = OP_ALG_ALGSEL_SHA512 | 3337 OP_ALG_AAI_HMAC_PRECOMP, 3338 .alg_op = OP_ALG_ALGSEL_SHA512 | OP_ALG_AAI_HMAC, 3339 .geniv = true, 3340 }, 3341 }, 3342 { 3343 .aead = { 3344 .base = { 3345 .cra_name = "authenc(hmac(md5),cbc(des3_ede))", 3346 .cra_driver_name = "authenc-hmac-md5-" 3347 "cbc-des3_ede-caam", 3348 .cra_blocksize = DES3_EDE_BLOCK_SIZE, 3349 }, 3350 .setkey = aead_setkey, 3351 .setauthsize = aead_setauthsize, 3352 .encrypt = aead_encrypt, 3353 .decrypt = aead_decrypt, 3354 .ivsize = DES3_EDE_BLOCK_SIZE, 3355 .maxauthsize = MD5_DIGEST_SIZE, 3356 }, 3357 .caam = { 3358 .class1_alg_type = OP_ALG_ALGSEL_3DES | OP_ALG_AAI_CBC, 3359 .class2_alg_type = OP_ALG_ALGSEL_MD5 | 3360 OP_ALG_AAI_HMAC_PRECOMP, 3361 .alg_op = OP_ALG_ALGSEL_MD5 | OP_ALG_AAI_HMAC, 3362 } 3363 }, 3364 { 3365 .aead = { 3366 .base = { 3367 .cra_name = "echainiv(authenc(hmac(md5)," 3368 "cbc(des3_ede)))", 3369 .cra_driver_name = "echainiv-authenc-hmac-md5-" 3370 "cbc-des3_ede-caam", 3371 .cra_blocksize = DES3_EDE_BLOCK_SIZE, 3372 }, 3373 .setkey = aead_setkey, 3374 .setauthsize = aead_setauthsize, 3375 .encrypt = aead_encrypt, 3376 .decrypt = aead_givdecrypt, 3377 .ivsize = DES3_EDE_BLOCK_SIZE, 3378 .maxauthsize = MD5_DIGEST_SIZE, 3379 }, 3380 .caam = { 3381 .class1_alg_type = OP_ALG_ALGSEL_3DES | OP_ALG_AAI_CBC, 3382 .class2_alg_type = OP_ALG_ALGSEL_MD5 | 3383 OP_ALG_AAI_HMAC_PRECOMP, 3384 .alg_op = OP_ALG_ALGSEL_MD5 | OP_ALG_AAI_HMAC, 3385 .geniv = true, 3386 } 3387 }, 3388 { 3389 .aead = { 3390 .base = { 3391 .cra_name = "authenc(hmac(sha1)," 3392 "cbc(des3_ede))", 3393 .cra_driver_name = "authenc-hmac-sha1-" 3394 "cbc-des3_ede-caam", 3395 .cra_blocksize = DES3_EDE_BLOCK_SIZE, 3396 }, 3397 .setkey = aead_setkey, 3398 .setauthsize = aead_setauthsize, 3399 .encrypt = aead_encrypt, 3400 .decrypt = aead_decrypt, 3401 .ivsize = DES3_EDE_BLOCK_SIZE, 3402 .maxauthsize = SHA1_DIGEST_SIZE, 3403 }, 3404 .caam = { 3405 .class1_alg_type = OP_ALG_ALGSEL_3DES | OP_ALG_AAI_CBC, 3406 .class2_alg_type = OP_ALG_ALGSEL_SHA1 | 3407 OP_ALG_AAI_HMAC_PRECOMP, 3408 .alg_op = OP_ALG_ALGSEL_SHA1 | OP_ALG_AAI_HMAC, 3409 }, 3410 }, 3411 { 3412 .aead = { 3413 .base = { 3414 .cra_name = "echainiv(authenc(hmac(sha1)," 3415 "cbc(des3_ede)))", 3416 .cra_driver_name = "echainiv-authenc-" 3417 "hmac-sha1-" 3418 "cbc-des3_ede-caam", 3419 .cra_blocksize = DES3_EDE_BLOCK_SIZE, 3420 }, 3421 .setkey = aead_setkey, 3422 .setauthsize = aead_setauthsize, 3423 .encrypt = aead_encrypt, 3424 .decrypt = aead_givdecrypt, 3425 .ivsize = DES3_EDE_BLOCK_SIZE, 3426 .maxauthsize = SHA1_DIGEST_SIZE, 3427 }, 3428 .caam = { 3429 .class1_alg_type = OP_ALG_ALGSEL_3DES | OP_ALG_AAI_CBC, 3430 .class2_alg_type = OP_ALG_ALGSEL_SHA1 | 3431 OP_ALG_AAI_HMAC_PRECOMP, 3432 .alg_op = OP_ALG_ALGSEL_SHA1 | OP_ALG_AAI_HMAC, 3433 .geniv = true, 3434 }, 3435 }, 3436 { 3437 .aead = { 3438 .base = { 3439 .cra_name = "authenc(hmac(sha224)," 3440 "cbc(des3_ede))", 3441 .cra_driver_name = "authenc-hmac-sha224-" 3442 "cbc-des3_ede-caam", 3443 .cra_blocksize = DES3_EDE_BLOCK_SIZE, 3444 }, 3445 .setkey = aead_setkey, 3446 .setauthsize = aead_setauthsize, 3447 .encrypt = aead_encrypt, 3448 .decrypt = aead_decrypt, 3449 .ivsize = DES3_EDE_BLOCK_SIZE, 3450 .maxauthsize = SHA224_DIGEST_SIZE, 3451 }, 3452 .caam = { 3453 .class1_alg_type = OP_ALG_ALGSEL_3DES | OP_ALG_AAI_CBC, 3454 .class2_alg_type = OP_ALG_ALGSEL_SHA224 | 3455 OP_ALG_AAI_HMAC_PRECOMP, 3456 .alg_op = OP_ALG_ALGSEL_SHA224 | OP_ALG_AAI_HMAC, 3457 }, 3458 }, 3459 { 3460 .aead = { 3461 .base = { 3462 .cra_name = "echainiv(authenc(hmac(sha224)," 3463 "cbc(des3_ede)))", 3464 .cra_driver_name = "echainiv-authenc-" 3465 "hmac-sha224-" 3466 "cbc-des3_ede-caam", 3467 .cra_blocksize = DES3_EDE_BLOCK_SIZE, 3468 }, 3469 .setkey = aead_setkey, 3470 .setauthsize = aead_setauthsize, 3471 .encrypt = aead_encrypt, 3472 .decrypt = aead_givdecrypt, 3473 .ivsize = DES3_EDE_BLOCK_SIZE, 3474 .maxauthsize = SHA224_DIGEST_SIZE, 3475 }, 3476 .caam = { 3477 .class1_alg_type = OP_ALG_ALGSEL_3DES | OP_ALG_AAI_CBC, 3478 .class2_alg_type = OP_ALG_ALGSEL_SHA224 | 3479 OP_ALG_AAI_HMAC_PRECOMP, 3480 .alg_op = OP_ALG_ALGSEL_SHA224 | OP_ALG_AAI_HMAC, 3481 .geniv = true, 3482 }, 3483 }, 3484 { 3485 .aead = { 3486 .base = { 3487 .cra_name = "authenc(hmac(sha256)," 3488 "cbc(des3_ede))", 3489 .cra_driver_name = "authenc-hmac-sha256-" 3490 "cbc-des3_ede-caam", 3491 .cra_blocksize = DES3_EDE_BLOCK_SIZE, 3492 }, 3493 .setkey = aead_setkey, 3494 .setauthsize = aead_setauthsize, 3495 .encrypt = aead_encrypt, 3496 .decrypt = aead_decrypt, 3497 .ivsize = DES3_EDE_BLOCK_SIZE, 3498 .maxauthsize = SHA256_DIGEST_SIZE, 3499 }, 3500 .caam = { 3501 .class1_alg_type = OP_ALG_ALGSEL_3DES | OP_ALG_AAI_CBC, 3502 .class2_alg_type = OP_ALG_ALGSEL_SHA256 | 3503 OP_ALG_AAI_HMAC_PRECOMP, 3504 .alg_op = OP_ALG_ALGSEL_SHA256 | OP_ALG_AAI_HMAC, 3505 }, 3506 }, 3507 { 3508 .aead = { 3509 .base = { 3510 .cra_name = "echainiv(authenc(hmac(sha256)," 3511 "cbc(des3_ede)))", 3512 .cra_driver_name = "echainiv-authenc-" 3513 "hmac-sha256-" 3514 "cbc-des3_ede-caam", 3515 .cra_blocksize = DES3_EDE_BLOCK_SIZE, 3516 }, 3517 .setkey = aead_setkey, 3518 .setauthsize = aead_setauthsize, 3519 .encrypt = aead_encrypt, 3520 .decrypt = aead_givdecrypt, 3521 .ivsize = DES3_EDE_BLOCK_SIZE, 3522 .maxauthsize = SHA256_DIGEST_SIZE, 3523 }, 3524 .caam = { 3525 .class1_alg_type = OP_ALG_ALGSEL_3DES | OP_ALG_AAI_CBC, 3526 .class2_alg_type = OP_ALG_ALGSEL_SHA256 | 3527 OP_ALG_AAI_HMAC_PRECOMP, 3528 .alg_op = OP_ALG_ALGSEL_SHA256 | OP_ALG_AAI_HMAC, 3529 .geniv = true, 3530 }, 3531 }, 3532 { 3533 .aead = { 3534 .base = { 3535 .cra_name = "authenc(hmac(sha384)," 3536 "cbc(des3_ede))", 3537 .cra_driver_name = "authenc-hmac-sha384-" 3538 "cbc-des3_ede-caam", 3539 .cra_blocksize = DES3_EDE_BLOCK_SIZE, 3540 }, 3541 .setkey = aead_setkey, 3542 .setauthsize = aead_setauthsize, 3543 .encrypt = aead_encrypt, 3544 .decrypt = aead_decrypt, 3545 .ivsize = DES3_EDE_BLOCK_SIZE, 3546 .maxauthsize = SHA384_DIGEST_SIZE, 3547 }, 3548 .caam = { 3549 .class1_alg_type = OP_ALG_ALGSEL_3DES | OP_ALG_AAI_CBC, 3550 .class2_alg_type = OP_ALG_ALGSEL_SHA384 | 3551 OP_ALG_AAI_HMAC_PRECOMP, 3552 .alg_op = OP_ALG_ALGSEL_SHA384 | OP_ALG_AAI_HMAC, 3553 }, 3554 }, 3555 { 3556 .aead = { 3557 .base = { 3558 .cra_name = "echainiv(authenc(hmac(sha384)," 3559 "cbc(des3_ede)))", 3560 .cra_driver_name = "echainiv-authenc-" 3561 "hmac-sha384-" 3562 "cbc-des3_ede-caam", 3563 .cra_blocksize = DES3_EDE_BLOCK_SIZE, 3564 }, 3565 .setkey = aead_setkey, 3566 .setauthsize = aead_setauthsize, 3567 .encrypt = aead_encrypt, 3568 .decrypt = aead_givdecrypt, 3569 .ivsize = DES3_EDE_BLOCK_SIZE, 3570 .maxauthsize = SHA384_DIGEST_SIZE, 3571 }, 3572 .caam = { 3573 .class1_alg_type = OP_ALG_ALGSEL_3DES | OP_ALG_AAI_CBC, 3574 .class2_alg_type = OP_ALG_ALGSEL_SHA384 | 3575 OP_ALG_AAI_HMAC_PRECOMP, 3576 .alg_op = OP_ALG_ALGSEL_SHA384 | OP_ALG_AAI_HMAC, 3577 .geniv = true, 3578 }, 3579 }, 3580 { 3581 .aead = { 3582 .base = { 3583 .cra_name = "authenc(hmac(sha512)," 3584 "cbc(des3_ede))", 3585 .cra_driver_name = "authenc-hmac-sha512-" 3586 "cbc-des3_ede-caam", 3587 .cra_blocksize = DES3_EDE_BLOCK_SIZE, 3588 }, 3589 .setkey = aead_setkey, 3590 .setauthsize = aead_setauthsize, 3591 .encrypt = aead_encrypt, 3592 .decrypt = aead_decrypt, 3593 .ivsize = DES3_EDE_BLOCK_SIZE, 3594 .maxauthsize = SHA512_DIGEST_SIZE, 3595 }, 3596 .caam = { 3597 .class1_alg_type = OP_ALG_ALGSEL_3DES | OP_ALG_AAI_CBC, 3598 .class2_alg_type = OP_ALG_ALGSEL_SHA512 | 3599 OP_ALG_AAI_HMAC_PRECOMP, 3600 .alg_op = OP_ALG_ALGSEL_SHA512 | OP_ALG_AAI_HMAC, 3601 }, 3602 }, 3603 { 3604 .aead = { 3605 .base = { 3606 .cra_name = "echainiv(authenc(hmac(sha512)," 3607 "cbc(des3_ede)))", 3608 .cra_driver_name = "echainiv-authenc-" 3609 "hmac-sha512-" 3610 "cbc-des3_ede-caam", 3611 .cra_blocksize = DES3_EDE_BLOCK_SIZE, 3612 }, 3613 .setkey = aead_setkey, 3614 .setauthsize = aead_setauthsize, 3615 .encrypt = aead_encrypt, 3616 .decrypt = aead_givdecrypt, 3617 .ivsize = DES3_EDE_BLOCK_SIZE, 3618 .maxauthsize = SHA512_DIGEST_SIZE, 3619 }, 3620 .caam = { 3621 .class1_alg_type = OP_ALG_ALGSEL_3DES | OP_ALG_AAI_CBC, 3622 .class2_alg_type = OP_ALG_ALGSEL_SHA512 | 3623 OP_ALG_AAI_HMAC_PRECOMP, 3624 .alg_op = OP_ALG_ALGSEL_SHA512 | OP_ALG_AAI_HMAC, 3625 .geniv = true, 3626 }, 3627 }, 3628 { 3629 .aead = { 3630 .base = { 3631 .cra_name = "authenc(hmac(md5),cbc(des))", 3632 .cra_driver_name = "authenc-hmac-md5-" 3633 "cbc-des-caam", 3634 .cra_blocksize = DES_BLOCK_SIZE, 3635 }, 3636 .setkey = aead_setkey, 3637 .setauthsize = aead_setauthsize, 3638 .encrypt = aead_encrypt, 3639 .decrypt = aead_decrypt, 3640 .ivsize = DES_BLOCK_SIZE, 3641 .maxauthsize = MD5_DIGEST_SIZE, 3642 }, 3643 .caam = { 3644 .class1_alg_type = OP_ALG_ALGSEL_DES | OP_ALG_AAI_CBC, 3645 .class2_alg_type = OP_ALG_ALGSEL_MD5 | 3646 OP_ALG_AAI_HMAC_PRECOMP, 3647 .alg_op = OP_ALG_ALGSEL_MD5 | OP_ALG_AAI_HMAC, 3648 }, 3649 }, 3650 { 3651 .aead = { 3652 .base = { 3653 .cra_name = "echainiv(authenc(hmac(md5)," 3654 "cbc(des)))", 3655 .cra_driver_name = "echainiv-authenc-hmac-md5-" 3656 "cbc-des-caam", 3657 .cra_blocksize = DES_BLOCK_SIZE, 3658 }, 3659 .setkey = aead_setkey, 3660 .setauthsize = aead_setauthsize, 3661 .encrypt = aead_encrypt, 3662 .decrypt = aead_givdecrypt, 3663 .ivsize = DES_BLOCK_SIZE, 3664 .maxauthsize = MD5_DIGEST_SIZE, 3665 }, 3666 .caam = { 3667 .class1_alg_type = OP_ALG_ALGSEL_DES | OP_ALG_AAI_CBC, 3668 .class2_alg_type = OP_ALG_ALGSEL_MD5 | 3669 OP_ALG_AAI_HMAC_PRECOMP, 3670 .alg_op = OP_ALG_ALGSEL_MD5 | OP_ALG_AAI_HMAC, 3671 .geniv = true, 3672 }, 3673 }, 3674 { 3675 .aead = { 3676 .base = { 3677 .cra_name = "authenc(hmac(sha1),cbc(des))", 3678 .cra_driver_name = "authenc-hmac-sha1-" 3679 "cbc-des-caam", 3680 .cra_blocksize = DES_BLOCK_SIZE, 3681 }, 3682 .setkey = aead_setkey, 3683 .setauthsize = aead_setauthsize, 3684 .encrypt = aead_encrypt, 3685 .decrypt = aead_decrypt, 3686 .ivsize = DES_BLOCK_SIZE, 3687 .maxauthsize = SHA1_DIGEST_SIZE, 3688 }, 3689 .caam = { 3690 .class1_alg_type = OP_ALG_ALGSEL_DES | OP_ALG_AAI_CBC, 3691 .class2_alg_type = OP_ALG_ALGSEL_SHA1 | 3692 OP_ALG_AAI_HMAC_PRECOMP, 3693 .alg_op = OP_ALG_ALGSEL_SHA1 | OP_ALG_AAI_HMAC, 3694 }, 3695 }, 3696 { 3697 .aead = { 3698 .base = { 3699 .cra_name = "echainiv(authenc(hmac(sha1)," 3700 "cbc(des)))", 3701 .cra_driver_name = "echainiv-authenc-" 3702 "hmac-sha1-cbc-des-caam", 3703 .cra_blocksize = DES_BLOCK_SIZE, 3704 }, 3705 .setkey = aead_setkey, 3706 .setauthsize = aead_setauthsize, 3707 .encrypt = aead_encrypt, 3708 .decrypt = aead_givdecrypt, 3709 .ivsize = DES_BLOCK_SIZE, 3710 .maxauthsize = SHA1_DIGEST_SIZE, 3711 }, 3712 .caam = { 3713 .class1_alg_type = OP_ALG_ALGSEL_DES | OP_ALG_AAI_CBC, 3714 .class2_alg_type = OP_ALG_ALGSEL_SHA1 | 3715 OP_ALG_AAI_HMAC_PRECOMP, 3716 .alg_op = OP_ALG_ALGSEL_SHA1 | OP_ALG_AAI_HMAC, 3717 .geniv = true, 3718 }, 3719 }, 3720 { 3721 .aead = { 3722 .base = { 3723 .cra_name = "authenc(hmac(sha224),cbc(des))", 3724 .cra_driver_name = "authenc-hmac-sha224-" 3725 "cbc-des-caam", 3726 .cra_blocksize = DES_BLOCK_SIZE, 3727 }, 3728 .setkey = aead_setkey, 3729 .setauthsize = aead_setauthsize, 3730 .encrypt = aead_encrypt, 3731 .decrypt = aead_decrypt, 3732 .ivsize = DES_BLOCK_SIZE, 3733 .maxauthsize = SHA224_DIGEST_SIZE, 3734 }, 3735 .caam = { 3736 .class1_alg_type = OP_ALG_ALGSEL_DES | OP_ALG_AAI_CBC, 3737 .class2_alg_type = OP_ALG_ALGSEL_SHA224 | 3738 OP_ALG_AAI_HMAC_PRECOMP, 3739 .alg_op = OP_ALG_ALGSEL_SHA224 | OP_ALG_AAI_HMAC, 3740 }, 3741 }, 3742 { 3743 .aead = { 3744 .base = { 3745 .cra_name = "echainiv(authenc(hmac(sha224)," 3746 "cbc(des)))", 3747 .cra_driver_name = "echainiv-authenc-" 3748 "hmac-sha224-cbc-des-caam", 3749 .cra_blocksize = DES_BLOCK_SIZE, 3750 }, 3751 .setkey = aead_setkey, 3752 .setauthsize = aead_setauthsize, 3753 .encrypt = aead_encrypt, 3754 .decrypt = aead_givdecrypt, 3755 .ivsize = DES_BLOCK_SIZE, 3756 .maxauthsize = SHA224_DIGEST_SIZE, 3757 }, 3758 .caam = { 3759 .class1_alg_type = OP_ALG_ALGSEL_DES | OP_ALG_AAI_CBC, 3760 .class2_alg_type = OP_ALG_ALGSEL_SHA224 | 3761 OP_ALG_AAI_HMAC_PRECOMP, 3762 .alg_op = OP_ALG_ALGSEL_SHA224 | OP_ALG_AAI_HMAC, 3763 .geniv = true, 3764 }, 3765 }, 3766 { 3767 .aead = { 3768 .base = { 3769 .cra_name = "authenc(hmac(sha256),cbc(des))", 3770 .cra_driver_name = "authenc-hmac-sha256-" 3771 "cbc-des-caam", 3772 .cra_blocksize = DES_BLOCK_SIZE, 3773 }, 3774 .setkey = aead_setkey, 3775 .setauthsize = aead_setauthsize, 3776 .encrypt = aead_encrypt, 3777 .decrypt = aead_decrypt, 3778 .ivsize = DES_BLOCK_SIZE, 3779 .maxauthsize = SHA256_DIGEST_SIZE, 3780 }, 3781 .caam = { 3782 .class1_alg_type = OP_ALG_ALGSEL_DES | OP_ALG_AAI_CBC, 3783 .class2_alg_type = OP_ALG_ALGSEL_SHA256 | 3784 OP_ALG_AAI_HMAC_PRECOMP, 3785 .alg_op = OP_ALG_ALGSEL_SHA256 | OP_ALG_AAI_HMAC, 3786 }, 3787 }, 3788 { 3789 .aead = { 3790 .base = { 3791 .cra_name = "echainiv(authenc(hmac(sha256)," 3792 "cbc(des)))", 3793 .cra_driver_name = "echainiv-authenc-" 3794 "hmac-sha256-cbc-des-caam", 3795 .cra_blocksize = DES_BLOCK_SIZE, 3796 }, 3797 .setkey = aead_setkey, 3798 .setauthsize = aead_setauthsize, 3799 .encrypt = aead_encrypt, 3800 .decrypt = aead_givdecrypt, 3801 .ivsize = DES_BLOCK_SIZE, 3802 .maxauthsize = SHA256_DIGEST_SIZE, 3803 }, 3804 .caam = { 3805 .class1_alg_type = OP_ALG_ALGSEL_DES | OP_ALG_AAI_CBC, 3806 .class2_alg_type = OP_ALG_ALGSEL_SHA256 | 3807 OP_ALG_AAI_HMAC_PRECOMP, 3808 .alg_op = OP_ALG_ALGSEL_SHA256 | OP_ALG_AAI_HMAC, 3809 .geniv = true, 3810 }, 3811 }, 3812 { 3813 .aead = { 3814 .base = { 3815 .cra_name = "authenc(hmac(sha384),cbc(des))", 3816 .cra_driver_name = "authenc-hmac-sha384-" 3817 "cbc-des-caam", 3818 .cra_blocksize = DES_BLOCK_SIZE, 3819 }, 3820 .setkey = aead_setkey, 3821 .setauthsize = aead_setauthsize, 3822 .encrypt = aead_encrypt, 3823 .decrypt = aead_decrypt, 3824 .ivsize = DES_BLOCK_SIZE, 3825 .maxauthsize = SHA384_DIGEST_SIZE, 3826 }, 3827 .caam = { 3828 .class1_alg_type = OP_ALG_ALGSEL_DES | OP_ALG_AAI_CBC, 3829 .class2_alg_type = OP_ALG_ALGSEL_SHA384 | 3830 OP_ALG_AAI_HMAC_PRECOMP, 3831 .alg_op = OP_ALG_ALGSEL_SHA384 | OP_ALG_AAI_HMAC, 3832 }, 3833 }, 3834 { 3835 .aead = { 3836 .base = { 3837 .cra_name = "echainiv(authenc(hmac(sha384)," 3838 "cbc(des)))", 3839 .cra_driver_name = "echainiv-authenc-" 3840 "hmac-sha384-cbc-des-caam", 3841 .cra_blocksize = DES_BLOCK_SIZE, 3842 }, 3843 .setkey = aead_setkey, 3844 .setauthsize = aead_setauthsize, 3845 .encrypt = aead_encrypt, 3846 .decrypt = aead_givdecrypt, 3847 .ivsize = DES_BLOCK_SIZE, 3848 .maxauthsize = SHA384_DIGEST_SIZE, 3849 }, 3850 .caam = { 3851 .class1_alg_type = OP_ALG_ALGSEL_DES | OP_ALG_AAI_CBC, 3852 .class2_alg_type = OP_ALG_ALGSEL_SHA384 | 3853 OP_ALG_AAI_HMAC_PRECOMP, 3854 .alg_op = OP_ALG_ALGSEL_SHA384 | OP_ALG_AAI_HMAC, 3855 .geniv = true, 3856 }, 3857 }, 3858 { 3859 .aead = { 3860 .base = { 3861 .cra_name = "authenc(hmac(sha512),cbc(des))", 3862 .cra_driver_name = "authenc-hmac-sha512-" 3863 "cbc-des-caam", 3864 .cra_blocksize = DES_BLOCK_SIZE, 3865 }, 3866 .setkey = aead_setkey, 3867 .setauthsize = aead_setauthsize, 3868 .encrypt = aead_encrypt, 3869 .decrypt = aead_decrypt, 3870 .ivsize = DES_BLOCK_SIZE, 3871 .maxauthsize = SHA512_DIGEST_SIZE, 3872 }, 3873 .caam = { 3874 .class1_alg_type = OP_ALG_ALGSEL_DES | OP_ALG_AAI_CBC, 3875 .class2_alg_type = OP_ALG_ALGSEL_SHA512 | 3876 OP_ALG_AAI_HMAC_PRECOMP, 3877 .alg_op = OP_ALG_ALGSEL_SHA512 | OP_ALG_AAI_HMAC, 3878 }, 3879 }, 3880 { 3881 .aead = { 3882 .base = { 3883 .cra_name = "echainiv(authenc(hmac(sha512)," 3884 "cbc(des)))", 3885 .cra_driver_name = "echainiv-authenc-" 3886 "hmac-sha512-cbc-des-caam", 3887 .cra_blocksize = DES_BLOCK_SIZE, 3888 }, 3889 .setkey = aead_setkey, 3890 .setauthsize = aead_setauthsize, 3891 .encrypt = aead_encrypt, 3892 .decrypt = aead_givdecrypt, 3893 .ivsize = DES_BLOCK_SIZE, 3894 .maxauthsize = SHA512_DIGEST_SIZE, 3895 }, 3896 .caam = { 3897 .class1_alg_type = OP_ALG_ALGSEL_DES | OP_ALG_AAI_CBC, 3898 .class2_alg_type = OP_ALG_ALGSEL_SHA512 | 3899 OP_ALG_AAI_HMAC_PRECOMP, 3900 .alg_op = OP_ALG_ALGSEL_SHA512 | OP_ALG_AAI_HMAC, 3901 .geniv = true, 3902 }, 3903 }, 3904 { 3905 .aead = { 3906 .base = { 3907 .cra_name = "authenc(hmac(md5)," 3908 "rfc3686(ctr(aes)))", 3909 .cra_driver_name = "authenc-hmac-md5-" 3910 "rfc3686-ctr-aes-caam", 3911 .cra_blocksize = 1, 3912 }, 3913 .setkey = aead_setkey, 3914 .setauthsize = aead_setauthsize, 3915 .encrypt = aead_encrypt, 3916 .decrypt = aead_decrypt, 3917 .ivsize = CTR_RFC3686_IV_SIZE, 3918 .maxauthsize = MD5_DIGEST_SIZE, 3919 }, 3920 .caam = { 3921 .class1_alg_type = OP_ALG_ALGSEL_AES | 3922 OP_ALG_AAI_CTR_MOD128, 3923 .class2_alg_type = OP_ALG_ALGSEL_MD5 | 3924 OP_ALG_AAI_HMAC_PRECOMP, 3925 .alg_op = OP_ALG_ALGSEL_MD5 | OP_ALG_AAI_HMAC, 3926 .rfc3686 = true, 3927 }, 3928 }, 3929 { 3930 .aead = { 3931 .base = { 3932 .cra_name = "seqiv(authenc(" 3933 "hmac(md5),rfc3686(ctr(aes))))", 3934 .cra_driver_name = "seqiv-authenc-hmac-md5-" 3935 "rfc3686-ctr-aes-caam", 3936 .cra_blocksize = 1, 3937 }, 3938 .setkey = aead_setkey, 3939 .setauthsize = aead_setauthsize, 3940 .encrypt = aead_encrypt, 3941 .decrypt = aead_givdecrypt, 3942 .ivsize = CTR_RFC3686_IV_SIZE, 3943 .maxauthsize = MD5_DIGEST_SIZE, 3944 }, 3945 .caam = { 3946 .class1_alg_type = OP_ALG_ALGSEL_AES | 3947 OP_ALG_AAI_CTR_MOD128, 3948 .class2_alg_type = OP_ALG_ALGSEL_MD5 | 3949 OP_ALG_AAI_HMAC_PRECOMP, 3950 .alg_op = OP_ALG_ALGSEL_MD5 | OP_ALG_AAI_HMAC, 3951 .rfc3686 = true, 3952 .geniv = true, 3953 }, 3954 }, 3955 { 3956 .aead = { 3957 .base = { 3958 .cra_name = "authenc(hmac(sha1)," 3959 "rfc3686(ctr(aes)))", 3960 .cra_driver_name = "authenc-hmac-sha1-" 3961 "rfc3686-ctr-aes-caam", 3962 .cra_blocksize = 1, 3963 }, 3964 .setkey = aead_setkey, 3965 .setauthsize = aead_setauthsize, 3966 .encrypt = aead_encrypt, 3967 .decrypt = aead_decrypt, 3968 .ivsize = CTR_RFC3686_IV_SIZE, 3969 .maxauthsize = SHA1_DIGEST_SIZE, 3970 }, 3971 .caam = { 3972 .class1_alg_type = OP_ALG_ALGSEL_AES | 3973 OP_ALG_AAI_CTR_MOD128, 3974 .class2_alg_type = OP_ALG_ALGSEL_SHA1 | 3975 OP_ALG_AAI_HMAC_PRECOMP, 3976 .alg_op = OP_ALG_ALGSEL_SHA1 | OP_ALG_AAI_HMAC, 3977 .rfc3686 = true, 3978 }, 3979 }, 3980 { 3981 .aead = { 3982 .base = { 3983 .cra_name = "seqiv(authenc(" 3984 "hmac(sha1),rfc3686(ctr(aes))))", 3985 .cra_driver_name = "seqiv-authenc-hmac-sha1-" 3986 "rfc3686-ctr-aes-caam", 3987 .cra_blocksize = 1, 3988 }, 3989 .setkey = aead_setkey, 3990 .setauthsize = aead_setauthsize, 3991 .encrypt = aead_encrypt, 3992 .decrypt = aead_givdecrypt, 3993 .ivsize = CTR_RFC3686_IV_SIZE, 3994 .maxauthsize = SHA1_DIGEST_SIZE, 3995 }, 3996 .caam = { 3997 .class1_alg_type = OP_ALG_ALGSEL_AES | 3998 OP_ALG_AAI_CTR_MOD128, 3999 .class2_alg_type = OP_ALG_ALGSEL_SHA1 | 4000 OP_ALG_AAI_HMAC_PRECOMP, 4001 .alg_op = OP_ALG_ALGSEL_SHA1 | OP_ALG_AAI_HMAC, 4002 .rfc3686 = true, 4003 .geniv = true, 4004 }, 4005 }, 4006 { 4007 .aead = { 4008 .base = { 4009 .cra_name = "authenc(hmac(sha224)," 4010 "rfc3686(ctr(aes)))", 4011 .cra_driver_name = "authenc-hmac-sha224-" 4012 "rfc3686-ctr-aes-caam", 4013 .cra_blocksize = 1, 4014 }, 4015 .setkey = aead_setkey, 4016 .setauthsize = aead_setauthsize, 4017 .encrypt = aead_encrypt, 4018 .decrypt = aead_decrypt, 4019 .ivsize = CTR_RFC3686_IV_SIZE, 4020 .maxauthsize = SHA224_DIGEST_SIZE, 4021 }, 4022 .caam = { 4023 .class1_alg_type = OP_ALG_ALGSEL_AES | 4024 OP_ALG_AAI_CTR_MOD128, 4025 .class2_alg_type = OP_ALG_ALGSEL_SHA224 | 4026 OP_ALG_AAI_HMAC_PRECOMP, 4027 .alg_op = OP_ALG_ALGSEL_SHA224 | OP_ALG_AAI_HMAC, 4028 .rfc3686 = true, 4029 }, 4030 }, 4031 { 4032 .aead = { 4033 .base = { 4034 .cra_name = "seqiv(authenc(" 4035 "hmac(sha224),rfc3686(ctr(aes))))", 4036 .cra_driver_name = "seqiv-authenc-hmac-sha224-" 4037 "rfc3686-ctr-aes-caam", 4038 .cra_blocksize = 1, 4039 }, 4040 .setkey = aead_setkey, 4041 .setauthsize = aead_setauthsize, 4042 .encrypt = aead_encrypt, 4043 .decrypt = aead_givdecrypt, 4044 .ivsize = CTR_RFC3686_IV_SIZE, 4045 .maxauthsize = SHA224_DIGEST_SIZE, 4046 }, 4047 .caam = { 4048 .class1_alg_type = OP_ALG_ALGSEL_AES | 4049 OP_ALG_AAI_CTR_MOD128, 4050 .class2_alg_type = OP_ALG_ALGSEL_SHA224 | 4051 OP_ALG_AAI_HMAC_PRECOMP, 4052 .alg_op = OP_ALG_ALGSEL_SHA224 | OP_ALG_AAI_HMAC, 4053 .rfc3686 = true, 4054 .geniv = true, 4055 }, 4056 }, 4057 { 4058 .aead = { 4059 .base = { 4060 .cra_name = "authenc(hmac(sha256)," 4061 "rfc3686(ctr(aes)))", 4062 .cra_driver_name = "authenc-hmac-sha256-" 4063 "rfc3686-ctr-aes-caam", 4064 .cra_blocksize = 1, 4065 }, 4066 .setkey = aead_setkey, 4067 .setauthsize = aead_setauthsize, 4068 .encrypt = aead_encrypt, 4069 .decrypt = aead_decrypt, 4070 .ivsize = CTR_RFC3686_IV_SIZE, 4071 .maxauthsize = SHA256_DIGEST_SIZE, 4072 }, 4073 .caam = { 4074 .class1_alg_type = OP_ALG_ALGSEL_AES | 4075 OP_ALG_AAI_CTR_MOD128, 4076 .class2_alg_type = OP_ALG_ALGSEL_SHA256 | 4077 OP_ALG_AAI_HMAC_PRECOMP, 4078 .alg_op = OP_ALG_ALGSEL_SHA256 | OP_ALG_AAI_HMAC, 4079 .rfc3686 = true, 4080 }, 4081 }, 4082 { 4083 .aead = { 4084 .base = { 4085 .cra_name = "seqiv(authenc(hmac(sha256)," 4086 "rfc3686(ctr(aes))))", 4087 .cra_driver_name = "seqiv-authenc-hmac-sha256-" 4088 "rfc3686-ctr-aes-caam", 4089 .cra_blocksize = 1, 4090 }, 4091 .setkey = aead_setkey, 4092 .setauthsize = aead_setauthsize, 4093 .encrypt = aead_encrypt, 4094 .decrypt = aead_givdecrypt, 4095 .ivsize = CTR_RFC3686_IV_SIZE, 4096 .maxauthsize = SHA256_DIGEST_SIZE, 4097 }, 4098 .caam = { 4099 .class1_alg_type = OP_ALG_ALGSEL_AES | 4100 OP_ALG_AAI_CTR_MOD128, 4101 .class2_alg_type = OP_ALG_ALGSEL_SHA256 | 4102 OP_ALG_AAI_HMAC_PRECOMP, 4103 .alg_op = OP_ALG_ALGSEL_SHA256 | OP_ALG_AAI_HMAC, 4104 .rfc3686 = true, 4105 .geniv = true, 4106 }, 4107 }, 4108 { 4109 .aead = { 4110 .base = { 4111 .cra_name = "authenc(hmac(sha384)," 4112 "rfc3686(ctr(aes)))", 4113 .cra_driver_name = "authenc-hmac-sha384-" 4114 "rfc3686-ctr-aes-caam", 4115 .cra_blocksize = 1, 4116 }, 4117 .setkey = aead_setkey, 4118 .setauthsize = aead_setauthsize, 4119 .encrypt = aead_encrypt, 4120 .decrypt = aead_decrypt, 4121 .ivsize = CTR_RFC3686_IV_SIZE, 4122 .maxauthsize = SHA384_DIGEST_SIZE, 4123 }, 4124 .caam = { 4125 .class1_alg_type = OP_ALG_ALGSEL_AES | 4126 OP_ALG_AAI_CTR_MOD128, 4127 .class2_alg_type = OP_ALG_ALGSEL_SHA384 | 4128 OP_ALG_AAI_HMAC_PRECOMP, 4129 .alg_op = OP_ALG_ALGSEL_SHA384 | OP_ALG_AAI_HMAC, 4130 .rfc3686 = true, 4131 }, 4132 }, 4133 { 4134 .aead = { 4135 .base = { 4136 .cra_name = "seqiv(authenc(hmac(sha384)," 4137 "rfc3686(ctr(aes))))", 4138 .cra_driver_name = "seqiv-authenc-hmac-sha384-" 4139 "rfc3686-ctr-aes-caam", 4140 .cra_blocksize = 1, 4141 }, 4142 .setkey = aead_setkey, 4143 .setauthsize = aead_setauthsize, 4144 .encrypt = aead_encrypt, 4145 .decrypt = aead_givdecrypt, 4146 .ivsize = CTR_RFC3686_IV_SIZE, 4147 .maxauthsize = SHA384_DIGEST_SIZE, 4148 }, 4149 .caam = { 4150 .class1_alg_type = OP_ALG_ALGSEL_AES | 4151 OP_ALG_AAI_CTR_MOD128, 4152 .class2_alg_type = OP_ALG_ALGSEL_SHA384 | 4153 OP_ALG_AAI_HMAC_PRECOMP, 4154 .alg_op = OP_ALG_ALGSEL_SHA384 | OP_ALG_AAI_HMAC, 4155 .rfc3686 = true, 4156 .geniv = true, 4157 }, 4158 }, 4159 { 4160 .aead = { 4161 .base = { 4162 .cra_name = "authenc(hmac(sha512)," 4163 "rfc3686(ctr(aes)))", 4164 .cra_driver_name = "authenc-hmac-sha512-" 4165 "rfc3686-ctr-aes-caam", 4166 .cra_blocksize = 1, 4167 }, 4168 .setkey = aead_setkey, 4169 .setauthsize = aead_setauthsize, 4170 .encrypt = aead_encrypt, 4171 .decrypt = aead_decrypt, 4172 .ivsize = CTR_RFC3686_IV_SIZE, 4173 .maxauthsize = SHA512_DIGEST_SIZE, 4174 }, 4175 .caam = { 4176 .class1_alg_type = OP_ALG_ALGSEL_AES | 4177 OP_ALG_AAI_CTR_MOD128, 4178 .class2_alg_type = OP_ALG_ALGSEL_SHA512 | 4179 OP_ALG_AAI_HMAC_PRECOMP, 4180 .alg_op = OP_ALG_ALGSEL_SHA512 | OP_ALG_AAI_HMAC, 4181 .rfc3686 = true, 4182 }, 4183 }, 4184 { 4185 .aead = { 4186 .base = { 4187 .cra_name = "seqiv(authenc(hmac(sha512)," 4188 "rfc3686(ctr(aes))))", 4189 .cra_driver_name = "seqiv-authenc-hmac-sha512-" 4190 "rfc3686-ctr-aes-caam", 4191 .cra_blocksize = 1, 4192 }, 4193 .setkey = aead_setkey, 4194 .setauthsize = aead_setauthsize, 4195 .encrypt = aead_encrypt, 4196 .decrypt = aead_givdecrypt, 4197 .ivsize = CTR_RFC3686_IV_SIZE, 4198 .maxauthsize = SHA512_DIGEST_SIZE, 4199 }, 4200 .caam = { 4201 .class1_alg_type = OP_ALG_ALGSEL_AES | 4202 OP_ALG_AAI_CTR_MOD128, 4203 .class2_alg_type = OP_ALG_ALGSEL_SHA512 | 4204 OP_ALG_AAI_HMAC_PRECOMP, 4205 .alg_op = OP_ALG_ALGSEL_SHA512 | OP_ALG_AAI_HMAC, 4206 .rfc3686 = true, 4207 .geniv = true, 4208 }, 4209 }, 4210 }; 4211 4212 struct caam_crypto_alg { 4213 struct crypto_alg crypto_alg; 4214 struct list_head entry; 4215 struct caam_alg_entry caam; 4216 }; 4217 4218 static int caam_init_common(struct caam_ctx *ctx, struct caam_alg_entry *caam) 4219 { 4220 ctx->jrdev = caam_jr_alloc(); 4221 if (IS_ERR(ctx->jrdev)) { 4222 pr_err("Job Ring Device allocation for transform failed\n"); 4223 return PTR_ERR(ctx->jrdev); 4224 } 4225 4226 /* copy descriptor header template value */ 4227 ctx->class1_alg_type = OP_TYPE_CLASS1_ALG | caam->class1_alg_type; 4228 ctx->class2_alg_type = OP_TYPE_CLASS2_ALG | caam->class2_alg_type; 4229 ctx->alg_op = OP_TYPE_CLASS2_ALG | caam->alg_op; 4230 4231 return 0; 4232 } 4233 4234 static int caam_cra_init(struct crypto_tfm *tfm) 4235 { 4236 struct crypto_alg *alg = tfm->__crt_alg; 4237 struct caam_crypto_alg *caam_alg = 4238 container_of(alg, struct caam_crypto_alg, crypto_alg); 4239 struct caam_ctx *ctx = crypto_tfm_ctx(tfm); 4240 4241 return caam_init_common(ctx, &caam_alg->caam); 4242 } 4243 4244 static int caam_aead_init(struct crypto_aead *tfm) 4245 { 4246 struct aead_alg *alg = crypto_aead_alg(tfm); 4247 struct caam_aead_alg *caam_alg = 4248 container_of(alg, struct caam_aead_alg, aead); 4249 struct caam_ctx *ctx = crypto_aead_ctx(tfm); 4250 4251 return caam_init_common(ctx, &caam_alg->caam); 4252 } 4253 4254 static void caam_exit_common(struct caam_ctx *ctx) 4255 { 4256 if (ctx->sh_desc_enc_dma && 4257 !dma_mapping_error(ctx->jrdev, ctx->sh_desc_enc_dma)) 4258 dma_unmap_single(ctx->jrdev, ctx->sh_desc_enc_dma, 4259 desc_bytes(ctx->sh_desc_enc), DMA_TO_DEVICE); 4260 if (ctx->sh_desc_dec_dma && 4261 !dma_mapping_error(ctx->jrdev, ctx->sh_desc_dec_dma)) 4262 dma_unmap_single(ctx->jrdev, ctx->sh_desc_dec_dma, 4263 desc_bytes(ctx->sh_desc_dec), DMA_TO_DEVICE); 4264 if (ctx->sh_desc_givenc_dma && 4265 !dma_mapping_error(ctx->jrdev, ctx->sh_desc_givenc_dma)) 4266 dma_unmap_single(ctx->jrdev, ctx->sh_desc_givenc_dma, 4267 desc_bytes(ctx->sh_desc_givenc), 4268 DMA_TO_DEVICE); 4269 if (ctx->key_dma && 4270 !dma_mapping_error(ctx->jrdev, ctx->key_dma)) 4271 dma_unmap_single(ctx->jrdev, ctx->key_dma, 4272 ctx->enckeylen + ctx->split_key_pad_len, 4273 DMA_TO_DEVICE); 4274 4275 caam_jr_free(ctx->jrdev); 4276 } 4277 4278 static void caam_cra_exit(struct crypto_tfm *tfm) 4279 { 4280 caam_exit_common(crypto_tfm_ctx(tfm)); 4281 } 4282 4283 static void caam_aead_exit(struct crypto_aead *tfm) 4284 { 4285 caam_exit_common(crypto_aead_ctx(tfm)); 4286 } 4287 4288 static void __exit caam_algapi_exit(void) 4289 { 4290 4291 struct caam_crypto_alg *t_alg, *n; 4292 int i; 4293 4294 for (i = 0; i < ARRAY_SIZE(driver_aeads); i++) { 4295 struct caam_aead_alg *t_alg = driver_aeads + i; 4296 4297 if (t_alg->registered) 4298 crypto_unregister_aead(&t_alg->aead); 4299 } 4300 4301 if (!alg_list.next) 4302 return; 4303 4304 list_for_each_entry_safe(t_alg, n, &alg_list, entry) { 4305 crypto_unregister_alg(&t_alg->crypto_alg); 4306 list_del(&t_alg->entry); 4307 kfree(t_alg); 4308 } 4309 } 4310 4311 static struct caam_crypto_alg *caam_alg_alloc(struct caam_alg_template 4312 *template) 4313 { 4314 struct caam_crypto_alg *t_alg; 4315 struct crypto_alg *alg; 4316 4317 t_alg = kzalloc(sizeof(*t_alg), GFP_KERNEL); 4318 if (!t_alg) { 4319 pr_err("failed to allocate t_alg\n"); 4320 return ERR_PTR(-ENOMEM); 4321 } 4322 4323 alg = &t_alg->crypto_alg; 4324 4325 snprintf(alg->cra_name, CRYPTO_MAX_ALG_NAME, "%s", template->name); 4326 snprintf(alg->cra_driver_name, CRYPTO_MAX_ALG_NAME, "%s", 4327 template->driver_name); 4328 alg->cra_module = THIS_MODULE; 4329 alg->cra_init = caam_cra_init; 4330 alg->cra_exit = caam_cra_exit; 4331 alg->cra_priority = CAAM_CRA_PRIORITY; 4332 alg->cra_blocksize = template->blocksize; 4333 alg->cra_alignmask = 0; 4334 alg->cra_ctxsize = sizeof(struct caam_ctx); 4335 alg->cra_flags = CRYPTO_ALG_ASYNC | CRYPTO_ALG_KERN_DRIVER_ONLY | 4336 template->type; 4337 switch (template->type) { 4338 case CRYPTO_ALG_TYPE_GIVCIPHER: 4339 alg->cra_type = &crypto_givcipher_type; 4340 alg->cra_ablkcipher = template->template_ablkcipher; 4341 break; 4342 case CRYPTO_ALG_TYPE_ABLKCIPHER: 4343 alg->cra_type = &crypto_ablkcipher_type; 4344 alg->cra_ablkcipher = template->template_ablkcipher; 4345 break; 4346 } 4347 4348 t_alg->caam.class1_alg_type = template->class1_alg_type; 4349 t_alg->caam.class2_alg_type = template->class2_alg_type; 4350 t_alg->caam.alg_op = template->alg_op; 4351 4352 return t_alg; 4353 } 4354 4355 static void caam_aead_alg_init(struct caam_aead_alg *t_alg) 4356 { 4357 struct aead_alg *alg = &t_alg->aead; 4358 4359 alg->base.cra_module = THIS_MODULE; 4360 alg->base.cra_priority = CAAM_CRA_PRIORITY; 4361 alg->base.cra_ctxsize = sizeof(struct caam_ctx); 4362 alg->base.cra_flags = CRYPTO_ALG_ASYNC | CRYPTO_ALG_KERN_DRIVER_ONLY; 4363 4364 alg->init = caam_aead_init; 4365 alg->exit = caam_aead_exit; 4366 } 4367 4368 static int __init caam_algapi_init(void) 4369 { 4370 struct device_node *dev_node; 4371 struct platform_device *pdev; 4372 struct device *ctrldev; 4373 struct caam_drv_private *priv; 4374 int i = 0, err = 0; 4375 u32 cha_vid, cha_inst, des_inst, aes_inst, md_inst; 4376 unsigned int md_limit = SHA512_DIGEST_SIZE; 4377 bool registered = false; 4378 4379 dev_node = of_find_compatible_node(NULL, NULL, "fsl,sec-v4.0"); 4380 if (!dev_node) { 4381 dev_node = of_find_compatible_node(NULL, NULL, "fsl,sec4.0"); 4382 if (!dev_node) 4383 return -ENODEV; 4384 } 4385 4386 pdev = of_find_device_by_node(dev_node); 4387 if (!pdev) { 4388 of_node_put(dev_node); 4389 return -ENODEV; 4390 } 4391 4392 ctrldev = &pdev->dev; 4393 priv = dev_get_drvdata(ctrldev); 4394 of_node_put(dev_node); 4395 4396 /* 4397 * If priv is NULL, it's probably because the caam driver wasn't 4398 * properly initialized (e.g. RNG4 init failed). Thus, bail out here. 4399 */ 4400 if (!priv) 4401 return -ENODEV; 4402 4403 4404 INIT_LIST_HEAD(&alg_list); 4405 4406 /* 4407 * Register crypto algorithms the device supports. 4408 * First, detect presence and attributes of DES, AES, and MD blocks. 4409 */ 4410 cha_vid = rd_reg32(&priv->ctrl->perfmon.cha_id_ls); 4411 cha_inst = rd_reg32(&priv->ctrl->perfmon.cha_num_ls); 4412 des_inst = (cha_inst & CHA_ID_LS_DES_MASK) >> CHA_ID_LS_DES_SHIFT; 4413 aes_inst = (cha_inst & CHA_ID_LS_AES_MASK) >> CHA_ID_LS_AES_SHIFT; 4414 md_inst = (cha_inst & CHA_ID_LS_MD_MASK) >> CHA_ID_LS_MD_SHIFT; 4415 4416 /* If MD is present, limit digest size based on LP256 */ 4417 if (md_inst && ((cha_vid & CHA_ID_LS_MD_MASK) == CHA_ID_LS_MD_LP256)) 4418 md_limit = SHA256_DIGEST_SIZE; 4419 4420 for (i = 0; i < ARRAY_SIZE(driver_algs); i++) { 4421 struct caam_crypto_alg *t_alg; 4422 struct caam_alg_template *alg = driver_algs + i; 4423 u32 alg_sel = alg->class1_alg_type & OP_ALG_ALGSEL_MASK; 4424 4425 /* Skip DES algorithms if not supported by device */ 4426 if (!des_inst && 4427 ((alg_sel == OP_ALG_ALGSEL_3DES) || 4428 (alg_sel == OP_ALG_ALGSEL_DES))) 4429 continue; 4430 4431 /* Skip AES algorithms if not supported by device */ 4432 if (!aes_inst && (alg_sel == OP_ALG_ALGSEL_AES)) 4433 continue; 4434 4435 t_alg = caam_alg_alloc(alg); 4436 if (IS_ERR(t_alg)) { 4437 err = PTR_ERR(t_alg); 4438 pr_warn("%s alg allocation failed\n", alg->driver_name); 4439 continue; 4440 } 4441 4442 err = crypto_register_alg(&t_alg->crypto_alg); 4443 if (err) { 4444 pr_warn("%s alg registration failed\n", 4445 t_alg->crypto_alg.cra_driver_name); 4446 kfree(t_alg); 4447 continue; 4448 } 4449 4450 list_add_tail(&t_alg->entry, &alg_list); 4451 registered = true; 4452 } 4453 4454 for (i = 0; i < ARRAY_SIZE(driver_aeads); i++) { 4455 struct caam_aead_alg *t_alg = driver_aeads + i; 4456 u32 c1_alg_sel = t_alg->caam.class1_alg_type & 4457 OP_ALG_ALGSEL_MASK; 4458 u32 c2_alg_sel = t_alg->caam.class2_alg_type & 4459 OP_ALG_ALGSEL_MASK; 4460 u32 alg_aai = t_alg->caam.class1_alg_type & OP_ALG_AAI_MASK; 4461 4462 /* Skip DES algorithms if not supported by device */ 4463 if (!des_inst && 4464 ((c1_alg_sel == OP_ALG_ALGSEL_3DES) || 4465 (c1_alg_sel == OP_ALG_ALGSEL_DES))) 4466 continue; 4467 4468 /* Skip AES algorithms if not supported by device */ 4469 if (!aes_inst && (c1_alg_sel == OP_ALG_ALGSEL_AES)) 4470 continue; 4471 4472 /* 4473 * Check support for AES algorithms not available 4474 * on LP devices. 4475 */ 4476 if ((cha_vid & CHA_ID_LS_AES_MASK) == CHA_ID_LS_AES_LP) 4477 if (alg_aai == OP_ALG_AAI_GCM) 4478 continue; 4479 4480 /* 4481 * Skip algorithms requiring message digests 4482 * if MD or MD size is not supported by device. 4483 */ 4484 if (c2_alg_sel && 4485 (!md_inst || (t_alg->aead.maxauthsize > md_limit))) 4486 continue; 4487 4488 caam_aead_alg_init(t_alg); 4489 4490 err = crypto_register_aead(&t_alg->aead); 4491 if (err) { 4492 pr_warn("%s alg registration failed\n", 4493 t_alg->aead.base.cra_driver_name); 4494 continue; 4495 } 4496 4497 t_alg->registered = true; 4498 registered = true; 4499 } 4500 4501 if (registered) 4502 pr_info("caam algorithms registered in /proc/crypto\n"); 4503 4504 return err; 4505 } 4506 4507 module_init(caam_algapi_init); 4508 module_exit(caam_algapi_exit); 4509 4510 MODULE_LICENSE("GPL"); 4511 MODULE_DESCRIPTION("FSL CAAM support for crypto API"); 4512 MODULE_AUTHOR("Freescale Semiconductor - NMG/STC"); 4513