1 /* 2 * caam - Freescale FSL CAAM support for crypto API 3 * 4 * Copyright 2008-2011 Freescale Semiconductor, Inc. 5 * 6 * Based on talitos crypto API driver. 7 * 8 * relationship of job descriptors to shared descriptors (SteveC Dec 10 2008): 9 * 10 * --------------- --------------- 11 * | JobDesc #1 |-------------------->| ShareDesc | 12 * | *(packet 1) | | (PDB) | 13 * --------------- |------------->| (hashKey) | 14 * . | | (cipherKey) | 15 * . | |-------->| (operation) | 16 * --------------- | | --------------- 17 * | JobDesc #2 |------| | 18 * | *(packet 2) | | 19 * --------------- | 20 * . | 21 * . | 22 * --------------- | 23 * | JobDesc #3 |------------ 24 * | *(packet 3) | 25 * --------------- 26 * 27 * The SharedDesc never changes for a connection unless rekeyed, but 28 * each packet will likely be in a different place. So all we need 29 * to know to process the packet is where the input is, where the 30 * output goes, and what context we want to process with. Context is 31 * in the SharedDesc, packet references in the JobDesc. 32 * 33 * So, a job desc looks like: 34 * 35 * --------------------- 36 * | Header | 37 * | ShareDesc Pointer | 38 * | SEQ_OUT_PTR | 39 * | (output buffer) | 40 * | SEQ_IN_PTR | 41 * | (input buffer) | 42 * | LOAD (to DECO) | 43 * --------------------- 44 */ 45 46 #include "compat.h" 47 48 #include "regs.h" 49 #include "intern.h" 50 #include "desc_constr.h" 51 #include "jr.h" 52 #include "error.h" 53 54 /* 55 * crypto alg 56 */ 57 #define CAAM_CRA_PRIORITY 3000 58 /* max key is sum of AES_MAX_KEY_SIZE, max split key size */ 59 #define CAAM_MAX_KEY_SIZE (AES_MAX_KEY_SIZE + \ 60 SHA512_DIGEST_SIZE * 2) 61 /* max IV is max of AES_BLOCK_SIZE, DES3_EDE_BLOCK_SIZE */ 62 #define CAAM_MAX_IV_LENGTH 16 63 64 /* length of descriptors text */ 65 #define DESC_JOB_IO_LEN (CAAM_CMD_SZ * 3 + CAAM_PTR_SZ * 3) 66 67 #define DESC_AEAD_BASE (4 * CAAM_CMD_SZ) 68 #define DESC_AEAD_ENC_LEN (DESC_AEAD_BASE + 16 * CAAM_CMD_SZ) 69 #define DESC_AEAD_DEC_LEN (DESC_AEAD_BASE + 21 * CAAM_CMD_SZ) 70 #define DESC_AEAD_GIVENC_LEN (DESC_AEAD_ENC_LEN + 7 * CAAM_CMD_SZ) 71 72 #define DESC_ABLKCIPHER_BASE (3 * CAAM_CMD_SZ) 73 #define DESC_ABLKCIPHER_ENC_LEN (DESC_ABLKCIPHER_BASE + \ 74 20 * CAAM_CMD_SZ) 75 #define DESC_ABLKCIPHER_DEC_LEN (DESC_ABLKCIPHER_BASE + \ 76 15 * CAAM_CMD_SZ) 77 78 #define DESC_MAX_USED_BYTES (DESC_AEAD_GIVENC_LEN + \ 79 CAAM_MAX_KEY_SIZE) 80 #define DESC_MAX_USED_LEN (DESC_MAX_USED_BYTES / CAAM_CMD_SZ) 81 82 #ifdef DEBUG 83 /* for print_hex_dumps with line references */ 84 #define xstr(s) str(s) 85 #define str(s) #s 86 #define debug(format, arg...) printk(format, arg) 87 #else 88 #define debug(format, arg...) 89 #endif 90 91 /* Set DK bit in class 1 operation if shared */ 92 static inline void append_dec_op1(u32 *desc, u32 type) 93 { 94 u32 *jump_cmd, *uncond_jump_cmd; 95 96 jump_cmd = append_jump(desc, JUMP_TEST_ALL | JUMP_COND_SHRD); 97 append_operation(desc, type | OP_ALG_AS_INITFINAL | 98 OP_ALG_DECRYPT); 99 uncond_jump_cmd = append_jump(desc, JUMP_TEST_ALL); 100 set_jump_tgt_here(desc, jump_cmd); 101 append_operation(desc, type | OP_ALG_AS_INITFINAL | 102 OP_ALG_DECRYPT | OP_ALG_AAI_DK); 103 set_jump_tgt_here(desc, uncond_jump_cmd); 104 } 105 106 /* 107 * Wait for completion of class 1 key loading before allowing 108 * error propagation 109 */ 110 static inline void append_dec_shr_done(u32 *desc) 111 { 112 u32 *jump_cmd; 113 114 jump_cmd = append_jump(desc, JUMP_CLASS_CLASS1 | JUMP_TEST_ALL); 115 set_jump_tgt_here(desc, jump_cmd); 116 append_cmd(desc, SET_OK_NO_PROP_ERRORS | CMD_LOAD); 117 } 118 119 /* 120 * For aead functions, read payload and write payload, 121 * both of which are specified in req->src and req->dst 122 */ 123 static inline void aead_append_src_dst(u32 *desc, u32 msg_type) 124 { 125 append_seq_fifo_load(desc, 0, FIFOLD_CLASS_BOTH | 126 KEY_VLF | msg_type | FIFOLD_TYPE_LASTBOTH); 127 append_seq_fifo_store(desc, 0, FIFOST_TYPE_MESSAGE_DATA | KEY_VLF); 128 } 129 130 /* 131 * For aead encrypt and decrypt, read iv for both classes 132 */ 133 static inline void aead_append_ld_iv(u32 *desc, int ivsize) 134 { 135 append_cmd(desc, CMD_SEQ_LOAD | LDST_SRCDST_BYTE_CONTEXT | 136 LDST_CLASS_1_CCB | ivsize); 137 append_move(desc, MOVE_SRC_CLASS1CTX | MOVE_DEST_CLASS2INFIFO | ivsize); 138 } 139 140 /* 141 * For ablkcipher encrypt and decrypt, read from req->src and 142 * write to req->dst 143 */ 144 static inline void ablkcipher_append_src_dst(u32 *desc) 145 { 146 append_math_add(desc, VARSEQOUTLEN, SEQINLEN, REG0, CAAM_CMD_SZ); \ 147 append_math_add(desc, VARSEQINLEN, SEQINLEN, REG0, CAAM_CMD_SZ); \ 148 append_seq_fifo_load(desc, 0, FIFOLD_CLASS_CLASS1 | \ 149 KEY_VLF | FIFOLD_TYPE_MSG | FIFOLD_TYPE_LAST1); \ 150 append_seq_fifo_store(desc, 0, FIFOST_TYPE_MESSAGE_DATA | KEY_VLF); \ 151 } 152 153 /* 154 * If all data, including src (with assoc and iv) or dst (with iv only) are 155 * contiguous 156 */ 157 #define GIV_SRC_CONTIG 1 158 #define GIV_DST_CONTIG (1 << 1) 159 160 /* 161 * per-session context 162 */ 163 struct caam_ctx { 164 struct device *jrdev; 165 u32 sh_desc_enc[DESC_MAX_USED_LEN]; 166 u32 sh_desc_dec[DESC_MAX_USED_LEN]; 167 u32 sh_desc_givenc[DESC_MAX_USED_LEN]; 168 dma_addr_t sh_desc_enc_dma; 169 dma_addr_t sh_desc_dec_dma; 170 dma_addr_t sh_desc_givenc_dma; 171 u32 class1_alg_type; 172 u32 class2_alg_type; 173 u32 alg_op; 174 u8 key[CAAM_MAX_KEY_SIZE]; 175 dma_addr_t key_dma; 176 unsigned int enckeylen; 177 unsigned int split_key_len; 178 unsigned int split_key_pad_len; 179 unsigned int authsize; 180 }; 181 182 static void append_key_aead(u32 *desc, struct caam_ctx *ctx, 183 int keys_fit_inline) 184 { 185 if (keys_fit_inline) { 186 append_key_as_imm(desc, ctx->key, ctx->split_key_pad_len, 187 ctx->split_key_len, CLASS_2 | 188 KEY_DEST_MDHA_SPLIT | KEY_ENC); 189 append_key_as_imm(desc, (void *)ctx->key + 190 ctx->split_key_pad_len, ctx->enckeylen, 191 ctx->enckeylen, CLASS_1 | KEY_DEST_CLASS_REG); 192 } else { 193 append_key(desc, ctx->key_dma, ctx->split_key_len, CLASS_2 | 194 KEY_DEST_MDHA_SPLIT | KEY_ENC); 195 append_key(desc, ctx->key_dma + ctx->split_key_pad_len, 196 ctx->enckeylen, CLASS_1 | KEY_DEST_CLASS_REG); 197 } 198 } 199 200 static void init_sh_desc_key_aead(u32 *desc, struct caam_ctx *ctx, 201 int keys_fit_inline) 202 { 203 u32 *key_jump_cmd; 204 205 init_sh_desc(desc, HDR_SHARE_WAIT); 206 207 /* Skip if already shared */ 208 key_jump_cmd = append_jump(desc, JUMP_JSL | JUMP_TEST_ALL | 209 JUMP_COND_SHRD); 210 211 append_key_aead(desc, ctx, keys_fit_inline); 212 213 set_jump_tgt_here(desc, key_jump_cmd); 214 215 /* Propagate errors from shared to job descriptor */ 216 append_cmd(desc, SET_OK_NO_PROP_ERRORS | CMD_LOAD); 217 } 218 219 static int aead_set_sh_desc(struct crypto_aead *aead) 220 { 221 struct aead_tfm *tfm = &aead->base.crt_aead; 222 struct caam_ctx *ctx = crypto_aead_ctx(aead); 223 struct device *jrdev = ctx->jrdev; 224 bool keys_fit_inline = 0; 225 u32 *key_jump_cmd, *jump_cmd; 226 u32 geniv, moveiv; 227 u32 *desc; 228 229 if (!ctx->enckeylen || !ctx->authsize) 230 return 0; 231 232 /* 233 * Job Descriptor and Shared Descriptors 234 * must all fit into the 64-word Descriptor h/w Buffer 235 */ 236 if (DESC_AEAD_ENC_LEN + DESC_JOB_IO_LEN + 237 ctx->split_key_pad_len + ctx->enckeylen <= 238 CAAM_DESC_BYTES_MAX) 239 keys_fit_inline = 1; 240 241 /* aead_encrypt shared descriptor */ 242 desc = ctx->sh_desc_enc; 243 244 init_sh_desc_key_aead(desc, ctx, keys_fit_inline); 245 246 /* Class 2 operation */ 247 append_operation(desc, ctx->class2_alg_type | 248 OP_ALG_AS_INITFINAL | OP_ALG_ENCRYPT); 249 250 /* cryptlen = seqoutlen - authsize */ 251 append_math_sub_imm_u32(desc, REG3, SEQOUTLEN, IMM, ctx->authsize); 252 253 /* assoclen + cryptlen = seqinlen - ivsize */ 254 append_math_sub_imm_u32(desc, REG2, SEQINLEN, IMM, tfm->ivsize); 255 256 /* assoclen + cryptlen = (assoclen + cryptlen) - cryptlen */ 257 append_math_sub(desc, VARSEQINLEN, REG2, REG3, CAAM_CMD_SZ); 258 259 /* read assoc before reading payload */ 260 append_seq_fifo_load(desc, 0, FIFOLD_CLASS_CLASS2 | FIFOLD_TYPE_MSG | 261 KEY_VLF); 262 aead_append_ld_iv(desc, tfm->ivsize); 263 264 /* Class 1 operation */ 265 append_operation(desc, ctx->class1_alg_type | 266 OP_ALG_AS_INITFINAL | OP_ALG_ENCRYPT); 267 268 /* Read and write cryptlen bytes */ 269 append_math_add(desc, VARSEQINLEN, ZERO, REG3, CAAM_CMD_SZ); 270 append_math_add(desc, VARSEQOUTLEN, ZERO, REG3, CAAM_CMD_SZ); 271 aead_append_src_dst(desc, FIFOLD_TYPE_MSG1OUT2); 272 273 /* Write ICV */ 274 append_seq_store(desc, ctx->authsize, LDST_CLASS_2_CCB | 275 LDST_SRCDST_BYTE_CONTEXT); 276 277 ctx->sh_desc_enc_dma = dma_map_single(jrdev, desc, 278 desc_bytes(desc), 279 DMA_TO_DEVICE); 280 if (dma_mapping_error(jrdev, ctx->sh_desc_enc_dma)) { 281 dev_err(jrdev, "unable to map shared descriptor\n"); 282 return -ENOMEM; 283 } 284 #ifdef DEBUG 285 print_hex_dump(KERN_ERR, "aead enc shdesc@"xstr(__LINE__)": ", 286 DUMP_PREFIX_ADDRESS, 16, 4, desc, 287 desc_bytes(desc), 1); 288 #endif 289 290 /* 291 * Job Descriptor and Shared Descriptors 292 * must all fit into the 64-word Descriptor h/w Buffer 293 */ 294 if (DESC_AEAD_DEC_LEN + DESC_JOB_IO_LEN + 295 ctx->split_key_pad_len + ctx->enckeylen <= 296 CAAM_DESC_BYTES_MAX) 297 keys_fit_inline = 1; 298 299 desc = ctx->sh_desc_dec; 300 301 /* aead_decrypt shared descriptor */ 302 init_sh_desc(desc, HDR_SHARE_WAIT); 303 304 /* Skip if already shared */ 305 key_jump_cmd = append_jump(desc, JUMP_JSL | JUMP_TEST_ALL | 306 JUMP_COND_SHRD); 307 308 append_key_aead(desc, ctx, keys_fit_inline); 309 310 /* Only propagate error immediately if shared */ 311 jump_cmd = append_jump(desc, JUMP_TEST_ALL); 312 set_jump_tgt_here(desc, key_jump_cmd); 313 append_cmd(desc, SET_OK_NO_PROP_ERRORS | CMD_LOAD); 314 set_jump_tgt_here(desc, jump_cmd); 315 316 /* Class 2 operation */ 317 append_operation(desc, ctx->class2_alg_type | 318 OP_ALG_AS_INITFINAL | OP_ALG_DECRYPT | OP_ALG_ICV_ON); 319 320 /* assoclen + cryptlen = seqinlen - ivsize */ 321 append_math_sub_imm_u32(desc, REG3, SEQINLEN, IMM, 322 ctx->authsize + tfm->ivsize) 323 /* assoclen = (assoclen + cryptlen) - cryptlen */ 324 append_math_sub(desc, REG2, SEQOUTLEN, REG0, CAAM_CMD_SZ); 325 append_math_sub(desc, VARSEQINLEN, REG3, REG2, CAAM_CMD_SZ); 326 327 /* read assoc before reading payload */ 328 append_seq_fifo_load(desc, 0, FIFOLD_CLASS_CLASS2 | FIFOLD_TYPE_MSG | 329 KEY_VLF); 330 331 aead_append_ld_iv(desc, tfm->ivsize); 332 333 append_dec_op1(desc, ctx->class1_alg_type); 334 335 /* Read and write cryptlen bytes */ 336 append_math_add(desc, VARSEQINLEN, ZERO, REG2, CAAM_CMD_SZ); 337 append_math_add(desc, VARSEQOUTLEN, ZERO, REG2, CAAM_CMD_SZ); 338 aead_append_src_dst(desc, FIFOLD_TYPE_MSG); 339 340 /* Load ICV */ 341 append_seq_fifo_load(desc, ctx->authsize, FIFOLD_CLASS_CLASS2 | 342 FIFOLD_TYPE_LAST2 | FIFOLD_TYPE_ICV); 343 append_dec_shr_done(desc); 344 345 ctx->sh_desc_dec_dma = dma_map_single(jrdev, desc, 346 desc_bytes(desc), 347 DMA_TO_DEVICE); 348 if (dma_mapping_error(jrdev, ctx->sh_desc_dec_dma)) { 349 dev_err(jrdev, "unable to map shared descriptor\n"); 350 return -ENOMEM; 351 } 352 #ifdef DEBUG 353 print_hex_dump(KERN_ERR, "aead dec shdesc@"xstr(__LINE__)": ", 354 DUMP_PREFIX_ADDRESS, 16, 4, desc, 355 desc_bytes(desc), 1); 356 #endif 357 358 /* 359 * Job Descriptor and Shared Descriptors 360 * must all fit into the 64-word Descriptor h/w Buffer 361 */ 362 if (DESC_AEAD_GIVENC_LEN + DESC_JOB_IO_LEN + 363 ctx->split_key_pad_len + ctx->enckeylen <= 364 CAAM_DESC_BYTES_MAX) 365 keys_fit_inline = 1; 366 367 /* aead_givencrypt shared descriptor */ 368 desc = ctx->sh_desc_givenc; 369 370 init_sh_desc_key_aead(desc, ctx, keys_fit_inline); 371 372 /* Generate IV */ 373 geniv = NFIFOENTRY_STYPE_PAD | NFIFOENTRY_DEST_DECO | 374 NFIFOENTRY_DTYPE_MSG | NFIFOENTRY_LC1 | 375 NFIFOENTRY_PTYPE_RND | (tfm->ivsize << NFIFOENTRY_DLEN_SHIFT); 376 append_load_imm_u32(desc, geniv, LDST_CLASS_IND_CCB | 377 LDST_SRCDST_WORD_INFO_FIFO | LDST_IMM); 378 append_cmd(desc, CMD_LOAD | DISABLE_AUTO_INFO_FIFO); 379 append_move(desc, MOVE_SRC_INFIFO | 380 MOVE_DEST_CLASS1CTX | (tfm->ivsize << MOVE_LEN_SHIFT)); 381 append_cmd(desc, CMD_LOAD | ENABLE_AUTO_INFO_FIFO); 382 383 /* Copy IV to class 1 context */ 384 append_move(desc, MOVE_SRC_CLASS1CTX | 385 MOVE_DEST_OUTFIFO | (tfm->ivsize << MOVE_LEN_SHIFT)); 386 387 /* Return to encryption */ 388 append_operation(desc, ctx->class2_alg_type | 389 OP_ALG_AS_INITFINAL | OP_ALG_ENCRYPT); 390 391 /* ivsize + cryptlen = seqoutlen - authsize */ 392 append_math_sub_imm_u32(desc, REG3, SEQOUTLEN, IMM, ctx->authsize); 393 394 /* assoclen = seqinlen - (ivsize + cryptlen) */ 395 append_math_sub(desc, VARSEQINLEN, SEQINLEN, REG3, CAAM_CMD_SZ); 396 397 /* read assoc before reading payload */ 398 append_seq_fifo_load(desc, 0, FIFOLD_CLASS_CLASS2 | FIFOLD_TYPE_MSG | 399 KEY_VLF); 400 401 /* Copy iv from class 1 ctx to class 2 fifo*/ 402 moveiv = NFIFOENTRY_STYPE_OFIFO | NFIFOENTRY_DEST_CLASS2 | 403 NFIFOENTRY_DTYPE_MSG | (tfm->ivsize << NFIFOENTRY_DLEN_SHIFT); 404 append_load_imm_u32(desc, moveiv, LDST_CLASS_IND_CCB | 405 LDST_SRCDST_WORD_INFO_FIFO | LDST_IMM); 406 append_load_imm_u32(desc, tfm->ivsize, LDST_CLASS_2_CCB | 407 LDST_SRCDST_WORD_DATASZ_REG | LDST_IMM); 408 409 /* Class 1 operation */ 410 append_operation(desc, ctx->class1_alg_type | 411 OP_ALG_AS_INITFINAL | OP_ALG_ENCRYPT); 412 413 /* Will write ivsize + cryptlen */ 414 append_math_add(desc, VARSEQOUTLEN, SEQINLEN, REG0, CAAM_CMD_SZ); 415 416 /* Not need to reload iv */ 417 append_seq_fifo_load(desc, tfm->ivsize, 418 FIFOLD_CLASS_SKIP); 419 420 /* Will read cryptlen */ 421 append_math_add(desc, VARSEQINLEN, SEQINLEN, REG0, CAAM_CMD_SZ); 422 aead_append_src_dst(desc, FIFOLD_TYPE_MSG1OUT2); 423 424 /* Write ICV */ 425 append_seq_store(desc, ctx->authsize, LDST_CLASS_2_CCB | 426 LDST_SRCDST_BYTE_CONTEXT); 427 428 ctx->sh_desc_givenc_dma = dma_map_single(jrdev, desc, 429 desc_bytes(desc), 430 DMA_TO_DEVICE); 431 if (dma_mapping_error(jrdev, ctx->sh_desc_givenc_dma)) { 432 dev_err(jrdev, "unable to map shared descriptor\n"); 433 return -ENOMEM; 434 } 435 #ifdef DEBUG 436 print_hex_dump(KERN_ERR, "aead givenc shdesc@"xstr(__LINE__)": ", 437 DUMP_PREFIX_ADDRESS, 16, 4, desc, 438 desc_bytes(desc), 1); 439 #endif 440 441 return 0; 442 } 443 444 static int aead_setauthsize(struct crypto_aead *authenc, 445 unsigned int authsize) 446 { 447 struct caam_ctx *ctx = crypto_aead_ctx(authenc); 448 449 ctx->authsize = authsize; 450 aead_set_sh_desc(authenc); 451 452 return 0; 453 } 454 455 struct split_key_result { 456 struct completion completion; 457 int err; 458 }; 459 460 static void split_key_done(struct device *dev, u32 *desc, u32 err, 461 void *context) 462 { 463 struct split_key_result *res = context; 464 465 #ifdef DEBUG 466 dev_err(dev, "%s %d: err 0x%x\n", __func__, __LINE__, err); 467 #endif 468 469 if (err) { 470 char tmp[CAAM_ERROR_STR_MAX]; 471 472 dev_err(dev, "%08x: %s\n", err, caam_jr_strstatus(tmp, err)); 473 } 474 475 res->err = err; 476 477 complete(&res->completion); 478 } 479 480 /* 481 get a split ipad/opad key 482 483 Split key generation----------------------------------------------- 484 485 [00] 0xb0810008 jobdesc: stidx=1 share=never len=8 486 [01] 0x04000014 key: class2->keyreg len=20 487 @0xffe01000 488 [03] 0x84410014 operation: cls2-op sha1 hmac init dec 489 [04] 0x24940000 fifold: class2 msgdata-last2 len=0 imm 490 [05] 0xa4000001 jump: class2 local all ->1 [06] 491 [06] 0x64260028 fifostr: class2 mdsplit-jdk len=40 492 @0xffe04000 493 */ 494 static u32 gen_split_key(struct caam_ctx *ctx, const u8 *key_in, u32 authkeylen) 495 { 496 struct device *jrdev = ctx->jrdev; 497 u32 *desc; 498 struct split_key_result result; 499 dma_addr_t dma_addr_in, dma_addr_out; 500 int ret = 0; 501 502 desc = kmalloc(CAAM_CMD_SZ * 6 + CAAM_PTR_SZ * 2, GFP_KERNEL | GFP_DMA); 503 504 init_job_desc(desc, 0); 505 506 dma_addr_in = dma_map_single(jrdev, (void *)key_in, authkeylen, 507 DMA_TO_DEVICE); 508 if (dma_mapping_error(jrdev, dma_addr_in)) { 509 dev_err(jrdev, "unable to map key input memory\n"); 510 kfree(desc); 511 return -ENOMEM; 512 } 513 append_key(desc, dma_addr_in, authkeylen, CLASS_2 | 514 KEY_DEST_CLASS_REG); 515 516 /* Sets MDHA up into an HMAC-INIT */ 517 append_operation(desc, ctx->alg_op | OP_ALG_DECRYPT | 518 OP_ALG_AS_INIT); 519 520 /* 521 * do a FIFO_LOAD of zero, this will trigger the internal key expansion 522 into both pads inside MDHA 523 */ 524 append_fifo_load_as_imm(desc, NULL, 0, LDST_CLASS_2_CCB | 525 FIFOLD_TYPE_MSG | FIFOLD_TYPE_LAST2); 526 527 /* 528 * FIFO_STORE with the explicit split-key content store 529 * (0x26 output type) 530 */ 531 dma_addr_out = dma_map_single(jrdev, ctx->key, ctx->split_key_pad_len, 532 DMA_FROM_DEVICE); 533 if (dma_mapping_error(jrdev, dma_addr_out)) { 534 dev_err(jrdev, "unable to map key output memory\n"); 535 kfree(desc); 536 return -ENOMEM; 537 } 538 append_fifo_store(desc, dma_addr_out, ctx->split_key_len, 539 LDST_CLASS_2_CCB | FIFOST_TYPE_SPLIT_KEK); 540 541 #ifdef DEBUG 542 print_hex_dump(KERN_ERR, "ctx.key@"xstr(__LINE__)": ", 543 DUMP_PREFIX_ADDRESS, 16, 4, key_in, authkeylen, 1); 544 print_hex_dump(KERN_ERR, "jobdesc@"xstr(__LINE__)": ", 545 DUMP_PREFIX_ADDRESS, 16, 4, desc, desc_bytes(desc), 1); 546 #endif 547 548 result.err = 0; 549 init_completion(&result.completion); 550 551 ret = caam_jr_enqueue(jrdev, desc, split_key_done, &result); 552 if (!ret) { 553 /* in progress */ 554 wait_for_completion_interruptible(&result.completion); 555 ret = result.err; 556 #ifdef DEBUG 557 print_hex_dump(KERN_ERR, "ctx.key@"xstr(__LINE__)": ", 558 DUMP_PREFIX_ADDRESS, 16, 4, ctx->key, 559 ctx->split_key_pad_len, 1); 560 #endif 561 } 562 563 dma_unmap_single(jrdev, dma_addr_out, ctx->split_key_pad_len, 564 DMA_FROM_DEVICE); 565 dma_unmap_single(jrdev, dma_addr_in, authkeylen, DMA_TO_DEVICE); 566 567 kfree(desc); 568 569 return ret; 570 } 571 572 static int aead_setkey(struct crypto_aead *aead, 573 const u8 *key, unsigned int keylen) 574 { 575 /* Sizes for MDHA pads (*not* keys): MD5, SHA1, 224, 256, 384, 512 */ 576 static const u8 mdpadlen[] = { 16, 20, 32, 32, 64, 64 }; 577 struct caam_ctx *ctx = crypto_aead_ctx(aead); 578 struct device *jrdev = ctx->jrdev; 579 struct rtattr *rta = (void *)key; 580 struct crypto_authenc_key_param *param; 581 unsigned int authkeylen; 582 unsigned int enckeylen; 583 int ret = 0; 584 585 param = RTA_DATA(rta); 586 enckeylen = be32_to_cpu(param->enckeylen); 587 588 key += RTA_ALIGN(rta->rta_len); 589 keylen -= RTA_ALIGN(rta->rta_len); 590 591 if (keylen < enckeylen) 592 goto badkey; 593 594 authkeylen = keylen - enckeylen; 595 596 if (keylen > CAAM_MAX_KEY_SIZE) 597 goto badkey; 598 599 /* Pick class 2 key length from algorithm submask */ 600 ctx->split_key_len = mdpadlen[(ctx->alg_op & OP_ALG_ALGSEL_SUBMASK) >> 601 OP_ALG_ALGSEL_SHIFT] * 2; 602 ctx->split_key_pad_len = ALIGN(ctx->split_key_len, 16); 603 604 #ifdef DEBUG 605 printk(KERN_ERR "keylen %d enckeylen %d authkeylen %d\n", 606 keylen, enckeylen, authkeylen); 607 printk(KERN_ERR "split_key_len %d split_key_pad_len %d\n", 608 ctx->split_key_len, ctx->split_key_pad_len); 609 print_hex_dump(KERN_ERR, "key in @"xstr(__LINE__)": ", 610 DUMP_PREFIX_ADDRESS, 16, 4, key, keylen, 1); 611 #endif 612 613 ret = gen_split_key(ctx, key, authkeylen); 614 if (ret) { 615 goto badkey; 616 } 617 618 /* postpend encryption key to auth split key */ 619 memcpy(ctx->key + ctx->split_key_pad_len, key + authkeylen, enckeylen); 620 621 ctx->key_dma = dma_map_single(jrdev, ctx->key, ctx->split_key_pad_len + 622 enckeylen, DMA_TO_DEVICE); 623 if (dma_mapping_error(jrdev, ctx->key_dma)) { 624 dev_err(jrdev, "unable to map key i/o memory\n"); 625 return -ENOMEM; 626 } 627 #ifdef DEBUG 628 print_hex_dump(KERN_ERR, "ctx.key@"xstr(__LINE__)": ", 629 DUMP_PREFIX_ADDRESS, 16, 4, ctx->key, 630 ctx->split_key_pad_len + enckeylen, 1); 631 #endif 632 633 ctx->enckeylen = enckeylen; 634 635 ret = aead_set_sh_desc(aead); 636 if (ret) { 637 dma_unmap_single(jrdev, ctx->key_dma, ctx->split_key_pad_len + 638 enckeylen, DMA_TO_DEVICE); 639 } 640 641 return ret; 642 badkey: 643 crypto_aead_set_flags(aead, CRYPTO_TFM_RES_BAD_KEY_LEN); 644 return -EINVAL; 645 } 646 647 static int ablkcipher_setkey(struct crypto_ablkcipher *ablkcipher, 648 const u8 *key, unsigned int keylen) 649 { 650 struct caam_ctx *ctx = crypto_ablkcipher_ctx(ablkcipher); 651 struct ablkcipher_tfm *tfm = &ablkcipher->base.crt_ablkcipher; 652 struct device *jrdev = ctx->jrdev; 653 int ret = 0; 654 u32 *key_jump_cmd, *jump_cmd; 655 u32 *desc; 656 657 #ifdef DEBUG 658 print_hex_dump(KERN_ERR, "key in @"xstr(__LINE__)": ", 659 DUMP_PREFIX_ADDRESS, 16, 4, key, keylen, 1); 660 #endif 661 662 memcpy(ctx->key, key, keylen); 663 ctx->key_dma = dma_map_single(jrdev, ctx->key, keylen, 664 DMA_TO_DEVICE); 665 if (dma_mapping_error(jrdev, ctx->key_dma)) { 666 dev_err(jrdev, "unable to map key i/o memory\n"); 667 return -ENOMEM; 668 } 669 ctx->enckeylen = keylen; 670 671 /* ablkcipher_encrypt shared descriptor */ 672 desc = ctx->sh_desc_enc; 673 init_sh_desc(desc, HDR_SHARE_WAIT); 674 /* Skip if already shared */ 675 key_jump_cmd = append_jump(desc, JUMP_JSL | JUMP_TEST_ALL | 676 JUMP_COND_SHRD); 677 678 /* Load class1 key only */ 679 append_key_as_imm(desc, (void *)ctx->key, ctx->enckeylen, 680 ctx->enckeylen, CLASS_1 | 681 KEY_DEST_CLASS_REG); 682 683 set_jump_tgt_here(desc, key_jump_cmd); 684 685 /* Propagate errors from shared to job descriptor */ 686 append_cmd(desc, SET_OK_NO_PROP_ERRORS | CMD_LOAD); 687 688 /* Load iv */ 689 append_cmd(desc, CMD_SEQ_LOAD | LDST_SRCDST_BYTE_CONTEXT | 690 LDST_CLASS_1_CCB | tfm->ivsize); 691 692 /* Load operation */ 693 append_operation(desc, ctx->class1_alg_type | 694 OP_ALG_AS_INITFINAL | OP_ALG_ENCRYPT); 695 696 /* Perform operation */ 697 ablkcipher_append_src_dst(desc); 698 699 ctx->sh_desc_enc_dma = dma_map_single(jrdev, desc, 700 desc_bytes(desc), 701 DMA_TO_DEVICE); 702 if (dma_mapping_error(jrdev, ctx->sh_desc_enc_dma)) { 703 dev_err(jrdev, "unable to map shared descriptor\n"); 704 return -ENOMEM; 705 } 706 #ifdef DEBUG 707 print_hex_dump(KERN_ERR, "ablkcipher enc shdesc@"xstr(__LINE__)": ", 708 DUMP_PREFIX_ADDRESS, 16, 4, desc, 709 desc_bytes(desc), 1); 710 #endif 711 /* ablkcipher_decrypt shared descriptor */ 712 desc = ctx->sh_desc_dec; 713 714 init_sh_desc(desc, HDR_SHARE_WAIT); 715 /* Skip if already shared */ 716 key_jump_cmd = append_jump(desc, JUMP_JSL | JUMP_TEST_ALL | 717 JUMP_COND_SHRD); 718 719 /* Load class1 key only */ 720 append_key_as_imm(desc, (void *)ctx->key, ctx->enckeylen, 721 ctx->enckeylen, CLASS_1 | 722 KEY_DEST_CLASS_REG); 723 724 /* For aead, only propagate error immediately if shared */ 725 jump_cmd = append_jump(desc, JUMP_TEST_ALL); 726 set_jump_tgt_here(desc, key_jump_cmd); 727 append_cmd(desc, SET_OK_NO_PROP_ERRORS | CMD_LOAD); 728 set_jump_tgt_here(desc, jump_cmd); 729 730 /* load IV */ 731 append_cmd(desc, CMD_SEQ_LOAD | LDST_SRCDST_BYTE_CONTEXT | 732 LDST_CLASS_1_CCB | tfm->ivsize); 733 734 /* Choose operation */ 735 append_dec_op1(desc, ctx->class1_alg_type); 736 737 /* Perform operation */ 738 ablkcipher_append_src_dst(desc); 739 740 /* Wait for key to load before allowing propagating error */ 741 append_dec_shr_done(desc); 742 743 ctx->sh_desc_dec_dma = dma_map_single(jrdev, desc, 744 desc_bytes(desc), 745 DMA_TO_DEVICE); 746 if (dma_mapping_error(jrdev, ctx->sh_desc_enc_dma)) { 747 dev_err(jrdev, "unable to map shared descriptor\n"); 748 return -ENOMEM; 749 } 750 751 #ifdef DEBUG 752 print_hex_dump(KERN_ERR, "ablkcipher dec shdesc@"xstr(__LINE__)": ", 753 DUMP_PREFIX_ADDRESS, 16, 4, desc, 754 desc_bytes(desc), 1); 755 #endif 756 757 return ret; 758 } 759 760 struct link_tbl_entry { 761 u64 ptr; 762 u32 len; 763 u8 reserved; 764 u8 buf_pool_id; 765 u16 offset; 766 }; 767 768 /* 769 * aead_edesc - s/w-extended aead descriptor 770 * @assoc_nents: number of segments in associated data (SPI+Seq) scatterlist 771 * @src_nents: number of segments in input scatterlist 772 * @dst_nents: number of segments in output scatterlist 773 * @iv_dma: dma address of iv for checking continuity and link table 774 * @desc: h/w descriptor (variable length; must not exceed MAX_CAAM_DESCSIZE) 775 * @link_tbl_bytes: length of dma mapped link_tbl space 776 * @link_tbl_dma: bus physical mapped address of h/w link table 777 * @hw_desc: the h/w job descriptor followed by any referenced link tables 778 */ 779 struct aead_edesc { 780 int assoc_nents; 781 int src_nents; 782 int dst_nents; 783 dma_addr_t iv_dma; 784 int link_tbl_bytes; 785 dma_addr_t link_tbl_dma; 786 struct link_tbl_entry *link_tbl; 787 u32 hw_desc[0]; 788 }; 789 790 /* 791 * ablkcipher_edesc - s/w-extended ablkcipher descriptor 792 * @src_nents: number of segments in input scatterlist 793 * @dst_nents: number of segments in output scatterlist 794 * @iv_dma: dma address of iv for checking continuity and link table 795 * @desc: h/w descriptor (variable length; must not exceed MAX_CAAM_DESCSIZE) 796 * @link_tbl_bytes: length of dma mapped link_tbl space 797 * @link_tbl_dma: bus physical mapped address of h/w link table 798 * @hw_desc: the h/w job descriptor followed by any referenced link tables 799 */ 800 struct ablkcipher_edesc { 801 int src_nents; 802 int dst_nents; 803 dma_addr_t iv_dma; 804 int link_tbl_bytes; 805 dma_addr_t link_tbl_dma; 806 struct link_tbl_entry *link_tbl; 807 u32 hw_desc[0]; 808 }; 809 810 static void caam_unmap(struct device *dev, struct scatterlist *src, 811 struct scatterlist *dst, int src_nents, int dst_nents, 812 dma_addr_t iv_dma, int ivsize, dma_addr_t link_tbl_dma, 813 int link_tbl_bytes) 814 { 815 if (unlikely(dst != src)) { 816 dma_unmap_sg(dev, src, src_nents, DMA_TO_DEVICE); 817 dma_unmap_sg(dev, dst, dst_nents, DMA_FROM_DEVICE); 818 } else { 819 dma_unmap_sg(dev, src, src_nents, DMA_BIDIRECTIONAL); 820 } 821 822 if (iv_dma) 823 dma_unmap_single(dev, iv_dma, ivsize, DMA_TO_DEVICE); 824 if (link_tbl_bytes) 825 dma_unmap_single(dev, link_tbl_dma, link_tbl_bytes, 826 DMA_TO_DEVICE); 827 } 828 829 static void aead_unmap(struct device *dev, 830 struct aead_edesc *edesc, 831 struct aead_request *req) 832 { 833 struct crypto_aead *aead = crypto_aead_reqtfm(req); 834 int ivsize = crypto_aead_ivsize(aead); 835 836 dma_unmap_sg(dev, req->assoc, edesc->assoc_nents, DMA_TO_DEVICE); 837 838 caam_unmap(dev, req->src, req->dst, 839 edesc->src_nents, edesc->dst_nents, 840 edesc->iv_dma, ivsize, edesc->link_tbl_dma, 841 edesc->link_tbl_bytes); 842 } 843 844 static void ablkcipher_unmap(struct device *dev, 845 struct ablkcipher_edesc *edesc, 846 struct ablkcipher_request *req) 847 { 848 struct crypto_ablkcipher *ablkcipher = crypto_ablkcipher_reqtfm(req); 849 int ivsize = crypto_ablkcipher_ivsize(ablkcipher); 850 851 caam_unmap(dev, req->src, req->dst, 852 edesc->src_nents, edesc->dst_nents, 853 edesc->iv_dma, ivsize, edesc->link_tbl_dma, 854 edesc->link_tbl_bytes); 855 } 856 857 static void aead_encrypt_done(struct device *jrdev, u32 *desc, u32 err, 858 void *context) 859 { 860 struct aead_request *req = context; 861 struct aead_edesc *edesc; 862 #ifdef DEBUG 863 struct crypto_aead *aead = crypto_aead_reqtfm(req); 864 struct caam_ctx *ctx = crypto_aead_ctx(aead); 865 int ivsize = crypto_aead_ivsize(aead); 866 867 dev_err(jrdev, "%s %d: err 0x%x\n", __func__, __LINE__, err); 868 #endif 869 870 edesc = (struct aead_edesc *)((char *)desc - 871 offsetof(struct aead_edesc, hw_desc)); 872 873 if (err) { 874 char tmp[CAAM_ERROR_STR_MAX]; 875 876 dev_err(jrdev, "%08x: %s\n", err, caam_jr_strstatus(tmp, err)); 877 } 878 879 aead_unmap(jrdev, edesc, req); 880 881 #ifdef DEBUG 882 print_hex_dump(KERN_ERR, "assoc @"xstr(__LINE__)": ", 883 DUMP_PREFIX_ADDRESS, 16, 4, sg_virt(req->assoc), 884 req->assoclen , 1); 885 print_hex_dump(KERN_ERR, "dstiv @"xstr(__LINE__)": ", 886 DUMP_PREFIX_ADDRESS, 16, 4, sg_virt(req->src) - ivsize, 887 edesc->src_nents ? 100 : ivsize, 1); 888 print_hex_dump(KERN_ERR, "dst @"xstr(__LINE__)": ", 889 DUMP_PREFIX_ADDRESS, 16, 4, sg_virt(req->src), 890 edesc->src_nents ? 100 : req->cryptlen + 891 ctx->authsize + 4, 1); 892 #endif 893 894 kfree(edesc); 895 896 aead_request_complete(req, err); 897 } 898 899 static void aead_decrypt_done(struct device *jrdev, u32 *desc, u32 err, 900 void *context) 901 { 902 struct aead_request *req = context; 903 struct aead_edesc *edesc; 904 #ifdef DEBUG 905 struct crypto_aead *aead = crypto_aead_reqtfm(req); 906 struct caam_ctx *ctx = crypto_aead_ctx(aead); 907 int ivsize = crypto_aead_ivsize(aead); 908 909 dev_err(jrdev, "%s %d: err 0x%x\n", __func__, __LINE__, err); 910 #endif 911 912 edesc = (struct aead_edesc *)((char *)desc - 913 offsetof(struct aead_edesc, hw_desc)); 914 915 #ifdef DEBUG 916 print_hex_dump(KERN_ERR, "dstiv @"xstr(__LINE__)": ", 917 DUMP_PREFIX_ADDRESS, 16, 4, req->iv, 918 ivsize, 1); 919 print_hex_dump(KERN_ERR, "dst @"xstr(__LINE__)": ", 920 DUMP_PREFIX_ADDRESS, 16, 4, sg_virt(req->dst), 921 req->cryptlen, 1); 922 #endif 923 924 if (err) { 925 char tmp[CAAM_ERROR_STR_MAX]; 926 927 dev_err(jrdev, "%08x: %s\n", err, caam_jr_strstatus(tmp, err)); 928 } 929 930 aead_unmap(jrdev, edesc, req); 931 932 /* 933 * verify hw auth check passed else return -EBADMSG 934 */ 935 if ((err & JRSTA_CCBERR_ERRID_MASK) == JRSTA_CCBERR_ERRID_ICVCHK) 936 err = -EBADMSG; 937 938 #ifdef DEBUG 939 print_hex_dump(KERN_ERR, "iphdrout@"xstr(__LINE__)": ", 940 DUMP_PREFIX_ADDRESS, 16, 4, 941 ((char *)sg_virt(req->assoc) - sizeof(struct iphdr)), 942 sizeof(struct iphdr) + req->assoclen + 943 ((req->cryptlen > 1500) ? 1500 : req->cryptlen) + 944 ctx->authsize + 36, 1); 945 if (!err && edesc->link_tbl_bytes) { 946 struct scatterlist *sg = sg_last(req->src, edesc->src_nents); 947 print_hex_dump(KERN_ERR, "sglastout@"xstr(__LINE__)": ", 948 DUMP_PREFIX_ADDRESS, 16, 4, sg_virt(sg), 949 sg->length + ctx->authsize + 16, 1); 950 } 951 #endif 952 953 kfree(edesc); 954 955 aead_request_complete(req, err); 956 } 957 958 static void ablkcipher_encrypt_done(struct device *jrdev, u32 *desc, u32 err, 959 void *context) 960 { 961 struct ablkcipher_request *req = context; 962 struct ablkcipher_edesc *edesc; 963 #ifdef DEBUG 964 struct crypto_ablkcipher *ablkcipher = crypto_ablkcipher_reqtfm(req); 965 int ivsize = crypto_ablkcipher_ivsize(ablkcipher); 966 967 dev_err(jrdev, "%s %d: err 0x%x\n", __func__, __LINE__, err); 968 #endif 969 970 edesc = (struct ablkcipher_edesc *)((char *)desc - 971 offsetof(struct ablkcipher_edesc, hw_desc)); 972 973 if (err) { 974 char tmp[CAAM_ERROR_STR_MAX]; 975 976 dev_err(jrdev, "%08x: %s\n", err, caam_jr_strstatus(tmp, err)); 977 } 978 979 #ifdef DEBUG 980 print_hex_dump(KERN_ERR, "dstiv @"xstr(__LINE__)": ", 981 DUMP_PREFIX_ADDRESS, 16, 4, req->info, 982 edesc->src_nents > 1 ? 100 : ivsize, 1); 983 print_hex_dump(KERN_ERR, "dst @"xstr(__LINE__)": ", 984 DUMP_PREFIX_ADDRESS, 16, 4, sg_virt(req->src), 985 edesc->dst_nents > 1 ? 100 : req->nbytes, 1); 986 #endif 987 988 ablkcipher_unmap(jrdev, edesc, req); 989 kfree(edesc); 990 991 ablkcipher_request_complete(req, err); 992 } 993 994 static void ablkcipher_decrypt_done(struct device *jrdev, u32 *desc, u32 err, 995 void *context) 996 { 997 struct ablkcipher_request *req = context; 998 struct ablkcipher_edesc *edesc; 999 #ifdef DEBUG 1000 struct crypto_ablkcipher *ablkcipher = crypto_ablkcipher_reqtfm(req); 1001 int ivsize = crypto_ablkcipher_ivsize(ablkcipher); 1002 1003 dev_err(jrdev, "%s %d: err 0x%x\n", __func__, __LINE__, err); 1004 #endif 1005 1006 edesc = (struct ablkcipher_edesc *)((char *)desc - 1007 offsetof(struct ablkcipher_edesc, hw_desc)); 1008 if (err) { 1009 char tmp[CAAM_ERROR_STR_MAX]; 1010 1011 dev_err(jrdev, "%08x: %s\n", err, caam_jr_strstatus(tmp, err)); 1012 } 1013 1014 #ifdef DEBUG 1015 print_hex_dump(KERN_ERR, "dstiv @"xstr(__LINE__)": ", 1016 DUMP_PREFIX_ADDRESS, 16, 4, req->info, 1017 ivsize, 1); 1018 print_hex_dump(KERN_ERR, "dst @"xstr(__LINE__)": ", 1019 DUMP_PREFIX_ADDRESS, 16, 4, sg_virt(req->src), 1020 edesc->dst_nents > 1 ? 100 : req->nbytes, 1); 1021 #endif 1022 1023 ablkcipher_unmap(jrdev, edesc, req); 1024 kfree(edesc); 1025 1026 ablkcipher_request_complete(req, err); 1027 } 1028 1029 static void sg_to_link_tbl_one(struct link_tbl_entry *link_tbl_ptr, 1030 dma_addr_t dma, u32 len, u32 offset) 1031 { 1032 link_tbl_ptr->ptr = dma; 1033 link_tbl_ptr->len = len; 1034 link_tbl_ptr->reserved = 0; 1035 link_tbl_ptr->buf_pool_id = 0; 1036 link_tbl_ptr->offset = offset; 1037 #ifdef DEBUG 1038 print_hex_dump(KERN_ERR, "link_tbl_ptr@"xstr(__LINE__)": ", 1039 DUMP_PREFIX_ADDRESS, 16, 4, link_tbl_ptr, 1040 sizeof(struct link_tbl_entry), 1); 1041 #endif 1042 } 1043 1044 /* 1045 * convert scatterlist to h/w link table format 1046 * but does not have final bit; instead, returns last entry 1047 */ 1048 static struct link_tbl_entry *sg_to_link_tbl(struct scatterlist *sg, 1049 int sg_count, struct link_tbl_entry 1050 *link_tbl_ptr, u32 offset) 1051 { 1052 while (sg_count) { 1053 sg_to_link_tbl_one(link_tbl_ptr, sg_dma_address(sg), 1054 sg_dma_len(sg), offset); 1055 link_tbl_ptr++; 1056 sg = sg_next(sg); 1057 sg_count--; 1058 } 1059 return link_tbl_ptr - 1; 1060 } 1061 1062 /* 1063 * convert scatterlist to h/w link table format 1064 * scatterlist must have been previously dma mapped 1065 */ 1066 static void sg_to_link_tbl_last(struct scatterlist *sg, int sg_count, 1067 struct link_tbl_entry *link_tbl_ptr, u32 offset) 1068 { 1069 link_tbl_ptr = sg_to_link_tbl(sg, sg_count, link_tbl_ptr, offset); 1070 link_tbl_ptr->len |= 0x40000000; 1071 } 1072 1073 /* 1074 * Fill in aead job descriptor 1075 */ 1076 static void init_aead_job(u32 *sh_desc, dma_addr_t ptr, 1077 struct aead_edesc *edesc, 1078 struct aead_request *req, 1079 bool all_contig, bool encrypt) 1080 { 1081 struct crypto_aead *aead = crypto_aead_reqtfm(req); 1082 struct caam_ctx *ctx = crypto_aead_ctx(aead); 1083 int ivsize = crypto_aead_ivsize(aead); 1084 int authsize = ctx->authsize; 1085 u32 *desc = edesc->hw_desc; 1086 u32 out_options = 0, in_options; 1087 dma_addr_t dst_dma, src_dma; 1088 int len, link_tbl_index = 0; 1089 1090 #ifdef DEBUG 1091 debug("assoclen %d cryptlen %d authsize %d\n", 1092 req->assoclen, req->cryptlen, authsize); 1093 print_hex_dump(KERN_ERR, "assoc @"xstr(__LINE__)": ", 1094 DUMP_PREFIX_ADDRESS, 16, 4, sg_virt(req->assoc), 1095 req->assoclen , 1); 1096 print_hex_dump(KERN_ERR, "presciv@"xstr(__LINE__)": ", 1097 DUMP_PREFIX_ADDRESS, 16, 4, req->iv, 1098 edesc->src_nents ? 100 : ivsize, 1); 1099 print_hex_dump(KERN_ERR, "src @"xstr(__LINE__)": ", 1100 DUMP_PREFIX_ADDRESS, 16, 4, sg_virt(req->src), 1101 edesc->src_nents ? 100 : req->cryptlen, 1); 1102 print_hex_dump(KERN_ERR, "shrdesc@"xstr(__LINE__)": ", 1103 DUMP_PREFIX_ADDRESS, 16, 4, sh_desc, 1104 desc_bytes(sh_desc), 1); 1105 #endif 1106 1107 len = desc_len(sh_desc); 1108 init_job_desc_shared(desc, ptr, len, HDR_SHARE_DEFER | HDR_REVERSE); 1109 1110 if (all_contig) { 1111 src_dma = sg_dma_address(req->assoc); 1112 in_options = 0; 1113 } else { 1114 src_dma = edesc->link_tbl_dma; 1115 link_tbl_index += (edesc->assoc_nents ? : 1) + 1 + 1116 (edesc->src_nents ? : 1); 1117 in_options = LDST_SGF; 1118 } 1119 if (encrypt) 1120 append_seq_in_ptr(desc, src_dma, req->assoclen + ivsize + 1121 req->cryptlen - authsize, in_options); 1122 else 1123 append_seq_in_ptr(desc, src_dma, req->assoclen + ivsize + 1124 req->cryptlen, in_options); 1125 1126 if (likely(req->src == req->dst)) { 1127 if (all_contig) { 1128 dst_dma = sg_dma_address(req->src); 1129 } else { 1130 dst_dma = src_dma + sizeof(struct link_tbl_entry) * 1131 ((edesc->assoc_nents ? : 1) + 1); 1132 out_options = LDST_SGF; 1133 } 1134 } else { 1135 if (!edesc->dst_nents) { 1136 dst_dma = sg_dma_address(req->dst); 1137 } else { 1138 dst_dma = edesc->link_tbl_dma + 1139 link_tbl_index * 1140 sizeof(struct link_tbl_entry); 1141 out_options = LDST_SGF; 1142 } 1143 } 1144 if (encrypt) 1145 append_seq_out_ptr(desc, dst_dma, req->cryptlen, out_options); 1146 else 1147 append_seq_out_ptr(desc, dst_dma, req->cryptlen - authsize, 1148 out_options); 1149 } 1150 1151 /* 1152 * Fill in aead givencrypt job descriptor 1153 */ 1154 static void init_aead_giv_job(u32 *sh_desc, dma_addr_t ptr, 1155 struct aead_edesc *edesc, 1156 struct aead_request *req, 1157 int contig) 1158 { 1159 struct crypto_aead *aead = crypto_aead_reqtfm(req); 1160 struct caam_ctx *ctx = crypto_aead_ctx(aead); 1161 int ivsize = crypto_aead_ivsize(aead); 1162 int authsize = ctx->authsize; 1163 u32 *desc = edesc->hw_desc; 1164 u32 out_options = 0, in_options; 1165 dma_addr_t dst_dma, src_dma; 1166 int len, link_tbl_index = 0; 1167 1168 #ifdef DEBUG 1169 debug("assoclen %d cryptlen %d authsize %d\n", 1170 req->assoclen, req->cryptlen, authsize); 1171 print_hex_dump(KERN_ERR, "assoc @"xstr(__LINE__)": ", 1172 DUMP_PREFIX_ADDRESS, 16, 4, sg_virt(req->assoc), 1173 req->assoclen , 1); 1174 print_hex_dump(KERN_ERR, "presciv@"xstr(__LINE__)": ", 1175 DUMP_PREFIX_ADDRESS, 16, 4, req->iv, ivsize, 1); 1176 print_hex_dump(KERN_ERR, "src @"xstr(__LINE__)": ", 1177 DUMP_PREFIX_ADDRESS, 16, 4, sg_virt(req->src), 1178 edesc->src_nents > 1 ? 100 : req->cryptlen, 1); 1179 print_hex_dump(KERN_ERR, "shrdesc@"xstr(__LINE__)": ", 1180 DUMP_PREFIX_ADDRESS, 16, 4, sh_desc, 1181 desc_bytes(sh_desc), 1); 1182 #endif 1183 1184 len = desc_len(sh_desc); 1185 init_job_desc_shared(desc, ptr, len, HDR_SHARE_DEFER | HDR_REVERSE); 1186 1187 if (contig & GIV_SRC_CONTIG) { 1188 src_dma = sg_dma_address(req->assoc); 1189 in_options = 0; 1190 } else { 1191 src_dma = edesc->link_tbl_dma; 1192 link_tbl_index += edesc->assoc_nents + 1 + edesc->src_nents; 1193 in_options = LDST_SGF; 1194 } 1195 append_seq_in_ptr(desc, src_dma, req->assoclen + ivsize + 1196 req->cryptlen - authsize, in_options); 1197 1198 if (contig & GIV_DST_CONTIG) { 1199 dst_dma = edesc->iv_dma; 1200 } else { 1201 if (likely(req->src == req->dst)) { 1202 dst_dma = src_dma + sizeof(struct link_tbl_entry) * 1203 edesc->assoc_nents; 1204 out_options = LDST_SGF; 1205 } else { 1206 dst_dma = edesc->link_tbl_dma + 1207 link_tbl_index * 1208 sizeof(struct link_tbl_entry); 1209 out_options = LDST_SGF; 1210 } 1211 } 1212 1213 append_seq_out_ptr(desc, dst_dma, ivsize + req->cryptlen, out_options); 1214 } 1215 1216 /* 1217 * Fill in ablkcipher job descriptor 1218 */ 1219 static void init_ablkcipher_job(u32 *sh_desc, dma_addr_t ptr, 1220 struct ablkcipher_edesc *edesc, 1221 struct ablkcipher_request *req, 1222 bool iv_contig) 1223 { 1224 struct crypto_ablkcipher *ablkcipher = crypto_ablkcipher_reqtfm(req); 1225 int ivsize = crypto_ablkcipher_ivsize(ablkcipher); 1226 u32 *desc = edesc->hw_desc; 1227 u32 out_options = 0, in_options; 1228 dma_addr_t dst_dma, src_dma; 1229 int len, link_tbl_index = 0; 1230 1231 #ifdef DEBUG 1232 print_hex_dump(KERN_ERR, "presciv@"xstr(__LINE__)": ", 1233 DUMP_PREFIX_ADDRESS, 16, 4, req->info, 1234 ivsize, 1); 1235 print_hex_dump(KERN_ERR, "src @"xstr(__LINE__)": ", 1236 DUMP_PREFIX_ADDRESS, 16, 4, sg_virt(req->src), 1237 edesc->src_nents ? 100 : req->nbytes, 1); 1238 #endif 1239 1240 len = desc_len(sh_desc); 1241 init_job_desc_shared(desc, ptr, len, HDR_SHARE_DEFER | HDR_REVERSE); 1242 1243 if (iv_contig) { 1244 src_dma = edesc->iv_dma; 1245 in_options = 0; 1246 } else { 1247 src_dma = edesc->link_tbl_dma; 1248 link_tbl_index += (iv_contig ? 0 : 1) + edesc->src_nents; 1249 in_options = LDST_SGF; 1250 } 1251 append_seq_in_ptr(desc, src_dma, req->nbytes + ivsize, in_options); 1252 1253 if (likely(req->src == req->dst)) { 1254 if (!edesc->src_nents && iv_contig) { 1255 dst_dma = sg_dma_address(req->src); 1256 } else { 1257 dst_dma = edesc->link_tbl_dma + 1258 sizeof(struct link_tbl_entry); 1259 out_options = LDST_SGF; 1260 } 1261 } else { 1262 if (!edesc->dst_nents) { 1263 dst_dma = sg_dma_address(req->dst); 1264 } else { 1265 dst_dma = edesc->link_tbl_dma + 1266 link_tbl_index * sizeof(struct link_tbl_entry); 1267 out_options = LDST_SGF; 1268 } 1269 } 1270 append_seq_out_ptr(desc, dst_dma, req->nbytes, out_options); 1271 } 1272 1273 /* 1274 * derive number of elements in scatterlist 1275 */ 1276 static int sg_count(struct scatterlist *sg_list, int nbytes) 1277 { 1278 struct scatterlist *sg = sg_list; 1279 int sg_nents = 0; 1280 1281 while (nbytes > 0) { 1282 sg_nents++; 1283 nbytes -= sg->length; 1284 if (!sg_is_last(sg) && (sg + 1)->length == 0) 1285 BUG(); /* Not support chaining */ 1286 sg = scatterwalk_sg_next(sg); 1287 } 1288 1289 if (likely(sg_nents == 1)) 1290 return 0; 1291 1292 return sg_nents; 1293 } 1294 1295 /* 1296 * allocate and map the aead extended descriptor 1297 */ 1298 static struct aead_edesc *aead_edesc_alloc(struct aead_request *req, 1299 int desc_bytes, bool *all_contig_ptr) 1300 { 1301 struct crypto_aead *aead = crypto_aead_reqtfm(req); 1302 struct caam_ctx *ctx = crypto_aead_ctx(aead); 1303 struct device *jrdev = ctx->jrdev; 1304 gfp_t flags = (req->base.flags & (CRYPTO_TFM_REQ_MAY_BACKLOG | 1305 CRYPTO_TFM_REQ_MAY_SLEEP)) ? GFP_KERNEL : GFP_ATOMIC; 1306 int assoc_nents, src_nents, dst_nents = 0; 1307 struct aead_edesc *edesc; 1308 dma_addr_t iv_dma = 0; 1309 int sgc; 1310 bool all_contig = true; 1311 int ivsize = crypto_aead_ivsize(aead); 1312 int link_tbl_index, link_tbl_len = 0, link_tbl_bytes; 1313 1314 assoc_nents = sg_count(req->assoc, req->assoclen); 1315 src_nents = sg_count(req->src, req->cryptlen); 1316 1317 if (unlikely(req->dst != req->src)) 1318 dst_nents = sg_count(req->dst, req->cryptlen); 1319 1320 sgc = dma_map_sg(jrdev, req->assoc, assoc_nents ? : 1, 1321 DMA_BIDIRECTIONAL); 1322 if (likely(req->src == req->dst)) { 1323 sgc = dma_map_sg(jrdev, req->src, src_nents ? : 1, 1324 DMA_BIDIRECTIONAL); 1325 } else { 1326 sgc = dma_map_sg(jrdev, req->src, src_nents ? : 1, 1327 DMA_TO_DEVICE); 1328 sgc = dma_map_sg(jrdev, req->dst, dst_nents ? : 1, 1329 DMA_FROM_DEVICE); 1330 } 1331 1332 /* Check if data are contiguous */ 1333 iv_dma = dma_map_single(jrdev, req->iv, ivsize, DMA_TO_DEVICE); 1334 if (assoc_nents || sg_dma_address(req->assoc) + req->assoclen != 1335 iv_dma || src_nents || iv_dma + ivsize != 1336 sg_dma_address(req->src)) { 1337 all_contig = false; 1338 assoc_nents = assoc_nents ? : 1; 1339 src_nents = src_nents ? : 1; 1340 link_tbl_len = assoc_nents + 1 + src_nents; 1341 } 1342 link_tbl_len += dst_nents; 1343 1344 link_tbl_bytes = link_tbl_len * sizeof(struct link_tbl_entry); 1345 1346 /* allocate space for base edesc and hw desc commands, link tables */ 1347 edesc = kmalloc(sizeof(struct aead_edesc) + desc_bytes + 1348 link_tbl_bytes, GFP_DMA | flags); 1349 if (!edesc) { 1350 dev_err(jrdev, "could not allocate extended descriptor\n"); 1351 return ERR_PTR(-ENOMEM); 1352 } 1353 1354 edesc->assoc_nents = assoc_nents; 1355 edesc->src_nents = src_nents; 1356 edesc->dst_nents = dst_nents; 1357 edesc->iv_dma = iv_dma; 1358 edesc->link_tbl_bytes = link_tbl_bytes; 1359 edesc->link_tbl = (void *)edesc + sizeof(struct aead_edesc) + 1360 desc_bytes; 1361 edesc->link_tbl_dma = dma_map_single(jrdev, edesc->link_tbl, 1362 link_tbl_bytes, DMA_TO_DEVICE); 1363 *all_contig_ptr = all_contig; 1364 1365 link_tbl_index = 0; 1366 if (!all_contig) { 1367 sg_to_link_tbl(req->assoc, 1368 (assoc_nents ? : 1), 1369 edesc->link_tbl + 1370 link_tbl_index, 0); 1371 link_tbl_index += assoc_nents ? : 1; 1372 sg_to_link_tbl_one(edesc->link_tbl + link_tbl_index, 1373 iv_dma, ivsize, 0); 1374 link_tbl_index += 1; 1375 sg_to_link_tbl_last(req->src, 1376 (src_nents ? : 1), 1377 edesc->link_tbl + 1378 link_tbl_index, 0); 1379 link_tbl_index += src_nents ? : 1; 1380 } 1381 if (dst_nents) { 1382 sg_to_link_tbl_last(req->dst, dst_nents, 1383 edesc->link_tbl + link_tbl_index, 0); 1384 } 1385 1386 return edesc; 1387 } 1388 1389 static int aead_encrypt(struct aead_request *req) 1390 { 1391 struct aead_edesc *edesc; 1392 struct crypto_aead *aead = crypto_aead_reqtfm(req); 1393 struct caam_ctx *ctx = crypto_aead_ctx(aead); 1394 struct device *jrdev = ctx->jrdev; 1395 bool all_contig; 1396 u32 *desc; 1397 int ret = 0; 1398 1399 req->cryptlen += ctx->authsize; 1400 1401 /* allocate extended descriptor */ 1402 edesc = aead_edesc_alloc(req, DESC_JOB_IO_LEN * 1403 CAAM_CMD_SZ, &all_contig); 1404 if (IS_ERR(edesc)) 1405 return PTR_ERR(edesc); 1406 1407 /* Create and submit job descriptor */ 1408 init_aead_job(ctx->sh_desc_enc, ctx->sh_desc_enc_dma, edesc, req, 1409 all_contig, true); 1410 #ifdef DEBUG 1411 print_hex_dump(KERN_ERR, "aead jobdesc@"xstr(__LINE__)": ", 1412 DUMP_PREFIX_ADDRESS, 16, 4, edesc->hw_desc, 1413 desc_bytes(edesc->hw_desc), 1); 1414 #endif 1415 1416 desc = edesc->hw_desc; 1417 ret = caam_jr_enqueue(jrdev, desc, aead_encrypt_done, req); 1418 if (!ret) { 1419 ret = -EINPROGRESS; 1420 } else { 1421 aead_unmap(jrdev, edesc, req); 1422 kfree(edesc); 1423 } 1424 1425 return ret; 1426 } 1427 1428 static int aead_decrypt(struct aead_request *req) 1429 { 1430 struct aead_edesc *edesc; 1431 struct crypto_aead *aead = crypto_aead_reqtfm(req); 1432 struct caam_ctx *ctx = crypto_aead_ctx(aead); 1433 struct device *jrdev = ctx->jrdev; 1434 bool all_contig; 1435 u32 *desc; 1436 int ret = 0; 1437 1438 /* allocate extended descriptor */ 1439 edesc = aead_edesc_alloc(req, DESC_JOB_IO_LEN * 1440 CAAM_CMD_SZ, &all_contig); 1441 if (IS_ERR(edesc)) 1442 return PTR_ERR(edesc); 1443 1444 #ifdef DEBUG 1445 print_hex_dump(KERN_ERR, "dec src@"xstr(__LINE__)": ", 1446 DUMP_PREFIX_ADDRESS, 16, 4, sg_virt(req->src), 1447 req->cryptlen, 1); 1448 #endif 1449 1450 /* Create and submit job descriptor*/ 1451 init_aead_job(ctx->sh_desc_dec, 1452 ctx->sh_desc_dec_dma, edesc, req, all_contig, false); 1453 #ifdef DEBUG 1454 print_hex_dump(KERN_ERR, "aead jobdesc@"xstr(__LINE__)": ", 1455 DUMP_PREFIX_ADDRESS, 16, 4, edesc->hw_desc, 1456 desc_bytes(edesc->hw_desc), 1); 1457 #endif 1458 1459 desc = edesc->hw_desc; 1460 ret = caam_jr_enqueue(jrdev, desc, aead_decrypt_done, req); 1461 if (!ret) { 1462 ret = -EINPROGRESS; 1463 } else { 1464 aead_unmap(jrdev, edesc, req); 1465 kfree(edesc); 1466 } 1467 1468 return ret; 1469 } 1470 1471 /* 1472 * allocate and map the aead extended descriptor for aead givencrypt 1473 */ 1474 static struct aead_edesc *aead_giv_edesc_alloc(struct aead_givcrypt_request 1475 *greq, int desc_bytes, 1476 u32 *contig_ptr) 1477 { 1478 struct aead_request *req = &greq->areq; 1479 struct crypto_aead *aead = crypto_aead_reqtfm(req); 1480 struct caam_ctx *ctx = crypto_aead_ctx(aead); 1481 struct device *jrdev = ctx->jrdev; 1482 gfp_t flags = (req->base.flags & (CRYPTO_TFM_REQ_MAY_BACKLOG | 1483 CRYPTO_TFM_REQ_MAY_SLEEP)) ? GFP_KERNEL : GFP_ATOMIC; 1484 int assoc_nents, src_nents, dst_nents = 0; 1485 struct aead_edesc *edesc; 1486 dma_addr_t iv_dma = 0; 1487 int sgc; 1488 u32 contig = GIV_SRC_CONTIG | GIV_DST_CONTIG; 1489 int ivsize = crypto_aead_ivsize(aead); 1490 int link_tbl_index, link_tbl_len = 0, link_tbl_bytes; 1491 1492 assoc_nents = sg_count(req->assoc, req->assoclen); 1493 src_nents = sg_count(req->src, req->cryptlen); 1494 1495 if (unlikely(req->dst != req->src)) 1496 dst_nents = sg_count(req->dst, req->cryptlen); 1497 1498 sgc = dma_map_sg(jrdev, req->assoc, assoc_nents ? : 1, 1499 DMA_BIDIRECTIONAL); 1500 if (likely(req->src == req->dst)) { 1501 sgc = dma_map_sg(jrdev, req->src, src_nents ? : 1, 1502 DMA_BIDIRECTIONAL); 1503 } else { 1504 sgc = dma_map_sg(jrdev, req->src, src_nents ? : 1, 1505 DMA_TO_DEVICE); 1506 sgc = dma_map_sg(jrdev, req->dst, dst_nents ? : 1, 1507 DMA_FROM_DEVICE); 1508 } 1509 1510 /* Check if data are contiguous */ 1511 iv_dma = dma_map_single(jrdev, greq->giv, ivsize, DMA_TO_DEVICE); 1512 if (assoc_nents || sg_dma_address(req->assoc) + req->assoclen != 1513 iv_dma || src_nents || iv_dma + ivsize != sg_dma_address(req->src)) 1514 contig &= ~GIV_SRC_CONTIG; 1515 if (dst_nents || iv_dma + ivsize != sg_dma_address(req->dst)) 1516 contig &= ~GIV_DST_CONTIG; 1517 if (unlikely(req->src != req->dst)) { 1518 dst_nents = dst_nents ? : 1; 1519 link_tbl_len += 1; 1520 } 1521 if (!(contig & GIV_SRC_CONTIG)) { 1522 assoc_nents = assoc_nents ? : 1; 1523 src_nents = src_nents ? : 1; 1524 link_tbl_len += assoc_nents + 1 + src_nents; 1525 if (likely(req->src == req->dst)) 1526 contig &= ~GIV_DST_CONTIG; 1527 } 1528 link_tbl_len += dst_nents; 1529 1530 link_tbl_bytes = link_tbl_len * sizeof(struct link_tbl_entry); 1531 1532 /* allocate space for base edesc and hw desc commands, link tables */ 1533 edesc = kmalloc(sizeof(struct aead_edesc) + desc_bytes + 1534 link_tbl_bytes, GFP_DMA | flags); 1535 if (!edesc) { 1536 dev_err(jrdev, "could not allocate extended descriptor\n"); 1537 return ERR_PTR(-ENOMEM); 1538 } 1539 1540 edesc->assoc_nents = assoc_nents; 1541 edesc->src_nents = src_nents; 1542 edesc->dst_nents = dst_nents; 1543 edesc->iv_dma = iv_dma; 1544 edesc->link_tbl_bytes = link_tbl_bytes; 1545 edesc->link_tbl = (void *)edesc + sizeof(struct aead_edesc) + 1546 desc_bytes; 1547 edesc->link_tbl_dma = dma_map_single(jrdev, edesc->link_tbl, 1548 link_tbl_bytes, DMA_TO_DEVICE); 1549 *contig_ptr = contig; 1550 1551 link_tbl_index = 0; 1552 if (!(contig & GIV_SRC_CONTIG)) { 1553 sg_to_link_tbl(req->assoc, assoc_nents, 1554 edesc->link_tbl + 1555 link_tbl_index, 0); 1556 link_tbl_index += assoc_nents; 1557 sg_to_link_tbl_one(edesc->link_tbl + link_tbl_index, 1558 iv_dma, ivsize, 0); 1559 link_tbl_index += 1; 1560 sg_to_link_tbl_last(req->src, src_nents, 1561 edesc->link_tbl + 1562 link_tbl_index, 0); 1563 link_tbl_index += src_nents; 1564 } 1565 if (unlikely(req->src != req->dst && !(contig & GIV_DST_CONTIG))) { 1566 sg_to_link_tbl_one(edesc->link_tbl + link_tbl_index, 1567 iv_dma, ivsize, 0); 1568 link_tbl_index += 1; 1569 sg_to_link_tbl_last(req->dst, dst_nents, 1570 edesc->link_tbl + link_tbl_index, 0); 1571 } 1572 1573 return edesc; 1574 } 1575 1576 static int aead_givencrypt(struct aead_givcrypt_request *areq) 1577 { 1578 struct aead_request *req = &areq->areq; 1579 struct aead_edesc *edesc; 1580 struct crypto_aead *aead = crypto_aead_reqtfm(req); 1581 struct caam_ctx *ctx = crypto_aead_ctx(aead); 1582 struct device *jrdev = ctx->jrdev; 1583 u32 contig; 1584 u32 *desc; 1585 int ret = 0; 1586 1587 req->cryptlen += ctx->authsize; 1588 1589 /* allocate extended descriptor */ 1590 edesc = aead_giv_edesc_alloc(areq, DESC_JOB_IO_LEN * 1591 CAAM_CMD_SZ, &contig); 1592 1593 if (IS_ERR(edesc)) 1594 return PTR_ERR(edesc); 1595 1596 #ifdef DEBUG 1597 print_hex_dump(KERN_ERR, "giv src@"xstr(__LINE__)": ", 1598 DUMP_PREFIX_ADDRESS, 16, 4, sg_virt(req->src), 1599 req->cryptlen, 1); 1600 #endif 1601 1602 /* Create and submit job descriptor*/ 1603 init_aead_giv_job(ctx->sh_desc_givenc, 1604 ctx->sh_desc_givenc_dma, edesc, req, contig); 1605 #ifdef DEBUG 1606 print_hex_dump(KERN_ERR, "aead jobdesc@"xstr(__LINE__)": ", 1607 DUMP_PREFIX_ADDRESS, 16, 4, edesc->hw_desc, 1608 desc_bytes(edesc->hw_desc), 1); 1609 #endif 1610 1611 desc = edesc->hw_desc; 1612 ret = caam_jr_enqueue(jrdev, desc, aead_encrypt_done, req); 1613 if (!ret) { 1614 ret = -EINPROGRESS; 1615 } else { 1616 aead_unmap(jrdev, edesc, req); 1617 kfree(edesc); 1618 } 1619 1620 return ret; 1621 } 1622 1623 /* 1624 * allocate and map the ablkcipher extended descriptor for ablkcipher 1625 */ 1626 static struct ablkcipher_edesc *ablkcipher_edesc_alloc(struct ablkcipher_request 1627 *req, int desc_bytes, 1628 bool *iv_contig_out) 1629 { 1630 struct crypto_ablkcipher *ablkcipher = crypto_ablkcipher_reqtfm(req); 1631 struct caam_ctx *ctx = crypto_ablkcipher_ctx(ablkcipher); 1632 struct device *jrdev = ctx->jrdev; 1633 gfp_t flags = (req->base.flags & (CRYPTO_TFM_REQ_MAY_BACKLOG | 1634 CRYPTO_TFM_REQ_MAY_SLEEP)) ? 1635 GFP_KERNEL : GFP_ATOMIC; 1636 int src_nents, dst_nents = 0, link_tbl_bytes; 1637 struct ablkcipher_edesc *edesc; 1638 dma_addr_t iv_dma = 0; 1639 bool iv_contig = false; 1640 int sgc; 1641 int ivsize = crypto_ablkcipher_ivsize(ablkcipher); 1642 int link_tbl_index; 1643 1644 src_nents = sg_count(req->src, req->nbytes); 1645 1646 if (unlikely(req->dst != req->src)) 1647 dst_nents = sg_count(req->dst, req->nbytes); 1648 1649 if (likely(req->src == req->dst)) { 1650 sgc = dma_map_sg(jrdev, req->src, src_nents ? : 1, 1651 DMA_BIDIRECTIONAL); 1652 } else { 1653 sgc = dma_map_sg(jrdev, req->src, src_nents ? : 1, 1654 DMA_TO_DEVICE); 1655 sgc = dma_map_sg(jrdev, req->dst, dst_nents ? : 1, 1656 DMA_FROM_DEVICE); 1657 } 1658 1659 /* 1660 * Check if iv can be contiguous with source and destination. 1661 * If so, include it. If not, create scatterlist. 1662 */ 1663 iv_dma = dma_map_single(jrdev, req->info, ivsize, DMA_TO_DEVICE); 1664 if (!src_nents && iv_dma + ivsize == sg_dma_address(req->src)) 1665 iv_contig = true; 1666 else 1667 src_nents = src_nents ? : 1; 1668 link_tbl_bytes = ((iv_contig ? 0 : 1) + src_nents + dst_nents) * 1669 sizeof(struct link_tbl_entry); 1670 1671 /* allocate space for base edesc and hw desc commands, link tables */ 1672 edesc = kmalloc(sizeof(struct ablkcipher_edesc) + desc_bytes + 1673 link_tbl_bytes, GFP_DMA | flags); 1674 if (!edesc) { 1675 dev_err(jrdev, "could not allocate extended descriptor\n"); 1676 return ERR_PTR(-ENOMEM); 1677 } 1678 1679 edesc->src_nents = src_nents; 1680 edesc->dst_nents = dst_nents; 1681 edesc->link_tbl_bytes = link_tbl_bytes; 1682 edesc->link_tbl = (void *)edesc + sizeof(struct ablkcipher_edesc) + 1683 desc_bytes; 1684 1685 link_tbl_index = 0; 1686 if (!iv_contig) { 1687 sg_to_link_tbl_one(edesc->link_tbl, iv_dma, ivsize, 0); 1688 sg_to_link_tbl_last(req->src, src_nents, 1689 edesc->link_tbl + 1, 0); 1690 link_tbl_index += 1 + src_nents; 1691 } 1692 1693 if (unlikely(dst_nents)) { 1694 sg_to_link_tbl_last(req->dst, dst_nents, 1695 edesc->link_tbl + link_tbl_index, 0); 1696 } 1697 1698 edesc->link_tbl_dma = dma_map_single(jrdev, edesc->link_tbl, 1699 link_tbl_bytes, DMA_TO_DEVICE); 1700 edesc->iv_dma = iv_dma; 1701 1702 #ifdef DEBUG 1703 print_hex_dump(KERN_ERR, "ablkcipher link_tbl@"xstr(__LINE__)": ", 1704 DUMP_PREFIX_ADDRESS, 16, 4, edesc->link_tbl, 1705 link_tbl_bytes, 1); 1706 #endif 1707 1708 *iv_contig_out = iv_contig; 1709 return edesc; 1710 } 1711 1712 static int ablkcipher_encrypt(struct ablkcipher_request *req) 1713 { 1714 struct ablkcipher_edesc *edesc; 1715 struct crypto_ablkcipher *ablkcipher = crypto_ablkcipher_reqtfm(req); 1716 struct caam_ctx *ctx = crypto_ablkcipher_ctx(ablkcipher); 1717 struct device *jrdev = ctx->jrdev; 1718 bool iv_contig; 1719 u32 *desc; 1720 int ret = 0; 1721 1722 /* allocate extended descriptor */ 1723 edesc = ablkcipher_edesc_alloc(req, DESC_JOB_IO_LEN * 1724 CAAM_CMD_SZ, &iv_contig); 1725 if (IS_ERR(edesc)) 1726 return PTR_ERR(edesc); 1727 1728 /* Create and submit job descriptor*/ 1729 init_ablkcipher_job(ctx->sh_desc_enc, 1730 ctx->sh_desc_enc_dma, edesc, req, iv_contig); 1731 #ifdef DEBUG 1732 print_hex_dump(KERN_ERR, "ablkcipher jobdesc@"xstr(__LINE__)": ", 1733 DUMP_PREFIX_ADDRESS, 16, 4, edesc->hw_desc, 1734 desc_bytes(edesc->hw_desc), 1); 1735 #endif 1736 desc = edesc->hw_desc; 1737 ret = caam_jr_enqueue(jrdev, desc, ablkcipher_encrypt_done, req); 1738 1739 if (!ret) { 1740 ret = -EINPROGRESS; 1741 } else { 1742 ablkcipher_unmap(jrdev, edesc, req); 1743 kfree(edesc); 1744 } 1745 1746 return ret; 1747 } 1748 1749 static int ablkcipher_decrypt(struct ablkcipher_request *req) 1750 { 1751 struct ablkcipher_edesc *edesc; 1752 struct crypto_ablkcipher *ablkcipher = crypto_ablkcipher_reqtfm(req); 1753 struct caam_ctx *ctx = crypto_ablkcipher_ctx(ablkcipher); 1754 struct device *jrdev = ctx->jrdev; 1755 bool iv_contig; 1756 u32 *desc; 1757 int ret = 0; 1758 1759 /* allocate extended descriptor */ 1760 edesc = ablkcipher_edesc_alloc(req, DESC_JOB_IO_LEN * 1761 CAAM_CMD_SZ, &iv_contig); 1762 if (IS_ERR(edesc)) 1763 return PTR_ERR(edesc); 1764 1765 /* Create and submit job descriptor*/ 1766 init_ablkcipher_job(ctx->sh_desc_dec, 1767 ctx->sh_desc_dec_dma, edesc, req, iv_contig); 1768 desc = edesc->hw_desc; 1769 #ifdef DEBUG 1770 print_hex_dump(KERN_ERR, "ablkcipher jobdesc@"xstr(__LINE__)": ", 1771 DUMP_PREFIX_ADDRESS, 16, 4, edesc->hw_desc, 1772 desc_bytes(edesc->hw_desc), 1); 1773 #endif 1774 1775 ret = caam_jr_enqueue(jrdev, desc, ablkcipher_decrypt_done, req); 1776 if (!ret) { 1777 ret = -EINPROGRESS; 1778 } else { 1779 ablkcipher_unmap(jrdev, edesc, req); 1780 kfree(edesc); 1781 } 1782 1783 return ret; 1784 } 1785 1786 #define template_aead template_u.aead 1787 #define template_ablkcipher template_u.ablkcipher 1788 struct caam_alg_template { 1789 char name[CRYPTO_MAX_ALG_NAME]; 1790 char driver_name[CRYPTO_MAX_ALG_NAME]; 1791 unsigned int blocksize; 1792 u32 type; 1793 union { 1794 struct ablkcipher_alg ablkcipher; 1795 struct aead_alg aead; 1796 struct blkcipher_alg blkcipher; 1797 struct cipher_alg cipher; 1798 struct compress_alg compress; 1799 struct rng_alg rng; 1800 } template_u; 1801 u32 class1_alg_type; 1802 u32 class2_alg_type; 1803 u32 alg_op; 1804 }; 1805 1806 static struct caam_alg_template driver_algs[] = { 1807 /* single-pass ipsec_esp descriptor */ 1808 { 1809 .name = "authenc(hmac(md5),cbc(aes))", 1810 .driver_name = "authenc-hmac-md5-cbc-aes-caam", 1811 .blocksize = AES_BLOCK_SIZE, 1812 .type = CRYPTO_ALG_TYPE_AEAD, 1813 .template_aead = { 1814 .setkey = aead_setkey, 1815 .setauthsize = aead_setauthsize, 1816 .encrypt = aead_encrypt, 1817 .decrypt = aead_decrypt, 1818 .givencrypt = aead_givencrypt, 1819 .geniv = "<built-in>", 1820 .ivsize = AES_BLOCK_SIZE, 1821 .maxauthsize = MD5_DIGEST_SIZE, 1822 }, 1823 .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CBC, 1824 .class2_alg_type = OP_ALG_ALGSEL_MD5 | OP_ALG_AAI_HMAC_PRECOMP, 1825 .alg_op = OP_ALG_ALGSEL_MD5 | OP_ALG_AAI_HMAC, 1826 }, 1827 { 1828 .name = "authenc(hmac(sha1),cbc(aes))", 1829 .driver_name = "authenc-hmac-sha1-cbc-aes-caam", 1830 .blocksize = AES_BLOCK_SIZE, 1831 .type = CRYPTO_ALG_TYPE_AEAD, 1832 .template_aead = { 1833 .setkey = aead_setkey, 1834 .setauthsize = aead_setauthsize, 1835 .encrypt = aead_encrypt, 1836 .decrypt = aead_decrypt, 1837 .givencrypt = aead_givencrypt, 1838 .geniv = "<built-in>", 1839 .ivsize = AES_BLOCK_SIZE, 1840 .maxauthsize = SHA1_DIGEST_SIZE, 1841 }, 1842 .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CBC, 1843 .class2_alg_type = OP_ALG_ALGSEL_SHA1 | OP_ALG_AAI_HMAC_PRECOMP, 1844 .alg_op = OP_ALG_ALGSEL_SHA1 | OP_ALG_AAI_HMAC, 1845 }, 1846 { 1847 .name = "authenc(hmac(sha256),cbc(aes))", 1848 .driver_name = "authenc-hmac-sha256-cbc-aes-caam", 1849 .blocksize = AES_BLOCK_SIZE, 1850 .type = CRYPTO_ALG_TYPE_AEAD, 1851 .template_aead = { 1852 .setkey = aead_setkey, 1853 .setauthsize = aead_setauthsize, 1854 .encrypt = aead_encrypt, 1855 .decrypt = aead_decrypt, 1856 .givencrypt = aead_givencrypt, 1857 .geniv = "<built-in>", 1858 .ivsize = AES_BLOCK_SIZE, 1859 .maxauthsize = SHA256_DIGEST_SIZE, 1860 }, 1861 .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CBC, 1862 .class2_alg_type = OP_ALG_ALGSEL_SHA256 | 1863 OP_ALG_AAI_HMAC_PRECOMP, 1864 .alg_op = OP_ALG_ALGSEL_SHA256 | OP_ALG_AAI_HMAC, 1865 }, 1866 { 1867 .name = "authenc(hmac(sha512),cbc(aes))", 1868 .driver_name = "authenc-hmac-sha512-cbc-aes-caam", 1869 .blocksize = AES_BLOCK_SIZE, 1870 .type = CRYPTO_ALG_TYPE_AEAD, 1871 .template_aead = { 1872 .setkey = aead_setkey, 1873 .setauthsize = aead_setauthsize, 1874 .encrypt = aead_encrypt, 1875 .decrypt = aead_decrypt, 1876 .givencrypt = aead_givencrypt, 1877 .geniv = "<built-in>", 1878 .ivsize = AES_BLOCK_SIZE, 1879 .maxauthsize = SHA512_DIGEST_SIZE, 1880 }, 1881 .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CBC, 1882 .class2_alg_type = OP_ALG_ALGSEL_SHA512 | 1883 OP_ALG_AAI_HMAC_PRECOMP, 1884 .alg_op = OP_ALG_ALGSEL_SHA512 | OP_ALG_AAI_HMAC, 1885 }, 1886 { 1887 .name = "authenc(hmac(md5),cbc(des3_ede))", 1888 .driver_name = "authenc-hmac-md5-cbc-des3_ede-caam", 1889 .blocksize = DES3_EDE_BLOCK_SIZE, 1890 .type = CRYPTO_ALG_TYPE_AEAD, 1891 .template_aead = { 1892 .setkey = aead_setkey, 1893 .setauthsize = aead_setauthsize, 1894 .encrypt = aead_encrypt, 1895 .decrypt = aead_decrypt, 1896 .givencrypt = aead_givencrypt, 1897 .geniv = "<built-in>", 1898 .ivsize = DES3_EDE_BLOCK_SIZE, 1899 .maxauthsize = MD5_DIGEST_SIZE, 1900 }, 1901 .class1_alg_type = OP_ALG_ALGSEL_3DES | OP_ALG_AAI_CBC, 1902 .class2_alg_type = OP_ALG_ALGSEL_MD5 | OP_ALG_AAI_HMAC_PRECOMP, 1903 .alg_op = OP_ALG_ALGSEL_MD5 | OP_ALG_AAI_HMAC, 1904 }, 1905 { 1906 .name = "authenc(hmac(sha1),cbc(des3_ede))", 1907 .driver_name = "authenc-hmac-sha1-cbc-des3_ede-caam", 1908 .blocksize = DES3_EDE_BLOCK_SIZE, 1909 .type = CRYPTO_ALG_TYPE_AEAD, 1910 .template_aead = { 1911 .setkey = aead_setkey, 1912 .setauthsize = aead_setauthsize, 1913 .encrypt = aead_encrypt, 1914 .decrypt = aead_decrypt, 1915 .givencrypt = aead_givencrypt, 1916 .geniv = "<built-in>", 1917 .ivsize = DES3_EDE_BLOCK_SIZE, 1918 .maxauthsize = SHA1_DIGEST_SIZE, 1919 }, 1920 .class1_alg_type = OP_ALG_ALGSEL_3DES | OP_ALG_AAI_CBC, 1921 .class2_alg_type = OP_ALG_ALGSEL_SHA1 | OP_ALG_AAI_HMAC_PRECOMP, 1922 .alg_op = OP_ALG_ALGSEL_SHA1 | OP_ALG_AAI_HMAC, 1923 }, 1924 { 1925 .name = "authenc(hmac(sha256),cbc(des3_ede))", 1926 .driver_name = "authenc-hmac-sha256-cbc-des3_ede-caam", 1927 .blocksize = DES3_EDE_BLOCK_SIZE, 1928 .type = CRYPTO_ALG_TYPE_AEAD, 1929 .template_aead = { 1930 .setkey = aead_setkey, 1931 .setauthsize = aead_setauthsize, 1932 .encrypt = aead_encrypt, 1933 .decrypt = aead_decrypt, 1934 .givencrypt = aead_givencrypt, 1935 .geniv = "<built-in>", 1936 .ivsize = DES3_EDE_BLOCK_SIZE, 1937 .maxauthsize = SHA256_DIGEST_SIZE, 1938 }, 1939 .class1_alg_type = OP_ALG_ALGSEL_3DES | OP_ALG_AAI_CBC, 1940 .class2_alg_type = OP_ALG_ALGSEL_SHA256 | 1941 OP_ALG_AAI_HMAC_PRECOMP, 1942 .alg_op = OP_ALG_ALGSEL_SHA256 | OP_ALG_AAI_HMAC, 1943 }, 1944 { 1945 .name = "authenc(hmac(sha512),cbc(des3_ede))", 1946 .driver_name = "authenc-hmac-sha512-cbc-des3_ede-caam", 1947 .blocksize = DES3_EDE_BLOCK_SIZE, 1948 .type = CRYPTO_ALG_TYPE_AEAD, 1949 .template_aead = { 1950 .setkey = aead_setkey, 1951 .setauthsize = aead_setauthsize, 1952 .encrypt = aead_encrypt, 1953 .decrypt = aead_decrypt, 1954 .givencrypt = aead_givencrypt, 1955 .geniv = "<built-in>", 1956 .ivsize = DES3_EDE_BLOCK_SIZE, 1957 .maxauthsize = SHA512_DIGEST_SIZE, 1958 }, 1959 .class1_alg_type = OP_ALG_ALGSEL_3DES | OP_ALG_AAI_CBC, 1960 .class2_alg_type = OP_ALG_ALGSEL_SHA512 | 1961 OP_ALG_AAI_HMAC_PRECOMP, 1962 .alg_op = OP_ALG_ALGSEL_SHA512 | OP_ALG_AAI_HMAC, 1963 }, 1964 { 1965 .name = "authenc(hmac(md5),cbc(des))", 1966 .driver_name = "authenc-hmac-md5-cbc-des-caam", 1967 .blocksize = DES_BLOCK_SIZE, 1968 .type = CRYPTO_ALG_TYPE_AEAD, 1969 .template_aead = { 1970 .setkey = aead_setkey, 1971 .setauthsize = aead_setauthsize, 1972 .encrypt = aead_encrypt, 1973 .decrypt = aead_decrypt, 1974 .givencrypt = aead_givencrypt, 1975 .geniv = "<built-in>", 1976 .ivsize = DES_BLOCK_SIZE, 1977 .maxauthsize = MD5_DIGEST_SIZE, 1978 }, 1979 .class1_alg_type = OP_ALG_ALGSEL_DES | OP_ALG_AAI_CBC, 1980 .class2_alg_type = OP_ALG_ALGSEL_MD5 | OP_ALG_AAI_HMAC_PRECOMP, 1981 .alg_op = OP_ALG_ALGSEL_MD5 | OP_ALG_AAI_HMAC, 1982 }, 1983 { 1984 .name = "authenc(hmac(sha1),cbc(des))", 1985 .driver_name = "authenc-hmac-sha1-cbc-des-caam", 1986 .blocksize = DES_BLOCK_SIZE, 1987 .type = CRYPTO_ALG_TYPE_AEAD, 1988 .template_aead = { 1989 .setkey = aead_setkey, 1990 .setauthsize = aead_setauthsize, 1991 .encrypt = aead_encrypt, 1992 .decrypt = aead_decrypt, 1993 .givencrypt = aead_givencrypt, 1994 .geniv = "<built-in>", 1995 .ivsize = DES_BLOCK_SIZE, 1996 .maxauthsize = SHA1_DIGEST_SIZE, 1997 }, 1998 .class1_alg_type = OP_ALG_ALGSEL_DES | OP_ALG_AAI_CBC, 1999 .class2_alg_type = OP_ALG_ALGSEL_SHA1 | OP_ALG_AAI_HMAC_PRECOMP, 2000 .alg_op = OP_ALG_ALGSEL_SHA1 | OP_ALG_AAI_HMAC, 2001 }, 2002 { 2003 .name = "authenc(hmac(sha256),cbc(des))", 2004 .driver_name = "authenc-hmac-sha256-cbc-des-caam", 2005 .blocksize = DES_BLOCK_SIZE, 2006 .type = CRYPTO_ALG_TYPE_AEAD, 2007 .template_aead = { 2008 .setkey = aead_setkey, 2009 .setauthsize = aead_setauthsize, 2010 .encrypt = aead_encrypt, 2011 .decrypt = aead_decrypt, 2012 .givencrypt = aead_givencrypt, 2013 .geniv = "<built-in>", 2014 .ivsize = DES_BLOCK_SIZE, 2015 .maxauthsize = SHA256_DIGEST_SIZE, 2016 }, 2017 .class1_alg_type = OP_ALG_ALGSEL_DES | OP_ALG_AAI_CBC, 2018 .class2_alg_type = OP_ALG_ALGSEL_SHA256 | 2019 OP_ALG_AAI_HMAC_PRECOMP, 2020 .alg_op = OP_ALG_ALGSEL_SHA256 | OP_ALG_AAI_HMAC, 2021 }, 2022 { 2023 .name = "authenc(hmac(sha512),cbc(des))", 2024 .driver_name = "authenc-hmac-sha512-cbc-des-caam", 2025 .blocksize = DES_BLOCK_SIZE, 2026 .type = CRYPTO_ALG_TYPE_AEAD, 2027 .template_aead = { 2028 .setkey = aead_setkey, 2029 .setauthsize = aead_setauthsize, 2030 .encrypt = aead_encrypt, 2031 .decrypt = aead_decrypt, 2032 .givencrypt = aead_givencrypt, 2033 .geniv = "<built-in>", 2034 .ivsize = DES_BLOCK_SIZE, 2035 .maxauthsize = SHA512_DIGEST_SIZE, 2036 }, 2037 .class1_alg_type = OP_ALG_ALGSEL_DES | OP_ALG_AAI_CBC, 2038 .class2_alg_type = OP_ALG_ALGSEL_SHA512 | 2039 OP_ALG_AAI_HMAC_PRECOMP, 2040 .alg_op = OP_ALG_ALGSEL_SHA512 | OP_ALG_AAI_HMAC, 2041 }, 2042 /* ablkcipher descriptor */ 2043 { 2044 .name = "cbc(aes)", 2045 .driver_name = "cbc-aes-caam", 2046 .blocksize = AES_BLOCK_SIZE, 2047 .type = CRYPTO_ALG_TYPE_ABLKCIPHER, 2048 .template_ablkcipher = { 2049 .setkey = ablkcipher_setkey, 2050 .encrypt = ablkcipher_encrypt, 2051 .decrypt = ablkcipher_decrypt, 2052 .geniv = "eseqiv", 2053 .min_keysize = AES_MIN_KEY_SIZE, 2054 .max_keysize = AES_MAX_KEY_SIZE, 2055 .ivsize = AES_BLOCK_SIZE, 2056 }, 2057 .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CBC, 2058 }, 2059 { 2060 .name = "cbc(des3_ede)", 2061 .driver_name = "cbc-3des-caam", 2062 .blocksize = DES3_EDE_BLOCK_SIZE, 2063 .type = CRYPTO_ALG_TYPE_ABLKCIPHER, 2064 .template_ablkcipher = { 2065 .setkey = ablkcipher_setkey, 2066 .encrypt = ablkcipher_encrypt, 2067 .decrypt = ablkcipher_decrypt, 2068 .geniv = "eseqiv", 2069 .min_keysize = DES3_EDE_KEY_SIZE, 2070 .max_keysize = DES3_EDE_KEY_SIZE, 2071 .ivsize = DES3_EDE_BLOCK_SIZE, 2072 }, 2073 .class1_alg_type = OP_ALG_ALGSEL_3DES | OP_ALG_AAI_CBC, 2074 }, 2075 { 2076 .name = "cbc(des)", 2077 .driver_name = "cbc-des-caam", 2078 .blocksize = DES_BLOCK_SIZE, 2079 .type = CRYPTO_ALG_TYPE_ABLKCIPHER, 2080 .template_ablkcipher = { 2081 .setkey = ablkcipher_setkey, 2082 .encrypt = ablkcipher_encrypt, 2083 .decrypt = ablkcipher_decrypt, 2084 .geniv = "eseqiv", 2085 .min_keysize = DES_KEY_SIZE, 2086 .max_keysize = DES_KEY_SIZE, 2087 .ivsize = DES_BLOCK_SIZE, 2088 }, 2089 .class1_alg_type = OP_ALG_ALGSEL_DES | OP_ALG_AAI_CBC, 2090 } 2091 }; 2092 2093 struct caam_crypto_alg { 2094 struct list_head entry; 2095 struct device *ctrldev; 2096 int class1_alg_type; 2097 int class2_alg_type; 2098 int alg_op; 2099 struct crypto_alg crypto_alg; 2100 }; 2101 2102 static int caam_cra_init(struct crypto_tfm *tfm) 2103 { 2104 struct crypto_alg *alg = tfm->__crt_alg; 2105 struct caam_crypto_alg *caam_alg = 2106 container_of(alg, struct caam_crypto_alg, crypto_alg); 2107 struct caam_ctx *ctx = crypto_tfm_ctx(tfm); 2108 struct caam_drv_private *priv = dev_get_drvdata(caam_alg->ctrldev); 2109 int tgt_jr = atomic_inc_return(&priv->tfm_count); 2110 2111 /* 2112 * distribute tfms across job rings to ensure in-order 2113 * crypto request processing per tfm 2114 */ 2115 ctx->jrdev = priv->algapi_jr[(tgt_jr / 2) % priv->num_jrs_for_algapi]; 2116 2117 /* copy descriptor header template value */ 2118 ctx->class1_alg_type = OP_TYPE_CLASS1_ALG | caam_alg->class1_alg_type; 2119 ctx->class2_alg_type = OP_TYPE_CLASS2_ALG | caam_alg->class2_alg_type; 2120 ctx->alg_op = OP_TYPE_CLASS2_ALG | caam_alg->alg_op; 2121 2122 return 0; 2123 } 2124 2125 static void caam_cra_exit(struct crypto_tfm *tfm) 2126 { 2127 struct caam_ctx *ctx = crypto_tfm_ctx(tfm); 2128 2129 if (ctx->sh_desc_enc_dma && 2130 !dma_mapping_error(ctx->jrdev, ctx->sh_desc_enc_dma)) 2131 dma_unmap_single(ctx->jrdev, ctx->sh_desc_enc_dma, 2132 desc_bytes(ctx->sh_desc_enc), DMA_TO_DEVICE); 2133 if (ctx->sh_desc_dec_dma && 2134 !dma_mapping_error(ctx->jrdev, ctx->sh_desc_dec_dma)) 2135 dma_unmap_single(ctx->jrdev, ctx->sh_desc_dec_dma, 2136 desc_bytes(ctx->sh_desc_dec), DMA_TO_DEVICE); 2137 if (ctx->sh_desc_givenc_dma && 2138 !dma_mapping_error(ctx->jrdev, ctx->sh_desc_givenc_dma)) 2139 dma_unmap_single(ctx->jrdev, ctx->sh_desc_givenc_dma, 2140 desc_bytes(ctx->sh_desc_givenc), 2141 DMA_TO_DEVICE); 2142 } 2143 2144 static void __exit caam_algapi_exit(void) 2145 { 2146 2147 struct device_node *dev_node; 2148 struct platform_device *pdev; 2149 struct device *ctrldev; 2150 struct caam_drv_private *priv; 2151 struct caam_crypto_alg *t_alg, *n; 2152 int i, err; 2153 2154 dev_node = of_find_compatible_node(NULL, NULL, "fsl,sec-v4.0"); 2155 if (!dev_node) 2156 return; 2157 2158 pdev = of_find_device_by_node(dev_node); 2159 if (!pdev) 2160 return; 2161 2162 ctrldev = &pdev->dev; 2163 of_node_put(dev_node); 2164 priv = dev_get_drvdata(ctrldev); 2165 2166 if (!priv->alg_list.next) 2167 return; 2168 2169 list_for_each_entry_safe(t_alg, n, &priv->alg_list, entry) { 2170 crypto_unregister_alg(&t_alg->crypto_alg); 2171 list_del(&t_alg->entry); 2172 kfree(t_alg); 2173 } 2174 2175 for (i = 0; i < priv->total_jobrs; i++) { 2176 err = caam_jr_deregister(priv->algapi_jr[i]); 2177 if (err < 0) 2178 break; 2179 } 2180 kfree(priv->algapi_jr); 2181 } 2182 2183 static struct caam_crypto_alg *caam_alg_alloc(struct device *ctrldev, 2184 struct caam_alg_template 2185 *template) 2186 { 2187 struct caam_crypto_alg *t_alg; 2188 struct crypto_alg *alg; 2189 2190 t_alg = kzalloc(sizeof(struct caam_crypto_alg), GFP_KERNEL); 2191 if (!t_alg) { 2192 dev_err(ctrldev, "failed to allocate t_alg\n"); 2193 return ERR_PTR(-ENOMEM); 2194 } 2195 2196 alg = &t_alg->crypto_alg; 2197 2198 snprintf(alg->cra_name, CRYPTO_MAX_ALG_NAME, "%s", template->name); 2199 snprintf(alg->cra_driver_name, CRYPTO_MAX_ALG_NAME, "%s", 2200 template->driver_name); 2201 alg->cra_module = THIS_MODULE; 2202 alg->cra_init = caam_cra_init; 2203 alg->cra_exit = caam_cra_exit; 2204 alg->cra_priority = CAAM_CRA_PRIORITY; 2205 alg->cra_blocksize = template->blocksize; 2206 alg->cra_alignmask = 0; 2207 alg->cra_ctxsize = sizeof(struct caam_ctx); 2208 alg->cra_flags = CRYPTO_ALG_ASYNC | template->type; 2209 switch (template->type) { 2210 case CRYPTO_ALG_TYPE_ABLKCIPHER: 2211 alg->cra_type = &crypto_ablkcipher_type; 2212 alg->cra_ablkcipher = template->template_ablkcipher; 2213 break; 2214 case CRYPTO_ALG_TYPE_AEAD: 2215 alg->cra_type = &crypto_aead_type; 2216 alg->cra_aead = template->template_aead; 2217 break; 2218 } 2219 2220 t_alg->class1_alg_type = template->class1_alg_type; 2221 t_alg->class2_alg_type = template->class2_alg_type; 2222 t_alg->alg_op = template->alg_op; 2223 t_alg->ctrldev = ctrldev; 2224 2225 return t_alg; 2226 } 2227 2228 static int __init caam_algapi_init(void) 2229 { 2230 struct device_node *dev_node; 2231 struct platform_device *pdev; 2232 struct device *ctrldev, **jrdev; 2233 struct caam_drv_private *priv; 2234 int i = 0, err = 0; 2235 2236 dev_node = of_find_compatible_node(NULL, NULL, "fsl,sec-v4.0"); 2237 if (!dev_node) 2238 return -ENODEV; 2239 2240 pdev = of_find_device_by_node(dev_node); 2241 if (!pdev) 2242 return -ENODEV; 2243 2244 ctrldev = &pdev->dev; 2245 priv = dev_get_drvdata(ctrldev); 2246 of_node_put(dev_node); 2247 2248 INIT_LIST_HEAD(&priv->alg_list); 2249 2250 jrdev = kmalloc(sizeof(*jrdev) * priv->total_jobrs, GFP_KERNEL); 2251 if (!jrdev) 2252 return -ENOMEM; 2253 2254 for (i = 0; i < priv->total_jobrs; i++) { 2255 err = caam_jr_register(ctrldev, &jrdev[i]); 2256 if (err < 0) 2257 break; 2258 } 2259 if (err < 0 && i == 0) { 2260 dev_err(ctrldev, "algapi error in job ring registration: %d\n", 2261 err); 2262 kfree(jrdev); 2263 return err; 2264 } 2265 2266 priv->num_jrs_for_algapi = i; 2267 priv->algapi_jr = jrdev; 2268 atomic_set(&priv->tfm_count, -1); 2269 2270 /* register crypto algorithms the device supports */ 2271 for (i = 0; i < ARRAY_SIZE(driver_algs); i++) { 2272 /* TODO: check if h/w supports alg */ 2273 struct caam_crypto_alg *t_alg; 2274 2275 t_alg = caam_alg_alloc(ctrldev, &driver_algs[i]); 2276 if (IS_ERR(t_alg)) { 2277 err = PTR_ERR(t_alg); 2278 dev_warn(ctrldev, "%s alg allocation failed\n", 2279 driver_algs[i].driver_name); 2280 continue; 2281 } 2282 2283 err = crypto_register_alg(&t_alg->crypto_alg); 2284 if (err) { 2285 dev_warn(ctrldev, "%s alg registration failed\n", 2286 t_alg->crypto_alg.cra_driver_name); 2287 kfree(t_alg); 2288 } else { 2289 list_add_tail(&t_alg->entry, &priv->alg_list); 2290 dev_info(ctrldev, "%s\n", 2291 t_alg->crypto_alg.cra_driver_name); 2292 } 2293 } 2294 2295 return err; 2296 } 2297 2298 module_init(caam_algapi_init); 2299 module_exit(caam_algapi_exit); 2300 2301 MODULE_LICENSE("GPL"); 2302 MODULE_DESCRIPTION("FSL CAAM support for crypto API"); 2303 MODULE_AUTHOR("Freescale Semiconductor - NMG/STC"); 2304