1 /* 2 * caam - Freescale FSL CAAM support for ahash functions of crypto API 3 * 4 * Copyright 2011 Freescale Semiconductor, Inc. 5 * 6 * Based on caamalg.c crypto API driver. 7 * 8 * relationship of digest job descriptor or first job descriptor after init to 9 * shared descriptors: 10 * 11 * --------------- --------------- 12 * | JobDesc #1 |-------------------->| ShareDesc | 13 * | *(packet 1) | | (hashKey) | 14 * --------------- | (operation) | 15 * --------------- 16 * 17 * relationship of subsequent job descriptors to shared descriptors: 18 * 19 * --------------- --------------- 20 * | JobDesc #2 |-------------------->| ShareDesc | 21 * | *(packet 2) | |------------->| (hashKey) | 22 * --------------- | |-------->| (operation) | 23 * . | | | (load ctx2) | 24 * . | | --------------- 25 * --------------- | | 26 * | JobDesc #3 |------| | 27 * | *(packet 3) | | 28 * --------------- | 29 * . | 30 * . | 31 * --------------- | 32 * | JobDesc #4 |------------ 33 * | *(packet 4) | 34 * --------------- 35 * 36 * The SharedDesc never changes for a connection unless rekeyed, but 37 * each packet will likely be in a different place. So all we need 38 * to know to process the packet is where the input is, where the 39 * output goes, and what context we want to process with. Context is 40 * in the SharedDesc, packet references in the JobDesc. 41 * 42 * So, a job desc looks like: 43 * 44 * --------------------- 45 * | Header | 46 * | ShareDesc Pointer | 47 * | SEQ_OUT_PTR | 48 * | (output buffer) | 49 * | (output length) | 50 * | SEQ_IN_PTR | 51 * | (input buffer) | 52 * | (input length) | 53 * --------------------- 54 */ 55 56 #include "compat.h" 57 58 #include "regs.h" 59 #include "intern.h" 60 #include "desc_constr.h" 61 #include "jr.h" 62 #include "error.h" 63 #include "sg_sw_sec4.h" 64 #include "key_gen.h" 65 66 #define CAAM_CRA_PRIORITY 3000 67 68 /* max hash key is max split key size */ 69 #define CAAM_MAX_HASH_KEY_SIZE (SHA512_DIGEST_SIZE * 2) 70 71 #define CAAM_MAX_HASH_BLOCK_SIZE SHA512_BLOCK_SIZE 72 #define CAAM_MAX_HASH_DIGEST_SIZE SHA512_DIGEST_SIZE 73 74 /* length of descriptors text */ 75 #define DESC_AHASH_BASE (4 * CAAM_CMD_SZ) 76 #define DESC_AHASH_UPDATE_LEN (6 * CAAM_CMD_SZ) 77 #define DESC_AHASH_UPDATE_FIRST_LEN (DESC_AHASH_BASE + 4 * CAAM_CMD_SZ) 78 #define DESC_AHASH_FINAL_LEN (DESC_AHASH_BASE + 5 * CAAM_CMD_SZ) 79 #define DESC_AHASH_FINUP_LEN (DESC_AHASH_BASE + 5 * CAAM_CMD_SZ) 80 #define DESC_AHASH_DIGEST_LEN (DESC_AHASH_BASE + 4 * CAAM_CMD_SZ) 81 82 #define DESC_HASH_MAX_USED_BYTES (DESC_AHASH_FINAL_LEN + \ 83 CAAM_MAX_HASH_KEY_SIZE) 84 #define DESC_HASH_MAX_USED_LEN (DESC_HASH_MAX_USED_BYTES / CAAM_CMD_SZ) 85 86 /* caam context sizes for hashes: running digest + 8 */ 87 #define HASH_MSG_LEN 8 88 #define MAX_CTX_LEN (HASH_MSG_LEN + SHA512_DIGEST_SIZE) 89 90 #ifdef DEBUG 91 /* for print_hex_dumps with line references */ 92 #define debug(format, arg...) printk(format, arg) 93 #else 94 #define debug(format, arg...) 95 #endif 96 97 98 static struct list_head hash_list; 99 100 /* ahash per-session context */ 101 struct caam_hash_ctx { 102 struct device *jrdev; 103 u32 sh_desc_update[DESC_HASH_MAX_USED_LEN]; 104 u32 sh_desc_update_first[DESC_HASH_MAX_USED_LEN]; 105 u32 sh_desc_fin[DESC_HASH_MAX_USED_LEN]; 106 u32 sh_desc_digest[DESC_HASH_MAX_USED_LEN]; 107 u32 sh_desc_finup[DESC_HASH_MAX_USED_LEN]; 108 dma_addr_t sh_desc_update_dma; 109 dma_addr_t sh_desc_update_first_dma; 110 dma_addr_t sh_desc_fin_dma; 111 dma_addr_t sh_desc_digest_dma; 112 dma_addr_t sh_desc_finup_dma; 113 u32 alg_type; 114 u32 alg_op; 115 u8 key[CAAM_MAX_HASH_KEY_SIZE]; 116 dma_addr_t key_dma; 117 int ctx_len; 118 unsigned int split_key_len; 119 unsigned int split_key_pad_len; 120 }; 121 122 /* ahash state */ 123 struct caam_hash_state { 124 dma_addr_t buf_dma; 125 dma_addr_t ctx_dma; 126 u8 buf_0[CAAM_MAX_HASH_BLOCK_SIZE] ____cacheline_aligned; 127 int buflen_0; 128 u8 buf_1[CAAM_MAX_HASH_BLOCK_SIZE] ____cacheline_aligned; 129 int buflen_1; 130 u8 caam_ctx[MAX_CTX_LEN] ____cacheline_aligned; 131 int (*update)(struct ahash_request *req); 132 int (*final)(struct ahash_request *req); 133 int (*finup)(struct ahash_request *req); 134 int current_buf; 135 }; 136 137 struct caam_export_state { 138 u8 buf[CAAM_MAX_HASH_BLOCK_SIZE]; 139 u8 caam_ctx[MAX_CTX_LEN]; 140 int buflen; 141 int (*update)(struct ahash_request *req); 142 int (*final)(struct ahash_request *req); 143 int (*finup)(struct ahash_request *req); 144 }; 145 146 /* Common job descriptor seq in/out ptr routines */ 147 148 /* Map state->caam_ctx, and append seq_out_ptr command that points to it */ 149 static inline int map_seq_out_ptr_ctx(u32 *desc, struct device *jrdev, 150 struct caam_hash_state *state, 151 int ctx_len) 152 { 153 state->ctx_dma = dma_map_single(jrdev, state->caam_ctx, 154 ctx_len, DMA_FROM_DEVICE); 155 if (dma_mapping_error(jrdev, state->ctx_dma)) { 156 dev_err(jrdev, "unable to map ctx\n"); 157 return -ENOMEM; 158 } 159 160 append_seq_out_ptr(desc, state->ctx_dma, ctx_len, 0); 161 162 return 0; 163 } 164 165 /* Map req->result, and append seq_out_ptr command that points to it */ 166 static inline dma_addr_t map_seq_out_ptr_result(u32 *desc, struct device *jrdev, 167 u8 *result, int digestsize) 168 { 169 dma_addr_t dst_dma; 170 171 dst_dma = dma_map_single(jrdev, result, digestsize, DMA_FROM_DEVICE); 172 append_seq_out_ptr(desc, dst_dma, digestsize, 0); 173 174 return dst_dma; 175 } 176 177 /* Map current buffer in state and put it in link table */ 178 static inline dma_addr_t buf_map_to_sec4_sg(struct device *jrdev, 179 struct sec4_sg_entry *sec4_sg, 180 u8 *buf, int buflen) 181 { 182 dma_addr_t buf_dma; 183 184 buf_dma = dma_map_single(jrdev, buf, buflen, DMA_TO_DEVICE); 185 dma_to_sec4_sg_one(sec4_sg, buf_dma, buflen, 0); 186 187 return buf_dma; 188 } 189 190 /* Map req->src and put it in link table */ 191 static inline void src_map_to_sec4_sg(struct device *jrdev, 192 struct scatterlist *src, int src_nents, 193 struct sec4_sg_entry *sec4_sg) 194 { 195 dma_map_sg(jrdev, src, src_nents, DMA_TO_DEVICE); 196 sg_to_sec4_sg_last(src, src_nents, sec4_sg, 0); 197 } 198 199 /* 200 * Only put buffer in link table if it contains data, which is possible, 201 * since a buffer has previously been used, and needs to be unmapped, 202 */ 203 static inline dma_addr_t 204 try_buf_map_to_sec4_sg(struct device *jrdev, struct sec4_sg_entry *sec4_sg, 205 u8 *buf, dma_addr_t buf_dma, int buflen, 206 int last_buflen) 207 { 208 if (buf_dma && !dma_mapping_error(jrdev, buf_dma)) 209 dma_unmap_single(jrdev, buf_dma, last_buflen, DMA_TO_DEVICE); 210 if (buflen) 211 buf_dma = buf_map_to_sec4_sg(jrdev, sec4_sg, buf, buflen); 212 else 213 buf_dma = 0; 214 215 return buf_dma; 216 } 217 218 /* Map state->caam_ctx, and add it to link table */ 219 static inline int ctx_map_to_sec4_sg(u32 *desc, struct device *jrdev, 220 struct caam_hash_state *state, int ctx_len, 221 struct sec4_sg_entry *sec4_sg, u32 flag) 222 { 223 state->ctx_dma = dma_map_single(jrdev, state->caam_ctx, ctx_len, flag); 224 if (dma_mapping_error(jrdev, state->ctx_dma)) { 225 dev_err(jrdev, "unable to map ctx\n"); 226 return -ENOMEM; 227 } 228 229 dma_to_sec4_sg_one(sec4_sg, state->ctx_dma, ctx_len, 0); 230 231 return 0; 232 } 233 234 /* Common shared descriptor commands */ 235 static inline void append_key_ahash(u32 *desc, struct caam_hash_ctx *ctx) 236 { 237 append_key_as_imm(desc, ctx->key, ctx->split_key_pad_len, 238 ctx->split_key_len, CLASS_2 | 239 KEY_DEST_MDHA_SPLIT | KEY_ENC); 240 } 241 242 /* Append key if it has been set */ 243 static inline void init_sh_desc_key_ahash(u32 *desc, struct caam_hash_ctx *ctx) 244 { 245 u32 *key_jump_cmd; 246 247 init_sh_desc(desc, HDR_SHARE_SERIAL); 248 249 if (ctx->split_key_len) { 250 /* Skip if already shared */ 251 key_jump_cmd = append_jump(desc, JUMP_JSL | JUMP_TEST_ALL | 252 JUMP_COND_SHRD); 253 254 append_key_ahash(desc, ctx); 255 256 set_jump_tgt_here(desc, key_jump_cmd); 257 } 258 259 /* Propagate errors from shared to job descriptor */ 260 append_cmd(desc, SET_OK_NO_PROP_ERRORS | CMD_LOAD); 261 } 262 263 /* 264 * For ahash read data from seqin following state->caam_ctx, 265 * and write resulting class2 context to seqout, which may be state->caam_ctx 266 * or req->result 267 */ 268 static inline void ahash_append_load_str(u32 *desc, int digestsize) 269 { 270 /* Calculate remaining bytes to read */ 271 append_math_add(desc, VARSEQINLEN, SEQINLEN, REG0, CAAM_CMD_SZ); 272 273 /* Read remaining bytes */ 274 append_seq_fifo_load(desc, 0, FIFOLD_CLASS_CLASS2 | FIFOLD_TYPE_LAST2 | 275 FIFOLD_TYPE_MSG | KEY_VLF); 276 277 /* Store class2 context bytes */ 278 append_seq_store(desc, digestsize, LDST_CLASS_2_CCB | 279 LDST_SRCDST_BYTE_CONTEXT); 280 } 281 282 /* 283 * For ahash update, final and finup, import context, read and write to seqout 284 */ 285 static inline void ahash_ctx_data_to_out(u32 *desc, u32 op, u32 state, 286 int digestsize, 287 struct caam_hash_ctx *ctx) 288 { 289 init_sh_desc_key_ahash(desc, ctx); 290 291 /* Import context from software */ 292 append_cmd(desc, CMD_SEQ_LOAD | LDST_SRCDST_BYTE_CONTEXT | 293 LDST_CLASS_2_CCB | ctx->ctx_len); 294 295 /* Class 2 operation */ 296 append_operation(desc, op | state | OP_ALG_ENCRYPT); 297 298 /* 299 * Load from buf and/or src and write to req->result or state->context 300 */ 301 ahash_append_load_str(desc, digestsize); 302 } 303 304 /* For ahash firsts and digest, read and write to seqout */ 305 static inline void ahash_data_to_out(u32 *desc, u32 op, u32 state, 306 int digestsize, struct caam_hash_ctx *ctx) 307 { 308 init_sh_desc_key_ahash(desc, ctx); 309 310 /* Class 2 operation */ 311 append_operation(desc, op | state | OP_ALG_ENCRYPT); 312 313 /* 314 * Load from buf and/or src and write to req->result or state->context 315 */ 316 ahash_append_load_str(desc, digestsize); 317 } 318 319 static int ahash_set_sh_desc(struct crypto_ahash *ahash) 320 { 321 struct caam_hash_ctx *ctx = crypto_ahash_ctx(ahash); 322 int digestsize = crypto_ahash_digestsize(ahash); 323 struct device *jrdev = ctx->jrdev; 324 u32 have_key = 0; 325 u32 *desc; 326 327 if (ctx->split_key_len) 328 have_key = OP_ALG_AAI_HMAC_PRECOMP; 329 330 /* ahash_update shared descriptor */ 331 desc = ctx->sh_desc_update; 332 333 init_sh_desc(desc, HDR_SHARE_SERIAL); 334 335 /* Import context from software */ 336 append_cmd(desc, CMD_SEQ_LOAD | LDST_SRCDST_BYTE_CONTEXT | 337 LDST_CLASS_2_CCB | ctx->ctx_len); 338 339 /* Class 2 operation */ 340 append_operation(desc, ctx->alg_type | OP_ALG_AS_UPDATE | 341 OP_ALG_ENCRYPT); 342 343 /* Load data and write to result or context */ 344 ahash_append_load_str(desc, ctx->ctx_len); 345 346 ctx->sh_desc_update_dma = dma_map_single(jrdev, desc, desc_bytes(desc), 347 DMA_TO_DEVICE); 348 if (dma_mapping_error(jrdev, ctx->sh_desc_update_dma)) { 349 dev_err(jrdev, "unable to map shared descriptor\n"); 350 return -ENOMEM; 351 } 352 #ifdef DEBUG 353 print_hex_dump(KERN_ERR, 354 "ahash update shdesc@"__stringify(__LINE__)": ", 355 DUMP_PREFIX_ADDRESS, 16, 4, desc, desc_bytes(desc), 1); 356 #endif 357 358 /* ahash_update_first shared descriptor */ 359 desc = ctx->sh_desc_update_first; 360 361 ahash_data_to_out(desc, have_key | ctx->alg_type, OP_ALG_AS_INIT, 362 ctx->ctx_len, ctx); 363 364 ctx->sh_desc_update_first_dma = dma_map_single(jrdev, desc, 365 desc_bytes(desc), 366 DMA_TO_DEVICE); 367 if (dma_mapping_error(jrdev, ctx->sh_desc_update_first_dma)) { 368 dev_err(jrdev, "unable to map shared descriptor\n"); 369 return -ENOMEM; 370 } 371 #ifdef DEBUG 372 print_hex_dump(KERN_ERR, 373 "ahash update first shdesc@"__stringify(__LINE__)": ", 374 DUMP_PREFIX_ADDRESS, 16, 4, desc, desc_bytes(desc), 1); 375 #endif 376 377 /* ahash_final shared descriptor */ 378 desc = ctx->sh_desc_fin; 379 380 ahash_ctx_data_to_out(desc, have_key | ctx->alg_type, 381 OP_ALG_AS_FINALIZE, digestsize, ctx); 382 383 ctx->sh_desc_fin_dma = dma_map_single(jrdev, desc, desc_bytes(desc), 384 DMA_TO_DEVICE); 385 if (dma_mapping_error(jrdev, ctx->sh_desc_fin_dma)) { 386 dev_err(jrdev, "unable to map shared descriptor\n"); 387 return -ENOMEM; 388 } 389 #ifdef DEBUG 390 print_hex_dump(KERN_ERR, "ahash final shdesc@"__stringify(__LINE__)": ", 391 DUMP_PREFIX_ADDRESS, 16, 4, desc, 392 desc_bytes(desc), 1); 393 #endif 394 395 /* ahash_finup shared descriptor */ 396 desc = ctx->sh_desc_finup; 397 398 ahash_ctx_data_to_out(desc, have_key | ctx->alg_type, 399 OP_ALG_AS_FINALIZE, digestsize, ctx); 400 401 ctx->sh_desc_finup_dma = dma_map_single(jrdev, desc, desc_bytes(desc), 402 DMA_TO_DEVICE); 403 if (dma_mapping_error(jrdev, ctx->sh_desc_finup_dma)) { 404 dev_err(jrdev, "unable to map shared descriptor\n"); 405 return -ENOMEM; 406 } 407 #ifdef DEBUG 408 print_hex_dump(KERN_ERR, "ahash finup shdesc@"__stringify(__LINE__)": ", 409 DUMP_PREFIX_ADDRESS, 16, 4, desc, 410 desc_bytes(desc), 1); 411 #endif 412 413 /* ahash_digest shared descriptor */ 414 desc = ctx->sh_desc_digest; 415 416 ahash_data_to_out(desc, have_key | ctx->alg_type, OP_ALG_AS_INITFINAL, 417 digestsize, ctx); 418 419 ctx->sh_desc_digest_dma = dma_map_single(jrdev, desc, 420 desc_bytes(desc), 421 DMA_TO_DEVICE); 422 if (dma_mapping_error(jrdev, ctx->sh_desc_digest_dma)) { 423 dev_err(jrdev, "unable to map shared descriptor\n"); 424 return -ENOMEM; 425 } 426 #ifdef DEBUG 427 print_hex_dump(KERN_ERR, 428 "ahash digest shdesc@"__stringify(__LINE__)": ", 429 DUMP_PREFIX_ADDRESS, 16, 4, desc, 430 desc_bytes(desc), 1); 431 #endif 432 433 return 0; 434 } 435 436 static int gen_split_hash_key(struct caam_hash_ctx *ctx, const u8 *key_in, 437 u32 keylen) 438 { 439 return gen_split_key(ctx->jrdev, ctx->key, ctx->split_key_len, 440 ctx->split_key_pad_len, key_in, keylen, 441 ctx->alg_op); 442 } 443 444 /* Digest hash size if it is too large */ 445 static int hash_digest_key(struct caam_hash_ctx *ctx, const u8 *key_in, 446 u32 *keylen, u8 *key_out, u32 digestsize) 447 { 448 struct device *jrdev = ctx->jrdev; 449 u32 *desc; 450 struct split_key_result result; 451 dma_addr_t src_dma, dst_dma; 452 int ret = 0; 453 454 desc = kmalloc(CAAM_CMD_SZ * 8 + CAAM_PTR_SZ * 2, GFP_KERNEL | GFP_DMA); 455 if (!desc) { 456 dev_err(jrdev, "unable to allocate key input memory\n"); 457 return -ENOMEM; 458 } 459 460 init_job_desc(desc, 0); 461 462 src_dma = dma_map_single(jrdev, (void *)key_in, *keylen, 463 DMA_TO_DEVICE); 464 if (dma_mapping_error(jrdev, src_dma)) { 465 dev_err(jrdev, "unable to map key input memory\n"); 466 kfree(desc); 467 return -ENOMEM; 468 } 469 dst_dma = dma_map_single(jrdev, (void *)key_out, digestsize, 470 DMA_FROM_DEVICE); 471 if (dma_mapping_error(jrdev, dst_dma)) { 472 dev_err(jrdev, "unable to map key output memory\n"); 473 dma_unmap_single(jrdev, src_dma, *keylen, DMA_TO_DEVICE); 474 kfree(desc); 475 return -ENOMEM; 476 } 477 478 /* Job descriptor to perform unkeyed hash on key_in */ 479 append_operation(desc, ctx->alg_type | OP_ALG_ENCRYPT | 480 OP_ALG_AS_INITFINAL); 481 append_seq_in_ptr(desc, src_dma, *keylen, 0); 482 append_seq_fifo_load(desc, *keylen, FIFOLD_CLASS_CLASS2 | 483 FIFOLD_TYPE_LAST2 | FIFOLD_TYPE_MSG); 484 append_seq_out_ptr(desc, dst_dma, digestsize, 0); 485 append_seq_store(desc, digestsize, LDST_CLASS_2_CCB | 486 LDST_SRCDST_BYTE_CONTEXT); 487 488 #ifdef DEBUG 489 print_hex_dump(KERN_ERR, "key_in@"__stringify(__LINE__)": ", 490 DUMP_PREFIX_ADDRESS, 16, 4, key_in, *keylen, 1); 491 print_hex_dump(KERN_ERR, "jobdesc@"__stringify(__LINE__)": ", 492 DUMP_PREFIX_ADDRESS, 16, 4, desc, desc_bytes(desc), 1); 493 #endif 494 495 result.err = 0; 496 init_completion(&result.completion); 497 498 ret = caam_jr_enqueue(jrdev, desc, split_key_done, &result); 499 if (!ret) { 500 /* in progress */ 501 wait_for_completion_interruptible(&result.completion); 502 ret = result.err; 503 #ifdef DEBUG 504 print_hex_dump(KERN_ERR, 505 "digested key@"__stringify(__LINE__)": ", 506 DUMP_PREFIX_ADDRESS, 16, 4, key_in, 507 digestsize, 1); 508 #endif 509 } 510 dma_unmap_single(jrdev, src_dma, *keylen, DMA_TO_DEVICE); 511 dma_unmap_single(jrdev, dst_dma, digestsize, DMA_FROM_DEVICE); 512 513 *keylen = digestsize; 514 515 kfree(desc); 516 517 return ret; 518 } 519 520 static int ahash_setkey(struct crypto_ahash *ahash, 521 const u8 *key, unsigned int keylen) 522 { 523 /* Sizes for MDHA pads (*not* keys): MD5, SHA1, 224, 256, 384, 512 */ 524 static const u8 mdpadlen[] = { 16, 20, 32, 32, 64, 64 }; 525 struct caam_hash_ctx *ctx = crypto_ahash_ctx(ahash); 526 struct device *jrdev = ctx->jrdev; 527 int blocksize = crypto_tfm_alg_blocksize(&ahash->base); 528 int digestsize = crypto_ahash_digestsize(ahash); 529 int ret = 0; 530 u8 *hashed_key = NULL; 531 532 #ifdef DEBUG 533 printk(KERN_ERR "keylen %d\n", keylen); 534 #endif 535 536 if (keylen > blocksize) { 537 hashed_key = kmalloc(sizeof(u8) * digestsize, GFP_KERNEL | 538 GFP_DMA); 539 if (!hashed_key) 540 return -ENOMEM; 541 ret = hash_digest_key(ctx, key, &keylen, hashed_key, 542 digestsize); 543 if (ret) 544 goto badkey; 545 key = hashed_key; 546 } 547 548 /* Pick class 2 key length from algorithm submask */ 549 ctx->split_key_len = mdpadlen[(ctx->alg_op & OP_ALG_ALGSEL_SUBMASK) >> 550 OP_ALG_ALGSEL_SHIFT] * 2; 551 ctx->split_key_pad_len = ALIGN(ctx->split_key_len, 16); 552 553 #ifdef DEBUG 554 printk(KERN_ERR "split_key_len %d split_key_pad_len %d\n", 555 ctx->split_key_len, ctx->split_key_pad_len); 556 print_hex_dump(KERN_ERR, "key in @"__stringify(__LINE__)": ", 557 DUMP_PREFIX_ADDRESS, 16, 4, key, keylen, 1); 558 #endif 559 560 ret = gen_split_hash_key(ctx, key, keylen); 561 if (ret) 562 goto badkey; 563 564 ctx->key_dma = dma_map_single(jrdev, ctx->key, ctx->split_key_pad_len, 565 DMA_TO_DEVICE); 566 if (dma_mapping_error(jrdev, ctx->key_dma)) { 567 dev_err(jrdev, "unable to map key i/o memory\n"); 568 ret = -ENOMEM; 569 goto map_err; 570 } 571 #ifdef DEBUG 572 print_hex_dump(KERN_ERR, "ctx.key@"__stringify(__LINE__)": ", 573 DUMP_PREFIX_ADDRESS, 16, 4, ctx->key, 574 ctx->split_key_pad_len, 1); 575 #endif 576 577 ret = ahash_set_sh_desc(ahash); 578 if (ret) { 579 dma_unmap_single(jrdev, ctx->key_dma, ctx->split_key_pad_len, 580 DMA_TO_DEVICE); 581 } 582 583 map_err: 584 kfree(hashed_key); 585 return ret; 586 badkey: 587 kfree(hashed_key); 588 crypto_ahash_set_flags(ahash, CRYPTO_TFM_RES_BAD_KEY_LEN); 589 return -EINVAL; 590 } 591 592 /* 593 * ahash_edesc - s/w-extended ahash descriptor 594 * @dst_dma: physical mapped address of req->result 595 * @sec4_sg_dma: physical mapped address of h/w link table 596 * @src_nents: number of segments in input scatterlist 597 * @sec4_sg_bytes: length of dma mapped sec4_sg space 598 * @sec4_sg: pointer to h/w link table 599 * @hw_desc: the h/w job descriptor followed by any referenced link tables 600 */ 601 struct ahash_edesc { 602 dma_addr_t dst_dma; 603 dma_addr_t sec4_sg_dma; 604 int src_nents; 605 int sec4_sg_bytes; 606 struct sec4_sg_entry *sec4_sg; 607 u32 hw_desc[0]; 608 }; 609 610 static inline void ahash_unmap(struct device *dev, 611 struct ahash_edesc *edesc, 612 struct ahash_request *req, int dst_len) 613 { 614 if (edesc->src_nents) 615 dma_unmap_sg(dev, req->src, edesc->src_nents, DMA_TO_DEVICE); 616 if (edesc->dst_dma) 617 dma_unmap_single(dev, edesc->dst_dma, dst_len, DMA_FROM_DEVICE); 618 619 if (edesc->sec4_sg_bytes) 620 dma_unmap_single(dev, edesc->sec4_sg_dma, 621 edesc->sec4_sg_bytes, DMA_TO_DEVICE); 622 } 623 624 static inline void ahash_unmap_ctx(struct device *dev, 625 struct ahash_edesc *edesc, 626 struct ahash_request *req, int dst_len, u32 flag) 627 { 628 struct crypto_ahash *ahash = crypto_ahash_reqtfm(req); 629 struct caam_hash_ctx *ctx = crypto_ahash_ctx(ahash); 630 struct caam_hash_state *state = ahash_request_ctx(req); 631 632 if (state->ctx_dma) 633 dma_unmap_single(dev, state->ctx_dma, ctx->ctx_len, flag); 634 ahash_unmap(dev, edesc, req, dst_len); 635 } 636 637 static void ahash_done(struct device *jrdev, u32 *desc, u32 err, 638 void *context) 639 { 640 struct ahash_request *req = context; 641 struct ahash_edesc *edesc; 642 struct crypto_ahash *ahash = crypto_ahash_reqtfm(req); 643 int digestsize = crypto_ahash_digestsize(ahash); 644 #ifdef DEBUG 645 struct caam_hash_ctx *ctx = crypto_ahash_ctx(ahash); 646 struct caam_hash_state *state = ahash_request_ctx(req); 647 648 dev_err(jrdev, "%s %d: err 0x%x\n", __func__, __LINE__, err); 649 #endif 650 651 edesc = (struct ahash_edesc *)((char *)desc - 652 offsetof(struct ahash_edesc, hw_desc)); 653 if (err) 654 caam_jr_strstatus(jrdev, err); 655 656 ahash_unmap(jrdev, edesc, req, digestsize); 657 kfree(edesc); 658 659 #ifdef DEBUG 660 print_hex_dump(KERN_ERR, "ctx@"__stringify(__LINE__)": ", 661 DUMP_PREFIX_ADDRESS, 16, 4, state->caam_ctx, 662 ctx->ctx_len, 1); 663 if (req->result) 664 print_hex_dump(KERN_ERR, "result@"__stringify(__LINE__)": ", 665 DUMP_PREFIX_ADDRESS, 16, 4, req->result, 666 digestsize, 1); 667 #endif 668 669 req->base.complete(&req->base, err); 670 } 671 672 static void ahash_done_bi(struct device *jrdev, u32 *desc, u32 err, 673 void *context) 674 { 675 struct ahash_request *req = context; 676 struct ahash_edesc *edesc; 677 struct crypto_ahash *ahash = crypto_ahash_reqtfm(req); 678 struct caam_hash_ctx *ctx = crypto_ahash_ctx(ahash); 679 #ifdef DEBUG 680 struct caam_hash_state *state = ahash_request_ctx(req); 681 int digestsize = crypto_ahash_digestsize(ahash); 682 683 dev_err(jrdev, "%s %d: err 0x%x\n", __func__, __LINE__, err); 684 #endif 685 686 edesc = (struct ahash_edesc *)((char *)desc - 687 offsetof(struct ahash_edesc, hw_desc)); 688 if (err) 689 caam_jr_strstatus(jrdev, err); 690 691 ahash_unmap_ctx(jrdev, edesc, req, ctx->ctx_len, DMA_BIDIRECTIONAL); 692 kfree(edesc); 693 694 #ifdef DEBUG 695 print_hex_dump(KERN_ERR, "ctx@"__stringify(__LINE__)": ", 696 DUMP_PREFIX_ADDRESS, 16, 4, state->caam_ctx, 697 ctx->ctx_len, 1); 698 if (req->result) 699 print_hex_dump(KERN_ERR, "result@"__stringify(__LINE__)": ", 700 DUMP_PREFIX_ADDRESS, 16, 4, req->result, 701 digestsize, 1); 702 #endif 703 704 req->base.complete(&req->base, err); 705 } 706 707 static void ahash_done_ctx_src(struct device *jrdev, u32 *desc, u32 err, 708 void *context) 709 { 710 struct ahash_request *req = context; 711 struct ahash_edesc *edesc; 712 struct crypto_ahash *ahash = crypto_ahash_reqtfm(req); 713 int digestsize = crypto_ahash_digestsize(ahash); 714 #ifdef DEBUG 715 struct caam_hash_ctx *ctx = crypto_ahash_ctx(ahash); 716 struct caam_hash_state *state = ahash_request_ctx(req); 717 718 dev_err(jrdev, "%s %d: err 0x%x\n", __func__, __LINE__, err); 719 #endif 720 721 edesc = (struct ahash_edesc *)((char *)desc - 722 offsetof(struct ahash_edesc, hw_desc)); 723 if (err) 724 caam_jr_strstatus(jrdev, err); 725 726 ahash_unmap_ctx(jrdev, edesc, req, digestsize, DMA_TO_DEVICE); 727 kfree(edesc); 728 729 #ifdef DEBUG 730 print_hex_dump(KERN_ERR, "ctx@"__stringify(__LINE__)": ", 731 DUMP_PREFIX_ADDRESS, 16, 4, state->caam_ctx, 732 ctx->ctx_len, 1); 733 if (req->result) 734 print_hex_dump(KERN_ERR, "result@"__stringify(__LINE__)": ", 735 DUMP_PREFIX_ADDRESS, 16, 4, req->result, 736 digestsize, 1); 737 #endif 738 739 req->base.complete(&req->base, err); 740 } 741 742 static void ahash_done_ctx_dst(struct device *jrdev, u32 *desc, u32 err, 743 void *context) 744 { 745 struct ahash_request *req = context; 746 struct ahash_edesc *edesc; 747 struct crypto_ahash *ahash = crypto_ahash_reqtfm(req); 748 struct caam_hash_ctx *ctx = crypto_ahash_ctx(ahash); 749 #ifdef DEBUG 750 struct caam_hash_state *state = ahash_request_ctx(req); 751 int digestsize = crypto_ahash_digestsize(ahash); 752 753 dev_err(jrdev, "%s %d: err 0x%x\n", __func__, __LINE__, err); 754 #endif 755 756 edesc = (struct ahash_edesc *)((char *)desc - 757 offsetof(struct ahash_edesc, hw_desc)); 758 if (err) 759 caam_jr_strstatus(jrdev, err); 760 761 ahash_unmap_ctx(jrdev, edesc, req, ctx->ctx_len, DMA_FROM_DEVICE); 762 kfree(edesc); 763 764 #ifdef DEBUG 765 print_hex_dump(KERN_ERR, "ctx@"__stringify(__LINE__)": ", 766 DUMP_PREFIX_ADDRESS, 16, 4, state->caam_ctx, 767 ctx->ctx_len, 1); 768 if (req->result) 769 print_hex_dump(KERN_ERR, "result@"__stringify(__LINE__)": ", 770 DUMP_PREFIX_ADDRESS, 16, 4, req->result, 771 digestsize, 1); 772 #endif 773 774 req->base.complete(&req->base, err); 775 } 776 777 /* submit update job descriptor */ 778 static int ahash_update_ctx(struct ahash_request *req) 779 { 780 struct crypto_ahash *ahash = crypto_ahash_reqtfm(req); 781 struct caam_hash_ctx *ctx = crypto_ahash_ctx(ahash); 782 struct caam_hash_state *state = ahash_request_ctx(req); 783 struct device *jrdev = ctx->jrdev; 784 gfp_t flags = (req->base.flags & (CRYPTO_TFM_REQ_MAY_BACKLOG | 785 CRYPTO_TFM_REQ_MAY_SLEEP)) ? GFP_KERNEL : GFP_ATOMIC; 786 u8 *buf = state->current_buf ? state->buf_1 : state->buf_0; 787 int *buflen = state->current_buf ? &state->buflen_1 : &state->buflen_0; 788 u8 *next_buf = state->current_buf ? state->buf_0 : state->buf_1; 789 int *next_buflen = state->current_buf ? &state->buflen_0 : 790 &state->buflen_1, last_buflen; 791 int in_len = *buflen + req->nbytes, to_hash; 792 u32 *sh_desc = ctx->sh_desc_update, *desc; 793 dma_addr_t ptr = ctx->sh_desc_update_dma; 794 int src_nents, sec4_sg_bytes, sec4_sg_src_index; 795 struct ahash_edesc *edesc; 796 int ret = 0; 797 int sh_len; 798 799 last_buflen = *next_buflen; 800 *next_buflen = in_len & (crypto_tfm_alg_blocksize(&ahash->base) - 1); 801 to_hash = in_len - *next_buflen; 802 803 if (to_hash) { 804 src_nents = sg_nents_for_len(req->src, 805 req->nbytes - (*next_buflen)); 806 if (src_nents < 0) { 807 dev_err(jrdev, "Invalid number of src SG.\n"); 808 return src_nents; 809 } 810 sec4_sg_src_index = 1 + (*buflen ? 1 : 0); 811 sec4_sg_bytes = (sec4_sg_src_index + src_nents) * 812 sizeof(struct sec4_sg_entry); 813 814 /* 815 * allocate space for base edesc and hw desc commands, 816 * link tables 817 */ 818 edesc = kzalloc(sizeof(*edesc) + DESC_JOB_IO_LEN + 819 sec4_sg_bytes, GFP_DMA | flags); 820 if (!edesc) { 821 dev_err(jrdev, 822 "could not allocate extended descriptor\n"); 823 return -ENOMEM; 824 } 825 826 edesc->src_nents = src_nents; 827 edesc->sec4_sg_bytes = sec4_sg_bytes; 828 edesc->sec4_sg = (void *)edesc + sizeof(struct ahash_edesc) + 829 DESC_JOB_IO_LEN; 830 831 ret = ctx_map_to_sec4_sg(desc, jrdev, state, ctx->ctx_len, 832 edesc->sec4_sg, DMA_BIDIRECTIONAL); 833 if (ret) 834 return ret; 835 836 state->buf_dma = try_buf_map_to_sec4_sg(jrdev, 837 edesc->sec4_sg + 1, 838 buf, state->buf_dma, 839 *buflen, last_buflen); 840 841 if (src_nents) { 842 src_map_to_sec4_sg(jrdev, req->src, src_nents, 843 edesc->sec4_sg + sec4_sg_src_index); 844 if (*next_buflen) 845 scatterwalk_map_and_copy(next_buf, req->src, 846 to_hash - *buflen, 847 *next_buflen, 0); 848 } else { 849 (edesc->sec4_sg + sec4_sg_src_index - 1)->len |= 850 cpu_to_caam32(SEC4_SG_LEN_FIN); 851 } 852 853 state->current_buf = !state->current_buf; 854 855 sh_len = desc_len(sh_desc); 856 desc = edesc->hw_desc; 857 init_job_desc_shared(desc, ptr, sh_len, HDR_SHARE_DEFER | 858 HDR_REVERSE); 859 860 edesc->sec4_sg_dma = dma_map_single(jrdev, edesc->sec4_sg, 861 sec4_sg_bytes, 862 DMA_TO_DEVICE); 863 if (dma_mapping_error(jrdev, edesc->sec4_sg_dma)) { 864 dev_err(jrdev, "unable to map S/G table\n"); 865 return -ENOMEM; 866 } 867 868 append_seq_in_ptr(desc, edesc->sec4_sg_dma, ctx->ctx_len + 869 to_hash, LDST_SGF); 870 871 append_seq_out_ptr(desc, state->ctx_dma, ctx->ctx_len, 0); 872 873 #ifdef DEBUG 874 print_hex_dump(KERN_ERR, "jobdesc@"__stringify(__LINE__)": ", 875 DUMP_PREFIX_ADDRESS, 16, 4, desc, 876 desc_bytes(desc), 1); 877 #endif 878 879 ret = caam_jr_enqueue(jrdev, desc, ahash_done_bi, req); 880 if (!ret) { 881 ret = -EINPROGRESS; 882 } else { 883 ahash_unmap_ctx(jrdev, edesc, req, ctx->ctx_len, 884 DMA_BIDIRECTIONAL); 885 kfree(edesc); 886 } 887 } else if (*next_buflen) { 888 scatterwalk_map_and_copy(buf + *buflen, req->src, 0, 889 req->nbytes, 0); 890 *buflen = *next_buflen; 891 *next_buflen = last_buflen; 892 } 893 #ifdef DEBUG 894 print_hex_dump(KERN_ERR, "buf@"__stringify(__LINE__)": ", 895 DUMP_PREFIX_ADDRESS, 16, 4, buf, *buflen, 1); 896 print_hex_dump(KERN_ERR, "next buf@"__stringify(__LINE__)": ", 897 DUMP_PREFIX_ADDRESS, 16, 4, next_buf, 898 *next_buflen, 1); 899 #endif 900 901 return ret; 902 } 903 904 static int ahash_final_ctx(struct ahash_request *req) 905 { 906 struct crypto_ahash *ahash = crypto_ahash_reqtfm(req); 907 struct caam_hash_ctx *ctx = crypto_ahash_ctx(ahash); 908 struct caam_hash_state *state = ahash_request_ctx(req); 909 struct device *jrdev = ctx->jrdev; 910 gfp_t flags = (req->base.flags & (CRYPTO_TFM_REQ_MAY_BACKLOG | 911 CRYPTO_TFM_REQ_MAY_SLEEP)) ? GFP_KERNEL : GFP_ATOMIC; 912 u8 *buf = state->current_buf ? state->buf_1 : state->buf_0; 913 int buflen = state->current_buf ? state->buflen_1 : state->buflen_0; 914 int last_buflen = state->current_buf ? state->buflen_0 : 915 state->buflen_1; 916 u32 *sh_desc = ctx->sh_desc_fin, *desc; 917 dma_addr_t ptr = ctx->sh_desc_fin_dma; 918 int sec4_sg_bytes, sec4_sg_src_index; 919 int digestsize = crypto_ahash_digestsize(ahash); 920 struct ahash_edesc *edesc; 921 int ret = 0; 922 int sh_len; 923 924 sec4_sg_src_index = 1 + (buflen ? 1 : 0); 925 sec4_sg_bytes = sec4_sg_src_index * sizeof(struct sec4_sg_entry); 926 927 /* allocate space for base edesc and hw desc commands, link tables */ 928 edesc = kzalloc(sizeof(*edesc) + DESC_JOB_IO_LEN + sec4_sg_bytes, 929 GFP_DMA | flags); 930 if (!edesc) { 931 dev_err(jrdev, "could not allocate extended descriptor\n"); 932 return -ENOMEM; 933 } 934 935 sh_len = desc_len(sh_desc); 936 desc = edesc->hw_desc; 937 init_job_desc_shared(desc, ptr, sh_len, HDR_SHARE_DEFER | HDR_REVERSE); 938 939 edesc->sec4_sg_bytes = sec4_sg_bytes; 940 edesc->sec4_sg = (void *)edesc + sizeof(struct ahash_edesc) + 941 DESC_JOB_IO_LEN; 942 edesc->src_nents = 0; 943 944 ret = ctx_map_to_sec4_sg(desc, jrdev, state, ctx->ctx_len, 945 edesc->sec4_sg, DMA_TO_DEVICE); 946 if (ret) 947 return ret; 948 949 state->buf_dma = try_buf_map_to_sec4_sg(jrdev, edesc->sec4_sg + 1, 950 buf, state->buf_dma, buflen, 951 last_buflen); 952 (edesc->sec4_sg + sec4_sg_src_index - 1)->len |= 953 cpu_to_caam32(SEC4_SG_LEN_FIN); 954 955 edesc->sec4_sg_dma = dma_map_single(jrdev, edesc->sec4_sg, 956 sec4_sg_bytes, DMA_TO_DEVICE); 957 if (dma_mapping_error(jrdev, edesc->sec4_sg_dma)) { 958 dev_err(jrdev, "unable to map S/G table\n"); 959 return -ENOMEM; 960 } 961 962 append_seq_in_ptr(desc, edesc->sec4_sg_dma, ctx->ctx_len + buflen, 963 LDST_SGF); 964 965 edesc->dst_dma = map_seq_out_ptr_result(desc, jrdev, req->result, 966 digestsize); 967 if (dma_mapping_error(jrdev, edesc->dst_dma)) { 968 dev_err(jrdev, "unable to map dst\n"); 969 return -ENOMEM; 970 } 971 972 #ifdef DEBUG 973 print_hex_dump(KERN_ERR, "jobdesc@"__stringify(__LINE__)": ", 974 DUMP_PREFIX_ADDRESS, 16, 4, desc, desc_bytes(desc), 1); 975 #endif 976 977 ret = caam_jr_enqueue(jrdev, desc, ahash_done_ctx_src, req); 978 if (!ret) { 979 ret = -EINPROGRESS; 980 } else { 981 ahash_unmap_ctx(jrdev, edesc, req, digestsize, DMA_FROM_DEVICE); 982 kfree(edesc); 983 } 984 985 return ret; 986 } 987 988 static int ahash_finup_ctx(struct ahash_request *req) 989 { 990 struct crypto_ahash *ahash = crypto_ahash_reqtfm(req); 991 struct caam_hash_ctx *ctx = crypto_ahash_ctx(ahash); 992 struct caam_hash_state *state = ahash_request_ctx(req); 993 struct device *jrdev = ctx->jrdev; 994 gfp_t flags = (req->base.flags & (CRYPTO_TFM_REQ_MAY_BACKLOG | 995 CRYPTO_TFM_REQ_MAY_SLEEP)) ? GFP_KERNEL : GFP_ATOMIC; 996 u8 *buf = state->current_buf ? state->buf_1 : state->buf_0; 997 int buflen = state->current_buf ? state->buflen_1 : state->buflen_0; 998 int last_buflen = state->current_buf ? state->buflen_0 : 999 state->buflen_1; 1000 u32 *sh_desc = ctx->sh_desc_finup, *desc; 1001 dma_addr_t ptr = ctx->sh_desc_finup_dma; 1002 int sec4_sg_bytes, sec4_sg_src_index; 1003 int src_nents; 1004 int digestsize = crypto_ahash_digestsize(ahash); 1005 struct ahash_edesc *edesc; 1006 int ret = 0; 1007 int sh_len; 1008 1009 src_nents = sg_nents_for_len(req->src, req->nbytes); 1010 if (src_nents < 0) { 1011 dev_err(jrdev, "Invalid number of src SG.\n"); 1012 return src_nents; 1013 } 1014 sec4_sg_src_index = 1 + (buflen ? 1 : 0); 1015 sec4_sg_bytes = (sec4_sg_src_index + src_nents) * 1016 sizeof(struct sec4_sg_entry); 1017 1018 /* allocate space for base edesc and hw desc commands, link tables */ 1019 edesc = kzalloc(sizeof(*edesc) + DESC_JOB_IO_LEN + sec4_sg_bytes, 1020 GFP_DMA | flags); 1021 if (!edesc) { 1022 dev_err(jrdev, "could not allocate extended descriptor\n"); 1023 return -ENOMEM; 1024 } 1025 1026 sh_len = desc_len(sh_desc); 1027 desc = edesc->hw_desc; 1028 init_job_desc_shared(desc, ptr, sh_len, HDR_SHARE_DEFER | HDR_REVERSE); 1029 1030 edesc->src_nents = src_nents; 1031 edesc->sec4_sg_bytes = sec4_sg_bytes; 1032 edesc->sec4_sg = (void *)edesc + sizeof(struct ahash_edesc) + 1033 DESC_JOB_IO_LEN; 1034 1035 ret = ctx_map_to_sec4_sg(desc, jrdev, state, ctx->ctx_len, 1036 edesc->sec4_sg, DMA_TO_DEVICE); 1037 if (ret) 1038 return ret; 1039 1040 state->buf_dma = try_buf_map_to_sec4_sg(jrdev, edesc->sec4_sg + 1, 1041 buf, state->buf_dma, buflen, 1042 last_buflen); 1043 1044 src_map_to_sec4_sg(jrdev, req->src, src_nents, edesc->sec4_sg + 1045 sec4_sg_src_index); 1046 1047 edesc->sec4_sg_dma = dma_map_single(jrdev, edesc->sec4_sg, 1048 sec4_sg_bytes, DMA_TO_DEVICE); 1049 if (dma_mapping_error(jrdev, edesc->sec4_sg_dma)) { 1050 dev_err(jrdev, "unable to map S/G table\n"); 1051 return -ENOMEM; 1052 } 1053 1054 append_seq_in_ptr(desc, edesc->sec4_sg_dma, ctx->ctx_len + 1055 buflen + req->nbytes, LDST_SGF); 1056 1057 edesc->dst_dma = map_seq_out_ptr_result(desc, jrdev, req->result, 1058 digestsize); 1059 if (dma_mapping_error(jrdev, edesc->dst_dma)) { 1060 dev_err(jrdev, "unable to map dst\n"); 1061 return -ENOMEM; 1062 } 1063 1064 #ifdef DEBUG 1065 print_hex_dump(KERN_ERR, "jobdesc@"__stringify(__LINE__)": ", 1066 DUMP_PREFIX_ADDRESS, 16, 4, desc, desc_bytes(desc), 1); 1067 #endif 1068 1069 ret = caam_jr_enqueue(jrdev, desc, ahash_done_ctx_src, req); 1070 if (!ret) { 1071 ret = -EINPROGRESS; 1072 } else { 1073 ahash_unmap_ctx(jrdev, edesc, req, digestsize, DMA_FROM_DEVICE); 1074 kfree(edesc); 1075 } 1076 1077 return ret; 1078 } 1079 1080 static int ahash_digest(struct ahash_request *req) 1081 { 1082 struct crypto_ahash *ahash = crypto_ahash_reqtfm(req); 1083 struct caam_hash_ctx *ctx = crypto_ahash_ctx(ahash); 1084 struct device *jrdev = ctx->jrdev; 1085 gfp_t flags = (req->base.flags & (CRYPTO_TFM_REQ_MAY_BACKLOG | 1086 CRYPTO_TFM_REQ_MAY_SLEEP)) ? GFP_KERNEL : GFP_ATOMIC; 1087 u32 *sh_desc = ctx->sh_desc_digest, *desc; 1088 dma_addr_t ptr = ctx->sh_desc_digest_dma; 1089 int digestsize = crypto_ahash_digestsize(ahash); 1090 int src_nents, sec4_sg_bytes; 1091 dma_addr_t src_dma; 1092 struct ahash_edesc *edesc; 1093 int ret = 0; 1094 u32 options; 1095 int sh_len; 1096 1097 src_nents = sg_nents_for_len(req->src, req->nbytes); 1098 if (src_nents < 0) { 1099 dev_err(jrdev, "Invalid number of src SG.\n"); 1100 return src_nents; 1101 } 1102 dma_map_sg(jrdev, req->src, src_nents, DMA_TO_DEVICE); 1103 if (src_nents > 1) 1104 sec4_sg_bytes = src_nents * sizeof(struct sec4_sg_entry); 1105 else 1106 sec4_sg_bytes = 0; 1107 1108 /* allocate space for base edesc and hw desc commands, link tables */ 1109 edesc = kzalloc(sizeof(*edesc) + sec4_sg_bytes + DESC_JOB_IO_LEN, 1110 GFP_DMA | flags); 1111 if (!edesc) { 1112 dev_err(jrdev, "could not allocate extended descriptor\n"); 1113 return -ENOMEM; 1114 } 1115 edesc->sec4_sg = (void *)edesc + sizeof(struct ahash_edesc) + 1116 DESC_JOB_IO_LEN; 1117 edesc->sec4_sg_bytes = sec4_sg_bytes; 1118 edesc->src_nents = src_nents; 1119 1120 sh_len = desc_len(sh_desc); 1121 desc = edesc->hw_desc; 1122 init_job_desc_shared(desc, ptr, sh_len, HDR_SHARE_DEFER | HDR_REVERSE); 1123 1124 if (src_nents > 1) { 1125 sg_to_sec4_sg_last(req->src, src_nents, edesc->sec4_sg, 0); 1126 edesc->sec4_sg_dma = dma_map_single(jrdev, edesc->sec4_sg, 1127 sec4_sg_bytes, DMA_TO_DEVICE); 1128 if (dma_mapping_error(jrdev, edesc->sec4_sg_dma)) { 1129 dev_err(jrdev, "unable to map S/G table\n"); 1130 return -ENOMEM; 1131 } 1132 src_dma = edesc->sec4_sg_dma; 1133 options = LDST_SGF; 1134 } else { 1135 src_dma = sg_dma_address(req->src); 1136 options = 0; 1137 } 1138 append_seq_in_ptr(desc, src_dma, req->nbytes, options); 1139 1140 edesc->dst_dma = map_seq_out_ptr_result(desc, jrdev, req->result, 1141 digestsize); 1142 if (dma_mapping_error(jrdev, edesc->dst_dma)) { 1143 dev_err(jrdev, "unable to map dst\n"); 1144 return -ENOMEM; 1145 } 1146 1147 #ifdef DEBUG 1148 print_hex_dump(KERN_ERR, "jobdesc@"__stringify(__LINE__)": ", 1149 DUMP_PREFIX_ADDRESS, 16, 4, desc, desc_bytes(desc), 1); 1150 #endif 1151 1152 ret = caam_jr_enqueue(jrdev, desc, ahash_done, req); 1153 if (!ret) { 1154 ret = -EINPROGRESS; 1155 } else { 1156 ahash_unmap(jrdev, edesc, req, digestsize); 1157 kfree(edesc); 1158 } 1159 1160 return ret; 1161 } 1162 1163 /* submit ahash final if it the first job descriptor */ 1164 static int ahash_final_no_ctx(struct ahash_request *req) 1165 { 1166 struct crypto_ahash *ahash = crypto_ahash_reqtfm(req); 1167 struct caam_hash_ctx *ctx = crypto_ahash_ctx(ahash); 1168 struct caam_hash_state *state = ahash_request_ctx(req); 1169 struct device *jrdev = ctx->jrdev; 1170 gfp_t flags = (req->base.flags & (CRYPTO_TFM_REQ_MAY_BACKLOG | 1171 CRYPTO_TFM_REQ_MAY_SLEEP)) ? GFP_KERNEL : GFP_ATOMIC; 1172 u8 *buf = state->current_buf ? state->buf_1 : state->buf_0; 1173 int buflen = state->current_buf ? state->buflen_1 : state->buflen_0; 1174 u32 *sh_desc = ctx->sh_desc_digest, *desc; 1175 dma_addr_t ptr = ctx->sh_desc_digest_dma; 1176 int digestsize = crypto_ahash_digestsize(ahash); 1177 struct ahash_edesc *edesc; 1178 int ret = 0; 1179 int sh_len; 1180 1181 /* allocate space for base edesc and hw desc commands, link tables */ 1182 edesc = kzalloc(sizeof(*edesc) + DESC_JOB_IO_LEN, GFP_DMA | flags); 1183 if (!edesc) { 1184 dev_err(jrdev, "could not allocate extended descriptor\n"); 1185 return -ENOMEM; 1186 } 1187 1188 edesc->sec4_sg_bytes = 0; 1189 sh_len = desc_len(sh_desc); 1190 desc = edesc->hw_desc; 1191 init_job_desc_shared(desc, ptr, sh_len, HDR_SHARE_DEFER | HDR_REVERSE); 1192 1193 state->buf_dma = dma_map_single(jrdev, buf, buflen, DMA_TO_DEVICE); 1194 if (dma_mapping_error(jrdev, state->buf_dma)) { 1195 dev_err(jrdev, "unable to map src\n"); 1196 return -ENOMEM; 1197 } 1198 1199 append_seq_in_ptr(desc, state->buf_dma, buflen, 0); 1200 1201 edesc->dst_dma = map_seq_out_ptr_result(desc, jrdev, req->result, 1202 digestsize); 1203 if (dma_mapping_error(jrdev, edesc->dst_dma)) { 1204 dev_err(jrdev, "unable to map dst\n"); 1205 return -ENOMEM; 1206 } 1207 edesc->src_nents = 0; 1208 1209 #ifdef DEBUG 1210 print_hex_dump(KERN_ERR, "jobdesc@"__stringify(__LINE__)": ", 1211 DUMP_PREFIX_ADDRESS, 16, 4, desc, desc_bytes(desc), 1); 1212 #endif 1213 1214 ret = caam_jr_enqueue(jrdev, desc, ahash_done, req); 1215 if (!ret) { 1216 ret = -EINPROGRESS; 1217 } else { 1218 ahash_unmap(jrdev, edesc, req, digestsize); 1219 kfree(edesc); 1220 } 1221 1222 return ret; 1223 } 1224 1225 /* submit ahash update if it the first job descriptor after update */ 1226 static int ahash_update_no_ctx(struct ahash_request *req) 1227 { 1228 struct crypto_ahash *ahash = crypto_ahash_reqtfm(req); 1229 struct caam_hash_ctx *ctx = crypto_ahash_ctx(ahash); 1230 struct caam_hash_state *state = ahash_request_ctx(req); 1231 struct device *jrdev = ctx->jrdev; 1232 gfp_t flags = (req->base.flags & (CRYPTO_TFM_REQ_MAY_BACKLOG | 1233 CRYPTO_TFM_REQ_MAY_SLEEP)) ? GFP_KERNEL : GFP_ATOMIC; 1234 u8 *buf = state->current_buf ? state->buf_1 : state->buf_0; 1235 int *buflen = state->current_buf ? &state->buflen_1 : &state->buflen_0; 1236 u8 *next_buf = state->current_buf ? state->buf_0 : state->buf_1; 1237 int *next_buflen = state->current_buf ? &state->buflen_0 : 1238 &state->buflen_1; 1239 int in_len = *buflen + req->nbytes, to_hash; 1240 int sec4_sg_bytes, src_nents; 1241 struct ahash_edesc *edesc; 1242 u32 *desc, *sh_desc = ctx->sh_desc_update_first; 1243 dma_addr_t ptr = ctx->sh_desc_update_first_dma; 1244 int ret = 0; 1245 int sh_len; 1246 1247 *next_buflen = in_len & (crypto_tfm_alg_blocksize(&ahash->base) - 1); 1248 to_hash = in_len - *next_buflen; 1249 1250 if (to_hash) { 1251 src_nents = sg_nents_for_len(req->src, 1252 req->nbytes - *next_buflen); 1253 if (src_nents < 0) { 1254 dev_err(jrdev, "Invalid number of src SG.\n"); 1255 return src_nents; 1256 } 1257 sec4_sg_bytes = (1 + src_nents) * 1258 sizeof(struct sec4_sg_entry); 1259 1260 /* 1261 * allocate space for base edesc and hw desc commands, 1262 * link tables 1263 */ 1264 edesc = kzalloc(sizeof(*edesc) + DESC_JOB_IO_LEN + 1265 sec4_sg_bytes, GFP_DMA | flags); 1266 if (!edesc) { 1267 dev_err(jrdev, 1268 "could not allocate extended descriptor\n"); 1269 return -ENOMEM; 1270 } 1271 1272 edesc->src_nents = src_nents; 1273 edesc->sec4_sg_bytes = sec4_sg_bytes; 1274 edesc->sec4_sg = (void *)edesc + sizeof(struct ahash_edesc) + 1275 DESC_JOB_IO_LEN; 1276 edesc->dst_dma = 0; 1277 1278 state->buf_dma = buf_map_to_sec4_sg(jrdev, edesc->sec4_sg, 1279 buf, *buflen); 1280 src_map_to_sec4_sg(jrdev, req->src, src_nents, 1281 edesc->sec4_sg + 1); 1282 if (*next_buflen) { 1283 scatterwalk_map_and_copy(next_buf, req->src, 1284 to_hash - *buflen, 1285 *next_buflen, 0); 1286 } 1287 1288 state->current_buf = !state->current_buf; 1289 1290 sh_len = desc_len(sh_desc); 1291 desc = edesc->hw_desc; 1292 init_job_desc_shared(desc, ptr, sh_len, HDR_SHARE_DEFER | 1293 HDR_REVERSE); 1294 1295 edesc->sec4_sg_dma = dma_map_single(jrdev, edesc->sec4_sg, 1296 sec4_sg_bytes, 1297 DMA_TO_DEVICE); 1298 if (dma_mapping_error(jrdev, edesc->sec4_sg_dma)) { 1299 dev_err(jrdev, "unable to map S/G table\n"); 1300 return -ENOMEM; 1301 } 1302 1303 append_seq_in_ptr(desc, edesc->sec4_sg_dma, to_hash, LDST_SGF); 1304 1305 ret = map_seq_out_ptr_ctx(desc, jrdev, state, ctx->ctx_len); 1306 if (ret) 1307 return ret; 1308 1309 #ifdef DEBUG 1310 print_hex_dump(KERN_ERR, "jobdesc@"__stringify(__LINE__)": ", 1311 DUMP_PREFIX_ADDRESS, 16, 4, desc, 1312 desc_bytes(desc), 1); 1313 #endif 1314 1315 ret = caam_jr_enqueue(jrdev, desc, ahash_done_ctx_dst, req); 1316 if (!ret) { 1317 ret = -EINPROGRESS; 1318 state->update = ahash_update_ctx; 1319 state->finup = ahash_finup_ctx; 1320 state->final = ahash_final_ctx; 1321 } else { 1322 ahash_unmap_ctx(jrdev, edesc, req, ctx->ctx_len, 1323 DMA_TO_DEVICE); 1324 kfree(edesc); 1325 } 1326 } else if (*next_buflen) { 1327 scatterwalk_map_and_copy(buf + *buflen, req->src, 0, 1328 req->nbytes, 0); 1329 *buflen = *next_buflen; 1330 *next_buflen = 0; 1331 } 1332 #ifdef DEBUG 1333 print_hex_dump(KERN_ERR, "buf@"__stringify(__LINE__)": ", 1334 DUMP_PREFIX_ADDRESS, 16, 4, buf, *buflen, 1); 1335 print_hex_dump(KERN_ERR, "next buf@"__stringify(__LINE__)": ", 1336 DUMP_PREFIX_ADDRESS, 16, 4, next_buf, 1337 *next_buflen, 1); 1338 #endif 1339 1340 return ret; 1341 } 1342 1343 /* submit ahash finup if it the first job descriptor after update */ 1344 static int ahash_finup_no_ctx(struct ahash_request *req) 1345 { 1346 struct crypto_ahash *ahash = crypto_ahash_reqtfm(req); 1347 struct caam_hash_ctx *ctx = crypto_ahash_ctx(ahash); 1348 struct caam_hash_state *state = ahash_request_ctx(req); 1349 struct device *jrdev = ctx->jrdev; 1350 gfp_t flags = (req->base.flags & (CRYPTO_TFM_REQ_MAY_BACKLOG | 1351 CRYPTO_TFM_REQ_MAY_SLEEP)) ? GFP_KERNEL : GFP_ATOMIC; 1352 u8 *buf = state->current_buf ? state->buf_1 : state->buf_0; 1353 int buflen = state->current_buf ? state->buflen_1 : state->buflen_0; 1354 int last_buflen = state->current_buf ? state->buflen_0 : 1355 state->buflen_1; 1356 u32 *sh_desc = ctx->sh_desc_digest, *desc; 1357 dma_addr_t ptr = ctx->sh_desc_digest_dma; 1358 int sec4_sg_bytes, sec4_sg_src_index, src_nents; 1359 int digestsize = crypto_ahash_digestsize(ahash); 1360 struct ahash_edesc *edesc; 1361 int sh_len; 1362 int ret = 0; 1363 1364 src_nents = sg_nents_for_len(req->src, req->nbytes); 1365 if (src_nents < 0) { 1366 dev_err(jrdev, "Invalid number of src SG.\n"); 1367 return src_nents; 1368 } 1369 sec4_sg_src_index = 2; 1370 sec4_sg_bytes = (sec4_sg_src_index + src_nents) * 1371 sizeof(struct sec4_sg_entry); 1372 1373 /* allocate space for base edesc and hw desc commands, link tables */ 1374 edesc = kzalloc(sizeof(*edesc) + DESC_JOB_IO_LEN + sec4_sg_bytes, 1375 GFP_DMA | flags); 1376 if (!edesc) { 1377 dev_err(jrdev, "could not allocate extended descriptor\n"); 1378 return -ENOMEM; 1379 } 1380 1381 sh_len = desc_len(sh_desc); 1382 desc = edesc->hw_desc; 1383 init_job_desc_shared(desc, ptr, sh_len, HDR_SHARE_DEFER | HDR_REVERSE); 1384 1385 edesc->src_nents = src_nents; 1386 edesc->sec4_sg_bytes = sec4_sg_bytes; 1387 edesc->sec4_sg = (void *)edesc + sizeof(struct ahash_edesc) + 1388 DESC_JOB_IO_LEN; 1389 1390 state->buf_dma = try_buf_map_to_sec4_sg(jrdev, edesc->sec4_sg, buf, 1391 state->buf_dma, buflen, 1392 last_buflen); 1393 1394 src_map_to_sec4_sg(jrdev, req->src, src_nents, edesc->sec4_sg + 1); 1395 1396 edesc->sec4_sg_dma = dma_map_single(jrdev, edesc->sec4_sg, 1397 sec4_sg_bytes, DMA_TO_DEVICE); 1398 if (dma_mapping_error(jrdev, edesc->sec4_sg_dma)) { 1399 dev_err(jrdev, "unable to map S/G table\n"); 1400 return -ENOMEM; 1401 } 1402 1403 append_seq_in_ptr(desc, edesc->sec4_sg_dma, buflen + 1404 req->nbytes, LDST_SGF); 1405 1406 edesc->dst_dma = map_seq_out_ptr_result(desc, jrdev, req->result, 1407 digestsize); 1408 if (dma_mapping_error(jrdev, edesc->dst_dma)) { 1409 dev_err(jrdev, "unable to map dst\n"); 1410 return -ENOMEM; 1411 } 1412 1413 #ifdef DEBUG 1414 print_hex_dump(KERN_ERR, "jobdesc@"__stringify(__LINE__)": ", 1415 DUMP_PREFIX_ADDRESS, 16, 4, desc, desc_bytes(desc), 1); 1416 #endif 1417 1418 ret = caam_jr_enqueue(jrdev, desc, ahash_done, req); 1419 if (!ret) { 1420 ret = -EINPROGRESS; 1421 } else { 1422 ahash_unmap(jrdev, edesc, req, digestsize); 1423 kfree(edesc); 1424 } 1425 1426 return ret; 1427 } 1428 1429 /* submit first update job descriptor after init */ 1430 static int ahash_update_first(struct ahash_request *req) 1431 { 1432 struct crypto_ahash *ahash = crypto_ahash_reqtfm(req); 1433 struct caam_hash_ctx *ctx = crypto_ahash_ctx(ahash); 1434 struct caam_hash_state *state = ahash_request_ctx(req); 1435 struct device *jrdev = ctx->jrdev; 1436 gfp_t flags = (req->base.flags & (CRYPTO_TFM_REQ_MAY_BACKLOG | 1437 CRYPTO_TFM_REQ_MAY_SLEEP)) ? GFP_KERNEL : GFP_ATOMIC; 1438 u8 *next_buf = state->current_buf ? state->buf_1 : state->buf_0; 1439 int *next_buflen = state->current_buf ? 1440 &state->buflen_1 : &state->buflen_0; 1441 int to_hash; 1442 u32 *sh_desc = ctx->sh_desc_update_first, *desc; 1443 dma_addr_t ptr = ctx->sh_desc_update_first_dma; 1444 int sec4_sg_bytes, src_nents; 1445 dma_addr_t src_dma; 1446 u32 options; 1447 struct ahash_edesc *edesc; 1448 int ret = 0; 1449 int sh_len; 1450 1451 *next_buflen = req->nbytes & (crypto_tfm_alg_blocksize(&ahash->base) - 1452 1); 1453 to_hash = req->nbytes - *next_buflen; 1454 1455 if (to_hash) { 1456 src_nents = sg_nents_for_len(req->src, 1457 req->nbytes - *next_buflen); 1458 if (src_nents < 0) { 1459 dev_err(jrdev, "Invalid number of src SG.\n"); 1460 return src_nents; 1461 } 1462 dma_map_sg(jrdev, req->src, src_nents, DMA_TO_DEVICE); 1463 if (src_nents > 1) 1464 sec4_sg_bytes = src_nents * 1465 sizeof(struct sec4_sg_entry); 1466 else 1467 sec4_sg_bytes = 0; 1468 1469 /* 1470 * allocate space for base edesc and hw desc commands, 1471 * link tables 1472 */ 1473 edesc = kzalloc(sizeof(*edesc) + DESC_JOB_IO_LEN + 1474 sec4_sg_bytes, GFP_DMA | flags); 1475 if (!edesc) { 1476 dev_err(jrdev, 1477 "could not allocate extended descriptor\n"); 1478 return -ENOMEM; 1479 } 1480 1481 edesc->src_nents = src_nents; 1482 edesc->sec4_sg_bytes = sec4_sg_bytes; 1483 edesc->sec4_sg = (void *)edesc + sizeof(struct ahash_edesc) + 1484 DESC_JOB_IO_LEN; 1485 edesc->dst_dma = 0; 1486 1487 if (src_nents > 1) { 1488 sg_to_sec4_sg_last(req->src, src_nents, 1489 edesc->sec4_sg, 0); 1490 edesc->sec4_sg_dma = dma_map_single(jrdev, 1491 edesc->sec4_sg, 1492 sec4_sg_bytes, 1493 DMA_TO_DEVICE); 1494 if (dma_mapping_error(jrdev, edesc->sec4_sg_dma)) { 1495 dev_err(jrdev, "unable to map S/G table\n"); 1496 return -ENOMEM; 1497 } 1498 src_dma = edesc->sec4_sg_dma; 1499 options = LDST_SGF; 1500 } else { 1501 src_dma = sg_dma_address(req->src); 1502 options = 0; 1503 } 1504 1505 if (*next_buflen) 1506 scatterwalk_map_and_copy(next_buf, req->src, to_hash, 1507 *next_buflen, 0); 1508 1509 sh_len = desc_len(sh_desc); 1510 desc = edesc->hw_desc; 1511 init_job_desc_shared(desc, ptr, sh_len, HDR_SHARE_DEFER | 1512 HDR_REVERSE); 1513 1514 append_seq_in_ptr(desc, src_dma, to_hash, options); 1515 1516 ret = map_seq_out_ptr_ctx(desc, jrdev, state, ctx->ctx_len); 1517 if (ret) 1518 return ret; 1519 1520 #ifdef DEBUG 1521 print_hex_dump(KERN_ERR, "jobdesc@"__stringify(__LINE__)": ", 1522 DUMP_PREFIX_ADDRESS, 16, 4, desc, 1523 desc_bytes(desc), 1); 1524 #endif 1525 1526 ret = caam_jr_enqueue(jrdev, desc, ahash_done_ctx_dst, 1527 req); 1528 if (!ret) { 1529 ret = -EINPROGRESS; 1530 state->update = ahash_update_ctx; 1531 state->finup = ahash_finup_ctx; 1532 state->final = ahash_final_ctx; 1533 } else { 1534 ahash_unmap_ctx(jrdev, edesc, req, ctx->ctx_len, 1535 DMA_TO_DEVICE); 1536 kfree(edesc); 1537 } 1538 } else if (*next_buflen) { 1539 state->update = ahash_update_no_ctx; 1540 state->finup = ahash_finup_no_ctx; 1541 state->final = ahash_final_no_ctx; 1542 scatterwalk_map_and_copy(next_buf, req->src, 0, 1543 req->nbytes, 0); 1544 } 1545 #ifdef DEBUG 1546 print_hex_dump(KERN_ERR, "next buf@"__stringify(__LINE__)": ", 1547 DUMP_PREFIX_ADDRESS, 16, 4, next_buf, 1548 *next_buflen, 1); 1549 #endif 1550 1551 return ret; 1552 } 1553 1554 static int ahash_finup_first(struct ahash_request *req) 1555 { 1556 return ahash_digest(req); 1557 } 1558 1559 static int ahash_init(struct ahash_request *req) 1560 { 1561 struct caam_hash_state *state = ahash_request_ctx(req); 1562 1563 state->update = ahash_update_first; 1564 state->finup = ahash_finup_first; 1565 state->final = ahash_final_no_ctx; 1566 1567 state->current_buf = 0; 1568 state->buf_dma = 0; 1569 state->buflen_0 = 0; 1570 state->buflen_1 = 0; 1571 1572 return 0; 1573 } 1574 1575 static int ahash_update(struct ahash_request *req) 1576 { 1577 struct caam_hash_state *state = ahash_request_ctx(req); 1578 1579 return state->update(req); 1580 } 1581 1582 static int ahash_finup(struct ahash_request *req) 1583 { 1584 struct caam_hash_state *state = ahash_request_ctx(req); 1585 1586 return state->finup(req); 1587 } 1588 1589 static int ahash_final(struct ahash_request *req) 1590 { 1591 struct caam_hash_state *state = ahash_request_ctx(req); 1592 1593 return state->final(req); 1594 } 1595 1596 static int ahash_export(struct ahash_request *req, void *out) 1597 { 1598 struct caam_hash_state *state = ahash_request_ctx(req); 1599 struct caam_export_state *export = out; 1600 int len; 1601 u8 *buf; 1602 1603 if (state->current_buf) { 1604 buf = state->buf_1; 1605 len = state->buflen_1; 1606 } else { 1607 buf = state->buf_0; 1608 len = state->buflen_0; 1609 } 1610 1611 memcpy(export->buf, buf, len); 1612 memcpy(export->caam_ctx, state->caam_ctx, sizeof(export->caam_ctx)); 1613 export->buflen = len; 1614 export->update = state->update; 1615 export->final = state->final; 1616 export->finup = state->finup; 1617 1618 return 0; 1619 } 1620 1621 static int ahash_import(struct ahash_request *req, const void *in) 1622 { 1623 struct caam_hash_state *state = ahash_request_ctx(req); 1624 const struct caam_export_state *export = in; 1625 1626 memset(state, 0, sizeof(*state)); 1627 memcpy(state->buf_0, export->buf, export->buflen); 1628 memcpy(state->caam_ctx, export->caam_ctx, sizeof(state->caam_ctx)); 1629 state->buflen_0 = export->buflen; 1630 state->update = export->update; 1631 state->final = export->final; 1632 state->finup = export->finup; 1633 1634 return 0; 1635 } 1636 1637 struct caam_hash_template { 1638 char name[CRYPTO_MAX_ALG_NAME]; 1639 char driver_name[CRYPTO_MAX_ALG_NAME]; 1640 char hmac_name[CRYPTO_MAX_ALG_NAME]; 1641 char hmac_driver_name[CRYPTO_MAX_ALG_NAME]; 1642 unsigned int blocksize; 1643 struct ahash_alg template_ahash; 1644 u32 alg_type; 1645 u32 alg_op; 1646 }; 1647 1648 /* ahash descriptors */ 1649 static struct caam_hash_template driver_hash[] = { 1650 { 1651 .name = "sha1", 1652 .driver_name = "sha1-caam", 1653 .hmac_name = "hmac(sha1)", 1654 .hmac_driver_name = "hmac-sha1-caam", 1655 .blocksize = SHA1_BLOCK_SIZE, 1656 .template_ahash = { 1657 .init = ahash_init, 1658 .update = ahash_update, 1659 .final = ahash_final, 1660 .finup = ahash_finup, 1661 .digest = ahash_digest, 1662 .export = ahash_export, 1663 .import = ahash_import, 1664 .setkey = ahash_setkey, 1665 .halg = { 1666 .digestsize = SHA1_DIGEST_SIZE, 1667 .statesize = sizeof(struct caam_export_state), 1668 }, 1669 }, 1670 .alg_type = OP_ALG_ALGSEL_SHA1, 1671 .alg_op = OP_ALG_ALGSEL_SHA1 | OP_ALG_AAI_HMAC, 1672 }, { 1673 .name = "sha224", 1674 .driver_name = "sha224-caam", 1675 .hmac_name = "hmac(sha224)", 1676 .hmac_driver_name = "hmac-sha224-caam", 1677 .blocksize = SHA224_BLOCK_SIZE, 1678 .template_ahash = { 1679 .init = ahash_init, 1680 .update = ahash_update, 1681 .final = ahash_final, 1682 .finup = ahash_finup, 1683 .digest = ahash_digest, 1684 .export = ahash_export, 1685 .import = ahash_import, 1686 .setkey = ahash_setkey, 1687 .halg = { 1688 .digestsize = SHA224_DIGEST_SIZE, 1689 .statesize = sizeof(struct caam_export_state), 1690 }, 1691 }, 1692 .alg_type = OP_ALG_ALGSEL_SHA224, 1693 .alg_op = OP_ALG_ALGSEL_SHA224 | OP_ALG_AAI_HMAC, 1694 }, { 1695 .name = "sha256", 1696 .driver_name = "sha256-caam", 1697 .hmac_name = "hmac(sha256)", 1698 .hmac_driver_name = "hmac-sha256-caam", 1699 .blocksize = SHA256_BLOCK_SIZE, 1700 .template_ahash = { 1701 .init = ahash_init, 1702 .update = ahash_update, 1703 .final = ahash_final, 1704 .finup = ahash_finup, 1705 .digest = ahash_digest, 1706 .export = ahash_export, 1707 .import = ahash_import, 1708 .setkey = ahash_setkey, 1709 .halg = { 1710 .digestsize = SHA256_DIGEST_SIZE, 1711 .statesize = sizeof(struct caam_export_state), 1712 }, 1713 }, 1714 .alg_type = OP_ALG_ALGSEL_SHA256, 1715 .alg_op = OP_ALG_ALGSEL_SHA256 | OP_ALG_AAI_HMAC, 1716 }, { 1717 .name = "sha384", 1718 .driver_name = "sha384-caam", 1719 .hmac_name = "hmac(sha384)", 1720 .hmac_driver_name = "hmac-sha384-caam", 1721 .blocksize = SHA384_BLOCK_SIZE, 1722 .template_ahash = { 1723 .init = ahash_init, 1724 .update = ahash_update, 1725 .final = ahash_final, 1726 .finup = ahash_finup, 1727 .digest = ahash_digest, 1728 .export = ahash_export, 1729 .import = ahash_import, 1730 .setkey = ahash_setkey, 1731 .halg = { 1732 .digestsize = SHA384_DIGEST_SIZE, 1733 .statesize = sizeof(struct caam_export_state), 1734 }, 1735 }, 1736 .alg_type = OP_ALG_ALGSEL_SHA384, 1737 .alg_op = OP_ALG_ALGSEL_SHA384 | OP_ALG_AAI_HMAC, 1738 }, { 1739 .name = "sha512", 1740 .driver_name = "sha512-caam", 1741 .hmac_name = "hmac(sha512)", 1742 .hmac_driver_name = "hmac-sha512-caam", 1743 .blocksize = SHA512_BLOCK_SIZE, 1744 .template_ahash = { 1745 .init = ahash_init, 1746 .update = ahash_update, 1747 .final = ahash_final, 1748 .finup = ahash_finup, 1749 .digest = ahash_digest, 1750 .export = ahash_export, 1751 .import = ahash_import, 1752 .setkey = ahash_setkey, 1753 .halg = { 1754 .digestsize = SHA512_DIGEST_SIZE, 1755 .statesize = sizeof(struct caam_export_state), 1756 }, 1757 }, 1758 .alg_type = OP_ALG_ALGSEL_SHA512, 1759 .alg_op = OP_ALG_ALGSEL_SHA512 | OP_ALG_AAI_HMAC, 1760 }, { 1761 .name = "md5", 1762 .driver_name = "md5-caam", 1763 .hmac_name = "hmac(md5)", 1764 .hmac_driver_name = "hmac-md5-caam", 1765 .blocksize = MD5_BLOCK_WORDS * 4, 1766 .template_ahash = { 1767 .init = ahash_init, 1768 .update = ahash_update, 1769 .final = ahash_final, 1770 .finup = ahash_finup, 1771 .digest = ahash_digest, 1772 .export = ahash_export, 1773 .import = ahash_import, 1774 .setkey = ahash_setkey, 1775 .halg = { 1776 .digestsize = MD5_DIGEST_SIZE, 1777 .statesize = sizeof(struct caam_export_state), 1778 }, 1779 }, 1780 .alg_type = OP_ALG_ALGSEL_MD5, 1781 .alg_op = OP_ALG_ALGSEL_MD5 | OP_ALG_AAI_HMAC, 1782 }, 1783 }; 1784 1785 struct caam_hash_alg { 1786 struct list_head entry; 1787 int alg_type; 1788 int alg_op; 1789 struct ahash_alg ahash_alg; 1790 }; 1791 1792 static int caam_hash_cra_init(struct crypto_tfm *tfm) 1793 { 1794 struct crypto_ahash *ahash = __crypto_ahash_cast(tfm); 1795 struct crypto_alg *base = tfm->__crt_alg; 1796 struct hash_alg_common *halg = 1797 container_of(base, struct hash_alg_common, base); 1798 struct ahash_alg *alg = 1799 container_of(halg, struct ahash_alg, halg); 1800 struct caam_hash_alg *caam_hash = 1801 container_of(alg, struct caam_hash_alg, ahash_alg); 1802 struct caam_hash_ctx *ctx = crypto_tfm_ctx(tfm); 1803 /* Sizes for MDHA running digests: MD5, SHA1, 224, 256, 384, 512 */ 1804 static const u8 runninglen[] = { HASH_MSG_LEN + MD5_DIGEST_SIZE, 1805 HASH_MSG_LEN + SHA1_DIGEST_SIZE, 1806 HASH_MSG_LEN + 32, 1807 HASH_MSG_LEN + SHA256_DIGEST_SIZE, 1808 HASH_MSG_LEN + 64, 1809 HASH_MSG_LEN + SHA512_DIGEST_SIZE }; 1810 int ret = 0; 1811 1812 /* 1813 * Get a Job ring from Job Ring driver to ensure in-order 1814 * crypto request processing per tfm 1815 */ 1816 ctx->jrdev = caam_jr_alloc(); 1817 if (IS_ERR(ctx->jrdev)) { 1818 pr_err("Job Ring Device allocation for transform failed\n"); 1819 return PTR_ERR(ctx->jrdev); 1820 } 1821 /* copy descriptor header template value */ 1822 ctx->alg_type = OP_TYPE_CLASS2_ALG | caam_hash->alg_type; 1823 ctx->alg_op = OP_TYPE_CLASS2_ALG | caam_hash->alg_op; 1824 1825 ctx->ctx_len = runninglen[(ctx->alg_op & OP_ALG_ALGSEL_SUBMASK) >> 1826 OP_ALG_ALGSEL_SHIFT]; 1827 1828 crypto_ahash_set_reqsize(__crypto_ahash_cast(tfm), 1829 sizeof(struct caam_hash_state)); 1830 1831 ret = ahash_set_sh_desc(ahash); 1832 1833 return ret; 1834 } 1835 1836 static void caam_hash_cra_exit(struct crypto_tfm *tfm) 1837 { 1838 struct caam_hash_ctx *ctx = crypto_tfm_ctx(tfm); 1839 1840 if (ctx->sh_desc_update_dma && 1841 !dma_mapping_error(ctx->jrdev, ctx->sh_desc_update_dma)) 1842 dma_unmap_single(ctx->jrdev, ctx->sh_desc_update_dma, 1843 desc_bytes(ctx->sh_desc_update), 1844 DMA_TO_DEVICE); 1845 if (ctx->sh_desc_update_first_dma && 1846 !dma_mapping_error(ctx->jrdev, ctx->sh_desc_update_first_dma)) 1847 dma_unmap_single(ctx->jrdev, ctx->sh_desc_update_first_dma, 1848 desc_bytes(ctx->sh_desc_update_first), 1849 DMA_TO_DEVICE); 1850 if (ctx->sh_desc_fin_dma && 1851 !dma_mapping_error(ctx->jrdev, ctx->sh_desc_fin_dma)) 1852 dma_unmap_single(ctx->jrdev, ctx->sh_desc_fin_dma, 1853 desc_bytes(ctx->sh_desc_fin), DMA_TO_DEVICE); 1854 if (ctx->sh_desc_digest_dma && 1855 !dma_mapping_error(ctx->jrdev, ctx->sh_desc_digest_dma)) 1856 dma_unmap_single(ctx->jrdev, ctx->sh_desc_digest_dma, 1857 desc_bytes(ctx->sh_desc_digest), 1858 DMA_TO_DEVICE); 1859 if (ctx->sh_desc_finup_dma && 1860 !dma_mapping_error(ctx->jrdev, ctx->sh_desc_finup_dma)) 1861 dma_unmap_single(ctx->jrdev, ctx->sh_desc_finup_dma, 1862 desc_bytes(ctx->sh_desc_finup), DMA_TO_DEVICE); 1863 1864 caam_jr_free(ctx->jrdev); 1865 } 1866 1867 static void __exit caam_algapi_hash_exit(void) 1868 { 1869 struct caam_hash_alg *t_alg, *n; 1870 1871 if (!hash_list.next) 1872 return; 1873 1874 list_for_each_entry_safe(t_alg, n, &hash_list, entry) { 1875 crypto_unregister_ahash(&t_alg->ahash_alg); 1876 list_del(&t_alg->entry); 1877 kfree(t_alg); 1878 } 1879 } 1880 1881 static struct caam_hash_alg * 1882 caam_hash_alloc(struct caam_hash_template *template, 1883 bool keyed) 1884 { 1885 struct caam_hash_alg *t_alg; 1886 struct ahash_alg *halg; 1887 struct crypto_alg *alg; 1888 1889 t_alg = kzalloc(sizeof(*t_alg), GFP_KERNEL); 1890 if (!t_alg) { 1891 pr_err("failed to allocate t_alg\n"); 1892 return ERR_PTR(-ENOMEM); 1893 } 1894 1895 t_alg->ahash_alg = template->template_ahash; 1896 halg = &t_alg->ahash_alg; 1897 alg = &halg->halg.base; 1898 1899 if (keyed) { 1900 snprintf(alg->cra_name, CRYPTO_MAX_ALG_NAME, "%s", 1901 template->hmac_name); 1902 snprintf(alg->cra_driver_name, CRYPTO_MAX_ALG_NAME, "%s", 1903 template->hmac_driver_name); 1904 } else { 1905 snprintf(alg->cra_name, CRYPTO_MAX_ALG_NAME, "%s", 1906 template->name); 1907 snprintf(alg->cra_driver_name, CRYPTO_MAX_ALG_NAME, "%s", 1908 template->driver_name); 1909 t_alg->ahash_alg.setkey = NULL; 1910 } 1911 alg->cra_module = THIS_MODULE; 1912 alg->cra_init = caam_hash_cra_init; 1913 alg->cra_exit = caam_hash_cra_exit; 1914 alg->cra_ctxsize = sizeof(struct caam_hash_ctx); 1915 alg->cra_priority = CAAM_CRA_PRIORITY; 1916 alg->cra_blocksize = template->blocksize; 1917 alg->cra_alignmask = 0; 1918 alg->cra_flags = CRYPTO_ALG_ASYNC | CRYPTO_ALG_TYPE_AHASH; 1919 alg->cra_type = &crypto_ahash_type; 1920 1921 t_alg->alg_type = template->alg_type; 1922 t_alg->alg_op = template->alg_op; 1923 1924 return t_alg; 1925 } 1926 1927 static int __init caam_algapi_hash_init(void) 1928 { 1929 struct device_node *dev_node; 1930 struct platform_device *pdev; 1931 struct device *ctrldev; 1932 int i = 0, err = 0; 1933 struct caam_drv_private *priv; 1934 unsigned int md_limit = SHA512_DIGEST_SIZE; 1935 u32 cha_inst, cha_vid; 1936 1937 dev_node = of_find_compatible_node(NULL, NULL, "fsl,sec-v4.0"); 1938 if (!dev_node) { 1939 dev_node = of_find_compatible_node(NULL, NULL, "fsl,sec4.0"); 1940 if (!dev_node) 1941 return -ENODEV; 1942 } 1943 1944 pdev = of_find_device_by_node(dev_node); 1945 if (!pdev) { 1946 of_node_put(dev_node); 1947 return -ENODEV; 1948 } 1949 1950 ctrldev = &pdev->dev; 1951 priv = dev_get_drvdata(ctrldev); 1952 of_node_put(dev_node); 1953 1954 /* 1955 * If priv is NULL, it's probably because the caam driver wasn't 1956 * properly initialized (e.g. RNG4 init failed). Thus, bail out here. 1957 */ 1958 if (!priv) 1959 return -ENODEV; 1960 1961 /* 1962 * Register crypto algorithms the device supports. First, identify 1963 * presence and attributes of MD block. 1964 */ 1965 cha_vid = rd_reg32(&priv->ctrl->perfmon.cha_id_ls); 1966 cha_inst = rd_reg32(&priv->ctrl->perfmon.cha_num_ls); 1967 1968 /* 1969 * Skip registration of any hashing algorithms if MD block 1970 * is not present. 1971 */ 1972 if (!((cha_inst & CHA_ID_LS_MD_MASK) >> CHA_ID_LS_MD_SHIFT)) 1973 return -ENODEV; 1974 1975 /* Limit digest size based on LP256 */ 1976 if ((cha_vid & CHA_ID_LS_MD_MASK) == CHA_ID_LS_MD_LP256) 1977 md_limit = SHA256_DIGEST_SIZE; 1978 1979 INIT_LIST_HEAD(&hash_list); 1980 1981 /* register crypto algorithms the device supports */ 1982 for (i = 0; i < ARRAY_SIZE(driver_hash); i++) { 1983 struct caam_hash_alg *t_alg; 1984 struct caam_hash_template *alg = driver_hash + i; 1985 1986 /* If MD size is not supported by device, skip registration */ 1987 if (alg->template_ahash.halg.digestsize > md_limit) 1988 continue; 1989 1990 /* register hmac version */ 1991 t_alg = caam_hash_alloc(alg, true); 1992 if (IS_ERR(t_alg)) { 1993 err = PTR_ERR(t_alg); 1994 pr_warn("%s alg allocation failed\n", alg->driver_name); 1995 continue; 1996 } 1997 1998 err = crypto_register_ahash(&t_alg->ahash_alg); 1999 if (err) { 2000 pr_warn("%s alg registration failed: %d\n", 2001 t_alg->ahash_alg.halg.base.cra_driver_name, 2002 err); 2003 kfree(t_alg); 2004 } else 2005 list_add_tail(&t_alg->entry, &hash_list); 2006 2007 /* register unkeyed version */ 2008 t_alg = caam_hash_alloc(alg, false); 2009 if (IS_ERR(t_alg)) { 2010 err = PTR_ERR(t_alg); 2011 pr_warn("%s alg allocation failed\n", alg->driver_name); 2012 continue; 2013 } 2014 2015 err = crypto_register_ahash(&t_alg->ahash_alg); 2016 if (err) { 2017 pr_warn("%s alg registration failed: %d\n", 2018 t_alg->ahash_alg.halg.base.cra_driver_name, 2019 err); 2020 kfree(t_alg); 2021 } else 2022 list_add_tail(&t_alg->entry, &hash_list); 2023 } 2024 2025 return err; 2026 } 2027 2028 module_init(caam_algapi_hash_init); 2029 module_exit(caam_algapi_hash_exit); 2030 2031 MODULE_LICENSE("GPL"); 2032 MODULE_DESCRIPTION("FSL CAAM support for ahash functions of crypto API"); 2033 MODULE_AUTHOR("Freescale Semiconductor - NMG"); 2034