1 /* 2 * caam - Freescale FSL CAAM support for ahash functions of crypto API 3 * 4 * Copyright 2011 Freescale Semiconductor, Inc. 5 * 6 * Based on caamalg.c crypto API driver. 7 * 8 * relationship of digest job descriptor or first job descriptor after init to 9 * shared descriptors: 10 * 11 * --------------- --------------- 12 * | JobDesc #1 |-------------------->| ShareDesc | 13 * | *(packet 1) | | (hashKey) | 14 * --------------- | (operation) | 15 * --------------- 16 * 17 * relationship of subsequent job descriptors to shared descriptors: 18 * 19 * --------------- --------------- 20 * | JobDesc #2 |-------------------->| ShareDesc | 21 * | *(packet 2) | |------------->| (hashKey) | 22 * --------------- | |-------->| (operation) | 23 * . | | | (load ctx2) | 24 * . | | --------------- 25 * --------------- | | 26 * | JobDesc #3 |------| | 27 * | *(packet 3) | | 28 * --------------- | 29 * . | 30 * . | 31 * --------------- | 32 * | JobDesc #4 |------------ 33 * | *(packet 4) | 34 * --------------- 35 * 36 * The SharedDesc never changes for a connection unless rekeyed, but 37 * each packet will likely be in a different place. So all we need 38 * to know to process the packet is where the input is, where the 39 * output goes, and what context we want to process with. Context is 40 * in the SharedDesc, packet references in the JobDesc. 41 * 42 * So, a job desc looks like: 43 * 44 * --------------------- 45 * | Header | 46 * | ShareDesc Pointer | 47 * | SEQ_OUT_PTR | 48 * | (output buffer) | 49 * | (output length) | 50 * | SEQ_IN_PTR | 51 * | (input buffer) | 52 * | (input length) | 53 * --------------------- 54 */ 55 56 #include "compat.h" 57 58 #include "regs.h" 59 #include "intern.h" 60 #include "desc_constr.h" 61 #include "jr.h" 62 #include "error.h" 63 #include "sg_sw_sec4.h" 64 #include "key_gen.h" 65 66 #define CAAM_CRA_PRIORITY 3000 67 68 /* max hash key is max split key size */ 69 #define CAAM_MAX_HASH_KEY_SIZE (SHA512_DIGEST_SIZE * 2) 70 71 #define CAAM_MAX_HASH_BLOCK_SIZE SHA512_BLOCK_SIZE 72 #define CAAM_MAX_HASH_DIGEST_SIZE SHA512_DIGEST_SIZE 73 74 /* length of descriptors text */ 75 #define DESC_AHASH_BASE (3 * CAAM_CMD_SZ) 76 #define DESC_AHASH_UPDATE_LEN (6 * CAAM_CMD_SZ) 77 #define DESC_AHASH_UPDATE_FIRST_LEN (DESC_AHASH_BASE + 4 * CAAM_CMD_SZ) 78 #define DESC_AHASH_FINAL_LEN (DESC_AHASH_BASE + 5 * CAAM_CMD_SZ) 79 #define DESC_AHASH_FINUP_LEN (DESC_AHASH_BASE + 5 * CAAM_CMD_SZ) 80 #define DESC_AHASH_DIGEST_LEN (DESC_AHASH_BASE + 4 * CAAM_CMD_SZ) 81 82 #define DESC_HASH_MAX_USED_BYTES (DESC_AHASH_FINAL_LEN + \ 83 CAAM_MAX_HASH_KEY_SIZE) 84 #define DESC_HASH_MAX_USED_LEN (DESC_HASH_MAX_USED_BYTES / CAAM_CMD_SZ) 85 86 /* caam context sizes for hashes: running digest + 8 */ 87 #define HASH_MSG_LEN 8 88 #define MAX_CTX_LEN (HASH_MSG_LEN + SHA512_DIGEST_SIZE) 89 90 #ifdef DEBUG 91 /* for print_hex_dumps with line references */ 92 #define debug(format, arg...) printk(format, arg) 93 #else 94 #define debug(format, arg...) 95 #endif 96 97 98 static struct list_head hash_list; 99 100 /* ahash per-session context */ 101 struct caam_hash_ctx { 102 u32 sh_desc_update[DESC_HASH_MAX_USED_LEN] ____cacheline_aligned; 103 u32 sh_desc_update_first[DESC_HASH_MAX_USED_LEN] ____cacheline_aligned; 104 u32 sh_desc_fin[DESC_HASH_MAX_USED_LEN] ____cacheline_aligned; 105 u32 sh_desc_digest[DESC_HASH_MAX_USED_LEN] ____cacheline_aligned; 106 dma_addr_t sh_desc_update_dma ____cacheline_aligned; 107 dma_addr_t sh_desc_update_first_dma; 108 dma_addr_t sh_desc_fin_dma; 109 dma_addr_t sh_desc_digest_dma; 110 struct device *jrdev; 111 u8 key[CAAM_MAX_HASH_KEY_SIZE]; 112 int ctx_len; 113 struct alginfo adata; 114 }; 115 116 /* ahash state */ 117 struct caam_hash_state { 118 dma_addr_t buf_dma; 119 dma_addr_t ctx_dma; 120 u8 buf_0[CAAM_MAX_HASH_BLOCK_SIZE] ____cacheline_aligned; 121 int buflen_0; 122 u8 buf_1[CAAM_MAX_HASH_BLOCK_SIZE] ____cacheline_aligned; 123 int buflen_1; 124 u8 caam_ctx[MAX_CTX_LEN] ____cacheline_aligned; 125 int (*update)(struct ahash_request *req); 126 int (*final)(struct ahash_request *req); 127 int (*finup)(struct ahash_request *req); 128 int current_buf; 129 }; 130 131 struct caam_export_state { 132 u8 buf[CAAM_MAX_HASH_BLOCK_SIZE]; 133 u8 caam_ctx[MAX_CTX_LEN]; 134 int buflen; 135 int (*update)(struct ahash_request *req); 136 int (*final)(struct ahash_request *req); 137 int (*finup)(struct ahash_request *req); 138 }; 139 140 static inline void switch_buf(struct caam_hash_state *state) 141 { 142 state->current_buf ^= 1; 143 } 144 145 static inline u8 *current_buf(struct caam_hash_state *state) 146 { 147 return state->current_buf ? state->buf_1 : state->buf_0; 148 } 149 150 static inline u8 *alt_buf(struct caam_hash_state *state) 151 { 152 return state->current_buf ? state->buf_0 : state->buf_1; 153 } 154 155 static inline int *current_buflen(struct caam_hash_state *state) 156 { 157 return state->current_buf ? &state->buflen_1 : &state->buflen_0; 158 } 159 160 static inline int *alt_buflen(struct caam_hash_state *state) 161 { 162 return state->current_buf ? &state->buflen_0 : &state->buflen_1; 163 } 164 165 /* Common job descriptor seq in/out ptr routines */ 166 167 /* Map state->caam_ctx, and append seq_out_ptr command that points to it */ 168 static inline int map_seq_out_ptr_ctx(u32 *desc, struct device *jrdev, 169 struct caam_hash_state *state, 170 int ctx_len) 171 { 172 state->ctx_dma = dma_map_single(jrdev, state->caam_ctx, 173 ctx_len, DMA_FROM_DEVICE); 174 if (dma_mapping_error(jrdev, state->ctx_dma)) { 175 dev_err(jrdev, "unable to map ctx\n"); 176 state->ctx_dma = 0; 177 return -ENOMEM; 178 } 179 180 append_seq_out_ptr(desc, state->ctx_dma, ctx_len, 0); 181 182 return 0; 183 } 184 185 /* Map req->result, and append seq_out_ptr command that points to it */ 186 static inline dma_addr_t map_seq_out_ptr_result(u32 *desc, struct device *jrdev, 187 u8 *result, int digestsize) 188 { 189 dma_addr_t dst_dma; 190 191 dst_dma = dma_map_single(jrdev, result, digestsize, DMA_FROM_DEVICE); 192 append_seq_out_ptr(desc, dst_dma, digestsize, 0); 193 194 return dst_dma; 195 } 196 197 /* Map current buffer in state (if length > 0) and put it in link table */ 198 static inline int buf_map_to_sec4_sg(struct device *jrdev, 199 struct sec4_sg_entry *sec4_sg, 200 struct caam_hash_state *state) 201 { 202 int buflen = *current_buflen(state); 203 204 if (!buflen) 205 return 0; 206 207 state->buf_dma = dma_map_single(jrdev, current_buf(state), buflen, 208 DMA_TO_DEVICE); 209 if (dma_mapping_error(jrdev, state->buf_dma)) { 210 dev_err(jrdev, "unable to map buf\n"); 211 state->buf_dma = 0; 212 return -ENOMEM; 213 } 214 215 dma_to_sec4_sg_one(sec4_sg, state->buf_dma, buflen, 0); 216 217 return 0; 218 } 219 220 /* Map state->caam_ctx, and add it to link table */ 221 static inline int ctx_map_to_sec4_sg(struct device *jrdev, 222 struct caam_hash_state *state, int ctx_len, 223 struct sec4_sg_entry *sec4_sg, u32 flag) 224 { 225 state->ctx_dma = dma_map_single(jrdev, state->caam_ctx, ctx_len, flag); 226 if (dma_mapping_error(jrdev, state->ctx_dma)) { 227 dev_err(jrdev, "unable to map ctx\n"); 228 state->ctx_dma = 0; 229 return -ENOMEM; 230 } 231 232 dma_to_sec4_sg_one(sec4_sg, state->ctx_dma, ctx_len, 0); 233 234 return 0; 235 } 236 237 /* 238 * For ahash update, final and finup (import_ctx = true) 239 * import context, read and write to seqout 240 * For ahash firsts and digest (import_ctx = false) 241 * read and write to seqout 242 */ 243 static inline void ahash_gen_sh_desc(u32 *desc, u32 state, int digestsize, 244 struct caam_hash_ctx *ctx, bool import_ctx) 245 { 246 u32 op = ctx->adata.algtype; 247 u32 *skip_key_load; 248 249 init_sh_desc(desc, HDR_SHARE_SERIAL); 250 251 /* Append key if it has been set; ahash update excluded */ 252 if ((state != OP_ALG_AS_UPDATE) && (ctx->adata.keylen)) { 253 /* Skip key loading if already shared */ 254 skip_key_load = append_jump(desc, JUMP_JSL | JUMP_TEST_ALL | 255 JUMP_COND_SHRD); 256 257 append_key_as_imm(desc, ctx->key, ctx->adata.keylen_pad, 258 ctx->adata.keylen, CLASS_2 | 259 KEY_DEST_MDHA_SPLIT | KEY_ENC); 260 261 set_jump_tgt_here(desc, skip_key_load); 262 263 op |= OP_ALG_AAI_HMAC_PRECOMP; 264 } 265 266 /* If needed, import context from software */ 267 if (import_ctx) 268 append_seq_load(desc, ctx->ctx_len, LDST_CLASS_2_CCB | 269 LDST_SRCDST_BYTE_CONTEXT); 270 271 /* Class 2 operation */ 272 append_operation(desc, op | state | OP_ALG_ENCRYPT); 273 274 /* 275 * Load from buf and/or src and write to req->result or state->context 276 * Calculate remaining bytes to read 277 */ 278 append_math_add(desc, VARSEQINLEN, SEQINLEN, REG0, CAAM_CMD_SZ); 279 /* Read remaining bytes */ 280 append_seq_fifo_load(desc, 0, FIFOLD_CLASS_CLASS2 | FIFOLD_TYPE_LAST2 | 281 FIFOLD_TYPE_MSG | KEY_VLF); 282 /* Store class2 context bytes */ 283 append_seq_store(desc, digestsize, LDST_CLASS_2_CCB | 284 LDST_SRCDST_BYTE_CONTEXT); 285 } 286 287 static int ahash_set_sh_desc(struct crypto_ahash *ahash) 288 { 289 struct caam_hash_ctx *ctx = crypto_ahash_ctx(ahash); 290 int digestsize = crypto_ahash_digestsize(ahash); 291 struct device *jrdev = ctx->jrdev; 292 u32 *desc; 293 294 /* ahash_update shared descriptor */ 295 desc = ctx->sh_desc_update; 296 ahash_gen_sh_desc(desc, OP_ALG_AS_UPDATE, ctx->ctx_len, ctx, true); 297 dma_sync_single_for_device(jrdev, ctx->sh_desc_update_dma, 298 desc_bytes(desc), DMA_TO_DEVICE); 299 #ifdef DEBUG 300 print_hex_dump(KERN_ERR, 301 "ahash update shdesc@"__stringify(__LINE__)": ", 302 DUMP_PREFIX_ADDRESS, 16, 4, desc, desc_bytes(desc), 1); 303 #endif 304 305 /* ahash_update_first shared descriptor */ 306 desc = ctx->sh_desc_update_first; 307 ahash_gen_sh_desc(desc, OP_ALG_AS_INIT, ctx->ctx_len, ctx, false); 308 dma_sync_single_for_device(jrdev, ctx->sh_desc_update_first_dma, 309 desc_bytes(desc), DMA_TO_DEVICE); 310 #ifdef DEBUG 311 print_hex_dump(KERN_ERR, 312 "ahash update first shdesc@"__stringify(__LINE__)": ", 313 DUMP_PREFIX_ADDRESS, 16, 4, desc, desc_bytes(desc), 1); 314 #endif 315 316 /* ahash_final shared descriptor */ 317 desc = ctx->sh_desc_fin; 318 ahash_gen_sh_desc(desc, OP_ALG_AS_FINALIZE, digestsize, ctx, true); 319 dma_sync_single_for_device(jrdev, ctx->sh_desc_fin_dma, 320 desc_bytes(desc), DMA_TO_DEVICE); 321 #ifdef DEBUG 322 print_hex_dump(KERN_ERR, "ahash final shdesc@"__stringify(__LINE__)": ", 323 DUMP_PREFIX_ADDRESS, 16, 4, desc, 324 desc_bytes(desc), 1); 325 #endif 326 327 /* ahash_digest shared descriptor */ 328 desc = ctx->sh_desc_digest; 329 ahash_gen_sh_desc(desc, OP_ALG_AS_INITFINAL, digestsize, ctx, false); 330 dma_sync_single_for_device(jrdev, ctx->sh_desc_digest_dma, 331 desc_bytes(desc), DMA_TO_DEVICE); 332 #ifdef DEBUG 333 print_hex_dump(KERN_ERR, 334 "ahash digest shdesc@"__stringify(__LINE__)": ", 335 DUMP_PREFIX_ADDRESS, 16, 4, desc, 336 desc_bytes(desc), 1); 337 #endif 338 339 return 0; 340 } 341 342 /* Digest hash size if it is too large */ 343 static int hash_digest_key(struct caam_hash_ctx *ctx, const u8 *key_in, 344 u32 *keylen, u8 *key_out, u32 digestsize) 345 { 346 struct device *jrdev = ctx->jrdev; 347 u32 *desc; 348 struct split_key_result result; 349 dma_addr_t src_dma, dst_dma; 350 int ret; 351 352 desc = kmalloc(CAAM_CMD_SZ * 8 + CAAM_PTR_SZ * 2, GFP_KERNEL | GFP_DMA); 353 if (!desc) { 354 dev_err(jrdev, "unable to allocate key input memory\n"); 355 return -ENOMEM; 356 } 357 358 init_job_desc(desc, 0); 359 360 src_dma = dma_map_single(jrdev, (void *)key_in, *keylen, 361 DMA_TO_DEVICE); 362 if (dma_mapping_error(jrdev, src_dma)) { 363 dev_err(jrdev, "unable to map key input memory\n"); 364 kfree(desc); 365 return -ENOMEM; 366 } 367 dst_dma = dma_map_single(jrdev, (void *)key_out, digestsize, 368 DMA_FROM_DEVICE); 369 if (dma_mapping_error(jrdev, dst_dma)) { 370 dev_err(jrdev, "unable to map key output memory\n"); 371 dma_unmap_single(jrdev, src_dma, *keylen, DMA_TO_DEVICE); 372 kfree(desc); 373 return -ENOMEM; 374 } 375 376 /* Job descriptor to perform unkeyed hash on key_in */ 377 append_operation(desc, ctx->adata.algtype | OP_ALG_ENCRYPT | 378 OP_ALG_AS_INITFINAL); 379 append_seq_in_ptr(desc, src_dma, *keylen, 0); 380 append_seq_fifo_load(desc, *keylen, FIFOLD_CLASS_CLASS2 | 381 FIFOLD_TYPE_LAST2 | FIFOLD_TYPE_MSG); 382 append_seq_out_ptr(desc, dst_dma, digestsize, 0); 383 append_seq_store(desc, digestsize, LDST_CLASS_2_CCB | 384 LDST_SRCDST_BYTE_CONTEXT); 385 386 #ifdef DEBUG 387 print_hex_dump(KERN_ERR, "key_in@"__stringify(__LINE__)": ", 388 DUMP_PREFIX_ADDRESS, 16, 4, key_in, *keylen, 1); 389 print_hex_dump(KERN_ERR, "jobdesc@"__stringify(__LINE__)": ", 390 DUMP_PREFIX_ADDRESS, 16, 4, desc, desc_bytes(desc), 1); 391 #endif 392 393 result.err = 0; 394 init_completion(&result.completion); 395 396 ret = caam_jr_enqueue(jrdev, desc, split_key_done, &result); 397 if (!ret) { 398 /* in progress */ 399 wait_for_completion(&result.completion); 400 ret = result.err; 401 #ifdef DEBUG 402 print_hex_dump(KERN_ERR, 403 "digested key@"__stringify(__LINE__)": ", 404 DUMP_PREFIX_ADDRESS, 16, 4, key_in, 405 digestsize, 1); 406 #endif 407 } 408 dma_unmap_single(jrdev, src_dma, *keylen, DMA_TO_DEVICE); 409 dma_unmap_single(jrdev, dst_dma, digestsize, DMA_FROM_DEVICE); 410 411 *keylen = digestsize; 412 413 kfree(desc); 414 415 return ret; 416 } 417 418 static int ahash_setkey(struct crypto_ahash *ahash, 419 const u8 *key, unsigned int keylen) 420 { 421 struct caam_hash_ctx *ctx = crypto_ahash_ctx(ahash); 422 int blocksize = crypto_tfm_alg_blocksize(&ahash->base); 423 int digestsize = crypto_ahash_digestsize(ahash); 424 int ret; 425 u8 *hashed_key = NULL; 426 427 #ifdef DEBUG 428 printk(KERN_ERR "keylen %d\n", keylen); 429 #endif 430 431 if (keylen > blocksize) { 432 hashed_key = kmalloc_array(digestsize, 433 sizeof(*hashed_key), 434 GFP_KERNEL | GFP_DMA); 435 if (!hashed_key) 436 return -ENOMEM; 437 ret = hash_digest_key(ctx, key, &keylen, hashed_key, 438 digestsize); 439 if (ret) 440 goto bad_free_key; 441 key = hashed_key; 442 } 443 444 ret = gen_split_key(ctx->jrdev, ctx->key, &ctx->adata, key, keylen, 445 CAAM_MAX_HASH_KEY_SIZE); 446 if (ret) 447 goto bad_free_key; 448 449 #ifdef DEBUG 450 print_hex_dump(KERN_ERR, "ctx.key@"__stringify(__LINE__)": ", 451 DUMP_PREFIX_ADDRESS, 16, 4, ctx->key, 452 ctx->adata.keylen_pad, 1); 453 #endif 454 455 kfree(hashed_key); 456 return ahash_set_sh_desc(ahash); 457 bad_free_key: 458 kfree(hashed_key); 459 crypto_ahash_set_flags(ahash, CRYPTO_TFM_RES_BAD_KEY_LEN); 460 return -EINVAL; 461 } 462 463 /* 464 * ahash_edesc - s/w-extended ahash descriptor 465 * @dst_dma: physical mapped address of req->result 466 * @sec4_sg_dma: physical mapped address of h/w link table 467 * @src_nents: number of segments in input scatterlist 468 * @sec4_sg_bytes: length of dma mapped sec4_sg space 469 * @hw_desc: the h/w job descriptor followed by any referenced link tables 470 * @sec4_sg: h/w link table 471 */ 472 struct ahash_edesc { 473 dma_addr_t dst_dma; 474 dma_addr_t sec4_sg_dma; 475 int src_nents; 476 int sec4_sg_bytes; 477 u32 hw_desc[DESC_JOB_IO_LEN / sizeof(u32)] ____cacheline_aligned; 478 struct sec4_sg_entry sec4_sg[0]; 479 }; 480 481 static inline void ahash_unmap(struct device *dev, 482 struct ahash_edesc *edesc, 483 struct ahash_request *req, int dst_len) 484 { 485 struct caam_hash_state *state = ahash_request_ctx(req); 486 487 if (edesc->src_nents) 488 dma_unmap_sg(dev, req->src, edesc->src_nents, DMA_TO_DEVICE); 489 if (edesc->dst_dma) 490 dma_unmap_single(dev, edesc->dst_dma, dst_len, DMA_FROM_DEVICE); 491 492 if (edesc->sec4_sg_bytes) 493 dma_unmap_single(dev, edesc->sec4_sg_dma, 494 edesc->sec4_sg_bytes, DMA_TO_DEVICE); 495 496 if (state->buf_dma) { 497 dma_unmap_single(dev, state->buf_dma, *current_buflen(state), 498 DMA_TO_DEVICE); 499 state->buf_dma = 0; 500 } 501 } 502 503 static inline void ahash_unmap_ctx(struct device *dev, 504 struct ahash_edesc *edesc, 505 struct ahash_request *req, int dst_len, u32 flag) 506 { 507 struct crypto_ahash *ahash = crypto_ahash_reqtfm(req); 508 struct caam_hash_ctx *ctx = crypto_ahash_ctx(ahash); 509 struct caam_hash_state *state = ahash_request_ctx(req); 510 511 if (state->ctx_dma) { 512 dma_unmap_single(dev, state->ctx_dma, ctx->ctx_len, flag); 513 state->ctx_dma = 0; 514 } 515 ahash_unmap(dev, edesc, req, dst_len); 516 } 517 518 static void ahash_done(struct device *jrdev, u32 *desc, u32 err, 519 void *context) 520 { 521 struct ahash_request *req = context; 522 struct ahash_edesc *edesc; 523 struct crypto_ahash *ahash = crypto_ahash_reqtfm(req); 524 int digestsize = crypto_ahash_digestsize(ahash); 525 #ifdef DEBUG 526 struct caam_hash_ctx *ctx = crypto_ahash_ctx(ahash); 527 struct caam_hash_state *state = ahash_request_ctx(req); 528 529 dev_err(jrdev, "%s %d: err 0x%x\n", __func__, __LINE__, err); 530 #endif 531 532 edesc = container_of(desc, struct ahash_edesc, hw_desc[0]); 533 if (err) 534 caam_jr_strstatus(jrdev, err); 535 536 ahash_unmap(jrdev, edesc, req, digestsize); 537 kfree(edesc); 538 539 #ifdef DEBUG 540 print_hex_dump(KERN_ERR, "ctx@"__stringify(__LINE__)": ", 541 DUMP_PREFIX_ADDRESS, 16, 4, state->caam_ctx, 542 ctx->ctx_len, 1); 543 if (req->result) 544 print_hex_dump(KERN_ERR, "result@"__stringify(__LINE__)": ", 545 DUMP_PREFIX_ADDRESS, 16, 4, req->result, 546 digestsize, 1); 547 #endif 548 549 req->base.complete(&req->base, err); 550 } 551 552 static void ahash_done_bi(struct device *jrdev, u32 *desc, u32 err, 553 void *context) 554 { 555 struct ahash_request *req = context; 556 struct ahash_edesc *edesc; 557 struct crypto_ahash *ahash = crypto_ahash_reqtfm(req); 558 struct caam_hash_ctx *ctx = crypto_ahash_ctx(ahash); 559 struct caam_hash_state *state = ahash_request_ctx(req); 560 #ifdef DEBUG 561 int digestsize = crypto_ahash_digestsize(ahash); 562 563 dev_err(jrdev, "%s %d: err 0x%x\n", __func__, __LINE__, err); 564 #endif 565 566 edesc = container_of(desc, struct ahash_edesc, hw_desc[0]); 567 if (err) 568 caam_jr_strstatus(jrdev, err); 569 570 ahash_unmap_ctx(jrdev, edesc, req, ctx->ctx_len, DMA_BIDIRECTIONAL); 571 switch_buf(state); 572 kfree(edesc); 573 574 #ifdef DEBUG 575 print_hex_dump(KERN_ERR, "ctx@"__stringify(__LINE__)": ", 576 DUMP_PREFIX_ADDRESS, 16, 4, state->caam_ctx, 577 ctx->ctx_len, 1); 578 if (req->result) 579 print_hex_dump(KERN_ERR, "result@"__stringify(__LINE__)": ", 580 DUMP_PREFIX_ADDRESS, 16, 4, req->result, 581 digestsize, 1); 582 #endif 583 584 req->base.complete(&req->base, err); 585 } 586 587 static void ahash_done_ctx_src(struct device *jrdev, u32 *desc, u32 err, 588 void *context) 589 { 590 struct ahash_request *req = context; 591 struct ahash_edesc *edesc; 592 struct crypto_ahash *ahash = crypto_ahash_reqtfm(req); 593 int digestsize = crypto_ahash_digestsize(ahash); 594 #ifdef DEBUG 595 struct caam_hash_ctx *ctx = crypto_ahash_ctx(ahash); 596 struct caam_hash_state *state = ahash_request_ctx(req); 597 598 dev_err(jrdev, "%s %d: err 0x%x\n", __func__, __LINE__, err); 599 #endif 600 601 edesc = container_of(desc, struct ahash_edesc, hw_desc[0]); 602 if (err) 603 caam_jr_strstatus(jrdev, err); 604 605 ahash_unmap_ctx(jrdev, edesc, req, digestsize, DMA_TO_DEVICE); 606 kfree(edesc); 607 608 #ifdef DEBUG 609 print_hex_dump(KERN_ERR, "ctx@"__stringify(__LINE__)": ", 610 DUMP_PREFIX_ADDRESS, 16, 4, state->caam_ctx, 611 ctx->ctx_len, 1); 612 if (req->result) 613 print_hex_dump(KERN_ERR, "result@"__stringify(__LINE__)": ", 614 DUMP_PREFIX_ADDRESS, 16, 4, req->result, 615 digestsize, 1); 616 #endif 617 618 req->base.complete(&req->base, err); 619 } 620 621 static void ahash_done_ctx_dst(struct device *jrdev, u32 *desc, u32 err, 622 void *context) 623 { 624 struct ahash_request *req = context; 625 struct ahash_edesc *edesc; 626 struct crypto_ahash *ahash = crypto_ahash_reqtfm(req); 627 struct caam_hash_ctx *ctx = crypto_ahash_ctx(ahash); 628 struct caam_hash_state *state = ahash_request_ctx(req); 629 #ifdef DEBUG 630 int digestsize = crypto_ahash_digestsize(ahash); 631 632 dev_err(jrdev, "%s %d: err 0x%x\n", __func__, __LINE__, err); 633 #endif 634 635 edesc = container_of(desc, struct ahash_edesc, hw_desc[0]); 636 if (err) 637 caam_jr_strstatus(jrdev, err); 638 639 ahash_unmap_ctx(jrdev, edesc, req, ctx->ctx_len, DMA_FROM_DEVICE); 640 switch_buf(state); 641 kfree(edesc); 642 643 #ifdef DEBUG 644 print_hex_dump(KERN_ERR, "ctx@"__stringify(__LINE__)": ", 645 DUMP_PREFIX_ADDRESS, 16, 4, state->caam_ctx, 646 ctx->ctx_len, 1); 647 if (req->result) 648 print_hex_dump(KERN_ERR, "result@"__stringify(__LINE__)": ", 649 DUMP_PREFIX_ADDRESS, 16, 4, req->result, 650 digestsize, 1); 651 #endif 652 653 req->base.complete(&req->base, err); 654 } 655 656 /* 657 * Allocate an enhanced descriptor, which contains the hardware descriptor 658 * and space for hardware scatter table containing sg_num entries. 659 */ 660 static struct ahash_edesc *ahash_edesc_alloc(struct caam_hash_ctx *ctx, 661 int sg_num, u32 *sh_desc, 662 dma_addr_t sh_desc_dma, 663 gfp_t flags) 664 { 665 struct ahash_edesc *edesc; 666 unsigned int sg_size = sg_num * sizeof(struct sec4_sg_entry); 667 668 edesc = kzalloc(sizeof(*edesc) + sg_size, GFP_DMA | flags); 669 if (!edesc) { 670 dev_err(ctx->jrdev, "could not allocate extended descriptor\n"); 671 return NULL; 672 } 673 674 init_job_desc_shared(edesc->hw_desc, sh_desc_dma, desc_len(sh_desc), 675 HDR_SHARE_DEFER | HDR_REVERSE); 676 677 return edesc; 678 } 679 680 static int ahash_edesc_add_src(struct caam_hash_ctx *ctx, 681 struct ahash_edesc *edesc, 682 struct ahash_request *req, int nents, 683 unsigned int first_sg, 684 unsigned int first_bytes, size_t to_hash) 685 { 686 dma_addr_t src_dma; 687 u32 options; 688 689 if (nents > 1 || first_sg) { 690 struct sec4_sg_entry *sg = edesc->sec4_sg; 691 unsigned int sgsize = sizeof(*sg) * (first_sg + nents); 692 693 sg_to_sec4_sg_last(req->src, nents, sg + first_sg, 0); 694 695 src_dma = dma_map_single(ctx->jrdev, sg, sgsize, DMA_TO_DEVICE); 696 if (dma_mapping_error(ctx->jrdev, src_dma)) { 697 dev_err(ctx->jrdev, "unable to map S/G table\n"); 698 return -ENOMEM; 699 } 700 701 edesc->sec4_sg_bytes = sgsize; 702 edesc->sec4_sg_dma = src_dma; 703 options = LDST_SGF; 704 } else { 705 src_dma = sg_dma_address(req->src); 706 options = 0; 707 } 708 709 append_seq_in_ptr(edesc->hw_desc, src_dma, first_bytes + to_hash, 710 options); 711 712 return 0; 713 } 714 715 /* submit update job descriptor */ 716 static int ahash_update_ctx(struct ahash_request *req) 717 { 718 struct crypto_ahash *ahash = crypto_ahash_reqtfm(req); 719 struct caam_hash_ctx *ctx = crypto_ahash_ctx(ahash); 720 struct caam_hash_state *state = ahash_request_ctx(req); 721 struct device *jrdev = ctx->jrdev; 722 gfp_t flags = (req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP) ? 723 GFP_KERNEL : GFP_ATOMIC; 724 u8 *buf = current_buf(state); 725 int *buflen = current_buflen(state); 726 u8 *next_buf = alt_buf(state); 727 int *next_buflen = alt_buflen(state), last_buflen; 728 int in_len = *buflen + req->nbytes, to_hash; 729 u32 *desc; 730 int src_nents, mapped_nents, sec4_sg_bytes, sec4_sg_src_index; 731 struct ahash_edesc *edesc; 732 int ret = 0; 733 734 last_buflen = *next_buflen; 735 *next_buflen = in_len & (crypto_tfm_alg_blocksize(&ahash->base) - 1); 736 to_hash = in_len - *next_buflen; 737 738 if (to_hash) { 739 src_nents = sg_nents_for_len(req->src, 740 req->nbytes - (*next_buflen)); 741 if (src_nents < 0) { 742 dev_err(jrdev, "Invalid number of src SG.\n"); 743 return src_nents; 744 } 745 746 if (src_nents) { 747 mapped_nents = dma_map_sg(jrdev, req->src, src_nents, 748 DMA_TO_DEVICE); 749 if (!mapped_nents) { 750 dev_err(jrdev, "unable to DMA map source\n"); 751 return -ENOMEM; 752 } 753 } else { 754 mapped_nents = 0; 755 } 756 757 sec4_sg_src_index = 1 + (*buflen ? 1 : 0); 758 sec4_sg_bytes = (sec4_sg_src_index + mapped_nents) * 759 sizeof(struct sec4_sg_entry); 760 761 /* 762 * allocate space for base edesc and hw desc commands, 763 * link tables 764 */ 765 edesc = ahash_edesc_alloc(ctx, sec4_sg_src_index + mapped_nents, 766 ctx->sh_desc_update, 767 ctx->sh_desc_update_dma, flags); 768 if (!edesc) { 769 dma_unmap_sg(jrdev, req->src, src_nents, DMA_TO_DEVICE); 770 return -ENOMEM; 771 } 772 773 edesc->src_nents = src_nents; 774 edesc->sec4_sg_bytes = sec4_sg_bytes; 775 776 ret = ctx_map_to_sec4_sg(jrdev, state, ctx->ctx_len, 777 edesc->sec4_sg, DMA_BIDIRECTIONAL); 778 if (ret) 779 goto unmap_ctx; 780 781 ret = buf_map_to_sec4_sg(jrdev, edesc->sec4_sg + 1, state); 782 if (ret) 783 goto unmap_ctx; 784 785 if (mapped_nents) { 786 sg_to_sec4_sg_last(req->src, mapped_nents, 787 edesc->sec4_sg + sec4_sg_src_index, 788 0); 789 if (*next_buflen) 790 scatterwalk_map_and_copy(next_buf, req->src, 791 to_hash - *buflen, 792 *next_buflen, 0); 793 } else { 794 sg_to_sec4_set_last(edesc->sec4_sg + sec4_sg_src_index - 795 1); 796 } 797 798 desc = edesc->hw_desc; 799 800 edesc->sec4_sg_dma = dma_map_single(jrdev, edesc->sec4_sg, 801 sec4_sg_bytes, 802 DMA_TO_DEVICE); 803 if (dma_mapping_error(jrdev, edesc->sec4_sg_dma)) { 804 dev_err(jrdev, "unable to map S/G table\n"); 805 ret = -ENOMEM; 806 goto unmap_ctx; 807 } 808 809 append_seq_in_ptr(desc, edesc->sec4_sg_dma, ctx->ctx_len + 810 to_hash, LDST_SGF); 811 812 append_seq_out_ptr(desc, state->ctx_dma, ctx->ctx_len, 0); 813 814 #ifdef DEBUG 815 print_hex_dump(KERN_ERR, "jobdesc@"__stringify(__LINE__)": ", 816 DUMP_PREFIX_ADDRESS, 16, 4, desc, 817 desc_bytes(desc), 1); 818 #endif 819 820 ret = caam_jr_enqueue(jrdev, desc, ahash_done_bi, req); 821 if (ret) 822 goto unmap_ctx; 823 824 ret = -EINPROGRESS; 825 } else if (*next_buflen) { 826 scatterwalk_map_and_copy(buf + *buflen, req->src, 0, 827 req->nbytes, 0); 828 *buflen = *next_buflen; 829 *next_buflen = last_buflen; 830 } 831 #ifdef DEBUG 832 print_hex_dump(KERN_ERR, "buf@"__stringify(__LINE__)": ", 833 DUMP_PREFIX_ADDRESS, 16, 4, buf, *buflen, 1); 834 print_hex_dump(KERN_ERR, "next buf@"__stringify(__LINE__)": ", 835 DUMP_PREFIX_ADDRESS, 16, 4, next_buf, 836 *next_buflen, 1); 837 #endif 838 839 return ret; 840 unmap_ctx: 841 ahash_unmap_ctx(jrdev, edesc, req, ctx->ctx_len, DMA_BIDIRECTIONAL); 842 kfree(edesc); 843 return ret; 844 } 845 846 static int ahash_final_ctx(struct ahash_request *req) 847 { 848 struct crypto_ahash *ahash = crypto_ahash_reqtfm(req); 849 struct caam_hash_ctx *ctx = crypto_ahash_ctx(ahash); 850 struct caam_hash_state *state = ahash_request_ctx(req); 851 struct device *jrdev = ctx->jrdev; 852 gfp_t flags = (req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP) ? 853 GFP_KERNEL : GFP_ATOMIC; 854 int buflen = *current_buflen(state); 855 u32 *desc; 856 int sec4_sg_bytes, sec4_sg_src_index; 857 int digestsize = crypto_ahash_digestsize(ahash); 858 struct ahash_edesc *edesc; 859 int ret; 860 861 sec4_sg_src_index = 1 + (buflen ? 1 : 0); 862 sec4_sg_bytes = sec4_sg_src_index * sizeof(struct sec4_sg_entry); 863 864 /* allocate space for base edesc and hw desc commands, link tables */ 865 edesc = ahash_edesc_alloc(ctx, sec4_sg_src_index, 866 ctx->sh_desc_fin, ctx->sh_desc_fin_dma, 867 flags); 868 if (!edesc) 869 return -ENOMEM; 870 871 desc = edesc->hw_desc; 872 873 edesc->sec4_sg_bytes = sec4_sg_bytes; 874 875 ret = ctx_map_to_sec4_sg(jrdev, state, ctx->ctx_len, 876 edesc->sec4_sg, DMA_TO_DEVICE); 877 if (ret) 878 goto unmap_ctx; 879 880 ret = buf_map_to_sec4_sg(jrdev, edesc->sec4_sg + 1, state); 881 if (ret) 882 goto unmap_ctx; 883 884 sg_to_sec4_set_last(edesc->sec4_sg + sec4_sg_src_index - 1); 885 886 edesc->sec4_sg_dma = dma_map_single(jrdev, edesc->sec4_sg, 887 sec4_sg_bytes, DMA_TO_DEVICE); 888 if (dma_mapping_error(jrdev, edesc->sec4_sg_dma)) { 889 dev_err(jrdev, "unable to map S/G table\n"); 890 ret = -ENOMEM; 891 goto unmap_ctx; 892 } 893 894 append_seq_in_ptr(desc, edesc->sec4_sg_dma, ctx->ctx_len + buflen, 895 LDST_SGF); 896 897 edesc->dst_dma = map_seq_out_ptr_result(desc, jrdev, req->result, 898 digestsize); 899 if (dma_mapping_error(jrdev, edesc->dst_dma)) { 900 dev_err(jrdev, "unable to map dst\n"); 901 ret = -ENOMEM; 902 goto unmap_ctx; 903 } 904 905 #ifdef DEBUG 906 print_hex_dump(KERN_ERR, "jobdesc@"__stringify(__LINE__)": ", 907 DUMP_PREFIX_ADDRESS, 16, 4, desc, desc_bytes(desc), 1); 908 #endif 909 910 ret = caam_jr_enqueue(jrdev, desc, ahash_done_ctx_src, req); 911 if (ret) 912 goto unmap_ctx; 913 914 return -EINPROGRESS; 915 unmap_ctx: 916 ahash_unmap_ctx(jrdev, edesc, req, digestsize, DMA_FROM_DEVICE); 917 kfree(edesc); 918 return ret; 919 } 920 921 static int ahash_finup_ctx(struct ahash_request *req) 922 { 923 struct crypto_ahash *ahash = crypto_ahash_reqtfm(req); 924 struct caam_hash_ctx *ctx = crypto_ahash_ctx(ahash); 925 struct caam_hash_state *state = ahash_request_ctx(req); 926 struct device *jrdev = ctx->jrdev; 927 gfp_t flags = (req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP) ? 928 GFP_KERNEL : GFP_ATOMIC; 929 int buflen = *current_buflen(state); 930 u32 *desc; 931 int sec4_sg_src_index; 932 int src_nents, mapped_nents; 933 int digestsize = crypto_ahash_digestsize(ahash); 934 struct ahash_edesc *edesc; 935 int ret; 936 937 src_nents = sg_nents_for_len(req->src, req->nbytes); 938 if (src_nents < 0) { 939 dev_err(jrdev, "Invalid number of src SG.\n"); 940 return src_nents; 941 } 942 943 if (src_nents) { 944 mapped_nents = dma_map_sg(jrdev, req->src, src_nents, 945 DMA_TO_DEVICE); 946 if (!mapped_nents) { 947 dev_err(jrdev, "unable to DMA map source\n"); 948 return -ENOMEM; 949 } 950 } else { 951 mapped_nents = 0; 952 } 953 954 sec4_sg_src_index = 1 + (buflen ? 1 : 0); 955 956 /* allocate space for base edesc and hw desc commands, link tables */ 957 edesc = ahash_edesc_alloc(ctx, sec4_sg_src_index + mapped_nents, 958 ctx->sh_desc_fin, ctx->sh_desc_fin_dma, 959 flags); 960 if (!edesc) { 961 dma_unmap_sg(jrdev, req->src, src_nents, DMA_TO_DEVICE); 962 return -ENOMEM; 963 } 964 965 desc = edesc->hw_desc; 966 967 edesc->src_nents = src_nents; 968 969 ret = ctx_map_to_sec4_sg(jrdev, state, ctx->ctx_len, 970 edesc->sec4_sg, DMA_TO_DEVICE); 971 if (ret) 972 goto unmap_ctx; 973 974 ret = buf_map_to_sec4_sg(jrdev, edesc->sec4_sg + 1, state); 975 if (ret) 976 goto unmap_ctx; 977 978 ret = ahash_edesc_add_src(ctx, edesc, req, mapped_nents, 979 sec4_sg_src_index, ctx->ctx_len + buflen, 980 req->nbytes); 981 if (ret) 982 goto unmap_ctx; 983 984 edesc->dst_dma = map_seq_out_ptr_result(desc, jrdev, req->result, 985 digestsize); 986 if (dma_mapping_error(jrdev, edesc->dst_dma)) { 987 dev_err(jrdev, "unable to map dst\n"); 988 ret = -ENOMEM; 989 goto unmap_ctx; 990 } 991 992 #ifdef DEBUG 993 print_hex_dump(KERN_ERR, "jobdesc@"__stringify(__LINE__)": ", 994 DUMP_PREFIX_ADDRESS, 16, 4, desc, desc_bytes(desc), 1); 995 #endif 996 997 ret = caam_jr_enqueue(jrdev, desc, ahash_done_ctx_src, req); 998 if (ret) 999 goto unmap_ctx; 1000 1001 return -EINPROGRESS; 1002 unmap_ctx: 1003 ahash_unmap_ctx(jrdev, edesc, req, digestsize, DMA_FROM_DEVICE); 1004 kfree(edesc); 1005 return ret; 1006 } 1007 1008 static int ahash_digest(struct ahash_request *req) 1009 { 1010 struct crypto_ahash *ahash = crypto_ahash_reqtfm(req); 1011 struct caam_hash_ctx *ctx = crypto_ahash_ctx(ahash); 1012 struct caam_hash_state *state = ahash_request_ctx(req); 1013 struct device *jrdev = ctx->jrdev; 1014 gfp_t flags = (req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP) ? 1015 GFP_KERNEL : GFP_ATOMIC; 1016 u32 *desc; 1017 int digestsize = crypto_ahash_digestsize(ahash); 1018 int src_nents, mapped_nents; 1019 struct ahash_edesc *edesc; 1020 int ret; 1021 1022 state->buf_dma = 0; 1023 1024 src_nents = sg_nents_for_len(req->src, req->nbytes); 1025 if (src_nents < 0) { 1026 dev_err(jrdev, "Invalid number of src SG.\n"); 1027 return src_nents; 1028 } 1029 1030 if (src_nents) { 1031 mapped_nents = dma_map_sg(jrdev, req->src, src_nents, 1032 DMA_TO_DEVICE); 1033 if (!mapped_nents) { 1034 dev_err(jrdev, "unable to map source for DMA\n"); 1035 return -ENOMEM; 1036 } 1037 } else { 1038 mapped_nents = 0; 1039 } 1040 1041 /* allocate space for base edesc and hw desc commands, link tables */ 1042 edesc = ahash_edesc_alloc(ctx, mapped_nents > 1 ? mapped_nents : 0, 1043 ctx->sh_desc_digest, ctx->sh_desc_digest_dma, 1044 flags); 1045 if (!edesc) { 1046 dma_unmap_sg(jrdev, req->src, src_nents, DMA_TO_DEVICE); 1047 return -ENOMEM; 1048 } 1049 1050 edesc->src_nents = src_nents; 1051 1052 ret = ahash_edesc_add_src(ctx, edesc, req, mapped_nents, 0, 0, 1053 req->nbytes); 1054 if (ret) { 1055 ahash_unmap(jrdev, edesc, req, digestsize); 1056 kfree(edesc); 1057 return ret; 1058 } 1059 1060 desc = edesc->hw_desc; 1061 1062 edesc->dst_dma = map_seq_out_ptr_result(desc, jrdev, req->result, 1063 digestsize); 1064 if (dma_mapping_error(jrdev, edesc->dst_dma)) { 1065 dev_err(jrdev, "unable to map dst\n"); 1066 ahash_unmap(jrdev, edesc, req, digestsize); 1067 kfree(edesc); 1068 return -ENOMEM; 1069 } 1070 1071 #ifdef DEBUG 1072 print_hex_dump(KERN_ERR, "jobdesc@"__stringify(__LINE__)": ", 1073 DUMP_PREFIX_ADDRESS, 16, 4, desc, desc_bytes(desc), 1); 1074 #endif 1075 1076 ret = caam_jr_enqueue(jrdev, desc, ahash_done, req); 1077 if (!ret) { 1078 ret = -EINPROGRESS; 1079 } else { 1080 ahash_unmap(jrdev, edesc, req, digestsize); 1081 kfree(edesc); 1082 } 1083 1084 return ret; 1085 } 1086 1087 /* submit ahash final if it the first job descriptor */ 1088 static int ahash_final_no_ctx(struct ahash_request *req) 1089 { 1090 struct crypto_ahash *ahash = crypto_ahash_reqtfm(req); 1091 struct caam_hash_ctx *ctx = crypto_ahash_ctx(ahash); 1092 struct caam_hash_state *state = ahash_request_ctx(req); 1093 struct device *jrdev = ctx->jrdev; 1094 gfp_t flags = (req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP) ? 1095 GFP_KERNEL : GFP_ATOMIC; 1096 u8 *buf = current_buf(state); 1097 int buflen = *current_buflen(state); 1098 u32 *desc; 1099 int digestsize = crypto_ahash_digestsize(ahash); 1100 struct ahash_edesc *edesc; 1101 int ret; 1102 1103 /* allocate space for base edesc and hw desc commands, link tables */ 1104 edesc = ahash_edesc_alloc(ctx, 0, ctx->sh_desc_digest, 1105 ctx->sh_desc_digest_dma, flags); 1106 if (!edesc) 1107 return -ENOMEM; 1108 1109 desc = edesc->hw_desc; 1110 1111 state->buf_dma = dma_map_single(jrdev, buf, buflen, DMA_TO_DEVICE); 1112 if (dma_mapping_error(jrdev, state->buf_dma)) { 1113 dev_err(jrdev, "unable to map src\n"); 1114 goto unmap; 1115 } 1116 1117 append_seq_in_ptr(desc, state->buf_dma, buflen, 0); 1118 1119 edesc->dst_dma = map_seq_out_ptr_result(desc, jrdev, req->result, 1120 digestsize); 1121 if (dma_mapping_error(jrdev, edesc->dst_dma)) { 1122 dev_err(jrdev, "unable to map dst\n"); 1123 goto unmap; 1124 } 1125 1126 #ifdef DEBUG 1127 print_hex_dump(KERN_ERR, "jobdesc@"__stringify(__LINE__)": ", 1128 DUMP_PREFIX_ADDRESS, 16, 4, desc, desc_bytes(desc), 1); 1129 #endif 1130 1131 ret = caam_jr_enqueue(jrdev, desc, ahash_done, req); 1132 if (!ret) { 1133 ret = -EINPROGRESS; 1134 } else { 1135 ahash_unmap(jrdev, edesc, req, digestsize); 1136 kfree(edesc); 1137 } 1138 1139 return ret; 1140 unmap: 1141 ahash_unmap(jrdev, edesc, req, digestsize); 1142 kfree(edesc); 1143 return -ENOMEM; 1144 1145 } 1146 1147 /* submit ahash update if it the first job descriptor after update */ 1148 static int ahash_update_no_ctx(struct ahash_request *req) 1149 { 1150 struct crypto_ahash *ahash = crypto_ahash_reqtfm(req); 1151 struct caam_hash_ctx *ctx = crypto_ahash_ctx(ahash); 1152 struct caam_hash_state *state = ahash_request_ctx(req); 1153 struct device *jrdev = ctx->jrdev; 1154 gfp_t flags = (req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP) ? 1155 GFP_KERNEL : GFP_ATOMIC; 1156 u8 *buf = current_buf(state); 1157 int *buflen = current_buflen(state); 1158 u8 *next_buf = alt_buf(state); 1159 int *next_buflen = alt_buflen(state); 1160 int in_len = *buflen + req->nbytes, to_hash; 1161 int sec4_sg_bytes, src_nents, mapped_nents; 1162 struct ahash_edesc *edesc; 1163 u32 *desc; 1164 int ret = 0; 1165 1166 *next_buflen = in_len & (crypto_tfm_alg_blocksize(&ahash->base) - 1); 1167 to_hash = in_len - *next_buflen; 1168 1169 if (to_hash) { 1170 src_nents = sg_nents_for_len(req->src, 1171 req->nbytes - *next_buflen); 1172 if (src_nents < 0) { 1173 dev_err(jrdev, "Invalid number of src SG.\n"); 1174 return src_nents; 1175 } 1176 1177 if (src_nents) { 1178 mapped_nents = dma_map_sg(jrdev, req->src, src_nents, 1179 DMA_TO_DEVICE); 1180 if (!mapped_nents) { 1181 dev_err(jrdev, "unable to DMA map source\n"); 1182 return -ENOMEM; 1183 } 1184 } else { 1185 mapped_nents = 0; 1186 } 1187 1188 sec4_sg_bytes = (1 + mapped_nents) * 1189 sizeof(struct sec4_sg_entry); 1190 1191 /* 1192 * allocate space for base edesc and hw desc commands, 1193 * link tables 1194 */ 1195 edesc = ahash_edesc_alloc(ctx, 1 + mapped_nents, 1196 ctx->sh_desc_update_first, 1197 ctx->sh_desc_update_first_dma, 1198 flags); 1199 if (!edesc) { 1200 dma_unmap_sg(jrdev, req->src, src_nents, DMA_TO_DEVICE); 1201 return -ENOMEM; 1202 } 1203 1204 edesc->src_nents = src_nents; 1205 edesc->sec4_sg_bytes = sec4_sg_bytes; 1206 1207 ret = buf_map_to_sec4_sg(jrdev, edesc->sec4_sg, state); 1208 if (ret) 1209 goto unmap_ctx; 1210 1211 sg_to_sec4_sg_last(req->src, mapped_nents, 1212 edesc->sec4_sg + 1, 0); 1213 1214 if (*next_buflen) { 1215 scatterwalk_map_and_copy(next_buf, req->src, 1216 to_hash - *buflen, 1217 *next_buflen, 0); 1218 } 1219 1220 desc = edesc->hw_desc; 1221 1222 edesc->sec4_sg_dma = dma_map_single(jrdev, edesc->sec4_sg, 1223 sec4_sg_bytes, 1224 DMA_TO_DEVICE); 1225 if (dma_mapping_error(jrdev, edesc->sec4_sg_dma)) { 1226 dev_err(jrdev, "unable to map S/G table\n"); 1227 ret = -ENOMEM; 1228 goto unmap_ctx; 1229 } 1230 1231 append_seq_in_ptr(desc, edesc->sec4_sg_dma, to_hash, LDST_SGF); 1232 1233 ret = map_seq_out_ptr_ctx(desc, jrdev, state, ctx->ctx_len); 1234 if (ret) 1235 goto unmap_ctx; 1236 1237 #ifdef DEBUG 1238 print_hex_dump(KERN_ERR, "jobdesc@"__stringify(__LINE__)": ", 1239 DUMP_PREFIX_ADDRESS, 16, 4, desc, 1240 desc_bytes(desc), 1); 1241 #endif 1242 1243 ret = caam_jr_enqueue(jrdev, desc, ahash_done_ctx_dst, req); 1244 if (ret) 1245 goto unmap_ctx; 1246 1247 ret = -EINPROGRESS; 1248 state->update = ahash_update_ctx; 1249 state->finup = ahash_finup_ctx; 1250 state->final = ahash_final_ctx; 1251 } else if (*next_buflen) { 1252 scatterwalk_map_and_copy(buf + *buflen, req->src, 0, 1253 req->nbytes, 0); 1254 *buflen = *next_buflen; 1255 *next_buflen = 0; 1256 } 1257 #ifdef DEBUG 1258 print_hex_dump(KERN_ERR, "buf@"__stringify(__LINE__)": ", 1259 DUMP_PREFIX_ADDRESS, 16, 4, buf, *buflen, 1); 1260 print_hex_dump(KERN_ERR, "next buf@"__stringify(__LINE__)": ", 1261 DUMP_PREFIX_ADDRESS, 16, 4, next_buf, 1262 *next_buflen, 1); 1263 #endif 1264 1265 return ret; 1266 unmap_ctx: 1267 ahash_unmap_ctx(jrdev, edesc, req, ctx->ctx_len, DMA_TO_DEVICE); 1268 kfree(edesc); 1269 return ret; 1270 } 1271 1272 /* submit ahash finup if it the first job descriptor after update */ 1273 static int ahash_finup_no_ctx(struct ahash_request *req) 1274 { 1275 struct crypto_ahash *ahash = crypto_ahash_reqtfm(req); 1276 struct caam_hash_ctx *ctx = crypto_ahash_ctx(ahash); 1277 struct caam_hash_state *state = ahash_request_ctx(req); 1278 struct device *jrdev = ctx->jrdev; 1279 gfp_t flags = (req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP) ? 1280 GFP_KERNEL : GFP_ATOMIC; 1281 int buflen = *current_buflen(state); 1282 u32 *desc; 1283 int sec4_sg_bytes, sec4_sg_src_index, src_nents, mapped_nents; 1284 int digestsize = crypto_ahash_digestsize(ahash); 1285 struct ahash_edesc *edesc; 1286 int ret; 1287 1288 src_nents = sg_nents_for_len(req->src, req->nbytes); 1289 if (src_nents < 0) { 1290 dev_err(jrdev, "Invalid number of src SG.\n"); 1291 return src_nents; 1292 } 1293 1294 if (src_nents) { 1295 mapped_nents = dma_map_sg(jrdev, req->src, src_nents, 1296 DMA_TO_DEVICE); 1297 if (!mapped_nents) { 1298 dev_err(jrdev, "unable to DMA map source\n"); 1299 return -ENOMEM; 1300 } 1301 } else { 1302 mapped_nents = 0; 1303 } 1304 1305 sec4_sg_src_index = 2; 1306 sec4_sg_bytes = (sec4_sg_src_index + mapped_nents) * 1307 sizeof(struct sec4_sg_entry); 1308 1309 /* allocate space for base edesc and hw desc commands, link tables */ 1310 edesc = ahash_edesc_alloc(ctx, sec4_sg_src_index + mapped_nents, 1311 ctx->sh_desc_digest, ctx->sh_desc_digest_dma, 1312 flags); 1313 if (!edesc) { 1314 dma_unmap_sg(jrdev, req->src, src_nents, DMA_TO_DEVICE); 1315 return -ENOMEM; 1316 } 1317 1318 desc = edesc->hw_desc; 1319 1320 edesc->src_nents = src_nents; 1321 edesc->sec4_sg_bytes = sec4_sg_bytes; 1322 1323 ret = buf_map_to_sec4_sg(jrdev, edesc->sec4_sg, state); 1324 if (ret) 1325 goto unmap; 1326 1327 ret = ahash_edesc_add_src(ctx, edesc, req, mapped_nents, 1, buflen, 1328 req->nbytes); 1329 if (ret) { 1330 dev_err(jrdev, "unable to map S/G table\n"); 1331 goto unmap; 1332 } 1333 1334 edesc->dst_dma = map_seq_out_ptr_result(desc, jrdev, req->result, 1335 digestsize); 1336 if (dma_mapping_error(jrdev, edesc->dst_dma)) { 1337 dev_err(jrdev, "unable to map dst\n"); 1338 goto unmap; 1339 } 1340 1341 #ifdef DEBUG 1342 print_hex_dump(KERN_ERR, "jobdesc@"__stringify(__LINE__)": ", 1343 DUMP_PREFIX_ADDRESS, 16, 4, desc, desc_bytes(desc), 1); 1344 #endif 1345 1346 ret = caam_jr_enqueue(jrdev, desc, ahash_done, req); 1347 if (!ret) { 1348 ret = -EINPROGRESS; 1349 } else { 1350 ahash_unmap(jrdev, edesc, req, digestsize); 1351 kfree(edesc); 1352 } 1353 1354 return ret; 1355 unmap: 1356 ahash_unmap(jrdev, edesc, req, digestsize); 1357 kfree(edesc); 1358 return -ENOMEM; 1359 1360 } 1361 1362 /* submit first update job descriptor after init */ 1363 static int ahash_update_first(struct ahash_request *req) 1364 { 1365 struct crypto_ahash *ahash = crypto_ahash_reqtfm(req); 1366 struct caam_hash_ctx *ctx = crypto_ahash_ctx(ahash); 1367 struct caam_hash_state *state = ahash_request_ctx(req); 1368 struct device *jrdev = ctx->jrdev; 1369 gfp_t flags = (req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP) ? 1370 GFP_KERNEL : GFP_ATOMIC; 1371 u8 *next_buf = alt_buf(state); 1372 int *next_buflen = alt_buflen(state); 1373 int to_hash; 1374 u32 *desc; 1375 int src_nents, mapped_nents; 1376 struct ahash_edesc *edesc; 1377 int ret = 0; 1378 1379 *next_buflen = req->nbytes & (crypto_tfm_alg_blocksize(&ahash->base) - 1380 1); 1381 to_hash = req->nbytes - *next_buflen; 1382 1383 if (to_hash) { 1384 src_nents = sg_nents_for_len(req->src, 1385 req->nbytes - *next_buflen); 1386 if (src_nents < 0) { 1387 dev_err(jrdev, "Invalid number of src SG.\n"); 1388 return src_nents; 1389 } 1390 1391 if (src_nents) { 1392 mapped_nents = dma_map_sg(jrdev, req->src, src_nents, 1393 DMA_TO_DEVICE); 1394 if (!mapped_nents) { 1395 dev_err(jrdev, "unable to map source for DMA\n"); 1396 return -ENOMEM; 1397 } 1398 } else { 1399 mapped_nents = 0; 1400 } 1401 1402 /* 1403 * allocate space for base edesc and hw desc commands, 1404 * link tables 1405 */ 1406 edesc = ahash_edesc_alloc(ctx, mapped_nents > 1 ? 1407 mapped_nents : 0, 1408 ctx->sh_desc_update_first, 1409 ctx->sh_desc_update_first_dma, 1410 flags); 1411 if (!edesc) { 1412 dma_unmap_sg(jrdev, req->src, src_nents, DMA_TO_DEVICE); 1413 return -ENOMEM; 1414 } 1415 1416 edesc->src_nents = src_nents; 1417 1418 ret = ahash_edesc_add_src(ctx, edesc, req, mapped_nents, 0, 0, 1419 to_hash); 1420 if (ret) 1421 goto unmap_ctx; 1422 1423 if (*next_buflen) 1424 scatterwalk_map_and_copy(next_buf, req->src, to_hash, 1425 *next_buflen, 0); 1426 1427 desc = edesc->hw_desc; 1428 1429 ret = map_seq_out_ptr_ctx(desc, jrdev, state, ctx->ctx_len); 1430 if (ret) 1431 goto unmap_ctx; 1432 1433 #ifdef DEBUG 1434 print_hex_dump(KERN_ERR, "jobdesc@"__stringify(__LINE__)": ", 1435 DUMP_PREFIX_ADDRESS, 16, 4, desc, 1436 desc_bytes(desc), 1); 1437 #endif 1438 1439 ret = caam_jr_enqueue(jrdev, desc, ahash_done_ctx_dst, req); 1440 if (ret) 1441 goto unmap_ctx; 1442 1443 ret = -EINPROGRESS; 1444 state->update = ahash_update_ctx; 1445 state->finup = ahash_finup_ctx; 1446 state->final = ahash_final_ctx; 1447 } else if (*next_buflen) { 1448 state->update = ahash_update_no_ctx; 1449 state->finup = ahash_finup_no_ctx; 1450 state->final = ahash_final_no_ctx; 1451 scatterwalk_map_and_copy(next_buf, req->src, 0, 1452 req->nbytes, 0); 1453 switch_buf(state); 1454 } 1455 #ifdef DEBUG 1456 print_hex_dump(KERN_ERR, "next buf@"__stringify(__LINE__)": ", 1457 DUMP_PREFIX_ADDRESS, 16, 4, next_buf, 1458 *next_buflen, 1); 1459 #endif 1460 1461 return ret; 1462 unmap_ctx: 1463 ahash_unmap_ctx(jrdev, edesc, req, ctx->ctx_len, DMA_TO_DEVICE); 1464 kfree(edesc); 1465 return ret; 1466 } 1467 1468 static int ahash_finup_first(struct ahash_request *req) 1469 { 1470 return ahash_digest(req); 1471 } 1472 1473 static int ahash_init(struct ahash_request *req) 1474 { 1475 struct caam_hash_state *state = ahash_request_ctx(req); 1476 1477 state->update = ahash_update_first; 1478 state->finup = ahash_finup_first; 1479 state->final = ahash_final_no_ctx; 1480 1481 state->ctx_dma = 0; 1482 state->current_buf = 0; 1483 state->buf_dma = 0; 1484 state->buflen_0 = 0; 1485 state->buflen_1 = 0; 1486 1487 return 0; 1488 } 1489 1490 static int ahash_update(struct ahash_request *req) 1491 { 1492 struct caam_hash_state *state = ahash_request_ctx(req); 1493 1494 return state->update(req); 1495 } 1496 1497 static int ahash_finup(struct ahash_request *req) 1498 { 1499 struct caam_hash_state *state = ahash_request_ctx(req); 1500 1501 return state->finup(req); 1502 } 1503 1504 static int ahash_final(struct ahash_request *req) 1505 { 1506 struct caam_hash_state *state = ahash_request_ctx(req); 1507 1508 return state->final(req); 1509 } 1510 1511 static int ahash_export(struct ahash_request *req, void *out) 1512 { 1513 struct caam_hash_state *state = ahash_request_ctx(req); 1514 struct caam_export_state *export = out; 1515 int len; 1516 u8 *buf; 1517 1518 if (state->current_buf) { 1519 buf = state->buf_1; 1520 len = state->buflen_1; 1521 } else { 1522 buf = state->buf_0; 1523 len = state->buflen_0; 1524 } 1525 1526 memcpy(export->buf, buf, len); 1527 memcpy(export->caam_ctx, state->caam_ctx, sizeof(export->caam_ctx)); 1528 export->buflen = len; 1529 export->update = state->update; 1530 export->final = state->final; 1531 export->finup = state->finup; 1532 1533 return 0; 1534 } 1535 1536 static int ahash_import(struct ahash_request *req, const void *in) 1537 { 1538 struct caam_hash_state *state = ahash_request_ctx(req); 1539 const struct caam_export_state *export = in; 1540 1541 memset(state, 0, sizeof(*state)); 1542 memcpy(state->buf_0, export->buf, export->buflen); 1543 memcpy(state->caam_ctx, export->caam_ctx, sizeof(state->caam_ctx)); 1544 state->buflen_0 = export->buflen; 1545 state->update = export->update; 1546 state->final = export->final; 1547 state->finup = export->finup; 1548 1549 return 0; 1550 } 1551 1552 struct caam_hash_template { 1553 char name[CRYPTO_MAX_ALG_NAME]; 1554 char driver_name[CRYPTO_MAX_ALG_NAME]; 1555 char hmac_name[CRYPTO_MAX_ALG_NAME]; 1556 char hmac_driver_name[CRYPTO_MAX_ALG_NAME]; 1557 unsigned int blocksize; 1558 struct ahash_alg template_ahash; 1559 u32 alg_type; 1560 }; 1561 1562 /* ahash descriptors */ 1563 static struct caam_hash_template driver_hash[] = { 1564 { 1565 .name = "sha1", 1566 .driver_name = "sha1-caam", 1567 .hmac_name = "hmac(sha1)", 1568 .hmac_driver_name = "hmac-sha1-caam", 1569 .blocksize = SHA1_BLOCK_SIZE, 1570 .template_ahash = { 1571 .init = ahash_init, 1572 .update = ahash_update, 1573 .final = ahash_final, 1574 .finup = ahash_finup, 1575 .digest = ahash_digest, 1576 .export = ahash_export, 1577 .import = ahash_import, 1578 .setkey = ahash_setkey, 1579 .halg = { 1580 .digestsize = SHA1_DIGEST_SIZE, 1581 .statesize = sizeof(struct caam_export_state), 1582 }, 1583 }, 1584 .alg_type = OP_ALG_ALGSEL_SHA1, 1585 }, { 1586 .name = "sha224", 1587 .driver_name = "sha224-caam", 1588 .hmac_name = "hmac(sha224)", 1589 .hmac_driver_name = "hmac-sha224-caam", 1590 .blocksize = SHA224_BLOCK_SIZE, 1591 .template_ahash = { 1592 .init = ahash_init, 1593 .update = ahash_update, 1594 .final = ahash_final, 1595 .finup = ahash_finup, 1596 .digest = ahash_digest, 1597 .export = ahash_export, 1598 .import = ahash_import, 1599 .setkey = ahash_setkey, 1600 .halg = { 1601 .digestsize = SHA224_DIGEST_SIZE, 1602 .statesize = sizeof(struct caam_export_state), 1603 }, 1604 }, 1605 .alg_type = OP_ALG_ALGSEL_SHA224, 1606 }, { 1607 .name = "sha256", 1608 .driver_name = "sha256-caam", 1609 .hmac_name = "hmac(sha256)", 1610 .hmac_driver_name = "hmac-sha256-caam", 1611 .blocksize = SHA256_BLOCK_SIZE, 1612 .template_ahash = { 1613 .init = ahash_init, 1614 .update = ahash_update, 1615 .final = ahash_final, 1616 .finup = ahash_finup, 1617 .digest = ahash_digest, 1618 .export = ahash_export, 1619 .import = ahash_import, 1620 .setkey = ahash_setkey, 1621 .halg = { 1622 .digestsize = SHA256_DIGEST_SIZE, 1623 .statesize = sizeof(struct caam_export_state), 1624 }, 1625 }, 1626 .alg_type = OP_ALG_ALGSEL_SHA256, 1627 }, { 1628 .name = "sha384", 1629 .driver_name = "sha384-caam", 1630 .hmac_name = "hmac(sha384)", 1631 .hmac_driver_name = "hmac-sha384-caam", 1632 .blocksize = SHA384_BLOCK_SIZE, 1633 .template_ahash = { 1634 .init = ahash_init, 1635 .update = ahash_update, 1636 .final = ahash_final, 1637 .finup = ahash_finup, 1638 .digest = ahash_digest, 1639 .export = ahash_export, 1640 .import = ahash_import, 1641 .setkey = ahash_setkey, 1642 .halg = { 1643 .digestsize = SHA384_DIGEST_SIZE, 1644 .statesize = sizeof(struct caam_export_state), 1645 }, 1646 }, 1647 .alg_type = OP_ALG_ALGSEL_SHA384, 1648 }, { 1649 .name = "sha512", 1650 .driver_name = "sha512-caam", 1651 .hmac_name = "hmac(sha512)", 1652 .hmac_driver_name = "hmac-sha512-caam", 1653 .blocksize = SHA512_BLOCK_SIZE, 1654 .template_ahash = { 1655 .init = ahash_init, 1656 .update = ahash_update, 1657 .final = ahash_final, 1658 .finup = ahash_finup, 1659 .digest = ahash_digest, 1660 .export = ahash_export, 1661 .import = ahash_import, 1662 .setkey = ahash_setkey, 1663 .halg = { 1664 .digestsize = SHA512_DIGEST_SIZE, 1665 .statesize = sizeof(struct caam_export_state), 1666 }, 1667 }, 1668 .alg_type = OP_ALG_ALGSEL_SHA512, 1669 }, { 1670 .name = "md5", 1671 .driver_name = "md5-caam", 1672 .hmac_name = "hmac(md5)", 1673 .hmac_driver_name = "hmac-md5-caam", 1674 .blocksize = MD5_BLOCK_WORDS * 4, 1675 .template_ahash = { 1676 .init = ahash_init, 1677 .update = ahash_update, 1678 .final = ahash_final, 1679 .finup = ahash_finup, 1680 .digest = ahash_digest, 1681 .export = ahash_export, 1682 .import = ahash_import, 1683 .setkey = ahash_setkey, 1684 .halg = { 1685 .digestsize = MD5_DIGEST_SIZE, 1686 .statesize = sizeof(struct caam_export_state), 1687 }, 1688 }, 1689 .alg_type = OP_ALG_ALGSEL_MD5, 1690 }, 1691 }; 1692 1693 struct caam_hash_alg { 1694 struct list_head entry; 1695 int alg_type; 1696 struct ahash_alg ahash_alg; 1697 }; 1698 1699 static int caam_hash_cra_init(struct crypto_tfm *tfm) 1700 { 1701 struct crypto_ahash *ahash = __crypto_ahash_cast(tfm); 1702 struct crypto_alg *base = tfm->__crt_alg; 1703 struct hash_alg_common *halg = 1704 container_of(base, struct hash_alg_common, base); 1705 struct ahash_alg *alg = 1706 container_of(halg, struct ahash_alg, halg); 1707 struct caam_hash_alg *caam_hash = 1708 container_of(alg, struct caam_hash_alg, ahash_alg); 1709 struct caam_hash_ctx *ctx = crypto_tfm_ctx(tfm); 1710 /* Sizes for MDHA running digests: MD5, SHA1, 224, 256, 384, 512 */ 1711 static const u8 runninglen[] = { HASH_MSG_LEN + MD5_DIGEST_SIZE, 1712 HASH_MSG_LEN + SHA1_DIGEST_SIZE, 1713 HASH_MSG_LEN + 32, 1714 HASH_MSG_LEN + SHA256_DIGEST_SIZE, 1715 HASH_MSG_LEN + 64, 1716 HASH_MSG_LEN + SHA512_DIGEST_SIZE }; 1717 dma_addr_t dma_addr; 1718 1719 /* 1720 * Get a Job ring from Job Ring driver to ensure in-order 1721 * crypto request processing per tfm 1722 */ 1723 ctx->jrdev = caam_jr_alloc(); 1724 if (IS_ERR(ctx->jrdev)) { 1725 pr_err("Job Ring Device allocation for transform failed\n"); 1726 return PTR_ERR(ctx->jrdev); 1727 } 1728 1729 dma_addr = dma_map_single_attrs(ctx->jrdev, ctx->sh_desc_update, 1730 offsetof(struct caam_hash_ctx, 1731 sh_desc_update_dma), 1732 DMA_TO_DEVICE, DMA_ATTR_SKIP_CPU_SYNC); 1733 if (dma_mapping_error(ctx->jrdev, dma_addr)) { 1734 dev_err(ctx->jrdev, "unable to map shared descriptors\n"); 1735 caam_jr_free(ctx->jrdev); 1736 return -ENOMEM; 1737 } 1738 1739 ctx->sh_desc_update_dma = dma_addr; 1740 ctx->sh_desc_update_first_dma = dma_addr + 1741 offsetof(struct caam_hash_ctx, 1742 sh_desc_update_first); 1743 ctx->sh_desc_fin_dma = dma_addr + offsetof(struct caam_hash_ctx, 1744 sh_desc_fin); 1745 ctx->sh_desc_digest_dma = dma_addr + offsetof(struct caam_hash_ctx, 1746 sh_desc_digest); 1747 1748 /* copy descriptor header template value */ 1749 ctx->adata.algtype = OP_TYPE_CLASS2_ALG | caam_hash->alg_type; 1750 1751 ctx->ctx_len = runninglen[(ctx->adata.algtype & 1752 OP_ALG_ALGSEL_SUBMASK) >> 1753 OP_ALG_ALGSEL_SHIFT]; 1754 1755 crypto_ahash_set_reqsize(__crypto_ahash_cast(tfm), 1756 sizeof(struct caam_hash_state)); 1757 return ahash_set_sh_desc(ahash); 1758 } 1759 1760 static void caam_hash_cra_exit(struct crypto_tfm *tfm) 1761 { 1762 struct caam_hash_ctx *ctx = crypto_tfm_ctx(tfm); 1763 1764 dma_unmap_single_attrs(ctx->jrdev, ctx->sh_desc_update_dma, 1765 offsetof(struct caam_hash_ctx, 1766 sh_desc_update_dma), 1767 DMA_TO_DEVICE, DMA_ATTR_SKIP_CPU_SYNC); 1768 caam_jr_free(ctx->jrdev); 1769 } 1770 1771 static void __exit caam_algapi_hash_exit(void) 1772 { 1773 struct caam_hash_alg *t_alg, *n; 1774 1775 if (!hash_list.next) 1776 return; 1777 1778 list_for_each_entry_safe(t_alg, n, &hash_list, entry) { 1779 crypto_unregister_ahash(&t_alg->ahash_alg); 1780 list_del(&t_alg->entry); 1781 kfree(t_alg); 1782 } 1783 } 1784 1785 static struct caam_hash_alg * 1786 caam_hash_alloc(struct caam_hash_template *template, 1787 bool keyed) 1788 { 1789 struct caam_hash_alg *t_alg; 1790 struct ahash_alg *halg; 1791 struct crypto_alg *alg; 1792 1793 t_alg = kzalloc(sizeof(*t_alg), GFP_KERNEL); 1794 if (!t_alg) { 1795 pr_err("failed to allocate t_alg\n"); 1796 return ERR_PTR(-ENOMEM); 1797 } 1798 1799 t_alg->ahash_alg = template->template_ahash; 1800 halg = &t_alg->ahash_alg; 1801 alg = &halg->halg.base; 1802 1803 if (keyed) { 1804 snprintf(alg->cra_name, CRYPTO_MAX_ALG_NAME, "%s", 1805 template->hmac_name); 1806 snprintf(alg->cra_driver_name, CRYPTO_MAX_ALG_NAME, "%s", 1807 template->hmac_driver_name); 1808 } else { 1809 snprintf(alg->cra_name, CRYPTO_MAX_ALG_NAME, "%s", 1810 template->name); 1811 snprintf(alg->cra_driver_name, CRYPTO_MAX_ALG_NAME, "%s", 1812 template->driver_name); 1813 t_alg->ahash_alg.setkey = NULL; 1814 } 1815 alg->cra_module = THIS_MODULE; 1816 alg->cra_init = caam_hash_cra_init; 1817 alg->cra_exit = caam_hash_cra_exit; 1818 alg->cra_ctxsize = sizeof(struct caam_hash_ctx); 1819 alg->cra_priority = CAAM_CRA_PRIORITY; 1820 alg->cra_blocksize = template->blocksize; 1821 alg->cra_alignmask = 0; 1822 alg->cra_flags = CRYPTO_ALG_ASYNC | CRYPTO_ALG_TYPE_AHASH; 1823 alg->cra_type = &crypto_ahash_type; 1824 1825 t_alg->alg_type = template->alg_type; 1826 1827 return t_alg; 1828 } 1829 1830 static int __init caam_algapi_hash_init(void) 1831 { 1832 struct device_node *dev_node; 1833 struct platform_device *pdev; 1834 struct device *ctrldev; 1835 int i = 0, err = 0; 1836 struct caam_drv_private *priv; 1837 unsigned int md_limit = SHA512_DIGEST_SIZE; 1838 u32 cha_inst, cha_vid; 1839 1840 dev_node = of_find_compatible_node(NULL, NULL, "fsl,sec-v4.0"); 1841 if (!dev_node) { 1842 dev_node = of_find_compatible_node(NULL, NULL, "fsl,sec4.0"); 1843 if (!dev_node) 1844 return -ENODEV; 1845 } 1846 1847 pdev = of_find_device_by_node(dev_node); 1848 if (!pdev) { 1849 of_node_put(dev_node); 1850 return -ENODEV; 1851 } 1852 1853 ctrldev = &pdev->dev; 1854 priv = dev_get_drvdata(ctrldev); 1855 of_node_put(dev_node); 1856 1857 /* 1858 * If priv is NULL, it's probably because the caam driver wasn't 1859 * properly initialized (e.g. RNG4 init failed). Thus, bail out here. 1860 */ 1861 if (!priv) 1862 return -ENODEV; 1863 1864 /* 1865 * Register crypto algorithms the device supports. First, identify 1866 * presence and attributes of MD block. 1867 */ 1868 cha_vid = rd_reg32(&priv->ctrl->perfmon.cha_id_ls); 1869 cha_inst = rd_reg32(&priv->ctrl->perfmon.cha_num_ls); 1870 1871 /* 1872 * Skip registration of any hashing algorithms if MD block 1873 * is not present. 1874 */ 1875 if (!((cha_inst & CHA_ID_LS_MD_MASK) >> CHA_ID_LS_MD_SHIFT)) 1876 return -ENODEV; 1877 1878 /* Limit digest size based on LP256 */ 1879 if ((cha_vid & CHA_ID_LS_MD_MASK) == CHA_ID_LS_MD_LP256) 1880 md_limit = SHA256_DIGEST_SIZE; 1881 1882 INIT_LIST_HEAD(&hash_list); 1883 1884 /* register crypto algorithms the device supports */ 1885 for (i = 0; i < ARRAY_SIZE(driver_hash); i++) { 1886 struct caam_hash_alg *t_alg; 1887 struct caam_hash_template *alg = driver_hash + i; 1888 1889 /* If MD size is not supported by device, skip registration */ 1890 if (alg->template_ahash.halg.digestsize > md_limit) 1891 continue; 1892 1893 /* register hmac version */ 1894 t_alg = caam_hash_alloc(alg, true); 1895 if (IS_ERR(t_alg)) { 1896 err = PTR_ERR(t_alg); 1897 pr_warn("%s alg allocation failed\n", alg->driver_name); 1898 continue; 1899 } 1900 1901 err = crypto_register_ahash(&t_alg->ahash_alg); 1902 if (err) { 1903 pr_warn("%s alg registration failed: %d\n", 1904 t_alg->ahash_alg.halg.base.cra_driver_name, 1905 err); 1906 kfree(t_alg); 1907 } else 1908 list_add_tail(&t_alg->entry, &hash_list); 1909 1910 /* register unkeyed version */ 1911 t_alg = caam_hash_alloc(alg, false); 1912 if (IS_ERR(t_alg)) { 1913 err = PTR_ERR(t_alg); 1914 pr_warn("%s alg allocation failed\n", alg->driver_name); 1915 continue; 1916 } 1917 1918 err = crypto_register_ahash(&t_alg->ahash_alg); 1919 if (err) { 1920 pr_warn("%s alg registration failed: %d\n", 1921 t_alg->ahash_alg.halg.base.cra_driver_name, 1922 err); 1923 kfree(t_alg); 1924 } else 1925 list_add_tail(&t_alg->entry, &hash_list); 1926 } 1927 1928 return err; 1929 } 1930 1931 module_init(caam_algapi_hash_init); 1932 module_exit(caam_algapi_hash_exit); 1933 1934 MODULE_LICENSE("GPL"); 1935 MODULE_DESCRIPTION("FSL CAAM support for ahash functions of crypto API"); 1936 MODULE_AUTHOR("Freescale Semiconductor - NMG"); 1937