1 /* 2 * caam - Freescale FSL CAAM support for ahash functions of crypto API 3 * 4 * Copyright 2011 Freescale Semiconductor, Inc. 5 * 6 * Based on caamalg.c crypto API driver. 7 * 8 * relationship of digest job descriptor or first job descriptor after init to 9 * shared descriptors: 10 * 11 * --------------- --------------- 12 * | JobDesc #1 |-------------------->| ShareDesc | 13 * | *(packet 1) | | (hashKey) | 14 * --------------- | (operation) | 15 * --------------- 16 * 17 * relationship of subsequent job descriptors to shared descriptors: 18 * 19 * --------------- --------------- 20 * | JobDesc #2 |-------------------->| ShareDesc | 21 * | *(packet 2) | |------------->| (hashKey) | 22 * --------------- | |-------->| (operation) | 23 * . | | | (load ctx2) | 24 * . | | --------------- 25 * --------------- | | 26 * | JobDesc #3 |------| | 27 * | *(packet 3) | | 28 * --------------- | 29 * . | 30 * . | 31 * --------------- | 32 * | JobDesc #4 |------------ 33 * | *(packet 4) | 34 * --------------- 35 * 36 * The SharedDesc never changes for a connection unless rekeyed, but 37 * each packet will likely be in a different place. So all we need 38 * to know to process the packet is where the input is, where the 39 * output goes, and what context we want to process with. Context is 40 * in the SharedDesc, packet references in the JobDesc. 41 * 42 * So, a job desc looks like: 43 * 44 * --------------------- 45 * | Header | 46 * | ShareDesc Pointer | 47 * | SEQ_OUT_PTR | 48 * | (output buffer) | 49 * | (output length) | 50 * | SEQ_IN_PTR | 51 * | (input buffer) | 52 * | (input length) | 53 * --------------------- 54 */ 55 56 #include "compat.h" 57 58 #include "regs.h" 59 #include "intern.h" 60 #include "desc_constr.h" 61 #include "jr.h" 62 #include "error.h" 63 #include "sg_sw_sec4.h" 64 #include "key_gen.h" 65 66 #define CAAM_CRA_PRIORITY 3000 67 68 /* max hash key is max split key size */ 69 #define CAAM_MAX_HASH_KEY_SIZE (SHA512_DIGEST_SIZE * 2) 70 71 #define CAAM_MAX_HASH_BLOCK_SIZE SHA512_BLOCK_SIZE 72 #define CAAM_MAX_HASH_DIGEST_SIZE SHA512_DIGEST_SIZE 73 74 /* length of descriptors text */ 75 #define DESC_AHASH_BASE (3 * CAAM_CMD_SZ) 76 #define DESC_AHASH_UPDATE_LEN (6 * CAAM_CMD_SZ) 77 #define DESC_AHASH_UPDATE_FIRST_LEN (DESC_AHASH_BASE + 4 * CAAM_CMD_SZ) 78 #define DESC_AHASH_FINAL_LEN (DESC_AHASH_BASE + 5 * CAAM_CMD_SZ) 79 #define DESC_AHASH_FINUP_LEN (DESC_AHASH_BASE + 5 * CAAM_CMD_SZ) 80 #define DESC_AHASH_DIGEST_LEN (DESC_AHASH_BASE + 4 * CAAM_CMD_SZ) 81 82 #define DESC_HASH_MAX_USED_BYTES (DESC_AHASH_FINAL_LEN + \ 83 CAAM_MAX_HASH_KEY_SIZE) 84 #define DESC_HASH_MAX_USED_LEN (DESC_HASH_MAX_USED_BYTES / CAAM_CMD_SZ) 85 86 /* caam context sizes for hashes: running digest + 8 */ 87 #define HASH_MSG_LEN 8 88 #define MAX_CTX_LEN (HASH_MSG_LEN + SHA512_DIGEST_SIZE) 89 90 #ifdef DEBUG 91 /* for print_hex_dumps with line references */ 92 #define debug(format, arg...) printk(format, arg) 93 #else 94 #define debug(format, arg...) 95 #endif 96 97 98 static struct list_head hash_list; 99 100 /* ahash per-session context */ 101 struct caam_hash_ctx { 102 u32 sh_desc_update[DESC_HASH_MAX_USED_LEN] ____cacheline_aligned; 103 u32 sh_desc_update_first[DESC_HASH_MAX_USED_LEN] ____cacheline_aligned; 104 u32 sh_desc_fin[DESC_HASH_MAX_USED_LEN] ____cacheline_aligned; 105 u32 sh_desc_digest[DESC_HASH_MAX_USED_LEN] ____cacheline_aligned; 106 dma_addr_t sh_desc_update_dma ____cacheline_aligned; 107 dma_addr_t sh_desc_update_first_dma; 108 dma_addr_t sh_desc_fin_dma; 109 dma_addr_t sh_desc_digest_dma; 110 struct device *jrdev; 111 u8 key[CAAM_MAX_HASH_KEY_SIZE]; 112 int ctx_len; 113 struct alginfo adata; 114 }; 115 116 /* ahash state */ 117 struct caam_hash_state { 118 dma_addr_t buf_dma; 119 dma_addr_t ctx_dma; 120 u8 buf_0[CAAM_MAX_HASH_BLOCK_SIZE] ____cacheline_aligned; 121 int buflen_0; 122 u8 buf_1[CAAM_MAX_HASH_BLOCK_SIZE] ____cacheline_aligned; 123 int buflen_1; 124 u8 caam_ctx[MAX_CTX_LEN] ____cacheline_aligned; 125 int (*update)(struct ahash_request *req); 126 int (*final)(struct ahash_request *req); 127 int (*finup)(struct ahash_request *req); 128 int current_buf; 129 }; 130 131 struct caam_export_state { 132 u8 buf[CAAM_MAX_HASH_BLOCK_SIZE]; 133 u8 caam_ctx[MAX_CTX_LEN]; 134 int buflen; 135 int (*update)(struct ahash_request *req); 136 int (*final)(struct ahash_request *req); 137 int (*finup)(struct ahash_request *req); 138 }; 139 140 static inline void switch_buf(struct caam_hash_state *state) 141 { 142 state->current_buf ^= 1; 143 } 144 145 static inline u8 *current_buf(struct caam_hash_state *state) 146 { 147 return state->current_buf ? state->buf_1 : state->buf_0; 148 } 149 150 static inline u8 *alt_buf(struct caam_hash_state *state) 151 { 152 return state->current_buf ? state->buf_0 : state->buf_1; 153 } 154 155 static inline int *current_buflen(struct caam_hash_state *state) 156 { 157 return state->current_buf ? &state->buflen_1 : &state->buflen_0; 158 } 159 160 static inline int *alt_buflen(struct caam_hash_state *state) 161 { 162 return state->current_buf ? &state->buflen_0 : &state->buflen_1; 163 } 164 165 /* Common job descriptor seq in/out ptr routines */ 166 167 /* Map state->caam_ctx, and append seq_out_ptr command that points to it */ 168 static inline int map_seq_out_ptr_ctx(u32 *desc, struct device *jrdev, 169 struct caam_hash_state *state, 170 int ctx_len) 171 { 172 state->ctx_dma = dma_map_single(jrdev, state->caam_ctx, 173 ctx_len, DMA_FROM_DEVICE); 174 if (dma_mapping_error(jrdev, state->ctx_dma)) { 175 dev_err(jrdev, "unable to map ctx\n"); 176 state->ctx_dma = 0; 177 return -ENOMEM; 178 } 179 180 append_seq_out_ptr(desc, state->ctx_dma, ctx_len, 0); 181 182 return 0; 183 } 184 185 /* Map req->result, and append seq_out_ptr command that points to it */ 186 static inline dma_addr_t map_seq_out_ptr_result(u32 *desc, struct device *jrdev, 187 u8 *result, int digestsize) 188 { 189 dma_addr_t dst_dma; 190 191 dst_dma = dma_map_single(jrdev, result, digestsize, DMA_FROM_DEVICE); 192 append_seq_out_ptr(desc, dst_dma, digestsize, 0); 193 194 return dst_dma; 195 } 196 197 /* Map current buffer in state (if length > 0) and put it in link table */ 198 static inline int buf_map_to_sec4_sg(struct device *jrdev, 199 struct sec4_sg_entry *sec4_sg, 200 struct caam_hash_state *state) 201 { 202 int buflen = *current_buflen(state); 203 204 if (!buflen) 205 return 0; 206 207 state->buf_dma = dma_map_single(jrdev, current_buf(state), buflen, 208 DMA_TO_DEVICE); 209 if (dma_mapping_error(jrdev, state->buf_dma)) { 210 dev_err(jrdev, "unable to map buf\n"); 211 state->buf_dma = 0; 212 return -ENOMEM; 213 } 214 215 dma_to_sec4_sg_one(sec4_sg, state->buf_dma, buflen, 0); 216 217 return 0; 218 } 219 220 /* Map state->caam_ctx, and add it to link table */ 221 static inline int ctx_map_to_sec4_sg(u32 *desc, struct device *jrdev, 222 struct caam_hash_state *state, int ctx_len, 223 struct sec4_sg_entry *sec4_sg, u32 flag) 224 { 225 state->ctx_dma = dma_map_single(jrdev, state->caam_ctx, ctx_len, flag); 226 if (dma_mapping_error(jrdev, state->ctx_dma)) { 227 dev_err(jrdev, "unable to map ctx\n"); 228 state->ctx_dma = 0; 229 return -ENOMEM; 230 } 231 232 dma_to_sec4_sg_one(sec4_sg, state->ctx_dma, ctx_len, 0); 233 234 return 0; 235 } 236 237 /* 238 * For ahash update, final and finup (import_ctx = true) 239 * import context, read and write to seqout 240 * For ahash firsts and digest (import_ctx = false) 241 * read and write to seqout 242 */ 243 static inline void ahash_gen_sh_desc(u32 *desc, u32 state, int digestsize, 244 struct caam_hash_ctx *ctx, bool import_ctx) 245 { 246 u32 op = ctx->adata.algtype; 247 u32 *skip_key_load; 248 249 init_sh_desc(desc, HDR_SHARE_SERIAL); 250 251 /* Append key if it has been set; ahash update excluded */ 252 if ((state != OP_ALG_AS_UPDATE) && (ctx->adata.keylen)) { 253 /* Skip key loading if already shared */ 254 skip_key_load = append_jump(desc, JUMP_JSL | JUMP_TEST_ALL | 255 JUMP_COND_SHRD); 256 257 append_key_as_imm(desc, ctx->key, ctx->adata.keylen_pad, 258 ctx->adata.keylen, CLASS_2 | 259 KEY_DEST_MDHA_SPLIT | KEY_ENC); 260 261 set_jump_tgt_here(desc, skip_key_load); 262 263 op |= OP_ALG_AAI_HMAC_PRECOMP; 264 } 265 266 /* If needed, import context from software */ 267 if (import_ctx) 268 append_seq_load(desc, ctx->ctx_len, LDST_CLASS_2_CCB | 269 LDST_SRCDST_BYTE_CONTEXT); 270 271 /* Class 2 operation */ 272 append_operation(desc, op | state | OP_ALG_ENCRYPT); 273 274 /* 275 * Load from buf and/or src and write to req->result or state->context 276 * Calculate remaining bytes to read 277 */ 278 append_math_add(desc, VARSEQINLEN, SEQINLEN, REG0, CAAM_CMD_SZ); 279 /* Read remaining bytes */ 280 append_seq_fifo_load(desc, 0, FIFOLD_CLASS_CLASS2 | FIFOLD_TYPE_LAST2 | 281 FIFOLD_TYPE_MSG | KEY_VLF); 282 /* Store class2 context bytes */ 283 append_seq_store(desc, digestsize, LDST_CLASS_2_CCB | 284 LDST_SRCDST_BYTE_CONTEXT); 285 } 286 287 static int ahash_set_sh_desc(struct crypto_ahash *ahash) 288 { 289 struct caam_hash_ctx *ctx = crypto_ahash_ctx(ahash); 290 int digestsize = crypto_ahash_digestsize(ahash); 291 struct device *jrdev = ctx->jrdev; 292 u32 *desc; 293 294 /* ahash_update shared descriptor */ 295 desc = ctx->sh_desc_update; 296 ahash_gen_sh_desc(desc, OP_ALG_AS_UPDATE, ctx->ctx_len, ctx, true); 297 dma_sync_single_for_device(jrdev, ctx->sh_desc_update_dma, 298 desc_bytes(desc), DMA_TO_DEVICE); 299 #ifdef DEBUG 300 print_hex_dump(KERN_ERR, 301 "ahash update shdesc@"__stringify(__LINE__)": ", 302 DUMP_PREFIX_ADDRESS, 16, 4, desc, desc_bytes(desc), 1); 303 #endif 304 305 /* ahash_update_first shared descriptor */ 306 desc = ctx->sh_desc_update_first; 307 ahash_gen_sh_desc(desc, OP_ALG_AS_INIT, ctx->ctx_len, ctx, false); 308 dma_sync_single_for_device(jrdev, ctx->sh_desc_update_first_dma, 309 desc_bytes(desc), DMA_TO_DEVICE); 310 #ifdef DEBUG 311 print_hex_dump(KERN_ERR, 312 "ahash update first shdesc@"__stringify(__LINE__)": ", 313 DUMP_PREFIX_ADDRESS, 16, 4, desc, desc_bytes(desc), 1); 314 #endif 315 316 /* ahash_final shared descriptor */ 317 desc = ctx->sh_desc_fin; 318 ahash_gen_sh_desc(desc, OP_ALG_AS_FINALIZE, digestsize, ctx, true); 319 dma_sync_single_for_device(jrdev, ctx->sh_desc_fin_dma, 320 desc_bytes(desc), DMA_TO_DEVICE); 321 #ifdef DEBUG 322 print_hex_dump(KERN_ERR, "ahash final shdesc@"__stringify(__LINE__)": ", 323 DUMP_PREFIX_ADDRESS, 16, 4, desc, 324 desc_bytes(desc), 1); 325 #endif 326 327 /* ahash_digest shared descriptor */ 328 desc = ctx->sh_desc_digest; 329 ahash_gen_sh_desc(desc, OP_ALG_AS_INITFINAL, digestsize, ctx, false); 330 dma_sync_single_for_device(jrdev, ctx->sh_desc_digest_dma, 331 desc_bytes(desc), DMA_TO_DEVICE); 332 #ifdef DEBUG 333 print_hex_dump(KERN_ERR, 334 "ahash digest shdesc@"__stringify(__LINE__)": ", 335 DUMP_PREFIX_ADDRESS, 16, 4, desc, 336 desc_bytes(desc), 1); 337 #endif 338 339 return 0; 340 } 341 342 /* Digest hash size if it is too large */ 343 static int hash_digest_key(struct caam_hash_ctx *ctx, const u8 *key_in, 344 u32 *keylen, u8 *key_out, u32 digestsize) 345 { 346 struct device *jrdev = ctx->jrdev; 347 u32 *desc; 348 struct split_key_result result; 349 dma_addr_t src_dma, dst_dma; 350 int ret; 351 352 desc = kmalloc(CAAM_CMD_SZ * 8 + CAAM_PTR_SZ * 2, GFP_KERNEL | GFP_DMA); 353 if (!desc) { 354 dev_err(jrdev, "unable to allocate key input memory\n"); 355 return -ENOMEM; 356 } 357 358 init_job_desc(desc, 0); 359 360 src_dma = dma_map_single(jrdev, (void *)key_in, *keylen, 361 DMA_TO_DEVICE); 362 if (dma_mapping_error(jrdev, src_dma)) { 363 dev_err(jrdev, "unable to map key input memory\n"); 364 kfree(desc); 365 return -ENOMEM; 366 } 367 dst_dma = dma_map_single(jrdev, (void *)key_out, digestsize, 368 DMA_FROM_DEVICE); 369 if (dma_mapping_error(jrdev, dst_dma)) { 370 dev_err(jrdev, "unable to map key output memory\n"); 371 dma_unmap_single(jrdev, src_dma, *keylen, DMA_TO_DEVICE); 372 kfree(desc); 373 return -ENOMEM; 374 } 375 376 /* Job descriptor to perform unkeyed hash on key_in */ 377 append_operation(desc, ctx->adata.algtype | OP_ALG_ENCRYPT | 378 OP_ALG_AS_INITFINAL); 379 append_seq_in_ptr(desc, src_dma, *keylen, 0); 380 append_seq_fifo_load(desc, *keylen, FIFOLD_CLASS_CLASS2 | 381 FIFOLD_TYPE_LAST2 | FIFOLD_TYPE_MSG); 382 append_seq_out_ptr(desc, dst_dma, digestsize, 0); 383 append_seq_store(desc, digestsize, LDST_CLASS_2_CCB | 384 LDST_SRCDST_BYTE_CONTEXT); 385 386 #ifdef DEBUG 387 print_hex_dump(KERN_ERR, "key_in@"__stringify(__LINE__)": ", 388 DUMP_PREFIX_ADDRESS, 16, 4, key_in, *keylen, 1); 389 print_hex_dump(KERN_ERR, "jobdesc@"__stringify(__LINE__)": ", 390 DUMP_PREFIX_ADDRESS, 16, 4, desc, desc_bytes(desc), 1); 391 #endif 392 393 result.err = 0; 394 init_completion(&result.completion); 395 396 ret = caam_jr_enqueue(jrdev, desc, split_key_done, &result); 397 if (!ret) { 398 /* in progress */ 399 wait_for_completion(&result.completion); 400 ret = result.err; 401 #ifdef DEBUG 402 print_hex_dump(KERN_ERR, 403 "digested key@"__stringify(__LINE__)": ", 404 DUMP_PREFIX_ADDRESS, 16, 4, key_in, 405 digestsize, 1); 406 #endif 407 } 408 dma_unmap_single(jrdev, src_dma, *keylen, DMA_TO_DEVICE); 409 dma_unmap_single(jrdev, dst_dma, digestsize, DMA_FROM_DEVICE); 410 411 *keylen = digestsize; 412 413 kfree(desc); 414 415 return ret; 416 } 417 418 static int ahash_setkey(struct crypto_ahash *ahash, 419 const u8 *key, unsigned int keylen) 420 { 421 struct caam_hash_ctx *ctx = crypto_ahash_ctx(ahash); 422 int blocksize = crypto_tfm_alg_blocksize(&ahash->base); 423 int digestsize = crypto_ahash_digestsize(ahash); 424 int ret; 425 u8 *hashed_key = NULL; 426 427 #ifdef DEBUG 428 printk(KERN_ERR "keylen %d\n", keylen); 429 #endif 430 431 if (keylen > blocksize) { 432 hashed_key = kmalloc_array(digestsize, 433 sizeof(*hashed_key), 434 GFP_KERNEL | GFP_DMA); 435 if (!hashed_key) 436 return -ENOMEM; 437 ret = hash_digest_key(ctx, key, &keylen, hashed_key, 438 digestsize); 439 if (ret) 440 goto bad_free_key; 441 key = hashed_key; 442 } 443 444 ret = gen_split_key(ctx->jrdev, ctx->key, &ctx->adata, key, keylen, 445 CAAM_MAX_HASH_KEY_SIZE); 446 if (ret) 447 goto bad_free_key; 448 449 #ifdef DEBUG 450 print_hex_dump(KERN_ERR, "ctx.key@"__stringify(__LINE__)": ", 451 DUMP_PREFIX_ADDRESS, 16, 4, ctx->key, 452 ctx->adata.keylen_pad, 1); 453 #endif 454 455 kfree(hashed_key); 456 return ahash_set_sh_desc(ahash); 457 bad_free_key: 458 kfree(hashed_key); 459 crypto_ahash_set_flags(ahash, CRYPTO_TFM_RES_BAD_KEY_LEN); 460 return -EINVAL; 461 } 462 463 /* 464 * ahash_edesc - s/w-extended ahash descriptor 465 * @dst_dma: physical mapped address of req->result 466 * @sec4_sg_dma: physical mapped address of h/w link table 467 * @src_nents: number of segments in input scatterlist 468 * @sec4_sg_bytes: length of dma mapped sec4_sg space 469 * @hw_desc: the h/w job descriptor followed by any referenced link tables 470 * @sec4_sg: h/w link table 471 */ 472 struct ahash_edesc { 473 dma_addr_t dst_dma; 474 dma_addr_t sec4_sg_dma; 475 int src_nents; 476 int sec4_sg_bytes; 477 u32 hw_desc[DESC_JOB_IO_LEN / sizeof(u32)] ____cacheline_aligned; 478 struct sec4_sg_entry sec4_sg[0]; 479 }; 480 481 static inline void ahash_unmap(struct device *dev, 482 struct ahash_edesc *edesc, 483 struct ahash_request *req, int dst_len) 484 { 485 struct caam_hash_state *state = ahash_request_ctx(req); 486 487 if (edesc->src_nents) 488 dma_unmap_sg(dev, req->src, edesc->src_nents, DMA_TO_DEVICE); 489 if (edesc->dst_dma) 490 dma_unmap_single(dev, edesc->dst_dma, dst_len, DMA_FROM_DEVICE); 491 492 if (edesc->sec4_sg_bytes) 493 dma_unmap_single(dev, edesc->sec4_sg_dma, 494 edesc->sec4_sg_bytes, DMA_TO_DEVICE); 495 496 if (state->buf_dma) { 497 dma_unmap_single(dev, state->buf_dma, *current_buflen(state), 498 DMA_TO_DEVICE); 499 state->buf_dma = 0; 500 } 501 } 502 503 static inline void ahash_unmap_ctx(struct device *dev, 504 struct ahash_edesc *edesc, 505 struct ahash_request *req, int dst_len, u32 flag) 506 { 507 struct crypto_ahash *ahash = crypto_ahash_reqtfm(req); 508 struct caam_hash_ctx *ctx = crypto_ahash_ctx(ahash); 509 struct caam_hash_state *state = ahash_request_ctx(req); 510 511 if (state->ctx_dma) { 512 dma_unmap_single(dev, state->ctx_dma, ctx->ctx_len, flag); 513 state->ctx_dma = 0; 514 } 515 ahash_unmap(dev, edesc, req, dst_len); 516 } 517 518 static void ahash_done(struct device *jrdev, u32 *desc, u32 err, 519 void *context) 520 { 521 struct ahash_request *req = context; 522 struct ahash_edesc *edesc; 523 struct crypto_ahash *ahash = crypto_ahash_reqtfm(req); 524 int digestsize = crypto_ahash_digestsize(ahash); 525 #ifdef DEBUG 526 struct caam_hash_ctx *ctx = crypto_ahash_ctx(ahash); 527 struct caam_hash_state *state = ahash_request_ctx(req); 528 529 dev_err(jrdev, "%s %d: err 0x%x\n", __func__, __LINE__, err); 530 #endif 531 532 edesc = container_of(desc, struct ahash_edesc, hw_desc[0]); 533 if (err) 534 caam_jr_strstatus(jrdev, err); 535 536 ahash_unmap(jrdev, edesc, req, digestsize); 537 kfree(edesc); 538 539 #ifdef DEBUG 540 print_hex_dump(KERN_ERR, "ctx@"__stringify(__LINE__)": ", 541 DUMP_PREFIX_ADDRESS, 16, 4, state->caam_ctx, 542 ctx->ctx_len, 1); 543 if (req->result) 544 print_hex_dump(KERN_ERR, "result@"__stringify(__LINE__)": ", 545 DUMP_PREFIX_ADDRESS, 16, 4, req->result, 546 digestsize, 1); 547 #endif 548 549 req->base.complete(&req->base, err); 550 } 551 552 static void ahash_done_bi(struct device *jrdev, u32 *desc, u32 err, 553 void *context) 554 { 555 struct ahash_request *req = context; 556 struct ahash_edesc *edesc; 557 struct crypto_ahash *ahash = crypto_ahash_reqtfm(req); 558 struct caam_hash_ctx *ctx = crypto_ahash_ctx(ahash); 559 struct caam_hash_state *state = ahash_request_ctx(req); 560 #ifdef DEBUG 561 int digestsize = crypto_ahash_digestsize(ahash); 562 563 dev_err(jrdev, "%s %d: err 0x%x\n", __func__, __LINE__, err); 564 #endif 565 566 edesc = container_of(desc, struct ahash_edesc, hw_desc[0]); 567 if (err) 568 caam_jr_strstatus(jrdev, err); 569 570 ahash_unmap_ctx(jrdev, edesc, req, ctx->ctx_len, DMA_BIDIRECTIONAL); 571 switch_buf(state); 572 kfree(edesc); 573 574 #ifdef DEBUG 575 print_hex_dump(KERN_ERR, "ctx@"__stringify(__LINE__)": ", 576 DUMP_PREFIX_ADDRESS, 16, 4, state->caam_ctx, 577 ctx->ctx_len, 1); 578 if (req->result) 579 print_hex_dump(KERN_ERR, "result@"__stringify(__LINE__)": ", 580 DUMP_PREFIX_ADDRESS, 16, 4, req->result, 581 digestsize, 1); 582 #endif 583 584 req->base.complete(&req->base, err); 585 } 586 587 static void ahash_done_ctx_src(struct device *jrdev, u32 *desc, u32 err, 588 void *context) 589 { 590 struct ahash_request *req = context; 591 struct ahash_edesc *edesc; 592 struct crypto_ahash *ahash = crypto_ahash_reqtfm(req); 593 int digestsize = crypto_ahash_digestsize(ahash); 594 #ifdef DEBUG 595 struct caam_hash_ctx *ctx = crypto_ahash_ctx(ahash); 596 struct caam_hash_state *state = ahash_request_ctx(req); 597 598 dev_err(jrdev, "%s %d: err 0x%x\n", __func__, __LINE__, err); 599 #endif 600 601 edesc = container_of(desc, struct ahash_edesc, hw_desc[0]); 602 if (err) 603 caam_jr_strstatus(jrdev, err); 604 605 ahash_unmap_ctx(jrdev, edesc, req, digestsize, DMA_TO_DEVICE); 606 kfree(edesc); 607 608 #ifdef DEBUG 609 print_hex_dump(KERN_ERR, "ctx@"__stringify(__LINE__)": ", 610 DUMP_PREFIX_ADDRESS, 16, 4, state->caam_ctx, 611 ctx->ctx_len, 1); 612 if (req->result) 613 print_hex_dump(KERN_ERR, "result@"__stringify(__LINE__)": ", 614 DUMP_PREFIX_ADDRESS, 16, 4, req->result, 615 digestsize, 1); 616 #endif 617 618 req->base.complete(&req->base, err); 619 } 620 621 static void ahash_done_ctx_dst(struct device *jrdev, u32 *desc, u32 err, 622 void *context) 623 { 624 struct ahash_request *req = context; 625 struct ahash_edesc *edesc; 626 struct crypto_ahash *ahash = crypto_ahash_reqtfm(req); 627 struct caam_hash_ctx *ctx = crypto_ahash_ctx(ahash); 628 struct caam_hash_state *state = ahash_request_ctx(req); 629 #ifdef DEBUG 630 int digestsize = crypto_ahash_digestsize(ahash); 631 632 dev_err(jrdev, "%s %d: err 0x%x\n", __func__, __LINE__, err); 633 #endif 634 635 edesc = container_of(desc, struct ahash_edesc, hw_desc[0]); 636 if (err) 637 caam_jr_strstatus(jrdev, err); 638 639 ahash_unmap_ctx(jrdev, edesc, req, ctx->ctx_len, DMA_FROM_DEVICE); 640 switch_buf(state); 641 kfree(edesc); 642 643 #ifdef DEBUG 644 print_hex_dump(KERN_ERR, "ctx@"__stringify(__LINE__)": ", 645 DUMP_PREFIX_ADDRESS, 16, 4, state->caam_ctx, 646 ctx->ctx_len, 1); 647 if (req->result) 648 print_hex_dump(KERN_ERR, "result@"__stringify(__LINE__)": ", 649 DUMP_PREFIX_ADDRESS, 16, 4, req->result, 650 digestsize, 1); 651 #endif 652 653 req->base.complete(&req->base, err); 654 } 655 656 /* 657 * Allocate an enhanced descriptor, which contains the hardware descriptor 658 * and space for hardware scatter table containing sg_num entries. 659 */ 660 static struct ahash_edesc *ahash_edesc_alloc(struct caam_hash_ctx *ctx, 661 int sg_num, u32 *sh_desc, 662 dma_addr_t sh_desc_dma, 663 gfp_t flags) 664 { 665 struct ahash_edesc *edesc; 666 unsigned int sg_size = sg_num * sizeof(struct sec4_sg_entry); 667 668 edesc = kzalloc(sizeof(*edesc) + sg_size, GFP_DMA | flags); 669 if (!edesc) { 670 dev_err(ctx->jrdev, "could not allocate extended descriptor\n"); 671 return NULL; 672 } 673 674 init_job_desc_shared(edesc->hw_desc, sh_desc_dma, desc_len(sh_desc), 675 HDR_SHARE_DEFER | HDR_REVERSE); 676 677 return edesc; 678 } 679 680 static int ahash_edesc_add_src(struct caam_hash_ctx *ctx, 681 struct ahash_edesc *edesc, 682 struct ahash_request *req, int nents, 683 unsigned int first_sg, 684 unsigned int first_bytes, size_t to_hash) 685 { 686 dma_addr_t src_dma; 687 u32 options; 688 689 if (nents > 1 || first_sg) { 690 struct sec4_sg_entry *sg = edesc->sec4_sg; 691 unsigned int sgsize = sizeof(*sg) * (first_sg + nents); 692 693 sg_to_sec4_sg_last(req->src, nents, sg + first_sg, 0); 694 695 src_dma = dma_map_single(ctx->jrdev, sg, sgsize, DMA_TO_DEVICE); 696 if (dma_mapping_error(ctx->jrdev, src_dma)) { 697 dev_err(ctx->jrdev, "unable to map S/G table\n"); 698 return -ENOMEM; 699 } 700 701 edesc->sec4_sg_bytes = sgsize; 702 edesc->sec4_sg_dma = src_dma; 703 options = LDST_SGF; 704 } else { 705 src_dma = sg_dma_address(req->src); 706 options = 0; 707 } 708 709 append_seq_in_ptr(edesc->hw_desc, src_dma, first_bytes + to_hash, 710 options); 711 712 return 0; 713 } 714 715 /* submit update job descriptor */ 716 static int ahash_update_ctx(struct ahash_request *req) 717 { 718 struct crypto_ahash *ahash = crypto_ahash_reqtfm(req); 719 struct caam_hash_ctx *ctx = crypto_ahash_ctx(ahash); 720 struct caam_hash_state *state = ahash_request_ctx(req); 721 struct device *jrdev = ctx->jrdev; 722 gfp_t flags = (req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP) ? 723 GFP_KERNEL : GFP_ATOMIC; 724 u8 *buf = current_buf(state); 725 int *buflen = current_buflen(state); 726 u8 *next_buf = alt_buf(state); 727 int *next_buflen = alt_buflen(state), last_buflen; 728 int in_len = *buflen + req->nbytes, to_hash; 729 u32 *desc; 730 int src_nents, mapped_nents, sec4_sg_bytes, sec4_sg_src_index; 731 struct ahash_edesc *edesc; 732 int ret = 0; 733 734 last_buflen = *next_buflen; 735 *next_buflen = in_len & (crypto_tfm_alg_blocksize(&ahash->base) - 1); 736 to_hash = in_len - *next_buflen; 737 738 if (to_hash) { 739 src_nents = sg_nents_for_len(req->src, 740 req->nbytes - (*next_buflen)); 741 if (src_nents < 0) { 742 dev_err(jrdev, "Invalid number of src SG.\n"); 743 return src_nents; 744 } 745 746 if (src_nents) { 747 mapped_nents = dma_map_sg(jrdev, req->src, src_nents, 748 DMA_TO_DEVICE); 749 if (!mapped_nents) { 750 dev_err(jrdev, "unable to DMA map source\n"); 751 return -ENOMEM; 752 } 753 } else { 754 mapped_nents = 0; 755 } 756 757 sec4_sg_src_index = 1 + (*buflen ? 1 : 0); 758 sec4_sg_bytes = (sec4_sg_src_index + mapped_nents) * 759 sizeof(struct sec4_sg_entry); 760 761 /* 762 * allocate space for base edesc and hw desc commands, 763 * link tables 764 */ 765 edesc = ahash_edesc_alloc(ctx, sec4_sg_src_index + mapped_nents, 766 ctx->sh_desc_update, 767 ctx->sh_desc_update_dma, flags); 768 if (!edesc) { 769 dma_unmap_sg(jrdev, req->src, src_nents, DMA_TO_DEVICE); 770 return -ENOMEM; 771 } 772 773 edesc->src_nents = src_nents; 774 edesc->sec4_sg_bytes = sec4_sg_bytes; 775 776 ret = ctx_map_to_sec4_sg(desc, jrdev, state, ctx->ctx_len, 777 edesc->sec4_sg, DMA_BIDIRECTIONAL); 778 if (ret) 779 goto unmap_ctx; 780 781 ret = buf_map_to_sec4_sg(jrdev, edesc->sec4_sg + 1, state); 782 if (ret) 783 goto unmap_ctx; 784 785 if (mapped_nents) { 786 sg_to_sec4_sg_last(req->src, mapped_nents, 787 edesc->sec4_sg + sec4_sg_src_index, 788 0); 789 if (*next_buflen) 790 scatterwalk_map_and_copy(next_buf, req->src, 791 to_hash - *buflen, 792 *next_buflen, 0); 793 } else { 794 sg_to_sec4_set_last(edesc->sec4_sg + sec4_sg_src_index - 795 1); 796 } 797 798 desc = edesc->hw_desc; 799 800 edesc->sec4_sg_dma = dma_map_single(jrdev, edesc->sec4_sg, 801 sec4_sg_bytes, 802 DMA_TO_DEVICE); 803 if (dma_mapping_error(jrdev, edesc->sec4_sg_dma)) { 804 dev_err(jrdev, "unable to map S/G table\n"); 805 ret = -ENOMEM; 806 goto unmap_ctx; 807 } 808 809 append_seq_in_ptr(desc, edesc->sec4_sg_dma, ctx->ctx_len + 810 to_hash, LDST_SGF); 811 812 append_seq_out_ptr(desc, state->ctx_dma, ctx->ctx_len, 0); 813 814 #ifdef DEBUG 815 print_hex_dump(KERN_ERR, "jobdesc@"__stringify(__LINE__)": ", 816 DUMP_PREFIX_ADDRESS, 16, 4, desc, 817 desc_bytes(desc), 1); 818 #endif 819 820 ret = caam_jr_enqueue(jrdev, desc, ahash_done_bi, req); 821 if (ret) 822 goto unmap_ctx; 823 824 ret = -EINPROGRESS; 825 } else if (*next_buflen) { 826 scatterwalk_map_and_copy(buf + *buflen, req->src, 0, 827 req->nbytes, 0); 828 *buflen = *next_buflen; 829 *next_buflen = last_buflen; 830 } 831 #ifdef DEBUG 832 print_hex_dump(KERN_ERR, "buf@"__stringify(__LINE__)": ", 833 DUMP_PREFIX_ADDRESS, 16, 4, buf, *buflen, 1); 834 print_hex_dump(KERN_ERR, "next buf@"__stringify(__LINE__)": ", 835 DUMP_PREFIX_ADDRESS, 16, 4, next_buf, 836 *next_buflen, 1); 837 #endif 838 839 return ret; 840 unmap_ctx: 841 ahash_unmap_ctx(jrdev, edesc, req, ctx->ctx_len, DMA_BIDIRECTIONAL); 842 kfree(edesc); 843 return ret; 844 } 845 846 static int ahash_final_ctx(struct ahash_request *req) 847 { 848 struct crypto_ahash *ahash = crypto_ahash_reqtfm(req); 849 struct caam_hash_ctx *ctx = crypto_ahash_ctx(ahash); 850 struct caam_hash_state *state = ahash_request_ctx(req); 851 struct device *jrdev = ctx->jrdev; 852 gfp_t flags = (req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP) ? 853 GFP_KERNEL : GFP_ATOMIC; 854 int buflen = *current_buflen(state); 855 u32 *desc; 856 int sec4_sg_bytes, sec4_sg_src_index; 857 int digestsize = crypto_ahash_digestsize(ahash); 858 struct ahash_edesc *edesc; 859 int ret; 860 861 sec4_sg_src_index = 1 + (buflen ? 1 : 0); 862 sec4_sg_bytes = sec4_sg_src_index * sizeof(struct sec4_sg_entry); 863 864 /* allocate space for base edesc and hw desc commands, link tables */ 865 edesc = ahash_edesc_alloc(ctx, sec4_sg_src_index, 866 ctx->sh_desc_fin, ctx->sh_desc_fin_dma, 867 flags); 868 if (!edesc) 869 return -ENOMEM; 870 871 desc = edesc->hw_desc; 872 873 edesc->sec4_sg_bytes = sec4_sg_bytes; 874 edesc->src_nents = 0; 875 876 ret = ctx_map_to_sec4_sg(desc, jrdev, state, ctx->ctx_len, 877 edesc->sec4_sg, DMA_TO_DEVICE); 878 if (ret) 879 goto unmap_ctx; 880 881 ret = buf_map_to_sec4_sg(jrdev, edesc->sec4_sg + 1, state); 882 if (ret) 883 goto unmap_ctx; 884 885 sg_to_sec4_set_last(edesc->sec4_sg + sec4_sg_src_index - 1); 886 887 edesc->sec4_sg_dma = dma_map_single(jrdev, edesc->sec4_sg, 888 sec4_sg_bytes, DMA_TO_DEVICE); 889 if (dma_mapping_error(jrdev, edesc->sec4_sg_dma)) { 890 dev_err(jrdev, "unable to map S/G table\n"); 891 ret = -ENOMEM; 892 goto unmap_ctx; 893 } 894 895 append_seq_in_ptr(desc, edesc->sec4_sg_dma, ctx->ctx_len + buflen, 896 LDST_SGF); 897 898 edesc->dst_dma = map_seq_out_ptr_result(desc, jrdev, req->result, 899 digestsize); 900 if (dma_mapping_error(jrdev, edesc->dst_dma)) { 901 dev_err(jrdev, "unable to map dst\n"); 902 ret = -ENOMEM; 903 goto unmap_ctx; 904 } 905 906 #ifdef DEBUG 907 print_hex_dump(KERN_ERR, "jobdesc@"__stringify(__LINE__)": ", 908 DUMP_PREFIX_ADDRESS, 16, 4, desc, desc_bytes(desc), 1); 909 #endif 910 911 ret = caam_jr_enqueue(jrdev, desc, ahash_done_ctx_src, req); 912 if (ret) 913 goto unmap_ctx; 914 915 return -EINPROGRESS; 916 unmap_ctx: 917 ahash_unmap_ctx(jrdev, edesc, req, digestsize, DMA_FROM_DEVICE); 918 kfree(edesc); 919 return ret; 920 } 921 922 static int ahash_finup_ctx(struct ahash_request *req) 923 { 924 struct crypto_ahash *ahash = crypto_ahash_reqtfm(req); 925 struct caam_hash_ctx *ctx = crypto_ahash_ctx(ahash); 926 struct caam_hash_state *state = ahash_request_ctx(req); 927 struct device *jrdev = ctx->jrdev; 928 gfp_t flags = (req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP) ? 929 GFP_KERNEL : GFP_ATOMIC; 930 int buflen = *current_buflen(state); 931 u32 *desc; 932 int sec4_sg_src_index; 933 int src_nents, mapped_nents; 934 int digestsize = crypto_ahash_digestsize(ahash); 935 struct ahash_edesc *edesc; 936 int ret; 937 938 src_nents = sg_nents_for_len(req->src, req->nbytes); 939 if (src_nents < 0) { 940 dev_err(jrdev, "Invalid number of src SG.\n"); 941 return src_nents; 942 } 943 944 if (src_nents) { 945 mapped_nents = dma_map_sg(jrdev, req->src, src_nents, 946 DMA_TO_DEVICE); 947 if (!mapped_nents) { 948 dev_err(jrdev, "unable to DMA map source\n"); 949 return -ENOMEM; 950 } 951 } else { 952 mapped_nents = 0; 953 } 954 955 sec4_sg_src_index = 1 + (buflen ? 1 : 0); 956 957 /* allocate space for base edesc and hw desc commands, link tables */ 958 edesc = ahash_edesc_alloc(ctx, sec4_sg_src_index + mapped_nents, 959 ctx->sh_desc_fin, ctx->sh_desc_fin_dma, 960 flags); 961 if (!edesc) { 962 dma_unmap_sg(jrdev, req->src, src_nents, DMA_TO_DEVICE); 963 return -ENOMEM; 964 } 965 966 desc = edesc->hw_desc; 967 968 edesc->src_nents = src_nents; 969 970 ret = ctx_map_to_sec4_sg(desc, jrdev, state, ctx->ctx_len, 971 edesc->sec4_sg, DMA_TO_DEVICE); 972 if (ret) 973 goto unmap_ctx; 974 975 ret = buf_map_to_sec4_sg(jrdev, edesc->sec4_sg + 1, state); 976 if (ret) 977 goto unmap_ctx; 978 979 ret = ahash_edesc_add_src(ctx, edesc, req, mapped_nents, 980 sec4_sg_src_index, ctx->ctx_len + buflen, 981 req->nbytes); 982 if (ret) 983 goto unmap_ctx; 984 985 edesc->dst_dma = map_seq_out_ptr_result(desc, jrdev, req->result, 986 digestsize); 987 if (dma_mapping_error(jrdev, edesc->dst_dma)) { 988 dev_err(jrdev, "unable to map dst\n"); 989 ret = -ENOMEM; 990 goto unmap_ctx; 991 } 992 993 #ifdef DEBUG 994 print_hex_dump(KERN_ERR, "jobdesc@"__stringify(__LINE__)": ", 995 DUMP_PREFIX_ADDRESS, 16, 4, desc, desc_bytes(desc), 1); 996 #endif 997 998 ret = caam_jr_enqueue(jrdev, desc, ahash_done_ctx_src, req); 999 if (ret) 1000 goto unmap_ctx; 1001 1002 return -EINPROGRESS; 1003 unmap_ctx: 1004 ahash_unmap_ctx(jrdev, edesc, req, digestsize, DMA_FROM_DEVICE); 1005 kfree(edesc); 1006 return ret; 1007 } 1008 1009 static int ahash_digest(struct ahash_request *req) 1010 { 1011 struct crypto_ahash *ahash = crypto_ahash_reqtfm(req); 1012 struct caam_hash_ctx *ctx = crypto_ahash_ctx(ahash); 1013 struct caam_hash_state *state = ahash_request_ctx(req); 1014 struct device *jrdev = ctx->jrdev; 1015 gfp_t flags = (req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP) ? 1016 GFP_KERNEL : GFP_ATOMIC; 1017 u32 *desc; 1018 int digestsize = crypto_ahash_digestsize(ahash); 1019 int src_nents, mapped_nents; 1020 struct ahash_edesc *edesc; 1021 int ret; 1022 1023 state->buf_dma = 0; 1024 1025 src_nents = sg_nents_for_len(req->src, req->nbytes); 1026 if (src_nents < 0) { 1027 dev_err(jrdev, "Invalid number of src SG.\n"); 1028 return src_nents; 1029 } 1030 1031 if (src_nents) { 1032 mapped_nents = dma_map_sg(jrdev, req->src, src_nents, 1033 DMA_TO_DEVICE); 1034 if (!mapped_nents) { 1035 dev_err(jrdev, "unable to map source for DMA\n"); 1036 return -ENOMEM; 1037 } 1038 } else { 1039 mapped_nents = 0; 1040 } 1041 1042 /* allocate space for base edesc and hw desc commands, link tables */ 1043 edesc = ahash_edesc_alloc(ctx, mapped_nents > 1 ? mapped_nents : 0, 1044 ctx->sh_desc_digest, ctx->sh_desc_digest_dma, 1045 flags); 1046 if (!edesc) { 1047 dma_unmap_sg(jrdev, req->src, src_nents, DMA_TO_DEVICE); 1048 return -ENOMEM; 1049 } 1050 1051 edesc->src_nents = src_nents; 1052 1053 ret = ahash_edesc_add_src(ctx, edesc, req, mapped_nents, 0, 0, 1054 req->nbytes); 1055 if (ret) { 1056 ahash_unmap(jrdev, edesc, req, digestsize); 1057 kfree(edesc); 1058 return ret; 1059 } 1060 1061 desc = edesc->hw_desc; 1062 1063 edesc->dst_dma = map_seq_out_ptr_result(desc, jrdev, req->result, 1064 digestsize); 1065 if (dma_mapping_error(jrdev, edesc->dst_dma)) { 1066 dev_err(jrdev, "unable to map dst\n"); 1067 ahash_unmap(jrdev, edesc, req, digestsize); 1068 kfree(edesc); 1069 return -ENOMEM; 1070 } 1071 1072 #ifdef DEBUG 1073 print_hex_dump(KERN_ERR, "jobdesc@"__stringify(__LINE__)": ", 1074 DUMP_PREFIX_ADDRESS, 16, 4, desc, desc_bytes(desc), 1); 1075 #endif 1076 1077 ret = caam_jr_enqueue(jrdev, desc, ahash_done, req); 1078 if (!ret) { 1079 ret = -EINPROGRESS; 1080 } else { 1081 ahash_unmap(jrdev, edesc, req, digestsize); 1082 kfree(edesc); 1083 } 1084 1085 return ret; 1086 } 1087 1088 /* submit ahash final if it the first job descriptor */ 1089 static int ahash_final_no_ctx(struct ahash_request *req) 1090 { 1091 struct crypto_ahash *ahash = crypto_ahash_reqtfm(req); 1092 struct caam_hash_ctx *ctx = crypto_ahash_ctx(ahash); 1093 struct caam_hash_state *state = ahash_request_ctx(req); 1094 struct device *jrdev = ctx->jrdev; 1095 gfp_t flags = (req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP) ? 1096 GFP_KERNEL : GFP_ATOMIC; 1097 u8 *buf = current_buf(state); 1098 int buflen = *current_buflen(state); 1099 u32 *desc; 1100 int digestsize = crypto_ahash_digestsize(ahash); 1101 struct ahash_edesc *edesc; 1102 int ret; 1103 1104 /* allocate space for base edesc and hw desc commands, link tables */ 1105 edesc = ahash_edesc_alloc(ctx, 0, ctx->sh_desc_digest, 1106 ctx->sh_desc_digest_dma, flags); 1107 if (!edesc) 1108 return -ENOMEM; 1109 1110 desc = edesc->hw_desc; 1111 1112 state->buf_dma = dma_map_single(jrdev, buf, buflen, DMA_TO_DEVICE); 1113 if (dma_mapping_error(jrdev, state->buf_dma)) { 1114 dev_err(jrdev, "unable to map src\n"); 1115 goto unmap; 1116 } 1117 1118 append_seq_in_ptr(desc, state->buf_dma, buflen, 0); 1119 1120 edesc->dst_dma = map_seq_out_ptr_result(desc, jrdev, req->result, 1121 digestsize); 1122 if (dma_mapping_error(jrdev, edesc->dst_dma)) { 1123 dev_err(jrdev, "unable to map dst\n"); 1124 goto unmap; 1125 } 1126 edesc->src_nents = 0; 1127 1128 #ifdef DEBUG 1129 print_hex_dump(KERN_ERR, "jobdesc@"__stringify(__LINE__)": ", 1130 DUMP_PREFIX_ADDRESS, 16, 4, desc, desc_bytes(desc), 1); 1131 #endif 1132 1133 ret = caam_jr_enqueue(jrdev, desc, ahash_done, req); 1134 if (!ret) { 1135 ret = -EINPROGRESS; 1136 } else { 1137 ahash_unmap(jrdev, edesc, req, digestsize); 1138 kfree(edesc); 1139 } 1140 1141 return ret; 1142 unmap: 1143 ahash_unmap(jrdev, edesc, req, digestsize); 1144 kfree(edesc); 1145 return -ENOMEM; 1146 1147 } 1148 1149 /* submit ahash update if it the first job descriptor after update */ 1150 static int ahash_update_no_ctx(struct ahash_request *req) 1151 { 1152 struct crypto_ahash *ahash = crypto_ahash_reqtfm(req); 1153 struct caam_hash_ctx *ctx = crypto_ahash_ctx(ahash); 1154 struct caam_hash_state *state = ahash_request_ctx(req); 1155 struct device *jrdev = ctx->jrdev; 1156 gfp_t flags = (req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP) ? 1157 GFP_KERNEL : GFP_ATOMIC; 1158 u8 *buf = current_buf(state); 1159 int *buflen = current_buflen(state); 1160 u8 *next_buf = alt_buf(state); 1161 int *next_buflen = alt_buflen(state); 1162 int in_len = *buflen + req->nbytes, to_hash; 1163 int sec4_sg_bytes, src_nents, mapped_nents; 1164 struct ahash_edesc *edesc; 1165 u32 *desc; 1166 int ret = 0; 1167 1168 *next_buflen = in_len & (crypto_tfm_alg_blocksize(&ahash->base) - 1); 1169 to_hash = in_len - *next_buflen; 1170 1171 if (to_hash) { 1172 src_nents = sg_nents_for_len(req->src, 1173 req->nbytes - *next_buflen); 1174 if (src_nents < 0) { 1175 dev_err(jrdev, "Invalid number of src SG.\n"); 1176 return src_nents; 1177 } 1178 1179 if (src_nents) { 1180 mapped_nents = dma_map_sg(jrdev, req->src, src_nents, 1181 DMA_TO_DEVICE); 1182 if (!mapped_nents) { 1183 dev_err(jrdev, "unable to DMA map source\n"); 1184 return -ENOMEM; 1185 } 1186 } else { 1187 mapped_nents = 0; 1188 } 1189 1190 sec4_sg_bytes = (1 + mapped_nents) * 1191 sizeof(struct sec4_sg_entry); 1192 1193 /* 1194 * allocate space for base edesc and hw desc commands, 1195 * link tables 1196 */ 1197 edesc = ahash_edesc_alloc(ctx, 1 + mapped_nents, 1198 ctx->sh_desc_update_first, 1199 ctx->sh_desc_update_first_dma, 1200 flags); 1201 if (!edesc) { 1202 dma_unmap_sg(jrdev, req->src, src_nents, DMA_TO_DEVICE); 1203 return -ENOMEM; 1204 } 1205 1206 edesc->src_nents = src_nents; 1207 edesc->sec4_sg_bytes = sec4_sg_bytes; 1208 edesc->dst_dma = 0; 1209 1210 ret = buf_map_to_sec4_sg(jrdev, edesc->sec4_sg, state); 1211 if (ret) 1212 goto unmap_ctx; 1213 1214 sg_to_sec4_sg_last(req->src, mapped_nents, 1215 edesc->sec4_sg + 1, 0); 1216 1217 if (*next_buflen) { 1218 scatterwalk_map_and_copy(next_buf, req->src, 1219 to_hash - *buflen, 1220 *next_buflen, 0); 1221 } 1222 1223 desc = edesc->hw_desc; 1224 1225 edesc->sec4_sg_dma = dma_map_single(jrdev, edesc->sec4_sg, 1226 sec4_sg_bytes, 1227 DMA_TO_DEVICE); 1228 if (dma_mapping_error(jrdev, edesc->sec4_sg_dma)) { 1229 dev_err(jrdev, "unable to map S/G table\n"); 1230 ret = -ENOMEM; 1231 goto unmap_ctx; 1232 } 1233 1234 append_seq_in_ptr(desc, edesc->sec4_sg_dma, to_hash, LDST_SGF); 1235 1236 ret = map_seq_out_ptr_ctx(desc, jrdev, state, ctx->ctx_len); 1237 if (ret) 1238 goto unmap_ctx; 1239 1240 #ifdef DEBUG 1241 print_hex_dump(KERN_ERR, "jobdesc@"__stringify(__LINE__)": ", 1242 DUMP_PREFIX_ADDRESS, 16, 4, desc, 1243 desc_bytes(desc), 1); 1244 #endif 1245 1246 ret = caam_jr_enqueue(jrdev, desc, ahash_done_ctx_dst, req); 1247 if (ret) 1248 goto unmap_ctx; 1249 1250 ret = -EINPROGRESS; 1251 state->update = ahash_update_ctx; 1252 state->finup = ahash_finup_ctx; 1253 state->final = ahash_final_ctx; 1254 } else if (*next_buflen) { 1255 scatterwalk_map_and_copy(buf + *buflen, req->src, 0, 1256 req->nbytes, 0); 1257 *buflen = *next_buflen; 1258 *next_buflen = 0; 1259 } 1260 #ifdef DEBUG 1261 print_hex_dump(KERN_ERR, "buf@"__stringify(__LINE__)": ", 1262 DUMP_PREFIX_ADDRESS, 16, 4, buf, *buflen, 1); 1263 print_hex_dump(KERN_ERR, "next buf@"__stringify(__LINE__)": ", 1264 DUMP_PREFIX_ADDRESS, 16, 4, next_buf, 1265 *next_buflen, 1); 1266 #endif 1267 1268 return ret; 1269 unmap_ctx: 1270 ahash_unmap_ctx(jrdev, edesc, req, ctx->ctx_len, DMA_TO_DEVICE); 1271 kfree(edesc); 1272 return ret; 1273 } 1274 1275 /* submit ahash finup if it the first job descriptor after update */ 1276 static int ahash_finup_no_ctx(struct ahash_request *req) 1277 { 1278 struct crypto_ahash *ahash = crypto_ahash_reqtfm(req); 1279 struct caam_hash_ctx *ctx = crypto_ahash_ctx(ahash); 1280 struct caam_hash_state *state = ahash_request_ctx(req); 1281 struct device *jrdev = ctx->jrdev; 1282 gfp_t flags = (req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP) ? 1283 GFP_KERNEL : GFP_ATOMIC; 1284 int buflen = *current_buflen(state); 1285 u32 *desc; 1286 int sec4_sg_bytes, sec4_sg_src_index, src_nents, mapped_nents; 1287 int digestsize = crypto_ahash_digestsize(ahash); 1288 struct ahash_edesc *edesc; 1289 int ret; 1290 1291 src_nents = sg_nents_for_len(req->src, req->nbytes); 1292 if (src_nents < 0) { 1293 dev_err(jrdev, "Invalid number of src SG.\n"); 1294 return src_nents; 1295 } 1296 1297 if (src_nents) { 1298 mapped_nents = dma_map_sg(jrdev, req->src, src_nents, 1299 DMA_TO_DEVICE); 1300 if (!mapped_nents) { 1301 dev_err(jrdev, "unable to DMA map source\n"); 1302 return -ENOMEM; 1303 } 1304 } else { 1305 mapped_nents = 0; 1306 } 1307 1308 sec4_sg_src_index = 2; 1309 sec4_sg_bytes = (sec4_sg_src_index + mapped_nents) * 1310 sizeof(struct sec4_sg_entry); 1311 1312 /* allocate space for base edesc and hw desc commands, link tables */ 1313 edesc = ahash_edesc_alloc(ctx, sec4_sg_src_index + mapped_nents, 1314 ctx->sh_desc_digest, ctx->sh_desc_digest_dma, 1315 flags); 1316 if (!edesc) { 1317 dma_unmap_sg(jrdev, req->src, src_nents, DMA_TO_DEVICE); 1318 return -ENOMEM; 1319 } 1320 1321 desc = edesc->hw_desc; 1322 1323 edesc->src_nents = src_nents; 1324 edesc->sec4_sg_bytes = sec4_sg_bytes; 1325 1326 ret = buf_map_to_sec4_sg(jrdev, edesc->sec4_sg, state); 1327 if (ret) 1328 goto unmap; 1329 1330 ret = ahash_edesc_add_src(ctx, edesc, req, mapped_nents, 1, buflen, 1331 req->nbytes); 1332 if (ret) { 1333 dev_err(jrdev, "unable to map S/G table\n"); 1334 goto unmap; 1335 } 1336 1337 edesc->dst_dma = map_seq_out_ptr_result(desc, jrdev, req->result, 1338 digestsize); 1339 if (dma_mapping_error(jrdev, edesc->dst_dma)) { 1340 dev_err(jrdev, "unable to map dst\n"); 1341 goto unmap; 1342 } 1343 1344 #ifdef DEBUG 1345 print_hex_dump(KERN_ERR, "jobdesc@"__stringify(__LINE__)": ", 1346 DUMP_PREFIX_ADDRESS, 16, 4, desc, desc_bytes(desc), 1); 1347 #endif 1348 1349 ret = caam_jr_enqueue(jrdev, desc, ahash_done, req); 1350 if (!ret) { 1351 ret = -EINPROGRESS; 1352 } else { 1353 ahash_unmap(jrdev, edesc, req, digestsize); 1354 kfree(edesc); 1355 } 1356 1357 return ret; 1358 unmap: 1359 ahash_unmap(jrdev, edesc, req, digestsize); 1360 kfree(edesc); 1361 return -ENOMEM; 1362 1363 } 1364 1365 /* submit first update job descriptor after init */ 1366 static int ahash_update_first(struct ahash_request *req) 1367 { 1368 struct crypto_ahash *ahash = crypto_ahash_reqtfm(req); 1369 struct caam_hash_ctx *ctx = crypto_ahash_ctx(ahash); 1370 struct caam_hash_state *state = ahash_request_ctx(req); 1371 struct device *jrdev = ctx->jrdev; 1372 gfp_t flags = (req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP) ? 1373 GFP_KERNEL : GFP_ATOMIC; 1374 u8 *next_buf = alt_buf(state); 1375 int *next_buflen = alt_buflen(state); 1376 int to_hash; 1377 u32 *desc; 1378 int src_nents, mapped_nents; 1379 struct ahash_edesc *edesc; 1380 int ret = 0; 1381 1382 *next_buflen = req->nbytes & (crypto_tfm_alg_blocksize(&ahash->base) - 1383 1); 1384 to_hash = req->nbytes - *next_buflen; 1385 1386 if (to_hash) { 1387 src_nents = sg_nents_for_len(req->src, 1388 req->nbytes - *next_buflen); 1389 if (src_nents < 0) { 1390 dev_err(jrdev, "Invalid number of src SG.\n"); 1391 return src_nents; 1392 } 1393 1394 if (src_nents) { 1395 mapped_nents = dma_map_sg(jrdev, req->src, src_nents, 1396 DMA_TO_DEVICE); 1397 if (!mapped_nents) { 1398 dev_err(jrdev, "unable to map source for DMA\n"); 1399 return -ENOMEM; 1400 } 1401 } else { 1402 mapped_nents = 0; 1403 } 1404 1405 /* 1406 * allocate space for base edesc and hw desc commands, 1407 * link tables 1408 */ 1409 edesc = ahash_edesc_alloc(ctx, mapped_nents > 1 ? 1410 mapped_nents : 0, 1411 ctx->sh_desc_update_first, 1412 ctx->sh_desc_update_first_dma, 1413 flags); 1414 if (!edesc) { 1415 dma_unmap_sg(jrdev, req->src, src_nents, DMA_TO_DEVICE); 1416 return -ENOMEM; 1417 } 1418 1419 edesc->src_nents = src_nents; 1420 edesc->dst_dma = 0; 1421 1422 ret = ahash_edesc_add_src(ctx, edesc, req, mapped_nents, 0, 0, 1423 to_hash); 1424 if (ret) 1425 goto unmap_ctx; 1426 1427 if (*next_buflen) 1428 scatterwalk_map_and_copy(next_buf, req->src, to_hash, 1429 *next_buflen, 0); 1430 1431 desc = edesc->hw_desc; 1432 1433 ret = map_seq_out_ptr_ctx(desc, jrdev, state, ctx->ctx_len); 1434 if (ret) 1435 goto unmap_ctx; 1436 1437 #ifdef DEBUG 1438 print_hex_dump(KERN_ERR, "jobdesc@"__stringify(__LINE__)": ", 1439 DUMP_PREFIX_ADDRESS, 16, 4, desc, 1440 desc_bytes(desc), 1); 1441 #endif 1442 1443 ret = caam_jr_enqueue(jrdev, desc, ahash_done_ctx_dst, req); 1444 if (ret) 1445 goto unmap_ctx; 1446 1447 ret = -EINPROGRESS; 1448 state->update = ahash_update_ctx; 1449 state->finup = ahash_finup_ctx; 1450 state->final = ahash_final_ctx; 1451 } else if (*next_buflen) { 1452 state->update = ahash_update_no_ctx; 1453 state->finup = ahash_finup_no_ctx; 1454 state->final = ahash_final_no_ctx; 1455 scatterwalk_map_and_copy(next_buf, req->src, 0, 1456 req->nbytes, 0); 1457 switch_buf(state); 1458 } 1459 #ifdef DEBUG 1460 print_hex_dump(KERN_ERR, "next buf@"__stringify(__LINE__)": ", 1461 DUMP_PREFIX_ADDRESS, 16, 4, next_buf, 1462 *next_buflen, 1); 1463 #endif 1464 1465 return ret; 1466 unmap_ctx: 1467 ahash_unmap_ctx(jrdev, edesc, req, ctx->ctx_len, DMA_TO_DEVICE); 1468 kfree(edesc); 1469 return ret; 1470 } 1471 1472 static int ahash_finup_first(struct ahash_request *req) 1473 { 1474 return ahash_digest(req); 1475 } 1476 1477 static int ahash_init(struct ahash_request *req) 1478 { 1479 struct caam_hash_state *state = ahash_request_ctx(req); 1480 1481 state->update = ahash_update_first; 1482 state->finup = ahash_finup_first; 1483 state->final = ahash_final_no_ctx; 1484 1485 state->ctx_dma = 0; 1486 state->current_buf = 0; 1487 state->buf_dma = 0; 1488 state->buflen_0 = 0; 1489 state->buflen_1 = 0; 1490 1491 return 0; 1492 } 1493 1494 static int ahash_update(struct ahash_request *req) 1495 { 1496 struct caam_hash_state *state = ahash_request_ctx(req); 1497 1498 return state->update(req); 1499 } 1500 1501 static int ahash_finup(struct ahash_request *req) 1502 { 1503 struct caam_hash_state *state = ahash_request_ctx(req); 1504 1505 return state->finup(req); 1506 } 1507 1508 static int ahash_final(struct ahash_request *req) 1509 { 1510 struct caam_hash_state *state = ahash_request_ctx(req); 1511 1512 return state->final(req); 1513 } 1514 1515 static int ahash_export(struct ahash_request *req, void *out) 1516 { 1517 struct caam_hash_state *state = ahash_request_ctx(req); 1518 struct caam_export_state *export = out; 1519 int len; 1520 u8 *buf; 1521 1522 if (state->current_buf) { 1523 buf = state->buf_1; 1524 len = state->buflen_1; 1525 } else { 1526 buf = state->buf_0; 1527 len = state->buflen_0; 1528 } 1529 1530 memcpy(export->buf, buf, len); 1531 memcpy(export->caam_ctx, state->caam_ctx, sizeof(export->caam_ctx)); 1532 export->buflen = len; 1533 export->update = state->update; 1534 export->final = state->final; 1535 export->finup = state->finup; 1536 1537 return 0; 1538 } 1539 1540 static int ahash_import(struct ahash_request *req, const void *in) 1541 { 1542 struct caam_hash_state *state = ahash_request_ctx(req); 1543 const struct caam_export_state *export = in; 1544 1545 memset(state, 0, sizeof(*state)); 1546 memcpy(state->buf_0, export->buf, export->buflen); 1547 memcpy(state->caam_ctx, export->caam_ctx, sizeof(state->caam_ctx)); 1548 state->buflen_0 = export->buflen; 1549 state->update = export->update; 1550 state->final = export->final; 1551 state->finup = export->finup; 1552 1553 return 0; 1554 } 1555 1556 struct caam_hash_template { 1557 char name[CRYPTO_MAX_ALG_NAME]; 1558 char driver_name[CRYPTO_MAX_ALG_NAME]; 1559 char hmac_name[CRYPTO_MAX_ALG_NAME]; 1560 char hmac_driver_name[CRYPTO_MAX_ALG_NAME]; 1561 unsigned int blocksize; 1562 struct ahash_alg template_ahash; 1563 u32 alg_type; 1564 }; 1565 1566 /* ahash descriptors */ 1567 static struct caam_hash_template driver_hash[] = { 1568 { 1569 .name = "sha1", 1570 .driver_name = "sha1-caam", 1571 .hmac_name = "hmac(sha1)", 1572 .hmac_driver_name = "hmac-sha1-caam", 1573 .blocksize = SHA1_BLOCK_SIZE, 1574 .template_ahash = { 1575 .init = ahash_init, 1576 .update = ahash_update, 1577 .final = ahash_final, 1578 .finup = ahash_finup, 1579 .digest = ahash_digest, 1580 .export = ahash_export, 1581 .import = ahash_import, 1582 .setkey = ahash_setkey, 1583 .halg = { 1584 .digestsize = SHA1_DIGEST_SIZE, 1585 .statesize = sizeof(struct caam_export_state), 1586 }, 1587 }, 1588 .alg_type = OP_ALG_ALGSEL_SHA1, 1589 }, { 1590 .name = "sha224", 1591 .driver_name = "sha224-caam", 1592 .hmac_name = "hmac(sha224)", 1593 .hmac_driver_name = "hmac-sha224-caam", 1594 .blocksize = SHA224_BLOCK_SIZE, 1595 .template_ahash = { 1596 .init = ahash_init, 1597 .update = ahash_update, 1598 .final = ahash_final, 1599 .finup = ahash_finup, 1600 .digest = ahash_digest, 1601 .export = ahash_export, 1602 .import = ahash_import, 1603 .setkey = ahash_setkey, 1604 .halg = { 1605 .digestsize = SHA224_DIGEST_SIZE, 1606 .statesize = sizeof(struct caam_export_state), 1607 }, 1608 }, 1609 .alg_type = OP_ALG_ALGSEL_SHA224, 1610 }, { 1611 .name = "sha256", 1612 .driver_name = "sha256-caam", 1613 .hmac_name = "hmac(sha256)", 1614 .hmac_driver_name = "hmac-sha256-caam", 1615 .blocksize = SHA256_BLOCK_SIZE, 1616 .template_ahash = { 1617 .init = ahash_init, 1618 .update = ahash_update, 1619 .final = ahash_final, 1620 .finup = ahash_finup, 1621 .digest = ahash_digest, 1622 .export = ahash_export, 1623 .import = ahash_import, 1624 .setkey = ahash_setkey, 1625 .halg = { 1626 .digestsize = SHA256_DIGEST_SIZE, 1627 .statesize = sizeof(struct caam_export_state), 1628 }, 1629 }, 1630 .alg_type = OP_ALG_ALGSEL_SHA256, 1631 }, { 1632 .name = "sha384", 1633 .driver_name = "sha384-caam", 1634 .hmac_name = "hmac(sha384)", 1635 .hmac_driver_name = "hmac-sha384-caam", 1636 .blocksize = SHA384_BLOCK_SIZE, 1637 .template_ahash = { 1638 .init = ahash_init, 1639 .update = ahash_update, 1640 .final = ahash_final, 1641 .finup = ahash_finup, 1642 .digest = ahash_digest, 1643 .export = ahash_export, 1644 .import = ahash_import, 1645 .setkey = ahash_setkey, 1646 .halg = { 1647 .digestsize = SHA384_DIGEST_SIZE, 1648 .statesize = sizeof(struct caam_export_state), 1649 }, 1650 }, 1651 .alg_type = OP_ALG_ALGSEL_SHA384, 1652 }, { 1653 .name = "sha512", 1654 .driver_name = "sha512-caam", 1655 .hmac_name = "hmac(sha512)", 1656 .hmac_driver_name = "hmac-sha512-caam", 1657 .blocksize = SHA512_BLOCK_SIZE, 1658 .template_ahash = { 1659 .init = ahash_init, 1660 .update = ahash_update, 1661 .final = ahash_final, 1662 .finup = ahash_finup, 1663 .digest = ahash_digest, 1664 .export = ahash_export, 1665 .import = ahash_import, 1666 .setkey = ahash_setkey, 1667 .halg = { 1668 .digestsize = SHA512_DIGEST_SIZE, 1669 .statesize = sizeof(struct caam_export_state), 1670 }, 1671 }, 1672 .alg_type = OP_ALG_ALGSEL_SHA512, 1673 }, { 1674 .name = "md5", 1675 .driver_name = "md5-caam", 1676 .hmac_name = "hmac(md5)", 1677 .hmac_driver_name = "hmac-md5-caam", 1678 .blocksize = MD5_BLOCK_WORDS * 4, 1679 .template_ahash = { 1680 .init = ahash_init, 1681 .update = ahash_update, 1682 .final = ahash_final, 1683 .finup = ahash_finup, 1684 .digest = ahash_digest, 1685 .export = ahash_export, 1686 .import = ahash_import, 1687 .setkey = ahash_setkey, 1688 .halg = { 1689 .digestsize = MD5_DIGEST_SIZE, 1690 .statesize = sizeof(struct caam_export_state), 1691 }, 1692 }, 1693 .alg_type = OP_ALG_ALGSEL_MD5, 1694 }, 1695 }; 1696 1697 struct caam_hash_alg { 1698 struct list_head entry; 1699 int alg_type; 1700 struct ahash_alg ahash_alg; 1701 }; 1702 1703 static int caam_hash_cra_init(struct crypto_tfm *tfm) 1704 { 1705 struct crypto_ahash *ahash = __crypto_ahash_cast(tfm); 1706 struct crypto_alg *base = tfm->__crt_alg; 1707 struct hash_alg_common *halg = 1708 container_of(base, struct hash_alg_common, base); 1709 struct ahash_alg *alg = 1710 container_of(halg, struct ahash_alg, halg); 1711 struct caam_hash_alg *caam_hash = 1712 container_of(alg, struct caam_hash_alg, ahash_alg); 1713 struct caam_hash_ctx *ctx = crypto_tfm_ctx(tfm); 1714 /* Sizes for MDHA running digests: MD5, SHA1, 224, 256, 384, 512 */ 1715 static const u8 runninglen[] = { HASH_MSG_LEN + MD5_DIGEST_SIZE, 1716 HASH_MSG_LEN + SHA1_DIGEST_SIZE, 1717 HASH_MSG_LEN + 32, 1718 HASH_MSG_LEN + SHA256_DIGEST_SIZE, 1719 HASH_MSG_LEN + 64, 1720 HASH_MSG_LEN + SHA512_DIGEST_SIZE }; 1721 dma_addr_t dma_addr; 1722 1723 /* 1724 * Get a Job ring from Job Ring driver to ensure in-order 1725 * crypto request processing per tfm 1726 */ 1727 ctx->jrdev = caam_jr_alloc(); 1728 if (IS_ERR(ctx->jrdev)) { 1729 pr_err("Job Ring Device allocation for transform failed\n"); 1730 return PTR_ERR(ctx->jrdev); 1731 } 1732 1733 dma_addr = dma_map_single_attrs(ctx->jrdev, ctx->sh_desc_update, 1734 offsetof(struct caam_hash_ctx, 1735 sh_desc_update_dma), 1736 DMA_TO_DEVICE, DMA_ATTR_SKIP_CPU_SYNC); 1737 if (dma_mapping_error(ctx->jrdev, dma_addr)) { 1738 dev_err(ctx->jrdev, "unable to map shared descriptors\n"); 1739 caam_jr_free(ctx->jrdev); 1740 return -ENOMEM; 1741 } 1742 1743 ctx->sh_desc_update_dma = dma_addr; 1744 ctx->sh_desc_update_first_dma = dma_addr + 1745 offsetof(struct caam_hash_ctx, 1746 sh_desc_update_first); 1747 ctx->sh_desc_fin_dma = dma_addr + offsetof(struct caam_hash_ctx, 1748 sh_desc_fin); 1749 ctx->sh_desc_digest_dma = dma_addr + offsetof(struct caam_hash_ctx, 1750 sh_desc_digest); 1751 1752 /* copy descriptor header template value */ 1753 ctx->adata.algtype = OP_TYPE_CLASS2_ALG | caam_hash->alg_type; 1754 1755 ctx->ctx_len = runninglen[(ctx->adata.algtype & 1756 OP_ALG_ALGSEL_SUBMASK) >> 1757 OP_ALG_ALGSEL_SHIFT]; 1758 1759 crypto_ahash_set_reqsize(__crypto_ahash_cast(tfm), 1760 sizeof(struct caam_hash_state)); 1761 return ahash_set_sh_desc(ahash); 1762 } 1763 1764 static void caam_hash_cra_exit(struct crypto_tfm *tfm) 1765 { 1766 struct caam_hash_ctx *ctx = crypto_tfm_ctx(tfm); 1767 1768 dma_unmap_single_attrs(ctx->jrdev, ctx->sh_desc_update_dma, 1769 offsetof(struct caam_hash_ctx, 1770 sh_desc_update_dma), 1771 DMA_TO_DEVICE, DMA_ATTR_SKIP_CPU_SYNC); 1772 caam_jr_free(ctx->jrdev); 1773 } 1774 1775 static void __exit caam_algapi_hash_exit(void) 1776 { 1777 struct caam_hash_alg *t_alg, *n; 1778 1779 if (!hash_list.next) 1780 return; 1781 1782 list_for_each_entry_safe(t_alg, n, &hash_list, entry) { 1783 crypto_unregister_ahash(&t_alg->ahash_alg); 1784 list_del(&t_alg->entry); 1785 kfree(t_alg); 1786 } 1787 } 1788 1789 static struct caam_hash_alg * 1790 caam_hash_alloc(struct caam_hash_template *template, 1791 bool keyed) 1792 { 1793 struct caam_hash_alg *t_alg; 1794 struct ahash_alg *halg; 1795 struct crypto_alg *alg; 1796 1797 t_alg = kzalloc(sizeof(*t_alg), GFP_KERNEL); 1798 if (!t_alg) { 1799 pr_err("failed to allocate t_alg\n"); 1800 return ERR_PTR(-ENOMEM); 1801 } 1802 1803 t_alg->ahash_alg = template->template_ahash; 1804 halg = &t_alg->ahash_alg; 1805 alg = &halg->halg.base; 1806 1807 if (keyed) { 1808 snprintf(alg->cra_name, CRYPTO_MAX_ALG_NAME, "%s", 1809 template->hmac_name); 1810 snprintf(alg->cra_driver_name, CRYPTO_MAX_ALG_NAME, "%s", 1811 template->hmac_driver_name); 1812 } else { 1813 snprintf(alg->cra_name, CRYPTO_MAX_ALG_NAME, "%s", 1814 template->name); 1815 snprintf(alg->cra_driver_name, CRYPTO_MAX_ALG_NAME, "%s", 1816 template->driver_name); 1817 t_alg->ahash_alg.setkey = NULL; 1818 } 1819 alg->cra_module = THIS_MODULE; 1820 alg->cra_init = caam_hash_cra_init; 1821 alg->cra_exit = caam_hash_cra_exit; 1822 alg->cra_ctxsize = sizeof(struct caam_hash_ctx); 1823 alg->cra_priority = CAAM_CRA_PRIORITY; 1824 alg->cra_blocksize = template->blocksize; 1825 alg->cra_alignmask = 0; 1826 alg->cra_flags = CRYPTO_ALG_ASYNC | CRYPTO_ALG_TYPE_AHASH; 1827 alg->cra_type = &crypto_ahash_type; 1828 1829 t_alg->alg_type = template->alg_type; 1830 1831 return t_alg; 1832 } 1833 1834 static int __init caam_algapi_hash_init(void) 1835 { 1836 struct device_node *dev_node; 1837 struct platform_device *pdev; 1838 struct device *ctrldev; 1839 int i = 0, err = 0; 1840 struct caam_drv_private *priv; 1841 unsigned int md_limit = SHA512_DIGEST_SIZE; 1842 u32 cha_inst, cha_vid; 1843 1844 dev_node = of_find_compatible_node(NULL, NULL, "fsl,sec-v4.0"); 1845 if (!dev_node) { 1846 dev_node = of_find_compatible_node(NULL, NULL, "fsl,sec4.0"); 1847 if (!dev_node) 1848 return -ENODEV; 1849 } 1850 1851 pdev = of_find_device_by_node(dev_node); 1852 if (!pdev) { 1853 of_node_put(dev_node); 1854 return -ENODEV; 1855 } 1856 1857 ctrldev = &pdev->dev; 1858 priv = dev_get_drvdata(ctrldev); 1859 of_node_put(dev_node); 1860 1861 /* 1862 * If priv is NULL, it's probably because the caam driver wasn't 1863 * properly initialized (e.g. RNG4 init failed). Thus, bail out here. 1864 */ 1865 if (!priv) 1866 return -ENODEV; 1867 1868 /* 1869 * Register crypto algorithms the device supports. First, identify 1870 * presence and attributes of MD block. 1871 */ 1872 cha_vid = rd_reg32(&priv->ctrl->perfmon.cha_id_ls); 1873 cha_inst = rd_reg32(&priv->ctrl->perfmon.cha_num_ls); 1874 1875 /* 1876 * Skip registration of any hashing algorithms if MD block 1877 * is not present. 1878 */ 1879 if (!((cha_inst & CHA_ID_LS_MD_MASK) >> CHA_ID_LS_MD_SHIFT)) 1880 return -ENODEV; 1881 1882 /* Limit digest size based on LP256 */ 1883 if ((cha_vid & CHA_ID_LS_MD_MASK) == CHA_ID_LS_MD_LP256) 1884 md_limit = SHA256_DIGEST_SIZE; 1885 1886 INIT_LIST_HEAD(&hash_list); 1887 1888 /* register crypto algorithms the device supports */ 1889 for (i = 0; i < ARRAY_SIZE(driver_hash); i++) { 1890 struct caam_hash_alg *t_alg; 1891 struct caam_hash_template *alg = driver_hash + i; 1892 1893 /* If MD size is not supported by device, skip registration */ 1894 if (alg->template_ahash.halg.digestsize > md_limit) 1895 continue; 1896 1897 /* register hmac version */ 1898 t_alg = caam_hash_alloc(alg, true); 1899 if (IS_ERR(t_alg)) { 1900 err = PTR_ERR(t_alg); 1901 pr_warn("%s alg allocation failed\n", alg->driver_name); 1902 continue; 1903 } 1904 1905 err = crypto_register_ahash(&t_alg->ahash_alg); 1906 if (err) { 1907 pr_warn("%s alg registration failed: %d\n", 1908 t_alg->ahash_alg.halg.base.cra_driver_name, 1909 err); 1910 kfree(t_alg); 1911 } else 1912 list_add_tail(&t_alg->entry, &hash_list); 1913 1914 /* register unkeyed version */ 1915 t_alg = caam_hash_alloc(alg, false); 1916 if (IS_ERR(t_alg)) { 1917 err = PTR_ERR(t_alg); 1918 pr_warn("%s alg allocation failed\n", alg->driver_name); 1919 continue; 1920 } 1921 1922 err = crypto_register_ahash(&t_alg->ahash_alg); 1923 if (err) { 1924 pr_warn("%s alg registration failed: %d\n", 1925 t_alg->ahash_alg.halg.base.cra_driver_name, 1926 err); 1927 kfree(t_alg); 1928 } else 1929 list_add_tail(&t_alg->entry, &hash_list); 1930 } 1931 1932 return err; 1933 } 1934 1935 module_init(caam_algapi_hash_init); 1936 module_exit(caam_algapi_hash_exit); 1937 1938 MODULE_LICENSE("GPL"); 1939 MODULE_DESCRIPTION("FSL CAAM support for ahash functions of crypto API"); 1940 MODULE_AUTHOR("Freescale Semiconductor - NMG"); 1941