1 /* 2 * caam - Freescale FSL CAAM support for ahash functions of crypto API 3 * 4 * Copyright 2011 Freescale Semiconductor, Inc. 5 * 6 * Based on caamalg.c crypto API driver. 7 * 8 * relationship of digest job descriptor or first job descriptor after init to 9 * shared descriptors: 10 * 11 * --------------- --------------- 12 * | JobDesc #1 |-------------------->| ShareDesc | 13 * | *(packet 1) | | (hashKey) | 14 * --------------- | (operation) | 15 * --------------- 16 * 17 * relationship of subsequent job descriptors to shared descriptors: 18 * 19 * --------------- --------------- 20 * | JobDesc #2 |-------------------->| ShareDesc | 21 * | *(packet 2) | |------------->| (hashKey) | 22 * --------------- | |-------->| (operation) | 23 * . | | | (load ctx2) | 24 * . | | --------------- 25 * --------------- | | 26 * | JobDesc #3 |------| | 27 * | *(packet 3) | | 28 * --------------- | 29 * . | 30 * . | 31 * --------------- | 32 * | JobDesc #4 |------------ 33 * | *(packet 4) | 34 * --------------- 35 * 36 * The SharedDesc never changes for a connection unless rekeyed, but 37 * each packet will likely be in a different place. So all we need 38 * to know to process the packet is where the input is, where the 39 * output goes, and what context we want to process with. Context is 40 * in the SharedDesc, packet references in the JobDesc. 41 * 42 * So, a job desc looks like: 43 * 44 * --------------------- 45 * | Header | 46 * | ShareDesc Pointer | 47 * | SEQ_OUT_PTR | 48 * | (output buffer) | 49 * | (output length) | 50 * | SEQ_IN_PTR | 51 * | (input buffer) | 52 * | (input length) | 53 * --------------------- 54 */ 55 56 #include "compat.h" 57 58 #include "regs.h" 59 #include "intern.h" 60 #include "desc_constr.h" 61 #include "jr.h" 62 #include "error.h" 63 #include "sg_sw_sec4.h" 64 #include "key_gen.h" 65 66 #define CAAM_CRA_PRIORITY 3000 67 68 /* max hash key is max split key size */ 69 #define CAAM_MAX_HASH_KEY_SIZE (SHA512_DIGEST_SIZE * 2) 70 71 #define CAAM_MAX_HASH_BLOCK_SIZE SHA512_BLOCK_SIZE 72 #define CAAM_MAX_HASH_DIGEST_SIZE SHA512_DIGEST_SIZE 73 74 /* length of descriptors text */ 75 #define DESC_AHASH_BASE (4 * CAAM_CMD_SZ) 76 #define DESC_AHASH_UPDATE_LEN (6 * CAAM_CMD_SZ) 77 #define DESC_AHASH_UPDATE_FIRST_LEN (DESC_AHASH_BASE + 4 * CAAM_CMD_SZ) 78 #define DESC_AHASH_FINAL_LEN (DESC_AHASH_BASE + 5 * CAAM_CMD_SZ) 79 #define DESC_AHASH_FINUP_LEN (DESC_AHASH_BASE + 5 * CAAM_CMD_SZ) 80 #define DESC_AHASH_DIGEST_LEN (DESC_AHASH_BASE + 4 * CAAM_CMD_SZ) 81 82 #define DESC_HASH_MAX_USED_BYTES (DESC_AHASH_FINAL_LEN + \ 83 CAAM_MAX_HASH_KEY_SIZE) 84 #define DESC_HASH_MAX_USED_LEN (DESC_HASH_MAX_USED_BYTES / CAAM_CMD_SZ) 85 86 /* caam context sizes for hashes: running digest + 8 */ 87 #define HASH_MSG_LEN 8 88 #define MAX_CTX_LEN (HASH_MSG_LEN + SHA512_DIGEST_SIZE) 89 90 #ifdef DEBUG 91 /* for print_hex_dumps with line references */ 92 #define debug(format, arg...) printk(format, arg) 93 #else 94 #define debug(format, arg...) 95 #endif 96 97 98 static struct list_head hash_list; 99 100 /* ahash per-session context */ 101 struct caam_hash_ctx { 102 u32 sh_desc_update[DESC_HASH_MAX_USED_LEN] ____cacheline_aligned; 103 u32 sh_desc_update_first[DESC_HASH_MAX_USED_LEN] ____cacheline_aligned; 104 u32 sh_desc_fin[DESC_HASH_MAX_USED_LEN] ____cacheline_aligned; 105 u32 sh_desc_digest[DESC_HASH_MAX_USED_LEN] ____cacheline_aligned; 106 u32 sh_desc_finup[DESC_HASH_MAX_USED_LEN] ____cacheline_aligned; 107 dma_addr_t sh_desc_update_dma ____cacheline_aligned; 108 dma_addr_t sh_desc_update_first_dma; 109 dma_addr_t sh_desc_fin_dma; 110 dma_addr_t sh_desc_digest_dma; 111 dma_addr_t sh_desc_finup_dma; 112 struct device *jrdev; 113 u32 alg_type; 114 u32 alg_op; 115 u8 key[CAAM_MAX_HASH_KEY_SIZE]; 116 dma_addr_t key_dma; 117 int ctx_len; 118 unsigned int split_key_len; 119 unsigned int split_key_pad_len; 120 }; 121 122 /* ahash state */ 123 struct caam_hash_state { 124 dma_addr_t buf_dma; 125 dma_addr_t ctx_dma; 126 u8 buf_0[CAAM_MAX_HASH_BLOCK_SIZE] ____cacheline_aligned; 127 int buflen_0; 128 u8 buf_1[CAAM_MAX_HASH_BLOCK_SIZE] ____cacheline_aligned; 129 int buflen_1; 130 u8 caam_ctx[MAX_CTX_LEN] ____cacheline_aligned; 131 int (*update)(struct ahash_request *req); 132 int (*final)(struct ahash_request *req); 133 int (*finup)(struct ahash_request *req); 134 int current_buf; 135 }; 136 137 struct caam_export_state { 138 u8 buf[CAAM_MAX_HASH_BLOCK_SIZE]; 139 u8 caam_ctx[MAX_CTX_LEN]; 140 int buflen; 141 int (*update)(struct ahash_request *req); 142 int (*final)(struct ahash_request *req); 143 int (*finup)(struct ahash_request *req); 144 }; 145 146 /* Common job descriptor seq in/out ptr routines */ 147 148 /* Map state->caam_ctx, and append seq_out_ptr command that points to it */ 149 static inline int map_seq_out_ptr_ctx(u32 *desc, struct device *jrdev, 150 struct caam_hash_state *state, 151 int ctx_len) 152 { 153 state->ctx_dma = dma_map_single(jrdev, state->caam_ctx, 154 ctx_len, DMA_FROM_DEVICE); 155 if (dma_mapping_error(jrdev, state->ctx_dma)) { 156 dev_err(jrdev, "unable to map ctx\n"); 157 return -ENOMEM; 158 } 159 160 append_seq_out_ptr(desc, state->ctx_dma, ctx_len, 0); 161 162 return 0; 163 } 164 165 /* Map req->result, and append seq_out_ptr command that points to it */ 166 static inline dma_addr_t map_seq_out_ptr_result(u32 *desc, struct device *jrdev, 167 u8 *result, int digestsize) 168 { 169 dma_addr_t dst_dma; 170 171 dst_dma = dma_map_single(jrdev, result, digestsize, DMA_FROM_DEVICE); 172 append_seq_out_ptr(desc, dst_dma, digestsize, 0); 173 174 return dst_dma; 175 } 176 177 /* Map current buffer in state and put it in link table */ 178 static inline dma_addr_t buf_map_to_sec4_sg(struct device *jrdev, 179 struct sec4_sg_entry *sec4_sg, 180 u8 *buf, int buflen) 181 { 182 dma_addr_t buf_dma; 183 184 buf_dma = dma_map_single(jrdev, buf, buflen, DMA_TO_DEVICE); 185 dma_to_sec4_sg_one(sec4_sg, buf_dma, buflen, 0); 186 187 return buf_dma; 188 } 189 190 /* 191 * Only put buffer in link table if it contains data, which is possible, 192 * since a buffer has previously been used, and needs to be unmapped, 193 */ 194 static inline dma_addr_t 195 try_buf_map_to_sec4_sg(struct device *jrdev, struct sec4_sg_entry *sec4_sg, 196 u8 *buf, dma_addr_t buf_dma, int buflen, 197 int last_buflen) 198 { 199 if (buf_dma && !dma_mapping_error(jrdev, buf_dma)) 200 dma_unmap_single(jrdev, buf_dma, last_buflen, DMA_TO_DEVICE); 201 if (buflen) 202 buf_dma = buf_map_to_sec4_sg(jrdev, sec4_sg, buf, buflen); 203 else 204 buf_dma = 0; 205 206 return buf_dma; 207 } 208 209 /* Map state->caam_ctx, and add it to link table */ 210 static inline int ctx_map_to_sec4_sg(u32 *desc, struct device *jrdev, 211 struct caam_hash_state *state, int ctx_len, 212 struct sec4_sg_entry *sec4_sg, u32 flag) 213 { 214 state->ctx_dma = dma_map_single(jrdev, state->caam_ctx, ctx_len, flag); 215 if (dma_mapping_error(jrdev, state->ctx_dma)) { 216 dev_err(jrdev, "unable to map ctx\n"); 217 return -ENOMEM; 218 } 219 220 dma_to_sec4_sg_one(sec4_sg, state->ctx_dma, ctx_len, 0); 221 222 return 0; 223 } 224 225 /* Common shared descriptor commands */ 226 static inline void append_key_ahash(u32 *desc, struct caam_hash_ctx *ctx) 227 { 228 append_key_as_imm(desc, ctx->key, ctx->split_key_pad_len, 229 ctx->split_key_len, CLASS_2 | 230 KEY_DEST_MDHA_SPLIT | KEY_ENC); 231 } 232 233 /* Append key if it has been set */ 234 static inline void init_sh_desc_key_ahash(u32 *desc, struct caam_hash_ctx *ctx) 235 { 236 u32 *key_jump_cmd; 237 238 init_sh_desc(desc, HDR_SHARE_SERIAL); 239 240 if (ctx->split_key_len) { 241 /* Skip if already shared */ 242 key_jump_cmd = append_jump(desc, JUMP_JSL | JUMP_TEST_ALL | 243 JUMP_COND_SHRD); 244 245 append_key_ahash(desc, ctx); 246 247 set_jump_tgt_here(desc, key_jump_cmd); 248 } 249 250 /* Propagate errors from shared to job descriptor */ 251 append_cmd(desc, SET_OK_NO_PROP_ERRORS | CMD_LOAD); 252 } 253 254 /* 255 * For ahash read data from seqin following state->caam_ctx, 256 * and write resulting class2 context to seqout, which may be state->caam_ctx 257 * or req->result 258 */ 259 static inline void ahash_append_load_str(u32 *desc, int digestsize) 260 { 261 /* Calculate remaining bytes to read */ 262 append_math_add(desc, VARSEQINLEN, SEQINLEN, REG0, CAAM_CMD_SZ); 263 264 /* Read remaining bytes */ 265 append_seq_fifo_load(desc, 0, FIFOLD_CLASS_CLASS2 | FIFOLD_TYPE_LAST2 | 266 FIFOLD_TYPE_MSG | KEY_VLF); 267 268 /* Store class2 context bytes */ 269 append_seq_store(desc, digestsize, LDST_CLASS_2_CCB | 270 LDST_SRCDST_BYTE_CONTEXT); 271 } 272 273 /* 274 * For ahash update, final and finup, import context, read and write to seqout 275 */ 276 static inline void ahash_ctx_data_to_out(u32 *desc, u32 op, u32 state, 277 int digestsize, 278 struct caam_hash_ctx *ctx) 279 { 280 init_sh_desc_key_ahash(desc, ctx); 281 282 /* Import context from software */ 283 append_cmd(desc, CMD_SEQ_LOAD | LDST_SRCDST_BYTE_CONTEXT | 284 LDST_CLASS_2_CCB | ctx->ctx_len); 285 286 /* Class 2 operation */ 287 append_operation(desc, op | state | OP_ALG_ENCRYPT); 288 289 /* 290 * Load from buf and/or src and write to req->result or state->context 291 */ 292 ahash_append_load_str(desc, digestsize); 293 } 294 295 /* For ahash firsts and digest, read and write to seqout */ 296 static inline void ahash_data_to_out(u32 *desc, u32 op, u32 state, 297 int digestsize, struct caam_hash_ctx *ctx) 298 { 299 init_sh_desc_key_ahash(desc, ctx); 300 301 /* Class 2 operation */ 302 append_operation(desc, op | state | OP_ALG_ENCRYPT); 303 304 /* 305 * Load from buf and/or src and write to req->result or state->context 306 */ 307 ahash_append_load_str(desc, digestsize); 308 } 309 310 static int ahash_set_sh_desc(struct crypto_ahash *ahash) 311 { 312 struct caam_hash_ctx *ctx = crypto_ahash_ctx(ahash); 313 int digestsize = crypto_ahash_digestsize(ahash); 314 struct device *jrdev = ctx->jrdev; 315 u32 have_key = 0; 316 u32 *desc; 317 318 if (ctx->split_key_len) 319 have_key = OP_ALG_AAI_HMAC_PRECOMP; 320 321 /* ahash_update shared descriptor */ 322 desc = ctx->sh_desc_update; 323 324 init_sh_desc(desc, HDR_SHARE_SERIAL); 325 326 /* Import context from software */ 327 append_cmd(desc, CMD_SEQ_LOAD | LDST_SRCDST_BYTE_CONTEXT | 328 LDST_CLASS_2_CCB | ctx->ctx_len); 329 330 /* Class 2 operation */ 331 append_operation(desc, ctx->alg_type | OP_ALG_AS_UPDATE | 332 OP_ALG_ENCRYPT); 333 334 /* Load data and write to result or context */ 335 ahash_append_load_str(desc, ctx->ctx_len); 336 337 ctx->sh_desc_update_dma = dma_map_single(jrdev, desc, desc_bytes(desc), 338 DMA_TO_DEVICE); 339 if (dma_mapping_error(jrdev, ctx->sh_desc_update_dma)) { 340 dev_err(jrdev, "unable to map shared descriptor\n"); 341 return -ENOMEM; 342 } 343 #ifdef DEBUG 344 print_hex_dump(KERN_ERR, 345 "ahash update shdesc@"__stringify(__LINE__)": ", 346 DUMP_PREFIX_ADDRESS, 16, 4, desc, desc_bytes(desc), 1); 347 #endif 348 349 /* ahash_update_first shared descriptor */ 350 desc = ctx->sh_desc_update_first; 351 352 ahash_data_to_out(desc, have_key | ctx->alg_type, OP_ALG_AS_INIT, 353 ctx->ctx_len, ctx); 354 355 ctx->sh_desc_update_first_dma = dma_map_single(jrdev, desc, 356 desc_bytes(desc), 357 DMA_TO_DEVICE); 358 if (dma_mapping_error(jrdev, ctx->sh_desc_update_first_dma)) { 359 dev_err(jrdev, "unable to map shared descriptor\n"); 360 return -ENOMEM; 361 } 362 #ifdef DEBUG 363 print_hex_dump(KERN_ERR, 364 "ahash update first shdesc@"__stringify(__LINE__)": ", 365 DUMP_PREFIX_ADDRESS, 16, 4, desc, desc_bytes(desc), 1); 366 #endif 367 368 /* ahash_final shared descriptor */ 369 desc = ctx->sh_desc_fin; 370 371 ahash_ctx_data_to_out(desc, have_key | ctx->alg_type, 372 OP_ALG_AS_FINALIZE, digestsize, ctx); 373 374 ctx->sh_desc_fin_dma = dma_map_single(jrdev, desc, desc_bytes(desc), 375 DMA_TO_DEVICE); 376 if (dma_mapping_error(jrdev, ctx->sh_desc_fin_dma)) { 377 dev_err(jrdev, "unable to map shared descriptor\n"); 378 return -ENOMEM; 379 } 380 #ifdef DEBUG 381 print_hex_dump(KERN_ERR, "ahash final shdesc@"__stringify(__LINE__)": ", 382 DUMP_PREFIX_ADDRESS, 16, 4, desc, 383 desc_bytes(desc), 1); 384 #endif 385 386 /* ahash_finup shared descriptor */ 387 desc = ctx->sh_desc_finup; 388 389 ahash_ctx_data_to_out(desc, have_key | ctx->alg_type, 390 OP_ALG_AS_FINALIZE, digestsize, ctx); 391 392 ctx->sh_desc_finup_dma = dma_map_single(jrdev, desc, desc_bytes(desc), 393 DMA_TO_DEVICE); 394 if (dma_mapping_error(jrdev, ctx->sh_desc_finup_dma)) { 395 dev_err(jrdev, "unable to map shared descriptor\n"); 396 return -ENOMEM; 397 } 398 #ifdef DEBUG 399 print_hex_dump(KERN_ERR, "ahash finup shdesc@"__stringify(__LINE__)": ", 400 DUMP_PREFIX_ADDRESS, 16, 4, desc, 401 desc_bytes(desc), 1); 402 #endif 403 404 /* ahash_digest shared descriptor */ 405 desc = ctx->sh_desc_digest; 406 407 ahash_data_to_out(desc, have_key | ctx->alg_type, OP_ALG_AS_INITFINAL, 408 digestsize, ctx); 409 410 ctx->sh_desc_digest_dma = dma_map_single(jrdev, desc, 411 desc_bytes(desc), 412 DMA_TO_DEVICE); 413 if (dma_mapping_error(jrdev, ctx->sh_desc_digest_dma)) { 414 dev_err(jrdev, "unable to map shared descriptor\n"); 415 return -ENOMEM; 416 } 417 #ifdef DEBUG 418 print_hex_dump(KERN_ERR, 419 "ahash digest shdesc@"__stringify(__LINE__)": ", 420 DUMP_PREFIX_ADDRESS, 16, 4, desc, 421 desc_bytes(desc), 1); 422 #endif 423 424 return 0; 425 } 426 427 static int gen_split_hash_key(struct caam_hash_ctx *ctx, const u8 *key_in, 428 u32 keylen) 429 { 430 return gen_split_key(ctx->jrdev, ctx->key, ctx->split_key_len, 431 ctx->split_key_pad_len, key_in, keylen, 432 ctx->alg_op); 433 } 434 435 /* Digest hash size if it is too large */ 436 static int hash_digest_key(struct caam_hash_ctx *ctx, const u8 *key_in, 437 u32 *keylen, u8 *key_out, u32 digestsize) 438 { 439 struct device *jrdev = ctx->jrdev; 440 u32 *desc; 441 struct split_key_result result; 442 dma_addr_t src_dma, dst_dma; 443 int ret = 0; 444 445 desc = kmalloc(CAAM_CMD_SZ * 8 + CAAM_PTR_SZ * 2, GFP_KERNEL | GFP_DMA); 446 if (!desc) { 447 dev_err(jrdev, "unable to allocate key input memory\n"); 448 return -ENOMEM; 449 } 450 451 init_job_desc(desc, 0); 452 453 src_dma = dma_map_single(jrdev, (void *)key_in, *keylen, 454 DMA_TO_DEVICE); 455 if (dma_mapping_error(jrdev, src_dma)) { 456 dev_err(jrdev, "unable to map key input memory\n"); 457 kfree(desc); 458 return -ENOMEM; 459 } 460 dst_dma = dma_map_single(jrdev, (void *)key_out, digestsize, 461 DMA_FROM_DEVICE); 462 if (dma_mapping_error(jrdev, dst_dma)) { 463 dev_err(jrdev, "unable to map key output memory\n"); 464 dma_unmap_single(jrdev, src_dma, *keylen, DMA_TO_DEVICE); 465 kfree(desc); 466 return -ENOMEM; 467 } 468 469 /* Job descriptor to perform unkeyed hash on key_in */ 470 append_operation(desc, ctx->alg_type | OP_ALG_ENCRYPT | 471 OP_ALG_AS_INITFINAL); 472 append_seq_in_ptr(desc, src_dma, *keylen, 0); 473 append_seq_fifo_load(desc, *keylen, FIFOLD_CLASS_CLASS2 | 474 FIFOLD_TYPE_LAST2 | FIFOLD_TYPE_MSG); 475 append_seq_out_ptr(desc, dst_dma, digestsize, 0); 476 append_seq_store(desc, digestsize, LDST_CLASS_2_CCB | 477 LDST_SRCDST_BYTE_CONTEXT); 478 479 #ifdef DEBUG 480 print_hex_dump(KERN_ERR, "key_in@"__stringify(__LINE__)": ", 481 DUMP_PREFIX_ADDRESS, 16, 4, key_in, *keylen, 1); 482 print_hex_dump(KERN_ERR, "jobdesc@"__stringify(__LINE__)": ", 483 DUMP_PREFIX_ADDRESS, 16, 4, desc, desc_bytes(desc), 1); 484 #endif 485 486 result.err = 0; 487 init_completion(&result.completion); 488 489 ret = caam_jr_enqueue(jrdev, desc, split_key_done, &result); 490 if (!ret) { 491 /* in progress */ 492 wait_for_completion_interruptible(&result.completion); 493 ret = result.err; 494 #ifdef DEBUG 495 print_hex_dump(KERN_ERR, 496 "digested key@"__stringify(__LINE__)": ", 497 DUMP_PREFIX_ADDRESS, 16, 4, key_in, 498 digestsize, 1); 499 #endif 500 } 501 dma_unmap_single(jrdev, src_dma, *keylen, DMA_TO_DEVICE); 502 dma_unmap_single(jrdev, dst_dma, digestsize, DMA_FROM_DEVICE); 503 504 *keylen = digestsize; 505 506 kfree(desc); 507 508 return ret; 509 } 510 511 static int ahash_setkey(struct crypto_ahash *ahash, 512 const u8 *key, unsigned int keylen) 513 { 514 /* Sizes for MDHA pads (*not* keys): MD5, SHA1, 224, 256, 384, 512 */ 515 static const u8 mdpadlen[] = { 16, 20, 32, 32, 64, 64 }; 516 struct caam_hash_ctx *ctx = crypto_ahash_ctx(ahash); 517 struct device *jrdev = ctx->jrdev; 518 int blocksize = crypto_tfm_alg_blocksize(&ahash->base); 519 int digestsize = crypto_ahash_digestsize(ahash); 520 int ret = 0; 521 u8 *hashed_key = NULL; 522 523 #ifdef DEBUG 524 printk(KERN_ERR "keylen %d\n", keylen); 525 #endif 526 527 if (keylen > blocksize) { 528 hashed_key = kmalloc_array(digestsize, 529 sizeof(*hashed_key), 530 GFP_KERNEL | GFP_DMA); 531 if (!hashed_key) 532 return -ENOMEM; 533 ret = hash_digest_key(ctx, key, &keylen, hashed_key, 534 digestsize); 535 if (ret) 536 goto bad_free_key; 537 key = hashed_key; 538 } 539 540 /* Pick class 2 key length from algorithm submask */ 541 ctx->split_key_len = mdpadlen[(ctx->alg_op & OP_ALG_ALGSEL_SUBMASK) >> 542 OP_ALG_ALGSEL_SHIFT] * 2; 543 ctx->split_key_pad_len = ALIGN(ctx->split_key_len, 16); 544 545 #ifdef DEBUG 546 printk(KERN_ERR "split_key_len %d split_key_pad_len %d\n", 547 ctx->split_key_len, ctx->split_key_pad_len); 548 print_hex_dump(KERN_ERR, "key in @"__stringify(__LINE__)": ", 549 DUMP_PREFIX_ADDRESS, 16, 4, key, keylen, 1); 550 #endif 551 552 ret = gen_split_hash_key(ctx, key, keylen); 553 if (ret) 554 goto bad_free_key; 555 556 ctx->key_dma = dma_map_single(jrdev, ctx->key, ctx->split_key_pad_len, 557 DMA_TO_DEVICE); 558 if (dma_mapping_error(jrdev, ctx->key_dma)) { 559 dev_err(jrdev, "unable to map key i/o memory\n"); 560 ret = -ENOMEM; 561 goto error_free_key; 562 } 563 #ifdef DEBUG 564 print_hex_dump(KERN_ERR, "ctx.key@"__stringify(__LINE__)": ", 565 DUMP_PREFIX_ADDRESS, 16, 4, ctx->key, 566 ctx->split_key_pad_len, 1); 567 #endif 568 569 ret = ahash_set_sh_desc(ahash); 570 if (ret) { 571 dma_unmap_single(jrdev, ctx->key_dma, ctx->split_key_pad_len, 572 DMA_TO_DEVICE); 573 } 574 error_free_key: 575 kfree(hashed_key); 576 return ret; 577 bad_free_key: 578 kfree(hashed_key); 579 crypto_ahash_set_flags(ahash, CRYPTO_TFM_RES_BAD_KEY_LEN); 580 return -EINVAL; 581 } 582 583 /* 584 * ahash_edesc - s/w-extended ahash descriptor 585 * @dst_dma: physical mapped address of req->result 586 * @sec4_sg_dma: physical mapped address of h/w link table 587 * @src_nents: number of segments in input scatterlist 588 * @sec4_sg_bytes: length of dma mapped sec4_sg space 589 * @hw_desc: the h/w job descriptor followed by any referenced link tables 590 * @sec4_sg: h/w link table 591 */ 592 struct ahash_edesc { 593 dma_addr_t dst_dma; 594 dma_addr_t sec4_sg_dma; 595 int src_nents; 596 int sec4_sg_bytes; 597 u32 hw_desc[DESC_JOB_IO_LEN / sizeof(u32)] ____cacheline_aligned; 598 struct sec4_sg_entry sec4_sg[0]; 599 }; 600 601 static inline void ahash_unmap(struct device *dev, 602 struct ahash_edesc *edesc, 603 struct ahash_request *req, int dst_len) 604 { 605 if (edesc->src_nents) 606 dma_unmap_sg(dev, req->src, edesc->src_nents, DMA_TO_DEVICE); 607 if (edesc->dst_dma) 608 dma_unmap_single(dev, edesc->dst_dma, dst_len, DMA_FROM_DEVICE); 609 610 if (edesc->sec4_sg_bytes) 611 dma_unmap_single(dev, edesc->sec4_sg_dma, 612 edesc->sec4_sg_bytes, DMA_TO_DEVICE); 613 } 614 615 static inline void ahash_unmap_ctx(struct device *dev, 616 struct ahash_edesc *edesc, 617 struct ahash_request *req, int dst_len, u32 flag) 618 { 619 struct crypto_ahash *ahash = crypto_ahash_reqtfm(req); 620 struct caam_hash_ctx *ctx = crypto_ahash_ctx(ahash); 621 struct caam_hash_state *state = ahash_request_ctx(req); 622 623 if (state->ctx_dma) 624 dma_unmap_single(dev, state->ctx_dma, ctx->ctx_len, flag); 625 ahash_unmap(dev, edesc, req, dst_len); 626 } 627 628 static void ahash_done(struct device *jrdev, u32 *desc, u32 err, 629 void *context) 630 { 631 struct ahash_request *req = context; 632 struct ahash_edesc *edesc; 633 struct crypto_ahash *ahash = crypto_ahash_reqtfm(req); 634 int digestsize = crypto_ahash_digestsize(ahash); 635 #ifdef DEBUG 636 struct caam_hash_ctx *ctx = crypto_ahash_ctx(ahash); 637 struct caam_hash_state *state = ahash_request_ctx(req); 638 639 dev_err(jrdev, "%s %d: err 0x%x\n", __func__, __LINE__, err); 640 #endif 641 642 edesc = (struct ahash_edesc *)((char *)desc - 643 offsetof(struct ahash_edesc, hw_desc)); 644 if (err) 645 caam_jr_strstatus(jrdev, err); 646 647 ahash_unmap(jrdev, edesc, req, digestsize); 648 kfree(edesc); 649 650 #ifdef DEBUG 651 print_hex_dump(KERN_ERR, "ctx@"__stringify(__LINE__)": ", 652 DUMP_PREFIX_ADDRESS, 16, 4, state->caam_ctx, 653 ctx->ctx_len, 1); 654 if (req->result) 655 print_hex_dump(KERN_ERR, "result@"__stringify(__LINE__)": ", 656 DUMP_PREFIX_ADDRESS, 16, 4, req->result, 657 digestsize, 1); 658 #endif 659 660 req->base.complete(&req->base, err); 661 } 662 663 static void ahash_done_bi(struct device *jrdev, u32 *desc, u32 err, 664 void *context) 665 { 666 struct ahash_request *req = context; 667 struct ahash_edesc *edesc; 668 struct crypto_ahash *ahash = crypto_ahash_reqtfm(req); 669 struct caam_hash_ctx *ctx = crypto_ahash_ctx(ahash); 670 #ifdef DEBUG 671 struct caam_hash_state *state = ahash_request_ctx(req); 672 int digestsize = crypto_ahash_digestsize(ahash); 673 674 dev_err(jrdev, "%s %d: err 0x%x\n", __func__, __LINE__, err); 675 #endif 676 677 edesc = (struct ahash_edesc *)((char *)desc - 678 offsetof(struct ahash_edesc, hw_desc)); 679 if (err) 680 caam_jr_strstatus(jrdev, err); 681 682 ahash_unmap_ctx(jrdev, edesc, req, ctx->ctx_len, DMA_BIDIRECTIONAL); 683 kfree(edesc); 684 685 #ifdef DEBUG 686 print_hex_dump(KERN_ERR, "ctx@"__stringify(__LINE__)": ", 687 DUMP_PREFIX_ADDRESS, 16, 4, state->caam_ctx, 688 ctx->ctx_len, 1); 689 if (req->result) 690 print_hex_dump(KERN_ERR, "result@"__stringify(__LINE__)": ", 691 DUMP_PREFIX_ADDRESS, 16, 4, req->result, 692 digestsize, 1); 693 #endif 694 695 req->base.complete(&req->base, err); 696 } 697 698 static void ahash_done_ctx_src(struct device *jrdev, u32 *desc, u32 err, 699 void *context) 700 { 701 struct ahash_request *req = context; 702 struct ahash_edesc *edesc; 703 struct crypto_ahash *ahash = crypto_ahash_reqtfm(req); 704 int digestsize = crypto_ahash_digestsize(ahash); 705 #ifdef DEBUG 706 struct caam_hash_ctx *ctx = crypto_ahash_ctx(ahash); 707 struct caam_hash_state *state = ahash_request_ctx(req); 708 709 dev_err(jrdev, "%s %d: err 0x%x\n", __func__, __LINE__, err); 710 #endif 711 712 edesc = (struct ahash_edesc *)((char *)desc - 713 offsetof(struct ahash_edesc, hw_desc)); 714 if (err) 715 caam_jr_strstatus(jrdev, err); 716 717 ahash_unmap_ctx(jrdev, edesc, req, digestsize, DMA_TO_DEVICE); 718 kfree(edesc); 719 720 #ifdef DEBUG 721 print_hex_dump(KERN_ERR, "ctx@"__stringify(__LINE__)": ", 722 DUMP_PREFIX_ADDRESS, 16, 4, state->caam_ctx, 723 ctx->ctx_len, 1); 724 if (req->result) 725 print_hex_dump(KERN_ERR, "result@"__stringify(__LINE__)": ", 726 DUMP_PREFIX_ADDRESS, 16, 4, req->result, 727 digestsize, 1); 728 #endif 729 730 req->base.complete(&req->base, err); 731 } 732 733 static void ahash_done_ctx_dst(struct device *jrdev, u32 *desc, u32 err, 734 void *context) 735 { 736 struct ahash_request *req = context; 737 struct ahash_edesc *edesc; 738 struct crypto_ahash *ahash = crypto_ahash_reqtfm(req); 739 struct caam_hash_ctx *ctx = crypto_ahash_ctx(ahash); 740 #ifdef DEBUG 741 struct caam_hash_state *state = ahash_request_ctx(req); 742 int digestsize = crypto_ahash_digestsize(ahash); 743 744 dev_err(jrdev, "%s %d: err 0x%x\n", __func__, __LINE__, err); 745 #endif 746 747 edesc = (struct ahash_edesc *)((char *)desc - 748 offsetof(struct ahash_edesc, hw_desc)); 749 if (err) 750 caam_jr_strstatus(jrdev, err); 751 752 ahash_unmap_ctx(jrdev, edesc, req, ctx->ctx_len, DMA_FROM_DEVICE); 753 kfree(edesc); 754 755 #ifdef DEBUG 756 print_hex_dump(KERN_ERR, "ctx@"__stringify(__LINE__)": ", 757 DUMP_PREFIX_ADDRESS, 16, 4, state->caam_ctx, 758 ctx->ctx_len, 1); 759 if (req->result) 760 print_hex_dump(KERN_ERR, "result@"__stringify(__LINE__)": ", 761 DUMP_PREFIX_ADDRESS, 16, 4, req->result, 762 digestsize, 1); 763 #endif 764 765 req->base.complete(&req->base, err); 766 } 767 768 /* 769 * Allocate an enhanced descriptor, which contains the hardware descriptor 770 * and space for hardware scatter table containing sg_num entries. 771 */ 772 static struct ahash_edesc *ahash_edesc_alloc(struct caam_hash_ctx *ctx, 773 int sg_num, u32 *sh_desc, 774 dma_addr_t sh_desc_dma, 775 gfp_t flags) 776 { 777 struct ahash_edesc *edesc; 778 unsigned int sg_size = sg_num * sizeof(struct sec4_sg_entry); 779 780 edesc = kzalloc(sizeof(*edesc) + sg_size, GFP_DMA | flags); 781 if (!edesc) { 782 dev_err(ctx->jrdev, "could not allocate extended descriptor\n"); 783 return NULL; 784 } 785 786 init_job_desc_shared(edesc->hw_desc, sh_desc_dma, desc_len(sh_desc), 787 HDR_SHARE_DEFER | HDR_REVERSE); 788 789 return edesc; 790 } 791 792 static int ahash_edesc_add_src(struct caam_hash_ctx *ctx, 793 struct ahash_edesc *edesc, 794 struct ahash_request *req, int nents, 795 unsigned int first_sg, 796 unsigned int first_bytes, size_t to_hash) 797 { 798 dma_addr_t src_dma; 799 u32 options; 800 801 if (nents > 1 || first_sg) { 802 struct sec4_sg_entry *sg = edesc->sec4_sg; 803 unsigned int sgsize = sizeof(*sg) * (first_sg + nents); 804 805 sg_to_sec4_sg_last(req->src, nents, sg + first_sg, 0); 806 807 src_dma = dma_map_single(ctx->jrdev, sg, sgsize, DMA_TO_DEVICE); 808 if (dma_mapping_error(ctx->jrdev, src_dma)) { 809 dev_err(ctx->jrdev, "unable to map S/G table\n"); 810 return -ENOMEM; 811 } 812 813 edesc->sec4_sg_bytes = sgsize; 814 edesc->sec4_sg_dma = src_dma; 815 options = LDST_SGF; 816 } else { 817 src_dma = sg_dma_address(req->src); 818 options = 0; 819 } 820 821 append_seq_in_ptr(edesc->hw_desc, src_dma, first_bytes + to_hash, 822 options); 823 824 return 0; 825 } 826 827 /* submit update job descriptor */ 828 static int ahash_update_ctx(struct ahash_request *req) 829 { 830 struct crypto_ahash *ahash = crypto_ahash_reqtfm(req); 831 struct caam_hash_ctx *ctx = crypto_ahash_ctx(ahash); 832 struct caam_hash_state *state = ahash_request_ctx(req); 833 struct device *jrdev = ctx->jrdev; 834 gfp_t flags = (req->base.flags & (CRYPTO_TFM_REQ_MAY_BACKLOG | 835 CRYPTO_TFM_REQ_MAY_SLEEP)) ? GFP_KERNEL : GFP_ATOMIC; 836 u8 *buf = state->current_buf ? state->buf_1 : state->buf_0; 837 int *buflen = state->current_buf ? &state->buflen_1 : &state->buflen_0; 838 u8 *next_buf = state->current_buf ? state->buf_0 : state->buf_1; 839 int *next_buflen = state->current_buf ? &state->buflen_0 : 840 &state->buflen_1, last_buflen; 841 int in_len = *buflen + req->nbytes, to_hash; 842 u32 *desc; 843 int src_nents, mapped_nents, sec4_sg_bytes, sec4_sg_src_index; 844 struct ahash_edesc *edesc; 845 int ret = 0; 846 847 last_buflen = *next_buflen; 848 *next_buflen = in_len & (crypto_tfm_alg_blocksize(&ahash->base) - 1); 849 to_hash = in_len - *next_buflen; 850 851 if (to_hash) { 852 src_nents = sg_nents_for_len(req->src, 853 req->nbytes - (*next_buflen)); 854 if (src_nents < 0) { 855 dev_err(jrdev, "Invalid number of src SG.\n"); 856 return src_nents; 857 } 858 859 if (src_nents) { 860 mapped_nents = dma_map_sg(jrdev, req->src, src_nents, 861 DMA_TO_DEVICE); 862 if (!mapped_nents) { 863 dev_err(jrdev, "unable to DMA map source\n"); 864 return -ENOMEM; 865 } 866 } else { 867 mapped_nents = 0; 868 } 869 870 sec4_sg_src_index = 1 + (*buflen ? 1 : 0); 871 sec4_sg_bytes = (sec4_sg_src_index + mapped_nents) * 872 sizeof(struct sec4_sg_entry); 873 874 /* 875 * allocate space for base edesc and hw desc commands, 876 * link tables 877 */ 878 edesc = ahash_edesc_alloc(ctx, sec4_sg_src_index + mapped_nents, 879 ctx->sh_desc_update, 880 ctx->sh_desc_update_dma, flags); 881 if (!edesc) { 882 dma_unmap_sg(jrdev, req->src, src_nents, DMA_TO_DEVICE); 883 return -ENOMEM; 884 } 885 886 edesc->src_nents = src_nents; 887 edesc->sec4_sg_bytes = sec4_sg_bytes; 888 889 ret = ctx_map_to_sec4_sg(desc, jrdev, state, ctx->ctx_len, 890 edesc->sec4_sg, DMA_BIDIRECTIONAL); 891 if (ret) 892 goto unmap_ctx; 893 894 state->buf_dma = try_buf_map_to_sec4_sg(jrdev, 895 edesc->sec4_sg + 1, 896 buf, state->buf_dma, 897 *buflen, last_buflen); 898 899 if (mapped_nents) { 900 sg_to_sec4_sg_last(req->src, mapped_nents, 901 edesc->sec4_sg + sec4_sg_src_index, 902 0); 903 if (*next_buflen) 904 scatterwalk_map_and_copy(next_buf, req->src, 905 to_hash - *buflen, 906 *next_buflen, 0); 907 } else { 908 (edesc->sec4_sg + sec4_sg_src_index - 1)->len |= 909 cpu_to_caam32(SEC4_SG_LEN_FIN); 910 } 911 912 state->current_buf = !state->current_buf; 913 914 desc = edesc->hw_desc; 915 916 edesc->sec4_sg_dma = dma_map_single(jrdev, edesc->sec4_sg, 917 sec4_sg_bytes, 918 DMA_TO_DEVICE); 919 if (dma_mapping_error(jrdev, edesc->sec4_sg_dma)) { 920 dev_err(jrdev, "unable to map S/G table\n"); 921 ret = -ENOMEM; 922 goto unmap_ctx; 923 } 924 925 append_seq_in_ptr(desc, edesc->sec4_sg_dma, ctx->ctx_len + 926 to_hash, LDST_SGF); 927 928 append_seq_out_ptr(desc, state->ctx_dma, ctx->ctx_len, 0); 929 930 #ifdef DEBUG 931 print_hex_dump(KERN_ERR, "jobdesc@"__stringify(__LINE__)": ", 932 DUMP_PREFIX_ADDRESS, 16, 4, desc, 933 desc_bytes(desc), 1); 934 #endif 935 936 ret = caam_jr_enqueue(jrdev, desc, ahash_done_bi, req); 937 if (ret) 938 goto unmap_ctx; 939 940 ret = -EINPROGRESS; 941 } else if (*next_buflen) { 942 scatterwalk_map_and_copy(buf + *buflen, req->src, 0, 943 req->nbytes, 0); 944 *buflen = *next_buflen; 945 *next_buflen = last_buflen; 946 } 947 #ifdef DEBUG 948 print_hex_dump(KERN_ERR, "buf@"__stringify(__LINE__)": ", 949 DUMP_PREFIX_ADDRESS, 16, 4, buf, *buflen, 1); 950 print_hex_dump(KERN_ERR, "next buf@"__stringify(__LINE__)": ", 951 DUMP_PREFIX_ADDRESS, 16, 4, next_buf, 952 *next_buflen, 1); 953 #endif 954 955 return ret; 956 unmap_ctx: 957 ahash_unmap_ctx(jrdev, edesc, req, ctx->ctx_len, DMA_BIDIRECTIONAL); 958 kfree(edesc); 959 return ret; 960 } 961 962 static int ahash_final_ctx(struct ahash_request *req) 963 { 964 struct crypto_ahash *ahash = crypto_ahash_reqtfm(req); 965 struct caam_hash_ctx *ctx = crypto_ahash_ctx(ahash); 966 struct caam_hash_state *state = ahash_request_ctx(req); 967 struct device *jrdev = ctx->jrdev; 968 gfp_t flags = (req->base.flags & (CRYPTO_TFM_REQ_MAY_BACKLOG | 969 CRYPTO_TFM_REQ_MAY_SLEEP)) ? GFP_KERNEL : GFP_ATOMIC; 970 u8 *buf = state->current_buf ? state->buf_1 : state->buf_0; 971 int buflen = state->current_buf ? state->buflen_1 : state->buflen_0; 972 int last_buflen = state->current_buf ? state->buflen_0 : 973 state->buflen_1; 974 u32 *desc; 975 int sec4_sg_bytes, sec4_sg_src_index; 976 int digestsize = crypto_ahash_digestsize(ahash); 977 struct ahash_edesc *edesc; 978 int ret = 0; 979 980 sec4_sg_src_index = 1 + (buflen ? 1 : 0); 981 sec4_sg_bytes = sec4_sg_src_index * sizeof(struct sec4_sg_entry); 982 983 /* allocate space for base edesc and hw desc commands, link tables */ 984 edesc = ahash_edesc_alloc(ctx, sec4_sg_src_index, 985 ctx->sh_desc_fin, ctx->sh_desc_fin_dma, 986 flags); 987 if (!edesc) 988 return -ENOMEM; 989 990 desc = edesc->hw_desc; 991 992 edesc->sec4_sg_bytes = sec4_sg_bytes; 993 edesc->src_nents = 0; 994 995 ret = ctx_map_to_sec4_sg(desc, jrdev, state, ctx->ctx_len, 996 edesc->sec4_sg, DMA_TO_DEVICE); 997 if (ret) 998 goto unmap_ctx; 999 1000 state->buf_dma = try_buf_map_to_sec4_sg(jrdev, edesc->sec4_sg + 1, 1001 buf, state->buf_dma, buflen, 1002 last_buflen); 1003 (edesc->sec4_sg + sec4_sg_src_index - 1)->len |= 1004 cpu_to_caam32(SEC4_SG_LEN_FIN); 1005 1006 edesc->sec4_sg_dma = dma_map_single(jrdev, edesc->sec4_sg, 1007 sec4_sg_bytes, DMA_TO_DEVICE); 1008 if (dma_mapping_error(jrdev, edesc->sec4_sg_dma)) { 1009 dev_err(jrdev, "unable to map S/G table\n"); 1010 ret = -ENOMEM; 1011 goto unmap_ctx; 1012 } 1013 1014 append_seq_in_ptr(desc, edesc->sec4_sg_dma, ctx->ctx_len + buflen, 1015 LDST_SGF); 1016 1017 edesc->dst_dma = map_seq_out_ptr_result(desc, jrdev, req->result, 1018 digestsize); 1019 if (dma_mapping_error(jrdev, edesc->dst_dma)) { 1020 dev_err(jrdev, "unable to map dst\n"); 1021 ret = -ENOMEM; 1022 goto unmap_ctx; 1023 } 1024 1025 #ifdef DEBUG 1026 print_hex_dump(KERN_ERR, "jobdesc@"__stringify(__LINE__)": ", 1027 DUMP_PREFIX_ADDRESS, 16, 4, desc, desc_bytes(desc), 1); 1028 #endif 1029 1030 ret = caam_jr_enqueue(jrdev, desc, ahash_done_ctx_src, req); 1031 if (ret) 1032 goto unmap_ctx; 1033 1034 return -EINPROGRESS; 1035 unmap_ctx: 1036 ahash_unmap_ctx(jrdev, edesc, req, digestsize, DMA_FROM_DEVICE); 1037 kfree(edesc); 1038 return ret; 1039 } 1040 1041 static int ahash_finup_ctx(struct ahash_request *req) 1042 { 1043 struct crypto_ahash *ahash = crypto_ahash_reqtfm(req); 1044 struct caam_hash_ctx *ctx = crypto_ahash_ctx(ahash); 1045 struct caam_hash_state *state = ahash_request_ctx(req); 1046 struct device *jrdev = ctx->jrdev; 1047 gfp_t flags = (req->base.flags & (CRYPTO_TFM_REQ_MAY_BACKLOG | 1048 CRYPTO_TFM_REQ_MAY_SLEEP)) ? GFP_KERNEL : GFP_ATOMIC; 1049 u8 *buf = state->current_buf ? state->buf_1 : state->buf_0; 1050 int buflen = state->current_buf ? state->buflen_1 : state->buflen_0; 1051 int last_buflen = state->current_buf ? state->buflen_0 : 1052 state->buflen_1; 1053 u32 *desc; 1054 int sec4_sg_src_index; 1055 int src_nents, mapped_nents; 1056 int digestsize = crypto_ahash_digestsize(ahash); 1057 struct ahash_edesc *edesc; 1058 int ret = 0; 1059 1060 src_nents = sg_nents_for_len(req->src, req->nbytes); 1061 if (src_nents < 0) { 1062 dev_err(jrdev, "Invalid number of src SG.\n"); 1063 return src_nents; 1064 } 1065 1066 if (src_nents) { 1067 mapped_nents = dma_map_sg(jrdev, req->src, src_nents, 1068 DMA_TO_DEVICE); 1069 if (!mapped_nents) { 1070 dev_err(jrdev, "unable to DMA map source\n"); 1071 return -ENOMEM; 1072 } 1073 } else { 1074 mapped_nents = 0; 1075 } 1076 1077 sec4_sg_src_index = 1 + (buflen ? 1 : 0); 1078 1079 /* allocate space for base edesc and hw desc commands, link tables */ 1080 edesc = ahash_edesc_alloc(ctx, sec4_sg_src_index + mapped_nents, 1081 ctx->sh_desc_finup, ctx->sh_desc_finup_dma, 1082 flags); 1083 if (!edesc) { 1084 dma_unmap_sg(jrdev, req->src, src_nents, DMA_TO_DEVICE); 1085 return -ENOMEM; 1086 } 1087 1088 desc = edesc->hw_desc; 1089 1090 edesc->src_nents = src_nents; 1091 1092 ret = ctx_map_to_sec4_sg(desc, jrdev, state, ctx->ctx_len, 1093 edesc->sec4_sg, DMA_TO_DEVICE); 1094 if (ret) 1095 goto unmap_ctx; 1096 1097 state->buf_dma = try_buf_map_to_sec4_sg(jrdev, edesc->sec4_sg + 1, 1098 buf, state->buf_dma, buflen, 1099 last_buflen); 1100 1101 ret = ahash_edesc_add_src(ctx, edesc, req, mapped_nents, 1102 sec4_sg_src_index, ctx->ctx_len + buflen, 1103 req->nbytes); 1104 if (ret) 1105 goto unmap_ctx; 1106 1107 edesc->dst_dma = map_seq_out_ptr_result(desc, jrdev, req->result, 1108 digestsize); 1109 if (dma_mapping_error(jrdev, edesc->dst_dma)) { 1110 dev_err(jrdev, "unable to map dst\n"); 1111 ret = -ENOMEM; 1112 goto unmap_ctx; 1113 } 1114 1115 #ifdef DEBUG 1116 print_hex_dump(KERN_ERR, "jobdesc@"__stringify(__LINE__)": ", 1117 DUMP_PREFIX_ADDRESS, 16, 4, desc, desc_bytes(desc), 1); 1118 #endif 1119 1120 ret = caam_jr_enqueue(jrdev, desc, ahash_done_ctx_src, req); 1121 if (ret) 1122 goto unmap_ctx; 1123 1124 return -EINPROGRESS; 1125 unmap_ctx: 1126 ahash_unmap_ctx(jrdev, edesc, req, digestsize, DMA_FROM_DEVICE); 1127 kfree(edesc); 1128 return ret; 1129 } 1130 1131 static int ahash_digest(struct ahash_request *req) 1132 { 1133 struct crypto_ahash *ahash = crypto_ahash_reqtfm(req); 1134 struct caam_hash_ctx *ctx = crypto_ahash_ctx(ahash); 1135 struct device *jrdev = ctx->jrdev; 1136 gfp_t flags = (req->base.flags & (CRYPTO_TFM_REQ_MAY_BACKLOG | 1137 CRYPTO_TFM_REQ_MAY_SLEEP)) ? GFP_KERNEL : GFP_ATOMIC; 1138 u32 *desc; 1139 int digestsize = crypto_ahash_digestsize(ahash); 1140 int src_nents, mapped_nents; 1141 struct ahash_edesc *edesc; 1142 int ret = 0; 1143 1144 src_nents = sg_nents_for_len(req->src, req->nbytes); 1145 if (src_nents < 0) { 1146 dev_err(jrdev, "Invalid number of src SG.\n"); 1147 return src_nents; 1148 } 1149 1150 if (src_nents) { 1151 mapped_nents = dma_map_sg(jrdev, req->src, src_nents, 1152 DMA_TO_DEVICE); 1153 if (!mapped_nents) { 1154 dev_err(jrdev, "unable to map source for DMA\n"); 1155 return -ENOMEM; 1156 } 1157 } else { 1158 mapped_nents = 0; 1159 } 1160 1161 /* allocate space for base edesc and hw desc commands, link tables */ 1162 edesc = ahash_edesc_alloc(ctx, mapped_nents > 1 ? mapped_nents : 0, 1163 ctx->sh_desc_digest, ctx->sh_desc_digest_dma, 1164 flags); 1165 if (!edesc) { 1166 dma_unmap_sg(jrdev, req->src, src_nents, DMA_TO_DEVICE); 1167 return -ENOMEM; 1168 } 1169 1170 edesc->src_nents = src_nents; 1171 1172 ret = ahash_edesc_add_src(ctx, edesc, req, mapped_nents, 0, 0, 1173 req->nbytes); 1174 if (ret) { 1175 ahash_unmap(jrdev, edesc, req, digestsize); 1176 kfree(edesc); 1177 return ret; 1178 } 1179 1180 desc = edesc->hw_desc; 1181 1182 edesc->dst_dma = map_seq_out_ptr_result(desc, jrdev, req->result, 1183 digestsize); 1184 if (dma_mapping_error(jrdev, edesc->dst_dma)) { 1185 dev_err(jrdev, "unable to map dst\n"); 1186 ahash_unmap(jrdev, edesc, req, digestsize); 1187 kfree(edesc); 1188 return -ENOMEM; 1189 } 1190 1191 #ifdef DEBUG 1192 print_hex_dump(KERN_ERR, "jobdesc@"__stringify(__LINE__)": ", 1193 DUMP_PREFIX_ADDRESS, 16, 4, desc, desc_bytes(desc), 1); 1194 #endif 1195 1196 ret = caam_jr_enqueue(jrdev, desc, ahash_done, req); 1197 if (!ret) { 1198 ret = -EINPROGRESS; 1199 } else { 1200 ahash_unmap(jrdev, edesc, req, digestsize); 1201 kfree(edesc); 1202 } 1203 1204 return ret; 1205 } 1206 1207 /* submit ahash final if it the first job descriptor */ 1208 static int ahash_final_no_ctx(struct ahash_request *req) 1209 { 1210 struct crypto_ahash *ahash = crypto_ahash_reqtfm(req); 1211 struct caam_hash_ctx *ctx = crypto_ahash_ctx(ahash); 1212 struct caam_hash_state *state = ahash_request_ctx(req); 1213 struct device *jrdev = ctx->jrdev; 1214 gfp_t flags = (req->base.flags & (CRYPTO_TFM_REQ_MAY_BACKLOG | 1215 CRYPTO_TFM_REQ_MAY_SLEEP)) ? GFP_KERNEL : GFP_ATOMIC; 1216 u8 *buf = state->current_buf ? state->buf_1 : state->buf_0; 1217 int buflen = state->current_buf ? state->buflen_1 : state->buflen_0; 1218 u32 *desc; 1219 int digestsize = crypto_ahash_digestsize(ahash); 1220 struct ahash_edesc *edesc; 1221 int ret = 0; 1222 1223 /* allocate space for base edesc and hw desc commands, link tables */ 1224 edesc = ahash_edesc_alloc(ctx, 0, ctx->sh_desc_digest, 1225 ctx->sh_desc_digest_dma, flags); 1226 if (!edesc) 1227 return -ENOMEM; 1228 1229 desc = edesc->hw_desc; 1230 1231 state->buf_dma = dma_map_single(jrdev, buf, buflen, DMA_TO_DEVICE); 1232 if (dma_mapping_error(jrdev, state->buf_dma)) { 1233 dev_err(jrdev, "unable to map src\n"); 1234 ahash_unmap(jrdev, edesc, req, digestsize); 1235 kfree(edesc); 1236 return -ENOMEM; 1237 } 1238 1239 append_seq_in_ptr(desc, state->buf_dma, buflen, 0); 1240 1241 edesc->dst_dma = map_seq_out_ptr_result(desc, jrdev, req->result, 1242 digestsize); 1243 if (dma_mapping_error(jrdev, edesc->dst_dma)) { 1244 dev_err(jrdev, "unable to map dst\n"); 1245 ahash_unmap(jrdev, edesc, req, digestsize); 1246 kfree(edesc); 1247 return -ENOMEM; 1248 } 1249 edesc->src_nents = 0; 1250 1251 #ifdef DEBUG 1252 print_hex_dump(KERN_ERR, "jobdesc@"__stringify(__LINE__)": ", 1253 DUMP_PREFIX_ADDRESS, 16, 4, desc, desc_bytes(desc), 1); 1254 #endif 1255 1256 ret = caam_jr_enqueue(jrdev, desc, ahash_done, req); 1257 if (!ret) { 1258 ret = -EINPROGRESS; 1259 } else { 1260 ahash_unmap(jrdev, edesc, req, digestsize); 1261 kfree(edesc); 1262 } 1263 1264 return ret; 1265 } 1266 1267 /* submit ahash update if it the first job descriptor after update */ 1268 static int ahash_update_no_ctx(struct ahash_request *req) 1269 { 1270 struct crypto_ahash *ahash = crypto_ahash_reqtfm(req); 1271 struct caam_hash_ctx *ctx = crypto_ahash_ctx(ahash); 1272 struct caam_hash_state *state = ahash_request_ctx(req); 1273 struct device *jrdev = ctx->jrdev; 1274 gfp_t flags = (req->base.flags & (CRYPTO_TFM_REQ_MAY_BACKLOG | 1275 CRYPTO_TFM_REQ_MAY_SLEEP)) ? GFP_KERNEL : GFP_ATOMIC; 1276 u8 *buf = state->current_buf ? state->buf_1 : state->buf_0; 1277 int *buflen = state->current_buf ? &state->buflen_1 : &state->buflen_0; 1278 u8 *next_buf = state->current_buf ? state->buf_0 : state->buf_1; 1279 int *next_buflen = state->current_buf ? &state->buflen_0 : 1280 &state->buflen_1; 1281 int in_len = *buflen + req->nbytes, to_hash; 1282 int sec4_sg_bytes, src_nents, mapped_nents; 1283 struct ahash_edesc *edesc; 1284 u32 *desc; 1285 int ret = 0; 1286 1287 *next_buflen = in_len & (crypto_tfm_alg_blocksize(&ahash->base) - 1); 1288 to_hash = in_len - *next_buflen; 1289 1290 if (to_hash) { 1291 src_nents = sg_nents_for_len(req->src, 1292 req->nbytes - *next_buflen); 1293 if (src_nents < 0) { 1294 dev_err(jrdev, "Invalid number of src SG.\n"); 1295 return src_nents; 1296 } 1297 1298 if (src_nents) { 1299 mapped_nents = dma_map_sg(jrdev, req->src, src_nents, 1300 DMA_TO_DEVICE); 1301 if (!mapped_nents) { 1302 dev_err(jrdev, "unable to DMA map source\n"); 1303 return -ENOMEM; 1304 } 1305 } else { 1306 mapped_nents = 0; 1307 } 1308 1309 sec4_sg_bytes = (1 + mapped_nents) * 1310 sizeof(struct sec4_sg_entry); 1311 1312 /* 1313 * allocate space for base edesc and hw desc commands, 1314 * link tables 1315 */ 1316 edesc = ahash_edesc_alloc(ctx, 1 + mapped_nents, 1317 ctx->sh_desc_update_first, 1318 ctx->sh_desc_update_first_dma, 1319 flags); 1320 if (!edesc) { 1321 dma_unmap_sg(jrdev, req->src, src_nents, DMA_TO_DEVICE); 1322 return -ENOMEM; 1323 } 1324 1325 edesc->src_nents = src_nents; 1326 edesc->sec4_sg_bytes = sec4_sg_bytes; 1327 edesc->dst_dma = 0; 1328 1329 state->buf_dma = buf_map_to_sec4_sg(jrdev, edesc->sec4_sg, 1330 buf, *buflen); 1331 sg_to_sec4_sg_last(req->src, mapped_nents, 1332 edesc->sec4_sg + 1, 0); 1333 1334 if (*next_buflen) { 1335 scatterwalk_map_and_copy(next_buf, req->src, 1336 to_hash - *buflen, 1337 *next_buflen, 0); 1338 } 1339 1340 state->current_buf = !state->current_buf; 1341 1342 desc = edesc->hw_desc; 1343 1344 edesc->sec4_sg_dma = dma_map_single(jrdev, edesc->sec4_sg, 1345 sec4_sg_bytes, 1346 DMA_TO_DEVICE); 1347 if (dma_mapping_error(jrdev, edesc->sec4_sg_dma)) { 1348 dev_err(jrdev, "unable to map S/G table\n"); 1349 ret = -ENOMEM; 1350 goto unmap_ctx; 1351 } 1352 1353 append_seq_in_ptr(desc, edesc->sec4_sg_dma, to_hash, LDST_SGF); 1354 1355 ret = map_seq_out_ptr_ctx(desc, jrdev, state, ctx->ctx_len); 1356 if (ret) 1357 goto unmap_ctx; 1358 1359 #ifdef DEBUG 1360 print_hex_dump(KERN_ERR, "jobdesc@"__stringify(__LINE__)": ", 1361 DUMP_PREFIX_ADDRESS, 16, 4, desc, 1362 desc_bytes(desc), 1); 1363 #endif 1364 1365 ret = caam_jr_enqueue(jrdev, desc, ahash_done_ctx_dst, req); 1366 if (ret) 1367 goto unmap_ctx; 1368 1369 ret = -EINPROGRESS; 1370 state->update = ahash_update_ctx; 1371 state->finup = ahash_finup_ctx; 1372 state->final = ahash_final_ctx; 1373 } else if (*next_buflen) { 1374 scatterwalk_map_and_copy(buf + *buflen, req->src, 0, 1375 req->nbytes, 0); 1376 *buflen = *next_buflen; 1377 *next_buflen = 0; 1378 } 1379 #ifdef DEBUG 1380 print_hex_dump(KERN_ERR, "buf@"__stringify(__LINE__)": ", 1381 DUMP_PREFIX_ADDRESS, 16, 4, buf, *buflen, 1); 1382 print_hex_dump(KERN_ERR, "next buf@"__stringify(__LINE__)": ", 1383 DUMP_PREFIX_ADDRESS, 16, 4, next_buf, 1384 *next_buflen, 1); 1385 #endif 1386 1387 return ret; 1388 unmap_ctx: 1389 ahash_unmap_ctx(jrdev, edesc, req, ctx->ctx_len, DMA_TO_DEVICE); 1390 kfree(edesc); 1391 return ret; 1392 } 1393 1394 /* submit ahash finup if it the first job descriptor after update */ 1395 static int ahash_finup_no_ctx(struct ahash_request *req) 1396 { 1397 struct crypto_ahash *ahash = crypto_ahash_reqtfm(req); 1398 struct caam_hash_ctx *ctx = crypto_ahash_ctx(ahash); 1399 struct caam_hash_state *state = ahash_request_ctx(req); 1400 struct device *jrdev = ctx->jrdev; 1401 gfp_t flags = (req->base.flags & (CRYPTO_TFM_REQ_MAY_BACKLOG | 1402 CRYPTO_TFM_REQ_MAY_SLEEP)) ? GFP_KERNEL : GFP_ATOMIC; 1403 u8 *buf = state->current_buf ? state->buf_1 : state->buf_0; 1404 int buflen = state->current_buf ? state->buflen_1 : state->buflen_0; 1405 int last_buflen = state->current_buf ? state->buflen_0 : 1406 state->buflen_1; 1407 u32 *desc; 1408 int sec4_sg_bytes, sec4_sg_src_index, src_nents, mapped_nents; 1409 int digestsize = crypto_ahash_digestsize(ahash); 1410 struct ahash_edesc *edesc; 1411 int ret = 0; 1412 1413 src_nents = sg_nents_for_len(req->src, req->nbytes); 1414 if (src_nents < 0) { 1415 dev_err(jrdev, "Invalid number of src SG.\n"); 1416 return src_nents; 1417 } 1418 1419 if (src_nents) { 1420 mapped_nents = dma_map_sg(jrdev, req->src, src_nents, 1421 DMA_TO_DEVICE); 1422 if (!mapped_nents) { 1423 dev_err(jrdev, "unable to DMA map source\n"); 1424 return -ENOMEM; 1425 } 1426 } else { 1427 mapped_nents = 0; 1428 } 1429 1430 sec4_sg_src_index = 2; 1431 sec4_sg_bytes = (sec4_sg_src_index + mapped_nents) * 1432 sizeof(struct sec4_sg_entry); 1433 1434 /* allocate space for base edesc and hw desc commands, link tables */ 1435 edesc = ahash_edesc_alloc(ctx, sec4_sg_src_index + mapped_nents, 1436 ctx->sh_desc_digest, ctx->sh_desc_digest_dma, 1437 flags); 1438 if (!edesc) { 1439 dma_unmap_sg(jrdev, req->src, src_nents, DMA_TO_DEVICE); 1440 return -ENOMEM; 1441 } 1442 1443 desc = edesc->hw_desc; 1444 1445 edesc->src_nents = src_nents; 1446 edesc->sec4_sg_bytes = sec4_sg_bytes; 1447 1448 state->buf_dma = try_buf_map_to_sec4_sg(jrdev, edesc->sec4_sg, buf, 1449 state->buf_dma, buflen, 1450 last_buflen); 1451 1452 ret = ahash_edesc_add_src(ctx, edesc, req, mapped_nents, 1, buflen, 1453 req->nbytes); 1454 if (ret) { 1455 dev_err(jrdev, "unable to map S/G table\n"); 1456 ahash_unmap(jrdev, edesc, req, digestsize); 1457 kfree(edesc); 1458 return -ENOMEM; 1459 } 1460 1461 edesc->dst_dma = map_seq_out_ptr_result(desc, jrdev, req->result, 1462 digestsize); 1463 if (dma_mapping_error(jrdev, edesc->dst_dma)) { 1464 dev_err(jrdev, "unable to map dst\n"); 1465 ahash_unmap(jrdev, edesc, req, digestsize); 1466 kfree(edesc); 1467 return -ENOMEM; 1468 } 1469 1470 #ifdef DEBUG 1471 print_hex_dump(KERN_ERR, "jobdesc@"__stringify(__LINE__)": ", 1472 DUMP_PREFIX_ADDRESS, 16, 4, desc, desc_bytes(desc), 1); 1473 #endif 1474 1475 ret = caam_jr_enqueue(jrdev, desc, ahash_done, req); 1476 if (!ret) { 1477 ret = -EINPROGRESS; 1478 } else { 1479 ahash_unmap(jrdev, edesc, req, digestsize); 1480 kfree(edesc); 1481 } 1482 1483 return ret; 1484 } 1485 1486 /* submit first update job descriptor after init */ 1487 static int ahash_update_first(struct ahash_request *req) 1488 { 1489 struct crypto_ahash *ahash = crypto_ahash_reqtfm(req); 1490 struct caam_hash_ctx *ctx = crypto_ahash_ctx(ahash); 1491 struct caam_hash_state *state = ahash_request_ctx(req); 1492 struct device *jrdev = ctx->jrdev; 1493 gfp_t flags = (req->base.flags & (CRYPTO_TFM_REQ_MAY_BACKLOG | 1494 CRYPTO_TFM_REQ_MAY_SLEEP)) ? GFP_KERNEL : GFP_ATOMIC; 1495 u8 *next_buf = state->current_buf ? state->buf_1 : state->buf_0; 1496 int *next_buflen = state->current_buf ? 1497 &state->buflen_1 : &state->buflen_0; 1498 int to_hash; 1499 u32 *desc; 1500 int src_nents, mapped_nents; 1501 struct ahash_edesc *edesc; 1502 int ret = 0; 1503 1504 *next_buflen = req->nbytes & (crypto_tfm_alg_blocksize(&ahash->base) - 1505 1); 1506 to_hash = req->nbytes - *next_buflen; 1507 1508 if (to_hash) { 1509 src_nents = sg_nents_for_len(req->src, 1510 req->nbytes - *next_buflen); 1511 if (src_nents < 0) { 1512 dev_err(jrdev, "Invalid number of src SG.\n"); 1513 return src_nents; 1514 } 1515 1516 if (src_nents) { 1517 mapped_nents = dma_map_sg(jrdev, req->src, src_nents, 1518 DMA_TO_DEVICE); 1519 if (!mapped_nents) { 1520 dev_err(jrdev, "unable to map source for DMA\n"); 1521 return -ENOMEM; 1522 } 1523 } else { 1524 mapped_nents = 0; 1525 } 1526 1527 /* 1528 * allocate space for base edesc and hw desc commands, 1529 * link tables 1530 */ 1531 edesc = ahash_edesc_alloc(ctx, mapped_nents > 1 ? 1532 mapped_nents : 0, 1533 ctx->sh_desc_update_first, 1534 ctx->sh_desc_update_first_dma, 1535 flags); 1536 if (!edesc) { 1537 dma_unmap_sg(jrdev, req->src, src_nents, DMA_TO_DEVICE); 1538 return -ENOMEM; 1539 } 1540 1541 edesc->src_nents = src_nents; 1542 edesc->dst_dma = 0; 1543 1544 ret = ahash_edesc_add_src(ctx, edesc, req, mapped_nents, 0, 0, 1545 to_hash); 1546 if (ret) 1547 goto unmap_ctx; 1548 1549 if (*next_buflen) 1550 scatterwalk_map_and_copy(next_buf, req->src, to_hash, 1551 *next_buflen, 0); 1552 1553 desc = edesc->hw_desc; 1554 1555 ret = map_seq_out_ptr_ctx(desc, jrdev, state, ctx->ctx_len); 1556 if (ret) 1557 goto unmap_ctx; 1558 1559 #ifdef DEBUG 1560 print_hex_dump(KERN_ERR, "jobdesc@"__stringify(__LINE__)": ", 1561 DUMP_PREFIX_ADDRESS, 16, 4, desc, 1562 desc_bytes(desc), 1); 1563 #endif 1564 1565 ret = caam_jr_enqueue(jrdev, desc, ahash_done_ctx_dst, req); 1566 if (ret) 1567 goto unmap_ctx; 1568 1569 ret = -EINPROGRESS; 1570 state->update = ahash_update_ctx; 1571 state->finup = ahash_finup_ctx; 1572 state->final = ahash_final_ctx; 1573 } else if (*next_buflen) { 1574 state->update = ahash_update_no_ctx; 1575 state->finup = ahash_finup_no_ctx; 1576 state->final = ahash_final_no_ctx; 1577 scatterwalk_map_and_copy(next_buf, req->src, 0, 1578 req->nbytes, 0); 1579 } 1580 #ifdef DEBUG 1581 print_hex_dump(KERN_ERR, "next buf@"__stringify(__LINE__)": ", 1582 DUMP_PREFIX_ADDRESS, 16, 4, next_buf, 1583 *next_buflen, 1); 1584 #endif 1585 1586 return ret; 1587 unmap_ctx: 1588 ahash_unmap_ctx(jrdev, edesc, req, ctx->ctx_len, DMA_TO_DEVICE); 1589 kfree(edesc); 1590 return ret; 1591 } 1592 1593 static int ahash_finup_first(struct ahash_request *req) 1594 { 1595 return ahash_digest(req); 1596 } 1597 1598 static int ahash_init(struct ahash_request *req) 1599 { 1600 struct caam_hash_state *state = ahash_request_ctx(req); 1601 1602 state->update = ahash_update_first; 1603 state->finup = ahash_finup_first; 1604 state->final = ahash_final_no_ctx; 1605 1606 state->current_buf = 0; 1607 state->buf_dma = 0; 1608 state->buflen_0 = 0; 1609 state->buflen_1 = 0; 1610 1611 return 0; 1612 } 1613 1614 static int ahash_update(struct ahash_request *req) 1615 { 1616 struct caam_hash_state *state = ahash_request_ctx(req); 1617 1618 return state->update(req); 1619 } 1620 1621 static int ahash_finup(struct ahash_request *req) 1622 { 1623 struct caam_hash_state *state = ahash_request_ctx(req); 1624 1625 return state->finup(req); 1626 } 1627 1628 static int ahash_final(struct ahash_request *req) 1629 { 1630 struct caam_hash_state *state = ahash_request_ctx(req); 1631 1632 return state->final(req); 1633 } 1634 1635 static int ahash_export(struct ahash_request *req, void *out) 1636 { 1637 struct caam_hash_state *state = ahash_request_ctx(req); 1638 struct caam_export_state *export = out; 1639 int len; 1640 u8 *buf; 1641 1642 if (state->current_buf) { 1643 buf = state->buf_1; 1644 len = state->buflen_1; 1645 } else { 1646 buf = state->buf_0; 1647 len = state->buflen_0; 1648 } 1649 1650 memcpy(export->buf, buf, len); 1651 memcpy(export->caam_ctx, state->caam_ctx, sizeof(export->caam_ctx)); 1652 export->buflen = len; 1653 export->update = state->update; 1654 export->final = state->final; 1655 export->finup = state->finup; 1656 1657 return 0; 1658 } 1659 1660 static int ahash_import(struct ahash_request *req, const void *in) 1661 { 1662 struct caam_hash_state *state = ahash_request_ctx(req); 1663 const struct caam_export_state *export = in; 1664 1665 memset(state, 0, sizeof(*state)); 1666 memcpy(state->buf_0, export->buf, export->buflen); 1667 memcpy(state->caam_ctx, export->caam_ctx, sizeof(state->caam_ctx)); 1668 state->buflen_0 = export->buflen; 1669 state->update = export->update; 1670 state->final = export->final; 1671 state->finup = export->finup; 1672 1673 return 0; 1674 } 1675 1676 struct caam_hash_template { 1677 char name[CRYPTO_MAX_ALG_NAME]; 1678 char driver_name[CRYPTO_MAX_ALG_NAME]; 1679 char hmac_name[CRYPTO_MAX_ALG_NAME]; 1680 char hmac_driver_name[CRYPTO_MAX_ALG_NAME]; 1681 unsigned int blocksize; 1682 struct ahash_alg template_ahash; 1683 u32 alg_type; 1684 u32 alg_op; 1685 }; 1686 1687 /* ahash descriptors */ 1688 static struct caam_hash_template driver_hash[] = { 1689 { 1690 .name = "sha1", 1691 .driver_name = "sha1-caam", 1692 .hmac_name = "hmac(sha1)", 1693 .hmac_driver_name = "hmac-sha1-caam", 1694 .blocksize = SHA1_BLOCK_SIZE, 1695 .template_ahash = { 1696 .init = ahash_init, 1697 .update = ahash_update, 1698 .final = ahash_final, 1699 .finup = ahash_finup, 1700 .digest = ahash_digest, 1701 .export = ahash_export, 1702 .import = ahash_import, 1703 .setkey = ahash_setkey, 1704 .halg = { 1705 .digestsize = SHA1_DIGEST_SIZE, 1706 .statesize = sizeof(struct caam_export_state), 1707 }, 1708 }, 1709 .alg_type = OP_ALG_ALGSEL_SHA1, 1710 .alg_op = OP_ALG_ALGSEL_SHA1 | OP_ALG_AAI_HMAC, 1711 }, { 1712 .name = "sha224", 1713 .driver_name = "sha224-caam", 1714 .hmac_name = "hmac(sha224)", 1715 .hmac_driver_name = "hmac-sha224-caam", 1716 .blocksize = SHA224_BLOCK_SIZE, 1717 .template_ahash = { 1718 .init = ahash_init, 1719 .update = ahash_update, 1720 .final = ahash_final, 1721 .finup = ahash_finup, 1722 .digest = ahash_digest, 1723 .export = ahash_export, 1724 .import = ahash_import, 1725 .setkey = ahash_setkey, 1726 .halg = { 1727 .digestsize = SHA224_DIGEST_SIZE, 1728 .statesize = sizeof(struct caam_export_state), 1729 }, 1730 }, 1731 .alg_type = OP_ALG_ALGSEL_SHA224, 1732 .alg_op = OP_ALG_ALGSEL_SHA224 | OP_ALG_AAI_HMAC, 1733 }, { 1734 .name = "sha256", 1735 .driver_name = "sha256-caam", 1736 .hmac_name = "hmac(sha256)", 1737 .hmac_driver_name = "hmac-sha256-caam", 1738 .blocksize = SHA256_BLOCK_SIZE, 1739 .template_ahash = { 1740 .init = ahash_init, 1741 .update = ahash_update, 1742 .final = ahash_final, 1743 .finup = ahash_finup, 1744 .digest = ahash_digest, 1745 .export = ahash_export, 1746 .import = ahash_import, 1747 .setkey = ahash_setkey, 1748 .halg = { 1749 .digestsize = SHA256_DIGEST_SIZE, 1750 .statesize = sizeof(struct caam_export_state), 1751 }, 1752 }, 1753 .alg_type = OP_ALG_ALGSEL_SHA256, 1754 .alg_op = OP_ALG_ALGSEL_SHA256 | OP_ALG_AAI_HMAC, 1755 }, { 1756 .name = "sha384", 1757 .driver_name = "sha384-caam", 1758 .hmac_name = "hmac(sha384)", 1759 .hmac_driver_name = "hmac-sha384-caam", 1760 .blocksize = SHA384_BLOCK_SIZE, 1761 .template_ahash = { 1762 .init = ahash_init, 1763 .update = ahash_update, 1764 .final = ahash_final, 1765 .finup = ahash_finup, 1766 .digest = ahash_digest, 1767 .export = ahash_export, 1768 .import = ahash_import, 1769 .setkey = ahash_setkey, 1770 .halg = { 1771 .digestsize = SHA384_DIGEST_SIZE, 1772 .statesize = sizeof(struct caam_export_state), 1773 }, 1774 }, 1775 .alg_type = OP_ALG_ALGSEL_SHA384, 1776 .alg_op = OP_ALG_ALGSEL_SHA384 | OP_ALG_AAI_HMAC, 1777 }, { 1778 .name = "sha512", 1779 .driver_name = "sha512-caam", 1780 .hmac_name = "hmac(sha512)", 1781 .hmac_driver_name = "hmac-sha512-caam", 1782 .blocksize = SHA512_BLOCK_SIZE, 1783 .template_ahash = { 1784 .init = ahash_init, 1785 .update = ahash_update, 1786 .final = ahash_final, 1787 .finup = ahash_finup, 1788 .digest = ahash_digest, 1789 .export = ahash_export, 1790 .import = ahash_import, 1791 .setkey = ahash_setkey, 1792 .halg = { 1793 .digestsize = SHA512_DIGEST_SIZE, 1794 .statesize = sizeof(struct caam_export_state), 1795 }, 1796 }, 1797 .alg_type = OP_ALG_ALGSEL_SHA512, 1798 .alg_op = OP_ALG_ALGSEL_SHA512 | OP_ALG_AAI_HMAC, 1799 }, { 1800 .name = "md5", 1801 .driver_name = "md5-caam", 1802 .hmac_name = "hmac(md5)", 1803 .hmac_driver_name = "hmac-md5-caam", 1804 .blocksize = MD5_BLOCK_WORDS * 4, 1805 .template_ahash = { 1806 .init = ahash_init, 1807 .update = ahash_update, 1808 .final = ahash_final, 1809 .finup = ahash_finup, 1810 .digest = ahash_digest, 1811 .export = ahash_export, 1812 .import = ahash_import, 1813 .setkey = ahash_setkey, 1814 .halg = { 1815 .digestsize = MD5_DIGEST_SIZE, 1816 .statesize = sizeof(struct caam_export_state), 1817 }, 1818 }, 1819 .alg_type = OP_ALG_ALGSEL_MD5, 1820 .alg_op = OP_ALG_ALGSEL_MD5 | OP_ALG_AAI_HMAC, 1821 }, 1822 }; 1823 1824 struct caam_hash_alg { 1825 struct list_head entry; 1826 int alg_type; 1827 int alg_op; 1828 struct ahash_alg ahash_alg; 1829 }; 1830 1831 static int caam_hash_cra_init(struct crypto_tfm *tfm) 1832 { 1833 struct crypto_ahash *ahash = __crypto_ahash_cast(tfm); 1834 struct crypto_alg *base = tfm->__crt_alg; 1835 struct hash_alg_common *halg = 1836 container_of(base, struct hash_alg_common, base); 1837 struct ahash_alg *alg = 1838 container_of(halg, struct ahash_alg, halg); 1839 struct caam_hash_alg *caam_hash = 1840 container_of(alg, struct caam_hash_alg, ahash_alg); 1841 struct caam_hash_ctx *ctx = crypto_tfm_ctx(tfm); 1842 /* Sizes for MDHA running digests: MD5, SHA1, 224, 256, 384, 512 */ 1843 static const u8 runninglen[] = { HASH_MSG_LEN + MD5_DIGEST_SIZE, 1844 HASH_MSG_LEN + SHA1_DIGEST_SIZE, 1845 HASH_MSG_LEN + 32, 1846 HASH_MSG_LEN + SHA256_DIGEST_SIZE, 1847 HASH_MSG_LEN + 64, 1848 HASH_MSG_LEN + SHA512_DIGEST_SIZE }; 1849 int ret = 0; 1850 1851 /* 1852 * Get a Job ring from Job Ring driver to ensure in-order 1853 * crypto request processing per tfm 1854 */ 1855 ctx->jrdev = caam_jr_alloc(); 1856 if (IS_ERR(ctx->jrdev)) { 1857 pr_err("Job Ring Device allocation for transform failed\n"); 1858 return PTR_ERR(ctx->jrdev); 1859 } 1860 /* copy descriptor header template value */ 1861 ctx->alg_type = OP_TYPE_CLASS2_ALG | caam_hash->alg_type; 1862 ctx->alg_op = OP_TYPE_CLASS2_ALG | caam_hash->alg_op; 1863 1864 ctx->ctx_len = runninglen[(ctx->alg_op & OP_ALG_ALGSEL_SUBMASK) >> 1865 OP_ALG_ALGSEL_SHIFT]; 1866 1867 crypto_ahash_set_reqsize(__crypto_ahash_cast(tfm), 1868 sizeof(struct caam_hash_state)); 1869 1870 ret = ahash_set_sh_desc(ahash); 1871 1872 return ret; 1873 } 1874 1875 static void caam_hash_cra_exit(struct crypto_tfm *tfm) 1876 { 1877 struct caam_hash_ctx *ctx = crypto_tfm_ctx(tfm); 1878 1879 if (ctx->sh_desc_update_dma && 1880 !dma_mapping_error(ctx->jrdev, ctx->sh_desc_update_dma)) 1881 dma_unmap_single(ctx->jrdev, ctx->sh_desc_update_dma, 1882 desc_bytes(ctx->sh_desc_update), 1883 DMA_TO_DEVICE); 1884 if (ctx->sh_desc_update_first_dma && 1885 !dma_mapping_error(ctx->jrdev, ctx->sh_desc_update_first_dma)) 1886 dma_unmap_single(ctx->jrdev, ctx->sh_desc_update_first_dma, 1887 desc_bytes(ctx->sh_desc_update_first), 1888 DMA_TO_DEVICE); 1889 if (ctx->sh_desc_fin_dma && 1890 !dma_mapping_error(ctx->jrdev, ctx->sh_desc_fin_dma)) 1891 dma_unmap_single(ctx->jrdev, ctx->sh_desc_fin_dma, 1892 desc_bytes(ctx->sh_desc_fin), DMA_TO_DEVICE); 1893 if (ctx->sh_desc_digest_dma && 1894 !dma_mapping_error(ctx->jrdev, ctx->sh_desc_digest_dma)) 1895 dma_unmap_single(ctx->jrdev, ctx->sh_desc_digest_dma, 1896 desc_bytes(ctx->sh_desc_digest), 1897 DMA_TO_DEVICE); 1898 if (ctx->sh_desc_finup_dma && 1899 !dma_mapping_error(ctx->jrdev, ctx->sh_desc_finup_dma)) 1900 dma_unmap_single(ctx->jrdev, ctx->sh_desc_finup_dma, 1901 desc_bytes(ctx->sh_desc_finup), DMA_TO_DEVICE); 1902 1903 caam_jr_free(ctx->jrdev); 1904 } 1905 1906 static void __exit caam_algapi_hash_exit(void) 1907 { 1908 struct caam_hash_alg *t_alg, *n; 1909 1910 if (!hash_list.next) 1911 return; 1912 1913 list_for_each_entry_safe(t_alg, n, &hash_list, entry) { 1914 crypto_unregister_ahash(&t_alg->ahash_alg); 1915 list_del(&t_alg->entry); 1916 kfree(t_alg); 1917 } 1918 } 1919 1920 static struct caam_hash_alg * 1921 caam_hash_alloc(struct caam_hash_template *template, 1922 bool keyed) 1923 { 1924 struct caam_hash_alg *t_alg; 1925 struct ahash_alg *halg; 1926 struct crypto_alg *alg; 1927 1928 t_alg = kzalloc(sizeof(*t_alg), GFP_KERNEL); 1929 if (!t_alg) { 1930 pr_err("failed to allocate t_alg\n"); 1931 return ERR_PTR(-ENOMEM); 1932 } 1933 1934 t_alg->ahash_alg = template->template_ahash; 1935 halg = &t_alg->ahash_alg; 1936 alg = &halg->halg.base; 1937 1938 if (keyed) { 1939 snprintf(alg->cra_name, CRYPTO_MAX_ALG_NAME, "%s", 1940 template->hmac_name); 1941 snprintf(alg->cra_driver_name, CRYPTO_MAX_ALG_NAME, "%s", 1942 template->hmac_driver_name); 1943 } else { 1944 snprintf(alg->cra_name, CRYPTO_MAX_ALG_NAME, "%s", 1945 template->name); 1946 snprintf(alg->cra_driver_name, CRYPTO_MAX_ALG_NAME, "%s", 1947 template->driver_name); 1948 t_alg->ahash_alg.setkey = NULL; 1949 } 1950 alg->cra_module = THIS_MODULE; 1951 alg->cra_init = caam_hash_cra_init; 1952 alg->cra_exit = caam_hash_cra_exit; 1953 alg->cra_ctxsize = sizeof(struct caam_hash_ctx); 1954 alg->cra_priority = CAAM_CRA_PRIORITY; 1955 alg->cra_blocksize = template->blocksize; 1956 alg->cra_alignmask = 0; 1957 alg->cra_flags = CRYPTO_ALG_ASYNC | CRYPTO_ALG_TYPE_AHASH; 1958 alg->cra_type = &crypto_ahash_type; 1959 1960 t_alg->alg_type = template->alg_type; 1961 t_alg->alg_op = template->alg_op; 1962 1963 return t_alg; 1964 } 1965 1966 static int __init caam_algapi_hash_init(void) 1967 { 1968 struct device_node *dev_node; 1969 struct platform_device *pdev; 1970 struct device *ctrldev; 1971 int i = 0, err = 0; 1972 struct caam_drv_private *priv; 1973 unsigned int md_limit = SHA512_DIGEST_SIZE; 1974 u32 cha_inst, cha_vid; 1975 1976 dev_node = of_find_compatible_node(NULL, NULL, "fsl,sec-v4.0"); 1977 if (!dev_node) { 1978 dev_node = of_find_compatible_node(NULL, NULL, "fsl,sec4.0"); 1979 if (!dev_node) 1980 return -ENODEV; 1981 } 1982 1983 pdev = of_find_device_by_node(dev_node); 1984 if (!pdev) { 1985 of_node_put(dev_node); 1986 return -ENODEV; 1987 } 1988 1989 ctrldev = &pdev->dev; 1990 priv = dev_get_drvdata(ctrldev); 1991 of_node_put(dev_node); 1992 1993 /* 1994 * If priv is NULL, it's probably because the caam driver wasn't 1995 * properly initialized (e.g. RNG4 init failed). Thus, bail out here. 1996 */ 1997 if (!priv) 1998 return -ENODEV; 1999 2000 /* 2001 * Register crypto algorithms the device supports. First, identify 2002 * presence and attributes of MD block. 2003 */ 2004 cha_vid = rd_reg32(&priv->ctrl->perfmon.cha_id_ls); 2005 cha_inst = rd_reg32(&priv->ctrl->perfmon.cha_num_ls); 2006 2007 /* 2008 * Skip registration of any hashing algorithms if MD block 2009 * is not present. 2010 */ 2011 if (!((cha_inst & CHA_ID_LS_MD_MASK) >> CHA_ID_LS_MD_SHIFT)) 2012 return -ENODEV; 2013 2014 /* Limit digest size based on LP256 */ 2015 if ((cha_vid & CHA_ID_LS_MD_MASK) == CHA_ID_LS_MD_LP256) 2016 md_limit = SHA256_DIGEST_SIZE; 2017 2018 INIT_LIST_HEAD(&hash_list); 2019 2020 /* register crypto algorithms the device supports */ 2021 for (i = 0; i < ARRAY_SIZE(driver_hash); i++) { 2022 struct caam_hash_alg *t_alg; 2023 struct caam_hash_template *alg = driver_hash + i; 2024 2025 /* If MD size is not supported by device, skip registration */ 2026 if (alg->template_ahash.halg.digestsize > md_limit) 2027 continue; 2028 2029 /* register hmac version */ 2030 t_alg = caam_hash_alloc(alg, true); 2031 if (IS_ERR(t_alg)) { 2032 err = PTR_ERR(t_alg); 2033 pr_warn("%s alg allocation failed\n", alg->driver_name); 2034 continue; 2035 } 2036 2037 err = crypto_register_ahash(&t_alg->ahash_alg); 2038 if (err) { 2039 pr_warn("%s alg registration failed: %d\n", 2040 t_alg->ahash_alg.halg.base.cra_driver_name, 2041 err); 2042 kfree(t_alg); 2043 } else 2044 list_add_tail(&t_alg->entry, &hash_list); 2045 2046 /* register unkeyed version */ 2047 t_alg = caam_hash_alloc(alg, false); 2048 if (IS_ERR(t_alg)) { 2049 err = PTR_ERR(t_alg); 2050 pr_warn("%s alg allocation failed\n", alg->driver_name); 2051 continue; 2052 } 2053 2054 err = crypto_register_ahash(&t_alg->ahash_alg); 2055 if (err) { 2056 pr_warn("%s alg registration failed: %d\n", 2057 t_alg->ahash_alg.halg.base.cra_driver_name, 2058 err); 2059 kfree(t_alg); 2060 } else 2061 list_add_tail(&t_alg->entry, &hash_list); 2062 } 2063 2064 return err; 2065 } 2066 2067 module_init(caam_algapi_hash_init); 2068 module_exit(caam_algapi_hash_exit); 2069 2070 MODULE_LICENSE("GPL"); 2071 MODULE_DESCRIPTION("FSL CAAM support for ahash functions of crypto API"); 2072 MODULE_AUTHOR("Freescale Semiconductor - NMG"); 2073