1 // SPDX-License-Identifier: GPL-2.0 2 /* 3 * Copyright (C) 2017 Marvell 4 * 5 * Antoine Tenart <antoine.tenart@free-electrons.com> 6 */ 7 8 #include <crypto/hmac.h> 9 #include <crypto/md5.h> 10 #include <crypto/sha.h> 11 #include <linux/device.h> 12 #include <linux/dma-mapping.h> 13 #include <linux/dmapool.h> 14 15 #include "safexcel.h" 16 17 struct safexcel_ahash_ctx { 18 struct safexcel_context base; 19 struct safexcel_crypto_priv *priv; 20 21 u32 alg; 22 23 u32 ipad[SHA512_DIGEST_SIZE / sizeof(u32)]; 24 u32 opad[SHA512_DIGEST_SIZE / sizeof(u32)]; 25 }; 26 27 struct safexcel_ahash_req { 28 bool last_req; 29 bool finish; 30 bool hmac; 31 bool needs_inv; 32 33 int nents; 34 dma_addr_t result_dma; 35 36 u32 digest; 37 38 u8 state_sz; /* expected sate size, only set once */ 39 u32 state[SHA512_DIGEST_SIZE / sizeof(u32)] __aligned(sizeof(u32)); 40 41 u64 len[2]; 42 u64 processed[2]; 43 44 u8 cache[SHA512_BLOCK_SIZE << 1] __aligned(sizeof(u32)); 45 dma_addr_t cache_dma; 46 unsigned int cache_sz; 47 48 u8 cache_next[SHA512_BLOCK_SIZE << 1] __aligned(sizeof(u32)); 49 }; 50 51 static inline u64 safexcel_queued_len(struct safexcel_ahash_req *req) 52 { 53 u64 len, processed; 54 55 len = (0xffffffff * req->len[1]) + req->len[0]; 56 processed = (0xffffffff * req->processed[1]) + req->processed[0]; 57 58 return len - processed; 59 } 60 61 static void safexcel_hash_token(struct safexcel_command_desc *cdesc, 62 u32 input_length, u32 result_length) 63 { 64 struct safexcel_token *token = 65 (struct safexcel_token *)cdesc->control_data.token; 66 67 token[0].opcode = EIP197_TOKEN_OPCODE_DIRECTION; 68 token[0].packet_length = input_length; 69 token[0].stat = EIP197_TOKEN_STAT_LAST_HASH; 70 token[0].instructions = EIP197_TOKEN_INS_TYPE_HASH; 71 72 token[1].opcode = EIP197_TOKEN_OPCODE_INSERT; 73 token[1].packet_length = result_length; 74 token[1].stat = EIP197_TOKEN_STAT_LAST_HASH | 75 EIP197_TOKEN_STAT_LAST_PACKET; 76 token[1].instructions = EIP197_TOKEN_INS_TYPE_OUTPUT | 77 EIP197_TOKEN_INS_INSERT_HASH_DIGEST; 78 } 79 80 static void safexcel_context_control(struct safexcel_ahash_ctx *ctx, 81 struct safexcel_ahash_req *req, 82 struct safexcel_command_desc *cdesc, 83 unsigned int digestsize) 84 { 85 struct safexcel_crypto_priv *priv = ctx->priv; 86 int i; 87 88 cdesc->control_data.control0 |= CONTEXT_CONTROL_TYPE_HASH_OUT; 89 cdesc->control_data.control0 |= ctx->alg; 90 cdesc->control_data.control0 |= req->digest; 91 92 if (!req->finish) 93 cdesc->control_data.control0 |= CONTEXT_CONTROL_NO_FINISH_HASH; 94 95 if (req->digest == CONTEXT_CONTROL_DIGEST_PRECOMPUTED) { 96 if (req->processed[0] || req->processed[1]) { 97 if (ctx->alg == CONTEXT_CONTROL_CRYPTO_ALG_MD5) 98 cdesc->control_data.control0 |= CONTEXT_CONTROL_SIZE(5); 99 else if (ctx->alg == CONTEXT_CONTROL_CRYPTO_ALG_SHA1) 100 cdesc->control_data.control0 |= CONTEXT_CONTROL_SIZE(6); 101 else if (ctx->alg == CONTEXT_CONTROL_CRYPTO_ALG_SHA224 || 102 ctx->alg == CONTEXT_CONTROL_CRYPTO_ALG_SHA256) 103 cdesc->control_data.control0 |= CONTEXT_CONTROL_SIZE(9); 104 else if (ctx->alg == CONTEXT_CONTROL_CRYPTO_ALG_SHA384 || 105 ctx->alg == CONTEXT_CONTROL_CRYPTO_ALG_SHA512) 106 cdesc->control_data.control0 |= CONTEXT_CONTROL_SIZE(17); 107 108 cdesc->control_data.control1 |= CONTEXT_CONTROL_DIGEST_CNT; 109 } else { 110 cdesc->control_data.control0 |= CONTEXT_CONTROL_RESTART_HASH; 111 } 112 113 /* 114 * Copy the input digest if needed, and setup the context 115 * fields. Do this now as we need it to setup the first command 116 * descriptor. 117 */ 118 if (req->processed[0] || req->processed[1]) { 119 for (i = 0; i < digestsize / sizeof(u32); i++) 120 ctx->base.ctxr->data[i] = cpu_to_le32(req->state[i]); 121 122 if (req->finish) { 123 u64 count = req->processed[0] / EIP197_COUNTER_BLOCK_SIZE; 124 count += ((0xffffffff / EIP197_COUNTER_BLOCK_SIZE) * 125 req->processed[1]); 126 127 /* This is a haredware limitation, as the 128 * counter must fit into an u32. This represents 129 * a farily big amount of input data, so we 130 * shouldn't see this. 131 */ 132 if (unlikely(count & 0xffff0000)) { 133 dev_warn(priv->dev, 134 "Input data is too big\n"); 135 return; 136 } 137 138 ctx->base.ctxr->data[i] = cpu_to_le32(count); 139 } 140 } 141 } else if (req->digest == CONTEXT_CONTROL_DIGEST_HMAC) { 142 cdesc->control_data.control0 |= CONTEXT_CONTROL_SIZE(2 * req->state_sz / sizeof(u32)); 143 144 memcpy(ctx->base.ctxr->data, ctx->ipad, req->state_sz); 145 memcpy(ctx->base.ctxr->data + req->state_sz / sizeof(u32), 146 ctx->opad, req->state_sz); 147 } 148 } 149 150 static int safexcel_handle_req_result(struct safexcel_crypto_priv *priv, int ring, 151 struct crypto_async_request *async, 152 bool *should_complete, int *ret) 153 { 154 struct safexcel_result_desc *rdesc; 155 struct ahash_request *areq = ahash_request_cast(async); 156 struct crypto_ahash *ahash = crypto_ahash_reqtfm(areq); 157 struct safexcel_ahash_req *sreq = ahash_request_ctx(areq); 158 u64 cache_len; 159 160 *ret = 0; 161 162 rdesc = safexcel_ring_next_rptr(priv, &priv->ring[ring].rdr); 163 if (IS_ERR(rdesc)) { 164 dev_err(priv->dev, 165 "hash: result: could not retrieve the result descriptor\n"); 166 *ret = PTR_ERR(rdesc); 167 } else { 168 *ret = safexcel_rdesc_check_errors(priv, rdesc); 169 } 170 171 safexcel_complete(priv, ring); 172 173 if (sreq->nents) { 174 dma_unmap_sg(priv->dev, areq->src, sreq->nents, DMA_TO_DEVICE); 175 sreq->nents = 0; 176 } 177 178 if (sreq->result_dma) { 179 dma_unmap_single(priv->dev, sreq->result_dma, sreq->state_sz, 180 DMA_FROM_DEVICE); 181 sreq->result_dma = 0; 182 } 183 184 if (sreq->cache_dma) { 185 dma_unmap_single(priv->dev, sreq->cache_dma, sreq->cache_sz, 186 DMA_TO_DEVICE); 187 sreq->cache_dma = 0; 188 sreq->cache_sz = 0; 189 } 190 191 if (sreq->finish) 192 memcpy(areq->result, sreq->state, 193 crypto_ahash_digestsize(ahash)); 194 195 cache_len = safexcel_queued_len(sreq); 196 if (cache_len) 197 memcpy(sreq->cache, sreq->cache_next, cache_len); 198 199 *should_complete = true; 200 201 return 1; 202 } 203 204 static int safexcel_ahash_send_req(struct crypto_async_request *async, int ring, 205 int *commands, int *results) 206 { 207 struct ahash_request *areq = ahash_request_cast(async); 208 struct crypto_ahash *ahash = crypto_ahash_reqtfm(areq); 209 struct safexcel_ahash_req *req = ahash_request_ctx(areq); 210 struct safexcel_ahash_ctx *ctx = crypto_ahash_ctx(crypto_ahash_reqtfm(areq)); 211 struct safexcel_crypto_priv *priv = ctx->priv; 212 struct safexcel_command_desc *cdesc, *first_cdesc = NULL; 213 struct safexcel_result_desc *rdesc; 214 struct scatterlist *sg; 215 int i, extra = 0, n_cdesc = 0, ret = 0; 216 u64 queued, len, cache_len, cache_max; 217 218 cache_max = crypto_ahash_blocksize(ahash); 219 if (req->digest == CONTEXT_CONTROL_DIGEST_HMAC) 220 cache_max <<= 1; 221 222 queued = len = safexcel_queued_len(req); 223 if (queued <= cache_max) 224 cache_len = queued; 225 else 226 cache_len = queued - areq->nbytes; 227 228 if (!req->last_req) { 229 /* If this is not the last request and the queued data does not 230 * fit into full blocks, cache it for the next send() call. 231 */ 232 extra = queued & (crypto_ahash_blocksize(ahash) - 1); 233 234 if (req->digest == CONTEXT_CONTROL_DIGEST_HMAC && 235 extra < crypto_ahash_blocksize(ahash)) 236 extra += crypto_ahash_blocksize(ahash); 237 238 /* If this is not the last request and the queued data 239 * is a multiple of a block, cache the last one for now. 240 */ 241 if (!extra) 242 extra = crypto_ahash_blocksize(ahash); 243 244 sg_pcopy_to_buffer(areq->src, sg_nents(areq->src), 245 req->cache_next, extra, 246 areq->nbytes - extra); 247 248 queued -= extra; 249 len -= extra; 250 } 251 252 /* Add a command descriptor for the cached data, if any */ 253 if (cache_len) { 254 req->cache_dma = dma_map_single(priv->dev, req->cache, 255 cache_len, DMA_TO_DEVICE); 256 if (dma_mapping_error(priv->dev, req->cache_dma)) 257 return -EINVAL; 258 259 req->cache_sz = cache_len; 260 first_cdesc = safexcel_add_cdesc(priv, ring, 1, 261 (cache_len == len), 262 req->cache_dma, cache_len, len, 263 ctx->base.ctxr_dma); 264 if (IS_ERR(first_cdesc)) { 265 ret = PTR_ERR(first_cdesc); 266 goto unmap_cache; 267 } 268 n_cdesc++; 269 270 queued -= cache_len; 271 if (!queued) 272 goto send_command; 273 } 274 275 /* Now handle the current ahash request buffer(s) */ 276 req->nents = dma_map_sg(priv->dev, areq->src, sg_nents(areq->src), 277 DMA_TO_DEVICE); 278 if (!req->nents) { 279 ret = -ENOMEM; 280 goto cdesc_rollback; 281 } 282 283 for_each_sg(areq->src, sg, req->nents, i) { 284 int sglen = sg_dma_len(sg); 285 286 /* Do not overflow the request */ 287 if (queued < sglen) 288 sglen = queued; 289 290 cdesc = safexcel_add_cdesc(priv, ring, !n_cdesc, 291 !(queued - sglen), sg_dma_address(sg), 292 sglen, len, ctx->base.ctxr_dma); 293 if (IS_ERR(cdesc)) { 294 ret = PTR_ERR(cdesc); 295 goto unmap_sg; 296 } 297 n_cdesc++; 298 299 if (n_cdesc == 1) 300 first_cdesc = cdesc; 301 302 queued -= sglen; 303 if (!queued) 304 break; 305 } 306 307 send_command: 308 /* Setup the context options */ 309 safexcel_context_control(ctx, req, first_cdesc, req->state_sz); 310 311 /* Add the token */ 312 safexcel_hash_token(first_cdesc, len, req->state_sz); 313 314 req->result_dma = dma_map_single(priv->dev, req->state, req->state_sz, 315 DMA_FROM_DEVICE); 316 if (dma_mapping_error(priv->dev, req->result_dma)) { 317 ret = -EINVAL; 318 goto unmap_sg; 319 } 320 321 /* Add a result descriptor */ 322 rdesc = safexcel_add_rdesc(priv, ring, 1, 1, req->result_dma, 323 req->state_sz); 324 if (IS_ERR(rdesc)) { 325 ret = PTR_ERR(rdesc); 326 goto unmap_result; 327 } 328 329 safexcel_rdr_req_set(priv, ring, rdesc, &areq->base); 330 331 req->processed[0] += len; 332 if (req->processed[0] < len) 333 req->processed[1]++; 334 335 *commands = n_cdesc; 336 *results = 1; 337 return 0; 338 339 unmap_result: 340 dma_unmap_single(priv->dev, req->result_dma, req->state_sz, 341 DMA_FROM_DEVICE); 342 unmap_sg: 343 dma_unmap_sg(priv->dev, areq->src, req->nents, DMA_TO_DEVICE); 344 cdesc_rollback: 345 for (i = 0; i < n_cdesc; i++) 346 safexcel_ring_rollback_wptr(priv, &priv->ring[ring].cdr); 347 unmap_cache: 348 if (req->cache_dma) { 349 dma_unmap_single(priv->dev, req->cache_dma, req->cache_sz, 350 DMA_TO_DEVICE); 351 req->cache_dma = 0; 352 req->cache_sz = 0; 353 } 354 355 return ret; 356 } 357 358 static inline bool safexcel_ahash_needs_inv_get(struct ahash_request *areq) 359 { 360 struct safexcel_ahash_ctx *ctx = crypto_ahash_ctx(crypto_ahash_reqtfm(areq)); 361 struct safexcel_ahash_req *req = ahash_request_ctx(areq); 362 unsigned int state_w_sz = req->state_sz / sizeof(u32); 363 u64 processed; 364 int i; 365 366 processed = req->processed[0] / EIP197_COUNTER_BLOCK_SIZE; 367 processed += (0xffffffff / EIP197_COUNTER_BLOCK_SIZE) * req->processed[1]; 368 369 for (i = 0; i < state_w_sz; i++) 370 if (ctx->base.ctxr->data[i] != cpu_to_le32(req->state[i])) 371 return true; 372 373 if (ctx->base.ctxr->data[state_w_sz] != cpu_to_le32(processed)) 374 return true; 375 376 return false; 377 } 378 379 static int safexcel_handle_inv_result(struct safexcel_crypto_priv *priv, 380 int ring, 381 struct crypto_async_request *async, 382 bool *should_complete, int *ret) 383 { 384 struct safexcel_result_desc *rdesc; 385 struct ahash_request *areq = ahash_request_cast(async); 386 struct crypto_ahash *ahash = crypto_ahash_reqtfm(areq); 387 struct safexcel_ahash_ctx *ctx = crypto_ahash_ctx(ahash); 388 int enq_ret; 389 390 *ret = 0; 391 392 rdesc = safexcel_ring_next_rptr(priv, &priv->ring[ring].rdr); 393 if (IS_ERR(rdesc)) { 394 dev_err(priv->dev, 395 "hash: invalidate: could not retrieve the result descriptor\n"); 396 *ret = PTR_ERR(rdesc); 397 } else { 398 *ret = safexcel_rdesc_check_errors(priv, rdesc); 399 } 400 401 safexcel_complete(priv, ring); 402 403 if (ctx->base.exit_inv) { 404 dma_pool_free(priv->context_pool, ctx->base.ctxr, 405 ctx->base.ctxr_dma); 406 407 *should_complete = true; 408 return 1; 409 } 410 411 ring = safexcel_select_ring(priv); 412 ctx->base.ring = ring; 413 414 spin_lock_bh(&priv->ring[ring].queue_lock); 415 enq_ret = crypto_enqueue_request(&priv->ring[ring].queue, async); 416 spin_unlock_bh(&priv->ring[ring].queue_lock); 417 418 if (enq_ret != -EINPROGRESS) 419 *ret = enq_ret; 420 421 queue_work(priv->ring[ring].workqueue, 422 &priv->ring[ring].work_data.work); 423 424 *should_complete = false; 425 426 return 1; 427 } 428 429 static int safexcel_handle_result(struct safexcel_crypto_priv *priv, int ring, 430 struct crypto_async_request *async, 431 bool *should_complete, int *ret) 432 { 433 struct ahash_request *areq = ahash_request_cast(async); 434 struct safexcel_ahash_req *req = ahash_request_ctx(areq); 435 int err; 436 437 BUG_ON(!(priv->flags & EIP197_TRC_CACHE) && req->needs_inv); 438 439 if (req->needs_inv) { 440 req->needs_inv = false; 441 err = safexcel_handle_inv_result(priv, ring, async, 442 should_complete, ret); 443 } else { 444 err = safexcel_handle_req_result(priv, ring, async, 445 should_complete, ret); 446 } 447 448 return err; 449 } 450 451 static int safexcel_ahash_send_inv(struct crypto_async_request *async, 452 int ring, int *commands, int *results) 453 { 454 struct ahash_request *areq = ahash_request_cast(async); 455 struct safexcel_ahash_ctx *ctx = crypto_ahash_ctx(crypto_ahash_reqtfm(areq)); 456 int ret; 457 458 ret = safexcel_invalidate_cache(async, ctx->priv, 459 ctx->base.ctxr_dma, ring); 460 if (unlikely(ret)) 461 return ret; 462 463 *commands = 1; 464 *results = 1; 465 466 return 0; 467 } 468 469 static int safexcel_ahash_send(struct crypto_async_request *async, 470 int ring, int *commands, int *results) 471 { 472 struct ahash_request *areq = ahash_request_cast(async); 473 struct safexcel_ahash_req *req = ahash_request_ctx(areq); 474 int ret; 475 476 if (req->needs_inv) 477 ret = safexcel_ahash_send_inv(async, ring, commands, results); 478 else 479 ret = safexcel_ahash_send_req(async, ring, commands, results); 480 481 return ret; 482 } 483 484 static int safexcel_ahash_exit_inv(struct crypto_tfm *tfm) 485 { 486 struct safexcel_ahash_ctx *ctx = crypto_tfm_ctx(tfm); 487 struct safexcel_crypto_priv *priv = ctx->priv; 488 EIP197_REQUEST_ON_STACK(req, ahash, EIP197_AHASH_REQ_SIZE); 489 struct safexcel_ahash_req *rctx = ahash_request_ctx(req); 490 struct safexcel_inv_result result = {}; 491 int ring = ctx->base.ring; 492 493 memset(req, 0, EIP197_AHASH_REQ_SIZE); 494 495 /* create invalidation request */ 496 init_completion(&result.completion); 497 ahash_request_set_callback(req, CRYPTO_TFM_REQ_MAY_BACKLOG, 498 safexcel_inv_complete, &result); 499 500 ahash_request_set_tfm(req, __crypto_ahash_cast(tfm)); 501 ctx = crypto_tfm_ctx(req->base.tfm); 502 ctx->base.exit_inv = true; 503 rctx->needs_inv = true; 504 505 spin_lock_bh(&priv->ring[ring].queue_lock); 506 crypto_enqueue_request(&priv->ring[ring].queue, &req->base); 507 spin_unlock_bh(&priv->ring[ring].queue_lock); 508 509 queue_work(priv->ring[ring].workqueue, 510 &priv->ring[ring].work_data.work); 511 512 wait_for_completion(&result.completion); 513 514 if (result.error) { 515 dev_warn(priv->dev, "hash: completion error (%d)\n", 516 result.error); 517 return result.error; 518 } 519 520 return 0; 521 } 522 523 /* safexcel_ahash_cache: cache data until at least one request can be sent to 524 * the engine, aka. when there is at least 1 block size in the pipe. 525 */ 526 static int safexcel_ahash_cache(struct ahash_request *areq, u32 cache_max) 527 { 528 struct safexcel_ahash_req *req = ahash_request_ctx(areq); 529 u64 queued, cache_len; 530 531 /* queued: everything accepted by the driver which will be handled by 532 * the next send() calls. 533 * tot sz handled by update() - tot sz handled by send() 534 */ 535 queued = safexcel_queued_len(req); 536 /* cache_len: everything accepted by the driver but not sent yet, 537 * tot sz handled by update() - last req sz - tot sz handled by send() 538 */ 539 cache_len = queued - areq->nbytes; 540 541 /* 542 * In case there isn't enough bytes to proceed (less than a 543 * block size), cache the data until we have enough. 544 */ 545 if (cache_len + areq->nbytes <= cache_max) { 546 sg_pcopy_to_buffer(areq->src, sg_nents(areq->src), 547 req->cache + cache_len, 548 areq->nbytes, 0); 549 return areq->nbytes; 550 } 551 552 /* We couldn't cache all the data */ 553 return -E2BIG; 554 } 555 556 static int safexcel_ahash_enqueue(struct ahash_request *areq) 557 { 558 struct safexcel_ahash_ctx *ctx = crypto_ahash_ctx(crypto_ahash_reqtfm(areq)); 559 struct safexcel_ahash_req *req = ahash_request_ctx(areq); 560 struct safexcel_crypto_priv *priv = ctx->priv; 561 int ret, ring; 562 563 req->needs_inv = false; 564 565 if (ctx->base.ctxr) { 566 if (priv->flags & EIP197_TRC_CACHE && !ctx->base.needs_inv && 567 (req->processed[0] || req->processed[1]) && 568 req->digest == CONTEXT_CONTROL_DIGEST_PRECOMPUTED) 569 /* We're still setting needs_inv here, even though it is 570 * cleared right away, because the needs_inv flag can be 571 * set in other functions and we want to keep the same 572 * logic. 573 */ 574 ctx->base.needs_inv = safexcel_ahash_needs_inv_get(areq); 575 576 if (ctx->base.needs_inv) { 577 ctx->base.needs_inv = false; 578 req->needs_inv = true; 579 } 580 } else { 581 ctx->base.ring = safexcel_select_ring(priv); 582 ctx->base.ctxr = dma_pool_zalloc(priv->context_pool, 583 EIP197_GFP_FLAGS(areq->base), 584 &ctx->base.ctxr_dma); 585 if (!ctx->base.ctxr) 586 return -ENOMEM; 587 } 588 589 ring = ctx->base.ring; 590 591 spin_lock_bh(&priv->ring[ring].queue_lock); 592 ret = crypto_enqueue_request(&priv->ring[ring].queue, &areq->base); 593 spin_unlock_bh(&priv->ring[ring].queue_lock); 594 595 queue_work(priv->ring[ring].workqueue, 596 &priv->ring[ring].work_data.work); 597 598 return ret; 599 } 600 601 static int safexcel_ahash_update(struct ahash_request *areq) 602 { 603 struct safexcel_ahash_req *req = ahash_request_ctx(areq); 604 struct crypto_ahash *ahash = crypto_ahash_reqtfm(areq); 605 u32 cache_max; 606 607 /* If the request is 0 length, do nothing */ 608 if (!areq->nbytes) 609 return 0; 610 611 req->len[0] += areq->nbytes; 612 if (req->len[0] < areq->nbytes) 613 req->len[1]++; 614 615 cache_max = crypto_ahash_blocksize(ahash); 616 if (req->digest == CONTEXT_CONTROL_DIGEST_HMAC) 617 cache_max <<= 1; 618 619 safexcel_ahash_cache(areq, cache_max); 620 621 /* 622 * We're not doing partial updates when performing an hmac request. 623 * Everything will be handled by the final() call. 624 */ 625 if (req->digest == CONTEXT_CONTROL_DIGEST_HMAC) 626 return 0; 627 628 if (req->hmac) 629 return safexcel_ahash_enqueue(areq); 630 631 if (!req->last_req && 632 safexcel_queued_len(req) > cache_max) 633 return safexcel_ahash_enqueue(areq); 634 635 return 0; 636 } 637 638 static int safexcel_ahash_final(struct ahash_request *areq) 639 { 640 struct safexcel_ahash_req *req = ahash_request_ctx(areq); 641 struct safexcel_ahash_ctx *ctx = crypto_ahash_ctx(crypto_ahash_reqtfm(areq)); 642 643 req->last_req = true; 644 req->finish = true; 645 646 /* If we have an overall 0 length request */ 647 if (!req->len[0] && !req->len[1] && !areq->nbytes) { 648 if (ctx->alg == CONTEXT_CONTROL_CRYPTO_ALG_MD5) 649 memcpy(areq->result, md5_zero_message_hash, 650 MD5_DIGEST_SIZE); 651 else if (ctx->alg == CONTEXT_CONTROL_CRYPTO_ALG_SHA1) 652 memcpy(areq->result, sha1_zero_message_hash, 653 SHA1_DIGEST_SIZE); 654 else if (ctx->alg == CONTEXT_CONTROL_CRYPTO_ALG_SHA224) 655 memcpy(areq->result, sha224_zero_message_hash, 656 SHA224_DIGEST_SIZE); 657 else if (ctx->alg == CONTEXT_CONTROL_CRYPTO_ALG_SHA256) 658 memcpy(areq->result, sha256_zero_message_hash, 659 SHA256_DIGEST_SIZE); 660 else if (ctx->alg == CONTEXT_CONTROL_CRYPTO_ALG_SHA384) 661 memcpy(areq->result, sha384_zero_message_hash, 662 SHA384_DIGEST_SIZE); 663 else if (ctx->alg == CONTEXT_CONTROL_CRYPTO_ALG_SHA512) 664 memcpy(areq->result, sha512_zero_message_hash, 665 SHA512_DIGEST_SIZE); 666 667 return 0; 668 } 669 670 return safexcel_ahash_enqueue(areq); 671 } 672 673 static int safexcel_ahash_finup(struct ahash_request *areq) 674 { 675 struct safexcel_ahash_req *req = ahash_request_ctx(areq); 676 677 req->last_req = true; 678 req->finish = true; 679 680 safexcel_ahash_update(areq); 681 return safexcel_ahash_final(areq); 682 } 683 684 static int safexcel_ahash_export(struct ahash_request *areq, void *out) 685 { 686 struct crypto_ahash *ahash = crypto_ahash_reqtfm(areq); 687 struct safexcel_ahash_req *req = ahash_request_ctx(areq); 688 struct safexcel_ahash_export_state *export = out; 689 u32 cache_sz; 690 691 cache_sz = crypto_ahash_blocksize(ahash); 692 if (req->digest == CONTEXT_CONTROL_DIGEST_HMAC) 693 cache_sz <<= 1; 694 695 export->len[0] = req->len[0]; 696 export->len[1] = req->len[1]; 697 export->processed[0] = req->processed[0]; 698 export->processed[1] = req->processed[1]; 699 700 export->digest = req->digest; 701 702 memcpy(export->state, req->state, req->state_sz); 703 memcpy(export->cache, req->cache, cache_sz); 704 705 return 0; 706 } 707 708 static int safexcel_ahash_import(struct ahash_request *areq, const void *in) 709 { 710 struct crypto_ahash *ahash = crypto_ahash_reqtfm(areq); 711 struct safexcel_ahash_req *req = ahash_request_ctx(areq); 712 const struct safexcel_ahash_export_state *export = in; 713 u32 cache_sz; 714 int ret; 715 716 ret = crypto_ahash_init(areq); 717 if (ret) 718 return ret; 719 720 cache_sz = crypto_ahash_blocksize(ahash); 721 if (req->digest == CONTEXT_CONTROL_DIGEST_HMAC) 722 cache_sz <<= 1; 723 724 req->len[0] = export->len[0]; 725 req->len[1] = export->len[1]; 726 req->processed[0] = export->processed[0]; 727 req->processed[1] = export->processed[1]; 728 729 req->digest = export->digest; 730 731 memcpy(req->cache, export->cache, cache_sz); 732 memcpy(req->state, export->state, req->state_sz); 733 734 return 0; 735 } 736 737 static int safexcel_ahash_cra_init(struct crypto_tfm *tfm) 738 { 739 struct safexcel_ahash_ctx *ctx = crypto_tfm_ctx(tfm); 740 struct safexcel_alg_template *tmpl = 741 container_of(__crypto_ahash_alg(tfm->__crt_alg), 742 struct safexcel_alg_template, alg.ahash); 743 744 ctx->priv = tmpl->priv; 745 ctx->base.send = safexcel_ahash_send; 746 ctx->base.handle_result = safexcel_handle_result; 747 748 crypto_ahash_set_reqsize(__crypto_ahash_cast(tfm), 749 sizeof(struct safexcel_ahash_req)); 750 return 0; 751 } 752 753 static int safexcel_sha1_init(struct ahash_request *areq) 754 { 755 struct safexcel_ahash_ctx *ctx = crypto_ahash_ctx(crypto_ahash_reqtfm(areq)); 756 struct safexcel_ahash_req *req = ahash_request_ctx(areq); 757 758 memset(req, 0, sizeof(*req)); 759 760 req->state[0] = SHA1_H0; 761 req->state[1] = SHA1_H1; 762 req->state[2] = SHA1_H2; 763 req->state[3] = SHA1_H3; 764 req->state[4] = SHA1_H4; 765 766 ctx->alg = CONTEXT_CONTROL_CRYPTO_ALG_SHA1; 767 req->digest = CONTEXT_CONTROL_DIGEST_PRECOMPUTED; 768 req->state_sz = SHA1_DIGEST_SIZE; 769 770 return 0; 771 } 772 773 static int safexcel_sha1_digest(struct ahash_request *areq) 774 { 775 int ret = safexcel_sha1_init(areq); 776 777 if (ret) 778 return ret; 779 780 return safexcel_ahash_finup(areq); 781 } 782 783 static void safexcel_ahash_cra_exit(struct crypto_tfm *tfm) 784 { 785 struct safexcel_ahash_ctx *ctx = crypto_tfm_ctx(tfm); 786 struct safexcel_crypto_priv *priv = ctx->priv; 787 int ret; 788 789 /* context not allocated, skip invalidation */ 790 if (!ctx->base.ctxr) 791 return; 792 793 if (priv->flags & EIP197_TRC_CACHE) { 794 ret = safexcel_ahash_exit_inv(tfm); 795 if (ret) 796 dev_warn(priv->dev, "hash: invalidation error %d\n", ret); 797 } else { 798 dma_pool_free(priv->context_pool, ctx->base.ctxr, 799 ctx->base.ctxr_dma); 800 } 801 } 802 803 struct safexcel_alg_template safexcel_alg_sha1 = { 804 .type = SAFEXCEL_ALG_TYPE_AHASH, 805 .engines = EIP97IES | EIP197B | EIP197D, 806 .alg.ahash = { 807 .init = safexcel_sha1_init, 808 .update = safexcel_ahash_update, 809 .final = safexcel_ahash_final, 810 .finup = safexcel_ahash_finup, 811 .digest = safexcel_sha1_digest, 812 .export = safexcel_ahash_export, 813 .import = safexcel_ahash_import, 814 .halg = { 815 .digestsize = SHA1_DIGEST_SIZE, 816 .statesize = sizeof(struct safexcel_ahash_export_state), 817 .base = { 818 .cra_name = "sha1", 819 .cra_driver_name = "safexcel-sha1", 820 .cra_priority = 300, 821 .cra_flags = CRYPTO_ALG_ASYNC | 822 CRYPTO_ALG_KERN_DRIVER_ONLY, 823 .cra_blocksize = SHA1_BLOCK_SIZE, 824 .cra_ctxsize = sizeof(struct safexcel_ahash_ctx), 825 .cra_init = safexcel_ahash_cra_init, 826 .cra_exit = safexcel_ahash_cra_exit, 827 .cra_module = THIS_MODULE, 828 }, 829 }, 830 }, 831 }; 832 833 static int safexcel_hmac_sha1_init(struct ahash_request *areq) 834 { 835 struct safexcel_ahash_req *req = ahash_request_ctx(areq); 836 837 safexcel_sha1_init(areq); 838 req->digest = CONTEXT_CONTROL_DIGEST_HMAC; 839 return 0; 840 } 841 842 static int safexcel_hmac_sha1_digest(struct ahash_request *areq) 843 { 844 int ret = safexcel_hmac_sha1_init(areq); 845 846 if (ret) 847 return ret; 848 849 return safexcel_ahash_finup(areq); 850 } 851 852 struct safexcel_ahash_result { 853 struct completion completion; 854 int error; 855 }; 856 857 static void safexcel_ahash_complete(struct crypto_async_request *req, int error) 858 { 859 struct safexcel_ahash_result *result = req->data; 860 861 if (error == -EINPROGRESS) 862 return; 863 864 result->error = error; 865 complete(&result->completion); 866 } 867 868 static int safexcel_hmac_init_pad(struct ahash_request *areq, 869 unsigned int blocksize, const u8 *key, 870 unsigned int keylen, u8 *ipad, u8 *opad) 871 { 872 struct safexcel_ahash_result result; 873 struct scatterlist sg; 874 int ret, i; 875 u8 *keydup; 876 877 if (keylen <= blocksize) { 878 memcpy(ipad, key, keylen); 879 } else { 880 keydup = kmemdup(key, keylen, GFP_KERNEL); 881 if (!keydup) 882 return -ENOMEM; 883 884 ahash_request_set_callback(areq, CRYPTO_TFM_REQ_MAY_BACKLOG, 885 safexcel_ahash_complete, &result); 886 sg_init_one(&sg, keydup, keylen); 887 ahash_request_set_crypt(areq, &sg, ipad, keylen); 888 init_completion(&result.completion); 889 890 ret = crypto_ahash_digest(areq); 891 if (ret == -EINPROGRESS || ret == -EBUSY) { 892 wait_for_completion_interruptible(&result.completion); 893 ret = result.error; 894 } 895 896 /* Avoid leaking */ 897 memzero_explicit(keydup, keylen); 898 kfree(keydup); 899 900 if (ret) 901 return ret; 902 903 keylen = crypto_ahash_digestsize(crypto_ahash_reqtfm(areq)); 904 } 905 906 memset(ipad + keylen, 0, blocksize - keylen); 907 memcpy(opad, ipad, blocksize); 908 909 for (i = 0; i < blocksize; i++) { 910 ipad[i] ^= HMAC_IPAD_VALUE; 911 opad[i] ^= HMAC_OPAD_VALUE; 912 } 913 914 return 0; 915 } 916 917 static int safexcel_hmac_init_iv(struct ahash_request *areq, 918 unsigned int blocksize, u8 *pad, void *state) 919 { 920 struct safexcel_ahash_result result; 921 struct safexcel_ahash_req *req; 922 struct scatterlist sg; 923 int ret; 924 925 ahash_request_set_callback(areq, CRYPTO_TFM_REQ_MAY_BACKLOG, 926 safexcel_ahash_complete, &result); 927 sg_init_one(&sg, pad, blocksize); 928 ahash_request_set_crypt(areq, &sg, pad, blocksize); 929 init_completion(&result.completion); 930 931 ret = crypto_ahash_init(areq); 932 if (ret) 933 return ret; 934 935 req = ahash_request_ctx(areq); 936 req->hmac = true; 937 req->last_req = true; 938 939 ret = crypto_ahash_update(areq); 940 if (ret && ret != -EINPROGRESS && ret != -EBUSY) 941 return ret; 942 943 wait_for_completion_interruptible(&result.completion); 944 if (result.error) 945 return result.error; 946 947 return crypto_ahash_export(areq, state); 948 } 949 950 int safexcel_hmac_setkey(const char *alg, const u8 *key, unsigned int keylen, 951 void *istate, void *ostate) 952 { 953 struct ahash_request *areq; 954 struct crypto_ahash *tfm; 955 unsigned int blocksize; 956 u8 *ipad, *opad; 957 int ret; 958 959 tfm = crypto_alloc_ahash(alg, 0, 0); 960 if (IS_ERR(tfm)) 961 return PTR_ERR(tfm); 962 963 areq = ahash_request_alloc(tfm, GFP_KERNEL); 964 if (!areq) { 965 ret = -ENOMEM; 966 goto free_ahash; 967 } 968 969 crypto_ahash_clear_flags(tfm, ~0); 970 blocksize = crypto_tfm_alg_blocksize(crypto_ahash_tfm(tfm)); 971 972 ipad = kcalloc(2, blocksize, GFP_KERNEL); 973 if (!ipad) { 974 ret = -ENOMEM; 975 goto free_request; 976 } 977 978 opad = ipad + blocksize; 979 980 ret = safexcel_hmac_init_pad(areq, blocksize, key, keylen, ipad, opad); 981 if (ret) 982 goto free_ipad; 983 984 ret = safexcel_hmac_init_iv(areq, blocksize, ipad, istate); 985 if (ret) 986 goto free_ipad; 987 988 ret = safexcel_hmac_init_iv(areq, blocksize, opad, ostate); 989 990 free_ipad: 991 kfree(ipad); 992 free_request: 993 ahash_request_free(areq); 994 free_ahash: 995 crypto_free_ahash(tfm); 996 997 return ret; 998 } 999 1000 static int safexcel_hmac_alg_setkey(struct crypto_ahash *tfm, const u8 *key, 1001 unsigned int keylen, const char *alg, 1002 unsigned int state_sz) 1003 { 1004 struct safexcel_ahash_ctx *ctx = crypto_tfm_ctx(crypto_ahash_tfm(tfm)); 1005 struct safexcel_crypto_priv *priv = ctx->priv; 1006 struct safexcel_ahash_export_state istate, ostate; 1007 int ret, i; 1008 1009 ret = safexcel_hmac_setkey(alg, key, keylen, &istate, &ostate); 1010 if (ret) 1011 return ret; 1012 1013 if (priv->flags & EIP197_TRC_CACHE && ctx->base.ctxr) { 1014 for (i = 0; i < state_sz / sizeof(u32); i++) { 1015 if (ctx->ipad[i] != le32_to_cpu(istate.state[i]) || 1016 ctx->opad[i] != le32_to_cpu(ostate.state[i])) { 1017 ctx->base.needs_inv = true; 1018 break; 1019 } 1020 } 1021 } 1022 1023 memcpy(ctx->ipad, &istate.state, state_sz); 1024 memcpy(ctx->opad, &ostate.state, state_sz); 1025 1026 return 0; 1027 } 1028 1029 static int safexcel_hmac_sha1_setkey(struct crypto_ahash *tfm, const u8 *key, 1030 unsigned int keylen) 1031 { 1032 return safexcel_hmac_alg_setkey(tfm, key, keylen, "safexcel-sha1", 1033 SHA1_DIGEST_SIZE); 1034 } 1035 1036 struct safexcel_alg_template safexcel_alg_hmac_sha1 = { 1037 .type = SAFEXCEL_ALG_TYPE_AHASH, 1038 .engines = EIP97IES | EIP197B | EIP197D, 1039 .alg.ahash = { 1040 .init = safexcel_hmac_sha1_init, 1041 .update = safexcel_ahash_update, 1042 .final = safexcel_ahash_final, 1043 .finup = safexcel_ahash_finup, 1044 .digest = safexcel_hmac_sha1_digest, 1045 .setkey = safexcel_hmac_sha1_setkey, 1046 .export = safexcel_ahash_export, 1047 .import = safexcel_ahash_import, 1048 .halg = { 1049 .digestsize = SHA1_DIGEST_SIZE, 1050 .statesize = sizeof(struct safexcel_ahash_export_state), 1051 .base = { 1052 .cra_name = "hmac(sha1)", 1053 .cra_driver_name = "safexcel-hmac-sha1", 1054 .cra_priority = 300, 1055 .cra_flags = CRYPTO_ALG_ASYNC | 1056 CRYPTO_ALG_KERN_DRIVER_ONLY, 1057 .cra_blocksize = SHA1_BLOCK_SIZE, 1058 .cra_ctxsize = sizeof(struct safexcel_ahash_ctx), 1059 .cra_init = safexcel_ahash_cra_init, 1060 .cra_exit = safexcel_ahash_cra_exit, 1061 .cra_module = THIS_MODULE, 1062 }, 1063 }, 1064 }, 1065 }; 1066 1067 static int safexcel_sha256_init(struct ahash_request *areq) 1068 { 1069 struct safexcel_ahash_ctx *ctx = crypto_ahash_ctx(crypto_ahash_reqtfm(areq)); 1070 struct safexcel_ahash_req *req = ahash_request_ctx(areq); 1071 1072 memset(req, 0, sizeof(*req)); 1073 1074 req->state[0] = SHA256_H0; 1075 req->state[1] = SHA256_H1; 1076 req->state[2] = SHA256_H2; 1077 req->state[3] = SHA256_H3; 1078 req->state[4] = SHA256_H4; 1079 req->state[5] = SHA256_H5; 1080 req->state[6] = SHA256_H6; 1081 req->state[7] = SHA256_H7; 1082 1083 ctx->alg = CONTEXT_CONTROL_CRYPTO_ALG_SHA256; 1084 req->digest = CONTEXT_CONTROL_DIGEST_PRECOMPUTED; 1085 req->state_sz = SHA256_DIGEST_SIZE; 1086 1087 return 0; 1088 } 1089 1090 static int safexcel_sha256_digest(struct ahash_request *areq) 1091 { 1092 int ret = safexcel_sha256_init(areq); 1093 1094 if (ret) 1095 return ret; 1096 1097 return safexcel_ahash_finup(areq); 1098 } 1099 1100 struct safexcel_alg_template safexcel_alg_sha256 = { 1101 .type = SAFEXCEL_ALG_TYPE_AHASH, 1102 .engines = EIP97IES | EIP197B | EIP197D, 1103 .alg.ahash = { 1104 .init = safexcel_sha256_init, 1105 .update = safexcel_ahash_update, 1106 .final = safexcel_ahash_final, 1107 .finup = safexcel_ahash_finup, 1108 .digest = safexcel_sha256_digest, 1109 .export = safexcel_ahash_export, 1110 .import = safexcel_ahash_import, 1111 .halg = { 1112 .digestsize = SHA256_DIGEST_SIZE, 1113 .statesize = sizeof(struct safexcel_ahash_export_state), 1114 .base = { 1115 .cra_name = "sha256", 1116 .cra_driver_name = "safexcel-sha256", 1117 .cra_priority = 300, 1118 .cra_flags = CRYPTO_ALG_ASYNC | 1119 CRYPTO_ALG_KERN_DRIVER_ONLY, 1120 .cra_blocksize = SHA256_BLOCK_SIZE, 1121 .cra_ctxsize = sizeof(struct safexcel_ahash_ctx), 1122 .cra_init = safexcel_ahash_cra_init, 1123 .cra_exit = safexcel_ahash_cra_exit, 1124 .cra_module = THIS_MODULE, 1125 }, 1126 }, 1127 }, 1128 }; 1129 1130 static int safexcel_sha224_init(struct ahash_request *areq) 1131 { 1132 struct safexcel_ahash_ctx *ctx = crypto_ahash_ctx(crypto_ahash_reqtfm(areq)); 1133 struct safexcel_ahash_req *req = ahash_request_ctx(areq); 1134 1135 memset(req, 0, sizeof(*req)); 1136 1137 req->state[0] = SHA224_H0; 1138 req->state[1] = SHA224_H1; 1139 req->state[2] = SHA224_H2; 1140 req->state[3] = SHA224_H3; 1141 req->state[4] = SHA224_H4; 1142 req->state[5] = SHA224_H5; 1143 req->state[6] = SHA224_H6; 1144 req->state[7] = SHA224_H7; 1145 1146 ctx->alg = CONTEXT_CONTROL_CRYPTO_ALG_SHA224; 1147 req->digest = CONTEXT_CONTROL_DIGEST_PRECOMPUTED; 1148 req->state_sz = SHA256_DIGEST_SIZE; 1149 1150 return 0; 1151 } 1152 1153 static int safexcel_sha224_digest(struct ahash_request *areq) 1154 { 1155 int ret = safexcel_sha224_init(areq); 1156 1157 if (ret) 1158 return ret; 1159 1160 return safexcel_ahash_finup(areq); 1161 } 1162 1163 struct safexcel_alg_template safexcel_alg_sha224 = { 1164 .type = SAFEXCEL_ALG_TYPE_AHASH, 1165 .engines = EIP97IES | EIP197B | EIP197D, 1166 .alg.ahash = { 1167 .init = safexcel_sha224_init, 1168 .update = safexcel_ahash_update, 1169 .final = safexcel_ahash_final, 1170 .finup = safexcel_ahash_finup, 1171 .digest = safexcel_sha224_digest, 1172 .export = safexcel_ahash_export, 1173 .import = safexcel_ahash_import, 1174 .halg = { 1175 .digestsize = SHA224_DIGEST_SIZE, 1176 .statesize = sizeof(struct safexcel_ahash_export_state), 1177 .base = { 1178 .cra_name = "sha224", 1179 .cra_driver_name = "safexcel-sha224", 1180 .cra_priority = 300, 1181 .cra_flags = CRYPTO_ALG_ASYNC | 1182 CRYPTO_ALG_KERN_DRIVER_ONLY, 1183 .cra_blocksize = SHA224_BLOCK_SIZE, 1184 .cra_ctxsize = sizeof(struct safexcel_ahash_ctx), 1185 .cra_init = safexcel_ahash_cra_init, 1186 .cra_exit = safexcel_ahash_cra_exit, 1187 .cra_module = THIS_MODULE, 1188 }, 1189 }, 1190 }, 1191 }; 1192 1193 static int safexcel_hmac_sha224_setkey(struct crypto_ahash *tfm, const u8 *key, 1194 unsigned int keylen) 1195 { 1196 return safexcel_hmac_alg_setkey(tfm, key, keylen, "safexcel-sha224", 1197 SHA256_DIGEST_SIZE); 1198 } 1199 1200 static int safexcel_hmac_sha224_init(struct ahash_request *areq) 1201 { 1202 struct safexcel_ahash_req *req = ahash_request_ctx(areq); 1203 1204 safexcel_sha224_init(areq); 1205 req->digest = CONTEXT_CONTROL_DIGEST_HMAC; 1206 return 0; 1207 } 1208 1209 static int safexcel_hmac_sha224_digest(struct ahash_request *areq) 1210 { 1211 int ret = safexcel_hmac_sha224_init(areq); 1212 1213 if (ret) 1214 return ret; 1215 1216 return safexcel_ahash_finup(areq); 1217 } 1218 1219 struct safexcel_alg_template safexcel_alg_hmac_sha224 = { 1220 .type = SAFEXCEL_ALG_TYPE_AHASH, 1221 .engines = EIP97IES | EIP197B | EIP197D, 1222 .alg.ahash = { 1223 .init = safexcel_hmac_sha224_init, 1224 .update = safexcel_ahash_update, 1225 .final = safexcel_ahash_final, 1226 .finup = safexcel_ahash_finup, 1227 .digest = safexcel_hmac_sha224_digest, 1228 .setkey = safexcel_hmac_sha224_setkey, 1229 .export = safexcel_ahash_export, 1230 .import = safexcel_ahash_import, 1231 .halg = { 1232 .digestsize = SHA224_DIGEST_SIZE, 1233 .statesize = sizeof(struct safexcel_ahash_export_state), 1234 .base = { 1235 .cra_name = "hmac(sha224)", 1236 .cra_driver_name = "safexcel-hmac-sha224", 1237 .cra_priority = 300, 1238 .cra_flags = CRYPTO_ALG_ASYNC | 1239 CRYPTO_ALG_KERN_DRIVER_ONLY, 1240 .cra_blocksize = SHA224_BLOCK_SIZE, 1241 .cra_ctxsize = sizeof(struct safexcel_ahash_ctx), 1242 .cra_init = safexcel_ahash_cra_init, 1243 .cra_exit = safexcel_ahash_cra_exit, 1244 .cra_module = THIS_MODULE, 1245 }, 1246 }, 1247 }, 1248 }; 1249 1250 static int safexcel_hmac_sha256_setkey(struct crypto_ahash *tfm, const u8 *key, 1251 unsigned int keylen) 1252 { 1253 return safexcel_hmac_alg_setkey(tfm, key, keylen, "safexcel-sha256", 1254 SHA256_DIGEST_SIZE); 1255 } 1256 1257 static int safexcel_hmac_sha256_init(struct ahash_request *areq) 1258 { 1259 struct safexcel_ahash_req *req = ahash_request_ctx(areq); 1260 1261 safexcel_sha256_init(areq); 1262 req->digest = CONTEXT_CONTROL_DIGEST_HMAC; 1263 return 0; 1264 } 1265 1266 static int safexcel_hmac_sha256_digest(struct ahash_request *areq) 1267 { 1268 int ret = safexcel_hmac_sha256_init(areq); 1269 1270 if (ret) 1271 return ret; 1272 1273 return safexcel_ahash_finup(areq); 1274 } 1275 1276 struct safexcel_alg_template safexcel_alg_hmac_sha256 = { 1277 .type = SAFEXCEL_ALG_TYPE_AHASH, 1278 .engines = EIP97IES | EIP197B | EIP197D, 1279 .alg.ahash = { 1280 .init = safexcel_hmac_sha256_init, 1281 .update = safexcel_ahash_update, 1282 .final = safexcel_ahash_final, 1283 .finup = safexcel_ahash_finup, 1284 .digest = safexcel_hmac_sha256_digest, 1285 .setkey = safexcel_hmac_sha256_setkey, 1286 .export = safexcel_ahash_export, 1287 .import = safexcel_ahash_import, 1288 .halg = { 1289 .digestsize = SHA256_DIGEST_SIZE, 1290 .statesize = sizeof(struct safexcel_ahash_export_state), 1291 .base = { 1292 .cra_name = "hmac(sha256)", 1293 .cra_driver_name = "safexcel-hmac-sha256", 1294 .cra_priority = 300, 1295 .cra_flags = CRYPTO_ALG_ASYNC | 1296 CRYPTO_ALG_KERN_DRIVER_ONLY, 1297 .cra_blocksize = SHA256_BLOCK_SIZE, 1298 .cra_ctxsize = sizeof(struct safexcel_ahash_ctx), 1299 .cra_init = safexcel_ahash_cra_init, 1300 .cra_exit = safexcel_ahash_cra_exit, 1301 .cra_module = THIS_MODULE, 1302 }, 1303 }, 1304 }, 1305 }; 1306 1307 static int safexcel_sha512_init(struct ahash_request *areq) 1308 { 1309 struct safexcel_ahash_ctx *ctx = crypto_ahash_ctx(crypto_ahash_reqtfm(areq)); 1310 struct safexcel_ahash_req *req = ahash_request_ctx(areq); 1311 1312 memset(req, 0, sizeof(*req)); 1313 1314 req->state[0] = lower_32_bits(SHA512_H0); 1315 req->state[1] = upper_32_bits(SHA512_H0); 1316 req->state[2] = lower_32_bits(SHA512_H1); 1317 req->state[3] = upper_32_bits(SHA512_H1); 1318 req->state[4] = lower_32_bits(SHA512_H2); 1319 req->state[5] = upper_32_bits(SHA512_H2); 1320 req->state[6] = lower_32_bits(SHA512_H3); 1321 req->state[7] = upper_32_bits(SHA512_H3); 1322 req->state[8] = lower_32_bits(SHA512_H4); 1323 req->state[9] = upper_32_bits(SHA512_H4); 1324 req->state[10] = lower_32_bits(SHA512_H5); 1325 req->state[11] = upper_32_bits(SHA512_H5); 1326 req->state[12] = lower_32_bits(SHA512_H6); 1327 req->state[13] = upper_32_bits(SHA512_H6); 1328 req->state[14] = lower_32_bits(SHA512_H7); 1329 req->state[15] = upper_32_bits(SHA512_H7); 1330 1331 ctx->alg = CONTEXT_CONTROL_CRYPTO_ALG_SHA512; 1332 req->digest = CONTEXT_CONTROL_DIGEST_PRECOMPUTED; 1333 req->state_sz = SHA512_DIGEST_SIZE; 1334 1335 return 0; 1336 } 1337 1338 static int safexcel_sha512_digest(struct ahash_request *areq) 1339 { 1340 int ret = safexcel_sha512_init(areq); 1341 1342 if (ret) 1343 return ret; 1344 1345 return safexcel_ahash_finup(areq); 1346 } 1347 1348 struct safexcel_alg_template safexcel_alg_sha512 = { 1349 .type = SAFEXCEL_ALG_TYPE_AHASH, 1350 .engines = EIP97IES | EIP197B | EIP197D, 1351 .alg.ahash = { 1352 .init = safexcel_sha512_init, 1353 .update = safexcel_ahash_update, 1354 .final = safexcel_ahash_final, 1355 .finup = safexcel_ahash_finup, 1356 .digest = safexcel_sha512_digest, 1357 .export = safexcel_ahash_export, 1358 .import = safexcel_ahash_import, 1359 .halg = { 1360 .digestsize = SHA512_DIGEST_SIZE, 1361 .statesize = sizeof(struct safexcel_ahash_export_state), 1362 .base = { 1363 .cra_name = "sha512", 1364 .cra_driver_name = "safexcel-sha512", 1365 .cra_priority = 300, 1366 .cra_flags = CRYPTO_ALG_ASYNC | 1367 CRYPTO_ALG_KERN_DRIVER_ONLY, 1368 .cra_blocksize = SHA512_BLOCK_SIZE, 1369 .cra_ctxsize = sizeof(struct safexcel_ahash_ctx), 1370 .cra_init = safexcel_ahash_cra_init, 1371 .cra_exit = safexcel_ahash_cra_exit, 1372 .cra_module = THIS_MODULE, 1373 }, 1374 }, 1375 }, 1376 }; 1377 1378 static int safexcel_sha384_init(struct ahash_request *areq) 1379 { 1380 struct safexcel_ahash_ctx *ctx = crypto_ahash_ctx(crypto_ahash_reqtfm(areq)); 1381 struct safexcel_ahash_req *req = ahash_request_ctx(areq); 1382 1383 memset(req, 0, sizeof(*req)); 1384 1385 req->state[0] = lower_32_bits(SHA384_H0); 1386 req->state[1] = upper_32_bits(SHA384_H0); 1387 req->state[2] = lower_32_bits(SHA384_H1); 1388 req->state[3] = upper_32_bits(SHA384_H1); 1389 req->state[4] = lower_32_bits(SHA384_H2); 1390 req->state[5] = upper_32_bits(SHA384_H2); 1391 req->state[6] = lower_32_bits(SHA384_H3); 1392 req->state[7] = upper_32_bits(SHA384_H3); 1393 req->state[8] = lower_32_bits(SHA384_H4); 1394 req->state[9] = upper_32_bits(SHA384_H4); 1395 req->state[10] = lower_32_bits(SHA384_H5); 1396 req->state[11] = upper_32_bits(SHA384_H5); 1397 req->state[12] = lower_32_bits(SHA384_H6); 1398 req->state[13] = upper_32_bits(SHA384_H6); 1399 req->state[14] = lower_32_bits(SHA384_H7); 1400 req->state[15] = upper_32_bits(SHA384_H7); 1401 1402 ctx->alg = CONTEXT_CONTROL_CRYPTO_ALG_SHA384; 1403 req->digest = CONTEXT_CONTROL_DIGEST_PRECOMPUTED; 1404 req->state_sz = SHA512_DIGEST_SIZE; 1405 1406 return 0; 1407 } 1408 1409 static int safexcel_sha384_digest(struct ahash_request *areq) 1410 { 1411 int ret = safexcel_sha384_init(areq); 1412 1413 if (ret) 1414 return ret; 1415 1416 return safexcel_ahash_finup(areq); 1417 } 1418 1419 struct safexcel_alg_template safexcel_alg_sha384 = { 1420 .type = SAFEXCEL_ALG_TYPE_AHASH, 1421 .engines = EIP97IES | EIP197B | EIP197D, 1422 .alg.ahash = { 1423 .init = safexcel_sha384_init, 1424 .update = safexcel_ahash_update, 1425 .final = safexcel_ahash_final, 1426 .finup = safexcel_ahash_finup, 1427 .digest = safexcel_sha384_digest, 1428 .export = safexcel_ahash_export, 1429 .import = safexcel_ahash_import, 1430 .halg = { 1431 .digestsize = SHA384_DIGEST_SIZE, 1432 .statesize = sizeof(struct safexcel_ahash_export_state), 1433 .base = { 1434 .cra_name = "sha384", 1435 .cra_driver_name = "safexcel-sha384", 1436 .cra_priority = 300, 1437 .cra_flags = CRYPTO_ALG_ASYNC | 1438 CRYPTO_ALG_KERN_DRIVER_ONLY, 1439 .cra_blocksize = SHA384_BLOCK_SIZE, 1440 .cra_ctxsize = sizeof(struct safexcel_ahash_ctx), 1441 .cra_init = safexcel_ahash_cra_init, 1442 .cra_exit = safexcel_ahash_cra_exit, 1443 .cra_module = THIS_MODULE, 1444 }, 1445 }, 1446 }, 1447 }; 1448 1449 static int safexcel_hmac_sha512_setkey(struct crypto_ahash *tfm, const u8 *key, 1450 unsigned int keylen) 1451 { 1452 return safexcel_hmac_alg_setkey(tfm, key, keylen, "safexcel-sha512", 1453 SHA512_DIGEST_SIZE); 1454 } 1455 1456 static int safexcel_hmac_sha512_init(struct ahash_request *areq) 1457 { 1458 struct safexcel_ahash_req *req = ahash_request_ctx(areq); 1459 1460 safexcel_sha512_init(areq); 1461 req->digest = CONTEXT_CONTROL_DIGEST_HMAC; 1462 return 0; 1463 } 1464 1465 static int safexcel_hmac_sha512_digest(struct ahash_request *areq) 1466 { 1467 int ret = safexcel_hmac_sha512_init(areq); 1468 1469 if (ret) 1470 return ret; 1471 1472 return safexcel_ahash_finup(areq); 1473 } 1474 1475 struct safexcel_alg_template safexcel_alg_hmac_sha512 = { 1476 .type = SAFEXCEL_ALG_TYPE_AHASH, 1477 .engines = EIP97IES | EIP197B | EIP197D, 1478 .alg.ahash = { 1479 .init = safexcel_hmac_sha512_init, 1480 .update = safexcel_ahash_update, 1481 .final = safexcel_ahash_final, 1482 .finup = safexcel_ahash_finup, 1483 .digest = safexcel_hmac_sha512_digest, 1484 .setkey = safexcel_hmac_sha512_setkey, 1485 .export = safexcel_ahash_export, 1486 .import = safexcel_ahash_import, 1487 .halg = { 1488 .digestsize = SHA512_DIGEST_SIZE, 1489 .statesize = sizeof(struct safexcel_ahash_export_state), 1490 .base = { 1491 .cra_name = "hmac(sha512)", 1492 .cra_driver_name = "safexcel-hmac-sha512", 1493 .cra_priority = 300, 1494 .cra_flags = CRYPTO_ALG_ASYNC | 1495 CRYPTO_ALG_KERN_DRIVER_ONLY, 1496 .cra_blocksize = SHA512_BLOCK_SIZE, 1497 .cra_ctxsize = sizeof(struct safexcel_ahash_ctx), 1498 .cra_init = safexcel_ahash_cra_init, 1499 .cra_exit = safexcel_ahash_cra_exit, 1500 .cra_module = THIS_MODULE, 1501 }, 1502 }, 1503 }, 1504 }; 1505 1506 static int safexcel_hmac_sha384_setkey(struct crypto_ahash *tfm, const u8 *key, 1507 unsigned int keylen) 1508 { 1509 return safexcel_hmac_alg_setkey(tfm, key, keylen, "safexcel-sha384", 1510 SHA512_DIGEST_SIZE); 1511 } 1512 1513 static int safexcel_hmac_sha384_init(struct ahash_request *areq) 1514 { 1515 struct safexcel_ahash_req *req = ahash_request_ctx(areq); 1516 1517 safexcel_sha384_init(areq); 1518 req->digest = CONTEXT_CONTROL_DIGEST_HMAC; 1519 return 0; 1520 } 1521 1522 static int safexcel_hmac_sha384_digest(struct ahash_request *areq) 1523 { 1524 int ret = safexcel_hmac_sha384_init(areq); 1525 1526 if (ret) 1527 return ret; 1528 1529 return safexcel_ahash_finup(areq); 1530 } 1531 1532 struct safexcel_alg_template safexcel_alg_hmac_sha384 = { 1533 .type = SAFEXCEL_ALG_TYPE_AHASH, 1534 .engines = EIP97IES | EIP197B | EIP197D, 1535 .alg.ahash = { 1536 .init = safexcel_hmac_sha384_init, 1537 .update = safexcel_ahash_update, 1538 .final = safexcel_ahash_final, 1539 .finup = safexcel_ahash_finup, 1540 .digest = safexcel_hmac_sha384_digest, 1541 .setkey = safexcel_hmac_sha384_setkey, 1542 .export = safexcel_ahash_export, 1543 .import = safexcel_ahash_import, 1544 .halg = { 1545 .digestsize = SHA384_DIGEST_SIZE, 1546 .statesize = sizeof(struct safexcel_ahash_export_state), 1547 .base = { 1548 .cra_name = "hmac(sha384)", 1549 .cra_driver_name = "safexcel-hmac-sha384", 1550 .cra_priority = 300, 1551 .cra_flags = CRYPTO_ALG_ASYNC | 1552 CRYPTO_ALG_KERN_DRIVER_ONLY, 1553 .cra_blocksize = SHA384_BLOCK_SIZE, 1554 .cra_ctxsize = sizeof(struct safexcel_ahash_ctx), 1555 .cra_init = safexcel_ahash_cra_init, 1556 .cra_exit = safexcel_ahash_cra_exit, 1557 .cra_module = THIS_MODULE, 1558 }, 1559 }, 1560 }, 1561 }; 1562 1563 static int safexcel_md5_init(struct ahash_request *areq) 1564 { 1565 struct safexcel_ahash_ctx *ctx = crypto_ahash_ctx(crypto_ahash_reqtfm(areq)); 1566 struct safexcel_ahash_req *req = ahash_request_ctx(areq); 1567 1568 memset(req, 0, sizeof(*req)); 1569 1570 req->state[0] = MD5_H0; 1571 req->state[1] = MD5_H1; 1572 req->state[2] = MD5_H2; 1573 req->state[3] = MD5_H3; 1574 1575 ctx->alg = CONTEXT_CONTROL_CRYPTO_ALG_MD5; 1576 req->digest = CONTEXT_CONTROL_DIGEST_PRECOMPUTED; 1577 req->state_sz = MD5_DIGEST_SIZE; 1578 1579 return 0; 1580 } 1581 1582 static int safexcel_md5_digest(struct ahash_request *areq) 1583 { 1584 int ret = safexcel_md5_init(areq); 1585 1586 if (ret) 1587 return ret; 1588 1589 return safexcel_ahash_finup(areq); 1590 } 1591 1592 struct safexcel_alg_template safexcel_alg_md5 = { 1593 .type = SAFEXCEL_ALG_TYPE_AHASH, 1594 .engines = EIP97IES | EIP197B | EIP197D, 1595 .alg.ahash = { 1596 .init = safexcel_md5_init, 1597 .update = safexcel_ahash_update, 1598 .final = safexcel_ahash_final, 1599 .finup = safexcel_ahash_finup, 1600 .digest = safexcel_md5_digest, 1601 .export = safexcel_ahash_export, 1602 .import = safexcel_ahash_import, 1603 .halg = { 1604 .digestsize = MD5_DIGEST_SIZE, 1605 .statesize = sizeof(struct safexcel_ahash_export_state), 1606 .base = { 1607 .cra_name = "md5", 1608 .cra_driver_name = "safexcel-md5", 1609 .cra_priority = 300, 1610 .cra_flags = CRYPTO_ALG_ASYNC | 1611 CRYPTO_ALG_KERN_DRIVER_ONLY, 1612 .cra_blocksize = MD5_HMAC_BLOCK_SIZE, 1613 .cra_ctxsize = sizeof(struct safexcel_ahash_ctx), 1614 .cra_init = safexcel_ahash_cra_init, 1615 .cra_exit = safexcel_ahash_cra_exit, 1616 .cra_module = THIS_MODULE, 1617 }, 1618 }, 1619 }, 1620 }; 1621 1622 static int safexcel_hmac_md5_init(struct ahash_request *areq) 1623 { 1624 struct safexcel_ahash_req *req = ahash_request_ctx(areq); 1625 1626 safexcel_md5_init(areq); 1627 req->digest = CONTEXT_CONTROL_DIGEST_HMAC; 1628 return 0; 1629 } 1630 1631 static int safexcel_hmac_md5_setkey(struct crypto_ahash *tfm, const u8 *key, 1632 unsigned int keylen) 1633 { 1634 return safexcel_hmac_alg_setkey(tfm, key, keylen, "safexcel-md5", 1635 MD5_DIGEST_SIZE); 1636 } 1637 1638 static int safexcel_hmac_md5_digest(struct ahash_request *areq) 1639 { 1640 int ret = safexcel_hmac_md5_init(areq); 1641 1642 if (ret) 1643 return ret; 1644 1645 return safexcel_ahash_finup(areq); 1646 } 1647 1648 struct safexcel_alg_template safexcel_alg_hmac_md5 = { 1649 .type = SAFEXCEL_ALG_TYPE_AHASH, 1650 .engines = EIP97IES | EIP197B | EIP197D, 1651 .alg.ahash = { 1652 .init = safexcel_hmac_md5_init, 1653 .update = safexcel_ahash_update, 1654 .final = safexcel_ahash_final, 1655 .finup = safexcel_ahash_finup, 1656 .digest = safexcel_hmac_md5_digest, 1657 .setkey = safexcel_hmac_md5_setkey, 1658 .export = safexcel_ahash_export, 1659 .import = safexcel_ahash_import, 1660 .halg = { 1661 .digestsize = MD5_DIGEST_SIZE, 1662 .statesize = sizeof(struct safexcel_ahash_export_state), 1663 .base = { 1664 .cra_name = "hmac(md5)", 1665 .cra_driver_name = "safexcel-hmac-md5", 1666 .cra_priority = 300, 1667 .cra_flags = CRYPTO_ALG_ASYNC | 1668 CRYPTO_ALG_KERN_DRIVER_ONLY, 1669 .cra_blocksize = MD5_HMAC_BLOCK_SIZE, 1670 .cra_ctxsize = sizeof(struct safexcel_ahash_ctx), 1671 .cra_init = safexcel_ahash_cra_init, 1672 .cra_exit = safexcel_ahash_cra_exit, 1673 .cra_module = THIS_MODULE, 1674 }, 1675 }, 1676 }, 1677 }; 1678