1 // SPDX-License-Identifier: GPL-2.0-only 2 /* 3 * Copyright (c) 2014 Imagination Technologies 4 * Authors: Will Thomas, James Hartley 5 * 6 * Interface structure taken from omap-sham driver 7 */ 8 9 #include <linux/clk.h> 10 #include <linux/dmaengine.h> 11 #include <linux/interrupt.h> 12 #include <linux/io.h> 13 #include <linux/kernel.h> 14 #include <linux/module.h> 15 #include <linux/of_device.h> 16 #include <linux/platform_device.h> 17 #include <linux/scatterlist.h> 18 19 #include <crypto/internal/hash.h> 20 #include <crypto/md5.h> 21 #include <crypto/sha.h> 22 23 #define CR_RESET 0 24 #define CR_RESET_SET 1 25 #define CR_RESET_UNSET 0 26 27 #define CR_MESSAGE_LENGTH_H 0x4 28 #define CR_MESSAGE_LENGTH_L 0x8 29 30 #define CR_CONTROL 0xc 31 #define CR_CONTROL_BYTE_ORDER_3210 0 32 #define CR_CONTROL_BYTE_ORDER_0123 1 33 #define CR_CONTROL_BYTE_ORDER_2310 2 34 #define CR_CONTROL_BYTE_ORDER_1032 3 35 #define CR_CONTROL_BYTE_ORDER_SHIFT 8 36 #define CR_CONTROL_ALGO_MD5 0 37 #define CR_CONTROL_ALGO_SHA1 1 38 #define CR_CONTROL_ALGO_SHA224 2 39 #define CR_CONTROL_ALGO_SHA256 3 40 41 #define CR_INTSTAT 0x10 42 #define CR_INTENAB 0x14 43 #define CR_INTCLEAR 0x18 44 #define CR_INT_RESULTS_AVAILABLE BIT(0) 45 #define CR_INT_NEW_RESULTS_SET BIT(1) 46 #define CR_INT_RESULT_READ_ERR BIT(2) 47 #define CR_INT_MESSAGE_WRITE_ERROR BIT(3) 48 #define CR_INT_STATUS BIT(8) 49 50 #define CR_RESULT_QUEUE 0x1c 51 #define CR_RSD0 0x40 52 #define CR_CORE_REV 0x50 53 #define CR_CORE_DES1 0x60 54 #define CR_CORE_DES2 0x70 55 56 #define DRIVER_FLAGS_BUSY BIT(0) 57 #define DRIVER_FLAGS_FINAL BIT(1) 58 #define DRIVER_FLAGS_DMA_ACTIVE BIT(2) 59 #define DRIVER_FLAGS_OUTPUT_READY BIT(3) 60 #define DRIVER_FLAGS_INIT BIT(4) 61 #define DRIVER_FLAGS_CPU BIT(5) 62 #define DRIVER_FLAGS_DMA_READY BIT(6) 63 #define DRIVER_FLAGS_ERROR BIT(7) 64 #define DRIVER_FLAGS_SG BIT(8) 65 #define DRIVER_FLAGS_SHA1 BIT(18) 66 #define DRIVER_FLAGS_SHA224 BIT(19) 67 #define DRIVER_FLAGS_SHA256 BIT(20) 68 #define DRIVER_FLAGS_MD5 BIT(21) 69 70 #define IMG_HASH_QUEUE_LENGTH 20 71 #define IMG_HASH_DMA_BURST 4 72 #define IMG_HASH_DMA_THRESHOLD 64 73 74 #ifdef __LITTLE_ENDIAN 75 #define IMG_HASH_BYTE_ORDER CR_CONTROL_BYTE_ORDER_3210 76 #else 77 #define IMG_HASH_BYTE_ORDER CR_CONTROL_BYTE_ORDER_0123 78 #endif 79 80 struct img_hash_dev; 81 82 struct img_hash_request_ctx { 83 struct img_hash_dev *hdev; 84 u8 digest[SHA256_DIGEST_SIZE] __aligned(sizeof(u32)); 85 unsigned long flags; 86 size_t digsize; 87 88 dma_addr_t dma_addr; 89 size_t dma_ct; 90 91 /* sg root */ 92 struct scatterlist *sgfirst; 93 /* walk state */ 94 struct scatterlist *sg; 95 size_t nents; 96 size_t offset; 97 unsigned int total; 98 size_t sent; 99 100 unsigned long op; 101 102 size_t bufcnt; 103 struct ahash_request fallback_req; 104 105 /* Zero length buffer must remain last member of struct */ 106 u8 buffer[0] __aligned(sizeof(u32)); 107 }; 108 109 struct img_hash_ctx { 110 struct img_hash_dev *hdev; 111 unsigned long flags; 112 struct crypto_ahash *fallback; 113 }; 114 115 struct img_hash_dev { 116 struct list_head list; 117 struct device *dev; 118 struct clk *hash_clk; 119 struct clk *sys_clk; 120 void __iomem *io_base; 121 122 phys_addr_t bus_addr; 123 void __iomem *cpu_addr; 124 125 spinlock_t lock; 126 int err; 127 struct tasklet_struct done_task; 128 struct tasklet_struct dma_task; 129 130 unsigned long flags; 131 struct crypto_queue queue; 132 struct ahash_request *req; 133 134 struct dma_chan *dma_lch; 135 }; 136 137 struct img_hash_drv { 138 struct list_head dev_list; 139 spinlock_t lock; 140 }; 141 142 static struct img_hash_drv img_hash = { 143 .dev_list = LIST_HEAD_INIT(img_hash.dev_list), 144 .lock = __SPIN_LOCK_UNLOCKED(img_hash.lock), 145 }; 146 147 static inline u32 img_hash_read(struct img_hash_dev *hdev, u32 offset) 148 { 149 return readl_relaxed(hdev->io_base + offset); 150 } 151 152 static inline void img_hash_write(struct img_hash_dev *hdev, 153 u32 offset, u32 value) 154 { 155 writel_relaxed(value, hdev->io_base + offset); 156 } 157 158 static inline u32 img_hash_read_result_queue(struct img_hash_dev *hdev) 159 { 160 return be32_to_cpu(img_hash_read(hdev, CR_RESULT_QUEUE)); 161 } 162 163 static void img_hash_start(struct img_hash_dev *hdev, bool dma) 164 { 165 struct img_hash_request_ctx *ctx = ahash_request_ctx(hdev->req); 166 u32 cr = IMG_HASH_BYTE_ORDER << CR_CONTROL_BYTE_ORDER_SHIFT; 167 168 if (ctx->flags & DRIVER_FLAGS_MD5) 169 cr |= CR_CONTROL_ALGO_MD5; 170 else if (ctx->flags & DRIVER_FLAGS_SHA1) 171 cr |= CR_CONTROL_ALGO_SHA1; 172 else if (ctx->flags & DRIVER_FLAGS_SHA224) 173 cr |= CR_CONTROL_ALGO_SHA224; 174 else if (ctx->flags & DRIVER_FLAGS_SHA256) 175 cr |= CR_CONTROL_ALGO_SHA256; 176 dev_dbg(hdev->dev, "Starting hash process\n"); 177 img_hash_write(hdev, CR_CONTROL, cr); 178 179 /* 180 * The hardware block requires two cycles between writing the control 181 * register and writing the first word of data in non DMA mode, to 182 * ensure the first data write is not grouped in burst with the control 183 * register write a read is issued to 'flush' the bus. 184 */ 185 if (!dma) 186 img_hash_read(hdev, CR_CONTROL); 187 } 188 189 static int img_hash_xmit_cpu(struct img_hash_dev *hdev, const u8 *buf, 190 size_t length, int final) 191 { 192 u32 count, len32; 193 const u32 *buffer = (const u32 *)buf; 194 195 dev_dbg(hdev->dev, "xmit_cpu: length: %zu bytes\n", length); 196 197 if (final) 198 hdev->flags |= DRIVER_FLAGS_FINAL; 199 200 len32 = DIV_ROUND_UP(length, sizeof(u32)); 201 202 for (count = 0; count < len32; count++) 203 writel_relaxed(buffer[count], hdev->cpu_addr); 204 205 return -EINPROGRESS; 206 } 207 208 static void img_hash_dma_callback(void *data) 209 { 210 struct img_hash_dev *hdev = (struct img_hash_dev *)data; 211 struct img_hash_request_ctx *ctx = ahash_request_ctx(hdev->req); 212 213 if (ctx->bufcnt) { 214 img_hash_xmit_cpu(hdev, ctx->buffer, ctx->bufcnt, 0); 215 ctx->bufcnt = 0; 216 } 217 if (ctx->sg) 218 tasklet_schedule(&hdev->dma_task); 219 } 220 221 static int img_hash_xmit_dma(struct img_hash_dev *hdev, struct scatterlist *sg) 222 { 223 struct dma_async_tx_descriptor *desc; 224 struct img_hash_request_ctx *ctx = ahash_request_ctx(hdev->req); 225 226 ctx->dma_ct = dma_map_sg(hdev->dev, sg, 1, DMA_TO_DEVICE); 227 if (ctx->dma_ct == 0) { 228 dev_err(hdev->dev, "Invalid DMA sg\n"); 229 hdev->err = -EINVAL; 230 return -EINVAL; 231 } 232 233 desc = dmaengine_prep_slave_sg(hdev->dma_lch, 234 sg, 235 ctx->dma_ct, 236 DMA_MEM_TO_DEV, 237 DMA_PREP_INTERRUPT | DMA_CTRL_ACK); 238 if (!desc) { 239 dev_err(hdev->dev, "Null DMA descriptor\n"); 240 hdev->err = -EINVAL; 241 dma_unmap_sg(hdev->dev, sg, 1, DMA_TO_DEVICE); 242 return -EINVAL; 243 } 244 desc->callback = img_hash_dma_callback; 245 desc->callback_param = hdev; 246 dmaengine_submit(desc); 247 dma_async_issue_pending(hdev->dma_lch); 248 249 return 0; 250 } 251 252 static int img_hash_write_via_cpu(struct img_hash_dev *hdev) 253 { 254 struct img_hash_request_ctx *ctx = ahash_request_ctx(hdev->req); 255 256 ctx->bufcnt = sg_copy_to_buffer(hdev->req->src, sg_nents(ctx->sg), 257 ctx->buffer, hdev->req->nbytes); 258 259 ctx->total = hdev->req->nbytes; 260 ctx->bufcnt = 0; 261 262 hdev->flags |= (DRIVER_FLAGS_CPU | DRIVER_FLAGS_FINAL); 263 264 img_hash_start(hdev, false); 265 266 return img_hash_xmit_cpu(hdev, ctx->buffer, ctx->total, 1); 267 } 268 269 static int img_hash_finish(struct ahash_request *req) 270 { 271 struct img_hash_request_ctx *ctx = ahash_request_ctx(req); 272 273 if (!req->result) 274 return -EINVAL; 275 276 memcpy(req->result, ctx->digest, ctx->digsize); 277 278 return 0; 279 } 280 281 static void img_hash_copy_hash(struct ahash_request *req) 282 { 283 struct img_hash_request_ctx *ctx = ahash_request_ctx(req); 284 u32 *hash = (u32 *)ctx->digest; 285 int i; 286 287 for (i = (ctx->digsize / sizeof(u32)) - 1; i >= 0; i--) 288 hash[i] = img_hash_read_result_queue(ctx->hdev); 289 } 290 291 static void img_hash_finish_req(struct ahash_request *req, int err) 292 { 293 struct img_hash_request_ctx *ctx = ahash_request_ctx(req); 294 struct img_hash_dev *hdev = ctx->hdev; 295 296 if (!err) { 297 img_hash_copy_hash(req); 298 if (DRIVER_FLAGS_FINAL & hdev->flags) 299 err = img_hash_finish(req); 300 } else { 301 dev_warn(hdev->dev, "Hash failed with error %d\n", err); 302 ctx->flags |= DRIVER_FLAGS_ERROR; 303 } 304 305 hdev->flags &= ~(DRIVER_FLAGS_DMA_READY | DRIVER_FLAGS_OUTPUT_READY | 306 DRIVER_FLAGS_CPU | DRIVER_FLAGS_BUSY | DRIVER_FLAGS_FINAL); 307 308 if (req->base.complete) 309 req->base.complete(&req->base, err); 310 } 311 312 static int img_hash_write_via_dma(struct img_hash_dev *hdev) 313 { 314 struct img_hash_request_ctx *ctx = ahash_request_ctx(hdev->req); 315 316 img_hash_start(hdev, true); 317 318 dev_dbg(hdev->dev, "xmit dma size: %d\n", ctx->total); 319 320 if (!ctx->total) 321 hdev->flags |= DRIVER_FLAGS_FINAL; 322 323 hdev->flags |= DRIVER_FLAGS_DMA_ACTIVE | DRIVER_FLAGS_FINAL; 324 325 tasklet_schedule(&hdev->dma_task); 326 327 return -EINPROGRESS; 328 } 329 330 static int img_hash_dma_init(struct img_hash_dev *hdev) 331 { 332 struct dma_slave_config dma_conf; 333 int err = -EINVAL; 334 335 hdev->dma_lch = dma_request_slave_channel(hdev->dev, "tx"); 336 if (!hdev->dma_lch) { 337 dev_err(hdev->dev, "Couldn't acquire a slave DMA channel.\n"); 338 return -EBUSY; 339 } 340 dma_conf.direction = DMA_MEM_TO_DEV; 341 dma_conf.dst_addr = hdev->bus_addr; 342 dma_conf.dst_addr_width = DMA_SLAVE_BUSWIDTH_4_BYTES; 343 dma_conf.dst_maxburst = IMG_HASH_DMA_BURST; 344 dma_conf.device_fc = false; 345 346 err = dmaengine_slave_config(hdev->dma_lch, &dma_conf); 347 if (err) { 348 dev_err(hdev->dev, "Couldn't configure DMA slave.\n"); 349 dma_release_channel(hdev->dma_lch); 350 return err; 351 } 352 353 return 0; 354 } 355 356 static void img_hash_dma_task(unsigned long d) 357 { 358 struct img_hash_dev *hdev = (struct img_hash_dev *)d; 359 struct img_hash_request_ctx *ctx = ahash_request_ctx(hdev->req); 360 u8 *addr; 361 size_t nbytes, bleft, wsend, len, tbc; 362 struct scatterlist tsg; 363 364 if (!hdev->req || !ctx->sg) 365 return; 366 367 addr = sg_virt(ctx->sg); 368 nbytes = ctx->sg->length - ctx->offset; 369 370 /* 371 * The hash accelerator does not support a data valid mask. This means 372 * that if each dma (i.e. per page) is not a multiple of 4 bytes, the 373 * padding bytes in the last word written by that dma would erroneously 374 * be included in the hash. To avoid this we round down the transfer, 375 * and add the excess to the start of the next dma. It does not matter 376 * that the final dma may not be a multiple of 4 bytes as the hashing 377 * block is programmed to accept the correct number of bytes. 378 */ 379 380 bleft = nbytes % 4; 381 wsend = (nbytes / 4); 382 383 if (wsend) { 384 sg_init_one(&tsg, addr + ctx->offset, wsend * 4); 385 if (img_hash_xmit_dma(hdev, &tsg)) { 386 dev_err(hdev->dev, "DMA failed, falling back to CPU"); 387 ctx->flags |= DRIVER_FLAGS_CPU; 388 hdev->err = 0; 389 img_hash_xmit_cpu(hdev, addr + ctx->offset, 390 wsend * 4, 0); 391 ctx->sent += wsend * 4; 392 wsend = 0; 393 } else { 394 ctx->sent += wsend * 4; 395 } 396 } 397 398 if (bleft) { 399 ctx->bufcnt = sg_pcopy_to_buffer(ctx->sgfirst, ctx->nents, 400 ctx->buffer, bleft, ctx->sent); 401 tbc = 0; 402 ctx->sg = sg_next(ctx->sg); 403 while (ctx->sg && (ctx->bufcnt < 4)) { 404 len = ctx->sg->length; 405 if (likely(len > (4 - ctx->bufcnt))) 406 len = 4 - ctx->bufcnt; 407 tbc = sg_pcopy_to_buffer(ctx->sgfirst, ctx->nents, 408 ctx->buffer + ctx->bufcnt, len, 409 ctx->sent + ctx->bufcnt); 410 ctx->bufcnt += tbc; 411 if (tbc >= ctx->sg->length) { 412 ctx->sg = sg_next(ctx->sg); 413 tbc = 0; 414 } 415 } 416 417 ctx->sent += ctx->bufcnt; 418 ctx->offset = tbc; 419 420 if (!wsend) 421 img_hash_dma_callback(hdev); 422 } else { 423 ctx->offset = 0; 424 ctx->sg = sg_next(ctx->sg); 425 } 426 } 427 428 static int img_hash_write_via_dma_stop(struct img_hash_dev *hdev) 429 { 430 struct img_hash_request_ctx *ctx = ahash_request_ctx(hdev->req); 431 432 if (ctx->flags & DRIVER_FLAGS_SG) 433 dma_unmap_sg(hdev->dev, ctx->sg, ctx->dma_ct, DMA_TO_DEVICE); 434 435 return 0; 436 } 437 438 static int img_hash_process_data(struct img_hash_dev *hdev) 439 { 440 struct ahash_request *req = hdev->req; 441 struct img_hash_request_ctx *ctx = ahash_request_ctx(req); 442 int err = 0; 443 444 ctx->bufcnt = 0; 445 446 if (req->nbytes >= IMG_HASH_DMA_THRESHOLD) { 447 dev_dbg(hdev->dev, "process data request(%d bytes) using DMA\n", 448 req->nbytes); 449 err = img_hash_write_via_dma(hdev); 450 } else { 451 dev_dbg(hdev->dev, "process data request(%d bytes) using CPU\n", 452 req->nbytes); 453 err = img_hash_write_via_cpu(hdev); 454 } 455 return err; 456 } 457 458 static int img_hash_hw_init(struct img_hash_dev *hdev) 459 { 460 unsigned long long nbits; 461 u32 u, l; 462 463 img_hash_write(hdev, CR_RESET, CR_RESET_SET); 464 img_hash_write(hdev, CR_RESET, CR_RESET_UNSET); 465 img_hash_write(hdev, CR_INTENAB, CR_INT_NEW_RESULTS_SET); 466 467 nbits = (u64)hdev->req->nbytes << 3; 468 u = nbits >> 32; 469 l = nbits; 470 img_hash_write(hdev, CR_MESSAGE_LENGTH_H, u); 471 img_hash_write(hdev, CR_MESSAGE_LENGTH_L, l); 472 473 if (!(DRIVER_FLAGS_INIT & hdev->flags)) { 474 hdev->flags |= DRIVER_FLAGS_INIT; 475 hdev->err = 0; 476 } 477 dev_dbg(hdev->dev, "hw initialized, nbits: %llx\n", nbits); 478 return 0; 479 } 480 481 static int img_hash_init(struct ahash_request *req) 482 { 483 struct crypto_ahash *tfm = crypto_ahash_reqtfm(req); 484 struct img_hash_request_ctx *rctx = ahash_request_ctx(req); 485 struct img_hash_ctx *ctx = crypto_ahash_ctx(tfm); 486 487 ahash_request_set_tfm(&rctx->fallback_req, ctx->fallback); 488 rctx->fallback_req.base.flags = req->base.flags 489 & CRYPTO_TFM_REQ_MAY_SLEEP; 490 491 return crypto_ahash_init(&rctx->fallback_req); 492 } 493 494 static int img_hash_handle_queue(struct img_hash_dev *hdev, 495 struct ahash_request *req) 496 { 497 struct crypto_async_request *async_req, *backlog; 498 struct img_hash_request_ctx *ctx; 499 unsigned long flags; 500 int err = 0, res = 0; 501 502 spin_lock_irqsave(&hdev->lock, flags); 503 504 if (req) 505 res = ahash_enqueue_request(&hdev->queue, req); 506 507 if (DRIVER_FLAGS_BUSY & hdev->flags) { 508 spin_unlock_irqrestore(&hdev->lock, flags); 509 return res; 510 } 511 512 backlog = crypto_get_backlog(&hdev->queue); 513 async_req = crypto_dequeue_request(&hdev->queue); 514 if (async_req) 515 hdev->flags |= DRIVER_FLAGS_BUSY; 516 517 spin_unlock_irqrestore(&hdev->lock, flags); 518 519 if (!async_req) 520 return res; 521 522 if (backlog) 523 backlog->complete(backlog, -EINPROGRESS); 524 525 req = ahash_request_cast(async_req); 526 hdev->req = req; 527 528 ctx = ahash_request_ctx(req); 529 530 dev_info(hdev->dev, "processing req, op: %lu, bytes: %d\n", 531 ctx->op, req->nbytes); 532 533 err = img_hash_hw_init(hdev); 534 535 if (!err) 536 err = img_hash_process_data(hdev); 537 538 if (err != -EINPROGRESS) { 539 /* done_task will not finish so do it here */ 540 img_hash_finish_req(req, err); 541 } 542 return res; 543 } 544 545 static int img_hash_update(struct ahash_request *req) 546 { 547 struct img_hash_request_ctx *rctx = ahash_request_ctx(req); 548 struct crypto_ahash *tfm = crypto_ahash_reqtfm(req); 549 struct img_hash_ctx *ctx = crypto_ahash_ctx(tfm); 550 551 ahash_request_set_tfm(&rctx->fallback_req, ctx->fallback); 552 rctx->fallback_req.base.flags = req->base.flags 553 & CRYPTO_TFM_REQ_MAY_SLEEP; 554 rctx->fallback_req.nbytes = req->nbytes; 555 rctx->fallback_req.src = req->src; 556 557 return crypto_ahash_update(&rctx->fallback_req); 558 } 559 560 static int img_hash_final(struct ahash_request *req) 561 { 562 struct img_hash_request_ctx *rctx = ahash_request_ctx(req); 563 struct crypto_ahash *tfm = crypto_ahash_reqtfm(req); 564 struct img_hash_ctx *ctx = crypto_ahash_ctx(tfm); 565 566 ahash_request_set_tfm(&rctx->fallback_req, ctx->fallback); 567 rctx->fallback_req.base.flags = req->base.flags 568 & CRYPTO_TFM_REQ_MAY_SLEEP; 569 rctx->fallback_req.result = req->result; 570 571 return crypto_ahash_final(&rctx->fallback_req); 572 } 573 574 static int img_hash_finup(struct ahash_request *req) 575 { 576 struct img_hash_request_ctx *rctx = ahash_request_ctx(req); 577 struct crypto_ahash *tfm = crypto_ahash_reqtfm(req); 578 struct img_hash_ctx *ctx = crypto_ahash_ctx(tfm); 579 580 ahash_request_set_tfm(&rctx->fallback_req, ctx->fallback); 581 rctx->fallback_req.base.flags = req->base.flags 582 & CRYPTO_TFM_REQ_MAY_SLEEP; 583 rctx->fallback_req.nbytes = req->nbytes; 584 rctx->fallback_req.src = req->src; 585 rctx->fallback_req.result = req->result; 586 587 return crypto_ahash_finup(&rctx->fallback_req); 588 } 589 590 static int img_hash_import(struct ahash_request *req, const void *in) 591 { 592 struct img_hash_request_ctx *rctx = ahash_request_ctx(req); 593 struct crypto_ahash *tfm = crypto_ahash_reqtfm(req); 594 struct img_hash_ctx *ctx = crypto_ahash_ctx(tfm); 595 596 ahash_request_set_tfm(&rctx->fallback_req, ctx->fallback); 597 rctx->fallback_req.base.flags = req->base.flags 598 & CRYPTO_TFM_REQ_MAY_SLEEP; 599 600 return crypto_ahash_import(&rctx->fallback_req, in); 601 } 602 603 static int img_hash_export(struct ahash_request *req, void *out) 604 { 605 struct img_hash_request_ctx *rctx = ahash_request_ctx(req); 606 struct crypto_ahash *tfm = crypto_ahash_reqtfm(req); 607 struct img_hash_ctx *ctx = crypto_ahash_ctx(tfm); 608 609 ahash_request_set_tfm(&rctx->fallback_req, ctx->fallback); 610 rctx->fallback_req.base.flags = req->base.flags 611 & CRYPTO_TFM_REQ_MAY_SLEEP; 612 613 return crypto_ahash_export(&rctx->fallback_req, out); 614 } 615 616 static int img_hash_digest(struct ahash_request *req) 617 { 618 struct crypto_ahash *tfm = crypto_ahash_reqtfm(req); 619 struct img_hash_ctx *tctx = crypto_ahash_ctx(tfm); 620 struct img_hash_request_ctx *ctx = ahash_request_ctx(req); 621 struct img_hash_dev *hdev = NULL; 622 struct img_hash_dev *tmp; 623 int err; 624 625 spin_lock(&img_hash.lock); 626 if (!tctx->hdev) { 627 list_for_each_entry(tmp, &img_hash.dev_list, list) { 628 hdev = tmp; 629 break; 630 } 631 tctx->hdev = hdev; 632 633 } else { 634 hdev = tctx->hdev; 635 } 636 637 spin_unlock(&img_hash.lock); 638 ctx->hdev = hdev; 639 ctx->flags = 0; 640 ctx->digsize = crypto_ahash_digestsize(tfm); 641 642 switch (ctx->digsize) { 643 case SHA1_DIGEST_SIZE: 644 ctx->flags |= DRIVER_FLAGS_SHA1; 645 break; 646 case SHA256_DIGEST_SIZE: 647 ctx->flags |= DRIVER_FLAGS_SHA256; 648 break; 649 case SHA224_DIGEST_SIZE: 650 ctx->flags |= DRIVER_FLAGS_SHA224; 651 break; 652 case MD5_DIGEST_SIZE: 653 ctx->flags |= DRIVER_FLAGS_MD5; 654 break; 655 default: 656 return -EINVAL; 657 } 658 659 ctx->bufcnt = 0; 660 ctx->offset = 0; 661 ctx->sent = 0; 662 ctx->total = req->nbytes; 663 ctx->sg = req->src; 664 ctx->sgfirst = req->src; 665 ctx->nents = sg_nents(ctx->sg); 666 667 err = img_hash_handle_queue(tctx->hdev, req); 668 669 return err; 670 } 671 672 static int img_hash_cra_init(struct crypto_tfm *tfm, const char *alg_name) 673 { 674 struct img_hash_ctx *ctx = crypto_tfm_ctx(tfm); 675 int err = -ENOMEM; 676 677 ctx->fallback = crypto_alloc_ahash(alg_name, 0, 678 CRYPTO_ALG_NEED_FALLBACK); 679 if (IS_ERR(ctx->fallback)) { 680 pr_err("img_hash: Could not load fallback driver.\n"); 681 err = PTR_ERR(ctx->fallback); 682 goto err; 683 } 684 crypto_ahash_set_reqsize(__crypto_ahash_cast(tfm), 685 sizeof(struct img_hash_request_ctx) + 686 crypto_ahash_reqsize(ctx->fallback) + 687 IMG_HASH_DMA_THRESHOLD); 688 689 return 0; 690 691 err: 692 return err; 693 } 694 695 static int img_hash_cra_md5_init(struct crypto_tfm *tfm) 696 { 697 return img_hash_cra_init(tfm, "md5-generic"); 698 } 699 700 static int img_hash_cra_sha1_init(struct crypto_tfm *tfm) 701 { 702 return img_hash_cra_init(tfm, "sha1-generic"); 703 } 704 705 static int img_hash_cra_sha224_init(struct crypto_tfm *tfm) 706 { 707 return img_hash_cra_init(tfm, "sha224-generic"); 708 } 709 710 static int img_hash_cra_sha256_init(struct crypto_tfm *tfm) 711 { 712 return img_hash_cra_init(tfm, "sha256-generic"); 713 } 714 715 static void img_hash_cra_exit(struct crypto_tfm *tfm) 716 { 717 struct img_hash_ctx *tctx = crypto_tfm_ctx(tfm); 718 719 crypto_free_ahash(tctx->fallback); 720 } 721 722 static irqreturn_t img_irq_handler(int irq, void *dev_id) 723 { 724 struct img_hash_dev *hdev = dev_id; 725 u32 reg; 726 727 reg = img_hash_read(hdev, CR_INTSTAT); 728 img_hash_write(hdev, CR_INTCLEAR, reg); 729 730 if (reg & CR_INT_NEW_RESULTS_SET) { 731 dev_dbg(hdev->dev, "IRQ CR_INT_NEW_RESULTS_SET\n"); 732 if (DRIVER_FLAGS_BUSY & hdev->flags) { 733 hdev->flags |= DRIVER_FLAGS_OUTPUT_READY; 734 if (!(DRIVER_FLAGS_CPU & hdev->flags)) 735 hdev->flags |= DRIVER_FLAGS_DMA_READY; 736 tasklet_schedule(&hdev->done_task); 737 } else { 738 dev_warn(hdev->dev, 739 "HASH interrupt when no active requests.\n"); 740 } 741 } else if (reg & CR_INT_RESULTS_AVAILABLE) { 742 dev_warn(hdev->dev, 743 "IRQ triggered before the hash had completed\n"); 744 } else if (reg & CR_INT_RESULT_READ_ERR) { 745 dev_warn(hdev->dev, 746 "Attempt to read from an empty result queue\n"); 747 } else if (reg & CR_INT_MESSAGE_WRITE_ERROR) { 748 dev_warn(hdev->dev, 749 "Data written before the hardware was configured\n"); 750 } 751 return IRQ_HANDLED; 752 } 753 754 static struct ahash_alg img_algs[] = { 755 { 756 .init = img_hash_init, 757 .update = img_hash_update, 758 .final = img_hash_final, 759 .finup = img_hash_finup, 760 .export = img_hash_export, 761 .import = img_hash_import, 762 .digest = img_hash_digest, 763 .halg = { 764 .digestsize = MD5_DIGEST_SIZE, 765 .statesize = sizeof(struct md5_state), 766 .base = { 767 .cra_name = "md5", 768 .cra_driver_name = "img-md5", 769 .cra_priority = 300, 770 .cra_flags = 771 CRYPTO_ALG_ASYNC | 772 CRYPTO_ALG_NEED_FALLBACK, 773 .cra_blocksize = MD5_HMAC_BLOCK_SIZE, 774 .cra_ctxsize = sizeof(struct img_hash_ctx), 775 .cra_init = img_hash_cra_md5_init, 776 .cra_exit = img_hash_cra_exit, 777 .cra_module = THIS_MODULE, 778 } 779 } 780 }, 781 { 782 .init = img_hash_init, 783 .update = img_hash_update, 784 .final = img_hash_final, 785 .finup = img_hash_finup, 786 .export = img_hash_export, 787 .import = img_hash_import, 788 .digest = img_hash_digest, 789 .halg = { 790 .digestsize = SHA1_DIGEST_SIZE, 791 .statesize = sizeof(struct sha1_state), 792 .base = { 793 .cra_name = "sha1", 794 .cra_driver_name = "img-sha1", 795 .cra_priority = 300, 796 .cra_flags = 797 CRYPTO_ALG_ASYNC | 798 CRYPTO_ALG_NEED_FALLBACK, 799 .cra_blocksize = SHA1_BLOCK_SIZE, 800 .cra_ctxsize = sizeof(struct img_hash_ctx), 801 .cra_init = img_hash_cra_sha1_init, 802 .cra_exit = img_hash_cra_exit, 803 .cra_module = THIS_MODULE, 804 } 805 } 806 }, 807 { 808 .init = img_hash_init, 809 .update = img_hash_update, 810 .final = img_hash_final, 811 .finup = img_hash_finup, 812 .export = img_hash_export, 813 .import = img_hash_import, 814 .digest = img_hash_digest, 815 .halg = { 816 .digestsize = SHA224_DIGEST_SIZE, 817 .statesize = sizeof(struct sha256_state), 818 .base = { 819 .cra_name = "sha224", 820 .cra_driver_name = "img-sha224", 821 .cra_priority = 300, 822 .cra_flags = 823 CRYPTO_ALG_ASYNC | 824 CRYPTO_ALG_NEED_FALLBACK, 825 .cra_blocksize = SHA224_BLOCK_SIZE, 826 .cra_ctxsize = sizeof(struct img_hash_ctx), 827 .cra_init = img_hash_cra_sha224_init, 828 .cra_exit = img_hash_cra_exit, 829 .cra_module = THIS_MODULE, 830 } 831 } 832 }, 833 { 834 .init = img_hash_init, 835 .update = img_hash_update, 836 .final = img_hash_final, 837 .finup = img_hash_finup, 838 .export = img_hash_export, 839 .import = img_hash_import, 840 .digest = img_hash_digest, 841 .halg = { 842 .digestsize = SHA256_DIGEST_SIZE, 843 .statesize = sizeof(struct sha256_state), 844 .base = { 845 .cra_name = "sha256", 846 .cra_driver_name = "img-sha256", 847 .cra_priority = 300, 848 .cra_flags = 849 CRYPTO_ALG_ASYNC | 850 CRYPTO_ALG_NEED_FALLBACK, 851 .cra_blocksize = SHA256_BLOCK_SIZE, 852 .cra_ctxsize = sizeof(struct img_hash_ctx), 853 .cra_init = img_hash_cra_sha256_init, 854 .cra_exit = img_hash_cra_exit, 855 .cra_module = THIS_MODULE, 856 } 857 } 858 } 859 }; 860 861 static int img_register_algs(struct img_hash_dev *hdev) 862 { 863 int i, err; 864 865 for (i = 0; i < ARRAY_SIZE(img_algs); i++) { 866 err = crypto_register_ahash(&img_algs[i]); 867 if (err) 868 goto err_reg; 869 } 870 return 0; 871 872 err_reg: 873 for (; i--; ) 874 crypto_unregister_ahash(&img_algs[i]); 875 876 return err; 877 } 878 879 static int img_unregister_algs(struct img_hash_dev *hdev) 880 { 881 int i; 882 883 for (i = 0; i < ARRAY_SIZE(img_algs); i++) 884 crypto_unregister_ahash(&img_algs[i]); 885 return 0; 886 } 887 888 static void img_hash_done_task(unsigned long data) 889 { 890 struct img_hash_dev *hdev = (struct img_hash_dev *)data; 891 int err = 0; 892 893 if (hdev->err == -EINVAL) { 894 err = hdev->err; 895 goto finish; 896 } 897 898 if (!(DRIVER_FLAGS_BUSY & hdev->flags)) { 899 img_hash_handle_queue(hdev, NULL); 900 return; 901 } 902 903 if (DRIVER_FLAGS_CPU & hdev->flags) { 904 if (DRIVER_FLAGS_OUTPUT_READY & hdev->flags) { 905 hdev->flags &= ~DRIVER_FLAGS_OUTPUT_READY; 906 goto finish; 907 } 908 } else if (DRIVER_FLAGS_DMA_READY & hdev->flags) { 909 if (DRIVER_FLAGS_DMA_ACTIVE & hdev->flags) { 910 hdev->flags &= ~DRIVER_FLAGS_DMA_ACTIVE; 911 img_hash_write_via_dma_stop(hdev); 912 if (hdev->err) { 913 err = hdev->err; 914 goto finish; 915 } 916 } 917 if (DRIVER_FLAGS_OUTPUT_READY & hdev->flags) { 918 hdev->flags &= ~(DRIVER_FLAGS_DMA_READY | 919 DRIVER_FLAGS_OUTPUT_READY); 920 goto finish; 921 } 922 } 923 return; 924 925 finish: 926 img_hash_finish_req(hdev->req, err); 927 } 928 929 static const struct of_device_id img_hash_match[] = { 930 { .compatible = "img,hash-accelerator" }, 931 {} 932 }; 933 MODULE_DEVICE_TABLE(of, img_hash_match); 934 935 static int img_hash_probe(struct platform_device *pdev) 936 { 937 struct img_hash_dev *hdev; 938 struct device *dev = &pdev->dev; 939 struct resource *hash_res; 940 int irq; 941 int err; 942 943 hdev = devm_kzalloc(dev, sizeof(*hdev), GFP_KERNEL); 944 if (hdev == NULL) 945 return -ENOMEM; 946 947 spin_lock_init(&hdev->lock); 948 949 hdev->dev = dev; 950 951 platform_set_drvdata(pdev, hdev); 952 953 INIT_LIST_HEAD(&hdev->list); 954 955 tasklet_init(&hdev->done_task, img_hash_done_task, (unsigned long)hdev); 956 tasklet_init(&hdev->dma_task, img_hash_dma_task, (unsigned long)hdev); 957 958 crypto_init_queue(&hdev->queue, IMG_HASH_QUEUE_LENGTH); 959 960 /* Register bank */ 961 hash_res = platform_get_resource(pdev, IORESOURCE_MEM, 0); 962 963 hdev->io_base = devm_ioremap_resource(dev, hash_res); 964 if (IS_ERR(hdev->io_base)) { 965 err = PTR_ERR(hdev->io_base); 966 dev_err(dev, "can't ioremap, returned %d\n", err); 967 968 goto res_err; 969 } 970 971 /* Write port (DMA or CPU) */ 972 hash_res = platform_get_resource(pdev, IORESOURCE_MEM, 1); 973 hdev->cpu_addr = devm_ioremap_resource(dev, hash_res); 974 if (IS_ERR(hdev->cpu_addr)) { 975 dev_err(dev, "can't ioremap write port\n"); 976 err = PTR_ERR(hdev->cpu_addr); 977 goto res_err; 978 } 979 hdev->bus_addr = hash_res->start; 980 981 irq = platform_get_irq(pdev, 0); 982 if (irq < 0) { 983 dev_err(dev, "no IRQ resource info\n"); 984 err = irq; 985 goto res_err; 986 } 987 988 err = devm_request_irq(dev, irq, img_irq_handler, 0, 989 dev_name(dev), hdev); 990 if (err) { 991 dev_err(dev, "unable to request irq\n"); 992 goto res_err; 993 } 994 dev_dbg(dev, "using IRQ channel %d\n", irq); 995 996 hdev->hash_clk = devm_clk_get(&pdev->dev, "hash"); 997 if (IS_ERR(hdev->hash_clk)) { 998 dev_err(dev, "clock initialization failed.\n"); 999 err = PTR_ERR(hdev->hash_clk); 1000 goto res_err; 1001 } 1002 1003 hdev->sys_clk = devm_clk_get(&pdev->dev, "sys"); 1004 if (IS_ERR(hdev->sys_clk)) { 1005 dev_err(dev, "clock initialization failed.\n"); 1006 err = PTR_ERR(hdev->sys_clk); 1007 goto res_err; 1008 } 1009 1010 err = clk_prepare_enable(hdev->hash_clk); 1011 if (err) 1012 goto res_err; 1013 1014 err = clk_prepare_enable(hdev->sys_clk); 1015 if (err) 1016 goto clk_err; 1017 1018 err = img_hash_dma_init(hdev); 1019 if (err) 1020 goto dma_err; 1021 1022 dev_dbg(dev, "using %s for DMA transfers\n", 1023 dma_chan_name(hdev->dma_lch)); 1024 1025 spin_lock(&img_hash.lock); 1026 list_add_tail(&hdev->list, &img_hash.dev_list); 1027 spin_unlock(&img_hash.lock); 1028 1029 err = img_register_algs(hdev); 1030 if (err) 1031 goto err_algs; 1032 dev_info(dev, "Img MD5/SHA1/SHA224/SHA256 Hardware accelerator initialized\n"); 1033 1034 return 0; 1035 1036 err_algs: 1037 spin_lock(&img_hash.lock); 1038 list_del(&hdev->list); 1039 spin_unlock(&img_hash.lock); 1040 dma_release_channel(hdev->dma_lch); 1041 dma_err: 1042 clk_disable_unprepare(hdev->sys_clk); 1043 clk_err: 1044 clk_disable_unprepare(hdev->hash_clk); 1045 res_err: 1046 tasklet_kill(&hdev->done_task); 1047 tasklet_kill(&hdev->dma_task); 1048 1049 return err; 1050 } 1051 1052 static int img_hash_remove(struct platform_device *pdev) 1053 { 1054 struct img_hash_dev *hdev; 1055 1056 hdev = platform_get_drvdata(pdev); 1057 spin_lock(&img_hash.lock); 1058 list_del(&hdev->list); 1059 spin_unlock(&img_hash.lock); 1060 1061 img_unregister_algs(hdev); 1062 1063 tasklet_kill(&hdev->done_task); 1064 tasklet_kill(&hdev->dma_task); 1065 1066 dma_release_channel(hdev->dma_lch); 1067 1068 clk_disable_unprepare(hdev->hash_clk); 1069 clk_disable_unprepare(hdev->sys_clk); 1070 1071 return 0; 1072 } 1073 1074 #ifdef CONFIG_PM_SLEEP 1075 static int img_hash_suspend(struct device *dev) 1076 { 1077 struct img_hash_dev *hdev = dev_get_drvdata(dev); 1078 1079 clk_disable_unprepare(hdev->hash_clk); 1080 clk_disable_unprepare(hdev->sys_clk); 1081 1082 return 0; 1083 } 1084 1085 static int img_hash_resume(struct device *dev) 1086 { 1087 struct img_hash_dev *hdev = dev_get_drvdata(dev); 1088 int ret; 1089 1090 ret = clk_prepare_enable(hdev->hash_clk); 1091 if (ret) 1092 return ret; 1093 1094 ret = clk_prepare_enable(hdev->sys_clk); 1095 if (ret) { 1096 clk_disable_unprepare(hdev->hash_clk); 1097 return ret; 1098 } 1099 1100 return 0; 1101 } 1102 #endif /* CONFIG_PM_SLEEP */ 1103 1104 static const struct dev_pm_ops img_hash_pm_ops = { 1105 SET_SYSTEM_SLEEP_PM_OPS(img_hash_suspend, img_hash_resume) 1106 }; 1107 1108 static struct platform_driver img_hash_driver = { 1109 .probe = img_hash_probe, 1110 .remove = img_hash_remove, 1111 .driver = { 1112 .name = "img-hash-accelerator", 1113 .pm = &img_hash_pm_ops, 1114 .of_match_table = of_match_ptr(img_hash_match), 1115 } 1116 }; 1117 module_platform_driver(img_hash_driver); 1118 1119 MODULE_LICENSE("GPL v2"); 1120 MODULE_DESCRIPTION("Imgtec SHA1/224/256 & MD5 hw accelerator driver"); 1121 MODULE_AUTHOR("Will Thomas."); 1122 MODULE_AUTHOR("James Hartley <james.hartley@imgtec.com>"); 1123