1 /* 2 * Cryptographic API. 3 * 4 * Support for OMAP SHA1/MD5 HW acceleration. 5 * 6 * Copyright (c) 2010 Nokia Corporation 7 * Author: Dmitry Kasatkin <dmitry.kasatkin@nokia.com> 8 * 9 * This program is free software; you can redistribute it and/or modify 10 * it under the terms of the GNU General Public License version 2 as published 11 * by the Free Software Foundation. 12 * 13 * Some ideas are from old omap-sha1-md5.c driver. 14 */ 15 16 #define pr_fmt(fmt) "%s: " fmt, __func__ 17 18 #include <linux/err.h> 19 #include <linux/device.h> 20 #include <linux/module.h> 21 #include <linux/init.h> 22 #include <linux/errno.h> 23 #include <linux/interrupt.h> 24 #include <linux/kernel.h> 25 #include <linux/clk.h> 26 #include <linux/irq.h> 27 #include <linux/io.h> 28 #include <linux/platform_device.h> 29 #include <linux/scatterlist.h> 30 #include <linux/dma-mapping.h> 31 #include <linux/delay.h> 32 #include <linux/crypto.h> 33 #include <linux/cryptohash.h> 34 #include <crypto/scatterwalk.h> 35 #include <crypto/algapi.h> 36 #include <crypto/sha.h> 37 #include <crypto/hash.h> 38 #include <crypto/internal/hash.h> 39 40 #include <plat/cpu.h> 41 #include <plat/dma.h> 42 #include <mach/irqs.h> 43 44 #define SHA_REG_DIGEST(x) (0x00 + ((x) * 0x04)) 45 #define SHA_REG_DIN(x) (0x1C + ((x) * 0x04)) 46 47 #define SHA1_MD5_BLOCK_SIZE SHA1_BLOCK_SIZE 48 #define MD5_DIGEST_SIZE 16 49 50 #define SHA_REG_DIGCNT 0x14 51 52 #define SHA_REG_CTRL 0x18 53 #define SHA_REG_CTRL_LENGTH (0xFFFFFFFF << 5) 54 #define SHA_REG_CTRL_CLOSE_HASH (1 << 4) 55 #define SHA_REG_CTRL_ALGO_CONST (1 << 3) 56 #define SHA_REG_CTRL_ALGO (1 << 2) 57 #define SHA_REG_CTRL_INPUT_READY (1 << 1) 58 #define SHA_REG_CTRL_OUTPUT_READY (1 << 0) 59 60 #define SHA_REG_REV 0x5C 61 #define SHA_REG_REV_MAJOR 0xF0 62 #define SHA_REG_REV_MINOR 0x0F 63 64 #define SHA_REG_MASK 0x60 65 #define SHA_REG_MASK_DMA_EN (1 << 3) 66 #define SHA_REG_MASK_IT_EN (1 << 2) 67 #define SHA_REG_MASK_SOFTRESET (1 << 1) 68 #define SHA_REG_AUTOIDLE (1 << 0) 69 70 #define SHA_REG_SYSSTATUS 0x64 71 #define SHA_REG_SYSSTATUS_RESETDONE (1 << 0) 72 73 #define DEFAULT_TIMEOUT_INTERVAL HZ 74 75 #define FLAGS_FINUP 0x0002 76 #define FLAGS_FINAL 0x0004 77 #define FLAGS_SG 0x0008 78 #define FLAGS_SHA1 0x0010 79 #define FLAGS_DMA_ACTIVE 0x0020 80 #define FLAGS_OUTPUT_READY 0x0040 81 #define FLAGS_CLEAN 0x0080 82 #define FLAGS_INIT 0x0100 83 #define FLAGS_CPU 0x0200 84 #define FLAGS_HMAC 0x0400 85 #define FLAGS_ERROR 0x0800 86 #define FLAGS_BUSY 0x1000 87 88 #define OP_UPDATE 1 89 #define OP_FINAL 2 90 91 #define OMAP_ALIGN_MASK (sizeof(u32)-1) 92 #define OMAP_ALIGNED __attribute__((aligned(sizeof(u32)))) 93 94 #define BUFLEN PAGE_SIZE 95 96 struct omap_sham_dev; 97 98 struct omap_sham_reqctx { 99 struct omap_sham_dev *dd; 100 unsigned long flags; 101 unsigned long op; 102 103 u8 digest[SHA1_DIGEST_SIZE] OMAP_ALIGNED; 104 size_t digcnt; 105 size_t bufcnt; 106 size_t buflen; 107 dma_addr_t dma_addr; 108 109 /* walk state */ 110 struct scatterlist *sg; 111 unsigned int offset; /* offset in current sg */ 112 unsigned int total; /* total request */ 113 114 u8 buffer[0] OMAP_ALIGNED; 115 }; 116 117 struct omap_sham_hmac_ctx { 118 struct crypto_shash *shash; 119 u8 ipad[SHA1_MD5_BLOCK_SIZE]; 120 u8 opad[SHA1_MD5_BLOCK_SIZE]; 121 }; 122 123 struct omap_sham_ctx { 124 struct omap_sham_dev *dd; 125 126 unsigned long flags; 127 128 /* fallback stuff */ 129 struct crypto_shash *fallback; 130 131 struct omap_sham_hmac_ctx base[0]; 132 }; 133 134 #define OMAP_SHAM_QUEUE_LENGTH 1 135 136 struct omap_sham_dev { 137 struct list_head list; 138 unsigned long phys_base; 139 struct device *dev; 140 void __iomem *io_base; 141 int irq; 142 struct clk *iclk; 143 spinlock_t lock; 144 int err; 145 int dma; 146 int dma_lch; 147 struct tasklet_struct done_task; 148 struct tasklet_struct queue_task; 149 150 unsigned long flags; 151 struct crypto_queue queue; 152 struct ahash_request *req; 153 }; 154 155 struct omap_sham_drv { 156 struct list_head dev_list; 157 spinlock_t lock; 158 unsigned long flags; 159 }; 160 161 static struct omap_sham_drv sham = { 162 .dev_list = LIST_HEAD_INIT(sham.dev_list), 163 .lock = __SPIN_LOCK_UNLOCKED(sham.lock), 164 }; 165 166 static inline u32 omap_sham_read(struct omap_sham_dev *dd, u32 offset) 167 { 168 return __raw_readl(dd->io_base + offset); 169 } 170 171 static inline void omap_sham_write(struct omap_sham_dev *dd, 172 u32 offset, u32 value) 173 { 174 __raw_writel(value, dd->io_base + offset); 175 } 176 177 static inline void omap_sham_write_mask(struct omap_sham_dev *dd, u32 address, 178 u32 value, u32 mask) 179 { 180 u32 val; 181 182 val = omap_sham_read(dd, address); 183 val &= ~mask; 184 val |= value; 185 omap_sham_write(dd, address, val); 186 } 187 188 static inline int omap_sham_wait(struct omap_sham_dev *dd, u32 offset, u32 bit) 189 { 190 unsigned long timeout = jiffies + DEFAULT_TIMEOUT_INTERVAL; 191 192 while (!(omap_sham_read(dd, offset) & bit)) { 193 if (time_is_before_jiffies(timeout)) 194 return -ETIMEDOUT; 195 } 196 197 return 0; 198 } 199 200 static void omap_sham_copy_hash(struct ahash_request *req, int out) 201 { 202 struct omap_sham_reqctx *ctx = ahash_request_ctx(req); 203 u32 *hash = (u32 *)ctx->digest; 204 int i; 205 206 /* MD5 is almost unused. So copy sha1 size to reduce code */ 207 for (i = 0; i < SHA1_DIGEST_SIZE / sizeof(u32); i++) { 208 if (out) 209 hash[i] = omap_sham_read(ctx->dd, 210 SHA_REG_DIGEST(i)); 211 else 212 omap_sham_write(ctx->dd, 213 SHA_REG_DIGEST(i), hash[i]); 214 } 215 } 216 217 static void omap_sham_copy_ready_hash(struct ahash_request *req) 218 { 219 struct omap_sham_reqctx *ctx = ahash_request_ctx(req); 220 u32 *in = (u32 *)ctx->digest; 221 u32 *hash = (u32 *)req->result; 222 int i; 223 224 if (!hash) 225 return; 226 227 if (likely(ctx->flags & FLAGS_SHA1)) { 228 /* SHA1 results are in big endian */ 229 for (i = 0; i < SHA1_DIGEST_SIZE / sizeof(u32); i++) 230 hash[i] = be32_to_cpu(in[i]); 231 } else { 232 /* MD5 results are in little endian */ 233 for (i = 0; i < MD5_DIGEST_SIZE / sizeof(u32); i++) 234 hash[i] = le32_to_cpu(in[i]); 235 } 236 } 237 238 static int omap_sham_hw_init(struct omap_sham_dev *dd) 239 { 240 clk_enable(dd->iclk); 241 242 if (!(dd->flags & FLAGS_INIT)) { 243 omap_sham_write_mask(dd, SHA_REG_MASK, 244 SHA_REG_MASK_SOFTRESET, SHA_REG_MASK_SOFTRESET); 245 246 if (omap_sham_wait(dd, SHA_REG_SYSSTATUS, 247 SHA_REG_SYSSTATUS_RESETDONE)) 248 return -ETIMEDOUT; 249 250 dd->flags |= FLAGS_INIT; 251 dd->err = 0; 252 } 253 254 return 0; 255 } 256 257 static void omap_sham_write_ctrl(struct omap_sham_dev *dd, size_t length, 258 int final, int dma) 259 { 260 struct omap_sham_reqctx *ctx = ahash_request_ctx(dd->req); 261 u32 val = length << 5, mask; 262 263 if (likely(ctx->digcnt)) 264 omap_sham_write(dd, SHA_REG_DIGCNT, ctx->digcnt); 265 266 omap_sham_write_mask(dd, SHA_REG_MASK, 267 SHA_REG_MASK_IT_EN | (dma ? SHA_REG_MASK_DMA_EN : 0), 268 SHA_REG_MASK_IT_EN | SHA_REG_MASK_DMA_EN); 269 /* 270 * Setting ALGO_CONST only for the first iteration 271 * and CLOSE_HASH only for the last one. 272 */ 273 if (ctx->flags & FLAGS_SHA1) 274 val |= SHA_REG_CTRL_ALGO; 275 if (!ctx->digcnt) 276 val |= SHA_REG_CTRL_ALGO_CONST; 277 if (final) 278 val |= SHA_REG_CTRL_CLOSE_HASH; 279 280 mask = SHA_REG_CTRL_ALGO_CONST | SHA_REG_CTRL_CLOSE_HASH | 281 SHA_REG_CTRL_ALGO | SHA_REG_CTRL_LENGTH; 282 283 omap_sham_write_mask(dd, SHA_REG_CTRL, val, mask); 284 } 285 286 static int omap_sham_xmit_cpu(struct omap_sham_dev *dd, const u8 *buf, 287 size_t length, int final) 288 { 289 struct omap_sham_reqctx *ctx = ahash_request_ctx(dd->req); 290 int count, len32; 291 const u32 *buffer = (const u32 *)buf; 292 293 dev_dbg(dd->dev, "xmit_cpu: digcnt: %d, length: %d, final: %d\n", 294 ctx->digcnt, length, final); 295 296 omap_sham_write_ctrl(dd, length, final, 0); 297 298 /* should be non-zero before next lines to disable clocks later */ 299 ctx->digcnt += length; 300 301 if (omap_sham_wait(dd, SHA_REG_CTRL, SHA_REG_CTRL_INPUT_READY)) 302 return -ETIMEDOUT; 303 304 if (final) 305 ctx->flags |= FLAGS_FINAL; /* catch last interrupt */ 306 307 len32 = DIV_ROUND_UP(length, sizeof(u32)); 308 309 for (count = 0; count < len32; count++) 310 omap_sham_write(dd, SHA_REG_DIN(count), buffer[count]); 311 312 return -EINPROGRESS; 313 } 314 315 static int omap_sham_xmit_dma(struct omap_sham_dev *dd, dma_addr_t dma_addr, 316 size_t length, int final) 317 { 318 struct omap_sham_reqctx *ctx = ahash_request_ctx(dd->req); 319 int len32; 320 321 dev_dbg(dd->dev, "xmit_dma: digcnt: %d, length: %d, final: %d\n", 322 ctx->digcnt, length, final); 323 324 len32 = DIV_ROUND_UP(length, sizeof(u32)); 325 326 omap_set_dma_transfer_params(dd->dma_lch, OMAP_DMA_DATA_TYPE_S32, len32, 327 1, OMAP_DMA_SYNC_PACKET, dd->dma, 328 OMAP_DMA_DST_SYNC_PREFETCH); 329 330 omap_set_dma_src_params(dd->dma_lch, 0, OMAP_DMA_AMODE_POST_INC, 331 dma_addr, 0, 0); 332 333 omap_sham_write_ctrl(dd, length, final, 1); 334 335 ctx->digcnt += length; 336 337 if (final) 338 ctx->flags |= FLAGS_FINAL; /* catch last interrupt */ 339 340 dd->flags |= FLAGS_DMA_ACTIVE; 341 342 omap_start_dma(dd->dma_lch); 343 344 return -EINPROGRESS; 345 } 346 347 static size_t omap_sham_append_buffer(struct omap_sham_reqctx *ctx, 348 const u8 *data, size_t length) 349 { 350 size_t count = min(length, ctx->buflen - ctx->bufcnt); 351 352 count = min(count, ctx->total); 353 if (count <= 0) 354 return 0; 355 memcpy(ctx->buffer + ctx->bufcnt, data, count); 356 ctx->bufcnt += count; 357 358 return count; 359 } 360 361 static size_t omap_sham_append_sg(struct omap_sham_reqctx *ctx) 362 { 363 size_t count; 364 365 while (ctx->sg) { 366 count = omap_sham_append_buffer(ctx, 367 sg_virt(ctx->sg) + ctx->offset, 368 ctx->sg->length - ctx->offset); 369 if (!count) 370 break; 371 ctx->offset += count; 372 ctx->total -= count; 373 if (ctx->offset == ctx->sg->length) { 374 ctx->sg = sg_next(ctx->sg); 375 if (ctx->sg) 376 ctx->offset = 0; 377 else 378 ctx->total = 0; 379 } 380 } 381 382 return 0; 383 } 384 385 static int omap_sham_xmit_dma_map(struct omap_sham_dev *dd, 386 struct omap_sham_reqctx *ctx, 387 size_t length, int final) 388 { 389 ctx->dma_addr = dma_map_single(dd->dev, ctx->buffer, ctx->buflen, 390 DMA_TO_DEVICE); 391 if (dma_mapping_error(dd->dev, ctx->dma_addr)) { 392 dev_err(dd->dev, "dma %u bytes error\n", ctx->buflen); 393 return -EINVAL; 394 } 395 396 ctx->flags &= ~FLAGS_SG; 397 398 /* next call does not fail... so no unmap in the case of error */ 399 return omap_sham_xmit_dma(dd, ctx->dma_addr, length, final); 400 } 401 402 static int omap_sham_update_dma_slow(struct omap_sham_dev *dd) 403 { 404 struct omap_sham_reqctx *ctx = ahash_request_ctx(dd->req); 405 unsigned int final; 406 size_t count; 407 408 omap_sham_append_sg(ctx); 409 410 final = (ctx->flags & FLAGS_FINUP) && !ctx->total; 411 412 dev_dbg(dd->dev, "slow: bufcnt: %u, digcnt: %d, final: %d\n", 413 ctx->bufcnt, ctx->digcnt, final); 414 415 if (final || (ctx->bufcnt == ctx->buflen && ctx->total)) { 416 count = ctx->bufcnt; 417 ctx->bufcnt = 0; 418 return omap_sham_xmit_dma_map(dd, ctx, count, final); 419 } 420 421 return 0; 422 } 423 424 /* Start address alignment */ 425 #define SG_AA(sg) (IS_ALIGNED(sg->offset, sizeof(u32))) 426 /* SHA1 block size alignment */ 427 #define SG_SA(sg) (IS_ALIGNED(sg->length, SHA1_MD5_BLOCK_SIZE)) 428 429 static int omap_sham_update_dma_start(struct omap_sham_dev *dd) 430 { 431 struct omap_sham_reqctx *ctx = ahash_request_ctx(dd->req); 432 unsigned int length, final, tail; 433 struct scatterlist *sg; 434 435 if (!ctx->total) 436 return 0; 437 438 if (ctx->bufcnt || ctx->offset) 439 return omap_sham_update_dma_slow(dd); 440 441 dev_dbg(dd->dev, "fast: digcnt: %d, bufcnt: %u, total: %u\n", 442 ctx->digcnt, ctx->bufcnt, ctx->total); 443 444 sg = ctx->sg; 445 446 if (!SG_AA(sg)) 447 return omap_sham_update_dma_slow(dd); 448 449 if (!sg_is_last(sg) && !SG_SA(sg)) 450 /* size is not SHA1_BLOCK_SIZE aligned */ 451 return omap_sham_update_dma_slow(dd); 452 453 length = min(ctx->total, sg->length); 454 455 if (sg_is_last(sg)) { 456 if (!(ctx->flags & FLAGS_FINUP)) { 457 /* not last sg must be SHA1_MD5_BLOCK_SIZE aligned */ 458 tail = length & (SHA1_MD5_BLOCK_SIZE - 1); 459 /* without finup() we need one block to close hash */ 460 if (!tail) 461 tail = SHA1_MD5_BLOCK_SIZE; 462 length -= tail; 463 } 464 } 465 466 if (!dma_map_sg(dd->dev, ctx->sg, 1, DMA_TO_DEVICE)) { 467 dev_err(dd->dev, "dma_map_sg error\n"); 468 return -EINVAL; 469 } 470 471 ctx->flags |= FLAGS_SG; 472 473 ctx->total -= length; 474 ctx->offset = length; /* offset where to start slow */ 475 476 final = (ctx->flags & FLAGS_FINUP) && !ctx->total; 477 478 /* next call does not fail... so no unmap in the case of error */ 479 return omap_sham_xmit_dma(dd, sg_dma_address(ctx->sg), length, final); 480 } 481 482 static int omap_sham_update_cpu(struct omap_sham_dev *dd) 483 { 484 struct omap_sham_reqctx *ctx = ahash_request_ctx(dd->req); 485 int bufcnt; 486 487 omap_sham_append_sg(ctx); 488 bufcnt = ctx->bufcnt; 489 ctx->bufcnt = 0; 490 491 return omap_sham_xmit_cpu(dd, ctx->buffer, bufcnt, 1); 492 } 493 494 static int omap_sham_update_dma_stop(struct omap_sham_dev *dd) 495 { 496 struct omap_sham_reqctx *ctx = ahash_request_ctx(dd->req); 497 498 omap_stop_dma(dd->dma_lch); 499 if (ctx->flags & FLAGS_SG) { 500 dma_unmap_sg(dd->dev, ctx->sg, 1, DMA_TO_DEVICE); 501 if (ctx->sg->length == ctx->offset) { 502 ctx->sg = sg_next(ctx->sg); 503 if (ctx->sg) 504 ctx->offset = 0; 505 } 506 } else { 507 dma_unmap_single(dd->dev, ctx->dma_addr, ctx->buflen, 508 DMA_TO_DEVICE); 509 } 510 511 return 0; 512 } 513 514 static void omap_sham_cleanup(struct ahash_request *req) 515 { 516 struct omap_sham_reqctx *ctx = ahash_request_ctx(req); 517 struct omap_sham_dev *dd = ctx->dd; 518 unsigned long flags; 519 520 spin_lock_irqsave(&dd->lock, flags); 521 if (ctx->flags & FLAGS_CLEAN) { 522 spin_unlock_irqrestore(&dd->lock, flags); 523 return; 524 } 525 ctx->flags |= FLAGS_CLEAN; 526 spin_unlock_irqrestore(&dd->lock, flags); 527 528 if (ctx->digcnt) 529 omap_sham_copy_ready_hash(req); 530 531 dev_dbg(dd->dev, "digcnt: %d, bufcnt: %d\n", ctx->digcnt, ctx->bufcnt); 532 } 533 534 static int omap_sham_init(struct ahash_request *req) 535 { 536 struct crypto_ahash *tfm = crypto_ahash_reqtfm(req); 537 struct omap_sham_ctx *tctx = crypto_ahash_ctx(tfm); 538 struct omap_sham_reqctx *ctx = ahash_request_ctx(req); 539 struct omap_sham_dev *dd = NULL, *tmp; 540 541 spin_lock_bh(&sham.lock); 542 if (!tctx->dd) { 543 list_for_each_entry(tmp, &sham.dev_list, list) { 544 dd = tmp; 545 break; 546 } 547 tctx->dd = dd; 548 } else { 549 dd = tctx->dd; 550 } 551 spin_unlock_bh(&sham.lock); 552 553 ctx->dd = dd; 554 555 ctx->flags = 0; 556 557 dev_dbg(dd->dev, "init: digest size: %d\n", 558 crypto_ahash_digestsize(tfm)); 559 560 if (crypto_ahash_digestsize(tfm) == SHA1_DIGEST_SIZE) 561 ctx->flags |= FLAGS_SHA1; 562 563 ctx->bufcnt = 0; 564 ctx->digcnt = 0; 565 ctx->buflen = BUFLEN; 566 567 if (tctx->flags & FLAGS_HMAC) { 568 struct omap_sham_hmac_ctx *bctx = tctx->base; 569 570 memcpy(ctx->buffer, bctx->ipad, SHA1_MD5_BLOCK_SIZE); 571 ctx->bufcnt = SHA1_MD5_BLOCK_SIZE; 572 ctx->flags |= FLAGS_HMAC; 573 } 574 575 return 0; 576 577 } 578 579 static int omap_sham_update_req(struct omap_sham_dev *dd) 580 { 581 struct ahash_request *req = dd->req; 582 struct omap_sham_reqctx *ctx = ahash_request_ctx(req); 583 int err; 584 585 dev_dbg(dd->dev, "update_req: total: %u, digcnt: %d, finup: %d\n", 586 ctx->total, ctx->digcnt, (ctx->flags & FLAGS_FINUP) != 0); 587 588 if (ctx->flags & FLAGS_CPU) 589 err = omap_sham_update_cpu(dd); 590 else 591 err = omap_sham_update_dma_start(dd); 592 593 /* wait for dma completion before can take more data */ 594 dev_dbg(dd->dev, "update: err: %d, digcnt: %d\n", err, ctx->digcnt); 595 596 return err; 597 } 598 599 static int omap_sham_final_req(struct omap_sham_dev *dd) 600 { 601 struct ahash_request *req = dd->req; 602 struct omap_sham_reqctx *ctx = ahash_request_ctx(req); 603 int err = 0, use_dma = 1; 604 605 if (ctx->bufcnt <= 64) 606 /* faster to handle last block with cpu */ 607 use_dma = 0; 608 609 if (use_dma) 610 err = omap_sham_xmit_dma_map(dd, ctx, ctx->bufcnt, 1); 611 else 612 err = omap_sham_xmit_cpu(dd, ctx->buffer, ctx->bufcnt, 1); 613 614 ctx->bufcnt = 0; 615 616 dev_dbg(dd->dev, "final_req: err: %d\n", err); 617 618 return err; 619 } 620 621 static int omap_sham_finish_req_hmac(struct ahash_request *req) 622 { 623 struct omap_sham_reqctx *ctx = ahash_request_ctx(req); 624 struct omap_sham_ctx *tctx = crypto_tfm_ctx(req->base.tfm); 625 struct omap_sham_hmac_ctx *bctx = tctx->base; 626 int bs = crypto_shash_blocksize(bctx->shash); 627 int ds = crypto_shash_digestsize(bctx->shash); 628 struct { 629 struct shash_desc shash; 630 char ctx[crypto_shash_descsize(bctx->shash)]; 631 } desc; 632 633 desc.shash.tfm = bctx->shash; 634 desc.shash.flags = 0; /* not CRYPTO_TFM_REQ_MAY_SLEEP */ 635 636 return crypto_shash_init(&desc.shash) ?: 637 crypto_shash_update(&desc.shash, bctx->opad, bs) ?: 638 crypto_shash_finup(&desc.shash, ctx->digest, ds, ctx->digest); 639 } 640 641 static void omap_sham_finish_req(struct ahash_request *req, int err) 642 { 643 struct omap_sham_reqctx *ctx = ahash_request_ctx(req); 644 struct omap_sham_dev *dd = ctx->dd; 645 646 if (!err) { 647 omap_sham_copy_hash(ctx->dd->req, 1); 648 if (ctx->flags & FLAGS_HMAC) 649 err = omap_sham_finish_req_hmac(req); 650 } else { 651 ctx->flags |= FLAGS_ERROR; 652 } 653 654 if ((ctx->flags & FLAGS_FINAL) || err) 655 omap_sham_cleanup(req); 656 657 clk_disable(dd->iclk); 658 dd->flags &= ~FLAGS_BUSY; 659 660 if (req->base.complete) 661 req->base.complete(&req->base, err); 662 } 663 664 static int omap_sham_handle_queue(struct omap_sham_dev *dd, 665 struct ahash_request *req) 666 { 667 struct crypto_async_request *async_req, *backlog; 668 struct omap_sham_reqctx *ctx; 669 struct ahash_request *prev_req; 670 unsigned long flags; 671 int err = 0, ret = 0; 672 673 spin_lock_irqsave(&dd->lock, flags); 674 if (req) 675 ret = ahash_enqueue_request(&dd->queue, req); 676 if (dd->flags & FLAGS_BUSY) { 677 spin_unlock_irqrestore(&dd->lock, flags); 678 return ret; 679 } 680 backlog = crypto_get_backlog(&dd->queue); 681 async_req = crypto_dequeue_request(&dd->queue); 682 if (async_req) 683 dd->flags |= FLAGS_BUSY; 684 spin_unlock_irqrestore(&dd->lock, flags); 685 686 if (!async_req) 687 return ret; 688 689 if (backlog) 690 backlog->complete(backlog, -EINPROGRESS); 691 692 req = ahash_request_cast(async_req); 693 694 prev_req = dd->req; 695 dd->req = req; 696 697 ctx = ahash_request_ctx(req); 698 699 dev_dbg(dd->dev, "handling new req, op: %lu, nbytes: %d\n", 700 ctx->op, req->nbytes); 701 702 703 err = omap_sham_hw_init(dd); 704 if (err) 705 goto err1; 706 707 omap_set_dma_dest_params(dd->dma_lch, 0, 708 OMAP_DMA_AMODE_CONSTANT, 709 dd->phys_base + SHA_REG_DIN(0), 0, 16); 710 711 omap_set_dma_dest_burst_mode(dd->dma_lch, 712 OMAP_DMA_DATA_BURST_16); 713 714 omap_set_dma_src_burst_mode(dd->dma_lch, 715 OMAP_DMA_DATA_BURST_4); 716 717 if (ctx->digcnt) 718 /* request has changed - restore hash */ 719 omap_sham_copy_hash(req, 0); 720 721 if (ctx->op == OP_UPDATE) { 722 err = omap_sham_update_req(dd); 723 if (err != -EINPROGRESS && (ctx->flags & FLAGS_FINUP)) 724 /* no final() after finup() */ 725 err = omap_sham_final_req(dd); 726 } else if (ctx->op == OP_FINAL) { 727 err = omap_sham_final_req(dd); 728 } 729 err1: 730 if (err != -EINPROGRESS) { 731 /* done_task will not finish it, so do it here */ 732 omap_sham_finish_req(req, err); 733 tasklet_schedule(&dd->queue_task); 734 } 735 736 dev_dbg(dd->dev, "exit, err: %d\n", err); 737 738 return ret; 739 } 740 741 static int omap_sham_enqueue(struct ahash_request *req, unsigned int op) 742 { 743 struct omap_sham_reqctx *ctx = ahash_request_ctx(req); 744 struct omap_sham_ctx *tctx = crypto_tfm_ctx(req->base.tfm); 745 struct omap_sham_dev *dd = tctx->dd; 746 747 ctx->op = op; 748 749 return omap_sham_handle_queue(dd, req); 750 } 751 752 static int omap_sham_update(struct ahash_request *req) 753 { 754 struct omap_sham_reqctx *ctx = ahash_request_ctx(req); 755 756 if (!req->nbytes) 757 return 0; 758 759 ctx->total = req->nbytes; 760 ctx->sg = req->src; 761 ctx->offset = 0; 762 763 if (ctx->flags & FLAGS_FINUP) { 764 if ((ctx->digcnt + ctx->bufcnt + ctx->total) < 9) { 765 /* 766 * OMAP HW accel works only with buffers >= 9 767 * will switch to bypass in final() 768 * final has the same request and data 769 */ 770 omap_sham_append_sg(ctx); 771 return 0; 772 } else if (ctx->bufcnt + ctx->total <= SHA1_MD5_BLOCK_SIZE) { 773 /* 774 * faster to use CPU for short transfers 775 */ 776 ctx->flags |= FLAGS_CPU; 777 } 778 } else if (ctx->bufcnt + ctx->total < ctx->buflen) { 779 omap_sham_append_sg(ctx); 780 return 0; 781 } 782 783 return omap_sham_enqueue(req, OP_UPDATE); 784 } 785 786 static int omap_sham_shash_digest(struct crypto_shash *shash, u32 flags, 787 const u8 *data, unsigned int len, u8 *out) 788 { 789 struct { 790 struct shash_desc shash; 791 char ctx[crypto_shash_descsize(shash)]; 792 } desc; 793 794 desc.shash.tfm = shash; 795 desc.shash.flags = flags & CRYPTO_TFM_REQ_MAY_SLEEP; 796 797 return crypto_shash_digest(&desc.shash, data, len, out); 798 } 799 800 static int omap_sham_final_shash(struct ahash_request *req) 801 { 802 struct omap_sham_ctx *tctx = crypto_tfm_ctx(req->base.tfm); 803 struct omap_sham_reqctx *ctx = ahash_request_ctx(req); 804 805 return omap_sham_shash_digest(tctx->fallback, req->base.flags, 806 ctx->buffer, ctx->bufcnt, req->result); 807 } 808 809 static int omap_sham_final(struct ahash_request *req) 810 { 811 struct omap_sham_reqctx *ctx = ahash_request_ctx(req); 812 int err = 0; 813 814 ctx->flags |= FLAGS_FINUP; 815 816 if (!(ctx->flags & FLAGS_ERROR)) { 817 /* OMAP HW accel works only with buffers >= 9 */ 818 /* HMAC is always >= 9 because of ipad */ 819 if ((ctx->digcnt + ctx->bufcnt) < 9) 820 err = omap_sham_final_shash(req); 821 else if (ctx->bufcnt) 822 return omap_sham_enqueue(req, OP_FINAL); 823 } 824 825 omap_sham_cleanup(req); 826 827 return err; 828 } 829 830 static int omap_sham_finup(struct ahash_request *req) 831 { 832 struct omap_sham_reqctx *ctx = ahash_request_ctx(req); 833 int err1, err2; 834 835 ctx->flags |= FLAGS_FINUP; 836 837 err1 = omap_sham_update(req); 838 if (err1 == -EINPROGRESS) 839 return err1; 840 /* 841 * final() has to be always called to cleanup resources 842 * even if udpate() failed, except EINPROGRESS 843 */ 844 err2 = omap_sham_final(req); 845 846 return err1 ?: err2; 847 } 848 849 static int omap_sham_digest(struct ahash_request *req) 850 { 851 return omap_sham_init(req) ?: omap_sham_finup(req); 852 } 853 854 static int omap_sham_setkey(struct crypto_ahash *tfm, const u8 *key, 855 unsigned int keylen) 856 { 857 struct omap_sham_ctx *tctx = crypto_ahash_ctx(tfm); 858 struct omap_sham_hmac_ctx *bctx = tctx->base; 859 int bs = crypto_shash_blocksize(bctx->shash); 860 int ds = crypto_shash_digestsize(bctx->shash); 861 int err, i; 862 err = crypto_shash_setkey(tctx->fallback, key, keylen); 863 if (err) 864 return err; 865 866 if (keylen > bs) { 867 err = omap_sham_shash_digest(bctx->shash, 868 crypto_shash_get_flags(bctx->shash), 869 key, keylen, bctx->ipad); 870 if (err) 871 return err; 872 keylen = ds; 873 } else { 874 memcpy(bctx->ipad, key, keylen); 875 } 876 877 memset(bctx->ipad + keylen, 0, bs - keylen); 878 memcpy(bctx->opad, bctx->ipad, bs); 879 880 for (i = 0; i < bs; i++) { 881 bctx->ipad[i] ^= 0x36; 882 bctx->opad[i] ^= 0x5c; 883 } 884 885 return err; 886 } 887 888 static int omap_sham_cra_init_alg(struct crypto_tfm *tfm, const char *alg_base) 889 { 890 struct omap_sham_ctx *tctx = crypto_tfm_ctx(tfm); 891 const char *alg_name = crypto_tfm_alg_name(tfm); 892 893 pr_info("enter\n"); 894 895 /* Allocate a fallback and abort if it failed. */ 896 tctx->fallback = crypto_alloc_shash(alg_name, 0, 897 CRYPTO_ALG_NEED_FALLBACK); 898 if (IS_ERR(tctx->fallback)) { 899 pr_err("omap-sham: fallback driver '%s' " 900 "could not be loaded.\n", alg_name); 901 return PTR_ERR(tctx->fallback); 902 } 903 904 crypto_ahash_set_reqsize(__crypto_ahash_cast(tfm), 905 sizeof(struct omap_sham_reqctx) + BUFLEN); 906 907 if (alg_base) { 908 struct omap_sham_hmac_ctx *bctx = tctx->base; 909 tctx->flags |= FLAGS_HMAC; 910 bctx->shash = crypto_alloc_shash(alg_base, 0, 911 CRYPTO_ALG_NEED_FALLBACK); 912 if (IS_ERR(bctx->shash)) { 913 pr_err("omap-sham: base driver '%s' " 914 "could not be loaded.\n", alg_base); 915 crypto_free_shash(tctx->fallback); 916 return PTR_ERR(bctx->shash); 917 } 918 919 } 920 921 return 0; 922 } 923 924 static int omap_sham_cra_init(struct crypto_tfm *tfm) 925 { 926 return omap_sham_cra_init_alg(tfm, NULL); 927 } 928 929 static int omap_sham_cra_sha1_init(struct crypto_tfm *tfm) 930 { 931 return omap_sham_cra_init_alg(tfm, "sha1"); 932 } 933 934 static int omap_sham_cra_md5_init(struct crypto_tfm *tfm) 935 { 936 return omap_sham_cra_init_alg(tfm, "md5"); 937 } 938 939 static void omap_sham_cra_exit(struct crypto_tfm *tfm) 940 { 941 struct omap_sham_ctx *tctx = crypto_tfm_ctx(tfm); 942 943 crypto_free_shash(tctx->fallback); 944 tctx->fallback = NULL; 945 946 if (tctx->flags & FLAGS_HMAC) { 947 struct omap_sham_hmac_ctx *bctx = tctx->base; 948 crypto_free_shash(bctx->shash); 949 } 950 } 951 952 static struct ahash_alg algs[] = { 953 { 954 .init = omap_sham_init, 955 .update = omap_sham_update, 956 .final = omap_sham_final, 957 .finup = omap_sham_finup, 958 .digest = omap_sham_digest, 959 .halg.digestsize = SHA1_DIGEST_SIZE, 960 .halg.base = { 961 .cra_name = "sha1", 962 .cra_driver_name = "omap-sha1", 963 .cra_priority = 100, 964 .cra_flags = CRYPTO_ALG_TYPE_AHASH | 965 CRYPTO_ALG_ASYNC | 966 CRYPTO_ALG_NEED_FALLBACK, 967 .cra_blocksize = SHA1_BLOCK_SIZE, 968 .cra_ctxsize = sizeof(struct omap_sham_ctx), 969 .cra_alignmask = 0, 970 .cra_module = THIS_MODULE, 971 .cra_init = omap_sham_cra_init, 972 .cra_exit = omap_sham_cra_exit, 973 } 974 }, 975 { 976 .init = omap_sham_init, 977 .update = omap_sham_update, 978 .final = omap_sham_final, 979 .finup = omap_sham_finup, 980 .digest = omap_sham_digest, 981 .halg.digestsize = MD5_DIGEST_SIZE, 982 .halg.base = { 983 .cra_name = "md5", 984 .cra_driver_name = "omap-md5", 985 .cra_priority = 100, 986 .cra_flags = CRYPTO_ALG_TYPE_AHASH | 987 CRYPTO_ALG_ASYNC | 988 CRYPTO_ALG_NEED_FALLBACK, 989 .cra_blocksize = SHA1_BLOCK_SIZE, 990 .cra_ctxsize = sizeof(struct omap_sham_ctx), 991 .cra_alignmask = OMAP_ALIGN_MASK, 992 .cra_module = THIS_MODULE, 993 .cra_init = omap_sham_cra_init, 994 .cra_exit = omap_sham_cra_exit, 995 } 996 }, 997 { 998 .init = omap_sham_init, 999 .update = omap_sham_update, 1000 .final = omap_sham_final, 1001 .finup = omap_sham_finup, 1002 .digest = omap_sham_digest, 1003 .setkey = omap_sham_setkey, 1004 .halg.digestsize = SHA1_DIGEST_SIZE, 1005 .halg.base = { 1006 .cra_name = "hmac(sha1)", 1007 .cra_driver_name = "omap-hmac-sha1", 1008 .cra_priority = 100, 1009 .cra_flags = CRYPTO_ALG_TYPE_AHASH | 1010 CRYPTO_ALG_ASYNC | 1011 CRYPTO_ALG_NEED_FALLBACK, 1012 .cra_blocksize = SHA1_BLOCK_SIZE, 1013 .cra_ctxsize = sizeof(struct omap_sham_ctx) + 1014 sizeof(struct omap_sham_hmac_ctx), 1015 .cra_alignmask = OMAP_ALIGN_MASK, 1016 .cra_module = THIS_MODULE, 1017 .cra_init = omap_sham_cra_sha1_init, 1018 .cra_exit = omap_sham_cra_exit, 1019 } 1020 }, 1021 { 1022 .init = omap_sham_init, 1023 .update = omap_sham_update, 1024 .final = omap_sham_final, 1025 .finup = omap_sham_finup, 1026 .digest = omap_sham_digest, 1027 .setkey = omap_sham_setkey, 1028 .halg.digestsize = MD5_DIGEST_SIZE, 1029 .halg.base = { 1030 .cra_name = "hmac(md5)", 1031 .cra_driver_name = "omap-hmac-md5", 1032 .cra_priority = 100, 1033 .cra_flags = CRYPTO_ALG_TYPE_AHASH | 1034 CRYPTO_ALG_ASYNC | 1035 CRYPTO_ALG_NEED_FALLBACK, 1036 .cra_blocksize = SHA1_BLOCK_SIZE, 1037 .cra_ctxsize = sizeof(struct omap_sham_ctx) + 1038 sizeof(struct omap_sham_hmac_ctx), 1039 .cra_alignmask = OMAP_ALIGN_MASK, 1040 .cra_module = THIS_MODULE, 1041 .cra_init = omap_sham_cra_md5_init, 1042 .cra_exit = omap_sham_cra_exit, 1043 } 1044 } 1045 }; 1046 1047 static void omap_sham_done_task(unsigned long data) 1048 { 1049 struct omap_sham_dev *dd = (struct omap_sham_dev *)data; 1050 struct ahash_request *req = dd->req; 1051 struct omap_sham_reqctx *ctx = ahash_request_ctx(req); 1052 int ready = 0, err = 0; 1053 1054 if (ctx->flags & FLAGS_OUTPUT_READY) { 1055 ctx->flags &= ~FLAGS_OUTPUT_READY; 1056 ready = 1; 1057 } 1058 1059 if (dd->flags & FLAGS_DMA_ACTIVE) { 1060 dd->flags &= ~FLAGS_DMA_ACTIVE; 1061 omap_sham_update_dma_stop(dd); 1062 if (!dd->err) 1063 err = omap_sham_update_dma_start(dd); 1064 } 1065 1066 err = dd->err ? : err; 1067 1068 if (err != -EINPROGRESS && (ready || err)) { 1069 dev_dbg(dd->dev, "update done: err: %d\n", err); 1070 /* finish curent request */ 1071 omap_sham_finish_req(req, err); 1072 /* start new request */ 1073 omap_sham_handle_queue(dd, NULL); 1074 } 1075 } 1076 1077 static void omap_sham_queue_task(unsigned long data) 1078 { 1079 struct omap_sham_dev *dd = (struct omap_sham_dev *)data; 1080 1081 omap_sham_handle_queue(dd, NULL); 1082 } 1083 1084 static irqreturn_t omap_sham_irq(int irq, void *dev_id) 1085 { 1086 struct omap_sham_dev *dd = dev_id; 1087 struct omap_sham_reqctx *ctx = ahash_request_ctx(dd->req); 1088 1089 if (!ctx) { 1090 dev_err(dd->dev, "unknown interrupt.\n"); 1091 return IRQ_HANDLED; 1092 } 1093 1094 if (unlikely(ctx->flags & FLAGS_FINAL)) 1095 /* final -> allow device to go to power-saving mode */ 1096 omap_sham_write_mask(dd, SHA_REG_CTRL, 0, SHA_REG_CTRL_LENGTH); 1097 1098 omap_sham_write_mask(dd, SHA_REG_CTRL, SHA_REG_CTRL_OUTPUT_READY, 1099 SHA_REG_CTRL_OUTPUT_READY); 1100 omap_sham_read(dd, SHA_REG_CTRL); 1101 1102 ctx->flags |= FLAGS_OUTPUT_READY; 1103 dd->err = 0; 1104 tasklet_schedule(&dd->done_task); 1105 1106 return IRQ_HANDLED; 1107 } 1108 1109 static void omap_sham_dma_callback(int lch, u16 ch_status, void *data) 1110 { 1111 struct omap_sham_dev *dd = data; 1112 1113 if (ch_status != OMAP_DMA_BLOCK_IRQ) { 1114 pr_err("omap-sham DMA error status: 0x%hx\n", ch_status); 1115 dd->err = -EIO; 1116 dd->flags &= ~FLAGS_INIT; /* request to re-initialize */ 1117 } 1118 1119 tasklet_schedule(&dd->done_task); 1120 } 1121 1122 static int omap_sham_dma_init(struct omap_sham_dev *dd) 1123 { 1124 int err; 1125 1126 dd->dma_lch = -1; 1127 1128 err = omap_request_dma(dd->dma, dev_name(dd->dev), 1129 omap_sham_dma_callback, dd, &dd->dma_lch); 1130 if (err) { 1131 dev_err(dd->dev, "Unable to request DMA channel\n"); 1132 return err; 1133 } 1134 1135 return 0; 1136 } 1137 1138 static void omap_sham_dma_cleanup(struct omap_sham_dev *dd) 1139 { 1140 if (dd->dma_lch >= 0) { 1141 omap_free_dma(dd->dma_lch); 1142 dd->dma_lch = -1; 1143 } 1144 } 1145 1146 static int __devinit omap_sham_probe(struct platform_device *pdev) 1147 { 1148 struct omap_sham_dev *dd; 1149 struct device *dev = &pdev->dev; 1150 struct resource *res; 1151 int err, i, j; 1152 1153 dd = kzalloc(sizeof(struct omap_sham_dev), GFP_KERNEL); 1154 if (dd == NULL) { 1155 dev_err(dev, "unable to alloc data struct.\n"); 1156 err = -ENOMEM; 1157 goto data_err; 1158 } 1159 dd->dev = dev; 1160 platform_set_drvdata(pdev, dd); 1161 1162 INIT_LIST_HEAD(&dd->list); 1163 spin_lock_init(&dd->lock); 1164 tasklet_init(&dd->done_task, omap_sham_done_task, (unsigned long)dd); 1165 tasklet_init(&dd->queue_task, omap_sham_queue_task, (unsigned long)dd); 1166 crypto_init_queue(&dd->queue, OMAP_SHAM_QUEUE_LENGTH); 1167 1168 dd->irq = -1; 1169 1170 /* Get the base address */ 1171 res = platform_get_resource(pdev, IORESOURCE_MEM, 0); 1172 if (!res) { 1173 dev_err(dev, "no MEM resource info\n"); 1174 err = -ENODEV; 1175 goto res_err; 1176 } 1177 dd->phys_base = res->start; 1178 1179 /* Get the DMA */ 1180 res = platform_get_resource(pdev, IORESOURCE_DMA, 0); 1181 if (!res) { 1182 dev_err(dev, "no DMA resource info\n"); 1183 err = -ENODEV; 1184 goto res_err; 1185 } 1186 dd->dma = res->start; 1187 1188 /* Get the IRQ */ 1189 dd->irq = platform_get_irq(pdev, 0); 1190 if (dd->irq < 0) { 1191 dev_err(dev, "no IRQ resource info\n"); 1192 err = dd->irq; 1193 goto res_err; 1194 } 1195 1196 err = request_irq(dd->irq, omap_sham_irq, 1197 IRQF_TRIGGER_LOW, dev_name(dev), dd); 1198 if (err) { 1199 dev_err(dev, "unable to request irq.\n"); 1200 goto res_err; 1201 } 1202 1203 err = omap_sham_dma_init(dd); 1204 if (err) 1205 goto dma_err; 1206 1207 /* Initializing the clock */ 1208 dd->iclk = clk_get(dev, "ick"); 1209 if (!dd->iclk) { 1210 dev_err(dev, "clock intialization failed.\n"); 1211 err = -ENODEV; 1212 goto clk_err; 1213 } 1214 1215 dd->io_base = ioremap(dd->phys_base, SZ_4K); 1216 if (!dd->io_base) { 1217 dev_err(dev, "can't ioremap\n"); 1218 err = -ENOMEM; 1219 goto io_err; 1220 } 1221 1222 clk_enable(dd->iclk); 1223 dev_info(dev, "hw accel on OMAP rev %u.%u\n", 1224 (omap_sham_read(dd, SHA_REG_REV) & SHA_REG_REV_MAJOR) >> 4, 1225 omap_sham_read(dd, SHA_REG_REV) & SHA_REG_REV_MINOR); 1226 clk_disable(dd->iclk); 1227 1228 spin_lock(&sham.lock); 1229 list_add_tail(&dd->list, &sham.dev_list); 1230 spin_unlock(&sham.lock); 1231 1232 for (i = 0; i < ARRAY_SIZE(algs); i++) { 1233 err = crypto_register_ahash(&algs[i]); 1234 if (err) 1235 goto err_algs; 1236 } 1237 1238 return 0; 1239 1240 err_algs: 1241 for (j = 0; j < i; j++) 1242 crypto_unregister_ahash(&algs[j]); 1243 iounmap(dd->io_base); 1244 io_err: 1245 clk_put(dd->iclk); 1246 clk_err: 1247 omap_sham_dma_cleanup(dd); 1248 dma_err: 1249 if (dd->irq >= 0) 1250 free_irq(dd->irq, dd); 1251 res_err: 1252 kfree(dd); 1253 dd = NULL; 1254 data_err: 1255 dev_err(dev, "initialization failed.\n"); 1256 1257 return err; 1258 } 1259 1260 static int __devexit omap_sham_remove(struct platform_device *pdev) 1261 { 1262 static struct omap_sham_dev *dd; 1263 int i; 1264 1265 dd = platform_get_drvdata(pdev); 1266 if (!dd) 1267 return -ENODEV; 1268 spin_lock(&sham.lock); 1269 list_del(&dd->list); 1270 spin_unlock(&sham.lock); 1271 for (i = 0; i < ARRAY_SIZE(algs); i++) 1272 crypto_unregister_ahash(&algs[i]); 1273 tasklet_kill(&dd->done_task); 1274 tasklet_kill(&dd->queue_task); 1275 iounmap(dd->io_base); 1276 clk_put(dd->iclk); 1277 omap_sham_dma_cleanup(dd); 1278 if (dd->irq >= 0) 1279 free_irq(dd->irq, dd); 1280 kfree(dd); 1281 dd = NULL; 1282 1283 return 0; 1284 } 1285 1286 static struct platform_driver omap_sham_driver = { 1287 .probe = omap_sham_probe, 1288 .remove = omap_sham_remove, 1289 .driver = { 1290 .name = "omap-sham", 1291 .owner = THIS_MODULE, 1292 }, 1293 }; 1294 1295 static int __init omap_sham_mod_init(void) 1296 { 1297 pr_info("loading %s driver\n", "omap-sham"); 1298 1299 if (!cpu_class_is_omap2() || 1300 omap_type() != OMAP2_DEVICE_TYPE_SEC) { 1301 pr_err("Unsupported cpu\n"); 1302 return -ENODEV; 1303 } 1304 1305 return platform_driver_register(&omap_sham_driver); 1306 } 1307 1308 static void __exit omap_sham_mod_exit(void) 1309 { 1310 platform_driver_unregister(&omap_sham_driver); 1311 } 1312 1313 module_init(omap_sham_mod_init); 1314 module_exit(omap_sham_mod_exit); 1315 1316 MODULE_DESCRIPTION("OMAP SHA1/MD5 hw acceleration support."); 1317 MODULE_LICENSE("GPL v2"); 1318 MODULE_AUTHOR("Dmitry Kasatkin"); 1319