1 /* 2 * Cryptographic API. 3 * 4 * Support for ATMEL SHA1/SHA256 HW acceleration. 5 * 6 * Copyright (c) 2012 Eukréa Electromatique - ATMEL 7 * Author: Nicolas Royer <nicolas@eukrea.com> 8 * 9 * This program is free software; you can redistribute it and/or modify 10 * it under the terms of the GNU General Public License version 2 as published 11 * by the Free Software Foundation. 12 * 13 * Some ideas are from omap-sham.c drivers. 14 */ 15 16 17 #include <linux/kernel.h> 18 #include <linux/module.h> 19 #include <linux/slab.h> 20 #include <linux/err.h> 21 #include <linux/clk.h> 22 #include <linux/io.h> 23 #include <linux/hw_random.h> 24 #include <linux/platform_device.h> 25 26 #include <linux/device.h> 27 #include <linux/init.h> 28 #include <linux/errno.h> 29 #include <linux/interrupt.h> 30 #include <linux/irq.h> 31 #include <linux/scatterlist.h> 32 #include <linux/dma-mapping.h> 33 #include <linux/delay.h> 34 #include <linux/crypto.h> 35 #include <linux/cryptohash.h> 36 #include <crypto/scatterwalk.h> 37 #include <crypto/algapi.h> 38 #include <crypto/sha.h> 39 #include <crypto/hash.h> 40 #include <crypto/internal/hash.h> 41 #include <linux/platform_data/crypto-atmel.h> 42 #include "atmel-sha-regs.h" 43 44 /* SHA flags */ 45 #define SHA_FLAGS_BUSY BIT(0) 46 #define SHA_FLAGS_FINAL BIT(1) 47 #define SHA_FLAGS_DMA_ACTIVE BIT(2) 48 #define SHA_FLAGS_OUTPUT_READY BIT(3) 49 #define SHA_FLAGS_INIT BIT(4) 50 #define SHA_FLAGS_CPU BIT(5) 51 #define SHA_FLAGS_DMA_READY BIT(6) 52 53 #define SHA_FLAGS_FINUP BIT(16) 54 #define SHA_FLAGS_SG BIT(17) 55 #define SHA_FLAGS_SHA1 BIT(18) 56 #define SHA_FLAGS_SHA224 BIT(19) 57 #define SHA_FLAGS_SHA256 BIT(20) 58 #define SHA_FLAGS_SHA384 BIT(21) 59 #define SHA_FLAGS_SHA512 BIT(22) 60 #define SHA_FLAGS_ERROR BIT(23) 61 #define SHA_FLAGS_PAD BIT(24) 62 63 #define SHA_OP_UPDATE 1 64 #define SHA_OP_FINAL 2 65 66 #define SHA_BUFFER_LEN PAGE_SIZE 67 68 #define ATMEL_SHA_DMA_THRESHOLD 56 69 70 struct atmel_sha_caps { 71 bool has_dma; 72 bool has_dualbuff; 73 bool has_sha224; 74 bool has_sha_384_512; 75 }; 76 77 struct atmel_sha_dev; 78 79 struct atmel_sha_reqctx { 80 struct atmel_sha_dev *dd; 81 unsigned long flags; 82 unsigned long op; 83 84 u8 digest[SHA512_DIGEST_SIZE] __aligned(sizeof(u32)); 85 u64 digcnt[2]; 86 size_t bufcnt; 87 size_t buflen; 88 dma_addr_t dma_addr; 89 90 /* walk state */ 91 struct scatterlist *sg; 92 unsigned int offset; /* offset in current sg */ 93 unsigned int total; /* total request */ 94 95 size_t block_size; 96 97 u8 buffer[0] __aligned(sizeof(u32)); 98 }; 99 100 struct atmel_sha_ctx { 101 struct atmel_sha_dev *dd; 102 103 unsigned long flags; 104 105 /* fallback stuff */ 106 struct crypto_shash *fallback; 107 108 }; 109 110 #define ATMEL_SHA_QUEUE_LENGTH 50 111 112 struct atmel_sha_dma { 113 struct dma_chan *chan; 114 struct dma_slave_config dma_conf; 115 }; 116 117 struct atmel_sha_dev { 118 struct list_head list; 119 unsigned long phys_base; 120 struct device *dev; 121 struct clk *iclk; 122 int irq; 123 void __iomem *io_base; 124 125 spinlock_t lock; 126 int err; 127 struct tasklet_struct done_task; 128 129 unsigned long flags; 130 struct crypto_queue queue; 131 struct ahash_request *req; 132 133 struct atmel_sha_dma dma_lch_in; 134 135 struct atmel_sha_caps caps; 136 137 u32 hw_version; 138 }; 139 140 struct atmel_sha_drv { 141 struct list_head dev_list; 142 spinlock_t lock; 143 }; 144 145 static struct atmel_sha_drv atmel_sha = { 146 .dev_list = LIST_HEAD_INIT(atmel_sha.dev_list), 147 .lock = __SPIN_LOCK_UNLOCKED(atmel_sha.lock), 148 }; 149 150 static inline u32 atmel_sha_read(struct atmel_sha_dev *dd, u32 offset) 151 { 152 return readl_relaxed(dd->io_base + offset); 153 } 154 155 static inline void atmel_sha_write(struct atmel_sha_dev *dd, 156 u32 offset, u32 value) 157 { 158 writel_relaxed(value, dd->io_base + offset); 159 } 160 161 static size_t atmel_sha_append_sg(struct atmel_sha_reqctx *ctx) 162 { 163 size_t count; 164 165 while ((ctx->bufcnt < ctx->buflen) && ctx->total) { 166 count = min(ctx->sg->length - ctx->offset, ctx->total); 167 count = min(count, ctx->buflen - ctx->bufcnt); 168 169 if (count <= 0) 170 break; 171 172 scatterwalk_map_and_copy(ctx->buffer + ctx->bufcnt, ctx->sg, 173 ctx->offset, count, 0); 174 175 ctx->bufcnt += count; 176 ctx->offset += count; 177 ctx->total -= count; 178 179 if (ctx->offset == ctx->sg->length) { 180 ctx->sg = sg_next(ctx->sg); 181 if (ctx->sg) 182 ctx->offset = 0; 183 else 184 ctx->total = 0; 185 } 186 } 187 188 return 0; 189 } 190 191 /* 192 * The purpose of this padding is to ensure that the padded message is a 193 * multiple of 512 bits (SHA1/SHA224/SHA256) or 1024 bits (SHA384/SHA512). 194 * The bit "1" is appended at the end of the message followed by 195 * "padlen-1" zero bits. Then a 64 bits block (SHA1/SHA224/SHA256) or 196 * 128 bits block (SHA384/SHA512) equals to the message length in bits 197 * is appended. 198 * 199 * For SHA1/SHA224/SHA256, padlen is calculated as followed: 200 * - if message length < 56 bytes then padlen = 56 - message length 201 * - else padlen = 64 + 56 - message length 202 * 203 * For SHA384/SHA512, padlen is calculated as followed: 204 * - if message length < 112 bytes then padlen = 112 - message length 205 * - else padlen = 128 + 112 - message length 206 */ 207 static void atmel_sha_fill_padding(struct atmel_sha_reqctx *ctx, int length) 208 { 209 unsigned int index, padlen; 210 u64 bits[2]; 211 u64 size[2]; 212 213 size[0] = ctx->digcnt[0]; 214 size[1] = ctx->digcnt[1]; 215 216 size[0] += ctx->bufcnt; 217 if (size[0] < ctx->bufcnt) 218 size[1]++; 219 220 size[0] += length; 221 if (size[0] < length) 222 size[1]++; 223 224 bits[1] = cpu_to_be64(size[0] << 3); 225 bits[0] = cpu_to_be64(size[1] << 3 | size[0] >> 61); 226 227 if (ctx->flags & (SHA_FLAGS_SHA384 | SHA_FLAGS_SHA512)) { 228 index = ctx->bufcnt & 0x7f; 229 padlen = (index < 112) ? (112 - index) : ((128+112) - index); 230 *(ctx->buffer + ctx->bufcnt) = 0x80; 231 memset(ctx->buffer + ctx->bufcnt + 1, 0, padlen-1); 232 memcpy(ctx->buffer + ctx->bufcnt + padlen, bits, 16); 233 ctx->bufcnt += padlen + 16; 234 ctx->flags |= SHA_FLAGS_PAD; 235 } else { 236 index = ctx->bufcnt & 0x3f; 237 padlen = (index < 56) ? (56 - index) : ((64+56) - index); 238 *(ctx->buffer + ctx->bufcnt) = 0x80; 239 memset(ctx->buffer + ctx->bufcnt + 1, 0, padlen-1); 240 memcpy(ctx->buffer + ctx->bufcnt + padlen, &bits[1], 8); 241 ctx->bufcnt += padlen + 8; 242 ctx->flags |= SHA_FLAGS_PAD; 243 } 244 } 245 246 static int atmel_sha_init(struct ahash_request *req) 247 { 248 struct crypto_ahash *tfm = crypto_ahash_reqtfm(req); 249 struct atmel_sha_ctx *tctx = crypto_ahash_ctx(tfm); 250 struct atmel_sha_reqctx *ctx = ahash_request_ctx(req); 251 struct atmel_sha_dev *dd = NULL; 252 struct atmel_sha_dev *tmp; 253 254 spin_lock_bh(&atmel_sha.lock); 255 if (!tctx->dd) { 256 list_for_each_entry(tmp, &atmel_sha.dev_list, list) { 257 dd = tmp; 258 break; 259 } 260 tctx->dd = dd; 261 } else { 262 dd = tctx->dd; 263 } 264 265 spin_unlock_bh(&atmel_sha.lock); 266 267 ctx->dd = dd; 268 269 ctx->flags = 0; 270 271 dev_dbg(dd->dev, "init: digest size: %d\n", 272 crypto_ahash_digestsize(tfm)); 273 274 switch (crypto_ahash_digestsize(tfm)) { 275 case SHA1_DIGEST_SIZE: 276 ctx->flags |= SHA_FLAGS_SHA1; 277 ctx->block_size = SHA1_BLOCK_SIZE; 278 break; 279 case SHA224_DIGEST_SIZE: 280 ctx->flags |= SHA_FLAGS_SHA224; 281 ctx->block_size = SHA224_BLOCK_SIZE; 282 break; 283 case SHA256_DIGEST_SIZE: 284 ctx->flags |= SHA_FLAGS_SHA256; 285 ctx->block_size = SHA256_BLOCK_SIZE; 286 break; 287 case SHA384_DIGEST_SIZE: 288 ctx->flags |= SHA_FLAGS_SHA384; 289 ctx->block_size = SHA384_BLOCK_SIZE; 290 break; 291 case SHA512_DIGEST_SIZE: 292 ctx->flags |= SHA_FLAGS_SHA512; 293 ctx->block_size = SHA512_BLOCK_SIZE; 294 break; 295 default: 296 return -EINVAL; 297 break; 298 } 299 300 ctx->bufcnt = 0; 301 ctx->digcnt[0] = 0; 302 ctx->digcnt[1] = 0; 303 ctx->buflen = SHA_BUFFER_LEN; 304 305 return 0; 306 } 307 308 static void atmel_sha_write_ctrl(struct atmel_sha_dev *dd, int dma) 309 { 310 struct atmel_sha_reqctx *ctx = ahash_request_ctx(dd->req); 311 u32 valcr = 0, valmr = SHA_MR_MODE_AUTO; 312 313 if (likely(dma)) { 314 if (!dd->caps.has_dma) 315 atmel_sha_write(dd, SHA_IER, SHA_INT_TXBUFE); 316 valmr = SHA_MR_MODE_PDC; 317 if (dd->caps.has_dualbuff) 318 valmr |= SHA_MR_DUALBUFF; 319 } else { 320 atmel_sha_write(dd, SHA_IER, SHA_INT_DATARDY); 321 } 322 323 if (ctx->flags & SHA_FLAGS_SHA1) 324 valmr |= SHA_MR_ALGO_SHA1; 325 else if (ctx->flags & SHA_FLAGS_SHA224) 326 valmr |= SHA_MR_ALGO_SHA224; 327 else if (ctx->flags & SHA_FLAGS_SHA256) 328 valmr |= SHA_MR_ALGO_SHA256; 329 else if (ctx->flags & SHA_FLAGS_SHA384) 330 valmr |= SHA_MR_ALGO_SHA384; 331 else if (ctx->flags & SHA_FLAGS_SHA512) 332 valmr |= SHA_MR_ALGO_SHA512; 333 334 /* Setting CR_FIRST only for the first iteration */ 335 if (!(ctx->digcnt[0] || ctx->digcnt[1])) 336 valcr = SHA_CR_FIRST; 337 338 atmel_sha_write(dd, SHA_CR, valcr); 339 atmel_sha_write(dd, SHA_MR, valmr); 340 } 341 342 static int atmel_sha_xmit_cpu(struct atmel_sha_dev *dd, const u8 *buf, 343 size_t length, int final) 344 { 345 struct atmel_sha_reqctx *ctx = ahash_request_ctx(dd->req); 346 int count, len32; 347 const u32 *buffer = (const u32 *)buf; 348 349 dev_dbg(dd->dev, "xmit_cpu: digcnt: 0x%llx 0x%llx, length: %d, final: %d\n", 350 ctx->digcnt[1], ctx->digcnt[0], length, final); 351 352 atmel_sha_write_ctrl(dd, 0); 353 354 /* should be non-zero before next lines to disable clocks later */ 355 ctx->digcnt[0] += length; 356 if (ctx->digcnt[0] < length) 357 ctx->digcnt[1]++; 358 359 if (final) 360 dd->flags |= SHA_FLAGS_FINAL; /* catch last interrupt */ 361 362 len32 = DIV_ROUND_UP(length, sizeof(u32)); 363 364 dd->flags |= SHA_FLAGS_CPU; 365 366 for (count = 0; count < len32; count++) 367 atmel_sha_write(dd, SHA_REG_DIN(count), buffer[count]); 368 369 return -EINPROGRESS; 370 } 371 372 static int atmel_sha_xmit_pdc(struct atmel_sha_dev *dd, dma_addr_t dma_addr1, 373 size_t length1, dma_addr_t dma_addr2, size_t length2, int final) 374 { 375 struct atmel_sha_reqctx *ctx = ahash_request_ctx(dd->req); 376 int len32; 377 378 dev_dbg(dd->dev, "xmit_pdc: digcnt: 0x%llx 0x%llx, length: %d, final: %d\n", 379 ctx->digcnt[1], ctx->digcnt[0], length1, final); 380 381 len32 = DIV_ROUND_UP(length1, sizeof(u32)); 382 atmel_sha_write(dd, SHA_PTCR, SHA_PTCR_TXTDIS); 383 atmel_sha_write(dd, SHA_TPR, dma_addr1); 384 atmel_sha_write(dd, SHA_TCR, len32); 385 386 len32 = DIV_ROUND_UP(length2, sizeof(u32)); 387 atmel_sha_write(dd, SHA_TNPR, dma_addr2); 388 atmel_sha_write(dd, SHA_TNCR, len32); 389 390 atmel_sha_write_ctrl(dd, 1); 391 392 /* should be non-zero before next lines to disable clocks later */ 393 ctx->digcnt[0] += length1; 394 if (ctx->digcnt[0] < length1) 395 ctx->digcnt[1]++; 396 397 if (final) 398 dd->flags |= SHA_FLAGS_FINAL; /* catch last interrupt */ 399 400 dd->flags |= SHA_FLAGS_DMA_ACTIVE; 401 402 /* Start DMA transfer */ 403 atmel_sha_write(dd, SHA_PTCR, SHA_PTCR_TXTEN); 404 405 return -EINPROGRESS; 406 } 407 408 static void atmel_sha_dma_callback(void *data) 409 { 410 struct atmel_sha_dev *dd = data; 411 412 /* dma_lch_in - completed - wait DATRDY */ 413 atmel_sha_write(dd, SHA_IER, SHA_INT_DATARDY); 414 } 415 416 static int atmel_sha_xmit_dma(struct atmel_sha_dev *dd, dma_addr_t dma_addr1, 417 size_t length1, dma_addr_t dma_addr2, size_t length2, int final) 418 { 419 struct atmel_sha_reqctx *ctx = ahash_request_ctx(dd->req); 420 struct dma_async_tx_descriptor *in_desc; 421 struct scatterlist sg[2]; 422 423 dev_dbg(dd->dev, "xmit_dma: digcnt: 0x%llx 0x%llx, length: %d, final: %d\n", 424 ctx->digcnt[1], ctx->digcnt[0], length1, final); 425 426 if (ctx->flags & (SHA_FLAGS_SHA1 | SHA_FLAGS_SHA224 | 427 SHA_FLAGS_SHA256)) { 428 dd->dma_lch_in.dma_conf.src_maxburst = 16; 429 dd->dma_lch_in.dma_conf.dst_maxburst = 16; 430 } else { 431 dd->dma_lch_in.dma_conf.src_maxburst = 32; 432 dd->dma_lch_in.dma_conf.dst_maxburst = 32; 433 } 434 435 dmaengine_slave_config(dd->dma_lch_in.chan, &dd->dma_lch_in.dma_conf); 436 437 if (length2) { 438 sg_init_table(sg, 2); 439 sg_dma_address(&sg[0]) = dma_addr1; 440 sg_dma_len(&sg[0]) = length1; 441 sg_dma_address(&sg[1]) = dma_addr2; 442 sg_dma_len(&sg[1]) = length2; 443 in_desc = dmaengine_prep_slave_sg(dd->dma_lch_in.chan, sg, 2, 444 DMA_MEM_TO_DEV, DMA_PREP_INTERRUPT | DMA_CTRL_ACK); 445 } else { 446 sg_init_table(sg, 1); 447 sg_dma_address(&sg[0]) = dma_addr1; 448 sg_dma_len(&sg[0]) = length1; 449 in_desc = dmaengine_prep_slave_sg(dd->dma_lch_in.chan, sg, 1, 450 DMA_MEM_TO_DEV, DMA_PREP_INTERRUPT | DMA_CTRL_ACK); 451 } 452 if (!in_desc) 453 return -EINVAL; 454 455 in_desc->callback = atmel_sha_dma_callback; 456 in_desc->callback_param = dd; 457 458 atmel_sha_write_ctrl(dd, 1); 459 460 /* should be non-zero before next lines to disable clocks later */ 461 ctx->digcnt[0] += length1; 462 if (ctx->digcnt[0] < length1) 463 ctx->digcnt[1]++; 464 465 if (final) 466 dd->flags |= SHA_FLAGS_FINAL; /* catch last interrupt */ 467 468 dd->flags |= SHA_FLAGS_DMA_ACTIVE; 469 470 /* Start DMA transfer */ 471 dmaengine_submit(in_desc); 472 dma_async_issue_pending(dd->dma_lch_in.chan); 473 474 return -EINPROGRESS; 475 } 476 477 static int atmel_sha_xmit_start(struct atmel_sha_dev *dd, dma_addr_t dma_addr1, 478 size_t length1, dma_addr_t dma_addr2, size_t length2, int final) 479 { 480 if (dd->caps.has_dma) 481 return atmel_sha_xmit_dma(dd, dma_addr1, length1, 482 dma_addr2, length2, final); 483 else 484 return atmel_sha_xmit_pdc(dd, dma_addr1, length1, 485 dma_addr2, length2, final); 486 } 487 488 static int atmel_sha_update_cpu(struct atmel_sha_dev *dd) 489 { 490 struct atmel_sha_reqctx *ctx = ahash_request_ctx(dd->req); 491 int bufcnt; 492 493 atmel_sha_append_sg(ctx); 494 atmel_sha_fill_padding(ctx, 0); 495 bufcnt = ctx->bufcnt; 496 ctx->bufcnt = 0; 497 498 return atmel_sha_xmit_cpu(dd, ctx->buffer, bufcnt, 1); 499 } 500 501 static int atmel_sha_xmit_dma_map(struct atmel_sha_dev *dd, 502 struct atmel_sha_reqctx *ctx, 503 size_t length, int final) 504 { 505 ctx->dma_addr = dma_map_single(dd->dev, ctx->buffer, 506 ctx->buflen + ctx->block_size, DMA_TO_DEVICE); 507 if (dma_mapping_error(dd->dev, ctx->dma_addr)) { 508 dev_err(dd->dev, "dma %u bytes error\n", ctx->buflen + 509 ctx->block_size); 510 return -EINVAL; 511 } 512 513 ctx->flags &= ~SHA_FLAGS_SG; 514 515 /* next call does not fail... so no unmap in the case of error */ 516 return atmel_sha_xmit_start(dd, ctx->dma_addr, length, 0, 0, final); 517 } 518 519 static int atmel_sha_update_dma_slow(struct atmel_sha_dev *dd) 520 { 521 struct atmel_sha_reqctx *ctx = ahash_request_ctx(dd->req); 522 unsigned int final; 523 size_t count; 524 525 atmel_sha_append_sg(ctx); 526 527 final = (ctx->flags & SHA_FLAGS_FINUP) && !ctx->total; 528 529 dev_dbg(dd->dev, "slow: bufcnt: %u, digcnt: 0x%llx 0x%llx, final: %d\n", 530 ctx->bufcnt, ctx->digcnt[1], ctx->digcnt[0], final); 531 532 if (final) 533 atmel_sha_fill_padding(ctx, 0); 534 535 if (final || (ctx->bufcnt == ctx->buflen && ctx->total)) { 536 count = ctx->bufcnt; 537 ctx->bufcnt = 0; 538 return atmel_sha_xmit_dma_map(dd, ctx, count, final); 539 } 540 541 return 0; 542 } 543 544 static int atmel_sha_update_dma_start(struct atmel_sha_dev *dd) 545 { 546 struct atmel_sha_reqctx *ctx = ahash_request_ctx(dd->req); 547 unsigned int length, final, tail; 548 struct scatterlist *sg; 549 unsigned int count; 550 551 if (!ctx->total) 552 return 0; 553 554 if (ctx->bufcnt || ctx->offset) 555 return atmel_sha_update_dma_slow(dd); 556 557 dev_dbg(dd->dev, "fast: digcnt: 0x%llx 0x%llx, bufcnt: %u, total: %u\n", 558 ctx->digcnt[1], ctx->digcnt[0], ctx->bufcnt, ctx->total); 559 560 sg = ctx->sg; 561 562 if (!IS_ALIGNED(sg->offset, sizeof(u32))) 563 return atmel_sha_update_dma_slow(dd); 564 565 if (!sg_is_last(sg) && !IS_ALIGNED(sg->length, ctx->block_size)) 566 /* size is not ctx->block_size aligned */ 567 return atmel_sha_update_dma_slow(dd); 568 569 length = min(ctx->total, sg->length); 570 571 if (sg_is_last(sg)) { 572 if (!(ctx->flags & SHA_FLAGS_FINUP)) { 573 /* not last sg must be ctx->block_size aligned */ 574 tail = length & (ctx->block_size - 1); 575 length -= tail; 576 } 577 } 578 579 ctx->total -= length; 580 ctx->offset = length; /* offset where to start slow */ 581 582 final = (ctx->flags & SHA_FLAGS_FINUP) && !ctx->total; 583 584 /* Add padding */ 585 if (final) { 586 tail = length & (ctx->block_size - 1); 587 length -= tail; 588 ctx->total += tail; 589 ctx->offset = length; /* offset where to start slow */ 590 591 sg = ctx->sg; 592 atmel_sha_append_sg(ctx); 593 594 atmel_sha_fill_padding(ctx, length); 595 596 ctx->dma_addr = dma_map_single(dd->dev, ctx->buffer, 597 ctx->buflen + ctx->block_size, DMA_TO_DEVICE); 598 if (dma_mapping_error(dd->dev, ctx->dma_addr)) { 599 dev_err(dd->dev, "dma %u bytes error\n", 600 ctx->buflen + ctx->block_size); 601 return -EINVAL; 602 } 603 604 if (length == 0) { 605 ctx->flags &= ~SHA_FLAGS_SG; 606 count = ctx->bufcnt; 607 ctx->bufcnt = 0; 608 return atmel_sha_xmit_start(dd, ctx->dma_addr, count, 0, 609 0, final); 610 } else { 611 ctx->sg = sg; 612 if (!dma_map_sg(dd->dev, ctx->sg, 1, 613 DMA_TO_DEVICE)) { 614 dev_err(dd->dev, "dma_map_sg error\n"); 615 return -EINVAL; 616 } 617 618 ctx->flags |= SHA_FLAGS_SG; 619 620 count = ctx->bufcnt; 621 ctx->bufcnt = 0; 622 return atmel_sha_xmit_start(dd, sg_dma_address(ctx->sg), 623 length, ctx->dma_addr, count, final); 624 } 625 } 626 627 if (!dma_map_sg(dd->dev, ctx->sg, 1, DMA_TO_DEVICE)) { 628 dev_err(dd->dev, "dma_map_sg error\n"); 629 return -EINVAL; 630 } 631 632 ctx->flags |= SHA_FLAGS_SG; 633 634 /* next call does not fail... so no unmap in the case of error */ 635 return atmel_sha_xmit_start(dd, sg_dma_address(ctx->sg), length, 0, 636 0, final); 637 } 638 639 static int atmel_sha_update_dma_stop(struct atmel_sha_dev *dd) 640 { 641 struct atmel_sha_reqctx *ctx = ahash_request_ctx(dd->req); 642 643 if (ctx->flags & SHA_FLAGS_SG) { 644 dma_unmap_sg(dd->dev, ctx->sg, 1, DMA_TO_DEVICE); 645 if (ctx->sg->length == ctx->offset) { 646 ctx->sg = sg_next(ctx->sg); 647 if (ctx->sg) 648 ctx->offset = 0; 649 } 650 if (ctx->flags & SHA_FLAGS_PAD) { 651 dma_unmap_single(dd->dev, ctx->dma_addr, 652 ctx->buflen + ctx->block_size, DMA_TO_DEVICE); 653 } 654 } else { 655 dma_unmap_single(dd->dev, ctx->dma_addr, ctx->buflen + 656 ctx->block_size, DMA_TO_DEVICE); 657 } 658 659 return 0; 660 } 661 662 static int atmel_sha_update_req(struct atmel_sha_dev *dd) 663 { 664 struct ahash_request *req = dd->req; 665 struct atmel_sha_reqctx *ctx = ahash_request_ctx(req); 666 int err; 667 668 dev_dbg(dd->dev, "update_req: total: %u, digcnt: 0x%llx 0x%llx\n", 669 ctx->total, ctx->digcnt[1], ctx->digcnt[0]); 670 671 if (ctx->flags & SHA_FLAGS_CPU) 672 err = atmel_sha_update_cpu(dd); 673 else 674 err = atmel_sha_update_dma_start(dd); 675 676 /* wait for dma completion before can take more data */ 677 dev_dbg(dd->dev, "update: err: %d, digcnt: 0x%llx 0%llx\n", 678 err, ctx->digcnt[1], ctx->digcnt[0]); 679 680 return err; 681 } 682 683 static int atmel_sha_final_req(struct atmel_sha_dev *dd) 684 { 685 struct ahash_request *req = dd->req; 686 struct atmel_sha_reqctx *ctx = ahash_request_ctx(req); 687 int err = 0; 688 int count; 689 690 if (ctx->bufcnt >= ATMEL_SHA_DMA_THRESHOLD) { 691 atmel_sha_fill_padding(ctx, 0); 692 count = ctx->bufcnt; 693 ctx->bufcnt = 0; 694 err = atmel_sha_xmit_dma_map(dd, ctx, count, 1); 695 } 696 /* faster to handle last block with cpu */ 697 else { 698 atmel_sha_fill_padding(ctx, 0); 699 count = ctx->bufcnt; 700 ctx->bufcnt = 0; 701 err = atmel_sha_xmit_cpu(dd, ctx->buffer, count, 1); 702 } 703 704 dev_dbg(dd->dev, "final_req: err: %d\n", err); 705 706 return err; 707 } 708 709 static void atmel_sha_copy_hash(struct ahash_request *req) 710 { 711 struct atmel_sha_reqctx *ctx = ahash_request_ctx(req); 712 u32 *hash = (u32 *)ctx->digest; 713 int i; 714 715 if (ctx->flags & SHA_FLAGS_SHA1) 716 for (i = 0; i < SHA1_DIGEST_SIZE / sizeof(u32); i++) 717 hash[i] = atmel_sha_read(ctx->dd, SHA_REG_DIGEST(i)); 718 else if (ctx->flags & SHA_FLAGS_SHA224) 719 for (i = 0; i < SHA224_DIGEST_SIZE / sizeof(u32); i++) 720 hash[i] = atmel_sha_read(ctx->dd, SHA_REG_DIGEST(i)); 721 else if (ctx->flags & SHA_FLAGS_SHA256) 722 for (i = 0; i < SHA256_DIGEST_SIZE / sizeof(u32); i++) 723 hash[i] = atmel_sha_read(ctx->dd, SHA_REG_DIGEST(i)); 724 else if (ctx->flags & SHA_FLAGS_SHA384) 725 for (i = 0; i < SHA384_DIGEST_SIZE / sizeof(u32); i++) 726 hash[i] = atmel_sha_read(ctx->dd, SHA_REG_DIGEST(i)); 727 else 728 for (i = 0; i < SHA512_DIGEST_SIZE / sizeof(u32); i++) 729 hash[i] = atmel_sha_read(ctx->dd, SHA_REG_DIGEST(i)); 730 } 731 732 static void atmel_sha_copy_ready_hash(struct ahash_request *req) 733 { 734 struct atmel_sha_reqctx *ctx = ahash_request_ctx(req); 735 736 if (!req->result) 737 return; 738 739 if (ctx->flags & SHA_FLAGS_SHA1) 740 memcpy(req->result, ctx->digest, SHA1_DIGEST_SIZE); 741 else if (ctx->flags & SHA_FLAGS_SHA224) 742 memcpy(req->result, ctx->digest, SHA224_DIGEST_SIZE); 743 else if (ctx->flags & SHA_FLAGS_SHA256) 744 memcpy(req->result, ctx->digest, SHA256_DIGEST_SIZE); 745 else if (ctx->flags & SHA_FLAGS_SHA384) 746 memcpy(req->result, ctx->digest, SHA384_DIGEST_SIZE); 747 else 748 memcpy(req->result, ctx->digest, SHA512_DIGEST_SIZE); 749 } 750 751 static int atmel_sha_finish(struct ahash_request *req) 752 { 753 struct atmel_sha_reqctx *ctx = ahash_request_ctx(req); 754 struct atmel_sha_dev *dd = ctx->dd; 755 int err = 0; 756 757 if (ctx->digcnt[0] || ctx->digcnt[1]) 758 atmel_sha_copy_ready_hash(req); 759 760 dev_dbg(dd->dev, "digcnt: 0x%llx 0x%llx, bufcnt: %d\n", ctx->digcnt[1], 761 ctx->digcnt[0], ctx->bufcnt); 762 763 return err; 764 } 765 766 static void atmel_sha_finish_req(struct ahash_request *req, int err) 767 { 768 struct atmel_sha_reqctx *ctx = ahash_request_ctx(req); 769 struct atmel_sha_dev *dd = ctx->dd; 770 771 if (!err) { 772 atmel_sha_copy_hash(req); 773 if (SHA_FLAGS_FINAL & dd->flags) 774 err = atmel_sha_finish(req); 775 } else { 776 ctx->flags |= SHA_FLAGS_ERROR; 777 } 778 779 /* atomic operation is not needed here */ 780 dd->flags &= ~(SHA_FLAGS_BUSY | SHA_FLAGS_FINAL | SHA_FLAGS_CPU | 781 SHA_FLAGS_DMA_READY | SHA_FLAGS_OUTPUT_READY); 782 783 clk_disable_unprepare(dd->iclk); 784 785 if (req->base.complete) 786 req->base.complete(&req->base, err); 787 788 /* handle new request */ 789 tasklet_schedule(&dd->done_task); 790 } 791 792 static int atmel_sha_hw_init(struct atmel_sha_dev *dd) 793 { 794 clk_prepare_enable(dd->iclk); 795 796 if (!(SHA_FLAGS_INIT & dd->flags)) { 797 atmel_sha_write(dd, SHA_CR, SHA_CR_SWRST); 798 dd->flags |= SHA_FLAGS_INIT; 799 dd->err = 0; 800 } 801 802 return 0; 803 } 804 805 static inline unsigned int atmel_sha_get_version(struct atmel_sha_dev *dd) 806 { 807 return atmel_sha_read(dd, SHA_HW_VERSION) & 0x00000fff; 808 } 809 810 static void atmel_sha_hw_version_init(struct atmel_sha_dev *dd) 811 { 812 atmel_sha_hw_init(dd); 813 814 dd->hw_version = atmel_sha_get_version(dd); 815 816 dev_info(dd->dev, 817 "version: 0x%x\n", dd->hw_version); 818 819 clk_disable_unprepare(dd->iclk); 820 } 821 822 static int atmel_sha_handle_queue(struct atmel_sha_dev *dd, 823 struct ahash_request *req) 824 { 825 struct crypto_async_request *async_req, *backlog; 826 struct atmel_sha_reqctx *ctx; 827 unsigned long flags; 828 int err = 0, ret = 0; 829 830 spin_lock_irqsave(&dd->lock, flags); 831 if (req) 832 ret = ahash_enqueue_request(&dd->queue, req); 833 834 if (SHA_FLAGS_BUSY & dd->flags) { 835 spin_unlock_irqrestore(&dd->lock, flags); 836 return ret; 837 } 838 839 backlog = crypto_get_backlog(&dd->queue); 840 async_req = crypto_dequeue_request(&dd->queue); 841 if (async_req) 842 dd->flags |= SHA_FLAGS_BUSY; 843 844 spin_unlock_irqrestore(&dd->lock, flags); 845 846 if (!async_req) 847 return ret; 848 849 if (backlog) 850 backlog->complete(backlog, -EINPROGRESS); 851 852 req = ahash_request_cast(async_req); 853 dd->req = req; 854 ctx = ahash_request_ctx(req); 855 856 dev_dbg(dd->dev, "handling new req, op: %lu, nbytes: %d\n", 857 ctx->op, req->nbytes); 858 859 err = atmel_sha_hw_init(dd); 860 861 if (err) 862 goto err1; 863 864 if (ctx->op == SHA_OP_UPDATE) { 865 err = atmel_sha_update_req(dd); 866 if (err != -EINPROGRESS && (ctx->flags & SHA_FLAGS_FINUP)) 867 /* no final() after finup() */ 868 err = atmel_sha_final_req(dd); 869 } else if (ctx->op == SHA_OP_FINAL) { 870 err = atmel_sha_final_req(dd); 871 } 872 873 err1: 874 if (err != -EINPROGRESS) 875 /* done_task will not finish it, so do it here */ 876 atmel_sha_finish_req(req, err); 877 878 dev_dbg(dd->dev, "exit, err: %d\n", err); 879 880 return ret; 881 } 882 883 static int atmel_sha_enqueue(struct ahash_request *req, unsigned int op) 884 { 885 struct atmel_sha_reqctx *ctx = ahash_request_ctx(req); 886 struct atmel_sha_ctx *tctx = crypto_tfm_ctx(req->base.tfm); 887 struct atmel_sha_dev *dd = tctx->dd; 888 889 ctx->op = op; 890 891 return atmel_sha_handle_queue(dd, req); 892 } 893 894 static int atmel_sha_update(struct ahash_request *req) 895 { 896 struct atmel_sha_reqctx *ctx = ahash_request_ctx(req); 897 898 if (!req->nbytes) 899 return 0; 900 901 ctx->total = req->nbytes; 902 ctx->sg = req->src; 903 ctx->offset = 0; 904 905 if (ctx->flags & SHA_FLAGS_FINUP) { 906 if (ctx->bufcnt + ctx->total < ATMEL_SHA_DMA_THRESHOLD) 907 /* faster to use CPU for short transfers */ 908 ctx->flags |= SHA_FLAGS_CPU; 909 } else if (ctx->bufcnt + ctx->total < ctx->buflen) { 910 atmel_sha_append_sg(ctx); 911 return 0; 912 } 913 return atmel_sha_enqueue(req, SHA_OP_UPDATE); 914 } 915 916 static int atmel_sha_final(struct ahash_request *req) 917 { 918 struct atmel_sha_reqctx *ctx = ahash_request_ctx(req); 919 struct atmel_sha_ctx *tctx = crypto_tfm_ctx(req->base.tfm); 920 struct atmel_sha_dev *dd = tctx->dd; 921 922 int err = 0; 923 924 ctx->flags |= SHA_FLAGS_FINUP; 925 926 if (ctx->flags & SHA_FLAGS_ERROR) 927 return 0; /* uncompleted hash is not needed */ 928 929 if (ctx->bufcnt) { 930 return atmel_sha_enqueue(req, SHA_OP_FINAL); 931 } else if (!(ctx->flags & SHA_FLAGS_PAD)) { /* add padding */ 932 err = atmel_sha_hw_init(dd); 933 if (err) 934 goto err1; 935 936 dd->flags |= SHA_FLAGS_BUSY; 937 err = atmel_sha_final_req(dd); 938 } else { 939 /* copy ready hash (+ finalize hmac) */ 940 return atmel_sha_finish(req); 941 } 942 943 err1: 944 if (err != -EINPROGRESS) 945 /* done_task will not finish it, so do it here */ 946 atmel_sha_finish_req(req, err); 947 948 return err; 949 } 950 951 static int atmel_sha_finup(struct ahash_request *req) 952 { 953 struct atmel_sha_reqctx *ctx = ahash_request_ctx(req); 954 int err1, err2; 955 956 ctx->flags |= SHA_FLAGS_FINUP; 957 958 err1 = atmel_sha_update(req); 959 if (err1 == -EINPROGRESS || err1 == -EBUSY) 960 return err1; 961 962 /* 963 * final() has to be always called to cleanup resources 964 * even if udpate() failed, except EINPROGRESS 965 */ 966 err2 = atmel_sha_final(req); 967 968 return err1 ?: err2; 969 } 970 971 static int atmel_sha_digest(struct ahash_request *req) 972 { 973 return atmel_sha_init(req) ?: atmel_sha_finup(req); 974 } 975 976 static int atmel_sha_cra_init_alg(struct crypto_tfm *tfm, const char *alg_base) 977 { 978 struct atmel_sha_ctx *tctx = crypto_tfm_ctx(tfm); 979 const char *alg_name = crypto_tfm_alg_name(tfm); 980 981 /* Allocate a fallback and abort if it failed. */ 982 tctx->fallback = crypto_alloc_shash(alg_name, 0, 983 CRYPTO_ALG_NEED_FALLBACK); 984 if (IS_ERR(tctx->fallback)) { 985 pr_err("atmel-sha: fallback driver '%s' could not be loaded.\n", 986 alg_name); 987 return PTR_ERR(tctx->fallback); 988 } 989 crypto_ahash_set_reqsize(__crypto_ahash_cast(tfm), 990 sizeof(struct atmel_sha_reqctx) + 991 SHA_BUFFER_LEN + SHA512_BLOCK_SIZE); 992 993 return 0; 994 } 995 996 static int atmel_sha_cra_init(struct crypto_tfm *tfm) 997 { 998 return atmel_sha_cra_init_alg(tfm, NULL); 999 } 1000 1001 static void atmel_sha_cra_exit(struct crypto_tfm *tfm) 1002 { 1003 struct atmel_sha_ctx *tctx = crypto_tfm_ctx(tfm); 1004 1005 crypto_free_shash(tctx->fallback); 1006 tctx->fallback = NULL; 1007 } 1008 1009 static struct ahash_alg sha_1_256_algs[] = { 1010 { 1011 .init = atmel_sha_init, 1012 .update = atmel_sha_update, 1013 .final = atmel_sha_final, 1014 .finup = atmel_sha_finup, 1015 .digest = atmel_sha_digest, 1016 .halg = { 1017 .digestsize = SHA1_DIGEST_SIZE, 1018 .base = { 1019 .cra_name = "sha1", 1020 .cra_driver_name = "atmel-sha1", 1021 .cra_priority = 100, 1022 .cra_flags = CRYPTO_ALG_ASYNC | 1023 CRYPTO_ALG_NEED_FALLBACK, 1024 .cra_blocksize = SHA1_BLOCK_SIZE, 1025 .cra_ctxsize = sizeof(struct atmel_sha_ctx), 1026 .cra_alignmask = 0, 1027 .cra_module = THIS_MODULE, 1028 .cra_init = atmel_sha_cra_init, 1029 .cra_exit = atmel_sha_cra_exit, 1030 } 1031 } 1032 }, 1033 { 1034 .init = atmel_sha_init, 1035 .update = atmel_sha_update, 1036 .final = atmel_sha_final, 1037 .finup = atmel_sha_finup, 1038 .digest = atmel_sha_digest, 1039 .halg = { 1040 .digestsize = SHA256_DIGEST_SIZE, 1041 .base = { 1042 .cra_name = "sha256", 1043 .cra_driver_name = "atmel-sha256", 1044 .cra_priority = 100, 1045 .cra_flags = CRYPTO_ALG_ASYNC | 1046 CRYPTO_ALG_NEED_FALLBACK, 1047 .cra_blocksize = SHA256_BLOCK_SIZE, 1048 .cra_ctxsize = sizeof(struct atmel_sha_ctx), 1049 .cra_alignmask = 0, 1050 .cra_module = THIS_MODULE, 1051 .cra_init = atmel_sha_cra_init, 1052 .cra_exit = atmel_sha_cra_exit, 1053 } 1054 } 1055 }, 1056 }; 1057 1058 static struct ahash_alg sha_224_alg = { 1059 .init = atmel_sha_init, 1060 .update = atmel_sha_update, 1061 .final = atmel_sha_final, 1062 .finup = atmel_sha_finup, 1063 .digest = atmel_sha_digest, 1064 .halg = { 1065 .digestsize = SHA224_DIGEST_SIZE, 1066 .base = { 1067 .cra_name = "sha224", 1068 .cra_driver_name = "atmel-sha224", 1069 .cra_priority = 100, 1070 .cra_flags = CRYPTO_ALG_ASYNC | 1071 CRYPTO_ALG_NEED_FALLBACK, 1072 .cra_blocksize = SHA224_BLOCK_SIZE, 1073 .cra_ctxsize = sizeof(struct atmel_sha_ctx), 1074 .cra_alignmask = 0, 1075 .cra_module = THIS_MODULE, 1076 .cra_init = atmel_sha_cra_init, 1077 .cra_exit = atmel_sha_cra_exit, 1078 } 1079 } 1080 }; 1081 1082 static struct ahash_alg sha_384_512_algs[] = { 1083 { 1084 .init = atmel_sha_init, 1085 .update = atmel_sha_update, 1086 .final = atmel_sha_final, 1087 .finup = atmel_sha_finup, 1088 .digest = atmel_sha_digest, 1089 .halg = { 1090 .digestsize = SHA384_DIGEST_SIZE, 1091 .base = { 1092 .cra_name = "sha384", 1093 .cra_driver_name = "atmel-sha384", 1094 .cra_priority = 100, 1095 .cra_flags = CRYPTO_ALG_ASYNC | 1096 CRYPTO_ALG_NEED_FALLBACK, 1097 .cra_blocksize = SHA384_BLOCK_SIZE, 1098 .cra_ctxsize = sizeof(struct atmel_sha_ctx), 1099 .cra_alignmask = 0x3, 1100 .cra_module = THIS_MODULE, 1101 .cra_init = atmel_sha_cra_init, 1102 .cra_exit = atmel_sha_cra_exit, 1103 } 1104 } 1105 }, 1106 { 1107 .init = atmel_sha_init, 1108 .update = atmel_sha_update, 1109 .final = atmel_sha_final, 1110 .finup = atmel_sha_finup, 1111 .digest = atmel_sha_digest, 1112 .halg = { 1113 .digestsize = SHA512_DIGEST_SIZE, 1114 .base = { 1115 .cra_name = "sha512", 1116 .cra_driver_name = "atmel-sha512", 1117 .cra_priority = 100, 1118 .cra_flags = CRYPTO_ALG_ASYNC | 1119 CRYPTO_ALG_NEED_FALLBACK, 1120 .cra_blocksize = SHA512_BLOCK_SIZE, 1121 .cra_ctxsize = sizeof(struct atmel_sha_ctx), 1122 .cra_alignmask = 0x3, 1123 .cra_module = THIS_MODULE, 1124 .cra_init = atmel_sha_cra_init, 1125 .cra_exit = atmel_sha_cra_exit, 1126 } 1127 } 1128 }, 1129 }; 1130 1131 static void atmel_sha_done_task(unsigned long data) 1132 { 1133 struct atmel_sha_dev *dd = (struct atmel_sha_dev *)data; 1134 int err = 0; 1135 1136 if (!(SHA_FLAGS_BUSY & dd->flags)) { 1137 atmel_sha_handle_queue(dd, NULL); 1138 return; 1139 } 1140 1141 if (SHA_FLAGS_CPU & dd->flags) { 1142 if (SHA_FLAGS_OUTPUT_READY & dd->flags) { 1143 dd->flags &= ~SHA_FLAGS_OUTPUT_READY; 1144 goto finish; 1145 } 1146 } else if (SHA_FLAGS_DMA_READY & dd->flags) { 1147 if (SHA_FLAGS_DMA_ACTIVE & dd->flags) { 1148 dd->flags &= ~SHA_FLAGS_DMA_ACTIVE; 1149 atmel_sha_update_dma_stop(dd); 1150 if (dd->err) { 1151 err = dd->err; 1152 goto finish; 1153 } 1154 } 1155 if (SHA_FLAGS_OUTPUT_READY & dd->flags) { 1156 /* hash or semi-hash ready */ 1157 dd->flags &= ~(SHA_FLAGS_DMA_READY | 1158 SHA_FLAGS_OUTPUT_READY); 1159 err = atmel_sha_update_dma_start(dd); 1160 if (err != -EINPROGRESS) 1161 goto finish; 1162 } 1163 } 1164 return; 1165 1166 finish: 1167 /* finish curent request */ 1168 atmel_sha_finish_req(dd->req, err); 1169 } 1170 1171 static irqreturn_t atmel_sha_irq(int irq, void *dev_id) 1172 { 1173 struct atmel_sha_dev *sha_dd = dev_id; 1174 u32 reg; 1175 1176 reg = atmel_sha_read(sha_dd, SHA_ISR); 1177 if (reg & atmel_sha_read(sha_dd, SHA_IMR)) { 1178 atmel_sha_write(sha_dd, SHA_IDR, reg); 1179 if (SHA_FLAGS_BUSY & sha_dd->flags) { 1180 sha_dd->flags |= SHA_FLAGS_OUTPUT_READY; 1181 if (!(SHA_FLAGS_CPU & sha_dd->flags)) 1182 sha_dd->flags |= SHA_FLAGS_DMA_READY; 1183 tasklet_schedule(&sha_dd->done_task); 1184 } else { 1185 dev_warn(sha_dd->dev, "SHA interrupt when no active requests.\n"); 1186 } 1187 return IRQ_HANDLED; 1188 } 1189 1190 return IRQ_NONE; 1191 } 1192 1193 static void atmel_sha_unregister_algs(struct atmel_sha_dev *dd) 1194 { 1195 int i; 1196 1197 for (i = 0; i < ARRAY_SIZE(sha_1_256_algs); i++) 1198 crypto_unregister_ahash(&sha_1_256_algs[i]); 1199 1200 if (dd->caps.has_sha224) 1201 crypto_unregister_ahash(&sha_224_alg); 1202 1203 if (dd->caps.has_sha_384_512) { 1204 for (i = 0; i < ARRAY_SIZE(sha_384_512_algs); i++) 1205 crypto_unregister_ahash(&sha_384_512_algs[i]); 1206 } 1207 } 1208 1209 static int atmel_sha_register_algs(struct atmel_sha_dev *dd) 1210 { 1211 int err, i, j; 1212 1213 for (i = 0; i < ARRAY_SIZE(sha_1_256_algs); i++) { 1214 err = crypto_register_ahash(&sha_1_256_algs[i]); 1215 if (err) 1216 goto err_sha_1_256_algs; 1217 } 1218 1219 if (dd->caps.has_sha224) { 1220 err = crypto_register_ahash(&sha_224_alg); 1221 if (err) 1222 goto err_sha_224_algs; 1223 } 1224 1225 if (dd->caps.has_sha_384_512) { 1226 for (i = 0; i < ARRAY_SIZE(sha_384_512_algs); i++) { 1227 err = crypto_register_ahash(&sha_384_512_algs[i]); 1228 if (err) 1229 goto err_sha_384_512_algs; 1230 } 1231 } 1232 1233 return 0; 1234 1235 err_sha_384_512_algs: 1236 for (j = 0; j < i; j++) 1237 crypto_unregister_ahash(&sha_384_512_algs[j]); 1238 crypto_unregister_ahash(&sha_224_alg); 1239 err_sha_224_algs: 1240 i = ARRAY_SIZE(sha_1_256_algs); 1241 err_sha_1_256_algs: 1242 for (j = 0; j < i; j++) 1243 crypto_unregister_ahash(&sha_1_256_algs[j]); 1244 1245 return err; 1246 } 1247 1248 static bool atmel_sha_filter(struct dma_chan *chan, void *slave) 1249 { 1250 struct at_dma_slave *sl = slave; 1251 1252 if (sl && sl->dma_dev == chan->device->dev) { 1253 chan->private = sl; 1254 return true; 1255 } else { 1256 return false; 1257 } 1258 } 1259 1260 static int atmel_sha_dma_init(struct atmel_sha_dev *dd, 1261 struct crypto_platform_data *pdata) 1262 { 1263 int err = -ENOMEM; 1264 dma_cap_mask_t mask_in; 1265 1266 if (pdata && pdata->dma_slave->rxdata.dma_dev) { 1267 /* Try to grab DMA channel */ 1268 dma_cap_zero(mask_in); 1269 dma_cap_set(DMA_SLAVE, mask_in); 1270 1271 dd->dma_lch_in.chan = dma_request_channel(mask_in, 1272 atmel_sha_filter, &pdata->dma_slave->rxdata); 1273 1274 if (!dd->dma_lch_in.chan) 1275 return err; 1276 1277 dd->dma_lch_in.dma_conf.direction = DMA_MEM_TO_DEV; 1278 dd->dma_lch_in.dma_conf.dst_addr = dd->phys_base + 1279 SHA_REG_DIN(0); 1280 dd->dma_lch_in.dma_conf.src_maxburst = 1; 1281 dd->dma_lch_in.dma_conf.src_addr_width = 1282 DMA_SLAVE_BUSWIDTH_4_BYTES; 1283 dd->dma_lch_in.dma_conf.dst_maxburst = 1; 1284 dd->dma_lch_in.dma_conf.dst_addr_width = 1285 DMA_SLAVE_BUSWIDTH_4_BYTES; 1286 dd->dma_lch_in.dma_conf.device_fc = false; 1287 1288 return 0; 1289 } 1290 1291 return -ENODEV; 1292 } 1293 1294 static void atmel_sha_dma_cleanup(struct atmel_sha_dev *dd) 1295 { 1296 dma_release_channel(dd->dma_lch_in.chan); 1297 } 1298 1299 static void atmel_sha_get_cap(struct atmel_sha_dev *dd) 1300 { 1301 1302 dd->caps.has_dma = 0; 1303 dd->caps.has_dualbuff = 0; 1304 dd->caps.has_sha224 = 0; 1305 dd->caps.has_sha_384_512 = 0; 1306 1307 /* keep only major version number */ 1308 switch (dd->hw_version & 0xff0) { 1309 case 0x410: 1310 dd->caps.has_dma = 1; 1311 dd->caps.has_dualbuff = 1; 1312 dd->caps.has_sha224 = 1; 1313 dd->caps.has_sha_384_512 = 1; 1314 break; 1315 case 0x400: 1316 dd->caps.has_dma = 1; 1317 dd->caps.has_dualbuff = 1; 1318 dd->caps.has_sha224 = 1; 1319 break; 1320 case 0x320: 1321 break; 1322 default: 1323 dev_warn(dd->dev, 1324 "Unmanaged sha version, set minimum capabilities\n"); 1325 break; 1326 } 1327 } 1328 1329 static int atmel_sha_probe(struct platform_device *pdev) 1330 { 1331 struct atmel_sha_dev *sha_dd; 1332 struct crypto_platform_data *pdata; 1333 struct device *dev = &pdev->dev; 1334 struct resource *sha_res; 1335 unsigned long sha_phys_size; 1336 int err; 1337 1338 sha_dd = kzalloc(sizeof(struct atmel_sha_dev), GFP_KERNEL); 1339 if (sha_dd == NULL) { 1340 dev_err(dev, "unable to alloc data struct.\n"); 1341 err = -ENOMEM; 1342 goto sha_dd_err; 1343 } 1344 1345 sha_dd->dev = dev; 1346 1347 platform_set_drvdata(pdev, sha_dd); 1348 1349 INIT_LIST_HEAD(&sha_dd->list); 1350 1351 tasklet_init(&sha_dd->done_task, atmel_sha_done_task, 1352 (unsigned long)sha_dd); 1353 1354 crypto_init_queue(&sha_dd->queue, ATMEL_SHA_QUEUE_LENGTH); 1355 1356 sha_dd->irq = -1; 1357 1358 /* Get the base address */ 1359 sha_res = platform_get_resource(pdev, IORESOURCE_MEM, 0); 1360 if (!sha_res) { 1361 dev_err(dev, "no MEM resource info\n"); 1362 err = -ENODEV; 1363 goto res_err; 1364 } 1365 sha_dd->phys_base = sha_res->start; 1366 sha_phys_size = resource_size(sha_res); 1367 1368 /* Get the IRQ */ 1369 sha_dd->irq = platform_get_irq(pdev, 0); 1370 if (sha_dd->irq < 0) { 1371 dev_err(dev, "no IRQ resource info\n"); 1372 err = sha_dd->irq; 1373 goto res_err; 1374 } 1375 1376 err = request_irq(sha_dd->irq, atmel_sha_irq, IRQF_SHARED, "atmel-sha", 1377 sha_dd); 1378 if (err) { 1379 dev_err(dev, "unable to request sha irq.\n"); 1380 goto res_err; 1381 } 1382 1383 /* Initializing the clock */ 1384 sha_dd->iclk = clk_get(&pdev->dev, "sha_clk"); 1385 if (IS_ERR(sha_dd->iclk)) { 1386 dev_err(dev, "clock intialization failed.\n"); 1387 err = PTR_ERR(sha_dd->iclk); 1388 goto clk_err; 1389 } 1390 1391 sha_dd->io_base = ioremap(sha_dd->phys_base, sha_phys_size); 1392 if (!sha_dd->io_base) { 1393 dev_err(dev, "can't ioremap\n"); 1394 err = -ENOMEM; 1395 goto sha_io_err; 1396 } 1397 1398 atmel_sha_hw_version_init(sha_dd); 1399 1400 atmel_sha_get_cap(sha_dd); 1401 1402 if (sha_dd->caps.has_dma) { 1403 pdata = pdev->dev.platform_data; 1404 if (!pdata) { 1405 dev_err(&pdev->dev, "platform data not available\n"); 1406 err = -ENXIO; 1407 goto err_pdata; 1408 } 1409 err = atmel_sha_dma_init(sha_dd, pdata); 1410 if (err) 1411 goto err_sha_dma; 1412 } 1413 1414 spin_lock(&atmel_sha.lock); 1415 list_add_tail(&sha_dd->list, &atmel_sha.dev_list); 1416 spin_unlock(&atmel_sha.lock); 1417 1418 err = atmel_sha_register_algs(sha_dd); 1419 if (err) 1420 goto err_algs; 1421 1422 dev_info(dev, "Atmel SHA1/SHA256\n"); 1423 1424 return 0; 1425 1426 err_algs: 1427 spin_lock(&atmel_sha.lock); 1428 list_del(&sha_dd->list); 1429 spin_unlock(&atmel_sha.lock); 1430 if (sha_dd->caps.has_dma) 1431 atmel_sha_dma_cleanup(sha_dd); 1432 err_sha_dma: 1433 err_pdata: 1434 iounmap(sha_dd->io_base); 1435 sha_io_err: 1436 clk_put(sha_dd->iclk); 1437 clk_err: 1438 free_irq(sha_dd->irq, sha_dd); 1439 res_err: 1440 tasklet_kill(&sha_dd->done_task); 1441 kfree(sha_dd); 1442 sha_dd = NULL; 1443 sha_dd_err: 1444 dev_err(dev, "initialization failed.\n"); 1445 1446 return err; 1447 } 1448 1449 static int atmel_sha_remove(struct platform_device *pdev) 1450 { 1451 static struct atmel_sha_dev *sha_dd; 1452 1453 sha_dd = platform_get_drvdata(pdev); 1454 if (!sha_dd) 1455 return -ENODEV; 1456 spin_lock(&atmel_sha.lock); 1457 list_del(&sha_dd->list); 1458 spin_unlock(&atmel_sha.lock); 1459 1460 atmel_sha_unregister_algs(sha_dd); 1461 1462 tasklet_kill(&sha_dd->done_task); 1463 1464 if (sha_dd->caps.has_dma) 1465 atmel_sha_dma_cleanup(sha_dd); 1466 1467 iounmap(sha_dd->io_base); 1468 1469 clk_put(sha_dd->iclk); 1470 1471 if (sha_dd->irq >= 0) 1472 free_irq(sha_dd->irq, sha_dd); 1473 1474 kfree(sha_dd); 1475 sha_dd = NULL; 1476 1477 return 0; 1478 } 1479 1480 static struct platform_driver atmel_sha_driver = { 1481 .probe = atmel_sha_probe, 1482 .remove = atmel_sha_remove, 1483 .driver = { 1484 .name = "atmel_sha", 1485 .owner = THIS_MODULE, 1486 }, 1487 }; 1488 1489 module_platform_driver(atmel_sha_driver); 1490 1491 MODULE_DESCRIPTION("Atmel SHA (1/256/224/384/512) hw acceleration support."); 1492 MODULE_LICENSE("GPL v2"); 1493 MODULE_AUTHOR("Nicolas Royer - Eukréa Electromatique"); 1494