1 /* 2 * Cryptographic API. 3 * 4 * Support for ATMEL SHA1/SHA256 HW acceleration. 5 * 6 * Copyright (c) 2012 Eukréa Electromatique - ATMEL 7 * Author: Nicolas Royer <nicolas@eukrea.com> 8 * 9 * This program is free software; you can redistribute it and/or modify 10 * it under the terms of the GNU General Public License version 2 as published 11 * by the Free Software Foundation. 12 * 13 * Some ideas are from omap-sham.c drivers. 14 */ 15 16 17 #include <linux/kernel.h> 18 #include <linux/module.h> 19 #include <linux/slab.h> 20 #include <linux/err.h> 21 #include <linux/clk.h> 22 #include <linux/io.h> 23 #include <linux/hw_random.h> 24 #include <linux/platform_device.h> 25 26 #include <linux/device.h> 27 #include <linux/init.h> 28 #include <linux/errno.h> 29 #include <linux/interrupt.h> 30 #include <linux/irq.h> 31 #include <linux/scatterlist.h> 32 #include <linux/dma-mapping.h> 33 #include <linux/of_device.h> 34 #include <linux/delay.h> 35 #include <linux/crypto.h> 36 #include <linux/cryptohash.h> 37 #include <crypto/scatterwalk.h> 38 #include <crypto/algapi.h> 39 #include <crypto/sha.h> 40 #include <crypto/hash.h> 41 #include <crypto/internal/hash.h> 42 #include <linux/platform_data/crypto-atmel.h> 43 #include "atmel-sha-regs.h" 44 45 /* SHA flags */ 46 #define SHA_FLAGS_BUSY BIT(0) 47 #define SHA_FLAGS_FINAL BIT(1) 48 #define SHA_FLAGS_DMA_ACTIVE BIT(2) 49 #define SHA_FLAGS_OUTPUT_READY BIT(3) 50 #define SHA_FLAGS_INIT BIT(4) 51 #define SHA_FLAGS_CPU BIT(5) 52 #define SHA_FLAGS_DMA_READY BIT(6) 53 54 #define SHA_FLAGS_FINUP BIT(16) 55 #define SHA_FLAGS_SG BIT(17) 56 #define SHA_FLAGS_SHA1 BIT(18) 57 #define SHA_FLAGS_SHA224 BIT(19) 58 #define SHA_FLAGS_SHA256 BIT(20) 59 #define SHA_FLAGS_SHA384 BIT(21) 60 #define SHA_FLAGS_SHA512 BIT(22) 61 #define SHA_FLAGS_ERROR BIT(23) 62 #define SHA_FLAGS_PAD BIT(24) 63 64 #define SHA_OP_UPDATE 1 65 #define SHA_OP_FINAL 2 66 67 #define SHA_BUFFER_LEN PAGE_SIZE 68 69 #define ATMEL_SHA_DMA_THRESHOLD 56 70 71 struct atmel_sha_caps { 72 bool has_dma; 73 bool has_dualbuff; 74 bool has_sha224; 75 bool has_sha_384_512; 76 }; 77 78 struct atmel_sha_dev; 79 80 struct atmel_sha_reqctx { 81 struct atmel_sha_dev *dd; 82 unsigned long flags; 83 unsigned long op; 84 85 u8 digest[SHA512_DIGEST_SIZE] __aligned(sizeof(u32)); 86 u64 digcnt[2]; 87 size_t bufcnt; 88 size_t buflen; 89 dma_addr_t dma_addr; 90 91 /* walk state */ 92 struct scatterlist *sg; 93 unsigned int offset; /* offset in current sg */ 94 unsigned int total; /* total request */ 95 96 size_t block_size; 97 98 u8 buffer[0] __aligned(sizeof(u32)); 99 }; 100 101 struct atmel_sha_ctx { 102 struct atmel_sha_dev *dd; 103 104 unsigned long flags; 105 }; 106 107 #define ATMEL_SHA_QUEUE_LENGTH 50 108 109 struct atmel_sha_dma { 110 struct dma_chan *chan; 111 struct dma_slave_config dma_conf; 112 }; 113 114 struct atmel_sha_dev { 115 struct list_head list; 116 unsigned long phys_base; 117 struct device *dev; 118 struct clk *iclk; 119 int irq; 120 void __iomem *io_base; 121 122 spinlock_t lock; 123 int err; 124 struct tasklet_struct done_task; 125 126 unsigned long flags; 127 struct crypto_queue queue; 128 struct ahash_request *req; 129 130 struct atmel_sha_dma dma_lch_in; 131 132 struct atmel_sha_caps caps; 133 134 u32 hw_version; 135 }; 136 137 struct atmel_sha_drv { 138 struct list_head dev_list; 139 spinlock_t lock; 140 }; 141 142 static struct atmel_sha_drv atmel_sha = { 143 .dev_list = LIST_HEAD_INIT(atmel_sha.dev_list), 144 .lock = __SPIN_LOCK_UNLOCKED(atmel_sha.lock), 145 }; 146 147 static inline u32 atmel_sha_read(struct atmel_sha_dev *dd, u32 offset) 148 { 149 return readl_relaxed(dd->io_base + offset); 150 } 151 152 static inline void atmel_sha_write(struct atmel_sha_dev *dd, 153 u32 offset, u32 value) 154 { 155 writel_relaxed(value, dd->io_base + offset); 156 } 157 158 static size_t atmel_sha_append_sg(struct atmel_sha_reqctx *ctx) 159 { 160 size_t count; 161 162 while ((ctx->bufcnt < ctx->buflen) && ctx->total) { 163 count = min(ctx->sg->length - ctx->offset, ctx->total); 164 count = min(count, ctx->buflen - ctx->bufcnt); 165 166 if (count <= 0) { 167 /* 168 * Check if count <= 0 because the buffer is full or 169 * because the sg length is 0. In the latest case, 170 * check if there is another sg in the list, a 0 length 171 * sg doesn't necessarily mean the end of the sg list. 172 */ 173 if ((ctx->sg->length == 0) && !sg_is_last(ctx->sg)) { 174 ctx->sg = sg_next(ctx->sg); 175 continue; 176 } else { 177 break; 178 } 179 } 180 181 scatterwalk_map_and_copy(ctx->buffer + ctx->bufcnt, ctx->sg, 182 ctx->offset, count, 0); 183 184 ctx->bufcnt += count; 185 ctx->offset += count; 186 ctx->total -= count; 187 188 if (ctx->offset == ctx->sg->length) { 189 ctx->sg = sg_next(ctx->sg); 190 if (ctx->sg) 191 ctx->offset = 0; 192 else 193 ctx->total = 0; 194 } 195 } 196 197 return 0; 198 } 199 200 /* 201 * The purpose of this padding is to ensure that the padded message is a 202 * multiple of 512 bits (SHA1/SHA224/SHA256) or 1024 bits (SHA384/SHA512). 203 * The bit "1" is appended at the end of the message followed by 204 * "padlen-1" zero bits. Then a 64 bits block (SHA1/SHA224/SHA256) or 205 * 128 bits block (SHA384/SHA512) equals to the message length in bits 206 * is appended. 207 * 208 * For SHA1/SHA224/SHA256, padlen is calculated as followed: 209 * - if message length < 56 bytes then padlen = 56 - message length 210 * - else padlen = 64 + 56 - message length 211 * 212 * For SHA384/SHA512, padlen is calculated as followed: 213 * - if message length < 112 bytes then padlen = 112 - message length 214 * - else padlen = 128 + 112 - message length 215 */ 216 static void atmel_sha_fill_padding(struct atmel_sha_reqctx *ctx, int length) 217 { 218 unsigned int index, padlen; 219 u64 bits[2]; 220 u64 size[2]; 221 222 size[0] = ctx->digcnt[0]; 223 size[1] = ctx->digcnt[1]; 224 225 size[0] += ctx->bufcnt; 226 if (size[0] < ctx->bufcnt) 227 size[1]++; 228 229 size[0] += length; 230 if (size[0] < length) 231 size[1]++; 232 233 bits[1] = cpu_to_be64(size[0] << 3); 234 bits[0] = cpu_to_be64(size[1] << 3 | size[0] >> 61); 235 236 if (ctx->flags & (SHA_FLAGS_SHA384 | SHA_FLAGS_SHA512)) { 237 index = ctx->bufcnt & 0x7f; 238 padlen = (index < 112) ? (112 - index) : ((128+112) - index); 239 *(ctx->buffer + ctx->bufcnt) = 0x80; 240 memset(ctx->buffer + ctx->bufcnt + 1, 0, padlen-1); 241 memcpy(ctx->buffer + ctx->bufcnt + padlen, bits, 16); 242 ctx->bufcnt += padlen + 16; 243 ctx->flags |= SHA_FLAGS_PAD; 244 } else { 245 index = ctx->bufcnt & 0x3f; 246 padlen = (index < 56) ? (56 - index) : ((64+56) - index); 247 *(ctx->buffer + ctx->bufcnt) = 0x80; 248 memset(ctx->buffer + ctx->bufcnt + 1, 0, padlen-1); 249 memcpy(ctx->buffer + ctx->bufcnt + padlen, &bits[1], 8); 250 ctx->bufcnt += padlen + 8; 251 ctx->flags |= SHA_FLAGS_PAD; 252 } 253 } 254 255 static int atmel_sha_init(struct ahash_request *req) 256 { 257 struct crypto_ahash *tfm = crypto_ahash_reqtfm(req); 258 struct atmel_sha_ctx *tctx = crypto_ahash_ctx(tfm); 259 struct atmel_sha_reqctx *ctx = ahash_request_ctx(req); 260 struct atmel_sha_dev *dd = NULL; 261 struct atmel_sha_dev *tmp; 262 263 spin_lock_bh(&atmel_sha.lock); 264 if (!tctx->dd) { 265 list_for_each_entry(tmp, &atmel_sha.dev_list, list) { 266 dd = tmp; 267 break; 268 } 269 tctx->dd = dd; 270 } else { 271 dd = tctx->dd; 272 } 273 274 spin_unlock_bh(&atmel_sha.lock); 275 276 ctx->dd = dd; 277 278 ctx->flags = 0; 279 280 dev_dbg(dd->dev, "init: digest size: %d\n", 281 crypto_ahash_digestsize(tfm)); 282 283 switch (crypto_ahash_digestsize(tfm)) { 284 case SHA1_DIGEST_SIZE: 285 ctx->flags |= SHA_FLAGS_SHA1; 286 ctx->block_size = SHA1_BLOCK_SIZE; 287 break; 288 case SHA224_DIGEST_SIZE: 289 ctx->flags |= SHA_FLAGS_SHA224; 290 ctx->block_size = SHA224_BLOCK_SIZE; 291 break; 292 case SHA256_DIGEST_SIZE: 293 ctx->flags |= SHA_FLAGS_SHA256; 294 ctx->block_size = SHA256_BLOCK_SIZE; 295 break; 296 case SHA384_DIGEST_SIZE: 297 ctx->flags |= SHA_FLAGS_SHA384; 298 ctx->block_size = SHA384_BLOCK_SIZE; 299 break; 300 case SHA512_DIGEST_SIZE: 301 ctx->flags |= SHA_FLAGS_SHA512; 302 ctx->block_size = SHA512_BLOCK_SIZE; 303 break; 304 default: 305 return -EINVAL; 306 break; 307 } 308 309 ctx->bufcnt = 0; 310 ctx->digcnt[0] = 0; 311 ctx->digcnt[1] = 0; 312 ctx->buflen = SHA_BUFFER_LEN; 313 314 return 0; 315 } 316 317 static void atmel_sha_write_ctrl(struct atmel_sha_dev *dd, int dma) 318 { 319 struct atmel_sha_reqctx *ctx = ahash_request_ctx(dd->req); 320 u32 valcr = 0, valmr = SHA_MR_MODE_AUTO; 321 322 if (likely(dma)) { 323 if (!dd->caps.has_dma) 324 atmel_sha_write(dd, SHA_IER, SHA_INT_TXBUFE); 325 valmr = SHA_MR_MODE_PDC; 326 if (dd->caps.has_dualbuff) 327 valmr |= SHA_MR_DUALBUFF; 328 } else { 329 atmel_sha_write(dd, SHA_IER, SHA_INT_DATARDY); 330 } 331 332 if (ctx->flags & SHA_FLAGS_SHA1) 333 valmr |= SHA_MR_ALGO_SHA1; 334 else if (ctx->flags & SHA_FLAGS_SHA224) 335 valmr |= SHA_MR_ALGO_SHA224; 336 else if (ctx->flags & SHA_FLAGS_SHA256) 337 valmr |= SHA_MR_ALGO_SHA256; 338 else if (ctx->flags & SHA_FLAGS_SHA384) 339 valmr |= SHA_MR_ALGO_SHA384; 340 else if (ctx->flags & SHA_FLAGS_SHA512) 341 valmr |= SHA_MR_ALGO_SHA512; 342 343 /* Setting CR_FIRST only for the first iteration */ 344 if (!(ctx->digcnt[0] || ctx->digcnt[1])) 345 valcr = SHA_CR_FIRST; 346 347 atmel_sha_write(dd, SHA_CR, valcr); 348 atmel_sha_write(dd, SHA_MR, valmr); 349 } 350 351 static int atmel_sha_xmit_cpu(struct atmel_sha_dev *dd, const u8 *buf, 352 size_t length, int final) 353 { 354 struct atmel_sha_reqctx *ctx = ahash_request_ctx(dd->req); 355 int count, len32; 356 const u32 *buffer = (const u32 *)buf; 357 358 dev_dbg(dd->dev, "xmit_cpu: digcnt: 0x%llx 0x%llx, length: %d, final: %d\n", 359 ctx->digcnt[1], ctx->digcnt[0], length, final); 360 361 atmel_sha_write_ctrl(dd, 0); 362 363 /* should be non-zero before next lines to disable clocks later */ 364 ctx->digcnt[0] += length; 365 if (ctx->digcnt[0] < length) 366 ctx->digcnt[1]++; 367 368 if (final) 369 dd->flags |= SHA_FLAGS_FINAL; /* catch last interrupt */ 370 371 len32 = DIV_ROUND_UP(length, sizeof(u32)); 372 373 dd->flags |= SHA_FLAGS_CPU; 374 375 for (count = 0; count < len32; count++) 376 atmel_sha_write(dd, SHA_REG_DIN(count), buffer[count]); 377 378 return -EINPROGRESS; 379 } 380 381 static int atmel_sha_xmit_pdc(struct atmel_sha_dev *dd, dma_addr_t dma_addr1, 382 size_t length1, dma_addr_t dma_addr2, size_t length2, int final) 383 { 384 struct atmel_sha_reqctx *ctx = ahash_request_ctx(dd->req); 385 int len32; 386 387 dev_dbg(dd->dev, "xmit_pdc: digcnt: 0x%llx 0x%llx, length: %d, final: %d\n", 388 ctx->digcnt[1], ctx->digcnt[0], length1, final); 389 390 len32 = DIV_ROUND_UP(length1, sizeof(u32)); 391 atmel_sha_write(dd, SHA_PTCR, SHA_PTCR_TXTDIS); 392 atmel_sha_write(dd, SHA_TPR, dma_addr1); 393 atmel_sha_write(dd, SHA_TCR, len32); 394 395 len32 = DIV_ROUND_UP(length2, sizeof(u32)); 396 atmel_sha_write(dd, SHA_TNPR, dma_addr2); 397 atmel_sha_write(dd, SHA_TNCR, len32); 398 399 atmel_sha_write_ctrl(dd, 1); 400 401 /* should be non-zero before next lines to disable clocks later */ 402 ctx->digcnt[0] += length1; 403 if (ctx->digcnt[0] < length1) 404 ctx->digcnt[1]++; 405 406 if (final) 407 dd->flags |= SHA_FLAGS_FINAL; /* catch last interrupt */ 408 409 dd->flags |= SHA_FLAGS_DMA_ACTIVE; 410 411 /* Start DMA transfer */ 412 atmel_sha_write(dd, SHA_PTCR, SHA_PTCR_TXTEN); 413 414 return -EINPROGRESS; 415 } 416 417 static void atmel_sha_dma_callback(void *data) 418 { 419 struct atmel_sha_dev *dd = data; 420 421 /* dma_lch_in - completed - wait DATRDY */ 422 atmel_sha_write(dd, SHA_IER, SHA_INT_DATARDY); 423 } 424 425 static int atmel_sha_xmit_dma(struct atmel_sha_dev *dd, dma_addr_t dma_addr1, 426 size_t length1, dma_addr_t dma_addr2, size_t length2, int final) 427 { 428 struct atmel_sha_reqctx *ctx = ahash_request_ctx(dd->req); 429 struct dma_async_tx_descriptor *in_desc; 430 struct scatterlist sg[2]; 431 432 dev_dbg(dd->dev, "xmit_dma: digcnt: 0x%llx 0x%llx, length: %d, final: %d\n", 433 ctx->digcnt[1], ctx->digcnt[0], length1, final); 434 435 dd->dma_lch_in.dma_conf.src_maxburst = 16; 436 dd->dma_lch_in.dma_conf.dst_maxburst = 16; 437 438 dmaengine_slave_config(dd->dma_lch_in.chan, &dd->dma_lch_in.dma_conf); 439 440 if (length2) { 441 sg_init_table(sg, 2); 442 sg_dma_address(&sg[0]) = dma_addr1; 443 sg_dma_len(&sg[0]) = length1; 444 sg_dma_address(&sg[1]) = dma_addr2; 445 sg_dma_len(&sg[1]) = length2; 446 in_desc = dmaengine_prep_slave_sg(dd->dma_lch_in.chan, sg, 2, 447 DMA_MEM_TO_DEV, DMA_PREP_INTERRUPT | DMA_CTRL_ACK); 448 } else { 449 sg_init_table(sg, 1); 450 sg_dma_address(&sg[0]) = dma_addr1; 451 sg_dma_len(&sg[0]) = length1; 452 in_desc = dmaengine_prep_slave_sg(dd->dma_lch_in.chan, sg, 1, 453 DMA_MEM_TO_DEV, DMA_PREP_INTERRUPT | DMA_CTRL_ACK); 454 } 455 if (!in_desc) 456 return -EINVAL; 457 458 in_desc->callback = atmel_sha_dma_callback; 459 in_desc->callback_param = dd; 460 461 atmel_sha_write_ctrl(dd, 1); 462 463 /* should be non-zero before next lines to disable clocks later */ 464 ctx->digcnt[0] += length1; 465 if (ctx->digcnt[0] < length1) 466 ctx->digcnt[1]++; 467 468 if (final) 469 dd->flags |= SHA_FLAGS_FINAL; /* catch last interrupt */ 470 471 dd->flags |= SHA_FLAGS_DMA_ACTIVE; 472 473 /* Start DMA transfer */ 474 dmaengine_submit(in_desc); 475 dma_async_issue_pending(dd->dma_lch_in.chan); 476 477 return -EINPROGRESS; 478 } 479 480 static int atmel_sha_xmit_start(struct atmel_sha_dev *dd, dma_addr_t dma_addr1, 481 size_t length1, dma_addr_t dma_addr2, size_t length2, int final) 482 { 483 if (dd->caps.has_dma) 484 return atmel_sha_xmit_dma(dd, dma_addr1, length1, 485 dma_addr2, length2, final); 486 else 487 return atmel_sha_xmit_pdc(dd, dma_addr1, length1, 488 dma_addr2, length2, final); 489 } 490 491 static int atmel_sha_update_cpu(struct atmel_sha_dev *dd) 492 { 493 struct atmel_sha_reqctx *ctx = ahash_request_ctx(dd->req); 494 int bufcnt; 495 496 atmel_sha_append_sg(ctx); 497 atmel_sha_fill_padding(ctx, 0); 498 bufcnt = ctx->bufcnt; 499 ctx->bufcnt = 0; 500 501 return atmel_sha_xmit_cpu(dd, ctx->buffer, bufcnt, 1); 502 } 503 504 static int atmel_sha_xmit_dma_map(struct atmel_sha_dev *dd, 505 struct atmel_sha_reqctx *ctx, 506 size_t length, int final) 507 { 508 ctx->dma_addr = dma_map_single(dd->dev, ctx->buffer, 509 ctx->buflen + ctx->block_size, DMA_TO_DEVICE); 510 if (dma_mapping_error(dd->dev, ctx->dma_addr)) { 511 dev_err(dd->dev, "dma %u bytes error\n", ctx->buflen + 512 ctx->block_size); 513 return -EINVAL; 514 } 515 516 ctx->flags &= ~SHA_FLAGS_SG; 517 518 /* next call does not fail... so no unmap in the case of error */ 519 return atmel_sha_xmit_start(dd, ctx->dma_addr, length, 0, 0, final); 520 } 521 522 static int atmel_sha_update_dma_slow(struct atmel_sha_dev *dd) 523 { 524 struct atmel_sha_reqctx *ctx = ahash_request_ctx(dd->req); 525 unsigned int final; 526 size_t count; 527 528 atmel_sha_append_sg(ctx); 529 530 final = (ctx->flags & SHA_FLAGS_FINUP) && !ctx->total; 531 532 dev_dbg(dd->dev, "slow: bufcnt: %u, digcnt: 0x%llx 0x%llx, final: %d\n", 533 ctx->bufcnt, ctx->digcnt[1], ctx->digcnt[0], final); 534 535 if (final) 536 atmel_sha_fill_padding(ctx, 0); 537 538 if (final || (ctx->bufcnt == ctx->buflen)) { 539 count = ctx->bufcnt; 540 ctx->bufcnt = 0; 541 return atmel_sha_xmit_dma_map(dd, ctx, count, final); 542 } 543 544 return 0; 545 } 546 547 static int atmel_sha_update_dma_start(struct atmel_sha_dev *dd) 548 { 549 struct atmel_sha_reqctx *ctx = ahash_request_ctx(dd->req); 550 unsigned int length, final, tail; 551 struct scatterlist *sg; 552 unsigned int count; 553 554 if (!ctx->total) 555 return 0; 556 557 if (ctx->bufcnt || ctx->offset) 558 return atmel_sha_update_dma_slow(dd); 559 560 dev_dbg(dd->dev, "fast: digcnt: 0x%llx 0x%llx, bufcnt: %u, total: %u\n", 561 ctx->digcnt[1], ctx->digcnt[0], ctx->bufcnt, ctx->total); 562 563 sg = ctx->sg; 564 565 if (!IS_ALIGNED(sg->offset, sizeof(u32))) 566 return atmel_sha_update_dma_slow(dd); 567 568 if (!sg_is_last(sg) && !IS_ALIGNED(sg->length, ctx->block_size)) 569 /* size is not ctx->block_size aligned */ 570 return atmel_sha_update_dma_slow(dd); 571 572 length = min(ctx->total, sg->length); 573 574 if (sg_is_last(sg)) { 575 if (!(ctx->flags & SHA_FLAGS_FINUP)) { 576 /* not last sg must be ctx->block_size aligned */ 577 tail = length & (ctx->block_size - 1); 578 length -= tail; 579 } 580 } 581 582 ctx->total -= length; 583 ctx->offset = length; /* offset where to start slow */ 584 585 final = (ctx->flags & SHA_FLAGS_FINUP) && !ctx->total; 586 587 /* Add padding */ 588 if (final) { 589 tail = length & (ctx->block_size - 1); 590 length -= tail; 591 ctx->total += tail; 592 ctx->offset = length; /* offset where to start slow */ 593 594 sg = ctx->sg; 595 atmel_sha_append_sg(ctx); 596 597 atmel_sha_fill_padding(ctx, length); 598 599 ctx->dma_addr = dma_map_single(dd->dev, ctx->buffer, 600 ctx->buflen + ctx->block_size, DMA_TO_DEVICE); 601 if (dma_mapping_error(dd->dev, ctx->dma_addr)) { 602 dev_err(dd->dev, "dma %u bytes error\n", 603 ctx->buflen + ctx->block_size); 604 return -EINVAL; 605 } 606 607 if (length == 0) { 608 ctx->flags &= ~SHA_FLAGS_SG; 609 count = ctx->bufcnt; 610 ctx->bufcnt = 0; 611 return atmel_sha_xmit_start(dd, ctx->dma_addr, count, 0, 612 0, final); 613 } else { 614 ctx->sg = sg; 615 if (!dma_map_sg(dd->dev, ctx->sg, 1, 616 DMA_TO_DEVICE)) { 617 dev_err(dd->dev, "dma_map_sg error\n"); 618 return -EINVAL; 619 } 620 621 ctx->flags |= SHA_FLAGS_SG; 622 623 count = ctx->bufcnt; 624 ctx->bufcnt = 0; 625 return atmel_sha_xmit_start(dd, sg_dma_address(ctx->sg), 626 length, ctx->dma_addr, count, final); 627 } 628 } 629 630 if (!dma_map_sg(dd->dev, ctx->sg, 1, DMA_TO_DEVICE)) { 631 dev_err(dd->dev, "dma_map_sg error\n"); 632 return -EINVAL; 633 } 634 635 ctx->flags |= SHA_FLAGS_SG; 636 637 /* next call does not fail... so no unmap in the case of error */ 638 return atmel_sha_xmit_start(dd, sg_dma_address(ctx->sg), length, 0, 639 0, final); 640 } 641 642 static int atmel_sha_update_dma_stop(struct atmel_sha_dev *dd) 643 { 644 struct atmel_sha_reqctx *ctx = ahash_request_ctx(dd->req); 645 646 if (ctx->flags & SHA_FLAGS_SG) { 647 dma_unmap_sg(dd->dev, ctx->sg, 1, DMA_TO_DEVICE); 648 if (ctx->sg->length == ctx->offset) { 649 ctx->sg = sg_next(ctx->sg); 650 if (ctx->sg) 651 ctx->offset = 0; 652 } 653 if (ctx->flags & SHA_FLAGS_PAD) { 654 dma_unmap_single(dd->dev, ctx->dma_addr, 655 ctx->buflen + ctx->block_size, DMA_TO_DEVICE); 656 } 657 } else { 658 dma_unmap_single(dd->dev, ctx->dma_addr, ctx->buflen + 659 ctx->block_size, DMA_TO_DEVICE); 660 } 661 662 return 0; 663 } 664 665 static int atmel_sha_update_req(struct atmel_sha_dev *dd) 666 { 667 struct ahash_request *req = dd->req; 668 struct atmel_sha_reqctx *ctx = ahash_request_ctx(req); 669 int err; 670 671 dev_dbg(dd->dev, "update_req: total: %u, digcnt: 0x%llx 0x%llx\n", 672 ctx->total, ctx->digcnt[1], ctx->digcnt[0]); 673 674 if (ctx->flags & SHA_FLAGS_CPU) 675 err = atmel_sha_update_cpu(dd); 676 else 677 err = atmel_sha_update_dma_start(dd); 678 679 /* wait for dma completion before can take more data */ 680 dev_dbg(dd->dev, "update: err: %d, digcnt: 0x%llx 0%llx\n", 681 err, ctx->digcnt[1], ctx->digcnt[0]); 682 683 return err; 684 } 685 686 static int atmel_sha_final_req(struct atmel_sha_dev *dd) 687 { 688 struct ahash_request *req = dd->req; 689 struct atmel_sha_reqctx *ctx = ahash_request_ctx(req); 690 int err = 0; 691 int count; 692 693 if (ctx->bufcnt >= ATMEL_SHA_DMA_THRESHOLD) { 694 atmel_sha_fill_padding(ctx, 0); 695 count = ctx->bufcnt; 696 ctx->bufcnt = 0; 697 err = atmel_sha_xmit_dma_map(dd, ctx, count, 1); 698 } 699 /* faster to handle last block with cpu */ 700 else { 701 atmel_sha_fill_padding(ctx, 0); 702 count = ctx->bufcnt; 703 ctx->bufcnt = 0; 704 err = atmel_sha_xmit_cpu(dd, ctx->buffer, count, 1); 705 } 706 707 dev_dbg(dd->dev, "final_req: err: %d\n", err); 708 709 return err; 710 } 711 712 static void atmel_sha_copy_hash(struct ahash_request *req) 713 { 714 struct atmel_sha_reqctx *ctx = ahash_request_ctx(req); 715 u32 *hash = (u32 *)ctx->digest; 716 int i; 717 718 if (ctx->flags & SHA_FLAGS_SHA1) 719 for (i = 0; i < SHA1_DIGEST_SIZE / sizeof(u32); i++) 720 hash[i] = atmel_sha_read(ctx->dd, SHA_REG_DIGEST(i)); 721 else if (ctx->flags & SHA_FLAGS_SHA224) 722 for (i = 0; i < SHA224_DIGEST_SIZE / sizeof(u32); i++) 723 hash[i] = atmel_sha_read(ctx->dd, SHA_REG_DIGEST(i)); 724 else if (ctx->flags & SHA_FLAGS_SHA256) 725 for (i = 0; i < SHA256_DIGEST_SIZE / sizeof(u32); i++) 726 hash[i] = atmel_sha_read(ctx->dd, SHA_REG_DIGEST(i)); 727 else if (ctx->flags & SHA_FLAGS_SHA384) 728 for (i = 0; i < SHA384_DIGEST_SIZE / sizeof(u32); i++) 729 hash[i] = atmel_sha_read(ctx->dd, SHA_REG_DIGEST(i)); 730 else 731 for (i = 0; i < SHA512_DIGEST_SIZE / sizeof(u32); i++) 732 hash[i] = atmel_sha_read(ctx->dd, SHA_REG_DIGEST(i)); 733 } 734 735 static void atmel_sha_copy_ready_hash(struct ahash_request *req) 736 { 737 struct atmel_sha_reqctx *ctx = ahash_request_ctx(req); 738 739 if (!req->result) 740 return; 741 742 if (ctx->flags & SHA_FLAGS_SHA1) 743 memcpy(req->result, ctx->digest, SHA1_DIGEST_SIZE); 744 else if (ctx->flags & SHA_FLAGS_SHA224) 745 memcpy(req->result, ctx->digest, SHA224_DIGEST_SIZE); 746 else if (ctx->flags & SHA_FLAGS_SHA256) 747 memcpy(req->result, ctx->digest, SHA256_DIGEST_SIZE); 748 else if (ctx->flags & SHA_FLAGS_SHA384) 749 memcpy(req->result, ctx->digest, SHA384_DIGEST_SIZE); 750 else 751 memcpy(req->result, ctx->digest, SHA512_DIGEST_SIZE); 752 } 753 754 static int atmel_sha_finish(struct ahash_request *req) 755 { 756 struct atmel_sha_reqctx *ctx = ahash_request_ctx(req); 757 struct atmel_sha_dev *dd = ctx->dd; 758 int err = 0; 759 760 if (ctx->digcnt[0] || ctx->digcnt[1]) 761 atmel_sha_copy_ready_hash(req); 762 763 dev_dbg(dd->dev, "digcnt: 0x%llx 0x%llx, bufcnt: %d\n", ctx->digcnt[1], 764 ctx->digcnt[0], ctx->bufcnt); 765 766 return err; 767 } 768 769 static void atmel_sha_finish_req(struct ahash_request *req, int err) 770 { 771 struct atmel_sha_reqctx *ctx = ahash_request_ctx(req); 772 struct atmel_sha_dev *dd = ctx->dd; 773 774 if (!err) { 775 atmel_sha_copy_hash(req); 776 if (SHA_FLAGS_FINAL & dd->flags) 777 err = atmel_sha_finish(req); 778 } else { 779 ctx->flags |= SHA_FLAGS_ERROR; 780 } 781 782 /* atomic operation is not needed here */ 783 dd->flags &= ~(SHA_FLAGS_BUSY | SHA_FLAGS_FINAL | SHA_FLAGS_CPU | 784 SHA_FLAGS_DMA_READY | SHA_FLAGS_OUTPUT_READY); 785 786 clk_disable_unprepare(dd->iclk); 787 788 if (req->base.complete) 789 req->base.complete(&req->base, err); 790 791 /* handle new request */ 792 tasklet_schedule(&dd->done_task); 793 } 794 795 static int atmel_sha_hw_init(struct atmel_sha_dev *dd) 796 { 797 clk_prepare_enable(dd->iclk); 798 799 if (!(SHA_FLAGS_INIT & dd->flags)) { 800 atmel_sha_write(dd, SHA_CR, SHA_CR_SWRST); 801 dd->flags |= SHA_FLAGS_INIT; 802 dd->err = 0; 803 } 804 805 return 0; 806 } 807 808 static inline unsigned int atmel_sha_get_version(struct atmel_sha_dev *dd) 809 { 810 return atmel_sha_read(dd, SHA_HW_VERSION) & 0x00000fff; 811 } 812 813 static void atmel_sha_hw_version_init(struct atmel_sha_dev *dd) 814 { 815 atmel_sha_hw_init(dd); 816 817 dd->hw_version = atmel_sha_get_version(dd); 818 819 dev_info(dd->dev, 820 "version: 0x%x\n", dd->hw_version); 821 822 clk_disable_unprepare(dd->iclk); 823 } 824 825 static int atmel_sha_handle_queue(struct atmel_sha_dev *dd, 826 struct ahash_request *req) 827 { 828 struct crypto_async_request *async_req, *backlog; 829 struct atmel_sha_reqctx *ctx; 830 unsigned long flags; 831 int err = 0, ret = 0; 832 833 spin_lock_irqsave(&dd->lock, flags); 834 if (req) 835 ret = ahash_enqueue_request(&dd->queue, req); 836 837 if (SHA_FLAGS_BUSY & dd->flags) { 838 spin_unlock_irqrestore(&dd->lock, flags); 839 return ret; 840 } 841 842 backlog = crypto_get_backlog(&dd->queue); 843 async_req = crypto_dequeue_request(&dd->queue); 844 if (async_req) 845 dd->flags |= SHA_FLAGS_BUSY; 846 847 spin_unlock_irqrestore(&dd->lock, flags); 848 849 if (!async_req) 850 return ret; 851 852 if (backlog) 853 backlog->complete(backlog, -EINPROGRESS); 854 855 req = ahash_request_cast(async_req); 856 dd->req = req; 857 ctx = ahash_request_ctx(req); 858 859 dev_dbg(dd->dev, "handling new req, op: %lu, nbytes: %d\n", 860 ctx->op, req->nbytes); 861 862 err = atmel_sha_hw_init(dd); 863 864 if (err) 865 goto err1; 866 867 if (ctx->op == SHA_OP_UPDATE) { 868 err = atmel_sha_update_req(dd); 869 if (err != -EINPROGRESS && (ctx->flags & SHA_FLAGS_FINUP)) 870 /* no final() after finup() */ 871 err = atmel_sha_final_req(dd); 872 } else if (ctx->op == SHA_OP_FINAL) { 873 err = atmel_sha_final_req(dd); 874 } 875 876 err1: 877 if (err != -EINPROGRESS) 878 /* done_task will not finish it, so do it here */ 879 atmel_sha_finish_req(req, err); 880 881 dev_dbg(dd->dev, "exit, err: %d\n", err); 882 883 return ret; 884 } 885 886 static int atmel_sha_enqueue(struct ahash_request *req, unsigned int op) 887 { 888 struct atmel_sha_reqctx *ctx = ahash_request_ctx(req); 889 struct atmel_sha_ctx *tctx = crypto_tfm_ctx(req->base.tfm); 890 struct atmel_sha_dev *dd = tctx->dd; 891 892 ctx->op = op; 893 894 return atmel_sha_handle_queue(dd, req); 895 } 896 897 static int atmel_sha_update(struct ahash_request *req) 898 { 899 struct atmel_sha_reqctx *ctx = ahash_request_ctx(req); 900 901 if (!req->nbytes) 902 return 0; 903 904 ctx->total = req->nbytes; 905 ctx->sg = req->src; 906 ctx->offset = 0; 907 908 if (ctx->flags & SHA_FLAGS_FINUP) { 909 if (ctx->bufcnt + ctx->total < ATMEL_SHA_DMA_THRESHOLD) 910 /* faster to use CPU for short transfers */ 911 ctx->flags |= SHA_FLAGS_CPU; 912 } else if (ctx->bufcnt + ctx->total < ctx->buflen) { 913 atmel_sha_append_sg(ctx); 914 return 0; 915 } 916 return atmel_sha_enqueue(req, SHA_OP_UPDATE); 917 } 918 919 static int atmel_sha_final(struct ahash_request *req) 920 { 921 struct atmel_sha_reqctx *ctx = ahash_request_ctx(req); 922 struct atmel_sha_ctx *tctx = crypto_tfm_ctx(req->base.tfm); 923 struct atmel_sha_dev *dd = tctx->dd; 924 925 int err = 0; 926 927 ctx->flags |= SHA_FLAGS_FINUP; 928 929 if (ctx->flags & SHA_FLAGS_ERROR) 930 return 0; /* uncompleted hash is not needed */ 931 932 if (ctx->bufcnt) { 933 return atmel_sha_enqueue(req, SHA_OP_FINAL); 934 } else if (!(ctx->flags & SHA_FLAGS_PAD)) { /* add padding */ 935 err = atmel_sha_hw_init(dd); 936 if (err) 937 goto err1; 938 939 dd->flags |= SHA_FLAGS_BUSY; 940 err = atmel_sha_final_req(dd); 941 } else { 942 /* copy ready hash (+ finalize hmac) */ 943 return atmel_sha_finish(req); 944 } 945 946 err1: 947 if (err != -EINPROGRESS) 948 /* done_task will not finish it, so do it here */ 949 atmel_sha_finish_req(req, err); 950 951 return err; 952 } 953 954 static int atmel_sha_finup(struct ahash_request *req) 955 { 956 struct atmel_sha_reqctx *ctx = ahash_request_ctx(req); 957 int err1, err2; 958 959 ctx->flags |= SHA_FLAGS_FINUP; 960 961 err1 = atmel_sha_update(req); 962 if (err1 == -EINPROGRESS || err1 == -EBUSY) 963 return err1; 964 965 /* 966 * final() has to be always called to cleanup resources 967 * even if udpate() failed, except EINPROGRESS 968 */ 969 err2 = atmel_sha_final(req); 970 971 return err1 ?: err2; 972 } 973 974 static int atmel_sha_digest(struct ahash_request *req) 975 { 976 return atmel_sha_init(req) ?: atmel_sha_finup(req); 977 } 978 979 static int atmel_sha_cra_init(struct crypto_tfm *tfm) 980 { 981 crypto_ahash_set_reqsize(__crypto_ahash_cast(tfm), 982 sizeof(struct atmel_sha_reqctx) + 983 SHA_BUFFER_LEN + SHA512_BLOCK_SIZE); 984 985 return 0; 986 } 987 988 static struct ahash_alg sha_1_256_algs[] = { 989 { 990 .init = atmel_sha_init, 991 .update = atmel_sha_update, 992 .final = atmel_sha_final, 993 .finup = atmel_sha_finup, 994 .digest = atmel_sha_digest, 995 .halg = { 996 .digestsize = SHA1_DIGEST_SIZE, 997 .base = { 998 .cra_name = "sha1", 999 .cra_driver_name = "atmel-sha1", 1000 .cra_priority = 100, 1001 .cra_flags = CRYPTO_ALG_ASYNC, 1002 .cra_blocksize = SHA1_BLOCK_SIZE, 1003 .cra_ctxsize = sizeof(struct atmel_sha_ctx), 1004 .cra_alignmask = 0, 1005 .cra_module = THIS_MODULE, 1006 .cra_init = atmel_sha_cra_init, 1007 } 1008 } 1009 }, 1010 { 1011 .init = atmel_sha_init, 1012 .update = atmel_sha_update, 1013 .final = atmel_sha_final, 1014 .finup = atmel_sha_finup, 1015 .digest = atmel_sha_digest, 1016 .halg = { 1017 .digestsize = SHA256_DIGEST_SIZE, 1018 .base = { 1019 .cra_name = "sha256", 1020 .cra_driver_name = "atmel-sha256", 1021 .cra_priority = 100, 1022 .cra_flags = CRYPTO_ALG_ASYNC, 1023 .cra_blocksize = SHA256_BLOCK_SIZE, 1024 .cra_ctxsize = sizeof(struct atmel_sha_ctx), 1025 .cra_alignmask = 0, 1026 .cra_module = THIS_MODULE, 1027 .cra_init = atmel_sha_cra_init, 1028 } 1029 } 1030 }, 1031 }; 1032 1033 static struct ahash_alg sha_224_alg = { 1034 .init = atmel_sha_init, 1035 .update = atmel_sha_update, 1036 .final = atmel_sha_final, 1037 .finup = atmel_sha_finup, 1038 .digest = atmel_sha_digest, 1039 .halg = { 1040 .digestsize = SHA224_DIGEST_SIZE, 1041 .base = { 1042 .cra_name = "sha224", 1043 .cra_driver_name = "atmel-sha224", 1044 .cra_priority = 100, 1045 .cra_flags = CRYPTO_ALG_ASYNC, 1046 .cra_blocksize = SHA224_BLOCK_SIZE, 1047 .cra_ctxsize = sizeof(struct atmel_sha_ctx), 1048 .cra_alignmask = 0, 1049 .cra_module = THIS_MODULE, 1050 .cra_init = atmel_sha_cra_init, 1051 } 1052 } 1053 }; 1054 1055 static struct ahash_alg sha_384_512_algs[] = { 1056 { 1057 .init = atmel_sha_init, 1058 .update = atmel_sha_update, 1059 .final = atmel_sha_final, 1060 .finup = atmel_sha_finup, 1061 .digest = atmel_sha_digest, 1062 .halg = { 1063 .digestsize = SHA384_DIGEST_SIZE, 1064 .base = { 1065 .cra_name = "sha384", 1066 .cra_driver_name = "atmel-sha384", 1067 .cra_priority = 100, 1068 .cra_flags = CRYPTO_ALG_ASYNC, 1069 .cra_blocksize = SHA384_BLOCK_SIZE, 1070 .cra_ctxsize = sizeof(struct atmel_sha_ctx), 1071 .cra_alignmask = 0x3, 1072 .cra_module = THIS_MODULE, 1073 .cra_init = atmel_sha_cra_init, 1074 } 1075 } 1076 }, 1077 { 1078 .init = atmel_sha_init, 1079 .update = atmel_sha_update, 1080 .final = atmel_sha_final, 1081 .finup = atmel_sha_finup, 1082 .digest = atmel_sha_digest, 1083 .halg = { 1084 .digestsize = SHA512_DIGEST_SIZE, 1085 .base = { 1086 .cra_name = "sha512", 1087 .cra_driver_name = "atmel-sha512", 1088 .cra_priority = 100, 1089 .cra_flags = CRYPTO_ALG_ASYNC, 1090 .cra_blocksize = SHA512_BLOCK_SIZE, 1091 .cra_ctxsize = sizeof(struct atmel_sha_ctx), 1092 .cra_alignmask = 0x3, 1093 .cra_module = THIS_MODULE, 1094 .cra_init = atmel_sha_cra_init, 1095 } 1096 } 1097 }, 1098 }; 1099 1100 static void atmel_sha_done_task(unsigned long data) 1101 { 1102 struct atmel_sha_dev *dd = (struct atmel_sha_dev *)data; 1103 int err = 0; 1104 1105 if (!(SHA_FLAGS_BUSY & dd->flags)) { 1106 atmel_sha_handle_queue(dd, NULL); 1107 return; 1108 } 1109 1110 if (SHA_FLAGS_CPU & dd->flags) { 1111 if (SHA_FLAGS_OUTPUT_READY & dd->flags) { 1112 dd->flags &= ~SHA_FLAGS_OUTPUT_READY; 1113 goto finish; 1114 } 1115 } else if (SHA_FLAGS_DMA_READY & dd->flags) { 1116 if (SHA_FLAGS_DMA_ACTIVE & dd->flags) { 1117 dd->flags &= ~SHA_FLAGS_DMA_ACTIVE; 1118 atmel_sha_update_dma_stop(dd); 1119 if (dd->err) { 1120 err = dd->err; 1121 goto finish; 1122 } 1123 } 1124 if (SHA_FLAGS_OUTPUT_READY & dd->flags) { 1125 /* hash or semi-hash ready */ 1126 dd->flags &= ~(SHA_FLAGS_DMA_READY | 1127 SHA_FLAGS_OUTPUT_READY); 1128 err = atmel_sha_update_dma_start(dd); 1129 if (err != -EINPROGRESS) 1130 goto finish; 1131 } 1132 } 1133 return; 1134 1135 finish: 1136 /* finish curent request */ 1137 atmel_sha_finish_req(dd->req, err); 1138 } 1139 1140 static irqreturn_t atmel_sha_irq(int irq, void *dev_id) 1141 { 1142 struct atmel_sha_dev *sha_dd = dev_id; 1143 u32 reg; 1144 1145 reg = atmel_sha_read(sha_dd, SHA_ISR); 1146 if (reg & atmel_sha_read(sha_dd, SHA_IMR)) { 1147 atmel_sha_write(sha_dd, SHA_IDR, reg); 1148 if (SHA_FLAGS_BUSY & sha_dd->flags) { 1149 sha_dd->flags |= SHA_FLAGS_OUTPUT_READY; 1150 if (!(SHA_FLAGS_CPU & sha_dd->flags)) 1151 sha_dd->flags |= SHA_FLAGS_DMA_READY; 1152 tasklet_schedule(&sha_dd->done_task); 1153 } else { 1154 dev_warn(sha_dd->dev, "SHA interrupt when no active requests.\n"); 1155 } 1156 return IRQ_HANDLED; 1157 } 1158 1159 return IRQ_NONE; 1160 } 1161 1162 static void atmel_sha_unregister_algs(struct atmel_sha_dev *dd) 1163 { 1164 int i; 1165 1166 for (i = 0; i < ARRAY_SIZE(sha_1_256_algs); i++) 1167 crypto_unregister_ahash(&sha_1_256_algs[i]); 1168 1169 if (dd->caps.has_sha224) 1170 crypto_unregister_ahash(&sha_224_alg); 1171 1172 if (dd->caps.has_sha_384_512) { 1173 for (i = 0; i < ARRAY_SIZE(sha_384_512_algs); i++) 1174 crypto_unregister_ahash(&sha_384_512_algs[i]); 1175 } 1176 } 1177 1178 static int atmel_sha_register_algs(struct atmel_sha_dev *dd) 1179 { 1180 int err, i, j; 1181 1182 for (i = 0; i < ARRAY_SIZE(sha_1_256_algs); i++) { 1183 err = crypto_register_ahash(&sha_1_256_algs[i]); 1184 if (err) 1185 goto err_sha_1_256_algs; 1186 } 1187 1188 if (dd->caps.has_sha224) { 1189 err = crypto_register_ahash(&sha_224_alg); 1190 if (err) 1191 goto err_sha_224_algs; 1192 } 1193 1194 if (dd->caps.has_sha_384_512) { 1195 for (i = 0; i < ARRAY_SIZE(sha_384_512_algs); i++) { 1196 err = crypto_register_ahash(&sha_384_512_algs[i]); 1197 if (err) 1198 goto err_sha_384_512_algs; 1199 } 1200 } 1201 1202 return 0; 1203 1204 err_sha_384_512_algs: 1205 for (j = 0; j < i; j++) 1206 crypto_unregister_ahash(&sha_384_512_algs[j]); 1207 crypto_unregister_ahash(&sha_224_alg); 1208 err_sha_224_algs: 1209 i = ARRAY_SIZE(sha_1_256_algs); 1210 err_sha_1_256_algs: 1211 for (j = 0; j < i; j++) 1212 crypto_unregister_ahash(&sha_1_256_algs[j]); 1213 1214 return err; 1215 } 1216 1217 static bool atmel_sha_filter(struct dma_chan *chan, void *slave) 1218 { 1219 struct at_dma_slave *sl = slave; 1220 1221 if (sl && sl->dma_dev == chan->device->dev) { 1222 chan->private = sl; 1223 return true; 1224 } else { 1225 return false; 1226 } 1227 } 1228 1229 static int atmel_sha_dma_init(struct atmel_sha_dev *dd, 1230 struct crypto_platform_data *pdata) 1231 { 1232 int err = -ENOMEM; 1233 dma_cap_mask_t mask_in; 1234 1235 /* Try to grab DMA channel */ 1236 dma_cap_zero(mask_in); 1237 dma_cap_set(DMA_SLAVE, mask_in); 1238 1239 dd->dma_lch_in.chan = dma_request_slave_channel_compat(mask_in, 1240 atmel_sha_filter, &pdata->dma_slave->rxdata, dd->dev, "tx"); 1241 if (!dd->dma_lch_in.chan) { 1242 dev_warn(dd->dev, "no DMA channel available\n"); 1243 return err; 1244 } 1245 1246 dd->dma_lch_in.dma_conf.direction = DMA_MEM_TO_DEV; 1247 dd->dma_lch_in.dma_conf.dst_addr = dd->phys_base + 1248 SHA_REG_DIN(0); 1249 dd->dma_lch_in.dma_conf.src_maxburst = 1; 1250 dd->dma_lch_in.dma_conf.src_addr_width = 1251 DMA_SLAVE_BUSWIDTH_4_BYTES; 1252 dd->dma_lch_in.dma_conf.dst_maxburst = 1; 1253 dd->dma_lch_in.dma_conf.dst_addr_width = 1254 DMA_SLAVE_BUSWIDTH_4_BYTES; 1255 dd->dma_lch_in.dma_conf.device_fc = false; 1256 1257 return 0; 1258 } 1259 1260 static void atmel_sha_dma_cleanup(struct atmel_sha_dev *dd) 1261 { 1262 dma_release_channel(dd->dma_lch_in.chan); 1263 } 1264 1265 static void atmel_sha_get_cap(struct atmel_sha_dev *dd) 1266 { 1267 1268 dd->caps.has_dma = 0; 1269 dd->caps.has_dualbuff = 0; 1270 dd->caps.has_sha224 = 0; 1271 dd->caps.has_sha_384_512 = 0; 1272 1273 /* keep only major version number */ 1274 switch (dd->hw_version & 0xff0) { 1275 case 0x420: 1276 dd->caps.has_dma = 1; 1277 dd->caps.has_dualbuff = 1; 1278 dd->caps.has_sha224 = 1; 1279 dd->caps.has_sha_384_512 = 1; 1280 break; 1281 case 0x410: 1282 dd->caps.has_dma = 1; 1283 dd->caps.has_dualbuff = 1; 1284 dd->caps.has_sha224 = 1; 1285 dd->caps.has_sha_384_512 = 1; 1286 break; 1287 case 0x400: 1288 dd->caps.has_dma = 1; 1289 dd->caps.has_dualbuff = 1; 1290 dd->caps.has_sha224 = 1; 1291 break; 1292 case 0x320: 1293 break; 1294 default: 1295 dev_warn(dd->dev, 1296 "Unmanaged sha version, set minimum capabilities\n"); 1297 break; 1298 } 1299 } 1300 1301 #if defined(CONFIG_OF) 1302 static const struct of_device_id atmel_sha_dt_ids[] = { 1303 { .compatible = "atmel,at91sam9g46-sha" }, 1304 { /* sentinel */ } 1305 }; 1306 1307 MODULE_DEVICE_TABLE(of, atmel_sha_dt_ids); 1308 1309 static struct crypto_platform_data *atmel_sha_of_init(struct platform_device *pdev) 1310 { 1311 struct device_node *np = pdev->dev.of_node; 1312 struct crypto_platform_data *pdata; 1313 1314 if (!np) { 1315 dev_err(&pdev->dev, "device node not found\n"); 1316 return ERR_PTR(-EINVAL); 1317 } 1318 1319 pdata = devm_kzalloc(&pdev->dev, sizeof(*pdata), GFP_KERNEL); 1320 if (!pdata) { 1321 dev_err(&pdev->dev, "could not allocate memory for pdata\n"); 1322 return ERR_PTR(-ENOMEM); 1323 } 1324 1325 pdata->dma_slave = devm_kzalloc(&pdev->dev, 1326 sizeof(*(pdata->dma_slave)), 1327 GFP_KERNEL); 1328 if (!pdata->dma_slave) { 1329 dev_err(&pdev->dev, "could not allocate memory for dma_slave\n"); 1330 return ERR_PTR(-ENOMEM); 1331 } 1332 1333 return pdata; 1334 } 1335 #else /* CONFIG_OF */ 1336 static inline struct crypto_platform_data *atmel_sha_of_init(struct platform_device *dev) 1337 { 1338 return ERR_PTR(-EINVAL); 1339 } 1340 #endif 1341 1342 static int atmel_sha_probe(struct platform_device *pdev) 1343 { 1344 struct atmel_sha_dev *sha_dd; 1345 struct crypto_platform_data *pdata; 1346 struct device *dev = &pdev->dev; 1347 struct resource *sha_res; 1348 unsigned long sha_phys_size; 1349 int err; 1350 1351 sha_dd = devm_kzalloc(&pdev->dev, sizeof(struct atmel_sha_dev), 1352 GFP_KERNEL); 1353 if (sha_dd == NULL) { 1354 dev_err(dev, "unable to alloc data struct.\n"); 1355 err = -ENOMEM; 1356 goto sha_dd_err; 1357 } 1358 1359 sha_dd->dev = dev; 1360 1361 platform_set_drvdata(pdev, sha_dd); 1362 1363 INIT_LIST_HEAD(&sha_dd->list); 1364 spin_lock_init(&sha_dd->lock); 1365 1366 tasklet_init(&sha_dd->done_task, atmel_sha_done_task, 1367 (unsigned long)sha_dd); 1368 1369 crypto_init_queue(&sha_dd->queue, ATMEL_SHA_QUEUE_LENGTH); 1370 1371 sha_dd->irq = -1; 1372 1373 /* Get the base address */ 1374 sha_res = platform_get_resource(pdev, IORESOURCE_MEM, 0); 1375 if (!sha_res) { 1376 dev_err(dev, "no MEM resource info\n"); 1377 err = -ENODEV; 1378 goto res_err; 1379 } 1380 sha_dd->phys_base = sha_res->start; 1381 sha_phys_size = resource_size(sha_res); 1382 1383 /* Get the IRQ */ 1384 sha_dd->irq = platform_get_irq(pdev, 0); 1385 if (sha_dd->irq < 0) { 1386 dev_err(dev, "no IRQ resource info\n"); 1387 err = sha_dd->irq; 1388 goto res_err; 1389 } 1390 1391 err = request_irq(sha_dd->irq, atmel_sha_irq, IRQF_SHARED, "atmel-sha", 1392 sha_dd); 1393 if (err) { 1394 dev_err(dev, "unable to request sha irq.\n"); 1395 goto res_err; 1396 } 1397 1398 /* Initializing the clock */ 1399 sha_dd->iclk = clk_get(&pdev->dev, "sha_clk"); 1400 if (IS_ERR(sha_dd->iclk)) { 1401 dev_err(dev, "clock initialization failed.\n"); 1402 err = PTR_ERR(sha_dd->iclk); 1403 goto clk_err; 1404 } 1405 1406 sha_dd->io_base = ioremap(sha_dd->phys_base, sha_phys_size); 1407 if (!sha_dd->io_base) { 1408 dev_err(dev, "can't ioremap\n"); 1409 err = -ENOMEM; 1410 goto sha_io_err; 1411 } 1412 1413 atmel_sha_hw_version_init(sha_dd); 1414 1415 atmel_sha_get_cap(sha_dd); 1416 1417 if (sha_dd->caps.has_dma) { 1418 pdata = pdev->dev.platform_data; 1419 if (!pdata) { 1420 pdata = atmel_sha_of_init(pdev); 1421 if (IS_ERR(pdata)) { 1422 dev_err(&pdev->dev, "platform data not available\n"); 1423 err = PTR_ERR(pdata); 1424 goto err_pdata; 1425 } 1426 } 1427 if (!pdata->dma_slave) { 1428 err = -ENXIO; 1429 goto err_pdata; 1430 } 1431 err = atmel_sha_dma_init(sha_dd, pdata); 1432 if (err) 1433 goto err_sha_dma; 1434 1435 dev_info(dev, "using %s for DMA transfers\n", 1436 dma_chan_name(sha_dd->dma_lch_in.chan)); 1437 } 1438 1439 spin_lock(&atmel_sha.lock); 1440 list_add_tail(&sha_dd->list, &atmel_sha.dev_list); 1441 spin_unlock(&atmel_sha.lock); 1442 1443 err = atmel_sha_register_algs(sha_dd); 1444 if (err) 1445 goto err_algs; 1446 1447 dev_info(dev, "Atmel SHA1/SHA256%s%s\n", 1448 sha_dd->caps.has_sha224 ? "/SHA224" : "", 1449 sha_dd->caps.has_sha_384_512 ? "/SHA384/SHA512" : ""); 1450 1451 return 0; 1452 1453 err_algs: 1454 spin_lock(&atmel_sha.lock); 1455 list_del(&sha_dd->list); 1456 spin_unlock(&atmel_sha.lock); 1457 if (sha_dd->caps.has_dma) 1458 atmel_sha_dma_cleanup(sha_dd); 1459 err_sha_dma: 1460 err_pdata: 1461 iounmap(sha_dd->io_base); 1462 sha_io_err: 1463 clk_put(sha_dd->iclk); 1464 clk_err: 1465 free_irq(sha_dd->irq, sha_dd); 1466 res_err: 1467 tasklet_kill(&sha_dd->done_task); 1468 sha_dd_err: 1469 dev_err(dev, "initialization failed.\n"); 1470 1471 return err; 1472 } 1473 1474 static int atmel_sha_remove(struct platform_device *pdev) 1475 { 1476 static struct atmel_sha_dev *sha_dd; 1477 1478 sha_dd = platform_get_drvdata(pdev); 1479 if (!sha_dd) 1480 return -ENODEV; 1481 spin_lock(&atmel_sha.lock); 1482 list_del(&sha_dd->list); 1483 spin_unlock(&atmel_sha.lock); 1484 1485 atmel_sha_unregister_algs(sha_dd); 1486 1487 tasklet_kill(&sha_dd->done_task); 1488 1489 if (sha_dd->caps.has_dma) 1490 atmel_sha_dma_cleanup(sha_dd); 1491 1492 iounmap(sha_dd->io_base); 1493 1494 clk_put(sha_dd->iclk); 1495 1496 if (sha_dd->irq >= 0) 1497 free_irq(sha_dd->irq, sha_dd); 1498 1499 return 0; 1500 } 1501 1502 static struct platform_driver atmel_sha_driver = { 1503 .probe = atmel_sha_probe, 1504 .remove = atmel_sha_remove, 1505 .driver = { 1506 .name = "atmel_sha", 1507 .of_match_table = of_match_ptr(atmel_sha_dt_ids), 1508 }, 1509 }; 1510 1511 module_platform_driver(atmel_sha_driver); 1512 1513 MODULE_DESCRIPTION("Atmel SHA (1/256/224/384/512) hw acceleration support."); 1514 MODULE_LICENSE("GPL v2"); 1515 MODULE_AUTHOR("Nicolas Royer - Eukréa Electromatique"); 1516