1 /* 2 * Cryptographic API. 3 * 4 * Support for ATMEL SHA1/SHA256 HW acceleration. 5 * 6 * Copyright (c) 2012 Eukréa Electromatique - ATMEL 7 * Author: Nicolas Royer <nicolas@eukrea.com> 8 * 9 * This program is free software; you can redistribute it and/or modify 10 * it under the terms of the GNU General Public License version 2 as published 11 * by the Free Software Foundation. 12 * 13 * Some ideas are from omap-sham.c drivers. 14 */ 15 16 17 #include <linux/kernel.h> 18 #include <linux/module.h> 19 #include <linux/slab.h> 20 #include <linux/err.h> 21 #include <linux/clk.h> 22 #include <linux/io.h> 23 #include <linux/hw_random.h> 24 #include <linux/platform_device.h> 25 26 #include <linux/device.h> 27 #include <linux/init.h> 28 #include <linux/errno.h> 29 #include <linux/interrupt.h> 30 #include <linux/irq.h> 31 #include <linux/scatterlist.h> 32 #include <linux/dma-mapping.h> 33 #include <linux/of_device.h> 34 #include <linux/delay.h> 35 #include <linux/crypto.h> 36 #include <linux/cryptohash.h> 37 #include <crypto/scatterwalk.h> 38 #include <crypto/algapi.h> 39 #include <crypto/sha.h> 40 #include <crypto/hash.h> 41 #include <crypto/internal/hash.h> 42 #include <linux/platform_data/crypto-atmel.h> 43 #include "atmel-sha-regs.h" 44 45 /* SHA flags */ 46 #define SHA_FLAGS_BUSY BIT(0) 47 #define SHA_FLAGS_FINAL BIT(1) 48 #define SHA_FLAGS_DMA_ACTIVE BIT(2) 49 #define SHA_FLAGS_OUTPUT_READY BIT(3) 50 #define SHA_FLAGS_INIT BIT(4) 51 #define SHA_FLAGS_CPU BIT(5) 52 #define SHA_FLAGS_DMA_READY BIT(6) 53 54 #define SHA_FLAGS_FINUP BIT(16) 55 #define SHA_FLAGS_SG BIT(17) 56 #define SHA_FLAGS_ALGO_MASK GENMASK(22, 18) 57 #define SHA_FLAGS_SHA1 BIT(18) 58 #define SHA_FLAGS_SHA224 BIT(19) 59 #define SHA_FLAGS_SHA256 BIT(20) 60 #define SHA_FLAGS_SHA384 BIT(21) 61 #define SHA_FLAGS_SHA512 BIT(22) 62 #define SHA_FLAGS_ERROR BIT(23) 63 #define SHA_FLAGS_PAD BIT(24) 64 #define SHA_FLAGS_RESTORE BIT(25) 65 66 #define SHA_OP_UPDATE 1 67 #define SHA_OP_FINAL 2 68 69 #define SHA_BUFFER_LEN (PAGE_SIZE / 16) 70 71 #define ATMEL_SHA_DMA_THRESHOLD 56 72 73 struct atmel_sha_caps { 74 bool has_dma; 75 bool has_dualbuff; 76 bool has_sha224; 77 bool has_sha_384_512; 78 bool has_uihv; 79 }; 80 81 struct atmel_sha_dev; 82 83 /* 84 * .statesize = sizeof(struct atmel_sha_reqctx) must be <= PAGE_SIZE / 8 as 85 * tested by the ahash_prepare_alg() function. 86 */ 87 struct atmel_sha_reqctx { 88 struct atmel_sha_dev *dd; 89 unsigned long flags; 90 unsigned long op; 91 92 u8 digest[SHA512_DIGEST_SIZE] __aligned(sizeof(u32)); 93 u64 digcnt[2]; 94 size_t bufcnt; 95 size_t buflen; 96 dma_addr_t dma_addr; 97 98 /* walk state */ 99 struct scatterlist *sg; 100 unsigned int offset; /* offset in current sg */ 101 unsigned int total; /* total request */ 102 103 size_t block_size; 104 105 u8 buffer[SHA_BUFFER_LEN + SHA512_BLOCK_SIZE] __aligned(sizeof(u32)); 106 }; 107 108 struct atmel_sha_ctx { 109 struct atmel_sha_dev *dd; 110 111 unsigned long flags; 112 }; 113 114 #define ATMEL_SHA_QUEUE_LENGTH 50 115 116 struct atmel_sha_dma { 117 struct dma_chan *chan; 118 struct dma_slave_config dma_conf; 119 }; 120 121 struct atmel_sha_dev { 122 struct list_head list; 123 unsigned long phys_base; 124 struct device *dev; 125 struct clk *iclk; 126 int irq; 127 void __iomem *io_base; 128 129 spinlock_t lock; 130 int err; 131 struct tasklet_struct done_task; 132 struct tasklet_struct queue_task; 133 134 unsigned long flags; 135 struct crypto_queue queue; 136 struct ahash_request *req; 137 138 struct atmel_sha_dma dma_lch_in; 139 140 struct atmel_sha_caps caps; 141 142 u32 hw_version; 143 }; 144 145 struct atmel_sha_drv { 146 struct list_head dev_list; 147 spinlock_t lock; 148 }; 149 150 static struct atmel_sha_drv atmel_sha = { 151 .dev_list = LIST_HEAD_INIT(atmel_sha.dev_list), 152 .lock = __SPIN_LOCK_UNLOCKED(atmel_sha.lock), 153 }; 154 155 static inline u32 atmel_sha_read(struct atmel_sha_dev *dd, u32 offset) 156 { 157 return readl_relaxed(dd->io_base + offset); 158 } 159 160 static inline void atmel_sha_write(struct atmel_sha_dev *dd, 161 u32 offset, u32 value) 162 { 163 writel_relaxed(value, dd->io_base + offset); 164 } 165 166 static size_t atmel_sha_append_sg(struct atmel_sha_reqctx *ctx) 167 { 168 size_t count; 169 170 while ((ctx->bufcnt < ctx->buflen) && ctx->total) { 171 count = min(ctx->sg->length - ctx->offset, ctx->total); 172 count = min(count, ctx->buflen - ctx->bufcnt); 173 174 if (count <= 0) { 175 /* 176 * Check if count <= 0 because the buffer is full or 177 * because the sg length is 0. In the latest case, 178 * check if there is another sg in the list, a 0 length 179 * sg doesn't necessarily mean the end of the sg list. 180 */ 181 if ((ctx->sg->length == 0) && !sg_is_last(ctx->sg)) { 182 ctx->sg = sg_next(ctx->sg); 183 continue; 184 } else { 185 break; 186 } 187 } 188 189 scatterwalk_map_and_copy(ctx->buffer + ctx->bufcnt, ctx->sg, 190 ctx->offset, count, 0); 191 192 ctx->bufcnt += count; 193 ctx->offset += count; 194 ctx->total -= count; 195 196 if (ctx->offset == ctx->sg->length) { 197 ctx->sg = sg_next(ctx->sg); 198 if (ctx->sg) 199 ctx->offset = 0; 200 else 201 ctx->total = 0; 202 } 203 } 204 205 return 0; 206 } 207 208 /* 209 * The purpose of this padding is to ensure that the padded message is a 210 * multiple of 512 bits (SHA1/SHA224/SHA256) or 1024 bits (SHA384/SHA512). 211 * The bit "1" is appended at the end of the message followed by 212 * "padlen-1" zero bits. Then a 64 bits block (SHA1/SHA224/SHA256) or 213 * 128 bits block (SHA384/SHA512) equals to the message length in bits 214 * is appended. 215 * 216 * For SHA1/SHA224/SHA256, padlen is calculated as followed: 217 * - if message length < 56 bytes then padlen = 56 - message length 218 * - else padlen = 64 + 56 - message length 219 * 220 * For SHA384/SHA512, padlen is calculated as followed: 221 * - if message length < 112 bytes then padlen = 112 - message length 222 * - else padlen = 128 + 112 - message length 223 */ 224 static void atmel_sha_fill_padding(struct atmel_sha_reqctx *ctx, int length) 225 { 226 unsigned int index, padlen; 227 u64 bits[2]; 228 u64 size[2]; 229 230 size[0] = ctx->digcnt[0]; 231 size[1] = ctx->digcnt[1]; 232 233 size[0] += ctx->bufcnt; 234 if (size[0] < ctx->bufcnt) 235 size[1]++; 236 237 size[0] += length; 238 if (size[0] < length) 239 size[1]++; 240 241 bits[1] = cpu_to_be64(size[0] << 3); 242 bits[0] = cpu_to_be64(size[1] << 3 | size[0] >> 61); 243 244 if (ctx->flags & (SHA_FLAGS_SHA384 | SHA_FLAGS_SHA512)) { 245 index = ctx->bufcnt & 0x7f; 246 padlen = (index < 112) ? (112 - index) : ((128+112) - index); 247 *(ctx->buffer + ctx->bufcnt) = 0x80; 248 memset(ctx->buffer + ctx->bufcnt + 1, 0, padlen-1); 249 memcpy(ctx->buffer + ctx->bufcnt + padlen, bits, 16); 250 ctx->bufcnt += padlen + 16; 251 ctx->flags |= SHA_FLAGS_PAD; 252 } else { 253 index = ctx->bufcnt & 0x3f; 254 padlen = (index < 56) ? (56 - index) : ((64+56) - index); 255 *(ctx->buffer + ctx->bufcnt) = 0x80; 256 memset(ctx->buffer + ctx->bufcnt + 1, 0, padlen-1); 257 memcpy(ctx->buffer + ctx->bufcnt + padlen, &bits[1], 8); 258 ctx->bufcnt += padlen + 8; 259 ctx->flags |= SHA_FLAGS_PAD; 260 } 261 } 262 263 static int atmel_sha_init(struct ahash_request *req) 264 { 265 struct crypto_ahash *tfm = crypto_ahash_reqtfm(req); 266 struct atmel_sha_ctx *tctx = crypto_ahash_ctx(tfm); 267 struct atmel_sha_reqctx *ctx = ahash_request_ctx(req); 268 struct atmel_sha_dev *dd = NULL; 269 struct atmel_sha_dev *tmp; 270 271 spin_lock_bh(&atmel_sha.lock); 272 if (!tctx->dd) { 273 list_for_each_entry(tmp, &atmel_sha.dev_list, list) { 274 dd = tmp; 275 break; 276 } 277 tctx->dd = dd; 278 } else { 279 dd = tctx->dd; 280 } 281 282 spin_unlock_bh(&atmel_sha.lock); 283 284 ctx->dd = dd; 285 286 ctx->flags = 0; 287 288 dev_dbg(dd->dev, "init: digest size: %d\n", 289 crypto_ahash_digestsize(tfm)); 290 291 switch (crypto_ahash_digestsize(tfm)) { 292 case SHA1_DIGEST_SIZE: 293 ctx->flags |= SHA_FLAGS_SHA1; 294 ctx->block_size = SHA1_BLOCK_SIZE; 295 break; 296 case SHA224_DIGEST_SIZE: 297 ctx->flags |= SHA_FLAGS_SHA224; 298 ctx->block_size = SHA224_BLOCK_SIZE; 299 break; 300 case SHA256_DIGEST_SIZE: 301 ctx->flags |= SHA_FLAGS_SHA256; 302 ctx->block_size = SHA256_BLOCK_SIZE; 303 break; 304 case SHA384_DIGEST_SIZE: 305 ctx->flags |= SHA_FLAGS_SHA384; 306 ctx->block_size = SHA384_BLOCK_SIZE; 307 break; 308 case SHA512_DIGEST_SIZE: 309 ctx->flags |= SHA_FLAGS_SHA512; 310 ctx->block_size = SHA512_BLOCK_SIZE; 311 break; 312 default: 313 return -EINVAL; 314 break; 315 } 316 317 ctx->bufcnt = 0; 318 ctx->digcnt[0] = 0; 319 ctx->digcnt[1] = 0; 320 ctx->buflen = SHA_BUFFER_LEN; 321 322 return 0; 323 } 324 325 static void atmel_sha_write_ctrl(struct atmel_sha_dev *dd, int dma) 326 { 327 struct atmel_sha_reqctx *ctx = ahash_request_ctx(dd->req); 328 u32 valmr = SHA_MR_MODE_AUTO; 329 unsigned int i, hashsize = 0; 330 331 if (likely(dma)) { 332 if (!dd->caps.has_dma) 333 atmel_sha_write(dd, SHA_IER, SHA_INT_TXBUFE); 334 valmr = SHA_MR_MODE_PDC; 335 if (dd->caps.has_dualbuff) 336 valmr |= SHA_MR_DUALBUFF; 337 } else { 338 atmel_sha_write(dd, SHA_IER, SHA_INT_DATARDY); 339 } 340 341 switch (ctx->flags & SHA_FLAGS_ALGO_MASK) { 342 case SHA_FLAGS_SHA1: 343 valmr |= SHA_MR_ALGO_SHA1; 344 hashsize = SHA1_DIGEST_SIZE; 345 break; 346 347 case SHA_FLAGS_SHA224: 348 valmr |= SHA_MR_ALGO_SHA224; 349 hashsize = SHA256_DIGEST_SIZE; 350 break; 351 352 case SHA_FLAGS_SHA256: 353 valmr |= SHA_MR_ALGO_SHA256; 354 hashsize = SHA256_DIGEST_SIZE; 355 break; 356 357 case SHA_FLAGS_SHA384: 358 valmr |= SHA_MR_ALGO_SHA384; 359 hashsize = SHA512_DIGEST_SIZE; 360 break; 361 362 case SHA_FLAGS_SHA512: 363 valmr |= SHA_MR_ALGO_SHA512; 364 hashsize = SHA512_DIGEST_SIZE; 365 break; 366 367 default: 368 break; 369 } 370 371 /* Setting CR_FIRST only for the first iteration */ 372 if (!(ctx->digcnt[0] || ctx->digcnt[1])) { 373 atmel_sha_write(dd, SHA_CR, SHA_CR_FIRST); 374 } else if (dd->caps.has_uihv && (ctx->flags & SHA_FLAGS_RESTORE)) { 375 const u32 *hash = (const u32 *)ctx->digest; 376 377 /* 378 * Restore the hardware context: update the User Initialize 379 * Hash Value (UIHV) with the value saved when the latest 380 * 'update' operation completed on this very same crypto 381 * request. 382 */ 383 ctx->flags &= ~SHA_FLAGS_RESTORE; 384 atmel_sha_write(dd, SHA_CR, SHA_CR_WUIHV); 385 for (i = 0; i < hashsize / sizeof(u32); ++i) 386 atmel_sha_write(dd, SHA_REG_DIN(i), hash[i]); 387 atmel_sha_write(dd, SHA_CR, SHA_CR_FIRST); 388 valmr |= SHA_MR_UIHV; 389 } 390 /* 391 * WARNING: If the UIHV feature is not available, the hardware CANNOT 392 * process concurrent requests: the internal registers used to store 393 * the hash/digest are still set to the partial digest output values 394 * computed during the latest round. 395 */ 396 397 atmel_sha_write(dd, SHA_MR, valmr); 398 } 399 400 static int atmel_sha_xmit_cpu(struct atmel_sha_dev *dd, const u8 *buf, 401 size_t length, int final) 402 { 403 struct atmel_sha_reqctx *ctx = ahash_request_ctx(dd->req); 404 int count, len32; 405 const u32 *buffer = (const u32 *)buf; 406 407 dev_dbg(dd->dev, "xmit_cpu: digcnt: 0x%llx 0x%llx, length: %d, final: %d\n", 408 ctx->digcnt[1], ctx->digcnt[0], length, final); 409 410 atmel_sha_write_ctrl(dd, 0); 411 412 /* should be non-zero before next lines to disable clocks later */ 413 ctx->digcnt[0] += length; 414 if (ctx->digcnt[0] < length) 415 ctx->digcnt[1]++; 416 417 if (final) 418 dd->flags |= SHA_FLAGS_FINAL; /* catch last interrupt */ 419 420 len32 = DIV_ROUND_UP(length, sizeof(u32)); 421 422 dd->flags |= SHA_FLAGS_CPU; 423 424 for (count = 0; count < len32; count++) 425 atmel_sha_write(dd, SHA_REG_DIN(count), buffer[count]); 426 427 return -EINPROGRESS; 428 } 429 430 static int atmel_sha_xmit_pdc(struct atmel_sha_dev *dd, dma_addr_t dma_addr1, 431 size_t length1, dma_addr_t dma_addr2, size_t length2, int final) 432 { 433 struct atmel_sha_reqctx *ctx = ahash_request_ctx(dd->req); 434 int len32; 435 436 dev_dbg(dd->dev, "xmit_pdc: digcnt: 0x%llx 0x%llx, length: %d, final: %d\n", 437 ctx->digcnt[1], ctx->digcnt[0], length1, final); 438 439 len32 = DIV_ROUND_UP(length1, sizeof(u32)); 440 atmel_sha_write(dd, SHA_PTCR, SHA_PTCR_TXTDIS); 441 atmel_sha_write(dd, SHA_TPR, dma_addr1); 442 atmel_sha_write(dd, SHA_TCR, len32); 443 444 len32 = DIV_ROUND_UP(length2, sizeof(u32)); 445 atmel_sha_write(dd, SHA_TNPR, dma_addr2); 446 atmel_sha_write(dd, SHA_TNCR, len32); 447 448 atmel_sha_write_ctrl(dd, 1); 449 450 /* should be non-zero before next lines to disable clocks later */ 451 ctx->digcnt[0] += length1; 452 if (ctx->digcnt[0] < length1) 453 ctx->digcnt[1]++; 454 455 if (final) 456 dd->flags |= SHA_FLAGS_FINAL; /* catch last interrupt */ 457 458 dd->flags |= SHA_FLAGS_DMA_ACTIVE; 459 460 /* Start DMA transfer */ 461 atmel_sha_write(dd, SHA_PTCR, SHA_PTCR_TXTEN); 462 463 return -EINPROGRESS; 464 } 465 466 static void atmel_sha_dma_callback(void *data) 467 { 468 struct atmel_sha_dev *dd = data; 469 470 /* dma_lch_in - completed - wait DATRDY */ 471 atmel_sha_write(dd, SHA_IER, SHA_INT_DATARDY); 472 } 473 474 static int atmel_sha_xmit_dma(struct atmel_sha_dev *dd, dma_addr_t dma_addr1, 475 size_t length1, dma_addr_t dma_addr2, size_t length2, int final) 476 { 477 struct atmel_sha_reqctx *ctx = ahash_request_ctx(dd->req); 478 struct dma_async_tx_descriptor *in_desc; 479 struct scatterlist sg[2]; 480 481 dev_dbg(dd->dev, "xmit_dma: digcnt: 0x%llx 0x%llx, length: %d, final: %d\n", 482 ctx->digcnt[1], ctx->digcnt[0], length1, final); 483 484 dd->dma_lch_in.dma_conf.src_maxburst = 16; 485 dd->dma_lch_in.dma_conf.dst_maxburst = 16; 486 487 dmaengine_slave_config(dd->dma_lch_in.chan, &dd->dma_lch_in.dma_conf); 488 489 if (length2) { 490 sg_init_table(sg, 2); 491 sg_dma_address(&sg[0]) = dma_addr1; 492 sg_dma_len(&sg[0]) = length1; 493 sg_dma_address(&sg[1]) = dma_addr2; 494 sg_dma_len(&sg[1]) = length2; 495 in_desc = dmaengine_prep_slave_sg(dd->dma_lch_in.chan, sg, 2, 496 DMA_MEM_TO_DEV, DMA_PREP_INTERRUPT | DMA_CTRL_ACK); 497 } else { 498 sg_init_table(sg, 1); 499 sg_dma_address(&sg[0]) = dma_addr1; 500 sg_dma_len(&sg[0]) = length1; 501 in_desc = dmaengine_prep_slave_sg(dd->dma_lch_in.chan, sg, 1, 502 DMA_MEM_TO_DEV, DMA_PREP_INTERRUPT | DMA_CTRL_ACK); 503 } 504 if (!in_desc) 505 return -EINVAL; 506 507 in_desc->callback = atmel_sha_dma_callback; 508 in_desc->callback_param = dd; 509 510 atmel_sha_write_ctrl(dd, 1); 511 512 /* should be non-zero before next lines to disable clocks later */ 513 ctx->digcnt[0] += length1; 514 if (ctx->digcnt[0] < length1) 515 ctx->digcnt[1]++; 516 517 if (final) 518 dd->flags |= SHA_FLAGS_FINAL; /* catch last interrupt */ 519 520 dd->flags |= SHA_FLAGS_DMA_ACTIVE; 521 522 /* Start DMA transfer */ 523 dmaengine_submit(in_desc); 524 dma_async_issue_pending(dd->dma_lch_in.chan); 525 526 return -EINPROGRESS; 527 } 528 529 static int atmel_sha_xmit_start(struct atmel_sha_dev *dd, dma_addr_t dma_addr1, 530 size_t length1, dma_addr_t dma_addr2, size_t length2, int final) 531 { 532 if (dd->caps.has_dma) 533 return atmel_sha_xmit_dma(dd, dma_addr1, length1, 534 dma_addr2, length2, final); 535 else 536 return atmel_sha_xmit_pdc(dd, dma_addr1, length1, 537 dma_addr2, length2, final); 538 } 539 540 static int atmel_sha_update_cpu(struct atmel_sha_dev *dd) 541 { 542 struct atmel_sha_reqctx *ctx = ahash_request_ctx(dd->req); 543 int bufcnt; 544 545 atmel_sha_append_sg(ctx); 546 atmel_sha_fill_padding(ctx, 0); 547 bufcnt = ctx->bufcnt; 548 ctx->bufcnt = 0; 549 550 return atmel_sha_xmit_cpu(dd, ctx->buffer, bufcnt, 1); 551 } 552 553 static int atmel_sha_xmit_dma_map(struct atmel_sha_dev *dd, 554 struct atmel_sha_reqctx *ctx, 555 size_t length, int final) 556 { 557 ctx->dma_addr = dma_map_single(dd->dev, ctx->buffer, 558 ctx->buflen + ctx->block_size, DMA_TO_DEVICE); 559 if (dma_mapping_error(dd->dev, ctx->dma_addr)) { 560 dev_err(dd->dev, "dma %u bytes error\n", ctx->buflen + 561 ctx->block_size); 562 return -EINVAL; 563 } 564 565 ctx->flags &= ~SHA_FLAGS_SG; 566 567 /* next call does not fail... so no unmap in the case of error */ 568 return atmel_sha_xmit_start(dd, ctx->dma_addr, length, 0, 0, final); 569 } 570 571 static int atmel_sha_update_dma_slow(struct atmel_sha_dev *dd) 572 { 573 struct atmel_sha_reqctx *ctx = ahash_request_ctx(dd->req); 574 unsigned int final; 575 size_t count; 576 577 atmel_sha_append_sg(ctx); 578 579 final = (ctx->flags & SHA_FLAGS_FINUP) && !ctx->total; 580 581 dev_dbg(dd->dev, "slow: bufcnt: %u, digcnt: 0x%llx 0x%llx, final: %d\n", 582 ctx->bufcnt, ctx->digcnt[1], ctx->digcnt[0], final); 583 584 if (final) 585 atmel_sha_fill_padding(ctx, 0); 586 587 if (final || (ctx->bufcnt == ctx->buflen)) { 588 count = ctx->bufcnt; 589 ctx->bufcnt = 0; 590 return atmel_sha_xmit_dma_map(dd, ctx, count, final); 591 } 592 593 return 0; 594 } 595 596 static int atmel_sha_update_dma_start(struct atmel_sha_dev *dd) 597 { 598 struct atmel_sha_reqctx *ctx = ahash_request_ctx(dd->req); 599 unsigned int length, final, tail; 600 struct scatterlist *sg; 601 unsigned int count; 602 603 if (!ctx->total) 604 return 0; 605 606 if (ctx->bufcnt || ctx->offset) 607 return atmel_sha_update_dma_slow(dd); 608 609 dev_dbg(dd->dev, "fast: digcnt: 0x%llx 0x%llx, bufcnt: %u, total: %u\n", 610 ctx->digcnt[1], ctx->digcnt[0], ctx->bufcnt, ctx->total); 611 612 sg = ctx->sg; 613 614 if (!IS_ALIGNED(sg->offset, sizeof(u32))) 615 return atmel_sha_update_dma_slow(dd); 616 617 if (!sg_is_last(sg) && !IS_ALIGNED(sg->length, ctx->block_size)) 618 /* size is not ctx->block_size aligned */ 619 return atmel_sha_update_dma_slow(dd); 620 621 length = min(ctx->total, sg->length); 622 623 if (sg_is_last(sg)) { 624 if (!(ctx->flags & SHA_FLAGS_FINUP)) { 625 /* not last sg must be ctx->block_size aligned */ 626 tail = length & (ctx->block_size - 1); 627 length -= tail; 628 } 629 } 630 631 ctx->total -= length; 632 ctx->offset = length; /* offset where to start slow */ 633 634 final = (ctx->flags & SHA_FLAGS_FINUP) && !ctx->total; 635 636 /* Add padding */ 637 if (final) { 638 tail = length & (ctx->block_size - 1); 639 length -= tail; 640 ctx->total += tail; 641 ctx->offset = length; /* offset where to start slow */ 642 643 sg = ctx->sg; 644 atmel_sha_append_sg(ctx); 645 646 atmel_sha_fill_padding(ctx, length); 647 648 ctx->dma_addr = dma_map_single(dd->dev, ctx->buffer, 649 ctx->buflen + ctx->block_size, DMA_TO_DEVICE); 650 if (dma_mapping_error(dd->dev, ctx->dma_addr)) { 651 dev_err(dd->dev, "dma %u bytes error\n", 652 ctx->buflen + ctx->block_size); 653 return -EINVAL; 654 } 655 656 if (length == 0) { 657 ctx->flags &= ~SHA_FLAGS_SG; 658 count = ctx->bufcnt; 659 ctx->bufcnt = 0; 660 return atmel_sha_xmit_start(dd, ctx->dma_addr, count, 0, 661 0, final); 662 } else { 663 ctx->sg = sg; 664 if (!dma_map_sg(dd->dev, ctx->sg, 1, 665 DMA_TO_DEVICE)) { 666 dev_err(dd->dev, "dma_map_sg error\n"); 667 return -EINVAL; 668 } 669 670 ctx->flags |= SHA_FLAGS_SG; 671 672 count = ctx->bufcnt; 673 ctx->bufcnt = 0; 674 return atmel_sha_xmit_start(dd, sg_dma_address(ctx->sg), 675 length, ctx->dma_addr, count, final); 676 } 677 } 678 679 if (!dma_map_sg(dd->dev, ctx->sg, 1, DMA_TO_DEVICE)) { 680 dev_err(dd->dev, "dma_map_sg error\n"); 681 return -EINVAL; 682 } 683 684 ctx->flags |= SHA_FLAGS_SG; 685 686 /* next call does not fail... so no unmap in the case of error */ 687 return atmel_sha_xmit_start(dd, sg_dma_address(ctx->sg), length, 0, 688 0, final); 689 } 690 691 static int atmel_sha_update_dma_stop(struct atmel_sha_dev *dd) 692 { 693 struct atmel_sha_reqctx *ctx = ahash_request_ctx(dd->req); 694 695 if (ctx->flags & SHA_FLAGS_SG) { 696 dma_unmap_sg(dd->dev, ctx->sg, 1, DMA_TO_DEVICE); 697 if (ctx->sg->length == ctx->offset) { 698 ctx->sg = sg_next(ctx->sg); 699 if (ctx->sg) 700 ctx->offset = 0; 701 } 702 if (ctx->flags & SHA_FLAGS_PAD) { 703 dma_unmap_single(dd->dev, ctx->dma_addr, 704 ctx->buflen + ctx->block_size, DMA_TO_DEVICE); 705 } 706 } else { 707 dma_unmap_single(dd->dev, ctx->dma_addr, ctx->buflen + 708 ctx->block_size, DMA_TO_DEVICE); 709 } 710 711 return 0; 712 } 713 714 static int atmel_sha_update_req(struct atmel_sha_dev *dd) 715 { 716 struct ahash_request *req = dd->req; 717 struct atmel_sha_reqctx *ctx = ahash_request_ctx(req); 718 int err; 719 720 dev_dbg(dd->dev, "update_req: total: %u, digcnt: 0x%llx 0x%llx\n", 721 ctx->total, ctx->digcnt[1], ctx->digcnt[0]); 722 723 if (ctx->flags & SHA_FLAGS_CPU) 724 err = atmel_sha_update_cpu(dd); 725 else 726 err = atmel_sha_update_dma_start(dd); 727 728 /* wait for dma completion before can take more data */ 729 dev_dbg(dd->dev, "update: err: %d, digcnt: 0x%llx 0%llx\n", 730 err, ctx->digcnt[1], ctx->digcnt[0]); 731 732 return err; 733 } 734 735 static int atmel_sha_final_req(struct atmel_sha_dev *dd) 736 { 737 struct ahash_request *req = dd->req; 738 struct atmel_sha_reqctx *ctx = ahash_request_ctx(req); 739 int err = 0; 740 int count; 741 742 if (ctx->bufcnt >= ATMEL_SHA_DMA_THRESHOLD) { 743 atmel_sha_fill_padding(ctx, 0); 744 count = ctx->bufcnt; 745 ctx->bufcnt = 0; 746 err = atmel_sha_xmit_dma_map(dd, ctx, count, 1); 747 } 748 /* faster to handle last block with cpu */ 749 else { 750 atmel_sha_fill_padding(ctx, 0); 751 count = ctx->bufcnt; 752 ctx->bufcnt = 0; 753 err = atmel_sha_xmit_cpu(dd, ctx->buffer, count, 1); 754 } 755 756 dev_dbg(dd->dev, "final_req: err: %d\n", err); 757 758 return err; 759 } 760 761 static void atmel_sha_copy_hash(struct ahash_request *req) 762 { 763 struct atmel_sha_reqctx *ctx = ahash_request_ctx(req); 764 u32 *hash = (u32 *)ctx->digest; 765 unsigned int i, hashsize; 766 767 switch (ctx->flags & SHA_FLAGS_ALGO_MASK) { 768 case SHA_FLAGS_SHA1: 769 hashsize = SHA1_DIGEST_SIZE; 770 break; 771 772 case SHA_FLAGS_SHA224: 773 case SHA_FLAGS_SHA256: 774 hashsize = SHA256_DIGEST_SIZE; 775 break; 776 777 case SHA_FLAGS_SHA384: 778 case SHA_FLAGS_SHA512: 779 hashsize = SHA512_DIGEST_SIZE; 780 break; 781 782 default: 783 /* Should not happen... */ 784 return; 785 } 786 787 for (i = 0; i < hashsize / sizeof(u32); ++i) 788 hash[i] = atmel_sha_read(ctx->dd, SHA_REG_DIGEST(i)); 789 ctx->flags |= SHA_FLAGS_RESTORE; 790 } 791 792 static void atmel_sha_copy_ready_hash(struct ahash_request *req) 793 { 794 struct atmel_sha_reqctx *ctx = ahash_request_ctx(req); 795 796 if (!req->result) 797 return; 798 799 if (ctx->flags & SHA_FLAGS_SHA1) 800 memcpy(req->result, ctx->digest, SHA1_DIGEST_SIZE); 801 else if (ctx->flags & SHA_FLAGS_SHA224) 802 memcpy(req->result, ctx->digest, SHA224_DIGEST_SIZE); 803 else if (ctx->flags & SHA_FLAGS_SHA256) 804 memcpy(req->result, ctx->digest, SHA256_DIGEST_SIZE); 805 else if (ctx->flags & SHA_FLAGS_SHA384) 806 memcpy(req->result, ctx->digest, SHA384_DIGEST_SIZE); 807 else 808 memcpy(req->result, ctx->digest, SHA512_DIGEST_SIZE); 809 } 810 811 static int atmel_sha_finish(struct ahash_request *req) 812 { 813 struct atmel_sha_reqctx *ctx = ahash_request_ctx(req); 814 struct atmel_sha_dev *dd = ctx->dd; 815 816 if (ctx->digcnt[0] || ctx->digcnt[1]) 817 atmel_sha_copy_ready_hash(req); 818 819 dev_dbg(dd->dev, "digcnt: 0x%llx 0x%llx, bufcnt: %d\n", ctx->digcnt[1], 820 ctx->digcnt[0], ctx->bufcnt); 821 822 return 0; 823 } 824 825 static void atmel_sha_finish_req(struct ahash_request *req, int err) 826 { 827 struct atmel_sha_reqctx *ctx = ahash_request_ctx(req); 828 struct atmel_sha_dev *dd = ctx->dd; 829 830 if (!err) { 831 atmel_sha_copy_hash(req); 832 if (SHA_FLAGS_FINAL & dd->flags) 833 err = atmel_sha_finish(req); 834 } else { 835 ctx->flags |= SHA_FLAGS_ERROR; 836 } 837 838 /* atomic operation is not needed here */ 839 dd->flags &= ~(SHA_FLAGS_BUSY | SHA_FLAGS_FINAL | SHA_FLAGS_CPU | 840 SHA_FLAGS_DMA_READY | SHA_FLAGS_OUTPUT_READY); 841 842 clk_disable(dd->iclk); 843 844 if (req->base.complete) 845 req->base.complete(&req->base, err); 846 847 /* handle new request */ 848 tasklet_schedule(&dd->queue_task); 849 } 850 851 static int atmel_sha_hw_init(struct atmel_sha_dev *dd) 852 { 853 int err; 854 855 err = clk_enable(dd->iclk); 856 if (err) 857 return err; 858 859 if (!(SHA_FLAGS_INIT & dd->flags)) { 860 atmel_sha_write(dd, SHA_CR, SHA_CR_SWRST); 861 dd->flags |= SHA_FLAGS_INIT; 862 dd->err = 0; 863 } 864 865 return 0; 866 } 867 868 static inline unsigned int atmel_sha_get_version(struct atmel_sha_dev *dd) 869 { 870 return atmel_sha_read(dd, SHA_HW_VERSION) & 0x00000fff; 871 } 872 873 static void atmel_sha_hw_version_init(struct atmel_sha_dev *dd) 874 { 875 atmel_sha_hw_init(dd); 876 877 dd->hw_version = atmel_sha_get_version(dd); 878 879 dev_info(dd->dev, 880 "version: 0x%x\n", dd->hw_version); 881 882 clk_disable(dd->iclk); 883 } 884 885 static int atmel_sha_handle_queue(struct atmel_sha_dev *dd, 886 struct ahash_request *req) 887 { 888 struct crypto_async_request *async_req, *backlog; 889 struct atmel_sha_reqctx *ctx; 890 unsigned long flags; 891 int err = 0, ret = 0; 892 893 spin_lock_irqsave(&dd->lock, flags); 894 if (req) 895 ret = ahash_enqueue_request(&dd->queue, req); 896 897 if (SHA_FLAGS_BUSY & dd->flags) { 898 spin_unlock_irqrestore(&dd->lock, flags); 899 return ret; 900 } 901 902 backlog = crypto_get_backlog(&dd->queue); 903 async_req = crypto_dequeue_request(&dd->queue); 904 if (async_req) 905 dd->flags |= SHA_FLAGS_BUSY; 906 907 spin_unlock_irqrestore(&dd->lock, flags); 908 909 if (!async_req) 910 return ret; 911 912 if (backlog) 913 backlog->complete(backlog, -EINPROGRESS); 914 915 req = ahash_request_cast(async_req); 916 dd->req = req; 917 ctx = ahash_request_ctx(req); 918 919 dev_dbg(dd->dev, "handling new req, op: %lu, nbytes: %d\n", 920 ctx->op, req->nbytes); 921 922 err = atmel_sha_hw_init(dd); 923 924 if (err) 925 goto err1; 926 927 if (ctx->op == SHA_OP_UPDATE) { 928 err = atmel_sha_update_req(dd); 929 if (err != -EINPROGRESS && (ctx->flags & SHA_FLAGS_FINUP)) 930 /* no final() after finup() */ 931 err = atmel_sha_final_req(dd); 932 } else if (ctx->op == SHA_OP_FINAL) { 933 err = atmel_sha_final_req(dd); 934 } 935 936 err1: 937 if (err != -EINPROGRESS) 938 /* done_task will not finish it, so do it here */ 939 atmel_sha_finish_req(req, err); 940 941 dev_dbg(dd->dev, "exit, err: %d\n", err); 942 943 return ret; 944 } 945 946 static int atmel_sha_enqueue(struct ahash_request *req, unsigned int op) 947 { 948 struct atmel_sha_reqctx *ctx = ahash_request_ctx(req); 949 struct atmel_sha_ctx *tctx = crypto_tfm_ctx(req->base.tfm); 950 struct atmel_sha_dev *dd = tctx->dd; 951 952 ctx->op = op; 953 954 return atmel_sha_handle_queue(dd, req); 955 } 956 957 static int atmel_sha_update(struct ahash_request *req) 958 { 959 struct atmel_sha_reqctx *ctx = ahash_request_ctx(req); 960 961 if (!req->nbytes) 962 return 0; 963 964 ctx->total = req->nbytes; 965 ctx->sg = req->src; 966 ctx->offset = 0; 967 968 if (ctx->flags & SHA_FLAGS_FINUP) { 969 if (ctx->bufcnt + ctx->total < ATMEL_SHA_DMA_THRESHOLD) 970 /* faster to use CPU for short transfers */ 971 ctx->flags |= SHA_FLAGS_CPU; 972 } else if (ctx->bufcnt + ctx->total < ctx->buflen) { 973 atmel_sha_append_sg(ctx); 974 return 0; 975 } 976 return atmel_sha_enqueue(req, SHA_OP_UPDATE); 977 } 978 979 static int atmel_sha_final(struct ahash_request *req) 980 { 981 struct atmel_sha_reqctx *ctx = ahash_request_ctx(req); 982 983 ctx->flags |= SHA_FLAGS_FINUP; 984 985 if (ctx->flags & SHA_FLAGS_ERROR) 986 return 0; /* uncompleted hash is not needed */ 987 988 if (ctx->flags & SHA_FLAGS_PAD) 989 /* copy ready hash (+ finalize hmac) */ 990 return atmel_sha_finish(req); 991 992 return atmel_sha_enqueue(req, SHA_OP_FINAL); 993 } 994 995 static int atmel_sha_finup(struct ahash_request *req) 996 { 997 struct atmel_sha_reqctx *ctx = ahash_request_ctx(req); 998 int err1, err2; 999 1000 ctx->flags |= SHA_FLAGS_FINUP; 1001 1002 err1 = atmel_sha_update(req); 1003 if (err1 == -EINPROGRESS || err1 == -EBUSY) 1004 return err1; 1005 1006 /* 1007 * final() has to be always called to cleanup resources 1008 * even if udpate() failed, except EINPROGRESS 1009 */ 1010 err2 = atmel_sha_final(req); 1011 1012 return err1 ?: err2; 1013 } 1014 1015 static int atmel_sha_digest(struct ahash_request *req) 1016 { 1017 return atmel_sha_init(req) ?: atmel_sha_finup(req); 1018 } 1019 1020 1021 static int atmel_sha_export(struct ahash_request *req, void *out) 1022 { 1023 const struct atmel_sha_reqctx *ctx = ahash_request_ctx(req); 1024 1025 memcpy(out, ctx, sizeof(*ctx)); 1026 return 0; 1027 } 1028 1029 static int atmel_sha_import(struct ahash_request *req, const void *in) 1030 { 1031 struct atmel_sha_reqctx *ctx = ahash_request_ctx(req); 1032 1033 memcpy(ctx, in, sizeof(*ctx)); 1034 return 0; 1035 } 1036 1037 static int atmel_sha_cra_init(struct crypto_tfm *tfm) 1038 { 1039 crypto_ahash_set_reqsize(__crypto_ahash_cast(tfm), 1040 sizeof(struct atmel_sha_reqctx)); 1041 1042 return 0; 1043 } 1044 1045 static struct ahash_alg sha_1_256_algs[] = { 1046 { 1047 .init = atmel_sha_init, 1048 .update = atmel_sha_update, 1049 .final = atmel_sha_final, 1050 .finup = atmel_sha_finup, 1051 .digest = atmel_sha_digest, 1052 .export = atmel_sha_export, 1053 .import = atmel_sha_import, 1054 .halg = { 1055 .digestsize = SHA1_DIGEST_SIZE, 1056 .statesize = sizeof(struct atmel_sha_reqctx), 1057 .base = { 1058 .cra_name = "sha1", 1059 .cra_driver_name = "atmel-sha1", 1060 .cra_priority = 100, 1061 .cra_flags = CRYPTO_ALG_ASYNC, 1062 .cra_blocksize = SHA1_BLOCK_SIZE, 1063 .cra_ctxsize = sizeof(struct atmel_sha_ctx), 1064 .cra_alignmask = 0, 1065 .cra_module = THIS_MODULE, 1066 .cra_init = atmel_sha_cra_init, 1067 } 1068 } 1069 }, 1070 { 1071 .init = atmel_sha_init, 1072 .update = atmel_sha_update, 1073 .final = atmel_sha_final, 1074 .finup = atmel_sha_finup, 1075 .digest = atmel_sha_digest, 1076 .export = atmel_sha_export, 1077 .import = atmel_sha_import, 1078 .halg = { 1079 .digestsize = SHA256_DIGEST_SIZE, 1080 .statesize = sizeof(struct atmel_sha_reqctx), 1081 .base = { 1082 .cra_name = "sha256", 1083 .cra_driver_name = "atmel-sha256", 1084 .cra_priority = 100, 1085 .cra_flags = CRYPTO_ALG_ASYNC, 1086 .cra_blocksize = SHA256_BLOCK_SIZE, 1087 .cra_ctxsize = sizeof(struct atmel_sha_ctx), 1088 .cra_alignmask = 0, 1089 .cra_module = THIS_MODULE, 1090 .cra_init = atmel_sha_cra_init, 1091 } 1092 } 1093 }, 1094 }; 1095 1096 static struct ahash_alg sha_224_alg = { 1097 .init = atmel_sha_init, 1098 .update = atmel_sha_update, 1099 .final = atmel_sha_final, 1100 .finup = atmel_sha_finup, 1101 .digest = atmel_sha_digest, 1102 .export = atmel_sha_export, 1103 .import = atmel_sha_import, 1104 .halg = { 1105 .digestsize = SHA224_DIGEST_SIZE, 1106 .statesize = sizeof(struct atmel_sha_reqctx), 1107 .base = { 1108 .cra_name = "sha224", 1109 .cra_driver_name = "atmel-sha224", 1110 .cra_priority = 100, 1111 .cra_flags = CRYPTO_ALG_ASYNC, 1112 .cra_blocksize = SHA224_BLOCK_SIZE, 1113 .cra_ctxsize = sizeof(struct atmel_sha_ctx), 1114 .cra_alignmask = 0, 1115 .cra_module = THIS_MODULE, 1116 .cra_init = atmel_sha_cra_init, 1117 } 1118 } 1119 }; 1120 1121 static struct ahash_alg sha_384_512_algs[] = { 1122 { 1123 .init = atmel_sha_init, 1124 .update = atmel_sha_update, 1125 .final = atmel_sha_final, 1126 .finup = atmel_sha_finup, 1127 .digest = atmel_sha_digest, 1128 .export = atmel_sha_export, 1129 .import = atmel_sha_import, 1130 .halg = { 1131 .digestsize = SHA384_DIGEST_SIZE, 1132 .statesize = sizeof(struct atmel_sha_reqctx), 1133 .base = { 1134 .cra_name = "sha384", 1135 .cra_driver_name = "atmel-sha384", 1136 .cra_priority = 100, 1137 .cra_flags = CRYPTO_ALG_ASYNC, 1138 .cra_blocksize = SHA384_BLOCK_SIZE, 1139 .cra_ctxsize = sizeof(struct atmel_sha_ctx), 1140 .cra_alignmask = 0x3, 1141 .cra_module = THIS_MODULE, 1142 .cra_init = atmel_sha_cra_init, 1143 } 1144 } 1145 }, 1146 { 1147 .init = atmel_sha_init, 1148 .update = atmel_sha_update, 1149 .final = atmel_sha_final, 1150 .finup = atmel_sha_finup, 1151 .digest = atmel_sha_digest, 1152 .export = atmel_sha_export, 1153 .import = atmel_sha_import, 1154 .halg = { 1155 .digestsize = SHA512_DIGEST_SIZE, 1156 .statesize = sizeof(struct atmel_sha_reqctx), 1157 .base = { 1158 .cra_name = "sha512", 1159 .cra_driver_name = "atmel-sha512", 1160 .cra_priority = 100, 1161 .cra_flags = CRYPTO_ALG_ASYNC, 1162 .cra_blocksize = SHA512_BLOCK_SIZE, 1163 .cra_ctxsize = sizeof(struct atmel_sha_ctx), 1164 .cra_alignmask = 0x3, 1165 .cra_module = THIS_MODULE, 1166 .cra_init = atmel_sha_cra_init, 1167 } 1168 } 1169 }, 1170 }; 1171 1172 static void atmel_sha_queue_task(unsigned long data) 1173 { 1174 struct atmel_sha_dev *dd = (struct atmel_sha_dev *)data; 1175 1176 atmel_sha_handle_queue(dd, NULL); 1177 } 1178 1179 static void atmel_sha_done_task(unsigned long data) 1180 { 1181 struct atmel_sha_dev *dd = (struct atmel_sha_dev *)data; 1182 int err = 0; 1183 1184 if (SHA_FLAGS_CPU & dd->flags) { 1185 if (SHA_FLAGS_OUTPUT_READY & dd->flags) { 1186 dd->flags &= ~SHA_FLAGS_OUTPUT_READY; 1187 goto finish; 1188 } 1189 } else if (SHA_FLAGS_DMA_READY & dd->flags) { 1190 if (SHA_FLAGS_DMA_ACTIVE & dd->flags) { 1191 dd->flags &= ~SHA_FLAGS_DMA_ACTIVE; 1192 atmel_sha_update_dma_stop(dd); 1193 if (dd->err) { 1194 err = dd->err; 1195 goto finish; 1196 } 1197 } 1198 if (SHA_FLAGS_OUTPUT_READY & dd->flags) { 1199 /* hash or semi-hash ready */ 1200 dd->flags &= ~(SHA_FLAGS_DMA_READY | 1201 SHA_FLAGS_OUTPUT_READY); 1202 err = atmel_sha_update_dma_start(dd); 1203 if (err != -EINPROGRESS) 1204 goto finish; 1205 } 1206 } 1207 return; 1208 1209 finish: 1210 /* finish curent request */ 1211 atmel_sha_finish_req(dd->req, err); 1212 } 1213 1214 static irqreturn_t atmel_sha_irq(int irq, void *dev_id) 1215 { 1216 struct atmel_sha_dev *sha_dd = dev_id; 1217 u32 reg; 1218 1219 reg = atmel_sha_read(sha_dd, SHA_ISR); 1220 if (reg & atmel_sha_read(sha_dd, SHA_IMR)) { 1221 atmel_sha_write(sha_dd, SHA_IDR, reg); 1222 if (SHA_FLAGS_BUSY & sha_dd->flags) { 1223 sha_dd->flags |= SHA_FLAGS_OUTPUT_READY; 1224 if (!(SHA_FLAGS_CPU & sha_dd->flags)) 1225 sha_dd->flags |= SHA_FLAGS_DMA_READY; 1226 tasklet_schedule(&sha_dd->done_task); 1227 } else { 1228 dev_warn(sha_dd->dev, "SHA interrupt when no active requests.\n"); 1229 } 1230 return IRQ_HANDLED; 1231 } 1232 1233 return IRQ_NONE; 1234 } 1235 1236 static void atmel_sha_unregister_algs(struct atmel_sha_dev *dd) 1237 { 1238 int i; 1239 1240 for (i = 0; i < ARRAY_SIZE(sha_1_256_algs); i++) 1241 crypto_unregister_ahash(&sha_1_256_algs[i]); 1242 1243 if (dd->caps.has_sha224) 1244 crypto_unregister_ahash(&sha_224_alg); 1245 1246 if (dd->caps.has_sha_384_512) { 1247 for (i = 0; i < ARRAY_SIZE(sha_384_512_algs); i++) 1248 crypto_unregister_ahash(&sha_384_512_algs[i]); 1249 } 1250 } 1251 1252 static int atmel_sha_register_algs(struct atmel_sha_dev *dd) 1253 { 1254 int err, i, j; 1255 1256 for (i = 0; i < ARRAY_SIZE(sha_1_256_algs); i++) { 1257 err = crypto_register_ahash(&sha_1_256_algs[i]); 1258 if (err) 1259 goto err_sha_1_256_algs; 1260 } 1261 1262 if (dd->caps.has_sha224) { 1263 err = crypto_register_ahash(&sha_224_alg); 1264 if (err) 1265 goto err_sha_224_algs; 1266 } 1267 1268 if (dd->caps.has_sha_384_512) { 1269 for (i = 0; i < ARRAY_SIZE(sha_384_512_algs); i++) { 1270 err = crypto_register_ahash(&sha_384_512_algs[i]); 1271 if (err) 1272 goto err_sha_384_512_algs; 1273 } 1274 } 1275 1276 return 0; 1277 1278 err_sha_384_512_algs: 1279 for (j = 0; j < i; j++) 1280 crypto_unregister_ahash(&sha_384_512_algs[j]); 1281 crypto_unregister_ahash(&sha_224_alg); 1282 err_sha_224_algs: 1283 i = ARRAY_SIZE(sha_1_256_algs); 1284 err_sha_1_256_algs: 1285 for (j = 0; j < i; j++) 1286 crypto_unregister_ahash(&sha_1_256_algs[j]); 1287 1288 return err; 1289 } 1290 1291 static bool atmel_sha_filter(struct dma_chan *chan, void *slave) 1292 { 1293 struct at_dma_slave *sl = slave; 1294 1295 if (sl && sl->dma_dev == chan->device->dev) { 1296 chan->private = sl; 1297 return true; 1298 } else { 1299 return false; 1300 } 1301 } 1302 1303 static int atmel_sha_dma_init(struct atmel_sha_dev *dd, 1304 struct crypto_platform_data *pdata) 1305 { 1306 int err = -ENOMEM; 1307 dma_cap_mask_t mask_in; 1308 1309 /* Try to grab DMA channel */ 1310 dma_cap_zero(mask_in); 1311 dma_cap_set(DMA_SLAVE, mask_in); 1312 1313 dd->dma_lch_in.chan = dma_request_slave_channel_compat(mask_in, 1314 atmel_sha_filter, &pdata->dma_slave->rxdata, dd->dev, "tx"); 1315 if (!dd->dma_lch_in.chan) { 1316 dev_warn(dd->dev, "no DMA channel available\n"); 1317 return err; 1318 } 1319 1320 dd->dma_lch_in.dma_conf.direction = DMA_MEM_TO_DEV; 1321 dd->dma_lch_in.dma_conf.dst_addr = dd->phys_base + 1322 SHA_REG_DIN(0); 1323 dd->dma_lch_in.dma_conf.src_maxburst = 1; 1324 dd->dma_lch_in.dma_conf.src_addr_width = 1325 DMA_SLAVE_BUSWIDTH_4_BYTES; 1326 dd->dma_lch_in.dma_conf.dst_maxburst = 1; 1327 dd->dma_lch_in.dma_conf.dst_addr_width = 1328 DMA_SLAVE_BUSWIDTH_4_BYTES; 1329 dd->dma_lch_in.dma_conf.device_fc = false; 1330 1331 return 0; 1332 } 1333 1334 static void atmel_sha_dma_cleanup(struct atmel_sha_dev *dd) 1335 { 1336 dma_release_channel(dd->dma_lch_in.chan); 1337 } 1338 1339 static void atmel_sha_get_cap(struct atmel_sha_dev *dd) 1340 { 1341 1342 dd->caps.has_dma = 0; 1343 dd->caps.has_dualbuff = 0; 1344 dd->caps.has_sha224 = 0; 1345 dd->caps.has_sha_384_512 = 0; 1346 dd->caps.has_uihv = 0; 1347 1348 /* keep only major version number */ 1349 switch (dd->hw_version & 0xff0) { 1350 case 0x510: 1351 dd->caps.has_dma = 1; 1352 dd->caps.has_dualbuff = 1; 1353 dd->caps.has_sha224 = 1; 1354 dd->caps.has_sha_384_512 = 1; 1355 dd->caps.has_uihv = 1; 1356 break; 1357 case 0x420: 1358 dd->caps.has_dma = 1; 1359 dd->caps.has_dualbuff = 1; 1360 dd->caps.has_sha224 = 1; 1361 dd->caps.has_sha_384_512 = 1; 1362 dd->caps.has_uihv = 1; 1363 break; 1364 case 0x410: 1365 dd->caps.has_dma = 1; 1366 dd->caps.has_dualbuff = 1; 1367 dd->caps.has_sha224 = 1; 1368 dd->caps.has_sha_384_512 = 1; 1369 break; 1370 case 0x400: 1371 dd->caps.has_dma = 1; 1372 dd->caps.has_dualbuff = 1; 1373 dd->caps.has_sha224 = 1; 1374 break; 1375 case 0x320: 1376 break; 1377 default: 1378 dev_warn(dd->dev, 1379 "Unmanaged sha version, set minimum capabilities\n"); 1380 break; 1381 } 1382 } 1383 1384 #if defined(CONFIG_OF) 1385 static const struct of_device_id atmel_sha_dt_ids[] = { 1386 { .compatible = "atmel,at91sam9g46-sha" }, 1387 { /* sentinel */ } 1388 }; 1389 1390 MODULE_DEVICE_TABLE(of, atmel_sha_dt_ids); 1391 1392 static struct crypto_platform_data *atmel_sha_of_init(struct platform_device *pdev) 1393 { 1394 struct device_node *np = pdev->dev.of_node; 1395 struct crypto_platform_data *pdata; 1396 1397 if (!np) { 1398 dev_err(&pdev->dev, "device node not found\n"); 1399 return ERR_PTR(-EINVAL); 1400 } 1401 1402 pdata = devm_kzalloc(&pdev->dev, sizeof(*pdata), GFP_KERNEL); 1403 if (!pdata) { 1404 dev_err(&pdev->dev, "could not allocate memory for pdata\n"); 1405 return ERR_PTR(-ENOMEM); 1406 } 1407 1408 pdata->dma_slave = devm_kzalloc(&pdev->dev, 1409 sizeof(*(pdata->dma_slave)), 1410 GFP_KERNEL); 1411 if (!pdata->dma_slave) { 1412 dev_err(&pdev->dev, "could not allocate memory for dma_slave\n"); 1413 return ERR_PTR(-ENOMEM); 1414 } 1415 1416 return pdata; 1417 } 1418 #else /* CONFIG_OF */ 1419 static inline struct crypto_platform_data *atmel_sha_of_init(struct platform_device *dev) 1420 { 1421 return ERR_PTR(-EINVAL); 1422 } 1423 #endif 1424 1425 static int atmel_sha_probe(struct platform_device *pdev) 1426 { 1427 struct atmel_sha_dev *sha_dd; 1428 struct crypto_platform_data *pdata; 1429 struct device *dev = &pdev->dev; 1430 struct resource *sha_res; 1431 int err; 1432 1433 sha_dd = devm_kzalloc(&pdev->dev, sizeof(*sha_dd), GFP_KERNEL); 1434 if (sha_dd == NULL) { 1435 dev_err(dev, "unable to alloc data struct.\n"); 1436 err = -ENOMEM; 1437 goto sha_dd_err; 1438 } 1439 1440 sha_dd->dev = dev; 1441 1442 platform_set_drvdata(pdev, sha_dd); 1443 1444 INIT_LIST_HEAD(&sha_dd->list); 1445 spin_lock_init(&sha_dd->lock); 1446 1447 tasklet_init(&sha_dd->done_task, atmel_sha_done_task, 1448 (unsigned long)sha_dd); 1449 tasklet_init(&sha_dd->queue_task, atmel_sha_queue_task, 1450 (unsigned long)sha_dd); 1451 1452 crypto_init_queue(&sha_dd->queue, ATMEL_SHA_QUEUE_LENGTH); 1453 1454 sha_dd->irq = -1; 1455 1456 /* Get the base address */ 1457 sha_res = platform_get_resource(pdev, IORESOURCE_MEM, 0); 1458 if (!sha_res) { 1459 dev_err(dev, "no MEM resource info\n"); 1460 err = -ENODEV; 1461 goto res_err; 1462 } 1463 sha_dd->phys_base = sha_res->start; 1464 1465 /* Get the IRQ */ 1466 sha_dd->irq = platform_get_irq(pdev, 0); 1467 if (sha_dd->irq < 0) { 1468 dev_err(dev, "no IRQ resource info\n"); 1469 err = sha_dd->irq; 1470 goto res_err; 1471 } 1472 1473 err = devm_request_irq(&pdev->dev, sha_dd->irq, atmel_sha_irq, 1474 IRQF_SHARED, "atmel-sha", sha_dd); 1475 if (err) { 1476 dev_err(dev, "unable to request sha irq.\n"); 1477 goto res_err; 1478 } 1479 1480 /* Initializing the clock */ 1481 sha_dd->iclk = devm_clk_get(&pdev->dev, "sha_clk"); 1482 if (IS_ERR(sha_dd->iclk)) { 1483 dev_err(dev, "clock initialization failed.\n"); 1484 err = PTR_ERR(sha_dd->iclk); 1485 goto res_err; 1486 } 1487 1488 sha_dd->io_base = devm_ioremap_resource(&pdev->dev, sha_res); 1489 if (IS_ERR(sha_dd->io_base)) { 1490 dev_err(dev, "can't ioremap\n"); 1491 err = PTR_ERR(sha_dd->io_base); 1492 goto res_err; 1493 } 1494 1495 err = clk_prepare(sha_dd->iclk); 1496 if (err) 1497 goto res_err; 1498 1499 atmel_sha_hw_version_init(sha_dd); 1500 1501 atmel_sha_get_cap(sha_dd); 1502 1503 if (sha_dd->caps.has_dma) { 1504 pdata = pdev->dev.platform_data; 1505 if (!pdata) { 1506 pdata = atmel_sha_of_init(pdev); 1507 if (IS_ERR(pdata)) { 1508 dev_err(&pdev->dev, "platform data not available\n"); 1509 err = PTR_ERR(pdata); 1510 goto iclk_unprepare; 1511 } 1512 } 1513 if (!pdata->dma_slave) { 1514 err = -ENXIO; 1515 goto iclk_unprepare; 1516 } 1517 err = atmel_sha_dma_init(sha_dd, pdata); 1518 if (err) 1519 goto err_sha_dma; 1520 1521 dev_info(dev, "using %s for DMA transfers\n", 1522 dma_chan_name(sha_dd->dma_lch_in.chan)); 1523 } 1524 1525 spin_lock(&atmel_sha.lock); 1526 list_add_tail(&sha_dd->list, &atmel_sha.dev_list); 1527 spin_unlock(&atmel_sha.lock); 1528 1529 err = atmel_sha_register_algs(sha_dd); 1530 if (err) 1531 goto err_algs; 1532 1533 dev_info(dev, "Atmel SHA1/SHA256%s%s\n", 1534 sha_dd->caps.has_sha224 ? "/SHA224" : "", 1535 sha_dd->caps.has_sha_384_512 ? "/SHA384/SHA512" : ""); 1536 1537 return 0; 1538 1539 err_algs: 1540 spin_lock(&atmel_sha.lock); 1541 list_del(&sha_dd->list); 1542 spin_unlock(&atmel_sha.lock); 1543 if (sha_dd->caps.has_dma) 1544 atmel_sha_dma_cleanup(sha_dd); 1545 err_sha_dma: 1546 iclk_unprepare: 1547 clk_unprepare(sha_dd->iclk); 1548 res_err: 1549 tasklet_kill(&sha_dd->queue_task); 1550 tasklet_kill(&sha_dd->done_task); 1551 sha_dd_err: 1552 dev_err(dev, "initialization failed.\n"); 1553 1554 return err; 1555 } 1556 1557 static int atmel_sha_remove(struct platform_device *pdev) 1558 { 1559 static struct atmel_sha_dev *sha_dd; 1560 1561 sha_dd = platform_get_drvdata(pdev); 1562 if (!sha_dd) 1563 return -ENODEV; 1564 spin_lock(&atmel_sha.lock); 1565 list_del(&sha_dd->list); 1566 spin_unlock(&atmel_sha.lock); 1567 1568 atmel_sha_unregister_algs(sha_dd); 1569 1570 tasklet_kill(&sha_dd->queue_task); 1571 tasklet_kill(&sha_dd->done_task); 1572 1573 if (sha_dd->caps.has_dma) 1574 atmel_sha_dma_cleanup(sha_dd); 1575 1576 clk_unprepare(sha_dd->iclk); 1577 1578 return 0; 1579 } 1580 1581 static struct platform_driver atmel_sha_driver = { 1582 .probe = atmel_sha_probe, 1583 .remove = atmel_sha_remove, 1584 .driver = { 1585 .name = "atmel_sha", 1586 .of_match_table = of_match_ptr(atmel_sha_dt_ids), 1587 }, 1588 }; 1589 1590 module_platform_driver(atmel_sha_driver); 1591 1592 MODULE_DESCRIPTION("Atmel SHA (1/256/224/384/512) hw acceleration support."); 1593 MODULE_LICENSE("GPL v2"); 1594 MODULE_AUTHOR("Nicolas Royer - Eukréa Electromatique"); 1595