1 // SPDX-License-Identifier: GPL-2.0 2 /* 3 * Cryptographic API. 4 * 5 * Support for ATMEL SHA1/SHA256 HW acceleration. 6 * 7 * Copyright (c) 2012 Eukréa Electromatique - ATMEL 8 * Author: Nicolas Royer <nicolas@eukrea.com> 9 * 10 * Some ideas are from omap-sham.c drivers. 11 */ 12 13 14 #include <linux/kernel.h> 15 #include <linux/module.h> 16 #include <linux/slab.h> 17 #include <linux/err.h> 18 #include <linux/clk.h> 19 #include <linux/io.h> 20 #include <linux/hw_random.h> 21 #include <linux/platform_device.h> 22 23 #include <linux/device.h> 24 #include <linux/dmaengine.h> 25 #include <linux/init.h> 26 #include <linux/errno.h> 27 #include <linux/interrupt.h> 28 #include <linux/irq.h> 29 #include <linux/scatterlist.h> 30 #include <linux/dma-mapping.h> 31 #include <linux/of_device.h> 32 #include <linux/delay.h> 33 #include <linux/crypto.h> 34 #include <linux/cryptohash.h> 35 #include <crypto/scatterwalk.h> 36 #include <crypto/algapi.h> 37 #include <crypto/sha.h> 38 #include <crypto/hash.h> 39 #include <crypto/internal/hash.h> 40 #include "atmel-sha-regs.h" 41 #include "atmel-authenc.h" 42 43 #define ATMEL_SHA_PRIORITY 300 44 45 /* SHA flags */ 46 #define SHA_FLAGS_BUSY BIT(0) 47 #define SHA_FLAGS_FINAL BIT(1) 48 #define SHA_FLAGS_DMA_ACTIVE BIT(2) 49 #define SHA_FLAGS_OUTPUT_READY BIT(3) 50 #define SHA_FLAGS_INIT BIT(4) 51 #define SHA_FLAGS_CPU BIT(5) 52 #define SHA_FLAGS_DMA_READY BIT(6) 53 #define SHA_FLAGS_DUMP_REG BIT(7) 54 55 /* bits[11:8] are reserved. */ 56 57 #define SHA_FLAGS_FINUP BIT(16) 58 #define SHA_FLAGS_SG BIT(17) 59 #define SHA_FLAGS_ERROR BIT(23) 60 #define SHA_FLAGS_PAD BIT(24) 61 #define SHA_FLAGS_RESTORE BIT(25) 62 #define SHA_FLAGS_IDATAR0 BIT(26) 63 #define SHA_FLAGS_WAIT_DATARDY BIT(27) 64 65 #define SHA_OP_INIT 0 66 #define SHA_OP_UPDATE 1 67 #define SHA_OP_FINAL 2 68 #define SHA_OP_DIGEST 3 69 70 #define SHA_BUFFER_LEN (PAGE_SIZE / 16) 71 72 #define ATMEL_SHA_DMA_THRESHOLD 56 73 74 struct atmel_sha_caps { 75 bool has_dma; 76 bool has_dualbuff; 77 bool has_sha224; 78 bool has_sha_384_512; 79 bool has_uihv; 80 bool has_hmac; 81 }; 82 83 struct atmel_sha_dev; 84 85 /* 86 * .statesize = sizeof(struct atmel_sha_reqctx) must be <= PAGE_SIZE / 8 as 87 * tested by the ahash_prepare_alg() function. 88 */ 89 struct atmel_sha_reqctx { 90 struct atmel_sha_dev *dd; 91 unsigned long flags; 92 unsigned long op; 93 94 u8 digest[SHA512_DIGEST_SIZE] __aligned(sizeof(u32)); 95 u64 digcnt[2]; 96 size_t bufcnt; 97 size_t buflen; 98 dma_addr_t dma_addr; 99 100 /* walk state */ 101 struct scatterlist *sg; 102 unsigned int offset; /* offset in current sg */ 103 unsigned int total; /* total request */ 104 105 size_t block_size; 106 size_t hash_size; 107 108 u8 buffer[SHA_BUFFER_LEN + SHA512_BLOCK_SIZE] __aligned(sizeof(u32)); 109 }; 110 111 typedef int (*atmel_sha_fn_t)(struct atmel_sha_dev *); 112 113 struct atmel_sha_ctx { 114 struct atmel_sha_dev *dd; 115 atmel_sha_fn_t start; 116 117 unsigned long flags; 118 }; 119 120 #define ATMEL_SHA_QUEUE_LENGTH 50 121 122 struct atmel_sha_dma { 123 struct dma_chan *chan; 124 struct dma_slave_config dma_conf; 125 struct scatterlist *sg; 126 int nents; 127 unsigned int last_sg_length; 128 }; 129 130 struct atmel_sha_dev { 131 struct list_head list; 132 unsigned long phys_base; 133 struct device *dev; 134 struct clk *iclk; 135 int irq; 136 void __iomem *io_base; 137 138 spinlock_t lock; 139 struct tasklet_struct done_task; 140 struct tasklet_struct queue_task; 141 142 unsigned long flags; 143 struct crypto_queue queue; 144 struct ahash_request *req; 145 bool is_async; 146 bool force_complete; 147 atmel_sha_fn_t resume; 148 atmel_sha_fn_t cpu_transfer_complete; 149 150 struct atmel_sha_dma dma_lch_in; 151 152 struct atmel_sha_caps caps; 153 154 struct scatterlist tmp; 155 156 u32 hw_version; 157 }; 158 159 struct atmel_sha_drv { 160 struct list_head dev_list; 161 spinlock_t lock; 162 }; 163 164 static struct atmel_sha_drv atmel_sha = { 165 .dev_list = LIST_HEAD_INIT(atmel_sha.dev_list), 166 .lock = __SPIN_LOCK_UNLOCKED(atmel_sha.lock), 167 }; 168 169 #ifdef VERBOSE_DEBUG 170 static const char *atmel_sha_reg_name(u32 offset, char *tmp, size_t sz, bool wr) 171 { 172 switch (offset) { 173 case SHA_CR: 174 return "CR"; 175 176 case SHA_MR: 177 return "MR"; 178 179 case SHA_IER: 180 return "IER"; 181 182 case SHA_IDR: 183 return "IDR"; 184 185 case SHA_IMR: 186 return "IMR"; 187 188 case SHA_ISR: 189 return "ISR"; 190 191 case SHA_MSR: 192 return "MSR"; 193 194 case SHA_BCR: 195 return "BCR"; 196 197 case SHA_REG_DIN(0): 198 case SHA_REG_DIN(1): 199 case SHA_REG_DIN(2): 200 case SHA_REG_DIN(3): 201 case SHA_REG_DIN(4): 202 case SHA_REG_DIN(5): 203 case SHA_REG_DIN(6): 204 case SHA_REG_DIN(7): 205 case SHA_REG_DIN(8): 206 case SHA_REG_DIN(9): 207 case SHA_REG_DIN(10): 208 case SHA_REG_DIN(11): 209 case SHA_REG_DIN(12): 210 case SHA_REG_DIN(13): 211 case SHA_REG_DIN(14): 212 case SHA_REG_DIN(15): 213 snprintf(tmp, sz, "IDATAR[%u]", (offset - SHA_REG_DIN(0)) >> 2); 214 break; 215 216 case SHA_REG_DIGEST(0): 217 case SHA_REG_DIGEST(1): 218 case SHA_REG_DIGEST(2): 219 case SHA_REG_DIGEST(3): 220 case SHA_REG_DIGEST(4): 221 case SHA_REG_DIGEST(5): 222 case SHA_REG_DIGEST(6): 223 case SHA_REG_DIGEST(7): 224 case SHA_REG_DIGEST(8): 225 case SHA_REG_DIGEST(9): 226 case SHA_REG_DIGEST(10): 227 case SHA_REG_DIGEST(11): 228 case SHA_REG_DIGEST(12): 229 case SHA_REG_DIGEST(13): 230 case SHA_REG_DIGEST(14): 231 case SHA_REG_DIGEST(15): 232 if (wr) 233 snprintf(tmp, sz, "IDATAR[%u]", 234 16u + ((offset - SHA_REG_DIGEST(0)) >> 2)); 235 else 236 snprintf(tmp, sz, "ODATAR[%u]", 237 (offset - SHA_REG_DIGEST(0)) >> 2); 238 break; 239 240 case SHA_HW_VERSION: 241 return "HWVER"; 242 243 default: 244 snprintf(tmp, sz, "0x%02x", offset); 245 break; 246 } 247 248 return tmp; 249 } 250 251 #endif /* VERBOSE_DEBUG */ 252 253 static inline u32 atmel_sha_read(struct atmel_sha_dev *dd, u32 offset) 254 { 255 u32 value = readl_relaxed(dd->io_base + offset); 256 257 #ifdef VERBOSE_DEBUG 258 if (dd->flags & SHA_FLAGS_DUMP_REG) { 259 char tmp[16]; 260 261 dev_vdbg(dd->dev, "read 0x%08x from %s\n", value, 262 atmel_sha_reg_name(offset, tmp, sizeof(tmp), false)); 263 } 264 #endif /* VERBOSE_DEBUG */ 265 266 return value; 267 } 268 269 static inline void atmel_sha_write(struct atmel_sha_dev *dd, 270 u32 offset, u32 value) 271 { 272 #ifdef VERBOSE_DEBUG 273 if (dd->flags & SHA_FLAGS_DUMP_REG) { 274 char tmp[16]; 275 276 dev_vdbg(dd->dev, "write 0x%08x into %s\n", value, 277 atmel_sha_reg_name(offset, tmp, sizeof(tmp), true)); 278 } 279 #endif /* VERBOSE_DEBUG */ 280 281 writel_relaxed(value, dd->io_base + offset); 282 } 283 284 static inline int atmel_sha_complete(struct atmel_sha_dev *dd, int err) 285 { 286 struct ahash_request *req = dd->req; 287 288 dd->flags &= ~(SHA_FLAGS_BUSY | SHA_FLAGS_FINAL | SHA_FLAGS_CPU | 289 SHA_FLAGS_DMA_READY | SHA_FLAGS_OUTPUT_READY | 290 SHA_FLAGS_DUMP_REG); 291 292 clk_disable(dd->iclk); 293 294 if ((dd->is_async || dd->force_complete) && req->base.complete) 295 req->base.complete(&req->base, err); 296 297 /* handle new request */ 298 tasklet_schedule(&dd->queue_task); 299 300 return err; 301 } 302 303 static size_t atmel_sha_append_sg(struct atmel_sha_reqctx *ctx) 304 { 305 size_t count; 306 307 while ((ctx->bufcnt < ctx->buflen) && ctx->total) { 308 count = min(ctx->sg->length - ctx->offset, ctx->total); 309 count = min(count, ctx->buflen - ctx->bufcnt); 310 311 if (count <= 0) { 312 /* 313 * Check if count <= 0 because the buffer is full or 314 * because the sg length is 0. In the latest case, 315 * check if there is another sg in the list, a 0 length 316 * sg doesn't necessarily mean the end of the sg list. 317 */ 318 if ((ctx->sg->length == 0) && !sg_is_last(ctx->sg)) { 319 ctx->sg = sg_next(ctx->sg); 320 continue; 321 } else { 322 break; 323 } 324 } 325 326 scatterwalk_map_and_copy(ctx->buffer + ctx->bufcnt, ctx->sg, 327 ctx->offset, count, 0); 328 329 ctx->bufcnt += count; 330 ctx->offset += count; 331 ctx->total -= count; 332 333 if (ctx->offset == ctx->sg->length) { 334 ctx->sg = sg_next(ctx->sg); 335 if (ctx->sg) 336 ctx->offset = 0; 337 else 338 ctx->total = 0; 339 } 340 } 341 342 return 0; 343 } 344 345 /* 346 * The purpose of this padding is to ensure that the padded message is a 347 * multiple of 512 bits (SHA1/SHA224/SHA256) or 1024 bits (SHA384/SHA512). 348 * The bit "1" is appended at the end of the message followed by 349 * "padlen-1" zero bits. Then a 64 bits block (SHA1/SHA224/SHA256) or 350 * 128 bits block (SHA384/SHA512) equals to the message length in bits 351 * is appended. 352 * 353 * For SHA1/SHA224/SHA256, padlen is calculated as followed: 354 * - if message length < 56 bytes then padlen = 56 - message length 355 * - else padlen = 64 + 56 - message length 356 * 357 * For SHA384/SHA512, padlen is calculated as followed: 358 * - if message length < 112 bytes then padlen = 112 - message length 359 * - else padlen = 128 + 112 - message length 360 */ 361 static void atmel_sha_fill_padding(struct atmel_sha_reqctx *ctx, int length) 362 { 363 unsigned int index, padlen; 364 __be64 bits[2]; 365 u64 size[2]; 366 367 size[0] = ctx->digcnt[0]; 368 size[1] = ctx->digcnt[1]; 369 370 size[0] += ctx->bufcnt; 371 if (size[0] < ctx->bufcnt) 372 size[1]++; 373 374 size[0] += length; 375 if (size[0] < length) 376 size[1]++; 377 378 bits[1] = cpu_to_be64(size[0] << 3); 379 bits[0] = cpu_to_be64(size[1] << 3 | size[0] >> 61); 380 381 switch (ctx->flags & SHA_FLAGS_ALGO_MASK) { 382 case SHA_FLAGS_SHA384: 383 case SHA_FLAGS_SHA512: 384 index = ctx->bufcnt & 0x7f; 385 padlen = (index < 112) ? (112 - index) : ((128+112) - index); 386 *(ctx->buffer + ctx->bufcnt) = 0x80; 387 memset(ctx->buffer + ctx->bufcnt + 1, 0, padlen-1); 388 memcpy(ctx->buffer + ctx->bufcnt + padlen, bits, 16); 389 ctx->bufcnt += padlen + 16; 390 ctx->flags |= SHA_FLAGS_PAD; 391 break; 392 393 default: 394 index = ctx->bufcnt & 0x3f; 395 padlen = (index < 56) ? (56 - index) : ((64+56) - index); 396 *(ctx->buffer + ctx->bufcnt) = 0x80; 397 memset(ctx->buffer + ctx->bufcnt + 1, 0, padlen-1); 398 memcpy(ctx->buffer + ctx->bufcnt + padlen, &bits[1], 8); 399 ctx->bufcnt += padlen + 8; 400 ctx->flags |= SHA_FLAGS_PAD; 401 break; 402 } 403 } 404 405 static struct atmel_sha_dev *atmel_sha_find_dev(struct atmel_sha_ctx *tctx) 406 { 407 struct atmel_sha_dev *dd = NULL; 408 struct atmel_sha_dev *tmp; 409 410 spin_lock_bh(&atmel_sha.lock); 411 if (!tctx->dd) { 412 list_for_each_entry(tmp, &atmel_sha.dev_list, list) { 413 dd = tmp; 414 break; 415 } 416 tctx->dd = dd; 417 } else { 418 dd = tctx->dd; 419 } 420 421 spin_unlock_bh(&atmel_sha.lock); 422 423 return dd; 424 } 425 426 static int atmel_sha_init(struct ahash_request *req) 427 { 428 struct crypto_ahash *tfm = crypto_ahash_reqtfm(req); 429 struct atmel_sha_ctx *tctx = crypto_ahash_ctx(tfm); 430 struct atmel_sha_reqctx *ctx = ahash_request_ctx(req); 431 struct atmel_sha_dev *dd = atmel_sha_find_dev(tctx); 432 433 ctx->dd = dd; 434 435 ctx->flags = 0; 436 437 dev_dbg(dd->dev, "init: digest size: %d\n", 438 crypto_ahash_digestsize(tfm)); 439 440 switch (crypto_ahash_digestsize(tfm)) { 441 case SHA1_DIGEST_SIZE: 442 ctx->flags |= SHA_FLAGS_SHA1; 443 ctx->block_size = SHA1_BLOCK_SIZE; 444 break; 445 case SHA224_DIGEST_SIZE: 446 ctx->flags |= SHA_FLAGS_SHA224; 447 ctx->block_size = SHA224_BLOCK_SIZE; 448 break; 449 case SHA256_DIGEST_SIZE: 450 ctx->flags |= SHA_FLAGS_SHA256; 451 ctx->block_size = SHA256_BLOCK_SIZE; 452 break; 453 case SHA384_DIGEST_SIZE: 454 ctx->flags |= SHA_FLAGS_SHA384; 455 ctx->block_size = SHA384_BLOCK_SIZE; 456 break; 457 case SHA512_DIGEST_SIZE: 458 ctx->flags |= SHA_FLAGS_SHA512; 459 ctx->block_size = SHA512_BLOCK_SIZE; 460 break; 461 default: 462 return -EINVAL; 463 break; 464 } 465 466 ctx->bufcnt = 0; 467 ctx->digcnt[0] = 0; 468 ctx->digcnt[1] = 0; 469 ctx->buflen = SHA_BUFFER_LEN; 470 471 return 0; 472 } 473 474 static void atmel_sha_write_ctrl(struct atmel_sha_dev *dd, int dma) 475 { 476 struct atmel_sha_reqctx *ctx = ahash_request_ctx(dd->req); 477 u32 valmr = SHA_MR_MODE_AUTO; 478 unsigned int i, hashsize = 0; 479 480 if (likely(dma)) { 481 if (!dd->caps.has_dma) 482 atmel_sha_write(dd, SHA_IER, SHA_INT_TXBUFE); 483 valmr = SHA_MR_MODE_PDC; 484 if (dd->caps.has_dualbuff) 485 valmr |= SHA_MR_DUALBUFF; 486 } else { 487 atmel_sha_write(dd, SHA_IER, SHA_INT_DATARDY); 488 } 489 490 switch (ctx->flags & SHA_FLAGS_ALGO_MASK) { 491 case SHA_FLAGS_SHA1: 492 valmr |= SHA_MR_ALGO_SHA1; 493 hashsize = SHA1_DIGEST_SIZE; 494 break; 495 496 case SHA_FLAGS_SHA224: 497 valmr |= SHA_MR_ALGO_SHA224; 498 hashsize = SHA256_DIGEST_SIZE; 499 break; 500 501 case SHA_FLAGS_SHA256: 502 valmr |= SHA_MR_ALGO_SHA256; 503 hashsize = SHA256_DIGEST_SIZE; 504 break; 505 506 case SHA_FLAGS_SHA384: 507 valmr |= SHA_MR_ALGO_SHA384; 508 hashsize = SHA512_DIGEST_SIZE; 509 break; 510 511 case SHA_FLAGS_SHA512: 512 valmr |= SHA_MR_ALGO_SHA512; 513 hashsize = SHA512_DIGEST_SIZE; 514 break; 515 516 default: 517 break; 518 } 519 520 /* Setting CR_FIRST only for the first iteration */ 521 if (!(ctx->digcnt[0] || ctx->digcnt[1])) { 522 atmel_sha_write(dd, SHA_CR, SHA_CR_FIRST); 523 } else if (dd->caps.has_uihv && (ctx->flags & SHA_FLAGS_RESTORE)) { 524 const u32 *hash = (const u32 *)ctx->digest; 525 526 /* 527 * Restore the hardware context: update the User Initialize 528 * Hash Value (UIHV) with the value saved when the latest 529 * 'update' operation completed on this very same crypto 530 * request. 531 */ 532 ctx->flags &= ~SHA_FLAGS_RESTORE; 533 atmel_sha_write(dd, SHA_CR, SHA_CR_WUIHV); 534 for (i = 0; i < hashsize / sizeof(u32); ++i) 535 atmel_sha_write(dd, SHA_REG_DIN(i), hash[i]); 536 atmel_sha_write(dd, SHA_CR, SHA_CR_FIRST); 537 valmr |= SHA_MR_UIHV; 538 } 539 /* 540 * WARNING: If the UIHV feature is not available, the hardware CANNOT 541 * process concurrent requests: the internal registers used to store 542 * the hash/digest are still set to the partial digest output values 543 * computed during the latest round. 544 */ 545 546 atmel_sha_write(dd, SHA_MR, valmr); 547 } 548 549 static inline int atmel_sha_wait_for_data_ready(struct atmel_sha_dev *dd, 550 atmel_sha_fn_t resume) 551 { 552 u32 isr = atmel_sha_read(dd, SHA_ISR); 553 554 if (unlikely(isr & SHA_INT_DATARDY)) 555 return resume(dd); 556 557 dd->resume = resume; 558 atmel_sha_write(dd, SHA_IER, SHA_INT_DATARDY); 559 return -EINPROGRESS; 560 } 561 562 static int atmel_sha_xmit_cpu(struct atmel_sha_dev *dd, const u8 *buf, 563 size_t length, int final) 564 { 565 struct atmel_sha_reqctx *ctx = ahash_request_ctx(dd->req); 566 int count, len32; 567 const u32 *buffer = (const u32 *)buf; 568 569 dev_dbg(dd->dev, "xmit_cpu: digcnt: 0x%llx 0x%llx, length: %zd, final: %d\n", 570 ctx->digcnt[1], ctx->digcnt[0], length, final); 571 572 atmel_sha_write_ctrl(dd, 0); 573 574 /* should be non-zero before next lines to disable clocks later */ 575 ctx->digcnt[0] += length; 576 if (ctx->digcnt[0] < length) 577 ctx->digcnt[1]++; 578 579 if (final) 580 dd->flags |= SHA_FLAGS_FINAL; /* catch last interrupt */ 581 582 len32 = DIV_ROUND_UP(length, sizeof(u32)); 583 584 dd->flags |= SHA_FLAGS_CPU; 585 586 for (count = 0; count < len32; count++) 587 atmel_sha_write(dd, SHA_REG_DIN(count), buffer[count]); 588 589 return -EINPROGRESS; 590 } 591 592 static int atmel_sha_xmit_pdc(struct atmel_sha_dev *dd, dma_addr_t dma_addr1, 593 size_t length1, dma_addr_t dma_addr2, size_t length2, int final) 594 { 595 struct atmel_sha_reqctx *ctx = ahash_request_ctx(dd->req); 596 int len32; 597 598 dev_dbg(dd->dev, "xmit_pdc: digcnt: 0x%llx 0x%llx, length: %zd, final: %d\n", 599 ctx->digcnt[1], ctx->digcnt[0], length1, final); 600 601 len32 = DIV_ROUND_UP(length1, sizeof(u32)); 602 atmel_sha_write(dd, SHA_PTCR, SHA_PTCR_TXTDIS); 603 atmel_sha_write(dd, SHA_TPR, dma_addr1); 604 atmel_sha_write(dd, SHA_TCR, len32); 605 606 len32 = DIV_ROUND_UP(length2, sizeof(u32)); 607 atmel_sha_write(dd, SHA_TNPR, dma_addr2); 608 atmel_sha_write(dd, SHA_TNCR, len32); 609 610 atmel_sha_write_ctrl(dd, 1); 611 612 /* should be non-zero before next lines to disable clocks later */ 613 ctx->digcnt[0] += length1; 614 if (ctx->digcnt[0] < length1) 615 ctx->digcnt[1]++; 616 617 if (final) 618 dd->flags |= SHA_FLAGS_FINAL; /* catch last interrupt */ 619 620 dd->flags |= SHA_FLAGS_DMA_ACTIVE; 621 622 /* Start DMA transfer */ 623 atmel_sha_write(dd, SHA_PTCR, SHA_PTCR_TXTEN); 624 625 return -EINPROGRESS; 626 } 627 628 static void atmel_sha_dma_callback(void *data) 629 { 630 struct atmel_sha_dev *dd = data; 631 632 dd->is_async = true; 633 634 /* dma_lch_in - completed - wait DATRDY */ 635 atmel_sha_write(dd, SHA_IER, SHA_INT_DATARDY); 636 } 637 638 static int atmel_sha_xmit_dma(struct atmel_sha_dev *dd, dma_addr_t dma_addr1, 639 size_t length1, dma_addr_t dma_addr2, size_t length2, int final) 640 { 641 struct atmel_sha_reqctx *ctx = ahash_request_ctx(dd->req); 642 struct dma_async_tx_descriptor *in_desc; 643 struct scatterlist sg[2]; 644 645 dev_dbg(dd->dev, "xmit_dma: digcnt: 0x%llx 0x%llx, length: %zd, final: %d\n", 646 ctx->digcnt[1], ctx->digcnt[0], length1, final); 647 648 dd->dma_lch_in.dma_conf.src_maxburst = 16; 649 dd->dma_lch_in.dma_conf.dst_maxburst = 16; 650 651 dmaengine_slave_config(dd->dma_lch_in.chan, &dd->dma_lch_in.dma_conf); 652 653 if (length2) { 654 sg_init_table(sg, 2); 655 sg_dma_address(&sg[0]) = dma_addr1; 656 sg_dma_len(&sg[0]) = length1; 657 sg_dma_address(&sg[1]) = dma_addr2; 658 sg_dma_len(&sg[1]) = length2; 659 in_desc = dmaengine_prep_slave_sg(dd->dma_lch_in.chan, sg, 2, 660 DMA_MEM_TO_DEV, DMA_PREP_INTERRUPT | DMA_CTRL_ACK); 661 } else { 662 sg_init_table(sg, 1); 663 sg_dma_address(&sg[0]) = dma_addr1; 664 sg_dma_len(&sg[0]) = length1; 665 in_desc = dmaengine_prep_slave_sg(dd->dma_lch_in.chan, sg, 1, 666 DMA_MEM_TO_DEV, DMA_PREP_INTERRUPT | DMA_CTRL_ACK); 667 } 668 if (!in_desc) 669 return atmel_sha_complete(dd, -EINVAL); 670 671 in_desc->callback = atmel_sha_dma_callback; 672 in_desc->callback_param = dd; 673 674 atmel_sha_write_ctrl(dd, 1); 675 676 /* should be non-zero before next lines to disable clocks later */ 677 ctx->digcnt[0] += length1; 678 if (ctx->digcnt[0] < length1) 679 ctx->digcnt[1]++; 680 681 if (final) 682 dd->flags |= SHA_FLAGS_FINAL; /* catch last interrupt */ 683 684 dd->flags |= SHA_FLAGS_DMA_ACTIVE; 685 686 /* Start DMA transfer */ 687 dmaengine_submit(in_desc); 688 dma_async_issue_pending(dd->dma_lch_in.chan); 689 690 return -EINPROGRESS; 691 } 692 693 static int atmel_sha_xmit_start(struct atmel_sha_dev *dd, dma_addr_t dma_addr1, 694 size_t length1, dma_addr_t dma_addr2, size_t length2, int final) 695 { 696 if (dd->caps.has_dma) 697 return atmel_sha_xmit_dma(dd, dma_addr1, length1, 698 dma_addr2, length2, final); 699 else 700 return atmel_sha_xmit_pdc(dd, dma_addr1, length1, 701 dma_addr2, length2, final); 702 } 703 704 static int atmel_sha_update_cpu(struct atmel_sha_dev *dd) 705 { 706 struct atmel_sha_reqctx *ctx = ahash_request_ctx(dd->req); 707 int bufcnt; 708 709 atmel_sha_append_sg(ctx); 710 atmel_sha_fill_padding(ctx, 0); 711 bufcnt = ctx->bufcnt; 712 ctx->bufcnt = 0; 713 714 return atmel_sha_xmit_cpu(dd, ctx->buffer, bufcnt, 1); 715 } 716 717 static int atmel_sha_xmit_dma_map(struct atmel_sha_dev *dd, 718 struct atmel_sha_reqctx *ctx, 719 size_t length, int final) 720 { 721 ctx->dma_addr = dma_map_single(dd->dev, ctx->buffer, 722 ctx->buflen + ctx->block_size, DMA_TO_DEVICE); 723 if (dma_mapping_error(dd->dev, ctx->dma_addr)) { 724 dev_err(dd->dev, "dma %zu bytes error\n", ctx->buflen + 725 ctx->block_size); 726 return atmel_sha_complete(dd, -EINVAL); 727 } 728 729 ctx->flags &= ~SHA_FLAGS_SG; 730 731 /* next call does not fail... so no unmap in the case of error */ 732 return atmel_sha_xmit_start(dd, ctx->dma_addr, length, 0, 0, final); 733 } 734 735 static int atmel_sha_update_dma_slow(struct atmel_sha_dev *dd) 736 { 737 struct atmel_sha_reqctx *ctx = ahash_request_ctx(dd->req); 738 unsigned int final; 739 size_t count; 740 741 atmel_sha_append_sg(ctx); 742 743 final = (ctx->flags & SHA_FLAGS_FINUP) && !ctx->total; 744 745 dev_dbg(dd->dev, "slow: bufcnt: %zu, digcnt: 0x%llx 0x%llx, final: %d\n", 746 ctx->bufcnt, ctx->digcnt[1], ctx->digcnt[0], final); 747 748 if (final) 749 atmel_sha_fill_padding(ctx, 0); 750 751 if (final || (ctx->bufcnt == ctx->buflen)) { 752 count = ctx->bufcnt; 753 ctx->bufcnt = 0; 754 return atmel_sha_xmit_dma_map(dd, ctx, count, final); 755 } 756 757 return 0; 758 } 759 760 static int atmel_sha_update_dma_start(struct atmel_sha_dev *dd) 761 { 762 struct atmel_sha_reqctx *ctx = ahash_request_ctx(dd->req); 763 unsigned int length, final, tail; 764 struct scatterlist *sg; 765 unsigned int count; 766 767 if (!ctx->total) 768 return 0; 769 770 if (ctx->bufcnt || ctx->offset) 771 return atmel_sha_update_dma_slow(dd); 772 773 dev_dbg(dd->dev, "fast: digcnt: 0x%llx 0x%llx, bufcnt: %zd, total: %u\n", 774 ctx->digcnt[1], ctx->digcnt[0], ctx->bufcnt, ctx->total); 775 776 sg = ctx->sg; 777 778 if (!IS_ALIGNED(sg->offset, sizeof(u32))) 779 return atmel_sha_update_dma_slow(dd); 780 781 if (!sg_is_last(sg) && !IS_ALIGNED(sg->length, ctx->block_size)) 782 /* size is not ctx->block_size aligned */ 783 return atmel_sha_update_dma_slow(dd); 784 785 length = min(ctx->total, sg->length); 786 787 if (sg_is_last(sg)) { 788 if (!(ctx->flags & SHA_FLAGS_FINUP)) { 789 /* not last sg must be ctx->block_size aligned */ 790 tail = length & (ctx->block_size - 1); 791 length -= tail; 792 } 793 } 794 795 ctx->total -= length; 796 ctx->offset = length; /* offset where to start slow */ 797 798 final = (ctx->flags & SHA_FLAGS_FINUP) && !ctx->total; 799 800 /* Add padding */ 801 if (final) { 802 tail = length & (ctx->block_size - 1); 803 length -= tail; 804 ctx->total += tail; 805 ctx->offset = length; /* offset where to start slow */ 806 807 sg = ctx->sg; 808 atmel_sha_append_sg(ctx); 809 810 atmel_sha_fill_padding(ctx, length); 811 812 ctx->dma_addr = dma_map_single(dd->dev, ctx->buffer, 813 ctx->buflen + ctx->block_size, DMA_TO_DEVICE); 814 if (dma_mapping_error(dd->dev, ctx->dma_addr)) { 815 dev_err(dd->dev, "dma %zu bytes error\n", 816 ctx->buflen + ctx->block_size); 817 return atmel_sha_complete(dd, -EINVAL); 818 } 819 820 if (length == 0) { 821 ctx->flags &= ~SHA_FLAGS_SG; 822 count = ctx->bufcnt; 823 ctx->bufcnt = 0; 824 return atmel_sha_xmit_start(dd, ctx->dma_addr, count, 0, 825 0, final); 826 } else { 827 ctx->sg = sg; 828 if (!dma_map_sg(dd->dev, ctx->sg, 1, 829 DMA_TO_DEVICE)) { 830 dev_err(dd->dev, "dma_map_sg error\n"); 831 return atmel_sha_complete(dd, -EINVAL); 832 } 833 834 ctx->flags |= SHA_FLAGS_SG; 835 836 count = ctx->bufcnt; 837 ctx->bufcnt = 0; 838 return atmel_sha_xmit_start(dd, sg_dma_address(ctx->sg), 839 length, ctx->dma_addr, count, final); 840 } 841 } 842 843 if (!dma_map_sg(dd->dev, ctx->sg, 1, DMA_TO_DEVICE)) { 844 dev_err(dd->dev, "dma_map_sg error\n"); 845 return atmel_sha_complete(dd, -EINVAL); 846 } 847 848 ctx->flags |= SHA_FLAGS_SG; 849 850 /* next call does not fail... so no unmap in the case of error */ 851 return atmel_sha_xmit_start(dd, sg_dma_address(ctx->sg), length, 0, 852 0, final); 853 } 854 855 static void atmel_sha_update_dma_stop(struct atmel_sha_dev *dd) 856 { 857 struct atmel_sha_reqctx *ctx = ahash_request_ctx(dd->req); 858 859 if (ctx->flags & SHA_FLAGS_SG) { 860 dma_unmap_sg(dd->dev, ctx->sg, 1, DMA_TO_DEVICE); 861 if (ctx->sg->length == ctx->offset) { 862 ctx->sg = sg_next(ctx->sg); 863 if (ctx->sg) 864 ctx->offset = 0; 865 } 866 if (ctx->flags & SHA_FLAGS_PAD) { 867 dma_unmap_single(dd->dev, ctx->dma_addr, 868 ctx->buflen + ctx->block_size, DMA_TO_DEVICE); 869 } 870 } else { 871 dma_unmap_single(dd->dev, ctx->dma_addr, ctx->buflen + 872 ctx->block_size, DMA_TO_DEVICE); 873 } 874 } 875 876 static int atmel_sha_update_req(struct atmel_sha_dev *dd) 877 { 878 struct ahash_request *req = dd->req; 879 struct atmel_sha_reqctx *ctx = ahash_request_ctx(req); 880 int err; 881 882 dev_dbg(dd->dev, "update_req: total: %u, digcnt: 0x%llx 0x%llx\n", 883 ctx->total, ctx->digcnt[1], ctx->digcnt[0]); 884 885 if (ctx->flags & SHA_FLAGS_CPU) 886 err = atmel_sha_update_cpu(dd); 887 else 888 err = atmel_sha_update_dma_start(dd); 889 890 /* wait for dma completion before can take more data */ 891 dev_dbg(dd->dev, "update: err: %d, digcnt: 0x%llx 0%llx\n", 892 err, ctx->digcnt[1], ctx->digcnt[0]); 893 894 return err; 895 } 896 897 static int atmel_sha_final_req(struct atmel_sha_dev *dd) 898 { 899 struct ahash_request *req = dd->req; 900 struct atmel_sha_reqctx *ctx = ahash_request_ctx(req); 901 int err = 0; 902 int count; 903 904 if (ctx->bufcnt >= ATMEL_SHA_DMA_THRESHOLD) { 905 atmel_sha_fill_padding(ctx, 0); 906 count = ctx->bufcnt; 907 ctx->bufcnt = 0; 908 err = atmel_sha_xmit_dma_map(dd, ctx, count, 1); 909 } 910 /* faster to handle last block with cpu */ 911 else { 912 atmel_sha_fill_padding(ctx, 0); 913 count = ctx->bufcnt; 914 ctx->bufcnt = 0; 915 err = atmel_sha_xmit_cpu(dd, ctx->buffer, count, 1); 916 } 917 918 dev_dbg(dd->dev, "final_req: err: %d\n", err); 919 920 return err; 921 } 922 923 static void atmel_sha_copy_hash(struct ahash_request *req) 924 { 925 struct atmel_sha_reqctx *ctx = ahash_request_ctx(req); 926 u32 *hash = (u32 *)ctx->digest; 927 unsigned int i, hashsize; 928 929 switch (ctx->flags & SHA_FLAGS_ALGO_MASK) { 930 case SHA_FLAGS_SHA1: 931 hashsize = SHA1_DIGEST_SIZE; 932 break; 933 934 case SHA_FLAGS_SHA224: 935 case SHA_FLAGS_SHA256: 936 hashsize = SHA256_DIGEST_SIZE; 937 break; 938 939 case SHA_FLAGS_SHA384: 940 case SHA_FLAGS_SHA512: 941 hashsize = SHA512_DIGEST_SIZE; 942 break; 943 944 default: 945 /* Should not happen... */ 946 return; 947 } 948 949 for (i = 0; i < hashsize / sizeof(u32); ++i) 950 hash[i] = atmel_sha_read(ctx->dd, SHA_REG_DIGEST(i)); 951 ctx->flags |= SHA_FLAGS_RESTORE; 952 } 953 954 static void atmel_sha_copy_ready_hash(struct ahash_request *req) 955 { 956 struct atmel_sha_reqctx *ctx = ahash_request_ctx(req); 957 958 if (!req->result) 959 return; 960 961 switch (ctx->flags & SHA_FLAGS_ALGO_MASK) { 962 default: 963 case SHA_FLAGS_SHA1: 964 memcpy(req->result, ctx->digest, SHA1_DIGEST_SIZE); 965 break; 966 967 case SHA_FLAGS_SHA224: 968 memcpy(req->result, ctx->digest, SHA224_DIGEST_SIZE); 969 break; 970 971 case SHA_FLAGS_SHA256: 972 memcpy(req->result, ctx->digest, SHA256_DIGEST_SIZE); 973 break; 974 975 case SHA_FLAGS_SHA384: 976 memcpy(req->result, ctx->digest, SHA384_DIGEST_SIZE); 977 break; 978 979 case SHA_FLAGS_SHA512: 980 memcpy(req->result, ctx->digest, SHA512_DIGEST_SIZE); 981 break; 982 } 983 } 984 985 static int atmel_sha_finish(struct ahash_request *req) 986 { 987 struct atmel_sha_reqctx *ctx = ahash_request_ctx(req); 988 struct atmel_sha_dev *dd = ctx->dd; 989 990 if (ctx->digcnt[0] || ctx->digcnt[1]) 991 atmel_sha_copy_ready_hash(req); 992 993 dev_dbg(dd->dev, "digcnt: 0x%llx 0x%llx, bufcnt: %zd\n", ctx->digcnt[1], 994 ctx->digcnt[0], ctx->bufcnt); 995 996 return 0; 997 } 998 999 static void atmel_sha_finish_req(struct ahash_request *req, int err) 1000 { 1001 struct atmel_sha_reqctx *ctx = ahash_request_ctx(req); 1002 struct atmel_sha_dev *dd = ctx->dd; 1003 1004 if (!err) { 1005 atmel_sha_copy_hash(req); 1006 if (SHA_FLAGS_FINAL & dd->flags) 1007 err = atmel_sha_finish(req); 1008 } else { 1009 ctx->flags |= SHA_FLAGS_ERROR; 1010 } 1011 1012 /* atomic operation is not needed here */ 1013 (void)atmel_sha_complete(dd, err); 1014 } 1015 1016 static int atmel_sha_hw_init(struct atmel_sha_dev *dd) 1017 { 1018 int err; 1019 1020 err = clk_enable(dd->iclk); 1021 if (err) 1022 return err; 1023 1024 if (!(SHA_FLAGS_INIT & dd->flags)) { 1025 atmel_sha_write(dd, SHA_CR, SHA_CR_SWRST); 1026 dd->flags |= SHA_FLAGS_INIT; 1027 } 1028 1029 return 0; 1030 } 1031 1032 static inline unsigned int atmel_sha_get_version(struct atmel_sha_dev *dd) 1033 { 1034 return atmel_sha_read(dd, SHA_HW_VERSION) & 0x00000fff; 1035 } 1036 1037 static int atmel_sha_hw_version_init(struct atmel_sha_dev *dd) 1038 { 1039 int err; 1040 1041 err = atmel_sha_hw_init(dd); 1042 if (err) 1043 return err; 1044 1045 dd->hw_version = atmel_sha_get_version(dd); 1046 1047 dev_info(dd->dev, 1048 "version: 0x%x\n", dd->hw_version); 1049 1050 clk_disable(dd->iclk); 1051 1052 return 0; 1053 } 1054 1055 static int atmel_sha_handle_queue(struct atmel_sha_dev *dd, 1056 struct ahash_request *req) 1057 { 1058 struct crypto_async_request *async_req, *backlog; 1059 struct atmel_sha_ctx *ctx; 1060 unsigned long flags; 1061 bool start_async; 1062 int err = 0, ret = 0; 1063 1064 spin_lock_irqsave(&dd->lock, flags); 1065 if (req) 1066 ret = ahash_enqueue_request(&dd->queue, req); 1067 1068 if (SHA_FLAGS_BUSY & dd->flags) { 1069 spin_unlock_irqrestore(&dd->lock, flags); 1070 return ret; 1071 } 1072 1073 backlog = crypto_get_backlog(&dd->queue); 1074 async_req = crypto_dequeue_request(&dd->queue); 1075 if (async_req) 1076 dd->flags |= SHA_FLAGS_BUSY; 1077 1078 spin_unlock_irqrestore(&dd->lock, flags); 1079 1080 if (!async_req) 1081 return ret; 1082 1083 if (backlog) 1084 backlog->complete(backlog, -EINPROGRESS); 1085 1086 ctx = crypto_tfm_ctx(async_req->tfm); 1087 1088 dd->req = ahash_request_cast(async_req); 1089 start_async = (dd->req != req); 1090 dd->is_async = start_async; 1091 dd->force_complete = false; 1092 1093 /* WARNING: ctx->start() MAY change dd->is_async. */ 1094 err = ctx->start(dd); 1095 return (start_async) ? ret : err; 1096 } 1097 1098 static int atmel_sha_done(struct atmel_sha_dev *dd); 1099 1100 static int atmel_sha_start(struct atmel_sha_dev *dd) 1101 { 1102 struct ahash_request *req = dd->req; 1103 struct atmel_sha_reqctx *ctx = ahash_request_ctx(req); 1104 int err; 1105 1106 dev_dbg(dd->dev, "handling new req, op: %lu, nbytes: %d\n", 1107 ctx->op, req->nbytes); 1108 1109 err = atmel_sha_hw_init(dd); 1110 if (err) 1111 return atmel_sha_complete(dd, err); 1112 1113 /* 1114 * atmel_sha_update_req() and atmel_sha_final_req() can return either: 1115 * -EINPROGRESS: the hardware is busy and the SHA driver will resume 1116 * its job later in the done_task. 1117 * This is the main path. 1118 * 1119 * 0: the SHA driver can continue its job then release the hardware 1120 * later, if needed, with atmel_sha_finish_req(). 1121 * This is the alternate path. 1122 * 1123 * < 0: an error has occurred so atmel_sha_complete(dd, err) has already 1124 * been called, hence the hardware has been released. 1125 * The SHA driver must stop its job without calling 1126 * atmel_sha_finish_req(), otherwise atmel_sha_complete() would be 1127 * called a second time. 1128 * 1129 * Please note that currently, atmel_sha_final_req() never returns 0. 1130 */ 1131 1132 dd->resume = atmel_sha_done; 1133 if (ctx->op == SHA_OP_UPDATE) { 1134 err = atmel_sha_update_req(dd); 1135 if (!err && (ctx->flags & SHA_FLAGS_FINUP)) 1136 /* no final() after finup() */ 1137 err = atmel_sha_final_req(dd); 1138 } else if (ctx->op == SHA_OP_FINAL) { 1139 err = atmel_sha_final_req(dd); 1140 } 1141 1142 if (!err) 1143 /* done_task will not finish it, so do it here */ 1144 atmel_sha_finish_req(req, err); 1145 1146 dev_dbg(dd->dev, "exit, err: %d\n", err); 1147 1148 return err; 1149 } 1150 1151 static int atmel_sha_enqueue(struct ahash_request *req, unsigned int op) 1152 { 1153 struct atmel_sha_reqctx *ctx = ahash_request_ctx(req); 1154 struct atmel_sha_ctx *tctx = crypto_tfm_ctx(req->base.tfm); 1155 struct atmel_sha_dev *dd = tctx->dd; 1156 1157 ctx->op = op; 1158 1159 return atmel_sha_handle_queue(dd, req); 1160 } 1161 1162 static int atmel_sha_update(struct ahash_request *req) 1163 { 1164 struct atmel_sha_reqctx *ctx = ahash_request_ctx(req); 1165 1166 if (!req->nbytes) 1167 return 0; 1168 1169 ctx->total = req->nbytes; 1170 ctx->sg = req->src; 1171 ctx->offset = 0; 1172 1173 if (ctx->flags & SHA_FLAGS_FINUP) { 1174 if (ctx->bufcnt + ctx->total < ATMEL_SHA_DMA_THRESHOLD) 1175 /* faster to use CPU for short transfers */ 1176 ctx->flags |= SHA_FLAGS_CPU; 1177 } else if (ctx->bufcnt + ctx->total < ctx->buflen) { 1178 atmel_sha_append_sg(ctx); 1179 return 0; 1180 } 1181 return atmel_sha_enqueue(req, SHA_OP_UPDATE); 1182 } 1183 1184 static int atmel_sha_final(struct ahash_request *req) 1185 { 1186 struct atmel_sha_reqctx *ctx = ahash_request_ctx(req); 1187 1188 ctx->flags |= SHA_FLAGS_FINUP; 1189 1190 if (ctx->flags & SHA_FLAGS_ERROR) 1191 return 0; /* uncompleted hash is not needed */ 1192 1193 if (ctx->flags & SHA_FLAGS_PAD) 1194 /* copy ready hash (+ finalize hmac) */ 1195 return atmel_sha_finish(req); 1196 1197 return atmel_sha_enqueue(req, SHA_OP_FINAL); 1198 } 1199 1200 static int atmel_sha_finup(struct ahash_request *req) 1201 { 1202 struct atmel_sha_reqctx *ctx = ahash_request_ctx(req); 1203 int err1, err2; 1204 1205 ctx->flags |= SHA_FLAGS_FINUP; 1206 1207 err1 = atmel_sha_update(req); 1208 if (err1 == -EINPROGRESS || 1209 (err1 == -EBUSY && (ahash_request_flags(req) & 1210 CRYPTO_TFM_REQ_MAY_BACKLOG))) 1211 return err1; 1212 1213 /* 1214 * final() has to be always called to cleanup resources 1215 * even if udpate() failed, except EINPROGRESS 1216 */ 1217 err2 = atmel_sha_final(req); 1218 1219 return err1 ?: err2; 1220 } 1221 1222 static int atmel_sha_digest(struct ahash_request *req) 1223 { 1224 return atmel_sha_init(req) ?: atmel_sha_finup(req); 1225 } 1226 1227 1228 static int atmel_sha_export(struct ahash_request *req, void *out) 1229 { 1230 const struct atmel_sha_reqctx *ctx = ahash_request_ctx(req); 1231 1232 memcpy(out, ctx, sizeof(*ctx)); 1233 return 0; 1234 } 1235 1236 static int atmel_sha_import(struct ahash_request *req, const void *in) 1237 { 1238 struct atmel_sha_reqctx *ctx = ahash_request_ctx(req); 1239 1240 memcpy(ctx, in, sizeof(*ctx)); 1241 return 0; 1242 } 1243 1244 static int atmel_sha_cra_init(struct crypto_tfm *tfm) 1245 { 1246 struct atmel_sha_ctx *ctx = crypto_tfm_ctx(tfm); 1247 1248 crypto_ahash_set_reqsize(__crypto_ahash_cast(tfm), 1249 sizeof(struct atmel_sha_reqctx)); 1250 ctx->start = atmel_sha_start; 1251 1252 return 0; 1253 } 1254 1255 static void atmel_sha_alg_init(struct ahash_alg *alg) 1256 { 1257 alg->halg.base.cra_priority = ATMEL_SHA_PRIORITY; 1258 alg->halg.base.cra_flags = CRYPTO_ALG_ASYNC; 1259 alg->halg.base.cra_ctxsize = sizeof(struct atmel_sha_ctx); 1260 alg->halg.base.cra_module = THIS_MODULE; 1261 alg->halg.base.cra_init = atmel_sha_cra_init; 1262 1263 alg->halg.statesize = sizeof(struct atmel_sha_reqctx); 1264 1265 alg->init = atmel_sha_init; 1266 alg->update = atmel_sha_update; 1267 alg->final = atmel_sha_final; 1268 alg->finup = atmel_sha_finup; 1269 alg->digest = atmel_sha_digest; 1270 alg->export = atmel_sha_export; 1271 alg->import = atmel_sha_import; 1272 } 1273 1274 static struct ahash_alg sha_1_256_algs[] = { 1275 { 1276 .halg.base.cra_name = "sha1", 1277 .halg.base.cra_driver_name = "atmel-sha1", 1278 .halg.base.cra_blocksize = SHA1_BLOCK_SIZE, 1279 1280 .halg.digestsize = SHA1_DIGEST_SIZE, 1281 }, 1282 { 1283 .halg.base.cra_name = "sha256", 1284 .halg.base.cra_driver_name = "atmel-sha256", 1285 .halg.base.cra_blocksize = SHA256_BLOCK_SIZE, 1286 1287 .halg.digestsize = SHA256_DIGEST_SIZE, 1288 }, 1289 }; 1290 1291 static struct ahash_alg sha_224_alg = { 1292 .halg.base.cra_name = "sha224", 1293 .halg.base.cra_driver_name = "atmel-sha224", 1294 .halg.base.cra_blocksize = SHA224_BLOCK_SIZE, 1295 1296 .halg.digestsize = SHA224_DIGEST_SIZE, 1297 }; 1298 1299 static struct ahash_alg sha_384_512_algs[] = { 1300 { 1301 .halg.base.cra_name = "sha384", 1302 .halg.base.cra_driver_name = "atmel-sha384", 1303 .halg.base.cra_blocksize = SHA384_BLOCK_SIZE, 1304 .halg.base.cra_alignmask = 0x3, 1305 1306 .halg.digestsize = SHA384_DIGEST_SIZE, 1307 }, 1308 { 1309 .halg.base.cra_name = "sha512", 1310 .halg.base.cra_driver_name = "atmel-sha512", 1311 .halg.base.cra_blocksize = SHA512_BLOCK_SIZE, 1312 .halg.base.cra_alignmask = 0x3, 1313 1314 .halg.digestsize = SHA512_DIGEST_SIZE, 1315 }, 1316 }; 1317 1318 static void atmel_sha_queue_task(unsigned long data) 1319 { 1320 struct atmel_sha_dev *dd = (struct atmel_sha_dev *)data; 1321 1322 atmel_sha_handle_queue(dd, NULL); 1323 } 1324 1325 static int atmel_sha_done(struct atmel_sha_dev *dd) 1326 { 1327 int err = 0; 1328 1329 if (SHA_FLAGS_CPU & dd->flags) { 1330 if (SHA_FLAGS_OUTPUT_READY & dd->flags) { 1331 dd->flags &= ~SHA_FLAGS_OUTPUT_READY; 1332 goto finish; 1333 } 1334 } else if (SHA_FLAGS_DMA_READY & dd->flags) { 1335 if (SHA_FLAGS_DMA_ACTIVE & dd->flags) { 1336 dd->flags &= ~SHA_FLAGS_DMA_ACTIVE; 1337 atmel_sha_update_dma_stop(dd); 1338 } 1339 if (SHA_FLAGS_OUTPUT_READY & dd->flags) { 1340 /* hash or semi-hash ready */ 1341 dd->flags &= ~(SHA_FLAGS_DMA_READY | 1342 SHA_FLAGS_OUTPUT_READY); 1343 err = atmel_sha_update_dma_start(dd); 1344 if (err != -EINPROGRESS) 1345 goto finish; 1346 } 1347 } 1348 return err; 1349 1350 finish: 1351 /* finish curent request */ 1352 atmel_sha_finish_req(dd->req, err); 1353 1354 return err; 1355 } 1356 1357 static void atmel_sha_done_task(unsigned long data) 1358 { 1359 struct atmel_sha_dev *dd = (struct atmel_sha_dev *)data; 1360 1361 dd->is_async = true; 1362 (void)dd->resume(dd); 1363 } 1364 1365 static irqreturn_t atmel_sha_irq(int irq, void *dev_id) 1366 { 1367 struct atmel_sha_dev *sha_dd = dev_id; 1368 u32 reg; 1369 1370 reg = atmel_sha_read(sha_dd, SHA_ISR); 1371 if (reg & atmel_sha_read(sha_dd, SHA_IMR)) { 1372 atmel_sha_write(sha_dd, SHA_IDR, reg); 1373 if (SHA_FLAGS_BUSY & sha_dd->flags) { 1374 sha_dd->flags |= SHA_FLAGS_OUTPUT_READY; 1375 if (!(SHA_FLAGS_CPU & sha_dd->flags)) 1376 sha_dd->flags |= SHA_FLAGS_DMA_READY; 1377 tasklet_schedule(&sha_dd->done_task); 1378 } else { 1379 dev_warn(sha_dd->dev, "SHA interrupt when no active requests.\n"); 1380 } 1381 return IRQ_HANDLED; 1382 } 1383 1384 return IRQ_NONE; 1385 } 1386 1387 1388 /* DMA transfer functions */ 1389 1390 static bool atmel_sha_dma_check_aligned(struct atmel_sha_dev *dd, 1391 struct scatterlist *sg, 1392 size_t len) 1393 { 1394 struct atmel_sha_dma *dma = &dd->dma_lch_in; 1395 struct ahash_request *req = dd->req; 1396 struct atmel_sha_reqctx *ctx = ahash_request_ctx(req); 1397 size_t bs = ctx->block_size; 1398 int nents; 1399 1400 for (nents = 0; sg; sg = sg_next(sg), ++nents) { 1401 if (!IS_ALIGNED(sg->offset, sizeof(u32))) 1402 return false; 1403 1404 /* 1405 * This is the last sg, the only one that is allowed to 1406 * have an unaligned length. 1407 */ 1408 if (len <= sg->length) { 1409 dma->nents = nents + 1; 1410 dma->last_sg_length = sg->length; 1411 sg->length = ALIGN(len, sizeof(u32)); 1412 return true; 1413 } 1414 1415 /* All other sg lengths MUST be aligned to the block size. */ 1416 if (!IS_ALIGNED(sg->length, bs)) 1417 return false; 1418 1419 len -= sg->length; 1420 } 1421 1422 return false; 1423 } 1424 1425 static void atmel_sha_dma_callback2(void *data) 1426 { 1427 struct atmel_sha_dev *dd = data; 1428 struct atmel_sha_dma *dma = &dd->dma_lch_in; 1429 struct scatterlist *sg; 1430 int nents; 1431 1432 dma_unmap_sg(dd->dev, dma->sg, dma->nents, DMA_TO_DEVICE); 1433 1434 sg = dma->sg; 1435 for (nents = 0; nents < dma->nents - 1; ++nents) 1436 sg = sg_next(sg); 1437 sg->length = dma->last_sg_length; 1438 1439 dd->is_async = true; 1440 (void)atmel_sha_wait_for_data_ready(dd, dd->resume); 1441 } 1442 1443 static int atmel_sha_dma_start(struct atmel_sha_dev *dd, 1444 struct scatterlist *src, 1445 size_t len, 1446 atmel_sha_fn_t resume) 1447 { 1448 struct atmel_sha_dma *dma = &dd->dma_lch_in; 1449 struct dma_slave_config *config = &dma->dma_conf; 1450 struct dma_chan *chan = dma->chan; 1451 struct dma_async_tx_descriptor *desc; 1452 dma_cookie_t cookie; 1453 unsigned int sg_len; 1454 int err; 1455 1456 dd->resume = resume; 1457 1458 /* 1459 * dma->nents has already been initialized by 1460 * atmel_sha_dma_check_aligned(). 1461 */ 1462 dma->sg = src; 1463 sg_len = dma_map_sg(dd->dev, dma->sg, dma->nents, DMA_TO_DEVICE); 1464 if (!sg_len) { 1465 err = -ENOMEM; 1466 goto exit; 1467 } 1468 1469 config->src_maxburst = 16; 1470 config->dst_maxburst = 16; 1471 err = dmaengine_slave_config(chan, config); 1472 if (err) 1473 goto unmap_sg; 1474 1475 desc = dmaengine_prep_slave_sg(chan, dma->sg, sg_len, DMA_MEM_TO_DEV, 1476 DMA_PREP_INTERRUPT | DMA_CTRL_ACK); 1477 if (!desc) { 1478 err = -ENOMEM; 1479 goto unmap_sg; 1480 } 1481 1482 desc->callback = atmel_sha_dma_callback2; 1483 desc->callback_param = dd; 1484 cookie = dmaengine_submit(desc); 1485 err = dma_submit_error(cookie); 1486 if (err) 1487 goto unmap_sg; 1488 1489 dma_async_issue_pending(chan); 1490 1491 return -EINPROGRESS; 1492 1493 unmap_sg: 1494 dma_unmap_sg(dd->dev, dma->sg, dma->nents, DMA_TO_DEVICE); 1495 exit: 1496 return atmel_sha_complete(dd, err); 1497 } 1498 1499 1500 /* CPU transfer functions */ 1501 1502 static int atmel_sha_cpu_transfer(struct atmel_sha_dev *dd) 1503 { 1504 struct ahash_request *req = dd->req; 1505 struct atmel_sha_reqctx *ctx = ahash_request_ctx(req); 1506 const u32 *words = (const u32 *)ctx->buffer; 1507 size_t i, num_words; 1508 u32 isr, din, din_inc; 1509 1510 din_inc = (ctx->flags & SHA_FLAGS_IDATAR0) ? 0 : 1; 1511 for (;;) { 1512 /* Write data into the Input Data Registers. */ 1513 num_words = DIV_ROUND_UP(ctx->bufcnt, sizeof(u32)); 1514 for (i = 0, din = 0; i < num_words; ++i, din += din_inc) 1515 atmel_sha_write(dd, SHA_REG_DIN(din), words[i]); 1516 1517 ctx->offset += ctx->bufcnt; 1518 ctx->total -= ctx->bufcnt; 1519 1520 if (!ctx->total) 1521 break; 1522 1523 /* 1524 * Prepare next block: 1525 * Fill ctx->buffer now with the next data to be written into 1526 * IDATARx: it gives time for the SHA hardware to process 1527 * the current data so the SHA_INT_DATARDY flag might be set 1528 * in SHA_ISR when polling this register at the beginning of 1529 * the next loop. 1530 */ 1531 ctx->bufcnt = min_t(size_t, ctx->block_size, ctx->total); 1532 scatterwalk_map_and_copy(ctx->buffer, ctx->sg, 1533 ctx->offset, ctx->bufcnt, 0); 1534 1535 /* Wait for hardware to be ready again. */ 1536 isr = atmel_sha_read(dd, SHA_ISR); 1537 if (!(isr & SHA_INT_DATARDY)) { 1538 /* Not ready yet. */ 1539 dd->resume = atmel_sha_cpu_transfer; 1540 atmel_sha_write(dd, SHA_IER, SHA_INT_DATARDY); 1541 return -EINPROGRESS; 1542 } 1543 } 1544 1545 if (unlikely(!(ctx->flags & SHA_FLAGS_WAIT_DATARDY))) 1546 return dd->cpu_transfer_complete(dd); 1547 1548 return atmel_sha_wait_for_data_ready(dd, dd->cpu_transfer_complete); 1549 } 1550 1551 static int atmel_sha_cpu_start(struct atmel_sha_dev *dd, 1552 struct scatterlist *sg, 1553 unsigned int len, 1554 bool idatar0_only, 1555 bool wait_data_ready, 1556 atmel_sha_fn_t resume) 1557 { 1558 struct ahash_request *req = dd->req; 1559 struct atmel_sha_reqctx *ctx = ahash_request_ctx(req); 1560 1561 if (!len) 1562 return resume(dd); 1563 1564 ctx->flags &= ~(SHA_FLAGS_IDATAR0 | SHA_FLAGS_WAIT_DATARDY); 1565 1566 if (idatar0_only) 1567 ctx->flags |= SHA_FLAGS_IDATAR0; 1568 1569 if (wait_data_ready) 1570 ctx->flags |= SHA_FLAGS_WAIT_DATARDY; 1571 1572 ctx->sg = sg; 1573 ctx->total = len; 1574 ctx->offset = 0; 1575 1576 /* Prepare the first block to be written. */ 1577 ctx->bufcnt = min_t(size_t, ctx->block_size, ctx->total); 1578 scatterwalk_map_and_copy(ctx->buffer, ctx->sg, 1579 ctx->offset, ctx->bufcnt, 0); 1580 1581 dd->cpu_transfer_complete = resume; 1582 return atmel_sha_cpu_transfer(dd); 1583 } 1584 1585 static int atmel_sha_cpu_hash(struct atmel_sha_dev *dd, 1586 const void *data, unsigned int datalen, 1587 bool auto_padding, 1588 atmel_sha_fn_t resume) 1589 { 1590 struct ahash_request *req = dd->req; 1591 struct atmel_sha_reqctx *ctx = ahash_request_ctx(req); 1592 u32 msglen = (auto_padding) ? datalen : 0; 1593 u32 mr = SHA_MR_MODE_AUTO; 1594 1595 if (!(IS_ALIGNED(datalen, ctx->block_size) || auto_padding)) 1596 return atmel_sha_complete(dd, -EINVAL); 1597 1598 mr |= (ctx->flags & SHA_FLAGS_ALGO_MASK); 1599 atmel_sha_write(dd, SHA_MR, mr); 1600 atmel_sha_write(dd, SHA_MSR, msglen); 1601 atmel_sha_write(dd, SHA_BCR, msglen); 1602 atmel_sha_write(dd, SHA_CR, SHA_CR_FIRST); 1603 1604 sg_init_one(&dd->tmp, data, datalen); 1605 return atmel_sha_cpu_start(dd, &dd->tmp, datalen, false, true, resume); 1606 } 1607 1608 1609 /* hmac functions */ 1610 1611 struct atmel_sha_hmac_key { 1612 bool valid; 1613 unsigned int keylen; 1614 u8 buffer[SHA512_BLOCK_SIZE]; 1615 u8 *keydup; 1616 }; 1617 1618 static inline void atmel_sha_hmac_key_init(struct atmel_sha_hmac_key *hkey) 1619 { 1620 memset(hkey, 0, sizeof(*hkey)); 1621 } 1622 1623 static inline void atmel_sha_hmac_key_release(struct atmel_sha_hmac_key *hkey) 1624 { 1625 kfree(hkey->keydup); 1626 memset(hkey, 0, sizeof(*hkey)); 1627 } 1628 1629 static inline int atmel_sha_hmac_key_set(struct atmel_sha_hmac_key *hkey, 1630 const u8 *key, 1631 unsigned int keylen) 1632 { 1633 atmel_sha_hmac_key_release(hkey); 1634 1635 if (keylen > sizeof(hkey->buffer)) { 1636 hkey->keydup = kmemdup(key, keylen, GFP_KERNEL); 1637 if (!hkey->keydup) 1638 return -ENOMEM; 1639 1640 } else { 1641 memcpy(hkey->buffer, key, keylen); 1642 } 1643 1644 hkey->valid = true; 1645 hkey->keylen = keylen; 1646 return 0; 1647 } 1648 1649 static inline bool atmel_sha_hmac_key_get(const struct atmel_sha_hmac_key *hkey, 1650 const u8 **key, 1651 unsigned int *keylen) 1652 { 1653 if (!hkey->valid) 1654 return false; 1655 1656 *keylen = hkey->keylen; 1657 *key = (hkey->keydup) ? hkey->keydup : hkey->buffer; 1658 return true; 1659 } 1660 1661 1662 struct atmel_sha_hmac_ctx { 1663 struct atmel_sha_ctx base; 1664 1665 struct atmel_sha_hmac_key hkey; 1666 u32 ipad[SHA512_BLOCK_SIZE / sizeof(u32)]; 1667 u32 opad[SHA512_BLOCK_SIZE / sizeof(u32)]; 1668 atmel_sha_fn_t resume; 1669 }; 1670 1671 static int atmel_sha_hmac_setup(struct atmel_sha_dev *dd, 1672 atmel_sha_fn_t resume); 1673 static int atmel_sha_hmac_prehash_key(struct atmel_sha_dev *dd, 1674 const u8 *key, unsigned int keylen); 1675 static int atmel_sha_hmac_prehash_key_done(struct atmel_sha_dev *dd); 1676 static int atmel_sha_hmac_compute_ipad_hash(struct atmel_sha_dev *dd); 1677 static int atmel_sha_hmac_compute_opad_hash(struct atmel_sha_dev *dd); 1678 static int atmel_sha_hmac_setup_done(struct atmel_sha_dev *dd); 1679 1680 static int atmel_sha_hmac_init_done(struct atmel_sha_dev *dd); 1681 static int atmel_sha_hmac_final(struct atmel_sha_dev *dd); 1682 static int atmel_sha_hmac_final_done(struct atmel_sha_dev *dd); 1683 static int atmel_sha_hmac_digest2(struct atmel_sha_dev *dd); 1684 1685 static int atmel_sha_hmac_setup(struct atmel_sha_dev *dd, 1686 atmel_sha_fn_t resume) 1687 { 1688 struct ahash_request *req = dd->req; 1689 struct atmel_sha_reqctx *ctx = ahash_request_ctx(req); 1690 struct crypto_ahash *tfm = crypto_ahash_reqtfm(req); 1691 struct atmel_sha_hmac_ctx *hmac = crypto_ahash_ctx(tfm); 1692 unsigned int keylen; 1693 const u8 *key; 1694 size_t bs; 1695 1696 hmac->resume = resume; 1697 switch (ctx->flags & SHA_FLAGS_ALGO_MASK) { 1698 case SHA_FLAGS_SHA1: 1699 ctx->block_size = SHA1_BLOCK_SIZE; 1700 ctx->hash_size = SHA1_DIGEST_SIZE; 1701 break; 1702 1703 case SHA_FLAGS_SHA224: 1704 ctx->block_size = SHA224_BLOCK_SIZE; 1705 ctx->hash_size = SHA256_DIGEST_SIZE; 1706 break; 1707 1708 case SHA_FLAGS_SHA256: 1709 ctx->block_size = SHA256_BLOCK_SIZE; 1710 ctx->hash_size = SHA256_DIGEST_SIZE; 1711 break; 1712 1713 case SHA_FLAGS_SHA384: 1714 ctx->block_size = SHA384_BLOCK_SIZE; 1715 ctx->hash_size = SHA512_DIGEST_SIZE; 1716 break; 1717 1718 case SHA_FLAGS_SHA512: 1719 ctx->block_size = SHA512_BLOCK_SIZE; 1720 ctx->hash_size = SHA512_DIGEST_SIZE; 1721 break; 1722 1723 default: 1724 return atmel_sha_complete(dd, -EINVAL); 1725 } 1726 bs = ctx->block_size; 1727 1728 if (likely(!atmel_sha_hmac_key_get(&hmac->hkey, &key, &keylen))) 1729 return resume(dd); 1730 1731 /* Compute K' from K. */ 1732 if (unlikely(keylen > bs)) 1733 return atmel_sha_hmac_prehash_key(dd, key, keylen); 1734 1735 /* Prepare ipad. */ 1736 memcpy((u8 *)hmac->ipad, key, keylen); 1737 memset((u8 *)hmac->ipad + keylen, 0, bs - keylen); 1738 return atmel_sha_hmac_compute_ipad_hash(dd); 1739 } 1740 1741 static int atmel_sha_hmac_prehash_key(struct atmel_sha_dev *dd, 1742 const u8 *key, unsigned int keylen) 1743 { 1744 return atmel_sha_cpu_hash(dd, key, keylen, true, 1745 atmel_sha_hmac_prehash_key_done); 1746 } 1747 1748 static int atmel_sha_hmac_prehash_key_done(struct atmel_sha_dev *dd) 1749 { 1750 struct ahash_request *req = dd->req; 1751 struct crypto_ahash *tfm = crypto_ahash_reqtfm(req); 1752 struct atmel_sha_hmac_ctx *hmac = crypto_ahash_ctx(tfm); 1753 struct atmel_sha_reqctx *ctx = ahash_request_ctx(req); 1754 size_t ds = crypto_ahash_digestsize(tfm); 1755 size_t bs = ctx->block_size; 1756 size_t i, num_words = ds / sizeof(u32); 1757 1758 /* Prepare ipad. */ 1759 for (i = 0; i < num_words; ++i) 1760 hmac->ipad[i] = atmel_sha_read(dd, SHA_REG_DIGEST(i)); 1761 memset((u8 *)hmac->ipad + ds, 0, bs - ds); 1762 return atmel_sha_hmac_compute_ipad_hash(dd); 1763 } 1764 1765 static int atmel_sha_hmac_compute_ipad_hash(struct atmel_sha_dev *dd) 1766 { 1767 struct ahash_request *req = dd->req; 1768 struct crypto_ahash *tfm = crypto_ahash_reqtfm(req); 1769 struct atmel_sha_hmac_ctx *hmac = crypto_ahash_ctx(tfm); 1770 struct atmel_sha_reqctx *ctx = ahash_request_ctx(req); 1771 size_t bs = ctx->block_size; 1772 size_t i, num_words = bs / sizeof(u32); 1773 1774 memcpy(hmac->opad, hmac->ipad, bs); 1775 for (i = 0; i < num_words; ++i) { 1776 hmac->ipad[i] ^= 0x36363636; 1777 hmac->opad[i] ^= 0x5c5c5c5c; 1778 } 1779 1780 return atmel_sha_cpu_hash(dd, hmac->ipad, bs, false, 1781 atmel_sha_hmac_compute_opad_hash); 1782 } 1783 1784 static int atmel_sha_hmac_compute_opad_hash(struct atmel_sha_dev *dd) 1785 { 1786 struct ahash_request *req = dd->req; 1787 struct crypto_ahash *tfm = crypto_ahash_reqtfm(req); 1788 struct atmel_sha_hmac_ctx *hmac = crypto_ahash_ctx(tfm); 1789 struct atmel_sha_reqctx *ctx = ahash_request_ctx(req); 1790 size_t bs = ctx->block_size; 1791 size_t hs = ctx->hash_size; 1792 size_t i, num_words = hs / sizeof(u32); 1793 1794 for (i = 0; i < num_words; ++i) 1795 hmac->ipad[i] = atmel_sha_read(dd, SHA_REG_DIGEST(i)); 1796 return atmel_sha_cpu_hash(dd, hmac->opad, bs, false, 1797 atmel_sha_hmac_setup_done); 1798 } 1799 1800 static int atmel_sha_hmac_setup_done(struct atmel_sha_dev *dd) 1801 { 1802 struct ahash_request *req = dd->req; 1803 struct crypto_ahash *tfm = crypto_ahash_reqtfm(req); 1804 struct atmel_sha_hmac_ctx *hmac = crypto_ahash_ctx(tfm); 1805 struct atmel_sha_reqctx *ctx = ahash_request_ctx(req); 1806 size_t hs = ctx->hash_size; 1807 size_t i, num_words = hs / sizeof(u32); 1808 1809 for (i = 0; i < num_words; ++i) 1810 hmac->opad[i] = atmel_sha_read(dd, SHA_REG_DIGEST(i)); 1811 atmel_sha_hmac_key_release(&hmac->hkey); 1812 return hmac->resume(dd); 1813 } 1814 1815 static int atmel_sha_hmac_start(struct atmel_sha_dev *dd) 1816 { 1817 struct ahash_request *req = dd->req; 1818 struct atmel_sha_reqctx *ctx = ahash_request_ctx(req); 1819 int err; 1820 1821 err = atmel_sha_hw_init(dd); 1822 if (err) 1823 return atmel_sha_complete(dd, err); 1824 1825 switch (ctx->op) { 1826 case SHA_OP_INIT: 1827 err = atmel_sha_hmac_setup(dd, atmel_sha_hmac_init_done); 1828 break; 1829 1830 case SHA_OP_UPDATE: 1831 dd->resume = atmel_sha_done; 1832 err = atmel_sha_update_req(dd); 1833 break; 1834 1835 case SHA_OP_FINAL: 1836 dd->resume = atmel_sha_hmac_final; 1837 err = atmel_sha_final_req(dd); 1838 break; 1839 1840 case SHA_OP_DIGEST: 1841 err = atmel_sha_hmac_setup(dd, atmel_sha_hmac_digest2); 1842 break; 1843 1844 default: 1845 return atmel_sha_complete(dd, -EINVAL); 1846 } 1847 1848 return err; 1849 } 1850 1851 static int atmel_sha_hmac_setkey(struct crypto_ahash *tfm, const u8 *key, 1852 unsigned int keylen) 1853 { 1854 struct atmel_sha_hmac_ctx *hmac = crypto_ahash_ctx(tfm); 1855 1856 return atmel_sha_hmac_key_set(&hmac->hkey, key, keylen); 1857 } 1858 1859 static int atmel_sha_hmac_init(struct ahash_request *req) 1860 { 1861 int err; 1862 1863 err = atmel_sha_init(req); 1864 if (err) 1865 return err; 1866 1867 return atmel_sha_enqueue(req, SHA_OP_INIT); 1868 } 1869 1870 static int atmel_sha_hmac_init_done(struct atmel_sha_dev *dd) 1871 { 1872 struct ahash_request *req = dd->req; 1873 struct atmel_sha_reqctx *ctx = ahash_request_ctx(req); 1874 struct crypto_ahash *tfm = crypto_ahash_reqtfm(req); 1875 struct atmel_sha_hmac_ctx *hmac = crypto_ahash_ctx(tfm); 1876 size_t bs = ctx->block_size; 1877 size_t hs = ctx->hash_size; 1878 1879 ctx->bufcnt = 0; 1880 ctx->digcnt[0] = bs; 1881 ctx->digcnt[1] = 0; 1882 ctx->flags |= SHA_FLAGS_RESTORE; 1883 memcpy(ctx->digest, hmac->ipad, hs); 1884 return atmel_sha_complete(dd, 0); 1885 } 1886 1887 static int atmel_sha_hmac_final(struct atmel_sha_dev *dd) 1888 { 1889 struct ahash_request *req = dd->req; 1890 struct atmel_sha_reqctx *ctx = ahash_request_ctx(req); 1891 struct crypto_ahash *tfm = crypto_ahash_reqtfm(req); 1892 struct atmel_sha_hmac_ctx *hmac = crypto_ahash_ctx(tfm); 1893 u32 *digest = (u32 *)ctx->digest; 1894 size_t ds = crypto_ahash_digestsize(tfm); 1895 size_t bs = ctx->block_size; 1896 size_t hs = ctx->hash_size; 1897 size_t i, num_words; 1898 u32 mr; 1899 1900 /* Save d = SHA((K' + ipad) | msg). */ 1901 num_words = ds / sizeof(u32); 1902 for (i = 0; i < num_words; ++i) 1903 digest[i] = atmel_sha_read(dd, SHA_REG_DIGEST(i)); 1904 1905 /* Restore context to finish computing SHA((K' + opad) | d). */ 1906 atmel_sha_write(dd, SHA_CR, SHA_CR_WUIHV); 1907 num_words = hs / sizeof(u32); 1908 for (i = 0; i < num_words; ++i) 1909 atmel_sha_write(dd, SHA_REG_DIN(i), hmac->opad[i]); 1910 1911 mr = SHA_MR_MODE_AUTO | SHA_MR_UIHV; 1912 mr |= (ctx->flags & SHA_FLAGS_ALGO_MASK); 1913 atmel_sha_write(dd, SHA_MR, mr); 1914 atmel_sha_write(dd, SHA_MSR, bs + ds); 1915 atmel_sha_write(dd, SHA_BCR, ds); 1916 atmel_sha_write(dd, SHA_CR, SHA_CR_FIRST); 1917 1918 sg_init_one(&dd->tmp, digest, ds); 1919 return atmel_sha_cpu_start(dd, &dd->tmp, ds, false, true, 1920 atmel_sha_hmac_final_done); 1921 } 1922 1923 static int atmel_sha_hmac_final_done(struct atmel_sha_dev *dd) 1924 { 1925 /* 1926 * req->result might not be sizeof(u32) aligned, so copy the 1927 * digest into ctx->digest[] before memcpy() the data into 1928 * req->result. 1929 */ 1930 atmel_sha_copy_hash(dd->req); 1931 atmel_sha_copy_ready_hash(dd->req); 1932 return atmel_sha_complete(dd, 0); 1933 } 1934 1935 static int atmel_sha_hmac_digest(struct ahash_request *req) 1936 { 1937 int err; 1938 1939 err = atmel_sha_init(req); 1940 if (err) 1941 return err; 1942 1943 return atmel_sha_enqueue(req, SHA_OP_DIGEST); 1944 } 1945 1946 static int atmel_sha_hmac_digest2(struct atmel_sha_dev *dd) 1947 { 1948 struct ahash_request *req = dd->req; 1949 struct atmel_sha_reqctx *ctx = ahash_request_ctx(req); 1950 struct crypto_ahash *tfm = crypto_ahash_reqtfm(req); 1951 struct atmel_sha_hmac_ctx *hmac = crypto_ahash_ctx(tfm); 1952 size_t hs = ctx->hash_size; 1953 size_t i, num_words = hs / sizeof(u32); 1954 bool use_dma = false; 1955 u32 mr; 1956 1957 /* Special case for empty message. */ 1958 if (!req->nbytes) 1959 return atmel_sha_complete(dd, -EINVAL); // TODO: 1960 1961 /* Check DMA threshold and alignment. */ 1962 if (req->nbytes > ATMEL_SHA_DMA_THRESHOLD && 1963 atmel_sha_dma_check_aligned(dd, req->src, req->nbytes)) 1964 use_dma = true; 1965 1966 /* Write both initial hash values to compute a HMAC. */ 1967 atmel_sha_write(dd, SHA_CR, SHA_CR_WUIHV); 1968 for (i = 0; i < num_words; ++i) 1969 atmel_sha_write(dd, SHA_REG_DIN(i), hmac->ipad[i]); 1970 1971 atmel_sha_write(dd, SHA_CR, SHA_CR_WUIEHV); 1972 for (i = 0; i < num_words; ++i) 1973 atmel_sha_write(dd, SHA_REG_DIN(i), hmac->opad[i]); 1974 1975 /* Write the Mode, Message Size, Bytes Count then Control Registers. */ 1976 mr = (SHA_MR_HMAC | SHA_MR_DUALBUFF); 1977 mr |= ctx->flags & SHA_FLAGS_ALGO_MASK; 1978 if (use_dma) 1979 mr |= SHA_MR_MODE_IDATAR0; 1980 else 1981 mr |= SHA_MR_MODE_AUTO; 1982 atmel_sha_write(dd, SHA_MR, mr); 1983 1984 atmel_sha_write(dd, SHA_MSR, req->nbytes); 1985 atmel_sha_write(dd, SHA_BCR, req->nbytes); 1986 1987 atmel_sha_write(dd, SHA_CR, SHA_CR_FIRST); 1988 1989 /* Process data. */ 1990 if (use_dma) 1991 return atmel_sha_dma_start(dd, req->src, req->nbytes, 1992 atmel_sha_hmac_final_done); 1993 1994 return atmel_sha_cpu_start(dd, req->src, req->nbytes, false, true, 1995 atmel_sha_hmac_final_done); 1996 } 1997 1998 static int atmel_sha_hmac_cra_init(struct crypto_tfm *tfm) 1999 { 2000 struct atmel_sha_hmac_ctx *hmac = crypto_tfm_ctx(tfm); 2001 2002 crypto_ahash_set_reqsize(__crypto_ahash_cast(tfm), 2003 sizeof(struct atmel_sha_reqctx)); 2004 hmac->base.start = atmel_sha_hmac_start; 2005 atmel_sha_hmac_key_init(&hmac->hkey); 2006 2007 return 0; 2008 } 2009 2010 static void atmel_sha_hmac_cra_exit(struct crypto_tfm *tfm) 2011 { 2012 struct atmel_sha_hmac_ctx *hmac = crypto_tfm_ctx(tfm); 2013 2014 atmel_sha_hmac_key_release(&hmac->hkey); 2015 } 2016 2017 static void atmel_sha_hmac_alg_init(struct ahash_alg *alg) 2018 { 2019 alg->halg.base.cra_priority = ATMEL_SHA_PRIORITY; 2020 alg->halg.base.cra_flags = CRYPTO_ALG_ASYNC; 2021 alg->halg.base.cra_ctxsize = sizeof(struct atmel_sha_hmac_ctx); 2022 alg->halg.base.cra_module = THIS_MODULE; 2023 alg->halg.base.cra_init = atmel_sha_hmac_cra_init; 2024 alg->halg.base.cra_exit = atmel_sha_hmac_cra_exit; 2025 2026 alg->halg.statesize = sizeof(struct atmel_sha_reqctx); 2027 2028 alg->init = atmel_sha_hmac_init; 2029 alg->update = atmel_sha_update; 2030 alg->final = atmel_sha_final; 2031 alg->digest = atmel_sha_hmac_digest; 2032 alg->setkey = atmel_sha_hmac_setkey; 2033 alg->export = atmel_sha_export; 2034 alg->import = atmel_sha_import; 2035 } 2036 2037 static struct ahash_alg sha_hmac_algs[] = { 2038 { 2039 .halg.base.cra_name = "hmac(sha1)", 2040 .halg.base.cra_driver_name = "atmel-hmac-sha1", 2041 .halg.base.cra_blocksize = SHA1_BLOCK_SIZE, 2042 2043 .halg.digestsize = SHA1_DIGEST_SIZE, 2044 }, 2045 { 2046 .halg.base.cra_name = "hmac(sha224)", 2047 .halg.base.cra_driver_name = "atmel-hmac-sha224", 2048 .halg.base.cra_blocksize = SHA224_BLOCK_SIZE, 2049 2050 .halg.digestsize = SHA224_DIGEST_SIZE, 2051 }, 2052 { 2053 .halg.base.cra_name = "hmac(sha256)", 2054 .halg.base.cra_driver_name = "atmel-hmac-sha256", 2055 .halg.base.cra_blocksize = SHA256_BLOCK_SIZE, 2056 2057 .halg.digestsize = SHA256_DIGEST_SIZE, 2058 }, 2059 { 2060 .halg.base.cra_name = "hmac(sha384)", 2061 .halg.base.cra_driver_name = "atmel-hmac-sha384", 2062 .halg.base.cra_blocksize = SHA384_BLOCK_SIZE, 2063 2064 .halg.digestsize = SHA384_DIGEST_SIZE, 2065 }, 2066 { 2067 .halg.base.cra_name = "hmac(sha512)", 2068 .halg.base.cra_driver_name = "atmel-hmac-sha512", 2069 .halg.base.cra_blocksize = SHA512_BLOCK_SIZE, 2070 2071 .halg.digestsize = SHA512_DIGEST_SIZE, 2072 }, 2073 }; 2074 2075 #if IS_ENABLED(CONFIG_CRYPTO_DEV_ATMEL_AUTHENC) 2076 /* authenc functions */ 2077 2078 static int atmel_sha_authenc_init2(struct atmel_sha_dev *dd); 2079 static int atmel_sha_authenc_init_done(struct atmel_sha_dev *dd); 2080 static int atmel_sha_authenc_final_done(struct atmel_sha_dev *dd); 2081 2082 2083 struct atmel_sha_authenc_ctx { 2084 struct crypto_ahash *tfm; 2085 }; 2086 2087 struct atmel_sha_authenc_reqctx { 2088 struct atmel_sha_reqctx base; 2089 2090 atmel_aes_authenc_fn_t cb; 2091 struct atmel_aes_dev *aes_dev; 2092 2093 /* _init() parameters. */ 2094 struct scatterlist *assoc; 2095 u32 assoclen; 2096 u32 textlen; 2097 2098 /* _final() parameters. */ 2099 u32 *digest; 2100 unsigned int digestlen; 2101 }; 2102 2103 static void atmel_sha_authenc_complete(struct crypto_async_request *areq, 2104 int err) 2105 { 2106 struct ahash_request *req = areq->data; 2107 struct atmel_sha_authenc_reqctx *authctx = ahash_request_ctx(req); 2108 2109 authctx->cb(authctx->aes_dev, err, authctx->base.dd->is_async); 2110 } 2111 2112 static int atmel_sha_authenc_start(struct atmel_sha_dev *dd) 2113 { 2114 struct ahash_request *req = dd->req; 2115 struct atmel_sha_authenc_reqctx *authctx = ahash_request_ctx(req); 2116 int err; 2117 2118 /* 2119 * Force atmel_sha_complete() to call req->base.complete(), ie 2120 * atmel_sha_authenc_complete(), which in turn calls authctx->cb(). 2121 */ 2122 dd->force_complete = true; 2123 2124 err = atmel_sha_hw_init(dd); 2125 return authctx->cb(authctx->aes_dev, err, dd->is_async); 2126 } 2127 2128 bool atmel_sha_authenc_is_ready(void) 2129 { 2130 struct atmel_sha_ctx dummy; 2131 2132 dummy.dd = NULL; 2133 return (atmel_sha_find_dev(&dummy) != NULL); 2134 } 2135 EXPORT_SYMBOL_GPL(atmel_sha_authenc_is_ready); 2136 2137 unsigned int atmel_sha_authenc_get_reqsize(void) 2138 { 2139 return sizeof(struct atmel_sha_authenc_reqctx); 2140 } 2141 EXPORT_SYMBOL_GPL(atmel_sha_authenc_get_reqsize); 2142 2143 struct atmel_sha_authenc_ctx *atmel_sha_authenc_spawn(unsigned long mode) 2144 { 2145 struct atmel_sha_authenc_ctx *auth; 2146 struct crypto_ahash *tfm; 2147 struct atmel_sha_ctx *tctx; 2148 const char *name; 2149 int err = -EINVAL; 2150 2151 switch (mode & SHA_FLAGS_MODE_MASK) { 2152 case SHA_FLAGS_HMAC_SHA1: 2153 name = "atmel-hmac-sha1"; 2154 break; 2155 2156 case SHA_FLAGS_HMAC_SHA224: 2157 name = "atmel-hmac-sha224"; 2158 break; 2159 2160 case SHA_FLAGS_HMAC_SHA256: 2161 name = "atmel-hmac-sha256"; 2162 break; 2163 2164 case SHA_FLAGS_HMAC_SHA384: 2165 name = "atmel-hmac-sha384"; 2166 break; 2167 2168 case SHA_FLAGS_HMAC_SHA512: 2169 name = "atmel-hmac-sha512"; 2170 break; 2171 2172 default: 2173 goto error; 2174 } 2175 2176 tfm = crypto_alloc_ahash(name, 0, 0); 2177 if (IS_ERR(tfm)) { 2178 err = PTR_ERR(tfm); 2179 goto error; 2180 } 2181 tctx = crypto_ahash_ctx(tfm); 2182 tctx->start = atmel_sha_authenc_start; 2183 tctx->flags = mode; 2184 2185 auth = kzalloc(sizeof(*auth), GFP_KERNEL); 2186 if (!auth) { 2187 err = -ENOMEM; 2188 goto err_free_ahash; 2189 } 2190 auth->tfm = tfm; 2191 2192 return auth; 2193 2194 err_free_ahash: 2195 crypto_free_ahash(tfm); 2196 error: 2197 return ERR_PTR(err); 2198 } 2199 EXPORT_SYMBOL_GPL(atmel_sha_authenc_spawn); 2200 2201 void atmel_sha_authenc_free(struct atmel_sha_authenc_ctx *auth) 2202 { 2203 if (auth) 2204 crypto_free_ahash(auth->tfm); 2205 kfree(auth); 2206 } 2207 EXPORT_SYMBOL_GPL(atmel_sha_authenc_free); 2208 2209 int atmel_sha_authenc_setkey(struct atmel_sha_authenc_ctx *auth, 2210 const u8 *key, unsigned int keylen, u32 flags) 2211 { 2212 struct crypto_ahash *tfm = auth->tfm; 2213 2214 crypto_ahash_clear_flags(tfm, CRYPTO_TFM_REQ_MASK); 2215 crypto_ahash_set_flags(tfm, flags & CRYPTO_TFM_REQ_MASK); 2216 return crypto_ahash_setkey(tfm, key, keylen); 2217 } 2218 EXPORT_SYMBOL_GPL(atmel_sha_authenc_setkey); 2219 2220 int atmel_sha_authenc_schedule(struct ahash_request *req, 2221 struct atmel_sha_authenc_ctx *auth, 2222 atmel_aes_authenc_fn_t cb, 2223 struct atmel_aes_dev *aes_dev) 2224 { 2225 struct atmel_sha_authenc_reqctx *authctx = ahash_request_ctx(req); 2226 struct atmel_sha_reqctx *ctx = &authctx->base; 2227 struct crypto_ahash *tfm = auth->tfm; 2228 struct atmel_sha_ctx *tctx = crypto_ahash_ctx(tfm); 2229 struct atmel_sha_dev *dd; 2230 2231 /* Reset request context (MUST be done first). */ 2232 memset(authctx, 0, sizeof(*authctx)); 2233 2234 /* Get SHA device. */ 2235 dd = atmel_sha_find_dev(tctx); 2236 if (!dd) 2237 return cb(aes_dev, -ENODEV, false); 2238 2239 /* Init request context. */ 2240 ctx->dd = dd; 2241 ctx->buflen = SHA_BUFFER_LEN; 2242 authctx->cb = cb; 2243 authctx->aes_dev = aes_dev; 2244 ahash_request_set_tfm(req, tfm); 2245 ahash_request_set_callback(req, 0, atmel_sha_authenc_complete, req); 2246 2247 return atmel_sha_handle_queue(dd, req); 2248 } 2249 EXPORT_SYMBOL_GPL(atmel_sha_authenc_schedule); 2250 2251 int atmel_sha_authenc_init(struct ahash_request *req, 2252 struct scatterlist *assoc, unsigned int assoclen, 2253 unsigned int textlen, 2254 atmel_aes_authenc_fn_t cb, 2255 struct atmel_aes_dev *aes_dev) 2256 { 2257 struct atmel_sha_authenc_reqctx *authctx = ahash_request_ctx(req); 2258 struct atmel_sha_reqctx *ctx = &authctx->base; 2259 struct crypto_ahash *tfm = crypto_ahash_reqtfm(req); 2260 struct atmel_sha_hmac_ctx *hmac = crypto_ahash_ctx(tfm); 2261 struct atmel_sha_dev *dd = ctx->dd; 2262 2263 if (unlikely(!IS_ALIGNED(assoclen, sizeof(u32)))) 2264 return atmel_sha_complete(dd, -EINVAL); 2265 2266 authctx->cb = cb; 2267 authctx->aes_dev = aes_dev; 2268 authctx->assoc = assoc; 2269 authctx->assoclen = assoclen; 2270 authctx->textlen = textlen; 2271 2272 ctx->flags = hmac->base.flags; 2273 return atmel_sha_hmac_setup(dd, atmel_sha_authenc_init2); 2274 } 2275 EXPORT_SYMBOL_GPL(atmel_sha_authenc_init); 2276 2277 static int atmel_sha_authenc_init2(struct atmel_sha_dev *dd) 2278 { 2279 struct ahash_request *req = dd->req; 2280 struct atmel_sha_authenc_reqctx *authctx = ahash_request_ctx(req); 2281 struct atmel_sha_reqctx *ctx = &authctx->base; 2282 struct crypto_ahash *tfm = crypto_ahash_reqtfm(req); 2283 struct atmel_sha_hmac_ctx *hmac = crypto_ahash_ctx(tfm); 2284 size_t hs = ctx->hash_size; 2285 size_t i, num_words = hs / sizeof(u32); 2286 u32 mr, msg_size; 2287 2288 atmel_sha_write(dd, SHA_CR, SHA_CR_WUIHV); 2289 for (i = 0; i < num_words; ++i) 2290 atmel_sha_write(dd, SHA_REG_DIN(i), hmac->ipad[i]); 2291 2292 atmel_sha_write(dd, SHA_CR, SHA_CR_WUIEHV); 2293 for (i = 0; i < num_words; ++i) 2294 atmel_sha_write(dd, SHA_REG_DIN(i), hmac->opad[i]); 2295 2296 mr = (SHA_MR_MODE_IDATAR0 | 2297 SHA_MR_HMAC | 2298 SHA_MR_DUALBUFF); 2299 mr |= ctx->flags & SHA_FLAGS_ALGO_MASK; 2300 atmel_sha_write(dd, SHA_MR, mr); 2301 2302 msg_size = authctx->assoclen + authctx->textlen; 2303 atmel_sha_write(dd, SHA_MSR, msg_size); 2304 atmel_sha_write(dd, SHA_BCR, msg_size); 2305 2306 atmel_sha_write(dd, SHA_CR, SHA_CR_FIRST); 2307 2308 /* Process assoc data. */ 2309 return atmel_sha_cpu_start(dd, authctx->assoc, authctx->assoclen, 2310 true, false, 2311 atmel_sha_authenc_init_done); 2312 } 2313 2314 static int atmel_sha_authenc_init_done(struct atmel_sha_dev *dd) 2315 { 2316 struct ahash_request *req = dd->req; 2317 struct atmel_sha_authenc_reqctx *authctx = ahash_request_ctx(req); 2318 2319 return authctx->cb(authctx->aes_dev, 0, dd->is_async); 2320 } 2321 2322 int atmel_sha_authenc_final(struct ahash_request *req, 2323 u32 *digest, unsigned int digestlen, 2324 atmel_aes_authenc_fn_t cb, 2325 struct atmel_aes_dev *aes_dev) 2326 { 2327 struct atmel_sha_authenc_reqctx *authctx = ahash_request_ctx(req); 2328 struct atmel_sha_reqctx *ctx = &authctx->base; 2329 struct atmel_sha_dev *dd = ctx->dd; 2330 2331 switch (ctx->flags & SHA_FLAGS_ALGO_MASK) { 2332 case SHA_FLAGS_SHA1: 2333 authctx->digestlen = SHA1_DIGEST_SIZE; 2334 break; 2335 2336 case SHA_FLAGS_SHA224: 2337 authctx->digestlen = SHA224_DIGEST_SIZE; 2338 break; 2339 2340 case SHA_FLAGS_SHA256: 2341 authctx->digestlen = SHA256_DIGEST_SIZE; 2342 break; 2343 2344 case SHA_FLAGS_SHA384: 2345 authctx->digestlen = SHA384_DIGEST_SIZE; 2346 break; 2347 2348 case SHA_FLAGS_SHA512: 2349 authctx->digestlen = SHA512_DIGEST_SIZE; 2350 break; 2351 2352 default: 2353 return atmel_sha_complete(dd, -EINVAL); 2354 } 2355 if (authctx->digestlen > digestlen) 2356 authctx->digestlen = digestlen; 2357 2358 authctx->cb = cb; 2359 authctx->aes_dev = aes_dev; 2360 authctx->digest = digest; 2361 return atmel_sha_wait_for_data_ready(dd, 2362 atmel_sha_authenc_final_done); 2363 } 2364 EXPORT_SYMBOL_GPL(atmel_sha_authenc_final); 2365 2366 static int atmel_sha_authenc_final_done(struct atmel_sha_dev *dd) 2367 { 2368 struct ahash_request *req = dd->req; 2369 struct atmel_sha_authenc_reqctx *authctx = ahash_request_ctx(req); 2370 size_t i, num_words = authctx->digestlen / sizeof(u32); 2371 2372 for (i = 0; i < num_words; ++i) 2373 authctx->digest[i] = atmel_sha_read(dd, SHA_REG_DIGEST(i)); 2374 2375 return atmel_sha_complete(dd, 0); 2376 } 2377 2378 void atmel_sha_authenc_abort(struct ahash_request *req) 2379 { 2380 struct atmel_sha_authenc_reqctx *authctx = ahash_request_ctx(req); 2381 struct atmel_sha_reqctx *ctx = &authctx->base; 2382 struct atmel_sha_dev *dd = ctx->dd; 2383 2384 /* Prevent atmel_sha_complete() from calling req->base.complete(). */ 2385 dd->is_async = false; 2386 dd->force_complete = false; 2387 (void)atmel_sha_complete(dd, 0); 2388 } 2389 EXPORT_SYMBOL_GPL(atmel_sha_authenc_abort); 2390 2391 #endif /* CONFIG_CRYPTO_DEV_ATMEL_AUTHENC */ 2392 2393 2394 static void atmel_sha_unregister_algs(struct atmel_sha_dev *dd) 2395 { 2396 int i; 2397 2398 if (dd->caps.has_hmac) 2399 for (i = 0; i < ARRAY_SIZE(sha_hmac_algs); i++) 2400 crypto_unregister_ahash(&sha_hmac_algs[i]); 2401 2402 for (i = 0; i < ARRAY_SIZE(sha_1_256_algs); i++) 2403 crypto_unregister_ahash(&sha_1_256_algs[i]); 2404 2405 if (dd->caps.has_sha224) 2406 crypto_unregister_ahash(&sha_224_alg); 2407 2408 if (dd->caps.has_sha_384_512) { 2409 for (i = 0; i < ARRAY_SIZE(sha_384_512_algs); i++) 2410 crypto_unregister_ahash(&sha_384_512_algs[i]); 2411 } 2412 } 2413 2414 static int atmel_sha_register_algs(struct atmel_sha_dev *dd) 2415 { 2416 int err, i, j; 2417 2418 for (i = 0; i < ARRAY_SIZE(sha_1_256_algs); i++) { 2419 atmel_sha_alg_init(&sha_1_256_algs[i]); 2420 2421 err = crypto_register_ahash(&sha_1_256_algs[i]); 2422 if (err) 2423 goto err_sha_1_256_algs; 2424 } 2425 2426 if (dd->caps.has_sha224) { 2427 atmel_sha_alg_init(&sha_224_alg); 2428 2429 err = crypto_register_ahash(&sha_224_alg); 2430 if (err) 2431 goto err_sha_224_algs; 2432 } 2433 2434 if (dd->caps.has_sha_384_512) { 2435 for (i = 0; i < ARRAY_SIZE(sha_384_512_algs); i++) { 2436 atmel_sha_alg_init(&sha_384_512_algs[i]); 2437 2438 err = crypto_register_ahash(&sha_384_512_algs[i]); 2439 if (err) 2440 goto err_sha_384_512_algs; 2441 } 2442 } 2443 2444 if (dd->caps.has_hmac) { 2445 for (i = 0; i < ARRAY_SIZE(sha_hmac_algs); i++) { 2446 atmel_sha_hmac_alg_init(&sha_hmac_algs[i]); 2447 2448 err = crypto_register_ahash(&sha_hmac_algs[i]); 2449 if (err) 2450 goto err_sha_hmac_algs; 2451 } 2452 } 2453 2454 return 0; 2455 2456 /*i = ARRAY_SIZE(sha_hmac_algs);*/ 2457 err_sha_hmac_algs: 2458 for (j = 0; j < i; j++) 2459 crypto_unregister_ahash(&sha_hmac_algs[j]); 2460 i = ARRAY_SIZE(sha_384_512_algs); 2461 err_sha_384_512_algs: 2462 for (j = 0; j < i; j++) 2463 crypto_unregister_ahash(&sha_384_512_algs[j]); 2464 crypto_unregister_ahash(&sha_224_alg); 2465 err_sha_224_algs: 2466 i = ARRAY_SIZE(sha_1_256_algs); 2467 err_sha_1_256_algs: 2468 for (j = 0; j < i; j++) 2469 crypto_unregister_ahash(&sha_1_256_algs[j]); 2470 2471 return err; 2472 } 2473 2474 static int atmel_sha_dma_init(struct atmel_sha_dev *dd) 2475 { 2476 dd->dma_lch_in.chan = dma_request_chan(dd->dev, "tx"); 2477 if (IS_ERR(dd->dma_lch_in.chan)) { 2478 dev_err(dd->dev, "DMA channel is not available\n"); 2479 return PTR_ERR(dd->dma_lch_in.chan); 2480 } 2481 2482 dd->dma_lch_in.dma_conf.dst_addr = dd->phys_base + 2483 SHA_REG_DIN(0); 2484 dd->dma_lch_in.dma_conf.src_maxburst = 1; 2485 dd->dma_lch_in.dma_conf.src_addr_width = 2486 DMA_SLAVE_BUSWIDTH_4_BYTES; 2487 dd->dma_lch_in.dma_conf.dst_maxburst = 1; 2488 dd->dma_lch_in.dma_conf.dst_addr_width = 2489 DMA_SLAVE_BUSWIDTH_4_BYTES; 2490 dd->dma_lch_in.dma_conf.device_fc = false; 2491 2492 return 0; 2493 } 2494 2495 static void atmel_sha_dma_cleanup(struct atmel_sha_dev *dd) 2496 { 2497 dma_release_channel(dd->dma_lch_in.chan); 2498 } 2499 2500 static void atmel_sha_get_cap(struct atmel_sha_dev *dd) 2501 { 2502 2503 dd->caps.has_dma = 0; 2504 dd->caps.has_dualbuff = 0; 2505 dd->caps.has_sha224 = 0; 2506 dd->caps.has_sha_384_512 = 0; 2507 dd->caps.has_uihv = 0; 2508 dd->caps.has_hmac = 0; 2509 2510 /* keep only major version number */ 2511 switch (dd->hw_version & 0xff0) { 2512 case 0x510: 2513 dd->caps.has_dma = 1; 2514 dd->caps.has_dualbuff = 1; 2515 dd->caps.has_sha224 = 1; 2516 dd->caps.has_sha_384_512 = 1; 2517 dd->caps.has_uihv = 1; 2518 dd->caps.has_hmac = 1; 2519 break; 2520 case 0x420: 2521 dd->caps.has_dma = 1; 2522 dd->caps.has_dualbuff = 1; 2523 dd->caps.has_sha224 = 1; 2524 dd->caps.has_sha_384_512 = 1; 2525 dd->caps.has_uihv = 1; 2526 break; 2527 case 0x410: 2528 dd->caps.has_dma = 1; 2529 dd->caps.has_dualbuff = 1; 2530 dd->caps.has_sha224 = 1; 2531 dd->caps.has_sha_384_512 = 1; 2532 break; 2533 case 0x400: 2534 dd->caps.has_dma = 1; 2535 dd->caps.has_dualbuff = 1; 2536 dd->caps.has_sha224 = 1; 2537 break; 2538 case 0x320: 2539 break; 2540 default: 2541 dev_warn(dd->dev, 2542 "Unmanaged sha version, set minimum capabilities\n"); 2543 break; 2544 } 2545 } 2546 2547 #if defined(CONFIG_OF) 2548 static const struct of_device_id atmel_sha_dt_ids[] = { 2549 { .compatible = "atmel,at91sam9g46-sha" }, 2550 { /* sentinel */ } 2551 }; 2552 2553 MODULE_DEVICE_TABLE(of, atmel_sha_dt_ids); 2554 #endif 2555 2556 static int atmel_sha_probe(struct platform_device *pdev) 2557 { 2558 struct atmel_sha_dev *sha_dd; 2559 struct device *dev = &pdev->dev; 2560 struct resource *sha_res; 2561 int err; 2562 2563 sha_dd = devm_kzalloc(&pdev->dev, sizeof(*sha_dd), GFP_KERNEL); 2564 if (!sha_dd) 2565 return -ENOMEM; 2566 2567 sha_dd->dev = dev; 2568 2569 platform_set_drvdata(pdev, sha_dd); 2570 2571 INIT_LIST_HEAD(&sha_dd->list); 2572 spin_lock_init(&sha_dd->lock); 2573 2574 tasklet_init(&sha_dd->done_task, atmel_sha_done_task, 2575 (unsigned long)sha_dd); 2576 tasklet_init(&sha_dd->queue_task, atmel_sha_queue_task, 2577 (unsigned long)sha_dd); 2578 2579 crypto_init_queue(&sha_dd->queue, ATMEL_SHA_QUEUE_LENGTH); 2580 2581 /* Get the base address */ 2582 sha_res = platform_get_resource(pdev, IORESOURCE_MEM, 0); 2583 if (!sha_res) { 2584 dev_err(dev, "no MEM resource info\n"); 2585 err = -ENODEV; 2586 goto err_tasklet_kill; 2587 } 2588 sha_dd->phys_base = sha_res->start; 2589 2590 /* Get the IRQ */ 2591 sha_dd->irq = platform_get_irq(pdev, 0); 2592 if (sha_dd->irq < 0) { 2593 err = sha_dd->irq; 2594 goto err_tasklet_kill; 2595 } 2596 2597 err = devm_request_irq(&pdev->dev, sha_dd->irq, atmel_sha_irq, 2598 IRQF_SHARED, "atmel-sha", sha_dd); 2599 if (err) { 2600 dev_err(dev, "unable to request sha irq.\n"); 2601 goto err_tasklet_kill; 2602 } 2603 2604 /* Initializing the clock */ 2605 sha_dd->iclk = devm_clk_get(&pdev->dev, "sha_clk"); 2606 if (IS_ERR(sha_dd->iclk)) { 2607 dev_err(dev, "clock initialization failed.\n"); 2608 err = PTR_ERR(sha_dd->iclk); 2609 goto err_tasklet_kill; 2610 } 2611 2612 sha_dd->io_base = devm_ioremap_resource(&pdev->dev, sha_res); 2613 if (IS_ERR(sha_dd->io_base)) { 2614 dev_err(dev, "can't ioremap\n"); 2615 err = PTR_ERR(sha_dd->io_base); 2616 goto err_tasklet_kill; 2617 } 2618 2619 err = clk_prepare(sha_dd->iclk); 2620 if (err) 2621 goto err_tasklet_kill; 2622 2623 err = atmel_sha_hw_version_init(sha_dd); 2624 if (err) 2625 goto err_iclk_unprepare; 2626 2627 atmel_sha_get_cap(sha_dd); 2628 2629 if (sha_dd->caps.has_dma) { 2630 err = atmel_sha_dma_init(sha_dd); 2631 if (err) 2632 goto err_iclk_unprepare; 2633 2634 dev_info(dev, "using %s for DMA transfers\n", 2635 dma_chan_name(sha_dd->dma_lch_in.chan)); 2636 } 2637 2638 spin_lock(&atmel_sha.lock); 2639 list_add_tail(&sha_dd->list, &atmel_sha.dev_list); 2640 spin_unlock(&atmel_sha.lock); 2641 2642 err = atmel_sha_register_algs(sha_dd); 2643 if (err) 2644 goto err_algs; 2645 2646 dev_info(dev, "Atmel SHA1/SHA256%s%s\n", 2647 sha_dd->caps.has_sha224 ? "/SHA224" : "", 2648 sha_dd->caps.has_sha_384_512 ? "/SHA384/SHA512" : ""); 2649 2650 return 0; 2651 2652 err_algs: 2653 spin_lock(&atmel_sha.lock); 2654 list_del(&sha_dd->list); 2655 spin_unlock(&atmel_sha.lock); 2656 if (sha_dd->caps.has_dma) 2657 atmel_sha_dma_cleanup(sha_dd); 2658 err_iclk_unprepare: 2659 clk_unprepare(sha_dd->iclk); 2660 err_tasklet_kill: 2661 tasklet_kill(&sha_dd->queue_task); 2662 tasklet_kill(&sha_dd->done_task); 2663 2664 return err; 2665 } 2666 2667 static int atmel_sha_remove(struct platform_device *pdev) 2668 { 2669 struct atmel_sha_dev *sha_dd; 2670 2671 sha_dd = platform_get_drvdata(pdev); 2672 if (!sha_dd) 2673 return -ENODEV; 2674 spin_lock(&atmel_sha.lock); 2675 list_del(&sha_dd->list); 2676 spin_unlock(&atmel_sha.lock); 2677 2678 atmel_sha_unregister_algs(sha_dd); 2679 2680 tasklet_kill(&sha_dd->queue_task); 2681 tasklet_kill(&sha_dd->done_task); 2682 2683 if (sha_dd->caps.has_dma) 2684 atmel_sha_dma_cleanup(sha_dd); 2685 2686 clk_unprepare(sha_dd->iclk); 2687 2688 return 0; 2689 } 2690 2691 static struct platform_driver atmel_sha_driver = { 2692 .probe = atmel_sha_probe, 2693 .remove = atmel_sha_remove, 2694 .driver = { 2695 .name = "atmel_sha", 2696 .of_match_table = of_match_ptr(atmel_sha_dt_ids), 2697 }, 2698 }; 2699 2700 module_platform_driver(atmel_sha_driver); 2701 2702 MODULE_DESCRIPTION("Atmel SHA (1/256/224/384/512) hw acceleration support."); 2703 MODULE_LICENSE("GPL v2"); 2704 MODULE_AUTHOR("Nicolas Royer - Eukréa Electromatique"); 2705