1 /* 2 * Cryptographic API. 3 * 4 * Support for ATMEL AES HW acceleration. 5 * 6 * Copyright (c) 2012 Eukréa Electromatique - ATMEL 7 * Author: Nicolas Royer <nicolas@eukrea.com> 8 * 9 * This program is free software; you can redistribute it and/or modify 10 * it under the terms of the GNU General Public License version 2 as published 11 * by the Free Software Foundation. 12 * 13 * Some ideas are from omap-aes.c driver. 14 */ 15 16 17 #include <linux/kernel.h> 18 #include <linux/module.h> 19 #include <linux/slab.h> 20 #include <linux/err.h> 21 #include <linux/clk.h> 22 #include <linux/io.h> 23 #include <linux/hw_random.h> 24 #include <linux/platform_device.h> 25 26 #include <linux/device.h> 27 #include <linux/init.h> 28 #include <linux/errno.h> 29 #include <linux/interrupt.h> 30 #include <linux/irq.h> 31 #include <linux/scatterlist.h> 32 #include <linux/dma-mapping.h> 33 #include <linux/of_device.h> 34 #include <linux/delay.h> 35 #include <linux/crypto.h> 36 #include <crypto/scatterwalk.h> 37 #include <crypto/algapi.h> 38 #include <crypto/aes.h> 39 #include <crypto/xts.h> 40 #include <crypto/internal/aead.h> 41 #include <linux/platform_data/crypto-atmel.h> 42 #include <dt-bindings/dma/at91.h> 43 #include "atmel-aes-regs.h" 44 #include "atmel-authenc.h" 45 46 #define ATMEL_AES_PRIORITY 300 47 48 #define ATMEL_AES_BUFFER_ORDER 2 49 #define ATMEL_AES_BUFFER_SIZE (PAGE_SIZE << ATMEL_AES_BUFFER_ORDER) 50 51 #define CFB8_BLOCK_SIZE 1 52 #define CFB16_BLOCK_SIZE 2 53 #define CFB32_BLOCK_SIZE 4 54 #define CFB64_BLOCK_SIZE 8 55 56 #define SIZE_IN_WORDS(x) ((x) >> 2) 57 58 /* AES flags */ 59 /* Reserve bits [18:16] [14:12] [1:0] for mode (same as for AES_MR) */ 60 #define AES_FLAGS_ENCRYPT AES_MR_CYPHER_ENC 61 #define AES_FLAGS_GTAGEN AES_MR_GTAGEN 62 #define AES_FLAGS_OPMODE_MASK (AES_MR_OPMOD_MASK | AES_MR_CFBS_MASK) 63 #define AES_FLAGS_ECB AES_MR_OPMOD_ECB 64 #define AES_FLAGS_CBC AES_MR_OPMOD_CBC 65 #define AES_FLAGS_OFB AES_MR_OPMOD_OFB 66 #define AES_FLAGS_CFB128 (AES_MR_OPMOD_CFB | AES_MR_CFBS_128b) 67 #define AES_FLAGS_CFB64 (AES_MR_OPMOD_CFB | AES_MR_CFBS_64b) 68 #define AES_FLAGS_CFB32 (AES_MR_OPMOD_CFB | AES_MR_CFBS_32b) 69 #define AES_FLAGS_CFB16 (AES_MR_OPMOD_CFB | AES_MR_CFBS_16b) 70 #define AES_FLAGS_CFB8 (AES_MR_OPMOD_CFB | AES_MR_CFBS_8b) 71 #define AES_FLAGS_CTR AES_MR_OPMOD_CTR 72 #define AES_FLAGS_GCM AES_MR_OPMOD_GCM 73 #define AES_FLAGS_XTS AES_MR_OPMOD_XTS 74 75 #define AES_FLAGS_MODE_MASK (AES_FLAGS_OPMODE_MASK | \ 76 AES_FLAGS_ENCRYPT | \ 77 AES_FLAGS_GTAGEN) 78 79 #define AES_FLAGS_INIT BIT(2) 80 #define AES_FLAGS_BUSY BIT(3) 81 #define AES_FLAGS_DUMP_REG BIT(4) 82 #define AES_FLAGS_OWN_SHA BIT(5) 83 84 #define AES_FLAGS_PERSISTENT (AES_FLAGS_INIT | AES_FLAGS_BUSY) 85 86 #define ATMEL_AES_QUEUE_LENGTH 50 87 88 #define ATMEL_AES_DMA_THRESHOLD 256 89 90 91 struct atmel_aes_caps { 92 bool has_dualbuff; 93 bool has_cfb64; 94 bool has_ctr32; 95 bool has_gcm; 96 bool has_xts; 97 bool has_authenc; 98 u32 max_burst_size; 99 }; 100 101 struct atmel_aes_dev; 102 103 104 typedef int (*atmel_aes_fn_t)(struct atmel_aes_dev *); 105 106 107 struct atmel_aes_base_ctx { 108 struct atmel_aes_dev *dd; 109 atmel_aes_fn_t start; 110 int keylen; 111 u32 key[AES_KEYSIZE_256 / sizeof(u32)]; 112 u16 block_size; 113 }; 114 115 struct atmel_aes_ctx { 116 struct atmel_aes_base_ctx base; 117 }; 118 119 struct atmel_aes_ctr_ctx { 120 struct atmel_aes_base_ctx base; 121 122 u32 iv[AES_BLOCK_SIZE / sizeof(u32)]; 123 size_t offset; 124 struct scatterlist src[2]; 125 struct scatterlist dst[2]; 126 }; 127 128 struct atmel_aes_gcm_ctx { 129 struct atmel_aes_base_ctx base; 130 131 struct scatterlist src[2]; 132 struct scatterlist dst[2]; 133 134 u32 j0[AES_BLOCK_SIZE / sizeof(u32)]; 135 u32 tag[AES_BLOCK_SIZE / sizeof(u32)]; 136 u32 ghash[AES_BLOCK_SIZE / sizeof(u32)]; 137 size_t textlen; 138 139 const u32 *ghash_in; 140 u32 *ghash_out; 141 atmel_aes_fn_t ghash_resume; 142 }; 143 144 struct atmel_aes_xts_ctx { 145 struct atmel_aes_base_ctx base; 146 147 u32 key2[AES_KEYSIZE_256 / sizeof(u32)]; 148 }; 149 150 #ifdef CONFIG_CRYPTO_DEV_ATMEL_AUTHENC 151 struct atmel_aes_authenc_ctx { 152 struct atmel_aes_base_ctx base; 153 struct atmel_sha_authenc_ctx *auth; 154 }; 155 #endif 156 157 struct atmel_aes_reqctx { 158 unsigned long mode; 159 }; 160 161 #ifdef CONFIG_CRYPTO_DEV_ATMEL_AUTHENC 162 struct atmel_aes_authenc_reqctx { 163 struct atmel_aes_reqctx base; 164 165 struct scatterlist src[2]; 166 struct scatterlist dst[2]; 167 size_t textlen; 168 u32 digest[SHA512_DIGEST_SIZE / sizeof(u32)]; 169 170 /* auth_req MUST be place last. */ 171 struct ahash_request auth_req; 172 }; 173 #endif 174 175 struct atmel_aes_dma { 176 struct dma_chan *chan; 177 struct scatterlist *sg; 178 int nents; 179 unsigned int remainder; 180 unsigned int sg_len; 181 }; 182 183 struct atmel_aes_dev { 184 struct list_head list; 185 unsigned long phys_base; 186 void __iomem *io_base; 187 188 struct crypto_async_request *areq; 189 struct atmel_aes_base_ctx *ctx; 190 191 bool is_async; 192 atmel_aes_fn_t resume; 193 atmel_aes_fn_t cpu_transfer_complete; 194 195 struct device *dev; 196 struct clk *iclk; 197 int irq; 198 199 unsigned long flags; 200 201 spinlock_t lock; 202 struct crypto_queue queue; 203 204 struct tasklet_struct done_task; 205 struct tasklet_struct queue_task; 206 207 size_t total; 208 size_t datalen; 209 u32 *data; 210 211 struct atmel_aes_dma src; 212 struct atmel_aes_dma dst; 213 214 size_t buflen; 215 void *buf; 216 struct scatterlist aligned_sg; 217 struct scatterlist *real_dst; 218 219 struct atmel_aes_caps caps; 220 221 u32 hw_version; 222 }; 223 224 struct atmel_aes_drv { 225 struct list_head dev_list; 226 spinlock_t lock; 227 }; 228 229 static struct atmel_aes_drv atmel_aes = { 230 .dev_list = LIST_HEAD_INIT(atmel_aes.dev_list), 231 .lock = __SPIN_LOCK_UNLOCKED(atmel_aes.lock), 232 }; 233 234 #ifdef VERBOSE_DEBUG 235 static const char *atmel_aes_reg_name(u32 offset, char *tmp, size_t sz) 236 { 237 switch (offset) { 238 case AES_CR: 239 return "CR"; 240 241 case AES_MR: 242 return "MR"; 243 244 case AES_ISR: 245 return "ISR"; 246 247 case AES_IMR: 248 return "IMR"; 249 250 case AES_IER: 251 return "IER"; 252 253 case AES_IDR: 254 return "IDR"; 255 256 case AES_KEYWR(0): 257 case AES_KEYWR(1): 258 case AES_KEYWR(2): 259 case AES_KEYWR(3): 260 case AES_KEYWR(4): 261 case AES_KEYWR(5): 262 case AES_KEYWR(6): 263 case AES_KEYWR(7): 264 snprintf(tmp, sz, "KEYWR[%u]", (offset - AES_KEYWR(0)) >> 2); 265 break; 266 267 case AES_IDATAR(0): 268 case AES_IDATAR(1): 269 case AES_IDATAR(2): 270 case AES_IDATAR(3): 271 snprintf(tmp, sz, "IDATAR[%u]", (offset - AES_IDATAR(0)) >> 2); 272 break; 273 274 case AES_ODATAR(0): 275 case AES_ODATAR(1): 276 case AES_ODATAR(2): 277 case AES_ODATAR(3): 278 snprintf(tmp, sz, "ODATAR[%u]", (offset - AES_ODATAR(0)) >> 2); 279 break; 280 281 case AES_IVR(0): 282 case AES_IVR(1): 283 case AES_IVR(2): 284 case AES_IVR(3): 285 snprintf(tmp, sz, "IVR[%u]", (offset - AES_IVR(0)) >> 2); 286 break; 287 288 case AES_AADLENR: 289 return "AADLENR"; 290 291 case AES_CLENR: 292 return "CLENR"; 293 294 case AES_GHASHR(0): 295 case AES_GHASHR(1): 296 case AES_GHASHR(2): 297 case AES_GHASHR(3): 298 snprintf(tmp, sz, "GHASHR[%u]", (offset - AES_GHASHR(0)) >> 2); 299 break; 300 301 case AES_TAGR(0): 302 case AES_TAGR(1): 303 case AES_TAGR(2): 304 case AES_TAGR(3): 305 snprintf(tmp, sz, "TAGR[%u]", (offset - AES_TAGR(0)) >> 2); 306 break; 307 308 case AES_CTRR: 309 return "CTRR"; 310 311 case AES_GCMHR(0): 312 case AES_GCMHR(1): 313 case AES_GCMHR(2): 314 case AES_GCMHR(3): 315 snprintf(tmp, sz, "GCMHR[%u]", (offset - AES_GCMHR(0)) >> 2); 316 break; 317 318 case AES_EMR: 319 return "EMR"; 320 321 case AES_TWR(0): 322 case AES_TWR(1): 323 case AES_TWR(2): 324 case AES_TWR(3): 325 snprintf(tmp, sz, "TWR[%u]", (offset - AES_TWR(0)) >> 2); 326 break; 327 328 case AES_ALPHAR(0): 329 case AES_ALPHAR(1): 330 case AES_ALPHAR(2): 331 case AES_ALPHAR(3): 332 snprintf(tmp, sz, "ALPHAR[%u]", (offset - AES_ALPHAR(0)) >> 2); 333 break; 334 335 default: 336 snprintf(tmp, sz, "0x%02x", offset); 337 break; 338 } 339 340 return tmp; 341 } 342 #endif /* VERBOSE_DEBUG */ 343 344 /* Shared functions */ 345 346 static inline u32 atmel_aes_read(struct atmel_aes_dev *dd, u32 offset) 347 { 348 u32 value = readl_relaxed(dd->io_base + offset); 349 350 #ifdef VERBOSE_DEBUG 351 if (dd->flags & AES_FLAGS_DUMP_REG) { 352 char tmp[16]; 353 354 dev_vdbg(dd->dev, "read 0x%08x from %s\n", value, 355 atmel_aes_reg_name(offset, tmp, sizeof(tmp))); 356 } 357 #endif /* VERBOSE_DEBUG */ 358 359 return value; 360 } 361 362 static inline void atmel_aes_write(struct atmel_aes_dev *dd, 363 u32 offset, u32 value) 364 { 365 #ifdef VERBOSE_DEBUG 366 if (dd->flags & AES_FLAGS_DUMP_REG) { 367 char tmp[16]; 368 369 dev_vdbg(dd->dev, "write 0x%08x into %s\n", value, 370 atmel_aes_reg_name(offset, tmp, sizeof(tmp))); 371 } 372 #endif /* VERBOSE_DEBUG */ 373 374 writel_relaxed(value, dd->io_base + offset); 375 } 376 377 static void atmel_aes_read_n(struct atmel_aes_dev *dd, u32 offset, 378 u32 *value, int count) 379 { 380 for (; count--; value++, offset += 4) 381 *value = atmel_aes_read(dd, offset); 382 } 383 384 static void atmel_aes_write_n(struct atmel_aes_dev *dd, u32 offset, 385 const u32 *value, int count) 386 { 387 for (; count--; value++, offset += 4) 388 atmel_aes_write(dd, offset, *value); 389 } 390 391 static inline void atmel_aes_read_block(struct atmel_aes_dev *dd, u32 offset, 392 u32 *value) 393 { 394 atmel_aes_read_n(dd, offset, value, SIZE_IN_WORDS(AES_BLOCK_SIZE)); 395 } 396 397 static inline void atmel_aes_write_block(struct atmel_aes_dev *dd, u32 offset, 398 const u32 *value) 399 { 400 atmel_aes_write_n(dd, offset, value, SIZE_IN_WORDS(AES_BLOCK_SIZE)); 401 } 402 403 static inline int atmel_aes_wait_for_data_ready(struct atmel_aes_dev *dd, 404 atmel_aes_fn_t resume) 405 { 406 u32 isr = atmel_aes_read(dd, AES_ISR); 407 408 if (unlikely(isr & AES_INT_DATARDY)) 409 return resume(dd); 410 411 dd->resume = resume; 412 atmel_aes_write(dd, AES_IER, AES_INT_DATARDY); 413 return -EINPROGRESS; 414 } 415 416 static inline size_t atmel_aes_padlen(size_t len, size_t block_size) 417 { 418 len &= block_size - 1; 419 return len ? block_size - len : 0; 420 } 421 422 static struct atmel_aes_dev *atmel_aes_find_dev(struct atmel_aes_base_ctx *ctx) 423 { 424 struct atmel_aes_dev *aes_dd = NULL; 425 struct atmel_aes_dev *tmp; 426 427 spin_lock_bh(&atmel_aes.lock); 428 if (!ctx->dd) { 429 list_for_each_entry(tmp, &atmel_aes.dev_list, list) { 430 aes_dd = tmp; 431 break; 432 } 433 ctx->dd = aes_dd; 434 } else { 435 aes_dd = ctx->dd; 436 } 437 438 spin_unlock_bh(&atmel_aes.lock); 439 440 return aes_dd; 441 } 442 443 static int atmel_aes_hw_init(struct atmel_aes_dev *dd) 444 { 445 int err; 446 447 err = clk_enable(dd->iclk); 448 if (err) 449 return err; 450 451 if (!(dd->flags & AES_FLAGS_INIT)) { 452 atmel_aes_write(dd, AES_CR, AES_CR_SWRST); 453 atmel_aes_write(dd, AES_MR, 0xE << AES_MR_CKEY_OFFSET); 454 dd->flags |= AES_FLAGS_INIT; 455 } 456 457 return 0; 458 } 459 460 static inline unsigned int atmel_aes_get_version(struct atmel_aes_dev *dd) 461 { 462 return atmel_aes_read(dd, AES_HW_VERSION) & 0x00000fff; 463 } 464 465 static int atmel_aes_hw_version_init(struct atmel_aes_dev *dd) 466 { 467 int err; 468 469 err = atmel_aes_hw_init(dd); 470 if (err) 471 return err; 472 473 dd->hw_version = atmel_aes_get_version(dd); 474 475 dev_info(dd->dev, "version: 0x%x\n", dd->hw_version); 476 477 clk_disable(dd->iclk); 478 return 0; 479 } 480 481 static inline void atmel_aes_set_mode(struct atmel_aes_dev *dd, 482 const struct atmel_aes_reqctx *rctx) 483 { 484 /* Clear all but persistent flags and set request flags. */ 485 dd->flags = (dd->flags & AES_FLAGS_PERSISTENT) | rctx->mode; 486 } 487 488 static inline bool atmel_aes_is_encrypt(const struct atmel_aes_dev *dd) 489 { 490 return (dd->flags & AES_FLAGS_ENCRYPT); 491 } 492 493 #ifdef CONFIG_CRYPTO_DEV_ATMEL_AUTHENC 494 static void atmel_aes_authenc_complete(struct atmel_aes_dev *dd, int err); 495 #endif 496 497 static inline int atmel_aes_complete(struct atmel_aes_dev *dd, int err) 498 { 499 #ifdef CONFIG_CRYPTO_DEV_ATMEL_AUTHENC 500 atmel_aes_authenc_complete(dd, err); 501 #endif 502 503 clk_disable(dd->iclk); 504 dd->flags &= ~AES_FLAGS_BUSY; 505 506 if (dd->is_async) 507 dd->areq->complete(dd->areq, err); 508 509 tasklet_schedule(&dd->queue_task); 510 511 return err; 512 } 513 514 static void atmel_aes_write_ctrl_key(struct atmel_aes_dev *dd, bool use_dma, 515 const u32 *iv, const u32 *key, int keylen) 516 { 517 u32 valmr = 0; 518 519 /* MR register must be set before IV registers */ 520 if (keylen == AES_KEYSIZE_128) 521 valmr |= AES_MR_KEYSIZE_128; 522 else if (keylen == AES_KEYSIZE_192) 523 valmr |= AES_MR_KEYSIZE_192; 524 else 525 valmr |= AES_MR_KEYSIZE_256; 526 527 valmr |= dd->flags & AES_FLAGS_MODE_MASK; 528 529 if (use_dma) { 530 valmr |= AES_MR_SMOD_IDATAR0; 531 if (dd->caps.has_dualbuff) 532 valmr |= AES_MR_DUALBUFF; 533 } else { 534 valmr |= AES_MR_SMOD_AUTO; 535 } 536 537 atmel_aes_write(dd, AES_MR, valmr); 538 539 atmel_aes_write_n(dd, AES_KEYWR(0), key, SIZE_IN_WORDS(keylen)); 540 541 if (iv && (valmr & AES_MR_OPMOD_MASK) != AES_MR_OPMOD_ECB) 542 atmel_aes_write_block(dd, AES_IVR(0), iv); 543 } 544 545 static inline void atmel_aes_write_ctrl(struct atmel_aes_dev *dd, bool use_dma, 546 const u32 *iv) 547 548 { 549 atmel_aes_write_ctrl_key(dd, use_dma, iv, 550 dd->ctx->key, dd->ctx->keylen); 551 } 552 553 /* CPU transfer */ 554 555 static int atmel_aes_cpu_transfer(struct atmel_aes_dev *dd) 556 { 557 int err = 0; 558 u32 isr; 559 560 for (;;) { 561 atmel_aes_read_block(dd, AES_ODATAR(0), dd->data); 562 dd->data += 4; 563 dd->datalen -= AES_BLOCK_SIZE; 564 565 if (dd->datalen < AES_BLOCK_SIZE) 566 break; 567 568 atmel_aes_write_block(dd, AES_IDATAR(0), dd->data); 569 570 isr = atmel_aes_read(dd, AES_ISR); 571 if (!(isr & AES_INT_DATARDY)) { 572 dd->resume = atmel_aes_cpu_transfer; 573 atmel_aes_write(dd, AES_IER, AES_INT_DATARDY); 574 return -EINPROGRESS; 575 } 576 } 577 578 if (!sg_copy_from_buffer(dd->real_dst, sg_nents(dd->real_dst), 579 dd->buf, dd->total)) 580 err = -EINVAL; 581 582 if (err) 583 return atmel_aes_complete(dd, err); 584 585 return dd->cpu_transfer_complete(dd); 586 } 587 588 static int atmel_aes_cpu_start(struct atmel_aes_dev *dd, 589 struct scatterlist *src, 590 struct scatterlist *dst, 591 size_t len, 592 atmel_aes_fn_t resume) 593 { 594 size_t padlen = atmel_aes_padlen(len, AES_BLOCK_SIZE); 595 596 if (unlikely(len == 0)) 597 return -EINVAL; 598 599 sg_copy_to_buffer(src, sg_nents(src), dd->buf, len); 600 601 dd->total = len; 602 dd->real_dst = dst; 603 dd->cpu_transfer_complete = resume; 604 dd->datalen = len + padlen; 605 dd->data = (u32 *)dd->buf; 606 atmel_aes_write_block(dd, AES_IDATAR(0), dd->data); 607 return atmel_aes_wait_for_data_ready(dd, atmel_aes_cpu_transfer); 608 } 609 610 611 /* DMA transfer */ 612 613 static void atmel_aes_dma_callback(void *data); 614 615 static bool atmel_aes_check_aligned(struct atmel_aes_dev *dd, 616 struct scatterlist *sg, 617 size_t len, 618 struct atmel_aes_dma *dma) 619 { 620 int nents; 621 622 if (!IS_ALIGNED(len, dd->ctx->block_size)) 623 return false; 624 625 for (nents = 0; sg; sg = sg_next(sg), ++nents) { 626 if (!IS_ALIGNED(sg->offset, sizeof(u32))) 627 return false; 628 629 if (len <= sg->length) { 630 if (!IS_ALIGNED(len, dd->ctx->block_size)) 631 return false; 632 633 dma->nents = nents+1; 634 dma->remainder = sg->length - len; 635 sg->length = len; 636 return true; 637 } 638 639 if (!IS_ALIGNED(sg->length, dd->ctx->block_size)) 640 return false; 641 642 len -= sg->length; 643 } 644 645 return false; 646 } 647 648 static inline void atmel_aes_restore_sg(const struct atmel_aes_dma *dma) 649 { 650 struct scatterlist *sg = dma->sg; 651 int nents = dma->nents; 652 653 if (!dma->remainder) 654 return; 655 656 while (--nents > 0 && sg) 657 sg = sg_next(sg); 658 659 if (!sg) 660 return; 661 662 sg->length += dma->remainder; 663 } 664 665 static int atmel_aes_map(struct atmel_aes_dev *dd, 666 struct scatterlist *src, 667 struct scatterlist *dst, 668 size_t len) 669 { 670 bool src_aligned, dst_aligned; 671 size_t padlen; 672 673 dd->total = len; 674 dd->src.sg = src; 675 dd->dst.sg = dst; 676 dd->real_dst = dst; 677 678 src_aligned = atmel_aes_check_aligned(dd, src, len, &dd->src); 679 if (src == dst) 680 dst_aligned = src_aligned; 681 else 682 dst_aligned = atmel_aes_check_aligned(dd, dst, len, &dd->dst); 683 if (!src_aligned || !dst_aligned) { 684 padlen = atmel_aes_padlen(len, dd->ctx->block_size); 685 686 if (dd->buflen < len + padlen) 687 return -ENOMEM; 688 689 if (!src_aligned) { 690 sg_copy_to_buffer(src, sg_nents(src), dd->buf, len); 691 dd->src.sg = &dd->aligned_sg; 692 dd->src.nents = 1; 693 dd->src.remainder = 0; 694 } 695 696 if (!dst_aligned) { 697 dd->dst.sg = &dd->aligned_sg; 698 dd->dst.nents = 1; 699 dd->dst.remainder = 0; 700 } 701 702 sg_init_table(&dd->aligned_sg, 1); 703 sg_set_buf(&dd->aligned_sg, dd->buf, len + padlen); 704 } 705 706 if (dd->src.sg == dd->dst.sg) { 707 dd->src.sg_len = dma_map_sg(dd->dev, dd->src.sg, dd->src.nents, 708 DMA_BIDIRECTIONAL); 709 dd->dst.sg_len = dd->src.sg_len; 710 if (!dd->src.sg_len) 711 return -EFAULT; 712 } else { 713 dd->src.sg_len = dma_map_sg(dd->dev, dd->src.sg, dd->src.nents, 714 DMA_TO_DEVICE); 715 if (!dd->src.sg_len) 716 return -EFAULT; 717 718 dd->dst.sg_len = dma_map_sg(dd->dev, dd->dst.sg, dd->dst.nents, 719 DMA_FROM_DEVICE); 720 if (!dd->dst.sg_len) { 721 dma_unmap_sg(dd->dev, dd->src.sg, dd->src.nents, 722 DMA_TO_DEVICE); 723 return -EFAULT; 724 } 725 } 726 727 return 0; 728 } 729 730 static void atmel_aes_unmap(struct atmel_aes_dev *dd) 731 { 732 if (dd->src.sg == dd->dst.sg) { 733 dma_unmap_sg(dd->dev, dd->src.sg, dd->src.nents, 734 DMA_BIDIRECTIONAL); 735 736 if (dd->src.sg != &dd->aligned_sg) 737 atmel_aes_restore_sg(&dd->src); 738 } else { 739 dma_unmap_sg(dd->dev, dd->dst.sg, dd->dst.nents, 740 DMA_FROM_DEVICE); 741 742 if (dd->dst.sg != &dd->aligned_sg) 743 atmel_aes_restore_sg(&dd->dst); 744 745 dma_unmap_sg(dd->dev, dd->src.sg, dd->src.nents, 746 DMA_TO_DEVICE); 747 748 if (dd->src.sg != &dd->aligned_sg) 749 atmel_aes_restore_sg(&dd->src); 750 } 751 752 if (dd->dst.sg == &dd->aligned_sg) 753 sg_copy_from_buffer(dd->real_dst, sg_nents(dd->real_dst), 754 dd->buf, dd->total); 755 } 756 757 static int atmel_aes_dma_transfer_start(struct atmel_aes_dev *dd, 758 enum dma_slave_buswidth addr_width, 759 enum dma_transfer_direction dir, 760 u32 maxburst) 761 { 762 struct dma_async_tx_descriptor *desc; 763 struct dma_slave_config config; 764 dma_async_tx_callback callback; 765 struct atmel_aes_dma *dma; 766 int err; 767 768 memset(&config, 0, sizeof(config)); 769 config.direction = dir; 770 config.src_addr_width = addr_width; 771 config.dst_addr_width = addr_width; 772 config.src_maxburst = maxburst; 773 config.dst_maxburst = maxburst; 774 775 switch (dir) { 776 case DMA_MEM_TO_DEV: 777 dma = &dd->src; 778 callback = NULL; 779 config.dst_addr = dd->phys_base + AES_IDATAR(0); 780 break; 781 782 case DMA_DEV_TO_MEM: 783 dma = &dd->dst; 784 callback = atmel_aes_dma_callback; 785 config.src_addr = dd->phys_base + AES_ODATAR(0); 786 break; 787 788 default: 789 return -EINVAL; 790 } 791 792 err = dmaengine_slave_config(dma->chan, &config); 793 if (err) 794 return err; 795 796 desc = dmaengine_prep_slave_sg(dma->chan, dma->sg, dma->sg_len, dir, 797 DMA_PREP_INTERRUPT | DMA_CTRL_ACK); 798 if (!desc) 799 return -ENOMEM; 800 801 desc->callback = callback; 802 desc->callback_param = dd; 803 dmaengine_submit(desc); 804 dma_async_issue_pending(dma->chan); 805 806 return 0; 807 } 808 809 static void atmel_aes_dma_transfer_stop(struct atmel_aes_dev *dd, 810 enum dma_transfer_direction dir) 811 { 812 struct atmel_aes_dma *dma; 813 814 switch (dir) { 815 case DMA_MEM_TO_DEV: 816 dma = &dd->src; 817 break; 818 819 case DMA_DEV_TO_MEM: 820 dma = &dd->dst; 821 break; 822 823 default: 824 return; 825 } 826 827 dmaengine_terminate_all(dma->chan); 828 } 829 830 static int atmel_aes_dma_start(struct atmel_aes_dev *dd, 831 struct scatterlist *src, 832 struct scatterlist *dst, 833 size_t len, 834 atmel_aes_fn_t resume) 835 { 836 enum dma_slave_buswidth addr_width; 837 u32 maxburst; 838 int err; 839 840 switch (dd->ctx->block_size) { 841 case CFB8_BLOCK_SIZE: 842 addr_width = DMA_SLAVE_BUSWIDTH_1_BYTE; 843 maxburst = 1; 844 break; 845 846 case CFB16_BLOCK_SIZE: 847 addr_width = DMA_SLAVE_BUSWIDTH_2_BYTES; 848 maxburst = 1; 849 break; 850 851 case CFB32_BLOCK_SIZE: 852 case CFB64_BLOCK_SIZE: 853 addr_width = DMA_SLAVE_BUSWIDTH_4_BYTES; 854 maxburst = 1; 855 break; 856 857 case AES_BLOCK_SIZE: 858 addr_width = DMA_SLAVE_BUSWIDTH_4_BYTES; 859 maxburst = dd->caps.max_burst_size; 860 break; 861 862 default: 863 err = -EINVAL; 864 goto exit; 865 } 866 867 err = atmel_aes_map(dd, src, dst, len); 868 if (err) 869 goto exit; 870 871 dd->resume = resume; 872 873 /* Set output DMA transfer first */ 874 err = atmel_aes_dma_transfer_start(dd, addr_width, DMA_DEV_TO_MEM, 875 maxburst); 876 if (err) 877 goto unmap; 878 879 /* Then set input DMA transfer */ 880 err = atmel_aes_dma_transfer_start(dd, addr_width, DMA_MEM_TO_DEV, 881 maxburst); 882 if (err) 883 goto output_transfer_stop; 884 885 return -EINPROGRESS; 886 887 output_transfer_stop: 888 atmel_aes_dma_transfer_stop(dd, DMA_DEV_TO_MEM); 889 unmap: 890 atmel_aes_unmap(dd); 891 exit: 892 return atmel_aes_complete(dd, err); 893 } 894 895 static void atmel_aes_dma_stop(struct atmel_aes_dev *dd) 896 { 897 atmel_aes_dma_transfer_stop(dd, DMA_MEM_TO_DEV); 898 atmel_aes_dma_transfer_stop(dd, DMA_DEV_TO_MEM); 899 atmel_aes_unmap(dd); 900 } 901 902 static void atmel_aes_dma_callback(void *data) 903 { 904 struct atmel_aes_dev *dd = data; 905 906 atmel_aes_dma_stop(dd); 907 dd->is_async = true; 908 (void)dd->resume(dd); 909 } 910 911 static int atmel_aes_handle_queue(struct atmel_aes_dev *dd, 912 struct crypto_async_request *new_areq) 913 { 914 struct crypto_async_request *areq, *backlog; 915 struct atmel_aes_base_ctx *ctx; 916 unsigned long flags; 917 bool start_async; 918 int err, ret = 0; 919 920 spin_lock_irqsave(&dd->lock, flags); 921 if (new_areq) 922 ret = crypto_enqueue_request(&dd->queue, new_areq); 923 if (dd->flags & AES_FLAGS_BUSY) { 924 spin_unlock_irqrestore(&dd->lock, flags); 925 return ret; 926 } 927 backlog = crypto_get_backlog(&dd->queue); 928 areq = crypto_dequeue_request(&dd->queue); 929 if (areq) 930 dd->flags |= AES_FLAGS_BUSY; 931 spin_unlock_irqrestore(&dd->lock, flags); 932 933 if (!areq) 934 return ret; 935 936 if (backlog) 937 backlog->complete(backlog, -EINPROGRESS); 938 939 ctx = crypto_tfm_ctx(areq->tfm); 940 941 dd->areq = areq; 942 dd->ctx = ctx; 943 start_async = (areq != new_areq); 944 dd->is_async = start_async; 945 946 /* WARNING: ctx->start() MAY change dd->is_async. */ 947 err = ctx->start(dd); 948 return (start_async) ? ret : err; 949 } 950 951 952 /* AES async block ciphers */ 953 954 static int atmel_aes_transfer_complete(struct atmel_aes_dev *dd) 955 { 956 return atmel_aes_complete(dd, 0); 957 } 958 959 static int atmel_aes_start(struct atmel_aes_dev *dd) 960 { 961 struct ablkcipher_request *req = ablkcipher_request_cast(dd->areq); 962 struct atmel_aes_reqctx *rctx = ablkcipher_request_ctx(req); 963 bool use_dma = (req->nbytes >= ATMEL_AES_DMA_THRESHOLD || 964 dd->ctx->block_size != AES_BLOCK_SIZE); 965 int err; 966 967 atmel_aes_set_mode(dd, rctx); 968 969 err = atmel_aes_hw_init(dd); 970 if (err) 971 return atmel_aes_complete(dd, err); 972 973 atmel_aes_write_ctrl(dd, use_dma, req->info); 974 if (use_dma) 975 return atmel_aes_dma_start(dd, req->src, req->dst, req->nbytes, 976 atmel_aes_transfer_complete); 977 978 return atmel_aes_cpu_start(dd, req->src, req->dst, req->nbytes, 979 atmel_aes_transfer_complete); 980 } 981 982 static inline struct atmel_aes_ctr_ctx * 983 atmel_aes_ctr_ctx_cast(struct atmel_aes_base_ctx *ctx) 984 { 985 return container_of(ctx, struct atmel_aes_ctr_ctx, base); 986 } 987 988 static int atmel_aes_ctr_transfer(struct atmel_aes_dev *dd) 989 { 990 struct atmel_aes_ctr_ctx *ctx = atmel_aes_ctr_ctx_cast(dd->ctx); 991 struct ablkcipher_request *req = ablkcipher_request_cast(dd->areq); 992 struct scatterlist *src, *dst; 993 u32 ctr, blocks; 994 size_t datalen; 995 bool use_dma, fragmented = false; 996 997 /* Check for transfer completion. */ 998 ctx->offset += dd->total; 999 if (ctx->offset >= req->nbytes) 1000 return atmel_aes_transfer_complete(dd); 1001 1002 /* Compute data length. */ 1003 datalen = req->nbytes - ctx->offset; 1004 blocks = DIV_ROUND_UP(datalen, AES_BLOCK_SIZE); 1005 ctr = be32_to_cpu(ctx->iv[3]); 1006 if (dd->caps.has_ctr32) { 1007 /* Check 32bit counter overflow. */ 1008 u32 start = ctr; 1009 u32 end = start + blocks - 1; 1010 1011 if (end < start) { 1012 ctr |= 0xffffffff; 1013 datalen = AES_BLOCK_SIZE * -start; 1014 fragmented = true; 1015 } 1016 } else { 1017 /* Check 16bit counter overflow. */ 1018 u16 start = ctr & 0xffff; 1019 u16 end = start + (u16)blocks - 1; 1020 1021 if (blocks >> 16 || end < start) { 1022 ctr |= 0xffff; 1023 datalen = AES_BLOCK_SIZE * (0x10000-start); 1024 fragmented = true; 1025 } 1026 } 1027 use_dma = (datalen >= ATMEL_AES_DMA_THRESHOLD); 1028 1029 /* Jump to offset. */ 1030 src = scatterwalk_ffwd(ctx->src, req->src, ctx->offset); 1031 dst = ((req->src == req->dst) ? src : 1032 scatterwalk_ffwd(ctx->dst, req->dst, ctx->offset)); 1033 1034 /* Configure hardware. */ 1035 atmel_aes_write_ctrl(dd, use_dma, ctx->iv); 1036 if (unlikely(fragmented)) { 1037 /* 1038 * Increment the counter manually to cope with the hardware 1039 * counter overflow. 1040 */ 1041 ctx->iv[3] = cpu_to_be32(ctr); 1042 crypto_inc((u8 *)ctx->iv, AES_BLOCK_SIZE); 1043 } 1044 1045 if (use_dma) 1046 return atmel_aes_dma_start(dd, src, dst, datalen, 1047 atmel_aes_ctr_transfer); 1048 1049 return atmel_aes_cpu_start(dd, src, dst, datalen, 1050 atmel_aes_ctr_transfer); 1051 } 1052 1053 static int atmel_aes_ctr_start(struct atmel_aes_dev *dd) 1054 { 1055 struct atmel_aes_ctr_ctx *ctx = atmel_aes_ctr_ctx_cast(dd->ctx); 1056 struct ablkcipher_request *req = ablkcipher_request_cast(dd->areq); 1057 struct atmel_aes_reqctx *rctx = ablkcipher_request_ctx(req); 1058 int err; 1059 1060 atmel_aes_set_mode(dd, rctx); 1061 1062 err = atmel_aes_hw_init(dd); 1063 if (err) 1064 return atmel_aes_complete(dd, err); 1065 1066 memcpy(ctx->iv, req->info, AES_BLOCK_SIZE); 1067 ctx->offset = 0; 1068 dd->total = 0; 1069 return atmel_aes_ctr_transfer(dd); 1070 } 1071 1072 static int atmel_aes_crypt(struct ablkcipher_request *req, unsigned long mode) 1073 { 1074 struct atmel_aes_base_ctx *ctx; 1075 struct atmel_aes_reqctx *rctx; 1076 struct atmel_aes_dev *dd; 1077 1078 ctx = crypto_ablkcipher_ctx(crypto_ablkcipher_reqtfm(req)); 1079 switch (mode & AES_FLAGS_OPMODE_MASK) { 1080 case AES_FLAGS_CFB8: 1081 ctx->block_size = CFB8_BLOCK_SIZE; 1082 break; 1083 1084 case AES_FLAGS_CFB16: 1085 ctx->block_size = CFB16_BLOCK_SIZE; 1086 break; 1087 1088 case AES_FLAGS_CFB32: 1089 ctx->block_size = CFB32_BLOCK_SIZE; 1090 break; 1091 1092 case AES_FLAGS_CFB64: 1093 ctx->block_size = CFB64_BLOCK_SIZE; 1094 break; 1095 1096 default: 1097 ctx->block_size = AES_BLOCK_SIZE; 1098 break; 1099 } 1100 1101 dd = atmel_aes_find_dev(ctx); 1102 if (!dd) 1103 return -ENODEV; 1104 1105 rctx = ablkcipher_request_ctx(req); 1106 rctx->mode = mode; 1107 1108 return atmel_aes_handle_queue(dd, &req->base); 1109 } 1110 1111 static int atmel_aes_setkey(struct crypto_ablkcipher *tfm, const u8 *key, 1112 unsigned int keylen) 1113 { 1114 struct atmel_aes_base_ctx *ctx = crypto_ablkcipher_ctx(tfm); 1115 1116 if (keylen != AES_KEYSIZE_128 && 1117 keylen != AES_KEYSIZE_192 && 1118 keylen != AES_KEYSIZE_256) { 1119 crypto_ablkcipher_set_flags(tfm, CRYPTO_TFM_RES_BAD_KEY_LEN); 1120 return -EINVAL; 1121 } 1122 1123 memcpy(ctx->key, key, keylen); 1124 ctx->keylen = keylen; 1125 1126 return 0; 1127 } 1128 1129 static int atmel_aes_ecb_encrypt(struct ablkcipher_request *req) 1130 { 1131 return atmel_aes_crypt(req, AES_FLAGS_ECB | AES_FLAGS_ENCRYPT); 1132 } 1133 1134 static int atmel_aes_ecb_decrypt(struct ablkcipher_request *req) 1135 { 1136 return atmel_aes_crypt(req, AES_FLAGS_ECB); 1137 } 1138 1139 static int atmel_aes_cbc_encrypt(struct ablkcipher_request *req) 1140 { 1141 return atmel_aes_crypt(req, AES_FLAGS_CBC | AES_FLAGS_ENCRYPT); 1142 } 1143 1144 static int atmel_aes_cbc_decrypt(struct ablkcipher_request *req) 1145 { 1146 return atmel_aes_crypt(req, AES_FLAGS_CBC); 1147 } 1148 1149 static int atmel_aes_ofb_encrypt(struct ablkcipher_request *req) 1150 { 1151 return atmel_aes_crypt(req, AES_FLAGS_OFB | AES_FLAGS_ENCRYPT); 1152 } 1153 1154 static int atmel_aes_ofb_decrypt(struct ablkcipher_request *req) 1155 { 1156 return atmel_aes_crypt(req, AES_FLAGS_OFB); 1157 } 1158 1159 static int atmel_aes_cfb_encrypt(struct ablkcipher_request *req) 1160 { 1161 return atmel_aes_crypt(req, AES_FLAGS_CFB128 | AES_FLAGS_ENCRYPT); 1162 } 1163 1164 static int atmel_aes_cfb_decrypt(struct ablkcipher_request *req) 1165 { 1166 return atmel_aes_crypt(req, AES_FLAGS_CFB128); 1167 } 1168 1169 static int atmel_aes_cfb64_encrypt(struct ablkcipher_request *req) 1170 { 1171 return atmel_aes_crypt(req, AES_FLAGS_CFB64 | AES_FLAGS_ENCRYPT); 1172 } 1173 1174 static int atmel_aes_cfb64_decrypt(struct ablkcipher_request *req) 1175 { 1176 return atmel_aes_crypt(req, AES_FLAGS_CFB64); 1177 } 1178 1179 static int atmel_aes_cfb32_encrypt(struct ablkcipher_request *req) 1180 { 1181 return atmel_aes_crypt(req, AES_FLAGS_CFB32 | AES_FLAGS_ENCRYPT); 1182 } 1183 1184 static int atmel_aes_cfb32_decrypt(struct ablkcipher_request *req) 1185 { 1186 return atmel_aes_crypt(req, AES_FLAGS_CFB32); 1187 } 1188 1189 static int atmel_aes_cfb16_encrypt(struct ablkcipher_request *req) 1190 { 1191 return atmel_aes_crypt(req, AES_FLAGS_CFB16 | AES_FLAGS_ENCRYPT); 1192 } 1193 1194 static int atmel_aes_cfb16_decrypt(struct ablkcipher_request *req) 1195 { 1196 return atmel_aes_crypt(req, AES_FLAGS_CFB16); 1197 } 1198 1199 static int atmel_aes_cfb8_encrypt(struct ablkcipher_request *req) 1200 { 1201 return atmel_aes_crypt(req, AES_FLAGS_CFB8 | AES_FLAGS_ENCRYPT); 1202 } 1203 1204 static int atmel_aes_cfb8_decrypt(struct ablkcipher_request *req) 1205 { 1206 return atmel_aes_crypt(req, AES_FLAGS_CFB8); 1207 } 1208 1209 static int atmel_aes_ctr_encrypt(struct ablkcipher_request *req) 1210 { 1211 return atmel_aes_crypt(req, AES_FLAGS_CTR | AES_FLAGS_ENCRYPT); 1212 } 1213 1214 static int atmel_aes_ctr_decrypt(struct ablkcipher_request *req) 1215 { 1216 return atmel_aes_crypt(req, AES_FLAGS_CTR); 1217 } 1218 1219 static int atmel_aes_cra_init(struct crypto_tfm *tfm) 1220 { 1221 struct atmel_aes_ctx *ctx = crypto_tfm_ctx(tfm); 1222 1223 tfm->crt_ablkcipher.reqsize = sizeof(struct atmel_aes_reqctx); 1224 ctx->base.start = atmel_aes_start; 1225 1226 return 0; 1227 } 1228 1229 static int atmel_aes_ctr_cra_init(struct crypto_tfm *tfm) 1230 { 1231 struct atmel_aes_ctx *ctx = crypto_tfm_ctx(tfm); 1232 1233 tfm->crt_ablkcipher.reqsize = sizeof(struct atmel_aes_reqctx); 1234 ctx->base.start = atmel_aes_ctr_start; 1235 1236 return 0; 1237 } 1238 1239 static void atmel_aes_cra_exit(struct crypto_tfm *tfm) 1240 { 1241 } 1242 1243 static struct crypto_alg aes_algs[] = { 1244 { 1245 .cra_name = "ecb(aes)", 1246 .cra_driver_name = "atmel-ecb-aes", 1247 .cra_priority = ATMEL_AES_PRIORITY, 1248 .cra_flags = CRYPTO_ALG_TYPE_ABLKCIPHER | CRYPTO_ALG_ASYNC, 1249 .cra_blocksize = AES_BLOCK_SIZE, 1250 .cra_ctxsize = sizeof(struct atmel_aes_ctx), 1251 .cra_alignmask = 0xf, 1252 .cra_type = &crypto_ablkcipher_type, 1253 .cra_module = THIS_MODULE, 1254 .cra_init = atmel_aes_cra_init, 1255 .cra_exit = atmel_aes_cra_exit, 1256 .cra_u.ablkcipher = { 1257 .min_keysize = AES_MIN_KEY_SIZE, 1258 .max_keysize = AES_MAX_KEY_SIZE, 1259 .setkey = atmel_aes_setkey, 1260 .encrypt = atmel_aes_ecb_encrypt, 1261 .decrypt = atmel_aes_ecb_decrypt, 1262 } 1263 }, 1264 { 1265 .cra_name = "cbc(aes)", 1266 .cra_driver_name = "atmel-cbc-aes", 1267 .cra_priority = ATMEL_AES_PRIORITY, 1268 .cra_flags = CRYPTO_ALG_TYPE_ABLKCIPHER | CRYPTO_ALG_ASYNC, 1269 .cra_blocksize = AES_BLOCK_SIZE, 1270 .cra_ctxsize = sizeof(struct atmel_aes_ctx), 1271 .cra_alignmask = 0xf, 1272 .cra_type = &crypto_ablkcipher_type, 1273 .cra_module = THIS_MODULE, 1274 .cra_init = atmel_aes_cra_init, 1275 .cra_exit = atmel_aes_cra_exit, 1276 .cra_u.ablkcipher = { 1277 .min_keysize = AES_MIN_KEY_SIZE, 1278 .max_keysize = AES_MAX_KEY_SIZE, 1279 .ivsize = AES_BLOCK_SIZE, 1280 .setkey = atmel_aes_setkey, 1281 .encrypt = atmel_aes_cbc_encrypt, 1282 .decrypt = atmel_aes_cbc_decrypt, 1283 } 1284 }, 1285 { 1286 .cra_name = "ofb(aes)", 1287 .cra_driver_name = "atmel-ofb-aes", 1288 .cra_priority = ATMEL_AES_PRIORITY, 1289 .cra_flags = CRYPTO_ALG_TYPE_ABLKCIPHER | CRYPTO_ALG_ASYNC, 1290 .cra_blocksize = AES_BLOCK_SIZE, 1291 .cra_ctxsize = sizeof(struct atmel_aes_ctx), 1292 .cra_alignmask = 0xf, 1293 .cra_type = &crypto_ablkcipher_type, 1294 .cra_module = THIS_MODULE, 1295 .cra_init = atmel_aes_cra_init, 1296 .cra_exit = atmel_aes_cra_exit, 1297 .cra_u.ablkcipher = { 1298 .min_keysize = AES_MIN_KEY_SIZE, 1299 .max_keysize = AES_MAX_KEY_SIZE, 1300 .ivsize = AES_BLOCK_SIZE, 1301 .setkey = atmel_aes_setkey, 1302 .encrypt = atmel_aes_ofb_encrypt, 1303 .decrypt = atmel_aes_ofb_decrypt, 1304 } 1305 }, 1306 { 1307 .cra_name = "cfb(aes)", 1308 .cra_driver_name = "atmel-cfb-aes", 1309 .cra_priority = ATMEL_AES_PRIORITY, 1310 .cra_flags = CRYPTO_ALG_TYPE_ABLKCIPHER | CRYPTO_ALG_ASYNC, 1311 .cra_blocksize = AES_BLOCK_SIZE, 1312 .cra_ctxsize = sizeof(struct atmel_aes_ctx), 1313 .cra_alignmask = 0xf, 1314 .cra_type = &crypto_ablkcipher_type, 1315 .cra_module = THIS_MODULE, 1316 .cra_init = atmel_aes_cra_init, 1317 .cra_exit = atmel_aes_cra_exit, 1318 .cra_u.ablkcipher = { 1319 .min_keysize = AES_MIN_KEY_SIZE, 1320 .max_keysize = AES_MAX_KEY_SIZE, 1321 .ivsize = AES_BLOCK_SIZE, 1322 .setkey = atmel_aes_setkey, 1323 .encrypt = atmel_aes_cfb_encrypt, 1324 .decrypt = atmel_aes_cfb_decrypt, 1325 } 1326 }, 1327 { 1328 .cra_name = "cfb32(aes)", 1329 .cra_driver_name = "atmel-cfb32-aes", 1330 .cra_priority = ATMEL_AES_PRIORITY, 1331 .cra_flags = CRYPTO_ALG_TYPE_ABLKCIPHER | CRYPTO_ALG_ASYNC, 1332 .cra_blocksize = CFB32_BLOCK_SIZE, 1333 .cra_ctxsize = sizeof(struct atmel_aes_ctx), 1334 .cra_alignmask = 0x3, 1335 .cra_type = &crypto_ablkcipher_type, 1336 .cra_module = THIS_MODULE, 1337 .cra_init = atmel_aes_cra_init, 1338 .cra_exit = atmel_aes_cra_exit, 1339 .cra_u.ablkcipher = { 1340 .min_keysize = AES_MIN_KEY_SIZE, 1341 .max_keysize = AES_MAX_KEY_SIZE, 1342 .ivsize = AES_BLOCK_SIZE, 1343 .setkey = atmel_aes_setkey, 1344 .encrypt = atmel_aes_cfb32_encrypt, 1345 .decrypt = atmel_aes_cfb32_decrypt, 1346 } 1347 }, 1348 { 1349 .cra_name = "cfb16(aes)", 1350 .cra_driver_name = "atmel-cfb16-aes", 1351 .cra_priority = ATMEL_AES_PRIORITY, 1352 .cra_flags = CRYPTO_ALG_TYPE_ABLKCIPHER | CRYPTO_ALG_ASYNC, 1353 .cra_blocksize = CFB16_BLOCK_SIZE, 1354 .cra_ctxsize = sizeof(struct atmel_aes_ctx), 1355 .cra_alignmask = 0x1, 1356 .cra_type = &crypto_ablkcipher_type, 1357 .cra_module = THIS_MODULE, 1358 .cra_init = atmel_aes_cra_init, 1359 .cra_exit = atmel_aes_cra_exit, 1360 .cra_u.ablkcipher = { 1361 .min_keysize = AES_MIN_KEY_SIZE, 1362 .max_keysize = AES_MAX_KEY_SIZE, 1363 .ivsize = AES_BLOCK_SIZE, 1364 .setkey = atmel_aes_setkey, 1365 .encrypt = atmel_aes_cfb16_encrypt, 1366 .decrypt = atmel_aes_cfb16_decrypt, 1367 } 1368 }, 1369 { 1370 .cra_name = "cfb8(aes)", 1371 .cra_driver_name = "atmel-cfb8-aes", 1372 .cra_priority = ATMEL_AES_PRIORITY, 1373 .cra_flags = CRYPTO_ALG_TYPE_ABLKCIPHER | CRYPTO_ALG_ASYNC, 1374 .cra_blocksize = CFB8_BLOCK_SIZE, 1375 .cra_ctxsize = sizeof(struct atmel_aes_ctx), 1376 .cra_alignmask = 0x0, 1377 .cra_type = &crypto_ablkcipher_type, 1378 .cra_module = THIS_MODULE, 1379 .cra_init = atmel_aes_cra_init, 1380 .cra_exit = atmel_aes_cra_exit, 1381 .cra_u.ablkcipher = { 1382 .min_keysize = AES_MIN_KEY_SIZE, 1383 .max_keysize = AES_MAX_KEY_SIZE, 1384 .ivsize = AES_BLOCK_SIZE, 1385 .setkey = atmel_aes_setkey, 1386 .encrypt = atmel_aes_cfb8_encrypt, 1387 .decrypt = atmel_aes_cfb8_decrypt, 1388 } 1389 }, 1390 { 1391 .cra_name = "ctr(aes)", 1392 .cra_driver_name = "atmel-ctr-aes", 1393 .cra_priority = ATMEL_AES_PRIORITY, 1394 .cra_flags = CRYPTO_ALG_TYPE_ABLKCIPHER | CRYPTO_ALG_ASYNC, 1395 .cra_blocksize = 1, 1396 .cra_ctxsize = sizeof(struct atmel_aes_ctr_ctx), 1397 .cra_alignmask = 0xf, 1398 .cra_type = &crypto_ablkcipher_type, 1399 .cra_module = THIS_MODULE, 1400 .cra_init = atmel_aes_ctr_cra_init, 1401 .cra_exit = atmel_aes_cra_exit, 1402 .cra_u.ablkcipher = { 1403 .min_keysize = AES_MIN_KEY_SIZE, 1404 .max_keysize = AES_MAX_KEY_SIZE, 1405 .ivsize = AES_BLOCK_SIZE, 1406 .setkey = atmel_aes_setkey, 1407 .encrypt = atmel_aes_ctr_encrypt, 1408 .decrypt = atmel_aes_ctr_decrypt, 1409 } 1410 }, 1411 }; 1412 1413 static struct crypto_alg aes_cfb64_alg = { 1414 .cra_name = "cfb64(aes)", 1415 .cra_driver_name = "atmel-cfb64-aes", 1416 .cra_priority = ATMEL_AES_PRIORITY, 1417 .cra_flags = CRYPTO_ALG_TYPE_ABLKCIPHER | CRYPTO_ALG_ASYNC, 1418 .cra_blocksize = CFB64_BLOCK_SIZE, 1419 .cra_ctxsize = sizeof(struct atmel_aes_ctx), 1420 .cra_alignmask = 0x7, 1421 .cra_type = &crypto_ablkcipher_type, 1422 .cra_module = THIS_MODULE, 1423 .cra_init = atmel_aes_cra_init, 1424 .cra_exit = atmel_aes_cra_exit, 1425 .cra_u.ablkcipher = { 1426 .min_keysize = AES_MIN_KEY_SIZE, 1427 .max_keysize = AES_MAX_KEY_SIZE, 1428 .ivsize = AES_BLOCK_SIZE, 1429 .setkey = atmel_aes_setkey, 1430 .encrypt = atmel_aes_cfb64_encrypt, 1431 .decrypt = atmel_aes_cfb64_decrypt, 1432 } 1433 }; 1434 1435 1436 /* gcm aead functions */ 1437 1438 static int atmel_aes_gcm_ghash(struct atmel_aes_dev *dd, 1439 const u32 *data, size_t datalen, 1440 const u32 *ghash_in, u32 *ghash_out, 1441 atmel_aes_fn_t resume); 1442 static int atmel_aes_gcm_ghash_init(struct atmel_aes_dev *dd); 1443 static int atmel_aes_gcm_ghash_finalize(struct atmel_aes_dev *dd); 1444 1445 static int atmel_aes_gcm_start(struct atmel_aes_dev *dd); 1446 static int atmel_aes_gcm_process(struct atmel_aes_dev *dd); 1447 static int atmel_aes_gcm_length(struct atmel_aes_dev *dd); 1448 static int atmel_aes_gcm_data(struct atmel_aes_dev *dd); 1449 static int atmel_aes_gcm_tag_init(struct atmel_aes_dev *dd); 1450 static int atmel_aes_gcm_tag(struct atmel_aes_dev *dd); 1451 static int atmel_aes_gcm_finalize(struct atmel_aes_dev *dd); 1452 1453 static inline struct atmel_aes_gcm_ctx * 1454 atmel_aes_gcm_ctx_cast(struct atmel_aes_base_ctx *ctx) 1455 { 1456 return container_of(ctx, struct atmel_aes_gcm_ctx, base); 1457 } 1458 1459 static int atmel_aes_gcm_ghash(struct atmel_aes_dev *dd, 1460 const u32 *data, size_t datalen, 1461 const u32 *ghash_in, u32 *ghash_out, 1462 atmel_aes_fn_t resume) 1463 { 1464 struct atmel_aes_gcm_ctx *ctx = atmel_aes_gcm_ctx_cast(dd->ctx); 1465 1466 dd->data = (u32 *)data; 1467 dd->datalen = datalen; 1468 ctx->ghash_in = ghash_in; 1469 ctx->ghash_out = ghash_out; 1470 ctx->ghash_resume = resume; 1471 1472 atmel_aes_write_ctrl(dd, false, NULL); 1473 return atmel_aes_wait_for_data_ready(dd, atmel_aes_gcm_ghash_init); 1474 } 1475 1476 static int atmel_aes_gcm_ghash_init(struct atmel_aes_dev *dd) 1477 { 1478 struct atmel_aes_gcm_ctx *ctx = atmel_aes_gcm_ctx_cast(dd->ctx); 1479 1480 /* Set the data length. */ 1481 atmel_aes_write(dd, AES_AADLENR, dd->total); 1482 atmel_aes_write(dd, AES_CLENR, 0); 1483 1484 /* If needed, overwrite the GCM Intermediate Hash Word Registers */ 1485 if (ctx->ghash_in) 1486 atmel_aes_write_block(dd, AES_GHASHR(0), ctx->ghash_in); 1487 1488 return atmel_aes_gcm_ghash_finalize(dd); 1489 } 1490 1491 static int atmel_aes_gcm_ghash_finalize(struct atmel_aes_dev *dd) 1492 { 1493 struct atmel_aes_gcm_ctx *ctx = atmel_aes_gcm_ctx_cast(dd->ctx); 1494 u32 isr; 1495 1496 /* Write data into the Input Data Registers. */ 1497 while (dd->datalen > 0) { 1498 atmel_aes_write_block(dd, AES_IDATAR(0), dd->data); 1499 dd->data += 4; 1500 dd->datalen -= AES_BLOCK_SIZE; 1501 1502 isr = atmel_aes_read(dd, AES_ISR); 1503 if (!(isr & AES_INT_DATARDY)) { 1504 dd->resume = atmel_aes_gcm_ghash_finalize; 1505 atmel_aes_write(dd, AES_IER, AES_INT_DATARDY); 1506 return -EINPROGRESS; 1507 } 1508 } 1509 1510 /* Read the computed hash from GHASHRx. */ 1511 atmel_aes_read_block(dd, AES_GHASHR(0), ctx->ghash_out); 1512 1513 return ctx->ghash_resume(dd); 1514 } 1515 1516 1517 static int atmel_aes_gcm_start(struct atmel_aes_dev *dd) 1518 { 1519 struct atmel_aes_gcm_ctx *ctx = atmel_aes_gcm_ctx_cast(dd->ctx); 1520 struct aead_request *req = aead_request_cast(dd->areq); 1521 struct crypto_aead *tfm = crypto_aead_reqtfm(req); 1522 struct atmel_aes_reqctx *rctx = aead_request_ctx(req); 1523 size_t ivsize = crypto_aead_ivsize(tfm); 1524 size_t datalen, padlen; 1525 const void *iv = req->iv; 1526 u8 *data = dd->buf; 1527 int err; 1528 1529 atmel_aes_set_mode(dd, rctx); 1530 1531 err = atmel_aes_hw_init(dd); 1532 if (err) 1533 return atmel_aes_complete(dd, err); 1534 1535 if (likely(ivsize == 12)) { 1536 memcpy(ctx->j0, iv, ivsize); 1537 ctx->j0[3] = cpu_to_be32(1); 1538 return atmel_aes_gcm_process(dd); 1539 } 1540 1541 padlen = atmel_aes_padlen(ivsize, AES_BLOCK_SIZE); 1542 datalen = ivsize + padlen + AES_BLOCK_SIZE; 1543 if (datalen > dd->buflen) 1544 return atmel_aes_complete(dd, -EINVAL); 1545 1546 memcpy(data, iv, ivsize); 1547 memset(data + ivsize, 0, padlen + sizeof(u64)); 1548 ((u64 *)(data + datalen))[-1] = cpu_to_be64(ivsize * 8); 1549 1550 return atmel_aes_gcm_ghash(dd, (const u32 *)data, datalen, 1551 NULL, ctx->j0, atmel_aes_gcm_process); 1552 } 1553 1554 static int atmel_aes_gcm_process(struct atmel_aes_dev *dd) 1555 { 1556 struct atmel_aes_gcm_ctx *ctx = atmel_aes_gcm_ctx_cast(dd->ctx); 1557 struct aead_request *req = aead_request_cast(dd->areq); 1558 struct crypto_aead *tfm = crypto_aead_reqtfm(req); 1559 bool enc = atmel_aes_is_encrypt(dd); 1560 u32 authsize; 1561 1562 /* Compute text length. */ 1563 authsize = crypto_aead_authsize(tfm); 1564 ctx->textlen = req->cryptlen - (enc ? 0 : authsize); 1565 1566 /* 1567 * According to tcrypt test suite, the GCM Automatic Tag Generation 1568 * fails when both the message and its associated data are empty. 1569 */ 1570 if (likely(req->assoclen != 0 || ctx->textlen != 0)) 1571 dd->flags |= AES_FLAGS_GTAGEN; 1572 1573 atmel_aes_write_ctrl(dd, false, NULL); 1574 return atmel_aes_wait_for_data_ready(dd, atmel_aes_gcm_length); 1575 } 1576 1577 static int atmel_aes_gcm_length(struct atmel_aes_dev *dd) 1578 { 1579 struct atmel_aes_gcm_ctx *ctx = atmel_aes_gcm_ctx_cast(dd->ctx); 1580 struct aead_request *req = aead_request_cast(dd->areq); 1581 u32 j0_lsw, *j0 = ctx->j0; 1582 size_t padlen; 1583 1584 /* Write incr32(J0) into IV. */ 1585 j0_lsw = j0[3]; 1586 j0[3] = cpu_to_be32(be32_to_cpu(j0[3]) + 1); 1587 atmel_aes_write_block(dd, AES_IVR(0), j0); 1588 j0[3] = j0_lsw; 1589 1590 /* Set aad and text lengths. */ 1591 atmel_aes_write(dd, AES_AADLENR, req->assoclen); 1592 atmel_aes_write(dd, AES_CLENR, ctx->textlen); 1593 1594 /* Check whether AAD are present. */ 1595 if (unlikely(req->assoclen == 0)) { 1596 dd->datalen = 0; 1597 return atmel_aes_gcm_data(dd); 1598 } 1599 1600 /* Copy assoc data and add padding. */ 1601 padlen = atmel_aes_padlen(req->assoclen, AES_BLOCK_SIZE); 1602 if (unlikely(req->assoclen + padlen > dd->buflen)) 1603 return atmel_aes_complete(dd, -EINVAL); 1604 sg_copy_to_buffer(req->src, sg_nents(req->src), dd->buf, req->assoclen); 1605 1606 /* Write assoc data into the Input Data register. */ 1607 dd->data = (u32 *)dd->buf; 1608 dd->datalen = req->assoclen + padlen; 1609 return atmel_aes_gcm_data(dd); 1610 } 1611 1612 static int atmel_aes_gcm_data(struct atmel_aes_dev *dd) 1613 { 1614 struct atmel_aes_gcm_ctx *ctx = atmel_aes_gcm_ctx_cast(dd->ctx); 1615 struct aead_request *req = aead_request_cast(dd->areq); 1616 bool use_dma = (ctx->textlen >= ATMEL_AES_DMA_THRESHOLD); 1617 struct scatterlist *src, *dst; 1618 u32 isr, mr; 1619 1620 /* Write AAD first. */ 1621 while (dd->datalen > 0) { 1622 atmel_aes_write_block(dd, AES_IDATAR(0), dd->data); 1623 dd->data += 4; 1624 dd->datalen -= AES_BLOCK_SIZE; 1625 1626 isr = atmel_aes_read(dd, AES_ISR); 1627 if (!(isr & AES_INT_DATARDY)) { 1628 dd->resume = atmel_aes_gcm_data; 1629 atmel_aes_write(dd, AES_IER, AES_INT_DATARDY); 1630 return -EINPROGRESS; 1631 } 1632 } 1633 1634 /* GMAC only. */ 1635 if (unlikely(ctx->textlen == 0)) 1636 return atmel_aes_gcm_tag_init(dd); 1637 1638 /* Prepare src and dst scatter lists to transfer cipher/plain texts */ 1639 src = scatterwalk_ffwd(ctx->src, req->src, req->assoclen); 1640 dst = ((req->src == req->dst) ? src : 1641 scatterwalk_ffwd(ctx->dst, req->dst, req->assoclen)); 1642 1643 if (use_dma) { 1644 /* Update the Mode Register for DMA transfers. */ 1645 mr = atmel_aes_read(dd, AES_MR); 1646 mr &= ~(AES_MR_SMOD_MASK | AES_MR_DUALBUFF); 1647 mr |= AES_MR_SMOD_IDATAR0; 1648 if (dd->caps.has_dualbuff) 1649 mr |= AES_MR_DUALBUFF; 1650 atmel_aes_write(dd, AES_MR, mr); 1651 1652 return atmel_aes_dma_start(dd, src, dst, ctx->textlen, 1653 atmel_aes_gcm_tag_init); 1654 } 1655 1656 return atmel_aes_cpu_start(dd, src, dst, ctx->textlen, 1657 atmel_aes_gcm_tag_init); 1658 } 1659 1660 static int atmel_aes_gcm_tag_init(struct atmel_aes_dev *dd) 1661 { 1662 struct atmel_aes_gcm_ctx *ctx = atmel_aes_gcm_ctx_cast(dd->ctx); 1663 struct aead_request *req = aead_request_cast(dd->areq); 1664 u64 *data = dd->buf; 1665 1666 if (likely(dd->flags & AES_FLAGS_GTAGEN)) { 1667 if (!(atmel_aes_read(dd, AES_ISR) & AES_INT_TAGRDY)) { 1668 dd->resume = atmel_aes_gcm_tag_init; 1669 atmel_aes_write(dd, AES_IER, AES_INT_TAGRDY); 1670 return -EINPROGRESS; 1671 } 1672 1673 return atmel_aes_gcm_finalize(dd); 1674 } 1675 1676 /* Read the GCM Intermediate Hash Word Registers. */ 1677 atmel_aes_read_block(dd, AES_GHASHR(0), ctx->ghash); 1678 1679 data[0] = cpu_to_be64(req->assoclen * 8); 1680 data[1] = cpu_to_be64(ctx->textlen * 8); 1681 1682 return atmel_aes_gcm_ghash(dd, (const u32 *)data, AES_BLOCK_SIZE, 1683 ctx->ghash, ctx->ghash, atmel_aes_gcm_tag); 1684 } 1685 1686 static int atmel_aes_gcm_tag(struct atmel_aes_dev *dd) 1687 { 1688 struct atmel_aes_gcm_ctx *ctx = atmel_aes_gcm_ctx_cast(dd->ctx); 1689 unsigned long flags; 1690 1691 /* 1692 * Change mode to CTR to complete the tag generation. 1693 * Use J0 as Initialization Vector. 1694 */ 1695 flags = dd->flags; 1696 dd->flags &= ~(AES_FLAGS_OPMODE_MASK | AES_FLAGS_GTAGEN); 1697 dd->flags |= AES_FLAGS_CTR; 1698 atmel_aes_write_ctrl(dd, false, ctx->j0); 1699 dd->flags = flags; 1700 1701 atmel_aes_write_block(dd, AES_IDATAR(0), ctx->ghash); 1702 return atmel_aes_wait_for_data_ready(dd, atmel_aes_gcm_finalize); 1703 } 1704 1705 static int atmel_aes_gcm_finalize(struct atmel_aes_dev *dd) 1706 { 1707 struct atmel_aes_gcm_ctx *ctx = atmel_aes_gcm_ctx_cast(dd->ctx); 1708 struct aead_request *req = aead_request_cast(dd->areq); 1709 struct crypto_aead *tfm = crypto_aead_reqtfm(req); 1710 bool enc = atmel_aes_is_encrypt(dd); 1711 u32 offset, authsize, itag[4], *otag = ctx->tag; 1712 int err; 1713 1714 /* Read the computed tag. */ 1715 if (likely(dd->flags & AES_FLAGS_GTAGEN)) 1716 atmel_aes_read_block(dd, AES_TAGR(0), ctx->tag); 1717 else 1718 atmel_aes_read_block(dd, AES_ODATAR(0), ctx->tag); 1719 1720 offset = req->assoclen + ctx->textlen; 1721 authsize = crypto_aead_authsize(tfm); 1722 if (enc) { 1723 scatterwalk_map_and_copy(otag, req->dst, offset, authsize, 1); 1724 err = 0; 1725 } else { 1726 scatterwalk_map_and_copy(itag, req->src, offset, authsize, 0); 1727 err = crypto_memneq(itag, otag, authsize) ? -EBADMSG : 0; 1728 } 1729 1730 return atmel_aes_complete(dd, err); 1731 } 1732 1733 static int atmel_aes_gcm_crypt(struct aead_request *req, 1734 unsigned long mode) 1735 { 1736 struct atmel_aes_base_ctx *ctx; 1737 struct atmel_aes_reqctx *rctx; 1738 struct atmel_aes_dev *dd; 1739 1740 ctx = crypto_aead_ctx(crypto_aead_reqtfm(req)); 1741 ctx->block_size = AES_BLOCK_SIZE; 1742 1743 dd = atmel_aes_find_dev(ctx); 1744 if (!dd) 1745 return -ENODEV; 1746 1747 rctx = aead_request_ctx(req); 1748 rctx->mode = AES_FLAGS_GCM | mode; 1749 1750 return atmel_aes_handle_queue(dd, &req->base); 1751 } 1752 1753 static int atmel_aes_gcm_setkey(struct crypto_aead *tfm, const u8 *key, 1754 unsigned int keylen) 1755 { 1756 struct atmel_aes_base_ctx *ctx = crypto_aead_ctx(tfm); 1757 1758 if (keylen != AES_KEYSIZE_256 && 1759 keylen != AES_KEYSIZE_192 && 1760 keylen != AES_KEYSIZE_128) { 1761 crypto_aead_set_flags(tfm, CRYPTO_TFM_RES_BAD_KEY_LEN); 1762 return -EINVAL; 1763 } 1764 1765 memcpy(ctx->key, key, keylen); 1766 ctx->keylen = keylen; 1767 1768 return 0; 1769 } 1770 1771 static int atmel_aes_gcm_setauthsize(struct crypto_aead *tfm, 1772 unsigned int authsize) 1773 { 1774 /* Same as crypto_gcm_authsize() from crypto/gcm.c */ 1775 switch (authsize) { 1776 case 4: 1777 case 8: 1778 case 12: 1779 case 13: 1780 case 14: 1781 case 15: 1782 case 16: 1783 break; 1784 default: 1785 return -EINVAL; 1786 } 1787 1788 return 0; 1789 } 1790 1791 static int atmel_aes_gcm_encrypt(struct aead_request *req) 1792 { 1793 return atmel_aes_gcm_crypt(req, AES_FLAGS_ENCRYPT); 1794 } 1795 1796 static int atmel_aes_gcm_decrypt(struct aead_request *req) 1797 { 1798 return atmel_aes_gcm_crypt(req, 0); 1799 } 1800 1801 static int atmel_aes_gcm_init(struct crypto_aead *tfm) 1802 { 1803 struct atmel_aes_gcm_ctx *ctx = crypto_aead_ctx(tfm); 1804 1805 crypto_aead_set_reqsize(tfm, sizeof(struct atmel_aes_reqctx)); 1806 ctx->base.start = atmel_aes_gcm_start; 1807 1808 return 0; 1809 } 1810 1811 static void atmel_aes_gcm_exit(struct crypto_aead *tfm) 1812 { 1813 1814 } 1815 1816 static struct aead_alg aes_gcm_alg = { 1817 .setkey = atmel_aes_gcm_setkey, 1818 .setauthsize = atmel_aes_gcm_setauthsize, 1819 .encrypt = atmel_aes_gcm_encrypt, 1820 .decrypt = atmel_aes_gcm_decrypt, 1821 .init = atmel_aes_gcm_init, 1822 .exit = atmel_aes_gcm_exit, 1823 .ivsize = 12, 1824 .maxauthsize = AES_BLOCK_SIZE, 1825 1826 .base = { 1827 .cra_name = "gcm(aes)", 1828 .cra_driver_name = "atmel-gcm-aes", 1829 .cra_priority = ATMEL_AES_PRIORITY, 1830 .cra_flags = CRYPTO_ALG_ASYNC, 1831 .cra_blocksize = 1, 1832 .cra_ctxsize = sizeof(struct atmel_aes_gcm_ctx), 1833 .cra_alignmask = 0xf, 1834 .cra_module = THIS_MODULE, 1835 }, 1836 }; 1837 1838 1839 /* xts functions */ 1840 1841 static inline struct atmel_aes_xts_ctx * 1842 atmel_aes_xts_ctx_cast(struct atmel_aes_base_ctx *ctx) 1843 { 1844 return container_of(ctx, struct atmel_aes_xts_ctx, base); 1845 } 1846 1847 static int atmel_aes_xts_process_data(struct atmel_aes_dev *dd); 1848 1849 static int atmel_aes_xts_start(struct atmel_aes_dev *dd) 1850 { 1851 struct atmel_aes_xts_ctx *ctx = atmel_aes_xts_ctx_cast(dd->ctx); 1852 struct ablkcipher_request *req = ablkcipher_request_cast(dd->areq); 1853 struct atmel_aes_reqctx *rctx = ablkcipher_request_ctx(req); 1854 unsigned long flags; 1855 int err; 1856 1857 atmel_aes_set_mode(dd, rctx); 1858 1859 err = atmel_aes_hw_init(dd); 1860 if (err) 1861 return atmel_aes_complete(dd, err); 1862 1863 /* Compute the tweak value from req->info with ecb(aes). */ 1864 flags = dd->flags; 1865 dd->flags &= ~AES_FLAGS_MODE_MASK; 1866 dd->flags |= (AES_FLAGS_ECB | AES_FLAGS_ENCRYPT); 1867 atmel_aes_write_ctrl_key(dd, false, NULL, 1868 ctx->key2, ctx->base.keylen); 1869 dd->flags = flags; 1870 1871 atmel_aes_write_block(dd, AES_IDATAR(0), req->info); 1872 return atmel_aes_wait_for_data_ready(dd, atmel_aes_xts_process_data); 1873 } 1874 1875 static int atmel_aes_xts_process_data(struct atmel_aes_dev *dd) 1876 { 1877 struct ablkcipher_request *req = ablkcipher_request_cast(dd->areq); 1878 bool use_dma = (req->nbytes >= ATMEL_AES_DMA_THRESHOLD); 1879 u32 tweak[AES_BLOCK_SIZE / sizeof(u32)]; 1880 static const u32 one[AES_BLOCK_SIZE / sizeof(u32)] = {cpu_to_le32(1), }; 1881 u8 *tweak_bytes = (u8 *)tweak; 1882 int i; 1883 1884 /* Read the computed ciphered tweak value. */ 1885 atmel_aes_read_block(dd, AES_ODATAR(0), tweak); 1886 /* 1887 * Hardware quirk: 1888 * the order of the ciphered tweak bytes need to be reversed before 1889 * writing them into the ODATARx registers. 1890 */ 1891 for (i = 0; i < AES_BLOCK_SIZE/2; ++i) { 1892 u8 tmp = tweak_bytes[AES_BLOCK_SIZE - 1 - i]; 1893 1894 tweak_bytes[AES_BLOCK_SIZE - 1 - i] = tweak_bytes[i]; 1895 tweak_bytes[i] = tmp; 1896 } 1897 1898 /* Process the data. */ 1899 atmel_aes_write_ctrl(dd, use_dma, NULL); 1900 atmel_aes_write_block(dd, AES_TWR(0), tweak); 1901 atmel_aes_write_block(dd, AES_ALPHAR(0), one); 1902 if (use_dma) 1903 return atmel_aes_dma_start(dd, req->src, req->dst, req->nbytes, 1904 atmel_aes_transfer_complete); 1905 1906 return atmel_aes_cpu_start(dd, req->src, req->dst, req->nbytes, 1907 atmel_aes_transfer_complete); 1908 } 1909 1910 static int atmel_aes_xts_setkey(struct crypto_ablkcipher *tfm, const u8 *key, 1911 unsigned int keylen) 1912 { 1913 struct atmel_aes_xts_ctx *ctx = crypto_ablkcipher_ctx(tfm); 1914 int err; 1915 1916 err = xts_check_key(crypto_ablkcipher_tfm(tfm), key, keylen); 1917 if (err) 1918 return err; 1919 1920 memcpy(ctx->base.key, key, keylen/2); 1921 memcpy(ctx->key2, key + keylen/2, keylen/2); 1922 ctx->base.keylen = keylen/2; 1923 1924 return 0; 1925 } 1926 1927 static int atmel_aes_xts_encrypt(struct ablkcipher_request *req) 1928 { 1929 return atmel_aes_crypt(req, AES_FLAGS_XTS | AES_FLAGS_ENCRYPT); 1930 } 1931 1932 static int atmel_aes_xts_decrypt(struct ablkcipher_request *req) 1933 { 1934 return atmel_aes_crypt(req, AES_FLAGS_XTS); 1935 } 1936 1937 static int atmel_aes_xts_cra_init(struct crypto_tfm *tfm) 1938 { 1939 struct atmel_aes_xts_ctx *ctx = crypto_tfm_ctx(tfm); 1940 1941 tfm->crt_ablkcipher.reqsize = sizeof(struct atmel_aes_reqctx); 1942 ctx->base.start = atmel_aes_xts_start; 1943 1944 return 0; 1945 } 1946 1947 static struct crypto_alg aes_xts_alg = { 1948 .cra_name = "xts(aes)", 1949 .cra_driver_name = "atmel-xts-aes", 1950 .cra_priority = ATMEL_AES_PRIORITY, 1951 .cra_flags = CRYPTO_ALG_TYPE_ABLKCIPHER | CRYPTO_ALG_ASYNC, 1952 .cra_blocksize = AES_BLOCK_SIZE, 1953 .cra_ctxsize = sizeof(struct atmel_aes_xts_ctx), 1954 .cra_alignmask = 0xf, 1955 .cra_type = &crypto_ablkcipher_type, 1956 .cra_module = THIS_MODULE, 1957 .cra_init = atmel_aes_xts_cra_init, 1958 .cra_exit = atmel_aes_cra_exit, 1959 .cra_u.ablkcipher = { 1960 .min_keysize = 2 * AES_MIN_KEY_SIZE, 1961 .max_keysize = 2 * AES_MAX_KEY_SIZE, 1962 .ivsize = AES_BLOCK_SIZE, 1963 .setkey = atmel_aes_xts_setkey, 1964 .encrypt = atmel_aes_xts_encrypt, 1965 .decrypt = atmel_aes_xts_decrypt, 1966 } 1967 }; 1968 1969 #ifdef CONFIG_CRYPTO_DEV_ATMEL_AUTHENC 1970 /* authenc aead functions */ 1971 1972 static int atmel_aes_authenc_start(struct atmel_aes_dev *dd); 1973 static int atmel_aes_authenc_init(struct atmel_aes_dev *dd, int err, 1974 bool is_async); 1975 static int atmel_aes_authenc_transfer(struct atmel_aes_dev *dd, int err, 1976 bool is_async); 1977 static int atmel_aes_authenc_digest(struct atmel_aes_dev *dd); 1978 static int atmel_aes_authenc_final(struct atmel_aes_dev *dd, int err, 1979 bool is_async); 1980 1981 static void atmel_aes_authenc_complete(struct atmel_aes_dev *dd, int err) 1982 { 1983 struct aead_request *req = aead_request_cast(dd->areq); 1984 struct atmel_aes_authenc_reqctx *rctx = aead_request_ctx(req); 1985 1986 if (err && (dd->flags & AES_FLAGS_OWN_SHA)) 1987 atmel_sha_authenc_abort(&rctx->auth_req); 1988 dd->flags &= ~AES_FLAGS_OWN_SHA; 1989 } 1990 1991 static int atmel_aes_authenc_start(struct atmel_aes_dev *dd) 1992 { 1993 struct aead_request *req = aead_request_cast(dd->areq); 1994 struct atmel_aes_authenc_reqctx *rctx = aead_request_ctx(req); 1995 struct crypto_aead *tfm = crypto_aead_reqtfm(req); 1996 struct atmel_aes_authenc_ctx *ctx = crypto_aead_ctx(tfm); 1997 int err; 1998 1999 atmel_aes_set_mode(dd, &rctx->base); 2000 2001 err = atmel_aes_hw_init(dd); 2002 if (err) 2003 return atmel_aes_complete(dd, err); 2004 2005 return atmel_sha_authenc_schedule(&rctx->auth_req, ctx->auth, 2006 atmel_aes_authenc_init, dd); 2007 } 2008 2009 static int atmel_aes_authenc_init(struct atmel_aes_dev *dd, int err, 2010 bool is_async) 2011 { 2012 struct aead_request *req = aead_request_cast(dd->areq); 2013 struct atmel_aes_authenc_reqctx *rctx = aead_request_ctx(req); 2014 2015 if (is_async) 2016 dd->is_async = true; 2017 if (err) 2018 return atmel_aes_complete(dd, err); 2019 2020 /* If here, we've got the ownership of the SHA device. */ 2021 dd->flags |= AES_FLAGS_OWN_SHA; 2022 2023 /* Configure the SHA device. */ 2024 return atmel_sha_authenc_init(&rctx->auth_req, 2025 req->src, req->assoclen, 2026 rctx->textlen, 2027 atmel_aes_authenc_transfer, dd); 2028 } 2029 2030 static int atmel_aes_authenc_transfer(struct atmel_aes_dev *dd, int err, 2031 bool is_async) 2032 { 2033 struct aead_request *req = aead_request_cast(dd->areq); 2034 struct atmel_aes_authenc_reqctx *rctx = aead_request_ctx(req); 2035 bool enc = atmel_aes_is_encrypt(dd); 2036 struct scatterlist *src, *dst; 2037 u32 iv[AES_BLOCK_SIZE / sizeof(u32)]; 2038 u32 emr; 2039 2040 if (is_async) 2041 dd->is_async = true; 2042 if (err) 2043 return atmel_aes_complete(dd, err); 2044 2045 /* Prepare src and dst scatter-lists to transfer cipher/plain texts. */ 2046 src = scatterwalk_ffwd(rctx->src, req->src, req->assoclen); 2047 dst = src; 2048 2049 if (req->src != req->dst) 2050 dst = scatterwalk_ffwd(rctx->dst, req->dst, req->assoclen); 2051 2052 /* Configure the AES device. */ 2053 memcpy(iv, req->iv, sizeof(iv)); 2054 2055 /* 2056 * Here we always set the 2nd parameter of atmel_aes_write_ctrl() to 2057 * 'true' even if the data transfer is actually performed by the CPU (so 2058 * not by the DMA) because we must force the AES_MR_SMOD bitfield to the 2059 * value AES_MR_SMOD_IDATAR0. Indeed, both AES_MR_SMOD and SHA_MR_SMOD 2060 * must be set to *_MR_SMOD_IDATAR0. 2061 */ 2062 atmel_aes_write_ctrl(dd, true, iv); 2063 emr = AES_EMR_PLIPEN; 2064 if (!enc) 2065 emr |= AES_EMR_PLIPD; 2066 atmel_aes_write(dd, AES_EMR, emr); 2067 2068 /* Transfer data. */ 2069 return atmel_aes_dma_start(dd, src, dst, rctx->textlen, 2070 atmel_aes_authenc_digest); 2071 } 2072 2073 static int atmel_aes_authenc_digest(struct atmel_aes_dev *dd) 2074 { 2075 struct aead_request *req = aead_request_cast(dd->areq); 2076 struct atmel_aes_authenc_reqctx *rctx = aead_request_ctx(req); 2077 2078 /* atmel_sha_authenc_final() releases the SHA device. */ 2079 dd->flags &= ~AES_FLAGS_OWN_SHA; 2080 return atmel_sha_authenc_final(&rctx->auth_req, 2081 rctx->digest, sizeof(rctx->digest), 2082 atmel_aes_authenc_final, dd); 2083 } 2084 2085 static int atmel_aes_authenc_final(struct atmel_aes_dev *dd, int err, 2086 bool is_async) 2087 { 2088 struct aead_request *req = aead_request_cast(dd->areq); 2089 struct atmel_aes_authenc_reqctx *rctx = aead_request_ctx(req); 2090 struct crypto_aead *tfm = crypto_aead_reqtfm(req); 2091 bool enc = atmel_aes_is_encrypt(dd); 2092 u32 idigest[SHA512_DIGEST_SIZE / sizeof(u32)], *odigest = rctx->digest; 2093 u32 offs, authsize; 2094 2095 if (is_async) 2096 dd->is_async = true; 2097 if (err) 2098 goto complete; 2099 2100 offs = req->assoclen + rctx->textlen; 2101 authsize = crypto_aead_authsize(tfm); 2102 if (enc) { 2103 scatterwalk_map_and_copy(odigest, req->dst, offs, authsize, 1); 2104 } else { 2105 scatterwalk_map_and_copy(idigest, req->src, offs, authsize, 0); 2106 if (crypto_memneq(idigest, odigest, authsize)) 2107 err = -EBADMSG; 2108 } 2109 2110 complete: 2111 return atmel_aes_complete(dd, err); 2112 } 2113 2114 static int atmel_aes_authenc_setkey(struct crypto_aead *tfm, const u8 *key, 2115 unsigned int keylen) 2116 { 2117 struct atmel_aes_authenc_ctx *ctx = crypto_aead_ctx(tfm); 2118 struct crypto_authenc_keys keys; 2119 u32 flags; 2120 int err; 2121 2122 if (crypto_authenc_extractkeys(&keys, key, keylen) != 0) 2123 goto badkey; 2124 2125 if (keys.enckeylen > sizeof(ctx->base.key)) 2126 goto badkey; 2127 2128 /* Save auth key. */ 2129 flags = crypto_aead_get_flags(tfm); 2130 err = atmel_sha_authenc_setkey(ctx->auth, 2131 keys.authkey, keys.authkeylen, 2132 &flags); 2133 crypto_aead_set_flags(tfm, flags & CRYPTO_TFM_RES_MASK); 2134 if (err) { 2135 memzero_explicit(&keys, sizeof(keys)); 2136 return err; 2137 } 2138 2139 /* Save enc key. */ 2140 ctx->base.keylen = keys.enckeylen; 2141 memcpy(ctx->base.key, keys.enckey, keys.enckeylen); 2142 2143 memzero_explicit(&keys, sizeof(keys)); 2144 return 0; 2145 2146 badkey: 2147 crypto_aead_set_flags(tfm, CRYPTO_TFM_RES_BAD_KEY_LEN); 2148 memzero_explicit(&key, sizeof(keys)); 2149 return -EINVAL; 2150 } 2151 2152 static int atmel_aes_authenc_init_tfm(struct crypto_aead *tfm, 2153 unsigned long auth_mode) 2154 { 2155 struct atmel_aes_authenc_ctx *ctx = crypto_aead_ctx(tfm); 2156 unsigned int auth_reqsize = atmel_sha_authenc_get_reqsize(); 2157 2158 ctx->auth = atmel_sha_authenc_spawn(auth_mode); 2159 if (IS_ERR(ctx->auth)) 2160 return PTR_ERR(ctx->auth); 2161 2162 crypto_aead_set_reqsize(tfm, (sizeof(struct atmel_aes_authenc_reqctx) + 2163 auth_reqsize)); 2164 ctx->base.start = atmel_aes_authenc_start; 2165 2166 return 0; 2167 } 2168 2169 static int atmel_aes_authenc_hmac_sha1_init_tfm(struct crypto_aead *tfm) 2170 { 2171 return atmel_aes_authenc_init_tfm(tfm, SHA_FLAGS_HMAC_SHA1); 2172 } 2173 2174 static int atmel_aes_authenc_hmac_sha224_init_tfm(struct crypto_aead *tfm) 2175 { 2176 return atmel_aes_authenc_init_tfm(tfm, SHA_FLAGS_HMAC_SHA224); 2177 } 2178 2179 static int atmel_aes_authenc_hmac_sha256_init_tfm(struct crypto_aead *tfm) 2180 { 2181 return atmel_aes_authenc_init_tfm(tfm, SHA_FLAGS_HMAC_SHA256); 2182 } 2183 2184 static int atmel_aes_authenc_hmac_sha384_init_tfm(struct crypto_aead *tfm) 2185 { 2186 return atmel_aes_authenc_init_tfm(tfm, SHA_FLAGS_HMAC_SHA384); 2187 } 2188 2189 static int atmel_aes_authenc_hmac_sha512_init_tfm(struct crypto_aead *tfm) 2190 { 2191 return atmel_aes_authenc_init_tfm(tfm, SHA_FLAGS_HMAC_SHA512); 2192 } 2193 2194 static void atmel_aes_authenc_exit_tfm(struct crypto_aead *tfm) 2195 { 2196 struct atmel_aes_authenc_ctx *ctx = crypto_aead_ctx(tfm); 2197 2198 atmel_sha_authenc_free(ctx->auth); 2199 } 2200 2201 static int atmel_aes_authenc_crypt(struct aead_request *req, 2202 unsigned long mode) 2203 { 2204 struct atmel_aes_authenc_reqctx *rctx = aead_request_ctx(req); 2205 struct crypto_aead *tfm = crypto_aead_reqtfm(req); 2206 struct atmel_aes_base_ctx *ctx = crypto_aead_ctx(tfm); 2207 u32 authsize = crypto_aead_authsize(tfm); 2208 bool enc = (mode & AES_FLAGS_ENCRYPT); 2209 struct atmel_aes_dev *dd; 2210 2211 /* Compute text length. */ 2212 if (!enc && req->cryptlen < authsize) 2213 return -EINVAL; 2214 rctx->textlen = req->cryptlen - (enc ? 0 : authsize); 2215 2216 /* 2217 * Currently, empty messages are not supported yet: 2218 * the SHA auto-padding can be used only on non-empty messages. 2219 * Hence a special case needs to be implemented for empty message. 2220 */ 2221 if (!rctx->textlen && !req->assoclen) 2222 return -EINVAL; 2223 2224 rctx->base.mode = mode; 2225 ctx->block_size = AES_BLOCK_SIZE; 2226 2227 dd = atmel_aes_find_dev(ctx); 2228 if (!dd) 2229 return -ENODEV; 2230 2231 return atmel_aes_handle_queue(dd, &req->base); 2232 } 2233 2234 static int atmel_aes_authenc_cbc_aes_encrypt(struct aead_request *req) 2235 { 2236 return atmel_aes_authenc_crypt(req, AES_FLAGS_CBC | AES_FLAGS_ENCRYPT); 2237 } 2238 2239 static int atmel_aes_authenc_cbc_aes_decrypt(struct aead_request *req) 2240 { 2241 return atmel_aes_authenc_crypt(req, AES_FLAGS_CBC); 2242 } 2243 2244 static struct aead_alg aes_authenc_algs[] = { 2245 { 2246 .setkey = atmel_aes_authenc_setkey, 2247 .encrypt = atmel_aes_authenc_cbc_aes_encrypt, 2248 .decrypt = atmel_aes_authenc_cbc_aes_decrypt, 2249 .init = atmel_aes_authenc_hmac_sha1_init_tfm, 2250 .exit = atmel_aes_authenc_exit_tfm, 2251 .ivsize = AES_BLOCK_SIZE, 2252 .maxauthsize = SHA1_DIGEST_SIZE, 2253 2254 .base = { 2255 .cra_name = "authenc(hmac(sha1),cbc(aes))", 2256 .cra_driver_name = "atmel-authenc-hmac-sha1-cbc-aes", 2257 .cra_priority = ATMEL_AES_PRIORITY, 2258 .cra_flags = CRYPTO_ALG_ASYNC, 2259 .cra_blocksize = AES_BLOCK_SIZE, 2260 .cra_ctxsize = sizeof(struct atmel_aes_authenc_ctx), 2261 .cra_alignmask = 0xf, 2262 .cra_module = THIS_MODULE, 2263 }, 2264 }, 2265 { 2266 .setkey = atmel_aes_authenc_setkey, 2267 .encrypt = atmel_aes_authenc_cbc_aes_encrypt, 2268 .decrypt = atmel_aes_authenc_cbc_aes_decrypt, 2269 .init = atmel_aes_authenc_hmac_sha224_init_tfm, 2270 .exit = atmel_aes_authenc_exit_tfm, 2271 .ivsize = AES_BLOCK_SIZE, 2272 .maxauthsize = SHA224_DIGEST_SIZE, 2273 2274 .base = { 2275 .cra_name = "authenc(hmac(sha224),cbc(aes))", 2276 .cra_driver_name = "atmel-authenc-hmac-sha224-cbc-aes", 2277 .cra_priority = ATMEL_AES_PRIORITY, 2278 .cra_flags = CRYPTO_ALG_ASYNC, 2279 .cra_blocksize = AES_BLOCK_SIZE, 2280 .cra_ctxsize = sizeof(struct atmel_aes_authenc_ctx), 2281 .cra_alignmask = 0xf, 2282 .cra_module = THIS_MODULE, 2283 }, 2284 }, 2285 { 2286 .setkey = atmel_aes_authenc_setkey, 2287 .encrypt = atmel_aes_authenc_cbc_aes_encrypt, 2288 .decrypt = atmel_aes_authenc_cbc_aes_decrypt, 2289 .init = atmel_aes_authenc_hmac_sha256_init_tfm, 2290 .exit = atmel_aes_authenc_exit_tfm, 2291 .ivsize = AES_BLOCK_SIZE, 2292 .maxauthsize = SHA256_DIGEST_SIZE, 2293 2294 .base = { 2295 .cra_name = "authenc(hmac(sha256),cbc(aes))", 2296 .cra_driver_name = "atmel-authenc-hmac-sha256-cbc-aes", 2297 .cra_priority = ATMEL_AES_PRIORITY, 2298 .cra_flags = CRYPTO_ALG_ASYNC, 2299 .cra_blocksize = AES_BLOCK_SIZE, 2300 .cra_ctxsize = sizeof(struct atmel_aes_authenc_ctx), 2301 .cra_alignmask = 0xf, 2302 .cra_module = THIS_MODULE, 2303 }, 2304 }, 2305 { 2306 .setkey = atmel_aes_authenc_setkey, 2307 .encrypt = atmel_aes_authenc_cbc_aes_encrypt, 2308 .decrypt = atmel_aes_authenc_cbc_aes_decrypt, 2309 .init = atmel_aes_authenc_hmac_sha384_init_tfm, 2310 .exit = atmel_aes_authenc_exit_tfm, 2311 .ivsize = AES_BLOCK_SIZE, 2312 .maxauthsize = SHA384_DIGEST_SIZE, 2313 2314 .base = { 2315 .cra_name = "authenc(hmac(sha384),cbc(aes))", 2316 .cra_driver_name = "atmel-authenc-hmac-sha384-cbc-aes", 2317 .cra_priority = ATMEL_AES_PRIORITY, 2318 .cra_flags = CRYPTO_ALG_ASYNC, 2319 .cra_blocksize = AES_BLOCK_SIZE, 2320 .cra_ctxsize = sizeof(struct atmel_aes_authenc_ctx), 2321 .cra_alignmask = 0xf, 2322 .cra_module = THIS_MODULE, 2323 }, 2324 }, 2325 { 2326 .setkey = atmel_aes_authenc_setkey, 2327 .encrypt = atmel_aes_authenc_cbc_aes_encrypt, 2328 .decrypt = atmel_aes_authenc_cbc_aes_decrypt, 2329 .init = atmel_aes_authenc_hmac_sha512_init_tfm, 2330 .exit = atmel_aes_authenc_exit_tfm, 2331 .ivsize = AES_BLOCK_SIZE, 2332 .maxauthsize = SHA512_DIGEST_SIZE, 2333 2334 .base = { 2335 .cra_name = "authenc(hmac(sha512),cbc(aes))", 2336 .cra_driver_name = "atmel-authenc-hmac-sha512-cbc-aes", 2337 .cra_priority = ATMEL_AES_PRIORITY, 2338 .cra_flags = CRYPTO_ALG_ASYNC, 2339 .cra_blocksize = AES_BLOCK_SIZE, 2340 .cra_ctxsize = sizeof(struct atmel_aes_authenc_ctx), 2341 .cra_alignmask = 0xf, 2342 .cra_module = THIS_MODULE, 2343 }, 2344 }, 2345 }; 2346 #endif /* CONFIG_CRYPTO_DEV_ATMEL_AUTHENC */ 2347 2348 /* Probe functions */ 2349 2350 static int atmel_aes_buff_init(struct atmel_aes_dev *dd) 2351 { 2352 dd->buf = (void *)__get_free_pages(GFP_KERNEL, ATMEL_AES_BUFFER_ORDER); 2353 dd->buflen = ATMEL_AES_BUFFER_SIZE; 2354 dd->buflen &= ~(AES_BLOCK_SIZE - 1); 2355 2356 if (!dd->buf) { 2357 dev_err(dd->dev, "unable to alloc pages.\n"); 2358 return -ENOMEM; 2359 } 2360 2361 return 0; 2362 } 2363 2364 static void atmel_aes_buff_cleanup(struct atmel_aes_dev *dd) 2365 { 2366 free_page((unsigned long)dd->buf); 2367 } 2368 2369 static bool atmel_aes_filter(struct dma_chan *chan, void *slave) 2370 { 2371 struct at_dma_slave *sl = slave; 2372 2373 if (sl && sl->dma_dev == chan->device->dev) { 2374 chan->private = sl; 2375 return true; 2376 } else { 2377 return false; 2378 } 2379 } 2380 2381 static int atmel_aes_dma_init(struct atmel_aes_dev *dd, 2382 struct crypto_platform_data *pdata) 2383 { 2384 struct at_dma_slave *slave; 2385 int err = -ENOMEM; 2386 dma_cap_mask_t mask; 2387 2388 dma_cap_zero(mask); 2389 dma_cap_set(DMA_SLAVE, mask); 2390 2391 /* Try to grab 2 DMA channels */ 2392 slave = &pdata->dma_slave->rxdata; 2393 dd->src.chan = dma_request_slave_channel_compat(mask, atmel_aes_filter, 2394 slave, dd->dev, "tx"); 2395 if (!dd->src.chan) 2396 goto err_dma_in; 2397 2398 slave = &pdata->dma_slave->txdata; 2399 dd->dst.chan = dma_request_slave_channel_compat(mask, atmel_aes_filter, 2400 slave, dd->dev, "rx"); 2401 if (!dd->dst.chan) 2402 goto err_dma_out; 2403 2404 return 0; 2405 2406 err_dma_out: 2407 dma_release_channel(dd->src.chan); 2408 err_dma_in: 2409 dev_warn(dd->dev, "no DMA channel available\n"); 2410 return err; 2411 } 2412 2413 static void atmel_aes_dma_cleanup(struct atmel_aes_dev *dd) 2414 { 2415 dma_release_channel(dd->dst.chan); 2416 dma_release_channel(dd->src.chan); 2417 } 2418 2419 static void atmel_aes_queue_task(unsigned long data) 2420 { 2421 struct atmel_aes_dev *dd = (struct atmel_aes_dev *)data; 2422 2423 atmel_aes_handle_queue(dd, NULL); 2424 } 2425 2426 static void atmel_aes_done_task(unsigned long data) 2427 { 2428 struct atmel_aes_dev *dd = (struct atmel_aes_dev *)data; 2429 2430 dd->is_async = true; 2431 (void)dd->resume(dd); 2432 } 2433 2434 static irqreturn_t atmel_aes_irq(int irq, void *dev_id) 2435 { 2436 struct atmel_aes_dev *aes_dd = dev_id; 2437 u32 reg; 2438 2439 reg = atmel_aes_read(aes_dd, AES_ISR); 2440 if (reg & atmel_aes_read(aes_dd, AES_IMR)) { 2441 atmel_aes_write(aes_dd, AES_IDR, reg); 2442 if (AES_FLAGS_BUSY & aes_dd->flags) 2443 tasklet_schedule(&aes_dd->done_task); 2444 else 2445 dev_warn(aes_dd->dev, "AES interrupt when no active requests.\n"); 2446 return IRQ_HANDLED; 2447 } 2448 2449 return IRQ_NONE; 2450 } 2451 2452 static void atmel_aes_unregister_algs(struct atmel_aes_dev *dd) 2453 { 2454 int i; 2455 2456 #ifdef CONFIG_CRYPTO_DEV_ATMEL_AUTHENC 2457 if (dd->caps.has_authenc) 2458 for (i = 0; i < ARRAY_SIZE(aes_authenc_algs); i++) 2459 crypto_unregister_aead(&aes_authenc_algs[i]); 2460 #endif 2461 2462 if (dd->caps.has_xts) 2463 crypto_unregister_alg(&aes_xts_alg); 2464 2465 if (dd->caps.has_gcm) 2466 crypto_unregister_aead(&aes_gcm_alg); 2467 2468 if (dd->caps.has_cfb64) 2469 crypto_unregister_alg(&aes_cfb64_alg); 2470 2471 for (i = 0; i < ARRAY_SIZE(aes_algs); i++) 2472 crypto_unregister_alg(&aes_algs[i]); 2473 } 2474 2475 static int atmel_aes_register_algs(struct atmel_aes_dev *dd) 2476 { 2477 int err, i, j; 2478 2479 for (i = 0; i < ARRAY_SIZE(aes_algs); i++) { 2480 err = crypto_register_alg(&aes_algs[i]); 2481 if (err) 2482 goto err_aes_algs; 2483 } 2484 2485 if (dd->caps.has_cfb64) { 2486 err = crypto_register_alg(&aes_cfb64_alg); 2487 if (err) 2488 goto err_aes_cfb64_alg; 2489 } 2490 2491 if (dd->caps.has_gcm) { 2492 err = crypto_register_aead(&aes_gcm_alg); 2493 if (err) 2494 goto err_aes_gcm_alg; 2495 } 2496 2497 if (dd->caps.has_xts) { 2498 err = crypto_register_alg(&aes_xts_alg); 2499 if (err) 2500 goto err_aes_xts_alg; 2501 } 2502 2503 #ifdef CONFIG_CRYPTO_DEV_ATMEL_AUTHENC 2504 if (dd->caps.has_authenc) { 2505 for (i = 0; i < ARRAY_SIZE(aes_authenc_algs); i++) { 2506 err = crypto_register_aead(&aes_authenc_algs[i]); 2507 if (err) 2508 goto err_aes_authenc_alg; 2509 } 2510 } 2511 #endif 2512 2513 return 0; 2514 2515 #ifdef CONFIG_CRYPTO_DEV_ATMEL_AUTHENC 2516 /* i = ARRAY_SIZE(aes_authenc_algs); */ 2517 err_aes_authenc_alg: 2518 for (j = 0; j < i; j++) 2519 crypto_unregister_aead(&aes_authenc_algs[j]); 2520 crypto_unregister_alg(&aes_xts_alg); 2521 #endif 2522 err_aes_xts_alg: 2523 crypto_unregister_aead(&aes_gcm_alg); 2524 err_aes_gcm_alg: 2525 crypto_unregister_alg(&aes_cfb64_alg); 2526 err_aes_cfb64_alg: 2527 i = ARRAY_SIZE(aes_algs); 2528 err_aes_algs: 2529 for (j = 0; j < i; j++) 2530 crypto_unregister_alg(&aes_algs[j]); 2531 2532 return err; 2533 } 2534 2535 static void atmel_aes_get_cap(struct atmel_aes_dev *dd) 2536 { 2537 dd->caps.has_dualbuff = 0; 2538 dd->caps.has_cfb64 = 0; 2539 dd->caps.has_ctr32 = 0; 2540 dd->caps.has_gcm = 0; 2541 dd->caps.has_xts = 0; 2542 dd->caps.has_authenc = 0; 2543 dd->caps.max_burst_size = 1; 2544 2545 /* keep only major version number */ 2546 switch (dd->hw_version & 0xff0) { 2547 case 0x500: 2548 dd->caps.has_dualbuff = 1; 2549 dd->caps.has_cfb64 = 1; 2550 dd->caps.has_ctr32 = 1; 2551 dd->caps.has_gcm = 1; 2552 dd->caps.has_xts = 1; 2553 dd->caps.has_authenc = 1; 2554 dd->caps.max_burst_size = 4; 2555 break; 2556 case 0x200: 2557 dd->caps.has_dualbuff = 1; 2558 dd->caps.has_cfb64 = 1; 2559 dd->caps.has_ctr32 = 1; 2560 dd->caps.has_gcm = 1; 2561 dd->caps.max_burst_size = 4; 2562 break; 2563 case 0x130: 2564 dd->caps.has_dualbuff = 1; 2565 dd->caps.has_cfb64 = 1; 2566 dd->caps.max_burst_size = 4; 2567 break; 2568 case 0x120: 2569 break; 2570 default: 2571 dev_warn(dd->dev, 2572 "Unmanaged aes version, set minimum capabilities\n"); 2573 break; 2574 } 2575 } 2576 2577 #if defined(CONFIG_OF) 2578 static const struct of_device_id atmel_aes_dt_ids[] = { 2579 { .compatible = "atmel,at91sam9g46-aes" }, 2580 { /* sentinel */ } 2581 }; 2582 MODULE_DEVICE_TABLE(of, atmel_aes_dt_ids); 2583 2584 static struct crypto_platform_data *atmel_aes_of_init(struct platform_device *pdev) 2585 { 2586 struct device_node *np = pdev->dev.of_node; 2587 struct crypto_platform_data *pdata; 2588 2589 if (!np) { 2590 dev_err(&pdev->dev, "device node not found\n"); 2591 return ERR_PTR(-EINVAL); 2592 } 2593 2594 pdata = devm_kzalloc(&pdev->dev, sizeof(*pdata), GFP_KERNEL); 2595 if (!pdata) { 2596 dev_err(&pdev->dev, "could not allocate memory for pdata\n"); 2597 return ERR_PTR(-ENOMEM); 2598 } 2599 2600 pdata->dma_slave = devm_kzalloc(&pdev->dev, 2601 sizeof(*(pdata->dma_slave)), 2602 GFP_KERNEL); 2603 if (!pdata->dma_slave) { 2604 dev_err(&pdev->dev, "could not allocate memory for dma_slave\n"); 2605 devm_kfree(&pdev->dev, pdata); 2606 return ERR_PTR(-ENOMEM); 2607 } 2608 2609 return pdata; 2610 } 2611 #else 2612 static inline struct crypto_platform_data *atmel_aes_of_init(struct platform_device *pdev) 2613 { 2614 return ERR_PTR(-EINVAL); 2615 } 2616 #endif 2617 2618 static int atmel_aes_probe(struct platform_device *pdev) 2619 { 2620 struct atmel_aes_dev *aes_dd; 2621 struct crypto_platform_data *pdata; 2622 struct device *dev = &pdev->dev; 2623 struct resource *aes_res; 2624 int err; 2625 2626 pdata = pdev->dev.platform_data; 2627 if (!pdata) { 2628 pdata = atmel_aes_of_init(pdev); 2629 if (IS_ERR(pdata)) { 2630 err = PTR_ERR(pdata); 2631 goto aes_dd_err; 2632 } 2633 } 2634 2635 if (!pdata->dma_slave) { 2636 err = -ENXIO; 2637 goto aes_dd_err; 2638 } 2639 2640 aes_dd = devm_kzalloc(&pdev->dev, sizeof(*aes_dd), GFP_KERNEL); 2641 if (aes_dd == NULL) { 2642 dev_err(dev, "unable to alloc data struct.\n"); 2643 err = -ENOMEM; 2644 goto aes_dd_err; 2645 } 2646 2647 aes_dd->dev = dev; 2648 2649 platform_set_drvdata(pdev, aes_dd); 2650 2651 INIT_LIST_HEAD(&aes_dd->list); 2652 spin_lock_init(&aes_dd->lock); 2653 2654 tasklet_init(&aes_dd->done_task, atmel_aes_done_task, 2655 (unsigned long)aes_dd); 2656 tasklet_init(&aes_dd->queue_task, atmel_aes_queue_task, 2657 (unsigned long)aes_dd); 2658 2659 crypto_init_queue(&aes_dd->queue, ATMEL_AES_QUEUE_LENGTH); 2660 2661 aes_dd->irq = -1; 2662 2663 /* Get the base address */ 2664 aes_res = platform_get_resource(pdev, IORESOURCE_MEM, 0); 2665 if (!aes_res) { 2666 dev_err(dev, "no MEM resource info\n"); 2667 err = -ENODEV; 2668 goto res_err; 2669 } 2670 aes_dd->phys_base = aes_res->start; 2671 2672 /* Get the IRQ */ 2673 aes_dd->irq = platform_get_irq(pdev, 0); 2674 if (aes_dd->irq < 0) { 2675 dev_err(dev, "no IRQ resource info\n"); 2676 err = aes_dd->irq; 2677 goto res_err; 2678 } 2679 2680 err = devm_request_irq(&pdev->dev, aes_dd->irq, atmel_aes_irq, 2681 IRQF_SHARED, "atmel-aes", aes_dd); 2682 if (err) { 2683 dev_err(dev, "unable to request aes irq.\n"); 2684 goto res_err; 2685 } 2686 2687 /* Initializing the clock */ 2688 aes_dd->iclk = devm_clk_get(&pdev->dev, "aes_clk"); 2689 if (IS_ERR(aes_dd->iclk)) { 2690 dev_err(dev, "clock initialization failed.\n"); 2691 err = PTR_ERR(aes_dd->iclk); 2692 goto res_err; 2693 } 2694 2695 aes_dd->io_base = devm_ioremap_resource(&pdev->dev, aes_res); 2696 if (IS_ERR(aes_dd->io_base)) { 2697 dev_err(dev, "can't ioremap\n"); 2698 err = PTR_ERR(aes_dd->io_base); 2699 goto res_err; 2700 } 2701 2702 err = clk_prepare(aes_dd->iclk); 2703 if (err) 2704 goto res_err; 2705 2706 err = atmel_aes_hw_version_init(aes_dd); 2707 if (err) 2708 goto iclk_unprepare; 2709 2710 atmel_aes_get_cap(aes_dd); 2711 2712 #ifdef CONFIG_CRYPTO_DEV_ATMEL_AUTHENC 2713 if (aes_dd->caps.has_authenc && !atmel_sha_authenc_is_ready()) { 2714 err = -EPROBE_DEFER; 2715 goto iclk_unprepare; 2716 } 2717 #endif 2718 2719 err = atmel_aes_buff_init(aes_dd); 2720 if (err) 2721 goto err_aes_buff; 2722 2723 err = atmel_aes_dma_init(aes_dd, pdata); 2724 if (err) 2725 goto err_aes_dma; 2726 2727 spin_lock(&atmel_aes.lock); 2728 list_add_tail(&aes_dd->list, &atmel_aes.dev_list); 2729 spin_unlock(&atmel_aes.lock); 2730 2731 err = atmel_aes_register_algs(aes_dd); 2732 if (err) 2733 goto err_algs; 2734 2735 dev_info(dev, "Atmel AES - Using %s, %s for DMA transfers\n", 2736 dma_chan_name(aes_dd->src.chan), 2737 dma_chan_name(aes_dd->dst.chan)); 2738 2739 return 0; 2740 2741 err_algs: 2742 spin_lock(&atmel_aes.lock); 2743 list_del(&aes_dd->list); 2744 spin_unlock(&atmel_aes.lock); 2745 atmel_aes_dma_cleanup(aes_dd); 2746 err_aes_dma: 2747 atmel_aes_buff_cleanup(aes_dd); 2748 err_aes_buff: 2749 iclk_unprepare: 2750 clk_unprepare(aes_dd->iclk); 2751 res_err: 2752 tasklet_kill(&aes_dd->done_task); 2753 tasklet_kill(&aes_dd->queue_task); 2754 aes_dd_err: 2755 if (err != -EPROBE_DEFER) 2756 dev_err(dev, "initialization failed.\n"); 2757 2758 return err; 2759 } 2760 2761 static int atmel_aes_remove(struct platform_device *pdev) 2762 { 2763 struct atmel_aes_dev *aes_dd; 2764 2765 aes_dd = platform_get_drvdata(pdev); 2766 if (!aes_dd) 2767 return -ENODEV; 2768 spin_lock(&atmel_aes.lock); 2769 list_del(&aes_dd->list); 2770 spin_unlock(&atmel_aes.lock); 2771 2772 atmel_aes_unregister_algs(aes_dd); 2773 2774 tasklet_kill(&aes_dd->done_task); 2775 tasklet_kill(&aes_dd->queue_task); 2776 2777 atmel_aes_dma_cleanup(aes_dd); 2778 atmel_aes_buff_cleanup(aes_dd); 2779 2780 clk_unprepare(aes_dd->iclk); 2781 2782 return 0; 2783 } 2784 2785 static struct platform_driver atmel_aes_driver = { 2786 .probe = atmel_aes_probe, 2787 .remove = atmel_aes_remove, 2788 .driver = { 2789 .name = "atmel_aes", 2790 .of_match_table = of_match_ptr(atmel_aes_dt_ids), 2791 }, 2792 }; 2793 2794 module_platform_driver(atmel_aes_driver); 2795 2796 MODULE_DESCRIPTION("Atmel AES hw acceleration support."); 2797 MODULE_LICENSE("GPL v2"); 2798 MODULE_AUTHOR("Nicolas Royer - Eukréa Electromatique"); 2799