1 // SPDX-License-Identifier: GPL-2.0-only 2 /* 3 * Copyright (C) STMicroelectronics SA 2017 4 * Author: Fabien Dessenne <fabien.dessenne@st.com> 5 */ 6 7 #include <linux/clk.h> 8 #include <linux/delay.h> 9 #include <linux/interrupt.h> 10 #include <linux/iopoll.h> 11 #include <linux/module.h> 12 #include <linux/of_device.h> 13 #include <linux/platform_device.h> 14 #include <linux/pm_runtime.h> 15 #include <linux/reset.h> 16 17 #include <crypto/aes.h> 18 #include <crypto/internal/des.h> 19 #include <crypto/engine.h> 20 #include <crypto/scatterwalk.h> 21 #include <crypto/internal/aead.h> 22 #include <crypto/internal/skcipher.h> 23 24 #define DRIVER_NAME "stm32-cryp" 25 26 /* Bit [0] encrypt / decrypt */ 27 #define FLG_ENCRYPT BIT(0) 28 /* Bit [8..1] algo & operation mode */ 29 #define FLG_AES BIT(1) 30 #define FLG_DES BIT(2) 31 #define FLG_TDES BIT(3) 32 #define FLG_ECB BIT(4) 33 #define FLG_CBC BIT(5) 34 #define FLG_CTR BIT(6) 35 #define FLG_GCM BIT(7) 36 #define FLG_CCM BIT(8) 37 /* Mode mask = bits [15..0] */ 38 #define FLG_MODE_MASK GENMASK(15, 0) 39 /* Bit [31..16] status */ 40 #define FLG_CCM_PADDED_WA BIT(16) 41 42 /* Registers */ 43 #define CRYP_CR 0x00000000 44 #define CRYP_SR 0x00000004 45 #define CRYP_DIN 0x00000008 46 #define CRYP_DOUT 0x0000000C 47 #define CRYP_DMACR 0x00000010 48 #define CRYP_IMSCR 0x00000014 49 #define CRYP_RISR 0x00000018 50 #define CRYP_MISR 0x0000001C 51 #define CRYP_K0LR 0x00000020 52 #define CRYP_K0RR 0x00000024 53 #define CRYP_K1LR 0x00000028 54 #define CRYP_K1RR 0x0000002C 55 #define CRYP_K2LR 0x00000030 56 #define CRYP_K2RR 0x00000034 57 #define CRYP_K3LR 0x00000038 58 #define CRYP_K3RR 0x0000003C 59 #define CRYP_IV0LR 0x00000040 60 #define CRYP_IV0RR 0x00000044 61 #define CRYP_IV1LR 0x00000048 62 #define CRYP_IV1RR 0x0000004C 63 #define CRYP_CSGCMCCM0R 0x00000050 64 #define CRYP_CSGCM0R 0x00000070 65 66 /* Registers values */ 67 #define CR_DEC_NOT_ENC 0x00000004 68 #define CR_TDES_ECB 0x00000000 69 #define CR_TDES_CBC 0x00000008 70 #define CR_DES_ECB 0x00000010 71 #define CR_DES_CBC 0x00000018 72 #define CR_AES_ECB 0x00000020 73 #define CR_AES_CBC 0x00000028 74 #define CR_AES_CTR 0x00000030 75 #define CR_AES_KP 0x00000038 76 #define CR_AES_GCM 0x00080000 77 #define CR_AES_CCM 0x00080008 78 #define CR_AES_UNKNOWN 0xFFFFFFFF 79 #define CR_ALGO_MASK 0x00080038 80 #define CR_DATA32 0x00000000 81 #define CR_DATA16 0x00000040 82 #define CR_DATA8 0x00000080 83 #define CR_DATA1 0x000000C0 84 #define CR_KEY128 0x00000000 85 #define CR_KEY192 0x00000100 86 #define CR_KEY256 0x00000200 87 #define CR_FFLUSH 0x00004000 88 #define CR_CRYPEN 0x00008000 89 #define CR_PH_INIT 0x00000000 90 #define CR_PH_HEADER 0x00010000 91 #define CR_PH_PAYLOAD 0x00020000 92 #define CR_PH_FINAL 0x00030000 93 #define CR_PH_MASK 0x00030000 94 #define CR_NBPBL_SHIFT 20 95 96 #define SR_BUSY 0x00000010 97 #define SR_OFNE 0x00000004 98 99 #define IMSCR_IN BIT(0) 100 #define IMSCR_OUT BIT(1) 101 102 #define MISR_IN BIT(0) 103 #define MISR_OUT BIT(1) 104 105 /* Misc */ 106 #define AES_BLOCK_32 (AES_BLOCK_SIZE / sizeof(u32)) 107 #define GCM_CTR_INIT 2 108 #define _walked_in (cryp->in_walk.offset - cryp->in_sg->offset) 109 #define _walked_out (cryp->out_walk.offset - cryp->out_sg->offset) 110 #define CRYP_AUTOSUSPEND_DELAY 50 111 112 struct stm32_cryp_caps { 113 bool swap_final; 114 bool padding_wa; 115 }; 116 117 struct stm32_cryp_ctx { 118 struct crypto_engine_ctx enginectx; 119 struct stm32_cryp *cryp; 120 int keylen; 121 __be32 key[AES_KEYSIZE_256 / sizeof(u32)]; 122 unsigned long flags; 123 }; 124 125 struct stm32_cryp_reqctx { 126 unsigned long mode; 127 }; 128 129 struct stm32_cryp { 130 struct list_head list; 131 struct device *dev; 132 void __iomem *regs; 133 struct clk *clk; 134 unsigned long flags; 135 u32 irq_status; 136 const struct stm32_cryp_caps *caps; 137 struct stm32_cryp_ctx *ctx; 138 139 struct crypto_engine *engine; 140 141 struct skcipher_request *req; 142 struct aead_request *areq; 143 144 size_t authsize; 145 size_t hw_blocksize; 146 147 size_t total_in; 148 size_t total_in_save; 149 size_t total_out; 150 size_t total_out_save; 151 152 struct scatterlist *in_sg; 153 struct scatterlist *out_sg; 154 struct scatterlist *out_sg_save; 155 156 struct scatterlist in_sgl; 157 struct scatterlist out_sgl; 158 bool sgs_copied; 159 160 int in_sg_len; 161 int out_sg_len; 162 163 struct scatter_walk in_walk; 164 struct scatter_walk out_walk; 165 166 u32 last_ctr[4]; 167 u32 gcm_ctr; 168 }; 169 170 struct stm32_cryp_list { 171 struct list_head dev_list; 172 spinlock_t lock; /* protect dev_list */ 173 }; 174 175 static struct stm32_cryp_list cryp_list = { 176 .dev_list = LIST_HEAD_INIT(cryp_list.dev_list), 177 .lock = __SPIN_LOCK_UNLOCKED(cryp_list.lock), 178 }; 179 180 static inline bool is_aes(struct stm32_cryp *cryp) 181 { 182 return cryp->flags & FLG_AES; 183 } 184 185 static inline bool is_des(struct stm32_cryp *cryp) 186 { 187 return cryp->flags & FLG_DES; 188 } 189 190 static inline bool is_tdes(struct stm32_cryp *cryp) 191 { 192 return cryp->flags & FLG_TDES; 193 } 194 195 static inline bool is_ecb(struct stm32_cryp *cryp) 196 { 197 return cryp->flags & FLG_ECB; 198 } 199 200 static inline bool is_cbc(struct stm32_cryp *cryp) 201 { 202 return cryp->flags & FLG_CBC; 203 } 204 205 static inline bool is_ctr(struct stm32_cryp *cryp) 206 { 207 return cryp->flags & FLG_CTR; 208 } 209 210 static inline bool is_gcm(struct stm32_cryp *cryp) 211 { 212 return cryp->flags & FLG_GCM; 213 } 214 215 static inline bool is_ccm(struct stm32_cryp *cryp) 216 { 217 return cryp->flags & FLG_CCM; 218 } 219 220 static inline bool is_encrypt(struct stm32_cryp *cryp) 221 { 222 return cryp->flags & FLG_ENCRYPT; 223 } 224 225 static inline bool is_decrypt(struct stm32_cryp *cryp) 226 { 227 return !is_encrypt(cryp); 228 } 229 230 static inline u32 stm32_cryp_read(struct stm32_cryp *cryp, u32 ofst) 231 { 232 return readl_relaxed(cryp->regs + ofst); 233 } 234 235 static inline void stm32_cryp_write(struct stm32_cryp *cryp, u32 ofst, u32 val) 236 { 237 writel_relaxed(val, cryp->regs + ofst); 238 } 239 240 static inline int stm32_cryp_wait_busy(struct stm32_cryp *cryp) 241 { 242 u32 status; 243 244 return readl_relaxed_poll_timeout(cryp->regs + CRYP_SR, status, 245 !(status & SR_BUSY), 10, 100000); 246 } 247 248 static inline int stm32_cryp_wait_enable(struct stm32_cryp *cryp) 249 { 250 u32 status; 251 252 return readl_relaxed_poll_timeout(cryp->regs + CRYP_CR, status, 253 !(status & CR_CRYPEN), 10, 100000); 254 } 255 256 static inline int stm32_cryp_wait_output(struct stm32_cryp *cryp) 257 { 258 u32 status; 259 260 return readl_relaxed_poll_timeout(cryp->regs + CRYP_SR, status, 261 status & SR_OFNE, 10, 100000); 262 } 263 264 static int stm32_cryp_read_auth_tag(struct stm32_cryp *cryp); 265 266 static struct stm32_cryp *stm32_cryp_find_dev(struct stm32_cryp_ctx *ctx) 267 { 268 struct stm32_cryp *tmp, *cryp = NULL; 269 270 spin_lock_bh(&cryp_list.lock); 271 if (!ctx->cryp) { 272 list_for_each_entry(tmp, &cryp_list.dev_list, list) { 273 cryp = tmp; 274 break; 275 } 276 ctx->cryp = cryp; 277 } else { 278 cryp = ctx->cryp; 279 } 280 281 spin_unlock_bh(&cryp_list.lock); 282 283 return cryp; 284 } 285 286 static int stm32_cryp_check_aligned(struct scatterlist *sg, size_t total, 287 size_t align) 288 { 289 int len = 0; 290 291 if (!total) 292 return 0; 293 294 if (!IS_ALIGNED(total, align)) 295 return -EINVAL; 296 297 while (sg) { 298 if (!IS_ALIGNED(sg->offset, sizeof(u32))) 299 return -EINVAL; 300 301 if (!IS_ALIGNED(sg->length, align)) 302 return -EINVAL; 303 304 len += sg->length; 305 sg = sg_next(sg); 306 } 307 308 if (len != total) 309 return -EINVAL; 310 311 return 0; 312 } 313 314 static int stm32_cryp_check_io_aligned(struct stm32_cryp *cryp) 315 { 316 int ret; 317 318 ret = stm32_cryp_check_aligned(cryp->in_sg, cryp->total_in, 319 cryp->hw_blocksize); 320 if (ret) 321 return ret; 322 323 ret = stm32_cryp_check_aligned(cryp->out_sg, cryp->total_out, 324 cryp->hw_blocksize); 325 326 return ret; 327 } 328 329 static void sg_copy_buf(void *buf, struct scatterlist *sg, 330 unsigned int start, unsigned int nbytes, int out) 331 { 332 struct scatter_walk walk; 333 334 if (!nbytes) 335 return; 336 337 scatterwalk_start(&walk, sg); 338 scatterwalk_advance(&walk, start); 339 scatterwalk_copychunks(buf, &walk, nbytes, out); 340 scatterwalk_done(&walk, out, 0); 341 } 342 343 static int stm32_cryp_copy_sgs(struct stm32_cryp *cryp) 344 { 345 void *buf_in, *buf_out; 346 int pages, total_in, total_out; 347 348 if (!stm32_cryp_check_io_aligned(cryp)) { 349 cryp->sgs_copied = 0; 350 return 0; 351 } 352 353 total_in = ALIGN(cryp->total_in, cryp->hw_blocksize); 354 pages = total_in ? get_order(total_in) : 1; 355 buf_in = (void *)__get_free_pages(GFP_ATOMIC, pages); 356 357 total_out = ALIGN(cryp->total_out, cryp->hw_blocksize); 358 pages = total_out ? get_order(total_out) : 1; 359 buf_out = (void *)__get_free_pages(GFP_ATOMIC, pages); 360 361 if (!buf_in || !buf_out) { 362 dev_err(cryp->dev, "Can't allocate pages when unaligned\n"); 363 cryp->sgs_copied = 0; 364 return -EFAULT; 365 } 366 367 sg_copy_buf(buf_in, cryp->in_sg, 0, cryp->total_in, 0); 368 369 sg_init_one(&cryp->in_sgl, buf_in, total_in); 370 cryp->in_sg = &cryp->in_sgl; 371 cryp->in_sg_len = 1; 372 373 sg_init_one(&cryp->out_sgl, buf_out, total_out); 374 cryp->out_sg_save = cryp->out_sg; 375 cryp->out_sg = &cryp->out_sgl; 376 cryp->out_sg_len = 1; 377 378 cryp->sgs_copied = 1; 379 380 return 0; 381 } 382 383 static void stm32_cryp_hw_write_iv(struct stm32_cryp *cryp, __be32 *iv) 384 { 385 if (!iv) 386 return; 387 388 stm32_cryp_write(cryp, CRYP_IV0LR, be32_to_cpu(*iv++)); 389 stm32_cryp_write(cryp, CRYP_IV0RR, be32_to_cpu(*iv++)); 390 391 if (is_aes(cryp)) { 392 stm32_cryp_write(cryp, CRYP_IV1LR, be32_to_cpu(*iv++)); 393 stm32_cryp_write(cryp, CRYP_IV1RR, be32_to_cpu(*iv++)); 394 } 395 } 396 397 static void stm32_cryp_get_iv(struct stm32_cryp *cryp) 398 { 399 struct skcipher_request *req = cryp->req; 400 __be32 *tmp = (void *)req->iv; 401 402 if (!tmp) 403 return; 404 405 *tmp++ = cpu_to_be32(stm32_cryp_read(cryp, CRYP_IV0LR)); 406 *tmp++ = cpu_to_be32(stm32_cryp_read(cryp, CRYP_IV0RR)); 407 408 if (is_aes(cryp)) { 409 *tmp++ = cpu_to_be32(stm32_cryp_read(cryp, CRYP_IV1LR)); 410 *tmp++ = cpu_to_be32(stm32_cryp_read(cryp, CRYP_IV1RR)); 411 } 412 } 413 414 static void stm32_cryp_hw_write_key(struct stm32_cryp *c) 415 { 416 unsigned int i; 417 int r_id; 418 419 if (is_des(c)) { 420 stm32_cryp_write(c, CRYP_K1LR, be32_to_cpu(c->ctx->key[0])); 421 stm32_cryp_write(c, CRYP_K1RR, be32_to_cpu(c->ctx->key[1])); 422 } else { 423 r_id = CRYP_K3RR; 424 for (i = c->ctx->keylen / sizeof(u32); i > 0; i--, r_id -= 4) 425 stm32_cryp_write(c, r_id, 426 be32_to_cpu(c->ctx->key[i - 1])); 427 } 428 } 429 430 static u32 stm32_cryp_get_hw_mode(struct stm32_cryp *cryp) 431 { 432 if (is_aes(cryp) && is_ecb(cryp)) 433 return CR_AES_ECB; 434 435 if (is_aes(cryp) && is_cbc(cryp)) 436 return CR_AES_CBC; 437 438 if (is_aes(cryp) && is_ctr(cryp)) 439 return CR_AES_CTR; 440 441 if (is_aes(cryp) && is_gcm(cryp)) 442 return CR_AES_GCM; 443 444 if (is_aes(cryp) && is_ccm(cryp)) 445 return CR_AES_CCM; 446 447 if (is_des(cryp) && is_ecb(cryp)) 448 return CR_DES_ECB; 449 450 if (is_des(cryp) && is_cbc(cryp)) 451 return CR_DES_CBC; 452 453 if (is_tdes(cryp) && is_ecb(cryp)) 454 return CR_TDES_ECB; 455 456 if (is_tdes(cryp) && is_cbc(cryp)) 457 return CR_TDES_CBC; 458 459 dev_err(cryp->dev, "Unknown mode\n"); 460 return CR_AES_UNKNOWN; 461 } 462 463 static unsigned int stm32_cryp_get_input_text_len(struct stm32_cryp *cryp) 464 { 465 return is_encrypt(cryp) ? cryp->areq->cryptlen : 466 cryp->areq->cryptlen - cryp->authsize; 467 } 468 469 static int stm32_cryp_gcm_init(struct stm32_cryp *cryp, u32 cfg) 470 { 471 int ret; 472 __be32 iv[4]; 473 474 /* Phase 1 : init */ 475 memcpy(iv, cryp->areq->iv, 12); 476 iv[3] = cpu_to_be32(GCM_CTR_INIT); 477 cryp->gcm_ctr = GCM_CTR_INIT; 478 stm32_cryp_hw_write_iv(cryp, iv); 479 480 stm32_cryp_write(cryp, CRYP_CR, cfg | CR_PH_INIT | CR_CRYPEN); 481 482 /* Wait for end of processing */ 483 ret = stm32_cryp_wait_enable(cryp); 484 if (ret) 485 dev_err(cryp->dev, "Timeout (gcm init)\n"); 486 487 return ret; 488 } 489 490 static int stm32_cryp_ccm_init(struct stm32_cryp *cryp, u32 cfg) 491 { 492 int ret; 493 u8 iv[AES_BLOCK_SIZE], b0[AES_BLOCK_SIZE]; 494 __be32 *bd; 495 u32 *d; 496 unsigned int i, textlen; 497 498 /* Phase 1 : init. Firstly set the CTR value to 1 (not 0) */ 499 memcpy(iv, cryp->areq->iv, AES_BLOCK_SIZE); 500 memset(iv + AES_BLOCK_SIZE - 1 - iv[0], 0, iv[0] + 1); 501 iv[AES_BLOCK_SIZE - 1] = 1; 502 stm32_cryp_hw_write_iv(cryp, (__be32 *)iv); 503 504 /* Build B0 */ 505 memcpy(b0, iv, AES_BLOCK_SIZE); 506 507 b0[0] |= (8 * ((cryp->authsize - 2) / 2)); 508 509 if (cryp->areq->assoclen) 510 b0[0] |= 0x40; 511 512 textlen = stm32_cryp_get_input_text_len(cryp); 513 514 b0[AES_BLOCK_SIZE - 2] = textlen >> 8; 515 b0[AES_BLOCK_SIZE - 1] = textlen & 0xFF; 516 517 /* Enable HW */ 518 stm32_cryp_write(cryp, CRYP_CR, cfg | CR_PH_INIT | CR_CRYPEN); 519 520 /* Write B0 */ 521 d = (u32 *)b0; 522 bd = (__be32 *)b0; 523 524 for (i = 0; i < AES_BLOCK_32; i++) { 525 u32 xd = d[i]; 526 527 if (!cryp->caps->padding_wa) 528 xd = be32_to_cpu(bd[i]); 529 stm32_cryp_write(cryp, CRYP_DIN, xd); 530 } 531 532 /* Wait for end of processing */ 533 ret = stm32_cryp_wait_enable(cryp); 534 if (ret) 535 dev_err(cryp->dev, "Timeout (ccm init)\n"); 536 537 return ret; 538 } 539 540 static int stm32_cryp_hw_init(struct stm32_cryp *cryp) 541 { 542 int ret; 543 u32 cfg, hw_mode; 544 545 pm_runtime_resume_and_get(cryp->dev); 546 547 /* Disable interrupt */ 548 stm32_cryp_write(cryp, CRYP_IMSCR, 0); 549 550 /* Set key */ 551 stm32_cryp_hw_write_key(cryp); 552 553 /* Set configuration */ 554 cfg = CR_DATA8 | CR_FFLUSH; 555 556 switch (cryp->ctx->keylen) { 557 case AES_KEYSIZE_128: 558 cfg |= CR_KEY128; 559 break; 560 561 case AES_KEYSIZE_192: 562 cfg |= CR_KEY192; 563 break; 564 565 default: 566 case AES_KEYSIZE_256: 567 cfg |= CR_KEY256; 568 break; 569 } 570 571 hw_mode = stm32_cryp_get_hw_mode(cryp); 572 if (hw_mode == CR_AES_UNKNOWN) 573 return -EINVAL; 574 575 /* AES ECB/CBC decrypt: run key preparation first */ 576 if (is_decrypt(cryp) && 577 ((hw_mode == CR_AES_ECB) || (hw_mode == CR_AES_CBC))) { 578 stm32_cryp_write(cryp, CRYP_CR, cfg | CR_AES_KP | CR_CRYPEN); 579 580 /* Wait for end of processing */ 581 ret = stm32_cryp_wait_busy(cryp); 582 if (ret) { 583 dev_err(cryp->dev, "Timeout (key preparation)\n"); 584 return ret; 585 } 586 } 587 588 cfg |= hw_mode; 589 590 if (is_decrypt(cryp)) 591 cfg |= CR_DEC_NOT_ENC; 592 593 /* Apply config and flush (valid when CRYPEN = 0) */ 594 stm32_cryp_write(cryp, CRYP_CR, cfg); 595 596 switch (hw_mode) { 597 case CR_AES_GCM: 598 case CR_AES_CCM: 599 /* Phase 1 : init */ 600 if (hw_mode == CR_AES_CCM) 601 ret = stm32_cryp_ccm_init(cryp, cfg); 602 else 603 ret = stm32_cryp_gcm_init(cryp, cfg); 604 605 if (ret) 606 return ret; 607 608 /* Phase 2 : header (authenticated data) */ 609 if (cryp->areq->assoclen) { 610 cfg |= CR_PH_HEADER; 611 } else if (stm32_cryp_get_input_text_len(cryp)) { 612 cfg |= CR_PH_PAYLOAD; 613 stm32_cryp_write(cryp, CRYP_CR, cfg); 614 } else { 615 cfg |= CR_PH_INIT; 616 } 617 618 break; 619 620 case CR_DES_CBC: 621 case CR_TDES_CBC: 622 case CR_AES_CBC: 623 case CR_AES_CTR: 624 stm32_cryp_hw_write_iv(cryp, (__be32 *)cryp->req->iv); 625 break; 626 627 default: 628 break; 629 } 630 631 /* Enable now */ 632 cfg |= CR_CRYPEN; 633 634 stm32_cryp_write(cryp, CRYP_CR, cfg); 635 636 cryp->flags &= ~FLG_CCM_PADDED_WA; 637 638 return 0; 639 } 640 641 static void stm32_cryp_finish_req(struct stm32_cryp *cryp, int err) 642 { 643 if (!err && (is_gcm(cryp) || is_ccm(cryp))) 644 /* Phase 4 : output tag */ 645 err = stm32_cryp_read_auth_tag(cryp); 646 647 if (!err && (!(is_gcm(cryp) || is_ccm(cryp)))) 648 stm32_cryp_get_iv(cryp); 649 650 if (cryp->sgs_copied) { 651 void *buf_in, *buf_out; 652 int pages, len; 653 654 buf_in = sg_virt(&cryp->in_sgl); 655 buf_out = sg_virt(&cryp->out_sgl); 656 657 sg_copy_buf(buf_out, cryp->out_sg_save, 0, 658 cryp->total_out_save, 1); 659 660 len = ALIGN(cryp->total_in_save, cryp->hw_blocksize); 661 pages = len ? get_order(len) : 1; 662 free_pages((unsigned long)buf_in, pages); 663 664 len = ALIGN(cryp->total_out_save, cryp->hw_blocksize); 665 pages = len ? get_order(len) : 1; 666 free_pages((unsigned long)buf_out, pages); 667 } 668 669 pm_runtime_mark_last_busy(cryp->dev); 670 pm_runtime_put_autosuspend(cryp->dev); 671 672 if (is_gcm(cryp) || is_ccm(cryp)) 673 crypto_finalize_aead_request(cryp->engine, cryp->areq, err); 674 else 675 crypto_finalize_skcipher_request(cryp->engine, cryp->req, 676 err); 677 678 memset(cryp->ctx->key, 0, cryp->ctx->keylen); 679 } 680 681 static int stm32_cryp_cpu_start(struct stm32_cryp *cryp) 682 { 683 /* Enable interrupt and let the IRQ handler do everything */ 684 stm32_cryp_write(cryp, CRYP_IMSCR, IMSCR_IN | IMSCR_OUT); 685 686 return 0; 687 } 688 689 static int stm32_cryp_cipher_one_req(struct crypto_engine *engine, void *areq); 690 static int stm32_cryp_prepare_cipher_req(struct crypto_engine *engine, 691 void *areq); 692 693 static int stm32_cryp_init_tfm(struct crypto_skcipher *tfm) 694 { 695 struct stm32_cryp_ctx *ctx = crypto_skcipher_ctx(tfm); 696 697 crypto_skcipher_set_reqsize(tfm, sizeof(struct stm32_cryp_reqctx)); 698 699 ctx->enginectx.op.do_one_request = stm32_cryp_cipher_one_req; 700 ctx->enginectx.op.prepare_request = stm32_cryp_prepare_cipher_req; 701 ctx->enginectx.op.unprepare_request = NULL; 702 return 0; 703 } 704 705 static int stm32_cryp_aead_one_req(struct crypto_engine *engine, void *areq); 706 static int stm32_cryp_prepare_aead_req(struct crypto_engine *engine, 707 void *areq); 708 709 static int stm32_cryp_aes_aead_init(struct crypto_aead *tfm) 710 { 711 struct stm32_cryp_ctx *ctx = crypto_aead_ctx(tfm); 712 713 tfm->reqsize = sizeof(struct stm32_cryp_reqctx); 714 715 ctx->enginectx.op.do_one_request = stm32_cryp_aead_one_req; 716 ctx->enginectx.op.prepare_request = stm32_cryp_prepare_aead_req; 717 ctx->enginectx.op.unprepare_request = NULL; 718 719 return 0; 720 } 721 722 static int stm32_cryp_crypt(struct skcipher_request *req, unsigned long mode) 723 { 724 struct stm32_cryp_ctx *ctx = crypto_skcipher_ctx( 725 crypto_skcipher_reqtfm(req)); 726 struct stm32_cryp_reqctx *rctx = skcipher_request_ctx(req); 727 struct stm32_cryp *cryp = stm32_cryp_find_dev(ctx); 728 729 if (!cryp) 730 return -ENODEV; 731 732 rctx->mode = mode; 733 734 return crypto_transfer_skcipher_request_to_engine(cryp->engine, req); 735 } 736 737 static int stm32_cryp_aead_crypt(struct aead_request *req, unsigned long mode) 738 { 739 struct stm32_cryp_ctx *ctx = crypto_aead_ctx(crypto_aead_reqtfm(req)); 740 struct stm32_cryp_reqctx *rctx = aead_request_ctx(req); 741 struct stm32_cryp *cryp = stm32_cryp_find_dev(ctx); 742 743 if (!cryp) 744 return -ENODEV; 745 746 rctx->mode = mode; 747 748 return crypto_transfer_aead_request_to_engine(cryp->engine, req); 749 } 750 751 static int stm32_cryp_setkey(struct crypto_skcipher *tfm, const u8 *key, 752 unsigned int keylen) 753 { 754 struct stm32_cryp_ctx *ctx = crypto_skcipher_ctx(tfm); 755 756 memcpy(ctx->key, key, keylen); 757 ctx->keylen = keylen; 758 759 return 0; 760 } 761 762 static int stm32_cryp_aes_setkey(struct crypto_skcipher *tfm, const u8 *key, 763 unsigned int keylen) 764 { 765 if (keylen != AES_KEYSIZE_128 && keylen != AES_KEYSIZE_192 && 766 keylen != AES_KEYSIZE_256) 767 return -EINVAL; 768 else 769 return stm32_cryp_setkey(tfm, key, keylen); 770 } 771 772 static int stm32_cryp_des_setkey(struct crypto_skcipher *tfm, const u8 *key, 773 unsigned int keylen) 774 { 775 return verify_skcipher_des_key(tfm, key) ?: 776 stm32_cryp_setkey(tfm, key, keylen); 777 } 778 779 static int stm32_cryp_tdes_setkey(struct crypto_skcipher *tfm, const u8 *key, 780 unsigned int keylen) 781 { 782 return verify_skcipher_des3_key(tfm, key) ?: 783 stm32_cryp_setkey(tfm, key, keylen); 784 } 785 786 static int stm32_cryp_aes_aead_setkey(struct crypto_aead *tfm, const u8 *key, 787 unsigned int keylen) 788 { 789 struct stm32_cryp_ctx *ctx = crypto_aead_ctx(tfm); 790 791 if (keylen != AES_KEYSIZE_128 && keylen != AES_KEYSIZE_192 && 792 keylen != AES_KEYSIZE_256) 793 return -EINVAL; 794 795 memcpy(ctx->key, key, keylen); 796 ctx->keylen = keylen; 797 798 return 0; 799 } 800 801 static int stm32_cryp_aes_gcm_setauthsize(struct crypto_aead *tfm, 802 unsigned int authsize) 803 { 804 return authsize == AES_BLOCK_SIZE ? 0 : -EINVAL; 805 } 806 807 static int stm32_cryp_aes_ccm_setauthsize(struct crypto_aead *tfm, 808 unsigned int authsize) 809 { 810 switch (authsize) { 811 case 4: 812 case 6: 813 case 8: 814 case 10: 815 case 12: 816 case 14: 817 case 16: 818 break; 819 default: 820 return -EINVAL; 821 } 822 823 return 0; 824 } 825 826 static int stm32_cryp_aes_ecb_encrypt(struct skcipher_request *req) 827 { 828 return stm32_cryp_crypt(req, FLG_AES | FLG_ECB | FLG_ENCRYPT); 829 } 830 831 static int stm32_cryp_aes_ecb_decrypt(struct skcipher_request *req) 832 { 833 return stm32_cryp_crypt(req, FLG_AES | FLG_ECB); 834 } 835 836 static int stm32_cryp_aes_cbc_encrypt(struct skcipher_request *req) 837 { 838 return stm32_cryp_crypt(req, FLG_AES | FLG_CBC | FLG_ENCRYPT); 839 } 840 841 static int stm32_cryp_aes_cbc_decrypt(struct skcipher_request *req) 842 { 843 return stm32_cryp_crypt(req, FLG_AES | FLG_CBC); 844 } 845 846 static int stm32_cryp_aes_ctr_encrypt(struct skcipher_request *req) 847 { 848 return stm32_cryp_crypt(req, FLG_AES | FLG_CTR | FLG_ENCRYPT); 849 } 850 851 static int stm32_cryp_aes_ctr_decrypt(struct skcipher_request *req) 852 { 853 return stm32_cryp_crypt(req, FLG_AES | FLG_CTR); 854 } 855 856 static int stm32_cryp_aes_gcm_encrypt(struct aead_request *req) 857 { 858 return stm32_cryp_aead_crypt(req, FLG_AES | FLG_GCM | FLG_ENCRYPT); 859 } 860 861 static int stm32_cryp_aes_gcm_decrypt(struct aead_request *req) 862 { 863 return stm32_cryp_aead_crypt(req, FLG_AES | FLG_GCM); 864 } 865 866 static int stm32_cryp_aes_ccm_encrypt(struct aead_request *req) 867 { 868 return stm32_cryp_aead_crypt(req, FLG_AES | FLG_CCM | FLG_ENCRYPT); 869 } 870 871 static int stm32_cryp_aes_ccm_decrypt(struct aead_request *req) 872 { 873 return stm32_cryp_aead_crypt(req, FLG_AES | FLG_CCM); 874 } 875 876 static int stm32_cryp_des_ecb_encrypt(struct skcipher_request *req) 877 { 878 return stm32_cryp_crypt(req, FLG_DES | FLG_ECB | FLG_ENCRYPT); 879 } 880 881 static int stm32_cryp_des_ecb_decrypt(struct skcipher_request *req) 882 { 883 return stm32_cryp_crypt(req, FLG_DES | FLG_ECB); 884 } 885 886 static int stm32_cryp_des_cbc_encrypt(struct skcipher_request *req) 887 { 888 return stm32_cryp_crypt(req, FLG_DES | FLG_CBC | FLG_ENCRYPT); 889 } 890 891 static int stm32_cryp_des_cbc_decrypt(struct skcipher_request *req) 892 { 893 return stm32_cryp_crypt(req, FLG_DES | FLG_CBC); 894 } 895 896 static int stm32_cryp_tdes_ecb_encrypt(struct skcipher_request *req) 897 { 898 return stm32_cryp_crypt(req, FLG_TDES | FLG_ECB | FLG_ENCRYPT); 899 } 900 901 static int stm32_cryp_tdes_ecb_decrypt(struct skcipher_request *req) 902 { 903 return stm32_cryp_crypt(req, FLG_TDES | FLG_ECB); 904 } 905 906 static int stm32_cryp_tdes_cbc_encrypt(struct skcipher_request *req) 907 { 908 return stm32_cryp_crypt(req, FLG_TDES | FLG_CBC | FLG_ENCRYPT); 909 } 910 911 static int stm32_cryp_tdes_cbc_decrypt(struct skcipher_request *req) 912 { 913 return stm32_cryp_crypt(req, FLG_TDES | FLG_CBC); 914 } 915 916 static int stm32_cryp_prepare_req(struct skcipher_request *req, 917 struct aead_request *areq) 918 { 919 struct stm32_cryp_ctx *ctx; 920 struct stm32_cryp *cryp; 921 struct stm32_cryp_reqctx *rctx; 922 int ret; 923 924 if (!req && !areq) 925 return -EINVAL; 926 927 ctx = req ? crypto_skcipher_ctx(crypto_skcipher_reqtfm(req)) : 928 crypto_aead_ctx(crypto_aead_reqtfm(areq)); 929 930 cryp = ctx->cryp; 931 932 if (!cryp) 933 return -ENODEV; 934 935 rctx = req ? skcipher_request_ctx(req) : aead_request_ctx(areq); 936 rctx->mode &= FLG_MODE_MASK; 937 938 ctx->cryp = cryp; 939 940 cryp->flags = (cryp->flags & ~FLG_MODE_MASK) | rctx->mode; 941 cryp->hw_blocksize = is_aes(cryp) ? AES_BLOCK_SIZE : DES_BLOCK_SIZE; 942 cryp->ctx = ctx; 943 944 if (req) { 945 cryp->req = req; 946 cryp->areq = NULL; 947 cryp->total_in = req->cryptlen; 948 cryp->total_out = cryp->total_in; 949 } else { 950 /* 951 * Length of input and output data: 952 * Encryption case: 953 * INPUT = AssocData || PlainText 954 * <- assoclen -> <- cryptlen -> 955 * <------- total_in -----------> 956 * 957 * OUTPUT = AssocData || CipherText || AuthTag 958 * <- assoclen -> <- cryptlen -> <- authsize -> 959 * <---------------- total_out -----------------> 960 * 961 * Decryption case: 962 * INPUT = AssocData || CipherText || AuthTag 963 * <- assoclen -> <--------- cryptlen ---------> 964 * <- authsize -> 965 * <---------------- total_in ------------------> 966 * 967 * OUTPUT = AssocData || PlainText 968 * <- assoclen -> <- crypten - authsize -> 969 * <---------- total_out -----------------> 970 */ 971 cryp->areq = areq; 972 cryp->req = NULL; 973 cryp->authsize = crypto_aead_authsize(crypto_aead_reqtfm(areq)); 974 cryp->total_in = areq->assoclen + areq->cryptlen; 975 if (is_encrypt(cryp)) 976 /* Append auth tag to output */ 977 cryp->total_out = cryp->total_in + cryp->authsize; 978 else 979 /* No auth tag in output */ 980 cryp->total_out = cryp->total_in - cryp->authsize; 981 } 982 983 cryp->total_in_save = cryp->total_in; 984 cryp->total_out_save = cryp->total_out; 985 986 cryp->in_sg = req ? req->src : areq->src; 987 cryp->out_sg = req ? req->dst : areq->dst; 988 cryp->out_sg_save = cryp->out_sg; 989 990 cryp->in_sg_len = sg_nents_for_len(cryp->in_sg, cryp->total_in); 991 if (cryp->in_sg_len < 0) { 992 dev_err(cryp->dev, "Cannot get in_sg_len\n"); 993 ret = cryp->in_sg_len; 994 return ret; 995 } 996 997 cryp->out_sg_len = sg_nents_for_len(cryp->out_sg, cryp->total_out); 998 if (cryp->out_sg_len < 0) { 999 dev_err(cryp->dev, "Cannot get out_sg_len\n"); 1000 ret = cryp->out_sg_len; 1001 return ret; 1002 } 1003 1004 ret = stm32_cryp_copy_sgs(cryp); 1005 if (ret) 1006 return ret; 1007 1008 scatterwalk_start(&cryp->in_walk, cryp->in_sg); 1009 scatterwalk_start(&cryp->out_walk, cryp->out_sg); 1010 1011 if (is_gcm(cryp) || is_ccm(cryp)) { 1012 /* In output, jump after assoc data */ 1013 scatterwalk_advance(&cryp->out_walk, cryp->areq->assoclen); 1014 cryp->total_out -= cryp->areq->assoclen; 1015 } 1016 1017 ret = stm32_cryp_hw_init(cryp); 1018 return ret; 1019 } 1020 1021 static int stm32_cryp_prepare_cipher_req(struct crypto_engine *engine, 1022 void *areq) 1023 { 1024 struct skcipher_request *req = container_of(areq, 1025 struct skcipher_request, 1026 base); 1027 1028 return stm32_cryp_prepare_req(req, NULL); 1029 } 1030 1031 static int stm32_cryp_cipher_one_req(struct crypto_engine *engine, void *areq) 1032 { 1033 struct skcipher_request *req = container_of(areq, 1034 struct skcipher_request, 1035 base); 1036 struct stm32_cryp_ctx *ctx = crypto_skcipher_ctx( 1037 crypto_skcipher_reqtfm(req)); 1038 struct stm32_cryp *cryp = ctx->cryp; 1039 1040 if (!cryp) 1041 return -ENODEV; 1042 1043 return stm32_cryp_cpu_start(cryp); 1044 } 1045 1046 static int stm32_cryp_prepare_aead_req(struct crypto_engine *engine, void *areq) 1047 { 1048 struct aead_request *req = container_of(areq, struct aead_request, 1049 base); 1050 1051 return stm32_cryp_prepare_req(NULL, req); 1052 } 1053 1054 static int stm32_cryp_aead_one_req(struct crypto_engine *engine, void *areq) 1055 { 1056 struct aead_request *req = container_of(areq, struct aead_request, 1057 base); 1058 struct stm32_cryp_ctx *ctx = crypto_aead_ctx(crypto_aead_reqtfm(req)); 1059 struct stm32_cryp *cryp = ctx->cryp; 1060 1061 if (!cryp) 1062 return -ENODEV; 1063 1064 if (unlikely(!cryp->areq->assoclen && 1065 !stm32_cryp_get_input_text_len(cryp))) { 1066 /* No input data to process: get tag and finish */ 1067 stm32_cryp_finish_req(cryp, 0); 1068 return 0; 1069 } 1070 1071 return stm32_cryp_cpu_start(cryp); 1072 } 1073 1074 static u32 *stm32_cryp_next_out(struct stm32_cryp *cryp, u32 *dst, 1075 unsigned int n) 1076 { 1077 scatterwalk_advance(&cryp->out_walk, n); 1078 1079 if (unlikely(cryp->out_sg->length == _walked_out)) { 1080 cryp->out_sg = sg_next(cryp->out_sg); 1081 if (cryp->out_sg) { 1082 scatterwalk_start(&cryp->out_walk, cryp->out_sg); 1083 return (sg_virt(cryp->out_sg) + _walked_out); 1084 } 1085 } 1086 1087 return (u32 *)((u8 *)dst + n); 1088 } 1089 1090 static u32 *stm32_cryp_next_in(struct stm32_cryp *cryp, u32 *src, 1091 unsigned int n) 1092 { 1093 scatterwalk_advance(&cryp->in_walk, n); 1094 1095 if (unlikely(cryp->in_sg->length == _walked_in)) { 1096 cryp->in_sg = sg_next(cryp->in_sg); 1097 if (cryp->in_sg) { 1098 scatterwalk_start(&cryp->in_walk, cryp->in_sg); 1099 return (sg_virt(cryp->in_sg) + _walked_in); 1100 } 1101 } 1102 1103 return (u32 *)((u8 *)src + n); 1104 } 1105 1106 static int stm32_cryp_read_auth_tag(struct stm32_cryp *cryp) 1107 { 1108 u32 cfg, size_bit, *dst, d32; 1109 u8 *d8; 1110 unsigned int i, j; 1111 int ret = 0; 1112 1113 /* Update Config */ 1114 cfg = stm32_cryp_read(cryp, CRYP_CR); 1115 1116 cfg &= ~CR_PH_MASK; 1117 cfg |= CR_PH_FINAL; 1118 cfg &= ~CR_DEC_NOT_ENC; 1119 cfg |= CR_CRYPEN; 1120 1121 stm32_cryp_write(cryp, CRYP_CR, cfg); 1122 1123 if (is_gcm(cryp)) { 1124 /* GCM: write aad and payload size (in bits) */ 1125 size_bit = cryp->areq->assoclen * 8; 1126 if (cryp->caps->swap_final) 1127 size_bit = (__force u32)cpu_to_be32(size_bit); 1128 1129 stm32_cryp_write(cryp, CRYP_DIN, 0); 1130 stm32_cryp_write(cryp, CRYP_DIN, size_bit); 1131 1132 size_bit = is_encrypt(cryp) ? cryp->areq->cryptlen : 1133 cryp->areq->cryptlen - AES_BLOCK_SIZE; 1134 size_bit *= 8; 1135 if (cryp->caps->swap_final) 1136 size_bit = (__force u32)cpu_to_be32(size_bit); 1137 1138 stm32_cryp_write(cryp, CRYP_DIN, 0); 1139 stm32_cryp_write(cryp, CRYP_DIN, size_bit); 1140 } else { 1141 /* CCM: write CTR0 */ 1142 u8 iv[AES_BLOCK_SIZE]; 1143 u32 *iv32 = (u32 *)iv; 1144 __be32 *biv; 1145 1146 biv = (void *)iv; 1147 1148 memcpy(iv, cryp->areq->iv, AES_BLOCK_SIZE); 1149 memset(iv + AES_BLOCK_SIZE - 1 - iv[0], 0, iv[0] + 1); 1150 1151 for (i = 0; i < AES_BLOCK_32; i++) { 1152 u32 xiv = iv32[i]; 1153 1154 if (!cryp->caps->padding_wa) 1155 xiv = be32_to_cpu(biv[i]); 1156 stm32_cryp_write(cryp, CRYP_DIN, xiv); 1157 } 1158 } 1159 1160 /* Wait for output data */ 1161 ret = stm32_cryp_wait_output(cryp); 1162 if (ret) { 1163 dev_err(cryp->dev, "Timeout (read tag)\n"); 1164 return ret; 1165 } 1166 1167 if (is_encrypt(cryp)) { 1168 /* Get and write tag */ 1169 dst = sg_virt(cryp->out_sg) + _walked_out; 1170 1171 for (i = 0; i < AES_BLOCK_32; i++) { 1172 if (cryp->total_out >= sizeof(u32)) { 1173 /* Read a full u32 */ 1174 *dst = stm32_cryp_read(cryp, CRYP_DOUT); 1175 1176 dst = stm32_cryp_next_out(cryp, dst, 1177 sizeof(u32)); 1178 cryp->total_out -= sizeof(u32); 1179 } else if (!cryp->total_out) { 1180 /* Empty fifo out (data from input padding) */ 1181 stm32_cryp_read(cryp, CRYP_DOUT); 1182 } else { 1183 /* Read less than an u32 */ 1184 d32 = stm32_cryp_read(cryp, CRYP_DOUT); 1185 d8 = (u8 *)&d32; 1186 1187 for (j = 0; j < cryp->total_out; j++) { 1188 *((u8 *)dst) = *(d8++); 1189 dst = stm32_cryp_next_out(cryp, dst, 1); 1190 } 1191 cryp->total_out = 0; 1192 } 1193 } 1194 } else { 1195 /* Get and check tag */ 1196 u32 in_tag[AES_BLOCK_32], out_tag[AES_BLOCK_32]; 1197 1198 scatterwalk_map_and_copy(in_tag, cryp->in_sg, 1199 cryp->total_in_save - cryp->authsize, 1200 cryp->authsize, 0); 1201 1202 for (i = 0; i < AES_BLOCK_32; i++) 1203 out_tag[i] = stm32_cryp_read(cryp, CRYP_DOUT); 1204 1205 if (crypto_memneq(in_tag, out_tag, cryp->authsize)) 1206 ret = -EBADMSG; 1207 } 1208 1209 /* Disable cryp */ 1210 cfg &= ~CR_CRYPEN; 1211 stm32_cryp_write(cryp, CRYP_CR, cfg); 1212 1213 return ret; 1214 } 1215 1216 static void stm32_cryp_check_ctr_counter(struct stm32_cryp *cryp) 1217 { 1218 u32 cr; 1219 1220 if (unlikely(cryp->last_ctr[3] == 0xFFFFFFFF)) { 1221 cryp->last_ctr[3] = 0; 1222 cryp->last_ctr[2]++; 1223 if (!cryp->last_ctr[2]) { 1224 cryp->last_ctr[1]++; 1225 if (!cryp->last_ctr[1]) 1226 cryp->last_ctr[0]++; 1227 } 1228 1229 cr = stm32_cryp_read(cryp, CRYP_CR); 1230 stm32_cryp_write(cryp, CRYP_CR, cr & ~CR_CRYPEN); 1231 1232 stm32_cryp_hw_write_iv(cryp, (__be32 *)cryp->last_ctr); 1233 1234 stm32_cryp_write(cryp, CRYP_CR, cr); 1235 } 1236 1237 cryp->last_ctr[0] = stm32_cryp_read(cryp, CRYP_IV0LR); 1238 cryp->last_ctr[1] = stm32_cryp_read(cryp, CRYP_IV0RR); 1239 cryp->last_ctr[2] = stm32_cryp_read(cryp, CRYP_IV1LR); 1240 cryp->last_ctr[3] = stm32_cryp_read(cryp, CRYP_IV1RR); 1241 } 1242 1243 static bool stm32_cryp_irq_read_data(struct stm32_cryp *cryp) 1244 { 1245 unsigned int i, j; 1246 u32 d32, *dst; 1247 u8 *d8; 1248 size_t tag_size; 1249 1250 /* Do no read tag now (if any) */ 1251 if (is_encrypt(cryp) && (is_gcm(cryp) || is_ccm(cryp))) 1252 tag_size = cryp->authsize; 1253 else 1254 tag_size = 0; 1255 1256 dst = sg_virt(cryp->out_sg) + _walked_out; 1257 1258 for (i = 0; i < cryp->hw_blocksize / sizeof(u32); i++) { 1259 if (likely(cryp->total_out - tag_size >= sizeof(u32))) { 1260 /* Read a full u32 */ 1261 *dst = stm32_cryp_read(cryp, CRYP_DOUT); 1262 1263 dst = stm32_cryp_next_out(cryp, dst, sizeof(u32)); 1264 cryp->total_out -= sizeof(u32); 1265 } else if (cryp->total_out == tag_size) { 1266 /* Empty fifo out (data from input padding) */ 1267 d32 = stm32_cryp_read(cryp, CRYP_DOUT); 1268 } else { 1269 /* Read less than an u32 */ 1270 d32 = stm32_cryp_read(cryp, CRYP_DOUT); 1271 d8 = (u8 *)&d32; 1272 1273 for (j = 0; j < cryp->total_out - tag_size; j++) { 1274 *((u8 *)dst) = *(d8++); 1275 dst = stm32_cryp_next_out(cryp, dst, 1); 1276 } 1277 cryp->total_out = tag_size; 1278 } 1279 } 1280 1281 return !(cryp->total_out - tag_size) || !cryp->total_in; 1282 } 1283 1284 static void stm32_cryp_irq_write_block(struct stm32_cryp *cryp) 1285 { 1286 unsigned int i, j; 1287 u32 *src; 1288 u8 d8[4]; 1289 size_t tag_size; 1290 1291 /* Do no write tag (if any) */ 1292 if (is_decrypt(cryp) && (is_gcm(cryp) || is_ccm(cryp))) 1293 tag_size = cryp->authsize; 1294 else 1295 tag_size = 0; 1296 1297 src = sg_virt(cryp->in_sg) + _walked_in; 1298 1299 for (i = 0; i < cryp->hw_blocksize / sizeof(u32); i++) { 1300 if (likely(cryp->total_in - tag_size >= sizeof(u32))) { 1301 /* Write a full u32 */ 1302 stm32_cryp_write(cryp, CRYP_DIN, *src); 1303 1304 src = stm32_cryp_next_in(cryp, src, sizeof(u32)); 1305 cryp->total_in -= sizeof(u32); 1306 } else if (cryp->total_in == tag_size) { 1307 /* Write padding data */ 1308 stm32_cryp_write(cryp, CRYP_DIN, 0); 1309 } else { 1310 /* Write less than an u32 */ 1311 memset(d8, 0, sizeof(u32)); 1312 for (j = 0; j < cryp->total_in - tag_size; j++) { 1313 d8[j] = *((u8 *)src); 1314 src = stm32_cryp_next_in(cryp, src, 1); 1315 } 1316 1317 stm32_cryp_write(cryp, CRYP_DIN, *(u32 *)d8); 1318 cryp->total_in = tag_size; 1319 } 1320 } 1321 } 1322 1323 static void stm32_cryp_irq_write_gcm_padded_data(struct stm32_cryp *cryp) 1324 { 1325 int err; 1326 u32 cfg, tmp[AES_BLOCK_32]; 1327 size_t total_in_ori = cryp->total_in; 1328 struct scatterlist *out_sg_ori = cryp->out_sg; 1329 unsigned int i; 1330 1331 /* 'Special workaround' procedure described in the datasheet */ 1332 1333 /* a) disable ip */ 1334 stm32_cryp_write(cryp, CRYP_IMSCR, 0); 1335 cfg = stm32_cryp_read(cryp, CRYP_CR); 1336 cfg &= ~CR_CRYPEN; 1337 stm32_cryp_write(cryp, CRYP_CR, cfg); 1338 1339 /* b) Update IV1R */ 1340 stm32_cryp_write(cryp, CRYP_IV1RR, cryp->gcm_ctr - 2); 1341 1342 /* c) change mode to CTR */ 1343 cfg &= ~CR_ALGO_MASK; 1344 cfg |= CR_AES_CTR; 1345 stm32_cryp_write(cryp, CRYP_CR, cfg); 1346 1347 /* a) enable IP */ 1348 cfg |= CR_CRYPEN; 1349 stm32_cryp_write(cryp, CRYP_CR, cfg); 1350 1351 /* b) pad and write the last block */ 1352 stm32_cryp_irq_write_block(cryp); 1353 cryp->total_in = total_in_ori; 1354 err = stm32_cryp_wait_output(cryp); 1355 if (err) { 1356 dev_err(cryp->dev, "Timeout (write gcm header)\n"); 1357 return stm32_cryp_finish_req(cryp, err); 1358 } 1359 1360 /* c) get and store encrypted data */ 1361 stm32_cryp_irq_read_data(cryp); 1362 scatterwalk_map_and_copy(tmp, out_sg_ori, 1363 cryp->total_in_save - total_in_ori, 1364 total_in_ori, 0); 1365 1366 /* d) change mode back to AES GCM */ 1367 cfg &= ~CR_ALGO_MASK; 1368 cfg |= CR_AES_GCM; 1369 stm32_cryp_write(cryp, CRYP_CR, cfg); 1370 1371 /* e) change phase to Final */ 1372 cfg &= ~CR_PH_MASK; 1373 cfg |= CR_PH_FINAL; 1374 stm32_cryp_write(cryp, CRYP_CR, cfg); 1375 1376 /* f) write padded data */ 1377 for (i = 0; i < AES_BLOCK_32; i++) { 1378 if (cryp->total_in) 1379 stm32_cryp_write(cryp, CRYP_DIN, tmp[i]); 1380 else 1381 stm32_cryp_write(cryp, CRYP_DIN, 0); 1382 1383 cryp->total_in -= min_t(size_t, sizeof(u32), cryp->total_in); 1384 } 1385 1386 /* g) Empty fifo out */ 1387 err = stm32_cryp_wait_output(cryp); 1388 if (err) { 1389 dev_err(cryp->dev, "Timeout (write gcm header)\n"); 1390 return stm32_cryp_finish_req(cryp, err); 1391 } 1392 1393 for (i = 0; i < AES_BLOCK_32; i++) 1394 stm32_cryp_read(cryp, CRYP_DOUT); 1395 1396 /* h) run the he normal Final phase */ 1397 stm32_cryp_finish_req(cryp, 0); 1398 } 1399 1400 static void stm32_cryp_irq_set_npblb(struct stm32_cryp *cryp) 1401 { 1402 u32 cfg, payload_bytes; 1403 1404 /* disable ip, set NPBLB and reneable ip */ 1405 cfg = stm32_cryp_read(cryp, CRYP_CR); 1406 cfg &= ~CR_CRYPEN; 1407 stm32_cryp_write(cryp, CRYP_CR, cfg); 1408 1409 payload_bytes = is_decrypt(cryp) ? cryp->total_in - cryp->authsize : 1410 cryp->total_in; 1411 cfg |= (cryp->hw_blocksize - payload_bytes) << CR_NBPBL_SHIFT; 1412 cfg |= CR_CRYPEN; 1413 stm32_cryp_write(cryp, CRYP_CR, cfg); 1414 } 1415 1416 static void stm32_cryp_irq_write_ccm_padded_data(struct stm32_cryp *cryp) 1417 { 1418 int err = 0; 1419 u32 cfg, iv1tmp; 1420 u32 cstmp1[AES_BLOCK_32], cstmp2[AES_BLOCK_32], tmp[AES_BLOCK_32]; 1421 size_t last_total_out, total_in_ori = cryp->total_in; 1422 struct scatterlist *out_sg_ori = cryp->out_sg; 1423 unsigned int i; 1424 1425 /* 'Special workaround' procedure described in the datasheet */ 1426 cryp->flags |= FLG_CCM_PADDED_WA; 1427 1428 /* a) disable ip */ 1429 stm32_cryp_write(cryp, CRYP_IMSCR, 0); 1430 1431 cfg = stm32_cryp_read(cryp, CRYP_CR); 1432 cfg &= ~CR_CRYPEN; 1433 stm32_cryp_write(cryp, CRYP_CR, cfg); 1434 1435 /* b) get IV1 from CRYP_CSGCMCCM7 */ 1436 iv1tmp = stm32_cryp_read(cryp, CRYP_CSGCMCCM0R + 7 * 4); 1437 1438 /* c) Load CRYP_CSGCMCCMxR */ 1439 for (i = 0; i < ARRAY_SIZE(cstmp1); i++) 1440 cstmp1[i] = stm32_cryp_read(cryp, CRYP_CSGCMCCM0R + i * 4); 1441 1442 /* d) Write IV1R */ 1443 stm32_cryp_write(cryp, CRYP_IV1RR, iv1tmp); 1444 1445 /* e) change mode to CTR */ 1446 cfg &= ~CR_ALGO_MASK; 1447 cfg |= CR_AES_CTR; 1448 stm32_cryp_write(cryp, CRYP_CR, cfg); 1449 1450 /* a) enable IP */ 1451 cfg |= CR_CRYPEN; 1452 stm32_cryp_write(cryp, CRYP_CR, cfg); 1453 1454 /* b) pad and write the last block */ 1455 stm32_cryp_irq_write_block(cryp); 1456 cryp->total_in = total_in_ori; 1457 err = stm32_cryp_wait_output(cryp); 1458 if (err) { 1459 dev_err(cryp->dev, "Timeout (wite ccm padded data)\n"); 1460 return stm32_cryp_finish_req(cryp, err); 1461 } 1462 1463 /* c) get and store decrypted data */ 1464 last_total_out = cryp->total_out; 1465 stm32_cryp_irq_read_data(cryp); 1466 1467 memset(tmp, 0, sizeof(tmp)); 1468 scatterwalk_map_and_copy(tmp, out_sg_ori, 1469 cryp->total_out_save - last_total_out, 1470 last_total_out, 0); 1471 1472 /* d) Load again CRYP_CSGCMCCMxR */ 1473 for (i = 0; i < ARRAY_SIZE(cstmp2); i++) 1474 cstmp2[i] = stm32_cryp_read(cryp, CRYP_CSGCMCCM0R + i * 4); 1475 1476 /* e) change mode back to AES CCM */ 1477 cfg &= ~CR_ALGO_MASK; 1478 cfg |= CR_AES_CCM; 1479 stm32_cryp_write(cryp, CRYP_CR, cfg); 1480 1481 /* f) change phase to header */ 1482 cfg &= ~CR_PH_MASK; 1483 cfg |= CR_PH_HEADER; 1484 stm32_cryp_write(cryp, CRYP_CR, cfg); 1485 1486 /* g) XOR and write padded data */ 1487 for (i = 0; i < ARRAY_SIZE(tmp); i++) { 1488 tmp[i] ^= cstmp1[i]; 1489 tmp[i] ^= cstmp2[i]; 1490 stm32_cryp_write(cryp, CRYP_DIN, tmp[i]); 1491 } 1492 1493 /* h) wait for completion */ 1494 err = stm32_cryp_wait_busy(cryp); 1495 if (err) 1496 dev_err(cryp->dev, "Timeout (wite ccm padded data)\n"); 1497 1498 /* i) run the he normal Final phase */ 1499 stm32_cryp_finish_req(cryp, err); 1500 } 1501 1502 static void stm32_cryp_irq_write_data(struct stm32_cryp *cryp) 1503 { 1504 if (unlikely(!cryp->total_in)) { 1505 dev_warn(cryp->dev, "No more data to process\n"); 1506 return; 1507 } 1508 1509 if (unlikely(cryp->total_in < AES_BLOCK_SIZE && 1510 (stm32_cryp_get_hw_mode(cryp) == CR_AES_GCM) && 1511 is_encrypt(cryp))) { 1512 /* Padding for AES GCM encryption */ 1513 if (cryp->caps->padding_wa) 1514 /* Special case 1 */ 1515 return stm32_cryp_irq_write_gcm_padded_data(cryp); 1516 1517 /* Setting padding bytes (NBBLB) */ 1518 stm32_cryp_irq_set_npblb(cryp); 1519 } 1520 1521 if (unlikely((cryp->total_in - cryp->authsize < AES_BLOCK_SIZE) && 1522 (stm32_cryp_get_hw_mode(cryp) == CR_AES_CCM) && 1523 is_decrypt(cryp))) { 1524 /* Padding for AES CCM decryption */ 1525 if (cryp->caps->padding_wa) 1526 /* Special case 2 */ 1527 return stm32_cryp_irq_write_ccm_padded_data(cryp); 1528 1529 /* Setting padding bytes (NBBLB) */ 1530 stm32_cryp_irq_set_npblb(cryp); 1531 } 1532 1533 if (is_aes(cryp) && is_ctr(cryp)) 1534 stm32_cryp_check_ctr_counter(cryp); 1535 1536 stm32_cryp_irq_write_block(cryp); 1537 } 1538 1539 static void stm32_cryp_irq_write_gcm_header(struct stm32_cryp *cryp) 1540 { 1541 int err; 1542 unsigned int i, j; 1543 u32 cfg, *src; 1544 1545 src = sg_virt(cryp->in_sg) + _walked_in; 1546 1547 for (i = 0; i < AES_BLOCK_32; i++) { 1548 stm32_cryp_write(cryp, CRYP_DIN, *src); 1549 1550 src = stm32_cryp_next_in(cryp, src, sizeof(u32)); 1551 cryp->total_in -= min_t(size_t, sizeof(u32), cryp->total_in); 1552 1553 /* Check if whole header written */ 1554 if ((cryp->total_in_save - cryp->total_in) == 1555 cryp->areq->assoclen) { 1556 /* Write padding if needed */ 1557 for (j = i + 1; j < AES_BLOCK_32; j++) 1558 stm32_cryp_write(cryp, CRYP_DIN, 0); 1559 1560 /* Wait for completion */ 1561 err = stm32_cryp_wait_busy(cryp); 1562 if (err) { 1563 dev_err(cryp->dev, "Timeout (gcm header)\n"); 1564 return stm32_cryp_finish_req(cryp, err); 1565 } 1566 1567 if (stm32_cryp_get_input_text_len(cryp)) { 1568 /* Phase 3 : payload */ 1569 cfg = stm32_cryp_read(cryp, CRYP_CR); 1570 cfg &= ~CR_CRYPEN; 1571 stm32_cryp_write(cryp, CRYP_CR, cfg); 1572 1573 cfg &= ~CR_PH_MASK; 1574 cfg |= CR_PH_PAYLOAD; 1575 cfg |= CR_CRYPEN; 1576 stm32_cryp_write(cryp, CRYP_CR, cfg); 1577 } else { 1578 /* Phase 4 : tag */ 1579 stm32_cryp_write(cryp, CRYP_IMSCR, 0); 1580 stm32_cryp_finish_req(cryp, 0); 1581 } 1582 1583 break; 1584 } 1585 1586 if (!cryp->total_in) 1587 break; 1588 } 1589 } 1590 1591 static void stm32_cryp_irq_write_ccm_header(struct stm32_cryp *cryp) 1592 { 1593 int err; 1594 unsigned int i = 0, j, k; 1595 u32 alen, cfg, *src; 1596 u8 d8[4]; 1597 1598 src = sg_virt(cryp->in_sg) + _walked_in; 1599 alen = cryp->areq->assoclen; 1600 1601 if (!_walked_in) { 1602 if (cryp->areq->assoclen <= 65280) { 1603 /* Write first u32 of B1 */ 1604 d8[0] = (alen >> 8) & 0xFF; 1605 d8[1] = alen & 0xFF; 1606 d8[2] = *((u8 *)src); 1607 src = stm32_cryp_next_in(cryp, src, 1); 1608 d8[3] = *((u8 *)src); 1609 src = stm32_cryp_next_in(cryp, src, 1); 1610 1611 stm32_cryp_write(cryp, CRYP_DIN, *(u32 *)d8); 1612 i++; 1613 1614 cryp->total_in -= min_t(size_t, 2, cryp->total_in); 1615 } else { 1616 /* Build the two first u32 of B1 */ 1617 d8[0] = 0xFF; 1618 d8[1] = 0xFE; 1619 d8[2] = alen & 0xFF000000; 1620 d8[3] = alen & 0x00FF0000; 1621 1622 stm32_cryp_write(cryp, CRYP_DIN, *(u32 *)d8); 1623 i++; 1624 1625 d8[0] = alen & 0x0000FF00; 1626 d8[1] = alen & 0x000000FF; 1627 d8[2] = *((u8 *)src); 1628 src = stm32_cryp_next_in(cryp, src, 1); 1629 d8[3] = *((u8 *)src); 1630 src = stm32_cryp_next_in(cryp, src, 1); 1631 1632 stm32_cryp_write(cryp, CRYP_DIN, *(u32 *)d8); 1633 i++; 1634 1635 cryp->total_in -= min_t(size_t, 2, cryp->total_in); 1636 } 1637 } 1638 1639 /* Write next u32 */ 1640 for (; i < AES_BLOCK_32; i++) { 1641 /* Build an u32 */ 1642 memset(d8, 0, sizeof(u32)); 1643 for (k = 0; k < sizeof(u32); k++) { 1644 d8[k] = *((u8 *)src); 1645 src = stm32_cryp_next_in(cryp, src, 1); 1646 1647 cryp->total_in -= min_t(size_t, 1, cryp->total_in); 1648 if ((cryp->total_in_save - cryp->total_in) == alen) 1649 break; 1650 } 1651 1652 stm32_cryp_write(cryp, CRYP_DIN, *(u32 *)d8); 1653 1654 if ((cryp->total_in_save - cryp->total_in) == alen) { 1655 /* Write padding if needed */ 1656 for (j = i + 1; j < AES_BLOCK_32; j++) 1657 stm32_cryp_write(cryp, CRYP_DIN, 0); 1658 1659 /* Wait for completion */ 1660 err = stm32_cryp_wait_busy(cryp); 1661 if (err) { 1662 dev_err(cryp->dev, "Timeout (ccm header)\n"); 1663 return stm32_cryp_finish_req(cryp, err); 1664 } 1665 1666 if (stm32_cryp_get_input_text_len(cryp)) { 1667 /* Phase 3 : payload */ 1668 cfg = stm32_cryp_read(cryp, CRYP_CR); 1669 cfg &= ~CR_CRYPEN; 1670 stm32_cryp_write(cryp, CRYP_CR, cfg); 1671 1672 cfg &= ~CR_PH_MASK; 1673 cfg |= CR_PH_PAYLOAD; 1674 cfg |= CR_CRYPEN; 1675 stm32_cryp_write(cryp, CRYP_CR, cfg); 1676 } else { 1677 /* Phase 4 : tag */ 1678 stm32_cryp_write(cryp, CRYP_IMSCR, 0); 1679 stm32_cryp_finish_req(cryp, 0); 1680 } 1681 1682 break; 1683 } 1684 } 1685 } 1686 1687 static irqreturn_t stm32_cryp_irq_thread(int irq, void *arg) 1688 { 1689 struct stm32_cryp *cryp = arg; 1690 u32 ph; 1691 1692 if (cryp->irq_status & MISR_OUT) 1693 /* Output FIFO IRQ: read data */ 1694 if (unlikely(stm32_cryp_irq_read_data(cryp))) { 1695 /* All bytes processed, finish */ 1696 stm32_cryp_write(cryp, CRYP_IMSCR, 0); 1697 stm32_cryp_finish_req(cryp, 0); 1698 return IRQ_HANDLED; 1699 } 1700 1701 if (cryp->irq_status & MISR_IN) { 1702 if (is_gcm(cryp)) { 1703 ph = stm32_cryp_read(cryp, CRYP_CR) & CR_PH_MASK; 1704 if (unlikely(ph == CR_PH_HEADER)) 1705 /* Write Header */ 1706 stm32_cryp_irq_write_gcm_header(cryp); 1707 else 1708 /* Input FIFO IRQ: write data */ 1709 stm32_cryp_irq_write_data(cryp); 1710 cryp->gcm_ctr++; 1711 } else if (is_ccm(cryp)) { 1712 ph = stm32_cryp_read(cryp, CRYP_CR) & CR_PH_MASK; 1713 if (unlikely(ph == CR_PH_HEADER)) 1714 /* Write Header */ 1715 stm32_cryp_irq_write_ccm_header(cryp); 1716 else 1717 /* Input FIFO IRQ: write data */ 1718 stm32_cryp_irq_write_data(cryp); 1719 } else { 1720 /* Input FIFO IRQ: write data */ 1721 stm32_cryp_irq_write_data(cryp); 1722 } 1723 } 1724 1725 return IRQ_HANDLED; 1726 } 1727 1728 static irqreturn_t stm32_cryp_irq(int irq, void *arg) 1729 { 1730 struct stm32_cryp *cryp = arg; 1731 1732 cryp->irq_status = stm32_cryp_read(cryp, CRYP_MISR); 1733 1734 return IRQ_WAKE_THREAD; 1735 } 1736 1737 static struct skcipher_alg crypto_algs[] = { 1738 { 1739 .base.cra_name = "ecb(aes)", 1740 .base.cra_driver_name = "stm32-ecb-aes", 1741 .base.cra_priority = 200, 1742 .base.cra_flags = CRYPTO_ALG_ASYNC, 1743 .base.cra_blocksize = AES_BLOCK_SIZE, 1744 .base.cra_ctxsize = sizeof(struct stm32_cryp_ctx), 1745 .base.cra_alignmask = 0xf, 1746 .base.cra_module = THIS_MODULE, 1747 1748 .init = stm32_cryp_init_tfm, 1749 .min_keysize = AES_MIN_KEY_SIZE, 1750 .max_keysize = AES_MAX_KEY_SIZE, 1751 .setkey = stm32_cryp_aes_setkey, 1752 .encrypt = stm32_cryp_aes_ecb_encrypt, 1753 .decrypt = stm32_cryp_aes_ecb_decrypt, 1754 }, 1755 { 1756 .base.cra_name = "cbc(aes)", 1757 .base.cra_driver_name = "stm32-cbc-aes", 1758 .base.cra_priority = 200, 1759 .base.cra_flags = CRYPTO_ALG_ASYNC, 1760 .base.cra_blocksize = AES_BLOCK_SIZE, 1761 .base.cra_ctxsize = sizeof(struct stm32_cryp_ctx), 1762 .base.cra_alignmask = 0xf, 1763 .base.cra_module = THIS_MODULE, 1764 1765 .init = stm32_cryp_init_tfm, 1766 .min_keysize = AES_MIN_KEY_SIZE, 1767 .max_keysize = AES_MAX_KEY_SIZE, 1768 .ivsize = AES_BLOCK_SIZE, 1769 .setkey = stm32_cryp_aes_setkey, 1770 .encrypt = stm32_cryp_aes_cbc_encrypt, 1771 .decrypt = stm32_cryp_aes_cbc_decrypt, 1772 }, 1773 { 1774 .base.cra_name = "ctr(aes)", 1775 .base.cra_driver_name = "stm32-ctr-aes", 1776 .base.cra_priority = 200, 1777 .base.cra_flags = CRYPTO_ALG_ASYNC, 1778 .base.cra_blocksize = 1, 1779 .base.cra_ctxsize = sizeof(struct stm32_cryp_ctx), 1780 .base.cra_alignmask = 0xf, 1781 .base.cra_module = THIS_MODULE, 1782 1783 .init = stm32_cryp_init_tfm, 1784 .min_keysize = AES_MIN_KEY_SIZE, 1785 .max_keysize = AES_MAX_KEY_SIZE, 1786 .ivsize = AES_BLOCK_SIZE, 1787 .setkey = stm32_cryp_aes_setkey, 1788 .encrypt = stm32_cryp_aes_ctr_encrypt, 1789 .decrypt = stm32_cryp_aes_ctr_decrypt, 1790 }, 1791 { 1792 .base.cra_name = "ecb(des)", 1793 .base.cra_driver_name = "stm32-ecb-des", 1794 .base.cra_priority = 200, 1795 .base.cra_flags = CRYPTO_ALG_ASYNC, 1796 .base.cra_blocksize = DES_BLOCK_SIZE, 1797 .base.cra_ctxsize = sizeof(struct stm32_cryp_ctx), 1798 .base.cra_alignmask = 0xf, 1799 .base.cra_module = THIS_MODULE, 1800 1801 .init = stm32_cryp_init_tfm, 1802 .min_keysize = DES_BLOCK_SIZE, 1803 .max_keysize = DES_BLOCK_SIZE, 1804 .setkey = stm32_cryp_des_setkey, 1805 .encrypt = stm32_cryp_des_ecb_encrypt, 1806 .decrypt = stm32_cryp_des_ecb_decrypt, 1807 }, 1808 { 1809 .base.cra_name = "cbc(des)", 1810 .base.cra_driver_name = "stm32-cbc-des", 1811 .base.cra_priority = 200, 1812 .base.cra_flags = CRYPTO_ALG_ASYNC, 1813 .base.cra_blocksize = DES_BLOCK_SIZE, 1814 .base.cra_ctxsize = sizeof(struct stm32_cryp_ctx), 1815 .base.cra_alignmask = 0xf, 1816 .base.cra_module = THIS_MODULE, 1817 1818 .init = stm32_cryp_init_tfm, 1819 .min_keysize = DES_BLOCK_SIZE, 1820 .max_keysize = DES_BLOCK_SIZE, 1821 .ivsize = DES_BLOCK_SIZE, 1822 .setkey = stm32_cryp_des_setkey, 1823 .encrypt = stm32_cryp_des_cbc_encrypt, 1824 .decrypt = stm32_cryp_des_cbc_decrypt, 1825 }, 1826 { 1827 .base.cra_name = "ecb(des3_ede)", 1828 .base.cra_driver_name = "stm32-ecb-des3", 1829 .base.cra_priority = 200, 1830 .base.cra_flags = CRYPTO_ALG_ASYNC, 1831 .base.cra_blocksize = DES_BLOCK_SIZE, 1832 .base.cra_ctxsize = sizeof(struct stm32_cryp_ctx), 1833 .base.cra_alignmask = 0xf, 1834 .base.cra_module = THIS_MODULE, 1835 1836 .init = stm32_cryp_init_tfm, 1837 .min_keysize = 3 * DES_BLOCK_SIZE, 1838 .max_keysize = 3 * DES_BLOCK_SIZE, 1839 .setkey = stm32_cryp_tdes_setkey, 1840 .encrypt = stm32_cryp_tdes_ecb_encrypt, 1841 .decrypt = stm32_cryp_tdes_ecb_decrypt, 1842 }, 1843 { 1844 .base.cra_name = "cbc(des3_ede)", 1845 .base.cra_driver_name = "stm32-cbc-des3", 1846 .base.cra_priority = 200, 1847 .base.cra_flags = CRYPTO_ALG_ASYNC, 1848 .base.cra_blocksize = DES_BLOCK_SIZE, 1849 .base.cra_ctxsize = sizeof(struct stm32_cryp_ctx), 1850 .base.cra_alignmask = 0xf, 1851 .base.cra_module = THIS_MODULE, 1852 1853 .init = stm32_cryp_init_tfm, 1854 .min_keysize = 3 * DES_BLOCK_SIZE, 1855 .max_keysize = 3 * DES_BLOCK_SIZE, 1856 .ivsize = DES_BLOCK_SIZE, 1857 .setkey = stm32_cryp_tdes_setkey, 1858 .encrypt = stm32_cryp_tdes_cbc_encrypt, 1859 .decrypt = stm32_cryp_tdes_cbc_decrypt, 1860 }, 1861 }; 1862 1863 static struct aead_alg aead_algs[] = { 1864 { 1865 .setkey = stm32_cryp_aes_aead_setkey, 1866 .setauthsize = stm32_cryp_aes_gcm_setauthsize, 1867 .encrypt = stm32_cryp_aes_gcm_encrypt, 1868 .decrypt = stm32_cryp_aes_gcm_decrypt, 1869 .init = stm32_cryp_aes_aead_init, 1870 .ivsize = 12, 1871 .maxauthsize = AES_BLOCK_SIZE, 1872 1873 .base = { 1874 .cra_name = "gcm(aes)", 1875 .cra_driver_name = "stm32-gcm-aes", 1876 .cra_priority = 200, 1877 .cra_flags = CRYPTO_ALG_ASYNC, 1878 .cra_blocksize = 1, 1879 .cra_ctxsize = sizeof(struct stm32_cryp_ctx), 1880 .cra_alignmask = 0xf, 1881 .cra_module = THIS_MODULE, 1882 }, 1883 }, 1884 { 1885 .setkey = stm32_cryp_aes_aead_setkey, 1886 .setauthsize = stm32_cryp_aes_ccm_setauthsize, 1887 .encrypt = stm32_cryp_aes_ccm_encrypt, 1888 .decrypt = stm32_cryp_aes_ccm_decrypt, 1889 .init = stm32_cryp_aes_aead_init, 1890 .ivsize = AES_BLOCK_SIZE, 1891 .maxauthsize = AES_BLOCK_SIZE, 1892 1893 .base = { 1894 .cra_name = "ccm(aes)", 1895 .cra_driver_name = "stm32-ccm-aes", 1896 .cra_priority = 200, 1897 .cra_flags = CRYPTO_ALG_ASYNC, 1898 .cra_blocksize = 1, 1899 .cra_ctxsize = sizeof(struct stm32_cryp_ctx), 1900 .cra_alignmask = 0xf, 1901 .cra_module = THIS_MODULE, 1902 }, 1903 }, 1904 }; 1905 1906 static const struct stm32_cryp_caps f7_data = { 1907 .swap_final = true, 1908 .padding_wa = true, 1909 }; 1910 1911 static const struct stm32_cryp_caps mp1_data = { 1912 .swap_final = false, 1913 .padding_wa = false, 1914 }; 1915 1916 static const struct of_device_id stm32_dt_ids[] = { 1917 { .compatible = "st,stm32f756-cryp", .data = &f7_data}, 1918 { .compatible = "st,stm32mp1-cryp", .data = &mp1_data}, 1919 {}, 1920 }; 1921 MODULE_DEVICE_TABLE(of, stm32_dt_ids); 1922 1923 static int stm32_cryp_probe(struct platform_device *pdev) 1924 { 1925 struct device *dev = &pdev->dev; 1926 struct stm32_cryp *cryp; 1927 struct reset_control *rst; 1928 int irq, ret; 1929 1930 cryp = devm_kzalloc(dev, sizeof(*cryp), GFP_KERNEL); 1931 if (!cryp) 1932 return -ENOMEM; 1933 1934 cryp->caps = of_device_get_match_data(dev); 1935 if (!cryp->caps) 1936 return -ENODEV; 1937 1938 cryp->dev = dev; 1939 1940 cryp->regs = devm_platform_ioremap_resource(pdev, 0); 1941 if (IS_ERR(cryp->regs)) 1942 return PTR_ERR(cryp->regs); 1943 1944 irq = platform_get_irq(pdev, 0); 1945 if (irq < 0) 1946 return irq; 1947 1948 ret = devm_request_threaded_irq(dev, irq, stm32_cryp_irq, 1949 stm32_cryp_irq_thread, IRQF_ONESHOT, 1950 dev_name(dev), cryp); 1951 if (ret) { 1952 dev_err(dev, "Cannot grab IRQ\n"); 1953 return ret; 1954 } 1955 1956 cryp->clk = devm_clk_get(dev, NULL); 1957 if (IS_ERR(cryp->clk)) { 1958 dev_err(dev, "Could not get clock\n"); 1959 return PTR_ERR(cryp->clk); 1960 } 1961 1962 ret = clk_prepare_enable(cryp->clk); 1963 if (ret) { 1964 dev_err(cryp->dev, "Failed to enable clock\n"); 1965 return ret; 1966 } 1967 1968 pm_runtime_set_autosuspend_delay(dev, CRYP_AUTOSUSPEND_DELAY); 1969 pm_runtime_use_autosuspend(dev); 1970 1971 pm_runtime_get_noresume(dev); 1972 pm_runtime_set_active(dev); 1973 pm_runtime_enable(dev); 1974 1975 rst = devm_reset_control_get(dev, NULL); 1976 if (IS_ERR(rst)) { 1977 ret = PTR_ERR(rst); 1978 if (ret == -EPROBE_DEFER) 1979 goto err_rst; 1980 } else { 1981 reset_control_assert(rst); 1982 udelay(2); 1983 reset_control_deassert(rst); 1984 } 1985 1986 platform_set_drvdata(pdev, cryp); 1987 1988 spin_lock(&cryp_list.lock); 1989 list_add(&cryp->list, &cryp_list.dev_list); 1990 spin_unlock(&cryp_list.lock); 1991 1992 /* Initialize crypto engine */ 1993 cryp->engine = crypto_engine_alloc_init(dev, 1); 1994 if (!cryp->engine) { 1995 dev_err(dev, "Could not init crypto engine\n"); 1996 ret = -ENOMEM; 1997 goto err_engine1; 1998 } 1999 2000 ret = crypto_engine_start(cryp->engine); 2001 if (ret) { 2002 dev_err(dev, "Could not start crypto engine\n"); 2003 goto err_engine2; 2004 } 2005 2006 ret = crypto_register_skciphers(crypto_algs, ARRAY_SIZE(crypto_algs)); 2007 if (ret) { 2008 dev_err(dev, "Could not register algs\n"); 2009 goto err_algs; 2010 } 2011 2012 ret = crypto_register_aeads(aead_algs, ARRAY_SIZE(aead_algs)); 2013 if (ret) 2014 goto err_aead_algs; 2015 2016 dev_info(dev, "Initialized\n"); 2017 2018 pm_runtime_put_sync(dev); 2019 2020 return 0; 2021 2022 err_aead_algs: 2023 crypto_unregister_skciphers(crypto_algs, ARRAY_SIZE(crypto_algs)); 2024 err_algs: 2025 err_engine2: 2026 crypto_engine_exit(cryp->engine); 2027 err_engine1: 2028 spin_lock(&cryp_list.lock); 2029 list_del(&cryp->list); 2030 spin_unlock(&cryp_list.lock); 2031 err_rst: 2032 pm_runtime_disable(dev); 2033 pm_runtime_put_noidle(dev); 2034 pm_runtime_disable(dev); 2035 pm_runtime_put_noidle(dev); 2036 2037 clk_disable_unprepare(cryp->clk); 2038 2039 return ret; 2040 } 2041 2042 static int stm32_cryp_remove(struct platform_device *pdev) 2043 { 2044 struct stm32_cryp *cryp = platform_get_drvdata(pdev); 2045 int ret; 2046 2047 if (!cryp) 2048 return -ENODEV; 2049 2050 ret = pm_runtime_resume_and_get(cryp->dev); 2051 if (ret < 0) 2052 return ret; 2053 2054 crypto_unregister_aeads(aead_algs, ARRAY_SIZE(aead_algs)); 2055 crypto_unregister_skciphers(crypto_algs, ARRAY_SIZE(crypto_algs)); 2056 2057 crypto_engine_exit(cryp->engine); 2058 2059 spin_lock(&cryp_list.lock); 2060 list_del(&cryp->list); 2061 spin_unlock(&cryp_list.lock); 2062 2063 pm_runtime_disable(cryp->dev); 2064 pm_runtime_put_noidle(cryp->dev); 2065 2066 clk_disable_unprepare(cryp->clk); 2067 2068 return 0; 2069 } 2070 2071 #ifdef CONFIG_PM 2072 static int stm32_cryp_runtime_suspend(struct device *dev) 2073 { 2074 struct stm32_cryp *cryp = dev_get_drvdata(dev); 2075 2076 clk_disable_unprepare(cryp->clk); 2077 2078 return 0; 2079 } 2080 2081 static int stm32_cryp_runtime_resume(struct device *dev) 2082 { 2083 struct stm32_cryp *cryp = dev_get_drvdata(dev); 2084 int ret; 2085 2086 ret = clk_prepare_enable(cryp->clk); 2087 if (ret) { 2088 dev_err(cryp->dev, "Failed to prepare_enable clock\n"); 2089 return ret; 2090 } 2091 2092 return 0; 2093 } 2094 #endif 2095 2096 static const struct dev_pm_ops stm32_cryp_pm_ops = { 2097 SET_SYSTEM_SLEEP_PM_OPS(pm_runtime_force_suspend, 2098 pm_runtime_force_resume) 2099 SET_RUNTIME_PM_OPS(stm32_cryp_runtime_suspend, 2100 stm32_cryp_runtime_resume, NULL) 2101 }; 2102 2103 static struct platform_driver stm32_cryp_driver = { 2104 .probe = stm32_cryp_probe, 2105 .remove = stm32_cryp_remove, 2106 .driver = { 2107 .name = DRIVER_NAME, 2108 .pm = &stm32_cryp_pm_ops, 2109 .of_match_table = stm32_dt_ids, 2110 }, 2111 }; 2112 2113 module_platform_driver(stm32_cryp_driver); 2114 2115 MODULE_AUTHOR("Fabien Dessenne <fabien.dessenne@st.com>"); 2116 MODULE_DESCRIPTION("STMicrolectronics STM32 CRYP hardware driver"); 2117 MODULE_LICENSE("GPL"); 2118