1 // SPDX-License-Identifier: GPL-2.0-only 2 /* 3 * Copyright (C) STMicroelectronics SA 2017 4 * Author: Fabien Dessenne <fabien.dessenne@st.com> 5 * Ux500 support taken from snippets in the old Ux500 cryp driver 6 */ 7 8 #include <linux/clk.h> 9 #include <linux/delay.h> 10 #include <linux/interrupt.h> 11 #include <linux/iopoll.h> 12 #include <linux/module.h> 13 #include <linux/of_device.h> 14 #include <linux/platform_device.h> 15 #include <linux/pm_runtime.h> 16 #include <linux/reset.h> 17 18 #include <crypto/aes.h> 19 #include <crypto/internal/des.h> 20 #include <crypto/engine.h> 21 #include <crypto/scatterwalk.h> 22 #include <crypto/internal/aead.h> 23 #include <crypto/internal/skcipher.h> 24 25 #define DRIVER_NAME "stm32-cryp" 26 27 /* Bit [0] encrypt / decrypt */ 28 #define FLG_ENCRYPT BIT(0) 29 /* Bit [8..1] algo & operation mode */ 30 #define FLG_AES BIT(1) 31 #define FLG_DES BIT(2) 32 #define FLG_TDES BIT(3) 33 #define FLG_ECB BIT(4) 34 #define FLG_CBC BIT(5) 35 #define FLG_CTR BIT(6) 36 #define FLG_GCM BIT(7) 37 #define FLG_CCM BIT(8) 38 /* Mode mask = bits [15..0] */ 39 #define FLG_MODE_MASK GENMASK(15, 0) 40 /* Bit [31..16] status */ 41 42 /* Registers */ 43 #define CRYP_CR 0x00000000 44 #define CRYP_SR 0x00000004 45 #define CRYP_DIN 0x00000008 46 #define CRYP_DOUT 0x0000000C 47 #define CRYP_DMACR 0x00000010 48 #define CRYP_IMSCR 0x00000014 49 #define CRYP_RISR 0x00000018 50 #define CRYP_MISR 0x0000001C 51 #define CRYP_K0LR 0x00000020 52 #define CRYP_K0RR 0x00000024 53 #define CRYP_K1LR 0x00000028 54 #define CRYP_K1RR 0x0000002C 55 #define CRYP_K2LR 0x00000030 56 #define CRYP_K2RR 0x00000034 57 #define CRYP_K3LR 0x00000038 58 #define CRYP_K3RR 0x0000003C 59 #define CRYP_IV0LR 0x00000040 60 #define CRYP_IV0RR 0x00000044 61 #define CRYP_IV1LR 0x00000048 62 #define CRYP_IV1RR 0x0000004C 63 #define CRYP_CSGCMCCM0R 0x00000050 64 #define CRYP_CSGCM0R 0x00000070 65 66 #define UX500_CRYP_CR 0x00000000 67 #define UX500_CRYP_SR 0x00000004 68 #define UX500_CRYP_DIN 0x00000008 69 #define UX500_CRYP_DINSIZE 0x0000000C 70 #define UX500_CRYP_DOUT 0x00000010 71 #define UX500_CRYP_DOUSIZE 0x00000014 72 #define UX500_CRYP_DMACR 0x00000018 73 #define UX500_CRYP_IMSC 0x0000001C 74 #define UX500_CRYP_RIS 0x00000020 75 #define UX500_CRYP_MIS 0x00000024 76 #define UX500_CRYP_K1L 0x00000028 77 #define UX500_CRYP_K1R 0x0000002C 78 #define UX500_CRYP_K2L 0x00000030 79 #define UX500_CRYP_K2R 0x00000034 80 #define UX500_CRYP_K3L 0x00000038 81 #define UX500_CRYP_K3R 0x0000003C 82 #define UX500_CRYP_K4L 0x00000040 83 #define UX500_CRYP_K4R 0x00000044 84 #define UX500_CRYP_IV0L 0x00000048 85 #define UX500_CRYP_IV0R 0x0000004C 86 #define UX500_CRYP_IV1L 0x00000050 87 #define UX500_CRYP_IV1R 0x00000054 88 89 /* Registers values */ 90 #define CR_DEC_NOT_ENC 0x00000004 91 #define CR_TDES_ECB 0x00000000 92 #define CR_TDES_CBC 0x00000008 93 #define CR_DES_ECB 0x00000010 94 #define CR_DES_CBC 0x00000018 95 #define CR_AES_ECB 0x00000020 96 #define CR_AES_CBC 0x00000028 97 #define CR_AES_CTR 0x00000030 98 #define CR_AES_KP 0x00000038 /* Not on Ux500 */ 99 #define CR_AES_XTS 0x00000038 /* Only on Ux500 */ 100 #define CR_AES_GCM 0x00080000 101 #define CR_AES_CCM 0x00080008 102 #define CR_AES_UNKNOWN 0xFFFFFFFF 103 #define CR_ALGO_MASK 0x00080038 104 #define CR_DATA32 0x00000000 105 #define CR_DATA16 0x00000040 106 #define CR_DATA8 0x00000080 107 #define CR_DATA1 0x000000C0 108 #define CR_KEY128 0x00000000 109 #define CR_KEY192 0x00000100 110 #define CR_KEY256 0x00000200 111 #define CR_KEYRDEN 0x00000400 /* Only on Ux500 */ 112 #define CR_KSE 0x00000800 /* Only on Ux500 */ 113 #define CR_FFLUSH 0x00004000 114 #define CR_CRYPEN 0x00008000 115 #define CR_PH_INIT 0x00000000 116 #define CR_PH_HEADER 0x00010000 117 #define CR_PH_PAYLOAD 0x00020000 118 #define CR_PH_FINAL 0x00030000 119 #define CR_PH_MASK 0x00030000 120 #define CR_NBPBL_SHIFT 20 121 122 #define SR_BUSY 0x00000010 123 #define SR_OFNE 0x00000004 124 125 #define IMSCR_IN BIT(0) 126 #define IMSCR_OUT BIT(1) 127 128 #define MISR_IN BIT(0) 129 #define MISR_OUT BIT(1) 130 131 /* Misc */ 132 #define AES_BLOCK_32 (AES_BLOCK_SIZE / sizeof(u32)) 133 #define GCM_CTR_INIT 2 134 #define CRYP_AUTOSUSPEND_DELAY 50 135 136 struct stm32_cryp_caps { 137 bool aeads_support; 138 bool linear_aes_key; 139 bool kp_mode; 140 bool iv_protection; 141 bool swap_final; 142 bool padding_wa; 143 u32 cr; 144 u32 sr; 145 u32 din; 146 u32 dout; 147 u32 imsc; 148 u32 mis; 149 u32 k1l; 150 u32 k1r; 151 u32 k3r; 152 u32 iv0l; 153 u32 iv0r; 154 u32 iv1l; 155 u32 iv1r; 156 }; 157 158 struct stm32_cryp_ctx { 159 struct crypto_engine_ctx enginectx; 160 struct stm32_cryp *cryp; 161 int keylen; 162 __be32 key[AES_KEYSIZE_256 / sizeof(u32)]; 163 unsigned long flags; 164 }; 165 166 struct stm32_cryp_reqctx { 167 unsigned long mode; 168 }; 169 170 struct stm32_cryp { 171 struct list_head list; 172 struct device *dev; 173 void __iomem *regs; 174 struct clk *clk; 175 unsigned long flags; 176 u32 irq_status; 177 const struct stm32_cryp_caps *caps; 178 struct stm32_cryp_ctx *ctx; 179 180 struct crypto_engine *engine; 181 182 struct skcipher_request *req; 183 struct aead_request *areq; 184 185 size_t authsize; 186 size_t hw_blocksize; 187 188 size_t payload_in; 189 size_t header_in; 190 size_t payload_out; 191 192 struct scatterlist *out_sg; 193 194 struct scatter_walk in_walk; 195 struct scatter_walk out_walk; 196 197 __be32 last_ctr[4]; 198 u32 gcm_ctr; 199 }; 200 201 struct stm32_cryp_list { 202 struct list_head dev_list; 203 spinlock_t lock; /* protect dev_list */ 204 }; 205 206 static struct stm32_cryp_list cryp_list = { 207 .dev_list = LIST_HEAD_INIT(cryp_list.dev_list), 208 .lock = __SPIN_LOCK_UNLOCKED(cryp_list.lock), 209 }; 210 211 static inline bool is_aes(struct stm32_cryp *cryp) 212 { 213 return cryp->flags & FLG_AES; 214 } 215 216 static inline bool is_des(struct stm32_cryp *cryp) 217 { 218 return cryp->flags & FLG_DES; 219 } 220 221 static inline bool is_tdes(struct stm32_cryp *cryp) 222 { 223 return cryp->flags & FLG_TDES; 224 } 225 226 static inline bool is_ecb(struct stm32_cryp *cryp) 227 { 228 return cryp->flags & FLG_ECB; 229 } 230 231 static inline bool is_cbc(struct stm32_cryp *cryp) 232 { 233 return cryp->flags & FLG_CBC; 234 } 235 236 static inline bool is_ctr(struct stm32_cryp *cryp) 237 { 238 return cryp->flags & FLG_CTR; 239 } 240 241 static inline bool is_gcm(struct stm32_cryp *cryp) 242 { 243 return cryp->flags & FLG_GCM; 244 } 245 246 static inline bool is_ccm(struct stm32_cryp *cryp) 247 { 248 return cryp->flags & FLG_CCM; 249 } 250 251 static inline bool is_encrypt(struct stm32_cryp *cryp) 252 { 253 return cryp->flags & FLG_ENCRYPT; 254 } 255 256 static inline bool is_decrypt(struct stm32_cryp *cryp) 257 { 258 return !is_encrypt(cryp); 259 } 260 261 static inline u32 stm32_cryp_read(struct stm32_cryp *cryp, u32 ofst) 262 { 263 return readl_relaxed(cryp->regs + ofst); 264 } 265 266 static inline void stm32_cryp_write(struct stm32_cryp *cryp, u32 ofst, u32 val) 267 { 268 writel_relaxed(val, cryp->regs + ofst); 269 } 270 271 static inline int stm32_cryp_wait_busy(struct stm32_cryp *cryp) 272 { 273 u32 status; 274 275 return readl_relaxed_poll_timeout(cryp->regs + cryp->caps->sr, status, 276 !(status & SR_BUSY), 10, 100000); 277 } 278 279 static inline void stm32_cryp_enable(struct stm32_cryp *cryp) 280 { 281 writel_relaxed(readl_relaxed(cryp->regs + cryp->caps->cr) | CR_CRYPEN, 282 cryp->regs + cryp->caps->cr); 283 } 284 285 static inline int stm32_cryp_wait_enable(struct stm32_cryp *cryp) 286 { 287 u32 status; 288 289 return readl_relaxed_poll_timeout(cryp->regs + cryp->caps->cr, status, 290 !(status & CR_CRYPEN), 10, 100000); 291 } 292 293 static inline int stm32_cryp_wait_output(struct stm32_cryp *cryp) 294 { 295 u32 status; 296 297 return readl_relaxed_poll_timeout(cryp->regs + cryp->caps->sr, status, 298 status & SR_OFNE, 10, 100000); 299 } 300 301 static inline void stm32_cryp_key_read_enable(struct stm32_cryp *cryp) 302 { 303 writel_relaxed(readl_relaxed(cryp->regs + cryp->caps->cr) | CR_KEYRDEN, 304 cryp->regs + cryp->caps->cr); 305 } 306 307 static inline void stm32_cryp_key_read_disable(struct stm32_cryp *cryp) 308 { 309 writel_relaxed(readl_relaxed(cryp->regs + cryp->caps->cr) & ~CR_KEYRDEN, 310 cryp->regs + cryp->caps->cr); 311 } 312 313 static int stm32_cryp_read_auth_tag(struct stm32_cryp *cryp); 314 static void stm32_cryp_finish_req(struct stm32_cryp *cryp, int err); 315 316 static struct stm32_cryp *stm32_cryp_find_dev(struct stm32_cryp_ctx *ctx) 317 { 318 struct stm32_cryp *tmp, *cryp = NULL; 319 320 spin_lock_bh(&cryp_list.lock); 321 if (!ctx->cryp) { 322 list_for_each_entry(tmp, &cryp_list.dev_list, list) { 323 cryp = tmp; 324 break; 325 } 326 ctx->cryp = cryp; 327 } else { 328 cryp = ctx->cryp; 329 } 330 331 spin_unlock_bh(&cryp_list.lock); 332 333 return cryp; 334 } 335 336 static void stm32_cryp_hw_write_iv(struct stm32_cryp *cryp, __be32 *iv) 337 { 338 if (!iv) 339 return; 340 341 stm32_cryp_write(cryp, cryp->caps->iv0l, be32_to_cpu(*iv++)); 342 stm32_cryp_write(cryp, cryp->caps->iv0r, be32_to_cpu(*iv++)); 343 344 if (is_aes(cryp)) { 345 stm32_cryp_write(cryp, cryp->caps->iv1l, be32_to_cpu(*iv++)); 346 stm32_cryp_write(cryp, cryp->caps->iv1r, be32_to_cpu(*iv++)); 347 } 348 } 349 350 static void stm32_cryp_get_iv(struct stm32_cryp *cryp) 351 { 352 struct skcipher_request *req = cryp->req; 353 __be32 *tmp = (void *)req->iv; 354 355 if (!tmp) 356 return; 357 358 if (cryp->caps->iv_protection) 359 stm32_cryp_key_read_enable(cryp); 360 361 *tmp++ = cpu_to_be32(stm32_cryp_read(cryp, cryp->caps->iv0l)); 362 *tmp++ = cpu_to_be32(stm32_cryp_read(cryp, cryp->caps->iv0r)); 363 364 if (is_aes(cryp)) { 365 *tmp++ = cpu_to_be32(stm32_cryp_read(cryp, cryp->caps->iv1l)); 366 *tmp++ = cpu_to_be32(stm32_cryp_read(cryp, cryp->caps->iv1r)); 367 } 368 369 if (cryp->caps->iv_protection) 370 stm32_cryp_key_read_disable(cryp); 371 } 372 373 /** 374 * ux500_swap_bits_in_byte() - mirror the bits in a byte 375 * @b: the byte to be mirrored 376 * 377 * The bits are swapped the following way: 378 * Byte b include bits 0-7, nibble 1 (n1) include bits 0-3 and 379 * nibble 2 (n2) bits 4-7. 380 * 381 * Nibble 1 (n1): 382 * (The "old" (moved) bit is replaced with a zero) 383 * 1. Move bit 6 and 7, 4 positions to the left. 384 * 2. Move bit 3 and 5, 2 positions to the left. 385 * 3. Move bit 1-4, 1 position to the left. 386 * 387 * Nibble 2 (n2): 388 * 1. Move bit 0 and 1, 4 positions to the right. 389 * 2. Move bit 2 and 4, 2 positions to the right. 390 * 3. Move bit 3-6, 1 position to the right. 391 * 392 * Combine the two nibbles to a complete and swapped byte. 393 */ 394 static inline u8 ux500_swap_bits_in_byte(u8 b) 395 { 396 #define R_SHIFT_4_MASK 0xc0 /* Bits 6 and 7, right shift 4 */ 397 #define R_SHIFT_2_MASK 0x28 /* (After right shift 4) Bits 3 and 5, 398 right shift 2 */ 399 #define R_SHIFT_1_MASK 0x1e /* (After right shift 2) Bits 1-4, 400 right shift 1 */ 401 #define L_SHIFT_4_MASK 0x03 /* Bits 0 and 1, left shift 4 */ 402 #define L_SHIFT_2_MASK 0x14 /* (After left shift 4) Bits 2 and 4, 403 left shift 2 */ 404 #define L_SHIFT_1_MASK 0x78 /* (After left shift 1) Bits 3-6, 405 left shift 1 */ 406 407 u8 n1; 408 u8 n2; 409 410 /* Swap most significant nibble */ 411 /* Right shift 4, bits 6 and 7 */ 412 n1 = ((b & R_SHIFT_4_MASK) >> 4) | (b & ~(R_SHIFT_4_MASK >> 4)); 413 /* Right shift 2, bits 3 and 5 */ 414 n1 = ((n1 & R_SHIFT_2_MASK) >> 2) | (n1 & ~(R_SHIFT_2_MASK >> 2)); 415 /* Right shift 1, bits 1-4 */ 416 n1 = (n1 & R_SHIFT_1_MASK) >> 1; 417 418 /* Swap least significant nibble */ 419 /* Left shift 4, bits 0 and 1 */ 420 n2 = ((b & L_SHIFT_4_MASK) << 4) | (b & ~(L_SHIFT_4_MASK << 4)); 421 /* Left shift 2, bits 2 and 4 */ 422 n2 = ((n2 & L_SHIFT_2_MASK) << 2) | (n2 & ~(L_SHIFT_2_MASK << 2)); 423 /* Left shift 1, bits 3-6 */ 424 n2 = (n2 & L_SHIFT_1_MASK) << 1; 425 426 return n1 | n2; 427 } 428 429 /** 430 * ux500_swizzle_key() - Shuffle around words and bits in the AES key 431 * @in: key to swizzle 432 * @out: swizzled key 433 * @len: length of key, in bytes 434 * 435 * This "key swizzling procedure" is described in the examples in the 436 * DB8500 design specification. There is no real description of why 437 * the bits have been arranged like this in the hardware. 438 */ 439 static inline void ux500_swizzle_key(const u8 *in, u8 *out, u32 len) 440 { 441 int i = 0; 442 int bpw = sizeof(u32); 443 int j; 444 int index = 0; 445 446 j = len - bpw; 447 while (j >= 0) { 448 for (i = 0; i < bpw; i++) { 449 index = len - j - bpw + i; 450 out[j + i] = 451 ux500_swap_bits_in_byte(in[index]); 452 } 453 j -= bpw; 454 } 455 } 456 457 static void stm32_cryp_hw_write_key(struct stm32_cryp *c) 458 { 459 unsigned int i; 460 int r_id; 461 462 if (is_des(c)) { 463 stm32_cryp_write(c, c->caps->k1l, be32_to_cpu(c->ctx->key[0])); 464 stm32_cryp_write(c, c->caps->k1r, be32_to_cpu(c->ctx->key[1])); 465 return; 466 } 467 468 /* 469 * On the Ux500 the AES key is considered as a single bit sequence 470 * of 128, 192 or 256 bits length. It is written linearly into the 471 * registers from K1L and down, and need to be processed to become 472 * a proper big-endian bit sequence. 473 */ 474 if (is_aes(c) && c->caps->linear_aes_key) { 475 u32 tmpkey[8]; 476 477 ux500_swizzle_key((u8 *)c->ctx->key, 478 (u8 *)tmpkey, c->ctx->keylen); 479 480 r_id = c->caps->k1l; 481 for (i = 0; i < c->ctx->keylen / sizeof(u32); i++, r_id += 4) 482 stm32_cryp_write(c, r_id, tmpkey[i]); 483 484 return; 485 } 486 487 r_id = c->caps->k3r; 488 for (i = c->ctx->keylen / sizeof(u32); i > 0; i--, r_id -= 4) 489 stm32_cryp_write(c, r_id, be32_to_cpu(c->ctx->key[i - 1])); 490 } 491 492 static u32 stm32_cryp_get_hw_mode(struct stm32_cryp *cryp) 493 { 494 if (is_aes(cryp) && is_ecb(cryp)) 495 return CR_AES_ECB; 496 497 if (is_aes(cryp) && is_cbc(cryp)) 498 return CR_AES_CBC; 499 500 if (is_aes(cryp) && is_ctr(cryp)) 501 return CR_AES_CTR; 502 503 if (is_aes(cryp) && is_gcm(cryp)) 504 return CR_AES_GCM; 505 506 if (is_aes(cryp) && is_ccm(cryp)) 507 return CR_AES_CCM; 508 509 if (is_des(cryp) && is_ecb(cryp)) 510 return CR_DES_ECB; 511 512 if (is_des(cryp) && is_cbc(cryp)) 513 return CR_DES_CBC; 514 515 if (is_tdes(cryp) && is_ecb(cryp)) 516 return CR_TDES_ECB; 517 518 if (is_tdes(cryp) && is_cbc(cryp)) 519 return CR_TDES_CBC; 520 521 dev_err(cryp->dev, "Unknown mode\n"); 522 return CR_AES_UNKNOWN; 523 } 524 525 static unsigned int stm32_cryp_get_input_text_len(struct stm32_cryp *cryp) 526 { 527 return is_encrypt(cryp) ? cryp->areq->cryptlen : 528 cryp->areq->cryptlen - cryp->authsize; 529 } 530 531 static int stm32_cryp_gcm_init(struct stm32_cryp *cryp, u32 cfg) 532 { 533 int ret; 534 __be32 iv[4]; 535 536 /* Phase 1 : init */ 537 memcpy(iv, cryp->areq->iv, 12); 538 iv[3] = cpu_to_be32(GCM_CTR_INIT); 539 cryp->gcm_ctr = GCM_CTR_INIT; 540 stm32_cryp_hw_write_iv(cryp, iv); 541 542 stm32_cryp_write(cryp, cryp->caps->cr, cfg | CR_PH_INIT | CR_CRYPEN); 543 544 /* Wait for end of processing */ 545 ret = stm32_cryp_wait_enable(cryp); 546 if (ret) { 547 dev_err(cryp->dev, "Timeout (gcm init)\n"); 548 return ret; 549 } 550 551 /* Prepare next phase */ 552 if (cryp->areq->assoclen) { 553 cfg |= CR_PH_HEADER; 554 stm32_cryp_write(cryp, cryp->caps->cr, cfg); 555 } else if (stm32_cryp_get_input_text_len(cryp)) { 556 cfg |= CR_PH_PAYLOAD; 557 stm32_cryp_write(cryp, cryp->caps->cr, cfg); 558 } 559 560 return 0; 561 } 562 563 static void stm32_crypt_gcmccm_end_header(struct stm32_cryp *cryp) 564 { 565 u32 cfg; 566 int err; 567 568 /* Check if whole header written */ 569 if (!cryp->header_in) { 570 /* Wait for completion */ 571 err = stm32_cryp_wait_busy(cryp); 572 if (err) { 573 dev_err(cryp->dev, "Timeout (gcm/ccm header)\n"); 574 stm32_cryp_write(cryp, cryp->caps->imsc, 0); 575 stm32_cryp_finish_req(cryp, err); 576 return; 577 } 578 579 if (stm32_cryp_get_input_text_len(cryp)) { 580 /* Phase 3 : payload */ 581 cfg = stm32_cryp_read(cryp, cryp->caps->cr); 582 cfg &= ~CR_CRYPEN; 583 stm32_cryp_write(cryp, cryp->caps->cr, cfg); 584 585 cfg &= ~CR_PH_MASK; 586 cfg |= CR_PH_PAYLOAD | CR_CRYPEN; 587 stm32_cryp_write(cryp, cryp->caps->cr, cfg); 588 } else { 589 /* 590 * Phase 4 : tag. 591 * Nothing to read, nothing to write, caller have to 592 * end request 593 */ 594 } 595 } 596 } 597 598 static void stm32_cryp_write_ccm_first_header(struct stm32_cryp *cryp) 599 { 600 size_t written; 601 size_t len; 602 u32 alen = cryp->areq->assoclen; 603 u32 block[AES_BLOCK_32] = {0}; 604 u8 *b8 = (u8 *)block; 605 606 if (alen <= 65280) { 607 /* Write first u32 of B1 */ 608 b8[0] = (alen >> 8) & 0xFF; 609 b8[1] = alen & 0xFF; 610 len = 2; 611 } else { 612 /* Build the two first u32 of B1 */ 613 b8[0] = 0xFF; 614 b8[1] = 0xFE; 615 b8[2] = (alen & 0xFF000000) >> 24; 616 b8[3] = (alen & 0x00FF0000) >> 16; 617 b8[4] = (alen & 0x0000FF00) >> 8; 618 b8[5] = alen & 0x000000FF; 619 len = 6; 620 } 621 622 written = min_t(size_t, AES_BLOCK_SIZE - len, alen); 623 624 scatterwalk_copychunks((char *)block + len, &cryp->in_walk, written, 0); 625 626 writesl(cryp->regs + cryp->caps->din, block, AES_BLOCK_32); 627 628 cryp->header_in -= written; 629 630 stm32_crypt_gcmccm_end_header(cryp); 631 } 632 633 static int stm32_cryp_ccm_init(struct stm32_cryp *cryp, u32 cfg) 634 { 635 int ret; 636 u32 iv_32[AES_BLOCK_32], b0_32[AES_BLOCK_32]; 637 u8 *iv = (u8 *)iv_32, *b0 = (u8 *)b0_32; 638 __be32 *bd; 639 u32 *d; 640 unsigned int i, textlen; 641 642 /* Phase 1 : init. Firstly set the CTR value to 1 (not 0) */ 643 memcpy(iv, cryp->areq->iv, AES_BLOCK_SIZE); 644 memset(iv + AES_BLOCK_SIZE - 1 - iv[0], 0, iv[0] + 1); 645 iv[AES_BLOCK_SIZE - 1] = 1; 646 stm32_cryp_hw_write_iv(cryp, (__be32 *)iv); 647 648 /* Build B0 */ 649 memcpy(b0, iv, AES_BLOCK_SIZE); 650 651 b0[0] |= (8 * ((cryp->authsize - 2) / 2)); 652 653 if (cryp->areq->assoclen) 654 b0[0] |= 0x40; 655 656 textlen = stm32_cryp_get_input_text_len(cryp); 657 658 b0[AES_BLOCK_SIZE - 2] = textlen >> 8; 659 b0[AES_BLOCK_SIZE - 1] = textlen & 0xFF; 660 661 /* Enable HW */ 662 stm32_cryp_write(cryp, cryp->caps->cr, cfg | CR_PH_INIT | CR_CRYPEN); 663 664 /* Write B0 */ 665 d = (u32 *)b0; 666 bd = (__be32 *)b0; 667 668 for (i = 0; i < AES_BLOCK_32; i++) { 669 u32 xd = d[i]; 670 671 if (!cryp->caps->padding_wa) 672 xd = be32_to_cpu(bd[i]); 673 stm32_cryp_write(cryp, cryp->caps->din, xd); 674 } 675 676 /* Wait for end of processing */ 677 ret = stm32_cryp_wait_enable(cryp); 678 if (ret) { 679 dev_err(cryp->dev, "Timeout (ccm init)\n"); 680 return ret; 681 } 682 683 /* Prepare next phase */ 684 if (cryp->areq->assoclen) { 685 cfg |= CR_PH_HEADER | CR_CRYPEN; 686 stm32_cryp_write(cryp, cryp->caps->cr, cfg); 687 688 /* Write first (special) block (may move to next phase [payload]) */ 689 stm32_cryp_write_ccm_first_header(cryp); 690 } else if (stm32_cryp_get_input_text_len(cryp)) { 691 cfg |= CR_PH_PAYLOAD; 692 stm32_cryp_write(cryp, cryp->caps->cr, cfg); 693 } 694 695 return 0; 696 } 697 698 static int stm32_cryp_hw_init(struct stm32_cryp *cryp) 699 { 700 int ret; 701 u32 cfg, hw_mode; 702 703 pm_runtime_get_sync(cryp->dev); 704 705 /* Disable interrupt */ 706 stm32_cryp_write(cryp, cryp->caps->imsc, 0); 707 708 /* Set configuration */ 709 cfg = CR_DATA8 | CR_FFLUSH; 710 711 switch (cryp->ctx->keylen) { 712 case AES_KEYSIZE_128: 713 cfg |= CR_KEY128; 714 break; 715 716 case AES_KEYSIZE_192: 717 cfg |= CR_KEY192; 718 break; 719 720 default: 721 case AES_KEYSIZE_256: 722 cfg |= CR_KEY256; 723 break; 724 } 725 726 hw_mode = stm32_cryp_get_hw_mode(cryp); 727 if (hw_mode == CR_AES_UNKNOWN) 728 return -EINVAL; 729 730 /* AES ECB/CBC decrypt: run key preparation first */ 731 if (is_decrypt(cryp) && 732 ((hw_mode == CR_AES_ECB) || (hw_mode == CR_AES_CBC))) { 733 /* Configure in key preparation mode */ 734 if (cryp->caps->kp_mode) 735 stm32_cryp_write(cryp, cryp->caps->cr, 736 cfg | CR_AES_KP); 737 else 738 stm32_cryp_write(cryp, 739 cryp->caps->cr, cfg | CR_AES_ECB | CR_KSE); 740 741 /* Set key only after full configuration done */ 742 stm32_cryp_hw_write_key(cryp); 743 744 /* Start prepare key */ 745 stm32_cryp_enable(cryp); 746 /* Wait for end of processing */ 747 ret = stm32_cryp_wait_busy(cryp); 748 if (ret) { 749 dev_err(cryp->dev, "Timeout (key preparation)\n"); 750 return ret; 751 } 752 753 cfg |= hw_mode | CR_DEC_NOT_ENC; 754 755 /* Apply updated config (Decrypt + algo) and flush */ 756 stm32_cryp_write(cryp, cryp->caps->cr, cfg); 757 } else { 758 cfg |= hw_mode; 759 if (is_decrypt(cryp)) 760 cfg |= CR_DEC_NOT_ENC; 761 762 /* Apply config and flush */ 763 stm32_cryp_write(cryp, cryp->caps->cr, cfg); 764 765 /* Set key only after configuration done */ 766 stm32_cryp_hw_write_key(cryp); 767 } 768 769 switch (hw_mode) { 770 case CR_AES_GCM: 771 case CR_AES_CCM: 772 /* Phase 1 : init */ 773 if (hw_mode == CR_AES_CCM) 774 ret = stm32_cryp_ccm_init(cryp, cfg); 775 else 776 ret = stm32_cryp_gcm_init(cryp, cfg); 777 778 if (ret) 779 return ret; 780 781 break; 782 783 case CR_DES_CBC: 784 case CR_TDES_CBC: 785 case CR_AES_CBC: 786 case CR_AES_CTR: 787 stm32_cryp_hw_write_iv(cryp, (__be32 *)cryp->req->iv); 788 break; 789 790 default: 791 break; 792 } 793 794 /* Enable now */ 795 stm32_cryp_enable(cryp); 796 797 return 0; 798 } 799 800 static void stm32_cryp_finish_req(struct stm32_cryp *cryp, int err) 801 { 802 if (!err && (is_gcm(cryp) || is_ccm(cryp))) 803 /* Phase 4 : output tag */ 804 err = stm32_cryp_read_auth_tag(cryp); 805 806 if (!err && (!(is_gcm(cryp) || is_ccm(cryp) || is_ecb(cryp)))) 807 stm32_cryp_get_iv(cryp); 808 809 pm_runtime_mark_last_busy(cryp->dev); 810 pm_runtime_put_autosuspend(cryp->dev); 811 812 if (is_gcm(cryp) || is_ccm(cryp)) 813 crypto_finalize_aead_request(cryp->engine, cryp->areq, err); 814 else 815 crypto_finalize_skcipher_request(cryp->engine, cryp->req, 816 err); 817 } 818 819 static int stm32_cryp_cpu_start(struct stm32_cryp *cryp) 820 { 821 /* Enable interrupt and let the IRQ handler do everything */ 822 stm32_cryp_write(cryp, cryp->caps->imsc, IMSCR_IN | IMSCR_OUT); 823 824 return 0; 825 } 826 827 static int stm32_cryp_cipher_one_req(struct crypto_engine *engine, void *areq); 828 829 static int stm32_cryp_init_tfm(struct crypto_skcipher *tfm) 830 { 831 struct stm32_cryp_ctx *ctx = crypto_skcipher_ctx(tfm); 832 833 crypto_skcipher_set_reqsize(tfm, sizeof(struct stm32_cryp_reqctx)); 834 835 ctx->enginectx.op.do_one_request = stm32_cryp_cipher_one_req; 836 return 0; 837 } 838 839 static int stm32_cryp_aead_one_req(struct crypto_engine *engine, void *areq); 840 841 static int stm32_cryp_aes_aead_init(struct crypto_aead *tfm) 842 { 843 struct stm32_cryp_ctx *ctx = crypto_aead_ctx(tfm); 844 845 tfm->reqsize = sizeof(struct stm32_cryp_reqctx); 846 847 ctx->enginectx.op.do_one_request = stm32_cryp_aead_one_req; 848 849 return 0; 850 } 851 852 static int stm32_cryp_crypt(struct skcipher_request *req, unsigned long mode) 853 { 854 struct stm32_cryp_ctx *ctx = crypto_skcipher_ctx( 855 crypto_skcipher_reqtfm(req)); 856 struct stm32_cryp_reqctx *rctx = skcipher_request_ctx(req); 857 struct stm32_cryp *cryp = stm32_cryp_find_dev(ctx); 858 859 if (!cryp) 860 return -ENODEV; 861 862 rctx->mode = mode; 863 864 return crypto_transfer_skcipher_request_to_engine(cryp->engine, req); 865 } 866 867 static int stm32_cryp_aead_crypt(struct aead_request *req, unsigned long mode) 868 { 869 struct stm32_cryp_ctx *ctx = crypto_aead_ctx(crypto_aead_reqtfm(req)); 870 struct stm32_cryp_reqctx *rctx = aead_request_ctx(req); 871 struct stm32_cryp *cryp = stm32_cryp_find_dev(ctx); 872 873 if (!cryp) 874 return -ENODEV; 875 876 rctx->mode = mode; 877 878 return crypto_transfer_aead_request_to_engine(cryp->engine, req); 879 } 880 881 static int stm32_cryp_setkey(struct crypto_skcipher *tfm, const u8 *key, 882 unsigned int keylen) 883 { 884 struct stm32_cryp_ctx *ctx = crypto_skcipher_ctx(tfm); 885 886 memcpy(ctx->key, key, keylen); 887 ctx->keylen = keylen; 888 889 return 0; 890 } 891 892 static int stm32_cryp_aes_setkey(struct crypto_skcipher *tfm, const u8 *key, 893 unsigned int keylen) 894 { 895 if (keylen != AES_KEYSIZE_128 && keylen != AES_KEYSIZE_192 && 896 keylen != AES_KEYSIZE_256) 897 return -EINVAL; 898 else 899 return stm32_cryp_setkey(tfm, key, keylen); 900 } 901 902 static int stm32_cryp_des_setkey(struct crypto_skcipher *tfm, const u8 *key, 903 unsigned int keylen) 904 { 905 return verify_skcipher_des_key(tfm, key) ?: 906 stm32_cryp_setkey(tfm, key, keylen); 907 } 908 909 static int stm32_cryp_tdes_setkey(struct crypto_skcipher *tfm, const u8 *key, 910 unsigned int keylen) 911 { 912 return verify_skcipher_des3_key(tfm, key) ?: 913 stm32_cryp_setkey(tfm, key, keylen); 914 } 915 916 static int stm32_cryp_aes_aead_setkey(struct crypto_aead *tfm, const u8 *key, 917 unsigned int keylen) 918 { 919 struct stm32_cryp_ctx *ctx = crypto_aead_ctx(tfm); 920 921 if (keylen != AES_KEYSIZE_128 && keylen != AES_KEYSIZE_192 && 922 keylen != AES_KEYSIZE_256) 923 return -EINVAL; 924 925 memcpy(ctx->key, key, keylen); 926 ctx->keylen = keylen; 927 928 return 0; 929 } 930 931 static int stm32_cryp_aes_gcm_setauthsize(struct crypto_aead *tfm, 932 unsigned int authsize) 933 { 934 switch (authsize) { 935 case 4: 936 case 8: 937 case 12: 938 case 13: 939 case 14: 940 case 15: 941 case 16: 942 break; 943 default: 944 return -EINVAL; 945 } 946 947 return 0; 948 } 949 950 static int stm32_cryp_aes_ccm_setauthsize(struct crypto_aead *tfm, 951 unsigned int authsize) 952 { 953 switch (authsize) { 954 case 4: 955 case 6: 956 case 8: 957 case 10: 958 case 12: 959 case 14: 960 case 16: 961 break; 962 default: 963 return -EINVAL; 964 } 965 966 return 0; 967 } 968 969 static int stm32_cryp_aes_ecb_encrypt(struct skcipher_request *req) 970 { 971 if (req->cryptlen % AES_BLOCK_SIZE) 972 return -EINVAL; 973 974 if (req->cryptlen == 0) 975 return 0; 976 977 return stm32_cryp_crypt(req, FLG_AES | FLG_ECB | FLG_ENCRYPT); 978 } 979 980 static int stm32_cryp_aes_ecb_decrypt(struct skcipher_request *req) 981 { 982 if (req->cryptlen % AES_BLOCK_SIZE) 983 return -EINVAL; 984 985 if (req->cryptlen == 0) 986 return 0; 987 988 return stm32_cryp_crypt(req, FLG_AES | FLG_ECB); 989 } 990 991 static int stm32_cryp_aes_cbc_encrypt(struct skcipher_request *req) 992 { 993 if (req->cryptlen % AES_BLOCK_SIZE) 994 return -EINVAL; 995 996 if (req->cryptlen == 0) 997 return 0; 998 999 return stm32_cryp_crypt(req, FLG_AES | FLG_CBC | FLG_ENCRYPT); 1000 } 1001 1002 static int stm32_cryp_aes_cbc_decrypt(struct skcipher_request *req) 1003 { 1004 if (req->cryptlen % AES_BLOCK_SIZE) 1005 return -EINVAL; 1006 1007 if (req->cryptlen == 0) 1008 return 0; 1009 1010 return stm32_cryp_crypt(req, FLG_AES | FLG_CBC); 1011 } 1012 1013 static int stm32_cryp_aes_ctr_encrypt(struct skcipher_request *req) 1014 { 1015 if (req->cryptlen == 0) 1016 return 0; 1017 1018 return stm32_cryp_crypt(req, FLG_AES | FLG_CTR | FLG_ENCRYPT); 1019 } 1020 1021 static int stm32_cryp_aes_ctr_decrypt(struct skcipher_request *req) 1022 { 1023 if (req->cryptlen == 0) 1024 return 0; 1025 1026 return stm32_cryp_crypt(req, FLG_AES | FLG_CTR); 1027 } 1028 1029 static int stm32_cryp_aes_gcm_encrypt(struct aead_request *req) 1030 { 1031 return stm32_cryp_aead_crypt(req, FLG_AES | FLG_GCM | FLG_ENCRYPT); 1032 } 1033 1034 static int stm32_cryp_aes_gcm_decrypt(struct aead_request *req) 1035 { 1036 return stm32_cryp_aead_crypt(req, FLG_AES | FLG_GCM); 1037 } 1038 1039 static inline int crypto_ccm_check_iv(const u8 *iv) 1040 { 1041 /* 2 <= L <= 8, so 1 <= L' <= 7. */ 1042 if (iv[0] < 1 || iv[0] > 7) 1043 return -EINVAL; 1044 1045 return 0; 1046 } 1047 1048 static int stm32_cryp_aes_ccm_encrypt(struct aead_request *req) 1049 { 1050 int err; 1051 1052 err = crypto_ccm_check_iv(req->iv); 1053 if (err) 1054 return err; 1055 1056 return stm32_cryp_aead_crypt(req, FLG_AES | FLG_CCM | FLG_ENCRYPT); 1057 } 1058 1059 static int stm32_cryp_aes_ccm_decrypt(struct aead_request *req) 1060 { 1061 int err; 1062 1063 err = crypto_ccm_check_iv(req->iv); 1064 if (err) 1065 return err; 1066 1067 return stm32_cryp_aead_crypt(req, FLG_AES | FLG_CCM); 1068 } 1069 1070 static int stm32_cryp_des_ecb_encrypt(struct skcipher_request *req) 1071 { 1072 if (req->cryptlen % DES_BLOCK_SIZE) 1073 return -EINVAL; 1074 1075 if (req->cryptlen == 0) 1076 return 0; 1077 1078 return stm32_cryp_crypt(req, FLG_DES | FLG_ECB | FLG_ENCRYPT); 1079 } 1080 1081 static int stm32_cryp_des_ecb_decrypt(struct skcipher_request *req) 1082 { 1083 if (req->cryptlen % DES_BLOCK_SIZE) 1084 return -EINVAL; 1085 1086 if (req->cryptlen == 0) 1087 return 0; 1088 1089 return stm32_cryp_crypt(req, FLG_DES | FLG_ECB); 1090 } 1091 1092 static int stm32_cryp_des_cbc_encrypt(struct skcipher_request *req) 1093 { 1094 if (req->cryptlen % DES_BLOCK_SIZE) 1095 return -EINVAL; 1096 1097 if (req->cryptlen == 0) 1098 return 0; 1099 1100 return stm32_cryp_crypt(req, FLG_DES | FLG_CBC | FLG_ENCRYPT); 1101 } 1102 1103 static int stm32_cryp_des_cbc_decrypt(struct skcipher_request *req) 1104 { 1105 if (req->cryptlen % DES_BLOCK_SIZE) 1106 return -EINVAL; 1107 1108 if (req->cryptlen == 0) 1109 return 0; 1110 1111 return stm32_cryp_crypt(req, FLG_DES | FLG_CBC); 1112 } 1113 1114 static int stm32_cryp_tdes_ecb_encrypt(struct skcipher_request *req) 1115 { 1116 if (req->cryptlen % DES_BLOCK_SIZE) 1117 return -EINVAL; 1118 1119 if (req->cryptlen == 0) 1120 return 0; 1121 1122 return stm32_cryp_crypt(req, FLG_TDES | FLG_ECB | FLG_ENCRYPT); 1123 } 1124 1125 static int stm32_cryp_tdes_ecb_decrypt(struct skcipher_request *req) 1126 { 1127 if (req->cryptlen % DES_BLOCK_SIZE) 1128 return -EINVAL; 1129 1130 if (req->cryptlen == 0) 1131 return 0; 1132 1133 return stm32_cryp_crypt(req, FLG_TDES | FLG_ECB); 1134 } 1135 1136 static int stm32_cryp_tdes_cbc_encrypt(struct skcipher_request *req) 1137 { 1138 if (req->cryptlen % DES_BLOCK_SIZE) 1139 return -EINVAL; 1140 1141 if (req->cryptlen == 0) 1142 return 0; 1143 1144 return stm32_cryp_crypt(req, FLG_TDES | FLG_CBC | FLG_ENCRYPT); 1145 } 1146 1147 static int stm32_cryp_tdes_cbc_decrypt(struct skcipher_request *req) 1148 { 1149 if (req->cryptlen % DES_BLOCK_SIZE) 1150 return -EINVAL; 1151 1152 if (req->cryptlen == 0) 1153 return 0; 1154 1155 return stm32_cryp_crypt(req, FLG_TDES | FLG_CBC); 1156 } 1157 1158 static int stm32_cryp_prepare_req(struct skcipher_request *req, 1159 struct aead_request *areq) 1160 { 1161 struct stm32_cryp_ctx *ctx; 1162 struct stm32_cryp *cryp; 1163 struct stm32_cryp_reqctx *rctx; 1164 struct scatterlist *in_sg; 1165 int ret; 1166 1167 if (!req && !areq) 1168 return -EINVAL; 1169 1170 ctx = req ? crypto_skcipher_ctx(crypto_skcipher_reqtfm(req)) : 1171 crypto_aead_ctx(crypto_aead_reqtfm(areq)); 1172 1173 cryp = ctx->cryp; 1174 1175 rctx = req ? skcipher_request_ctx(req) : aead_request_ctx(areq); 1176 rctx->mode &= FLG_MODE_MASK; 1177 1178 ctx->cryp = cryp; 1179 1180 cryp->flags = (cryp->flags & ~FLG_MODE_MASK) | rctx->mode; 1181 cryp->hw_blocksize = is_aes(cryp) ? AES_BLOCK_SIZE : DES_BLOCK_SIZE; 1182 cryp->ctx = ctx; 1183 1184 if (req) { 1185 cryp->req = req; 1186 cryp->areq = NULL; 1187 cryp->header_in = 0; 1188 cryp->payload_in = req->cryptlen; 1189 cryp->payload_out = req->cryptlen; 1190 cryp->authsize = 0; 1191 } else { 1192 /* 1193 * Length of input and output data: 1194 * Encryption case: 1195 * INPUT = AssocData || PlainText 1196 * <- assoclen -> <- cryptlen -> 1197 * 1198 * OUTPUT = AssocData || CipherText || AuthTag 1199 * <- assoclen -> <-- cryptlen --> <- authsize -> 1200 * 1201 * Decryption case: 1202 * INPUT = AssocData || CipherTex || AuthTag 1203 * <- assoclen ---> <---------- cryptlen ----------> 1204 * 1205 * OUTPUT = AssocData || PlainText 1206 * <- assoclen -> <- cryptlen - authsize -> 1207 */ 1208 cryp->areq = areq; 1209 cryp->req = NULL; 1210 cryp->authsize = crypto_aead_authsize(crypto_aead_reqtfm(areq)); 1211 if (is_encrypt(cryp)) { 1212 cryp->payload_in = areq->cryptlen; 1213 cryp->header_in = areq->assoclen; 1214 cryp->payload_out = areq->cryptlen; 1215 } else { 1216 cryp->payload_in = areq->cryptlen - cryp->authsize; 1217 cryp->header_in = areq->assoclen; 1218 cryp->payload_out = cryp->payload_in; 1219 } 1220 } 1221 1222 in_sg = req ? req->src : areq->src; 1223 scatterwalk_start(&cryp->in_walk, in_sg); 1224 1225 cryp->out_sg = req ? req->dst : areq->dst; 1226 scatterwalk_start(&cryp->out_walk, cryp->out_sg); 1227 1228 if (is_gcm(cryp) || is_ccm(cryp)) { 1229 /* In output, jump after assoc data */ 1230 scatterwalk_copychunks(NULL, &cryp->out_walk, cryp->areq->assoclen, 2); 1231 } 1232 1233 if (is_ctr(cryp)) 1234 memset(cryp->last_ctr, 0, sizeof(cryp->last_ctr)); 1235 1236 ret = stm32_cryp_hw_init(cryp); 1237 return ret; 1238 } 1239 1240 static int stm32_cryp_cipher_one_req(struct crypto_engine *engine, void *areq) 1241 { 1242 struct skcipher_request *req = container_of(areq, 1243 struct skcipher_request, 1244 base); 1245 struct stm32_cryp_ctx *ctx = crypto_skcipher_ctx( 1246 crypto_skcipher_reqtfm(req)); 1247 struct stm32_cryp *cryp = ctx->cryp; 1248 1249 if (!cryp) 1250 return -ENODEV; 1251 1252 return stm32_cryp_prepare_req(req, NULL) ?: 1253 stm32_cryp_cpu_start(cryp); 1254 } 1255 1256 static int stm32_cryp_aead_one_req(struct crypto_engine *engine, void *areq) 1257 { 1258 struct aead_request *req = container_of(areq, struct aead_request, 1259 base); 1260 struct stm32_cryp_ctx *ctx = crypto_aead_ctx(crypto_aead_reqtfm(req)); 1261 struct stm32_cryp *cryp = ctx->cryp; 1262 int err; 1263 1264 if (!cryp) 1265 return -ENODEV; 1266 1267 err = stm32_cryp_prepare_req(NULL, req); 1268 if (err) 1269 return err; 1270 1271 if (unlikely(!cryp->payload_in && !cryp->header_in)) { 1272 /* No input data to process: get tag and finish */ 1273 stm32_cryp_finish_req(cryp, 0); 1274 return 0; 1275 } 1276 1277 return stm32_cryp_cpu_start(cryp); 1278 } 1279 1280 static int stm32_cryp_read_auth_tag(struct stm32_cryp *cryp) 1281 { 1282 u32 cfg, size_bit; 1283 unsigned int i; 1284 int ret = 0; 1285 1286 /* Update Config */ 1287 cfg = stm32_cryp_read(cryp, cryp->caps->cr); 1288 1289 cfg &= ~CR_PH_MASK; 1290 cfg |= CR_PH_FINAL; 1291 cfg &= ~CR_DEC_NOT_ENC; 1292 cfg |= CR_CRYPEN; 1293 1294 stm32_cryp_write(cryp, cryp->caps->cr, cfg); 1295 1296 if (is_gcm(cryp)) { 1297 /* GCM: write aad and payload size (in bits) */ 1298 size_bit = cryp->areq->assoclen * 8; 1299 if (cryp->caps->swap_final) 1300 size_bit = (__force u32)cpu_to_be32(size_bit); 1301 1302 stm32_cryp_write(cryp, cryp->caps->din, 0); 1303 stm32_cryp_write(cryp, cryp->caps->din, size_bit); 1304 1305 size_bit = is_encrypt(cryp) ? cryp->areq->cryptlen : 1306 cryp->areq->cryptlen - cryp->authsize; 1307 size_bit *= 8; 1308 if (cryp->caps->swap_final) 1309 size_bit = (__force u32)cpu_to_be32(size_bit); 1310 1311 stm32_cryp_write(cryp, cryp->caps->din, 0); 1312 stm32_cryp_write(cryp, cryp->caps->din, size_bit); 1313 } else { 1314 /* CCM: write CTR0 */ 1315 u32 iv32[AES_BLOCK_32]; 1316 u8 *iv = (u8 *)iv32; 1317 __be32 *biv = (__be32 *)iv32; 1318 1319 memcpy(iv, cryp->areq->iv, AES_BLOCK_SIZE); 1320 memset(iv + AES_BLOCK_SIZE - 1 - iv[0], 0, iv[0] + 1); 1321 1322 for (i = 0; i < AES_BLOCK_32; i++) { 1323 u32 xiv = iv32[i]; 1324 1325 if (!cryp->caps->padding_wa) 1326 xiv = be32_to_cpu(biv[i]); 1327 stm32_cryp_write(cryp, cryp->caps->din, xiv); 1328 } 1329 } 1330 1331 /* Wait for output data */ 1332 ret = stm32_cryp_wait_output(cryp); 1333 if (ret) { 1334 dev_err(cryp->dev, "Timeout (read tag)\n"); 1335 return ret; 1336 } 1337 1338 if (is_encrypt(cryp)) { 1339 u32 out_tag[AES_BLOCK_32]; 1340 1341 /* Get and write tag */ 1342 readsl(cryp->regs + cryp->caps->dout, out_tag, AES_BLOCK_32); 1343 scatterwalk_copychunks(out_tag, &cryp->out_walk, cryp->authsize, 1); 1344 } else { 1345 /* Get and check tag */ 1346 u32 in_tag[AES_BLOCK_32], out_tag[AES_BLOCK_32]; 1347 1348 scatterwalk_copychunks(in_tag, &cryp->in_walk, cryp->authsize, 0); 1349 readsl(cryp->regs + cryp->caps->dout, out_tag, AES_BLOCK_32); 1350 1351 if (crypto_memneq(in_tag, out_tag, cryp->authsize)) 1352 ret = -EBADMSG; 1353 } 1354 1355 /* Disable cryp */ 1356 cfg &= ~CR_CRYPEN; 1357 stm32_cryp_write(cryp, cryp->caps->cr, cfg); 1358 1359 return ret; 1360 } 1361 1362 static void stm32_cryp_check_ctr_counter(struct stm32_cryp *cryp) 1363 { 1364 u32 cr; 1365 1366 if (unlikely(cryp->last_ctr[3] == cpu_to_be32(0xFFFFFFFF))) { 1367 /* 1368 * In this case, we need to increment manually the ctr counter, 1369 * as HW doesn't handle the U32 carry. 1370 */ 1371 crypto_inc((u8 *)cryp->last_ctr, sizeof(cryp->last_ctr)); 1372 1373 cr = stm32_cryp_read(cryp, cryp->caps->cr); 1374 stm32_cryp_write(cryp, cryp->caps->cr, cr & ~CR_CRYPEN); 1375 1376 stm32_cryp_hw_write_iv(cryp, cryp->last_ctr); 1377 1378 stm32_cryp_write(cryp, cryp->caps->cr, cr); 1379 } 1380 1381 /* The IV registers are BE */ 1382 cryp->last_ctr[0] = cpu_to_be32(stm32_cryp_read(cryp, cryp->caps->iv0l)); 1383 cryp->last_ctr[1] = cpu_to_be32(stm32_cryp_read(cryp, cryp->caps->iv0r)); 1384 cryp->last_ctr[2] = cpu_to_be32(stm32_cryp_read(cryp, cryp->caps->iv1l)); 1385 cryp->last_ctr[3] = cpu_to_be32(stm32_cryp_read(cryp, cryp->caps->iv1r)); 1386 } 1387 1388 static void stm32_cryp_irq_read_data(struct stm32_cryp *cryp) 1389 { 1390 u32 block[AES_BLOCK_32]; 1391 1392 readsl(cryp->regs + cryp->caps->dout, block, cryp->hw_blocksize / sizeof(u32)); 1393 scatterwalk_copychunks(block, &cryp->out_walk, min_t(size_t, cryp->hw_blocksize, 1394 cryp->payload_out), 1); 1395 cryp->payload_out -= min_t(size_t, cryp->hw_blocksize, 1396 cryp->payload_out); 1397 } 1398 1399 static void stm32_cryp_irq_write_block(struct stm32_cryp *cryp) 1400 { 1401 u32 block[AES_BLOCK_32] = {0}; 1402 1403 scatterwalk_copychunks(block, &cryp->in_walk, min_t(size_t, cryp->hw_blocksize, 1404 cryp->payload_in), 0); 1405 writesl(cryp->regs + cryp->caps->din, block, cryp->hw_blocksize / sizeof(u32)); 1406 cryp->payload_in -= min_t(size_t, cryp->hw_blocksize, cryp->payload_in); 1407 } 1408 1409 static void stm32_cryp_irq_write_gcm_padded_data(struct stm32_cryp *cryp) 1410 { 1411 int err; 1412 u32 cfg, block[AES_BLOCK_32] = {0}; 1413 unsigned int i; 1414 1415 /* 'Special workaround' procedure described in the datasheet */ 1416 1417 /* a) disable ip */ 1418 stm32_cryp_write(cryp, cryp->caps->imsc, 0); 1419 cfg = stm32_cryp_read(cryp, cryp->caps->cr); 1420 cfg &= ~CR_CRYPEN; 1421 stm32_cryp_write(cryp, cryp->caps->cr, cfg); 1422 1423 /* b) Update IV1R */ 1424 stm32_cryp_write(cryp, cryp->caps->iv1r, cryp->gcm_ctr - 2); 1425 1426 /* c) change mode to CTR */ 1427 cfg &= ~CR_ALGO_MASK; 1428 cfg |= CR_AES_CTR; 1429 stm32_cryp_write(cryp, cryp->caps->cr, cfg); 1430 1431 /* a) enable IP */ 1432 cfg |= CR_CRYPEN; 1433 stm32_cryp_write(cryp, cryp->caps->cr, cfg); 1434 1435 /* b) pad and write the last block */ 1436 stm32_cryp_irq_write_block(cryp); 1437 /* wait end of process */ 1438 err = stm32_cryp_wait_output(cryp); 1439 if (err) { 1440 dev_err(cryp->dev, "Timeout (write gcm last data)\n"); 1441 return stm32_cryp_finish_req(cryp, err); 1442 } 1443 1444 /* c) get and store encrypted data */ 1445 /* 1446 * Same code as stm32_cryp_irq_read_data(), but we want to store 1447 * block value 1448 */ 1449 readsl(cryp->regs + cryp->caps->dout, block, cryp->hw_blocksize / sizeof(u32)); 1450 1451 scatterwalk_copychunks(block, &cryp->out_walk, min_t(size_t, cryp->hw_blocksize, 1452 cryp->payload_out), 1); 1453 cryp->payload_out -= min_t(size_t, cryp->hw_blocksize, 1454 cryp->payload_out); 1455 1456 /* d) change mode back to AES GCM */ 1457 cfg &= ~CR_ALGO_MASK; 1458 cfg |= CR_AES_GCM; 1459 stm32_cryp_write(cryp, cryp->caps->cr, cfg); 1460 1461 /* e) change phase to Final */ 1462 cfg &= ~CR_PH_MASK; 1463 cfg |= CR_PH_FINAL; 1464 stm32_cryp_write(cryp, cryp->caps->cr, cfg); 1465 1466 /* f) write padded data */ 1467 writesl(cryp->regs + cryp->caps->din, block, AES_BLOCK_32); 1468 1469 /* g) Empty fifo out */ 1470 err = stm32_cryp_wait_output(cryp); 1471 if (err) { 1472 dev_err(cryp->dev, "Timeout (write gcm padded data)\n"); 1473 return stm32_cryp_finish_req(cryp, err); 1474 } 1475 1476 for (i = 0; i < AES_BLOCK_32; i++) 1477 stm32_cryp_read(cryp, cryp->caps->dout); 1478 1479 /* h) run the he normal Final phase */ 1480 stm32_cryp_finish_req(cryp, 0); 1481 } 1482 1483 static void stm32_cryp_irq_set_npblb(struct stm32_cryp *cryp) 1484 { 1485 u32 cfg; 1486 1487 /* disable ip, set NPBLB and reneable ip */ 1488 cfg = stm32_cryp_read(cryp, cryp->caps->cr); 1489 cfg &= ~CR_CRYPEN; 1490 stm32_cryp_write(cryp, cryp->caps->cr, cfg); 1491 1492 cfg |= (cryp->hw_blocksize - cryp->payload_in) << CR_NBPBL_SHIFT; 1493 cfg |= CR_CRYPEN; 1494 stm32_cryp_write(cryp, cryp->caps->cr, cfg); 1495 } 1496 1497 static void stm32_cryp_irq_write_ccm_padded_data(struct stm32_cryp *cryp) 1498 { 1499 int err = 0; 1500 u32 cfg, iv1tmp; 1501 u32 cstmp1[AES_BLOCK_32], cstmp2[AES_BLOCK_32]; 1502 u32 block[AES_BLOCK_32] = {0}; 1503 unsigned int i; 1504 1505 /* 'Special workaround' procedure described in the datasheet */ 1506 1507 /* a) disable ip */ 1508 stm32_cryp_write(cryp, cryp->caps->imsc, 0); 1509 1510 cfg = stm32_cryp_read(cryp, cryp->caps->cr); 1511 cfg &= ~CR_CRYPEN; 1512 stm32_cryp_write(cryp, cryp->caps->cr, cfg); 1513 1514 /* b) get IV1 from CRYP_CSGCMCCM7 */ 1515 iv1tmp = stm32_cryp_read(cryp, CRYP_CSGCMCCM0R + 7 * 4); 1516 1517 /* c) Load CRYP_CSGCMCCMxR */ 1518 for (i = 0; i < ARRAY_SIZE(cstmp1); i++) 1519 cstmp1[i] = stm32_cryp_read(cryp, CRYP_CSGCMCCM0R + i * 4); 1520 1521 /* d) Write IV1R */ 1522 stm32_cryp_write(cryp, cryp->caps->iv1r, iv1tmp); 1523 1524 /* e) change mode to CTR */ 1525 cfg &= ~CR_ALGO_MASK; 1526 cfg |= CR_AES_CTR; 1527 stm32_cryp_write(cryp, cryp->caps->cr, cfg); 1528 1529 /* a) enable IP */ 1530 cfg |= CR_CRYPEN; 1531 stm32_cryp_write(cryp, cryp->caps->cr, cfg); 1532 1533 /* b) pad and write the last block */ 1534 stm32_cryp_irq_write_block(cryp); 1535 /* wait end of process */ 1536 err = stm32_cryp_wait_output(cryp); 1537 if (err) { 1538 dev_err(cryp->dev, "Timeout (write ccm padded data)\n"); 1539 return stm32_cryp_finish_req(cryp, err); 1540 } 1541 1542 /* c) get and store decrypted data */ 1543 /* 1544 * Same code as stm32_cryp_irq_read_data(), but we want to store 1545 * block value 1546 */ 1547 readsl(cryp->regs + cryp->caps->dout, block, cryp->hw_blocksize / sizeof(u32)); 1548 1549 scatterwalk_copychunks(block, &cryp->out_walk, min_t(size_t, cryp->hw_blocksize, 1550 cryp->payload_out), 1); 1551 cryp->payload_out -= min_t(size_t, cryp->hw_blocksize, cryp->payload_out); 1552 1553 /* d) Load again CRYP_CSGCMCCMxR */ 1554 for (i = 0; i < ARRAY_SIZE(cstmp2); i++) 1555 cstmp2[i] = stm32_cryp_read(cryp, CRYP_CSGCMCCM0R + i * 4); 1556 1557 /* e) change mode back to AES CCM */ 1558 cfg &= ~CR_ALGO_MASK; 1559 cfg |= CR_AES_CCM; 1560 stm32_cryp_write(cryp, cryp->caps->cr, cfg); 1561 1562 /* f) change phase to header */ 1563 cfg &= ~CR_PH_MASK; 1564 cfg |= CR_PH_HEADER; 1565 stm32_cryp_write(cryp, cryp->caps->cr, cfg); 1566 1567 /* g) XOR and write padded data */ 1568 for (i = 0; i < ARRAY_SIZE(block); i++) { 1569 block[i] ^= cstmp1[i]; 1570 block[i] ^= cstmp2[i]; 1571 stm32_cryp_write(cryp, cryp->caps->din, block[i]); 1572 } 1573 1574 /* h) wait for completion */ 1575 err = stm32_cryp_wait_busy(cryp); 1576 if (err) 1577 dev_err(cryp->dev, "Timeout (write ccm padded data)\n"); 1578 1579 /* i) run the he normal Final phase */ 1580 stm32_cryp_finish_req(cryp, err); 1581 } 1582 1583 static void stm32_cryp_irq_write_data(struct stm32_cryp *cryp) 1584 { 1585 if (unlikely(!cryp->payload_in)) { 1586 dev_warn(cryp->dev, "No more data to process\n"); 1587 return; 1588 } 1589 1590 if (unlikely(cryp->payload_in < AES_BLOCK_SIZE && 1591 (stm32_cryp_get_hw_mode(cryp) == CR_AES_GCM) && 1592 is_encrypt(cryp))) { 1593 /* Padding for AES GCM encryption */ 1594 if (cryp->caps->padding_wa) { 1595 /* Special case 1 */ 1596 stm32_cryp_irq_write_gcm_padded_data(cryp); 1597 return; 1598 } 1599 1600 /* Setting padding bytes (NBBLB) */ 1601 stm32_cryp_irq_set_npblb(cryp); 1602 } 1603 1604 if (unlikely((cryp->payload_in < AES_BLOCK_SIZE) && 1605 (stm32_cryp_get_hw_mode(cryp) == CR_AES_CCM) && 1606 is_decrypt(cryp))) { 1607 /* Padding for AES CCM decryption */ 1608 if (cryp->caps->padding_wa) { 1609 /* Special case 2 */ 1610 stm32_cryp_irq_write_ccm_padded_data(cryp); 1611 return; 1612 } 1613 1614 /* Setting padding bytes (NBBLB) */ 1615 stm32_cryp_irq_set_npblb(cryp); 1616 } 1617 1618 if (is_aes(cryp) && is_ctr(cryp)) 1619 stm32_cryp_check_ctr_counter(cryp); 1620 1621 stm32_cryp_irq_write_block(cryp); 1622 } 1623 1624 static void stm32_cryp_irq_write_gcmccm_header(struct stm32_cryp *cryp) 1625 { 1626 u32 block[AES_BLOCK_32] = {0}; 1627 size_t written; 1628 1629 written = min_t(size_t, AES_BLOCK_SIZE, cryp->header_in); 1630 1631 scatterwalk_copychunks(block, &cryp->in_walk, written, 0); 1632 1633 writesl(cryp->regs + cryp->caps->din, block, AES_BLOCK_32); 1634 1635 cryp->header_in -= written; 1636 1637 stm32_crypt_gcmccm_end_header(cryp); 1638 } 1639 1640 static irqreturn_t stm32_cryp_irq_thread(int irq, void *arg) 1641 { 1642 struct stm32_cryp *cryp = arg; 1643 u32 ph; 1644 u32 it_mask = stm32_cryp_read(cryp, cryp->caps->imsc); 1645 1646 if (cryp->irq_status & MISR_OUT) 1647 /* Output FIFO IRQ: read data */ 1648 stm32_cryp_irq_read_data(cryp); 1649 1650 if (cryp->irq_status & MISR_IN) { 1651 if (is_gcm(cryp) || is_ccm(cryp)) { 1652 ph = stm32_cryp_read(cryp, cryp->caps->cr) & CR_PH_MASK; 1653 if (unlikely(ph == CR_PH_HEADER)) 1654 /* Write Header */ 1655 stm32_cryp_irq_write_gcmccm_header(cryp); 1656 else 1657 /* Input FIFO IRQ: write data */ 1658 stm32_cryp_irq_write_data(cryp); 1659 if (is_gcm(cryp)) 1660 cryp->gcm_ctr++; 1661 } else { 1662 /* Input FIFO IRQ: write data */ 1663 stm32_cryp_irq_write_data(cryp); 1664 } 1665 } 1666 1667 /* Mask useless interrupts */ 1668 if (!cryp->payload_in && !cryp->header_in) 1669 it_mask &= ~IMSCR_IN; 1670 if (!cryp->payload_out) 1671 it_mask &= ~IMSCR_OUT; 1672 stm32_cryp_write(cryp, cryp->caps->imsc, it_mask); 1673 1674 if (!cryp->payload_in && !cryp->header_in && !cryp->payload_out) 1675 stm32_cryp_finish_req(cryp, 0); 1676 1677 return IRQ_HANDLED; 1678 } 1679 1680 static irqreturn_t stm32_cryp_irq(int irq, void *arg) 1681 { 1682 struct stm32_cryp *cryp = arg; 1683 1684 cryp->irq_status = stm32_cryp_read(cryp, cryp->caps->mis); 1685 1686 return IRQ_WAKE_THREAD; 1687 } 1688 1689 static struct skcipher_alg crypto_algs[] = { 1690 { 1691 .base.cra_name = "ecb(aes)", 1692 .base.cra_driver_name = "stm32-ecb-aes", 1693 .base.cra_priority = 200, 1694 .base.cra_flags = CRYPTO_ALG_ASYNC, 1695 .base.cra_blocksize = AES_BLOCK_SIZE, 1696 .base.cra_ctxsize = sizeof(struct stm32_cryp_ctx), 1697 .base.cra_alignmask = 0, 1698 .base.cra_module = THIS_MODULE, 1699 1700 .init = stm32_cryp_init_tfm, 1701 .min_keysize = AES_MIN_KEY_SIZE, 1702 .max_keysize = AES_MAX_KEY_SIZE, 1703 .setkey = stm32_cryp_aes_setkey, 1704 .encrypt = stm32_cryp_aes_ecb_encrypt, 1705 .decrypt = stm32_cryp_aes_ecb_decrypt, 1706 }, 1707 { 1708 .base.cra_name = "cbc(aes)", 1709 .base.cra_driver_name = "stm32-cbc-aes", 1710 .base.cra_priority = 200, 1711 .base.cra_flags = CRYPTO_ALG_ASYNC, 1712 .base.cra_blocksize = AES_BLOCK_SIZE, 1713 .base.cra_ctxsize = sizeof(struct stm32_cryp_ctx), 1714 .base.cra_alignmask = 0, 1715 .base.cra_module = THIS_MODULE, 1716 1717 .init = stm32_cryp_init_tfm, 1718 .min_keysize = AES_MIN_KEY_SIZE, 1719 .max_keysize = AES_MAX_KEY_SIZE, 1720 .ivsize = AES_BLOCK_SIZE, 1721 .setkey = stm32_cryp_aes_setkey, 1722 .encrypt = stm32_cryp_aes_cbc_encrypt, 1723 .decrypt = stm32_cryp_aes_cbc_decrypt, 1724 }, 1725 { 1726 .base.cra_name = "ctr(aes)", 1727 .base.cra_driver_name = "stm32-ctr-aes", 1728 .base.cra_priority = 200, 1729 .base.cra_flags = CRYPTO_ALG_ASYNC, 1730 .base.cra_blocksize = 1, 1731 .base.cra_ctxsize = sizeof(struct stm32_cryp_ctx), 1732 .base.cra_alignmask = 0, 1733 .base.cra_module = THIS_MODULE, 1734 1735 .init = stm32_cryp_init_tfm, 1736 .min_keysize = AES_MIN_KEY_SIZE, 1737 .max_keysize = AES_MAX_KEY_SIZE, 1738 .ivsize = AES_BLOCK_SIZE, 1739 .setkey = stm32_cryp_aes_setkey, 1740 .encrypt = stm32_cryp_aes_ctr_encrypt, 1741 .decrypt = stm32_cryp_aes_ctr_decrypt, 1742 }, 1743 { 1744 .base.cra_name = "ecb(des)", 1745 .base.cra_driver_name = "stm32-ecb-des", 1746 .base.cra_priority = 200, 1747 .base.cra_flags = CRYPTO_ALG_ASYNC, 1748 .base.cra_blocksize = DES_BLOCK_SIZE, 1749 .base.cra_ctxsize = sizeof(struct stm32_cryp_ctx), 1750 .base.cra_alignmask = 0, 1751 .base.cra_module = THIS_MODULE, 1752 1753 .init = stm32_cryp_init_tfm, 1754 .min_keysize = DES_BLOCK_SIZE, 1755 .max_keysize = DES_BLOCK_SIZE, 1756 .setkey = stm32_cryp_des_setkey, 1757 .encrypt = stm32_cryp_des_ecb_encrypt, 1758 .decrypt = stm32_cryp_des_ecb_decrypt, 1759 }, 1760 { 1761 .base.cra_name = "cbc(des)", 1762 .base.cra_driver_name = "stm32-cbc-des", 1763 .base.cra_priority = 200, 1764 .base.cra_flags = CRYPTO_ALG_ASYNC, 1765 .base.cra_blocksize = DES_BLOCK_SIZE, 1766 .base.cra_ctxsize = sizeof(struct stm32_cryp_ctx), 1767 .base.cra_alignmask = 0, 1768 .base.cra_module = THIS_MODULE, 1769 1770 .init = stm32_cryp_init_tfm, 1771 .min_keysize = DES_BLOCK_SIZE, 1772 .max_keysize = DES_BLOCK_SIZE, 1773 .ivsize = DES_BLOCK_SIZE, 1774 .setkey = stm32_cryp_des_setkey, 1775 .encrypt = stm32_cryp_des_cbc_encrypt, 1776 .decrypt = stm32_cryp_des_cbc_decrypt, 1777 }, 1778 { 1779 .base.cra_name = "ecb(des3_ede)", 1780 .base.cra_driver_name = "stm32-ecb-des3", 1781 .base.cra_priority = 200, 1782 .base.cra_flags = CRYPTO_ALG_ASYNC, 1783 .base.cra_blocksize = DES_BLOCK_SIZE, 1784 .base.cra_ctxsize = sizeof(struct stm32_cryp_ctx), 1785 .base.cra_alignmask = 0, 1786 .base.cra_module = THIS_MODULE, 1787 1788 .init = stm32_cryp_init_tfm, 1789 .min_keysize = 3 * DES_BLOCK_SIZE, 1790 .max_keysize = 3 * DES_BLOCK_SIZE, 1791 .setkey = stm32_cryp_tdes_setkey, 1792 .encrypt = stm32_cryp_tdes_ecb_encrypt, 1793 .decrypt = stm32_cryp_tdes_ecb_decrypt, 1794 }, 1795 { 1796 .base.cra_name = "cbc(des3_ede)", 1797 .base.cra_driver_name = "stm32-cbc-des3", 1798 .base.cra_priority = 200, 1799 .base.cra_flags = CRYPTO_ALG_ASYNC, 1800 .base.cra_blocksize = DES_BLOCK_SIZE, 1801 .base.cra_ctxsize = sizeof(struct stm32_cryp_ctx), 1802 .base.cra_alignmask = 0, 1803 .base.cra_module = THIS_MODULE, 1804 1805 .init = stm32_cryp_init_tfm, 1806 .min_keysize = 3 * DES_BLOCK_SIZE, 1807 .max_keysize = 3 * DES_BLOCK_SIZE, 1808 .ivsize = DES_BLOCK_SIZE, 1809 .setkey = stm32_cryp_tdes_setkey, 1810 .encrypt = stm32_cryp_tdes_cbc_encrypt, 1811 .decrypt = stm32_cryp_tdes_cbc_decrypt, 1812 }, 1813 }; 1814 1815 static struct aead_alg aead_algs[] = { 1816 { 1817 .setkey = stm32_cryp_aes_aead_setkey, 1818 .setauthsize = stm32_cryp_aes_gcm_setauthsize, 1819 .encrypt = stm32_cryp_aes_gcm_encrypt, 1820 .decrypt = stm32_cryp_aes_gcm_decrypt, 1821 .init = stm32_cryp_aes_aead_init, 1822 .ivsize = 12, 1823 .maxauthsize = AES_BLOCK_SIZE, 1824 1825 .base = { 1826 .cra_name = "gcm(aes)", 1827 .cra_driver_name = "stm32-gcm-aes", 1828 .cra_priority = 200, 1829 .cra_flags = CRYPTO_ALG_ASYNC, 1830 .cra_blocksize = 1, 1831 .cra_ctxsize = sizeof(struct stm32_cryp_ctx), 1832 .cra_alignmask = 0, 1833 .cra_module = THIS_MODULE, 1834 }, 1835 }, 1836 { 1837 .setkey = stm32_cryp_aes_aead_setkey, 1838 .setauthsize = stm32_cryp_aes_ccm_setauthsize, 1839 .encrypt = stm32_cryp_aes_ccm_encrypt, 1840 .decrypt = stm32_cryp_aes_ccm_decrypt, 1841 .init = stm32_cryp_aes_aead_init, 1842 .ivsize = AES_BLOCK_SIZE, 1843 .maxauthsize = AES_BLOCK_SIZE, 1844 1845 .base = { 1846 .cra_name = "ccm(aes)", 1847 .cra_driver_name = "stm32-ccm-aes", 1848 .cra_priority = 200, 1849 .cra_flags = CRYPTO_ALG_ASYNC, 1850 .cra_blocksize = 1, 1851 .cra_ctxsize = sizeof(struct stm32_cryp_ctx), 1852 .cra_alignmask = 0, 1853 .cra_module = THIS_MODULE, 1854 }, 1855 }, 1856 }; 1857 1858 static const struct stm32_cryp_caps ux500_data = { 1859 .aeads_support = false, 1860 .linear_aes_key = true, 1861 .kp_mode = false, 1862 .iv_protection = true, 1863 .swap_final = true, 1864 .padding_wa = true, 1865 .cr = UX500_CRYP_CR, 1866 .sr = UX500_CRYP_SR, 1867 .din = UX500_CRYP_DIN, 1868 .dout = UX500_CRYP_DOUT, 1869 .imsc = UX500_CRYP_IMSC, 1870 .mis = UX500_CRYP_MIS, 1871 .k1l = UX500_CRYP_K1L, 1872 .k1r = UX500_CRYP_K1R, 1873 .k3r = UX500_CRYP_K3R, 1874 .iv0l = UX500_CRYP_IV0L, 1875 .iv0r = UX500_CRYP_IV0R, 1876 .iv1l = UX500_CRYP_IV1L, 1877 .iv1r = UX500_CRYP_IV1R, 1878 }; 1879 1880 static const struct stm32_cryp_caps f7_data = { 1881 .aeads_support = true, 1882 .linear_aes_key = false, 1883 .kp_mode = true, 1884 .iv_protection = false, 1885 .swap_final = true, 1886 .padding_wa = true, 1887 .cr = CRYP_CR, 1888 .sr = CRYP_SR, 1889 .din = CRYP_DIN, 1890 .dout = CRYP_DOUT, 1891 .imsc = CRYP_IMSCR, 1892 .mis = CRYP_MISR, 1893 .k1l = CRYP_K1LR, 1894 .k1r = CRYP_K1RR, 1895 .k3r = CRYP_K3RR, 1896 .iv0l = CRYP_IV0LR, 1897 .iv0r = CRYP_IV0RR, 1898 .iv1l = CRYP_IV1LR, 1899 .iv1r = CRYP_IV1RR, 1900 }; 1901 1902 static const struct stm32_cryp_caps mp1_data = { 1903 .aeads_support = true, 1904 .linear_aes_key = false, 1905 .kp_mode = true, 1906 .iv_protection = false, 1907 .swap_final = false, 1908 .padding_wa = false, 1909 .cr = CRYP_CR, 1910 .sr = CRYP_SR, 1911 .din = CRYP_DIN, 1912 .dout = CRYP_DOUT, 1913 .imsc = CRYP_IMSCR, 1914 .mis = CRYP_MISR, 1915 .k1l = CRYP_K1LR, 1916 .k1r = CRYP_K1RR, 1917 .k3r = CRYP_K3RR, 1918 .iv0l = CRYP_IV0LR, 1919 .iv0r = CRYP_IV0RR, 1920 .iv1l = CRYP_IV1LR, 1921 .iv1r = CRYP_IV1RR, 1922 }; 1923 1924 static const struct of_device_id stm32_dt_ids[] = { 1925 { .compatible = "stericsson,ux500-cryp", .data = &ux500_data}, 1926 { .compatible = "st,stm32f756-cryp", .data = &f7_data}, 1927 { .compatible = "st,stm32mp1-cryp", .data = &mp1_data}, 1928 {}, 1929 }; 1930 MODULE_DEVICE_TABLE(of, stm32_dt_ids); 1931 1932 static int stm32_cryp_probe(struct platform_device *pdev) 1933 { 1934 struct device *dev = &pdev->dev; 1935 struct stm32_cryp *cryp; 1936 struct reset_control *rst; 1937 int irq, ret; 1938 1939 cryp = devm_kzalloc(dev, sizeof(*cryp), GFP_KERNEL); 1940 if (!cryp) 1941 return -ENOMEM; 1942 1943 cryp->caps = of_device_get_match_data(dev); 1944 if (!cryp->caps) 1945 return -ENODEV; 1946 1947 cryp->dev = dev; 1948 1949 cryp->regs = devm_platform_ioremap_resource(pdev, 0); 1950 if (IS_ERR(cryp->regs)) 1951 return PTR_ERR(cryp->regs); 1952 1953 irq = platform_get_irq(pdev, 0); 1954 if (irq < 0) 1955 return irq; 1956 1957 ret = devm_request_threaded_irq(dev, irq, stm32_cryp_irq, 1958 stm32_cryp_irq_thread, IRQF_ONESHOT, 1959 dev_name(dev), cryp); 1960 if (ret) { 1961 dev_err(dev, "Cannot grab IRQ\n"); 1962 return ret; 1963 } 1964 1965 cryp->clk = devm_clk_get(dev, NULL); 1966 if (IS_ERR(cryp->clk)) { 1967 dev_err_probe(dev, PTR_ERR(cryp->clk), "Could not get clock\n"); 1968 1969 return PTR_ERR(cryp->clk); 1970 } 1971 1972 ret = clk_prepare_enable(cryp->clk); 1973 if (ret) { 1974 dev_err(cryp->dev, "Failed to enable clock\n"); 1975 return ret; 1976 } 1977 1978 pm_runtime_set_autosuspend_delay(dev, CRYP_AUTOSUSPEND_DELAY); 1979 pm_runtime_use_autosuspend(dev); 1980 1981 pm_runtime_get_noresume(dev); 1982 pm_runtime_set_active(dev); 1983 pm_runtime_enable(dev); 1984 1985 rst = devm_reset_control_get(dev, NULL); 1986 if (IS_ERR(rst)) { 1987 ret = PTR_ERR(rst); 1988 if (ret == -EPROBE_DEFER) 1989 goto err_rst; 1990 } else { 1991 reset_control_assert(rst); 1992 udelay(2); 1993 reset_control_deassert(rst); 1994 } 1995 1996 platform_set_drvdata(pdev, cryp); 1997 1998 spin_lock(&cryp_list.lock); 1999 list_add(&cryp->list, &cryp_list.dev_list); 2000 spin_unlock(&cryp_list.lock); 2001 2002 /* Initialize crypto engine */ 2003 cryp->engine = crypto_engine_alloc_init(dev, 1); 2004 if (!cryp->engine) { 2005 dev_err(dev, "Could not init crypto engine\n"); 2006 ret = -ENOMEM; 2007 goto err_engine1; 2008 } 2009 2010 ret = crypto_engine_start(cryp->engine); 2011 if (ret) { 2012 dev_err(dev, "Could not start crypto engine\n"); 2013 goto err_engine2; 2014 } 2015 2016 ret = crypto_register_skciphers(crypto_algs, ARRAY_SIZE(crypto_algs)); 2017 if (ret) { 2018 dev_err(dev, "Could not register algs\n"); 2019 goto err_algs; 2020 } 2021 2022 if (cryp->caps->aeads_support) { 2023 ret = crypto_register_aeads(aead_algs, ARRAY_SIZE(aead_algs)); 2024 if (ret) 2025 goto err_aead_algs; 2026 } 2027 2028 dev_info(dev, "Initialized\n"); 2029 2030 pm_runtime_put_sync(dev); 2031 2032 return 0; 2033 2034 err_aead_algs: 2035 crypto_unregister_skciphers(crypto_algs, ARRAY_SIZE(crypto_algs)); 2036 err_algs: 2037 err_engine2: 2038 crypto_engine_exit(cryp->engine); 2039 err_engine1: 2040 spin_lock(&cryp_list.lock); 2041 list_del(&cryp->list); 2042 spin_unlock(&cryp_list.lock); 2043 err_rst: 2044 pm_runtime_disable(dev); 2045 pm_runtime_put_noidle(dev); 2046 2047 clk_disable_unprepare(cryp->clk); 2048 2049 return ret; 2050 } 2051 2052 static int stm32_cryp_remove(struct platform_device *pdev) 2053 { 2054 struct stm32_cryp *cryp = platform_get_drvdata(pdev); 2055 int ret; 2056 2057 if (!cryp) 2058 return -ENODEV; 2059 2060 ret = pm_runtime_resume_and_get(cryp->dev); 2061 if (ret < 0) 2062 return ret; 2063 2064 if (cryp->caps->aeads_support) 2065 crypto_unregister_aeads(aead_algs, ARRAY_SIZE(aead_algs)); 2066 crypto_unregister_skciphers(crypto_algs, ARRAY_SIZE(crypto_algs)); 2067 2068 crypto_engine_exit(cryp->engine); 2069 2070 spin_lock(&cryp_list.lock); 2071 list_del(&cryp->list); 2072 spin_unlock(&cryp_list.lock); 2073 2074 pm_runtime_disable(cryp->dev); 2075 pm_runtime_put_noidle(cryp->dev); 2076 2077 clk_disable_unprepare(cryp->clk); 2078 2079 return 0; 2080 } 2081 2082 #ifdef CONFIG_PM 2083 static int stm32_cryp_runtime_suspend(struct device *dev) 2084 { 2085 struct stm32_cryp *cryp = dev_get_drvdata(dev); 2086 2087 clk_disable_unprepare(cryp->clk); 2088 2089 return 0; 2090 } 2091 2092 static int stm32_cryp_runtime_resume(struct device *dev) 2093 { 2094 struct stm32_cryp *cryp = dev_get_drvdata(dev); 2095 int ret; 2096 2097 ret = clk_prepare_enable(cryp->clk); 2098 if (ret) { 2099 dev_err(cryp->dev, "Failed to prepare_enable clock\n"); 2100 return ret; 2101 } 2102 2103 return 0; 2104 } 2105 #endif 2106 2107 static const struct dev_pm_ops stm32_cryp_pm_ops = { 2108 SET_SYSTEM_SLEEP_PM_OPS(pm_runtime_force_suspend, 2109 pm_runtime_force_resume) 2110 SET_RUNTIME_PM_OPS(stm32_cryp_runtime_suspend, 2111 stm32_cryp_runtime_resume, NULL) 2112 }; 2113 2114 static struct platform_driver stm32_cryp_driver = { 2115 .probe = stm32_cryp_probe, 2116 .remove = stm32_cryp_remove, 2117 .driver = { 2118 .name = DRIVER_NAME, 2119 .pm = &stm32_cryp_pm_ops, 2120 .of_match_table = stm32_dt_ids, 2121 }, 2122 }; 2123 2124 module_platform_driver(stm32_cryp_driver); 2125 2126 MODULE_AUTHOR("Fabien Dessenne <fabien.dessenne@st.com>"); 2127 MODULE_DESCRIPTION("STMicrolectronics STM32 CRYP hardware driver"); 2128 MODULE_LICENSE("GPL"); 2129