1 // SPDX-License-Identifier: GPL-2.0 2 /* Copyright (C) 2012-2019 ARM Limited (or its affiliates). */ 3 4 #include <linux/kernel.h> 5 #include <linux/module.h> 6 #include <crypto/algapi.h> 7 #include <crypto/internal/skcipher.h> 8 #include <crypto/internal/des.h> 9 #include <crypto/xts.h> 10 #include <crypto/sm4.h> 11 #include <crypto/scatterwalk.h> 12 13 #include "cc_driver.h" 14 #include "cc_lli_defs.h" 15 #include "cc_buffer_mgr.h" 16 #include "cc_cipher.h" 17 #include "cc_request_mgr.h" 18 19 #define MAX_SKCIPHER_SEQ_LEN 6 20 21 #define template_skcipher template_u.skcipher 22 23 struct cc_user_key_info { 24 u8 *key; 25 dma_addr_t key_dma_addr; 26 }; 27 28 struct cc_hw_key_info { 29 enum cc_hw_crypto_key key1_slot; 30 enum cc_hw_crypto_key key2_slot; 31 }; 32 33 struct cc_cpp_key_info { 34 u8 slot; 35 enum cc_cpp_alg alg; 36 }; 37 38 enum cc_key_type { 39 CC_UNPROTECTED_KEY, /* User key */ 40 CC_HW_PROTECTED_KEY, /* HW (FDE) key */ 41 CC_POLICY_PROTECTED_KEY, /* CPP key */ 42 CC_INVALID_PROTECTED_KEY /* Invalid key */ 43 }; 44 45 struct cc_cipher_ctx { 46 struct cc_drvdata *drvdata; 47 int keylen; 48 int cipher_mode; 49 int flow_mode; 50 unsigned int flags; 51 enum cc_key_type key_type; 52 struct cc_user_key_info user; 53 union { 54 struct cc_hw_key_info hw; 55 struct cc_cpp_key_info cpp; 56 }; 57 struct crypto_shash *shash_tfm; 58 struct crypto_skcipher *fallback_tfm; 59 bool fallback_on; 60 }; 61 62 static void cc_cipher_complete(struct device *dev, void *cc_req, int err); 63 64 static inline enum cc_key_type cc_key_type(struct crypto_tfm *tfm) 65 { 66 struct cc_cipher_ctx *ctx_p = crypto_tfm_ctx(tfm); 67 68 return ctx_p->key_type; 69 } 70 71 static int validate_keys_sizes(struct cc_cipher_ctx *ctx_p, u32 size) 72 { 73 switch (ctx_p->flow_mode) { 74 case S_DIN_to_AES: 75 switch (size) { 76 case CC_AES_128_BIT_KEY_SIZE: 77 case CC_AES_192_BIT_KEY_SIZE: 78 if (ctx_p->cipher_mode != DRV_CIPHER_XTS) 79 return 0; 80 break; 81 case CC_AES_256_BIT_KEY_SIZE: 82 return 0; 83 case (CC_AES_192_BIT_KEY_SIZE * 2): 84 case (CC_AES_256_BIT_KEY_SIZE * 2): 85 if (ctx_p->cipher_mode == DRV_CIPHER_XTS || 86 ctx_p->cipher_mode == DRV_CIPHER_ESSIV) 87 return 0; 88 break; 89 default: 90 break; 91 } 92 break; 93 case S_DIN_to_DES: 94 if (size == DES3_EDE_KEY_SIZE || size == DES_KEY_SIZE) 95 return 0; 96 break; 97 case S_DIN_to_SM4: 98 if (size == SM4_KEY_SIZE) 99 return 0; 100 break; 101 default: 102 break; 103 } 104 return -EINVAL; 105 } 106 107 static int validate_data_size(struct cc_cipher_ctx *ctx_p, 108 unsigned int size) 109 { 110 switch (ctx_p->flow_mode) { 111 case S_DIN_to_AES: 112 switch (ctx_p->cipher_mode) { 113 case DRV_CIPHER_XTS: 114 case DRV_CIPHER_CBC_CTS: 115 if (size >= AES_BLOCK_SIZE) 116 return 0; 117 break; 118 case DRV_CIPHER_OFB: 119 case DRV_CIPHER_CTR: 120 return 0; 121 case DRV_CIPHER_ECB: 122 case DRV_CIPHER_CBC: 123 case DRV_CIPHER_ESSIV: 124 if (IS_ALIGNED(size, AES_BLOCK_SIZE)) 125 return 0; 126 break; 127 default: 128 break; 129 } 130 break; 131 case S_DIN_to_DES: 132 if (IS_ALIGNED(size, DES_BLOCK_SIZE)) 133 return 0; 134 break; 135 case S_DIN_to_SM4: 136 switch (ctx_p->cipher_mode) { 137 case DRV_CIPHER_CTR: 138 return 0; 139 case DRV_CIPHER_ECB: 140 case DRV_CIPHER_CBC: 141 if (IS_ALIGNED(size, SM4_BLOCK_SIZE)) 142 return 0; 143 break; 144 default: 145 break; 146 } 147 break; 148 default: 149 break; 150 } 151 return -EINVAL; 152 } 153 154 static int cc_cipher_init(struct crypto_tfm *tfm) 155 { 156 struct cc_cipher_ctx *ctx_p = crypto_tfm_ctx(tfm); 157 struct cc_crypto_alg *cc_alg = 158 container_of(tfm->__crt_alg, struct cc_crypto_alg, 159 skcipher_alg.base); 160 struct device *dev = drvdata_to_dev(cc_alg->drvdata); 161 unsigned int max_key_buf_size = cc_alg->skcipher_alg.max_keysize; 162 unsigned int fallback_req_size = 0; 163 164 dev_dbg(dev, "Initializing context @%p for %s\n", ctx_p, 165 crypto_tfm_alg_name(tfm)); 166 167 ctx_p->cipher_mode = cc_alg->cipher_mode; 168 ctx_p->flow_mode = cc_alg->flow_mode; 169 ctx_p->drvdata = cc_alg->drvdata; 170 171 if (ctx_p->cipher_mode == DRV_CIPHER_ESSIV) { 172 const char *name = crypto_tfm_alg_name(tfm); 173 174 /* Alloc hash tfm for essiv */ 175 ctx_p->shash_tfm = crypto_alloc_shash("sha256", 0, 0); 176 if (IS_ERR(ctx_p->shash_tfm)) { 177 dev_err(dev, "Error allocating hash tfm for ESSIV.\n"); 178 return PTR_ERR(ctx_p->shash_tfm); 179 } 180 max_key_buf_size <<= 1; 181 182 /* Alloc fallabck tfm or essiv when key size != 256 bit */ 183 ctx_p->fallback_tfm = 184 crypto_alloc_skcipher(name, 0, CRYPTO_ALG_NEED_FALLBACK | CRYPTO_ALG_ASYNC); 185 186 if (IS_ERR(ctx_p->fallback_tfm)) { 187 /* Note we're still allowing registration with no fallback since it's 188 * better to have most modes supported than none at all. 189 */ 190 dev_warn(dev, "Error allocating fallback algo %s. Some modes may be available.\n", 191 name); 192 ctx_p->fallback_tfm = NULL; 193 } else { 194 fallback_req_size = crypto_skcipher_reqsize(ctx_p->fallback_tfm); 195 } 196 } 197 198 crypto_skcipher_set_reqsize(__crypto_skcipher_cast(tfm), 199 sizeof(struct cipher_req_ctx) + fallback_req_size); 200 201 /* Allocate key buffer, cache line aligned */ 202 ctx_p->user.key = kzalloc(max_key_buf_size, GFP_KERNEL); 203 if (!ctx_p->user.key) 204 goto free_fallback; 205 206 dev_dbg(dev, "Allocated key buffer in context. key=@%p\n", 207 ctx_p->user.key); 208 209 /* Map key buffer */ 210 ctx_p->user.key_dma_addr = dma_map_single(dev, ctx_p->user.key, 211 max_key_buf_size, 212 DMA_TO_DEVICE); 213 if (dma_mapping_error(dev, ctx_p->user.key_dma_addr)) { 214 dev_err(dev, "Mapping Key %u B at va=%pK for DMA failed\n", 215 max_key_buf_size, ctx_p->user.key); 216 goto free_key; 217 } 218 dev_dbg(dev, "Mapped key %u B at va=%pK to dma=%pad\n", 219 max_key_buf_size, ctx_p->user.key, &ctx_p->user.key_dma_addr); 220 221 return 0; 222 223 free_key: 224 kfree(ctx_p->user.key); 225 free_fallback: 226 crypto_free_skcipher(ctx_p->fallback_tfm); 227 crypto_free_shash(ctx_p->shash_tfm); 228 229 return -ENOMEM; 230 } 231 232 static void cc_cipher_exit(struct crypto_tfm *tfm) 233 { 234 struct crypto_alg *alg = tfm->__crt_alg; 235 struct cc_crypto_alg *cc_alg = 236 container_of(alg, struct cc_crypto_alg, 237 skcipher_alg.base); 238 unsigned int max_key_buf_size = cc_alg->skcipher_alg.max_keysize; 239 struct cc_cipher_ctx *ctx_p = crypto_tfm_ctx(tfm); 240 struct device *dev = drvdata_to_dev(ctx_p->drvdata); 241 242 dev_dbg(dev, "Clearing context @%p for %s\n", 243 crypto_tfm_ctx(tfm), crypto_tfm_alg_name(tfm)); 244 245 if (ctx_p->cipher_mode == DRV_CIPHER_ESSIV) { 246 /* Free hash tfm for essiv */ 247 crypto_free_shash(ctx_p->shash_tfm); 248 ctx_p->shash_tfm = NULL; 249 crypto_free_skcipher(ctx_p->fallback_tfm); 250 ctx_p->fallback_tfm = NULL; 251 } 252 253 /* Unmap key buffer */ 254 dma_unmap_single(dev, ctx_p->user.key_dma_addr, max_key_buf_size, 255 DMA_TO_DEVICE); 256 dev_dbg(dev, "Unmapped key buffer key_dma_addr=%pad\n", 257 &ctx_p->user.key_dma_addr); 258 259 /* Free key buffer in context */ 260 dev_dbg(dev, "Free key buffer in context. key=@%p\n", ctx_p->user.key); 261 kfree_sensitive(ctx_p->user.key); 262 } 263 264 struct tdes_keys { 265 u8 key1[DES_KEY_SIZE]; 266 u8 key2[DES_KEY_SIZE]; 267 u8 key3[DES_KEY_SIZE]; 268 }; 269 270 static enum cc_hw_crypto_key cc_slot_to_hw_key(u8 slot_num) 271 { 272 switch (slot_num) { 273 case 0: 274 return KFDE0_KEY; 275 case 1: 276 return KFDE1_KEY; 277 case 2: 278 return KFDE2_KEY; 279 case 3: 280 return KFDE3_KEY; 281 } 282 return END_OF_KEYS; 283 } 284 285 static u8 cc_slot_to_cpp_key(u8 slot_num) 286 { 287 return (slot_num - CC_FIRST_CPP_KEY_SLOT); 288 } 289 290 static inline enum cc_key_type cc_slot_to_key_type(u8 slot_num) 291 { 292 if (slot_num >= CC_FIRST_HW_KEY_SLOT && slot_num <= CC_LAST_HW_KEY_SLOT) 293 return CC_HW_PROTECTED_KEY; 294 else if (slot_num >= CC_FIRST_CPP_KEY_SLOT && 295 slot_num <= CC_LAST_CPP_KEY_SLOT) 296 return CC_POLICY_PROTECTED_KEY; 297 else 298 return CC_INVALID_PROTECTED_KEY; 299 } 300 301 static int cc_cipher_sethkey(struct crypto_skcipher *sktfm, const u8 *key, 302 unsigned int keylen) 303 { 304 struct crypto_tfm *tfm = crypto_skcipher_tfm(sktfm); 305 struct cc_cipher_ctx *ctx_p = crypto_tfm_ctx(tfm); 306 struct device *dev = drvdata_to_dev(ctx_p->drvdata); 307 struct cc_hkey_info hki; 308 309 dev_dbg(dev, "Setting HW key in context @%p for %s. keylen=%u\n", 310 ctx_p, crypto_tfm_alg_name(tfm), keylen); 311 dump_byte_array("key", key, keylen); 312 313 /* STAT_PHASE_0: Init and sanity checks */ 314 315 /* This check the size of the protected key token */ 316 if (keylen != sizeof(hki)) { 317 dev_err(dev, "Unsupported protected key size %d.\n", keylen); 318 return -EINVAL; 319 } 320 321 memcpy(&hki, key, keylen); 322 323 /* The real key len for crypto op is the size of the HW key 324 * referenced by the HW key slot, not the hardware key token 325 */ 326 keylen = hki.keylen; 327 328 if (validate_keys_sizes(ctx_p, keylen)) { 329 dev_dbg(dev, "Unsupported key size %d.\n", keylen); 330 return -EINVAL; 331 } 332 333 ctx_p->keylen = keylen; 334 ctx_p->fallback_on = false; 335 336 switch (cc_slot_to_key_type(hki.hw_key1)) { 337 case CC_HW_PROTECTED_KEY: 338 if (ctx_p->flow_mode == S_DIN_to_SM4) { 339 dev_err(dev, "Only AES HW protected keys are supported\n"); 340 return -EINVAL; 341 } 342 343 ctx_p->hw.key1_slot = cc_slot_to_hw_key(hki.hw_key1); 344 if (ctx_p->hw.key1_slot == END_OF_KEYS) { 345 dev_err(dev, "Unsupported hw key1 number (%d)\n", 346 hki.hw_key1); 347 return -EINVAL; 348 } 349 350 if (ctx_p->cipher_mode == DRV_CIPHER_XTS || 351 ctx_p->cipher_mode == DRV_CIPHER_ESSIV) { 352 if (hki.hw_key1 == hki.hw_key2) { 353 dev_err(dev, "Illegal hw key numbers (%d,%d)\n", 354 hki.hw_key1, hki.hw_key2); 355 return -EINVAL; 356 } 357 358 ctx_p->hw.key2_slot = cc_slot_to_hw_key(hki.hw_key2); 359 if (ctx_p->hw.key2_slot == END_OF_KEYS) { 360 dev_err(dev, "Unsupported hw key2 number (%d)\n", 361 hki.hw_key2); 362 return -EINVAL; 363 } 364 } 365 366 ctx_p->key_type = CC_HW_PROTECTED_KEY; 367 dev_dbg(dev, "HW protected key %d/%d set\n.", 368 ctx_p->hw.key1_slot, ctx_p->hw.key2_slot); 369 break; 370 371 case CC_POLICY_PROTECTED_KEY: 372 if (ctx_p->drvdata->hw_rev < CC_HW_REV_713) { 373 dev_err(dev, "CPP keys not supported in this hardware revision.\n"); 374 return -EINVAL; 375 } 376 377 if (ctx_p->cipher_mode != DRV_CIPHER_CBC && 378 ctx_p->cipher_mode != DRV_CIPHER_CTR) { 379 dev_err(dev, "CPP keys only supported in CBC or CTR modes.\n"); 380 return -EINVAL; 381 } 382 383 ctx_p->cpp.slot = cc_slot_to_cpp_key(hki.hw_key1); 384 if (ctx_p->flow_mode == S_DIN_to_AES) 385 ctx_p->cpp.alg = CC_CPP_AES; 386 else /* Must be SM4 since due to sethkey registration */ 387 ctx_p->cpp.alg = CC_CPP_SM4; 388 ctx_p->key_type = CC_POLICY_PROTECTED_KEY; 389 dev_dbg(dev, "policy protected key alg: %d slot: %d.\n", 390 ctx_p->cpp.alg, ctx_p->cpp.slot); 391 break; 392 393 default: 394 dev_err(dev, "Unsupported protected key (%d)\n", hki.hw_key1); 395 return -EINVAL; 396 } 397 398 return 0; 399 } 400 401 static int cc_cipher_setkey(struct crypto_skcipher *sktfm, const u8 *key, 402 unsigned int keylen) 403 { 404 struct crypto_tfm *tfm = crypto_skcipher_tfm(sktfm); 405 struct cc_cipher_ctx *ctx_p = crypto_tfm_ctx(tfm); 406 struct device *dev = drvdata_to_dev(ctx_p->drvdata); 407 struct cc_crypto_alg *cc_alg = 408 container_of(tfm->__crt_alg, struct cc_crypto_alg, 409 skcipher_alg.base); 410 unsigned int max_key_buf_size = cc_alg->skcipher_alg.max_keysize; 411 412 dev_dbg(dev, "Setting key in context @%p for %s. keylen=%u\n", 413 ctx_p, crypto_tfm_alg_name(tfm), keylen); 414 dump_byte_array("key", key, keylen); 415 416 /* STAT_PHASE_0: Init and sanity checks */ 417 418 if (validate_keys_sizes(ctx_p, keylen)) { 419 dev_dbg(dev, "Invalid key size %d.\n", keylen); 420 return -EINVAL; 421 } 422 423 if (ctx_p->cipher_mode == DRV_CIPHER_ESSIV) { 424 425 /* We only support 256 bit ESSIV-CBC-AES keys */ 426 if (keylen != AES_KEYSIZE_256) { 427 unsigned int flags = crypto_tfm_get_flags(tfm) & CRYPTO_TFM_REQ_MASK; 428 429 if (likely(ctx_p->fallback_tfm)) { 430 ctx_p->fallback_on = true; 431 crypto_skcipher_clear_flags(ctx_p->fallback_tfm, 432 CRYPTO_TFM_REQ_MASK); 433 crypto_skcipher_clear_flags(ctx_p->fallback_tfm, flags); 434 return crypto_skcipher_setkey(ctx_p->fallback_tfm, key, keylen); 435 } 436 437 dev_dbg(dev, "Unsupported key size %d and no fallback.\n", keylen); 438 return -EINVAL; 439 } 440 441 /* Internal ESSIV key buffer is double sized */ 442 max_key_buf_size <<= 1; 443 } 444 445 ctx_p->fallback_on = false; 446 ctx_p->key_type = CC_UNPROTECTED_KEY; 447 448 /* 449 * Verify DES weak keys 450 * Note that we're dropping the expanded key since the 451 * HW does the expansion on its own. 452 */ 453 if (ctx_p->flow_mode == S_DIN_to_DES) { 454 if ((keylen == DES3_EDE_KEY_SIZE && 455 verify_skcipher_des3_key(sktfm, key)) || 456 verify_skcipher_des_key(sktfm, key)) { 457 dev_dbg(dev, "weak DES key"); 458 return -EINVAL; 459 } 460 } 461 462 if (ctx_p->cipher_mode == DRV_CIPHER_XTS && 463 xts_verify_key(sktfm, key, keylen)) { 464 dev_dbg(dev, "weak XTS key"); 465 return -EINVAL; 466 } 467 468 /* STAT_PHASE_1: Copy key to ctx */ 469 dma_sync_single_for_cpu(dev, ctx_p->user.key_dma_addr, 470 max_key_buf_size, DMA_TO_DEVICE); 471 472 memcpy(ctx_p->user.key, key, keylen); 473 474 if (ctx_p->cipher_mode == DRV_CIPHER_ESSIV) { 475 /* sha256 for key2 - use sw implementation */ 476 int err; 477 478 err = crypto_shash_tfm_digest(ctx_p->shash_tfm, 479 ctx_p->user.key, keylen, 480 ctx_p->user.key + keylen); 481 if (err) { 482 dev_err(dev, "Failed to hash ESSIV key.\n"); 483 return err; 484 } 485 486 keylen <<= 1; 487 } 488 dma_sync_single_for_device(dev, ctx_p->user.key_dma_addr, 489 max_key_buf_size, DMA_TO_DEVICE); 490 ctx_p->keylen = keylen; 491 492 dev_dbg(dev, "return safely"); 493 return 0; 494 } 495 496 static int cc_out_setup_mode(struct cc_cipher_ctx *ctx_p) 497 { 498 switch (ctx_p->flow_mode) { 499 case S_DIN_to_AES: 500 return S_AES_to_DOUT; 501 case S_DIN_to_DES: 502 return S_DES_to_DOUT; 503 case S_DIN_to_SM4: 504 return S_SM4_to_DOUT; 505 default: 506 return ctx_p->flow_mode; 507 } 508 } 509 510 static void cc_setup_readiv_desc(struct crypto_tfm *tfm, 511 struct cipher_req_ctx *req_ctx, 512 unsigned int ivsize, struct cc_hw_desc desc[], 513 unsigned int *seq_size) 514 { 515 struct cc_cipher_ctx *ctx_p = crypto_tfm_ctx(tfm); 516 struct device *dev = drvdata_to_dev(ctx_p->drvdata); 517 int cipher_mode = ctx_p->cipher_mode; 518 int flow_mode = cc_out_setup_mode(ctx_p); 519 int direction = req_ctx->gen_ctx.op_type; 520 dma_addr_t iv_dma_addr = req_ctx->gen_ctx.iv_dma_addr; 521 522 if (ctx_p->key_type == CC_POLICY_PROTECTED_KEY) 523 return; 524 525 switch (cipher_mode) { 526 case DRV_CIPHER_ECB: 527 break; 528 case DRV_CIPHER_CBC: 529 case DRV_CIPHER_CBC_CTS: 530 case DRV_CIPHER_CTR: 531 case DRV_CIPHER_OFB: 532 /* Read next IV */ 533 hw_desc_init(&desc[*seq_size]); 534 set_dout_dlli(&desc[*seq_size], iv_dma_addr, ivsize, NS_BIT, 1); 535 set_cipher_config0(&desc[*seq_size], direction); 536 set_flow_mode(&desc[*seq_size], flow_mode); 537 set_cipher_mode(&desc[*seq_size], cipher_mode); 538 if (cipher_mode == DRV_CIPHER_CTR || 539 cipher_mode == DRV_CIPHER_OFB) { 540 set_setup_mode(&desc[*seq_size], SETUP_WRITE_STATE1); 541 } else { 542 set_setup_mode(&desc[*seq_size], SETUP_WRITE_STATE0); 543 } 544 set_queue_last_ind(ctx_p->drvdata, &desc[*seq_size]); 545 (*seq_size)++; 546 break; 547 case DRV_CIPHER_XTS: 548 case DRV_CIPHER_ESSIV: 549 /* IV */ 550 hw_desc_init(&desc[*seq_size]); 551 set_setup_mode(&desc[*seq_size], SETUP_WRITE_STATE1); 552 set_cipher_mode(&desc[*seq_size], cipher_mode); 553 set_cipher_config0(&desc[*seq_size], direction); 554 set_flow_mode(&desc[*seq_size], flow_mode); 555 set_dout_dlli(&desc[*seq_size], iv_dma_addr, CC_AES_BLOCK_SIZE, 556 NS_BIT, 1); 557 set_queue_last_ind(ctx_p->drvdata, &desc[*seq_size]); 558 (*seq_size)++; 559 break; 560 default: 561 dev_err(dev, "Unsupported cipher mode (%d)\n", cipher_mode); 562 } 563 } 564 565 566 static void cc_setup_state_desc(struct crypto_tfm *tfm, 567 struct cipher_req_ctx *req_ctx, 568 unsigned int ivsize, unsigned int nbytes, 569 struct cc_hw_desc desc[], 570 unsigned int *seq_size) 571 { 572 struct cc_cipher_ctx *ctx_p = crypto_tfm_ctx(tfm); 573 struct device *dev = drvdata_to_dev(ctx_p->drvdata); 574 int cipher_mode = ctx_p->cipher_mode; 575 int flow_mode = ctx_p->flow_mode; 576 int direction = req_ctx->gen_ctx.op_type; 577 dma_addr_t iv_dma_addr = req_ctx->gen_ctx.iv_dma_addr; 578 579 switch (cipher_mode) { 580 case DRV_CIPHER_ECB: 581 break; 582 case DRV_CIPHER_CBC: 583 case DRV_CIPHER_CBC_CTS: 584 case DRV_CIPHER_CTR: 585 case DRV_CIPHER_OFB: 586 /* Load IV */ 587 hw_desc_init(&desc[*seq_size]); 588 set_din_type(&desc[*seq_size], DMA_DLLI, iv_dma_addr, ivsize, 589 NS_BIT); 590 set_cipher_config0(&desc[*seq_size], direction); 591 set_flow_mode(&desc[*seq_size], flow_mode); 592 set_cipher_mode(&desc[*seq_size], cipher_mode); 593 if (cipher_mode == DRV_CIPHER_CTR || 594 cipher_mode == DRV_CIPHER_OFB) { 595 set_setup_mode(&desc[*seq_size], SETUP_LOAD_STATE1); 596 } else { 597 set_setup_mode(&desc[*seq_size], SETUP_LOAD_STATE0); 598 } 599 (*seq_size)++; 600 break; 601 case DRV_CIPHER_XTS: 602 case DRV_CIPHER_ESSIV: 603 break; 604 default: 605 dev_err(dev, "Unsupported cipher mode (%d)\n", cipher_mode); 606 } 607 } 608 609 610 static void cc_setup_xex_state_desc(struct crypto_tfm *tfm, 611 struct cipher_req_ctx *req_ctx, 612 unsigned int ivsize, unsigned int nbytes, 613 struct cc_hw_desc desc[], 614 unsigned int *seq_size) 615 { 616 struct cc_cipher_ctx *ctx_p = crypto_tfm_ctx(tfm); 617 struct device *dev = drvdata_to_dev(ctx_p->drvdata); 618 int cipher_mode = ctx_p->cipher_mode; 619 int flow_mode = ctx_p->flow_mode; 620 int direction = req_ctx->gen_ctx.op_type; 621 dma_addr_t key_dma_addr = ctx_p->user.key_dma_addr; 622 unsigned int key_len = (ctx_p->keylen / 2); 623 dma_addr_t iv_dma_addr = req_ctx->gen_ctx.iv_dma_addr; 624 unsigned int key_offset = key_len; 625 626 switch (cipher_mode) { 627 case DRV_CIPHER_ECB: 628 break; 629 case DRV_CIPHER_CBC: 630 case DRV_CIPHER_CBC_CTS: 631 case DRV_CIPHER_CTR: 632 case DRV_CIPHER_OFB: 633 break; 634 case DRV_CIPHER_XTS: 635 case DRV_CIPHER_ESSIV: 636 637 if (cipher_mode == DRV_CIPHER_ESSIV) 638 key_len = SHA256_DIGEST_SIZE; 639 640 /* load XEX key */ 641 hw_desc_init(&desc[*seq_size]); 642 set_cipher_mode(&desc[*seq_size], cipher_mode); 643 set_cipher_config0(&desc[*seq_size], direction); 644 if (cc_key_type(tfm) == CC_HW_PROTECTED_KEY) { 645 set_hw_crypto_key(&desc[*seq_size], 646 ctx_p->hw.key2_slot); 647 } else { 648 set_din_type(&desc[*seq_size], DMA_DLLI, 649 (key_dma_addr + key_offset), 650 key_len, NS_BIT); 651 } 652 set_xex_data_unit_size(&desc[*seq_size], nbytes); 653 set_flow_mode(&desc[*seq_size], S_DIN_to_AES2); 654 set_key_size_aes(&desc[*seq_size], key_len); 655 set_setup_mode(&desc[*seq_size], SETUP_LOAD_XEX_KEY); 656 (*seq_size)++; 657 658 /* Load IV */ 659 hw_desc_init(&desc[*seq_size]); 660 set_setup_mode(&desc[*seq_size], SETUP_LOAD_STATE1); 661 set_cipher_mode(&desc[*seq_size], cipher_mode); 662 set_cipher_config0(&desc[*seq_size], direction); 663 set_key_size_aes(&desc[*seq_size], key_len); 664 set_flow_mode(&desc[*seq_size], flow_mode); 665 set_din_type(&desc[*seq_size], DMA_DLLI, iv_dma_addr, 666 CC_AES_BLOCK_SIZE, NS_BIT); 667 (*seq_size)++; 668 break; 669 default: 670 dev_err(dev, "Unsupported cipher mode (%d)\n", cipher_mode); 671 } 672 } 673 674 static int cc_out_flow_mode(struct cc_cipher_ctx *ctx_p) 675 { 676 switch (ctx_p->flow_mode) { 677 case S_DIN_to_AES: 678 return DIN_AES_DOUT; 679 case S_DIN_to_DES: 680 return DIN_DES_DOUT; 681 case S_DIN_to_SM4: 682 return DIN_SM4_DOUT; 683 default: 684 return ctx_p->flow_mode; 685 } 686 } 687 688 static void cc_setup_key_desc(struct crypto_tfm *tfm, 689 struct cipher_req_ctx *req_ctx, 690 unsigned int nbytes, struct cc_hw_desc desc[], 691 unsigned int *seq_size) 692 { 693 struct cc_cipher_ctx *ctx_p = crypto_tfm_ctx(tfm); 694 struct device *dev = drvdata_to_dev(ctx_p->drvdata); 695 int cipher_mode = ctx_p->cipher_mode; 696 int flow_mode = ctx_p->flow_mode; 697 int direction = req_ctx->gen_ctx.op_type; 698 dma_addr_t key_dma_addr = ctx_p->user.key_dma_addr; 699 unsigned int key_len = ctx_p->keylen; 700 unsigned int din_size; 701 702 switch (cipher_mode) { 703 case DRV_CIPHER_CBC: 704 case DRV_CIPHER_CBC_CTS: 705 case DRV_CIPHER_CTR: 706 case DRV_CIPHER_OFB: 707 case DRV_CIPHER_ECB: 708 /* Load key */ 709 hw_desc_init(&desc[*seq_size]); 710 set_cipher_mode(&desc[*seq_size], cipher_mode); 711 set_cipher_config0(&desc[*seq_size], direction); 712 713 if (cc_key_type(tfm) == CC_POLICY_PROTECTED_KEY) { 714 /* We use the AES key size coding for all CPP algs */ 715 set_key_size_aes(&desc[*seq_size], key_len); 716 set_cpp_crypto_key(&desc[*seq_size], ctx_p->cpp.slot); 717 flow_mode = cc_out_flow_mode(ctx_p); 718 } else { 719 if (flow_mode == S_DIN_to_AES) { 720 if (cc_key_type(tfm) == CC_HW_PROTECTED_KEY) { 721 set_hw_crypto_key(&desc[*seq_size], 722 ctx_p->hw.key1_slot); 723 } else { 724 /* CC_POLICY_UNPROTECTED_KEY 725 * Invalid keys are filtered out in 726 * sethkey() 727 */ 728 din_size = (key_len == 24) ? 729 AES_MAX_KEY_SIZE : key_len; 730 731 set_din_type(&desc[*seq_size], DMA_DLLI, 732 key_dma_addr, din_size, 733 NS_BIT); 734 } 735 set_key_size_aes(&desc[*seq_size], key_len); 736 } else { 737 /*des*/ 738 set_din_type(&desc[*seq_size], DMA_DLLI, 739 key_dma_addr, key_len, NS_BIT); 740 set_key_size_des(&desc[*seq_size], key_len); 741 } 742 set_setup_mode(&desc[*seq_size], SETUP_LOAD_KEY0); 743 } 744 set_flow_mode(&desc[*seq_size], flow_mode); 745 (*seq_size)++; 746 break; 747 case DRV_CIPHER_XTS: 748 case DRV_CIPHER_ESSIV: 749 /* Load AES key */ 750 hw_desc_init(&desc[*seq_size]); 751 set_cipher_mode(&desc[*seq_size], cipher_mode); 752 set_cipher_config0(&desc[*seq_size], direction); 753 if (cc_key_type(tfm) == CC_HW_PROTECTED_KEY) { 754 set_hw_crypto_key(&desc[*seq_size], 755 ctx_p->hw.key1_slot); 756 } else { 757 set_din_type(&desc[*seq_size], DMA_DLLI, key_dma_addr, 758 (key_len / 2), NS_BIT); 759 } 760 set_key_size_aes(&desc[*seq_size], (key_len / 2)); 761 set_flow_mode(&desc[*seq_size], flow_mode); 762 set_setup_mode(&desc[*seq_size], SETUP_LOAD_KEY0); 763 (*seq_size)++; 764 break; 765 default: 766 dev_err(dev, "Unsupported cipher mode (%d)\n", cipher_mode); 767 } 768 } 769 770 static void cc_setup_mlli_desc(struct crypto_tfm *tfm, 771 struct cipher_req_ctx *req_ctx, 772 struct scatterlist *dst, struct scatterlist *src, 773 unsigned int nbytes, void *areq, 774 struct cc_hw_desc desc[], unsigned int *seq_size) 775 { 776 struct cc_cipher_ctx *ctx_p = crypto_tfm_ctx(tfm); 777 struct device *dev = drvdata_to_dev(ctx_p->drvdata); 778 779 if (req_ctx->dma_buf_type == CC_DMA_BUF_MLLI) { 780 /* bypass */ 781 dev_dbg(dev, " bypass params addr %pad length 0x%X addr 0x%08X\n", 782 &req_ctx->mlli_params.mlli_dma_addr, 783 req_ctx->mlli_params.mlli_len, 784 ctx_p->drvdata->mlli_sram_addr); 785 hw_desc_init(&desc[*seq_size]); 786 set_din_type(&desc[*seq_size], DMA_DLLI, 787 req_ctx->mlli_params.mlli_dma_addr, 788 req_ctx->mlli_params.mlli_len, NS_BIT); 789 set_dout_sram(&desc[*seq_size], 790 ctx_p->drvdata->mlli_sram_addr, 791 req_ctx->mlli_params.mlli_len); 792 set_flow_mode(&desc[*seq_size], BYPASS); 793 (*seq_size)++; 794 } 795 } 796 797 static void cc_setup_flow_desc(struct crypto_tfm *tfm, 798 struct cipher_req_ctx *req_ctx, 799 struct scatterlist *dst, struct scatterlist *src, 800 unsigned int nbytes, struct cc_hw_desc desc[], 801 unsigned int *seq_size) 802 { 803 struct cc_cipher_ctx *ctx_p = crypto_tfm_ctx(tfm); 804 struct device *dev = drvdata_to_dev(ctx_p->drvdata); 805 unsigned int flow_mode = cc_out_flow_mode(ctx_p); 806 bool last_desc = (ctx_p->key_type == CC_POLICY_PROTECTED_KEY || 807 ctx_p->cipher_mode == DRV_CIPHER_ECB); 808 809 /* Process */ 810 if (req_ctx->dma_buf_type == CC_DMA_BUF_DLLI) { 811 dev_dbg(dev, " data params addr %pad length 0x%X\n", 812 &sg_dma_address(src), nbytes); 813 dev_dbg(dev, " data params addr %pad length 0x%X\n", 814 &sg_dma_address(dst), nbytes); 815 hw_desc_init(&desc[*seq_size]); 816 set_din_type(&desc[*seq_size], DMA_DLLI, sg_dma_address(src), 817 nbytes, NS_BIT); 818 set_dout_dlli(&desc[*seq_size], sg_dma_address(dst), 819 nbytes, NS_BIT, (!last_desc ? 0 : 1)); 820 if (last_desc) 821 set_queue_last_ind(ctx_p->drvdata, &desc[*seq_size]); 822 823 set_flow_mode(&desc[*seq_size], flow_mode); 824 (*seq_size)++; 825 } else { 826 hw_desc_init(&desc[*seq_size]); 827 set_din_type(&desc[*seq_size], DMA_MLLI, 828 ctx_p->drvdata->mlli_sram_addr, 829 req_ctx->in_mlli_nents, NS_BIT); 830 if (req_ctx->out_nents == 0) { 831 dev_dbg(dev, " din/dout params addr 0x%08X addr 0x%08X\n", 832 ctx_p->drvdata->mlli_sram_addr, 833 ctx_p->drvdata->mlli_sram_addr); 834 set_dout_mlli(&desc[*seq_size], 835 ctx_p->drvdata->mlli_sram_addr, 836 req_ctx->in_mlli_nents, NS_BIT, 837 (!last_desc ? 0 : 1)); 838 } else { 839 dev_dbg(dev, " din/dout params addr 0x%08X addr 0x%08X\n", 840 ctx_p->drvdata->mlli_sram_addr, 841 ctx_p->drvdata->mlli_sram_addr + 842 (u32)LLI_ENTRY_BYTE_SIZE * req_ctx->in_nents); 843 set_dout_mlli(&desc[*seq_size], 844 (ctx_p->drvdata->mlli_sram_addr + 845 (LLI_ENTRY_BYTE_SIZE * 846 req_ctx->in_mlli_nents)), 847 req_ctx->out_mlli_nents, NS_BIT, 848 (!last_desc ? 0 : 1)); 849 } 850 if (last_desc) 851 set_queue_last_ind(ctx_p->drvdata, &desc[*seq_size]); 852 853 set_flow_mode(&desc[*seq_size], flow_mode); 854 (*seq_size)++; 855 } 856 } 857 858 static void cc_cipher_complete(struct device *dev, void *cc_req, int err) 859 { 860 struct skcipher_request *req = (struct skcipher_request *)cc_req; 861 struct scatterlist *dst = req->dst; 862 struct scatterlist *src = req->src; 863 struct cipher_req_ctx *req_ctx = skcipher_request_ctx(req); 864 struct crypto_skcipher *sk_tfm = crypto_skcipher_reqtfm(req); 865 unsigned int ivsize = crypto_skcipher_ivsize(sk_tfm); 866 867 if (err != -EINPROGRESS) { 868 /* Not a BACKLOG notification */ 869 cc_unmap_cipher_request(dev, req_ctx, ivsize, src, dst); 870 memcpy(req->iv, req_ctx->iv, ivsize); 871 kfree_sensitive(req_ctx->iv); 872 } 873 874 skcipher_request_complete(req, err); 875 } 876 877 static int cc_cipher_process(struct skcipher_request *req, 878 enum drv_crypto_direction direction) 879 { 880 struct crypto_skcipher *sk_tfm = crypto_skcipher_reqtfm(req); 881 struct crypto_tfm *tfm = crypto_skcipher_tfm(sk_tfm); 882 struct cipher_req_ctx *req_ctx = skcipher_request_ctx(req); 883 unsigned int ivsize = crypto_skcipher_ivsize(sk_tfm); 884 struct scatterlist *dst = req->dst; 885 struct scatterlist *src = req->src; 886 unsigned int nbytes = req->cryptlen; 887 void *iv = req->iv; 888 struct cc_cipher_ctx *ctx_p = crypto_tfm_ctx(tfm); 889 struct device *dev = drvdata_to_dev(ctx_p->drvdata); 890 struct cc_hw_desc desc[MAX_SKCIPHER_SEQ_LEN]; 891 struct cc_crypto_req cc_req = {}; 892 int rc; 893 unsigned int seq_len = 0; 894 gfp_t flags = cc_gfp_flags(&req->base); 895 896 dev_dbg(dev, "%s req=%p iv=%p nbytes=%d\n", 897 ((direction == DRV_CRYPTO_DIRECTION_ENCRYPT) ? 898 "Encrypt" : "Decrypt"), req, iv, nbytes); 899 900 /* STAT_PHASE_0: Init and sanity checks */ 901 902 if (validate_data_size(ctx_p, nbytes)) { 903 dev_dbg(dev, "Unsupported data size %d.\n", nbytes); 904 rc = -EINVAL; 905 goto exit_process; 906 } 907 if (nbytes == 0) { 908 /* No data to process is valid */ 909 rc = 0; 910 goto exit_process; 911 } 912 913 if (ctx_p->fallback_on) { 914 struct skcipher_request *subreq = skcipher_request_ctx(req); 915 916 *subreq = *req; 917 skcipher_request_set_tfm(subreq, ctx_p->fallback_tfm); 918 if (direction == DRV_CRYPTO_DIRECTION_ENCRYPT) 919 return crypto_skcipher_encrypt(subreq); 920 else 921 return crypto_skcipher_decrypt(subreq); 922 } 923 924 /* The IV we are handed may be allocated from the stack so 925 * we must copy it to a DMAable buffer before use. 926 */ 927 req_ctx->iv = kmemdup(iv, ivsize, flags); 928 if (!req_ctx->iv) { 929 rc = -ENOMEM; 930 goto exit_process; 931 } 932 933 /* Setup request structure */ 934 cc_req.user_cb = cc_cipher_complete; 935 cc_req.user_arg = req; 936 937 /* Setup CPP operation details */ 938 if (ctx_p->key_type == CC_POLICY_PROTECTED_KEY) { 939 cc_req.cpp.is_cpp = true; 940 cc_req.cpp.alg = ctx_p->cpp.alg; 941 cc_req.cpp.slot = ctx_p->cpp.slot; 942 } 943 944 /* Setup request context */ 945 req_ctx->gen_ctx.op_type = direction; 946 947 /* STAT_PHASE_1: Map buffers */ 948 949 rc = cc_map_cipher_request(ctx_p->drvdata, req_ctx, ivsize, nbytes, 950 req_ctx->iv, src, dst, flags); 951 if (rc) { 952 dev_err(dev, "map_request() failed\n"); 953 goto exit_process; 954 } 955 956 /* STAT_PHASE_2: Create sequence */ 957 958 /* Setup state (IV) */ 959 cc_setup_state_desc(tfm, req_ctx, ivsize, nbytes, desc, &seq_len); 960 /* Setup MLLI line, if needed */ 961 cc_setup_mlli_desc(tfm, req_ctx, dst, src, nbytes, req, desc, &seq_len); 962 /* Setup key */ 963 cc_setup_key_desc(tfm, req_ctx, nbytes, desc, &seq_len); 964 /* Setup state (IV and XEX key) */ 965 cc_setup_xex_state_desc(tfm, req_ctx, ivsize, nbytes, desc, &seq_len); 966 /* Data processing */ 967 cc_setup_flow_desc(tfm, req_ctx, dst, src, nbytes, desc, &seq_len); 968 /* Read next IV */ 969 cc_setup_readiv_desc(tfm, req_ctx, ivsize, desc, &seq_len); 970 971 /* STAT_PHASE_3: Lock HW and push sequence */ 972 973 rc = cc_send_request(ctx_p->drvdata, &cc_req, desc, seq_len, 974 &req->base); 975 if (rc != -EINPROGRESS && rc != -EBUSY) { 976 /* Failed to send the request or request completed 977 * synchronously 978 */ 979 cc_unmap_cipher_request(dev, req_ctx, ivsize, src, dst); 980 } 981 982 exit_process: 983 if (rc != -EINPROGRESS && rc != -EBUSY) { 984 kfree_sensitive(req_ctx->iv); 985 } 986 987 return rc; 988 } 989 990 static int cc_cipher_encrypt(struct skcipher_request *req) 991 { 992 struct cipher_req_ctx *req_ctx = skcipher_request_ctx(req); 993 994 memset(req_ctx, 0, sizeof(*req_ctx)); 995 996 return cc_cipher_process(req, DRV_CRYPTO_DIRECTION_ENCRYPT); 997 } 998 999 static int cc_cipher_decrypt(struct skcipher_request *req) 1000 { 1001 struct cipher_req_ctx *req_ctx = skcipher_request_ctx(req); 1002 1003 memset(req_ctx, 0, sizeof(*req_ctx)); 1004 1005 return cc_cipher_process(req, DRV_CRYPTO_DIRECTION_DECRYPT); 1006 } 1007 1008 /* Block cipher alg */ 1009 static const struct cc_alg_template skcipher_algs[] = { 1010 { 1011 .name = "xts(paes)", 1012 .driver_name = "xts-paes-ccree", 1013 .blocksize = 1, 1014 .template_skcipher = { 1015 .setkey = cc_cipher_sethkey, 1016 .encrypt = cc_cipher_encrypt, 1017 .decrypt = cc_cipher_decrypt, 1018 .min_keysize = CC_HW_KEY_SIZE, 1019 .max_keysize = CC_HW_KEY_SIZE, 1020 .ivsize = AES_BLOCK_SIZE, 1021 }, 1022 .cipher_mode = DRV_CIPHER_XTS, 1023 .flow_mode = S_DIN_to_AES, 1024 .min_hw_rev = CC_HW_REV_630, 1025 .std_body = CC_STD_NIST, 1026 .sec_func = true, 1027 }, 1028 { 1029 .name = "essiv(cbc(paes),sha256)", 1030 .driver_name = "essiv-paes-ccree", 1031 .blocksize = AES_BLOCK_SIZE, 1032 .template_skcipher = { 1033 .setkey = cc_cipher_sethkey, 1034 .encrypt = cc_cipher_encrypt, 1035 .decrypt = cc_cipher_decrypt, 1036 .min_keysize = CC_HW_KEY_SIZE, 1037 .max_keysize = CC_HW_KEY_SIZE, 1038 .ivsize = AES_BLOCK_SIZE, 1039 }, 1040 .cipher_mode = DRV_CIPHER_ESSIV, 1041 .flow_mode = S_DIN_to_AES, 1042 .min_hw_rev = CC_HW_REV_712, 1043 .std_body = CC_STD_NIST, 1044 .sec_func = true, 1045 }, 1046 { 1047 .name = "ecb(paes)", 1048 .driver_name = "ecb-paes-ccree", 1049 .blocksize = AES_BLOCK_SIZE, 1050 .template_skcipher = { 1051 .setkey = cc_cipher_sethkey, 1052 .encrypt = cc_cipher_encrypt, 1053 .decrypt = cc_cipher_decrypt, 1054 .min_keysize = CC_HW_KEY_SIZE, 1055 .max_keysize = CC_HW_KEY_SIZE, 1056 .ivsize = 0, 1057 }, 1058 .cipher_mode = DRV_CIPHER_ECB, 1059 .flow_mode = S_DIN_to_AES, 1060 .min_hw_rev = CC_HW_REV_712, 1061 .std_body = CC_STD_NIST, 1062 .sec_func = true, 1063 }, 1064 { 1065 .name = "cbc(paes)", 1066 .driver_name = "cbc-paes-ccree", 1067 .blocksize = AES_BLOCK_SIZE, 1068 .template_skcipher = { 1069 .setkey = cc_cipher_sethkey, 1070 .encrypt = cc_cipher_encrypt, 1071 .decrypt = cc_cipher_decrypt, 1072 .min_keysize = CC_HW_KEY_SIZE, 1073 .max_keysize = CC_HW_KEY_SIZE, 1074 .ivsize = AES_BLOCK_SIZE, 1075 }, 1076 .cipher_mode = DRV_CIPHER_CBC, 1077 .flow_mode = S_DIN_to_AES, 1078 .min_hw_rev = CC_HW_REV_712, 1079 .std_body = CC_STD_NIST, 1080 .sec_func = true, 1081 }, 1082 { 1083 .name = "ofb(paes)", 1084 .driver_name = "ofb-paes-ccree", 1085 .blocksize = AES_BLOCK_SIZE, 1086 .template_skcipher = { 1087 .setkey = cc_cipher_sethkey, 1088 .encrypt = cc_cipher_encrypt, 1089 .decrypt = cc_cipher_decrypt, 1090 .min_keysize = CC_HW_KEY_SIZE, 1091 .max_keysize = CC_HW_KEY_SIZE, 1092 .ivsize = AES_BLOCK_SIZE, 1093 }, 1094 .cipher_mode = DRV_CIPHER_OFB, 1095 .flow_mode = S_DIN_to_AES, 1096 .min_hw_rev = CC_HW_REV_712, 1097 .std_body = CC_STD_NIST, 1098 .sec_func = true, 1099 }, 1100 { 1101 .name = "cts(cbc(paes))", 1102 .driver_name = "cts-cbc-paes-ccree", 1103 .blocksize = AES_BLOCK_SIZE, 1104 .template_skcipher = { 1105 .setkey = cc_cipher_sethkey, 1106 .encrypt = cc_cipher_encrypt, 1107 .decrypt = cc_cipher_decrypt, 1108 .min_keysize = CC_HW_KEY_SIZE, 1109 .max_keysize = CC_HW_KEY_SIZE, 1110 .ivsize = AES_BLOCK_SIZE, 1111 }, 1112 .cipher_mode = DRV_CIPHER_CBC_CTS, 1113 .flow_mode = S_DIN_to_AES, 1114 .min_hw_rev = CC_HW_REV_712, 1115 .std_body = CC_STD_NIST, 1116 .sec_func = true, 1117 }, 1118 { 1119 .name = "ctr(paes)", 1120 .driver_name = "ctr-paes-ccree", 1121 .blocksize = 1, 1122 .template_skcipher = { 1123 .setkey = cc_cipher_sethkey, 1124 .encrypt = cc_cipher_encrypt, 1125 .decrypt = cc_cipher_decrypt, 1126 .min_keysize = CC_HW_KEY_SIZE, 1127 .max_keysize = CC_HW_KEY_SIZE, 1128 .ivsize = AES_BLOCK_SIZE, 1129 }, 1130 .cipher_mode = DRV_CIPHER_CTR, 1131 .flow_mode = S_DIN_to_AES, 1132 .min_hw_rev = CC_HW_REV_712, 1133 .std_body = CC_STD_NIST, 1134 .sec_func = true, 1135 }, 1136 { 1137 /* See https://www.mail-archive.com/linux-crypto@vger.kernel.org/msg40576.html 1138 * for the reason why this differs from the generic 1139 * implementation. 1140 */ 1141 .name = "xts(aes)", 1142 .driver_name = "xts-aes-ccree", 1143 .blocksize = 1, 1144 .template_skcipher = { 1145 .setkey = cc_cipher_setkey, 1146 .encrypt = cc_cipher_encrypt, 1147 .decrypt = cc_cipher_decrypt, 1148 .min_keysize = AES_MIN_KEY_SIZE * 2, 1149 .max_keysize = AES_MAX_KEY_SIZE * 2, 1150 .ivsize = AES_BLOCK_SIZE, 1151 }, 1152 .cipher_mode = DRV_CIPHER_XTS, 1153 .flow_mode = S_DIN_to_AES, 1154 .min_hw_rev = CC_HW_REV_630, 1155 .std_body = CC_STD_NIST, 1156 }, 1157 { 1158 .name = "essiv(cbc(aes),sha256)", 1159 .driver_name = "essiv-aes-ccree", 1160 .blocksize = AES_BLOCK_SIZE, 1161 .template_skcipher = { 1162 .setkey = cc_cipher_setkey, 1163 .encrypt = cc_cipher_encrypt, 1164 .decrypt = cc_cipher_decrypt, 1165 .min_keysize = AES_MIN_KEY_SIZE, 1166 .max_keysize = AES_MAX_KEY_SIZE, 1167 .ivsize = AES_BLOCK_SIZE, 1168 }, 1169 .cipher_mode = DRV_CIPHER_ESSIV, 1170 .flow_mode = S_DIN_to_AES, 1171 .min_hw_rev = CC_HW_REV_712, 1172 .std_body = CC_STD_NIST, 1173 }, 1174 { 1175 .name = "ecb(aes)", 1176 .driver_name = "ecb-aes-ccree", 1177 .blocksize = AES_BLOCK_SIZE, 1178 .template_skcipher = { 1179 .setkey = cc_cipher_setkey, 1180 .encrypt = cc_cipher_encrypt, 1181 .decrypt = cc_cipher_decrypt, 1182 .min_keysize = AES_MIN_KEY_SIZE, 1183 .max_keysize = AES_MAX_KEY_SIZE, 1184 .ivsize = 0, 1185 }, 1186 .cipher_mode = DRV_CIPHER_ECB, 1187 .flow_mode = S_DIN_to_AES, 1188 .min_hw_rev = CC_HW_REV_630, 1189 .std_body = CC_STD_NIST, 1190 }, 1191 { 1192 .name = "cbc(aes)", 1193 .driver_name = "cbc-aes-ccree", 1194 .blocksize = AES_BLOCK_SIZE, 1195 .template_skcipher = { 1196 .setkey = cc_cipher_setkey, 1197 .encrypt = cc_cipher_encrypt, 1198 .decrypt = cc_cipher_decrypt, 1199 .min_keysize = AES_MIN_KEY_SIZE, 1200 .max_keysize = AES_MAX_KEY_SIZE, 1201 .ivsize = AES_BLOCK_SIZE, 1202 }, 1203 .cipher_mode = DRV_CIPHER_CBC, 1204 .flow_mode = S_DIN_to_AES, 1205 .min_hw_rev = CC_HW_REV_630, 1206 .std_body = CC_STD_NIST, 1207 }, 1208 { 1209 .name = "ofb(aes)", 1210 .driver_name = "ofb-aes-ccree", 1211 .blocksize = 1, 1212 .template_skcipher = { 1213 .setkey = cc_cipher_setkey, 1214 .encrypt = cc_cipher_encrypt, 1215 .decrypt = cc_cipher_decrypt, 1216 .min_keysize = AES_MIN_KEY_SIZE, 1217 .max_keysize = AES_MAX_KEY_SIZE, 1218 .ivsize = AES_BLOCK_SIZE, 1219 }, 1220 .cipher_mode = DRV_CIPHER_OFB, 1221 .flow_mode = S_DIN_to_AES, 1222 .min_hw_rev = CC_HW_REV_630, 1223 .std_body = CC_STD_NIST, 1224 }, 1225 { 1226 .name = "cts(cbc(aes))", 1227 .driver_name = "cts-cbc-aes-ccree", 1228 .blocksize = AES_BLOCK_SIZE, 1229 .template_skcipher = { 1230 .setkey = cc_cipher_setkey, 1231 .encrypt = cc_cipher_encrypt, 1232 .decrypt = cc_cipher_decrypt, 1233 .min_keysize = AES_MIN_KEY_SIZE, 1234 .max_keysize = AES_MAX_KEY_SIZE, 1235 .ivsize = AES_BLOCK_SIZE, 1236 }, 1237 .cipher_mode = DRV_CIPHER_CBC_CTS, 1238 .flow_mode = S_DIN_to_AES, 1239 .min_hw_rev = CC_HW_REV_630, 1240 .std_body = CC_STD_NIST, 1241 }, 1242 { 1243 .name = "ctr(aes)", 1244 .driver_name = "ctr-aes-ccree", 1245 .blocksize = 1, 1246 .template_skcipher = { 1247 .setkey = cc_cipher_setkey, 1248 .encrypt = cc_cipher_encrypt, 1249 .decrypt = cc_cipher_decrypt, 1250 .min_keysize = AES_MIN_KEY_SIZE, 1251 .max_keysize = AES_MAX_KEY_SIZE, 1252 .ivsize = AES_BLOCK_SIZE, 1253 }, 1254 .cipher_mode = DRV_CIPHER_CTR, 1255 .flow_mode = S_DIN_to_AES, 1256 .min_hw_rev = CC_HW_REV_630, 1257 .std_body = CC_STD_NIST, 1258 }, 1259 { 1260 .name = "cbc(des3_ede)", 1261 .driver_name = "cbc-3des-ccree", 1262 .blocksize = DES3_EDE_BLOCK_SIZE, 1263 .template_skcipher = { 1264 .setkey = cc_cipher_setkey, 1265 .encrypt = cc_cipher_encrypt, 1266 .decrypt = cc_cipher_decrypt, 1267 .min_keysize = DES3_EDE_KEY_SIZE, 1268 .max_keysize = DES3_EDE_KEY_SIZE, 1269 .ivsize = DES3_EDE_BLOCK_SIZE, 1270 }, 1271 .cipher_mode = DRV_CIPHER_CBC, 1272 .flow_mode = S_DIN_to_DES, 1273 .min_hw_rev = CC_HW_REV_630, 1274 .std_body = CC_STD_NIST, 1275 }, 1276 { 1277 .name = "ecb(des3_ede)", 1278 .driver_name = "ecb-3des-ccree", 1279 .blocksize = DES3_EDE_BLOCK_SIZE, 1280 .template_skcipher = { 1281 .setkey = cc_cipher_setkey, 1282 .encrypt = cc_cipher_encrypt, 1283 .decrypt = cc_cipher_decrypt, 1284 .min_keysize = DES3_EDE_KEY_SIZE, 1285 .max_keysize = DES3_EDE_KEY_SIZE, 1286 .ivsize = 0, 1287 }, 1288 .cipher_mode = DRV_CIPHER_ECB, 1289 .flow_mode = S_DIN_to_DES, 1290 .min_hw_rev = CC_HW_REV_630, 1291 .std_body = CC_STD_NIST, 1292 }, 1293 { 1294 .name = "cbc(des)", 1295 .driver_name = "cbc-des-ccree", 1296 .blocksize = DES_BLOCK_SIZE, 1297 .template_skcipher = { 1298 .setkey = cc_cipher_setkey, 1299 .encrypt = cc_cipher_encrypt, 1300 .decrypt = cc_cipher_decrypt, 1301 .min_keysize = DES_KEY_SIZE, 1302 .max_keysize = DES_KEY_SIZE, 1303 .ivsize = DES_BLOCK_SIZE, 1304 }, 1305 .cipher_mode = DRV_CIPHER_CBC, 1306 .flow_mode = S_DIN_to_DES, 1307 .min_hw_rev = CC_HW_REV_630, 1308 .std_body = CC_STD_NIST, 1309 }, 1310 { 1311 .name = "ecb(des)", 1312 .driver_name = "ecb-des-ccree", 1313 .blocksize = DES_BLOCK_SIZE, 1314 .template_skcipher = { 1315 .setkey = cc_cipher_setkey, 1316 .encrypt = cc_cipher_encrypt, 1317 .decrypt = cc_cipher_decrypt, 1318 .min_keysize = DES_KEY_SIZE, 1319 .max_keysize = DES_KEY_SIZE, 1320 .ivsize = 0, 1321 }, 1322 .cipher_mode = DRV_CIPHER_ECB, 1323 .flow_mode = S_DIN_to_DES, 1324 .min_hw_rev = CC_HW_REV_630, 1325 .std_body = CC_STD_NIST, 1326 }, 1327 { 1328 .name = "cbc(sm4)", 1329 .driver_name = "cbc-sm4-ccree", 1330 .blocksize = SM4_BLOCK_SIZE, 1331 .template_skcipher = { 1332 .setkey = cc_cipher_setkey, 1333 .encrypt = cc_cipher_encrypt, 1334 .decrypt = cc_cipher_decrypt, 1335 .min_keysize = SM4_KEY_SIZE, 1336 .max_keysize = SM4_KEY_SIZE, 1337 .ivsize = SM4_BLOCK_SIZE, 1338 }, 1339 .cipher_mode = DRV_CIPHER_CBC, 1340 .flow_mode = S_DIN_to_SM4, 1341 .min_hw_rev = CC_HW_REV_713, 1342 .std_body = CC_STD_OSCCA, 1343 }, 1344 { 1345 .name = "ecb(sm4)", 1346 .driver_name = "ecb-sm4-ccree", 1347 .blocksize = SM4_BLOCK_SIZE, 1348 .template_skcipher = { 1349 .setkey = cc_cipher_setkey, 1350 .encrypt = cc_cipher_encrypt, 1351 .decrypt = cc_cipher_decrypt, 1352 .min_keysize = SM4_KEY_SIZE, 1353 .max_keysize = SM4_KEY_SIZE, 1354 .ivsize = 0, 1355 }, 1356 .cipher_mode = DRV_CIPHER_ECB, 1357 .flow_mode = S_DIN_to_SM4, 1358 .min_hw_rev = CC_HW_REV_713, 1359 .std_body = CC_STD_OSCCA, 1360 }, 1361 { 1362 .name = "ctr(sm4)", 1363 .driver_name = "ctr-sm4-ccree", 1364 .blocksize = 1, 1365 .template_skcipher = { 1366 .setkey = cc_cipher_setkey, 1367 .encrypt = cc_cipher_encrypt, 1368 .decrypt = cc_cipher_decrypt, 1369 .min_keysize = SM4_KEY_SIZE, 1370 .max_keysize = SM4_KEY_SIZE, 1371 .ivsize = SM4_BLOCK_SIZE, 1372 }, 1373 .cipher_mode = DRV_CIPHER_CTR, 1374 .flow_mode = S_DIN_to_SM4, 1375 .min_hw_rev = CC_HW_REV_713, 1376 .std_body = CC_STD_OSCCA, 1377 }, 1378 { 1379 .name = "cbc(psm4)", 1380 .driver_name = "cbc-psm4-ccree", 1381 .blocksize = SM4_BLOCK_SIZE, 1382 .template_skcipher = { 1383 .setkey = cc_cipher_sethkey, 1384 .encrypt = cc_cipher_encrypt, 1385 .decrypt = cc_cipher_decrypt, 1386 .min_keysize = CC_HW_KEY_SIZE, 1387 .max_keysize = CC_HW_KEY_SIZE, 1388 .ivsize = SM4_BLOCK_SIZE, 1389 }, 1390 .cipher_mode = DRV_CIPHER_CBC, 1391 .flow_mode = S_DIN_to_SM4, 1392 .min_hw_rev = CC_HW_REV_713, 1393 .std_body = CC_STD_OSCCA, 1394 .sec_func = true, 1395 }, 1396 { 1397 .name = "ctr(psm4)", 1398 .driver_name = "ctr-psm4-ccree", 1399 .blocksize = SM4_BLOCK_SIZE, 1400 .template_skcipher = { 1401 .setkey = cc_cipher_sethkey, 1402 .encrypt = cc_cipher_encrypt, 1403 .decrypt = cc_cipher_decrypt, 1404 .min_keysize = CC_HW_KEY_SIZE, 1405 .max_keysize = CC_HW_KEY_SIZE, 1406 .ivsize = SM4_BLOCK_SIZE, 1407 }, 1408 .cipher_mode = DRV_CIPHER_CTR, 1409 .flow_mode = S_DIN_to_SM4, 1410 .min_hw_rev = CC_HW_REV_713, 1411 .std_body = CC_STD_OSCCA, 1412 .sec_func = true, 1413 }, 1414 }; 1415 1416 static struct cc_crypto_alg *cc_create_alg(const struct cc_alg_template *tmpl, 1417 struct device *dev) 1418 { 1419 struct cc_crypto_alg *t_alg; 1420 struct skcipher_alg *alg; 1421 1422 t_alg = devm_kzalloc(dev, sizeof(*t_alg), GFP_KERNEL); 1423 if (!t_alg) 1424 return ERR_PTR(-ENOMEM); 1425 1426 alg = &t_alg->skcipher_alg; 1427 1428 memcpy(alg, &tmpl->template_skcipher, sizeof(*alg)); 1429 1430 snprintf(alg->base.cra_name, CRYPTO_MAX_ALG_NAME, "%s", tmpl->name); 1431 snprintf(alg->base.cra_driver_name, CRYPTO_MAX_ALG_NAME, "%s", 1432 tmpl->driver_name); 1433 alg->base.cra_module = THIS_MODULE; 1434 alg->base.cra_priority = CC_CRA_PRIO; 1435 alg->base.cra_blocksize = tmpl->blocksize; 1436 alg->base.cra_alignmask = 0; 1437 alg->base.cra_ctxsize = sizeof(struct cc_cipher_ctx); 1438 1439 alg->base.cra_init = cc_cipher_init; 1440 alg->base.cra_exit = cc_cipher_exit; 1441 alg->base.cra_flags = CRYPTO_ALG_ASYNC | CRYPTO_ALG_KERN_DRIVER_ONLY; 1442 1443 t_alg->cipher_mode = tmpl->cipher_mode; 1444 t_alg->flow_mode = tmpl->flow_mode; 1445 1446 return t_alg; 1447 } 1448 1449 int cc_cipher_free(struct cc_drvdata *drvdata) 1450 { 1451 struct cc_crypto_alg *t_alg, *n; 1452 1453 /* Remove registered algs */ 1454 list_for_each_entry_safe(t_alg, n, &drvdata->alg_list, entry) { 1455 crypto_unregister_skcipher(&t_alg->skcipher_alg); 1456 list_del(&t_alg->entry); 1457 } 1458 return 0; 1459 } 1460 1461 int cc_cipher_alloc(struct cc_drvdata *drvdata) 1462 { 1463 struct cc_crypto_alg *t_alg; 1464 struct device *dev = drvdata_to_dev(drvdata); 1465 int rc = -ENOMEM; 1466 int alg; 1467 1468 INIT_LIST_HEAD(&drvdata->alg_list); 1469 1470 /* Linux crypto */ 1471 dev_dbg(dev, "Number of algorithms = %zu\n", 1472 ARRAY_SIZE(skcipher_algs)); 1473 for (alg = 0; alg < ARRAY_SIZE(skcipher_algs); alg++) { 1474 if ((skcipher_algs[alg].min_hw_rev > drvdata->hw_rev) || 1475 !(drvdata->std_bodies & skcipher_algs[alg].std_body) || 1476 (drvdata->sec_disabled && skcipher_algs[alg].sec_func)) 1477 continue; 1478 1479 dev_dbg(dev, "creating %s\n", skcipher_algs[alg].driver_name); 1480 t_alg = cc_create_alg(&skcipher_algs[alg], dev); 1481 if (IS_ERR(t_alg)) { 1482 rc = PTR_ERR(t_alg); 1483 dev_err(dev, "%s alg allocation failed\n", 1484 skcipher_algs[alg].driver_name); 1485 goto fail0; 1486 } 1487 t_alg->drvdata = drvdata; 1488 1489 dev_dbg(dev, "registering %s\n", 1490 skcipher_algs[alg].driver_name); 1491 rc = crypto_register_skcipher(&t_alg->skcipher_alg); 1492 dev_dbg(dev, "%s alg registration rc = %x\n", 1493 t_alg->skcipher_alg.base.cra_driver_name, rc); 1494 if (rc) { 1495 dev_err(dev, "%s alg registration failed\n", 1496 t_alg->skcipher_alg.base.cra_driver_name); 1497 goto fail0; 1498 } 1499 1500 list_add_tail(&t_alg->entry, &drvdata->alg_list); 1501 dev_dbg(dev, "Registered %s\n", 1502 t_alg->skcipher_alg.base.cra_driver_name); 1503 } 1504 return 0; 1505 1506 fail0: 1507 cc_cipher_free(drvdata); 1508 return rc; 1509 } 1510