1 // SPDX-License-Identifier: GPL-2.0 2 /* Copyright (C) 2012-2018 ARM Limited or its affiliates. */ 3 4 #include <linux/kernel.h> 5 #include <linux/module.h> 6 #include <crypto/algapi.h> 7 #include <crypto/internal/skcipher.h> 8 #include <crypto/des.h> 9 #include <crypto/xts.h> 10 #include <crypto/scatterwalk.h> 11 12 #include "cc_driver.h" 13 #include "cc_lli_defs.h" 14 #include "cc_buffer_mgr.h" 15 #include "cc_cipher.h" 16 #include "cc_request_mgr.h" 17 18 #define MAX_ABLKCIPHER_SEQ_LEN 6 19 20 #define template_skcipher template_u.skcipher 21 22 struct cc_cipher_handle { 23 struct list_head alg_list; 24 }; 25 26 struct cc_user_key_info { 27 u8 *key; 28 dma_addr_t key_dma_addr; 29 }; 30 31 struct cc_hw_key_info { 32 enum cc_hw_crypto_key key1_slot; 33 enum cc_hw_crypto_key key2_slot; 34 }; 35 36 struct cc_cipher_ctx { 37 struct cc_drvdata *drvdata; 38 int keylen; 39 int key_round_number; 40 int cipher_mode; 41 int flow_mode; 42 unsigned int flags; 43 bool hw_key; 44 struct cc_user_key_info user; 45 struct cc_hw_key_info hw; 46 struct crypto_shash *shash_tfm; 47 }; 48 49 static void cc_cipher_complete(struct device *dev, void *cc_req, int err); 50 51 static inline bool cc_is_hw_key(struct crypto_tfm *tfm) 52 { 53 struct cc_cipher_ctx *ctx_p = crypto_tfm_ctx(tfm); 54 55 return ctx_p->hw_key; 56 } 57 58 static int validate_keys_sizes(struct cc_cipher_ctx *ctx_p, u32 size) 59 { 60 switch (ctx_p->flow_mode) { 61 case S_DIN_to_AES: 62 switch (size) { 63 case CC_AES_128_BIT_KEY_SIZE: 64 case CC_AES_192_BIT_KEY_SIZE: 65 if (ctx_p->cipher_mode != DRV_CIPHER_XTS && 66 ctx_p->cipher_mode != DRV_CIPHER_ESSIV && 67 ctx_p->cipher_mode != DRV_CIPHER_BITLOCKER) 68 return 0; 69 break; 70 case CC_AES_256_BIT_KEY_SIZE: 71 return 0; 72 case (CC_AES_192_BIT_KEY_SIZE * 2): 73 case (CC_AES_256_BIT_KEY_SIZE * 2): 74 if (ctx_p->cipher_mode == DRV_CIPHER_XTS || 75 ctx_p->cipher_mode == DRV_CIPHER_ESSIV || 76 ctx_p->cipher_mode == DRV_CIPHER_BITLOCKER) 77 return 0; 78 break; 79 default: 80 break; 81 } 82 case S_DIN_to_DES: 83 if (size == DES3_EDE_KEY_SIZE || size == DES_KEY_SIZE) 84 return 0; 85 break; 86 default: 87 break; 88 } 89 return -EINVAL; 90 } 91 92 static int validate_data_size(struct cc_cipher_ctx *ctx_p, 93 unsigned int size) 94 { 95 switch (ctx_p->flow_mode) { 96 case S_DIN_to_AES: 97 switch (ctx_p->cipher_mode) { 98 case DRV_CIPHER_XTS: 99 if (size >= AES_BLOCK_SIZE && 100 IS_ALIGNED(size, AES_BLOCK_SIZE)) 101 return 0; 102 break; 103 case DRV_CIPHER_CBC_CTS: 104 if (size >= AES_BLOCK_SIZE) 105 return 0; 106 break; 107 case DRV_CIPHER_OFB: 108 case DRV_CIPHER_CTR: 109 return 0; 110 case DRV_CIPHER_ECB: 111 case DRV_CIPHER_CBC: 112 case DRV_CIPHER_ESSIV: 113 case DRV_CIPHER_BITLOCKER: 114 if (IS_ALIGNED(size, AES_BLOCK_SIZE)) 115 return 0; 116 break; 117 default: 118 break; 119 } 120 break; 121 case S_DIN_to_DES: 122 if (IS_ALIGNED(size, DES_BLOCK_SIZE)) 123 return 0; 124 break; 125 default: 126 break; 127 } 128 return -EINVAL; 129 } 130 131 static int cc_cipher_init(struct crypto_tfm *tfm) 132 { 133 struct cc_cipher_ctx *ctx_p = crypto_tfm_ctx(tfm); 134 struct cc_crypto_alg *cc_alg = 135 container_of(tfm->__crt_alg, struct cc_crypto_alg, 136 skcipher_alg.base); 137 struct device *dev = drvdata_to_dev(cc_alg->drvdata); 138 unsigned int max_key_buf_size = cc_alg->skcipher_alg.max_keysize; 139 int rc = 0; 140 141 dev_dbg(dev, "Initializing context @%p for %s\n", ctx_p, 142 crypto_tfm_alg_name(tfm)); 143 144 crypto_skcipher_set_reqsize(__crypto_skcipher_cast(tfm), 145 sizeof(struct cipher_req_ctx)); 146 147 ctx_p->cipher_mode = cc_alg->cipher_mode; 148 ctx_p->flow_mode = cc_alg->flow_mode; 149 ctx_p->drvdata = cc_alg->drvdata; 150 151 /* Allocate key buffer, cache line aligned */ 152 ctx_p->user.key = kmalloc(max_key_buf_size, GFP_KERNEL); 153 if (!ctx_p->user.key) 154 return -ENOMEM; 155 156 dev_dbg(dev, "Allocated key buffer in context. key=@%p\n", 157 ctx_p->user.key); 158 159 /* Map key buffer */ 160 ctx_p->user.key_dma_addr = dma_map_single(dev, (void *)ctx_p->user.key, 161 max_key_buf_size, 162 DMA_TO_DEVICE); 163 if (dma_mapping_error(dev, ctx_p->user.key_dma_addr)) { 164 dev_err(dev, "Mapping Key %u B at va=%pK for DMA failed\n", 165 max_key_buf_size, ctx_p->user.key); 166 return -ENOMEM; 167 } 168 dev_dbg(dev, "Mapped key %u B at va=%pK to dma=%pad\n", 169 max_key_buf_size, ctx_p->user.key, &ctx_p->user.key_dma_addr); 170 171 if (ctx_p->cipher_mode == DRV_CIPHER_ESSIV) { 172 /* Alloc hash tfm for essiv */ 173 ctx_p->shash_tfm = crypto_alloc_shash("sha256-generic", 0, 0); 174 if (IS_ERR(ctx_p->shash_tfm)) { 175 dev_err(dev, "Error allocating hash tfm for ESSIV.\n"); 176 return PTR_ERR(ctx_p->shash_tfm); 177 } 178 } 179 180 return rc; 181 } 182 183 static void cc_cipher_exit(struct crypto_tfm *tfm) 184 { 185 struct crypto_alg *alg = tfm->__crt_alg; 186 struct cc_crypto_alg *cc_alg = 187 container_of(alg, struct cc_crypto_alg, 188 skcipher_alg.base); 189 unsigned int max_key_buf_size = cc_alg->skcipher_alg.max_keysize; 190 struct cc_cipher_ctx *ctx_p = crypto_tfm_ctx(tfm); 191 struct device *dev = drvdata_to_dev(ctx_p->drvdata); 192 193 dev_dbg(dev, "Clearing context @%p for %s\n", 194 crypto_tfm_ctx(tfm), crypto_tfm_alg_name(tfm)); 195 196 if (ctx_p->cipher_mode == DRV_CIPHER_ESSIV) { 197 /* Free hash tfm for essiv */ 198 crypto_free_shash(ctx_p->shash_tfm); 199 ctx_p->shash_tfm = NULL; 200 } 201 202 /* Unmap key buffer */ 203 dma_unmap_single(dev, ctx_p->user.key_dma_addr, max_key_buf_size, 204 DMA_TO_DEVICE); 205 dev_dbg(dev, "Unmapped key buffer key_dma_addr=%pad\n", 206 &ctx_p->user.key_dma_addr); 207 208 /* Free key buffer in context */ 209 kzfree(ctx_p->user.key); 210 dev_dbg(dev, "Free key buffer in context. key=@%p\n", ctx_p->user.key); 211 } 212 213 struct tdes_keys { 214 u8 key1[DES_KEY_SIZE]; 215 u8 key2[DES_KEY_SIZE]; 216 u8 key3[DES_KEY_SIZE]; 217 }; 218 219 static enum cc_hw_crypto_key cc_slot_to_hw_key(int slot_num) 220 { 221 switch (slot_num) { 222 case 0: 223 return KFDE0_KEY; 224 case 1: 225 return KFDE1_KEY; 226 case 2: 227 return KFDE2_KEY; 228 case 3: 229 return KFDE3_KEY; 230 } 231 return END_OF_KEYS; 232 } 233 234 static int cc_cipher_sethkey(struct crypto_skcipher *sktfm, const u8 *key, 235 unsigned int keylen) 236 { 237 struct crypto_tfm *tfm = crypto_skcipher_tfm(sktfm); 238 struct cc_cipher_ctx *ctx_p = crypto_tfm_ctx(tfm); 239 struct device *dev = drvdata_to_dev(ctx_p->drvdata); 240 struct cc_hkey_info hki; 241 242 dev_dbg(dev, "Setting HW key in context @%p for %s. keylen=%u\n", 243 ctx_p, crypto_tfm_alg_name(tfm), keylen); 244 dump_byte_array("key", (u8 *)key, keylen); 245 246 /* STAT_PHASE_0: Init and sanity checks */ 247 248 /* This check the size of the hardware key token */ 249 if (keylen != sizeof(hki)) { 250 dev_err(dev, "Unsupported HW key size %d.\n", keylen); 251 crypto_tfm_set_flags(tfm, CRYPTO_TFM_RES_BAD_KEY_LEN); 252 return -EINVAL; 253 } 254 255 if (ctx_p->flow_mode != S_DIN_to_AES) { 256 dev_err(dev, "HW key not supported for non-AES flows\n"); 257 return -EINVAL; 258 } 259 260 memcpy(&hki, key, keylen); 261 262 /* The real key len for crypto op is the size of the HW key 263 * referenced by the HW key slot, not the hardware key token 264 */ 265 keylen = hki.keylen; 266 267 if (validate_keys_sizes(ctx_p, keylen)) { 268 dev_err(dev, "Unsupported key size %d.\n", keylen); 269 crypto_tfm_set_flags(tfm, CRYPTO_TFM_RES_BAD_KEY_LEN); 270 return -EINVAL; 271 } 272 273 ctx_p->hw.key1_slot = cc_slot_to_hw_key(hki.hw_key1); 274 if (ctx_p->hw.key1_slot == END_OF_KEYS) { 275 dev_err(dev, "Unsupported hw key1 number (%d)\n", hki.hw_key1); 276 return -EINVAL; 277 } 278 279 if (ctx_p->cipher_mode == DRV_CIPHER_XTS || 280 ctx_p->cipher_mode == DRV_CIPHER_ESSIV || 281 ctx_p->cipher_mode == DRV_CIPHER_BITLOCKER) { 282 if (hki.hw_key1 == hki.hw_key2) { 283 dev_err(dev, "Illegal hw key numbers (%d,%d)\n", 284 hki.hw_key1, hki.hw_key2); 285 return -EINVAL; 286 } 287 ctx_p->hw.key2_slot = cc_slot_to_hw_key(hki.hw_key2); 288 if (ctx_p->hw.key2_slot == END_OF_KEYS) { 289 dev_err(dev, "Unsupported hw key2 number (%d)\n", 290 hki.hw_key2); 291 return -EINVAL; 292 } 293 } 294 295 ctx_p->keylen = keylen; 296 ctx_p->hw_key = true; 297 dev_dbg(dev, "cc_is_hw_key ret 0"); 298 299 return 0; 300 } 301 302 static int cc_cipher_setkey(struct crypto_skcipher *sktfm, const u8 *key, 303 unsigned int keylen) 304 { 305 struct crypto_tfm *tfm = crypto_skcipher_tfm(sktfm); 306 struct cc_cipher_ctx *ctx_p = crypto_tfm_ctx(tfm); 307 struct device *dev = drvdata_to_dev(ctx_p->drvdata); 308 u32 tmp[DES3_EDE_EXPKEY_WORDS]; 309 struct cc_crypto_alg *cc_alg = 310 container_of(tfm->__crt_alg, struct cc_crypto_alg, 311 skcipher_alg.base); 312 unsigned int max_key_buf_size = cc_alg->skcipher_alg.max_keysize; 313 314 dev_dbg(dev, "Setting key in context @%p for %s. keylen=%u\n", 315 ctx_p, crypto_tfm_alg_name(tfm), keylen); 316 dump_byte_array("key", (u8 *)key, keylen); 317 318 /* STAT_PHASE_0: Init and sanity checks */ 319 320 if (validate_keys_sizes(ctx_p, keylen)) { 321 dev_err(dev, "Unsupported key size %d.\n", keylen); 322 crypto_tfm_set_flags(tfm, CRYPTO_TFM_RES_BAD_KEY_LEN); 323 return -EINVAL; 324 } 325 326 ctx_p->hw_key = false; 327 328 /* 329 * Verify DES weak keys 330 * Note that we're dropping the expanded key since the 331 * HW does the expansion on its own. 332 */ 333 if (ctx_p->flow_mode == S_DIN_to_DES) { 334 if (keylen == DES3_EDE_KEY_SIZE && 335 __des3_ede_setkey(tmp, &tfm->crt_flags, key, 336 DES3_EDE_KEY_SIZE)) { 337 dev_dbg(dev, "weak 3DES key"); 338 return -EINVAL; 339 } else if (!des_ekey(tmp, key) && 340 (crypto_tfm_get_flags(tfm) & CRYPTO_TFM_REQ_WEAK_KEY)) { 341 tfm->crt_flags |= CRYPTO_TFM_RES_WEAK_KEY; 342 dev_dbg(dev, "weak DES key"); 343 return -EINVAL; 344 } 345 } 346 347 if (ctx_p->cipher_mode == DRV_CIPHER_XTS && 348 xts_check_key(tfm, key, keylen)) { 349 dev_dbg(dev, "weak XTS key"); 350 return -EINVAL; 351 } 352 353 /* STAT_PHASE_1: Copy key to ctx */ 354 dma_sync_single_for_cpu(dev, ctx_p->user.key_dma_addr, 355 max_key_buf_size, DMA_TO_DEVICE); 356 357 memcpy(ctx_p->user.key, key, keylen); 358 if (keylen == 24) 359 memset(ctx_p->user.key + 24, 0, CC_AES_KEY_SIZE_MAX - 24); 360 361 if (ctx_p->cipher_mode == DRV_CIPHER_ESSIV) { 362 /* sha256 for key2 - use sw implementation */ 363 int key_len = keylen >> 1; 364 int err; 365 366 SHASH_DESC_ON_STACK(desc, ctx_p->shash_tfm); 367 368 desc->tfm = ctx_p->shash_tfm; 369 370 err = crypto_shash_digest(desc, ctx_p->user.key, key_len, 371 ctx_p->user.key + key_len); 372 if (err) { 373 dev_err(dev, "Failed to hash ESSIV key.\n"); 374 return err; 375 } 376 } 377 dma_sync_single_for_device(dev, ctx_p->user.key_dma_addr, 378 max_key_buf_size, DMA_TO_DEVICE); 379 ctx_p->keylen = keylen; 380 381 dev_dbg(dev, "return safely"); 382 return 0; 383 } 384 385 static void cc_setup_cipher_desc(struct crypto_tfm *tfm, 386 struct cipher_req_ctx *req_ctx, 387 unsigned int ivsize, unsigned int nbytes, 388 struct cc_hw_desc desc[], 389 unsigned int *seq_size) 390 { 391 struct cc_cipher_ctx *ctx_p = crypto_tfm_ctx(tfm); 392 struct device *dev = drvdata_to_dev(ctx_p->drvdata); 393 int cipher_mode = ctx_p->cipher_mode; 394 int flow_mode = ctx_p->flow_mode; 395 int direction = req_ctx->gen_ctx.op_type; 396 dma_addr_t key_dma_addr = ctx_p->user.key_dma_addr; 397 unsigned int key_len = ctx_p->keylen; 398 dma_addr_t iv_dma_addr = req_ctx->gen_ctx.iv_dma_addr; 399 unsigned int du_size = nbytes; 400 401 struct cc_crypto_alg *cc_alg = 402 container_of(tfm->__crt_alg, struct cc_crypto_alg, 403 skcipher_alg.base); 404 405 if (cc_alg->data_unit) 406 du_size = cc_alg->data_unit; 407 408 switch (cipher_mode) { 409 case DRV_CIPHER_CBC: 410 case DRV_CIPHER_CBC_CTS: 411 case DRV_CIPHER_CTR: 412 case DRV_CIPHER_OFB: 413 /* Load cipher state */ 414 hw_desc_init(&desc[*seq_size]); 415 set_din_type(&desc[*seq_size], DMA_DLLI, iv_dma_addr, ivsize, 416 NS_BIT); 417 set_cipher_config0(&desc[*seq_size], direction); 418 set_flow_mode(&desc[*seq_size], flow_mode); 419 set_cipher_mode(&desc[*seq_size], cipher_mode); 420 if (cipher_mode == DRV_CIPHER_CTR || 421 cipher_mode == DRV_CIPHER_OFB) { 422 set_setup_mode(&desc[*seq_size], SETUP_LOAD_STATE1); 423 } else { 424 set_setup_mode(&desc[*seq_size], SETUP_LOAD_STATE0); 425 } 426 (*seq_size)++; 427 /*FALLTHROUGH*/ 428 case DRV_CIPHER_ECB: 429 /* Load key */ 430 hw_desc_init(&desc[*seq_size]); 431 set_cipher_mode(&desc[*seq_size], cipher_mode); 432 set_cipher_config0(&desc[*seq_size], direction); 433 if (flow_mode == S_DIN_to_AES) { 434 if (cc_is_hw_key(tfm)) { 435 set_hw_crypto_key(&desc[*seq_size], 436 ctx_p->hw.key1_slot); 437 } else { 438 set_din_type(&desc[*seq_size], DMA_DLLI, 439 key_dma_addr, ((key_len == 24) ? 440 AES_MAX_KEY_SIZE : 441 key_len), NS_BIT); 442 } 443 set_key_size_aes(&desc[*seq_size], key_len); 444 } else { 445 /*des*/ 446 set_din_type(&desc[*seq_size], DMA_DLLI, key_dma_addr, 447 key_len, NS_BIT); 448 set_key_size_des(&desc[*seq_size], key_len); 449 } 450 set_flow_mode(&desc[*seq_size], flow_mode); 451 set_setup_mode(&desc[*seq_size], SETUP_LOAD_KEY0); 452 (*seq_size)++; 453 break; 454 case DRV_CIPHER_XTS: 455 case DRV_CIPHER_ESSIV: 456 case DRV_CIPHER_BITLOCKER: 457 /* Load AES key */ 458 hw_desc_init(&desc[*seq_size]); 459 set_cipher_mode(&desc[*seq_size], cipher_mode); 460 set_cipher_config0(&desc[*seq_size], direction); 461 if (cc_is_hw_key(tfm)) { 462 set_hw_crypto_key(&desc[*seq_size], 463 ctx_p->hw.key1_slot); 464 } else { 465 set_din_type(&desc[*seq_size], DMA_DLLI, key_dma_addr, 466 (key_len / 2), NS_BIT); 467 } 468 set_key_size_aes(&desc[*seq_size], (key_len / 2)); 469 set_flow_mode(&desc[*seq_size], flow_mode); 470 set_setup_mode(&desc[*seq_size], SETUP_LOAD_KEY0); 471 (*seq_size)++; 472 473 /* load XEX key */ 474 hw_desc_init(&desc[*seq_size]); 475 set_cipher_mode(&desc[*seq_size], cipher_mode); 476 set_cipher_config0(&desc[*seq_size], direction); 477 if (cc_is_hw_key(tfm)) { 478 set_hw_crypto_key(&desc[*seq_size], 479 ctx_p->hw.key2_slot); 480 } else { 481 set_din_type(&desc[*seq_size], DMA_DLLI, 482 (key_dma_addr + (key_len / 2)), 483 (key_len / 2), NS_BIT); 484 } 485 set_xex_data_unit_size(&desc[*seq_size], du_size); 486 set_flow_mode(&desc[*seq_size], S_DIN_to_AES2); 487 set_key_size_aes(&desc[*seq_size], (key_len / 2)); 488 set_setup_mode(&desc[*seq_size], SETUP_LOAD_XEX_KEY); 489 (*seq_size)++; 490 491 /* Set state */ 492 hw_desc_init(&desc[*seq_size]); 493 set_setup_mode(&desc[*seq_size], SETUP_LOAD_STATE1); 494 set_cipher_mode(&desc[*seq_size], cipher_mode); 495 set_cipher_config0(&desc[*seq_size], direction); 496 set_key_size_aes(&desc[*seq_size], (key_len / 2)); 497 set_flow_mode(&desc[*seq_size], flow_mode); 498 set_din_type(&desc[*seq_size], DMA_DLLI, iv_dma_addr, 499 CC_AES_BLOCK_SIZE, NS_BIT); 500 (*seq_size)++; 501 break; 502 default: 503 dev_err(dev, "Unsupported cipher mode (%d)\n", cipher_mode); 504 } 505 } 506 507 static void cc_setup_cipher_data(struct crypto_tfm *tfm, 508 struct cipher_req_ctx *req_ctx, 509 struct scatterlist *dst, 510 struct scatterlist *src, unsigned int nbytes, 511 void *areq, struct cc_hw_desc desc[], 512 unsigned int *seq_size) 513 { 514 struct cc_cipher_ctx *ctx_p = crypto_tfm_ctx(tfm); 515 struct device *dev = drvdata_to_dev(ctx_p->drvdata); 516 unsigned int flow_mode = ctx_p->flow_mode; 517 518 switch (ctx_p->flow_mode) { 519 case S_DIN_to_AES: 520 flow_mode = DIN_AES_DOUT; 521 break; 522 case S_DIN_to_DES: 523 flow_mode = DIN_DES_DOUT; 524 break; 525 default: 526 dev_err(dev, "invalid flow mode, flow_mode = %d\n", flow_mode); 527 return; 528 } 529 /* Process */ 530 if (req_ctx->dma_buf_type == CC_DMA_BUF_DLLI) { 531 dev_dbg(dev, " data params addr %pad length 0x%X\n", 532 &sg_dma_address(src), nbytes); 533 dev_dbg(dev, " data params addr %pad length 0x%X\n", 534 &sg_dma_address(dst), nbytes); 535 hw_desc_init(&desc[*seq_size]); 536 set_din_type(&desc[*seq_size], DMA_DLLI, sg_dma_address(src), 537 nbytes, NS_BIT); 538 set_dout_dlli(&desc[*seq_size], sg_dma_address(dst), 539 nbytes, NS_BIT, (!areq ? 0 : 1)); 540 if (areq) 541 set_queue_last_ind(ctx_p->drvdata, &desc[*seq_size]); 542 543 set_flow_mode(&desc[*seq_size], flow_mode); 544 (*seq_size)++; 545 } else { 546 /* bypass */ 547 dev_dbg(dev, " bypass params addr %pad length 0x%X addr 0x%08X\n", 548 &req_ctx->mlli_params.mlli_dma_addr, 549 req_ctx->mlli_params.mlli_len, 550 (unsigned int)ctx_p->drvdata->mlli_sram_addr); 551 hw_desc_init(&desc[*seq_size]); 552 set_din_type(&desc[*seq_size], DMA_DLLI, 553 req_ctx->mlli_params.mlli_dma_addr, 554 req_ctx->mlli_params.mlli_len, NS_BIT); 555 set_dout_sram(&desc[*seq_size], 556 ctx_p->drvdata->mlli_sram_addr, 557 req_ctx->mlli_params.mlli_len); 558 set_flow_mode(&desc[*seq_size], BYPASS); 559 (*seq_size)++; 560 561 hw_desc_init(&desc[*seq_size]); 562 set_din_type(&desc[*seq_size], DMA_MLLI, 563 ctx_p->drvdata->mlli_sram_addr, 564 req_ctx->in_mlli_nents, NS_BIT); 565 if (req_ctx->out_nents == 0) { 566 dev_dbg(dev, " din/dout params addr 0x%08X addr 0x%08X\n", 567 (unsigned int)ctx_p->drvdata->mlli_sram_addr, 568 (unsigned int)ctx_p->drvdata->mlli_sram_addr); 569 set_dout_mlli(&desc[*seq_size], 570 ctx_p->drvdata->mlli_sram_addr, 571 req_ctx->in_mlli_nents, NS_BIT, 572 (!areq ? 0 : 1)); 573 } else { 574 dev_dbg(dev, " din/dout params addr 0x%08X addr 0x%08X\n", 575 (unsigned int)ctx_p->drvdata->mlli_sram_addr, 576 (unsigned int)ctx_p->drvdata->mlli_sram_addr + 577 (u32)LLI_ENTRY_BYTE_SIZE * req_ctx->in_nents); 578 set_dout_mlli(&desc[*seq_size], 579 (ctx_p->drvdata->mlli_sram_addr + 580 (LLI_ENTRY_BYTE_SIZE * 581 req_ctx->in_mlli_nents)), 582 req_ctx->out_mlli_nents, NS_BIT, 583 (!areq ? 0 : 1)); 584 } 585 if (areq) 586 set_queue_last_ind(ctx_p->drvdata, &desc[*seq_size]); 587 588 set_flow_mode(&desc[*seq_size], flow_mode); 589 (*seq_size)++; 590 } 591 } 592 593 /* 594 * Update a CTR-AES 128 bit counter 595 */ 596 static void cc_update_ctr(u8 *ctr, unsigned int increment) 597 { 598 if (IS_ENABLED(CONFIG_HAVE_EFFICIENT_UNALIGNED_ACCESS) || 599 IS_ALIGNED((unsigned long)ctr, 8)) { 600 601 __be64 *high_be = (__be64 *)ctr; 602 __be64 *low_be = high_be + 1; 603 u64 orig_low = __be64_to_cpu(*low_be); 604 u64 new_low = orig_low + (u64)increment; 605 606 *low_be = __cpu_to_be64(new_low); 607 608 if (new_low < orig_low) 609 *high_be = __cpu_to_be64(__be64_to_cpu(*high_be) + 1); 610 } else { 611 u8 *pos = (ctr + AES_BLOCK_SIZE); 612 u8 val; 613 unsigned int size; 614 615 for (; increment; increment--) 616 for (size = AES_BLOCK_SIZE; size; size--) { 617 val = *--pos + 1; 618 *pos = val; 619 if (val) 620 break; 621 } 622 } 623 } 624 625 static void cc_cipher_complete(struct device *dev, void *cc_req, int err) 626 { 627 struct skcipher_request *req = (struct skcipher_request *)cc_req; 628 struct scatterlist *dst = req->dst; 629 struct scatterlist *src = req->src; 630 struct cipher_req_ctx *req_ctx = skcipher_request_ctx(req); 631 struct crypto_skcipher *sk_tfm = crypto_skcipher_reqtfm(req); 632 struct crypto_tfm *tfm = crypto_skcipher_tfm(sk_tfm); 633 struct cc_cipher_ctx *ctx_p = crypto_tfm_ctx(tfm); 634 unsigned int ivsize = crypto_skcipher_ivsize(sk_tfm); 635 unsigned int len; 636 637 switch (ctx_p->cipher_mode) { 638 case DRV_CIPHER_CBC: 639 /* 640 * The crypto API expects us to set the req->iv to the last 641 * ciphertext block. For encrypt, simply copy from the result. 642 * For decrypt, we must copy from a saved buffer since this 643 * could be an in-place decryption operation and the src is 644 * lost by this point. 645 */ 646 if (req_ctx->gen_ctx.op_type == DRV_CRYPTO_DIRECTION_DECRYPT) { 647 memcpy(req->iv, req_ctx->backup_info, ivsize); 648 kzfree(req_ctx->backup_info); 649 } else if (!err) { 650 len = req->cryptlen - ivsize; 651 scatterwalk_map_and_copy(req->iv, req->dst, len, 652 ivsize, 0); 653 } 654 break; 655 656 case DRV_CIPHER_CTR: 657 /* Compute the counter of the last block */ 658 len = ALIGN(req->cryptlen, AES_BLOCK_SIZE) / AES_BLOCK_SIZE; 659 cc_update_ctr((u8 *)req->iv, len); 660 break; 661 662 default: 663 break; 664 } 665 666 cc_unmap_cipher_request(dev, req_ctx, ivsize, src, dst); 667 kzfree(req_ctx->iv); 668 669 skcipher_request_complete(req, err); 670 } 671 672 static int cc_cipher_process(struct skcipher_request *req, 673 enum drv_crypto_direction direction) 674 { 675 struct crypto_skcipher *sk_tfm = crypto_skcipher_reqtfm(req); 676 struct crypto_tfm *tfm = crypto_skcipher_tfm(sk_tfm); 677 struct cipher_req_ctx *req_ctx = skcipher_request_ctx(req); 678 unsigned int ivsize = crypto_skcipher_ivsize(sk_tfm); 679 struct scatterlist *dst = req->dst; 680 struct scatterlist *src = req->src; 681 unsigned int nbytes = req->cryptlen; 682 void *iv = req->iv; 683 struct cc_cipher_ctx *ctx_p = crypto_tfm_ctx(tfm); 684 struct device *dev = drvdata_to_dev(ctx_p->drvdata); 685 struct cc_hw_desc desc[MAX_ABLKCIPHER_SEQ_LEN]; 686 struct cc_crypto_req cc_req = {}; 687 int rc; 688 unsigned int seq_len = 0; 689 gfp_t flags = cc_gfp_flags(&req->base); 690 691 dev_dbg(dev, "%s req=%p iv=%p nbytes=%d\n", 692 ((direction == DRV_CRYPTO_DIRECTION_ENCRYPT) ? 693 "Encrypt" : "Decrypt"), req, iv, nbytes); 694 695 /* STAT_PHASE_0: Init and sanity checks */ 696 697 /* TODO: check data length according to mode */ 698 if (validate_data_size(ctx_p, nbytes)) { 699 dev_err(dev, "Unsupported data size %d.\n", nbytes); 700 crypto_tfm_set_flags(tfm, CRYPTO_TFM_RES_BAD_BLOCK_LEN); 701 rc = -EINVAL; 702 goto exit_process; 703 } 704 if (nbytes == 0) { 705 /* No data to process is valid */ 706 rc = 0; 707 goto exit_process; 708 } 709 710 /* The IV we are handed may be allocted from the stack so 711 * we must copy it to a DMAable buffer before use. 712 */ 713 req_ctx->iv = kmemdup(iv, ivsize, flags); 714 if (!req_ctx->iv) { 715 rc = -ENOMEM; 716 goto exit_process; 717 } 718 719 /* Setup request structure */ 720 cc_req.user_cb = (void *)cc_cipher_complete; 721 cc_req.user_arg = (void *)req; 722 723 /* Setup request context */ 724 req_ctx->gen_ctx.op_type = direction; 725 726 /* STAT_PHASE_1: Map buffers */ 727 728 rc = cc_map_cipher_request(ctx_p->drvdata, req_ctx, ivsize, nbytes, 729 req_ctx->iv, src, dst, flags); 730 if (rc) { 731 dev_err(dev, "map_request() failed\n"); 732 goto exit_process; 733 } 734 735 /* STAT_PHASE_2: Create sequence */ 736 737 /* Setup processing */ 738 cc_setup_cipher_desc(tfm, req_ctx, ivsize, nbytes, desc, &seq_len); 739 /* Data processing */ 740 cc_setup_cipher_data(tfm, req_ctx, dst, src, nbytes, req, desc, 741 &seq_len); 742 743 /* STAT_PHASE_3: Lock HW and push sequence */ 744 745 rc = cc_send_request(ctx_p->drvdata, &cc_req, desc, seq_len, 746 &req->base); 747 if (rc != -EINPROGRESS && rc != -EBUSY) { 748 /* Failed to send the request or request completed 749 * synchronously 750 */ 751 cc_unmap_cipher_request(dev, req_ctx, ivsize, src, dst); 752 } 753 754 exit_process: 755 if (rc != -EINPROGRESS && rc != -EBUSY) { 756 kzfree(req_ctx->backup_info); 757 kzfree(req_ctx->iv); 758 } 759 760 return rc; 761 } 762 763 static int cc_cipher_encrypt(struct skcipher_request *req) 764 { 765 struct cipher_req_ctx *req_ctx = skcipher_request_ctx(req); 766 767 memset(req_ctx, 0, sizeof(*req_ctx)); 768 769 return cc_cipher_process(req, DRV_CRYPTO_DIRECTION_ENCRYPT); 770 } 771 772 static int cc_cipher_decrypt(struct skcipher_request *req) 773 { 774 struct crypto_skcipher *sk_tfm = crypto_skcipher_reqtfm(req); 775 struct crypto_tfm *tfm = crypto_skcipher_tfm(sk_tfm); 776 struct cc_cipher_ctx *ctx_p = crypto_tfm_ctx(tfm); 777 struct cipher_req_ctx *req_ctx = skcipher_request_ctx(req); 778 unsigned int ivsize = crypto_skcipher_ivsize(sk_tfm); 779 gfp_t flags = cc_gfp_flags(&req->base); 780 unsigned int len; 781 782 memset(req_ctx, 0, sizeof(*req_ctx)); 783 784 if (ctx_p->cipher_mode == DRV_CIPHER_CBC) { 785 786 /* Allocate and save the last IV sized bytes of the source, 787 * which will be lost in case of in-place decryption. 788 */ 789 req_ctx->backup_info = kzalloc(ivsize, flags); 790 if (!req_ctx->backup_info) 791 return -ENOMEM; 792 793 len = req->cryptlen - ivsize; 794 scatterwalk_map_and_copy(req_ctx->backup_info, req->src, len, 795 ivsize, 0); 796 } 797 798 return cc_cipher_process(req, DRV_CRYPTO_DIRECTION_DECRYPT); 799 } 800 801 /* Block cipher alg */ 802 static const struct cc_alg_template skcipher_algs[] = { 803 { 804 .name = "xts(paes)", 805 .driver_name = "xts-paes-ccree", 806 .blocksize = AES_BLOCK_SIZE, 807 .template_skcipher = { 808 .setkey = cc_cipher_sethkey, 809 .encrypt = cc_cipher_encrypt, 810 .decrypt = cc_cipher_decrypt, 811 .min_keysize = CC_HW_KEY_SIZE, 812 .max_keysize = CC_HW_KEY_SIZE, 813 .ivsize = AES_BLOCK_SIZE, 814 }, 815 .cipher_mode = DRV_CIPHER_XTS, 816 .flow_mode = S_DIN_to_AES, 817 .min_hw_rev = CC_HW_REV_630, 818 }, 819 { 820 .name = "xts512(paes)", 821 .driver_name = "xts-paes-du512-ccree", 822 .blocksize = AES_BLOCK_SIZE, 823 .template_skcipher = { 824 .setkey = cc_cipher_sethkey, 825 .encrypt = cc_cipher_encrypt, 826 .decrypt = cc_cipher_decrypt, 827 .min_keysize = CC_HW_KEY_SIZE, 828 .max_keysize = CC_HW_KEY_SIZE, 829 .ivsize = AES_BLOCK_SIZE, 830 }, 831 .cipher_mode = DRV_CIPHER_XTS, 832 .flow_mode = S_DIN_to_AES, 833 .data_unit = 512, 834 .min_hw_rev = CC_HW_REV_712, 835 }, 836 { 837 .name = "xts4096(paes)", 838 .driver_name = "xts-paes-du4096-ccree", 839 .blocksize = AES_BLOCK_SIZE, 840 .template_skcipher = { 841 .setkey = cc_cipher_sethkey, 842 .encrypt = cc_cipher_encrypt, 843 .decrypt = cc_cipher_decrypt, 844 .min_keysize = CC_HW_KEY_SIZE, 845 .max_keysize = CC_HW_KEY_SIZE, 846 .ivsize = AES_BLOCK_SIZE, 847 }, 848 .cipher_mode = DRV_CIPHER_XTS, 849 .flow_mode = S_DIN_to_AES, 850 .data_unit = 4096, 851 .min_hw_rev = CC_HW_REV_712, 852 }, 853 { 854 .name = "essiv(paes)", 855 .driver_name = "essiv-paes-ccree", 856 .blocksize = AES_BLOCK_SIZE, 857 .template_skcipher = { 858 .setkey = cc_cipher_sethkey, 859 .encrypt = cc_cipher_encrypt, 860 .decrypt = cc_cipher_decrypt, 861 .min_keysize = CC_HW_KEY_SIZE, 862 .max_keysize = CC_HW_KEY_SIZE, 863 .ivsize = AES_BLOCK_SIZE, 864 }, 865 .cipher_mode = DRV_CIPHER_ESSIV, 866 .flow_mode = S_DIN_to_AES, 867 .min_hw_rev = CC_HW_REV_712, 868 }, 869 { 870 .name = "essiv512(paes)", 871 .driver_name = "essiv-paes-du512-ccree", 872 .blocksize = AES_BLOCK_SIZE, 873 .template_skcipher = { 874 .setkey = cc_cipher_sethkey, 875 .encrypt = cc_cipher_encrypt, 876 .decrypt = cc_cipher_decrypt, 877 .min_keysize = CC_HW_KEY_SIZE, 878 .max_keysize = CC_HW_KEY_SIZE, 879 .ivsize = AES_BLOCK_SIZE, 880 }, 881 .cipher_mode = DRV_CIPHER_ESSIV, 882 .flow_mode = S_DIN_to_AES, 883 .data_unit = 512, 884 .min_hw_rev = CC_HW_REV_712, 885 }, 886 { 887 .name = "essiv4096(paes)", 888 .driver_name = "essiv-paes-du4096-ccree", 889 .blocksize = AES_BLOCK_SIZE, 890 .template_skcipher = { 891 .setkey = cc_cipher_sethkey, 892 .encrypt = cc_cipher_encrypt, 893 .decrypt = cc_cipher_decrypt, 894 .min_keysize = CC_HW_KEY_SIZE, 895 .max_keysize = CC_HW_KEY_SIZE, 896 .ivsize = AES_BLOCK_SIZE, 897 }, 898 .cipher_mode = DRV_CIPHER_ESSIV, 899 .flow_mode = S_DIN_to_AES, 900 .data_unit = 4096, 901 .min_hw_rev = CC_HW_REV_712, 902 }, 903 { 904 .name = "bitlocker(paes)", 905 .driver_name = "bitlocker-paes-ccree", 906 .blocksize = AES_BLOCK_SIZE, 907 .template_skcipher = { 908 .setkey = cc_cipher_sethkey, 909 .encrypt = cc_cipher_encrypt, 910 .decrypt = cc_cipher_decrypt, 911 .min_keysize = CC_HW_KEY_SIZE, 912 .max_keysize = CC_HW_KEY_SIZE, 913 .ivsize = AES_BLOCK_SIZE, 914 }, 915 .cipher_mode = DRV_CIPHER_BITLOCKER, 916 .flow_mode = S_DIN_to_AES, 917 .min_hw_rev = CC_HW_REV_712, 918 }, 919 { 920 .name = "bitlocker512(paes)", 921 .driver_name = "bitlocker-paes-du512-ccree", 922 .blocksize = AES_BLOCK_SIZE, 923 .template_skcipher = { 924 .setkey = cc_cipher_sethkey, 925 .encrypt = cc_cipher_encrypt, 926 .decrypt = cc_cipher_decrypt, 927 .min_keysize = CC_HW_KEY_SIZE, 928 .max_keysize = CC_HW_KEY_SIZE, 929 .ivsize = AES_BLOCK_SIZE, 930 }, 931 .cipher_mode = DRV_CIPHER_BITLOCKER, 932 .flow_mode = S_DIN_to_AES, 933 .data_unit = 512, 934 .min_hw_rev = CC_HW_REV_712, 935 }, 936 { 937 .name = "bitlocker4096(paes)", 938 .driver_name = "bitlocker-paes-du4096-ccree", 939 .blocksize = AES_BLOCK_SIZE, 940 .template_skcipher = { 941 .setkey = cc_cipher_sethkey, 942 .encrypt = cc_cipher_encrypt, 943 .decrypt = cc_cipher_decrypt, 944 .min_keysize = CC_HW_KEY_SIZE, 945 .max_keysize = CC_HW_KEY_SIZE, 946 .ivsize = AES_BLOCK_SIZE, 947 }, 948 .cipher_mode = DRV_CIPHER_BITLOCKER, 949 .flow_mode = S_DIN_to_AES, 950 .data_unit = 4096, 951 .min_hw_rev = CC_HW_REV_712, 952 }, 953 { 954 .name = "ecb(paes)", 955 .driver_name = "ecb-paes-ccree", 956 .blocksize = AES_BLOCK_SIZE, 957 .template_skcipher = { 958 .setkey = cc_cipher_sethkey, 959 .encrypt = cc_cipher_encrypt, 960 .decrypt = cc_cipher_decrypt, 961 .min_keysize = CC_HW_KEY_SIZE, 962 .max_keysize = CC_HW_KEY_SIZE, 963 .ivsize = 0, 964 }, 965 .cipher_mode = DRV_CIPHER_ECB, 966 .flow_mode = S_DIN_to_AES, 967 .min_hw_rev = CC_HW_REV_712, 968 }, 969 { 970 .name = "cbc(paes)", 971 .driver_name = "cbc-paes-ccree", 972 .blocksize = AES_BLOCK_SIZE, 973 .template_skcipher = { 974 .setkey = cc_cipher_sethkey, 975 .encrypt = cc_cipher_encrypt, 976 .decrypt = cc_cipher_decrypt, 977 .min_keysize = CC_HW_KEY_SIZE, 978 .max_keysize = CC_HW_KEY_SIZE, 979 .ivsize = AES_BLOCK_SIZE, 980 }, 981 .cipher_mode = DRV_CIPHER_CBC, 982 .flow_mode = S_DIN_to_AES, 983 .min_hw_rev = CC_HW_REV_712, 984 }, 985 { 986 .name = "ofb(paes)", 987 .driver_name = "ofb-paes-ccree", 988 .blocksize = AES_BLOCK_SIZE, 989 .template_skcipher = { 990 .setkey = cc_cipher_sethkey, 991 .encrypt = cc_cipher_encrypt, 992 .decrypt = cc_cipher_decrypt, 993 .min_keysize = CC_HW_KEY_SIZE, 994 .max_keysize = CC_HW_KEY_SIZE, 995 .ivsize = AES_BLOCK_SIZE, 996 }, 997 .cipher_mode = DRV_CIPHER_OFB, 998 .flow_mode = S_DIN_to_AES, 999 .min_hw_rev = CC_HW_REV_712, 1000 }, 1001 { 1002 .name = "cts(cbc(paes))", 1003 .driver_name = "cts-cbc-paes-ccree", 1004 .blocksize = AES_BLOCK_SIZE, 1005 .template_skcipher = { 1006 .setkey = cc_cipher_sethkey, 1007 .encrypt = cc_cipher_encrypt, 1008 .decrypt = cc_cipher_decrypt, 1009 .min_keysize = CC_HW_KEY_SIZE, 1010 .max_keysize = CC_HW_KEY_SIZE, 1011 .ivsize = AES_BLOCK_SIZE, 1012 }, 1013 .cipher_mode = DRV_CIPHER_CBC_CTS, 1014 .flow_mode = S_DIN_to_AES, 1015 .min_hw_rev = CC_HW_REV_712, 1016 }, 1017 { 1018 .name = "ctr(paes)", 1019 .driver_name = "ctr-paes-ccree", 1020 .blocksize = 1, 1021 .template_skcipher = { 1022 .setkey = cc_cipher_sethkey, 1023 .encrypt = cc_cipher_encrypt, 1024 .decrypt = cc_cipher_decrypt, 1025 .min_keysize = CC_HW_KEY_SIZE, 1026 .max_keysize = CC_HW_KEY_SIZE, 1027 .ivsize = AES_BLOCK_SIZE, 1028 }, 1029 .cipher_mode = DRV_CIPHER_CTR, 1030 .flow_mode = S_DIN_to_AES, 1031 .min_hw_rev = CC_HW_REV_712, 1032 }, 1033 { 1034 .name = "xts(aes)", 1035 .driver_name = "xts-aes-ccree", 1036 .blocksize = AES_BLOCK_SIZE, 1037 .template_skcipher = { 1038 .setkey = cc_cipher_setkey, 1039 .encrypt = cc_cipher_encrypt, 1040 .decrypt = cc_cipher_decrypt, 1041 .min_keysize = AES_MIN_KEY_SIZE * 2, 1042 .max_keysize = AES_MAX_KEY_SIZE * 2, 1043 .ivsize = AES_BLOCK_SIZE, 1044 }, 1045 .cipher_mode = DRV_CIPHER_XTS, 1046 .flow_mode = S_DIN_to_AES, 1047 .min_hw_rev = CC_HW_REV_630, 1048 }, 1049 { 1050 .name = "xts512(aes)", 1051 .driver_name = "xts-aes-du512-ccree", 1052 .blocksize = AES_BLOCK_SIZE, 1053 .template_skcipher = { 1054 .setkey = cc_cipher_setkey, 1055 .encrypt = cc_cipher_encrypt, 1056 .decrypt = cc_cipher_decrypt, 1057 .min_keysize = AES_MIN_KEY_SIZE * 2, 1058 .max_keysize = AES_MAX_KEY_SIZE * 2, 1059 .ivsize = AES_BLOCK_SIZE, 1060 }, 1061 .cipher_mode = DRV_CIPHER_XTS, 1062 .flow_mode = S_DIN_to_AES, 1063 .data_unit = 512, 1064 .min_hw_rev = CC_HW_REV_712, 1065 }, 1066 { 1067 .name = "xts4096(aes)", 1068 .driver_name = "xts-aes-du4096-ccree", 1069 .blocksize = AES_BLOCK_SIZE, 1070 .template_skcipher = { 1071 .setkey = cc_cipher_setkey, 1072 .encrypt = cc_cipher_encrypt, 1073 .decrypt = cc_cipher_decrypt, 1074 .min_keysize = AES_MIN_KEY_SIZE * 2, 1075 .max_keysize = AES_MAX_KEY_SIZE * 2, 1076 .ivsize = AES_BLOCK_SIZE, 1077 }, 1078 .cipher_mode = DRV_CIPHER_XTS, 1079 .flow_mode = S_DIN_to_AES, 1080 .data_unit = 4096, 1081 .min_hw_rev = CC_HW_REV_712, 1082 }, 1083 { 1084 .name = "essiv(aes)", 1085 .driver_name = "essiv-aes-ccree", 1086 .blocksize = AES_BLOCK_SIZE, 1087 .template_skcipher = { 1088 .setkey = cc_cipher_setkey, 1089 .encrypt = cc_cipher_encrypt, 1090 .decrypt = cc_cipher_decrypt, 1091 .min_keysize = AES_MIN_KEY_SIZE * 2, 1092 .max_keysize = AES_MAX_KEY_SIZE * 2, 1093 .ivsize = AES_BLOCK_SIZE, 1094 }, 1095 .cipher_mode = DRV_CIPHER_ESSIV, 1096 .flow_mode = S_DIN_to_AES, 1097 .min_hw_rev = CC_HW_REV_712, 1098 }, 1099 { 1100 .name = "essiv512(aes)", 1101 .driver_name = "essiv-aes-du512-ccree", 1102 .blocksize = AES_BLOCK_SIZE, 1103 .template_skcipher = { 1104 .setkey = cc_cipher_setkey, 1105 .encrypt = cc_cipher_encrypt, 1106 .decrypt = cc_cipher_decrypt, 1107 .min_keysize = AES_MIN_KEY_SIZE * 2, 1108 .max_keysize = AES_MAX_KEY_SIZE * 2, 1109 .ivsize = AES_BLOCK_SIZE, 1110 }, 1111 .cipher_mode = DRV_CIPHER_ESSIV, 1112 .flow_mode = S_DIN_to_AES, 1113 .data_unit = 512, 1114 .min_hw_rev = CC_HW_REV_712, 1115 }, 1116 { 1117 .name = "essiv4096(aes)", 1118 .driver_name = "essiv-aes-du4096-ccree", 1119 .blocksize = AES_BLOCK_SIZE, 1120 .template_skcipher = { 1121 .setkey = cc_cipher_setkey, 1122 .encrypt = cc_cipher_encrypt, 1123 .decrypt = cc_cipher_decrypt, 1124 .min_keysize = AES_MIN_KEY_SIZE * 2, 1125 .max_keysize = AES_MAX_KEY_SIZE * 2, 1126 .ivsize = AES_BLOCK_SIZE, 1127 }, 1128 .cipher_mode = DRV_CIPHER_ESSIV, 1129 .flow_mode = S_DIN_to_AES, 1130 .data_unit = 4096, 1131 .min_hw_rev = CC_HW_REV_712, 1132 }, 1133 { 1134 .name = "bitlocker(aes)", 1135 .driver_name = "bitlocker-aes-ccree", 1136 .blocksize = AES_BLOCK_SIZE, 1137 .template_skcipher = { 1138 .setkey = cc_cipher_setkey, 1139 .encrypt = cc_cipher_encrypt, 1140 .decrypt = cc_cipher_decrypt, 1141 .min_keysize = AES_MIN_KEY_SIZE * 2, 1142 .max_keysize = AES_MAX_KEY_SIZE * 2, 1143 .ivsize = AES_BLOCK_SIZE, 1144 }, 1145 .cipher_mode = DRV_CIPHER_BITLOCKER, 1146 .flow_mode = S_DIN_to_AES, 1147 .min_hw_rev = CC_HW_REV_712, 1148 }, 1149 { 1150 .name = "bitlocker512(aes)", 1151 .driver_name = "bitlocker-aes-du512-ccree", 1152 .blocksize = AES_BLOCK_SIZE, 1153 .template_skcipher = { 1154 .setkey = cc_cipher_setkey, 1155 .encrypt = cc_cipher_encrypt, 1156 .decrypt = cc_cipher_decrypt, 1157 .min_keysize = AES_MIN_KEY_SIZE * 2, 1158 .max_keysize = AES_MAX_KEY_SIZE * 2, 1159 .ivsize = AES_BLOCK_SIZE, 1160 }, 1161 .cipher_mode = DRV_CIPHER_BITLOCKER, 1162 .flow_mode = S_DIN_to_AES, 1163 .data_unit = 512, 1164 .min_hw_rev = CC_HW_REV_712, 1165 }, 1166 { 1167 .name = "bitlocker4096(aes)", 1168 .driver_name = "bitlocker-aes-du4096-ccree", 1169 .blocksize = AES_BLOCK_SIZE, 1170 .template_skcipher = { 1171 .setkey = cc_cipher_setkey, 1172 .encrypt = cc_cipher_encrypt, 1173 .decrypt = cc_cipher_decrypt, 1174 .min_keysize = AES_MIN_KEY_SIZE * 2, 1175 .max_keysize = AES_MAX_KEY_SIZE * 2, 1176 .ivsize = AES_BLOCK_SIZE, 1177 }, 1178 .cipher_mode = DRV_CIPHER_BITLOCKER, 1179 .flow_mode = S_DIN_to_AES, 1180 .data_unit = 4096, 1181 .min_hw_rev = CC_HW_REV_712, 1182 }, 1183 { 1184 .name = "ecb(aes)", 1185 .driver_name = "ecb-aes-ccree", 1186 .blocksize = AES_BLOCK_SIZE, 1187 .template_skcipher = { 1188 .setkey = cc_cipher_setkey, 1189 .encrypt = cc_cipher_encrypt, 1190 .decrypt = cc_cipher_decrypt, 1191 .min_keysize = AES_MIN_KEY_SIZE, 1192 .max_keysize = AES_MAX_KEY_SIZE, 1193 .ivsize = 0, 1194 }, 1195 .cipher_mode = DRV_CIPHER_ECB, 1196 .flow_mode = S_DIN_to_AES, 1197 .min_hw_rev = CC_HW_REV_630, 1198 }, 1199 { 1200 .name = "cbc(aes)", 1201 .driver_name = "cbc-aes-ccree", 1202 .blocksize = AES_BLOCK_SIZE, 1203 .template_skcipher = { 1204 .setkey = cc_cipher_setkey, 1205 .encrypt = cc_cipher_encrypt, 1206 .decrypt = cc_cipher_decrypt, 1207 .min_keysize = AES_MIN_KEY_SIZE, 1208 .max_keysize = AES_MAX_KEY_SIZE, 1209 .ivsize = AES_BLOCK_SIZE, 1210 }, 1211 .cipher_mode = DRV_CIPHER_CBC, 1212 .flow_mode = S_DIN_to_AES, 1213 .min_hw_rev = CC_HW_REV_630, 1214 }, 1215 { 1216 .name = "ofb(aes)", 1217 .driver_name = "ofb-aes-ccree", 1218 .blocksize = AES_BLOCK_SIZE, 1219 .template_skcipher = { 1220 .setkey = cc_cipher_setkey, 1221 .encrypt = cc_cipher_encrypt, 1222 .decrypt = cc_cipher_decrypt, 1223 .min_keysize = AES_MIN_KEY_SIZE, 1224 .max_keysize = AES_MAX_KEY_SIZE, 1225 .ivsize = AES_BLOCK_SIZE, 1226 }, 1227 .cipher_mode = DRV_CIPHER_OFB, 1228 .flow_mode = S_DIN_to_AES, 1229 .min_hw_rev = CC_HW_REV_630, 1230 }, 1231 { 1232 .name = "cts(cbc(aes))", 1233 .driver_name = "cts-cbc-aes-ccree", 1234 .blocksize = AES_BLOCK_SIZE, 1235 .template_skcipher = { 1236 .setkey = cc_cipher_setkey, 1237 .encrypt = cc_cipher_encrypt, 1238 .decrypt = cc_cipher_decrypt, 1239 .min_keysize = AES_MIN_KEY_SIZE, 1240 .max_keysize = AES_MAX_KEY_SIZE, 1241 .ivsize = AES_BLOCK_SIZE, 1242 }, 1243 .cipher_mode = DRV_CIPHER_CBC_CTS, 1244 .flow_mode = S_DIN_to_AES, 1245 .min_hw_rev = CC_HW_REV_630, 1246 }, 1247 { 1248 .name = "ctr(aes)", 1249 .driver_name = "ctr-aes-ccree", 1250 .blocksize = 1, 1251 .template_skcipher = { 1252 .setkey = cc_cipher_setkey, 1253 .encrypt = cc_cipher_encrypt, 1254 .decrypt = cc_cipher_decrypt, 1255 .min_keysize = AES_MIN_KEY_SIZE, 1256 .max_keysize = AES_MAX_KEY_SIZE, 1257 .ivsize = AES_BLOCK_SIZE, 1258 }, 1259 .cipher_mode = DRV_CIPHER_CTR, 1260 .flow_mode = S_DIN_to_AES, 1261 .min_hw_rev = CC_HW_REV_630, 1262 }, 1263 { 1264 .name = "cbc(des3_ede)", 1265 .driver_name = "cbc-3des-ccree", 1266 .blocksize = DES3_EDE_BLOCK_SIZE, 1267 .template_skcipher = { 1268 .setkey = cc_cipher_setkey, 1269 .encrypt = cc_cipher_encrypt, 1270 .decrypt = cc_cipher_decrypt, 1271 .min_keysize = DES3_EDE_KEY_SIZE, 1272 .max_keysize = DES3_EDE_KEY_SIZE, 1273 .ivsize = DES3_EDE_BLOCK_SIZE, 1274 }, 1275 .cipher_mode = DRV_CIPHER_CBC, 1276 .flow_mode = S_DIN_to_DES, 1277 .min_hw_rev = CC_HW_REV_630, 1278 }, 1279 { 1280 .name = "ecb(des3_ede)", 1281 .driver_name = "ecb-3des-ccree", 1282 .blocksize = DES3_EDE_BLOCK_SIZE, 1283 .template_skcipher = { 1284 .setkey = cc_cipher_setkey, 1285 .encrypt = cc_cipher_encrypt, 1286 .decrypt = cc_cipher_decrypt, 1287 .min_keysize = DES3_EDE_KEY_SIZE, 1288 .max_keysize = DES3_EDE_KEY_SIZE, 1289 .ivsize = 0, 1290 }, 1291 .cipher_mode = DRV_CIPHER_ECB, 1292 .flow_mode = S_DIN_to_DES, 1293 .min_hw_rev = CC_HW_REV_630, 1294 }, 1295 { 1296 .name = "cbc(des)", 1297 .driver_name = "cbc-des-ccree", 1298 .blocksize = DES_BLOCK_SIZE, 1299 .template_skcipher = { 1300 .setkey = cc_cipher_setkey, 1301 .encrypt = cc_cipher_encrypt, 1302 .decrypt = cc_cipher_decrypt, 1303 .min_keysize = DES_KEY_SIZE, 1304 .max_keysize = DES_KEY_SIZE, 1305 .ivsize = DES_BLOCK_SIZE, 1306 }, 1307 .cipher_mode = DRV_CIPHER_CBC, 1308 .flow_mode = S_DIN_to_DES, 1309 .min_hw_rev = CC_HW_REV_630, 1310 }, 1311 { 1312 .name = "ecb(des)", 1313 .driver_name = "ecb-des-ccree", 1314 .blocksize = DES_BLOCK_SIZE, 1315 .template_skcipher = { 1316 .setkey = cc_cipher_setkey, 1317 .encrypt = cc_cipher_encrypt, 1318 .decrypt = cc_cipher_decrypt, 1319 .min_keysize = DES_KEY_SIZE, 1320 .max_keysize = DES_KEY_SIZE, 1321 .ivsize = 0, 1322 }, 1323 .cipher_mode = DRV_CIPHER_ECB, 1324 .flow_mode = S_DIN_to_DES, 1325 .min_hw_rev = CC_HW_REV_630, 1326 }, 1327 }; 1328 1329 static struct cc_crypto_alg *cc_create_alg(const struct cc_alg_template *tmpl, 1330 struct device *dev) 1331 { 1332 struct cc_crypto_alg *t_alg; 1333 struct skcipher_alg *alg; 1334 1335 t_alg = kzalloc(sizeof(*t_alg), GFP_KERNEL); 1336 if (!t_alg) 1337 return ERR_PTR(-ENOMEM); 1338 1339 alg = &t_alg->skcipher_alg; 1340 1341 memcpy(alg, &tmpl->template_skcipher, sizeof(*alg)); 1342 1343 snprintf(alg->base.cra_name, CRYPTO_MAX_ALG_NAME, "%s", tmpl->name); 1344 snprintf(alg->base.cra_driver_name, CRYPTO_MAX_ALG_NAME, "%s", 1345 tmpl->driver_name); 1346 alg->base.cra_module = THIS_MODULE; 1347 alg->base.cra_priority = CC_CRA_PRIO; 1348 alg->base.cra_blocksize = tmpl->blocksize; 1349 alg->base.cra_alignmask = 0; 1350 alg->base.cra_ctxsize = sizeof(struct cc_cipher_ctx); 1351 1352 alg->base.cra_init = cc_cipher_init; 1353 alg->base.cra_exit = cc_cipher_exit; 1354 alg->base.cra_flags = CRYPTO_ALG_ASYNC | CRYPTO_ALG_KERN_DRIVER_ONLY; 1355 1356 t_alg->cipher_mode = tmpl->cipher_mode; 1357 t_alg->flow_mode = tmpl->flow_mode; 1358 t_alg->data_unit = tmpl->data_unit; 1359 1360 return t_alg; 1361 } 1362 1363 int cc_cipher_free(struct cc_drvdata *drvdata) 1364 { 1365 struct cc_crypto_alg *t_alg, *n; 1366 struct cc_cipher_handle *cipher_handle = drvdata->cipher_handle; 1367 1368 if (cipher_handle) { 1369 /* Remove registered algs */ 1370 list_for_each_entry_safe(t_alg, n, &cipher_handle->alg_list, 1371 entry) { 1372 crypto_unregister_skcipher(&t_alg->skcipher_alg); 1373 list_del(&t_alg->entry); 1374 kfree(t_alg); 1375 } 1376 kfree(cipher_handle); 1377 drvdata->cipher_handle = NULL; 1378 } 1379 return 0; 1380 } 1381 1382 int cc_cipher_alloc(struct cc_drvdata *drvdata) 1383 { 1384 struct cc_cipher_handle *cipher_handle; 1385 struct cc_crypto_alg *t_alg; 1386 struct device *dev = drvdata_to_dev(drvdata); 1387 int rc = -ENOMEM; 1388 int alg; 1389 1390 cipher_handle = kmalloc(sizeof(*cipher_handle), GFP_KERNEL); 1391 if (!cipher_handle) 1392 return -ENOMEM; 1393 1394 INIT_LIST_HEAD(&cipher_handle->alg_list); 1395 drvdata->cipher_handle = cipher_handle; 1396 1397 /* Linux crypto */ 1398 dev_dbg(dev, "Number of algorithms = %zu\n", 1399 ARRAY_SIZE(skcipher_algs)); 1400 for (alg = 0; alg < ARRAY_SIZE(skcipher_algs); alg++) { 1401 if (skcipher_algs[alg].min_hw_rev > drvdata->hw_rev) 1402 continue; 1403 1404 dev_dbg(dev, "creating %s\n", skcipher_algs[alg].driver_name); 1405 t_alg = cc_create_alg(&skcipher_algs[alg], dev); 1406 if (IS_ERR(t_alg)) { 1407 rc = PTR_ERR(t_alg); 1408 dev_err(dev, "%s alg allocation failed\n", 1409 skcipher_algs[alg].driver_name); 1410 goto fail0; 1411 } 1412 t_alg->drvdata = drvdata; 1413 1414 dev_dbg(dev, "registering %s\n", 1415 skcipher_algs[alg].driver_name); 1416 rc = crypto_register_skcipher(&t_alg->skcipher_alg); 1417 dev_dbg(dev, "%s alg registration rc = %x\n", 1418 t_alg->skcipher_alg.base.cra_driver_name, rc); 1419 if (rc) { 1420 dev_err(dev, "%s alg registration failed\n", 1421 t_alg->skcipher_alg.base.cra_driver_name); 1422 kfree(t_alg); 1423 goto fail0; 1424 } else { 1425 list_add_tail(&t_alg->entry, 1426 &cipher_handle->alg_list); 1427 dev_dbg(dev, "Registered %s\n", 1428 t_alg->skcipher_alg.base.cra_driver_name); 1429 } 1430 } 1431 return 0; 1432 1433 fail0: 1434 cc_cipher_free(drvdata); 1435 return rc; 1436 } 1437