1 // SPDX-License-Identifier: GPL-2.0+ 2 /* 3 * Freescale FSL CAAM support for crypto API over QI backend. 4 * Based on caamalg.c 5 * 6 * Copyright 2013-2016 Freescale Semiconductor, Inc. 7 * Copyright 2016-2018 NXP 8 */ 9 10 #include "compat.h" 11 #include "ctrl.h" 12 #include "regs.h" 13 #include "intern.h" 14 #include "desc_constr.h" 15 #include "error.h" 16 #include "sg_sw_qm.h" 17 #include "key_gen.h" 18 #include "qi.h" 19 #include "jr.h" 20 #include "caamalg_desc.h" 21 22 /* 23 * crypto alg 24 */ 25 #define CAAM_CRA_PRIORITY 2000 26 /* max key is sum of AES_MAX_KEY_SIZE, max split key size */ 27 #define CAAM_MAX_KEY_SIZE (AES_MAX_KEY_SIZE + \ 28 SHA512_DIGEST_SIZE * 2) 29 30 #define DESC_MAX_USED_BYTES (DESC_QI_AEAD_GIVENC_LEN + \ 31 CAAM_MAX_KEY_SIZE) 32 #define DESC_MAX_USED_LEN (DESC_MAX_USED_BYTES / CAAM_CMD_SZ) 33 34 struct caam_alg_entry { 35 int class1_alg_type; 36 int class2_alg_type; 37 bool rfc3686; 38 bool geniv; 39 }; 40 41 struct caam_aead_alg { 42 struct aead_alg aead; 43 struct caam_alg_entry caam; 44 bool registered; 45 }; 46 47 struct caam_skcipher_alg { 48 struct skcipher_alg skcipher; 49 struct caam_alg_entry caam; 50 bool registered; 51 }; 52 53 /* 54 * per-session context 55 */ 56 struct caam_ctx { 57 struct device *jrdev; 58 u32 sh_desc_enc[DESC_MAX_USED_LEN]; 59 u32 sh_desc_dec[DESC_MAX_USED_LEN]; 60 u8 key[CAAM_MAX_KEY_SIZE]; 61 dma_addr_t key_dma; 62 enum dma_data_direction dir; 63 struct alginfo adata; 64 struct alginfo cdata; 65 unsigned int authsize; 66 struct device *qidev; 67 spinlock_t lock; /* Protects multiple init of driver context */ 68 struct caam_drv_ctx *drv_ctx[NUM_OP]; 69 }; 70 71 static int aead_set_sh_desc(struct crypto_aead *aead) 72 { 73 struct caam_aead_alg *alg = container_of(crypto_aead_alg(aead), 74 typeof(*alg), aead); 75 struct caam_ctx *ctx = crypto_aead_ctx(aead); 76 unsigned int ivsize = crypto_aead_ivsize(aead); 77 u32 ctx1_iv_off = 0; 78 u32 *nonce = NULL; 79 unsigned int data_len[2]; 80 u32 inl_mask; 81 const bool ctr_mode = ((ctx->cdata.algtype & OP_ALG_AAI_MASK) == 82 OP_ALG_AAI_CTR_MOD128); 83 const bool is_rfc3686 = alg->caam.rfc3686; 84 struct caam_drv_private *ctrlpriv = dev_get_drvdata(ctx->jrdev->parent); 85 86 if (!ctx->cdata.keylen || !ctx->authsize) 87 return 0; 88 89 /* 90 * AES-CTR needs to load IV in CONTEXT1 reg 91 * at an offset of 128bits (16bytes) 92 * CONTEXT1[255:128] = IV 93 */ 94 if (ctr_mode) 95 ctx1_iv_off = 16; 96 97 /* 98 * RFC3686 specific: 99 * CONTEXT1[255:128] = {NONCE, IV, COUNTER} 100 */ 101 if (is_rfc3686) { 102 ctx1_iv_off = 16 + CTR_RFC3686_NONCE_SIZE; 103 nonce = (u32 *)((void *)ctx->key + ctx->adata.keylen_pad + 104 ctx->cdata.keylen - CTR_RFC3686_NONCE_SIZE); 105 } 106 107 data_len[0] = ctx->adata.keylen_pad; 108 data_len[1] = ctx->cdata.keylen; 109 110 if (alg->caam.geniv) 111 goto skip_enc; 112 113 /* aead_encrypt shared descriptor */ 114 if (desc_inline_query(DESC_QI_AEAD_ENC_LEN + 115 (is_rfc3686 ? DESC_AEAD_CTR_RFC3686_LEN : 0), 116 DESC_JOB_IO_LEN, data_len, &inl_mask, 117 ARRAY_SIZE(data_len)) < 0) 118 return -EINVAL; 119 120 if (inl_mask & 1) 121 ctx->adata.key_virt = ctx->key; 122 else 123 ctx->adata.key_dma = ctx->key_dma; 124 125 if (inl_mask & 2) 126 ctx->cdata.key_virt = ctx->key + ctx->adata.keylen_pad; 127 else 128 ctx->cdata.key_dma = ctx->key_dma + ctx->adata.keylen_pad; 129 130 ctx->adata.key_inline = !!(inl_mask & 1); 131 ctx->cdata.key_inline = !!(inl_mask & 2); 132 133 cnstr_shdsc_aead_encap(ctx->sh_desc_enc, &ctx->cdata, &ctx->adata, 134 ivsize, ctx->authsize, is_rfc3686, nonce, 135 ctx1_iv_off, true, ctrlpriv->era); 136 137 skip_enc: 138 /* aead_decrypt shared descriptor */ 139 if (desc_inline_query(DESC_QI_AEAD_DEC_LEN + 140 (is_rfc3686 ? DESC_AEAD_CTR_RFC3686_LEN : 0), 141 DESC_JOB_IO_LEN, data_len, &inl_mask, 142 ARRAY_SIZE(data_len)) < 0) 143 return -EINVAL; 144 145 if (inl_mask & 1) 146 ctx->adata.key_virt = ctx->key; 147 else 148 ctx->adata.key_dma = ctx->key_dma; 149 150 if (inl_mask & 2) 151 ctx->cdata.key_virt = ctx->key + ctx->adata.keylen_pad; 152 else 153 ctx->cdata.key_dma = ctx->key_dma + ctx->adata.keylen_pad; 154 155 ctx->adata.key_inline = !!(inl_mask & 1); 156 ctx->cdata.key_inline = !!(inl_mask & 2); 157 158 cnstr_shdsc_aead_decap(ctx->sh_desc_dec, &ctx->cdata, &ctx->adata, 159 ivsize, ctx->authsize, alg->caam.geniv, 160 is_rfc3686, nonce, ctx1_iv_off, true, 161 ctrlpriv->era); 162 163 if (!alg->caam.geniv) 164 goto skip_givenc; 165 166 /* aead_givencrypt shared descriptor */ 167 if (desc_inline_query(DESC_QI_AEAD_GIVENC_LEN + 168 (is_rfc3686 ? DESC_AEAD_CTR_RFC3686_LEN : 0), 169 DESC_JOB_IO_LEN, data_len, &inl_mask, 170 ARRAY_SIZE(data_len)) < 0) 171 return -EINVAL; 172 173 if (inl_mask & 1) 174 ctx->adata.key_virt = ctx->key; 175 else 176 ctx->adata.key_dma = ctx->key_dma; 177 178 if (inl_mask & 2) 179 ctx->cdata.key_virt = ctx->key + ctx->adata.keylen_pad; 180 else 181 ctx->cdata.key_dma = ctx->key_dma + ctx->adata.keylen_pad; 182 183 ctx->adata.key_inline = !!(inl_mask & 1); 184 ctx->cdata.key_inline = !!(inl_mask & 2); 185 186 cnstr_shdsc_aead_givencap(ctx->sh_desc_enc, &ctx->cdata, &ctx->adata, 187 ivsize, ctx->authsize, is_rfc3686, nonce, 188 ctx1_iv_off, true, ctrlpriv->era); 189 190 skip_givenc: 191 return 0; 192 } 193 194 static int aead_setauthsize(struct crypto_aead *authenc, unsigned int authsize) 195 { 196 struct caam_ctx *ctx = crypto_aead_ctx(authenc); 197 198 ctx->authsize = authsize; 199 aead_set_sh_desc(authenc); 200 201 return 0; 202 } 203 204 static int aead_setkey(struct crypto_aead *aead, const u8 *key, 205 unsigned int keylen) 206 { 207 struct caam_ctx *ctx = crypto_aead_ctx(aead); 208 struct device *jrdev = ctx->jrdev; 209 struct caam_drv_private *ctrlpriv = dev_get_drvdata(jrdev->parent); 210 struct crypto_authenc_keys keys; 211 int ret = 0; 212 213 if (crypto_authenc_extractkeys(&keys, key, keylen) != 0) 214 goto badkey; 215 216 #ifdef DEBUG 217 dev_err(jrdev, "keylen %d enckeylen %d authkeylen %d\n", 218 keys.authkeylen + keys.enckeylen, keys.enckeylen, 219 keys.authkeylen); 220 print_hex_dump(KERN_ERR, "key in @" __stringify(__LINE__)": ", 221 DUMP_PREFIX_ADDRESS, 16, 4, key, keylen, 1); 222 #endif 223 224 /* 225 * If DKP is supported, use it in the shared descriptor to generate 226 * the split key. 227 */ 228 if (ctrlpriv->era >= 6) { 229 ctx->adata.keylen = keys.authkeylen; 230 ctx->adata.keylen_pad = split_key_len(ctx->adata.algtype & 231 OP_ALG_ALGSEL_MASK); 232 233 if (ctx->adata.keylen_pad + keys.enckeylen > CAAM_MAX_KEY_SIZE) 234 goto badkey; 235 236 memcpy(ctx->key, keys.authkey, keys.authkeylen); 237 memcpy(ctx->key + ctx->adata.keylen_pad, keys.enckey, 238 keys.enckeylen); 239 dma_sync_single_for_device(jrdev, ctx->key_dma, 240 ctx->adata.keylen_pad + 241 keys.enckeylen, ctx->dir); 242 goto skip_split_key; 243 } 244 245 ret = gen_split_key(jrdev, ctx->key, &ctx->adata, keys.authkey, 246 keys.authkeylen, CAAM_MAX_KEY_SIZE - 247 keys.enckeylen); 248 if (ret) 249 goto badkey; 250 251 /* postpend encryption key to auth split key */ 252 memcpy(ctx->key + ctx->adata.keylen_pad, keys.enckey, keys.enckeylen); 253 dma_sync_single_for_device(jrdev, ctx->key_dma, ctx->adata.keylen_pad + 254 keys.enckeylen, ctx->dir); 255 #ifdef DEBUG 256 print_hex_dump(KERN_ERR, "ctx.key@" __stringify(__LINE__)": ", 257 DUMP_PREFIX_ADDRESS, 16, 4, ctx->key, 258 ctx->adata.keylen_pad + keys.enckeylen, 1); 259 #endif 260 261 skip_split_key: 262 ctx->cdata.keylen = keys.enckeylen; 263 264 ret = aead_set_sh_desc(aead); 265 if (ret) 266 goto badkey; 267 268 /* Now update the driver contexts with the new shared descriptor */ 269 if (ctx->drv_ctx[ENCRYPT]) { 270 ret = caam_drv_ctx_update(ctx->drv_ctx[ENCRYPT], 271 ctx->sh_desc_enc); 272 if (ret) { 273 dev_err(jrdev, "driver enc context update failed\n"); 274 goto badkey; 275 } 276 } 277 278 if (ctx->drv_ctx[DECRYPT]) { 279 ret = caam_drv_ctx_update(ctx->drv_ctx[DECRYPT], 280 ctx->sh_desc_dec); 281 if (ret) { 282 dev_err(jrdev, "driver dec context update failed\n"); 283 goto badkey; 284 } 285 } 286 287 memzero_explicit(&keys, sizeof(keys)); 288 return ret; 289 badkey: 290 crypto_aead_set_flags(aead, CRYPTO_TFM_RES_BAD_KEY_LEN); 291 memzero_explicit(&keys, sizeof(keys)); 292 return -EINVAL; 293 } 294 295 static int gcm_set_sh_desc(struct crypto_aead *aead) 296 { 297 struct caam_ctx *ctx = crypto_aead_ctx(aead); 298 unsigned int ivsize = crypto_aead_ivsize(aead); 299 int rem_bytes = CAAM_DESC_BYTES_MAX - DESC_JOB_IO_LEN - 300 ctx->cdata.keylen; 301 302 if (!ctx->cdata.keylen || !ctx->authsize) 303 return 0; 304 305 /* 306 * Job Descriptor and Shared Descriptor 307 * must fit into the 64-word Descriptor h/w Buffer 308 */ 309 if (rem_bytes >= DESC_QI_GCM_ENC_LEN) { 310 ctx->cdata.key_inline = true; 311 ctx->cdata.key_virt = ctx->key; 312 } else { 313 ctx->cdata.key_inline = false; 314 ctx->cdata.key_dma = ctx->key_dma; 315 } 316 317 cnstr_shdsc_gcm_encap(ctx->sh_desc_enc, &ctx->cdata, ivsize, 318 ctx->authsize, true); 319 320 /* 321 * Job Descriptor and Shared Descriptor 322 * must fit into the 64-word Descriptor h/w Buffer 323 */ 324 if (rem_bytes >= DESC_QI_GCM_DEC_LEN) { 325 ctx->cdata.key_inline = true; 326 ctx->cdata.key_virt = ctx->key; 327 } else { 328 ctx->cdata.key_inline = false; 329 ctx->cdata.key_dma = ctx->key_dma; 330 } 331 332 cnstr_shdsc_gcm_decap(ctx->sh_desc_dec, &ctx->cdata, ivsize, 333 ctx->authsize, true); 334 335 return 0; 336 } 337 338 static int gcm_setauthsize(struct crypto_aead *authenc, unsigned int authsize) 339 { 340 struct caam_ctx *ctx = crypto_aead_ctx(authenc); 341 342 ctx->authsize = authsize; 343 gcm_set_sh_desc(authenc); 344 345 return 0; 346 } 347 348 static int gcm_setkey(struct crypto_aead *aead, 349 const u8 *key, unsigned int keylen) 350 { 351 struct caam_ctx *ctx = crypto_aead_ctx(aead); 352 struct device *jrdev = ctx->jrdev; 353 int ret; 354 355 #ifdef DEBUG 356 print_hex_dump(KERN_ERR, "key in @" __stringify(__LINE__)": ", 357 DUMP_PREFIX_ADDRESS, 16, 4, key, keylen, 1); 358 #endif 359 360 memcpy(ctx->key, key, keylen); 361 dma_sync_single_for_device(jrdev, ctx->key_dma, keylen, ctx->dir); 362 ctx->cdata.keylen = keylen; 363 364 ret = gcm_set_sh_desc(aead); 365 if (ret) 366 return ret; 367 368 /* Now update the driver contexts with the new shared descriptor */ 369 if (ctx->drv_ctx[ENCRYPT]) { 370 ret = caam_drv_ctx_update(ctx->drv_ctx[ENCRYPT], 371 ctx->sh_desc_enc); 372 if (ret) { 373 dev_err(jrdev, "driver enc context update failed\n"); 374 return ret; 375 } 376 } 377 378 if (ctx->drv_ctx[DECRYPT]) { 379 ret = caam_drv_ctx_update(ctx->drv_ctx[DECRYPT], 380 ctx->sh_desc_dec); 381 if (ret) { 382 dev_err(jrdev, "driver dec context update failed\n"); 383 return ret; 384 } 385 } 386 387 return 0; 388 } 389 390 static int rfc4106_set_sh_desc(struct crypto_aead *aead) 391 { 392 struct caam_ctx *ctx = crypto_aead_ctx(aead); 393 unsigned int ivsize = crypto_aead_ivsize(aead); 394 int rem_bytes = CAAM_DESC_BYTES_MAX - DESC_JOB_IO_LEN - 395 ctx->cdata.keylen; 396 397 if (!ctx->cdata.keylen || !ctx->authsize) 398 return 0; 399 400 ctx->cdata.key_virt = ctx->key; 401 402 /* 403 * Job Descriptor and Shared Descriptor 404 * must fit into the 64-word Descriptor h/w Buffer 405 */ 406 if (rem_bytes >= DESC_QI_RFC4106_ENC_LEN) { 407 ctx->cdata.key_inline = true; 408 } else { 409 ctx->cdata.key_inline = false; 410 ctx->cdata.key_dma = ctx->key_dma; 411 } 412 413 cnstr_shdsc_rfc4106_encap(ctx->sh_desc_enc, &ctx->cdata, ivsize, 414 ctx->authsize, true); 415 416 /* 417 * Job Descriptor and Shared Descriptor 418 * must fit into the 64-word Descriptor h/w Buffer 419 */ 420 if (rem_bytes >= DESC_QI_RFC4106_DEC_LEN) { 421 ctx->cdata.key_inline = true; 422 } else { 423 ctx->cdata.key_inline = false; 424 ctx->cdata.key_dma = ctx->key_dma; 425 } 426 427 cnstr_shdsc_rfc4106_decap(ctx->sh_desc_dec, &ctx->cdata, ivsize, 428 ctx->authsize, true); 429 430 return 0; 431 } 432 433 static int rfc4106_setauthsize(struct crypto_aead *authenc, 434 unsigned int authsize) 435 { 436 struct caam_ctx *ctx = crypto_aead_ctx(authenc); 437 438 ctx->authsize = authsize; 439 rfc4106_set_sh_desc(authenc); 440 441 return 0; 442 } 443 444 static int rfc4106_setkey(struct crypto_aead *aead, 445 const u8 *key, unsigned int keylen) 446 { 447 struct caam_ctx *ctx = crypto_aead_ctx(aead); 448 struct device *jrdev = ctx->jrdev; 449 int ret; 450 451 if (keylen < 4) 452 return -EINVAL; 453 454 #ifdef DEBUG 455 print_hex_dump(KERN_ERR, "key in @" __stringify(__LINE__)": ", 456 DUMP_PREFIX_ADDRESS, 16, 4, key, keylen, 1); 457 #endif 458 459 memcpy(ctx->key, key, keylen); 460 /* 461 * The last four bytes of the key material are used as the salt value 462 * in the nonce. Update the AES key length. 463 */ 464 ctx->cdata.keylen = keylen - 4; 465 dma_sync_single_for_device(jrdev, ctx->key_dma, ctx->cdata.keylen, 466 ctx->dir); 467 468 ret = rfc4106_set_sh_desc(aead); 469 if (ret) 470 return ret; 471 472 /* Now update the driver contexts with the new shared descriptor */ 473 if (ctx->drv_ctx[ENCRYPT]) { 474 ret = caam_drv_ctx_update(ctx->drv_ctx[ENCRYPT], 475 ctx->sh_desc_enc); 476 if (ret) { 477 dev_err(jrdev, "driver enc context update failed\n"); 478 return ret; 479 } 480 } 481 482 if (ctx->drv_ctx[DECRYPT]) { 483 ret = caam_drv_ctx_update(ctx->drv_ctx[DECRYPT], 484 ctx->sh_desc_dec); 485 if (ret) { 486 dev_err(jrdev, "driver dec context update failed\n"); 487 return ret; 488 } 489 } 490 491 return 0; 492 } 493 494 static int rfc4543_set_sh_desc(struct crypto_aead *aead) 495 { 496 struct caam_ctx *ctx = crypto_aead_ctx(aead); 497 unsigned int ivsize = crypto_aead_ivsize(aead); 498 int rem_bytes = CAAM_DESC_BYTES_MAX - DESC_JOB_IO_LEN - 499 ctx->cdata.keylen; 500 501 if (!ctx->cdata.keylen || !ctx->authsize) 502 return 0; 503 504 ctx->cdata.key_virt = ctx->key; 505 506 /* 507 * Job Descriptor and Shared Descriptor 508 * must fit into the 64-word Descriptor h/w Buffer 509 */ 510 if (rem_bytes >= DESC_QI_RFC4543_ENC_LEN) { 511 ctx->cdata.key_inline = true; 512 } else { 513 ctx->cdata.key_inline = false; 514 ctx->cdata.key_dma = ctx->key_dma; 515 } 516 517 cnstr_shdsc_rfc4543_encap(ctx->sh_desc_enc, &ctx->cdata, ivsize, 518 ctx->authsize, true); 519 520 /* 521 * Job Descriptor and Shared Descriptor 522 * must fit into the 64-word Descriptor h/w Buffer 523 */ 524 if (rem_bytes >= DESC_QI_RFC4543_DEC_LEN) { 525 ctx->cdata.key_inline = true; 526 } else { 527 ctx->cdata.key_inline = false; 528 ctx->cdata.key_dma = ctx->key_dma; 529 } 530 531 cnstr_shdsc_rfc4543_decap(ctx->sh_desc_dec, &ctx->cdata, ivsize, 532 ctx->authsize, true); 533 534 return 0; 535 } 536 537 static int rfc4543_setauthsize(struct crypto_aead *authenc, 538 unsigned int authsize) 539 { 540 struct caam_ctx *ctx = crypto_aead_ctx(authenc); 541 542 ctx->authsize = authsize; 543 rfc4543_set_sh_desc(authenc); 544 545 return 0; 546 } 547 548 static int rfc4543_setkey(struct crypto_aead *aead, 549 const u8 *key, unsigned int keylen) 550 { 551 struct caam_ctx *ctx = crypto_aead_ctx(aead); 552 struct device *jrdev = ctx->jrdev; 553 int ret; 554 555 if (keylen < 4) 556 return -EINVAL; 557 558 #ifdef DEBUG 559 print_hex_dump(KERN_ERR, "key in @" __stringify(__LINE__)": ", 560 DUMP_PREFIX_ADDRESS, 16, 4, key, keylen, 1); 561 #endif 562 563 memcpy(ctx->key, key, keylen); 564 /* 565 * The last four bytes of the key material are used as the salt value 566 * in the nonce. Update the AES key length. 567 */ 568 ctx->cdata.keylen = keylen - 4; 569 dma_sync_single_for_device(jrdev, ctx->key_dma, ctx->cdata.keylen, 570 ctx->dir); 571 572 ret = rfc4543_set_sh_desc(aead); 573 if (ret) 574 return ret; 575 576 /* Now update the driver contexts with the new shared descriptor */ 577 if (ctx->drv_ctx[ENCRYPT]) { 578 ret = caam_drv_ctx_update(ctx->drv_ctx[ENCRYPT], 579 ctx->sh_desc_enc); 580 if (ret) { 581 dev_err(jrdev, "driver enc context update failed\n"); 582 return ret; 583 } 584 } 585 586 if (ctx->drv_ctx[DECRYPT]) { 587 ret = caam_drv_ctx_update(ctx->drv_ctx[DECRYPT], 588 ctx->sh_desc_dec); 589 if (ret) { 590 dev_err(jrdev, "driver dec context update failed\n"); 591 return ret; 592 } 593 } 594 595 return 0; 596 } 597 598 static int skcipher_setkey(struct crypto_skcipher *skcipher, const u8 *key, 599 unsigned int keylen) 600 { 601 struct caam_ctx *ctx = crypto_skcipher_ctx(skcipher); 602 struct caam_skcipher_alg *alg = 603 container_of(crypto_skcipher_alg(skcipher), typeof(*alg), 604 skcipher); 605 struct device *jrdev = ctx->jrdev; 606 unsigned int ivsize = crypto_skcipher_ivsize(skcipher); 607 u32 ctx1_iv_off = 0; 608 const bool ctr_mode = ((ctx->cdata.algtype & OP_ALG_AAI_MASK) == 609 OP_ALG_AAI_CTR_MOD128); 610 const bool is_rfc3686 = alg->caam.rfc3686; 611 int ret = 0; 612 613 #ifdef DEBUG 614 print_hex_dump(KERN_ERR, "key in @" __stringify(__LINE__)": ", 615 DUMP_PREFIX_ADDRESS, 16, 4, key, keylen, 1); 616 #endif 617 /* 618 * AES-CTR needs to load IV in CONTEXT1 reg 619 * at an offset of 128bits (16bytes) 620 * CONTEXT1[255:128] = IV 621 */ 622 if (ctr_mode) 623 ctx1_iv_off = 16; 624 625 /* 626 * RFC3686 specific: 627 * | CONTEXT1[255:128] = {NONCE, IV, COUNTER} 628 * | *key = {KEY, NONCE} 629 */ 630 if (is_rfc3686) { 631 ctx1_iv_off = 16 + CTR_RFC3686_NONCE_SIZE; 632 keylen -= CTR_RFC3686_NONCE_SIZE; 633 } 634 635 ctx->cdata.keylen = keylen; 636 ctx->cdata.key_virt = key; 637 ctx->cdata.key_inline = true; 638 639 /* skcipher encrypt, decrypt shared descriptors */ 640 cnstr_shdsc_skcipher_encap(ctx->sh_desc_enc, &ctx->cdata, ivsize, 641 is_rfc3686, ctx1_iv_off); 642 cnstr_shdsc_skcipher_decap(ctx->sh_desc_dec, &ctx->cdata, ivsize, 643 is_rfc3686, ctx1_iv_off); 644 645 /* Now update the driver contexts with the new shared descriptor */ 646 if (ctx->drv_ctx[ENCRYPT]) { 647 ret = caam_drv_ctx_update(ctx->drv_ctx[ENCRYPT], 648 ctx->sh_desc_enc); 649 if (ret) { 650 dev_err(jrdev, "driver enc context update failed\n"); 651 goto badkey; 652 } 653 } 654 655 if (ctx->drv_ctx[DECRYPT]) { 656 ret = caam_drv_ctx_update(ctx->drv_ctx[DECRYPT], 657 ctx->sh_desc_dec); 658 if (ret) { 659 dev_err(jrdev, "driver dec context update failed\n"); 660 goto badkey; 661 } 662 } 663 664 return ret; 665 badkey: 666 crypto_skcipher_set_flags(skcipher, CRYPTO_TFM_RES_BAD_KEY_LEN); 667 return -EINVAL; 668 } 669 670 static int xts_skcipher_setkey(struct crypto_skcipher *skcipher, const u8 *key, 671 unsigned int keylen) 672 { 673 struct caam_ctx *ctx = crypto_skcipher_ctx(skcipher); 674 struct device *jrdev = ctx->jrdev; 675 int ret = 0; 676 677 if (keylen != 2 * AES_MIN_KEY_SIZE && keylen != 2 * AES_MAX_KEY_SIZE) { 678 dev_err(jrdev, "key size mismatch\n"); 679 goto badkey; 680 } 681 682 ctx->cdata.keylen = keylen; 683 ctx->cdata.key_virt = key; 684 ctx->cdata.key_inline = true; 685 686 /* xts skcipher encrypt, decrypt shared descriptors */ 687 cnstr_shdsc_xts_skcipher_encap(ctx->sh_desc_enc, &ctx->cdata); 688 cnstr_shdsc_xts_skcipher_decap(ctx->sh_desc_dec, &ctx->cdata); 689 690 /* Now update the driver contexts with the new shared descriptor */ 691 if (ctx->drv_ctx[ENCRYPT]) { 692 ret = caam_drv_ctx_update(ctx->drv_ctx[ENCRYPT], 693 ctx->sh_desc_enc); 694 if (ret) { 695 dev_err(jrdev, "driver enc context update failed\n"); 696 goto badkey; 697 } 698 } 699 700 if (ctx->drv_ctx[DECRYPT]) { 701 ret = caam_drv_ctx_update(ctx->drv_ctx[DECRYPT], 702 ctx->sh_desc_dec); 703 if (ret) { 704 dev_err(jrdev, "driver dec context update failed\n"); 705 goto badkey; 706 } 707 } 708 709 return ret; 710 badkey: 711 crypto_skcipher_set_flags(skcipher, CRYPTO_TFM_RES_BAD_KEY_LEN); 712 return -EINVAL; 713 } 714 715 /* 716 * aead_edesc - s/w-extended aead descriptor 717 * @src_nents: number of segments in input scatterlist 718 * @dst_nents: number of segments in output scatterlist 719 * @iv_dma: dma address of iv for checking continuity and link table 720 * @qm_sg_bytes: length of dma mapped h/w link table 721 * @qm_sg_dma: bus physical mapped address of h/w link table 722 * @assoclen: associated data length, in CAAM endianness 723 * @assoclen_dma: bus physical mapped address of req->assoclen 724 * @drv_req: driver-specific request structure 725 * @sgt: the h/w link table, followed by IV 726 */ 727 struct aead_edesc { 728 int src_nents; 729 int dst_nents; 730 dma_addr_t iv_dma; 731 int qm_sg_bytes; 732 dma_addr_t qm_sg_dma; 733 unsigned int assoclen; 734 dma_addr_t assoclen_dma; 735 struct caam_drv_req drv_req; 736 struct qm_sg_entry sgt[0]; 737 }; 738 739 /* 740 * skcipher_edesc - s/w-extended skcipher descriptor 741 * @src_nents: number of segments in input scatterlist 742 * @dst_nents: number of segments in output scatterlist 743 * @iv_dma: dma address of iv for checking continuity and link table 744 * @qm_sg_bytes: length of dma mapped h/w link table 745 * @qm_sg_dma: bus physical mapped address of h/w link table 746 * @drv_req: driver-specific request structure 747 * @sgt: the h/w link table, followed by IV 748 */ 749 struct skcipher_edesc { 750 int src_nents; 751 int dst_nents; 752 dma_addr_t iv_dma; 753 int qm_sg_bytes; 754 dma_addr_t qm_sg_dma; 755 struct caam_drv_req drv_req; 756 struct qm_sg_entry sgt[0]; 757 }; 758 759 static struct caam_drv_ctx *get_drv_ctx(struct caam_ctx *ctx, 760 enum optype type) 761 { 762 /* 763 * This function is called on the fast path with values of 'type' 764 * known at compile time. Invalid arguments are not expected and 765 * thus no checks are made. 766 */ 767 struct caam_drv_ctx *drv_ctx = ctx->drv_ctx[type]; 768 u32 *desc; 769 770 if (unlikely(!drv_ctx)) { 771 spin_lock(&ctx->lock); 772 773 /* Read again to check if some other core init drv_ctx */ 774 drv_ctx = ctx->drv_ctx[type]; 775 if (!drv_ctx) { 776 int cpu; 777 778 if (type == ENCRYPT) 779 desc = ctx->sh_desc_enc; 780 else /* (type == DECRYPT) */ 781 desc = ctx->sh_desc_dec; 782 783 cpu = smp_processor_id(); 784 drv_ctx = caam_drv_ctx_init(ctx->qidev, &cpu, desc); 785 if (!IS_ERR_OR_NULL(drv_ctx)) 786 drv_ctx->op_type = type; 787 788 ctx->drv_ctx[type] = drv_ctx; 789 } 790 791 spin_unlock(&ctx->lock); 792 } 793 794 return drv_ctx; 795 } 796 797 static void caam_unmap(struct device *dev, struct scatterlist *src, 798 struct scatterlist *dst, int src_nents, 799 int dst_nents, dma_addr_t iv_dma, int ivsize, 800 dma_addr_t qm_sg_dma, int qm_sg_bytes) 801 { 802 if (dst != src) { 803 if (src_nents) 804 dma_unmap_sg(dev, src, src_nents, DMA_TO_DEVICE); 805 if (dst_nents) 806 dma_unmap_sg(dev, dst, dst_nents, DMA_FROM_DEVICE); 807 } else { 808 dma_unmap_sg(dev, src, src_nents, DMA_BIDIRECTIONAL); 809 } 810 811 if (iv_dma) 812 dma_unmap_single(dev, iv_dma, ivsize, DMA_TO_DEVICE); 813 if (qm_sg_bytes) 814 dma_unmap_single(dev, qm_sg_dma, qm_sg_bytes, DMA_TO_DEVICE); 815 } 816 817 static void aead_unmap(struct device *dev, 818 struct aead_edesc *edesc, 819 struct aead_request *req) 820 { 821 struct crypto_aead *aead = crypto_aead_reqtfm(req); 822 int ivsize = crypto_aead_ivsize(aead); 823 824 caam_unmap(dev, req->src, req->dst, edesc->src_nents, edesc->dst_nents, 825 edesc->iv_dma, ivsize, edesc->qm_sg_dma, edesc->qm_sg_bytes); 826 dma_unmap_single(dev, edesc->assoclen_dma, 4, DMA_TO_DEVICE); 827 } 828 829 static void skcipher_unmap(struct device *dev, struct skcipher_edesc *edesc, 830 struct skcipher_request *req) 831 { 832 struct crypto_skcipher *skcipher = crypto_skcipher_reqtfm(req); 833 int ivsize = crypto_skcipher_ivsize(skcipher); 834 835 caam_unmap(dev, req->src, req->dst, edesc->src_nents, edesc->dst_nents, 836 edesc->iv_dma, ivsize, edesc->qm_sg_dma, edesc->qm_sg_bytes); 837 } 838 839 static void aead_done(struct caam_drv_req *drv_req, u32 status) 840 { 841 struct device *qidev; 842 struct aead_edesc *edesc; 843 struct aead_request *aead_req = drv_req->app_ctx; 844 struct crypto_aead *aead = crypto_aead_reqtfm(aead_req); 845 struct caam_ctx *caam_ctx = crypto_aead_ctx(aead); 846 int ecode = 0; 847 848 qidev = caam_ctx->qidev; 849 850 if (unlikely(status)) { 851 u32 ssrc = status & JRSTA_SSRC_MASK; 852 u8 err_id = status & JRSTA_CCBERR_ERRID_MASK; 853 854 caam_jr_strstatus(qidev, status); 855 /* 856 * verify hw auth check passed else return -EBADMSG 857 */ 858 if (ssrc == JRSTA_SSRC_CCB_ERROR && 859 err_id == JRSTA_CCBERR_ERRID_ICVCHK) 860 ecode = -EBADMSG; 861 else 862 ecode = -EIO; 863 } 864 865 edesc = container_of(drv_req, typeof(*edesc), drv_req); 866 aead_unmap(qidev, edesc, aead_req); 867 868 aead_request_complete(aead_req, ecode); 869 qi_cache_free(edesc); 870 } 871 872 /* 873 * allocate and map the aead extended descriptor 874 */ 875 static struct aead_edesc *aead_edesc_alloc(struct aead_request *req, 876 bool encrypt) 877 { 878 struct crypto_aead *aead = crypto_aead_reqtfm(req); 879 struct caam_ctx *ctx = crypto_aead_ctx(aead); 880 struct caam_aead_alg *alg = container_of(crypto_aead_alg(aead), 881 typeof(*alg), aead); 882 struct device *qidev = ctx->qidev; 883 gfp_t flags = (req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP) ? 884 GFP_KERNEL : GFP_ATOMIC; 885 int src_nents, mapped_src_nents, dst_nents = 0, mapped_dst_nents = 0; 886 struct aead_edesc *edesc; 887 dma_addr_t qm_sg_dma, iv_dma = 0; 888 int ivsize = 0; 889 unsigned int authsize = ctx->authsize; 890 int qm_sg_index = 0, qm_sg_ents = 0, qm_sg_bytes; 891 int in_len, out_len; 892 struct qm_sg_entry *sg_table, *fd_sgt; 893 struct caam_drv_ctx *drv_ctx; 894 895 drv_ctx = get_drv_ctx(ctx, encrypt ? ENCRYPT : DECRYPT); 896 if (IS_ERR_OR_NULL(drv_ctx)) 897 return (struct aead_edesc *)drv_ctx; 898 899 /* allocate space for base edesc and hw desc commands, link tables */ 900 edesc = qi_cache_alloc(GFP_DMA | flags); 901 if (unlikely(!edesc)) { 902 dev_err(qidev, "could not allocate extended descriptor\n"); 903 return ERR_PTR(-ENOMEM); 904 } 905 906 if (likely(req->src == req->dst)) { 907 src_nents = sg_nents_for_len(req->src, req->assoclen + 908 req->cryptlen + 909 (encrypt ? authsize : 0)); 910 if (unlikely(src_nents < 0)) { 911 dev_err(qidev, "Insufficient bytes (%d) in src S/G\n", 912 req->assoclen + req->cryptlen + 913 (encrypt ? authsize : 0)); 914 qi_cache_free(edesc); 915 return ERR_PTR(src_nents); 916 } 917 918 mapped_src_nents = dma_map_sg(qidev, req->src, src_nents, 919 DMA_BIDIRECTIONAL); 920 if (unlikely(!mapped_src_nents)) { 921 dev_err(qidev, "unable to map source\n"); 922 qi_cache_free(edesc); 923 return ERR_PTR(-ENOMEM); 924 } 925 } else { 926 src_nents = sg_nents_for_len(req->src, req->assoclen + 927 req->cryptlen); 928 if (unlikely(src_nents < 0)) { 929 dev_err(qidev, "Insufficient bytes (%d) in src S/G\n", 930 req->assoclen + req->cryptlen); 931 qi_cache_free(edesc); 932 return ERR_PTR(src_nents); 933 } 934 935 dst_nents = sg_nents_for_len(req->dst, req->assoclen + 936 req->cryptlen + 937 (encrypt ? authsize : 938 (-authsize))); 939 if (unlikely(dst_nents < 0)) { 940 dev_err(qidev, "Insufficient bytes (%d) in dst S/G\n", 941 req->assoclen + req->cryptlen + 942 (encrypt ? authsize : (-authsize))); 943 qi_cache_free(edesc); 944 return ERR_PTR(dst_nents); 945 } 946 947 if (src_nents) { 948 mapped_src_nents = dma_map_sg(qidev, req->src, 949 src_nents, DMA_TO_DEVICE); 950 if (unlikely(!mapped_src_nents)) { 951 dev_err(qidev, "unable to map source\n"); 952 qi_cache_free(edesc); 953 return ERR_PTR(-ENOMEM); 954 } 955 } else { 956 mapped_src_nents = 0; 957 } 958 959 if (dst_nents) { 960 mapped_dst_nents = dma_map_sg(qidev, req->dst, 961 dst_nents, 962 DMA_FROM_DEVICE); 963 if (unlikely(!mapped_dst_nents)) { 964 dev_err(qidev, "unable to map destination\n"); 965 dma_unmap_sg(qidev, req->src, src_nents, 966 DMA_TO_DEVICE); 967 qi_cache_free(edesc); 968 return ERR_PTR(-ENOMEM); 969 } 970 } else { 971 mapped_dst_nents = 0; 972 } 973 } 974 975 if ((alg->caam.rfc3686 && encrypt) || !alg->caam.geniv) 976 ivsize = crypto_aead_ivsize(aead); 977 978 /* 979 * Create S/G table: req->assoclen, [IV,] req->src [, req->dst]. 980 * Input is not contiguous. 981 */ 982 qm_sg_ents = 1 + !!ivsize + mapped_src_nents + 983 (mapped_dst_nents > 1 ? mapped_dst_nents : 0); 984 sg_table = &edesc->sgt[0]; 985 qm_sg_bytes = qm_sg_ents * sizeof(*sg_table); 986 if (unlikely(offsetof(struct aead_edesc, sgt) + qm_sg_bytes + ivsize > 987 CAAM_QI_MEMCACHE_SIZE)) { 988 dev_err(qidev, "No space for %d S/G entries and/or %dB IV\n", 989 qm_sg_ents, ivsize); 990 caam_unmap(qidev, req->src, req->dst, src_nents, dst_nents, 0, 991 0, 0, 0); 992 qi_cache_free(edesc); 993 return ERR_PTR(-ENOMEM); 994 } 995 996 if (ivsize) { 997 u8 *iv = (u8 *)(sg_table + qm_sg_ents); 998 999 /* Make sure IV is located in a DMAable area */ 1000 memcpy(iv, req->iv, ivsize); 1001 1002 iv_dma = dma_map_single(qidev, iv, ivsize, DMA_TO_DEVICE); 1003 if (dma_mapping_error(qidev, iv_dma)) { 1004 dev_err(qidev, "unable to map IV\n"); 1005 caam_unmap(qidev, req->src, req->dst, src_nents, 1006 dst_nents, 0, 0, 0, 0); 1007 qi_cache_free(edesc); 1008 return ERR_PTR(-ENOMEM); 1009 } 1010 } 1011 1012 edesc->src_nents = src_nents; 1013 edesc->dst_nents = dst_nents; 1014 edesc->iv_dma = iv_dma; 1015 edesc->drv_req.app_ctx = req; 1016 edesc->drv_req.cbk = aead_done; 1017 edesc->drv_req.drv_ctx = drv_ctx; 1018 1019 edesc->assoclen = cpu_to_caam32(req->assoclen); 1020 edesc->assoclen_dma = dma_map_single(qidev, &edesc->assoclen, 4, 1021 DMA_TO_DEVICE); 1022 if (dma_mapping_error(qidev, edesc->assoclen_dma)) { 1023 dev_err(qidev, "unable to map assoclen\n"); 1024 caam_unmap(qidev, req->src, req->dst, src_nents, dst_nents, 1025 iv_dma, ivsize, 0, 0); 1026 qi_cache_free(edesc); 1027 return ERR_PTR(-ENOMEM); 1028 } 1029 1030 dma_to_qm_sg_one(sg_table, edesc->assoclen_dma, 4, 0); 1031 qm_sg_index++; 1032 if (ivsize) { 1033 dma_to_qm_sg_one(sg_table + qm_sg_index, iv_dma, ivsize, 0); 1034 qm_sg_index++; 1035 } 1036 sg_to_qm_sg_last(req->src, mapped_src_nents, sg_table + qm_sg_index, 0); 1037 qm_sg_index += mapped_src_nents; 1038 1039 if (mapped_dst_nents > 1) 1040 sg_to_qm_sg_last(req->dst, mapped_dst_nents, sg_table + 1041 qm_sg_index, 0); 1042 1043 qm_sg_dma = dma_map_single(qidev, sg_table, qm_sg_bytes, DMA_TO_DEVICE); 1044 if (dma_mapping_error(qidev, qm_sg_dma)) { 1045 dev_err(qidev, "unable to map S/G table\n"); 1046 dma_unmap_single(qidev, edesc->assoclen_dma, 4, DMA_TO_DEVICE); 1047 caam_unmap(qidev, req->src, req->dst, src_nents, dst_nents, 1048 iv_dma, ivsize, 0, 0); 1049 qi_cache_free(edesc); 1050 return ERR_PTR(-ENOMEM); 1051 } 1052 1053 edesc->qm_sg_dma = qm_sg_dma; 1054 edesc->qm_sg_bytes = qm_sg_bytes; 1055 1056 out_len = req->assoclen + req->cryptlen + 1057 (encrypt ? ctx->authsize : (-ctx->authsize)); 1058 in_len = 4 + ivsize + req->assoclen + req->cryptlen; 1059 1060 fd_sgt = &edesc->drv_req.fd_sgt[0]; 1061 dma_to_qm_sg_one_last_ext(&fd_sgt[1], qm_sg_dma, in_len, 0); 1062 1063 if (req->dst == req->src) { 1064 if (mapped_src_nents == 1) 1065 dma_to_qm_sg_one(&fd_sgt[0], sg_dma_address(req->src), 1066 out_len, 0); 1067 else 1068 dma_to_qm_sg_one_ext(&fd_sgt[0], qm_sg_dma + 1069 (1 + !!ivsize) * sizeof(*sg_table), 1070 out_len, 0); 1071 } else if (mapped_dst_nents == 1) { 1072 dma_to_qm_sg_one(&fd_sgt[0], sg_dma_address(req->dst), out_len, 1073 0); 1074 } else { 1075 dma_to_qm_sg_one_ext(&fd_sgt[0], qm_sg_dma + sizeof(*sg_table) * 1076 qm_sg_index, out_len, 0); 1077 } 1078 1079 return edesc; 1080 } 1081 1082 static inline int aead_crypt(struct aead_request *req, bool encrypt) 1083 { 1084 struct aead_edesc *edesc; 1085 struct crypto_aead *aead = crypto_aead_reqtfm(req); 1086 struct caam_ctx *ctx = crypto_aead_ctx(aead); 1087 int ret; 1088 1089 if (unlikely(caam_congested)) 1090 return -EAGAIN; 1091 1092 /* allocate extended descriptor */ 1093 edesc = aead_edesc_alloc(req, encrypt); 1094 if (IS_ERR_OR_NULL(edesc)) 1095 return PTR_ERR(edesc); 1096 1097 /* Create and submit job descriptor */ 1098 ret = caam_qi_enqueue(ctx->qidev, &edesc->drv_req); 1099 if (!ret) { 1100 ret = -EINPROGRESS; 1101 } else { 1102 aead_unmap(ctx->qidev, edesc, req); 1103 qi_cache_free(edesc); 1104 } 1105 1106 return ret; 1107 } 1108 1109 static int aead_encrypt(struct aead_request *req) 1110 { 1111 return aead_crypt(req, true); 1112 } 1113 1114 static int aead_decrypt(struct aead_request *req) 1115 { 1116 return aead_crypt(req, false); 1117 } 1118 1119 static int ipsec_gcm_encrypt(struct aead_request *req) 1120 { 1121 if (req->assoclen < 8) 1122 return -EINVAL; 1123 1124 return aead_crypt(req, true); 1125 } 1126 1127 static int ipsec_gcm_decrypt(struct aead_request *req) 1128 { 1129 if (req->assoclen < 8) 1130 return -EINVAL; 1131 1132 return aead_crypt(req, false); 1133 } 1134 1135 static void skcipher_done(struct caam_drv_req *drv_req, u32 status) 1136 { 1137 struct skcipher_edesc *edesc; 1138 struct skcipher_request *req = drv_req->app_ctx; 1139 struct crypto_skcipher *skcipher = crypto_skcipher_reqtfm(req); 1140 struct caam_ctx *caam_ctx = crypto_skcipher_ctx(skcipher); 1141 struct device *qidev = caam_ctx->qidev; 1142 int ivsize = crypto_skcipher_ivsize(skcipher); 1143 1144 #ifdef DEBUG 1145 dev_err(qidev, "%s %d: status 0x%x\n", __func__, __LINE__, status); 1146 #endif 1147 1148 edesc = container_of(drv_req, typeof(*edesc), drv_req); 1149 1150 if (status) 1151 caam_jr_strstatus(qidev, status); 1152 1153 #ifdef DEBUG 1154 print_hex_dump(KERN_ERR, "dstiv @" __stringify(__LINE__)": ", 1155 DUMP_PREFIX_ADDRESS, 16, 4, req->iv, 1156 edesc->src_nents > 1 ? 100 : ivsize, 1); 1157 caam_dump_sg(KERN_ERR, "dst @" __stringify(__LINE__)": ", 1158 DUMP_PREFIX_ADDRESS, 16, 4, req->dst, 1159 edesc->dst_nents > 1 ? 100 : req->cryptlen, 1); 1160 #endif 1161 1162 skcipher_unmap(qidev, edesc, req); 1163 1164 /* 1165 * The crypto API expects us to set the IV (req->iv) to the last 1166 * ciphertext block. This is used e.g. by the CTS mode. 1167 */ 1168 if (edesc->drv_req.drv_ctx->op_type == ENCRYPT) 1169 scatterwalk_map_and_copy(req->iv, req->dst, req->cryptlen - 1170 ivsize, ivsize, 0); 1171 1172 qi_cache_free(edesc); 1173 skcipher_request_complete(req, status); 1174 } 1175 1176 static struct skcipher_edesc *skcipher_edesc_alloc(struct skcipher_request *req, 1177 bool encrypt) 1178 { 1179 struct crypto_skcipher *skcipher = crypto_skcipher_reqtfm(req); 1180 struct caam_ctx *ctx = crypto_skcipher_ctx(skcipher); 1181 struct device *qidev = ctx->qidev; 1182 gfp_t flags = (req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP) ? 1183 GFP_KERNEL : GFP_ATOMIC; 1184 int src_nents, mapped_src_nents, dst_nents = 0, mapped_dst_nents = 0; 1185 struct skcipher_edesc *edesc; 1186 dma_addr_t iv_dma; 1187 u8 *iv; 1188 int ivsize = crypto_skcipher_ivsize(skcipher); 1189 int dst_sg_idx, qm_sg_ents, qm_sg_bytes; 1190 struct qm_sg_entry *sg_table, *fd_sgt; 1191 struct caam_drv_ctx *drv_ctx; 1192 1193 drv_ctx = get_drv_ctx(ctx, encrypt ? ENCRYPT : DECRYPT); 1194 if (IS_ERR_OR_NULL(drv_ctx)) 1195 return (struct skcipher_edesc *)drv_ctx; 1196 1197 src_nents = sg_nents_for_len(req->src, req->cryptlen); 1198 if (unlikely(src_nents < 0)) { 1199 dev_err(qidev, "Insufficient bytes (%d) in src S/G\n", 1200 req->cryptlen); 1201 return ERR_PTR(src_nents); 1202 } 1203 1204 if (unlikely(req->src != req->dst)) { 1205 dst_nents = sg_nents_for_len(req->dst, req->cryptlen); 1206 if (unlikely(dst_nents < 0)) { 1207 dev_err(qidev, "Insufficient bytes (%d) in dst S/G\n", 1208 req->cryptlen); 1209 return ERR_PTR(dst_nents); 1210 } 1211 1212 mapped_src_nents = dma_map_sg(qidev, req->src, src_nents, 1213 DMA_TO_DEVICE); 1214 if (unlikely(!mapped_src_nents)) { 1215 dev_err(qidev, "unable to map source\n"); 1216 return ERR_PTR(-ENOMEM); 1217 } 1218 1219 mapped_dst_nents = dma_map_sg(qidev, req->dst, dst_nents, 1220 DMA_FROM_DEVICE); 1221 if (unlikely(!mapped_dst_nents)) { 1222 dev_err(qidev, "unable to map destination\n"); 1223 dma_unmap_sg(qidev, req->src, src_nents, DMA_TO_DEVICE); 1224 return ERR_PTR(-ENOMEM); 1225 } 1226 } else { 1227 mapped_src_nents = dma_map_sg(qidev, req->src, src_nents, 1228 DMA_BIDIRECTIONAL); 1229 if (unlikely(!mapped_src_nents)) { 1230 dev_err(qidev, "unable to map source\n"); 1231 return ERR_PTR(-ENOMEM); 1232 } 1233 } 1234 1235 qm_sg_ents = 1 + mapped_src_nents; 1236 dst_sg_idx = qm_sg_ents; 1237 1238 qm_sg_ents += mapped_dst_nents > 1 ? mapped_dst_nents : 0; 1239 qm_sg_bytes = qm_sg_ents * sizeof(struct qm_sg_entry); 1240 if (unlikely(offsetof(struct skcipher_edesc, sgt) + qm_sg_bytes + 1241 ivsize > CAAM_QI_MEMCACHE_SIZE)) { 1242 dev_err(qidev, "No space for %d S/G entries and/or %dB IV\n", 1243 qm_sg_ents, ivsize); 1244 caam_unmap(qidev, req->src, req->dst, src_nents, dst_nents, 0, 1245 0, 0, 0); 1246 return ERR_PTR(-ENOMEM); 1247 } 1248 1249 /* allocate space for base edesc, link tables and IV */ 1250 edesc = qi_cache_alloc(GFP_DMA | flags); 1251 if (unlikely(!edesc)) { 1252 dev_err(qidev, "could not allocate extended descriptor\n"); 1253 caam_unmap(qidev, req->src, req->dst, src_nents, dst_nents, 0, 1254 0, 0, 0); 1255 return ERR_PTR(-ENOMEM); 1256 } 1257 1258 /* Make sure IV is located in a DMAable area */ 1259 sg_table = &edesc->sgt[0]; 1260 iv = (u8 *)(sg_table + qm_sg_ents); 1261 memcpy(iv, req->iv, ivsize); 1262 1263 iv_dma = dma_map_single(qidev, iv, ivsize, DMA_TO_DEVICE); 1264 if (dma_mapping_error(qidev, iv_dma)) { 1265 dev_err(qidev, "unable to map IV\n"); 1266 caam_unmap(qidev, req->src, req->dst, src_nents, dst_nents, 0, 1267 0, 0, 0); 1268 qi_cache_free(edesc); 1269 return ERR_PTR(-ENOMEM); 1270 } 1271 1272 edesc->src_nents = src_nents; 1273 edesc->dst_nents = dst_nents; 1274 edesc->iv_dma = iv_dma; 1275 edesc->qm_sg_bytes = qm_sg_bytes; 1276 edesc->drv_req.app_ctx = req; 1277 edesc->drv_req.cbk = skcipher_done; 1278 edesc->drv_req.drv_ctx = drv_ctx; 1279 1280 dma_to_qm_sg_one(sg_table, iv_dma, ivsize, 0); 1281 sg_to_qm_sg_last(req->src, mapped_src_nents, sg_table + 1, 0); 1282 1283 if (mapped_dst_nents > 1) 1284 sg_to_qm_sg_last(req->dst, mapped_dst_nents, sg_table + 1285 dst_sg_idx, 0); 1286 1287 edesc->qm_sg_dma = dma_map_single(qidev, sg_table, edesc->qm_sg_bytes, 1288 DMA_TO_DEVICE); 1289 if (dma_mapping_error(qidev, edesc->qm_sg_dma)) { 1290 dev_err(qidev, "unable to map S/G table\n"); 1291 caam_unmap(qidev, req->src, req->dst, src_nents, dst_nents, 1292 iv_dma, ivsize, 0, 0); 1293 qi_cache_free(edesc); 1294 return ERR_PTR(-ENOMEM); 1295 } 1296 1297 fd_sgt = &edesc->drv_req.fd_sgt[0]; 1298 1299 dma_to_qm_sg_one_last_ext(&fd_sgt[1], edesc->qm_sg_dma, 1300 ivsize + req->cryptlen, 0); 1301 1302 if (req->src == req->dst) { 1303 dma_to_qm_sg_one_ext(&fd_sgt[0], edesc->qm_sg_dma + 1304 sizeof(*sg_table), req->cryptlen, 0); 1305 } else if (mapped_dst_nents > 1) { 1306 dma_to_qm_sg_one_ext(&fd_sgt[0], edesc->qm_sg_dma + dst_sg_idx * 1307 sizeof(*sg_table), req->cryptlen, 0); 1308 } else { 1309 dma_to_qm_sg_one(&fd_sgt[0], sg_dma_address(req->dst), 1310 req->cryptlen, 0); 1311 } 1312 1313 return edesc; 1314 } 1315 1316 static inline int skcipher_crypt(struct skcipher_request *req, bool encrypt) 1317 { 1318 struct skcipher_edesc *edesc; 1319 struct crypto_skcipher *skcipher = crypto_skcipher_reqtfm(req); 1320 struct caam_ctx *ctx = crypto_skcipher_ctx(skcipher); 1321 int ivsize = crypto_skcipher_ivsize(skcipher); 1322 int ret; 1323 1324 if (unlikely(caam_congested)) 1325 return -EAGAIN; 1326 1327 /* allocate extended descriptor */ 1328 edesc = skcipher_edesc_alloc(req, encrypt); 1329 if (IS_ERR(edesc)) 1330 return PTR_ERR(edesc); 1331 1332 /* 1333 * The crypto API expects us to set the IV (req->iv) to the last 1334 * ciphertext block. 1335 */ 1336 if (!encrypt) 1337 scatterwalk_map_and_copy(req->iv, req->src, req->cryptlen - 1338 ivsize, ivsize, 0); 1339 1340 ret = caam_qi_enqueue(ctx->qidev, &edesc->drv_req); 1341 if (!ret) { 1342 ret = -EINPROGRESS; 1343 } else { 1344 skcipher_unmap(ctx->qidev, edesc, req); 1345 qi_cache_free(edesc); 1346 } 1347 1348 return ret; 1349 } 1350 1351 static int skcipher_encrypt(struct skcipher_request *req) 1352 { 1353 return skcipher_crypt(req, true); 1354 } 1355 1356 static int skcipher_decrypt(struct skcipher_request *req) 1357 { 1358 return skcipher_crypt(req, false); 1359 } 1360 1361 static struct caam_skcipher_alg driver_algs[] = { 1362 { 1363 .skcipher = { 1364 .base = { 1365 .cra_name = "cbc(aes)", 1366 .cra_driver_name = "cbc-aes-caam-qi", 1367 .cra_blocksize = AES_BLOCK_SIZE, 1368 }, 1369 .setkey = skcipher_setkey, 1370 .encrypt = skcipher_encrypt, 1371 .decrypt = skcipher_decrypt, 1372 .min_keysize = AES_MIN_KEY_SIZE, 1373 .max_keysize = AES_MAX_KEY_SIZE, 1374 .ivsize = AES_BLOCK_SIZE, 1375 }, 1376 .caam.class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CBC, 1377 }, 1378 { 1379 .skcipher = { 1380 .base = { 1381 .cra_name = "cbc(des3_ede)", 1382 .cra_driver_name = "cbc-3des-caam-qi", 1383 .cra_blocksize = DES3_EDE_BLOCK_SIZE, 1384 }, 1385 .setkey = skcipher_setkey, 1386 .encrypt = skcipher_encrypt, 1387 .decrypt = skcipher_decrypt, 1388 .min_keysize = DES3_EDE_KEY_SIZE, 1389 .max_keysize = DES3_EDE_KEY_SIZE, 1390 .ivsize = DES3_EDE_BLOCK_SIZE, 1391 }, 1392 .caam.class1_alg_type = OP_ALG_ALGSEL_3DES | OP_ALG_AAI_CBC, 1393 }, 1394 { 1395 .skcipher = { 1396 .base = { 1397 .cra_name = "cbc(des)", 1398 .cra_driver_name = "cbc-des-caam-qi", 1399 .cra_blocksize = DES_BLOCK_SIZE, 1400 }, 1401 .setkey = skcipher_setkey, 1402 .encrypt = skcipher_encrypt, 1403 .decrypt = skcipher_decrypt, 1404 .min_keysize = DES_KEY_SIZE, 1405 .max_keysize = DES_KEY_SIZE, 1406 .ivsize = DES_BLOCK_SIZE, 1407 }, 1408 .caam.class1_alg_type = OP_ALG_ALGSEL_DES | OP_ALG_AAI_CBC, 1409 }, 1410 { 1411 .skcipher = { 1412 .base = { 1413 .cra_name = "ctr(aes)", 1414 .cra_driver_name = "ctr-aes-caam-qi", 1415 .cra_blocksize = 1, 1416 }, 1417 .setkey = skcipher_setkey, 1418 .encrypt = skcipher_encrypt, 1419 .decrypt = skcipher_decrypt, 1420 .min_keysize = AES_MIN_KEY_SIZE, 1421 .max_keysize = AES_MAX_KEY_SIZE, 1422 .ivsize = AES_BLOCK_SIZE, 1423 .chunksize = AES_BLOCK_SIZE, 1424 }, 1425 .caam.class1_alg_type = OP_ALG_ALGSEL_AES | 1426 OP_ALG_AAI_CTR_MOD128, 1427 }, 1428 { 1429 .skcipher = { 1430 .base = { 1431 .cra_name = "rfc3686(ctr(aes))", 1432 .cra_driver_name = "rfc3686-ctr-aes-caam-qi", 1433 .cra_blocksize = 1, 1434 }, 1435 .setkey = skcipher_setkey, 1436 .encrypt = skcipher_encrypt, 1437 .decrypt = skcipher_decrypt, 1438 .min_keysize = AES_MIN_KEY_SIZE + 1439 CTR_RFC3686_NONCE_SIZE, 1440 .max_keysize = AES_MAX_KEY_SIZE + 1441 CTR_RFC3686_NONCE_SIZE, 1442 .ivsize = CTR_RFC3686_IV_SIZE, 1443 .chunksize = AES_BLOCK_SIZE, 1444 }, 1445 .caam = { 1446 .class1_alg_type = OP_ALG_ALGSEL_AES | 1447 OP_ALG_AAI_CTR_MOD128, 1448 .rfc3686 = true, 1449 }, 1450 }, 1451 { 1452 .skcipher = { 1453 .base = { 1454 .cra_name = "xts(aes)", 1455 .cra_driver_name = "xts-aes-caam-qi", 1456 .cra_blocksize = AES_BLOCK_SIZE, 1457 }, 1458 .setkey = xts_skcipher_setkey, 1459 .encrypt = skcipher_encrypt, 1460 .decrypt = skcipher_decrypt, 1461 .min_keysize = 2 * AES_MIN_KEY_SIZE, 1462 .max_keysize = 2 * AES_MAX_KEY_SIZE, 1463 .ivsize = AES_BLOCK_SIZE, 1464 }, 1465 .caam.class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_XTS, 1466 }, 1467 }; 1468 1469 static struct caam_aead_alg driver_aeads[] = { 1470 { 1471 .aead = { 1472 .base = { 1473 .cra_name = "rfc4106(gcm(aes))", 1474 .cra_driver_name = "rfc4106-gcm-aes-caam-qi", 1475 .cra_blocksize = 1, 1476 }, 1477 .setkey = rfc4106_setkey, 1478 .setauthsize = rfc4106_setauthsize, 1479 .encrypt = ipsec_gcm_encrypt, 1480 .decrypt = ipsec_gcm_decrypt, 1481 .ivsize = 8, 1482 .maxauthsize = AES_BLOCK_SIZE, 1483 }, 1484 .caam = { 1485 .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_GCM, 1486 }, 1487 }, 1488 { 1489 .aead = { 1490 .base = { 1491 .cra_name = "rfc4543(gcm(aes))", 1492 .cra_driver_name = "rfc4543-gcm-aes-caam-qi", 1493 .cra_blocksize = 1, 1494 }, 1495 .setkey = rfc4543_setkey, 1496 .setauthsize = rfc4543_setauthsize, 1497 .encrypt = ipsec_gcm_encrypt, 1498 .decrypt = ipsec_gcm_decrypt, 1499 .ivsize = 8, 1500 .maxauthsize = AES_BLOCK_SIZE, 1501 }, 1502 .caam = { 1503 .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_GCM, 1504 }, 1505 }, 1506 /* Galois Counter Mode */ 1507 { 1508 .aead = { 1509 .base = { 1510 .cra_name = "gcm(aes)", 1511 .cra_driver_name = "gcm-aes-caam-qi", 1512 .cra_blocksize = 1, 1513 }, 1514 .setkey = gcm_setkey, 1515 .setauthsize = gcm_setauthsize, 1516 .encrypt = aead_encrypt, 1517 .decrypt = aead_decrypt, 1518 .ivsize = 12, 1519 .maxauthsize = AES_BLOCK_SIZE, 1520 }, 1521 .caam = { 1522 .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_GCM, 1523 } 1524 }, 1525 /* single-pass ipsec_esp descriptor */ 1526 { 1527 .aead = { 1528 .base = { 1529 .cra_name = "authenc(hmac(md5),cbc(aes))", 1530 .cra_driver_name = "authenc-hmac-md5-" 1531 "cbc-aes-caam-qi", 1532 .cra_blocksize = AES_BLOCK_SIZE, 1533 }, 1534 .setkey = aead_setkey, 1535 .setauthsize = aead_setauthsize, 1536 .encrypt = aead_encrypt, 1537 .decrypt = aead_decrypt, 1538 .ivsize = AES_BLOCK_SIZE, 1539 .maxauthsize = MD5_DIGEST_SIZE, 1540 }, 1541 .caam = { 1542 .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CBC, 1543 .class2_alg_type = OP_ALG_ALGSEL_MD5 | 1544 OP_ALG_AAI_HMAC_PRECOMP, 1545 } 1546 }, 1547 { 1548 .aead = { 1549 .base = { 1550 .cra_name = "echainiv(authenc(hmac(md5)," 1551 "cbc(aes)))", 1552 .cra_driver_name = "echainiv-authenc-hmac-md5-" 1553 "cbc-aes-caam-qi", 1554 .cra_blocksize = AES_BLOCK_SIZE, 1555 }, 1556 .setkey = aead_setkey, 1557 .setauthsize = aead_setauthsize, 1558 .encrypt = aead_encrypt, 1559 .decrypt = aead_decrypt, 1560 .ivsize = AES_BLOCK_SIZE, 1561 .maxauthsize = MD5_DIGEST_SIZE, 1562 }, 1563 .caam = { 1564 .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CBC, 1565 .class2_alg_type = OP_ALG_ALGSEL_MD5 | 1566 OP_ALG_AAI_HMAC_PRECOMP, 1567 .geniv = true, 1568 } 1569 }, 1570 { 1571 .aead = { 1572 .base = { 1573 .cra_name = "authenc(hmac(sha1),cbc(aes))", 1574 .cra_driver_name = "authenc-hmac-sha1-" 1575 "cbc-aes-caam-qi", 1576 .cra_blocksize = AES_BLOCK_SIZE, 1577 }, 1578 .setkey = aead_setkey, 1579 .setauthsize = aead_setauthsize, 1580 .encrypt = aead_encrypt, 1581 .decrypt = aead_decrypt, 1582 .ivsize = AES_BLOCK_SIZE, 1583 .maxauthsize = SHA1_DIGEST_SIZE, 1584 }, 1585 .caam = { 1586 .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CBC, 1587 .class2_alg_type = OP_ALG_ALGSEL_SHA1 | 1588 OP_ALG_AAI_HMAC_PRECOMP, 1589 } 1590 }, 1591 { 1592 .aead = { 1593 .base = { 1594 .cra_name = "echainiv(authenc(hmac(sha1)," 1595 "cbc(aes)))", 1596 .cra_driver_name = "echainiv-authenc-" 1597 "hmac-sha1-cbc-aes-caam-qi", 1598 .cra_blocksize = AES_BLOCK_SIZE, 1599 }, 1600 .setkey = aead_setkey, 1601 .setauthsize = aead_setauthsize, 1602 .encrypt = aead_encrypt, 1603 .decrypt = aead_decrypt, 1604 .ivsize = AES_BLOCK_SIZE, 1605 .maxauthsize = SHA1_DIGEST_SIZE, 1606 }, 1607 .caam = { 1608 .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CBC, 1609 .class2_alg_type = OP_ALG_ALGSEL_SHA1 | 1610 OP_ALG_AAI_HMAC_PRECOMP, 1611 .geniv = true, 1612 }, 1613 }, 1614 { 1615 .aead = { 1616 .base = { 1617 .cra_name = "authenc(hmac(sha224),cbc(aes))", 1618 .cra_driver_name = "authenc-hmac-sha224-" 1619 "cbc-aes-caam-qi", 1620 .cra_blocksize = AES_BLOCK_SIZE, 1621 }, 1622 .setkey = aead_setkey, 1623 .setauthsize = aead_setauthsize, 1624 .encrypt = aead_encrypt, 1625 .decrypt = aead_decrypt, 1626 .ivsize = AES_BLOCK_SIZE, 1627 .maxauthsize = SHA224_DIGEST_SIZE, 1628 }, 1629 .caam = { 1630 .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CBC, 1631 .class2_alg_type = OP_ALG_ALGSEL_SHA224 | 1632 OP_ALG_AAI_HMAC_PRECOMP, 1633 } 1634 }, 1635 { 1636 .aead = { 1637 .base = { 1638 .cra_name = "echainiv(authenc(hmac(sha224)," 1639 "cbc(aes)))", 1640 .cra_driver_name = "echainiv-authenc-" 1641 "hmac-sha224-cbc-aes-caam-qi", 1642 .cra_blocksize = AES_BLOCK_SIZE, 1643 }, 1644 .setkey = aead_setkey, 1645 .setauthsize = aead_setauthsize, 1646 .encrypt = aead_encrypt, 1647 .decrypt = aead_decrypt, 1648 .ivsize = AES_BLOCK_SIZE, 1649 .maxauthsize = SHA224_DIGEST_SIZE, 1650 }, 1651 .caam = { 1652 .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CBC, 1653 .class2_alg_type = OP_ALG_ALGSEL_SHA224 | 1654 OP_ALG_AAI_HMAC_PRECOMP, 1655 .geniv = true, 1656 } 1657 }, 1658 { 1659 .aead = { 1660 .base = { 1661 .cra_name = "authenc(hmac(sha256),cbc(aes))", 1662 .cra_driver_name = "authenc-hmac-sha256-" 1663 "cbc-aes-caam-qi", 1664 .cra_blocksize = AES_BLOCK_SIZE, 1665 }, 1666 .setkey = aead_setkey, 1667 .setauthsize = aead_setauthsize, 1668 .encrypt = aead_encrypt, 1669 .decrypt = aead_decrypt, 1670 .ivsize = AES_BLOCK_SIZE, 1671 .maxauthsize = SHA256_DIGEST_SIZE, 1672 }, 1673 .caam = { 1674 .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CBC, 1675 .class2_alg_type = OP_ALG_ALGSEL_SHA256 | 1676 OP_ALG_AAI_HMAC_PRECOMP, 1677 } 1678 }, 1679 { 1680 .aead = { 1681 .base = { 1682 .cra_name = "echainiv(authenc(hmac(sha256)," 1683 "cbc(aes)))", 1684 .cra_driver_name = "echainiv-authenc-" 1685 "hmac-sha256-cbc-aes-" 1686 "caam-qi", 1687 .cra_blocksize = AES_BLOCK_SIZE, 1688 }, 1689 .setkey = aead_setkey, 1690 .setauthsize = aead_setauthsize, 1691 .encrypt = aead_encrypt, 1692 .decrypt = aead_decrypt, 1693 .ivsize = AES_BLOCK_SIZE, 1694 .maxauthsize = SHA256_DIGEST_SIZE, 1695 }, 1696 .caam = { 1697 .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CBC, 1698 .class2_alg_type = OP_ALG_ALGSEL_SHA256 | 1699 OP_ALG_AAI_HMAC_PRECOMP, 1700 .geniv = true, 1701 } 1702 }, 1703 { 1704 .aead = { 1705 .base = { 1706 .cra_name = "authenc(hmac(sha384),cbc(aes))", 1707 .cra_driver_name = "authenc-hmac-sha384-" 1708 "cbc-aes-caam-qi", 1709 .cra_blocksize = AES_BLOCK_SIZE, 1710 }, 1711 .setkey = aead_setkey, 1712 .setauthsize = aead_setauthsize, 1713 .encrypt = aead_encrypt, 1714 .decrypt = aead_decrypt, 1715 .ivsize = AES_BLOCK_SIZE, 1716 .maxauthsize = SHA384_DIGEST_SIZE, 1717 }, 1718 .caam = { 1719 .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CBC, 1720 .class2_alg_type = OP_ALG_ALGSEL_SHA384 | 1721 OP_ALG_AAI_HMAC_PRECOMP, 1722 } 1723 }, 1724 { 1725 .aead = { 1726 .base = { 1727 .cra_name = "echainiv(authenc(hmac(sha384)," 1728 "cbc(aes)))", 1729 .cra_driver_name = "echainiv-authenc-" 1730 "hmac-sha384-cbc-aes-" 1731 "caam-qi", 1732 .cra_blocksize = AES_BLOCK_SIZE, 1733 }, 1734 .setkey = aead_setkey, 1735 .setauthsize = aead_setauthsize, 1736 .encrypt = aead_encrypt, 1737 .decrypt = aead_decrypt, 1738 .ivsize = AES_BLOCK_SIZE, 1739 .maxauthsize = SHA384_DIGEST_SIZE, 1740 }, 1741 .caam = { 1742 .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CBC, 1743 .class2_alg_type = OP_ALG_ALGSEL_SHA384 | 1744 OP_ALG_AAI_HMAC_PRECOMP, 1745 .geniv = true, 1746 } 1747 }, 1748 { 1749 .aead = { 1750 .base = { 1751 .cra_name = "authenc(hmac(sha512),cbc(aes))", 1752 .cra_driver_name = "authenc-hmac-sha512-" 1753 "cbc-aes-caam-qi", 1754 .cra_blocksize = AES_BLOCK_SIZE, 1755 }, 1756 .setkey = aead_setkey, 1757 .setauthsize = aead_setauthsize, 1758 .encrypt = aead_encrypt, 1759 .decrypt = aead_decrypt, 1760 .ivsize = AES_BLOCK_SIZE, 1761 .maxauthsize = SHA512_DIGEST_SIZE, 1762 }, 1763 .caam = { 1764 .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CBC, 1765 .class2_alg_type = OP_ALG_ALGSEL_SHA512 | 1766 OP_ALG_AAI_HMAC_PRECOMP, 1767 } 1768 }, 1769 { 1770 .aead = { 1771 .base = { 1772 .cra_name = "echainiv(authenc(hmac(sha512)," 1773 "cbc(aes)))", 1774 .cra_driver_name = "echainiv-authenc-" 1775 "hmac-sha512-cbc-aes-" 1776 "caam-qi", 1777 .cra_blocksize = AES_BLOCK_SIZE, 1778 }, 1779 .setkey = aead_setkey, 1780 .setauthsize = aead_setauthsize, 1781 .encrypt = aead_encrypt, 1782 .decrypt = aead_decrypt, 1783 .ivsize = AES_BLOCK_SIZE, 1784 .maxauthsize = SHA512_DIGEST_SIZE, 1785 }, 1786 .caam = { 1787 .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CBC, 1788 .class2_alg_type = OP_ALG_ALGSEL_SHA512 | 1789 OP_ALG_AAI_HMAC_PRECOMP, 1790 .geniv = true, 1791 } 1792 }, 1793 { 1794 .aead = { 1795 .base = { 1796 .cra_name = "authenc(hmac(md5),cbc(des3_ede))", 1797 .cra_driver_name = "authenc-hmac-md5-" 1798 "cbc-des3_ede-caam-qi", 1799 .cra_blocksize = DES3_EDE_BLOCK_SIZE, 1800 }, 1801 .setkey = aead_setkey, 1802 .setauthsize = aead_setauthsize, 1803 .encrypt = aead_encrypt, 1804 .decrypt = aead_decrypt, 1805 .ivsize = DES3_EDE_BLOCK_SIZE, 1806 .maxauthsize = MD5_DIGEST_SIZE, 1807 }, 1808 .caam = { 1809 .class1_alg_type = OP_ALG_ALGSEL_3DES | OP_ALG_AAI_CBC, 1810 .class2_alg_type = OP_ALG_ALGSEL_MD5 | 1811 OP_ALG_AAI_HMAC_PRECOMP, 1812 } 1813 }, 1814 { 1815 .aead = { 1816 .base = { 1817 .cra_name = "echainiv(authenc(hmac(md5)," 1818 "cbc(des3_ede)))", 1819 .cra_driver_name = "echainiv-authenc-hmac-md5-" 1820 "cbc-des3_ede-caam-qi", 1821 .cra_blocksize = DES3_EDE_BLOCK_SIZE, 1822 }, 1823 .setkey = aead_setkey, 1824 .setauthsize = aead_setauthsize, 1825 .encrypt = aead_encrypt, 1826 .decrypt = aead_decrypt, 1827 .ivsize = DES3_EDE_BLOCK_SIZE, 1828 .maxauthsize = MD5_DIGEST_SIZE, 1829 }, 1830 .caam = { 1831 .class1_alg_type = OP_ALG_ALGSEL_3DES | OP_ALG_AAI_CBC, 1832 .class2_alg_type = OP_ALG_ALGSEL_MD5 | 1833 OP_ALG_AAI_HMAC_PRECOMP, 1834 .geniv = true, 1835 } 1836 }, 1837 { 1838 .aead = { 1839 .base = { 1840 .cra_name = "authenc(hmac(sha1)," 1841 "cbc(des3_ede))", 1842 .cra_driver_name = "authenc-hmac-sha1-" 1843 "cbc-des3_ede-caam-qi", 1844 .cra_blocksize = DES3_EDE_BLOCK_SIZE, 1845 }, 1846 .setkey = aead_setkey, 1847 .setauthsize = aead_setauthsize, 1848 .encrypt = aead_encrypt, 1849 .decrypt = aead_decrypt, 1850 .ivsize = DES3_EDE_BLOCK_SIZE, 1851 .maxauthsize = SHA1_DIGEST_SIZE, 1852 }, 1853 .caam = { 1854 .class1_alg_type = OP_ALG_ALGSEL_3DES | OP_ALG_AAI_CBC, 1855 .class2_alg_type = OP_ALG_ALGSEL_SHA1 | 1856 OP_ALG_AAI_HMAC_PRECOMP, 1857 }, 1858 }, 1859 { 1860 .aead = { 1861 .base = { 1862 .cra_name = "echainiv(authenc(hmac(sha1)," 1863 "cbc(des3_ede)))", 1864 .cra_driver_name = "echainiv-authenc-" 1865 "hmac-sha1-" 1866 "cbc-des3_ede-caam-qi", 1867 .cra_blocksize = DES3_EDE_BLOCK_SIZE, 1868 }, 1869 .setkey = aead_setkey, 1870 .setauthsize = aead_setauthsize, 1871 .encrypt = aead_encrypt, 1872 .decrypt = aead_decrypt, 1873 .ivsize = DES3_EDE_BLOCK_SIZE, 1874 .maxauthsize = SHA1_DIGEST_SIZE, 1875 }, 1876 .caam = { 1877 .class1_alg_type = OP_ALG_ALGSEL_3DES | OP_ALG_AAI_CBC, 1878 .class2_alg_type = OP_ALG_ALGSEL_SHA1 | 1879 OP_ALG_AAI_HMAC_PRECOMP, 1880 .geniv = true, 1881 } 1882 }, 1883 { 1884 .aead = { 1885 .base = { 1886 .cra_name = "authenc(hmac(sha224)," 1887 "cbc(des3_ede))", 1888 .cra_driver_name = "authenc-hmac-sha224-" 1889 "cbc-des3_ede-caam-qi", 1890 .cra_blocksize = DES3_EDE_BLOCK_SIZE, 1891 }, 1892 .setkey = aead_setkey, 1893 .setauthsize = aead_setauthsize, 1894 .encrypt = aead_encrypt, 1895 .decrypt = aead_decrypt, 1896 .ivsize = DES3_EDE_BLOCK_SIZE, 1897 .maxauthsize = SHA224_DIGEST_SIZE, 1898 }, 1899 .caam = { 1900 .class1_alg_type = OP_ALG_ALGSEL_3DES | OP_ALG_AAI_CBC, 1901 .class2_alg_type = OP_ALG_ALGSEL_SHA224 | 1902 OP_ALG_AAI_HMAC_PRECOMP, 1903 }, 1904 }, 1905 { 1906 .aead = { 1907 .base = { 1908 .cra_name = "echainiv(authenc(hmac(sha224)," 1909 "cbc(des3_ede)))", 1910 .cra_driver_name = "echainiv-authenc-" 1911 "hmac-sha224-" 1912 "cbc-des3_ede-caam-qi", 1913 .cra_blocksize = DES3_EDE_BLOCK_SIZE, 1914 }, 1915 .setkey = aead_setkey, 1916 .setauthsize = aead_setauthsize, 1917 .encrypt = aead_encrypt, 1918 .decrypt = aead_decrypt, 1919 .ivsize = DES3_EDE_BLOCK_SIZE, 1920 .maxauthsize = SHA224_DIGEST_SIZE, 1921 }, 1922 .caam = { 1923 .class1_alg_type = OP_ALG_ALGSEL_3DES | OP_ALG_AAI_CBC, 1924 .class2_alg_type = OP_ALG_ALGSEL_SHA224 | 1925 OP_ALG_AAI_HMAC_PRECOMP, 1926 .geniv = true, 1927 } 1928 }, 1929 { 1930 .aead = { 1931 .base = { 1932 .cra_name = "authenc(hmac(sha256)," 1933 "cbc(des3_ede))", 1934 .cra_driver_name = "authenc-hmac-sha256-" 1935 "cbc-des3_ede-caam-qi", 1936 .cra_blocksize = DES3_EDE_BLOCK_SIZE, 1937 }, 1938 .setkey = aead_setkey, 1939 .setauthsize = aead_setauthsize, 1940 .encrypt = aead_encrypt, 1941 .decrypt = aead_decrypt, 1942 .ivsize = DES3_EDE_BLOCK_SIZE, 1943 .maxauthsize = SHA256_DIGEST_SIZE, 1944 }, 1945 .caam = { 1946 .class1_alg_type = OP_ALG_ALGSEL_3DES | OP_ALG_AAI_CBC, 1947 .class2_alg_type = OP_ALG_ALGSEL_SHA256 | 1948 OP_ALG_AAI_HMAC_PRECOMP, 1949 }, 1950 }, 1951 { 1952 .aead = { 1953 .base = { 1954 .cra_name = "echainiv(authenc(hmac(sha256)," 1955 "cbc(des3_ede)))", 1956 .cra_driver_name = "echainiv-authenc-" 1957 "hmac-sha256-" 1958 "cbc-des3_ede-caam-qi", 1959 .cra_blocksize = DES3_EDE_BLOCK_SIZE, 1960 }, 1961 .setkey = aead_setkey, 1962 .setauthsize = aead_setauthsize, 1963 .encrypt = aead_encrypt, 1964 .decrypt = aead_decrypt, 1965 .ivsize = DES3_EDE_BLOCK_SIZE, 1966 .maxauthsize = SHA256_DIGEST_SIZE, 1967 }, 1968 .caam = { 1969 .class1_alg_type = OP_ALG_ALGSEL_3DES | OP_ALG_AAI_CBC, 1970 .class2_alg_type = OP_ALG_ALGSEL_SHA256 | 1971 OP_ALG_AAI_HMAC_PRECOMP, 1972 .geniv = true, 1973 } 1974 }, 1975 { 1976 .aead = { 1977 .base = { 1978 .cra_name = "authenc(hmac(sha384)," 1979 "cbc(des3_ede))", 1980 .cra_driver_name = "authenc-hmac-sha384-" 1981 "cbc-des3_ede-caam-qi", 1982 .cra_blocksize = DES3_EDE_BLOCK_SIZE, 1983 }, 1984 .setkey = aead_setkey, 1985 .setauthsize = aead_setauthsize, 1986 .encrypt = aead_encrypt, 1987 .decrypt = aead_decrypt, 1988 .ivsize = DES3_EDE_BLOCK_SIZE, 1989 .maxauthsize = SHA384_DIGEST_SIZE, 1990 }, 1991 .caam = { 1992 .class1_alg_type = OP_ALG_ALGSEL_3DES | OP_ALG_AAI_CBC, 1993 .class2_alg_type = OP_ALG_ALGSEL_SHA384 | 1994 OP_ALG_AAI_HMAC_PRECOMP, 1995 }, 1996 }, 1997 { 1998 .aead = { 1999 .base = { 2000 .cra_name = "echainiv(authenc(hmac(sha384)," 2001 "cbc(des3_ede)))", 2002 .cra_driver_name = "echainiv-authenc-" 2003 "hmac-sha384-" 2004 "cbc-des3_ede-caam-qi", 2005 .cra_blocksize = DES3_EDE_BLOCK_SIZE, 2006 }, 2007 .setkey = aead_setkey, 2008 .setauthsize = aead_setauthsize, 2009 .encrypt = aead_encrypt, 2010 .decrypt = aead_decrypt, 2011 .ivsize = DES3_EDE_BLOCK_SIZE, 2012 .maxauthsize = SHA384_DIGEST_SIZE, 2013 }, 2014 .caam = { 2015 .class1_alg_type = OP_ALG_ALGSEL_3DES | OP_ALG_AAI_CBC, 2016 .class2_alg_type = OP_ALG_ALGSEL_SHA384 | 2017 OP_ALG_AAI_HMAC_PRECOMP, 2018 .geniv = true, 2019 } 2020 }, 2021 { 2022 .aead = { 2023 .base = { 2024 .cra_name = "authenc(hmac(sha512)," 2025 "cbc(des3_ede))", 2026 .cra_driver_name = "authenc-hmac-sha512-" 2027 "cbc-des3_ede-caam-qi", 2028 .cra_blocksize = DES3_EDE_BLOCK_SIZE, 2029 }, 2030 .setkey = aead_setkey, 2031 .setauthsize = aead_setauthsize, 2032 .encrypt = aead_encrypt, 2033 .decrypt = aead_decrypt, 2034 .ivsize = DES3_EDE_BLOCK_SIZE, 2035 .maxauthsize = SHA512_DIGEST_SIZE, 2036 }, 2037 .caam = { 2038 .class1_alg_type = OP_ALG_ALGSEL_3DES | OP_ALG_AAI_CBC, 2039 .class2_alg_type = OP_ALG_ALGSEL_SHA512 | 2040 OP_ALG_AAI_HMAC_PRECOMP, 2041 }, 2042 }, 2043 { 2044 .aead = { 2045 .base = { 2046 .cra_name = "echainiv(authenc(hmac(sha512)," 2047 "cbc(des3_ede)))", 2048 .cra_driver_name = "echainiv-authenc-" 2049 "hmac-sha512-" 2050 "cbc-des3_ede-caam-qi", 2051 .cra_blocksize = DES3_EDE_BLOCK_SIZE, 2052 }, 2053 .setkey = aead_setkey, 2054 .setauthsize = aead_setauthsize, 2055 .encrypt = aead_encrypt, 2056 .decrypt = aead_decrypt, 2057 .ivsize = DES3_EDE_BLOCK_SIZE, 2058 .maxauthsize = SHA512_DIGEST_SIZE, 2059 }, 2060 .caam = { 2061 .class1_alg_type = OP_ALG_ALGSEL_3DES | OP_ALG_AAI_CBC, 2062 .class2_alg_type = OP_ALG_ALGSEL_SHA512 | 2063 OP_ALG_AAI_HMAC_PRECOMP, 2064 .geniv = true, 2065 } 2066 }, 2067 { 2068 .aead = { 2069 .base = { 2070 .cra_name = "authenc(hmac(md5),cbc(des))", 2071 .cra_driver_name = "authenc-hmac-md5-" 2072 "cbc-des-caam-qi", 2073 .cra_blocksize = DES_BLOCK_SIZE, 2074 }, 2075 .setkey = aead_setkey, 2076 .setauthsize = aead_setauthsize, 2077 .encrypt = aead_encrypt, 2078 .decrypt = aead_decrypt, 2079 .ivsize = DES_BLOCK_SIZE, 2080 .maxauthsize = MD5_DIGEST_SIZE, 2081 }, 2082 .caam = { 2083 .class1_alg_type = OP_ALG_ALGSEL_DES | OP_ALG_AAI_CBC, 2084 .class2_alg_type = OP_ALG_ALGSEL_MD5 | 2085 OP_ALG_AAI_HMAC_PRECOMP, 2086 }, 2087 }, 2088 { 2089 .aead = { 2090 .base = { 2091 .cra_name = "echainiv(authenc(hmac(md5)," 2092 "cbc(des)))", 2093 .cra_driver_name = "echainiv-authenc-hmac-md5-" 2094 "cbc-des-caam-qi", 2095 .cra_blocksize = DES_BLOCK_SIZE, 2096 }, 2097 .setkey = aead_setkey, 2098 .setauthsize = aead_setauthsize, 2099 .encrypt = aead_encrypt, 2100 .decrypt = aead_decrypt, 2101 .ivsize = DES_BLOCK_SIZE, 2102 .maxauthsize = MD5_DIGEST_SIZE, 2103 }, 2104 .caam = { 2105 .class1_alg_type = OP_ALG_ALGSEL_DES | OP_ALG_AAI_CBC, 2106 .class2_alg_type = OP_ALG_ALGSEL_MD5 | 2107 OP_ALG_AAI_HMAC_PRECOMP, 2108 .geniv = true, 2109 } 2110 }, 2111 { 2112 .aead = { 2113 .base = { 2114 .cra_name = "authenc(hmac(sha1),cbc(des))", 2115 .cra_driver_name = "authenc-hmac-sha1-" 2116 "cbc-des-caam-qi", 2117 .cra_blocksize = DES_BLOCK_SIZE, 2118 }, 2119 .setkey = aead_setkey, 2120 .setauthsize = aead_setauthsize, 2121 .encrypt = aead_encrypt, 2122 .decrypt = aead_decrypt, 2123 .ivsize = DES_BLOCK_SIZE, 2124 .maxauthsize = SHA1_DIGEST_SIZE, 2125 }, 2126 .caam = { 2127 .class1_alg_type = OP_ALG_ALGSEL_DES | OP_ALG_AAI_CBC, 2128 .class2_alg_type = OP_ALG_ALGSEL_SHA1 | 2129 OP_ALG_AAI_HMAC_PRECOMP, 2130 }, 2131 }, 2132 { 2133 .aead = { 2134 .base = { 2135 .cra_name = "echainiv(authenc(hmac(sha1)," 2136 "cbc(des)))", 2137 .cra_driver_name = "echainiv-authenc-" 2138 "hmac-sha1-cbc-des-caam-qi", 2139 .cra_blocksize = DES_BLOCK_SIZE, 2140 }, 2141 .setkey = aead_setkey, 2142 .setauthsize = aead_setauthsize, 2143 .encrypt = aead_encrypt, 2144 .decrypt = aead_decrypt, 2145 .ivsize = DES_BLOCK_SIZE, 2146 .maxauthsize = SHA1_DIGEST_SIZE, 2147 }, 2148 .caam = { 2149 .class1_alg_type = OP_ALG_ALGSEL_DES | OP_ALG_AAI_CBC, 2150 .class2_alg_type = OP_ALG_ALGSEL_SHA1 | 2151 OP_ALG_AAI_HMAC_PRECOMP, 2152 .geniv = true, 2153 } 2154 }, 2155 { 2156 .aead = { 2157 .base = { 2158 .cra_name = "authenc(hmac(sha224),cbc(des))", 2159 .cra_driver_name = "authenc-hmac-sha224-" 2160 "cbc-des-caam-qi", 2161 .cra_blocksize = DES_BLOCK_SIZE, 2162 }, 2163 .setkey = aead_setkey, 2164 .setauthsize = aead_setauthsize, 2165 .encrypt = aead_encrypt, 2166 .decrypt = aead_decrypt, 2167 .ivsize = DES_BLOCK_SIZE, 2168 .maxauthsize = SHA224_DIGEST_SIZE, 2169 }, 2170 .caam = { 2171 .class1_alg_type = OP_ALG_ALGSEL_DES | OP_ALG_AAI_CBC, 2172 .class2_alg_type = OP_ALG_ALGSEL_SHA224 | 2173 OP_ALG_AAI_HMAC_PRECOMP, 2174 }, 2175 }, 2176 { 2177 .aead = { 2178 .base = { 2179 .cra_name = "echainiv(authenc(hmac(sha224)," 2180 "cbc(des)))", 2181 .cra_driver_name = "echainiv-authenc-" 2182 "hmac-sha224-cbc-des-" 2183 "caam-qi", 2184 .cra_blocksize = DES_BLOCK_SIZE, 2185 }, 2186 .setkey = aead_setkey, 2187 .setauthsize = aead_setauthsize, 2188 .encrypt = aead_encrypt, 2189 .decrypt = aead_decrypt, 2190 .ivsize = DES_BLOCK_SIZE, 2191 .maxauthsize = SHA224_DIGEST_SIZE, 2192 }, 2193 .caam = { 2194 .class1_alg_type = OP_ALG_ALGSEL_DES | OP_ALG_AAI_CBC, 2195 .class2_alg_type = OP_ALG_ALGSEL_SHA224 | 2196 OP_ALG_AAI_HMAC_PRECOMP, 2197 .geniv = true, 2198 } 2199 }, 2200 { 2201 .aead = { 2202 .base = { 2203 .cra_name = "authenc(hmac(sha256),cbc(des))", 2204 .cra_driver_name = "authenc-hmac-sha256-" 2205 "cbc-des-caam-qi", 2206 .cra_blocksize = DES_BLOCK_SIZE, 2207 }, 2208 .setkey = aead_setkey, 2209 .setauthsize = aead_setauthsize, 2210 .encrypt = aead_encrypt, 2211 .decrypt = aead_decrypt, 2212 .ivsize = DES_BLOCK_SIZE, 2213 .maxauthsize = SHA256_DIGEST_SIZE, 2214 }, 2215 .caam = { 2216 .class1_alg_type = OP_ALG_ALGSEL_DES | OP_ALG_AAI_CBC, 2217 .class2_alg_type = OP_ALG_ALGSEL_SHA256 | 2218 OP_ALG_AAI_HMAC_PRECOMP, 2219 }, 2220 }, 2221 { 2222 .aead = { 2223 .base = { 2224 .cra_name = "echainiv(authenc(hmac(sha256)," 2225 "cbc(des)))", 2226 .cra_driver_name = "echainiv-authenc-" 2227 "hmac-sha256-cbc-des-" 2228 "caam-qi", 2229 .cra_blocksize = DES_BLOCK_SIZE, 2230 }, 2231 .setkey = aead_setkey, 2232 .setauthsize = aead_setauthsize, 2233 .encrypt = aead_encrypt, 2234 .decrypt = aead_decrypt, 2235 .ivsize = DES_BLOCK_SIZE, 2236 .maxauthsize = SHA256_DIGEST_SIZE, 2237 }, 2238 .caam = { 2239 .class1_alg_type = OP_ALG_ALGSEL_DES | OP_ALG_AAI_CBC, 2240 .class2_alg_type = OP_ALG_ALGSEL_SHA256 | 2241 OP_ALG_AAI_HMAC_PRECOMP, 2242 .geniv = true, 2243 }, 2244 }, 2245 { 2246 .aead = { 2247 .base = { 2248 .cra_name = "authenc(hmac(sha384),cbc(des))", 2249 .cra_driver_name = "authenc-hmac-sha384-" 2250 "cbc-des-caam-qi", 2251 .cra_blocksize = DES_BLOCK_SIZE, 2252 }, 2253 .setkey = aead_setkey, 2254 .setauthsize = aead_setauthsize, 2255 .encrypt = aead_encrypt, 2256 .decrypt = aead_decrypt, 2257 .ivsize = DES_BLOCK_SIZE, 2258 .maxauthsize = SHA384_DIGEST_SIZE, 2259 }, 2260 .caam = { 2261 .class1_alg_type = OP_ALG_ALGSEL_DES | OP_ALG_AAI_CBC, 2262 .class2_alg_type = OP_ALG_ALGSEL_SHA384 | 2263 OP_ALG_AAI_HMAC_PRECOMP, 2264 }, 2265 }, 2266 { 2267 .aead = { 2268 .base = { 2269 .cra_name = "echainiv(authenc(hmac(sha384)," 2270 "cbc(des)))", 2271 .cra_driver_name = "echainiv-authenc-" 2272 "hmac-sha384-cbc-des-" 2273 "caam-qi", 2274 .cra_blocksize = DES_BLOCK_SIZE, 2275 }, 2276 .setkey = aead_setkey, 2277 .setauthsize = aead_setauthsize, 2278 .encrypt = aead_encrypt, 2279 .decrypt = aead_decrypt, 2280 .ivsize = DES_BLOCK_SIZE, 2281 .maxauthsize = SHA384_DIGEST_SIZE, 2282 }, 2283 .caam = { 2284 .class1_alg_type = OP_ALG_ALGSEL_DES | OP_ALG_AAI_CBC, 2285 .class2_alg_type = OP_ALG_ALGSEL_SHA384 | 2286 OP_ALG_AAI_HMAC_PRECOMP, 2287 .geniv = true, 2288 } 2289 }, 2290 { 2291 .aead = { 2292 .base = { 2293 .cra_name = "authenc(hmac(sha512),cbc(des))", 2294 .cra_driver_name = "authenc-hmac-sha512-" 2295 "cbc-des-caam-qi", 2296 .cra_blocksize = DES_BLOCK_SIZE, 2297 }, 2298 .setkey = aead_setkey, 2299 .setauthsize = aead_setauthsize, 2300 .encrypt = aead_encrypt, 2301 .decrypt = aead_decrypt, 2302 .ivsize = DES_BLOCK_SIZE, 2303 .maxauthsize = SHA512_DIGEST_SIZE, 2304 }, 2305 .caam = { 2306 .class1_alg_type = OP_ALG_ALGSEL_DES | OP_ALG_AAI_CBC, 2307 .class2_alg_type = OP_ALG_ALGSEL_SHA512 | 2308 OP_ALG_AAI_HMAC_PRECOMP, 2309 } 2310 }, 2311 { 2312 .aead = { 2313 .base = { 2314 .cra_name = "echainiv(authenc(hmac(sha512)," 2315 "cbc(des)))", 2316 .cra_driver_name = "echainiv-authenc-" 2317 "hmac-sha512-cbc-des-" 2318 "caam-qi", 2319 .cra_blocksize = DES_BLOCK_SIZE, 2320 }, 2321 .setkey = aead_setkey, 2322 .setauthsize = aead_setauthsize, 2323 .encrypt = aead_encrypt, 2324 .decrypt = aead_decrypt, 2325 .ivsize = DES_BLOCK_SIZE, 2326 .maxauthsize = SHA512_DIGEST_SIZE, 2327 }, 2328 .caam = { 2329 .class1_alg_type = OP_ALG_ALGSEL_DES | OP_ALG_AAI_CBC, 2330 .class2_alg_type = OP_ALG_ALGSEL_SHA512 | 2331 OP_ALG_AAI_HMAC_PRECOMP, 2332 .geniv = true, 2333 } 2334 }, 2335 }; 2336 2337 static int caam_init_common(struct caam_ctx *ctx, struct caam_alg_entry *caam, 2338 bool uses_dkp) 2339 { 2340 struct caam_drv_private *priv; 2341 2342 /* 2343 * distribute tfms across job rings to ensure in-order 2344 * crypto request processing per tfm 2345 */ 2346 ctx->jrdev = caam_jr_alloc(); 2347 if (IS_ERR(ctx->jrdev)) { 2348 pr_err("Job Ring Device allocation for transform failed\n"); 2349 return PTR_ERR(ctx->jrdev); 2350 } 2351 2352 priv = dev_get_drvdata(ctx->jrdev->parent); 2353 if (priv->era >= 6 && uses_dkp) 2354 ctx->dir = DMA_BIDIRECTIONAL; 2355 else 2356 ctx->dir = DMA_TO_DEVICE; 2357 2358 ctx->key_dma = dma_map_single(ctx->jrdev, ctx->key, sizeof(ctx->key), 2359 ctx->dir); 2360 if (dma_mapping_error(ctx->jrdev, ctx->key_dma)) { 2361 dev_err(ctx->jrdev, "unable to map key\n"); 2362 caam_jr_free(ctx->jrdev); 2363 return -ENOMEM; 2364 } 2365 2366 /* copy descriptor header template value */ 2367 ctx->cdata.algtype = OP_TYPE_CLASS1_ALG | caam->class1_alg_type; 2368 ctx->adata.algtype = OP_TYPE_CLASS2_ALG | caam->class2_alg_type; 2369 2370 ctx->qidev = priv->qidev; 2371 2372 spin_lock_init(&ctx->lock); 2373 ctx->drv_ctx[ENCRYPT] = NULL; 2374 ctx->drv_ctx[DECRYPT] = NULL; 2375 2376 return 0; 2377 } 2378 2379 static int caam_cra_init(struct crypto_skcipher *tfm) 2380 { 2381 struct skcipher_alg *alg = crypto_skcipher_alg(tfm); 2382 struct caam_skcipher_alg *caam_alg = 2383 container_of(alg, typeof(*caam_alg), skcipher); 2384 2385 return caam_init_common(crypto_skcipher_ctx(tfm), &caam_alg->caam, 2386 false); 2387 } 2388 2389 static int caam_aead_init(struct crypto_aead *tfm) 2390 { 2391 struct aead_alg *alg = crypto_aead_alg(tfm); 2392 struct caam_aead_alg *caam_alg = container_of(alg, typeof(*caam_alg), 2393 aead); 2394 struct caam_ctx *ctx = crypto_aead_ctx(tfm); 2395 2396 return caam_init_common(ctx, &caam_alg->caam, 2397 alg->setkey == aead_setkey); 2398 } 2399 2400 static void caam_exit_common(struct caam_ctx *ctx) 2401 { 2402 caam_drv_ctx_rel(ctx->drv_ctx[ENCRYPT]); 2403 caam_drv_ctx_rel(ctx->drv_ctx[DECRYPT]); 2404 2405 dma_unmap_single(ctx->jrdev, ctx->key_dma, sizeof(ctx->key), ctx->dir); 2406 2407 caam_jr_free(ctx->jrdev); 2408 } 2409 2410 static void caam_cra_exit(struct crypto_skcipher *tfm) 2411 { 2412 caam_exit_common(crypto_skcipher_ctx(tfm)); 2413 } 2414 2415 static void caam_aead_exit(struct crypto_aead *tfm) 2416 { 2417 caam_exit_common(crypto_aead_ctx(tfm)); 2418 } 2419 2420 static void __exit caam_qi_algapi_exit(void) 2421 { 2422 int i; 2423 2424 for (i = 0; i < ARRAY_SIZE(driver_aeads); i++) { 2425 struct caam_aead_alg *t_alg = driver_aeads + i; 2426 2427 if (t_alg->registered) 2428 crypto_unregister_aead(&t_alg->aead); 2429 } 2430 2431 for (i = 0; i < ARRAY_SIZE(driver_algs); i++) { 2432 struct caam_skcipher_alg *t_alg = driver_algs + i; 2433 2434 if (t_alg->registered) 2435 crypto_unregister_skcipher(&t_alg->skcipher); 2436 } 2437 } 2438 2439 static void caam_skcipher_alg_init(struct caam_skcipher_alg *t_alg) 2440 { 2441 struct skcipher_alg *alg = &t_alg->skcipher; 2442 2443 alg->base.cra_module = THIS_MODULE; 2444 alg->base.cra_priority = CAAM_CRA_PRIORITY; 2445 alg->base.cra_ctxsize = sizeof(struct caam_ctx); 2446 alg->base.cra_flags = CRYPTO_ALG_ASYNC | CRYPTO_ALG_KERN_DRIVER_ONLY; 2447 2448 alg->init = caam_cra_init; 2449 alg->exit = caam_cra_exit; 2450 } 2451 2452 static void caam_aead_alg_init(struct caam_aead_alg *t_alg) 2453 { 2454 struct aead_alg *alg = &t_alg->aead; 2455 2456 alg->base.cra_module = THIS_MODULE; 2457 alg->base.cra_priority = CAAM_CRA_PRIORITY; 2458 alg->base.cra_ctxsize = sizeof(struct caam_ctx); 2459 alg->base.cra_flags = CRYPTO_ALG_ASYNC | CRYPTO_ALG_KERN_DRIVER_ONLY; 2460 2461 alg->init = caam_aead_init; 2462 alg->exit = caam_aead_exit; 2463 } 2464 2465 static int __init caam_qi_algapi_init(void) 2466 { 2467 struct device_node *dev_node; 2468 struct platform_device *pdev; 2469 struct device *ctrldev; 2470 struct caam_drv_private *priv; 2471 int i = 0, err = 0; 2472 u32 aes_vid, aes_inst, des_inst, md_vid, md_inst; 2473 unsigned int md_limit = SHA512_DIGEST_SIZE; 2474 bool registered = false; 2475 2476 dev_node = of_find_compatible_node(NULL, NULL, "fsl,sec-v4.0"); 2477 if (!dev_node) { 2478 dev_node = of_find_compatible_node(NULL, NULL, "fsl,sec4.0"); 2479 if (!dev_node) 2480 return -ENODEV; 2481 } 2482 2483 pdev = of_find_device_by_node(dev_node); 2484 of_node_put(dev_node); 2485 if (!pdev) 2486 return -ENODEV; 2487 2488 ctrldev = &pdev->dev; 2489 priv = dev_get_drvdata(ctrldev); 2490 2491 /* 2492 * If priv is NULL, it's probably because the caam driver wasn't 2493 * properly initialized (e.g. RNG4 init failed). Thus, bail out here. 2494 */ 2495 if (!priv || !priv->qi_present) { 2496 err = -ENODEV; 2497 goto out_put_dev; 2498 } 2499 2500 if (caam_dpaa2) { 2501 dev_info(ctrldev, "caam/qi frontend driver not suitable for DPAA 2.x, aborting...\n"); 2502 err = -ENODEV; 2503 goto out_put_dev; 2504 } 2505 2506 /* 2507 * Register crypto algorithms the device supports. 2508 * First, detect presence and attributes of DES, AES, and MD blocks. 2509 */ 2510 if (priv->era < 10) { 2511 u32 cha_vid, cha_inst; 2512 2513 cha_vid = rd_reg32(&priv->ctrl->perfmon.cha_id_ls); 2514 aes_vid = cha_vid & CHA_ID_LS_AES_MASK; 2515 md_vid = (cha_vid & CHA_ID_LS_MD_MASK) >> CHA_ID_LS_MD_SHIFT; 2516 2517 cha_inst = rd_reg32(&priv->ctrl->perfmon.cha_num_ls); 2518 des_inst = (cha_inst & CHA_ID_LS_DES_MASK) >> 2519 CHA_ID_LS_DES_SHIFT; 2520 aes_inst = cha_inst & CHA_ID_LS_AES_MASK; 2521 md_inst = (cha_inst & CHA_ID_LS_MD_MASK) >> CHA_ID_LS_MD_SHIFT; 2522 } else { 2523 u32 aesa, mdha; 2524 2525 aesa = rd_reg32(&priv->ctrl->vreg.aesa); 2526 mdha = rd_reg32(&priv->ctrl->vreg.mdha); 2527 2528 aes_vid = (aesa & CHA_VER_VID_MASK) >> CHA_VER_VID_SHIFT; 2529 md_vid = (mdha & CHA_VER_VID_MASK) >> CHA_VER_VID_SHIFT; 2530 2531 des_inst = rd_reg32(&priv->ctrl->vreg.desa) & CHA_VER_NUM_MASK; 2532 aes_inst = aesa & CHA_VER_NUM_MASK; 2533 md_inst = mdha & CHA_VER_NUM_MASK; 2534 } 2535 2536 /* If MD is present, limit digest size based on LP256 */ 2537 if (md_inst && md_vid == CHA_VER_VID_MD_LP256) 2538 md_limit = SHA256_DIGEST_SIZE; 2539 2540 for (i = 0; i < ARRAY_SIZE(driver_algs); i++) { 2541 struct caam_skcipher_alg *t_alg = driver_algs + i; 2542 u32 alg_sel = t_alg->caam.class1_alg_type & OP_ALG_ALGSEL_MASK; 2543 2544 /* Skip DES algorithms if not supported by device */ 2545 if (!des_inst && 2546 ((alg_sel == OP_ALG_ALGSEL_3DES) || 2547 (alg_sel == OP_ALG_ALGSEL_DES))) 2548 continue; 2549 2550 /* Skip AES algorithms if not supported by device */ 2551 if (!aes_inst && (alg_sel == OP_ALG_ALGSEL_AES)) 2552 continue; 2553 2554 caam_skcipher_alg_init(t_alg); 2555 2556 err = crypto_register_skcipher(&t_alg->skcipher); 2557 if (err) { 2558 dev_warn(priv->qidev, "%s alg registration failed\n", 2559 t_alg->skcipher.base.cra_driver_name); 2560 continue; 2561 } 2562 2563 t_alg->registered = true; 2564 registered = true; 2565 } 2566 2567 for (i = 0; i < ARRAY_SIZE(driver_aeads); i++) { 2568 struct caam_aead_alg *t_alg = driver_aeads + i; 2569 u32 c1_alg_sel = t_alg->caam.class1_alg_type & 2570 OP_ALG_ALGSEL_MASK; 2571 u32 c2_alg_sel = t_alg->caam.class2_alg_type & 2572 OP_ALG_ALGSEL_MASK; 2573 u32 alg_aai = t_alg->caam.class1_alg_type & OP_ALG_AAI_MASK; 2574 2575 /* Skip DES algorithms if not supported by device */ 2576 if (!des_inst && 2577 ((c1_alg_sel == OP_ALG_ALGSEL_3DES) || 2578 (c1_alg_sel == OP_ALG_ALGSEL_DES))) 2579 continue; 2580 2581 /* Skip AES algorithms if not supported by device */ 2582 if (!aes_inst && (c1_alg_sel == OP_ALG_ALGSEL_AES)) 2583 continue; 2584 2585 /* 2586 * Check support for AES algorithms not available 2587 * on LP devices. 2588 */ 2589 if (aes_vid == CHA_VER_VID_AES_LP && alg_aai == OP_ALG_AAI_GCM) 2590 continue; 2591 2592 /* 2593 * Skip algorithms requiring message digests 2594 * if MD or MD size is not supported by device. 2595 */ 2596 if (c2_alg_sel && 2597 (!md_inst || (t_alg->aead.maxauthsize > md_limit))) 2598 continue; 2599 2600 caam_aead_alg_init(t_alg); 2601 2602 err = crypto_register_aead(&t_alg->aead); 2603 if (err) { 2604 pr_warn("%s alg registration failed\n", 2605 t_alg->aead.base.cra_driver_name); 2606 continue; 2607 } 2608 2609 t_alg->registered = true; 2610 registered = true; 2611 } 2612 2613 if (registered) 2614 dev_info(priv->qidev, "algorithms registered in /proc/crypto\n"); 2615 2616 out_put_dev: 2617 put_device(ctrldev); 2618 return err; 2619 } 2620 2621 module_init(caam_qi_algapi_init); 2622 module_exit(caam_qi_algapi_exit); 2623 2624 MODULE_LICENSE("GPL"); 2625 MODULE_DESCRIPTION("Support for crypto API using CAAM-QI backend"); 2626 MODULE_AUTHOR("Freescale Semiconductor"); 2627