1 // SPDX-License-Identifier: GPL-2.0+ 2 /* 3 * Freescale FSL CAAM support for crypto API over QI backend. 4 * Based on caamalg.c 5 * 6 * Copyright 2013-2016 Freescale Semiconductor, Inc. 7 * Copyright 2016-2019 NXP 8 */ 9 10 #include "compat.h" 11 #include "ctrl.h" 12 #include "regs.h" 13 #include "intern.h" 14 #include "desc_constr.h" 15 #include "error.h" 16 #include "sg_sw_qm.h" 17 #include "key_gen.h" 18 #include "qi.h" 19 #include "jr.h" 20 #include "caamalg_desc.h" 21 #include <crypto/xts.h> 22 #include <asm/unaligned.h> 23 24 /* 25 * crypto alg 26 */ 27 #define CAAM_CRA_PRIORITY 2000 28 /* max key is sum of AES_MAX_KEY_SIZE, max split key size */ 29 #define CAAM_MAX_KEY_SIZE (AES_MAX_KEY_SIZE + \ 30 SHA512_DIGEST_SIZE * 2) 31 32 #define DESC_MAX_USED_BYTES (DESC_QI_AEAD_GIVENC_LEN + \ 33 CAAM_MAX_KEY_SIZE) 34 #define DESC_MAX_USED_LEN (DESC_MAX_USED_BYTES / CAAM_CMD_SZ) 35 36 struct caam_alg_entry { 37 int class1_alg_type; 38 int class2_alg_type; 39 bool rfc3686; 40 bool geniv; 41 bool nodkp; 42 }; 43 44 struct caam_aead_alg { 45 struct aead_alg aead; 46 struct caam_alg_entry caam; 47 bool registered; 48 }; 49 50 struct caam_skcipher_alg { 51 struct skcipher_alg skcipher; 52 struct caam_alg_entry caam; 53 bool registered; 54 }; 55 56 /* 57 * per-session context 58 */ 59 struct caam_ctx { 60 struct device *jrdev; 61 u32 sh_desc_enc[DESC_MAX_USED_LEN]; 62 u32 sh_desc_dec[DESC_MAX_USED_LEN]; 63 u8 key[CAAM_MAX_KEY_SIZE]; 64 dma_addr_t key_dma; 65 enum dma_data_direction dir; 66 struct alginfo adata; 67 struct alginfo cdata; 68 unsigned int authsize; 69 struct device *qidev; 70 spinlock_t lock; /* Protects multiple init of driver context */ 71 struct caam_drv_ctx *drv_ctx[NUM_OP]; 72 bool xts_key_fallback; 73 struct crypto_skcipher *fallback; 74 }; 75 76 struct caam_skcipher_req_ctx { 77 struct skcipher_request fallback_req; 78 }; 79 80 static int aead_set_sh_desc(struct crypto_aead *aead) 81 { 82 struct caam_aead_alg *alg = container_of(crypto_aead_alg(aead), 83 typeof(*alg), aead); 84 struct caam_ctx *ctx = crypto_aead_ctx_dma(aead); 85 unsigned int ivsize = crypto_aead_ivsize(aead); 86 u32 ctx1_iv_off = 0; 87 u32 *nonce = NULL; 88 unsigned int data_len[2]; 89 u32 inl_mask; 90 const bool ctr_mode = ((ctx->cdata.algtype & OP_ALG_AAI_MASK) == 91 OP_ALG_AAI_CTR_MOD128); 92 const bool is_rfc3686 = alg->caam.rfc3686; 93 struct caam_drv_private *ctrlpriv = dev_get_drvdata(ctx->jrdev->parent); 94 95 if (!ctx->cdata.keylen || !ctx->authsize) 96 return 0; 97 98 /* 99 * AES-CTR needs to load IV in CONTEXT1 reg 100 * at an offset of 128bits (16bytes) 101 * CONTEXT1[255:128] = IV 102 */ 103 if (ctr_mode) 104 ctx1_iv_off = 16; 105 106 /* 107 * RFC3686 specific: 108 * CONTEXT1[255:128] = {NONCE, IV, COUNTER} 109 */ 110 if (is_rfc3686) { 111 ctx1_iv_off = 16 + CTR_RFC3686_NONCE_SIZE; 112 nonce = (u32 *)((void *)ctx->key + ctx->adata.keylen_pad + 113 ctx->cdata.keylen - CTR_RFC3686_NONCE_SIZE); 114 } 115 116 /* 117 * In case |user key| > |derived key|, using DKP<imm,imm> would result 118 * in invalid opcodes (last bytes of user key) in the resulting 119 * descriptor. Use DKP<ptr,imm> instead => both virtual and dma key 120 * addresses are needed. 121 */ 122 ctx->adata.key_virt = ctx->key; 123 ctx->adata.key_dma = ctx->key_dma; 124 125 ctx->cdata.key_virt = ctx->key + ctx->adata.keylen_pad; 126 ctx->cdata.key_dma = ctx->key_dma + ctx->adata.keylen_pad; 127 128 data_len[0] = ctx->adata.keylen_pad; 129 data_len[1] = ctx->cdata.keylen; 130 131 if (alg->caam.geniv) 132 goto skip_enc; 133 134 /* aead_encrypt shared descriptor */ 135 if (desc_inline_query(DESC_QI_AEAD_ENC_LEN + 136 (is_rfc3686 ? DESC_AEAD_CTR_RFC3686_LEN : 0), 137 DESC_JOB_IO_LEN, data_len, &inl_mask, 138 ARRAY_SIZE(data_len)) < 0) 139 return -EINVAL; 140 141 ctx->adata.key_inline = !!(inl_mask & 1); 142 ctx->cdata.key_inline = !!(inl_mask & 2); 143 144 cnstr_shdsc_aead_encap(ctx->sh_desc_enc, &ctx->cdata, &ctx->adata, 145 ivsize, ctx->authsize, is_rfc3686, nonce, 146 ctx1_iv_off, true, ctrlpriv->era); 147 148 skip_enc: 149 /* aead_decrypt shared descriptor */ 150 if (desc_inline_query(DESC_QI_AEAD_DEC_LEN + 151 (is_rfc3686 ? DESC_AEAD_CTR_RFC3686_LEN : 0), 152 DESC_JOB_IO_LEN, data_len, &inl_mask, 153 ARRAY_SIZE(data_len)) < 0) 154 return -EINVAL; 155 156 ctx->adata.key_inline = !!(inl_mask & 1); 157 ctx->cdata.key_inline = !!(inl_mask & 2); 158 159 cnstr_shdsc_aead_decap(ctx->sh_desc_dec, &ctx->cdata, &ctx->adata, 160 ivsize, ctx->authsize, alg->caam.geniv, 161 is_rfc3686, nonce, ctx1_iv_off, true, 162 ctrlpriv->era); 163 164 if (!alg->caam.geniv) 165 goto skip_givenc; 166 167 /* aead_givencrypt shared descriptor */ 168 if (desc_inline_query(DESC_QI_AEAD_GIVENC_LEN + 169 (is_rfc3686 ? DESC_AEAD_CTR_RFC3686_LEN : 0), 170 DESC_JOB_IO_LEN, data_len, &inl_mask, 171 ARRAY_SIZE(data_len)) < 0) 172 return -EINVAL; 173 174 ctx->adata.key_inline = !!(inl_mask & 1); 175 ctx->cdata.key_inline = !!(inl_mask & 2); 176 177 cnstr_shdsc_aead_givencap(ctx->sh_desc_enc, &ctx->cdata, &ctx->adata, 178 ivsize, ctx->authsize, is_rfc3686, nonce, 179 ctx1_iv_off, true, ctrlpriv->era); 180 181 skip_givenc: 182 return 0; 183 } 184 185 static int aead_setauthsize(struct crypto_aead *authenc, unsigned int authsize) 186 { 187 struct caam_ctx *ctx = crypto_aead_ctx_dma(authenc); 188 189 ctx->authsize = authsize; 190 aead_set_sh_desc(authenc); 191 192 return 0; 193 } 194 195 static int aead_setkey(struct crypto_aead *aead, const u8 *key, 196 unsigned int keylen) 197 { 198 struct caam_ctx *ctx = crypto_aead_ctx_dma(aead); 199 struct device *jrdev = ctx->jrdev; 200 struct caam_drv_private *ctrlpriv = dev_get_drvdata(jrdev->parent); 201 struct crypto_authenc_keys keys; 202 int ret = 0; 203 204 if (crypto_authenc_extractkeys(&keys, key, keylen) != 0) 205 goto badkey; 206 207 dev_dbg(jrdev, "keylen %d enckeylen %d authkeylen %d\n", 208 keys.authkeylen + keys.enckeylen, keys.enckeylen, 209 keys.authkeylen); 210 print_hex_dump_debug("key in @" __stringify(__LINE__)": ", 211 DUMP_PREFIX_ADDRESS, 16, 4, key, keylen, 1); 212 213 /* 214 * If DKP is supported, use it in the shared descriptor to generate 215 * the split key. 216 */ 217 if (ctrlpriv->era >= 6) { 218 ctx->adata.keylen = keys.authkeylen; 219 ctx->adata.keylen_pad = split_key_len(ctx->adata.algtype & 220 OP_ALG_ALGSEL_MASK); 221 222 if (ctx->adata.keylen_pad + keys.enckeylen > CAAM_MAX_KEY_SIZE) 223 goto badkey; 224 225 memcpy(ctx->key, keys.authkey, keys.authkeylen); 226 memcpy(ctx->key + ctx->adata.keylen_pad, keys.enckey, 227 keys.enckeylen); 228 dma_sync_single_for_device(jrdev->parent, ctx->key_dma, 229 ctx->adata.keylen_pad + 230 keys.enckeylen, ctx->dir); 231 goto skip_split_key; 232 } 233 234 ret = gen_split_key(jrdev, ctx->key, &ctx->adata, keys.authkey, 235 keys.authkeylen, CAAM_MAX_KEY_SIZE - 236 keys.enckeylen); 237 if (ret) 238 goto badkey; 239 240 /* postpend encryption key to auth split key */ 241 memcpy(ctx->key + ctx->adata.keylen_pad, keys.enckey, keys.enckeylen); 242 dma_sync_single_for_device(jrdev->parent, ctx->key_dma, 243 ctx->adata.keylen_pad + keys.enckeylen, 244 ctx->dir); 245 246 print_hex_dump_debug("ctx.key@" __stringify(__LINE__)": ", 247 DUMP_PREFIX_ADDRESS, 16, 4, ctx->key, 248 ctx->adata.keylen_pad + keys.enckeylen, 1); 249 250 skip_split_key: 251 ctx->cdata.keylen = keys.enckeylen; 252 253 ret = aead_set_sh_desc(aead); 254 if (ret) 255 goto badkey; 256 257 /* Now update the driver contexts with the new shared descriptor */ 258 if (ctx->drv_ctx[ENCRYPT]) { 259 ret = caam_drv_ctx_update(ctx->drv_ctx[ENCRYPT], 260 ctx->sh_desc_enc); 261 if (ret) { 262 dev_err(jrdev, "driver enc context update failed\n"); 263 goto badkey; 264 } 265 } 266 267 if (ctx->drv_ctx[DECRYPT]) { 268 ret = caam_drv_ctx_update(ctx->drv_ctx[DECRYPT], 269 ctx->sh_desc_dec); 270 if (ret) { 271 dev_err(jrdev, "driver dec context update failed\n"); 272 goto badkey; 273 } 274 } 275 276 memzero_explicit(&keys, sizeof(keys)); 277 return ret; 278 badkey: 279 memzero_explicit(&keys, sizeof(keys)); 280 return -EINVAL; 281 } 282 283 static int des3_aead_setkey(struct crypto_aead *aead, const u8 *key, 284 unsigned int keylen) 285 { 286 struct crypto_authenc_keys keys; 287 int err; 288 289 err = crypto_authenc_extractkeys(&keys, key, keylen); 290 if (unlikely(err)) 291 return err; 292 293 err = verify_aead_des3_key(aead, keys.enckey, keys.enckeylen) ?: 294 aead_setkey(aead, key, keylen); 295 296 memzero_explicit(&keys, sizeof(keys)); 297 return err; 298 } 299 300 static int gcm_set_sh_desc(struct crypto_aead *aead) 301 { 302 struct caam_ctx *ctx = crypto_aead_ctx_dma(aead); 303 unsigned int ivsize = crypto_aead_ivsize(aead); 304 int rem_bytes = CAAM_DESC_BYTES_MAX - DESC_JOB_IO_LEN - 305 ctx->cdata.keylen; 306 307 if (!ctx->cdata.keylen || !ctx->authsize) 308 return 0; 309 310 /* 311 * Job Descriptor and Shared Descriptor 312 * must fit into the 64-word Descriptor h/w Buffer 313 */ 314 if (rem_bytes >= DESC_QI_GCM_ENC_LEN) { 315 ctx->cdata.key_inline = true; 316 ctx->cdata.key_virt = ctx->key; 317 } else { 318 ctx->cdata.key_inline = false; 319 ctx->cdata.key_dma = ctx->key_dma; 320 } 321 322 cnstr_shdsc_gcm_encap(ctx->sh_desc_enc, &ctx->cdata, ivsize, 323 ctx->authsize, true); 324 325 /* 326 * Job Descriptor and Shared Descriptor 327 * must fit into the 64-word Descriptor h/w Buffer 328 */ 329 if (rem_bytes >= DESC_QI_GCM_DEC_LEN) { 330 ctx->cdata.key_inline = true; 331 ctx->cdata.key_virt = ctx->key; 332 } else { 333 ctx->cdata.key_inline = false; 334 ctx->cdata.key_dma = ctx->key_dma; 335 } 336 337 cnstr_shdsc_gcm_decap(ctx->sh_desc_dec, &ctx->cdata, ivsize, 338 ctx->authsize, true); 339 340 return 0; 341 } 342 343 static int gcm_setauthsize(struct crypto_aead *authenc, unsigned int authsize) 344 { 345 struct caam_ctx *ctx = crypto_aead_ctx_dma(authenc); 346 int err; 347 348 err = crypto_gcm_check_authsize(authsize); 349 if (err) 350 return err; 351 352 ctx->authsize = authsize; 353 gcm_set_sh_desc(authenc); 354 355 return 0; 356 } 357 358 static int gcm_setkey(struct crypto_aead *aead, 359 const u8 *key, unsigned int keylen) 360 { 361 struct caam_ctx *ctx = crypto_aead_ctx_dma(aead); 362 struct device *jrdev = ctx->jrdev; 363 int ret; 364 365 ret = aes_check_keylen(keylen); 366 if (ret) 367 return ret; 368 369 print_hex_dump_debug("key in @" __stringify(__LINE__)": ", 370 DUMP_PREFIX_ADDRESS, 16, 4, key, keylen, 1); 371 372 memcpy(ctx->key, key, keylen); 373 dma_sync_single_for_device(jrdev->parent, ctx->key_dma, keylen, 374 ctx->dir); 375 ctx->cdata.keylen = keylen; 376 377 ret = gcm_set_sh_desc(aead); 378 if (ret) 379 return ret; 380 381 /* Now update the driver contexts with the new shared descriptor */ 382 if (ctx->drv_ctx[ENCRYPT]) { 383 ret = caam_drv_ctx_update(ctx->drv_ctx[ENCRYPT], 384 ctx->sh_desc_enc); 385 if (ret) { 386 dev_err(jrdev, "driver enc context update failed\n"); 387 return ret; 388 } 389 } 390 391 if (ctx->drv_ctx[DECRYPT]) { 392 ret = caam_drv_ctx_update(ctx->drv_ctx[DECRYPT], 393 ctx->sh_desc_dec); 394 if (ret) { 395 dev_err(jrdev, "driver dec context update failed\n"); 396 return ret; 397 } 398 } 399 400 return 0; 401 } 402 403 static int rfc4106_set_sh_desc(struct crypto_aead *aead) 404 { 405 struct caam_ctx *ctx = crypto_aead_ctx_dma(aead); 406 unsigned int ivsize = crypto_aead_ivsize(aead); 407 int rem_bytes = CAAM_DESC_BYTES_MAX - DESC_JOB_IO_LEN - 408 ctx->cdata.keylen; 409 410 if (!ctx->cdata.keylen || !ctx->authsize) 411 return 0; 412 413 ctx->cdata.key_virt = ctx->key; 414 415 /* 416 * Job Descriptor and Shared Descriptor 417 * must fit into the 64-word Descriptor h/w Buffer 418 */ 419 if (rem_bytes >= DESC_QI_RFC4106_ENC_LEN) { 420 ctx->cdata.key_inline = true; 421 } else { 422 ctx->cdata.key_inline = false; 423 ctx->cdata.key_dma = ctx->key_dma; 424 } 425 426 cnstr_shdsc_rfc4106_encap(ctx->sh_desc_enc, &ctx->cdata, ivsize, 427 ctx->authsize, true); 428 429 /* 430 * Job Descriptor and Shared Descriptor 431 * must fit into the 64-word Descriptor h/w Buffer 432 */ 433 if (rem_bytes >= DESC_QI_RFC4106_DEC_LEN) { 434 ctx->cdata.key_inline = true; 435 } else { 436 ctx->cdata.key_inline = false; 437 ctx->cdata.key_dma = ctx->key_dma; 438 } 439 440 cnstr_shdsc_rfc4106_decap(ctx->sh_desc_dec, &ctx->cdata, ivsize, 441 ctx->authsize, true); 442 443 return 0; 444 } 445 446 static int rfc4106_setauthsize(struct crypto_aead *authenc, 447 unsigned int authsize) 448 { 449 struct caam_ctx *ctx = crypto_aead_ctx_dma(authenc); 450 int err; 451 452 err = crypto_rfc4106_check_authsize(authsize); 453 if (err) 454 return err; 455 456 ctx->authsize = authsize; 457 rfc4106_set_sh_desc(authenc); 458 459 return 0; 460 } 461 462 static int rfc4106_setkey(struct crypto_aead *aead, 463 const u8 *key, unsigned int keylen) 464 { 465 struct caam_ctx *ctx = crypto_aead_ctx_dma(aead); 466 struct device *jrdev = ctx->jrdev; 467 int ret; 468 469 ret = aes_check_keylen(keylen - 4); 470 if (ret) 471 return ret; 472 473 print_hex_dump_debug("key in @" __stringify(__LINE__)": ", 474 DUMP_PREFIX_ADDRESS, 16, 4, key, keylen, 1); 475 476 memcpy(ctx->key, key, keylen); 477 /* 478 * The last four bytes of the key material are used as the salt value 479 * in the nonce. Update the AES key length. 480 */ 481 ctx->cdata.keylen = keylen - 4; 482 dma_sync_single_for_device(jrdev->parent, ctx->key_dma, 483 ctx->cdata.keylen, ctx->dir); 484 485 ret = rfc4106_set_sh_desc(aead); 486 if (ret) 487 return ret; 488 489 /* Now update the driver contexts with the new shared descriptor */ 490 if (ctx->drv_ctx[ENCRYPT]) { 491 ret = caam_drv_ctx_update(ctx->drv_ctx[ENCRYPT], 492 ctx->sh_desc_enc); 493 if (ret) { 494 dev_err(jrdev, "driver enc context update failed\n"); 495 return ret; 496 } 497 } 498 499 if (ctx->drv_ctx[DECRYPT]) { 500 ret = caam_drv_ctx_update(ctx->drv_ctx[DECRYPT], 501 ctx->sh_desc_dec); 502 if (ret) { 503 dev_err(jrdev, "driver dec context update failed\n"); 504 return ret; 505 } 506 } 507 508 return 0; 509 } 510 511 static int rfc4543_set_sh_desc(struct crypto_aead *aead) 512 { 513 struct caam_ctx *ctx = crypto_aead_ctx_dma(aead); 514 unsigned int ivsize = crypto_aead_ivsize(aead); 515 int rem_bytes = CAAM_DESC_BYTES_MAX - DESC_JOB_IO_LEN - 516 ctx->cdata.keylen; 517 518 if (!ctx->cdata.keylen || !ctx->authsize) 519 return 0; 520 521 ctx->cdata.key_virt = ctx->key; 522 523 /* 524 * Job Descriptor and Shared Descriptor 525 * must fit into the 64-word Descriptor h/w Buffer 526 */ 527 if (rem_bytes >= DESC_QI_RFC4543_ENC_LEN) { 528 ctx->cdata.key_inline = true; 529 } else { 530 ctx->cdata.key_inline = false; 531 ctx->cdata.key_dma = ctx->key_dma; 532 } 533 534 cnstr_shdsc_rfc4543_encap(ctx->sh_desc_enc, &ctx->cdata, ivsize, 535 ctx->authsize, true); 536 537 /* 538 * Job Descriptor and Shared Descriptor 539 * must fit into the 64-word Descriptor h/w Buffer 540 */ 541 if (rem_bytes >= DESC_QI_RFC4543_DEC_LEN) { 542 ctx->cdata.key_inline = true; 543 } else { 544 ctx->cdata.key_inline = false; 545 ctx->cdata.key_dma = ctx->key_dma; 546 } 547 548 cnstr_shdsc_rfc4543_decap(ctx->sh_desc_dec, &ctx->cdata, ivsize, 549 ctx->authsize, true); 550 551 return 0; 552 } 553 554 static int rfc4543_setauthsize(struct crypto_aead *authenc, 555 unsigned int authsize) 556 { 557 struct caam_ctx *ctx = crypto_aead_ctx_dma(authenc); 558 559 if (authsize != 16) 560 return -EINVAL; 561 562 ctx->authsize = authsize; 563 rfc4543_set_sh_desc(authenc); 564 565 return 0; 566 } 567 568 static int rfc4543_setkey(struct crypto_aead *aead, 569 const u8 *key, unsigned int keylen) 570 { 571 struct caam_ctx *ctx = crypto_aead_ctx_dma(aead); 572 struct device *jrdev = ctx->jrdev; 573 int ret; 574 575 ret = aes_check_keylen(keylen - 4); 576 if (ret) 577 return ret; 578 579 print_hex_dump_debug("key in @" __stringify(__LINE__)": ", 580 DUMP_PREFIX_ADDRESS, 16, 4, key, keylen, 1); 581 582 memcpy(ctx->key, key, keylen); 583 /* 584 * The last four bytes of the key material are used as the salt value 585 * in the nonce. Update the AES key length. 586 */ 587 ctx->cdata.keylen = keylen - 4; 588 dma_sync_single_for_device(jrdev->parent, ctx->key_dma, 589 ctx->cdata.keylen, ctx->dir); 590 591 ret = rfc4543_set_sh_desc(aead); 592 if (ret) 593 return ret; 594 595 /* Now update the driver contexts with the new shared descriptor */ 596 if (ctx->drv_ctx[ENCRYPT]) { 597 ret = caam_drv_ctx_update(ctx->drv_ctx[ENCRYPT], 598 ctx->sh_desc_enc); 599 if (ret) { 600 dev_err(jrdev, "driver enc context update failed\n"); 601 return ret; 602 } 603 } 604 605 if (ctx->drv_ctx[DECRYPT]) { 606 ret = caam_drv_ctx_update(ctx->drv_ctx[DECRYPT], 607 ctx->sh_desc_dec); 608 if (ret) { 609 dev_err(jrdev, "driver dec context update failed\n"); 610 return ret; 611 } 612 } 613 614 return 0; 615 } 616 617 static int skcipher_setkey(struct crypto_skcipher *skcipher, const u8 *key, 618 unsigned int keylen, const u32 ctx1_iv_off) 619 { 620 struct caam_ctx *ctx = crypto_skcipher_ctx_dma(skcipher); 621 struct caam_skcipher_alg *alg = 622 container_of(crypto_skcipher_alg(skcipher), typeof(*alg), 623 skcipher); 624 struct device *jrdev = ctx->jrdev; 625 unsigned int ivsize = crypto_skcipher_ivsize(skcipher); 626 const bool is_rfc3686 = alg->caam.rfc3686; 627 int ret = 0; 628 629 print_hex_dump_debug("key in @" __stringify(__LINE__)": ", 630 DUMP_PREFIX_ADDRESS, 16, 4, key, keylen, 1); 631 632 ctx->cdata.keylen = keylen; 633 ctx->cdata.key_virt = key; 634 ctx->cdata.key_inline = true; 635 636 /* skcipher encrypt, decrypt shared descriptors */ 637 cnstr_shdsc_skcipher_encap(ctx->sh_desc_enc, &ctx->cdata, ivsize, 638 is_rfc3686, ctx1_iv_off); 639 cnstr_shdsc_skcipher_decap(ctx->sh_desc_dec, &ctx->cdata, ivsize, 640 is_rfc3686, ctx1_iv_off); 641 642 /* Now update the driver contexts with the new shared descriptor */ 643 if (ctx->drv_ctx[ENCRYPT]) { 644 ret = caam_drv_ctx_update(ctx->drv_ctx[ENCRYPT], 645 ctx->sh_desc_enc); 646 if (ret) { 647 dev_err(jrdev, "driver enc context update failed\n"); 648 return -EINVAL; 649 } 650 } 651 652 if (ctx->drv_ctx[DECRYPT]) { 653 ret = caam_drv_ctx_update(ctx->drv_ctx[DECRYPT], 654 ctx->sh_desc_dec); 655 if (ret) { 656 dev_err(jrdev, "driver dec context update failed\n"); 657 return -EINVAL; 658 } 659 } 660 661 return ret; 662 } 663 664 static int aes_skcipher_setkey(struct crypto_skcipher *skcipher, 665 const u8 *key, unsigned int keylen) 666 { 667 int err; 668 669 err = aes_check_keylen(keylen); 670 if (err) 671 return err; 672 673 return skcipher_setkey(skcipher, key, keylen, 0); 674 } 675 676 static int rfc3686_skcipher_setkey(struct crypto_skcipher *skcipher, 677 const u8 *key, unsigned int keylen) 678 { 679 u32 ctx1_iv_off; 680 int err; 681 682 /* 683 * RFC3686 specific: 684 * | CONTEXT1[255:128] = {NONCE, IV, COUNTER} 685 * | *key = {KEY, NONCE} 686 */ 687 ctx1_iv_off = 16 + CTR_RFC3686_NONCE_SIZE; 688 keylen -= CTR_RFC3686_NONCE_SIZE; 689 690 err = aes_check_keylen(keylen); 691 if (err) 692 return err; 693 694 return skcipher_setkey(skcipher, key, keylen, ctx1_iv_off); 695 } 696 697 static int ctr_skcipher_setkey(struct crypto_skcipher *skcipher, 698 const u8 *key, unsigned int keylen) 699 { 700 u32 ctx1_iv_off; 701 int err; 702 703 /* 704 * AES-CTR needs to load IV in CONTEXT1 reg 705 * at an offset of 128bits (16bytes) 706 * CONTEXT1[255:128] = IV 707 */ 708 ctx1_iv_off = 16; 709 710 err = aes_check_keylen(keylen); 711 if (err) 712 return err; 713 714 return skcipher_setkey(skcipher, key, keylen, ctx1_iv_off); 715 } 716 717 static int des3_skcipher_setkey(struct crypto_skcipher *skcipher, 718 const u8 *key, unsigned int keylen) 719 { 720 return verify_skcipher_des3_key(skcipher, key) ?: 721 skcipher_setkey(skcipher, key, keylen, 0); 722 } 723 724 static int des_skcipher_setkey(struct crypto_skcipher *skcipher, 725 const u8 *key, unsigned int keylen) 726 { 727 return verify_skcipher_des_key(skcipher, key) ?: 728 skcipher_setkey(skcipher, key, keylen, 0); 729 } 730 731 static int xts_skcipher_setkey(struct crypto_skcipher *skcipher, const u8 *key, 732 unsigned int keylen) 733 { 734 struct caam_ctx *ctx = crypto_skcipher_ctx_dma(skcipher); 735 struct device *jrdev = ctx->jrdev; 736 struct caam_drv_private *ctrlpriv = dev_get_drvdata(jrdev->parent); 737 int ret = 0; 738 int err; 739 740 err = xts_verify_key(skcipher, key, keylen); 741 if (err) { 742 dev_dbg(jrdev, "key size mismatch\n"); 743 return err; 744 } 745 746 if (keylen != 2 * AES_KEYSIZE_128 && keylen != 2 * AES_KEYSIZE_256) 747 ctx->xts_key_fallback = true; 748 749 if (ctrlpriv->era <= 8 || ctx->xts_key_fallback) { 750 err = crypto_skcipher_setkey(ctx->fallback, key, keylen); 751 if (err) 752 return err; 753 } 754 755 ctx->cdata.keylen = keylen; 756 ctx->cdata.key_virt = key; 757 ctx->cdata.key_inline = true; 758 759 /* xts skcipher encrypt, decrypt shared descriptors */ 760 cnstr_shdsc_xts_skcipher_encap(ctx->sh_desc_enc, &ctx->cdata); 761 cnstr_shdsc_xts_skcipher_decap(ctx->sh_desc_dec, &ctx->cdata); 762 763 /* Now update the driver contexts with the new shared descriptor */ 764 if (ctx->drv_ctx[ENCRYPT]) { 765 ret = caam_drv_ctx_update(ctx->drv_ctx[ENCRYPT], 766 ctx->sh_desc_enc); 767 if (ret) { 768 dev_err(jrdev, "driver enc context update failed\n"); 769 return -EINVAL; 770 } 771 } 772 773 if (ctx->drv_ctx[DECRYPT]) { 774 ret = caam_drv_ctx_update(ctx->drv_ctx[DECRYPT], 775 ctx->sh_desc_dec); 776 if (ret) { 777 dev_err(jrdev, "driver dec context update failed\n"); 778 return -EINVAL; 779 } 780 } 781 782 return ret; 783 } 784 785 /* 786 * aead_edesc - s/w-extended aead descriptor 787 * @src_nents: number of segments in input scatterlist 788 * @dst_nents: number of segments in output scatterlist 789 * @iv_dma: dma address of iv for checking continuity and link table 790 * @qm_sg_bytes: length of dma mapped h/w link table 791 * @qm_sg_dma: bus physical mapped address of h/w link table 792 * @assoclen: associated data length, in CAAM endianness 793 * @assoclen_dma: bus physical mapped address of req->assoclen 794 * @drv_req: driver-specific request structure 795 * @sgt: the h/w link table, followed by IV 796 */ 797 struct aead_edesc { 798 int src_nents; 799 int dst_nents; 800 dma_addr_t iv_dma; 801 int qm_sg_bytes; 802 dma_addr_t qm_sg_dma; 803 unsigned int assoclen; 804 dma_addr_t assoclen_dma; 805 struct caam_drv_req drv_req; 806 struct qm_sg_entry sgt[]; 807 }; 808 809 /* 810 * skcipher_edesc - s/w-extended skcipher descriptor 811 * @src_nents: number of segments in input scatterlist 812 * @dst_nents: number of segments in output scatterlist 813 * @iv_dma: dma address of iv for checking continuity and link table 814 * @qm_sg_bytes: length of dma mapped h/w link table 815 * @qm_sg_dma: bus physical mapped address of h/w link table 816 * @drv_req: driver-specific request structure 817 * @sgt: the h/w link table, followed by IV 818 */ 819 struct skcipher_edesc { 820 int src_nents; 821 int dst_nents; 822 dma_addr_t iv_dma; 823 int qm_sg_bytes; 824 dma_addr_t qm_sg_dma; 825 struct caam_drv_req drv_req; 826 struct qm_sg_entry sgt[]; 827 }; 828 829 static struct caam_drv_ctx *get_drv_ctx(struct caam_ctx *ctx, 830 enum optype type) 831 { 832 /* 833 * This function is called on the fast path with values of 'type' 834 * known at compile time. Invalid arguments are not expected and 835 * thus no checks are made. 836 */ 837 struct caam_drv_ctx *drv_ctx = ctx->drv_ctx[type]; 838 u32 *desc; 839 840 if (unlikely(!drv_ctx)) { 841 spin_lock(&ctx->lock); 842 843 /* Read again to check if some other core init drv_ctx */ 844 drv_ctx = ctx->drv_ctx[type]; 845 if (!drv_ctx) { 846 int cpu; 847 848 if (type == ENCRYPT) 849 desc = ctx->sh_desc_enc; 850 else /* (type == DECRYPT) */ 851 desc = ctx->sh_desc_dec; 852 853 cpu = smp_processor_id(); 854 drv_ctx = caam_drv_ctx_init(ctx->qidev, &cpu, desc); 855 if (!IS_ERR(drv_ctx)) 856 drv_ctx->op_type = type; 857 858 ctx->drv_ctx[type] = drv_ctx; 859 } 860 861 spin_unlock(&ctx->lock); 862 } 863 864 return drv_ctx; 865 } 866 867 static void caam_unmap(struct device *dev, struct scatterlist *src, 868 struct scatterlist *dst, int src_nents, 869 int dst_nents, dma_addr_t iv_dma, int ivsize, 870 enum dma_data_direction iv_dir, dma_addr_t qm_sg_dma, 871 int qm_sg_bytes) 872 { 873 if (dst != src) { 874 if (src_nents) 875 dma_unmap_sg(dev, src, src_nents, DMA_TO_DEVICE); 876 if (dst_nents) 877 dma_unmap_sg(dev, dst, dst_nents, DMA_FROM_DEVICE); 878 } else { 879 dma_unmap_sg(dev, src, src_nents, DMA_BIDIRECTIONAL); 880 } 881 882 if (iv_dma) 883 dma_unmap_single(dev, iv_dma, ivsize, iv_dir); 884 if (qm_sg_bytes) 885 dma_unmap_single(dev, qm_sg_dma, qm_sg_bytes, DMA_TO_DEVICE); 886 } 887 888 static void aead_unmap(struct device *dev, 889 struct aead_edesc *edesc, 890 struct aead_request *req) 891 { 892 struct crypto_aead *aead = crypto_aead_reqtfm(req); 893 int ivsize = crypto_aead_ivsize(aead); 894 895 caam_unmap(dev, req->src, req->dst, edesc->src_nents, edesc->dst_nents, 896 edesc->iv_dma, ivsize, DMA_TO_DEVICE, edesc->qm_sg_dma, 897 edesc->qm_sg_bytes); 898 dma_unmap_single(dev, edesc->assoclen_dma, 4, DMA_TO_DEVICE); 899 } 900 901 static void skcipher_unmap(struct device *dev, struct skcipher_edesc *edesc, 902 struct skcipher_request *req) 903 { 904 struct crypto_skcipher *skcipher = crypto_skcipher_reqtfm(req); 905 int ivsize = crypto_skcipher_ivsize(skcipher); 906 907 caam_unmap(dev, req->src, req->dst, edesc->src_nents, edesc->dst_nents, 908 edesc->iv_dma, ivsize, DMA_BIDIRECTIONAL, edesc->qm_sg_dma, 909 edesc->qm_sg_bytes); 910 } 911 912 static void aead_done(struct caam_drv_req *drv_req, u32 status) 913 { 914 struct device *qidev; 915 struct aead_edesc *edesc; 916 struct aead_request *aead_req = drv_req->app_ctx; 917 struct crypto_aead *aead = crypto_aead_reqtfm(aead_req); 918 struct caam_ctx *caam_ctx = crypto_aead_ctx_dma(aead); 919 int ecode = 0; 920 921 qidev = caam_ctx->qidev; 922 923 if (unlikely(status)) 924 ecode = caam_jr_strstatus(qidev, status); 925 926 edesc = container_of(drv_req, typeof(*edesc), drv_req); 927 aead_unmap(qidev, edesc, aead_req); 928 929 aead_request_complete(aead_req, ecode); 930 qi_cache_free(edesc); 931 } 932 933 /* 934 * allocate and map the aead extended descriptor 935 */ 936 static struct aead_edesc *aead_edesc_alloc(struct aead_request *req, 937 bool encrypt) 938 { 939 struct crypto_aead *aead = crypto_aead_reqtfm(req); 940 struct caam_ctx *ctx = crypto_aead_ctx_dma(aead); 941 struct caam_aead_alg *alg = container_of(crypto_aead_alg(aead), 942 typeof(*alg), aead); 943 struct device *qidev = ctx->qidev; 944 gfp_t flags = (req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP) ? 945 GFP_KERNEL : GFP_ATOMIC; 946 int src_nents, mapped_src_nents, dst_nents = 0, mapped_dst_nents = 0; 947 int src_len, dst_len = 0; 948 struct aead_edesc *edesc; 949 dma_addr_t qm_sg_dma, iv_dma = 0; 950 int ivsize = 0; 951 unsigned int authsize = ctx->authsize; 952 int qm_sg_index = 0, qm_sg_ents = 0, qm_sg_bytes; 953 int in_len, out_len; 954 struct qm_sg_entry *sg_table, *fd_sgt; 955 struct caam_drv_ctx *drv_ctx; 956 957 drv_ctx = get_drv_ctx(ctx, encrypt ? ENCRYPT : DECRYPT); 958 if (IS_ERR(drv_ctx)) 959 return (struct aead_edesc *)drv_ctx; 960 961 /* allocate space for base edesc and hw desc commands, link tables */ 962 edesc = qi_cache_alloc(GFP_DMA | flags); 963 if (unlikely(!edesc)) { 964 dev_err(qidev, "could not allocate extended descriptor\n"); 965 return ERR_PTR(-ENOMEM); 966 } 967 968 if (likely(req->src == req->dst)) { 969 src_len = req->assoclen + req->cryptlen + 970 (encrypt ? authsize : 0); 971 972 src_nents = sg_nents_for_len(req->src, src_len); 973 if (unlikely(src_nents < 0)) { 974 dev_err(qidev, "Insufficient bytes (%d) in src S/G\n", 975 src_len); 976 qi_cache_free(edesc); 977 return ERR_PTR(src_nents); 978 } 979 980 mapped_src_nents = dma_map_sg(qidev, req->src, src_nents, 981 DMA_BIDIRECTIONAL); 982 if (unlikely(!mapped_src_nents)) { 983 dev_err(qidev, "unable to map source\n"); 984 qi_cache_free(edesc); 985 return ERR_PTR(-ENOMEM); 986 } 987 } else { 988 src_len = req->assoclen + req->cryptlen; 989 dst_len = src_len + (encrypt ? authsize : (-authsize)); 990 991 src_nents = sg_nents_for_len(req->src, src_len); 992 if (unlikely(src_nents < 0)) { 993 dev_err(qidev, "Insufficient bytes (%d) in src S/G\n", 994 src_len); 995 qi_cache_free(edesc); 996 return ERR_PTR(src_nents); 997 } 998 999 dst_nents = sg_nents_for_len(req->dst, dst_len); 1000 if (unlikely(dst_nents < 0)) { 1001 dev_err(qidev, "Insufficient bytes (%d) in dst S/G\n", 1002 dst_len); 1003 qi_cache_free(edesc); 1004 return ERR_PTR(dst_nents); 1005 } 1006 1007 if (src_nents) { 1008 mapped_src_nents = dma_map_sg(qidev, req->src, 1009 src_nents, DMA_TO_DEVICE); 1010 if (unlikely(!mapped_src_nents)) { 1011 dev_err(qidev, "unable to map source\n"); 1012 qi_cache_free(edesc); 1013 return ERR_PTR(-ENOMEM); 1014 } 1015 } else { 1016 mapped_src_nents = 0; 1017 } 1018 1019 if (dst_nents) { 1020 mapped_dst_nents = dma_map_sg(qidev, req->dst, 1021 dst_nents, 1022 DMA_FROM_DEVICE); 1023 if (unlikely(!mapped_dst_nents)) { 1024 dev_err(qidev, "unable to map destination\n"); 1025 dma_unmap_sg(qidev, req->src, src_nents, 1026 DMA_TO_DEVICE); 1027 qi_cache_free(edesc); 1028 return ERR_PTR(-ENOMEM); 1029 } 1030 } else { 1031 mapped_dst_nents = 0; 1032 } 1033 } 1034 1035 if ((alg->caam.rfc3686 && encrypt) || !alg->caam.geniv) 1036 ivsize = crypto_aead_ivsize(aead); 1037 1038 /* 1039 * Create S/G table: req->assoclen, [IV,] req->src [, req->dst]. 1040 * Input is not contiguous. 1041 * HW reads 4 S/G entries at a time; make sure the reads don't go beyond 1042 * the end of the table by allocating more S/G entries. Logic: 1043 * if (src != dst && output S/G) 1044 * pad output S/G, if needed 1045 * else if (src == dst && S/G) 1046 * overlapping S/Gs; pad one of them 1047 * else if (input S/G) ... 1048 * pad input S/G, if needed 1049 */ 1050 qm_sg_ents = 1 + !!ivsize + mapped_src_nents; 1051 if (mapped_dst_nents > 1) 1052 qm_sg_ents += pad_sg_nents(mapped_dst_nents); 1053 else if ((req->src == req->dst) && (mapped_src_nents > 1)) 1054 qm_sg_ents = max(pad_sg_nents(qm_sg_ents), 1055 1 + !!ivsize + pad_sg_nents(mapped_src_nents)); 1056 else 1057 qm_sg_ents = pad_sg_nents(qm_sg_ents); 1058 1059 sg_table = &edesc->sgt[0]; 1060 qm_sg_bytes = qm_sg_ents * sizeof(*sg_table); 1061 if (unlikely(offsetof(struct aead_edesc, sgt) + qm_sg_bytes + ivsize > 1062 CAAM_QI_MEMCACHE_SIZE)) { 1063 dev_err(qidev, "No space for %d S/G entries and/or %dB IV\n", 1064 qm_sg_ents, ivsize); 1065 caam_unmap(qidev, req->src, req->dst, src_nents, dst_nents, 0, 1066 0, DMA_NONE, 0, 0); 1067 qi_cache_free(edesc); 1068 return ERR_PTR(-ENOMEM); 1069 } 1070 1071 if (ivsize) { 1072 u8 *iv = (u8 *)(sg_table + qm_sg_ents); 1073 1074 /* Make sure IV is located in a DMAable area */ 1075 memcpy(iv, req->iv, ivsize); 1076 1077 iv_dma = dma_map_single(qidev, iv, ivsize, DMA_TO_DEVICE); 1078 if (dma_mapping_error(qidev, iv_dma)) { 1079 dev_err(qidev, "unable to map IV\n"); 1080 caam_unmap(qidev, req->src, req->dst, src_nents, 1081 dst_nents, 0, 0, DMA_NONE, 0, 0); 1082 qi_cache_free(edesc); 1083 return ERR_PTR(-ENOMEM); 1084 } 1085 } 1086 1087 edesc->src_nents = src_nents; 1088 edesc->dst_nents = dst_nents; 1089 edesc->iv_dma = iv_dma; 1090 edesc->drv_req.app_ctx = req; 1091 edesc->drv_req.cbk = aead_done; 1092 edesc->drv_req.drv_ctx = drv_ctx; 1093 1094 edesc->assoclen = cpu_to_caam32(req->assoclen); 1095 edesc->assoclen_dma = dma_map_single(qidev, &edesc->assoclen, 4, 1096 DMA_TO_DEVICE); 1097 if (dma_mapping_error(qidev, edesc->assoclen_dma)) { 1098 dev_err(qidev, "unable to map assoclen\n"); 1099 caam_unmap(qidev, req->src, req->dst, src_nents, dst_nents, 1100 iv_dma, ivsize, DMA_TO_DEVICE, 0, 0); 1101 qi_cache_free(edesc); 1102 return ERR_PTR(-ENOMEM); 1103 } 1104 1105 dma_to_qm_sg_one(sg_table, edesc->assoclen_dma, 4, 0); 1106 qm_sg_index++; 1107 if (ivsize) { 1108 dma_to_qm_sg_one(sg_table + qm_sg_index, iv_dma, ivsize, 0); 1109 qm_sg_index++; 1110 } 1111 sg_to_qm_sg_last(req->src, src_len, sg_table + qm_sg_index, 0); 1112 qm_sg_index += mapped_src_nents; 1113 1114 if (mapped_dst_nents > 1) 1115 sg_to_qm_sg_last(req->dst, dst_len, sg_table + qm_sg_index, 0); 1116 1117 qm_sg_dma = dma_map_single(qidev, sg_table, qm_sg_bytes, DMA_TO_DEVICE); 1118 if (dma_mapping_error(qidev, qm_sg_dma)) { 1119 dev_err(qidev, "unable to map S/G table\n"); 1120 dma_unmap_single(qidev, edesc->assoclen_dma, 4, DMA_TO_DEVICE); 1121 caam_unmap(qidev, req->src, req->dst, src_nents, dst_nents, 1122 iv_dma, ivsize, DMA_TO_DEVICE, 0, 0); 1123 qi_cache_free(edesc); 1124 return ERR_PTR(-ENOMEM); 1125 } 1126 1127 edesc->qm_sg_dma = qm_sg_dma; 1128 edesc->qm_sg_bytes = qm_sg_bytes; 1129 1130 out_len = req->assoclen + req->cryptlen + 1131 (encrypt ? ctx->authsize : (-ctx->authsize)); 1132 in_len = 4 + ivsize + req->assoclen + req->cryptlen; 1133 1134 fd_sgt = &edesc->drv_req.fd_sgt[0]; 1135 dma_to_qm_sg_one_last_ext(&fd_sgt[1], qm_sg_dma, in_len, 0); 1136 1137 if (req->dst == req->src) { 1138 if (mapped_src_nents == 1) 1139 dma_to_qm_sg_one(&fd_sgt[0], sg_dma_address(req->src), 1140 out_len, 0); 1141 else 1142 dma_to_qm_sg_one_ext(&fd_sgt[0], qm_sg_dma + 1143 (1 + !!ivsize) * sizeof(*sg_table), 1144 out_len, 0); 1145 } else if (mapped_dst_nents <= 1) { 1146 dma_to_qm_sg_one(&fd_sgt[0], sg_dma_address(req->dst), out_len, 1147 0); 1148 } else { 1149 dma_to_qm_sg_one_ext(&fd_sgt[0], qm_sg_dma + sizeof(*sg_table) * 1150 qm_sg_index, out_len, 0); 1151 } 1152 1153 return edesc; 1154 } 1155 1156 static inline int aead_crypt(struct aead_request *req, bool encrypt) 1157 { 1158 struct aead_edesc *edesc; 1159 struct crypto_aead *aead = crypto_aead_reqtfm(req); 1160 struct caam_ctx *ctx = crypto_aead_ctx_dma(aead); 1161 int ret; 1162 1163 if (unlikely(caam_congested)) 1164 return -EAGAIN; 1165 1166 /* allocate extended descriptor */ 1167 edesc = aead_edesc_alloc(req, encrypt); 1168 if (IS_ERR(edesc)) 1169 return PTR_ERR(edesc); 1170 1171 /* Create and submit job descriptor */ 1172 ret = caam_qi_enqueue(ctx->qidev, &edesc->drv_req); 1173 if (!ret) { 1174 ret = -EINPROGRESS; 1175 } else { 1176 aead_unmap(ctx->qidev, edesc, req); 1177 qi_cache_free(edesc); 1178 } 1179 1180 return ret; 1181 } 1182 1183 static int aead_encrypt(struct aead_request *req) 1184 { 1185 return aead_crypt(req, true); 1186 } 1187 1188 static int aead_decrypt(struct aead_request *req) 1189 { 1190 return aead_crypt(req, false); 1191 } 1192 1193 static int ipsec_gcm_encrypt(struct aead_request *req) 1194 { 1195 return crypto_ipsec_check_assoclen(req->assoclen) ? : aead_crypt(req, 1196 true); 1197 } 1198 1199 static int ipsec_gcm_decrypt(struct aead_request *req) 1200 { 1201 return crypto_ipsec_check_assoclen(req->assoclen) ? : aead_crypt(req, 1202 false); 1203 } 1204 1205 static void skcipher_done(struct caam_drv_req *drv_req, u32 status) 1206 { 1207 struct skcipher_edesc *edesc; 1208 struct skcipher_request *req = drv_req->app_ctx; 1209 struct crypto_skcipher *skcipher = crypto_skcipher_reqtfm(req); 1210 struct caam_ctx *caam_ctx = crypto_skcipher_ctx_dma(skcipher); 1211 struct device *qidev = caam_ctx->qidev; 1212 int ivsize = crypto_skcipher_ivsize(skcipher); 1213 int ecode = 0; 1214 1215 dev_dbg(qidev, "%s %d: status 0x%x\n", __func__, __LINE__, status); 1216 1217 edesc = container_of(drv_req, typeof(*edesc), drv_req); 1218 1219 if (status) 1220 ecode = caam_jr_strstatus(qidev, status); 1221 1222 print_hex_dump_debug("dstiv @" __stringify(__LINE__)": ", 1223 DUMP_PREFIX_ADDRESS, 16, 4, req->iv, 1224 edesc->src_nents > 1 ? 100 : ivsize, 1); 1225 caam_dump_sg("dst @" __stringify(__LINE__)": ", 1226 DUMP_PREFIX_ADDRESS, 16, 4, req->dst, 1227 edesc->dst_nents > 1 ? 100 : req->cryptlen, 1); 1228 1229 skcipher_unmap(qidev, edesc, req); 1230 1231 /* 1232 * The crypto API expects us to set the IV (req->iv) to the last 1233 * ciphertext block (CBC mode) or last counter (CTR mode). 1234 * This is used e.g. by the CTS mode. 1235 */ 1236 if (!ecode) 1237 memcpy(req->iv, (u8 *)&edesc->sgt[0] + edesc->qm_sg_bytes, 1238 ivsize); 1239 1240 qi_cache_free(edesc); 1241 skcipher_request_complete(req, ecode); 1242 } 1243 1244 static struct skcipher_edesc *skcipher_edesc_alloc(struct skcipher_request *req, 1245 bool encrypt) 1246 { 1247 struct crypto_skcipher *skcipher = crypto_skcipher_reqtfm(req); 1248 struct caam_ctx *ctx = crypto_skcipher_ctx_dma(skcipher); 1249 struct device *qidev = ctx->qidev; 1250 gfp_t flags = (req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP) ? 1251 GFP_KERNEL : GFP_ATOMIC; 1252 int src_nents, mapped_src_nents, dst_nents = 0, mapped_dst_nents = 0; 1253 struct skcipher_edesc *edesc; 1254 dma_addr_t iv_dma; 1255 u8 *iv; 1256 int ivsize = crypto_skcipher_ivsize(skcipher); 1257 int dst_sg_idx, qm_sg_ents, qm_sg_bytes; 1258 struct qm_sg_entry *sg_table, *fd_sgt; 1259 struct caam_drv_ctx *drv_ctx; 1260 1261 drv_ctx = get_drv_ctx(ctx, encrypt ? ENCRYPT : DECRYPT); 1262 if (IS_ERR(drv_ctx)) 1263 return (struct skcipher_edesc *)drv_ctx; 1264 1265 src_nents = sg_nents_for_len(req->src, req->cryptlen); 1266 if (unlikely(src_nents < 0)) { 1267 dev_err(qidev, "Insufficient bytes (%d) in src S/G\n", 1268 req->cryptlen); 1269 return ERR_PTR(src_nents); 1270 } 1271 1272 if (unlikely(req->src != req->dst)) { 1273 dst_nents = sg_nents_for_len(req->dst, req->cryptlen); 1274 if (unlikely(dst_nents < 0)) { 1275 dev_err(qidev, "Insufficient bytes (%d) in dst S/G\n", 1276 req->cryptlen); 1277 return ERR_PTR(dst_nents); 1278 } 1279 1280 mapped_src_nents = dma_map_sg(qidev, req->src, src_nents, 1281 DMA_TO_DEVICE); 1282 if (unlikely(!mapped_src_nents)) { 1283 dev_err(qidev, "unable to map source\n"); 1284 return ERR_PTR(-ENOMEM); 1285 } 1286 1287 mapped_dst_nents = dma_map_sg(qidev, req->dst, dst_nents, 1288 DMA_FROM_DEVICE); 1289 if (unlikely(!mapped_dst_nents)) { 1290 dev_err(qidev, "unable to map destination\n"); 1291 dma_unmap_sg(qidev, req->src, src_nents, DMA_TO_DEVICE); 1292 return ERR_PTR(-ENOMEM); 1293 } 1294 } else { 1295 mapped_src_nents = dma_map_sg(qidev, req->src, src_nents, 1296 DMA_BIDIRECTIONAL); 1297 if (unlikely(!mapped_src_nents)) { 1298 dev_err(qidev, "unable to map source\n"); 1299 return ERR_PTR(-ENOMEM); 1300 } 1301 } 1302 1303 qm_sg_ents = 1 + mapped_src_nents; 1304 dst_sg_idx = qm_sg_ents; 1305 1306 /* 1307 * Input, output HW S/G tables: [IV, src][dst, IV] 1308 * IV entries point to the same buffer 1309 * If src == dst, S/G entries are reused (S/G tables overlap) 1310 * 1311 * HW reads 4 S/G entries at a time; make sure the reads don't go beyond 1312 * the end of the table by allocating more S/G entries. 1313 */ 1314 if (req->src != req->dst) 1315 qm_sg_ents += pad_sg_nents(mapped_dst_nents + 1); 1316 else 1317 qm_sg_ents = 1 + pad_sg_nents(qm_sg_ents); 1318 1319 qm_sg_bytes = qm_sg_ents * sizeof(struct qm_sg_entry); 1320 if (unlikely(offsetof(struct skcipher_edesc, sgt) + qm_sg_bytes + 1321 ivsize > CAAM_QI_MEMCACHE_SIZE)) { 1322 dev_err(qidev, "No space for %d S/G entries and/or %dB IV\n", 1323 qm_sg_ents, ivsize); 1324 caam_unmap(qidev, req->src, req->dst, src_nents, dst_nents, 0, 1325 0, DMA_NONE, 0, 0); 1326 return ERR_PTR(-ENOMEM); 1327 } 1328 1329 /* allocate space for base edesc, link tables and IV */ 1330 edesc = qi_cache_alloc(GFP_DMA | flags); 1331 if (unlikely(!edesc)) { 1332 dev_err(qidev, "could not allocate extended descriptor\n"); 1333 caam_unmap(qidev, req->src, req->dst, src_nents, dst_nents, 0, 1334 0, DMA_NONE, 0, 0); 1335 return ERR_PTR(-ENOMEM); 1336 } 1337 1338 /* Make sure IV is located in a DMAable area */ 1339 sg_table = &edesc->sgt[0]; 1340 iv = (u8 *)(sg_table + qm_sg_ents); 1341 memcpy(iv, req->iv, ivsize); 1342 1343 iv_dma = dma_map_single(qidev, iv, ivsize, DMA_BIDIRECTIONAL); 1344 if (dma_mapping_error(qidev, iv_dma)) { 1345 dev_err(qidev, "unable to map IV\n"); 1346 caam_unmap(qidev, req->src, req->dst, src_nents, dst_nents, 0, 1347 0, DMA_NONE, 0, 0); 1348 qi_cache_free(edesc); 1349 return ERR_PTR(-ENOMEM); 1350 } 1351 1352 edesc->src_nents = src_nents; 1353 edesc->dst_nents = dst_nents; 1354 edesc->iv_dma = iv_dma; 1355 edesc->qm_sg_bytes = qm_sg_bytes; 1356 edesc->drv_req.app_ctx = req; 1357 edesc->drv_req.cbk = skcipher_done; 1358 edesc->drv_req.drv_ctx = drv_ctx; 1359 1360 dma_to_qm_sg_one(sg_table, iv_dma, ivsize, 0); 1361 sg_to_qm_sg(req->src, req->cryptlen, sg_table + 1, 0); 1362 1363 if (req->src != req->dst) 1364 sg_to_qm_sg(req->dst, req->cryptlen, sg_table + dst_sg_idx, 0); 1365 1366 dma_to_qm_sg_one(sg_table + dst_sg_idx + mapped_dst_nents, iv_dma, 1367 ivsize, 0); 1368 1369 edesc->qm_sg_dma = dma_map_single(qidev, sg_table, edesc->qm_sg_bytes, 1370 DMA_TO_DEVICE); 1371 if (dma_mapping_error(qidev, edesc->qm_sg_dma)) { 1372 dev_err(qidev, "unable to map S/G table\n"); 1373 caam_unmap(qidev, req->src, req->dst, src_nents, dst_nents, 1374 iv_dma, ivsize, DMA_BIDIRECTIONAL, 0, 0); 1375 qi_cache_free(edesc); 1376 return ERR_PTR(-ENOMEM); 1377 } 1378 1379 fd_sgt = &edesc->drv_req.fd_sgt[0]; 1380 1381 dma_to_qm_sg_one_last_ext(&fd_sgt[1], edesc->qm_sg_dma, 1382 ivsize + req->cryptlen, 0); 1383 1384 if (req->src == req->dst) 1385 dma_to_qm_sg_one_ext(&fd_sgt[0], edesc->qm_sg_dma + 1386 sizeof(*sg_table), req->cryptlen + ivsize, 1387 0); 1388 else 1389 dma_to_qm_sg_one_ext(&fd_sgt[0], edesc->qm_sg_dma + dst_sg_idx * 1390 sizeof(*sg_table), req->cryptlen + ivsize, 1391 0); 1392 1393 return edesc; 1394 } 1395 1396 static inline bool xts_skcipher_ivsize(struct skcipher_request *req) 1397 { 1398 struct crypto_skcipher *skcipher = crypto_skcipher_reqtfm(req); 1399 unsigned int ivsize = crypto_skcipher_ivsize(skcipher); 1400 1401 return !!get_unaligned((u64 *)(req->iv + (ivsize / 2))); 1402 } 1403 1404 static inline int skcipher_crypt(struct skcipher_request *req, bool encrypt) 1405 { 1406 struct skcipher_edesc *edesc; 1407 struct crypto_skcipher *skcipher = crypto_skcipher_reqtfm(req); 1408 struct caam_ctx *ctx = crypto_skcipher_ctx_dma(skcipher); 1409 struct caam_drv_private *ctrlpriv = dev_get_drvdata(ctx->jrdev->parent); 1410 int ret; 1411 1412 /* 1413 * XTS is expected to return an error even for input length = 0 1414 * Note that the case input length < block size will be caught during 1415 * HW offloading and return an error. 1416 */ 1417 if (!req->cryptlen && !ctx->fallback) 1418 return 0; 1419 1420 if (ctx->fallback && ((ctrlpriv->era <= 8 && xts_skcipher_ivsize(req)) || 1421 ctx->xts_key_fallback)) { 1422 struct caam_skcipher_req_ctx *rctx = skcipher_request_ctx(req); 1423 1424 skcipher_request_set_tfm(&rctx->fallback_req, ctx->fallback); 1425 skcipher_request_set_callback(&rctx->fallback_req, 1426 req->base.flags, 1427 req->base.complete, 1428 req->base.data); 1429 skcipher_request_set_crypt(&rctx->fallback_req, req->src, 1430 req->dst, req->cryptlen, req->iv); 1431 1432 return encrypt ? crypto_skcipher_encrypt(&rctx->fallback_req) : 1433 crypto_skcipher_decrypt(&rctx->fallback_req); 1434 } 1435 1436 if (unlikely(caam_congested)) 1437 return -EAGAIN; 1438 1439 /* allocate extended descriptor */ 1440 edesc = skcipher_edesc_alloc(req, encrypt); 1441 if (IS_ERR(edesc)) 1442 return PTR_ERR(edesc); 1443 1444 ret = caam_qi_enqueue(ctx->qidev, &edesc->drv_req); 1445 if (!ret) { 1446 ret = -EINPROGRESS; 1447 } else { 1448 skcipher_unmap(ctx->qidev, edesc, req); 1449 qi_cache_free(edesc); 1450 } 1451 1452 return ret; 1453 } 1454 1455 static int skcipher_encrypt(struct skcipher_request *req) 1456 { 1457 return skcipher_crypt(req, true); 1458 } 1459 1460 static int skcipher_decrypt(struct skcipher_request *req) 1461 { 1462 return skcipher_crypt(req, false); 1463 } 1464 1465 static struct caam_skcipher_alg driver_algs[] = { 1466 { 1467 .skcipher = { 1468 .base = { 1469 .cra_name = "cbc(aes)", 1470 .cra_driver_name = "cbc-aes-caam-qi", 1471 .cra_blocksize = AES_BLOCK_SIZE, 1472 }, 1473 .setkey = aes_skcipher_setkey, 1474 .encrypt = skcipher_encrypt, 1475 .decrypt = skcipher_decrypt, 1476 .min_keysize = AES_MIN_KEY_SIZE, 1477 .max_keysize = AES_MAX_KEY_SIZE, 1478 .ivsize = AES_BLOCK_SIZE, 1479 }, 1480 .caam.class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CBC, 1481 }, 1482 { 1483 .skcipher = { 1484 .base = { 1485 .cra_name = "cbc(des3_ede)", 1486 .cra_driver_name = "cbc-3des-caam-qi", 1487 .cra_blocksize = DES3_EDE_BLOCK_SIZE, 1488 }, 1489 .setkey = des3_skcipher_setkey, 1490 .encrypt = skcipher_encrypt, 1491 .decrypt = skcipher_decrypt, 1492 .min_keysize = DES3_EDE_KEY_SIZE, 1493 .max_keysize = DES3_EDE_KEY_SIZE, 1494 .ivsize = DES3_EDE_BLOCK_SIZE, 1495 }, 1496 .caam.class1_alg_type = OP_ALG_ALGSEL_3DES | OP_ALG_AAI_CBC, 1497 }, 1498 { 1499 .skcipher = { 1500 .base = { 1501 .cra_name = "cbc(des)", 1502 .cra_driver_name = "cbc-des-caam-qi", 1503 .cra_blocksize = DES_BLOCK_SIZE, 1504 }, 1505 .setkey = des_skcipher_setkey, 1506 .encrypt = skcipher_encrypt, 1507 .decrypt = skcipher_decrypt, 1508 .min_keysize = DES_KEY_SIZE, 1509 .max_keysize = DES_KEY_SIZE, 1510 .ivsize = DES_BLOCK_SIZE, 1511 }, 1512 .caam.class1_alg_type = OP_ALG_ALGSEL_DES | OP_ALG_AAI_CBC, 1513 }, 1514 { 1515 .skcipher = { 1516 .base = { 1517 .cra_name = "ctr(aes)", 1518 .cra_driver_name = "ctr-aes-caam-qi", 1519 .cra_blocksize = 1, 1520 }, 1521 .setkey = ctr_skcipher_setkey, 1522 .encrypt = skcipher_encrypt, 1523 .decrypt = skcipher_decrypt, 1524 .min_keysize = AES_MIN_KEY_SIZE, 1525 .max_keysize = AES_MAX_KEY_SIZE, 1526 .ivsize = AES_BLOCK_SIZE, 1527 .chunksize = AES_BLOCK_SIZE, 1528 }, 1529 .caam.class1_alg_type = OP_ALG_ALGSEL_AES | 1530 OP_ALG_AAI_CTR_MOD128, 1531 }, 1532 { 1533 .skcipher = { 1534 .base = { 1535 .cra_name = "rfc3686(ctr(aes))", 1536 .cra_driver_name = "rfc3686-ctr-aes-caam-qi", 1537 .cra_blocksize = 1, 1538 }, 1539 .setkey = rfc3686_skcipher_setkey, 1540 .encrypt = skcipher_encrypt, 1541 .decrypt = skcipher_decrypt, 1542 .min_keysize = AES_MIN_KEY_SIZE + 1543 CTR_RFC3686_NONCE_SIZE, 1544 .max_keysize = AES_MAX_KEY_SIZE + 1545 CTR_RFC3686_NONCE_SIZE, 1546 .ivsize = CTR_RFC3686_IV_SIZE, 1547 .chunksize = AES_BLOCK_SIZE, 1548 }, 1549 .caam = { 1550 .class1_alg_type = OP_ALG_ALGSEL_AES | 1551 OP_ALG_AAI_CTR_MOD128, 1552 .rfc3686 = true, 1553 }, 1554 }, 1555 { 1556 .skcipher = { 1557 .base = { 1558 .cra_name = "xts(aes)", 1559 .cra_driver_name = "xts-aes-caam-qi", 1560 .cra_flags = CRYPTO_ALG_NEED_FALLBACK, 1561 .cra_blocksize = AES_BLOCK_SIZE, 1562 }, 1563 .setkey = xts_skcipher_setkey, 1564 .encrypt = skcipher_encrypt, 1565 .decrypt = skcipher_decrypt, 1566 .min_keysize = 2 * AES_MIN_KEY_SIZE, 1567 .max_keysize = 2 * AES_MAX_KEY_SIZE, 1568 .ivsize = AES_BLOCK_SIZE, 1569 }, 1570 .caam.class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_XTS, 1571 }, 1572 }; 1573 1574 static struct caam_aead_alg driver_aeads[] = { 1575 { 1576 .aead = { 1577 .base = { 1578 .cra_name = "rfc4106(gcm(aes))", 1579 .cra_driver_name = "rfc4106-gcm-aes-caam-qi", 1580 .cra_blocksize = 1, 1581 }, 1582 .setkey = rfc4106_setkey, 1583 .setauthsize = rfc4106_setauthsize, 1584 .encrypt = ipsec_gcm_encrypt, 1585 .decrypt = ipsec_gcm_decrypt, 1586 .ivsize = 8, 1587 .maxauthsize = AES_BLOCK_SIZE, 1588 }, 1589 .caam = { 1590 .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_GCM, 1591 .nodkp = true, 1592 }, 1593 }, 1594 { 1595 .aead = { 1596 .base = { 1597 .cra_name = "rfc4543(gcm(aes))", 1598 .cra_driver_name = "rfc4543-gcm-aes-caam-qi", 1599 .cra_blocksize = 1, 1600 }, 1601 .setkey = rfc4543_setkey, 1602 .setauthsize = rfc4543_setauthsize, 1603 .encrypt = ipsec_gcm_encrypt, 1604 .decrypt = ipsec_gcm_decrypt, 1605 .ivsize = 8, 1606 .maxauthsize = AES_BLOCK_SIZE, 1607 }, 1608 .caam = { 1609 .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_GCM, 1610 .nodkp = true, 1611 }, 1612 }, 1613 /* Galois Counter Mode */ 1614 { 1615 .aead = { 1616 .base = { 1617 .cra_name = "gcm(aes)", 1618 .cra_driver_name = "gcm-aes-caam-qi", 1619 .cra_blocksize = 1, 1620 }, 1621 .setkey = gcm_setkey, 1622 .setauthsize = gcm_setauthsize, 1623 .encrypt = aead_encrypt, 1624 .decrypt = aead_decrypt, 1625 .ivsize = 12, 1626 .maxauthsize = AES_BLOCK_SIZE, 1627 }, 1628 .caam = { 1629 .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_GCM, 1630 .nodkp = true, 1631 } 1632 }, 1633 /* single-pass ipsec_esp descriptor */ 1634 { 1635 .aead = { 1636 .base = { 1637 .cra_name = "authenc(hmac(md5),cbc(aes))", 1638 .cra_driver_name = "authenc-hmac-md5-" 1639 "cbc-aes-caam-qi", 1640 .cra_blocksize = AES_BLOCK_SIZE, 1641 }, 1642 .setkey = aead_setkey, 1643 .setauthsize = aead_setauthsize, 1644 .encrypt = aead_encrypt, 1645 .decrypt = aead_decrypt, 1646 .ivsize = AES_BLOCK_SIZE, 1647 .maxauthsize = MD5_DIGEST_SIZE, 1648 }, 1649 .caam = { 1650 .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CBC, 1651 .class2_alg_type = OP_ALG_ALGSEL_MD5 | 1652 OP_ALG_AAI_HMAC_PRECOMP, 1653 } 1654 }, 1655 { 1656 .aead = { 1657 .base = { 1658 .cra_name = "echainiv(authenc(hmac(md5)," 1659 "cbc(aes)))", 1660 .cra_driver_name = "echainiv-authenc-hmac-md5-" 1661 "cbc-aes-caam-qi", 1662 .cra_blocksize = AES_BLOCK_SIZE, 1663 }, 1664 .setkey = aead_setkey, 1665 .setauthsize = aead_setauthsize, 1666 .encrypt = aead_encrypt, 1667 .decrypt = aead_decrypt, 1668 .ivsize = AES_BLOCK_SIZE, 1669 .maxauthsize = MD5_DIGEST_SIZE, 1670 }, 1671 .caam = { 1672 .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CBC, 1673 .class2_alg_type = OP_ALG_ALGSEL_MD5 | 1674 OP_ALG_AAI_HMAC_PRECOMP, 1675 .geniv = true, 1676 } 1677 }, 1678 { 1679 .aead = { 1680 .base = { 1681 .cra_name = "authenc(hmac(sha1),cbc(aes))", 1682 .cra_driver_name = "authenc-hmac-sha1-" 1683 "cbc-aes-caam-qi", 1684 .cra_blocksize = AES_BLOCK_SIZE, 1685 }, 1686 .setkey = aead_setkey, 1687 .setauthsize = aead_setauthsize, 1688 .encrypt = aead_encrypt, 1689 .decrypt = aead_decrypt, 1690 .ivsize = AES_BLOCK_SIZE, 1691 .maxauthsize = SHA1_DIGEST_SIZE, 1692 }, 1693 .caam = { 1694 .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CBC, 1695 .class2_alg_type = OP_ALG_ALGSEL_SHA1 | 1696 OP_ALG_AAI_HMAC_PRECOMP, 1697 } 1698 }, 1699 { 1700 .aead = { 1701 .base = { 1702 .cra_name = "echainiv(authenc(hmac(sha1)," 1703 "cbc(aes)))", 1704 .cra_driver_name = "echainiv-authenc-" 1705 "hmac-sha1-cbc-aes-caam-qi", 1706 .cra_blocksize = AES_BLOCK_SIZE, 1707 }, 1708 .setkey = aead_setkey, 1709 .setauthsize = aead_setauthsize, 1710 .encrypt = aead_encrypt, 1711 .decrypt = aead_decrypt, 1712 .ivsize = AES_BLOCK_SIZE, 1713 .maxauthsize = SHA1_DIGEST_SIZE, 1714 }, 1715 .caam = { 1716 .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CBC, 1717 .class2_alg_type = OP_ALG_ALGSEL_SHA1 | 1718 OP_ALG_AAI_HMAC_PRECOMP, 1719 .geniv = true, 1720 }, 1721 }, 1722 { 1723 .aead = { 1724 .base = { 1725 .cra_name = "authenc(hmac(sha224),cbc(aes))", 1726 .cra_driver_name = "authenc-hmac-sha224-" 1727 "cbc-aes-caam-qi", 1728 .cra_blocksize = AES_BLOCK_SIZE, 1729 }, 1730 .setkey = aead_setkey, 1731 .setauthsize = aead_setauthsize, 1732 .encrypt = aead_encrypt, 1733 .decrypt = aead_decrypt, 1734 .ivsize = AES_BLOCK_SIZE, 1735 .maxauthsize = SHA224_DIGEST_SIZE, 1736 }, 1737 .caam = { 1738 .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CBC, 1739 .class2_alg_type = OP_ALG_ALGSEL_SHA224 | 1740 OP_ALG_AAI_HMAC_PRECOMP, 1741 } 1742 }, 1743 { 1744 .aead = { 1745 .base = { 1746 .cra_name = "echainiv(authenc(hmac(sha224)," 1747 "cbc(aes)))", 1748 .cra_driver_name = "echainiv-authenc-" 1749 "hmac-sha224-cbc-aes-caam-qi", 1750 .cra_blocksize = AES_BLOCK_SIZE, 1751 }, 1752 .setkey = aead_setkey, 1753 .setauthsize = aead_setauthsize, 1754 .encrypt = aead_encrypt, 1755 .decrypt = aead_decrypt, 1756 .ivsize = AES_BLOCK_SIZE, 1757 .maxauthsize = SHA224_DIGEST_SIZE, 1758 }, 1759 .caam = { 1760 .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CBC, 1761 .class2_alg_type = OP_ALG_ALGSEL_SHA224 | 1762 OP_ALG_AAI_HMAC_PRECOMP, 1763 .geniv = true, 1764 } 1765 }, 1766 { 1767 .aead = { 1768 .base = { 1769 .cra_name = "authenc(hmac(sha256),cbc(aes))", 1770 .cra_driver_name = "authenc-hmac-sha256-" 1771 "cbc-aes-caam-qi", 1772 .cra_blocksize = AES_BLOCK_SIZE, 1773 }, 1774 .setkey = aead_setkey, 1775 .setauthsize = aead_setauthsize, 1776 .encrypt = aead_encrypt, 1777 .decrypt = aead_decrypt, 1778 .ivsize = AES_BLOCK_SIZE, 1779 .maxauthsize = SHA256_DIGEST_SIZE, 1780 }, 1781 .caam = { 1782 .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CBC, 1783 .class2_alg_type = OP_ALG_ALGSEL_SHA256 | 1784 OP_ALG_AAI_HMAC_PRECOMP, 1785 } 1786 }, 1787 { 1788 .aead = { 1789 .base = { 1790 .cra_name = "echainiv(authenc(hmac(sha256)," 1791 "cbc(aes)))", 1792 .cra_driver_name = "echainiv-authenc-" 1793 "hmac-sha256-cbc-aes-" 1794 "caam-qi", 1795 .cra_blocksize = AES_BLOCK_SIZE, 1796 }, 1797 .setkey = aead_setkey, 1798 .setauthsize = aead_setauthsize, 1799 .encrypt = aead_encrypt, 1800 .decrypt = aead_decrypt, 1801 .ivsize = AES_BLOCK_SIZE, 1802 .maxauthsize = SHA256_DIGEST_SIZE, 1803 }, 1804 .caam = { 1805 .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CBC, 1806 .class2_alg_type = OP_ALG_ALGSEL_SHA256 | 1807 OP_ALG_AAI_HMAC_PRECOMP, 1808 .geniv = true, 1809 } 1810 }, 1811 { 1812 .aead = { 1813 .base = { 1814 .cra_name = "authenc(hmac(sha384),cbc(aes))", 1815 .cra_driver_name = "authenc-hmac-sha384-" 1816 "cbc-aes-caam-qi", 1817 .cra_blocksize = AES_BLOCK_SIZE, 1818 }, 1819 .setkey = aead_setkey, 1820 .setauthsize = aead_setauthsize, 1821 .encrypt = aead_encrypt, 1822 .decrypt = aead_decrypt, 1823 .ivsize = AES_BLOCK_SIZE, 1824 .maxauthsize = SHA384_DIGEST_SIZE, 1825 }, 1826 .caam = { 1827 .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CBC, 1828 .class2_alg_type = OP_ALG_ALGSEL_SHA384 | 1829 OP_ALG_AAI_HMAC_PRECOMP, 1830 } 1831 }, 1832 { 1833 .aead = { 1834 .base = { 1835 .cra_name = "echainiv(authenc(hmac(sha384)," 1836 "cbc(aes)))", 1837 .cra_driver_name = "echainiv-authenc-" 1838 "hmac-sha384-cbc-aes-" 1839 "caam-qi", 1840 .cra_blocksize = AES_BLOCK_SIZE, 1841 }, 1842 .setkey = aead_setkey, 1843 .setauthsize = aead_setauthsize, 1844 .encrypt = aead_encrypt, 1845 .decrypt = aead_decrypt, 1846 .ivsize = AES_BLOCK_SIZE, 1847 .maxauthsize = SHA384_DIGEST_SIZE, 1848 }, 1849 .caam = { 1850 .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CBC, 1851 .class2_alg_type = OP_ALG_ALGSEL_SHA384 | 1852 OP_ALG_AAI_HMAC_PRECOMP, 1853 .geniv = true, 1854 } 1855 }, 1856 { 1857 .aead = { 1858 .base = { 1859 .cra_name = "authenc(hmac(sha512),cbc(aes))", 1860 .cra_driver_name = "authenc-hmac-sha512-" 1861 "cbc-aes-caam-qi", 1862 .cra_blocksize = AES_BLOCK_SIZE, 1863 }, 1864 .setkey = aead_setkey, 1865 .setauthsize = aead_setauthsize, 1866 .encrypt = aead_encrypt, 1867 .decrypt = aead_decrypt, 1868 .ivsize = AES_BLOCK_SIZE, 1869 .maxauthsize = SHA512_DIGEST_SIZE, 1870 }, 1871 .caam = { 1872 .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CBC, 1873 .class2_alg_type = OP_ALG_ALGSEL_SHA512 | 1874 OP_ALG_AAI_HMAC_PRECOMP, 1875 } 1876 }, 1877 { 1878 .aead = { 1879 .base = { 1880 .cra_name = "echainiv(authenc(hmac(sha512)," 1881 "cbc(aes)))", 1882 .cra_driver_name = "echainiv-authenc-" 1883 "hmac-sha512-cbc-aes-" 1884 "caam-qi", 1885 .cra_blocksize = AES_BLOCK_SIZE, 1886 }, 1887 .setkey = aead_setkey, 1888 .setauthsize = aead_setauthsize, 1889 .encrypt = aead_encrypt, 1890 .decrypt = aead_decrypt, 1891 .ivsize = AES_BLOCK_SIZE, 1892 .maxauthsize = SHA512_DIGEST_SIZE, 1893 }, 1894 .caam = { 1895 .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CBC, 1896 .class2_alg_type = OP_ALG_ALGSEL_SHA512 | 1897 OP_ALG_AAI_HMAC_PRECOMP, 1898 .geniv = true, 1899 } 1900 }, 1901 { 1902 .aead = { 1903 .base = { 1904 .cra_name = "authenc(hmac(md5),cbc(des3_ede))", 1905 .cra_driver_name = "authenc-hmac-md5-" 1906 "cbc-des3_ede-caam-qi", 1907 .cra_blocksize = DES3_EDE_BLOCK_SIZE, 1908 }, 1909 .setkey = des3_aead_setkey, 1910 .setauthsize = aead_setauthsize, 1911 .encrypt = aead_encrypt, 1912 .decrypt = aead_decrypt, 1913 .ivsize = DES3_EDE_BLOCK_SIZE, 1914 .maxauthsize = MD5_DIGEST_SIZE, 1915 }, 1916 .caam = { 1917 .class1_alg_type = OP_ALG_ALGSEL_3DES | OP_ALG_AAI_CBC, 1918 .class2_alg_type = OP_ALG_ALGSEL_MD5 | 1919 OP_ALG_AAI_HMAC_PRECOMP, 1920 } 1921 }, 1922 { 1923 .aead = { 1924 .base = { 1925 .cra_name = "echainiv(authenc(hmac(md5)," 1926 "cbc(des3_ede)))", 1927 .cra_driver_name = "echainiv-authenc-hmac-md5-" 1928 "cbc-des3_ede-caam-qi", 1929 .cra_blocksize = DES3_EDE_BLOCK_SIZE, 1930 }, 1931 .setkey = des3_aead_setkey, 1932 .setauthsize = aead_setauthsize, 1933 .encrypt = aead_encrypt, 1934 .decrypt = aead_decrypt, 1935 .ivsize = DES3_EDE_BLOCK_SIZE, 1936 .maxauthsize = MD5_DIGEST_SIZE, 1937 }, 1938 .caam = { 1939 .class1_alg_type = OP_ALG_ALGSEL_3DES | OP_ALG_AAI_CBC, 1940 .class2_alg_type = OP_ALG_ALGSEL_MD5 | 1941 OP_ALG_AAI_HMAC_PRECOMP, 1942 .geniv = true, 1943 } 1944 }, 1945 { 1946 .aead = { 1947 .base = { 1948 .cra_name = "authenc(hmac(sha1)," 1949 "cbc(des3_ede))", 1950 .cra_driver_name = "authenc-hmac-sha1-" 1951 "cbc-des3_ede-caam-qi", 1952 .cra_blocksize = DES3_EDE_BLOCK_SIZE, 1953 }, 1954 .setkey = des3_aead_setkey, 1955 .setauthsize = aead_setauthsize, 1956 .encrypt = aead_encrypt, 1957 .decrypt = aead_decrypt, 1958 .ivsize = DES3_EDE_BLOCK_SIZE, 1959 .maxauthsize = SHA1_DIGEST_SIZE, 1960 }, 1961 .caam = { 1962 .class1_alg_type = OP_ALG_ALGSEL_3DES | OP_ALG_AAI_CBC, 1963 .class2_alg_type = OP_ALG_ALGSEL_SHA1 | 1964 OP_ALG_AAI_HMAC_PRECOMP, 1965 }, 1966 }, 1967 { 1968 .aead = { 1969 .base = { 1970 .cra_name = "echainiv(authenc(hmac(sha1)," 1971 "cbc(des3_ede)))", 1972 .cra_driver_name = "echainiv-authenc-" 1973 "hmac-sha1-" 1974 "cbc-des3_ede-caam-qi", 1975 .cra_blocksize = DES3_EDE_BLOCK_SIZE, 1976 }, 1977 .setkey = des3_aead_setkey, 1978 .setauthsize = aead_setauthsize, 1979 .encrypt = aead_encrypt, 1980 .decrypt = aead_decrypt, 1981 .ivsize = DES3_EDE_BLOCK_SIZE, 1982 .maxauthsize = SHA1_DIGEST_SIZE, 1983 }, 1984 .caam = { 1985 .class1_alg_type = OP_ALG_ALGSEL_3DES | OP_ALG_AAI_CBC, 1986 .class2_alg_type = OP_ALG_ALGSEL_SHA1 | 1987 OP_ALG_AAI_HMAC_PRECOMP, 1988 .geniv = true, 1989 } 1990 }, 1991 { 1992 .aead = { 1993 .base = { 1994 .cra_name = "authenc(hmac(sha224)," 1995 "cbc(des3_ede))", 1996 .cra_driver_name = "authenc-hmac-sha224-" 1997 "cbc-des3_ede-caam-qi", 1998 .cra_blocksize = DES3_EDE_BLOCK_SIZE, 1999 }, 2000 .setkey = des3_aead_setkey, 2001 .setauthsize = aead_setauthsize, 2002 .encrypt = aead_encrypt, 2003 .decrypt = aead_decrypt, 2004 .ivsize = DES3_EDE_BLOCK_SIZE, 2005 .maxauthsize = SHA224_DIGEST_SIZE, 2006 }, 2007 .caam = { 2008 .class1_alg_type = OP_ALG_ALGSEL_3DES | OP_ALG_AAI_CBC, 2009 .class2_alg_type = OP_ALG_ALGSEL_SHA224 | 2010 OP_ALG_AAI_HMAC_PRECOMP, 2011 }, 2012 }, 2013 { 2014 .aead = { 2015 .base = { 2016 .cra_name = "echainiv(authenc(hmac(sha224)," 2017 "cbc(des3_ede)))", 2018 .cra_driver_name = "echainiv-authenc-" 2019 "hmac-sha224-" 2020 "cbc-des3_ede-caam-qi", 2021 .cra_blocksize = DES3_EDE_BLOCK_SIZE, 2022 }, 2023 .setkey = des3_aead_setkey, 2024 .setauthsize = aead_setauthsize, 2025 .encrypt = aead_encrypt, 2026 .decrypt = aead_decrypt, 2027 .ivsize = DES3_EDE_BLOCK_SIZE, 2028 .maxauthsize = SHA224_DIGEST_SIZE, 2029 }, 2030 .caam = { 2031 .class1_alg_type = OP_ALG_ALGSEL_3DES | OP_ALG_AAI_CBC, 2032 .class2_alg_type = OP_ALG_ALGSEL_SHA224 | 2033 OP_ALG_AAI_HMAC_PRECOMP, 2034 .geniv = true, 2035 } 2036 }, 2037 { 2038 .aead = { 2039 .base = { 2040 .cra_name = "authenc(hmac(sha256)," 2041 "cbc(des3_ede))", 2042 .cra_driver_name = "authenc-hmac-sha256-" 2043 "cbc-des3_ede-caam-qi", 2044 .cra_blocksize = DES3_EDE_BLOCK_SIZE, 2045 }, 2046 .setkey = des3_aead_setkey, 2047 .setauthsize = aead_setauthsize, 2048 .encrypt = aead_encrypt, 2049 .decrypt = aead_decrypt, 2050 .ivsize = DES3_EDE_BLOCK_SIZE, 2051 .maxauthsize = SHA256_DIGEST_SIZE, 2052 }, 2053 .caam = { 2054 .class1_alg_type = OP_ALG_ALGSEL_3DES | OP_ALG_AAI_CBC, 2055 .class2_alg_type = OP_ALG_ALGSEL_SHA256 | 2056 OP_ALG_AAI_HMAC_PRECOMP, 2057 }, 2058 }, 2059 { 2060 .aead = { 2061 .base = { 2062 .cra_name = "echainiv(authenc(hmac(sha256)," 2063 "cbc(des3_ede)))", 2064 .cra_driver_name = "echainiv-authenc-" 2065 "hmac-sha256-" 2066 "cbc-des3_ede-caam-qi", 2067 .cra_blocksize = DES3_EDE_BLOCK_SIZE, 2068 }, 2069 .setkey = des3_aead_setkey, 2070 .setauthsize = aead_setauthsize, 2071 .encrypt = aead_encrypt, 2072 .decrypt = aead_decrypt, 2073 .ivsize = DES3_EDE_BLOCK_SIZE, 2074 .maxauthsize = SHA256_DIGEST_SIZE, 2075 }, 2076 .caam = { 2077 .class1_alg_type = OP_ALG_ALGSEL_3DES | OP_ALG_AAI_CBC, 2078 .class2_alg_type = OP_ALG_ALGSEL_SHA256 | 2079 OP_ALG_AAI_HMAC_PRECOMP, 2080 .geniv = true, 2081 } 2082 }, 2083 { 2084 .aead = { 2085 .base = { 2086 .cra_name = "authenc(hmac(sha384)," 2087 "cbc(des3_ede))", 2088 .cra_driver_name = "authenc-hmac-sha384-" 2089 "cbc-des3_ede-caam-qi", 2090 .cra_blocksize = DES3_EDE_BLOCK_SIZE, 2091 }, 2092 .setkey = des3_aead_setkey, 2093 .setauthsize = aead_setauthsize, 2094 .encrypt = aead_encrypt, 2095 .decrypt = aead_decrypt, 2096 .ivsize = DES3_EDE_BLOCK_SIZE, 2097 .maxauthsize = SHA384_DIGEST_SIZE, 2098 }, 2099 .caam = { 2100 .class1_alg_type = OP_ALG_ALGSEL_3DES | OP_ALG_AAI_CBC, 2101 .class2_alg_type = OP_ALG_ALGSEL_SHA384 | 2102 OP_ALG_AAI_HMAC_PRECOMP, 2103 }, 2104 }, 2105 { 2106 .aead = { 2107 .base = { 2108 .cra_name = "echainiv(authenc(hmac(sha384)," 2109 "cbc(des3_ede)))", 2110 .cra_driver_name = "echainiv-authenc-" 2111 "hmac-sha384-" 2112 "cbc-des3_ede-caam-qi", 2113 .cra_blocksize = DES3_EDE_BLOCK_SIZE, 2114 }, 2115 .setkey = des3_aead_setkey, 2116 .setauthsize = aead_setauthsize, 2117 .encrypt = aead_encrypt, 2118 .decrypt = aead_decrypt, 2119 .ivsize = DES3_EDE_BLOCK_SIZE, 2120 .maxauthsize = SHA384_DIGEST_SIZE, 2121 }, 2122 .caam = { 2123 .class1_alg_type = OP_ALG_ALGSEL_3DES | OP_ALG_AAI_CBC, 2124 .class2_alg_type = OP_ALG_ALGSEL_SHA384 | 2125 OP_ALG_AAI_HMAC_PRECOMP, 2126 .geniv = true, 2127 } 2128 }, 2129 { 2130 .aead = { 2131 .base = { 2132 .cra_name = "authenc(hmac(sha512)," 2133 "cbc(des3_ede))", 2134 .cra_driver_name = "authenc-hmac-sha512-" 2135 "cbc-des3_ede-caam-qi", 2136 .cra_blocksize = DES3_EDE_BLOCK_SIZE, 2137 }, 2138 .setkey = des3_aead_setkey, 2139 .setauthsize = aead_setauthsize, 2140 .encrypt = aead_encrypt, 2141 .decrypt = aead_decrypt, 2142 .ivsize = DES3_EDE_BLOCK_SIZE, 2143 .maxauthsize = SHA512_DIGEST_SIZE, 2144 }, 2145 .caam = { 2146 .class1_alg_type = OP_ALG_ALGSEL_3DES | OP_ALG_AAI_CBC, 2147 .class2_alg_type = OP_ALG_ALGSEL_SHA512 | 2148 OP_ALG_AAI_HMAC_PRECOMP, 2149 }, 2150 }, 2151 { 2152 .aead = { 2153 .base = { 2154 .cra_name = "echainiv(authenc(hmac(sha512)," 2155 "cbc(des3_ede)))", 2156 .cra_driver_name = "echainiv-authenc-" 2157 "hmac-sha512-" 2158 "cbc-des3_ede-caam-qi", 2159 .cra_blocksize = DES3_EDE_BLOCK_SIZE, 2160 }, 2161 .setkey = des3_aead_setkey, 2162 .setauthsize = aead_setauthsize, 2163 .encrypt = aead_encrypt, 2164 .decrypt = aead_decrypt, 2165 .ivsize = DES3_EDE_BLOCK_SIZE, 2166 .maxauthsize = SHA512_DIGEST_SIZE, 2167 }, 2168 .caam = { 2169 .class1_alg_type = OP_ALG_ALGSEL_3DES | OP_ALG_AAI_CBC, 2170 .class2_alg_type = OP_ALG_ALGSEL_SHA512 | 2171 OP_ALG_AAI_HMAC_PRECOMP, 2172 .geniv = true, 2173 } 2174 }, 2175 { 2176 .aead = { 2177 .base = { 2178 .cra_name = "authenc(hmac(md5),cbc(des))", 2179 .cra_driver_name = "authenc-hmac-md5-" 2180 "cbc-des-caam-qi", 2181 .cra_blocksize = DES_BLOCK_SIZE, 2182 }, 2183 .setkey = aead_setkey, 2184 .setauthsize = aead_setauthsize, 2185 .encrypt = aead_encrypt, 2186 .decrypt = aead_decrypt, 2187 .ivsize = DES_BLOCK_SIZE, 2188 .maxauthsize = MD5_DIGEST_SIZE, 2189 }, 2190 .caam = { 2191 .class1_alg_type = OP_ALG_ALGSEL_DES | OP_ALG_AAI_CBC, 2192 .class2_alg_type = OP_ALG_ALGSEL_MD5 | 2193 OP_ALG_AAI_HMAC_PRECOMP, 2194 }, 2195 }, 2196 { 2197 .aead = { 2198 .base = { 2199 .cra_name = "echainiv(authenc(hmac(md5)," 2200 "cbc(des)))", 2201 .cra_driver_name = "echainiv-authenc-hmac-md5-" 2202 "cbc-des-caam-qi", 2203 .cra_blocksize = DES_BLOCK_SIZE, 2204 }, 2205 .setkey = aead_setkey, 2206 .setauthsize = aead_setauthsize, 2207 .encrypt = aead_encrypt, 2208 .decrypt = aead_decrypt, 2209 .ivsize = DES_BLOCK_SIZE, 2210 .maxauthsize = MD5_DIGEST_SIZE, 2211 }, 2212 .caam = { 2213 .class1_alg_type = OP_ALG_ALGSEL_DES | OP_ALG_AAI_CBC, 2214 .class2_alg_type = OP_ALG_ALGSEL_MD5 | 2215 OP_ALG_AAI_HMAC_PRECOMP, 2216 .geniv = true, 2217 } 2218 }, 2219 { 2220 .aead = { 2221 .base = { 2222 .cra_name = "authenc(hmac(sha1),cbc(des))", 2223 .cra_driver_name = "authenc-hmac-sha1-" 2224 "cbc-des-caam-qi", 2225 .cra_blocksize = DES_BLOCK_SIZE, 2226 }, 2227 .setkey = aead_setkey, 2228 .setauthsize = aead_setauthsize, 2229 .encrypt = aead_encrypt, 2230 .decrypt = aead_decrypt, 2231 .ivsize = DES_BLOCK_SIZE, 2232 .maxauthsize = SHA1_DIGEST_SIZE, 2233 }, 2234 .caam = { 2235 .class1_alg_type = OP_ALG_ALGSEL_DES | OP_ALG_AAI_CBC, 2236 .class2_alg_type = OP_ALG_ALGSEL_SHA1 | 2237 OP_ALG_AAI_HMAC_PRECOMP, 2238 }, 2239 }, 2240 { 2241 .aead = { 2242 .base = { 2243 .cra_name = "echainiv(authenc(hmac(sha1)," 2244 "cbc(des)))", 2245 .cra_driver_name = "echainiv-authenc-" 2246 "hmac-sha1-cbc-des-caam-qi", 2247 .cra_blocksize = DES_BLOCK_SIZE, 2248 }, 2249 .setkey = aead_setkey, 2250 .setauthsize = aead_setauthsize, 2251 .encrypt = aead_encrypt, 2252 .decrypt = aead_decrypt, 2253 .ivsize = DES_BLOCK_SIZE, 2254 .maxauthsize = SHA1_DIGEST_SIZE, 2255 }, 2256 .caam = { 2257 .class1_alg_type = OP_ALG_ALGSEL_DES | OP_ALG_AAI_CBC, 2258 .class2_alg_type = OP_ALG_ALGSEL_SHA1 | 2259 OP_ALG_AAI_HMAC_PRECOMP, 2260 .geniv = true, 2261 } 2262 }, 2263 { 2264 .aead = { 2265 .base = { 2266 .cra_name = "authenc(hmac(sha224),cbc(des))", 2267 .cra_driver_name = "authenc-hmac-sha224-" 2268 "cbc-des-caam-qi", 2269 .cra_blocksize = DES_BLOCK_SIZE, 2270 }, 2271 .setkey = aead_setkey, 2272 .setauthsize = aead_setauthsize, 2273 .encrypt = aead_encrypt, 2274 .decrypt = aead_decrypt, 2275 .ivsize = DES_BLOCK_SIZE, 2276 .maxauthsize = SHA224_DIGEST_SIZE, 2277 }, 2278 .caam = { 2279 .class1_alg_type = OP_ALG_ALGSEL_DES | OP_ALG_AAI_CBC, 2280 .class2_alg_type = OP_ALG_ALGSEL_SHA224 | 2281 OP_ALG_AAI_HMAC_PRECOMP, 2282 }, 2283 }, 2284 { 2285 .aead = { 2286 .base = { 2287 .cra_name = "echainiv(authenc(hmac(sha224)," 2288 "cbc(des)))", 2289 .cra_driver_name = "echainiv-authenc-" 2290 "hmac-sha224-cbc-des-" 2291 "caam-qi", 2292 .cra_blocksize = DES_BLOCK_SIZE, 2293 }, 2294 .setkey = aead_setkey, 2295 .setauthsize = aead_setauthsize, 2296 .encrypt = aead_encrypt, 2297 .decrypt = aead_decrypt, 2298 .ivsize = DES_BLOCK_SIZE, 2299 .maxauthsize = SHA224_DIGEST_SIZE, 2300 }, 2301 .caam = { 2302 .class1_alg_type = OP_ALG_ALGSEL_DES | OP_ALG_AAI_CBC, 2303 .class2_alg_type = OP_ALG_ALGSEL_SHA224 | 2304 OP_ALG_AAI_HMAC_PRECOMP, 2305 .geniv = true, 2306 } 2307 }, 2308 { 2309 .aead = { 2310 .base = { 2311 .cra_name = "authenc(hmac(sha256),cbc(des))", 2312 .cra_driver_name = "authenc-hmac-sha256-" 2313 "cbc-des-caam-qi", 2314 .cra_blocksize = DES_BLOCK_SIZE, 2315 }, 2316 .setkey = aead_setkey, 2317 .setauthsize = aead_setauthsize, 2318 .encrypt = aead_encrypt, 2319 .decrypt = aead_decrypt, 2320 .ivsize = DES_BLOCK_SIZE, 2321 .maxauthsize = SHA256_DIGEST_SIZE, 2322 }, 2323 .caam = { 2324 .class1_alg_type = OP_ALG_ALGSEL_DES | OP_ALG_AAI_CBC, 2325 .class2_alg_type = OP_ALG_ALGSEL_SHA256 | 2326 OP_ALG_AAI_HMAC_PRECOMP, 2327 }, 2328 }, 2329 { 2330 .aead = { 2331 .base = { 2332 .cra_name = "echainiv(authenc(hmac(sha256)," 2333 "cbc(des)))", 2334 .cra_driver_name = "echainiv-authenc-" 2335 "hmac-sha256-cbc-des-" 2336 "caam-qi", 2337 .cra_blocksize = DES_BLOCK_SIZE, 2338 }, 2339 .setkey = aead_setkey, 2340 .setauthsize = aead_setauthsize, 2341 .encrypt = aead_encrypt, 2342 .decrypt = aead_decrypt, 2343 .ivsize = DES_BLOCK_SIZE, 2344 .maxauthsize = SHA256_DIGEST_SIZE, 2345 }, 2346 .caam = { 2347 .class1_alg_type = OP_ALG_ALGSEL_DES | OP_ALG_AAI_CBC, 2348 .class2_alg_type = OP_ALG_ALGSEL_SHA256 | 2349 OP_ALG_AAI_HMAC_PRECOMP, 2350 .geniv = true, 2351 }, 2352 }, 2353 { 2354 .aead = { 2355 .base = { 2356 .cra_name = "authenc(hmac(sha384),cbc(des))", 2357 .cra_driver_name = "authenc-hmac-sha384-" 2358 "cbc-des-caam-qi", 2359 .cra_blocksize = DES_BLOCK_SIZE, 2360 }, 2361 .setkey = aead_setkey, 2362 .setauthsize = aead_setauthsize, 2363 .encrypt = aead_encrypt, 2364 .decrypt = aead_decrypt, 2365 .ivsize = DES_BLOCK_SIZE, 2366 .maxauthsize = SHA384_DIGEST_SIZE, 2367 }, 2368 .caam = { 2369 .class1_alg_type = OP_ALG_ALGSEL_DES | OP_ALG_AAI_CBC, 2370 .class2_alg_type = OP_ALG_ALGSEL_SHA384 | 2371 OP_ALG_AAI_HMAC_PRECOMP, 2372 }, 2373 }, 2374 { 2375 .aead = { 2376 .base = { 2377 .cra_name = "echainiv(authenc(hmac(sha384)," 2378 "cbc(des)))", 2379 .cra_driver_name = "echainiv-authenc-" 2380 "hmac-sha384-cbc-des-" 2381 "caam-qi", 2382 .cra_blocksize = DES_BLOCK_SIZE, 2383 }, 2384 .setkey = aead_setkey, 2385 .setauthsize = aead_setauthsize, 2386 .encrypt = aead_encrypt, 2387 .decrypt = aead_decrypt, 2388 .ivsize = DES_BLOCK_SIZE, 2389 .maxauthsize = SHA384_DIGEST_SIZE, 2390 }, 2391 .caam = { 2392 .class1_alg_type = OP_ALG_ALGSEL_DES | OP_ALG_AAI_CBC, 2393 .class2_alg_type = OP_ALG_ALGSEL_SHA384 | 2394 OP_ALG_AAI_HMAC_PRECOMP, 2395 .geniv = true, 2396 } 2397 }, 2398 { 2399 .aead = { 2400 .base = { 2401 .cra_name = "authenc(hmac(sha512),cbc(des))", 2402 .cra_driver_name = "authenc-hmac-sha512-" 2403 "cbc-des-caam-qi", 2404 .cra_blocksize = DES_BLOCK_SIZE, 2405 }, 2406 .setkey = aead_setkey, 2407 .setauthsize = aead_setauthsize, 2408 .encrypt = aead_encrypt, 2409 .decrypt = aead_decrypt, 2410 .ivsize = DES_BLOCK_SIZE, 2411 .maxauthsize = SHA512_DIGEST_SIZE, 2412 }, 2413 .caam = { 2414 .class1_alg_type = OP_ALG_ALGSEL_DES | OP_ALG_AAI_CBC, 2415 .class2_alg_type = OP_ALG_ALGSEL_SHA512 | 2416 OP_ALG_AAI_HMAC_PRECOMP, 2417 } 2418 }, 2419 { 2420 .aead = { 2421 .base = { 2422 .cra_name = "echainiv(authenc(hmac(sha512)," 2423 "cbc(des)))", 2424 .cra_driver_name = "echainiv-authenc-" 2425 "hmac-sha512-cbc-des-" 2426 "caam-qi", 2427 .cra_blocksize = DES_BLOCK_SIZE, 2428 }, 2429 .setkey = aead_setkey, 2430 .setauthsize = aead_setauthsize, 2431 .encrypt = aead_encrypt, 2432 .decrypt = aead_decrypt, 2433 .ivsize = DES_BLOCK_SIZE, 2434 .maxauthsize = SHA512_DIGEST_SIZE, 2435 }, 2436 .caam = { 2437 .class1_alg_type = OP_ALG_ALGSEL_DES | OP_ALG_AAI_CBC, 2438 .class2_alg_type = OP_ALG_ALGSEL_SHA512 | 2439 OP_ALG_AAI_HMAC_PRECOMP, 2440 .geniv = true, 2441 } 2442 }, 2443 }; 2444 2445 static int caam_init_common(struct caam_ctx *ctx, struct caam_alg_entry *caam, 2446 bool uses_dkp) 2447 { 2448 struct caam_drv_private *priv; 2449 struct device *dev; 2450 2451 /* 2452 * distribute tfms across job rings to ensure in-order 2453 * crypto request processing per tfm 2454 */ 2455 ctx->jrdev = caam_jr_alloc(); 2456 if (IS_ERR(ctx->jrdev)) { 2457 pr_err("Job Ring Device allocation for transform failed\n"); 2458 return PTR_ERR(ctx->jrdev); 2459 } 2460 2461 dev = ctx->jrdev->parent; 2462 priv = dev_get_drvdata(dev); 2463 if (priv->era >= 6 && uses_dkp) 2464 ctx->dir = DMA_BIDIRECTIONAL; 2465 else 2466 ctx->dir = DMA_TO_DEVICE; 2467 2468 ctx->key_dma = dma_map_single(dev, ctx->key, sizeof(ctx->key), 2469 ctx->dir); 2470 if (dma_mapping_error(dev, ctx->key_dma)) { 2471 dev_err(dev, "unable to map key\n"); 2472 caam_jr_free(ctx->jrdev); 2473 return -ENOMEM; 2474 } 2475 2476 /* copy descriptor header template value */ 2477 ctx->cdata.algtype = OP_TYPE_CLASS1_ALG | caam->class1_alg_type; 2478 ctx->adata.algtype = OP_TYPE_CLASS2_ALG | caam->class2_alg_type; 2479 2480 ctx->qidev = dev; 2481 2482 spin_lock_init(&ctx->lock); 2483 ctx->drv_ctx[ENCRYPT] = NULL; 2484 ctx->drv_ctx[DECRYPT] = NULL; 2485 2486 return 0; 2487 } 2488 2489 static int caam_cra_init(struct crypto_skcipher *tfm) 2490 { 2491 struct skcipher_alg *alg = crypto_skcipher_alg(tfm); 2492 struct caam_skcipher_alg *caam_alg = 2493 container_of(alg, typeof(*caam_alg), skcipher); 2494 struct caam_ctx *ctx = crypto_skcipher_ctx_dma(tfm); 2495 u32 alg_aai = caam_alg->caam.class1_alg_type & OP_ALG_AAI_MASK; 2496 int ret = 0; 2497 2498 if (alg_aai == OP_ALG_AAI_XTS) { 2499 const char *tfm_name = crypto_tfm_alg_name(&tfm->base); 2500 struct crypto_skcipher *fallback; 2501 2502 fallback = crypto_alloc_skcipher(tfm_name, 0, 2503 CRYPTO_ALG_NEED_FALLBACK); 2504 if (IS_ERR(fallback)) { 2505 pr_err("Failed to allocate %s fallback: %ld\n", 2506 tfm_name, PTR_ERR(fallback)); 2507 return PTR_ERR(fallback); 2508 } 2509 2510 ctx->fallback = fallback; 2511 crypto_skcipher_set_reqsize(tfm, sizeof(struct caam_skcipher_req_ctx) + 2512 crypto_skcipher_reqsize(fallback)); 2513 } 2514 2515 ret = caam_init_common(ctx, &caam_alg->caam, false); 2516 if (ret && ctx->fallback) 2517 crypto_free_skcipher(ctx->fallback); 2518 2519 return ret; 2520 } 2521 2522 static int caam_aead_init(struct crypto_aead *tfm) 2523 { 2524 struct aead_alg *alg = crypto_aead_alg(tfm); 2525 struct caam_aead_alg *caam_alg = container_of(alg, typeof(*caam_alg), 2526 aead); 2527 struct caam_ctx *ctx = crypto_aead_ctx_dma(tfm); 2528 2529 return caam_init_common(ctx, &caam_alg->caam, !caam_alg->caam.nodkp); 2530 } 2531 2532 static void caam_exit_common(struct caam_ctx *ctx) 2533 { 2534 caam_drv_ctx_rel(ctx->drv_ctx[ENCRYPT]); 2535 caam_drv_ctx_rel(ctx->drv_ctx[DECRYPT]); 2536 2537 dma_unmap_single(ctx->jrdev->parent, ctx->key_dma, sizeof(ctx->key), 2538 ctx->dir); 2539 2540 caam_jr_free(ctx->jrdev); 2541 } 2542 2543 static void caam_cra_exit(struct crypto_skcipher *tfm) 2544 { 2545 struct caam_ctx *ctx = crypto_skcipher_ctx_dma(tfm); 2546 2547 if (ctx->fallback) 2548 crypto_free_skcipher(ctx->fallback); 2549 caam_exit_common(ctx); 2550 } 2551 2552 static void caam_aead_exit(struct crypto_aead *tfm) 2553 { 2554 caam_exit_common(crypto_aead_ctx_dma(tfm)); 2555 } 2556 2557 void caam_qi_algapi_exit(void) 2558 { 2559 int i; 2560 2561 for (i = 0; i < ARRAY_SIZE(driver_aeads); i++) { 2562 struct caam_aead_alg *t_alg = driver_aeads + i; 2563 2564 if (t_alg->registered) 2565 crypto_unregister_aead(&t_alg->aead); 2566 } 2567 2568 for (i = 0; i < ARRAY_SIZE(driver_algs); i++) { 2569 struct caam_skcipher_alg *t_alg = driver_algs + i; 2570 2571 if (t_alg->registered) 2572 crypto_unregister_skcipher(&t_alg->skcipher); 2573 } 2574 } 2575 2576 static void caam_skcipher_alg_init(struct caam_skcipher_alg *t_alg) 2577 { 2578 struct skcipher_alg *alg = &t_alg->skcipher; 2579 2580 alg->base.cra_module = THIS_MODULE; 2581 alg->base.cra_priority = CAAM_CRA_PRIORITY; 2582 alg->base.cra_ctxsize = sizeof(struct caam_ctx) + crypto_dma_padding(); 2583 alg->base.cra_flags |= (CRYPTO_ALG_ASYNC | CRYPTO_ALG_ALLOCATES_MEMORY | 2584 CRYPTO_ALG_KERN_DRIVER_ONLY); 2585 2586 alg->init = caam_cra_init; 2587 alg->exit = caam_cra_exit; 2588 } 2589 2590 static void caam_aead_alg_init(struct caam_aead_alg *t_alg) 2591 { 2592 struct aead_alg *alg = &t_alg->aead; 2593 2594 alg->base.cra_module = THIS_MODULE; 2595 alg->base.cra_priority = CAAM_CRA_PRIORITY; 2596 alg->base.cra_ctxsize = sizeof(struct caam_ctx) + crypto_dma_padding(); 2597 alg->base.cra_flags = CRYPTO_ALG_ASYNC | CRYPTO_ALG_ALLOCATES_MEMORY | 2598 CRYPTO_ALG_KERN_DRIVER_ONLY; 2599 2600 alg->init = caam_aead_init; 2601 alg->exit = caam_aead_exit; 2602 } 2603 2604 int caam_qi_algapi_init(struct device *ctrldev) 2605 { 2606 struct caam_drv_private *priv = dev_get_drvdata(ctrldev); 2607 int i = 0, err = 0; 2608 u32 aes_vid, aes_inst, des_inst, md_vid, md_inst; 2609 unsigned int md_limit = SHA512_DIGEST_SIZE; 2610 bool registered = false; 2611 2612 /* Make sure this runs only on (DPAA 1.x) QI */ 2613 if (!priv->qi_present || caam_dpaa2) 2614 return 0; 2615 2616 /* 2617 * Register crypto algorithms the device supports. 2618 * First, detect presence and attributes of DES, AES, and MD blocks. 2619 */ 2620 if (priv->era < 10) { 2621 u32 cha_vid, cha_inst; 2622 2623 cha_vid = rd_reg32(&priv->ctrl->perfmon.cha_id_ls); 2624 aes_vid = cha_vid & CHA_ID_LS_AES_MASK; 2625 md_vid = (cha_vid & CHA_ID_LS_MD_MASK) >> CHA_ID_LS_MD_SHIFT; 2626 2627 cha_inst = rd_reg32(&priv->ctrl->perfmon.cha_num_ls); 2628 des_inst = (cha_inst & CHA_ID_LS_DES_MASK) >> 2629 CHA_ID_LS_DES_SHIFT; 2630 aes_inst = cha_inst & CHA_ID_LS_AES_MASK; 2631 md_inst = (cha_inst & CHA_ID_LS_MD_MASK) >> CHA_ID_LS_MD_SHIFT; 2632 } else { 2633 u32 aesa, mdha; 2634 2635 aesa = rd_reg32(&priv->ctrl->vreg.aesa); 2636 mdha = rd_reg32(&priv->ctrl->vreg.mdha); 2637 2638 aes_vid = (aesa & CHA_VER_VID_MASK) >> CHA_VER_VID_SHIFT; 2639 md_vid = (mdha & CHA_VER_VID_MASK) >> CHA_VER_VID_SHIFT; 2640 2641 des_inst = rd_reg32(&priv->ctrl->vreg.desa) & CHA_VER_NUM_MASK; 2642 aes_inst = aesa & CHA_VER_NUM_MASK; 2643 md_inst = mdha & CHA_VER_NUM_MASK; 2644 } 2645 2646 /* If MD is present, limit digest size based on LP256 */ 2647 if (md_inst && md_vid == CHA_VER_VID_MD_LP256) 2648 md_limit = SHA256_DIGEST_SIZE; 2649 2650 for (i = 0; i < ARRAY_SIZE(driver_algs); i++) { 2651 struct caam_skcipher_alg *t_alg = driver_algs + i; 2652 u32 alg_sel = t_alg->caam.class1_alg_type & OP_ALG_ALGSEL_MASK; 2653 2654 /* Skip DES algorithms if not supported by device */ 2655 if (!des_inst && 2656 ((alg_sel == OP_ALG_ALGSEL_3DES) || 2657 (alg_sel == OP_ALG_ALGSEL_DES))) 2658 continue; 2659 2660 /* Skip AES algorithms if not supported by device */ 2661 if (!aes_inst && (alg_sel == OP_ALG_ALGSEL_AES)) 2662 continue; 2663 2664 caam_skcipher_alg_init(t_alg); 2665 2666 err = crypto_register_skcipher(&t_alg->skcipher); 2667 if (err) { 2668 dev_warn(ctrldev, "%s alg registration failed\n", 2669 t_alg->skcipher.base.cra_driver_name); 2670 continue; 2671 } 2672 2673 t_alg->registered = true; 2674 registered = true; 2675 } 2676 2677 for (i = 0; i < ARRAY_SIZE(driver_aeads); i++) { 2678 struct caam_aead_alg *t_alg = driver_aeads + i; 2679 u32 c1_alg_sel = t_alg->caam.class1_alg_type & 2680 OP_ALG_ALGSEL_MASK; 2681 u32 c2_alg_sel = t_alg->caam.class2_alg_type & 2682 OP_ALG_ALGSEL_MASK; 2683 u32 alg_aai = t_alg->caam.class1_alg_type & OP_ALG_AAI_MASK; 2684 2685 /* Skip DES algorithms if not supported by device */ 2686 if (!des_inst && 2687 ((c1_alg_sel == OP_ALG_ALGSEL_3DES) || 2688 (c1_alg_sel == OP_ALG_ALGSEL_DES))) 2689 continue; 2690 2691 /* Skip AES algorithms if not supported by device */ 2692 if (!aes_inst && (c1_alg_sel == OP_ALG_ALGSEL_AES)) 2693 continue; 2694 2695 /* 2696 * Check support for AES algorithms not available 2697 * on LP devices. 2698 */ 2699 if (aes_vid == CHA_VER_VID_AES_LP && alg_aai == OP_ALG_AAI_GCM) 2700 continue; 2701 2702 /* 2703 * Skip algorithms requiring message digests 2704 * if MD or MD size is not supported by device. 2705 */ 2706 if (c2_alg_sel && 2707 (!md_inst || (t_alg->aead.maxauthsize > md_limit))) 2708 continue; 2709 2710 caam_aead_alg_init(t_alg); 2711 2712 err = crypto_register_aead(&t_alg->aead); 2713 if (err) { 2714 pr_warn("%s alg registration failed\n", 2715 t_alg->aead.base.cra_driver_name); 2716 continue; 2717 } 2718 2719 t_alg->registered = true; 2720 registered = true; 2721 } 2722 2723 if (registered) 2724 dev_info(ctrldev, "algorithms registered in /proc/crypto\n"); 2725 2726 return err; 2727 } 2728