1 // SPDX-License-Identifier: GPL-2.0+ 2 /* 3 * Freescale FSL CAAM support for crypto API over QI backend. 4 * Based on caamalg.c 5 * 6 * Copyright 2013-2016 Freescale Semiconductor, Inc. 7 * Copyright 2016-2019 NXP 8 */ 9 10 #include "compat.h" 11 #include "ctrl.h" 12 #include "regs.h" 13 #include "intern.h" 14 #include "desc_constr.h" 15 #include "error.h" 16 #include "sg_sw_qm.h" 17 #include "key_gen.h" 18 #include "qi.h" 19 #include "jr.h" 20 #include "caamalg_desc.h" 21 22 /* 23 * crypto alg 24 */ 25 #define CAAM_CRA_PRIORITY 2000 26 /* max key is sum of AES_MAX_KEY_SIZE, max split key size */ 27 #define CAAM_MAX_KEY_SIZE (AES_MAX_KEY_SIZE + \ 28 SHA512_DIGEST_SIZE * 2) 29 30 #define DESC_MAX_USED_BYTES (DESC_QI_AEAD_GIVENC_LEN + \ 31 CAAM_MAX_KEY_SIZE) 32 #define DESC_MAX_USED_LEN (DESC_MAX_USED_BYTES / CAAM_CMD_SZ) 33 34 struct caam_alg_entry { 35 int class1_alg_type; 36 int class2_alg_type; 37 bool rfc3686; 38 bool geniv; 39 bool nodkp; 40 }; 41 42 struct caam_aead_alg { 43 struct aead_alg aead; 44 struct caam_alg_entry caam; 45 bool registered; 46 }; 47 48 struct caam_skcipher_alg { 49 struct skcipher_alg skcipher; 50 struct caam_alg_entry caam; 51 bool registered; 52 }; 53 54 /* 55 * per-session context 56 */ 57 struct caam_ctx { 58 struct device *jrdev; 59 u32 sh_desc_enc[DESC_MAX_USED_LEN]; 60 u32 sh_desc_dec[DESC_MAX_USED_LEN]; 61 u8 key[CAAM_MAX_KEY_SIZE]; 62 dma_addr_t key_dma; 63 enum dma_data_direction dir; 64 struct alginfo adata; 65 struct alginfo cdata; 66 unsigned int authsize; 67 struct device *qidev; 68 spinlock_t lock; /* Protects multiple init of driver context */ 69 struct caam_drv_ctx *drv_ctx[NUM_OP]; 70 }; 71 72 static int aead_set_sh_desc(struct crypto_aead *aead) 73 { 74 struct caam_aead_alg *alg = container_of(crypto_aead_alg(aead), 75 typeof(*alg), aead); 76 struct caam_ctx *ctx = crypto_aead_ctx(aead); 77 unsigned int ivsize = crypto_aead_ivsize(aead); 78 u32 ctx1_iv_off = 0; 79 u32 *nonce = NULL; 80 unsigned int data_len[2]; 81 u32 inl_mask; 82 const bool ctr_mode = ((ctx->cdata.algtype & OP_ALG_AAI_MASK) == 83 OP_ALG_AAI_CTR_MOD128); 84 const bool is_rfc3686 = alg->caam.rfc3686; 85 struct caam_drv_private *ctrlpriv = dev_get_drvdata(ctx->jrdev->parent); 86 87 if (!ctx->cdata.keylen || !ctx->authsize) 88 return 0; 89 90 /* 91 * AES-CTR needs to load IV in CONTEXT1 reg 92 * at an offset of 128bits (16bytes) 93 * CONTEXT1[255:128] = IV 94 */ 95 if (ctr_mode) 96 ctx1_iv_off = 16; 97 98 /* 99 * RFC3686 specific: 100 * CONTEXT1[255:128] = {NONCE, IV, COUNTER} 101 */ 102 if (is_rfc3686) { 103 ctx1_iv_off = 16 + CTR_RFC3686_NONCE_SIZE; 104 nonce = (u32 *)((void *)ctx->key + ctx->adata.keylen_pad + 105 ctx->cdata.keylen - CTR_RFC3686_NONCE_SIZE); 106 } 107 108 /* 109 * In case |user key| > |derived key|, using DKP<imm,imm> would result 110 * in invalid opcodes (last bytes of user key) in the resulting 111 * descriptor. Use DKP<ptr,imm> instead => both virtual and dma key 112 * addresses are needed. 113 */ 114 ctx->adata.key_virt = ctx->key; 115 ctx->adata.key_dma = ctx->key_dma; 116 117 ctx->cdata.key_virt = ctx->key + ctx->adata.keylen_pad; 118 ctx->cdata.key_dma = ctx->key_dma + ctx->adata.keylen_pad; 119 120 data_len[0] = ctx->adata.keylen_pad; 121 data_len[1] = ctx->cdata.keylen; 122 123 if (alg->caam.geniv) 124 goto skip_enc; 125 126 /* aead_encrypt shared descriptor */ 127 if (desc_inline_query(DESC_QI_AEAD_ENC_LEN + 128 (is_rfc3686 ? DESC_AEAD_CTR_RFC3686_LEN : 0), 129 DESC_JOB_IO_LEN, data_len, &inl_mask, 130 ARRAY_SIZE(data_len)) < 0) 131 return -EINVAL; 132 133 ctx->adata.key_inline = !!(inl_mask & 1); 134 ctx->cdata.key_inline = !!(inl_mask & 2); 135 136 cnstr_shdsc_aead_encap(ctx->sh_desc_enc, &ctx->cdata, &ctx->adata, 137 ivsize, ctx->authsize, is_rfc3686, nonce, 138 ctx1_iv_off, true, ctrlpriv->era); 139 140 skip_enc: 141 /* aead_decrypt shared descriptor */ 142 if (desc_inline_query(DESC_QI_AEAD_DEC_LEN + 143 (is_rfc3686 ? DESC_AEAD_CTR_RFC3686_LEN : 0), 144 DESC_JOB_IO_LEN, data_len, &inl_mask, 145 ARRAY_SIZE(data_len)) < 0) 146 return -EINVAL; 147 148 ctx->adata.key_inline = !!(inl_mask & 1); 149 ctx->cdata.key_inline = !!(inl_mask & 2); 150 151 cnstr_shdsc_aead_decap(ctx->sh_desc_dec, &ctx->cdata, &ctx->adata, 152 ivsize, ctx->authsize, alg->caam.geniv, 153 is_rfc3686, nonce, ctx1_iv_off, true, 154 ctrlpriv->era); 155 156 if (!alg->caam.geniv) 157 goto skip_givenc; 158 159 /* aead_givencrypt shared descriptor */ 160 if (desc_inline_query(DESC_QI_AEAD_GIVENC_LEN + 161 (is_rfc3686 ? DESC_AEAD_CTR_RFC3686_LEN : 0), 162 DESC_JOB_IO_LEN, data_len, &inl_mask, 163 ARRAY_SIZE(data_len)) < 0) 164 return -EINVAL; 165 166 ctx->adata.key_inline = !!(inl_mask & 1); 167 ctx->cdata.key_inline = !!(inl_mask & 2); 168 169 cnstr_shdsc_aead_givencap(ctx->sh_desc_enc, &ctx->cdata, &ctx->adata, 170 ivsize, ctx->authsize, is_rfc3686, nonce, 171 ctx1_iv_off, true, ctrlpriv->era); 172 173 skip_givenc: 174 return 0; 175 } 176 177 static int aead_setauthsize(struct crypto_aead *authenc, unsigned int authsize) 178 { 179 struct caam_ctx *ctx = crypto_aead_ctx(authenc); 180 181 ctx->authsize = authsize; 182 aead_set_sh_desc(authenc); 183 184 return 0; 185 } 186 187 static int aead_setkey(struct crypto_aead *aead, const u8 *key, 188 unsigned int keylen) 189 { 190 struct caam_ctx *ctx = crypto_aead_ctx(aead); 191 struct device *jrdev = ctx->jrdev; 192 struct caam_drv_private *ctrlpriv = dev_get_drvdata(jrdev->parent); 193 struct crypto_authenc_keys keys; 194 int ret = 0; 195 196 if (crypto_authenc_extractkeys(&keys, key, keylen) != 0) 197 goto badkey; 198 199 dev_dbg(jrdev, "keylen %d enckeylen %d authkeylen %d\n", 200 keys.authkeylen + keys.enckeylen, keys.enckeylen, 201 keys.authkeylen); 202 print_hex_dump_debug("key in @" __stringify(__LINE__)": ", 203 DUMP_PREFIX_ADDRESS, 16, 4, key, keylen, 1); 204 205 /* 206 * If DKP is supported, use it in the shared descriptor to generate 207 * the split key. 208 */ 209 if (ctrlpriv->era >= 6) { 210 ctx->adata.keylen = keys.authkeylen; 211 ctx->adata.keylen_pad = split_key_len(ctx->adata.algtype & 212 OP_ALG_ALGSEL_MASK); 213 214 if (ctx->adata.keylen_pad + keys.enckeylen > CAAM_MAX_KEY_SIZE) 215 goto badkey; 216 217 memcpy(ctx->key, keys.authkey, keys.authkeylen); 218 memcpy(ctx->key + ctx->adata.keylen_pad, keys.enckey, 219 keys.enckeylen); 220 dma_sync_single_for_device(jrdev->parent, ctx->key_dma, 221 ctx->adata.keylen_pad + 222 keys.enckeylen, ctx->dir); 223 goto skip_split_key; 224 } 225 226 ret = gen_split_key(jrdev, ctx->key, &ctx->adata, keys.authkey, 227 keys.authkeylen, CAAM_MAX_KEY_SIZE - 228 keys.enckeylen); 229 if (ret) 230 goto badkey; 231 232 /* postpend encryption key to auth split key */ 233 memcpy(ctx->key + ctx->adata.keylen_pad, keys.enckey, keys.enckeylen); 234 dma_sync_single_for_device(jrdev->parent, ctx->key_dma, 235 ctx->adata.keylen_pad + keys.enckeylen, 236 ctx->dir); 237 238 print_hex_dump_debug("ctx.key@" __stringify(__LINE__)": ", 239 DUMP_PREFIX_ADDRESS, 16, 4, ctx->key, 240 ctx->adata.keylen_pad + keys.enckeylen, 1); 241 242 skip_split_key: 243 ctx->cdata.keylen = keys.enckeylen; 244 245 ret = aead_set_sh_desc(aead); 246 if (ret) 247 goto badkey; 248 249 /* Now update the driver contexts with the new shared descriptor */ 250 if (ctx->drv_ctx[ENCRYPT]) { 251 ret = caam_drv_ctx_update(ctx->drv_ctx[ENCRYPT], 252 ctx->sh_desc_enc); 253 if (ret) { 254 dev_err(jrdev, "driver enc context update failed\n"); 255 goto badkey; 256 } 257 } 258 259 if (ctx->drv_ctx[DECRYPT]) { 260 ret = caam_drv_ctx_update(ctx->drv_ctx[DECRYPT], 261 ctx->sh_desc_dec); 262 if (ret) { 263 dev_err(jrdev, "driver dec context update failed\n"); 264 goto badkey; 265 } 266 } 267 268 memzero_explicit(&keys, sizeof(keys)); 269 return ret; 270 badkey: 271 memzero_explicit(&keys, sizeof(keys)); 272 return -EINVAL; 273 } 274 275 static int des3_aead_setkey(struct crypto_aead *aead, const u8 *key, 276 unsigned int keylen) 277 { 278 struct crypto_authenc_keys keys; 279 int err; 280 281 err = crypto_authenc_extractkeys(&keys, key, keylen); 282 if (unlikely(err)) 283 return err; 284 285 err = verify_aead_des3_key(aead, keys.enckey, keys.enckeylen) ?: 286 aead_setkey(aead, key, keylen); 287 288 memzero_explicit(&keys, sizeof(keys)); 289 return err; 290 } 291 292 static int gcm_set_sh_desc(struct crypto_aead *aead) 293 { 294 struct caam_ctx *ctx = crypto_aead_ctx(aead); 295 unsigned int ivsize = crypto_aead_ivsize(aead); 296 int rem_bytes = CAAM_DESC_BYTES_MAX - DESC_JOB_IO_LEN - 297 ctx->cdata.keylen; 298 299 if (!ctx->cdata.keylen || !ctx->authsize) 300 return 0; 301 302 /* 303 * Job Descriptor and Shared Descriptor 304 * must fit into the 64-word Descriptor h/w Buffer 305 */ 306 if (rem_bytes >= DESC_QI_GCM_ENC_LEN) { 307 ctx->cdata.key_inline = true; 308 ctx->cdata.key_virt = ctx->key; 309 } else { 310 ctx->cdata.key_inline = false; 311 ctx->cdata.key_dma = ctx->key_dma; 312 } 313 314 cnstr_shdsc_gcm_encap(ctx->sh_desc_enc, &ctx->cdata, ivsize, 315 ctx->authsize, true); 316 317 /* 318 * Job Descriptor and Shared Descriptor 319 * must fit into the 64-word Descriptor h/w Buffer 320 */ 321 if (rem_bytes >= DESC_QI_GCM_DEC_LEN) { 322 ctx->cdata.key_inline = true; 323 ctx->cdata.key_virt = ctx->key; 324 } else { 325 ctx->cdata.key_inline = false; 326 ctx->cdata.key_dma = ctx->key_dma; 327 } 328 329 cnstr_shdsc_gcm_decap(ctx->sh_desc_dec, &ctx->cdata, ivsize, 330 ctx->authsize, true); 331 332 return 0; 333 } 334 335 static int gcm_setauthsize(struct crypto_aead *authenc, unsigned int authsize) 336 { 337 struct caam_ctx *ctx = crypto_aead_ctx(authenc); 338 int err; 339 340 err = crypto_gcm_check_authsize(authsize); 341 if (err) 342 return err; 343 344 ctx->authsize = authsize; 345 gcm_set_sh_desc(authenc); 346 347 return 0; 348 } 349 350 static int gcm_setkey(struct crypto_aead *aead, 351 const u8 *key, unsigned int keylen) 352 { 353 struct caam_ctx *ctx = crypto_aead_ctx(aead); 354 struct device *jrdev = ctx->jrdev; 355 int ret; 356 357 ret = aes_check_keylen(keylen); 358 if (ret) 359 return ret; 360 361 print_hex_dump_debug("key in @" __stringify(__LINE__)": ", 362 DUMP_PREFIX_ADDRESS, 16, 4, key, keylen, 1); 363 364 memcpy(ctx->key, key, keylen); 365 dma_sync_single_for_device(jrdev->parent, ctx->key_dma, keylen, 366 ctx->dir); 367 ctx->cdata.keylen = keylen; 368 369 ret = gcm_set_sh_desc(aead); 370 if (ret) 371 return ret; 372 373 /* Now update the driver contexts with the new shared descriptor */ 374 if (ctx->drv_ctx[ENCRYPT]) { 375 ret = caam_drv_ctx_update(ctx->drv_ctx[ENCRYPT], 376 ctx->sh_desc_enc); 377 if (ret) { 378 dev_err(jrdev, "driver enc context update failed\n"); 379 return ret; 380 } 381 } 382 383 if (ctx->drv_ctx[DECRYPT]) { 384 ret = caam_drv_ctx_update(ctx->drv_ctx[DECRYPT], 385 ctx->sh_desc_dec); 386 if (ret) { 387 dev_err(jrdev, "driver dec context update failed\n"); 388 return ret; 389 } 390 } 391 392 return 0; 393 } 394 395 static int rfc4106_set_sh_desc(struct crypto_aead *aead) 396 { 397 struct caam_ctx *ctx = crypto_aead_ctx(aead); 398 unsigned int ivsize = crypto_aead_ivsize(aead); 399 int rem_bytes = CAAM_DESC_BYTES_MAX - DESC_JOB_IO_LEN - 400 ctx->cdata.keylen; 401 402 if (!ctx->cdata.keylen || !ctx->authsize) 403 return 0; 404 405 ctx->cdata.key_virt = ctx->key; 406 407 /* 408 * Job Descriptor and Shared Descriptor 409 * must fit into the 64-word Descriptor h/w Buffer 410 */ 411 if (rem_bytes >= DESC_QI_RFC4106_ENC_LEN) { 412 ctx->cdata.key_inline = true; 413 } else { 414 ctx->cdata.key_inline = false; 415 ctx->cdata.key_dma = ctx->key_dma; 416 } 417 418 cnstr_shdsc_rfc4106_encap(ctx->sh_desc_enc, &ctx->cdata, ivsize, 419 ctx->authsize, true); 420 421 /* 422 * Job Descriptor and Shared Descriptor 423 * must fit into the 64-word Descriptor h/w Buffer 424 */ 425 if (rem_bytes >= DESC_QI_RFC4106_DEC_LEN) { 426 ctx->cdata.key_inline = true; 427 } else { 428 ctx->cdata.key_inline = false; 429 ctx->cdata.key_dma = ctx->key_dma; 430 } 431 432 cnstr_shdsc_rfc4106_decap(ctx->sh_desc_dec, &ctx->cdata, ivsize, 433 ctx->authsize, true); 434 435 return 0; 436 } 437 438 static int rfc4106_setauthsize(struct crypto_aead *authenc, 439 unsigned int authsize) 440 { 441 struct caam_ctx *ctx = crypto_aead_ctx(authenc); 442 int err; 443 444 err = crypto_rfc4106_check_authsize(authsize); 445 if (err) 446 return err; 447 448 ctx->authsize = authsize; 449 rfc4106_set_sh_desc(authenc); 450 451 return 0; 452 } 453 454 static int rfc4106_setkey(struct crypto_aead *aead, 455 const u8 *key, unsigned int keylen) 456 { 457 struct caam_ctx *ctx = crypto_aead_ctx(aead); 458 struct device *jrdev = ctx->jrdev; 459 int ret; 460 461 ret = aes_check_keylen(keylen - 4); 462 if (ret) 463 return ret; 464 465 print_hex_dump_debug("key in @" __stringify(__LINE__)": ", 466 DUMP_PREFIX_ADDRESS, 16, 4, key, keylen, 1); 467 468 memcpy(ctx->key, key, keylen); 469 /* 470 * The last four bytes of the key material are used as the salt value 471 * in the nonce. Update the AES key length. 472 */ 473 ctx->cdata.keylen = keylen - 4; 474 dma_sync_single_for_device(jrdev->parent, ctx->key_dma, 475 ctx->cdata.keylen, ctx->dir); 476 477 ret = rfc4106_set_sh_desc(aead); 478 if (ret) 479 return ret; 480 481 /* Now update the driver contexts with the new shared descriptor */ 482 if (ctx->drv_ctx[ENCRYPT]) { 483 ret = caam_drv_ctx_update(ctx->drv_ctx[ENCRYPT], 484 ctx->sh_desc_enc); 485 if (ret) { 486 dev_err(jrdev, "driver enc context update failed\n"); 487 return ret; 488 } 489 } 490 491 if (ctx->drv_ctx[DECRYPT]) { 492 ret = caam_drv_ctx_update(ctx->drv_ctx[DECRYPT], 493 ctx->sh_desc_dec); 494 if (ret) { 495 dev_err(jrdev, "driver dec context update failed\n"); 496 return ret; 497 } 498 } 499 500 return 0; 501 } 502 503 static int rfc4543_set_sh_desc(struct crypto_aead *aead) 504 { 505 struct caam_ctx *ctx = crypto_aead_ctx(aead); 506 unsigned int ivsize = crypto_aead_ivsize(aead); 507 int rem_bytes = CAAM_DESC_BYTES_MAX - DESC_JOB_IO_LEN - 508 ctx->cdata.keylen; 509 510 if (!ctx->cdata.keylen || !ctx->authsize) 511 return 0; 512 513 ctx->cdata.key_virt = ctx->key; 514 515 /* 516 * Job Descriptor and Shared Descriptor 517 * must fit into the 64-word Descriptor h/w Buffer 518 */ 519 if (rem_bytes >= DESC_QI_RFC4543_ENC_LEN) { 520 ctx->cdata.key_inline = true; 521 } else { 522 ctx->cdata.key_inline = false; 523 ctx->cdata.key_dma = ctx->key_dma; 524 } 525 526 cnstr_shdsc_rfc4543_encap(ctx->sh_desc_enc, &ctx->cdata, ivsize, 527 ctx->authsize, true); 528 529 /* 530 * Job Descriptor and Shared Descriptor 531 * must fit into the 64-word Descriptor h/w Buffer 532 */ 533 if (rem_bytes >= DESC_QI_RFC4543_DEC_LEN) { 534 ctx->cdata.key_inline = true; 535 } else { 536 ctx->cdata.key_inline = false; 537 ctx->cdata.key_dma = ctx->key_dma; 538 } 539 540 cnstr_shdsc_rfc4543_decap(ctx->sh_desc_dec, &ctx->cdata, ivsize, 541 ctx->authsize, true); 542 543 return 0; 544 } 545 546 static int rfc4543_setauthsize(struct crypto_aead *authenc, 547 unsigned int authsize) 548 { 549 struct caam_ctx *ctx = crypto_aead_ctx(authenc); 550 551 if (authsize != 16) 552 return -EINVAL; 553 554 ctx->authsize = authsize; 555 rfc4543_set_sh_desc(authenc); 556 557 return 0; 558 } 559 560 static int rfc4543_setkey(struct crypto_aead *aead, 561 const u8 *key, unsigned int keylen) 562 { 563 struct caam_ctx *ctx = crypto_aead_ctx(aead); 564 struct device *jrdev = ctx->jrdev; 565 int ret; 566 567 ret = aes_check_keylen(keylen - 4); 568 if (ret) 569 return ret; 570 571 print_hex_dump_debug("key in @" __stringify(__LINE__)": ", 572 DUMP_PREFIX_ADDRESS, 16, 4, key, keylen, 1); 573 574 memcpy(ctx->key, key, keylen); 575 /* 576 * The last four bytes of the key material are used as the salt value 577 * in the nonce. Update the AES key length. 578 */ 579 ctx->cdata.keylen = keylen - 4; 580 dma_sync_single_for_device(jrdev->parent, ctx->key_dma, 581 ctx->cdata.keylen, ctx->dir); 582 583 ret = rfc4543_set_sh_desc(aead); 584 if (ret) 585 return ret; 586 587 /* Now update the driver contexts with the new shared descriptor */ 588 if (ctx->drv_ctx[ENCRYPT]) { 589 ret = caam_drv_ctx_update(ctx->drv_ctx[ENCRYPT], 590 ctx->sh_desc_enc); 591 if (ret) { 592 dev_err(jrdev, "driver enc context update failed\n"); 593 return ret; 594 } 595 } 596 597 if (ctx->drv_ctx[DECRYPT]) { 598 ret = caam_drv_ctx_update(ctx->drv_ctx[DECRYPT], 599 ctx->sh_desc_dec); 600 if (ret) { 601 dev_err(jrdev, "driver dec context update failed\n"); 602 return ret; 603 } 604 } 605 606 return 0; 607 } 608 609 static int skcipher_setkey(struct crypto_skcipher *skcipher, const u8 *key, 610 unsigned int keylen, const u32 ctx1_iv_off) 611 { 612 struct caam_ctx *ctx = crypto_skcipher_ctx(skcipher); 613 struct caam_skcipher_alg *alg = 614 container_of(crypto_skcipher_alg(skcipher), typeof(*alg), 615 skcipher); 616 struct device *jrdev = ctx->jrdev; 617 unsigned int ivsize = crypto_skcipher_ivsize(skcipher); 618 const bool is_rfc3686 = alg->caam.rfc3686; 619 int ret = 0; 620 621 print_hex_dump_debug("key in @" __stringify(__LINE__)": ", 622 DUMP_PREFIX_ADDRESS, 16, 4, key, keylen, 1); 623 624 ctx->cdata.keylen = keylen; 625 ctx->cdata.key_virt = key; 626 ctx->cdata.key_inline = true; 627 628 /* skcipher encrypt, decrypt shared descriptors */ 629 cnstr_shdsc_skcipher_encap(ctx->sh_desc_enc, &ctx->cdata, ivsize, 630 is_rfc3686, ctx1_iv_off); 631 cnstr_shdsc_skcipher_decap(ctx->sh_desc_dec, &ctx->cdata, ivsize, 632 is_rfc3686, ctx1_iv_off); 633 634 /* Now update the driver contexts with the new shared descriptor */ 635 if (ctx->drv_ctx[ENCRYPT]) { 636 ret = caam_drv_ctx_update(ctx->drv_ctx[ENCRYPT], 637 ctx->sh_desc_enc); 638 if (ret) { 639 dev_err(jrdev, "driver enc context update failed\n"); 640 return -EINVAL; 641 } 642 } 643 644 if (ctx->drv_ctx[DECRYPT]) { 645 ret = caam_drv_ctx_update(ctx->drv_ctx[DECRYPT], 646 ctx->sh_desc_dec); 647 if (ret) { 648 dev_err(jrdev, "driver dec context update failed\n"); 649 return -EINVAL; 650 } 651 } 652 653 return ret; 654 } 655 656 static int aes_skcipher_setkey(struct crypto_skcipher *skcipher, 657 const u8 *key, unsigned int keylen) 658 { 659 int err; 660 661 err = aes_check_keylen(keylen); 662 if (err) 663 return err; 664 665 return skcipher_setkey(skcipher, key, keylen, 0); 666 } 667 668 static int rfc3686_skcipher_setkey(struct crypto_skcipher *skcipher, 669 const u8 *key, unsigned int keylen) 670 { 671 u32 ctx1_iv_off; 672 int err; 673 674 /* 675 * RFC3686 specific: 676 * | CONTEXT1[255:128] = {NONCE, IV, COUNTER} 677 * | *key = {KEY, NONCE} 678 */ 679 ctx1_iv_off = 16 + CTR_RFC3686_NONCE_SIZE; 680 keylen -= CTR_RFC3686_NONCE_SIZE; 681 682 err = aes_check_keylen(keylen); 683 if (err) 684 return err; 685 686 return skcipher_setkey(skcipher, key, keylen, ctx1_iv_off); 687 } 688 689 static int ctr_skcipher_setkey(struct crypto_skcipher *skcipher, 690 const u8 *key, unsigned int keylen) 691 { 692 u32 ctx1_iv_off; 693 int err; 694 695 /* 696 * AES-CTR needs to load IV in CONTEXT1 reg 697 * at an offset of 128bits (16bytes) 698 * CONTEXT1[255:128] = IV 699 */ 700 ctx1_iv_off = 16; 701 702 err = aes_check_keylen(keylen); 703 if (err) 704 return err; 705 706 return skcipher_setkey(skcipher, key, keylen, ctx1_iv_off); 707 } 708 709 static int des3_skcipher_setkey(struct crypto_skcipher *skcipher, 710 const u8 *key, unsigned int keylen) 711 { 712 return verify_skcipher_des3_key(skcipher, key) ?: 713 skcipher_setkey(skcipher, key, keylen, 0); 714 } 715 716 static int des_skcipher_setkey(struct crypto_skcipher *skcipher, 717 const u8 *key, unsigned int keylen) 718 { 719 return verify_skcipher_des_key(skcipher, key) ?: 720 skcipher_setkey(skcipher, key, keylen, 0); 721 } 722 723 static int xts_skcipher_setkey(struct crypto_skcipher *skcipher, const u8 *key, 724 unsigned int keylen) 725 { 726 struct caam_ctx *ctx = crypto_skcipher_ctx(skcipher); 727 struct device *jrdev = ctx->jrdev; 728 int ret = 0; 729 730 if (keylen != 2 * AES_MIN_KEY_SIZE && keylen != 2 * AES_MAX_KEY_SIZE) { 731 dev_err(jrdev, "key size mismatch\n"); 732 return -EINVAL; 733 } 734 735 ctx->cdata.keylen = keylen; 736 ctx->cdata.key_virt = key; 737 ctx->cdata.key_inline = true; 738 739 /* xts skcipher encrypt, decrypt shared descriptors */ 740 cnstr_shdsc_xts_skcipher_encap(ctx->sh_desc_enc, &ctx->cdata); 741 cnstr_shdsc_xts_skcipher_decap(ctx->sh_desc_dec, &ctx->cdata); 742 743 /* Now update the driver contexts with the new shared descriptor */ 744 if (ctx->drv_ctx[ENCRYPT]) { 745 ret = caam_drv_ctx_update(ctx->drv_ctx[ENCRYPT], 746 ctx->sh_desc_enc); 747 if (ret) { 748 dev_err(jrdev, "driver enc context update failed\n"); 749 return -EINVAL; 750 } 751 } 752 753 if (ctx->drv_ctx[DECRYPT]) { 754 ret = caam_drv_ctx_update(ctx->drv_ctx[DECRYPT], 755 ctx->sh_desc_dec); 756 if (ret) { 757 dev_err(jrdev, "driver dec context update failed\n"); 758 return -EINVAL; 759 } 760 } 761 762 return ret; 763 } 764 765 /* 766 * aead_edesc - s/w-extended aead descriptor 767 * @src_nents: number of segments in input scatterlist 768 * @dst_nents: number of segments in output scatterlist 769 * @iv_dma: dma address of iv for checking continuity and link table 770 * @qm_sg_bytes: length of dma mapped h/w link table 771 * @qm_sg_dma: bus physical mapped address of h/w link table 772 * @assoclen: associated data length, in CAAM endianness 773 * @assoclen_dma: bus physical mapped address of req->assoclen 774 * @drv_req: driver-specific request structure 775 * @sgt: the h/w link table, followed by IV 776 */ 777 struct aead_edesc { 778 int src_nents; 779 int dst_nents; 780 dma_addr_t iv_dma; 781 int qm_sg_bytes; 782 dma_addr_t qm_sg_dma; 783 unsigned int assoclen; 784 dma_addr_t assoclen_dma; 785 struct caam_drv_req drv_req; 786 struct qm_sg_entry sgt[]; 787 }; 788 789 /* 790 * skcipher_edesc - s/w-extended skcipher descriptor 791 * @src_nents: number of segments in input scatterlist 792 * @dst_nents: number of segments in output scatterlist 793 * @iv_dma: dma address of iv for checking continuity and link table 794 * @qm_sg_bytes: length of dma mapped h/w link table 795 * @qm_sg_dma: bus physical mapped address of h/w link table 796 * @drv_req: driver-specific request structure 797 * @sgt: the h/w link table, followed by IV 798 */ 799 struct skcipher_edesc { 800 int src_nents; 801 int dst_nents; 802 dma_addr_t iv_dma; 803 int qm_sg_bytes; 804 dma_addr_t qm_sg_dma; 805 struct caam_drv_req drv_req; 806 struct qm_sg_entry sgt[]; 807 }; 808 809 static struct caam_drv_ctx *get_drv_ctx(struct caam_ctx *ctx, 810 enum optype type) 811 { 812 /* 813 * This function is called on the fast path with values of 'type' 814 * known at compile time. Invalid arguments are not expected and 815 * thus no checks are made. 816 */ 817 struct caam_drv_ctx *drv_ctx = ctx->drv_ctx[type]; 818 u32 *desc; 819 820 if (unlikely(!drv_ctx)) { 821 spin_lock(&ctx->lock); 822 823 /* Read again to check if some other core init drv_ctx */ 824 drv_ctx = ctx->drv_ctx[type]; 825 if (!drv_ctx) { 826 int cpu; 827 828 if (type == ENCRYPT) 829 desc = ctx->sh_desc_enc; 830 else /* (type == DECRYPT) */ 831 desc = ctx->sh_desc_dec; 832 833 cpu = smp_processor_id(); 834 drv_ctx = caam_drv_ctx_init(ctx->qidev, &cpu, desc); 835 if (!IS_ERR_OR_NULL(drv_ctx)) 836 drv_ctx->op_type = type; 837 838 ctx->drv_ctx[type] = drv_ctx; 839 } 840 841 spin_unlock(&ctx->lock); 842 } 843 844 return drv_ctx; 845 } 846 847 static void caam_unmap(struct device *dev, struct scatterlist *src, 848 struct scatterlist *dst, int src_nents, 849 int dst_nents, dma_addr_t iv_dma, int ivsize, 850 enum dma_data_direction iv_dir, dma_addr_t qm_sg_dma, 851 int qm_sg_bytes) 852 { 853 if (dst != src) { 854 if (src_nents) 855 dma_unmap_sg(dev, src, src_nents, DMA_TO_DEVICE); 856 if (dst_nents) 857 dma_unmap_sg(dev, dst, dst_nents, DMA_FROM_DEVICE); 858 } else { 859 dma_unmap_sg(dev, src, src_nents, DMA_BIDIRECTIONAL); 860 } 861 862 if (iv_dma) 863 dma_unmap_single(dev, iv_dma, ivsize, iv_dir); 864 if (qm_sg_bytes) 865 dma_unmap_single(dev, qm_sg_dma, qm_sg_bytes, DMA_TO_DEVICE); 866 } 867 868 static void aead_unmap(struct device *dev, 869 struct aead_edesc *edesc, 870 struct aead_request *req) 871 { 872 struct crypto_aead *aead = crypto_aead_reqtfm(req); 873 int ivsize = crypto_aead_ivsize(aead); 874 875 caam_unmap(dev, req->src, req->dst, edesc->src_nents, edesc->dst_nents, 876 edesc->iv_dma, ivsize, DMA_TO_DEVICE, edesc->qm_sg_dma, 877 edesc->qm_sg_bytes); 878 dma_unmap_single(dev, edesc->assoclen_dma, 4, DMA_TO_DEVICE); 879 } 880 881 static void skcipher_unmap(struct device *dev, struct skcipher_edesc *edesc, 882 struct skcipher_request *req) 883 { 884 struct crypto_skcipher *skcipher = crypto_skcipher_reqtfm(req); 885 int ivsize = crypto_skcipher_ivsize(skcipher); 886 887 caam_unmap(dev, req->src, req->dst, edesc->src_nents, edesc->dst_nents, 888 edesc->iv_dma, ivsize, DMA_BIDIRECTIONAL, edesc->qm_sg_dma, 889 edesc->qm_sg_bytes); 890 } 891 892 static void aead_done(struct caam_drv_req *drv_req, u32 status) 893 { 894 struct device *qidev; 895 struct aead_edesc *edesc; 896 struct aead_request *aead_req = drv_req->app_ctx; 897 struct crypto_aead *aead = crypto_aead_reqtfm(aead_req); 898 struct caam_ctx *caam_ctx = crypto_aead_ctx(aead); 899 int ecode = 0; 900 901 qidev = caam_ctx->qidev; 902 903 if (unlikely(status)) 904 ecode = caam_jr_strstatus(qidev, status); 905 906 edesc = container_of(drv_req, typeof(*edesc), drv_req); 907 aead_unmap(qidev, edesc, aead_req); 908 909 aead_request_complete(aead_req, ecode); 910 qi_cache_free(edesc); 911 } 912 913 /* 914 * allocate and map the aead extended descriptor 915 */ 916 static struct aead_edesc *aead_edesc_alloc(struct aead_request *req, 917 bool encrypt) 918 { 919 struct crypto_aead *aead = crypto_aead_reqtfm(req); 920 struct caam_ctx *ctx = crypto_aead_ctx(aead); 921 struct caam_aead_alg *alg = container_of(crypto_aead_alg(aead), 922 typeof(*alg), aead); 923 struct device *qidev = ctx->qidev; 924 gfp_t flags = (req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP) ? 925 GFP_KERNEL : GFP_ATOMIC; 926 int src_nents, mapped_src_nents, dst_nents = 0, mapped_dst_nents = 0; 927 int src_len, dst_len = 0; 928 struct aead_edesc *edesc; 929 dma_addr_t qm_sg_dma, iv_dma = 0; 930 int ivsize = 0; 931 unsigned int authsize = ctx->authsize; 932 int qm_sg_index = 0, qm_sg_ents = 0, qm_sg_bytes; 933 int in_len, out_len; 934 struct qm_sg_entry *sg_table, *fd_sgt; 935 struct caam_drv_ctx *drv_ctx; 936 937 drv_ctx = get_drv_ctx(ctx, encrypt ? ENCRYPT : DECRYPT); 938 if (IS_ERR_OR_NULL(drv_ctx)) 939 return (struct aead_edesc *)drv_ctx; 940 941 /* allocate space for base edesc and hw desc commands, link tables */ 942 edesc = qi_cache_alloc(GFP_DMA | flags); 943 if (unlikely(!edesc)) { 944 dev_err(qidev, "could not allocate extended descriptor\n"); 945 return ERR_PTR(-ENOMEM); 946 } 947 948 if (likely(req->src == req->dst)) { 949 src_len = req->assoclen + req->cryptlen + 950 (encrypt ? authsize : 0); 951 952 src_nents = sg_nents_for_len(req->src, src_len); 953 if (unlikely(src_nents < 0)) { 954 dev_err(qidev, "Insufficient bytes (%d) in src S/G\n", 955 src_len); 956 qi_cache_free(edesc); 957 return ERR_PTR(src_nents); 958 } 959 960 mapped_src_nents = dma_map_sg(qidev, req->src, src_nents, 961 DMA_BIDIRECTIONAL); 962 if (unlikely(!mapped_src_nents)) { 963 dev_err(qidev, "unable to map source\n"); 964 qi_cache_free(edesc); 965 return ERR_PTR(-ENOMEM); 966 } 967 } else { 968 src_len = req->assoclen + req->cryptlen; 969 dst_len = src_len + (encrypt ? authsize : (-authsize)); 970 971 src_nents = sg_nents_for_len(req->src, src_len); 972 if (unlikely(src_nents < 0)) { 973 dev_err(qidev, "Insufficient bytes (%d) in src S/G\n", 974 src_len); 975 qi_cache_free(edesc); 976 return ERR_PTR(src_nents); 977 } 978 979 dst_nents = sg_nents_for_len(req->dst, dst_len); 980 if (unlikely(dst_nents < 0)) { 981 dev_err(qidev, "Insufficient bytes (%d) in dst S/G\n", 982 dst_len); 983 qi_cache_free(edesc); 984 return ERR_PTR(dst_nents); 985 } 986 987 if (src_nents) { 988 mapped_src_nents = dma_map_sg(qidev, req->src, 989 src_nents, DMA_TO_DEVICE); 990 if (unlikely(!mapped_src_nents)) { 991 dev_err(qidev, "unable to map source\n"); 992 qi_cache_free(edesc); 993 return ERR_PTR(-ENOMEM); 994 } 995 } else { 996 mapped_src_nents = 0; 997 } 998 999 if (dst_nents) { 1000 mapped_dst_nents = dma_map_sg(qidev, req->dst, 1001 dst_nents, 1002 DMA_FROM_DEVICE); 1003 if (unlikely(!mapped_dst_nents)) { 1004 dev_err(qidev, "unable to map destination\n"); 1005 dma_unmap_sg(qidev, req->src, src_nents, 1006 DMA_TO_DEVICE); 1007 qi_cache_free(edesc); 1008 return ERR_PTR(-ENOMEM); 1009 } 1010 } else { 1011 mapped_dst_nents = 0; 1012 } 1013 } 1014 1015 if ((alg->caam.rfc3686 && encrypt) || !alg->caam.geniv) 1016 ivsize = crypto_aead_ivsize(aead); 1017 1018 /* 1019 * Create S/G table: req->assoclen, [IV,] req->src [, req->dst]. 1020 * Input is not contiguous. 1021 * HW reads 4 S/G entries at a time; make sure the reads don't go beyond 1022 * the end of the table by allocating more S/G entries. Logic: 1023 * if (src != dst && output S/G) 1024 * pad output S/G, if needed 1025 * else if (src == dst && S/G) 1026 * overlapping S/Gs; pad one of them 1027 * else if (input S/G) ... 1028 * pad input S/G, if needed 1029 */ 1030 qm_sg_ents = 1 + !!ivsize + mapped_src_nents; 1031 if (mapped_dst_nents > 1) 1032 qm_sg_ents += pad_sg_nents(mapped_dst_nents); 1033 else if ((req->src == req->dst) && (mapped_src_nents > 1)) 1034 qm_sg_ents = max(pad_sg_nents(qm_sg_ents), 1035 1 + !!ivsize + pad_sg_nents(mapped_src_nents)); 1036 else 1037 qm_sg_ents = pad_sg_nents(qm_sg_ents); 1038 1039 sg_table = &edesc->sgt[0]; 1040 qm_sg_bytes = qm_sg_ents * sizeof(*sg_table); 1041 if (unlikely(offsetof(struct aead_edesc, sgt) + qm_sg_bytes + ivsize > 1042 CAAM_QI_MEMCACHE_SIZE)) { 1043 dev_err(qidev, "No space for %d S/G entries and/or %dB IV\n", 1044 qm_sg_ents, ivsize); 1045 caam_unmap(qidev, req->src, req->dst, src_nents, dst_nents, 0, 1046 0, DMA_NONE, 0, 0); 1047 qi_cache_free(edesc); 1048 return ERR_PTR(-ENOMEM); 1049 } 1050 1051 if (ivsize) { 1052 u8 *iv = (u8 *)(sg_table + qm_sg_ents); 1053 1054 /* Make sure IV is located in a DMAable area */ 1055 memcpy(iv, req->iv, ivsize); 1056 1057 iv_dma = dma_map_single(qidev, iv, ivsize, DMA_TO_DEVICE); 1058 if (dma_mapping_error(qidev, iv_dma)) { 1059 dev_err(qidev, "unable to map IV\n"); 1060 caam_unmap(qidev, req->src, req->dst, src_nents, 1061 dst_nents, 0, 0, DMA_NONE, 0, 0); 1062 qi_cache_free(edesc); 1063 return ERR_PTR(-ENOMEM); 1064 } 1065 } 1066 1067 edesc->src_nents = src_nents; 1068 edesc->dst_nents = dst_nents; 1069 edesc->iv_dma = iv_dma; 1070 edesc->drv_req.app_ctx = req; 1071 edesc->drv_req.cbk = aead_done; 1072 edesc->drv_req.drv_ctx = drv_ctx; 1073 1074 edesc->assoclen = cpu_to_caam32(req->assoclen); 1075 edesc->assoclen_dma = dma_map_single(qidev, &edesc->assoclen, 4, 1076 DMA_TO_DEVICE); 1077 if (dma_mapping_error(qidev, edesc->assoclen_dma)) { 1078 dev_err(qidev, "unable to map assoclen\n"); 1079 caam_unmap(qidev, req->src, req->dst, src_nents, dst_nents, 1080 iv_dma, ivsize, DMA_TO_DEVICE, 0, 0); 1081 qi_cache_free(edesc); 1082 return ERR_PTR(-ENOMEM); 1083 } 1084 1085 dma_to_qm_sg_one(sg_table, edesc->assoclen_dma, 4, 0); 1086 qm_sg_index++; 1087 if (ivsize) { 1088 dma_to_qm_sg_one(sg_table + qm_sg_index, iv_dma, ivsize, 0); 1089 qm_sg_index++; 1090 } 1091 sg_to_qm_sg_last(req->src, src_len, sg_table + qm_sg_index, 0); 1092 qm_sg_index += mapped_src_nents; 1093 1094 if (mapped_dst_nents > 1) 1095 sg_to_qm_sg_last(req->dst, dst_len, sg_table + qm_sg_index, 0); 1096 1097 qm_sg_dma = dma_map_single(qidev, sg_table, qm_sg_bytes, DMA_TO_DEVICE); 1098 if (dma_mapping_error(qidev, qm_sg_dma)) { 1099 dev_err(qidev, "unable to map S/G table\n"); 1100 dma_unmap_single(qidev, edesc->assoclen_dma, 4, DMA_TO_DEVICE); 1101 caam_unmap(qidev, req->src, req->dst, src_nents, dst_nents, 1102 iv_dma, ivsize, DMA_TO_DEVICE, 0, 0); 1103 qi_cache_free(edesc); 1104 return ERR_PTR(-ENOMEM); 1105 } 1106 1107 edesc->qm_sg_dma = qm_sg_dma; 1108 edesc->qm_sg_bytes = qm_sg_bytes; 1109 1110 out_len = req->assoclen + req->cryptlen + 1111 (encrypt ? ctx->authsize : (-ctx->authsize)); 1112 in_len = 4 + ivsize + req->assoclen + req->cryptlen; 1113 1114 fd_sgt = &edesc->drv_req.fd_sgt[0]; 1115 dma_to_qm_sg_one_last_ext(&fd_sgt[1], qm_sg_dma, in_len, 0); 1116 1117 if (req->dst == req->src) { 1118 if (mapped_src_nents == 1) 1119 dma_to_qm_sg_one(&fd_sgt[0], sg_dma_address(req->src), 1120 out_len, 0); 1121 else 1122 dma_to_qm_sg_one_ext(&fd_sgt[0], qm_sg_dma + 1123 (1 + !!ivsize) * sizeof(*sg_table), 1124 out_len, 0); 1125 } else if (mapped_dst_nents <= 1) { 1126 dma_to_qm_sg_one(&fd_sgt[0], sg_dma_address(req->dst), out_len, 1127 0); 1128 } else { 1129 dma_to_qm_sg_one_ext(&fd_sgt[0], qm_sg_dma + sizeof(*sg_table) * 1130 qm_sg_index, out_len, 0); 1131 } 1132 1133 return edesc; 1134 } 1135 1136 static inline int aead_crypt(struct aead_request *req, bool encrypt) 1137 { 1138 struct aead_edesc *edesc; 1139 struct crypto_aead *aead = crypto_aead_reqtfm(req); 1140 struct caam_ctx *ctx = crypto_aead_ctx(aead); 1141 int ret; 1142 1143 if (unlikely(caam_congested)) 1144 return -EAGAIN; 1145 1146 /* allocate extended descriptor */ 1147 edesc = aead_edesc_alloc(req, encrypt); 1148 if (IS_ERR_OR_NULL(edesc)) 1149 return PTR_ERR(edesc); 1150 1151 /* Create and submit job descriptor */ 1152 ret = caam_qi_enqueue(ctx->qidev, &edesc->drv_req); 1153 if (!ret) { 1154 ret = -EINPROGRESS; 1155 } else { 1156 aead_unmap(ctx->qidev, edesc, req); 1157 qi_cache_free(edesc); 1158 } 1159 1160 return ret; 1161 } 1162 1163 static int aead_encrypt(struct aead_request *req) 1164 { 1165 return aead_crypt(req, true); 1166 } 1167 1168 static int aead_decrypt(struct aead_request *req) 1169 { 1170 return aead_crypt(req, false); 1171 } 1172 1173 static int ipsec_gcm_encrypt(struct aead_request *req) 1174 { 1175 return crypto_ipsec_check_assoclen(req->assoclen) ? : aead_crypt(req, 1176 true); 1177 } 1178 1179 static int ipsec_gcm_decrypt(struct aead_request *req) 1180 { 1181 return crypto_ipsec_check_assoclen(req->assoclen) ? : aead_crypt(req, 1182 false); 1183 } 1184 1185 static void skcipher_done(struct caam_drv_req *drv_req, u32 status) 1186 { 1187 struct skcipher_edesc *edesc; 1188 struct skcipher_request *req = drv_req->app_ctx; 1189 struct crypto_skcipher *skcipher = crypto_skcipher_reqtfm(req); 1190 struct caam_ctx *caam_ctx = crypto_skcipher_ctx(skcipher); 1191 struct device *qidev = caam_ctx->qidev; 1192 int ivsize = crypto_skcipher_ivsize(skcipher); 1193 int ecode = 0; 1194 1195 dev_dbg(qidev, "%s %d: status 0x%x\n", __func__, __LINE__, status); 1196 1197 edesc = container_of(drv_req, typeof(*edesc), drv_req); 1198 1199 if (status) 1200 ecode = caam_jr_strstatus(qidev, status); 1201 1202 print_hex_dump_debug("dstiv @" __stringify(__LINE__)": ", 1203 DUMP_PREFIX_ADDRESS, 16, 4, req->iv, 1204 edesc->src_nents > 1 ? 100 : ivsize, 1); 1205 caam_dump_sg("dst @" __stringify(__LINE__)": ", 1206 DUMP_PREFIX_ADDRESS, 16, 4, req->dst, 1207 edesc->dst_nents > 1 ? 100 : req->cryptlen, 1); 1208 1209 skcipher_unmap(qidev, edesc, req); 1210 1211 /* 1212 * The crypto API expects us to set the IV (req->iv) to the last 1213 * ciphertext block (CBC mode) or last counter (CTR mode). 1214 * This is used e.g. by the CTS mode. 1215 */ 1216 if (!ecode) 1217 memcpy(req->iv, (u8 *)&edesc->sgt[0] + edesc->qm_sg_bytes, 1218 ivsize); 1219 1220 qi_cache_free(edesc); 1221 skcipher_request_complete(req, ecode); 1222 } 1223 1224 static struct skcipher_edesc *skcipher_edesc_alloc(struct skcipher_request *req, 1225 bool encrypt) 1226 { 1227 struct crypto_skcipher *skcipher = crypto_skcipher_reqtfm(req); 1228 struct caam_ctx *ctx = crypto_skcipher_ctx(skcipher); 1229 struct device *qidev = ctx->qidev; 1230 gfp_t flags = (req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP) ? 1231 GFP_KERNEL : GFP_ATOMIC; 1232 int src_nents, mapped_src_nents, dst_nents = 0, mapped_dst_nents = 0; 1233 struct skcipher_edesc *edesc; 1234 dma_addr_t iv_dma; 1235 u8 *iv; 1236 int ivsize = crypto_skcipher_ivsize(skcipher); 1237 int dst_sg_idx, qm_sg_ents, qm_sg_bytes; 1238 struct qm_sg_entry *sg_table, *fd_sgt; 1239 struct caam_drv_ctx *drv_ctx; 1240 1241 drv_ctx = get_drv_ctx(ctx, encrypt ? ENCRYPT : DECRYPT); 1242 if (IS_ERR_OR_NULL(drv_ctx)) 1243 return (struct skcipher_edesc *)drv_ctx; 1244 1245 src_nents = sg_nents_for_len(req->src, req->cryptlen); 1246 if (unlikely(src_nents < 0)) { 1247 dev_err(qidev, "Insufficient bytes (%d) in src S/G\n", 1248 req->cryptlen); 1249 return ERR_PTR(src_nents); 1250 } 1251 1252 if (unlikely(req->src != req->dst)) { 1253 dst_nents = sg_nents_for_len(req->dst, req->cryptlen); 1254 if (unlikely(dst_nents < 0)) { 1255 dev_err(qidev, "Insufficient bytes (%d) in dst S/G\n", 1256 req->cryptlen); 1257 return ERR_PTR(dst_nents); 1258 } 1259 1260 mapped_src_nents = dma_map_sg(qidev, req->src, src_nents, 1261 DMA_TO_DEVICE); 1262 if (unlikely(!mapped_src_nents)) { 1263 dev_err(qidev, "unable to map source\n"); 1264 return ERR_PTR(-ENOMEM); 1265 } 1266 1267 mapped_dst_nents = dma_map_sg(qidev, req->dst, dst_nents, 1268 DMA_FROM_DEVICE); 1269 if (unlikely(!mapped_dst_nents)) { 1270 dev_err(qidev, "unable to map destination\n"); 1271 dma_unmap_sg(qidev, req->src, src_nents, DMA_TO_DEVICE); 1272 return ERR_PTR(-ENOMEM); 1273 } 1274 } else { 1275 mapped_src_nents = dma_map_sg(qidev, req->src, src_nents, 1276 DMA_BIDIRECTIONAL); 1277 if (unlikely(!mapped_src_nents)) { 1278 dev_err(qidev, "unable to map source\n"); 1279 return ERR_PTR(-ENOMEM); 1280 } 1281 } 1282 1283 qm_sg_ents = 1 + mapped_src_nents; 1284 dst_sg_idx = qm_sg_ents; 1285 1286 /* 1287 * Input, output HW S/G tables: [IV, src][dst, IV] 1288 * IV entries point to the same buffer 1289 * If src == dst, S/G entries are reused (S/G tables overlap) 1290 * 1291 * HW reads 4 S/G entries at a time; make sure the reads don't go beyond 1292 * the end of the table by allocating more S/G entries. 1293 */ 1294 if (req->src != req->dst) 1295 qm_sg_ents += pad_sg_nents(mapped_dst_nents + 1); 1296 else 1297 qm_sg_ents = 1 + pad_sg_nents(qm_sg_ents); 1298 1299 qm_sg_bytes = qm_sg_ents * sizeof(struct qm_sg_entry); 1300 if (unlikely(offsetof(struct skcipher_edesc, sgt) + qm_sg_bytes + 1301 ivsize > CAAM_QI_MEMCACHE_SIZE)) { 1302 dev_err(qidev, "No space for %d S/G entries and/or %dB IV\n", 1303 qm_sg_ents, ivsize); 1304 caam_unmap(qidev, req->src, req->dst, src_nents, dst_nents, 0, 1305 0, DMA_NONE, 0, 0); 1306 return ERR_PTR(-ENOMEM); 1307 } 1308 1309 /* allocate space for base edesc, link tables and IV */ 1310 edesc = qi_cache_alloc(GFP_DMA | flags); 1311 if (unlikely(!edesc)) { 1312 dev_err(qidev, "could not allocate extended descriptor\n"); 1313 caam_unmap(qidev, req->src, req->dst, src_nents, dst_nents, 0, 1314 0, DMA_NONE, 0, 0); 1315 return ERR_PTR(-ENOMEM); 1316 } 1317 1318 /* Make sure IV is located in a DMAable area */ 1319 sg_table = &edesc->sgt[0]; 1320 iv = (u8 *)(sg_table + qm_sg_ents); 1321 memcpy(iv, req->iv, ivsize); 1322 1323 iv_dma = dma_map_single(qidev, iv, ivsize, DMA_BIDIRECTIONAL); 1324 if (dma_mapping_error(qidev, iv_dma)) { 1325 dev_err(qidev, "unable to map IV\n"); 1326 caam_unmap(qidev, req->src, req->dst, src_nents, dst_nents, 0, 1327 0, DMA_NONE, 0, 0); 1328 qi_cache_free(edesc); 1329 return ERR_PTR(-ENOMEM); 1330 } 1331 1332 edesc->src_nents = src_nents; 1333 edesc->dst_nents = dst_nents; 1334 edesc->iv_dma = iv_dma; 1335 edesc->qm_sg_bytes = qm_sg_bytes; 1336 edesc->drv_req.app_ctx = req; 1337 edesc->drv_req.cbk = skcipher_done; 1338 edesc->drv_req.drv_ctx = drv_ctx; 1339 1340 dma_to_qm_sg_one(sg_table, iv_dma, ivsize, 0); 1341 sg_to_qm_sg(req->src, req->cryptlen, sg_table + 1, 0); 1342 1343 if (req->src != req->dst) 1344 sg_to_qm_sg(req->dst, req->cryptlen, sg_table + dst_sg_idx, 0); 1345 1346 dma_to_qm_sg_one(sg_table + dst_sg_idx + mapped_dst_nents, iv_dma, 1347 ivsize, 0); 1348 1349 edesc->qm_sg_dma = dma_map_single(qidev, sg_table, edesc->qm_sg_bytes, 1350 DMA_TO_DEVICE); 1351 if (dma_mapping_error(qidev, edesc->qm_sg_dma)) { 1352 dev_err(qidev, "unable to map S/G table\n"); 1353 caam_unmap(qidev, req->src, req->dst, src_nents, dst_nents, 1354 iv_dma, ivsize, DMA_BIDIRECTIONAL, 0, 0); 1355 qi_cache_free(edesc); 1356 return ERR_PTR(-ENOMEM); 1357 } 1358 1359 fd_sgt = &edesc->drv_req.fd_sgt[0]; 1360 1361 dma_to_qm_sg_one_last_ext(&fd_sgt[1], edesc->qm_sg_dma, 1362 ivsize + req->cryptlen, 0); 1363 1364 if (req->src == req->dst) 1365 dma_to_qm_sg_one_ext(&fd_sgt[0], edesc->qm_sg_dma + 1366 sizeof(*sg_table), req->cryptlen + ivsize, 1367 0); 1368 else 1369 dma_to_qm_sg_one_ext(&fd_sgt[0], edesc->qm_sg_dma + dst_sg_idx * 1370 sizeof(*sg_table), req->cryptlen + ivsize, 1371 0); 1372 1373 return edesc; 1374 } 1375 1376 static inline int skcipher_crypt(struct skcipher_request *req, bool encrypt) 1377 { 1378 struct skcipher_edesc *edesc; 1379 struct crypto_skcipher *skcipher = crypto_skcipher_reqtfm(req); 1380 struct caam_ctx *ctx = crypto_skcipher_ctx(skcipher); 1381 int ret; 1382 1383 if (!req->cryptlen) 1384 return 0; 1385 1386 if (unlikely(caam_congested)) 1387 return -EAGAIN; 1388 1389 /* allocate extended descriptor */ 1390 edesc = skcipher_edesc_alloc(req, encrypt); 1391 if (IS_ERR(edesc)) 1392 return PTR_ERR(edesc); 1393 1394 ret = caam_qi_enqueue(ctx->qidev, &edesc->drv_req); 1395 if (!ret) { 1396 ret = -EINPROGRESS; 1397 } else { 1398 skcipher_unmap(ctx->qidev, edesc, req); 1399 qi_cache_free(edesc); 1400 } 1401 1402 return ret; 1403 } 1404 1405 static int skcipher_encrypt(struct skcipher_request *req) 1406 { 1407 return skcipher_crypt(req, true); 1408 } 1409 1410 static int skcipher_decrypt(struct skcipher_request *req) 1411 { 1412 return skcipher_crypt(req, false); 1413 } 1414 1415 static struct caam_skcipher_alg driver_algs[] = { 1416 { 1417 .skcipher = { 1418 .base = { 1419 .cra_name = "cbc(aes)", 1420 .cra_driver_name = "cbc-aes-caam-qi", 1421 .cra_blocksize = AES_BLOCK_SIZE, 1422 }, 1423 .setkey = aes_skcipher_setkey, 1424 .encrypt = skcipher_encrypt, 1425 .decrypt = skcipher_decrypt, 1426 .min_keysize = AES_MIN_KEY_SIZE, 1427 .max_keysize = AES_MAX_KEY_SIZE, 1428 .ivsize = AES_BLOCK_SIZE, 1429 }, 1430 .caam.class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CBC, 1431 }, 1432 { 1433 .skcipher = { 1434 .base = { 1435 .cra_name = "cbc(des3_ede)", 1436 .cra_driver_name = "cbc-3des-caam-qi", 1437 .cra_blocksize = DES3_EDE_BLOCK_SIZE, 1438 }, 1439 .setkey = des3_skcipher_setkey, 1440 .encrypt = skcipher_encrypt, 1441 .decrypt = skcipher_decrypt, 1442 .min_keysize = DES3_EDE_KEY_SIZE, 1443 .max_keysize = DES3_EDE_KEY_SIZE, 1444 .ivsize = DES3_EDE_BLOCK_SIZE, 1445 }, 1446 .caam.class1_alg_type = OP_ALG_ALGSEL_3DES | OP_ALG_AAI_CBC, 1447 }, 1448 { 1449 .skcipher = { 1450 .base = { 1451 .cra_name = "cbc(des)", 1452 .cra_driver_name = "cbc-des-caam-qi", 1453 .cra_blocksize = DES_BLOCK_SIZE, 1454 }, 1455 .setkey = des_skcipher_setkey, 1456 .encrypt = skcipher_encrypt, 1457 .decrypt = skcipher_decrypt, 1458 .min_keysize = DES_KEY_SIZE, 1459 .max_keysize = DES_KEY_SIZE, 1460 .ivsize = DES_BLOCK_SIZE, 1461 }, 1462 .caam.class1_alg_type = OP_ALG_ALGSEL_DES | OP_ALG_AAI_CBC, 1463 }, 1464 { 1465 .skcipher = { 1466 .base = { 1467 .cra_name = "ctr(aes)", 1468 .cra_driver_name = "ctr-aes-caam-qi", 1469 .cra_blocksize = 1, 1470 }, 1471 .setkey = ctr_skcipher_setkey, 1472 .encrypt = skcipher_encrypt, 1473 .decrypt = skcipher_decrypt, 1474 .min_keysize = AES_MIN_KEY_SIZE, 1475 .max_keysize = AES_MAX_KEY_SIZE, 1476 .ivsize = AES_BLOCK_SIZE, 1477 .chunksize = AES_BLOCK_SIZE, 1478 }, 1479 .caam.class1_alg_type = OP_ALG_ALGSEL_AES | 1480 OP_ALG_AAI_CTR_MOD128, 1481 }, 1482 { 1483 .skcipher = { 1484 .base = { 1485 .cra_name = "rfc3686(ctr(aes))", 1486 .cra_driver_name = "rfc3686-ctr-aes-caam-qi", 1487 .cra_blocksize = 1, 1488 }, 1489 .setkey = rfc3686_skcipher_setkey, 1490 .encrypt = skcipher_encrypt, 1491 .decrypt = skcipher_decrypt, 1492 .min_keysize = AES_MIN_KEY_SIZE + 1493 CTR_RFC3686_NONCE_SIZE, 1494 .max_keysize = AES_MAX_KEY_SIZE + 1495 CTR_RFC3686_NONCE_SIZE, 1496 .ivsize = CTR_RFC3686_IV_SIZE, 1497 .chunksize = AES_BLOCK_SIZE, 1498 }, 1499 .caam = { 1500 .class1_alg_type = OP_ALG_ALGSEL_AES | 1501 OP_ALG_AAI_CTR_MOD128, 1502 .rfc3686 = true, 1503 }, 1504 }, 1505 { 1506 .skcipher = { 1507 .base = { 1508 .cra_name = "xts(aes)", 1509 .cra_driver_name = "xts-aes-caam-qi", 1510 .cra_blocksize = AES_BLOCK_SIZE, 1511 }, 1512 .setkey = xts_skcipher_setkey, 1513 .encrypt = skcipher_encrypt, 1514 .decrypt = skcipher_decrypt, 1515 .min_keysize = 2 * AES_MIN_KEY_SIZE, 1516 .max_keysize = 2 * AES_MAX_KEY_SIZE, 1517 .ivsize = AES_BLOCK_SIZE, 1518 }, 1519 .caam.class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_XTS, 1520 }, 1521 }; 1522 1523 static struct caam_aead_alg driver_aeads[] = { 1524 { 1525 .aead = { 1526 .base = { 1527 .cra_name = "rfc4106(gcm(aes))", 1528 .cra_driver_name = "rfc4106-gcm-aes-caam-qi", 1529 .cra_blocksize = 1, 1530 }, 1531 .setkey = rfc4106_setkey, 1532 .setauthsize = rfc4106_setauthsize, 1533 .encrypt = ipsec_gcm_encrypt, 1534 .decrypt = ipsec_gcm_decrypt, 1535 .ivsize = 8, 1536 .maxauthsize = AES_BLOCK_SIZE, 1537 }, 1538 .caam = { 1539 .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_GCM, 1540 .nodkp = true, 1541 }, 1542 }, 1543 { 1544 .aead = { 1545 .base = { 1546 .cra_name = "rfc4543(gcm(aes))", 1547 .cra_driver_name = "rfc4543-gcm-aes-caam-qi", 1548 .cra_blocksize = 1, 1549 }, 1550 .setkey = rfc4543_setkey, 1551 .setauthsize = rfc4543_setauthsize, 1552 .encrypt = ipsec_gcm_encrypt, 1553 .decrypt = ipsec_gcm_decrypt, 1554 .ivsize = 8, 1555 .maxauthsize = AES_BLOCK_SIZE, 1556 }, 1557 .caam = { 1558 .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_GCM, 1559 .nodkp = true, 1560 }, 1561 }, 1562 /* Galois Counter Mode */ 1563 { 1564 .aead = { 1565 .base = { 1566 .cra_name = "gcm(aes)", 1567 .cra_driver_name = "gcm-aes-caam-qi", 1568 .cra_blocksize = 1, 1569 }, 1570 .setkey = gcm_setkey, 1571 .setauthsize = gcm_setauthsize, 1572 .encrypt = aead_encrypt, 1573 .decrypt = aead_decrypt, 1574 .ivsize = 12, 1575 .maxauthsize = AES_BLOCK_SIZE, 1576 }, 1577 .caam = { 1578 .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_GCM, 1579 .nodkp = true, 1580 } 1581 }, 1582 /* single-pass ipsec_esp descriptor */ 1583 { 1584 .aead = { 1585 .base = { 1586 .cra_name = "authenc(hmac(md5),cbc(aes))", 1587 .cra_driver_name = "authenc-hmac-md5-" 1588 "cbc-aes-caam-qi", 1589 .cra_blocksize = AES_BLOCK_SIZE, 1590 }, 1591 .setkey = aead_setkey, 1592 .setauthsize = aead_setauthsize, 1593 .encrypt = aead_encrypt, 1594 .decrypt = aead_decrypt, 1595 .ivsize = AES_BLOCK_SIZE, 1596 .maxauthsize = MD5_DIGEST_SIZE, 1597 }, 1598 .caam = { 1599 .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CBC, 1600 .class2_alg_type = OP_ALG_ALGSEL_MD5 | 1601 OP_ALG_AAI_HMAC_PRECOMP, 1602 } 1603 }, 1604 { 1605 .aead = { 1606 .base = { 1607 .cra_name = "echainiv(authenc(hmac(md5)," 1608 "cbc(aes)))", 1609 .cra_driver_name = "echainiv-authenc-hmac-md5-" 1610 "cbc-aes-caam-qi", 1611 .cra_blocksize = AES_BLOCK_SIZE, 1612 }, 1613 .setkey = aead_setkey, 1614 .setauthsize = aead_setauthsize, 1615 .encrypt = aead_encrypt, 1616 .decrypt = aead_decrypt, 1617 .ivsize = AES_BLOCK_SIZE, 1618 .maxauthsize = MD5_DIGEST_SIZE, 1619 }, 1620 .caam = { 1621 .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CBC, 1622 .class2_alg_type = OP_ALG_ALGSEL_MD5 | 1623 OP_ALG_AAI_HMAC_PRECOMP, 1624 .geniv = true, 1625 } 1626 }, 1627 { 1628 .aead = { 1629 .base = { 1630 .cra_name = "authenc(hmac(sha1),cbc(aes))", 1631 .cra_driver_name = "authenc-hmac-sha1-" 1632 "cbc-aes-caam-qi", 1633 .cra_blocksize = AES_BLOCK_SIZE, 1634 }, 1635 .setkey = aead_setkey, 1636 .setauthsize = aead_setauthsize, 1637 .encrypt = aead_encrypt, 1638 .decrypt = aead_decrypt, 1639 .ivsize = AES_BLOCK_SIZE, 1640 .maxauthsize = SHA1_DIGEST_SIZE, 1641 }, 1642 .caam = { 1643 .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CBC, 1644 .class2_alg_type = OP_ALG_ALGSEL_SHA1 | 1645 OP_ALG_AAI_HMAC_PRECOMP, 1646 } 1647 }, 1648 { 1649 .aead = { 1650 .base = { 1651 .cra_name = "echainiv(authenc(hmac(sha1)," 1652 "cbc(aes)))", 1653 .cra_driver_name = "echainiv-authenc-" 1654 "hmac-sha1-cbc-aes-caam-qi", 1655 .cra_blocksize = AES_BLOCK_SIZE, 1656 }, 1657 .setkey = aead_setkey, 1658 .setauthsize = aead_setauthsize, 1659 .encrypt = aead_encrypt, 1660 .decrypt = aead_decrypt, 1661 .ivsize = AES_BLOCK_SIZE, 1662 .maxauthsize = SHA1_DIGEST_SIZE, 1663 }, 1664 .caam = { 1665 .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CBC, 1666 .class2_alg_type = OP_ALG_ALGSEL_SHA1 | 1667 OP_ALG_AAI_HMAC_PRECOMP, 1668 .geniv = true, 1669 }, 1670 }, 1671 { 1672 .aead = { 1673 .base = { 1674 .cra_name = "authenc(hmac(sha224),cbc(aes))", 1675 .cra_driver_name = "authenc-hmac-sha224-" 1676 "cbc-aes-caam-qi", 1677 .cra_blocksize = AES_BLOCK_SIZE, 1678 }, 1679 .setkey = aead_setkey, 1680 .setauthsize = aead_setauthsize, 1681 .encrypt = aead_encrypt, 1682 .decrypt = aead_decrypt, 1683 .ivsize = AES_BLOCK_SIZE, 1684 .maxauthsize = SHA224_DIGEST_SIZE, 1685 }, 1686 .caam = { 1687 .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CBC, 1688 .class2_alg_type = OP_ALG_ALGSEL_SHA224 | 1689 OP_ALG_AAI_HMAC_PRECOMP, 1690 } 1691 }, 1692 { 1693 .aead = { 1694 .base = { 1695 .cra_name = "echainiv(authenc(hmac(sha224)," 1696 "cbc(aes)))", 1697 .cra_driver_name = "echainiv-authenc-" 1698 "hmac-sha224-cbc-aes-caam-qi", 1699 .cra_blocksize = AES_BLOCK_SIZE, 1700 }, 1701 .setkey = aead_setkey, 1702 .setauthsize = aead_setauthsize, 1703 .encrypt = aead_encrypt, 1704 .decrypt = aead_decrypt, 1705 .ivsize = AES_BLOCK_SIZE, 1706 .maxauthsize = SHA224_DIGEST_SIZE, 1707 }, 1708 .caam = { 1709 .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CBC, 1710 .class2_alg_type = OP_ALG_ALGSEL_SHA224 | 1711 OP_ALG_AAI_HMAC_PRECOMP, 1712 .geniv = true, 1713 } 1714 }, 1715 { 1716 .aead = { 1717 .base = { 1718 .cra_name = "authenc(hmac(sha256),cbc(aes))", 1719 .cra_driver_name = "authenc-hmac-sha256-" 1720 "cbc-aes-caam-qi", 1721 .cra_blocksize = AES_BLOCK_SIZE, 1722 }, 1723 .setkey = aead_setkey, 1724 .setauthsize = aead_setauthsize, 1725 .encrypt = aead_encrypt, 1726 .decrypt = aead_decrypt, 1727 .ivsize = AES_BLOCK_SIZE, 1728 .maxauthsize = SHA256_DIGEST_SIZE, 1729 }, 1730 .caam = { 1731 .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CBC, 1732 .class2_alg_type = OP_ALG_ALGSEL_SHA256 | 1733 OP_ALG_AAI_HMAC_PRECOMP, 1734 } 1735 }, 1736 { 1737 .aead = { 1738 .base = { 1739 .cra_name = "echainiv(authenc(hmac(sha256)," 1740 "cbc(aes)))", 1741 .cra_driver_name = "echainiv-authenc-" 1742 "hmac-sha256-cbc-aes-" 1743 "caam-qi", 1744 .cra_blocksize = AES_BLOCK_SIZE, 1745 }, 1746 .setkey = aead_setkey, 1747 .setauthsize = aead_setauthsize, 1748 .encrypt = aead_encrypt, 1749 .decrypt = aead_decrypt, 1750 .ivsize = AES_BLOCK_SIZE, 1751 .maxauthsize = SHA256_DIGEST_SIZE, 1752 }, 1753 .caam = { 1754 .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CBC, 1755 .class2_alg_type = OP_ALG_ALGSEL_SHA256 | 1756 OP_ALG_AAI_HMAC_PRECOMP, 1757 .geniv = true, 1758 } 1759 }, 1760 { 1761 .aead = { 1762 .base = { 1763 .cra_name = "authenc(hmac(sha384),cbc(aes))", 1764 .cra_driver_name = "authenc-hmac-sha384-" 1765 "cbc-aes-caam-qi", 1766 .cra_blocksize = AES_BLOCK_SIZE, 1767 }, 1768 .setkey = aead_setkey, 1769 .setauthsize = aead_setauthsize, 1770 .encrypt = aead_encrypt, 1771 .decrypt = aead_decrypt, 1772 .ivsize = AES_BLOCK_SIZE, 1773 .maxauthsize = SHA384_DIGEST_SIZE, 1774 }, 1775 .caam = { 1776 .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CBC, 1777 .class2_alg_type = OP_ALG_ALGSEL_SHA384 | 1778 OP_ALG_AAI_HMAC_PRECOMP, 1779 } 1780 }, 1781 { 1782 .aead = { 1783 .base = { 1784 .cra_name = "echainiv(authenc(hmac(sha384)," 1785 "cbc(aes)))", 1786 .cra_driver_name = "echainiv-authenc-" 1787 "hmac-sha384-cbc-aes-" 1788 "caam-qi", 1789 .cra_blocksize = AES_BLOCK_SIZE, 1790 }, 1791 .setkey = aead_setkey, 1792 .setauthsize = aead_setauthsize, 1793 .encrypt = aead_encrypt, 1794 .decrypt = aead_decrypt, 1795 .ivsize = AES_BLOCK_SIZE, 1796 .maxauthsize = SHA384_DIGEST_SIZE, 1797 }, 1798 .caam = { 1799 .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CBC, 1800 .class2_alg_type = OP_ALG_ALGSEL_SHA384 | 1801 OP_ALG_AAI_HMAC_PRECOMP, 1802 .geniv = true, 1803 } 1804 }, 1805 { 1806 .aead = { 1807 .base = { 1808 .cra_name = "authenc(hmac(sha512),cbc(aes))", 1809 .cra_driver_name = "authenc-hmac-sha512-" 1810 "cbc-aes-caam-qi", 1811 .cra_blocksize = AES_BLOCK_SIZE, 1812 }, 1813 .setkey = aead_setkey, 1814 .setauthsize = aead_setauthsize, 1815 .encrypt = aead_encrypt, 1816 .decrypt = aead_decrypt, 1817 .ivsize = AES_BLOCK_SIZE, 1818 .maxauthsize = SHA512_DIGEST_SIZE, 1819 }, 1820 .caam = { 1821 .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CBC, 1822 .class2_alg_type = OP_ALG_ALGSEL_SHA512 | 1823 OP_ALG_AAI_HMAC_PRECOMP, 1824 } 1825 }, 1826 { 1827 .aead = { 1828 .base = { 1829 .cra_name = "echainiv(authenc(hmac(sha512)," 1830 "cbc(aes)))", 1831 .cra_driver_name = "echainiv-authenc-" 1832 "hmac-sha512-cbc-aes-" 1833 "caam-qi", 1834 .cra_blocksize = AES_BLOCK_SIZE, 1835 }, 1836 .setkey = aead_setkey, 1837 .setauthsize = aead_setauthsize, 1838 .encrypt = aead_encrypt, 1839 .decrypt = aead_decrypt, 1840 .ivsize = AES_BLOCK_SIZE, 1841 .maxauthsize = SHA512_DIGEST_SIZE, 1842 }, 1843 .caam = { 1844 .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CBC, 1845 .class2_alg_type = OP_ALG_ALGSEL_SHA512 | 1846 OP_ALG_AAI_HMAC_PRECOMP, 1847 .geniv = true, 1848 } 1849 }, 1850 { 1851 .aead = { 1852 .base = { 1853 .cra_name = "authenc(hmac(md5),cbc(des3_ede))", 1854 .cra_driver_name = "authenc-hmac-md5-" 1855 "cbc-des3_ede-caam-qi", 1856 .cra_blocksize = DES3_EDE_BLOCK_SIZE, 1857 }, 1858 .setkey = des3_aead_setkey, 1859 .setauthsize = aead_setauthsize, 1860 .encrypt = aead_encrypt, 1861 .decrypt = aead_decrypt, 1862 .ivsize = DES3_EDE_BLOCK_SIZE, 1863 .maxauthsize = MD5_DIGEST_SIZE, 1864 }, 1865 .caam = { 1866 .class1_alg_type = OP_ALG_ALGSEL_3DES | OP_ALG_AAI_CBC, 1867 .class2_alg_type = OP_ALG_ALGSEL_MD5 | 1868 OP_ALG_AAI_HMAC_PRECOMP, 1869 } 1870 }, 1871 { 1872 .aead = { 1873 .base = { 1874 .cra_name = "echainiv(authenc(hmac(md5)," 1875 "cbc(des3_ede)))", 1876 .cra_driver_name = "echainiv-authenc-hmac-md5-" 1877 "cbc-des3_ede-caam-qi", 1878 .cra_blocksize = DES3_EDE_BLOCK_SIZE, 1879 }, 1880 .setkey = des3_aead_setkey, 1881 .setauthsize = aead_setauthsize, 1882 .encrypt = aead_encrypt, 1883 .decrypt = aead_decrypt, 1884 .ivsize = DES3_EDE_BLOCK_SIZE, 1885 .maxauthsize = MD5_DIGEST_SIZE, 1886 }, 1887 .caam = { 1888 .class1_alg_type = OP_ALG_ALGSEL_3DES | OP_ALG_AAI_CBC, 1889 .class2_alg_type = OP_ALG_ALGSEL_MD5 | 1890 OP_ALG_AAI_HMAC_PRECOMP, 1891 .geniv = true, 1892 } 1893 }, 1894 { 1895 .aead = { 1896 .base = { 1897 .cra_name = "authenc(hmac(sha1)," 1898 "cbc(des3_ede))", 1899 .cra_driver_name = "authenc-hmac-sha1-" 1900 "cbc-des3_ede-caam-qi", 1901 .cra_blocksize = DES3_EDE_BLOCK_SIZE, 1902 }, 1903 .setkey = des3_aead_setkey, 1904 .setauthsize = aead_setauthsize, 1905 .encrypt = aead_encrypt, 1906 .decrypt = aead_decrypt, 1907 .ivsize = DES3_EDE_BLOCK_SIZE, 1908 .maxauthsize = SHA1_DIGEST_SIZE, 1909 }, 1910 .caam = { 1911 .class1_alg_type = OP_ALG_ALGSEL_3DES | OP_ALG_AAI_CBC, 1912 .class2_alg_type = OP_ALG_ALGSEL_SHA1 | 1913 OP_ALG_AAI_HMAC_PRECOMP, 1914 }, 1915 }, 1916 { 1917 .aead = { 1918 .base = { 1919 .cra_name = "echainiv(authenc(hmac(sha1)," 1920 "cbc(des3_ede)))", 1921 .cra_driver_name = "echainiv-authenc-" 1922 "hmac-sha1-" 1923 "cbc-des3_ede-caam-qi", 1924 .cra_blocksize = DES3_EDE_BLOCK_SIZE, 1925 }, 1926 .setkey = des3_aead_setkey, 1927 .setauthsize = aead_setauthsize, 1928 .encrypt = aead_encrypt, 1929 .decrypt = aead_decrypt, 1930 .ivsize = DES3_EDE_BLOCK_SIZE, 1931 .maxauthsize = SHA1_DIGEST_SIZE, 1932 }, 1933 .caam = { 1934 .class1_alg_type = OP_ALG_ALGSEL_3DES | OP_ALG_AAI_CBC, 1935 .class2_alg_type = OP_ALG_ALGSEL_SHA1 | 1936 OP_ALG_AAI_HMAC_PRECOMP, 1937 .geniv = true, 1938 } 1939 }, 1940 { 1941 .aead = { 1942 .base = { 1943 .cra_name = "authenc(hmac(sha224)," 1944 "cbc(des3_ede))", 1945 .cra_driver_name = "authenc-hmac-sha224-" 1946 "cbc-des3_ede-caam-qi", 1947 .cra_blocksize = DES3_EDE_BLOCK_SIZE, 1948 }, 1949 .setkey = des3_aead_setkey, 1950 .setauthsize = aead_setauthsize, 1951 .encrypt = aead_encrypt, 1952 .decrypt = aead_decrypt, 1953 .ivsize = DES3_EDE_BLOCK_SIZE, 1954 .maxauthsize = SHA224_DIGEST_SIZE, 1955 }, 1956 .caam = { 1957 .class1_alg_type = OP_ALG_ALGSEL_3DES | OP_ALG_AAI_CBC, 1958 .class2_alg_type = OP_ALG_ALGSEL_SHA224 | 1959 OP_ALG_AAI_HMAC_PRECOMP, 1960 }, 1961 }, 1962 { 1963 .aead = { 1964 .base = { 1965 .cra_name = "echainiv(authenc(hmac(sha224)," 1966 "cbc(des3_ede)))", 1967 .cra_driver_name = "echainiv-authenc-" 1968 "hmac-sha224-" 1969 "cbc-des3_ede-caam-qi", 1970 .cra_blocksize = DES3_EDE_BLOCK_SIZE, 1971 }, 1972 .setkey = des3_aead_setkey, 1973 .setauthsize = aead_setauthsize, 1974 .encrypt = aead_encrypt, 1975 .decrypt = aead_decrypt, 1976 .ivsize = DES3_EDE_BLOCK_SIZE, 1977 .maxauthsize = SHA224_DIGEST_SIZE, 1978 }, 1979 .caam = { 1980 .class1_alg_type = OP_ALG_ALGSEL_3DES | OP_ALG_AAI_CBC, 1981 .class2_alg_type = OP_ALG_ALGSEL_SHA224 | 1982 OP_ALG_AAI_HMAC_PRECOMP, 1983 .geniv = true, 1984 } 1985 }, 1986 { 1987 .aead = { 1988 .base = { 1989 .cra_name = "authenc(hmac(sha256)," 1990 "cbc(des3_ede))", 1991 .cra_driver_name = "authenc-hmac-sha256-" 1992 "cbc-des3_ede-caam-qi", 1993 .cra_blocksize = DES3_EDE_BLOCK_SIZE, 1994 }, 1995 .setkey = des3_aead_setkey, 1996 .setauthsize = aead_setauthsize, 1997 .encrypt = aead_encrypt, 1998 .decrypt = aead_decrypt, 1999 .ivsize = DES3_EDE_BLOCK_SIZE, 2000 .maxauthsize = SHA256_DIGEST_SIZE, 2001 }, 2002 .caam = { 2003 .class1_alg_type = OP_ALG_ALGSEL_3DES | OP_ALG_AAI_CBC, 2004 .class2_alg_type = OP_ALG_ALGSEL_SHA256 | 2005 OP_ALG_AAI_HMAC_PRECOMP, 2006 }, 2007 }, 2008 { 2009 .aead = { 2010 .base = { 2011 .cra_name = "echainiv(authenc(hmac(sha256)," 2012 "cbc(des3_ede)))", 2013 .cra_driver_name = "echainiv-authenc-" 2014 "hmac-sha256-" 2015 "cbc-des3_ede-caam-qi", 2016 .cra_blocksize = DES3_EDE_BLOCK_SIZE, 2017 }, 2018 .setkey = des3_aead_setkey, 2019 .setauthsize = aead_setauthsize, 2020 .encrypt = aead_encrypt, 2021 .decrypt = aead_decrypt, 2022 .ivsize = DES3_EDE_BLOCK_SIZE, 2023 .maxauthsize = SHA256_DIGEST_SIZE, 2024 }, 2025 .caam = { 2026 .class1_alg_type = OP_ALG_ALGSEL_3DES | OP_ALG_AAI_CBC, 2027 .class2_alg_type = OP_ALG_ALGSEL_SHA256 | 2028 OP_ALG_AAI_HMAC_PRECOMP, 2029 .geniv = true, 2030 } 2031 }, 2032 { 2033 .aead = { 2034 .base = { 2035 .cra_name = "authenc(hmac(sha384)," 2036 "cbc(des3_ede))", 2037 .cra_driver_name = "authenc-hmac-sha384-" 2038 "cbc-des3_ede-caam-qi", 2039 .cra_blocksize = DES3_EDE_BLOCK_SIZE, 2040 }, 2041 .setkey = des3_aead_setkey, 2042 .setauthsize = aead_setauthsize, 2043 .encrypt = aead_encrypt, 2044 .decrypt = aead_decrypt, 2045 .ivsize = DES3_EDE_BLOCK_SIZE, 2046 .maxauthsize = SHA384_DIGEST_SIZE, 2047 }, 2048 .caam = { 2049 .class1_alg_type = OP_ALG_ALGSEL_3DES | OP_ALG_AAI_CBC, 2050 .class2_alg_type = OP_ALG_ALGSEL_SHA384 | 2051 OP_ALG_AAI_HMAC_PRECOMP, 2052 }, 2053 }, 2054 { 2055 .aead = { 2056 .base = { 2057 .cra_name = "echainiv(authenc(hmac(sha384)," 2058 "cbc(des3_ede)))", 2059 .cra_driver_name = "echainiv-authenc-" 2060 "hmac-sha384-" 2061 "cbc-des3_ede-caam-qi", 2062 .cra_blocksize = DES3_EDE_BLOCK_SIZE, 2063 }, 2064 .setkey = des3_aead_setkey, 2065 .setauthsize = aead_setauthsize, 2066 .encrypt = aead_encrypt, 2067 .decrypt = aead_decrypt, 2068 .ivsize = DES3_EDE_BLOCK_SIZE, 2069 .maxauthsize = SHA384_DIGEST_SIZE, 2070 }, 2071 .caam = { 2072 .class1_alg_type = OP_ALG_ALGSEL_3DES | OP_ALG_AAI_CBC, 2073 .class2_alg_type = OP_ALG_ALGSEL_SHA384 | 2074 OP_ALG_AAI_HMAC_PRECOMP, 2075 .geniv = true, 2076 } 2077 }, 2078 { 2079 .aead = { 2080 .base = { 2081 .cra_name = "authenc(hmac(sha512)," 2082 "cbc(des3_ede))", 2083 .cra_driver_name = "authenc-hmac-sha512-" 2084 "cbc-des3_ede-caam-qi", 2085 .cra_blocksize = DES3_EDE_BLOCK_SIZE, 2086 }, 2087 .setkey = des3_aead_setkey, 2088 .setauthsize = aead_setauthsize, 2089 .encrypt = aead_encrypt, 2090 .decrypt = aead_decrypt, 2091 .ivsize = DES3_EDE_BLOCK_SIZE, 2092 .maxauthsize = SHA512_DIGEST_SIZE, 2093 }, 2094 .caam = { 2095 .class1_alg_type = OP_ALG_ALGSEL_3DES | OP_ALG_AAI_CBC, 2096 .class2_alg_type = OP_ALG_ALGSEL_SHA512 | 2097 OP_ALG_AAI_HMAC_PRECOMP, 2098 }, 2099 }, 2100 { 2101 .aead = { 2102 .base = { 2103 .cra_name = "echainiv(authenc(hmac(sha512)," 2104 "cbc(des3_ede)))", 2105 .cra_driver_name = "echainiv-authenc-" 2106 "hmac-sha512-" 2107 "cbc-des3_ede-caam-qi", 2108 .cra_blocksize = DES3_EDE_BLOCK_SIZE, 2109 }, 2110 .setkey = des3_aead_setkey, 2111 .setauthsize = aead_setauthsize, 2112 .encrypt = aead_encrypt, 2113 .decrypt = aead_decrypt, 2114 .ivsize = DES3_EDE_BLOCK_SIZE, 2115 .maxauthsize = SHA512_DIGEST_SIZE, 2116 }, 2117 .caam = { 2118 .class1_alg_type = OP_ALG_ALGSEL_3DES | OP_ALG_AAI_CBC, 2119 .class2_alg_type = OP_ALG_ALGSEL_SHA512 | 2120 OP_ALG_AAI_HMAC_PRECOMP, 2121 .geniv = true, 2122 } 2123 }, 2124 { 2125 .aead = { 2126 .base = { 2127 .cra_name = "authenc(hmac(md5),cbc(des))", 2128 .cra_driver_name = "authenc-hmac-md5-" 2129 "cbc-des-caam-qi", 2130 .cra_blocksize = DES_BLOCK_SIZE, 2131 }, 2132 .setkey = aead_setkey, 2133 .setauthsize = aead_setauthsize, 2134 .encrypt = aead_encrypt, 2135 .decrypt = aead_decrypt, 2136 .ivsize = DES_BLOCK_SIZE, 2137 .maxauthsize = MD5_DIGEST_SIZE, 2138 }, 2139 .caam = { 2140 .class1_alg_type = OP_ALG_ALGSEL_DES | OP_ALG_AAI_CBC, 2141 .class2_alg_type = OP_ALG_ALGSEL_MD5 | 2142 OP_ALG_AAI_HMAC_PRECOMP, 2143 }, 2144 }, 2145 { 2146 .aead = { 2147 .base = { 2148 .cra_name = "echainiv(authenc(hmac(md5)," 2149 "cbc(des)))", 2150 .cra_driver_name = "echainiv-authenc-hmac-md5-" 2151 "cbc-des-caam-qi", 2152 .cra_blocksize = DES_BLOCK_SIZE, 2153 }, 2154 .setkey = aead_setkey, 2155 .setauthsize = aead_setauthsize, 2156 .encrypt = aead_encrypt, 2157 .decrypt = aead_decrypt, 2158 .ivsize = DES_BLOCK_SIZE, 2159 .maxauthsize = MD5_DIGEST_SIZE, 2160 }, 2161 .caam = { 2162 .class1_alg_type = OP_ALG_ALGSEL_DES | OP_ALG_AAI_CBC, 2163 .class2_alg_type = OP_ALG_ALGSEL_MD5 | 2164 OP_ALG_AAI_HMAC_PRECOMP, 2165 .geniv = true, 2166 } 2167 }, 2168 { 2169 .aead = { 2170 .base = { 2171 .cra_name = "authenc(hmac(sha1),cbc(des))", 2172 .cra_driver_name = "authenc-hmac-sha1-" 2173 "cbc-des-caam-qi", 2174 .cra_blocksize = DES_BLOCK_SIZE, 2175 }, 2176 .setkey = aead_setkey, 2177 .setauthsize = aead_setauthsize, 2178 .encrypt = aead_encrypt, 2179 .decrypt = aead_decrypt, 2180 .ivsize = DES_BLOCK_SIZE, 2181 .maxauthsize = SHA1_DIGEST_SIZE, 2182 }, 2183 .caam = { 2184 .class1_alg_type = OP_ALG_ALGSEL_DES | OP_ALG_AAI_CBC, 2185 .class2_alg_type = OP_ALG_ALGSEL_SHA1 | 2186 OP_ALG_AAI_HMAC_PRECOMP, 2187 }, 2188 }, 2189 { 2190 .aead = { 2191 .base = { 2192 .cra_name = "echainiv(authenc(hmac(sha1)," 2193 "cbc(des)))", 2194 .cra_driver_name = "echainiv-authenc-" 2195 "hmac-sha1-cbc-des-caam-qi", 2196 .cra_blocksize = DES_BLOCK_SIZE, 2197 }, 2198 .setkey = aead_setkey, 2199 .setauthsize = aead_setauthsize, 2200 .encrypt = aead_encrypt, 2201 .decrypt = aead_decrypt, 2202 .ivsize = DES_BLOCK_SIZE, 2203 .maxauthsize = SHA1_DIGEST_SIZE, 2204 }, 2205 .caam = { 2206 .class1_alg_type = OP_ALG_ALGSEL_DES | OP_ALG_AAI_CBC, 2207 .class2_alg_type = OP_ALG_ALGSEL_SHA1 | 2208 OP_ALG_AAI_HMAC_PRECOMP, 2209 .geniv = true, 2210 } 2211 }, 2212 { 2213 .aead = { 2214 .base = { 2215 .cra_name = "authenc(hmac(sha224),cbc(des))", 2216 .cra_driver_name = "authenc-hmac-sha224-" 2217 "cbc-des-caam-qi", 2218 .cra_blocksize = DES_BLOCK_SIZE, 2219 }, 2220 .setkey = aead_setkey, 2221 .setauthsize = aead_setauthsize, 2222 .encrypt = aead_encrypt, 2223 .decrypt = aead_decrypt, 2224 .ivsize = DES_BLOCK_SIZE, 2225 .maxauthsize = SHA224_DIGEST_SIZE, 2226 }, 2227 .caam = { 2228 .class1_alg_type = OP_ALG_ALGSEL_DES | OP_ALG_AAI_CBC, 2229 .class2_alg_type = OP_ALG_ALGSEL_SHA224 | 2230 OP_ALG_AAI_HMAC_PRECOMP, 2231 }, 2232 }, 2233 { 2234 .aead = { 2235 .base = { 2236 .cra_name = "echainiv(authenc(hmac(sha224)," 2237 "cbc(des)))", 2238 .cra_driver_name = "echainiv-authenc-" 2239 "hmac-sha224-cbc-des-" 2240 "caam-qi", 2241 .cra_blocksize = DES_BLOCK_SIZE, 2242 }, 2243 .setkey = aead_setkey, 2244 .setauthsize = aead_setauthsize, 2245 .encrypt = aead_encrypt, 2246 .decrypt = aead_decrypt, 2247 .ivsize = DES_BLOCK_SIZE, 2248 .maxauthsize = SHA224_DIGEST_SIZE, 2249 }, 2250 .caam = { 2251 .class1_alg_type = OP_ALG_ALGSEL_DES | OP_ALG_AAI_CBC, 2252 .class2_alg_type = OP_ALG_ALGSEL_SHA224 | 2253 OP_ALG_AAI_HMAC_PRECOMP, 2254 .geniv = true, 2255 } 2256 }, 2257 { 2258 .aead = { 2259 .base = { 2260 .cra_name = "authenc(hmac(sha256),cbc(des))", 2261 .cra_driver_name = "authenc-hmac-sha256-" 2262 "cbc-des-caam-qi", 2263 .cra_blocksize = DES_BLOCK_SIZE, 2264 }, 2265 .setkey = aead_setkey, 2266 .setauthsize = aead_setauthsize, 2267 .encrypt = aead_encrypt, 2268 .decrypt = aead_decrypt, 2269 .ivsize = DES_BLOCK_SIZE, 2270 .maxauthsize = SHA256_DIGEST_SIZE, 2271 }, 2272 .caam = { 2273 .class1_alg_type = OP_ALG_ALGSEL_DES | OP_ALG_AAI_CBC, 2274 .class2_alg_type = OP_ALG_ALGSEL_SHA256 | 2275 OP_ALG_AAI_HMAC_PRECOMP, 2276 }, 2277 }, 2278 { 2279 .aead = { 2280 .base = { 2281 .cra_name = "echainiv(authenc(hmac(sha256)," 2282 "cbc(des)))", 2283 .cra_driver_name = "echainiv-authenc-" 2284 "hmac-sha256-cbc-des-" 2285 "caam-qi", 2286 .cra_blocksize = DES_BLOCK_SIZE, 2287 }, 2288 .setkey = aead_setkey, 2289 .setauthsize = aead_setauthsize, 2290 .encrypt = aead_encrypt, 2291 .decrypt = aead_decrypt, 2292 .ivsize = DES_BLOCK_SIZE, 2293 .maxauthsize = SHA256_DIGEST_SIZE, 2294 }, 2295 .caam = { 2296 .class1_alg_type = OP_ALG_ALGSEL_DES | OP_ALG_AAI_CBC, 2297 .class2_alg_type = OP_ALG_ALGSEL_SHA256 | 2298 OP_ALG_AAI_HMAC_PRECOMP, 2299 .geniv = true, 2300 }, 2301 }, 2302 { 2303 .aead = { 2304 .base = { 2305 .cra_name = "authenc(hmac(sha384),cbc(des))", 2306 .cra_driver_name = "authenc-hmac-sha384-" 2307 "cbc-des-caam-qi", 2308 .cra_blocksize = DES_BLOCK_SIZE, 2309 }, 2310 .setkey = aead_setkey, 2311 .setauthsize = aead_setauthsize, 2312 .encrypt = aead_encrypt, 2313 .decrypt = aead_decrypt, 2314 .ivsize = DES_BLOCK_SIZE, 2315 .maxauthsize = SHA384_DIGEST_SIZE, 2316 }, 2317 .caam = { 2318 .class1_alg_type = OP_ALG_ALGSEL_DES | OP_ALG_AAI_CBC, 2319 .class2_alg_type = OP_ALG_ALGSEL_SHA384 | 2320 OP_ALG_AAI_HMAC_PRECOMP, 2321 }, 2322 }, 2323 { 2324 .aead = { 2325 .base = { 2326 .cra_name = "echainiv(authenc(hmac(sha384)," 2327 "cbc(des)))", 2328 .cra_driver_name = "echainiv-authenc-" 2329 "hmac-sha384-cbc-des-" 2330 "caam-qi", 2331 .cra_blocksize = DES_BLOCK_SIZE, 2332 }, 2333 .setkey = aead_setkey, 2334 .setauthsize = aead_setauthsize, 2335 .encrypt = aead_encrypt, 2336 .decrypt = aead_decrypt, 2337 .ivsize = DES_BLOCK_SIZE, 2338 .maxauthsize = SHA384_DIGEST_SIZE, 2339 }, 2340 .caam = { 2341 .class1_alg_type = OP_ALG_ALGSEL_DES | OP_ALG_AAI_CBC, 2342 .class2_alg_type = OP_ALG_ALGSEL_SHA384 | 2343 OP_ALG_AAI_HMAC_PRECOMP, 2344 .geniv = true, 2345 } 2346 }, 2347 { 2348 .aead = { 2349 .base = { 2350 .cra_name = "authenc(hmac(sha512),cbc(des))", 2351 .cra_driver_name = "authenc-hmac-sha512-" 2352 "cbc-des-caam-qi", 2353 .cra_blocksize = DES_BLOCK_SIZE, 2354 }, 2355 .setkey = aead_setkey, 2356 .setauthsize = aead_setauthsize, 2357 .encrypt = aead_encrypt, 2358 .decrypt = aead_decrypt, 2359 .ivsize = DES_BLOCK_SIZE, 2360 .maxauthsize = SHA512_DIGEST_SIZE, 2361 }, 2362 .caam = { 2363 .class1_alg_type = OP_ALG_ALGSEL_DES | OP_ALG_AAI_CBC, 2364 .class2_alg_type = OP_ALG_ALGSEL_SHA512 | 2365 OP_ALG_AAI_HMAC_PRECOMP, 2366 } 2367 }, 2368 { 2369 .aead = { 2370 .base = { 2371 .cra_name = "echainiv(authenc(hmac(sha512)," 2372 "cbc(des)))", 2373 .cra_driver_name = "echainiv-authenc-" 2374 "hmac-sha512-cbc-des-" 2375 "caam-qi", 2376 .cra_blocksize = DES_BLOCK_SIZE, 2377 }, 2378 .setkey = aead_setkey, 2379 .setauthsize = aead_setauthsize, 2380 .encrypt = aead_encrypt, 2381 .decrypt = aead_decrypt, 2382 .ivsize = DES_BLOCK_SIZE, 2383 .maxauthsize = SHA512_DIGEST_SIZE, 2384 }, 2385 .caam = { 2386 .class1_alg_type = OP_ALG_ALGSEL_DES | OP_ALG_AAI_CBC, 2387 .class2_alg_type = OP_ALG_ALGSEL_SHA512 | 2388 OP_ALG_AAI_HMAC_PRECOMP, 2389 .geniv = true, 2390 } 2391 }, 2392 }; 2393 2394 static int caam_init_common(struct caam_ctx *ctx, struct caam_alg_entry *caam, 2395 bool uses_dkp) 2396 { 2397 struct caam_drv_private *priv; 2398 struct device *dev; 2399 2400 /* 2401 * distribute tfms across job rings to ensure in-order 2402 * crypto request processing per tfm 2403 */ 2404 ctx->jrdev = caam_jr_alloc(); 2405 if (IS_ERR(ctx->jrdev)) { 2406 pr_err("Job Ring Device allocation for transform failed\n"); 2407 return PTR_ERR(ctx->jrdev); 2408 } 2409 2410 dev = ctx->jrdev->parent; 2411 priv = dev_get_drvdata(dev); 2412 if (priv->era >= 6 && uses_dkp) 2413 ctx->dir = DMA_BIDIRECTIONAL; 2414 else 2415 ctx->dir = DMA_TO_DEVICE; 2416 2417 ctx->key_dma = dma_map_single(dev, ctx->key, sizeof(ctx->key), 2418 ctx->dir); 2419 if (dma_mapping_error(dev, ctx->key_dma)) { 2420 dev_err(dev, "unable to map key\n"); 2421 caam_jr_free(ctx->jrdev); 2422 return -ENOMEM; 2423 } 2424 2425 /* copy descriptor header template value */ 2426 ctx->cdata.algtype = OP_TYPE_CLASS1_ALG | caam->class1_alg_type; 2427 ctx->adata.algtype = OP_TYPE_CLASS2_ALG | caam->class2_alg_type; 2428 2429 ctx->qidev = dev; 2430 2431 spin_lock_init(&ctx->lock); 2432 ctx->drv_ctx[ENCRYPT] = NULL; 2433 ctx->drv_ctx[DECRYPT] = NULL; 2434 2435 return 0; 2436 } 2437 2438 static int caam_cra_init(struct crypto_skcipher *tfm) 2439 { 2440 struct skcipher_alg *alg = crypto_skcipher_alg(tfm); 2441 struct caam_skcipher_alg *caam_alg = 2442 container_of(alg, typeof(*caam_alg), skcipher); 2443 2444 return caam_init_common(crypto_skcipher_ctx(tfm), &caam_alg->caam, 2445 false); 2446 } 2447 2448 static int caam_aead_init(struct crypto_aead *tfm) 2449 { 2450 struct aead_alg *alg = crypto_aead_alg(tfm); 2451 struct caam_aead_alg *caam_alg = container_of(alg, typeof(*caam_alg), 2452 aead); 2453 struct caam_ctx *ctx = crypto_aead_ctx(tfm); 2454 2455 return caam_init_common(ctx, &caam_alg->caam, !caam_alg->caam.nodkp); 2456 } 2457 2458 static void caam_exit_common(struct caam_ctx *ctx) 2459 { 2460 caam_drv_ctx_rel(ctx->drv_ctx[ENCRYPT]); 2461 caam_drv_ctx_rel(ctx->drv_ctx[DECRYPT]); 2462 2463 dma_unmap_single(ctx->jrdev->parent, ctx->key_dma, sizeof(ctx->key), 2464 ctx->dir); 2465 2466 caam_jr_free(ctx->jrdev); 2467 } 2468 2469 static void caam_cra_exit(struct crypto_skcipher *tfm) 2470 { 2471 caam_exit_common(crypto_skcipher_ctx(tfm)); 2472 } 2473 2474 static void caam_aead_exit(struct crypto_aead *tfm) 2475 { 2476 caam_exit_common(crypto_aead_ctx(tfm)); 2477 } 2478 2479 void caam_qi_algapi_exit(void) 2480 { 2481 int i; 2482 2483 for (i = 0; i < ARRAY_SIZE(driver_aeads); i++) { 2484 struct caam_aead_alg *t_alg = driver_aeads + i; 2485 2486 if (t_alg->registered) 2487 crypto_unregister_aead(&t_alg->aead); 2488 } 2489 2490 for (i = 0; i < ARRAY_SIZE(driver_algs); i++) { 2491 struct caam_skcipher_alg *t_alg = driver_algs + i; 2492 2493 if (t_alg->registered) 2494 crypto_unregister_skcipher(&t_alg->skcipher); 2495 } 2496 } 2497 2498 static void caam_skcipher_alg_init(struct caam_skcipher_alg *t_alg) 2499 { 2500 struct skcipher_alg *alg = &t_alg->skcipher; 2501 2502 alg->base.cra_module = THIS_MODULE; 2503 alg->base.cra_priority = CAAM_CRA_PRIORITY; 2504 alg->base.cra_ctxsize = sizeof(struct caam_ctx); 2505 alg->base.cra_flags = CRYPTO_ALG_ASYNC | CRYPTO_ALG_KERN_DRIVER_ONLY; 2506 2507 alg->init = caam_cra_init; 2508 alg->exit = caam_cra_exit; 2509 } 2510 2511 static void caam_aead_alg_init(struct caam_aead_alg *t_alg) 2512 { 2513 struct aead_alg *alg = &t_alg->aead; 2514 2515 alg->base.cra_module = THIS_MODULE; 2516 alg->base.cra_priority = CAAM_CRA_PRIORITY; 2517 alg->base.cra_ctxsize = sizeof(struct caam_ctx); 2518 alg->base.cra_flags = CRYPTO_ALG_ASYNC | CRYPTO_ALG_KERN_DRIVER_ONLY; 2519 2520 alg->init = caam_aead_init; 2521 alg->exit = caam_aead_exit; 2522 } 2523 2524 int caam_qi_algapi_init(struct device *ctrldev) 2525 { 2526 struct caam_drv_private *priv = dev_get_drvdata(ctrldev); 2527 int i = 0, err = 0; 2528 u32 aes_vid, aes_inst, des_inst, md_vid, md_inst; 2529 unsigned int md_limit = SHA512_DIGEST_SIZE; 2530 bool registered = false; 2531 2532 /* Make sure this runs only on (DPAA 1.x) QI */ 2533 if (!priv->qi_present || caam_dpaa2) 2534 return 0; 2535 2536 /* 2537 * Register crypto algorithms the device supports. 2538 * First, detect presence and attributes of DES, AES, and MD blocks. 2539 */ 2540 if (priv->era < 10) { 2541 u32 cha_vid, cha_inst; 2542 2543 cha_vid = rd_reg32(&priv->ctrl->perfmon.cha_id_ls); 2544 aes_vid = cha_vid & CHA_ID_LS_AES_MASK; 2545 md_vid = (cha_vid & CHA_ID_LS_MD_MASK) >> CHA_ID_LS_MD_SHIFT; 2546 2547 cha_inst = rd_reg32(&priv->ctrl->perfmon.cha_num_ls); 2548 des_inst = (cha_inst & CHA_ID_LS_DES_MASK) >> 2549 CHA_ID_LS_DES_SHIFT; 2550 aes_inst = cha_inst & CHA_ID_LS_AES_MASK; 2551 md_inst = (cha_inst & CHA_ID_LS_MD_MASK) >> CHA_ID_LS_MD_SHIFT; 2552 } else { 2553 u32 aesa, mdha; 2554 2555 aesa = rd_reg32(&priv->ctrl->vreg.aesa); 2556 mdha = rd_reg32(&priv->ctrl->vreg.mdha); 2557 2558 aes_vid = (aesa & CHA_VER_VID_MASK) >> CHA_VER_VID_SHIFT; 2559 md_vid = (mdha & CHA_VER_VID_MASK) >> CHA_VER_VID_SHIFT; 2560 2561 des_inst = rd_reg32(&priv->ctrl->vreg.desa) & CHA_VER_NUM_MASK; 2562 aes_inst = aesa & CHA_VER_NUM_MASK; 2563 md_inst = mdha & CHA_VER_NUM_MASK; 2564 } 2565 2566 /* If MD is present, limit digest size based on LP256 */ 2567 if (md_inst && md_vid == CHA_VER_VID_MD_LP256) 2568 md_limit = SHA256_DIGEST_SIZE; 2569 2570 for (i = 0; i < ARRAY_SIZE(driver_algs); i++) { 2571 struct caam_skcipher_alg *t_alg = driver_algs + i; 2572 u32 alg_sel = t_alg->caam.class1_alg_type & OP_ALG_ALGSEL_MASK; 2573 2574 /* Skip DES algorithms if not supported by device */ 2575 if (!des_inst && 2576 ((alg_sel == OP_ALG_ALGSEL_3DES) || 2577 (alg_sel == OP_ALG_ALGSEL_DES))) 2578 continue; 2579 2580 /* Skip AES algorithms if not supported by device */ 2581 if (!aes_inst && (alg_sel == OP_ALG_ALGSEL_AES)) 2582 continue; 2583 2584 caam_skcipher_alg_init(t_alg); 2585 2586 err = crypto_register_skcipher(&t_alg->skcipher); 2587 if (err) { 2588 dev_warn(ctrldev, "%s alg registration failed\n", 2589 t_alg->skcipher.base.cra_driver_name); 2590 continue; 2591 } 2592 2593 t_alg->registered = true; 2594 registered = true; 2595 } 2596 2597 for (i = 0; i < ARRAY_SIZE(driver_aeads); i++) { 2598 struct caam_aead_alg *t_alg = driver_aeads + i; 2599 u32 c1_alg_sel = t_alg->caam.class1_alg_type & 2600 OP_ALG_ALGSEL_MASK; 2601 u32 c2_alg_sel = t_alg->caam.class2_alg_type & 2602 OP_ALG_ALGSEL_MASK; 2603 u32 alg_aai = t_alg->caam.class1_alg_type & OP_ALG_AAI_MASK; 2604 2605 /* Skip DES algorithms if not supported by device */ 2606 if (!des_inst && 2607 ((c1_alg_sel == OP_ALG_ALGSEL_3DES) || 2608 (c1_alg_sel == OP_ALG_ALGSEL_DES))) 2609 continue; 2610 2611 /* Skip AES algorithms if not supported by device */ 2612 if (!aes_inst && (c1_alg_sel == OP_ALG_ALGSEL_AES)) 2613 continue; 2614 2615 /* 2616 * Check support for AES algorithms not available 2617 * on LP devices. 2618 */ 2619 if (aes_vid == CHA_VER_VID_AES_LP && alg_aai == OP_ALG_AAI_GCM) 2620 continue; 2621 2622 /* 2623 * Skip algorithms requiring message digests 2624 * if MD or MD size is not supported by device. 2625 */ 2626 if (c2_alg_sel && 2627 (!md_inst || (t_alg->aead.maxauthsize > md_limit))) 2628 continue; 2629 2630 caam_aead_alg_init(t_alg); 2631 2632 err = crypto_register_aead(&t_alg->aead); 2633 if (err) { 2634 pr_warn("%s alg registration failed\n", 2635 t_alg->aead.base.cra_driver_name); 2636 continue; 2637 } 2638 2639 t_alg->registered = true; 2640 registered = true; 2641 } 2642 2643 if (registered) 2644 dev_info(ctrldev, "algorithms registered in /proc/crypto\n"); 2645 2646 return err; 2647 } 2648