1 // SPDX-License-Identifier: GPL-2.0+ 2 /* 3 * caam - Freescale FSL CAAM support for crypto API 4 * 5 * Copyright 2008-2011 Freescale Semiconductor, Inc. 6 * Copyright 2016-2019 NXP 7 * 8 * Based on talitos crypto API driver. 9 * 10 * relationship of job descriptors to shared descriptors (SteveC Dec 10 2008): 11 * 12 * --------------- --------------- 13 * | JobDesc #1 |-------------------->| ShareDesc | 14 * | *(packet 1) | | (PDB) | 15 * --------------- |------------->| (hashKey) | 16 * . | | (cipherKey) | 17 * . | |-------->| (operation) | 18 * --------------- | | --------------- 19 * | JobDesc #2 |------| | 20 * | *(packet 2) | | 21 * --------------- | 22 * . | 23 * . | 24 * --------------- | 25 * | JobDesc #3 |------------ 26 * | *(packet 3) | 27 * --------------- 28 * 29 * The SharedDesc never changes for a connection unless rekeyed, but 30 * each packet will likely be in a different place. So all we need 31 * to know to process the packet is where the input is, where the 32 * output goes, and what context we want to process with. Context is 33 * in the SharedDesc, packet references in the JobDesc. 34 * 35 * So, a job desc looks like: 36 * 37 * --------------------- 38 * | Header | 39 * | ShareDesc Pointer | 40 * | SEQ_OUT_PTR | 41 * | (output buffer) | 42 * | (output length) | 43 * | SEQ_IN_PTR | 44 * | (input buffer) | 45 * | (input length) | 46 * --------------------- 47 */ 48 49 #include "compat.h" 50 51 #include "regs.h" 52 #include "intern.h" 53 #include "desc_constr.h" 54 #include "jr.h" 55 #include "error.h" 56 #include "sg_sw_sec4.h" 57 #include "key_gen.h" 58 #include "caamalg_desc.h" 59 #include <crypto/engine.h> 60 61 /* 62 * crypto alg 63 */ 64 #define CAAM_CRA_PRIORITY 3000 65 /* max key is sum of AES_MAX_KEY_SIZE, max split key size */ 66 #define CAAM_MAX_KEY_SIZE (AES_MAX_KEY_SIZE + \ 67 CTR_RFC3686_NONCE_SIZE + \ 68 SHA512_DIGEST_SIZE * 2) 69 70 #define AEAD_DESC_JOB_IO_LEN (DESC_JOB_IO_LEN + CAAM_CMD_SZ * 2) 71 #define GCM_DESC_JOB_IO_LEN (AEAD_DESC_JOB_IO_LEN + \ 72 CAAM_CMD_SZ * 4) 73 #define AUTHENC_DESC_JOB_IO_LEN (AEAD_DESC_JOB_IO_LEN + \ 74 CAAM_CMD_SZ * 5) 75 76 #define CHACHAPOLY_DESC_JOB_IO_LEN (AEAD_DESC_JOB_IO_LEN + CAAM_CMD_SZ * 6) 77 78 #define DESC_MAX_USED_BYTES (CAAM_DESC_BYTES_MAX - DESC_JOB_IO_LEN_MIN) 79 #define DESC_MAX_USED_LEN (DESC_MAX_USED_BYTES / CAAM_CMD_SZ) 80 81 struct caam_alg_entry { 82 int class1_alg_type; 83 int class2_alg_type; 84 bool rfc3686; 85 bool geniv; 86 bool nodkp; 87 }; 88 89 struct caam_aead_alg { 90 struct aead_alg aead; 91 struct caam_alg_entry caam; 92 bool registered; 93 }; 94 95 struct caam_skcipher_alg { 96 struct skcipher_alg skcipher; 97 struct caam_alg_entry caam; 98 bool registered; 99 }; 100 101 /* 102 * per-session context 103 */ 104 struct caam_ctx { 105 struct crypto_engine_ctx enginectx; 106 u32 sh_desc_enc[DESC_MAX_USED_LEN]; 107 u32 sh_desc_dec[DESC_MAX_USED_LEN]; 108 u8 key[CAAM_MAX_KEY_SIZE]; 109 dma_addr_t sh_desc_enc_dma; 110 dma_addr_t sh_desc_dec_dma; 111 dma_addr_t key_dma; 112 enum dma_data_direction dir; 113 struct device *jrdev; 114 struct alginfo adata; 115 struct alginfo cdata; 116 unsigned int authsize; 117 }; 118 119 struct caam_skcipher_req_ctx { 120 struct skcipher_edesc *edesc; 121 }; 122 123 struct caam_aead_req_ctx { 124 struct aead_edesc *edesc; 125 }; 126 127 static int aead_null_set_sh_desc(struct crypto_aead *aead) 128 { 129 struct caam_ctx *ctx = crypto_aead_ctx(aead); 130 struct device *jrdev = ctx->jrdev; 131 struct caam_drv_private *ctrlpriv = dev_get_drvdata(jrdev->parent); 132 u32 *desc; 133 int rem_bytes = CAAM_DESC_BYTES_MAX - AEAD_DESC_JOB_IO_LEN - 134 ctx->adata.keylen_pad; 135 136 /* 137 * Job Descriptor and Shared Descriptors 138 * must all fit into the 64-word Descriptor h/w Buffer 139 */ 140 if (rem_bytes >= DESC_AEAD_NULL_ENC_LEN) { 141 ctx->adata.key_inline = true; 142 ctx->adata.key_virt = ctx->key; 143 } else { 144 ctx->adata.key_inline = false; 145 ctx->adata.key_dma = ctx->key_dma; 146 } 147 148 /* aead_encrypt shared descriptor */ 149 desc = ctx->sh_desc_enc; 150 cnstr_shdsc_aead_null_encap(desc, &ctx->adata, ctx->authsize, 151 ctrlpriv->era); 152 dma_sync_single_for_device(jrdev, ctx->sh_desc_enc_dma, 153 desc_bytes(desc), ctx->dir); 154 155 /* 156 * Job Descriptor and Shared Descriptors 157 * must all fit into the 64-word Descriptor h/w Buffer 158 */ 159 if (rem_bytes >= DESC_AEAD_NULL_DEC_LEN) { 160 ctx->adata.key_inline = true; 161 ctx->adata.key_virt = ctx->key; 162 } else { 163 ctx->adata.key_inline = false; 164 ctx->adata.key_dma = ctx->key_dma; 165 } 166 167 /* aead_decrypt shared descriptor */ 168 desc = ctx->sh_desc_dec; 169 cnstr_shdsc_aead_null_decap(desc, &ctx->adata, ctx->authsize, 170 ctrlpriv->era); 171 dma_sync_single_for_device(jrdev, ctx->sh_desc_dec_dma, 172 desc_bytes(desc), ctx->dir); 173 174 return 0; 175 } 176 177 static int aead_set_sh_desc(struct crypto_aead *aead) 178 { 179 struct caam_aead_alg *alg = container_of(crypto_aead_alg(aead), 180 struct caam_aead_alg, aead); 181 unsigned int ivsize = crypto_aead_ivsize(aead); 182 struct caam_ctx *ctx = crypto_aead_ctx(aead); 183 struct device *jrdev = ctx->jrdev; 184 struct caam_drv_private *ctrlpriv = dev_get_drvdata(jrdev->parent); 185 u32 ctx1_iv_off = 0; 186 u32 *desc, *nonce = NULL; 187 u32 inl_mask; 188 unsigned int data_len[2]; 189 const bool ctr_mode = ((ctx->cdata.algtype & OP_ALG_AAI_MASK) == 190 OP_ALG_AAI_CTR_MOD128); 191 const bool is_rfc3686 = alg->caam.rfc3686; 192 193 if (!ctx->authsize) 194 return 0; 195 196 /* NULL encryption / decryption */ 197 if (!ctx->cdata.keylen) 198 return aead_null_set_sh_desc(aead); 199 200 /* 201 * AES-CTR needs to load IV in CONTEXT1 reg 202 * at an offset of 128bits (16bytes) 203 * CONTEXT1[255:128] = IV 204 */ 205 if (ctr_mode) 206 ctx1_iv_off = 16; 207 208 /* 209 * RFC3686 specific: 210 * CONTEXT1[255:128] = {NONCE, IV, COUNTER} 211 */ 212 if (is_rfc3686) { 213 ctx1_iv_off = 16 + CTR_RFC3686_NONCE_SIZE; 214 nonce = (u32 *)((void *)ctx->key + ctx->adata.keylen_pad + 215 ctx->cdata.keylen - CTR_RFC3686_NONCE_SIZE); 216 } 217 218 /* 219 * In case |user key| > |derived key|, using DKP<imm,imm> 220 * would result in invalid opcodes (last bytes of user key) in 221 * the resulting descriptor. Use DKP<ptr,imm> instead => both 222 * virtual and dma key addresses are needed. 223 */ 224 ctx->adata.key_virt = ctx->key; 225 ctx->adata.key_dma = ctx->key_dma; 226 227 ctx->cdata.key_virt = ctx->key + ctx->adata.keylen_pad; 228 ctx->cdata.key_dma = ctx->key_dma + ctx->adata.keylen_pad; 229 230 data_len[0] = ctx->adata.keylen_pad; 231 data_len[1] = ctx->cdata.keylen; 232 233 if (alg->caam.geniv) 234 goto skip_enc; 235 236 /* 237 * Job Descriptor and Shared Descriptors 238 * must all fit into the 64-word Descriptor h/w Buffer 239 */ 240 if (desc_inline_query(DESC_AEAD_ENC_LEN + 241 (is_rfc3686 ? DESC_AEAD_CTR_RFC3686_LEN : 0), 242 AUTHENC_DESC_JOB_IO_LEN, data_len, &inl_mask, 243 ARRAY_SIZE(data_len)) < 0) 244 return -EINVAL; 245 246 ctx->adata.key_inline = !!(inl_mask & 1); 247 ctx->cdata.key_inline = !!(inl_mask & 2); 248 249 /* aead_encrypt shared descriptor */ 250 desc = ctx->sh_desc_enc; 251 cnstr_shdsc_aead_encap(desc, &ctx->cdata, &ctx->adata, ivsize, 252 ctx->authsize, is_rfc3686, nonce, ctx1_iv_off, 253 false, ctrlpriv->era); 254 dma_sync_single_for_device(jrdev, ctx->sh_desc_enc_dma, 255 desc_bytes(desc), ctx->dir); 256 257 skip_enc: 258 /* 259 * Job Descriptor and Shared Descriptors 260 * must all fit into the 64-word Descriptor h/w Buffer 261 */ 262 if (desc_inline_query(DESC_AEAD_DEC_LEN + 263 (is_rfc3686 ? DESC_AEAD_CTR_RFC3686_LEN : 0), 264 AUTHENC_DESC_JOB_IO_LEN, data_len, &inl_mask, 265 ARRAY_SIZE(data_len)) < 0) 266 return -EINVAL; 267 268 ctx->adata.key_inline = !!(inl_mask & 1); 269 ctx->cdata.key_inline = !!(inl_mask & 2); 270 271 /* aead_decrypt shared descriptor */ 272 desc = ctx->sh_desc_dec; 273 cnstr_shdsc_aead_decap(desc, &ctx->cdata, &ctx->adata, ivsize, 274 ctx->authsize, alg->caam.geniv, is_rfc3686, 275 nonce, ctx1_iv_off, false, ctrlpriv->era); 276 dma_sync_single_for_device(jrdev, ctx->sh_desc_dec_dma, 277 desc_bytes(desc), ctx->dir); 278 279 if (!alg->caam.geniv) 280 goto skip_givenc; 281 282 /* 283 * Job Descriptor and Shared Descriptors 284 * must all fit into the 64-word Descriptor h/w Buffer 285 */ 286 if (desc_inline_query(DESC_AEAD_GIVENC_LEN + 287 (is_rfc3686 ? DESC_AEAD_CTR_RFC3686_LEN : 0), 288 AUTHENC_DESC_JOB_IO_LEN, data_len, &inl_mask, 289 ARRAY_SIZE(data_len)) < 0) 290 return -EINVAL; 291 292 ctx->adata.key_inline = !!(inl_mask & 1); 293 ctx->cdata.key_inline = !!(inl_mask & 2); 294 295 /* aead_givencrypt shared descriptor */ 296 desc = ctx->sh_desc_enc; 297 cnstr_shdsc_aead_givencap(desc, &ctx->cdata, &ctx->adata, ivsize, 298 ctx->authsize, is_rfc3686, nonce, 299 ctx1_iv_off, false, ctrlpriv->era); 300 dma_sync_single_for_device(jrdev, ctx->sh_desc_enc_dma, 301 desc_bytes(desc), ctx->dir); 302 303 skip_givenc: 304 return 0; 305 } 306 307 static int aead_setauthsize(struct crypto_aead *authenc, 308 unsigned int authsize) 309 { 310 struct caam_ctx *ctx = crypto_aead_ctx(authenc); 311 312 ctx->authsize = authsize; 313 aead_set_sh_desc(authenc); 314 315 return 0; 316 } 317 318 static int gcm_set_sh_desc(struct crypto_aead *aead) 319 { 320 struct caam_ctx *ctx = crypto_aead_ctx(aead); 321 struct device *jrdev = ctx->jrdev; 322 unsigned int ivsize = crypto_aead_ivsize(aead); 323 u32 *desc; 324 int rem_bytes = CAAM_DESC_BYTES_MAX - GCM_DESC_JOB_IO_LEN - 325 ctx->cdata.keylen; 326 327 if (!ctx->cdata.keylen || !ctx->authsize) 328 return 0; 329 330 /* 331 * AES GCM encrypt shared descriptor 332 * Job Descriptor and Shared Descriptor 333 * must fit into the 64-word Descriptor h/w Buffer 334 */ 335 if (rem_bytes >= DESC_GCM_ENC_LEN) { 336 ctx->cdata.key_inline = true; 337 ctx->cdata.key_virt = ctx->key; 338 } else { 339 ctx->cdata.key_inline = false; 340 ctx->cdata.key_dma = ctx->key_dma; 341 } 342 343 desc = ctx->sh_desc_enc; 344 cnstr_shdsc_gcm_encap(desc, &ctx->cdata, ivsize, ctx->authsize, false); 345 dma_sync_single_for_device(jrdev, ctx->sh_desc_enc_dma, 346 desc_bytes(desc), ctx->dir); 347 348 /* 349 * Job Descriptor and Shared Descriptors 350 * must all fit into the 64-word Descriptor h/w Buffer 351 */ 352 if (rem_bytes >= DESC_GCM_DEC_LEN) { 353 ctx->cdata.key_inline = true; 354 ctx->cdata.key_virt = ctx->key; 355 } else { 356 ctx->cdata.key_inline = false; 357 ctx->cdata.key_dma = ctx->key_dma; 358 } 359 360 desc = ctx->sh_desc_dec; 361 cnstr_shdsc_gcm_decap(desc, &ctx->cdata, ivsize, ctx->authsize, false); 362 dma_sync_single_for_device(jrdev, ctx->sh_desc_dec_dma, 363 desc_bytes(desc), ctx->dir); 364 365 return 0; 366 } 367 368 static int gcm_setauthsize(struct crypto_aead *authenc, unsigned int authsize) 369 { 370 struct caam_ctx *ctx = crypto_aead_ctx(authenc); 371 int err; 372 373 err = crypto_gcm_check_authsize(authsize); 374 if (err) 375 return err; 376 377 ctx->authsize = authsize; 378 gcm_set_sh_desc(authenc); 379 380 return 0; 381 } 382 383 static int rfc4106_set_sh_desc(struct crypto_aead *aead) 384 { 385 struct caam_ctx *ctx = crypto_aead_ctx(aead); 386 struct device *jrdev = ctx->jrdev; 387 unsigned int ivsize = crypto_aead_ivsize(aead); 388 u32 *desc; 389 int rem_bytes = CAAM_DESC_BYTES_MAX - GCM_DESC_JOB_IO_LEN - 390 ctx->cdata.keylen; 391 392 if (!ctx->cdata.keylen || !ctx->authsize) 393 return 0; 394 395 /* 396 * RFC4106 encrypt shared descriptor 397 * Job Descriptor and Shared Descriptor 398 * must fit into the 64-word Descriptor h/w Buffer 399 */ 400 if (rem_bytes >= DESC_RFC4106_ENC_LEN) { 401 ctx->cdata.key_inline = true; 402 ctx->cdata.key_virt = ctx->key; 403 } else { 404 ctx->cdata.key_inline = false; 405 ctx->cdata.key_dma = ctx->key_dma; 406 } 407 408 desc = ctx->sh_desc_enc; 409 cnstr_shdsc_rfc4106_encap(desc, &ctx->cdata, ivsize, ctx->authsize, 410 false); 411 dma_sync_single_for_device(jrdev, ctx->sh_desc_enc_dma, 412 desc_bytes(desc), ctx->dir); 413 414 /* 415 * Job Descriptor and Shared Descriptors 416 * must all fit into the 64-word Descriptor h/w Buffer 417 */ 418 if (rem_bytes >= DESC_RFC4106_DEC_LEN) { 419 ctx->cdata.key_inline = true; 420 ctx->cdata.key_virt = ctx->key; 421 } else { 422 ctx->cdata.key_inline = false; 423 ctx->cdata.key_dma = ctx->key_dma; 424 } 425 426 desc = ctx->sh_desc_dec; 427 cnstr_shdsc_rfc4106_decap(desc, &ctx->cdata, ivsize, ctx->authsize, 428 false); 429 dma_sync_single_for_device(jrdev, ctx->sh_desc_dec_dma, 430 desc_bytes(desc), ctx->dir); 431 432 return 0; 433 } 434 435 static int rfc4106_setauthsize(struct crypto_aead *authenc, 436 unsigned int authsize) 437 { 438 struct caam_ctx *ctx = crypto_aead_ctx(authenc); 439 int err; 440 441 err = crypto_rfc4106_check_authsize(authsize); 442 if (err) 443 return err; 444 445 ctx->authsize = authsize; 446 rfc4106_set_sh_desc(authenc); 447 448 return 0; 449 } 450 451 static int rfc4543_set_sh_desc(struct crypto_aead *aead) 452 { 453 struct caam_ctx *ctx = crypto_aead_ctx(aead); 454 struct device *jrdev = ctx->jrdev; 455 unsigned int ivsize = crypto_aead_ivsize(aead); 456 u32 *desc; 457 int rem_bytes = CAAM_DESC_BYTES_MAX - GCM_DESC_JOB_IO_LEN - 458 ctx->cdata.keylen; 459 460 if (!ctx->cdata.keylen || !ctx->authsize) 461 return 0; 462 463 /* 464 * RFC4543 encrypt shared descriptor 465 * Job Descriptor and Shared Descriptor 466 * must fit into the 64-word Descriptor h/w Buffer 467 */ 468 if (rem_bytes >= DESC_RFC4543_ENC_LEN) { 469 ctx->cdata.key_inline = true; 470 ctx->cdata.key_virt = ctx->key; 471 } else { 472 ctx->cdata.key_inline = false; 473 ctx->cdata.key_dma = ctx->key_dma; 474 } 475 476 desc = ctx->sh_desc_enc; 477 cnstr_shdsc_rfc4543_encap(desc, &ctx->cdata, ivsize, ctx->authsize, 478 false); 479 dma_sync_single_for_device(jrdev, ctx->sh_desc_enc_dma, 480 desc_bytes(desc), ctx->dir); 481 482 /* 483 * Job Descriptor and Shared Descriptors 484 * must all fit into the 64-word Descriptor h/w Buffer 485 */ 486 if (rem_bytes >= DESC_RFC4543_DEC_LEN) { 487 ctx->cdata.key_inline = true; 488 ctx->cdata.key_virt = ctx->key; 489 } else { 490 ctx->cdata.key_inline = false; 491 ctx->cdata.key_dma = ctx->key_dma; 492 } 493 494 desc = ctx->sh_desc_dec; 495 cnstr_shdsc_rfc4543_decap(desc, &ctx->cdata, ivsize, ctx->authsize, 496 false); 497 dma_sync_single_for_device(jrdev, ctx->sh_desc_dec_dma, 498 desc_bytes(desc), ctx->dir); 499 500 return 0; 501 } 502 503 static int rfc4543_setauthsize(struct crypto_aead *authenc, 504 unsigned int authsize) 505 { 506 struct caam_ctx *ctx = crypto_aead_ctx(authenc); 507 508 if (authsize != 16) 509 return -EINVAL; 510 511 ctx->authsize = authsize; 512 rfc4543_set_sh_desc(authenc); 513 514 return 0; 515 } 516 517 static int chachapoly_set_sh_desc(struct crypto_aead *aead) 518 { 519 struct caam_ctx *ctx = crypto_aead_ctx(aead); 520 struct device *jrdev = ctx->jrdev; 521 unsigned int ivsize = crypto_aead_ivsize(aead); 522 u32 *desc; 523 524 if (!ctx->cdata.keylen || !ctx->authsize) 525 return 0; 526 527 desc = ctx->sh_desc_enc; 528 cnstr_shdsc_chachapoly(desc, &ctx->cdata, &ctx->adata, ivsize, 529 ctx->authsize, true, false); 530 dma_sync_single_for_device(jrdev, ctx->sh_desc_enc_dma, 531 desc_bytes(desc), ctx->dir); 532 533 desc = ctx->sh_desc_dec; 534 cnstr_shdsc_chachapoly(desc, &ctx->cdata, &ctx->adata, ivsize, 535 ctx->authsize, false, false); 536 dma_sync_single_for_device(jrdev, ctx->sh_desc_dec_dma, 537 desc_bytes(desc), ctx->dir); 538 539 return 0; 540 } 541 542 static int chachapoly_setauthsize(struct crypto_aead *aead, 543 unsigned int authsize) 544 { 545 struct caam_ctx *ctx = crypto_aead_ctx(aead); 546 547 if (authsize != POLY1305_DIGEST_SIZE) 548 return -EINVAL; 549 550 ctx->authsize = authsize; 551 return chachapoly_set_sh_desc(aead); 552 } 553 554 static int chachapoly_setkey(struct crypto_aead *aead, const u8 *key, 555 unsigned int keylen) 556 { 557 struct caam_ctx *ctx = crypto_aead_ctx(aead); 558 unsigned int ivsize = crypto_aead_ivsize(aead); 559 unsigned int saltlen = CHACHAPOLY_IV_SIZE - ivsize; 560 561 if (keylen != CHACHA_KEY_SIZE + saltlen) 562 return -EINVAL; 563 564 ctx->cdata.key_virt = key; 565 ctx->cdata.keylen = keylen - saltlen; 566 567 return chachapoly_set_sh_desc(aead); 568 } 569 570 static int aead_setkey(struct crypto_aead *aead, 571 const u8 *key, unsigned int keylen) 572 { 573 struct caam_ctx *ctx = crypto_aead_ctx(aead); 574 struct device *jrdev = ctx->jrdev; 575 struct caam_drv_private *ctrlpriv = dev_get_drvdata(jrdev->parent); 576 struct crypto_authenc_keys keys; 577 int ret = 0; 578 579 if (crypto_authenc_extractkeys(&keys, key, keylen) != 0) 580 goto badkey; 581 582 dev_dbg(jrdev, "keylen %d enckeylen %d authkeylen %d\n", 583 keys.authkeylen + keys.enckeylen, keys.enckeylen, 584 keys.authkeylen); 585 print_hex_dump_debug("key in @"__stringify(__LINE__)": ", 586 DUMP_PREFIX_ADDRESS, 16, 4, key, keylen, 1); 587 588 /* 589 * If DKP is supported, use it in the shared descriptor to generate 590 * the split key. 591 */ 592 if (ctrlpriv->era >= 6) { 593 ctx->adata.keylen = keys.authkeylen; 594 ctx->adata.keylen_pad = split_key_len(ctx->adata.algtype & 595 OP_ALG_ALGSEL_MASK); 596 597 if (ctx->adata.keylen_pad + keys.enckeylen > CAAM_MAX_KEY_SIZE) 598 goto badkey; 599 600 memcpy(ctx->key, keys.authkey, keys.authkeylen); 601 memcpy(ctx->key + ctx->adata.keylen_pad, keys.enckey, 602 keys.enckeylen); 603 dma_sync_single_for_device(jrdev, ctx->key_dma, 604 ctx->adata.keylen_pad + 605 keys.enckeylen, ctx->dir); 606 goto skip_split_key; 607 } 608 609 ret = gen_split_key(ctx->jrdev, ctx->key, &ctx->adata, keys.authkey, 610 keys.authkeylen, CAAM_MAX_KEY_SIZE - 611 keys.enckeylen); 612 if (ret) { 613 goto badkey; 614 } 615 616 /* postpend encryption key to auth split key */ 617 memcpy(ctx->key + ctx->adata.keylen_pad, keys.enckey, keys.enckeylen); 618 dma_sync_single_for_device(jrdev, ctx->key_dma, ctx->adata.keylen_pad + 619 keys.enckeylen, ctx->dir); 620 621 print_hex_dump_debug("ctx.key@"__stringify(__LINE__)": ", 622 DUMP_PREFIX_ADDRESS, 16, 4, ctx->key, 623 ctx->adata.keylen_pad + keys.enckeylen, 1); 624 625 skip_split_key: 626 ctx->cdata.keylen = keys.enckeylen; 627 memzero_explicit(&keys, sizeof(keys)); 628 return aead_set_sh_desc(aead); 629 badkey: 630 memzero_explicit(&keys, sizeof(keys)); 631 return -EINVAL; 632 } 633 634 static int des3_aead_setkey(struct crypto_aead *aead, const u8 *key, 635 unsigned int keylen) 636 { 637 struct crypto_authenc_keys keys; 638 int err; 639 640 err = crypto_authenc_extractkeys(&keys, key, keylen); 641 if (unlikely(err)) 642 return err; 643 644 err = verify_aead_des3_key(aead, keys.enckey, keys.enckeylen) ?: 645 aead_setkey(aead, key, keylen); 646 647 memzero_explicit(&keys, sizeof(keys)); 648 return err; 649 } 650 651 static int gcm_setkey(struct crypto_aead *aead, 652 const u8 *key, unsigned int keylen) 653 { 654 struct caam_ctx *ctx = crypto_aead_ctx(aead); 655 struct device *jrdev = ctx->jrdev; 656 int err; 657 658 err = aes_check_keylen(keylen); 659 if (err) 660 return err; 661 662 print_hex_dump_debug("key in @"__stringify(__LINE__)": ", 663 DUMP_PREFIX_ADDRESS, 16, 4, key, keylen, 1); 664 665 memcpy(ctx->key, key, keylen); 666 dma_sync_single_for_device(jrdev, ctx->key_dma, keylen, ctx->dir); 667 ctx->cdata.keylen = keylen; 668 669 return gcm_set_sh_desc(aead); 670 } 671 672 static int rfc4106_setkey(struct crypto_aead *aead, 673 const u8 *key, unsigned int keylen) 674 { 675 struct caam_ctx *ctx = crypto_aead_ctx(aead); 676 struct device *jrdev = ctx->jrdev; 677 int err; 678 679 err = aes_check_keylen(keylen - 4); 680 if (err) 681 return err; 682 683 print_hex_dump_debug("key in @"__stringify(__LINE__)": ", 684 DUMP_PREFIX_ADDRESS, 16, 4, key, keylen, 1); 685 686 memcpy(ctx->key, key, keylen); 687 688 /* 689 * The last four bytes of the key material are used as the salt value 690 * in the nonce. Update the AES key length. 691 */ 692 ctx->cdata.keylen = keylen - 4; 693 dma_sync_single_for_device(jrdev, ctx->key_dma, ctx->cdata.keylen, 694 ctx->dir); 695 return rfc4106_set_sh_desc(aead); 696 } 697 698 static int rfc4543_setkey(struct crypto_aead *aead, 699 const u8 *key, unsigned int keylen) 700 { 701 struct caam_ctx *ctx = crypto_aead_ctx(aead); 702 struct device *jrdev = ctx->jrdev; 703 int err; 704 705 err = aes_check_keylen(keylen - 4); 706 if (err) 707 return err; 708 709 print_hex_dump_debug("key in @"__stringify(__LINE__)": ", 710 DUMP_PREFIX_ADDRESS, 16, 4, key, keylen, 1); 711 712 memcpy(ctx->key, key, keylen); 713 714 /* 715 * The last four bytes of the key material are used as the salt value 716 * in the nonce. Update the AES key length. 717 */ 718 ctx->cdata.keylen = keylen - 4; 719 dma_sync_single_for_device(jrdev, ctx->key_dma, ctx->cdata.keylen, 720 ctx->dir); 721 return rfc4543_set_sh_desc(aead); 722 } 723 724 static int skcipher_setkey(struct crypto_skcipher *skcipher, const u8 *key, 725 unsigned int keylen, const u32 ctx1_iv_off) 726 { 727 struct caam_ctx *ctx = crypto_skcipher_ctx(skcipher); 728 struct caam_skcipher_alg *alg = 729 container_of(crypto_skcipher_alg(skcipher), typeof(*alg), 730 skcipher); 731 struct device *jrdev = ctx->jrdev; 732 unsigned int ivsize = crypto_skcipher_ivsize(skcipher); 733 u32 *desc; 734 const bool is_rfc3686 = alg->caam.rfc3686; 735 736 print_hex_dump_debug("key in @"__stringify(__LINE__)": ", 737 DUMP_PREFIX_ADDRESS, 16, 4, key, keylen, 1); 738 739 ctx->cdata.keylen = keylen; 740 ctx->cdata.key_virt = key; 741 ctx->cdata.key_inline = true; 742 743 /* skcipher_encrypt shared descriptor */ 744 desc = ctx->sh_desc_enc; 745 cnstr_shdsc_skcipher_encap(desc, &ctx->cdata, ivsize, is_rfc3686, 746 ctx1_iv_off); 747 dma_sync_single_for_device(jrdev, ctx->sh_desc_enc_dma, 748 desc_bytes(desc), ctx->dir); 749 750 /* skcipher_decrypt shared descriptor */ 751 desc = ctx->sh_desc_dec; 752 cnstr_shdsc_skcipher_decap(desc, &ctx->cdata, ivsize, is_rfc3686, 753 ctx1_iv_off); 754 dma_sync_single_for_device(jrdev, ctx->sh_desc_dec_dma, 755 desc_bytes(desc), ctx->dir); 756 757 return 0; 758 } 759 760 static int aes_skcipher_setkey(struct crypto_skcipher *skcipher, 761 const u8 *key, unsigned int keylen) 762 { 763 int err; 764 765 err = aes_check_keylen(keylen); 766 if (err) 767 return err; 768 769 return skcipher_setkey(skcipher, key, keylen, 0); 770 } 771 772 static int rfc3686_skcipher_setkey(struct crypto_skcipher *skcipher, 773 const u8 *key, unsigned int keylen) 774 { 775 u32 ctx1_iv_off; 776 int err; 777 778 /* 779 * RFC3686 specific: 780 * | CONTEXT1[255:128] = {NONCE, IV, COUNTER} 781 * | *key = {KEY, NONCE} 782 */ 783 ctx1_iv_off = 16 + CTR_RFC3686_NONCE_SIZE; 784 keylen -= CTR_RFC3686_NONCE_SIZE; 785 786 err = aes_check_keylen(keylen); 787 if (err) 788 return err; 789 790 return skcipher_setkey(skcipher, key, keylen, ctx1_iv_off); 791 } 792 793 static int ctr_skcipher_setkey(struct crypto_skcipher *skcipher, 794 const u8 *key, unsigned int keylen) 795 { 796 u32 ctx1_iv_off; 797 int err; 798 799 /* 800 * AES-CTR needs to load IV in CONTEXT1 reg 801 * at an offset of 128bits (16bytes) 802 * CONTEXT1[255:128] = IV 803 */ 804 ctx1_iv_off = 16; 805 806 err = aes_check_keylen(keylen); 807 if (err) 808 return err; 809 810 return skcipher_setkey(skcipher, key, keylen, ctx1_iv_off); 811 } 812 813 static int arc4_skcipher_setkey(struct crypto_skcipher *skcipher, 814 const u8 *key, unsigned int keylen) 815 { 816 return skcipher_setkey(skcipher, key, keylen, 0); 817 } 818 819 static int des_skcipher_setkey(struct crypto_skcipher *skcipher, 820 const u8 *key, unsigned int keylen) 821 { 822 return verify_skcipher_des_key(skcipher, key) ?: 823 skcipher_setkey(skcipher, key, keylen, 0); 824 } 825 826 static int des3_skcipher_setkey(struct crypto_skcipher *skcipher, 827 const u8 *key, unsigned int keylen) 828 { 829 return verify_skcipher_des3_key(skcipher, key) ?: 830 skcipher_setkey(skcipher, key, keylen, 0); 831 } 832 833 static int xts_skcipher_setkey(struct crypto_skcipher *skcipher, const u8 *key, 834 unsigned int keylen) 835 { 836 struct caam_ctx *ctx = crypto_skcipher_ctx(skcipher); 837 struct device *jrdev = ctx->jrdev; 838 u32 *desc; 839 840 if (keylen != 2 * AES_MIN_KEY_SIZE && keylen != 2 * AES_MAX_KEY_SIZE) { 841 dev_err(jrdev, "key size mismatch\n"); 842 return -EINVAL; 843 } 844 845 ctx->cdata.keylen = keylen; 846 ctx->cdata.key_virt = key; 847 ctx->cdata.key_inline = true; 848 849 /* xts_skcipher_encrypt shared descriptor */ 850 desc = ctx->sh_desc_enc; 851 cnstr_shdsc_xts_skcipher_encap(desc, &ctx->cdata); 852 dma_sync_single_for_device(jrdev, ctx->sh_desc_enc_dma, 853 desc_bytes(desc), ctx->dir); 854 855 /* xts_skcipher_decrypt shared descriptor */ 856 desc = ctx->sh_desc_dec; 857 cnstr_shdsc_xts_skcipher_decap(desc, &ctx->cdata); 858 dma_sync_single_for_device(jrdev, ctx->sh_desc_dec_dma, 859 desc_bytes(desc), ctx->dir); 860 861 return 0; 862 } 863 864 /* 865 * aead_edesc - s/w-extended aead descriptor 866 * @src_nents: number of segments in input s/w scatterlist 867 * @dst_nents: number of segments in output s/w scatterlist 868 * @mapped_src_nents: number of segments in input h/w link table 869 * @mapped_dst_nents: number of segments in output h/w link table 870 * @sec4_sg_bytes: length of dma mapped sec4_sg space 871 * @bklog: stored to determine if the request needs backlog 872 * @sec4_sg_dma: bus physical mapped address of h/w link table 873 * @sec4_sg: pointer to h/w link table 874 * @hw_desc: the h/w job descriptor followed by any referenced link tables 875 */ 876 struct aead_edesc { 877 int src_nents; 878 int dst_nents; 879 int mapped_src_nents; 880 int mapped_dst_nents; 881 int sec4_sg_bytes; 882 bool bklog; 883 dma_addr_t sec4_sg_dma; 884 struct sec4_sg_entry *sec4_sg; 885 u32 hw_desc[]; 886 }; 887 888 /* 889 * skcipher_edesc - s/w-extended skcipher descriptor 890 * @src_nents: number of segments in input s/w scatterlist 891 * @dst_nents: number of segments in output s/w scatterlist 892 * @mapped_src_nents: number of segments in input h/w link table 893 * @mapped_dst_nents: number of segments in output h/w link table 894 * @iv_dma: dma address of iv for checking continuity and link table 895 * @sec4_sg_bytes: length of dma mapped sec4_sg space 896 * @bklog: stored to determine if the request needs backlog 897 * @sec4_sg_dma: bus physical mapped address of h/w link table 898 * @sec4_sg: pointer to h/w link table 899 * @hw_desc: the h/w job descriptor followed by any referenced link tables 900 * and IV 901 */ 902 struct skcipher_edesc { 903 int src_nents; 904 int dst_nents; 905 int mapped_src_nents; 906 int mapped_dst_nents; 907 dma_addr_t iv_dma; 908 int sec4_sg_bytes; 909 bool bklog; 910 dma_addr_t sec4_sg_dma; 911 struct sec4_sg_entry *sec4_sg; 912 u32 hw_desc[]; 913 }; 914 915 static void caam_unmap(struct device *dev, struct scatterlist *src, 916 struct scatterlist *dst, int src_nents, 917 int dst_nents, 918 dma_addr_t iv_dma, int ivsize, dma_addr_t sec4_sg_dma, 919 int sec4_sg_bytes) 920 { 921 if (dst != src) { 922 if (src_nents) 923 dma_unmap_sg(dev, src, src_nents, DMA_TO_DEVICE); 924 if (dst_nents) 925 dma_unmap_sg(dev, dst, dst_nents, DMA_FROM_DEVICE); 926 } else { 927 dma_unmap_sg(dev, src, src_nents, DMA_BIDIRECTIONAL); 928 } 929 930 if (iv_dma) 931 dma_unmap_single(dev, iv_dma, ivsize, DMA_BIDIRECTIONAL); 932 if (sec4_sg_bytes) 933 dma_unmap_single(dev, sec4_sg_dma, sec4_sg_bytes, 934 DMA_TO_DEVICE); 935 } 936 937 static void aead_unmap(struct device *dev, 938 struct aead_edesc *edesc, 939 struct aead_request *req) 940 { 941 caam_unmap(dev, req->src, req->dst, 942 edesc->src_nents, edesc->dst_nents, 0, 0, 943 edesc->sec4_sg_dma, edesc->sec4_sg_bytes); 944 } 945 946 static void skcipher_unmap(struct device *dev, struct skcipher_edesc *edesc, 947 struct skcipher_request *req) 948 { 949 struct crypto_skcipher *skcipher = crypto_skcipher_reqtfm(req); 950 int ivsize = crypto_skcipher_ivsize(skcipher); 951 952 caam_unmap(dev, req->src, req->dst, 953 edesc->src_nents, edesc->dst_nents, 954 edesc->iv_dma, ivsize, 955 edesc->sec4_sg_dma, edesc->sec4_sg_bytes); 956 } 957 958 static void aead_crypt_done(struct device *jrdev, u32 *desc, u32 err, 959 void *context) 960 { 961 struct aead_request *req = context; 962 struct caam_aead_req_ctx *rctx = aead_request_ctx(req); 963 struct caam_drv_private_jr *jrp = dev_get_drvdata(jrdev); 964 struct aead_edesc *edesc; 965 int ecode = 0; 966 bool has_bklog; 967 968 dev_dbg(jrdev, "%s %d: err 0x%x\n", __func__, __LINE__, err); 969 970 edesc = rctx->edesc; 971 has_bklog = edesc->bklog; 972 973 if (err) 974 ecode = caam_jr_strstatus(jrdev, err); 975 976 aead_unmap(jrdev, edesc, req); 977 978 kfree(edesc); 979 980 /* 981 * If no backlog flag, the completion of the request is done 982 * by CAAM, not crypto engine. 983 */ 984 if (!has_bklog) 985 aead_request_complete(req, ecode); 986 else 987 crypto_finalize_aead_request(jrp->engine, req, ecode); 988 } 989 990 static void skcipher_crypt_done(struct device *jrdev, u32 *desc, u32 err, 991 void *context) 992 { 993 struct skcipher_request *req = context; 994 struct skcipher_edesc *edesc; 995 struct caam_skcipher_req_ctx *rctx = skcipher_request_ctx(req); 996 struct crypto_skcipher *skcipher = crypto_skcipher_reqtfm(req); 997 struct caam_drv_private_jr *jrp = dev_get_drvdata(jrdev); 998 int ivsize = crypto_skcipher_ivsize(skcipher); 999 int ecode = 0; 1000 bool has_bklog; 1001 1002 dev_dbg(jrdev, "%s %d: err 0x%x\n", __func__, __LINE__, err); 1003 1004 edesc = rctx->edesc; 1005 has_bklog = edesc->bklog; 1006 if (err) 1007 ecode = caam_jr_strstatus(jrdev, err); 1008 1009 skcipher_unmap(jrdev, edesc, req); 1010 1011 /* 1012 * The crypto API expects us to set the IV (req->iv) to the last 1013 * ciphertext block (CBC mode) or last counter (CTR mode). 1014 * This is used e.g. by the CTS mode. 1015 */ 1016 if (ivsize && !ecode) { 1017 memcpy(req->iv, (u8 *)edesc->sec4_sg + edesc->sec4_sg_bytes, 1018 ivsize); 1019 1020 print_hex_dump_debug("dstiv @" __stringify(__LINE__)": ", 1021 DUMP_PREFIX_ADDRESS, 16, 4, req->iv, 1022 ivsize, 1); 1023 } 1024 1025 caam_dump_sg("dst @" __stringify(__LINE__)": ", 1026 DUMP_PREFIX_ADDRESS, 16, 4, req->dst, 1027 edesc->dst_nents > 1 ? 100 : req->cryptlen, 1); 1028 1029 kfree(edesc); 1030 1031 /* 1032 * If no backlog flag, the completion of the request is done 1033 * by CAAM, not crypto engine. 1034 */ 1035 if (!has_bklog) 1036 skcipher_request_complete(req, ecode); 1037 else 1038 crypto_finalize_skcipher_request(jrp->engine, req, ecode); 1039 } 1040 1041 /* 1042 * Fill in aead job descriptor 1043 */ 1044 static void init_aead_job(struct aead_request *req, 1045 struct aead_edesc *edesc, 1046 bool all_contig, bool encrypt) 1047 { 1048 struct crypto_aead *aead = crypto_aead_reqtfm(req); 1049 struct caam_ctx *ctx = crypto_aead_ctx(aead); 1050 int authsize = ctx->authsize; 1051 u32 *desc = edesc->hw_desc; 1052 u32 out_options, in_options; 1053 dma_addr_t dst_dma, src_dma; 1054 int len, sec4_sg_index = 0; 1055 dma_addr_t ptr; 1056 u32 *sh_desc; 1057 1058 sh_desc = encrypt ? ctx->sh_desc_enc : ctx->sh_desc_dec; 1059 ptr = encrypt ? ctx->sh_desc_enc_dma : ctx->sh_desc_dec_dma; 1060 1061 len = desc_len(sh_desc); 1062 init_job_desc_shared(desc, ptr, len, HDR_SHARE_DEFER | HDR_REVERSE); 1063 1064 if (all_contig) { 1065 src_dma = edesc->mapped_src_nents ? sg_dma_address(req->src) : 1066 0; 1067 in_options = 0; 1068 } else { 1069 src_dma = edesc->sec4_sg_dma; 1070 sec4_sg_index += edesc->mapped_src_nents; 1071 in_options = LDST_SGF; 1072 } 1073 1074 append_seq_in_ptr(desc, src_dma, req->assoclen + req->cryptlen, 1075 in_options); 1076 1077 dst_dma = src_dma; 1078 out_options = in_options; 1079 1080 if (unlikely(req->src != req->dst)) { 1081 if (!edesc->mapped_dst_nents) { 1082 dst_dma = 0; 1083 out_options = 0; 1084 } else if (edesc->mapped_dst_nents == 1) { 1085 dst_dma = sg_dma_address(req->dst); 1086 out_options = 0; 1087 } else { 1088 dst_dma = edesc->sec4_sg_dma + 1089 sec4_sg_index * 1090 sizeof(struct sec4_sg_entry); 1091 out_options = LDST_SGF; 1092 } 1093 } 1094 1095 if (encrypt) 1096 append_seq_out_ptr(desc, dst_dma, 1097 req->assoclen + req->cryptlen + authsize, 1098 out_options); 1099 else 1100 append_seq_out_ptr(desc, dst_dma, 1101 req->assoclen + req->cryptlen - authsize, 1102 out_options); 1103 } 1104 1105 static void init_gcm_job(struct aead_request *req, 1106 struct aead_edesc *edesc, 1107 bool all_contig, bool encrypt) 1108 { 1109 struct crypto_aead *aead = crypto_aead_reqtfm(req); 1110 struct caam_ctx *ctx = crypto_aead_ctx(aead); 1111 unsigned int ivsize = crypto_aead_ivsize(aead); 1112 u32 *desc = edesc->hw_desc; 1113 bool generic_gcm = (ivsize == GCM_AES_IV_SIZE); 1114 unsigned int last; 1115 1116 init_aead_job(req, edesc, all_contig, encrypt); 1117 append_math_add_imm_u32(desc, REG3, ZERO, IMM, req->assoclen); 1118 1119 /* BUG This should not be specific to generic GCM. */ 1120 last = 0; 1121 if (encrypt && generic_gcm && !(req->assoclen + req->cryptlen)) 1122 last = FIFOLD_TYPE_LAST1; 1123 1124 /* Read GCM IV */ 1125 append_cmd(desc, CMD_FIFO_LOAD | FIFOLD_CLASS_CLASS1 | IMMEDIATE | 1126 FIFOLD_TYPE_IV | FIFOLD_TYPE_FLUSH1 | GCM_AES_IV_SIZE | last); 1127 /* Append Salt */ 1128 if (!generic_gcm) 1129 append_data(desc, ctx->key + ctx->cdata.keylen, 4); 1130 /* Append IV */ 1131 append_data(desc, req->iv, ivsize); 1132 /* End of blank commands */ 1133 } 1134 1135 static void init_chachapoly_job(struct aead_request *req, 1136 struct aead_edesc *edesc, bool all_contig, 1137 bool encrypt) 1138 { 1139 struct crypto_aead *aead = crypto_aead_reqtfm(req); 1140 unsigned int ivsize = crypto_aead_ivsize(aead); 1141 unsigned int assoclen = req->assoclen; 1142 u32 *desc = edesc->hw_desc; 1143 u32 ctx_iv_off = 4; 1144 1145 init_aead_job(req, edesc, all_contig, encrypt); 1146 1147 if (ivsize != CHACHAPOLY_IV_SIZE) { 1148 /* IPsec specific: CONTEXT1[223:128] = {NONCE, IV} */ 1149 ctx_iv_off += 4; 1150 1151 /* 1152 * The associated data comes already with the IV but we need 1153 * to skip it when we authenticate or encrypt... 1154 */ 1155 assoclen -= ivsize; 1156 } 1157 1158 append_math_add_imm_u32(desc, REG3, ZERO, IMM, assoclen); 1159 1160 /* 1161 * For IPsec load the IV further in the same register. 1162 * For RFC7539 simply load the 12 bytes nonce in a single operation 1163 */ 1164 append_load_as_imm(desc, req->iv, ivsize, LDST_CLASS_1_CCB | 1165 LDST_SRCDST_BYTE_CONTEXT | 1166 ctx_iv_off << LDST_OFFSET_SHIFT); 1167 } 1168 1169 static void init_authenc_job(struct aead_request *req, 1170 struct aead_edesc *edesc, 1171 bool all_contig, bool encrypt) 1172 { 1173 struct crypto_aead *aead = crypto_aead_reqtfm(req); 1174 struct caam_aead_alg *alg = container_of(crypto_aead_alg(aead), 1175 struct caam_aead_alg, aead); 1176 unsigned int ivsize = crypto_aead_ivsize(aead); 1177 struct caam_ctx *ctx = crypto_aead_ctx(aead); 1178 struct caam_drv_private *ctrlpriv = dev_get_drvdata(ctx->jrdev->parent); 1179 const bool ctr_mode = ((ctx->cdata.algtype & OP_ALG_AAI_MASK) == 1180 OP_ALG_AAI_CTR_MOD128); 1181 const bool is_rfc3686 = alg->caam.rfc3686; 1182 u32 *desc = edesc->hw_desc; 1183 u32 ivoffset = 0; 1184 1185 /* 1186 * AES-CTR needs to load IV in CONTEXT1 reg 1187 * at an offset of 128bits (16bytes) 1188 * CONTEXT1[255:128] = IV 1189 */ 1190 if (ctr_mode) 1191 ivoffset = 16; 1192 1193 /* 1194 * RFC3686 specific: 1195 * CONTEXT1[255:128] = {NONCE, IV, COUNTER} 1196 */ 1197 if (is_rfc3686) 1198 ivoffset = 16 + CTR_RFC3686_NONCE_SIZE; 1199 1200 init_aead_job(req, edesc, all_contig, encrypt); 1201 1202 /* 1203 * {REG3, DPOVRD} = assoclen, depending on whether MATH command supports 1204 * having DPOVRD as destination. 1205 */ 1206 if (ctrlpriv->era < 3) 1207 append_math_add_imm_u32(desc, REG3, ZERO, IMM, req->assoclen); 1208 else 1209 append_math_add_imm_u32(desc, DPOVRD, ZERO, IMM, req->assoclen); 1210 1211 if (ivsize && ((is_rfc3686 && encrypt) || !alg->caam.geniv)) 1212 append_load_as_imm(desc, req->iv, ivsize, 1213 LDST_CLASS_1_CCB | 1214 LDST_SRCDST_BYTE_CONTEXT | 1215 (ivoffset << LDST_OFFSET_SHIFT)); 1216 } 1217 1218 /* 1219 * Fill in skcipher job descriptor 1220 */ 1221 static void init_skcipher_job(struct skcipher_request *req, 1222 struct skcipher_edesc *edesc, 1223 const bool encrypt) 1224 { 1225 struct crypto_skcipher *skcipher = crypto_skcipher_reqtfm(req); 1226 struct caam_ctx *ctx = crypto_skcipher_ctx(skcipher); 1227 struct device *jrdev = ctx->jrdev; 1228 int ivsize = crypto_skcipher_ivsize(skcipher); 1229 u32 *desc = edesc->hw_desc; 1230 u32 *sh_desc; 1231 u32 in_options = 0, out_options = 0; 1232 dma_addr_t src_dma, dst_dma, ptr; 1233 int len, sec4_sg_index = 0; 1234 1235 print_hex_dump_debug("presciv@"__stringify(__LINE__)": ", 1236 DUMP_PREFIX_ADDRESS, 16, 4, req->iv, ivsize, 1); 1237 dev_dbg(jrdev, "asked=%d, cryptlen%d\n", 1238 (int)edesc->src_nents > 1 ? 100 : req->cryptlen, req->cryptlen); 1239 1240 caam_dump_sg("src @" __stringify(__LINE__)": ", 1241 DUMP_PREFIX_ADDRESS, 16, 4, req->src, 1242 edesc->src_nents > 1 ? 100 : req->cryptlen, 1); 1243 1244 sh_desc = encrypt ? ctx->sh_desc_enc : ctx->sh_desc_dec; 1245 ptr = encrypt ? ctx->sh_desc_enc_dma : ctx->sh_desc_dec_dma; 1246 1247 len = desc_len(sh_desc); 1248 init_job_desc_shared(desc, ptr, len, HDR_SHARE_DEFER | HDR_REVERSE); 1249 1250 if (ivsize || edesc->mapped_src_nents > 1) { 1251 src_dma = edesc->sec4_sg_dma; 1252 sec4_sg_index = edesc->mapped_src_nents + !!ivsize; 1253 in_options = LDST_SGF; 1254 } else { 1255 src_dma = sg_dma_address(req->src); 1256 } 1257 1258 append_seq_in_ptr(desc, src_dma, req->cryptlen + ivsize, in_options); 1259 1260 if (likely(req->src == req->dst)) { 1261 dst_dma = src_dma + !!ivsize * sizeof(struct sec4_sg_entry); 1262 out_options = in_options; 1263 } else if (!ivsize && edesc->mapped_dst_nents == 1) { 1264 dst_dma = sg_dma_address(req->dst); 1265 } else { 1266 dst_dma = edesc->sec4_sg_dma + sec4_sg_index * 1267 sizeof(struct sec4_sg_entry); 1268 out_options = LDST_SGF; 1269 } 1270 1271 append_seq_out_ptr(desc, dst_dma, req->cryptlen + ivsize, out_options); 1272 } 1273 1274 /* 1275 * allocate and map the aead extended descriptor 1276 */ 1277 static struct aead_edesc *aead_edesc_alloc(struct aead_request *req, 1278 int desc_bytes, bool *all_contig_ptr, 1279 bool encrypt) 1280 { 1281 struct crypto_aead *aead = crypto_aead_reqtfm(req); 1282 struct caam_ctx *ctx = crypto_aead_ctx(aead); 1283 struct device *jrdev = ctx->jrdev; 1284 struct caam_aead_req_ctx *rctx = aead_request_ctx(req); 1285 gfp_t flags = (req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP) ? 1286 GFP_KERNEL : GFP_ATOMIC; 1287 int src_nents, mapped_src_nents, dst_nents = 0, mapped_dst_nents = 0; 1288 int src_len, dst_len = 0; 1289 struct aead_edesc *edesc; 1290 int sec4_sg_index, sec4_sg_len, sec4_sg_bytes; 1291 unsigned int authsize = ctx->authsize; 1292 1293 if (unlikely(req->dst != req->src)) { 1294 src_len = req->assoclen + req->cryptlen; 1295 dst_len = src_len + (encrypt ? authsize : (-authsize)); 1296 1297 src_nents = sg_nents_for_len(req->src, src_len); 1298 if (unlikely(src_nents < 0)) { 1299 dev_err(jrdev, "Insufficient bytes (%d) in src S/G\n", 1300 src_len); 1301 return ERR_PTR(src_nents); 1302 } 1303 1304 dst_nents = sg_nents_for_len(req->dst, dst_len); 1305 if (unlikely(dst_nents < 0)) { 1306 dev_err(jrdev, "Insufficient bytes (%d) in dst S/G\n", 1307 dst_len); 1308 return ERR_PTR(dst_nents); 1309 } 1310 } else { 1311 src_len = req->assoclen + req->cryptlen + 1312 (encrypt ? authsize : 0); 1313 1314 src_nents = sg_nents_for_len(req->src, src_len); 1315 if (unlikely(src_nents < 0)) { 1316 dev_err(jrdev, "Insufficient bytes (%d) in src S/G\n", 1317 src_len); 1318 return ERR_PTR(src_nents); 1319 } 1320 } 1321 1322 if (likely(req->src == req->dst)) { 1323 mapped_src_nents = dma_map_sg(jrdev, req->src, src_nents, 1324 DMA_BIDIRECTIONAL); 1325 if (unlikely(!mapped_src_nents)) { 1326 dev_err(jrdev, "unable to map source\n"); 1327 return ERR_PTR(-ENOMEM); 1328 } 1329 } else { 1330 /* Cover also the case of null (zero length) input data */ 1331 if (src_nents) { 1332 mapped_src_nents = dma_map_sg(jrdev, req->src, 1333 src_nents, DMA_TO_DEVICE); 1334 if (unlikely(!mapped_src_nents)) { 1335 dev_err(jrdev, "unable to map source\n"); 1336 return ERR_PTR(-ENOMEM); 1337 } 1338 } else { 1339 mapped_src_nents = 0; 1340 } 1341 1342 /* Cover also the case of null (zero length) output data */ 1343 if (dst_nents) { 1344 mapped_dst_nents = dma_map_sg(jrdev, req->dst, 1345 dst_nents, 1346 DMA_FROM_DEVICE); 1347 if (unlikely(!mapped_dst_nents)) { 1348 dev_err(jrdev, "unable to map destination\n"); 1349 dma_unmap_sg(jrdev, req->src, src_nents, 1350 DMA_TO_DEVICE); 1351 return ERR_PTR(-ENOMEM); 1352 } 1353 } else { 1354 mapped_dst_nents = 0; 1355 } 1356 } 1357 1358 /* 1359 * HW reads 4 S/G entries at a time; make sure the reads don't go beyond 1360 * the end of the table by allocating more S/G entries. 1361 */ 1362 sec4_sg_len = mapped_src_nents > 1 ? mapped_src_nents : 0; 1363 if (mapped_dst_nents > 1) 1364 sec4_sg_len += pad_sg_nents(mapped_dst_nents); 1365 else 1366 sec4_sg_len = pad_sg_nents(sec4_sg_len); 1367 1368 sec4_sg_bytes = sec4_sg_len * sizeof(struct sec4_sg_entry); 1369 1370 /* allocate space for base edesc and hw desc commands, link tables */ 1371 edesc = kzalloc(sizeof(*edesc) + desc_bytes + sec4_sg_bytes, 1372 GFP_DMA | flags); 1373 if (!edesc) { 1374 caam_unmap(jrdev, req->src, req->dst, src_nents, dst_nents, 0, 1375 0, 0, 0); 1376 return ERR_PTR(-ENOMEM); 1377 } 1378 1379 edesc->src_nents = src_nents; 1380 edesc->dst_nents = dst_nents; 1381 edesc->mapped_src_nents = mapped_src_nents; 1382 edesc->mapped_dst_nents = mapped_dst_nents; 1383 edesc->sec4_sg = (void *)edesc + sizeof(struct aead_edesc) + 1384 desc_bytes; 1385 1386 rctx->edesc = edesc; 1387 1388 *all_contig_ptr = !(mapped_src_nents > 1); 1389 1390 sec4_sg_index = 0; 1391 if (mapped_src_nents > 1) { 1392 sg_to_sec4_sg_last(req->src, src_len, 1393 edesc->sec4_sg + sec4_sg_index, 0); 1394 sec4_sg_index += mapped_src_nents; 1395 } 1396 if (mapped_dst_nents > 1) { 1397 sg_to_sec4_sg_last(req->dst, dst_len, 1398 edesc->sec4_sg + sec4_sg_index, 0); 1399 } 1400 1401 if (!sec4_sg_bytes) 1402 return edesc; 1403 1404 edesc->sec4_sg_dma = dma_map_single(jrdev, edesc->sec4_sg, 1405 sec4_sg_bytes, DMA_TO_DEVICE); 1406 if (dma_mapping_error(jrdev, edesc->sec4_sg_dma)) { 1407 dev_err(jrdev, "unable to map S/G table\n"); 1408 aead_unmap(jrdev, edesc, req); 1409 kfree(edesc); 1410 return ERR_PTR(-ENOMEM); 1411 } 1412 1413 edesc->sec4_sg_bytes = sec4_sg_bytes; 1414 1415 return edesc; 1416 } 1417 1418 static int aead_enqueue_req(struct device *jrdev, struct aead_request *req) 1419 { 1420 struct caam_drv_private_jr *jrpriv = dev_get_drvdata(jrdev); 1421 struct caam_aead_req_ctx *rctx = aead_request_ctx(req); 1422 struct aead_edesc *edesc = rctx->edesc; 1423 u32 *desc = edesc->hw_desc; 1424 int ret; 1425 1426 /* 1427 * Only the backlog request are sent to crypto-engine since the others 1428 * can be handled by CAAM, if free, especially since JR has up to 1024 1429 * entries (more than the 10 entries from crypto-engine). 1430 */ 1431 if (req->base.flags & CRYPTO_TFM_REQ_MAY_BACKLOG) 1432 ret = crypto_transfer_aead_request_to_engine(jrpriv->engine, 1433 req); 1434 else 1435 ret = caam_jr_enqueue(jrdev, desc, aead_crypt_done, req); 1436 1437 if ((ret != -EINPROGRESS) && (ret != -EBUSY)) { 1438 aead_unmap(jrdev, edesc, req); 1439 kfree(rctx->edesc); 1440 } 1441 1442 return ret; 1443 } 1444 1445 static inline int chachapoly_crypt(struct aead_request *req, bool encrypt) 1446 { 1447 struct aead_edesc *edesc; 1448 struct crypto_aead *aead = crypto_aead_reqtfm(req); 1449 struct caam_ctx *ctx = crypto_aead_ctx(aead); 1450 struct device *jrdev = ctx->jrdev; 1451 bool all_contig; 1452 u32 *desc; 1453 1454 edesc = aead_edesc_alloc(req, CHACHAPOLY_DESC_JOB_IO_LEN, &all_contig, 1455 encrypt); 1456 if (IS_ERR(edesc)) 1457 return PTR_ERR(edesc); 1458 1459 desc = edesc->hw_desc; 1460 1461 init_chachapoly_job(req, edesc, all_contig, encrypt); 1462 print_hex_dump_debug("chachapoly jobdesc@" __stringify(__LINE__)": ", 1463 DUMP_PREFIX_ADDRESS, 16, 4, desc, desc_bytes(desc), 1464 1); 1465 1466 return aead_enqueue_req(jrdev, req); 1467 } 1468 1469 static int chachapoly_encrypt(struct aead_request *req) 1470 { 1471 return chachapoly_crypt(req, true); 1472 } 1473 1474 static int chachapoly_decrypt(struct aead_request *req) 1475 { 1476 return chachapoly_crypt(req, false); 1477 } 1478 1479 static inline int aead_crypt(struct aead_request *req, bool encrypt) 1480 { 1481 struct aead_edesc *edesc; 1482 struct crypto_aead *aead = crypto_aead_reqtfm(req); 1483 struct caam_ctx *ctx = crypto_aead_ctx(aead); 1484 struct device *jrdev = ctx->jrdev; 1485 bool all_contig; 1486 1487 /* allocate extended descriptor */ 1488 edesc = aead_edesc_alloc(req, AUTHENC_DESC_JOB_IO_LEN, 1489 &all_contig, encrypt); 1490 if (IS_ERR(edesc)) 1491 return PTR_ERR(edesc); 1492 1493 /* Create and submit job descriptor */ 1494 init_authenc_job(req, edesc, all_contig, encrypt); 1495 1496 print_hex_dump_debug("aead jobdesc@"__stringify(__LINE__)": ", 1497 DUMP_PREFIX_ADDRESS, 16, 4, edesc->hw_desc, 1498 desc_bytes(edesc->hw_desc), 1); 1499 1500 return aead_enqueue_req(jrdev, req); 1501 } 1502 1503 static int aead_encrypt(struct aead_request *req) 1504 { 1505 return aead_crypt(req, true); 1506 } 1507 1508 static int aead_decrypt(struct aead_request *req) 1509 { 1510 return aead_crypt(req, false); 1511 } 1512 1513 static int aead_do_one_req(struct crypto_engine *engine, void *areq) 1514 { 1515 struct aead_request *req = aead_request_cast(areq); 1516 struct caam_ctx *ctx = crypto_aead_ctx(crypto_aead_reqtfm(req)); 1517 struct caam_aead_req_ctx *rctx = aead_request_ctx(req); 1518 u32 *desc = rctx->edesc->hw_desc; 1519 int ret; 1520 1521 rctx->edesc->bklog = true; 1522 1523 ret = caam_jr_enqueue(ctx->jrdev, desc, aead_crypt_done, req); 1524 1525 if (ret != -EINPROGRESS) { 1526 aead_unmap(ctx->jrdev, rctx->edesc, req); 1527 kfree(rctx->edesc); 1528 } else { 1529 ret = 0; 1530 } 1531 1532 return ret; 1533 } 1534 1535 static inline int gcm_crypt(struct aead_request *req, bool encrypt) 1536 { 1537 struct aead_edesc *edesc; 1538 struct crypto_aead *aead = crypto_aead_reqtfm(req); 1539 struct caam_ctx *ctx = crypto_aead_ctx(aead); 1540 struct device *jrdev = ctx->jrdev; 1541 bool all_contig; 1542 1543 /* allocate extended descriptor */ 1544 edesc = aead_edesc_alloc(req, GCM_DESC_JOB_IO_LEN, &all_contig, 1545 encrypt); 1546 if (IS_ERR(edesc)) 1547 return PTR_ERR(edesc); 1548 1549 /* Create and submit job descriptor */ 1550 init_gcm_job(req, edesc, all_contig, encrypt); 1551 1552 print_hex_dump_debug("aead jobdesc@"__stringify(__LINE__)": ", 1553 DUMP_PREFIX_ADDRESS, 16, 4, edesc->hw_desc, 1554 desc_bytes(edesc->hw_desc), 1); 1555 1556 return aead_enqueue_req(jrdev, req); 1557 } 1558 1559 static int gcm_encrypt(struct aead_request *req) 1560 { 1561 return gcm_crypt(req, true); 1562 } 1563 1564 static int gcm_decrypt(struct aead_request *req) 1565 { 1566 return gcm_crypt(req, false); 1567 } 1568 1569 static int ipsec_gcm_encrypt(struct aead_request *req) 1570 { 1571 return crypto_ipsec_check_assoclen(req->assoclen) ? : gcm_encrypt(req); 1572 } 1573 1574 static int ipsec_gcm_decrypt(struct aead_request *req) 1575 { 1576 return crypto_ipsec_check_assoclen(req->assoclen) ? : gcm_decrypt(req); 1577 } 1578 1579 /* 1580 * allocate and map the skcipher extended descriptor for skcipher 1581 */ 1582 static struct skcipher_edesc *skcipher_edesc_alloc(struct skcipher_request *req, 1583 int desc_bytes) 1584 { 1585 struct crypto_skcipher *skcipher = crypto_skcipher_reqtfm(req); 1586 struct caam_ctx *ctx = crypto_skcipher_ctx(skcipher); 1587 struct caam_skcipher_req_ctx *rctx = skcipher_request_ctx(req); 1588 struct device *jrdev = ctx->jrdev; 1589 gfp_t flags = (req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP) ? 1590 GFP_KERNEL : GFP_ATOMIC; 1591 int src_nents, mapped_src_nents, dst_nents = 0, mapped_dst_nents = 0; 1592 struct skcipher_edesc *edesc; 1593 dma_addr_t iv_dma = 0; 1594 u8 *iv; 1595 int ivsize = crypto_skcipher_ivsize(skcipher); 1596 int dst_sg_idx, sec4_sg_ents, sec4_sg_bytes; 1597 1598 src_nents = sg_nents_for_len(req->src, req->cryptlen); 1599 if (unlikely(src_nents < 0)) { 1600 dev_err(jrdev, "Insufficient bytes (%d) in src S/G\n", 1601 req->cryptlen); 1602 return ERR_PTR(src_nents); 1603 } 1604 1605 if (req->dst != req->src) { 1606 dst_nents = sg_nents_for_len(req->dst, req->cryptlen); 1607 if (unlikely(dst_nents < 0)) { 1608 dev_err(jrdev, "Insufficient bytes (%d) in dst S/G\n", 1609 req->cryptlen); 1610 return ERR_PTR(dst_nents); 1611 } 1612 } 1613 1614 if (likely(req->src == req->dst)) { 1615 mapped_src_nents = dma_map_sg(jrdev, req->src, src_nents, 1616 DMA_BIDIRECTIONAL); 1617 if (unlikely(!mapped_src_nents)) { 1618 dev_err(jrdev, "unable to map source\n"); 1619 return ERR_PTR(-ENOMEM); 1620 } 1621 } else { 1622 mapped_src_nents = dma_map_sg(jrdev, req->src, src_nents, 1623 DMA_TO_DEVICE); 1624 if (unlikely(!mapped_src_nents)) { 1625 dev_err(jrdev, "unable to map source\n"); 1626 return ERR_PTR(-ENOMEM); 1627 } 1628 mapped_dst_nents = dma_map_sg(jrdev, req->dst, dst_nents, 1629 DMA_FROM_DEVICE); 1630 if (unlikely(!mapped_dst_nents)) { 1631 dev_err(jrdev, "unable to map destination\n"); 1632 dma_unmap_sg(jrdev, req->src, src_nents, DMA_TO_DEVICE); 1633 return ERR_PTR(-ENOMEM); 1634 } 1635 } 1636 1637 if (!ivsize && mapped_src_nents == 1) 1638 sec4_sg_ents = 0; // no need for an input hw s/g table 1639 else 1640 sec4_sg_ents = mapped_src_nents + !!ivsize; 1641 dst_sg_idx = sec4_sg_ents; 1642 1643 /* 1644 * Input, output HW S/G tables: [IV, src][dst, IV] 1645 * IV entries point to the same buffer 1646 * If src == dst, S/G entries are reused (S/G tables overlap) 1647 * 1648 * HW reads 4 S/G entries at a time; make sure the reads don't go beyond 1649 * the end of the table by allocating more S/G entries. Logic: 1650 * if (output S/G) 1651 * pad output S/G, if needed 1652 * else if (input S/G) ... 1653 * pad input S/G, if needed 1654 */ 1655 if (ivsize || mapped_dst_nents > 1) { 1656 if (req->src == req->dst) 1657 sec4_sg_ents = !!ivsize + pad_sg_nents(sec4_sg_ents); 1658 else 1659 sec4_sg_ents += pad_sg_nents(mapped_dst_nents + 1660 !!ivsize); 1661 } else { 1662 sec4_sg_ents = pad_sg_nents(sec4_sg_ents); 1663 } 1664 1665 sec4_sg_bytes = sec4_sg_ents * sizeof(struct sec4_sg_entry); 1666 1667 /* 1668 * allocate space for base edesc and hw desc commands, link tables, IV 1669 */ 1670 edesc = kzalloc(sizeof(*edesc) + desc_bytes + sec4_sg_bytes + ivsize, 1671 GFP_DMA | flags); 1672 if (!edesc) { 1673 dev_err(jrdev, "could not allocate extended descriptor\n"); 1674 caam_unmap(jrdev, req->src, req->dst, src_nents, dst_nents, 0, 1675 0, 0, 0); 1676 return ERR_PTR(-ENOMEM); 1677 } 1678 1679 edesc->src_nents = src_nents; 1680 edesc->dst_nents = dst_nents; 1681 edesc->mapped_src_nents = mapped_src_nents; 1682 edesc->mapped_dst_nents = mapped_dst_nents; 1683 edesc->sec4_sg_bytes = sec4_sg_bytes; 1684 edesc->sec4_sg = (struct sec4_sg_entry *)((u8 *)edesc->hw_desc + 1685 desc_bytes); 1686 rctx->edesc = edesc; 1687 1688 /* Make sure IV is located in a DMAable area */ 1689 if (ivsize) { 1690 iv = (u8 *)edesc->sec4_sg + sec4_sg_bytes; 1691 memcpy(iv, req->iv, ivsize); 1692 1693 iv_dma = dma_map_single(jrdev, iv, ivsize, DMA_BIDIRECTIONAL); 1694 if (dma_mapping_error(jrdev, iv_dma)) { 1695 dev_err(jrdev, "unable to map IV\n"); 1696 caam_unmap(jrdev, req->src, req->dst, src_nents, 1697 dst_nents, 0, 0, 0, 0); 1698 kfree(edesc); 1699 return ERR_PTR(-ENOMEM); 1700 } 1701 1702 dma_to_sec4_sg_one(edesc->sec4_sg, iv_dma, ivsize, 0); 1703 } 1704 if (dst_sg_idx) 1705 sg_to_sec4_sg(req->src, req->cryptlen, edesc->sec4_sg + 1706 !!ivsize, 0); 1707 1708 if (req->src != req->dst && (ivsize || mapped_dst_nents > 1)) 1709 sg_to_sec4_sg(req->dst, req->cryptlen, edesc->sec4_sg + 1710 dst_sg_idx, 0); 1711 1712 if (ivsize) 1713 dma_to_sec4_sg_one(edesc->sec4_sg + dst_sg_idx + 1714 mapped_dst_nents, iv_dma, ivsize, 0); 1715 1716 if (ivsize || mapped_dst_nents > 1) 1717 sg_to_sec4_set_last(edesc->sec4_sg + dst_sg_idx + 1718 mapped_dst_nents - 1 + !!ivsize); 1719 1720 if (sec4_sg_bytes) { 1721 edesc->sec4_sg_dma = dma_map_single(jrdev, edesc->sec4_sg, 1722 sec4_sg_bytes, 1723 DMA_TO_DEVICE); 1724 if (dma_mapping_error(jrdev, edesc->sec4_sg_dma)) { 1725 dev_err(jrdev, "unable to map S/G table\n"); 1726 caam_unmap(jrdev, req->src, req->dst, src_nents, 1727 dst_nents, iv_dma, ivsize, 0, 0); 1728 kfree(edesc); 1729 return ERR_PTR(-ENOMEM); 1730 } 1731 } 1732 1733 edesc->iv_dma = iv_dma; 1734 1735 print_hex_dump_debug("skcipher sec4_sg@" __stringify(__LINE__)": ", 1736 DUMP_PREFIX_ADDRESS, 16, 4, edesc->sec4_sg, 1737 sec4_sg_bytes, 1); 1738 1739 return edesc; 1740 } 1741 1742 static int skcipher_do_one_req(struct crypto_engine *engine, void *areq) 1743 { 1744 struct skcipher_request *req = skcipher_request_cast(areq); 1745 struct caam_ctx *ctx = crypto_skcipher_ctx(crypto_skcipher_reqtfm(req)); 1746 struct caam_skcipher_req_ctx *rctx = skcipher_request_ctx(req); 1747 u32 *desc = rctx->edesc->hw_desc; 1748 int ret; 1749 1750 rctx->edesc->bklog = true; 1751 1752 ret = caam_jr_enqueue(ctx->jrdev, desc, skcipher_crypt_done, req); 1753 1754 if (ret != -EINPROGRESS) { 1755 skcipher_unmap(ctx->jrdev, rctx->edesc, req); 1756 kfree(rctx->edesc); 1757 } else { 1758 ret = 0; 1759 } 1760 1761 return ret; 1762 } 1763 1764 static inline int skcipher_crypt(struct skcipher_request *req, bool encrypt) 1765 { 1766 struct skcipher_edesc *edesc; 1767 struct crypto_skcipher *skcipher = crypto_skcipher_reqtfm(req); 1768 struct caam_ctx *ctx = crypto_skcipher_ctx(skcipher); 1769 struct device *jrdev = ctx->jrdev; 1770 struct caam_drv_private_jr *jrpriv = dev_get_drvdata(jrdev); 1771 u32 *desc; 1772 int ret = 0; 1773 1774 if (!req->cryptlen) 1775 return 0; 1776 1777 /* allocate extended descriptor */ 1778 edesc = skcipher_edesc_alloc(req, DESC_JOB_IO_LEN * CAAM_CMD_SZ); 1779 if (IS_ERR(edesc)) 1780 return PTR_ERR(edesc); 1781 1782 /* Create and submit job descriptor*/ 1783 init_skcipher_job(req, edesc, encrypt); 1784 1785 print_hex_dump_debug("skcipher jobdesc@" __stringify(__LINE__)": ", 1786 DUMP_PREFIX_ADDRESS, 16, 4, edesc->hw_desc, 1787 desc_bytes(edesc->hw_desc), 1); 1788 1789 desc = edesc->hw_desc; 1790 /* 1791 * Only the backlog request are sent to crypto-engine since the others 1792 * can be handled by CAAM, if free, especially since JR has up to 1024 1793 * entries (more than the 10 entries from crypto-engine). 1794 */ 1795 if (req->base.flags & CRYPTO_TFM_REQ_MAY_BACKLOG) 1796 ret = crypto_transfer_skcipher_request_to_engine(jrpriv->engine, 1797 req); 1798 else 1799 ret = caam_jr_enqueue(jrdev, desc, skcipher_crypt_done, req); 1800 1801 if ((ret != -EINPROGRESS) && (ret != -EBUSY)) { 1802 skcipher_unmap(jrdev, edesc, req); 1803 kfree(edesc); 1804 } 1805 1806 return ret; 1807 } 1808 1809 static int skcipher_encrypt(struct skcipher_request *req) 1810 { 1811 return skcipher_crypt(req, true); 1812 } 1813 1814 static int skcipher_decrypt(struct skcipher_request *req) 1815 { 1816 return skcipher_crypt(req, false); 1817 } 1818 1819 static struct caam_skcipher_alg driver_algs[] = { 1820 { 1821 .skcipher = { 1822 .base = { 1823 .cra_name = "cbc(aes)", 1824 .cra_driver_name = "cbc-aes-caam", 1825 .cra_blocksize = AES_BLOCK_SIZE, 1826 }, 1827 .setkey = aes_skcipher_setkey, 1828 .encrypt = skcipher_encrypt, 1829 .decrypt = skcipher_decrypt, 1830 .min_keysize = AES_MIN_KEY_SIZE, 1831 .max_keysize = AES_MAX_KEY_SIZE, 1832 .ivsize = AES_BLOCK_SIZE, 1833 }, 1834 .caam.class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CBC, 1835 }, 1836 { 1837 .skcipher = { 1838 .base = { 1839 .cra_name = "cbc(des3_ede)", 1840 .cra_driver_name = "cbc-3des-caam", 1841 .cra_blocksize = DES3_EDE_BLOCK_SIZE, 1842 }, 1843 .setkey = des3_skcipher_setkey, 1844 .encrypt = skcipher_encrypt, 1845 .decrypt = skcipher_decrypt, 1846 .min_keysize = DES3_EDE_KEY_SIZE, 1847 .max_keysize = DES3_EDE_KEY_SIZE, 1848 .ivsize = DES3_EDE_BLOCK_SIZE, 1849 }, 1850 .caam.class1_alg_type = OP_ALG_ALGSEL_3DES | OP_ALG_AAI_CBC, 1851 }, 1852 { 1853 .skcipher = { 1854 .base = { 1855 .cra_name = "cbc(des)", 1856 .cra_driver_name = "cbc-des-caam", 1857 .cra_blocksize = DES_BLOCK_SIZE, 1858 }, 1859 .setkey = des_skcipher_setkey, 1860 .encrypt = skcipher_encrypt, 1861 .decrypt = skcipher_decrypt, 1862 .min_keysize = DES_KEY_SIZE, 1863 .max_keysize = DES_KEY_SIZE, 1864 .ivsize = DES_BLOCK_SIZE, 1865 }, 1866 .caam.class1_alg_type = OP_ALG_ALGSEL_DES | OP_ALG_AAI_CBC, 1867 }, 1868 { 1869 .skcipher = { 1870 .base = { 1871 .cra_name = "ctr(aes)", 1872 .cra_driver_name = "ctr-aes-caam", 1873 .cra_blocksize = 1, 1874 }, 1875 .setkey = ctr_skcipher_setkey, 1876 .encrypt = skcipher_encrypt, 1877 .decrypt = skcipher_decrypt, 1878 .min_keysize = AES_MIN_KEY_SIZE, 1879 .max_keysize = AES_MAX_KEY_SIZE, 1880 .ivsize = AES_BLOCK_SIZE, 1881 .chunksize = AES_BLOCK_SIZE, 1882 }, 1883 .caam.class1_alg_type = OP_ALG_ALGSEL_AES | 1884 OP_ALG_AAI_CTR_MOD128, 1885 }, 1886 { 1887 .skcipher = { 1888 .base = { 1889 .cra_name = "rfc3686(ctr(aes))", 1890 .cra_driver_name = "rfc3686-ctr-aes-caam", 1891 .cra_blocksize = 1, 1892 }, 1893 .setkey = rfc3686_skcipher_setkey, 1894 .encrypt = skcipher_encrypt, 1895 .decrypt = skcipher_decrypt, 1896 .min_keysize = AES_MIN_KEY_SIZE + 1897 CTR_RFC3686_NONCE_SIZE, 1898 .max_keysize = AES_MAX_KEY_SIZE + 1899 CTR_RFC3686_NONCE_SIZE, 1900 .ivsize = CTR_RFC3686_IV_SIZE, 1901 .chunksize = AES_BLOCK_SIZE, 1902 }, 1903 .caam = { 1904 .class1_alg_type = OP_ALG_ALGSEL_AES | 1905 OP_ALG_AAI_CTR_MOD128, 1906 .rfc3686 = true, 1907 }, 1908 }, 1909 { 1910 .skcipher = { 1911 .base = { 1912 .cra_name = "xts(aes)", 1913 .cra_driver_name = "xts-aes-caam", 1914 .cra_blocksize = AES_BLOCK_SIZE, 1915 }, 1916 .setkey = xts_skcipher_setkey, 1917 .encrypt = skcipher_encrypt, 1918 .decrypt = skcipher_decrypt, 1919 .min_keysize = 2 * AES_MIN_KEY_SIZE, 1920 .max_keysize = 2 * AES_MAX_KEY_SIZE, 1921 .ivsize = AES_BLOCK_SIZE, 1922 }, 1923 .caam.class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_XTS, 1924 }, 1925 { 1926 .skcipher = { 1927 .base = { 1928 .cra_name = "ecb(des)", 1929 .cra_driver_name = "ecb-des-caam", 1930 .cra_blocksize = DES_BLOCK_SIZE, 1931 }, 1932 .setkey = des_skcipher_setkey, 1933 .encrypt = skcipher_encrypt, 1934 .decrypt = skcipher_decrypt, 1935 .min_keysize = DES_KEY_SIZE, 1936 .max_keysize = DES_KEY_SIZE, 1937 }, 1938 .caam.class1_alg_type = OP_ALG_ALGSEL_DES | OP_ALG_AAI_ECB, 1939 }, 1940 { 1941 .skcipher = { 1942 .base = { 1943 .cra_name = "ecb(aes)", 1944 .cra_driver_name = "ecb-aes-caam", 1945 .cra_blocksize = AES_BLOCK_SIZE, 1946 }, 1947 .setkey = aes_skcipher_setkey, 1948 .encrypt = skcipher_encrypt, 1949 .decrypt = skcipher_decrypt, 1950 .min_keysize = AES_MIN_KEY_SIZE, 1951 .max_keysize = AES_MAX_KEY_SIZE, 1952 }, 1953 .caam.class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_ECB, 1954 }, 1955 { 1956 .skcipher = { 1957 .base = { 1958 .cra_name = "ecb(des3_ede)", 1959 .cra_driver_name = "ecb-des3-caam", 1960 .cra_blocksize = DES3_EDE_BLOCK_SIZE, 1961 }, 1962 .setkey = des3_skcipher_setkey, 1963 .encrypt = skcipher_encrypt, 1964 .decrypt = skcipher_decrypt, 1965 .min_keysize = DES3_EDE_KEY_SIZE, 1966 .max_keysize = DES3_EDE_KEY_SIZE, 1967 }, 1968 .caam.class1_alg_type = OP_ALG_ALGSEL_3DES | OP_ALG_AAI_ECB, 1969 }, 1970 { 1971 .skcipher = { 1972 .base = { 1973 .cra_name = "ecb(arc4)", 1974 .cra_driver_name = "ecb-arc4-caam", 1975 .cra_blocksize = ARC4_BLOCK_SIZE, 1976 }, 1977 .setkey = arc4_skcipher_setkey, 1978 .encrypt = skcipher_encrypt, 1979 .decrypt = skcipher_decrypt, 1980 .min_keysize = ARC4_MIN_KEY_SIZE, 1981 .max_keysize = ARC4_MAX_KEY_SIZE, 1982 }, 1983 .caam.class1_alg_type = OP_ALG_ALGSEL_ARC4 | OP_ALG_AAI_ECB, 1984 }, 1985 }; 1986 1987 static struct caam_aead_alg driver_aeads[] = { 1988 { 1989 .aead = { 1990 .base = { 1991 .cra_name = "rfc4106(gcm(aes))", 1992 .cra_driver_name = "rfc4106-gcm-aes-caam", 1993 .cra_blocksize = 1, 1994 }, 1995 .setkey = rfc4106_setkey, 1996 .setauthsize = rfc4106_setauthsize, 1997 .encrypt = ipsec_gcm_encrypt, 1998 .decrypt = ipsec_gcm_decrypt, 1999 .ivsize = GCM_RFC4106_IV_SIZE, 2000 .maxauthsize = AES_BLOCK_SIZE, 2001 }, 2002 .caam = { 2003 .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_GCM, 2004 .nodkp = true, 2005 }, 2006 }, 2007 { 2008 .aead = { 2009 .base = { 2010 .cra_name = "rfc4543(gcm(aes))", 2011 .cra_driver_name = "rfc4543-gcm-aes-caam", 2012 .cra_blocksize = 1, 2013 }, 2014 .setkey = rfc4543_setkey, 2015 .setauthsize = rfc4543_setauthsize, 2016 .encrypt = ipsec_gcm_encrypt, 2017 .decrypt = ipsec_gcm_decrypt, 2018 .ivsize = GCM_RFC4543_IV_SIZE, 2019 .maxauthsize = AES_BLOCK_SIZE, 2020 }, 2021 .caam = { 2022 .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_GCM, 2023 .nodkp = true, 2024 }, 2025 }, 2026 /* Galois Counter Mode */ 2027 { 2028 .aead = { 2029 .base = { 2030 .cra_name = "gcm(aes)", 2031 .cra_driver_name = "gcm-aes-caam", 2032 .cra_blocksize = 1, 2033 }, 2034 .setkey = gcm_setkey, 2035 .setauthsize = gcm_setauthsize, 2036 .encrypt = gcm_encrypt, 2037 .decrypt = gcm_decrypt, 2038 .ivsize = GCM_AES_IV_SIZE, 2039 .maxauthsize = AES_BLOCK_SIZE, 2040 }, 2041 .caam = { 2042 .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_GCM, 2043 .nodkp = true, 2044 }, 2045 }, 2046 /* single-pass ipsec_esp descriptor */ 2047 { 2048 .aead = { 2049 .base = { 2050 .cra_name = "authenc(hmac(md5)," 2051 "ecb(cipher_null))", 2052 .cra_driver_name = "authenc-hmac-md5-" 2053 "ecb-cipher_null-caam", 2054 .cra_blocksize = NULL_BLOCK_SIZE, 2055 }, 2056 .setkey = aead_setkey, 2057 .setauthsize = aead_setauthsize, 2058 .encrypt = aead_encrypt, 2059 .decrypt = aead_decrypt, 2060 .ivsize = NULL_IV_SIZE, 2061 .maxauthsize = MD5_DIGEST_SIZE, 2062 }, 2063 .caam = { 2064 .class2_alg_type = OP_ALG_ALGSEL_MD5 | 2065 OP_ALG_AAI_HMAC_PRECOMP, 2066 }, 2067 }, 2068 { 2069 .aead = { 2070 .base = { 2071 .cra_name = "authenc(hmac(sha1)," 2072 "ecb(cipher_null))", 2073 .cra_driver_name = "authenc-hmac-sha1-" 2074 "ecb-cipher_null-caam", 2075 .cra_blocksize = NULL_BLOCK_SIZE, 2076 }, 2077 .setkey = aead_setkey, 2078 .setauthsize = aead_setauthsize, 2079 .encrypt = aead_encrypt, 2080 .decrypt = aead_decrypt, 2081 .ivsize = NULL_IV_SIZE, 2082 .maxauthsize = SHA1_DIGEST_SIZE, 2083 }, 2084 .caam = { 2085 .class2_alg_type = OP_ALG_ALGSEL_SHA1 | 2086 OP_ALG_AAI_HMAC_PRECOMP, 2087 }, 2088 }, 2089 { 2090 .aead = { 2091 .base = { 2092 .cra_name = "authenc(hmac(sha224)," 2093 "ecb(cipher_null))", 2094 .cra_driver_name = "authenc-hmac-sha224-" 2095 "ecb-cipher_null-caam", 2096 .cra_blocksize = NULL_BLOCK_SIZE, 2097 }, 2098 .setkey = aead_setkey, 2099 .setauthsize = aead_setauthsize, 2100 .encrypt = aead_encrypt, 2101 .decrypt = aead_decrypt, 2102 .ivsize = NULL_IV_SIZE, 2103 .maxauthsize = SHA224_DIGEST_SIZE, 2104 }, 2105 .caam = { 2106 .class2_alg_type = OP_ALG_ALGSEL_SHA224 | 2107 OP_ALG_AAI_HMAC_PRECOMP, 2108 }, 2109 }, 2110 { 2111 .aead = { 2112 .base = { 2113 .cra_name = "authenc(hmac(sha256)," 2114 "ecb(cipher_null))", 2115 .cra_driver_name = "authenc-hmac-sha256-" 2116 "ecb-cipher_null-caam", 2117 .cra_blocksize = NULL_BLOCK_SIZE, 2118 }, 2119 .setkey = aead_setkey, 2120 .setauthsize = aead_setauthsize, 2121 .encrypt = aead_encrypt, 2122 .decrypt = aead_decrypt, 2123 .ivsize = NULL_IV_SIZE, 2124 .maxauthsize = SHA256_DIGEST_SIZE, 2125 }, 2126 .caam = { 2127 .class2_alg_type = OP_ALG_ALGSEL_SHA256 | 2128 OP_ALG_AAI_HMAC_PRECOMP, 2129 }, 2130 }, 2131 { 2132 .aead = { 2133 .base = { 2134 .cra_name = "authenc(hmac(sha384)," 2135 "ecb(cipher_null))", 2136 .cra_driver_name = "authenc-hmac-sha384-" 2137 "ecb-cipher_null-caam", 2138 .cra_blocksize = NULL_BLOCK_SIZE, 2139 }, 2140 .setkey = aead_setkey, 2141 .setauthsize = aead_setauthsize, 2142 .encrypt = aead_encrypt, 2143 .decrypt = aead_decrypt, 2144 .ivsize = NULL_IV_SIZE, 2145 .maxauthsize = SHA384_DIGEST_SIZE, 2146 }, 2147 .caam = { 2148 .class2_alg_type = OP_ALG_ALGSEL_SHA384 | 2149 OP_ALG_AAI_HMAC_PRECOMP, 2150 }, 2151 }, 2152 { 2153 .aead = { 2154 .base = { 2155 .cra_name = "authenc(hmac(sha512)," 2156 "ecb(cipher_null))", 2157 .cra_driver_name = "authenc-hmac-sha512-" 2158 "ecb-cipher_null-caam", 2159 .cra_blocksize = NULL_BLOCK_SIZE, 2160 }, 2161 .setkey = aead_setkey, 2162 .setauthsize = aead_setauthsize, 2163 .encrypt = aead_encrypt, 2164 .decrypt = aead_decrypt, 2165 .ivsize = NULL_IV_SIZE, 2166 .maxauthsize = SHA512_DIGEST_SIZE, 2167 }, 2168 .caam = { 2169 .class2_alg_type = OP_ALG_ALGSEL_SHA512 | 2170 OP_ALG_AAI_HMAC_PRECOMP, 2171 }, 2172 }, 2173 { 2174 .aead = { 2175 .base = { 2176 .cra_name = "authenc(hmac(md5),cbc(aes))", 2177 .cra_driver_name = "authenc-hmac-md5-" 2178 "cbc-aes-caam", 2179 .cra_blocksize = AES_BLOCK_SIZE, 2180 }, 2181 .setkey = aead_setkey, 2182 .setauthsize = aead_setauthsize, 2183 .encrypt = aead_encrypt, 2184 .decrypt = aead_decrypt, 2185 .ivsize = AES_BLOCK_SIZE, 2186 .maxauthsize = MD5_DIGEST_SIZE, 2187 }, 2188 .caam = { 2189 .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CBC, 2190 .class2_alg_type = OP_ALG_ALGSEL_MD5 | 2191 OP_ALG_AAI_HMAC_PRECOMP, 2192 }, 2193 }, 2194 { 2195 .aead = { 2196 .base = { 2197 .cra_name = "echainiv(authenc(hmac(md5)," 2198 "cbc(aes)))", 2199 .cra_driver_name = "echainiv-authenc-hmac-md5-" 2200 "cbc-aes-caam", 2201 .cra_blocksize = AES_BLOCK_SIZE, 2202 }, 2203 .setkey = aead_setkey, 2204 .setauthsize = aead_setauthsize, 2205 .encrypt = aead_encrypt, 2206 .decrypt = aead_decrypt, 2207 .ivsize = AES_BLOCK_SIZE, 2208 .maxauthsize = MD5_DIGEST_SIZE, 2209 }, 2210 .caam = { 2211 .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CBC, 2212 .class2_alg_type = OP_ALG_ALGSEL_MD5 | 2213 OP_ALG_AAI_HMAC_PRECOMP, 2214 .geniv = true, 2215 }, 2216 }, 2217 { 2218 .aead = { 2219 .base = { 2220 .cra_name = "authenc(hmac(sha1),cbc(aes))", 2221 .cra_driver_name = "authenc-hmac-sha1-" 2222 "cbc-aes-caam", 2223 .cra_blocksize = AES_BLOCK_SIZE, 2224 }, 2225 .setkey = aead_setkey, 2226 .setauthsize = aead_setauthsize, 2227 .encrypt = aead_encrypt, 2228 .decrypt = aead_decrypt, 2229 .ivsize = AES_BLOCK_SIZE, 2230 .maxauthsize = SHA1_DIGEST_SIZE, 2231 }, 2232 .caam = { 2233 .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CBC, 2234 .class2_alg_type = OP_ALG_ALGSEL_SHA1 | 2235 OP_ALG_AAI_HMAC_PRECOMP, 2236 }, 2237 }, 2238 { 2239 .aead = { 2240 .base = { 2241 .cra_name = "echainiv(authenc(hmac(sha1)," 2242 "cbc(aes)))", 2243 .cra_driver_name = "echainiv-authenc-" 2244 "hmac-sha1-cbc-aes-caam", 2245 .cra_blocksize = AES_BLOCK_SIZE, 2246 }, 2247 .setkey = aead_setkey, 2248 .setauthsize = aead_setauthsize, 2249 .encrypt = aead_encrypt, 2250 .decrypt = aead_decrypt, 2251 .ivsize = AES_BLOCK_SIZE, 2252 .maxauthsize = SHA1_DIGEST_SIZE, 2253 }, 2254 .caam = { 2255 .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CBC, 2256 .class2_alg_type = OP_ALG_ALGSEL_SHA1 | 2257 OP_ALG_AAI_HMAC_PRECOMP, 2258 .geniv = true, 2259 }, 2260 }, 2261 { 2262 .aead = { 2263 .base = { 2264 .cra_name = "authenc(hmac(sha224),cbc(aes))", 2265 .cra_driver_name = "authenc-hmac-sha224-" 2266 "cbc-aes-caam", 2267 .cra_blocksize = AES_BLOCK_SIZE, 2268 }, 2269 .setkey = aead_setkey, 2270 .setauthsize = aead_setauthsize, 2271 .encrypt = aead_encrypt, 2272 .decrypt = aead_decrypt, 2273 .ivsize = AES_BLOCK_SIZE, 2274 .maxauthsize = SHA224_DIGEST_SIZE, 2275 }, 2276 .caam = { 2277 .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CBC, 2278 .class2_alg_type = OP_ALG_ALGSEL_SHA224 | 2279 OP_ALG_AAI_HMAC_PRECOMP, 2280 }, 2281 }, 2282 { 2283 .aead = { 2284 .base = { 2285 .cra_name = "echainiv(authenc(hmac(sha224)," 2286 "cbc(aes)))", 2287 .cra_driver_name = "echainiv-authenc-" 2288 "hmac-sha224-cbc-aes-caam", 2289 .cra_blocksize = AES_BLOCK_SIZE, 2290 }, 2291 .setkey = aead_setkey, 2292 .setauthsize = aead_setauthsize, 2293 .encrypt = aead_encrypt, 2294 .decrypt = aead_decrypt, 2295 .ivsize = AES_BLOCK_SIZE, 2296 .maxauthsize = SHA224_DIGEST_SIZE, 2297 }, 2298 .caam = { 2299 .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CBC, 2300 .class2_alg_type = OP_ALG_ALGSEL_SHA224 | 2301 OP_ALG_AAI_HMAC_PRECOMP, 2302 .geniv = true, 2303 }, 2304 }, 2305 { 2306 .aead = { 2307 .base = { 2308 .cra_name = "authenc(hmac(sha256),cbc(aes))", 2309 .cra_driver_name = "authenc-hmac-sha256-" 2310 "cbc-aes-caam", 2311 .cra_blocksize = AES_BLOCK_SIZE, 2312 }, 2313 .setkey = aead_setkey, 2314 .setauthsize = aead_setauthsize, 2315 .encrypt = aead_encrypt, 2316 .decrypt = aead_decrypt, 2317 .ivsize = AES_BLOCK_SIZE, 2318 .maxauthsize = SHA256_DIGEST_SIZE, 2319 }, 2320 .caam = { 2321 .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CBC, 2322 .class2_alg_type = OP_ALG_ALGSEL_SHA256 | 2323 OP_ALG_AAI_HMAC_PRECOMP, 2324 }, 2325 }, 2326 { 2327 .aead = { 2328 .base = { 2329 .cra_name = "echainiv(authenc(hmac(sha256)," 2330 "cbc(aes)))", 2331 .cra_driver_name = "echainiv-authenc-" 2332 "hmac-sha256-cbc-aes-caam", 2333 .cra_blocksize = AES_BLOCK_SIZE, 2334 }, 2335 .setkey = aead_setkey, 2336 .setauthsize = aead_setauthsize, 2337 .encrypt = aead_encrypt, 2338 .decrypt = aead_decrypt, 2339 .ivsize = AES_BLOCK_SIZE, 2340 .maxauthsize = SHA256_DIGEST_SIZE, 2341 }, 2342 .caam = { 2343 .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CBC, 2344 .class2_alg_type = OP_ALG_ALGSEL_SHA256 | 2345 OP_ALG_AAI_HMAC_PRECOMP, 2346 .geniv = true, 2347 }, 2348 }, 2349 { 2350 .aead = { 2351 .base = { 2352 .cra_name = "authenc(hmac(sha384),cbc(aes))", 2353 .cra_driver_name = "authenc-hmac-sha384-" 2354 "cbc-aes-caam", 2355 .cra_blocksize = AES_BLOCK_SIZE, 2356 }, 2357 .setkey = aead_setkey, 2358 .setauthsize = aead_setauthsize, 2359 .encrypt = aead_encrypt, 2360 .decrypt = aead_decrypt, 2361 .ivsize = AES_BLOCK_SIZE, 2362 .maxauthsize = SHA384_DIGEST_SIZE, 2363 }, 2364 .caam = { 2365 .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CBC, 2366 .class2_alg_type = OP_ALG_ALGSEL_SHA384 | 2367 OP_ALG_AAI_HMAC_PRECOMP, 2368 }, 2369 }, 2370 { 2371 .aead = { 2372 .base = { 2373 .cra_name = "echainiv(authenc(hmac(sha384)," 2374 "cbc(aes)))", 2375 .cra_driver_name = "echainiv-authenc-" 2376 "hmac-sha384-cbc-aes-caam", 2377 .cra_blocksize = AES_BLOCK_SIZE, 2378 }, 2379 .setkey = aead_setkey, 2380 .setauthsize = aead_setauthsize, 2381 .encrypt = aead_encrypt, 2382 .decrypt = aead_decrypt, 2383 .ivsize = AES_BLOCK_SIZE, 2384 .maxauthsize = SHA384_DIGEST_SIZE, 2385 }, 2386 .caam = { 2387 .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CBC, 2388 .class2_alg_type = OP_ALG_ALGSEL_SHA384 | 2389 OP_ALG_AAI_HMAC_PRECOMP, 2390 .geniv = true, 2391 }, 2392 }, 2393 { 2394 .aead = { 2395 .base = { 2396 .cra_name = "authenc(hmac(sha512),cbc(aes))", 2397 .cra_driver_name = "authenc-hmac-sha512-" 2398 "cbc-aes-caam", 2399 .cra_blocksize = AES_BLOCK_SIZE, 2400 }, 2401 .setkey = aead_setkey, 2402 .setauthsize = aead_setauthsize, 2403 .encrypt = aead_encrypt, 2404 .decrypt = aead_decrypt, 2405 .ivsize = AES_BLOCK_SIZE, 2406 .maxauthsize = SHA512_DIGEST_SIZE, 2407 }, 2408 .caam = { 2409 .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CBC, 2410 .class2_alg_type = OP_ALG_ALGSEL_SHA512 | 2411 OP_ALG_AAI_HMAC_PRECOMP, 2412 }, 2413 }, 2414 { 2415 .aead = { 2416 .base = { 2417 .cra_name = "echainiv(authenc(hmac(sha512)," 2418 "cbc(aes)))", 2419 .cra_driver_name = "echainiv-authenc-" 2420 "hmac-sha512-cbc-aes-caam", 2421 .cra_blocksize = AES_BLOCK_SIZE, 2422 }, 2423 .setkey = aead_setkey, 2424 .setauthsize = aead_setauthsize, 2425 .encrypt = aead_encrypt, 2426 .decrypt = aead_decrypt, 2427 .ivsize = AES_BLOCK_SIZE, 2428 .maxauthsize = SHA512_DIGEST_SIZE, 2429 }, 2430 .caam = { 2431 .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CBC, 2432 .class2_alg_type = OP_ALG_ALGSEL_SHA512 | 2433 OP_ALG_AAI_HMAC_PRECOMP, 2434 .geniv = true, 2435 }, 2436 }, 2437 { 2438 .aead = { 2439 .base = { 2440 .cra_name = "authenc(hmac(md5),cbc(des3_ede))", 2441 .cra_driver_name = "authenc-hmac-md5-" 2442 "cbc-des3_ede-caam", 2443 .cra_blocksize = DES3_EDE_BLOCK_SIZE, 2444 }, 2445 .setkey = des3_aead_setkey, 2446 .setauthsize = aead_setauthsize, 2447 .encrypt = aead_encrypt, 2448 .decrypt = aead_decrypt, 2449 .ivsize = DES3_EDE_BLOCK_SIZE, 2450 .maxauthsize = MD5_DIGEST_SIZE, 2451 }, 2452 .caam = { 2453 .class1_alg_type = OP_ALG_ALGSEL_3DES | OP_ALG_AAI_CBC, 2454 .class2_alg_type = OP_ALG_ALGSEL_MD5 | 2455 OP_ALG_AAI_HMAC_PRECOMP, 2456 } 2457 }, 2458 { 2459 .aead = { 2460 .base = { 2461 .cra_name = "echainiv(authenc(hmac(md5)," 2462 "cbc(des3_ede)))", 2463 .cra_driver_name = "echainiv-authenc-hmac-md5-" 2464 "cbc-des3_ede-caam", 2465 .cra_blocksize = DES3_EDE_BLOCK_SIZE, 2466 }, 2467 .setkey = des3_aead_setkey, 2468 .setauthsize = aead_setauthsize, 2469 .encrypt = aead_encrypt, 2470 .decrypt = aead_decrypt, 2471 .ivsize = DES3_EDE_BLOCK_SIZE, 2472 .maxauthsize = MD5_DIGEST_SIZE, 2473 }, 2474 .caam = { 2475 .class1_alg_type = OP_ALG_ALGSEL_3DES | OP_ALG_AAI_CBC, 2476 .class2_alg_type = OP_ALG_ALGSEL_MD5 | 2477 OP_ALG_AAI_HMAC_PRECOMP, 2478 .geniv = true, 2479 } 2480 }, 2481 { 2482 .aead = { 2483 .base = { 2484 .cra_name = "authenc(hmac(sha1)," 2485 "cbc(des3_ede))", 2486 .cra_driver_name = "authenc-hmac-sha1-" 2487 "cbc-des3_ede-caam", 2488 .cra_blocksize = DES3_EDE_BLOCK_SIZE, 2489 }, 2490 .setkey = des3_aead_setkey, 2491 .setauthsize = aead_setauthsize, 2492 .encrypt = aead_encrypt, 2493 .decrypt = aead_decrypt, 2494 .ivsize = DES3_EDE_BLOCK_SIZE, 2495 .maxauthsize = SHA1_DIGEST_SIZE, 2496 }, 2497 .caam = { 2498 .class1_alg_type = OP_ALG_ALGSEL_3DES | OP_ALG_AAI_CBC, 2499 .class2_alg_type = OP_ALG_ALGSEL_SHA1 | 2500 OP_ALG_AAI_HMAC_PRECOMP, 2501 }, 2502 }, 2503 { 2504 .aead = { 2505 .base = { 2506 .cra_name = "echainiv(authenc(hmac(sha1)," 2507 "cbc(des3_ede)))", 2508 .cra_driver_name = "echainiv-authenc-" 2509 "hmac-sha1-" 2510 "cbc-des3_ede-caam", 2511 .cra_blocksize = DES3_EDE_BLOCK_SIZE, 2512 }, 2513 .setkey = des3_aead_setkey, 2514 .setauthsize = aead_setauthsize, 2515 .encrypt = aead_encrypt, 2516 .decrypt = aead_decrypt, 2517 .ivsize = DES3_EDE_BLOCK_SIZE, 2518 .maxauthsize = SHA1_DIGEST_SIZE, 2519 }, 2520 .caam = { 2521 .class1_alg_type = OP_ALG_ALGSEL_3DES | OP_ALG_AAI_CBC, 2522 .class2_alg_type = OP_ALG_ALGSEL_SHA1 | 2523 OP_ALG_AAI_HMAC_PRECOMP, 2524 .geniv = true, 2525 }, 2526 }, 2527 { 2528 .aead = { 2529 .base = { 2530 .cra_name = "authenc(hmac(sha224)," 2531 "cbc(des3_ede))", 2532 .cra_driver_name = "authenc-hmac-sha224-" 2533 "cbc-des3_ede-caam", 2534 .cra_blocksize = DES3_EDE_BLOCK_SIZE, 2535 }, 2536 .setkey = des3_aead_setkey, 2537 .setauthsize = aead_setauthsize, 2538 .encrypt = aead_encrypt, 2539 .decrypt = aead_decrypt, 2540 .ivsize = DES3_EDE_BLOCK_SIZE, 2541 .maxauthsize = SHA224_DIGEST_SIZE, 2542 }, 2543 .caam = { 2544 .class1_alg_type = OP_ALG_ALGSEL_3DES | OP_ALG_AAI_CBC, 2545 .class2_alg_type = OP_ALG_ALGSEL_SHA224 | 2546 OP_ALG_AAI_HMAC_PRECOMP, 2547 }, 2548 }, 2549 { 2550 .aead = { 2551 .base = { 2552 .cra_name = "echainiv(authenc(hmac(sha224)," 2553 "cbc(des3_ede)))", 2554 .cra_driver_name = "echainiv-authenc-" 2555 "hmac-sha224-" 2556 "cbc-des3_ede-caam", 2557 .cra_blocksize = DES3_EDE_BLOCK_SIZE, 2558 }, 2559 .setkey = des3_aead_setkey, 2560 .setauthsize = aead_setauthsize, 2561 .encrypt = aead_encrypt, 2562 .decrypt = aead_decrypt, 2563 .ivsize = DES3_EDE_BLOCK_SIZE, 2564 .maxauthsize = SHA224_DIGEST_SIZE, 2565 }, 2566 .caam = { 2567 .class1_alg_type = OP_ALG_ALGSEL_3DES | OP_ALG_AAI_CBC, 2568 .class2_alg_type = OP_ALG_ALGSEL_SHA224 | 2569 OP_ALG_AAI_HMAC_PRECOMP, 2570 .geniv = true, 2571 }, 2572 }, 2573 { 2574 .aead = { 2575 .base = { 2576 .cra_name = "authenc(hmac(sha256)," 2577 "cbc(des3_ede))", 2578 .cra_driver_name = "authenc-hmac-sha256-" 2579 "cbc-des3_ede-caam", 2580 .cra_blocksize = DES3_EDE_BLOCK_SIZE, 2581 }, 2582 .setkey = des3_aead_setkey, 2583 .setauthsize = aead_setauthsize, 2584 .encrypt = aead_encrypt, 2585 .decrypt = aead_decrypt, 2586 .ivsize = DES3_EDE_BLOCK_SIZE, 2587 .maxauthsize = SHA256_DIGEST_SIZE, 2588 }, 2589 .caam = { 2590 .class1_alg_type = OP_ALG_ALGSEL_3DES | OP_ALG_AAI_CBC, 2591 .class2_alg_type = OP_ALG_ALGSEL_SHA256 | 2592 OP_ALG_AAI_HMAC_PRECOMP, 2593 }, 2594 }, 2595 { 2596 .aead = { 2597 .base = { 2598 .cra_name = "echainiv(authenc(hmac(sha256)," 2599 "cbc(des3_ede)))", 2600 .cra_driver_name = "echainiv-authenc-" 2601 "hmac-sha256-" 2602 "cbc-des3_ede-caam", 2603 .cra_blocksize = DES3_EDE_BLOCK_SIZE, 2604 }, 2605 .setkey = des3_aead_setkey, 2606 .setauthsize = aead_setauthsize, 2607 .encrypt = aead_encrypt, 2608 .decrypt = aead_decrypt, 2609 .ivsize = DES3_EDE_BLOCK_SIZE, 2610 .maxauthsize = SHA256_DIGEST_SIZE, 2611 }, 2612 .caam = { 2613 .class1_alg_type = OP_ALG_ALGSEL_3DES | OP_ALG_AAI_CBC, 2614 .class2_alg_type = OP_ALG_ALGSEL_SHA256 | 2615 OP_ALG_AAI_HMAC_PRECOMP, 2616 .geniv = true, 2617 }, 2618 }, 2619 { 2620 .aead = { 2621 .base = { 2622 .cra_name = "authenc(hmac(sha384)," 2623 "cbc(des3_ede))", 2624 .cra_driver_name = "authenc-hmac-sha384-" 2625 "cbc-des3_ede-caam", 2626 .cra_blocksize = DES3_EDE_BLOCK_SIZE, 2627 }, 2628 .setkey = des3_aead_setkey, 2629 .setauthsize = aead_setauthsize, 2630 .encrypt = aead_encrypt, 2631 .decrypt = aead_decrypt, 2632 .ivsize = DES3_EDE_BLOCK_SIZE, 2633 .maxauthsize = SHA384_DIGEST_SIZE, 2634 }, 2635 .caam = { 2636 .class1_alg_type = OP_ALG_ALGSEL_3DES | OP_ALG_AAI_CBC, 2637 .class2_alg_type = OP_ALG_ALGSEL_SHA384 | 2638 OP_ALG_AAI_HMAC_PRECOMP, 2639 }, 2640 }, 2641 { 2642 .aead = { 2643 .base = { 2644 .cra_name = "echainiv(authenc(hmac(sha384)," 2645 "cbc(des3_ede)))", 2646 .cra_driver_name = "echainiv-authenc-" 2647 "hmac-sha384-" 2648 "cbc-des3_ede-caam", 2649 .cra_blocksize = DES3_EDE_BLOCK_SIZE, 2650 }, 2651 .setkey = des3_aead_setkey, 2652 .setauthsize = aead_setauthsize, 2653 .encrypt = aead_encrypt, 2654 .decrypt = aead_decrypt, 2655 .ivsize = DES3_EDE_BLOCK_SIZE, 2656 .maxauthsize = SHA384_DIGEST_SIZE, 2657 }, 2658 .caam = { 2659 .class1_alg_type = OP_ALG_ALGSEL_3DES | OP_ALG_AAI_CBC, 2660 .class2_alg_type = OP_ALG_ALGSEL_SHA384 | 2661 OP_ALG_AAI_HMAC_PRECOMP, 2662 .geniv = true, 2663 }, 2664 }, 2665 { 2666 .aead = { 2667 .base = { 2668 .cra_name = "authenc(hmac(sha512)," 2669 "cbc(des3_ede))", 2670 .cra_driver_name = "authenc-hmac-sha512-" 2671 "cbc-des3_ede-caam", 2672 .cra_blocksize = DES3_EDE_BLOCK_SIZE, 2673 }, 2674 .setkey = des3_aead_setkey, 2675 .setauthsize = aead_setauthsize, 2676 .encrypt = aead_encrypt, 2677 .decrypt = aead_decrypt, 2678 .ivsize = DES3_EDE_BLOCK_SIZE, 2679 .maxauthsize = SHA512_DIGEST_SIZE, 2680 }, 2681 .caam = { 2682 .class1_alg_type = OP_ALG_ALGSEL_3DES | OP_ALG_AAI_CBC, 2683 .class2_alg_type = OP_ALG_ALGSEL_SHA512 | 2684 OP_ALG_AAI_HMAC_PRECOMP, 2685 }, 2686 }, 2687 { 2688 .aead = { 2689 .base = { 2690 .cra_name = "echainiv(authenc(hmac(sha512)," 2691 "cbc(des3_ede)))", 2692 .cra_driver_name = "echainiv-authenc-" 2693 "hmac-sha512-" 2694 "cbc-des3_ede-caam", 2695 .cra_blocksize = DES3_EDE_BLOCK_SIZE, 2696 }, 2697 .setkey = des3_aead_setkey, 2698 .setauthsize = aead_setauthsize, 2699 .encrypt = aead_encrypt, 2700 .decrypt = aead_decrypt, 2701 .ivsize = DES3_EDE_BLOCK_SIZE, 2702 .maxauthsize = SHA512_DIGEST_SIZE, 2703 }, 2704 .caam = { 2705 .class1_alg_type = OP_ALG_ALGSEL_3DES | OP_ALG_AAI_CBC, 2706 .class2_alg_type = OP_ALG_ALGSEL_SHA512 | 2707 OP_ALG_AAI_HMAC_PRECOMP, 2708 .geniv = true, 2709 }, 2710 }, 2711 { 2712 .aead = { 2713 .base = { 2714 .cra_name = "authenc(hmac(md5),cbc(des))", 2715 .cra_driver_name = "authenc-hmac-md5-" 2716 "cbc-des-caam", 2717 .cra_blocksize = DES_BLOCK_SIZE, 2718 }, 2719 .setkey = aead_setkey, 2720 .setauthsize = aead_setauthsize, 2721 .encrypt = aead_encrypt, 2722 .decrypt = aead_decrypt, 2723 .ivsize = DES_BLOCK_SIZE, 2724 .maxauthsize = MD5_DIGEST_SIZE, 2725 }, 2726 .caam = { 2727 .class1_alg_type = OP_ALG_ALGSEL_DES | OP_ALG_AAI_CBC, 2728 .class2_alg_type = OP_ALG_ALGSEL_MD5 | 2729 OP_ALG_AAI_HMAC_PRECOMP, 2730 }, 2731 }, 2732 { 2733 .aead = { 2734 .base = { 2735 .cra_name = "echainiv(authenc(hmac(md5)," 2736 "cbc(des)))", 2737 .cra_driver_name = "echainiv-authenc-hmac-md5-" 2738 "cbc-des-caam", 2739 .cra_blocksize = DES_BLOCK_SIZE, 2740 }, 2741 .setkey = aead_setkey, 2742 .setauthsize = aead_setauthsize, 2743 .encrypt = aead_encrypt, 2744 .decrypt = aead_decrypt, 2745 .ivsize = DES_BLOCK_SIZE, 2746 .maxauthsize = MD5_DIGEST_SIZE, 2747 }, 2748 .caam = { 2749 .class1_alg_type = OP_ALG_ALGSEL_DES | OP_ALG_AAI_CBC, 2750 .class2_alg_type = OP_ALG_ALGSEL_MD5 | 2751 OP_ALG_AAI_HMAC_PRECOMP, 2752 .geniv = true, 2753 }, 2754 }, 2755 { 2756 .aead = { 2757 .base = { 2758 .cra_name = "authenc(hmac(sha1),cbc(des))", 2759 .cra_driver_name = "authenc-hmac-sha1-" 2760 "cbc-des-caam", 2761 .cra_blocksize = DES_BLOCK_SIZE, 2762 }, 2763 .setkey = aead_setkey, 2764 .setauthsize = aead_setauthsize, 2765 .encrypt = aead_encrypt, 2766 .decrypt = aead_decrypt, 2767 .ivsize = DES_BLOCK_SIZE, 2768 .maxauthsize = SHA1_DIGEST_SIZE, 2769 }, 2770 .caam = { 2771 .class1_alg_type = OP_ALG_ALGSEL_DES | OP_ALG_AAI_CBC, 2772 .class2_alg_type = OP_ALG_ALGSEL_SHA1 | 2773 OP_ALG_AAI_HMAC_PRECOMP, 2774 }, 2775 }, 2776 { 2777 .aead = { 2778 .base = { 2779 .cra_name = "echainiv(authenc(hmac(sha1)," 2780 "cbc(des)))", 2781 .cra_driver_name = "echainiv-authenc-" 2782 "hmac-sha1-cbc-des-caam", 2783 .cra_blocksize = DES_BLOCK_SIZE, 2784 }, 2785 .setkey = aead_setkey, 2786 .setauthsize = aead_setauthsize, 2787 .encrypt = aead_encrypt, 2788 .decrypt = aead_decrypt, 2789 .ivsize = DES_BLOCK_SIZE, 2790 .maxauthsize = SHA1_DIGEST_SIZE, 2791 }, 2792 .caam = { 2793 .class1_alg_type = OP_ALG_ALGSEL_DES | OP_ALG_AAI_CBC, 2794 .class2_alg_type = OP_ALG_ALGSEL_SHA1 | 2795 OP_ALG_AAI_HMAC_PRECOMP, 2796 .geniv = true, 2797 }, 2798 }, 2799 { 2800 .aead = { 2801 .base = { 2802 .cra_name = "authenc(hmac(sha224),cbc(des))", 2803 .cra_driver_name = "authenc-hmac-sha224-" 2804 "cbc-des-caam", 2805 .cra_blocksize = DES_BLOCK_SIZE, 2806 }, 2807 .setkey = aead_setkey, 2808 .setauthsize = aead_setauthsize, 2809 .encrypt = aead_encrypt, 2810 .decrypt = aead_decrypt, 2811 .ivsize = DES_BLOCK_SIZE, 2812 .maxauthsize = SHA224_DIGEST_SIZE, 2813 }, 2814 .caam = { 2815 .class1_alg_type = OP_ALG_ALGSEL_DES | OP_ALG_AAI_CBC, 2816 .class2_alg_type = OP_ALG_ALGSEL_SHA224 | 2817 OP_ALG_AAI_HMAC_PRECOMP, 2818 }, 2819 }, 2820 { 2821 .aead = { 2822 .base = { 2823 .cra_name = "echainiv(authenc(hmac(sha224)," 2824 "cbc(des)))", 2825 .cra_driver_name = "echainiv-authenc-" 2826 "hmac-sha224-cbc-des-caam", 2827 .cra_blocksize = DES_BLOCK_SIZE, 2828 }, 2829 .setkey = aead_setkey, 2830 .setauthsize = aead_setauthsize, 2831 .encrypt = aead_encrypt, 2832 .decrypt = aead_decrypt, 2833 .ivsize = DES_BLOCK_SIZE, 2834 .maxauthsize = SHA224_DIGEST_SIZE, 2835 }, 2836 .caam = { 2837 .class1_alg_type = OP_ALG_ALGSEL_DES | OP_ALG_AAI_CBC, 2838 .class2_alg_type = OP_ALG_ALGSEL_SHA224 | 2839 OP_ALG_AAI_HMAC_PRECOMP, 2840 .geniv = true, 2841 }, 2842 }, 2843 { 2844 .aead = { 2845 .base = { 2846 .cra_name = "authenc(hmac(sha256),cbc(des))", 2847 .cra_driver_name = "authenc-hmac-sha256-" 2848 "cbc-des-caam", 2849 .cra_blocksize = DES_BLOCK_SIZE, 2850 }, 2851 .setkey = aead_setkey, 2852 .setauthsize = aead_setauthsize, 2853 .encrypt = aead_encrypt, 2854 .decrypt = aead_decrypt, 2855 .ivsize = DES_BLOCK_SIZE, 2856 .maxauthsize = SHA256_DIGEST_SIZE, 2857 }, 2858 .caam = { 2859 .class1_alg_type = OP_ALG_ALGSEL_DES | OP_ALG_AAI_CBC, 2860 .class2_alg_type = OP_ALG_ALGSEL_SHA256 | 2861 OP_ALG_AAI_HMAC_PRECOMP, 2862 }, 2863 }, 2864 { 2865 .aead = { 2866 .base = { 2867 .cra_name = "echainiv(authenc(hmac(sha256)," 2868 "cbc(des)))", 2869 .cra_driver_name = "echainiv-authenc-" 2870 "hmac-sha256-cbc-des-caam", 2871 .cra_blocksize = DES_BLOCK_SIZE, 2872 }, 2873 .setkey = aead_setkey, 2874 .setauthsize = aead_setauthsize, 2875 .encrypt = aead_encrypt, 2876 .decrypt = aead_decrypt, 2877 .ivsize = DES_BLOCK_SIZE, 2878 .maxauthsize = SHA256_DIGEST_SIZE, 2879 }, 2880 .caam = { 2881 .class1_alg_type = OP_ALG_ALGSEL_DES | OP_ALG_AAI_CBC, 2882 .class2_alg_type = OP_ALG_ALGSEL_SHA256 | 2883 OP_ALG_AAI_HMAC_PRECOMP, 2884 .geniv = true, 2885 }, 2886 }, 2887 { 2888 .aead = { 2889 .base = { 2890 .cra_name = "authenc(hmac(sha384),cbc(des))", 2891 .cra_driver_name = "authenc-hmac-sha384-" 2892 "cbc-des-caam", 2893 .cra_blocksize = DES_BLOCK_SIZE, 2894 }, 2895 .setkey = aead_setkey, 2896 .setauthsize = aead_setauthsize, 2897 .encrypt = aead_encrypt, 2898 .decrypt = aead_decrypt, 2899 .ivsize = DES_BLOCK_SIZE, 2900 .maxauthsize = SHA384_DIGEST_SIZE, 2901 }, 2902 .caam = { 2903 .class1_alg_type = OP_ALG_ALGSEL_DES | OP_ALG_AAI_CBC, 2904 .class2_alg_type = OP_ALG_ALGSEL_SHA384 | 2905 OP_ALG_AAI_HMAC_PRECOMP, 2906 }, 2907 }, 2908 { 2909 .aead = { 2910 .base = { 2911 .cra_name = "echainiv(authenc(hmac(sha384)," 2912 "cbc(des)))", 2913 .cra_driver_name = "echainiv-authenc-" 2914 "hmac-sha384-cbc-des-caam", 2915 .cra_blocksize = DES_BLOCK_SIZE, 2916 }, 2917 .setkey = aead_setkey, 2918 .setauthsize = aead_setauthsize, 2919 .encrypt = aead_encrypt, 2920 .decrypt = aead_decrypt, 2921 .ivsize = DES_BLOCK_SIZE, 2922 .maxauthsize = SHA384_DIGEST_SIZE, 2923 }, 2924 .caam = { 2925 .class1_alg_type = OP_ALG_ALGSEL_DES | OP_ALG_AAI_CBC, 2926 .class2_alg_type = OP_ALG_ALGSEL_SHA384 | 2927 OP_ALG_AAI_HMAC_PRECOMP, 2928 .geniv = true, 2929 }, 2930 }, 2931 { 2932 .aead = { 2933 .base = { 2934 .cra_name = "authenc(hmac(sha512),cbc(des))", 2935 .cra_driver_name = "authenc-hmac-sha512-" 2936 "cbc-des-caam", 2937 .cra_blocksize = DES_BLOCK_SIZE, 2938 }, 2939 .setkey = aead_setkey, 2940 .setauthsize = aead_setauthsize, 2941 .encrypt = aead_encrypt, 2942 .decrypt = aead_decrypt, 2943 .ivsize = DES_BLOCK_SIZE, 2944 .maxauthsize = SHA512_DIGEST_SIZE, 2945 }, 2946 .caam = { 2947 .class1_alg_type = OP_ALG_ALGSEL_DES | OP_ALG_AAI_CBC, 2948 .class2_alg_type = OP_ALG_ALGSEL_SHA512 | 2949 OP_ALG_AAI_HMAC_PRECOMP, 2950 }, 2951 }, 2952 { 2953 .aead = { 2954 .base = { 2955 .cra_name = "echainiv(authenc(hmac(sha512)," 2956 "cbc(des)))", 2957 .cra_driver_name = "echainiv-authenc-" 2958 "hmac-sha512-cbc-des-caam", 2959 .cra_blocksize = DES_BLOCK_SIZE, 2960 }, 2961 .setkey = aead_setkey, 2962 .setauthsize = aead_setauthsize, 2963 .encrypt = aead_encrypt, 2964 .decrypt = aead_decrypt, 2965 .ivsize = DES_BLOCK_SIZE, 2966 .maxauthsize = SHA512_DIGEST_SIZE, 2967 }, 2968 .caam = { 2969 .class1_alg_type = OP_ALG_ALGSEL_DES | OP_ALG_AAI_CBC, 2970 .class2_alg_type = OP_ALG_ALGSEL_SHA512 | 2971 OP_ALG_AAI_HMAC_PRECOMP, 2972 .geniv = true, 2973 }, 2974 }, 2975 { 2976 .aead = { 2977 .base = { 2978 .cra_name = "authenc(hmac(md5)," 2979 "rfc3686(ctr(aes)))", 2980 .cra_driver_name = "authenc-hmac-md5-" 2981 "rfc3686-ctr-aes-caam", 2982 .cra_blocksize = 1, 2983 }, 2984 .setkey = aead_setkey, 2985 .setauthsize = aead_setauthsize, 2986 .encrypt = aead_encrypt, 2987 .decrypt = aead_decrypt, 2988 .ivsize = CTR_RFC3686_IV_SIZE, 2989 .maxauthsize = MD5_DIGEST_SIZE, 2990 }, 2991 .caam = { 2992 .class1_alg_type = OP_ALG_ALGSEL_AES | 2993 OP_ALG_AAI_CTR_MOD128, 2994 .class2_alg_type = OP_ALG_ALGSEL_MD5 | 2995 OP_ALG_AAI_HMAC_PRECOMP, 2996 .rfc3686 = true, 2997 }, 2998 }, 2999 { 3000 .aead = { 3001 .base = { 3002 .cra_name = "seqiv(authenc(" 3003 "hmac(md5),rfc3686(ctr(aes))))", 3004 .cra_driver_name = "seqiv-authenc-hmac-md5-" 3005 "rfc3686-ctr-aes-caam", 3006 .cra_blocksize = 1, 3007 }, 3008 .setkey = aead_setkey, 3009 .setauthsize = aead_setauthsize, 3010 .encrypt = aead_encrypt, 3011 .decrypt = aead_decrypt, 3012 .ivsize = CTR_RFC3686_IV_SIZE, 3013 .maxauthsize = MD5_DIGEST_SIZE, 3014 }, 3015 .caam = { 3016 .class1_alg_type = OP_ALG_ALGSEL_AES | 3017 OP_ALG_AAI_CTR_MOD128, 3018 .class2_alg_type = OP_ALG_ALGSEL_MD5 | 3019 OP_ALG_AAI_HMAC_PRECOMP, 3020 .rfc3686 = true, 3021 .geniv = true, 3022 }, 3023 }, 3024 { 3025 .aead = { 3026 .base = { 3027 .cra_name = "authenc(hmac(sha1)," 3028 "rfc3686(ctr(aes)))", 3029 .cra_driver_name = "authenc-hmac-sha1-" 3030 "rfc3686-ctr-aes-caam", 3031 .cra_blocksize = 1, 3032 }, 3033 .setkey = aead_setkey, 3034 .setauthsize = aead_setauthsize, 3035 .encrypt = aead_encrypt, 3036 .decrypt = aead_decrypt, 3037 .ivsize = CTR_RFC3686_IV_SIZE, 3038 .maxauthsize = SHA1_DIGEST_SIZE, 3039 }, 3040 .caam = { 3041 .class1_alg_type = OP_ALG_ALGSEL_AES | 3042 OP_ALG_AAI_CTR_MOD128, 3043 .class2_alg_type = OP_ALG_ALGSEL_SHA1 | 3044 OP_ALG_AAI_HMAC_PRECOMP, 3045 .rfc3686 = true, 3046 }, 3047 }, 3048 { 3049 .aead = { 3050 .base = { 3051 .cra_name = "seqiv(authenc(" 3052 "hmac(sha1),rfc3686(ctr(aes))))", 3053 .cra_driver_name = "seqiv-authenc-hmac-sha1-" 3054 "rfc3686-ctr-aes-caam", 3055 .cra_blocksize = 1, 3056 }, 3057 .setkey = aead_setkey, 3058 .setauthsize = aead_setauthsize, 3059 .encrypt = aead_encrypt, 3060 .decrypt = aead_decrypt, 3061 .ivsize = CTR_RFC3686_IV_SIZE, 3062 .maxauthsize = SHA1_DIGEST_SIZE, 3063 }, 3064 .caam = { 3065 .class1_alg_type = OP_ALG_ALGSEL_AES | 3066 OP_ALG_AAI_CTR_MOD128, 3067 .class2_alg_type = OP_ALG_ALGSEL_SHA1 | 3068 OP_ALG_AAI_HMAC_PRECOMP, 3069 .rfc3686 = true, 3070 .geniv = true, 3071 }, 3072 }, 3073 { 3074 .aead = { 3075 .base = { 3076 .cra_name = "authenc(hmac(sha224)," 3077 "rfc3686(ctr(aes)))", 3078 .cra_driver_name = "authenc-hmac-sha224-" 3079 "rfc3686-ctr-aes-caam", 3080 .cra_blocksize = 1, 3081 }, 3082 .setkey = aead_setkey, 3083 .setauthsize = aead_setauthsize, 3084 .encrypt = aead_encrypt, 3085 .decrypt = aead_decrypt, 3086 .ivsize = CTR_RFC3686_IV_SIZE, 3087 .maxauthsize = SHA224_DIGEST_SIZE, 3088 }, 3089 .caam = { 3090 .class1_alg_type = OP_ALG_ALGSEL_AES | 3091 OP_ALG_AAI_CTR_MOD128, 3092 .class2_alg_type = OP_ALG_ALGSEL_SHA224 | 3093 OP_ALG_AAI_HMAC_PRECOMP, 3094 .rfc3686 = true, 3095 }, 3096 }, 3097 { 3098 .aead = { 3099 .base = { 3100 .cra_name = "seqiv(authenc(" 3101 "hmac(sha224),rfc3686(ctr(aes))))", 3102 .cra_driver_name = "seqiv-authenc-hmac-sha224-" 3103 "rfc3686-ctr-aes-caam", 3104 .cra_blocksize = 1, 3105 }, 3106 .setkey = aead_setkey, 3107 .setauthsize = aead_setauthsize, 3108 .encrypt = aead_encrypt, 3109 .decrypt = aead_decrypt, 3110 .ivsize = CTR_RFC3686_IV_SIZE, 3111 .maxauthsize = SHA224_DIGEST_SIZE, 3112 }, 3113 .caam = { 3114 .class1_alg_type = OP_ALG_ALGSEL_AES | 3115 OP_ALG_AAI_CTR_MOD128, 3116 .class2_alg_type = OP_ALG_ALGSEL_SHA224 | 3117 OP_ALG_AAI_HMAC_PRECOMP, 3118 .rfc3686 = true, 3119 .geniv = true, 3120 }, 3121 }, 3122 { 3123 .aead = { 3124 .base = { 3125 .cra_name = "authenc(hmac(sha256)," 3126 "rfc3686(ctr(aes)))", 3127 .cra_driver_name = "authenc-hmac-sha256-" 3128 "rfc3686-ctr-aes-caam", 3129 .cra_blocksize = 1, 3130 }, 3131 .setkey = aead_setkey, 3132 .setauthsize = aead_setauthsize, 3133 .encrypt = aead_encrypt, 3134 .decrypt = aead_decrypt, 3135 .ivsize = CTR_RFC3686_IV_SIZE, 3136 .maxauthsize = SHA256_DIGEST_SIZE, 3137 }, 3138 .caam = { 3139 .class1_alg_type = OP_ALG_ALGSEL_AES | 3140 OP_ALG_AAI_CTR_MOD128, 3141 .class2_alg_type = OP_ALG_ALGSEL_SHA256 | 3142 OP_ALG_AAI_HMAC_PRECOMP, 3143 .rfc3686 = true, 3144 }, 3145 }, 3146 { 3147 .aead = { 3148 .base = { 3149 .cra_name = "seqiv(authenc(hmac(sha256)," 3150 "rfc3686(ctr(aes))))", 3151 .cra_driver_name = "seqiv-authenc-hmac-sha256-" 3152 "rfc3686-ctr-aes-caam", 3153 .cra_blocksize = 1, 3154 }, 3155 .setkey = aead_setkey, 3156 .setauthsize = aead_setauthsize, 3157 .encrypt = aead_encrypt, 3158 .decrypt = aead_decrypt, 3159 .ivsize = CTR_RFC3686_IV_SIZE, 3160 .maxauthsize = SHA256_DIGEST_SIZE, 3161 }, 3162 .caam = { 3163 .class1_alg_type = OP_ALG_ALGSEL_AES | 3164 OP_ALG_AAI_CTR_MOD128, 3165 .class2_alg_type = OP_ALG_ALGSEL_SHA256 | 3166 OP_ALG_AAI_HMAC_PRECOMP, 3167 .rfc3686 = true, 3168 .geniv = true, 3169 }, 3170 }, 3171 { 3172 .aead = { 3173 .base = { 3174 .cra_name = "authenc(hmac(sha384)," 3175 "rfc3686(ctr(aes)))", 3176 .cra_driver_name = "authenc-hmac-sha384-" 3177 "rfc3686-ctr-aes-caam", 3178 .cra_blocksize = 1, 3179 }, 3180 .setkey = aead_setkey, 3181 .setauthsize = aead_setauthsize, 3182 .encrypt = aead_encrypt, 3183 .decrypt = aead_decrypt, 3184 .ivsize = CTR_RFC3686_IV_SIZE, 3185 .maxauthsize = SHA384_DIGEST_SIZE, 3186 }, 3187 .caam = { 3188 .class1_alg_type = OP_ALG_ALGSEL_AES | 3189 OP_ALG_AAI_CTR_MOD128, 3190 .class2_alg_type = OP_ALG_ALGSEL_SHA384 | 3191 OP_ALG_AAI_HMAC_PRECOMP, 3192 .rfc3686 = true, 3193 }, 3194 }, 3195 { 3196 .aead = { 3197 .base = { 3198 .cra_name = "seqiv(authenc(hmac(sha384)," 3199 "rfc3686(ctr(aes))))", 3200 .cra_driver_name = "seqiv-authenc-hmac-sha384-" 3201 "rfc3686-ctr-aes-caam", 3202 .cra_blocksize = 1, 3203 }, 3204 .setkey = aead_setkey, 3205 .setauthsize = aead_setauthsize, 3206 .encrypt = aead_encrypt, 3207 .decrypt = aead_decrypt, 3208 .ivsize = CTR_RFC3686_IV_SIZE, 3209 .maxauthsize = SHA384_DIGEST_SIZE, 3210 }, 3211 .caam = { 3212 .class1_alg_type = OP_ALG_ALGSEL_AES | 3213 OP_ALG_AAI_CTR_MOD128, 3214 .class2_alg_type = OP_ALG_ALGSEL_SHA384 | 3215 OP_ALG_AAI_HMAC_PRECOMP, 3216 .rfc3686 = true, 3217 .geniv = true, 3218 }, 3219 }, 3220 { 3221 .aead = { 3222 .base = { 3223 .cra_name = "authenc(hmac(sha512)," 3224 "rfc3686(ctr(aes)))", 3225 .cra_driver_name = "authenc-hmac-sha512-" 3226 "rfc3686-ctr-aes-caam", 3227 .cra_blocksize = 1, 3228 }, 3229 .setkey = aead_setkey, 3230 .setauthsize = aead_setauthsize, 3231 .encrypt = aead_encrypt, 3232 .decrypt = aead_decrypt, 3233 .ivsize = CTR_RFC3686_IV_SIZE, 3234 .maxauthsize = SHA512_DIGEST_SIZE, 3235 }, 3236 .caam = { 3237 .class1_alg_type = OP_ALG_ALGSEL_AES | 3238 OP_ALG_AAI_CTR_MOD128, 3239 .class2_alg_type = OP_ALG_ALGSEL_SHA512 | 3240 OP_ALG_AAI_HMAC_PRECOMP, 3241 .rfc3686 = true, 3242 }, 3243 }, 3244 { 3245 .aead = { 3246 .base = { 3247 .cra_name = "seqiv(authenc(hmac(sha512)," 3248 "rfc3686(ctr(aes))))", 3249 .cra_driver_name = "seqiv-authenc-hmac-sha512-" 3250 "rfc3686-ctr-aes-caam", 3251 .cra_blocksize = 1, 3252 }, 3253 .setkey = aead_setkey, 3254 .setauthsize = aead_setauthsize, 3255 .encrypt = aead_encrypt, 3256 .decrypt = aead_decrypt, 3257 .ivsize = CTR_RFC3686_IV_SIZE, 3258 .maxauthsize = SHA512_DIGEST_SIZE, 3259 }, 3260 .caam = { 3261 .class1_alg_type = OP_ALG_ALGSEL_AES | 3262 OP_ALG_AAI_CTR_MOD128, 3263 .class2_alg_type = OP_ALG_ALGSEL_SHA512 | 3264 OP_ALG_AAI_HMAC_PRECOMP, 3265 .rfc3686 = true, 3266 .geniv = true, 3267 }, 3268 }, 3269 { 3270 .aead = { 3271 .base = { 3272 .cra_name = "rfc7539(chacha20,poly1305)", 3273 .cra_driver_name = "rfc7539-chacha20-poly1305-" 3274 "caam", 3275 .cra_blocksize = 1, 3276 }, 3277 .setkey = chachapoly_setkey, 3278 .setauthsize = chachapoly_setauthsize, 3279 .encrypt = chachapoly_encrypt, 3280 .decrypt = chachapoly_decrypt, 3281 .ivsize = CHACHAPOLY_IV_SIZE, 3282 .maxauthsize = POLY1305_DIGEST_SIZE, 3283 }, 3284 .caam = { 3285 .class1_alg_type = OP_ALG_ALGSEL_CHACHA20 | 3286 OP_ALG_AAI_AEAD, 3287 .class2_alg_type = OP_ALG_ALGSEL_POLY1305 | 3288 OP_ALG_AAI_AEAD, 3289 .nodkp = true, 3290 }, 3291 }, 3292 { 3293 .aead = { 3294 .base = { 3295 .cra_name = "rfc7539esp(chacha20,poly1305)", 3296 .cra_driver_name = "rfc7539esp-chacha20-" 3297 "poly1305-caam", 3298 .cra_blocksize = 1, 3299 }, 3300 .setkey = chachapoly_setkey, 3301 .setauthsize = chachapoly_setauthsize, 3302 .encrypt = chachapoly_encrypt, 3303 .decrypt = chachapoly_decrypt, 3304 .ivsize = 8, 3305 .maxauthsize = POLY1305_DIGEST_SIZE, 3306 }, 3307 .caam = { 3308 .class1_alg_type = OP_ALG_ALGSEL_CHACHA20 | 3309 OP_ALG_AAI_AEAD, 3310 .class2_alg_type = OP_ALG_ALGSEL_POLY1305 | 3311 OP_ALG_AAI_AEAD, 3312 .nodkp = true, 3313 }, 3314 }, 3315 }; 3316 3317 static int caam_init_common(struct caam_ctx *ctx, struct caam_alg_entry *caam, 3318 bool uses_dkp) 3319 { 3320 dma_addr_t dma_addr; 3321 struct caam_drv_private *priv; 3322 const size_t sh_desc_enc_offset = offsetof(struct caam_ctx, 3323 sh_desc_enc); 3324 3325 ctx->jrdev = caam_jr_alloc(); 3326 if (IS_ERR(ctx->jrdev)) { 3327 pr_err("Job Ring Device allocation for transform failed\n"); 3328 return PTR_ERR(ctx->jrdev); 3329 } 3330 3331 priv = dev_get_drvdata(ctx->jrdev->parent); 3332 if (priv->era >= 6 && uses_dkp) 3333 ctx->dir = DMA_BIDIRECTIONAL; 3334 else 3335 ctx->dir = DMA_TO_DEVICE; 3336 3337 dma_addr = dma_map_single_attrs(ctx->jrdev, ctx->sh_desc_enc, 3338 offsetof(struct caam_ctx, 3339 sh_desc_enc_dma) - 3340 sh_desc_enc_offset, 3341 ctx->dir, DMA_ATTR_SKIP_CPU_SYNC); 3342 if (dma_mapping_error(ctx->jrdev, dma_addr)) { 3343 dev_err(ctx->jrdev, "unable to map key, shared descriptors\n"); 3344 caam_jr_free(ctx->jrdev); 3345 return -ENOMEM; 3346 } 3347 3348 ctx->sh_desc_enc_dma = dma_addr; 3349 ctx->sh_desc_dec_dma = dma_addr + offsetof(struct caam_ctx, 3350 sh_desc_dec) - 3351 sh_desc_enc_offset; 3352 ctx->key_dma = dma_addr + offsetof(struct caam_ctx, key) - 3353 sh_desc_enc_offset; 3354 3355 /* copy descriptor header template value */ 3356 ctx->cdata.algtype = OP_TYPE_CLASS1_ALG | caam->class1_alg_type; 3357 ctx->adata.algtype = OP_TYPE_CLASS2_ALG | caam->class2_alg_type; 3358 3359 return 0; 3360 } 3361 3362 static int caam_cra_init(struct crypto_skcipher *tfm) 3363 { 3364 struct skcipher_alg *alg = crypto_skcipher_alg(tfm); 3365 struct caam_skcipher_alg *caam_alg = 3366 container_of(alg, typeof(*caam_alg), skcipher); 3367 struct caam_ctx *ctx = crypto_skcipher_ctx(tfm); 3368 3369 crypto_skcipher_set_reqsize(tfm, sizeof(struct caam_skcipher_req_ctx)); 3370 3371 ctx->enginectx.op.do_one_request = skcipher_do_one_req; 3372 3373 return caam_init_common(crypto_skcipher_ctx(tfm), &caam_alg->caam, 3374 false); 3375 } 3376 3377 static int caam_aead_init(struct crypto_aead *tfm) 3378 { 3379 struct aead_alg *alg = crypto_aead_alg(tfm); 3380 struct caam_aead_alg *caam_alg = 3381 container_of(alg, struct caam_aead_alg, aead); 3382 struct caam_ctx *ctx = crypto_aead_ctx(tfm); 3383 3384 crypto_aead_set_reqsize(tfm, sizeof(struct caam_aead_req_ctx)); 3385 3386 ctx->enginectx.op.do_one_request = aead_do_one_req; 3387 3388 return caam_init_common(ctx, &caam_alg->caam, !caam_alg->caam.nodkp); 3389 } 3390 3391 static void caam_exit_common(struct caam_ctx *ctx) 3392 { 3393 dma_unmap_single_attrs(ctx->jrdev, ctx->sh_desc_enc_dma, 3394 offsetof(struct caam_ctx, sh_desc_enc_dma) - 3395 offsetof(struct caam_ctx, sh_desc_enc), 3396 ctx->dir, DMA_ATTR_SKIP_CPU_SYNC); 3397 caam_jr_free(ctx->jrdev); 3398 } 3399 3400 static void caam_cra_exit(struct crypto_skcipher *tfm) 3401 { 3402 caam_exit_common(crypto_skcipher_ctx(tfm)); 3403 } 3404 3405 static void caam_aead_exit(struct crypto_aead *tfm) 3406 { 3407 caam_exit_common(crypto_aead_ctx(tfm)); 3408 } 3409 3410 void caam_algapi_exit(void) 3411 { 3412 int i; 3413 3414 for (i = 0; i < ARRAY_SIZE(driver_aeads); i++) { 3415 struct caam_aead_alg *t_alg = driver_aeads + i; 3416 3417 if (t_alg->registered) 3418 crypto_unregister_aead(&t_alg->aead); 3419 } 3420 3421 for (i = 0; i < ARRAY_SIZE(driver_algs); i++) { 3422 struct caam_skcipher_alg *t_alg = driver_algs + i; 3423 3424 if (t_alg->registered) 3425 crypto_unregister_skcipher(&t_alg->skcipher); 3426 } 3427 } 3428 3429 static void caam_skcipher_alg_init(struct caam_skcipher_alg *t_alg) 3430 { 3431 struct skcipher_alg *alg = &t_alg->skcipher; 3432 3433 alg->base.cra_module = THIS_MODULE; 3434 alg->base.cra_priority = CAAM_CRA_PRIORITY; 3435 alg->base.cra_ctxsize = sizeof(struct caam_ctx); 3436 alg->base.cra_flags = CRYPTO_ALG_ASYNC | CRYPTO_ALG_KERN_DRIVER_ONLY; 3437 3438 alg->init = caam_cra_init; 3439 alg->exit = caam_cra_exit; 3440 } 3441 3442 static void caam_aead_alg_init(struct caam_aead_alg *t_alg) 3443 { 3444 struct aead_alg *alg = &t_alg->aead; 3445 3446 alg->base.cra_module = THIS_MODULE; 3447 alg->base.cra_priority = CAAM_CRA_PRIORITY; 3448 alg->base.cra_ctxsize = sizeof(struct caam_ctx); 3449 alg->base.cra_flags = CRYPTO_ALG_ASYNC | CRYPTO_ALG_KERN_DRIVER_ONLY; 3450 3451 alg->init = caam_aead_init; 3452 alg->exit = caam_aead_exit; 3453 } 3454 3455 int caam_algapi_init(struct device *ctrldev) 3456 { 3457 struct caam_drv_private *priv = dev_get_drvdata(ctrldev); 3458 int i = 0, err = 0; 3459 u32 aes_vid, aes_inst, des_inst, md_vid, md_inst, ccha_inst, ptha_inst; 3460 u32 arc4_inst; 3461 unsigned int md_limit = SHA512_DIGEST_SIZE; 3462 bool registered = false, gcm_support; 3463 3464 /* 3465 * Register crypto algorithms the device supports. 3466 * First, detect presence and attributes of DES, AES, and MD blocks. 3467 */ 3468 if (priv->era < 10) { 3469 u32 cha_vid, cha_inst, aes_rn; 3470 3471 cha_vid = rd_reg32(&priv->ctrl->perfmon.cha_id_ls); 3472 aes_vid = cha_vid & CHA_ID_LS_AES_MASK; 3473 md_vid = (cha_vid & CHA_ID_LS_MD_MASK) >> CHA_ID_LS_MD_SHIFT; 3474 3475 cha_inst = rd_reg32(&priv->ctrl->perfmon.cha_num_ls); 3476 des_inst = (cha_inst & CHA_ID_LS_DES_MASK) >> 3477 CHA_ID_LS_DES_SHIFT; 3478 aes_inst = cha_inst & CHA_ID_LS_AES_MASK; 3479 md_inst = (cha_inst & CHA_ID_LS_MD_MASK) >> CHA_ID_LS_MD_SHIFT; 3480 arc4_inst = (cha_inst & CHA_ID_LS_ARC4_MASK) >> 3481 CHA_ID_LS_ARC4_SHIFT; 3482 ccha_inst = 0; 3483 ptha_inst = 0; 3484 3485 aes_rn = rd_reg32(&priv->ctrl->perfmon.cha_rev_ls) & 3486 CHA_ID_LS_AES_MASK; 3487 gcm_support = !(aes_vid == CHA_VER_VID_AES_LP && aes_rn < 8); 3488 } else { 3489 u32 aesa, mdha; 3490 3491 aesa = rd_reg32(&priv->ctrl->vreg.aesa); 3492 mdha = rd_reg32(&priv->ctrl->vreg.mdha); 3493 3494 aes_vid = (aesa & CHA_VER_VID_MASK) >> CHA_VER_VID_SHIFT; 3495 md_vid = (mdha & CHA_VER_VID_MASK) >> CHA_VER_VID_SHIFT; 3496 3497 des_inst = rd_reg32(&priv->ctrl->vreg.desa) & CHA_VER_NUM_MASK; 3498 aes_inst = aesa & CHA_VER_NUM_MASK; 3499 md_inst = mdha & CHA_VER_NUM_MASK; 3500 ccha_inst = rd_reg32(&priv->ctrl->vreg.ccha) & CHA_VER_NUM_MASK; 3501 ptha_inst = rd_reg32(&priv->ctrl->vreg.ptha) & CHA_VER_NUM_MASK; 3502 arc4_inst = rd_reg32(&priv->ctrl->vreg.afha) & CHA_VER_NUM_MASK; 3503 3504 gcm_support = aesa & CHA_VER_MISC_AES_GCM; 3505 } 3506 3507 /* If MD is present, limit digest size based on LP256 */ 3508 if (md_inst && md_vid == CHA_VER_VID_MD_LP256) 3509 md_limit = SHA256_DIGEST_SIZE; 3510 3511 for (i = 0; i < ARRAY_SIZE(driver_algs); i++) { 3512 struct caam_skcipher_alg *t_alg = driver_algs + i; 3513 u32 alg_sel = t_alg->caam.class1_alg_type & OP_ALG_ALGSEL_MASK; 3514 3515 /* Skip DES algorithms if not supported by device */ 3516 if (!des_inst && 3517 ((alg_sel == OP_ALG_ALGSEL_3DES) || 3518 (alg_sel == OP_ALG_ALGSEL_DES))) 3519 continue; 3520 3521 /* Skip AES algorithms if not supported by device */ 3522 if (!aes_inst && (alg_sel == OP_ALG_ALGSEL_AES)) 3523 continue; 3524 3525 /* Skip ARC4 algorithms if not supported by device */ 3526 if (!arc4_inst && alg_sel == OP_ALG_ALGSEL_ARC4) 3527 continue; 3528 3529 /* 3530 * Check support for AES modes not available 3531 * on LP devices. 3532 */ 3533 if (aes_vid == CHA_VER_VID_AES_LP && 3534 (t_alg->caam.class1_alg_type & OP_ALG_AAI_MASK) == 3535 OP_ALG_AAI_XTS) 3536 continue; 3537 3538 caam_skcipher_alg_init(t_alg); 3539 3540 err = crypto_register_skcipher(&t_alg->skcipher); 3541 if (err) { 3542 pr_warn("%s alg registration failed\n", 3543 t_alg->skcipher.base.cra_driver_name); 3544 continue; 3545 } 3546 3547 t_alg->registered = true; 3548 registered = true; 3549 } 3550 3551 for (i = 0; i < ARRAY_SIZE(driver_aeads); i++) { 3552 struct caam_aead_alg *t_alg = driver_aeads + i; 3553 u32 c1_alg_sel = t_alg->caam.class1_alg_type & 3554 OP_ALG_ALGSEL_MASK; 3555 u32 c2_alg_sel = t_alg->caam.class2_alg_type & 3556 OP_ALG_ALGSEL_MASK; 3557 u32 alg_aai = t_alg->caam.class1_alg_type & OP_ALG_AAI_MASK; 3558 3559 /* Skip DES algorithms if not supported by device */ 3560 if (!des_inst && 3561 ((c1_alg_sel == OP_ALG_ALGSEL_3DES) || 3562 (c1_alg_sel == OP_ALG_ALGSEL_DES))) 3563 continue; 3564 3565 /* Skip AES algorithms if not supported by device */ 3566 if (!aes_inst && (c1_alg_sel == OP_ALG_ALGSEL_AES)) 3567 continue; 3568 3569 /* Skip CHACHA20 algorithms if not supported by device */ 3570 if (c1_alg_sel == OP_ALG_ALGSEL_CHACHA20 && !ccha_inst) 3571 continue; 3572 3573 /* Skip POLY1305 algorithms if not supported by device */ 3574 if (c2_alg_sel == OP_ALG_ALGSEL_POLY1305 && !ptha_inst) 3575 continue; 3576 3577 /* Skip GCM algorithms if not supported by device */ 3578 if (c1_alg_sel == OP_ALG_ALGSEL_AES && 3579 alg_aai == OP_ALG_AAI_GCM && !gcm_support) 3580 continue; 3581 3582 /* 3583 * Skip algorithms requiring message digests 3584 * if MD or MD size is not supported by device. 3585 */ 3586 if (is_mdha(c2_alg_sel) && 3587 (!md_inst || t_alg->aead.maxauthsize > md_limit)) 3588 continue; 3589 3590 caam_aead_alg_init(t_alg); 3591 3592 err = crypto_register_aead(&t_alg->aead); 3593 if (err) { 3594 pr_warn("%s alg registration failed\n", 3595 t_alg->aead.base.cra_driver_name); 3596 continue; 3597 } 3598 3599 t_alg->registered = true; 3600 registered = true; 3601 } 3602 3603 if (registered) 3604 pr_info("caam algorithms registered in /proc/crypto\n"); 3605 3606 return err; 3607 } 3608