1 // SPDX-License-Identifier: GPL-2.0+ 2 /* 3 * caam - Freescale FSL CAAM support for crypto API 4 * 5 * Copyright 2008-2011 Freescale Semiconductor, Inc. 6 * Copyright 2016-2019 NXP 7 * 8 * Based on talitos crypto API driver. 9 * 10 * relationship of job descriptors to shared descriptors (SteveC Dec 10 2008): 11 * 12 * --------------- --------------- 13 * | JobDesc #1 |-------------------->| ShareDesc | 14 * | *(packet 1) | | (PDB) | 15 * --------------- |------------->| (hashKey) | 16 * . | | (cipherKey) | 17 * . | |-------->| (operation) | 18 * --------------- | | --------------- 19 * | JobDesc #2 |------| | 20 * | *(packet 2) | | 21 * --------------- | 22 * . | 23 * . | 24 * --------------- | 25 * | JobDesc #3 |------------ 26 * | *(packet 3) | 27 * --------------- 28 * 29 * The SharedDesc never changes for a connection unless rekeyed, but 30 * each packet will likely be in a different place. So all we need 31 * to know to process the packet is where the input is, where the 32 * output goes, and what context we want to process with. Context is 33 * in the SharedDesc, packet references in the JobDesc. 34 * 35 * So, a job desc looks like: 36 * 37 * --------------------- 38 * | Header | 39 * | ShareDesc Pointer | 40 * | SEQ_OUT_PTR | 41 * | (output buffer) | 42 * | (output length) | 43 * | SEQ_IN_PTR | 44 * | (input buffer) | 45 * | (input length) | 46 * --------------------- 47 */ 48 49 #include "compat.h" 50 51 #include "regs.h" 52 #include "intern.h" 53 #include "desc_constr.h" 54 #include "jr.h" 55 #include "error.h" 56 #include "sg_sw_sec4.h" 57 #include "key_gen.h" 58 #include "caamalg_desc.h" 59 #include <crypto/engine.h> 60 61 /* 62 * crypto alg 63 */ 64 #define CAAM_CRA_PRIORITY 3000 65 /* max key is sum of AES_MAX_KEY_SIZE, max split key size */ 66 #define CAAM_MAX_KEY_SIZE (AES_MAX_KEY_SIZE + \ 67 CTR_RFC3686_NONCE_SIZE + \ 68 SHA512_DIGEST_SIZE * 2) 69 70 #define AEAD_DESC_JOB_IO_LEN (DESC_JOB_IO_LEN + CAAM_CMD_SZ * 2) 71 #define GCM_DESC_JOB_IO_LEN (AEAD_DESC_JOB_IO_LEN + \ 72 CAAM_CMD_SZ * 4) 73 #define AUTHENC_DESC_JOB_IO_LEN (AEAD_DESC_JOB_IO_LEN + \ 74 CAAM_CMD_SZ * 5) 75 76 #define CHACHAPOLY_DESC_JOB_IO_LEN (AEAD_DESC_JOB_IO_LEN + CAAM_CMD_SZ * 6) 77 78 #define DESC_MAX_USED_BYTES (CAAM_DESC_BYTES_MAX - DESC_JOB_IO_LEN_MIN) 79 #define DESC_MAX_USED_LEN (DESC_MAX_USED_BYTES / CAAM_CMD_SZ) 80 81 struct caam_alg_entry { 82 int class1_alg_type; 83 int class2_alg_type; 84 bool rfc3686; 85 bool geniv; 86 bool nodkp; 87 }; 88 89 struct caam_aead_alg { 90 struct aead_alg aead; 91 struct caam_alg_entry caam; 92 bool registered; 93 }; 94 95 struct caam_skcipher_alg { 96 struct skcipher_alg skcipher; 97 struct caam_alg_entry caam; 98 bool registered; 99 }; 100 101 /* 102 * per-session context 103 */ 104 struct caam_ctx { 105 struct crypto_engine_ctx enginectx; 106 u32 sh_desc_enc[DESC_MAX_USED_LEN]; 107 u32 sh_desc_dec[DESC_MAX_USED_LEN]; 108 u8 key[CAAM_MAX_KEY_SIZE]; 109 dma_addr_t sh_desc_enc_dma; 110 dma_addr_t sh_desc_dec_dma; 111 dma_addr_t key_dma; 112 enum dma_data_direction dir; 113 struct device *jrdev; 114 struct alginfo adata; 115 struct alginfo cdata; 116 unsigned int authsize; 117 }; 118 119 struct caam_skcipher_req_ctx { 120 struct skcipher_edesc *edesc; 121 }; 122 123 struct caam_aead_req_ctx { 124 struct aead_edesc *edesc; 125 }; 126 127 static int aead_null_set_sh_desc(struct crypto_aead *aead) 128 { 129 struct caam_ctx *ctx = crypto_aead_ctx(aead); 130 struct device *jrdev = ctx->jrdev; 131 struct caam_drv_private *ctrlpriv = dev_get_drvdata(jrdev->parent); 132 u32 *desc; 133 int rem_bytes = CAAM_DESC_BYTES_MAX - AEAD_DESC_JOB_IO_LEN - 134 ctx->adata.keylen_pad; 135 136 /* 137 * Job Descriptor and Shared Descriptors 138 * must all fit into the 64-word Descriptor h/w Buffer 139 */ 140 if (rem_bytes >= DESC_AEAD_NULL_ENC_LEN) { 141 ctx->adata.key_inline = true; 142 ctx->adata.key_virt = ctx->key; 143 } else { 144 ctx->adata.key_inline = false; 145 ctx->adata.key_dma = ctx->key_dma; 146 } 147 148 /* aead_encrypt shared descriptor */ 149 desc = ctx->sh_desc_enc; 150 cnstr_shdsc_aead_null_encap(desc, &ctx->adata, ctx->authsize, 151 ctrlpriv->era); 152 dma_sync_single_for_device(jrdev, ctx->sh_desc_enc_dma, 153 desc_bytes(desc), ctx->dir); 154 155 /* 156 * Job Descriptor and Shared Descriptors 157 * must all fit into the 64-word Descriptor h/w Buffer 158 */ 159 if (rem_bytes >= DESC_AEAD_NULL_DEC_LEN) { 160 ctx->adata.key_inline = true; 161 ctx->adata.key_virt = ctx->key; 162 } else { 163 ctx->adata.key_inline = false; 164 ctx->adata.key_dma = ctx->key_dma; 165 } 166 167 /* aead_decrypt shared descriptor */ 168 desc = ctx->sh_desc_dec; 169 cnstr_shdsc_aead_null_decap(desc, &ctx->adata, ctx->authsize, 170 ctrlpriv->era); 171 dma_sync_single_for_device(jrdev, ctx->sh_desc_dec_dma, 172 desc_bytes(desc), ctx->dir); 173 174 return 0; 175 } 176 177 static int aead_set_sh_desc(struct crypto_aead *aead) 178 { 179 struct caam_aead_alg *alg = container_of(crypto_aead_alg(aead), 180 struct caam_aead_alg, aead); 181 unsigned int ivsize = crypto_aead_ivsize(aead); 182 struct caam_ctx *ctx = crypto_aead_ctx(aead); 183 struct device *jrdev = ctx->jrdev; 184 struct caam_drv_private *ctrlpriv = dev_get_drvdata(jrdev->parent); 185 u32 ctx1_iv_off = 0; 186 u32 *desc, *nonce = NULL; 187 u32 inl_mask; 188 unsigned int data_len[2]; 189 const bool ctr_mode = ((ctx->cdata.algtype & OP_ALG_AAI_MASK) == 190 OP_ALG_AAI_CTR_MOD128); 191 const bool is_rfc3686 = alg->caam.rfc3686; 192 193 if (!ctx->authsize) 194 return 0; 195 196 /* NULL encryption / decryption */ 197 if (!ctx->cdata.keylen) 198 return aead_null_set_sh_desc(aead); 199 200 /* 201 * AES-CTR needs to load IV in CONTEXT1 reg 202 * at an offset of 128bits (16bytes) 203 * CONTEXT1[255:128] = IV 204 */ 205 if (ctr_mode) 206 ctx1_iv_off = 16; 207 208 /* 209 * RFC3686 specific: 210 * CONTEXT1[255:128] = {NONCE, IV, COUNTER} 211 */ 212 if (is_rfc3686) { 213 ctx1_iv_off = 16 + CTR_RFC3686_NONCE_SIZE; 214 nonce = (u32 *)((void *)ctx->key + ctx->adata.keylen_pad + 215 ctx->cdata.keylen - CTR_RFC3686_NONCE_SIZE); 216 } 217 218 /* 219 * In case |user key| > |derived key|, using DKP<imm,imm> 220 * would result in invalid opcodes (last bytes of user key) in 221 * the resulting descriptor. Use DKP<ptr,imm> instead => both 222 * virtual and dma key addresses are needed. 223 */ 224 ctx->adata.key_virt = ctx->key; 225 ctx->adata.key_dma = ctx->key_dma; 226 227 ctx->cdata.key_virt = ctx->key + ctx->adata.keylen_pad; 228 ctx->cdata.key_dma = ctx->key_dma + ctx->adata.keylen_pad; 229 230 data_len[0] = ctx->adata.keylen_pad; 231 data_len[1] = ctx->cdata.keylen; 232 233 if (alg->caam.geniv) 234 goto skip_enc; 235 236 /* 237 * Job Descriptor and Shared Descriptors 238 * must all fit into the 64-word Descriptor h/w Buffer 239 */ 240 if (desc_inline_query(DESC_AEAD_ENC_LEN + 241 (is_rfc3686 ? DESC_AEAD_CTR_RFC3686_LEN : 0), 242 AUTHENC_DESC_JOB_IO_LEN, data_len, &inl_mask, 243 ARRAY_SIZE(data_len)) < 0) 244 return -EINVAL; 245 246 ctx->adata.key_inline = !!(inl_mask & 1); 247 ctx->cdata.key_inline = !!(inl_mask & 2); 248 249 /* aead_encrypt shared descriptor */ 250 desc = ctx->sh_desc_enc; 251 cnstr_shdsc_aead_encap(desc, &ctx->cdata, &ctx->adata, ivsize, 252 ctx->authsize, is_rfc3686, nonce, ctx1_iv_off, 253 false, ctrlpriv->era); 254 dma_sync_single_for_device(jrdev, ctx->sh_desc_enc_dma, 255 desc_bytes(desc), ctx->dir); 256 257 skip_enc: 258 /* 259 * Job Descriptor and Shared Descriptors 260 * must all fit into the 64-word Descriptor h/w Buffer 261 */ 262 if (desc_inline_query(DESC_AEAD_DEC_LEN + 263 (is_rfc3686 ? DESC_AEAD_CTR_RFC3686_LEN : 0), 264 AUTHENC_DESC_JOB_IO_LEN, data_len, &inl_mask, 265 ARRAY_SIZE(data_len)) < 0) 266 return -EINVAL; 267 268 ctx->adata.key_inline = !!(inl_mask & 1); 269 ctx->cdata.key_inline = !!(inl_mask & 2); 270 271 /* aead_decrypt shared descriptor */ 272 desc = ctx->sh_desc_dec; 273 cnstr_shdsc_aead_decap(desc, &ctx->cdata, &ctx->adata, ivsize, 274 ctx->authsize, alg->caam.geniv, is_rfc3686, 275 nonce, ctx1_iv_off, false, ctrlpriv->era); 276 dma_sync_single_for_device(jrdev, ctx->sh_desc_dec_dma, 277 desc_bytes(desc), ctx->dir); 278 279 if (!alg->caam.geniv) 280 goto skip_givenc; 281 282 /* 283 * Job Descriptor and Shared Descriptors 284 * must all fit into the 64-word Descriptor h/w Buffer 285 */ 286 if (desc_inline_query(DESC_AEAD_GIVENC_LEN + 287 (is_rfc3686 ? DESC_AEAD_CTR_RFC3686_LEN : 0), 288 AUTHENC_DESC_JOB_IO_LEN, data_len, &inl_mask, 289 ARRAY_SIZE(data_len)) < 0) 290 return -EINVAL; 291 292 ctx->adata.key_inline = !!(inl_mask & 1); 293 ctx->cdata.key_inline = !!(inl_mask & 2); 294 295 /* aead_givencrypt shared descriptor */ 296 desc = ctx->sh_desc_enc; 297 cnstr_shdsc_aead_givencap(desc, &ctx->cdata, &ctx->adata, ivsize, 298 ctx->authsize, is_rfc3686, nonce, 299 ctx1_iv_off, false, ctrlpriv->era); 300 dma_sync_single_for_device(jrdev, ctx->sh_desc_enc_dma, 301 desc_bytes(desc), ctx->dir); 302 303 skip_givenc: 304 return 0; 305 } 306 307 static int aead_setauthsize(struct crypto_aead *authenc, 308 unsigned int authsize) 309 { 310 struct caam_ctx *ctx = crypto_aead_ctx(authenc); 311 312 ctx->authsize = authsize; 313 aead_set_sh_desc(authenc); 314 315 return 0; 316 } 317 318 static int gcm_set_sh_desc(struct crypto_aead *aead) 319 { 320 struct caam_ctx *ctx = crypto_aead_ctx(aead); 321 struct device *jrdev = ctx->jrdev; 322 unsigned int ivsize = crypto_aead_ivsize(aead); 323 u32 *desc; 324 int rem_bytes = CAAM_DESC_BYTES_MAX - GCM_DESC_JOB_IO_LEN - 325 ctx->cdata.keylen; 326 327 if (!ctx->cdata.keylen || !ctx->authsize) 328 return 0; 329 330 /* 331 * AES GCM encrypt shared descriptor 332 * Job Descriptor and Shared Descriptor 333 * must fit into the 64-word Descriptor h/w Buffer 334 */ 335 if (rem_bytes >= DESC_GCM_ENC_LEN) { 336 ctx->cdata.key_inline = true; 337 ctx->cdata.key_virt = ctx->key; 338 } else { 339 ctx->cdata.key_inline = false; 340 ctx->cdata.key_dma = ctx->key_dma; 341 } 342 343 desc = ctx->sh_desc_enc; 344 cnstr_shdsc_gcm_encap(desc, &ctx->cdata, ivsize, ctx->authsize, false); 345 dma_sync_single_for_device(jrdev, ctx->sh_desc_enc_dma, 346 desc_bytes(desc), ctx->dir); 347 348 /* 349 * Job Descriptor and Shared Descriptors 350 * must all fit into the 64-word Descriptor h/w Buffer 351 */ 352 if (rem_bytes >= DESC_GCM_DEC_LEN) { 353 ctx->cdata.key_inline = true; 354 ctx->cdata.key_virt = ctx->key; 355 } else { 356 ctx->cdata.key_inline = false; 357 ctx->cdata.key_dma = ctx->key_dma; 358 } 359 360 desc = ctx->sh_desc_dec; 361 cnstr_shdsc_gcm_decap(desc, &ctx->cdata, ivsize, ctx->authsize, false); 362 dma_sync_single_for_device(jrdev, ctx->sh_desc_dec_dma, 363 desc_bytes(desc), ctx->dir); 364 365 return 0; 366 } 367 368 static int gcm_setauthsize(struct crypto_aead *authenc, unsigned int authsize) 369 { 370 struct caam_ctx *ctx = crypto_aead_ctx(authenc); 371 int err; 372 373 err = crypto_gcm_check_authsize(authsize); 374 if (err) 375 return err; 376 377 ctx->authsize = authsize; 378 gcm_set_sh_desc(authenc); 379 380 return 0; 381 } 382 383 static int rfc4106_set_sh_desc(struct crypto_aead *aead) 384 { 385 struct caam_ctx *ctx = crypto_aead_ctx(aead); 386 struct device *jrdev = ctx->jrdev; 387 unsigned int ivsize = crypto_aead_ivsize(aead); 388 u32 *desc; 389 int rem_bytes = CAAM_DESC_BYTES_MAX - GCM_DESC_JOB_IO_LEN - 390 ctx->cdata.keylen; 391 392 if (!ctx->cdata.keylen || !ctx->authsize) 393 return 0; 394 395 /* 396 * RFC4106 encrypt shared descriptor 397 * Job Descriptor and Shared Descriptor 398 * must fit into the 64-word Descriptor h/w Buffer 399 */ 400 if (rem_bytes >= DESC_RFC4106_ENC_LEN) { 401 ctx->cdata.key_inline = true; 402 ctx->cdata.key_virt = ctx->key; 403 } else { 404 ctx->cdata.key_inline = false; 405 ctx->cdata.key_dma = ctx->key_dma; 406 } 407 408 desc = ctx->sh_desc_enc; 409 cnstr_shdsc_rfc4106_encap(desc, &ctx->cdata, ivsize, ctx->authsize, 410 false); 411 dma_sync_single_for_device(jrdev, ctx->sh_desc_enc_dma, 412 desc_bytes(desc), ctx->dir); 413 414 /* 415 * Job Descriptor and Shared Descriptors 416 * must all fit into the 64-word Descriptor h/w Buffer 417 */ 418 if (rem_bytes >= DESC_RFC4106_DEC_LEN) { 419 ctx->cdata.key_inline = true; 420 ctx->cdata.key_virt = ctx->key; 421 } else { 422 ctx->cdata.key_inline = false; 423 ctx->cdata.key_dma = ctx->key_dma; 424 } 425 426 desc = ctx->sh_desc_dec; 427 cnstr_shdsc_rfc4106_decap(desc, &ctx->cdata, ivsize, ctx->authsize, 428 false); 429 dma_sync_single_for_device(jrdev, ctx->sh_desc_dec_dma, 430 desc_bytes(desc), ctx->dir); 431 432 return 0; 433 } 434 435 static int rfc4106_setauthsize(struct crypto_aead *authenc, 436 unsigned int authsize) 437 { 438 struct caam_ctx *ctx = crypto_aead_ctx(authenc); 439 int err; 440 441 err = crypto_rfc4106_check_authsize(authsize); 442 if (err) 443 return err; 444 445 ctx->authsize = authsize; 446 rfc4106_set_sh_desc(authenc); 447 448 return 0; 449 } 450 451 static int rfc4543_set_sh_desc(struct crypto_aead *aead) 452 { 453 struct caam_ctx *ctx = crypto_aead_ctx(aead); 454 struct device *jrdev = ctx->jrdev; 455 unsigned int ivsize = crypto_aead_ivsize(aead); 456 u32 *desc; 457 int rem_bytes = CAAM_DESC_BYTES_MAX - GCM_DESC_JOB_IO_LEN - 458 ctx->cdata.keylen; 459 460 if (!ctx->cdata.keylen || !ctx->authsize) 461 return 0; 462 463 /* 464 * RFC4543 encrypt shared descriptor 465 * Job Descriptor and Shared Descriptor 466 * must fit into the 64-word Descriptor h/w Buffer 467 */ 468 if (rem_bytes >= DESC_RFC4543_ENC_LEN) { 469 ctx->cdata.key_inline = true; 470 ctx->cdata.key_virt = ctx->key; 471 } else { 472 ctx->cdata.key_inline = false; 473 ctx->cdata.key_dma = ctx->key_dma; 474 } 475 476 desc = ctx->sh_desc_enc; 477 cnstr_shdsc_rfc4543_encap(desc, &ctx->cdata, ivsize, ctx->authsize, 478 false); 479 dma_sync_single_for_device(jrdev, ctx->sh_desc_enc_dma, 480 desc_bytes(desc), ctx->dir); 481 482 /* 483 * Job Descriptor and Shared Descriptors 484 * must all fit into the 64-word Descriptor h/w Buffer 485 */ 486 if (rem_bytes >= DESC_RFC4543_DEC_LEN) { 487 ctx->cdata.key_inline = true; 488 ctx->cdata.key_virt = ctx->key; 489 } else { 490 ctx->cdata.key_inline = false; 491 ctx->cdata.key_dma = ctx->key_dma; 492 } 493 494 desc = ctx->sh_desc_dec; 495 cnstr_shdsc_rfc4543_decap(desc, &ctx->cdata, ivsize, ctx->authsize, 496 false); 497 dma_sync_single_for_device(jrdev, ctx->sh_desc_dec_dma, 498 desc_bytes(desc), ctx->dir); 499 500 return 0; 501 } 502 503 static int rfc4543_setauthsize(struct crypto_aead *authenc, 504 unsigned int authsize) 505 { 506 struct caam_ctx *ctx = crypto_aead_ctx(authenc); 507 508 if (authsize != 16) 509 return -EINVAL; 510 511 ctx->authsize = authsize; 512 rfc4543_set_sh_desc(authenc); 513 514 return 0; 515 } 516 517 static int chachapoly_set_sh_desc(struct crypto_aead *aead) 518 { 519 struct caam_ctx *ctx = crypto_aead_ctx(aead); 520 struct device *jrdev = ctx->jrdev; 521 unsigned int ivsize = crypto_aead_ivsize(aead); 522 u32 *desc; 523 524 if (!ctx->cdata.keylen || !ctx->authsize) 525 return 0; 526 527 desc = ctx->sh_desc_enc; 528 cnstr_shdsc_chachapoly(desc, &ctx->cdata, &ctx->adata, ivsize, 529 ctx->authsize, true, false); 530 dma_sync_single_for_device(jrdev, ctx->sh_desc_enc_dma, 531 desc_bytes(desc), ctx->dir); 532 533 desc = ctx->sh_desc_dec; 534 cnstr_shdsc_chachapoly(desc, &ctx->cdata, &ctx->adata, ivsize, 535 ctx->authsize, false, false); 536 dma_sync_single_for_device(jrdev, ctx->sh_desc_dec_dma, 537 desc_bytes(desc), ctx->dir); 538 539 return 0; 540 } 541 542 static int chachapoly_setauthsize(struct crypto_aead *aead, 543 unsigned int authsize) 544 { 545 struct caam_ctx *ctx = crypto_aead_ctx(aead); 546 547 if (authsize != POLY1305_DIGEST_SIZE) 548 return -EINVAL; 549 550 ctx->authsize = authsize; 551 return chachapoly_set_sh_desc(aead); 552 } 553 554 static int chachapoly_setkey(struct crypto_aead *aead, const u8 *key, 555 unsigned int keylen) 556 { 557 struct caam_ctx *ctx = crypto_aead_ctx(aead); 558 unsigned int ivsize = crypto_aead_ivsize(aead); 559 unsigned int saltlen = CHACHAPOLY_IV_SIZE - ivsize; 560 561 if (keylen != CHACHA_KEY_SIZE + saltlen) 562 return -EINVAL; 563 564 ctx->cdata.key_virt = key; 565 ctx->cdata.keylen = keylen - saltlen; 566 567 return chachapoly_set_sh_desc(aead); 568 } 569 570 static int aead_setkey(struct crypto_aead *aead, 571 const u8 *key, unsigned int keylen) 572 { 573 struct caam_ctx *ctx = crypto_aead_ctx(aead); 574 struct device *jrdev = ctx->jrdev; 575 struct caam_drv_private *ctrlpriv = dev_get_drvdata(jrdev->parent); 576 struct crypto_authenc_keys keys; 577 int ret = 0; 578 579 if (crypto_authenc_extractkeys(&keys, key, keylen) != 0) 580 goto badkey; 581 582 dev_dbg(jrdev, "keylen %d enckeylen %d authkeylen %d\n", 583 keys.authkeylen + keys.enckeylen, keys.enckeylen, 584 keys.authkeylen); 585 print_hex_dump_debug("key in @"__stringify(__LINE__)": ", 586 DUMP_PREFIX_ADDRESS, 16, 4, key, keylen, 1); 587 588 /* 589 * If DKP is supported, use it in the shared descriptor to generate 590 * the split key. 591 */ 592 if (ctrlpriv->era >= 6) { 593 ctx->adata.keylen = keys.authkeylen; 594 ctx->adata.keylen_pad = split_key_len(ctx->adata.algtype & 595 OP_ALG_ALGSEL_MASK); 596 597 if (ctx->adata.keylen_pad + keys.enckeylen > CAAM_MAX_KEY_SIZE) 598 goto badkey; 599 600 memcpy(ctx->key, keys.authkey, keys.authkeylen); 601 memcpy(ctx->key + ctx->adata.keylen_pad, keys.enckey, 602 keys.enckeylen); 603 dma_sync_single_for_device(jrdev, ctx->key_dma, 604 ctx->adata.keylen_pad + 605 keys.enckeylen, ctx->dir); 606 goto skip_split_key; 607 } 608 609 ret = gen_split_key(ctx->jrdev, ctx->key, &ctx->adata, keys.authkey, 610 keys.authkeylen, CAAM_MAX_KEY_SIZE - 611 keys.enckeylen); 612 if (ret) { 613 goto badkey; 614 } 615 616 /* postpend encryption key to auth split key */ 617 memcpy(ctx->key + ctx->adata.keylen_pad, keys.enckey, keys.enckeylen); 618 dma_sync_single_for_device(jrdev, ctx->key_dma, ctx->adata.keylen_pad + 619 keys.enckeylen, ctx->dir); 620 621 print_hex_dump_debug("ctx.key@"__stringify(__LINE__)": ", 622 DUMP_PREFIX_ADDRESS, 16, 4, ctx->key, 623 ctx->adata.keylen_pad + keys.enckeylen, 1); 624 625 skip_split_key: 626 ctx->cdata.keylen = keys.enckeylen; 627 memzero_explicit(&keys, sizeof(keys)); 628 return aead_set_sh_desc(aead); 629 badkey: 630 memzero_explicit(&keys, sizeof(keys)); 631 return -EINVAL; 632 } 633 634 static int des3_aead_setkey(struct crypto_aead *aead, const u8 *key, 635 unsigned int keylen) 636 { 637 struct crypto_authenc_keys keys; 638 int err; 639 640 err = crypto_authenc_extractkeys(&keys, key, keylen); 641 if (unlikely(err)) 642 return err; 643 644 err = verify_aead_des3_key(aead, keys.enckey, keys.enckeylen) ?: 645 aead_setkey(aead, key, keylen); 646 647 memzero_explicit(&keys, sizeof(keys)); 648 return err; 649 } 650 651 static int gcm_setkey(struct crypto_aead *aead, 652 const u8 *key, unsigned int keylen) 653 { 654 struct caam_ctx *ctx = crypto_aead_ctx(aead); 655 struct device *jrdev = ctx->jrdev; 656 int err; 657 658 err = aes_check_keylen(keylen); 659 if (err) 660 return err; 661 662 print_hex_dump_debug("key in @"__stringify(__LINE__)": ", 663 DUMP_PREFIX_ADDRESS, 16, 4, key, keylen, 1); 664 665 memcpy(ctx->key, key, keylen); 666 dma_sync_single_for_device(jrdev, ctx->key_dma, keylen, ctx->dir); 667 ctx->cdata.keylen = keylen; 668 669 return gcm_set_sh_desc(aead); 670 } 671 672 static int rfc4106_setkey(struct crypto_aead *aead, 673 const u8 *key, unsigned int keylen) 674 { 675 struct caam_ctx *ctx = crypto_aead_ctx(aead); 676 struct device *jrdev = ctx->jrdev; 677 int err; 678 679 err = aes_check_keylen(keylen - 4); 680 if (err) 681 return err; 682 683 print_hex_dump_debug("key in @"__stringify(__LINE__)": ", 684 DUMP_PREFIX_ADDRESS, 16, 4, key, keylen, 1); 685 686 memcpy(ctx->key, key, keylen); 687 688 /* 689 * The last four bytes of the key material are used as the salt value 690 * in the nonce. Update the AES key length. 691 */ 692 ctx->cdata.keylen = keylen - 4; 693 dma_sync_single_for_device(jrdev, ctx->key_dma, ctx->cdata.keylen, 694 ctx->dir); 695 return rfc4106_set_sh_desc(aead); 696 } 697 698 static int rfc4543_setkey(struct crypto_aead *aead, 699 const u8 *key, unsigned int keylen) 700 { 701 struct caam_ctx *ctx = crypto_aead_ctx(aead); 702 struct device *jrdev = ctx->jrdev; 703 int err; 704 705 err = aes_check_keylen(keylen - 4); 706 if (err) 707 return err; 708 709 print_hex_dump_debug("key in @"__stringify(__LINE__)": ", 710 DUMP_PREFIX_ADDRESS, 16, 4, key, keylen, 1); 711 712 memcpy(ctx->key, key, keylen); 713 714 /* 715 * The last four bytes of the key material are used as the salt value 716 * in the nonce. Update the AES key length. 717 */ 718 ctx->cdata.keylen = keylen - 4; 719 dma_sync_single_for_device(jrdev, ctx->key_dma, ctx->cdata.keylen, 720 ctx->dir); 721 return rfc4543_set_sh_desc(aead); 722 } 723 724 static int skcipher_setkey(struct crypto_skcipher *skcipher, const u8 *key, 725 unsigned int keylen, const u32 ctx1_iv_off) 726 { 727 struct caam_ctx *ctx = crypto_skcipher_ctx(skcipher); 728 struct caam_skcipher_alg *alg = 729 container_of(crypto_skcipher_alg(skcipher), typeof(*alg), 730 skcipher); 731 struct device *jrdev = ctx->jrdev; 732 unsigned int ivsize = crypto_skcipher_ivsize(skcipher); 733 u32 *desc; 734 const bool is_rfc3686 = alg->caam.rfc3686; 735 736 print_hex_dump_debug("key in @"__stringify(__LINE__)": ", 737 DUMP_PREFIX_ADDRESS, 16, 4, key, keylen, 1); 738 739 ctx->cdata.keylen = keylen; 740 ctx->cdata.key_virt = key; 741 ctx->cdata.key_inline = true; 742 743 /* skcipher_encrypt shared descriptor */ 744 desc = ctx->sh_desc_enc; 745 cnstr_shdsc_skcipher_encap(desc, &ctx->cdata, ivsize, is_rfc3686, 746 ctx1_iv_off); 747 dma_sync_single_for_device(jrdev, ctx->sh_desc_enc_dma, 748 desc_bytes(desc), ctx->dir); 749 750 /* skcipher_decrypt shared descriptor */ 751 desc = ctx->sh_desc_dec; 752 cnstr_shdsc_skcipher_decap(desc, &ctx->cdata, ivsize, is_rfc3686, 753 ctx1_iv_off); 754 dma_sync_single_for_device(jrdev, ctx->sh_desc_dec_dma, 755 desc_bytes(desc), ctx->dir); 756 757 return 0; 758 } 759 760 static int aes_skcipher_setkey(struct crypto_skcipher *skcipher, 761 const u8 *key, unsigned int keylen) 762 { 763 int err; 764 765 err = aes_check_keylen(keylen); 766 if (err) 767 return err; 768 769 return skcipher_setkey(skcipher, key, keylen, 0); 770 } 771 772 static int rfc3686_skcipher_setkey(struct crypto_skcipher *skcipher, 773 const u8 *key, unsigned int keylen) 774 { 775 u32 ctx1_iv_off; 776 int err; 777 778 /* 779 * RFC3686 specific: 780 * | CONTEXT1[255:128] = {NONCE, IV, COUNTER} 781 * | *key = {KEY, NONCE} 782 */ 783 ctx1_iv_off = 16 + CTR_RFC3686_NONCE_SIZE; 784 keylen -= CTR_RFC3686_NONCE_SIZE; 785 786 err = aes_check_keylen(keylen); 787 if (err) 788 return err; 789 790 return skcipher_setkey(skcipher, key, keylen, ctx1_iv_off); 791 } 792 793 static int ctr_skcipher_setkey(struct crypto_skcipher *skcipher, 794 const u8 *key, unsigned int keylen) 795 { 796 u32 ctx1_iv_off; 797 int err; 798 799 /* 800 * AES-CTR needs to load IV in CONTEXT1 reg 801 * at an offset of 128bits (16bytes) 802 * CONTEXT1[255:128] = IV 803 */ 804 ctx1_iv_off = 16; 805 806 err = aes_check_keylen(keylen); 807 if (err) 808 return err; 809 810 return skcipher_setkey(skcipher, key, keylen, ctx1_iv_off); 811 } 812 813 static int arc4_skcipher_setkey(struct crypto_skcipher *skcipher, 814 const u8 *key, unsigned int keylen) 815 { 816 return skcipher_setkey(skcipher, key, keylen, 0); 817 } 818 819 static int des_skcipher_setkey(struct crypto_skcipher *skcipher, 820 const u8 *key, unsigned int keylen) 821 { 822 return verify_skcipher_des_key(skcipher, key) ?: 823 skcipher_setkey(skcipher, key, keylen, 0); 824 } 825 826 static int des3_skcipher_setkey(struct crypto_skcipher *skcipher, 827 const u8 *key, unsigned int keylen) 828 { 829 return verify_skcipher_des3_key(skcipher, key) ?: 830 skcipher_setkey(skcipher, key, keylen, 0); 831 } 832 833 static int xts_skcipher_setkey(struct crypto_skcipher *skcipher, const u8 *key, 834 unsigned int keylen) 835 { 836 struct caam_ctx *ctx = crypto_skcipher_ctx(skcipher); 837 struct device *jrdev = ctx->jrdev; 838 u32 *desc; 839 840 if (keylen != 2 * AES_MIN_KEY_SIZE && keylen != 2 * AES_MAX_KEY_SIZE) { 841 dev_err(jrdev, "key size mismatch\n"); 842 return -EINVAL; 843 } 844 845 ctx->cdata.keylen = keylen; 846 ctx->cdata.key_virt = key; 847 ctx->cdata.key_inline = true; 848 849 /* xts_skcipher_encrypt shared descriptor */ 850 desc = ctx->sh_desc_enc; 851 cnstr_shdsc_xts_skcipher_encap(desc, &ctx->cdata); 852 dma_sync_single_for_device(jrdev, ctx->sh_desc_enc_dma, 853 desc_bytes(desc), ctx->dir); 854 855 /* xts_skcipher_decrypt shared descriptor */ 856 desc = ctx->sh_desc_dec; 857 cnstr_shdsc_xts_skcipher_decap(desc, &ctx->cdata); 858 dma_sync_single_for_device(jrdev, ctx->sh_desc_dec_dma, 859 desc_bytes(desc), ctx->dir); 860 861 return 0; 862 } 863 864 /* 865 * aead_edesc - s/w-extended aead descriptor 866 * @src_nents: number of segments in input s/w scatterlist 867 * @dst_nents: number of segments in output s/w scatterlist 868 * @mapped_src_nents: number of segments in input h/w link table 869 * @mapped_dst_nents: number of segments in output h/w link table 870 * @sec4_sg_bytes: length of dma mapped sec4_sg space 871 * @bklog: stored to determine if the request needs backlog 872 * @sec4_sg_dma: bus physical mapped address of h/w link table 873 * @sec4_sg: pointer to h/w link table 874 * @hw_desc: the h/w job descriptor followed by any referenced link tables 875 */ 876 struct aead_edesc { 877 int src_nents; 878 int dst_nents; 879 int mapped_src_nents; 880 int mapped_dst_nents; 881 int sec4_sg_bytes; 882 bool bklog; 883 dma_addr_t sec4_sg_dma; 884 struct sec4_sg_entry *sec4_sg; 885 u32 hw_desc[]; 886 }; 887 888 /* 889 * skcipher_edesc - s/w-extended skcipher descriptor 890 * @src_nents: number of segments in input s/w scatterlist 891 * @dst_nents: number of segments in output s/w scatterlist 892 * @mapped_src_nents: number of segments in input h/w link table 893 * @mapped_dst_nents: number of segments in output h/w link table 894 * @iv_dma: dma address of iv for checking continuity and link table 895 * @sec4_sg_bytes: length of dma mapped sec4_sg space 896 * @bklog: stored to determine if the request needs backlog 897 * @sec4_sg_dma: bus physical mapped address of h/w link table 898 * @sec4_sg: pointer to h/w link table 899 * @hw_desc: the h/w job descriptor followed by any referenced link tables 900 * and IV 901 */ 902 struct skcipher_edesc { 903 int src_nents; 904 int dst_nents; 905 int mapped_src_nents; 906 int mapped_dst_nents; 907 dma_addr_t iv_dma; 908 int sec4_sg_bytes; 909 bool bklog; 910 dma_addr_t sec4_sg_dma; 911 struct sec4_sg_entry *sec4_sg; 912 u32 hw_desc[]; 913 }; 914 915 static void caam_unmap(struct device *dev, struct scatterlist *src, 916 struct scatterlist *dst, int src_nents, 917 int dst_nents, 918 dma_addr_t iv_dma, int ivsize, dma_addr_t sec4_sg_dma, 919 int sec4_sg_bytes) 920 { 921 if (dst != src) { 922 if (src_nents) 923 dma_unmap_sg(dev, src, src_nents, DMA_TO_DEVICE); 924 if (dst_nents) 925 dma_unmap_sg(dev, dst, dst_nents, DMA_FROM_DEVICE); 926 } else { 927 dma_unmap_sg(dev, src, src_nents, DMA_BIDIRECTIONAL); 928 } 929 930 if (iv_dma) 931 dma_unmap_single(dev, iv_dma, ivsize, DMA_BIDIRECTIONAL); 932 if (sec4_sg_bytes) 933 dma_unmap_single(dev, sec4_sg_dma, sec4_sg_bytes, 934 DMA_TO_DEVICE); 935 } 936 937 static void aead_unmap(struct device *dev, 938 struct aead_edesc *edesc, 939 struct aead_request *req) 940 { 941 caam_unmap(dev, req->src, req->dst, 942 edesc->src_nents, edesc->dst_nents, 0, 0, 943 edesc->sec4_sg_dma, edesc->sec4_sg_bytes); 944 } 945 946 static void skcipher_unmap(struct device *dev, struct skcipher_edesc *edesc, 947 struct skcipher_request *req) 948 { 949 struct crypto_skcipher *skcipher = crypto_skcipher_reqtfm(req); 950 int ivsize = crypto_skcipher_ivsize(skcipher); 951 952 caam_unmap(dev, req->src, req->dst, 953 edesc->src_nents, edesc->dst_nents, 954 edesc->iv_dma, ivsize, 955 edesc->sec4_sg_dma, edesc->sec4_sg_bytes); 956 } 957 958 static void aead_crypt_done(struct device *jrdev, u32 *desc, u32 err, 959 void *context) 960 { 961 struct aead_request *req = context; 962 struct caam_aead_req_ctx *rctx = aead_request_ctx(req); 963 struct caam_drv_private_jr *jrp = dev_get_drvdata(jrdev); 964 struct aead_edesc *edesc; 965 int ecode = 0; 966 967 dev_dbg(jrdev, "%s %d: err 0x%x\n", __func__, __LINE__, err); 968 969 edesc = rctx->edesc; 970 971 if (err) 972 ecode = caam_jr_strstatus(jrdev, err); 973 974 aead_unmap(jrdev, edesc, req); 975 976 kfree(edesc); 977 978 /* 979 * If no backlog flag, the completion of the request is done 980 * by CAAM, not crypto engine. 981 */ 982 if (!edesc->bklog) 983 aead_request_complete(req, ecode); 984 else 985 crypto_finalize_aead_request(jrp->engine, req, ecode); 986 } 987 988 static void skcipher_crypt_done(struct device *jrdev, u32 *desc, u32 err, 989 void *context) 990 { 991 struct skcipher_request *req = context; 992 struct skcipher_edesc *edesc; 993 struct caam_skcipher_req_ctx *rctx = skcipher_request_ctx(req); 994 struct crypto_skcipher *skcipher = crypto_skcipher_reqtfm(req); 995 struct caam_drv_private_jr *jrp = dev_get_drvdata(jrdev); 996 int ivsize = crypto_skcipher_ivsize(skcipher); 997 int ecode = 0; 998 999 dev_dbg(jrdev, "%s %d: err 0x%x\n", __func__, __LINE__, err); 1000 1001 edesc = rctx->edesc; 1002 if (err) 1003 ecode = caam_jr_strstatus(jrdev, err); 1004 1005 skcipher_unmap(jrdev, edesc, req); 1006 1007 /* 1008 * The crypto API expects us to set the IV (req->iv) to the last 1009 * ciphertext block (CBC mode) or last counter (CTR mode). 1010 * This is used e.g. by the CTS mode. 1011 */ 1012 if (ivsize && !ecode) { 1013 memcpy(req->iv, (u8 *)edesc->sec4_sg + edesc->sec4_sg_bytes, 1014 ivsize); 1015 1016 print_hex_dump_debug("dstiv @" __stringify(__LINE__)": ", 1017 DUMP_PREFIX_ADDRESS, 16, 4, req->iv, 1018 ivsize, 1); 1019 } 1020 1021 caam_dump_sg("dst @" __stringify(__LINE__)": ", 1022 DUMP_PREFIX_ADDRESS, 16, 4, req->dst, 1023 edesc->dst_nents > 1 ? 100 : req->cryptlen, 1); 1024 1025 kfree(edesc); 1026 1027 /* 1028 * If no backlog flag, the completion of the request is done 1029 * by CAAM, not crypto engine. 1030 */ 1031 if (!edesc->bklog) 1032 skcipher_request_complete(req, ecode); 1033 else 1034 crypto_finalize_skcipher_request(jrp->engine, req, ecode); 1035 } 1036 1037 /* 1038 * Fill in aead job descriptor 1039 */ 1040 static void init_aead_job(struct aead_request *req, 1041 struct aead_edesc *edesc, 1042 bool all_contig, bool encrypt) 1043 { 1044 struct crypto_aead *aead = crypto_aead_reqtfm(req); 1045 struct caam_ctx *ctx = crypto_aead_ctx(aead); 1046 int authsize = ctx->authsize; 1047 u32 *desc = edesc->hw_desc; 1048 u32 out_options, in_options; 1049 dma_addr_t dst_dma, src_dma; 1050 int len, sec4_sg_index = 0; 1051 dma_addr_t ptr; 1052 u32 *sh_desc; 1053 1054 sh_desc = encrypt ? ctx->sh_desc_enc : ctx->sh_desc_dec; 1055 ptr = encrypt ? ctx->sh_desc_enc_dma : ctx->sh_desc_dec_dma; 1056 1057 len = desc_len(sh_desc); 1058 init_job_desc_shared(desc, ptr, len, HDR_SHARE_DEFER | HDR_REVERSE); 1059 1060 if (all_contig) { 1061 src_dma = edesc->mapped_src_nents ? sg_dma_address(req->src) : 1062 0; 1063 in_options = 0; 1064 } else { 1065 src_dma = edesc->sec4_sg_dma; 1066 sec4_sg_index += edesc->mapped_src_nents; 1067 in_options = LDST_SGF; 1068 } 1069 1070 append_seq_in_ptr(desc, src_dma, req->assoclen + req->cryptlen, 1071 in_options); 1072 1073 dst_dma = src_dma; 1074 out_options = in_options; 1075 1076 if (unlikely(req->src != req->dst)) { 1077 if (!edesc->mapped_dst_nents) { 1078 dst_dma = 0; 1079 out_options = 0; 1080 } else if (edesc->mapped_dst_nents == 1) { 1081 dst_dma = sg_dma_address(req->dst); 1082 out_options = 0; 1083 } else { 1084 dst_dma = edesc->sec4_sg_dma + 1085 sec4_sg_index * 1086 sizeof(struct sec4_sg_entry); 1087 out_options = LDST_SGF; 1088 } 1089 } 1090 1091 if (encrypt) 1092 append_seq_out_ptr(desc, dst_dma, 1093 req->assoclen + req->cryptlen + authsize, 1094 out_options); 1095 else 1096 append_seq_out_ptr(desc, dst_dma, 1097 req->assoclen + req->cryptlen - authsize, 1098 out_options); 1099 } 1100 1101 static void init_gcm_job(struct aead_request *req, 1102 struct aead_edesc *edesc, 1103 bool all_contig, bool encrypt) 1104 { 1105 struct crypto_aead *aead = crypto_aead_reqtfm(req); 1106 struct caam_ctx *ctx = crypto_aead_ctx(aead); 1107 unsigned int ivsize = crypto_aead_ivsize(aead); 1108 u32 *desc = edesc->hw_desc; 1109 bool generic_gcm = (ivsize == GCM_AES_IV_SIZE); 1110 unsigned int last; 1111 1112 init_aead_job(req, edesc, all_contig, encrypt); 1113 append_math_add_imm_u32(desc, REG3, ZERO, IMM, req->assoclen); 1114 1115 /* BUG This should not be specific to generic GCM. */ 1116 last = 0; 1117 if (encrypt && generic_gcm && !(req->assoclen + req->cryptlen)) 1118 last = FIFOLD_TYPE_LAST1; 1119 1120 /* Read GCM IV */ 1121 append_cmd(desc, CMD_FIFO_LOAD | FIFOLD_CLASS_CLASS1 | IMMEDIATE | 1122 FIFOLD_TYPE_IV | FIFOLD_TYPE_FLUSH1 | GCM_AES_IV_SIZE | last); 1123 /* Append Salt */ 1124 if (!generic_gcm) 1125 append_data(desc, ctx->key + ctx->cdata.keylen, 4); 1126 /* Append IV */ 1127 append_data(desc, req->iv, ivsize); 1128 /* End of blank commands */ 1129 } 1130 1131 static void init_chachapoly_job(struct aead_request *req, 1132 struct aead_edesc *edesc, bool all_contig, 1133 bool encrypt) 1134 { 1135 struct crypto_aead *aead = crypto_aead_reqtfm(req); 1136 unsigned int ivsize = crypto_aead_ivsize(aead); 1137 unsigned int assoclen = req->assoclen; 1138 u32 *desc = edesc->hw_desc; 1139 u32 ctx_iv_off = 4; 1140 1141 init_aead_job(req, edesc, all_contig, encrypt); 1142 1143 if (ivsize != CHACHAPOLY_IV_SIZE) { 1144 /* IPsec specific: CONTEXT1[223:128] = {NONCE, IV} */ 1145 ctx_iv_off += 4; 1146 1147 /* 1148 * The associated data comes already with the IV but we need 1149 * to skip it when we authenticate or encrypt... 1150 */ 1151 assoclen -= ivsize; 1152 } 1153 1154 append_math_add_imm_u32(desc, REG3, ZERO, IMM, assoclen); 1155 1156 /* 1157 * For IPsec load the IV further in the same register. 1158 * For RFC7539 simply load the 12 bytes nonce in a single operation 1159 */ 1160 append_load_as_imm(desc, req->iv, ivsize, LDST_CLASS_1_CCB | 1161 LDST_SRCDST_BYTE_CONTEXT | 1162 ctx_iv_off << LDST_OFFSET_SHIFT); 1163 } 1164 1165 static void init_authenc_job(struct aead_request *req, 1166 struct aead_edesc *edesc, 1167 bool all_contig, bool encrypt) 1168 { 1169 struct crypto_aead *aead = crypto_aead_reqtfm(req); 1170 struct caam_aead_alg *alg = container_of(crypto_aead_alg(aead), 1171 struct caam_aead_alg, aead); 1172 unsigned int ivsize = crypto_aead_ivsize(aead); 1173 struct caam_ctx *ctx = crypto_aead_ctx(aead); 1174 struct caam_drv_private *ctrlpriv = dev_get_drvdata(ctx->jrdev->parent); 1175 const bool ctr_mode = ((ctx->cdata.algtype & OP_ALG_AAI_MASK) == 1176 OP_ALG_AAI_CTR_MOD128); 1177 const bool is_rfc3686 = alg->caam.rfc3686; 1178 u32 *desc = edesc->hw_desc; 1179 u32 ivoffset = 0; 1180 1181 /* 1182 * AES-CTR needs to load IV in CONTEXT1 reg 1183 * at an offset of 128bits (16bytes) 1184 * CONTEXT1[255:128] = IV 1185 */ 1186 if (ctr_mode) 1187 ivoffset = 16; 1188 1189 /* 1190 * RFC3686 specific: 1191 * CONTEXT1[255:128] = {NONCE, IV, COUNTER} 1192 */ 1193 if (is_rfc3686) 1194 ivoffset = 16 + CTR_RFC3686_NONCE_SIZE; 1195 1196 init_aead_job(req, edesc, all_contig, encrypt); 1197 1198 /* 1199 * {REG3, DPOVRD} = assoclen, depending on whether MATH command supports 1200 * having DPOVRD as destination. 1201 */ 1202 if (ctrlpriv->era < 3) 1203 append_math_add_imm_u32(desc, REG3, ZERO, IMM, req->assoclen); 1204 else 1205 append_math_add_imm_u32(desc, DPOVRD, ZERO, IMM, req->assoclen); 1206 1207 if (ivsize && ((is_rfc3686 && encrypt) || !alg->caam.geniv)) 1208 append_load_as_imm(desc, req->iv, ivsize, 1209 LDST_CLASS_1_CCB | 1210 LDST_SRCDST_BYTE_CONTEXT | 1211 (ivoffset << LDST_OFFSET_SHIFT)); 1212 } 1213 1214 /* 1215 * Fill in skcipher job descriptor 1216 */ 1217 static void init_skcipher_job(struct skcipher_request *req, 1218 struct skcipher_edesc *edesc, 1219 const bool encrypt) 1220 { 1221 struct crypto_skcipher *skcipher = crypto_skcipher_reqtfm(req); 1222 struct caam_ctx *ctx = crypto_skcipher_ctx(skcipher); 1223 struct device *jrdev = ctx->jrdev; 1224 int ivsize = crypto_skcipher_ivsize(skcipher); 1225 u32 *desc = edesc->hw_desc; 1226 u32 *sh_desc; 1227 u32 in_options = 0, out_options = 0; 1228 dma_addr_t src_dma, dst_dma, ptr; 1229 int len, sec4_sg_index = 0; 1230 1231 print_hex_dump_debug("presciv@"__stringify(__LINE__)": ", 1232 DUMP_PREFIX_ADDRESS, 16, 4, req->iv, ivsize, 1); 1233 dev_dbg(jrdev, "asked=%d, cryptlen%d\n", 1234 (int)edesc->src_nents > 1 ? 100 : req->cryptlen, req->cryptlen); 1235 1236 caam_dump_sg("src @" __stringify(__LINE__)": ", 1237 DUMP_PREFIX_ADDRESS, 16, 4, req->src, 1238 edesc->src_nents > 1 ? 100 : req->cryptlen, 1); 1239 1240 sh_desc = encrypt ? ctx->sh_desc_enc : ctx->sh_desc_dec; 1241 ptr = encrypt ? ctx->sh_desc_enc_dma : ctx->sh_desc_dec_dma; 1242 1243 len = desc_len(sh_desc); 1244 init_job_desc_shared(desc, ptr, len, HDR_SHARE_DEFER | HDR_REVERSE); 1245 1246 if (ivsize || edesc->mapped_src_nents > 1) { 1247 src_dma = edesc->sec4_sg_dma; 1248 sec4_sg_index = edesc->mapped_src_nents + !!ivsize; 1249 in_options = LDST_SGF; 1250 } else { 1251 src_dma = sg_dma_address(req->src); 1252 } 1253 1254 append_seq_in_ptr(desc, src_dma, req->cryptlen + ivsize, in_options); 1255 1256 if (likely(req->src == req->dst)) { 1257 dst_dma = src_dma + !!ivsize * sizeof(struct sec4_sg_entry); 1258 out_options = in_options; 1259 } else if (!ivsize && edesc->mapped_dst_nents == 1) { 1260 dst_dma = sg_dma_address(req->dst); 1261 } else { 1262 dst_dma = edesc->sec4_sg_dma + sec4_sg_index * 1263 sizeof(struct sec4_sg_entry); 1264 out_options = LDST_SGF; 1265 } 1266 1267 append_seq_out_ptr(desc, dst_dma, req->cryptlen + ivsize, out_options); 1268 } 1269 1270 /* 1271 * allocate and map the aead extended descriptor 1272 */ 1273 static struct aead_edesc *aead_edesc_alloc(struct aead_request *req, 1274 int desc_bytes, bool *all_contig_ptr, 1275 bool encrypt) 1276 { 1277 struct crypto_aead *aead = crypto_aead_reqtfm(req); 1278 struct caam_ctx *ctx = crypto_aead_ctx(aead); 1279 struct device *jrdev = ctx->jrdev; 1280 struct caam_aead_req_ctx *rctx = aead_request_ctx(req); 1281 gfp_t flags = (req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP) ? 1282 GFP_KERNEL : GFP_ATOMIC; 1283 int src_nents, mapped_src_nents, dst_nents = 0, mapped_dst_nents = 0; 1284 int src_len, dst_len = 0; 1285 struct aead_edesc *edesc; 1286 int sec4_sg_index, sec4_sg_len, sec4_sg_bytes; 1287 unsigned int authsize = ctx->authsize; 1288 1289 if (unlikely(req->dst != req->src)) { 1290 src_len = req->assoclen + req->cryptlen; 1291 dst_len = src_len + (encrypt ? authsize : (-authsize)); 1292 1293 src_nents = sg_nents_for_len(req->src, src_len); 1294 if (unlikely(src_nents < 0)) { 1295 dev_err(jrdev, "Insufficient bytes (%d) in src S/G\n", 1296 src_len); 1297 return ERR_PTR(src_nents); 1298 } 1299 1300 dst_nents = sg_nents_for_len(req->dst, dst_len); 1301 if (unlikely(dst_nents < 0)) { 1302 dev_err(jrdev, "Insufficient bytes (%d) in dst S/G\n", 1303 dst_len); 1304 return ERR_PTR(dst_nents); 1305 } 1306 } else { 1307 src_len = req->assoclen + req->cryptlen + 1308 (encrypt ? authsize : 0); 1309 1310 src_nents = sg_nents_for_len(req->src, src_len); 1311 if (unlikely(src_nents < 0)) { 1312 dev_err(jrdev, "Insufficient bytes (%d) in src S/G\n", 1313 src_len); 1314 return ERR_PTR(src_nents); 1315 } 1316 } 1317 1318 if (likely(req->src == req->dst)) { 1319 mapped_src_nents = dma_map_sg(jrdev, req->src, src_nents, 1320 DMA_BIDIRECTIONAL); 1321 if (unlikely(!mapped_src_nents)) { 1322 dev_err(jrdev, "unable to map source\n"); 1323 return ERR_PTR(-ENOMEM); 1324 } 1325 } else { 1326 /* Cover also the case of null (zero length) input data */ 1327 if (src_nents) { 1328 mapped_src_nents = dma_map_sg(jrdev, req->src, 1329 src_nents, DMA_TO_DEVICE); 1330 if (unlikely(!mapped_src_nents)) { 1331 dev_err(jrdev, "unable to map source\n"); 1332 return ERR_PTR(-ENOMEM); 1333 } 1334 } else { 1335 mapped_src_nents = 0; 1336 } 1337 1338 /* Cover also the case of null (zero length) output data */ 1339 if (dst_nents) { 1340 mapped_dst_nents = dma_map_sg(jrdev, req->dst, 1341 dst_nents, 1342 DMA_FROM_DEVICE); 1343 if (unlikely(!mapped_dst_nents)) { 1344 dev_err(jrdev, "unable to map destination\n"); 1345 dma_unmap_sg(jrdev, req->src, src_nents, 1346 DMA_TO_DEVICE); 1347 return ERR_PTR(-ENOMEM); 1348 } 1349 } else { 1350 mapped_dst_nents = 0; 1351 } 1352 } 1353 1354 /* 1355 * HW reads 4 S/G entries at a time; make sure the reads don't go beyond 1356 * the end of the table by allocating more S/G entries. 1357 */ 1358 sec4_sg_len = mapped_src_nents > 1 ? mapped_src_nents : 0; 1359 if (mapped_dst_nents > 1) 1360 sec4_sg_len += pad_sg_nents(mapped_dst_nents); 1361 else 1362 sec4_sg_len = pad_sg_nents(sec4_sg_len); 1363 1364 sec4_sg_bytes = sec4_sg_len * sizeof(struct sec4_sg_entry); 1365 1366 /* allocate space for base edesc and hw desc commands, link tables */ 1367 edesc = kzalloc(sizeof(*edesc) + desc_bytes + sec4_sg_bytes, 1368 GFP_DMA | flags); 1369 if (!edesc) { 1370 caam_unmap(jrdev, req->src, req->dst, src_nents, dst_nents, 0, 1371 0, 0, 0); 1372 return ERR_PTR(-ENOMEM); 1373 } 1374 1375 edesc->src_nents = src_nents; 1376 edesc->dst_nents = dst_nents; 1377 edesc->mapped_src_nents = mapped_src_nents; 1378 edesc->mapped_dst_nents = mapped_dst_nents; 1379 edesc->sec4_sg = (void *)edesc + sizeof(struct aead_edesc) + 1380 desc_bytes; 1381 1382 rctx->edesc = edesc; 1383 1384 *all_contig_ptr = !(mapped_src_nents > 1); 1385 1386 sec4_sg_index = 0; 1387 if (mapped_src_nents > 1) { 1388 sg_to_sec4_sg_last(req->src, src_len, 1389 edesc->sec4_sg + sec4_sg_index, 0); 1390 sec4_sg_index += mapped_src_nents; 1391 } 1392 if (mapped_dst_nents > 1) { 1393 sg_to_sec4_sg_last(req->dst, dst_len, 1394 edesc->sec4_sg + sec4_sg_index, 0); 1395 } 1396 1397 if (!sec4_sg_bytes) 1398 return edesc; 1399 1400 edesc->sec4_sg_dma = dma_map_single(jrdev, edesc->sec4_sg, 1401 sec4_sg_bytes, DMA_TO_DEVICE); 1402 if (dma_mapping_error(jrdev, edesc->sec4_sg_dma)) { 1403 dev_err(jrdev, "unable to map S/G table\n"); 1404 aead_unmap(jrdev, edesc, req); 1405 kfree(edesc); 1406 return ERR_PTR(-ENOMEM); 1407 } 1408 1409 edesc->sec4_sg_bytes = sec4_sg_bytes; 1410 1411 return edesc; 1412 } 1413 1414 static int aead_enqueue_req(struct device *jrdev, struct aead_request *req) 1415 { 1416 struct caam_drv_private_jr *jrpriv = dev_get_drvdata(jrdev); 1417 struct caam_aead_req_ctx *rctx = aead_request_ctx(req); 1418 struct aead_edesc *edesc = rctx->edesc; 1419 u32 *desc = edesc->hw_desc; 1420 int ret; 1421 1422 /* 1423 * Only the backlog request are sent to crypto-engine since the others 1424 * can be handled by CAAM, if free, especially since JR has up to 1024 1425 * entries (more than the 10 entries from crypto-engine). 1426 */ 1427 if (req->base.flags & CRYPTO_TFM_REQ_MAY_BACKLOG) 1428 ret = crypto_transfer_aead_request_to_engine(jrpriv->engine, 1429 req); 1430 else 1431 ret = caam_jr_enqueue(jrdev, desc, aead_crypt_done, req); 1432 1433 if ((ret != -EINPROGRESS) && (ret != -EBUSY)) { 1434 aead_unmap(jrdev, edesc, req); 1435 kfree(rctx->edesc); 1436 } 1437 1438 return ret; 1439 } 1440 1441 static inline int chachapoly_crypt(struct aead_request *req, bool encrypt) 1442 { 1443 struct aead_edesc *edesc; 1444 struct crypto_aead *aead = crypto_aead_reqtfm(req); 1445 struct caam_ctx *ctx = crypto_aead_ctx(aead); 1446 struct device *jrdev = ctx->jrdev; 1447 bool all_contig; 1448 u32 *desc; 1449 1450 edesc = aead_edesc_alloc(req, CHACHAPOLY_DESC_JOB_IO_LEN, &all_contig, 1451 encrypt); 1452 if (IS_ERR(edesc)) 1453 return PTR_ERR(edesc); 1454 1455 desc = edesc->hw_desc; 1456 1457 init_chachapoly_job(req, edesc, all_contig, encrypt); 1458 print_hex_dump_debug("chachapoly jobdesc@" __stringify(__LINE__)": ", 1459 DUMP_PREFIX_ADDRESS, 16, 4, desc, desc_bytes(desc), 1460 1); 1461 1462 return aead_enqueue_req(jrdev, req); 1463 } 1464 1465 static int chachapoly_encrypt(struct aead_request *req) 1466 { 1467 return chachapoly_crypt(req, true); 1468 } 1469 1470 static int chachapoly_decrypt(struct aead_request *req) 1471 { 1472 return chachapoly_crypt(req, false); 1473 } 1474 1475 static inline int aead_crypt(struct aead_request *req, bool encrypt) 1476 { 1477 struct aead_edesc *edesc; 1478 struct crypto_aead *aead = crypto_aead_reqtfm(req); 1479 struct caam_ctx *ctx = crypto_aead_ctx(aead); 1480 struct device *jrdev = ctx->jrdev; 1481 bool all_contig; 1482 1483 /* allocate extended descriptor */ 1484 edesc = aead_edesc_alloc(req, AUTHENC_DESC_JOB_IO_LEN, 1485 &all_contig, encrypt); 1486 if (IS_ERR(edesc)) 1487 return PTR_ERR(edesc); 1488 1489 /* Create and submit job descriptor */ 1490 init_authenc_job(req, edesc, all_contig, encrypt); 1491 1492 print_hex_dump_debug("aead jobdesc@"__stringify(__LINE__)": ", 1493 DUMP_PREFIX_ADDRESS, 16, 4, edesc->hw_desc, 1494 desc_bytes(edesc->hw_desc), 1); 1495 1496 return aead_enqueue_req(jrdev, req); 1497 } 1498 1499 static int aead_encrypt(struct aead_request *req) 1500 { 1501 return aead_crypt(req, true); 1502 } 1503 1504 static int aead_decrypt(struct aead_request *req) 1505 { 1506 return aead_crypt(req, false); 1507 } 1508 1509 static int aead_do_one_req(struct crypto_engine *engine, void *areq) 1510 { 1511 struct aead_request *req = aead_request_cast(areq); 1512 struct caam_ctx *ctx = crypto_aead_ctx(crypto_aead_reqtfm(req)); 1513 struct caam_aead_req_ctx *rctx = aead_request_ctx(req); 1514 u32 *desc = rctx->edesc->hw_desc; 1515 int ret; 1516 1517 rctx->edesc->bklog = true; 1518 1519 ret = caam_jr_enqueue(ctx->jrdev, desc, aead_crypt_done, req); 1520 1521 if (ret != -EINPROGRESS) { 1522 aead_unmap(ctx->jrdev, rctx->edesc, req); 1523 kfree(rctx->edesc); 1524 } else { 1525 ret = 0; 1526 } 1527 1528 return ret; 1529 } 1530 1531 static inline int gcm_crypt(struct aead_request *req, bool encrypt) 1532 { 1533 struct aead_edesc *edesc; 1534 struct crypto_aead *aead = crypto_aead_reqtfm(req); 1535 struct caam_ctx *ctx = crypto_aead_ctx(aead); 1536 struct device *jrdev = ctx->jrdev; 1537 bool all_contig; 1538 1539 /* allocate extended descriptor */ 1540 edesc = aead_edesc_alloc(req, GCM_DESC_JOB_IO_LEN, &all_contig, 1541 encrypt); 1542 if (IS_ERR(edesc)) 1543 return PTR_ERR(edesc); 1544 1545 /* Create and submit job descriptor */ 1546 init_gcm_job(req, edesc, all_contig, encrypt); 1547 1548 print_hex_dump_debug("aead jobdesc@"__stringify(__LINE__)": ", 1549 DUMP_PREFIX_ADDRESS, 16, 4, edesc->hw_desc, 1550 desc_bytes(edesc->hw_desc), 1); 1551 1552 return aead_enqueue_req(jrdev, req); 1553 } 1554 1555 static int gcm_encrypt(struct aead_request *req) 1556 { 1557 return gcm_crypt(req, true); 1558 } 1559 1560 static int gcm_decrypt(struct aead_request *req) 1561 { 1562 return gcm_crypt(req, false); 1563 } 1564 1565 static int ipsec_gcm_encrypt(struct aead_request *req) 1566 { 1567 return crypto_ipsec_check_assoclen(req->assoclen) ? : gcm_encrypt(req); 1568 } 1569 1570 static int ipsec_gcm_decrypt(struct aead_request *req) 1571 { 1572 return crypto_ipsec_check_assoclen(req->assoclen) ? : gcm_decrypt(req); 1573 } 1574 1575 /* 1576 * allocate and map the skcipher extended descriptor for skcipher 1577 */ 1578 static struct skcipher_edesc *skcipher_edesc_alloc(struct skcipher_request *req, 1579 int desc_bytes) 1580 { 1581 struct crypto_skcipher *skcipher = crypto_skcipher_reqtfm(req); 1582 struct caam_ctx *ctx = crypto_skcipher_ctx(skcipher); 1583 struct caam_skcipher_req_ctx *rctx = skcipher_request_ctx(req); 1584 struct device *jrdev = ctx->jrdev; 1585 gfp_t flags = (req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP) ? 1586 GFP_KERNEL : GFP_ATOMIC; 1587 int src_nents, mapped_src_nents, dst_nents = 0, mapped_dst_nents = 0; 1588 struct skcipher_edesc *edesc; 1589 dma_addr_t iv_dma = 0; 1590 u8 *iv; 1591 int ivsize = crypto_skcipher_ivsize(skcipher); 1592 int dst_sg_idx, sec4_sg_ents, sec4_sg_bytes; 1593 1594 src_nents = sg_nents_for_len(req->src, req->cryptlen); 1595 if (unlikely(src_nents < 0)) { 1596 dev_err(jrdev, "Insufficient bytes (%d) in src S/G\n", 1597 req->cryptlen); 1598 return ERR_PTR(src_nents); 1599 } 1600 1601 if (req->dst != req->src) { 1602 dst_nents = sg_nents_for_len(req->dst, req->cryptlen); 1603 if (unlikely(dst_nents < 0)) { 1604 dev_err(jrdev, "Insufficient bytes (%d) in dst S/G\n", 1605 req->cryptlen); 1606 return ERR_PTR(dst_nents); 1607 } 1608 } 1609 1610 if (likely(req->src == req->dst)) { 1611 mapped_src_nents = dma_map_sg(jrdev, req->src, src_nents, 1612 DMA_BIDIRECTIONAL); 1613 if (unlikely(!mapped_src_nents)) { 1614 dev_err(jrdev, "unable to map source\n"); 1615 return ERR_PTR(-ENOMEM); 1616 } 1617 } else { 1618 mapped_src_nents = dma_map_sg(jrdev, req->src, src_nents, 1619 DMA_TO_DEVICE); 1620 if (unlikely(!mapped_src_nents)) { 1621 dev_err(jrdev, "unable to map source\n"); 1622 return ERR_PTR(-ENOMEM); 1623 } 1624 mapped_dst_nents = dma_map_sg(jrdev, req->dst, dst_nents, 1625 DMA_FROM_DEVICE); 1626 if (unlikely(!mapped_dst_nents)) { 1627 dev_err(jrdev, "unable to map destination\n"); 1628 dma_unmap_sg(jrdev, req->src, src_nents, DMA_TO_DEVICE); 1629 return ERR_PTR(-ENOMEM); 1630 } 1631 } 1632 1633 if (!ivsize && mapped_src_nents == 1) 1634 sec4_sg_ents = 0; // no need for an input hw s/g table 1635 else 1636 sec4_sg_ents = mapped_src_nents + !!ivsize; 1637 dst_sg_idx = sec4_sg_ents; 1638 1639 /* 1640 * Input, output HW S/G tables: [IV, src][dst, IV] 1641 * IV entries point to the same buffer 1642 * If src == dst, S/G entries are reused (S/G tables overlap) 1643 * 1644 * HW reads 4 S/G entries at a time; make sure the reads don't go beyond 1645 * the end of the table by allocating more S/G entries. Logic: 1646 * if (output S/G) 1647 * pad output S/G, if needed 1648 * else if (input S/G) ... 1649 * pad input S/G, if needed 1650 */ 1651 if (ivsize || mapped_dst_nents > 1) { 1652 if (req->src == req->dst) 1653 sec4_sg_ents = !!ivsize + pad_sg_nents(sec4_sg_ents); 1654 else 1655 sec4_sg_ents += pad_sg_nents(mapped_dst_nents + 1656 !!ivsize); 1657 } else { 1658 sec4_sg_ents = pad_sg_nents(sec4_sg_ents); 1659 } 1660 1661 sec4_sg_bytes = sec4_sg_ents * sizeof(struct sec4_sg_entry); 1662 1663 /* 1664 * allocate space for base edesc and hw desc commands, link tables, IV 1665 */ 1666 edesc = kzalloc(sizeof(*edesc) + desc_bytes + sec4_sg_bytes + ivsize, 1667 GFP_DMA | flags); 1668 if (!edesc) { 1669 dev_err(jrdev, "could not allocate extended descriptor\n"); 1670 caam_unmap(jrdev, req->src, req->dst, src_nents, dst_nents, 0, 1671 0, 0, 0); 1672 return ERR_PTR(-ENOMEM); 1673 } 1674 1675 edesc->src_nents = src_nents; 1676 edesc->dst_nents = dst_nents; 1677 edesc->mapped_src_nents = mapped_src_nents; 1678 edesc->mapped_dst_nents = mapped_dst_nents; 1679 edesc->sec4_sg_bytes = sec4_sg_bytes; 1680 edesc->sec4_sg = (struct sec4_sg_entry *)((u8 *)edesc->hw_desc + 1681 desc_bytes); 1682 rctx->edesc = edesc; 1683 1684 /* Make sure IV is located in a DMAable area */ 1685 if (ivsize) { 1686 iv = (u8 *)edesc->sec4_sg + sec4_sg_bytes; 1687 memcpy(iv, req->iv, ivsize); 1688 1689 iv_dma = dma_map_single(jrdev, iv, ivsize, DMA_BIDIRECTIONAL); 1690 if (dma_mapping_error(jrdev, iv_dma)) { 1691 dev_err(jrdev, "unable to map IV\n"); 1692 caam_unmap(jrdev, req->src, req->dst, src_nents, 1693 dst_nents, 0, 0, 0, 0); 1694 kfree(edesc); 1695 return ERR_PTR(-ENOMEM); 1696 } 1697 1698 dma_to_sec4_sg_one(edesc->sec4_sg, iv_dma, ivsize, 0); 1699 } 1700 if (dst_sg_idx) 1701 sg_to_sec4_sg(req->src, req->cryptlen, edesc->sec4_sg + 1702 !!ivsize, 0); 1703 1704 if (req->src != req->dst && (ivsize || mapped_dst_nents > 1)) 1705 sg_to_sec4_sg(req->dst, req->cryptlen, edesc->sec4_sg + 1706 dst_sg_idx, 0); 1707 1708 if (ivsize) 1709 dma_to_sec4_sg_one(edesc->sec4_sg + dst_sg_idx + 1710 mapped_dst_nents, iv_dma, ivsize, 0); 1711 1712 if (ivsize || mapped_dst_nents > 1) 1713 sg_to_sec4_set_last(edesc->sec4_sg + dst_sg_idx + 1714 mapped_dst_nents); 1715 1716 if (sec4_sg_bytes) { 1717 edesc->sec4_sg_dma = dma_map_single(jrdev, edesc->sec4_sg, 1718 sec4_sg_bytes, 1719 DMA_TO_DEVICE); 1720 if (dma_mapping_error(jrdev, edesc->sec4_sg_dma)) { 1721 dev_err(jrdev, "unable to map S/G table\n"); 1722 caam_unmap(jrdev, req->src, req->dst, src_nents, 1723 dst_nents, iv_dma, ivsize, 0, 0); 1724 kfree(edesc); 1725 return ERR_PTR(-ENOMEM); 1726 } 1727 } 1728 1729 edesc->iv_dma = iv_dma; 1730 1731 print_hex_dump_debug("skcipher sec4_sg@" __stringify(__LINE__)": ", 1732 DUMP_PREFIX_ADDRESS, 16, 4, edesc->sec4_sg, 1733 sec4_sg_bytes, 1); 1734 1735 return edesc; 1736 } 1737 1738 static int skcipher_do_one_req(struct crypto_engine *engine, void *areq) 1739 { 1740 struct skcipher_request *req = skcipher_request_cast(areq); 1741 struct caam_ctx *ctx = crypto_skcipher_ctx(crypto_skcipher_reqtfm(req)); 1742 struct caam_skcipher_req_ctx *rctx = skcipher_request_ctx(req); 1743 u32 *desc = rctx->edesc->hw_desc; 1744 int ret; 1745 1746 rctx->edesc->bklog = true; 1747 1748 ret = caam_jr_enqueue(ctx->jrdev, desc, skcipher_crypt_done, req); 1749 1750 if (ret != -EINPROGRESS) { 1751 skcipher_unmap(ctx->jrdev, rctx->edesc, req); 1752 kfree(rctx->edesc); 1753 } else { 1754 ret = 0; 1755 } 1756 1757 return ret; 1758 } 1759 1760 static inline int skcipher_crypt(struct skcipher_request *req, bool encrypt) 1761 { 1762 struct skcipher_edesc *edesc; 1763 struct crypto_skcipher *skcipher = crypto_skcipher_reqtfm(req); 1764 struct caam_ctx *ctx = crypto_skcipher_ctx(skcipher); 1765 struct device *jrdev = ctx->jrdev; 1766 struct caam_drv_private_jr *jrpriv = dev_get_drvdata(jrdev); 1767 u32 *desc; 1768 int ret = 0; 1769 1770 if (!req->cryptlen) 1771 return 0; 1772 1773 /* allocate extended descriptor */ 1774 edesc = skcipher_edesc_alloc(req, DESC_JOB_IO_LEN * CAAM_CMD_SZ); 1775 if (IS_ERR(edesc)) 1776 return PTR_ERR(edesc); 1777 1778 /* Create and submit job descriptor*/ 1779 init_skcipher_job(req, edesc, encrypt); 1780 1781 print_hex_dump_debug("skcipher jobdesc@" __stringify(__LINE__)": ", 1782 DUMP_PREFIX_ADDRESS, 16, 4, edesc->hw_desc, 1783 desc_bytes(edesc->hw_desc), 1); 1784 1785 desc = edesc->hw_desc; 1786 /* 1787 * Only the backlog request are sent to crypto-engine since the others 1788 * can be handled by CAAM, if free, especially since JR has up to 1024 1789 * entries (more than the 10 entries from crypto-engine). 1790 */ 1791 if (req->base.flags & CRYPTO_TFM_REQ_MAY_BACKLOG) 1792 ret = crypto_transfer_skcipher_request_to_engine(jrpriv->engine, 1793 req); 1794 else 1795 ret = caam_jr_enqueue(jrdev, desc, skcipher_crypt_done, req); 1796 1797 if ((ret != -EINPROGRESS) && (ret != -EBUSY)) { 1798 skcipher_unmap(jrdev, edesc, req); 1799 kfree(edesc); 1800 } 1801 1802 return ret; 1803 } 1804 1805 static int skcipher_encrypt(struct skcipher_request *req) 1806 { 1807 return skcipher_crypt(req, true); 1808 } 1809 1810 static int skcipher_decrypt(struct skcipher_request *req) 1811 { 1812 return skcipher_crypt(req, false); 1813 } 1814 1815 static struct caam_skcipher_alg driver_algs[] = { 1816 { 1817 .skcipher = { 1818 .base = { 1819 .cra_name = "cbc(aes)", 1820 .cra_driver_name = "cbc-aes-caam", 1821 .cra_blocksize = AES_BLOCK_SIZE, 1822 }, 1823 .setkey = aes_skcipher_setkey, 1824 .encrypt = skcipher_encrypt, 1825 .decrypt = skcipher_decrypt, 1826 .min_keysize = AES_MIN_KEY_SIZE, 1827 .max_keysize = AES_MAX_KEY_SIZE, 1828 .ivsize = AES_BLOCK_SIZE, 1829 }, 1830 .caam.class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CBC, 1831 }, 1832 { 1833 .skcipher = { 1834 .base = { 1835 .cra_name = "cbc(des3_ede)", 1836 .cra_driver_name = "cbc-3des-caam", 1837 .cra_blocksize = DES3_EDE_BLOCK_SIZE, 1838 }, 1839 .setkey = des3_skcipher_setkey, 1840 .encrypt = skcipher_encrypt, 1841 .decrypt = skcipher_decrypt, 1842 .min_keysize = DES3_EDE_KEY_SIZE, 1843 .max_keysize = DES3_EDE_KEY_SIZE, 1844 .ivsize = DES3_EDE_BLOCK_SIZE, 1845 }, 1846 .caam.class1_alg_type = OP_ALG_ALGSEL_3DES | OP_ALG_AAI_CBC, 1847 }, 1848 { 1849 .skcipher = { 1850 .base = { 1851 .cra_name = "cbc(des)", 1852 .cra_driver_name = "cbc-des-caam", 1853 .cra_blocksize = DES_BLOCK_SIZE, 1854 }, 1855 .setkey = des_skcipher_setkey, 1856 .encrypt = skcipher_encrypt, 1857 .decrypt = skcipher_decrypt, 1858 .min_keysize = DES_KEY_SIZE, 1859 .max_keysize = DES_KEY_SIZE, 1860 .ivsize = DES_BLOCK_SIZE, 1861 }, 1862 .caam.class1_alg_type = OP_ALG_ALGSEL_DES | OP_ALG_AAI_CBC, 1863 }, 1864 { 1865 .skcipher = { 1866 .base = { 1867 .cra_name = "ctr(aes)", 1868 .cra_driver_name = "ctr-aes-caam", 1869 .cra_blocksize = 1, 1870 }, 1871 .setkey = ctr_skcipher_setkey, 1872 .encrypt = skcipher_encrypt, 1873 .decrypt = skcipher_decrypt, 1874 .min_keysize = AES_MIN_KEY_SIZE, 1875 .max_keysize = AES_MAX_KEY_SIZE, 1876 .ivsize = AES_BLOCK_SIZE, 1877 .chunksize = AES_BLOCK_SIZE, 1878 }, 1879 .caam.class1_alg_type = OP_ALG_ALGSEL_AES | 1880 OP_ALG_AAI_CTR_MOD128, 1881 }, 1882 { 1883 .skcipher = { 1884 .base = { 1885 .cra_name = "rfc3686(ctr(aes))", 1886 .cra_driver_name = "rfc3686-ctr-aes-caam", 1887 .cra_blocksize = 1, 1888 }, 1889 .setkey = rfc3686_skcipher_setkey, 1890 .encrypt = skcipher_encrypt, 1891 .decrypt = skcipher_decrypt, 1892 .min_keysize = AES_MIN_KEY_SIZE + 1893 CTR_RFC3686_NONCE_SIZE, 1894 .max_keysize = AES_MAX_KEY_SIZE + 1895 CTR_RFC3686_NONCE_SIZE, 1896 .ivsize = CTR_RFC3686_IV_SIZE, 1897 .chunksize = AES_BLOCK_SIZE, 1898 }, 1899 .caam = { 1900 .class1_alg_type = OP_ALG_ALGSEL_AES | 1901 OP_ALG_AAI_CTR_MOD128, 1902 .rfc3686 = true, 1903 }, 1904 }, 1905 { 1906 .skcipher = { 1907 .base = { 1908 .cra_name = "xts(aes)", 1909 .cra_driver_name = "xts-aes-caam", 1910 .cra_blocksize = AES_BLOCK_SIZE, 1911 }, 1912 .setkey = xts_skcipher_setkey, 1913 .encrypt = skcipher_encrypt, 1914 .decrypt = skcipher_decrypt, 1915 .min_keysize = 2 * AES_MIN_KEY_SIZE, 1916 .max_keysize = 2 * AES_MAX_KEY_SIZE, 1917 .ivsize = AES_BLOCK_SIZE, 1918 }, 1919 .caam.class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_XTS, 1920 }, 1921 { 1922 .skcipher = { 1923 .base = { 1924 .cra_name = "ecb(des)", 1925 .cra_driver_name = "ecb-des-caam", 1926 .cra_blocksize = DES_BLOCK_SIZE, 1927 }, 1928 .setkey = des_skcipher_setkey, 1929 .encrypt = skcipher_encrypt, 1930 .decrypt = skcipher_decrypt, 1931 .min_keysize = DES_KEY_SIZE, 1932 .max_keysize = DES_KEY_SIZE, 1933 }, 1934 .caam.class1_alg_type = OP_ALG_ALGSEL_DES | OP_ALG_AAI_ECB, 1935 }, 1936 { 1937 .skcipher = { 1938 .base = { 1939 .cra_name = "ecb(aes)", 1940 .cra_driver_name = "ecb-aes-caam", 1941 .cra_blocksize = AES_BLOCK_SIZE, 1942 }, 1943 .setkey = aes_skcipher_setkey, 1944 .encrypt = skcipher_encrypt, 1945 .decrypt = skcipher_decrypt, 1946 .min_keysize = AES_MIN_KEY_SIZE, 1947 .max_keysize = AES_MAX_KEY_SIZE, 1948 }, 1949 .caam.class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_ECB, 1950 }, 1951 { 1952 .skcipher = { 1953 .base = { 1954 .cra_name = "ecb(des3_ede)", 1955 .cra_driver_name = "ecb-des3-caam", 1956 .cra_blocksize = DES3_EDE_BLOCK_SIZE, 1957 }, 1958 .setkey = des3_skcipher_setkey, 1959 .encrypt = skcipher_encrypt, 1960 .decrypt = skcipher_decrypt, 1961 .min_keysize = DES3_EDE_KEY_SIZE, 1962 .max_keysize = DES3_EDE_KEY_SIZE, 1963 }, 1964 .caam.class1_alg_type = OP_ALG_ALGSEL_3DES | OP_ALG_AAI_ECB, 1965 }, 1966 { 1967 .skcipher = { 1968 .base = { 1969 .cra_name = "ecb(arc4)", 1970 .cra_driver_name = "ecb-arc4-caam", 1971 .cra_blocksize = ARC4_BLOCK_SIZE, 1972 }, 1973 .setkey = arc4_skcipher_setkey, 1974 .encrypt = skcipher_encrypt, 1975 .decrypt = skcipher_decrypt, 1976 .min_keysize = ARC4_MIN_KEY_SIZE, 1977 .max_keysize = ARC4_MAX_KEY_SIZE, 1978 }, 1979 .caam.class1_alg_type = OP_ALG_ALGSEL_ARC4 | OP_ALG_AAI_ECB, 1980 }, 1981 }; 1982 1983 static struct caam_aead_alg driver_aeads[] = { 1984 { 1985 .aead = { 1986 .base = { 1987 .cra_name = "rfc4106(gcm(aes))", 1988 .cra_driver_name = "rfc4106-gcm-aes-caam", 1989 .cra_blocksize = 1, 1990 }, 1991 .setkey = rfc4106_setkey, 1992 .setauthsize = rfc4106_setauthsize, 1993 .encrypt = ipsec_gcm_encrypt, 1994 .decrypt = ipsec_gcm_decrypt, 1995 .ivsize = GCM_RFC4106_IV_SIZE, 1996 .maxauthsize = AES_BLOCK_SIZE, 1997 }, 1998 .caam = { 1999 .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_GCM, 2000 .nodkp = true, 2001 }, 2002 }, 2003 { 2004 .aead = { 2005 .base = { 2006 .cra_name = "rfc4543(gcm(aes))", 2007 .cra_driver_name = "rfc4543-gcm-aes-caam", 2008 .cra_blocksize = 1, 2009 }, 2010 .setkey = rfc4543_setkey, 2011 .setauthsize = rfc4543_setauthsize, 2012 .encrypt = ipsec_gcm_encrypt, 2013 .decrypt = ipsec_gcm_decrypt, 2014 .ivsize = GCM_RFC4543_IV_SIZE, 2015 .maxauthsize = AES_BLOCK_SIZE, 2016 }, 2017 .caam = { 2018 .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_GCM, 2019 .nodkp = true, 2020 }, 2021 }, 2022 /* Galois Counter Mode */ 2023 { 2024 .aead = { 2025 .base = { 2026 .cra_name = "gcm(aes)", 2027 .cra_driver_name = "gcm-aes-caam", 2028 .cra_blocksize = 1, 2029 }, 2030 .setkey = gcm_setkey, 2031 .setauthsize = gcm_setauthsize, 2032 .encrypt = gcm_encrypt, 2033 .decrypt = gcm_decrypt, 2034 .ivsize = GCM_AES_IV_SIZE, 2035 .maxauthsize = AES_BLOCK_SIZE, 2036 }, 2037 .caam = { 2038 .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_GCM, 2039 .nodkp = true, 2040 }, 2041 }, 2042 /* single-pass ipsec_esp descriptor */ 2043 { 2044 .aead = { 2045 .base = { 2046 .cra_name = "authenc(hmac(md5)," 2047 "ecb(cipher_null))", 2048 .cra_driver_name = "authenc-hmac-md5-" 2049 "ecb-cipher_null-caam", 2050 .cra_blocksize = NULL_BLOCK_SIZE, 2051 }, 2052 .setkey = aead_setkey, 2053 .setauthsize = aead_setauthsize, 2054 .encrypt = aead_encrypt, 2055 .decrypt = aead_decrypt, 2056 .ivsize = NULL_IV_SIZE, 2057 .maxauthsize = MD5_DIGEST_SIZE, 2058 }, 2059 .caam = { 2060 .class2_alg_type = OP_ALG_ALGSEL_MD5 | 2061 OP_ALG_AAI_HMAC_PRECOMP, 2062 }, 2063 }, 2064 { 2065 .aead = { 2066 .base = { 2067 .cra_name = "authenc(hmac(sha1)," 2068 "ecb(cipher_null))", 2069 .cra_driver_name = "authenc-hmac-sha1-" 2070 "ecb-cipher_null-caam", 2071 .cra_blocksize = NULL_BLOCK_SIZE, 2072 }, 2073 .setkey = aead_setkey, 2074 .setauthsize = aead_setauthsize, 2075 .encrypt = aead_encrypt, 2076 .decrypt = aead_decrypt, 2077 .ivsize = NULL_IV_SIZE, 2078 .maxauthsize = SHA1_DIGEST_SIZE, 2079 }, 2080 .caam = { 2081 .class2_alg_type = OP_ALG_ALGSEL_SHA1 | 2082 OP_ALG_AAI_HMAC_PRECOMP, 2083 }, 2084 }, 2085 { 2086 .aead = { 2087 .base = { 2088 .cra_name = "authenc(hmac(sha224)," 2089 "ecb(cipher_null))", 2090 .cra_driver_name = "authenc-hmac-sha224-" 2091 "ecb-cipher_null-caam", 2092 .cra_blocksize = NULL_BLOCK_SIZE, 2093 }, 2094 .setkey = aead_setkey, 2095 .setauthsize = aead_setauthsize, 2096 .encrypt = aead_encrypt, 2097 .decrypt = aead_decrypt, 2098 .ivsize = NULL_IV_SIZE, 2099 .maxauthsize = SHA224_DIGEST_SIZE, 2100 }, 2101 .caam = { 2102 .class2_alg_type = OP_ALG_ALGSEL_SHA224 | 2103 OP_ALG_AAI_HMAC_PRECOMP, 2104 }, 2105 }, 2106 { 2107 .aead = { 2108 .base = { 2109 .cra_name = "authenc(hmac(sha256)," 2110 "ecb(cipher_null))", 2111 .cra_driver_name = "authenc-hmac-sha256-" 2112 "ecb-cipher_null-caam", 2113 .cra_blocksize = NULL_BLOCK_SIZE, 2114 }, 2115 .setkey = aead_setkey, 2116 .setauthsize = aead_setauthsize, 2117 .encrypt = aead_encrypt, 2118 .decrypt = aead_decrypt, 2119 .ivsize = NULL_IV_SIZE, 2120 .maxauthsize = SHA256_DIGEST_SIZE, 2121 }, 2122 .caam = { 2123 .class2_alg_type = OP_ALG_ALGSEL_SHA256 | 2124 OP_ALG_AAI_HMAC_PRECOMP, 2125 }, 2126 }, 2127 { 2128 .aead = { 2129 .base = { 2130 .cra_name = "authenc(hmac(sha384)," 2131 "ecb(cipher_null))", 2132 .cra_driver_name = "authenc-hmac-sha384-" 2133 "ecb-cipher_null-caam", 2134 .cra_blocksize = NULL_BLOCK_SIZE, 2135 }, 2136 .setkey = aead_setkey, 2137 .setauthsize = aead_setauthsize, 2138 .encrypt = aead_encrypt, 2139 .decrypt = aead_decrypt, 2140 .ivsize = NULL_IV_SIZE, 2141 .maxauthsize = SHA384_DIGEST_SIZE, 2142 }, 2143 .caam = { 2144 .class2_alg_type = OP_ALG_ALGSEL_SHA384 | 2145 OP_ALG_AAI_HMAC_PRECOMP, 2146 }, 2147 }, 2148 { 2149 .aead = { 2150 .base = { 2151 .cra_name = "authenc(hmac(sha512)," 2152 "ecb(cipher_null))", 2153 .cra_driver_name = "authenc-hmac-sha512-" 2154 "ecb-cipher_null-caam", 2155 .cra_blocksize = NULL_BLOCK_SIZE, 2156 }, 2157 .setkey = aead_setkey, 2158 .setauthsize = aead_setauthsize, 2159 .encrypt = aead_encrypt, 2160 .decrypt = aead_decrypt, 2161 .ivsize = NULL_IV_SIZE, 2162 .maxauthsize = SHA512_DIGEST_SIZE, 2163 }, 2164 .caam = { 2165 .class2_alg_type = OP_ALG_ALGSEL_SHA512 | 2166 OP_ALG_AAI_HMAC_PRECOMP, 2167 }, 2168 }, 2169 { 2170 .aead = { 2171 .base = { 2172 .cra_name = "authenc(hmac(md5),cbc(aes))", 2173 .cra_driver_name = "authenc-hmac-md5-" 2174 "cbc-aes-caam", 2175 .cra_blocksize = AES_BLOCK_SIZE, 2176 }, 2177 .setkey = aead_setkey, 2178 .setauthsize = aead_setauthsize, 2179 .encrypt = aead_encrypt, 2180 .decrypt = aead_decrypt, 2181 .ivsize = AES_BLOCK_SIZE, 2182 .maxauthsize = MD5_DIGEST_SIZE, 2183 }, 2184 .caam = { 2185 .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CBC, 2186 .class2_alg_type = OP_ALG_ALGSEL_MD5 | 2187 OP_ALG_AAI_HMAC_PRECOMP, 2188 }, 2189 }, 2190 { 2191 .aead = { 2192 .base = { 2193 .cra_name = "echainiv(authenc(hmac(md5)," 2194 "cbc(aes)))", 2195 .cra_driver_name = "echainiv-authenc-hmac-md5-" 2196 "cbc-aes-caam", 2197 .cra_blocksize = AES_BLOCK_SIZE, 2198 }, 2199 .setkey = aead_setkey, 2200 .setauthsize = aead_setauthsize, 2201 .encrypt = aead_encrypt, 2202 .decrypt = aead_decrypt, 2203 .ivsize = AES_BLOCK_SIZE, 2204 .maxauthsize = MD5_DIGEST_SIZE, 2205 }, 2206 .caam = { 2207 .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CBC, 2208 .class2_alg_type = OP_ALG_ALGSEL_MD5 | 2209 OP_ALG_AAI_HMAC_PRECOMP, 2210 .geniv = true, 2211 }, 2212 }, 2213 { 2214 .aead = { 2215 .base = { 2216 .cra_name = "authenc(hmac(sha1),cbc(aes))", 2217 .cra_driver_name = "authenc-hmac-sha1-" 2218 "cbc-aes-caam", 2219 .cra_blocksize = AES_BLOCK_SIZE, 2220 }, 2221 .setkey = aead_setkey, 2222 .setauthsize = aead_setauthsize, 2223 .encrypt = aead_encrypt, 2224 .decrypt = aead_decrypt, 2225 .ivsize = AES_BLOCK_SIZE, 2226 .maxauthsize = SHA1_DIGEST_SIZE, 2227 }, 2228 .caam = { 2229 .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CBC, 2230 .class2_alg_type = OP_ALG_ALGSEL_SHA1 | 2231 OP_ALG_AAI_HMAC_PRECOMP, 2232 }, 2233 }, 2234 { 2235 .aead = { 2236 .base = { 2237 .cra_name = "echainiv(authenc(hmac(sha1)," 2238 "cbc(aes)))", 2239 .cra_driver_name = "echainiv-authenc-" 2240 "hmac-sha1-cbc-aes-caam", 2241 .cra_blocksize = AES_BLOCK_SIZE, 2242 }, 2243 .setkey = aead_setkey, 2244 .setauthsize = aead_setauthsize, 2245 .encrypt = aead_encrypt, 2246 .decrypt = aead_decrypt, 2247 .ivsize = AES_BLOCK_SIZE, 2248 .maxauthsize = SHA1_DIGEST_SIZE, 2249 }, 2250 .caam = { 2251 .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CBC, 2252 .class2_alg_type = OP_ALG_ALGSEL_SHA1 | 2253 OP_ALG_AAI_HMAC_PRECOMP, 2254 .geniv = true, 2255 }, 2256 }, 2257 { 2258 .aead = { 2259 .base = { 2260 .cra_name = "authenc(hmac(sha224),cbc(aes))", 2261 .cra_driver_name = "authenc-hmac-sha224-" 2262 "cbc-aes-caam", 2263 .cra_blocksize = AES_BLOCK_SIZE, 2264 }, 2265 .setkey = aead_setkey, 2266 .setauthsize = aead_setauthsize, 2267 .encrypt = aead_encrypt, 2268 .decrypt = aead_decrypt, 2269 .ivsize = AES_BLOCK_SIZE, 2270 .maxauthsize = SHA224_DIGEST_SIZE, 2271 }, 2272 .caam = { 2273 .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CBC, 2274 .class2_alg_type = OP_ALG_ALGSEL_SHA224 | 2275 OP_ALG_AAI_HMAC_PRECOMP, 2276 }, 2277 }, 2278 { 2279 .aead = { 2280 .base = { 2281 .cra_name = "echainiv(authenc(hmac(sha224)," 2282 "cbc(aes)))", 2283 .cra_driver_name = "echainiv-authenc-" 2284 "hmac-sha224-cbc-aes-caam", 2285 .cra_blocksize = AES_BLOCK_SIZE, 2286 }, 2287 .setkey = aead_setkey, 2288 .setauthsize = aead_setauthsize, 2289 .encrypt = aead_encrypt, 2290 .decrypt = aead_decrypt, 2291 .ivsize = AES_BLOCK_SIZE, 2292 .maxauthsize = SHA224_DIGEST_SIZE, 2293 }, 2294 .caam = { 2295 .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CBC, 2296 .class2_alg_type = OP_ALG_ALGSEL_SHA224 | 2297 OP_ALG_AAI_HMAC_PRECOMP, 2298 .geniv = true, 2299 }, 2300 }, 2301 { 2302 .aead = { 2303 .base = { 2304 .cra_name = "authenc(hmac(sha256),cbc(aes))", 2305 .cra_driver_name = "authenc-hmac-sha256-" 2306 "cbc-aes-caam", 2307 .cra_blocksize = AES_BLOCK_SIZE, 2308 }, 2309 .setkey = aead_setkey, 2310 .setauthsize = aead_setauthsize, 2311 .encrypt = aead_encrypt, 2312 .decrypt = aead_decrypt, 2313 .ivsize = AES_BLOCK_SIZE, 2314 .maxauthsize = SHA256_DIGEST_SIZE, 2315 }, 2316 .caam = { 2317 .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CBC, 2318 .class2_alg_type = OP_ALG_ALGSEL_SHA256 | 2319 OP_ALG_AAI_HMAC_PRECOMP, 2320 }, 2321 }, 2322 { 2323 .aead = { 2324 .base = { 2325 .cra_name = "echainiv(authenc(hmac(sha256)," 2326 "cbc(aes)))", 2327 .cra_driver_name = "echainiv-authenc-" 2328 "hmac-sha256-cbc-aes-caam", 2329 .cra_blocksize = AES_BLOCK_SIZE, 2330 }, 2331 .setkey = aead_setkey, 2332 .setauthsize = aead_setauthsize, 2333 .encrypt = aead_encrypt, 2334 .decrypt = aead_decrypt, 2335 .ivsize = AES_BLOCK_SIZE, 2336 .maxauthsize = SHA256_DIGEST_SIZE, 2337 }, 2338 .caam = { 2339 .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CBC, 2340 .class2_alg_type = OP_ALG_ALGSEL_SHA256 | 2341 OP_ALG_AAI_HMAC_PRECOMP, 2342 .geniv = true, 2343 }, 2344 }, 2345 { 2346 .aead = { 2347 .base = { 2348 .cra_name = "authenc(hmac(sha384),cbc(aes))", 2349 .cra_driver_name = "authenc-hmac-sha384-" 2350 "cbc-aes-caam", 2351 .cra_blocksize = AES_BLOCK_SIZE, 2352 }, 2353 .setkey = aead_setkey, 2354 .setauthsize = aead_setauthsize, 2355 .encrypt = aead_encrypt, 2356 .decrypt = aead_decrypt, 2357 .ivsize = AES_BLOCK_SIZE, 2358 .maxauthsize = SHA384_DIGEST_SIZE, 2359 }, 2360 .caam = { 2361 .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CBC, 2362 .class2_alg_type = OP_ALG_ALGSEL_SHA384 | 2363 OP_ALG_AAI_HMAC_PRECOMP, 2364 }, 2365 }, 2366 { 2367 .aead = { 2368 .base = { 2369 .cra_name = "echainiv(authenc(hmac(sha384)," 2370 "cbc(aes)))", 2371 .cra_driver_name = "echainiv-authenc-" 2372 "hmac-sha384-cbc-aes-caam", 2373 .cra_blocksize = AES_BLOCK_SIZE, 2374 }, 2375 .setkey = aead_setkey, 2376 .setauthsize = aead_setauthsize, 2377 .encrypt = aead_encrypt, 2378 .decrypt = aead_decrypt, 2379 .ivsize = AES_BLOCK_SIZE, 2380 .maxauthsize = SHA384_DIGEST_SIZE, 2381 }, 2382 .caam = { 2383 .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CBC, 2384 .class2_alg_type = OP_ALG_ALGSEL_SHA384 | 2385 OP_ALG_AAI_HMAC_PRECOMP, 2386 .geniv = true, 2387 }, 2388 }, 2389 { 2390 .aead = { 2391 .base = { 2392 .cra_name = "authenc(hmac(sha512),cbc(aes))", 2393 .cra_driver_name = "authenc-hmac-sha512-" 2394 "cbc-aes-caam", 2395 .cra_blocksize = AES_BLOCK_SIZE, 2396 }, 2397 .setkey = aead_setkey, 2398 .setauthsize = aead_setauthsize, 2399 .encrypt = aead_encrypt, 2400 .decrypt = aead_decrypt, 2401 .ivsize = AES_BLOCK_SIZE, 2402 .maxauthsize = SHA512_DIGEST_SIZE, 2403 }, 2404 .caam = { 2405 .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CBC, 2406 .class2_alg_type = OP_ALG_ALGSEL_SHA512 | 2407 OP_ALG_AAI_HMAC_PRECOMP, 2408 }, 2409 }, 2410 { 2411 .aead = { 2412 .base = { 2413 .cra_name = "echainiv(authenc(hmac(sha512)," 2414 "cbc(aes)))", 2415 .cra_driver_name = "echainiv-authenc-" 2416 "hmac-sha512-cbc-aes-caam", 2417 .cra_blocksize = AES_BLOCK_SIZE, 2418 }, 2419 .setkey = aead_setkey, 2420 .setauthsize = aead_setauthsize, 2421 .encrypt = aead_encrypt, 2422 .decrypt = aead_decrypt, 2423 .ivsize = AES_BLOCK_SIZE, 2424 .maxauthsize = SHA512_DIGEST_SIZE, 2425 }, 2426 .caam = { 2427 .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CBC, 2428 .class2_alg_type = OP_ALG_ALGSEL_SHA512 | 2429 OP_ALG_AAI_HMAC_PRECOMP, 2430 .geniv = true, 2431 }, 2432 }, 2433 { 2434 .aead = { 2435 .base = { 2436 .cra_name = "authenc(hmac(md5),cbc(des3_ede))", 2437 .cra_driver_name = "authenc-hmac-md5-" 2438 "cbc-des3_ede-caam", 2439 .cra_blocksize = DES3_EDE_BLOCK_SIZE, 2440 }, 2441 .setkey = des3_aead_setkey, 2442 .setauthsize = aead_setauthsize, 2443 .encrypt = aead_encrypt, 2444 .decrypt = aead_decrypt, 2445 .ivsize = DES3_EDE_BLOCK_SIZE, 2446 .maxauthsize = MD5_DIGEST_SIZE, 2447 }, 2448 .caam = { 2449 .class1_alg_type = OP_ALG_ALGSEL_3DES | OP_ALG_AAI_CBC, 2450 .class2_alg_type = OP_ALG_ALGSEL_MD5 | 2451 OP_ALG_AAI_HMAC_PRECOMP, 2452 } 2453 }, 2454 { 2455 .aead = { 2456 .base = { 2457 .cra_name = "echainiv(authenc(hmac(md5)," 2458 "cbc(des3_ede)))", 2459 .cra_driver_name = "echainiv-authenc-hmac-md5-" 2460 "cbc-des3_ede-caam", 2461 .cra_blocksize = DES3_EDE_BLOCK_SIZE, 2462 }, 2463 .setkey = des3_aead_setkey, 2464 .setauthsize = aead_setauthsize, 2465 .encrypt = aead_encrypt, 2466 .decrypt = aead_decrypt, 2467 .ivsize = DES3_EDE_BLOCK_SIZE, 2468 .maxauthsize = MD5_DIGEST_SIZE, 2469 }, 2470 .caam = { 2471 .class1_alg_type = OP_ALG_ALGSEL_3DES | OP_ALG_AAI_CBC, 2472 .class2_alg_type = OP_ALG_ALGSEL_MD5 | 2473 OP_ALG_AAI_HMAC_PRECOMP, 2474 .geniv = true, 2475 } 2476 }, 2477 { 2478 .aead = { 2479 .base = { 2480 .cra_name = "authenc(hmac(sha1)," 2481 "cbc(des3_ede))", 2482 .cra_driver_name = "authenc-hmac-sha1-" 2483 "cbc-des3_ede-caam", 2484 .cra_blocksize = DES3_EDE_BLOCK_SIZE, 2485 }, 2486 .setkey = des3_aead_setkey, 2487 .setauthsize = aead_setauthsize, 2488 .encrypt = aead_encrypt, 2489 .decrypt = aead_decrypt, 2490 .ivsize = DES3_EDE_BLOCK_SIZE, 2491 .maxauthsize = SHA1_DIGEST_SIZE, 2492 }, 2493 .caam = { 2494 .class1_alg_type = OP_ALG_ALGSEL_3DES | OP_ALG_AAI_CBC, 2495 .class2_alg_type = OP_ALG_ALGSEL_SHA1 | 2496 OP_ALG_AAI_HMAC_PRECOMP, 2497 }, 2498 }, 2499 { 2500 .aead = { 2501 .base = { 2502 .cra_name = "echainiv(authenc(hmac(sha1)," 2503 "cbc(des3_ede)))", 2504 .cra_driver_name = "echainiv-authenc-" 2505 "hmac-sha1-" 2506 "cbc-des3_ede-caam", 2507 .cra_blocksize = DES3_EDE_BLOCK_SIZE, 2508 }, 2509 .setkey = des3_aead_setkey, 2510 .setauthsize = aead_setauthsize, 2511 .encrypt = aead_encrypt, 2512 .decrypt = aead_decrypt, 2513 .ivsize = DES3_EDE_BLOCK_SIZE, 2514 .maxauthsize = SHA1_DIGEST_SIZE, 2515 }, 2516 .caam = { 2517 .class1_alg_type = OP_ALG_ALGSEL_3DES | OP_ALG_AAI_CBC, 2518 .class2_alg_type = OP_ALG_ALGSEL_SHA1 | 2519 OP_ALG_AAI_HMAC_PRECOMP, 2520 .geniv = true, 2521 }, 2522 }, 2523 { 2524 .aead = { 2525 .base = { 2526 .cra_name = "authenc(hmac(sha224)," 2527 "cbc(des3_ede))", 2528 .cra_driver_name = "authenc-hmac-sha224-" 2529 "cbc-des3_ede-caam", 2530 .cra_blocksize = DES3_EDE_BLOCK_SIZE, 2531 }, 2532 .setkey = des3_aead_setkey, 2533 .setauthsize = aead_setauthsize, 2534 .encrypt = aead_encrypt, 2535 .decrypt = aead_decrypt, 2536 .ivsize = DES3_EDE_BLOCK_SIZE, 2537 .maxauthsize = SHA224_DIGEST_SIZE, 2538 }, 2539 .caam = { 2540 .class1_alg_type = OP_ALG_ALGSEL_3DES | OP_ALG_AAI_CBC, 2541 .class2_alg_type = OP_ALG_ALGSEL_SHA224 | 2542 OP_ALG_AAI_HMAC_PRECOMP, 2543 }, 2544 }, 2545 { 2546 .aead = { 2547 .base = { 2548 .cra_name = "echainiv(authenc(hmac(sha224)," 2549 "cbc(des3_ede)))", 2550 .cra_driver_name = "echainiv-authenc-" 2551 "hmac-sha224-" 2552 "cbc-des3_ede-caam", 2553 .cra_blocksize = DES3_EDE_BLOCK_SIZE, 2554 }, 2555 .setkey = des3_aead_setkey, 2556 .setauthsize = aead_setauthsize, 2557 .encrypt = aead_encrypt, 2558 .decrypt = aead_decrypt, 2559 .ivsize = DES3_EDE_BLOCK_SIZE, 2560 .maxauthsize = SHA224_DIGEST_SIZE, 2561 }, 2562 .caam = { 2563 .class1_alg_type = OP_ALG_ALGSEL_3DES | OP_ALG_AAI_CBC, 2564 .class2_alg_type = OP_ALG_ALGSEL_SHA224 | 2565 OP_ALG_AAI_HMAC_PRECOMP, 2566 .geniv = true, 2567 }, 2568 }, 2569 { 2570 .aead = { 2571 .base = { 2572 .cra_name = "authenc(hmac(sha256)," 2573 "cbc(des3_ede))", 2574 .cra_driver_name = "authenc-hmac-sha256-" 2575 "cbc-des3_ede-caam", 2576 .cra_blocksize = DES3_EDE_BLOCK_SIZE, 2577 }, 2578 .setkey = des3_aead_setkey, 2579 .setauthsize = aead_setauthsize, 2580 .encrypt = aead_encrypt, 2581 .decrypt = aead_decrypt, 2582 .ivsize = DES3_EDE_BLOCK_SIZE, 2583 .maxauthsize = SHA256_DIGEST_SIZE, 2584 }, 2585 .caam = { 2586 .class1_alg_type = OP_ALG_ALGSEL_3DES | OP_ALG_AAI_CBC, 2587 .class2_alg_type = OP_ALG_ALGSEL_SHA256 | 2588 OP_ALG_AAI_HMAC_PRECOMP, 2589 }, 2590 }, 2591 { 2592 .aead = { 2593 .base = { 2594 .cra_name = "echainiv(authenc(hmac(sha256)," 2595 "cbc(des3_ede)))", 2596 .cra_driver_name = "echainiv-authenc-" 2597 "hmac-sha256-" 2598 "cbc-des3_ede-caam", 2599 .cra_blocksize = DES3_EDE_BLOCK_SIZE, 2600 }, 2601 .setkey = des3_aead_setkey, 2602 .setauthsize = aead_setauthsize, 2603 .encrypt = aead_encrypt, 2604 .decrypt = aead_decrypt, 2605 .ivsize = DES3_EDE_BLOCK_SIZE, 2606 .maxauthsize = SHA256_DIGEST_SIZE, 2607 }, 2608 .caam = { 2609 .class1_alg_type = OP_ALG_ALGSEL_3DES | OP_ALG_AAI_CBC, 2610 .class2_alg_type = OP_ALG_ALGSEL_SHA256 | 2611 OP_ALG_AAI_HMAC_PRECOMP, 2612 .geniv = true, 2613 }, 2614 }, 2615 { 2616 .aead = { 2617 .base = { 2618 .cra_name = "authenc(hmac(sha384)," 2619 "cbc(des3_ede))", 2620 .cra_driver_name = "authenc-hmac-sha384-" 2621 "cbc-des3_ede-caam", 2622 .cra_blocksize = DES3_EDE_BLOCK_SIZE, 2623 }, 2624 .setkey = des3_aead_setkey, 2625 .setauthsize = aead_setauthsize, 2626 .encrypt = aead_encrypt, 2627 .decrypt = aead_decrypt, 2628 .ivsize = DES3_EDE_BLOCK_SIZE, 2629 .maxauthsize = SHA384_DIGEST_SIZE, 2630 }, 2631 .caam = { 2632 .class1_alg_type = OP_ALG_ALGSEL_3DES | OP_ALG_AAI_CBC, 2633 .class2_alg_type = OP_ALG_ALGSEL_SHA384 | 2634 OP_ALG_AAI_HMAC_PRECOMP, 2635 }, 2636 }, 2637 { 2638 .aead = { 2639 .base = { 2640 .cra_name = "echainiv(authenc(hmac(sha384)," 2641 "cbc(des3_ede)))", 2642 .cra_driver_name = "echainiv-authenc-" 2643 "hmac-sha384-" 2644 "cbc-des3_ede-caam", 2645 .cra_blocksize = DES3_EDE_BLOCK_SIZE, 2646 }, 2647 .setkey = des3_aead_setkey, 2648 .setauthsize = aead_setauthsize, 2649 .encrypt = aead_encrypt, 2650 .decrypt = aead_decrypt, 2651 .ivsize = DES3_EDE_BLOCK_SIZE, 2652 .maxauthsize = SHA384_DIGEST_SIZE, 2653 }, 2654 .caam = { 2655 .class1_alg_type = OP_ALG_ALGSEL_3DES | OP_ALG_AAI_CBC, 2656 .class2_alg_type = OP_ALG_ALGSEL_SHA384 | 2657 OP_ALG_AAI_HMAC_PRECOMP, 2658 .geniv = true, 2659 }, 2660 }, 2661 { 2662 .aead = { 2663 .base = { 2664 .cra_name = "authenc(hmac(sha512)," 2665 "cbc(des3_ede))", 2666 .cra_driver_name = "authenc-hmac-sha512-" 2667 "cbc-des3_ede-caam", 2668 .cra_blocksize = DES3_EDE_BLOCK_SIZE, 2669 }, 2670 .setkey = des3_aead_setkey, 2671 .setauthsize = aead_setauthsize, 2672 .encrypt = aead_encrypt, 2673 .decrypt = aead_decrypt, 2674 .ivsize = DES3_EDE_BLOCK_SIZE, 2675 .maxauthsize = SHA512_DIGEST_SIZE, 2676 }, 2677 .caam = { 2678 .class1_alg_type = OP_ALG_ALGSEL_3DES | OP_ALG_AAI_CBC, 2679 .class2_alg_type = OP_ALG_ALGSEL_SHA512 | 2680 OP_ALG_AAI_HMAC_PRECOMP, 2681 }, 2682 }, 2683 { 2684 .aead = { 2685 .base = { 2686 .cra_name = "echainiv(authenc(hmac(sha512)," 2687 "cbc(des3_ede)))", 2688 .cra_driver_name = "echainiv-authenc-" 2689 "hmac-sha512-" 2690 "cbc-des3_ede-caam", 2691 .cra_blocksize = DES3_EDE_BLOCK_SIZE, 2692 }, 2693 .setkey = des3_aead_setkey, 2694 .setauthsize = aead_setauthsize, 2695 .encrypt = aead_encrypt, 2696 .decrypt = aead_decrypt, 2697 .ivsize = DES3_EDE_BLOCK_SIZE, 2698 .maxauthsize = SHA512_DIGEST_SIZE, 2699 }, 2700 .caam = { 2701 .class1_alg_type = OP_ALG_ALGSEL_3DES | OP_ALG_AAI_CBC, 2702 .class2_alg_type = OP_ALG_ALGSEL_SHA512 | 2703 OP_ALG_AAI_HMAC_PRECOMP, 2704 .geniv = true, 2705 }, 2706 }, 2707 { 2708 .aead = { 2709 .base = { 2710 .cra_name = "authenc(hmac(md5),cbc(des))", 2711 .cra_driver_name = "authenc-hmac-md5-" 2712 "cbc-des-caam", 2713 .cra_blocksize = DES_BLOCK_SIZE, 2714 }, 2715 .setkey = aead_setkey, 2716 .setauthsize = aead_setauthsize, 2717 .encrypt = aead_encrypt, 2718 .decrypt = aead_decrypt, 2719 .ivsize = DES_BLOCK_SIZE, 2720 .maxauthsize = MD5_DIGEST_SIZE, 2721 }, 2722 .caam = { 2723 .class1_alg_type = OP_ALG_ALGSEL_DES | OP_ALG_AAI_CBC, 2724 .class2_alg_type = OP_ALG_ALGSEL_MD5 | 2725 OP_ALG_AAI_HMAC_PRECOMP, 2726 }, 2727 }, 2728 { 2729 .aead = { 2730 .base = { 2731 .cra_name = "echainiv(authenc(hmac(md5)," 2732 "cbc(des)))", 2733 .cra_driver_name = "echainiv-authenc-hmac-md5-" 2734 "cbc-des-caam", 2735 .cra_blocksize = DES_BLOCK_SIZE, 2736 }, 2737 .setkey = aead_setkey, 2738 .setauthsize = aead_setauthsize, 2739 .encrypt = aead_encrypt, 2740 .decrypt = aead_decrypt, 2741 .ivsize = DES_BLOCK_SIZE, 2742 .maxauthsize = MD5_DIGEST_SIZE, 2743 }, 2744 .caam = { 2745 .class1_alg_type = OP_ALG_ALGSEL_DES | OP_ALG_AAI_CBC, 2746 .class2_alg_type = OP_ALG_ALGSEL_MD5 | 2747 OP_ALG_AAI_HMAC_PRECOMP, 2748 .geniv = true, 2749 }, 2750 }, 2751 { 2752 .aead = { 2753 .base = { 2754 .cra_name = "authenc(hmac(sha1),cbc(des))", 2755 .cra_driver_name = "authenc-hmac-sha1-" 2756 "cbc-des-caam", 2757 .cra_blocksize = DES_BLOCK_SIZE, 2758 }, 2759 .setkey = aead_setkey, 2760 .setauthsize = aead_setauthsize, 2761 .encrypt = aead_encrypt, 2762 .decrypt = aead_decrypt, 2763 .ivsize = DES_BLOCK_SIZE, 2764 .maxauthsize = SHA1_DIGEST_SIZE, 2765 }, 2766 .caam = { 2767 .class1_alg_type = OP_ALG_ALGSEL_DES | OP_ALG_AAI_CBC, 2768 .class2_alg_type = OP_ALG_ALGSEL_SHA1 | 2769 OP_ALG_AAI_HMAC_PRECOMP, 2770 }, 2771 }, 2772 { 2773 .aead = { 2774 .base = { 2775 .cra_name = "echainiv(authenc(hmac(sha1)," 2776 "cbc(des)))", 2777 .cra_driver_name = "echainiv-authenc-" 2778 "hmac-sha1-cbc-des-caam", 2779 .cra_blocksize = DES_BLOCK_SIZE, 2780 }, 2781 .setkey = aead_setkey, 2782 .setauthsize = aead_setauthsize, 2783 .encrypt = aead_encrypt, 2784 .decrypt = aead_decrypt, 2785 .ivsize = DES_BLOCK_SIZE, 2786 .maxauthsize = SHA1_DIGEST_SIZE, 2787 }, 2788 .caam = { 2789 .class1_alg_type = OP_ALG_ALGSEL_DES | OP_ALG_AAI_CBC, 2790 .class2_alg_type = OP_ALG_ALGSEL_SHA1 | 2791 OP_ALG_AAI_HMAC_PRECOMP, 2792 .geniv = true, 2793 }, 2794 }, 2795 { 2796 .aead = { 2797 .base = { 2798 .cra_name = "authenc(hmac(sha224),cbc(des))", 2799 .cra_driver_name = "authenc-hmac-sha224-" 2800 "cbc-des-caam", 2801 .cra_blocksize = DES_BLOCK_SIZE, 2802 }, 2803 .setkey = aead_setkey, 2804 .setauthsize = aead_setauthsize, 2805 .encrypt = aead_encrypt, 2806 .decrypt = aead_decrypt, 2807 .ivsize = DES_BLOCK_SIZE, 2808 .maxauthsize = SHA224_DIGEST_SIZE, 2809 }, 2810 .caam = { 2811 .class1_alg_type = OP_ALG_ALGSEL_DES | OP_ALG_AAI_CBC, 2812 .class2_alg_type = OP_ALG_ALGSEL_SHA224 | 2813 OP_ALG_AAI_HMAC_PRECOMP, 2814 }, 2815 }, 2816 { 2817 .aead = { 2818 .base = { 2819 .cra_name = "echainiv(authenc(hmac(sha224)," 2820 "cbc(des)))", 2821 .cra_driver_name = "echainiv-authenc-" 2822 "hmac-sha224-cbc-des-caam", 2823 .cra_blocksize = DES_BLOCK_SIZE, 2824 }, 2825 .setkey = aead_setkey, 2826 .setauthsize = aead_setauthsize, 2827 .encrypt = aead_encrypt, 2828 .decrypt = aead_decrypt, 2829 .ivsize = DES_BLOCK_SIZE, 2830 .maxauthsize = SHA224_DIGEST_SIZE, 2831 }, 2832 .caam = { 2833 .class1_alg_type = OP_ALG_ALGSEL_DES | OP_ALG_AAI_CBC, 2834 .class2_alg_type = OP_ALG_ALGSEL_SHA224 | 2835 OP_ALG_AAI_HMAC_PRECOMP, 2836 .geniv = true, 2837 }, 2838 }, 2839 { 2840 .aead = { 2841 .base = { 2842 .cra_name = "authenc(hmac(sha256),cbc(des))", 2843 .cra_driver_name = "authenc-hmac-sha256-" 2844 "cbc-des-caam", 2845 .cra_blocksize = DES_BLOCK_SIZE, 2846 }, 2847 .setkey = aead_setkey, 2848 .setauthsize = aead_setauthsize, 2849 .encrypt = aead_encrypt, 2850 .decrypt = aead_decrypt, 2851 .ivsize = DES_BLOCK_SIZE, 2852 .maxauthsize = SHA256_DIGEST_SIZE, 2853 }, 2854 .caam = { 2855 .class1_alg_type = OP_ALG_ALGSEL_DES | OP_ALG_AAI_CBC, 2856 .class2_alg_type = OP_ALG_ALGSEL_SHA256 | 2857 OP_ALG_AAI_HMAC_PRECOMP, 2858 }, 2859 }, 2860 { 2861 .aead = { 2862 .base = { 2863 .cra_name = "echainiv(authenc(hmac(sha256)," 2864 "cbc(des)))", 2865 .cra_driver_name = "echainiv-authenc-" 2866 "hmac-sha256-cbc-des-caam", 2867 .cra_blocksize = DES_BLOCK_SIZE, 2868 }, 2869 .setkey = aead_setkey, 2870 .setauthsize = aead_setauthsize, 2871 .encrypt = aead_encrypt, 2872 .decrypt = aead_decrypt, 2873 .ivsize = DES_BLOCK_SIZE, 2874 .maxauthsize = SHA256_DIGEST_SIZE, 2875 }, 2876 .caam = { 2877 .class1_alg_type = OP_ALG_ALGSEL_DES | OP_ALG_AAI_CBC, 2878 .class2_alg_type = OP_ALG_ALGSEL_SHA256 | 2879 OP_ALG_AAI_HMAC_PRECOMP, 2880 .geniv = true, 2881 }, 2882 }, 2883 { 2884 .aead = { 2885 .base = { 2886 .cra_name = "authenc(hmac(sha384),cbc(des))", 2887 .cra_driver_name = "authenc-hmac-sha384-" 2888 "cbc-des-caam", 2889 .cra_blocksize = DES_BLOCK_SIZE, 2890 }, 2891 .setkey = aead_setkey, 2892 .setauthsize = aead_setauthsize, 2893 .encrypt = aead_encrypt, 2894 .decrypt = aead_decrypt, 2895 .ivsize = DES_BLOCK_SIZE, 2896 .maxauthsize = SHA384_DIGEST_SIZE, 2897 }, 2898 .caam = { 2899 .class1_alg_type = OP_ALG_ALGSEL_DES | OP_ALG_AAI_CBC, 2900 .class2_alg_type = OP_ALG_ALGSEL_SHA384 | 2901 OP_ALG_AAI_HMAC_PRECOMP, 2902 }, 2903 }, 2904 { 2905 .aead = { 2906 .base = { 2907 .cra_name = "echainiv(authenc(hmac(sha384)," 2908 "cbc(des)))", 2909 .cra_driver_name = "echainiv-authenc-" 2910 "hmac-sha384-cbc-des-caam", 2911 .cra_blocksize = DES_BLOCK_SIZE, 2912 }, 2913 .setkey = aead_setkey, 2914 .setauthsize = aead_setauthsize, 2915 .encrypt = aead_encrypt, 2916 .decrypt = aead_decrypt, 2917 .ivsize = DES_BLOCK_SIZE, 2918 .maxauthsize = SHA384_DIGEST_SIZE, 2919 }, 2920 .caam = { 2921 .class1_alg_type = OP_ALG_ALGSEL_DES | OP_ALG_AAI_CBC, 2922 .class2_alg_type = OP_ALG_ALGSEL_SHA384 | 2923 OP_ALG_AAI_HMAC_PRECOMP, 2924 .geniv = true, 2925 }, 2926 }, 2927 { 2928 .aead = { 2929 .base = { 2930 .cra_name = "authenc(hmac(sha512),cbc(des))", 2931 .cra_driver_name = "authenc-hmac-sha512-" 2932 "cbc-des-caam", 2933 .cra_blocksize = DES_BLOCK_SIZE, 2934 }, 2935 .setkey = aead_setkey, 2936 .setauthsize = aead_setauthsize, 2937 .encrypt = aead_encrypt, 2938 .decrypt = aead_decrypt, 2939 .ivsize = DES_BLOCK_SIZE, 2940 .maxauthsize = SHA512_DIGEST_SIZE, 2941 }, 2942 .caam = { 2943 .class1_alg_type = OP_ALG_ALGSEL_DES | OP_ALG_AAI_CBC, 2944 .class2_alg_type = OP_ALG_ALGSEL_SHA512 | 2945 OP_ALG_AAI_HMAC_PRECOMP, 2946 }, 2947 }, 2948 { 2949 .aead = { 2950 .base = { 2951 .cra_name = "echainiv(authenc(hmac(sha512)," 2952 "cbc(des)))", 2953 .cra_driver_name = "echainiv-authenc-" 2954 "hmac-sha512-cbc-des-caam", 2955 .cra_blocksize = DES_BLOCK_SIZE, 2956 }, 2957 .setkey = aead_setkey, 2958 .setauthsize = aead_setauthsize, 2959 .encrypt = aead_encrypt, 2960 .decrypt = aead_decrypt, 2961 .ivsize = DES_BLOCK_SIZE, 2962 .maxauthsize = SHA512_DIGEST_SIZE, 2963 }, 2964 .caam = { 2965 .class1_alg_type = OP_ALG_ALGSEL_DES | OP_ALG_AAI_CBC, 2966 .class2_alg_type = OP_ALG_ALGSEL_SHA512 | 2967 OP_ALG_AAI_HMAC_PRECOMP, 2968 .geniv = true, 2969 }, 2970 }, 2971 { 2972 .aead = { 2973 .base = { 2974 .cra_name = "authenc(hmac(md5)," 2975 "rfc3686(ctr(aes)))", 2976 .cra_driver_name = "authenc-hmac-md5-" 2977 "rfc3686-ctr-aes-caam", 2978 .cra_blocksize = 1, 2979 }, 2980 .setkey = aead_setkey, 2981 .setauthsize = aead_setauthsize, 2982 .encrypt = aead_encrypt, 2983 .decrypt = aead_decrypt, 2984 .ivsize = CTR_RFC3686_IV_SIZE, 2985 .maxauthsize = MD5_DIGEST_SIZE, 2986 }, 2987 .caam = { 2988 .class1_alg_type = OP_ALG_ALGSEL_AES | 2989 OP_ALG_AAI_CTR_MOD128, 2990 .class2_alg_type = OP_ALG_ALGSEL_MD5 | 2991 OP_ALG_AAI_HMAC_PRECOMP, 2992 .rfc3686 = true, 2993 }, 2994 }, 2995 { 2996 .aead = { 2997 .base = { 2998 .cra_name = "seqiv(authenc(" 2999 "hmac(md5),rfc3686(ctr(aes))))", 3000 .cra_driver_name = "seqiv-authenc-hmac-md5-" 3001 "rfc3686-ctr-aes-caam", 3002 .cra_blocksize = 1, 3003 }, 3004 .setkey = aead_setkey, 3005 .setauthsize = aead_setauthsize, 3006 .encrypt = aead_encrypt, 3007 .decrypt = aead_decrypt, 3008 .ivsize = CTR_RFC3686_IV_SIZE, 3009 .maxauthsize = MD5_DIGEST_SIZE, 3010 }, 3011 .caam = { 3012 .class1_alg_type = OP_ALG_ALGSEL_AES | 3013 OP_ALG_AAI_CTR_MOD128, 3014 .class2_alg_type = OP_ALG_ALGSEL_MD5 | 3015 OP_ALG_AAI_HMAC_PRECOMP, 3016 .rfc3686 = true, 3017 .geniv = true, 3018 }, 3019 }, 3020 { 3021 .aead = { 3022 .base = { 3023 .cra_name = "authenc(hmac(sha1)," 3024 "rfc3686(ctr(aes)))", 3025 .cra_driver_name = "authenc-hmac-sha1-" 3026 "rfc3686-ctr-aes-caam", 3027 .cra_blocksize = 1, 3028 }, 3029 .setkey = aead_setkey, 3030 .setauthsize = aead_setauthsize, 3031 .encrypt = aead_encrypt, 3032 .decrypt = aead_decrypt, 3033 .ivsize = CTR_RFC3686_IV_SIZE, 3034 .maxauthsize = SHA1_DIGEST_SIZE, 3035 }, 3036 .caam = { 3037 .class1_alg_type = OP_ALG_ALGSEL_AES | 3038 OP_ALG_AAI_CTR_MOD128, 3039 .class2_alg_type = OP_ALG_ALGSEL_SHA1 | 3040 OP_ALG_AAI_HMAC_PRECOMP, 3041 .rfc3686 = true, 3042 }, 3043 }, 3044 { 3045 .aead = { 3046 .base = { 3047 .cra_name = "seqiv(authenc(" 3048 "hmac(sha1),rfc3686(ctr(aes))))", 3049 .cra_driver_name = "seqiv-authenc-hmac-sha1-" 3050 "rfc3686-ctr-aes-caam", 3051 .cra_blocksize = 1, 3052 }, 3053 .setkey = aead_setkey, 3054 .setauthsize = aead_setauthsize, 3055 .encrypt = aead_encrypt, 3056 .decrypt = aead_decrypt, 3057 .ivsize = CTR_RFC3686_IV_SIZE, 3058 .maxauthsize = SHA1_DIGEST_SIZE, 3059 }, 3060 .caam = { 3061 .class1_alg_type = OP_ALG_ALGSEL_AES | 3062 OP_ALG_AAI_CTR_MOD128, 3063 .class2_alg_type = OP_ALG_ALGSEL_SHA1 | 3064 OP_ALG_AAI_HMAC_PRECOMP, 3065 .rfc3686 = true, 3066 .geniv = true, 3067 }, 3068 }, 3069 { 3070 .aead = { 3071 .base = { 3072 .cra_name = "authenc(hmac(sha224)," 3073 "rfc3686(ctr(aes)))", 3074 .cra_driver_name = "authenc-hmac-sha224-" 3075 "rfc3686-ctr-aes-caam", 3076 .cra_blocksize = 1, 3077 }, 3078 .setkey = aead_setkey, 3079 .setauthsize = aead_setauthsize, 3080 .encrypt = aead_encrypt, 3081 .decrypt = aead_decrypt, 3082 .ivsize = CTR_RFC3686_IV_SIZE, 3083 .maxauthsize = SHA224_DIGEST_SIZE, 3084 }, 3085 .caam = { 3086 .class1_alg_type = OP_ALG_ALGSEL_AES | 3087 OP_ALG_AAI_CTR_MOD128, 3088 .class2_alg_type = OP_ALG_ALGSEL_SHA224 | 3089 OP_ALG_AAI_HMAC_PRECOMP, 3090 .rfc3686 = true, 3091 }, 3092 }, 3093 { 3094 .aead = { 3095 .base = { 3096 .cra_name = "seqiv(authenc(" 3097 "hmac(sha224),rfc3686(ctr(aes))))", 3098 .cra_driver_name = "seqiv-authenc-hmac-sha224-" 3099 "rfc3686-ctr-aes-caam", 3100 .cra_blocksize = 1, 3101 }, 3102 .setkey = aead_setkey, 3103 .setauthsize = aead_setauthsize, 3104 .encrypt = aead_encrypt, 3105 .decrypt = aead_decrypt, 3106 .ivsize = CTR_RFC3686_IV_SIZE, 3107 .maxauthsize = SHA224_DIGEST_SIZE, 3108 }, 3109 .caam = { 3110 .class1_alg_type = OP_ALG_ALGSEL_AES | 3111 OP_ALG_AAI_CTR_MOD128, 3112 .class2_alg_type = OP_ALG_ALGSEL_SHA224 | 3113 OP_ALG_AAI_HMAC_PRECOMP, 3114 .rfc3686 = true, 3115 .geniv = true, 3116 }, 3117 }, 3118 { 3119 .aead = { 3120 .base = { 3121 .cra_name = "authenc(hmac(sha256)," 3122 "rfc3686(ctr(aes)))", 3123 .cra_driver_name = "authenc-hmac-sha256-" 3124 "rfc3686-ctr-aes-caam", 3125 .cra_blocksize = 1, 3126 }, 3127 .setkey = aead_setkey, 3128 .setauthsize = aead_setauthsize, 3129 .encrypt = aead_encrypt, 3130 .decrypt = aead_decrypt, 3131 .ivsize = CTR_RFC3686_IV_SIZE, 3132 .maxauthsize = SHA256_DIGEST_SIZE, 3133 }, 3134 .caam = { 3135 .class1_alg_type = OP_ALG_ALGSEL_AES | 3136 OP_ALG_AAI_CTR_MOD128, 3137 .class2_alg_type = OP_ALG_ALGSEL_SHA256 | 3138 OP_ALG_AAI_HMAC_PRECOMP, 3139 .rfc3686 = true, 3140 }, 3141 }, 3142 { 3143 .aead = { 3144 .base = { 3145 .cra_name = "seqiv(authenc(hmac(sha256)," 3146 "rfc3686(ctr(aes))))", 3147 .cra_driver_name = "seqiv-authenc-hmac-sha256-" 3148 "rfc3686-ctr-aes-caam", 3149 .cra_blocksize = 1, 3150 }, 3151 .setkey = aead_setkey, 3152 .setauthsize = aead_setauthsize, 3153 .encrypt = aead_encrypt, 3154 .decrypt = aead_decrypt, 3155 .ivsize = CTR_RFC3686_IV_SIZE, 3156 .maxauthsize = SHA256_DIGEST_SIZE, 3157 }, 3158 .caam = { 3159 .class1_alg_type = OP_ALG_ALGSEL_AES | 3160 OP_ALG_AAI_CTR_MOD128, 3161 .class2_alg_type = OP_ALG_ALGSEL_SHA256 | 3162 OP_ALG_AAI_HMAC_PRECOMP, 3163 .rfc3686 = true, 3164 .geniv = true, 3165 }, 3166 }, 3167 { 3168 .aead = { 3169 .base = { 3170 .cra_name = "authenc(hmac(sha384)," 3171 "rfc3686(ctr(aes)))", 3172 .cra_driver_name = "authenc-hmac-sha384-" 3173 "rfc3686-ctr-aes-caam", 3174 .cra_blocksize = 1, 3175 }, 3176 .setkey = aead_setkey, 3177 .setauthsize = aead_setauthsize, 3178 .encrypt = aead_encrypt, 3179 .decrypt = aead_decrypt, 3180 .ivsize = CTR_RFC3686_IV_SIZE, 3181 .maxauthsize = SHA384_DIGEST_SIZE, 3182 }, 3183 .caam = { 3184 .class1_alg_type = OP_ALG_ALGSEL_AES | 3185 OP_ALG_AAI_CTR_MOD128, 3186 .class2_alg_type = OP_ALG_ALGSEL_SHA384 | 3187 OP_ALG_AAI_HMAC_PRECOMP, 3188 .rfc3686 = true, 3189 }, 3190 }, 3191 { 3192 .aead = { 3193 .base = { 3194 .cra_name = "seqiv(authenc(hmac(sha384)," 3195 "rfc3686(ctr(aes))))", 3196 .cra_driver_name = "seqiv-authenc-hmac-sha384-" 3197 "rfc3686-ctr-aes-caam", 3198 .cra_blocksize = 1, 3199 }, 3200 .setkey = aead_setkey, 3201 .setauthsize = aead_setauthsize, 3202 .encrypt = aead_encrypt, 3203 .decrypt = aead_decrypt, 3204 .ivsize = CTR_RFC3686_IV_SIZE, 3205 .maxauthsize = SHA384_DIGEST_SIZE, 3206 }, 3207 .caam = { 3208 .class1_alg_type = OP_ALG_ALGSEL_AES | 3209 OP_ALG_AAI_CTR_MOD128, 3210 .class2_alg_type = OP_ALG_ALGSEL_SHA384 | 3211 OP_ALG_AAI_HMAC_PRECOMP, 3212 .rfc3686 = true, 3213 .geniv = true, 3214 }, 3215 }, 3216 { 3217 .aead = { 3218 .base = { 3219 .cra_name = "authenc(hmac(sha512)," 3220 "rfc3686(ctr(aes)))", 3221 .cra_driver_name = "authenc-hmac-sha512-" 3222 "rfc3686-ctr-aes-caam", 3223 .cra_blocksize = 1, 3224 }, 3225 .setkey = aead_setkey, 3226 .setauthsize = aead_setauthsize, 3227 .encrypt = aead_encrypt, 3228 .decrypt = aead_decrypt, 3229 .ivsize = CTR_RFC3686_IV_SIZE, 3230 .maxauthsize = SHA512_DIGEST_SIZE, 3231 }, 3232 .caam = { 3233 .class1_alg_type = OP_ALG_ALGSEL_AES | 3234 OP_ALG_AAI_CTR_MOD128, 3235 .class2_alg_type = OP_ALG_ALGSEL_SHA512 | 3236 OP_ALG_AAI_HMAC_PRECOMP, 3237 .rfc3686 = true, 3238 }, 3239 }, 3240 { 3241 .aead = { 3242 .base = { 3243 .cra_name = "seqiv(authenc(hmac(sha512)," 3244 "rfc3686(ctr(aes))))", 3245 .cra_driver_name = "seqiv-authenc-hmac-sha512-" 3246 "rfc3686-ctr-aes-caam", 3247 .cra_blocksize = 1, 3248 }, 3249 .setkey = aead_setkey, 3250 .setauthsize = aead_setauthsize, 3251 .encrypt = aead_encrypt, 3252 .decrypt = aead_decrypt, 3253 .ivsize = CTR_RFC3686_IV_SIZE, 3254 .maxauthsize = SHA512_DIGEST_SIZE, 3255 }, 3256 .caam = { 3257 .class1_alg_type = OP_ALG_ALGSEL_AES | 3258 OP_ALG_AAI_CTR_MOD128, 3259 .class2_alg_type = OP_ALG_ALGSEL_SHA512 | 3260 OP_ALG_AAI_HMAC_PRECOMP, 3261 .rfc3686 = true, 3262 .geniv = true, 3263 }, 3264 }, 3265 { 3266 .aead = { 3267 .base = { 3268 .cra_name = "rfc7539(chacha20,poly1305)", 3269 .cra_driver_name = "rfc7539-chacha20-poly1305-" 3270 "caam", 3271 .cra_blocksize = 1, 3272 }, 3273 .setkey = chachapoly_setkey, 3274 .setauthsize = chachapoly_setauthsize, 3275 .encrypt = chachapoly_encrypt, 3276 .decrypt = chachapoly_decrypt, 3277 .ivsize = CHACHAPOLY_IV_SIZE, 3278 .maxauthsize = POLY1305_DIGEST_SIZE, 3279 }, 3280 .caam = { 3281 .class1_alg_type = OP_ALG_ALGSEL_CHACHA20 | 3282 OP_ALG_AAI_AEAD, 3283 .class2_alg_type = OP_ALG_ALGSEL_POLY1305 | 3284 OP_ALG_AAI_AEAD, 3285 .nodkp = true, 3286 }, 3287 }, 3288 { 3289 .aead = { 3290 .base = { 3291 .cra_name = "rfc7539esp(chacha20,poly1305)", 3292 .cra_driver_name = "rfc7539esp-chacha20-" 3293 "poly1305-caam", 3294 .cra_blocksize = 1, 3295 }, 3296 .setkey = chachapoly_setkey, 3297 .setauthsize = chachapoly_setauthsize, 3298 .encrypt = chachapoly_encrypt, 3299 .decrypt = chachapoly_decrypt, 3300 .ivsize = 8, 3301 .maxauthsize = POLY1305_DIGEST_SIZE, 3302 }, 3303 .caam = { 3304 .class1_alg_type = OP_ALG_ALGSEL_CHACHA20 | 3305 OP_ALG_AAI_AEAD, 3306 .class2_alg_type = OP_ALG_ALGSEL_POLY1305 | 3307 OP_ALG_AAI_AEAD, 3308 .nodkp = true, 3309 }, 3310 }, 3311 }; 3312 3313 static int caam_init_common(struct caam_ctx *ctx, struct caam_alg_entry *caam, 3314 bool uses_dkp) 3315 { 3316 dma_addr_t dma_addr; 3317 struct caam_drv_private *priv; 3318 const size_t sh_desc_enc_offset = offsetof(struct caam_ctx, 3319 sh_desc_enc); 3320 3321 ctx->jrdev = caam_jr_alloc(); 3322 if (IS_ERR(ctx->jrdev)) { 3323 pr_err("Job Ring Device allocation for transform failed\n"); 3324 return PTR_ERR(ctx->jrdev); 3325 } 3326 3327 priv = dev_get_drvdata(ctx->jrdev->parent); 3328 if (priv->era >= 6 && uses_dkp) 3329 ctx->dir = DMA_BIDIRECTIONAL; 3330 else 3331 ctx->dir = DMA_TO_DEVICE; 3332 3333 dma_addr = dma_map_single_attrs(ctx->jrdev, ctx->sh_desc_enc, 3334 offsetof(struct caam_ctx, 3335 sh_desc_enc_dma) - 3336 sh_desc_enc_offset, 3337 ctx->dir, DMA_ATTR_SKIP_CPU_SYNC); 3338 if (dma_mapping_error(ctx->jrdev, dma_addr)) { 3339 dev_err(ctx->jrdev, "unable to map key, shared descriptors\n"); 3340 caam_jr_free(ctx->jrdev); 3341 return -ENOMEM; 3342 } 3343 3344 ctx->sh_desc_enc_dma = dma_addr; 3345 ctx->sh_desc_dec_dma = dma_addr + offsetof(struct caam_ctx, 3346 sh_desc_dec) - 3347 sh_desc_enc_offset; 3348 ctx->key_dma = dma_addr + offsetof(struct caam_ctx, key) - 3349 sh_desc_enc_offset; 3350 3351 /* copy descriptor header template value */ 3352 ctx->cdata.algtype = OP_TYPE_CLASS1_ALG | caam->class1_alg_type; 3353 ctx->adata.algtype = OP_TYPE_CLASS2_ALG | caam->class2_alg_type; 3354 3355 return 0; 3356 } 3357 3358 static int caam_cra_init(struct crypto_skcipher *tfm) 3359 { 3360 struct skcipher_alg *alg = crypto_skcipher_alg(tfm); 3361 struct caam_skcipher_alg *caam_alg = 3362 container_of(alg, typeof(*caam_alg), skcipher); 3363 struct caam_ctx *ctx = crypto_skcipher_ctx(tfm); 3364 3365 crypto_skcipher_set_reqsize(tfm, sizeof(struct caam_skcipher_req_ctx)); 3366 3367 ctx->enginectx.op.do_one_request = skcipher_do_one_req; 3368 3369 return caam_init_common(crypto_skcipher_ctx(tfm), &caam_alg->caam, 3370 false); 3371 } 3372 3373 static int caam_aead_init(struct crypto_aead *tfm) 3374 { 3375 struct aead_alg *alg = crypto_aead_alg(tfm); 3376 struct caam_aead_alg *caam_alg = 3377 container_of(alg, struct caam_aead_alg, aead); 3378 struct caam_ctx *ctx = crypto_aead_ctx(tfm); 3379 3380 crypto_aead_set_reqsize(tfm, sizeof(struct caam_aead_req_ctx)); 3381 3382 ctx->enginectx.op.do_one_request = aead_do_one_req; 3383 3384 return caam_init_common(ctx, &caam_alg->caam, !caam_alg->caam.nodkp); 3385 } 3386 3387 static void caam_exit_common(struct caam_ctx *ctx) 3388 { 3389 dma_unmap_single_attrs(ctx->jrdev, ctx->sh_desc_enc_dma, 3390 offsetof(struct caam_ctx, sh_desc_enc_dma) - 3391 offsetof(struct caam_ctx, sh_desc_enc), 3392 ctx->dir, DMA_ATTR_SKIP_CPU_SYNC); 3393 caam_jr_free(ctx->jrdev); 3394 } 3395 3396 static void caam_cra_exit(struct crypto_skcipher *tfm) 3397 { 3398 caam_exit_common(crypto_skcipher_ctx(tfm)); 3399 } 3400 3401 static void caam_aead_exit(struct crypto_aead *tfm) 3402 { 3403 caam_exit_common(crypto_aead_ctx(tfm)); 3404 } 3405 3406 void caam_algapi_exit(void) 3407 { 3408 int i; 3409 3410 for (i = 0; i < ARRAY_SIZE(driver_aeads); i++) { 3411 struct caam_aead_alg *t_alg = driver_aeads + i; 3412 3413 if (t_alg->registered) 3414 crypto_unregister_aead(&t_alg->aead); 3415 } 3416 3417 for (i = 0; i < ARRAY_SIZE(driver_algs); i++) { 3418 struct caam_skcipher_alg *t_alg = driver_algs + i; 3419 3420 if (t_alg->registered) 3421 crypto_unregister_skcipher(&t_alg->skcipher); 3422 } 3423 } 3424 3425 static void caam_skcipher_alg_init(struct caam_skcipher_alg *t_alg) 3426 { 3427 struct skcipher_alg *alg = &t_alg->skcipher; 3428 3429 alg->base.cra_module = THIS_MODULE; 3430 alg->base.cra_priority = CAAM_CRA_PRIORITY; 3431 alg->base.cra_ctxsize = sizeof(struct caam_ctx); 3432 alg->base.cra_flags = CRYPTO_ALG_ASYNC | CRYPTO_ALG_KERN_DRIVER_ONLY; 3433 3434 alg->init = caam_cra_init; 3435 alg->exit = caam_cra_exit; 3436 } 3437 3438 static void caam_aead_alg_init(struct caam_aead_alg *t_alg) 3439 { 3440 struct aead_alg *alg = &t_alg->aead; 3441 3442 alg->base.cra_module = THIS_MODULE; 3443 alg->base.cra_priority = CAAM_CRA_PRIORITY; 3444 alg->base.cra_ctxsize = sizeof(struct caam_ctx); 3445 alg->base.cra_flags = CRYPTO_ALG_ASYNC | CRYPTO_ALG_KERN_DRIVER_ONLY; 3446 3447 alg->init = caam_aead_init; 3448 alg->exit = caam_aead_exit; 3449 } 3450 3451 int caam_algapi_init(struct device *ctrldev) 3452 { 3453 struct caam_drv_private *priv = dev_get_drvdata(ctrldev); 3454 int i = 0, err = 0; 3455 u32 aes_vid, aes_inst, des_inst, md_vid, md_inst, ccha_inst, ptha_inst; 3456 u32 arc4_inst; 3457 unsigned int md_limit = SHA512_DIGEST_SIZE; 3458 bool registered = false, gcm_support; 3459 3460 /* 3461 * Register crypto algorithms the device supports. 3462 * First, detect presence and attributes of DES, AES, and MD blocks. 3463 */ 3464 if (priv->era < 10) { 3465 u32 cha_vid, cha_inst, aes_rn; 3466 3467 cha_vid = rd_reg32(&priv->ctrl->perfmon.cha_id_ls); 3468 aes_vid = cha_vid & CHA_ID_LS_AES_MASK; 3469 md_vid = (cha_vid & CHA_ID_LS_MD_MASK) >> CHA_ID_LS_MD_SHIFT; 3470 3471 cha_inst = rd_reg32(&priv->ctrl->perfmon.cha_num_ls); 3472 des_inst = (cha_inst & CHA_ID_LS_DES_MASK) >> 3473 CHA_ID_LS_DES_SHIFT; 3474 aes_inst = cha_inst & CHA_ID_LS_AES_MASK; 3475 md_inst = (cha_inst & CHA_ID_LS_MD_MASK) >> CHA_ID_LS_MD_SHIFT; 3476 arc4_inst = (cha_inst & CHA_ID_LS_ARC4_MASK) >> 3477 CHA_ID_LS_ARC4_SHIFT; 3478 ccha_inst = 0; 3479 ptha_inst = 0; 3480 3481 aes_rn = rd_reg32(&priv->ctrl->perfmon.cha_rev_ls) & 3482 CHA_ID_LS_AES_MASK; 3483 gcm_support = !(aes_vid == CHA_VER_VID_AES_LP && aes_rn < 8); 3484 } else { 3485 u32 aesa, mdha; 3486 3487 aesa = rd_reg32(&priv->ctrl->vreg.aesa); 3488 mdha = rd_reg32(&priv->ctrl->vreg.mdha); 3489 3490 aes_vid = (aesa & CHA_VER_VID_MASK) >> CHA_VER_VID_SHIFT; 3491 md_vid = (mdha & CHA_VER_VID_MASK) >> CHA_VER_VID_SHIFT; 3492 3493 des_inst = rd_reg32(&priv->ctrl->vreg.desa) & CHA_VER_NUM_MASK; 3494 aes_inst = aesa & CHA_VER_NUM_MASK; 3495 md_inst = mdha & CHA_VER_NUM_MASK; 3496 ccha_inst = rd_reg32(&priv->ctrl->vreg.ccha) & CHA_VER_NUM_MASK; 3497 ptha_inst = rd_reg32(&priv->ctrl->vreg.ptha) & CHA_VER_NUM_MASK; 3498 arc4_inst = rd_reg32(&priv->ctrl->vreg.afha) & CHA_VER_NUM_MASK; 3499 3500 gcm_support = aesa & CHA_VER_MISC_AES_GCM; 3501 } 3502 3503 /* If MD is present, limit digest size based on LP256 */ 3504 if (md_inst && md_vid == CHA_VER_VID_MD_LP256) 3505 md_limit = SHA256_DIGEST_SIZE; 3506 3507 for (i = 0; i < ARRAY_SIZE(driver_algs); i++) { 3508 struct caam_skcipher_alg *t_alg = driver_algs + i; 3509 u32 alg_sel = t_alg->caam.class1_alg_type & OP_ALG_ALGSEL_MASK; 3510 3511 /* Skip DES algorithms if not supported by device */ 3512 if (!des_inst && 3513 ((alg_sel == OP_ALG_ALGSEL_3DES) || 3514 (alg_sel == OP_ALG_ALGSEL_DES))) 3515 continue; 3516 3517 /* Skip AES algorithms if not supported by device */ 3518 if (!aes_inst && (alg_sel == OP_ALG_ALGSEL_AES)) 3519 continue; 3520 3521 /* Skip ARC4 algorithms if not supported by device */ 3522 if (!arc4_inst && alg_sel == OP_ALG_ALGSEL_ARC4) 3523 continue; 3524 3525 /* 3526 * Check support for AES modes not available 3527 * on LP devices. 3528 */ 3529 if (aes_vid == CHA_VER_VID_AES_LP && 3530 (t_alg->caam.class1_alg_type & OP_ALG_AAI_MASK) == 3531 OP_ALG_AAI_XTS) 3532 continue; 3533 3534 caam_skcipher_alg_init(t_alg); 3535 3536 err = crypto_register_skcipher(&t_alg->skcipher); 3537 if (err) { 3538 pr_warn("%s alg registration failed\n", 3539 t_alg->skcipher.base.cra_driver_name); 3540 continue; 3541 } 3542 3543 t_alg->registered = true; 3544 registered = true; 3545 } 3546 3547 for (i = 0; i < ARRAY_SIZE(driver_aeads); i++) { 3548 struct caam_aead_alg *t_alg = driver_aeads + i; 3549 u32 c1_alg_sel = t_alg->caam.class1_alg_type & 3550 OP_ALG_ALGSEL_MASK; 3551 u32 c2_alg_sel = t_alg->caam.class2_alg_type & 3552 OP_ALG_ALGSEL_MASK; 3553 u32 alg_aai = t_alg->caam.class1_alg_type & OP_ALG_AAI_MASK; 3554 3555 /* Skip DES algorithms if not supported by device */ 3556 if (!des_inst && 3557 ((c1_alg_sel == OP_ALG_ALGSEL_3DES) || 3558 (c1_alg_sel == OP_ALG_ALGSEL_DES))) 3559 continue; 3560 3561 /* Skip AES algorithms if not supported by device */ 3562 if (!aes_inst && (c1_alg_sel == OP_ALG_ALGSEL_AES)) 3563 continue; 3564 3565 /* Skip CHACHA20 algorithms if not supported by device */ 3566 if (c1_alg_sel == OP_ALG_ALGSEL_CHACHA20 && !ccha_inst) 3567 continue; 3568 3569 /* Skip POLY1305 algorithms if not supported by device */ 3570 if (c2_alg_sel == OP_ALG_ALGSEL_POLY1305 && !ptha_inst) 3571 continue; 3572 3573 /* Skip GCM algorithms if not supported by device */ 3574 if (c1_alg_sel == OP_ALG_ALGSEL_AES && 3575 alg_aai == OP_ALG_AAI_GCM && !gcm_support) 3576 continue; 3577 3578 /* 3579 * Skip algorithms requiring message digests 3580 * if MD or MD size is not supported by device. 3581 */ 3582 if (is_mdha(c2_alg_sel) && 3583 (!md_inst || t_alg->aead.maxauthsize > md_limit)) 3584 continue; 3585 3586 caam_aead_alg_init(t_alg); 3587 3588 err = crypto_register_aead(&t_alg->aead); 3589 if (err) { 3590 pr_warn("%s alg registration failed\n", 3591 t_alg->aead.base.cra_driver_name); 3592 continue; 3593 } 3594 3595 t_alg->registered = true; 3596 registered = true; 3597 } 3598 3599 if (registered) 3600 pr_info("caam algorithms registered in /proc/crypto\n"); 3601 3602 return err; 3603 } 3604