1 // SPDX-License-Identifier: GPL-2.0+ 2 /* 3 * caam - Freescale FSL CAAM support for crypto API 4 * 5 * Copyright 2008-2011 Freescale Semiconductor, Inc. 6 * Copyright 2016-2019 NXP 7 * 8 * Based on talitos crypto API driver. 9 * 10 * relationship of job descriptors to shared descriptors (SteveC Dec 10 2008): 11 * 12 * --------------- --------------- 13 * | JobDesc #1 |-------------------->| ShareDesc | 14 * | *(packet 1) | | (PDB) | 15 * --------------- |------------->| (hashKey) | 16 * . | | (cipherKey) | 17 * . | |-------->| (operation) | 18 * --------------- | | --------------- 19 * | JobDesc #2 |------| | 20 * | *(packet 2) | | 21 * --------------- | 22 * . | 23 * . | 24 * --------------- | 25 * | JobDesc #3 |------------ 26 * | *(packet 3) | 27 * --------------- 28 * 29 * The SharedDesc never changes for a connection unless rekeyed, but 30 * each packet will likely be in a different place. So all we need 31 * to know to process the packet is where the input is, where the 32 * output goes, and what context we want to process with. Context is 33 * in the SharedDesc, packet references in the JobDesc. 34 * 35 * So, a job desc looks like: 36 * 37 * --------------------- 38 * | Header | 39 * | ShareDesc Pointer | 40 * | SEQ_OUT_PTR | 41 * | (output buffer) | 42 * | (output length) | 43 * | SEQ_IN_PTR | 44 * | (input buffer) | 45 * | (input length) | 46 * --------------------- 47 */ 48 49 #include "compat.h" 50 51 #include "regs.h" 52 #include "intern.h" 53 #include "desc_constr.h" 54 #include "jr.h" 55 #include "error.h" 56 #include "sg_sw_sec4.h" 57 #include "key_gen.h" 58 #include "caamalg_desc.h" 59 #include <crypto/engine.h> 60 #include <crypto/xts.h> 61 #include <asm/unaligned.h> 62 63 /* 64 * crypto alg 65 */ 66 #define CAAM_CRA_PRIORITY 3000 67 /* max key is sum of AES_MAX_KEY_SIZE, max split key size */ 68 #define CAAM_MAX_KEY_SIZE (AES_MAX_KEY_SIZE + \ 69 CTR_RFC3686_NONCE_SIZE + \ 70 SHA512_DIGEST_SIZE * 2) 71 72 #define AEAD_DESC_JOB_IO_LEN (DESC_JOB_IO_LEN + CAAM_CMD_SZ * 2) 73 #define GCM_DESC_JOB_IO_LEN (AEAD_DESC_JOB_IO_LEN + \ 74 CAAM_CMD_SZ * 4) 75 #define AUTHENC_DESC_JOB_IO_LEN (AEAD_DESC_JOB_IO_LEN + \ 76 CAAM_CMD_SZ * 5) 77 78 #define CHACHAPOLY_DESC_JOB_IO_LEN (AEAD_DESC_JOB_IO_LEN + CAAM_CMD_SZ * 6) 79 80 #define DESC_MAX_USED_BYTES (CAAM_DESC_BYTES_MAX - DESC_JOB_IO_LEN_MIN) 81 #define DESC_MAX_USED_LEN (DESC_MAX_USED_BYTES / CAAM_CMD_SZ) 82 83 struct caam_alg_entry { 84 int class1_alg_type; 85 int class2_alg_type; 86 bool rfc3686; 87 bool geniv; 88 bool nodkp; 89 }; 90 91 struct caam_aead_alg { 92 struct aead_alg aead; 93 struct caam_alg_entry caam; 94 bool registered; 95 }; 96 97 struct caam_skcipher_alg { 98 struct skcipher_alg skcipher; 99 struct caam_alg_entry caam; 100 bool registered; 101 }; 102 103 /* 104 * per-session context 105 */ 106 struct caam_ctx { 107 struct crypto_engine_ctx enginectx; 108 u32 sh_desc_enc[DESC_MAX_USED_LEN]; 109 u32 sh_desc_dec[DESC_MAX_USED_LEN]; 110 u8 key[CAAM_MAX_KEY_SIZE]; 111 dma_addr_t sh_desc_enc_dma; 112 dma_addr_t sh_desc_dec_dma; 113 dma_addr_t key_dma; 114 enum dma_data_direction dir; 115 struct device *jrdev; 116 struct alginfo adata; 117 struct alginfo cdata; 118 unsigned int authsize; 119 bool xts_key_fallback; 120 struct crypto_skcipher *fallback; 121 }; 122 123 struct caam_skcipher_req_ctx { 124 struct skcipher_edesc *edesc; 125 struct skcipher_request fallback_req; 126 }; 127 128 struct caam_aead_req_ctx { 129 struct aead_edesc *edesc; 130 }; 131 132 static int aead_null_set_sh_desc(struct crypto_aead *aead) 133 { 134 struct caam_ctx *ctx = crypto_aead_ctx_dma(aead); 135 struct device *jrdev = ctx->jrdev; 136 struct caam_drv_private *ctrlpriv = dev_get_drvdata(jrdev->parent); 137 u32 *desc; 138 int rem_bytes = CAAM_DESC_BYTES_MAX - AEAD_DESC_JOB_IO_LEN - 139 ctx->adata.keylen_pad; 140 141 /* 142 * Job Descriptor and Shared Descriptors 143 * must all fit into the 64-word Descriptor h/w Buffer 144 */ 145 if (rem_bytes >= DESC_AEAD_NULL_ENC_LEN) { 146 ctx->adata.key_inline = true; 147 ctx->adata.key_virt = ctx->key; 148 } else { 149 ctx->adata.key_inline = false; 150 ctx->adata.key_dma = ctx->key_dma; 151 } 152 153 /* aead_encrypt shared descriptor */ 154 desc = ctx->sh_desc_enc; 155 cnstr_shdsc_aead_null_encap(desc, &ctx->adata, ctx->authsize, 156 ctrlpriv->era); 157 dma_sync_single_for_device(jrdev, ctx->sh_desc_enc_dma, 158 desc_bytes(desc), ctx->dir); 159 160 /* 161 * Job Descriptor and Shared Descriptors 162 * must all fit into the 64-word Descriptor h/w Buffer 163 */ 164 if (rem_bytes >= DESC_AEAD_NULL_DEC_LEN) { 165 ctx->adata.key_inline = true; 166 ctx->adata.key_virt = ctx->key; 167 } else { 168 ctx->adata.key_inline = false; 169 ctx->adata.key_dma = ctx->key_dma; 170 } 171 172 /* aead_decrypt shared descriptor */ 173 desc = ctx->sh_desc_dec; 174 cnstr_shdsc_aead_null_decap(desc, &ctx->adata, ctx->authsize, 175 ctrlpriv->era); 176 dma_sync_single_for_device(jrdev, ctx->sh_desc_dec_dma, 177 desc_bytes(desc), ctx->dir); 178 179 return 0; 180 } 181 182 static int aead_set_sh_desc(struct crypto_aead *aead) 183 { 184 struct caam_aead_alg *alg = container_of(crypto_aead_alg(aead), 185 struct caam_aead_alg, aead); 186 unsigned int ivsize = crypto_aead_ivsize(aead); 187 struct caam_ctx *ctx = crypto_aead_ctx_dma(aead); 188 struct device *jrdev = ctx->jrdev; 189 struct caam_drv_private *ctrlpriv = dev_get_drvdata(jrdev->parent); 190 u32 ctx1_iv_off = 0; 191 u32 *desc, *nonce = NULL; 192 u32 inl_mask; 193 unsigned int data_len[2]; 194 const bool ctr_mode = ((ctx->cdata.algtype & OP_ALG_AAI_MASK) == 195 OP_ALG_AAI_CTR_MOD128); 196 const bool is_rfc3686 = alg->caam.rfc3686; 197 198 if (!ctx->authsize) 199 return 0; 200 201 /* NULL encryption / decryption */ 202 if (!ctx->cdata.keylen) 203 return aead_null_set_sh_desc(aead); 204 205 /* 206 * AES-CTR needs to load IV in CONTEXT1 reg 207 * at an offset of 128bits (16bytes) 208 * CONTEXT1[255:128] = IV 209 */ 210 if (ctr_mode) 211 ctx1_iv_off = 16; 212 213 /* 214 * RFC3686 specific: 215 * CONTEXT1[255:128] = {NONCE, IV, COUNTER} 216 */ 217 if (is_rfc3686) { 218 ctx1_iv_off = 16 + CTR_RFC3686_NONCE_SIZE; 219 nonce = (u32 *)((void *)ctx->key + ctx->adata.keylen_pad + 220 ctx->cdata.keylen - CTR_RFC3686_NONCE_SIZE); 221 } 222 223 /* 224 * In case |user key| > |derived key|, using DKP<imm,imm> 225 * would result in invalid opcodes (last bytes of user key) in 226 * the resulting descriptor. Use DKP<ptr,imm> instead => both 227 * virtual and dma key addresses are needed. 228 */ 229 ctx->adata.key_virt = ctx->key; 230 ctx->adata.key_dma = ctx->key_dma; 231 232 ctx->cdata.key_virt = ctx->key + ctx->adata.keylen_pad; 233 ctx->cdata.key_dma = ctx->key_dma + ctx->adata.keylen_pad; 234 235 data_len[0] = ctx->adata.keylen_pad; 236 data_len[1] = ctx->cdata.keylen; 237 238 if (alg->caam.geniv) 239 goto skip_enc; 240 241 /* 242 * Job Descriptor and Shared Descriptors 243 * must all fit into the 64-word Descriptor h/w Buffer 244 */ 245 if (desc_inline_query(DESC_AEAD_ENC_LEN + 246 (is_rfc3686 ? DESC_AEAD_CTR_RFC3686_LEN : 0), 247 AUTHENC_DESC_JOB_IO_LEN, data_len, &inl_mask, 248 ARRAY_SIZE(data_len)) < 0) 249 return -EINVAL; 250 251 ctx->adata.key_inline = !!(inl_mask & 1); 252 ctx->cdata.key_inline = !!(inl_mask & 2); 253 254 /* aead_encrypt shared descriptor */ 255 desc = ctx->sh_desc_enc; 256 cnstr_shdsc_aead_encap(desc, &ctx->cdata, &ctx->adata, ivsize, 257 ctx->authsize, is_rfc3686, nonce, ctx1_iv_off, 258 false, ctrlpriv->era); 259 dma_sync_single_for_device(jrdev, ctx->sh_desc_enc_dma, 260 desc_bytes(desc), ctx->dir); 261 262 skip_enc: 263 /* 264 * Job Descriptor and Shared Descriptors 265 * must all fit into the 64-word Descriptor h/w Buffer 266 */ 267 if (desc_inline_query(DESC_AEAD_DEC_LEN + 268 (is_rfc3686 ? DESC_AEAD_CTR_RFC3686_LEN : 0), 269 AUTHENC_DESC_JOB_IO_LEN, data_len, &inl_mask, 270 ARRAY_SIZE(data_len)) < 0) 271 return -EINVAL; 272 273 ctx->adata.key_inline = !!(inl_mask & 1); 274 ctx->cdata.key_inline = !!(inl_mask & 2); 275 276 /* aead_decrypt shared descriptor */ 277 desc = ctx->sh_desc_dec; 278 cnstr_shdsc_aead_decap(desc, &ctx->cdata, &ctx->adata, ivsize, 279 ctx->authsize, alg->caam.geniv, is_rfc3686, 280 nonce, ctx1_iv_off, false, ctrlpriv->era); 281 dma_sync_single_for_device(jrdev, ctx->sh_desc_dec_dma, 282 desc_bytes(desc), ctx->dir); 283 284 if (!alg->caam.geniv) 285 goto skip_givenc; 286 287 /* 288 * Job Descriptor and Shared Descriptors 289 * must all fit into the 64-word Descriptor h/w Buffer 290 */ 291 if (desc_inline_query(DESC_AEAD_GIVENC_LEN + 292 (is_rfc3686 ? DESC_AEAD_CTR_RFC3686_LEN : 0), 293 AUTHENC_DESC_JOB_IO_LEN, data_len, &inl_mask, 294 ARRAY_SIZE(data_len)) < 0) 295 return -EINVAL; 296 297 ctx->adata.key_inline = !!(inl_mask & 1); 298 ctx->cdata.key_inline = !!(inl_mask & 2); 299 300 /* aead_givencrypt shared descriptor */ 301 desc = ctx->sh_desc_enc; 302 cnstr_shdsc_aead_givencap(desc, &ctx->cdata, &ctx->adata, ivsize, 303 ctx->authsize, is_rfc3686, nonce, 304 ctx1_iv_off, false, ctrlpriv->era); 305 dma_sync_single_for_device(jrdev, ctx->sh_desc_enc_dma, 306 desc_bytes(desc), ctx->dir); 307 308 skip_givenc: 309 return 0; 310 } 311 312 static int aead_setauthsize(struct crypto_aead *authenc, 313 unsigned int authsize) 314 { 315 struct caam_ctx *ctx = crypto_aead_ctx_dma(authenc); 316 317 ctx->authsize = authsize; 318 aead_set_sh_desc(authenc); 319 320 return 0; 321 } 322 323 static int gcm_set_sh_desc(struct crypto_aead *aead) 324 { 325 struct caam_ctx *ctx = crypto_aead_ctx_dma(aead); 326 struct device *jrdev = ctx->jrdev; 327 unsigned int ivsize = crypto_aead_ivsize(aead); 328 u32 *desc; 329 int rem_bytes = CAAM_DESC_BYTES_MAX - GCM_DESC_JOB_IO_LEN - 330 ctx->cdata.keylen; 331 332 if (!ctx->cdata.keylen || !ctx->authsize) 333 return 0; 334 335 /* 336 * AES GCM encrypt shared descriptor 337 * Job Descriptor and Shared Descriptor 338 * must fit into the 64-word Descriptor h/w Buffer 339 */ 340 if (rem_bytes >= DESC_GCM_ENC_LEN) { 341 ctx->cdata.key_inline = true; 342 ctx->cdata.key_virt = ctx->key; 343 } else { 344 ctx->cdata.key_inline = false; 345 ctx->cdata.key_dma = ctx->key_dma; 346 } 347 348 desc = ctx->sh_desc_enc; 349 cnstr_shdsc_gcm_encap(desc, &ctx->cdata, ivsize, ctx->authsize, false); 350 dma_sync_single_for_device(jrdev, ctx->sh_desc_enc_dma, 351 desc_bytes(desc), ctx->dir); 352 353 /* 354 * Job Descriptor and Shared Descriptors 355 * must all fit into the 64-word Descriptor h/w Buffer 356 */ 357 if (rem_bytes >= DESC_GCM_DEC_LEN) { 358 ctx->cdata.key_inline = true; 359 ctx->cdata.key_virt = ctx->key; 360 } else { 361 ctx->cdata.key_inline = false; 362 ctx->cdata.key_dma = ctx->key_dma; 363 } 364 365 desc = ctx->sh_desc_dec; 366 cnstr_shdsc_gcm_decap(desc, &ctx->cdata, ivsize, ctx->authsize, false); 367 dma_sync_single_for_device(jrdev, ctx->sh_desc_dec_dma, 368 desc_bytes(desc), ctx->dir); 369 370 return 0; 371 } 372 373 static int gcm_setauthsize(struct crypto_aead *authenc, unsigned int authsize) 374 { 375 struct caam_ctx *ctx = crypto_aead_ctx_dma(authenc); 376 int err; 377 378 err = crypto_gcm_check_authsize(authsize); 379 if (err) 380 return err; 381 382 ctx->authsize = authsize; 383 gcm_set_sh_desc(authenc); 384 385 return 0; 386 } 387 388 static int rfc4106_set_sh_desc(struct crypto_aead *aead) 389 { 390 struct caam_ctx *ctx = crypto_aead_ctx_dma(aead); 391 struct device *jrdev = ctx->jrdev; 392 unsigned int ivsize = crypto_aead_ivsize(aead); 393 u32 *desc; 394 int rem_bytes = CAAM_DESC_BYTES_MAX - GCM_DESC_JOB_IO_LEN - 395 ctx->cdata.keylen; 396 397 if (!ctx->cdata.keylen || !ctx->authsize) 398 return 0; 399 400 /* 401 * RFC4106 encrypt shared descriptor 402 * Job Descriptor and Shared Descriptor 403 * must fit into the 64-word Descriptor h/w Buffer 404 */ 405 if (rem_bytes >= DESC_RFC4106_ENC_LEN) { 406 ctx->cdata.key_inline = true; 407 ctx->cdata.key_virt = ctx->key; 408 } else { 409 ctx->cdata.key_inline = false; 410 ctx->cdata.key_dma = ctx->key_dma; 411 } 412 413 desc = ctx->sh_desc_enc; 414 cnstr_shdsc_rfc4106_encap(desc, &ctx->cdata, ivsize, ctx->authsize, 415 false); 416 dma_sync_single_for_device(jrdev, ctx->sh_desc_enc_dma, 417 desc_bytes(desc), ctx->dir); 418 419 /* 420 * Job Descriptor and Shared Descriptors 421 * must all fit into the 64-word Descriptor h/w Buffer 422 */ 423 if (rem_bytes >= DESC_RFC4106_DEC_LEN) { 424 ctx->cdata.key_inline = true; 425 ctx->cdata.key_virt = ctx->key; 426 } else { 427 ctx->cdata.key_inline = false; 428 ctx->cdata.key_dma = ctx->key_dma; 429 } 430 431 desc = ctx->sh_desc_dec; 432 cnstr_shdsc_rfc4106_decap(desc, &ctx->cdata, ivsize, ctx->authsize, 433 false); 434 dma_sync_single_for_device(jrdev, ctx->sh_desc_dec_dma, 435 desc_bytes(desc), ctx->dir); 436 437 return 0; 438 } 439 440 static int rfc4106_setauthsize(struct crypto_aead *authenc, 441 unsigned int authsize) 442 { 443 struct caam_ctx *ctx = crypto_aead_ctx_dma(authenc); 444 int err; 445 446 err = crypto_rfc4106_check_authsize(authsize); 447 if (err) 448 return err; 449 450 ctx->authsize = authsize; 451 rfc4106_set_sh_desc(authenc); 452 453 return 0; 454 } 455 456 static int rfc4543_set_sh_desc(struct crypto_aead *aead) 457 { 458 struct caam_ctx *ctx = crypto_aead_ctx_dma(aead); 459 struct device *jrdev = ctx->jrdev; 460 unsigned int ivsize = crypto_aead_ivsize(aead); 461 u32 *desc; 462 int rem_bytes = CAAM_DESC_BYTES_MAX - GCM_DESC_JOB_IO_LEN - 463 ctx->cdata.keylen; 464 465 if (!ctx->cdata.keylen || !ctx->authsize) 466 return 0; 467 468 /* 469 * RFC4543 encrypt shared descriptor 470 * Job Descriptor and Shared Descriptor 471 * must fit into the 64-word Descriptor h/w Buffer 472 */ 473 if (rem_bytes >= DESC_RFC4543_ENC_LEN) { 474 ctx->cdata.key_inline = true; 475 ctx->cdata.key_virt = ctx->key; 476 } else { 477 ctx->cdata.key_inline = false; 478 ctx->cdata.key_dma = ctx->key_dma; 479 } 480 481 desc = ctx->sh_desc_enc; 482 cnstr_shdsc_rfc4543_encap(desc, &ctx->cdata, ivsize, ctx->authsize, 483 false); 484 dma_sync_single_for_device(jrdev, ctx->sh_desc_enc_dma, 485 desc_bytes(desc), ctx->dir); 486 487 /* 488 * Job Descriptor and Shared Descriptors 489 * must all fit into the 64-word Descriptor h/w Buffer 490 */ 491 if (rem_bytes >= DESC_RFC4543_DEC_LEN) { 492 ctx->cdata.key_inline = true; 493 ctx->cdata.key_virt = ctx->key; 494 } else { 495 ctx->cdata.key_inline = false; 496 ctx->cdata.key_dma = ctx->key_dma; 497 } 498 499 desc = ctx->sh_desc_dec; 500 cnstr_shdsc_rfc4543_decap(desc, &ctx->cdata, ivsize, ctx->authsize, 501 false); 502 dma_sync_single_for_device(jrdev, ctx->sh_desc_dec_dma, 503 desc_bytes(desc), ctx->dir); 504 505 return 0; 506 } 507 508 static int rfc4543_setauthsize(struct crypto_aead *authenc, 509 unsigned int authsize) 510 { 511 struct caam_ctx *ctx = crypto_aead_ctx_dma(authenc); 512 513 if (authsize != 16) 514 return -EINVAL; 515 516 ctx->authsize = authsize; 517 rfc4543_set_sh_desc(authenc); 518 519 return 0; 520 } 521 522 static int chachapoly_set_sh_desc(struct crypto_aead *aead) 523 { 524 struct caam_ctx *ctx = crypto_aead_ctx_dma(aead); 525 struct device *jrdev = ctx->jrdev; 526 unsigned int ivsize = crypto_aead_ivsize(aead); 527 u32 *desc; 528 529 if (!ctx->cdata.keylen || !ctx->authsize) 530 return 0; 531 532 desc = ctx->sh_desc_enc; 533 cnstr_shdsc_chachapoly(desc, &ctx->cdata, &ctx->adata, ivsize, 534 ctx->authsize, true, false); 535 dma_sync_single_for_device(jrdev, ctx->sh_desc_enc_dma, 536 desc_bytes(desc), ctx->dir); 537 538 desc = ctx->sh_desc_dec; 539 cnstr_shdsc_chachapoly(desc, &ctx->cdata, &ctx->adata, ivsize, 540 ctx->authsize, false, false); 541 dma_sync_single_for_device(jrdev, ctx->sh_desc_dec_dma, 542 desc_bytes(desc), ctx->dir); 543 544 return 0; 545 } 546 547 static int chachapoly_setauthsize(struct crypto_aead *aead, 548 unsigned int authsize) 549 { 550 struct caam_ctx *ctx = crypto_aead_ctx_dma(aead); 551 552 if (authsize != POLY1305_DIGEST_SIZE) 553 return -EINVAL; 554 555 ctx->authsize = authsize; 556 return chachapoly_set_sh_desc(aead); 557 } 558 559 static int chachapoly_setkey(struct crypto_aead *aead, const u8 *key, 560 unsigned int keylen) 561 { 562 struct caam_ctx *ctx = crypto_aead_ctx_dma(aead); 563 unsigned int ivsize = crypto_aead_ivsize(aead); 564 unsigned int saltlen = CHACHAPOLY_IV_SIZE - ivsize; 565 566 if (keylen != CHACHA_KEY_SIZE + saltlen) 567 return -EINVAL; 568 569 ctx->cdata.key_virt = key; 570 ctx->cdata.keylen = keylen - saltlen; 571 572 return chachapoly_set_sh_desc(aead); 573 } 574 575 static int aead_setkey(struct crypto_aead *aead, 576 const u8 *key, unsigned int keylen) 577 { 578 struct caam_ctx *ctx = crypto_aead_ctx_dma(aead); 579 struct device *jrdev = ctx->jrdev; 580 struct caam_drv_private *ctrlpriv = dev_get_drvdata(jrdev->parent); 581 struct crypto_authenc_keys keys; 582 int ret = 0; 583 584 if (crypto_authenc_extractkeys(&keys, key, keylen) != 0) 585 goto badkey; 586 587 dev_dbg(jrdev, "keylen %d enckeylen %d authkeylen %d\n", 588 keys.authkeylen + keys.enckeylen, keys.enckeylen, 589 keys.authkeylen); 590 print_hex_dump_debug("key in @"__stringify(__LINE__)": ", 591 DUMP_PREFIX_ADDRESS, 16, 4, key, keylen, 1); 592 593 /* 594 * If DKP is supported, use it in the shared descriptor to generate 595 * the split key. 596 */ 597 if (ctrlpriv->era >= 6) { 598 ctx->adata.keylen = keys.authkeylen; 599 ctx->adata.keylen_pad = split_key_len(ctx->adata.algtype & 600 OP_ALG_ALGSEL_MASK); 601 602 if (ctx->adata.keylen_pad + keys.enckeylen > CAAM_MAX_KEY_SIZE) 603 goto badkey; 604 605 memcpy(ctx->key, keys.authkey, keys.authkeylen); 606 memcpy(ctx->key + ctx->adata.keylen_pad, keys.enckey, 607 keys.enckeylen); 608 dma_sync_single_for_device(jrdev, ctx->key_dma, 609 ctx->adata.keylen_pad + 610 keys.enckeylen, ctx->dir); 611 goto skip_split_key; 612 } 613 614 ret = gen_split_key(ctx->jrdev, ctx->key, &ctx->adata, keys.authkey, 615 keys.authkeylen, CAAM_MAX_KEY_SIZE - 616 keys.enckeylen); 617 if (ret) { 618 goto badkey; 619 } 620 621 /* postpend encryption key to auth split key */ 622 memcpy(ctx->key + ctx->adata.keylen_pad, keys.enckey, keys.enckeylen); 623 dma_sync_single_for_device(jrdev, ctx->key_dma, ctx->adata.keylen_pad + 624 keys.enckeylen, ctx->dir); 625 626 print_hex_dump_debug("ctx.key@"__stringify(__LINE__)": ", 627 DUMP_PREFIX_ADDRESS, 16, 4, ctx->key, 628 ctx->adata.keylen_pad + keys.enckeylen, 1); 629 630 skip_split_key: 631 ctx->cdata.keylen = keys.enckeylen; 632 memzero_explicit(&keys, sizeof(keys)); 633 return aead_set_sh_desc(aead); 634 badkey: 635 memzero_explicit(&keys, sizeof(keys)); 636 return -EINVAL; 637 } 638 639 static int des3_aead_setkey(struct crypto_aead *aead, const u8 *key, 640 unsigned int keylen) 641 { 642 struct crypto_authenc_keys keys; 643 int err; 644 645 err = crypto_authenc_extractkeys(&keys, key, keylen); 646 if (unlikely(err)) 647 return err; 648 649 err = verify_aead_des3_key(aead, keys.enckey, keys.enckeylen) ?: 650 aead_setkey(aead, key, keylen); 651 652 memzero_explicit(&keys, sizeof(keys)); 653 return err; 654 } 655 656 static int gcm_setkey(struct crypto_aead *aead, 657 const u8 *key, unsigned int keylen) 658 { 659 struct caam_ctx *ctx = crypto_aead_ctx_dma(aead); 660 struct device *jrdev = ctx->jrdev; 661 int err; 662 663 err = aes_check_keylen(keylen); 664 if (err) 665 return err; 666 667 print_hex_dump_debug("key in @"__stringify(__LINE__)": ", 668 DUMP_PREFIX_ADDRESS, 16, 4, key, keylen, 1); 669 670 memcpy(ctx->key, key, keylen); 671 dma_sync_single_for_device(jrdev, ctx->key_dma, keylen, ctx->dir); 672 ctx->cdata.keylen = keylen; 673 674 return gcm_set_sh_desc(aead); 675 } 676 677 static int rfc4106_setkey(struct crypto_aead *aead, 678 const u8 *key, unsigned int keylen) 679 { 680 struct caam_ctx *ctx = crypto_aead_ctx_dma(aead); 681 struct device *jrdev = ctx->jrdev; 682 int err; 683 684 err = aes_check_keylen(keylen - 4); 685 if (err) 686 return err; 687 688 print_hex_dump_debug("key in @"__stringify(__LINE__)": ", 689 DUMP_PREFIX_ADDRESS, 16, 4, key, keylen, 1); 690 691 memcpy(ctx->key, key, keylen); 692 693 /* 694 * The last four bytes of the key material are used as the salt value 695 * in the nonce. Update the AES key length. 696 */ 697 ctx->cdata.keylen = keylen - 4; 698 dma_sync_single_for_device(jrdev, ctx->key_dma, ctx->cdata.keylen, 699 ctx->dir); 700 return rfc4106_set_sh_desc(aead); 701 } 702 703 static int rfc4543_setkey(struct crypto_aead *aead, 704 const u8 *key, unsigned int keylen) 705 { 706 struct caam_ctx *ctx = crypto_aead_ctx_dma(aead); 707 struct device *jrdev = ctx->jrdev; 708 int err; 709 710 err = aes_check_keylen(keylen - 4); 711 if (err) 712 return err; 713 714 print_hex_dump_debug("key in @"__stringify(__LINE__)": ", 715 DUMP_PREFIX_ADDRESS, 16, 4, key, keylen, 1); 716 717 memcpy(ctx->key, key, keylen); 718 719 /* 720 * The last four bytes of the key material are used as the salt value 721 * in the nonce. Update the AES key length. 722 */ 723 ctx->cdata.keylen = keylen - 4; 724 dma_sync_single_for_device(jrdev, ctx->key_dma, ctx->cdata.keylen, 725 ctx->dir); 726 return rfc4543_set_sh_desc(aead); 727 } 728 729 static int skcipher_setkey(struct crypto_skcipher *skcipher, const u8 *key, 730 unsigned int keylen, const u32 ctx1_iv_off) 731 { 732 struct caam_ctx *ctx = crypto_skcipher_ctx_dma(skcipher); 733 struct caam_skcipher_alg *alg = 734 container_of(crypto_skcipher_alg(skcipher), typeof(*alg), 735 skcipher); 736 struct device *jrdev = ctx->jrdev; 737 unsigned int ivsize = crypto_skcipher_ivsize(skcipher); 738 u32 *desc; 739 const bool is_rfc3686 = alg->caam.rfc3686; 740 741 print_hex_dump_debug("key in @"__stringify(__LINE__)": ", 742 DUMP_PREFIX_ADDRESS, 16, 4, key, keylen, 1); 743 744 ctx->cdata.keylen = keylen; 745 ctx->cdata.key_virt = key; 746 ctx->cdata.key_inline = true; 747 748 /* skcipher_encrypt shared descriptor */ 749 desc = ctx->sh_desc_enc; 750 cnstr_shdsc_skcipher_encap(desc, &ctx->cdata, ivsize, is_rfc3686, 751 ctx1_iv_off); 752 dma_sync_single_for_device(jrdev, ctx->sh_desc_enc_dma, 753 desc_bytes(desc), ctx->dir); 754 755 /* skcipher_decrypt shared descriptor */ 756 desc = ctx->sh_desc_dec; 757 cnstr_shdsc_skcipher_decap(desc, &ctx->cdata, ivsize, is_rfc3686, 758 ctx1_iv_off); 759 dma_sync_single_for_device(jrdev, ctx->sh_desc_dec_dma, 760 desc_bytes(desc), ctx->dir); 761 762 return 0; 763 } 764 765 static int aes_skcipher_setkey(struct crypto_skcipher *skcipher, 766 const u8 *key, unsigned int keylen) 767 { 768 int err; 769 770 err = aes_check_keylen(keylen); 771 if (err) 772 return err; 773 774 return skcipher_setkey(skcipher, key, keylen, 0); 775 } 776 777 static int rfc3686_skcipher_setkey(struct crypto_skcipher *skcipher, 778 const u8 *key, unsigned int keylen) 779 { 780 u32 ctx1_iv_off; 781 int err; 782 783 /* 784 * RFC3686 specific: 785 * | CONTEXT1[255:128] = {NONCE, IV, COUNTER} 786 * | *key = {KEY, NONCE} 787 */ 788 ctx1_iv_off = 16 + CTR_RFC3686_NONCE_SIZE; 789 keylen -= CTR_RFC3686_NONCE_SIZE; 790 791 err = aes_check_keylen(keylen); 792 if (err) 793 return err; 794 795 return skcipher_setkey(skcipher, key, keylen, ctx1_iv_off); 796 } 797 798 static int ctr_skcipher_setkey(struct crypto_skcipher *skcipher, 799 const u8 *key, unsigned int keylen) 800 { 801 u32 ctx1_iv_off; 802 int err; 803 804 /* 805 * AES-CTR needs to load IV in CONTEXT1 reg 806 * at an offset of 128bits (16bytes) 807 * CONTEXT1[255:128] = IV 808 */ 809 ctx1_iv_off = 16; 810 811 err = aes_check_keylen(keylen); 812 if (err) 813 return err; 814 815 return skcipher_setkey(skcipher, key, keylen, ctx1_iv_off); 816 } 817 818 static int des_skcipher_setkey(struct crypto_skcipher *skcipher, 819 const u8 *key, unsigned int keylen) 820 { 821 return verify_skcipher_des_key(skcipher, key) ?: 822 skcipher_setkey(skcipher, key, keylen, 0); 823 } 824 825 static int des3_skcipher_setkey(struct crypto_skcipher *skcipher, 826 const u8 *key, unsigned int keylen) 827 { 828 return verify_skcipher_des3_key(skcipher, key) ?: 829 skcipher_setkey(skcipher, key, keylen, 0); 830 } 831 832 static int xts_skcipher_setkey(struct crypto_skcipher *skcipher, const u8 *key, 833 unsigned int keylen) 834 { 835 struct caam_ctx *ctx = crypto_skcipher_ctx_dma(skcipher); 836 struct device *jrdev = ctx->jrdev; 837 struct caam_drv_private *ctrlpriv = dev_get_drvdata(jrdev->parent); 838 u32 *desc; 839 int err; 840 841 err = xts_verify_key(skcipher, key, keylen); 842 if (err) { 843 dev_dbg(jrdev, "key size mismatch\n"); 844 return err; 845 } 846 847 if (keylen != 2 * AES_KEYSIZE_128 && keylen != 2 * AES_KEYSIZE_256) 848 ctx->xts_key_fallback = true; 849 850 if (ctrlpriv->era <= 8 || ctx->xts_key_fallback) { 851 err = crypto_skcipher_setkey(ctx->fallback, key, keylen); 852 if (err) 853 return err; 854 } 855 856 ctx->cdata.keylen = keylen; 857 ctx->cdata.key_virt = key; 858 ctx->cdata.key_inline = true; 859 860 /* xts_skcipher_encrypt shared descriptor */ 861 desc = ctx->sh_desc_enc; 862 cnstr_shdsc_xts_skcipher_encap(desc, &ctx->cdata); 863 dma_sync_single_for_device(jrdev, ctx->sh_desc_enc_dma, 864 desc_bytes(desc), ctx->dir); 865 866 /* xts_skcipher_decrypt shared descriptor */ 867 desc = ctx->sh_desc_dec; 868 cnstr_shdsc_xts_skcipher_decap(desc, &ctx->cdata); 869 dma_sync_single_for_device(jrdev, ctx->sh_desc_dec_dma, 870 desc_bytes(desc), ctx->dir); 871 872 return 0; 873 } 874 875 /* 876 * aead_edesc - s/w-extended aead descriptor 877 * @src_nents: number of segments in input s/w scatterlist 878 * @dst_nents: number of segments in output s/w scatterlist 879 * @mapped_src_nents: number of segments in input h/w link table 880 * @mapped_dst_nents: number of segments in output h/w link table 881 * @sec4_sg_bytes: length of dma mapped sec4_sg space 882 * @bklog: stored to determine if the request needs backlog 883 * @sec4_sg_dma: bus physical mapped address of h/w link table 884 * @sec4_sg: pointer to h/w link table 885 * @hw_desc: the h/w job descriptor followed by any referenced link tables 886 */ 887 struct aead_edesc { 888 int src_nents; 889 int dst_nents; 890 int mapped_src_nents; 891 int mapped_dst_nents; 892 int sec4_sg_bytes; 893 bool bklog; 894 dma_addr_t sec4_sg_dma; 895 struct sec4_sg_entry *sec4_sg; 896 u32 hw_desc[]; 897 }; 898 899 /* 900 * skcipher_edesc - s/w-extended skcipher descriptor 901 * @src_nents: number of segments in input s/w scatterlist 902 * @dst_nents: number of segments in output s/w scatterlist 903 * @mapped_src_nents: number of segments in input h/w link table 904 * @mapped_dst_nents: number of segments in output h/w link table 905 * @iv_dma: dma address of iv for checking continuity and link table 906 * @sec4_sg_bytes: length of dma mapped sec4_sg space 907 * @bklog: stored to determine if the request needs backlog 908 * @sec4_sg_dma: bus physical mapped address of h/w link table 909 * @sec4_sg: pointer to h/w link table 910 * @hw_desc: the h/w job descriptor followed by any referenced link tables 911 * and IV 912 */ 913 struct skcipher_edesc { 914 int src_nents; 915 int dst_nents; 916 int mapped_src_nents; 917 int mapped_dst_nents; 918 dma_addr_t iv_dma; 919 int sec4_sg_bytes; 920 bool bklog; 921 dma_addr_t sec4_sg_dma; 922 struct sec4_sg_entry *sec4_sg; 923 u32 hw_desc[]; 924 }; 925 926 static void caam_unmap(struct device *dev, struct scatterlist *src, 927 struct scatterlist *dst, int src_nents, 928 int dst_nents, 929 dma_addr_t iv_dma, int ivsize, dma_addr_t sec4_sg_dma, 930 int sec4_sg_bytes) 931 { 932 if (dst != src) { 933 if (src_nents) 934 dma_unmap_sg(dev, src, src_nents, DMA_TO_DEVICE); 935 if (dst_nents) 936 dma_unmap_sg(dev, dst, dst_nents, DMA_FROM_DEVICE); 937 } else { 938 dma_unmap_sg(dev, src, src_nents, DMA_BIDIRECTIONAL); 939 } 940 941 if (iv_dma) 942 dma_unmap_single(dev, iv_dma, ivsize, DMA_BIDIRECTIONAL); 943 if (sec4_sg_bytes) 944 dma_unmap_single(dev, sec4_sg_dma, sec4_sg_bytes, 945 DMA_TO_DEVICE); 946 } 947 948 static void aead_unmap(struct device *dev, 949 struct aead_edesc *edesc, 950 struct aead_request *req) 951 { 952 caam_unmap(dev, req->src, req->dst, 953 edesc->src_nents, edesc->dst_nents, 0, 0, 954 edesc->sec4_sg_dma, edesc->sec4_sg_bytes); 955 } 956 957 static void skcipher_unmap(struct device *dev, struct skcipher_edesc *edesc, 958 struct skcipher_request *req) 959 { 960 struct crypto_skcipher *skcipher = crypto_skcipher_reqtfm(req); 961 int ivsize = crypto_skcipher_ivsize(skcipher); 962 963 caam_unmap(dev, req->src, req->dst, 964 edesc->src_nents, edesc->dst_nents, 965 edesc->iv_dma, ivsize, 966 edesc->sec4_sg_dma, edesc->sec4_sg_bytes); 967 } 968 969 static void aead_crypt_done(struct device *jrdev, u32 *desc, u32 err, 970 void *context) 971 { 972 struct aead_request *req = context; 973 struct caam_aead_req_ctx *rctx = aead_request_ctx(req); 974 struct caam_drv_private_jr *jrp = dev_get_drvdata(jrdev); 975 struct aead_edesc *edesc; 976 int ecode = 0; 977 bool has_bklog; 978 979 dev_dbg(jrdev, "%s %d: err 0x%x\n", __func__, __LINE__, err); 980 981 edesc = rctx->edesc; 982 has_bklog = edesc->bklog; 983 984 if (err) 985 ecode = caam_jr_strstatus(jrdev, err); 986 987 aead_unmap(jrdev, edesc, req); 988 989 kfree(edesc); 990 991 /* 992 * If no backlog flag, the completion of the request is done 993 * by CAAM, not crypto engine. 994 */ 995 if (!has_bklog) 996 aead_request_complete(req, ecode); 997 else 998 crypto_finalize_aead_request(jrp->engine, req, ecode); 999 } 1000 1001 static void skcipher_crypt_done(struct device *jrdev, u32 *desc, u32 err, 1002 void *context) 1003 { 1004 struct skcipher_request *req = context; 1005 struct skcipher_edesc *edesc; 1006 struct caam_skcipher_req_ctx *rctx = skcipher_request_ctx(req); 1007 struct crypto_skcipher *skcipher = crypto_skcipher_reqtfm(req); 1008 struct caam_drv_private_jr *jrp = dev_get_drvdata(jrdev); 1009 int ivsize = crypto_skcipher_ivsize(skcipher); 1010 int ecode = 0; 1011 bool has_bklog; 1012 1013 dev_dbg(jrdev, "%s %d: err 0x%x\n", __func__, __LINE__, err); 1014 1015 edesc = rctx->edesc; 1016 has_bklog = edesc->bklog; 1017 if (err) 1018 ecode = caam_jr_strstatus(jrdev, err); 1019 1020 skcipher_unmap(jrdev, edesc, req); 1021 1022 /* 1023 * The crypto API expects us to set the IV (req->iv) to the last 1024 * ciphertext block (CBC mode) or last counter (CTR mode). 1025 * This is used e.g. by the CTS mode. 1026 */ 1027 if (ivsize && !ecode) { 1028 memcpy(req->iv, (u8 *)edesc->sec4_sg + edesc->sec4_sg_bytes, 1029 ivsize); 1030 1031 print_hex_dump_debug("dstiv @" __stringify(__LINE__)": ", 1032 DUMP_PREFIX_ADDRESS, 16, 4, req->iv, 1033 ivsize, 1); 1034 } 1035 1036 caam_dump_sg("dst @" __stringify(__LINE__)": ", 1037 DUMP_PREFIX_ADDRESS, 16, 4, req->dst, 1038 edesc->dst_nents > 1 ? 100 : req->cryptlen, 1); 1039 1040 kfree(edesc); 1041 1042 /* 1043 * If no backlog flag, the completion of the request is done 1044 * by CAAM, not crypto engine. 1045 */ 1046 if (!has_bklog) 1047 skcipher_request_complete(req, ecode); 1048 else 1049 crypto_finalize_skcipher_request(jrp->engine, req, ecode); 1050 } 1051 1052 /* 1053 * Fill in aead job descriptor 1054 */ 1055 static void init_aead_job(struct aead_request *req, 1056 struct aead_edesc *edesc, 1057 bool all_contig, bool encrypt) 1058 { 1059 struct crypto_aead *aead = crypto_aead_reqtfm(req); 1060 struct caam_ctx *ctx = crypto_aead_ctx_dma(aead); 1061 int authsize = ctx->authsize; 1062 u32 *desc = edesc->hw_desc; 1063 u32 out_options, in_options; 1064 dma_addr_t dst_dma, src_dma; 1065 int len, sec4_sg_index = 0; 1066 dma_addr_t ptr; 1067 u32 *sh_desc; 1068 1069 sh_desc = encrypt ? ctx->sh_desc_enc : ctx->sh_desc_dec; 1070 ptr = encrypt ? ctx->sh_desc_enc_dma : ctx->sh_desc_dec_dma; 1071 1072 len = desc_len(sh_desc); 1073 init_job_desc_shared(desc, ptr, len, HDR_SHARE_DEFER | HDR_REVERSE); 1074 1075 if (all_contig) { 1076 src_dma = edesc->mapped_src_nents ? sg_dma_address(req->src) : 1077 0; 1078 in_options = 0; 1079 } else { 1080 src_dma = edesc->sec4_sg_dma; 1081 sec4_sg_index += edesc->mapped_src_nents; 1082 in_options = LDST_SGF; 1083 } 1084 1085 append_seq_in_ptr(desc, src_dma, req->assoclen + req->cryptlen, 1086 in_options); 1087 1088 dst_dma = src_dma; 1089 out_options = in_options; 1090 1091 if (unlikely(req->src != req->dst)) { 1092 if (!edesc->mapped_dst_nents) { 1093 dst_dma = 0; 1094 out_options = 0; 1095 } else if (edesc->mapped_dst_nents == 1) { 1096 dst_dma = sg_dma_address(req->dst); 1097 out_options = 0; 1098 } else { 1099 dst_dma = edesc->sec4_sg_dma + 1100 sec4_sg_index * 1101 sizeof(struct sec4_sg_entry); 1102 out_options = LDST_SGF; 1103 } 1104 } 1105 1106 if (encrypt) 1107 append_seq_out_ptr(desc, dst_dma, 1108 req->assoclen + req->cryptlen + authsize, 1109 out_options); 1110 else 1111 append_seq_out_ptr(desc, dst_dma, 1112 req->assoclen + req->cryptlen - authsize, 1113 out_options); 1114 } 1115 1116 static void init_gcm_job(struct aead_request *req, 1117 struct aead_edesc *edesc, 1118 bool all_contig, bool encrypt) 1119 { 1120 struct crypto_aead *aead = crypto_aead_reqtfm(req); 1121 struct caam_ctx *ctx = crypto_aead_ctx_dma(aead); 1122 unsigned int ivsize = crypto_aead_ivsize(aead); 1123 u32 *desc = edesc->hw_desc; 1124 bool generic_gcm = (ivsize == GCM_AES_IV_SIZE); 1125 unsigned int last; 1126 1127 init_aead_job(req, edesc, all_contig, encrypt); 1128 append_math_add_imm_u32(desc, REG3, ZERO, IMM, req->assoclen); 1129 1130 /* BUG This should not be specific to generic GCM. */ 1131 last = 0; 1132 if (encrypt && generic_gcm && !(req->assoclen + req->cryptlen)) 1133 last = FIFOLD_TYPE_LAST1; 1134 1135 /* Read GCM IV */ 1136 append_cmd(desc, CMD_FIFO_LOAD | FIFOLD_CLASS_CLASS1 | IMMEDIATE | 1137 FIFOLD_TYPE_IV | FIFOLD_TYPE_FLUSH1 | GCM_AES_IV_SIZE | last); 1138 /* Append Salt */ 1139 if (!generic_gcm) 1140 append_data(desc, ctx->key + ctx->cdata.keylen, 4); 1141 /* Append IV */ 1142 append_data(desc, req->iv, ivsize); 1143 /* End of blank commands */ 1144 } 1145 1146 static void init_chachapoly_job(struct aead_request *req, 1147 struct aead_edesc *edesc, bool all_contig, 1148 bool encrypt) 1149 { 1150 struct crypto_aead *aead = crypto_aead_reqtfm(req); 1151 unsigned int ivsize = crypto_aead_ivsize(aead); 1152 unsigned int assoclen = req->assoclen; 1153 u32 *desc = edesc->hw_desc; 1154 u32 ctx_iv_off = 4; 1155 1156 init_aead_job(req, edesc, all_contig, encrypt); 1157 1158 if (ivsize != CHACHAPOLY_IV_SIZE) { 1159 /* IPsec specific: CONTEXT1[223:128] = {NONCE, IV} */ 1160 ctx_iv_off += 4; 1161 1162 /* 1163 * The associated data comes already with the IV but we need 1164 * to skip it when we authenticate or encrypt... 1165 */ 1166 assoclen -= ivsize; 1167 } 1168 1169 append_math_add_imm_u32(desc, REG3, ZERO, IMM, assoclen); 1170 1171 /* 1172 * For IPsec load the IV further in the same register. 1173 * For RFC7539 simply load the 12 bytes nonce in a single operation 1174 */ 1175 append_load_as_imm(desc, req->iv, ivsize, LDST_CLASS_1_CCB | 1176 LDST_SRCDST_BYTE_CONTEXT | 1177 ctx_iv_off << LDST_OFFSET_SHIFT); 1178 } 1179 1180 static void init_authenc_job(struct aead_request *req, 1181 struct aead_edesc *edesc, 1182 bool all_contig, bool encrypt) 1183 { 1184 struct crypto_aead *aead = crypto_aead_reqtfm(req); 1185 struct caam_aead_alg *alg = container_of(crypto_aead_alg(aead), 1186 struct caam_aead_alg, aead); 1187 unsigned int ivsize = crypto_aead_ivsize(aead); 1188 struct caam_ctx *ctx = crypto_aead_ctx_dma(aead); 1189 struct caam_drv_private *ctrlpriv = dev_get_drvdata(ctx->jrdev->parent); 1190 const bool ctr_mode = ((ctx->cdata.algtype & OP_ALG_AAI_MASK) == 1191 OP_ALG_AAI_CTR_MOD128); 1192 const bool is_rfc3686 = alg->caam.rfc3686; 1193 u32 *desc = edesc->hw_desc; 1194 u32 ivoffset = 0; 1195 1196 /* 1197 * AES-CTR needs to load IV in CONTEXT1 reg 1198 * at an offset of 128bits (16bytes) 1199 * CONTEXT1[255:128] = IV 1200 */ 1201 if (ctr_mode) 1202 ivoffset = 16; 1203 1204 /* 1205 * RFC3686 specific: 1206 * CONTEXT1[255:128] = {NONCE, IV, COUNTER} 1207 */ 1208 if (is_rfc3686) 1209 ivoffset = 16 + CTR_RFC3686_NONCE_SIZE; 1210 1211 init_aead_job(req, edesc, all_contig, encrypt); 1212 1213 /* 1214 * {REG3, DPOVRD} = assoclen, depending on whether MATH command supports 1215 * having DPOVRD as destination. 1216 */ 1217 if (ctrlpriv->era < 3) 1218 append_math_add_imm_u32(desc, REG3, ZERO, IMM, req->assoclen); 1219 else 1220 append_math_add_imm_u32(desc, DPOVRD, ZERO, IMM, req->assoclen); 1221 1222 if (ivsize && ((is_rfc3686 && encrypt) || !alg->caam.geniv)) 1223 append_load_as_imm(desc, req->iv, ivsize, 1224 LDST_CLASS_1_CCB | 1225 LDST_SRCDST_BYTE_CONTEXT | 1226 (ivoffset << LDST_OFFSET_SHIFT)); 1227 } 1228 1229 /* 1230 * Fill in skcipher job descriptor 1231 */ 1232 static void init_skcipher_job(struct skcipher_request *req, 1233 struct skcipher_edesc *edesc, 1234 const bool encrypt) 1235 { 1236 struct crypto_skcipher *skcipher = crypto_skcipher_reqtfm(req); 1237 struct caam_ctx *ctx = crypto_skcipher_ctx_dma(skcipher); 1238 struct device *jrdev = ctx->jrdev; 1239 int ivsize = crypto_skcipher_ivsize(skcipher); 1240 u32 *desc = edesc->hw_desc; 1241 u32 *sh_desc; 1242 u32 in_options = 0, out_options = 0; 1243 dma_addr_t src_dma, dst_dma, ptr; 1244 int len, sec4_sg_index = 0; 1245 1246 print_hex_dump_debug("presciv@"__stringify(__LINE__)": ", 1247 DUMP_PREFIX_ADDRESS, 16, 4, req->iv, ivsize, 1); 1248 dev_dbg(jrdev, "asked=%d, cryptlen%d\n", 1249 (int)edesc->src_nents > 1 ? 100 : req->cryptlen, req->cryptlen); 1250 1251 caam_dump_sg("src @" __stringify(__LINE__)": ", 1252 DUMP_PREFIX_ADDRESS, 16, 4, req->src, 1253 edesc->src_nents > 1 ? 100 : req->cryptlen, 1); 1254 1255 sh_desc = encrypt ? ctx->sh_desc_enc : ctx->sh_desc_dec; 1256 ptr = encrypt ? ctx->sh_desc_enc_dma : ctx->sh_desc_dec_dma; 1257 1258 len = desc_len(sh_desc); 1259 init_job_desc_shared(desc, ptr, len, HDR_SHARE_DEFER | HDR_REVERSE); 1260 1261 if (ivsize || edesc->mapped_src_nents > 1) { 1262 src_dma = edesc->sec4_sg_dma; 1263 sec4_sg_index = edesc->mapped_src_nents + !!ivsize; 1264 in_options = LDST_SGF; 1265 } else { 1266 src_dma = sg_dma_address(req->src); 1267 } 1268 1269 append_seq_in_ptr(desc, src_dma, req->cryptlen + ivsize, in_options); 1270 1271 if (likely(req->src == req->dst)) { 1272 dst_dma = src_dma + !!ivsize * sizeof(struct sec4_sg_entry); 1273 out_options = in_options; 1274 } else if (!ivsize && edesc->mapped_dst_nents == 1) { 1275 dst_dma = sg_dma_address(req->dst); 1276 } else { 1277 dst_dma = edesc->sec4_sg_dma + sec4_sg_index * 1278 sizeof(struct sec4_sg_entry); 1279 out_options = LDST_SGF; 1280 } 1281 1282 append_seq_out_ptr(desc, dst_dma, req->cryptlen + ivsize, out_options); 1283 } 1284 1285 /* 1286 * allocate and map the aead extended descriptor 1287 */ 1288 static struct aead_edesc *aead_edesc_alloc(struct aead_request *req, 1289 int desc_bytes, bool *all_contig_ptr, 1290 bool encrypt) 1291 { 1292 struct crypto_aead *aead = crypto_aead_reqtfm(req); 1293 struct caam_ctx *ctx = crypto_aead_ctx_dma(aead); 1294 struct device *jrdev = ctx->jrdev; 1295 struct caam_aead_req_ctx *rctx = aead_request_ctx(req); 1296 gfp_t flags = (req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP) ? 1297 GFP_KERNEL : GFP_ATOMIC; 1298 int src_nents, mapped_src_nents, dst_nents = 0, mapped_dst_nents = 0; 1299 int src_len, dst_len = 0; 1300 struct aead_edesc *edesc; 1301 int sec4_sg_index, sec4_sg_len, sec4_sg_bytes; 1302 unsigned int authsize = ctx->authsize; 1303 1304 if (unlikely(req->dst != req->src)) { 1305 src_len = req->assoclen + req->cryptlen; 1306 dst_len = src_len + (encrypt ? authsize : (-authsize)); 1307 1308 src_nents = sg_nents_for_len(req->src, src_len); 1309 if (unlikely(src_nents < 0)) { 1310 dev_err(jrdev, "Insufficient bytes (%d) in src S/G\n", 1311 src_len); 1312 return ERR_PTR(src_nents); 1313 } 1314 1315 dst_nents = sg_nents_for_len(req->dst, dst_len); 1316 if (unlikely(dst_nents < 0)) { 1317 dev_err(jrdev, "Insufficient bytes (%d) in dst S/G\n", 1318 dst_len); 1319 return ERR_PTR(dst_nents); 1320 } 1321 } else { 1322 src_len = req->assoclen + req->cryptlen + 1323 (encrypt ? authsize : 0); 1324 1325 src_nents = sg_nents_for_len(req->src, src_len); 1326 if (unlikely(src_nents < 0)) { 1327 dev_err(jrdev, "Insufficient bytes (%d) in src S/G\n", 1328 src_len); 1329 return ERR_PTR(src_nents); 1330 } 1331 } 1332 1333 if (likely(req->src == req->dst)) { 1334 mapped_src_nents = dma_map_sg(jrdev, req->src, src_nents, 1335 DMA_BIDIRECTIONAL); 1336 if (unlikely(!mapped_src_nents)) { 1337 dev_err(jrdev, "unable to map source\n"); 1338 return ERR_PTR(-ENOMEM); 1339 } 1340 } else { 1341 /* Cover also the case of null (zero length) input data */ 1342 if (src_nents) { 1343 mapped_src_nents = dma_map_sg(jrdev, req->src, 1344 src_nents, DMA_TO_DEVICE); 1345 if (unlikely(!mapped_src_nents)) { 1346 dev_err(jrdev, "unable to map source\n"); 1347 return ERR_PTR(-ENOMEM); 1348 } 1349 } else { 1350 mapped_src_nents = 0; 1351 } 1352 1353 /* Cover also the case of null (zero length) output data */ 1354 if (dst_nents) { 1355 mapped_dst_nents = dma_map_sg(jrdev, req->dst, 1356 dst_nents, 1357 DMA_FROM_DEVICE); 1358 if (unlikely(!mapped_dst_nents)) { 1359 dev_err(jrdev, "unable to map destination\n"); 1360 dma_unmap_sg(jrdev, req->src, src_nents, 1361 DMA_TO_DEVICE); 1362 return ERR_PTR(-ENOMEM); 1363 } 1364 } else { 1365 mapped_dst_nents = 0; 1366 } 1367 } 1368 1369 /* 1370 * HW reads 4 S/G entries at a time; make sure the reads don't go beyond 1371 * the end of the table by allocating more S/G entries. 1372 */ 1373 sec4_sg_len = mapped_src_nents > 1 ? mapped_src_nents : 0; 1374 if (mapped_dst_nents > 1) 1375 sec4_sg_len += pad_sg_nents(mapped_dst_nents); 1376 else 1377 sec4_sg_len = pad_sg_nents(sec4_sg_len); 1378 1379 sec4_sg_bytes = sec4_sg_len * sizeof(struct sec4_sg_entry); 1380 1381 /* allocate space for base edesc and hw desc commands, link tables */ 1382 edesc = kzalloc(sizeof(*edesc) + desc_bytes + sec4_sg_bytes, 1383 GFP_DMA | flags); 1384 if (!edesc) { 1385 caam_unmap(jrdev, req->src, req->dst, src_nents, dst_nents, 0, 1386 0, 0, 0); 1387 return ERR_PTR(-ENOMEM); 1388 } 1389 1390 edesc->src_nents = src_nents; 1391 edesc->dst_nents = dst_nents; 1392 edesc->mapped_src_nents = mapped_src_nents; 1393 edesc->mapped_dst_nents = mapped_dst_nents; 1394 edesc->sec4_sg = (void *)edesc + sizeof(struct aead_edesc) + 1395 desc_bytes; 1396 1397 rctx->edesc = edesc; 1398 1399 *all_contig_ptr = !(mapped_src_nents > 1); 1400 1401 sec4_sg_index = 0; 1402 if (mapped_src_nents > 1) { 1403 sg_to_sec4_sg_last(req->src, src_len, 1404 edesc->sec4_sg + sec4_sg_index, 0); 1405 sec4_sg_index += mapped_src_nents; 1406 } 1407 if (mapped_dst_nents > 1) { 1408 sg_to_sec4_sg_last(req->dst, dst_len, 1409 edesc->sec4_sg + sec4_sg_index, 0); 1410 } 1411 1412 if (!sec4_sg_bytes) 1413 return edesc; 1414 1415 edesc->sec4_sg_dma = dma_map_single(jrdev, edesc->sec4_sg, 1416 sec4_sg_bytes, DMA_TO_DEVICE); 1417 if (dma_mapping_error(jrdev, edesc->sec4_sg_dma)) { 1418 dev_err(jrdev, "unable to map S/G table\n"); 1419 aead_unmap(jrdev, edesc, req); 1420 kfree(edesc); 1421 return ERR_PTR(-ENOMEM); 1422 } 1423 1424 edesc->sec4_sg_bytes = sec4_sg_bytes; 1425 1426 return edesc; 1427 } 1428 1429 static int aead_enqueue_req(struct device *jrdev, struct aead_request *req) 1430 { 1431 struct caam_drv_private_jr *jrpriv = dev_get_drvdata(jrdev); 1432 struct caam_aead_req_ctx *rctx = aead_request_ctx(req); 1433 struct aead_edesc *edesc = rctx->edesc; 1434 u32 *desc = edesc->hw_desc; 1435 int ret; 1436 1437 /* 1438 * Only the backlog request are sent to crypto-engine since the others 1439 * can be handled by CAAM, if free, especially since JR has up to 1024 1440 * entries (more than the 10 entries from crypto-engine). 1441 */ 1442 if (req->base.flags & CRYPTO_TFM_REQ_MAY_BACKLOG) 1443 ret = crypto_transfer_aead_request_to_engine(jrpriv->engine, 1444 req); 1445 else 1446 ret = caam_jr_enqueue(jrdev, desc, aead_crypt_done, req); 1447 1448 if ((ret != -EINPROGRESS) && (ret != -EBUSY)) { 1449 aead_unmap(jrdev, edesc, req); 1450 kfree(rctx->edesc); 1451 } 1452 1453 return ret; 1454 } 1455 1456 static inline int chachapoly_crypt(struct aead_request *req, bool encrypt) 1457 { 1458 struct aead_edesc *edesc; 1459 struct crypto_aead *aead = crypto_aead_reqtfm(req); 1460 struct caam_ctx *ctx = crypto_aead_ctx_dma(aead); 1461 struct device *jrdev = ctx->jrdev; 1462 bool all_contig; 1463 u32 *desc; 1464 1465 edesc = aead_edesc_alloc(req, CHACHAPOLY_DESC_JOB_IO_LEN, &all_contig, 1466 encrypt); 1467 if (IS_ERR(edesc)) 1468 return PTR_ERR(edesc); 1469 1470 desc = edesc->hw_desc; 1471 1472 init_chachapoly_job(req, edesc, all_contig, encrypt); 1473 print_hex_dump_debug("chachapoly jobdesc@" __stringify(__LINE__)": ", 1474 DUMP_PREFIX_ADDRESS, 16, 4, desc, desc_bytes(desc), 1475 1); 1476 1477 return aead_enqueue_req(jrdev, req); 1478 } 1479 1480 static int chachapoly_encrypt(struct aead_request *req) 1481 { 1482 return chachapoly_crypt(req, true); 1483 } 1484 1485 static int chachapoly_decrypt(struct aead_request *req) 1486 { 1487 return chachapoly_crypt(req, false); 1488 } 1489 1490 static inline int aead_crypt(struct aead_request *req, bool encrypt) 1491 { 1492 struct aead_edesc *edesc; 1493 struct crypto_aead *aead = crypto_aead_reqtfm(req); 1494 struct caam_ctx *ctx = crypto_aead_ctx_dma(aead); 1495 struct device *jrdev = ctx->jrdev; 1496 bool all_contig; 1497 1498 /* allocate extended descriptor */ 1499 edesc = aead_edesc_alloc(req, AUTHENC_DESC_JOB_IO_LEN, 1500 &all_contig, encrypt); 1501 if (IS_ERR(edesc)) 1502 return PTR_ERR(edesc); 1503 1504 /* Create and submit job descriptor */ 1505 init_authenc_job(req, edesc, all_contig, encrypt); 1506 1507 print_hex_dump_debug("aead jobdesc@"__stringify(__LINE__)": ", 1508 DUMP_PREFIX_ADDRESS, 16, 4, edesc->hw_desc, 1509 desc_bytes(edesc->hw_desc), 1); 1510 1511 return aead_enqueue_req(jrdev, req); 1512 } 1513 1514 static int aead_encrypt(struct aead_request *req) 1515 { 1516 return aead_crypt(req, true); 1517 } 1518 1519 static int aead_decrypt(struct aead_request *req) 1520 { 1521 return aead_crypt(req, false); 1522 } 1523 1524 static int aead_do_one_req(struct crypto_engine *engine, void *areq) 1525 { 1526 struct aead_request *req = aead_request_cast(areq); 1527 struct caam_ctx *ctx = crypto_aead_ctx_dma(crypto_aead_reqtfm(req)); 1528 struct caam_aead_req_ctx *rctx = aead_request_ctx(req); 1529 u32 *desc = rctx->edesc->hw_desc; 1530 int ret; 1531 1532 rctx->edesc->bklog = true; 1533 1534 ret = caam_jr_enqueue(ctx->jrdev, desc, aead_crypt_done, req); 1535 1536 if (ret == -ENOSPC && engine->retry_support) 1537 return ret; 1538 1539 if (ret != -EINPROGRESS) { 1540 aead_unmap(ctx->jrdev, rctx->edesc, req); 1541 kfree(rctx->edesc); 1542 } else { 1543 ret = 0; 1544 } 1545 1546 return ret; 1547 } 1548 1549 static inline int gcm_crypt(struct aead_request *req, bool encrypt) 1550 { 1551 struct aead_edesc *edesc; 1552 struct crypto_aead *aead = crypto_aead_reqtfm(req); 1553 struct caam_ctx *ctx = crypto_aead_ctx_dma(aead); 1554 struct device *jrdev = ctx->jrdev; 1555 bool all_contig; 1556 1557 /* allocate extended descriptor */ 1558 edesc = aead_edesc_alloc(req, GCM_DESC_JOB_IO_LEN, &all_contig, 1559 encrypt); 1560 if (IS_ERR(edesc)) 1561 return PTR_ERR(edesc); 1562 1563 /* Create and submit job descriptor */ 1564 init_gcm_job(req, edesc, all_contig, encrypt); 1565 1566 print_hex_dump_debug("aead jobdesc@"__stringify(__LINE__)": ", 1567 DUMP_PREFIX_ADDRESS, 16, 4, edesc->hw_desc, 1568 desc_bytes(edesc->hw_desc), 1); 1569 1570 return aead_enqueue_req(jrdev, req); 1571 } 1572 1573 static int gcm_encrypt(struct aead_request *req) 1574 { 1575 return gcm_crypt(req, true); 1576 } 1577 1578 static int gcm_decrypt(struct aead_request *req) 1579 { 1580 return gcm_crypt(req, false); 1581 } 1582 1583 static int ipsec_gcm_encrypt(struct aead_request *req) 1584 { 1585 return crypto_ipsec_check_assoclen(req->assoclen) ? : gcm_encrypt(req); 1586 } 1587 1588 static int ipsec_gcm_decrypt(struct aead_request *req) 1589 { 1590 return crypto_ipsec_check_assoclen(req->assoclen) ? : gcm_decrypt(req); 1591 } 1592 1593 /* 1594 * allocate and map the skcipher extended descriptor for skcipher 1595 */ 1596 static struct skcipher_edesc *skcipher_edesc_alloc(struct skcipher_request *req, 1597 int desc_bytes) 1598 { 1599 struct crypto_skcipher *skcipher = crypto_skcipher_reqtfm(req); 1600 struct caam_ctx *ctx = crypto_skcipher_ctx_dma(skcipher); 1601 struct caam_skcipher_req_ctx *rctx = skcipher_request_ctx(req); 1602 struct device *jrdev = ctx->jrdev; 1603 gfp_t flags = (req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP) ? 1604 GFP_KERNEL : GFP_ATOMIC; 1605 int src_nents, mapped_src_nents, dst_nents = 0, mapped_dst_nents = 0; 1606 struct skcipher_edesc *edesc; 1607 dma_addr_t iv_dma = 0; 1608 u8 *iv; 1609 int ivsize = crypto_skcipher_ivsize(skcipher); 1610 int dst_sg_idx, sec4_sg_ents, sec4_sg_bytes; 1611 1612 src_nents = sg_nents_for_len(req->src, req->cryptlen); 1613 if (unlikely(src_nents < 0)) { 1614 dev_err(jrdev, "Insufficient bytes (%d) in src S/G\n", 1615 req->cryptlen); 1616 return ERR_PTR(src_nents); 1617 } 1618 1619 if (req->dst != req->src) { 1620 dst_nents = sg_nents_for_len(req->dst, req->cryptlen); 1621 if (unlikely(dst_nents < 0)) { 1622 dev_err(jrdev, "Insufficient bytes (%d) in dst S/G\n", 1623 req->cryptlen); 1624 return ERR_PTR(dst_nents); 1625 } 1626 } 1627 1628 if (likely(req->src == req->dst)) { 1629 mapped_src_nents = dma_map_sg(jrdev, req->src, src_nents, 1630 DMA_BIDIRECTIONAL); 1631 if (unlikely(!mapped_src_nents)) { 1632 dev_err(jrdev, "unable to map source\n"); 1633 return ERR_PTR(-ENOMEM); 1634 } 1635 } else { 1636 mapped_src_nents = dma_map_sg(jrdev, req->src, src_nents, 1637 DMA_TO_DEVICE); 1638 if (unlikely(!mapped_src_nents)) { 1639 dev_err(jrdev, "unable to map source\n"); 1640 return ERR_PTR(-ENOMEM); 1641 } 1642 mapped_dst_nents = dma_map_sg(jrdev, req->dst, dst_nents, 1643 DMA_FROM_DEVICE); 1644 if (unlikely(!mapped_dst_nents)) { 1645 dev_err(jrdev, "unable to map destination\n"); 1646 dma_unmap_sg(jrdev, req->src, src_nents, DMA_TO_DEVICE); 1647 return ERR_PTR(-ENOMEM); 1648 } 1649 } 1650 1651 if (!ivsize && mapped_src_nents == 1) 1652 sec4_sg_ents = 0; // no need for an input hw s/g table 1653 else 1654 sec4_sg_ents = mapped_src_nents + !!ivsize; 1655 dst_sg_idx = sec4_sg_ents; 1656 1657 /* 1658 * Input, output HW S/G tables: [IV, src][dst, IV] 1659 * IV entries point to the same buffer 1660 * If src == dst, S/G entries are reused (S/G tables overlap) 1661 * 1662 * HW reads 4 S/G entries at a time; make sure the reads don't go beyond 1663 * the end of the table by allocating more S/G entries. Logic: 1664 * if (output S/G) 1665 * pad output S/G, if needed 1666 * else if (input S/G) ... 1667 * pad input S/G, if needed 1668 */ 1669 if (ivsize || mapped_dst_nents > 1) { 1670 if (req->src == req->dst) 1671 sec4_sg_ents = !!ivsize + pad_sg_nents(sec4_sg_ents); 1672 else 1673 sec4_sg_ents += pad_sg_nents(mapped_dst_nents + 1674 !!ivsize); 1675 } else { 1676 sec4_sg_ents = pad_sg_nents(sec4_sg_ents); 1677 } 1678 1679 sec4_sg_bytes = sec4_sg_ents * sizeof(struct sec4_sg_entry); 1680 1681 /* 1682 * allocate space for base edesc and hw desc commands, link tables, IV 1683 */ 1684 edesc = kzalloc(sizeof(*edesc) + desc_bytes + sec4_sg_bytes + ivsize, 1685 GFP_DMA | flags); 1686 if (!edesc) { 1687 dev_err(jrdev, "could not allocate extended descriptor\n"); 1688 caam_unmap(jrdev, req->src, req->dst, src_nents, dst_nents, 0, 1689 0, 0, 0); 1690 return ERR_PTR(-ENOMEM); 1691 } 1692 1693 edesc->src_nents = src_nents; 1694 edesc->dst_nents = dst_nents; 1695 edesc->mapped_src_nents = mapped_src_nents; 1696 edesc->mapped_dst_nents = mapped_dst_nents; 1697 edesc->sec4_sg_bytes = sec4_sg_bytes; 1698 edesc->sec4_sg = (struct sec4_sg_entry *)((u8 *)edesc->hw_desc + 1699 desc_bytes); 1700 rctx->edesc = edesc; 1701 1702 /* Make sure IV is located in a DMAable area */ 1703 if (ivsize) { 1704 iv = (u8 *)edesc->sec4_sg + sec4_sg_bytes; 1705 memcpy(iv, req->iv, ivsize); 1706 1707 iv_dma = dma_map_single(jrdev, iv, ivsize, DMA_BIDIRECTIONAL); 1708 if (dma_mapping_error(jrdev, iv_dma)) { 1709 dev_err(jrdev, "unable to map IV\n"); 1710 caam_unmap(jrdev, req->src, req->dst, src_nents, 1711 dst_nents, 0, 0, 0, 0); 1712 kfree(edesc); 1713 return ERR_PTR(-ENOMEM); 1714 } 1715 1716 dma_to_sec4_sg_one(edesc->sec4_sg, iv_dma, ivsize, 0); 1717 } 1718 if (dst_sg_idx) 1719 sg_to_sec4_sg(req->src, req->cryptlen, edesc->sec4_sg + 1720 !!ivsize, 0); 1721 1722 if (req->src != req->dst && (ivsize || mapped_dst_nents > 1)) 1723 sg_to_sec4_sg(req->dst, req->cryptlen, edesc->sec4_sg + 1724 dst_sg_idx, 0); 1725 1726 if (ivsize) 1727 dma_to_sec4_sg_one(edesc->sec4_sg + dst_sg_idx + 1728 mapped_dst_nents, iv_dma, ivsize, 0); 1729 1730 if (ivsize || mapped_dst_nents > 1) 1731 sg_to_sec4_set_last(edesc->sec4_sg + dst_sg_idx + 1732 mapped_dst_nents - 1 + !!ivsize); 1733 1734 if (sec4_sg_bytes) { 1735 edesc->sec4_sg_dma = dma_map_single(jrdev, edesc->sec4_sg, 1736 sec4_sg_bytes, 1737 DMA_TO_DEVICE); 1738 if (dma_mapping_error(jrdev, edesc->sec4_sg_dma)) { 1739 dev_err(jrdev, "unable to map S/G table\n"); 1740 caam_unmap(jrdev, req->src, req->dst, src_nents, 1741 dst_nents, iv_dma, ivsize, 0, 0); 1742 kfree(edesc); 1743 return ERR_PTR(-ENOMEM); 1744 } 1745 } 1746 1747 edesc->iv_dma = iv_dma; 1748 1749 print_hex_dump_debug("skcipher sec4_sg@" __stringify(__LINE__)": ", 1750 DUMP_PREFIX_ADDRESS, 16, 4, edesc->sec4_sg, 1751 sec4_sg_bytes, 1); 1752 1753 return edesc; 1754 } 1755 1756 static int skcipher_do_one_req(struct crypto_engine *engine, void *areq) 1757 { 1758 struct skcipher_request *req = skcipher_request_cast(areq); 1759 struct caam_ctx *ctx = crypto_skcipher_ctx_dma(crypto_skcipher_reqtfm(req)); 1760 struct caam_skcipher_req_ctx *rctx = skcipher_request_ctx(req); 1761 u32 *desc = rctx->edesc->hw_desc; 1762 int ret; 1763 1764 rctx->edesc->bklog = true; 1765 1766 ret = caam_jr_enqueue(ctx->jrdev, desc, skcipher_crypt_done, req); 1767 1768 if (ret == -ENOSPC && engine->retry_support) 1769 return ret; 1770 1771 if (ret != -EINPROGRESS) { 1772 skcipher_unmap(ctx->jrdev, rctx->edesc, req); 1773 kfree(rctx->edesc); 1774 } else { 1775 ret = 0; 1776 } 1777 1778 return ret; 1779 } 1780 1781 static inline bool xts_skcipher_ivsize(struct skcipher_request *req) 1782 { 1783 struct crypto_skcipher *skcipher = crypto_skcipher_reqtfm(req); 1784 unsigned int ivsize = crypto_skcipher_ivsize(skcipher); 1785 1786 return !!get_unaligned((u64 *)(req->iv + (ivsize / 2))); 1787 } 1788 1789 static inline int skcipher_crypt(struct skcipher_request *req, bool encrypt) 1790 { 1791 struct skcipher_edesc *edesc; 1792 struct crypto_skcipher *skcipher = crypto_skcipher_reqtfm(req); 1793 struct caam_ctx *ctx = crypto_skcipher_ctx_dma(skcipher); 1794 struct device *jrdev = ctx->jrdev; 1795 struct caam_drv_private_jr *jrpriv = dev_get_drvdata(jrdev); 1796 struct caam_drv_private *ctrlpriv = dev_get_drvdata(jrdev->parent); 1797 u32 *desc; 1798 int ret = 0; 1799 1800 /* 1801 * XTS is expected to return an error even for input length = 0 1802 * Note that the case input length < block size will be caught during 1803 * HW offloading and return an error. 1804 */ 1805 if (!req->cryptlen && !ctx->fallback) 1806 return 0; 1807 1808 if (ctx->fallback && ((ctrlpriv->era <= 8 && xts_skcipher_ivsize(req)) || 1809 ctx->xts_key_fallback)) { 1810 struct caam_skcipher_req_ctx *rctx = skcipher_request_ctx(req); 1811 1812 skcipher_request_set_tfm(&rctx->fallback_req, ctx->fallback); 1813 skcipher_request_set_callback(&rctx->fallback_req, 1814 req->base.flags, 1815 req->base.complete, 1816 req->base.data); 1817 skcipher_request_set_crypt(&rctx->fallback_req, req->src, 1818 req->dst, req->cryptlen, req->iv); 1819 1820 return encrypt ? crypto_skcipher_encrypt(&rctx->fallback_req) : 1821 crypto_skcipher_decrypt(&rctx->fallback_req); 1822 } 1823 1824 /* allocate extended descriptor */ 1825 edesc = skcipher_edesc_alloc(req, DESC_JOB_IO_LEN * CAAM_CMD_SZ); 1826 if (IS_ERR(edesc)) 1827 return PTR_ERR(edesc); 1828 1829 /* Create and submit job descriptor*/ 1830 init_skcipher_job(req, edesc, encrypt); 1831 1832 print_hex_dump_debug("skcipher jobdesc@" __stringify(__LINE__)": ", 1833 DUMP_PREFIX_ADDRESS, 16, 4, edesc->hw_desc, 1834 desc_bytes(edesc->hw_desc), 1); 1835 1836 desc = edesc->hw_desc; 1837 /* 1838 * Only the backlog request are sent to crypto-engine since the others 1839 * can be handled by CAAM, if free, especially since JR has up to 1024 1840 * entries (more than the 10 entries from crypto-engine). 1841 */ 1842 if (req->base.flags & CRYPTO_TFM_REQ_MAY_BACKLOG) 1843 ret = crypto_transfer_skcipher_request_to_engine(jrpriv->engine, 1844 req); 1845 else 1846 ret = caam_jr_enqueue(jrdev, desc, skcipher_crypt_done, req); 1847 1848 if ((ret != -EINPROGRESS) && (ret != -EBUSY)) { 1849 skcipher_unmap(jrdev, edesc, req); 1850 kfree(edesc); 1851 } 1852 1853 return ret; 1854 } 1855 1856 static int skcipher_encrypt(struct skcipher_request *req) 1857 { 1858 return skcipher_crypt(req, true); 1859 } 1860 1861 static int skcipher_decrypt(struct skcipher_request *req) 1862 { 1863 return skcipher_crypt(req, false); 1864 } 1865 1866 static struct caam_skcipher_alg driver_algs[] = { 1867 { 1868 .skcipher = { 1869 .base = { 1870 .cra_name = "cbc(aes)", 1871 .cra_driver_name = "cbc-aes-caam", 1872 .cra_blocksize = AES_BLOCK_SIZE, 1873 }, 1874 .setkey = aes_skcipher_setkey, 1875 .encrypt = skcipher_encrypt, 1876 .decrypt = skcipher_decrypt, 1877 .min_keysize = AES_MIN_KEY_SIZE, 1878 .max_keysize = AES_MAX_KEY_SIZE, 1879 .ivsize = AES_BLOCK_SIZE, 1880 }, 1881 .caam.class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CBC, 1882 }, 1883 { 1884 .skcipher = { 1885 .base = { 1886 .cra_name = "cbc(des3_ede)", 1887 .cra_driver_name = "cbc-3des-caam", 1888 .cra_blocksize = DES3_EDE_BLOCK_SIZE, 1889 }, 1890 .setkey = des3_skcipher_setkey, 1891 .encrypt = skcipher_encrypt, 1892 .decrypt = skcipher_decrypt, 1893 .min_keysize = DES3_EDE_KEY_SIZE, 1894 .max_keysize = DES3_EDE_KEY_SIZE, 1895 .ivsize = DES3_EDE_BLOCK_SIZE, 1896 }, 1897 .caam.class1_alg_type = OP_ALG_ALGSEL_3DES | OP_ALG_AAI_CBC, 1898 }, 1899 { 1900 .skcipher = { 1901 .base = { 1902 .cra_name = "cbc(des)", 1903 .cra_driver_name = "cbc-des-caam", 1904 .cra_blocksize = DES_BLOCK_SIZE, 1905 }, 1906 .setkey = des_skcipher_setkey, 1907 .encrypt = skcipher_encrypt, 1908 .decrypt = skcipher_decrypt, 1909 .min_keysize = DES_KEY_SIZE, 1910 .max_keysize = DES_KEY_SIZE, 1911 .ivsize = DES_BLOCK_SIZE, 1912 }, 1913 .caam.class1_alg_type = OP_ALG_ALGSEL_DES | OP_ALG_AAI_CBC, 1914 }, 1915 { 1916 .skcipher = { 1917 .base = { 1918 .cra_name = "ctr(aes)", 1919 .cra_driver_name = "ctr-aes-caam", 1920 .cra_blocksize = 1, 1921 }, 1922 .setkey = ctr_skcipher_setkey, 1923 .encrypt = skcipher_encrypt, 1924 .decrypt = skcipher_decrypt, 1925 .min_keysize = AES_MIN_KEY_SIZE, 1926 .max_keysize = AES_MAX_KEY_SIZE, 1927 .ivsize = AES_BLOCK_SIZE, 1928 .chunksize = AES_BLOCK_SIZE, 1929 }, 1930 .caam.class1_alg_type = OP_ALG_ALGSEL_AES | 1931 OP_ALG_AAI_CTR_MOD128, 1932 }, 1933 { 1934 .skcipher = { 1935 .base = { 1936 .cra_name = "rfc3686(ctr(aes))", 1937 .cra_driver_name = "rfc3686-ctr-aes-caam", 1938 .cra_blocksize = 1, 1939 }, 1940 .setkey = rfc3686_skcipher_setkey, 1941 .encrypt = skcipher_encrypt, 1942 .decrypt = skcipher_decrypt, 1943 .min_keysize = AES_MIN_KEY_SIZE + 1944 CTR_RFC3686_NONCE_SIZE, 1945 .max_keysize = AES_MAX_KEY_SIZE + 1946 CTR_RFC3686_NONCE_SIZE, 1947 .ivsize = CTR_RFC3686_IV_SIZE, 1948 .chunksize = AES_BLOCK_SIZE, 1949 }, 1950 .caam = { 1951 .class1_alg_type = OP_ALG_ALGSEL_AES | 1952 OP_ALG_AAI_CTR_MOD128, 1953 .rfc3686 = true, 1954 }, 1955 }, 1956 { 1957 .skcipher = { 1958 .base = { 1959 .cra_name = "xts(aes)", 1960 .cra_driver_name = "xts-aes-caam", 1961 .cra_flags = CRYPTO_ALG_NEED_FALLBACK, 1962 .cra_blocksize = AES_BLOCK_SIZE, 1963 }, 1964 .setkey = xts_skcipher_setkey, 1965 .encrypt = skcipher_encrypt, 1966 .decrypt = skcipher_decrypt, 1967 .min_keysize = 2 * AES_MIN_KEY_SIZE, 1968 .max_keysize = 2 * AES_MAX_KEY_SIZE, 1969 .ivsize = AES_BLOCK_SIZE, 1970 }, 1971 .caam.class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_XTS, 1972 }, 1973 { 1974 .skcipher = { 1975 .base = { 1976 .cra_name = "ecb(des)", 1977 .cra_driver_name = "ecb-des-caam", 1978 .cra_blocksize = DES_BLOCK_SIZE, 1979 }, 1980 .setkey = des_skcipher_setkey, 1981 .encrypt = skcipher_encrypt, 1982 .decrypt = skcipher_decrypt, 1983 .min_keysize = DES_KEY_SIZE, 1984 .max_keysize = DES_KEY_SIZE, 1985 }, 1986 .caam.class1_alg_type = OP_ALG_ALGSEL_DES | OP_ALG_AAI_ECB, 1987 }, 1988 { 1989 .skcipher = { 1990 .base = { 1991 .cra_name = "ecb(aes)", 1992 .cra_driver_name = "ecb-aes-caam", 1993 .cra_blocksize = AES_BLOCK_SIZE, 1994 }, 1995 .setkey = aes_skcipher_setkey, 1996 .encrypt = skcipher_encrypt, 1997 .decrypt = skcipher_decrypt, 1998 .min_keysize = AES_MIN_KEY_SIZE, 1999 .max_keysize = AES_MAX_KEY_SIZE, 2000 }, 2001 .caam.class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_ECB, 2002 }, 2003 { 2004 .skcipher = { 2005 .base = { 2006 .cra_name = "ecb(des3_ede)", 2007 .cra_driver_name = "ecb-des3-caam", 2008 .cra_blocksize = DES3_EDE_BLOCK_SIZE, 2009 }, 2010 .setkey = des3_skcipher_setkey, 2011 .encrypt = skcipher_encrypt, 2012 .decrypt = skcipher_decrypt, 2013 .min_keysize = DES3_EDE_KEY_SIZE, 2014 .max_keysize = DES3_EDE_KEY_SIZE, 2015 }, 2016 .caam.class1_alg_type = OP_ALG_ALGSEL_3DES | OP_ALG_AAI_ECB, 2017 }, 2018 }; 2019 2020 static struct caam_aead_alg driver_aeads[] = { 2021 { 2022 .aead = { 2023 .base = { 2024 .cra_name = "rfc4106(gcm(aes))", 2025 .cra_driver_name = "rfc4106-gcm-aes-caam", 2026 .cra_blocksize = 1, 2027 }, 2028 .setkey = rfc4106_setkey, 2029 .setauthsize = rfc4106_setauthsize, 2030 .encrypt = ipsec_gcm_encrypt, 2031 .decrypt = ipsec_gcm_decrypt, 2032 .ivsize = GCM_RFC4106_IV_SIZE, 2033 .maxauthsize = AES_BLOCK_SIZE, 2034 }, 2035 .caam = { 2036 .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_GCM, 2037 .nodkp = true, 2038 }, 2039 }, 2040 { 2041 .aead = { 2042 .base = { 2043 .cra_name = "rfc4543(gcm(aes))", 2044 .cra_driver_name = "rfc4543-gcm-aes-caam", 2045 .cra_blocksize = 1, 2046 }, 2047 .setkey = rfc4543_setkey, 2048 .setauthsize = rfc4543_setauthsize, 2049 .encrypt = ipsec_gcm_encrypt, 2050 .decrypt = ipsec_gcm_decrypt, 2051 .ivsize = GCM_RFC4543_IV_SIZE, 2052 .maxauthsize = AES_BLOCK_SIZE, 2053 }, 2054 .caam = { 2055 .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_GCM, 2056 .nodkp = true, 2057 }, 2058 }, 2059 /* Galois Counter Mode */ 2060 { 2061 .aead = { 2062 .base = { 2063 .cra_name = "gcm(aes)", 2064 .cra_driver_name = "gcm-aes-caam", 2065 .cra_blocksize = 1, 2066 }, 2067 .setkey = gcm_setkey, 2068 .setauthsize = gcm_setauthsize, 2069 .encrypt = gcm_encrypt, 2070 .decrypt = gcm_decrypt, 2071 .ivsize = GCM_AES_IV_SIZE, 2072 .maxauthsize = AES_BLOCK_SIZE, 2073 }, 2074 .caam = { 2075 .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_GCM, 2076 .nodkp = true, 2077 }, 2078 }, 2079 /* single-pass ipsec_esp descriptor */ 2080 { 2081 .aead = { 2082 .base = { 2083 .cra_name = "authenc(hmac(md5)," 2084 "ecb(cipher_null))", 2085 .cra_driver_name = "authenc-hmac-md5-" 2086 "ecb-cipher_null-caam", 2087 .cra_blocksize = NULL_BLOCK_SIZE, 2088 }, 2089 .setkey = aead_setkey, 2090 .setauthsize = aead_setauthsize, 2091 .encrypt = aead_encrypt, 2092 .decrypt = aead_decrypt, 2093 .ivsize = NULL_IV_SIZE, 2094 .maxauthsize = MD5_DIGEST_SIZE, 2095 }, 2096 .caam = { 2097 .class2_alg_type = OP_ALG_ALGSEL_MD5 | 2098 OP_ALG_AAI_HMAC_PRECOMP, 2099 }, 2100 }, 2101 { 2102 .aead = { 2103 .base = { 2104 .cra_name = "authenc(hmac(sha1)," 2105 "ecb(cipher_null))", 2106 .cra_driver_name = "authenc-hmac-sha1-" 2107 "ecb-cipher_null-caam", 2108 .cra_blocksize = NULL_BLOCK_SIZE, 2109 }, 2110 .setkey = aead_setkey, 2111 .setauthsize = aead_setauthsize, 2112 .encrypt = aead_encrypt, 2113 .decrypt = aead_decrypt, 2114 .ivsize = NULL_IV_SIZE, 2115 .maxauthsize = SHA1_DIGEST_SIZE, 2116 }, 2117 .caam = { 2118 .class2_alg_type = OP_ALG_ALGSEL_SHA1 | 2119 OP_ALG_AAI_HMAC_PRECOMP, 2120 }, 2121 }, 2122 { 2123 .aead = { 2124 .base = { 2125 .cra_name = "authenc(hmac(sha224)," 2126 "ecb(cipher_null))", 2127 .cra_driver_name = "authenc-hmac-sha224-" 2128 "ecb-cipher_null-caam", 2129 .cra_blocksize = NULL_BLOCK_SIZE, 2130 }, 2131 .setkey = aead_setkey, 2132 .setauthsize = aead_setauthsize, 2133 .encrypt = aead_encrypt, 2134 .decrypt = aead_decrypt, 2135 .ivsize = NULL_IV_SIZE, 2136 .maxauthsize = SHA224_DIGEST_SIZE, 2137 }, 2138 .caam = { 2139 .class2_alg_type = OP_ALG_ALGSEL_SHA224 | 2140 OP_ALG_AAI_HMAC_PRECOMP, 2141 }, 2142 }, 2143 { 2144 .aead = { 2145 .base = { 2146 .cra_name = "authenc(hmac(sha256)," 2147 "ecb(cipher_null))", 2148 .cra_driver_name = "authenc-hmac-sha256-" 2149 "ecb-cipher_null-caam", 2150 .cra_blocksize = NULL_BLOCK_SIZE, 2151 }, 2152 .setkey = aead_setkey, 2153 .setauthsize = aead_setauthsize, 2154 .encrypt = aead_encrypt, 2155 .decrypt = aead_decrypt, 2156 .ivsize = NULL_IV_SIZE, 2157 .maxauthsize = SHA256_DIGEST_SIZE, 2158 }, 2159 .caam = { 2160 .class2_alg_type = OP_ALG_ALGSEL_SHA256 | 2161 OP_ALG_AAI_HMAC_PRECOMP, 2162 }, 2163 }, 2164 { 2165 .aead = { 2166 .base = { 2167 .cra_name = "authenc(hmac(sha384)," 2168 "ecb(cipher_null))", 2169 .cra_driver_name = "authenc-hmac-sha384-" 2170 "ecb-cipher_null-caam", 2171 .cra_blocksize = NULL_BLOCK_SIZE, 2172 }, 2173 .setkey = aead_setkey, 2174 .setauthsize = aead_setauthsize, 2175 .encrypt = aead_encrypt, 2176 .decrypt = aead_decrypt, 2177 .ivsize = NULL_IV_SIZE, 2178 .maxauthsize = SHA384_DIGEST_SIZE, 2179 }, 2180 .caam = { 2181 .class2_alg_type = OP_ALG_ALGSEL_SHA384 | 2182 OP_ALG_AAI_HMAC_PRECOMP, 2183 }, 2184 }, 2185 { 2186 .aead = { 2187 .base = { 2188 .cra_name = "authenc(hmac(sha512)," 2189 "ecb(cipher_null))", 2190 .cra_driver_name = "authenc-hmac-sha512-" 2191 "ecb-cipher_null-caam", 2192 .cra_blocksize = NULL_BLOCK_SIZE, 2193 }, 2194 .setkey = aead_setkey, 2195 .setauthsize = aead_setauthsize, 2196 .encrypt = aead_encrypt, 2197 .decrypt = aead_decrypt, 2198 .ivsize = NULL_IV_SIZE, 2199 .maxauthsize = SHA512_DIGEST_SIZE, 2200 }, 2201 .caam = { 2202 .class2_alg_type = OP_ALG_ALGSEL_SHA512 | 2203 OP_ALG_AAI_HMAC_PRECOMP, 2204 }, 2205 }, 2206 { 2207 .aead = { 2208 .base = { 2209 .cra_name = "authenc(hmac(md5),cbc(aes))", 2210 .cra_driver_name = "authenc-hmac-md5-" 2211 "cbc-aes-caam", 2212 .cra_blocksize = AES_BLOCK_SIZE, 2213 }, 2214 .setkey = aead_setkey, 2215 .setauthsize = aead_setauthsize, 2216 .encrypt = aead_encrypt, 2217 .decrypt = aead_decrypt, 2218 .ivsize = AES_BLOCK_SIZE, 2219 .maxauthsize = MD5_DIGEST_SIZE, 2220 }, 2221 .caam = { 2222 .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CBC, 2223 .class2_alg_type = OP_ALG_ALGSEL_MD5 | 2224 OP_ALG_AAI_HMAC_PRECOMP, 2225 }, 2226 }, 2227 { 2228 .aead = { 2229 .base = { 2230 .cra_name = "echainiv(authenc(hmac(md5)," 2231 "cbc(aes)))", 2232 .cra_driver_name = "echainiv-authenc-hmac-md5-" 2233 "cbc-aes-caam", 2234 .cra_blocksize = AES_BLOCK_SIZE, 2235 }, 2236 .setkey = aead_setkey, 2237 .setauthsize = aead_setauthsize, 2238 .encrypt = aead_encrypt, 2239 .decrypt = aead_decrypt, 2240 .ivsize = AES_BLOCK_SIZE, 2241 .maxauthsize = MD5_DIGEST_SIZE, 2242 }, 2243 .caam = { 2244 .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CBC, 2245 .class2_alg_type = OP_ALG_ALGSEL_MD5 | 2246 OP_ALG_AAI_HMAC_PRECOMP, 2247 .geniv = true, 2248 }, 2249 }, 2250 { 2251 .aead = { 2252 .base = { 2253 .cra_name = "authenc(hmac(sha1),cbc(aes))", 2254 .cra_driver_name = "authenc-hmac-sha1-" 2255 "cbc-aes-caam", 2256 .cra_blocksize = AES_BLOCK_SIZE, 2257 }, 2258 .setkey = aead_setkey, 2259 .setauthsize = aead_setauthsize, 2260 .encrypt = aead_encrypt, 2261 .decrypt = aead_decrypt, 2262 .ivsize = AES_BLOCK_SIZE, 2263 .maxauthsize = SHA1_DIGEST_SIZE, 2264 }, 2265 .caam = { 2266 .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CBC, 2267 .class2_alg_type = OP_ALG_ALGSEL_SHA1 | 2268 OP_ALG_AAI_HMAC_PRECOMP, 2269 }, 2270 }, 2271 { 2272 .aead = { 2273 .base = { 2274 .cra_name = "echainiv(authenc(hmac(sha1)," 2275 "cbc(aes)))", 2276 .cra_driver_name = "echainiv-authenc-" 2277 "hmac-sha1-cbc-aes-caam", 2278 .cra_blocksize = AES_BLOCK_SIZE, 2279 }, 2280 .setkey = aead_setkey, 2281 .setauthsize = aead_setauthsize, 2282 .encrypt = aead_encrypt, 2283 .decrypt = aead_decrypt, 2284 .ivsize = AES_BLOCK_SIZE, 2285 .maxauthsize = SHA1_DIGEST_SIZE, 2286 }, 2287 .caam = { 2288 .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CBC, 2289 .class2_alg_type = OP_ALG_ALGSEL_SHA1 | 2290 OP_ALG_AAI_HMAC_PRECOMP, 2291 .geniv = true, 2292 }, 2293 }, 2294 { 2295 .aead = { 2296 .base = { 2297 .cra_name = "authenc(hmac(sha224),cbc(aes))", 2298 .cra_driver_name = "authenc-hmac-sha224-" 2299 "cbc-aes-caam", 2300 .cra_blocksize = AES_BLOCK_SIZE, 2301 }, 2302 .setkey = aead_setkey, 2303 .setauthsize = aead_setauthsize, 2304 .encrypt = aead_encrypt, 2305 .decrypt = aead_decrypt, 2306 .ivsize = AES_BLOCK_SIZE, 2307 .maxauthsize = SHA224_DIGEST_SIZE, 2308 }, 2309 .caam = { 2310 .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CBC, 2311 .class2_alg_type = OP_ALG_ALGSEL_SHA224 | 2312 OP_ALG_AAI_HMAC_PRECOMP, 2313 }, 2314 }, 2315 { 2316 .aead = { 2317 .base = { 2318 .cra_name = "echainiv(authenc(hmac(sha224)," 2319 "cbc(aes)))", 2320 .cra_driver_name = "echainiv-authenc-" 2321 "hmac-sha224-cbc-aes-caam", 2322 .cra_blocksize = AES_BLOCK_SIZE, 2323 }, 2324 .setkey = aead_setkey, 2325 .setauthsize = aead_setauthsize, 2326 .encrypt = aead_encrypt, 2327 .decrypt = aead_decrypt, 2328 .ivsize = AES_BLOCK_SIZE, 2329 .maxauthsize = SHA224_DIGEST_SIZE, 2330 }, 2331 .caam = { 2332 .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CBC, 2333 .class2_alg_type = OP_ALG_ALGSEL_SHA224 | 2334 OP_ALG_AAI_HMAC_PRECOMP, 2335 .geniv = true, 2336 }, 2337 }, 2338 { 2339 .aead = { 2340 .base = { 2341 .cra_name = "authenc(hmac(sha256),cbc(aes))", 2342 .cra_driver_name = "authenc-hmac-sha256-" 2343 "cbc-aes-caam", 2344 .cra_blocksize = AES_BLOCK_SIZE, 2345 }, 2346 .setkey = aead_setkey, 2347 .setauthsize = aead_setauthsize, 2348 .encrypt = aead_encrypt, 2349 .decrypt = aead_decrypt, 2350 .ivsize = AES_BLOCK_SIZE, 2351 .maxauthsize = SHA256_DIGEST_SIZE, 2352 }, 2353 .caam = { 2354 .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CBC, 2355 .class2_alg_type = OP_ALG_ALGSEL_SHA256 | 2356 OP_ALG_AAI_HMAC_PRECOMP, 2357 }, 2358 }, 2359 { 2360 .aead = { 2361 .base = { 2362 .cra_name = "echainiv(authenc(hmac(sha256)," 2363 "cbc(aes)))", 2364 .cra_driver_name = "echainiv-authenc-" 2365 "hmac-sha256-cbc-aes-caam", 2366 .cra_blocksize = AES_BLOCK_SIZE, 2367 }, 2368 .setkey = aead_setkey, 2369 .setauthsize = aead_setauthsize, 2370 .encrypt = aead_encrypt, 2371 .decrypt = aead_decrypt, 2372 .ivsize = AES_BLOCK_SIZE, 2373 .maxauthsize = SHA256_DIGEST_SIZE, 2374 }, 2375 .caam = { 2376 .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CBC, 2377 .class2_alg_type = OP_ALG_ALGSEL_SHA256 | 2378 OP_ALG_AAI_HMAC_PRECOMP, 2379 .geniv = true, 2380 }, 2381 }, 2382 { 2383 .aead = { 2384 .base = { 2385 .cra_name = "authenc(hmac(sha384),cbc(aes))", 2386 .cra_driver_name = "authenc-hmac-sha384-" 2387 "cbc-aes-caam", 2388 .cra_blocksize = AES_BLOCK_SIZE, 2389 }, 2390 .setkey = aead_setkey, 2391 .setauthsize = aead_setauthsize, 2392 .encrypt = aead_encrypt, 2393 .decrypt = aead_decrypt, 2394 .ivsize = AES_BLOCK_SIZE, 2395 .maxauthsize = SHA384_DIGEST_SIZE, 2396 }, 2397 .caam = { 2398 .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CBC, 2399 .class2_alg_type = OP_ALG_ALGSEL_SHA384 | 2400 OP_ALG_AAI_HMAC_PRECOMP, 2401 }, 2402 }, 2403 { 2404 .aead = { 2405 .base = { 2406 .cra_name = "echainiv(authenc(hmac(sha384)," 2407 "cbc(aes)))", 2408 .cra_driver_name = "echainiv-authenc-" 2409 "hmac-sha384-cbc-aes-caam", 2410 .cra_blocksize = AES_BLOCK_SIZE, 2411 }, 2412 .setkey = aead_setkey, 2413 .setauthsize = aead_setauthsize, 2414 .encrypt = aead_encrypt, 2415 .decrypt = aead_decrypt, 2416 .ivsize = AES_BLOCK_SIZE, 2417 .maxauthsize = SHA384_DIGEST_SIZE, 2418 }, 2419 .caam = { 2420 .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CBC, 2421 .class2_alg_type = OP_ALG_ALGSEL_SHA384 | 2422 OP_ALG_AAI_HMAC_PRECOMP, 2423 .geniv = true, 2424 }, 2425 }, 2426 { 2427 .aead = { 2428 .base = { 2429 .cra_name = "authenc(hmac(sha512),cbc(aes))", 2430 .cra_driver_name = "authenc-hmac-sha512-" 2431 "cbc-aes-caam", 2432 .cra_blocksize = AES_BLOCK_SIZE, 2433 }, 2434 .setkey = aead_setkey, 2435 .setauthsize = aead_setauthsize, 2436 .encrypt = aead_encrypt, 2437 .decrypt = aead_decrypt, 2438 .ivsize = AES_BLOCK_SIZE, 2439 .maxauthsize = SHA512_DIGEST_SIZE, 2440 }, 2441 .caam = { 2442 .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CBC, 2443 .class2_alg_type = OP_ALG_ALGSEL_SHA512 | 2444 OP_ALG_AAI_HMAC_PRECOMP, 2445 }, 2446 }, 2447 { 2448 .aead = { 2449 .base = { 2450 .cra_name = "echainiv(authenc(hmac(sha512)," 2451 "cbc(aes)))", 2452 .cra_driver_name = "echainiv-authenc-" 2453 "hmac-sha512-cbc-aes-caam", 2454 .cra_blocksize = AES_BLOCK_SIZE, 2455 }, 2456 .setkey = aead_setkey, 2457 .setauthsize = aead_setauthsize, 2458 .encrypt = aead_encrypt, 2459 .decrypt = aead_decrypt, 2460 .ivsize = AES_BLOCK_SIZE, 2461 .maxauthsize = SHA512_DIGEST_SIZE, 2462 }, 2463 .caam = { 2464 .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CBC, 2465 .class2_alg_type = OP_ALG_ALGSEL_SHA512 | 2466 OP_ALG_AAI_HMAC_PRECOMP, 2467 .geniv = true, 2468 }, 2469 }, 2470 { 2471 .aead = { 2472 .base = { 2473 .cra_name = "authenc(hmac(md5),cbc(des3_ede))", 2474 .cra_driver_name = "authenc-hmac-md5-" 2475 "cbc-des3_ede-caam", 2476 .cra_blocksize = DES3_EDE_BLOCK_SIZE, 2477 }, 2478 .setkey = des3_aead_setkey, 2479 .setauthsize = aead_setauthsize, 2480 .encrypt = aead_encrypt, 2481 .decrypt = aead_decrypt, 2482 .ivsize = DES3_EDE_BLOCK_SIZE, 2483 .maxauthsize = MD5_DIGEST_SIZE, 2484 }, 2485 .caam = { 2486 .class1_alg_type = OP_ALG_ALGSEL_3DES | OP_ALG_AAI_CBC, 2487 .class2_alg_type = OP_ALG_ALGSEL_MD5 | 2488 OP_ALG_AAI_HMAC_PRECOMP, 2489 } 2490 }, 2491 { 2492 .aead = { 2493 .base = { 2494 .cra_name = "echainiv(authenc(hmac(md5)," 2495 "cbc(des3_ede)))", 2496 .cra_driver_name = "echainiv-authenc-hmac-md5-" 2497 "cbc-des3_ede-caam", 2498 .cra_blocksize = DES3_EDE_BLOCK_SIZE, 2499 }, 2500 .setkey = des3_aead_setkey, 2501 .setauthsize = aead_setauthsize, 2502 .encrypt = aead_encrypt, 2503 .decrypt = aead_decrypt, 2504 .ivsize = DES3_EDE_BLOCK_SIZE, 2505 .maxauthsize = MD5_DIGEST_SIZE, 2506 }, 2507 .caam = { 2508 .class1_alg_type = OP_ALG_ALGSEL_3DES | OP_ALG_AAI_CBC, 2509 .class2_alg_type = OP_ALG_ALGSEL_MD5 | 2510 OP_ALG_AAI_HMAC_PRECOMP, 2511 .geniv = true, 2512 } 2513 }, 2514 { 2515 .aead = { 2516 .base = { 2517 .cra_name = "authenc(hmac(sha1)," 2518 "cbc(des3_ede))", 2519 .cra_driver_name = "authenc-hmac-sha1-" 2520 "cbc-des3_ede-caam", 2521 .cra_blocksize = DES3_EDE_BLOCK_SIZE, 2522 }, 2523 .setkey = des3_aead_setkey, 2524 .setauthsize = aead_setauthsize, 2525 .encrypt = aead_encrypt, 2526 .decrypt = aead_decrypt, 2527 .ivsize = DES3_EDE_BLOCK_SIZE, 2528 .maxauthsize = SHA1_DIGEST_SIZE, 2529 }, 2530 .caam = { 2531 .class1_alg_type = OP_ALG_ALGSEL_3DES | OP_ALG_AAI_CBC, 2532 .class2_alg_type = OP_ALG_ALGSEL_SHA1 | 2533 OP_ALG_AAI_HMAC_PRECOMP, 2534 }, 2535 }, 2536 { 2537 .aead = { 2538 .base = { 2539 .cra_name = "echainiv(authenc(hmac(sha1)," 2540 "cbc(des3_ede)))", 2541 .cra_driver_name = "echainiv-authenc-" 2542 "hmac-sha1-" 2543 "cbc-des3_ede-caam", 2544 .cra_blocksize = DES3_EDE_BLOCK_SIZE, 2545 }, 2546 .setkey = des3_aead_setkey, 2547 .setauthsize = aead_setauthsize, 2548 .encrypt = aead_encrypt, 2549 .decrypt = aead_decrypt, 2550 .ivsize = DES3_EDE_BLOCK_SIZE, 2551 .maxauthsize = SHA1_DIGEST_SIZE, 2552 }, 2553 .caam = { 2554 .class1_alg_type = OP_ALG_ALGSEL_3DES | OP_ALG_AAI_CBC, 2555 .class2_alg_type = OP_ALG_ALGSEL_SHA1 | 2556 OP_ALG_AAI_HMAC_PRECOMP, 2557 .geniv = true, 2558 }, 2559 }, 2560 { 2561 .aead = { 2562 .base = { 2563 .cra_name = "authenc(hmac(sha224)," 2564 "cbc(des3_ede))", 2565 .cra_driver_name = "authenc-hmac-sha224-" 2566 "cbc-des3_ede-caam", 2567 .cra_blocksize = DES3_EDE_BLOCK_SIZE, 2568 }, 2569 .setkey = des3_aead_setkey, 2570 .setauthsize = aead_setauthsize, 2571 .encrypt = aead_encrypt, 2572 .decrypt = aead_decrypt, 2573 .ivsize = DES3_EDE_BLOCK_SIZE, 2574 .maxauthsize = SHA224_DIGEST_SIZE, 2575 }, 2576 .caam = { 2577 .class1_alg_type = OP_ALG_ALGSEL_3DES | OP_ALG_AAI_CBC, 2578 .class2_alg_type = OP_ALG_ALGSEL_SHA224 | 2579 OP_ALG_AAI_HMAC_PRECOMP, 2580 }, 2581 }, 2582 { 2583 .aead = { 2584 .base = { 2585 .cra_name = "echainiv(authenc(hmac(sha224)," 2586 "cbc(des3_ede)))", 2587 .cra_driver_name = "echainiv-authenc-" 2588 "hmac-sha224-" 2589 "cbc-des3_ede-caam", 2590 .cra_blocksize = DES3_EDE_BLOCK_SIZE, 2591 }, 2592 .setkey = des3_aead_setkey, 2593 .setauthsize = aead_setauthsize, 2594 .encrypt = aead_encrypt, 2595 .decrypt = aead_decrypt, 2596 .ivsize = DES3_EDE_BLOCK_SIZE, 2597 .maxauthsize = SHA224_DIGEST_SIZE, 2598 }, 2599 .caam = { 2600 .class1_alg_type = OP_ALG_ALGSEL_3DES | OP_ALG_AAI_CBC, 2601 .class2_alg_type = OP_ALG_ALGSEL_SHA224 | 2602 OP_ALG_AAI_HMAC_PRECOMP, 2603 .geniv = true, 2604 }, 2605 }, 2606 { 2607 .aead = { 2608 .base = { 2609 .cra_name = "authenc(hmac(sha256)," 2610 "cbc(des3_ede))", 2611 .cra_driver_name = "authenc-hmac-sha256-" 2612 "cbc-des3_ede-caam", 2613 .cra_blocksize = DES3_EDE_BLOCK_SIZE, 2614 }, 2615 .setkey = des3_aead_setkey, 2616 .setauthsize = aead_setauthsize, 2617 .encrypt = aead_encrypt, 2618 .decrypt = aead_decrypt, 2619 .ivsize = DES3_EDE_BLOCK_SIZE, 2620 .maxauthsize = SHA256_DIGEST_SIZE, 2621 }, 2622 .caam = { 2623 .class1_alg_type = OP_ALG_ALGSEL_3DES | OP_ALG_AAI_CBC, 2624 .class2_alg_type = OP_ALG_ALGSEL_SHA256 | 2625 OP_ALG_AAI_HMAC_PRECOMP, 2626 }, 2627 }, 2628 { 2629 .aead = { 2630 .base = { 2631 .cra_name = "echainiv(authenc(hmac(sha256)," 2632 "cbc(des3_ede)))", 2633 .cra_driver_name = "echainiv-authenc-" 2634 "hmac-sha256-" 2635 "cbc-des3_ede-caam", 2636 .cra_blocksize = DES3_EDE_BLOCK_SIZE, 2637 }, 2638 .setkey = des3_aead_setkey, 2639 .setauthsize = aead_setauthsize, 2640 .encrypt = aead_encrypt, 2641 .decrypt = aead_decrypt, 2642 .ivsize = DES3_EDE_BLOCK_SIZE, 2643 .maxauthsize = SHA256_DIGEST_SIZE, 2644 }, 2645 .caam = { 2646 .class1_alg_type = OP_ALG_ALGSEL_3DES | OP_ALG_AAI_CBC, 2647 .class2_alg_type = OP_ALG_ALGSEL_SHA256 | 2648 OP_ALG_AAI_HMAC_PRECOMP, 2649 .geniv = true, 2650 }, 2651 }, 2652 { 2653 .aead = { 2654 .base = { 2655 .cra_name = "authenc(hmac(sha384)," 2656 "cbc(des3_ede))", 2657 .cra_driver_name = "authenc-hmac-sha384-" 2658 "cbc-des3_ede-caam", 2659 .cra_blocksize = DES3_EDE_BLOCK_SIZE, 2660 }, 2661 .setkey = des3_aead_setkey, 2662 .setauthsize = aead_setauthsize, 2663 .encrypt = aead_encrypt, 2664 .decrypt = aead_decrypt, 2665 .ivsize = DES3_EDE_BLOCK_SIZE, 2666 .maxauthsize = SHA384_DIGEST_SIZE, 2667 }, 2668 .caam = { 2669 .class1_alg_type = OP_ALG_ALGSEL_3DES | OP_ALG_AAI_CBC, 2670 .class2_alg_type = OP_ALG_ALGSEL_SHA384 | 2671 OP_ALG_AAI_HMAC_PRECOMP, 2672 }, 2673 }, 2674 { 2675 .aead = { 2676 .base = { 2677 .cra_name = "echainiv(authenc(hmac(sha384)," 2678 "cbc(des3_ede)))", 2679 .cra_driver_name = "echainiv-authenc-" 2680 "hmac-sha384-" 2681 "cbc-des3_ede-caam", 2682 .cra_blocksize = DES3_EDE_BLOCK_SIZE, 2683 }, 2684 .setkey = des3_aead_setkey, 2685 .setauthsize = aead_setauthsize, 2686 .encrypt = aead_encrypt, 2687 .decrypt = aead_decrypt, 2688 .ivsize = DES3_EDE_BLOCK_SIZE, 2689 .maxauthsize = SHA384_DIGEST_SIZE, 2690 }, 2691 .caam = { 2692 .class1_alg_type = OP_ALG_ALGSEL_3DES | OP_ALG_AAI_CBC, 2693 .class2_alg_type = OP_ALG_ALGSEL_SHA384 | 2694 OP_ALG_AAI_HMAC_PRECOMP, 2695 .geniv = true, 2696 }, 2697 }, 2698 { 2699 .aead = { 2700 .base = { 2701 .cra_name = "authenc(hmac(sha512)," 2702 "cbc(des3_ede))", 2703 .cra_driver_name = "authenc-hmac-sha512-" 2704 "cbc-des3_ede-caam", 2705 .cra_blocksize = DES3_EDE_BLOCK_SIZE, 2706 }, 2707 .setkey = des3_aead_setkey, 2708 .setauthsize = aead_setauthsize, 2709 .encrypt = aead_encrypt, 2710 .decrypt = aead_decrypt, 2711 .ivsize = DES3_EDE_BLOCK_SIZE, 2712 .maxauthsize = SHA512_DIGEST_SIZE, 2713 }, 2714 .caam = { 2715 .class1_alg_type = OP_ALG_ALGSEL_3DES | OP_ALG_AAI_CBC, 2716 .class2_alg_type = OP_ALG_ALGSEL_SHA512 | 2717 OP_ALG_AAI_HMAC_PRECOMP, 2718 }, 2719 }, 2720 { 2721 .aead = { 2722 .base = { 2723 .cra_name = "echainiv(authenc(hmac(sha512)," 2724 "cbc(des3_ede)))", 2725 .cra_driver_name = "echainiv-authenc-" 2726 "hmac-sha512-" 2727 "cbc-des3_ede-caam", 2728 .cra_blocksize = DES3_EDE_BLOCK_SIZE, 2729 }, 2730 .setkey = des3_aead_setkey, 2731 .setauthsize = aead_setauthsize, 2732 .encrypt = aead_encrypt, 2733 .decrypt = aead_decrypt, 2734 .ivsize = DES3_EDE_BLOCK_SIZE, 2735 .maxauthsize = SHA512_DIGEST_SIZE, 2736 }, 2737 .caam = { 2738 .class1_alg_type = OP_ALG_ALGSEL_3DES | OP_ALG_AAI_CBC, 2739 .class2_alg_type = OP_ALG_ALGSEL_SHA512 | 2740 OP_ALG_AAI_HMAC_PRECOMP, 2741 .geniv = true, 2742 }, 2743 }, 2744 { 2745 .aead = { 2746 .base = { 2747 .cra_name = "authenc(hmac(md5),cbc(des))", 2748 .cra_driver_name = "authenc-hmac-md5-" 2749 "cbc-des-caam", 2750 .cra_blocksize = DES_BLOCK_SIZE, 2751 }, 2752 .setkey = aead_setkey, 2753 .setauthsize = aead_setauthsize, 2754 .encrypt = aead_encrypt, 2755 .decrypt = aead_decrypt, 2756 .ivsize = DES_BLOCK_SIZE, 2757 .maxauthsize = MD5_DIGEST_SIZE, 2758 }, 2759 .caam = { 2760 .class1_alg_type = OP_ALG_ALGSEL_DES | OP_ALG_AAI_CBC, 2761 .class2_alg_type = OP_ALG_ALGSEL_MD5 | 2762 OP_ALG_AAI_HMAC_PRECOMP, 2763 }, 2764 }, 2765 { 2766 .aead = { 2767 .base = { 2768 .cra_name = "echainiv(authenc(hmac(md5)," 2769 "cbc(des)))", 2770 .cra_driver_name = "echainiv-authenc-hmac-md5-" 2771 "cbc-des-caam", 2772 .cra_blocksize = DES_BLOCK_SIZE, 2773 }, 2774 .setkey = aead_setkey, 2775 .setauthsize = aead_setauthsize, 2776 .encrypt = aead_encrypt, 2777 .decrypt = aead_decrypt, 2778 .ivsize = DES_BLOCK_SIZE, 2779 .maxauthsize = MD5_DIGEST_SIZE, 2780 }, 2781 .caam = { 2782 .class1_alg_type = OP_ALG_ALGSEL_DES | OP_ALG_AAI_CBC, 2783 .class2_alg_type = OP_ALG_ALGSEL_MD5 | 2784 OP_ALG_AAI_HMAC_PRECOMP, 2785 .geniv = true, 2786 }, 2787 }, 2788 { 2789 .aead = { 2790 .base = { 2791 .cra_name = "authenc(hmac(sha1),cbc(des))", 2792 .cra_driver_name = "authenc-hmac-sha1-" 2793 "cbc-des-caam", 2794 .cra_blocksize = DES_BLOCK_SIZE, 2795 }, 2796 .setkey = aead_setkey, 2797 .setauthsize = aead_setauthsize, 2798 .encrypt = aead_encrypt, 2799 .decrypt = aead_decrypt, 2800 .ivsize = DES_BLOCK_SIZE, 2801 .maxauthsize = SHA1_DIGEST_SIZE, 2802 }, 2803 .caam = { 2804 .class1_alg_type = OP_ALG_ALGSEL_DES | OP_ALG_AAI_CBC, 2805 .class2_alg_type = OP_ALG_ALGSEL_SHA1 | 2806 OP_ALG_AAI_HMAC_PRECOMP, 2807 }, 2808 }, 2809 { 2810 .aead = { 2811 .base = { 2812 .cra_name = "echainiv(authenc(hmac(sha1)," 2813 "cbc(des)))", 2814 .cra_driver_name = "echainiv-authenc-" 2815 "hmac-sha1-cbc-des-caam", 2816 .cra_blocksize = DES_BLOCK_SIZE, 2817 }, 2818 .setkey = aead_setkey, 2819 .setauthsize = aead_setauthsize, 2820 .encrypt = aead_encrypt, 2821 .decrypt = aead_decrypt, 2822 .ivsize = DES_BLOCK_SIZE, 2823 .maxauthsize = SHA1_DIGEST_SIZE, 2824 }, 2825 .caam = { 2826 .class1_alg_type = OP_ALG_ALGSEL_DES | OP_ALG_AAI_CBC, 2827 .class2_alg_type = OP_ALG_ALGSEL_SHA1 | 2828 OP_ALG_AAI_HMAC_PRECOMP, 2829 .geniv = true, 2830 }, 2831 }, 2832 { 2833 .aead = { 2834 .base = { 2835 .cra_name = "authenc(hmac(sha224),cbc(des))", 2836 .cra_driver_name = "authenc-hmac-sha224-" 2837 "cbc-des-caam", 2838 .cra_blocksize = DES_BLOCK_SIZE, 2839 }, 2840 .setkey = aead_setkey, 2841 .setauthsize = aead_setauthsize, 2842 .encrypt = aead_encrypt, 2843 .decrypt = aead_decrypt, 2844 .ivsize = DES_BLOCK_SIZE, 2845 .maxauthsize = SHA224_DIGEST_SIZE, 2846 }, 2847 .caam = { 2848 .class1_alg_type = OP_ALG_ALGSEL_DES | OP_ALG_AAI_CBC, 2849 .class2_alg_type = OP_ALG_ALGSEL_SHA224 | 2850 OP_ALG_AAI_HMAC_PRECOMP, 2851 }, 2852 }, 2853 { 2854 .aead = { 2855 .base = { 2856 .cra_name = "echainiv(authenc(hmac(sha224)," 2857 "cbc(des)))", 2858 .cra_driver_name = "echainiv-authenc-" 2859 "hmac-sha224-cbc-des-caam", 2860 .cra_blocksize = DES_BLOCK_SIZE, 2861 }, 2862 .setkey = aead_setkey, 2863 .setauthsize = aead_setauthsize, 2864 .encrypt = aead_encrypt, 2865 .decrypt = aead_decrypt, 2866 .ivsize = DES_BLOCK_SIZE, 2867 .maxauthsize = SHA224_DIGEST_SIZE, 2868 }, 2869 .caam = { 2870 .class1_alg_type = OP_ALG_ALGSEL_DES | OP_ALG_AAI_CBC, 2871 .class2_alg_type = OP_ALG_ALGSEL_SHA224 | 2872 OP_ALG_AAI_HMAC_PRECOMP, 2873 .geniv = true, 2874 }, 2875 }, 2876 { 2877 .aead = { 2878 .base = { 2879 .cra_name = "authenc(hmac(sha256),cbc(des))", 2880 .cra_driver_name = "authenc-hmac-sha256-" 2881 "cbc-des-caam", 2882 .cra_blocksize = DES_BLOCK_SIZE, 2883 }, 2884 .setkey = aead_setkey, 2885 .setauthsize = aead_setauthsize, 2886 .encrypt = aead_encrypt, 2887 .decrypt = aead_decrypt, 2888 .ivsize = DES_BLOCK_SIZE, 2889 .maxauthsize = SHA256_DIGEST_SIZE, 2890 }, 2891 .caam = { 2892 .class1_alg_type = OP_ALG_ALGSEL_DES | OP_ALG_AAI_CBC, 2893 .class2_alg_type = OP_ALG_ALGSEL_SHA256 | 2894 OP_ALG_AAI_HMAC_PRECOMP, 2895 }, 2896 }, 2897 { 2898 .aead = { 2899 .base = { 2900 .cra_name = "echainiv(authenc(hmac(sha256)," 2901 "cbc(des)))", 2902 .cra_driver_name = "echainiv-authenc-" 2903 "hmac-sha256-cbc-des-caam", 2904 .cra_blocksize = DES_BLOCK_SIZE, 2905 }, 2906 .setkey = aead_setkey, 2907 .setauthsize = aead_setauthsize, 2908 .encrypt = aead_encrypt, 2909 .decrypt = aead_decrypt, 2910 .ivsize = DES_BLOCK_SIZE, 2911 .maxauthsize = SHA256_DIGEST_SIZE, 2912 }, 2913 .caam = { 2914 .class1_alg_type = OP_ALG_ALGSEL_DES | OP_ALG_AAI_CBC, 2915 .class2_alg_type = OP_ALG_ALGSEL_SHA256 | 2916 OP_ALG_AAI_HMAC_PRECOMP, 2917 .geniv = true, 2918 }, 2919 }, 2920 { 2921 .aead = { 2922 .base = { 2923 .cra_name = "authenc(hmac(sha384),cbc(des))", 2924 .cra_driver_name = "authenc-hmac-sha384-" 2925 "cbc-des-caam", 2926 .cra_blocksize = DES_BLOCK_SIZE, 2927 }, 2928 .setkey = aead_setkey, 2929 .setauthsize = aead_setauthsize, 2930 .encrypt = aead_encrypt, 2931 .decrypt = aead_decrypt, 2932 .ivsize = DES_BLOCK_SIZE, 2933 .maxauthsize = SHA384_DIGEST_SIZE, 2934 }, 2935 .caam = { 2936 .class1_alg_type = OP_ALG_ALGSEL_DES | OP_ALG_AAI_CBC, 2937 .class2_alg_type = OP_ALG_ALGSEL_SHA384 | 2938 OP_ALG_AAI_HMAC_PRECOMP, 2939 }, 2940 }, 2941 { 2942 .aead = { 2943 .base = { 2944 .cra_name = "echainiv(authenc(hmac(sha384)," 2945 "cbc(des)))", 2946 .cra_driver_name = "echainiv-authenc-" 2947 "hmac-sha384-cbc-des-caam", 2948 .cra_blocksize = DES_BLOCK_SIZE, 2949 }, 2950 .setkey = aead_setkey, 2951 .setauthsize = aead_setauthsize, 2952 .encrypt = aead_encrypt, 2953 .decrypt = aead_decrypt, 2954 .ivsize = DES_BLOCK_SIZE, 2955 .maxauthsize = SHA384_DIGEST_SIZE, 2956 }, 2957 .caam = { 2958 .class1_alg_type = OP_ALG_ALGSEL_DES | OP_ALG_AAI_CBC, 2959 .class2_alg_type = OP_ALG_ALGSEL_SHA384 | 2960 OP_ALG_AAI_HMAC_PRECOMP, 2961 .geniv = true, 2962 }, 2963 }, 2964 { 2965 .aead = { 2966 .base = { 2967 .cra_name = "authenc(hmac(sha512),cbc(des))", 2968 .cra_driver_name = "authenc-hmac-sha512-" 2969 "cbc-des-caam", 2970 .cra_blocksize = DES_BLOCK_SIZE, 2971 }, 2972 .setkey = aead_setkey, 2973 .setauthsize = aead_setauthsize, 2974 .encrypt = aead_encrypt, 2975 .decrypt = aead_decrypt, 2976 .ivsize = DES_BLOCK_SIZE, 2977 .maxauthsize = SHA512_DIGEST_SIZE, 2978 }, 2979 .caam = { 2980 .class1_alg_type = OP_ALG_ALGSEL_DES | OP_ALG_AAI_CBC, 2981 .class2_alg_type = OP_ALG_ALGSEL_SHA512 | 2982 OP_ALG_AAI_HMAC_PRECOMP, 2983 }, 2984 }, 2985 { 2986 .aead = { 2987 .base = { 2988 .cra_name = "echainiv(authenc(hmac(sha512)," 2989 "cbc(des)))", 2990 .cra_driver_name = "echainiv-authenc-" 2991 "hmac-sha512-cbc-des-caam", 2992 .cra_blocksize = DES_BLOCK_SIZE, 2993 }, 2994 .setkey = aead_setkey, 2995 .setauthsize = aead_setauthsize, 2996 .encrypt = aead_encrypt, 2997 .decrypt = aead_decrypt, 2998 .ivsize = DES_BLOCK_SIZE, 2999 .maxauthsize = SHA512_DIGEST_SIZE, 3000 }, 3001 .caam = { 3002 .class1_alg_type = OP_ALG_ALGSEL_DES | OP_ALG_AAI_CBC, 3003 .class2_alg_type = OP_ALG_ALGSEL_SHA512 | 3004 OP_ALG_AAI_HMAC_PRECOMP, 3005 .geniv = true, 3006 }, 3007 }, 3008 { 3009 .aead = { 3010 .base = { 3011 .cra_name = "authenc(hmac(md5)," 3012 "rfc3686(ctr(aes)))", 3013 .cra_driver_name = "authenc-hmac-md5-" 3014 "rfc3686-ctr-aes-caam", 3015 .cra_blocksize = 1, 3016 }, 3017 .setkey = aead_setkey, 3018 .setauthsize = aead_setauthsize, 3019 .encrypt = aead_encrypt, 3020 .decrypt = aead_decrypt, 3021 .ivsize = CTR_RFC3686_IV_SIZE, 3022 .maxauthsize = MD5_DIGEST_SIZE, 3023 }, 3024 .caam = { 3025 .class1_alg_type = OP_ALG_ALGSEL_AES | 3026 OP_ALG_AAI_CTR_MOD128, 3027 .class2_alg_type = OP_ALG_ALGSEL_MD5 | 3028 OP_ALG_AAI_HMAC_PRECOMP, 3029 .rfc3686 = true, 3030 }, 3031 }, 3032 { 3033 .aead = { 3034 .base = { 3035 .cra_name = "seqiv(authenc(" 3036 "hmac(md5),rfc3686(ctr(aes))))", 3037 .cra_driver_name = "seqiv-authenc-hmac-md5-" 3038 "rfc3686-ctr-aes-caam", 3039 .cra_blocksize = 1, 3040 }, 3041 .setkey = aead_setkey, 3042 .setauthsize = aead_setauthsize, 3043 .encrypt = aead_encrypt, 3044 .decrypt = aead_decrypt, 3045 .ivsize = CTR_RFC3686_IV_SIZE, 3046 .maxauthsize = MD5_DIGEST_SIZE, 3047 }, 3048 .caam = { 3049 .class1_alg_type = OP_ALG_ALGSEL_AES | 3050 OP_ALG_AAI_CTR_MOD128, 3051 .class2_alg_type = OP_ALG_ALGSEL_MD5 | 3052 OP_ALG_AAI_HMAC_PRECOMP, 3053 .rfc3686 = true, 3054 .geniv = true, 3055 }, 3056 }, 3057 { 3058 .aead = { 3059 .base = { 3060 .cra_name = "authenc(hmac(sha1)," 3061 "rfc3686(ctr(aes)))", 3062 .cra_driver_name = "authenc-hmac-sha1-" 3063 "rfc3686-ctr-aes-caam", 3064 .cra_blocksize = 1, 3065 }, 3066 .setkey = aead_setkey, 3067 .setauthsize = aead_setauthsize, 3068 .encrypt = aead_encrypt, 3069 .decrypt = aead_decrypt, 3070 .ivsize = CTR_RFC3686_IV_SIZE, 3071 .maxauthsize = SHA1_DIGEST_SIZE, 3072 }, 3073 .caam = { 3074 .class1_alg_type = OP_ALG_ALGSEL_AES | 3075 OP_ALG_AAI_CTR_MOD128, 3076 .class2_alg_type = OP_ALG_ALGSEL_SHA1 | 3077 OP_ALG_AAI_HMAC_PRECOMP, 3078 .rfc3686 = true, 3079 }, 3080 }, 3081 { 3082 .aead = { 3083 .base = { 3084 .cra_name = "seqiv(authenc(" 3085 "hmac(sha1),rfc3686(ctr(aes))))", 3086 .cra_driver_name = "seqiv-authenc-hmac-sha1-" 3087 "rfc3686-ctr-aes-caam", 3088 .cra_blocksize = 1, 3089 }, 3090 .setkey = aead_setkey, 3091 .setauthsize = aead_setauthsize, 3092 .encrypt = aead_encrypt, 3093 .decrypt = aead_decrypt, 3094 .ivsize = CTR_RFC3686_IV_SIZE, 3095 .maxauthsize = SHA1_DIGEST_SIZE, 3096 }, 3097 .caam = { 3098 .class1_alg_type = OP_ALG_ALGSEL_AES | 3099 OP_ALG_AAI_CTR_MOD128, 3100 .class2_alg_type = OP_ALG_ALGSEL_SHA1 | 3101 OP_ALG_AAI_HMAC_PRECOMP, 3102 .rfc3686 = true, 3103 .geniv = true, 3104 }, 3105 }, 3106 { 3107 .aead = { 3108 .base = { 3109 .cra_name = "authenc(hmac(sha224)," 3110 "rfc3686(ctr(aes)))", 3111 .cra_driver_name = "authenc-hmac-sha224-" 3112 "rfc3686-ctr-aes-caam", 3113 .cra_blocksize = 1, 3114 }, 3115 .setkey = aead_setkey, 3116 .setauthsize = aead_setauthsize, 3117 .encrypt = aead_encrypt, 3118 .decrypt = aead_decrypt, 3119 .ivsize = CTR_RFC3686_IV_SIZE, 3120 .maxauthsize = SHA224_DIGEST_SIZE, 3121 }, 3122 .caam = { 3123 .class1_alg_type = OP_ALG_ALGSEL_AES | 3124 OP_ALG_AAI_CTR_MOD128, 3125 .class2_alg_type = OP_ALG_ALGSEL_SHA224 | 3126 OP_ALG_AAI_HMAC_PRECOMP, 3127 .rfc3686 = true, 3128 }, 3129 }, 3130 { 3131 .aead = { 3132 .base = { 3133 .cra_name = "seqiv(authenc(" 3134 "hmac(sha224),rfc3686(ctr(aes))))", 3135 .cra_driver_name = "seqiv-authenc-hmac-sha224-" 3136 "rfc3686-ctr-aes-caam", 3137 .cra_blocksize = 1, 3138 }, 3139 .setkey = aead_setkey, 3140 .setauthsize = aead_setauthsize, 3141 .encrypt = aead_encrypt, 3142 .decrypt = aead_decrypt, 3143 .ivsize = CTR_RFC3686_IV_SIZE, 3144 .maxauthsize = SHA224_DIGEST_SIZE, 3145 }, 3146 .caam = { 3147 .class1_alg_type = OP_ALG_ALGSEL_AES | 3148 OP_ALG_AAI_CTR_MOD128, 3149 .class2_alg_type = OP_ALG_ALGSEL_SHA224 | 3150 OP_ALG_AAI_HMAC_PRECOMP, 3151 .rfc3686 = true, 3152 .geniv = true, 3153 }, 3154 }, 3155 { 3156 .aead = { 3157 .base = { 3158 .cra_name = "authenc(hmac(sha256)," 3159 "rfc3686(ctr(aes)))", 3160 .cra_driver_name = "authenc-hmac-sha256-" 3161 "rfc3686-ctr-aes-caam", 3162 .cra_blocksize = 1, 3163 }, 3164 .setkey = aead_setkey, 3165 .setauthsize = aead_setauthsize, 3166 .encrypt = aead_encrypt, 3167 .decrypt = aead_decrypt, 3168 .ivsize = CTR_RFC3686_IV_SIZE, 3169 .maxauthsize = SHA256_DIGEST_SIZE, 3170 }, 3171 .caam = { 3172 .class1_alg_type = OP_ALG_ALGSEL_AES | 3173 OP_ALG_AAI_CTR_MOD128, 3174 .class2_alg_type = OP_ALG_ALGSEL_SHA256 | 3175 OP_ALG_AAI_HMAC_PRECOMP, 3176 .rfc3686 = true, 3177 }, 3178 }, 3179 { 3180 .aead = { 3181 .base = { 3182 .cra_name = "seqiv(authenc(hmac(sha256)," 3183 "rfc3686(ctr(aes))))", 3184 .cra_driver_name = "seqiv-authenc-hmac-sha256-" 3185 "rfc3686-ctr-aes-caam", 3186 .cra_blocksize = 1, 3187 }, 3188 .setkey = aead_setkey, 3189 .setauthsize = aead_setauthsize, 3190 .encrypt = aead_encrypt, 3191 .decrypt = aead_decrypt, 3192 .ivsize = CTR_RFC3686_IV_SIZE, 3193 .maxauthsize = SHA256_DIGEST_SIZE, 3194 }, 3195 .caam = { 3196 .class1_alg_type = OP_ALG_ALGSEL_AES | 3197 OP_ALG_AAI_CTR_MOD128, 3198 .class2_alg_type = OP_ALG_ALGSEL_SHA256 | 3199 OP_ALG_AAI_HMAC_PRECOMP, 3200 .rfc3686 = true, 3201 .geniv = true, 3202 }, 3203 }, 3204 { 3205 .aead = { 3206 .base = { 3207 .cra_name = "authenc(hmac(sha384)," 3208 "rfc3686(ctr(aes)))", 3209 .cra_driver_name = "authenc-hmac-sha384-" 3210 "rfc3686-ctr-aes-caam", 3211 .cra_blocksize = 1, 3212 }, 3213 .setkey = aead_setkey, 3214 .setauthsize = aead_setauthsize, 3215 .encrypt = aead_encrypt, 3216 .decrypt = aead_decrypt, 3217 .ivsize = CTR_RFC3686_IV_SIZE, 3218 .maxauthsize = SHA384_DIGEST_SIZE, 3219 }, 3220 .caam = { 3221 .class1_alg_type = OP_ALG_ALGSEL_AES | 3222 OP_ALG_AAI_CTR_MOD128, 3223 .class2_alg_type = OP_ALG_ALGSEL_SHA384 | 3224 OP_ALG_AAI_HMAC_PRECOMP, 3225 .rfc3686 = true, 3226 }, 3227 }, 3228 { 3229 .aead = { 3230 .base = { 3231 .cra_name = "seqiv(authenc(hmac(sha384)," 3232 "rfc3686(ctr(aes))))", 3233 .cra_driver_name = "seqiv-authenc-hmac-sha384-" 3234 "rfc3686-ctr-aes-caam", 3235 .cra_blocksize = 1, 3236 }, 3237 .setkey = aead_setkey, 3238 .setauthsize = aead_setauthsize, 3239 .encrypt = aead_encrypt, 3240 .decrypt = aead_decrypt, 3241 .ivsize = CTR_RFC3686_IV_SIZE, 3242 .maxauthsize = SHA384_DIGEST_SIZE, 3243 }, 3244 .caam = { 3245 .class1_alg_type = OP_ALG_ALGSEL_AES | 3246 OP_ALG_AAI_CTR_MOD128, 3247 .class2_alg_type = OP_ALG_ALGSEL_SHA384 | 3248 OP_ALG_AAI_HMAC_PRECOMP, 3249 .rfc3686 = true, 3250 .geniv = true, 3251 }, 3252 }, 3253 { 3254 .aead = { 3255 .base = { 3256 .cra_name = "authenc(hmac(sha512)," 3257 "rfc3686(ctr(aes)))", 3258 .cra_driver_name = "authenc-hmac-sha512-" 3259 "rfc3686-ctr-aes-caam", 3260 .cra_blocksize = 1, 3261 }, 3262 .setkey = aead_setkey, 3263 .setauthsize = aead_setauthsize, 3264 .encrypt = aead_encrypt, 3265 .decrypt = aead_decrypt, 3266 .ivsize = CTR_RFC3686_IV_SIZE, 3267 .maxauthsize = SHA512_DIGEST_SIZE, 3268 }, 3269 .caam = { 3270 .class1_alg_type = OP_ALG_ALGSEL_AES | 3271 OP_ALG_AAI_CTR_MOD128, 3272 .class2_alg_type = OP_ALG_ALGSEL_SHA512 | 3273 OP_ALG_AAI_HMAC_PRECOMP, 3274 .rfc3686 = true, 3275 }, 3276 }, 3277 { 3278 .aead = { 3279 .base = { 3280 .cra_name = "seqiv(authenc(hmac(sha512)," 3281 "rfc3686(ctr(aes))))", 3282 .cra_driver_name = "seqiv-authenc-hmac-sha512-" 3283 "rfc3686-ctr-aes-caam", 3284 .cra_blocksize = 1, 3285 }, 3286 .setkey = aead_setkey, 3287 .setauthsize = aead_setauthsize, 3288 .encrypt = aead_encrypt, 3289 .decrypt = aead_decrypt, 3290 .ivsize = CTR_RFC3686_IV_SIZE, 3291 .maxauthsize = SHA512_DIGEST_SIZE, 3292 }, 3293 .caam = { 3294 .class1_alg_type = OP_ALG_ALGSEL_AES | 3295 OP_ALG_AAI_CTR_MOD128, 3296 .class2_alg_type = OP_ALG_ALGSEL_SHA512 | 3297 OP_ALG_AAI_HMAC_PRECOMP, 3298 .rfc3686 = true, 3299 .geniv = true, 3300 }, 3301 }, 3302 { 3303 .aead = { 3304 .base = { 3305 .cra_name = "rfc7539(chacha20,poly1305)", 3306 .cra_driver_name = "rfc7539-chacha20-poly1305-" 3307 "caam", 3308 .cra_blocksize = 1, 3309 }, 3310 .setkey = chachapoly_setkey, 3311 .setauthsize = chachapoly_setauthsize, 3312 .encrypt = chachapoly_encrypt, 3313 .decrypt = chachapoly_decrypt, 3314 .ivsize = CHACHAPOLY_IV_SIZE, 3315 .maxauthsize = POLY1305_DIGEST_SIZE, 3316 }, 3317 .caam = { 3318 .class1_alg_type = OP_ALG_ALGSEL_CHACHA20 | 3319 OP_ALG_AAI_AEAD, 3320 .class2_alg_type = OP_ALG_ALGSEL_POLY1305 | 3321 OP_ALG_AAI_AEAD, 3322 .nodkp = true, 3323 }, 3324 }, 3325 { 3326 .aead = { 3327 .base = { 3328 .cra_name = "rfc7539esp(chacha20,poly1305)", 3329 .cra_driver_name = "rfc7539esp-chacha20-" 3330 "poly1305-caam", 3331 .cra_blocksize = 1, 3332 }, 3333 .setkey = chachapoly_setkey, 3334 .setauthsize = chachapoly_setauthsize, 3335 .encrypt = chachapoly_encrypt, 3336 .decrypt = chachapoly_decrypt, 3337 .ivsize = 8, 3338 .maxauthsize = POLY1305_DIGEST_SIZE, 3339 }, 3340 .caam = { 3341 .class1_alg_type = OP_ALG_ALGSEL_CHACHA20 | 3342 OP_ALG_AAI_AEAD, 3343 .class2_alg_type = OP_ALG_ALGSEL_POLY1305 | 3344 OP_ALG_AAI_AEAD, 3345 .nodkp = true, 3346 }, 3347 }, 3348 }; 3349 3350 static int caam_init_common(struct caam_ctx *ctx, struct caam_alg_entry *caam, 3351 bool uses_dkp) 3352 { 3353 dma_addr_t dma_addr; 3354 struct caam_drv_private *priv; 3355 const size_t sh_desc_enc_offset = offsetof(struct caam_ctx, 3356 sh_desc_enc); 3357 3358 ctx->jrdev = caam_jr_alloc(); 3359 if (IS_ERR(ctx->jrdev)) { 3360 pr_err("Job Ring Device allocation for transform failed\n"); 3361 return PTR_ERR(ctx->jrdev); 3362 } 3363 3364 priv = dev_get_drvdata(ctx->jrdev->parent); 3365 if (priv->era >= 6 && uses_dkp) 3366 ctx->dir = DMA_BIDIRECTIONAL; 3367 else 3368 ctx->dir = DMA_TO_DEVICE; 3369 3370 dma_addr = dma_map_single_attrs(ctx->jrdev, ctx->sh_desc_enc, 3371 offsetof(struct caam_ctx, 3372 sh_desc_enc_dma) - 3373 sh_desc_enc_offset, 3374 ctx->dir, DMA_ATTR_SKIP_CPU_SYNC); 3375 if (dma_mapping_error(ctx->jrdev, dma_addr)) { 3376 dev_err(ctx->jrdev, "unable to map key, shared descriptors\n"); 3377 caam_jr_free(ctx->jrdev); 3378 return -ENOMEM; 3379 } 3380 3381 ctx->sh_desc_enc_dma = dma_addr; 3382 ctx->sh_desc_dec_dma = dma_addr + offsetof(struct caam_ctx, 3383 sh_desc_dec) - 3384 sh_desc_enc_offset; 3385 ctx->key_dma = dma_addr + offsetof(struct caam_ctx, key) - 3386 sh_desc_enc_offset; 3387 3388 /* copy descriptor header template value */ 3389 ctx->cdata.algtype = OP_TYPE_CLASS1_ALG | caam->class1_alg_type; 3390 ctx->adata.algtype = OP_TYPE_CLASS2_ALG | caam->class2_alg_type; 3391 3392 return 0; 3393 } 3394 3395 static int caam_cra_init(struct crypto_skcipher *tfm) 3396 { 3397 struct skcipher_alg *alg = crypto_skcipher_alg(tfm); 3398 struct caam_skcipher_alg *caam_alg = 3399 container_of(alg, typeof(*caam_alg), skcipher); 3400 struct caam_ctx *ctx = crypto_skcipher_ctx_dma(tfm); 3401 u32 alg_aai = caam_alg->caam.class1_alg_type & OP_ALG_AAI_MASK; 3402 int ret = 0; 3403 3404 ctx->enginectx.op.do_one_request = skcipher_do_one_req; 3405 3406 if (alg_aai == OP_ALG_AAI_XTS) { 3407 const char *tfm_name = crypto_tfm_alg_name(&tfm->base); 3408 struct crypto_skcipher *fallback; 3409 3410 fallback = crypto_alloc_skcipher(tfm_name, 0, 3411 CRYPTO_ALG_NEED_FALLBACK); 3412 if (IS_ERR(fallback)) { 3413 pr_err("Failed to allocate %s fallback: %ld\n", 3414 tfm_name, PTR_ERR(fallback)); 3415 return PTR_ERR(fallback); 3416 } 3417 3418 ctx->fallback = fallback; 3419 crypto_skcipher_set_reqsize(tfm, sizeof(struct caam_skcipher_req_ctx) + 3420 crypto_skcipher_reqsize(fallback)); 3421 } else { 3422 crypto_skcipher_set_reqsize(tfm, sizeof(struct caam_skcipher_req_ctx)); 3423 } 3424 3425 ret = caam_init_common(ctx, &caam_alg->caam, false); 3426 if (ret && ctx->fallback) 3427 crypto_free_skcipher(ctx->fallback); 3428 3429 return ret; 3430 } 3431 3432 static int caam_aead_init(struct crypto_aead *tfm) 3433 { 3434 struct aead_alg *alg = crypto_aead_alg(tfm); 3435 struct caam_aead_alg *caam_alg = 3436 container_of(alg, struct caam_aead_alg, aead); 3437 struct caam_ctx *ctx = crypto_aead_ctx_dma(tfm); 3438 3439 crypto_aead_set_reqsize(tfm, sizeof(struct caam_aead_req_ctx)); 3440 3441 ctx->enginectx.op.do_one_request = aead_do_one_req; 3442 3443 return caam_init_common(ctx, &caam_alg->caam, !caam_alg->caam.nodkp); 3444 } 3445 3446 static void caam_exit_common(struct caam_ctx *ctx) 3447 { 3448 dma_unmap_single_attrs(ctx->jrdev, ctx->sh_desc_enc_dma, 3449 offsetof(struct caam_ctx, sh_desc_enc_dma) - 3450 offsetof(struct caam_ctx, sh_desc_enc), 3451 ctx->dir, DMA_ATTR_SKIP_CPU_SYNC); 3452 caam_jr_free(ctx->jrdev); 3453 } 3454 3455 static void caam_cra_exit(struct crypto_skcipher *tfm) 3456 { 3457 struct caam_ctx *ctx = crypto_skcipher_ctx_dma(tfm); 3458 3459 if (ctx->fallback) 3460 crypto_free_skcipher(ctx->fallback); 3461 caam_exit_common(ctx); 3462 } 3463 3464 static void caam_aead_exit(struct crypto_aead *tfm) 3465 { 3466 caam_exit_common(crypto_aead_ctx_dma(tfm)); 3467 } 3468 3469 void caam_algapi_exit(void) 3470 { 3471 int i; 3472 3473 for (i = 0; i < ARRAY_SIZE(driver_aeads); i++) { 3474 struct caam_aead_alg *t_alg = driver_aeads + i; 3475 3476 if (t_alg->registered) 3477 crypto_unregister_aead(&t_alg->aead); 3478 } 3479 3480 for (i = 0; i < ARRAY_SIZE(driver_algs); i++) { 3481 struct caam_skcipher_alg *t_alg = driver_algs + i; 3482 3483 if (t_alg->registered) 3484 crypto_unregister_skcipher(&t_alg->skcipher); 3485 } 3486 } 3487 3488 static void caam_skcipher_alg_init(struct caam_skcipher_alg *t_alg) 3489 { 3490 struct skcipher_alg *alg = &t_alg->skcipher; 3491 3492 alg->base.cra_module = THIS_MODULE; 3493 alg->base.cra_priority = CAAM_CRA_PRIORITY; 3494 alg->base.cra_ctxsize = sizeof(struct caam_ctx) + crypto_dma_padding(); 3495 alg->base.cra_flags |= (CRYPTO_ALG_ASYNC | CRYPTO_ALG_ALLOCATES_MEMORY | 3496 CRYPTO_ALG_KERN_DRIVER_ONLY); 3497 3498 alg->init = caam_cra_init; 3499 alg->exit = caam_cra_exit; 3500 } 3501 3502 static void caam_aead_alg_init(struct caam_aead_alg *t_alg) 3503 { 3504 struct aead_alg *alg = &t_alg->aead; 3505 3506 alg->base.cra_module = THIS_MODULE; 3507 alg->base.cra_priority = CAAM_CRA_PRIORITY; 3508 alg->base.cra_ctxsize = sizeof(struct caam_ctx) + crypto_dma_padding(); 3509 alg->base.cra_flags = CRYPTO_ALG_ASYNC | CRYPTO_ALG_ALLOCATES_MEMORY | 3510 CRYPTO_ALG_KERN_DRIVER_ONLY; 3511 3512 alg->init = caam_aead_init; 3513 alg->exit = caam_aead_exit; 3514 } 3515 3516 int caam_algapi_init(struct device *ctrldev) 3517 { 3518 struct caam_drv_private *priv = dev_get_drvdata(ctrldev); 3519 int i = 0, err = 0; 3520 u32 aes_vid, aes_inst, des_inst, md_vid, md_inst, ccha_inst, ptha_inst; 3521 unsigned int md_limit = SHA512_DIGEST_SIZE; 3522 bool registered = false, gcm_support; 3523 3524 /* 3525 * Register crypto algorithms the device supports. 3526 * First, detect presence and attributes of DES, AES, and MD blocks. 3527 */ 3528 if (priv->era < 10) { 3529 u32 cha_vid, cha_inst, aes_rn; 3530 3531 cha_vid = rd_reg32(&priv->ctrl->perfmon.cha_id_ls); 3532 aes_vid = cha_vid & CHA_ID_LS_AES_MASK; 3533 md_vid = (cha_vid & CHA_ID_LS_MD_MASK) >> CHA_ID_LS_MD_SHIFT; 3534 3535 cha_inst = rd_reg32(&priv->ctrl->perfmon.cha_num_ls); 3536 des_inst = (cha_inst & CHA_ID_LS_DES_MASK) >> 3537 CHA_ID_LS_DES_SHIFT; 3538 aes_inst = cha_inst & CHA_ID_LS_AES_MASK; 3539 md_inst = (cha_inst & CHA_ID_LS_MD_MASK) >> CHA_ID_LS_MD_SHIFT; 3540 ccha_inst = 0; 3541 ptha_inst = 0; 3542 3543 aes_rn = rd_reg32(&priv->ctrl->perfmon.cha_rev_ls) & 3544 CHA_ID_LS_AES_MASK; 3545 gcm_support = !(aes_vid == CHA_VER_VID_AES_LP && aes_rn < 8); 3546 } else { 3547 u32 aesa, mdha; 3548 3549 aesa = rd_reg32(&priv->ctrl->vreg.aesa); 3550 mdha = rd_reg32(&priv->ctrl->vreg.mdha); 3551 3552 aes_vid = (aesa & CHA_VER_VID_MASK) >> CHA_VER_VID_SHIFT; 3553 md_vid = (mdha & CHA_VER_VID_MASK) >> CHA_VER_VID_SHIFT; 3554 3555 des_inst = rd_reg32(&priv->ctrl->vreg.desa) & CHA_VER_NUM_MASK; 3556 aes_inst = aesa & CHA_VER_NUM_MASK; 3557 md_inst = mdha & CHA_VER_NUM_MASK; 3558 ccha_inst = rd_reg32(&priv->ctrl->vreg.ccha) & CHA_VER_NUM_MASK; 3559 ptha_inst = rd_reg32(&priv->ctrl->vreg.ptha) & CHA_VER_NUM_MASK; 3560 3561 gcm_support = aesa & CHA_VER_MISC_AES_GCM; 3562 } 3563 3564 /* If MD is present, limit digest size based on LP256 */ 3565 if (md_inst && md_vid == CHA_VER_VID_MD_LP256) 3566 md_limit = SHA256_DIGEST_SIZE; 3567 3568 for (i = 0; i < ARRAY_SIZE(driver_algs); i++) { 3569 struct caam_skcipher_alg *t_alg = driver_algs + i; 3570 u32 alg_sel = t_alg->caam.class1_alg_type & OP_ALG_ALGSEL_MASK; 3571 3572 /* Skip DES algorithms if not supported by device */ 3573 if (!des_inst && 3574 ((alg_sel == OP_ALG_ALGSEL_3DES) || 3575 (alg_sel == OP_ALG_ALGSEL_DES))) 3576 continue; 3577 3578 /* Skip AES algorithms if not supported by device */ 3579 if (!aes_inst && (alg_sel == OP_ALG_ALGSEL_AES)) 3580 continue; 3581 3582 /* 3583 * Check support for AES modes not available 3584 * on LP devices. 3585 */ 3586 if (aes_vid == CHA_VER_VID_AES_LP && 3587 (t_alg->caam.class1_alg_type & OP_ALG_AAI_MASK) == 3588 OP_ALG_AAI_XTS) 3589 continue; 3590 3591 caam_skcipher_alg_init(t_alg); 3592 3593 err = crypto_register_skcipher(&t_alg->skcipher); 3594 if (err) { 3595 pr_warn("%s alg registration failed\n", 3596 t_alg->skcipher.base.cra_driver_name); 3597 continue; 3598 } 3599 3600 t_alg->registered = true; 3601 registered = true; 3602 } 3603 3604 for (i = 0; i < ARRAY_SIZE(driver_aeads); i++) { 3605 struct caam_aead_alg *t_alg = driver_aeads + i; 3606 u32 c1_alg_sel = t_alg->caam.class1_alg_type & 3607 OP_ALG_ALGSEL_MASK; 3608 u32 c2_alg_sel = t_alg->caam.class2_alg_type & 3609 OP_ALG_ALGSEL_MASK; 3610 u32 alg_aai = t_alg->caam.class1_alg_type & OP_ALG_AAI_MASK; 3611 3612 /* Skip DES algorithms if not supported by device */ 3613 if (!des_inst && 3614 ((c1_alg_sel == OP_ALG_ALGSEL_3DES) || 3615 (c1_alg_sel == OP_ALG_ALGSEL_DES))) 3616 continue; 3617 3618 /* Skip AES algorithms if not supported by device */ 3619 if (!aes_inst && (c1_alg_sel == OP_ALG_ALGSEL_AES)) 3620 continue; 3621 3622 /* Skip CHACHA20 algorithms if not supported by device */ 3623 if (c1_alg_sel == OP_ALG_ALGSEL_CHACHA20 && !ccha_inst) 3624 continue; 3625 3626 /* Skip POLY1305 algorithms if not supported by device */ 3627 if (c2_alg_sel == OP_ALG_ALGSEL_POLY1305 && !ptha_inst) 3628 continue; 3629 3630 /* Skip GCM algorithms if not supported by device */ 3631 if (c1_alg_sel == OP_ALG_ALGSEL_AES && 3632 alg_aai == OP_ALG_AAI_GCM && !gcm_support) 3633 continue; 3634 3635 /* 3636 * Skip algorithms requiring message digests 3637 * if MD or MD size is not supported by device. 3638 */ 3639 if (is_mdha(c2_alg_sel) && 3640 (!md_inst || t_alg->aead.maxauthsize > md_limit)) 3641 continue; 3642 3643 caam_aead_alg_init(t_alg); 3644 3645 err = crypto_register_aead(&t_alg->aead); 3646 if (err) { 3647 pr_warn("%s alg registration failed\n", 3648 t_alg->aead.base.cra_driver_name); 3649 continue; 3650 } 3651 3652 t_alg->registered = true; 3653 registered = true; 3654 } 3655 3656 if (registered) 3657 pr_info("caam algorithms registered in /proc/crypto\n"); 3658 3659 return err; 3660 } 3661