1 /* 2 * caam - Freescale FSL CAAM support for crypto API 3 * 4 * Copyright 2008-2011 Freescale Semiconductor, Inc. 5 * Copyright 2016 NXP 6 * 7 * Based on talitos crypto API driver. 8 * 9 * relationship of job descriptors to shared descriptors (SteveC Dec 10 2008): 10 * 11 * --------------- --------------- 12 * | JobDesc #1 |-------------------->| ShareDesc | 13 * | *(packet 1) | | (PDB) | 14 * --------------- |------------->| (hashKey) | 15 * . | | (cipherKey) | 16 * . | |-------->| (operation) | 17 * --------------- | | --------------- 18 * | JobDesc #2 |------| | 19 * | *(packet 2) | | 20 * --------------- | 21 * . | 22 * . | 23 * --------------- | 24 * | JobDesc #3 |------------ 25 * | *(packet 3) | 26 * --------------- 27 * 28 * The SharedDesc never changes for a connection unless rekeyed, but 29 * each packet will likely be in a different place. So all we need 30 * to know to process the packet is where the input is, where the 31 * output goes, and what context we want to process with. Context is 32 * in the SharedDesc, packet references in the JobDesc. 33 * 34 * So, a job desc looks like: 35 * 36 * --------------------- 37 * | Header | 38 * | ShareDesc Pointer | 39 * | SEQ_OUT_PTR | 40 * | (output buffer) | 41 * | (output length) | 42 * | SEQ_IN_PTR | 43 * | (input buffer) | 44 * | (input length) | 45 * --------------------- 46 */ 47 48 #include "compat.h" 49 50 #include "regs.h" 51 #include "intern.h" 52 #include "desc_constr.h" 53 #include "jr.h" 54 #include "error.h" 55 #include "sg_sw_sec4.h" 56 #include "key_gen.h" 57 #include "caamalg_desc.h" 58 59 /* 60 * crypto alg 61 */ 62 #define CAAM_CRA_PRIORITY 3000 63 /* max key is sum of AES_MAX_KEY_SIZE, max split key size */ 64 #define CAAM_MAX_KEY_SIZE (AES_MAX_KEY_SIZE + \ 65 CTR_RFC3686_NONCE_SIZE + \ 66 SHA512_DIGEST_SIZE * 2) 67 68 #define AEAD_DESC_JOB_IO_LEN (DESC_JOB_IO_LEN + CAAM_CMD_SZ * 2) 69 #define GCM_DESC_JOB_IO_LEN (AEAD_DESC_JOB_IO_LEN + \ 70 CAAM_CMD_SZ * 4) 71 #define AUTHENC_DESC_JOB_IO_LEN (AEAD_DESC_JOB_IO_LEN + \ 72 CAAM_CMD_SZ * 5) 73 74 #define DESC_MAX_USED_BYTES (CAAM_DESC_BYTES_MAX - DESC_JOB_IO_LEN) 75 #define DESC_MAX_USED_LEN (DESC_MAX_USED_BYTES / CAAM_CMD_SZ) 76 77 #ifdef DEBUG 78 /* for print_hex_dumps with line references */ 79 #define debug(format, arg...) printk(format, arg) 80 #else 81 #define debug(format, arg...) 82 #endif 83 84 static struct list_head alg_list; 85 86 struct caam_alg_entry { 87 int class1_alg_type; 88 int class2_alg_type; 89 bool rfc3686; 90 bool geniv; 91 }; 92 93 struct caam_aead_alg { 94 struct aead_alg aead; 95 struct caam_alg_entry caam; 96 bool registered; 97 }; 98 99 /* 100 * per-session context 101 */ 102 struct caam_ctx { 103 u32 sh_desc_enc[DESC_MAX_USED_LEN]; 104 u32 sh_desc_dec[DESC_MAX_USED_LEN]; 105 u32 sh_desc_givenc[DESC_MAX_USED_LEN]; 106 u8 key[CAAM_MAX_KEY_SIZE]; 107 dma_addr_t sh_desc_enc_dma; 108 dma_addr_t sh_desc_dec_dma; 109 dma_addr_t sh_desc_givenc_dma; 110 dma_addr_t key_dma; 111 enum dma_data_direction dir; 112 struct device *jrdev; 113 struct alginfo adata; 114 struct alginfo cdata; 115 unsigned int authsize; 116 }; 117 118 static int aead_null_set_sh_desc(struct crypto_aead *aead) 119 { 120 struct caam_ctx *ctx = crypto_aead_ctx(aead); 121 struct device *jrdev = ctx->jrdev; 122 struct caam_drv_private *ctrlpriv = dev_get_drvdata(jrdev->parent); 123 u32 *desc; 124 int rem_bytes = CAAM_DESC_BYTES_MAX - AEAD_DESC_JOB_IO_LEN - 125 ctx->adata.keylen_pad; 126 127 /* 128 * Job Descriptor and Shared Descriptors 129 * must all fit into the 64-word Descriptor h/w Buffer 130 */ 131 if (rem_bytes >= DESC_AEAD_NULL_ENC_LEN) { 132 ctx->adata.key_inline = true; 133 ctx->adata.key_virt = ctx->key; 134 } else { 135 ctx->adata.key_inline = false; 136 ctx->adata.key_dma = ctx->key_dma; 137 } 138 139 /* aead_encrypt shared descriptor */ 140 desc = ctx->sh_desc_enc; 141 cnstr_shdsc_aead_null_encap(desc, &ctx->adata, ctx->authsize, 142 ctrlpriv->era); 143 dma_sync_single_for_device(jrdev, ctx->sh_desc_enc_dma, 144 desc_bytes(desc), ctx->dir); 145 146 /* 147 * Job Descriptor and Shared Descriptors 148 * must all fit into the 64-word Descriptor h/w Buffer 149 */ 150 if (rem_bytes >= DESC_AEAD_NULL_DEC_LEN) { 151 ctx->adata.key_inline = true; 152 ctx->adata.key_virt = ctx->key; 153 } else { 154 ctx->adata.key_inline = false; 155 ctx->adata.key_dma = ctx->key_dma; 156 } 157 158 /* aead_decrypt shared descriptor */ 159 desc = ctx->sh_desc_dec; 160 cnstr_shdsc_aead_null_decap(desc, &ctx->adata, ctx->authsize, 161 ctrlpriv->era); 162 dma_sync_single_for_device(jrdev, ctx->sh_desc_dec_dma, 163 desc_bytes(desc), ctx->dir); 164 165 return 0; 166 } 167 168 static int aead_set_sh_desc(struct crypto_aead *aead) 169 { 170 struct caam_aead_alg *alg = container_of(crypto_aead_alg(aead), 171 struct caam_aead_alg, aead); 172 unsigned int ivsize = crypto_aead_ivsize(aead); 173 struct caam_ctx *ctx = crypto_aead_ctx(aead); 174 struct device *jrdev = ctx->jrdev; 175 struct caam_drv_private *ctrlpriv = dev_get_drvdata(jrdev->parent); 176 u32 ctx1_iv_off = 0; 177 u32 *desc, *nonce = NULL; 178 u32 inl_mask; 179 unsigned int data_len[2]; 180 const bool ctr_mode = ((ctx->cdata.algtype & OP_ALG_AAI_MASK) == 181 OP_ALG_AAI_CTR_MOD128); 182 const bool is_rfc3686 = alg->caam.rfc3686; 183 184 if (!ctx->authsize) 185 return 0; 186 187 /* NULL encryption / decryption */ 188 if (!ctx->cdata.keylen) 189 return aead_null_set_sh_desc(aead); 190 191 /* 192 * AES-CTR needs to load IV in CONTEXT1 reg 193 * at an offset of 128bits (16bytes) 194 * CONTEXT1[255:128] = IV 195 */ 196 if (ctr_mode) 197 ctx1_iv_off = 16; 198 199 /* 200 * RFC3686 specific: 201 * CONTEXT1[255:128] = {NONCE, IV, COUNTER} 202 */ 203 if (is_rfc3686) { 204 ctx1_iv_off = 16 + CTR_RFC3686_NONCE_SIZE; 205 nonce = (u32 *)((void *)ctx->key + ctx->adata.keylen_pad + 206 ctx->cdata.keylen - CTR_RFC3686_NONCE_SIZE); 207 } 208 209 data_len[0] = ctx->adata.keylen_pad; 210 data_len[1] = ctx->cdata.keylen; 211 212 if (alg->caam.geniv) 213 goto skip_enc; 214 215 /* 216 * Job Descriptor and Shared Descriptors 217 * must all fit into the 64-word Descriptor h/w Buffer 218 */ 219 if (desc_inline_query(DESC_AEAD_ENC_LEN + 220 (is_rfc3686 ? DESC_AEAD_CTR_RFC3686_LEN : 0), 221 AUTHENC_DESC_JOB_IO_LEN, data_len, &inl_mask, 222 ARRAY_SIZE(data_len)) < 0) 223 return -EINVAL; 224 225 if (inl_mask & 1) 226 ctx->adata.key_virt = ctx->key; 227 else 228 ctx->adata.key_dma = ctx->key_dma; 229 230 if (inl_mask & 2) 231 ctx->cdata.key_virt = ctx->key + ctx->adata.keylen_pad; 232 else 233 ctx->cdata.key_dma = ctx->key_dma + ctx->adata.keylen_pad; 234 235 ctx->adata.key_inline = !!(inl_mask & 1); 236 ctx->cdata.key_inline = !!(inl_mask & 2); 237 238 /* aead_encrypt shared descriptor */ 239 desc = ctx->sh_desc_enc; 240 cnstr_shdsc_aead_encap(desc, &ctx->cdata, &ctx->adata, ivsize, 241 ctx->authsize, is_rfc3686, nonce, ctx1_iv_off, 242 false, ctrlpriv->era); 243 dma_sync_single_for_device(jrdev, ctx->sh_desc_enc_dma, 244 desc_bytes(desc), ctx->dir); 245 246 skip_enc: 247 /* 248 * Job Descriptor and Shared Descriptors 249 * must all fit into the 64-word Descriptor h/w Buffer 250 */ 251 if (desc_inline_query(DESC_AEAD_DEC_LEN + 252 (is_rfc3686 ? DESC_AEAD_CTR_RFC3686_LEN : 0), 253 AUTHENC_DESC_JOB_IO_LEN, data_len, &inl_mask, 254 ARRAY_SIZE(data_len)) < 0) 255 return -EINVAL; 256 257 if (inl_mask & 1) 258 ctx->adata.key_virt = ctx->key; 259 else 260 ctx->adata.key_dma = ctx->key_dma; 261 262 if (inl_mask & 2) 263 ctx->cdata.key_virt = ctx->key + ctx->adata.keylen_pad; 264 else 265 ctx->cdata.key_dma = ctx->key_dma + ctx->adata.keylen_pad; 266 267 ctx->adata.key_inline = !!(inl_mask & 1); 268 ctx->cdata.key_inline = !!(inl_mask & 2); 269 270 /* aead_decrypt shared descriptor */ 271 desc = ctx->sh_desc_dec; 272 cnstr_shdsc_aead_decap(desc, &ctx->cdata, &ctx->adata, ivsize, 273 ctx->authsize, alg->caam.geniv, is_rfc3686, 274 nonce, ctx1_iv_off, false, ctrlpriv->era); 275 dma_sync_single_for_device(jrdev, ctx->sh_desc_dec_dma, 276 desc_bytes(desc), ctx->dir); 277 278 if (!alg->caam.geniv) 279 goto skip_givenc; 280 281 /* 282 * Job Descriptor and Shared Descriptors 283 * must all fit into the 64-word Descriptor h/w Buffer 284 */ 285 if (desc_inline_query(DESC_AEAD_GIVENC_LEN + 286 (is_rfc3686 ? DESC_AEAD_CTR_RFC3686_LEN : 0), 287 AUTHENC_DESC_JOB_IO_LEN, data_len, &inl_mask, 288 ARRAY_SIZE(data_len)) < 0) 289 return -EINVAL; 290 291 if (inl_mask & 1) 292 ctx->adata.key_virt = ctx->key; 293 else 294 ctx->adata.key_dma = ctx->key_dma; 295 296 if (inl_mask & 2) 297 ctx->cdata.key_virt = ctx->key + ctx->adata.keylen_pad; 298 else 299 ctx->cdata.key_dma = ctx->key_dma + ctx->adata.keylen_pad; 300 301 ctx->adata.key_inline = !!(inl_mask & 1); 302 ctx->cdata.key_inline = !!(inl_mask & 2); 303 304 /* aead_givencrypt shared descriptor */ 305 desc = ctx->sh_desc_enc; 306 cnstr_shdsc_aead_givencap(desc, &ctx->cdata, &ctx->adata, ivsize, 307 ctx->authsize, is_rfc3686, nonce, 308 ctx1_iv_off, false, ctrlpriv->era); 309 dma_sync_single_for_device(jrdev, ctx->sh_desc_enc_dma, 310 desc_bytes(desc), ctx->dir); 311 312 skip_givenc: 313 return 0; 314 } 315 316 static int aead_setauthsize(struct crypto_aead *authenc, 317 unsigned int authsize) 318 { 319 struct caam_ctx *ctx = crypto_aead_ctx(authenc); 320 321 ctx->authsize = authsize; 322 aead_set_sh_desc(authenc); 323 324 return 0; 325 } 326 327 static int gcm_set_sh_desc(struct crypto_aead *aead) 328 { 329 struct caam_ctx *ctx = crypto_aead_ctx(aead); 330 struct device *jrdev = ctx->jrdev; 331 u32 *desc; 332 int rem_bytes = CAAM_DESC_BYTES_MAX - GCM_DESC_JOB_IO_LEN - 333 ctx->cdata.keylen; 334 335 if (!ctx->cdata.keylen || !ctx->authsize) 336 return 0; 337 338 /* 339 * AES GCM encrypt shared descriptor 340 * Job Descriptor and Shared Descriptor 341 * must fit into the 64-word Descriptor h/w Buffer 342 */ 343 if (rem_bytes >= DESC_GCM_ENC_LEN) { 344 ctx->cdata.key_inline = true; 345 ctx->cdata.key_virt = ctx->key; 346 } else { 347 ctx->cdata.key_inline = false; 348 ctx->cdata.key_dma = ctx->key_dma; 349 } 350 351 desc = ctx->sh_desc_enc; 352 cnstr_shdsc_gcm_encap(desc, &ctx->cdata, ctx->authsize); 353 dma_sync_single_for_device(jrdev, ctx->sh_desc_enc_dma, 354 desc_bytes(desc), ctx->dir); 355 356 /* 357 * Job Descriptor and Shared Descriptors 358 * must all fit into the 64-word Descriptor h/w Buffer 359 */ 360 if (rem_bytes >= DESC_GCM_DEC_LEN) { 361 ctx->cdata.key_inline = true; 362 ctx->cdata.key_virt = ctx->key; 363 } else { 364 ctx->cdata.key_inline = false; 365 ctx->cdata.key_dma = ctx->key_dma; 366 } 367 368 desc = ctx->sh_desc_dec; 369 cnstr_shdsc_gcm_decap(desc, &ctx->cdata, ctx->authsize); 370 dma_sync_single_for_device(jrdev, ctx->sh_desc_dec_dma, 371 desc_bytes(desc), ctx->dir); 372 373 return 0; 374 } 375 376 static int gcm_setauthsize(struct crypto_aead *authenc, unsigned int authsize) 377 { 378 struct caam_ctx *ctx = crypto_aead_ctx(authenc); 379 380 ctx->authsize = authsize; 381 gcm_set_sh_desc(authenc); 382 383 return 0; 384 } 385 386 static int rfc4106_set_sh_desc(struct crypto_aead *aead) 387 { 388 struct caam_ctx *ctx = crypto_aead_ctx(aead); 389 struct device *jrdev = ctx->jrdev; 390 u32 *desc; 391 int rem_bytes = CAAM_DESC_BYTES_MAX - GCM_DESC_JOB_IO_LEN - 392 ctx->cdata.keylen; 393 394 if (!ctx->cdata.keylen || !ctx->authsize) 395 return 0; 396 397 /* 398 * RFC4106 encrypt shared descriptor 399 * Job Descriptor and Shared Descriptor 400 * must fit into the 64-word Descriptor h/w Buffer 401 */ 402 if (rem_bytes >= DESC_RFC4106_ENC_LEN) { 403 ctx->cdata.key_inline = true; 404 ctx->cdata.key_virt = ctx->key; 405 } else { 406 ctx->cdata.key_inline = false; 407 ctx->cdata.key_dma = ctx->key_dma; 408 } 409 410 desc = ctx->sh_desc_enc; 411 cnstr_shdsc_rfc4106_encap(desc, &ctx->cdata, ctx->authsize); 412 dma_sync_single_for_device(jrdev, ctx->sh_desc_enc_dma, 413 desc_bytes(desc), ctx->dir); 414 415 /* 416 * Job Descriptor and Shared Descriptors 417 * must all fit into the 64-word Descriptor h/w Buffer 418 */ 419 if (rem_bytes >= DESC_RFC4106_DEC_LEN) { 420 ctx->cdata.key_inline = true; 421 ctx->cdata.key_virt = ctx->key; 422 } else { 423 ctx->cdata.key_inline = false; 424 ctx->cdata.key_dma = ctx->key_dma; 425 } 426 427 desc = ctx->sh_desc_dec; 428 cnstr_shdsc_rfc4106_decap(desc, &ctx->cdata, ctx->authsize); 429 dma_sync_single_for_device(jrdev, ctx->sh_desc_dec_dma, 430 desc_bytes(desc), ctx->dir); 431 432 return 0; 433 } 434 435 static int rfc4106_setauthsize(struct crypto_aead *authenc, 436 unsigned int authsize) 437 { 438 struct caam_ctx *ctx = crypto_aead_ctx(authenc); 439 440 ctx->authsize = authsize; 441 rfc4106_set_sh_desc(authenc); 442 443 return 0; 444 } 445 446 static int rfc4543_set_sh_desc(struct crypto_aead *aead) 447 { 448 struct caam_ctx *ctx = crypto_aead_ctx(aead); 449 struct device *jrdev = ctx->jrdev; 450 u32 *desc; 451 int rem_bytes = CAAM_DESC_BYTES_MAX - GCM_DESC_JOB_IO_LEN - 452 ctx->cdata.keylen; 453 454 if (!ctx->cdata.keylen || !ctx->authsize) 455 return 0; 456 457 /* 458 * RFC4543 encrypt shared descriptor 459 * Job Descriptor and Shared Descriptor 460 * must fit into the 64-word Descriptor h/w Buffer 461 */ 462 if (rem_bytes >= DESC_RFC4543_ENC_LEN) { 463 ctx->cdata.key_inline = true; 464 ctx->cdata.key_virt = ctx->key; 465 } else { 466 ctx->cdata.key_inline = false; 467 ctx->cdata.key_dma = ctx->key_dma; 468 } 469 470 desc = ctx->sh_desc_enc; 471 cnstr_shdsc_rfc4543_encap(desc, &ctx->cdata, ctx->authsize); 472 dma_sync_single_for_device(jrdev, ctx->sh_desc_enc_dma, 473 desc_bytes(desc), ctx->dir); 474 475 /* 476 * Job Descriptor and Shared Descriptors 477 * must all fit into the 64-word Descriptor h/w Buffer 478 */ 479 if (rem_bytes >= DESC_RFC4543_DEC_LEN) { 480 ctx->cdata.key_inline = true; 481 ctx->cdata.key_virt = ctx->key; 482 } else { 483 ctx->cdata.key_inline = false; 484 ctx->cdata.key_dma = ctx->key_dma; 485 } 486 487 desc = ctx->sh_desc_dec; 488 cnstr_shdsc_rfc4543_decap(desc, &ctx->cdata, ctx->authsize); 489 dma_sync_single_for_device(jrdev, ctx->sh_desc_dec_dma, 490 desc_bytes(desc), ctx->dir); 491 492 return 0; 493 } 494 495 static int rfc4543_setauthsize(struct crypto_aead *authenc, 496 unsigned int authsize) 497 { 498 struct caam_ctx *ctx = crypto_aead_ctx(authenc); 499 500 ctx->authsize = authsize; 501 rfc4543_set_sh_desc(authenc); 502 503 return 0; 504 } 505 506 static int aead_setkey(struct crypto_aead *aead, 507 const u8 *key, unsigned int keylen) 508 { 509 struct caam_ctx *ctx = crypto_aead_ctx(aead); 510 struct device *jrdev = ctx->jrdev; 511 struct caam_drv_private *ctrlpriv = dev_get_drvdata(jrdev->parent); 512 struct crypto_authenc_keys keys; 513 int ret = 0; 514 515 if (crypto_authenc_extractkeys(&keys, key, keylen) != 0) 516 goto badkey; 517 518 #ifdef DEBUG 519 printk(KERN_ERR "keylen %d enckeylen %d authkeylen %d\n", 520 keys.authkeylen + keys.enckeylen, keys.enckeylen, 521 keys.authkeylen); 522 print_hex_dump(KERN_ERR, "key in @"__stringify(__LINE__)": ", 523 DUMP_PREFIX_ADDRESS, 16, 4, key, keylen, 1); 524 #endif 525 526 /* 527 * If DKP is supported, use it in the shared descriptor to generate 528 * the split key. 529 */ 530 if (ctrlpriv->era >= 6) { 531 ctx->adata.keylen = keys.authkeylen; 532 ctx->adata.keylen_pad = split_key_len(ctx->adata.algtype & 533 OP_ALG_ALGSEL_MASK); 534 535 if (ctx->adata.keylen_pad + keys.enckeylen > CAAM_MAX_KEY_SIZE) 536 goto badkey; 537 538 memcpy(ctx->key, keys.authkey, keys.authkeylen); 539 memcpy(ctx->key + ctx->adata.keylen_pad, keys.enckey, 540 keys.enckeylen); 541 dma_sync_single_for_device(jrdev, ctx->key_dma, 542 ctx->adata.keylen_pad + 543 keys.enckeylen, ctx->dir); 544 goto skip_split_key; 545 } 546 547 ret = gen_split_key(ctx->jrdev, ctx->key, &ctx->adata, keys.authkey, 548 keys.authkeylen, CAAM_MAX_KEY_SIZE - 549 keys.enckeylen); 550 if (ret) { 551 goto badkey; 552 } 553 554 /* postpend encryption key to auth split key */ 555 memcpy(ctx->key + ctx->adata.keylen_pad, keys.enckey, keys.enckeylen); 556 dma_sync_single_for_device(jrdev, ctx->key_dma, ctx->adata.keylen_pad + 557 keys.enckeylen, ctx->dir); 558 #ifdef DEBUG 559 print_hex_dump(KERN_ERR, "ctx.key@"__stringify(__LINE__)": ", 560 DUMP_PREFIX_ADDRESS, 16, 4, ctx->key, 561 ctx->adata.keylen_pad + keys.enckeylen, 1); 562 #endif 563 564 skip_split_key: 565 ctx->cdata.keylen = keys.enckeylen; 566 return aead_set_sh_desc(aead); 567 badkey: 568 crypto_aead_set_flags(aead, CRYPTO_TFM_RES_BAD_KEY_LEN); 569 return -EINVAL; 570 } 571 572 static int gcm_setkey(struct crypto_aead *aead, 573 const u8 *key, unsigned int keylen) 574 { 575 struct caam_ctx *ctx = crypto_aead_ctx(aead); 576 struct device *jrdev = ctx->jrdev; 577 578 #ifdef DEBUG 579 print_hex_dump(KERN_ERR, "key in @"__stringify(__LINE__)": ", 580 DUMP_PREFIX_ADDRESS, 16, 4, key, keylen, 1); 581 #endif 582 583 memcpy(ctx->key, key, keylen); 584 dma_sync_single_for_device(jrdev, ctx->key_dma, keylen, ctx->dir); 585 ctx->cdata.keylen = keylen; 586 587 return gcm_set_sh_desc(aead); 588 } 589 590 static int rfc4106_setkey(struct crypto_aead *aead, 591 const u8 *key, unsigned int keylen) 592 { 593 struct caam_ctx *ctx = crypto_aead_ctx(aead); 594 struct device *jrdev = ctx->jrdev; 595 596 if (keylen < 4) 597 return -EINVAL; 598 599 #ifdef DEBUG 600 print_hex_dump(KERN_ERR, "key in @"__stringify(__LINE__)": ", 601 DUMP_PREFIX_ADDRESS, 16, 4, key, keylen, 1); 602 #endif 603 604 memcpy(ctx->key, key, keylen); 605 606 /* 607 * The last four bytes of the key material are used as the salt value 608 * in the nonce. Update the AES key length. 609 */ 610 ctx->cdata.keylen = keylen - 4; 611 dma_sync_single_for_device(jrdev, ctx->key_dma, ctx->cdata.keylen, 612 ctx->dir); 613 return rfc4106_set_sh_desc(aead); 614 } 615 616 static int rfc4543_setkey(struct crypto_aead *aead, 617 const u8 *key, unsigned int keylen) 618 { 619 struct caam_ctx *ctx = crypto_aead_ctx(aead); 620 struct device *jrdev = ctx->jrdev; 621 622 if (keylen < 4) 623 return -EINVAL; 624 625 #ifdef DEBUG 626 print_hex_dump(KERN_ERR, "key in @"__stringify(__LINE__)": ", 627 DUMP_PREFIX_ADDRESS, 16, 4, key, keylen, 1); 628 #endif 629 630 memcpy(ctx->key, key, keylen); 631 632 /* 633 * The last four bytes of the key material are used as the salt value 634 * in the nonce. Update the AES key length. 635 */ 636 ctx->cdata.keylen = keylen - 4; 637 dma_sync_single_for_device(jrdev, ctx->key_dma, ctx->cdata.keylen, 638 ctx->dir); 639 return rfc4543_set_sh_desc(aead); 640 } 641 642 static int ablkcipher_setkey(struct crypto_ablkcipher *ablkcipher, 643 const u8 *key, unsigned int keylen) 644 { 645 struct caam_ctx *ctx = crypto_ablkcipher_ctx(ablkcipher); 646 struct crypto_tfm *tfm = crypto_ablkcipher_tfm(ablkcipher); 647 const char *alg_name = crypto_tfm_alg_name(tfm); 648 struct device *jrdev = ctx->jrdev; 649 unsigned int ivsize = crypto_ablkcipher_ivsize(ablkcipher); 650 u32 *desc; 651 u32 ctx1_iv_off = 0; 652 const bool ctr_mode = ((ctx->cdata.algtype & OP_ALG_AAI_MASK) == 653 OP_ALG_AAI_CTR_MOD128); 654 const bool is_rfc3686 = (ctr_mode && 655 (strstr(alg_name, "rfc3686") != NULL)); 656 657 #ifdef DEBUG 658 print_hex_dump(KERN_ERR, "key in @"__stringify(__LINE__)": ", 659 DUMP_PREFIX_ADDRESS, 16, 4, key, keylen, 1); 660 #endif 661 /* 662 * AES-CTR needs to load IV in CONTEXT1 reg 663 * at an offset of 128bits (16bytes) 664 * CONTEXT1[255:128] = IV 665 */ 666 if (ctr_mode) 667 ctx1_iv_off = 16; 668 669 /* 670 * RFC3686 specific: 671 * | CONTEXT1[255:128] = {NONCE, IV, COUNTER} 672 * | *key = {KEY, NONCE} 673 */ 674 if (is_rfc3686) { 675 ctx1_iv_off = 16 + CTR_RFC3686_NONCE_SIZE; 676 keylen -= CTR_RFC3686_NONCE_SIZE; 677 } 678 679 ctx->cdata.keylen = keylen; 680 ctx->cdata.key_virt = key; 681 ctx->cdata.key_inline = true; 682 683 /* ablkcipher_encrypt shared descriptor */ 684 desc = ctx->sh_desc_enc; 685 cnstr_shdsc_ablkcipher_encap(desc, &ctx->cdata, ivsize, is_rfc3686, 686 ctx1_iv_off); 687 dma_sync_single_for_device(jrdev, ctx->sh_desc_enc_dma, 688 desc_bytes(desc), ctx->dir); 689 690 /* ablkcipher_decrypt shared descriptor */ 691 desc = ctx->sh_desc_dec; 692 cnstr_shdsc_ablkcipher_decap(desc, &ctx->cdata, ivsize, is_rfc3686, 693 ctx1_iv_off); 694 dma_sync_single_for_device(jrdev, ctx->sh_desc_dec_dma, 695 desc_bytes(desc), ctx->dir); 696 697 /* ablkcipher_givencrypt shared descriptor */ 698 desc = ctx->sh_desc_givenc; 699 cnstr_shdsc_ablkcipher_givencap(desc, &ctx->cdata, ivsize, is_rfc3686, 700 ctx1_iv_off); 701 dma_sync_single_for_device(jrdev, ctx->sh_desc_givenc_dma, 702 desc_bytes(desc), ctx->dir); 703 704 return 0; 705 } 706 707 static int xts_ablkcipher_setkey(struct crypto_ablkcipher *ablkcipher, 708 const u8 *key, unsigned int keylen) 709 { 710 struct caam_ctx *ctx = crypto_ablkcipher_ctx(ablkcipher); 711 struct device *jrdev = ctx->jrdev; 712 u32 *desc; 713 714 if (keylen != 2 * AES_MIN_KEY_SIZE && keylen != 2 * AES_MAX_KEY_SIZE) { 715 crypto_ablkcipher_set_flags(ablkcipher, 716 CRYPTO_TFM_RES_BAD_KEY_LEN); 717 dev_err(jrdev, "key size mismatch\n"); 718 return -EINVAL; 719 } 720 721 ctx->cdata.keylen = keylen; 722 ctx->cdata.key_virt = key; 723 ctx->cdata.key_inline = true; 724 725 /* xts_ablkcipher_encrypt shared descriptor */ 726 desc = ctx->sh_desc_enc; 727 cnstr_shdsc_xts_ablkcipher_encap(desc, &ctx->cdata); 728 dma_sync_single_for_device(jrdev, ctx->sh_desc_enc_dma, 729 desc_bytes(desc), ctx->dir); 730 731 /* xts_ablkcipher_decrypt shared descriptor */ 732 desc = ctx->sh_desc_dec; 733 cnstr_shdsc_xts_ablkcipher_decap(desc, &ctx->cdata); 734 dma_sync_single_for_device(jrdev, ctx->sh_desc_dec_dma, 735 desc_bytes(desc), ctx->dir); 736 737 return 0; 738 } 739 740 /* 741 * aead_edesc - s/w-extended aead descriptor 742 * @src_nents: number of segments in input s/w scatterlist 743 * @dst_nents: number of segments in output s/w scatterlist 744 * @sec4_sg_bytes: length of dma mapped sec4_sg space 745 * @sec4_sg_dma: bus physical mapped address of h/w link table 746 * @sec4_sg: pointer to h/w link table 747 * @hw_desc: the h/w job descriptor followed by any referenced link tables 748 */ 749 struct aead_edesc { 750 int src_nents; 751 int dst_nents; 752 int sec4_sg_bytes; 753 dma_addr_t sec4_sg_dma; 754 struct sec4_sg_entry *sec4_sg; 755 u32 hw_desc[]; 756 }; 757 758 /* 759 * ablkcipher_edesc - s/w-extended ablkcipher descriptor 760 * @src_nents: number of segments in input s/w scatterlist 761 * @dst_nents: number of segments in output s/w scatterlist 762 * @iv_dma: dma address of iv for checking continuity and link table 763 * @sec4_sg_bytes: length of dma mapped sec4_sg space 764 * @sec4_sg_dma: bus physical mapped address of h/w link table 765 * @sec4_sg: pointer to h/w link table 766 * @hw_desc: the h/w job descriptor followed by any referenced link tables 767 */ 768 struct ablkcipher_edesc { 769 int src_nents; 770 int dst_nents; 771 dma_addr_t iv_dma; 772 int sec4_sg_bytes; 773 dma_addr_t sec4_sg_dma; 774 struct sec4_sg_entry *sec4_sg; 775 u32 hw_desc[0]; 776 }; 777 778 static void caam_unmap(struct device *dev, struct scatterlist *src, 779 struct scatterlist *dst, int src_nents, 780 int dst_nents, 781 dma_addr_t iv_dma, int ivsize, dma_addr_t sec4_sg_dma, 782 int sec4_sg_bytes) 783 { 784 if (dst != src) { 785 if (src_nents) 786 dma_unmap_sg(dev, src, src_nents, DMA_TO_DEVICE); 787 dma_unmap_sg(dev, dst, dst_nents, DMA_FROM_DEVICE); 788 } else { 789 dma_unmap_sg(dev, src, src_nents, DMA_BIDIRECTIONAL); 790 } 791 792 if (iv_dma) 793 dma_unmap_single(dev, iv_dma, ivsize, DMA_TO_DEVICE); 794 if (sec4_sg_bytes) 795 dma_unmap_single(dev, sec4_sg_dma, sec4_sg_bytes, 796 DMA_TO_DEVICE); 797 } 798 799 static void aead_unmap(struct device *dev, 800 struct aead_edesc *edesc, 801 struct aead_request *req) 802 { 803 caam_unmap(dev, req->src, req->dst, 804 edesc->src_nents, edesc->dst_nents, 0, 0, 805 edesc->sec4_sg_dma, edesc->sec4_sg_bytes); 806 } 807 808 static void ablkcipher_unmap(struct device *dev, 809 struct ablkcipher_edesc *edesc, 810 struct ablkcipher_request *req) 811 { 812 struct crypto_ablkcipher *ablkcipher = crypto_ablkcipher_reqtfm(req); 813 int ivsize = crypto_ablkcipher_ivsize(ablkcipher); 814 815 caam_unmap(dev, req->src, req->dst, 816 edesc->src_nents, edesc->dst_nents, 817 edesc->iv_dma, ivsize, 818 edesc->sec4_sg_dma, edesc->sec4_sg_bytes); 819 } 820 821 static void aead_encrypt_done(struct device *jrdev, u32 *desc, u32 err, 822 void *context) 823 { 824 struct aead_request *req = context; 825 struct aead_edesc *edesc; 826 827 #ifdef DEBUG 828 dev_err(jrdev, "%s %d: err 0x%x\n", __func__, __LINE__, err); 829 #endif 830 831 edesc = container_of(desc, struct aead_edesc, hw_desc[0]); 832 833 if (err) 834 caam_jr_strstatus(jrdev, err); 835 836 aead_unmap(jrdev, edesc, req); 837 838 kfree(edesc); 839 840 aead_request_complete(req, err); 841 } 842 843 static void aead_decrypt_done(struct device *jrdev, u32 *desc, u32 err, 844 void *context) 845 { 846 struct aead_request *req = context; 847 struct aead_edesc *edesc; 848 849 #ifdef DEBUG 850 dev_err(jrdev, "%s %d: err 0x%x\n", __func__, __LINE__, err); 851 #endif 852 853 edesc = container_of(desc, struct aead_edesc, hw_desc[0]); 854 855 if (err) 856 caam_jr_strstatus(jrdev, err); 857 858 aead_unmap(jrdev, edesc, req); 859 860 /* 861 * verify hw auth check passed else return -EBADMSG 862 */ 863 if ((err & JRSTA_CCBERR_ERRID_MASK) == JRSTA_CCBERR_ERRID_ICVCHK) 864 err = -EBADMSG; 865 866 kfree(edesc); 867 868 aead_request_complete(req, err); 869 } 870 871 static void ablkcipher_encrypt_done(struct device *jrdev, u32 *desc, u32 err, 872 void *context) 873 { 874 struct ablkcipher_request *req = context; 875 struct ablkcipher_edesc *edesc; 876 struct crypto_ablkcipher *ablkcipher = crypto_ablkcipher_reqtfm(req); 877 int ivsize = crypto_ablkcipher_ivsize(ablkcipher); 878 879 #ifdef DEBUG 880 dev_err(jrdev, "%s %d: err 0x%x\n", __func__, __LINE__, err); 881 #endif 882 883 edesc = container_of(desc, struct ablkcipher_edesc, hw_desc[0]); 884 885 if (err) 886 caam_jr_strstatus(jrdev, err); 887 888 #ifdef DEBUG 889 print_hex_dump(KERN_ERR, "dstiv @"__stringify(__LINE__)": ", 890 DUMP_PREFIX_ADDRESS, 16, 4, req->info, 891 edesc->src_nents > 1 ? 100 : ivsize, 1); 892 #endif 893 caam_dump_sg(KERN_ERR, "dst @" __stringify(__LINE__)": ", 894 DUMP_PREFIX_ADDRESS, 16, 4, req->dst, 895 edesc->dst_nents > 1 ? 100 : req->nbytes, 1); 896 897 ablkcipher_unmap(jrdev, edesc, req); 898 899 /* 900 * The crypto API expects us to set the IV (req->info) to the last 901 * ciphertext block. This is used e.g. by the CTS mode. 902 */ 903 scatterwalk_map_and_copy(req->info, req->dst, req->nbytes - ivsize, 904 ivsize, 0); 905 906 kfree(edesc); 907 908 ablkcipher_request_complete(req, err); 909 } 910 911 static void ablkcipher_decrypt_done(struct device *jrdev, u32 *desc, u32 err, 912 void *context) 913 { 914 struct ablkcipher_request *req = context; 915 struct ablkcipher_edesc *edesc; 916 struct crypto_ablkcipher *ablkcipher = crypto_ablkcipher_reqtfm(req); 917 int ivsize = crypto_ablkcipher_ivsize(ablkcipher); 918 919 #ifdef DEBUG 920 dev_err(jrdev, "%s %d: err 0x%x\n", __func__, __LINE__, err); 921 #endif 922 923 edesc = container_of(desc, struct ablkcipher_edesc, hw_desc[0]); 924 if (err) 925 caam_jr_strstatus(jrdev, err); 926 927 #ifdef DEBUG 928 print_hex_dump(KERN_ERR, "dstiv @"__stringify(__LINE__)": ", 929 DUMP_PREFIX_ADDRESS, 16, 4, req->info, 930 ivsize, 1); 931 #endif 932 caam_dump_sg(KERN_ERR, "dst @" __stringify(__LINE__)": ", 933 DUMP_PREFIX_ADDRESS, 16, 4, req->dst, 934 edesc->dst_nents > 1 ? 100 : req->nbytes, 1); 935 936 ablkcipher_unmap(jrdev, edesc, req); 937 938 /* 939 * The crypto API expects us to set the IV (req->info) to the last 940 * ciphertext block. 941 */ 942 scatterwalk_map_and_copy(req->info, req->src, req->nbytes - ivsize, 943 ivsize, 0); 944 945 kfree(edesc); 946 947 ablkcipher_request_complete(req, err); 948 } 949 950 /* 951 * Fill in aead job descriptor 952 */ 953 static void init_aead_job(struct aead_request *req, 954 struct aead_edesc *edesc, 955 bool all_contig, bool encrypt) 956 { 957 struct crypto_aead *aead = crypto_aead_reqtfm(req); 958 struct caam_ctx *ctx = crypto_aead_ctx(aead); 959 int authsize = ctx->authsize; 960 u32 *desc = edesc->hw_desc; 961 u32 out_options, in_options; 962 dma_addr_t dst_dma, src_dma; 963 int len, sec4_sg_index = 0; 964 dma_addr_t ptr; 965 u32 *sh_desc; 966 967 sh_desc = encrypt ? ctx->sh_desc_enc : ctx->sh_desc_dec; 968 ptr = encrypt ? ctx->sh_desc_enc_dma : ctx->sh_desc_dec_dma; 969 970 len = desc_len(sh_desc); 971 init_job_desc_shared(desc, ptr, len, HDR_SHARE_DEFER | HDR_REVERSE); 972 973 if (all_contig) { 974 src_dma = edesc->src_nents ? sg_dma_address(req->src) : 0; 975 in_options = 0; 976 } else { 977 src_dma = edesc->sec4_sg_dma; 978 sec4_sg_index += edesc->src_nents; 979 in_options = LDST_SGF; 980 } 981 982 append_seq_in_ptr(desc, src_dma, req->assoclen + req->cryptlen, 983 in_options); 984 985 dst_dma = src_dma; 986 out_options = in_options; 987 988 if (unlikely(req->src != req->dst)) { 989 if (edesc->dst_nents == 1) { 990 dst_dma = sg_dma_address(req->dst); 991 } else { 992 dst_dma = edesc->sec4_sg_dma + 993 sec4_sg_index * 994 sizeof(struct sec4_sg_entry); 995 out_options = LDST_SGF; 996 } 997 } 998 999 if (encrypt) 1000 append_seq_out_ptr(desc, dst_dma, 1001 req->assoclen + req->cryptlen + authsize, 1002 out_options); 1003 else 1004 append_seq_out_ptr(desc, dst_dma, 1005 req->assoclen + req->cryptlen - authsize, 1006 out_options); 1007 } 1008 1009 static void init_gcm_job(struct aead_request *req, 1010 struct aead_edesc *edesc, 1011 bool all_contig, bool encrypt) 1012 { 1013 struct crypto_aead *aead = crypto_aead_reqtfm(req); 1014 struct caam_ctx *ctx = crypto_aead_ctx(aead); 1015 unsigned int ivsize = crypto_aead_ivsize(aead); 1016 u32 *desc = edesc->hw_desc; 1017 bool generic_gcm = (ivsize == GCM_AES_IV_SIZE); 1018 unsigned int last; 1019 1020 init_aead_job(req, edesc, all_contig, encrypt); 1021 append_math_add_imm_u32(desc, REG3, ZERO, IMM, req->assoclen); 1022 1023 /* BUG This should not be specific to generic GCM. */ 1024 last = 0; 1025 if (encrypt && generic_gcm && !(req->assoclen + req->cryptlen)) 1026 last = FIFOLD_TYPE_LAST1; 1027 1028 /* Read GCM IV */ 1029 append_cmd(desc, CMD_FIFO_LOAD | FIFOLD_CLASS_CLASS1 | IMMEDIATE | 1030 FIFOLD_TYPE_IV | FIFOLD_TYPE_FLUSH1 | GCM_AES_IV_SIZE | last); 1031 /* Append Salt */ 1032 if (!generic_gcm) 1033 append_data(desc, ctx->key + ctx->cdata.keylen, 4); 1034 /* Append IV */ 1035 append_data(desc, req->iv, ivsize); 1036 /* End of blank commands */ 1037 } 1038 1039 static void init_authenc_job(struct aead_request *req, 1040 struct aead_edesc *edesc, 1041 bool all_contig, bool encrypt) 1042 { 1043 struct crypto_aead *aead = crypto_aead_reqtfm(req); 1044 struct caam_aead_alg *alg = container_of(crypto_aead_alg(aead), 1045 struct caam_aead_alg, aead); 1046 unsigned int ivsize = crypto_aead_ivsize(aead); 1047 struct caam_ctx *ctx = crypto_aead_ctx(aead); 1048 struct caam_drv_private *ctrlpriv = dev_get_drvdata(ctx->jrdev->parent); 1049 const bool ctr_mode = ((ctx->cdata.algtype & OP_ALG_AAI_MASK) == 1050 OP_ALG_AAI_CTR_MOD128); 1051 const bool is_rfc3686 = alg->caam.rfc3686; 1052 u32 *desc = edesc->hw_desc; 1053 u32 ivoffset = 0; 1054 1055 /* 1056 * AES-CTR needs to load IV in CONTEXT1 reg 1057 * at an offset of 128bits (16bytes) 1058 * CONTEXT1[255:128] = IV 1059 */ 1060 if (ctr_mode) 1061 ivoffset = 16; 1062 1063 /* 1064 * RFC3686 specific: 1065 * CONTEXT1[255:128] = {NONCE, IV, COUNTER} 1066 */ 1067 if (is_rfc3686) 1068 ivoffset = 16 + CTR_RFC3686_NONCE_SIZE; 1069 1070 init_aead_job(req, edesc, all_contig, encrypt); 1071 1072 /* 1073 * {REG3, DPOVRD} = assoclen, depending on whether MATH command supports 1074 * having DPOVRD as destination. 1075 */ 1076 if (ctrlpriv->era < 3) 1077 append_math_add_imm_u32(desc, REG3, ZERO, IMM, req->assoclen); 1078 else 1079 append_math_add_imm_u32(desc, DPOVRD, ZERO, IMM, req->assoclen); 1080 1081 if (ivsize && ((is_rfc3686 && encrypt) || !alg->caam.geniv)) 1082 append_load_as_imm(desc, req->iv, ivsize, 1083 LDST_CLASS_1_CCB | 1084 LDST_SRCDST_BYTE_CONTEXT | 1085 (ivoffset << LDST_OFFSET_SHIFT)); 1086 } 1087 1088 /* 1089 * Fill in ablkcipher job descriptor 1090 */ 1091 static void init_ablkcipher_job(u32 *sh_desc, dma_addr_t ptr, 1092 struct ablkcipher_edesc *edesc, 1093 struct ablkcipher_request *req, 1094 bool iv_contig) 1095 { 1096 struct crypto_ablkcipher *ablkcipher = crypto_ablkcipher_reqtfm(req); 1097 int ivsize = crypto_ablkcipher_ivsize(ablkcipher); 1098 u32 *desc = edesc->hw_desc; 1099 u32 out_options = 0, in_options; 1100 dma_addr_t dst_dma, src_dma; 1101 int len, sec4_sg_index = 0; 1102 1103 #ifdef DEBUG 1104 print_hex_dump(KERN_ERR, "presciv@"__stringify(__LINE__)": ", 1105 DUMP_PREFIX_ADDRESS, 16, 4, req->info, 1106 ivsize, 1); 1107 pr_err("asked=%d, nbytes%d\n", 1108 (int)edesc->src_nents > 1 ? 100 : req->nbytes, req->nbytes); 1109 #endif 1110 caam_dump_sg(KERN_ERR, "src @" __stringify(__LINE__)": ", 1111 DUMP_PREFIX_ADDRESS, 16, 4, req->src, 1112 edesc->src_nents > 1 ? 100 : req->nbytes, 1); 1113 1114 len = desc_len(sh_desc); 1115 init_job_desc_shared(desc, ptr, len, HDR_SHARE_DEFER | HDR_REVERSE); 1116 1117 if (iv_contig) { 1118 src_dma = edesc->iv_dma; 1119 in_options = 0; 1120 } else { 1121 src_dma = edesc->sec4_sg_dma; 1122 sec4_sg_index += edesc->src_nents + 1; 1123 in_options = LDST_SGF; 1124 } 1125 append_seq_in_ptr(desc, src_dma, req->nbytes + ivsize, in_options); 1126 1127 if (likely(req->src == req->dst)) { 1128 if (edesc->src_nents == 1 && iv_contig) { 1129 dst_dma = sg_dma_address(req->src); 1130 } else { 1131 dst_dma = edesc->sec4_sg_dma + 1132 sizeof(struct sec4_sg_entry); 1133 out_options = LDST_SGF; 1134 } 1135 } else { 1136 if (edesc->dst_nents == 1) { 1137 dst_dma = sg_dma_address(req->dst); 1138 } else { 1139 dst_dma = edesc->sec4_sg_dma + 1140 sec4_sg_index * sizeof(struct sec4_sg_entry); 1141 out_options = LDST_SGF; 1142 } 1143 } 1144 append_seq_out_ptr(desc, dst_dma, req->nbytes, out_options); 1145 } 1146 1147 /* 1148 * Fill in ablkcipher givencrypt job descriptor 1149 */ 1150 static void init_ablkcipher_giv_job(u32 *sh_desc, dma_addr_t ptr, 1151 struct ablkcipher_edesc *edesc, 1152 struct ablkcipher_request *req, 1153 bool iv_contig) 1154 { 1155 struct crypto_ablkcipher *ablkcipher = crypto_ablkcipher_reqtfm(req); 1156 int ivsize = crypto_ablkcipher_ivsize(ablkcipher); 1157 u32 *desc = edesc->hw_desc; 1158 u32 out_options, in_options; 1159 dma_addr_t dst_dma, src_dma; 1160 int len, sec4_sg_index = 0; 1161 1162 #ifdef DEBUG 1163 print_hex_dump(KERN_ERR, "presciv@" __stringify(__LINE__) ": ", 1164 DUMP_PREFIX_ADDRESS, 16, 4, req->info, 1165 ivsize, 1); 1166 #endif 1167 caam_dump_sg(KERN_ERR, "src @" __stringify(__LINE__) ": ", 1168 DUMP_PREFIX_ADDRESS, 16, 4, req->src, 1169 edesc->src_nents > 1 ? 100 : req->nbytes, 1); 1170 1171 len = desc_len(sh_desc); 1172 init_job_desc_shared(desc, ptr, len, HDR_SHARE_DEFER | HDR_REVERSE); 1173 1174 if (edesc->src_nents == 1) { 1175 src_dma = sg_dma_address(req->src); 1176 in_options = 0; 1177 } else { 1178 src_dma = edesc->sec4_sg_dma; 1179 sec4_sg_index += edesc->src_nents; 1180 in_options = LDST_SGF; 1181 } 1182 append_seq_in_ptr(desc, src_dma, req->nbytes, in_options); 1183 1184 if (iv_contig) { 1185 dst_dma = edesc->iv_dma; 1186 out_options = 0; 1187 } else { 1188 dst_dma = edesc->sec4_sg_dma + 1189 sec4_sg_index * sizeof(struct sec4_sg_entry); 1190 out_options = LDST_SGF; 1191 } 1192 append_seq_out_ptr(desc, dst_dma, req->nbytes + ivsize, out_options); 1193 } 1194 1195 /* 1196 * allocate and map the aead extended descriptor 1197 */ 1198 static struct aead_edesc *aead_edesc_alloc(struct aead_request *req, 1199 int desc_bytes, bool *all_contig_ptr, 1200 bool encrypt) 1201 { 1202 struct crypto_aead *aead = crypto_aead_reqtfm(req); 1203 struct caam_ctx *ctx = crypto_aead_ctx(aead); 1204 struct device *jrdev = ctx->jrdev; 1205 gfp_t flags = (req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP) ? 1206 GFP_KERNEL : GFP_ATOMIC; 1207 int src_nents, mapped_src_nents, dst_nents = 0, mapped_dst_nents = 0; 1208 struct aead_edesc *edesc; 1209 int sec4_sg_index, sec4_sg_len, sec4_sg_bytes; 1210 unsigned int authsize = ctx->authsize; 1211 1212 if (unlikely(req->dst != req->src)) { 1213 src_nents = sg_nents_for_len(req->src, req->assoclen + 1214 req->cryptlen); 1215 if (unlikely(src_nents < 0)) { 1216 dev_err(jrdev, "Insufficient bytes (%d) in src S/G\n", 1217 req->assoclen + req->cryptlen); 1218 return ERR_PTR(src_nents); 1219 } 1220 1221 dst_nents = sg_nents_for_len(req->dst, req->assoclen + 1222 req->cryptlen + 1223 (encrypt ? authsize : 1224 (-authsize))); 1225 if (unlikely(dst_nents < 0)) { 1226 dev_err(jrdev, "Insufficient bytes (%d) in dst S/G\n", 1227 req->assoclen + req->cryptlen + 1228 (encrypt ? authsize : (-authsize))); 1229 return ERR_PTR(dst_nents); 1230 } 1231 } else { 1232 src_nents = sg_nents_for_len(req->src, req->assoclen + 1233 req->cryptlen + 1234 (encrypt ? authsize : 0)); 1235 if (unlikely(src_nents < 0)) { 1236 dev_err(jrdev, "Insufficient bytes (%d) in src S/G\n", 1237 req->assoclen + req->cryptlen + 1238 (encrypt ? authsize : 0)); 1239 return ERR_PTR(src_nents); 1240 } 1241 } 1242 1243 if (likely(req->src == req->dst)) { 1244 mapped_src_nents = dma_map_sg(jrdev, req->src, src_nents, 1245 DMA_BIDIRECTIONAL); 1246 if (unlikely(!mapped_src_nents)) { 1247 dev_err(jrdev, "unable to map source\n"); 1248 return ERR_PTR(-ENOMEM); 1249 } 1250 } else { 1251 /* Cover also the case of null (zero length) input data */ 1252 if (src_nents) { 1253 mapped_src_nents = dma_map_sg(jrdev, req->src, 1254 src_nents, DMA_TO_DEVICE); 1255 if (unlikely(!mapped_src_nents)) { 1256 dev_err(jrdev, "unable to map source\n"); 1257 return ERR_PTR(-ENOMEM); 1258 } 1259 } else { 1260 mapped_src_nents = 0; 1261 } 1262 1263 mapped_dst_nents = dma_map_sg(jrdev, req->dst, dst_nents, 1264 DMA_FROM_DEVICE); 1265 if (unlikely(!mapped_dst_nents)) { 1266 dev_err(jrdev, "unable to map destination\n"); 1267 dma_unmap_sg(jrdev, req->src, src_nents, DMA_TO_DEVICE); 1268 return ERR_PTR(-ENOMEM); 1269 } 1270 } 1271 1272 sec4_sg_len = mapped_src_nents > 1 ? mapped_src_nents : 0; 1273 sec4_sg_len += mapped_dst_nents > 1 ? mapped_dst_nents : 0; 1274 sec4_sg_bytes = sec4_sg_len * sizeof(struct sec4_sg_entry); 1275 1276 /* allocate space for base edesc and hw desc commands, link tables */ 1277 edesc = kzalloc(sizeof(*edesc) + desc_bytes + sec4_sg_bytes, 1278 GFP_DMA | flags); 1279 if (!edesc) { 1280 caam_unmap(jrdev, req->src, req->dst, src_nents, dst_nents, 0, 1281 0, 0, 0); 1282 return ERR_PTR(-ENOMEM); 1283 } 1284 1285 edesc->src_nents = src_nents; 1286 edesc->dst_nents = dst_nents; 1287 edesc->sec4_sg = (void *)edesc + sizeof(struct aead_edesc) + 1288 desc_bytes; 1289 *all_contig_ptr = !(mapped_src_nents > 1); 1290 1291 sec4_sg_index = 0; 1292 if (mapped_src_nents > 1) { 1293 sg_to_sec4_sg_last(req->src, mapped_src_nents, 1294 edesc->sec4_sg + sec4_sg_index, 0); 1295 sec4_sg_index += mapped_src_nents; 1296 } 1297 if (mapped_dst_nents > 1) { 1298 sg_to_sec4_sg_last(req->dst, mapped_dst_nents, 1299 edesc->sec4_sg + sec4_sg_index, 0); 1300 } 1301 1302 if (!sec4_sg_bytes) 1303 return edesc; 1304 1305 edesc->sec4_sg_dma = dma_map_single(jrdev, edesc->sec4_sg, 1306 sec4_sg_bytes, DMA_TO_DEVICE); 1307 if (dma_mapping_error(jrdev, edesc->sec4_sg_dma)) { 1308 dev_err(jrdev, "unable to map S/G table\n"); 1309 aead_unmap(jrdev, edesc, req); 1310 kfree(edesc); 1311 return ERR_PTR(-ENOMEM); 1312 } 1313 1314 edesc->sec4_sg_bytes = sec4_sg_bytes; 1315 1316 return edesc; 1317 } 1318 1319 static int gcm_encrypt(struct aead_request *req) 1320 { 1321 struct aead_edesc *edesc; 1322 struct crypto_aead *aead = crypto_aead_reqtfm(req); 1323 struct caam_ctx *ctx = crypto_aead_ctx(aead); 1324 struct device *jrdev = ctx->jrdev; 1325 bool all_contig; 1326 u32 *desc; 1327 int ret = 0; 1328 1329 /* allocate extended descriptor */ 1330 edesc = aead_edesc_alloc(req, GCM_DESC_JOB_IO_LEN, &all_contig, true); 1331 if (IS_ERR(edesc)) 1332 return PTR_ERR(edesc); 1333 1334 /* Create and submit job descriptor */ 1335 init_gcm_job(req, edesc, all_contig, true); 1336 #ifdef DEBUG 1337 print_hex_dump(KERN_ERR, "aead jobdesc@"__stringify(__LINE__)": ", 1338 DUMP_PREFIX_ADDRESS, 16, 4, edesc->hw_desc, 1339 desc_bytes(edesc->hw_desc), 1); 1340 #endif 1341 1342 desc = edesc->hw_desc; 1343 ret = caam_jr_enqueue(jrdev, desc, aead_encrypt_done, req); 1344 if (!ret) { 1345 ret = -EINPROGRESS; 1346 } else { 1347 aead_unmap(jrdev, edesc, req); 1348 kfree(edesc); 1349 } 1350 1351 return ret; 1352 } 1353 1354 static int ipsec_gcm_encrypt(struct aead_request *req) 1355 { 1356 if (req->assoclen < 8) 1357 return -EINVAL; 1358 1359 return gcm_encrypt(req); 1360 } 1361 1362 static int aead_encrypt(struct aead_request *req) 1363 { 1364 struct aead_edesc *edesc; 1365 struct crypto_aead *aead = crypto_aead_reqtfm(req); 1366 struct caam_ctx *ctx = crypto_aead_ctx(aead); 1367 struct device *jrdev = ctx->jrdev; 1368 bool all_contig; 1369 u32 *desc; 1370 int ret = 0; 1371 1372 /* allocate extended descriptor */ 1373 edesc = aead_edesc_alloc(req, AUTHENC_DESC_JOB_IO_LEN, 1374 &all_contig, true); 1375 if (IS_ERR(edesc)) 1376 return PTR_ERR(edesc); 1377 1378 /* Create and submit job descriptor */ 1379 init_authenc_job(req, edesc, all_contig, true); 1380 #ifdef DEBUG 1381 print_hex_dump(KERN_ERR, "aead jobdesc@"__stringify(__LINE__)": ", 1382 DUMP_PREFIX_ADDRESS, 16, 4, edesc->hw_desc, 1383 desc_bytes(edesc->hw_desc), 1); 1384 #endif 1385 1386 desc = edesc->hw_desc; 1387 ret = caam_jr_enqueue(jrdev, desc, aead_encrypt_done, req); 1388 if (!ret) { 1389 ret = -EINPROGRESS; 1390 } else { 1391 aead_unmap(jrdev, edesc, req); 1392 kfree(edesc); 1393 } 1394 1395 return ret; 1396 } 1397 1398 static int gcm_decrypt(struct aead_request *req) 1399 { 1400 struct aead_edesc *edesc; 1401 struct crypto_aead *aead = crypto_aead_reqtfm(req); 1402 struct caam_ctx *ctx = crypto_aead_ctx(aead); 1403 struct device *jrdev = ctx->jrdev; 1404 bool all_contig; 1405 u32 *desc; 1406 int ret = 0; 1407 1408 /* allocate extended descriptor */ 1409 edesc = aead_edesc_alloc(req, GCM_DESC_JOB_IO_LEN, &all_contig, false); 1410 if (IS_ERR(edesc)) 1411 return PTR_ERR(edesc); 1412 1413 /* Create and submit job descriptor*/ 1414 init_gcm_job(req, edesc, all_contig, false); 1415 #ifdef DEBUG 1416 print_hex_dump(KERN_ERR, "aead jobdesc@"__stringify(__LINE__)": ", 1417 DUMP_PREFIX_ADDRESS, 16, 4, edesc->hw_desc, 1418 desc_bytes(edesc->hw_desc), 1); 1419 #endif 1420 1421 desc = edesc->hw_desc; 1422 ret = caam_jr_enqueue(jrdev, desc, aead_decrypt_done, req); 1423 if (!ret) { 1424 ret = -EINPROGRESS; 1425 } else { 1426 aead_unmap(jrdev, edesc, req); 1427 kfree(edesc); 1428 } 1429 1430 return ret; 1431 } 1432 1433 static int ipsec_gcm_decrypt(struct aead_request *req) 1434 { 1435 if (req->assoclen < 8) 1436 return -EINVAL; 1437 1438 return gcm_decrypt(req); 1439 } 1440 1441 static int aead_decrypt(struct aead_request *req) 1442 { 1443 struct aead_edesc *edesc; 1444 struct crypto_aead *aead = crypto_aead_reqtfm(req); 1445 struct caam_ctx *ctx = crypto_aead_ctx(aead); 1446 struct device *jrdev = ctx->jrdev; 1447 bool all_contig; 1448 u32 *desc; 1449 int ret = 0; 1450 1451 caam_dump_sg(KERN_ERR, "dec src@" __stringify(__LINE__)": ", 1452 DUMP_PREFIX_ADDRESS, 16, 4, req->src, 1453 req->assoclen + req->cryptlen, 1); 1454 1455 /* allocate extended descriptor */ 1456 edesc = aead_edesc_alloc(req, AUTHENC_DESC_JOB_IO_LEN, 1457 &all_contig, false); 1458 if (IS_ERR(edesc)) 1459 return PTR_ERR(edesc); 1460 1461 /* Create and submit job descriptor*/ 1462 init_authenc_job(req, edesc, all_contig, false); 1463 #ifdef DEBUG 1464 print_hex_dump(KERN_ERR, "aead jobdesc@"__stringify(__LINE__)": ", 1465 DUMP_PREFIX_ADDRESS, 16, 4, edesc->hw_desc, 1466 desc_bytes(edesc->hw_desc), 1); 1467 #endif 1468 1469 desc = edesc->hw_desc; 1470 ret = caam_jr_enqueue(jrdev, desc, aead_decrypt_done, req); 1471 if (!ret) { 1472 ret = -EINPROGRESS; 1473 } else { 1474 aead_unmap(jrdev, edesc, req); 1475 kfree(edesc); 1476 } 1477 1478 return ret; 1479 } 1480 1481 /* 1482 * allocate and map the ablkcipher extended descriptor for ablkcipher 1483 */ 1484 static struct ablkcipher_edesc *ablkcipher_edesc_alloc(struct ablkcipher_request 1485 *req, int desc_bytes, 1486 bool *iv_contig_out) 1487 { 1488 struct crypto_ablkcipher *ablkcipher = crypto_ablkcipher_reqtfm(req); 1489 struct caam_ctx *ctx = crypto_ablkcipher_ctx(ablkcipher); 1490 struct device *jrdev = ctx->jrdev; 1491 gfp_t flags = (req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP) ? 1492 GFP_KERNEL : GFP_ATOMIC; 1493 int src_nents, mapped_src_nents, dst_nents = 0, mapped_dst_nents = 0; 1494 struct ablkcipher_edesc *edesc; 1495 dma_addr_t iv_dma = 0; 1496 bool in_contig; 1497 int ivsize = crypto_ablkcipher_ivsize(ablkcipher); 1498 int dst_sg_idx, sec4_sg_ents, sec4_sg_bytes; 1499 1500 src_nents = sg_nents_for_len(req->src, req->nbytes); 1501 if (unlikely(src_nents < 0)) { 1502 dev_err(jrdev, "Insufficient bytes (%d) in src S/G\n", 1503 req->nbytes); 1504 return ERR_PTR(src_nents); 1505 } 1506 1507 if (req->dst != req->src) { 1508 dst_nents = sg_nents_for_len(req->dst, req->nbytes); 1509 if (unlikely(dst_nents < 0)) { 1510 dev_err(jrdev, "Insufficient bytes (%d) in dst S/G\n", 1511 req->nbytes); 1512 return ERR_PTR(dst_nents); 1513 } 1514 } 1515 1516 if (likely(req->src == req->dst)) { 1517 mapped_src_nents = dma_map_sg(jrdev, req->src, src_nents, 1518 DMA_BIDIRECTIONAL); 1519 if (unlikely(!mapped_src_nents)) { 1520 dev_err(jrdev, "unable to map source\n"); 1521 return ERR_PTR(-ENOMEM); 1522 } 1523 } else { 1524 mapped_src_nents = dma_map_sg(jrdev, req->src, src_nents, 1525 DMA_TO_DEVICE); 1526 if (unlikely(!mapped_src_nents)) { 1527 dev_err(jrdev, "unable to map source\n"); 1528 return ERR_PTR(-ENOMEM); 1529 } 1530 1531 mapped_dst_nents = dma_map_sg(jrdev, req->dst, dst_nents, 1532 DMA_FROM_DEVICE); 1533 if (unlikely(!mapped_dst_nents)) { 1534 dev_err(jrdev, "unable to map destination\n"); 1535 dma_unmap_sg(jrdev, req->src, src_nents, DMA_TO_DEVICE); 1536 return ERR_PTR(-ENOMEM); 1537 } 1538 } 1539 1540 iv_dma = dma_map_single(jrdev, req->info, ivsize, DMA_TO_DEVICE); 1541 if (dma_mapping_error(jrdev, iv_dma)) { 1542 dev_err(jrdev, "unable to map IV\n"); 1543 caam_unmap(jrdev, req->src, req->dst, src_nents, dst_nents, 0, 1544 0, 0, 0); 1545 return ERR_PTR(-ENOMEM); 1546 } 1547 1548 if (mapped_src_nents == 1 && 1549 iv_dma + ivsize == sg_dma_address(req->src)) { 1550 in_contig = true; 1551 sec4_sg_ents = 0; 1552 } else { 1553 in_contig = false; 1554 sec4_sg_ents = 1 + mapped_src_nents; 1555 } 1556 dst_sg_idx = sec4_sg_ents; 1557 sec4_sg_ents += mapped_dst_nents > 1 ? mapped_dst_nents : 0; 1558 sec4_sg_bytes = sec4_sg_ents * sizeof(struct sec4_sg_entry); 1559 1560 /* allocate space for base edesc and hw desc commands, link tables */ 1561 edesc = kzalloc(sizeof(*edesc) + desc_bytes + sec4_sg_bytes, 1562 GFP_DMA | flags); 1563 if (!edesc) { 1564 dev_err(jrdev, "could not allocate extended descriptor\n"); 1565 caam_unmap(jrdev, req->src, req->dst, src_nents, dst_nents, 1566 iv_dma, ivsize, 0, 0); 1567 return ERR_PTR(-ENOMEM); 1568 } 1569 1570 edesc->src_nents = src_nents; 1571 edesc->dst_nents = dst_nents; 1572 edesc->sec4_sg_bytes = sec4_sg_bytes; 1573 edesc->sec4_sg = (void *)edesc + sizeof(struct ablkcipher_edesc) + 1574 desc_bytes; 1575 1576 if (!in_contig) { 1577 dma_to_sec4_sg_one(edesc->sec4_sg, iv_dma, ivsize, 0); 1578 sg_to_sec4_sg_last(req->src, mapped_src_nents, 1579 edesc->sec4_sg + 1, 0); 1580 } 1581 1582 if (mapped_dst_nents > 1) { 1583 sg_to_sec4_sg_last(req->dst, mapped_dst_nents, 1584 edesc->sec4_sg + dst_sg_idx, 0); 1585 } 1586 1587 edesc->sec4_sg_dma = dma_map_single(jrdev, edesc->sec4_sg, 1588 sec4_sg_bytes, DMA_TO_DEVICE); 1589 if (dma_mapping_error(jrdev, edesc->sec4_sg_dma)) { 1590 dev_err(jrdev, "unable to map S/G table\n"); 1591 caam_unmap(jrdev, req->src, req->dst, src_nents, dst_nents, 1592 iv_dma, ivsize, 0, 0); 1593 kfree(edesc); 1594 return ERR_PTR(-ENOMEM); 1595 } 1596 1597 edesc->iv_dma = iv_dma; 1598 1599 #ifdef DEBUG 1600 print_hex_dump(KERN_ERR, "ablkcipher sec4_sg@"__stringify(__LINE__)": ", 1601 DUMP_PREFIX_ADDRESS, 16, 4, edesc->sec4_sg, 1602 sec4_sg_bytes, 1); 1603 #endif 1604 1605 *iv_contig_out = in_contig; 1606 return edesc; 1607 } 1608 1609 static int ablkcipher_encrypt(struct ablkcipher_request *req) 1610 { 1611 struct ablkcipher_edesc *edesc; 1612 struct crypto_ablkcipher *ablkcipher = crypto_ablkcipher_reqtfm(req); 1613 struct caam_ctx *ctx = crypto_ablkcipher_ctx(ablkcipher); 1614 struct device *jrdev = ctx->jrdev; 1615 bool iv_contig; 1616 u32 *desc; 1617 int ret = 0; 1618 1619 /* allocate extended descriptor */ 1620 edesc = ablkcipher_edesc_alloc(req, DESC_JOB_IO_LEN * 1621 CAAM_CMD_SZ, &iv_contig); 1622 if (IS_ERR(edesc)) 1623 return PTR_ERR(edesc); 1624 1625 /* Create and submit job descriptor*/ 1626 init_ablkcipher_job(ctx->sh_desc_enc, 1627 ctx->sh_desc_enc_dma, edesc, req, iv_contig); 1628 #ifdef DEBUG 1629 print_hex_dump(KERN_ERR, "ablkcipher jobdesc@"__stringify(__LINE__)": ", 1630 DUMP_PREFIX_ADDRESS, 16, 4, edesc->hw_desc, 1631 desc_bytes(edesc->hw_desc), 1); 1632 #endif 1633 desc = edesc->hw_desc; 1634 ret = caam_jr_enqueue(jrdev, desc, ablkcipher_encrypt_done, req); 1635 1636 if (!ret) { 1637 ret = -EINPROGRESS; 1638 } else { 1639 ablkcipher_unmap(jrdev, edesc, req); 1640 kfree(edesc); 1641 } 1642 1643 return ret; 1644 } 1645 1646 static int ablkcipher_decrypt(struct ablkcipher_request *req) 1647 { 1648 struct ablkcipher_edesc *edesc; 1649 struct crypto_ablkcipher *ablkcipher = crypto_ablkcipher_reqtfm(req); 1650 struct caam_ctx *ctx = crypto_ablkcipher_ctx(ablkcipher); 1651 struct device *jrdev = ctx->jrdev; 1652 bool iv_contig; 1653 u32 *desc; 1654 int ret = 0; 1655 1656 /* allocate extended descriptor */ 1657 edesc = ablkcipher_edesc_alloc(req, DESC_JOB_IO_LEN * 1658 CAAM_CMD_SZ, &iv_contig); 1659 if (IS_ERR(edesc)) 1660 return PTR_ERR(edesc); 1661 1662 /* Create and submit job descriptor*/ 1663 init_ablkcipher_job(ctx->sh_desc_dec, 1664 ctx->sh_desc_dec_dma, edesc, req, iv_contig); 1665 desc = edesc->hw_desc; 1666 #ifdef DEBUG 1667 print_hex_dump(KERN_ERR, "ablkcipher jobdesc@"__stringify(__LINE__)": ", 1668 DUMP_PREFIX_ADDRESS, 16, 4, edesc->hw_desc, 1669 desc_bytes(edesc->hw_desc), 1); 1670 #endif 1671 1672 ret = caam_jr_enqueue(jrdev, desc, ablkcipher_decrypt_done, req); 1673 if (!ret) { 1674 ret = -EINPROGRESS; 1675 } else { 1676 ablkcipher_unmap(jrdev, edesc, req); 1677 kfree(edesc); 1678 } 1679 1680 return ret; 1681 } 1682 1683 /* 1684 * allocate and map the ablkcipher extended descriptor 1685 * for ablkcipher givencrypt 1686 */ 1687 static struct ablkcipher_edesc *ablkcipher_giv_edesc_alloc( 1688 struct skcipher_givcrypt_request *greq, 1689 int desc_bytes, 1690 bool *iv_contig_out) 1691 { 1692 struct ablkcipher_request *req = &greq->creq; 1693 struct crypto_ablkcipher *ablkcipher = crypto_ablkcipher_reqtfm(req); 1694 struct caam_ctx *ctx = crypto_ablkcipher_ctx(ablkcipher); 1695 struct device *jrdev = ctx->jrdev; 1696 gfp_t flags = (req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP) ? 1697 GFP_KERNEL : GFP_ATOMIC; 1698 int src_nents, mapped_src_nents, dst_nents, mapped_dst_nents; 1699 struct ablkcipher_edesc *edesc; 1700 dma_addr_t iv_dma = 0; 1701 bool out_contig; 1702 int ivsize = crypto_ablkcipher_ivsize(ablkcipher); 1703 int dst_sg_idx, sec4_sg_ents, sec4_sg_bytes; 1704 1705 src_nents = sg_nents_for_len(req->src, req->nbytes); 1706 if (unlikely(src_nents < 0)) { 1707 dev_err(jrdev, "Insufficient bytes (%d) in src S/G\n", 1708 req->nbytes); 1709 return ERR_PTR(src_nents); 1710 } 1711 1712 if (likely(req->src == req->dst)) { 1713 mapped_src_nents = dma_map_sg(jrdev, req->src, src_nents, 1714 DMA_BIDIRECTIONAL); 1715 if (unlikely(!mapped_src_nents)) { 1716 dev_err(jrdev, "unable to map source\n"); 1717 return ERR_PTR(-ENOMEM); 1718 } 1719 1720 dst_nents = src_nents; 1721 mapped_dst_nents = src_nents; 1722 } else { 1723 mapped_src_nents = dma_map_sg(jrdev, req->src, src_nents, 1724 DMA_TO_DEVICE); 1725 if (unlikely(!mapped_src_nents)) { 1726 dev_err(jrdev, "unable to map source\n"); 1727 return ERR_PTR(-ENOMEM); 1728 } 1729 1730 dst_nents = sg_nents_for_len(req->dst, req->nbytes); 1731 if (unlikely(dst_nents < 0)) { 1732 dev_err(jrdev, "Insufficient bytes (%d) in dst S/G\n", 1733 req->nbytes); 1734 return ERR_PTR(dst_nents); 1735 } 1736 1737 mapped_dst_nents = dma_map_sg(jrdev, req->dst, dst_nents, 1738 DMA_FROM_DEVICE); 1739 if (unlikely(!mapped_dst_nents)) { 1740 dev_err(jrdev, "unable to map destination\n"); 1741 dma_unmap_sg(jrdev, req->src, src_nents, DMA_TO_DEVICE); 1742 return ERR_PTR(-ENOMEM); 1743 } 1744 } 1745 1746 /* 1747 * Check if iv can be contiguous with source and destination. 1748 * If so, include it. If not, create scatterlist. 1749 */ 1750 iv_dma = dma_map_single(jrdev, greq->giv, ivsize, DMA_TO_DEVICE); 1751 if (dma_mapping_error(jrdev, iv_dma)) { 1752 dev_err(jrdev, "unable to map IV\n"); 1753 caam_unmap(jrdev, req->src, req->dst, src_nents, dst_nents, 0, 1754 0, 0, 0); 1755 return ERR_PTR(-ENOMEM); 1756 } 1757 1758 sec4_sg_ents = mapped_src_nents > 1 ? mapped_src_nents : 0; 1759 dst_sg_idx = sec4_sg_ents; 1760 if (mapped_dst_nents == 1 && 1761 iv_dma + ivsize == sg_dma_address(req->dst)) { 1762 out_contig = true; 1763 } else { 1764 out_contig = false; 1765 sec4_sg_ents += 1 + mapped_dst_nents; 1766 } 1767 1768 /* allocate space for base edesc and hw desc commands, link tables */ 1769 sec4_sg_bytes = sec4_sg_ents * sizeof(struct sec4_sg_entry); 1770 edesc = kzalloc(sizeof(*edesc) + desc_bytes + sec4_sg_bytes, 1771 GFP_DMA | flags); 1772 if (!edesc) { 1773 dev_err(jrdev, "could not allocate extended descriptor\n"); 1774 caam_unmap(jrdev, req->src, req->dst, src_nents, dst_nents, 1775 iv_dma, ivsize, 0, 0); 1776 return ERR_PTR(-ENOMEM); 1777 } 1778 1779 edesc->src_nents = src_nents; 1780 edesc->dst_nents = dst_nents; 1781 edesc->sec4_sg_bytes = sec4_sg_bytes; 1782 edesc->sec4_sg = (void *)edesc + sizeof(struct ablkcipher_edesc) + 1783 desc_bytes; 1784 1785 if (mapped_src_nents > 1) 1786 sg_to_sec4_sg_last(req->src, mapped_src_nents, edesc->sec4_sg, 1787 0); 1788 1789 if (!out_contig) { 1790 dma_to_sec4_sg_one(edesc->sec4_sg + dst_sg_idx, 1791 iv_dma, ivsize, 0); 1792 sg_to_sec4_sg_last(req->dst, mapped_dst_nents, 1793 edesc->sec4_sg + dst_sg_idx + 1, 0); 1794 } 1795 1796 edesc->sec4_sg_dma = dma_map_single(jrdev, edesc->sec4_sg, 1797 sec4_sg_bytes, DMA_TO_DEVICE); 1798 if (dma_mapping_error(jrdev, edesc->sec4_sg_dma)) { 1799 dev_err(jrdev, "unable to map S/G table\n"); 1800 caam_unmap(jrdev, req->src, req->dst, src_nents, dst_nents, 1801 iv_dma, ivsize, 0, 0); 1802 kfree(edesc); 1803 return ERR_PTR(-ENOMEM); 1804 } 1805 edesc->iv_dma = iv_dma; 1806 1807 #ifdef DEBUG 1808 print_hex_dump(KERN_ERR, 1809 "ablkcipher sec4_sg@" __stringify(__LINE__) ": ", 1810 DUMP_PREFIX_ADDRESS, 16, 4, edesc->sec4_sg, 1811 sec4_sg_bytes, 1); 1812 #endif 1813 1814 *iv_contig_out = out_contig; 1815 return edesc; 1816 } 1817 1818 static int ablkcipher_givencrypt(struct skcipher_givcrypt_request *creq) 1819 { 1820 struct ablkcipher_request *req = &creq->creq; 1821 struct ablkcipher_edesc *edesc; 1822 struct crypto_ablkcipher *ablkcipher = crypto_ablkcipher_reqtfm(req); 1823 struct caam_ctx *ctx = crypto_ablkcipher_ctx(ablkcipher); 1824 struct device *jrdev = ctx->jrdev; 1825 bool iv_contig = false; 1826 u32 *desc; 1827 int ret = 0; 1828 1829 /* allocate extended descriptor */ 1830 edesc = ablkcipher_giv_edesc_alloc(creq, DESC_JOB_IO_LEN * 1831 CAAM_CMD_SZ, &iv_contig); 1832 if (IS_ERR(edesc)) 1833 return PTR_ERR(edesc); 1834 1835 /* Create and submit job descriptor*/ 1836 init_ablkcipher_giv_job(ctx->sh_desc_givenc, ctx->sh_desc_givenc_dma, 1837 edesc, req, iv_contig); 1838 #ifdef DEBUG 1839 print_hex_dump(KERN_ERR, 1840 "ablkcipher jobdesc@" __stringify(__LINE__) ": ", 1841 DUMP_PREFIX_ADDRESS, 16, 4, edesc->hw_desc, 1842 desc_bytes(edesc->hw_desc), 1); 1843 #endif 1844 desc = edesc->hw_desc; 1845 ret = caam_jr_enqueue(jrdev, desc, ablkcipher_encrypt_done, req); 1846 1847 if (!ret) { 1848 ret = -EINPROGRESS; 1849 } else { 1850 ablkcipher_unmap(jrdev, edesc, req); 1851 kfree(edesc); 1852 } 1853 1854 return ret; 1855 } 1856 1857 #define template_aead template_u.aead 1858 #define template_ablkcipher template_u.ablkcipher 1859 struct caam_alg_template { 1860 char name[CRYPTO_MAX_ALG_NAME]; 1861 char driver_name[CRYPTO_MAX_ALG_NAME]; 1862 unsigned int blocksize; 1863 u32 type; 1864 union { 1865 struct ablkcipher_alg ablkcipher; 1866 } template_u; 1867 u32 class1_alg_type; 1868 u32 class2_alg_type; 1869 }; 1870 1871 static struct caam_alg_template driver_algs[] = { 1872 /* ablkcipher descriptor */ 1873 { 1874 .name = "cbc(aes)", 1875 .driver_name = "cbc-aes-caam", 1876 .blocksize = AES_BLOCK_SIZE, 1877 .type = CRYPTO_ALG_TYPE_GIVCIPHER, 1878 .template_ablkcipher = { 1879 .setkey = ablkcipher_setkey, 1880 .encrypt = ablkcipher_encrypt, 1881 .decrypt = ablkcipher_decrypt, 1882 .givencrypt = ablkcipher_givencrypt, 1883 .geniv = "<built-in>", 1884 .min_keysize = AES_MIN_KEY_SIZE, 1885 .max_keysize = AES_MAX_KEY_SIZE, 1886 .ivsize = AES_BLOCK_SIZE, 1887 }, 1888 .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CBC, 1889 }, 1890 { 1891 .name = "cbc(des3_ede)", 1892 .driver_name = "cbc-3des-caam", 1893 .blocksize = DES3_EDE_BLOCK_SIZE, 1894 .type = CRYPTO_ALG_TYPE_GIVCIPHER, 1895 .template_ablkcipher = { 1896 .setkey = ablkcipher_setkey, 1897 .encrypt = ablkcipher_encrypt, 1898 .decrypt = ablkcipher_decrypt, 1899 .givencrypt = ablkcipher_givencrypt, 1900 .geniv = "<built-in>", 1901 .min_keysize = DES3_EDE_KEY_SIZE, 1902 .max_keysize = DES3_EDE_KEY_SIZE, 1903 .ivsize = DES3_EDE_BLOCK_SIZE, 1904 }, 1905 .class1_alg_type = OP_ALG_ALGSEL_3DES | OP_ALG_AAI_CBC, 1906 }, 1907 { 1908 .name = "cbc(des)", 1909 .driver_name = "cbc-des-caam", 1910 .blocksize = DES_BLOCK_SIZE, 1911 .type = CRYPTO_ALG_TYPE_GIVCIPHER, 1912 .template_ablkcipher = { 1913 .setkey = ablkcipher_setkey, 1914 .encrypt = ablkcipher_encrypt, 1915 .decrypt = ablkcipher_decrypt, 1916 .givencrypt = ablkcipher_givencrypt, 1917 .geniv = "<built-in>", 1918 .min_keysize = DES_KEY_SIZE, 1919 .max_keysize = DES_KEY_SIZE, 1920 .ivsize = DES_BLOCK_SIZE, 1921 }, 1922 .class1_alg_type = OP_ALG_ALGSEL_DES | OP_ALG_AAI_CBC, 1923 }, 1924 { 1925 .name = "ctr(aes)", 1926 .driver_name = "ctr-aes-caam", 1927 .blocksize = 1, 1928 .type = CRYPTO_ALG_TYPE_ABLKCIPHER, 1929 .template_ablkcipher = { 1930 .setkey = ablkcipher_setkey, 1931 .encrypt = ablkcipher_encrypt, 1932 .decrypt = ablkcipher_decrypt, 1933 .geniv = "chainiv", 1934 .min_keysize = AES_MIN_KEY_SIZE, 1935 .max_keysize = AES_MAX_KEY_SIZE, 1936 .ivsize = AES_BLOCK_SIZE, 1937 }, 1938 .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CTR_MOD128, 1939 }, 1940 { 1941 .name = "rfc3686(ctr(aes))", 1942 .driver_name = "rfc3686-ctr-aes-caam", 1943 .blocksize = 1, 1944 .type = CRYPTO_ALG_TYPE_GIVCIPHER, 1945 .template_ablkcipher = { 1946 .setkey = ablkcipher_setkey, 1947 .encrypt = ablkcipher_encrypt, 1948 .decrypt = ablkcipher_decrypt, 1949 .givencrypt = ablkcipher_givencrypt, 1950 .geniv = "<built-in>", 1951 .min_keysize = AES_MIN_KEY_SIZE + 1952 CTR_RFC3686_NONCE_SIZE, 1953 .max_keysize = AES_MAX_KEY_SIZE + 1954 CTR_RFC3686_NONCE_SIZE, 1955 .ivsize = CTR_RFC3686_IV_SIZE, 1956 }, 1957 .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CTR_MOD128, 1958 }, 1959 { 1960 .name = "xts(aes)", 1961 .driver_name = "xts-aes-caam", 1962 .blocksize = AES_BLOCK_SIZE, 1963 .type = CRYPTO_ALG_TYPE_ABLKCIPHER, 1964 .template_ablkcipher = { 1965 .setkey = xts_ablkcipher_setkey, 1966 .encrypt = ablkcipher_encrypt, 1967 .decrypt = ablkcipher_decrypt, 1968 .geniv = "eseqiv", 1969 .min_keysize = 2 * AES_MIN_KEY_SIZE, 1970 .max_keysize = 2 * AES_MAX_KEY_SIZE, 1971 .ivsize = AES_BLOCK_SIZE, 1972 }, 1973 .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_XTS, 1974 }, 1975 }; 1976 1977 static struct caam_aead_alg driver_aeads[] = { 1978 { 1979 .aead = { 1980 .base = { 1981 .cra_name = "rfc4106(gcm(aes))", 1982 .cra_driver_name = "rfc4106-gcm-aes-caam", 1983 .cra_blocksize = 1, 1984 }, 1985 .setkey = rfc4106_setkey, 1986 .setauthsize = rfc4106_setauthsize, 1987 .encrypt = ipsec_gcm_encrypt, 1988 .decrypt = ipsec_gcm_decrypt, 1989 .ivsize = GCM_RFC4106_IV_SIZE, 1990 .maxauthsize = AES_BLOCK_SIZE, 1991 }, 1992 .caam = { 1993 .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_GCM, 1994 }, 1995 }, 1996 { 1997 .aead = { 1998 .base = { 1999 .cra_name = "rfc4543(gcm(aes))", 2000 .cra_driver_name = "rfc4543-gcm-aes-caam", 2001 .cra_blocksize = 1, 2002 }, 2003 .setkey = rfc4543_setkey, 2004 .setauthsize = rfc4543_setauthsize, 2005 .encrypt = ipsec_gcm_encrypt, 2006 .decrypt = ipsec_gcm_decrypt, 2007 .ivsize = GCM_RFC4543_IV_SIZE, 2008 .maxauthsize = AES_BLOCK_SIZE, 2009 }, 2010 .caam = { 2011 .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_GCM, 2012 }, 2013 }, 2014 /* Galois Counter Mode */ 2015 { 2016 .aead = { 2017 .base = { 2018 .cra_name = "gcm(aes)", 2019 .cra_driver_name = "gcm-aes-caam", 2020 .cra_blocksize = 1, 2021 }, 2022 .setkey = gcm_setkey, 2023 .setauthsize = gcm_setauthsize, 2024 .encrypt = gcm_encrypt, 2025 .decrypt = gcm_decrypt, 2026 .ivsize = GCM_AES_IV_SIZE, 2027 .maxauthsize = AES_BLOCK_SIZE, 2028 }, 2029 .caam = { 2030 .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_GCM, 2031 }, 2032 }, 2033 /* single-pass ipsec_esp descriptor */ 2034 { 2035 .aead = { 2036 .base = { 2037 .cra_name = "authenc(hmac(md5)," 2038 "ecb(cipher_null))", 2039 .cra_driver_name = "authenc-hmac-md5-" 2040 "ecb-cipher_null-caam", 2041 .cra_blocksize = NULL_BLOCK_SIZE, 2042 }, 2043 .setkey = aead_setkey, 2044 .setauthsize = aead_setauthsize, 2045 .encrypt = aead_encrypt, 2046 .decrypt = aead_decrypt, 2047 .ivsize = NULL_IV_SIZE, 2048 .maxauthsize = MD5_DIGEST_SIZE, 2049 }, 2050 .caam = { 2051 .class2_alg_type = OP_ALG_ALGSEL_MD5 | 2052 OP_ALG_AAI_HMAC_PRECOMP, 2053 }, 2054 }, 2055 { 2056 .aead = { 2057 .base = { 2058 .cra_name = "authenc(hmac(sha1)," 2059 "ecb(cipher_null))", 2060 .cra_driver_name = "authenc-hmac-sha1-" 2061 "ecb-cipher_null-caam", 2062 .cra_blocksize = NULL_BLOCK_SIZE, 2063 }, 2064 .setkey = aead_setkey, 2065 .setauthsize = aead_setauthsize, 2066 .encrypt = aead_encrypt, 2067 .decrypt = aead_decrypt, 2068 .ivsize = NULL_IV_SIZE, 2069 .maxauthsize = SHA1_DIGEST_SIZE, 2070 }, 2071 .caam = { 2072 .class2_alg_type = OP_ALG_ALGSEL_SHA1 | 2073 OP_ALG_AAI_HMAC_PRECOMP, 2074 }, 2075 }, 2076 { 2077 .aead = { 2078 .base = { 2079 .cra_name = "authenc(hmac(sha224)," 2080 "ecb(cipher_null))", 2081 .cra_driver_name = "authenc-hmac-sha224-" 2082 "ecb-cipher_null-caam", 2083 .cra_blocksize = NULL_BLOCK_SIZE, 2084 }, 2085 .setkey = aead_setkey, 2086 .setauthsize = aead_setauthsize, 2087 .encrypt = aead_encrypt, 2088 .decrypt = aead_decrypt, 2089 .ivsize = NULL_IV_SIZE, 2090 .maxauthsize = SHA224_DIGEST_SIZE, 2091 }, 2092 .caam = { 2093 .class2_alg_type = OP_ALG_ALGSEL_SHA224 | 2094 OP_ALG_AAI_HMAC_PRECOMP, 2095 }, 2096 }, 2097 { 2098 .aead = { 2099 .base = { 2100 .cra_name = "authenc(hmac(sha256)," 2101 "ecb(cipher_null))", 2102 .cra_driver_name = "authenc-hmac-sha256-" 2103 "ecb-cipher_null-caam", 2104 .cra_blocksize = NULL_BLOCK_SIZE, 2105 }, 2106 .setkey = aead_setkey, 2107 .setauthsize = aead_setauthsize, 2108 .encrypt = aead_encrypt, 2109 .decrypt = aead_decrypt, 2110 .ivsize = NULL_IV_SIZE, 2111 .maxauthsize = SHA256_DIGEST_SIZE, 2112 }, 2113 .caam = { 2114 .class2_alg_type = OP_ALG_ALGSEL_SHA256 | 2115 OP_ALG_AAI_HMAC_PRECOMP, 2116 }, 2117 }, 2118 { 2119 .aead = { 2120 .base = { 2121 .cra_name = "authenc(hmac(sha384)," 2122 "ecb(cipher_null))", 2123 .cra_driver_name = "authenc-hmac-sha384-" 2124 "ecb-cipher_null-caam", 2125 .cra_blocksize = NULL_BLOCK_SIZE, 2126 }, 2127 .setkey = aead_setkey, 2128 .setauthsize = aead_setauthsize, 2129 .encrypt = aead_encrypt, 2130 .decrypt = aead_decrypt, 2131 .ivsize = NULL_IV_SIZE, 2132 .maxauthsize = SHA384_DIGEST_SIZE, 2133 }, 2134 .caam = { 2135 .class2_alg_type = OP_ALG_ALGSEL_SHA384 | 2136 OP_ALG_AAI_HMAC_PRECOMP, 2137 }, 2138 }, 2139 { 2140 .aead = { 2141 .base = { 2142 .cra_name = "authenc(hmac(sha512)," 2143 "ecb(cipher_null))", 2144 .cra_driver_name = "authenc-hmac-sha512-" 2145 "ecb-cipher_null-caam", 2146 .cra_blocksize = NULL_BLOCK_SIZE, 2147 }, 2148 .setkey = aead_setkey, 2149 .setauthsize = aead_setauthsize, 2150 .encrypt = aead_encrypt, 2151 .decrypt = aead_decrypt, 2152 .ivsize = NULL_IV_SIZE, 2153 .maxauthsize = SHA512_DIGEST_SIZE, 2154 }, 2155 .caam = { 2156 .class2_alg_type = OP_ALG_ALGSEL_SHA512 | 2157 OP_ALG_AAI_HMAC_PRECOMP, 2158 }, 2159 }, 2160 { 2161 .aead = { 2162 .base = { 2163 .cra_name = "authenc(hmac(md5),cbc(aes))", 2164 .cra_driver_name = "authenc-hmac-md5-" 2165 "cbc-aes-caam", 2166 .cra_blocksize = AES_BLOCK_SIZE, 2167 }, 2168 .setkey = aead_setkey, 2169 .setauthsize = aead_setauthsize, 2170 .encrypt = aead_encrypt, 2171 .decrypt = aead_decrypt, 2172 .ivsize = AES_BLOCK_SIZE, 2173 .maxauthsize = MD5_DIGEST_SIZE, 2174 }, 2175 .caam = { 2176 .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CBC, 2177 .class2_alg_type = OP_ALG_ALGSEL_MD5 | 2178 OP_ALG_AAI_HMAC_PRECOMP, 2179 }, 2180 }, 2181 { 2182 .aead = { 2183 .base = { 2184 .cra_name = "echainiv(authenc(hmac(md5)," 2185 "cbc(aes)))", 2186 .cra_driver_name = "echainiv-authenc-hmac-md5-" 2187 "cbc-aes-caam", 2188 .cra_blocksize = AES_BLOCK_SIZE, 2189 }, 2190 .setkey = aead_setkey, 2191 .setauthsize = aead_setauthsize, 2192 .encrypt = aead_encrypt, 2193 .decrypt = aead_decrypt, 2194 .ivsize = AES_BLOCK_SIZE, 2195 .maxauthsize = MD5_DIGEST_SIZE, 2196 }, 2197 .caam = { 2198 .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CBC, 2199 .class2_alg_type = OP_ALG_ALGSEL_MD5 | 2200 OP_ALG_AAI_HMAC_PRECOMP, 2201 .geniv = true, 2202 }, 2203 }, 2204 { 2205 .aead = { 2206 .base = { 2207 .cra_name = "authenc(hmac(sha1),cbc(aes))", 2208 .cra_driver_name = "authenc-hmac-sha1-" 2209 "cbc-aes-caam", 2210 .cra_blocksize = AES_BLOCK_SIZE, 2211 }, 2212 .setkey = aead_setkey, 2213 .setauthsize = aead_setauthsize, 2214 .encrypt = aead_encrypt, 2215 .decrypt = aead_decrypt, 2216 .ivsize = AES_BLOCK_SIZE, 2217 .maxauthsize = SHA1_DIGEST_SIZE, 2218 }, 2219 .caam = { 2220 .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CBC, 2221 .class2_alg_type = OP_ALG_ALGSEL_SHA1 | 2222 OP_ALG_AAI_HMAC_PRECOMP, 2223 }, 2224 }, 2225 { 2226 .aead = { 2227 .base = { 2228 .cra_name = "echainiv(authenc(hmac(sha1)," 2229 "cbc(aes)))", 2230 .cra_driver_name = "echainiv-authenc-" 2231 "hmac-sha1-cbc-aes-caam", 2232 .cra_blocksize = AES_BLOCK_SIZE, 2233 }, 2234 .setkey = aead_setkey, 2235 .setauthsize = aead_setauthsize, 2236 .encrypt = aead_encrypt, 2237 .decrypt = aead_decrypt, 2238 .ivsize = AES_BLOCK_SIZE, 2239 .maxauthsize = SHA1_DIGEST_SIZE, 2240 }, 2241 .caam = { 2242 .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CBC, 2243 .class2_alg_type = OP_ALG_ALGSEL_SHA1 | 2244 OP_ALG_AAI_HMAC_PRECOMP, 2245 .geniv = true, 2246 }, 2247 }, 2248 { 2249 .aead = { 2250 .base = { 2251 .cra_name = "authenc(hmac(sha224),cbc(aes))", 2252 .cra_driver_name = "authenc-hmac-sha224-" 2253 "cbc-aes-caam", 2254 .cra_blocksize = AES_BLOCK_SIZE, 2255 }, 2256 .setkey = aead_setkey, 2257 .setauthsize = aead_setauthsize, 2258 .encrypt = aead_encrypt, 2259 .decrypt = aead_decrypt, 2260 .ivsize = AES_BLOCK_SIZE, 2261 .maxauthsize = SHA224_DIGEST_SIZE, 2262 }, 2263 .caam = { 2264 .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CBC, 2265 .class2_alg_type = OP_ALG_ALGSEL_SHA224 | 2266 OP_ALG_AAI_HMAC_PRECOMP, 2267 }, 2268 }, 2269 { 2270 .aead = { 2271 .base = { 2272 .cra_name = "echainiv(authenc(hmac(sha224)," 2273 "cbc(aes)))", 2274 .cra_driver_name = "echainiv-authenc-" 2275 "hmac-sha224-cbc-aes-caam", 2276 .cra_blocksize = AES_BLOCK_SIZE, 2277 }, 2278 .setkey = aead_setkey, 2279 .setauthsize = aead_setauthsize, 2280 .encrypt = aead_encrypt, 2281 .decrypt = aead_decrypt, 2282 .ivsize = AES_BLOCK_SIZE, 2283 .maxauthsize = SHA224_DIGEST_SIZE, 2284 }, 2285 .caam = { 2286 .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CBC, 2287 .class2_alg_type = OP_ALG_ALGSEL_SHA224 | 2288 OP_ALG_AAI_HMAC_PRECOMP, 2289 .geniv = true, 2290 }, 2291 }, 2292 { 2293 .aead = { 2294 .base = { 2295 .cra_name = "authenc(hmac(sha256),cbc(aes))", 2296 .cra_driver_name = "authenc-hmac-sha256-" 2297 "cbc-aes-caam", 2298 .cra_blocksize = AES_BLOCK_SIZE, 2299 }, 2300 .setkey = aead_setkey, 2301 .setauthsize = aead_setauthsize, 2302 .encrypt = aead_encrypt, 2303 .decrypt = aead_decrypt, 2304 .ivsize = AES_BLOCK_SIZE, 2305 .maxauthsize = SHA256_DIGEST_SIZE, 2306 }, 2307 .caam = { 2308 .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CBC, 2309 .class2_alg_type = OP_ALG_ALGSEL_SHA256 | 2310 OP_ALG_AAI_HMAC_PRECOMP, 2311 }, 2312 }, 2313 { 2314 .aead = { 2315 .base = { 2316 .cra_name = "echainiv(authenc(hmac(sha256)," 2317 "cbc(aes)))", 2318 .cra_driver_name = "echainiv-authenc-" 2319 "hmac-sha256-cbc-aes-caam", 2320 .cra_blocksize = AES_BLOCK_SIZE, 2321 }, 2322 .setkey = aead_setkey, 2323 .setauthsize = aead_setauthsize, 2324 .encrypt = aead_encrypt, 2325 .decrypt = aead_decrypt, 2326 .ivsize = AES_BLOCK_SIZE, 2327 .maxauthsize = SHA256_DIGEST_SIZE, 2328 }, 2329 .caam = { 2330 .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CBC, 2331 .class2_alg_type = OP_ALG_ALGSEL_SHA256 | 2332 OP_ALG_AAI_HMAC_PRECOMP, 2333 .geniv = true, 2334 }, 2335 }, 2336 { 2337 .aead = { 2338 .base = { 2339 .cra_name = "authenc(hmac(sha384),cbc(aes))", 2340 .cra_driver_name = "authenc-hmac-sha384-" 2341 "cbc-aes-caam", 2342 .cra_blocksize = AES_BLOCK_SIZE, 2343 }, 2344 .setkey = aead_setkey, 2345 .setauthsize = aead_setauthsize, 2346 .encrypt = aead_encrypt, 2347 .decrypt = aead_decrypt, 2348 .ivsize = AES_BLOCK_SIZE, 2349 .maxauthsize = SHA384_DIGEST_SIZE, 2350 }, 2351 .caam = { 2352 .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CBC, 2353 .class2_alg_type = OP_ALG_ALGSEL_SHA384 | 2354 OP_ALG_AAI_HMAC_PRECOMP, 2355 }, 2356 }, 2357 { 2358 .aead = { 2359 .base = { 2360 .cra_name = "echainiv(authenc(hmac(sha384)," 2361 "cbc(aes)))", 2362 .cra_driver_name = "echainiv-authenc-" 2363 "hmac-sha384-cbc-aes-caam", 2364 .cra_blocksize = AES_BLOCK_SIZE, 2365 }, 2366 .setkey = aead_setkey, 2367 .setauthsize = aead_setauthsize, 2368 .encrypt = aead_encrypt, 2369 .decrypt = aead_decrypt, 2370 .ivsize = AES_BLOCK_SIZE, 2371 .maxauthsize = SHA384_DIGEST_SIZE, 2372 }, 2373 .caam = { 2374 .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CBC, 2375 .class2_alg_type = OP_ALG_ALGSEL_SHA384 | 2376 OP_ALG_AAI_HMAC_PRECOMP, 2377 .geniv = true, 2378 }, 2379 }, 2380 { 2381 .aead = { 2382 .base = { 2383 .cra_name = "authenc(hmac(sha512),cbc(aes))", 2384 .cra_driver_name = "authenc-hmac-sha512-" 2385 "cbc-aes-caam", 2386 .cra_blocksize = AES_BLOCK_SIZE, 2387 }, 2388 .setkey = aead_setkey, 2389 .setauthsize = aead_setauthsize, 2390 .encrypt = aead_encrypt, 2391 .decrypt = aead_decrypt, 2392 .ivsize = AES_BLOCK_SIZE, 2393 .maxauthsize = SHA512_DIGEST_SIZE, 2394 }, 2395 .caam = { 2396 .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CBC, 2397 .class2_alg_type = OP_ALG_ALGSEL_SHA512 | 2398 OP_ALG_AAI_HMAC_PRECOMP, 2399 }, 2400 }, 2401 { 2402 .aead = { 2403 .base = { 2404 .cra_name = "echainiv(authenc(hmac(sha512)," 2405 "cbc(aes)))", 2406 .cra_driver_name = "echainiv-authenc-" 2407 "hmac-sha512-cbc-aes-caam", 2408 .cra_blocksize = AES_BLOCK_SIZE, 2409 }, 2410 .setkey = aead_setkey, 2411 .setauthsize = aead_setauthsize, 2412 .encrypt = aead_encrypt, 2413 .decrypt = aead_decrypt, 2414 .ivsize = AES_BLOCK_SIZE, 2415 .maxauthsize = SHA512_DIGEST_SIZE, 2416 }, 2417 .caam = { 2418 .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CBC, 2419 .class2_alg_type = OP_ALG_ALGSEL_SHA512 | 2420 OP_ALG_AAI_HMAC_PRECOMP, 2421 .geniv = true, 2422 }, 2423 }, 2424 { 2425 .aead = { 2426 .base = { 2427 .cra_name = "authenc(hmac(md5),cbc(des3_ede))", 2428 .cra_driver_name = "authenc-hmac-md5-" 2429 "cbc-des3_ede-caam", 2430 .cra_blocksize = DES3_EDE_BLOCK_SIZE, 2431 }, 2432 .setkey = aead_setkey, 2433 .setauthsize = aead_setauthsize, 2434 .encrypt = aead_encrypt, 2435 .decrypt = aead_decrypt, 2436 .ivsize = DES3_EDE_BLOCK_SIZE, 2437 .maxauthsize = MD5_DIGEST_SIZE, 2438 }, 2439 .caam = { 2440 .class1_alg_type = OP_ALG_ALGSEL_3DES | OP_ALG_AAI_CBC, 2441 .class2_alg_type = OP_ALG_ALGSEL_MD5 | 2442 OP_ALG_AAI_HMAC_PRECOMP, 2443 } 2444 }, 2445 { 2446 .aead = { 2447 .base = { 2448 .cra_name = "echainiv(authenc(hmac(md5)," 2449 "cbc(des3_ede)))", 2450 .cra_driver_name = "echainiv-authenc-hmac-md5-" 2451 "cbc-des3_ede-caam", 2452 .cra_blocksize = DES3_EDE_BLOCK_SIZE, 2453 }, 2454 .setkey = aead_setkey, 2455 .setauthsize = aead_setauthsize, 2456 .encrypt = aead_encrypt, 2457 .decrypt = aead_decrypt, 2458 .ivsize = DES3_EDE_BLOCK_SIZE, 2459 .maxauthsize = MD5_DIGEST_SIZE, 2460 }, 2461 .caam = { 2462 .class1_alg_type = OP_ALG_ALGSEL_3DES | OP_ALG_AAI_CBC, 2463 .class2_alg_type = OP_ALG_ALGSEL_MD5 | 2464 OP_ALG_AAI_HMAC_PRECOMP, 2465 .geniv = true, 2466 } 2467 }, 2468 { 2469 .aead = { 2470 .base = { 2471 .cra_name = "authenc(hmac(sha1)," 2472 "cbc(des3_ede))", 2473 .cra_driver_name = "authenc-hmac-sha1-" 2474 "cbc-des3_ede-caam", 2475 .cra_blocksize = DES3_EDE_BLOCK_SIZE, 2476 }, 2477 .setkey = aead_setkey, 2478 .setauthsize = aead_setauthsize, 2479 .encrypt = aead_encrypt, 2480 .decrypt = aead_decrypt, 2481 .ivsize = DES3_EDE_BLOCK_SIZE, 2482 .maxauthsize = SHA1_DIGEST_SIZE, 2483 }, 2484 .caam = { 2485 .class1_alg_type = OP_ALG_ALGSEL_3DES | OP_ALG_AAI_CBC, 2486 .class2_alg_type = OP_ALG_ALGSEL_SHA1 | 2487 OP_ALG_AAI_HMAC_PRECOMP, 2488 }, 2489 }, 2490 { 2491 .aead = { 2492 .base = { 2493 .cra_name = "echainiv(authenc(hmac(sha1)," 2494 "cbc(des3_ede)))", 2495 .cra_driver_name = "echainiv-authenc-" 2496 "hmac-sha1-" 2497 "cbc-des3_ede-caam", 2498 .cra_blocksize = DES3_EDE_BLOCK_SIZE, 2499 }, 2500 .setkey = aead_setkey, 2501 .setauthsize = aead_setauthsize, 2502 .encrypt = aead_encrypt, 2503 .decrypt = aead_decrypt, 2504 .ivsize = DES3_EDE_BLOCK_SIZE, 2505 .maxauthsize = SHA1_DIGEST_SIZE, 2506 }, 2507 .caam = { 2508 .class1_alg_type = OP_ALG_ALGSEL_3DES | OP_ALG_AAI_CBC, 2509 .class2_alg_type = OP_ALG_ALGSEL_SHA1 | 2510 OP_ALG_AAI_HMAC_PRECOMP, 2511 .geniv = true, 2512 }, 2513 }, 2514 { 2515 .aead = { 2516 .base = { 2517 .cra_name = "authenc(hmac(sha224)," 2518 "cbc(des3_ede))", 2519 .cra_driver_name = "authenc-hmac-sha224-" 2520 "cbc-des3_ede-caam", 2521 .cra_blocksize = DES3_EDE_BLOCK_SIZE, 2522 }, 2523 .setkey = aead_setkey, 2524 .setauthsize = aead_setauthsize, 2525 .encrypt = aead_encrypt, 2526 .decrypt = aead_decrypt, 2527 .ivsize = DES3_EDE_BLOCK_SIZE, 2528 .maxauthsize = SHA224_DIGEST_SIZE, 2529 }, 2530 .caam = { 2531 .class1_alg_type = OP_ALG_ALGSEL_3DES | OP_ALG_AAI_CBC, 2532 .class2_alg_type = OP_ALG_ALGSEL_SHA224 | 2533 OP_ALG_AAI_HMAC_PRECOMP, 2534 }, 2535 }, 2536 { 2537 .aead = { 2538 .base = { 2539 .cra_name = "echainiv(authenc(hmac(sha224)," 2540 "cbc(des3_ede)))", 2541 .cra_driver_name = "echainiv-authenc-" 2542 "hmac-sha224-" 2543 "cbc-des3_ede-caam", 2544 .cra_blocksize = DES3_EDE_BLOCK_SIZE, 2545 }, 2546 .setkey = aead_setkey, 2547 .setauthsize = aead_setauthsize, 2548 .encrypt = aead_encrypt, 2549 .decrypt = aead_decrypt, 2550 .ivsize = DES3_EDE_BLOCK_SIZE, 2551 .maxauthsize = SHA224_DIGEST_SIZE, 2552 }, 2553 .caam = { 2554 .class1_alg_type = OP_ALG_ALGSEL_3DES | OP_ALG_AAI_CBC, 2555 .class2_alg_type = OP_ALG_ALGSEL_SHA224 | 2556 OP_ALG_AAI_HMAC_PRECOMP, 2557 .geniv = true, 2558 }, 2559 }, 2560 { 2561 .aead = { 2562 .base = { 2563 .cra_name = "authenc(hmac(sha256)," 2564 "cbc(des3_ede))", 2565 .cra_driver_name = "authenc-hmac-sha256-" 2566 "cbc-des3_ede-caam", 2567 .cra_blocksize = DES3_EDE_BLOCK_SIZE, 2568 }, 2569 .setkey = aead_setkey, 2570 .setauthsize = aead_setauthsize, 2571 .encrypt = aead_encrypt, 2572 .decrypt = aead_decrypt, 2573 .ivsize = DES3_EDE_BLOCK_SIZE, 2574 .maxauthsize = SHA256_DIGEST_SIZE, 2575 }, 2576 .caam = { 2577 .class1_alg_type = OP_ALG_ALGSEL_3DES | OP_ALG_AAI_CBC, 2578 .class2_alg_type = OP_ALG_ALGSEL_SHA256 | 2579 OP_ALG_AAI_HMAC_PRECOMP, 2580 }, 2581 }, 2582 { 2583 .aead = { 2584 .base = { 2585 .cra_name = "echainiv(authenc(hmac(sha256)," 2586 "cbc(des3_ede)))", 2587 .cra_driver_name = "echainiv-authenc-" 2588 "hmac-sha256-" 2589 "cbc-des3_ede-caam", 2590 .cra_blocksize = DES3_EDE_BLOCK_SIZE, 2591 }, 2592 .setkey = aead_setkey, 2593 .setauthsize = aead_setauthsize, 2594 .encrypt = aead_encrypt, 2595 .decrypt = aead_decrypt, 2596 .ivsize = DES3_EDE_BLOCK_SIZE, 2597 .maxauthsize = SHA256_DIGEST_SIZE, 2598 }, 2599 .caam = { 2600 .class1_alg_type = OP_ALG_ALGSEL_3DES | OP_ALG_AAI_CBC, 2601 .class2_alg_type = OP_ALG_ALGSEL_SHA256 | 2602 OP_ALG_AAI_HMAC_PRECOMP, 2603 .geniv = true, 2604 }, 2605 }, 2606 { 2607 .aead = { 2608 .base = { 2609 .cra_name = "authenc(hmac(sha384)," 2610 "cbc(des3_ede))", 2611 .cra_driver_name = "authenc-hmac-sha384-" 2612 "cbc-des3_ede-caam", 2613 .cra_blocksize = DES3_EDE_BLOCK_SIZE, 2614 }, 2615 .setkey = aead_setkey, 2616 .setauthsize = aead_setauthsize, 2617 .encrypt = aead_encrypt, 2618 .decrypt = aead_decrypt, 2619 .ivsize = DES3_EDE_BLOCK_SIZE, 2620 .maxauthsize = SHA384_DIGEST_SIZE, 2621 }, 2622 .caam = { 2623 .class1_alg_type = OP_ALG_ALGSEL_3DES | OP_ALG_AAI_CBC, 2624 .class2_alg_type = OP_ALG_ALGSEL_SHA384 | 2625 OP_ALG_AAI_HMAC_PRECOMP, 2626 }, 2627 }, 2628 { 2629 .aead = { 2630 .base = { 2631 .cra_name = "echainiv(authenc(hmac(sha384)," 2632 "cbc(des3_ede)))", 2633 .cra_driver_name = "echainiv-authenc-" 2634 "hmac-sha384-" 2635 "cbc-des3_ede-caam", 2636 .cra_blocksize = DES3_EDE_BLOCK_SIZE, 2637 }, 2638 .setkey = aead_setkey, 2639 .setauthsize = aead_setauthsize, 2640 .encrypt = aead_encrypt, 2641 .decrypt = aead_decrypt, 2642 .ivsize = DES3_EDE_BLOCK_SIZE, 2643 .maxauthsize = SHA384_DIGEST_SIZE, 2644 }, 2645 .caam = { 2646 .class1_alg_type = OP_ALG_ALGSEL_3DES | OP_ALG_AAI_CBC, 2647 .class2_alg_type = OP_ALG_ALGSEL_SHA384 | 2648 OP_ALG_AAI_HMAC_PRECOMP, 2649 .geniv = true, 2650 }, 2651 }, 2652 { 2653 .aead = { 2654 .base = { 2655 .cra_name = "authenc(hmac(sha512)," 2656 "cbc(des3_ede))", 2657 .cra_driver_name = "authenc-hmac-sha512-" 2658 "cbc-des3_ede-caam", 2659 .cra_blocksize = DES3_EDE_BLOCK_SIZE, 2660 }, 2661 .setkey = aead_setkey, 2662 .setauthsize = aead_setauthsize, 2663 .encrypt = aead_encrypt, 2664 .decrypt = aead_decrypt, 2665 .ivsize = DES3_EDE_BLOCK_SIZE, 2666 .maxauthsize = SHA512_DIGEST_SIZE, 2667 }, 2668 .caam = { 2669 .class1_alg_type = OP_ALG_ALGSEL_3DES | OP_ALG_AAI_CBC, 2670 .class2_alg_type = OP_ALG_ALGSEL_SHA512 | 2671 OP_ALG_AAI_HMAC_PRECOMP, 2672 }, 2673 }, 2674 { 2675 .aead = { 2676 .base = { 2677 .cra_name = "echainiv(authenc(hmac(sha512)," 2678 "cbc(des3_ede)))", 2679 .cra_driver_name = "echainiv-authenc-" 2680 "hmac-sha512-" 2681 "cbc-des3_ede-caam", 2682 .cra_blocksize = DES3_EDE_BLOCK_SIZE, 2683 }, 2684 .setkey = aead_setkey, 2685 .setauthsize = aead_setauthsize, 2686 .encrypt = aead_encrypt, 2687 .decrypt = aead_decrypt, 2688 .ivsize = DES3_EDE_BLOCK_SIZE, 2689 .maxauthsize = SHA512_DIGEST_SIZE, 2690 }, 2691 .caam = { 2692 .class1_alg_type = OP_ALG_ALGSEL_3DES | OP_ALG_AAI_CBC, 2693 .class2_alg_type = OP_ALG_ALGSEL_SHA512 | 2694 OP_ALG_AAI_HMAC_PRECOMP, 2695 .geniv = true, 2696 }, 2697 }, 2698 { 2699 .aead = { 2700 .base = { 2701 .cra_name = "authenc(hmac(md5),cbc(des))", 2702 .cra_driver_name = "authenc-hmac-md5-" 2703 "cbc-des-caam", 2704 .cra_blocksize = DES_BLOCK_SIZE, 2705 }, 2706 .setkey = aead_setkey, 2707 .setauthsize = aead_setauthsize, 2708 .encrypt = aead_encrypt, 2709 .decrypt = aead_decrypt, 2710 .ivsize = DES_BLOCK_SIZE, 2711 .maxauthsize = MD5_DIGEST_SIZE, 2712 }, 2713 .caam = { 2714 .class1_alg_type = OP_ALG_ALGSEL_DES | OP_ALG_AAI_CBC, 2715 .class2_alg_type = OP_ALG_ALGSEL_MD5 | 2716 OP_ALG_AAI_HMAC_PRECOMP, 2717 }, 2718 }, 2719 { 2720 .aead = { 2721 .base = { 2722 .cra_name = "echainiv(authenc(hmac(md5)," 2723 "cbc(des)))", 2724 .cra_driver_name = "echainiv-authenc-hmac-md5-" 2725 "cbc-des-caam", 2726 .cra_blocksize = DES_BLOCK_SIZE, 2727 }, 2728 .setkey = aead_setkey, 2729 .setauthsize = aead_setauthsize, 2730 .encrypt = aead_encrypt, 2731 .decrypt = aead_decrypt, 2732 .ivsize = DES_BLOCK_SIZE, 2733 .maxauthsize = MD5_DIGEST_SIZE, 2734 }, 2735 .caam = { 2736 .class1_alg_type = OP_ALG_ALGSEL_DES | OP_ALG_AAI_CBC, 2737 .class2_alg_type = OP_ALG_ALGSEL_MD5 | 2738 OP_ALG_AAI_HMAC_PRECOMP, 2739 .geniv = true, 2740 }, 2741 }, 2742 { 2743 .aead = { 2744 .base = { 2745 .cra_name = "authenc(hmac(sha1),cbc(des))", 2746 .cra_driver_name = "authenc-hmac-sha1-" 2747 "cbc-des-caam", 2748 .cra_blocksize = DES_BLOCK_SIZE, 2749 }, 2750 .setkey = aead_setkey, 2751 .setauthsize = aead_setauthsize, 2752 .encrypt = aead_encrypt, 2753 .decrypt = aead_decrypt, 2754 .ivsize = DES_BLOCK_SIZE, 2755 .maxauthsize = SHA1_DIGEST_SIZE, 2756 }, 2757 .caam = { 2758 .class1_alg_type = OP_ALG_ALGSEL_DES | OP_ALG_AAI_CBC, 2759 .class2_alg_type = OP_ALG_ALGSEL_SHA1 | 2760 OP_ALG_AAI_HMAC_PRECOMP, 2761 }, 2762 }, 2763 { 2764 .aead = { 2765 .base = { 2766 .cra_name = "echainiv(authenc(hmac(sha1)," 2767 "cbc(des)))", 2768 .cra_driver_name = "echainiv-authenc-" 2769 "hmac-sha1-cbc-des-caam", 2770 .cra_blocksize = DES_BLOCK_SIZE, 2771 }, 2772 .setkey = aead_setkey, 2773 .setauthsize = aead_setauthsize, 2774 .encrypt = aead_encrypt, 2775 .decrypt = aead_decrypt, 2776 .ivsize = DES_BLOCK_SIZE, 2777 .maxauthsize = SHA1_DIGEST_SIZE, 2778 }, 2779 .caam = { 2780 .class1_alg_type = OP_ALG_ALGSEL_DES | OP_ALG_AAI_CBC, 2781 .class2_alg_type = OP_ALG_ALGSEL_SHA1 | 2782 OP_ALG_AAI_HMAC_PRECOMP, 2783 .geniv = true, 2784 }, 2785 }, 2786 { 2787 .aead = { 2788 .base = { 2789 .cra_name = "authenc(hmac(sha224),cbc(des))", 2790 .cra_driver_name = "authenc-hmac-sha224-" 2791 "cbc-des-caam", 2792 .cra_blocksize = DES_BLOCK_SIZE, 2793 }, 2794 .setkey = aead_setkey, 2795 .setauthsize = aead_setauthsize, 2796 .encrypt = aead_encrypt, 2797 .decrypt = aead_decrypt, 2798 .ivsize = DES_BLOCK_SIZE, 2799 .maxauthsize = SHA224_DIGEST_SIZE, 2800 }, 2801 .caam = { 2802 .class1_alg_type = OP_ALG_ALGSEL_DES | OP_ALG_AAI_CBC, 2803 .class2_alg_type = OP_ALG_ALGSEL_SHA224 | 2804 OP_ALG_AAI_HMAC_PRECOMP, 2805 }, 2806 }, 2807 { 2808 .aead = { 2809 .base = { 2810 .cra_name = "echainiv(authenc(hmac(sha224)," 2811 "cbc(des)))", 2812 .cra_driver_name = "echainiv-authenc-" 2813 "hmac-sha224-cbc-des-caam", 2814 .cra_blocksize = DES_BLOCK_SIZE, 2815 }, 2816 .setkey = aead_setkey, 2817 .setauthsize = aead_setauthsize, 2818 .encrypt = aead_encrypt, 2819 .decrypt = aead_decrypt, 2820 .ivsize = DES_BLOCK_SIZE, 2821 .maxauthsize = SHA224_DIGEST_SIZE, 2822 }, 2823 .caam = { 2824 .class1_alg_type = OP_ALG_ALGSEL_DES | OP_ALG_AAI_CBC, 2825 .class2_alg_type = OP_ALG_ALGSEL_SHA224 | 2826 OP_ALG_AAI_HMAC_PRECOMP, 2827 .geniv = true, 2828 }, 2829 }, 2830 { 2831 .aead = { 2832 .base = { 2833 .cra_name = "authenc(hmac(sha256),cbc(des))", 2834 .cra_driver_name = "authenc-hmac-sha256-" 2835 "cbc-des-caam", 2836 .cra_blocksize = DES_BLOCK_SIZE, 2837 }, 2838 .setkey = aead_setkey, 2839 .setauthsize = aead_setauthsize, 2840 .encrypt = aead_encrypt, 2841 .decrypt = aead_decrypt, 2842 .ivsize = DES_BLOCK_SIZE, 2843 .maxauthsize = SHA256_DIGEST_SIZE, 2844 }, 2845 .caam = { 2846 .class1_alg_type = OP_ALG_ALGSEL_DES | OP_ALG_AAI_CBC, 2847 .class2_alg_type = OP_ALG_ALGSEL_SHA256 | 2848 OP_ALG_AAI_HMAC_PRECOMP, 2849 }, 2850 }, 2851 { 2852 .aead = { 2853 .base = { 2854 .cra_name = "echainiv(authenc(hmac(sha256)," 2855 "cbc(des)))", 2856 .cra_driver_name = "echainiv-authenc-" 2857 "hmac-sha256-cbc-des-caam", 2858 .cra_blocksize = DES_BLOCK_SIZE, 2859 }, 2860 .setkey = aead_setkey, 2861 .setauthsize = aead_setauthsize, 2862 .encrypt = aead_encrypt, 2863 .decrypt = aead_decrypt, 2864 .ivsize = DES_BLOCK_SIZE, 2865 .maxauthsize = SHA256_DIGEST_SIZE, 2866 }, 2867 .caam = { 2868 .class1_alg_type = OP_ALG_ALGSEL_DES | OP_ALG_AAI_CBC, 2869 .class2_alg_type = OP_ALG_ALGSEL_SHA256 | 2870 OP_ALG_AAI_HMAC_PRECOMP, 2871 .geniv = true, 2872 }, 2873 }, 2874 { 2875 .aead = { 2876 .base = { 2877 .cra_name = "authenc(hmac(sha384),cbc(des))", 2878 .cra_driver_name = "authenc-hmac-sha384-" 2879 "cbc-des-caam", 2880 .cra_blocksize = DES_BLOCK_SIZE, 2881 }, 2882 .setkey = aead_setkey, 2883 .setauthsize = aead_setauthsize, 2884 .encrypt = aead_encrypt, 2885 .decrypt = aead_decrypt, 2886 .ivsize = DES_BLOCK_SIZE, 2887 .maxauthsize = SHA384_DIGEST_SIZE, 2888 }, 2889 .caam = { 2890 .class1_alg_type = OP_ALG_ALGSEL_DES | OP_ALG_AAI_CBC, 2891 .class2_alg_type = OP_ALG_ALGSEL_SHA384 | 2892 OP_ALG_AAI_HMAC_PRECOMP, 2893 }, 2894 }, 2895 { 2896 .aead = { 2897 .base = { 2898 .cra_name = "echainiv(authenc(hmac(sha384)," 2899 "cbc(des)))", 2900 .cra_driver_name = "echainiv-authenc-" 2901 "hmac-sha384-cbc-des-caam", 2902 .cra_blocksize = DES_BLOCK_SIZE, 2903 }, 2904 .setkey = aead_setkey, 2905 .setauthsize = aead_setauthsize, 2906 .encrypt = aead_encrypt, 2907 .decrypt = aead_decrypt, 2908 .ivsize = DES_BLOCK_SIZE, 2909 .maxauthsize = SHA384_DIGEST_SIZE, 2910 }, 2911 .caam = { 2912 .class1_alg_type = OP_ALG_ALGSEL_DES | OP_ALG_AAI_CBC, 2913 .class2_alg_type = OP_ALG_ALGSEL_SHA384 | 2914 OP_ALG_AAI_HMAC_PRECOMP, 2915 .geniv = true, 2916 }, 2917 }, 2918 { 2919 .aead = { 2920 .base = { 2921 .cra_name = "authenc(hmac(sha512),cbc(des))", 2922 .cra_driver_name = "authenc-hmac-sha512-" 2923 "cbc-des-caam", 2924 .cra_blocksize = DES_BLOCK_SIZE, 2925 }, 2926 .setkey = aead_setkey, 2927 .setauthsize = aead_setauthsize, 2928 .encrypt = aead_encrypt, 2929 .decrypt = aead_decrypt, 2930 .ivsize = DES_BLOCK_SIZE, 2931 .maxauthsize = SHA512_DIGEST_SIZE, 2932 }, 2933 .caam = { 2934 .class1_alg_type = OP_ALG_ALGSEL_DES | OP_ALG_AAI_CBC, 2935 .class2_alg_type = OP_ALG_ALGSEL_SHA512 | 2936 OP_ALG_AAI_HMAC_PRECOMP, 2937 }, 2938 }, 2939 { 2940 .aead = { 2941 .base = { 2942 .cra_name = "echainiv(authenc(hmac(sha512)," 2943 "cbc(des)))", 2944 .cra_driver_name = "echainiv-authenc-" 2945 "hmac-sha512-cbc-des-caam", 2946 .cra_blocksize = DES_BLOCK_SIZE, 2947 }, 2948 .setkey = aead_setkey, 2949 .setauthsize = aead_setauthsize, 2950 .encrypt = aead_encrypt, 2951 .decrypt = aead_decrypt, 2952 .ivsize = DES_BLOCK_SIZE, 2953 .maxauthsize = SHA512_DIGEST_SIZE, 2954 }, 2955 .caam = { 2956 .class1_alg_type = OP_ALG_ALGSEL_DES | OP_ALG_AAI_CBC, 2957 .class2_alg_type = OP_ALG_ALGSEL_SHA512 | 2958 OP_ALG_AAI_HMAC_PRECOMP, 2959 .geniv = true, 2960 }, 2961 }, 2962 { 2963 .aead = { 2964 .base = { 2965 .cra_name = "authenc(hmac(md5)," 2966 "rfc3686(ctr(aes)))", 2967 .cra_driver_name = "authenc-hmac-md5-" 2968 "rfc3686-ctr-aes-caam", 2969 .cra_blocksize = 1, 2970 }, 2971 .setkey = aead_setkey, 2972 .setauthsize = aead_setauthsize, 2973 .encrypt = aead_encrypt, 2974 .decrypt = aead_decrypt, 2975 .ivsize = CTR_RFC3686_IV_SIZE, 2976 .maxauthsize = MD5_DIGEST_SIZE, 2977 }, 2978 .caam = { 2979 .class1_alg_type = OP_ALG_ALGSEL_AES | 2980 OP_ALG_AAI_CTR_MOD128, 2981 .class2_alg_type = OP_ALG_ALGSEL_MD5 | 2982 OP_ALG_AAI_HMAC_PRECOMP, 2983 .rfc3686 = true, 2984 }, 2985 }, 2986 { 2987 .aead = { 2988 .base = { 2989 .cra_name = "seqiv(authenc(" 2990 "hmac(md5),rfc3686(ctr(aes))))", 2991 .cra_driver_name = "seqiv-authenc-hmac-md5-" 2992 "rfc3686-ctr-aes-caam", 2993 .cra_blocksize = 1, 2994 }, 2995 .setkey = aead_setkey, 2996 .setauthsize = aead_setauthsize, 2997 .encrypt = aead_encrypt, 2998 .decrypt = aead_decrypt, 2999 .ivsize = CTR_RFC3686_IV_SIZE, 3000 .maxauthsize = MD5_DIGEST_SIZE, 3001 }, 3002 .caam = { 3003 .class1_alg_type = OP_ALG_ALGSEL_AES | 3004 OP_ALG_AAI_CTR_MOD128, 3005 .class2_alg_type = OP_ALG_ALGSEL_MD5 | 3006 OP_ALG_AAI_HMAC_PRECOMP, 3007 .rfc3686 = true, 3008 .geniv = true, 3009 }, 3010 }, 3011 { 3012 .aead = { 3013 .base = { 3014 .cra_name = "authenc(hmac(sha1)," 3015 "rfc3686(ctr(aes)))", 3016 .cra_driver_name = "authenc-hmac-sha1-" 3017 "rfc3686-ctr-aes-caam", 3018 .cra_blocksize = 1, 3019 }, 3020 .setkey = aead_setkey, 3021 .setauthsize = aead_setauthsize, 3022 .encrypt = aead_encrypt, 3023 .decrypt = aead_decrypt, 3024 .ivsize = CTR_RFC3686_IV_SIZE, 3025 .maxauthsize = SHA1_DIGEST_SIZE, 3026 }, 3027 .caam = { 3028 .class1_alg_type = OP_ALG_ALGSEL_AES | 3029 OP_ALG_AAI_CTR_MOD128, 3030 .class2_alg_type = OP_ALG_ALGSEL_SHA1 | 3031 OP_ALG_AAI_HMAC_PRECOMP, 3032 .rfc3686 = true, 3033 }, 3034 }, 3035 { 3036 .aead = { 3037 .base = { 3038 .cra_name = "seqiv(authenc(" 3039 "hmac(sha1),rfc3686(ctr(aes))))", 3040 .cra_driver_name = "seqiv-authenc-hmac-sha1-" 3041 "rfc3686-ctr-aes-caam", 3042 .cra_blocksize = 1, 3043 }, 3044 .setkey = aead_setkey, 3045 .setauthsize = aead_setauthsize, 3046 .encrypt = aead_encrypt, 3047 .decrypt = aead_decrypt, 3048 .ivsize = CTR_RFC3686_IV_SIZE, 3049 .maxauthsize = SHA1_DIGEST_SIZE, 3050 }, 3051 .caam = { 3052 .class1_alg_type = OP_ALG_ALGSEL_AES | 3053 OP_ALG_AAI_CTR_MOD128, 3054 .class2_alg_type = OP_ALG_ALGSEL_SHA1 | 3055 OP_ALG_AAI_HMAC_PRECOMP, 3056 .rfc3686 = true, 3057 .geniv = true, 3058 }, 3059 }, 3060 { 3061 .aead = { 3062 .base = { 3063 .cra_name = "authenc(hmac(sha224)," 3064 "rfc3686(ctr(aes)))", 3065 .cra_driver_name = "authenc-hmac-sha224-" 3066 "rfc3686-ctr-aes-caam", 3067 .cra_blocksize = 1, 3068 }, 3069 .setkey = aead_setkey, 3070 .setauthsize = aead_setauthsize, 3071 .encrypt = aead_encrypt, 3072 .decrypt = aead_decrypt, 3073 .ivsize = CTR_RFC3686_IV_SIZE, 3074 .maxauthsize = SHA224_DIGEST_SIZE, 3075 }, 3076 .caam = { 3077 .class1_alg_type = OP_ALG_ALGSEL_AES | 3078 OP_ALG_AAI_CTR_MOD128, 3079 .class2_alg_type = OP_ALG_ALGSEL_SHA224 | 3080 OP_ALG_AAI_HMAC_PRECOMP, 3081 .rfc3686 = true, 3082 }, 3083 }, 3084 { 3085 .aead = { 3086 .base = { 3087 .cra_name = "seqiv(authenc(" 3088 "hmac(sha224),rfc3686(ctr(aes))))", 3089 .cra_driver_name = "seqiv-authenc-hmac-sha224-" 3090 "rfc3686-ctr-aes-caam", 3091 .cra_blocksize = 1, 3092 }, 3093 .setkey = aead_setkey, 3094 .setauthsize = aead_setauthsize, 3095 .encrypt = aead_encrypt, 3096 .decrypt = aead_decrypt, 3097 .ivsize = CTR_RFC3686_IV_SIZE, 3098 .maxauthsize = SHA224_DIGEST_SIZE, 3099 }, 3100 .caam = { 3101 .class1_alg_type = OP_ALG_ALGSEL_AES | 3102 OP_ALG_AAI_CTR_MOD128, 3103 .class2_alg_type = OP_ALG_ALGSEL_SHA224 | 3104 OP_ALG_AAI_HMAC_PRECOMP, 3105 .rfc3686 = true, 3106 .geniv = true, 3107 }, 3108 }, 3109 { 3110 .aead = { 3111 .base = { 3112 .cra_name = "authenc(hmac(sha256)," 3113 "rfc3686(ctr(aes)))", 3114 .cra_driver_name = "authenc-hmac-sha256-" 3115 "rfc3686-ctr-aes-caam", 3116 .cra_blocksize = 1, 3117 }, 3118 .setkey = aead_setkey, 3119 .setauthsize = aead_setauthsize, 3120 .encrypt = aead_encrypt, 3121 .decrypt = aead_decrypt, 3122 .ivsize = CTR_RFC3686_IV_SIZE, 3123 .maxauthsize = SHA256_DIGEST_SIZE, 3124 }, 3125 .caam = { 3126 .class1_alg_type = OP_ALG_ALGSEL_AES | 3127 OP_ALG_AAI_CTR_MOD128, 3128 .class2_alg_type = OP_ALG_ALGSEL_SHA256 | 3129 OP_ALG_AAI_HMAC_PRECOMP, 3130 .rfc3686 = true, 3131 }, 3132 }, 3133 { 3134 .aead = { 3135 .base = { 3136 .cra_name = "seqiv(authenc(hmac(sha256)," 3137 "rfc3686(ctr(aes))))", 3138 .cra_driver_name = "seqiv-authenc-hmac-sha256-" 3139 "rfc3686-ctr-aes-caam", 3140 .cra_blocksize = 1, 3141 }, 3142 .setkey = aead_setkey, 3143 .setauthsize = aead_setauthsize, 3144 .encrypt = aead_encrypt, 3145 .decrypt = aead_decrypt, 3146 .ivsize = CTR_RFC3686_IV_SIZE, 3147 .maxauthsize = SHA256_DIGEST_SIZE, 3148 }, 3149 .caam = { 3150 .class1_alg_type = OP_ALG_ALGSEL_AES | 3151 OP_ALG_AAI_CTR_MOD128, 3152 .class2_alg_type = OP_ALG_ALGSEL_SHA256 | 3153 OP_ALG_AAI_HMAC_PRECOMP, 3154 .rfc3686 = true, 3155 .geniv = true, 3156 }, 3157 }, 3158 { 3159 .aead = { 3160 .base = { 3161 .cra_name = "authenc(hmac(sha384)," 3162 "rfc3686(ctr(aes)))", 3163 .cra_driver_name = "authenc-hmac-sha384-" 3164 "rfc3686-ctr-aes-caam", 3165 .cra_blocksize = 1, 3166 }, 3167 .setkey = aead_setkey, 3168 .setauthsize = aead_setauthsize, 3169 .encrypt = aead_encrypt, 3170 .decrypt = aead_decrypt, 3171 .ivsize = CTR_RFC3686_IV_SIZE, 3172 .maxauthsize = SHA384_DIGEST_SIZE, 3173 }, 3174 .caam = { 3175 .class1_alg_type = OP_ALG_ALGSEL_AES | 3176 OP_ALG_AAI_CTR_MOD128, 3177 .class2_alg_type = OP_ALG_ALGSEL_SHA384 | 3178 OP_ALG_AAI_HMAC_PRECOMP, 3179 .rfc3686 = true, 3180 }, 3181 }, 3182 { 3183 .aead = { 3184 .base = { 3185 .cra_name = "seqiv(authenc(hmac(sha384)," 3186 "rfc3686(ctr(aes))))", 3187 .cra_driver_name = "seqiv-authenc-hmac-sha384-" 3188 "rfc3686-ctr-aes-caam", 3189 .cra_blocksize = 1, 3190 }, 3191 .setkey = aead_setkey, 3192 .setauthsize = aead_setauthsize, 3193 .encrypt = aead_encrypt, 3194 .decrypt = aead_decrypt, 3195 .ivsize = CTR_RFC3686_IV_SIZE, 3196 .maxauthsize = SHA384_DIGEST_SIZE, 3197 }, 3198 .caam = { 3199 .class1_alg_type = OP_ALG_ALGSEL_AES | 3200 OP_ALG_AAI_CTR_MOD128, 3201 .class2_alg_type = OP_ALG_ALGSEL_SHA384 | 3202 OP_ALG_AAI_HMAC_PRECOMP, 3203 .rfc3686 = true, 3204 .geniv = true, 3205 }, 3206 }, 3207 { 3208 .aead = { 3209 .base = { 3210 .cra_name = "authenc(hmac(sha512)," 3211 "rfc3686(ctr(aes)))", 3212 .cra_driver_name = "authenc-hmac-sha512-" 3213 "rfc3686-ctr-aes-caam", 3214 .cra_blocksize = 1, 3215 }, 3216 .setkey = aead_setkey, 3217 .setauthsize = aead_setauthsize, 3218 .encrypt = aead_encrypt, 3219 .decrypt = aead_decrypt, 3220 .ivsize = CTR_RFC3686_IV_SIZE, 3221 .maxauthsize = SHA512_DIGEST_SIZE, 3222 }, 3223 .caam = { 3224 .class1_alg_type = OP_ALG_ALGSEL_AES | 3225 OP_ALG_AAI_CTR_MOD128, 3226 .class2_alg_type = OP_ALG_ALGSEL_SHA512 | 3227 OP_ALG_AAI_HMAC_PRECOMP, 3228 .rfc3686 = true, 3229 }, 3230 }, 3231 { 3232 .aead = { 3233 .base = { 3234 .cra_name = "seqiv(authenc(hmac(sha512)," 3235 "rfc3686(ctr(aes))))", 3236 .cra_driver_name = "seqiv-authenc-hmac-sha512-" 3237 "rfc3686-ctr-aes-caam", 3238 .cra_blocksize = 1, 3239 }, 3240 .setkey = aead_setkey, 3241 .setauthsize = aead_setauthsize, 3242 .encrypt = aead_encrypt, 3243 .decrypt = aead_decrypt, 3244 .ivsize = CTR_RFC3686_IV_SIZE, 3245 .maxauthsize = SHA512_DIGEST_SIZE, 3246 }, 3247 .caam = { 3248 .class1_alg_type = OP_ALG_ALGSEL_AES | 3249 OP_ALG_AAI_CTR_MOD128, 3250 .class2_alg_type = OP_ALG_ALGSEL_SHA512 | 3251 OP_ALG_AAI_HMAC_PRECOMP, 3252 .rfc3686 = true, 3253 .geniv = true, 3254 }, 3255 }, 3256 }; 3257 3258 struct caam_crypto_alg { 3259 struct crypto_alg crypto_alg; 3260 struct list_head entry; 3261 struct caam_alg_entry caam; 3262 }; 3263 3264 static int caam_init_common(struct caam_ctx *ctx, struct caam_alg_entry *caam, 3265 bool uses_dkp) 3266 { 3267 dma_addr_t dma_addr; 3268 struct caam_drv_private *priv; 3269 3270 ctx->jrdev = caam_jr_alloc(); 3271 if (IS_ERR(ctx->jrdev)) { 3272 pr_err("Job Ring Device allocation for transform failed\n"); 3273 return PTR_ERR(ctx->jrdev); 3274 } 3275 3276 priv = dev_get_drvdata(ctx->jrdev->parent); 3277 if (priv->era >= 6 && uses_dkp) 3278 ctx->dir = DMA_BIDIRECTIONAL; 3279 else 3280 ctx->dir = DMA_TO_DEVICE; 3281 3282 dma_addr = dma_map_single_attrs(ctx->jrdev, ctx->sh_desc_enc, 3283 offsetof(struct caam_ctx, 3284 sh_desc_enc_dma), 3285 ctx->dir, DMA_ATTR_SKIP_CPU_SYNC); 3286 if (dma_mapping_error(ctx->jrdev, dma_addr)) { 3287 dev_err(ctx->jrdev, "unable to map key, shared descriptors\n"); 3288 caam_jr_free(ctx->jrdev); 3289 return -ENOMEM; 3290 } 3291 3292 ctx->sh_desc_enc_dma = dma_addr; 3293 ctx->sh_desc_dec_dma = dma_addr + offsetof(struct caam_ctx, 3294 sh_desc_dec); 3295 ctx->sh_desc_givenc_dma = dma_addr + offsetof(struct caam_ctx, 3296 sh_desc_givenc); 3297 ctx->key_dma = dma_addr + offsetof(struct caam_ctx, key); 3298 3299 /* copy descriptor header template value */ 3300 ctx->cdata.algtype = OP_TYPE_CLASS1_ALG | caam->class1_alg_type; 3301 ctx->adata.algtype = OP_TYPE_CLASS2_ALG | caam->class2_alg_type; 3302 3303 return 0; 3304 } 3305 3306 static int caam_cra_init(struct crypto_tfm *tfm) 3307 { 3308 struct crypto_alg *alg = tfm->__crt_alg; 3309 struct caam_crypto_alg *caam_alg = 3310 container_of(alg, struct caam_crypto_alg, crypto_alg); 3311 struct caam_ctx *ctx = crypto_tfm_ctx(tfm); 3312 3313 return caam_init_common(ctx, &caam_alg->caam, false); 3314 } 3315 3316 static int caam_aead_init(struct crypto_aead *tfm) 3317 { 3318 struct aead_alg *alg = crypto_aead_alg(tfm); 3319 struct caam_aead_alg *caam_alg = 3320 container_of(alg, struct caam_aead_alg, aead); 3321 struct caam_ctx *ctx = crypto_aead_ctx(tfm); 3322 3323 return caam_init_common(ctx, &caam_alg->caam, 3324 alg->setkey == aead_setkey); 3325 } 3326 3327 static void caam_exit_common(struct caam_ctx *ctx) 3328 { 3329 dma_unmap_single_attrs(ctx->jrdev, ctx->sh_desc_enc_dma, 3330 offsetof(struct caam_ctx, sh_desc_enc_dma), 3331 ctx->dir, DMA_ATTR_SKIP_CPU_SYNC); 3332 caam_jr_free(ctx->jrdev); 3333 } 3334 3335 static void caam_cra_exit(struct crypto_tfm *tfm) 3336 { 3337 caam_exit_common(crypto_tfm_ctx(tfm)); 3338 } 3339 3340 static void caam_aead_exit(struct crypto_aead *tfm) 3341 { 3342 caam_exit_common(crypto_aead_ctx(tfm)); 3343 } 3344 3345 static void __exit caam_algapi_exit(void) 3346 { 3347 3348 struct caam_crypto_alg *t_alg, *n; 3349 int i; 3350 3351 for (i = 0; i < ARRAY_SIZE(driver_aeads); i++) { 3352 struct caam_aead_alg *t_alg = driver_aeads + i; 3353 3354 if (t_alg->registered) 3355 crypto_unregister_aead(&t_alg->aead); 3356 } 3357 3358 if (!alg_list.next) 3359 return; 3360 3361 list_for_each_entry_safe(t_alg, n, &alg_list, entry) { 3362 crypto_unregister_alg(&t_alg->crypto_alg); 3363 list_del(&t_alg->entry); 3364 kfree(t_alg); 3365 } 3366 } 3367 3368 static struct caam_crypto_alg *caam_alg_alloc(struct caam_alg_template 3369 *template) 3370 { 3371 struct caam_crypto_alg *t_alg; 3372 struct crypto_alg *alg; 3373 3374 t_alg = kzalloc(sizeof(*t_alg), GFP_KERNEL); 3375 if (!t_alg) { 3376 pr_err("failed to allocate t_alg\n"); 3377 return ERR_PTR(-ENOMEM); 3378 } 3379 3380 alg = &t_alg->crypto_alg; 3381 3382 snprintf(alg->cra_name, CRYPTO_MAX_ALG_NAME, "%s", template->name); 3383 snprintf(alg->cra_driver_name, CRYPTO_MAX_ALG_NAME, "%s", 3384 template->driver_name); 3385 alg->cra_module = THIS_MODULE; 3386 alg->cra_init = caam_cra_init; 3387 alg->cra_exit = caam_cra_exit; 3388 alg->cra_priority = CAAM_CRA_PRIORITY; 3389 alg->cra_blocksize = template->blocksize; 3390 alg->cra_alignmask = 0; 3391 alg->cra_ctxsize = sizeof(struct caam_ctx); 3392 alg->cra_flags = CRYPTO_ALG_ASYNC | CRYPTO_ALG_KERN_DRIVER_ONLY | 3393 template->type; 3394 switch (template->type) { 3395 case CRYPTO_ALG_TYPE_GIVCIPHER: 3396 alg->cra_type = &crypto_givcipher_type; 3397 alg->cra_ablkcipher = template->template_ablkcipher; 3398 break; 3399 case CRYPTO_ALG_TYPE_ABLKCIPHER: 3400 alg->cra_type = &crypto_ablkcipher_type; 3401 alg->cra_ablkcipher = template->template_ablkcipher; 3402 break; 3403 } 3404 3405 t_alg->caam.class1_alg_type = template->class1_alg_type; 3406 t_alg->caam.class2_alg_type = template->class2_alg_type; 3407 3408 return t_alg; 3409 } 3410 3411 static void caam_aead_alg_init(struct caam_aead_alg *t_alg) 3412 { 3413 struct aead_alg *alg = &t_alg->aead; 3414 3415 alg->base.cra_module = THIS_MODULE; 3416 alg->base.cra_priority = CAAM_CRA_PRIORITY; 3417 alg->base.cra_ctxsize = sizeof(struct caam_ctx); 3418 alg->base.cra_flags = CRYPTO_ALG_ASYNC | CRYPTO_ALG_KERN_DRIVER_ONLY; 3419 3420 alg->init = caam_aead_init; 3421 alg->exit = caam_aead_exit; 3422 } 3423 3424 static int __init caam_algapi_init(void) 3425 { 3426 struct device_node *dev_node; 3427 struct platform_device *pdev; 3428 struct device *ctrldev; 3429 struct caam_drv_private *priv; 3430 int i = 0, err = 0; 3431 u32 cha_vid, cha_inst, des_inst, aes_inst, md_inst; 3432 unsigned int md_limit = SHA512_DIGEST_SIZE; 3433 bool registered = false; 3434 3435 dev_node = of_find_compatible_node(NULL, NULL, "fsl,sec-v4.0"); 3436 if (!dev_node) { 3437 dev_node = of_find_compatible_node(NULL, NULL, "fsl,sec4.0"); 3438 if (!dev_node) 3439 return -ENODEV; 3440 } 3441 3442 pdev = of_find_device_by_node(dev_node); 3443 if (!pdev) { 3444 of_node_put(dev_node); 3445 return -ENODEV; 3446 } 3447 3448 ctrldev = &pdev->dev; 3449 priv = dev_get_drvdata(ctrldev); 3450 of_node_put(dev_node); 3451 3452 /* 3453 * If priv is NULL, it's probably because the caam driver wasn't 3454 * properly initialized (e.g. RNG4 init failed). Thus, bail out here. 3455 */ 3456 if (!priv) 3457 return -ENODEV; 3458 3459 3460 INIT_LIST_HEAD(&alg_list); 3461 3462 /* 3463 * Register crypto algorithms the device supports. 3464 * First, detect presence and attributes of DES, AES, and MD blocks. 3465 */ 3466 cha_vid = rd_reg32(&priv->ctrl->perfmon.cha_id_ls); 3467 cha_inst = rd_reg32(&priv->ctrl->perfmon.cha_num_ls); 3468 des_inst = (cha_inst & CHA_ID_LS_DES_MASK) >> CHA_ID_LS_DES_SHIFT; 3469 aes_inst = (cha_inst & CHA_ID_LS_AES_MASK) >> CHA_ID_LS_AES_SHIFT; 3470 md_inst = (cha_inst & CHA_ID_LS_MD_MASK) >> CHA_ID_LS_MD_SHIFT; 3471 3472 /* If MD is present, limit digest size based on LP256 */ 3473 if (md_inst && ((cha_vid & CHA_ID_LS_MD_MASK) == CHA_ID_LS_MD_LP256)) 3474 md_limit = SHA256_DIGEST_SIZE; 3475 3476 for (i = 0; i < ARRAY_SIZE(driver_algs); i++) { 3477 struct caam_crypto_alg *t_alg; 3478 struct caam_alg_template *alg = driver_algs + i; 3479 u32 alg_sel = alg->class1_alg_type & OP_ALG_ALGSEL_MASK; 3480 3481 /* Skip DES algorithms if not supported by device */ 3482 if (!des_inst && 3483 ((alg_sel == OP_ALG_ALGSEL_3DES) || 3484 (alg_sel == OP_ALG_ALGSEL_DES))) 3485 continue; 3486 3487 /* Skip AES algorithms if not supported by device */ 3488 if (!aes_inst && (alg_sel == OP_ALG_ALGSEL_AES)) 3489 continue; 3490 3491 /* 3492 * Check support for AES modes not available 3493 * on LP devices. 3494 */ 3495 if ((cha_vid & CHA_ID_LS_AES_MASK) == CHA_ID_LS_AES_LP) 3496 if ((alg->class1_alg_type & OP_ALG_AAI_MASK) == 3497 OP_ALG_AAI_XTS) 3498 continue; 3499 3500 t_alg = caam_alg_alloc(alg); 3501 if (IS_ERR(t_alg)) { 3502 err = PTR_ERR(t_alg); 3503 pr_warn("%s alg allocation failed\n", alg->driver_name); 3504 continue; 3505 } 3506 3507 err = crypto_register_alg(&t_alg->crypto_alg); 3508 if (err) { 3509 pr_warn("%s alg registration failed\n", 3510 t_alg->crypto_alg.cra_driver_name); 3511 kfree(t_alg); 3512 continue; 3513 } 3514 3515 list_add_tail(&t_alg->entry, &alg_list); 3516 registered = true; 3517 } 3518 3519 for (i = 0; i < ARRAY_SIZE(driver_aeads); i++) { 3520 struct caam_aead_alg *t_alg = driver_aeads + i; 3521 u32 c1_alg_sel = t_alg->caam.class1_alg_type & 3522 OP_ALG_ALGSEL_MASK; 3523 u32 c2_alg_sel = t_alg->caam.class2_alg_type & 3524 OP_ALG_ALGSEL_MASK; 3525 u32 alg_aai = t_alg->caam.class1_alg_type & OP_ALG_AAI_MASK; 3526 3527 /* Skip DES algorithms if not supported by device */ 3528 if (!des_inst && 3529 ((c1_alg_sel == OP_ALG_ALGSEL_3DES) || 3530 (c1_alg_sel == OP_ALG_ALGSEL_DES))) 3531 continue; 3532 3533 /* Skip AES algorithms if not supported by device */ 3534 if (!aes_inst && (c1_alg_sel == OP_ALG_ALGSEL_AES)) 3535 continue; 3536 3537 /* 3538 * Check support for AES algorithms not available 3539 * on LP devices. 3540 */ 3541 if ((cha_vid & CHA_ID_LS_AES_MASK) == CHA_ID_LS_AES_LP) 3542 if (alg_aai == OP_ALG_AAI_GCM) 3543 continue; 3544 3545 /* 3546 * Skip algorithms requiring message digests 3547 * if MD or MD size is not supported by device. 3548 */ 3549 if (c2_alg_sel && 3550 (!md_inst || (t_alg->aead.maxauthsize > md_limit))) 3551 continue; 3552 3553 caam_aead_alg_init(t_alg); 3554 3555 err = crypto_register_aead(&t_alg->aead); 3556 if (err) { 3557 pr_warn("%s alg registration failed\n", 3558 t_alg->aead.base.cra_driver_name); 3559 continue; 3560 } 3561 3562 t_alg->registered = true; 3563 registered = true; 3564 } 3565 3566 if (registered) 3567 pr_info("caam algorithms registered in /proc/crypto\n"); 3568 3569 return err; 3570 } 3571 3572 module_init(caam_algapi_init); 3573 module_exit(caam_algapi_exit); 3574 3575 MODULE_LICENSE("GPL"); 3576 MODULE_DESCRIPTION("FSL CAAM support for crypto API"); 3577 MODULE_AUTHOR("Freescale Semiconductor - NMG/STC"); 3578