1 /* 2 * caam - Freescale FSL CAAM support for crypto API 3 * 4 * Copyright 2008-2011 Freescale Semiconductor, Inc. 5 * Copyright 2016 NXP 6 * 7 * Based on talitos crypto API driver. 8 * 9 * relationship of job descriptors to shared descriptors (SteveC Dec 10 2008): 10 * 11 * --------------- --------------- 12 * | JobDesc #1 |-------------------->| ShareDesc | 13 * | *(packet 1) | | (PDB) | 14 * --------------- |------------->| (hashKey) | 15 * . | | (cipherKey) | 16 * . | |-------->| (operation) | 17 * --------------- | | --------------- 18 * | JobDesc #2 |------| | 19 * | *(packet 2) | | 20 * --------------- | 21 * . | 22 * . | 23 * --------------- | 24 * | JobDesc #3 |------------ 25 * | *(packet 3) | 26 * --------------- 27 * 28 * The SharedDesc never changes for a connection unless rekeyed, but 29 * each packet will likely be in a different place. So all we need 30 * to know to process the packet is where the input is, where the 31 * output goes, and what context we want to process with. Context is 32 * in the SharedDesc, packet references in the JobDesc. 33 * 34 * So, a job desc looks like: 35 * 36 * --------------------- 37 * | Header | 38 * | ShareDesc Pointer | 39 * | SEQ_OUT_PTR | 40 * | (output buffer) | 41 * | (output length) | 42 * | SEQ_IN_PTR | 43 * | (input buffer) | 44 * | (input length) | 45 * --------------------- 46 */ 47 48 #include "compat.h" 49 50 #include "regs.h" 51 #include "intern.h" 52 #include "desc_constr.h" 53 #include "jr.h" 54 #include "error.h" 55 #include "sg_sw_sec4.h" 56 #include "key_gen.h" 57 #include "caamalg_desc.h" 58 59 /* 60 * crypto alg 61 */ 62 #define CAAM_CRA_PRIORITY 3000 63 /* max key is sum of AES_MAX_KEY_SIZE, max split key size */ 64 #define CAAM_MAX_KEY_SIZE (AES_MAX_KEY_SIZE + \ 65 CTR_RFC3686_NONCE_SIZE + \ 66 SHA512_DIGEST_SIZE * 2) 67 68 #define AEAD_DESC_JOB_IO_LEN (DESC_JOB_IO_LEN + CAAM_CMD_SZ * 2) 69 #define GCM_DESC_JOB_IO_LEN (AEAD_DESC_JOB_IO_LEN + \ 70 CAAM_CMD_SZ * 4) 71 #define AUTHENC_DESC_JOB_IO_LEN (AEAD_DESC_JOB_IO_LEN + \ 72 CAAM_CMD_SZ * 5) 73 74 #define DESC_MAX_USED_BYTES (CAAM_DESC_BYTES_MAX - DESC_JOB_IO_LEN) 75 #define DESC_MAX_USED_LEN (DESC_MAX_USED_BYTES / CAAM_CMD_SZ) 76 77 #ifdef DEBUG 78 /* for print_hex_dumps with line references */ 79 #define debug(format, arg...) printk(format, arg) 80 #else 81 #define debug(format, arg...) 82 #endif 83 84 #ifdef DEBUG 85 #include <linux/highmem.h> 86 87 static void dbg_dump_sg(const char *level, const char *prefix_str, 88 int prefix_type, int rowsize, int groupsize, 89 struct scatterlist *sg, size_t tlen, bool ascii) 90 { 91 struct scatterlist *it; 92 void *it_page; 93 size_t len; 94 void *buf; 95 96 for (it = sg; it != NULL && tlen > 0 ; it = sg_next(sg)) { 97 /* 98 * make sure the scatterlist's page 99 * has a valid virtual memory mapping 100 */ 101 it_page = kmap_atomic(sg_page(it)); 102 if (unlikely(!it_page)) { 103 printk(KERN_ERR "dbg_dump_sg: kmap failed\n"); 104 return; 105 } 106 107 buf = it_page + it->offset; 108 len = min_t(size_t, tlen, it->length); 109 print_hex_dump(level, prefix_str, prefix_type, rowsize, 110 groupsize, buf, len, ascii); 111 tlen -= len; 112 113 kunmap_atomic(it_page); 114 } 115 } 116 #endif 117 118 static struct list_head alg_list; 119 120 struct caam_alg_entry { 121 int class1_alg_type; 122 int class2_alg_type; 123 bool rfc3686; 124 bool geniv; 125 }; 126 127 struct caam_aead_alg { 128 struct aead_alg aead; 129 struct caam_alg_entry caam; 130 bool registered; 131 }; 132 133 /* 134 * per-session context 135 */ 136 struct caam_ctx { 137 u32 sh_desc_enc[DESC_MAX_USED_LEN]; 138 u32 sh_desc_dec[DESC_MAX_USED_LEN]; 139 u32 sh_desc_givenc[DESC_MAX_USED_LEN]; 140 u8 key[CAAM_MAX_KEY_SIZE]; 141 dma_addr_t sh_desc_enc_dma; 142 dma_addr_t sh_desc_dec_dma; 143 dma_addr_t sh_desc_givenc_dma; 144 dma_addr_t key_dma; 145 struct device *jrdev; 146 struct alginfo adata; 147 struct alginfo cdata; 148 unsigned int authsize; 149 }; 150 151 static int aead_null_set_sh_desc(struct crypto_aead *aead) 152 { 153 struct caam_ctx *ctx = crypto_aead_ctx(aead); 154 struct device *jrdev = ctx->jrdev; 155 u32 *desc; 156 int rem_bytes = CAAM_DESC_BYTES_MAX - AEAD_DESC_JOB_IO_LEN - 157 ctx->adata.keylen_pad; 158 159 /* 160 * Job Descriptor and Shared Descriptors 161 * must all fit into the 64-word Descriptor h/w Buffer 162 */ 163 if (rem_bytes >= DESC_AEAD_NULL_ENC_LEN) { 164 ctx->adata.key_inline = true; 165 ctx->adata.key_virt = ctx->key; 166 } else { 167 ctx->adata.key_inline = false; 168 ctx->adata.key_dma = ctx->key_dma; 169 } 170 171 /* aead_encrypt shared descriptor */ 172 desc = ctx->sh_desc_enc; 173 cnstr_shdsc_aead_null_encap(desc, &ctx->adata, ctx->authsize); 174 dma_sync_single_for_device(jrdev, ctx->sh_desc_enc_dma, 175 desc_bytes(desc), DMA_TO_DEVICE); 176 177 /* 178 * Job Descriptor and Shared Descriptors 179 * must all fit into the 64-word Descriptor h/w Buffer 180 */ 181 if (rem_bytes >= DESC_AEAD_NULL_DEC_LEN) { 182 ctx->adata.key_inline = true; 183 ctx->adata.key_virt = ctx->key; 184 } else { 185 ctx->adata.key_inline = false; 186 ctx->adata.key_dma = ctx->key_dma; 187 } 188 189 /* aead_decrypt shared descriptor */ 190 desc = ctx->sh_desc_dec; 191 cnstr_shdsc_aead_null_decap(desc, &ctx->adata, ctx->authsize); 192 dma_sync_single_for_device(jrdev, ctx->sh_desc_dec_dma, 193 desc_bytes(desc), DMA_TO_DEVICE); 194 195 return 0; 196 } 197 198 static int aead_set_sh_desc(struct crypto_aead *aead) 199 { 200 struct caam_aead_alg *alg = container_of(crypto_aead_alg(aead), 201 struct caam_aead_alg, aead); 202 unsigned int ivsize = crypto_aead_ivsize(aead); 203 struct caam_ctx *ctx = crypto_aead_ctx(aead); 204 struct device *jrdev = ctx->jrdev; 205 u32 ctx1_iv_off = 0; 206 u32 *desc, *nonce = NULL; 207 u32 inl_mask; 208 unsigned int data_len[2]; 209 const bool ctr_mode = ((ctx->cdata.algtype & OP_ALG_AAI_MASK) == 210 OP_ALG_AAI_CTR_MOD128); 211 const bool is_rfc3686 = alg->caam.rfc3686; 212 213 if (!ctx->authsize) 214 return 0; 215 216 /* NULL encryption / decryption */ 217 if (!ctx->cdata.keylen) 218 return aead_null_set_sh_desc(aead); 219 220 /* 221 * AES-CTR needs to load IV in CONTEXT1 reg 222 * at an offset of 128bits (16bytes) 223 * CONTEXT1[255:128] = IV 224 */ 225 if (ctr_mode) 226 ctx1_iv_off = 16; 227 228 /* 229 * RFC3686 specific: 230 * CONTEXT1[255:128] = {NONCE, IV, COUNTER} 231 */ 232 if (is_rfc3686) { 233 ctx1_iv_off = 16 + CTR_RFC3686_NONCE_SIZE; 234 nonce = (u32 *)((void *)ctx->key + ctx->adata.keylen_pad + 235 ctx->cdata.keylen - CTR_RFC3686_NONCE_SIZE); 236 } 237 238 data_len[0] = ctx->adata.keylen_pad; 239 data_len[1] = ctx->cdata.keylen; 240 241 if (alg->caam.geniv) 242 goto skip_enc; 243 244 /* 245 * Job Descriptor and Shared Descriptors 246 * must all fit into the 64-word Descriptor h/w Buffer 247 */ 248 if (desc_inline_query(DESC_AEAD_ENC_LEN + 249 (is_rfc3686 ? DESC_AEAD_CTR_RFC3686_LEN : 0), 250 AUTHENC_DESC_JOB_IO_LEN, data_len, &inl_mask, 251 ARRAY_SIZE(data_len)) < 0) 252 return -EINVAL; 253 254 if (inl_mask & 1) 255 ctx->adata.key_virt = ctx->key; 256 else 257 ctx->adata.key_dma = ctx->key_dma; 258 259 if (inl_mask & 2) 260 ctx->cdata.key_virt = ctx->key + ctx->adata.keylen_pad; 261 else 262 ctx->cdata.key_dma = ctx->key_dma + ctx->adata.keylen_pad; 263 264 ctx->adata.key_inline = !!(inl_mask & 1); 265 ctx->cdata.key_inline = !!(inl_mask & 2); 266 267 /* aead_encrypt shared descriptor */ 268 desc = ctx->sh_desc_enc; 269 cnstr_shdsc_aead_encap(desc, &ctx->cdata, &ctx->adata, ivsize, 270 ctx->authsize, is_rfc3686, nonce, ctx1_iv_off, 271 false); 272 dma_sync_single_for_device(jrdev, ctx->sh_desc_enc_dma, 273 desc_bytes(desc), DMA_TO_DEVICE); 274 275 skip_enc: 276 /* 277 * Job Descriptor and Shared Descriptors 278 * must all fit into the 64-word Descriptor h/w Buffer 279 */ 280 if (desc_inline_query(DESC_AEAD_DEC_LEN + 281 (is_rfc3686 ? DESC_AEAD_CTR_RFC3686_LEN : 0), 282 AUTHENC_DESC_JOB_IO_LEN, data_len, &inl_mask, 283 ARRAY_SIZE(data_len)) < 0) 284 return -EINVAL; 285 286 if (inl_mask & 1) 287 ctx->adata.key_virt = ctx->key; 288 else 289 ctx->adata.key_dma = ctx->key_dma; 290 291 if (inl_mask & 2) 292 ctx->cdata.key_virt = ctx->key + ctx->adata.keylen_pad; 293 else 294 ctx->cdata.key_dma = ctx->key_dma + ctx->adata.keylen_pad; 295 296 ctx->adata.key_inline = !!(inl_mask & 1); 297 ctx->cdata.key_inline = !!(inl_mask & 2); 298 299 /* aead_decrypt shared descriptor */ 300 desc = ctx->sh_desc_dec; 301 cnstr_shdsc_aead_decap(desc, &ctx->cdata, &ctx->adata, ivsize, 302 ctx->authsize, alg->caam.geniv, is_rfc3686, 303 nonce, ctx1_iv_off, false); 304 dma_sync_single_for_device(jrdev, ctx->sh_desc_dec_dma, 305 desc_bytes(desc), DMA_TO_DEVICE); 306 307 if (!alg->caam.geniv) 308 goto skip_givenc; 309 310 /* 311 * Job Descriptor and Shared Descriptors 312 * must all fit into the 64-word Descriptor h/w Buffer 313 */ 314 if (desc_inline_query(DESC_AEAD_GIVENC_LEN + 315 (is_rfc3686 ? DESC_AEAD_CTR_RFC3686_LEN : 0), 316 AUTHENC_DESC_JOB_IO_LEN, data_len, &inl_mask, 317 ARRAY_SIZE(data_len)) < 0) 318 return -EINVAL; 319 320 if (inl_mask & 1) 321 ctx->adata.key_virt = ctx->key; 322 else 323 ctx->adata.key_dma = ctx->key_dma; 324 325 if (inl_mask & 2) 326 ctx->cdata.key_virt = ctx->key + ctx->adata.keylen_pad; 327 else 328 ctx->cdata.key_dma = ctx->key_dma + ctx->adata.keylen_pad; 329 330 ctx->adata.key_inline = !!(inl_mask & 1); 331 ctx->cdata.key_inline = !!(inl_mask & 2); 332 333 /* aead_givencrypt shared descriptor */ 334 desc = ctx->sh_desc_enc; 335 cnstr_shdsc_aead_givencap(desc, &ctx->cdata, &ctx->adata, ivsize, 336 ctx->authsize, is_rfc3686, nonce, 337 ctx1_iv_off, false); 338 dma_sync_single_for_device(jrdev, ctx->sh_desc_enc_dma, 339 desc_bytes(desc), DMA_TO_DEVICE); 340 341 skip_givenc: 342 return 0; 343 } 344 345 static int aead_setauthsize(struct crypto_aead *authenc, 346 unsigned int authsize) 347 { 348 struct caam_ctx *ctx = crypto_aead_ctx(authenc); 349 350 ctx->authsize = authsize; 351 aead_set_sh_desc(authenc); 352 353 return 0; 354 } 355 356 static int gcm_set_sh_desc(struct crypto_aead *aead) 357 { 358 struct caam_ctx *ctx = crypto_aead_ctx(aead); 359 struct device *jrdev = ctx->jrdev; 360 u32 *desc; 361 int rem_bytes = CAAM_DESC_BYTES_MAX - GCM_DESC_JOB_IO_LEN - 362 ctx->cdata.keylen; 363 364 if (!ctx->cdata.keylen || !ctx->authsize) 365 return 0; 366 367 /* 368 * AES GCM encrypt shared descriptor 369 * Job Descriptor and Shared Descriptor 370 * must fit into the 64-word Descriptor h/w Buffer 371 */ 372 if (rem_bytes >= DESC_GCM_ENC_LEN) { 373 ctx->cdata.key_inline = true; 374 ctx->cdata.key_virt = ctx->key; 375 } else { 376 ctx->cdata.key_inline = false; 377 ctx->cdata.key_dma = ctx->key_dma; 378 } 379 380 desc = ctx->sh_desc_enc; 381 cnstr_shdsc_gcm_encap(desc, &ctx->cdata, ctx->authsize); 382 dma_sync_single_for_device(jrdev, ctx->sh_desc_enc_dma, 383 desc_bytes(desc), DMA_TO_DEVICE); 384 385 /* 386 * Job Descriptor and Shared Descriptors 387 * must all fit into the 64-word Descriptor h/w Buffer 388 */ 389 if (rem_bytes >= DESC_GCM_DEC_LEN) { 390 ctx->cdata.key_inline = true; 391 ctx->cdata.key_virt = ctx->key; 392 } else { 393 ctx->cdata.key_inline = false; 394 ctx->cdata.key_dma = ctx->key_dma; 395 } 396 397 desc = ctx->sh_desc_dec; 398 cnstr_shdsc_gcm_decap(desc, &ctx->cdata, ctx->authsize); 399 dma_sync_single_for_device(jrdev, ctx->sh_desc_dec_dma, 400 desc_bytes(desc), DMA_TO_DEVICE); 401 402 return 0; 403 } 404 405 static int gcm_setauthsize(struct crypto_aead *authenc, unsigned int authsize) 406 { 407 struct caam_ctx *ctx = crypto_aead_ctx(authenc); 408 409 ctx->authsize = authsize; 410 gcm_set_sh_desc(authenc); 411 412 return 0; 413 } 414 415 static int rfc4106_set_sh_desc(struct crypto_aead *aead) 416 { 417 struct caam_ctx *ctx = crypto_aead_ctx(aead); 418 struct device *jrdev = ctx->jrdev; 419 u32 *desc; 420 int rem_bytes = CAAM_DESC_BYTES_MAX - GCM_DESC_JOB_IO_LEN - 421 ctx->cdata.keylen; 422 423 if (!ctx->cdata.keylen || !ctx->authsize) 424 return 0; 425 426 /* 427 * RFC4106 encrypt shared descriptor 428 * Job Descriptor and Shared Descriptor 429 * must fit into the 64-word Descriptor h/w Buffer 430 */ 431 if (rem_bytes >= DESC_RFC4106_ENC_LEN) { 432 ctx->cdata.key_inline = true; 433 ctx->cdata.key_virt = ctx->key; 434 } else { 435 ctx->cdata.key_inline = false; 436 ctx->cdata.key_dma = ctx->key_dma; 437 } 438 439 desc = ctx->sh_desc_enc; 440 cnstr_shdsc_rfc4106_encap(desc, &ctx->cdata, ctx->authsize); 441 dma_sync_single_for_device(jrdev, ctx->sh_desc_enc_dma, 442 desc_bytes(desc), DMA_TO_DEVICE); 443 444 /* 445 * Job Descriptor and Shared Descriptors 446 * must all fit into the 64-word Descriptor h/w Buffer 447 */ 448 if (rem_bytes >= DESC_RFC4106_DEC_LEN) { 449 ctx->cdata.key_inline = true; 450 ctx->cdata.key_virt = ctx->key; 451 } else { 452 ctx->cdata.key_inline = false; 453 ctx->cdata.key_dma = ctx->key_dma; 454 } 455 456 desc = ctx->sh_desc_dec; 457 cnstr_shdsc_rfc4106_decap(desc, &ctx->cdata, ctx->authsize); 458 dma_sync_single_for_device(jrdev, ctx->sh_desc_dec_dma, 459 desc_bytes(desc), DMA_TO_DEVICE); 460 461 return 0; 462 } 463 464 static int rfc4106_setauthsize(struct crypto_aead *authenc, 465 unsigned int authsize) 466 { 467 struct caam_ctx *ctx = crypto_aead_ctx(authenc); 468 469 ctx->authsize = authsize; 470 rfc4106_set_sh_desc(authenc); 471 472 return 0; 473 } 474 475 static int rfc4543_set_sh_desc(struct crypto_aead *aead) 476 { 477 struct caam_ctx *ctx = crypto_aead_ctx(aead); 478 struct device *jrdev = ctx->jrdev; 479 u32 *desc; 480 int rem_bytes = CAAM_DESC_BYTES_MAX - GCM_DESC_JOB_IO_LEN - 481 ctx->cdata.keylen; 482 483 if (!ctx->cdata.keylen || !ctx->authsize) 484 return 0; 485 486 /* 487 * RFC4543 encrypt shared descriptor 488 * Job Descriptor and Shared Descriptor 489 * must fit into the 64-word Descriptor h/w Buffer 490 */ 491 if (rem_bytes >= DESC_RFC4543_ENC_LEN) { 492 ctx->cdata.key_inline = true; 493 ctx->cdata.key_virt = ctx->key; 494 } else { 495 ctx->cdata.key_inline = false; 496 ctx->cdata.key_dma = ctx->key_dma; 497 } 498 499 desc = ctx->sh_desc_enc; 500 cnstr_shdsc_rfc4543_encap(desc, &ctx->cdata, ctx->authsize); 501 dma_sync_single_for_device(jrdev, ctx->sh_desc_enc_dma, 502 desc_bytes(desc), DMA_TO_DEVICE); 503 504 /* 505 * Job Descriptor and Shared Descriptors 506 * must all fit into the 64-word Descriptor h/w Buffer 507 */ 508 if (rem_bytes >= DESC_RFC4543_DEC_LEN) { 509 ctx->cdata.key_inline = true; 510 ctx->cdata.key_virt = ctx->key; 511 } else { 512 ctx->cdata.key_inline = false; 513 ctx->cdata.key_dma = ctx->key_dma; 514 } 515 516 desc = ctx->sh_desc_dec; 517 cnstr_shdsc_rfc4543_decap(desc, &ctx->cdata, ctx->authsize); 518 dma_sync_single_for_device(jrdev, ctx->sh_desc_dec_dma, 519 desc_bytes(desc), DMA_TO_DEVICE); 520 521 return 0; 522 } 523 524 static int rfc4543_setauthsize(struct crypto_aead *authenc, 525 unsigned int authsize) 526 { 527 struct caam_ctx *ctx = crypto_aead_ctx(authenc); 528 529 ctx->authsize = authsize; 530 rfc4543_set_sh_desc(authenc); 531 532 return 0; 533 } 534 535 static int aead_setkey(struct crypto_aead *aead, 536 const u8 *key, unsigned int keylen) 537 { 538 struct caam_ctx *ctx = crypto_aead_ctx(aead); 539 struct device *jrdev = ctx->jrdev; 540 struct crypto_authenc_keys keys; 541 int ret = 0; 542 543 if (crypto_authenc_extractkeys(&keys, key, keylen) != 0) 544 goto badkey; 545 546 #ifdef DEBUG 547 printk(KERN_ERR "keylen %d enckeylen %d authkeylen %d\n", 548 keys.authkeylen + keys.enckeylen, keys.enckeylen, 549 keys.authkeylen); 550 print_hex_dump(KERN_ERR, "key in @"__stringify(__LINE__)": ", 551 DUMP_PREFIX_ADDRESS, 16, 4, key, keylen, 1); 552 #endif 553 554 ret = gen_split_key(ctx->jrdev, ctx->key, &ctx->adata, keys.authkey, 555 keys.authkeylen, CAAM_MAX_KEY_SIZE - 556 keys.enckeylen); 557 if (ret) { 558 goto badkey; 559 } 560 561 /* postpend encryption key to auth split key */ 562 memcpy(ctx->key + ctx->adata.keylen_pad, keys.enckey, keys.enckeylen); 563 dma_sync_single_for_device(jrdev, ctx->key_dma, ctx->adata.keylen_pad + 564 keys.enckeylen, DMA_TO_DEVICE); 565 #ifdef DEBUG 566 print_hex_dump(KERN_ERR, "ctx.key@"__stringify(__LINE__)": ", 567 DUMP_PREFIX_ADDRESS, 16, 4, ctx->key, 568 ctx->adata.keylen_pad + keys.enckeylen, 1); 569 #endif 570 ctx->cdata.keylen = keys.enckeylen; 571 return aead_set_sh_desc(aead); 572 badkey: 573 crypto_aead_set_flags(aead, CRYPTO_TFM_RES_BAD_KEY_LEN); 574 return -EINVAL; 575 } 576 577 static int gcm_setkey(struct crypto_aead *aead, 578 const u8 *key, unsigned int keylen) 579 { 580 struct caam_ctx *ctx = crypto_aead_ctx(aead); 581 struct device *jrdev = ctx->jrdev; 582 583 #ifdef DEBUG 584 print_hex_dump(KERN_ERR, "key in @"__stringify(__LINE__)": ", 585 DUMP_PREFIX_ADDRESS, 16, 4, key, keylen, 1); 586 #endif 587 588 memcpy(ctx->key, key, keylen); 589 dma_sync_single_for_device(jrdev, ctx->key_dma, keylen, DMA_TO_DEVICE); 590 ctx->cdata.keylen = keylen; 591 592 return gcm_set_sh_desc(aead); 593 } 594 595 static int rfc4106_setkey(struct crypto_aead *aead, 596 const u8 *key, unsigned int keylen) 597 { 598 struct caam_ctx *ctx = crypto_aead_ctx(aead); 599 struct device *jrdev = ctx->jrdev; 600 601 if (keylen < 4) 602 return -EINVAL; 603 604 #ifdef DEBUG 605 print_hex_dump(KERN_ERR, "key in @"__stringify(__LINE__)": ", 606 DUMP_PREFIX_ADDRESS, 16, 4, key, keylen, 1); 607 #endif 608 609 memcpy(ctx->key, key, keylen); 610 611 /* 612 * The last four bytes of the key material are used as the salt value 613 * in the nonce. Update the AES key length. 614 */ 615 ctx->cdata.keylen = keylen - 4; 616 dma_sync_single_for_device(jrdev, ctx->key_dma, ctx->cdata.keylen, 617 DMA_TO_DEVICE); 618 return rfc4106_set_sh_desc(aead); 619 } 620 621 static int rfc4543_setkey(struct crypto_aead *aead, 622 const u8 *key, unsigned int keylen) 623 { 624 struct caam_ctx *ctx = crypto_aead_ctx(aead); 625 struct device *jrdev = ctx->jrdev; 626 627 if (keylen < 4) 628 return -EINVAL; 629 630 #ifdef DEBUG 631 print_hex_dump(KERN_ERR, "key in @"__stringify(__LINE__)": ", 632 DUMP_PREFIX_ADDRESS, 16, 4, key, keylen, 1); 633 #endif 634 635 memcpy(ctx->key, key, keylen); 636 637 /* 638 * The last four bytes of the key material are used as the salt value 639 * in the nonce. Update the AES key length. 640 */ 641 ctx->cdata.keylen = keylen - 4; 642 dma_sync_single_for_device(jrdev, ctx->key_dma, ctx->cdata.keylen, 643 DMA_TO_DEVICE); 644 return rfc4543_set_sh_desc(aead); 645 } 646 647 static int ablkcipher_setkey(struct crypto_ablkcipher *ablkcipher, 648 const u8 *key, unsigned int keylen) 649 { 650 struct caam_ctx *ctx = crypto_ablkcipher_ctx(ablkcipher); 651 struct crypto_tfm *tfm = crypto_ablkcipher_tfm(ablkcipher); 652 const char *alg_name = crypto_tfm_alg_name(tfm); 653 struct device *jrdev = ctx->jrdev; 654 unsigned int ivsize = crypto_ablkcipher_ivsize(ablkcipher); 655 u32 *desc; 656 u32 ctx1_iv_off = 0; 657 const bool ctr_mode = ((ctx->cdata.algtype & OP_ALG_AAI_MASK) == 658 OP_ALG_AAI_CTR_MOD128); 659 const bool is_rfc3686 = (ctr_mode && 660 (strstr(alg_name, "rfc3686") != NULL)); 661 662 memcpy(ctx->key, key, keylen); 663 #ifdef DEBUG 664 print_hex_dump(KERN_ERR, "key in @"__stringify(__LINE__)": ", 665 DUMP_PREFIX_ADDRESS, 16, 4, key, keylen, 1); 666 #endif 667 /* 668 * AES-CTR needs to load IV in CONTEXT1 reg 669 * at an offset of 128bits (16bytes) 670 * CONTEXT1[255:128] = IV 671 */ 672 if (ctr_mode) 673 ctx1_iv_off = 16; 674 675 /* 676 * RFC3686 specific: 677 * | CONTEXT1[255:128] = {NONCE, IV, COUNTER} 678 * | *key = {KEY, NONCE} 679 */ 680 if (is_rfc3686) { 681 ctx1_iv_off = 16 + CTR_RFC3686_NONCE_SIZE; 682 keylen -= CTR_RFC3686_NONCE_SIZE; 683 } 684 685 dma_sync_single_for_device(jrdev, ctx->key_dma, keylen, DMA_TO_DEVICE); 686 ctx->cdata.keylen = keylen; 687 ctx->cdata.key_virt = ctx->key; 688 ctx->cdata.key_inline = true; 689 690 /* ablkcipher_encrypt shared descriptor */ 691 desc = ctx->sh_desc_enc; 692 cnstr_shdsc_ablkcipher_encap(desc, &ctx->cdata, ivsize, is_rfc3686, 693 ctx1_iv_off); 694 dma_sync_single_for_device(jrdev, ctx->sh_desc_enc_dma, 695 desc_bytes(desc), DMA_TO_DEVICE); 696 697 /* ablkcipher_decrypt shared descriptor */ 698 desc = ctx->sh_desc_dec; 699 cnstr_shdsc_ablkcipher_decap(desc, &ctx->cdata, ivsize, is_rfc3686, 700 ctx1_iv_off); 701 dma_sync_single_for_device(jrdev, ctx->sh_desc_dec_dma, 702 desc_bytes(desc), DMA_TO_DEVICE); 703 704 /* ablkcipher_givencrypt shared descriptor */ 705 desc = ctx->sh_desc_givenc; 706 cnstr_shdsc_ablkcipher_givencap(desc, &ctx->cdata, ivsize, is_rfc3686, 707 ctx1_iv_off); 708 dma_sync_single_for_device(jrdev, ctx->sh_desc_givenc_dma, 709 desc_bytes(desc), DMA_TO_DEVICE); 710 711 return 0; 712 } 713 714 static int xts_ablkcipher_setkey(struct crypto_ablkcipher *ablkcipher, 715 const u8 *key, unsigned int keylen) 716 { 717 struct caam_ctx *ctx = crypto_ablkcipher_ctx(ablkcipher); 718 struct device *jrdev = ctx->jrdev; 719 u32 *desc; 720 721 if (keylen != 2 * AES_MIN_KEY_SIZE && keylen != 2 * AES_MAX_KEY_SIZE) { 722 crypto_ablkcipher_set_flags(ablkcipher, 723 CRYPTO_TFM_RES_BAD_KEY_LEN); 724 dev_err(jrdev, "key size mismatch\n"); 725 return -EINVAL; 726 } 727 728 memcpy(ctx->key, key, keylen); 729 dma_sync_single_for_device(jrdev, ctx->key_dma, keylen, DMA_TO_DEVICE); 730 ctx->cdata.keylen = keylen; 731 ctx->cdata.key_virt = ctx->key; 732 ctx->cdata.key_inline = true; 733 734 /* xts_ablkcipher_encrypt shared descriptor */ 735 desc = ctx->sh_desc_enc; 736 cnstr_shdsc_xts_ablkcipher_encap(desc, &ctx->cdata); 737 dma_sync_single_for_device(jrdev, ctx->sh_desc_enc_dma, 738 desc_bytes(desc), DMA_TO_DEVICE); 739 740 /* xts_ablkcipher_decrypt shared descriptor */ 741 desc = ctx->sh_desc_dec; 742 cnstr_shdsc_xts_ablkcipher_decap(desc, &ctx->cdata); 743 dma_sync_single_for_device(jrdev, ctx->sh_desc_dec_dma, 744 desc_bytes(desc), DMA_TO_DEVICE); 745 746 return 0; 747 } 748 749 /* 750 * aead_edesc - s/w-extended aead descriptor 751 * @src_nents: number of segments in input s/w scatterlist 752 * @dst_nents: number of segments in output s/w scatterlist 753 * @sec4_sg_bytes: length of dma mapped sec4_sg space 754 * @sec4_sg_dma: bus physical mapped address of h/w link table 755 * @sec4_sg: pointer to h/w link table 756 * @hw_desc: the h/w job descriptor followed by any referenced link tables 757 */ 758 struct aead_edesc { 759 int src_nents; 760 int dst_nents; 761 int sec4_sg_bytes; 762 dma_addr_t sec4_sg_dma; 763 struct sec4_sg_entry *sec4_sg; 764 u32 hw_desc[]; 765 }; 766 767 /* 768 * ablkcipher_edesc - s/w-extended ablkcipher descriptor 769 * @src_nents: number of segments in input s/w scatterlist 770 * @dst_nents: number of segments in output s/w scatterlist 771 * @iv_dma: dma address of iv for checking continuity and link table 772 * @sec4_sg_bytes: length of dma mapped sec4_sg space 773 * @sec4_sg_dma: bus physical mapped address of h/w link table 774 * @sec4_sg: pointer to h/w link table 775 * @hw_desc: the h/w job descriptor followed by any referenced link tables 776 */ 777 struct ablkcipher_edesc { 778 int src_nents; 779 int dst_nents; 780 dma_addr_t iv_dma; 781 int sec4_sg_bytes; 782 dma_addr_t sec4_sg_dma; 783 struct sec4_sg_entry *sec4_sg; 784 u32 hw_desc[0]; 785 }; 786 787 static void caam_unmap(struct device *dev, struct scatterlist *src, 788 struct scatterlist *dst, int src_nents, 789 int dst_nents, 790 dma_addr_t iv_dma, int ivsize, dma_addr_t sec4_sg_dma, 791 int sec4_sg_bytes) 792 { 793 if (dst != src) { 794 if (src_nents) 795 dma_unmap_sg(dev, src, src_nents, DMA_TO_DEVICE); 796 dma_unmap_sg(dev, dst, dst_nents, DMA_FROM_DEVICE); 797 } else { 798 dma_unmap_sg(dev, src, src_nents, DMA_BIDIRECTIONAL); 799 } 800 801 if (iv_dma) 802 dma_unmap_single(dev, iv_dma, ivsize, DMA_TO_DEVICE); 803 if (sec4_sg_bytes) 804 dma_unmap_single(dev, sec4_sg_dma, sec4_sg_bytes, 805 DMA_TO_DEVICE); 806 } 807 808 static void aead_unmap(struct device *dev, 809 struct aead_edesc *edesc, 810 struct aead_request *req) 811 { 812 caam_unmap(dev, req->src, req->dst, 813 edesc->src_nents, edesc->dst_nents, 0, 0, 814 edesc->sec4_sg_dma, edesc->sec4_sg_bytes); 815 } 816 817 static void ablkcipher_unmap(struct device *dev, 818 struct ablkcipher_edesc *edesc, 819 struct ablkcipher_request *req) 820 { 821 struct crypto_ablkcipher *ablkcipher = crypto_ablkcipher_reqtfm(req); 822 int ivsize = crypto_ablkcipher_ivsize(ablkcipher); 823 824 caam_unmap(dev, req->src, req->dst, 825 edesc->src_nents, edesc->dst_nents, 826 edesc->iv_dma, ivsize, 827 edesc->sec4_sg_dma, edesc->sec4_sg_bytes); 828 } 829 830 static void aead_encrypt_done(struct device *jrdev, u32 *desc, u32 err, 831 void *context) 832 { 833 struct aead_request *req = context; 834 struct aead_edesc *edesc; 835 836 #ifdef DEBUG 837 dev_err(jrdev, "%s %d: err 0x%x\n", __func__, __LINE__, err); 838 #endif 839 840 edesc = container_of(desc, struct aead_edesc, hw_desc[0]); 841 842 if (err) 843 caam_jr_strstatus(jrdev, err); 844 845 aead_unmap(jrdev, edesc, req); 846 847 kfree(edesc); 848 849 aead_request_complete(req, err); 850 } 851 852 static void aead_decrypt_done(struct device *jrdev, u32 *desc, u32 err, 853 void *context) 854 { 855 struct aead_request *req = context; 856 struct aead_edesc *edesc; 857 858 #ifdef DEBUG 859 dev_err(jrdev, "%s %d: err 0x%x\n", __func__, __LINE__, err); 860 #endif 861 862 edesc = container_of(desc, struct aead_edesc, hw_desc[0]); 863 864 if (err) 865 caam_jr_strstatus(jrdev, err); 866 867 aead_unmap(jrdev, edesc, req); 868 869 /* 870 * verify hw auth check passed else return -EBADMSG 871 */ 872 if ((err & JRSTA_CCBERR_ERRID_MASK) == JRSTA_CCBERR_ERRID_ICVCHK) 873 err = -EBADMSG; 874 875 kfree(edesc); 876 877 aead_request_complete(req, err); 878 } 879 880 static void ablkcipher_encrypt_done(struct device *jrdev, u32 *desc, u32 err, 881 void *context) 882 { 883 struct ablkcipher_request *req = context; 884 struct ablkcipher_edesc *edesc; 885 struct crypto_ablkcipher *ablkcipher = crypto_ablkcipher_reqtfm(req); 886 int ivsize = crypto_ablkcipher_ivsize(ablkcipher); 887 888 #ifdef DEBUG 889 dev_err(jrdev, "%s %d: err 0x%x\n", __func__, __LINE__, err); 890 #endif 891 892 edesc = container_of(desc, struct ablkcipher_edesc, hw_desc[0]); 893 894 if (err) 895 caam_jr_strstatus(jrdev, err); 896 897 #ifdef DEBUG 898 print_hex_dump(KERN_ERR, "dstiv @"__stringify(__LINE__)": ", 899 DUMP_PREFIX_ADDRESS, 16, 4, req->info, 900 edesc->src_nents > 1 ? 100 : ivsize, 1); 901 dbg_dump_sg(KERN_ERR, "dst @"__stringify(__LINE__)": ", 902 DUMP_PREFIX_ADDRESS, 16, 4, req->dst, 903 edesc->dst_nents > 1 ? 100 : req->nbytes, 1); 904 #endif 905 906 ablkcipher_unmap(jrdev, edesc, req); 907 908 /* 909 * The crypto API expects us to set the IV (req->info) to the last 910 * ciphertext block. This is used e.g. by the CTS mode. 911 */ 912 scatterwalk_map_and_copy(req->info, req->dst, req->nbytes - ivsize, 913 ivsize, 0); 914 915 kfree(edesc); 916 917 ablkcipher_request_complete(req, err); 918 } 919 920 static void ablkcipher_decrypt_done(struct device *jrdev, u32 *desc, u32 err, 921 void *context) 922 { 923 struct ablkcipher_request *req = context; 924 struct ablkcipher_edesc *edesc; 925 struct crypto_ablkcipher *ablkcipher = crypto_ablkcipher_reqtfm(req); 926 int ivsize = crypto_ablkcipher_ivsize(ablkcipher); 927 928 #ifdef DEBUG 929 dev_err(jrdev, "%s %d: err 0x%x\n", __func__, __LINE__, err); 930 #endif 931 932 edesc = container_of(desc, struct ablkcipher_edesc, hw_desc[0]); 933 if (err) 934 caam_jr_strstatus(jrdev, err); 935 936 #ifdef DEBUG 937 print_hex_dump(KERN_ERR, "dstiv @"__stringify(__LINE__)": ", 938 DUMP_PREFIX_ADDRESS, 16, 4, req->info, 939 ivsize, 1); 940 dbg_dump_sg(KERN_ERR, "dst @"__stringify(__LINE__)": ", 941 DUMP_PREFIX_ADDRESS, 16, 4, req->dst, 942 edesc->dst_nents > 1 ? 100 : req->nbytes, 1); 943 #endif 944 945 ablkcipher_unmap(jrdev, edesc, req); 946 947 /* 948 * The crypto API expects us to set the IV (req->info) to the last 949 * ciphertext block. 950 */ 951 scatterwalk_map_and_copy(req->info, req->src, req->nbytes - ivsize, 952 ivsize, 0); 953 954 kfree(edesc); 955 956 ablkcipher_request_complete(req, err); 957 } 958 959 /* 960 * Fill in aead job descriptor 961 */ 962 static void init_aead_job(struct aead_request *req, 963 struct aead_edesc *edesc, 964 bool all_contig, bool encrypt) 965 { 966 struct crypto_aead *aead = crypto_aead_reqtfm(req); 967 struct caam_ctx *ctx = crypto_aead_ctx(aead); 968 int authsize = ctx->authsize; 969 u32 *desc = edesc->hw_desc; 970 u32 out_options, in_options; 971 dma_addr_t dst_dma, src_dma; 972 int len, sec4_sg_index = 0; 973 dma_addr_t ptr; 974 u32 *sh_desc; 975 976 sh_desc = encrypt ? ctx->sh_desc_enc : ctx->sh_desc_dec; 977 ptr = encrypt ? ctx->sh_desc_enc_dma : ctx->sh_desc_dec_dma; 978 979 len = desc_len(sh_desc); 980 init_job_desc_shared(desc, ptr, len, HDR_SHARE_DEFER | HDR_REVERSE); 981 982 if (all_contig) { 983 src_dma = edesc->src_nents ? sg_dma_address(req->src) : 0; 984 in_options = 0; 985 } else { 986 src_dma = edesc->sec4_sg_dma; 987 sec4_sg_index += edesc->src_nents; 988 in_options = LDST_SGF; 989 } 990 991 append_seq_in_ptr(desc, src_dma, req->assoclen + req->cryptlen, 992 in_options); 993 994 dst_dma = src_dma; 995 out_options = in_options; 996 997 if (unlikely(req->src != req->dst)) { 998 if (edesc->dst_nents == 1) { 999 dst_dma = sg_dma_address(req->dst); 1000 } else { 1001 dst_dma = edesc->sec4_sg_dma + 1002 sec4_sg_index * 1003 sizeof(struct sec4_sg_entry); 1004 out_options = LDST_SGF; 1005 } 1006 } 1007 1008 if (encrypt) 1009 append_seq_out_ptr(desc, dst_dma, 1010 req->assoclen + req->cryptlen + authsize, 1011 out_options); 1012 else 1013 append_seq_out_ptr(desc, dst_dma, 1014 req->assoclen + req->cryptlen - authsize, 1015 out_options); 1016 1017 /* REG3 = assoclen */ 1018 append_math_add_imm_u32(desc, REG3, ZERO, IMM, req->assoclen); 1019 } 1020 1021 static void init_gcm_job(struct aead_request *req, 1022 struct aead_edesc *edesc, 1023 bool all_contig, bool encrypt) 1024 { 1025 struct crypto_aead *aead = crypto_aead_reqtfm(req); 1026 struct caam_ctx *ctx = crypto_aead_ctx(aead); 1027 unsigned int ivsize = crypto_aead_ivsize(aead); 1028 u32 *desc = edesc->hw_desc; 1029 bool generic_gcm = (ivsize == 12); 1030 unsigned int last; 1031 1032 init_aead_job(req, edesc, all_contig, encrypt); 1033 1034 /* BUG This should not be specific to generic GCM. */ 1035 last = 0; 1036 if (encrypt && generic_gcm && !(req->assoclen + req->cryptlen)) 1037 last = FIFOLD_TYPE_LAST1; 1038 1039 /* Read GCM IV */ 1040 append_cmd(desc, CMD_FIFO_LOAD | FIFOLD_CLASS_CLASS1 | IMMEDIATE | 1041 FIFOLD_TYPE_IV | FIFOLD_TYPE_FLUSH1 | 12 | last); 1042 /* Append Salt */ 1043 if (!generic_gcm) 1044 append_data(desc, ctx->key + ctx->cdata.keylen, 4); 1045 /* Append IV */ 1046 append_data(desc, req->iv, ivsize); 1047 /* End of blank commands */ 1048 } 1049 1050 static void init_authenc_job(struct aead_request *req, 1051 struct aead_edesc *edesc, 1052 bool all_contig, bool encrypt) 1053 { 1054 struct crypto_aead *aead = crypto_aead_reqtfm(req); 1055 struct caam_aead_alg *alg = container_of(crypto_aead_alg(aead), 1056 struct caam_aead_alg, aead); 1057 unsigned int ivsize = crypto_aead_ivsize(aead); 1058 struct caam_ctx *ctx = crypto_aead_ctx(aead); 1059 const bool ctr_mode = ((ctx->cdata.algtype & OP_ALG_AAI_MASK) == 1060 OP_ALG_AAI_CTR_MOD128); 1061 const bool is_rfc3686 = alg->caam.rfc3686; 1062 u32 *desc = edesc->hw_desc; 1063 u32 ivoffset = 0; 1064 1065 /* 1066 * AES-CTR needs to load IV in CONTEXT1 reg 1067 * at an offset of 128bits (16bytes) 1068 * CONTEXT1[255:128] = IV 1069 */ 1070 if (ctr_mode) 1071 ivoffset = 16; 1072 1073 /* 1074 * RFC3686 specific: 1075 * CONTEXT1[255:128] = {NONCE, IV, COUNTER} 1076 */ 1077 if (is_rfc3686) 1078 ivoffset = 16 + CTR_RFC3686_NONCE_SIZE; 1079 1080 init_aead_job(req, edesc, all_contig, encrypt); 1081 1082 if (ivsize && ((is_rfc3686 && encrypt) || !alg->caam.geniv)) 1083 append_load_as_imm(desc, req->iv, ivsize, 1084 LDST_CLASS_1_CCB | 1085 LDST_SRCDST_BYTE_CONTEXT | 1086 (ivoffset << LDST_OFFSET_SHIFT)); 1087 } 1088 1089 /* 1090 * Fill in ablkcipher job descriptor 1091 */ 1092 static void init_ablkcipher_job(u32 *sh_desc, dma_addr_t ptr, 1093 struct ablkcipher_edesc *edesc, 1094 struct ablkcipher_request *req, 1095 bool iv_contig) 1096 { 1097 struct crypto_ablkcipher *ablkcipher = crypto_ablkcipher_reqtfm(req); 1098 int ivsize = crypto_ablkcipher_ivsize(ablkcipher); 1099 u32 *desc = edesc->hw_desc; 1100 u32 out_options = 0, in_options; 1101 dma_addr_t dst_dma, src_dma; 1102 int len, sec4_sg_index = 0; 1103 1104 #ifdef DEBUG 1105 print_hex_dump(KERN_ERR, "presciv@"__stringify(__LINE__)": ", 1106 DUMP_PREFIX_ADDRESS, 16, 4, req->info, 1107 ivsize, 1); 1108 pr_err("asked=%d, nbytes%d\n", 1109 (int)edesc->src_nents > 1 ? 100 : req->nbytes, req->nbytes); 1110 dbg_dump_sg(KERN_ERR, "src @"__stringify(__LINE__)": ", 1111 DUMP_PREFIX_ADDRESS, 16, 4, req->src, 1112 edesc->src_nents > 1 ? 100 : req->nbytes, 1); 1113 #endif 1114 1115 len = desc_len(sh_desc); 1116 init_job_desc_shared(desc, ptr, len, HDR_SHARE_DEFER | HDR_REVERSE); 1117 1118 if (iv_contig) { 1119 src_dma = edesc->iv_dma; 1120 in_options = 0; 1121 } else { 1122 src_dma = edesc->sec4_sg_dma; 1123 sec4_sg_index += edesc->src_nents + 1; 1124 in_options = LDST_SGF; 1125 } 1126 append_seq_in_ptr(desc, src_dma, req->nbytes + ivsize, in_options); 1127 1128 if (likely(req->src == req->dst)) { 1129 if (edesc->src_nents == 1 && iv_contig) { 1130 dst_dma = sg_dma_address(req->src); 1131 } else { 1132 dst_dma = edesc->sec4_sg_dma + 1133 sizeof(struct sec4_sg_entry); 1134 out_options = LDST_SGF; 1135 } 1136 } else { 1137 if (edesc->dst_nents == 1) { 1138 dst_dma = sg_dma_address(req->dst); 1139 } else { 1140 dst_dma = edesc->sec4_sg_dma + 1141 sec4_sg_index * sizeof(struct sec4_sg_entry); 1142 out_options = LDST_SGF; 1143 } 1144 } 1145 append_seq_out_ptr(desc, dst_dma, req->nbytes, out_options); 1146 } 1147 1148 /* 1149 * Fill in ablkcipher givencrypt job descriptor 1150 */ 1151 static void init_ablkcipher_giv_job(u32 *sh_desc, dma_addr_t ptr, 1152 struct ablkcipher_edesc *edesc, 1153 struct ablkcipher_request *req, 1154 bool iv_contig) 1155 { 1156 struct crypto_ablkcipher *ablkcipher = crypto_ablkcipher_reqtfm(req); 1157 int ivsize = crypto_ablkcipher_ivsize(ablkcipher); 1158 u32 *desc = edesc->hw_desc; 1159 u32 out_options, in_options; 1160 dma_addr_t dst_dma, src_dma; 1161 int len, sec4_sg_index = 0; 1162 1163 #ifdef DEBUG 1164 print_hex_dump(KERN_ERR, "presciv@" __stringify(__LINE__) ": ", 1165 DUMP_PREFIX_ADDRESS, 16, 4, req->info, 1166 ivsize, 1); 1167 dbg_dump_sg(KERN_ERR, "src @" __stringify(__LINE__) ": ", 1168 DUMP_PREFIX_ADDRESS, 16, 4, req->src, 1169 edesc->src_nents > 1 ? 100 : req->nbytes, 1); 1170 #endif 1171 1172 len = desc_len(sh_desc); 1173 init_job_desc_shared(desc, ptr, len, HDR_SHARE_DEFER | HDR_REVERSE); 1174 1175 if (edesc->src_nents == 1) { 1176 src_dma = sg_dma_address(req->src); 1177 in_options = 0; 1178 } else { 1179 src_dma = edesc->sec4_sg_dma; 1180 sec4_sg_index += edesc->src_nents; 1181 in_options = LDST_SGF; 1182 } 1183 append_seq_in_ptr(desc, src_dma, req->nbytes, in_options); 1184 1185 if (iv_contig) { 1186 dst_dma = edesc->iv_dma; 1187 out_options = 0; 1188 } else { 1189 dst_dma = edesc->sec4_sg_dma + 1190 sec4_sg_index * sizeof(struct sec4_sg_entry); 1191 out_options = LDST_SGF; 1192 } 1193 append_seq_out_ptr(desc, dst_dma, req->nbytes + ivsize, out_options); 1194 } 1195 1196 /* 1197 * allocate and map the aead extended descriptor 1198 */ 1199 static struct aead_edesc *aead_edesc_alloc(struct aead_request *req, 1200 int desc_bytes, bool *all_contig_ptr, 1201 bool encrypt) 1202 { 1203 struct crypto_aead *aead = crypto_aead_reqtfm(req); 1204 struct caam_ctx *ctx = crypto_aead_ctx(aead); 1205 struct device *jrdev = ctx->jrdev; 1206 gfp_t flags = (req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP) ? 1207 GFP_KERNEL : GFP_ATOMIC; 1208 int src_nents, mapped_src_nents, dst_nents = 0, mapped_dst_nents = 0; 1209 struct aead_edesc *edesc; 1210 int sec4_sg_index, sec4_sg_len, sec4_sg_bytes; 1211 unsigned int authsize = ctx->authsize; 1212 1213 if (unlikely(req->dst != req->src)) { 1214 src_nents = sg_nents_for_len(req->src, req->assoclen + 1215 req->cryptlen); 1216 if (unlikely(src_nents < 0)) { 1217 dev_err(jrdev, "Insufficient bytes (%d) in src S/G\n", 1218 req->assoclen + req->cryptlen); 1219 return ERR_PTR(src_nents); 1220 } 1221 1222 dst_nents = sg_nents_for_len(req->dst, req->assoclen + 1223 req->cryptlen + 1224 (encrypt ? authsize : 1225 (-authsize))); 1226 if (unlikely(dst_nents < 0)) { 1227 dev_err(jrdev, "Insufficient bytes (%d) in dst S/G\n", 1228 req->assoclen + req->cryptlen + 1229 (encrypt ? authsize : (-authsize))); 1230 return ERR_PTR(dst_nents); 1231 } 1232 } else { 1233 src_nents = sg_nents_for_len(req->src, req->assoclen + 1234 req->cryptlen + 1235 (encrypt ? authsize : 0)); 1236 if (unlikely(src_nents < 0)) { 1237 dev_err(jrdev, "Insufficient bytes (%d) in src S/G\n", 1238 req->assoclen + req->cryptlen + 1239 (encrypt ? authsize : 0)); 1240 return ERR_PTR(src_nents); 1241 } 1242 } 1243 1244 if (likely(req->src == req->dst)) { 1245 mapped_src_nents = dma_map_sg(jrdev, req->src, src_nents, 1246 DMA_BIDIRECTIONAL); 1247 if (unlikely(!mapped_src_nents)) { 1248 dev_err(jrdev, "unable to map source\n"); 1249 return ERR_PTR(-ENOMEM); 1250 } 1251 } else { 1252 /* Cover also the case of null (zero length) input data */ 1253 if (src_nents) { 1254 mapped_src_nents = dma_map_sg(jrdev, req->src, 1255 src_nents, DMA_TO_DEVICE); 1256 if (unlikely(!mapped_src_nents)) { 1257 dev_err(jrdev, "unable to map source\n"); 1258 return ERR_PTR(-ENOMEM); 1259 } 1260 } else { 1261 mapped_src_nents = 0; 1262 } 1263 1264 mapped_dst_nents = dma_map_sg(jrdev, req->dst, dst_nents, 1265 DMA_FROM_DEVICE); 1266 if (unlikely(!mapped_dst_nents)) { 1267 dev_err(jrdev, "unable to map destination\n"); 1268 dma_unmap_sg(jrdev, req->src, src_nents, DMA_TO_DEVICE); 1269 return ERR_PTR(-ENOMEM); 1270 } 1271 } 1272 1273 sec4_sg_len = mapped_src_nents > 1 ? mapped_src_nents : 0; 1274 sec4_sg_len += mapped_dst_nents > 1 ? mapped_dst_nents : 0; 1275 sec4_sg_bytes = sec4_sg_len * sizeof(struct sec4_sg_entry); 1276 1277 /* allocate space for base edesc and hw desc commands, link tables */ 1278 edesc = kzalloc(sizeof(*edesc) + desc_bytes + sec4_sg_bytes, 1279 GFP_DMA | flags); 1280 if (!edesc) { 1281 caam_unmap(jrdev, req->src, req->dst, src_nents, dst_nents, 0, 1282 0, 0, 0); 1283 return ERR_PTR(-ENOMEM); 1284 } 1285 1286 edesc->src_nents = src_nents; 1287 edesc->dst_nents = dst_nents; 1288 edesc->sec4_sg = (void *)edesc + sizeof(struct aead_edesc) + 1289 desc_bytes; 1290 *all_contig_ptr = !(mapped_src_nents > 1); 1291 1292 sec4_sg_index = 0; 1293 if (mapped_src_nents > 1) { 1294 sg_to_sec4_sg_last(req->src, mapped_src_nents, 1295 edesc->sec4_sg + sec4_sg_index, 0); 1296 sec4_sg_index += mapped_src_nents; 1297 } 1298 if (mapped_dst_nents > 1) { 1299 sg_to_sec4_sg_last(req->dst, mapped_dst_nents, 1300 edesc->sec4_sg + sec4_sg_index, 0); 1301 } 1302 1303 if (!sec4_sg_bytes) 1304 return edesc; 1305 1306 edesc->sec4_sg_dma = dma_map_single(jrdev, edesc->sec4_sg, 1307 sec4_sg_bytes, DMA_TO_DEVICE); 1308 if (dma_mapping_error(jrdev, edesc->sec4_sg_dma)) { 1309 dev_err(jrdev, "unable to map S/G table\n"); 1310 aead_unmap(jrdev, edesc, req); 1311 kfree(edesc); 1312 return ERR_PTR(-ENOMEM); 1313 } 1314 1315 edesc->sec4_sg_bytes = sec4_sg_bytes; 1316 1317 return edesc; 1318 } 1319 1320 static int gcm_encrypt(struct aead_request *req) 1321 { 1322 struct aead_edesc *edesc; 1323 struct crypto_aead *aead = crypto_aead_reqtfm(req); 1324 struct caam_ctx *ctx = crypto_aead_ctx(aead); 1325 struct device *jrdev = ctx->jrdev; 1326 bool all_contig; 1327 u32 *desc; 1328 int ret = 0; 1329 1330 /* allocate extended descriptor */ 1331 edesc = aead_edesc_alloc(req, GCM_DESC_JOB_IO_LEN, &all_contig, true); 1332 if (IS_ERR(edesc)) 1333 return PTR_ERR(edesc); 1334 1335 /* Create and submit job descriptor */ 1336 init_gcm_job(req, edesc, all_contig, true); 1337 #ifdef DEBUG 1338 print_hex_dump(KERN_ERR, "aead jobdesc@"__stringify(__LINE__)": ", 1339 DUMP_PREFIX_ADDRESS, 16, 4, edesc->hw_desc, 1340 desc_bytes(edesc->hw_desc), 1); 1341 #endif 1342 1343 desc = edesc->hw_desc; 1344 ret = caam_jr_enqueue(jrdev, desc, aead_encrypt_done, req); 1345 if (!ret) { 1346 ret = -EINPROGRESS; 1347 } else { 1348 aead_unmap(jrdev, edesc, req); 1349 kfree(edesc); 1350 } 1351 1352 return ret; 1353 } 1354 1355 static int ipsec_gcm_encrypt(struct aead_request *req) 1356 { 1357 if (req->assoclen < 8) 1358 return -EINVAL; 1359 1360 return gcm_encrypt(req); 1361 } 1362 1363 static int aead_encrypt(struct aead_request *req) 1364 { 1365 struct aead_edesc *edesc; 1366 struct crypto_aead *aead = crypto_aead_reqtfm(req); 1367 struct caam_ctx *ctx = crypto_aead_ctx(aead); 1368 struct device *jrdev = ctx->jrdev; 1369 bool all_contig; 1370 u32 *desc; 1371 int ret = 0; 1372 1373 /* allocate extended descriptor */ 1374 edesc = aead_edesc_alloc(req, AUTHENC_DESC_JOB_IO_LEN, 1375 &all_contig, true); 1376 if (IS_ERR(edesc)) 1377 return PTR_ERR(edesc); 1378 1379 /* Create and submit job descriptor */ 1380 init_authenc_job(req, edesc, all_contig, true); 1381 #ifdef DEBUG 1382 print_hex_dump(KERN_ERR, "aead jobdesc@"__stringify(__LINE__)": ", 1383 DUMP_PREFIX_ADDRESS, 16, 4, edesc->hw_desc, 1384 desc_bytes(edesc->hw_desc), 1); 1385 #endif 1386 1387 desc = edesc->hw_desc; 1388 ret = caam_jr_enqueue(jrdev, desc, aead_encrypt_done, req); 1389 if (!ret) { 1390 ret = -EINPROGRESS; 1391 } else { 1392 aead_unmap(jrdev, edesc, req); 1393 kfree(edesc); 1394 } 1395 1396 return ret; 1397 } 1398 1399 static int gcm_decrypt(struct aead_request *req) 1400 { 1401 struct aead_edesc *edesc; 1402 struct crypto_aead *aead = crypto_aead_reqtfm(req); 1403 struct caam_ctx *ctx = crypto_aead_ctx(aead); 1404 struct device *jrdev = ctx->jrdev; 1405 bool all_contig; 1406 u32 *desc; 1407 int ret = 0; 1408 1409 /* allocate extended descriptor */ 1410 edesc = aead_edesc_alloc(req, GCM_DESC_JOB_IO_LEN, &all_contig, false); 1411 if (IS_ERR(edesc)) 1412 return PTR_ERR(edesc); 1413 1414 /* Create and submit job descriptor*/ 1415 init_gcm_job(req, edesc, all_contig, false); 1416 #ifdef DEBUG 1417 print_hex_dump(KERN_ERR, "aead jobdesc@"__stringify(__LINE__)": ", 1418 DUMP_PREFIX_ADDRESS, 16, 4, edesc->hw_desc, 1419 desc_bytes(edesc->hw_desc), 1); 1420 #endif 1421 1422 desc = edesc->hw_desc; 1423 ret = caam_jr_enqueue(jrdev, desc, aead_decrypt_done, req); 1424 if (!ret) { 1425 ret = -EINPROGRESS; 1426 } else { 1427 aead_unmap(jrdev, edesc, req); 1428 kfree(edesc); 1429 } 1430 1431 return ret; 1432 } 1433 1434 static int ipsec_gcm_decrypt(struct aead_request *req) 1435 { 1436 if (req->assoclen < 8) 1437 return -EINVAL; 1438 1439 return gcm_decrypt(req); 1440 } 1441 1442 static int aead_decrypt(struct aead_request *req) 1443 { 1444 struct aead_edesc *edesc; 1445 struct crypto_aead *aead = crypto_aead_reqtfm(req); 1446 struct caam_ctx *ctx = crypto_aead_ctx(aead); 1447 struct device *jrdev = ctx->jrdev; 1448 bool all_contig; 1449 u32 *desc; 1450 int ret = 0; 1451 1452 #ifdef DEBUG 1453 dbg_dump_sg(KERN_ERR, "dec src@"__stringify(__LINE__)": ", 1454 DUMP_PREFIX_ADDRESS, 16, 4, req->src, 1455 req->assoclen + req->cryptlen, 1); 1456 #endif 1457 1458 /* allocate extended descriptor */ 1459 edesc = aead_edesc_alloc(req, AUTHENC_DESC_JOB_IO_LEN, 1460 &all_contig, false); 1461 if (IS_ERR(edesc)) 1462 return PTR_ERR(edesc); 1463 1464 /* Create and submit job descriptor*/ 1465 init_authenc_job(req, edesc, all_contig, false); 1466 #ifdef DEBUG 1467 print_hex_dump(KERN_ERR, "aead jobdesc@"__stringify(__LINE__)": ", 1468 DUMP_PREFIX_ADDRESS, 16, 4, edesc->hw_desc, 1469 desc_bytes(edesc->hw_desc), 1); 1470 #endif 1471 1472 desc = edesc->hw_desc; 1473 ret = caam_jr_enqueue(jrdev, desc, aead_decrypt_done, req); 1474 if (!ret) { 1475 ret = -EINPROGRESS; 1476 } else { 1477 aead_unmap(jrdev, edesc, req); 1478 kfree(edesc); 1479 } 1480 1481 return ret; 1482 } 1483 1484 /* 1485 * allocate and map the ablkcipher extended descriptor for ablkcipher 1486 */ 1487 static struct ablkcipher_edesc *ablkcipher_edesc_alloc(struct ablkcipher_request 1488 *req, int desc_bytes, 1489 bool *iv_contig_out) 1490 { 1491 struct crypto_ablkcipher *ablkcipher = crypto_ablkcipher_reqtfm(req); 1492 struct caam_ctx *ctx = crypto_ablkcipher_ctx(ablkcipher); 1493 struct device *jrdev = ctx->jrdev; 1494 gfp_t flags = (req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP) ? 1495 GFP_KERNEL : GFP_ATOMIC; 1496 int src_nents, mapped_src_nents, dst_nents = 0, mapped_dst_nents = 0; 1497 struct ablkcipher_edesc *edesc; 1498 dma_addr_t iv_dma = 0; 1499 bool in_contig; 1500 int ivsize = crypto_ablkcipher_ivsize(ablkcipher); 1501 int dst_sg_idx, sec4_sg_ents, sec4_sg_bytes; 1502 1503 src_nents = sg_nents_for_len(req->src, req->nbytes); 1504 if (unlikely(src_nents < 0)) { 1505 dev_err(jrdev, "Insufficient bytes (%d) in src S/G\n", 1506 req->nbytes); 1507 return ERR_PTR(src_nents); 1508 } 1509 1510 if (req->dst != req->src) { 1511 dst_nents = sg_nents_for_len(req->dst, req->nbytes); 1512 if (unlikely(dst_nents < 0)) { 1513 dev_err(jrdev, "Insufficient bytes (%d) in dst S/G\n", 1514 req->nbytes); 1515 return ERR_PTR(dst_nents); 1516 } 1517 } 1518 1519 if (likely(req->src == req->dst)) { 1520 mapped_src_nents = dma_map_sg(jrdev, req->src, src_nents, 1521 DMA_BIDIRECTIONAL); 1522 if (unlikely(!mapped_src_nents)) { 1523 dev_err(jrdev, "unable to map source\n"); 1524 return ERR_PTR(-ENOMEM); 1525 } 1526 } else { 1527 mapped_src_nents = dma_map_sg(jrdev, req->src, src_nents, 1528 DMA_TO_DEVICE); 1529 if (unlikely(!mapped_src_nents)) { 1530 dev_err(jrdev, "unable to map source\n"); 1531 return ERR_PTR(-ENOMEM); 1532 } 1533 1534 mapped_dst_nents = dma_map_sg(jrdev, req->dst, dst_nents, 1535 DMA_FROM_DEVICE); 1536 if (unlikely(!mapped_dst_nents)) { 1537 dev_err(jrdev, "unable to map destination\n"); 1538 dma_unmap_sg(jrdev, req->src, src_nents, DMA_TO_DEVICE); 1539 return ERR_PTR(-ENOMEM); 1540 } 1541 } 1542 1543 iv_dma = dma_map_single(jrdev, req->info, ivsize, DMA_TO_DEVICE); 1544 if (dma_mapping_error(jrdev, iv_dma)) { 1545 dev_err(jrdev, "unable to map IV\n"); 1546 caam_unmap(jrdev, req->src, req->dst, src_nents, dst_nents, 0, 1547 0, 0, 0); 1548 return ERR_PTR(-ENOMEM); 1549 } 1550 1551 if (mapped_src_nents == 1 && 1552 iv_dma + ivsize == sg_dma_address(req->src)) { 1553 in_contig = true; 1554 sec4_sg_ents = 0; 1555 } else { 1556 in_contig = false; 1557 sec4_sg_ents = 1 + mapped_src_nents; 1558 } 1559 dst_sg_idx = sec4_sg_ents; 1560 sec4_sg_ents += mapped_dst_nents > 1 ? mapped_dst_nents : 0; 1561 sec4_sg_bytes = sec4_sg_ents * sizeof(struct sec4_sg_entry); 1562 1563 /* allocate space for base edesc and hw desc commands, link tables */ 1564 edesc = kzalloc(sizeof(*edesc) + desc_bytes + sec4_sg_bytes, 1565 GFP_DMA | flags); 1566 if (!edesc) { 1567 dev_err(jrdev, "could not allocate extended descriptor\n"); 1568 caam_unmap(jrdev, req->src, req->dst, src_nents, dst_nents, 1569 iv_dma, ivsize, 0, 0); 1570 return ERR_PTR(-ENOMEM); 1571 } 1572 1573 edesc->src_nents = src_nents; 1574 edesc->dst_nents = dst_nents; 1575 edesc->sec4_sg_bytes = sec4_sg_bytes; 1576 edesc->sec4_sg = (void *)edesc + sizeof(struct ablkcipher_edesc) + 1577 desc_bytes; 1578 1579 if (!in_contig) { 1580 dma_to_sec4_sg_one(edesc->sec4_sg, iv_dma, ivsize, 0); 1581 sg_to_sec4_sg_last(req->src, mapped_src_nents, 1582 edesc->sec4_sg + 1, 0); 1583 } 1584 1585 if (mapped_dst_nents > 1) { 1586 sg_to_sec4_sg_last(req->dst, mapped_dst_nents, 1587 edesc->sec4_sg + dst_sg_idx, 0); 1588 } 1589 1590 edesc->sec4_sg_dma = dma_map_single(jrdev, edesc->sec4_sg, 1591 sec4_sg_bytes, DMA_TO_DEVICE); 1592 if (dma_mapping_error(jrdev, edesc->sec4_sg_dma)) { 1593 dev_err(jrdev, "unable to map S/G table\n"); 1594 caam_unmap(jrdev, req->src, req->dst, src_nents, dst_nents, 1595 iv_dma, ivsize, 0, 0); 1596 kfree(edesc); 1597 return ERR_PTR(-ENOMEM); 1598 } 1599 1600 edesc->iv_dma = iv_dma; 1601 1602 #ifdef DEBUG 1603 print_hex_dump(KERN_ERR, "ablkcipher sec4_sg@"__stringify(__LINE__)": ", 1604 DUMP_PREFIX_ADDRESS, 16, 4, edesc->sec4_sg, 1605 sec4_sg_bytes, 1); 1606 #endif 1607 1608 *iv_contig_out = in_contig; 1609 return edesc; 1610 } 1611 1612 static int ablkcipher_encrypt(struct ablkcipher_request *req) 1613 { 1614 struct ablkcipher_edesc *edesc; 1615 struct crypto_ablkcipher *ablkcipher = crypto_ablkcipher_reqtfm(req); 1616 struct caam_ctx *ctx = crypto_ablkcipher_ctx(ablkcipher); 1617 struct device *jrdev = ctx->jrdev; 1618 bool iv_contig; 1619 u32 *desc; 1620 int ret = 0; 1621 1622 /* allocate extended descriptor */ 1623 edesc = ablkcipher_edesc_alloc(req, DESC_JOB_IO_LEN * 1624 CAAM_CMD_SZ, &iv_contig); 1625 if (IS_ERR(edesc)) 1626 return PTR_ERR(edesc); 1627 1628 /* Create and submit job descriptor*/ 1629 init_ablkcipher_job(ctx->sh_desc_enc, 1630 ctx->sh_desc_enc_dma, edesc, req, iv_contig); 1631 #ifdef DEBUG 1632 print_hex_dump(KERN_ERR, "ablkcipher jobdesc@"__stringify(__LINE__)": ", 1633 DUMP_PREFIX_ADDRESS, 16, 4, edesc->hw_desc, 1634 desc_bytes(edesc->hw_desc), 1); 1635 #endif 1636 desc = edesc->hw_desc; 1637 ret = caam_jr_enqueue(jrdev, desc, ablkcipher_encrypt_done, req); 1638 1639 if (!ret) { 1640 ret = -EINPROGRESS; 1641 } else { 1642 ablkcipher_unmap(jrdev, edesc, req); 1643 kfree(edesc); 1644 } 1645 1646 return ret; 1647 } 1648 1649 static int ablkcipher_decrypt(struct ablkcipher_request *req) 1650 { 1651 struct ablkcipher_edesc *edesc; 1652 struct crypto_ablkcipher *ablkcipher = crypto_ablkcipher_reqtfm(req); 1653 struct caam_ctx *ctx = crypto_ablkcipher_ctx(ablkcipher); 1654 struct device *jrdev = ctx->jrdev; 1655 bool iv_contig; 1656 u32 *desc; 1657 int ret = 0; 1658 1659 /* allocate extended descriptor */ 1660 edesc = ablkcipher_edesc_alloc(req, DESC_JOB_IO_LEN * 1661 CAAM_CMD_SZ, &iv_contig); 1662 if (IS_ERR(edesc)) 1663 return PTR_ERR(edesc); 1664 1665 /* Create and submit job descriptor*/ 1666 init_ablkcipher_job(ctx->sh_desc_dec, 1667 ctx->sh_desc_dec_dma, edesc, req, iv_contig); 1668 desc = edesc->hw_desc; 1669 #ifdef DEBUG 1670 print_hex_dump(KERN_ERR, "ablkcipher jobdesc@"__stringify(__LINE__)": ", 1671 DUMP_PREFIX_ADDRESS, 16, 4, edesc->hw_desc, 1672 desc_bytes(edesc->hw_desc), 1); 1673 #endif 1674 1675 ret = caam_jr_enqueue(jrdev, desc, ablkcipher_decrypt_done, req); 1676 if (!ret) { 1677 ret = -EINPROGRESS; 1678 } else { 1679 ablkcipher_unmap(jrdev, edesc, req); 1680 kfree(edesc); 1681 } 1682 1683 return ret; 1684 } 1685 1686 /* 1687 * allocate and map the ablkcipher extended descriptor 1688 * for ablkcipher givencrypt 1689 */ 1690 static struct ablkcipher_edesc *ablkcipher_giv_edesc_alloc( 1691 struct skcipher_givcrypt_request *greq, 1692 int desc_bytes, 1693 bool *iv_contig_out) 1694 { 1695 struct ablkcipher_request *req = &greq->creq; 1696 struct crypto_ablkcipher *ablkcipher = crypto_ablkcipher_reqtfm(req); 1697 struct caam_ctx *ctx = crypto_ablkcipher_ctx(ablkcipher); 1698 struct device *jrdev = ctx->jrdev; 1699 gfp_t flags = (req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP) ? 1700 GFP_KERNEL : GFP_ATOMIC; 1701 int src_nents, mapped_src_nents, dst_nents, mapped_dst_nents; 1702 struct ablkcipher_edesc *edesc; 1703 dma_addr_t iv_dma = 0; 1704 bool out_contig; 1705 int ivsize = crypto_ablkcipher_ivsize(ablkcipher); 1706 int dst_sg_idx, sec4_sg_ents, sec4_sg_bytes; 1707 1708 src_nents = sg_nents_for_len(req->src, req->nbytes); 1709 if (unlikely(src_nents < 0)) { 1710 dev_err(jrdev, "Insufficient bytes (%d) in src S/G\n", 1711 req->nbytes); 1712 return ERR_PTR(src_nents); 1713 } 1714 1715 if (likely(req->src == req->dst)) { 1716 mapped_src_nents = dma_map_sg(jrdev, req->src, src_nents, 1717 DMA_BIDIRECTIONAL); 1718 if (unlikely(!mapped_src_nents)) { 1719 dev_err(jrdev, "unable to map source\n"); 1720 return ERR_PTR(-ENOMEM); 1721 } 1722 1723 dst_nents = src_nents; 1724 mapped_dst_nents = src_nents; 1725 } else { 1726 mapped_src_nents = dma_map_sg(jrdev, req->src, src_nents, 1727 DMA_TO_DEVICE); 1728 if (unlikely(!mapped_src_nents)) { 1729 dev_err(jrdev, "unable to map source\n"); 1730 return ERR_PTR(-ENOMEM); 1731 } 1732 1733 dst_nents = sg_nents_for_len(req->dst, req->nbytes); 1734 if (unlikely(dst_nents < 0)) { 1735 dev_err(jrdev, "Insufficient bytes (%d) in dst S/G\n", 1736 req->nbytes); 1737 return ERR_PTR(dst_nents); 1738 } 1739 1740 mapped_dst_nents = dma_map_sg(jrdev, req->dst, dst_nents, 1741 DMA_FROM_DEVICE); 1742 if (unlikely(!mapped_dst_nents)) { 1743 dev_err(jrdev, "unable to map destination\n"); 1744 dma_unmap_sg(jrdev, req->src, src_nents, DMA_TO_DEVICE); 1745 return ERR_PTR(-ENOMEM); 1746 } 1747 } 1748 1749 /* 1750 * Check if iv can be contiguous with source and destination. 1751 * If so, include it. If not, create scatterlist. 1752 */ 1753 iv_dma = dma_map_single(jrdev, greq->giv, ivsize, DMA_TO_DEVICE); 1754 if (dma_mapping_error(jrdev, iv_dma)) { 1755 dev_err(jrdev, "unable to map IV\n"); 1756 caam_unmap(jrdev, req->src, req->dst, src_nents, dst_nents, 0, 1757 0, 0, 0); 1758 return ERR_PTR(-ENOMEM); 1759 } 1760 1761 sec4_sg_ents = mapped_src_nents > 1 ? mapped_src_nents : 0; 1762 dst_sg_idx = sec4_sg_ents; 1763 if (mapped_dst_nents == 1 && 1764 iv_dma + ivsize == sg_dma_address(req->dst)) { 1765 out_contig = true; 1766 } else { 1767 out_contig = false; 1768 sec4_sg_ents += 1 + mapped_dst_nents; 1769 } 1770 1771 /* allocate space for base edesc and hw desc commands, link tables */ 1772 sec4_sg_bytes = sec4_sg_ents * sizeof(struct sec4_sg_entry); 1773 edesc = kzalloc(sizeof(*edesc) + desc_bytes + sec4_sg_bytes, 1774 GFP_DMA | flags); 1775 if (!edesc) { 1776 dev_err(jrdev, "could not allocate extended descriptor\n"); 1777 caam_unmap(jrdev, req->src, req->dst, src_nents, dst_nents, 1778 iv_dma, ivsize, 0, 0); 1779 return ERR_PTR(-ENOMEM); 1780 } 1781 1782 edesc->src_nents = src_nents; 1783 edesc->dst_nents = dst_nents; 1784 edesc->sec4_sg_bytes = sec4_sg_bytes; 1785 edesc->sec4_sg = (void *)edesc + sizeof(struct ablkcipher_edesc) + 1786 desc_bytes; 1787 1788 if (mapped_src_nents > 1) 1789 sg_to_sec4_sg_last(req->src, mapped_src_nents, edesc->sec4_sg, 1790 0); 1791 1792 if (!out_contig) { 1793 dma_to_sec4_sg_one(edesc->sec4_sg + dst_sg_idx, 1794 iv_dma, ivsize, 0); 1795 sg_to_sec4_sg_last(req->dst, mapped_dst_nents, 1796 edesc->sec4_sg + dst_sg_idx + 1, 0); 1797 } 1798 1799 edesc->sec4_sg_dma = dma_map_single(jrdev, edesc->sec4_sg, 1800 sec4_sg_bytes, DMA_TO_DEVICE); 1801 if (dma_mapping_error(jrdev, edesc->sec4_sg_dma)) { 1802 dev_err(jrdev, "unable to map S/G table\n"); 1803 caam_unmap(jrdev, req->src, req->dst, src_nents, dst_nents, 1804 iv_dma, ivsize, 0, 0); 1805 kfree(edesc); 1806 return ERR_PTR(-ENOMEM); 1807 } 1808 edesc->iv_dma = iv_dma; 1809 1810 #ifdef DEBUG 1811 print_hex_dump(KERN_ERR, 1812 "ablkcipher sec4_sg@" __stringify(__LINE__) ": ", 1813 DUMP_PREFIX_ADDRESS, 16, 4, edesc->sec4_sg, 1814 sec4_sg_bytes, 1); 1815 #endif 1816 1817 *iv_contig_out = out_contig; 1818 return edesc; 1819 } 1820 1821 static int ablkcipher_givencrypt(struct skcipher_givcrypt_request *creq) 1822 { 1823 struct ablkcipher_request *req = &creq->creq; 1824 struct ablkcipher_edesc *edesc; 1825 struct crypto_ablkcipher *ablkcipher = crypto_ablkcipher_reqtfm(req); 1826 struct caam_ctx *ctx = crypto_ablkcipher_ctx(ablkcipher); 1827 struct device *jrdev = ctx->jrdev; 1828 bool iv_contig = false; 1829 u32 *desc; 1830 int ret = 0; 1831 1832 /* allocate extended descriptor */ 1833 edesc = ablkcipher_giv_edesc_alloc(creq, DESC_JOB_IO_LEN * 1834 CAAM_CMD_SZ, &iv_contig); 1835 if (IS_ERR(edesc)) 1836 return PTR_ERR(edesc); 1837 1838 /* Create and submit job descriptor*/ 1839 init_ablkcipher_giv_job(ctx->sh_desc_givenc, ctx->sh_desc_givenc_dma, 1840 edesc, req, iv_contig); 1841 #ifdef DEBUG 1842 print_hex_dump(KERN_ERR, 1843 "ablkcipher jobdesc@" __stringify(__LINE__) ": ", 1844 DUMP_PREFIX_ADDRESS, 16, 4, edesc->hw_desc, 1845 desc_bytes(edesc->hw_desc), 1); 1846 #endif 1847 desc = edesc->hw_desc; 1848 ret = caam_jr_enqueue(jrdev, desc, ablkcipher_encrypt_done, req); 1849 1850 if (!ret) { 1851 ret = -EINPROGRESS; 1852 } else { 1853 ablkcipher_unmap(jrdev, edesc, req); 1854 kfree(edesc); 1855 } 1856 1857 return ret; 1858 } 1859 1860 #define template_aead template_u.aead 1861 #define template_ablkcipher template_u.ablkcipher 1862 struct caam_alg_template { 1863 char name[CRYPTO_MAX_ALG_NAME]; 1864 char driver_name[CRYPTO_MAX_ALG_NAME]; 1865 unsigned int blocksize; 1866 u32 type; 1867 union { 1868 struct ablkcipher_alg ablkcipher; 1869 } template_u; 1870 u32 class1_alg_type; 1871 u32 class2_alg_type; 1872 }; 1873 1874 static struct caam_alg_template driver_algs[] = { 1875 /* ablkcipher descriptor */ 1876 { 1877 .name = "cbc(aes)", 1878 .driver_name = "cbc-aes-caam", 1879 .blocksize = AES_BLOCK_SIZE, 1880 .type = CRYPTO_ALG_TYPE_GIVCIPHER, 1881 .template_ablkcipher = { 1882 .setkey = ablkcipher_setkey, 1883 .encrypt = ablkcipher_encrypt, 1884 .decrypt = ablkcipher_decrypt, 1885 .givencrypt = ablkcipher_givencrypt, 1886 .geniv = "<built-in>", 1887 .min_keysize = AES_MIN_KEY_SIZE, 1888 .max_keysize = AES_MAX_KEY_SIZE, 1889 .ivsize = AES_BLOCK_SIZE, 1890 }, 1891 .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CBC, 1892 }, 1893 { 1894 .name = "cbc(des3_ede)", 1895 .driver_name = "cbc-3des-caam", 1896 .blocksize = DES3_EDE_BLOCK_SIZE, 1897 .type = CRYPTO_ALG_TYPE_GIVCIPHER, 1898 .template_ablkcipher = { 1899 .setkey = ablkcipher_setkey, 1900 .encrypt = ablkcipher_encrypt, 1901 .decrypt = ablkcipher_decrypt, 1902 .givencrypt = ablkcipher_givencrypt, 1903 .geniv = "<built-in>", 1904 .min_keysize = DES3_EDE_KEY_SIZE, 1905 .max_keysize = DES3_EDE_KEY_SIZE, 1906 .ivsize = DES3_EDE_BLOCK_SIZE, 1907 }, 1908 .class1_alg_type = OP_ALG_ALGSEL_3DES | OP_ALG_AAI_CBC, 1909 }, 1910 { 1911 .name = "cbc(des)", 1912 .driver_name = "cbc-des-caam", 1913 .blocksize = DES_BLOCK_SIZE, 1914 .type = CRYPTO_ALG_TYPE_GIVCIPHER, 1915 .template_ablkcipher = { 1916 .setkey = ablkcipher_setkey, 1917 .encrypt = ablkcipher_encrypt, 1918 .decrypt = ablkcipher_decrypt, 1919 .givencrypt = ablkcipher_givencrypt, 1920 .geniv = "<built-in>", 1921 .min_keysize = DES_KEY_SIZE, 1922 .max_keysize = DES_KEY_SIZE, 1923 .ivsize = DES_BLOCK_SIZE, 1924 }, 1925 .class1_alg_type = OP_ALG_ALGSEL_DES | OP_ALG_AAI_CBC, 1926 }, 1927 { 1928 .name = "ctr(aes)", 1929 .driver_name = "ctr-aes-caam", 1930 .blocksize = 1, 1931 .type = CRYPTO_ALG_TYPE_ABLKCIPHER, 1932 .template_ablkcipher = { 1933 .setkey = ablkcipher_setkey, 1934 .encrypt = ablkcipher_encrypt, 1935 .decrypt = ablkcipher_decrypt, 1936 .geniv = "chainiv", 1937 .min_keysize = AES_MIN_KEY_SIZE, 1938 .max_keysize = AES_MAX_KEY_SIZE, 1939 .ivsize = AES_BLOCK_SIZE, 1940 }, 1941 .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CTR_MOD128, 1942 }, 1943 { 1944 .name = "rfc3686(ctr(aes))", 1945 .driver_name = "rfc3686-ctr-aes-caam", 1946 .blocksize = 1, 1947 .type = CRYPTO_ALG_TYPE_GIVCIPHER, 1948 .template_ablkcipher = { 1949 .setkey = ablkcipher_setkey, 1950 .encrypt = ablkcipher_encrypt, 1951 .decrypt = ablkcipher_decrypt, 1952 .givencrypt = ablkcipher_givencrypt, 1953 .geniv = "<built-in>", 1954 .min_keysize = AES_MIN_KEY_SIZE + 1955 CTR_RFC3686_NONCE_SIZE, 1956 .max_keysize = AES_MAX_KEY_SIZE + 1957 CTR_RFC3686_NONCE_SIZE, 1958 .ivsize = CTR_RFC3686_IV_SIZE, 1959 }, 1960 .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CTR_MOD128, 1961 }, 1962 { 1963 .name = "xts(aes)", 1964 .driver_name = "xts-aes-caam", 1965 .blocksize = AES_BLOCK_SIZE, 1966 .type = CRYPTO_ALG_TYPE_ABLKCIPHER, 1967 .template_ablkcipher = { 1968 .setkey = xts_ablkcipher_setkey, 1969 .encrypt = ablkcipher_encrypt, 1970 .decrypt = ablkcipher_decrypt, 1971 .geniv = "eseqiv", 1972 .min_keysize = 2 * AES_MIN_KEY_SIZE, 1973 .max_keysize = 2 * AES_MAX_KEY_SIZE, 1974 .ivsize = AES_BLOCK_SIZE, 1975 }, 1976 .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_XTS, 1977 }, 1978 }; 1979 1980 static struct caam_aead_alg driver_aeads[] = { 1981 { 1982 .aead = { 1983 .base = { 1984 .cra_name = "rfc4106(gcm(aes))", 1985 .cra_driver_name = "rfc4106-gcm-aes-caam", 1986 .cra_blocksize = 1, 1987 }, 1988 .setkey = rfc4106_setkey, 1989 .setauthsize = rfc4106_setauthsize, 1990 .encrypt = ipsec_gcm_encrypt, 1991 .decrypt = ipsec_gcm_decrypt, 1992 .ivsize = 8, 1993 .maxauthsize = AES_BLOCK_SIZE, 1994 }, 1995 .caam = { 1996 .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_GCM, 1997 }, 1998 }, 1999 { 2000 .aead = { 2001 .base = { 2002 .cra_name = "rfc4543(gcm(aes))", 2003 .cra_driver_name = "rfc4543-gcm-aes-caam", 2004 .cra_blocksize = 1, 2005 }, 2006 .setkey = rfc4543_setkey, 2007 .setauthsize = rfc4543_setauthsize, 2008 .encrypt = ipsec_gcm_encrypt, 2009 .decrypt = ipsec_gcm_decrypt, 2010 .ivsize = 8, 2011 .maxauthsize = AES_BLOCK_SIZE, 2012 }, 2013 .caam = { 2014 .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_GCM, 2015 }, 2016 }, 2017 /* Galois Counter Mode */ 2018 { 2019 .aead = { 2020 .base = { 2021 .cra_name = "gcm(aes)", 2022 .cra_driver_name = "gcm-aes-caam", 2023 .cra_blocksize = 1, 2024 }, 2025 .setkey = gcm_setkey, 2026 .setauthsize = gcm_setauthsize, 2027 .encrypt = gcm_encrypt, 2028 .decrypt = gcm_decrypt, 2029 .ivsize = 12, 2030 .maxauthsize = AES_BLOCK_SIZE, 2031 }, 2032 .caam = { 2033 .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_GCM, 2034 }, 2035 }, 2036 /* single-pass ipsec_esp descriptor */ 2037 { 2038 .aead = { 2039 .base = { 2040 .cra_name = "authenc(hmac(md5)," 2041 "ecb(cipher_null))", 2042 .cra_driver_name = "authenc-hmac-md5-" 2043 "ecb-cipher_null-caam", 2044 .cra_blocksize = NULL_BLOCK_SIZE, 2045 }, 2046 .setkey = aead_setkey, 2047 .setauthsize = aead_setauthsize, 2048 .encrypt = aead_encrypt, 2049 .decrypt = aead_decrypt, 2050 .ivsize = NULL_IV_SIZE, 2051 .maxauthsize = MD5_DIGEST_SIZE, 2052 }, 2053 .caam = { 2054 .class2_alg_type = OP_ALG_ALGSEL_MD5 | 2055 OP_ALG_AAI_HMAC_PRECOMP, 2056 }, 2057 }, 2058 { 2059 .aead = { 2060 .base = { 2061 .cra_name = "authenc(hmac(sha1)," 2062 "ecb(cipher_null))", 2063 .cra_driver_name = "authenc-hmac-sha1-" 2064 "ecb-cipher_null-caam", 2065 .cra_blocksize = NULL_BLOCK_SIZE, 2066 }, 2067 .setkey = aead_setkey, 2068 .setauthsize = aead_setauthsize, 2069 .encrypt = aead_encrypt, 2070 .decrypt = aead_decrypt, 2071 .ivsize = NULL_IV_SIZE, 2072 .maxauthsize = SHA1_DIGEST_SIZE, 2073 }, 2074 .caam = { 2075 .class2_alg_type = OP_ALG_ALGSEL_SHA1 | 2076 OP_ALG_AAI_HMAC_PRECOMP, 2077 }, 2078 }, 2079 { 2080 .aead = { 2081 .base = { 2082 .cra_name = "authenc(hmac(sha224)," 2083 "ecb(cipher_null))", 2084 .cra_driver_name = "authenc-hmac-sha224-" 2085 "ecb-cipher_null-caam", 2086 .cra_blocksize = NULL_BLOCK_SIZE, 2087 }, 2088 .setkey = aead_setkey, 2089 .setauthsize = aead_setauthsize, 2090 .encrypt = aead_encrypt, 2091 .decrypt = aead_decrypt, 2092 .ivsize = NULL_IV_SIZE, 2093 .maxauthsize = SHA224_DIGEST_SIZE, 2094 }, 2095 .caam = { 2096 .class2_alg_type = OP_ALG_ALGSEL_SHA224 | 2097 OP_ALG_AAI_HMAC_PRECOMP, 2098 }, 2099 }, 2100 { 2101 .aead = { 2102 .base = { 2103 .cra_name = "authenc(hmac(sha256)," 2104 "ecb(cipher_null))", 2105 .cra_driver_name = "authenc-hmac-sha256-" 2106 "ecb-cipher_null-caam", 2107 .cra_blocksize = NULL_BLOCK_SIZE, 2108 }, 2109 .setkey = aead_setkey, 2110 .setauthsize = aead_setauthsize, 2111 .encrypt = aead_encrypt, 2112 .decrypt = aead_decrypt, 2113 .ivsize = NULL_IV_SIZE, 2114 .maxauthsize = SHA256_DIGEST_SIZE, 2115 }, 2116 .caam = { 2117 .class2_alg_type = OP_ALG_ALGSEL_SHA256 | 2118 OP_ALG_AAI_HMAC_PRECOMP, 2119 }, 2120 }, 2121 { 2122 .aead = { 2123 .base = { 2124 .cra_name = "authenc(hmac(sha384)," 2125 "ecb(cipher_null))", 2126 .cra_driver_name = "authenc-hmac-sha384-" 2127 "ecb-cipher_null-caam", 2128 .cra_blocksize = NULL_BLOCK_SIZE, 2129 }, 2130 .setkey = aead_setkey, 2131 .setauthsize = aead_setauthsize, 2132 .encrypt = aead_encrypt, 2133 .decrypt = aead_decrypt, 2134 .ivsize = NULL_IV_SIZE, 2135 .maxauthsize = SHA384_DIGEST_SIZE, 2136 }, 2137 .caam = { 2138 .class2_alg_type = OP_ALG_ALGSEL_SHA384 | 2139 OP_ALG_AAI_HMAC_PRECOMP, 2140 }, 2141 }, 2142 { 2143 .aead = { 2144 .base = { 2145 .cra_name = "authenc(hmac(sha512)," 2146 "ecb(cipher_null))", 2147 .cra_driver_name = "authenc-hmac-sha512-" 2148 "ecb-cipher_null-caam", 2149 .cra_blocksize = NULL_BLOCK_SIZE, 2150 }, 2151 .setkey = aead_setkey, 2152 .setauthsize = aead_setauthsize, 2153 .encrypt = aead_encrypt, 2154 .decrypt = aead_decrypt, 2155 .ivsize = NULL_IV_SIZE, 2156 .maxauthsize = SHA512_DIGEST_SIZE, 2157 }, 2158 .caam = { 2159 .class2_alg_type = OP_ALG_ALGSEL_SHA512 | 2160 OP_ALG_AAI_HMAC_PRECOMP, 2161 }, 2162 }, 2163 { 2164 .aead = { 2165 .base = { 2166 .cra_name = "authenc(hmac(md5),cbc(aes))", 2167 .cra_driver_name = "authenc-hmac-md5-" 2168 "cbc-aes-caam", 2169 .cra_blocksize = AES_BLOCK_SIZE, 2170 }, 2171 .setkey = aead_setkey, 2172 .setauthsize = aead_setauthsize, 2173 .encrypt = aead_encrypt, 2174 .decrypt = aead_decrypt, 2175 .ivsize = AES_BLOCK_SIZE, 2176 .maxauthsize = MD5_DIGEST_SIZE, 2177 }, 2178 .caam = { 2179 .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CBC, 2180 .class2_alg_type = OP_ALG_ALGSEL_MD5 | 2181 OP_ALG_AAI_HMAC_PRECOMP, 2182 }, 2183 }, 2184 { 2185 .aead = { 2186 .base = { 2187 .cra_name = "echainiv(authenc(hmac(md5)," 2188 "cbc(aes)))", 2189 .cra_driver_name = "echainiv-authenc-hmac-md5-" 2190 "cbc-aes-caam", 2191 .cra_blocksize = AES_BLOCK_SIZE, 2192 }, 2193 .setkey = aead_setkey, 2194 .setauthsize = aead_setauthsize, 2195 .encrypt = aead_encrypt, 2196 .decrypt = aead_decrypt, 2197 .ivsize = AES_BLOCK_SIZE, 2198 .maxauthsize = MD5_DIGEST_SIZE, 2199 }, 2200 .caam = { 2201 .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CBC, 2202 .class2_alg_type = OP_ALG_ALGSEL_MD5 | 2203 OP_ALG_AAI_HMAC_PRECOMP, 2204 .geniv = true, 2205 }, 2206 }, 2207 { 2208 .aead = { 2209 .base = { 2210 .cra_name = "authenc(hmac(sha1),cbc(aes))", 2211 .cra_driver_name = "authenc-hmac-sha1-" 2212 "cbc-aes-caam", 2213 .cra_blocksize = AES_BLOCK_SIZE, 2214 }, 2215 .setkey = aead_setkey, 2216 .setauthsize = aead_setauthsize, 2217 .encrypt = aead_encrypt, 2218 .decrypt = aead_decrypt, 2219 .ivsize = AES_BLOCK_SIZE, 2220 .maxauthsize = SHA1_DIGEST_SIZE, 2221 }, 2222 .caam = { 2223 .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CBC, 2224 .class2_alg_type = OP_ALG_ALGSEL_SHA1 | 2225 OP_ALG_AAI_HMAC_PRECOMP, 2226 }, 2227 }, 2228 { 2229 .aead = { 2230 .base = { 2231 .cra_name = "echainiv(authenc(hmac(sha1)," 2232 "cbc(aes)))", 2233 .cra_driver_name = "echainiv-authenc-" 2234 "hmac-sha1-cbc-aes-caam", 2235 .cra_blocksize = AES_BLOCK_SIZE, 2236 }, 2237 .setkey = aead_setkey, 2238 .setauthsize = aead_setauthsize, 2239 .encrypt = aead_encrypt, 2240 .decrypt = aead_decrypt, 2241 .ivsize = AES_BLOCK_SIZE, 2242 .maxauthsize = SHA1_DIGEST_SIZE, 2243 }, 2244 .caam = { 2245 .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CBC, 2246 .class2_alg_type = OP_ALG_ALGSEL_SHA1 | 2247 OP_ALG_AAI_HMAC_PRECOMP, 2248 .geniv = true, 2249 }, 2250 }, 2251 { 2252 .aead = { 2253 .base = { 2254 .cra_name = "authenc(hmac(sha224),cbc(aes))", 2255 .cra_driver_name = "authenc-hmac-sha224-" 2256 "cbc-aes-caam", 2257 .cra_blocksize = AES_BLOCK_SIZE, 2258 }, 2259 .setkey = aead_setkey, 2260 .setauthsize = aead_setauthsize, 2261 .encrypt = aead_encrypt, 2262 .decrypt = aead_decrypt, 2263 .ivsize = AES_BLOCK_SIZE, 2264 .maxauthsize = SHA224_DIGEST_SIZE, 2265 }, 2266 .caam = { 2267 .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CBC, 2268 .class2_alg_type = OP_ALG_ALGSEL_SHA224 | 2269 OP_ALG_AAI_HMAC_PRECOMP, 2270 }, 2271 }, 2272 { 2273 .aead = { 2274 .base = { 2275 .cra_name = "echainiv(authenc(hmac(sha224)," 2276 "cbc(aes)))", 2277 .cra_driver_name = "echainiv-authenc-" 2278 "hmac-sha224-cbc-aes-caam", 2279 .cra_blocksize = AES_BLOCK_SIZE, 2280 }, 2281 .setkey = aead_setkey, 2282 .setauthsize = aead_setauthsize, 2283 .encrypt = aead_encrypt, 2284 .decrypt = aead_decrypt, 2285 .ivsize = AES_BLOCK_SIZE, 2286 .maxauthsize = SHA224_DIGEST_SIZE, 2287 }, 2288 .caam = { 2289 .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CBC, 2290 .class2_alg_type = OP_ALG_ALGSEL_SHA224 | 2291 OP_ALG_AAI_HMAC_PRECOMP, 2292 .geniv = true, 2293 }, 2294 }, 2295 { 2296 .aead = { 2297 .base = { 2298 .cra_name = "authenc(hmac(sha256),cbc(aes))", 2299 .cra_driver_name = "authenc-hmac-sha256-" 2300 "cbc-aes-caam", 2301 .cra_blocksize = AES_BLOCK_SIZE, 2302 }, 2303 .setkey = aead_setkey, 2304 .setauthsize = aead_setauthsize, 2305 .encrypt = aead_encrypt, 2306 .decrypt = aead_decrypt, 2307 .ivsize = AES_BLOCK_SIZE, 2308 .maxauthsize = SHA256_DIGEST_SIZE, 2309 }, 2310 .caam = { 2311 .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CBC, 2312 .class2_alg_type = OP_ALG_ALGSEL_SHA256 | 2313 OP_ALG_AAI_HMAC_PRECOMP, 2314 }, 2315 }, 2316 { 2317 .aead = { 2318 .base = { 2319 .cra_name = "echainiv(authenc(hmac(sha256)," 2320 "cbc(aes)))", 2321 .cra_driver_name = "echainiv-authenc-" 2322 "hmac-sha256-cbc-aes-caam", 2323 .cra_blocksize = AES_BLOCK_SIZE, 2324 }, 2325 .setkey = aead_setkey, 2326 .setauthsize = aead_setauthsize, 2327 .encrypt = aead_encrypt, 2328 .decrypt = aead_decrypt, 2329 .ivsize = AES_BLOCK_SIZE, 2330 .maxauthsize = SHA256_DIGEST_SIZE, 2331 }, 2332 .caam = { 2333 .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CBC, 2334 .class2_alg_type = OP_ALG_ALGSEL_SHA256 | 2335 OP_ALG_AAI_HMAC_PRECOMP, 2336 .geniv = true, 2337 }, 2338 }, 2339 { 2340 .aead = { 2341 .base = { 2342 .cra_name = "authenc(hmac(sha384),cbc(aes))", 2343 .cra_driver_name = "authenc-hmac-sha384-" 2344 "cbc-aes-caam", 2345 .cra_blocksize = AES_BLOCK_SIZE, 2346 }, 2347 .setkey = aead_setkey, 2348 .setauthsize = aead_setauthsize, 2349 .encrypt = aead_encrypt, 2350 .decrypt = aead_decrypt, 2351 .ivsize = AES_BLOCK_SIZE, 2352 .maxauthsize = SHA384_DIGEST_SIZE, 2353 }, 2354 .caam = { 2355 .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CBC, 2356 .class2_alg_type = OP_ALG_ALGSEL_SHA384 | 2357 OP_ALG_AAI_HMAC_PRECOMP, 2358 }, 2359 }, 2360 { 2361 .aead = { 2362 .base = { 2363 .cra_name = "echainiv(authenc(hmac(sha384)," 2364 "cbc(aes)))", 2365 .cra_driver_name = "echainiv-authenc-" 2366 "hmac-sha384-cbc-aes-caam", 2367 .cra_blocksize = AES_BLOCK_SIZE, 2368 }, 2369 .setkey = aead_setkey, 2370 .setauthsize = aead_setauthsize, 2371 .encrypt = aead_encrypt, 2372 .decrypt = aead_decrypt, 2373 .ivsize = AES_BLOCK_SIZE, 2374 .maxauthsize = SHA384_DIGEST_SIZE, 2375 }, 2376 .caam = { 2377 .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CBC, 2378 .class2_alg_type = OP_ALG_ALGSEL_SHA384 | 2379 OP_ALG_AAI_HMAC_PRECOMP, 2380 .geniv = true, 2381 }, 2382 }, 2383 { 2384 .aead = { 2385 .base = { 2386 .cra_name = "authenc(hmac(sha512),cbc(aes))", 2387 .cra_driver_name = "authenc-hmac-sha512-" 2388 "cbc-aes-caam", 2389 .cra_blocksize = AES_BLOCK_SIZE, 2390 }, 2391 .setkey = aead_setkey, 2392 .setauthsize = aead_setauthsize, 2393 .encrypt = aead_encrypt, 2394 .decrypt = aead_decrypt, 2395 .ivsize = AES_BLOCK_SIZE, 2396 .maxauthsize = SHA512_DIGEST_SIZE, 2397 }, 2398 .caam = { 2399 .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CBC, 2400 .class2_alg_type = OP_ALG_ALGSEL_SHA512 | 2401 OP_ALG_AAI_HMAC_PRECOMP, 2402 }, 2403 }, 2404 { 2405 .aead = { 2406 .base = { 2407 .cra_name = "echainiv(authenc(hmac(sha512)," 2408 "cbc(aes)))", 2409 .cra_driver_name = "echainiv-authenc-" 2410 "hmac-sha512-cbc-aes-caam", 2411 .cra_blocksize = AES_BLOCK_SIZE, 2412 }, 2413 .setkey = aead_setkey, 2414 .setauthsize = aead_setauthsize, 2415 .encrypt = aead_encrypt, 2416 .decrypt = aead_decrypt, 2417 .ivsize = AES_BLOCK_SIZE, 2418 .maxauthsize = SHA512_DIGEST_SIZE, 2419 }, 2420 .caam = { 2421 .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CBC, 2422 .class2_alg_type = OP_ALG_ALGSEL_SHA512 | 2423 OP_ALG_AAI_HMAC_PRECOMP, 2424 .geniv = true, 2425 }, 2426 }, 2427 { 2428 .aead = { 2429 .base = { 2430 .cra_name = "authenc(hmac(md5),cbc(des3_ede))", 2431 .cra_driver_name = "authenc-hmac-md5-" 2432 "cbc-des3_ede-caam", 2433 .cra_blocksize = DES3_EDE_BLOCK_SIZE, 2434 }, 2435 .setkey = aead_setkey, 2436 .setauthsize = aead_setauthsize, 2437 .encrypt = aead_encrypt, 2438 .decrypt = aead_decrypt, 2439 .ivsize = DES3_EDE_BLOCK_SIZE, 2440 .maxauthsize = MD5_DIGEST_SIZE, 2441 }, 2442 .caam = { 2443 .class1_alg_type = OP_ALG_ALGSEL_3DES | OP_ALG_AAI_CBC, 2444 .class2_alg_type = OP_ALG_ALGSEL_MD5 | 2445 OP_ALG_AAI_HMAC_PRECOMP, 2446 } 2447 }, 2448 { 2449 .aead = { 2450 .base = { 2451 .cra_name = "echainiv(authenc(hmac(md5)," 2452 "cbc(des3_ede)))", 2453 .cra_driver_name = "echainiv-authenc-hmac-md5-" 2454 "cbc-des3_ede-caam", 2455 .cra_blocksize = DES3_EDE_BLOCK_SIZE, 2456 }, 2457 .setkey = aead_setkey, 2458 .setauthsize = aead_setauthsize, 2459 .encrypt = aead_encrypt, 2460 .decrypt = aead_decrypt, 2461 .ivsize = DES3_EDE_BLOCK_SIZE, 2462 .maxauthsize = MD5_DIGEST_SIZE, 2463 }, 2464 .caam = { 2465 .class1_alg_type = OP_ALG_ALGSEL_3DES | OP_ALG_AAI_CBC, 2466 .class2_alg_type = OP_ALG_ALGSEL_MD5 | 2467 OP_ALG_AAI_HMAC_PRECOMP, 2468 .geniv = true, 2469 } 2470 }, 2471 { 2472 .aead = { 2473 .base = { 2474 .cra_name = "authenc(hmac(sha1)," 2475 "cbc(des3_ede))", 2476 .cra_driver_name = "authenc-hmac-sha1-" 2477 "cbc-des3_ede-caam", 2478 .cra_blocksize = DES3_EDE_BLOCK_SIZE, 2479 }, 2480 .setkey = aead_setkey, 2481 .setauthsize = aead_setauthsize, 2482 .encrypt = aead_encrypt, 2483 .decrypt = aead_decrypt, 2484 .ivsize = DES3_EDE_BLOCK_SIZE, 2485 .maxauthsize = SHA1_DIGEST_SIZE, 2486 }, 2487 .caam = { 2488 .class1_alg_type = OP_ALG_ALGSEL_3DES | OP_ALG_AAI_CBC, 2489 .class2_alg_type = OP_ALG_ALGSEL_SHA1 | 2490 OP_ALG_AAI_HMAC_PRECOMP, 2491 }, 2492 }, 2493 { 2494 .aead = { 2495 .base = { 2496 .cra_name = "echainiv(authenc(hmac(sha1)," 2497 "cbc(des3_ede)))", 2498 .cra_driver_name = "echainiv-authenc-" 2499 "hmac-sha1-" 2500 "cbc-des3_ede-caam", 2501 .cra_blocksize = DES3_EDE_BLOCK_SIZE, 2502 }, 2503 .setkey = aead_setkey, 2504 .setauthsize = aead_setauthsize, 2505 .encrypt = aead_encrypt, 2506 .decrypt = aead_decrypt, 2507 .ivsize = DES3_EDE_BLOCK_SIZE, 2508 .maxauthsize = SHA1_DIGEST_SIZE, 2509 }, 2510 .caam = { 2511 .class1_alg_type = OP_ALG_ALGSEL_3DES | OP_ALG_AAI_CBC, 2512 .class2_alg_type = OP_ALG_ALGSEL_SHA1 | 2513 OP_ALG_AAI_HMAC_PRECOMP, 2514 .geniv = true, 2515 }, 2516 }, 2517 { 2518 .aead = { 2519 .base = { 2520 .cra_name = "authenc(hmac(sha224)," 2521 "cbc(des3_ede))", 2522 .cra_driver_name = "authenc-hmac-sha224-" 2523 "cbc-des3_ede-caam", 2524 .cra_blocksize = DES3_EDE_BLOCK_SIZE, 2525 }, 2526 .setkey = aead_setkey, 2527 .setauthsize = aead_setauthsize, 2528 .encrypt = aead_encrypt, 2529 .decrypt = aead_decrypt, 2530 .ivsize = DES3_EDE_BLOCK_SIZE, 2531 .maxauthsize = SHA224_DIGEST_SIZE, 2532 }, 2533 .caam = { 2534 .class1_alg_type = OP_ALG_ALGSEL_3DES | OP_ALG_AAI_CBC, 2535 .class2_alg_type = OP_ALG_ALGSEL_SHA224 | 2536 OP_ALG_AAI_HMAC_PRECOMP, 2537 }, 2538 }, 2539 { 2540 .aead = { 2541 .base = { 2542 .cra_name = "echainiv(authenc(hmac(sha224)," 2543 "cbc(des3_ede)))", 2544 .cra_driver_name = "echainiv-authenc-" 2545 "hmac-sha224-" 2546 "cbc-des3_ede-caam", 2547 .cra_blocksize = DES3_EDE_BLOCK_SIZE, 2548 }, 2549 .setkey = aead_setkey, 2550 .setauthsize = aead_setauthsize, 2551 .encrypt = aead_encrypt, 2552 .decrypt = aead_decrypt, 2553 .ivsize = DES3_EDE_BLOCK_SIZE, 2554 .maxauthsize = SHA224_DIGEST_SIZE, 2555 }, 2556 .caam = { 2557 .class1_alg_type = OP_ALG_ALGSEL_3DES | OP_ALG_AAI_CBC, 2558 .class2_alg_type = OP_ALG_ALGSEL_SHA224 | 2559 OP_ALG_AAI_HMAC_PRECOMP, 2560 .geniv = true, 2561 }, 2562 }, 2563 { 2564 .aead = { 2565 .base = { 2566 .cra_name = "authenc(hmac(sha256)," 2567 "cbc(des3_ede))", 2568 .cra_driver_name = "authenc-hmac-sha256-" 2569 "cbc-des3_ede-caam", 2570 .cra_blocksize = DES3_EDE_BLOCK_SIZE, 2571 }, 2572 .setkey = aead_setkey, 2573 .setauthsize = aead_setauthsize, 2574 .encrypt = aead_encrypt, 2575 .decrypt = aead_decrypt, 2576 .ivsize = DES3_EDE_BLOCK_SIZE, 2577 .maxauthsize = SHA256_DIGEST_SIZE, 2578 }, 2579 .caam = { 2580 .class1_alg_type = OP_ALG_ALGSEL_3DES | OP_ALG_AAI_CBC, 2581 .class2_alg_type = OP_ALG_ALGSEL_SHA256 | 2582 OP_ALG_AAI_HMAC_PRECOMP, 2583 }, 2584 }, 2585 { 2586 .aead = { 2587 .base = { 2588 .cra_name = "echainiv(authenc(hmac(sha256)," 2589 "cbc(des3_ede)))", 2590 .cra_driver_name = "echainiv-authenc-" 2591 "hmac-sha256-" 2592 "cbc-des3_ede-caam", 2593 .cra_blocksize = DES3_EDE_BLOCK_SIZE, 2594 }, 2595 .setkey = aead_setkey, 2596 .setauthsize = aead_setauthsize, 2597 .encrypt = aead_encrypt, 2598 .decrypt = aead_decrypt, 2599 .ivsize = DES3_EDE_BLOCK_SIZE, 2600 .maxauthsize = SHA256_DIGEST_SIZE, 2601 }, 2602 .caam = { 2603 .class1_alg_type = OP_ALG_ALGSEL_3DES | OP_ALG_AAI_CBC, 2604 .class2_alg_type = OP_ALG_ALGSEL_SHA256 | 2605 OP_ALG_AAI_HMAC_PRECOMP, 2606 .geniv = true, 2607 }, 2608 }, 2609 { 2610 .aead = { 2611 .base = { 2612 .cra_name = "authenc(hmac(sha384)," 2613 "cbc(des3_ede))", 2614 .cra_driver_name = "authenc-hmac-sha384-" 2615 "cbc-des3_ede-caam", 2616 .cra_blocksize = DES3_EDE_BLOCK_SIZE, 2617 }, 2618 .setkey = aead_setkey, 2619 .setauthsize = aead_setauthsize, 2620 .encrypt = aead_encrypt, 2621 .decrypt = aead_decrypt, 2622 .ivsize = DES3_EDE_BLOCK_SIZE, 2623 .maxauthsize = SHA384_DIGEST_SIZE, 2624 }, 2625 .caam = { 2626 .class1_alg_type = OP_ALG_ALGSEL_3DES | OP_ALG_AAI_CBC, 2627 .class2_alg_type = OP_ALG_ALGSEL_SHA384 | 2628 OP_ALG_AAI_HMAC_PRECOMP, 2629 }, 2630 }, 2631 { 2632 .aead = { 2633 .base = { 2634 .cra_name = "echainiv(authenc(hmac(sha384)," 2635 "cbc(des3_ede)))", 2636 .cra_driver_name = "echainiv-authenc-" 2637 "hmac-sha384-" 2638 "cbc-des3_ede-caam", 2639 .cra_blocksize = DES3_EDE_BLOCK_SIZE, 2640 }, 2641 .setkey = aead_setkey, 2642 .setauthsize = aead_setauthsize, 2643 .encrypt = aead_encrypt, 2644 .decrypt = aead_decrypt, 2645 .ivsize = DES3_EDE_BLOCK_SIZE, 2646 .maxauthsize = SHA384_DIGEST_SIZE, 2647 }, 2648 .caam = { 2649 .class1_alg_type = OP_ALG_ALGSEL_3DES | OP_ALG_AAI_CBC, 2650 .class2_alg_type = OP_ALG_ALGSEL_SHA384 | 2651 OP_ALG_AAI_HMAC_PRECOMP, 2652 .geniv = true, 2653 }, 2654 }, 2655 { 2656 .aead = { 2657 .base = { 2658 .cra_name = "authenc(hmac(sha512)," 2659 "cbc(des3_ede))", 2660 .cra_driver_name = "authenc-hmac-sha512-" 2661 "cbc-des3_ede-caam", 2662 .cra_blocksize = DES3_EDE_BLOCK_SIZE, 2663 }, 2664 .setkey = aead_setkey, 2665 .setauthsize = aead_setauthsize, 2666 .encrypt = aead_encrypt, 2667 .decrypt = aead_decrypt, 2668 .ivsize = DES3_EDE_BLOCK_SIZE, 2669 .maxauthsize = SHA512_DIGEST_SIZE, 2670 }, 2671 .caam = { 2672 .class1_alg_type = OP_ALG_ALGSEL_3DES | OP_ALG_AAI_CBC, 2673 .class2_alg_type = OP_ALG_ALGSEL_SHA512 | 2674 OP_ALG_AAI_HMAC_PRECOMP, 2675 }, 2676 }, 2677 { 2678 .aead = { 2679 .base = { 2680 .cra_name = "echainiv(authenc(hmac(sha512)," 2681 "cbc(des3_ede)))", 2682 .cra_driver_name = "echainiv-authenc-" 2683 "hmac-sha512-" 2684 "cbc-des3_ede-caam", 2685 .cra_blocksize = DES3_EDE_BLOCK_SIZE, 2686 }, 2687 .setkey = aead_setkey, 2688 .setauthsize = aead_setauthsize, 2689 .encrypt = aead_encrypt, 2690 .decrypt = aead_decrypt, 2691 .ivsize = DES3_EDE_BLOCK_SIZE, 2692 .maxauthsize = SHA512_DIGEST_SIZE, 2693 }, 2694 .caam = { 2695 .class1_alg_type = OP_ALG_ALGSEL_3DES | OP_ALG_AAI_CBC, 2696 .class2_alg_type = OP_ALG_ALGSEL_SHA512 | 2697 OP_ALG_AAI_HMAC_PRECOMP, 2698 .geniv = true, 2699 }, 2700 }, 2701 { 2702 .aead = { 2703 .base = { 2704 .cra_name = "authenc(hmac(md5),cbc(des))", 2705 .cra_driver_name = "authenc-hmac-md5-" 2706 "cbc-des-caam", 2707 .cra_blocksize = DES_BLOCK_SIZE, 2708 }, 2709 .setkey = aead_setkey, 2710 .setauthsize = aead_setauthsize, 2711 .encrypt = aead_encrypt, 2712 .decrypt = aead_decrypt, 2713 .ivsize = DES_BLOCK_SIZE, 2714 .maxauthsize = MD5_DIGEST_SIZE, 2715 }, 2716 .caam = { 2717 .class1_alg_type = OP_ALG_ALGSEL_DES | OP_ALG_AAI_CBC, 2718 .class2_alg_type = OP_ALG_ALGSEL_MD5 | 2719 OP_ALG_AAI_HMAC_PRECOMP, 2720 }, 2721 }, 2722 { 2723 .aead = { 2724 .base = { 2725 .cra_name = "echainiv(authenc(hmac(md5)," 2726 "cbc(des)))", 2727 .cra_driver_name = "echainiv-authenc-hmac-md5-" 2728 "cbc-des-caam", 2729 .cra_blocksize = DES_BLOCK_SIZE, 2730 }, 2731 .setkey = aead_setkey, 2732 .setauthsize = aead_setauthsize, 2733 .encrypt = aead_encrypt, 2734 .decrypt = aead_decrypt, 2735 .ivsize = DES_BLOCK_SIZE, 2736 .maxauthsize = MD5_DIGEST_SIZE, 2737 }, 2738 .caam = { 2739 .class1_alg_type = OP_ALG_ALGSEL_DES | OP_ALG_AAI_CBC, 2740 .class2_alg_type = OP_ALG_ALGSEL_MD5 | 2741 OP_ALG_AAI_HMAC_PRECOMP, 2742 .geniv = true, 2743 }, 2744 }, 2745 { 2746 .aead = { 2747 .base = { 2748 .cra_name = "authenc(hmac(sha1),cbc(des))", 2749 .cra_driver_name = "authenc-hmac-sha1-" 2750 "cbc-des-caam", 2751 .cra_blocksize = DES_BLOCK_SIZE, 2752 }, 2753 .setkey = aead_setkey, 2754 .setauthsize = aead_setauthsize, 2755 .encrypt = aead_encrypt, 2756 .decrypt = aead_decrypt, 2757 .ivsize = DES_BLOCK_SIZE, 2758 .maxauthsize = SHA1_DIGEST_SIZE, 2759 }, 2760 .caam = { 2761 .class1_alg_type = OP_ALG_ALGSEL_DES | OP_ALG_AAI_CBC, 2762 .class2_alg_type = OP_ALG_ALGSEL_SHA1 | 2763 OP_ALG_AAI_HMAC_PRECOMP, 2764 }, 2765 }, 2766 { 2767 .aead = { 2768 .base = { 2769 .cra_name = "echainiv(authenc(hmac(sha1)," 2770 "cbc(des)))", 2771 .cra_driver_name = "echainiv-authenc-" 2772 "hmac-sha1-cbc-des-caam", 2773 .cra_blocksize = DES_BLOCK_SIZE, 2774 }, 2775 .setkey = aead_setkey, 2776 .setauthsize = aead_setauthsize, 2777 .encrypt = aead_encrypt, 2778 .decrypt = aead_decrypt, 2779 .ivsize = DES_BLOCK_SIZE, 2780 .maxauthsize = SHA1_DIGEST_SIZE, 2781 }, 2782 .caam = { 2783 .class1_alg_type = OP_ALG_ALGSEL_DES | OP_ALG_AAI_CBC, 2784 .class2_alg_type = OP_ALG_ALGSEL_SHA1 | 2785 OP_ALG_AAI_HMAC_PRECOMP, 2786 .geniv = true, 2787 }, 2788 }, 2789 { 2790 .aead = { 2791 .base = { 2792 .cra_name = "authenc(hmac(sha224),cbc(des))", 2793 .cra_driver_name = "authenc-hmac-sha224-" 2794 "cbc-des-caam", 2795 .cra_blocksize = DES_BLOCK_SIZE, 2796 }, 2797 .setkey = aead_setkey, 2798 .setauthsize = aead_setauthsize, 2799 .encrypt = aead_encrypt, 2800 .decrypt = aead_decrypt, 2801 .ivsize = DES_BLOCK_SIZE, 2802 .maxauthsize = SHA224_DIGEST_SIZE, 2803 }, 2804 .caam = { 2805 .class1_alg_type = OP_ALG_ALGSEL_DES | OP_ALG_AAI_CBC, 2806 .class2_alg_type = OP_ALG_ALGSEL_SHA224 | 2807 OP_ALG_AAI_HMAC_PRECOMP, 2808 }, 2809 }, 2810 { 2811 .aead = { 2812 .base = { 2813 .cra_name = "echainiv(authenc(hmac(sha224)," 2814 "cbc(des)))", 2815 .cra_driver_name = "echainiv-authenc-" 2816 "hmac-sha224-cbc-des-caam", 2817 .cra_blocksize = DES_BLOCK_SIZE, 2818 }, 2819 .setkey = aead_setkey, 2820 .setauthsize = aead_setauthsize, 2821 .encrypt = aead_encrypt, 2822 .decrypt = aead_decrypt, 2823 .ivsize = DES_BLOCK_SIZE, 2824 .maxauthsize = SHA224_DIGEST_SIZE, 2825 }, 2826 .caam = { 2827 .class1_alg_type = OP_ALG_ALGSEL_DES | OP_ALG_AAI_CBC, 2828 .class2_alg_type = OP_ALG_ALGSEL_SHA224 | 2829 OP_ALG_AAI_HMAC_PRECOMP, 2830 .geniv = true, 2831 }, 2832 }, 2833 { 2834 .aead = { 2835 .base = { 2836 .cra_name = "authenc(hmac(sha256),cbc(des))", 2837 .cra_driver_name = "authenc-hmac-sha256-" 2838 "cbc-des-caam", 2839 .cra_blocksize = DES_BLOCK_SIZE, 2840 }, 2841 .setkey = aead_setkey, 2842 .setauthsize = aead_setauthsize, 2843 .encrypt = aead_encrypt, 2844 .decrypt = aead_decrypt, 2845 .ivsize = DES_BLOCK_SIZE, 2846 .maxauthsize = SHA256_DIGEST_SIZE, 2847 }, 2848 .caam = { 2849 .class1_alg_type = OP_ALG_ALGSEL_DES | OP_ALG_AAI_CBC, 2850 .class2_alg_type = OP_ALG_ALGSEL_SHA256 | 2851 OP_ALG_AAI_HMAC_PRECOMP, 2852 }, 2853 }, 2854 { 2855 .aead = { 2856 .base = { 2857 .cra_name = "echainiv(authenc(hmac(sha256)," 2858 "cbc(des)))", 2859 .cra_driver_name = "echainiv-authenc-" 2860 "hmac-sha256-cbc-des-caam", 2861 .cra_blocksize = DES_BLOCK_SIZE, 2862 }, 2863 .setkey = aead_setkey, 2864 .setauthsize = aead_setauthsize, 2865 .encrypt = aead_encrypt, 2866 .decrypt = aead_decrypt, 2867 .ivsize = DES_BLOCK_SIZE, 2868 .maxauthsize = SHA256_DIGEST_SIZE, 2869 }, 2870 .caam = { 2871 .class1_alg_type = OP_ALG_ALGSEL_DES | OP_ALG_AAI_CBC, 2872 .class2_alg_type = OP_ALG_ALGSEL_SHA256 | 2873 OP_ALG_AAI_HMAC_PRECOMP, 2874 .geniv = true, 2875 }, 2876 }, 2877 { 2878 .aead = { 2879 .base = { 2880 .cra_name = "authenc(hmac(sha384),cbc(des))", 2881 .cra_driver_name = "authenc-hmac-sha384-" 2882 "cbc-des-caam", 2883 .cra_blocksize = DES_BLOCK_SIZE, 2884 }, 2885 .setkey = aead_setkey, 2886 .setauthsize = aead_setauthsize, 2887 .encrypt = aead_encrypt, 2888 .decrypt = aead_decrypt, 2889 .ivsize = DES_BLOCK_SIZE, 2890 .maxauthsize = SHA384_DIGEST_SIZE, 2891 }, 2892 .caam = { 2893 .class1_alg_type = OP_ALG_ALGSEL_DES | OP_ALG_AAI_CBC, 2894 .class2_alg_type = OP_ALG_ALGSEL_SHA384 | 2895 OP_ALG_AAI_HMAC_PRECOMP, 2896 }, 2897 }, 2898 { 2899 .aead = { 2900 .base = { 2901 .cra_name = "echainiv(authenc(hmac(sha384)," 2902 "cbc(des)))", 2903 .cra_driver_name = "echainiv-authenc-" 2904 "hmac-sha384-cbc-des-caam", 2905 .cra_blocksize = DES_BLOCK_SIZE, 2906 }, 2907 .setkey = aead_setkey, 2908 .setauthsize = aead_setauthsize, 2909 .encrypt = aead_encrypt, 2910 .decrypt = aead_decrypt, 2911 .ivsize = DES_BLOCK_SIZE, 2912 .maxauthsize = SHA384_DIGEST_SIZE, 2913 }, 2914 .caam = { 2915 .class1_alg_type = OP_ALG_ALGSEL_DES | OP_ALG_AAI_CBC, 2916 .class2_alg_type = OP_ALG_ALGSEL_SHA384 | 2917 OP_ALG_AAI_HMAC_PRECOMP, 2918 .geniv = true, 2919 }, 2920 }, 2921 { 2922 .aead = { 2923 .base = { 2924 .cra_name = "authenc(hmac(sha512),cbc(des))", 2925 .cra_driver_name = "authenc-hmac-sha512-" 2926 "cbc-des-caam", 2927 .cra_blocksize = DES_BLOCK_SIZE, 2928 }, 2929 .setkey = aead_setkey, 2930 .setauthsize = aead_setauthsize, 2931 .encrypt = aead_encrypt, 2932 .decrypt = aead_decrypt, 2933 .ivsize = DES_BLOCK_SIZE, 2934 .maxauthsize = SHA512_DIGEST_SIZE, 2935 }, 2936 .caam = { 2937 .class1_alg_type = OP_ALG_ALGSEL_DES | OP_ALG_AAI_CBC, 2938 .class2_alg_type = OP_ALG_ALGSEL_SHA512 | 2939 OP_ALG_AAI_HMAC_PRECOMP, 2940 }, 2941 }, 2942 { 2943 .aead = { 2944 .base = { 2945 .cra_name = "echainiv(authenc(hmac(sha512)," 2946 "cbc(des)))", 2947 .cra_driver_name = "echainiv-authenc-" 2948 "hmac-sha512-cbc-des-caam", 2949 .cra_blocksize = DES_BLOCK_SIZE, 2950 }, 2951 .setkey = aead_setkey, 2952 .setauthsize = aead_setauthsize, 2953 .encrypt = aead_encrypt, 2954 .decrypt = aead_decrypt, 2955 .ivsize = DES_BLOCK_SIZE, 2956 .maxauthsize = SHA512_DIGEST_SIZE, 2957 }, 2958 .caam = { 2959 .class1_alg_type = OP_ALG_ALGSEL_DES | OP_ALG_AAI_CBC, 2960 .class2_alg_type = OP_ALG_ALGSEL_SHA512 | 2961 OP_ALG_AAI_HMAC_PRECOMP, 2962 .geniv = true, 2963 }, 2964 }, 2965 { 2966 .aead = { 2967 .base = { 2968 .cra_name = "authenc(hmac(md5)," 2969 "rfc3686(ctr(aes)))", 2970 .cra_driver_name = "authenc-hmac-md5-" 2971 "rfc3686-ctr-aes-caam", 2972 .cra_blocksize = 1, 2973 }, 2974 .setkey = aead_setkey, 2975 .setauthsize = aead_setauthsize, 2976 .encrypt = aead_encrypt, 2977 .decrypt = aead_decrypt, 2978 .ivsize = CTR_RFC3686_IV_SIZE, 2979 .maxauthsize = MD5_DIGEST_SIZE, 2980 }, 2981 .caam = { 2982 .class1_alg_type = OP_ALG_ALGSEL_AES | 2983 OP_ALG_AAI_CTR_MOD128, 2984 .class2_alg_type = OP_ALG_ALGSEL_MD5 | 2985 OP_ALG_AAI_HMAC_PRECOMP, 2986 .rfc3686 = true, 2987 }, 2988 }, 2989 { 2990 .aead = { 2991 .base = { 2992 .cra_name = "seqiv(authenc(" 2993 "hmac(md5),rfc3686(ctr(aes))))", 2994 .cra_driver_name = "seqiv-authenc-hmac-md5-" 2995 "rfc3686-ctr-aes-caam", 2996 .cra_blocksize = 1, 2997 }, 2998 .setkey = aead_setkey, 2999 .setauthsize = aead_setauthsize, 3000 .encrypt = aead_encrypt, 3001 .decrypt = aead_decrypt, 3002 .ivsize = CTR_RFC3686_IV_SIZE, 3003 .maxauthsize = MD5_DIGEST_SIZE, 3004 }, 3005 .caam = { 3006 .class1_alg_type = OP_ALG_ALGSEL_AES | 3007 OP_ALG_AAI_CTR_MOD128, 3008 .class2_alg_type = OP_ALG_ALGSEL_MD5 | 3009 OP_ALG_AAI_HMAC_PRECOMP, 3010 .rfc3686 = true, 3011 .geniv = true, 3012 }, 3013 }, 3014 { 3015 .aead = { 3016 .base = { 3017 .cra_name = "authenc(hmac(sha1)," 3018 "rfc3686(ctr(aes)))", 3019 .cra_driver_name = "authenc-hmac-sha1-" 3020 "rfc3686-ctr-aes-caam", 3021 .cra_blocksize = 1, 3022 }, 3023 .setkey = aead_setkey, 3024 .setauthsize = aead_setauthsize, 3025 .encrypt = aead_encrypt, 3026 .decrypt = aead_decrypt, 3027 .ivsize = CTR_RFC3686_IV_SIZE, 3028 .maxauthsize = SHA1_DIGEST_SIZE, 3029 }, 3030 .caam = { 3031 .class1_alg_type = OP_ALG_ALGSEL_AES | 3032 OP_ALG_AAI_CTR_MOD128, 3033 .class2_alg_type = OP_ALG_ALGSEL_SHA1 | 3034 OP_ALG_AAI_HMAC_PRECOMP, 3035 .rfc3686 = true, 3036 }, 3037 }, 3038 { 3039 .aead = { 3040 .base = { 3041 .cra_name = "seqiv(authenc(" 3042 "hmac(sha1),rfc3686(ctr(aes))))", 3043 .cra_driver_name = "seqiv-authenc-hmac-sha1-" 3044 "rfc3686-ctr-aes-caam", 3045 .cra_blocksize = 1, 3046 }, 3047 .setkey = aead_setkey, 3048 .setauthsize = aead_setauthsize, 3049 .encrypt = aead_encrypt, 3050 .decrypt = aead_decrypt, 3051 .ivsize = CTR_RFC3686_IV_SIZE, 3052 .maxauthsize = SHA1_DIGEST_SIZE, 3053 }, 3054 .caam = { 3055 .class1_alg_type = OP_ALG_ALGSEL_AES | 3056 OP_ALG_AAI_CTR_MOD128, 3057 .class2_alg_type = OP_ALG_ALGSEL_SHA1 | 3058 OP_ALG_AAI_HMAC_PRECOMP, 3059 .rfc3686 = true, 3060 .geniv = true, 3061 }, 3062 }, 3063 { 3064 .aead = { 3065 .base = { 3066 .cra_name = "authenc(hmac(sha224)," 3067 "rfc3686(ctr(aes)))", 3068 .cra_driver_name = "authenc-hmac-sha224-" 3069 "rfc3686-ctr-aes-caam", 3070 .cra_blocksize = 1, 3071 }, 3072 .setkey = aead_setkey, 3073 .setauthsize = aead_setauthsize, 3074 .encrypt = aead_encrypt, 3075 .decrypt = aead_decrypt, 3076 .ivsize = CTR_RFC3686_IV_SIZE, 3077 .maxauthsize = SHA224_DIGEST_SIZE, 3078 }, 3079 .caam = { 3080 .class1_alg_type = OP_ALG_ALGSEL_AES | 3081 OP_ALG_AAI_CTR_MOD128, 3082 .class2_alg_type = OP_ALG_ALGSEL_SHA224 | 3083 OP_ALG_AAI_HMAC_PRECOMP, 3084 .rfc3686 = true, 3085 }, 3086 }, 3087 { 3088 .aead = { 3089 .base = { 3090 .cra_name = "seqiv(authenc(" 3091 "hmac(sha224),rfc3686(ctr(aes))))", 3092 .cra_driver_name = "seqiv-authenc-hmac-sha224-" 3093 "rfc3686-ctr-aes-caam", 3094 .cra_blocksize = 1, 3095 }, 3096 .setkey = aead_setkey, 3097 .setauthsize = aead_setauthsize, 3098 .encrypt = aead_encrypt, 3099 .decrypt = aead_decrypt, 3100 .ivsize = CTR_RFC3686_IV_SIZE, 3101 .maxauthsize = SHA224_DIGEST_SIZE, 3102 }, 3103 .caam = { 3104 .class1_alg_type = OP_ALG_ALGSEL_AES | 3105 OP_ALG_AAI_CTR_MOD128, 3106 .class2_alg_type = OP_ALG_ALGSEL_SHA224 | 3107 OP_ALG_AAI_HMAC_PRECOMP, 3108 .rfc3686 = true, 3109 .geniv = true, 3110 }, 3111 }, 3112 { 3113 .aead = { 3114 .base = { 3115 .cra_name = "authenc(hmac(sha256)," 3116 "rfc3686(ctr(aes)))", 3117 .cra_driver_name = "authenc-hmac-sha256-" 3118 "rfc3686-ctr-aes-caam", 3119 .cra_blocksize = 1, 3120 }, 3121 .setkey = aead_setkey, 3122 .setauthsize = aead_setauthsize, 3123 .encrypt = aead_encrypt, 3124 .decrypt = aead_decrypt, 3125 .ivsize = CTR_RFC3686_IV_SIZE, 3126 .maxauthsize = SHA256_DIGEST_SIZE, 3127 }, 3128 .caam = { 3129 .class1_alg_type = OP_ALG_ALGSEL_AES | 3130 OP_ALG_AAI_CTR_MOD128, 3131 .class2_alg_type = OP_ALG_ALGSEL_SHA256 | 3132 OP_ALG_AAI_HMAC_PRECOMP, 3133 .rfc3686 = true, 3134 }, 3135 }, 3136 { 3137 .aead = { 3138 .base = { 3139 .cra_name = "seqiv(authenc(hmac(sha256)," 3140 "rfc3686(ctr(aes))))", 3141 .cra_driver_name = "seqiv-authenc-hmac-sha256-" 3142 "rfc3686-ctr-aes-caam", 3143 .cra_blocksize = 1, 3144 }, 3145 .setkey = aead_setkey, 3146 .setauthsize = aead_setauthsize, 3147 .encrypt = aead_encrypt, 3148 .decrypt = aead_decrypt, 3149 .ivsize = CTR_RFC3686_IV_SIZE, 3150 .maxauthsize = SHA256_DIGEST_SIZE, 3151 }, 3152 .caam = { 3153 .class1_alg_type = OP_ALG_ALGSEL_AES | 3154 OP_ALG_AAI_CTR_MOD128, 3155 .class2_alg_type = OP_ALG_ALGSEL_SHA256 | 3156 OP_ALG_AAI_HMAC_PRECOMP, 3157 .rfc3686 = true, 3158 .geniv = true, 3159 }, 3160 }, 3161 { 3162 .aead = { 3163 .base = { 3164 .cra_name = "authenc(hmac(sha384)," 3165 "rfc3686(ctr(aes)))", 3166 .cra_driver_name = "authenc-hmac-sha384-" 3167 "rfc3686-ctr-aes-caam", 3168 .cra_blocksize = 1, 3169 }, 3170 .setkey = aead_setkey, 3171 .setauthsize = aead_setauthsize, 3172 .encrypt = aead_encrypt, 3173 .decrypt = aead_decrypt, 3174 .ivsize = CTR_RFC3686_IV_SIZE, 3175 .maxauthsize = SHA384_DIGEST_SIZE, 3176 }, 3177 .caam = { 3178 .class1_alg_type = OP_ALG_ALGSEL_AES | 3179 OP_ALG_AAI_CTR_MOD128, 3180 .class2_alg_type = OP_ALG_ALGSEL_SHA384 | 3181 OP_ALG_AAI_HMAC_PRECOMP, 3182 .rfc3686 = true, 3183 }, 3184 }, 3185 { 3186 .aead = { 3187 .base = { 3188 .cra_name = "seqiv(authenc(hmac(sha384)," 3189 "rfc3686(ctr(aes))))", 3190 .cra_driver_name = "seqiv-authenc-hmac-sha384-" 3191 "rfc3686-ctr-aes-caam", 3192 .cra_blocksize = 1, 3193 }, 3194 .setkey = aead_setkey, 3195 .setauthsize = aead_setauthsize, 3196 .encrypt = aead_encrypt, 3197 .decrypt = aead_decrypt, 3198 .ivsize = CTR_RFC3686_IV_SIZE, 3199 .maxauthsize = SHA384_DIGEST_SIZE, 3200 }, 3201 .caam = { 3202 .class1_alg_type = OP_ALG_ALGSEL_AES | 3203 OP_ALG_AAI_CTR_MOD128, 3204 .class2_alg_type = OP_ALG_ALGSEL_SHA384 | 3205 OP_ALG_AAI_HMAC_PRECOMP, 3206 .rfc3686 = true, 3207 .geniv = true, 3208 }, 3209 }, 3210 { 3211 .aead = { 3212 .base = { 3213 .cra_name = "authenc(hmac(sha512)," 3214 "rfc3686(ctr(aes)))", 3215 .cra_driver_name = "authenc-hmac-sha512-" 3216 "rfc3686-ctr-aes-caam", 3217 .cra_blocksize = 1, 3218 }, 3219 .setkey = aead_setkey, 3220 .setauthsize = aead_setauthsize, 3221 .encrypt = aead_encrypt, 3222 .decrypt = aead_decrypt, 3223 .ivsize = CTR_RFC3686_IV_SIZE, 3224 .maxauthsize = SHA512_DIGEST_SIZE, 3225 }, 3226 .caam = { 3227 .class1_alg_type = OP_ALG_ALGSEL_AES | 3228 OP_ALG_AAI_CTR_MOD128, 3229 .class2_alg_type = OP_ALG_ALGSEL_SHA512 | 3230 OP_ALG_AAI_HMAC_PRECOMP, 3231 .rfc3686 = true, 3232 }, 3233 }, 3234 { 3235 .aead = { 3236 .base = { 3237 .cra_name = "seqiv(authenc(hmac(sha512)," 3238 "rfc3686(ctr(aes))))", 3239 .cra_driver_name = "seqiv-authenc-hmac-sha512-" 3240 "rfc3686-ctr-aes-caam", 3241 .cra_blocksize = 1, 3242 }, 3243 .setkey = aead_setkey, 3244 .setauthsize = aead_setauthsize, 3245 .encrypt = aead_encrypt, 3246 .decrypt = aead_decrypt, 3247 .ivsize = CTR_RFC3686_IV_SIZE, 3248 .maxauthsize = SHA512_DIGEST_SIZE, 3249 }, 3250 .caam = { 3251 .class1_alg_type = OP_ALG_ALGSEL_AES | 3252 OP_ALG_AAI_CTR_MOD128, 3253 .class2_alg_type = OP_ALG_ALGSEL_SHA512 | 3254 OP_ALG_AAI_HMAC_PRECOMP, 3255 .rfc3686 = true, 3256 .geniv = true, 3257 }, 3258 }, 3259 }; 3260 3261 struct caam_crypto_alg { 3262 struct crypto_alg crypto_alg; 3263 struct list_head entry; 3264 struct caam_alg_entry caam; 3265 }; 3266 3267 static int caam_init_common(struct caam_ctx *ctx, struct caam_alg_entry *caam) 3268 { 3269 dma_addr_t dma_addr; 3270 3271 ctx->jrdev = caam_jr_alloc(); 3272 if (IS_ERR(ctx->jrdev)) { 3273 pr_err("Job Ring Device allocation for transform failed\n"); 3274 return PTR_ERR(ctx->jrdev); 3275 } 3276 3277 dma_addr = dma_map_single_attrs(ctx->jrdev, ctx->sh_desc_enc, 3278 offsetof(struct caam_ctx, 3279 sh_desc_enc_dma), 3280 DMA_TO_DEVICE, DMA_ATTR_SKIP_CPU_SYNC); 3281 if (dma_mapping_error(ctx->jrdev, dma_addr)) { 3282 dev_err(ctx->jrdev, "unable to map key, shared descriptors\n"); 3283 caam_jr_free(ctx->jrdev); 3284 return -ENOMEM; 3285 } 3286 3287 ctx->sh_desc_enc_dma = dma_addr; 3288 ctx->sh_desc_dec_dma = dma_addr + offsetof(struct caam_ctx, 3289 sh_desc_dec); 3290 ctx->sh_desc_givenc_dma = dma_addr + offsetof(struct caam_ctx, 3291 sh_desc_givenc); 3292 ctx->key_dma = dma_addr + offsetof(struct caam_ctx, key); 3293 3294 /* copy descriptor header template value */ 3295 ctx->cdata.algtype = OP_TYPE_CLASS1_ALG | caam->class1_alg_type; 3296 ctx->adata.algtype = OP_TYPE_CLASS2_ALG | caam->class2_alg_type; 3297 3298 return 0; 3299 } 3300 3301 static int caam_cra_init(struct crypto_tfm *tfm) 3302 { 3303 struct crypto_alg *alg = tfm->__crt_alg; 3304 struct caam_crypto_alg *caam_alg = 3305 container_of(alg, struct caam_crypto_alg, crypto_alg); 3306 struct caam_ctx *ctx = crypto_tfm_ctx(tfm); 3307 3308 return caam_init_common(ctx, &caam_alg->caam); 3309 } 3310 3311 static int caam_aead_init(struct crypto_aead *tfm) 3312 { 3313 struct aead_alg *alg = crypto_aead_alg(tfm); 3314 struct caam_aead_alg *caam_alg = 3315 container_of(alg, struct caam_aead_alg, aead); 3316 struct caam_ctx *ctx = crypto_aead_ctx(tfm); 3317 3318 return caam_init_common(ctx, &caam_alg->caam); 3319 } 3320 3321 static void caam_exit_common(struct caam_ctx *ctx) 3322 { 3323 dma_unmap_single_attrs(ctx->jrdev, ctx->sh_desc_enc_dma, 3324 offsetof(struct caam_ctx, sh_desc_enc_dma), 3325 DMA_TO_DEVICE, DMA_ATTR_SKIP_CPU_SYNC); 3326 caam_jr_free(ctx->jrdev); 3327 } 3328 3329 static void caam_cra_exit(struct crypto_tfm *tfm) 3330 { 3331 caam_exit_common(crypto_tfm_ctx(tfm)); 3332 } 3333 3334 static void caam_aead_exit(struct crypto_aead *tfm) 3335 { 3336 caam_exit_common(crypto_aead_ctx(tfm)); 3337 } 3338 3339 static void __exit caam_algapi_exit(void) 3340 { 3341 3342 struct caam_crypto_alg *t_alg, *n; 3343 int i; 3344 3345 for (i = 0; i < ARRAY_SIZE(driver_aeads); i++) { 3346 struct caam_aead_alg *t_alg = driver_aeads + i; 3347 3348 if (t_alg->registered) 3349 crypto_unregister_aead(&t_alg->aead); 3350 } 3351 3352 if (!alg_list.next) 3353 return; 3354 3355 list_for_each_entry_safe(t_alg, n, &alg_list, entry) { 3356 crypto_unregister_alg(&t_alg->crypto_alg); 3357 list_del(&t_alg->entry); 3358 kfree(t_alg); 3359 } 3360 } 3361 3362 static struct caam_crypto_alg *caam_alg_alloc(struct caam_alg_template 3363 *template) 3364 { 3365 struct caam_crypto_alg *t_alg; 3366 struct crypto_alg *alg; 3367 3368 t_alg = kzalloc(sizeof(*t_alg), GFP_KERNEL); 3369 if (!t_alg) { 3370 pr_err("failed to allocate t_alg\n"); 3371 return ERR_PTR(-ENOMEM); 3372 } 3373 3374 alg = &t_alg->crypto_alg; 3375 3376 snprintf(alg->cra_name, CRYPTO_MAX_ALG_NAME, "%s", template->name); 3377 snprintf(alg->cra_driver_name, CRYPTO_MAX_ALG_NAME, "%s", 3378 template->driver_name); 3379 alg->cra_module = THIS_MODULE; 3380 alg->cra_init = caam_cra_init; 3381 alg->cra_exit = caam_cra_exit; 3382 alg->cra_priority = CAAM_CRA_PRIORITY; 3383 alg->cra_blocksize = template->blocksize; 3384 alg->cra_alignmask = 0; 3385 alg->cra_ctxsize = sizeof(struct caam_ctx); 3386 alg->cra_flags = CRYPTO_ALG_ASYNC | CRYPTO_ALG_KERN_DRIVER_ONLY | 3387 template->type; 3388 switch (template->type) { 3389 case CRYPTO_ALG_TYPE_GIVCIPHER: 3390 alg->cra_type = &crypto_givcipher_type; 3391 alg->cra_ablkcipher = template->template_ablkcipher; 3392 break; 3393 case CRYPTO_ALG_TYPE_ABLKCIPHER: 3394 alg->cra_type = &crypto_ablkcipher_type; 3395 alg->cra_ablkcipher = template->template_ablkcipher; 3396 break; 3397 } 3398 3399 t_alg->caam.class1_alg_type = template->class1_alg_type; 3400 t_alg->caam.class2_alg_type = template->class2_alg_type; 3401 3402 return t_alg; 3403 } 3404 3405 static void caam_aead_alg_init(struct caam_aead_alg *t_alg) 3406 { 3407 struct aead_alg *alg = &t_alg->aead; 3408 3409 alg->base.cra_module = THIS_MODULE; 3410 alg->base.cra_priority = CAAM_CRA_PRIORITY; 3411 alg->base.cra_ctxsize = sizeof(struct caam_ctx); 3412 alg->base.cra_flags = CRYPTO_ALG_ASYNC | CRYPTO_ALG_KERN_DRIVER_ONLY; 3413 3414 alg->init = caam_aead_init; 3415 alg->exit = caam_aead_exit; 3416 } 3417 3418 static int __init caam_algapi_init(void) 3419 { 3420 struct device_node *dev_node; 3421 struct platform_device *pdev; 3422 struct device *ctrldev; 3423 struct caam_drv_private *priv; 3424 int i = 0, err = 0; 3425 u32 cha_vid, cha_inst, des_inst, aes_inst, md_inst; 3426 unsigned int md_limit = SHA512_DIGEST_SIZE; 3427 bool registered = false; 3428 3429 dev_node = of_find_compatible_node(NULL, NULL, "fsl,sec-v4.0"); 3430 if (!dev_node) { 3431 dev_node = of_find_compatible_node(NULL, NULL, "fsl,sec4.0"); 3432 if (!dev_node) 3433 return -ENODEV; 3434 } 3435 3436 pdev = of_find_device_by_node(dev_node); 3437 if (!pdev) { 3438 of_node_put(dev_node); 3439 return -ENODEV; 3440 } 3441 3442 ctrldev = &pdev->dev; 3443 priv = dev_get_drvdata(ctrldev); 3444 of_node_put(dev_node); 3445 3446 /* 3447 * If priv is NULL, it's probably because the caam driver wasn't 3448 * properly initialized (e.g. RNG4 init failed). Thus, bail out here. 3449 */ 3450 if (!priv) 3451 return -ENODEV; 3452 3453 3454 INIT_LIST_HEAD(&alg_list); 3455 3456 /* 3457 * Register crypto algorithms the device supports. 3458 * First, detect presence and attributes of DES, AES, and MD blocks. 3459 */ 3460 cha_vid = rd_reg32(&priv->ctrl->perfmon.cha_id_ls); 3461 cha_inst = rd_reg32(&priv->ctrl->perfmon.cha_num_ls); 3462 des_inst = (cha_inst & CHA_ID_LS_DES_MASK) >> CHA_ID_LS_DES_SHIFT; 3463 aes_inst = (cha_inst & CHA_ID_LS_AES_MASK) >> CHA_ID_LS_AES_SHIFT; 3464 md_inst = (cha_inst & CHA_ID_LS_MD_MASK) >> CHA_ID_LS_MD_SHIFT; 3465 3466 /* If MD is present, limit digest size based on LP256 */ 3467 if (md_inst && ((cha_vid & CHA_ID_LS_MD_MASK) == CHA_ID_LS_MD_LP256)) 3468 md_limit = SHA256_DIGEST_SIZE; 3469 3470 for (i = 0; i < ARRAY_SIZE(driver_algs); i++) { 3471 struct caam_crypto_alg *t_alg; 3472 struct caam_alg_template *alg = driver_algs + i; 3473 u32 alg_sel = alg->class1_alg_type & OP_ALG_ALGSEL_MASK; 3474 3475 /* Skip DES algorithms if not supported by device */ 3476 if (!des_inst && 3477 ((alg_sel == OP_ALG_ALGSEL_3DES) || 3478 (alg_sel == OP_ALG_ALGSEL_DES))) 3479 continue; 3480 3481 /* Skip AES algorithms if not supported by device */ 3482 if (!aes_inst && (alg_sel == OP_ALG_ALGSEL_AES)) 3483 continue; 3484 3485 /* 3486 * Check support for AES modes not available 3487 * on LP devices. 3488 */ 3489 if ((cha_vid & CHA_ID_LS_AES_MASK) == CHA_ID_LS_AES_LP) 3490 if ((alg->class1_alg_type & OP_ALG_AAI_MASK) == 3491 OP_ALG_AAI_XTS) 3492 continue; 3493 3494 t_alg = caam_alg_alloc(alg); 3495 if (IS_ERR(t_alg)) { 3496 err = PTR_ERR(t_alg); 3497 pr_warn("%s alg allocation failed\n", alg->driver_name); 3498 continue; 3499 } 3500 3501 err = crypto_register_alg(&t_alg->crypto_alg); 3502 if (err) { 3503 pr_warn("%s alg registration failed\n", 3504 t_alg->crypto_alg.cra_driver_name); 3505 kfree(t_alg); 3506 continue; 3507 } 3508 3509 list_add_tail(&t_alg->entry, &alg_list); 3510 registered = true; 3511 } 3512 3513 for (i = 0; i < ARRAY_SIZE(driver_aeads); i++) { 3514 struct caam_aead_alg *t_alg = driver_aeads + i; 3515 u32 c1_alg_sel = t_alg->caam.class1_alg_type & 3516 OP_ALG_ALGSEL_MASK; 3517 u32 c2_alg_sel = t_alg->caam.class2_alg_type & 3518 OP_ALG_ALGSEL_MASK; 3519 u32 alg_aai = t_alg->caam.class1_alg_type & OP_ALG_AAI_MASK; 3520 3521 /* Skip DES algorithms if not supported by device */ 3522 if (!des_inst && 3523 ((c1_alg_sel == OP_ALG_ALGSEL_3DES) || 3524 (c1_alg_sel == OP_ALG_ALGSEL_DES))) 3525 continue; 3526 3527 /* Skip AES algorithms if not supported by device */ 3528 if (!aes_inst && (c1_alg_sel == OP_ALG_ALGSEL_AES)) 3529 continue; 3530 3531 /* 3532 * Check support for AES algorithms not available 3533 * on LP devices. 3534 */ 3535 if ((cha_vid & CHA_ID_LS_AES_MASK) == CHA_ID_LS_AES_LP) 3536 if (alg_aai == OP_ALG_AAI_GCM) 3537 continue; 3538 3539 /* 3540 * Skip algorithms requiring message digests 3541 * if MD or MD size is not supported by device. 3542 */ 3543 if (c2_alg_sel && 3544 (!md_inst || (t_alg->aead.maxauthsize > md_limit))) 3545 continue; 3546 3547 caam_aead_alg_init(t_alg); 3548 3549 err = crypto_register_aead(&t_alg->aead); 3550 if (err) { 3551 pr_warn("%s alg registration failed\n", 3552 t_alg->aead.base.cra_driver_name); 3553 continue; 3554 } 3555 3556 t_alg->registered = true; 3557 registered = true; 3558 } 3559 3560 if (registered) 3561 pr_info("caam algorithms registered in /proc/crypto\n"); 3562 3563 return err; 3564 } 3565 3566 module_init(caam_algapi_init); 3567 module_exit(caam_algapi_exit); 3568 3569 MODULE_LICENSE("GPL"); 3570 MODULE_DESCRIPTION("FSL CAAM support for crypto API"); 3571 MODULE_AUTHOR("Freescale Semiconductor - NMG/STC"); 3572