1 // SPDX-License-Identifier: GPL-2.0-or-later 2 /* 3 * Cryptographic API for algorithms (i.e., low-level API). 4 * 5 * Copyright (c) 2006 Herbert Xu <herbert@gondor.apana.org.au> 6 */ 7 8 #include <crypto/algapi.h> 9 #include <linux/err.h> 10 #include <linux/errno.h> 11 #include <linux/fips.h> 12 #include <linux/init.h> 13 #include <linux/kernel.h> 14 #include <linux/list.h> 15 #include <linux/module.h> 16 #include <linux/rtnetlink.h> 17 #include <linux/slab.h> 18 #include <linux/string.h> 19 20 #include "internal.h" 21 22 static LIST_HEAD(crypto_template_list); 23 24 static inline void crypto_check_module_sig(struct module *mod) 25 { 26 if (fips_enabled && mod && !module_sig_ok(mod)) 27 panic("Module %s signature verification failed in FIPS mode\n", 28 module_name(mod)); 29 } 30 31 static int crypto_check_alg(struct crypto_alg *alg) 32 { 33 crypto_check_module_sig(alg->cra_module); 34 35 if (!alg->cra_name[0] || !alg->cra_driver_name[0]) 36 return -EINVAL; 37 38 if (alg->cra_alignmask & (alg->cra_alignmask + 1)) 39 return -EINVAL; 40 41 /* General maximums for all algs. */ 42 if (alg->cra_alignmask > MAX_ALGAPI_ALIGNMASK) 43 return -EINVAL; 44 45 if (alg->cra_blocksize > MAX_ALGAPI_BLOCKSIZE) 46 return -EINVAL; 47 48 /* Lower maximums for specific alg types. */ 49 if (!alg->cra_type && (alg->cra_flags & CRYPTO_ALG_TYPE_MASK) == 50 CRYPTO_ALG_TYPE_CIPHER) { 51 if (alg->cra_alignmask > MAX_CIPHER_ALIGNMASK) 52 return -EINVAL; 53 54 if (alg->cra_blocksize > MAX_CIPHER_BLOCKSIZE) 55 return -EINVAL; 56 } 57 58 if (alg->cra_priority < 0) 59 return -EINVAL; 60 61 refcount_set(&alg->cra_refcnt, 1); 62 63 return 0; 64 } 65 66 static void crypto_free_instance(struct crypto_instance *inst) 67 { 68 if (!inst->alg.cra_type->free) { 69 inst->tmpl->free(inst); 70 return; 71 } 72 73 inst->alg.cra_type->free(inst); 74 } 75 76 static void crypto_destroy_instance(struct crypto_alg *alg) 77 { 78 struct crypto_instance *inst = (void *)alg; 79 struct crypto_template *tmpl = inst->tmpl; 80 81 crypto_free_instance(inst); 82 crypto_tmpl_put(tmpl); 83 } 84 85 static struct list_head *crypto_more_spawns(struct crypto_alg *alg, 86 struct list_head *stack, 87 struct list_head *top, 88 struct list_head *secondary_spawns) 89 { 90 struct crypto_spawn *spawn, *n; 91 92 spawn = list_first_entry_or_null(stack, struct crypto_spawn, list); 93 if (!spawn) 94 return NULL; 95 96 n = list_next_entry(spawn, list); 97 98 if (spawn->alg && &n->list != stack && !n->alg) 99 n->alg = (n->list.next == stack) ? alg : 100 &list_next_entry(n, list)->inst->alg; 101 102 list_move(&spawn->list, secondary_spawns); 103 104 return &n->list == stack ? top : &n->inst->alg.cra_users; 105 } 106 107 static void crypto_remove_instance(struct crypto_instance *inst, 108 struct list_head *list) 109 { 110 struct crypto_template *tmpl = inst->tmpl; 111 112 if (crypto_is_dead(&inst->alg)) 113 return; 114 115 inst->alg.cra_flags |= CRYPTO_ALG_DEAD; 116 if (hlist_unhashed(&inst->list)) 117 return; 118 119 if (!tmpl || !crypto_tmpl_get(tmpl)) 120 return; 121 122 list_move(&inst->alg.cra_list, list); 123 hlist_del(&inst->list); 124 inst->alg.cra_destroy = crypto_destroy_instance; 125 126 BUG_ON(!list_empty(&inst->alg.cra_users)); 127 } 128 129 void crypto_remove_spawns(struct crypto_alg *alg, struct list_head *list, 130 struct crypto_alg *nalg) 131 { 132 u32 new_type = (nalg ?: alg)->cra_flags; 133 struct crypto_spawn *spawn, *n; 134 LIST_HEAD(secondary_spawns); 135 struct list_head *spawns; 136 LIST_HEAD(stack); 137 LIST_HEAD(top); 138 139 spawns = &alg->cra_users; 140 list_for_each_entry_safe(spawn, n, spawns, list) { 141 if ((spawn->alg->cra_flags ^ new_type) & spawn->mask) 142 continue; 143 144 list_move(&spawn->list, &top); 145 } 146 147 spawns = ⊤ 148 do { 149 while (!list_empty(spawns)) { 150 struct crypto_instance *inst; 151 152 spawn = list_first_entry(spawns, struct crypto_spawn, 153 list); 154 inst = spawn->inst; 155 156 BUG_ON(&inst->alg == alg); 157 158 list_move(&spawn->list, &stack); 159 160 if (&inst->alg == nalg) 161 break; 162 163 spawn->alg = NULL; 164 spawns = &inst->alg.cra_users; 165 166 /* 167 * We may encounter an unregistered instance here, since 168 * an instance's spawns are set up prior to the instance 169 * being registered. An unregistered instance will have 170 * NULL ->cra_users.next, since ->cra_users isn't 171 * properly initialized until registration. But an 172 * unregistered instance cannot have any users, so treat 173 * it the same as ->cra_users being empty. 174 */ 175 if (spawns->next == NULL) 176 break; 177 } 178 } while ((spawns = crypto_more_spawns(alg, &stack, &top, 179 &secondary_spawns))); 180 181 list_for_each_entry_safe(spawn, n, &secondary_spawns, list) { 182 if (spawn->alg) 183 list_move(&spawn->list, &spawn->alg->cra_users); 184 else 185 crypto_remove_instance(spawn->inst, list); 186 } 187 } 188 EXPORT_SYMBOL_GPL(crypto_remove_spawns); 189 190 static struct crypto_larval *__crypto_register_alg(struct crypto_alg *alg) 191 { 192 struct crypto_alg *q; 193 struct crypto_larval *larval; 194 int ret = -EAGAIN; 195 196 if (crypto_is_dead(alg)) 197 goto err; 198 199 INIT_LIST_HEAD(&alg->cra_users); 200 201 /* No cheating! */ 202 alg->cra_flags &= ~CRYPTO_ALG_TESTED; 203 204 ret = -EEXIST; 205 206 list_for_each_entry(q, &crypto_alg_list, cra_list) { 207 if (q == alg) 208 goto err; 209 210 if (crypto_is_moribund(q)) 211 continue; 212 213 if (crypto_is_larval(q)) { 214 if (!strcmp(alg->cra_driver_name, q->cra_driver_name)) 215 goto err; 216 continue; 217 } 218 219 if (!strcmp(q->cra_driver_name, alg->cra_name) || 220 !strcmp(q->cra_name, alg->cra_driver_name)) 221 goto err; 222 } 223 224 larval = crypto_larval_alloc(alg->cra_name, 225 alg->cra_flags | CRYPTO_ALG_TESTED, 0); 226 if (IS_ERR(larval)) 227 goto out; 228 229 ret = -ENOENT; 230 larval->adult = crypto_mod_get(alg); 231 if (!larval->adult) 232 goto free_larval; 233 234 refcount_set(&larval->alg.cra_refcnt, 1); 235 memcpy(larval->alg.cra_driver_name, alg->cra_driver_name, 236 CRYPTO_MAX_ALG_NAME); 237 larval->alg.cra_priority = alg->cra_priority; 238 239 list_add(&alg->cra_list, &crypto_alg_list); 240 list_add(&larval->alg.cra_list, &crypto_alg_list); 241 242 crypto_stats_init(alg); 243 244 out: 245 return larval; 246 247 free_larval: 248 kfree(larval); 249 err: 250 larval = ERR_PTR(ret); 251 goto out; 252 } 253 254 void crypto_alg_tested(const char *name, int err) 255 { 256 struct crypto_larval *test; 257 struct crypto_alg *alg; 258 struct crypto_alg *q; 259 LIST_HEAD(list); 260 261 down_write(&crypto_alg_sem); 262 list_for_each_entry(q, &crypto_alg_list, cra_list) { 263 if (crypto_is_moribund(q) || !crypto_is_larval(q)) 264 continue; 265 266 test = (struct crypto_larval *)q; 267 268 if (!strcmp(q->cra_driver_name, name)) 269 goto found; 270 } 271 272 pr_err("alg: Unexpected test result for %s: %d\n", name, err); 273 goto unlock; 274 275 found: 276 q->cra_flags |= CRYPTO_ALG_DEAD; 277 alg = test->adult; 278 if (err || list_empty(&alg->cra_list)) 279 goto complete; 280 281 alg->cra_flags |= CRYPTO_ALG_TESTED; 282 283 list_for_each_entry(q, &crypto_alg_list, cra_list) { 284 if (q == alg) 285 continue; 286 287 if (crypto_is_moribund(q)) 288 continue; 289 290 if (crypto_is_larval(q)) { 291 struct crypto_larval *larval = (void *)q; 292 293 /* 294 * Check to see if either our generic name or 295 * specific name can satisfy the name requested 296 * by the larval entry q. 297 */ 298 if (strcmp(alg->cra_name, q->cra_name) && 299 strcmp(alg->cra_driver_name, q->cra_name)) 300 continue; 301 302 if (larval->adult) 303 continue; 304 if ((q->cra_flags ^ alg->cra_flags) & larval->mask) 305 continue; 306 if (!crypto_mod_get(alg)) 307 continue; 308 309 larval->adult = alg; 310 continue; 311 } 312 313 if (strcmp(alg->cra_name, q->cra_name)) 314 continue; 315 316 if (strcmp(alg->cra_driver_name, q->cra_driver_name) && 317 q->cra_priority > alg->cra_priority) 318 continue; 319 320 crypto_remove_spawns(q, &list, alg); 321 } 322 323 complete: 324 complete_all(&test->completion); 325 326 unlock: 327 up_write(&crypto_alg_sem); 328 329 crypto_remove_final(&list); 330 } 331 EXPORT_SYMBOL_GPL(crypto_alg_tested); 332 333 void crypto_remove_final(struct list_head *list) 334 { 335 struct crypto_alg *alg; 336 struct crypto_alg *n; 337 338 list_for_each_entry_safe(alg, n, list, cra_list) { 339 list_del_init(&alg->cra_list); 340 crypto_alg_put(alg); 341 } 342 } 343 EXPORT_SYMBOL_GPL(crypto_remove_final); 344 345 static void crypto_wait_for_test(struct crypto_larval *larval) 346 { 347 int err; 348 349 err = crypto_probing_notify(CRYPTO_MSG_ALG_REGISTER, larval->adult); 350 if (err != NOTIFY_STOP) { 351 if (WARN_ON(err != NOTIFY_DONE)) 352 goto out; 353 crypto_alg_tested(larval->alg.cra_driver_name, 0); 354 } 355 356 err = wait_for_completion_killable(&larval->completion); 357 WARN_ON(err); 358 if (!err) 359 crypto_probing_notify(CRYPTO_MSG_ALG_LOADED, larval); 360 361 out: 362 crypto_larval_kill(&larval->alg); 363 } 364 365 int crypto_register_alg(struct crypto_alg *alg) 366 { 367 struct crypto_larval *larval; 368 int err; 369 370 alg->cra_flags &= ~CRYPTO_ALG_DEAD; 371 err = crypto_check_alg(alg); 372 if (err) 373 return err; 374 375 down_write(&crypto_alg_sem); 376 larval = __crypto_register_alg(alg); 377 up_write(&crypto_alg_sem); 378 379 if (IS_ERR(larval)) 380 return PTR_ERR(larval); 381 382 crypto_wait_for_test(larval); 383 return 0; 384 } 385 EXPORT_SYMBOL_GPL(crypto_register_alg); 386 387 static int crypto_remove_alg(struct crypto_alg *alg, struct list_head *list) 388 { 389 if (unlikely(list_empty(&alg->cra_list))) 390 return -ENOENT; 391 392 alg->cra_flags |= CRYPTO_ALG_DEAD; 393 394 list_del_init(&alg->cra_list); 395 crypto_remove_spawns(alg, list, NULL); 396 397 return 0; 398 } 399 400 int crypto_unregister_alg(struct crypto_alg *alg) 401 { 402 int ret; 403 LIST_HEAD(list); 404 405 down_write(&crypto_alg_sem); 406 ret = crypto_remove_alg(alg, &list); 407 up_write(&crypto_alg_sem); 408 409 if (ret) 410 return ret; 411 412 BUG_ON(refcount_read(&alg->cra_refcnt) != 1); 413 if (alg->cra_destroy) 414 alg->cra_destroy(alg); 415 416 crypto_remove_final(&list); 417 return 0; 418 } 419 EXPORT_SYMBOL_GPL(crypto_unregister_alg); 420 421 int crypto_register_algs(struct crypto_alg *algs, int count) 422 { 423 int i, ret; 424 425 for (i = 0; i < count; i++) { 426 ret = crypto_register_alg(&algs[i]); 427 if (ret) 428 goto err; 429 } 430 431 return 0; 432 433 err: 434 for (--i; i >= 0; --i) 435 crypto_unregister_alg(&algs[i]); 436 437 return ret; 438 } 439 EXPORT_SYMBOL_GPL(crypto_register_algs); 440 441 int crypto_unregister_algs(struct crypto_alg *algs, int count) 442 { 443 int i, ret; 444 445 for (i = 0; i < count; i++) { 446 ret = crypto_unregister_alg(&algs[i]); 447 if (ret) 448 pr_err("Failed to unregister %s %s: %d\n", 449 algs[i].cra_driver_name, algs[i].cra_name, ret); 450 } 451 452 return 0; 453 } 454 EXPORT_SYMBOL_GPL(crypto_unregister_algs); 455 456 int crypto_register_template(struct crypto_template *tmpl) 457 { 458 struct crypto_template *q; 459 int err = -EEXIST; 460 461 down_write(&crypto_alg_sem); 462 463 crypto_check_module_sig(tmpl->module); 464 465 list_for_each_entry(q, &crypto_template_list, list) { 466 if (q == tmpl) 467 goto out; 468 } 469 470 list_add(&tmpl->list, &crypto_template_list); 471 err = 0; 472 out: 473 up_write(&crypto_alg_sem); 474 return err; 475 } 476 EXPORT_SYMBOL_GPL(crypto_register_template); 477 478 int crypto_register_templates(struct crypto_template *tmpls, int count) 479 { 480 int i, err; 481 482 for (i = 0; i < count; i++) { 483 err = crypto_register_template(&tmpls[i]); 484 if (err) 485 goto out; 486 } 487 return 0; 488 489 out: 490 for (--i; i >= 0; --i) 491 crypto_unregister_template(&tmpls[i]); 492 return err; 493 } 494 EXPORT_SYMBOL_GPL(crypto_register_templates); 495 496 void crypto_unregister_template(struct crypto_template *tmpl) 497 { 498 struct crypto_instance *inst; 499 struct hlist_node *n; 500 struct hlist_head *list; 501 LIST_HEAD(users); 502 503 down_write(&crypto_alg_sem); 504 505 BUG_ON(list_empty(&tmpl->list)); 506 list_del_init(&tmpl->list); 507 508 list = &tmpl->instances; 509 hlist_for_each_entry(inst, list, list) { 510 int err = crypto_remove_alg(&inst->alg, &users); 511 512 BUG_ON(err); 513 } 514 515 up_write(&crypto_alg_sem); 516 517 hlist_for_each_entry_safe(inst, n, list, list) { 518 BUG_ON(refcount_read(&inst->alg.cra_refcnt) != 1); 519 crypto_free_instance(inst); 520 } 521 crypto_remove_final(&users); 522 } 523 EXPORT_SYMBOL_GPL(crypto_unregister_template); 524 525 void crypto_unregister_templates(struct crypto_template *tmpls, int count) 526 { 527 int i; 528 529 for (i = count - 1; i >= 0; --i) 530 crypto_unregister_template(&tmpls[i]); 531 } 532 EXPORT_SYMBOL_GPL(crypto_unregister_templates); 533 534 static struct crypto_template *__crypto_lookup_template(const char *name) 535 { 536 struct crypto_template *q, *tmpl = NULL; 537 538 down_read(&crypto_alg_sem); 539 list_for_each_entry(q, &crypto_template_list, list) { 540 if (strcmp(q->name, name)) 541 continue; 542 if (unlikely(!crypto_tmpl_get(q))) 543 continue; 544 545 tmpl = q; 546 break; 547 } 548 up_read(&crypto_alg_sem); 549 550 return tmpl; 551 } 552 553 struct crypto_template *crypto_lookup_template(const char *name) 554 { 555 return try_then_request_module(__crypto_lookup_template(name), 556 "crypto-%s", name); 557 } 558 EXPORT_SYMBOL_GPL(crypto_lookup_template); 559 560 int crypto_register_instance(struct crypto_template *tmpl, 561 struct crypto_instance *inst) 562 { 563 struct crypto_larval *larval; 564 int err; 565 566 err = crypto_check_alg(&inst->alg); 567 if (err) 568 return err; 569 570 inst->alg.cra_module = tmpl->module; 571 inst->alg.cra_flags |= CRYPTO_ALG_INSTANCE; 572 573 down_write(&crypto_alg_sem); 574 575 larval = __crypto_register_alg(&inst->alg); 576 if (IS_ERR(larval)) 577 goto unlock; 578 579 hlist_add_head(&inst->list, &tmpl->instances); 580 inst->tmpl = tmpl; 581 582 unlock: 583 up_write(&crypto_alg_sem); 584 585 err = PTR_ERR(larval); 586 if (IS_ERR(larval)) 587 goto err; 588 589 crypto_wait_for_test(larval); 590 err = 0; 591 592 err: 593 return err; 594 } 595 EXPORT_SYMBOL_GPL(crypto_register_instance); 596 597 int crypto_unregister_instance(struct crypto_instance *inst) 598 { 599 LIST_HEAD(list); 600 601 down_write(&crypto_alg_sem); 602 603 crypto_remove_spawns(&inst->alg, &list, NULL); 604 crypto_remove_instance(inst, &list); 605 606 up_write(&crypto_alg_sem); 607 608 crypto_remove_final(&list); 609 610 return 0; 611 } 612 EXPORT_SYMBOL_GPL(crypto_unregister_instance); 613 614 int crypto_init_spawn(struct crypto_spawn *spawn, struct crypto_alg *alg, 615 struct crypto_instance *inst, u32 mask) 616 { 617 int err = -EAGAIN; 618 619 if (WARN_ON_ONCE(inst == NULL)) 620 return -EINVAL; 621 622 spawn->inst = inst; 623 spawn->mask = mask; 624 625 down_write(&crypto_alg_sem); 626 if (!crypto_is_moribund(alg)) { 627 list_add(&spawn->list, &alg->cra_users); 628 spawn->alg = alg; 629 err = 0; 630 } 631 up_write(&crypto_alg_sem); 632 633 return err; 634 } 635 EXPORT_SYMBOL_GPL(crypto_init_spawn); 636 637 int crypto_init_spawn2(struct crypto_spawn *spawn, struct crypto_alg *alg, 638 struct crypto_instance *inst, 639 const struct crypto_type *frontend) 640 { 641 int err = -EINVAL; 642 643 if ((alg->cra_flags ^ frontend->type) & frontend->maskset) 644 goto out; 645 646 spawn->frontend = frontend; 647 err = crypto_init_spawn(spawn, alg, inst, frontend->maskset); 648 649 out: 650 return err; 651 } 652 EXPORT_SYMBOL_GPL(crypto_init_spawn2); 653 654 int crypto_grab_spawn(struct crypto_spawn *spawn, const char *name, 655 u32 type, u32 mask) 656 { 657 struct crypto_alg *alg; 658 int err; 659 660 alg = crypto_find_alg(name, spawn->frontend, type, mask); 661 if (IS_ERR(alg)) 662 return PTR_ERR(alg); 663 664 err = crypto_init_spawn(spawn, alg, spawn->inst, mask); 665 crypto_mod_put(alg); 666 return err; 667 } 668 EXPORT_SYMBOL_GPL(crypto_grab_spawn); 669 670 void crypto_drop_spawn(struct crypto_spawn *spawn) 671 { 672 if (!spawn->alg) 673 return; 674 675 down_write(&crypto_alg_sem); 676 list_del(&spawn->list); 677 up_write(&crypto_alg_sem); 678 } 679 EXPORT_SYMBOL_GPL(crypto_drop_spawn); 680 681 static struct crypto_alg *crypto_spawn_alg(struct crypto_spawn *spawn) 682 { 683 struct crypto_alg *alg; 684 struct crypto_alg *alg2; 685 686 down_read(&crypto_alg_sem); 687 alg = spawn->alg; 688 alg2 = alg; 689 if (alg2) 690 alg2 = crypto_mod_get(alg2); 691 up_read(&crypto_alg_sem); 692 693 if (!alg2) { 694 if (alg) 695 crypto_shoot_alg(alg); 696 return ERR_PTR(-EAGAIN); 697 } 698 699 return alg; 700 } 701 702 struct crypto_tfm *crypto_spawn_tfm(struct crypto_spawn *spawn, u32 type, 703 u32 mask) 704 { 705 struct crypto_alg *alg; 706 struct crypto_tfm *tfm; 707 708 alg = crypto_spawn_alg(spawn); 709 if (IS_ERR(alg)) 710 return ERR_CAST(alg); 711 712 tfm = ERR_PTR(-EINVAL); 713 if (unlikely((alg->cra_flags ^ type) & mask)) 714 goto out_put_alg; 715 716 tfm = __crypto_alloc_tfm(alg, type, mask); 717 if (IS_ERR(tfm)) 718 goto out_put_alg; 719 720 return tfm; 721 722 out_put_alg: 723 crypto_mod_put(alg); 724 return tfm; 725 } 726 EXPORT_SYMBOL_GPL(crypto_spawn_tfm); 727 728 void *crypto_spawn_tfm2(struct crypto_spawn *spawn) 729 { 730 struct crypto_alg *alg; 731 struct crypto_tfm *tfm; 732 733 alg = crypto_spawn_alg(spawn); 734 if (IS_ERR(alg)) 735 return ERR_CAST(alg); 736 737 tfm = crypto_create_tfm(alg, spawn->frontend); 738 if (IS_ERR(tfm)) 739 goto out_put_alg; 740 741 return tfm; 742 743 out_put_alg: 744 crypto_mod_put(alg); 745 return tfm; 746 } 747 EXPORT_SYMBOL_GPL(crypto_spawn_tfm2); 748 749 int crypto_register_notifier(struct notifier_block *nb) 750 { 751 return blocking_notifier_chain_register(&crypto_chain, nb); 752 } 753 EXPORT_SYMBOL_GPL(crypto_register_notifier); 754 755 int crypto_unregister_notifier(struct notifier_block *nb) 756 { 757 return blocking_notifier_chain_unregister(&crypto_chain, nb); 758 } 759 EXPORT_SYMBOL_GPL(crypto_unregister_notifier); 760 761 struct crypto_attr_type *crypto_get_attr_type(struct rtattr **tb) 762 { 763 struct rtattr *rta = tb[0]; 764 struct crypto_attr_type *algt; 765 766 if (!rta) 767 return ERR_PTR(-ENOENT); 768 if (RTA_PAYLOAD(rta) < sizeof(*algt)) 769 return ERR_PTR(-EINVAL); 770 if (rta->rta_type != CRYPTOA_TYPE) 771 return ERR_PTR(-EINVAL); 772 773 algt = RTA_DATA(rta); 774 775 return algt; 776 } 777 EXPORT_SYMBOL_GPL(crypto_get_attr_type); 778 779 int crypto_check_attr_type(struct rtattr **tb, u32 type) 780 { 781 struct crypto_attr_type *algt; 782 783 algt = crypto_get_attr_type(tb); 784 if (IS_ERR(algt)) 785 return PTR_ERR(algt); 786 787 if ((algt->type ^ type) & algt->mask) 788 return -EINVAL; 789 790 return 0; 791 } 792 EXPORT_SYMBOL_GPL(crypto_check_attr_type); 793 794 const char *crypto_attr_alg_name(struct rtattr *rta) 795 { 796 struct crypto_attr_alg *alga; 797 798 if (!rta) 799 return ERR_PTR(-ENOENT); 800 if (RTA_PAYLOAD(rta) < sizeof(*alga)) 801 return ERR_PTR(-EINVAL); 802 if (rta->rta_type != CRYPTOA_ALG) 803 return ERR_PTR(-EINVAL); 804 805 alga = RTA_DATA(rta); 806 alga->name[CRYPTO_MAX_ALG_NAME - 1] = 0; 807 808 return alga->name; 809 } 810 EXPORT_SYMBOL_GPL(crypto_attr_alg_name); 811 812 struct crypto_alg *crypto_attr_alg2(struct rtattr *rta, 813 const struct crypto_type *frontend, 814 u32 type, u32 mask) 815 { 816 const char *name; 817 818 name = crypto_attr_alg_name(rta); 819 if (IS_ERR(name)) 820 return ERR_CAST(name); 821 822 return crypto_find_alg(name, frontend, type, mask); 823 } 824 EXPORT_SYMBOL_GPL(crypto_attr_alg2); 825 826 int crypto_attr_u32(struct rtattr *rta, u32 *num) 827 { 828 struct crypto_attr_u32 *nu32; 829 830 if (!rta) 831 return -ENOENT; 832 if (RTA_PAYLOAD(rta) < sizeof(*nu32)) 833 return -EINVAL; 834 if (rta->rta_type != CRYPTOA_U32) 835 return -EINVAL; 836 837 nu32 = RTA_DATA(rta); 838 *num = nu32->num; 839 840 return 0; 841 } 842 EXPORT_SYMBOL_GPL(crypto_attr_u32); 843 844 int crypto_inst_setname(struct crypto_instance *inst, const char *name, 845 struct crypto_alg *alg) 846 { 847 if (snprintf(inst->alg.cra_name, CRYPTO_MAX_ALG_NAME, "%s(%s)", name, 848 alg->cra_name) >= CRYPTO_MAX_ALG_NAME) 849 return -ENAMETOOLONG; 850 851 if (snprintf(inst->alg.cra_driver_name, CRYPTO_MAX_ALG_NAME, "%s(%s)", 852 name, alg->cra_driver_name) >= CRYPTO_MAX_ALG_NAME) 853 return -ENAMETOOLONG; 854 855 return 0; 856 } 857 EXPORT_SYMBOL_GPL(crypto_inst_setname); 858 859 void *crypto_alloc_instance(const char *name, struct crypto_alg *alg, 860 unsigned int head) 861 { 862 struct crypto_instance *inst; 863 char *p; 864 int err; 865 866 p = kzalloc(head + sizeof(*inst) + sizeof(struct crypto_spawn), 867 GFP_KERNEL); 868 if (!p) 869 return ERR_PTR(-ENOMEM); 870 871 inst = (void *)(p + head); 872 873 err = crypto_inst_setname(inst, name, alg); 874 if (err) 875 goto err_free_inst; 876 877 return p; 878 879 err_free_inst: 880 kfree(p); 881 return ERR_PTR(err); 882 } 883 EXPORT_SYMBOL_GPL(crypto_alloc_instance); 884 885 void crypto_init_queue(struct crypto_queue *queue, unsigned int max_qlen) 886 { 887 INIT_LIST_HEAD(&queue->list); 888 queue->backlog = &queue->list; 889 queue->qlen = 0; 890 queue->max_qlen = max_qlen; 891 } 892 EXPORT_SYMBOL_GPL(crypto_init_queue); 893 894 int crypto_enqueue_request(struct crypto_queue *queue, 895 struct crypto_async_request *request) 896 { 897 int err = -EINPROGRESS; 898 899 if (unlikely(queue->qlen >= queue->max_qlen)) { 900 if (!(request->flags & CRYPTO_TFM_REQ_MAY_BACKLOG)) { 901 err = -ENOSPC; 902 goto out; 903 } 904 err = -EBUSY; 905 if (queue->backlog == &queue->list) 906 queue->backlog = &request->list; 907 } 908 909 queue->qlen++; 910 list_add_tail(&request->list, &queue->list); 911 912 out: 913 return err; 914 } 915 EXPORT_SYMBOL_GPL(crypto_enqueue_request); 916 917 struct crypto_async_request *crypto_dequeue_request(struct crypto_queue *queue) 918 { 919 struct list_head *request; 920 921 if (unlikely(!queue->qlen)) 922 return NULL; 923 924 queue->qlen--; 925 926 if (queue->backlog != &queue->list) 927 queue->backlog = queue->backlog->next; 928 929 request = queue->list.next; 930 list_del(request); 931 932 return list_entry(request, struct crypto_async_request, list); 933 } 934 EXPORT_SYMBOL_GPL(crypto_dequeue_request); 935 936 static inline void crypto_inc_byte(u8 *a, unsigned int size) 937 { 938 u8 *b = (a + size); 939 u8 c; 940 941 for (; size; size--) { 942 c = *--b + 1; 943 *b = c; 944 if (c) 945 break; 946 } 947 } 948 949 void crypto_inc(u8 *a, unsigned int size) 950 { 951 __be32 *b = (__be32 *)(a + size); 952 u32 c; 953 954 if (IS_ENABLED(CONFIG_HAVE_EFFICIENT_UNALIGNED_ACCESS) || 955 IS_ALIGNED((unsigned long)b, __alignof__(*b))) 956 for (; size >= 4; size -= 4) { 957 c = be32_to_cpu(*--b) + 1; 958 *b = cpu_to_be32(c); 959 if (likely(c)) 960 return; 961 } 962 963 crypto_inc_byte(a, size); 964 } 965 EXPORT_SYMBOL_GPL(crypto_inc); 966 967 void __crypto_xor(u8 *dst, const u8 *src1, const u8 *src2, unsigned int len) 968 { 969 int relalign = 0; 970 971 if (!IS_ENABLED(CONFIG_HAVE_EFFICIENT_UNALIGNED_ACCESS)) { 972 int size = sizeof(unsigned long); 973 int d = (((unsigned long)dst ^ (unsigned long)src1) | 974 ((unsigned long)dst ^ (unsigned long)src2)) & 975 (size - 1); 976 977 relalign = d ? 1 << __ffs(d) : size; 978 979 /* 980 * If we care about alignment, process as many bytes as 981 * needed to advance dst and src to values whose alignments 982 * equal their relative alignment. This will allow us to 983 * process the remainder of the input using optimal strides. 984 */ 985 while (((unsigned long)dst & (relalign - 1)) && len > 0) { 986 *dst++ = *src1++ ^ *src2++; 987 len--; 988 } 989 } 990 991 while (IS_ENABLED(CONFIG_64BIT) && len >= 8 && !(relalign & 7)) { 992 *(u64 *)dst = *(u64 *)src1 ^ *(u64 *)src2; 993 dst += 8; 994 src1 += 8; 995 src2 += 8; 996 len -= 8; 997 } 998 999 while (len >= 4 && !(relalign & 3)) { 1000 *(u32 *)dst = *(u32 *)src1 ^ *(u32 *)src2; 1001 dst += 4; 1002 src1 += 4; 1003 src2 += 4; 1004 len -= 4; 1005 } 1006 1007 while (len >= 2 && !(relalign & 1)) { 1008 *(u16 *)dst = *(u16 *)src1 ^ *(u16 *)src2; 1009 dst += 2; 1010 src1 += 2; 1011 src2 += 2; 1012 len -= 2; 1013 } 1014 1015 while (len--) 1016 *dst++ = *src1++ ^ *src2++; 1017 } 1018 EXPORT_SYMBOL_GPL(__crypto_xor); 1019 1020 unsigned int crypto_alg_extsize(struct crypto_alg *alg) 1021 { 1022 return alg->cra_ctxsize + 1023 (alg->cra_alignmask & ~(crypto_tfm_ctx_alignment() - 1)); 1024 } 1025 EXPORT_SYMBOL_GPL(crypto_alg_extsize); 1026 1027 int crypto_type_has_alg(const char *name, const struct crypto_type *frontend, 1028 u32 type, u32 mask) 1029 { 1030 int ret = 0; 1031 struct crypto_alg *alg = crypto_find_alg(name, frontend, type, mask); 1032 1033 if (!IS_ERR(alg)) { 1034 crypto_mod_put(alg); 1035 ret = 1; 1036 } 1037 1038 return ret; 1039 } 1040 EXPORT_SYMBOL_GPL(crypto_type_has_alg); 1041 1042 #ifdef CONFIG_CRYPTO_STATS 1043 void crypto_stats_init(struct crypto_alg *alg) 1044 { 1045 memset(&alg->stats, 0, sizeof(alg->stats)); 1046 } 1047 EXPORT_SYMBOL_GPL(crypto_stats_init); 1048 1049 void crypto_stats_get(struct crypto_alg *alg) 1050 { 1051 crypto_alg_get(alg); 1052 } 1053 EXPORT_SYMBOL_GPL(crypto_stats_get); 1054 1055 void crypto_stats_ablkcipher_encrypt(unsigned int nbytes, int ret, 1056 struct crypto_alg *alg) 1057 { 1058 if (ret && ret != -EINPROGRESS && ret != -EBUSY) { 1059 atomic64_inc(&alg->stats.cipher.err_cnt); 1060 } else { 1061 atomic64_inc(&alg->stats.cipher.encrypt_cnt); 1062 atomic64_add(nbytes, &alg->stats.cipher.encrypt_tlen); 1063 } 1064 crypto_alg_put(alg); 1065 } 1066 EXPORT_SYMBOL_GPL(crypto_stats_ablkcipher_encrypt); 1067 1068 void crypto_stats_ablkcipher_decrypt(unsigned int nbytes, int ret, 1069 struct crypto_alg *alg) 1070 { 1071 if (ret && ret != -EINPROGRESS && ret != -EBUSY) { 1072 atomic64_inc(&alg->stats.cipher.err_cnt); 1073 } else { 1074 atomic64_inc(&alg->stats.cipher.decrypt_cnt); 1075 atomic64_add(nbytes, &alg->stats.cipher.decrypt_tlen); 1076 } 1077 crypto_alg_put(alg); 1078 } 1079 EXPORT_SYMBOL_GPL(crypto_stats_ablkcipher_decrypt); 1080 1081 void crypto_stats_aead_encrypt(unsigned int cryptlen, struct crypto_alg *alg, 1082 int ret) 1083 { 1084 if (ret && ret != -EINPROGRESS && ret != -EBUSY) { 1085 atomic64_inc(&alg->stats.aead.err_cnt); 1086 } else { 1087 atomic64_inc(&alg->stats.aead.encrypt_cnt); 1088 atomic64_add(cryptlen, &alg->stats.aead.encrypt_tlen); 1089 } 1090 crypto_alg_put(alg); 1091 } 1092 EXPORT_SYMBOL_GPL(crypto_stats_aead_encrypt); 1093 1094 void crypto_stats_aead_decrypt(unsigned int cryptlen, struct crypto_alg *alg, 1095 int ret) 1096 { 1097 if (ret && ret != -EINPROGRESS && ret != -EBUSY) { 1098 atomic64_inc(&alg->stats.aead.err_cnt); 1099 } else { 1100 atomic64_inc(&alg->stats.aead.decrypt_cnt); 1101 atomic64_add(cryptlen, &alg->stats.aead.decrypt_tlen); 1102 } 1103 crypto_alg_put(alg); 1104 } 1105 EXPORT_SYMBOL_GPL(crypto_stats_aead_decrypt); 1106 1107 void crypto_stats_akcipher_encrypt(unsigned int src_len, int ret, 1108 struct crypto_alg *alg) 1109 { 1110 if (ret && ret != -EINPROGRESS && ret != -EBUSY) { 1111 atomic64_inc(&alg->stats.akcipher.err_cnt); 1112 } else { 1113 atomic64_inc(&alg->stats.akcipher.encrypt_cnt); 1114 atomic64_add(src_len, &alg->stats.akcipher.encrypt_tlen); 1115 } 1116 crypto_alg_put(alg); 1117 } 1118 EXPORT_SYMBOL_GPL(crypto_stats_akcipher_encrypt); 1119 1120 void crypto_stats_akcipher_decrypt(unsigned int src_len, int ret, 1121 struct crypto_alg *alg) 1122 { 1123 if (ret && ret != -EINPROGRESS && ret != -EBUSY) { 1124 atomic64_inc(&alg->stats.akcipher.err_cnt); 1125 } else { 1126 atomic64_inc(&alg->stats.akcipher.decrypt_cnt); 1127 atomic64_add(src_len, &alg->stats.akcipher.decrypt_tlen); 1128 } 1129 crypto_alg_put(alg); 1130 } 1131 EXPORT_SYMBOL_GPL(crypto_stats_akcipher_decrypt); 1132 1133 void crypto_stats_akcipher_sign(int ret, struct crypto_alg *alg) 1134 { 1135 if (ret && ret != -EINPROGRESS && ret != -EBUSY) 1136 atomic64_inc(&alg->stats.akcipher.err_cnt); 1137 else 1138 atomic64_inc(&alg->stats.akcipher.sign_cnt); 1139 crypto_alg_put(alg); 1140 } 1141 EXPORT_SYMBOL_GPL(crypto_stats_akcipher_sign); 1142 1143 void crypto_stats_akcipher_verify(int ret, struct crypto_alg *alg) 1144 { 1145 if (ret && ret != -EINPROGRESS && ret != -EBUSY) 1146 atomic64_inc(&alg->stats.akcipher.err_cnt); 1147 else 1148 atomic64_inc(&alg->stats.akcipher.verify_cnt); 1149 crypto_alg_put(alg); 1150 } 1151 EXPORT_SYMBOL_GPL(crypto_stats_akcipher_verify); 1152 1153 void crypto_stats_compress(unsigned int slen, int ret, struct crypto_alg *alg) 1154 { 1155 if (ret && ret != -EINPROGRESS && ret != -EBUSY) { 1156 atomic64_inc(&alg->stats.compress.err_cnt); 1157 } else { 1158 atomic64_inc(&alg->stats.compress.compress_cnt); 1159 atomic64_add(slen, &alg->stats.compress.compress_tlen); 1160 } 1161 crypto_alg_put(alg); 1162 } 1163 EXPORT_SYMBOL_GPL(crypto_stats_compress); 1164 1165 void crypto_stats_decompress(unsigned int slen, int ret, struct crypto_alg *alg) 1166 { 1167 if (ret && ret != -EINPROGRESS && ret != -EBUSY) { 1168 atomic64_inc(&alg->stats.compress.err_cnt); 1169 } else { 1170 atomic64_inc(&alg->stats.compress.decompress_cnt); 1171 atomic64_add(slen, &alg->stats.compress.decompress_tlen); 1172 } 1173 crypto_alg_put(alg); 1174 } 1175 EXPORT_SYMBOL_GPL(crypto_stats_decompress); 1176 1177 void crypto_stats_ahash_update(unsigned int nbytes, int ret, 1178 struct crypto_alg *alg) 1179 { 1180 if (ret && ret != -EINPROGRESS && ret != -EBUSY) 1181 atomic64_inc(&alg->stats.hash.err_cnt); 1182 else 1183 atomic64_add(nbytes, &alg->stats.hash.hash_tlen); 1184 crypto_alg_put(alg); 1185 } 1186 EXPORT_SYMBOL_GPL(crypto_stats_ahash_update); 1187 1188 void crypto_stats_ahash_final(unsigned int nbytes, int ret, 1189 struct crypto_alg *alg) 1190 { 1191 if (ret && ret != -EINPROGRESS && ret != -EBUSY) { 1192 atomic64_inc(&alg->stats.hash.err_cnt); 1193 } else { 1194 atomic64_inc(&alg->stats.hash.hash_cnt); 1195 atomic64_add(nbytes, &alg->stats.hash.hash_tlen); 1196 } 1197 crypto_alg_put(alg); 1198 } 1199 EXPORT_SYMBOL_GPL(crypto_stats_ahash_final); 1200 1201 void crypto_stats_kpp_set_secret(struct crypto_alg *alg, int ret) 1202 { 1203 if (ret) 1204 atomic64_inc(&alg->stats.kpp.err_cnt); 1205 else 1206 atomic64_inc(&alg->stats.kpp.setsecret_cnt); 1207 crypto_alg_put(alg); 1208 } 1209 EXPORT_SYMBOL_GPL(crypto_stats_kpp_set_secret); 1210 1211 void crypto_stats_kpp_generate_public_key(struct crypto_alg *alg, int ret) 1212 { 1213 if (ret) 1214 atomic64_inc(&alg->stats.kpp.err_cnt); 1215 else 1216 atomic64_inc(&alg->stats.kpp.generate_public_key_cnt); 1217 crypto_alg_put(alg); 1218 } 1219 EXPORT_SYMBOL_GPL(crypto_stats_kpp_generate_public_key); 1220 1221 void crypto_stats_kpp_compute_shared_secret(struct crypto_alg *alg, int ret) 1222 { 1223 if (ret) 1224 atomic64_inc(&alg->stats.kpp.err_cnt); 1225 else 1226 atomic64_inc(&alg->stats.kpp.compute_shared_secret_cnt); 1227 crypto_alg_put(alg); 1228 } 1229 EXPORT_SYMBOL_GPL(crypto_stats_kpp_compute_shared_secret); 1230 1231 void crypto_stats_rng_seed(struct crypto_alg *alg, int ret) 1232 { 1233 if (ret && ret != -EINPROGRESS && ret != -EBUSY) 1234 atomic64_inc(&alg->stats.rng.err_cnt); 1235 else 1236 atomic64_inc(&alg->stats.rng.seed_cnt); 1237 crypto_alg_put(alg); 1238 } 1239 EXPORT_SYMBOL_GPL(crypto_stats_rng_seed); 1240 1241 void crypto_stats_rng_generate(struct crypto_alg *alg, unsigned int dlen, 1242 int ret) 1243 { 1244 if (ret && ret != -EINPROGRESS && ret != -EBUSY) { 1245 atomic64_inc(&alg->stats.rng.err_cnt); 1246 } else { 1247 atomic64_inc(&alg->stats.rng.generate_cnt); 1248 atomic64_add(dlen, &alg->stats.rng.generate_tlen); 1249 } 1250 crypto_alg_put(alg); 1251 } 1252 EXPORT_SYMBOL_GPL(crypto_stats_rng_generate); 1253 1254 void crypto_stats_skcipher_encrypt(unsigned int cryptlen, int ret, 1255 struct crypto_alg *alg) 1256 { 1257 if (ret && ret != -EINPROGRESS && ret != -EBUSY) { 1258 atomic64_inc(&alg->stats.cipher.err_cnt); 1259 } else { 1260 atomic64_inc(&alg->stats.cipher.encrypt_cnt); 1261 atomic64_add(cryptlen, &alg->stats.cipher.encrypt_tlen); 1262 } 1263 crypto_alg_put(alg); 1264 } 1265 EXPORT_SYMBOL_GPL(crypto_stats_skcipher_encrypt); 1266 1267 void crypto_stats_skcipher_decrypt(unsigned int cryptlen, int ret, 1268 struct crypto_alg *alg) 1269 { 1270 if (ret && ret != -EINPROGRESS && ret != -EBUSY) { 1271 atomic64_inc(&alg->stats.cipher.err_cnt); 1272 } else { 1273 atomic64_inc(&alg->stats.cipher.decrypt_cnt); 1274 atomic64_add(cryptlen, &alg->stats.cipher.decrypt_tlen); 1275 } 1276 crypto_alg_put(alg); 1277 } 1278 EXPORT_SYMBOL_GPL(crypto_stats_skcipher_decrypt); 1279 #endif 1280 1281 static int __init crypto_algapi_init(void) 1282 { 1283 crypto_init_proc(); 1284 return 0; 1285 } 1286 1287 static void __exit crypto_algapi_exit(void) 1288 { 1289 crypto_exit_proc(); 1290 } 1291 1292 module_init(crypto_algapi_init); 1293 module_exit(crypto_algapi_exit); 1294 1295 MODULE_LICENSE("GPL"); 1296 MODULE_DESCRIPTION("Cryptographic algorithms API"); 1297