1 /* 2 * Software async crypto daemon. 3 * 4 * Copyright (c) 2006 Herbert Xu <herbert@gondor.apana.org.au> 5 * 6 * Added AEAD support to cryptd. 7 * Authors: Tadeusz Struk (tadeusz.struk@intel.com) 8 * Adrian Hoban <adrian.hoban@intel.com> 9 * Gabriele Paoloni <gabriele.paoloni@intel.com> 10 * Aidan O'Mahony (aidan.o.mahony@intel.com) 11 * Copyright (c) 2010, Intel Corporation. 12 * 13 * This program is free software; you can redistribute it and/or modify it 14 * under the terms of the GNU General Public License as published by the Free 15 * Software Foundation; either version 2 of the License, or (at your option) 16 * any later version. 17 * 18 */ 19 20 #include <crypto/algapi.h> 21 #include <crypto/internal/hash.h> 22 #include <crypto/internal/aead.h> 23 #include <crypto/cryptd.h> 24 #include <crypto/crypto_wq.h> 25 #include <linux/err.h> 26 #include <linux/init.h> 27 #include <linux/kernel.h> 28 #include <linux/list.h> 29 #include <linux/module.h> 30 #include <linux/scatterlist.h> 31 #include <linux/sched.h> 32 #include <linux/slab.h> 33 34 #define CRYPTD_MAX_CPU_QLEN 100 35 36 struct cryptd_cpu_queue { 37 struct crypto_queue queue; 38 struct work_struct work; 39 }; 40 41 struct cryptd_queue { 42 struct cryptd_cpu_queue __percpu *cpu_queue; 43 }; 44 45 struct cryptd_instance_ctx { 46 struct crypto_spawn spawn; 47 struct cryptd_queue *queue; 48 }; 49 50 struct hashd_instance_ctx { 51 struct crypto_shash_spawn spawn; 52 struct cryptd_queue *queue; 53 }; 54 55 struct aead_instance_ctx { 56 struct crypto_aead_spawn aead_spawn; 57 struct cryptd_queue *queue; 58 }; 59 60 struct cryptd_blkcipher_ctx { 61 struct crypto_blkcipher *child; 62 }; 63 64 struct cryptd_blkcipher_request_ctx { 65 crypto_completion_t complete; 66 }; 67 68 struct cryptd_hash_ctx { 69 struct crypto_shash *child; 70 }; 71 72 struct cryptd_hash_request_ctx { 73 crypto_completion_t complete; 74 struct shash_desc desc; 75 }; 76 77 struct cryptd_aead_ctx { 78 struct crypto_aead *child; 79 }; 80 81 struct cryptd_aead_request_ctx { 82 crypto_completion_t complete; 83 }; 84 85 static void cryptd_queue_worker(struct work_struct *work); 86 87 static int cryptd_init_queue(struct cryptd_queue *queue, 88 unsigned int max_cpu_qlen) 89 { 90 int cpu; 91 struct cryptd_cpu_queue *cpu_queue; 92 93 queue->cpu_queue = alloc_percpu(struct cryptd_cpu_queue); 94 if (!queue->cpu_queue) 95 return -ENOMEM; 96 for_each_possible_cpu(cpu) { 97 cpu_queue = per_cpu_ptr(queue->cpu_queue, cpu); 98 crypto_init_queue(&cpu_queue->queue, max_cpu_qlen); 99 INIT_WORK(&cpu_queue->work, cryptd_queue_worker); 100 } 101 return 0; 102 } 103 104 static void cryptd_fini_queue(struct cryptd_queue *queue) 105 { 106 int cpu; 107 struct cryptd_cpu_queue *cpu_queue; 108 109 for_each_possible_cpu(cpu) { 110 cpu_queue = per_cpu_ptr(queue->cpu_queue, cpu); 111 BUG_ON(cpu_queue->queue.qlen); 112 } 113 free_percpu(queue->cpu_queue); 114 } 115 116 static int cryptd_enqueue_request(struct cryptd_queue *queue, 117 struct crypto_async_request *request) 118 { 119 int cpu, err; 120 struct cryptd_cpu_queue *cpu_queue; 121 122 cpu = get_cpu(); 123 cpu_queue = this_cpu_ptr(queue->cpu_queue); 124 err = crypto_enqueue_request(&cpu_queue->queue, request); 125 queue_work_on(cpu, kcrypto_wq, &cpu_queue->work); 126 put_cpu(); 127 128 return err; 129 } 130 131 /* Called in workqueue context, do one real cryption work (via 132 * req->complete) and reschedule itself if there are more work to 133 * do. */ 134 static void cryptd_queue_worker(struct work_struct *work) 135 { 136 struct cryptd_cpu_queue *cpu_queue; 137 struct crypto_async_request *req, *backlog; 138 139 cpu_queue = container_of(work, struct cryptd_cpu_queue, work); 140 /* 141 * Only handle one request at a time to avoid hogging crypto workqueue. 142 * preempt_disable/enable is used to prevent being preempted by 143 * cryptd_enqueue_request(). local_bh_disable/enable is used to prevent 144 * cryptd_enqueue_request() being accessed from software interrupts. 145 */ 146 local_bh_disable(); 147 preempt_disable(); 148 backlog = crypto_get_backlog(&cpu_queue->queue); 149 req = crypto_dequeue_request(&cpu_queue->queue); 150 preempt_enable(); 151 local_bh_enable(); 152 153 if (!req) 154 return; 155 156 if (backlog) 157 backlog->complete(backlog, -EINPROGRESS); 158 req->complete(req, 0); 159 160 if (cpu_queue->queue.qlen) 161 queue_work(kcrypto_wq, &cpu_queue->work); 162 } 163 164 static inline struct cryptd_queue *cryptd_get_queue(struct crypto_tfm *tfm) 165 { 166 struct crypto_instance *inst = crypto_tfm_alg_instance(tfm); 167 struct cryptd_instance_ctx *ictx = crypto_instance_ctx(inst); 168 return ictx->queue; 169 } 170 171 static inline void cryptd_check_internal(struct rtattr **tb, u32 *type, 172 u32 *mask) 173 { 174 struct crypto_attr_type *algt; 175 176 algt = crypto_get_attr_type(tb); 177 if (IS_ERR(algt)) 178 return; 179 180 *type |= algt->type & CRYPTO_ALG_INTERNAL; 181 *mask |= algt->mask & CRYPTO_ALG_INTERNAL; 182 } 183 184 static int cryptd_blkcipher_setkey(struct crypto_ablkcipher *parent, 185 const u8 *key, unsigned int keylen) 186 { 187 struct cryptd_blkcipher_ctx *ctx = crypto_ablkcipher_ctx(parent); 188 struct crypto_blkcipher *child = ctx->child; 189 int err; 190 191 crypto_blkcipher_clear_flags(child, CRYPTO_TFM_REQ_MASK); 192 crypto_blkcipher_set_flags(child, crypto_ablkcipher_get_flags(parent) & 193 CRYPTO_TFM_REQ_MASK); 194 err = crypto_blkcipher_setkey(child, key, keylen); 195 crypto_ablkcipher_set_flags(parent, crypto_blkcipher_get_flags(child) & 196 CRYPTO_TFM_RES_MASK); 197 return err; 198 } 199 200 static void cryptd_blkcipher_crypt(struct ablkcipher_request *req, 201 struct crypto_blkcipher *child, 202 int err, 203 int (*crypt)(struct blkcipher_desc *desc, 204 struct scatterlist *dst, 205 struct scatterlist *src, 206 unsigned int len)) 207 { 208 struct cryptd_blkcipher_request_ctx *rctx; 209 struct blkcipher_desc desc; 210 211 rctx = ablkcipher_request_ctx(req); 212 213 if (unlikely(err == -EINPROGRESS)) 214 goto out; 215 216 desc.tfm = child; 217 desc.info = req->info; 218 desc.flags = CRYPTO_TFM_REQ_MAY_SLEEP; 219 220 err = crypt(&desc, req->dst, req->src, req->nbytes); 221 222 req->base.complete = rctx->complete; 223 224 out: 225 local_bh_disable(); 226 rctx->complete(&req->base, err); 227 local_bh_enable(); 228 } 229 230 static void cryptd_blkcipher_encrypt(struct crypto_async_request *req, int err) 231 { 232 struct cryptd_blkcipher_ctx *ctx = crypto_tfm_ctx(req->tfm); 233 struct crypto_blkcipher *child = ctx->child; 234 235 cryptd_blkcipher_crypt(ablkcipher_request_cast(req), child, err, 236 crypto_blkcipher_crt(child)->encrypt); 237 } 238 239 static void cryptd_blkcipher_decrypt(struct crypto_async_request *req, int err) 240 { 241 struct cryptd_blkcipher_ctx *ctx = crypto_tfm_ctx(req->tfm); 242 struct crypto_blkcipher *child = ctx->child; 243 244 cryptd_blkcipher_crypt(ablkcipher_request_cast(req), child, err, 245 crypto_blkcipher_crt(child)->decrypt); 246 } 247 248 static int cryptd_blkcipher_enqueue(struct ablkcipher_request *req, 249 crypto_completion_t compl) 250 { 251 struct cryptd_blkcipher_request_ctx *rctx = ablkcipher_request_ctx(req); 252 struct crypto_ablkcipher *tfm = crypto_ablkcipher_reqtfm(req); 253 struct cryptd_queue *queue; 254 255 queue = cryptd_get_queue(crypto_ablkcipher_tfm(tfm)); 256 rctx->complete = req->base.complete; 257 req->base.complete = compl; 258 259 return cryptd_enqueue_request(queue, &req->base); 260 } 261 262 static int cryptd_blkcipher_encrypt_enqueue(struct ablkcipher_request *req) 263 { 264 return cryptd_blkcipher_enqueue(req, cryptd_blkcipher_encrypt); 265 } 266 267 static int cryptd_blkcipher_decrypt_enqueue(struct ablkcipher_request *req) 268 { 269 return cryptd_blkcipher_enqueue(req, cryptd_blkcipher_decrypt); 270 } 271 272 static int cryptd_blkcipher_init_tfm(struct crypto_tfm *tfm) 273 { 274 struct crypto_instance *inst = crypto_tfm_alg_instance(tfm); 275 struct cryptd_instance_ctx *ictx = crypto_instance_ctx(inst); 276 struct crypto_spawn *spawn = &ictx->spawn; 277 struct cryptd_blkcipher_ctx *ctx = crypto_tfm_ctx(tfm); 278 struct crypto_blkcipher *cipher; 279 280 cipher = crypto_spawn_blkcipher(spawn); 281 if (IS_ERR(cipher)) 282 return PTR_ERR(cipher); 283 284 ctx->child = cipher; 285 tfm->crt_ablkcipher.reqsize = 286 sizeof(struct cryptd_blkcipher_request_ctx); 287 return 0; 288 } 289 290 static void cryptd_blkcipher_exit_tfm(struct crypto_tfm *tfm) 291 { 292 struct cryptd_blkcipher_ctx *ctx = crypto_tfm_ctx(tfm); 293 294 crypto_free_blkcipher(ctx->child); 295 } 296 297 static int cryptd_init_instance(struct crypto_instance *inst, 298 struct crypto_alg *alg) 299 { 300 if (snprintf(inst->alg.cra_driver_name, CRYPTO_MAX_ALG_NAME, 301 "cryptd(%s)", 302 alg->cra_driver_name) >= CRYPTO_MAX_ALG_NAME) 303 return -ENAMETOOLONG; 304 305 memcpy(inst->alg.cra_name, alg->cra_name, CRYPTO_MAX_ALG_NAME); 306 307 inst->alg.cra_priority = alg->cra_priority + 50; 308 inst->alg.cra_blocksize = alg->cra_blocksize; 309 inst->alg.cra_alignmask = alg->cra_alignmask; 310 311 return 0; 312 } 313 314 static void *cryptd_alloc_instance(struct crypto_alg *alg, unsigned int head, 315 unsigned int tail) 316 { 317 char *p; 318 struct crypto_instance *inst; 319 int err; 320 321 p = kzalloc(head + sizeof(*inst) + tail, GFP_KERNEL); 322 if (!p) 323 return ERR_PTR(-ENOMEM); 324 325 inst = (void *)(p + head); 326 327 err = cryptd_init_instance(inst, alg); 328 if (err) 329 goto out_free_inst; 330 331 out: 332 return p; 333 334 out_free_inst: 335 kfree(p); 336 p = ERR_PTR(err); 337 goto out; 338 } 339 340 static int cryptd_create_blkcipher(struct crypto_template *tmpl, 341 struct rtattr **tb, 342 struct cryptd_queue *queue) 343 { 344 struct cryptd_instance_ctx *ctx; 345 struct crypto_instance *inst; 346 struct crypto_alg *alg; 347 u32 type = CRYPTO_ALG_TYPE_BLKCIPHER; 348 u32 mask = CRYPTO_ALG_TYPE_MASK; 349 int err; 350 351 cryptd_check_internal(tb, &type, &mask); 352 353 alg = crypto_get_attr_alg(tb, type, mask); 354 if (IS_ERR(alg)) 355 return PTR_ERR(alg); 356 357 inst = cryptd_alloc_instance(alg, 0, sizeof(*ctx)); 358 err = PTR_ERR(inst); 359 if (IS_ERR(inst)) 360 goto out_put_alg; 361 362 ctx = crypto_instance_ctx(inst); 363 ctx->queue = queue; 364 365 err = crypto_init_spawn(&ctx->spawn, alg, inst, 366 CRYPTO_ALG_TYPE_MASK | CRYPTO_ALG_ASYNC); 367 if (err) 368 goto out_free_inst; 369 370 type = CRYPTO_ALG_TYPE_ABLKCIPHER | CRYPTO_ALG_ASYNC; 371 if (alg->cra_flags & CRYPTO_ALG_INTERNAL) 372 type |= CRYPTO_ALG_INTERNAL; 373 inst->alg.cra_flags = type; 374 inst->alg.cra_type = &crypto_ablkcipher_type; 375 376 inst->alg.cra_ablkcipher.ivsize = alg->cra_blkcipher.ivsize; 377 inst->alg.cra_ablkcipher.min_keysize = alg->cra_blkcipher.min_keysize; 378 inst->alg.cra_ablkcipher.max_keysize = alg->cra_blkcipher.max_keysize; 379 380 inst->alg.cra_ablkcipher.geniv = alg->cra_blkcipher.geniv; 381 382 inst->alg.cra_ctxsize = sizeof(struct cryptd_blkcipher_ctx); 383 384 inst->alg.cra_init = cryptd_blkcipher_init_tfm; 385 inst->alg.cra_exit = cryptd_blkcipher_exit_tfm; 386 387 inst->alg.cra_ablkcipher.setkey = cryptd_blkcipher_setkey; 388 inst->alg.cra_ablkcipher.encrypt = cryptd_blkcipher_encrypt_enqueue; 389 inst->alg.cra_ablkcipher.decrypt = cryptd_blkcipher_decrypt_enqueue; 390 391 err = crypto_register_instance(tmpl, inst); 392 if (err) { 393 crypto_drop_spawn(&ctx->spawn); 394 out_free_inst: 395 kfree(inst); 396 } 397 398 out_put_alg: 399 crypto_mod_put(alg); 400 return err; 401 } 402 403 static int cryptd_hash_init_tfm(struct crypto_tfm *tfm) 404 { 405 struct crypto_instance *inst = crypto_tfm_alg_instance(tfm); 406 struct hashd_instance_ctx *ictx = crypto_instance_ctx(inst); 407 struct crypto_shash_spawn *spawn = &ictx->spawn; 408 struct cryptd_hash_ctx *ctx = crypto_tfm_ctx(tfm); 409 struct crypto_shash *hash; 410 411 hash = crypto_spawn_shash(spawn); 412 if (IS_ERR(hash)) 413 return PTR_ERR(hash); 414 415 ctx->child = hash; 416 crypto_ahash_set_reqsize(__crypto_ahash_cast(tfm), 417 sizeof(struct cryptd_hash_request_ctx) + 418 crypto_shash_descsize(hash)); 419 return 0; 420 } 421 422 static void cryptd_hash_exit_tfm(struct crypto_tfm *tfm) 423 { 424 struct cryptd_hash_ctx *ctx = crypto_tfm_ctx(tfm); 425 426 crypto_free_shash(ctx->child); 427 } 428 429 static int cryptd_hash_setkey(struct crypto_ahash *parent, 430 const u8 *key, unsigned int keylen) 431 { 432 struct cryptd_hash_ctx *ctx = crypto_ahash_ctx(parent); 433 struct crypto_shash *child = ctx->child; 434 int err; 435 436 crypto_shash_clear_flags(child, CRYPTO_TFM_REQ_MASK); 437 crypto_shash_set_flags(child, crypto_ahash_get_flags(parent) & 438 CRYPTO_TFM_REQ_MASK); 439 err = crypto_shash_setkey(child, key, keylen); 440 crypto_ahash_set_flags(parent, crypto_shash_get_flags(child) & 441 CRYPTO_TFM_RES_MASK); 442 return err; 443 } 444 445 static int cryptd_hash_enqueue(struct ahash_request *req, 446 crypto_completion_t compl) 447 { 448 struct cryptd_hash_request_ctx *rctx = ahash_request_ctx(req); 449 struct crypto_ahash *tfm = crypto_ahash_reqtfm(req); 450 struct cryptd_queue *queue = 451 cryptd_get_queue(crypto_ahash_tfm(tfm)); 452 453 rctx->complete = req->base.complete; 454 req->base.complete = compl; 455 456 return cryptd_enqueue_request(queue, &req->base); 457 } 458 459 static void cryptd_hash_init(struct crypto_async_request *req_async, int err) 460 { 461 struct cryptd_hash_ctx *ctx = crypto_tfm_ctx(req_async->tfm); 462 struct crypto_shash *child = ctx->child; 463 struct ahash_request *req = ahash_request_cast(req_async); 464 struct cryptd_hash_request_ctx *rctx = ahash_request_ctx(req); 465 struct shash_desc *desc = &rctx->desc; 466 467 if (unlikely(err == -EINPROGRESS)) 468 goto out; 469 470 desc->tfm = child; 471 desc->flags = CRYPTO_TFM_REQ_MAY_SLEEP; 472 473 err = crypto_shash_init(desc); 474 475 req->base.complete = rctx->complete; 476 477 out: 478 local_bh_disable(); 479 rctx->complete(&req->base, err); 480 local_bh_enable(); 481 } 482 483 static int cryptd_hash_init_enqueue(struct ahash_request *req) 484 { 485 return cryptd_hash_enqueue(req, cryptd_hash_init); 486 } 487 488 static void cryptd_hash_update(struct crypto_async_request *req_async, int err) 489 { 490 struct ahash_request *req = ahash_request_cast(req_async); 491 struct cryptd_hash_request_ctx *rctx; 492 493 rctx = ahash_request_ctx(req); 494 495 if (unlikely(err == -EINPROGRESS)) 496 goto out; 497 498 err = shash_ahash_update(req, &rctx->desc); 499 500 req->base.complete = rctx->complete; 501 502 out: 503 local_bh_disable(); 504 rctx->complete(&req->base, err); 505 local_bh_enable(); 506 } 507 508 static int cryptd_hash_update_enqueue(struct ahash_request *req) 509 { 510 return cryptd_hash_enqueue(req, cryptd_hash_update); 511 } 512 513 static void cryptd_hash_final(struct crypto_async_request *req_async, int err) 514 { 515 struct ahash_request *req = ahash_request_cast(req_async); 516 struct cryptd_hash_request_ctx *rctx = ahash_request_ctx(req); 517 518 if (unlikely(err == -EINPROGRESS)) 519 goto out; 520 521 err = crypto_shash_final(&rctx->desc, req->result); 522 523 req->base.complete = rctx->complete; 524 525 out: 526 local_bh_disable(); 527 rctx->complete(&req->base, err); 528 local_bh_enable(); 529 } 530 531 static int cryptd_hash_final_enqueue(struct ahash_request *req) 532 { 533 return cryptd_hash_enqueue(req, cryptd_hash_final); 534 } 535 536 static void cryptd_hash_finup(struct crypto_async_request *req_async, int err) 537 { 538 struct ahash_request *req = ahash_request_cast(req_async); 539 struct cryptd_hash_request_ctx *rctx = ahash_request_ctx(req); 540 541 if (unlikely(err == -EINPROGRESS)) 542 goto out; 543 544 err = shash_ahash_finup(req, &rctx->desc); 545 546 req->base.complete = rctx->complete; 547 548 out: 549 local_bh_disable(); 550 rctx->complete(&req->base, err); 551 local_bh_enable(); 552 } 553 554 static int cryptd_hash_finup_enqueue(struct ahash_request *req) 555 { 556 return cryptd_hash_enqueue(req, cryptd_hash_finup); 557 } 558 559 static void cryptd_hash_digest(struct crypto_async_request *req_async, int err) 560 { 561 struct cryptd_hash_ctx *ctx = crypto_tfm_ctx(req_async->tfm); 562 struct crypto_shash *child = ctx->child; 563 struct ahash_request *req = ahash_request_cast(req_async); 564 struct cryptd_hash_request_ctx *rctx = ahash_request_ctx(req); 565 struct shash_desc *desc = &rctx->desc; 566 567 if (unlikely(err == -EINPROGRESS)) 568 goto out; 569 570 desc->tfm = child; 571 desc->flags = CRYPTO_TFM_REQ_MAY_SLEEP; 572 573 err = shash_ahash_digest(req, desc); 574 575 req->base.complete = rctx->complete; 576 577 out: 578 local_bh_disable(); 579 rctx->complete(&req->base, err); 580 local_bh_enable(); 581 } 582 583 static int cryptd_hash_digest_enqueue(struct ahash_request *req) 584 { 585 return cryptd_hash_enqueue(req, cryptd_hash_digest); 586 } 587 588 static int cryptd_hash_export(struct ahash_request *req, void *out) 589 { 590 struct cryptd_hash_request_ctx *rctx = ahash_request_ctx(req); 591 592 return crypto_shash_export(&rctx->desc, out); 593 } 594 595 static int cryptd_hash_import(struct ahash_request *req, const void *in) 596 { 597 struct cryptd_hash_request_ctx *rctx = ahash_request_ctx(req); 598 599 return crypto_shash_import(&rctx->desc, in); 600 } 601 602 static int cryptd_create_hash(struct crypto_template *tmpl, struct rtattr **tb, 603 struct cryptd_queue *queue) 604 { 605 struct hashd_instance_ctx *ctx; 606 struct ahash_instance *inst; 607 struct shash_alg *salg; 608 struct crypto_alg *alg; 609 u32 type = 0; 610 u32 mask = 0; 611 int err; 612 613 cryptd_check_internal(tb, &type, &mask); 614 615 salg = shash_attr_alg(tb[1], type, mask); 616 if (IS_ERR(salg)) 617 return PTR_ERR(salg); 618 619 alg = &salg->base; 620 inst = cryptd_alloc_instance(alg, ahash_instance_headroom(), 621 sizeof(*ctx)); 622 err = PTR_ERR(inst); 623 if (IS_ERR(inst)) 624 goto out_put_alg; 625 626 ctx = ahash_instance_ctx(inst); 627 ctx->queue = queue; 628 629 err = crypto_init_shash_spawn(&ctx->spawn, salg, 630 ahash_crypto_instance(inst)); 631 if (err) 632 goto out_free_inst; 633 634 type = CRYPTO_ALG_ASYNC; 635 if (alg->cra_flags & CRYPTO_ALG_INTERNAL) 636 type |= CRYPTO_ALG_INTERNAL; 637 inst->alg.halg.base.cra_flags = type; 638 639 inst->alg.halg.digestsize = salg->digestsize; 640 inst->alg.halg.base.cra_ctxsize = sizeof(struct cryptd_hash_ctx); 641 642 inst->alg.halg.base.cra_init = cryptd_hash_init_tfm; 643 inst->alg.halg.base.cra_exit = cryptd_hash_exit_tfm; 644 645 inst->alg.init = cryptd_hash_init_enqueue; 646 inst->alg.update = cryptd_hash_update_enqueue; 647 inst->alg.final = cryptd_hash_final_enqueue; 648 inst->alg.finup = cryptd_hash_finup_enqueue; 649 inst->alg.export = cryptd_hash_export; 650 inst->alg.import = cryptd_hash_import; 651 inst->alg.setkey = cryptd_hash_setkey; 652 inst->alg.digest = cryptd_hash_digest_enqueue; 653 654 err = ahash_register_instance(tmpl, inst); 655 if (err) { 656 crypto_drop_shash(&ctx->spawn); 657 out_free_inst: 658 kfree(inst); 659 } 660 661 out_put_alg: 662 crypto_mod_put(alg); 663 return err; 664 } 665 666 static int cryptd_aead_setkey(struct crypto_aead *parent, 667 const u8 *key, unsigned int keylen) 668 { 669 struct cryptd_aead_ctx *ctx = crypto_aead_ctx(parent); 670 struct crypto_aead *child = ctx->child; 671 672 return crypto_aead_setkey(child, key, keylen); 673 } 674 675 static int cryptd_aead_setauthsize(struct crypto_aead *parent, 676 unsigned int authsize) 677 { 678 struct cryptd_aead_ctx *ctx = crypto_aead_ctx(parent); 679 struct crypto_aead *child = ctx->child; 680 681 return crypto_aead_setauthsize(child, authsize); 682 } 683 684 static void cryptd_aead_crypt(struct aead_request *req, 685 struct crypto_aead *child, 686 int err, 687 int (*crypt)(struct aead_request *req)) 688 { 689 struct cryptd_aead_request_ctx *rctx; 690 crypto_completion_t compl; 691 692 rctx = aead_request_ctx(req); 693 compl = rctx->complete; 694 695 if (unlikely(err == -EINPROGRESS)) 696 goto out; 697 aead_request_set_tfm(req, child); 698 err = crypt( req ); 699 out: 700 local_bh_disable(); 701 compl(&req->base, err); 702 local_bh_enable(); 703 } 704 705 static void cryptd_aead_encrypt(struct crypto_async_request *areq, int err) 706 { 707 struct cryptd_aead_ctx *ctx = crypto_tfm_ctx(areq->tfm); 708 struct crypto_aead *child = ctx->child; 709 struct aead_request *req; 710 711 req = container_of(areq, struct aead_request, base); 712 cryptd_aead_crypt(req, child, err, crypto_aead_alg(child)->encrypt); 713 } 714 715 static void cryptd_aead_decrypt(struct crypto_async_request *areq, int err) 716 { 717 struct cryptd_aead_ctx *ctx = crypto_tfm_ctx(areq->tfm); 718 struct crypto_aead *child = ctx->child; 719 struct aead_request *req; 720 721 req = container_of(areq, struct aead_request, base); 722 cryptd_aead_crypt(req, child, err, crypto_aead_alg(child)->decrypt); 723 } 724 725 static int cryptd_aead_enqueue(struct aead_request *req, 726 crypto_completion_t compl) 727 { 728 struct cryptd_aead_request_ctx *rctx = aead_request_ctx(req); 729 struct crypto_aead *tfm = crypto_aead_reqtfm(req); 730 struct cryptd_queue *queue = cryptd_get_queue(crypto_aead_tfm(tfm)); 731 732 rctx->complete = req->base.complete; 733 req->base.complete = compl; 734 return cryptd_enqueue_request(queue, &req->base); 735 } 736 737 static int cryptd_aead_encrypt_enqueue(struct aead_request *req) 738 { 739 return cryptd_aead_enqueue(req, cryptd_aead_encrypt ); 740 } 741 742 static int cryptd_aead_decrypt_enqueue(struct aead_request *req) 743 { 744 return cryptd_aead_enqueue(req, cryptd_aead_decrypt ); 745 } 746 747 static int cryptd_aead_init_tfm(struct crypto_aead *tfm) 748 { 749 struct aead_instance *inst = aead_alg_instance(tfm); 750 struct aead_instance_ctx *ictx = aead_instance_ctx(inst); 751 struct crypto_aead_spawn *spawn = &ictx->aead_spawn; 752 struct cryptd_aead_ctx *ctx = crypto_aead_ctx(tfm); 753 struct crypto_aead *cipher; 754 755 cipher = crypto_spawn_aead(spawn); 756 if (IS_ERR(cipher)) 757 return PTR_ERR(cipher); 758 759 ctx->child = cipher; 760 crypto_aead_set_reqsize( 761 tfm, max((unsigned)sizeof(struct cryptd_aead_request_ctx), 762 crypto_aead_reqsize(cipher))); 763 return 0; 764 } 765 766 static void cryptd_aead_exit_tfm(struct crypto_aead *tfm) 767 { 768 struct cryptd_aead_ctx *ctx = crypto_aead_ctx(tfm); 769 crypto_free_aead(ctx->child); 770 } 771 772 static int cryptd_create_aead(struct crypto_template *tmpl, 773 struct rtattr **tb, 774 struct cryptd_queue *queue) 775 { 776 struct aead_instance_ctx *ctx; 777 struct aead_instance *inst; 778 struct aead_alg *alg; 779 const char *name; 780 u32 type = 0; 781 u32 mask = CRYPTO_ALG_ASYNC; 782 int err; 783 784 cryptd_check_internal(tb, &type, &mask); 785 786 name = crypto_attr_alg_name(tb[1]); 787 if (IS_ERR(name)) 788 return PTR_ERR(name); 789 790 inst = kzalloc(sizeof(*inst) + sizeof(*ctx), GFP_KERNEL); 791 if (!inst) 792 return -ENOMEM; 793 794 ctx = aead_instance_ctx(inst); 795 ctx->queue = queue; 796 797 crypto_set_aead_spawn(&ctx->aead_spawn, aead_crypto_instance(inst)); 798 err = crypto_grab_aead(&ctx->aead_spawn, name, type, mask); 799 if (err) 800 goto out_free_inst; 801 802 alg = crypto_spawn_aead_alg(&ctx->aead_spawn); 803 err = cryptd_init_instance(aead_crypto_instance(inst), &alg->base); 804 if (err) 805 goto out_drop_aead; 806 807 inst->alg.base.cra_flags = CRYPTO_ALG_ASYNC | 808 (alg->base.cra_flags & CRYPTO_ALG_INTERNAL); 809 inst->alg.base.cra_ctxsize = sizeof(struct cryptd_aead_ctx); 810 811 inst->alg.ivsize = crypto_aead_alg_ivsize(alg); 812 inst->alg.maxauthsize = crypto_aead_alg_maxauthsize(alg); 813 814 inst->alg.init = cryptd_aead_init_tfm; 815 inst->alg.exit = cryptd_aead_exit_tfm; 816 inst->alg.setkey = cryptd_aead_setkey; 817 inst->alg.setauthsize = cryptd_aead_setauthsize; 818 inst->alg.encrypt = cryptd_aead_encrypt_enqueue; 819 inst->alg.decrypt = cryptd_aead_decrypt_enqueue; 820 821 err = aead_register_instance(tmpl, inst); 822 if (err) { 823 out_drop_aead: 824 crypto_drop_aead(&ctx->aead_spawn); 825 out_free_inst: 826 kfree(inst); 827 } 828 return err; 829 } 830 831 static struct cryptd_queue queue; 832 833 static int cryptd_create(struct crypto_template *tmpl, struct rtattr **tb) 834 { 835 struct crypto_attr_type *algt; 836 837 algt = crypto_get_attr_type(tb); 838 if (IS_ERR(algt)) 839 return PTR_ERR(algt); 840 841 switch (algt->type & algt->mask & CRYPTO_ALG_TYPE_MASK) { 842 case CRYPTO_ALG_TYPE_BLKCIPHER: 843 return cryptd_create_blkcipher(tmpl, tb, &queue); 844 case CRYPTO_ALG_TYPE_DIGEST: 845 return cryptd_create_hash(tmpl, tb, &queue); 846 case CRYPTO_ALG_TYPE_AEAD: 847 return cryptd_create_aead(tmpl, tb, &queue); 848 } 849 850 return -EINVAL; 851 } 852 853 static void cryptd_free(struct crypto_instance *inst) 854 { 855 struct cryptd_instance_ctx *ctx = crypto_instance_ctx(inst); 856 struct hashd_instance_ctx *hctx = crypto_instance_ctx(inst); 857 struct aead_instance_ctx *aead_ctx = crypto_instance_ctx(inst); 858 859 switch (inst->alg.cra_flags & CRYPTO_ALG_TYPE_MASK) { 860 case CRYPTO_ALG_TYPE_AHASH: 861 crypto_drop_shash(&hctx->spawn); 862 kfree(ahash_instance(inst)); 863 return; 864 case CRYPTO_ALG_TYPE_AEAD: 865 crypto_drop_aead(&aead_ctx->aead_spawn); 866 kfree(aead_instance(inst)); 867 return; 868 default: 869 crypto_drop_spawn(&ctx->spawn); 870 kfree(inst); 871 } 872 } 873 874 static struct crypto_template cryptd_tmpl = { 875 .name = "cryptd", 876 .create = cryptd_create, 877 .free = cryptd_free, 878 .module = THIS_MODULE, 879 }; 880 881 struct cryptd_ablkcipher *cryptd_alloc_ablkcipher(const char *alg_name, 882 u32 type, u32 mask) 883 { 884 char cryptd_alg_name[CRYPTO_MAX_ALG_NAME]; 885 struct crypto_tfm *tfm; 886 887 if (snprintf(cryptd_alg_name, CRYPTO_MAX_ALG_NAME, 888 "cryptd(%s)", alg_name) >= CRYPTO_MAX_ALG_NAME) 889 return ERR_PTR(-EINVAL); 890 type &= ~(CRYPTO_ALG_TYPE_MASK | CRYPTO_ALG_GENIV); 891 type |= CRYPTO_ALG_TYPE_BLKCIPHER; 892 mask &= ~CRYPTO_ALG_TYPE_MASK; 893 mask |= (CRYPTO_ALG_GENIV | CRYPTO_ALG_TYPE_BLKCIPHER_MASK); 894 tfm = crypto_alloc_base(cryptd_alg_name, type, mask); 895 if (IS_ERR(tfm)) 896 return ERR_CAST(tfm); 897 if (tfm->__crt_alg->cra_module != THIS_MODULE) { 898 crypto_free_tfm(tfm); 899 return ERR_PTR(-EINVAL); 900 } 901 902 return __cryptd_ablkcipher_cast(__crypto_ablkcipher_cast(tfm)); 903 } 904 EXPORT_SYMBOL_GPL(cryptd_alloc_ablkcipher); 905 906 struct crypto_blkcipher *cryptd_ablkcipher_child(struct cryptd_ablkcipher *tfm) 907 { 908 struct cryptd_blkcipher_ctx *ctx = crypto_ablkcipher_ctx(&tfm->base); 909 return ctx->child; 910 } 911 EXPORT_SYMBOL_GPL(cryptd_ablkcipher_child); 912 913 void cryptd_free_ablkcipher(struct cryptd_ablkcipher *tfm) 914 { 915 crypto_free_ablkcipher(&tfm->base); 916 } 917 EXPORT_SYMBOL_GPL(cryptd_free_ablkcipher); 918 919 struct cryptd_ahash *cryptd_alloc_ahash(const char *alg_name, 920 u32 type, u32 mask) 921 { 922 char cryptd_alg_name[CRYPTO_MAX_ALG_NAME]; 923 struct crypto_ahash *tfm; 924 925 if (snprintf(cryptd_alg_name, CRYPTO_MAX_ALG_NAME, 926 "cryptd(%s)", alg_name) >= CRYPTO_MAX_ALG_NAME) 927 return ERR_PTR(-EINVAL); 928 tfm = crypto_alloc_ahash(cryptd_alg_name, type, mask); 929 if (IS_ERR(tfm)) 930 return ERR_CAST(tfm); 931 if (tfm->base.__crt_alg->cra_module != THIS_MODULE) { 932 crypto_free_ahash(tfm); 933 return ERR_PTR(-EINVAL); 934 } 935 936 return __cryptd_ahash_cast(tfm); 937 } 938 EXPORT_SYMBOL_GPL(cryptd_alloc_ahash); 939 940 struct crypto_shash *cryptd_ahash_child(struct cryptd_ahash *tfm) 941 { 942 struct cryptd_hash_ctx *ctx = crypto_ahash_ctx(&tfm->base); 943 944 return ctx->child; 945 } 946 EXPORT_SYMBOL_GPL(cryptd_ahash_child); 947 948 struct shash_desc *cryptd_shash_desc(struct ahash_request *req) 949 { 950 struct cryptd_hash_request_ctx *rctx = ahash_request_ctx(req); 951 return &rctx->desc; 952 } 953 EXPORT_SYMBOL_GPL(cryptd_shash_desc); 954 955 void cryptd_free_ahash(struct cryptd_ahash *tfm) 956 { 957 crypto_free_ahash(&tfm->base); 958 } 959 EXPORT_SYMBOL_GPL(cryptd_free_ahash); 960 961 struct cryptd_aead *cryptd_alloc_aead(const char *alg_name, 962 u32 type, u32 mask) 963 { 964 char cryptd_alg_name[CRYPTO_MAX_ALG_NAME]; 965 struct crypto_aead *tfm; 966 967 if (snprintf(cryptd_alg_name, CRYPTO_MAX_ALG_NAME, 968 "cryptd(%s)", alg_name) >= CRYPTO_MAX_ALG_NAME) 969 return ERR_PTR(-EINVAL); 970 tfm = crypto_alloc_aead(cryptd_alg_name, type, mask); 971 if (IS_ERR(tfm)) 972 return ERR_CAST(tfm); 973 if (tfm->base.__crt_alg->cra_module != THIS_MODULE) { 974 crypto_free_aead(tfm); 975 return ERR_PTR(-EINVAL); 976 } 977 return __cryptd_aead_cast(tfm); 978 } 979 EXPORT_SYMBOL_GPL(cryptd_alloc_aead); 980 981 struct crypto_aead *cryptd_aead_child(struct cryptd_aead *tfm) 982 { 983 struct cryptd_aead_ctx *ctx; 984 ctx = crypto_aead_ctx(&tfm->base); 985 return ctx->child; 986 } 987 EXPORT_SYMBOL_GPL(cryptd_aead_child); 988 989 void cryptd_free_aead(struct cryptd_aead *tfm) 990 { 991 crypto_free_aead(&tfm->base); 992 } 993 EXPORT_SYMBOL_GPL(cryptd_free_aead); 994 995 static int __init cryptd_init(void) 996 { 997 int err; 998 999 err = cryptd_init_queue(&queue, CRYPTD_MAX_CPU_QLEN); 1000 if (err) 1001 return err; 1002 1003 err = crypto_register_template(&cryptd_tmpl); 1004 if (err) 1005 cryptd_fini_queue(&queue); 1006 1007 return err; 1008 } 1009 1010 static void __exit cryptd_exit(void) 1011 { 1012 cryptd_fini_queue(&queue); 1013 crypto_unregister_template(&cryptd_tmpl); 1014 } 1015 1016 subsys_initcall(cryptd_init); 1017 module_exit(cryptd_exit); 1018 1019 MODULE_LICENSE("GPL"); 1020 MODULE_DESCRIPTION("Software async crypto daemon"); 1021 MODULE_ALIAS_CRYPTO("cryptd"); 1022