xref: /openbmc/linux/crypto/cryptd.c (revision b830f94f)
1 // SPDX-License-Identifier: GPL-2.0-or-later
2 /*
3  * Software async crypto daemon.
4  *
5  * Copyright (c) 2006 Herbert Xu <herbert@gondor.apana.org.au>
6  *
7  * Added AEAD support to cryptd.
8  *    Authors: Tadeusz Struk (tadeusz.struk@intel.com)
9  *             Adrian Hoban <adrian.hoban@intel.com>
10  *             Gabriele Paoloni <gabriele.paoloni@intel.com>
11  *             Aidan O'Mahony (aidan.o.mahony@intel.com)
12  *    Copyright (c) 2010, Intel Corporation.
13  */
14 
15 #include <crypto/internal/hash.h>
16 #include <crypto/internal/aead.h>
17 #include <crypto/internal/skcipher.h>
18 #include <crypto/cryptd.h>
19 #include <linux/atomic.h>
20 #include <linux/err.h>
21 #include <linux/init.h>
22 #include <linux/kernel.h>
23 #include <linux/list.h>
24 #include <linux/module.h>
25 #include <linux/scatterlist.h>
26 #include <linux/sched.h>
27 #include <linux/slab.h>
28 #include <linux/workqueue.h>
29 
30 static unsigned int cryptd_max_cpu_qlen = 1000;
31 module_param(cryptd_max_cpu_qlen, uint, 0);
32 MODULE_PARM_DESC(cryptd_max_cpu_qlen, "Set cryptd Max queue depth");
33 
34 static struct workqueue_struct *cryptd_wq;
35 
36 struct cryptd_cpu_queue {
37 	struct crypto_queue queue;
38 	struct work_struct work;
39 };
40 
41 struct cryptd_queue {
42 	struct cryptd_cpu_queue __percpu *cpu_queue;
43 };
44 
45 struct cryptd_instance_ctx {
46 	struct crypto_spawn spawn;
47 	struct cryptd_queue *queue;
48 };
49 
50 struct skcipherd_instance_ctx {
51 	struct crypto_skcipher_spawn spawn;
52 	struct cryptd_queue *queue;
53 };
54 
55 struct hashd_instance_ctx {
56 	struct crypto_shash_spawn spawn;
57 	struct cryptd_queue *queue;
58 };
59 
60 struct aead_instance_ctx {
61 	struct crypto_aead_spawn aead_spawn;
62 	struct cryptd_queue *queue;
63 };
64 
65 struct cryptd_skcipher_ctx {
66 	atomic_t refcnt;
67 	struct crypto_sync_skcipher *child;
68 };
69 
70 struct cryptd_skcipher_request_ctx {
71 	crypto_completion_t complete;
72 };
73 
74 struct cryptd_hash_ctx {
75 	atomic_t refcnt;
76 	struct crypto_shash *child;
77 };
78 
79 struct cryptd_hash_request_ctx {
80 	crypto_completion_t complete;
81 	struct shash_desc desc;
82 };
83 
84 struct cryptd_aead_ctx {
85 	atomic_t refcnt;
86 	struct crypto_aead *child;
87 };
88 
89 struct cryptd_aead_request_ctx {
90 	crypto_completion_t complete;
91 };
92 
93 static void cryptd_queue_worker(struct work_struct *work);
94 
95 static int cryptd_init_queue(struct cryptd_queue *queue,
96 			     unsigned int max_cpu_qlen)
97 {
98 	int cpu;
99 	struct cryptd_cpu_queue *cpu_queue;
100 
101 	queue->cpu_queue = alloc_percpu(struct cryptd_cpu_queue);
102 	if (!queue->cpu_queue)
103 		return -ENOMEM;
104 	for_each_possible_cpu(cpu) {
105 		cpu_queue = per_cpu_ptr(queue->cpu_queue, cpu);
106 		crypto_init_queue(&cpu_queue->queue, max_cpu_qlen);
107 		INIT_WORK(&cpu_queue->work, cryptd_queue_worker);
108 	}
109 	pr_info("cryptd: max_cpu_qlen set to %d\n", max_cpu_qlen);
110 	return 0;
111 }
112 
113 static void cryptd_fini_queue(struct cryptd_queue *queue)
114 {
115 	int cpu;
116 	struct cryptd_cpu_queue *cpu_queue;
117 
118 	for_each_possible_cpu(cpu) {
119 		cpu_queue = per_cpu_ptr(queue->cpu_queue, cpu);
120 		BUG_ON(cpu_queue->queue.qlen);
121 	}
122 	free_percpu(queue->cpu_queue);
123 }
124 
125 static int cryptd_enqueue_request(struct cryptd_queue *queue,
126 				  struct crypto_async_request *request)
127 {
128 	int cpu, err;
129 	struct cryptd_cpu_queue *cpu_queue;
130 	atomic_t *refcnt;
131 
132 	cpu = get_cpu();
133 	cpu_queue = this_cpu_ptr(queue->cpu_queue);
134 	err = crypto_enqueue_request(&cpu_queue->queue, request);
135 
136 	refcnt = crypto_tfm_ctx(request->tfm);
137 
138 	if (err == -ENOSPC)
139 		goto out_put_cpu;
140 
141 	queue_work_on(cpu, cryptd_wq, &cpu_queue->work);
142 
143 	if (!atomic_read(refcnt))
144 		goto out_put_cpu;
145 
146 	atomic_inc(refcnt);
147 
148 out_put_cpu:
149 	put_cpu();
150 
151 	return err;
152 }
153 
154 /* Called in workqueue context, do one real cryption work (via
155  * req->complete) and reschedule itself if there are more work to
156  * do. */
157 static void cryptd_queue_worker(struct work_struct *work)
158 {
159 	struct cryptd_cpu_queue *cpu_queue;
160 	struct crypto_async_request *req, *backlog;
161 
162 	cpu_queue = container_of(work, struct cryptd_cpu_queue, work);
163 	/*
164 	 * Only handle one request at a time to avoid hogging crypto workqueue.
165 	 * preempt_disable/enable is used to prevent being preempted by
166 	 * cryptd_enqueue_request(). local_bh_disable/enable is used to prevent
167 	 * cryptd_enqueue_request() being accessed from software interrupts.
168 	 */
169 	local_bh_disable();
170 	preempt_disable();
171 	backlog = crypto_get_backlog(&cpu_queue->queue);
172 	req = crypto_dequeue_request(&cpu_queue->queue);
173 	preempt_enable();
174 	local_bh_enable();
175 
176 	if (!req)
177 		return;
178 
179 	if (backlog)
180 		backlog->complete(backlog, -EINPROGRESS);
181 	req->complete(req, 0);
182 
183 	if (cpu_queue->queue.qlen)
184 		queue_work(cryptd_wq, &cpu_queue->work);
185 }
186 
187 static inline struct cryptd_queue *cryptd_get_queue(struct crypto_tfm *tfm)
188 {
189 	struct crypto_instance *inst = crypto_tfm_alg_instance(tfm);
190 	struct cryptd_instance_ctx *ictx = crypto_instance_ctx(inst);
191 	return ictx->queue;
192 }
193 
194 static inline void cryptd_check_internal(struct rtattr **tb, u32 *type,
195 					 u32 *mask)
196 {
197 	struct crypto_attr_type *algt;
198 
199 	algt = crypto_get_attr_type(tb);
200 	if (IS_ERR(algt))
201 		return;
202 
203 	*type |= algt->type & CRYPTO_ALG_INTERNAL;
204 	*mask |= algt->mask & CRYPTO_ALG_INTERNAL;
205 }
206 
207 static int cryptd_init_instance(struct crypto_instance *inst,
208 				struct crypto_alg *alg)
209 {
210 	if (snprintf(inst->alg.cra_driver_name, CRYPTO_MAX_ALG_NAME,
211 		     "cryptd(%s)",
212 		     alg->cra_driver_name) >= CRYPTO_MAX_ALG_NAME)
213 		return -ENAMETOOLONG;
214 
215 	memcpy(inst->alg.cra_name, alg->cra_name, CRYPTO_MAX_ALG_NAME);
216 
217 	inst->alg.cra_priority = alg->cra_priority + 50;
218 	inst->alg.cra_blocksize = alg->cra_blocksize;
219 	inst->alg.cra_alignmask = alg->cra_alignmask;
220 
221 	return 0;
222 }
223 
224 static void *cryptd_alloc_instance(struct crypto_alg *alg, unsigned int head,
225 				   unsigned int tail)
226 {
227 	char *p;
228 	struct crypto_instance *inst;
229 	int err;
230 
231 	p = kzalloc(head + sizeof(*inst) + tail, GFP_KERNEL);
232 	if (!p)
233 		return ERR_PTR(-ENOMEM);
234 
235 	inst = (void *)(p + head);
236 
237 	err = cryptd_init_instance(inst, alg);
238 	if (err)
239 		goto out_free_inst;
240 
241 out:
242 	return p;
243 
244 out_free_inst:
245 	kfree(p);
246 	p = ERR_PTR(err);
247 	goto out;
248 }
249 
250 static int cryptd_skcipher_setkey(struct crypto_skcipher *parent,
251 				  const u8 *key, unsigned int keylen)
252 {
253 	struct cryptd_skcipher_ctx *ctx = crypto_skcipher_ctx(parent);
254 	struct crypto_sync_skcipher *child = ctx->child;
255 	int err;
256 
257 	crypto_sync_skcipher_clear_flags(child, CRYPTO_TFM_REQ_MASK);
258 	crypto_sync_skcipher_set_flags(child,
259 				       crypto_skcipher_get_flags(parent) &
260 					 CRYPTO_TFM_REQ_MASK);
261 	err = crypto_sync_skcipher_setkey(child, key, keylen);
262 	crypto_skcipher_set_flags(parent,
263 				  crypto_sync_skcipher_get_flags(child) &
264 					  CRYPTO_TFM_RES_MASK);
265 	return err;
266 }
267 
268 static void cryptd_skcipher_complete(struct skcipher_request *req, int err)
269 {
270 	struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req);
271 	struct cryptd_skcipher_ctx *ctx = crypto_skcipher_ctx(tfm);
272 	struct cryptd_skcipher_request_ctx *rctx = skcipher_request_ctx(req);
273 	int refcnt = atomic_read(&ctx->refcnt);
274 
275 	local_bh_disable();
276 	rctx->complete(&req->base, err);
277 	local_bh_enable();
278 
279 	if (err != -EINPROGRESS && refcnt && atomic_dec_and_test(&ctx->refcnt))
280 		crypto_free_skcipher(tfm);
281 }
282 
283 static void cryptd_skcipher_encrypt(struct crypto_async_request *base,
284 				    int err)
285 {
286 	struct skcipher_request *req = skcipher_request_cast(base);
287 	struct cryptd_skcipher_request_ctx *rctx = skcipher_request_ctx(req);
288 	struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req);
289 	struct cryptd_skcipher_ctx *ctx = crypto_skcipher_ctx(tfm);
290 	struct crypto_sync_skcipher *child = ctx->child;
291 	SYNC_SKCIPHER_REQUEST_ON_STACK(subreq, child);
292 
293 	if (unlikely(err == -EINPROGRESS))
294 		goto out;
295 
296 	skcipher_request_set_sync_tfm(subreq, child);
297 	skcipher_request_set_callback(subreq, CRYPTO_TFM_REQ_MAY_SLEEP,
298 				      NULL, NULL);
299 	skcipher_request_set_crypt(subreq, req->src, req->dst, req->cryptlen,
300 				   req->iv);
301 
302 	err = crypto_skcipher_encrypt(subreq);
303 	skcipher_request_zero(subreq);
304 
305 	req->base.complete = rctx->complete;
306 
307 out:
308 	cryptd_skcipher_complete(req, err);
309 }
310 
311 static void cryptd_skcipher_decrypt(struct crypto_async_request *base,
312 				    int err)
313 {
314 	struct skcipher_request *req = skcipher_request_cast(base);
315 	struct cryptd_skcipher_request_ctx *rctx = skcipher_request_ctx(req);
316 	struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req);
317 	struct cryptd_skcipher_ctx *ctx = crypto_skcipher_ctx(tfm);
318 	struct crypto_sync_skcipher *child = ctx->child;
319 	SYNC_SKCIPHER_REQUEST_ON_STACK(subreq, child);
320 
321 	if (unlikely(err == -EINPROGRESS))
322 		goto out;
323 
324 	skcipher_request_set_sync_tfm(subreq, child);
325 	skcipher_request_set_callback(subreq, CRYPTO_TFM_REQ_MAY_SLEEP,
326 				      NULL, NULL);
327 	skcipher_request_set_crypt(subreq, req->src, req->dst, req->cryptlen,
328 				   req->iv);
329 
330 	err = crypto_skcipher_decrypt(subreq);
331 	skcipher_request_zero(subreq);
332 
333 	req->base.complete = rctx->complete;
334 
335 out:
336 	cryptd_skcipher_complete(req, err);
337 }
338 
339 static int cryptd_skcipher_enqueue(struct skcipher_request *req,
340 				   crypto_completion_t compl)
341 {
342 	struct cryptd_skcipher_request_ctx *rctx = skcipher_request_ctx(req);
343 	struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req);
344 	struct cryptd_queue *queue;
345 
346 	queue = cryptd_get_queue(crypto_skcipher_tfm(tfm));
347 	rctx->complete = req->base.complete;
348 	req->base.complete = compl;
349 
350 	return cryptd_enqueue_request(queue, &req->base);
351 }
352 
353 static int cryptd_skcipher_encrypt_enqueue(struct skcipher_request *req)
354 {
355 	return cryptd_skcipher_enqueue(req, cryptd_skcipher_encrypt);
356 }
357 
358 static int cryptd_skcipher_decrypt_enqueue(struct skcipher_request *req)
359 {
360 	return cryptd_skcipher_enqueue(req, cryptd_skcipher_decrypt);
361 }
362 
363 static int cryptd_skcipher_init_tfm(struct crypto_skcipher *tfm)
364 {
365 	struct skcipher_instance *inst = skcipher_alg_instance(tfm);
366 	struct skcipherd_instance_ctx *ictx = skcipher_instance_ctx(inst);
367 	struct crypto_skcipher_spawn *spawn = &ictx->spawn;
368 	struct cryptd_skcipher_ctx *ctx = crypto_skcipher_ctx(tfm);
369 	struct crypto_skcipher *cipher;
370 
371 	cipher = crypto_spawn_skcipher(spawn);
372 	if (IS_ERR(cipher))
373 		return PTR_ERR(cipher);
374 
375 	ctx->child = (struct crypto_sync_skcipher *)cipher;
376 	crypto_skcipher_set_reqsize(
377 		tfm, sizeof(struct cryptd_skcipher_request_ctx));
378 	return 0;
379 }
380 
381 static void cryptd_skcipher_exit_tfm(struct crypto_skcipher *tfm)
382 {
383 	struct cryptd_skcipher_ctx *ctx = crypto_skcipher_ctx(tfm);
384 
385 	crypto_free_sync_skcipher(ctx->child);
386 }
387 
388 static void cryptd_skcipher_free(struct skcipher_instance *inst)
389 {
390 	struct skcipherd_instance_ctx *ctx = skcipher_instance_ctx(inst);
391 
392 	crypto_drop_skcipher(&ctx->spawn);
393 	kfree(inst);
394 }
395 
396 static int cryptd_create_skcipher(struct crypto_template *tmpl,
397 				  struct rtattr **tb,
398 				  struct cryptd_queue *queue)
399 {
400 	struct skcipherd_instance_ctx *ctx;
401 	struct skcipher_instance *inst;
402 	struct skcipher_alg *alg;
403 	const char *name;
404 	u32 type;
405 	u32 mask;
406 	int err;
407 
408 	type = 0;
409 	mask = CRYPTO_ALG_ASYNC;
410 
411 	cryptd_check_internal(tb, &type, &mask);
412 
413 	name = crypto_attr_alg_name(tb[1]);
414 	if (IS_ERR(name))
415 		return PTR_ERR(name);
416 
417 	inst = kzalloc(sizeof(*inst) + sizeof(*ctx), GFP_KERNEL);
418 	if (!inst)
419 		return -ENOMEM;
420 
421 	ctx = skcipher_instance_ctx(inst);
422 	ctx->queue = queue;
423 
424 	crypto_set_skcipher_spawn(&ctx->spawn, skcipher_crypto_instance(inst));
425 	err = crypto_grab_skcipher(&ctx->spawn, name, type, mask);
426 	if (err)
427 		goto out_free_inst;
428 
429 	alg = crypto_spawn_skcipher_alg(&ctx->spawn);
430 	err = cryptd_init_instance(skcipher_crypto_instance(inst), &alg->base);
431 	if (err)
432 		goto out_drop_skcipher;
433 
434 	inst->alg.base.cra_flags = CRYPTO_ALG_ASYNC |
435 				   (alg->base.cra_flags & CRYPTO_ALG_INTERNAL);
436 
437 	inst->alg.ivsize = crypto_skcipher_alg_ivsize(alg);
438 	inst->alg.chunksize = crypto_skcipher_alg_chunksize(alg);
439 	inst->alg.min_keysize = crypto_skcipher_alg_min_keysize(alg);
440 	inst->alg.max_keysize = crypto_skcipher_alg_max_keysize(alg);
441 
442 	inst->alg.base.cra_ctxsize = sizeof(struct cryptd_skcipher_ctx);
443 
444 	inst->alg.init = cryptd_skcipher_init_tfm;
445 	inst->alg.exit = cryptd_skcipher_exit_tfm;
446 
447 	inst->alg.setkey = cryptd_skcipher_setkey;
448 	inst->alg.encrypt = cryptd_skcipher_encrypt_enqueue;
449 	inst->alg.decrypt = cryptd_skcipher_decrypt_enqueue;
450 
451 	inst->free = cryptd_skcipher_free;
452 
453 	err = skcipher_register_instance(tmpl, inst);
454 	if (err) {
455 out_drop_skcipher:
456 		crypto_drop_skcipher(&ctx->spawn);
457 out_free_inst:
458 		kfree(inst);
459 	}
460 	return err;
461 }
462 
463 static int cryptd_hash_init_tfm(struct crypto_tfm *tfm)
464 {
465 	struct crypto_instance *inst = crypto_tfm_alg_instance(tfm);
466 	struct hashd_instance_ctx *ictx = crypto_instance_ctx(inst);
467 	struct crypto_shash_spawn *spawn = &ictx->spawn;
468 	struct cryptd_hash_ctx *ctx = crypto_tfm_ctx(tfm);
469 	struct crypto_shash *hash;
470 
471 	hash = crypto_spawn_shash(spawn);
472 	if (IS_ERR(hash))
473 		return PTR_ERR(hash);
474 
475 	ctx->child = hash;
476 	crypto_ahash_set_reqsize(__crypto_ahash_cast(tfm),
477 				 sizeof(struct cryptd_hash_request_ctx) +
478 				 crypto_shash_descsize(hash));
479 	return 0;
480 }
481 
482 static void cryptd_hash_exit_tfm(struct crypto_tfm *tfm)
483 {
484 	struct cryptd_hash_ctx *ctx = crypto_tfm_ctx(tfm);
485 
486 	crypto_free_shash(ctx->child);
487 }
488 
489 static int cryptd_hash_setkey(struct crypto_ahash *parent,
490 				   const u8 *key, unsigned int keylen)
491 {
492 	struct cryptd_hash_ctx *ctx   = crypto_ahash_ctx(parent);
493 	struct crypto_shash *child = ctx->child;
494 	int err;
495 
496 	crypto_shash_clear_flags(child, CRYPTO_TFM_REQ_MASK);
497 	crypto_shash_set_flags(child, crypto_ahash_get_flags(parent) &
498 				      CRYPTO_TFM_REQ_MASK);
499 	err = crypto_shash_setkey(child, key, keylen);
500 	crypto_ahash_set_flags(parent, crypto_shash_get_flags(child) &
501 				       CRYPTO_TFM_RES_MASK);
502 	return err;
503 }
504 
505 static int cryptd_hash_enqueue(struct ahash_request *req,
506 				crypto_completion_t compl)
507 {
508 	struct cryptd_hash_request_ctx *rctx = ahash_request_ctx(req);
509 	struct crypto_ahash *tfm = crypto_ahash_reqtfm(req);
510 	struct cryptd_queue *queue =
511 		cryptd_get_queue(crypto_ahash_tfm(tfm));
512 
513 	rctx->complete = req->base.complete;
514 	req->base.complete = compl;
515 
516 	return cryptd_enqueue_request(queue, &req->base);
517 }
518 
519 static void cryptd_hash_complete(struct ahash_request *req, int err)
520 {
521 	struct crypto_ahash *tfm = crypto_ahash_reqtfm(req);
522 	struct cryptd_hash_ctx *ctx = crypto_ahash_ctx(tfm);
523 	struct cryptd_hash_request_ctx *rctx = ahash_request_ctx(req);
524 	int refcnt = atomic_read(&ctx->refcnt);
525 
526 	local_bh_disable();
527 	rctx->complete(&req->base, err);
528 	local_bh_enable();
529 
530 	if (err != -EINPROGRESS && refcnt && atomic_dec_and_test(&ctx->refcnt))
531 		crypto_free_ahash(tfm);
532 }
533 
534 static void cryptd_hash_init(struct crypto_async_request *req_async, int err)
535 {
536 	struct cryptd_hash_ctx *ctx = crypto_tfm_ctx(req_async->tfm);
537 	struct crypto_shash *child = ctx->child;
538 	struct ahash_request *req = ahash_request_cast(req_async);
539 	struct cryptd_hash_request_ctx *rctx = ahash_request_ctx(req);
540 	struct shash_desc *desc = &rctx->desc;
541 
542 	if (unlikely(err == -EINPROGRESS))
543 		goto out;
544 
545 	desc->tfm = child;
546 
547 	err = crypto_shash_init(desc);
548 
549 	req->base.complete = rctx->complete;
550 
551 out:
552 	cryptd_hash_complete(req, err);
553 }
554 
555 static int cryptd_hash_init_enqueue(struct ahash_request *req)
556 {
557 	return cryptd_hash_enqueue(req, cryptd_hash_init);
558 }
559 
560 static void cryptd_hash_update(struct crypto_async_request *req_async, int err)
561 {
562 	struct ahash_request *req = ahash_request_cast(req_async);
563 	struct cryptd_hash_request_ctx *rctx;
564 
565 	rctx = ahash_request_ctx(req);
566 
567 	if (unlikely(err == -EINPROGRESS))
568 		goto out;
569 
570 	err = shash_ahash_update(req, &rctx->desc);
571 
572 	req->base.complete = rctx->complete;
573 
574 out:
575 	cryptd_hash_complete(req, err);
576 }
577 
578 static int cryptd_hash_update_enqueue(struct ahash_request *req)
579 {
580 	return cryptd_hash_enqueue(req, cryptd_hash_update);
581 }
582 
583 static void cryptd_hash_final(struct crypto_async_request *req_async, int err)
584 {
585 	struct ahash_request *req = ahash_request_cast(req_async);
586 	struct cryptd_hash_request_ctx *rctx = ahash_request_ctx(req);
587 
588 	if (unlikely(err == -EINPROGRESS))
589 		goto out;
590 
591 	err = crypto_shash_final(&rctx->desc, req->result);
592 
593 	req->base.complete = rctx->complete;
594 
595 out:
596 	cryptd_hash_complete(req, err);
597 }
598 
599 static int cryptd_hash_final_enqueue(struct ahash_request *req)
600 {
601 	return cryptd_hash_enqueue(req, cryptd_hash_final);
602 }
603 
604 static void cryptd_hash_finup(struct crypto_async_request *req_async, int err)
605 {
606 	struct ahash_request *req = ahash_request_cast(req_async);
607 	struct cryptd_hash_request_ctx *rctx = ahash_request_ctx(req);
608 
609 	if (unlikely(err == -EINPROGRESS))
610 		goto out;
611 
612 	err = shash_ahash_finup(req, &rctx->desc);
613 
614 	req->base.complete = rctx->complete;
615 
616 out:
617 	cryptd_hash_complete(req, err);
618 }
619 
620 static int cryptd_hash_finup_enqueue(struct ahash_request *req)
621 {
622 	return cryptd_hash_enqueue(req, cryptd_hash_finup);
623 }
624 
625 static void cryptd_hash_digest(struct crypto_async_request *req_async, int err)
626 {
627 	struct cryptd_hash_ctx *ctx = crypto_tfm_ctx(req_async->tfm);
628 	struct crypto_shash *child = ctx->child;
629 	struct ahash_request *req = ahash_request_cast(req_async);
630 	struct cryptd_hash_request_ctx *rctx = ahash_request_ctx(req);
631 	struct shash_desc *desc = &rctx->desc;
632 
633 	if (unlikely(err == -EINPROGRESS))
634 		goto out;
635 
636 	desc->tfm = child;
637 
638 	err = shash_ahash_digest(req, desc);
639 
640 	req->base.complete = rctx->complete;
641 
642 out:
643 	cryptd_hash_complete(req, err);
644 }
645 
646 static int cryptd_hash_digest_enqueue(struct ahash_request *req)
647 {
648 	return cryptd_hash_enqueue(req, cryptd_hash_digest);
649 }
650 
651 static int cryptd_hash_export(struct ahash_request *req, void *out)
652 {
653 	struct cryptd_hash_request_ctx *rctx = ahash_request_ctx(req);
654 
655 	return crypto_shash_export(&rctx->desc, out);
656 }
657 
658 static int cryptd_hash_import(struct ahash_request *req, const void *in)
659 {
660 	struct crypto_ahash *tfm = crypto_ahash_reqtfm(req);
661 	struct cryptd_hash_ctx *ctx = crypto_ahash_ctx(tfm);
662 	struct shash_desc *desc = cryptd_shash_desc(req);
663 
664 	desc->tfm = ctx->child;
665 
666 	return crypto_shash_import(desc, in);
667 }
668 
669 static int cryptd_create_hash(struct crypto_template *tmpl, struct rtattr **tb,
670 			      struct cryptd_queue *queue)
671 {
672 	struct hashd_instance_ctx *ctx;
673 	struct ahash_instance *inst;
674 	struct shash_alg *salg;
675 	struct crypto_alg *alg;
676 	u32 type = 0;
677 	u32 mask = 0;
678 	int err;
679 
680 	cryptd_check_internal(tb, &type, &mask);
681 
682 	salg = shash_attr_alg(tb[1], type, mask);
683 	if (IS_ERR(salg))
684 		return PTR_ERR(salg);
685 
686 	alg = &salg->base;
687 	inst = cryptd_alloc_instance(alg, ahash_instance_headroom(),
688 				     sizeof(*ctx));
689 	err = PTR_ERR(inst);
690 	if (IS_ERR(inst))
691 		goto out_put_alg;
692 
693 	ctx = ahash_instance_ctx(inst);
694 	ctx->queue = queue;
695 
696 	err = crypto_init_shash_spawn(&ctx->spawn, salg,
697 				      ahash_crypto_instance(inst));
698 	if (err)
699 		goto out_free_inst;
700 
701 	inst->alg.halg.base.cra_flags = CRYPTO_ALG_ASYNC |
702 		(alg->cra_flags & (CRYPTO_ALG_INTERNAL |
703 				   CRYPTO_ALG_OPTIONAL_KEY));
704 
705 	inst->alg.halg.digestsize = salg->digestsize;
706 	inst->alg.halg.statesize = salg->statesize;
707 	inst->alg.halg.base.cra_ctxsize = sizeof(struct cryptd_hash_ctx);
708 
709 	inst->alg.halg.base.cra_init = cryptd_hash_init_tfm;
710 	inst->alg.halg.base.cra_exit = cryptd_hash_exit_tfm;
711 
712 	inst->alg.init   = cryptd_hash_init_enqueue;
713 	inst->alg.update = cryptd_hash_update_enqueue;
714 	inst->alg.final  = cryptd_hash_final_enqueue;
715 	inst->alg.finup  = cryptd_hash_finup_enqueue;
716 	inst->alg.export = cryptd_hash_export;
717 	inst->alg.import = cryptd_hash_import;
718 	if (crypto_shash_alg_has_setkey(salg))
719 		inst->alg.setkey = cryptd_hash_setkey;
720 	inst->alg.digest = cryptd_hash_digest_enqueue;
721 
722 	err = ahash_register_instance(tmpl, inst);
723 	if (err) {
724 		crypto_drop_shash(&ctx->spawn);
725 out_free_inst:
726 		kfree(inst);
727 	}
728 
729 out_put_alg:
730 	crypto_mod_put(alg);
731 	return err;
732 }
733 
734 static int cryptd_aead_setkey(struct crypto_aead *parent,
735 			      const u8 *key, unsigned int keylen)
736 {
737 	struct cryptd_aead_ctx *ctx = crypto_aead_ctx(parent);
738 	struct crypto_aead *child = ctx->child;
739 
740 	return crypto_aead_setkey(child, key, keylen);
741 }
742 
743 static int cryptd_aead_setauthsize(struct crypto_aead *parent,
744 				   unsigned int authsize)
745 {
746 	struct cryptd_aead_ctx *ctx = crypto_aead_ctx(parent);
747 	struct crypto_aead *child = ctx->child;
748 
749 	return crypto_aead_setauthsize(child, authsize);
750 }
751 
752 static void cryptd_aead_crypt(struct aead_request *req,
753 			struct crypto_aead *child,
754 			int err,
755 			int (*crypt)(struct aead_request *req))
756 {
757 	struct cryptd_aead_request_ctx *rctx;
758 	struct cryptd_aead_ctx *ctx;
759 	crypto_completion_t compl;
760 	struct crypto_aead *tfm;
761 	int refcnt;
762 
763 	rctx = aead_request_ctx(req);
764 	compl = rctx->complete;
765 
766 	tfm = crypto_aead_reqtfm(req);
767 
768 	if (unlikely(err == -EINPROGRESS))
769 		goto out;
770 	aead_request_set_tfm(req, child);
771 	err = crypt( req );
772 
773 out:
774 	ctx = crypto_aead_ctx(tfm);
775 	refcnt = atomic_read(&ctx->refcnt);
776 
777 	local_bh_disable();
778 	compl(&req->base, err);
779 	local_bh_enable();
780 
781 	if (err != -EINPROGRESS && refcnt && atomic_dec_and_test(&ctx->refcnt))
782 		crypto_free_aead(tfm);
783 }
784 
785 static void cryptd_aead_encrypt(struct crypto_async_request *areq, int err)
786 {
787 	struct cryptd_aead_ctx *ctx = crypto_tfm_ctx(areq->tfm);
788 	struct crypto_aead *child = ctx->child;
789 	struct aead_request *req;
790 
791 	req = container_of(areq, struct aead_request, base);
792 	cryptd_aead_crypt(req, child, err, crypto_aead_alg(child)->encrypt);
793 }
794 
795 static void cryptd_aead_decrypt(struct crypto_async_request *areq, int err)
796 {
797 	struct cryptd_aead_ctx *ctx = crypto_tfm_ctx(areq->tfm);
798 	struct crypto_aead *child = ctx->child;
799 	struct aead_request *req;
800 
801 	req = container_of(areq, struct aead_request, base);
802 	cryptd_aead_crypt(req, child, err, crypto_aead_alg(child)->decrypt);
803 }
804 
805 static int cryptd_aead_enqueue(struct aead_request *req,
806 				    crypto_completion_t compl)
807 {
808 	struct cryptd_aead_request_ctx *rctx = aead_request_ctx(req);
809 	struct crypto_aead *tfm = crypto_aead_reqtfm(req);
810 	struct cryptd_queue *queue = cryptd_get_queue(crypto_aead_tfm(tfm));
811 
812 	rctx->complete = req->base.complete;
813 	req->base.complete = compl;
814 	return cryptd_enqueue_request(queue, &req->base);
815 }
816 
817 static int cryptd_aead_encrypt_enqueue(struct aead_request *req)
818 {
819 	return cryptd_aead_enqueue(req, cryptd_aead_encrypt );
820 }
821 
822 static int cryptd_aead_decrypt_enqueue(struct aead_request *req)
823 {
824 	return cryptd_aead_enqueue(req, cryptd_aead_decrypt );
825 }
826 
827 static int cryptd_aead_init_tfm(struct crypto_aead *tfm)
828 {
829 	struct aead_instance *inst = aead_alg_instance(tfm);
830 	struct aead_instance_ctx *ictx = aead_instance_ctx(inst);
831 	struct crypto_aead_spawn *spawn = &ictx->aead_spawn;
832 	struct cryptd_aead_ctx *ctx = crypto_aead_ctx(tfm);
833 	struct crypto_aead *cipher;
834 
835 	cipher = crypto_spawn_aead(spawn);
836 	if (IS_ERR(cipher))
837 		return PTR_ERR(cipher);
838 
839 	ctx->child = cipher;
840 	crypto_aead_set_reqsize(
841 		tfm, max((unsigned)sizeof(struct cryptd_aead_request_ctx),
842 			 crypto_aead_reqsize(cipher)));
843 	return 0;
844 }
845 
846 static void cryptd_aead_exit_tfm(struct crypto_aead *tfm)
847 {
848 	struct cryptd_aead_ctx *ctx = crypto_aead_ctx(tfm);
849 	crypto_free_aead(ctx->child);
850 }
851 
852 static int cryptd_create_aead(struct crypto_template *tmpl,
853 		              struct rtattr **tb,
854 			      struct cryptd_queue *queue)
855 {
856 	struct aead_instance_ctx *ctx;
857 	struct aead_instance *inst;
858 	struct aead_alg *alg;
859 	const char *name;
860 	u32 type = 0;
861 	u32 mask = CRYPTO_ALG_ASYNC;
862 	int err;
863 
864 	cryptd_check_internal(tb, &type, &mask);
865 
866 	name = crypto_attr_alg_name(tb[1]);
867 	if (IS_ERR(name))
868 		return PTR_ERR(name);
869 
870 	inst = kzalloc(sizeof(*inst) + sizeof(*ctx), GFP_KERNEL);
871 	if (!inst)
872 		return -ENOMEM;
873 
874 	ctx = aead_instance_ctx(inst);
875 	ctx->queue = queue;
876 
877 	crypto_set_aead_spawn(&ctx->aead_spawn, aead_crypto_instance(inst));
878 	err = crypto_grab_aead(&ctx->aead_spawn, name, type, mask);
879 	if (err)
880 		goto out_free_inst;
881 
882 	alg = crypto_spawn_aead_alg(&ctx->aead_spawn);
883 	err = cryptd_init_instance(aead_crypto_instance(inst), &alg->base);
884 	if (err)
885 		goto out_drop_aead;
886 
887 	inst->alg.base.cra_flags = CRYPTO_ALG_ASYNC |
888 				   (alg->base.cra_flags & CRYPTO_ALG_INTERNAL);
889 	inst->alg.base.cra_ctxsize = sizeof(struct cryptd_aead_ctx);
890 
891 	inst->alg.ivsize = crypto_aead_alg_ivsize(alg);
892 	inst->alg.maxauthsize = crypto_aead_alg_maxauthsize(alg);
893 
894 	inst->alg.init = cryptd_aead_init_tfm;
895 	inst->alg.exit = cryptd_aead_exit_tfm;
896 	inst->alg.setkey = cryptd_aead_setkey;
897 	inst->alg.setauthsize = cryptd_aead_setauthsize;
898 	inst->alg.encrypt = cryptd_aead_encrypt_enqueue;
899 	inst->alg.decrypt = cryptd_aead_decrypt_enqueue;
900 
901 	err = aead_register_instance(tmpl, inst);
902 	if (err) {
903 out_drop_aead:
904 		crypto_drop_aead(&ctx->aead_spawn);
905 out_free_inst:
906 		kfree(inst);
907 	}
908 	return err;
909 }
910 
911 static struct cryptd_queue queue;
912 
913 static int cryptd_create(struct crypto_template *tmpl, struct rtattr **tb)
914 {
915 	struct crypto_attr_type *algt;
916 
917 	algt = crypto_get_attr_type(tb);
918 	if (IS_ERR(algt))
919 		return PTR_ERR(algt);
920 
921 	switch (algt->type & algt->mask & CRYPTO_ALG_TYPE_MASK) {
922 	case CRYPTO_ALG_TYPE_BLKCIPHER:
923 		return cryptd_create_skcipher(tmpl, tb, &queue);
924 	case CRYPTO_ALG_TYPE_HASH:
925 		return cryptd_create_hash(tmpl, tb, &queue);
926 	case CRYPTO_ALG_TYPE_AEAD:
927 		return cryptd_create_aead(tmpl, tb, &queue);
928 	}
929 
930 	return -EINVAL;
931 }
932 
933 static void cryptd_free(struct crypto_instance *inst)
934 {
935 	struct cryptd_instance_ctx *ctx = crypto_instance_ctx(inst);
936 	struct hashd_instance_ctx *hctx = crypto_instance_ctx(inst);
937 	struct aead_instance_ctx *aead_ctx = crypto_instance_ctx(inst);
938 
939 	switch (inst->alg.cra_flags & CRYPTO_ALG_TYPE_MASK) {
940 	case CRYPTO_ALG_TYPE_AHASH:
941 		crypto_drop_shash(&hctx->spawn);
942 		kfree(ahash_instance(inst));
943 		return;
944 	case CRYPTO_ALG_TYPE_AEAD:
945 		crypto_drop_aead(&aead_ctx->aead_spawn);
946 		kfree(aead_instance(inst));
947 		return;
948 	default:
949 		crypto_drop_spawn(&ctx->spawn);
950 		kfree(inst);
951 	}
952 }
953 
954 static struct crypto_template cryptd_tmpl = {
955 	.name = "cryptd",
956 	.create = cryptd_create,
957 	.free = cryptd_free,
958 	.module = THIS_MODULE,
959 };
960 
961 struct cryptd_skcipher *cryptd_alloc_skcipher(const char *alg_name,
962 					      u32 type, u32 mask)
963 {
964 	char cryptd_alg_name[CRYPTO_MAX_ALG_NAME];
965 	struct cryptd_skcipher_ctx *ctx;
966 	struct crypto_skcipher *tfm;
967 
968 	if (snprintf(cryptd_alg_name, CRYPTO_MAX_ALG_NAME,
969 		     "cryptd(%s)", alg_name) >= CRYPTO_MAX_ALG_NAME)
970 		return ERR_PTR(-EINVAL);
971 
972 	tfm = crypto_alloc_skcipher(cryptd_alg_name, type, mask);
973 	if (IS_ERR(tfm))
974 		return ERR_CAST(tfm);
975 
976 	if (tfm->base.__crt_alg->cra_module != THIS_MODULE) {
977 		crypto_free_skcipher(tfm);
978 		return ERR_PTR(-EINVAL);
979 	}
980 
981 	ctx = crypto_skcipher_ctx(tfm);
982 	atomic_set(&ctx->refcnt, 1);
983 
984 	return container_of(tfm, struct cryptd_skcipher, base);
985 }
986 EXPORT_SYMBOL_GPL(cryptd_alloc_skcipher);
987 
988 struct crypto_skcipher *cryptd_skcipher_child(struct cryptd_skcipher *tfm)
989 {
990 	struct cryptd_skcipher_ctx *ctx = crypto_skcipher_ctx(&tfm->base);
991 
992 	return &ctx->child->base;
993 }
994 EXPORT_SYMBOL_GPL(cryptd_skcipher_child);
995 
996 bool cryptd_skcipher_queued(struct cryptd_skcipher *tfm)
997 {
998 	struct cryptd_skcipher_ctx *ctx = crypto_skcipher_ctx(&tfm->base);
999 
1000 	return atomic_read(&ctx->refcnt) - 1;
1001 }
1002 EXPORT_SYMBOL_GPL(cryptd_skcipher_queued);
1003 
1004 void cryptd_free_skcipher(struct cryptd_skcipher *tfm)
1005 {
1006 	struct cryptd_skcipher_ctx *ctx = crypto_skcipher_ctx(&tfm->base);
1007 
1008 	if (atomic_dec_and_test(&ctx->refcnt))
1009 		crypto_free_skcipher(&tfm->base);
1010 }
1011 EXPORT_SYMBOL_GPL(cryptd_free_skcipher);
1012 
1013 struct cryptd_ahash *cryptd_alloc_ahash(const char *alg_name,
1014 					u32 type, u32 mask)
1015 {
1016 	char cryptd_alg_name[CRYPTO_MAX_ALG_NAME];
1017 	struct cryptd_hash_ctx *ctx;
1018 	struct crypto_ahash *tfm;
1019 
1020 	if (snprintf(cryptd_alg_name, CRYPTO_MAX_ALG_NAME,
1021 		     "cryptd(%s)", alg_name) >= CRYPTO_MAX_ALG_NAME)
1022 		return ERR_PTR(-EINVAL);
1023 	tfm = crypto_alloc_ahash(cryptd_alg_name, type, mask);
1024 	if (IS_ERR(tfm))
1025 		return ERR_CAST(tfm);
1026 	if (tfm->base.__crt_alg->cra_module != THIS_MODULE) {
1027 		crypto_free_ahash(tfm);
1028 		return ERR_PTR(-EINVAL);
1029 	}
1030 
1031 	ctx = crypto_ahash_ctx(tfm);
1032 	atomic_set(&ctx->refcnt, 1);
1033 
1034 	return __cryptd_ahash_cast(tfm);
1035 }
1036 EXPORT_SYMBOL_GPL(cryptd_alloc_ahash);
1037 
1038 struct crypto_shash *cryptd_ahash_child(struct cryptd_ahash *tfm)
1039 {
1040 	struct cryptd_hash_ctx *ctx = crypto_ahash_ctx(&tfm->base);
1041 
1042 	return ctx->child;
1043 }
1044 EXPORT_SYMBOL_GPL(cryptd_ahash_child);
1045 
1046 struct shash_desc *cryptd_shash_desc(struct ahash_request *req)
1047 {
1048 	struct cryptd_hash_request_ctx *rctx = ahash_request_ctx(req);
1049 	return &rctx->desc;
1050 }
1051 EXPORT_SYMBOL_GPL(cryptd_shash_desc);
1052 
1053 bool cryptd_ahash_queued(struct cryptd_ahash *tfm)
1054 {
1055 	struct cryptd_hash_ctx *ctx = crypto_ahash_ctx(&tfm->base);
1056 
1057 	return atomic_read(&ctx->refcnt) - 1;
1058 }
1059 EXPORT_SYMBOL_GPL(cryptd_ahash_queued);
1060 
1061 void cryptd_free_ahash(struct cryptd_ahash *tfm)
1062 {
1063 	struct cryptd_hash_ctx *ctx = crypto_ahash_ctx(&tfm->base);
1064 
1065 	if (atomic_dec_and_test(&ctx->refcnt))
1066 		crypto_free_ahash(&tfm->base);
1067 }
1068 EXPORT_SYMBOL_GPL(cryptd_free_ahash);
1069 
1070 struct cryptd_aead *cryptd_alloc_aead(const char *alg_name,
1071 						  u32 type, u32 mask)
1072 {
1073 	char cryptd_alg_name[CRYPTO_MAX_ALG_NAME];
1074 	struct cryptd_aead_ctx *ctx;
1075 	struct crypto_aead *tfm;
1076 
1077 	if (snprintf(cryptd_alg_name, CRYPTO_MAX_ALG_NAME,
1078 		     "cryptd(%s)", alg_name) >= CRYPTO_MAX_ALG_NAME)
1079 		return ERR_PTR(-EINVAL);
1080 	tfm = crypto_alloc_aead(cryptd_alg_name, type, mask);
1081 	if (IS_ERR(tfm))
1082 		return ERR_CAST(tfm);
1083 	if (tfm->base.__crt_alg->cra_module != THIS_MODULE) {
1084 		crypto_free_aead(tfm);
1085 		return ERR_PTR(-EINVAL);
1086 	}
1087 
1088 	ctx = crypto_aead_ctx(tfm);
1089 	atomic_set(&ctx->refcnt, 1);
1090 
1091 	return __cryptd_aead_cast(tfm);
1092 }
1093 EXPORT_SYMBOL_GPL(cryptd_alloc_aead);
1094 
1095 struct crypto_aead *cryptd_aead_child(struct cryptd_aead *tfm)
1096 {
1097 	struct cryptd_aead_ctx *ctx;
1098 	ctx = crypto_aead_ctx(&tfm->base);
1099 	return ctx->child;
1100 }
1101 EXPORT_SYMBOL_GPL(cryptd_aead_child);
1102 
1103 bool cryptd_aead_queued(struct cryptd_aead *tfm)
1104 {
1105 	struct cryptd_aead_ctx *ctx = crypto_aead_ctx(&tfm->base);
1106 
1107 	return atomic_read(&ctx->refcnt) - 1;
1108 }
1109 EXPORT_SYMBOL_GPL(cryptd_aead_queued);
1110 
1111 void cryptd_free_aead(struct cryptd_aead *tfm)
1112 {
1113 	struct cryptd_aead_ctx *ctx = crypto_aead_ctx(&tfm->base);
1114 
1115 	if (atomic_dec_and_test(&ctx->refcnt))
1116 		crypto_free_aead(&tfm->base);
1117 }
1118 EXPORT_SYMBOL_GPL(cryptd_free_aead);
1119 
1120 static int __init cryptd_init(void)
1121 {
1122 	int err;
1123 
1124 	cryptd_wq = alloc_workqueue("cryptd", WQ_MEM_RECLAIM | WQ_CPU_INTENSIVE,
1125 				    1);
1126 	if (!cryptd_wq)
1127 		return -ENOMEM;
1128 
1129 	err = cryptd_init_queue(&queue, cryptd_max_cpu_qlen);
1130 	if (err)
1131 		goto err_destroy_wq;
1132 
1133 	err = crypto_register_template(&cryptd_tmpl);
1134 	if (err)
1135 		goto err_fini_queue;
1136 
1137 	return 0;
1138 
1139 err_fini_queue:
1140 	cryptd_fini_queue(&queue);
1141 err_destroy_wq:
1142 	destroy_workqueue(cryptd_wq);
1143 	return err;
1144 }
1145 
1146 static void __exit cryptd_exit(void)
1147 {
1148 	destroy_workqueue(cryptd_wq);
1149 	cryptd_fini_queue(&queue);
1150 	crypto_unregister_template(&cryptd_tmpl);
1151 }
1152 
1153 subsys_initcall(cryptd_init);
1154 module_exit(cryptd_exit);
1155 
1156 MODULE_LICENSE("GPL");
1157 MODULE_DESCRIPTION("Software async crypto daemon");
1158 MODULE_ALIAS_CRYPTO("cryptd");
1159