xref: /openbmc/linux/crypto/algif_aead.c (revision 2a598d0b)
1 // SPDX-License-Identifier: GPL-2.0-or-later
2 /*
3  * algif_aead: User-space interface for AEAD algorithms
4  *
5  * Copyright (C) 2014, Stephan Mueller <smueller@chronox.de>
6  *
7  * This file provides the user-space API for AEAD ciphers.
8  *
9  * The following concept of the memory management is used:
10  *
11  * The kernel maintains two SGLs, the TX SGL and the RX SGL. The TX SGL is
12  * filled by user space with the data submitted via sendmsg (maybe with
13  * MSG_SPLICE_PAGES).  Filling up the TX SGL does not cause a crypto operation
14  * -- the data will only be tracked by the kernel. Upon receipt of one recvmsg
15  * call, the caller must provide a buffer which is tracked with the RX SGL.
16  *
17  * During the processing of the recvmsg operation, the cipher request is
18  * allocated and prepared. As part of the recvmsg operation, the processed
19  * TX buffers are extracted from the TX SGL into a separate SGL.
20  *
21  * After the completion of the crypto operation, the RX SGL and the cipher
22  * request is released. The extracted TX SGL parts are released together with
23  * the RX SGL release.
24  */
25 
26 #include <crypto/internal/aead.h>
27 #include <crypto/scatterwalk.h>
28 #include <crypto/if_alg.h>
29 #include <crypto/skcipher.h>
30 #include <crypto/null.h>
31 #include <linux/init.h>
32 #include <linux/list.h>
33 #include <linux/kernel.h>
34 #include <linux/mm.h>
35 #include <linux/module.h>
36 #include <linux/net.h>
37 #include <net/sock.h>
38 
39 struct aead_tfm {
40 	struct crypto_aead *aead;
41 	struct crypto_sync_skcipher *null_tfm;
42 };
43 
44 static inline bool aead_sufficient_data(struct sock *sk)
45 {
46 	struct alg_sock *ask = alg_sk(sk);
47 	struct sock *psk = ask->parent;
48 	struct alg_sock *pask = alg_sk(psk);
49 	struct af_alg_ctx *ctx = ask->private;
50 	struct aead_tfm *aeadc = pask->private;
51 	struct crypto_aead *tfm = aeadc->aead;
52 	unsigned int as = crypto_aead_authsize(tfm);
53 
54 	/*
55 	 * The minimum amount of memory needed for an AEAD cipher is
56 	 * the AAD and in case of decryption the tag.
57 	 */
58 	return ctx->used >= ctx->aead_assoclen + (ctx->enc ? 0 : as);
59 }
60 
61 static int aead_sendmsg(struct socket *sock, struct msghdr *msg, size_t size)
62 {
63 	struct sock *sk = sock->sk;
64 	struct alg_sock *ask = alg_sk(sk);
65 	struct sock *psk = ask->parent;
66 	struct alg_sock *pask = alg_sk(psk);
67 	struct aead_tfm *aeadc = pask->private;
68 	struct crypto_aead *tfm = aeadc->aead;
69 	unsigned int ivsize = crypto_aead_ivsize(tfm);
70 
71 	return af_alg_sendmsg(sock, msg, size, ivsize);
72 }
73 
74 static int crypto_aead_copy_sgl(struct crypto_sync_skcipher *null_tfm,
75 				struct scatterlist *src,
76 				struct scatterlist *dst, unsigned int len)
77 {
78 	SYNC_SKCIPHER_REQUEST_ON_STACK(skreq, null_tfm);
79 
80 	skcipher_request_set_sync_tfm(skreq, null_tfm);
81 	skcipher_request_set_callback(skreq, CRYPTO_TFM_REQ_MAY_SLEEP,
82 				      NULL, NULL);
83 	skcipher_request_set_crypt(skreq, src, dst, len, NULL);
84 
85 	return crypto_skcipher_encrypt(skreq);
86 }
87 
88 static int _aead_recvmsg(struct socket *sock, struct msghdr *msg,
89 			 size_t ignored, int flags)
90 {
91 	struct sock *sk = sock->sk;
92 	struct alg_sock *ask = alg_sk(sk);
93 	struct sock *psk = ask->parent;
94 	struct alg_sock *pask = alg_sk(psk);
95 	struct af_alg_ctx *ctx = ask->private;
96 	struct aead_tfm *aeadc = pask->private;
97 	struct crypto_aead *tfm = aeadc->aead;
98 	struct crypto_sync_skcipher *null_tfm = aeadc->null_tfm;
99 	unsigned int i, as = crypto_aead_authsize(tfm);
100 	struct af_alg_async_req *areq;
101 	struct af_alg_tsgl *tsgl, *tmp;
102 	struct scatterlist *rsgl_src, *tsgl_src = NULL;
103 	int err = 0;
104 	size_t used = 0;		/* [in]  TX bufs to be en/decrypted */
105 	size_t outlen = 0;		/* [out] RX bufs produced by kernel */
106 	size_t usedpages = 0;		/* [in]  RX bufs to be used from user */
107 	size_t processed = 0;		/* [in]  TX bufs to be consumed */
108 
109 	if (!ctx->init || ctx->more) {
110 		err = af_alg_wait_for_data(sk, flags, 0);
111 		if (err)
112 			return err;
113 	}
114 
115 	/*
116 	 * Data length provided by caller via sendmsg that has not yet been
117 	 * processed.
118 	 */
119 	used = ctx->used;
120 
121 	/*
122 	 * Make sure sufficient data is present -- note, the same check is also
123 	 * present in sendmsg. The checks in sendmsg shall provide an
124 	 * information to the data sender that something is wrong, but they are
125 	 * irrelevant to maintain the kernel integrity.  We need this check
126 	 * here too in case user space decides to not honor the error message
127 	 * in sendmsg and still call recvmsg. This check here protects the
128 	 * kernel integrity.
129 	 */
130 	if (!aead_sufficient_data(sk))
131 		return -EINVAL;
132 
133 	/*
134 	 * Calculate the minimum output buffer size holding the result of the
135 	 * cipher operation. When encrypting data, the receiving buffer is
136 	 * larger by the tag length compared to the input buffer as the
137 	 * encryption operation generates the tag. For decryption, the input
138 	 * buffer provides the tag which is consumed resulting in only the
139 	 * plaintext without a buffer for the tag returned to the caller.
140 	 */
141 	if (ctx->enc)
142 		outlen = used + as;
143 	else
144 		outlen = used - as;
145 
146 	/*
147 	 * The cipher operation input data is reduced by the associated data
148 	 * length as this data is processed separately later on.
149 	 */
150 	used -= ctx->aead_assoclen;
151 
152 	/* Allocate cipher request for current operation. */
153 	areq = af_alg_alloc_areq(sk, sizeof(struct af_alg_async_req) +
154 				     crypto_aead_reqsize(tfm));
155 	if (IS_ERR(areq))
156 		return PTR_ERR(areq);
157 
158 	/* convert iovecs of output buffers into RX SGL */
159 	err = af_alg_get_rsgl(sk, msg, flags, areq, outlen, &usedpages);
160 	if (err)
161 		goto free;
162 
163 	/*
164 	 * Ensure output buffer is sufficiently large. If the caller provides
165 	 * less buffer space, only use the relative required input size. This
166 	 * allows AIO operation where the caller sent all data to be processed
167 	 * and the AIO operation performs the operation on the different chunks
168 	 * of the input data.
169 	 */
170 	if (usedpages < outlen) {
171 		size_t less = outlen - usedpages;
172 
173 		if (used < less) {
174 			err = -EINVAL;
175 			goto free;
176 		}
177 		used -= less;
178 		outlen -= less;
179 	}
180 
181 	processed = used + ctx->aead_assoclen;
182 	list_for_each_entry_safe(tsgl, tmp, &ctx->tsgl_list, list) {
183 		for (i = 0; i < tsgl->cur; i++) {
184 			struct scatterlist *process_sg = tsgl->sg + i;
185 
186 			if (!(process_sg->length) || !sg_page(process_sg))
187 				continue;
188 			tsgl_src = process_sg;
189 			break;
190 		}
191 		if (tsgl_src)
192 			break;
193 	}
194 	if (processed && !tsgl_src) {
195 		err = -EFAULT;
196 		goto free;
197 	}
198 
199 	/*
200 	 * Copy of AAD from source to destination
201 	 *
202 	 * The AAD is copied to the destination buffer without change. Even
203 	 * when user space uses an in-place cipher operation, the kernel
204 	 * will copy the data as it does not see whether such in-place operation
205 	 * is initiated.
206 	 *
207 	 * To ensure efficiency, the following implementation ensure that the
208 	 * ciphers are invoked to perform a crypto operation in-place. This
209 	 * is achieved by memory management specified as follows.
210 	 */
211 
212 	/* Use the RX SGL as source (and destination) for crypto op. */
213 	rsgl_src = areq->first_rsgl.sgl.sgt.sgl;
214 
215 	if (ctx->enc) {
216 		/*
217 		 * Encryption operation - The in-place cipher operation is
218 		 * achieved by the following operation:
219 		 *
220 		 * TX SGL: AAD || PT
221 		 *	    |	   |
222 		 *	    | copy |
223 		 *	    v	   v
224 		 * RX SGL: AAD || PT || Tag
225 		 */
226 		err = crypto_aead_copy_sgl(null_tfm, tsgl_src,
227 					   areq->first_rsgl.sgl.sgt.sgl,
228 					   processed);
229 		if (err)
230 			goto free;
231 		af_alg_pull_tsgl(sk, processed, NULL, 0);
232 	} else {
233 		/*
234 		 * Decryption operation - To achieve an in-place cipher
235 		 * operation, the following  SGL structure is used:
236 		 *
237 		 * TX SGL: AAD || CT || Tag
238 		 *	    |	   |	 ^
239 		 *	    | copy |	 | Create SGL link.
240 		 *	    v	   v	 |
241 		 * RX SGL: AAD || CT ----+
242 		 */
243 
244 		 /* Copy AAD || CT to RX SGL buffer for in-place operation. */
245 		err = crypto_aead_copy_sgl(null_tfm, tsgl_src,
246 					   areq->first_rsgl.sgl.sgt.sgl,
247 					   outlen);
248 		if (err)
249 			goto free;
250 
251 		/* Create TX SGL for tag and chain it to RX SGL. */
252 		areq->tsgl_entries = af_alg_count_tsgl(sk, processed,
253 						       processed - as);
254 		if (!areq->tsgl_entries)
255 			areq->tsgl_entries = 1;
256 		areq->tsgl = sock_kmalloc(sk, array_size(sizeof(*areq->tsgl),
257 							 areq->tsgl_entries),
258 					  GFP_KERNEL);
259 		if (!areq->tsgl) {
260 			err = -ENOMEM;
261 			goto free;
262 		}
263 		sg_init_table(areq->tsgl, areq->tsgl_entries);
264 
265 		/* Release TX SGL, except for tag data and reassign tag data. */
266 		af_alg_pull_tsgl(sk, processed, areq->tsgl, processed - as);
267 
268 		/* chain the areq TX SGL holding the tag with RX SGL */
269 		if (usedpages) {
270 			/* RX SGL present */
271 			struct af_alg_sgl *sgl_prev = &areq->last_rsgl->sgl;
272 			struct scatterlist *sg = sgl_prev->sgt.sgl;
273 
274 			sg_unmark_end(sg + sgl_prev->sgt.nents - 1);
275 			sg_chain(sg, sgl_prev->sgt.nents + 1, areq->tsgl);
276 		} else
277 			/* no RX SGL present (e.g. authentication only) */
278 			rsgl_src = areq->tsgl;
279 	}
280 
281 	/* Initialize the crypto operation */
282 	aead_request_set_crypt(&areq->cra_u.aead_req, rsgl_src,
283 			       areq->first_rsgl.sgl.sgt.sgl, used, ctx->iv);
284 	aead_request_set_ad(&areq->cra_u.aead_req, ctx->aead_assoclen);
285 	aead_request_set_tfm(&areq->cra_u.aead_req, tfm);
286 
287 	if (msg->msg_iocb && !is_sync_kiocb(msg->msg_iocb)) {
288 		/* AIO operation */
289 		sock_hold(sk);
290 		areq->iocb = msg->msg_iocb;
291 
292 		/* Remember output size that will be generated. */
293 		areq->outlen = outlen;
294 
295 		aead_request_set_callback(&areq->cra_u.aead_req,
296 					  CRYPTO_TFM_REQ_MAY_SLEEP,
297 					  af_alg_async_cb, areq);
298 		err = ctx->enc ? crypto_aead_encrypt(&areq->cra_u.aead_req) :
299 				 crypto_aead_decrypt(&areq->cra_u.aead_req);
300 
301 		/* AIO operation in progress */
302 		if (err == -EINPROGRESS)
303 			return -EIOCBQUEUED;
304 
305 		sock_put(sk);
306 	} else {
307 		/* Synchronous operation */
308 		aead_request_set_callback(&areq->cra_u.aead_req,
309 					  CRYPTO_TFM_REQ_MAY_SLEEP |
310 					  CRYPTO_TFM_REQ_MAY_BACKLOG,
311 					  crypto_req_done, &ctx->wait);
312 		err = crypto_wait_req(ctx->enc ?
313 				crypto_aead_encrypt(&areq->cra_u.aead_req) :
314 				crypto_aead_decrypt(&areq->cra_u.aead_req),
315 				&ctx->wait);
316 	}
317 
318 
319 free:
320 	af_alg_free_resources(areq);
321 
322 	return err ? err : outlen;
323 }
324 
325 static int aead_recvmsg(struct socket *sock, struct msghdr *msg,
326 			size_t ignored, int flags)
327 {
328 	struct sock *sk = sock->sk;
329 	int ret = 0;
330 
331 	lock_sock(sk);
332 	while (msg_data_left(msg)) {
333 		int err = _aead_recvmsg(sock, msg, ignored, flags);
334 
335 		/*
336 		 * This error covers -EIOCBQUEUED which implies that we can
337 		 * only handle one AIO request. If the caller wants to have
338 		 * multiple AIO requests in parallel, he must make multiple
339 		 * separate AIO calls.
340 		 *
341 		 * Also return the error if no data has been processed so far.
342 		 */
343 		if (err <= 0) {
344 			if (err == -EIOCBQUEUED || err == -EBADMSG || !ret)
345 				ret = err;
346 			goto out;
347 		}
348 
349 		ret += err;
350 	}
351 
352 out:
353 	af_alg_wmem_wakeup(sk);
354 	release_sock(sk);
355 	return ret;
356 }
357 
358 static struct proto_ops algif_aead_ops = {
359 	.family		=	PF_ALG,
360 
361 	.connect	=	sock_no_connect,
362 	.socketpair	=	sock_no_socketpair,
363 	.getname	=	sock_no_getname,
364 	.ioctl		=	sock_no_ioctl,
365 	.listen		=	sock_no_listen,
366 	.shutdown	=	sock_no_shutdown,
367 	.mmap		=	sock_no_mmap,
368 	.bind		=	sock_no_bind,
369 	.accept		=	sock_no_accept,
370 
371 	.release	=	af_alg_release,
372 	.sendmsg	=	aead_sendmsg,
373 	.recvmsg	=	aead_recvmsg,
374 	.poll		=	af_alg_poll,
375 };
376 
377 static int aead_check_key(struct socket *sock)
378 {
379 	int err = 0;
380 	struct sock *psk;
381 	struct alg_sock *pask;
382 	struct aead_tfm *tfm;
383 	struct sock *sk = sock->sk;
384 	struct alg_sock *ask = alg_sk(sk);
385 
386 	lock_sock(sk);
387 	if (!atomic_read(&ask->nokey_refcnt))
388 		goto unlock_child;
389 
390 	psk = ask->parent;
391 	pask = alg_sk(ask->parent);
392 	tfm = pask->private;
393 
394 	err = -ENOKEY;
395 	lock_sock_nested(psk, SINGLE_DEPTH_NESTING);
396 	if (crypto_aead_get_flags(tfm->aead) & CRYPTO_TFM_NEED_KEY)
397 		goto unlock;
398 
399 	atomic_dec(&pask->nokey_refcnt);
400 	atomic_set(&ask->nokey_refcnt, 0);
401 
402 	err = 0;
403 
404 unlock:
405 	release_sock(psk);
406 unlock_child:
407 	release_sock(sk);
408 
409 	return err;
410 }
411 
412 static int aead_sendmsg_nokey(struct socket *sock, struct msghdr *msg,
413 				  size_t size)
414 {
415 	int err;
416 
417 	err = aead_check_key(sock);
418 	if (err)
419 		return err;
420 
421 	return aead_sendmsg(sock, msg, size);
422 }
423 
424 static int aead_recvmsg_nokey(struct socket *sock, struct msghdr *msg,
425 				  size_t ignored, int flags)
426 {
427 	int err;
428 
429 	err = aead_check_key(sock);
430 	if (err)
431 		return err;
432 
433 	return aead_recvmsg(sock, msg, ignored, flags);
434 }
435 
436 static struct proto_ops algif_aead_ops_nokey = {
437 	.family		=	PF_ALG,
438 
439 	.connect	=	sock_no_connect,
440 	.socketpair	=	sock_no_socketpair,
441 	.getname	=	sock_no_getname,
442 	.ioctl		=	sock_no_ioctl,
443 	.listen		=	sock_no_listen,
444 	.shutdown	=	sock_no_shutdown,
445 	.mmap		=	sock_no_mmap,
446 	.bind		=	sock_no_bind,
447 	.accept		=	sock_no_accept,
448 
449 	.release	=	af_alg_release,
450 	.sendmsg	=	aead_sendmsg_nokey,
451 	.recvmsg	=	aead_recvmsg_nokey,
452 	.poll		=	af_alg_poll,
453 };
454 
455 static void *aead_bind(const char *name, u32 type, u32 mask)
456 {
457 	struct aead_tfm *tfm;
458 	struct crypto_aead *aead;
459 	struct crypto_sync_skcipher *null_tfm;
460 
461 	tfm = kzalloc(sizeof(*tfm), GFP_KERNEL);
462 	if (!tfm)
463 		return ERR_PTR(-ENOMEM);
464 
465 	aead = crypto_alloc_aead(name, type, mask);
466 	if (IS_ERR(aead)) {
467 		kfree(tfm);
468 		return ERR_CAST(aead);
469 	}
470 
471 	null_tfm = crypto_get_default_null_skcipher();
472 	if (IS_ERR(null_tfm)) {
473 		crypto_free_aead(aead);
474 		kfree(tfm);
475 		return ERR_CAST(null_tfm);
476 	}
477 
478 	tfm->aead = aead;
479 	tfm->null_tfm = null_tfm;
480 
481 	return tfm;
482 }
483 
484 static void aead_release(void *private)
485 {
486 	struct aead_tfm *tfm = private;
487 
488 	crypto_free_aead(tfm->aead);
489 	crypto_put_default_null_skcipher();
490 	kfree(tfm);
491 }
492 
493 static int aead_setauthsize(void *private, unsigned int authsize)
494 {
495 	struct aead_tfm *tfm = private;
496 
497 	return crypto_aead_setauthsize(tfm->aead, authsize);
498 }
499 
500 static int aead_setkey(void *private, const u8 *key, unsigned int keylen)
501 {
502 	struct aead_tfm *tfm = private;
503 
504 	return crypto_aead_setkey(tfm->aead, key, keylen);
505 }
506 
507 static void aead_sock_destruct(struct sock *sk)
508 {
509 	struct alg_sock *ask = alg_sk(sk);
510 	struct af_alg_ctx *ctx = ask->private;
511 	struct sock *psk = ask->parent;
512 	struct alg_sock *pask = alg_sk(psk);
513 	struct aead_tfm *aeadc = pask->private;
514 	struct crypto_aead *tfm = aeadc->aead;
515 	unsigned int ivlen = crypto_aead_ivsize(tfm);
516 
517 	af_alg_pull_tsgl(sk, ctx->used, NULL, 0);
518 	sock_kzfree_s(sk, ctx->iv, ivlen);
519 	sock_kfree_s(sk, ctx, ctx->len);
520 	af_alg_release_parent(sk);
521 }
522 
523 static int aead_accept_parent_nokey(void *private, struct sock *sk)
524 {
525 	struct af_alg_ctx *ctx;
526 	struct alg_sock *ask = alg_sk(sk);
527 	struct aead_tfm *tfm = private;
528 	struct crypto_aead *aead = tfm->aead;
529 	unsigned int len = sizeof(*ctx);
530 	unsigned int ivlen = crypto_aead_ivsize(aead);
531 
532 	ctx = sock_kmalloc(sk, len, GFP_KERNEL);
533 	if (!ctx)
534 		return -ENOMEM;
535 	memset(ctx, 0, len);
536 
537 	ctx->iv = sock_kmalloc(sk, ivlen, GFP_KERNEL);
538 	if (!ctx->iv) {
539 		sock_kfree_s(sk, ctx, len);
540 		return -ENOMEM;
541 	}
542 	memset(ctx->iv, 0, ivlen);
543 
544 	INIT_LIST_HEAD(&ctx->tsgl_list);
545 	ctx->len = len;
546 	crypto_init_wait(&ctx->wait);
547 
548 	ask->private = ctx;
549 
550 	sk->sk_destruct = aead_sock_destruct;
551 
552 	return 0;
553 }
554 
555 static int aead_accept_parent(void *private, struct sock *sk)
556 {
557 	struct aead_tfm *tfm = private;
558 
559 	if (crypto_aead_get_flags(tfm->aead) & CRYPTO_TFM_NEED_KEY)
560 		return -ENOKEY;
561 
562 	return aead_accept_parent_nokey(private, sk);
563 }
564 
565 static const struct af_alg_type algif_type_aead = {
566 	.bind		=	aead_bind,
567 	.release	=	aead_release,
568 	.setkey		=	aead_setkey,
569 	.setauthsize	=	aead_setauthsize,
570 	.accept		=	aead_accept_parent,
571 	.accept_nokey	=	aead_accept_parent_nokey,
572 	.ops		=	&algif_aead_ops,
573 	.ops_nokey	=	&algif_aead_ops_nokey,
574 	.name		=	"aead",
575 	.owner		=	THIS_MODULE
576 };
577 
578 static int __init algif_aead_init(void)
579 {
580 	return af_alg_register_type(&algif_type_aead);
581 }
582 
583 static void __exit algif_aead_exit(void)
584 {
585 	int err = af_alg_unregister_type(&algif_type_aead);
586 	BUG_ON(err);
587 }
588 
589 module_init(algif_aead_init);
590 module_exit(algif_aead_exit);
591 MODULE_LICENSE("GPL");
592 MODULE_AUTHOR("Stephan Mueller <smueller@chronox.de>");
593 MODULE_DESCRIPTION("AEAD kernel crypto API user space interface");
594