xref: /openbmc/linux/net/tls/tls_strp.c (revision f0931824) 
1 // SPDX-License-Identifier: GPL-2.0-only
2 /* Copyright (c) 2016 Tom Herbert <tom@herbertland.com> */
3 
4 #include <linux/skbuff.h>
5 #include <linux/workqueue.h>
6 #include <net/strparser.h>
7 #include <net/tcp.h>
8 #include <net/sock.h>
9 #include <net/tls.h>
10 
11 #include "tls.h"
12 
13 static struct workqueue_struct *tls_strp_wq;
14 
15 static void tls_strp_abort_strp(struct tls_strparser *strp, int err)
16 {
17 	if (strp->stopped)
18 		return;
19 
20 	strp->stopped = 1;
21 
22 	/* Report an error on the lower socket */
23 	WRITE_ONCE(strp->sk->sk_err, -err);
24 	/* Paired with smp_rmb() in tcp_poll() */
25 	smp_wmb();
26 	sk_error_report(strp->sk);
27 }
28 
29 static void tls_strp_anchor_free(struct tls_strparser *strp)
30 {
31 	struct skb_shared_info *shinfo = skb_shinfo(strp->anchor);
32 
33 	DEBUG_NET_WARN_ON_ONCE(atomic_read(&shinfo->dataref) != 1);
34 	if (!strp->copy_mode)
35 		shinfo->frag_list = NULL;
36 	consume_skb(strp->anchor);
37 	strp->anchor = NULL;
38 }
39 
40 static struct sk_buff *
41 tls_strp_skb_copy(struct tls_strparser *strp, struct sk_buff *in_skb,
42 		  int offset, int len)
43 {
44 	struct sk_buff *skb;
45 	int i, err;
46 
47 	skb = alloc_skb_with_frags(0, len, TLS_PAGE_ORDER,
48 				   &err, strp->sk->sk_allocation);
49 	if (!skb)
50 		return NULL;
51 
52 	for (i = 0; i < skb_shinfo(skb)->nr_frags; i++) {
53 		skb_frag_t *frag = &skb_shinfo(skb)->frags[i];
54 
55 		WARN_ON_ONCE(skb_copy_bits(in_skb, offset,
56 					   skb_frag_address(frag),
57 					   skb_frag_size(frag)));
58 		offset += skb_frag_size(frag);
59 	}
60 
61 	skb->len = len;
62 	skb->data_len = len;
63 	skb_copy_header(skb, in_skb);
64 	return skb;
65 }
66 
67 /* Create a new skb with the contents of input copied to its page frags */
68 static struct sk_buff *tls_strp_msg_make_copy(struct tls_strparser *strp)
69 {
70 	struct strp_msg *rxm;
71 	struct sk_buff *skb;
72 
73 	skb = tls_strp_skb_copy(strp, strp->anchor, strp->stm.offset,
74 				strp->stm.full_len);
75 	if (!skb)
76 		return NULL;
77 
78 	rxm = strp_msg(skb);
79 	rxm->offset = 0;
80 	return skb;
81 }
82 
83 /* Steal the input skb, input msg is invalid after calling this function */
84 struct sk_buff *tls_strp_msg_detach(struct tls_sw_context_rx *ctx)
85 {
86 	struct tls_strparser *strp = &ctx->strp;
87 
88 #ifdef CONFIG_TLS_DEVICE
89 	DEBUG_NET_WARN_ON_ONCE(!strp->anchor->decrypted);
90 #else
91 	/* This function turns an input into an output,
92 	 * that can only happen if we have offload.
93 	 */
94 	WARN_ON(1);
95 #endif
96 
97 	if (strp->copy_mode) {
98 		struct sk_buff *skb;
99 
100 		/* Replace anchor with an empty skb, this is a little
101 		 * dangerous but __tls_cur_msg() warns on empty skbs
102 		 * so hopefully we'll catch abuses.
103 		 */
104 		skb = alloc_skb(0, strp->sk->sk_allocation);
105 		if (!skb)
106 			return NULL;
107 
108 		swap(strp->anchor, skb);
109 		return skb;
110 	}
111 
112 	return tls_strp_msg_make_copy(strp);
113 }
114 
115 /* Force the input skb to be in copy mode. The data ownership remains
116  * with the input skb itself (meaning unpause will wipe it) but it can
117  * be modified.
118  */
119 int tls_strp_msg_cow(struct tls_sw_context_rx *ctx)
120 {
121 	struct tls_strparser *strp = &ctx->strp;
122 	struct sk_buff *skb;
123 
124 	if (strp->copy_mode)
125 		return 0;
126 
127 	skb = tls_strp_msg_make_copy(strp);
128 	if (!skb)
129 		return -ENOMEM;
130 
131 	tls_strp_anchor_free(strp);
132 	strp->anchor = skb;
133 
134 	tcp_read_done(strp->sk, strp->stm.full_len);
135 	strp->copy_mode = 1;
136 
137 	return 0;
138 }
139 
140 /* Make a clone (in the skb sense) of the input msg to keep a reference
141  * to the underlying data. The reference-holding skbs get placed on
142  * @dst.
143  */
144 int tls_strp_msg_hold(struct tls_strparser *strp, struct sk_buff_head *dst)
145 {
146 	struct skb_shared_info *shinfo = skb_shinfo(strp->anchor);
147 
148 	if (strp->copy_mode) {
149 		struct sk_buff *skb;
150 
151 		WARN_ON_ONCE(!shinfo->nr_frags);
152 
153 		/* We can't skb_clone() the anchor, it gets wiped by unpause */
154 		skb = alloc_skb(0, strp->sk->sk_allocation);
155 		if (!skb)
156 			return -ENOMEM;
157 
158 		__skb_queue_tail(dst, strp->anchor);
159 		strp->anchor = skb;
160 	} else {
161 		struct sk_buff *iter, *clone;
162 		int chunk, len, offset;
163 
164 		offset = strp->stm.offset;
165 		len = strp->stm.full_len;
166 		iter = shinfo->frag_list;
167 
168 		while (len > 0) {
169 			if (iter->len <= offset) {
170 				offset -= iter->len;
171 				goto next;
172 			}
173 
174 			chunk = iter->len - offset;
175 			offset = 0;
176 
177 			clone = skb_clone(iter, strp->sk->sk_allocation);
178 			if (!clone)
179 				return -ENOMEM;
180 			__skb_queue_tail(dst, clone);
181 
182 			len -= chunk;
183 next:
184 			iter = iter->next;
185 		}
186 	}
187 
188 	return 0;
189 }
190 
191 static void tls_strp_flush_anchor_copy(struct tls_strparser *strp)
192 {
193 	struct skb_shared_info *shinfo = skb_shinfo(strp->anchor);
194 	int i;
195 
196 	DEBUG_NET_WARN_ON_ONCE(atomic_read(&shinfo->dataref) != 1);
197 
198 	for (i = 0; i < shinfo->nr_frags; i++)
199 		__skb_frag_unref(&shinfo->frags[i], false);
200 	shinfo->nr_frags = 0;
201 	if (strp->copy_mode) {
202 		kfree_skb_list(shinfo->frag_list);
203 		shinfo->frag_list = NULL;
204 	}
205 	strp->copy_mode = 0;
206 	strp->mixed_decrypted = 0;
207 }
208 
209 static int tls_strp_copyin_frag(struct tls_strparser *strp, struct sk_buff *skb,
210 				struct sk_buff *in_skb, unsigned int offset,
211 				size_t in_len)
212 {
213 	size_t len, chunk;
214 	skb_frag_t *frag;
215 	int sz;
216 
217 	frag = &skb_shinfo(skb)->frags[skb->len / PAGE_SIZE];
218 
219 	len = in_len;
220 	/* First make sure we got the header */
221 	if (!strp->stm.full_len) {
222 		/* Assume one page is more than enough for headers */
223 		chunk =	min_t(size_t, len, PAGE_SIZE - skb_frag_size(frag));
224 		WARN_ON_ONCE(skb_copy_bits(in_skb, offset,
225 					   skb_frag_address(frag) +
226 					   skb_frag_size(frag),
227 					   chunk));
228 
229 		skb->len += chunk;
230 		skb->data_len += chunk;
231 		skb_frag_size_add(frag, chunk);
232 
233 		sz = tls_rx_msg_size(strp, skb);
234 		if (sz < 0)
235 			return sz;
236 
237 		/* We may have over-read, sz == 0 is guaranteed under-read */
238 		if (unlikely(sz && sz < skb->len)) {
239 			int over = skb->len - sz;
240 
241 			WARN_ON_ONCE(over > chunk);
242 			skb->len -= over;
243 			skb->data_len -= over;
244 			skb_frag_size_add(frag, -over);
245 
246 			chunk -= over;
247 		}
248 
249 		frag++;
250 		len -= chunk;
251 		offset += chunk;
252 
253 		strp->stm.full_len = sz;
254 		if (!strp->stm.full_len)
255 			goto read_done;
256 	}
257 
258 	/* Load up more data */
259 	while (len && strp->stm.full_len > skb->len) {
260 		chunk =	min_t(size_t, len, strp->stm.full_len - skb->len);
261 		chunk = min_t(size_t, chunk, PAGE_SIZE - skb_frag_size(frag));
262 		WARN_ON_ONCE(skb_copy_bits(in_skb, offset,
263 					   skb_frag_address(frag) +
264 					   skb_frag_size(frag),
265 					   chunk));
266 
267 		skb->len += chunk;
268 		skb->data_len += chunk;
269 		skb_frag_size_add(frag, chunk);
270 		frag++;
271 		len -= chunk;
272 		offset += chunk;
273 	}
274 
275 read_done:
276 	return in_len - len;
277 }
278 
279 static int tls_strp_copyin_skb(struct tls_strparser *strp, struct sk_buff *skb,
280 			       struct sk_buff *in_skb, unsigned int offset,
281 			       size_t in_len)
282 {
283 	struct sk_buff *nskb, *first, *last;
284 	struct skb_shared_info *shinfo;
285 	size_t chunk;
286 	int sz;
287 
288 	if (strp->stm.full_len)
289 		chunk = strp->stm.full_len - skb->len;
290 	else
291 		chunk = TLS_MAX_PAYLOAD_SIZE + PAGE_SIZE;
292 	chunk = min(chunk, in_len);
293 
294 	nskb = tls_strp_skb_copy(strp, in_skb, offset, chunk);
295 	if (!nskb)
296 		return -ENOMEM;
297 
298 	shinfo = skb_shinfo(skb);
299 	if (!shinfo->frag_list) {
300 		shinfo->frag_list = nskb;
301 		nskb->prev = nskb;
302 	} else {
303 		first = shinfo->frag_list;
304 		last = first->prev;
305 		last->next = nskb;
306 		first->prev = nskb;
307 	}
308 
309 	skb->len += chunk;
310 	skb->data_len += chunk;
311 
312 	if (!strp->stm.full_len) {
313 		sz = tls_rx_msg_size(strp, skb);
314 		if (sz < 0)
315 			return sz;
316 
317 		/* We may have over-read, sz == 0 is guaranteed under-read */
318 		if (unlikely(sz && sz < skb->len)) {
319 			int over = skb->len - sz;
320 
321 			WARN_ON_ONCE(over > chunk);
322 			skb->len -= over;
323 			skb->data_len -= over;
324 			__pskb_trim(nskb, nskb->len - over);
325 
326 			chunk -= over;
327 		}
328 
329 		strp->stm.full_len = sz;
330 	}
331 
332 	return chunk;
333 }
334 
335 static int tls_strp_copyin(read_descriptor_t *desc, struct sk_buff *in_skb,
336 			   unsigned int offset, size_t in_len)
337 {
338 	struct tls_strparser *strp = (struct tls_strparser *)desc->arg.data;
339 	struct sk_buff *skb;
340 	int ret;
341 
342 	if (strp->msg_ready)
343 		return 0;
344 
345 	skb = strp->anchor;
346 	if (!skb->len)
347 		skb_copy_decrypted(skb, in_skb);
348 	else
349 		strp->mixed_decrypted |= !!skb_cmp_decrypted(skb, in_skb);
350 
351 	if (IS_ENABLED(CONFIG_TLS_DEVICE) && strp->mixed_decrypted)
352 		ret = tls_strp_copyin_skb(strp, skb, in_skb, offset, in_len);
353 	else
354 		ret = tls_strp_copyin_frag(strp, skb, in_skb, offset, in_len);
355 	if (ret < 0) {
356 		desc->error = ret;
357 		ret = 0;
358 	}
359 
360 	if (strp->stm.full_len && strp->stm.full_len == skb->len) {
361 		desc->count = 0;
362 
363 		WRITE_ONCE(strp->msg_ready, 1);
364 		tls_rx_msg_ready(strp);
365 	}
366 
367 	return ret;
368 }
369 
370 static int tls_strp_read_copyin(struct tls_strparser *strp)
371 {
372 	read_descriptor_t desc;
373 
374 	desc.arg.data = strp;
375 	desc.error = 0;
376 	desc.count = 1; /* give more than one skb per call */
377 
378 	/* sk should be locked here, so okay to do read_sock */
379 	tcp_read_sock(strp->sk, &desc, tls_strp_copyin);
380 
381 	return desc.error;
382 }
383 
384 static int tls_strp_read_copy(struct tls_strparser *strp, bool qshort)
385 {
386 	struct skb_shared_info *shinfo;
387 	struct page *page;
388 	int need_spc, len;
389 
390 	/* If the rbuf is small or rcv window has collapsed to 0 we need
391 	 * to read the data out. Otherwise the connection will stall.
392 	 * Without pressure threshold of INT_MAX will never be ready.
393 	 */
394 	if (likely(qshort && !tcp_epollin_ready(strp->sk, INT_MAX)))
395 		return 0;
396 
397 	shinfo = skb_shinfo(strp->anchor);
398 	shinfo->frag_list = NULL;
399 
400 	/* If we don't know the length go max plus page for cipher overhead */
401 	need_spc = strp->stm.full_len ?: TLS_MAX_PAYLOAD_SIZE + PAGE_SIZE;
402 
403 	for (len = need_spc; len > 0; len -= PAGE_SIZE) {
404 		page = alloc_page(strp->sk->sk_allocation);
405 		if (!page) {
406 			tls_strp_flush_anchor_copy(strp);
407 			return -ENOMEM;
408 		}
409 
410 		skb_fill_page_desc(strp->anchor, shinfo->nr_frags++,
411 				   page, 0, 0);
412 	}
413 
414 	strp->copy_mode = 1;
415 	strp->stm.offset = 0;
416 
417 	strp->anchor->len = 0;
418 	strp->anchor->data_len = 0;
419 	strp->anchor->truesize = round_up(need_spc, PAGE_SIZE);
420 
421 	tls_strp_read_copyin(strp);
422 
423 	return 0;
424 }
425 
426 static bool tls_strp_check_queue_ok(struct tls_strparser *strp)
427 {
428 	unsigned int len = strp->stm.offset + strp->stm.full_len;
429 	struct sk_buff *first, *skb;
430 	u32 seq;
431 
432 	first = skb_shinfo(strp->anchor)->frag_list;
433 	skb = first;
434 	seq = TCP_SKB_CB(first)->seq;
435 
436 	/* Make sure there's no duplicate data in the queue,
437 	 * and the decrypted status matches.
438 	 */
439 	while (skb->len < len) {
440 		seq += skb->len;
441 		len -= skb->len;
442 		skb = skb->next;
443 
444 		if (TCP_SKB_CB(skb)->seq != seq)
445 			return false;
446 		if (skb_cmp_decrypted(first, skb))
447 			return false;
448 	}
449 
450 	return true;
451 }
452 
453 static void tls_strp_load_anchor_with_queue(struct tls_strparser *strp, int len)
454 {
455 	struct tcp_sock *tp = tcp_sk(strp->sk);
456 	struct sk_buff *first;
457 	u32 offset;
458 
459 	first = tcp_recv_skb(strp->sk, tp->copied_seq, &offset);
460 	if (WARN_ON_ONCE(!first))
461 		return;
462 
463 	/* Bestow the state onto the anchor */
464 	strp->anchor->len = offset + len;
465 	strp->anchor->data_len = offset + len;
466 	strp->anchor->truesize = offset + len;
467 
468 	skb_shinfo(strp->anchor)->frag_list = first;
469 
470 	skb_copy_header(strp->anchor, first);
471 	strp->anchor->destructor = NULL;
472 
473 	strp->stm.offset = offset;
474 }
475 
476 void tls_strp_msg_load(struct tls_strparser *strp, bool force_refresh)
477 {
478 	struct strp_msg *rxm;
479 	struct tls_msg *tlm;
480 
481 	DEBUG_NET_WARN_ON_ONCE(!strp->msg_ready);
482 	DEBUG_NET_WARN_ON_ONCE(!strp->stm.full_len);
483 
484 	if (!strp->copy_mode && force_refresh) {
485 		if (WARN_ON(tcp_inq(strp->sk) < strp->stm.full_len))
486 			return;
487 
488 		tls_strp_load_anchor_with_queue(strp, strp->stm.full_len);
489 	}
490 
491 	rxm = strp_msg(strp->anchor);
492 	rxm->full_len	= strp->stm.full_len;
493 	rxm->offset	= strp->stm.offset;
494 	tlm = tls_msg(strp->anchor);
495 	tlm->control	= strp->mark;
496 }
497 
498 /* Called with lock held on lower socket */
499 static int tls_strp_read_sock(struct tls_strparser *strp)
500 {
501 	int sz, inq;
502 
503 	inq = tcp_inq(strp->sk);
504 	if (inq < 1)
505 		return 0;
506 
507 	if (unlikely(strp->copy_mode))
508 		return tls_strp_read_copyin(strp);
509 
510 	if (inq < strp->stm.full_len)
511 		return tls_strp_read_copy(strp, true);
512 
513 	if (!strp->stm.full_len) {
514 		tls_strp_load_anchor_with_queue(strp, inq);
515 
516 		sz = tls_rx_msg_size(strp, strp->anchor);
517 		if (sz < 0) {
518 			tls_strp_abort_strp(strp, sz);
519 			return sz;
520 		}
521 
522 		strp->stm.full_len = sz;
523 
524 		if (!strp->stm.full_len || inq < strp->stm.full_len)
525 			return tls_strp_read_copy(strp, true);
526 	}
527 
528 	if (!tls_strp_check_queue_ok(strp))
529 		return tls_strp_read_copy(strp, false);
530 
531 	WRITE_ONCE(strp->msg_ready, 1);
532 	tls_rx_msg_ready(strp);
533 
534 	return 0;
535 }
536 
537 void tls_strp_check_rcv(struct tls_strparser *strp)
538 {
539 	if (unlikely(strp->stopped) || strp->msg_ready)
540 		return;
541 
542 	if (tls_strp_read_sock(strp) == -ENOMEM)
543 		queue_work(tls_strp_wq, &strp->work);
544 }
545 
546 /* Lower sock lock held */
547 void tls_strp_data_ready(struct tls_strparser *strp)
548 {
549 	/* This check is needed to synchronize with do_tls_strp_work.
550 	 * do_tls_strp_work acquires a process lock (lock_sock) whereas
551 	 * the lock held here is bh_lock_sock. The two locks can be
552 	 * held by different threads at the same time, but bh_lock_sock
553 	 * allows a thread in BH context to safely check if the process
554 	 * lock is held. In this case, if the lock is held, queue work.
555 	 */
556 	if (sock_owned_by_user_nocheck(strp->sk)) {
557 		queue_work(tls_strp_wq, &strp->work);
558 		return;
559 	}
560 
561 	tls_strp_check_rcv(strp);
562 }
563 
564 static void tls_strp_work(struct work_struct *w)
565 {
566 	struct tls_strparser *strp =
567 		container_of(w, struct tls_strparser, work);
568 
569 	lock_sock(strp->sk);
570 	tls_strp_check_rcv(strp);
571 	release_sock(strp->sk);
572 }
573 
574 void tls_strp_msg_done(struct tls_strparser *strp)
575 {
576 	WARN_ON(!strp->stm.full_len);
577 
578 	if (likely(!strp->copy_mode))
579 		tcp_read_done(strp->sk, strp->stm.full_len);
580 	else
581 		tls_strp_flush_anchor_copy(strp);
582 
583 	WRITE_ONCE(strp->msg_ready, 0);
584 	memset(&strp->stm, 0, sizeof(strp->stm));
585 
586 	tls_strp_check_rcv(strp);
587 }
588 
589 void tls_strp_stop(struct tls_strparser *strp)
590 {
591 	strp->stopped = 1;
592 }
593 
594 int tls_strp_init(struct tls_strparser *strp, struct sock *sk)
595 {
596 	memset(strp, 0, sizeof(*strp));
597 
598 	strp->sk = sk;
599 
600 	strp->anchor = alloc_skb(0, GFP_KERNEL);
601 	if (!strp->anchor)
602 		return -ENOMEM;
603 
604 	INIT_WORK(&strp->work, tls_strp_work);
605 
606 	return 0;
607 }
608 
609 /* strp must already be stopped so that tls_strp_recv will no longer be called.
610  * Note that tls_strp_done is not called with the lower socket held.
611  */
612 void tls_strp_done(struct tls_strparser *strp)
613 {
614 	WARN_ON(!strp->stopped);
615 
616 	cancel_work_sync(&strp->work);
617 	tls_strp_anchor_free(strp);
618 }
619 
620 int __init tls_strp_dev_init(void)
621 {
622 	tls_strp_wq = create_workqueue("tls-strp");
623 	if (unlikely(!tls_strp_wq))
624 		return -ENOMEM;
625 
626 	return 0;
627 }
628 
629 void tls_strp_dev_exit(void)
630 {
631 	destroy_workqueue(tls_strp_wq);
632 }
633