1c618db2aSJakub Kicinski // SPDX-License-Identifier: GPL-2.0-only 284c61fe1SJakub Kicinski /* Copyright (c) 2016 Tom Herbert <tom@herbertland.com> */ 3c618db2aSJakub Kicinski 4c618db2aSJakub Kicinski #include <linux/skbuff.h> 584c61fe1SJakub Kicinski #include <linux/workqueue.h> 684c61fe1SJakub Kicinski #include <net/strparser.h> 784c61fe1SJakub Kicinski #include <net/tcp.h> 884c61fe1SJakub Kicinski #include <net/sock.h> 984c61fe1SJakub Kicinski #include <net/tls.h> 10c618db2aSJakub Kicinski 11c618db2aSJakub Kicinski #include "tls.h" 12c618db2aSJakub Kicinski 1384c61fe1SJakub Kicinski static struct workqueue_struct *tls_strp_wq; 14d4e5db64SJakub Kicinski 1584c61fe1SJakub Kicinski static void tls_strp_abort_strp(struct tls_strparser *strp, int err) 1684c61fe1SJakub Kicinski { 1784c61fe1SJakub Kicinski if (strp->stopped) 1884c61fe1SJakub Kicinski return; 1984c61fe1SJakub Kicinski 2084c61fe1SJakub Kicinski strp->stopped = 1; 2184c61fe1SJakub Kicinski 2284c61fe1SJakub Kicinski /* Report an error on the lower socket */ 2384c61fe1SJakub Kicinski strp->sk->sk_err = -err; 2484c61fe1SJakub Kicinski sk_error_report(strp->sk); 2584c61fe1SJakub Kicinski } 2684c61fe1SJakub Kicinski 2784c61fe1SJakub Kicinski static void tls_strp_anchor_free(struct tls_strparser *strp) 2884c61fe1SJakub Kicinski { 2984c61fe1SJakub Kicinski struct skb_shared_info *shinfo = skb_shinfo(strp->anchor); 3084c61fe1SJakub Kicinski 3184c61fe1SJakub Kicinski DEBUG_NET_WARN_ON_ONCE(atomic_read(&shinfo->dataref) != 1); 3284c61fe1SJakub Kicinski shinfo->frag_list = NULL; 3384c61fe1SJakub Kicinski consume_skb(strp->anchor); 3484c61fe1SJakub Kicinski strp->anchor = NULL; 3584c61fe1SJakub Kicinski } 3684c61fe1SJakub Kicinski 3784c61fe1SJakub Kicinski /* Create a new skb with the contents of input copied to its page frags */ 3884c61fe1SJakub Kicinski static struct sk_buff *tls_strp_msg_make_copy(struct tls_strparser *strp) 3984c61fe1SJakub Kicinski { 4084c61fe1SJakub Kicinski struct strp_msg *rxm; 4184c61fe1SJakub Kicinski struct sk_buff *skb; 4284c61fe1SJakub Kicinski int i, err, offset; 4384c61fe1SJakub Kicinski 44d800a7b3SJakub Kicinski skb = alloc_skb_with_frags(0, strp->stm.full_len, TLS_PAGE_ORDER, 4584c61fe1SJakub Kicinski &err, strp->sk->sk_allocation); 4684c61fe1SJakub Kicinski if (!skb) 4784c61fe1SJakub Kicinski return NULL; 4884c61fe1SJakub Kicinski 4984c61fe1SJakub Kicinski offset = strp->stm.offset; 5084c61fe1SJakub Kicinski for (i = 0; i < skb_shinfo(skb)->nr_frags; i++) { 5184c61fe1SJakub Kicinski skb_frag_t *frag = &skb_shinfo(skb)->frags[i]; 5284c61fe1SJakub Kicinski 5384c61fe1SJakub Kicinski WARN_ON_ONCE(skb_copy_bits(strp->anchor, offset, 5484c61fe1SJakub Kicinski skb_frag_address(frag), 5584c61fe1SJakub Kicinski skb_frag_size(frag))); 5684c61fe1SJakub Kicinski offset += skb_frag_size(frag); 5784c61fe1SJakub Kicinski } 5884c61fe1SJakub Kicinski 59210620aeSJakub Kicinski skb->len = strp->stm.full_len; 60210620aeSJakub Kicinski skb->data_len = strp->stm.full_len; 6184c61fe1SJakub Kicinski skb_copy_header(skb, strp->anchor); 6284c61fe1SJakub Kicinski rxm = strp_msg(skb); 6384c61fe1SJakub Kicinski rxm->offset = 0; 64d4e5db64SJakub Kicinski return skb; 65d4e5db64SJakub Kicinski } 66d4e5db64SJakub Kicinski 6784c61fe1SJakub Kicinski /* Steal the input skb, input msg is invalid after calling this function */ 6884c61fe1SJakub Kicinski struct sk_buff *tls_strp_msg_detach(struct tls_sw_context_rx *ctx) 6984c61fe1SJakub Kicinski { 7084c61fe1SJakub Kicinski struct tls_strparser *strp = &ctx->strp; 7184c61fe1SJakub Kicinski 7284c61fe1SJakub Kicinski #ifdef CONFIG_TLS_DEVICE 7384c61fe1SJakub Kicinski DEBUG_NET_WARN_ON_ONCE(!strp->anchor->decrypted); 7484c61fe1SJakub Kicinski #else 7584c61fe1SJakub Kicinski /* This function turns an input into an output, 7684c61fe1SJakub Kicinski * that can only happen if we have offload. 7784c61fe1SJakub Kicinski */ 7884c61fe1SJakub Kicinski WARN_ON(1); 7984c61fe1SJakub Kicinski #endif 8084c61fe1SJakub Kicinski 8184c61fe1SJakub Kicinski if (strp->copy_mode) { 8284c61fe1SJakub Kicinski struct sk_buff *skb; 8384c61fe1SJakub Kicinski 8484c61fe1SJakub Kicinski /* Replace anchor with an empty skb, this is a little 8584c61fe1SJakub Kicinski * dangerous but __tls_cur_msg() warns on empty skbs 8684c61fe1SJakub Kicinski * so hopefully we'll catch abuses. 8784c61fe1SJakub Kicinski */ 8884c61fe1SJakub Kicinski skb = alloc_skb(0, strp->sk->sk_allocation); 8984c61fe1SJakub Kicinski if (!skb) 9084c61fe1SJakub Kicinski return NULL; 9184c61fe1SJakub Kicinski 9284c61fe1SJakub Kicinski swap(strp->anchor, skb); 9384c61fe1SJakub Kicinski return skb; 9484c61fe1SJakub Kicinski } 9584c61fe1SJakub Kicinski 9684c61fe1SJakub Kicinski return tls_strp_msg_make_copy(strp); 9784c61fe1SJakub Kicinski } 9884c61fe1SJakub Kicinski 9984c61fe1SJakub Kicinski /* Force the input skb to be in copy mode. The data ownership remains 10084c61fe1SJakub Kicinski * with the input skb itself (meaning unpause will wipe it) but it can 10184c61fe1SJakub Kicinski * be modified. 10284c61fe1SJakub Kicinski */ 1038b3c59a7SJakub Kicinski int tls_strp_msg_cow(struct tls_sw_context_rx *ctx) 1048b3c59a7SJakub Kicinski { 10584c61fe1SJakub Kicinski struct tls_strparser *strp = &ctx->strp; 10684c61fe1SJakub Kicinski struct sk_buff *skb; 1078b3c59a7SJakub Kicinski 10884c61fe1SJakub Kicinski if (strp->copy_mode) 10984c61fe1SJakub Kicinski return 0; 11084c61fe1SJakub Kicinski 11184c61fe1SJakub Kicinski skb = tls_strp_msg_make_copy(strp); 11284c61fe1SJakub Kicinski if (!skb) 11384c61fe1SJakub Kicinski return -ENOMEM; 11484c61fe1SJakub Kicinski 11584c61fe1SJakub Kicinski tls_strp_anchor_free(strp); 11684c61fe1SJakub Kicinski strp->anchor = skb; 11784c61fe1SJakub Kicinski 11884c61fe1SJakub Kicinski tcp_read_done(strp->sk, strp->stm.full_len); 11984c61fe1SJakub Kicinski strp->copy_mode = 1; 12084c61fe1SJakub Kicinski 1218b3c59a7SJakub Kicinski return 0; 1228b3c59a7SJakub Kicinski } 1238b3c59a7SJakub Kicinski 12484c61fe1SJakub Kicinski /* Make a clone (in the skb sense) of the input msg to keep a reference 12584c61fe1SJakub Kicinski * to the underlying data. The reference-holding skbs get placed on 12684c61fe1SJakub Kicinski * @dst. 12784c61fe1SJakub Kicinski */ 12884c61fe1SJakub Kicinski int tls_strp_msg_hold(struct tls_strparser *strp, struct sk_buff_head *dst) 129c618db2aSJakub Kicinski { 13084c61fe1SJakub Kicinski struct skb_shared_info *shinfo = skb_shinfo(strp->anchor); 131c618db2aSJakub Kicinski 13284c61fe1SJakub Kicinski if (strp->copy_mode) { 13384c61fe1SJakub Kicinski struct sk_buff *skb; 13484c61fe1SJakub Kicinski 13584c61fe1SJakub Kicinski WARN_ON_ONCE(!shinfo->nr_frags); 13684c61fe1SJakub Kicinski 13784c61fe1SJakub Kicinski /* We can't skb_clone() the anchor, it gets wiped by unpause */ 13884c61fe1SJakub Kicinski skb = alloc_skb(0, strp->sk->sk_allocation); 13984c61fe1SJakub Kicinski if (!skb) 14084c61fe1SJakub Kicinski return -ENOMEM; 14184c61fe1SJakub Kicinski 14284c61fe1SJakub Kicinski __skb_queue_tail(dst, strp->anchor); 14384c61fe1SJakub Kicinski strp->anchor = skb; 14484c61fe1SJakub Kicinski } else { 14584c61fe1SJakub Kicinski struct sk_buff *iter, *clone; 14684c61fe1SJakub Kicinski int chunk, len, offset; 14784c61fe1SJakub Kicinski 14884c61fe1SJakub Kicinski offset = strp->stm.offset; 14984c61fe1SJakub Kicinski len = strp->stm.full_len; 15084c61fe1SJakub Kicinski iter = shinfo->frag_list; 15184c61fe1SJakub Kicinski 15284c61fe1SJakub Kicinski while (len > 0) { 15384c61fe1SJakub Kicinski if (iter->len <= offset) { 15484c61fe1SJakub Kicinski offset -= iter->len; 15584c61fe1SJakub Kicinski goto next; 15684c61fe1SJakub Kicinski } 15784c61fe1SJakub Kicinski 15884c61fe1SJakub Kicinski chunk = iter->len - offset; 15984c61fe1SJakub Kicinski offset = 0; 16084c61fe1SJakub Kicinski 16184c61fe1SJakub Kicinski clone = skb_clone(iter, strp->sk->sk_allocation); 162c618db2aSJakub Kicinski if (!clone) 163c618db2aSJakub Kicinski return -ENOMEM; 164c618db2aSJakub Kicinski __skb_queue_tail(dst, clone); 16584c61fe1SJakub Kicinski 16684c61fe1SJakub Kicinski len -= chunk; 16784c61fe1SJakub Kicinski next: 16884c61fe1SJakub Kicinski iter = iter->next; 16984c61fe1SJakub Kicinski } 17084c61fe1SJakub Kicinski } 17184c61fe1SJakub Kicinski 172c618db2aSJakub Kicinski return 0; 173c618db2aSJakub Kicinski } 17484c61fe1SJakub Kicinski 17584c61fe1SJakub Kicinski static void tls_strp_flush_anchor_copy(struct tls_strparser *strp) 17684c61fe1SJakub Kicinski { 17784c61fe1SJakub Kicinski struct skb_shared_info *shinfo = skb_shinfo(strp->anchor); 17884c61fe1SJakub Kicinski int i; 17984c61fe1SJakub Kicinski 18084c61fe1SJakub Kicinski DEBUG_NET_WARN_ON_ONCE(atomic_read(&shinfo->dataref) != 1); 18184c61fe1SJakub Kicinski 18284c61fe1SJakub Kicinski for (i = 0; i < shinfo->nr_frags; i++) 18384c61fe1SJakub Kicinski __skb_frag_unref(&shinfo->frags[i], false); 18484c61fe1SJakub Kicinski shinfo->nr_frags = 0; 18584c61fe1SJakub Kicinski strp->copy_mode = 0; 18684c61fe1SJakub Kicinski } 18784c61fe1SJakub Kicinski 18884c61fe1SJakub Kicinski static int tls_strp_copyin(read_descriptor_t *desc, struct sk_buff *in_skb, 18984c61fe1SJakub Kicinski unsigned int offset, size_t in_len) 19084c61fe1SJakub Kicinski { 19184c61fe1SJakub Kicinski struct tls_strparser *strp = (struct tls_strparser *)desc->arg.data; 19284c61fe1SJakub Kicinski struct sk_buff *skb; 19384c61fe1SJakub Kicinski skb_frag_t *frag; 1948fd1e151SYang Li size_t len, chunk; 1958fd1e151SYang Li int sz; 19684c61fe1SJakub Kicinski 19784c61fe1SJakub Kicinski if (strp->msg_ready) 19884c61fe1SJakub Kicinski return 0; 19984c61fe1SJakub Kicinski 20084c61fe1SJakub Kicinski skb = strp->anchor; 20184c61fe1SJakub Kicinski frag = &skb_shinfo(skb)->frags[skb->len / PAGE_SIZE]; 20284c61fe1SJakub Kicinski 20384c61fe1SJakub Kicinski len = in_len; 20484c61fe1SJakub Kicinski /* First make sure we got the header */ 20584c61fe1SJakub Kicinski if (!strp->stm.full_len) { 20684c61fe1SJakub Kicinski /* Assume one page is more than enough for headers */ 20784c61fe1SJakub Kicinski chunk = min_t(size_t, len, PAGE_SIZE - skb_frag_size(frag)); 20884c61fe1SJakub Kicinski WARN_ON_ONCE(skb_copy_bits(in_skb, offset, 20984c61fe1SJakub Kicinski skb_frag_address(frag) + 21084c61fe1SJakub Kicinski skb_frag_size(frag), 21184c61fe1SJakub Kicinski chunk)); 21284c61fe1SJakub Kicinski 213*8b0c0dc9SJakub Kicinski skb->len += chunk; 214*8b0c0dc9SJakub Kicinski skb->data_len += chunk; 215*8b0c0dc9SJakub Kicinski skb_frag_size_add(frag, chunk); 216*8b0c0dc9SJakub Kicinski 217*8b0c0dc9SJakub Kicinski sz = tls_rx_msg_size(strp, skb); 21884c61fe1SJakub Kicinski if (sz < 0) { 21984c61fe1SJakub Kicinski desc->error = sz; 22084c61fe1SJakub Kicinski return 0; 22184c61fe1SJakub Kicinski } 22284c61fe1SJakub Kicinski 22384c61fe1SJakub Kicinski /* We may have over-read, sz == 0 is guaranteed under-read */ 224*8b0c0dc9SJakub Kicinski if (unlikely(sz && sz < skb->len)) { 225*8b0c0dc9SJakub Kicinski int over = skb->len - sz; 22684c61fe1SJakub Kicinski 227*8b0c0dc9SJakub Kicinski WARN_ON_ONCE(over > chunk); 228*8b0c0dc9SJakub Kicinski skb->len -= over; 229*8b0c0dc9SJakub Kicinski skb->data_len -= over; 230*8b0c0dc9SJakub Kicinski skb_frag_size_add(frag, -over); 231*8b0c0dc9SJakub Kicinski 232*8b0c0dc9SJakub Kicinski chunk -= over; 233*8b0c0dc9SJakub Kicinski } 234*8b0c0dc9SJakub Kicinski 23584c61fe1SJakub Kicinski frag++; 23684c61fe1SJakub Kicinski len -= chunk; 23784c61fe1SJakub Kicinski offset += chunk; 23884c61fe1SJakub Kicinski 23984c61fe1SJakub Kicinski strp->stm.full_len = sz; 24084c61fe1SJakub Kicinski if (!strp->stm.full_len) 24184c61fe1SJakub Kicinski goto read_done; 24284c61fe1SJakub Kicinski } 24384c61fe1SJakub Kicinski 24484c61fe1SJakub Kicinski /* Load up more data */ 24584c61fe1SJakub Kicinski while (len && strp->stm.full_len > skb->len) { 24684c61fe1SJakub Kicinski chunk = min_t(size_t, len, strp->stm.full_len - skb->len); 24784c61fe1SJakub Kicinski chunk = min_t(size_t, chunk, PAGE_SIZE - skb_frag_size(frag)); 24884c61fe1SJakub Kicinski WARN_ON_ONCE(skb_copy_bits(in_skb, offset, 24984c61fe1SJakub Kicinski skb_frag_address(frag) + 25084c61fe1SJakub Kicinski skb_frag_size(frag), 25184c61fe1SJakub Kicinski chunk)); 25284c61fe1SJakub Kicinski 25384c61fe1SJakub Kicinski skb->len += chunk; 25484c61fe1SJakub Kicinski skb->data_len += chunk; 25584c61fe1SJakub Kicinski skb_frag_size_add(frag, chunk); 25684c61fe1SJakub Kicinski frag++; 25784c61fe1SJakub Kicinski len -= chunk; 25884c61fe1SJakub Kicinski offset += chunk; 25984c61fe1SJakub Kicinski } 26084c61fe1SJakub Kicinski 26184c61fe1SJakub Kicinski if (strp->stm.full_len == skb->len) { 26284c61fe1SJakub Kicinski desc->count = 0; 26384c61fe1SJakub Kicinski 26484c61fe1SJakub Kicinski strp->msg_ready = 1; 26584c61fe1SJakub Kicinski tls_rx_msg_ready(strp); 26684c61fe1SJakub Kicinski } 26784c61fe1SJakub Kicinski 26884c61fe1SJakub Kicinski read_done: 26984c61fe1SJakub Kicinski return in_len - len; 27084c61fe1SJakub Kicinski } 27184c61fe1SJakub Kicinski 27284c61fe1SJakub Kicinski static int tls_strp_read_copyin(struct tls_strparser *strp) 27384c61fe1SJakub Kicinski { 27484c61fe1SJakub Kicinski struct socket *sock = strp->sk->sk_socket; 27584c61fe1SJakub Kicinski read_descriptor_t desc; 27684c61fe1SJakub Kicinski 27784c61fe1SJakub Kicinski desc.arg.data = strp; 27884c61fe1SJakub Kicinski desc.error = 0; 27984c61fe1SJakub Kicinski desc.count = 1; /* give more than one skb per call */ 28084c61fe1SJakub Kicinski 28184c61fe1SJakub Kicinski /* sk should be locked here, so okay to do read_sock */ 28284c61fe1SJakub Kicinski sock->ops->read_sock(strp->sk, &desc, tls_strp_copyin); 28384c61fe1SJakub Kicinski 28484c61fe1SJakub Kicinski return desc.error; 28584c61fe1SJakub Kicinski } 28684c61fe1SJakub Kicinski 2870d87bbd3SJakub Kicinski static int tls_strp_read_copy(struct tls_strparser *strp, bool qshort) 28884c61fe1SJakub Kicinski { 28984c61fe1SJakub Kicinski struct skb_shared_info *shinfo; 29084c61fe1SJakub Kicinski struct page *page; 29184c61fe1SJakub Kicinski int need_spc, len; 29284c61fe1SJakub Kicinski 29384c61fe1SJakub Kicinski /* If the rbuf is small or rcv window has collapsed to 0 we need 29484c61fe1SJakub Kicinski * to read the data out. Otherwise the connection will stall. 29584c61fe1SJakub Kicinski * Without pressure threshold of INT_MAX will never be ready. 29684c61fe1SJakub Kicinski */ 2970d87bbd3SJakub Kicinski if (likely(qshort && !tcp_epollin_ready(strp->sk, INT_MAX))) 29884c61fe1SJakub Kicinski return 0; 29984c61fe1SJakub Kicinski 30084c61fe1SJakub Kicinski shinfo = skb_shinfo(strp->anchor); 30184c61fe1SJakub Kicinski shinfo->frag_list = NULL; 30284c61fe1SJakub Kicinski 30384c61fe1SJakub Kicinski /* If we don't know the length go max plus page for cipher overhead */ 30484c61fe1SJakub Kicinski need_spc = strp->stm.full_len ?: TLS_MAX_PAYLOAD_SIZE + PAGE_SIZE; 30584c61fe1SJakub Kicinski 30684c61fe1SJakub Kicinski for (len = need_spc; len > 0; len -= PAGE_SIZE) { 30784c61fe1SJakub Kicinski page = alloc_page(strp->sk->sk_allocation); 30884c61fe1SJakub Kicinski if (!page) { 30984c61fe1SJakub Kicinski tls_strp_flush_anchor_copy(strp); 31084c61fe1SJakub Kicinski return -ENOMEM; 31184c61fe1SJakub Kicinski } 31284c61fe1SJakub Kicinski 31384c61fe1SJakub Kicinski skb_fill_page_desc(strp->anchor, shinfo->nr_frags++, 31484c61fe1SJakub Kicinski page, 0, 0); 31584c61fe1SJakub Kicinski } 31684c61fe1SJakub Kicinski 31784c61fe1SJakub Kicinski strp->copy_mode = 1; 31884c61fe1SJakub Kicinski strp->stm.offset = 0; 31984c61fe1SJakub Kicinski 32084c61fe1SJakub Kicinski strp->anchor->len = 0; 32184c61fe1SJakub Kicinski strp->anchor->data_len = 0; 32284c61fe1SJakub Kicinski strp->anchor->truesize = round_up(need_spc, PAGE_SIZE); 32384c61fe1SJakub Kicinski 32484c61fe1SJakub Kicinski tls_strp_read_copyin(strp); 32584c61fe1SJakub Kicinski 32684c61fe1SJakub Kicinski return 0; 32784c61fe1SJakub Kicinski } 32884c61fe1SJakub Kicinski 32914c4be92SJakub Kicinski static bool tls_strp_check_queue_ok(struct tls_strparser *strp) 3300d87bbd3SJakub Kicinski { 3310d87bbd3SJakub Kicinski unsigned int len = strp->stm.offset + strp->stm.full_len; 33214c4be92SJakub Kicinski struct sk_buff *first, *skb; 3330d87bbd3SJakub Kicinski u32 seq; 3340d87bbd3SJakub Kicinski 33514c4be92SJakub Kicinski first = skb_shinfo(strp->anchor)->frag_list; 33614c4be92SJakub Kicinski skb = first; 33714c4be92SJakub Kicinski seq = TCP_SKB_CB(first)->seq; 3380d87bbd3SJakub Kicinski 33914c4be92SJakub Kicinski /* Make sure there's no duplicate data in the queue, 34014c4be92SJakub Kicinski * and the decrypted status matches. 34114c4be92SJakub Kicinski */ 3420d87bbd3SJakub Kicinski while (skb->len < len) { 3430d87bbd3SJakub Kicinski seq += skb->len; 3440d87bbd3SJakub Kicinski len -= skb->len; 3450d87bbd3SJakub Kicinski skb = skb->next; 3460d87bbd3SJakub Kicinski 3470d87bbd3SJakub Kicinski if (TCP_SKB_CB(skb)->seq != seq) 3480d87bbd3SJakub Kicinski return false; 34914c4be92SJakub Kicinski if (skb_cmp_decrypted(first, skb)) 35014c4be92SJakub Kicinski return false; 3510d87bbd3SJakub Kicinski } 3520d87bbd3SJakub Kicinski 3530d87bbd3SJakub Kicinski return true; 3540d87bbd3SJakub Kicinski } 3550d87bbd3SJakub Kicinski 35684c61fe1SJakub Kicinski static void tls_strp_load_anchor_with_queue(struct tls_strparser *strp, int len) 35784c61fe1SJakub Kicinski { 35884c61fe1SJakub Kicinski struct tcp_sock *tp = tcp_sk(strp->sk); 35984c61fe1SJakub Kicinski struct sk_buff *first; 36084c61fe1SJakub Kicinski u32 offset; 36184c61fe1SJakub Kicinski 36284c61fe1SJakub Kicinski first = tcp_recv_skb(strp->sk, tp->copied_seq, &offset); 36384c61fe1SJakub Kicinski if (WARN_ON_ONCE(!first)) 36484c61fe1SJakub Kicinski return; 36584c61fe1SJakub Kicinski 36684c61fe1SJakub Kicinski /* Bestow the state onto the anchor */ 36784c61fe1SJakub Kicinski strp->anchor->len = offset + len; 36884c61fe1SJakub Kicinski strp->anchor->data_len = offset + len; 36984c61fe1SJakub Kicinski strp->anchor->truesize = offset + len; 37084c61fe1SJakub Kicinski 37184c61fe1SJakub Kicinski skb_shinfo(strp->anchor)->frag_list = first; 37284c61fe1SJakub Kicinski 37384c61fe1SJakub Kicinski skb_copy_header(strp->anchor, first); 37484c61fe1SJakub Kicinski strp->anchor->destructor = NULL; 37584c61fe1SJakub Kicinski 37684c61fe1SJakub Kicinski strp->stm.offset = offset; 37784c61fe1SJakub Kicinski } 37884c61fe1SJakub Kicinski 37984c61fe1SJakub Kicinski void tls_strp_msg_load(struct tls_strparser *strp, bool force_refresh) 38084c61fe1SJakub Kicinski { 38184c61fe1SJakub Kicinski struct strp_msg *rxm; 38284c61fe1SJakub Kicinski struct tls_msg *tlm; 38384c61fe1SJakub Kicinski 38484c61fe1SJakub Kicinski DEBUG_NET_WARN_ON_ONCE(!strp->msg_ready); 38584c61fe1SJakub Kicinski DEBUG_NET_WARN_ON_ONCE(!strp->stm.full_len); 38684c61fe1SJakub Kicinski 38784c61fe1SJakub Kicinski if (!strp->copy_mode && force_refresh) { 38884c61fe1SJakub Kicinski if (WARN_ON(tcp_inq(strp->sk) < strp->stm.full_len)) 38984c61fe1SJakub Kicinski return; 39084c61fe1SJakub Kicinski 39184c61fe1SJakub Kicinski tls_strp_load_anchor_with_queue(strp, strp->stm.full_len); 39284c61fe1SJakub Kicinski } 39384c61fe1SJakub Kicinski 39484c61fe1SJakub Kicinski rxm = strp_msg(strp->anchor); 39584c61fe1SJakub Kicinski rxm->full_len = strp->stm.full_len; 39684c61fe1SJakub Kicinski rxm->offset = strp->stm.offset; 39784c61fe1SJakub Kicinski tlm = tls_msg(strp->anchor); 39884c61fe1SJakub Kicinski tlm->control = strp->mark; 39984c61fe1SJakub Kicinski } 40084c61fe1SJakub Kicinski 40184c61fe1SJakub Kicinski /* Called with lock held on lower socket */ 40284c61fe1SJakub Kicinski static int tls_strp_read_sock(struct tls_strparser *strp) 40384c61fe1SJakub Kicinski { 40484c61fe1SJakub Kicinski int sz, inq; 40584c61fe1SJakub Kicinski 40684c61fe1SJakub Kicinski inq = tcp_inq(strp->sk); 40784c61fe1SJakub Kicinski if (inq < 1) 40884c61fe1SJakub Kicinski return 0; 40984c61fe1SJakub Kicinski 41084c61fe1SJakub Kicinski if (unlikely(strp->copy_mode)) 41184c61fe1SJakub Kicinski return tls_strp_read_copyin(strp); 41284c61fe1SJakub Kicinski 41384c61fe1SJakub Kicinski if (inq < strp->stm.full_len) 4140d87bbd3SJakub Kicinski return tls_strp_read_copy(strp, true); 41584c61fe1SJakub Kicinski 41684c61fe1SJakub Kicinski if (!strp->stm.full_len) { 41784c61fe1SJakub Kicinski tls_strp_load_anchor_with_queue(strp, inq); 41884c61fe1SJakub Kicinski 41984c61fe1SJakub Kicinski sz = tls_rx_msg_size(strp, strp->anchor); 42084c61fe1SJakub Kicinski if (sz < 0) { 42184c61fe1SJakub Kicinski tls_strp_abort_strp(strp, sz); 42284c61fe1SJakub Kicinski return sz; 42384c61fe1SJakub Kicinski } 42484c61fe1SJakub Kicinski 42584c61fe1SJakub Kicinski strp->stm.full_len = sz; 42684c61fe1SJakub Kicinski 42784c61fe1SJakub Kicinski if (!strp->stm.full_len || inq < strp->stm.full_len) 4280d87bbd3SJakub Kicinski return tls_strp_read_copy(strp, true); 42984c61fe1SJakub Kicinski } 43084c61fe1SJakub Kicinski 43114c4be92SJakub Kicinski if (!tls_strp_check_queue_ok(strp)) 4320d87bbd3SJakub Kicinski return tls_strp_read_copy(strp, false); 4330d87bbd3SJakub Kicinski 43484c61fe1SJakub Kicinski strp->msg_ready = 1; 43584c61fe1SJakub Kicinski tls_rx_msg_ready(strp); 43684c61fe1SJakub Kicinski 43784c61fe1SJakub Kicinski return 0; 43884c61fe1SJakub Kicinski } 43984c61fe1SJakub Kicinski 44084c61fe1SJakub Kicinski void tls_strp_check_rcv(struct tls_strparser *strp) 44184c61fe1SJakub Kicinski { 44284c61fe1SJakub Kicinski if (unlikely(strp->stopped) || strp->msg_ready) 44384c61fe1SJakub Kicinski return; 44484c61fe1SJakub Kicinski 44584c61fe1SJakub Kicinski if (tls_strp_read_sock(strp) == -ENOMEM) 44684c61fe1SJakub Kicinski queue_work(tls_strp_wq, &strp->work); 44784c61fe1SJakub Kicinski } 44884c61fe1SJakub Kicinski 44984c61fe1SJakub Kicinski /* Lower sock lock held */ 45084c61fe1SJakub Kicinski void tls_strp_data_ready(struct tls_strparser *strp) 45184c61fe1SJakub Kicinski { 45284c61fe1SJakub Kicinski /* This check is needed to synchronize with do_tls_strp_work. 45384c61fe1SJakub Kicinski * do_tls_strp_work acquires a process lock (lock_sock) whereas 45484c61fe1SJakub Kicinski * the lock held here is bh_lock_sock. The two locks can be 45584c61fe1SJakub Kicinski * held by different threads at the same time, but bh_lock_sock 45684c61fe1SJakub Kicinski * allows a thread in BH context to safely check if the process 45784c61fe1SJakub Kicinski * lock is held. In this case, if the lock is held, queue work. 45884c61fe1SJakub Kicinski */ 45984c61fe1SJakub Kicinski if (sock_owned_by_user_nocheck(strp->sk)) { 46084c61fe1SJakub Kicinski queue_work(tls_strp_wq, &strp->work); 46184c61fe1SJakub Kicinski return; 46284c61fe1SJakub Kicinski } 46384c61fe1SJakub Kicinski 46484c61fe1SJakub Kicinski tls_strp_check_rcv(strp); 46584c61fe1SJakub Kicinski } 46684c61fe1SJakub Kicinski 46784c61fe1SJakub Kicinski static void tls_strp_work(struct work_struct *w) 46884c61fe1SJakub Kicinski { 46984c61fe1SJakub Kicinski struct tls_strparser *strp = 47084c61fe1SJakub Kicinski container_of(w, struct tls_strparser, work); 47184c61fe1SJakub Kicinski 47284c61fe1SJakub Kicinski lock_sock(strp->sk); 47384c61fe1SJakub Kicinski tls_strp_check_rcv(strp); 47484c61fe1SJakub Kicinski release_sock(strp->sk); 47584c61fe1SJakub Kicinski } 47684c61fe1SJakub Kicinski 47784c61fe1SJakub Kicinski void tls_strp_msg_done(struct tls_strparser *strp) 47884c61fe1SJakub Kicinski { 47984c61fe1SJakub Kicinski WARN_ON(!strp->stm.full_len); 48084c61fe1SJakub Kicinski 48184c61fe1SJakub Kicinski if (likely(!strp->copy_mode)) 48284c61fe1SJakub Kicinski tcp_read_done(strp->sk, strp->stm.full_len); 48384c61fe1SJakub Kicinski else 48484c61fe1SJakub Kicinski tls_strp_flush_anchor_copy(strp); 48584c61fe1SJakub Kicinski 48684c61fe1SJakub Kicinski strp->msg_ready = 0; 48784c61fe1SJakub Kicinski memset(&strp->stm, 0, sizeof(strp->stm)); 48884c61fe1SJakub Kicinski 48984c61fe1SJakub Kicinski tls_strp_check_rcv(strp); 49084c61fe1SJakub Kicinski } 49184c61fe1SJakub Kicinski 49284c61fe1SJakub Kicinski void tls_strp_stop(struct tls_strparser *strp) 49384c61fe1SJakub Kicinski { 49484c61fe1SJakub Kicinski strp->stopped = 1; 49584c61fe1SJakub Kicinski } 49684c61fe1SJakub Kicinski 49784c61fe1SJakub Kicinski int tls_strp_init(struct tls_strparser *strp, struct sock *sk) 49884c61fe1SJakub Kicinski { 49984c61fe1SJakub Kicinski memset(strp, 0, sizeof(*strp)); 50084c61fe1SJakub Kicinski 50184c61fe1SJakub Kicinski strp->sk = sk; 50284c61fe1SJakub Kicinski 50384c61fe1SJakub Kicinski strp->anchor = alloc_skb(0, GFP_KERNEL); 50484c61fe1SJakub Kicinski if (!strp->anchor) 50584c61fe1SJakub Kicinski return -ENOMEM; 50684c61fe1SJakub Kicinski 50784c61fe1SJakub Kicinski INIT_WORK(&strp->work, tls_strp_work); 50884c61fe1SJakub Kicinski 50984c61fe1SJakub Kicinski return 0; 51084c61fe1SJakub Kicinski } 51184c61fe1SJakub Kicinski 51284c61fe1SJakub Kicinski /* strp must already be stopped so that tls_strp_recv will no longer be called. 51384c61fe1SJakub Kicinski * Note that tls_strp_done is not called with the lower socket held. 51484c61fe1SJakub Kicinski */ 51584c61fe1SJakub Kicinski void tls_strp_done(struct tls_strparser *strp) 51684c61fe1SJakub Kicinski { 51784c61fe1SJakub Kicinski WARN_ON(!strp->stopped); 51884c61fe1SJakub Kicinski 51984c61fe1SJakub Kicinski cancel_work_sync(&strp->work); 52084c61fe1SJakub Kicinski tls_strp_anchor_free(strp); 52184c61fe1SJakub Kicinski } 52284c61fe1SJakub Kicinski 52384c61fe1SJakub Kicinski int __init tls_strp_dev_init(void) 52484c61fe1SJakub Kicinski { 525d11ef9ccSJakub Kicinski tls_strp_wq = create_workqueue("tls-strp"); 52684c61fe1SJakub Kicinski if (unlikely(!tls_strp_wq)) 52784c61fe1SJakub Kicinski return -ENOMEM; 52884c61fe1SJakub Kicinski 52984c61fe1SJakub Kicinski return 0; 53084c61fe1SJakub Kicinski } 53184c61fe1SJakub Kicinski 53284c61fe1SJakub Kicinski void tls_strp_dev_exit(void) 53384c61fe1SJakub Kicinski { 53484c61fe1SJakub Kicinski destroy_workqueue(tls_strp_wq); 53584c61fe1SJakub Kicinski } 536