1c618db2aSJakub Kicinski // SPDX-License-Identifier: GPL-2.0-only 284c61fe1SJakub Kicinski /* Copyright (c) 2016 Tom Herbert <tom@herbertland.com> */ 3c618db2aSJakub Kicinski 4c618db2aSJakub Kicinski #include <linux/skbuff.h> 584c61fe1SJakub Kicinski #include <linux/workqueue.h> 684c61fe1SJakub Kicinski #include <net/strparser.h> 784c61fe1SJakub Kicinski #include <net/tcp.h> 884c61fe1SJakub Kicinski #include <net/sock.h> 984c61fe1SJakub Kicinski #include <net/tls.h> 10c618db2aSJakub Kicinski 11c618db2aSJakub Kicinski #include "tls.h" 12c618db2aSJakub Kicinski 1384c61fe1SJakub Kicinski static struct workqueue_struct *tls_strp_wq; 14d4e5db64SJakub Kicinski 1584c61fe1SJakub Kicinski static void tls_strp_abort_strp(struct tls_strparser *strp, int err) 1684c61fe1SJakub Kicinski { 1784c61fe1SJakub Kicinski if (strp->stopped) 1884c61fe1SJakub Kicinski return; 1984c61fe1SJakub Kicinski 2084c61fe1SJakub Kicinski strp->stopped = 1; 2184c61fe1SJakub Kicinski 2284c61fe1SJakub Kicinski /* Report an error on the lower socket */ 2384c61fe1SJakub Kicinski strp->sk->sk_err = -err; 2484c61fe1SJakub Kicinski sk_error_report(strp->sk); 2584c61fe1SJakub Kicinski } 2684c61fe1SJakub Kicinski 2784c61fe1SJakub Kicinski static void tls_strp_anchor_free(struct tls_strparser *strp) 2884c61fe1SJakub Kicinski { 2984c61fe1SJakub Kicinski struct skb_shared_info *shinfo = skb_shinfo(strp->anchor); 3084c61fe1SJakub Kicinski 3184c61fe1SJakub Kicinski DEBUG_NET_WARN_ON_ONCE(atomic_read(&shinfo->dataref) != 1); 3284c61fe1SJakub Kicinski shinfo->frag_list = NULL; 3384c61fe1SJakub Kicinski consume_skb(strp->anchor); 3484c61fe1SJakub Kicinski strp->anchor = NULL; 3584c61fe1SJakub Kicinski } 3684c61fe1SJakub Kicinski 37*c1c607b1SJakub Kicinski static struct sk_buff * 38*c1c607b1SJakub Kicinski tls_strp_skb_copy(struct tls_strparser *strp, struct sk_buff *in_skb, 39*c1c607b1SJakub Kicinski int offset, int len) 4084c61fe1SJakub Kicinski { 4184c61fe1SJakub Kicinski struct sk_buff *skb; 42*c1c607b1SJakub Kicinski int i, err; 4384c61fe1SJakub Kicinski 44*c1c607b1SJakub Kicinski skb = alloc_skb_with_frags(0, len, TLS_PAGE_ORDER, 4584c61fe1SJakub Kicinski &err, strp->sk->sk_allocation); 4684c61fe1SJakub Kicinski if (!skb) 4784c61fe1SJakub Kicinski return NULL; 4884c61fe1SJakub Kicinski 4984c61fe1SJakub Kicinski for (i = 0; i < skb_shinfo(skb)->nr_frags; i++) { 5084c61fe1SJakub Kicinski skb_frag_t *frag = &skb_shinfo(skb)->frags[i]; 5184c61fe1SJakub Kicinski 52*c1c607b1SJakub Kicinski WARN_ON_ONCE(skb_copy_bits(in_skb, offset, 5384c61fe1SJakub Kicinski skb_frag_address(frag), 5484c61fe1SJakub Kicinski skb_frag_size(frag))); 5584c61fe1SJakub Kicinski offset += skb_frag_size(frag); 5684c61fe1SJakub Kicinski } 5784c61fe1SJakub Kicinski 58*c1c607b1SJakub Kicinski skb->len = len; 59*c1c607b1SJakub Kicinski skb->data_len = len; 60*c1c607b1SJakub Kicinski skb_copy_header(skb, in_skb); 61*c1c607b1SJakub Kicinski return skb; 62*c1c607b1SJakub Kicinski } 63*c1c607b1SJakub Kicinski 64*c1c607b1SJakub Kicinski /* Create a new skb with the contents of input copied to its page frags */ 65*c1c607b1SJakub Kicinski static struct sk_buff *tls_strp_msg_make_copy(struct tls_strparser *strp) 66*c1c607b1SJakub Kicinski { 67*c1c607b1SJakub Kicinski struct strp_msg *rxm; 68*c1c607b1SJakub Kicinski struct sk_buff *skb; 69*c1c607b1SJakub Kicinski 70*c1c607b1SJakub Kicinski skb = tls_strp_skb_copy(strp, strp->anchor, strp->stm.offset, 71*c1c607b1SJakub Kicinski strp->stm.full_len); 72*c1c607b1SJakub Kicinski if (!skb) 73*c1c607b1SJakub Kicinski return NULL; 74*c1c607b1SJakub Kicinski 7584c61fe1SJakub Kicinski rxm = strp_msg(skb); 7684c61fe1SJakub Kicinski rxm->offset = 0; 77d4e5db64SJakub Kicinski return skb; 78d4e5db64SJakub Kicinski } 79d4e5db64SJakub Kicinski 8084c61fe1SJakub Kicinski /* Steal the input skb, input msg is invalid after calling this function */ 8184c61fe1SJakub Kicinski struct sk_buff *tls_strp_msg_detach(struct tls_sw_context_rx *ctx) 8284c61fe1SJakub Kicinski { 8384c61fe1SJakub Kicinski struct tls_strparser *strp = &ctx->strp; 8484c61fe1SJakub Kicinski 8584c61fe1SJakub Kicinski #ifdef CONFIG_TLS_DEVICE 8684c61fe1SJakub Kicinski DEBUG_NET_WARN_ON_ONCE(!strp->anchor->decrypted); 8784c61fe1SJakub Kicinski #else 8884c61fe1SJakub Kicinski /* This function turns an input into an output, 8984c61fe1SJakub Kicinski * that can only happen if we have offload. 9084c61fe1SJakub Kicinski */ 9184c61fe1SJakub Kicinski WARN_ON(1); 9284c61fe1SJakub Kicinski #endif 9384c61fe1SJakub Kicinski 9484c61fe1SJakub Kicinski if (strp->copy_mode) { 9584c61fe1SJakub Kicinski struct sk_buff *skb; 9684c61fe1SJakub Kicinski 9784c61fe1SJakub Kicinski /* Replace anchor with an empty skb, this is a little 9884c61fe1SJakub Kicinski * dangerous but __tls_cur_msg() warns on empty skbs 9984c61fe1SJakub Kicinski * so hopefully we'll catch abuses. 10084c61fe1SJakub Kicinski */ 10184c61fe1SJakub Kicinski skb = alloc_skb(0, strp->sk->sk_allocation); 10284c61fe1SJakub Kicinski if (!skb) 10384c61fe1SJakub Kicinski return NULL; 10484c61fe1SJakub Kicinski 10584c61fe1SJakub Kicinski swap(strp->anchor, skb); 10684c61fe1SJakub Kicinski return skb; 10784c61fe1SJakub Kicinski } 10884c61fe1SJakub Kicinski 10984c61fe1SJakub Kicinski return tls_strp_msg_make_copy(strp); 11084c61fe1SJakub Kicinski } 11184c61fe1SJakub Kicinski 11284c61fe1SJakub Kicinski /* Force the input skb to be in copy mode. The data ownership remains 11384c61fe1SJakub Kicinski * with the input skb itself (meaning unpause will wipe it) but it can 11484c61fe1SJakub Kicinski * be modified. 11584c61fe1SJakub Kicinski */ 1168b3c59a7SJakub Kicinski int tls_strp_msg_cow(struct tls_sw_context_rx *ctx) 1178b3c59a7SJakub Kicinski { 11884c61fe1SJakub Kicinski struct tls_strparser *strp = &ctx->strp; 11984c61fe1SJakub Kicinski struct sk_buff *skb; 1208b3c59a7SJakub Kicinski 12184c61fe1SJakub Kicinski if (strp->copy_mode) 12284c61fe1SJakub Kicinski return 0; 12384c61fe1SJakub Kicinski 12484c61fe1SJakub Kicinski skb = tls_strp_msg_make_copy(strp); 12584c61fe1SJakub Kicinski if (!skb) 12684c61fe1SJakub Kicinski return -ENOMEM; 12784c61fe1SJakub Kicinski 12884c61fe1SJakub Kicinski tls_strp_anchor_free(strp); 12984c61fe1SJakub Kicinski strp->anchor = skb; 13084c61fe1SJakub Kicinski 13184c61fe1SJakub Kicinski tcp_read_done(strp->sk, strp->stm.full_len); 13284c61fe1SJakub Kicinski strp->copy_mode = 1; 13384c61fe1SJakub Kicinski 1348b3c59a7SJakub Kicinski return 0; 1358b3c59a7SJakub Kicinski } 1368b3c59a7SJakub Kicinski 13784c61fe1SJakub Kicinski /* Make a clone (in the skb sense) of the input msg to keep a reference 13884c61fe1SJakub Kicinski * to the underlying data. The reference-holding skbs get placed on 13984c61fe1SJakub Kicinski * @dst. 14084c61fe1SJakub Kicinski */ 14184c61fe1SJakub Kicinski int tls_strp_msg_hold(struct tls_strparser *strp, struct sk_buff_head *dst) 142c618db2aSJakub Kicinski { 14384c61fe1SJakub Kicinski struct skb_shared_info *shinfo = skb_shinfo(strp->anchor); 144c618db2aSJakub Kicinski 14584c61fe1SJakub Kicinski if (strp->copy_mode) { 14684c61fe1SJakub Kicinski struct sk_buff *skb; 14784c61fe1SJakub Kicinski 14884c61fe1SJakub Kicinski WARN_ON_ONCE(!shinfo->nr_frags); 14984c61fe1SJakub Kicinski 15084c61fe1SJakub Kicinski /* We can't skb_clone() the anchor, it gets wiped by unpause */ 15184c61fe1SJakub Kicinski skb = alloc_skb(0, strp->sk->sk_allocation); 15284c61fe1SJakub Kicinski if (!skb) 15384c61fe1SJakub Kicinski return -ENOMEM; 15484c61fe1SJakub Kicinski 15584c61fe1SJakub Kicinski __skb_queue_tail(dst, strp->anchor); 15684c61fe1SJakub Kicinski strp->anchor = skb; 15784c61fe1SJakub Kicinski } else { 15884c61fe1SJakub Kicinski struct sk_buff *iter, *clone; 15984c61fe1SJakub Kicinski int chunk, len, offset; 16084c61fe1SJakub Kicinski 16184c61fe1SJakub Kicinski offset = strp->stm.offset; 16284c61fe1SJakub Kicinski len = strp->stm.full_len; 16384c61fe1SJakub Kicinski iter = shinfo->frag_list; 16484c61fe1SJakub Kicinski 16584c61fe1SJakub Kicinski while (len > 0) { 16684c61fe1SJakub Kicinski if (iter->len <= offset) { 16784c61fe1SJakub Kicinski offset -= iter->len; 16884c61fe1SJakub Kicinski goto next; 16984c61fe1SJakub Kicinski } 17084c61fe1SJakub Kicinski 17184c61fe1SJakub Kicinski chunk = iter->len - offset; 17284c61fe1SJakub Kicinski offset = 0; 17384c61fe1SJakub Kicinski 17484c61fe1SJakub Kicinski clone = skb_clone(iter, strp->sk->sk_allocation); 175c618db2aSJakub Kicinski if (!clone) 176c618db2aSJakub Kicinski return -ENOMEM; 177c618db2aSJakub Kicinski __skb_queue_tail(dst, clone); 17884c61fe1SJakub Kicinski 17984c61fe1SJakub Kicinski len -= chunk; 18084c61fe1SJakub Kicinski next: 18184c61fe1SJakub Kicinski iter = iter->next; 18284c61fe1SJakub Kicinski } 18384c61fe1SJakub Kicinski } 18484c61fe1SJakub Kicinski 185c618db2aSJakub Kicinski return 0; 186c618db2aSJakub Kicinski } 18784c61fe1SJakub Kicinski 18884c61fe1SJakub Kicinski static void tls_strp_flush_anchor_copy(struct tls_strparser *strp) 18984c61fe1SJakub Kicinski { 19084c61fe1SJakub Kicinski struct skb_shared_info *shinfo = skb_shinfo(strp->anchor); 19184c61fe1SJakub Kicinski int i; 19284c61fe1SJakub Kicinski 19384c61fe1SJakub Kicinski DEBUG_NET_WARN_ON_ONCE(atomic_read(&shinfo->dataref) != 1); 19484c61fe1SJakub Kicinski 19584c61fe1SJakub Kicinski for (i = 0; i < shinfo->nr_frags; i++) 19684c61fe1SJakub Kicinski __skb_frag_unref(&shinfo->frags[i], false); 19784c61fe1SJakub Kicinski shinfo->nr_frags = 0; 19884c61fe1SJakub Kicinski strp->copy_mode = 0; 19984c61fe1SJakub Kicinski } 20084c61fe1SJakub Kicinski 20184c61fe1SJakub Kicinski static int tls_strp_copyin(read_descriptor_t *desc, struct sk_buff *in_skb, 20284c61fe1SJakub Kicinski unsigned int offset, size_t in_len) 20384c61fe1SJakub Kicinski { 20484c61fe1SJakub Kicinski struct tls_strparser *strp = (struct tls_strparser *)desc->arg.data; 20584c61fe1SJakub Kicinski struct sk_buff *skb; 20684c61fe1SJakub Kicinski skb_frag_t *frag; 2078fd1e151SYang Li size_t len, chunk; 2088fd1e151SYang Li int sz; 20984c61fe1SJakub Kicinski 21084c61fe1SJakub Kicinski if (strp->msg_ready) 21184c61fe1SJakub Kicinski return 0; 21284c61fe1SJakub Kicinski 21384c61fe1SJakub Kicinski skb = strp->anchor; 21484c61fe1SJakub Kicinski frag = &skb_shinfo(skb)->frags[skb->len / PAGE_SIZE]; 21584c61fe1SJakub Kicinski 21684c61fe1SJakub Kicinski len = in_len; 21784c61fe1SJakub Kicinski /* First make sure we got the header */ 21884c61fe1SJakub Kicinski if (!strp->stm.full_len) { 21984c61fe1SJakub Kicinski /* Assume one page is more than enough for headers */ 22084c61fe1SJakub Kicinski chunk = min_t(size_t, len, PAGE_SIZE - skb_frag_size(frag)); 22184c61fe1SJakub Kicinski WARN_ON_ONCE(skb_copy_bits(in_skb, offset, 22284c61fe1SJakub Kicinski skb_frag_address(frag) + 22384c61fe1SJakub Kicinski skb_frag_size(frag), 22484c61fe1SJakub Kicinski chunk)); 22584c61fe1SJakub Kicinski 2268b0c0dc9SJakub Kicinski skb->len += chunk; 2278b0c0dc9SJakub Kicinski skb->data_len += chunk; 2288b0c0dc9SJakub Kicinski skb_frag_size_add(frag, chunk); 2298b0c0dc9SJakub Kicinski 2308b0c0dc9SJakub Kicinski sz = tls_rx_msg_size(strp, skb); 23184c61fe1SJakub Kicinski if (sz < 0) { 23284c61fe1SJakub Kicinski desc->error = sz; 23384c61fe1SJakub Kicinski return 0; 23484c61fe1SJakub Kicinski } 23584c61fe1SJakub Kicinski 23684c61fe1SJakub Kicinski /* We may have over-read, sz == 0 is guaranteed under-read */ 2378b0c0dc9SJakub Kicinski if (unlikely(sz && sz < skb->len)) { 2388b0c0dc9SJakub Kicinski int over = skb->len - sz; 23984c61fe1SJakub Kicinski 2408b0c0dc9SJakub Kicinski WARN_ON_ONCE(over > chunk); 2418b0c0dc9SJakub Kicinski skb->len -= over; 2428b0c0dc9SJakub Kicinski skb->data_len -= over; 2438b0c0dc9SJakub Kicinski skb_frag_size_add(frag, -over); 2448b0c0dc9SJakub Kicinski 2458b0c0dc9SJakub Kicinski chunk -= over; 2468b0c0dc9SJakub Kicinski } 2478b0c0dc9SJakub Kicinski 24884c61fe1SJakub Kicinski frag++; 24984c61fe1SJakub Kicinski len -= chunk; 25084c61fe1SJakub Kicinski offset += chunk; 25184c61fe1SJakub Kicinski 25284c61fe1SJakub Kicinski strp->stm.full_len = sz; 25384c61fe1SJakub Kicinski if (!strp->stm.full_len) 25484c61fe1SJakub Kicinski goto read_done; 25584c61fe1SJakub Kicinski } 25684c61fe1SJakub Kicinski 25784c61fe1SJakub Kicinski /* Load up more data */ 25884c61fe1SJakub Kicinski while (len && strp->stm.full_len > skb->len) { 25984c61fe1SJakub Kicinski chunk = min_t(size_t, len, strp->stm.full_len - skb->len); 26084c61fe1SJakub Kicinski chunk = min_t(size_t, chunk, PAGE_SIZE - skb_frag_size(frag)); 26184c61fe1SJakub Kicinski WARN_ON_ONCE(skb_copy_bits(in_skb, offset, 26284c61fe1SJakub Kicinski skb_frag_address(frag) + 26384c61fe1SJakub Kicinski skb_frag_size(frag), 26484c61fe1SJakub Kicinski chunk)); 26584c61fe1SJakub Kicinski 26684c61fe1SJakub Kicinski skb->len += chunk; 26784c61fe1SJakub Kicinski skb->data_len += chunk; 26884c61fe1SJakub Kicinski skb_frag_size_add(frag, chunk); 26984c61fe1SJakub Kicinski frag++; 27084c61fe1SJakub Kicinski len -= chunk; 27184c61fe1SJakub Kicinski offset += chunk; 27284c61fe1SJakub Kicinski } 27384c61fe1SJakub Kicinski 27484c61fe1SJakub Kicinski if (strp->stm.full_len == skb->len) { 27584c61fe1SJakub Kicinski desc->count = 0; 27684c61fe1SJakub Kicinski 27784c61fe1SJakub Kicinski strp->msg_ready = 1; 27884c61fe1SJakub Kicinski tls_rx_msg_ready(strp); 27984c61fe1SJakub Kicinski } 28084c61fe1SJakub Kicinski 28184c61fe1SJakub Kicinski read_done: 28284c61fe1SJakub Kicinski return in_len - len; 28384c61fe1SJakub Kicinski } 28484c61fe1SJakub Kicinski 28584c61fe1SJakub Kicinski static int tls_strp_read_copyin(struct tls_strparser *strp) 28684c61fe1SJakub Kicinski { 28784c61fe1SJakub Kicinski struct socket *sock = strp->sk->sk_socket; 28884c61fe1SJakub Kicinski read_descriptor_t desc; 28984c61fe1SJakub Kicinski 29084c61fe1SJakub Kicinski desc.arg.data = strp; 29184c61fe1SJakub Kicinski desc.error = 0; 29284c61fe1SJakub Kicinski desc.count = 1; /* give more than one skb per call */ 29384c61fe1SJakub Kicinski 29484c61fe1SJakub Kicinski /* sk should be locked here, so okay to do read_sock */ 29584c61fe1SJakub Kicinski sock->ops->read_sock(strp->sk, &desc, tls_strp_copyin); 29684c61fe1SJakub Kicinski 29784c61fe1SJakub Kicinski return desc.error; 29884c61fe1SJakub Kicinski } 29984c61fe1SJakub Kicinski 3000d87bbd3SJakub Kicinski static int tls_strp_read_copy(struct tls_strparser *strp, bool qshort) 30184c61fe1SJakub Kicinski { 30284c61fe1SJakub Kicinski struct skb_shared_info *shinfo; 30384c61fe1SJakub Kicinski struct page *page; 30484c61fe1SJakub Kicinski int need_spc, len; 30584c61fe1SJakub Kicinski 30684c61fe1SJakub Kicinski /* If the rbuf is small or rcv window has collapsed to 0 we need 30784c61fe1SJakub Kicinski * to read the data out. Otherwise the connection will stall. 30884c61fe1SJakub Kicinski * Without pressure threshold of INT_MAX will never be ready. 30984c61fe1SJakub Kicinski */ 3100d87bbd3SJakub Kicinski if (likely(qshort && !tcp_epollin_ready(strp->sk, INT_MAX))) 31184c61fe1SJakub Kicinski return 0; 31284c61fe1SJakub Kicinski 31384c61fe1SJakub Kicinski shinfo = skb_shinfo(strp->anchor); 31484c61fe1SJakub Kicinski shinfo->frag_list = NULL; 31584c61fe1SJakub Kicinski 31684c61fe1SJakub Kicinski /* If we don't know the length go max plus page for cipher overhead */ 31784c61fe1SJakub Kicinski need_spc = strp->stm.full_len ?: TLS_MAX_PAYLOAD_SIZE + PAGE_SIZE; 31884c61fe1SJakub Kicinski 31984c61fe1SJakub Kicinski for (len = need_spc; len > 0; len -= PAGE_SIZE) { 32084c61fe1SJakub Kicinski page = alloc_page(strp->sk->sk_allocation); 32184c61fe1SJakub Kicinski if (!page) { 32284c61fe1SJakub Kicinski tls_strp_flush_anchor_copy(strp); 32384c61fe1SJakub Kicinski return -ENOMEM; 32484c61fe1SJakub Kicinski } 32584c61fe1SJakub Kicinski 32684c61fe1SJakub Kicinski skb_fill_page_desc(strp->anchor, shinfo->nr_frags++, 32784c61fe1SJakub Kicinski page, 0, 0); 32884c61fe1SJakub Kicinski } 32984c61fe1SJakub Kicinski 33084c61fe1SJakub Kicinski strp->copy_mode = 1; 33184c61fe1SJakub Kicinski strp->stm.offset = 0; 33284c61fe1SJakub Kicinski 33384c61fe1SJakub Kicinski strp->anchor->len = 0; 33484c61fe1SJakub Kicinski strp->anchor->data_len = 0; 33584c61fe1SJakub Kicinski strp->anchor->truesize = round_up(need_spc, PAGE_SIZE); 33684c61fe1SJakub Kicinski 33784c61fe1SJakub Kicinski tls_strp_read_copyin(strp); 33884c61fe1SJakub Kicinski 33984c61fe1SJakub Kicinski return 0; 34084c61fe1SJakub Kicinski } 34184c61fe1SJakub Kicinski 34214c4be92SJakub Kicinski static bool tls_strp_check_queue_ok(struct tls_strparser *strp) 3430d87bbd3SJakub Kicinski { 3440d87bbd3SJakub Kicinski unsigned int len = strp->stm.offset + strp->stm.full_len; 34514c4be92SJakub Kicinski struct sk_buff *first, *skb; 3460d87bbd3SJakub Kicinski u32 seq; 3470d87bbd3SJakub Kicinski 34814c4be92SJakub Kicinski first = skb_shinfo(strp->anchor)->frag_list; 34914c4be92SJakub Kicinski skb = first; 35014c4be92SJakub Kicinski seq = TCP_SKB_CB(first)->seq; 3510d87bbd3SJakub Kicinski 35214c4be92SJakub Kicinski /* Make sure there's no duplicate data in the queue, 35314c4be92SJakub Kicinski * and the decrypted status matches. 35414c4be92SJakub Kicinski */ 3550d87bbd3SJakub Kicinski while (skb->len < len) { 3560d87bbd3SJakub Kicinski seq += skb->len; 3570d87bbd3SJakub Kicinski len -= skb->len; 3580d87bbd3SJakub Kicinski skb = skb->next; 3590d87bbd3SJakub Kicinski 3600d87bbd3SJakub Kicinski if (TCP_SKB_CB(skb)->seq != seq) 3610d87bbd3SJakub Kicinski return false; 36214c4be92SJakub Kicinski if (skb_cmp_decrypted(first, skb)) 36314c4be92SJakub Kicinski return false; 3640d87bbd3SJakub Kicinski } 3650d87bbd3SJakub Kicinski 3660d87bbd3SJakub Kicinski return true; 3670d87bbd3SJakub Kicinski } 3680d87bbd3SJakub Kicinski 36984c61fe1SJakub Kicinski static void tls_strp_load_anchor_with_queue(struct tls_strparser *strp, int len) 37084c61fe1SJakub Kicinski { 37184c61fe1SJakub Kicinski struct tcp_sock *tp = tcp_sk(strp->sk); 37284c61fe1SJakub Kicinski struct sk_buff *first; 37384c61fe1SJakub Kicinski u32 offset; 37484c61fe1SJakub Kicinski 37584c61fe1SJakub Kicinski first = tcp_recv_skb(strp->sk, tp->copied_seq, &offset); 37684c61fe1SJakub Kicinski if (WARN_ON_ONCE(!first)) 37784c61fe1SJakub Kicinski return; 37884c61fe1SJakub Kicinski 37984c61fe1SJakub Kicinski /* Bestow the state onto the anchor */ 38084c61fe1SJakub Kicinski strp->anchor->len = offset + len; 38184c61fe1SJakub Kicinski strp->anchor->data_len = offset + len; 38284c61fe1SJakub Kicinski strp->anchor->truesize = offset + len; 38384c61fe1SJakub Kicinski 38484c61fe1SJakub Kicinski skb_shinfo(strp->anchor)->frag_list = first; 38584c61fe1SJakub Kicinski 38684c61fe1SJakub Kicinski skb_copy_header(strp->anchor, first); 38784c61fe1SJakub Kicinski strp->anchor->destructor = NULL; 38884c61fe1SJakub Kicinski 38984c61fe1SJakub Kicinski strp->stm.offset = offset; 39084c61fe1SJakub Kicinski } 39184c61fe1SJakub Kicinski 39284c61fe1SJakub Kicinski void tls_strp_msg_load(struct tls_strparser *strp, bool force_refresh) 39384c61fe1SJakub Kicinski { 39484c61fe1SJakub Kicinski struct strp_msg *rxm; 39584c61fe1SJakub Kicinski struct tls_msg *tlm; 39684c61fe1SJakub Kicinski 39784c61fe1SJakub Kicinski DEBUG_NET_WARN_ON_ONCE(!strp->msg_ready); 39884c61fe1SJakub Kicinski DEBUG_NET_WARN_ON_ONCE(!strp->stm.full_len); 39984c61fe1SJakub Kicinski 40084c61fe1SJakub Kicinski if (!strp->copy_mode && force_refresh) { 40184c61fe1SJakub Kicinski if (WARN_ON(tcp_inq(strp->sk) < strp->stm.full_len)) 40284c61fe1SJakub Kicinski return; 40384c61fe1SJakub Kicinski 40484c61fe1SJakub Kicinski tls_strp_load_anchor_with_queue(strp, strp->stm.full_len); 40584c61fe1SJakub Kicinski } 40684c61fe1SJakub Kicinski 40784c61fe1SJakub Kicinski rxm = strp_msg(strp->anchor); 40884c61fe1SJakub Kicinski rxm->full_len = strp->stm.full_len; 40984c61fe1SJakub Kicinski rxm->offset = strp->stm.offset; 41084c61fe1SJakub Kicinski tlm = tls_msg(strp->anchor); 41184c61fe1SJakub Kicinski tlm->control = strp->mark; 41284c61fe1SJakub Kicinski } 41384c61fe1SJakub Kicinski 41484c61fe1SJakub Kicinski /* Called with lock held on lower socket */ 41584c61fe1SJakub Kicinski static int tls_strp_read_sock(struct tls_strparser *strp) 41684c61fe1SJakub Kicinski { 41784c61fe1SJakub Kicinski int sz, inq; 41884c61fe1SJakub Kicinski 41984c61fe1SJakub Kicinski inq = tcp_inq(strp->sk); 42084c61fe1SJakub Kicinski if (inq < 1) 42184c61fe1SJakub Kicinski return 0; 42284c61fe1SJakub Kicinski 42384c61fe1SJakub Kicinski if (unlikely(strp->copy_mode)) 42484c61fe1SJakub Kicinski return tls_strp_read_copyin(strp); 42584c61fe1SJakub Kicinski 42684c61fe1SJakub Kicinski if (inq < strp->stm.full_len) 4270d87bbd3SJakub Kicinski return tls_strp_read_copy(strp, true); 42884c61fe1SJakub Kicinski 42984c61fe1SJakub Kicinski if (!strp->stm.full_len) { 43084c61fe1SJakub Kicinski tls_strp_load_anchor_with_queue(strp, inq); 43184c61fe1SJakub Kicinski 43284c61fe1SJakub Kicinski sz = tls_rx_msg_size(strp, strp->anchor); 43384c61fe1SJakub Kicinski if (sz < 0) { 43484c61fe1SJakub Kicinski tls_strp_abort_strp(strp, sz); 43584c61fe1SJakub Kicinski return sz; 43684c61fe1SJakub Kicinski } 43784c61fe1SJakub Kicinski 43884c61fe1SJakub Kicinski strp->stm.full_len = sz; 43984c61fe1SJakub Kicinski 44084c61fe1SJakub Kicinski if (!strp->stm.full_len || inq < strp->stm.full_len) 4410d87bbd3SJakub Kicinski return tls_strp_read_copy(strp, true); 44284c61fe1SJakub Kicinski } 44384c61fe1SJakub Kicinski 44414c4be92SJakub Kicinski if (!tls_strp_check_queue_ok(strp)) 4450d87bbd3SJakub Kicinski return tls_strp_read_copy(strp, false); 4460d87bbd3SJakub Kicinski 44784c61fe1SJakub Kicinski strp->msg_ready = 1; 44884c61fe1SJakub Kicinski tls_rx_msg_ready(strp); 44984c61fe1SJakub Kicinski 45084c61fe1SJakub Kicinski return 0; 45184c61fe1SJakub Kicinski } 45284c61fe1SJakub Kicinski 45384c61fe1SJakub Kicinski void tls_strp_check_rcv(struct tls_strparser *strp) 45484c61fe1SJakub Kicinski { 45584c61fe1SJakub Kicinski if (unlikely(strp->stopped) || strp->msg_ready) 45684c61fe1SJakub Kicinski return; 45784c61fe1SJakub Kicinski 45884c61fe1SJakub Kicinski if (tls_strp_read_sock(strp) == -ENOMEM) 45984c61fe1SJakub Kicinski queue_work(tls_strp_wq, &strp->work); 46084c61fe1SJakub Kicinski } 46184c61fe1SJakub Kicinski 46284c61fe1SJakub Kicinski /* Lower sock lock held */ 46384c61fe1SJakub Kicinski void tls_strp_data_ready(struct tls_strparser *strp) 46484c61fe1SJakub Kicinski { 46584c61fe1SJakub Kicinski /* This check is needed to synchronize with do_tls_strp_work. 46684c61fe1SJakub Kicinski * do_tls_strp_work acquires a process lock (lock_sock) whereas 46784c61fe1SJakub Kicinski * the lock held here is bh_lock_sock. The two locks can be 46884c61fe1SJakub Kicinski * held by different threads at the same time, but bh_lock_sock 46984c61fe1SJakub Kicinski * allows a thread in BH context to safely check if the process 47084c61fe1SJakub Kicinski * lock is held. In this case, if the lock is held, queue work. 47184c61fe1SJakub Kicinski */ 47284c61fe1SJakub Kicinski if (sock_owned_by_user_nocheck(strp->sk)) { 47384c61fe1SJakub Kicinski queue_work(tls_strp_wq, &strp->work); 47484c61fe1SJakub Kicinski return; 47584c61fe1SJakub Kicinski } 47684c61fe1SJakub Kicinski 47784c61fe1SJakub Kicinski tls_strp_check_rcv(strp); 47884c61fe1SJakub Kicinski } 47984c61fe1SJakub Kicinski 48084c61fe1SJakub Kicinski static void tls_strp_work(struct work_struct *w) 48184c61fe1SJakub Kicinski { 48284c61fe1SJakub Kicinski struct tls_strparser *strp = 48384c61fe1SJakub Kicinski container_of(w, struct tls_strparser, work); 48484c61fe1SJakub Kicinski 48584c61fe1SJakub Kicinski lock_sock(strp->sk); 48684c61fe1SJakub Kicinski tls_strp_check_rcv(strp); 48784c61fe1SJakub Kicinski release_sock(strp->sk); 48884c61fe1SJakub Kicinski } 48984c61fe1SJakub Kicinski 49084c61fe1SJakub Kicinski void tls_strp_msg_done(struct tls_strparser *strp) 49184c61fe1SJakub Kicinski { 49284c61fe1SJakub Kicinski WARN_ON(!strp->stm.full_len); 49384c61fe1SJakub Kicinski 49484c61fe1SJakub Kicinski if (likely(!strp->copy_mode)) 49584c61fe1SJakub Kicinski tcp_read_done(strp->sk, strp->stm.full_len); 49684c61fe1SJakub Kicinski else 49784c61fe1SJakub Kicinski tls_strp_flush_anchor_copy(strp); 49884c61fe1SJakub Kicinski 49984c61fe1SJakub Kicinski strp->msg_ready = 0; 50084c61fe1SJakub Kicinski memset(&strp->stm, 0, sizeof(strp->stm)); 50184c61fe1SJakub Kicinski 50284c61fe1SJakub Kicinski tls_strp_check_rcv(strp); 50384c61fe1SJakub Kicinski } 50484c61fe1SJakub Kicinski 50584c61fe1SJakub Kicinski void tls_strp_stop(struct tls_strparser *strp) 50684c61fe1SJakub Kicinski { 50784c61fe1SJakub Kicinski strp->stopped = 1; 50884c61fe1SJakub Kicinski } 50984c61fe1SJakub Kicinski 51084c61fe1SJakub Kicinski int tls_strp_init(struct tls_strparser *strp, struct sock *sk) 51184c61fe1SJakub Kicinski { 51284c61fe1SJakub Kicinski memset(strp, 0, sizeof(*strp)); 51384c61fe1SJakub Kicinski 51484c61fe1SJakub Kicinski strp->sk = sk; 51584c61fe1SJakub Kicinski 51684c61fe1SJakub Kicinski strp->anchor = alloc_skb(0, GFP_KERNEL); 51784c61fe1SJakub Kicinski if (!strp->anchor) 51884c61fe1SJakub Kicinski return -ENOMEM; 51984c61fe1SJakub Kicinski 52084c61fe1SJakub Kicinski INIT_WORK(&strp->work, tls_strp_work); 52184c61fe1SJakub Kicinski 52284c61fe1SJakub Kicinski return 0; 52384c61fe1SJakub Kicinski } 52484c61fe1SJakub Kicinski 52584c61fe1SJakub Kicinski /* strp must already be stopped so that tls_strp_recv will no longer be called. 52684c61fe1SJakub Kicinski * Note that tls_strp_done is not called with the lower socket held. 52784c61fe1SJakub Kicinski */ 52884c61fe1SJakub Kicinski void tls_strp_done(struct tls_strparser *strp) 52984c61fe1SJakub Kicinski { 53084c61fe1SJakub Kicinski WARN_ON(!strp->stopped); 53184c61fe1SJakub Kicinski 53284c61fe1SJakub Kicinski cancel_work_sync(&strp->work); 53384c61fe1SJakub Kicinski tls_strp_anchor_free(strp); 53484c61fe1SJakub Kicinski } 53584c61fe1SJakub Kicinski 53684c61fe1SJakub Kicinski int __init tls_strp_dev_init(void) 53784c61fe1SJakub Kicinski { 538d11ef9ccSJakub Kicinski tls_strp_wq = create_workqueue("tls-strp"); 53984c61fe1SJakub Kicinski if (unlikely(!tls_strp_wq)) 54084c61fe1SJakub Kicinski return -ENOMEM; 54184c61fe1SJakub Kicinski 54284c61fe1SJakub Kicinski return 0; 54384c61fe1SJakub Kicinski } 54484c61fe1SJakub Kicinski 54584c61fe1SJakub Kicinski void tls_strp_dev_exit(void) 54684c61fe1SJakub Kicinski { 54784c61fe1SJakub Kicinski destroy_workqueue(tls_strp_wq); 54884c61fe1SJakub Kicinski } 549