1c618db2aSJakub Kicinski // SPDX-License-Identifier: GPL-2.0-only 284c61fe1SJakub Kicinski /* Copyright (c) 2016 Tom Herbert <tom@herbertland.com> */ 3c618db2aSJakub Kicinski 4c618db2aSJakub Kicinski #include <linux/skbuff.h> 584c61fe1SJakub Kicinski #include <linux/workqueue.h> 684c61fe1SJakub Kicinski #include <net/strparser.h> 784c61fe1SJakub Kicinski #include <net/tcp.h> 884c61fe1SJakub Kicinski #include <net/sock.h> 984c61fe1SJakub Kicinski #include <net/tls.h> 10c618db2aSJakub Kicinski 11c618db2aSJakub Kicinski #include "tls.h" 12c618db2aSJakub Kicinski 1384c61fe1SJakub Kicinski static struct workqueue_struct *tls_strp_wq; 14d4e5db64SJakub Kicinski 1584c61fe1SJakub Kicinski static void tls_strp_abort_strp(struct tls_strparser *strp, int err) 1684c61fe1SJakub Kicinski { 1784c61fe1SJakub Kicinski if (strp->stopped) 1884c61fe1SJakub Kicinski return; 1984c61fe1SJakub Kicinski 2084c61fe1SJakub Kicinski strp->stopped = 1; 2184c61fe1SJakub Kicinski 2284c61fe1SJakub Kicinski /* Report an error on the lower socket */ 2384c61fe1SJakub Kicinski strp->sk->sk_err = -err; 2484c61fe1SJakub Kicinski sk_error_report(strp->sk); 2584c61fe1SJakub Kicinski } 2684c61fe1SJakub Kicinski 2784c61fe1SJakub Kicinski static void tls_strp_anchor_free(struct tls_strparser *strp) 2884c61fe1SJakub Kicinski { 2984c61fe1SJakub Kicinski struct skb_shared_info *shinfo = skb_shinfo(strp->anchor); 3084c61fe1SJakub Kicinski 3184c61fe1SJakub Kicinski DEBUG_NET_WARN_ON_ONCE(atomic_read(&shinfo->dataref) != 1); 3284c61fe1SJakub Kicinski shinfo->frag_list = NULL; 3384c61fe1SJakub Kicinski consume_skb(strp->anchor); 3484c61fe1SJakub Kicinski strp->anchor = NULL; 3584c61fe1SJakub Kicinski } 3684c61fe1SJakub Kicinski 3784c61fe1SJakub Kicinski /* Create a new skb with the contents of input copied to its page frags */ 3884c61fe1SJakub Kicinski static struct sk_buff *tls_strp_msg_make_copy(struct tls_strparser *strp) 3984c61fe1SJakub Kicinski { 4084c61fe1SJakub Kicinski struct strp_msg *rxm; 4184c61fe1SJakub Kicinski struct sk_buff *skb; 4284c61fe1SJakub Kicinski int i, err, offset; 4384c61fe1SJakub Kicinski 44d800a7b3SJakub Kicinski skb = alloc_skb_with_frags(0, strp->stm.full_len, TLS_PAGE_ORDER, 4584c61fe1SJakub Kicinski &err, strp->sk->sk_allocation); 4684c61fe1SJakub Kicinski if (!skb) 4784c61fe1SJakub Kicinski return NULL; 4884c61fe1SJakub Kicinski 4984c61fe1SJakub Kicinski offset = strp->stm.offset; 5084c61fe1SJakub Kicinski for (i = 0; i < skb_shinfo(skb)->nr_frags; i++) { 5184c61fe1SJakub Kicinski skb_frag_t *frag = &skb_shinfo(skb)->frags[i]; 5284c61fe1SJakub Kicinski 5384c61fe1SJakub Kicinski WARN_ON_ONCE(skb_copy_bits(strp->anchor, offset, 5484c61fe1SJakub Kicinski skb_frag_address(frag), 5584c61fe1SJakub Kicinski skb_frag_size(frag))); 5684c61fe1SJakub Kicinski offset += skb_frag_size(frag); 5784c61fe1SJakub Kicinski } 5884c61fe1SJakub Kicinski 5984c61fe1SJakub Kicinski skb_copy_header(skb, strp->anchor); 6084c61fe1SJakub Kicinski rxm = strp_msg(skb); 6184c61fe1SJakub Kicinski rxm->offset = 0; 62d4e5db64SJakub Kicinski return skb; 63d4e5db64SJakub Kicinski } 64d4e5db64SJakub Kicinski 6584c61fe1SJakub Kicinski /* Steal the input skb, input msg is invalid after calling this function */ 6684c61fe1SJakub Kicinski struct sk_buff *tls_strp_msg_detach(struct tls_sw_context_rx *ctx) 6784c61fe1SJakub Kicinski { 6884c61fe1SJakub Kicinski struct tls_strparser *strp = &ctx->strp; 6984c61fe1SJakub Kicinski 7084c61fe1SJakub Kicinski #ifdef CONFIG_TLS_DEVICE 7184c61fe1SJakub Kicinski DEBUG_NET_WARN_ON_ONCE(!strp->anchor->decrypted); 7284c61fe1SJakub Kicinski #else 7384c61fe1SJakub Kicinski /* This function turns an input into an output, 7484c61fe1SJakub Kicinski * that can only happen if we have offload. 7584c61fe1SJakub Kicinski */ 7684c61fe1SJakub Kicinski WARN_ON(1); 7784c61fe1SJakub Kicinski #endif 7884c61fe1SJakub Kicinski 7984c61fe1SJakub Kicinski if (strp->copy_mode) { 8084c61fe1SJakub Kicinski struct sk_buff *skb; 8184c61fe1SJakub Kicinski 8284c61fe1SJakub Kicinski /* Replace anchor with an empty skb, this is a little 8384c61fe1SJakub Kicinski * dangerous but __tls_cur_msg() warns on empty skbs 8484c61fe1SJakub Kicinski * so hopefully we'll catch abuses. 8584c61fe1SJakub Kicinski */ 8684c61fe1SJakub Kicinski skb = alloc_skb(0, strp->sk->sk_allocation); 8784c61fe1SJakub Kicinski if (!skb) 8884c61fe1SJakub Kicinski return NULL; 8984c61fe1SJakub Kicinski 9084c61fe1SJakub Kicinski swap(strp->anchor, skb); 9184c61fe1SJakub Kicinski return skb; 9284c61fe1SJakub Kicinski } 9384c61fe1SJakub Kicinski 9484c61fe1SJakub Kicinski return tls_strp_msg_make_copy(strp); 9584c61fe1SJakub Kicinski } 9684c61fe1SJakub Kicinski 9784c61fe1SJakub Kicinski /* Force the input skb to be in copy mode. The data ownership remains 9884c61fe1SJakub Kicinski * with the input skb itself (meaning unpause will wipe it) but it can 9984c61fe1SJakub Kicinski * be modified. 10084c61fe1SJakub Kicinski */ 1018b3c59a7SJakub Kicinski int tls_strp_msg_cow(struct tls_sw_context_rx *ctx) 1028b3c59a7SJakub Kicinski { 10384c61fe1SJakub Kicinski struct tls_strparser *strp = &ctx->strp; 10484c61fe1SJakub Kicinski struct sk_buff *skb; 1058b3c59a7SJakub Kicinski 10684c61fe1SJakub Kicinski if (strp->copy_mode) 10784c61fe1SJakub Kicinski return 0; 10884c61fe1SJakub Kicinski 10984c61fe1SJakub Kicinski skb = tls_strp_msg_make_copy(strp); 11084c61fe1SJakub Kicinski if (!skb) 11184c61fe1SJakub Kicinski return -ENOMEM; 11284c61fe1SJakub Kicinski 11384c61fe1SJakub Kicinski tls_strp_anchor_free(strp); 11484c61fe1SJakub Kicinski strp->anchor = skb; 11584c61fe1SJakub Kicinski 11684c61fe1SJakub Kicinski tcp_read_done(strp->sk, strp->stm.full_len); 11784c61fe1SJakub Kicinski strp->copy_mode = 1; 11884c61fe1SJakub Kicinski 1198b3c59a7SJakub Kicinski return 0; 1208b3c59a7SJakub Kicinski } 1218b3c59a7SJakub Kicinski 12284c61fe1SJakub Kicinski /* Make a clone (in the skb sense) of the input msg to keep a reference 12384c61fe1SJakub Kicinski * to the underlying data. The reference-holding skbs get placed on 12484c61fe1SJakub Kicinski * @dst. 12584c61fe1SJakub Kicinski */ 12684c61fe1SJakub Kicinski int tls_strp_msg_hold(struct tls_strparser *strp, struct sk_buff_head *dst) 127c618db2aSJakub Kicinski { 12884c61fe1SJakub Kicinski struct skb_shared_info *shinfo = skb_shinfo(strp->anchor); 129c618db2aSJakub Kicinski 13084c61fe1SJakub Kicinski if (strp->copy_mode) { 13184c61fe1SJakub Kicinski struct sk_buff *skb; 13284c61fe1SJakub Kicinski 13384c61fe1SJakub Kicinski WARN_ON_ONCE(!shinfo->nr_frags); 13484c61fe1SJakub Kicinski 13584c61fe1SJakub Kicinski /* We can't skb_clone() the anchor, it gets wiped by unpause */ 13684c61fe1SJakub Kicinski skb = alloc_skb(0, strp->sk->sk_allocation); 13784c61fe1SJakub Kicinski if (!skb) 13884c61fe1SJakub Kicinski return -ENOMEM; 13984c61fe1SJakub Kicinski 14084c61fe1SJakub Kicinski __skb_queue_tail(dst, strp->anchor); 14184c61fe1SJakub Kicinski strp->anchor = skb; 14284c61fe1SJakub Kicinski } else { 14384c61fe1SJakub Kicinski struct sk_buff *iter, *clone; 14484c61fe1SJakub Kicinski int chunk, len, offset; 14584c61fe1SJakub Kicinski 14684c61fe1SJakub Kicinski offset = strp->stm.offset; 14784c61fe1SJakub Kicinski len = strp->stm.full_len; 14884c61fe1SJakub Kicinski iter = shinfo->frag_list; 14984c61fe1SJakub Kicinski 15084c61fe1SJakub Kicinski while (len > 0) { 15184c61fe1SJakub Kicinski if (iter->len <= offset) { 15284c61fe1SJakub Kicinski offset -= iter->len; 15384c61fe1SJakub Kicinski goto next; 15484c61fe1SJakub Kicinski } 15584c61fe1SJakub Kicinski 15684c61fe1SJakub Kicinski chunk = iter->len - offset; 15784c61fe1SJakub Kicinski offset = 0; 15884c61fe1SJakub Kicinski 15984c61fe1SJakub Kicinski clone = skb_clone(iter, strp->sk->sk_allocation); 160c618db2aSJakub Kicinski if (!clone) 161c618db2aSJakub Kicinski return -ENOMEM; 162c618db2aSJakub Kicinski __skb_queue_tail(dst, clone); 16384c61fe1SJakub Kicinski 16484c61fe1SJakub Kicinski len -= chunk; 16584c61fe1SJakub Kicinski next: 16684c61fe1SJakub Kicinski iter = iter->next; 16784c61fe1SJakub Kicinski } 16884c61fe1SJakub Kicinski } 16984c61fe1SJakub Kicinski 170c618db2aSJakub Kicinski return 0; 171c618db2aSJakub Kicinski } 17284c61fe1SJakub Kicinski 17384c61fe1SJakub Kicinski static void tls_strp_flush_anchor_copy(struct tls_strparser *strp) 17484c61fe1SJakub Kicinski { 17584c61fe1SJakub Kicinski struct skb_shared_info *shinfo = skb_shinfo(strp->anchor); 17684c61fe1SJakub Kicinski int i; 17784c61fe1SJakub Kicinski 17884c61fe1SJakub Kicinski DEBUG_NET_WARN_ON_ONCE(atomic_read(&shinfo->dataref) != 1); 17984c61fe1SJakub Kicinski 18084c61fe1SJakub Kicinski for (i = 0; i < shinfo->nr_frags; i++) 18184c61fe1SJakub Kicinski __skb_frag_unref(&shinfo->frags[i], false); 18284c61fe1SJakub Kicinski shinfo->nr_frags = 0; 18384c61fe1SJakub Kicinski strp->copy_mode = 0; 18484c61fe1SJakub Kicinski } 18584c61fe1SJakub Kicinski 18684c61fe1SJakub Kicinski static int tls_strp_copyin(read_descriptor_t *desc, struct sk_buff *in_skb, 18784c61fe1SJakub Kicinski unsigned int offset, size_t in_len) 18884c61fe1SJakub Kicinski { 18984c61fe1SJakub Kicinski struct tls_strparser *strp = (struct tls_strparser *)desc->arg.data; 19084c61fe1SJakub Kicinski struct sk_buff *skb; 19184c61fe1SJakub Kicinski skb_frag_t *frag; 1928fd1e151SYang Li size_t len, chunk; 1938fd1e151SYang Li int sz; 19484c61fe1SJakub Kicinski 19584c61fe1SJakub Kicinski if (strp->msg_ready) 19684c61fe1SJakub Kicinski return 0; 19784c61fe1SJakub Kicinski 19884c61fe1SJakub Kicinski skb = strp->anchor; 19984c61fe1SJakub Kicinski frag = &skb_shinfo(skb)->frags[skb->len / PAGE_SIZE]; 20084c61fe1SJakub Kicinski 20184c61fe1SJakub Kicinski len = in_len; 20284c61fe1SJakub Kicinski /* First make sure we got the header */ 20384c61fe1SJakub Kicinski if (!strp->stm.full_len) { 20484c61fe1SJakub Kicinski /* Assume one page is more than enough for headers */ 20584c61fe1SJakub Kicinski chunk = min_t(size_t, len, PAGE_SIZE - skb_frag_size(frag)); 20684c61fe1SJakub Kicinski WARN_ON_ONCE(skb_copy_bits(in_skb, offset, 20784c61fe1SJakub Kicinski skb_frag_address(frag) + 20884c61fe1SJakub Kicinski skb_frag_size(frag), 20984c61fe1SJakub Kicinski chunk)); 21084c61fe1SJakub Kicinski 21184c61fe1SJakub Kicinski sz = tls_rx_msg_size(strp, strp->anchor); 21284c61fe1SJakub Kicinski if (sz < 0) { 21384c61fe1SJakub Kicinski desc->error = sz; 21484c61fe1SJakub Kicinski return 0; 21584c61fe1SJakub Kicinski } 21684c61fe1SJakub Kicinski 21784c61fe1SJakub Kicinski /* We may have over-read, sz == 0 is guaranteed under-read */ 21884c61fe1SJakub Kicinski if (sz > 0) 21984c61fe1SJakub Kicinski chunk = min_t(size_t, chunk, sz - skb->len); 22084c61fe1SJakub Kicinski 22184c61fe1SJakub Kicinski skb->len += chunk; 22284c61fe1SJakub Kicinski skb->data_len += chunk; 22384c61fe1SJakub Kicinski skb_frag_size_add(frag, chunk); 22484c61fe1SJakub Kicinski frag++; 22584c61fe1SJakub Kicinski len -= chunk; 22684c61fe1SJakub Kicinski offset += chunk; 22784c61fe1SJakub Kicinski 22884c61fe1SJakub Kicinski strp->stm.full_len = sz; 22984c61fe1SJakub Kicinski if (!strp->stm.full_len) 23084c61fe1SJakub Kicinski goto read_done; 23184c61fe1SJakub Kicinski } 23284c61fe1SJakub Kicinski 23384c61fe1SJakub Kicinski /* Load up more data */ 23484c61fe1SJakub Kicinski while (len && strp->stm.full_len > skb->len) { 23584c61fe1SJakub Kicinski chunk = min_t(size_t, len, strp->stm.full_len - skb->len); 23684c61fe1SJakub Kicinski chunk = min_t(size_t, chunk, PAGE_SIZE - skb_frag_size(frag)); 23784c61fe1SJakub Kicinski WARN_ON_ONCE(skb_copy_bits(in_skb, offset, 23884c61fe1SJakub Kicinski skb_frag_address(frag) + 23984c61fe1SJakub Kicinski skb_frag_size(frag), 24084c61fe1SJakub Kicinski chunk)); 24184c61fe1SJakub Kicinski 24284c61fe1SJakub Kicinski skb->len += chunk; 24384c61fe1SJakub Kicinski skb->data_len += chunk; 24484c61fe1SJakub Kicinski skb_frag_size_add(frag, chunk); 24584c61fe1SJakub Kicinski frag++; 24684c61fe1SJakub Kicinski len -= chunk; 24784c61fe1SJakub Kicinski offset += chunk; 24884c61fe1SJakub Kicinski } 24984c61fe1SJakub Kicinski 25084c61fe1SJakub Kicinski if (strp->stm.full_len == skb->len) { 25184c61fe1SJakub Kicinski desc->count = 0; 25284c61fe1SJakub Kicinski 25384c61fe1SJakub Kicinski strp->msg_ready = 1; 25484c61fe1SJakub Kicinski tls_rx_msg_ready(strp); 25584c61fe1SJakub Kicinski } 25684c61fe1SJakub Kicinski 25784c61fe1SJakub Kicinski read_done: 25884c61fe1SJakub Kicinski return in_len - len; 25984c61fe1SJakub Kicinski } 26084c61fe1SJakub Kicinski 26184c61fe1SJakub Kicinski static int tls_strp_read_copyin(struct tls_strparser *strp) 26284c61fe1SJakub Kicinski { 26384c61fe1SJakub Kicinski struct socket *sock = strp->sk->sk_socket; 26484c61fe1SJakub Kicinski read_descriptor_t desc; 26584c61fe1SJakub Kicinski 26684c61fe1SJakub Kicinski desc.arg.data = strp; 26784c61fe1SJakub Kicinski desc.error = 0; 26884c61fe1SJakub Kicinski desc.count = 1; /* give more than one skb per call */ 26984c61fe1SJakub Kicinski 27084c61fe1SJakub Kicinski /* sk should be locked here, so okay to do read_sock */ 27184c61fe1SJakub Kicinski sock->ops->read_sock(strp->sk, &desc, tls_strp_copyin); 27284c61fe1SJakub Kicinski 27384c61fe1SJakub Kicinski return desc.error; 27484c61fe1SJakub Kicinski } 27584c61fe1SJakub Kicinski 276*0d87bbd3SJakub Kicinski static int tls_strp_read_copy(struct tls_strparser *strp, bool qshort) 27784c61fe1SJakub Kicinski { 27884c61fe1SJakub Kicinski struct skb_shared_info *shinfo; 27984c61fe1SJakub Kicinski struct page *page; 28084c61fe1SJakub Kicinski int need_spc, len; 28184c61fe1SJakub Kicinski 28284c61fe1SJakub Kicinski /* If the rbuf is small or rcv window has collapsed to 0 we need 28384c61fe1SJakub Kicinski * to read the data out. Otherwise the connection will stall. 28484c61fe1SJakub Kicinski * Without pressure threshold of INT_MAX will never be ready. 28584c61fe1SJakub Kicinski */ 286*0d87bbd3SJakub Kicinski if (likely(qshort && !tcp_epollin_ready(strp->sk, INT_MAX))) 28784c61fe1SJakub Kicinski return 0; 28884c61fe1SJakub Kicinski 28984c61fe1SJakub Kicinski shinfo = skb_shinfo(strp->anchor); 29084c61fe1SJakub Kicinski shinfo->frag_list = NULL; 29184c61fe1SJakub Kicinski 29284c61fe1SJakub Kicinski /* If we don't know the length go max plus page for cipher overhead */ 29384c61fe1SJakub Kicinski need_spc = strp->stm.full_len ?: TLS_MAX_PAYLOAD_SIZE + PAGE_SIZE; 29484c61fe1SJakub Kicinski 29584c61fe1SJakub Kicinski for (len = need_spc; len > 0; len -= PAGE_SIZE) { 29684c61fe1SJakub Kicinski page = alloc_page(strp->sk->sk_allocation); 29784c61fe1SJakub Kicinski if (!page) { 29884c61fe1SJakub Kicinski tls_strp_flush_anchor_copy(strp); 29984c61fe1SJakub Kicinski return -ENOMEM; 30084c61fe1SJakub Kicinski } 30184c61fe1SJakub Kicinski 30284c61fe1SJakub Kicinski skb_fill_page_desc(strp->anchor, shinfo->nr_frags++, 30384c61fe1SJakub Kicinski page, 0, 0); 30484c61fe1SJakub Kicinski } 30584c61fe1SJakub Kicinski 30684c61fe1SJakub Kicinski strp->copy_mode = 1; 30784c61fe1SJakub Kicinski strp->stm.offset = 0; 30884c61fe1SJakub Kicinski 30984c61fe1SJakub Kicinski strp->anchor->len = 0; 31084c61fe1SJakub Kicinski strp->anchor->data_len = 0; 31184c61fe1SJakub Kicinski strp->anchor->truesize = round_up(need_spc, PAGE_SIZE); 31284c61fe1SJakub Kicinski 31384c61fe1SJakub Kicinski tls_strp_read_copyin(strp); 31484c61fe1SJakub Kicinski 31584c61fe1SJakub Kicinski return 0; 31684c61fe1SJakub Kicinski } 31784c61fe1SJakub Kicinski 318*0d87bbd3SJakub Kicinski static bool tls_strp_check_no_dup(struct tls_strparser *strp) 319*0d87bbd3SJakub Kicinski { 320*0d87bbd3SJakub Kicinski unsigned int len = strp->stm.offset + strp->stm.full_len; 321*0d87bbd3SJakub Kicinski struct sk_buff *skb; 322*0d87bbd3SJakub Kicinski u32 seq; 323*0d87bbd3SJakub Kicinski 324*0d87bbd3SJakub Kicinski skb = skb_shinfo(strp->anchor)->frag_list; 325*0d87bbd3SJakub Kicinski seq = TCP_SKB_CB(skb)->seq; 326*0d87bbd3SJakub Kicinski 327*0d87bbd3SJakub Kicinski while (skb->len < len) { 328*0d87bbd3SJakub Kicinski seq += skb->len; 329*0d87bbd3SJakub Kicinski len -= skb->len; 330*0d87bbd3SJakub Kicinski skb = skb->next; 331*0d87bbd3SJakub Kicinski 332*0d87bbd3SJakub Kicinski if (TCP_SKB_CB(skb)->seq != seq) 333*0d87bbd3SJakub Kicinski return false; 334*0d87bbd3SJakub Kicinski } 335*0d87bbd3SJakub Kicinski 336*0d87bbd3SJakub Kicinski return true; 337*0d87bbd3SJakub Kicinski } 338*0d87bbd3SJakub Kicinski 33984c61fe1SJakub Kicinski static void tls_strp_load_anchor_with_queue(struct tls_strparser *strp, int len) 34084c61fe1SJakub Kicinski { 34184c61fe1SJakub Kicinski struct tcp_sock *tp = tcp_sk(strp->sk); 34284c61fe1SJakub Kicinski struct sk_buff *first; 34384c61fe1SJakub Kicinski u32 offset; 34484c61fe1SJakub Kicinski 34584c61fe1SJakub Kicinski first = tcp_recv_skb(strp->sk, tp->copied_seq, &offset); 34684c61fe1SJakub Kicinski if (WARN_ON_ONCE(!first)) 34784c61fe1SJakub Kicinski return; 34884c61fe1SJakub Kicinski 34984c61fe1SJakub Kicinski /* Bestow the state onto the anchor */ 35084c61fe1SJakub Kicinski strp->anchor->len = offset + len; 35184c61fe1SJakub Kicinski strp->anchor->data_len = offset + len; 35284c61fe1SJakub Kicinski strp->anchor->truesize = offset + len; 35384c61fe1SJakub Kicinski 35484c61fe1SJakub Kicinski skb_shinfo(strp->anchor)->frag_list = first; 35584c61fe1SJakub Kicinski 35684c61fe1SJakub Kicinski skb_copy_header(strp->anchor, first); 35784c61fe1SJakub Kicinski strp->anchor->destructor = NULL; 35884c61fe1SJakub Kicinski 35984c61fe1SJakub Kicinski strp->stm.offset = offset; 36084c61fe1SJakub Kicinski } 36184c61fe1SJakub Kicinski 36284c61fe1SJakub Kicinski void tls_strp_msg_load(struct tls_strparser *strp, bool force_refresh) 36384c61fe1SJakub Kicinski { 36484c61fe1SJakub Kicinski struct strp_msg *rxm; 36584c61fe1SJakub Kicinski struct tls_msg *tlm; 36684c61fe1SJakub Kicinski 36784c61fe1SJakub Kicinski DEBUG_NET_WARN_ON_ONCE(!strp->msg_ready); 36884c61fe1SJakub Kicinski DEBUG_NET_WARN_ON_ONCE(!strp->stm.full_len); 36984c61fe1SJakub Kicinski 37084c61fe1SJakub Kicinski if (!strp->copy_mode && force_refresh) { 37184c61fe1SJakub Kicinski if (WARN_ON(tcp_inq(strp->sk) < strp->stm.full_len)) 37284c61fe1SJakub Kicinski return; 37384c61fe1SJakub Kicinski 37484c61fe1SJakub Kicinski tls_strp_load_anchor_with_queue(strp, strp->stm.full_len); 37584c61fe1SJakub Kicinski } 37684c61fe1SJakub Kicinski 37784c61fe1SJakub Kicinski rxm = strp_msg(strp->anchor); 37884c61fe1SJakub Kicinski rxm->full_len = strp->stm.full_len; 37984c61fe1SJakub Kicinski rxm->offset = strp->stm.offset; 38084c61fe1SJakub Kicinski tlm = tls_msg(strp->anchor); 38184c61fe1SJakub Kicinski tlm->control = strp->mark; 38284c61fe1SJakub Kicinski } 38384c61fe1SJakub Kicinski 38484c61fe1SJakub Kicinski /* Called with lock held on lower socket */ 38584c61fe1SJakub Kicinski static int tls_strp_read_sock(struct tls_strparser *strp) 38684c61fe1SJakub Kicinski { 38784c61fe1SJakub Kicinski int sz, inq; 38884c61fe1SJakub Kicinski 38984c61fe1SJakub Kicinski inq = tcp_inq(strp->sk); 39084c61fe1SJakub Kicinski if (inq < 1) 39184c61fe1SJakub Kicinski return 0; 39284c61fe1SJakub Kicinski 39384c61fe1SJakub Kicinski if (unlikely(strp->copy_mode)) 39484c61fe1SJakub Kicinski return tls_strp_read_copyin(strp); 39584c61fe1SJakub Kicinski 39684c61fe1SJakub Kicinski if (inq < strp->stm.full_len) 397*0d87bbd3SJakub Kicinski return tls_strp_read_copy(strp, true); 39884c61fe1SJakub Kicinski 39984c61fe1SJakub Kicinski if (!strp->stm.full_len) { 40084c61fe1SJakub Kicinski tls_strp_load_anchor_with_queue(strp, inq); 40184c61fe1SJakub Kicinski 40284c61fe1SJakub Kicinski sz = tls_rx_msg_size(strp, strp->anchor); 40384c61fe1SJakub Kicinski if (sz < 0) { 40484c61fe1SJakub Kicinski tls_strp_abort_strp(strp, sz); 40584c61fe1SJakub Kicinski return sz; 40684c61fe1SJakub Kicinski } 40784c61fe1SJakub Kicinski 40884c61fe1SJakub Kicinski strp->stm.full_len = sz; 40984c61fe1SJakub Kicinski 41084c61fe1SJakub Kicinski if (!strp->stm.full_len || inq < strp->stm.full_len) 411*0d87bbd3SJakub Kicinski return tls_strp_read_copy(strp, true); 41284c61fe1SJakub Kicinski } 41384c61fe1SJakub Kicinski 414*0d87bbd3SJakub Kicinski if (!tls_strp_check_no_dup(strp)) 415*0d87bbd3SJakub Kicinski return tls_strp_read_copy(strp, false); 416*0d87bbd3SJakub Kicinski 41784c61fe1SJakub Kicinski strp->msg_ready = 1; 41884c61fe1SJakub Kicinski tls_rx_msg_ready(strp); 41984c61fe1SJakub Kicinski 42084c61fe1SJakub Kicinski return 0; 42184c61fe1SJakub Kicinski } 42284c61fe1SJakub Kicinski 42384c61fe1SJakub Kicinski void tls_strp_check_rcv(struct tls_strparser *strp) 42484c61fe1SJakub Kicinski { 42584c61fe1SJakub Kicinski if (unlikely(strp->stopped) || strp->msg_ready) 42684c61fe1SJakub Kicinski return; 42784c61fe1SJakub Kicinski 42884c61fe1SJakub Kicinski if (tls_strp_read_sock(strp) == -ENOMEM) 42984c61fe1SJakub Kicinski queue_work(tls_strp_wq, &strp->work); 43084c61fe1SJakub Kicinski } 43184c61fe1SJakub Kicinski 43284c61fe1SJakub Kicinski /* Lower sock lock held */ 43384c61fe1SJakub Kicinski void tls_strp_data_ready(struct tls_strparser *strp) 43484c61fe1SJakub Kicinski { 43584c61fe1SJakub Kicinski /* This check is needed to synchronize with do_tls_strp_work. 43684c61fe1SJakub Kicinski * do_tls_strp_work acquires a process lock (lock_sock) whereas 43784c61fe1SJakub Kicinski * the lock held here is bh_lock_sock. The two locks can be 43884c61fe1SJakub Kicinski * held by different threads at the same time, but bh_lock_sock 43984c61fe1SJakub Kicinski * allows a thread in BH context to safely check if the process 44084c61fe1SJakub Kicinski * lock is held. In this case, if the lock is held, queue work. 44184c61fe1SJakub Kicinski */ 44284c61fe1SJakub Kicinski if (sock_owned_by_user_nocheck(strp->sk)) { 44384c61fe1SJakub Kicinski queue_work(tls_strp_wq, &strp->work); 44484c61fe1SJakub Kicinski return; 44584c61fe1SJakub Kicinski } 44684c61fe1SJakub Kicinski 44784c61fe1SJakub Kicinski tls_strp_check_rcv(strp); 44884c61fe1SJakub Kicinski } 44984c61fe1SJakub Kicinski 45084c61fe1SJakub Kicinski static void tls_strp_work(struct work_struct *w) 45184c61fe1SJakub Kicinski { 45284c61fe1SJakub Kicinski struct tls_strparser *strp = 45384c61fe1SJakub Kicinski container_of(w, struct tls_strparser, work); 45484c61fe1SJakub Kicinski 45584c61fe1SJakub Kicinski lock_sock(strp->sk); 45684c61fe1SJakub Kicinski tls_strp_check_rcv(strp); 45784c61fe1SJakub Kicinski release_sock(strp->sk); 45884c61fe1SJakub Kicinski } 45984c61fe1SJakub Kicinski 46084c61fe1SJakub Kicinski void tls_strp_msg_done(struct tls_strparser *strp) 46184c61fe1SJakub Kicinski { 46284c61fe1SJakub Kicinski WARN_ON(!strp->stm.full_len); 46384c61fe1SJakub Kicinski 46484c61fe1SJakub Kicinski if (likely(!strp->copy_mode)) 46584c61fe1SJakub Kicinski tcp_read_done(strp->sk, strp->stm.full_len); 46684c61fe1SJakub Kicinski else 46784c61fe1SJakub Kicinski tls_strp_flush_anchor_copy(strp); 46884c61fe1SJakub Kicinski 46984c61fe1SJakub Kicinski strp->msg_ready = 0; 47084c61fe1SJakub Kicinski memset(&strp->stm, 0, sizeof(strp->stm)); 47184c61fe1SJakub Kicinski 47284c61fe1SJakub Kicinski tls_strp_check_rcv(strp); 47384c61fe1SJakub Kicinski } 47484c61fe1SJakub Kicinski 47584c61fe1SJakub Kicinski void tls_strp_stop(struct tls_strparser *strp) 47684c61fe1SJakub Kicinski { 47784c61fe1SJakub Kicinski strp->stopped = 1; 47884c61fe1SJakub Kicinski } 47984c61fe1SJakub Kicinski 48084c61fe1SJakub Kicinski int tls_strp_init(struct tls_strparser *strp, struct sock *sk) 48184c61fe1SJakub Kicinski { 48284c61fe1SJakub Kicinski memset(strp, 0, sizeof(*strp)); 48384c61fe1SJakub Kicinski 48484c61fe1SJakub Kicinski strp->sk = sk; 48584c61fe1SJakub Kicinski 48684c61fe1SJakub Kicinski strp->anchor = alloc_skb(0, GFP_KERNEL); 48784c61fe1SJakub Kicinski if (!strp->anchor) 48884c61fe1SJakub Kicinski return -ENOMEM; 48984c61fe1SJakub Kicinski 49084c61fe1SJakub Kicinski INIT_WORK(&strp->work, tls_strp_work); 49184c61fe1SJakub Kicinski 49284c61fe1SJakub Kicinski return 0; 49384c61fe1SJakub Kicinski } 49484c61fe1SJakub Kicinski 49584c61fe1SJakub Kicinski /* strp must already be stopped so that tls_strp_recv will no longer be called. 49684c61fe1SJakub Kicinski * Note that tls_strp_done is not called with the lower socket held. 49784c61fe1SJakub Kicinski */ 49884c61fe1SJakub Kicinski void tls_strp_done(struct tls_strparser *strp) 49984c61fe1SJakub Kicinski { 50084c61fe1SJakub Kicinski WARN_ON(!strp->stopped); 50184c61fe1SJakub Kicinski 50284c61fe1SJakub Kicinski cancel_work_sync(&strp->work); 50384c61fe1SJakub Kicinski tls_strp_anchor_free(strp); 50484c61fe1SJakub Kicinski } 50584c61fe1SJakub Kicinski 50684c61fe1SJakub Kicinski int __init tls_strp_dev_init(void) 50784c61fe1SJakub Kicinski { 508d11ef9ccSJakub Kicinski tls_strp_wq = create_workqueue("tls-strp"); 50984c61fe1SJakub Kicinski if (unlikely(!tls_strp_wq)) 51084c61fe1SJakub Kicinski return -ENOMEM; 51184c61fe1SJakub Kicinski 51284c61fe1SJakub Kicinski return 0; 51384c61fe1SJakub Kicinski } 51484c61fe1SJakub Kicinski 51584c61fe1SJakub Kicinski void tls_strp_dev_exit(void) 51684c61fe1SJakub Kicinski { 51784c61fe1SJakub Kicinski destroy_workqueue(tls_strp_wq); 51884c61fe1SJakub Kicinski } 519