1c618db2aSJakub Kicinski // SPDX-License-Identifier: GPL-2.0-only 284c61fe1SJakub Kicinski /* Copyright (c) 2016 Tom Herbert <tom@herbertland.com> */ 3c618db2aSJakub Kicinski 4c618db2aSJakub Kicinski #include <linux/skbuff.h> 584c61fe1SJakub Kicinski #include <linux/workqueue.h> 684c61fe1SJakub Kicinski #include <net/strparser.h> 784c61fe1SJakub Kicinski #include <net/tcp.h> 884c61fe1SJakub Kicinski #include <net/sock.h> 984c61fe1SJakub Kicinski #include <net/tls.h> 10c618db2aSJakub Kicinski 11c618db2aSJakub Kicinski #include "tls.h" 12c618db2aSJakub Kicinski 1384c61fe1SJakub Kicinski static struct workqueue_struct *tls_strp_wq; 14d4e5db64SJakub Kicinski 1584c61fe1SJakub Kicinski static void tls_strp_abort_strp(struct tls_strparser *strp, int err) 1684c61fe1SJakub Kicinski { 1784c61fe1SJakub Kicinski if (strp->stopped) 1884c61fe1SJakub Kicinski return; 1984c61fe1SJakub Kicinski 2084c61fe1SJakub Kicinski strp->stopped = 1; 2184c61fe1SJakub Kicinski 2284c61fe1SJakub Kicinski /* Report an error on the lower socket */ 2384c61fe1SJakub Kicinski strp->sk->sk_err = -err; 2484c61fe1SJakub Kicinski sk_error_report(strp->sk); 2584c61fe1SJakub Kicinski } 2684c61fe1SJakub Kicinski 2784c61fe1SJakub Kicinski static void tls_strp_anchor_free(struct tls_strparser *strp) 2884c61fe1SJakub Kicinski { 2984c61fe1SJakub Kicinski struct skb_shared_info *shinfo = skb_shinfo(strp->anchor); 3084c61fe1SJakub Kicinski 3184c61fe1SJakub Kicinski DEBUG_NET_WARN_ON_ONCE(atomic_read(&shinfo->dataref) != 1); 3284c61fe1SJakub Kicinski shinfo->frag_list = NULL; 3384c61fe1SJakub Kicinski consume_skb(strp->anchor); 3484c61fe1SJakub Kicinski strp->anchor = NULL; 3584c61fe1SJakub Kicinski } 3684c61fe1SJakub Kicinski 3784c61fe1SJakub Kicinski /* Create a new skb with the contents of input copied to its page frags */ 3884c61fe1SJakub Kicinski static struct sk_buff *tls_strp_msg_make_copy(struct tls_strparser *strp) 3984c61fe1SJakub Kicinski { 4084c61fe1SJakub Kicinski struct strp_msg *rxm; 4184c61fe1SJakub Kicinski struct sk_buff *skb; 4284c61fe1SJakub Kicinski int i, err, offset; 4384c61fe1SJakub Kicinski 44d800a7b3SJakub Kicinski skb = alloc_skb_with_frags(0, strp->stm.full_len, TLS_PAGE_ORDER, 4584c61fe1SJakub Kicinski &err, strp->sk->sk_allocation); 4684c61fe1SJakub Kicinski if (!skb) 4784c61fe1SJakub Kicinski return NULL; 4884c61fe1SJakub Kicinski 4984c61fe1SJakub Kicinski offset = strp->stm.offset; 5084c61fe1SJakub Kicinski for (i = 0; i < skb_shinfo(skb)->nr_frags; i++) { 5184c61fe1SJakub Kicinski skb_frag_t *frag = &skb_shinfo(skb)->frags[i]; 5284c61fe1SJakub Kicinski 5384c61fe1SJakub Kicinski WARN_ON_ONCE(skb_copy_bits(strp->anchor, offset, 5484c61fe1SJakub Kicinski skb_frag_address(frag), 5584c61fe1SJakub Kicinski skb_frag_size(frag))); 5684c61fe1SJakub Kicinski offset += skb_frag_size(frag); 5784c61fe1SJakub Kicinski } 5884c61fe1SJakub Kicinski 59210620aeSJakub Kicinski skb->len = strp->stm.full_len; 60210620aeSJakub Kicinski skb->data_len = strp->stm.full_len; 6184c61fe1SJakub Kicinski skb_copy_header(skb, strp->anchor); 6284c61fe1SJakub Kicinski rxm = strp_msg(skb); 6384c61fe1SJakub Kicinski rxm->offset = 0; 64d4e5db64SJakub Kicinski return skb; 65d4e5db64SJakub Kicinski } 66d4e5db64SJakub Kicinski 6784c61fe1SJakub Kicinski /* Steal the input skb, input msg is invalid after calling this function */ 6884c61fe1SJakub Kicinski struct sk_buff *tls_strp_msg_detach(struct tls_sw_context_rx *ctx) 6984c61fe1SJakub Kicinski { 7084c61fe1SJakub Kicinski struct tls_strparser *strp = &ctx->strp; 7184c61fe1SJakub Kicinski 7284c61fe1SJakub Kicinski #ifdef CONFIG_TLS_DEVICE 7384c61fe1SJakub Kicinski DEBUG_NET_WARN_ON_ONCE(!strp->anchor->decrypted); 7484c61fe1SJakub Kicinski #else 7584c61fe1SJakub Kicinski /* This function turns an input into an output, 7684c61fe1SJakub Kicinski * that can only happen if we have offload. 7784c61fe1SJakub Kicinski */ 7884c61fe1SJakub Kicinski WARN_ON(1); 7984c61fe1SJakub Kicinski #endif 8084c61fe1SJakub Kicinski 8184c61fe1SJakub Kicinski if (strp->copy_mode) { 8284c61fe1SJakub Kicinski struct sk_buff *skb; 8384c61fe1SJakub Kicinski 8484c61fe1SJakub Kicinski /* Replace anchor with an empty skb, this is a little 8584c61fe1SJakub Kicinski * dangerous but __tls_cur_msg() warns on empty skbs 8684c61fe1SJakub Kicinski * so hopefully we'll catch abuses. 8784c61fe1SJakub Kicinski */ 8884c61fe1SJakub Kicinski skb = alloc_skb(0, strp->sk->sk_allocation); 8984c61fe1SJakub Kicinski if (!skb) 9084c61fe1SJakub Kicinski return NULL; 9184c61fe1SJakub Kicinski 9284c61fe1SJakub Kicinski swap(strp->anchor, skb); 9384c61fe1SJakub Kicinski return skb; 9484c61fe1SJakub Kicinski } 9584c61fe1SJakub Kicinski 9684c61fe1SJakub Kicinski return tls_strp_msg_make_copy(strp); 9784c61fe1SJakub Kicinski } 9884c61fe1SJakub Kicinski 9984c61fe1SJakub Kicinski /* Force the input skb to be in copy mode. The data ownership remains 10084c61fe1SJakub Kicinski * with the input skb itself (meaning unpause will wipe it) but it can 10184c61fe1SJakub Kicinski * be modified. 10284c61fe1SJakub Kicinski */ 1038b3c59a7SJakub Kicinski int tls_strp_msg_cow(struct tls_sw_context_rx *ctx) 1048b3c59a7SJakub Kicinski { 10584c61fe1SJakub Kicinski struct tls_strparser *strp = &ctx->strp; 10684c61fe1SJakub Kicinski struct sk_buff *skb; 1078b3c59a7SJakub Kicinski 10884c61fe1SJakub Kicinski if (strp->copy_mode) 10984c61fe1SJakub Kicinski return 0; 11084c61fe1SJakub Kicinski 11184c61fe1SJakub Kicinski skb = tls_strp_msg_make_copy(strp); 11284c61fe1SJakub Kicinski if (!skb) 11384c61fe1SJakub Kicinski return -ENOMEM; 11484c61fe1SJakub Kicinski 11584c61fe1SJakub Kicinski tls_strp_anchor_free(strp); 11684c61fe1SJakub Kicinski strp->anchor = skb; 11784c61fe1SJakub Kicinski 11884c61fe1SJakub Kicinski tcp_read_done(strp->sk, strp->stm.full_len); 11984c61fe1SJakub Kicinski strp->copy_mode = 1; 12084c61fe1SJakub Kicinski 1218b3c59a7SJakub Kicinski return 0; 1228b3c59a7SJakub Kicinski } 1238b3c59a7SJakub Kicinski 12484c61fe1SJakub Kicinski /* Make a clone (in the skb sense) of the input msg to keep a reference 12584c61fe1SJakub Kicinski * to the underlying data. The reference-holding skbs get placed on 12684c61fe1SJakub Kicinski * @dst. 12784c61fe1SJakub Kicinski */ 12884c61fe1SJakub Kicinski int tls_strp_msg_hold(struct tls_strparser *strp, struct sk_buff_head *dst) 129c618db2aSJakub Kicinski { 13084c61fe1SJakub Kicinski struct skb_shared_info *shinfo = skb_shinfo(strp->anchor); 131c618db2aSJakub Kicinski 13284c61fe1SJakub Kicinski if (strp->copy_mode) { 13384c61fe1SJakub Kicinski struct sk_buff *skb; 13484c61fe1SJakub Kicinski 13584c61fe1SJakub Kicinski WARN_ON_ONCE(!shinfo->nr_frags); 13684c61fe1SJakub Kicinski 13784c61fe1SJakub Kicinski /* We can't skb_clone() the anchor, it gets wiped by unpause */ 13884c61fe1SJakub Kicinski skb = alloc_skb(0, strp->sk->sk_allocation); 13984c61fe1SJakub Kicinski if (!skb) 14084c61fe1SJakub Kicinski return -ENOMEM; 14184c61fe1SJakub Kicinski 14284c61fe1SJakub Kicinski __skb_queue_tail(dst, strp->anchor); 14384c61fe1SJakub Kicinski strp->anchor = skb; 14484c61fe1SJakub Kicinski } else { 14584c61fe1SJakub Kicinski struct sk_buff *iter, *clone; 14684c61fe1SJakub Kicinski int chunk, len, offset; 14784c61fe1SJakub Kicinski 14884c61fe1SJakub Kicinski offset = strp->stm.offset; 14984c61fe1SJakub Kicinski len = strp->stm.full_len; 15084c61fe1SJakub Kicinski iter = shinfo->frag_list; 15184c61fe1SJakub Kicinski 15284c61fe1SJakub Kicinski while (len > 0) { 15384c61fe1SJakub Kicinski if (iter->len <= offset) { 15484c61fe1SJakub Kicinski offset -= iter->len; 15584c61fe1SJakub Kicinski goto next; 15684c61fe1SJakub Kicinski } 15784c61fe1SJakub Kicinski 15884c61fe1SJakub Kicinski chunk = iter->len - offset; 15984c61fe1SJakub Kicinski offset = 0; 16084c61fe1SJakub Kicinski 16184c61fe1SJakub Kicinski clone = skb_clone(iter, strp->sk->sk_allocation); 162c618db2aSJakub Kicinski if (!clone) 163c618db2aSJakub Kicinski return -ENOMEM; 164c618db2aSJakub Kicinski __skb_queue_tail(dst, clone); 16584c61fe1SJakub Kicinski 16684c61fe1SJakub Kicinski len -= chunk; 16784c61fe1SJakub Kicinski next: 16884c61fe1SJakub Kicinski iter = iter->next; 16984c61fe1SJakub Kicinski } 17084c61fe1SJakub Kicinski } 17184c61fe1SJakub Kicinski 172c618db2aSJakub Kicinski return 0; 173c618db2aSJakub Kicinski } 17484c61fe1SJakub Kicinski 17584c61fe1SJakub Kicinski static void tls_strp_flush_anchor_copy(struct tls_strparser *strp) 17684c61fe1SJakub Kicinski { 17784c61fe1SJakub Kicinski struct skb_shared_info *shinfo = skb_shinfo(strp->anchor); 17884c61fe1SJakub Kicinski int i; 17984c61fe1SJakub Kicinski 18084c61fe1SJakub Kicinski DEBUG_NET_WARN_ON_ONCE(atomic_read(&shinfo->dataref) != 1); 18184c61fe1SJakub Kicinski 18284c61fe1SJakub Kicinski for (i = 0; i < shinfo->nr_frags; i++) 18384c61fe1SJakub Kicinski __skb_frag_unref(&shinfo->frags[i], false); 18484c61fe1SJakub Kicinski shinfo->nr_frags = 0; 18584c61fe1SJakub Kicinski strp->copy_mode = 0; 18684c61fe1SJakub Kicinski } 18784c61fe1SJakub Kicinski 18884c61fe1SJakub Kicinski static int tls_strp_copyin(read_descriptor_t *desc, struct sk_buff *in_skb, 18984c61fe1SJakub Kicinski unsigned int offset, size_t in_len) 19084c61fe1SJakub Kicinski { 19184c61fe1SJakub Kicinski struct tls_strparser *strp = (struct tls_strparser *)desc->arg.data; 19284c61fe1SJakub Kicinski struct sk_buff *skb; 19384c61fe1SJakub Kicinski skb_frag_t *frag; 1948fd1e151SYang Li size_t len, chunk; 1958fd1e151SYang Li int sz; 19684c61fe1SJakub Kicinski 19784c61fe1SJakub Kicinski if (strp->msg_ready) 19884c61fe1SJakub Kicinski return 0; 19984c61fe1SJakub Kicinski 20084c61fe1SJakub Kicinski skb = strp->anchor; 20184c61fe1SJakub Kicinski frag = &skb_shinfo(skb)->frags[skb->len / PAGE_SIZE]; 20284c61fe1SJakub Kicinski 20384c61fe1SJakub Kicinski len = in_len; 20484c61fe1SJakub Kicinski /* First make sure we got the header */ 20584c61fe1SJakub Kicinski if (!strp->stm.full_len) { 20684c61fe1SJakub Kicinski /* Assume one page is more than enough for headers */ 20784c61fe1SJakub Kicinski chunk = min_t(size_t, len, PAGE_SIZE - skb_frag_size(frag)); 20884c61fe1SJakub Kicinski WARN_ON_ONCE(skb_copy_bits(in_skb, offset, 20984c61fe1SJakub Kicinski skb_frag_address(frag) + 21084c61fe1SJakub Kicinski skb_frag_size(frag), 21184c61fe1SJakub Kicinski chunk)); 21284c61fe1SJakub Kicinski 21384c61fe1SJakub Kicinski sz = tls_rx_msg_size(strp, strp->anchor); 21484c61fe1SJakub Kicinski if (sz < 0) { 21584c61fe1SJakub Kicinski desc->error = sz; 21684c61fe1SJakub Kicinski return 0; 21784c61fe1SJakub Kicinski } 21884c61fe1SJakub Kicinski 21984c61fe1SJakub Kicinski /* We may have over-read, sz == 0 is guaranteed under-read */ 22084c61fe1SJakub Kicinski if (sz > 0) 22184c61fe1SJakub Kicinski chunk = min_t(size_t, chunk, sz - skb->len); 22284c61fe1SJakub Kicinski 22384c61fe1SJakub Kicinski skb->len += chunk; 22484c61fe1SJakub Kicinski skb->data_len += chunk; 22584c61fe1SJakub Kicinski skb_frag_size_add(frag, chunk); 22684c61fe1SJakub Kicinski frag++; 22784c61fe1SJakub Kicinski len -= chunk; 22884c61fe1SJakub Kicinski offset += chunk; 22984c61fe1SJakub Kicinski 23084c61fe1SJakub Kicinski strp->stm.full_len = sz; 23184c61fe1SJakub Kicinski if (!strp->stm.full_len) 23284c61fe1SJakub Kicinski goto read_done; 23384c61fe1SJakub Kicinski } 23484c61fe1SJakub Kicinski 23584c61fe1SJakub Kicinski /* Load up more data */ 23684c61fe1SJakub Kicinski while (len && strp->stm.full_len > skb->len) { 23784c61fe1SJakub Kicinski chunk = min_t(size_t, len, strp->stm.full_len - skb->len); 23884c61fe1SJakub Kicinski chunk = min_t(size_t, chunk, PAGE_SIZE - skb_frag_size(frag)); 23984c61fe1SJakub Kicinski WARN_ON_ONCE(skb_copy_bits(in_skb, offset, 24084c61fe1SJakub Kicinski skb_frag_address(frag) + 24184c61fe1SJakub Kicinski skb_frag_size(frag), 24284c61fe1SJakub Kicinski chunk)); 24384c61fe1SJakub Kicinski 24484c61fe1SJakub Kicinski skb->len += chunk; 24584c61fe1SJakub Kicinski skb->data_len += chunk; 24684c61fe1SJakub Kicinski skb_frag_size_add(frag, chunk); 24784c61fe1SJakub Kicinski frag++; 24884c61fe1SJakub Kicinski len -= chunk; 24984c61fe1SJakub Kicinski offset += chunk; 25084c61fe1SJakub Kicinski } 25184c61fe1SJakub Kicinski 25284c61fe1SJakub Kicinski if (strp->stm.full_len == skb->len) { 25384c61fe1SJakub Kicinski desc->count = 0; 25484c61fe1SJakub Kicinski 25584c61fe1SJakub Kicinski strp->msg_ready = 1; 25684c61fe1SJakub Kicinski tls_rx_msg_ready(strp); 25784c61fe1SJakub Kicinski } 25884c61fe1SJakub Kicinski 25984c61fe1SJakub Kicinski read_done: 26084c61fe1SJakub Kicinski return in_len - len; 26184c61fe1SJakub Kicinski } 26284c61fe1SJakub Kicinski 26384c61fe1SJakub Kicinski static int tls_strp_read_copyin(struct tls_strparser *strp) 26484c61fe1SJakub Kicinski { 26584c61fe1SJakub Kicinski struct socket *sock = strp->sk->sk_socket; 26684c61fe1SJakub Kicinski read_descriptor_t desc; 26784c61fe1SJakub Kicinski 26884c61fe1SJakub Kicinski desc.arg.data = strp; 26984c61fe1SJakub Kicinski desc.error = 0; 27084c61fe1SJakub Kicinski desc.count = 1; /* give more than one skb per call */ 27184c61fe1SJakub Kicinski 27284c61fe1SJakub Kicinski /* sk should be locked here, so okay to do read_sock */ 27384c61fe1SJakub Kicinski sock->ops->read_sock(strp->sk, &desc, tls_strp_copyin); 27484c61fe1SJakub Kicinski 27584c61fe1SJakub Kicinski return desc.error; 27684c61fe1SJakub Kicinski } 27784c61fe1SJakub Kicinski 2780d87bbd3SJakub Kicinski static int tls_strp_read_copy(struct tls_strparser *strp, bool qshort) 27984c61fe1SJakub Kicinski { 28084c61fe1SJakub Kicinski struct skb_shared_info *shinfo; 28184c61fe1SJakub Kicinski struct page *page; 28284c61fe1SJakub Kicinski int need_spc, len; 28384c61fe1SJakub Kicinski 28484c61fe1SJakub Kicinski /* If the rbuf is small or rcv window has collapsed to 0 we need 28584c61fe1SJakub Kicinski * to read the data out. Otherwise the connection will stall. 28684c61fe1SJakub Kicinski * Without pressure threshold of INT_MAX will never be ready. 28784c61fe1SJakub Kicinski */ 2880d87bbd3SJakub Kicinski if (likely(qshort && !tcp_epollin_ready(strp->sk, INT_MAX))) 28984c61fe1SJakub Kicinski return 0; 29084c61fe1SJakub Kicinski 29184c61fe1SJakub Kicinski shinfo = skb_shinfo(strp->anchor); 29284c61fe1SJakub Kicinski shinfo->frag_list = NULL; 29384c61fe1SJakub Kicinski 29484c61fe1SJakub Kicinski /* If we don't know the length go max plus page for cipher overhead */ 29584c61fe1SJakub Kicinski need_spc = strp->stm.full_len ?: TLS_MAX_PAYLOAD_SIZE + PAGE_SIZE; 29684c61fe1SJakub Kicinski 29784c61fe1SJakub Kicinski for (len = need_spc; len > 0; len -= PAGE_SIZE) { 29884c61fe1SJakub Kicinski page = alloc_page(strp->sk->sk_allocation); 29984c61fe1SJakub Kicinski if (!page) { 30084c61fe1SJakub Kicinski tls_strp_flush_anchor_copy(strp); 30184c61fe1SJakub Kicinski return -ENOMEM; 30284c61fe1SJakub Kicinski } 30384c61fe1SJakub Kicinski 30484c61fe1SJakub Kicinski skb_fill_page_desc(strp->anchor, shinfo->nr_frags++, 30584c61fe1SJakub Kicinski page, 0, 0); 30684c61fe1SJakub Kicinski } 30784c61fe1SJakub Kicinski 30884c61fe1SJakub Kicinski strp->copy_mode = 1; 30984c61fe1SJakub Kicinski strp->stm.offset = 0; 31084c61fe1SJakub Kicinski 31184c61fe1SJakub Kicinski strp->anchor->len = 0; 31284c61fe1SJakub Kicinski strp->anchor->data_len = 0; 31384c61fe1SJakub Kicinski strp->anchor->truesize = round_up(need_spc, PAGE_SIZE); 31484c61fe1SJakub Kicinski 31584c61fe1SJakub Kicinski tls_strp_read_copyin(strp); 31684c61fe1SJakub Kicinski 31784c61fe1SJakub Kicinski return 0; 31884c61fe1SJakub Kicinski } 31984c61fe1SJakub Kicinski 320*14c4be92SJakub Kicinski static bool tls_strp_check_queue_ok(struct tls_strparser *strp) 3210d87bbd3SJakub Kicinski { 3220d87bbd3SJakub Kicinski unsigned int len = strp->stm.offset + strp->stm.full_len; 323*14c4be92SJakub Kicinski struct sk_buff *first, *skb; 3240d87bbd3SJakub Kicinski u32 seq; 3250d87bbd3SJakub Kicinski 326*14c4be92SJakub Kicinski first = skb_shinfo(strp->anchor)->frag_list; 327*14c4be92SJakub Kicinski skb = first; 328*14c4be92SJakub Kicinski seq = TCP_SKB_CB(first)->seq; 3290d87bbd3SJakub Kicinski 330*14c4be92SJakub Kicinski /* Make sure there's no duplicate data in the queue, 331*14c4be92SJakub Kicinski * and the decrypted status matches. 332*14c4be92SJakub Kicinski */ 3330d87bbd3SJakub Kicinski while (skb->len < len) { 3340d87bbd3SJakub Kicinski seq += skb->len; 3350d87bbd3SJakub Kicinski len -= skb->len; 3360d87bbd3SJakub Kicinski skb = skb->next; 3370d87bbd3SJakub Kicinski 3380d87bbd3SJakub Kicinski if (TCP_SKB_CB(skb)->seq != seq) 3390d87bbd3SJakub Kicinski return false; 340*14c4be92SJakub Kicinski if (skb_cmp_decrypted(first, skb)) 341*14c4be92SJakub Kicinski return false; 3420d87bbd3SJakub Kicinski } 3430d87bbd3SJakub Kicinski 3440d87bbd3SJakub Kicinski return true; 3450d87bbd3SJakub Kicinski } 3460d87bbd3SJakub Kicinski 34784c61fe1SJakub Kicinski static void tls_strp_load_anchor_with_queue(struct tls_strparser *strp, int len) 34884c61fe1SJakub Kicinski { 34984c61fe1SJakub Kicinski struct tcp_sock *tp = tcp_sk(strp->sk); 35084c61fe1SJakub Kicinski struct sk_buff *first; 35184c61fe1SJakub Kicinski u32 offset; 35284c61fe1SJakub Kicinski 35384c61fe1SJakub Kicinski first = tcp_recv_skb(strp->sk, tp->copied_seq, &offset); 35484c61fe1SJakub Kicinski if (WARN_ON_ONCE(!first)) 35584c61fe1SJakub Kicinski return; 35684c61fe1SJakub Kicinski 35784c61fe1SJakub Kicinski /* Bestow the state onto the anchor */ 35884c61fe1SJakub Kicinski strp->anchor->len = offset + len; 35984c61fe1SJakub Kicinski strp->anchor->data_len = offset + len; 36084c61fe1SJakub Kicinski strp->anchor->truesize = offset + len; 36184c61fe1SJakub Kicinski 36284c61fe1SJakub Kicinski skb_shinfo(strp->anchor)->frag_list = first; 36384c61fe1SJakub Kicinski 36484c61fe1SJakub Kicinski skb_copy_header(strp->anchor, first); 36584c61fe1SJakub Kicinski strp->anchor->destructor = NULL; 36684c61fe1SJakub Kicinski 36784c61fe1SJakub Kicinski strp->stm.offset = offset; 36884c61fe1SJakub Kicinski } 36984c61fe1SJakub Kicinski 37084c61fe1SJakub Kicinski void tls_strp_msg_load(struct tls_strparser *strp, bool force_refresh) 37184c61fe1SJakub Kicinski { 37284c61fe1SJakub Kicinski struct strp_msg *rxm; 37384c61fe1SJakub Kicinski struct tls_msg *tlm; 37484c61fe1SJakub Kicinski 37584c61fe1SJakub Kicinski DEBUG_NET_WARN_ON_ONCE(!strp->msg_ready); 37684c61fe1SJakub Kicinski DEBUG_NET_WARN_ON_ONCE(!strp->stm.full_len); 37784c61fe1SJakub Kicinski 37884c61fe1SJakub Kicinski if (!strp->copy_mode && force_refresh) { 37984c61fe1SJakub Kicinski if (WARN_ON(tcp_inq(strp->sk) < strp->stm.full_len)) 38084c61fe1SJakub Kicinski return; 38184c61fe1SJakub Kicinski 38284c61fe1SJakub Kicinski tls_strp_load_anchor_with_queue(strp, strp->stm.full_len); 38384c61fe1SJakub Kicinski } 38484c61fe1SJakub Kicinski 38584c61fe1SJakub Kicinski rxm = strp_msg(strp->anchor); 38684c61fe1SJakub Kicinski rxm->full_len = strp->stm.full_len; 38784c61fe1SJakub Kicinski rxm->offset = strp->stm.offset; 38884c61fe1SJakub Kicinski tlm = tls_msg(strp->anchor); 38984c61fe1SJakub Kicinski tlm->control = strp->mark; 39084c61fe1SJakub Kicinski } 39184c61fe1SJakub Kicinski 39284c61fe1SJakub Kicinski /* Called with lock held on lower socket */ 39384c61fe1SJakub Kicinski static int tls_strp_read_sock(struct tls_strparser *strp) 39484c61fe1SJakub Kicinski { 39584c61fe1SJakub Kicinski int sz, inq; 39684c61fe1SJakub Kicinski 39784c61fe1SJakub Kicinski inq = tcp_inq(strp->sk); 39884c61fe1SJakub Kicinski if (inq < 1) 39984c61fe1SJakub Kicinski return 0; 40084c61fe1SJakub Kicinski 40184c61fe1SJakub Kicinski if (unlikely(strp->copy_mode)) 40284c61fe1SJakub Kicinski return tls_strp_read_copyin(strp); 40384c61fe1SJakub Kicinski 40484c61fe1SJakub Kicinski if (inq < strp->stm.full_len) 4050d87bbd3SJakub Kicinski return tls_strp_read_copy(strp, true); 40684c61fe1SJakub Kicinski 40784c61fe1SJakub Kicinski if (!strp->stm.full_len) { 40884c61fe1SJakub Kicinski tls_strp_load_anchor_with_queue(strp, inq); 40984c61fe1SJakub Kicinski 41084c61fe1SJakub Kicinski sz = tls_rx_msg_size(strp, strp->anchor); 41184c61fe1SJakub Kicinski if (sz < 0) { 41284c61fe1SJakub Kicinski tls_strp_abort_strp(strp, sz); 41384c61fe1SJakub Kicinski return sz; 41484c61fe1SJakub Kicinski } 41584c61fe1SJakub Kicinski 41684c61fe1SJakub Kicinski strp->stm.full_len = sz; 41784c61fe1SJakub Kicinski 41884c61fe1SJakub Kicinski if (!strp->stm.full_len || inq < strp->stm.full_len) 4190d87bbd3SJakub Kicinski return tls_strp_read_copy(strp, true); 42084c61fe1SJakub Kicinski } 42184c61fe1SJakub Kicinski 422*14c4be92SJakub Kicinski if (!tls_strp_check_queue_ok(strp)) 4230d87bbd3SJakub Kicinski return tls_strp_read_copy(strp, false); 4240d87bbd3SJakub Kicinski 42584c61fe1SJakub Kicinski strp->msg_ready = 1; 42684c61fe1SJakub Kicinski tls_rx_msg_ready(strp); 42784c61fe1SJakub Kicinski 42884c61fe1SJakub Kicinski return 0; 42984c61fe1SJakub Kicinski } 43084c61fe1SJakub Kicinski 43184c61fe1SJakub Kicinski void tls_strp_check_rcv(struct tls_strparser *strp) 43284c61fe1SJakub Kicinski { 43384c61fe1SJakub Kicinski if (unlikely(strp->stopped) || strp->msg_ready) 43484c61fe1SJakub Kicinski return; 43584c61fe1SJakub Kicinski 43684c61fe1SJakub Kicinski if (tls_strp_read_sock(strp) == -ENOMEM) 43784c61fe1SJakub Kicinski queue_work(tls_strp_wq, &strp->work); 43884c61fe1SJakub Kicinski } 43984c61fe1SJakub Kicinski 44084c61fe1SJakub Kicinski /* Lower sock lock held */ 44184c61fe1SJakub Kicinski void tls_strp_data_ready(struct tls_strparser *strp) 44284c61fe1SJakub Kicinski { 44384c61fe1SJakub Kicinski /* This check is needed to synchronize with do_tls_strp_work. 44484c61fe1SJakub Kicinski * do_tls_strp_work acquires a process lock (lock_sock) whereas 44584c61fe1SJakub Kicinski * the lock held here is bh_lock_sock. The two locks can be 44684c61fe1SJakub Kicinski * held by different threads at the same time, but bh_lock_sock 44784c61fe1SJakub Kicinski * allows a thread in BH context to safely check if the process 44884c61fe1SJakub Kicinski * lock is held. In this case, if the lock is held, queue work. 44984c61fe1SJakub Kicinski */ 45084c61fe1SJakub Kicinski if (sock_owned_by_user_nocheck(strp->sk)) { 45184c61fe1SJakub Kicinski queue_work(tls_strp_wq, &strp->work); 45284c61fe1SJakub Kicinski return; 45384c61fe1SJakub Kicinski } 45484c61fe1SJakub Kicinski 45584c61fe1SJakub Kicinski tls_strp_check_rcv(strp); 45684c61fe1SJakub Kicinski } 45784c61fe1SJakub Kicinski 45884c61fe1SJakub Kicinski static void tls_strp_work(struct work_struct *w) 45984c61fe1SJakub Kicinski { 46084c61fe1SJakub Kicinski struct tls_strparser *strp = 46184c61fe1SJakub Kicinski container_of(w, struct tls_strparser, work); 46284c61fe1SJakub Kicinski 46384c61fe1SJakub Kicinski lock_sock(strp->sk); 46484c61fe1SJakub Kicinski tls_strp_check_rcv(strp); 46584c61fe1SJakub Kicinski release_sock(strp->sk); 46684c61fe1SJakub Kicinski } 46784c61fe1SJakub Kicinski 46884c61fe1SJakub Kicinski void tls_strp_msg_done(struct tls_strparser *strp) 46984c61fe1SJakub Kicinski { 47084c61fe1SJakub Kicinski WARN_ON(!strp->stm.full_len); 47184c61fe1SJakub Kicinski 47284c61fe1SJakub Kicinski if (likely(!strp->copy_mode)) 47384c61fe1SJakub Kicinski tcp_read_done(strp->sk, strp->stm.full_len); 47484c61fe1SJakub Kicinski else 47584c61fe1SJakub Kicinski tls_strp_flush_anchor_copy(strp); 47684c61fe1SJakub Kicinski 47784c61fe1SJakub Kicinski strp->msg_ready = 0; 47884c61fe1SJakub Kicinski memset(&strp->stm, 0, sizeof(strp->stm)); 47984c61fe1SJakub Kicinski 48084c61fe1SJakub Kicinski tls_strp_check_rcv(strp); 48184c61fe1SJakub Kicinski } 48284c61fe1SJakub Kicinski 48384c61fe1SJakub Kicinski void tls_strp_stop(struct tls_strparser *strp) 48484c61fe1SJakub Kicinski { 48584c61fe1SJakub Kicinski strp->stopped = 1; 48684c61fe1SJakub Kicinski } 48784c61fe1SJakub Kicinski 48884c61fe1SJakub Kicinski int tls_strp_init(struct tls_strparser *strp, struct sock *sk) 48984c61fe1SJakub Kicinski { 49084c61fe1SJakub Kicinski memset(strp, 0, sizeof(*strp)); 49184c61fe1SJakub Kicinski 49284c61fe1SJakub Kicinski strp->sk = sk; 49384c61fe1SJakub Kicinski 49484c61fe1SJakub Kicinski strp->anchor = alloc_skb(0, GFP_KERNEL); 49584c61fe1SJakub Kicinski if (!strp->anchor) 49684c61fe1SJakub Kicinski return -ENOMEM; 49784c61fe1SJakub Kicinski 49884c61fe1SJakub Kicinski INIT_WORK(&strp->work, tls_strp_work); 49984c61fe1SJakub Kicinski 50084c61fe1SJakub Kicinski return 0; 50184c61fe1SJakub Kicinski } 50284c61fe1SJakub Kicinski 50384c61fe1SJakub Kicinski /* strp must already be stopped so that tls_strp_recv will no longer be called. 50484c61fe1SJakub Kicinski * Note that tls_strp_done is not called with the lower socket held. 50584c61fe1SJakub Kicinski */ 50684c61fe1SJakub Kicinski void tls_strp_done(struct tls_strparser *strp) 50784c61fe1SJakub Kicinski { 50884c61fe1SJakub Kicinski WARN_ON(!strp->stopped); 50984c61fe1SJakub Kicinski 51084c61fe1SJakub Kicinski cancel_work_sync(&strp->work); 51184c61fe1SJakub Kicinski tls_strp_anchor_free(strp); 51284c61fe1SJakub Kicinski } 51384c61fe1SJakub Kicinski 51484c61fe1SJakub Kicinski int __init tls_strp_dev_init(void) 51584c61fe1SJakub Kicinski { 516d11ef9ccSJakub Kicinski tls_strp_wq = create_workqueue("tls-strp"); 51784c61fe1SJakub Kicinski if (unlikely(!tls_strp_wq)) 51884c61fe1SJakub Kicinski return -ENOMEM; 51984c61fe1SJakub Kicinski 52084c61fe1SJakub Kicinski return 0; 52184c61fe1SJakub Kicinski } 52284c61fe1SJakub Kicinski 52384c61fe1SJakub Kicinski void tls_strp_dev_exit(void) 52484c61fe1SJakub Kicinski { 52584c61fe1SJakub Kicinski destroy_workqueue(tls_strp_wq); 52684c61fe1SJakub Kicinski } 527