1604326b4SDaniel Borkmann // SPDX-License-Identifier: GPL-2.0 2604326b4SDaniel Borkmann /* Copyright (c) 2017 - 2018 Covalent IO, Inc. http://covalent.io */ 3604326b4SDaniel Borkmann 4604326b4SDaniel Borkmann #include <linux/skmsg.h> 5604326b4SDaniel Borkmann #include <linux/filter.h> 6604326b4SDaniel Borkmann #include <linux/bpf.h> 7604326b4SDaniel Borkmann #include <linux/init.h> 8604326b4SDaniel Borkmann #include <linux/wait.h> 9604326b4SDaniel Borkmann 10604326b4SDaniel Borkmann #include <net/inet_common.h> 110608c69cSJohn Fastabend #include <net/tls.h> 12604326b4SDaniel Borkmann 13604326b4SDaniel Borkmann static int bpf_tcp_ingress(struct sock *sk, struct sk_psock *psock, 14604326b4SDaniel Borkmann struct sk_msg *msg, u32 apply_bytes, int flags) 15604326b4SDaniel Borkmann { 16604326b4SDaniel Borkmann bool apply = apply_bytes; 17604326b4SDaniel Borkmann struct scatterlist *sge; 18604326b4SDaniel Borkmann u32 size, copied = 0; 19604326b4SDaniel Borkmann struct sk_msg *tmp; 20604326b4SDaniel Borkmann int i, ret = 0; 21604326b4SDaniel Borkmann 22604326b4SDaniel Borkmann tmp = kzalloc(sizeof(*tmp), __GFP_NOWARN | GFP_KERNEL); 23604326b4SDaniel Borkmann if (unlikely(!tmp)) 24604326b4SDaniel Borkmann return -ENOMEM; 25604326b4SDaniel Borkmann 26604326b4SDaniel Borkmann lock_sock(sk); 27604326b4SDaniel Borkmann tmp->sg.start = msg->sg.start; 28604326b4SDaniel Borkmann i = msg->sg.start; 29604326b4SDaniel Borkmann do { 30604326b4SDaniel Borkmann sge = sk_msg_elem(msg, i); 31604326b4SDaniel Borkmann size = (apply && apply_bytes < sge->length) ? 32604326b4SDaniel Borkmann apply_bytes : sge->length; 33604326b4SDaniel Borkmann if (!sk_wmem_schedule(sk, size)) { 34604326b4SDaniel Borkmann if (!copied) 35604326b4SDaniel Borkmann ret = -ENOMEM; 36604326b4SDaniel Borkmann break; 37604326b4SDaniel Borkmann } 38604326b4SDaniel Borkmann 39604326b4SDaniel Borkmann sk_mem_charge(sk, size); 40604326b4SDaniel Borkmann sk_msg_xfer(tmp, msg, i, size); 41604326b4SDaniel Borkmann copied += size; 42604326b4SDaniel Borkmann if (sge->length) 43604326b4SDaniel Borkmann get_page(sk_msg_page(tmp, i)); 44604326b4SDaniel Borkmann sk_msg_iter_var_next(i); 45604326b4SDaniel Borkmann tmp->sg.end = i; 46604326b4SDaniel Borkmann if (apply) { 47604326b4SDaniel Borkmann apply_bytes -= size; 48*9072931fSPengcheng Yang if (!apply_bytes) { 49*9072931fSPengcheng Yang if (sge->length) 50*9072931fSPengcheng Yang sk_msg_iter_var_prev(i); 51604326b4SDaniel Borkmann break; 52604326b4SDaniel Borkmann } 53*9072931fSPengcheng Yang } 54604326b4SDaniel Borkmann } while (i != msg->sg.end); 55604326b4SDaniel Borkmann 56604326b4SDaniel Borkmann if (!ret) { 57604326b4SDaniel Borkmann msg->sg.start = i; 58604326b4SDaniel Borkmann sk_psock_queue_msg(psock, tmp); 59552de910SJohn Fastabend sk_psock_data_ready(sk, psock); 60604326b4SDaniel Borkmann } else { 61604326b4SDaniel Borkmann sk_msg_free(sk, tmp); 62604326b4SDaniel Borkmann kfree(tmp); 63604326b4SDaniel Borkmann } 64604326b4SDaniel Borkmann 65604326b4SDaniel Borkmann release_sock(sk); 66604326b4SDaniel Borkmann return ret; 67604326b4SDaniel Borkmann } 68604326b4SDaniel Borkmann 69604326b4SDaniel Borkmann static int tcp_bpf_push(struct sock *sk, struct sk_msg *msg, u32 apply_bytes, 70604326b4SDaniel Borkmann int flags, bool uncharge) 71604326b4SDaniel Borkmann { 72604326b4SDaniel Borkmann bool apply = apply_bytes; 73604326b4SDaniel Borkmann struct scatterlist *sge; 74604326b4SDaniel Borkmann struct page *page; 75604326b4SDaniel Borkmann int size, ret = 0; 76604326b4SDaniel Borkmann u32 off; 77604326b4SDaniel Borkmann 78604326b4SDaniel Borkmann while (1) { 790608c69cSJohn Fastabend bool has_tx_ulp; 800608c69cSJohn Fastabend 81604326b4SDaniel Borkmann sge = sk_msg_elem(msg, msg->sg.start); 82604326b4SDaniel Borkmann size = (apply && apply_bytes < sge->length) ? 83604326b4SDaniel Borkmann apply_bytes : sge->length; 84604326b4SDaniel Borkmann off = sge->offset; 85604326b4SDaniel Borkmann page = sg_page(sge); 86604326b4SDaniel Borkmann 87604326b4SDaniel Borkmann tcp_rate_check_app_limited(sk); 88604326b4SDaniel Borkmann retry: 890608c69cSJohn Fastabend has_tx_ulp = tls_sw_has_ctx_tx(sk); 900608c69cSJohn Fastabend if (has_tx_ulp) { 910608c69cSJohn Fastabend flags |= MSG_SENDPAGE_NOPOLICY; 920608c69cSJohn Fastabend ret = kernel_sendpage_locked(sk, 930608c69cSJohn Fastabend page, off, size, flags); 940608c69cSJohn Fastabend } else { 95604326b4SDaniel Borkmann ret = do_tcp_sendpages(sk, page, off, size, flags); 960608c69cSJohn Fastabend } 970608c69cSJohn Fastabend 98604326b4SDaniel Borkmann if (ret <= 0) 99604326b4SDaniel Borkmann return ret; 100604326b4SDaniel Borkmann if (apply) 101604326b4SDaniel Borkmann apply_bytes -= ret; 102604326b4SDaniel Borkmann msg->sg.size -= ret; 103604326b4SDaniel Borkmann sge->offset += ret; 104604326b4SDaniel Borkmann sge->length -= ret; 105604326b4SDaniel Borkmann if (uncharge) 106604326b4SDaniel Borkmann sk_mem_uncharge(sk, ret); 107604326b4SDaniel Borkmann if (ret != size) { 108604326b4SDaniel Borkmann size -= ret; 109604326b4SDaniel Borkmann off += ret; 110604326b4SDaniel Borkmann goto retry; 111604326b4SDaniel Borkmann } 112604326b4SDaniel Borkmann if (!sge->length) { 113604326b4SDaniel Borkmann put_page(page); 114604326b4SDaniel Borkmann sk_msg_iter_next(msg, start); 115604326b4SDaniel Borkmann sg_init_table(sge, 1); 116604326b4SDaniel Borkmann if (msg->sg.start == msg->sg.end) 117604326b4SDaniel Borkmann break; 118604326b4SDaniel Borkmann } 119604326b4SDaniel Borkmann if (apply && !apply_bytes) 120604326b4SDaniel Borkmann break; 121604326b4SDaniel Borkmann } 122604326b4SDaniel Borkmann 123604326b4SDaniel Borkmann return 0; 124604326b4SDaniel Borkmann } 125604326b4SDaniel Borkmann 126604326b4SDaniel Borkmann static int tcp_bpf_push_locked(struct sock *sk, struct sk_msg *msg, 127604326b4SDaniel Borkmann u32 apply_bytes, int flags, bool uncharge) 128604326b4SDaniel Borkmann { 129604326b4SDaniel Borkmann int ret; 130604326b4SDaniel Borkmann 131604326b4SDaniel Borkmann lock_sock(sk); 132604326b4SDaniel Borkmann ret = tcp_bpf_push(sk, msg, apply_bytes, flags, uncharge); 133604326b4SDaniel Borkmann release_sock(sk); 134604326b4SDaniel Borkmann return ret; 135604326b4SDaniel Borkmann } 136604326b4SDaniel Borkmann 137a351d608SPengcheng Yang int tcp_bpf_sendmsg_redir(struct sock *sk, bool ingress, 138a351d608SPengcheng Yang struct sk_msg *msg, u32 bytes, int flags) 139604326b4SDaniel Borkmann { 140604326b4SDaniel Borkmann struct sk_psock *psock = sk_psock_get(sk); 141604326b4SDaniel Borkmann int ret; 142604326b4SDaniel Borkmann 1432486ab43SWang Yufen if (unlikely(!psock)) 1442486ab43SWang Yufen return -EPIPE; 1452486ab43SWang Yufen 146604326b4SDaniel Borkmann ret = ingress ? bpf_tcp_ingress(sk, psock, msg, bytes, flags) : 147604326b4SDaniel Borkmann tcp_bpf_push_locked(sk, msg, bytes, flags, false); 148604326b4SDaniel Borkmann sk_psock_put(sk, psock); 149604326b4SDaniel Borkmann return ret; 150604326b4SDaniel Borkmann } 151604326b4SDaniel Borkmann EXPORT_SYMBOL_GPL(tcp_bpf_sendmsg_redir); 152604326b4SDaniel Borkmann 15388759609SCong Wang #ifdef CONFIG_BPF_SYSCALL 154b6df0078SJakub Kicinski static int tcp_msg_wait_data(struct sock *sk, struct sk_psock *psock, 155b6df0078SJakub Kicinski long timeo) 1569f2470fbSCong Wang { 1579f2470fbSCong Wang DEFINE_WAIT_FUNC(wait, woken_wake_function); 1589f2470fbSCong Wang int ret = 0; 1599f2470fbSCong Wang 1609f2470fbSCong Wang if (sk->sk_shutdown & RCV_SHUTDOWN) 1619f2470fbSCong Wang return 1; 1629f2470fbSCong Wang 1639f2470fbSCong Wang if (!timeo) 1649f2470fbSCong Wang return ret; 1659f2470fbSCong Wang 1669f2470fbSCong Wang add_wait_queue(sk_sleep(sk), &wait); 1679f2470fbSCong Wang sk_set_bit(SOCKWQ_ASYNC_WAITDATA, sk); 1689f2470fbSCong Wang ret = sk_wait_event(sk, &timeo, 1699f2470fbSCong Wang !list_empty(&psock->ingress_msg) || 1709f2470fbSCong Wang !skb_queue_empty(&sk->sk_receive_queue), &wait); 1719f2470fbSCong Wang sk_clear_bit(SOCKWQ_ASYNC_WAITDATA, sk); 1729f2470fbSCong Wang remove_wait_queue(sk_sleep(sk), &wait); 1739f2470fbSCong Wang return ret; 1749f2470fbSCong Wang } 1759f2470fbSCong Wang 176c5d2177aSJohn Fastabend static int tcp_bpf_recvmsg_parser(struct sock *sk, 177c5d2177aSJohn Fastabend struct msghdr *msg, 178c5d2177aSJohn Fastabend size_t len, 179c5d2177aSJohn Fastabend int flags, 180c5d2177aSJohn Fastabend int *addr_len) 181c5d2177aSJohn Fastabend { 182c5d2177aSJohn Fastabend struct sk_psock *psock; 183c5d2177aSJohn Fastabend int copied; 184c5d2177aSJohn Fastabend 185c5d2177aSJohn Fastabend if (unlikely(flags & MSG_ERRQUEUE)) 186c5d2177aSJohn Fastabend return inet_recv_error(sk, msg, len, addr_len); 187c5d2177aSJohn Fastabend 188c5d2177aSJohn Fastabend psock = sk_psock_get(sk); 189c5d2177aSJohn Fastabend if (unlikely(!psock)) 190ec095263SOliver Hartkopp return tcp_recvmsg(sk, msg, len, flags, addr_len); 191c5d2177aSJohn Fastabend 192c5d2177aSJohn Fastabend lock_sock(sk); 193c5d2177aSJohn Fastabend msg_bytes_ready: 194c5d2177aSJohn Fastabend copied = sk_msg_recvmsg(sk, psock, msg, len, flags); 195c5d2177aSJohn Fastabend if (!copied) { 196c5d2177aSJohn Fastabend long timeo; 197c5d2177aSJohn Fastabend int data; 198c5d2177aSJohn Fastabend 1995b2c5540SJohn Fastabend if (sock_flag(sk, SOCK_DONE)) 2005b2c5540SJohn Fastabend goto out; 2015b2c5540SJohn Fastabend 2025b2c5540SJohn Fastabend if (sk->sk_err) { 2035b2c5540SJohn Fastabend copied = sock_error(sk); 2045b2c5540SJohn Fastabend goto out; 2055b2c5540SJohn Fastabend } 2065b2c5540SJohn Fastabend 2075b2c5540SJohn Fastabend if (sk->sk_shutdown & RCV_SHUTDOWN) 2085b2c5540SJohn Fastabend goto out; 2095b2c5540SJohn Fastabend 2105b2c5540SJohn Fastabend if (sk->sk_state == TCP_CLOSE) { 2115b2c5540SJohn Fastabend copied = -ENOTCONN; 2125b2c5540SJohn Fastabend goto out; 2135b2c5540SJohn Fastabend } 2145b2c5540SJohn Fastabend 215ec095263SOliver Hartkopp timeo = sock_rcvtimeo(sk, flags & MSG_DONTWAIT); 2165b2c5540SJohn Fastabend if (!timeo) { 2175b2c5540SJohn Fastabend copied = -EAGAIN; 2185b2c5540SJohn Fastabend goto out; 2195b2c5540SJohn Fastabend } 2205b2c5540SJohn Fastabend 2215b2c5540SJohn Fastabend if (signal_pending(current)) { 2225b2c5540SJohn Fastabend copied = sock_intr_errno(timeo); 2235b2c5540SJohn Fastabend goto out; 2245b2c5540SJohn Fastabend } 2255b2c5540SJohn Fastabend 226c5d2177aSJohn Fastabend data = tcp_msg_wait_data(sk, psock, timeo); 227c5d2177aSJohn Fastabend if (data && !sk_psock_queue_empty(psock)) 228c5d2177aSJohn Fastabend goto msg_bytes_ready; 229c5d2177aSJohn Fastabend copied = -EAGAIN; 230c5d2177aSJohn Fastabend } 2315b2c5540SJohn Fastabend out: 232c5d2177aSJohn Fastabend release_sock(sk); 233c5d2177aSJohn Fastabend sk_psock_put(sk, psock); 234c5d2177aSJohn Fastabend return copied; 235c5d2177aSJohn Fastabend } 236c5d2177aSJohn Fastabend 237c0fd336eSYueHaibing static int tcp_bpf_recvmsg(struct sock *sk, struct msghdr *msg, size_t len, 238ec095263SOliver Hartkopp int flags, int *addr_len) 239c0fd336eSYueHaibing { 240c0fd336eSYueHaibing struct sk_psock *psock; 241c0fd336eSYueHaibing int copied, ret; 242c0fd336eSYueHaibing 24318f02ad1SXiyu Yang if (unlikely(flags & MSG_ERRQUEUE)) 24418f02ad1SXiyu Yang return inet_recv_error(sk, msg, len, addr_len); 24518f02ad1SXiyu Yang 246c0fd336eSYueHaibing psock = sk_psock_get(sk); 247c0fd336eSYueHaibing if (unlikely(!psock)) 248ec095263SOliver Hartkopp return tcp_recvmsg(sk, msg, len, flags, addr_len); 249c0fd336eSYueHaibing if (!skb_queue_empty(&sk->sk_receive_queue) && 25018f02ad1SXiyu Yang sk_psock_queue_empty(psock)) { 25118f02ad1SXiyu Yang sk_psock_put(sk, psock); 252ec095263SOliver Hartkopp return tcp_recvmsg(sk, msg, len, flags, addr_len); 25318f02ad1SXiyu Yang } 254c0fd336eSYueHaibing lock_sock(sk); 255c0fd336eSYueHaibing msg_bytes_ready: 2562bc793e3SCong Wang copied = sk_msg_recvmsg(sk, psock, msg, len, flags); 257c0fd336eSYueHaibing if (!copied) { 258c0fd336eSYueHaibing long timeo; 259c49661aaSCong Wang int data; 260c0fd336eSYueHaibing 261ec095263SOliver Hartkopp timeo = sock_rcvtimeo(sk, flags & MSG_DONTWAIT); 262b6df0078SJakub Kicinski data = tcp_msg_wait_data(sk, psock, timeo); 263c0fd336eSYueHaibing if (data) { 264c0fd336eSYueHaibing if (!sk_psock_queue_empty(psock)) 265c0fd336eSYueHaibing goto msg_bytes_ready; 266c0fd336eSYueHaibing release_sock(sk); 267c0fd336eSYueHaibing sk_psock_put(sk, psock); 268ec095263SOliver Hartkopp return tcp_recvmsg(sk, msg, len, flags, addr_len); 269c0fd336eSYueHaibing } 270c0fd336eSYueHaibing copied = -EAGAIN; 271c0fd336eSYueHaibing } 272c0fd336eSYueHaibing ret = copied; 273c0fd336eSYueHaibing release_sock(sk); 274c0fd336eSYueHaibing sk_psock_put(sk, psock); 275c0fd336eSYueHaibing return ret; 276c0fd336eSYueHaibing } 277c0fd336eSYueHaibing 278604326b4SDaniel Borkmann static int tcp_bpf_send_verdict(struct sock *sk, struct sk_psock *psock, 279604326b4SDaniel Borkmann struct sk_msg *msg, int *copied, int flags) 280604326b4SDaniel Borkmann { 281a351d608SPengcheng Yang bool cork = false, enospc = sk_msg_full(msg), redir_ingress; 282604326b4SDaniel Borkmann struct sock *sk_redir; 2838ec95b94SWang Yufen u32 tosend, origsize, sent, delta = 0; 2847a9841caSPengcheng Yang u32 eval; 285604326b4SDaniel Borkmann int ret; 286604326b4SDaniel Borkmann 287604326b4SDaniel Borkmann more_data: 2887246d8edSJohn Fastabend if (psock->eval == __SK_NONE) { 2897246d8edSJohn Fastabend /* Track delta in msg size to add/subtract it on SK_DROP from 2907246d8edSJohn Fastabend * returned to user copied size. This ensures user doesn't 2917246d8edSJohn Fastabend * get a positive return code with msg_cut_data and SK_DROP 2927246d8edSJohn Fastabend * verdict. 2937246d8edSJohn Fastabend */ 2947246d8edSJohn Fastabend delta = msg->sg.size; 295604326b4SDaniel Borkmann psock->eval = sk_psock_msg_verdict(sk, psock, msg); 2967246d8edSJohn Fastabend delta -= msg->sg.size; 2977246d8edSJohn Fastabend } 298604326b4SDaniel Borkmann 299604326b4SDaniel Borkmann if (msg->cork_bytes && 300604326b4SDaniel Borkmann msg->cork_bytes > msg->sg.size && !enospc) { 301604326b4SDaniel Borkmann psock->cork_bytes = msg->cork_bytes - msg->sg.size; 302604326b4SDaniel Borkmann if (!psock->cork) { 303604326b4SDaniel Borkmann psock->cork = kzalloc(sizeof(*psock->cork), 304604326b4SDaniel Borkmann GFP_ATOMIC | __GFP_NOWARN); 305604326b4SDaniel Borkmann if (!psock->cork) 306604326b4SDaniel Borkmann return -ENOMEM; 307604326b4SDaniel Borkmann } 308604326b4SDaniel Borkmann memcpy(psock->cork, msg, sizeof(*msg)); 309604326b4SDaniel Borkmann return 0; 310604326b4SDaniel Borkmann } 311604326b4SDaniel Borkmann 312604326b4SDaniel Borkmann tosend = msg->sg.size; 313604326b4SDaniel Borkmann if (psock->apply_bytes && psock->apply_bytes < tosend) 314604326b4SDaniel Borkmann tosend = psock->apply_bytes; 3157a9841caSPengcheng Yang eval = __SK_NONE; 316604326b4SDaniel Borkmann 317604326b4SDaniel Borkmann switch (psock->eval) { 318604326b4SDaniel Borkmann case __SK_PASS: 319604326b4SDaniel Borkmann ret = tcp_bpf_push(sk, msg, tosend, flags, true); 320604326b4SDaniel Borkmann if (unlikely(ret)) { 321604326b4SDaniel Borkmann *copied -= sk_msg_free(sk, msg); 322604326b4SDaniel Borkmann break; 323604326b4SDaniel Borkmann } 324604326b4SDaniel Borkmann sk_msg_apply_bytes(psock, tosend); 325604326b4SDaniel Borkmann break; 326604326b4SDaniel Borkmann case __SK_REDIRECT: 327a351d608SPengcheng Yang redir_ingress = psock->redir_ingress; 328604326b4SDaniel Borkmann sk_redir = psock->sk_redir; 329604326b4SDaniel Borkmann sk_msg_apply_bytes(psock, tosend); 330cd9733f5SLiu Jian if (!psock->apply_bytes) { 331cd9733f5SLiu Jian /* Clean up before releasing the sock lock. */ 332cd9733f5SLiu Jian eval = psock->eval; 333cd9733f5SLiu Jian psock->eval = __SK_NONE; 334cd9733f5SLiu Jian psock->sk_redir = NULL; 335cd9733f5SLiu Jian } 336604326b4SDaniel Borkmann if (psock->cork) { 337604326b4SDaniel Borkmann cork = true; 338604326b4SDaniel Borkmann psock->cork = NULL; 339604326b4SDaniel Borkmann } 3408ec95b94SWang Yufen sk_msg_return(sk, msg, tosend); 341604326b4SDaniel Borkmann release_sock(sk); 342cd9733f5SLiu Jian 3438ec95b94SWang Yufen origsize = msg->sg.size; 344a351d608SPengcheng Yang ret = tcp_bpf_sendmsg_redir(sk_redir, redir_ingress, 345a351d608SPengcheng Yang msg, tosend, flags); 3468ec95b94SWang Yufen sent = origsize - msg->sg.size; 347cd9733f5SLiu Jian 348cd9733f5SLiu Jian if (eval == __SK_REDIRECT) 349cd9733f5SLiu Jian sock_put(sk_redir); 350cd9733f5SLiu Jian 351604326b4SDaniel Borkmann lock_sock(sk); 352604326b4SDaniel Borkmann if (unlikely(ret < 0)) { 353604326b4SDaniel Borkmann int free = sk_msg_free_nocharge(sk, msg); 354604326b4SDaniel Borkmann 355604326b4SDaniel Borkmann if (!cork) 356604326b4SDaniel Borkmann *copied -= free; 357604326b4SDaniel Borkmann } 358604326b4SDaniel Borkmann if (cork) { 359604326b4SDaniel Borkmann sk_msg_free(sk, msg); 360604326b4SDaniel Borkmann kfree(msg); 361604326b4SDaniel Borkmann msg = NULL; 362604326b4SDaniel Borkmann ret = 0; 363604326b4SDaniel Borkmann } 364604326b4SDaniel Borkmann break; 365604326b4SDaniel Borkmann case __SK_DROP: 366604326b4SDaniel Borkmann default: 367604326b4SDaniel Borkmann sk_msg_free_partial(sk, msg, tosend); 368604326b4SDaniel Borkmann sk_msg_apply_bytes(psock, tosend); 3697246d8edSJohn Fastabend *copied -= (tosend + delta); 370604326b4SDaniel Borkmann return -EACCES; 371604326b4SDaniel Borkmann } 372604326b4SDaniel Borkmann 373604326b4SDaniel Borkmann if (likely(!ret)) { 374604326b4SDaniel Borkmann if (!psock->apply_bytes) { 375604326b4SDaniel Borkmann psock->eval = __SK_NONE; 376604326b4SDaniel Borkmann if (psock->sk_redir) { 377604326b4SDaniel Borkmann sock_put(psock->sk_redir); 378604326b4SDaniel Borkmann psock->sk_redir = NULL; 379604326b4SDaniel Borkmann } 380604326b4SDaniel Borkmann } 381604326b4SDaniel Borkmann if (msg && 382604326b4SDaniel Borkmann msg->sg.data[msg->sg.start].page_link && 38384472b43SWang Yufen msg->sg.data[msg->sg.start].length) { 38484472b43SWang Yufen if (eval == __SK_REDIRECT) 3858ec95b94SWang Yufen sk_mem_charge(sk, tosend - sent); 386604326b4SDaniel Borkmann goto more_data; 387604326b4SDaniel Borkmann } 38884472b43SWang Yufen } 389604326b4SDaniel Borkmann return ret; 390604326b4SDaniel Borkmann } 391604326b4SDaniel Borkmann 392604326b4SDaniel Borkmann static int tcp_bpf_sendmsg(struct sock *sk, struct msghdr *msg, size_t size) 393604326b4SDaniel Borkmann { 394604326b4SDaniel Borkmann struct sk_msg tmp, *msg_tx = NULL; 395604326b4SDaniel Borkmann int copied = 0, err = 0; 396604326b4SDaniel Borkmann struct sk_psock *psock; 397604326b4SDaniel Borkmann long timeo; 39841477662SJakub Kicinski int flags; 39941477662SJakub Kicinski 40041477662SJakub Kicinski /* Don't let internal do_tcp_sendpages() flags through */ 40141477662SJakub Kicinski flags = (msg->msg_flags & ~MSG_SENDPAGE_DECRYPTED); 40241477662SJakub Kicinski flags |= MSG_NO_SHARED_FRAGS; 403604326b4SDaniel Borkmann 404604326b4SDaniel Borkmann psock = sk_psock_get(sk); 405604326b4SDaniel Borkmann if (unlikely(!psock)) 406604326b4SDaniel Borkmann return tcp_sendmsg(sk, msg, size); 407604326b4SDaniel Borkmann 408604326b4SDaniel Borkmann lock_sock(sk); 409604326b4SDaniel Borkmann timeo = sock_sndtimeo(sk, msg->msg_flags & MSG_DONTWAIT); 410604326b4SDaniel Borkmann while (msg_data_left(msg)) { 411604326b4SDaniel Borkmann bool enospc = false; 412604326b4SDaniel Borkmann u32 copy, osize; 413604326b4SDaniel Borkmann 414604326b4SDaniel Borkmann if (sk->sk_err) { 415604326b4SDaniel Borkmann err = -sk->sk_err; 416604326b4SDaniel Borkmann goto out_err; 417604326b4SDaniel Borkmann } 418604326b4SDaniel Borkmann 419604326b4SDaniel Borkmann copy = msg_data_left(msg); 420604326b4SDaniel Borkmann if (!sk_stream_memory_free(sk)) 421604326b4SDaniel Borkmann goto wait_for_sndbuf; 422604326b4SDaniel Borkmann if (psock->cork) { 423604326b4SDaniel Borkmann msg_tx = psock->cork; 424604326b4SDaniel Borkmann } else { 425604326b4SDaniel Borkmann msg_tx = &tmp; 426604326b4SDaniel Borkmann sk_msg_init(msg_tx); 427604326b4SDaniel Borkmann } 428604326b4SDaniel Borkmann 429604326b4SDaniel Borkmann osize = msg_tx->sg.size; 430604326b4SDaniel Borkmann err = sk_msg_alloc(sk, msg_tx, msg_tx->sg.size + copy, msg_tx->sg.end - 1); 431604326b4SDaniel Borkmann if (err) { 432604326b4SDaniel Borkmann if (err != -ENOSPC) 433604326b4SDaniel Borkmann goto wait_for_memory; 434604326b4SDaniel Borkmann enospc = true; 435604326b4SDaniel Borkmann copy = msg_tx->sg.size - osize; 436604326b4SDaniel Borkmann } 437604326b4SDaniel Borkmann 438604326b4SDaniel Borkmann err = sk_msg_memcopy_from_iter(sk, &msg->msg_iter, msg_tx, 439604326b4SDaniel Borkmann copy); 440604326b4SDaniel Borkmann if (err < 0) { 441604326b4SDaniel Borkmann sk_msg_trim(sk, msg_tx, osize); 442604326b4SDaniel Borkmann goto out_err; 443604326b4SDaniel Borkmann } 444604326b4SDaniel Borkmann 445604326b4SDaniel Borkmann copied += copy; 446604326b4SDaniel Borkmann if (psock->cork_bytes) { 447604326b4SDaniel Borkmann if (size > psock->cork_bytes) 448604326b4SDaniel Borkmann psock->cork_bytes = 0; 449604326b4SDaniel Borkmann else 450604326b4SDaniel Borkmann psock->cork_bytes -= size; 451604326b4SDaniel Borkmann if (psock->cork_bytes && !enospc) 452604326b4SDaniel Borkmann goto out_err; 453604326b4SDaniel Borkmann /* All cork bytes are accounted, rerun the prog. */ 454604326b4SDaniel Borkmann psock->eval = __SK_NONE; 455604326b4SDaniel Borkmann psock->cork_bytes = 0; 456604326b4SDaniel Borkmann } 457604326b4SDaniel Borkmann 458604326b4SDaniel Borkmann err = tcp_bpf_send_verdict(sk, psock, msg_tx, &copied, flags); 459604326b4SDaniel Borkmann if (unlikely(err < 0)) 460604326b4SDaniel Borkmann goto out_err; 461604326b4SDaniel Borkmann continue; 462604326b4SDaniel Borkmann wait_for_sndbuf: 463604326b4SDaniel Borkmann set_bit(SOCK_NOSPACE, &sk->sk_socket->flags); 464604326b4SDaniel Borkmann wait_for_memory: 465604326b4SDaniel Borkmann err = sk_stream_wait_memory(sk, &timeo); 466604326b4SDaniel Borkmann if (err) { 467604326b4SDaniel Borkmann if (msg_tx && msg_tx != psock->cork) 468604326b4SDaniel Borkmann sk_msg_free(sk, msg_tx); 469604326b4SDaniel Borkmann goto out_err; 470604326b4SDaniel Borkmann } 471604326b4SDaniel Borkmann } 472604326b4SDaniel Borkmann out_err: 473604326b4SDaniel Borkmann if (err < 0) 474604326b4SDaniel Borkmann err = sk_stream_error(sk, msg->msg_flags, err); 475604326b4SDaniel Borkmann release_sock(sk); 476604326b4SDaniel Borkmann sk_psock_put(sk, psock); 477604326b4SDaniel Borkmann return copied ? copied : err; 478604326b4SDaniel Borkmann } 479604326b4SDaniel Borkmann 480604326b4SDaniel Borkmann static int tcp_bpf_sendpage(struct sock *sk, struct page *page, int offset, 481604326b4SDaniel Borkmann size_t size, int flags) 482604326b4SDaniel Borkmann { 483604326b4SDaniel Borkmann struct sk_msg tmp, *msg = NULL; 484604326b4SDaniel Borkmann int err = 0, copied = 0; 485604326b4SDaniel Borkmann struct sk_psock *psock; 486604326b4SDaniel Borkmann bool enospc = false; 487604326b4SDaniel Borkmann 488604326b4SDaniel Borkmann psock = sk_psock_get(sk); 489604326b4SDaniel Borkmann if (unlikely(!psock)) 490604326b4SDaniel Borkmann return tcp_sendpage(sk, page, offset, size, flags); 491604326b4SDaniel Borkmann 492604326b4SDaniel Borkmann lock_sock(sk); 493604326b4SDaniel Borkmann if (psock->cork) { 494604326b4SDaniel Borkmann msg = psock->cork; 495604326b4SDaniel Borkmann } else { 496604326b4SDaniel Borkmann msg = &tmp; 497604326b4SDaniel Borkmann sk_msg_init(msg); 498604326b4SDaniel Borkmann } 499604326b4SDaniel Borkmann 500604326b4SDaniel Borkmann /* Catch case where ring is full and sendpage is stalled. */ 501604326b4SDaniel Borkmann if (unlikely(sk_msg_full(msg))) 502604326b4SDaniel Borkmann goto out_err; 503604326b4SDaniel Borkmann 504604326b4SDaniel Borkmann sk_msg_page_add(msg, page, size, offset); 505604326b4SDaniel Borkmann sk_mem_charge(sk, size); 506604326b4SDaniel Borkmann copied = size; 507604326b4SDaniel Borkmann if (sk_msg_full(msg)) 508604326b4SDaniel Borkmann enospc = true; 509604326b4SDaniel Borkmann if (psock->cork_bytes) { 510604326b4SDaniel Borkmann if (size > psock->cork_bytes) 511604326b4SDaniel Borkmann psock->cork_bytes = 0; 512604326b4SDaniel Borkmann else 513604326b4SDaniel Borkmann psock->cork_bytes -= size; 514604326b4SDaniel Borkmann if (psock->cork_bytes && !enospc) 515604326b4SDaniel Borkmann goto out_err; 516604326b4SDaniel Borkmann /* All cork bytes are accounted, rerun the prog. */ 517604326b4SDaniel Borkmann psock->eval = __SK_NONE; 518604326b4SDaniel Borkmann psock->cork_bytes = 0; 519604326b4SDaniel Borkmann } 520604326b4SDaniel Borkmann 521604326b4SDaniel Borkmann err = tcp_bpf_send_verdict(sk, psock, msg, &copied, flags); 522604326b4SDaniel Borkmann out_err: 523604326b4SDaniel Borkmann release_sock(sk); 524604326b4SDaniel Borkmann sk_psock_put(sk, psock); 525604326b4SDaniel Borkmann return copied ? copied : err; 526604326b4SDaniel Borkmann } 527604326b4SDaniel Borkmann 528604326b4SDaniel Borkmann enum { 529604326b4SDaniel Borkmann TCP_BPF_IPV4, 530604326b4SDaniel Borkmann TCP_BPF_IPV6, 531604326b4SDaniel Borkmann TCP_BPF_NUM_PROTS, 532604326b4SDaniel Borkmann }; 533604326b4SDaniel Borkmann 534604326b4SDaniel Borkmann enum { 535604326b4SDaniel Borkmann TCP_BPF_BASE, 536604326b4SDaniel Borkmann TCP_BPF_TX, 537c5d2177aSJohn Fastabend TCP_BPF_RX, 538c5d2177aSJohn Fastabend TCP_BPF_TXRX, 539604326b4SDaniel Borkmann TCP_BPF_NUM_CFGS, 540604326b4SDaniel Borkmann }; 541604326b4SDaniel Borkmann 542604326b4SDaniel Borkmann static struct proto *tcpv6_prot_saved __read_mostly; 543604326b4SDaniel Borkmann static DEFINE_SPINLOCK(tcpv6_prot_lock); 544604326b4SDaniel Borkmann static struct proto tcp_bpf_prots[TCP_BPF_NUM_PROTS][TCP_BPF_NUM_CFGS]; 545604326b4SDaniel Borkmann 546604326b4SDaniel Borkmann static void tcp_bpf_rebuild_protos(struct proto prot[TCP_BPF_NUM_CFGS], 547604326b4SDaniel Borkmann struct proto *base) 548604326b4SDaniel Borkmann { 549604326b4SDaniel Borkmann prot[TCP_BPF_BASE] = *base; 550d8616ee2SWang Yufen prot[TCP_BPF_BASE].destroy = sock_map_destroy; 551f747632bSLorenz Bauer prot[TCP_BPF_BASE].close = sock_map_close; 552604326b4SDaniel Borkmann prot[TCP_BPF_BASE].recvmsg = tcp_bpf_recvmsg; 553fb4e0a5eSCong Wang prot[TCP_BPF_BASE].sock_is_readable = sk_msg_is_readable; 554604326b4SDaniel Borkmann 555604326b4SDaniel Borkmann prot[TCP_BPF_TX] = prot[TCP_BPF_BASE]; 556604326b4SDaniel Borkmann prot[TCP_BPF_TX].sendmsg = tcp_bpf_sendmsg; 557604326b4SDaniel Borkmann prot[TCP_BPF_TX].sendpage = tcp_bpf_sendpage; 558c5d2177aSJohn Fastabend 559c5d2177aSJohn Fastabend prot[TCP_BPF_RX] = prot[TCP_BPF_BASE]; 560c5d2177aSJohn Fastabend prot[TCP_BPF_RX].recvmsg = tcp_bpf_recvmsg_parser; 561c5d2177aSJohn Fastabend 562c5d2177aSJohn Fastabend prot[TCP_BPF_TXRX] = prot[TCP_BPF_TX]; 563c5d2177aSJohn Fastabend prot[TCP_BPF_TXRX].recvmsg = tcp_bpf_recvmsg_parser; 564604326b4SDaniel Borkmann } 565604326b4SDaniel Borkmann 5667b219da4SLorenz Bauer static void tcp_bpf_check_v6_needs_rebuild(struct proto *ops) 567604326b4SDaniel Borkmann { 5687b219da4SLorenz Bauer if (unlikely(ops != smp_load_acquire(&tcpv6_prot_saved))) { 569604326b4SDaniel Borkmann spin_lock_bh(&tcpv6_prot_lock); 570604326b4SDaniel Borkmann if (likely(ops != tcpv6_prot_saved)) { 571604326b4SDaniel Borkmann tcp_bpf_rebuild_protos(tcp_bpf_prots[TCP_BPF_IPV6], ops); 572604326b4SDaniel Borkmann smp_store_release(&tcpv6_prot_saved, ops); 573604326b4SDaniel Borkmann } 574604326b4SDaniel Borkmann spin_unlock_bh(&tcpv6_prot_lock); 575604326b4SDaniel Borkmann } 576604326b4SDaniel Borkmann } 577604326b4SDaniel Borkmann 578604326b4SDaniel Borkmann static int __init tcp_bpf_v4_build_proto(void) 579604326b4SDaniel Borkmann { 580604326b4SDaniel Borkmann tcp_bpf_rebuild_protos(tcp_bpf_prots[TCP_BPF_IPV4], &tcp_prot); 581604326b4SDaniel Borkmann return 0; 582604326b4SDaniel Borkmann } 583228a4a7bSJohn Fastabend late_initcall(tcp_bpf_v4_build_proto); 584604326b4SDaniel Borkmann 585604326b4SDaniel Borkmann static int tcp_bpf_assert_proto_ops(struct proto *ops) 586604326b4SDaniel Borkmann { 587604326b4SDaniel Borkmann /* In order to avoid retpoline, we make assumptions when we call 588604326b4SDaniel Borkmann * into ops if e.g. a psock is not present. Make sure they are 589604326b4SDaniel Borkmann * indeed valid assumptions. 590604326b4SDaniel Borkmann */ 591604326b4SDaniel Borkmann return ops->recvmsg == tcp_recvmsg && 592604326b4SDaniel Borkmann ops->sendmsg == tcp_sendmsg && 593604326b4SDaniel Borkmann ops->sendpage == tcp_sendpage ? 0 : -ENOTSUPP; 594604326b4SDaniel Borkmann } 595604326b4SDaniel Borkmann 59651e0158aSCong Wang int tcp_bpf_update_proto(struct sock *sk, struct sk_psock *psock, bool restore) 597604326b4SDaniel Borkmann { 598d19da360SLorenz Bauer int family = sk->sk_family == AF_INET6 ? TCP_BPF_IPV6 : TCP_BPF_IPV4; 599d19da360SLorenz Bauer int config = psock->progs.msg_parser ? TCP_BPF_TX : TCP_BPF_BASE; 600604326b4SDaniel Borkmann 601c5d2177aSJohn Fastabend if (psock->progs.stream_verdict || psock->progs.skb_verdict) { 602c5d2177aSJohn Fastabend config = (config == TCP_BPF_TX) ? TCP_BPF_TXRX : TCP_BPF_RX; 603c5d2177aSJohn Fastabend } 604c5d2177aSJohn Fastabend 6058a59f9d1SCong Wang if (restore) { 6068a59f9d1SCong Wang if (inet_csk_has_ulp(sk)) { 6078859a44eSJakub Kicinski /* TLS does not have an unhash proto in SW cases, 6088859a44eSJakub Kicinski * but we need to ensure we stop using the sock_map 6098859a44eSJakub Kicinski * unhash routine because the associated psock is being 6108859a44eSJakub Kicinski * removed. So use the original unhash handler. 6118859a44eSJakub Kicinski */ 6128859a44eSJakub Kicinski WRITE_ONCE(sk->sk_prot->unhash, psock->saved_unhash); 6138a59f9d1SCong Wang tcp_update_ulp(sk, psock->sk_proto, psock->saved_write_space); 6148a59f9d1SCong Wang } else { 6158a59f9d1SCong Wang sk->sk_write_space = psock->saved_write_space; 6168a59f9d1SCong Wang /* Pairs with lockless read in sk_clone_lock() */ 617fee9ac06SPavel Begunkov sock_replace_proto(sk, psock->sk_proto); 6188a59f9d1SCong Wang } 6198a59f9d1SCong Wang return 0; 6208a59f9d1SCong Wang } 6218a59f9d1SCong Wang 6227b219da4SLorenz Bauer if (sk->sk_family == AF_INET6) { 6237b219da4SLorenz Bauer if (tcp_bpf_assert_proto_ops(psock->sk_proto)) 6248a59f9d1SCong Wang return -EINVAL; 625d19da360SLorenz Bauer 6267b219da4SLorenz Bauer tcp_bpf_check_v6_needs_rebuild(psock->sk_proto); 627d19da360SLorenz Bauer } 628d19da360SLorenz Bauer 6298a59f9d1SCong Wang /* Pairs with lockless read in sk_clone_lock() */ 630fee9ac06SPavel Begunkov sock_replace_proto(sk, &tcp_bpf_prots[family][config]); 6318a59f9d1SCong Wang return 0; 632604326b4SDaniel Borkmann } 6338a59f9d1SCong Wang EXPORT_SYMBOL_GPL(tcp_bpf_update_proto); 634604326b4SDaniel Borkmann 635e8025155SJakub Sitnicki /* If a child got cloned from a listening socket that had tcp_bpf 636e8025155SJakub Sitnicki * protocol callbacks installed, we need to restore the callbacks to 637e8025155SJakub Sitnicki * the default ones because the child does not inherit the psock state 638e8025155SJakub Sitnicki * that tcp_bpf callbacks expect. 639e8025155SJakub Sitnicki */ 640e8025155SJakub Sitnicki void tcp_bpf_clone(const struct sock *sk, struct sock *newsk) 641e8025155SJakub Sitnicki { 642e8025155SJakub Sitnicki int family = sk->sk_family == AF_INET6 ? TCP_BPF_IPV6 : TCP_BPF_IPV4; 643e8025155SJakub Sitnicki struct proto *prot = newsk->sk_prot; 644e8025155SJakub Sitnicki 645e8025155SJakub Sitnicki if (prot == &tcp_bpf_prots[family][TCP_BPF_BASE]) 646e8025155SJakub Sitnicki newsk->sk_prot = sk->sk_prot_creator; 647e8025155SJakub Sitnicki } 64888759609SCong Wang #endif /* CONFIG_BPF_SYSCALL */ 649