1604326b4SDaniel Borkmann // SPDX-License-Identifier: GPL-2.0 2604326b4SDaniel Borkmann /* Copyright (c) 2017 - 2018 Covalent IO, Inc. http://covalent.io */ 3604326b4SDaniel Borkmann 4604326b4SDaniel Borkmann #include <linux/skmsg.h> 5604326b4SDaniel Borkmann #include <linux/filter.h> 6604326b4SDaniel Borkmann #include <linux/bpf.h> 7604326b4SDaniel Borkmann #include <linux/init.h> 8604326b4SDaniel Borkmann #include <linux/wait.h> 9ddce1e09SJakub Sitnicki #include <linux/util_macros.h> 10604326b4SDaniel Borkmann 11604326b4SDaniel Borkmann #include <net/inet_common.h> 120608c69cSJohn Fastabend #include <net/tls.h> 13604326b4SDaniel Borkmann 14604326b4SDaniel Borkmann static int bpf_tcp_ingress(struct sock *sk, struct sk_psock *psock, 15604326b4SDaniel Borkmann struct sk_msg *msg, u32 apply_bytes, int flags) 16604326b4SDaniel Borkmann { 17604326b4SDaniel Borkmann bool apply = apply_bytes; 18604326b4SDaniel Borkmann struct scatterlist *sge; 19604326b4SDaniel Borkmann u32 size, copied = 0; 20604326b4SDaniel Borkmann struct sk_msg *tmp; 21604326b4SDaniel Borkmann int i, ret = 0; 22604326b4SDaniel Borkmann 23604326b4SDaniel Borkmann tmp = kzalloc(sizeof(*tmp), __GFP_NOWARN | GFP_KERNEL); 24604326b4SDaniel Borkmann if (unlikely(!tmp)) 25604326b4SDaniel Borkmann return -ENOMEM; 26604326b4SDaniel Borkmann 27604326b4SDaniel Borkmann lock_sock(sk); 28604326b4SDaniel Borkmann tmp->sg.start = msg->sg.start; 29604326b4SDaniel Borkmann i = msg->sg.start; 30604326b4SDaniel Borkmann do { 31604326b4SDaniel Borkmann sge = sk_msg_elem(msg, i); 32604326b4SDaniel Borkmann size = (apply && apply_bytes < sge->length) ? 33604326b4SDaniel Borkmann apply_bytes : sge->length; 34604326b4SDaniel Borkmann if (!sk_wmem_schedule(sk, size)) { 35604326b4SDaniel Borkmann if (!copied) 36604326b4SDaniel Borkmann ret = -ENOMEM; 37604326b4SDaniel Borkmann break; 38604326b4SDaniel Borkmann } 39604326b4SDaniel Borkmann 40604326b4SDaniel Borkmann sk_mem_charge(sk, size); 41604326b4SDaniel Borkmann sk_msg_xfer(tmp, msg, i, size); 42604326b4SDaniel Borkmann copied += size; 43604326b4SDaniel Borkmann if (sge->length) 44604326b4SDaniel Borkmann get_page(sk_msg_page(tmp, i)); 45604326b4SDaniel Borkmann sk_msg_iter_var_next(i); 46604326b4SDaniel Borkmann tmp->sg.end = i; 47604326b4SDaniel Borkmann if (apply) { 48604326b4SDaniel Borkmann apply_bytes -= size; 499072931fSPengcheng Yang if (!apply_bytes) { 509072931fSPengcheng Yang if (sge->length) 519072931fSPengcheng Yang sk_msg_iter_var_prev(i); 52604326b4SDaniel Borkmann break; 53604326b4SDaniel Borkmann } 549072931fSPengcheng Yang } 55604326b4SDaniel Borkmann } while (i != msg->sg.end); 56604326b4SDaniel Borkmann 57604326b4SDaniel Borkmann if (!ret) { 58604326b4SDaniel Borkmann msg->sg.start = i; 59604326b4SDaniel Borkmann sk_psock_queue_msg(psock, tmp); 60552de910SJohn Fastabend sk_psock_data_ready(sk, psock); 61604326b4SDaniel Borkmann } else { 62604326b4SDaniel Borkmann sk_msg_free(sk, tmp); 63604326b4SDaniel Borkmann kfree(tmp); 64604326b4SDaniel Borkmann } 65604326b4SDaniel Borkmann 66604326b4SDaniel Borkmann release_sock(sk); 67604326b4SDaniel Borkmann return ret; 68604326b4SDaniel Borkmann } 69604326b4SDaniel Borkmann 70604326b4SDaniel Borkmann static int tcp_bpf_push(struct sock *sk, struct sk_msg *msg, u32 apply_bytes, 71604326b4SDaniel Borkmann int flags, bool uncharge) 72604326b4SDaniel Borkmann { 73604326b4SDaniel Borkmann bool apply = apply_bytes; 74604326b4SDaniel Borkmann struct scatterlist *sge; 75604326b4SDaniel Borkmann struct page *page; 76604326b4SDaniel Borkmann int size, ret = 0; 77604326b4SDaniel Borkmann u32 off; 78604326b4SDaniel Borkmann 79604326b4SDaniel Borkmann while (1) { 800608c69cSJohn Fastabend bool has_tx_ulp; 810608c69cSJohn Fastabend 82604326b4SDaniel Borkmann sge = sk_msg_elem(msg, msg->sg.start); 83604326b4SDaniel Borkmann size = (apply && apply_bytes < sge->length) ? 84604326b4SDaniel Borkmann apply_bytes : sge->length; 85604326b4SDaniel Borkmann off = sge->offset; 86604326b4SDaniel Borkmann page = sg_page(sge); 87604326b4SDaniel Borkmann 88604326b4SDaniel Borkmann tcp_rate_check_app_limited(sk); 89604326b4SDaniel Borkmann retry: 900608c69cSJohn Fastabend has_tx_ulp = tls_sw_has_ctx_tx(sk); 910608c69cSJohn Fastabend if (has_tx_ulp) { 920608c69cSJohn Fastabend flags |= MSG_SENDPAGE_NOPOLICY; 930608c69cSJohn Fastabend ret = kernel_sendpage_locked(sk, 940608c69cSJohn Fastabend page, off, size, flags); 950608c69cSJohn Fastabend } else { 96604326b4SDaniel Borkmann ret = do_tcp_sendpages(sk, page, off, size, flags); 970608c69cSJohn Fastabend } 980608c69cSJohn Fastabend 99604326b4SDaniel Borkmann if (ret <= 0) 100604326b4SDaniel Borkmann return ret; 101604326b4SDaniel Borkmann if (apply) 102604326b4SDaniel Borkmann apply_bytes -= ret; 103604326b4SDaniel Borkmann msg->sg.size -= ret; 104604326b4SDaniel Borkmann sge->offset += ret; 105604326b4SDaniel Borkmann sge->length -= ret; 106604326b4SDaniel Borkmann if (uncharge) 107604326b4SDaniel Borkmann sk_mem_uncharge(sk, ret); 108604326b4SDaniel Borkmann if (ret != size) { 109604326b4SDaniel Borkmann size -= ret; 110604326b4SDaniel Borkmann off += ret; 111604326b4SDaniel Borkmann goto retry; 112604326b4SDaniel Borkmann } 113604326b4SDaniel Borkmann if (!sge->length) { 114604326b4SDaniel Borkmann put_page(page); 115604326b4SDaniel Borkmann sk_msg_iter_next(msg, start); 116604326b4SDaniel Borkmann sg_init_table(sge, 1); 117604326b4SDaniel Borkmann if (msg->sg.start == msg->sg.end) 118604326b4SDaniel Borkmann break; 119604326b4SDaniel Borkmann } 120604326b4SDaniel Borkmann if (apply && !apply_bytes) 121604326b4SDaniel Borkmann break; 122604326b4SDaniel Borkmann } 123604326b4SDaniel Borkmann 124604326b4SDaniel Borkmann return 0; 125604326b4SDaniel Borkmann } 126604326b4SDaniel Borkmann 127604326b4SDaniel Borkmann static int tcp_bpf_push_locked(struct sock *sk, struct sk_msg *msg, 128604326b4SDaniel Borkmann u32 apply_bytes, int flags, bool uncharge) 129604326b4SDaniel Borkmann { 130604326b4SDaniel Borkmann int ret; 131604326b4SDaniel Borkmann 132604326b4SDaniel Borkmann lock_sock(sk); 133604326b4SDaniel Borkmann ret = tcp_bpf_push(sk, msg, apply_bytes, flags, uncharge); 134604326b4SDaniel Borkmann release_sock(sk); 135604326b4SDaniel Borkmann return ret; 136604326b4SDaniel Borkmann } 137604326b4SDaniel Borkmann 138a351d608SPengcheng Yang int tcp_bpf_sendmsg_redir(struct sock *sk, bool ingress, 139a351d608SPengcheng Yang struct sk_msg *msg, u32 bytes, int flags) 140604326b4SDaniel Borkmann { 141604326b4SDaniel Borkmann struct sk_psock *psock = sk_psock_get(sk); 142604326b4SDaniel Borkmann int ret; 143604326b4SDaniel Borkmann 1442486ab43SWang Yufen if (unlikely(!psock)) 1452486ab43SWang Yufen return -EPIPE; 1462486ab43SWang Yufen 147604326b4SDaniel Borkmann ret = ingress ? bpf_tcp_ingress(sk, psock, msg, bytes, flags) : 148604326b4SDaniel Borkmann tcp_bpf_push_locked(sk, msg, bytes, flags, false); 149604326b4SDaniel Borkmann sk_psock_put(sk, psock); 150604326b4SDaniel Borkmann return ret; 151604326b4SDaniel Borkmann } 152604326b4SDaniel Borkmann EXPORT_SYMBOL_GPL(tcp_bpf_sendmsg_redir); 153604326b4SDaniel Borkmann 15488759609SCong Wang #ifdef CONFIG_BPF_SYSCALL 155b6df0078SJakub Kicinski static int tcp_msg_wait_data(struct sock *sk, struct sk_psock *psock, 156b6df0078SJakub Kicinski long timeo) 1579f2470fbSCong Wang { 1589f2470fbSCong Wang DEFINE_WAIT_FUNC(wait, woken_wake_function); 1599f2470fbSCong Wang int ret = 0; 1609f2470fbSCong Wang 1619f2470fbSCong Wang if (sk->sk_shutdown & RCV_SHUTDOWN) 1629f2470fbSCong Wang return 1; 1639f2470fbSCong Wang 1649f2470fbSCong Wang if (!timeo) 1659f2470fbSCong Wang return ret; 1669f2470fbSCong Wang 1679f2470fbSCong Wang add_wait_queue(sk_sleep(sk), &wait); 1689f2470fbSCong Wang sk_set_bit(SOCKWQ_ASYNC_WAITDATA, sk); 1699f2470fbSCong Wang ret = sk_wait_event(sk, &timeo, 1709f2470fbSCong Wang !list_empty(&psock->ingress_msg) || 171d0ac89f6SEric Dumazet !skb_queue_empty_lockless(&sk->sk_receive_queue), &wait); 1729f2470fbSCong Wang sk_clear_bit(SOCKWQ_ASYNC_WAITDATA, sk); 1739f2470fbSCong Wang remove_wait_queue(sk_sleep(sk), &wait); 1749f2470fbSCong Wang return ret; 1759f2470fbSCong Wang } 1769f2470fbSCong Wang 177901546fdSJohn Fastabend static bool is_next_msg_fin(struct sk_psock *psock) 178901546fdSJohn Fastabend { 179901546fdSJohn Fastabend struct scatterlist *sge; 180901546fdSJohn Fastabend struct sk_msg *msg_rx; 181901546fdSJohn Fastabend int i; 182901546fdSJohn Fastabend 183901546fdSJohn Fastabend msg_rx = sk_psock_peek_msg(psock); 184901546fdSJohn Fastabend i = msg_rx->sg.start; 185901546fdSJohn Fastabend sge = sk_msg_elem(msg_rx, i); 186901546fdSJohn Fastabend if (!sge->length) { 187901546fdSJohn Fastabend struct sk_buff *skb = msg_rx->skb; 188901546fdSJohn Fastabend 189901546fdSJohn Fastabend if (skb && TCP_SKB_CB(skb)->tcp_flags & TCPHDR_FIN) 190901546fdSJohn Fastabend return true; 191901546fdSJohn Fastabend } 192901546fdSJohn Fastabend return false; 193901546fdSJohn Fastabend } 194901546fdSJohn Fastabend 195c5d2177aSJohn Fastabend static int tcp_bpf_recvmsg_parser(struct sock *sk, 196c5d2177aSJohn Fastabend struct msghdr *msg, 197c5d2177aSJohn Fastabend size_t len, 198c5d2177aSJohn Fastabend int flags, 199c5d2177aSJohn Fastabend int *addr_len) 200c5d2177aSJohn Fastabend { 201c5d2177aSJohn Fastabend struct sk_psock *psock; 202c5d2177aSJohn Fastabend int copied; 203c5d2177aSJohn Fastabend 204c5d2177aSJohn Fastabend if (unlikely(flags & MSG_ERRQUEUE)) 205c5d2177aSJohn Fastabend return inet_recv_error(sk, msg, len, addr_len); 206c5d2177aSJohn Fastabend 207d900f3d2SLiu Jian if (!len) 208d900f3d2SLiu Jian return 0; 209d900f3d2SLiu Jian 210c5d2177aSJohn Fastabend psock = sk_psock_get(sk); 211c5d2177aSJohn Fastabend if (unlikely(!psock)) 212ec095263SOliver Hartkopp return tcp_recvmsg(sk, msg, len, flags, addr_len); 213c5d2177aSJohn Fastabend 214c5d2177aSJohn Fastabend lock_sock(sk); 215*ea444185SJohn Fastabend 216*ea444185SJohn Fastabend /* We may have received data on the sk_receive_queue pre-accept and 217*ea444185SJohn Fastabend * then we can not use read_skb in this context because we haven't 218*ea444185SJohn Fastabend * assigned a sk_socket yet so have no link to the ops. The work-around 219*ea444185SJohn Fastabend * is to check the sk_receive_queue and in these cases read skbs off 220*ea444185SJohn Fastabend * queue again. The read_skb hook is not running at this point because 221*ea444185SJohn Fastabend * of lock_sock so we avoid having multiple runners in read_skb. 222*ea444185SJohn Fastabend */ 223*ea444185SJohn Fastabend if (unlikely(!skb_queue_empty(&sk->sk_receive_queue))) { 224*ea444185SJohn Fastabend tcp_data_ready(sk); 225*ea444185SJohn Fastabend /* This handles the ENOMEM errors if we both receive data 226*ea444185SJohn Fastabend * pre accept and are already under memory pressure. At least 227*ea444185SJohn Fastabend * let user know to retry. 228*ea444185SJohn Fastabend */ 229*ea444185SJohn Fastabend if (unlikely(!skb_queue_empty(&sk->sk_receive_queue))) { 230*ea444185SJohn Fastabend copied = -EAGAIN; 231*ea444185SJohn Fastabend goto out; 232*ea444185SJohn Fastabend } 233*ea444185SJohn Fastabend } 234*ea444185SJohn Fastabend 235c5d2177aSJohn Fastabend msg_bytes_ready: 236c5d2177aSJohn Fastabend copied = sk_msg_recvmsg(sk, psock, msg, len, flags); 237901546fdSJohn Fastabend /* The typical case for EFAULT is the socket was gracefully 238901546fdSJohn Fastabend * shutdown with a FIN pkt. So check here the other case is 239901546fdSJohn Fastabend * some error on copy_page_to_iter which would be unexpected. 240901546fdSJohn Fastabend * On fin return correct return code to zero. 241901546fdSJohn Fastabend */ 242901546fdSJohn Fastabend if (copied == -EFAULT) { 243901546fdSJohn Fastabend bool is_fin = is_next_msg_fin(psock); 244901546fdSJohn Fastabend 245901546fdSJohn Fastabend if (is_fin) { 246901546fdSJohn Fastabend copied = 0; 247901546fdSJohn Fastabend goto out; 248901546fdSJohn Fastabend } 249901546fdSJohn Fastabend } 250c5d2177aSJohn Fastabend if (!copied) { 251c5d2177aSJohn Fastabend long timeo; 252c5d2177aSJohn Fastabend int data; 253c5d2177aSJohn Fastabend 2545b2c5540SJohn Fastabend if (sock_flag(sk, SOCK_DONE)) 2555b2c5540SJohn Fastabend goto out; 2565b2c5540SJohn Fastabend 2575b2c5540SJohn Fastabend if (sk->sk_err) { 2585b2c5540SJohn Fastabend copied = sock_error(sk); 2595b2c5540SJohn Fastabend goto out; 2605b2c5540SJohn Fastabend } 2615b2c5540SJohn Fastabend 2625b2c5540SJohn Fastabend if (sk->sk_shutdown & RCV_SHUTDOWN) 2635b2c5540SJohn Fastabend goto out; 2645b2c5540SJohn Fastabend 2655b2c5540SJohn Fastabend if (sk->sk_state == TCP_CLOSE) { 2665b2c5540SJohn Fastabend copied = -ENOTCONN; 2675b2c5540SJohn Fastabend goto out; 2685b2c5540SJohn Fastabend } 2695b2c5540SJohn Fastabend 270ec095263SOliver Hartkopp timeo = sock_rcvtimeo(sk, flags & MSG_DONTWAIT); 2715b2c5540SJohn Fastabend if (!timeo) { 2725b2c5540SJohn Fastabend copied = -EAGAIN; 2735b2c5540SJohn Fastabend goto out; 2745b2c5540SJohn Fastabend } 2755b2c5540SJohn Fastabend 2765b2c5540SJohn Fastabend if (signal_pending(current)) { 2775b2c5540SJohn Fastabend copied = sock_intr_errno(timeo); 2785b2c5540SJohn Fastabend goto out; 2795b2c5540SJohn Fastabend } 2805b2c5540SJohn Fastabend 281c5d2177aSJohn Fastabend data = tcp_msg_wait_data(sk, psock, timeo); 282c5d2177aSJohn Fastabend if (data && !sk_psock_queue_empty(psock)) 283c5d2177aSJohn Fastabend goto msg_bytes_ready; 284c5d2177aSJohn Fastabend copied = -EAGAIN; 285c5d2177aSJohn Fastabend } 2865b2c5540SJohn Fastabend out: 287c5d2177aSJohn Fastabend release_sock(sk); 288c5d2177aSJohn Fastabend sk_psock_put(sk, psock); 289c5d2177aSJohn Fastabend return copied; 290c5d2177aSJohn Fastabend } 291c5d2177aSJohn Fastabend 292c0fd336eSYueHaibing static int tcp_bpf_recvmsg(struct sock *sk, struct msghdr *msg, size_t len, 293ec095263SOliver Hartkopp int flags, int *addr_len) 294c0fd336eSYueHaibing { 295c0fd336eSYueHaibing struct sk_psock *psock; 296c0fd336eSYueHaibing int copied, ret; 297c0fd336eSYueHaibing 29818f02ad1SXiyu Yang if (unlikely(flags & MSG_ERRQUEUE)) 29918f02ad1SXiyu Yang return inet_recv_error(sk, msg, len, addr_len); 30018f02ad1SXiyu Yang 301d900f3d2SLiu Jian if (!len) 302d900f3d2SLiu Jian return 0; 303d900f3d2SLiu Jian 304c0fd336eSYueHaibing psock = sk_psock_get(sk); 305c0fd336eSYueHaibing if (unlikely(!psock)) 306ec095263SOliver Hartkopp return tcp_recvmsg(sk, msg, len, flags, addr_len); 307c0fd336eSYueHaibing if (!skb_queue_empty(&sk->sk_receive_queue) && 30818f02ad1SXiyu Yang sk_psock_queue_empty(psock)) { 30918f02ad1SXiyu Yang sk_psock_put(sk, psock); 310ec095263SOliver Hartkopp return tcp_recvmsg(sk, msg, len, flags, addr_len); 31118f02ad1SXiyu Yang } 312c0fd336eSYueHaibing lock_sock(sk); 313c0fd336eSYueHaibing msg_bytes_ready: 3142bc793e3SCong Wang copied = sk_msg_recvmsg(sk, psock, msg, len, flags); 315c0fd336eSYueHaibing if (!copied) { 316c0fd336eSYueHaibing long timeo; 317c49661aaSCong Wang int data; 318c0fd336eSYueHaibing 319ec095263SOliver Hartkopp timeo = sock_rcvtimeo(sk, flags & MSG_DONTWAIT); 320b6df0078SJakub Kicinski data = tcp_msg_wait_data(sk, psock, timeo); 321c0fd336eSYueHaibing if (data) { 322c0fd336eSYueHaibing if (!sk_psock_queue_empty(psock)) 323c0fd336eSYueHaibing goto msg_bytes_ready; 324c0fd336eSYueHaibing release_sock(sk); 325c0fd336eSYueHaibing sk_psock_put(sk, psock); 326ec095263SOliver Hartkopp return tcp_recvmsg(sk, msg, len, flags, addr_len); 327c0fd336eSYueHaibing } 328c0fd336eSYueHaibing copied = -EAGAIN; 329c0fd336eSYueHaibing } 330c0fd336eSYueHaibing ret = copied; 331c0fd336eSYueHaibing release_sock(sk); 332c0fd336eSYueHaibing sk_psock_put(sk, psock); 333c0fd336eSYueHaibing return ret; 334c0fd336eSYueHaibing } 335c0fd336eSYueHaibing 336604326b4SDaniel Borkmann static int tcp_bpf_send_verdict(struct sock *sk, struct sk_psock *psock, 337604326b4SDaniel Borkmann struct sk_msg *msg, int *copied, int flags) 338604326b4SDaniel Borkmann { 339a351d608SPengcheng Yang bool cork = false, enospc = sk_msg_full(msg), redir_ingress; 340604326b4SDaniel Borkmann struct sock *sk_redir; 3418ec95b94SWang Yufen u32 tosend, origsize, sent, delta = 0; 3427a9841caSPengcheng Yang u32 eval; 343604326b4SDaniel Borkmann int ret; 344604326b4SDaniel Borkmann 345604326b4SDaniel Borkmann more_data: 3467246d8edSJohn Fastabend if (psock->eval == __SK_NONE) { 3477246d8edSJohn Fastabend /* Track delta in msg size to add/subtract it on SK_DROP from 3487246d8edSJohn Fastabend * returned to user copied size. This ensures user doesn't 3497246d8edSJohn Fastabend * get a positive return code with msg_cut_data and SK_DROP 3507246d8edSJohn Fastabend * verdict. 3517246d8edSJohn Fastabend */ 3527246d8edSJohn Fastabend delta = msg->sg.size; 353604326b4SDaniel Borkmann psock->eval = sk_psock_msg_verdict(sk, psock, msg); 3547246d8edSJohn Fastabend delta -= msg->sg.size; 3557246d8edSJohn Fastabend } 356604326b4SDaniel Borkmann 357604326b4SDaniel Borkmann if (msg->cork_bytes && 358604326b4SDaniel Borkmann msg->cork_bytes > msg->sg.size && !enospc) { 359604326b4SDaniel Borkmann psock->cork_bytes = msg->cork_bytes - msg->sg.size; 360604326b4SDaniel Borkmann if (!psock->cork) { 361604326b4SDaniel Borkmann psock->cork = kzalloc(sizeof(*psock->cork), 362604326b4SDaniel Borkmann GFP_ATOMIC | __GFP_NOWARN); 363604326b4SDaniel Borkmann if (!psock->cork) 364604326b4SDaniel Borkmann return -ENOMEM; 365604326b4SDaniel Borkmann } 366604326b4SDaniel Borkmann memcpy(psock->cork, msg, sizeof(*msg)); 367604326b4SDaniel Borkmann return 0; 368604326b4SDaniel Borkmann } 369604326b4SDaniel Borkmann 370604326b4SDaniel Borkmann tosend = msg->sg.size; 371604326b4SDaniel Borkmann if (psock->apply_bytes && psock->apply_bytes < tosend) 372604326b4SDaniel Borkmann tosend = psock->apply_bytes; 3737a9841caSPengcheng Yang eval = __SK_NONE; 374604326b4SDaniel Borkmann 375604326b4SDaniel Borkmann switch (psock->eval) { 376604326b4SDaniel Borkmann case __SK_PASS: 377604326b4SDaniel Borkmann ret = tcp_bpf_push(sk, msg, tosend, flags, true); 378604326b4SDaniel Borkmann if (unlikely(ret)) { 379604326b4SDaniel Borkmann *copied -= sk_msg_free(sk, msg); 380604326b4SDaniel Borkmann break; 381604326b4SDaniel Borkmann } 382604326b4SDaniel Borkmann sk_msg_apply_bytes(psock, tosend); 383604326b4SDaniel Borkmann break; 384604326b4SDaniel Borkmann case __SK_REDIRECT: 385a351d608SPengcheng Yang redir_ingress = psock->redir_ingress; 386604326b4SDaniel Borkmann sk_redir = psock->sk_redir; 387604326b4SDaniel Borkmann sk_msg_apply_bytes(psock, tosend); 388cd9733f5SLiu Jian if (!psock->apply_bytes) { 389cd9733f5SLiu Jian /* Clean up before releasing the sock lock. */ 390cd9733f5SLiu Jian eval = psock->eval; 391cd9733f5SLiu Jian psock->eval = __SK_NONE; 392cd9733f5SLiu Jian psock->sk_redir = NULL; 393cd9733f5SLiu Jian } 394604326b4SDaniel Borkmann if (psock->cork) { 395604326b4SDaniel Borkmann cork = true; 396604326b4SDaniel Borkmann psock->cork = NULL; 397604326b4SDaniel Borkmann } 3988ec95b94SWang Yufen sk_msg_return(sk, msg, tosend); 399604326b4SDaniel Borkmann release_sock(sk); 400cd9733f5SLiu Jian 4018ec95b94SWang Yufen origsize = msg->sg.size; 402a351d608SPengcheng Yang ret = tcp_bpf_sendmsg_redir(sk_redir, redir_ingress, 403a351d608SPengcheng Yang msg, tosend, flags); 4048ec95b94SWang Yufen sent = origsize - msg->sg.size; 405cd9733f5SLiu Jian 406cd9733f5SLiu Jian if (eval == __SK_REDIRECT) 407cd9733f5SLiu Jian sock_put(sk_redir); 408cd9733f5SLiu Jian 409604326b4SDaniel Borkmann lock_sock(sk); 410604326b4SDaniel Borkmann if (unlikely(ret < 0)) { 411604326b4SDaniel Borkmann int free = sk_msg_free_nocharge(sk, msg); 412604326b4SDaniel Borkmann 413604326b4SDaniel Borkmann if (!cork) 414604326b4SDaniel Borkmann *copied -= free; 415604326b4SDaniel Borkmann } 416604326b4SDaniel Borkmann if (cork) { 417604326b4SDaniel Borkmann sk_msg_free(sk, msg); 418604326b4SDaniel Borkmann kfree(msg); 419604326b4SDaniel Borkmann msg = NULL; 420604326b4SDaniel Borkmann ret = 0; 421604326b4SDaniel Borkmann } 422604326b4SDaniel Borkmann break; 423604326b4SDaniel Borkmann case __SK_DROP: 424604326b4SDaniel Borkmann default: 425604326b4SDaniel Borkmann sk_msg_free_partial(sk, msg, tosend); 426604326b4SDaniel Borkmann sk_msg_apply_bytes(psock, tosend); 4277246d8edSJohn Fastabend *copied -= (tosend + delta); 428604326b4SDaniel Borkmann return -EACCES; 429604326b4SDaniel Borkmann } 430604326b4SDaniel Borkmann 431604326b4SDaniel Borkmann if (likely(!ret)) { 432604326b4SDaniel Borkmann if (!psock->apply_bytes) { 433604326b4SDaniel Borkmann psock->eval = __SK_NONE; 434604326b4SDaniel Borkmann if (psock->sk_redir) { 435604326b4SDaniel Borkmann sock_put(psock->sk_redir); 436604326b4SDaniel Borkmann psock->sk_redir = NULL; 437604326b4SDaniel Borkmann } 438604326b4SDaniel Borkmann } 439604326b4SDaniel Borkmann if (msg && 440604326b4SDaniel Borkmann msg->sg.data[msg->sg.start].page_link && 44184472b43SWang Yufen msg->sg.data[msg->sg.start].length) { 44284472b43SWang Yufen if (eval == __SK_REDIRECT) 4438ec95b94SWang Yufen sk_mem_charge(sk, tosend - sent); 444604326b4SDaniel Borkmann goto more_data; 445604326b4SDaniel Borkmann } 44684472b43SWang Yufen } 447604326b4SDaniel Borkmann return ret; 448604326b4SDaniel Borkmann } 449604326b4SDaniel Borkmann 450604326b4SDaniel Borkmann static int tcp_bpf_sendmsg(struct sock *sk, struct msghdr *msg, size_t size) 451604326b4SDaniel Borkmann { 452604326b4SDaniel Borkmann struct sk_msg tmp, *msg_tx = NULL; 453604326b4SDaniel Borkmann int copied = 0, err = 0; 454604326b4SDaniel Borkmann struct sk_psock *psock; 455604326b4SDaniel Borkmann long timeo; 45641477662SJakub Kicinski int flags; 45741477662SJakub Kicinski 45841477662SJakub Kicinski /* Don't let internal do_tcp_sendpages() flags through */ 45941477662SJakub Kicinski flags = (msg->msg_flags & ~MSG_SENDPAGE_DECRYPTED); 46041477662SJakub Kicinski flags |= MSG_NO_SHARED_FRAGS; 461604326b4SDaniel Borkmann 462604326b4SDaniel Borkmann psock = sk_psock_get(sk); 463604326b4SDaniel Borkmann if (unlikely(!psock)) 464604326b4SDaniel Borkmann return tcp_sendmsg(sk, msg, size); 465604326b4SDaniel Borkmann 466604326b4SDaniel Borkmann lock_sock(sk); 467604326b4SDaniel Borkmann timeo = sock_sndtimeo(sk, msg->msg_flags & MSG_DONTWAIT); 468604326b4SDaniel Borkmann while (msg_data_left(msg)) { 469604326b4SDaniel Borkmann bool enospc = false; 470604326b4SDaniel Borkmann u32 copy, osize; 471604326b4SDaniel Borkmann 472604326b4SDaniel Borkmann if (sk->sk_err) { 473604326b4SDaniel Borkmann err = -sk->sk_err; 474604326b4SDaniel Borkmann goto out_err; 475604326b4SDaniel Borkmann } 476604326b4SDaniel Borkmann 477604326b4SDaniel Borkmann copy = msg_data_left(msg); 478604326b4SDaniel Borkmann if (!sk_stream_memory_free(sk)) 479604326b4SDaniel Borkmann goto wait_for_sndbuf; 480604326b4SDaniel Borkmann if (psock->cork) { 481604326b4SDaniel Borkmann msg_tx = psock->cork; 482604326b4SDaniel Borkmann } else { 483604326b4SDaniel Borkmann msg_tx = &tmp; 484604326b4SDaniel Borkmann sk_msg_init(msg_tx); 485604326b4SDaniel Borkmann } 486604326b4SDaniel Borkmann 487604326b4SDaniel Borkmann osize = msg_tx->sg.size; 488604326b4SDaniel Borkmann err = sk_msg_alloc(sk, msg_tx, msg_tx->sg.size + copy, msg_tx->sg.end - 1); 489604326b4SDaniel Borkmann if (err) { 490604326b4SDaniel Borkmann if (err != -ENOSPC) 491604326b4SDaniel Borkmann goto wait_for_memory; 492604326b4SDaniel Borkmann enospc = true; 493604326b4SDaniel Borkmann copy = msg_tx->sg.size - osize; 494604326b4SDaniel Borkmann } 495604326b4SDaniel Borkmann 496604326b4SDaniel Borkmann err = sk_msg_memcopy_from_iter(sk, &msg->msg_iter, msg_tx, 497604326b4SDaniel Borkmann copy); 498604326b4SDaniel Borkmann if (err < 0) { 499604326b4SDaniel Borkmann sk_msg_trim(sk, msg_tx, osize); 500604326b4SDaniel Borkmann goto out_err; 501604326b4SDaniel Borkmann } 502604326b4SDaniel Borkmann 503604326b4SDaniel Borkmann copied += copy; 504604326b4SDaniel Borkmann if (psock->cork_bytes) { 505604326b4SDaniel Borkmann if (size > psock->cork_bytes) 506604326b4SDaniel Borkmann psock->cork_bytes = 0; 507604326b4SDaniel Borkmann else 508604326b4SDaniel Borkmann psock->cork_bytes -= size; 509604326b4SDaniel Borkmann if (psock->cork_bytes && !enospc) 510604326b4SDaniel Borkmann goto out_err; 511604326b4SDaniel Borkmann /* All cork bytes are accounted, rerun the prog. */ 512604326b4SDaniel Borkmann psock->eval = __SK_NONE; 513604326b4SDaniel Borkmann psock->cork_bytes = 0; 514604326b4SDaniel Borkmann } 515604326b4SDaniel Borkmann 516604326b4SDaniel Borkmann err = tcp_bpf_send_verdict(sk, psock, msg_tx, &copied, flags); 517604326b4SDaniel Borkmann if (unlikely(err < 0)) 518604326b4SDaniel Borkmann goto out_err; 519604326b4SDaniel Borkmann continue; 520604326b4SDaniel Borkmann wait_for_sndbuf: 521604326b4SDaniel Borkmann set_bit(SOCK_NOSPACE, &sk->sk_socket->flags); 522604326b4SDaniel Borkmann wait_for_memory: 523604326b4SDaniel Borkmann err = sk_stream_wait_memory(sk, &timeo); 524604326b4SDaniel Borkmann if (err) { 525604326b4SDaniel Borkmann if (msg_tx && msg_tx != psock->cork) 526604326b4SDaniel Borkmann sk_msg_free(sk, msg_tx); 527604326b4SDaniel Borkmann goto out_err; 528604326b4SDaniel Borkmann } 529604326b4SDaniel Borkmann } 530604326b4SDaniel Borkmann out_err: 531604326b4SDaniel Borkmann if (err < 0) 532604326b4SDaniel Borkmann err = sk_stream_error(sk, msg->msg_flags, err); 533604326b4SDaniel Borkmann release_sock(sk); 534604326b4SDaniel Borkmann sk_psock_put(sk, psock); 535604326b4SDaniel Borkmann return copied ? copied : err; 536604326b4SDaniel Borkmann } 537604326b4SDaniel Borkmann 538604326b4SDaniel Borkmann static int tcp_bpf_sendpage(struct sock *sk, struct page *page, int offset, 539604326b4SDaniel Borkmann size_t size, int flags) 540604326b4SDaniel Borkmann { 541604326b4SDaniel Borkmann struct sk_msg tmp, *msg = NULL; 542604326b4SDaniel Borkmann int err = 0, copied = 0; 543604326b4SDaniel Borkmann struct sk_psock *psock; 544604326b4SDaniel Borkmann bool enospc = false; 545604326b4SDaniel Borkmann 546604326b4SDaniel Borkmann psock = sk_psock_get(sk); 547604326b4SDaniel Borkmann if (unlikely(!psock)) 548604326b4SDaniel Borkmann return tcp_sendpage(sk, page, offset, size, flags); 549604326b4SDaniel Borkmann 550604326b4SDaniel Borkmann lock_sock(sk); 551604326b4SDaniel Borkmann if (psock->cork) { 552604326b4SDaniel Borkmann msg = psock->cork; 553604326b4SDaniel Borkmann } else { 554604326b4SDaniel Borkmann msg = &tmp; 555604326b4SDaniel Borkmann sk_msg_init(msg); 556604326b4SDaniel Borkmann } 557604326b4SDaniel Borkmann 558604326b4SDaniel Borkmann /* Catch case where ring is full and sendpage is stalled. */ 559604326b4SDaniel Borkmann if (unlikely(sk_msg_full(msg))) 560604326b4SDaniel Borkmann goto out_err; 561604326b4SDaniel Borkmann 562604326b4SDaniel Borkmann sk_msg_page_add(msg, page, size, offset); 563604326b4SDaniel Borkmann sk_mem_charge(sk, size); 564604326b4SDaniel Borkmann copied = size; 565604326b4SDaniel Borkmann if (sk_msg_full(msg)) 566604326b4SDaniel Borkmann enospc = true; 567604326b4SDaniel Borkmann if (psock->cork_bytes) { 568604326b4SDaniel Borkmann if (size > psock->cork_bytes) 569604326b4SDaniel Borkmann psock->cork_bytes = 0; 570604326b4SDaniel Borkmann else 571604326b4SDaniel Borkmann psock->cork_bytes -= size; 572604326b4SDaniel Borkmann if (psock->cork_bytes && !enospc) 573604326b4SDaniel Borkmann goto out_err; 574604326b4SDaniel Borkmann /* All cork bytes are accounted, rerun the prog. */ 575604326b4SDaniel Borkmann psock->eval = __SK_NONE; 576604326b4SDaniel Borkmann psock->cork_bytes = 0; 577604326b4SDaniel Borkmann } 578604326b4SDaniel Borkmann 579604326b4SDaniel Borkmann err = tcp_bpf_send_verdict(sk, psock, msg, &copied, flags); 580604326b4SDaniel Borkmann out_err: 581604326b4SDaniel Borkmann release_sock(sk); 582604326b4SDaniel Borkmann sk_psock_put(sk, psock); 583604326b4SDaniel Borkmann return copied ? copied : err; 584604326b4SDaniel Borkmann } 585604326b4SDaniel Borkmann 586604326b4SDaniel Borkmann enum { 587604326b4SDaniel Borkmann TCP_BPF_IPV4, 588604326b4SDaniel Borkmann TCP_BPF_IPV6, 589604326b4SDaniel Borkmann TCP_BPF_NUM_PROTS, 590604326b4SDaniel Borkmann }; 591604326b4SDaniel Borkmann 592604326b4SDaniel Borkmann enum { 593604326b4SDaniel Borkmann TCP_BPF_BASE, 594604326b4SDaniel Borkmann TCP_BPF_TX, 595c5d2177aSJohn Fastabend TCP_BPF_RX, 596c5d2177aSJohn Fastabend TCP_BPF_TXRX, 597604326b4SDaniel Borkmann TCP_BPF_NUM_CFGS, 598604326b4SDaniel Borkmann }; 599604326b4SDaniel Borkmann 600604326b4SDaniel Borkmann static struct proto *tcpv6_prot_saved __read_mostly; 601604326b4SDaniel Borkmann static DEFINE_SPINLOCK(tcpv6_prot_lock); 602604326b4SDaniel Borkmann static struct proto tcp_bpf_prots[TCP_BPF_NUM_PROTS][TCP_BPF_NUM_CFGS]; 603604326b4SDaniel Borkmann 604604326b4SDaniel Borkmann static void tcp_bpf_rebuild_protos(struct proto prot[TCP_BPF_NUM_CFGS], 605604326b4SDaniel Borkmann struct proto *base) 606604326b4SDaniel Borkmann { 607604326b4SDaniel Borkmann prot[TCP_BPF_BASE] = *base; 608d8616ee2SWang Yufen prot[TCP_BPF_BASE].destroy = sock_map_destroy; 609f747632bSLorenz Bauer prot[TCP_BPF_BASE].close = sock_map_close; 610604326b4SDaniel Borkmann prot[TCP_BPF_BASE].recvmsg = tcp_bpf_recvmsg; 611fb4e0a5eSCong Wang prot[TCP_BPF_BASE].sock_is_readable = sk_msg_is_readable; 612604326b4SDaniel Borkmann 613604326b4SDaniel Borkmann prot[TCP_BPF_TX] = prot[TCP_BPF_BASE]; 614604326b4SDaniel Borkmann prot[TCP_BPF_TX].sendmsg = tcp_bpf_sendmsg; 615604326b4SDaniel Borkmann prot[TCP_BPF_TX].sendpage = tcp_bpf_sendpage; 616c5d2177aSJohn Fastabend 617c5d2177aSJohn Fastabend prot[TCP_BPF_RX] = prot[TCP_BPF_BASE]; 618c5d2177aSJohn Fastabend prot[TCP_BPF_RX].recvmsg = tcp_bpf_recvmsg_parser; 619c5d2177aSJohn Fastabend 620c5d2177aSJohn Fastabend prot[TCP_BPF_TXRX] = prot[TCP_BPF_TX]; 621c5d2177aSJohn Fastabend prot[TCP_BPF_TXRX].recvmsg = tcp_bpf_recvmsg_parser; 622604326b4SDaniel Borkmann } 623604326b4SDaniel Borkmann 6247b219da4SLorenz Bauer static void tcp_bpf_check_v6_needs_rebuild(struct proto *ops) 625604326b4SDaniel Borkmann { 6267b219da4SLorenz Bauer if (unlikely(ops != smp_load_acquire(&tcpv6_prot_saved))) { 627604326b4SDaniel Borkmann spin_lock_bh(&tcpv6_prot_lock); 628604326b4SDaniel Borkmann if (likely(ops != tcpv6_prot_saved)) { 629604326b4SDaniel Borkmann tcp_bpf_rebuild_protos(tcp_bpf_prots[TCP_BPF_IPV6], ops); 630604326b4SDaniel Borkmann smp_store_release(&tcpv6_prot_saved, ops); 631604326b4SDaniel Borkmann } 632604326b4SDaniel Borkmann spin_unlock_bh(&tcpv6_prot_lock); 633604326b4SDaniel Borkmann } 634604326b4SDaniel Borkmann } 635604326b4SDaniel Borkmann 636604326b4SDaniel Borkmann static int __init tcp_bpf_v4_build_proto(void) 637604326b4SDaniel Borkmann { 638604326b4SDaniel Borkmann tcp_bpf_rebuild_protos(tcp_bpf_prots[TCP_BPF_IPV4], &tcp_prot); 639604326b4SDaniel Borkmann return 0; 640604326b4SDaniel Borkmann } 641228a4a7bSJohn Fastabend late_initcall(tcp_bpf_v4_build_proto); 642604326b4SDaniel Borkmann 643604326b4SDaniel Borkmann static int tcp_bpf_assert_proto_ops(struct proto *ops) 644604326b4SDaniel Borkmann { 645604326b4SDaniel Borkmann /* In order to avoid retpoline, we make assumptions when we call 646604326b4SDaniel Borkmann * into ops if e.g. a psock is not present. Make sure they are 647604326b4SDaniel Borkmann * indeed valid assumptions. 648604326b4SDaniel Borkmann */ 649604326b4SDaniel Borkmann return ops->recvmsg == tcp_recvmsg && 650604326b4SDaniel Borkmann ops->sendmsg == tcp_sendmsg && 651604326b4SDaniel Borkmann ops->sendpage == tcp_sendpage ? 0 : -ENOTSUPP; 652604326b4SDaniel Borkmann } 653604326b4SDaniel Borkmann 65451e0158aSCong Wang int tcp_bpf_update_proto(struct sock *sk, struct sk_psock *psock, bool restore) 655604326b4SDaniel Borkmann { 656d19da360SLorenz Bauer int family = sk->sk_family == AF_INET6 ? TCP_BPF_IPV6 : TCP_BPF_IPV4; 657d19da360SLorenz Bauer int config = psock->progs.msg_parser ? TCP_BPF_TX : TCP_BPF_BASE; 658604326b4SDaniel Borkmann 659c5d2177aSJohn Fastabend if (psock->progs.stream_verdict || psock->progs.skb_verdict) { 660c5d2177aSJohn Fastabend config = (config == TCP_BPF_TX) ? TCP_BPF_TXRX : TCP_BPF_RX; 661c5d2177aSJohn Fastabend } 662c5d2177aSJohn Fastabend 6638a59f9d1SCong Wang if (restore) { 6648a59f9d1SCong Wang if (inet_csk_has_ulp(sk)) { 6658859a44eSJakub Kicinski /* TLS does not have an unhash proto in SW cases, 6668859a44eSJakub Kicinski * but we need to ensure we stop using the sock_map 6678859a44eSJakub Kicinski * unhash routine because the associated psock is being 6688859a44eSJakub Kicinski * removed. So use the original unhash handler. 6698859a44eSJakub Kicinski */ 6708859a44eSJakub Kicinski WRITE_ONCE(sk->sk_prot->unhash, psock->saved_unhash); 6718a59f9d1SCong Wang tcp_update_ulp(sk, psock->sk_proto, psock->saved_write_space); 6728a59f9d1SCong Wang } else { 6738a59f9d1SCong Wang sk->sk_write_space = psock->saved_write_space; 6748a59f9d1SCong Wang /* Pairs with lockless read in sk_clone_lock() */ 675fee9ac06SPavel Begunkov sock_replace_proto(sk, psock->sk_proto); 6768a59f9d1SCong Wang } 6778a59f9d1SCong Wang return 0; 6788a59f9d1SCong Wang } 6798a59f9d1SCong Wang 6807b219da4SLorenz Bauer if (sk->sk_family == AF_INET6) { 6817b219da4SLorenz Bauer if (tcp_bpf_assert_proto_ops(psock->sk_proto)) 6828a59f9d1SCong Wang return -EINVAL; 683d19da360SLorenz Bauer 6847b219da4SLorenz Bauer tcp_bpf_check_v6_needs_rebuild(psock->sk_proto); 685d19da360SLorenz Bauer } 686d19da360SLorenz Bauer 6878a59f9d1SCong Wang /* Pairs with lockless read in sk_clone_lock() */ 688fee9ac06SPavel Begunkov sock_replace_proto(sk, &tcp_bpf_prots[family][config]); 6898a59f9d1SCong Wang return 0; 690604326b4SDaniel Borkmann } 6918a59f9d1SCong Wang EXPORT_SYMBOL_GPL(tcp_bpf_update_proto); 692604326b4SDaniel Borkmann 693e8025155SJakub Sitnicki /* If a child got cloned from a listening socket that had tcp_bpf 694e8025155SJakub Sitnicki * protocol callbacks installed, we need to restore the callbacks to 695e8025155SJakub Sitnicki * the default ones because the child does not inherit the psock state 696e8025155SJakub Sitnicki * that tcp_bpf callbacks expect. 697e8025155SJakub Sitnicki */ 698e8025155SJakub Sitnicki void tcp_bpf_clone(const struct sock *sk, struct sock *newsk) 699e8025155SJakub Sitnicki { 700e8025155SJakub Sitnicki struct proto *prot = newsk->sk_prot; 701e8025155SJakub Sitnicki 702ddce1e09SJakub Sitnicki if (is_insidevar(prot, tcp_bpf_prots)) 703e8025155SJakub Sitnicki newsk->sk_prot = sk->sk_prot_creator; 704e8025155SJakub Sitnicki } 70588759609SCong Wang #endif /* CONFIG_BPF_SYSCALL */ 706