1 // SPDX-License-Identifier: GPL-2.0 2 /* Copyright (c) 2021 Cong Wang <cong.wang@bytedance.com> */ 3 4 #include <linux/skmsg.h> 5 #include <linux/bpf.h> 6 #include <net/sock.h> 7 #include <net/af_unix.h> 8 9 #define unix_sk_has_data(__sk, __psock) \ 10 ({ !skb_queue_empty(&__sk->sk_receive_queue) || \ 11 !skb_queue_empty(&__psock->ingress_skb) || \ 12 !list_empty(&__psock->ingress_msg); \ 13 }) 14 15 static int unix_msg_wait_data(struct sock *sk, struct sk_psock *psock, 16 long timeo) 17 { 18 DEFINE_WAIT_FUNC(wait, woken_wake_function); 19 struct unix_sock *u = unix_sk(sk); 20 int ret = 0; 21 22 if (sk->sk_shutdown & RCV_SHUTDOWN) 23 return 1; 24 25 if (!timeo) 26 return ret; 27 28 add_wait_queue(sk_sleep(sk), &wait); 29 sk_set_bit(SOCKWQ_ASYNC_WAITDATA, sk); 30 if (!unix_sk_has_data(sk, psock)) { 31 mutex_unlock(&u->iolock); 32 wait_woken(&wait, TASK_INTERRUPTIBLE, timeo); 33 mutex_lock(&u->iolock); 34 ret = unix_sk_has_data(sk, psock); 35 } 36 sk_clear_bit(SOCKWQ_ASYNC_WAITDATA, sk); 37 remove_wait_queue(sk_sleep(sk), &wait); 38 return ret; 39 } 40 41 static int unix_dgram_bpf_recvmsg(struct sock *sk, struct msghdr *msg, 42 size_t len, int nonblock, int flags, 43 int *addr_len) 44 { 45 struct unix_sock *u = unix_sk(sk); 46 struct sk_psock *psock; 47 int copied; 48 49 psock = sk_psock_get(sk); 50 if (unlikely(!psock)) 51 return __unix_dgram_recvmsg(sk, msg, len, flags); 52 53 mutex_lock(&u->iolock); 54 if (!skb_queue_empty(&sk->sk_receive_queue) && 55 sk_psock_queue_empty(psock)) { 56 mutex_unlock(&u->iolock); 57 sk_psock_put(sk, psock); 58 return __unix_dgram_recvmsg(sk, msg, len, flags); 59 } 60 61 msg_bytes_ready: 62 copied = sk_msg_recvmsg(sk, psock, msg, len, flags); 63 if (!copied) { 64 long timeo; 65 int data; 66 67 timeo = sock_rcvtimeo(sk, nonblock); 68 data = unix_msg_wait_data(sk, psock, timeo); 69 if (data) { 70 if (!sk_psock_queue_empty(psock)) 71 goto msg_bytes_ready; 72 mutex_unlock(&u->iolock); 73 sk_psock_put(sk, psock); 74 return __unix_dgram_recvmsg(sk, msg, len, flags); 75 } 76 copied = -EAGAIN; 77 } 78 mutex_unlock(&u->iolock); 79 sk_psock_put(sk, psock); 80 return copied; 81 } 82 83 static struct proto *unix_prot_saved __read_mostly; 84 static DEFINE_SPINLOCK(unix_prot_lock); 85 static struct proto unix_bpf_prot; 86 87 static void unix_bpf_rebuild_protos(struct proto *prot, const struct proto *base) 88 { 89 *prot = *base; 90 prot->close = sock_map_close; 91 prot->recvmsg = unix_dgram_bpf_recvmsg; 92 } 93 94 static void unix_bpf_check_needs_rebuild(struct proto *ops) 95 { 96 if (unlikely(ops != smp_load_acquire(&unix_prot_saved))) { 97 spin_lock_bh(&unix_prot_lock); 98 if (likely(ops != unix_prot_saved)) { 99 unix_bpf_rebuild_protos(&unix_bpf_prot, ops); 100 smp_store_release(&unix_prot_saved, ops); 101 } 102 spin_unlock_bh(&unix_prot_lock); 103 } 104 } 105 106 int unix_bpf_update_proto(struct sock *sk, struct sk_psock *psock, bool restore) 107 { 108 if (restore) { 109 sk->sk_write_space = psock->saved_write_space; 110 WRITE_ONCE(sk->sk_prot, psock->sk_proto); 111 return 0; 112 } 113 114 unix_bpf_check_needs_rebuild(psock->sk_proto); 115 WRITE_ONCE(sk->sk_prot, &unix_bpf_prot); 116 return 0; 117 } 118 119 void __init unix_bpf_build_proto(void) 120 { 121 unix_bpf_rebuild_protos(&unix_bpf_prot, &unix_proto); 122 } 123