1d2912cb1SThomas Gleixner // SPDX-License-Identifier: GPL-2.0-only
29b73896aSTom Herbert /*
39b73896aSTom Herbert * Kernel Connection Multiplexor
49b73896aSTom Herbert *
59b73896aSTom Herbert * Copyright (c) 2016 Tom Herbert <tom@herbertland.com>
69b73896aSTom Herbert */
79b73896aSTom Herbert
8ab7ac4ebSTom Herbert #include <linux/bpf.h>
9ab7ac4ebSTom Herbert #include <linux/errno.h>
10ab7ac4ebSTom Herbert #include <linux/errqueue.h>
11ab7ac4ebSTom Herbert #include <linux/file.h>
12b6459415SJakub Kicinski #include <linux/filter.h>
13ab7ac4ebSTom Herbert #include <linux/in.h>
14ab7ac4ebSTom Herbert #include <linux/kernel.h>
15ab7ac4ebSTom Herbert #include <linux/module.h>
16ab7ac4ebSTom Herbert #include <linux/net.h>
17ab7ac4ebSTom Herbert #include <linux/netdevice.h>
18ab7ac4ebSTom Herbert #include <linux/poll.h>
19ab7ac4ebSTom Herbert #include <linux/rculist.h>
20ab7ac4ebSTom Herbert #include <linux/skbuff.h>
21ab7ac4ebSTom Herbert #include <linux/socket.h>
22ab7ac4ebSTom Herbert #include <linux/uaccess.h>
23ab7ac4ebSTom Herbert #include <linux/workqueue.h>
24c0338affSWANG Cong #include <linux/syscalls.h>
25174cd4b1SIngo Molnar #include <linux/sched/signal.h>
26174cd4b1SIngo Molnar
27ab7ac4ebSTom Herbert #include <net/kcm.h>
28ab7ac4ebSTom Herbert #include <net/netns/generic.h>
29ab7ac4ebSTom Herbert #include <net/sock.h>
30ab7ac4ebSTom Herbert #include <uapi/linux/kcm.h>
3140e0b090SPeilin Ye #include <trace/events/sock.h>
32ab7ac4ebSTom Herbert
33ab7ac4ebSTom Herbert unsigned int kcm_net_id;
34ab7ac4ebSTom Herbert
35ab7ac4ebSTom Herbert static struct kmem_cache *kcm_psockp __read_mostly;
36ab7ac4ebSTom Herbert static struct kmem_cache *kcm_muxp __read_mostly;
37ab7ac4ebSTom Herbert static struct workqueue_struct *kcm_wq;
38ab7ac4ebSTom Herbert
kcm_sk(const struct sock * sk)39ab7ac4ebSTom Herbert static inline struct kcm_sock *kcm_sk(const struct sock *sk)
40ab7ac4ebSTom Herbert {
41ab7ac4ebSTom Herbert return (struct kcm_sock *)sk;
42ab7ac4ebSTom Herbert }
43ab7ac4ebSTom Herbert
kcm_tx_msg(struct sk_buff * skb)44ab7ac4ebSTom Herbert static inline struct kcm_tx_msg *kcm_tx_msg(struct sk_buff *skb)
45ab7ac4ebSTom Herbert {
46ab7ac4ebSTom Herbert return (struct kcm_tx_msg *)skb->cb;
47ab7ac4ebSTom Herbert }
48ab7ac4ebSTom Herbert
report_csk_error(struct sock * csk,int err)49ab7ac4ebSTom Herbert static void report_csk_error(struct sock *csk, int err)
50ab7ac4ebSTom Herbert {
51ab7ac4ebSTom Herbert csk->sk_err = EPIPE;
52e3ae2365SAlexander Aring sk_error_report(csk);
53ab7ac4ebSTom Herbert }
54ab7ac4ebSTom Herbert
kcm_abort_tx_psock(struct kcm_psock * psock,int err,bool wakeup_kcm)55ab7ac4ebSTom Herbert static void kcm_abort_tx_psock(struct kcm_psock *psock, int err,
56ab7ac4ebSTom Herbert bool wakeup_kcm)
57ab7ac4ebSTom Herbert {
58ab7ac4ebSTom Herbert struct sock *csk = psock->sk;
59ab7ac4ebSTom Herbert struct kcm_mux *mux = psock->mux;
60ab7ac4ebSTom Herbert
61ab7ac4ebSTom Herbert /* Unrecoverable error in transmit */
62ab7ac4ebSTom Herbert
63ab7ac4ebSTom Herbert spin_lock_bh(&mux->lock);
64ab7ac4ebSTom Herbert
65ab7ac4ebSTom Herbert if (psock->tx_stopped) {
66ab7ac4ebSTom Herbert spin_unlock_bh(&mux->lock);
67ab7ac4ebSTom Herbert return;
68ab7ac4ebSTom Herbert }
69ab7ac4ebSTom Herbert
70ab7ac4ebSTom Herbert psock->tx_stopped = 1;
71cd6e111bSTom Herbert KCM_STATS_INCR(psock->stats.tx_aborts);
72ab7ac4ebSTom Herbert
73ab7ac4ebSTom Herbert if (!psock->tx_kcm) {
74ab7ac4ebSTom Herbert /* Take off psocks_avail list */
75ab7ac4ebSTom Herbert list_del(&psock->psock_avail_list);
76ab7ac4ebSTom Herbert } else if (wakeup_kcm) {
77ab7ac4ebSTom Herbert /* In this case psock is being aborted while outside of
78ab7ac4ebSTom Herbert * write_msgs and psock is reserved. Schedule tx_work
79ab7ac4ebSTom Herbert * to handle the failure there. Need to commit tx_stopped
80ab7ac4ebSTom Herbert * before queuing work.
81ab7ac4ebSTom Herbert */
82ab7ac4ebSTom Herbert smp_mb();
83ab7ac4ebSTom Herbert
84ab7ac4ebSTom Herbert queue_work(kcm_wq, &psock->tx_kcm->tx_work);
85ab7ac4ebSTom Herbert }
86ab7ac4ebSTom Herbert
87ab7ac4ebSTom Herbert spin_unlock_bh(&mux->lock);
88ab7ac4ebSTom Herbert
89ab7ac4ebSTom Herbert /* Report error on lower socket */
90ab7ac4ebSTom Herbert report_csk_error(csk, err);
91ab7ac4ebSTom Herbert }
92ab7ac4ebSTom Herbert
93cd6e111bSTom Herbert /* RX mux lock held. */
kcm_update_rx_mux_stats(struct kcm_mux * mux,struct kcm_psock * psock)94cd6e111bSTom Herbert static void kcm_update_rx_mux_stats(struct kcm_mux *mux,
95cd6e111bSTom Herbert struct kcm_psock *psock)
96cd6e111bSTom Herbert {
979b73896aSTom Herbert STRP_STATS_ADD(mux->stats.rx_bytes,
98bbb03029STom Herbert psock->strp.stats.bytes -
999b73896aSTom Herbert psock->saved_rx_bytes);
100cd6e111bSTom Herbert mux->stats.rx_msgs +=
101bbb03029STom Herbert psock->strp.stats.msgs - psock->saved_rx_msgs;
102bbb03029STom Herbert psock->saved_rx_msgs = psock->strp.stats.msgs;
103bbb03029STom Herbert psock->saved_rx_bytes = psock->strp.stats.bytes;
104cd6e111bSTom Herbert }
105cd6e111bSTom Herbert
kcm_update_tx_mux_stats(struct kcm_mux * mux,struct kcm_psock * psock)106cd6e111bSTom Herbert static void kcm_update_tx_mux_stats(struct kcm_mux *mux,
107cd6e111bSTom Herbert struct kcm_psock *psock)
108cd6e111bSTom Herbert {
109cd6e111bSTom Herbert KCM_STATS_ADD(mux->stats.tx_bytes,
110cd6e111bSTom Herbert psock->stats.tx_bytes - psock->saved_tx_bytes);
111cd6e111bSTom Herbert mux->stats.tx_msgs +=
112cd6e111bSTom Herbert psock->stats.tx_msgs - psock->saved_tx_msgs;
113cd6e111bSTom Herbert psock->saved_tx_msgs = psock->stats.tx_msgs;
114cd6e111bSTom Herbert psock->saved_tx_bytes = psock->stats.tx_bytes;
115cd6e111bSTom Herbert }
116cd6e111bSTom Herbert
117ab7ac4ebSTom Herbert static int kcm_queue_rcv_skb(struct sock *sk, struct sk_buff *skb);
118ab7ac4ebSTom Herbert
119ab7ac4ebSTom Herbert /* KCM is ready to receive messages on its queue-- either the KCM is new or
120ab7ac4ebSTom Herbert * has become unblocked after being blocked on full socket buffer. Queue any
121ab7ac4ebSTom Herbert * pending ready messages on a psock. RX mux lock held.
122ab7ac4ebSTom Herbert */
kcm_rcv_ready(struct kcm_sock * kcm)123ab7ac4ebSTom Herbert static void kcm_rcv_ready(struct kcm_sock *kcm)
124ab7ac4ebSTom Herbert {
125ab7ac4ebSTom Herbert struct kcm_mux *mux = kcm->mux;
126ab7ac4ebSTom Herbert struct kcm_psock *psock;
127ab7ac4ebSTom Herbert struct sk_buff *skb;
128ab7ac4ebSTom Herbert
129ab7ac4ebSTom Herbert if (unlikely(kcm->rx_wait || kcm->rx_psock || kcm->rx_disabled))
130ab7ac4ebSTom Herbert return;
131ab7ac4ebSTom Herbert
132ab7ac4ebSTom Herbert while (unlikely((skb = __skb_dequeue(&mux->rx_hold_queue)))) {
133ab7ac4ebSTom Herbert if (kcm_queue_rcv_skb(&kcm->sk, skb)) {
134ab7ac4ebSTom Herbert /* Assuming buffer limit has been reached */
135ab7ac4ebSTom Herbert skb_queue_head(&mux->rx_hold_queue, skb);
136ab7ac4ebSTom Herbert WARN_ON(!sk_rmem_alloc_get(&kcm->sk));
137ab7ac4ebSTom Herbert return;
138ab7ac4ebSTom Herbert }
139ab7ac4ebSTom Herbert }
140ab7ac4ebSTom Herbert
141ab7ac4ebSTom Herbert while (!list_empty(&mux->psocks_ready)) {
142ab7ac4ebSTom Herbert psock = list_first_entry(&mux->psocks_ready, struct kcm_psock,
143ab7ac4ebSTom Herbert psock_ready_list);
144ab7ac4ebSTom Herbert
145ab7ac4ebSTom Herbert if (kcm_queue_rcv_skb(&kcm->sk, psock->ready_rx_msg)) {
146ab7ac4ebSTom Herbert /* Assuming buffer limit has been reached */
147ab7ac4ebSTom Herbert WARN_ON(!sk_rmem_alloc_get(&kcm->sk));
148ab7ac4ebSTom Herbert return;
149ab7ac4ebSTom Herbert }
150ab7ac4ebSTom Herbert
151ab7ac4ebSTom Herbert /* Consumed the ready message on the psock. Schedule rx_work to
152ab7ac4ebSTom Herbert * get more messages.
153ab7ac4ebSTom Herbert */
154ab7ac4ebSTom Herbert list_del(&psock->psock_ready_list);
155ab7ac4ebSTom Herbert psock->ready_rx_msg = NULL;
156ab7ac4ebSTom Herbert /* Commit clearing of ready_rx_msg for queuing work */
157ab7ac4ebSTom Herbert smp_mb();
158ab7ac4ebSTom Herbert
1599b73896aSTom Herbert strp_unpause(&psock->strp);
1609b73896aSTom Herbert strp_check_rcv(&psock->strp);
161ab7ac4ebSTom Herbert }
162ab7ac4ebSTom Herbert
163ab7ac4ebSTom Herbert /* Buffer limit is okay now, add to ready list */
164ab7ac4ebSTom Herbert list_add_tail(&kcm->wait_rx_list,
165ab7ac4ebSTom Herbert &kcm->mux->kcm_rx_waiters);
1660c745b51SEric Dumazet /* paired with lockless reads in kcm_rfree() */
1670c745b51SEric Dumazet WRITE_ONCE(kcm->rx_wait, true);
168ab7ac4ebSTom Herbert }
169ab7ac4ebSTom Herbert
kcm_rfree(struct sk_buff * skb)170ab7ac4ebSTom Herbert static void kcm_rfree(struct sk_buff *skb)
171ab7ac4ebSTom Herbert {
172ab7ac4ebSTom Herbert struct sock *sk = skb->sk;
173ab7ac4ebSTom Herbert struct kcm_sock *kcm = kcm_sk(sk);
174ab7ac4ebSTom Herbert struct kcm_mux *mux = kcm->mux;
175ab7ac4ebSTom Herbert unsigned int len = skb->truesize;
176ab7ac4ebSTom Herbert
177ab7ac4ebSTom Herbert sk_mem_uncharge(sk, len);
178ab7ac4ebSTom Herbert atomic_sub(len, &sk->sk_rmem_alloc);
179ab7ac4ebSTom Herbert
180ab7ac4ebSTom Herbert /* For reading rx_wait and rx_psock without holding lock */
181ab7ac4ebSTom Herbert smp_mb__after_atomic();
182ab7ac4ebSTom Herbert
1830c745b51SEric Dumazet if (!READ_ONCE(kcm->rx_wait) && !READ_ONCE(kcm->rx_psock) &&
184ab7ac4ebSTom Herbert sk_rmem_alloc_get(sk) < sk->sk_rcvlowat) {
185ab7ac4ebSTom Herbert spin_lock_bh(&mux->rx_lock);
186ab7ac4ebSTom Herbert kcm_rcv_ready(kcm);
187ab7ac4ebSTom Herbert spin_unlock_bh(&mux->rx_lock);
188ab7ac4ebSTom Herbert }
189ab7ac4ebSTom Herbert }
190ab7ac4ebSTom Herbert
kcm_queue_rcv_skb(struct sock * sk,struct sk_buff * skb)191ab7ac4ebSTom Herbert static int kcm_queue_rcv_skb(struct sock *sk, struct sk_buff *skb)
192ab7ac4ebSTom Herbert {
193ab7ac4ebSTom Herbert struct sk_buff_head *list = &sk->sk_receive_queue;
194ab7ac4ebSTom Herbert
195ab7ac4ebSTom Herbert if (atomic_read(&sk->sk_rmem_alloc) >= sk->sk_rcvbuf)
196ab7ac4ebSTom Herbert return -ENOMEM;
197ab7ac4ebSTom Herbert
198ab7ac4ebSTom Herbert if (!sk_rmem_schedule(sk, skb, skb->truesize))
199ab7ac4ebSTom Herbert return -ENOBUFS;
200ab7ac4ebSTom Herbert
201ab7ac4ebSTom Herbert skb->dev = NULL;
202ab7ac4ebSTom Herbert
203ab7ac4ebSTom Herbert skb_orphan(skb);
204ab7ac4ebSTom Herbert skb->sk = sk;
205ab7ac4ebSTom Herbert skb->destructor = kcm_rfree;
206ab7ac4ebSTom Herbert atomic_add(skb->truesize, &sk->sk_rmem_alloc);
207ab7ac4ebSTom Herbert sk_mem_charge(sk, skb->truesize);
208ab7ac4ebSTom Herbert
209ab7ac4ebSTom Herbert skb_queue_tail(list, skb);
210ab7ac4ebSTom Herbert
211ab7ac4ebSTom Herbert if (!sock_flag(sk, SOCK_DEAD))
212ab7ac4ebSTom Herbert sk->sk_data_ready(sk);
213ab7ac4ebSTom Herbert
214ab7ac4ebSTom Herbert return 0;
215ab7ac4ebSTom Herbert }
216ab7ac4ebSTom Herbert
217ab7ac4ebSTom Herbert /* Requeue received messages for a kcm socket to other kcm sockets. This is
218ab7ac4ebSTom Herbert * called with a kcm socket is receive disabled.
219ab7ac4ebSTom Herbert * RX mux lock held.
220ab7ac4ebSTom Herbert */
requeue_rx_msgs(struct kcm_mux * mux,struct sk_buff_head * head)221ab7ac4ebSTom Herbert static void requeue_rx_msgs(struct kcm_mux *mux, struct sk_buff_head *head)
222ab7ac4ebSTom Herbert {
223ab7ac4ebSTom Herbert struct sk_buff *skb;
224ab7ac4ebSTom Herbert struct kcm_sock *kcm;
225ab7ac4ebSTom Herbert
2265121197eSCong Wang while ((skb = skb_dequeue(head))) {
227ab7ac4ebSTom Herbert /* Reset destructor to avoid calling kcm_rcv_ready */
228ab7ac4ebSTom Herbert skb->destructor = sock_rfree;
229ab7ac4ebSTom Herbert skb_orphan(skb);
230ab7ac4ebSTom Herbert try_again:
231ab7ac4ebSTom Herbert if (list_empty(&mux->kcm_rx_waiters)) {
232ab7ac4ebSTom Herbert skb_queue_tail(&mux->rx_hold_queue, skb);
233ab7ac4ebSTom Herbert continue;
234ab7ac4ebSTom Herbert }
235ab7ac4ebSTom Herbert
236ab7ac4ebSTom Herbert kcm = list_first_entry(&mux->kcm_rx_waiters,
237ab7ac4ebSTom Herbert struct kcm_sock, wait_rx_list);
238ab7ac4ebSTom Herbert
239ab7ac4ebSTom Herbert if (kcm_queue_rcv_skb(&kcm->sk, skb)) {
240ab7ac4ebSTom Herbert /* Should mean socket buffer full */
241ab7ac4ebSTom Herbert list_del(&kcm->wait_rx_list);
2420c745b51SEric Dumazet /* paired with lockless reads in kcm_rfree() */
2430c745b51SEric Dumazet WRITE_ONCE(kcm->rx_wait, false);
244ab7ac4ebSTom Herbert
245ab7ac4ebSTom Herbert /* Commit rx_wait to read in kcm_free */
246ab7ac4ebSTom Herbert smp_wmb();
247ab7ac4ebSTom Herbert
248ab7ac4ebSTom Herbert goto try_again;
249ab7ac4ebSTom Herbert }
250ab7ac4ebSTom Herbert }
251ab7ac4ebSTom Herbert }
252ab7ac4ebSTom Herbert
253ab7ac4ebSTom Herbert /* Lower sock lock held */
reserve_rx_kcm(struct kcm_psock * psock,struct sk_buff * head)254ab7ac4ebSTom Herbert static struct kcm_sock *reserve_rx_kcm(struct kcm_psock *psock,
255ab7ac4ebSTom Herbert struct sk_buff *head)
256ab7ac4ebSTom Herbert {
257ab7ac4ebSTom Herbert struct kcm_mux *mux = psock->mux;
258ab7ac4ebSTom Herbert struct kcm_sock *kcm;
259ab7ac4ebSTom Herbert
260ab7ac4ebSTom Herbert WARN_ON(psock->ready_rx_msg);
261ab7ac4ebSTom Herbert
262ab7ac4ebSTom Herbert if (psock->rx_kcm)
263ab7ac4ebSTom Herbert return psock->rx_kcm;
264ab7ac4ebSTom Herbert
265ab7ac4ebSTom Herbert spin_lock_bh(&mux->rx_lock);
266ab7ac4ebSTom Herbert
267ab7ac4ebSTom Herbert if (psock->rx_kcm) {
268ab7ac4ebSTom Herbert spin_unlock_bh(&mux->rx_lock);
269ab7ac4ebSTom Herbert return psock->rx_kcm;
270ab7ac4ebSTom Herbert }
271ab7ac4ebSTom Herbert
272cd6e111bSTom Herbert kcm_update_rx_mux_stats(mux, psock);
273cd6e111bSTom Herbert
274ab7ac4ebSTom Herbert if (list_empty(&mux->kcm_rx_waiters)) {
275ab7ac4ebSTom Herbert psock->ready_rx_msg = head;
2769b73896aSTom Herbert strp_pause(&psock->strp);
277ab7ac4ebSTom Herbert list_add_tail(&psock->psock_ready_list,
278ab7ac4ebSTom Herbert &mux->psocks_ready);
279ab7ac4ebSTom Herbert spin_unlock_bh(&mux->rx_lock);
280ab7ac4ebSTom Herbert return NULL;
281ab7ac4ebSTom Herbert }
282ab7ac4ebSTom Herbert
283ab7ac4ebSTom Herbert kcm = list_first_entry(&mux->kcm_rx_waiters,
284ab7ac4ebSTom Herbert struct kcm_sock, wait_rx_list);
285ab7ac4ebSTom Herbert list_del(&kcm->wait_rx_list);
2860c745b51SEric Dumazet /* paired with lockless reads in kcm_rfree() */
2870c745b51SEric Dumazet WRITE_ONCE(kcm->rx_wait, false);
288ab7ac4ebSTom Herbert
289ab7ac4ebSTom Herbert psock->rx_kcm = kcm;
29015e4dabdSEric Dumazet /* paired with lockless reads in kcm_rfree() */
29115e4dabdSEric Dumazet WRITE_ONCE(kcm->rx_psock, psock);
292ab7ac4ebSTom Herbert
293ab7ac4ebSTom Herbert spin_unlock_bh(&mux->rx_lock);
294ab7ac4ebSTom Herbert
295ab7ac4ebSTom Herbert return kcm;
296ab7ac4ebSTom Herbert }
297ab7ac4ebSTom Herbert
298ab7ac4ebSTom Herbert static void kcm_done(struct kcm_sock *kcm);
299ab7ac4ebSTom Herbert
kcm_done_work(struct work_struct * w)300ab7ac4ebSTom Herbert static void kcm_done_work(struct work_struct *w)
301ab7ac4ebSTom Herbert {
302ab7ac4ebSTom Herbert kcm_done(container_of(w, struct kcm_sock, done_work));
303ab7ac4ebSTom Herbert }
304ab7ac4ebSTom Herbert
305ab7ac4ebSTom Herbert /* Lower sock held */
unreserve_rx_kcm(struct kcm_psock * psock,bool rcv_ready)306ab7ac4ebSTom Herbert static void unreserve_rx_kcm(struct kcm_psock *psock,
307ab7ac4ebSTom Herbert bool rcv_ready)
308ab7ac4ebSTom Herbert {
309ab7ac4ebSTom Herbert struct kcm_sock *kcm = psock->rx_kcm;
310ab7ac4ebSTom Herbert struct kcm_mux *mux = psock->mux;
311ab7ac4ebSTom Herbert
312ab7ac4ebSTom Herbert if (!kcm)
313ab7ac4ebSTom Herbert return;
314ab7ac4ebSTom Herbert
315ab7ac4ebSTom Herbert spin_lock_bh(&mux->rx_lock);
316ab7ac4ebSTom Herbert
317ab7ac4ebSTom Herbert psock->rx_kcm = NULL;
31815e4dabdSEric Dumazet /* paired with lockless reads in kcm_rfree() */
31915e4dabdSEric Dumazet WRITE_ONCE(kcm->rx_psock, NULL);
320ab7ac4ebSTom Herbert
321ab7ac4ebSTom Herbert /* Commit kcm->rx_psock before sk_rmem_alloc_get to sync with
322ab7ac4ebSTom Herbert * kcm_rfree
323ab7ac4ebSTom Herbert */
324ab7ac4ebSTom Herbert smp_mb();
325ab7ac4ebSTom Herbert
326ab7ac4ebSTom Herbert if (unlikely(kcm->done)) {
327ab7ac4ebSTom Herbert spin_unlock_bh(&mux->rx_lock);
328ab7ac4ebSTom Herbert
329ab7ac4ebSTom Herbert /* Need to run kcm_done in a task since we need to qcquire
330ab7ac4ebSTom Herbert * callback locks which may already be held here.
331ab7ac4ebSTom Herbert */
332ab7ac4ebSTom Herbert INIT_WORK(&kcm->done_work, kcm_done_work);
333ab7ac4ebSTom Herbert schedule_work(&kcm->done_work);
334ab7ac4ebSTom Herbert return;
335ab7ac4ebSTom Herbert }
336ab7ac4ebSTom Herbert
337ab7ac4ebSTom Herbert if (unlikely(kcm->rx_disabled)) {
338ab7ac4ebSTom Herbert requeue_rx_msgs(mux, &kcm->sk.sk_receive_queue);
339ab7ac4ebSTom Herbert } else if (rcv_ready || unlikely(!sk_rmem_alloc_get(&kcm->sk))) {
340ab7ac4ebSTom Herbert /* Check for degenerative race with rx_wait that all
341ab7ac4ebSTom Herbert * data was dequeued (accounted for in kcm_rfree).
342ab7ac4ebSTom Herbert */
343ab7ac4ebSTom Herbert kcm_rcv_ready(kcm);
344ab7ac4ebSTom Herbert }
345ab7ac4ebSTom Herbert spin_unlock_bh(&mux->rx_lock);
346ab7ac4ebSTom Herbert }
347ab7ac4ebSTom Herbert
348ab7ac4ebSTom Herbert /* Lower sock lock held */
psock_data_ready(struct sock * sk)34996a59083STom Herbert static void psock_data_ready(struct sock *sk)
350ab7ac4ebSTom Herbert {
351ab7ac4ebSTom Herbert struct kcm_psock *psock;
352ab7ac4ebSTom Herbert
35340e0b090SPeilin Ye trace_sk_data_ready(sk);
35440e0b090SPeilin Ye
355ab7ac4ebSTom Herbert read_lock_bh(&sk->sk_callback_lock);
356ab7ac4ebSTom Herbert
357ab7ac4ebSTom Herbert psock = (struct kcm_psock *)sk->sk_user_data;
3589b73896aSTom Herbert if (likely(psock))
35996a59083STom Herbert strp_data_ready(&psock->strp);
360ab7ac4ebSTom Herbert
361ab7ac4ebSTom Herbert read_unlock_bh(&sk->sk_callback_lock);
362ab7ac4ebSTom Herbert }
363ab7ac4ebSTom Herbert
3649b73896aSTom Herbert /* Called with lower sock held */
kcm_rcv_strparser(struct strparser * strp,struct sk_buff * skb)3659b73896aSTom Herbert static void kcm_rcv_strparser(struct strparser *strp, struct sk_buff *skb)
366ab7ac4ebSTom Herbert {
3679b73896aSTom Herbert struct kcm_psock *psock = container_of(strp, struct kcm_psock, strp);
3689b73896aSTom Herbert struct kcm_sock *kcm;
369ab7ac4ebSTom Herbert
3709b73896aSTom Herbert try_queue:
3719b73896aSTom Herbert kcm = reserve_rx_kcm(psock, skb);
3729b73896aSTom Herbert if (!kcm) {
3739b73896aSTom Herbert /* Unable to reserve a KCM, message is held in psock and strp
3749b73896aSTom Herbert * is paused.
375ab7ac4ebSTom Herbert */
3769b73896aSTom Herbert return;
377ab7ac4ebSTom Herbert }
378ab7ac4ebSTom Herbert
3799b73896aSTom Herbert if (kcm_queue_rcv_skb(&kcm->sk, skb)) {
3809b73896aSTom Herbert /* Should mean socket buffer full */
3819b73896aSTom Herbert unreserve_rx_kcm(psock, false);
3829b73896aSTom Herbert goto try_queue;
3839b73896aSTom Herbert }
384ab7ac4ebSTom Herbert }
385ab7ac4ebSTom Herbert
kcm_parse_func_strparser(struct strparser * strp,struct sk_buff * skb)3869b73896aSTom Herbert static int kcm_parse_func_strparser(struct strparser *strp, struct sk_buff *skb)
387ab7ac4ebSTom Herbert {
3889b73896aSTom Herbert struct kcm_psock *psock = container_of(strp, struct kcm_psock, strp);
3899b73896aSTom Herbert struct bpf_prog *prog = psock->bpf_prog;
3900355d6c1SEric Dumazet int res;
3919b73896aSTom Herbert
3923d9f773cSDavid Miller res = bpf_prog_run_pin_on_cpu(prog, skb);
3930355d6c1SEric Dumazet return res;
3949b73896aSTom Herbert }
3959b73896aSTom Herbert
kcm_read_sock_done(struct strparser * strp,int err)3969b73896aSTom Herbert static int kcm_read_sock_done(struct strparser *strp, int err)
3979b73896aSTom Herbert {
3989b73896aSTom Herbert struct kcm_psock *psock = container_of(strp, struct kcm_psock, strp);
3999b73896aSTom Herbert
4009b73896aSTom Herbert unreserve_rx_kcm(psock, true);
4019b73896aSTom Herbert
4029b73896aSTom Herbert return err;
403ab7ac4ebSTom Herbert }
404ab7ac4ebSTom Herbert
psock_state_change(struct sock * sk)40596a59083STom Herbert static void psock_state_change(struct sock *sk)
406ab7ac4ebSTom Herbert {
407a9a08845SLinus Torvalds /* TCP only does a EPOLLIN for a half close. Do a EPOLLHUP here
408a9a08845SLinus Torvalds * since application will normally not poll with EPOLLIN
409ab7ac4ebSTom Herbert * on the TCP sockets.
410ab7ac4ebSTom Herbert */
411ab7ac4ebSTom Herbert
412ab7ac4ebSTom Herbert report_csk_error(sk, EPIPE);
413ab7ac4ebSTom Herbert }
414ab7ac4ebSTom Herbert
psock_write_space(struct sock * sk)41596a59083STom Herbert static void psock_write_space(struct sock *sk)
416ab7ac4ebSTom Herbert {
417ab7ac4ebSTom Herbert struct kcm_psock *psock;
418ab7ac4ebSTom Herbert struct kcm_mux *mux;
419ab7ac4ebSTom Herbert struct kcm_sock *kcm;
420ab7ac4ebSTom Herbert
421ab7ac4ebSTom Herbert read_lock_bh(&sk->sk_callback_lock);
422ab7ac4ebSTom Herbert
423ab7ac4ebSTom Herbert psock = (struct kcm_psock *)sk->sk_user_data;
424ab7ac4ebSTom Herbert if (unlikely(!psock))
425ab7ac4ebSTom Herbert goto out;
426ab7ac4ebSTom Herbert mux = psock->mux;
427ab7ac4ebSTom Herbert
428ab7ac4ebSTom Herbert spin_lock_bh(&mux->lock);
429ab7ac4ebSTom Herbert
430ab7ac4ebSTom Herbert /* Check if the socket is reserved so someone is waiting for sending. */
431ab7ac4ebSTom Herbert kcm = psock->tx_kcm;
4329b73896aSTom Herbert if (kcm && !unlikely(kcm->tx_stopped))
433ab7ac4ebSTom Herbert queue_work(kcm_wq, &kcm->tx_work);
434ab7ac4ebSTom Herbert
435ab7ac4ebSTom Herbert spin_unlock_bh(&mux->lock);
436ab7ac4ebSTom Herbert out:
437ab7ac4ebSTom Herbert read_unlock_bh(&sk->sk_callback_lock);
438ab7ac4ebSTom Herbert }
439ab7ac4ebSTom Herbert
440ab7ac4ebSTom Herbert static void unreserve_psock(struct kcm_sock *kcm);
441ab7ac4ebSTom Herbert
442ab7ac4ebSTom Herbert /* kcm sock is locked. */
reserve_psock(struct kcm_sock * kcm)443ab7ac4ebSTom Herbert static struct kcm_psock *reserve_psock(struct kcm_sock *kcm)
444ab7ac4ebSTom Herbert {
445ab7ac4ebSTom Herbert struct kcm_mux *mux = kcm->mux;
446ab7ac4ebSTom Herbert struct kcm_psock *psock;
447ab7ac4ebSTom Herbert
448ab7ac4ebSTom Herbert psock = kcm->tx_psock;
449ab7ac4ebSTom Herbert
450ab7ac4ebSTom Herbert smp_rmb(); /* Must read tx_psock before tx_wait */
451ab7ac4ebSTom Herbert
452ab7ac4ebSTom Herbert if (psock) {
453ab7ac4ebSTom Herbert WARN_ON(kcm->tx_wait);
454ab7ac4ebSTom Herbert if (unlikely(psock->tx_stopped))
455ab7ac4ebSTom Herbert unreserve_psock(kcm);
456ab7ac4ebSTom Herbert else
457ab7ac4ebSTom Herbert return kcm->tx_psock;
458ab7ac4ebSTom Herbert }
459ab7ac4ebSTom Herbert
460ab7ac4ebSTom Herbert spin_lock_bh(&mux->lock);
461ab7ac4ebSTom Herbert
462ab7ac4ebSTom Herbert /* Check again under lock to see if psock was reserved for this
463ab7ac4ebSTom Herbert * psock via psock_unreserve.
464ab7ac4ebSTom Herbert */
465ab7ac4ebSTom Herbert psock = kcm->tx_psock;
466ab7ac4ebSTom Herbert if (unlikely(psock)) {
467ab7ac4ebSTom Herbert WARN_ON(kcm->tx_wait);
468ab7ac4ebSTom Herbert spin_unlock_bh(&mux->lock);
469ab7ac4ebSTom Herbert return kcm->tx_psock;
470ab7ac4ebSTom Herbert }
471ab7ac4ebSTom Herbert
472ab7ac4ebSTom Herbert if (!list_empty(&mux->psocks_avail)) {
473ab7ac4ebSTom Herbert psock = list_first_entry(&mux->psocks_avail,
474ab7ac4ebSTom Herbert struct kcm_psock,
475ab7ac4ebSTom Herbert psock_avail_list);
476ab7ac4ebSTom Herbert list_del(&psock->psock_avail_list);
477ab7ac4ebSTom Herbert if (kcm->tx_wait) {
478ab7ac4ebSTom Herbert list_del(&kcm->wait_psock_list);
479ab7ac4ebSTom Herbert kcm->tx_wait = false;
480ab7ac4ebSTom Herbert }
481ab7ac4ebSTom Herbert kcm->tx_psock = psock;
482ab7ac4ebSTom Herbert psock->tx_kcm = kcm;
483cd6e111bSTom Herbert KCM_STATS_INCR(psock->stats.reserved);
484ab7ac4ebSTom Herbert } else if (!kcm->tx_wait) {
485ab7ac4ebSTom Herbert list_add_tail(&kcm->wait_psock_list,
486ab7ac4ebSTom Herbert &mux->kcm_tx_waiters);
487ab7ac4ebSTom Herbert kcm->tx_wait = true;
488ab7ac4ebSTom Herbert }
489ab7ac4ebSTom Herbert
490ab7ac4ebSTom Herbert spin_unlock_bh(&mux->lock);
491ab7ac4ebSTom Herbert
492ab7ac4ebSTom Herbert return psock;
493ab7ac4ebSTom Herbert }
494ab7ac4ebSTom Herbert
495ab7ac4ebSTom Herbert /* mux lock held */
psock_now_avail(struct kcm_psock * psock)496ab7ac4ebSTom Herbert static void psock_now_avail(struct kcm_psock *psock)
497ab7ac4ebSTom Herbert {
498ab7ac4ebSTom Herbert struct kcm_mux *mux = psock->mux;
499ab7ac4ebSTom Herbert struct kcm_sock *kcm;
500ab7ac4ebSTom Herbert
501ab7ac4ebSTom Herbert if (list_empty(&mux->kcm_tx_waiters)) {
502ab7ac4ebSTom Herbert list_add_tail(&psock->psock_avail_list,
503ab7ac4ebSTom Herbert &mux->psocks_avail);
504ab7ac4ebSTom Herbert } else {
505ab7ac4ebSTom Herbert kcm = list_first_entry(&mux->kcm_tx_waiters,
506ab7ac4ebSTom Herbert struct kcm_sock,
507ab7ac4ebSTom Herbert wait_psock_list);
508ab7ac4ebSTom Herbert list_del(&kcm->wait_psock_list);
509ab7ac4ebSTom Herbert kcm->tx_wait = false;
510ab7ac4ebSTom Herbert psock->tx_kcm = kcm;
511ab7ac4ebSTom Herbert
512ab7ac4ebSTom Herbert /* Commit before changing tx_psock since that is read in
513ab7ac4ebSTom Herbert * reserve_psock before queuing work.
514ab7ac4ebSTom Herbert */
515ab7ac4ebSTom Herbert smp_mb();
516ab7ac4ebSTom Herbert
517ab7ac4ebSTom Herbert kcm->tx_psock = psock;
518cd6e111bSTom Herbert KCM_STATS_INCR(psock->stats.reserved);
519ab7ac4ebSTom Herbert queue_work(kcm_wq, &kcm->tx_work);
520ab7ac4ebSTom Herbert }
521ab7ac4ebSTom Herbert }
522ab7ac4ebSTom Herbert
523ab7ac4ebSTom Herbert /* kcm sock is locked. */
unreserve_psock(struct kcm_sock * kcm)524ab7ac4ebSTom Herbert static void unreserve_psock(struct kcm_sock *kcm)
525ab7ac4ebSTom Herbert {
526ab7ac4ebSTom Herbert struct kcm_psock *psock;
527ab7ac4ebSTom Herbert struct kcm_mux *mux = kcm->mux;
528ab7ac4ebSTom Herbert
529ab7ac4ebSTom Herbert spin_lock_bh(&mux->lock);
530ab7ac4ebSTom Herbert
531ab7ac4ebSTom Herbert psock = kcm->tx_psock;
532ab7ac4ebSTom Herbert
533ab7ac4ebSTom Herbert if (WARN_ON(!psock)) {
534ab7ac4ebSTom Herbert spin_unlock_bh(&mux->lock);
535ab7ac4ebSTom Herbert return;
536ab7ac4ebSTom Herbert }
537ab7ac4ebSTom Herbert
538ab7ac4ebSTom Herbert smp_rmb(); /* Read tx_psock before tx_wait */
539ab7ac4ebSTom Herbert
540cd6e111bSTom Herbert kcm_update_tx_mux_stats(mux, psock);
541cd6e111bSTom Herbert
542ab7ac4ebSTom Herbert WARN_ON(kcm->tx_wait);
543ab7ac4ebSTom Herbert
544ab7ac4ebSTom Herbert kcm->tx_psock = NULL;
545ab7ac4ebSTom Herbert psock->tx_kcm = NULL;
546cd6e111bSTom Herbert KCM_STATS_INCR(psock->stats.unreserved);
547ab7ac4ebSTom Herbert
548ab7ac4ebSTom Herbert if (unlikely(psock->tx_stopped)) {
549ab7ac4ebSTom Herbert if (psock->done) {
550ab7ac4ebSTom Herbert /* Deferred free */
551ab7ac4ebSTom Herbert list_del(&psock->psock_list);
552ab7ac4ebSTom Herbert mux->psocks_cnt--;
553ab7ac4ebSTom Herbert sock_put(psock->sk);
554ab7ac4ebSTom Herbert fput(psock->sk->sk_socket->file);
555ab7ac4ebSTom Herbert kmem_cache_free(kcm_psockp, psock);
556ab7ac4ebSTom Herbert }
557ab7ac4ebSTom Herbert
558ab7ac4ebSTom Herbert /* Don't put back on available list */
559ab7ac4ebSTom Herbert
560ab7ac4ebSTom Herbert spin_unlock_bh(&mux->lock);
561ab7ac4ebSTom Herbert
562ab7ac4ebSTom Herbert return;
563ab7ac4ebSTom Herbert }
564ab7ac4ebSTom Herbert
565ab7ac4ebSTom Herbert psock_now_avail(psock);
566ab7ac4ebSTom Herbert
567ab7ac4ebSTom Herbert spin_unlock_bh(&mux->lock);
568ab7ac4ebSTom Herbert }
569ab7ac4ebSTom Herbert
kcm_report_tx_retry(struct kcm_sock * kcm)570cd6e111bSTom Herbert static void kcm_report_tx_retry(struct kcm_sock *kcm)
571cd6e111bSTom Herbert {
572cd6e111bSTom Herbert struct kcm_mux *mux = kcm->mux;
573cd6e111bSTom Herbert
574cd6e111bSTom Herbert spin_lock_bh(&mux->lock);
575cd6e111bSTom Herbert KCM_STATS_INCR(mux->stats.tx_retries);
576cd6e111bSTom Herbert spin_unlock_bh(&mux->lock);
577cd6e111bSTom Herbert }
578cd6e111bSTom Herbert
579ab7ac4ebSTom Herbert /* Write any messages ready on the kcm socket. Called with kcm sock lock
580ab7ac4ebSTom Herbert * held. Return bytes actually sent or error.
581ab7ac4ebSTom Herbert */
kcm_write_msgs(struct kcm_sock * kcm)582ab7ac4ebSTom Herbert static int kcm_write_msgs(struct kcm_sock *kcm)
583ab7ac4ebSTom Herbert {
584c31a25e1SDavid Howells unsigned int total_sent = 0;
585ab7ac4ebSTom Herbert struct sock *sk = &kcm->sk;
586ab7ac4ebSTom Herbert struct kcm_psock *psock;
587c31a25e1SDavid Howells struct sk_buff *head;
588ab7ac4ebSTom Herbert int ret = 0;
589ab7ac4ebSTom Herbert
590ab7ac4ebSTom Herbert kcm->tx_wait_more = false;
591ab7ac4ebSTom Herbert psock = kcm->tx_psock;
592ab7ac4ebSTom Herbert if (unlikely(psock && psock->tx_stopped)) {
593ab7ac4ebSTom Herbert /* A reserved psock was aborted asynchronously. Unreserve
594ab7ac4ebSTom Herbert * it and we'll retry the message.
595ab7ac4ebSTom Herbert */
596ab7ac4ebSTom Herbert unreserve_psock(kcm);
597cd6e111bSTom Herbert kcm_report_tx_retry(kcm);
598ab7ac4ebSTom Herbert if (skb_queue_empty(&sk->sk_write_queue))
599ab7ac4ebSTom Herbert return 0;
600ab7ac4ebSTom Herbert
601c31a25e1SDavid Howells kcm_tx_msg(skb_peek(&sk->sk_write_queue))->started_tx = false;
602ab7ac4ebSTom Herbert }
603ab7ac4ebSTom Herbert
604c31a25e1SDavid Howells retry:
605c31a25e1SDavid Howells while ((head = skb_peek(&sk->sk_write_queue))) {
606c31a25e1SDavid Howells struct msghdr msg = {
607c31a25e1SDavid Howells .msg_flags = MSG_DONTWAIT | MSG_SPLICE_PAGES,
608c31a25e1SDavid Howells };
609c31a25e1SDavid Howells struct kcm_tx_msg *txm = kcm_tx_msg(head);
610c31a25e1SDavid Howells struct sk_buff *skb;
611c31a25e1SDavid Howells unsigned int msize;
612c31a25e1SDavid Howells int i;
613ab7ac4ebSTom Herbert
614c31a25e1SDavid Howells if (!txm->started_tx) {
615c31a25e1SDavid Howells psock = reserve_psock(kcm);
616c31a25e1SDavid Howells if (!psock)
617c31a25e1SDavid Howells goto out;
618c31a25e1SDavid Howells skb = head;
619c31a25e1SDavid Howells txm->frag_offset = 0;
620c31a25e1SDavid Howells txm->sent = 0;
621c31a25e1SDavid Howells txm->started_tx = true;
622c31a25e1SDavid Howells } else {
623ab7ac4ebSTom Herbert if (WARN_ON(!psock)) {
624ab7ac4ebSTom Herbert ret = -EINVAL;
625ab7ac4ebSTom Herbert goto out;
626ab7ac4ebSTom Herbert }
627ab7ac4ebSTom Herbert skb = txm->frag_skb;
628ab7ac4ebSTom Herbert }
629ab7ac4ebSTom Herbert
630ab7ac4ebSTom Herbert if (WARN_ON(!skb_shinfo(skb)->nr_frags)) {
631ab7ac4ebSTom Herbert ret = -EINVAL;
632ab7ac4ebSTom Herbert goto out;
633ab7ac4ebSTom Herbert }
634ab7ac4ebSTom Herbert
635c31a25e1SDavid Howells msize = 0;
636c31a25e1SDavid Howells for (i = 0; i < skb_shinfo(skb)->nr_frags; i++)
63745d201c4SMina Almasry msize += skb_frag_size(&skb_shinfo(skb)->frags[i]);
638ab7ac4ebSTom Herbert
639c31a25e1SDavid Howells iov_iter_bvec(&msg.msg_iter, ITER_SOURCE,
640c31a25e1SDavid Howells skb_shinfo(skb)->frags, skb_shinfo(skb)->nr_frags,
641c31a25e1SDavid Howells msize);
642c31a25e1SDavid Howells iov_iter_advance(&msg.msg_iter, txm->frag_offset);
643ab7ac4ebSTom Herbert
644c31a25e1SDavid Howells do {
645264ba53fSDavid Howells ret = sock_sendmsg(psock->sk->sk_socket, &msg);
646ab7ac4ebSTom Herbert if (ret <= 0) {
647ab7ac4ebSTom Herbert if (ret == -EAGAIN) {
648ab7ac4ebSTom Herbert /* Save state to try again when there's
649ab7ac4ebSTom Herbert * write space on the socket
650ab7ac4ebSTom Herbert */
651ab7ac4ebSTom Herbert txm->frag_skb = skb;
652ab7ac4ebSTom Herbert ret = 0;
653ab7ac4ebSTom Herbert goto out;
654ab7ac4ebSTom Herbert }
655ab7ac4ebSTom Herbert
656ab7ac4ebSTom Herbert /* Hard failure in sending message, abort this
657ab7ac4ebSTom Herbert * psock since it has lost framing
65871a2fae5SBhaskar Chowdhury * synchronization and retry sending the
659ab7ac4ebSTom Herbert * message from the beginning.
660ab7ac4ebSTom Herbert */
661ab7ac4ebSTom Herbert kcm_abort_tx_psock(psock, ret ? -ret : EPIPE,
662ab7ac4ebSTom Herbert true);
663ab7ac4ebSTom Herbert unreserve_psock(kcm);
6649f8d0dc0SDavid Howells psock = NULL;
665ab7ac4ebSTom Herbert
666c31a25e1SDavid Howells txm->started_tx = false;
667cd6e111bSTom Herbert kcm_report_tx_retry(kcm);
668ab7ac4ebSTom Herbert ret = 0;
669c31a25e1SDavid Howells goto retry;
670ab7ac4ebSTom Herbert }
671ab7ac4ebSTom Herbert
672c31a25e1SDavid Howells txm->sent += ret;
673c31a25e1SDavid Howells txm->frag_offset += ret;
674cd6e111bSTom Herbert KCM_STATS_ADD(psock->stats.tx_bytes, ret);
675c31a25e1SDavid Howells } while (msg.msg_iter.count > 0);
676ab7ac4ebSTom Herbert
677ab7ac4ebSTom Herbert if (skb == head) {
678ab7ac4ebSTom Herbert if (skb_has_frag_list(skb)) {
679c31a25e1SDavid Howells txm->frag_skb = skb_shinfo(skb)->frag_list;
680c31a25e1SDavid Howells txm->frag_offset = 0;
681c31a25e1SDavid Howells continue;
682ab7ac4ebSTom Herbert }
683ab7ac4ebSTom Herbert } else if (skb->next) {
684c31a25e1SDavid Howells txm->frag_skb = skb->next;
685c31a25e1SDavid Howells txm->frag_offset = 0;
686c31a25e1SDavid Howells continue;
687ab7ac4ebSTom Herbert }
688ab7ac4ebSTom Herbert
689ab7ac4ebSTom Herbert /* Successfully sent the whole packet, account for it. */
690c31a25e1SDavid Howells sk->sk_wmem_queued -= txm->sent;
691c31a25e1SDavid Howells total_sent += txm->sent;
692ab7ac4ebSTom Herbert skb_dequeue(&sk->sk_write_queue);
693ab7ac4ebSTom Herbert kfree_skb(head);
694cd6e111bSTom Herbert KCM_STATS_INCR(psock->stats.tx_msgs);
695c31a25e1SDavid Howells }
696ab7ac4ebSTom Herbert out:
697ab7ac4ebSTom Herbert if (!head) {
698ab7ac4ebSTom Herbert /* Done with all queued messages. */
699ab7ac4ebSTom Herbert WARN_ON(!skb_queue_empty(&sk->sk_write_queue));
7009f8d0dc0SDavid Howells if (psock)
701ab7ac4ebSTom Herbert unreserve_psock(kcm);
702ab7ac4ebSTom Herbert }
703ab7ac4ebSTom Herbert
704ab7ac4ebSTom Herbert /* Check if write space is available */
705ab7ac4ebSTom Herbert sk->sk_write_space(sk);
706ab7ac4ebSTom Herbert
707ab7ac4ebSTom Herbert return total_sent ? : ret;
708ab7ac4ebSTom Herbert }
709ab7ac4ebSTom Herbert
kcm_tx_work(struct work_struct * w)710ab7ac4ebSTom Herbert static void kcm_tx_work(struct work_struct *w)
711ab7ac4ebSTom Herbert {
712ab7ac4ebSTom Herbert struct kcm_sock *kcm = container_of(w, struct kcm_sock, tx_work);
713ab7ac4ebSTom Herbert struct sock *sk = &kcm->sk;
714ab7ac4ebSTom Herbert int err;
715ab7ac4ebSTom Herbert
716ab7ac4ebSTom Herbert lock_sock(sk);
717ab7ac4ebSTom Herbert
718ab7ac4ebSTom Herbert /* Primarily for SOCK_DGRAM sockets, also handle asynchronous tx
719ab7ac4ebSTom Herbert * aborts
720ab7ac4ebSTom Herbert */
721ab7ac4ebSTom Herbert err = kcm_write_msgs(kcm);
722ab7ac4ebSTom Herbert if (err < 0) {
723ab7ac4ebSTom Herbert /* Hard failure in write, report error on KCM socket */
724ab7ac4ebSTom Herbert pr_warn("KCM: Hard failure on kcm_write_msgs %d\n", err);
725ab7ac4ebSTom Herbert report_csk_error(&kcm->sk, -err);
726ab7ac4ebSTom Herbert goto out;
727ab7ac4ebSTom Herbert }
728ab7ac4ebSTom Herbert
729ab7ac4ebSTom Herbert /* Primarily for SOCK_SEQPACKET sockets */
730ab7ac4ebSTom Herbert if (likely(sk->sk_socket) &&
731ab7ac4ebSTom Herbert test_bit(SOCK_NOSPACE, &sk->sk_socket->flags)) {
732ab7ac4ebSTom Herbert clear_bit(SOCK_NOSPACE, &sk->sk_socket->flags);
733ab7ac4ebSTom Herbert sk->sk_write_space(sk);
734ab7ac4ebSTom Herbert }
735ab7ac4ebSTom Herbert
736ab7ac4ebSTom Herbert out:
737ab7ac4ebSTom Herbert release_sock(sk);
738ab7ac4ebSTom Herbert }
739ab7ac4ebSTom Herbert
kcm_push(struct kcm_sock * kcm)740ab7ac4ebSTom Herbert static void kcm_push(struct kcm_sock *kcm)
741ab7ac4ebSTom Herbert {
742ab7ac4ebSTom Herbert if (kcm->tx_wait_more)
743ab7ac4ebSTom Herbert kcm_write_msgs(kcm);
744ab7ac4ebSTom Herbert }
745ab7ac4ebSTom Herbert
kcm_sendmsg(struct socket * sock,struct msghdr * msg,size_t len)746ab7ac4ebSTom Herbert static int kcm_sendmsg(struct socket *sock, struct msghdr *msg, size_t len)
747ab7ac4ebSTom Herbert {
748ab7ac4ebSTom Herbert struct sock *sk = sock->sk;
749ab7ac4ebSTom Herbert struct kcm_sock *kcm = kcm_sk(sk);
750ab7ac4ebSTom Herbert struct sk_buff *skb = NULL, *head = NULL;
751ab7ac4ebSTom Herbert size_t copy, copied = 0;
752ab7ac4ebSTom Herbert long timeo = sock_sndtimeo(sk, msg->msg_flags & MSG_DONTWAIT);
753ab7ac4ebSTom Herbert int eor = (sock->type == SOCK_DGRAM) ?
754ab7ac4ebSTom Herbert !(msg->msg_flags & MSG_MORE) : !!(msg->msg_flags & MSG_EOR);
755ab7ac4ebSTom Herbert int err = -EPIPE;
756ab7ac4ebSTom Herbert
757*00425508SKuniyuki Iwashima mutex_lock(&kcm->tx_mutex);
758ab7ac4ebSTom Herbert lock_sock(sk);
759ab7ac4ebSTom Herbert
760ab7ac4ebSTom Herbert /* Per tcp_sendmsg this should be in poll */
761ab7ac4ebSTom Herbert sk_clear_bit(SOCKWQ_ASYNC_NOSPACE, sk);
762ab7ac4ebSTom Herbert
763ab7ac4ebSTom Herbert if (sk->sk_err)
764ab7ac4ebSTom Herbert goto out_error;
765ab7ac4ebSTom Herbert
766ab7ac4ebSTom Herbert if (kcm->seq_skb) {
767ab7ac4ebSTom Herbert /* Previously opened message */
768ab7ac4ebSTom Herbert head = kcm->seq_skb;
769ab7ac4ebSTom Herbert skb = kcm_tx_msg(head)->last_skb;
770ab7ac4ebSTom Herbert goto start;
771ab7ac4ebSTom Herbert }
772ab7ac4ebSTom Herbert
773ab7ac4ebSTom Herbert /* Call the sk_stream functions to manage the sndbuf mem. */
774ab7ac4ebSTom Herbert if (!sk_stream_memory_free(sk)) {
775ab7ac4ebSTom Herbert kcm_push(kcm);
776ab7ac4ebSTom Herbert set_bit(SOCK_NOSPACE, &sk->sk_socket->flags);
777ab7ac4ebSTom Herbert err = sk_stream_wait_memory(sk, &timeo);
778ab7ac4ebSTom Herbert if (err)
779ab7ac4ebSTom Herbert goto out_error;
780ab7ac4ebSTom Herbert }
781ab7ac4ebSTom Herbert
78298e3862cSWANG Cong if (msg_data_left(msg)) {
783ab7ac4ebSTom Herbert /* New message, alloc head skb */
784ab7ac4ebSTom Herbert head = alloc_skb(0, sk->sk_allocation);
785ab7ac4ebSTom Herbert while (!head) {
786ab7ac4ebSTom Herbert kcm_push(kcm);
787ab7ac4ebSTom Herbert err = sk_stream_wait_memory(sk, &timeo);
788ab7ac4ebSTom Herbert if (err)
789ab7ac4ebSTom Herbert goto out_error;
790ab7ac4ebSTom Herbert
791ab7ac4ebSTom Herbert head = alloc_skb(0, sk->sk_allocation);
792ab7ac4ebSTom Herbert }
793ab7ac4ebSTom Herbert
794ab7ac4ebSTom Herbert skb = head;
795ab7ac4ebSTom Herbert
796ab7ac4ebSTom Herbert /* Set ip_summed to CHECKSUM_UNNECESSARY to avoid calling
797ab7ac4ebSTom Herbert * csum_and_copy_from_iter from skb_do_copy_data_nocache.
798ab7ac4ebSTom Herbert */
799ab7ac4ebSTom Herbert skb->ip_summed = CHECKSUM_UNNECESSARY;
80098e3862cSWANG Cong }
801ab7ac4ebSTom Herbert
802ab7ac4ebSTom Herbert start:
803ab7ac4ebSTom Herbert while (msg_data_left(msg)) {
804ab7ac4ebSTom Herbert bool merge = true;
805ab7ac4ebSTom Herbert int i = skb_shinfo(skb)->nr_frags;
806ab7ac4ebSTom Herbert struct page_frag *pfrag = sk_page_frag(sk);
807ab7ac4ebSTom Herbert
808ab7ac4ebSTom Herbert if (!sk_page_frag_refill(sk, pfrag))
809ab7ac4ebSTom Herbert goto wait_for_memory;
810ab7ac4ebSTom Herbert
811ab7ac4ebSTom Herbert if (!skb_can_coalesce(skb, i, pfrag->page,
812ab7ac4ebSTom Herbert pfrag->offset)) {
813ab7ac4ebSTom Herbert if (i == MAX_SKB_FRAGS) {
814ab7ac4ebSTom Herbert struct sk_buff *tskb;
815ab7ac4ebSTom Herbert
816ab7ac4ebSTom Herbert tskb = alloc_skb(0, sk->sk_allocation);
817ab7ac4ebSTom Herbert if (!tskb)
818ab7ac4ebSTom Herbert goto wait_for_memory;
819ab7ac4ebSTom Herbert
820ab7ac4ebSTom Herbert if (head == skb)
821ab7ac4ebSTom Herbert skb_shinfo(head)->frag_list = tskb;
822ab7ac4ebSTom Herbert else
823ab7ac4ebSTom Herbert skb->next = tskb;
824ab7ac4ebSTom Herbert
825ab7ac4ebSTom Herbert skb = tskb;
826ab7ac4ebSTom Herbert skb->ip_summed = CHECKSUM_UNNECESSARY;
827ab7ac4ebSTom Herbert continue;
828ab7ac4ebSTom Herbert }
829ab7ac4ebSTom Herbert merge = false;
830ab7ac4ebSTom Herbert }
831ab7ac4ebSTom Herbert
8322b03bcaeSDavid Howells if (msg->msg_flags & MSG_SPLICE_PAGES) {
8332b03bcaeSDavid Howells copy = msg_data_left(msg);
8342b03bcaeSDavid Howells if (!sk_wmem_schedule(sk, copy))
8352b03bcaeSDavid Howells goto wait_for_memory;
8362b03bcaeSDavid Howells
8372b03bcaeSDavid Howells err = skb_splice_from_iter(skb, &msg->msg_iter, copy,
8382b03bcaeSDavid Howells sk->sk_allocation);
8392b03bcaeSDavid Howells if (err < 0) {
8402b03bcaeSDavid Howells if (err == -EMSGSIZE)
8412b03bcaeSDavid Howells goto wait_for_memory;
8422b03bcaeSDavid Howells goto out_error;
8432b03bcaeSDavid Howells }
8442b03bcaeSDavid Howells
8452b03bcaeSDavid Howells copy = err;
8462b03bcaeSDavid Howells skb_shinfo(skb)->flags |= SKBFL_SHARED_FRAG;
8472b03bcaeSDavid Howells sk_wmem_queued_add(sk, copy);
8482b03bcaeSDavid Howells sk_mem_charge(sk, copy);
8492b03bcaeSDavid Howells
8502b03bcaeSDavid Howells if (head != skb)
8512b03bcaeSDavid Howells head->truesize += copy;
8522b03bcaeSDavid Howells } else {
853ab7ac4ebSTom Herbert copy = min_t(int, msg_data_left(msg),
854ab7ac4ebSTom Herbert pfrag->size - pfrag->offset);
855ab7ac4ebSTom Herbert if (!sk_wmem_schedule(sk, copy))
856ab7ac4ebSTom Herbert goto wait_for_memory;
857ab7ac4ebSTom Herbert
858ab7ac4ebSTom Herbert err = skb_copy_to_page_nocache(sk, &msg->msg_iter, skb,
859ab7ac4ebSTom Herbert pfrag->page,
860ab7ac4ebSTom Herbert pfrag->offset,
861ab7ac4ebSTom Herbert copy);
862ab7ac4ebSTom Herbert if (err)
863ab7ac4ebSTom Herbert goto out_error;
864ab7ac4ebSTom Herbert
865ab7ac4ebSTom Herbert /* Update the skb. */
866ab7ac4ebSTom Herbert if (merge) {
8672b03bcaeSDavid Howells skb_frag_size_add(
8682b03bcaeSDavid Howells &skb_shinfo(skb)->frags[i - 1], copy);
869ab7ac4ebSTom Herbert } else {
870ab7ac4ebSTom Herbert skb_fill_page_desc(skb, i, pfrag->page,
871ab7ac4ebSTom Herbert pfrag->offset, copy);
872ab7ac4ebSTom Herbert get_page(pfrag->page);
873ab7ac4ebSTom Herbert }
874ab7ac4ebSTom Herbert
875ab7ac4ebSTom Herbert pfrag->offset += copy;
8762b03bcaeSDavid Howells }
8772b03bcaeSDavid Howells
878ab7ac4ebSTom Herbert copied += copy;
879ab7ac4ebSTom Herbert if (head != skb) {
880ab7ac4ebSTom Herbert head->len += copy;
881ab7ac4ebSTom Herbert head->data_len += copy;
882ab7ac4ebSTom Herbert }
883ab7ac4ebSTom Herbert
884ab7ac4ebSTom Herbert continue;
885ab7ac4ebSTom Herbert
886ab7ac4ebSTom Herbert wait_for_memory:
887ab7ac4ebSTom Herbert kcm_push(kcm);
888ab7ac4ebSTom Herbert err = sk_stream_wait_memory(sk, &timeo);
889ab7ac4ebSTom Herbert if (err)
890ab7ac4ebSTom Herbert goto out_error;
891ab7ac4ebSTom Herbert }
892ab7ac4ebSTom Herbert
893ab7ac4ebSTom Herbert if (eor) {
894ab7ac4ebSTom Herbert bool not_busy = skb_queue_empty(&sk->sk_write_queue);
895ab7ac4ebSTom Herbert
89698e3862cSWANG Cong if (head) {
897ab7ac4ebSTom Herbert /* Message complete, queue it on send buffer */
898ab7ac4ebSTom Herbert __skb_queue_tail(&sk->sk_write_queue, head);
899ab7ac4ebSTom Herbert kcm->seq_skb = NULL;
900cd6e111bSTom Herbert KCM_STATS_INCR(kcm->stats.tx_msgs);
90198e3862cSWANG Cong }
902ab7ac4ebSTom Herbert
903ab7ac4ebSTom Herbert if (msg->msg_flags & MSG_BATCH) {
904ab7ac4ebSTom Herbert kcm->tx_wait_more = true;
905ab7ac4ebSTom Herbert } else if (kcm->tx_wait_more || not_busy) {
906ab7ac4ebSTom Herbert err = kcm_write_msgs(kcm);
907ab7ac4ebSTom Herbert if (err < 0) {
908ab7ac4ebSTom Herbert /* We got a hard error in write_msgs but have
909ab7ac4ebSTom Herbert * already queued this message. Report an error
910ab7ac4ebSTom Herbert * in the socket, but don't affect return value
911ab7ac4ebSTom Herbert * from sendmsg
912ab7ac4ebSTom Herbert */
913ab7ac4ebSTom Herbert pr_warn("KCM: Hard failure on kcm_write_msgs\n");
914ab7ac4ebSTom Herbert report_csk_error(&kcm->sk, -err);
915ab7ac4ebSTom Herbert }
916ab7ac4ebSTom Herbert }
917ab7ac4ebSTom Herbert } else {
918ab7ac4ebSTom Herbert /* Message not complete, save state */
919ab7ac4ebSTom Herbert partial_message:
920cd27b96bSWANG Cong if (head) {
921ab7ac4ebSTom Herbert kcm->seq_skb = head;
922ab7ac4ebSTom Herbert kcm_tx_msg(head)->last_skb = skb;
923ab7ac4ebSTom Herbert }
924cd27b96bSWANG Cong }
925ab7ac4ebSTom Herbert
926cd6e111bSTom Herbert KCM_STATS_ADD(kcm->stats.tx_bytes, copied);
927cd6e111bSTom Herbert
928ab7ac4ebSTom Herbert release_sock(sk);
929*00425508SKuniyuki Iwashima mutex_unlock(&kcm->tx_mutex);
930ab7ac4ebSTom Herbert return copied;
931ab7ac4ebSTom Herbert
932ab7ac4ebSTom Herbert out_error:
933ab7ac4ebSTom Herbert kcm_push(kcm);
934ab7ac4ebSTom Herbert
935a22730b1SKuniyuki Iwashima if (sock->type == SOCK_SEQPACKET) {
936ab7ac4ebSTom Herbert /* Wrote some bytes before encountering an
937ab7ac4ebSTom Herbert * error, return partial success.
938ab7ac4ebSTom Herbert */
939a22730b1SKuniyuki Iwashima if (copied)
940ab7ac4ebSTom Herbert goto partial_message;
941ab7ac4ebSTom Herbert if (head != kcm->seq_skb)
942ab7ac4ebSTom Herbert kfree_skb(head);
943a22730b1SKuniyuki Iwashima } else {
944a22730b1SKuniyuki Iwashima kfree_skb(head);
945a22730b1SKuniyuki Iwashima kcm->seq_skb = NULL;
946a22730b1SKuniyuki Iwashima }
947ab7ac4ebSTom Herbert
948ab7ac4ebSTom Herbert err = sk_stream_error(sk, msg->msg_flags, err);
949ab7ac4ebSTom Herbert
950ab7ac4ebSTom Herbert /* make sure we wake any epoll edge trigger waiter */
951ab7ac4ebSTom Herbert if (unlikely(skb_queue_len(&sk->sk_write_queue) == 0 && err == -EAGAIN))
952ab7ac4ebSTom Herbert sk->sk_write_space(sk);
953ab7ac4ebSTom Herbert
954ab7ac4ebSTom Herbert release_sock(sk);
955*00425508SKuniyuki Iwashima mutex_unlock(&kcm->tx_mutex);
956ab7ac4ebSTom Herbert return err;
957ab7ac4ebSTom Herbert }
958ab7ac4ebSTom Herbert
kcm_splice_eof(struct socket * sock)959951ace99SDavid Howells static void kcm_splice_eof(struct socket *sock)
960951ace99SDavid Howells {
961951ace99SDavid Howells struct sock *sk = sock->sk;
962951ace99SDavid Howells struct kcm_sock *kcm = kcm_sk(sk);
963951ace99SDavid Howells
964951ace99SDavid Howells if (skb_queue_empty_lockless(&sk->sk_write_queue))
965951ace99SDavid Howells return;
966951ace99SDavid Howells
967951ace99SDavid Howells lock_sock(sk);
968951ace99SDavid Howells kcm_write_msgs(kcm);
969951ace99SDavid Howells release_sock(sk);
970951ace99SDavid Howells }
971951ace99SDavid Howells
kcm_recvmsg(struct socket * sock,struct msghdr * msg,size_t len,int flags)972ab7ac4ebSTom Herbert static int kcm_recvmsg(struct socket *sock, struct msghdr *msg,
973ab7ac4ebSTom Herbert size_t len, int flags)
974ab7ac4ebSTom Herbert {
975ab7ac4ebSTom Herbert struct sock *sk = sock->sk;
976cd6e111bSTom Herbert struct kcm_sock *kcm = kcm_sk(sk);
977ab7ac4ebSTom Herbert int err = 0;
978bbb03029STom Herbert struct strp_msg *stm;
979ab7ac4ebSTom Herbert int copied = 0;
980ab7ac4ebSTom Herbert struct sk_buff *skb;
981ab7ac4ebSTom Herbert
9825121197eSCong Wang skb = skb_recv_datagram(sk, flags, &err);
983ab7ac4ebSTom Herbert if (!skb)
984ab7ac4ebSTom Herbert goto out;
985ab7ac4ebSTom Herbert
986ab7ac4ebSTom Herbert /* Okay, have a message on the receive queue */
987ab7ac4ebSTom Herbert
988bbb03029STom Herbert stm = strp_msg(skb);
989ab7ac4ebSTom Herbert
990bbb03029STom Herbert if (len > stm->full_len)
991bbb03029STom Herbert len = stm->full_len;
992ab7ac4ebSTom Herbert
993bbb03029STom Herbert err = skb_copy_datagram_msg(skb, stm->offset, msg, len);
994ab7ac4ebSTom Herbert if (err < 0)
995ab7ac4ebSTom Herbert goto out;
996ab7ac4ebSTom Herbert
997ab7ac4ebSTom Herbert copied = len;
998ab7ac4ebSTom Herbert if (likely(!(flags & MSG_PEEK))) {
999cd6e111bSTom Herbert KCM_STATS_ADD(kcm->stats.rx_bytes, copied);
1000bbb03029STom Herbert if (copied < stm->full_len) {
1001ab7ac4ebSTom Herbert if (sock->type == SOCK_DGRAM) {
1002ab7ac4ebSTom Herbert /* Truncated message */
1003ab7ac4ebSTom Herbert msg->msg_flags |= MSG_TRUNC;
1004ab7ac4ebSTom Herbert goto msg_finished;
1005ab7ac4ebSTom Herbert }
1006bbb03029STom Herbert stm->offset += copied;
1007bbb03029STom Herbert stm->full_len -= copied;
1008ab7ac4ebSTom Herbert } else {
1009ab7ac4ebSTom Herbert msg_finished:
1010ab7ac4ebSTom Herbert /* Finished with message */
1011ab7ac4ebSTom Herbert msg->msg_flags |= MSG_EOR;
1012cd6e111bSTom Herbert KCM_STATS_INCR(kcm->stats.rx_msgs);
1013ab7ac4ebSTom Herbert }
1014ab7ac4ebSTom Herbert }
1015ab7ac4ebSTom Herbert
1016ab7ac4ebSTom Herbert out:
10175121197eSCong Wang skb_free_datagram(sk, skb);
1018ab7ac4ebSTom Herbert return copied ? : err;
1019ab7ac4ebSTom Herbert }
1020ab7ac4ebSTom Herbert
kcm_splice_read(struct socket * sock,loff_t * ppos,struct pipe_inode_info * pipe,size_t len,unsigned int flags)102191687355STom Herbert static ssize_t kcm_splice_read(struct socket *sock, loff_t *ppos,
102291687355STom Herbert struct pipe_inode_info *pipe, size_t len,
102391687355STom Herbert unsigned int flags)
102491687355STom Herbert {
102591687355STom Herbert struct sock *sk = sock->sk;
102691687355STom Herbert struct kcm_sock *kcm = kcm_sk(sk);
1027bbb03029STom Herbert struct strp_msg *stm;
102891687355STom Herbert int err = 0;
1029f1971a2eSWANG Cong ssize_t copied;
103091687355STom Herbert struct sk_buff *skb;
103191687355STom Herbert
103291687355STom Herbert /* Only support splice for SOCKSEQPACKET */
103391687355STom Herbert
10345121197eSCong Wang skb = skb_recv_datagram(sk, flags, &err);
103591687355STom Herbert if (!skb)
103691687355STom Herbert goto err_out;
103791687355STom Herbert
103891687355STom Herbert /* Okay, have a message on the receive queue */
103991687355STom Herbert
1040bbb03029STom Herbert stm = strp_msg(skb);
104191687355STom Herbert
1042bbb03029STom Herbert if (len > stm->full_len)
1043bbb03029STom Herbert len = stm->full_len;
104491687355STom Herbert
1045bbb03029STom Herbert copied = skb_splice_bits(skb, sk, stm->offset, pipe, len, flags);
104691687355STom Herbert if (copied < 0) {
104791687355STom Herbert err = copied;
104891687355STom Herbert goto err_out;
104991687355STom Herbert }
105091687355STom Herbert
105191687355STom Herbert KCM_STATS_ADD(kcm->stats.rx_bytes, copied);
105291687355STom Herbert
1053bbb03029STom Herbert stm->offset += copied;
1054bbb03029STom Herbert stm->full_len -= copied;
105591687355STom Herbert
105691687355STom Herbert /* We have no way to return MSG_EOR. If all the bytes have been
105791687355STom Herbert * read we still leave the message in the receive socket buffer.
105891687355STom Herbert * A subsequent recvmsg needs to be done to return MSG_EOR and
105991687355STom Herbert * finish reading the message.
106091687355STom Herbert */
106191687355STom Herbert
10625121197eSCong Wang skb_free_datagram(sk, skb);
106391687355STom Herbert return copied;
106491687355STom Herbert
106591687355STom Herbert err_out:
10665121197eSCong Wang skb_free_datagram(sk, skb);
106791687355STom Herbert return err;
106891687355STom Herbert }
106991687355STom Herbert
1070ab7ac4ebSTom Herbert /* kcm sock lock held */
kcm_recv_disable(struct kcm_sock * kcm)1071ab7ac4ebSTom Herbert static void kcm_recv_disable(struct kcm_sock *kcm)
1072ab7ac4ebSTom Herbert {
1073ab7ac4ebSTom Herbert struct kcm_mux *mux = kcm->mux;
1074ab7ac4ebSTom Herbert
1075ab7ac4ebSTom Herbert if (kcm->rx_disabled)
1076ab7ac4ebSTom Herbert return;
1077ab7ac4ebSTom Herbert
1078ab7ac4ebSTom Herbert spin_lock_bh(&mux->rx_lock);
1079ab7ac4ebSTom Herbert
1080ab7ac4ebSTom Herbert kcm->rx_disabled = 1;
1081ab7ac4ebSTom Herbert
1082ab7ac4ebSTom Herbert /* If a psock is reserved we'll do cleanup in unreserve */
1083ab7ac4ebSTom Herbert if (!kcm->rx_psock) {
1084ab7ac4ebSTom Herbert if (kcm->rx_wait) {
1085ab7ac4ebSTom Herbert list_del(&kcm->wait_rx_list);
10860c745b51SEric Dumazet /* paired with lockless reads in kcm_rfree() */
10870c745b51SEric Dumazet WRITE_ONCE(kcm->rx_wait, false);
1088ab7ac4ebSTom Herbert }
1089ab7ac4ebSTom Herbert
1090ab7ac4ebSTom Herbert requeue_rx_msgs(mux, &kcm->sk.sk_receive_queue);
1091ab7ac4ebSTom Herbert }
1092ab7ac4ebSTom Herbert
1093ab7ac4ebSTom Herbert spin_unlock_bh(&mux->rx_lock);
1094ab7ac4ebSTom Herbert }
1095ab7ac4ebSTom Herbert
1096ab7ac4ebSTom Herbert /* kcm sock lock held */
kcm_recv_enable(struct kcm_sock * kcm)1097ab7ac4ebSTom Herbert static void kcm_recv_enable(struct kcm_sock *kcm)
1098ab7ac4ebSTom Herbert {
1099ab7ac4ebSTom Herbert struct kcm_mux *mux = kcm->mux;
1100ab7ac4ebSTom Herbert
1101ab7ac4ebSTom Herbert if (!kcm->rx_disabled)
1102ab7ac4ebSTom Herbert return;
1103ab7ac4ebSTom Herbert
1104ab7ac4ebSTom Herbert spin_lock_bh(&mux->rx_lock);
1105ab7ac4ebSTom Herbert
1106ab7ac4ebSTom Herbert kcm->rx_disabled = 0;
1107ab7ac4ebSTom Herbert kcm_rcv_ready(kcm);
1108ab7ac4ebSTom Herbert
1109ab7ac4ebSTom Herbert spin_unlock_bh(&mux->rx_lock);
1110ab7ac4ebSTom Herbert }
1111ab7ac4ebSTom Herbert
kcm_setsockopt(struct socket * sock,int level,int optname,sockptr_t optval,unsigned int optlen)1112ab7ac4ebSTom Herbert static int kcm_setsockopt(struct socket *sock, int level, int optname,
1113a7b75c5aSChristoph Hellwig sockptr_t optval, unsigned int optlen)
1114ab7ac4ebSTom Herbert {
1115ab7ac4ebSTom Herbert struct kcm_sock *kcm = kcm_sk(sock->sk);
1116ab7ac4ebSTom Herbert int val, valbool;
1117ab7ac4ebSTom Herbert int err = 0;
1118ab7ac4ebSTom Herbert
1119ab7ac4ebSTom Herbert if (level != SOL_KCM)
1120ab7ac4ebSTom Herbert return -ENOPROTOOPT;
1121ab7ac4ebSTom Herbert
1122ab7ac4ebSTom Herbert if (optlen < sizeof(int))
1123ab7ac4ebSTom Herbert return -EINVAL;
1124ab7ac4ebSTom Herbert
1125a7b75c5aSChristoph Hellwig if (copy_from_sockptr(&val, optval, sizeof(int)))
1126a7b75c5aSChristoph Hellwig return -EFAULT;
1127ab7ac4ebSTom Herbert
1128ab7ac4ebSTom Herbert valbool = val ? 1 : 0;
1129ab7ac4ebSTom Herbert
1130ab7ac4ebSTom Herbert switch (optname) {
1131ab7ac4ebSTom Herbert case KCM_RECV_DISABLE:
1132ab7ac4ebSTom Herbert lock_sock(&kcm->sk);
1133ab7ac4ebSTom Herbert if (valbool)
1134ab7ac4ebSTom Herbert kcm_recv_disable(kcm);
1135ab7ac4ebSTom Herbert else
1136ab7ac4ebSTom Herbert kcm_recv_enable(kcm);
1137ab7ac4ebSTom Herbert release_sock(&kcm->sk);
1138ab7ac4ebSTom Herbert break;
1139ab7ac4ebSTom Herbert default:
1140ab7ac4ebSTom Herbert err = -ENOPROTOOPT;
1141ab7ac4ebSTom Herbert }
1142ab7ac4ebSTom Herbert
1143ab7ac4ebSTom Herbert return err;
1144ab7ac4ebSTom Herbert }
1145ab7ac4ebSTom Herbert
kcm_getsockopt(struct socket * sock,int level,int optname,char __user * optval,int __user * optlen)1146ab7ac4ebSTom Herbert static int kcm_getsockopt(struct socket *sock, int level, int optname,
1147ab7ac4ebSTom Herbert char __user *optval, int __user *optlen)
1148ab7ac4ebSTom Herbert {
1149ab7ac4ebSTom Herbert struct kcm_sock *kcm = kcm_sk(sock->sk);
1150ab7ac4ebSTom Herbert int val, len;
1151ab7ac4ebSTom Herbert
1152ab7ac4ebSTom Herbert if (level != SOL_KCM)
1153ab7ac4ebSTom Herbert return -ENOPROTOOPT;
1154ab7ac4ebSTom Herbert
1155ab7ac4ebSTom Herbert if (get_user(len, optlen))
1156ab7ac4ebSTom Herbert return -EFAULT;
1157ab7ac4ebSTom Herbert
1158ab7ac4ebSTom Herbert if (len < 0)
1159ab7ac4ebSTom Herbert return -EINVAL;
1160ab7ac4ebSTom Herbert
1161cddf9fc7SGavrilov Ilia len = min_t(unsigned int, len, sizeof(int));
1162cddf9fc7SGavrilov Ilia
1163ab7ac4ebSTom Herbert switch (optname) {
1164ab7ac4ebSTom Herbert case KCM_RECV_DISABLE:
1165ab7ac4ebSTom Herbert val = kcm->rx_disabled;
1166ab7ac4ebSTom Herbert break;
1167ab7ac4ebSTom Herbert default:
1168ab7ac4ebSTom Herbert return -ENOPROTOOPT;
1169ab7ac4ebSTom Herbert }
1170ab7ac4ebSTom Herbert
1171ab7ac4ebSTom Herbert if (put_user(len, optlen))
1172ab7ac4ebSTom Herbert return -EFAULT;
1173ab7ac4ebSTom Herbert if (copy_to_user(optval, &val, len))
1174ab7ac4ebSTom Herbert return -EFAULT;
1175ab7ac4ebSTom Herbert return 0;
1176ab7ac4ebSTom Herbert }
1177ab7ac4ebSTom Herbert
init_kcm_sock(struct kcm_sock * kcm,struct kcm_mux * mux)1178ab7ac4ebSTom Herbert static void init_kcm_sock(struct kcm_sock *kcm, struct kcm_mux *mux)
1179ab7ac4ebSTom Herbert {
1180ab7ac4ebSTom Herbert struct kcm_sock *tkcm;
1181ab7ac4ebSTom Herbert struct list_head *head;
1182ab7ac4ebSTom Herbert int index = 0;
1183ab7ac4ebSTom Herbert
1184a11e1d43SLinus Torvalds /* For SOCK_SEQPACKET sock type, datagram_poll checks the sk_state, so
1185a11e1d43SLinus Torvalds * we set sk_state, otherwise epoll_wait always returns right away with
1186a11e1d43SLinus Torvalds * EPOLLHUP
1187ab7ac4ebSTom Herbert */
1188ab7ac4ebSTom Herbert kcm->sk.sk_state = TCP_ESTABLISHED;
1189ab7ac4ebSTom Herbert
1190ab7ac4ebSTom Herbert /* Add to mux's kcm sockets list */
1191ab7ac4ebSTom Herbert kcm->mux = mux;
1192ab7ac4ebSTom Herbert spin_lock_bh(&mux->lock);
1193ab7ac4ebSTom Herbert
1194ab7ac4ebSTom Herbert head = &mux->kcm_socks;
1195ab7ac4ebSTom Herbert list_for_each_entry(tkcm, &mux->kcm_socks, kcm_sock_list) {
1196ab7ac4ebSTom Herbert if (tkcm->index != index)
1197ab7ac4ebSTom Herbert break;
1198ab7ac4ebSTom Herbert head = &tkcm->kcm_sock_list;
1199ab7ac4ebSTom Herbert index++;
1200ab7ac4ebSTom Herbert }
1201ab7ac4ebSTom Herbert
1202ab7ac4ebSTom Herbert list_add(&kcm->kcm_sock_list, head);
1203ab7ac4ebSTom Herbert kcm->index = index;
1204ab7ac4ebSTom Herbert
1205ab7ac4ebSTom Herbert mux->kcm_socks_cnt++;
1206ab7ac4ebSTom Herbert spin_unlock_bh(&mux->lock);
1207ab7ac4ebSTom Herbert
1208ab7ac4ebSTom Herbert INIT_WORK(&kcm->tx_work, kcm_tx_work);
1209*00425508SKuniyuki Iwashima mutex_init(&kcm->tx_mutex);
1210ab7ac4ebSTom Herbert
1211ab7ac4ebSTom Herbert spin_lock_bh(&mux->rx_lock);
1212ab7ac4ebSTom Herbert kcm_rcv_ready(kcm);
1213ab7ac4ebSTom Herbert spin_unlock_bh(&mux->rx_lock);
1214ab7ac4ebSTom Herbert }
1215ab7ac4ebSTom Herbert
kcm_attach(struct socket * sock,struct socket * csock,struct bpf_prog * prog)1216ab7ac4ebSTom Herbert static int kcm_attach(struct socket *sock, struct socket *csock,
1217ab7ac4ebSTom Herbert struct bpf_prog *prog)
1218ab7ac4ebSTom Herbert {
1219ab7ac4ebSTom Herbert struct kcm_sock *kcm = kcm_sk(sock->sk);
1220ab7ac4ebSTom Herbert struct kcm_mux *mux = kcm->mux;
1221ab7ac4ebSTom Herbert struct sock *csk;
1222ab7ac4ebSTom Herbert struct kcm_psock *psock = NULL, *tpsock;
1223ab7ac4ebSTom Herbert struct list_head *head;
1224ab7ac4ebSTom Herbert int index = 0;
12253fd87127SEric Biggers static const struct strp_callbacks cb = {
12263fd87127SEric Biggers .rcv_msg = kcm_rcv_strparser,
12273fd87127SEric Biggers .parse_msg = kcm_parse_func_strparser,
12283fd87127SEric Biggers .read_sock_done = kcm_read_sock_done,
12293fd87127SEric Biggers };
12302cc683e8STom Herbert int err = 0;
1231ab7ac4ebSTom Herbert
1232ab7ac4ebSTom Herbert csk = csock->sk;
1233ab7ac4ebSTom Herbert if (!csk)
1234ab7ac4ebSTom Herbert return -EINVAL;
1235ab7ac4ebSTom Herbert
12362cc683e8STom Herbert lock_sock(csk);
12372cc683e8STom Herbert
1238581e7226STom Herbert /* Only allow TCP sockets to be attached for now */
1239581e7226STom Herbert if ((csk->sk_family != AF_INET && csk->sk_family != AF_INET6) ||
12402cc683e8STom Herbert csk->sk_protocol != IPPROTO_TCP) {
12412cc683e8STom Herbert err = -EOPNOTSUPP;
12422cc683e8STom Herbert goto out;
12432cc683e8STom Herbert }
1244581e7226STom Herbert
1245581e7226STom Herbert /* Don't allow listeners or closed sockets */
12462cc683e8STom Herbert if (csk->sk_state == TCP_LISTEN || csk->sk_state == TCP_CLOSE) {
12472cc683e8STom Herbert err = -EOPNOTSUPP;
12482cc683e8STom Herbert goto out;
12492cc683e8STom Herbert }
1250351050ecSEric Dumazet
1251ab7ac4ebSTom Herbert psock = kmem_cache_zalloc(kcm_psockp, GFP_KERNEL);
12522cc683e8STom Herbert if (!psock) {
12532cc683e8STom Herbert err = -ENOMEM;
12542cc683e8STom Herbert goto out;
12552cc683e8STom Herbert }
1256ab7ac4ebSTom Herbert
1257ab7ac4ebSTom Herbert psock->mux = mux;
1258ab7ac4ebSTom Herbert psock->sk = csk;
1259ab7ac4ebSTom Herbert psock->bpf_prog = prog;
126029152a34STom Herbert
1261ab7ac4ebSTom Herbert write_lock_bh(&csk->sk_callback_lock);
1262e5571240STom Herbert
126371a2fae5SBhaskar Chowdhury /* Check if sk_user_data is already by KCM or someone else.
1264e5571240STom Herbert * Must be done under lock to prevent race conditions.
1265e5571240STom Herbert */
1266e5571240STom Herbert if (csk->sk_user_data) {
1267e5571240STom Herbert write_unlock_bh(&csk->sk_callback_lock);
1268e5571240STom Herbert kmem_cache_free(kcm_psockp, psock);
12692cc683e8STom Herbert err = -EALREADY;
12702cc683e8STom Herbert goto out;
1271e5571240STom Herbert }
1272e5571240STom Herbert
12738fc29ff3SCong Wang err = strp_init(&psock->strp, csk, &cb);
12748fc29ff3SCong Wang if (err) {
12758fc29ff3SCong Wang write_unlock_bh(&csk->sk_callback_lock);
12768fc29ff3SCong Wang kmem_cache_free(kcm_psockp, psock);
12778fc29ff3SCong Wang goto out;
12788fc29ff3SCong Wang }
12798fc29ff3SCong Wang
1280ab7ac4ebSTom Herbert psock->save_data_ready = csk->sk_data_ready;
1281ab7ac4ebSTom Herbert psock->save_write_space = csk->sk_write_space;
1282ab7ac4ebSTom Herbert psock->save_state_change = csk->sk_state_change;
1283ab7ac4ebSTom Herbert csk->sk_user_data = psock;
128496a59083STom Herbert csk->sk_data_ready = psock_data_ready;
128596a59083STom Herbert csk->sk_write_space = psock_write_space;
128696a59083STom Herbert csk->sk_state_change = psock_state_change;
1287e5571240STom Herbert
1288ab7ac4ebSTom Herbert write_unlock_bh(&csk->sk_callback_lock);
1289ab7ac4ebSTom Herbert
1290e5571240STom Herbert sock_hold(csk);
1291e5571240STom Herbert
1292ab7ac4ebSTom Herbert /* Finished initialization, now add the psock to the MUX. */
1293ab7ac4ebSTom Herbert spin_lock_bh(&mux->lock);
1294ab7ac4ebSTom Herbert head = &mux->psocks;
1295ab7ac4ebSTom Herbert list_for_each_entry(tpsock, &mux->psocks, psock_list) {
1296ab7ac4ebSTom Herbert if (tpsock->index != index)
1297ab7ac4ebSTom Herbert break;
1298ab7ac4ebSTom Herbert head = &tpsock->psock_list;
1299ab7ac4ebSTom Herbert index++;
1300ab7ac4ebSTom Herbert }
1301ab7ac4ebSTom Herbert
1302ab7ac4ebSTom Herbert list_add(&psock->psock_list, head);
1303ab7ac4ebSTom Herbert psock->index = index;
1304ab7ac4ebSTom Herbert
1305cd6e111bSTom Herbert KCM_STATS_INCR(mux->stats.psock_attach);
1306ab7ac4ebSTom Herbert mux->psocks_cnt++;
1307ab7ac4ebSTom Herbert psock_now_avail(psock);
1308ab7ac4ebSTom Herbert spin_unlock_bh(&mux->lock);
1309ab7ac4ebSTom Herbert
1310ab7ac4ebSTom Herbert /* Schedule RX work in case there are already bytes queued */
13119b73896aSTom Herbert strp_check_rcv(&psock->strp);
1312ab7ac4ebSTom Herbert
13132cc683e8STom Herbert out:
13142cc683e8STom Herbert release_sock(csk);
13152cc683e8STom Herbert
13162cc683e8STom Herbert return err;
1317ab7ac4ebSTom Herbert }
1318ab7ac4ebSTom Herbert
kcm_attach_ioctl(struct socket * sock,struct kcm_attach * info)1319ab7ac4ebSTom Herbert static int kcm_attach_ioctl(struct socket *sock, struct kcm_attach *info)
1320ab7ac4ebSTom Herbert {
1321ab7ac4ebSTom Herbert struct socket *csock;
1322ab7ac4ebSTom Herbert struct bpf_prog *prog;
1323ab7ac4ebSTom Herbert int err;
1324ab7ac4ebSTom Herbert
1325ab7ac4ebSTom Herbert csock = sockfd_lookup(info->fd, &err);
1326ab7ac4ebSTom Herbert if (!csock)
1327ab7ac4ebSTom Herbert return -ENOENT;
1328ab7ac4ebSTom Herbert
1329113214beSDaniel Borkmann prog = bpf_prog_get_type(info->bpf_fd, BPF_PROG_TYPE_SOCKET_FILTER);
1330ab7ac4ebSTom Herbert if (IS_ERR(prog)) {
1331ab7ac4ebSTom Herbert err = PTR_ERR(prog);
1332ab7ac4ebSTom Herbert goto out;
1333ab7ac4ebSTom Herbert }
1334ab7ac4ebSTom Herbert
1335ab7ac4ebSTom Herbert err = kcm_attach(sock, csock, prog);
1336ab7ac4ebSTom Herbert if (err) {
1337ab7ac4ebSTom Herbert bpf_prog_put(prog);
1338ab7ac4ebSTom Herbert goto out;
1339ab7ac4ebSTom Herbert }
1340ab7ac4ebSTom Herbert
1341ab7ac4ebSTom Herbert /* Keep reference on file also */
1342ab7ac4ebSTom Herbert
1343ab7ac4ebSTom Herbert return 0;
1344ab7ac4ebSTom Herbert out:
13458dc879a1SZheng Yongjun sockfd_put(csock);
1346ab7ac4ebSTom Herbert return err;
1347ab7ac4ebSTom Herbert }
1348ab7ac4ebSTom Herbert
kcm_unattach(struct kcm_psock * psock)1349ab7ac4ebSTom Herbert static void kcm_unattach(struct kcm_psock *psock)
1350ab7ac4ebSTom Herbert {
1351ab7ac4ebSTom Herbert struct sock *csk = psock->sk;
1352ab7ac4ebSTom Herbert struct kcm_mux *mux = psock->mux;
1353ab7ac4ebSTom Herbert
13541616b38fSTom Herbert lock_sock(csk);
13551616b38fSTom Herbert
1356ab7ac4ebSTom Herbert /* Stop getting callbacks from TCP socket. After this there should
1357ab7ac4ebSTom Herbert * be no way to reserve a kcm for this psock.
1358ab7ac4ebSTom Herbert */
1359ab7ac4ebSTom Herbert write_lock_bh(&csk->sk_callback_lock);
1360ab7ac4ebSTom Herbert csk->sk_user_data = NULL;
1361ab7ac4ebSTom Herbert csk->sk_data_ready = psock->save_data_ready;
1362ab7ac4ebSTom Herbert csk->sk_write_space = psock->save_write_space;
1363ab7ac4ebSTom Herbert csk->sk_state_change = psock->save_state_change;
13649b73896aSTom Herbert strp_stop(&psock->strp);
1365ab7ac4ebSTom Herbert
1366ab7ac4ebSTom Herbert if (WARN_ON(psock->rx_kcm)) {
1367ab7ac4ebSTom Herbert write_unlock_bh(&csk->sk_callback_lock);
13682cc683e8STom Herbert release_sock(csk);
1369ab7ac4ebSTom Herbert return;
1370ab7ac4ebSTom Herbert }
1371ab7ac4ebSTom Herbert
1372ab7ac4ebSTom Herbert spin_lock_bh(&mux->rx_lock);
1373ab7ac4ebSTom Herbert
1374ab7ac4ebSTom Herbert /* Stop receiver activities. After this point psock should not be
1375ab7ac4ebSTom Herbert * able to get onto ready list either through callbacks or work.
1376ab7ac4ebSTom Herbert */
1377ab7ac4ebSTom Herbert if (psock->ready_rx_msg) {
1378ab7ac4ebSTom Herbert list_del(&psock->psock_ready_list);
1379ab7ac4ebSTom Herbert kfree_skb(psock->ready_rx_msg);
1380ab7ac4ebSTom Herbert psock->ready_rx_msg = NULL;
1381cd6e111bSTom Herbert KCM_STATS_INCR(mux->stats.rx_ready_drops);
1382ab7ac4ebSTom Herbert }
1383ab7ac4ebSTom Herbert
1384ab7ac4ebSTom Herbert spin_unlock_bh(&mux->rx_lock);
1385ab7ac4ebSTom Herbert
1386ab7ac4ebSTom Herbert write_unlock_bh(&csk->sk_callback_lock);
1387ab7ac4ebSTom Herbert
13881616b38fSTom Herbert /* Call strp_done without sock lock */
13891616b38fSTom Herbert release_sock(csk);
13909b73896aSTom Herbert strp_done(&psock->strp);
13911616b38fSTom Herbert lock_sock(csk);
1392ab7ac4ebSTom Herbert
1393ab7ac4ebSTom Herbert bpf_prog_put(psock->bpf_prog);
1394ab7ac4ebSTom Herbert
1395ab7ac4ebSTom Herbert spin_lock_bh(&mux->lock);
1396ab7ac4ebSTom Herbert
1397cd6e111bSTom Herbert aggregate_psock_stats(&psock->stats, &mux->aggregate_psock_stats);
13989b73896aSTom Herbert save_strp_stats(&psock->strp, &mux->aggregate_strp_stats);
1399cd6e111bSTom Herbert
1400cd6e111bSTom Herbert KCM_STATS_INCR(mux->stats.psock_unattach);
1401cd6e111bSTom Herbert
1402ab7ac4ebSTom Herbert if (psock->tx_kcm) {
1403ab7ac4ebSTom Herbert /* psock was reserved. Just mark it finished and we will clean
1404ab7ac4ebSTom Herbert * up in the kcm paths, we need kcm lock which can not be
1405ab7ac4ebSTom Herbert * acquired here.
1406ab7ac4ebSTom Herbert */
1407cd6e111bSTom Herbert KCM_STATS_INCR(mux->stats.psock_unattach_rsvd);
1408ab7ac4ebSTom Herbert spin_unlock_bh(&mux->lock);
1409ab7ac4ebSTom Herbert
1410ab7ac4ebSTom Herbert /* We are unattaching a socket that is reserved. Abort the
1411ab7ac4ebSTom Herbert * socket since we may be out of sync in sending on it. We need
1412ab7ac4ebSTom Herbert * to do this without the mux lock.
1413ab7ac4ebSTom Herbert */
1414ab7ac4ebSTom Herbert kcm_abort_tx_psock(psock, EPIPE, false);
1415ab7ac4ebSTom Herbert
1416ab7ac4ebSTom Herbert spin_lock_bh(&mux->lock);
1417ab7ac4ebSTom Herbert if (!psock->tx_kcm) {
1418ab7ac4ebSTom Herbert /* psock now unreserved in window mux was unlocked */
1419ab7ac4ebSTom Herbert goto no_reserved;
1420ab7ac4ebSTom Herbert }
1421ab7ac4ebSTom Herbert psock->done = 1;
1422ab7ac4ebSTom Herbert
1423ab7ac4ebSTom Herbert /* Commit done before queuing work to process it */
1424ab7ac4ebSTom Herbert smp_mb();
1425ab7ac4ebSTom Herbert
1426ab7ac4ebSTom Herbert /* Queue tx work to make sure psock->done is handled */
1427ab7ac4ebSTom Herbert queue_work(kcm_wq, &psock->tx_kcm->tx_work);
1428ab7ac4ebSTom Herbert spin_unlock_bh(&mux->lock);
1429ab7ac4ebSTom Herbert } else {
1430ab7ac4ebSTom Herbert no_reserved:
1431ab7ac4ebSTom Herbert if (!psock->tx_stopped)
1432ab7ac4ebSTom Herbert list_del(&psock->psock_avail_list);
1433ab7ac4ebSTom Herbert list_del(&psock->psock_list);
1434ab7ac4ebSTom Herbert mux->psocks_cnt--;
1435ab7ac4ebSTom Herbert spin_unlock_bh(&mux->lock);
1436ab7ac4ebSTom Herbert
1437ab7ac4ebSTom Herbert sock_put(csk);
1438ab7ac4ebSTom Herbert fput(csk->sk_socket->file);
1439ab7ac4ebSTom Herbert kmem_cache_free(kcm_psockp, psock);
1440ab7ac4ebSTom Herbert }
14411616b38fSTom Herbert
14421616b38fSTom Herbert release_sock(csk);
1443ab7ac4ebSTom Herbert }
1444ab7ac4ebSTom Herbert
kcm_unattach_ioctl(struct socket * sock,struct kcm_unattach * info)1445ab7ac4ebSTom Herbert static int kcm_unattach_ioctl(struct socket *sock, struct kcm_unattach *info)
1446ab7ac4ebSTom Herbert {
1447ab7ac4ebSTom Herbert struct kcm_sock *kcm = kcm_sk(sock->sk);
1448ab7ac4ebSTom Herbert struct kcm_mux *mux = kcm->mux;
1449ab7ac4ebSTom Herbert struct kcm_psock *psock;
1450ab7ac4ebSTom Herbert struct socket *csock;
1451ab7ac4ebSTom Herbert struct sock *csk;
1452ab7ac4ebSTom Herbert int err;
1453ab7ac4ebSTom Herbert
1454ab7ac4ebSTom Herbert csock = sockfd_lookup(info->fd, &err);
1455ab7ac4ebSTom Herbert if (!csock)
1456ab7ac4ebSTom Herbert return -ENOENT;
1457ab7ac4ebSTom Herbert
1458ab7ac4ebSTom Herbert csk = csock->sk;
1459ab7ac4ebSTom Herbert if (!csk) {
1460ab7ac4ebSTom Herbert err = -EINVAL;
1461ab7ac4ebSTom Herbert goto out;
1462ab7ac4ebSTom Herbert }
1463ab7ac4ebSTom Herbert
1464ab7ac4ebSTom Herbert err = -ENOENT;
1465ab7ac4ebSTom Herbert
1466ab7ac4ebSTom Herbert spin_lock_bh(&mux->lock);
1467ab7ac4ebSTom Herbert
1468ab7ac4ebSTom Herbert list_for_each_entry(psock, &mux->psocks, psock_list) {
1469ab7ac4ebSTom Herbert if (psock->sk != csk)
1470ab7ac4ebSTom Herbert continue;
1471ab7ac4ebSTom Herbert
1472ab7ac4ebSTom Herbert /* Found the matching psock */
1473ab7ac4ebSTom Herbert
1474ab7ac4ebSTom Herbert if (psock->unattaching || WARN_ON(psock->done)) {
1475ab7ac4ebSTom Herbert err = -EALREADY;
1476ab7ac4ebSTom Herbert break;
1477ab7ac4ebSTom Herbert }
1478ab7ac4ebSTom Herbert
1479ab7ac4ebSTom Herbert psock->unattaching = 1;
1480ab7ac4ebSTom Herbert
1481ab7ac4ebSTom Herbert spin_unlock_bh(&mux->lock);
1482ab7ac4ebSTom Herbert
14839b73896aSTom Herbert /* Lower socket lock should already be held */
1484ab7ac4ebSTom Herbert kcm_unattach(psock);
1485ab7ac4ebSTom Herbert
1486ab7ac4ebSTom Herbert err = 0;
1487ab7ac4ebSTom Herbert goto out;
1488ab7ac4ebSTom Herbert }
1489ab7ac4ebSTom Herbert
1490ab7ac4ebSTom Herbert spin_unlock_bh(&mux->lock);
1491ab7ac4ebSTom Herbert
1492ab7ac4ebSTom Herbert out:
14938dc879a1SZheng Yongjun sockfd_put(csock);
1494ab7ac4ebSTom Herbert return err;
1495ab7ac4ebSTom Herbert }
1496ab7ac4ebSTom Herbert
1497ab7ac4ebSTom Herbert static struct proto kcm_proto = {
1498ab7ac4ebSTom Herbert .name = "KCM",
1499ab7ac4ebSTom Herbert .owner = THIS_MODULE,
1500ab7ac4ebSTom Herbert .obj_size = sizeof(struct kcm_sock),
1501ab7ac4ebSTom Herbert };
1502ab7ac4ebSTom Herbert
1503ab7ac4ebSTom Herbert /* Clone a kcm socket. */
kcm_clone(struct socket * osock)1504a5739435SAl Viro static struct file *kcm_clone(struct socket *osock)
1505ab7ac4ebSTom Herbert {
1506ab7ac4ebSTom Herbert struct socket *newsock;
1507ab7ac4ebSTom Herbert struct sock *newsk;
1508ab7ac4ebSTom Herbert
1509ab7ac4ebSTom Herbert newsock = sock_alloc();
1510ab7ac4ebSTom Herbert if (!newsock)
1511a5739435SAl Viro return ERR_PTR(-ENFILE);
1512ab7ac4ebSTom Herbert
1513ab7ac4ebSTom Herbert newsock->type = osock->type;
1514ab7ac4ebSTom Herbert newsock->ops = osock->ops;
1515ab7ac4ebSTom Herbert
1516ab7ac4ebSTom Herbert __module_get(newsock->ops->owner);
1517ab7ac4ebSTom Herbert
1518ab7ac4ebSTom Herbert newsk = sk_alloc(sock_net(osock->sk), PF_KCM, GFP_KERNEL,
1519eb7f54b9SKirill Tkhai &kcm_proto, false);
1520ab7ac4ebSTom Herbert if (!newsk) {
1521a5739435SAl Viro sock_release(newsock);
1522a5739435SAl Viro return ERR_PTR(-ENOMEM);
1523ab7ac4ebSTom Herbert }
1524ab7ac4ebSTom Herbert sock_init_data(newsock, newsk);
1525ab7ac4ebSTom Herbert init_kcm_sock(kcm_sk(newsk), kcm_sk(osock->sk)->mux);
1526ab7ac4ebSTom Herbert
15278e1611e2SAl Viro return sock_alloc_file(newsock, 0, osock->sk->sk_prot_creator->name);
1528ab7ac4ebSTom Herbert }
1529ab7ac4ebSTom Herbert
kcm_ioctl(struct socket * sock,unsigned int cmd,unsigned long arg)1530ab7ac4ebSTom Herbert static int kcm_ioctl(struct socket *sock, unsigned int cmd, unsigned long arg)
1531ab7ac4ebSTom Herbert {
1532ab7ac4ebSTom Herbert int err;
1533ab7ac4ebSTom Herbert
1534ab7ac4ebSTom Herbert switch (cmd) {
1535ab7ac4ebSTom Herbert case SIOCKCMATTACH: {
1536ab7ac4ebSTom Herbert struct kcm_attach info;
1537ab7ac4ebSTom Herbert
1538ab7ac4ebSTom Herbert if (copy_from_user(&info, (void __user *)arg, sizeof(info)))
1539a80db69eSWANG Cong return -EFAULT;
1540ab7ac4ebSTom Herbert
1541ab7ac4ebSTom Herbert err = kcm_attach_ioctl(sock, &info);
1542ab7ac4ebSTom Herbert
1543ab7ac4ebSTom Herbert break;
1544ab7ac4ebSTom Herbert }
1545ab7ac4ebSTom Herbert case SIOCKCMUNATTACH: {
1546ab7ac4ebSTom Herbert struct kcm_unattach info;
1547ab7ac4ebSTom Herbert
1548ab7ac4ebSTom Herbert if (copy_from_user(&info, (void __user *)arg, sizeof(info)))
1549a80db69eSWANG Cong return -EFAULT;
1550ab7ac4ebSTom Herbert
1551ab7ac4ebSTom Herbert err = kcm_unattach_ioctl(sock, &info);
1552ab7ac4ebSTom Herbert
1553ab7ac4ebSTom Herbert break;
1554ab7ac4ebSTom Herbert }
1555ab7ac4ebSTom Herbert case SIOCKCMCLONE: {
1556ab7ac4ebSTom Herbert struct kcm_clone info;
1557a5739435SAl Viro struct file *file;
1558ab7ac4ebSTom Herbert
1559a5739435SAl Viro info.fd = get_unused_fd_flags(0);
1560a5739435SAl Viro if (unlikely(info.fd < 0))
1561a5739435SAl Viro return info.fd;
1562a5739435SAl Viro
1563a5739435SAl Viro file = kcm_clone(sock);
1564a5739435SAl Viro if (IS_ERR(file)) {
1565a5739435SAl Viro put_unused_fd(info.fd);
1566a5739435SAl Viro return PTR_ERR(file);
1567a5739435SAl Viro }
1568ab7ac4ebSTom Herbert if (copy_to_user((void __user *)arg, &info,
1569ab7ac4ebSTom Herbert sizeof(info))) {
1570a5739435SAl Viro put_unused_fd(info.fd);
1571a5739435SAl Viro fput(file);
1572a5739435SAl Viro return -EFAULT;
1573ab7ac4ebSTom Herbert }
1574a5739435SAl Viro fd_install(info.fd, file);
1575a5739435SAl Viro err = 0;
1576ab7ac4ebSTom Herbert break;
1577ab7ac4ebSTom Herbert }
1578ab7ac4ebSTom Herbert default:
1579ab7ac4ebSTom Herbert err = -ENOIOCTLCMD;
1580ab7ac4ebSTom Herbert break;
1581ab7ac4ebSTom Herbert }
1582ab7ac4ebSTom Herbert
1583ab7ac4ebSTom Herbert return err;
1584ab7ac4ebSTom Herbert }
1585ab7ac4ebSTom Herbert
free_mux(struct rcu_head * rcu)1586ab7ac4ebSTom Herbert static void free_mux(struct rcu_head *rcu)
1587ab7ac4ebSTom Herbert {
1588ab7ac4ebSTom Herbert struct kcm_mux *mux = container_of(rcu,
1589ab7ac4ebSTom Herbert struct kcm_mux, rcu);
1590ab7ac4ebSTom Herbert
1591ab7ac4ebSTom Herbert kmem_cache_free(kcm_muxp, mux);
1592ab7ac4ebSTom Herbert }
1593ab7ac4ebSTom Herbert
release_mux(struct kcm_mux * mux)1594ab7ac4ebSTom Herbert static void release_mux(struct kcm_mux *mux)
1595ab7ac4ebSTom Herbert {
1596ab7ac4ebSTom Herbert struct kcm_net *knet = mux->knet;
1597ab7ac4ebSTom Herbert struct kcm_psock *psock, *tmp_psock;
1598ab7ac4ebSTom Herbert
1599ab7ac4ebSTom Herbert /* Release psocks */
1600ab7ac4ebSTom Herbert list_for_each_entry_safe(psock, tmp_psock,
1601ab7ac4ebSTom Herbert &mux->psocks, psock_list) {
16021616b38fSTom Herbert if (!WARN_ON(psock->unattaching))
1603ab7ac4ebSTom Herbert kcm_unattach(psock);
1604ab7ac4ebSTom Herbert }
1605ab7ac4ebSTom Herbert
1606ab7ac4ebSTom Herbert if (WARN_ON(mux->psocks_cnt))
1607ab7ac4ebSTom Herbert return;
1608ab7ac4ebSTom Herbert
1609ab7ac4ebSTom Herbert __skb_queue_purge(&mux->rx_hold_queue);
1610ab7ac4ebSTom Herbert
1611ab7ac4ebSTom Herbert mutex_lock(&knet->mutex);
1612cd6e111bSTom Herbert aggregate_mux_stats(&mux->stats, &knet->aggregate_mux_stats);
1613cd6e111bSTom Herbert aggregate_psock_stats(&mux->aggregate_psock_stats,
1614cd6e111bSTom Herbert &knet->aggregate_psock_stats);
16159b73896aSTom Herbert aggregate_strp_stats(&mux->aggregate_strp_stats,
16169b73896aSTom Herbert &knet->aggregate_strp_stats);
1617ab7ac4ebSTom Herbert list_del_rcu(&mux->kcm_mux_list);
1618ab7ac4ebSTom Herbert knet->count--;
1619ab7ac4ebSTom Herbert mutex_unlock(&knet->mutex);
1620ab7ac4ebSTom Herbert
1621ab7ac4ebSTom Herbert call_rcu(&mux->rcu, free_mux);
1622ab7ac4ebSTom Herbert }
1623ab7ac4ebSTom Herbert
kcm_done(struct kcm_sock * kcm)1624ab7ac4ebSTom Herbert static void kcm_done(struct kcm_sock *kcm)
1625ab7ac4ebSTom Herbert {
1626ab7ac4ebSTom Herbert struct kcm_mux *mux = kcm->mux;
1627ab7ac4ebSTom Herbert struct sock *sk = &kcm->sk;
1628ab7ac4ebSTom Herbert int socks_cnt;
1629ab7ac4ebSTom Herbert
1630ab7ac4ebSTom Herbert spin_lock_bh(&mux->rx_lock);
1631ab7ac4ebSTom Herbert if (kcm->rx_psock) {
1632ab7ac4ebSTom Herbert /* Cleanup in unreserve_rx_kcm */
1633ab7ac4ebSTom Herbert WARN_ON(kcm->done);
1634ab7ac4ebSTom Herbert kcm->rx_disabled = 1;
1635ab7ac4ebSTom Herbert kcm->done = 1;
1636ab7ac4ebSTom Herbert spin_unlock_bh(&mux->rx_lock);
1637ab7ac4ebSTom Herbert return;
1638ab7ac4ebSTom Herbert }
1639ab7ac4ebSTom Herbert
1640ab7ac4ebSTom Herbert if (kcm->rx_wait) {
1641ab7ac4ebSTom Herbert list_del(&kcm->wait_rx_list);
16420c745b51SEric Dumazet /* paired with lockless reads in kcm_rfree() */
16430c745b51SEric Dumazet WRITE_ONCE(kcm->rx_wait, false);
1644ab7ac4ebSTom Herbert }
1645ab7ac4ebSTom Herbert /* Move any pending receive messages to other kcm sockets */
1646ab7ac4ebSTom Herbert requeue_rx_msgs(mux, &sk->sk_receive_queue);
1647ab7ac4ebSTom Herbert
1648ab7ac4ebSTom Herbert spin_unlock_bh(&mux->rx_lock);
1649ab7ac4ebSTom Herbert
1650ab7ac4ebSTom Herbert if (WARN_ON(sk_rmem_alloc_get(sk)))
1651ab7ac4ebSTom Herbert return;
1652ab7ac4ebSTom Herbert
1653ab7ac4ebSTom Herbert /* Detach from MUX */
1654ab7ac4ebSTom Herbert spin_lock_bh(&mux->lock);
1655ab7ac4ebSTom Herbert
1656ab7ac4ebSTom Herbert list_del(&kcm->kcm_sock_list);
1657ab7ac4ebSTom Herbert mux->kcm_socks_cnt--;
1658ab7ac4ebSTom Herbert socks_cnt = mux->kcm_socks_cnt;
1659ab7ac4ebSTom Herbert
1660ab7ac4ebSTom Herbert spin_unlock_bh(&mux->lock);
1661ab7ac4ebSTom Herbert
1662ab7ac4ebSTom Herbert if (!socks_cnt) {
1663ab7ac4ebSTom Herbert /* We are done with the mux now. */
1664ab7ac4ebSTom Herbert release_mux(mux);
1665ab7ac4ebSTom Herbert }
1666ab7ac4ebSTom Herbert
1667ab7ac4ebSTom Herbert WARN_ON(kcm->rx_wait);
1668ab7ac4ebSTom Herbert
1669ab7ac4ebSTom Herbert sock_put(&kcm->sk);
1670ab7ac4ebSTom Herbert }
1671ab7ac4ebSTom Herbert
1672ab7ac4ebSTom Herbert /* Called by kcm_release to close a KCM socket.
1673ab7ac4ebSTom Herbert * If this is the last KCM socket on the MUX, destroy the MUX.
1674ab7ac4ebSTom Herbert */
kcm_release(struct socket * sock)1675ab7ac4ebSTom Herbert static int kcm_release(struct socket *sock)
1676ab7ac4ebSTom Herbert {
1677ab7ac4ebSTom Herbert struct sock *sk = sock->sk;
1678ab7ac4ebSTom Herbert struct kcm_sock *kcm;
1679ab7ac4ebSTom Herbert struct kcm_mux *mux;
1680ab7ac4ebSTom Herbert struct kcm_psock *psock;
1681ab7ac4ebSTom Herbert
1682ab7ac4ebSTom Herbert if (!sk)
1683ab7ac4ebSTom Herbert return 0;
1684ab7ac4ebSTom Herbert
1685ab7ac4ebSTom Herbert kcm = kcm_sk(sk);
1686ab7ac4ebSTom Herbert mux = kcm->mux;
1687ab7ac4ebSTom Herbert
1688ec7eede3SEric Dumazet lock_sock(sk);
1689ab7ac4ebSTom Herbert sock_orphan(sk);
1690ab7ac4ebSTom Herbert kfree_skb(kcm->seq_skb);
1691ab7ac4ebSTom Herbert
1692ab7ac4ebSTom Herbert /* Purge queue under lock to avoid race condition with tx_work trying
1693ab7ac4ebSTom Herbert * to act when queue is nonempty. If tx_work runs after this point
1694ab7ac4ebSTom Herbert * it will just return.
1695ab7ac4ebSTom Herbert */
1696ab7ac4ebSTom Herbert __skb_queue_purge(&sk->sk_write_queue);
16979b73896aSTom Herbert
16989b73896aSTom Herbert /* Set tx_stopped. This is checked when psock is bound to a kcm and we
16999b73896aSTom Herbert * get a writespace callback. This prevents further work being queued
17009b73896aSTom Herbert * from the callback (unbinding the psock occurs after canceling work.
17019b73896aSTom Herbert */
17029b73896aSTom Herbert kcm->tx_stopped = 1;
17039b73896aSTom Herbert
1704ab7ac4ebSTom Herbert release_sock(sk);
1705ab7ac4ebSTom Herbert
1706ab7ac4ebSTom Herbert spin_lock_bh(&mux->lock);
1707ab7ac4ebSTom Herbert if (kcm->tx_wait) {
1708ab7ac4ebSTom Herbert /* Take of tx_wait list, after this point there should be no way
1709ab7ac4ebSTom Herbert * that a psock will be assigned to this kcm.
1710ab7ac4ebSTom Herbert */
1711ab7ac4ebSTom Herbert list_del(&kcm->wait_psock_list);
1712ab7ac4ebSTom Herbert kcm->tx_wait = false;
1713ab7ac4ebSTom Herbert }
1714ab7ac4ebSTom Herbert spin_unlock_bh(&mux->lock);
1715ab7ac4ebSTom Herbert
1716ab7ac4ebSTom Herbert /* Cancel work. After this point there should be no outside references
1717ab7ac4ebSTom Herbert * to the kcm socket.
1718ab7ac4ebSTom Herbert */
1719ab7ac4ebSTom Herbert cancel_work_sync(&kcm->tx_work);
1720ab7ac4ebSTom Herbert
1721ab7ac4ebSTom Herbert lock_sock(sk);
1722ab7ac4ebSTom Herbert psock = kcm->tx_psock;
1723ab7ac4ebSTom Herbert if (psock) {
1724ab7ac4ebSTom Herbert /* A psock was reserved, so we need to kill it since it
1725ab7ac4ebSTom Herbert * may already have some bytes queued from a message. We
1726ab7ac4ebSTom Herbert * need to do this after removing kcm from tx_wait list.
1727ab7ac4ebSTom Herbert */
1728ab7ac4ebSTom Herbert kcm_abort_tx_psock(psock, EPIPE, false);
1729ab7ac4ebSTom Herbert unreserve_psock(kcm);
1730ab7ac4ebSTom Herbert }
1731ab7ac4ebSTom Herbert release_sock(sk);
1732ab7ac4ebSTom Herbert
1733ab7ac4ebSTom Herbert WARN_ON(kcm->tx_wait);
1734ab7ac4ebSTom Herbert WARN_ON(kcm->tx_psock);
1735ab7ac4ebSTom Herbert
1736ab7ac4ebSTom Herbert sock->sk = NULL;
1737ab7ac4ebSTom Herbert
1738ab7ac4ebSTom Herbert kcm_done(kcm);
1739ab7ac4ebSTom Herbert
1740ab7ac4ebSTom Herbert return 0;
1741ab7ac4ebSTom Herbert }
1742ab7ac4ebSTom Herbert
174391687355STom Herbert static const struct proto_ops kcm_dgram_ops = {
1744ab7ac4ebSTom Herbert .family = PF_KCM,
1745ab7ac4ebSTom Herbert .owner = THIS_MODULE,
1746ab7ac4ebSTom Herbert .release = kcm_release,
1747ab7ac4ebSTom Herbert .bind = sock_no_bind,
1748ab7ac4ebSTom Herbert .connect = sock_no_connect,
1749ab7ac4ebSTom Herbert .socketpair = sock_no_socketpair,
1750ab7ac4ebSTom Herbert .accept = sock_no_accept,
1751ab7ac4ebSTom Herbert .getname = sock_no_getname,
1752a11e1d43SLinus Torvalds .poll = datagram_poll,
1753ab7ac4ebSTom Herbert .ioctl = kcm_ioctl,
1754ab7ac4ebSTom Herbert .listen = sock_no_listen,
1755ab7ac4ebSTom Herbert .shutdown = sock_no_shutdown,
1756ab7ac4ebSTom Herbert .setsockopt = kcm_setsockopt,
1757ab7ac4ebSTom Herbert .getsockopt = kcm_getsockopt,
1758ab7ac4ebSTom Herbert .sendmsg = kcm_sendmsg,
1759ab7ac4ebSTom Herbert .recvmsg = kcm_recvmsg,
1760ab7ac4ebSTom Herbert .mmap = sock_no_mmap,
1761951ace99SDavid Howells .splice_eof = kcm_splice_eof,
1762ab7ac4ebSTom Herbert };
1763ab7ac4ebSTom Herbert
176491687355STom Herbert static const struct proto_ops kcm_seqpacket_ops = {
176591687355STom Herbert .family = PF_KCM,
176691687355STom Herbert .owner = THIS_MODULE,
176791687355STom Herbert .release = kcm_release,
176891687355STom Herbert .bind = sock_no_bind,
176991687355STom Herbert .connect = sock_no_connect,
177091687355STom Herbert .socketpair = sock_no_socketpair,
177191687355STom Herbert .accept = sock_no_accept,
177291687355STom Herbert .getname = sock_no_getname,
1773a11e1d43SLinus Torvalds .poll = datagram_poll,
177491687355STom Herbert .ioctl = kcm_ioctl,
177591687355STom Herbert .listen = sock_no_listen,
177691687355STom Herbert .shutdown = sock_no_shutdown,
177791687355STom Herbert .setsockopt = kcm_setsockopt,
177891687355STom Herbert .getsockopt = kcm_getsockopt,
177991687355STom Herbert .sendmsg = kcm_sendmsg,
178091687355STom Herbert .recvmsg = kcm_recvmsg,
178191687355STom Herbert .mmap = sock_no_mmap,
1782951ace99SDavid Howells .splice_eof = kcm_splice_eof,
178391687355STom Herbert .splice_read = kcm_splice_read,
178491687355STom Herbert };
178591687355STom Herbert
1786ab7ac4ebSTom Herbert /* Create proto operation for kcm sockets */
kcm_create(struct net * net,struct socket * sock,int protocol,int kern)1787ab7ac4ebSTom Herbert static int kcm_create(struct net *net, struct socket *sock,
1788ab7ac4ebSTom Herbert int protocol, int kern)
1789ab7ac4ebSTom Herbert {
1790ab7ac4ebSTom Herbert struct kcm_net *knet = net_generic(net, kcm_net_id);
1791ab7ac4ebSTom Herbert struct sock *sk;
1792ab7ac4ebSTom Herbert struct kcm_mux *mux;
1793ab7ac4ebSTom Herbert
1794ab7ac4ebSTom Herbert switch (sock->type) {
1795ab7ac4ebSTom Herbert case SOCK_DGRAM:
179691687355STom Herbert sock->ops = &kcm_dgram_ops;
179791687355STom Herbert break;
1798ab7ac4ebSTom Herbert case SOCK_SEQPACKET:
179991687355STom Herbert sock->ops = &kcm_seqpacket_ops;
1800ab7ac4ebSTom Herbert break;
1801ab7ac4ebSTom Herbert default:
1802ab7ac4ebSTom Herbert return -ESOCKTNOSUPPORT;
1803ab7ac4ebSTom Herbert }
1804ab7ac4ebSTom Herbert
1805ab7ac4ebSTom Herbert if (protocol != KCMPROTO_CONNECTED)
1806ab7ac4ebSTom Herbert return -EPROTONOSUPPORT;
1807ab7ac4ebSTom Herbert
1808ab7ac4ebSTom Herbert sk = sk_alloc(net, PF_KCM, GFP_KERNEL, &kcm_proto, kern);
1809ab7ac4ebSTom Herbert if (!sk)
1810ab7ac4ebSTom Herbert return -ENOMEM;
1811ab7ac4ebSTom Herbert
1812ab7ac4ebSTom Herbert /* Allocate a kcm mux, shared between KCM sockets */
1813ab7ac4ebSTom Herbert mux = kmem_cache_zalloc(kcm_muxp, GFP_KERNEL);
1814ab7ac4ebSTom Herbert if (!mux) {
1815ab7ac4ebSTom Herbert sk_free(sk);
1816ab7ac4ebSTom Herbert return -ENOMEM;
1817ab7ac4ebSTom Herbert }
1818ab7ac4ebSTom Herbert
1819ab7ac4ebSTom Herbert spin_lock_init(&mux->lock);
1820ab7ac4ebSTom Herbert spin_lock_init(&mux->rx_lock);
1821ab7ac4ebSTom Herbert INIT_LIST_HEAD(&mux->kcm_socks);
1822ab7ac4ebSTom Herbert INIT_LIST_HEAD(&mux->kcm_rx_waiters);
1823ab7ac4ebSTom Herbert INIT_LIST_HEAD(&mux->kcm_tx_waiters);
1824ab7ac4ebSTom Herbert
1825ab7ac4ebSTom Herbert INIT_LIST_HEAD(&mux->psocks);
1826ab7ac4ebSTom Herbert INIT_LIST_HEAD(&mux->psocks_ready);
1827ab7ac4ebSTom Herbert INIT_LIST_HEAD(&mux->psocks_avail);
1828ab7ac4ebSTom Herbert
1829ab7ac4ebSTom Herbert mux->knet = knet;
1830ab7ac4ebSTom Herbert
1831ab7ac4ebSTom Herbert /* Add new MUX to list */
1832ab7ac4ebSTom Herbert mutex_lock(&knet->mutex);
1833ab7ac4ebSTom Herbert list_add_rcu(&mux->kcm_mux_list, &knet->mux_list);
1834ab7ac4ebSTom Herbert knet->count++;
1835ab7ac4ebSTom Herbert mutex_unlock(&knet->mutex);
1836ab7ac4ebSTom Herbert
1837ab7ac4ebSTom Herbert skb_queue_head_init(&mux->rx_hold_queue);
1838ab7ac4ebSTom Herbert
1839ab7ac4ebSTom Herbert /* Init KCM socket */
1840ab7ac4ebSTom Herbert sock_init_data(sock, sk);
1841ab7ac4ebSTom Herbert init_kcm_sock(kcm_sk(sk), mux);
1842ab7ac4ebSTom Herbert
1843ab7ac4ebSTom Herbert return 0;
1844ab7ac4ebSTom Herbert }
1845ab7ac4ebSTom Herbert
1846173e7837Slinzhang static const struct net_proto_family kcm_family_ops = {
1847ab7ac4ebSTom Herbert .family = PF_KCM,
1848ab7ac4ebSTom Herbert .create = kcm_create,
1849ab7ac4ebSTom Herbert .owner = THIS_MODULE,
1850ab7ac4ebSTom Herbert };
1851ab7ac4ebSTom Herbert
kcm_init_net(struct net * net)1852ab7ac4ebSTom Herbert static __net_init int kcm_init_net(struct net *net)
1853ab7ac4ebSTom Herbert {
1854ab7ac4ebSTom Herbert struct kcm_net *knet = net_generic(net, kcm_net_id);
1855ab7ac4ebSTom Herbert
1856ab7ac4ebSTom Herbert INIT_LIST_HEAD_RCU(&knet->mux_list);
1857ab7ac4ebSTom Herbert mutex_init(&knet->mutex);
1858ab7ac4ebSTom Herbert
1859ab7ac4ebSTom Herbert return 0;
1860ab7ac4ebSTom Herbert }
1861ab7ac4ebSTom Herbert
kcm_exit_net(struct net * net)1862ab7ac4ebSTom Herbert static __net_exit void kcm_exit_net(struct net *net)
1863ab7ac4ebSTom Herbert {
1864ab7ac4ebSTom Herbert struct kcm_net *knet = net_generic(net, kcm_net_id);
1865ab7ac4ebSTom Herbert
1866ab7ac4ebSTom Herbert /* All KCM sockets should be closed at this point, which should mean
1867ab7ac4ebSTom Herbert * that all multiplexors and psocks have been destroyed.
1868ab7ac4ebSTom Herbert */
1869ab7ac4ebSTom Herbert WARN_ON(!list_empty(&knet->mux_list));
18706ad40b36SShigeru Yoshida
18716ad40b36SShigeru Yoshida mutex_destroy(&knet->mutex);
1872ab7ac4ebSTom Herbert }
1873ab7ac4ebSTom Herbert
1874ab7ac4ebSTom Herbert static struct pernet_operations kcm_net_ops = {
1875ab7ac4ebSTom Herbert .init = kcm_init_net,
1876ab7ac4ebSTom Herbert .exit = kcm_exit_net,
1877ab7ac4ebSTom Herbert .id = &kcm_net_id,
1878ab7ac4ebSTom Herbert .size = sizeof(struct kcm_net),
1879ab7ac4ebSTom Herbert };
1880ab7ac4ebSTom Herbert
kcm_init(void)1881ab7ac4ebSTom Herbert static int __init kcm_init(void)
1882ab7ac4ebSTom Herbert {
1883ab7ac4ebSTom Herbert int err = -ENOMEM;
1884ab7ac4ebSTom Herbert
1885ab7ac4ebSTom Herbert kcm_muxp = kmem_cache_create("kcm_mux_cache",
1886ab7ac4ebSTom Herbert sizeof(struct kcm_mux), 0,
1887c2115240SYueHaibing SLAB_HWCACHE_ALIGN, NULL);
1888ab7ac4ebSTom Herbert if (!kcm_muxp)
1889ab7ac4ebSTom Herbert goto fail;
1890ab7ac4ebSTom Herbert
1891ab7ac4ebSTom Herbert kcm_psockp = kmem_cache_create("kcm_psock_cache",
1892ab7ac4ebSTom Herbert sizeof(struct kcm_psock), 0,
1893c2115240SYueHaibing SLAB_HWCACHE_ALIGN, NULL);
1894ab7ac4ebSTom Herbert if (!kcm_psockp)
1895ab7ac4ebSTom Herbert goto fail;
1896ab7ac4ebSTom Herbert
1897ab7ac4ebSTom Herbert kcm_wq = create_singlethread_workqueue("kkcmd");
1898ab7ac4ebSTom Herbert if (!kcm_wq)
1899ab7ac4ebSTom Herbert goto fail;
1900ab7ac4ebSTom Herbert
1901ab7ac4ebSTom Herbert err = proto_register(&kcm_proto, 1);
1902ab7ac4ebSTom Herbert if (err)
1903ab7ac4ebSTom Herbert goto fail;
1904ab7ac4ebSTom Herbert
1905ab7ac4ebSTom Herbert err = register_pernet_device(&kcm_net_ops);
1906ab7ac4ebSTom Herbert if (err)
1907ab7ac4ebSTom Herbert goto net_ops_fail;
1908ab7ac4ebSTom Herbert
19093c446e6fSJiri Slaby err = sock_register(&kcm_family_ops);
19103c446e6fSJiri Slaby if (err)
19113c446e6fSJiri Slaby goto sock_register_fail;
19123c446e6fSJiri Slaby
1913cd6e111bSTom Herbert err = kcm_proc_init();
1914cd6e111bSTom Herbert if (err)
1915cd6e111bSTom Herbert goto proc_init_fail;
1916cd6e111bSTom Herbert
1917ab7ac4ebSTom Herbert return 0;
1918ab7ac4ebSTom Herbert
1919cd6e111bSTom Herbert proc_init_fail:
1920ab7ac4ebSTom Herbert sock_unregister(PF_KCM);
1921ab7ac4ebSTom Herbert
1922ab7ac4ebSTom Herbert sock_register_fail:
19233c446e6fSJiri Slaby unregister_pernet_device(&kcm_net_ops);
19243c446e6fSJiri Slaby
19253c446e6fSJiri Slaby net_ops_fail:
1926ab7ac4ebSTom Herbert proto_unregister(&kcm_proto);
1927ab7ac4ebSTom Herbert
1928ab7ac4ebSTom Herbert fail:
1929ab7ac4ebSTom Herbert kmem_cache_destroy(kcm_muxp);
1930ab7ac4ebSTom Herbert kmem_cache_destroy(kcm_psockp);
1931ab7ac4ebSTom Herbert
1932ab7ac4ebSTom Herbert if (kcm_wq)
1933ab7ac4ebSTom Herbert destroy_workqueue(kcm_wq);
1934ab7ac4ebSTom Herbert
1935ab7ac4ebSTom Herbert return err;
1936ab7ac4ebSTom Herbert }
1937ab7ac4ebSTom Herbert
kcm_exit(void)1938ab7ac4ebSTom Herbert static void __exit kcm_exit(void)
1939ab7ac4ebSTom Herbert {
1940cd6e111bSTom Herbert kcm_proc_exit();
1941ab7ac4ebSTom Herbert sock_unregister(PF_KCM);
19423c446e6fSJiri Slaby unregister_pernet_device(&kcm_net_ops);
1943ab7ac4ebSTom Herbert proto_unregister(&kcm_proto);
1944ab7ac4ebSTom Herbert destroy_workqueue(kcm_wq);
1945ab7ac4ebSTom Herbert
1946ab7ac4ebSTom Herbert kmem_cache_destroy(kcm_muxp);
1947ab7ac4ebSTom Herbert kmem_cache_destroy(kcm_psockp);
1948ab7ac4ebSTom Herbert }
1949ab7ac4ebSTom Herbert
1950ab7ac4ebSTom Herbert module_init(kcm_init);
1951ab7ac4ebSTom Herbert module_exit(kcm_exit);
1952ab7ac4ebSTom Herbert
1953ab7ac4ebSTom Herbert MODULE_LICENSE("GPL");
1954ab7ac4ebSTom Herbert MODULE_ALIAS_NETPROTO(PF_KCM);
1955