xref: /openbmc/linux/net/core/sock_reuseport.c (revision b261eda8)
1 // SPDX-License-Identifier: GPL-2.0
2 /*
3  * To speed up listener socket lookup, create an array to store all sockets
4  * listening on the same port.  This allows a decision to be made after finding
5  * the first socket.  An optional BPF program can also be configured for
6  * selecting the socket index from the array of available sockets.
7  */
8 
9 #include <net/ip.h>
10 #include <net/sock_reuseport.h>
11 #include <linux/bpf.h>
12 #include <linux/idr.h>
13 #include <linux/filter.h>
14 #include <linux/rcupdate.h>
15 
16 #define INIT_SOCKS 128
17 
18 DEFINE_SPINLOCK(reuseport_lock);
19 
20 static DEFINE_IDA(reuseport_ida);
21 static int reuseport_resurrect(struct sock *sk, struct sock_reuseport *old_reuse,
22 			       struct sock_reuseport *reuse, bool bind_inany);
23 
reuseport_has_conns_set(struct sock * sk)24 void reuseport_has_conns_set(struct sock *sk)
25 {
26 	struct sock_reuseport *reuse;
27 
28 	if (!rcu_access_pointer(sk->sk_reuseport_cb))
29 		return;
30 
31 	spin_lock_bh(&reuseport_lock);
32 	reuse = rcu_dereference_protected(sk->sk_reuseport_cb,
33 					  lockdep_is_held(&reuseport_lock));
34 	if (likely(reuse))
35 		reuse->has_conns = 1;
36 	spin_unlock_bh(&reuseport_lock);
37 }
38 EXPORT_SYMBOL(reuseport_has_conns_set);
39 
__reuseport_get_incoming_cpu(struct sock_reuseport * reuse)40 static void __reuseport_get_incoming_cpu(struct sock_reuseport *reuse)
41 {
42 	/* Paired with READ_ONCE() in reuseport_select_sock_by_hash(). */
43 	WRITE_ONCE(reuse->incoming_cpu, reuse->incoming_cpu + 1);
44 }
45 
__reuseport_put_incoming_cpu(struct sock_reuseport * reuse)46 static void __reuseport_put_incoming_cpu(struct sock_reuseport *reuse)
47 {
48 	/* Paired with READ_ONCE() in reuseport_select_sock_by_hash(). */
49 	WRITE_ONCE(reuse->incoming_cpu, reuse->incoming_cpu - 1);
50 }
51 
reuseport_get_incoming_cpu(struct sock * sk,struct sock_reuseport * reuse)52 static void reuseport_get_incoming_cpu(struct sock *sk, struct sock_reuseport *reuse)
53 {
54 	if (sk->sk_incoming_cpu >= 0)
55 		__reuseport_get_incoming_cpu(reuse);
56 }
57 
reuseport_put_incoming_cpu(struct sock * sk,struct sock_reuseport * reuse)58 static void reuseport_put_incoming_cpu(struct sock *sk, struct sock_reuseport *reuse)
59 {
60 	if (sk->sk_incoming_cpu >= 0)
61 		__reuseport_put_incoming_cpu(reuse);
62 }
63 
reuseport_update_incoming_cpu(struct sock * sk,int val)64 void reuseport_update_incoming_cpu(struct sock *sk, int val)
65 {
66 	struct sock_reuseport *reuse;
67 	int old_sk_incoming_cpu;
68 
69 	if (unlikely(!rcu_access_pointer(sk->sk_reuseport_cb))) {
70 		/* Paired with REAE_ONCE() in sk_incoming_cpu_update()
71 		 * and compute_score().
72 		 */
73 		WRITE_ONCE(sk->sk_incoming_cpu, val);
74 		return;
75 	}
76 
77 	spin_lock_bh(&reuseport_lock);
78 
79 	/* This must be done under reuseport_lock to avoid a race with
80 	 * reuseport_grow(), which accesses sk->sk_incoming_cpu without
81 	 * lock_sock() when detaching a shutdown()ed sk.
82 	 *
83 	 * Paired with READ_ONCE() in reuseport_select_sock_by_hash().
84 	 */
85 	old_sk_incoming_cpu = sk->sk_incoming_cpu;
86 	WRITE_ONCE(sk->sk_incoming_cpu, val);
87 
88 	reuse = rcu_dereference_protected(sk->sk_reuseport_cb,
89 					  lockdep_is_held(&reuseport_lock));
90 
91 	/* reuseport_grow() has detached a closed sk. */
92 	if (!reuse)
93 		goto out;
94 
95 	if (old_sk_incoming_cpu < 0 && val >= 0)
96 		__reuseport_get_incoming_cpu(reuse);
97 	else if (old_sk_incoming_cpu >= 0 && val < 0)
98 		__reuseport_put_incoming_cpu(reuse);
99 
100 out:
101 	spin_unlock_bh(&reuseport_lock);
102 }
103 
reuseport_sock_index(struct sock * sk,const struct sock_reuseport * reuse,bool closed)104 static int reuseport_sock_index(struct sock *sk,
105 				const struct sock_reuseport *reuse,
106 				bool closed)
107 {
108 	int left, right;
109 
110 	if (!closed) {
111 		left = 0;
112 		right = reuse->num_socks;
113 	} else {
114 		left = reuse->max_socks - reuse->num_closed_socks;
115 		right = reuse->max_socks;
116 	}
117 
118 	for (; left < right; left++)
119 		if (reuse->socks[left] == sk)
120 			return left;
121 	return -1;
122 }
123 
__reuseport_add_sock(struct sock * sk,struct sock_reuseport * reuse)124 static void __reuseport_add_sock(struct sock *sk,
125 				 struct sock_reuseport *reuse)
126 {
127 	reuse->socks[reuse->num_socks] = sk;
128 	/* paired with smp_rmb() in reuseport_(select|migrate)_sock() */
129 	smp_wmb();
130 	reuse->num_socks++;
131 	reuseport_get_incoming_cpu(sk, reuse);
132 }
133 
__reuseport_detach_sock(struct sock * sk,struct sock_reuseport * reuse)134 static bool __reuseport_detach_sock(struct sock *sk,
135 				    struct sock_reuseport *reuse)
136 {
137 	int i = reuseport_sock_index(sk, reuse, false);
138 
139 	if (i == -1)
140 		return false;
141 
142 	reuse->socks[i] = reuse->socks[reuse->num_socks - 1];
143 	reuse->num_socks--;
144 	reuseport_put_incoming_cpu(sk, reuse);
145 
146 	return true;
147 }
148 
__reuseport_add_closed_sock(struct sock * sk,struct sock_reuseport * reuse)149 static void __reuseport_add_closed_sock(struct sock *sk,
150 					struct sock_reuseport *reuse)
151 {
152 	reuse->socks[reuse->max_socks - reuse->num_closed_socks - 1] = sk;
153 	/* paired with READ_ONCE() in inet_csk_bind_conflict() */
154 	WRITE_ONCE(reuse->num_closed_socks, reuse->num_closed_socks + 1);
155 	reuseport_get_incoming_cpu(sk, reuse);
156 }
157 
__reuseport_detach_closed_sock(struct sock * sk,struct sock_reuseport * reuse)158 static bool __reuseport_detach_closed_sock(struct sock *sk,
159 					   struct sock_reuseport *reuse)
160 {
161 	int i = reuseport_sock_index(sk, reuse, true);
162 
163 	if (i == -1)
164 		return false;
165 
166 	reuse->socks[i] = reuse->socks[reuse->max_socks - reuse->num_closed_socks];
167 	/* paired with READ_ONCE() in inet_csk_bind_conflict() */
168 	WRITE_ONCE(reuse->num_closed_socks, reuse->num_closed_socks - 1);
169 	reuseport_put_incoming_cpu(sk, reuse);
170 
171 	return true;
172 }
173 
__reuseport_alloc(unsigned int max_socks)174 static struct sock_reuseport *__reuseport_alloc(unsigned int max_socks)
175 {
176 	unsigned int size = sizeof(struct sock_reuseport) +
177 		      sizeof(struct sock *) * max_socks;
178 	struct sock_reuseport *reuse = kzalloc(size, GFP_ATOMIC);
179 
180 	if (!reuse)
181 		return NULL;
182 
183 	reuse->max_socks = max_socks;
184 
185 	RCU_INIT_POINTER(reuse->prog, NULL);
186 	return reuse;
187 }
188 
reuseport_alloc(struct sock * sk,bool bind_inany)189 int reuseport_alloc(struct sock *sk, bool bind_inany)
190 {
191 	struct sock_reuseport *reuse;
192 	int id, ret = 0;
193 
194 	/* bh lock used since this function call may precede hlist lock in
195 	 * soft irq of receive path or setsockopt from process context
196 	 */
197 	spin_lock_bh(&reuseport_lock);
198 
199 	/* Allocation attempts can occur concurrently via the setsockopt path
200 	 * and the bind/hash path.  Nothing to do when we lose the race.
201 	 */
202 	reuse = rcu_dereference_protected(sk->sk_reuseport_cb,
203 					  lockdep_is_held(&reuseport_lock));
204 	if (reuse) {
205 		if (reuse->num_closed_socks) {
206 			/* sk was shutdown()ed before */
207 			ret = reuseport_resurrect(sk, reuse, NULL, bind_inany);
208 			goto out;
209 		}
210 
211 		/* Only set reuse->bind_inany if the bind_inany is true.
212 		 * Otherwise, it will overwrite the reuse->bind_inany
213 		 * which was set by the bind/hash path.
214 		 */
215 		if (bind_inany)
216 			reuse->bind_inany = bind_inany;
217 		goto out;
218 	}
219 
220 	reuse = __reuseport_alloc(INIT_SOCKS);
221 	if (!reuse) {
222 		ret = -ENOMEM;
223 		goto out;
224 	}
225 
226 	id = ida_alloc(&reuseport_ida, GFP_ATOMIC);
227 	if (id < 0) {
228 		kfree(reuse);
229 		ret = id;
230 		goto out;
231 	}
232 
233 	reuse->reuseport_id = id;
234 	reuse->bind_inany = bind_inany;
235 	reuse->socks[0] = sk;
236 	reuse->num_socks = 1;
237 	reuseport_get_incoming_cpu(sk, reuse);
238 	rcu_assign_pointer(sk->sk_reuseport_cb, reuse);
239 
240 out:
241 	spin_unlock_bh(&reuseport_lock);
242 
243 	return ret;
244 }
245 EXPORT_SYMBOL(reuseport_alloc);
246 
reuseport_grow(struct sock_reuseport * reuse)247 static struct sock_reuseport *reuseport_grow(struct sock_reuseport *reuse)
248 {
249 	struct sock_reuseport *more_reuse;
250 	u32 more_socks_size, i;
251 
252 	more_socks_size = reuse->max_socks * 2U;
253 	if (more_socks_size > U16_MAX) {
254 		if (reuse->num_closed_socks) {
255 			/* Make room by removing a closed sk.
256 			 * The child has already been migrated.
257 			 * Only reqsk left at this point.
258 			 */
259 			struct sock *sk;
260 
261 			sk = reuse->socks[reuse->max_socks - reuse->num_closed_socks];
262 			RCU_INIT_POINTER(sk->sk_reuseport_cb, NULL);
263 			__reuseport_detach_closed_sock(sk, reuse);
264 
265 			return reuse;
266 		}
267 
268 		return NULL;
269 	}
270 
271 	more_reuse = __reuseport_alloc(more_socks_size);
272 	if (!more_reuse)
273 		return NULL;
274 
275 	more_reuse->num_socks = reuse->num_socks;
276 	more_reuse->num_closed_socks = reuse->num_closed_socks;
277 	more_reuse->prog = reuse->prog;
278 	more_reuse->reuseport_id = reuse->reuseport_id;
279 	more_reuse->bind_inany = reuse->bind_inany;
280 	more_reuse->has_conns = reuse->has_conns;
281 	more_reuse->incoming_cpu = reuse->incoming_cpu;
282 
283 	memcpy(more_reuse->socks, reuse->socks,
284 	       reuse->num_socks * sizeof(struct sock *));
285 	memcpy(more_reuse->socks +
286 	       (more_reuse->max_socks - more_reuse->num_closed_socks),
287 	       reuse->socks + (reuse->max_socks - reuse->num_closed_socks),
288 	       reuse->num_closed_socks * sizeof(struct sock *));
289 	more_reuse->synq_overflow_ts = READ_ONCE(reuse->synq_overflow_ts);
290 
291 	for (i = 0; i < reuse->max_socks; ++i)
292 		rcu_assign_pointer(reuse->socks[i]->sk_reuseport_cb,
293 				   more_reuse);
294 
295 	/* Note: we use kfree_rcu here instead of reuseport_free_rcu so
296 	 * that reuse and more_reuse can temporarily share a reference
297 	 * to prog.
298 	 */
299 	kfree_rcu(reuse, rcu);
300 	return more_reuse;
301 }
302 
reuseport_free_rcu(struct rcu_head * head)303 static void reuseport_free_rcu(struct rcu_head *head)
304 {
305 	struct sock_reuseport *reuse;
306 
307 	reuse = container_of(head, struct sock_reuseport, rcu);
308 	sk_reuseport_prog_free(rcu_dereference_protected(reuse->prog, 1));
309 	ida_free(&reuseport_ida, reuse->reuseport_id);
310 	kfree(reuse);
311 }
312 
313 /**
314  *  reuseport_add_sock - Add a socket to the reuseport group of another.
315  *  @sk:  New socket to add to the group.
316  *  @sk2: Socket belonging to the existing reuseport group.
317  *  @bind_inany: Whether or not the group is bound to a local INANY address.
318  *
319  *  May return ENOMEM and not add socket to group under memory pressure.
320  */
reuseport_add_sock(struct sock * sk,struct sock * sk2,bool bind_inany)321 int reuseport_add_sock(struct sock *sk, struct sock *sk2, bool bind_inany)
322 {
323 	struct sock_reuseport *old_reuse, *reuse;
324 
325 	if (!rcu_access_pointer(sk2->sk_reuseport_cb)) {
326 		int err = reuseport_alloc(sk2, bind_inany);
327 
328 		if (err)
329 			return err;
330 	}
331 
332 	spin_lock_bh(&reuseport_lock);
333 	reuse = rcu_dereference_protected(sk2->sk_reuseport_cb,
334 					  lockdep_is_held(&reuseport_lock));
335 	old_reuse = rcu_dereference_protected(sk->sk_reuseport_cb,
336 					      lockdep_is_held(&reuseport_lock));
337 	if (old_reuse && old_reuse->num_closed_socks) {
338 		/* sk was shutdown()ed before */
339 		int err = reuseport_resurrect(sk, old_reuse, reuse, reuse->bind_inany);
340 
341 		spin_unlock_bh(&reuseport_lock);
342 		return err;
343 	}
344 
345 	if (old_reuse && old_reuse->num_socks != 1) {
346 		spin_unlock_bh(&reuseport_lock);
347 		return -EBUSY;
348 	}
349 
350 	if (reuse->num_socks + reuse->num_closed_socks == reuse->max_socks) {
351 		reuse = reuseport_grow(reuse);
352 		if (!reuse) {
353 			spin_unlock_bh(&reuseport_lock);
354 			return -ENOMEM;
355 		}
356 	}
357 
358 	__reuseport_add_sock(sk, reuse);
359 	rcu_assign_pointer(sk->sk_reuseport_cb, reuse);
360 
361 	spin_unlock_bh(&reuseport_lock);
362 
363 	if (old_reuse)
364 		call_rcu(&old_reuse->rcu, reuseport_free_rcu);
365 	return 0;
366 }
367 EXPORT_SYMBOL(reuseport_add_sock);
368 
reuseport_resurrect(struct sock * sk,struct sock_reuseport * old_reuse,struct sock_reuseport * reuse,bool bind_inany)369 static int reuseport_resurrect(struct sock *sk, struct sock_reuseport *old_reuse,
370 			       struct sock_reuseport *reuse, bool bind_inany)
371 {
372 	if (old_reuse == reuse) {
373 		/* If sk was in the same reuseport group, just pop sk out of
374 		 * the closed section and push sk into the listening section.
375 		 */
376 		__reuseport_detach_closed_sock(sk, old_reuse);
377 		__reuseport_add_sock(sk, old_reuse);
378 		return 0;
379 	}
380 
381 	if (!reuse) {
382 		/* In bind()/listen() path, we cannot carry over the eBPF prog
383 		 * for the shutdown()ed socket. In setsockopt() path, we should
384 		 * not change the eBPF prog of listening sockets by attaching a
385 		 * prog to the shutdown()ed socket. Thus, we will allocate a new
386 		 * reuseport group and detach sk from the old group.
387 		 */
388 		int id;
389 
390 		reuse = __reuseport_alloc(INIT_SOCKS);
391 		if (!reuse)
392 			return -ENOMEM;
393 
394 		id = ida_alloc(&reuseport_ida, GFP_ATOMIC);
395 		if (id < 0) {
396 			kfree(reuse);
397 			return id;
398 		}
399 
400 		reuse->reuseport_id = id;
401 		reuse->bind_inany = bind_inany;
402 	} else {
403 		/* Move sk from the old group to the new one if
404 		 * - all the other listeners in the old group were close()d or
405 		 *   shutdown()ed, and then sk2 has listen()ed on the same port
406 		 * OR
407 		 * - sk listen()ed without bind() (or with autobind), was
408 		 *   shutdown()ed, and then listen()s on another port which
409 		 *   sk2 listen()s on.
410 		 */
411 		if (reuse->num_socks + reuse->num_closed_socks == reuse->max_socks) {
412 			reuse = reuseport_grow(reuse);
413 			if (!reuse)
414 				return -ENOMEM;
415 		}
416 	}
417 
418 	__reuseport_detach_closed_sock(sk, old_reuse);
419 	__reuseport_add_sock(sk, reuse);
420 	rcu_assign_pointer(sk->sk_reuseport_cb, reuse);
421 
422 	if (old_reuse->num_socks + old_reuse->num_closed_socks == 0)
423 		call_rcu(&old_reuse->rcu, reuseport_free_rcu);
424 
425 	return 0;
426 }
427 
reuseport_detach_sock(struct sock * sk)428 void reuseport_detach_sock(struct sock *sk)
429 {
430 	struct sock_reuseport *reuse;
431 
432 	spin_lock_bh(&reuseport_lock);
433 	reuse = rcu_dereference_protected(sk->sk_reuseport_cb,
434 					  lockdep_is_held(&reuseport_lock));
435 
436 	/* reuseport_grow() has detached a closed sk */
437 	if (!reuse)
438 		goto out;
439 
440 	/* Notify the bpf side. The sk may be added to a sockarray
441 	 * map. If so, sockarray logic will remove it from the map.
442 	 *
443 	 * Other bpf map types that work with reuseport, like sockmap,
444 	 * don't need an explicit callback from here. They override sk
445 	 * unhash/close ops to remove the sk from the map before we
446 	 * get to this point.
447 	 */
448 	bpf_sk_reuseport_detach(sk);
449 
450 	rcu_assign_pointer(sk->sk_reuseport_cb, NULL);
451 
452 	if (!__reuseport_detach_closed_sock(sk, reuse))
453 		__reuseport_detach_sock(sk, reuse);
454 
455 	if (reuse->num_socks + reuse->num_closed_socks == 0)
456 		call_rcu(&reuse->rcu, reuseport_free_rcu);
457 
458 out:
459 	spin_unlock_bh(&reuseport_lock);
460 }
461 EXPORT_SYMBOL(reuseport_detach_sock);
462 
reuseport_stop_listen_sock(struct sock * sk)463 void reuseport_stop_listen_sock(struct sock *sk)
464 {
465 	if (sk->sk_protocol == IPPROTO_TCP) {
466 		struct sock_reuseport *reuse;
467 		struct bpf_prog *prog;
468 
469 		spin_lock_bh(&reuseport_lock);
470 
471 		reuse = rcu_dereference_protected(sk->sk_reuseport_cb,
472 						  lockdep_is_held(&reuseport_lock));
473 		prog = rcu_dereference_protected(reuse->prog,
474 						 lockdep_is_held(&reuseport_lock));
475 
476 		if (READ_ONCE(sock_net(sk)->ipv4.sysctl_tcp_migrate_req) ||
477 		    (prog && prog->expected_attach_type == BPF_SK_REUSEPORT_SELECT_OR_MIGRATE)) {
478 			/* Migration capable, move sk from the listening section
479 			 * to the closed section.
480 			 */
481 			bpf_sk_reuseport_detach(sk);
482 
483 			__reuseport_detach_sock(sk, reuse);
484 			__reuseport_add_closed_sock(sk, reuse);
485 
486 			spin_unlock_bh(&reuseport_lock);
487 			return;
488 		}
489 
490 		spin_unlock_bh(&reuseport_lock);
491 	}
492 
493 	/* Not capable to do migration, detach immediately */
494 	reuseport_detach_sock(sk);
495 }
496 EXPORT_SYMBOL(reuseport_stop_listen_sock);
497 
run_bpf_filter(struct sock_reuseport * reuse,u16 socks,struct bpf_prog * prog,struct sk_buff * skb,int hdr_len)498 static struct sock *run_bpf_filter(struct sock_reuseport *reuse, u16 socks,
499 				   struct bpf_prog *prog, struct sk_buff *skb,
500 				   int hdr_len)
501 {
502 	struct sk_buff *nskb = NULL;
503 	u32 index;
504 
505 	if (skb_shared(skb)) {
506 		nskb = skb_clone(skb, GFP_ATOMIC);
507 		if (!nskb)
508 			return NULL;
509 		skb = nskb;
510 	}
511 
512 	/* temporarily advance data past protocol header */
513 	if (!pskb_pull(skb, hdr_len)) {
514 		kfree_skb(nskb);
515 		return NULL;
516 	}
517 	index = bpf_prog_run_save_cb(prog, skb);
518 	__skb_push(skb, hdr_len);
519 
520 	consume_skb(nskb);
521 
522 	if (index >= socks)
523 		return NULL;
524 
525 	return reuse->socks[index];
526 }
527 
reuseport_select_sock_by_hash(struct sock_reuseport * reuse,u32 hash,u16 num_socks)528 static struct sock *reuseport_select_sock_by_hash(struct sock_reuseport *reuse,
529 						  u32 hash, u16 num_socks)
530 {
531 	struct sock *first_valid_sk = NULL;
532 	int i, j;
533 
534 	i = j = reciprocal_scale(hash, num_socks);
535 	do {
536 		struct sock *sk = reuse->socks[i];
537 
538 		if (sk->sk_state != TCP_ESTABLISHED) {
539 			/* Paired with WRITE_ONCE() in __reuseport_(get|put)_incoming_cpu(). */
540 			if (!READ_ONCE(reuse->incoming_cpu))
541 				return sk;
542 
543 			/* Paired with WRITE_ONCE() in reuseport_update_incoming_cpu(). */
544 			if (READ_ONCE(sk->sk_incoming_cpu) == raw_smp_processor_id())
545 				return sk;
546 
547 			if (!first_valid_sk)
548 				first_valid_sk = sk;
549 		}
550 
551 		i++;
552 		if (i >= num_socks)
553 			i = 0;
554 	} while (i != j);
555 
556 	return first_valid_sk;
557 }
558 
559 /**
560  *  reuseport_select_sock - Select a socket from an SO_REUSEPORT group.
561  *  @sk: First socket in the group.
562  *  @hash: When no BPF filter is available, use this hash to select.
563  *  @skb: skb to run through BPF filter.
564  *  @hdr_len: BPF filter expects skb data pointer at payload data.  If
565  *    the skb does not yet point at the payload, this parameter represents
566  *    how far the pointer needs to advance to reach the payload.
567  *  Returns a socket that should receive the packet (or NULL on error).
568  */
reuseport_select_sock(struct sock * sk,u32 hash,struct sk_buff * skb,int hdr_len)569 struct sock *reuseport_select_sock(struct sock *sk,
570 				   u32 hash,
571 				   struct sk_buff *skb,
572 				   int hdr_len)
573 {
574 	struct sock_reuseport *reuse;
575 	struct bpf_prog *prog;
576 	struct sock *sk2 = NULL;
577 	u16 socks;
578 
579 	rcu_read_lock();
580 	reuse = rcu_dereference(sk->sk_reuseport_cb);
581 
582 	/* if memory allocation failed or add call is not yet complete */
583 	if (!reuse)
584 		goto out;
585 
586 	prog = rcu_dereference(reuse->prog);
587 	socks = READ_ONCE(reuse->num_socks);
588 	if (likely(socks)) {
589 		/* paired with smp_wmb() in __reuseport_add_sock() */
590 		smp_rmb();
591 
592 		if (!prog || !skb)
593 			goto select_by_hash;
594 
595 		if (prog->type == BPF_PROG_TYPE_SK_REUSEPORT)
596 			sk2 = bpf_run_sk_reuseport(reuse, sk, prog, skb, NULL, hash);
597 		else
598 			sk2 = run_bpf_filter(reuse, socks, prog, skb, hdr_len);
599 
600 select_by_hash:
601 		/* no bpf or invalid bpf result: fall back to hash usage */
602 		if (!sk2)
603 			sk2 = reuseport_select_sock_by_hash(reuse, hash, socks);
604 	}
605 
606 out:
607 	rcu_read_unlock();
608 	return sk2;
609 }
610 EXPORT_SYMBOL(reuseport_select_sock);
611 
612 /**
613  *  reuseport_migrate_sock - Select a socket from an SO_REUSEPORT group.
614  *  @sk: close()ed or shutdown()ed socket in the group.
615  *  @migrating_sk: ESTABLISHED/SYN_RECV full socket in the accept queue or
616  *    NEW_SYN_RECV request socket during 3WHS.
617  *  @skb: skb to run through BPF filter.
618  *  Returns a socket (with sk_refcnt +1) that should accept the child socket
619  *  (or NULL on error).
620  */
reuseport_migrate_sock(struct sock * sk,struct sock * migrating_sk,struct sk_buff * skb)621 struct sock *reuseport_migrate_sock(struct sock *sk,
622 				    struct sock *migrating_sk,
623 				    struct sk_buff *skb)
624 {
625 	struct sock_reuseport *reuse;
626 	struct sock *nsk = NULL;
627 	bool allocated = false;
628 	struct bpf_prog *prog;
629 	u16 socks;
630 	u32 hash;
631 
632 	rcu_read_lock();
633 
634 	reuse = rcu_dereference(sk->sk_reuseport_cb);
635 	if (!reuse)
636 		goto out;
637 
638 	socks = READ_ONCE(reuse->num_socks);
639 	if (unlikely(!socks))
640 		goto failure;
641 
642 	/* paired with smp_wmb() in __reuseport_add_sock() */
643 	smp_rmb();
644 
645 	hash = migrating_sk->sk_hash;
646 	prog = rcu_dereference(reuse->prog);
647 	if (!prog || prog->expected_attach_type != BPF_SK_REUSEPORT_SELECT_OR_MIGRATE) {
648 		if (READ_ONCE(sock_net(sk)->ipv4.sysctl_tcp_migrate_req))
649 			goto select_by_hash;
650 		goto failure;
651 	}
652 
653 	if (!skb) {
654 		skb = alloc_skb(0, GFP_ATOMIC);
655 		if (!skb)
656 			goto failure;
657 		allocated = true;
658 	}
659 
660 	nsk = bpf_run_sk_reuseport(reuse, sk, prog, skb, migrating_sk, hash);
661 
662 	if (allocated)
663 		kfree_skb(skb);
664 
665 select_by_hash:
666 	if (!nsk)
667 		nsk = reuseport_select_sock_by_hash(reuse, hash, socks);
668 
669 	if (IS_ERR_OR_NULL(nsk) || unlikely(!refcount_inc_not_zero(&nsk->sk_refcnt))) {
670 		nsk = NULL;
671 		goto failure;
672 	}
673 
674 out:
675 	rcu_read_unlock();
676 	return nsk;
677 
678 failure:
679 	__NET_INC_STATS(sock_net(sk), LINUX_MIB_TCPMIGRATEREQFAILURE);
680 	goto out;
681 }
682 EXPORT_SYMBOL(reuseport_migrate_sock);
683 
reuseport_attach_prog(struct sock * sk,struct bpf_prog * prog)684 int reuseport_attach_prog(struct sock *sk, struct bpf_prog *prog)
685 {
686 	struct sock_reuseport *reuse;
687 	struct bpf_prog *old_prog;
688 
689 	if (sk_unhashed(sk)) {
690 		int err;
691 
692 		if (!sk->sk_reuseport)
693 			return -EINVAL;
694 
695 		err = reuseport_alloc(sk, false);
696 		if (err)
697 			return err;
698 	} else if (!rcu_access_pointer(sk->sk_reuseport_cb)) {
699 		/* The socket wasn't bound with SO_REUSEPORT */
700 		return -EINVAL;
701 	}
702 
703 	spin_lock_bh(&reuseport_lock);
704 	reuse = rcu_dereference_protected(sk->sk_reuseport_cb,
705 					  lockdep_is_held(&reuseport_lock));
706 	old_prog = rcu_dereference_protected(reuse->prog,
707 					     lockdep_is_held(&reuseport_lock));
708 	rcu_assign_pointer(reuse->prog, prog);
709 	spin_unlock_bh(&reuseport_lock);
710 
711 	sk_reuseport_prog_free(old_prog);
712 	return 0;
713 }
714 EXPORT_SYMBOL(reuseport_attach_prog);
715 
reuseport_detach_prog(struct sock * sk)716 int reuseport_detach_prog(struct sock *sk)
717 {
718 	struct sock_reuseport *reuse;
719 	struct bpf_prog *old_prog;
720 
721 	old_prog = NULL;
722 	spin_lock_bh(&reuseport_lock);
723 	reuse = rcu_dereference_protected(sk->sk_reuseport_cb,
724 					  lockdep_is_held(&reuseport_lock));
725 
726 	/* reuse must be checked after acquiring the reuseport_lock
727 	 * because reuseport_grow() can detach a closed sk.
728 	 */
729 	if (!reuse) {
730 		spin_unlock_bh(&reuseport_lock);
731 		return sk->sk_reuseport ? -ENOENT : -EINVAL;
732 	}
733 
734 	if (sk_unhashed(sk) && reuse->num_closed_socks) {
735 		spin_unlock_bh(&reuseport_lock);
736 		return -ENOENT;
737 	}
738 
739 	old_prog = rcu_replace_pointer(reuse->prog, old_prog,
740 				       lockdep_is_held(&reuseport_lock));
741 	spin_unlock_bh(&reuseport_lock);
742 
743 	if (!old_prog)
744 		return -ENOENT;
745 
746 	sk_reuseport_prog_free(old_prog);
747 	return 0;
748 }
749 EXPORT_SYMBOL(reuseport_detach_prog);
750