xref: /openbmc/linux/kernel/bpf/memalloc.c (revision 3bc29c78)
1 // SPDX-License-Identifier: GPL-2.0-only
2 /* Copyright (c) 2022 Meta Platforms, Inc. and affiliates. */
3 #include <linux/mm.h>
4 #include <linux/llist.h>
5 #include <linux/bpf.h>
6 #include <linux/irq_work.h>
7 #include <linux/bpf_mem_alloc.h>
8 #include <linux/memcontrol.h>
9 #include <asm/local.h>
10 
11 /* Any context (including NMI) BPF specific memory allocator.
12  *
13  * Tracing BPF programs can attach to kprobe and fentry. Hence they
14  * run in unknown context where calling plain kmalloc() might not be safe.
15  *
16  * Front-end kmalloc() with per-cpu per-bucket cache of free elements.
17  * Refill this cache asynchronously from irq_work.
18  *
19  * CPU_0 buckets
20  * 16 32 64 96 128 196 256 512 1024 2048 4096
21  * ...
22  * CPU_N buckets
23  * 16 32 64 96 128 196 256 512 1024 2048 4096
24  *
25  * The buckets are prefilled at the start.
26  * BPF programs always run with migration disabled.
27  * It's safe to allocate from cache of the current cpu with irqs disabled.
28  * Free-ing is always done into bucket of the current cpu as well.
29  * irq_work trims extra free elements from buckets with kfree
30  * and refills them with kmalloc, so global kmalloc logic takes care
31  * of freeing objects allocated by one cpu and freed on another.
32  *
33  * Every allocated objected is padded with extra 8 bytes that contains
34  * struct llist_node.
35  */
36 #define LLIST_NODE_SZ sizeof(struct llist_node)
37 
38 /* similar to kmalloc, but sizeof == 8 bucket is gone */
39 static u8 size_index[24] __ro_after_init = {
40 	3,	/* 8 */
41 	3,	/* 16 */
42 	4,	/* 24 */
43 	4,	/* 32 */
44 	5,	/* 40 */
45 	5,	/* 48 */
46 	5,	/* 56 */
47 	5,	/* 64 */
48 	1,	/* 72 */
49 	1,	/* 80 */
50 	1,	/* 88 */
51 	1,	/* 96 */
52 	6,	/* 104 */
53 	6,	/* 112 */
54 	6,	/* 120 */
55 	6,	/* 128 */
56 	2,	/* 136 */
57 	2,	/* 144 */
58 	2,	/* 152 */
59 	2,	/* 160 */
60 	2,	/* 168 */
61 	2,	/* 176 */
62 	2,	/* 184 */
63 	2	/* 192 */
64 };
65 
bpf_mem_cache_idx(size_t size)66 static int bpf_mem_cache_idx(size_t size)
67 {
68 	if (!size || size > 4096)
69 		return -1;
70 
71 	if (size <= 192)
72 		return size_index[(size - 1) / 8] - 1;
73 
74 	return fls(size - 1) - 2;
75 }
76 
77 #define NUM_CACHES 11
78 
79 struct bpf_mem_cache {
80 	/* per-cpu list of free objects of size 'unit_size'.
81 	 * All accesses are done with interrupts disabled and 'active' counter
82 	 * protection with __llist_add() and __llist_del_first().
83 	 */
84 	struct llist_head free_llist;
85 	local_t active;
86 
87 	/* Operations on the free_list from unit_alloc/unit_free/bpf_mem_refill
88 	 * are sequenced by per-cpu 'active' counter. But unit_free() cannot
89 	 * fail. When 'active' is busy the unit_free() will add an object to
90 	 * free_llist_extra.
91 	 */
92 	struct llist_head free_llist_extra;
93 
94 	struct irq_work refill_work;
95 	struct obj_cgroup *objcg;
96 	int unit_size;
97 	/* count of objects in free_llist */
98 	int free_cnt;
99 	int low_watermark, high_watermark, batch;
100 	int percpu_size;
101 	bool draining;
102 	struct bpf_mem_cache *tgt;
103 
104 	/* list of objects to be freed after RCU GP */
105 	struct llist_head free_by_rcu;
106 	struct llist_node *free_by_rcu_tail;
107 	struct llist_head waiting_for_gp;
108 	struct llist_node *waiting_for_gp_tail;
109 	struct rcu_head rcu;
110 	atomic_t call_rcu_in_progress;
111 	struct llist_head free_llist_extra_rcu;
112 
113 	/* list of objects to be freed after RCU tasks trace GP */
114 	struct llist_head free_by_rcu_ttrace;
115 	struct llist_head waiting_for_gp_ttrace;
116 	struct rcu_head rcu_ttrace;
117 	atomic_t call_rcu_ttrace_in_progress;
118 };
119 
120 struct bpf_mem_caches {
121 	struct bpf_mem_cache cache[NUM_CACHES];
122 };
123 
__llist_del_first(struct llist_head * head)124 static struct llist_node notrace *__llist_del_first(struct llist_head *head)
125 {
126 	struct llist_node *entry, *next;
127 
128 	entry = head->first;
129 	if (!entry)
130 		return NULL;
131 	next = entry->next;
132 	head->first = next;
133 	return entry;
134 }
135 
__alloc(struct bpf_mem_cache * c,int node,gfp_t flags)136 static void *__alloc(struct bpf_mem_cache *c, int node, gfp_t flags)
137 {
138 	if (c->percpu_size) {
139 		void **obj = kmalloc_node(c->percpu_size, flags, node);
140 		void *pptr = __alloc_percpu_gfp(c->unit_size, 8, flags);
141 
142 		if (!obj || !pptr) {
143 			free_percpu(pptr);
144 			kfree(obj);
145 			return NULL;
146 		}
147 		obj[1] = pptr;
148 		return obj;
149 	}
150 
151 	return kmalloc_node(c->unit_size, flags | __GFP_ZERO, node);
152 }
153 
get_memcg(const struct bpf_mem_cache * c)154 static struct mem_cgroup *get_memcg(const struct bpf_mem_cache *c)
155 {
156 #ifdef CONFIG_MEMCG_KMEM
157 	if (c->objcg)
158 		return get_mem_cgroup_from_objcg(c->objcg);
159 #endif
160 
161 #ifdef CONFIG_MEMCG
162 	return root_mem_cgroup;
163 #else
164 	return NULL;
165 #endif
166 }
167 
inc_active(struct bpf_mem_cache * c,unsigned long * flags)168 static void inc_active(struct bpf_mem_cache *c, unsigned long *flags)
169 {
170 	if (IS_ENABLED(CONFIG_PREEMPT_RT))
171 		/* In RT irq_work runs in per-cpu kthread, so disable
172 		 * interrupts to avoid preemption and interrupts and
173 		 * reduce the chance of bpf prog executing on this cpu
174 		 * when active counter is busy.
175 		 */
176 		local_irq_save(*flags);
177 	/* alloc_bulk runs from irq_work which will not preempt a bpf
178 	 * program that does unit_alloc/unit_free since IRQs are
179 	 * disabled there. There is no race to increment 'active'
180 	 * counter. It protects free_llist from corruption in case NMI
181 	 * bpf prog preempted this loop.
182 	 */
183 	WARN_ON_ONCE(local_inc_return(&c->active) != 1);
184 }
185 
dec_active(struct bpf_mem_cache * c,unsigned long * flags)186 static void dec_active(struct bpf_mem_cache *c, unsigned long *flags)
187 {
188 	local_dec(&c->active);
189 	if (IS_ENABLED(CONFIG_PREEMPT_RT))
190 		local_irq_restore(*flags);
191 }
192 
add_obj_to_free_list(struct bpf_mem_cache * c,void * obj)193 static void add_obj_to_free_list(struct bpf_mem_cache *c, void *obj)
194 {
195 	unsigned long flags;
196 
197 	inc_active(c, &flags);
198 	__llist_add(obj, &c->free_llist);
199 	c->free_cnt++;
200 	dec_active(c, &flags);
201 }
202 
203 /* Mostly runs from irq_work except __init phase. */
alloc_bulk(struct bpf_mem_cache * c,int cnt,int node,bool atomic)204 static void alloc_bulk(struct bpf_mem_cache *c, int cnt, int node, bool atomic)
205 {
206 	struct mem_cgroup *memcg = NULL, *old_memcg;
207 	gfp_t gfp;
208 	void *obj;
209 	int i;
210 
211 	gfp = __GFP_NOWARN | __GFP_ACCOUNT;
212 	gfp |= atomic ? GFP_NOWAIT : GFP_KERNEL;
213 
214 	for (i = 0; i < cnt; i++) {
215 		/*
216 		 * For every 'c' llist_del_first(&c->free_by_rcu_ttrace); is
217 		 * done only by one CPU == current CPU. Other CPUs might
218 		 * llist_add() and llist_del_all() in parallel.
219 		 */
220 		obj = llist_del_first(&c->free_by_rcu_ttrace);
221 		if (!obj)
222 			break;
223 		add_obj_to_free_list(c, obj);
224 	}
225 	if (i >= cnt)
226 		return;
227 
228 	for (; i < cnt; i++) {
229 		obj = llist_del_first(&c->waiting_for_gp_ttrace);
230 		if (!obj)
231 			break;
232 		add_obj_to_free_list(c, obj);
233 	}
234 	if (i >= cnt)
235 		return;
236 
237 	memcg = get_memcg(c);
238 	old_memcg = set_active_memcg(memcg);
239 	for (; i < cnt; i++) {
240 		/* Allocate, but don't deplete atomic reserves that typical
241 		 * GFP_ATOMIC would do. irq_work runs on this cpu and kmalloc
242 		 * will allocate from the current numa node which is what we
243 		 * want here.
244 		 */
245 		obj = __alloc(c, node, gfp);
246 		if (!obj)
247 			break;
248 		add_obj_to_free_list(c, obj);
249 	}
250 	set_active_memcg(old_memcg);
251 	mem_cgroup_put(memcg);
252 }
253 
free_one(void * obj,bool percpu)254 static void free_one(void *obj, bool percpu)
255 {
256 	if (percpu) {
257 		free_percpu(((void **)obj)[1]);
258 		kfree(obj);
259 		return;
260 	}
261 
262 	kfree(obj);
263 }
264 
free_all(struct llist_node * llnode,bool percpu)265 static int free_all(struct llist_node *llnode, bool percpu)
266 {
267 	struct llist_node *pos, *t;
268 	int cnt = 0;
269 
270 	llist_for_each_safe(pos, t, llnode) {
271 		free_one(pos, percpu);
272 		cnt++;
273 	}
274 	return cnt;
275 }
276 
__free_rcu(struct rcu_head * head)277 static void __free_rcu(struct rcu_head *head)
278 {
279 	struct bpf_mem_cache *c = container_of(head, struct bpf_mem_cache, rcu_ttrace);
280 
281 	free_all(llist_del_all(&c->waiting_for_gp_ttrace), !!c->percpu_size);
282 	atomic_set(&c->call_rcu_ttrace_in_progress, 0);
283 }
284 
__free_rcu_tasks_trace(struct rcu_head * head)285 static void __free_rcu_tasks_trace(struct rcu_head *head)
286 {
287 	/* If RCU Tasks Trace grace period implies RCU grace period,
288 	 * there is no need to invoke call_rcu().
289 	 */
290 	if (rcu_trace_implies_rcu_gp())
291 		__free_rcu(head);
292 	else
293 		call_rcu(head, __free_rcu);
294 }
295 
enque_to_free(struct bpf_mem_cache * c,void * obj)296 static void enque_to_free(struct bpf_mem_cache *c, void *obj)
297 {
298 	struct llist_node *llnode = obj;
299 
300 	/* bpf_mem_cache is a per-cpu object. Freeing happens in irq_work.
301 	 * Nothing races to add to free_by_rcu_ttrace list.
302 	 */
303 	llist_add(llnode, &c->free_by_rcu_ttrace);
304 }
305 
do_call_rcu_ttrace(struct bpf_mem_cache * c)306 static void do_call_rcu_ttrace(struct bpf_mem_cache *c)
307 {
308 	struct llist_node *llnode, *t;
309 
310 	if (atomic_xchg(&c->call_rcu_ttrace_in_progress, 1)) {
311 		if (unlikely(READ_ONCE(c->draining))) {
312 			llnode = llist_del_all(&c->free_by_rcu_ttrace);
313 			free_all(llnode, !!c->percpu_size);
314 		}
315 		return;
316 	}
317 
318 	WARN_ON_ONCE(!llist_empty(&c->waiting_for_gp_ttrace));
319 	llist_for_each_safe(llnode, t, llist_del_all(&c->free_by_rcu_ttrace))
320 		llist_add(llnode, &c->waiting_for_gp_ttrace);
321 
322 	if (unlikely(READ_ONCE(c->draining))) {
323 		__free_rcu(&c->rcu_ttrace);
324 		return;
325 	}
326 
327 	/* Use call_rcu_tasks_trace() to wait for sleepable progs to finish.
328 	 * If RCU Tasks Trace grace period implies RCU grace period, free
329 	 * these elements directly, else use call_rcu() to wait for normal
330 	 * progs to finish and finally do free_one() on each element.
331 	 */
332 	call_rcu_tasks_trace(&c->rcu_ttrace, __free_rcu_tasks_trace);
333 }
334 
free_bulk(struct bpf_mem_cache * c)335 static void free_bulk(struct bpf_mem_cache *c)
336 {
337 	struct bpf_mem_cache *tgt = c->tgt;
338 	struct llist_node *llnode, *t;
339 	unsigned long flags;
340 	int cnt;
341 
342 	WARN_ON_ONCE(tgt->unit_size != c->unit_size);
343 
344 	do {
345 		inc_active(c, &flags);
346 		llnode = __llist_del_first(&c->free_llist);
347 		if (llnode)
348 			cnt = --c->free_cnt;
349 		else
350 			cnt = 0;
351 		dec_active(c, &flags);
352 		if (llnode)
353 			enque_to_free(tgt, llnode);
354 	} while (cnt > (c->high_watermark + c->low_watermark) / 2);
355 
356 	/* and drain free_llist_extra */
357 	llist_for_each_safe(llnode, t, llist_del_all(&c->free_llist_extra))
358 		enque_to_free(tgt, llnode);
359 	do_call_rcu_ttrace(tgt);
360 }
361 
__free_by_rcu(struct rcu_head * head)362 static void __free_by_rcu(struct rcu_head *head)
363 {
364 	struct bpf_mem_cache *c = container_of(head, struct bpf_mem_cache, rcu);
365 	struct bpf_mem_cache *tgt = c->tgt;
366 	struct llist_node *llnode;
367 
368 	llnode = llist_del_all(&c->waiting_for_gp);
369 	if (!llnode)
370 		goto out;
371 
372 	llist_add_batch(llnode, c->waiting_for_gp_tail, &tgt->free_by_rcu_ttrace);
373 
374 	/* Objects went through regular RCU GP. Send them to RCU tasks trace */
375 	do_call_rcu_ttrace(tgt);
376 out:
377 	atomic_set(&c->call_rcu_in_progress, 0);
378 }
379 
check_free_by_rcu(struct bpf_mem_cache * c)380 static void check_free_by_rcu(struct bpf_mem_cache *c)
381 {
382 	struct llist_node *llnode, *t;
383 	unsigned long flags;
384 
385 	/* drain free_llist_extra_rcu */
386 	if (unlikely(!llist_empty(&c->free_llist_extra_rcu))) {
387 		inc_active(c, &flags);
388 		llist_for_each_safe(llnode, t, llist_del_all(&c->free_llist_extra_rcu))
389 			if (__llist_add(llnode, &c->free_by_rcu))
390 				c->free_by_rcu_tail = llnode;
391 		dec_active(c, &flags);
392 	}
393 
394 	if (llist_empty(&c->free_by_rcu))
395 		return;
396 
397 	if (atomic_xchg(&c->call_rcu_in_progress, 1)) {
398 		/*
399 		 * Instead of kmalloc-ing new rcu_head and triggering 10k
400 		 * call_rcu() to hit rcutree.qhimark and force RCU to notice
401 		 * the overload just ask RCU to hurry up. There could be many
402 		 * objects in free_by_rcu list.
403 		 * This hint reduces memory consumption for an artificial
404 		 * benchmark from 2 Gbyte to 150 Mbyte.
405 		 */
406 		rcu_request_urgent_qs_task(current);
407 		return;
408 	}
409 
410 	WARN_ON_ONCE(!llist_empty(&c->waiting_for_gp));
411 
412 	inc_active(c, &flags);
413 	WRITE_ONCE(c->waiting_for_gp.first, __llist_del_all(&c->free_by_rcu));
414 	c->waiting_for_gp_tail = c->free_by_rcu_tail;
415 	dec_active(c, &flags);
416 
417 	if (unlikely(READ_ONCE(c->draining))) {
418 		free_all(llist_del_all(&c->waiting_for_gp), !!c->percpu_size);
419 		atomic_set(&c->call_rcu_in_progress, 0);
420 	} else {
421 		call_rcu_hurry(&c->rcu, __free_by_rcu);
422 	}
423 }
424 
bpf_mem_refill(struct irq_work * work)425 static void bpf_mem_refill(struct irq_work *work)
426 {
427 	struct bpf_mem_cache *c = container_of(work, struct bpf_mem_cache, refill_work);
428 	int cnt;
429 
430 	/* Racy access to free_cnt. It doesn't need to be 100% accurate */
431 	cnt = c->free_cnt;
432 	if (cnt < c->low_watermark)
433 		/* irq_work runs on this cpu and kmalloc will allocate
434 		 * from the current numa node which is what we want here.
435 		 */
436 		alloc_bulk(c, c->batch, NUMA_NO_NODE, true);
437 	else if (cnt > c->high_watermark)
438 		free_bulk(c);
439 
440 	check_free_by_rcu(c);
441 }
442 
irq_work_raise(struct bpf_mem_cache * c)443 static void notrace irq_work_raise(struct bpf_mem_cache *c)
444 {
445 	irq_work_queue(&c->refill_work);
446 }
447 
448 /* For typical bpf map case that uses bpf_mem_cache_alloc and single bucket
449  * the freelist cache will be elem_size * 64 (or less) on each cpu.
450  *
451  * For bpf programs that don't have statically known allocation sizes and
452  * assuming (low_mark + high_mark) / 2 as an average number of elements per
453  * bucket and all buckets are used the total amount of memory in freelists
454  * on each cpu will be:
455  * 64*16 + 64*32 + 64*64 + 64*96 + 64*128 + 64*196 + 64*256 + 32*512 + 16*1024 + 8*2048 + 4*4096
456  * == ~ 116 Kbyte using below heuristic.
457  * Initialized, but unused bpf allocator (not bpf map specific one) will
458  * consume ~ 11 Kbyte per cpu.
459  * Typical case will be between 11K and 116K closer to 11K.
460  * bpf progs can and should share bpf_mem_cache when possible.
461  */
init_refill_work(struct bpf_mem_cache * c)462 static void init_refill_work(struct bpf_mem_cache *c)
463 {
464 	init_irq_work(&c->refill_work, bpf_mem_refill);
465 	if (c->unit_size <= 256) {
466 		c->low_watermark = 32;
467 		c->high_watermark = 96;
468 	} else {
469 		/* When page_size == 4k, order-0 cache will have low_mark == 2
470 		 * and high_mark == 6 with batch alloc of 3 individual pages at
471 		 * a time.
472 		 * 8k allocs and above low == 1, high == 3, batch == 1.
473 		 */
474 		c->low_watermark = max(32 * 256 / c->unit_size, 1);
475 		c->high_watermark = max(96 * 256 / c->unit_size, 3);
476 	}
477 	c->batch = max((c->high_watermark - c->low_watermark) / 4 * 3, 1);
478 }
479 
prefill_mem_cache(struct bpf_mem_cache * c,int cpu)480 static void prefill_mem_cache(struct bpf_mem_cache *c, int cpu)
481 {
482 	/* To avoid consuming memory assume that 1st run of bpf
483 	 * prog won't be doing more than 4 map_update_elem from
484 	 * irq disabled region
485 	 */
486 	alloc_bulk(c, c->unit_size <= 256 ? 4 : 1, cpu_to_node(cpu), false);
487 }
488 
489 /* When size != 0 bpf_mem_cache for each cpu.
490  * This is typical bpf hash map use case when all elements have equal size.
491  *
492  * When size == 0 allocate 11 bpf_mem_cache-s for each cpu, then rely on
493  * kmalloc/kfree. Max allocation size is 4096 in this case.
494  * This is bpf_dynptr and bpf_kptr use case.
495  */
bpf_mem_alloc_init(struct bpf_mem_alloc * ma,int size,bool percpu)496 int bpf_mem_alloc_init(struct bpf_mem_alloc *ma, int size, bool percpu)
497 {
498 	static u16 sizes[NUM_CACHES] = {96, 192, 16, 32, 64, 128, 256, 512, 1024, 2048, 4096};
499 	struct bpf_mem_caches *cc, __percpu *pcc;
500 	struct bpf_mem_cache *c, __percpu *pc;
501 	struct obj_cgroup *objcg = NULL;
502 	int cpu, i, unit_size, percpu_size = 0;
503 
504 	ma->percpu = percpu;
505 
506 	if (size) {
507 		pc = __alloc_percpu_gfp(sizeof(*pc), 8, GFP_KERNEL);
508 		if (!pc)
509 			return -ENOMEM;
510 
511 		if (percpu)
512 			/* room for llist_node and per-cpu pointer */
513 			percpu_size = LLIST_NODE_SZ + sizeof(void *);
514 		else
515 			size += LLIST_NODE_SZ; /* room for llist_node */
516 		unit_size = size;
517 
518 #ifdef CONFIG_MEMCG_KMEM
519 		if (memcg_bpf_enabled())
520 			objcg = get_obj_cgroup_from_current();
521 #endif
522 		for_each_possible_cpu(cpu) {
523 			c = per_cpu_ptr(pc, cpu);
524 			c->unit_size = unit_size;
525 			c->objcg = objcg;
526 			c->percpu_size = percpu_size;
527 			c->tgt = c;
528 			init_refill_work(c);
529 			prefill_mem_cache(c, cpu);
530 		}
531 		ma->cache = pc;
532 		return 0;
533 	}
534 
535 	/* size == 0 && percpu is an invalid combination */
536 	if (WARN_ON_ONCE(percpu))
537 		return -EINVAL;
538 
539 	pcc = __alloc_percpu_gfp(sizeof(*cc), 8, GFP_KERNEL);
540 	if (!pcc)
541 		return -ENOMEM;
542 #ifdef CONFIG_MEMCG_KMEM
543 	objcg = get_obj_cgroup_from_current();
544 #endif
545 	for_each_possible_cpu(cpu) {
546 		cc = per_cpu_ptr(pcc, cpu);
547 		for (i = 0; i < NUM_CACHES; i++) {
548 			c = &cc->cache[i];
549 			c->unit_size = sizes[i];
550 			c->objcg = objcg;
551 			c->tgt = c;
552 
553 			init_refill_work(c);
554 			prefill_mem_cache(c, cpu);
555 		}
556 	}
557 
558 	ma->caches = pcc;
559 	return 0;
560 }
561 
drain_mem_cache(struct bpf_mem_cache * c)562 static void drain_mem_cache(struct bpf_mem_cache *c)
563 {
564 	bool percpu = !!c->percpu_size;
565 
566 	/* No progs are using this bpf_mem_cache, but htab_map_free() called
567 	 * bpf_mem_cache_free() for all remaining elements and they can be in
568 	 * free_by_rcu_ttrace or in waiting_for_gp_ttrace lists, so drain those lists now.
569 	 *
570 	 * Except for waiting_for_gp_ttrace list, there are no concurrent operations
571 	 * on these lists, so it is safe to use __llist_del_all().
572 	 */
573 	free_all(llist_del_all(&c->free_by_rcu_ttrace), percpu);
574 	free_all(llist_del_all(&c->waiting_for_gp_ttrace), percpu);
575 	free_all(__llist_del_all(&c->free_llist), percpu);
576 	free_all(__llist_del_all(&c->free_llist_extra), percpu);
577 	free_all(__llist_del_all(&c->free_by_rcu), percpu);
578 	free_all(__llist_del_all(&c->free_llist_extra_rcu), percpu);
579 	free_all(llist_del_all(&c->waiting_for_gp), percpu);
580 }
581 
check_mem_cache(struct bpf_mem_cache * c)582 static void check_mem_cache(struct bpf_mem_cache *c)
583 {
584 	WARN_ON_ONCE(!llist_empty(&c->free_by_rcu_ttrace));
585 	WARN_ON_ONCE(!llist_empty(&c->waiting_for_gp_ttrace));
586 	WARN_ON_ONCE(!llist_empty(&c->free_llist));
587 	WARN_ON_ONCE(!llist_empty(&c->free_llist_extra));
588 	WARN_ON_ONCE(!llist_empty(&c->free_by_rcu));
589 	WARN_ON_ONCE(!llist_empty(&c->free_llist_extra_rcu));
590 	WARN_ON_ONCE(!llist_empty(&c->waiting_for_gp));
591 }
592 
check_leaked_objs(struct bpf_mem_alloc * ma)593 static void check_leaked_objs(struct bpf_mem_alloc *ma)
594 {
595 	struct bpf_mem_caches *cc;
596 	struct bpf_mem_cache *c;
597 	int cpu, i;
598 
599 	if (ma->cache) {
600 		for_each_possible_cpu(cpu) {
601 			c = per_cpu_ptr(ma->cache, cpu);
602 			check_mem_cache(c);
603 		}
604 	}
605 	if (ma->caches) {
606 		for_each_possible_cpu(cpu) {
607 			cc = per_cpu_ptr(ma->caches, cpu);
608 			for (i = 0; i < NUM_CACHES; i++) {
609 				c = &cc->cache[i];
610 				check_mem_cache(c);
611 			}
612 		}
613 	}
614 }
615 
free_mem_alloc_no_barrier(struct bpf_mem_alloc * ma)616 static void free_mem_alloc_no_barrier(struct bpf_mem_alloc *ma)
617 {
618 	check_leaked_objs(ma);
619 	free_percpu(ma->cache);
620 	free_percpu(ma->caches);
621 	ma->cache = NULL;
622 	ma->caches = NULL;
623 }
624 
free_mem_alloc(struct bpf_mem_alloc * ma)625 static void free_mem_alloc(struct bpf_mem_alloc *ma)
626 {
627 	/* waiting_for_gp[_ttrace] lists were drained, but RCU callbacks
628 	 * might still execute. Wait for them.
629 	 *
630 	 * rcu_barrier_tasks_trace() doesn't imply synchronize_rcu_tasks_trace(),
631 	 * but rcu_barrier_tasks_trace() and rcu_barrier() below are only used
632 	 * to wait for the pending __free_rcu_tasks_trace() and __free_rcu(),
633 	 * so if call_rcu(head, __free_rcu) is skipped due to
634 	 * rcu_trace_implies_rcu_gp(), it will be OK to skip rcu_barrier() by
635 	 * using rcu_trace_implies_rcu_gp() as well.
636 	 */
637 	rcu_barrier(); /* wait for __free_by_rcu */
638 	rcu_barrier_tasks_trace(); /* wait for __free_rcu */
639 	if (!rcu_trace_implies_rcu_gp())
640 		rcu_barrier();
641 	free_mem_alloc_no_barrier(ma);
642 }
643 
free_mem_alloc_deferred(struct work_struct * work)644 static void free_mem_alloc_deferred(struct work_struct *work)
645 {
646 	struct bpf_mem_alloc *ma = container_of(work, struct bpf_mem_alloc, work);
647 
648 	free_mem_alloc(ma);
649 	kfree(ma);
650 }
651 
destroy_mem_alloc(struct bpf_mem_alloc * ma,int rcu_in_progress)652 static void destroy_mem_alloc(struct bpf_mem_alloc *ma, int rcu_in_progress)
653 {
654 	struct bpf_mem_alloc *copy;
655 
656 	if (!rcu_in_progress) {
657 		/* Fast path. No callbacks are pending, hence no need to do
658 		 * rcu_barrier-s.
659 		 */
660 		free_mem_alloc_no_barrier(ma);
661 		return;
662 	}
663 
664 	copy = kmemdup(ma, sizeof(*ma), GFP_KERNEL);
665 	if (!copy) {
666 		/* Slow path with inline barrier-s */
667 		free_mem_alloc(ma);
668 		return;
669 	}
670 
671 	/* Defer barriers into worker to let the rest of map memory to be freed */
672 	memset(ma, 0, sizeof(*ma));
673 	INIT_WORK(&copy->work, free_mem_alloc_deferred);
674 	queue_work(system_unbound_wq, &copy->work);
675 }
676 
bpf_mem_alloc_destroy(struct bpf_mem_alloc * ma)677 void bpf_mem_alloc_destroy(struct bpf_mem_alloc *ma)
678 {
679 	struct bpf_mem_caches *cc;
680 	struct bpf_mem_cache *c;
681 	int cpu, i, rcu_in_progress;
682 
683 	if (ma->cache) {
684 		rcu_in_progress = 0;
685 		for_each_possible_cpu(cpu) {
686 			c = per_cpu_ptr(ma->cache, cpu);
687 			WRITE_ONCE(c->draining, true);
688 			irq_work_sync(&c->refill_work);
689 			drain_mem_cache(c);
690 			rcu_in_progress += atomic_read(&c->call_rcu_ttrace_in_progress);
691 			rcu_in_progress += atomic_read(&c->call_rcu_in_progress);
692 		}
693 		/* objcg is the same across cpus */
694 		if (c->objcg)
695 			obj_cgroup_put(c->objcg);
696 		destroy_mem_alloc(ma, rcu_in_progress);
697 	}
698 	if (ma->caches) {
699 		rcu_in_progress = 0;
700 		for_each_possible_cpu(cpu) {
701 			cc = per_cpu_ptr(ma->caches, cpu);
702 			for (i = 0; i < NUM_CACHES; i++) {
703 				c = &cc->cache[i];
704 				WRITE_ONCE(c->draining, true);
705 				irq_work_sync(&c->refill_work);
706 				drain_mem_cache(c);
707 				rcu_in_progress += atomic_read(&c->call_rcu_ttrace_in_progress);
708 				rcu_in_progress += atomic_read(&c->call_rcu_in_progress);
709 			}
710 		}
711 		if (c->objcg)
712 			obj_cgroup_put(c->objcg);
713 		destroy_mem_alloc(ma, rcu_in_progress);
714 	}
715 }
716 
717 /* notrace is necessary here and in other functions to make sure
718  * bpf programs cannot attach to them and cause llist corruptions.
719  */
unit_alloc(struct bpf_mem_cache * c)720 static void notrace *unit_alloc(struct bpf_mem_cache *c)
721 {
722 	struct llist_node *llnode = NULL;
723 	unsigned long flags;
724 	int cnt = 0;
725 
726 	/* Disable irqs to prevent the following race for majority of prog types:
727 	 * prog_A
728 	 *   bpf_mem_alloc
729 	 *      preemption or irq -> prog_B
730 	 *        bpf_mem_alloc
731 	 *
732 	 * but prog_B could be a perf_event NMI prog.
733 	 * Use per-cpu 'active' counter to order free_list access between
734 	 * unit_alloc/unit_free/bpf_mem_refill.
735 	 */
736 	local_irq_save(flags);
737 	if (local_inc_return(&c->active) == 1) {
738 		llnode = __llist_del_first(&c->free_llist);
739 		if (llnode) {
740 			cnt = --c->free_cnt;
741 			*(struct bpf_mem_cache **)llnode = c;
742 		}
743 	}
744 	local_dec(&c->active);
745 	local_irq_restore(flags);
746 
747 	WARN_ON(cnt < 0);
748 
749 	if (cnt < c->low_watermark)
750 		irq_work_raise(c);
751 	return llnode;
752 }
753 
754 /* Though 'ptr' object could have been allocated on a different cpu
755  * add it to the free_llist of the current cpu.
756  * Let kfree() logic deal with it when it's later called from irq_work.
757  */
unit_free(struct bpf_mem_cache * c,void * ptr)758 static void notrace unit_free(struct bpf_mem_cache *c, void *ptr)
759 {
760 	struct llist_node *llnode = ptr - LLIST_NODE_SZ;
761 	unsigned long flags;
762 	int cnt = 0;
763 
764 	BUILD_BUG_ON(LLIST_NODE_SZ > 8);
765 
766 	/*
767 	 * Remember bpf_mem_cache that allocated this object.
768 	 * The hint is not accurate.
769 	 */
770 	c->tgt = *(struct bpf_mem_cache **)llnode;
771 
772 	local_irq_save(flags);
773 	if (local_inc_return(&c->active) == 1) {
774 		__llist_add(llnode, &c->free_llist);
775 		cnt = ++c->free_cnt;
776 	} else {
777 		/* unit_free() cannot fail. Therefore add an object to atomic
778 		 * llist. free_bulk() will drain it. Though free_llist_extra is
779 		 * a per-cpu list we have to use atomic llist_add here, since
780 		 * it also can be interrupted by bpf nmi prog that does another
781 		 * unit_free() into the same free_llist_extra.
782 		 */
783 		llist_add(llnode, &c->free_llist_extra);
784 	}
785 	local_dec(&c->active);
786 	local_irq_restore(flags);
787 
788 	if (cnt > c->high_watermark)
789 		/* free few objects from current cpu into global kmalloc pool */
790 		irq_work_raise(c);
791 }
792 
unit_free_rcu(struct bpf_mem_cache * c,void * ptr)793 static void notrace unit_free_rcu(struct bpf_mem_cache *c, void *ptr)
794 {
795 	struct llist_node *llnode = ptr - LLIST_NODE_SZ;
796 	unsigned long flags;
797 
798 	c->tgt = *(struct bpf_mem_cache **)llnode;
799 
800 	local_irq_save(flags);
801 	if (local_inc_return(&c->active) == 1) {
802 		if (__llist_add(llnode, &c->free_by_rcu))
803 			c->free_by_rcu_tail = llnode;
804 	} else {
805 		llist_add(llnode, &c->free_llist_extra_rcu);
806 	}
807 	local_dec(&c->active);
808 	local_irq_restore(flags);
809 
810 	if (!atomic_read(&c->call_rcu_in_progress))
811 		irq_work_raise(c);
812 }
813 
814 /* Called from BPF program or from sys_bpf syscall.
815  * In both cases migration is disabled.
816  */
bpf_mem_alloc(struct bpf_mem_alloc * ma,size_t size)817 void notrace *bpf_mem_alloc(struct bpf_mem_alloc *ma, size_t size)
818 {
819 	int idx;
820 	void *ret;
821 
822 	if (!size)
823 		return NULL;
824 
825 	idx = bpf_mem_cache_idx(size + LLIST_NODE_SZ);
826 	if (idx < 0)
827 		return NULL;
828 
829 	ret = unit_alloc(this_cpu_ptr(ma->caches)->cache + idx);
830 	return !ret ? NULL : ret + LLIST_NODE_SZ;
831 }
832 
bpf_mem_free(struct bpf_mem_alloc * ma,void * ptr)833 void notrace bpf_mem_free(struct bpf_mem_alloc *ma, void *ptr)
834 {
835 	struct bpf_mem_cache *c;
836 	int idx;
837 
838 	if (!ptr)
839 		return;
840 
841 	c = *(void **)(ptr - LLIST_NODE_SZ);
842 	idx = bpf_mem_cache_idx(c->unit_size);
843 	if (WARN_ON_ONCE(idx < 0))
844 		return;
845 
846 	unit_free(this_cpu_ptr(ma->caches)->cache + idx, ptr);
847 }
848 
bpf_mem_free_rcu(struct bpf_mem_alloc * ma,void * ptr)849 void notrace bpf_mem_free_rcu(struct bpf_mem_alloc *ma, void *ptr)
850 {
851 	struct bpf_mem_cache *c;
852 	int idx;
853 
854 	if (!ptr)
855 		return;
856 
857 	c = *(void **)(ptr - LLIST_NODE_SZ);
858 	idx = bpf_mem_cache_idx(c->unit_size);
859 	if (WARN_ON_ONCE(idx < 0))
860 		return;
861 
862 	unit_free_rcu(this_cpu_ptr(ma->caches)->cache + idx, ptr);
863 }
864 
bpf_mem_cache_alloc(struct bpf_mem_alloc * ma)865 void notrace *bpf_mem_cache_alloc(struct bpf_mem_alloc *ma)
866 {
867 	void *ret;
868 
869 	ret = unit_alloc(this_cpu_ptr(ma->cache));
870 	return !ret ? NULL : ret + LLIST_NODE_SZ;
871 }
872 
bpf_mem_cache_free(struct bpf_mem_alloc * ma,void * ptr)873 void notrace bpf_mem_cache_free(struct bpf_mem_alloc *ma, void *ptr)
874 {
875 	if (!ptr)
876 		return;
877 
878 	unit_free(this_cpu_ptr(ma->cache), ptr);
879 }
880 
bpf_mem_cache_free_rcu(struct bpf_mem_alloc * ma,void * ptr)881 void notrace bpf_mem_cache_free_rcu(struct bpf_mem_alloc *ma, void *ptr)
882 {
883 	if (!ptr)
884 		return;
885 
886 	unit_free_rcu(this_cpu_ptr(ma->cache), ptr);
887 }
888 
889 /* Directly does a kfree() without putting 'ptr' back to the free_llist
890  * for reuse and without waiting for a rcu_tasks_trace gp.
891  * The caller must first go through the rcu_tasks_trace gp for 'ptr'
892  * before calling bpf_mem_cache_raw_free().
893  * It could be used when the rcu_tasks_trace callback does not have
894  * a hold on the original bpf_mem_alloc object that allocated the
895  * 'ptr'. This should only be used in the uncommon code path.
896  * Otherwise, the bpf_mem_alloc's free_llist cannot be refilled
897  * and may affect performance.
898  */
bpf_mem_cache_raw_free(void * ptr)899 void bpf_mem_cache_raw_free(void *ptr)
900 {
901 	if (!ptr)
902 		return;
903 
904 	kfree(ptr - LLIST_NODE_SZ);
905 }
906 
907 /* When flags == GFP_KERNEL, it signals that the caller will not cause
908  * deadlock when using kmalloc. bpf_mem_cache_alloc_flags() will use
909  * kmalloc if the free_llist is empty.
910  */
bpf_mem_cache_alloc_flags(struct bpf_mem_alloc * ma,gfp_t flags)911 void notrace *bpf_mem_cache_alloc_flags(struct bpf_mem_alloc *ma, gfp_t flags)
912 {
913 	struct bpf_mem_cache *c;
914 	void *ret;
915 
916 	c = this_cpu_ptr(ma->cache);
917 
918 	ret = unit_alloc(c);
919 	if (!ret && flags == GFP_KERNEL) {
920 		struct mem_cgroup *memcg, *old_memcg;
921 
922 		memcg = get_memcg(c);
923 		old_memcg = set_active_memcg(memcg);
924 		ret = __alloc(c, NUMA_NO_NODE, GFP_KERNEL | __GFP_NOWARN | __GFP_ACCOUNT);
925 		if (ret)
926 			*(struct bpf_mem_cache **)ret = c;
927 		set_active_memcg(old_memcg);
928 		mem_cgroup_put(memcg);
929 	}
930 
931 	return !ret ? NULL : ret + LLIST_NODE_SZ;
932 }
933