17c8199e2SAlexei Starovoitov // SPDX-License-Identifier: GPL-2.0-only 27c8199e2SAlexei Starovoitov /* Copyright (c) 2022 Meta Platforms, Inc. and affiliates. */ 37c8199e2SAlexei Starovoitov #include <linux/mm.h> 47c8199e2SAlexei Starovoitov #include <linux/llist.h> 57c8199e2SAlexei Starovoitov #include <linux/bpf.h> 67c8199e2SAlexei Starovoitov #include <linux/irq_work.h> 77c8199e2SAlexei Starovoitov #include <linux/bpf_mem_alloc.h> 87c8199e2SAlexei Starovoitov #include <linux/memcontrol.h> 97c8199e2SAlexei Starovoitov #include <asm/local.h> 107c8199e2SAlexei Starovoitov 117c8199e2SAlexei Starovoitov /* Any context (including NMI) BPF specific memory allocator. 127c8199e2SAlexei Starovoitov * 137c8199e2SAlexei Starovoitov * Tracing BPF programs can attach to kprobe and fentry. Hence they 147c8199e2SAlexei Starovoitov * run in unknown context where calling plain kmalloc() might not be safe. 157c8199e2SAlexei Starovoitov * 167c8199e2SAlexei Starovoitov * Front-end kmalloc() with per-cpu per-bucket cache of free elements. 177c8199e2SAlexei Starovoitov * Refill this cache asynchronously from irq_work. 187c8199e2SAlexei Starovoitov * 197c8199e2SAlexei Starovoitov * CPU_0 buckets 207c8199e2SAlexei Starovoitov * 16 32 64 96 128 196 256 512 1024 2048 4096 217c8199e2SAlexei Starovoitov * ... 227c8199e2SAlexei Starovoitov * CPU_N buckets 237c8199e2SAlexei Starovoitov * 16 32 64 96 128 196 256 512 1024 2048 4096 247c8199e2SAlexei Starovoitov * 257c8199e2SAlexei Starovoitov * The buckets are prefilled at the start. 267c8199e2SAlexei Starovoitov * BPF programs always run with migration disabled. 277c8199e2SAlexei Starovoitov * It's safe to allocate from cache of the current cpu with irqs disabled. 287c8199e2SAlexei Starovoitov * Free-ing is always done into bucket of the current cpu as well. 297c8199e2SAlexei Starovoitov * irq_work trims extra free elements from buckets with kfree 307c8199e2SAlexei Starovoitov * and refills them with kmalloc, so global kmalloc logic takes care 317c8199e2SAlexei Starovoitov * of freeing objects allocated by one cpu and freed on another. 327c8199e2SAlexei Starovoitov * 337c8199e2SAlexei Starovoitov * Every allocated objected is padded with extra 8 bytes that contains 347c8199e2SAlexei Starovoitov * struct llist_node. 357c8199e2SAlexei Starovoitov */ 367c8199e2SAlexei Starovoitov #define LLIST_NODE_SZ sizeof(struct llist_node) 377c8199e2SAlexei Starovoitov 387c8199e2SAlexei Starovoitov /* similar to kmalloc, but sizeof == 8 bucket is gone */ 397c8199e2SAlexei Starovoitov static u8 size_index[24] __ro_after_init = { 407c8199e2SAlexei Starovoitov 3, /* 8 */ 417c8199e2SAlexei Starovoitov 3, /* 16 */ 427c8199e2SAlexei Starovoitov 4, /* 24 */ 437c8199e2SAlexei Starovoitov 4, /* 32 */ 447c8199e2SAlexei Starovoitov 5, /* 40 */ 457c8199e2SAlexei Starovoitov 5, /* 48 */ 467c8199e2SAlexei Starovoitov 5, /* 56 */ 477c8199e2SAlexei Starovoitov 5, /* 64 */ 487c8199e2SAlexei Starovoitov 1, /* 72 */ 497c8199e2SAlexei Starovoitov 1, /* 80 */ 507c8199e2SAlexei Starovoitov 1, /* 88 */ 517c8199e2SAlexei Starovoitov 1, /* 96 */ 527c8199e2SAlexei Starovoitov 6, /* 104 */ 537c8199e2SAlexei Starovoitov 6, /* 112 */ 547c8199e2SAlexei Starovoitov 6, /* 120 */ 557c8199e2SAlexei Starovoitov 6, /* 128 */ 567c8199e2SAlexei Starovoitov 2, /* 136 */ 577c8199e2SAlexei Starovoitov 2, /* 144 */ 587c8199e2SAlexei Starovoitov 2, /* 152 */ 597c8199e2SAlexei Starovoitov 2, /* 160 */ 607c8199e2SAlexei Starovoitov 2, /* 168 */ 617c8199e2SAlexei Starovoitov 2, /* 176 */ 627c8199e2SAlexei Starovoitov 2, /* 184 */ 637c8199e2SAlexei Starovoitov 2 /* 192 */ 647c8199e2SAlexei Starovoitov }; 657c8199e2SAlexei Starovoitov 667c8199e2SAlexei Starovoitov static int bpf_mem_cache_idx(size_t size) 677c8199e2SAlexei Starovoitov { 687c8199e2SAlexei Starovoitov if (!size || size > 4096) 697c8199e2SAlexei Starovoitov return -1; 707c8199e2SAlexei Starovoitov 717c8199e2SAlexei Starovoitov if (size <= 192) 727c8199e2SAlexei Starovoitov return size_index[(size - 1) / 8] - 1; 737c8199e2SAlexei Starovoitov 7436024d02SHou Tao return fls(size - 1) - 2; 757c8199e2SAlexei Starovoitov } 767c8199e2SAlexei Starovoitov 777c8199e2SAlexei Starovoitov #define NUM_CACHES 11 787c8199e2SAlexei Starovoitov 797c8199e2SAlexei Starovoitov struct bpf_mem_cache { 807c8199e2SAlexei Starovoitov /* per-cpu list of free objects of size 'unit_size'. 817c8199e2SAlexei Starovoitov * All accesses are done with interrupts disabled and 'active' counter 827c8199e2SAlexei Starovoitov * protection with __llist_add() and __llist_del_first(). 837c8199e2SAlexei Starovoitov */ 847c8199e2SAlexei Starovoitov struct llist_head free_llist; 857c8199e2SAlexei Starovoitov local_t active; 867c8199e2SAlexei Starovoitov 877c8199e2SAlexei Starovoitov /* Operations on the free_list from unit_alloc/unit_free/bpf_mem_refill 887c8199e2SAlexei Starovoitov * are sequenced by per-cpu 'active' counter. But unit_free() cannot 897c8199e2SAlexei Starovoitov * fail. When 'active' is busy the unit_free() will add an object to 907c8199e2SAlexei Starovoitov * free_llist_extra. 917c8199e2SAlexei Starovoitov */ 927c8199e2SAlexei Starovoitov struct llist_head free_llist_extra; 937c8199e2SAlexei Starovoitov 947c8199e2SAlexei Starovoitov struct irq_work refill_work; 957c8199e2SAlexei Starovoitov struct obj_cgroup *objcg; 967c8199e2SAlexei Starovoitov int unit_size; 977c8199e2SAlexei Starovoitov /* count of objects in free_llist */ 987c8199e2SAlexei Starovoitov int free_cnt; 997c266178SAlexei Starovoitov int low_watermark, high_watermark, batch; 100bfc03c15SAlexei Starovoitov int percpu_size; 1018d5a8011SAlexei Starovoitov 10212c8d0f4SAlexei Starovoitov /* list of objects to be freed after RCU tasks trace GP */ 10312c8d0f4SAlexei Starovoitov struct llist_head free_by_rcu_ttrace; 10412c8d0f4SAlexei Starovoitov struct llist_head waiting_for_gp_ttrace; 10512c8d0f4SAlexei Starovoitov struct rcu_head rcu_ttrace; 10612c8d0f4SAlexei Starovoitov atomic_t call_rcu_ttrace_in_progress; 1077c8199e2SAlexei Starovoitov }; 1087c8199e2SAlexei Starovoitov 1097c8199e2SAlexei Starovoitov struct bpf_mem_caches { 1107c8199e2SAlexei Starovoitov struct bpf_mem_cache cache[NUM_CACHES]; 1117c8199e2SAlexei Starovoitov }; 1127c8199e2SAlexei Starovoitov 1137c8199e2SAlexei Starovoitov static struct llist_node notrace *__llist_del_first(struct llist_head *head) 1147c8199e2SAlexei Starovoitov { 1157c8199e2SAlexei Starovoitov struct llist_node *entry, *next; 1167c8199e2SAlexei Starovoitov 1177c8199e2SAlexei Starovoitov entry = head->first; 1187c8199e2SAlexei Starovoitov if (!entry) 1197c8199e2SAlexei Starovoitov return NULL; 1207c8199e2SAlexei Starovoitov next = entry->next; 1217c8199e2SAlexei Starovoitov head->first = next; 1227c8199e2SAlexei Starovoitov return entry; 1237c8199e2SAlexei Starovoitov } 1247c8199e2SAlexei Starovoitov 125e65a5c6eSMartin KaFai Lau static void *__alloc(struct bpf_mem_cache *c, int node, gfp_t flags) 1267c8199e2SAlexei Starovoitov { 127bfc03c15SAlexei Starovoitov if (c->percpu_size) { 128bfc03c15SAlexei Starovoitov void **obj = kmalloc_node(c->percpu_size, flags, node); 1294ab67149SAlexei Starovoitov void *pptr = __alloc_percpu_gfp(c->unit_size, 8, flags); 1304ab67149SAlexei Starovoitov 1314ab67149SAlexei Starovoitov if (!obj || !pptr) { 1324ab67149SAlexei Starovoitov free_percpu(pptr); 1334ab67149SAlexei Starovoitov kfree(obj); 1344ab67149SAlexei Starovoitov return NULL; 1354ab67149SAlexei Starovoitov } 1364ab67149SAlexei Starovoitov obj[1] = pptr; 1374ab67149SAlexei Starovoitov return obj; 1384ab67149SAlexei Starovoitov } 1394ab67149SAlexei Starovoitov 140997849c4SHou Tao return kmalloc_node(c->unit_size, flags | __GFP_ZERO, node); 1417c8199e2SAlexei Starovoitov } 1427c8199e2SAlexei Starovoitov 1437c8199e2SAlexei Starovoitov static struct mem_cgroup *get_memcg(const struct bpf_mem_cache *c) 1447c8199e2SAlexei Starovoitov { 1457c8199e2SAlexei Starovoitov #ifdef CONFIG_MEMCG_KMEM 1467c8199e2SAlexei Starovoitov if (c->objcg) 1477c8199e2SAlexei Starovoitov return get_mem_cgroup_from_objcg(c->objcg); 1487c8199e2SAlexei Starovoitov #endif 1497c8199e2SAlexei Starovoitov 1507c8199e2SAlexei Starovoitov #ifdef CONFIG_MEMCG 1517c8199e2SAlexei Starovoitov return root_mem_cgroup; 1527c8199e2SAlexei Starovoitov #else 1537c8199e2SAlexei Starovoitov return NULL; 1547c8199e2SAlexei Starovoitov #endif 1557c8199e2SAlexei Starovoitov } 1567c8199e2SAlexei Starovoitov 1577c8199e2SAlexei Starovoitov /* Mostly runs from irq_work except __init phase. */ 1587c8199e2SAlexei Starovoitov static void alloc_bulk(struct bpf_mem_cache *c, int cnt, int node) 1597c8199e2SAlexei Starovoitov { 1607c8199e2SAlexei Starovoitov struct mem_cgroup *memcg = NULL, *old_memcg; 1617c8199e2SAlexei Starovoitov unsigned long flags; 1627c8199e2SAlexei Starovoitov void *obj; 1637c8199e2SAlexei Starovoitov int i; 1647c8199e2SAlexei Starovoitov 1657c8199e2SAlexei Starovoitov memcg = get_memcg(c); 1667c8199e2SAlexei Starovoitov old_memcg = set_active_memcg(memcg); 1677c8199e2SAlexei Starovoitov for (i = 0; i < cnt; i++) { 1680893d600SHou Tao /* 16912c8d0f4SAlexei Starovoitov * free_by_rcu_ttrace is only manipulated by irq work refill_work(). 1700893d600SHou Tao * IRQ works on the same CPU are called sequentially, so it is 1710893d600SHou Tao * safe to use __llist_del_first() here. If alloc_bulk() is 1720893d600SHou Tao * invoked by the initial prefill, there will be no running 1730893d600SHou Tao * refill_work(), so __llist_del_first() is fine as well. 1740893d600SHou Tao * 17512c8d0f4SAlexei Starovoitov * In most cases, objects on free_by_rcu_ttrace are from the same CPU. 1760893d600SHou Tao * If some objects come from other CPUs, it doesn't incur any 1770893d600SHou Tao * harm because NUMA_NO_NODE means the preference for current 1780893d600SHou Tao * numa node and it is not a guarantee. 1790893d600SHou Tao */ 18012c8d0f4SAlexei Starovoitov obj = __llist_del_first(&c->free_by_rcu_ttrace); 1810893d600SHou Tao if (!obj) { 182e65a5c6eSMartin KaFai Lau /* Allocate, but don't deplete atomic reserves that typical 183e65a5c6eSMartin KaFai Lau * GFP_ATOMIC would do. irq_work runs on this cpu and kmalloc 184e65a5c6eSMartin KaFai Lau * will allocate from the current numa node which is what we 185e65a5c6eSMartin KaFai Lau * want here. 186e65a5c6eSMartin KaFai Lau */ 187e65a5c6eSMartin KaFai Lau obj = __alloc(c, node, GFP_NOWAIT | __GFP_NOWARN | __GFP_ACCOUNT); 1887c8199e2SAlexei Starovoitov if (!obj) 1897c8199e2SAlexei Starovoitov break; 1900893d600SHou Tao } 1917c8199e2SAlexei Starovoitov if (IS_ENABLED(CONFIG_PREEMPT_RT)) 1927c8199e2SAlexei Starovoitov /* In RT irq_work runs in per-cpu kthread, so disable 1937c8199e2SAlexei Starovoitov * interrupts to avoid preemption and interrupts and 1947c8199e2SAlexei Starovoitov * reduce the chance of bpf prog executing on this cpu 1957c8199e2SAlexei Starovoitov * when active counter is busy. 1967c8199e2SAlexei Starovoitov */ 1977c8199e2SAlexei Starovoitov local_irq_save(flags); 1987c8199e2SAlexei Starovoitov /* alloc_bulk runs from irq_work which will not preempt a bpf 1997c8199e2SAlexei Starovoitov * program that does unit_alloc/unit_free since IRQs are 2007c8199e2SAlexei Starovoitov * disabled there. There is no race to increment 'active' 2017c8199e2SAlexei Starovoitov * counter. It protects free_llist from corruption in case NMI 2027c8199e2SAlexei Starovoitov * bpf prog preempted this loop. 2037c8199e2SAlexei Starovoitov */ 2047c8199e2SAlexei Starovoitov WARN_ON_ONCE(local_inc_return(&c->active) != 1); 2057c8199e2SAlexei Starovoitov __llist_add(obj, &c->free_llist); 2067c8199e2SAlexei Starovoitov c->free_cnt++; 2077c8199e2SAlexei Starovoitov local_dec(&c->active); 2087c8199e2SAlexei Starovoitov if (IS_ENABLED(CONFIG_PREEMPT_RT)) 2097c8199e2SAlexei Starovoitov local_irq_restore(flags); 2107c8199e2SAlexei Starovoitov } 2117c8199e2SAlexei Starovoitov set_active_memcg(old_memcg); 2127c8199e2SAlexei Starovoitov mem_cgroup_put(memcg); 2137c8199e2SAlexei Starovoitov } 2147c8199e2SAlexei Starovoitov 215aa7881fcSHou Tao static void free_one(void *obj, bool percpu) 2167c8199e2SAlexei Starovoitov { 217aa7881fcSHou Tao if (percpu) { 2184ab67149SAlexei Starovoitov free_percpu(((void **)obj)[1]); 219bfc03c15SAlexei Starovoitov kfree(obj); 2204ab67149SAlexei Starovoitov return; 2214ab67149SAlexei Starovoitov } 2224ab67149SAlexei Starovoitov 2237c8199e2SAlexei Starovoitov kfree(obj); 2247c8199e2SAlexei Starovoitov } 2257c8199e2SAlexei Starovoitov 226*9de3e815SAlexei Starovoitov static int free_all(struct llist_node *llnode, bool percpu) 2278d5a8011SAlexei Starovoitov { 2288d5a8011SAlexei Starovoitov struct llist_node *pos, *t; 229*9de3e815SAlexei Starovoitov int cnt = 0; 2308d5a8011SAlexei Starovoitov 231*9de3e815SAlexei Starovoitov llist_for_each_safe(pos, t, llnode) { 232aa7881fcSHou Tao free_one(pos, percpu); 233*9de3e815SAlexei Starovoitov cnt++; 234*9de3e815SAlexei Starovoitov } 235*9de3e815SAlexei Starovoitov return cnt; 236aa7881fcSHou Tao } 237aa7881fcSHou Tao 238aa7881fcSHou Tao static void __free_rcu(struct rcu_head *head) 239aa7881fcSHou Tao { 24012c8d0f4SAlexei Starovoitov struct bpf_mem_cache *c = container_of(head, struct bpf_mem_cache, rcu_ttrace); 241aa7881fcSHou Tao 24212c8d0f4SAlexei Starovoitov free_all(llist_del_all(&c->waiting_for_gp_ttrace), !!c->percpu_size); 24312c8d0f4SAlexei Starovoitov atomic_set(&c->call_rcu_ttrace_in_progress, 0); 2448d5a8011SAlexei Starovoitov } 2458d5a8011SAlexei Starovoitov 246dccb4a90SAlexei Starovoitov static void __free_rcu_tasks_trace(struct rcu_head *head) 247dccb4a90SAlexei Starovoitov { 24859be91e5SHou Tao /* If RCU Tasks Trace grace period implies RCU grace period, 24959be91e5SHou Tao * there is no need to invoke call_rcu(). 25059be91e5SHou Tao */ 25159be91e5SHou Tao if (rcu_trace_implies_rcu_gp()) 25259be91e5SHou Tao __free_rcu(head); 25359be91e5SHou Tao else 25459be91e5SHou Tao call_rcu(head, __free_rcu); 255dccb4a90SAlexei Starovoitov } 256dccb4a90SAlexei Starovoitov 2578d5a8011SAlexei Starovoitov static void enque_to_free(struct bpf_mem_cache *c, void *obj) 2588d5a8011SAlexei Starovoitov { 2598d5a8011SAlexei Starovoitov struct llist_node *llnode = obj; 2608d5a8011SAlexei Starovoitov 2618d5a8011SAlexei Starovoitov /* bpf_mem_cache is a per-cpu object. Freeing happens in irq_work. 26212c8d0f4SAlexei Starovoitov * Nothing races to add to free_by_rcu_ttrace list. 2638d5a8011SAlexei Starovoitov */ 26412c8d0f4SAlexei Starovoitov __llist_add(llnode, &c->free_by_rcu_ttrace); 2658d5a8011SAlexei Starovoitov } 2668d5a8011SAlexei Starovoitov 26712c8d0f4SAlexei Starovoitov static void do_call_rcu_ttrace(struct bpf_mem_cache *c) 2688d5a8011SAlexei Starovoitov { 2698d5a8011SAlexei Starovoitov struct llist_node *llnode, *t; 2708d5a8011SAlexei Starovoitov 27112c8d0f4SAlexei Starovoitov if (atomic_xchg(&c->call_rcu_ttrace_in_progress, 1)) 2728d5a8011SAlexei Starovoitov return; 2738d5a8011SAlexei Starovoitov 27412c8d0f4SAlexei Starovoitov WARN_ON_ONCE(!llist_empty(&c->waiting_for_gp_ttrace)); 27512c8d0f4SAlexei Starovoitov llist_for_each_safe(llnode, t, __llist_del_all(&c->free_by_rcu_ttrace)) 27612c8d0f4SAlexei Starovoitov /* There is no concurrent __llist_add(waiting_for_gp_ttrace) access. 2778d5a8011SAlexei Starovoitov * It doesn't race with llist_del_all either. 27812c8d0f4SAlexei Starovoitov * But there could be two concurrent llist_del_all(waiting_for_gp_ttrace): 2798d5a8011SAlexei Starovoitov * from __free_rcu() and from drain_mem_cache(). 2808d5a8011SAlexei Starovoitov */ 28112c8d0f4SAlexei Starovoitov __llist_add(llnode, &c->waiting_for_gp_ttrace); 282dccb4a90SAlexei Starovoitov /* Use call_rcu_tasks_trace() to wait for sleepable progs to finish. 28359be91e5SHou Tao * If RCU Tasks Trace grace period implies RCU grace period, free 28459be91e5SHou Tao * these elements directly, else use call_rcu() to wait for normal 28559be91e5SHou Tao * progs to finish and finally do free_one() on each element. 286dccb4a90SAlexei Starovoitov */ 28712c8d0f4SAlexei Starovoitov call_rcu_tasks_trace(&c->rcu_ttrace, __free_rcu_tasks_trace); 2888d5a8011SAlexei Starovoitov } 2898d5a8011SAlexei Starovoitov 2907c8199e2SAlexei Starovoitov static void free_bulk(struct bpf_mem_cache *c) 2917c8199e2SAlexei Starovoitov { 2927c8199e2SAlexei Starovoitov struct llist_node *llnode, *t; 2937c8199e2SAlexei Starovoitov unsigned long flags; 2947c8199e2SAlexei Starovoitov int cnt; 2957c8199e2SAlexei Starovoitov 2967c8199e2SAlexei Starovoitov do { 2977c8199e2SAlexei Starovoitov if (IS_ENABLED(CONFIG_PREEMPT_RT)) 2987c8199e2SAlexei Starovoitov local_irq_save(flags); 2997c8199e2SAlexei Starovoitov WARN_ON_ONCE(local_inc_return(&c->active) != 1); 3007c8199e2SAlexei Starovoitov llnode = __llist_del_first(&c->free_llist); 3017c8199e2SAlexei Starovoitov if (llnode) 3027c8199e2SAlexei Starovoitov cnt = --c->free_cnt; 3037c8199e2SAlexei Starovoitov else 3047c8199e2SAlexei Starovoitov cnt = 0; 3057c8199e2SAlexei Starovoitov local_dec(&c->active); 3067c8199e2SAlexei Starovoitov if (IS_ENABLED(CONFIG_PREEMPT_RT)) 3077c8199e2SAlexei Starovoitov local_irq_restore(flags); 308c31b38cbSHou Tao if (llnode) 3098d5a8011SAlexei Starovoitov enque_to_free(c, llnode); 3107c266178SAlexei Starovoitov } while (cnt > (c->high_watermark + c->low_watermark) / 2); 3117c8199e2SAlexei Starovoitov 3127c8199e2SAlexei Starovoitov /* and drain free_llist_extra */ 3137c8199e2SAlexei Starovoitov llist_for_each_safe(llnode, t, llist_del_all(&c->free_llist_extra)) 3148d5a8011SAlexei Starovoitov enque_to_free(c, llnode); 31512c8d0f4SAlexei Starovoitov do_call_rcu_ttrace(c); 3167c8199e2SAlexei Starovoitov } 3177c8199e2SAlexei Starovoitov 3187c8199e2SAlexei Starovoitov static void bpf_mem_refill(struct irq_work *work) 3197c8199e2SAlexei Starovoitov { 3207c8199e2SAlexei Starovoitov struct bpf_mem_cache *c = container_of(work, struct bpf_mem_cache, refill_work); 3217c8199e2SAlexei Starovoitov int cnt; 3227c8199e2SAlexei Starovoitov 3237c8199e2SAlexei Starovoitov /* Racy access to free_cnt. It doesn't need to be 100% accurate */ 3247c8199e2SAlexei Starovoitov cnt = c->free_cnt; 3257c266178SAlexei Starovoitov if (cnt < c->low_watermark) 3267c8199e2SAlexei Starovoitov /* irq_work runs on this cpu and kmalloc will allocate 3277c8199e2SAlexei Starovoitov * from the current numa node which is what we want here. 3287c8199e2SAlexei Starovoitov */ 3297c266178SAlexei Starovoitov alloc_bulk(c, c->batch, NUMA_NO_NODE); 3307c266178SAlexei Starovoitov else if (cnt > c->high_watermark) 3317c8199e2SAlexei Starovoitov free_bulk(c); 3327c8199e2SAlexei Starovoitov } 3337c8199e2SAlexei Starovoitov 3347c8199e2SAlexei Starovoitov static void notrace irq_work_raise(struct bpf_mem_cache *c) 3357c8199e2SAlexei Starovoitov { 3367c8199e2SAlexei Starovoitov irq_work_queue(&c->refill_work); 3377c8199e2SAlexei Starovoitov } 3387c8199e2SAlexei Starovoitov 3397c266178SAlexei Starovoitov /* For typical bpf map case that uses bpf_mem_cache_alloc and single bucket 3407c266178SAlexei Starovoitov * the freelist cache will be elem_size * 64 (or less) on each cpu. 3417c266178SAlexei Starovoitov * 3427c266178SAlexei Starovoitov * For bpf programs that don't have statically known allocation sizes and 3437c266178SAlexei Starovoitov * assuming (low_mark + high_mark) / 2 as an average number of elements per 3447c266178SAlexei Starovoitov * bucket and all buckets are used the total amount of memory in freelists 3457c266178SAlexei Starovoitov * on each cpu will be: 3467c266178SAlexei Starovoitov * 64*16 + 64*32 + 64*64 + 64*96 + 64*128 + 64*196 + 64*256 + 32*512 + 16*1024 + 8*2048 + 4*4096 3477c266178SAlexei Starovoitov * == ~ 116 Kbyte using below heuristic. 3487c266178SAlexei Starovoitov * Initialized, but unused bpf allocator (not bpf map specific one) will 3497c266178SAlexei Starovoitov * consume ~ 11 Kbyte per cpu. 3507c266178SAlexei Starovoitov * Typical case will be between 11K and 116K closer to 11K. 3517c266178SAlexei Starovoitov * bpf progs can and should share bpf_mem_cache when possible. 3527c266178SAlexei Starovoitov */ 3537c266178SAlexei Starovoitov 3547c8199e2SAlexei Starovoitov static void prefill_mem_cache(struct bpf_mem_cache *c, int cpu) 3557c8199e2SAlexei Starovoitov { 3567c8199e2SAlexei Starovoitov init_irq_work(&c->refill_work, bpf_mem_refill); 3577c266178SAlexei Starovoitov if (c->unit_size <= 256) { 3587c266178SAlexei Starovoitov c->low_watermark = 32; 3597c266178SAlexei Starovoitov c->high_watermark = 96; 3607c266178SAlexei Starovoitov } else { 3617c266178SAlexei Starovoitov /* When page_size == 4k, order-0 cache will have low_mark == 2 3627c266178SAlexei Starovoitov * and high_mark == 6 with batch alloc of 3 individual pages at 3637c266178SAlexei Starovoitov * a time. 3647c266178SAlexei Starovoitov * 8k allocs and above low == 1, high == 3, batch == 1. 3657c266178SAlexei Starovoitov */ 3667c266178SAlexei Starovoitov c->low_watermark = max(32 * 256 / c->unit_size, 1); 3677c266178SAlexei Starovoitov c->high_watermark = max(96 * 256 / c->unit_size, 3); 3687c266178SAlexei Starovoitov } 3697c266178SAlexei Starovoitov c->batch = max((c->high_watermark - c->low_watermark) / 4 * 3, 1); 3707c266178SAlexei Starovoitov 3717c8199e2SAlexei Starovoitov /* To avoid consuming memory assume that 1st run of bpf 3727c8199e2SAlexei Starovoitov * prog won't be doing more than 4 map_update_elem from 3737c8199e2SAlexei Starovoitov * irq disabled region 3747c8199e2SAlexei Starovoitov */ 3757c8199e2SAlexei Starovoitov alloc_bulk(c, c->unit_size <= 256 ? 4 : 1, cpu_to_node(cpu)); 3767c8199e2SAlexei Starovoitov } 3777c8199e2SAlexei Starovoitov 378bfc03c15SAlexei Starovoitov /* When size != 0 bpf_mem_cache for each cpu. 3797c8199e2SAlexei Starovoitov * This is typical bpf hash map use case when all elements have equal size. 3807c8199e2SAlexei Starovoitov * 3817c8199e2SAlexei Starovoitov * When size == 0 allocate 11 bpf_mem_cache-s for each cpu, then rely on 3827c8199e2SAlexei Starovoitov * kmalloc/kfree. Max allocation size is 4096 in this case. 3837c8199e2SAlexei Starovoitov * This is bpf_dynptr and bpf_kptr use case. 3847c8199e2SAlexei Starovoitov */ 3854ab67149SAlexei Starovoitov int bpf_mem_alloc_init(struct bpf_mem_alloc *ma, int size, bool percpu) 3867c8199e2SAlexei Starovoitov { 3877c8199e2SAlexei Starovoitov static u16 sizes[NUM_CACHES] = {96, 192, 16, 32, 64, 128, 256, 512, 1024, 2048, 4096}; 3887c8199e2SAlexei Starovoitov struct bpf_mem_caches *cc, __percpu *pcc; 3897c8199e2SAlexei Starovoitov struct bpf_mem_cache *c, __percpu *pc; 3907c8199e2SAlexei Starovoitov struct obj_cgroup *objcg = NULL; 391bfc03c15SAlexei Starovoitov int cpu, i, unit_size, percpu_size = 0; 3927c8199e2SAlexei Starovoitov 3937c8199e2SAlexei Starovoitov if (size) { 3947c8199e2SAlexei Starovoitov pc = __alloc_percpu_gfp(sizeof(*pc), 8, GFP_KERNEL); 3957c8199e2SAlexei Starovoitov if (!pc) 3967c8199e2SAlexei Starovoitov return -ENOMEM; 3974ab67149SAlexei Starovoitov 398bfc03c15SAlexei Starovoitov if (percpu) 3994ab67149SAlexei Starovoitov /* room for llist_node and per-cpu pointer */ 400bfc03c15SAlexei Starovoitov percpu_size = LLIST_NODE_SZ + sizeof(void *); 401bfc03c15SAlexei Starovoitov else 4027c8199e2SAlexei Starovoitov size += LLIST_NODE_SZ; /* room for llist_node */ 4034ab67149SAlexei Starovoitov unit_size = size; 4044ab67149SAlexei Starovoitov 4057c8199e2SAlexei Starovoitov #ifdef CONFIG_MEMCG_KMEM 406ee53cbfbSYafang Shao if (memcg_bpf_enabled()) 4077c8199e2SAlexei Starovoitov objcg = get_obj_cgroup_from_current(); 4087c8199e2SAlexei Starovoitov #endif 4097c8199e2SAlexei Starovoitov for_each_possible_cpu(cpu) { 4107c8199e2SAlexei Starovoitov c = per_cpu_ptr(pc, cpu); 4114ab67149SAlexei Starovoitov c->unit_size = unit_size; 4127c8199e2SAlexei Starovoitov c->objcg = objcg; 413bfc03c15SAlexei Starovoitov c->percpu_size = percpu_size; 4147c8199e2SAlexei Starovoitov prefill_mem_cache(c, cpu); 4157c8199e2SAlexei Starovoitov } 4167c8199e2SAlexei Starovoitov ma->cache = pc; 4177c8199e2SAlexei Starovoitov return 0; 4187c8199e2SAlexei Starovoitov } 4197c8199e2SAlexei Starovoitov 4204ab67149SAlexei Starovoitov /* size == 0 && percpu is an invalid combination */ 4214ab67149SAlexei Starovoitov if (WARN_ON_ONCE(percpu)) 4224ab67149SAlexei Starovoitov return -EINVAL; 4234ab67149SAlexei Starovoitov 4247c8199e2SAlexei Starovoitov pcc = __alloc_percpu_gfp(sizeof(*cc), 8, GFP_KERNEL); 4257c8199e2SAlexei Starovoitov if (!pcc) 4267c8199e2SAlexei Starovoitov return -ENOMEM; 4277c8199e2SAlexei Starovoitov #ifdef CONFIG_MEMCG_KMEM 4287c8199e2SAlexei Starovoitov objcg = get_obj_cgroup_from_current(); 4297c8199e2SAlexei Starovoitov #endif 4307c8199e2SAlexei Starovoitov for_each_possible_cpu(cpu) { 4317c8199e2SAlexei Starovoitov cc = per_cpu_ptr(pcc, cpu); 4327c8199e2SAlexei Starovoitov for (i = 0; i < NUM_CACHES; i++) { 4337c8199e2SAlexei Starovoitov c = &cc->cache[i]; 4347c8199e2SAlexei Starovoitov c->unit_size = sizes[i]; 4357c8199e2SAlexei Starovoitov c->objcg = objcg; 4367c8199e2SAlexei Starovoitov prefill_mem_cache(c, cpu); 4377c8199e2SAlexei Starovoitov } 4387c8199e2SAlexei Starovoitov } 4397c8199e2SAlexei Starovoitov ma->caches = pcc; 4407c8199e2SAlexei Starovoitov return 0; 4417c8199e2SAlexei Starovoitov } 4427c8199e2SAlexei Starovoitov 4437c8199e2SAlexei Starovoitov static void drain_mem_cache(struct bpf_mem_cache *c) 4447c8199e2SAlexei Starovoitov { 445aa7881fcSHou Tao bool percpu = !!c->percpu_size; 4467c8199e2SAlexei Starovoitov 4479f2c6e96SAlexei Starovoitov /* No progs are using this bpf_mem_cache, but htab_map_free() called 4489f2c6e96SAlexei Starovoitov * bpf_mem_cache_free() for all remaining elements and they can be in 44912c8d0f4SAlexei Starovoitov * free_by_rcu_ttrace or in waiting_for_gp_ttrace lists, so drain those lists now. 450fa4447cbSHou Tao * 45112c8d0f4SAlexei Starovoitov * Except for waiting_for_gp_ttrace list, there are no concurrent operations 452fa4447cbSHou Tao * on these lists, so it is safe to use __llist_del_all(). 4538d5a8011SAlexei Starovoitov */ 45412c8d0f4SAlexei Starovoitov free_all(__llist_del_all(&c->free_by_rcu_ttrace), percpu); 45512c8d0f4SAlexei Starovoitov free_all(llist_del_all(&c->waiting_for_gp_ttrace), percpu); 456aa7881fcSHou Tao free_all(__llist_del_all(&c->free_llist), percpu); 457aa7881fcSHou Tao free_all(__llist_del_all(&c->free_llist_extra), percpu); 4587c8199e2SAlexei Starovoitov } 4597c8199e2SAlexei Starovoitov 4609f2c6e96SAlexei Starovoitov static void free_mem_alloc_no_barrier(struct bpf_mem_alloc *ma) 4619f2c6e96SAlexei Starovoitov { 4629f2c6e96SAlexei Starovoitov free_percpu(ma->cache); 4639f2c6e96SAlexei Starovoitov free_percpu(ma->caches); 4649f2c6e96SAlexei Starovoitov ma->cache = NULL; 4659f2c6e96SAlexei Starovoitov ma->caches = NULL; 4669f2c6e96SAlexei Starovoitov } 4679f2c6e96SAlexei Starovoitov 4689f2c6e96SAlexei Starovoitov static void free_mem_alloc(struct bpf_mem_alloc *ma) 4699f2c6e96SAlexei Starovoitov { 47012c8d0f4SAlexei Starovoitov /* waiting_for_gp_ttrace lists was drained, but __free_rcu might 4719f2c6e96SAlexei Starovoitov * still execute. Wait for it now before we freeing percpu caches. 472822ed78fSHou Tao * 473822ed78fSHou Tao * rcu_barrier_tasks_trace() doesn't imply synchronize_rcu_tasks_trace(), 474822ed78fSHou Tao * but rcu_barrier_tasks_trace() and rcu_barrier() below are only used 475822ed78fSHou Tao * to wait for the pending __free_rcu_tasks_trace() and __free_rcu(), 476822ed78fSHou Tao * so if call_rcu(head, __free_rcu) is skipped due to 477822ed78fSHou Tao * rcu_trace_implies_rcu_gp(), it will be OK to skip rcu_barrier() by 478822ed78fSHou Tao * using rcu_trace_implies_rcu_gp() as well. 4799f2c6e96SAlexei Starovoitov */ 4809f2c6e96SAlexei Starovoitov rcu_barrier_tasks_trace(); 481822ed78fSHou Tao if (!rcu_trace_implies_rcu_gp()) 4829f2c6e96SAlexei Starovoitov rcu_barrier(); 4839f2c6e96SAlexei Starovoitov free_mem_alloc_no_barrier(ma); 4849f2c6e96SAlexei Starovoitov } 4859f2c6e96SAlexei Starovoitov 4869f2c6e96SAlexei Starovoitov static void free_mem_alloc_deferred(struct work_struct *work) 4879f2c6e96SAlexei Starovoitov { 4889f2c6e96SAlexei Starovoitov struct bpf_mem_alloc *ma = container_of(work, struct bpf_mem_alloc, work); 4899f2c6e96SAlexei Starovoitov 4909f2c6e96SAlexei Starovoitov free_mem_alloc(ma); 4919f2c6e96SAlexei Starovoitov kfree(ma); 4929f2c6e96SAlexei Starovoitov } 4939f2c6e96SAlexei Starovoitov 4949f2c6e96SAlexei Starovoitov static void destroy_mem_alloc(struct bpf_mem_alloc *ma, int rcu_in_progress) 4959f2c6e96SAlexei Starovoitov { 4969f2c6e96SAlexei Starovoitov struct bpf_mem_alloc *copy; 4979f2c6e96SAlexei Starovoitov 4989f2c6e96SAlexei Starovoitov if (!rcu_in_progress) { 4999f2c6e96SAlexei Starovoitov /* Fast path. No callbacks are pending, hence no need to do 5009f2c6e96SAlexei Starovoitov * rcu_barrier-s. 5019f2c6e96SAlexei Starovoitov */ 5029f2c6e96SAlexei Starovoitov free_mem_alloc_no_barrier(ma); 5039f2c6e96SAlexei Starovoitov return; 5049f2c6e96SAlexei Starovoitov } 5059f2c6e96SAlexei Starovoitov 506a80672d7SAlexei Starovoitov copy = kmemdup(ma, sizeof(*ma), GFP_KERNEL); 5079f2c6e96SAlexei Starovoitov if (!copy) { 5089f2c6e96SAlexei Starovoitov /* Slow path with inline barrier-s */ 5099f2c6e96SAlexei Starovoitov free_mem_alloc(ma); 5109f2c6e96SAlexei Starovoitov return; 5119f2c6e96SAlexei Starovoitov } 5129f2c6e96SAlexei Starovoitov 5139f2c6e96SAlexei Starovoitov /* Defer barriers into worker to let the rest of map memory to be freed */ 514a80672d7SAlexei Starovoitov memset(ma, 0, sizeof(*ma)); 5159f2c6e96SAlexei Starovoitov INIT_WORK(©->work, free_mem_alloc_deferred); 5169f2c6e96SAlexei Starovoitov queue_work(system_unbound_wq, ©->work); 5179f2c6e96SAlexei Starovoitov } 5189f2c6e96SAlexei Starovoitov 5197c8199e2SAlexei Starovoitov void bpf_mem_alloc_destroy(struct bpf_mem_alloc *ma) 5207c8199e2SAlexei Starovoitov { 5217c8199e2SAlexei Starovoitov struct bpf_mem_caches *cc; 5227c8199e2SAlexei Starovoitov struct bpf_mem_cache *c; 5239f2c6e96SAlexei Starovoitov int cpu, i, rcu_in_progress; 5247c8199e2SAlexei Starovoitov 5257c8199e2SAlexei Starovoitov if (ma->cache) { 5269f2c6e96SAlexei Starovoitov rcu_in_progress = 0; 5277c8199e2SAlexei Starovoitov for_each_possible_cpu(cpu) { 5287c8199e2SAlexei Starovoitov c = per_cpu_ptr(ma->cache, cpu); 5293d058187SHou Tao /* 5303d058187SHou Tao * refill_work may be unfinished for PREEMPT_RT kernel 5313d058187SHou Tao * in which irq work is invoked in a per-CPU RT thread. 5323d058187SHou Tao * It is also possible for kernel with 5333d058187SHou Tao * arch_irq_work_has_interrupt() being false and irq 5343d058187SHou Tao * work is invoked in timer interrupt. So waiting for 5353d058187SHou Tao * the completion of irq work to ease the handling of 5363d058187SHou Tao * concurrency. 5373d058187SHou Tao */ 5383d058187SHou Tao irq_work_sync(&c->refill_work); 5397c8199e2SAlexei Starovoitov drain_mem_cache(c); 54012c8d0f4SAlexei Starovoitov rcu_in_progress += atomic_read(&c->call_rcu_ttrace_in_progress); 5417c8199e2SAlexei Starovoitov } 542bfc03c15SAlexei Starovoitov /* objcg is the same across cpus */ 5437c8199e2SAlexei Starovoitov if (c->objcg) 5447c8199e2SAlexei Starovoitov obj_cgroup_put(c->objcg); 5459f2c6e96SAlexei Starovoitov destroy_mem_alloc(ma, rcu_in_progress); 5467c8199e2SAlexei Starovoitov } 5477c8199e2SAlexei Starovoitov if (ma->caches) { 5489f2c6e96SAlexei Starovoitov rcu_in_progress = 0; 5497c8199e2SAlexei Starovoitov for_each_possible_cpu(cpu) { 5507c8199e2SAlexei Starovoitov cc = per_cpu_ptr(ma->caches, cpu); 5517c8199e2SAlexei Starovoitov for (i = 0; i < NUM_CACHES; i++) { 5527c8199e2SAlexei Starovoitov c = &cc->cache[i]; 5533d058187SHou Tao irq_work_sync(&c->refill_work); 5547c8199e2SAlexei Starovoitov drain_mem_cache(c); 55512c8d0f4SAlexei Starovoitov rcu_in_progress += atomic_read(&c->call_rcu_ttrace_in_progress); 5567c8199e2SAlexei Starovoitov } 5577c8199e2SAlexei Starovoitov } 5587c8199e2SAlexei Starovoitov if (c->objcg) 5597c8199e2SAlexei Starovoitov obj_cgroup_put(c->objcg); 5609f2c6e96SAlexei Starovoitov destroy_mem_alloc(ma, rcu_in_progress); 5617c8199e2SAlexei Starovoitov } 5627c8199e2SAlexei Starovoitov } 5637c8199e2SAlexei Starovoitov 5647c8199e2SAlexei Starovoitov /* notrace is necessary here and in other functions to make sure 5657c8199e2SAlexei Starovoitov * bpf programs cannot attach to them and cause llist corruptions. 5667c8199e2SAlexei Starovoitov */ 5677c8199e2SAlexei Starovoitov static void notrace *unit_alloc(struct bpf_mem_cache *c) 5687c8199e2SAlexei Starovoitov { 5697c8199e2SAlexei Starovoitov struct llist_node *llnode = NULL; 5707c8199e2SAlexei Starovoitov unsigned long flags; 5717c8199e2SAlexei Starovoitov int cnt = 0; 5727c8199e2SAlexei Starovoitov 5737c8199e2SAlexei Starovoitov /* Disable irqs to prevent the following race for majority of prog types: 5747c8199e2SAlexei Starovoitov * prog_A 5757c8199e2SAlexei Starovoitov * bpf_mem_alloc 5767c8199e2SAlexei Starovoitov * preemption or irq -> prog_B 5777c8199e2SAlexei Starovoitov * bpf_mem_alloc 5787c8199e2SAlexei Starovoitov * 5797c8199e2SAlexei Starovoitov * but prog_B could be a perf_event NMI prog. 5807c8199e2SAlexei Starovoitov * Use per-cpu 'active' counter to order free_list access between 5817c8199e2SAlexei Starovoitov * unit_alloc/unit_free/bpf_mem_refill. 5827c8199e2SAlexei Starovoitov */ 5837c8199e2SAlexei Starovoitov local_irq_save(flags); 5847c8199e2SAlexei Starovoitov if (local_inc_return(&c->active) == 1) { 5857c8199e2SAlexei Starovoitov llnode = __llist_del_first(&c->free_llist); 5867c8199e2SAlexei Starovoitov if (llnode) 5877c8199e2SAlexei Starovoitov cnt = --c->free_cnt; 5887c8199e2SAlexei Starovoitov } 5897c8199e2SAlexei Starovoitov local_dec(&c->active); 5907c8199e2SAlexei Starovoitov local_irq_restore(flags); 5917c8199e2SAlexei Starovoitov 5927c8199e2SAlexei Starovoitov WARN_ON(cnt < 0); 5937c8199e2SAlexei Starovoitov 5947c266178SAlexei Starovoitov if (cnt < c->low_watermark) 5957c8199e2SAlexei Starovoitov irq_work_raise(c); 5967c8199e2SAlexei Starovoitov return llnode; 5977c8199e2SAlexei Starovoitov } 5987c8199e2SAlexei Starovoitov 5997c8199e2SAlexei Starovoitov /* Though 'ptr' object could have been allocated on a different cpu 6007c8199e2SAlexei Starovoitov * add it to the free_llist of the current cpu. 6017c8199e2SAlexei Starovoitov * Let kfree() logic deal with it when it's later called from irq_work. 6027c8199e2SAlexei Starovoitov */ 6037c8199e2SAlexei Starovoitov static void notrace unit_free(struct bpf_mem_cache *c, void *ptr) 6047c8199e2SAlexei Starovoitov { 6057c8199e2SAlexei Starovoitov struct llist_node *llnode = ptr - LLIST_NODE_SZ; 6067c8199e2SAlexei Starovoitov unsigned long flags; 6077c8199e2SAlexei Starovoitov int cnt = 0; 6087c8199e2SAlexei Starovoitov 6097c8199e2SAlexei Starovoitov BUILD_BUG_ON(LLIST_NODE_SZ > 8); 6107c8199e2SAlexei Starovoitov 6117c8199e2SAlexei Starovoitov local_irq_save(flags); 6127c8199e2SAlexei Starovoitov if (local_inc_return(&c->active) == 1) { 6137c8199e2SAlexei Starovoitov __llist_add(llnode, &c->free_llist); 6147c8199e2SAlexei Starovoitov cnt = ++c->free_cnt; 6157c8199e2SAlexei Starovoitov } else { 6167c8199e2SAlexei Starovoitov /* unit_free() cannot fail. Therefore add an object to atomic 6177c8199e2SAlexei Starovoitov * llist. free_bulk() will drain it. Though free_llist_extra is 6187c8199e2SAlexei Starovoitov * a per-cpu list we have to use atomic llist_add here, since 6197c8199e2SAlexei Starovoitov * it also can be interrupted by bpf nmi prog that does another 6207c8199e2SAlexei Starovoitov * unit_free() into the same free_llist_extra. 6217c8199e2SAlexei Starovoitov */ 6227c8199e2SAlexei Starovoitov llist_add(llnode, &c->free_llist_extra); 6237c8199e2SAlexei Starovoitov } 6247c8199e2SAlexei Starovoitov local_dec(&c->active); 6257c8199e2SAlexei Starovoitov local_irq_restore(flags); 6267c8199e2SAlexei Starovoitov 6277c266178SAlexei Starovoitov if (cnt > c->high_watermark) 6287c8199e2SAlexei Starovoitov /* free few objects from current cpu into global kmalloc pool */ 6297c8199e2SAlexei Starovoitov irq_work_raise(c); 6307c8199e2SAlexei Starovoitov } 6317c8199e2SAlexei Starovoitov 6327c8199e2SAlexei Starovoitov /* Called from BPF program or from sys_bpf syscall. 6337c8199e2SAlexei Starovoitov * In both cases migration is disabled. 6347c8199e2SAlexei Starovoitov */ 6357c8199e2SAlexei Starovoitov void notrace *bpf_mem_alloc(struct bpf_mem_alloc *ma, size_t size) 6367c8199e2SAlexei Starovoitov { 6377c8199e2SAlexei Starovoitov int idx; 6387c8199e2SAlexei Starovoitov void *ret; 6397c8199e2SAlexei Starovoitov 6407c8199e2SAlexei Starovoitov if (!size) 6417c8199e2SAlexei Starovoitov return ZERO_SIZE_PTR; 6427c8199e2SAlexei Starovoitov 6437c8199e2SAlexei Starovoitov idx = bpf_mem_cache_idx(size + LLIST_NODE_SZ); 6447c8199e2SAlexei Starovoitov if (idx < 0) 6457c8199e2SAlexei Starovoitov return NULL; 6467c8199e2SAlexei Starovoitov 6477c8199e2SAlexei Starovoitov ret = unit_alloc(this_cpu_ptr(ma->caches)->cache + idx); 6487c8199e2SAlexei Starovoitov return !ret ? NULL : ret + LLIST_NODE_SZ; 6497c8199e2SAlexei Starovoitov } 6507c8199e2SAlexei Starovoitov 6517c8199e2SAlexei Starovoitov void notrace bpf_mem_free(struct bpf_mem_alloc *ma, void *ptr) 6527c8199e2SAlexei Starovoitov { 6537c8199e2SAlexei Starovoitov int idx; 6547c8199e2SAlexei Starovoitov 6557c8199e2SAlexei Starovoitov if (!ptr) 6567c8199e2SAlexei Starovoitov return; 6577c8199e2SAlexei Starovoitov 6581e660f7eSAlexei Starovoitov idx = bpf_mem_cache_idx(ksize(ptr - LLIST_NODE_SZ)); 6597c8199e2SAlexei Starovoitov if (idx < 0) 6607c8199e2SAlexei Starovoitov return; 6617c8199e2SAlexei Starovoitov 6627c8199e2SAlexei Starovoitov unit_free(this_cpu_ptr(ma->caches)->cache + idx, ptr); 6637c8199e2SAlexei Starovoitov } 6647c8199e2SAlexei Starovoitov 6657c8199e2SAlexei Starovoitov void notrace *bpf_mem_cache_alloc(struct bpf_mem_alloc *ma) 6667c8199e2SAlexei Starovoitov { 6677c8199e2SAlexei Starovoitov void *ret; 6687c8199e2SAlexei Starovoitov 6697c8199e2SAlexei Starovoitov ret = unit_alloc(this_cpu_ptr(ma->cache)); 6707c8199e2SAlexei Starovoitov return !ret ? NULL : ret + LLIST_NODE_SZ; 6717c8199e2SAlexei Starovoitov } 6727c8199e2SAlexei Starovoitov 6737c8199e2SAlexei Starovoitov void notrace bpf_mem_cache_free(struct bpf_mem_alloc *ma, void *ptr) 6747c8199e2SAlexei Starovoitov { 6757c8199e2SAlexei Starovoitov if (!ptr) 6767c8199e2SAlexei Starovoitov return; 6777c8199e2SAlexei Starovoitov 6787c8199e2SAlexei Starovoitov unit_free(this_cpu_ptr(ma->cache), ptr); 6797c8199e2SAlexei Starovoitov } 680e65a5c6eSMartin KaFai Lau 681e65a5c6eSMartin KaFai Lau /* Directly does a kfree() without putting 'ptr' back to the free_llist 682e65a5c6eSMartin KaFai Lau * for reuse and without waiting for a rcu_tasks_trace gp. 683e65a5c6eSMartin KaFai Lau * The caller must first go through the rcu_tasks_trace gp for 'ptr' 684e65a5c6eSMartin KaFai Lau * before calling bpf_mem_cache_raw_free(). 685e65a5c6eSMartin KaFai Lau * It could be used when the rcu_tasks_trace callback does not have 686e65a5c6eSMartin KaFai Lau * a hold on the original bpf_mem_alloc object that allocated the 687e65a5c6eSMartin KaFai Lau * 'ptr'. This should only be used in the uncommon code path. 688e65a5c6eSMartin KaFai Lau * Otherwise, the bpf_mem_alloc's free_llist cannot be refilled 689e65a5c6eSMartin KaFai Lau * and may affect performance. 690e65a5c6eSMartin KaFai Lau */ 691e65a5c6eSMartin KaFai Lau void bpf_mem_cache_raw_free(void *ptr) 692e65a5c6eSMartin KaFai Lau { 693e65a5c6eSMartin KaFai Lau if (!ptr) 694e65a5c6eSMartin KaFai Lau return; 695e65a5c6eSMartin KaFai Lau 696e65a5c6eSMartin KaFai Lau kfree(ptr - LLIST_NODE_SZ); 697e65a5c6eSMartin KaFai Lau } 698e65a5c6eSMartin KaFai Lau 699e65a5c6eSMartin KaFai Lau /* When flags == GFP_KERNEL, it signals that the caller will not cause 700e65a5c6eSMartin KaFai Lau * deadlock when using kmalloc. bpf_mem_cache_alloc_flags() will use 701e65a5c6eSMartin KaFai Lau * kmalloc if the free_llist is empty. 702e65a5c6eSMartin KaFai Lau */ 703e65a5c6eSMartin KaFai Lau void notrace *bpf_mem_cache_alloc_flags(struct bpf_mem_alloc *ma, gfp_t flags) 704e65a5c6eSMartin KaFai Lau { 705e65a5c6eSMartin KaFai Lau struct bpf_mem_cache *c; 706e65a5c6eSMartin KaFai Lau void *ret; 707e65a5c6eSMartin KaFai Lau 708e65a5c6eSMartin KaFai Lau c = this_cpu_ptr(ma->cache); 709e65a5c6eSMartin KaFai Lau 710e65a5c6eSMartin KaFai Lau ret = unit_alloc(c); 711e65a5c6eSMartin KaFai Lau if (!ret && flags == GFP_KERNEL) { 712e65a5c6eSMartin KaFai Lau struct mem_cgroup *memcg, *old_memcg; 713e65a5c6eSMartin KaFai Lau 714e65a5c6eSMartin KaFai Lau memcg = get_memcg(c); 715e65a5c6eSMartin KaFai Lau old_memcg = set_active_memcg(memcg); 716e65a5c6eSMartin KaFai Lau ret = __alloc(c, NUMA_NO_NODE, GFP_KERNEL | __GFP_NOWARN | __GFP_ACCOUNT); 717e65a5c6eSMartin KaFai Lau set_active_memcg(old_memcg); 718e65a5c6eSMartin KaFai Lau mem_cgroup_put(memcg); 719e65a5c6eSMartin KaFai Lau } 720e65a5c6eSMartin KaFai Lau 721e65a5c6eSMartin KaFai Lau return !ret ? NULL : ret + LLIST_NODE_SZ; 722e65a5c6eSMartin KaFai Lau } 723