15b497af4SThomas Gleixner // SPDX-License-Identifier: GPL-2.0-only
299c55f7dSAlexei Starovoitov /* Copyright (c) 2011-2014 PLUMgrid, http://plumgrid.com
399c55f7dSAlexei Starovoitov */
499c55f7dSAlexei Starovoitov #include <linux/bpf.h>
5aef2fedaSJakub Kicinski #include <linux/bpf-cgroup.h>
6a67edbf4SDaniel Borkmann #include <linux/bpf_trace.h>
7f4364dcfSSean Young #include <linux/bpf_lirc.h>
84a1e7c0cSToke Høiland-Jørgensen #include <linux/bpf_verifier.h>
961df10c7SKumar Kartikeya Dwivedi #include <linux/bsearch.h>
10f56a653cSMartin KaFai Lau #include <linux/btf.h>
1199c55f7dSAlexei Starovoitov #include <linux/syscalls.h>
1299c55f7dSAlexei Starovoitov #include <linux/slab.h>
133f07c014SIngo Molnar #include <linux/sched/signal.h>
14d407bd25SDaniel Borkmann #include <linux/vmalloc.h>
15d407bd25SDaniel Borkmann #include <linux/mmzone.h>
1699c55f7dSAlexei Starovoitov #include <linux/anon_inodes.h>
1741bdc4b4SYonghong Song #include <linux/fdtable.h>
18db20fd2bSAlexei Starovoitov #include <linux/file.h>
1941bdc4b4SYonghong Song #include <linux/fs.h>
2009756af4SAlexei Starovoitov #include <linux/license.h>
2109756af4SAlexei Starovoitov #include <linux/filter.h>
22535e7b4bSMickaël Salaün #include <linux/kernel.h>
23dc4bb0e2SMartin KaFai Lau #include <linux/idr.h>
24cb4d2b3fSMartin KaFai Lau #include <linux/cred.h>
25cb4d2b3fSMartin KaFai Lau #include <linux/timekeeping.h>
26cb4d2b3fSMartin KaFai Lau #include <linux/ctype.h>
279ef09e35SMark Rutland #include <linux/nospec.h>
28bae141f5SDaniel Borkmann #include <linux/audit.h>
29ccfe29ebSAlexei Starovoitov #include <uapi/linux/btf.h>
30ca5999fdSMike Rapoport #include <linux/pgtable.h>
319e4e01dfSKP Singh #include <linux/bpf_lsm.h>
32457f4436SAndrii Nakryiko #include <linux/poll.h>
334d7d7f69SKumar Kartikeya Dwivedi #include <linux/sort.h>
34a3fd7ceeSJakub Sitnicki #include <linux/bpf-netns.h>
351e6c62a8SAlexei Starovoitov #include <linux/rcupdate_trace.h>
3648edc1f7SRoman Gushchin #include <linux/memcontrol.h>
370dcac272SJiri Olsa #include <linux/trace_events.h>
3884601d6eSFlorian Westphal #include <net/netfilter/nf_bpf_link.h>
3999c55f7dSAlexei Starovoitov
40e420bed0SDaniel Borkmann #include <net/tcx.h>
41e420bed0SDaniel Borkmann
42da765a2fSDaniel Borkmann #define IS_FD_ARRAY(map) ((map)->map_type == BPF_MAP_TYPE_PERF_EVENT_ARRAY || \
4314dc6f04SMartin KaFai Lau (map)->map_type == BPF_MAP_TYPE_CGROUP_ARRAY || \
4414dc6f04SMartin KaFai Lau (map)->map_type == BPF_MAP_TYPE_ARRAY_OF_MAPS)
45da765a2fSDaniel Borkmann #define IS_FD_PROG_ARRAY(map) ((map)->map_type == BPF_MAP_TYPE_PROG_ARRAY)
4614dc6f04SMartin KaFai Lau #define IS_FD_HASH(map) ((map)->map_type == BPF_MAP_TYPE_HASH_OF_MAPS)
47da765a2fSDaniel Borkmann #define IS_FD_MAP(map) (IS_FD_ARRAY(map) || IS_FD_PROG_ARRAY(map) || \
48da765a2fSDaniel Borkmann IS_FD_HASH(map))
4914dc6f04SMartin KaFai Lau
506e71b04aSChenbo Feng #define BPF_OBJ_FLAG_MASK (BPF_F_RDONLY | BPF_F_WRONLY)
516e71b04aSChenbo Feng
52b121d1e7SAlexei Starovoitov DEFINE_PER_CPU(int, bpf_prog_active);
53dc4bb0e2SMartin KaFai Lau static DEFINE_IDR(prog_idr);
54dc4bb0e2SMartin KaFai Lau static DEFINE_SPINLOCK(prog_idr_lock);
55f3f1c054SMartin KaFai Lau static DEFINE_IDR(map_idr);
56f3f1c054SMartin KaFai Lau static DEFINE_SPINLOCK(map_idr_lock);
57a3b80e10SAndrii Nakryiko static DEFINE_IDR(link_idr);
58a3b80e10SAndrii Nakryiko static DEFINE_SPINLOCK(link_idr_lock);
59b121d1e7SAlexei Starovoitov
6008389d88SDaniel Borkmann int sysctl_unprivileged_bpf_disabled __read_mostly =
6108389d88SDaniel Borkmann IS_BUILTIN(CONFIG_BPF_UNPRIV_DEFAULT_OFF) ? 2 : 0;
621be7f75dSAlexei Starovoitov
6340077e0cSJohannes Berg static const struct bpf_map_ops * const bpf_map_types[] = {
6491cc1a99SAlexei Starovoitov #define BPF_PROG_TYPE(_id, _name, prog_ctx_type, kern_ctx_type)
6540077e0cSJohannes Berg #define BPF_MAP_TYPE(_id, _ops) \
6640077e0cSJohannes Berg [_id] = &_ops,
67f2e10bffSAndrii Nakryiko #define BPF_LINK_TYPE(_id, _name)
6840077e0cSJohannes Berg #include <linux/bpf_types.h>
6940077e0cSJohannes Berg #undef BPF_PROG_TYPE
7040077e0cSJohannes Berg #undef BPF_MAP_TYPE
71f2e10bffSAndrii Nakryiko #undef BPF_LINK_TYPE
7240077e0cSJohannes Berg };
7399c55f7dSAlexei Starovoitov
74752ba56fSMickaël Salaün /*
75752ba56fSMickaël Salaün * If we're handed a bigger struct than we know of, ensure all the unknown bits
76752ba56fSMickaël Salaün * are 0 - i.e. new user-space does not rely on any kernel feature extensions
77752ba56fSMickaël Salaün * we don't know about yet.
78752ba56fSMickaël Salaün *
79752ba56fSMickaël Salaün * There is a ToCToU between this function call and the following
80752ba56fSMickaël Salaün * copy_from_user() call. However, this is not a concern since this function is
81752ba56fSMickaël Salaün * meant to be a future-proofing of bits.
82752ba56fSMickaël Salaün */
bpf_check_uarg_tail_zero(bpfptr_t uaddr,size_t expected_size,size_t actual_size)83af2ac3e1SAlexei Starovoitov int bpf_check_uarg_tail_zero(bpfptr_t uaddr,
8458291a74SMickaël Salaün size_t expected_size,
8558291a74SMickaël Salaün size_t actual_size)
8658291a74SMickaël Salaün {
87b7e4b65fSAl Viro int res;
8858291a74SMickaël Salaün
89752ba56fSMickaël Salaün if (unlikely(actual_size > PAGE_SIZE)) /* silly large */
90752ba56fSMickaël Salaün return -E2BIG;
91752ba56fSMickaël Salaün
9258291a74SMickaël Salaün if (actual_size <= expected_size)
9358291a74SMickaël Salaün return 0;
9458291a74SMickaël Salaün
95af2ac3e1SAlexei Starovoitov if (uaddr.is_kernel)
96af2ac3e1SAlexei Starovoitov res = memchr_inv(uaddr.kernel + expected_size, 0,
97af2ac3e1SAlexei Starovoitov actual_size - expected_size) == NULL;
98af2ac3e1SAlexei Starovoitov else
99af2ac3e1SAlexei Starovoitov res = check_zeroed_user(uaddr.user + expected_size,
100af2ac3e1SAlexei Starovoitov actual_size - expected_size);
101b7e4b65fSAl Viro if (res < 0)
102b7e4b65fSAl Viro return res;
103b7e4b65fSAl Viro return res ? 0 : -E2BIG;
10458291a74SMickaël Salaün }
10558291a74SMickaël Salaün
106a3884572SJakub Kicinski const struct bpf_map_ops bpf_map_offload_ops = {
107f4d05259SMartin KaFai Lau .map_meta_equal = bpf_map_meta_equal,
108a3884572SJakub Kicinski .map_alloc = bpf_map_offload_map_alloc,
109a3884572SJakub Kicinski .map_free = bpf_map_offload_map_free,
110e8d2bec0SDaniel Borkmann .map_check_btf = map_check_no_btf,
1119629363cSYafang Shao .map_mem_usage = bpf_map_offload_map_mem_usage,
112a3884572SJakub Kicinski };
113a3884572SJakub Kicinski
bpf_map_write_active_inc(struct bpf_map * map)114353050beSDaniel Borkmann static void bpf_map_write_active_inc(struct bpf_map *map)
115353050beSDaniel Borkmann {
116353050beSDaniel Borkmann atomic64_inc(&map->writecnt);
117353050beSDaniel Borkmann }
118353050beSDaniel Borkmann
bpf_map_write_active_dec(struct bpf_map * map)119353050beSDaniel Borkmann static void bpf_map_write_active_dec(struct bpf_map *map)
120353050beSDaniel Borkmann {
121353050beSDaniel Borkmann atomic64_dec(&map->writecnt);
122353050beSDaniel Borkmann }
123353050beSDaniel Borkmann
bpf_map_write_active(const struct bpf_map * map)124353050beSDaniel Borkmann bool bpf_map_write_active(const struct bpf_map *map)
125353050beSDaniel Borkmann {
126353050beSDaniel Borkmann return atomic64_read(&map->writecnt) != 0;
127353050beSDaniel Borkmann }
128353050beSDaniel Borkmann
bpf_map_value_size(const struct bpf_map * map)12980ee81e0SRoman Gushchin static u32 bpf_map_value_size(const struct bpf_map *map)
13015c14a3dSBrian Vazquez {
13115c14a3dSBrian Vazquez if (map->map_type == BPF_MAP_TYPE_PERCPU_HASH ||
13215c14a3dSBrian Vazquez map->map_type == BPF_MAP_TYPE_LRU_PERCPU_HASH ||
13315c14a3dSBrian Vazquez map->map_type == BPF_MAP_TYPE_PERCPU_ARRAY ||
13415c14a3dSBrian Vazquez map->map_type == BPF_MAP_TYPE_PERCPU_CGROUP_STORAGE)
13515c14a3dSBrian Vazquez return round_up(map->value_size, 8) * num_possible_cpus();
13615c14a3dSBrian Vazquez else if (IS_FD_MAP(map))
13715c14a3dSBrian Vazquez return sizeof(u32);
13815c14a3dSBrian Vazquez else
13915c14a3dSBrian Vazquez return map->value_size;
14015c14a3dSBrian Vazquez }
14115c14a3dSBrian Vazquez
maybe_wait_bpf_programs(struct bpf_map * map)14215c14a3dSBrian Vazquez static void maybe_wait_bpf_programs(struct bpf_map *map)
14315c14a3dSBrian Vazquez {
14415c14a3dSBrian Vazquez /* Wait for any running BPF programs to complete so that
14515c14a3dSBrian Vazquez * userspace, when we return to it, knows that all programs
14615c14a3dSBrian Vazquez * that could be running use the new map value.
14715c14a3dSBrian Vazquez */
14815c14a3dSBrian Vazquez if (map->map_type == BPF_MAP_TYPE_HASH_OF_MAPS ||
14915c14a3dSBrian Vazquez map->map_type == BPF_MAP_TYPE_ARRAY_OF_MAPS)
15015c14a3dSBrian Vazquez synchronize_rcu();
15115c14a3dSBrian Vazquez }
15215c14a3dSBrian Vazquez
bpf_map_update_value(struct bpf_map * map,struct file * map_file,void * key,void * value,__u64 flags)1533af43ba4SHou Tao static int bpf_map_update_value(struct bpf_map *map, struct file *map_file,
1543af43ba4SHou Tao void *key, void *value, __u64 flags)
15515c14a3dSBrian Vazquez {
15615c14a3dSBrian Vazquez int err;
15715c14a3dSBrian Vazquez
15815c14a3dSBrian Vazquez /* Need to create a kthread, thus must support schedule */
1599d03ebc7SStanislav Fomichev if (bpf_map_is_offloaded(map)) {
16015c14a3dSBrian Vazquez return bpf_map_offload_update_elem(map, key, value, flags);
16115c14a3dSBrian Vazquez } else if (map->map_type == BPF_MAP_TYPE_CPUMAP ||
16215c14a3dSBrian Vazquez map->map_type == BPF_MAP_TYPE_STRUCT_OPS) {
16315c14a3dSBrian Vazquez return map->ops->map_update_elem(map, key, value, flags);
16413b79d3fSLorenz Bauer } else if (map->map_type == BPF_MAP_TYPE_SOCKHASH ||
16513b79d3fSLorenz Bauer map->map_type == BPF_MAP_TYPE_SOCKMAP) {
16613b79d3fSLorenz Bauer return sock_map_update_elem_sys(map, key, value, flags);
16715c14a3dSBrian Vazquez } else if (IS_FD_PROG_ARRAY(map)) {
1683af43ba4SHou Tao return bpf_fd_array_map_update_elem(map, map_file, key, value,
16915c14a3dSBrian Vazquez flags);
17015c14a3dSBrian Vazquez }
17115c14a3dSBrian Vazquez
172b6e5dae1SThomas Gleixner bpf_disable_instrumentation();
17315c14a3dSBrian Vazquez if (map->map_type == BPF_MAP_TYPE_PERCPU_HASH ||
17415c14a3dSBrian Vazquez map->map_type == BPF_MAP_TYPE_LRU_PERCPU_HASH) {
17515c14a3dSBrian Vazquez err = bpf_percpu_hash_update(map, key, value, flags);
17615c14a3dSBrian Vazquez } else if (map->map_type == BPF_MAP_TYPE_PERCPU_ARRAY) {
17715c14a3dSBrian Vazquez err = bpf_percpu_array_update(map, key, value, flags);
17815c14a3dSBrian Vazquez } else if (map->map_type == BPF_MAP_TYPE_PERCPU_CGROUP_STORAGE) {
17915c14a3dSBrian Vazquez err = bpf_percpu_cgroup_storage_update(map, key, value,
18015c14a3dSBrian Vazquez flags);
18115c14a3dSBrian Vazquez } else if (IS_FD_ARRAY(map)) {
18215c14a3dSBrian Vazquez rcu_read_lock();
1833af43ba4SHou Tao err = bpf_fd_array_map_update_elem(map, map_file, key, value,
18415c14a3dSBrian Vazquez flags);
18515c14a3dSBrian Vazquez rcu_read_unlock();
18615c14a3dSBrian Vazquez } else if (map->map_type == BPF_MAP_TYPE_HASH_OF_MAPS) {
18715c14a3dSBrian Vazquez rcu_read_lock();
1883af43ba4SHou Tao err = bpf_fd_htab_map_update_elem(map, map_file, key, value,
18915c14a3dSBrian Vazquez flags);
19015c14a3dSBrian Vazquez rcu_read_unlock();
19115c14a3dSBrian Vazquez } else if (map->map_type == BPF_MAP_TYPE_REUSEPORT_SOCKARRAY) {
19215c14a3dSBrian Vazquez /* rcu_read_lock() is not needed */
19315c14a3dSBrian Vazquez err = bpf_fd_reuseport_array_update_elem(map, key, value,
19415c14a3dSBrian Vazquez flags);
19515c14a3dSBrian Vazquez } else if (map->map_type == BPF_MAP_TYPE_QUEUE ||
1969330986cSJoanne Koong map->map_type == BPF_MAP_TYPE_STACK ||
1979330986cSJoanne Koong map->map_type == BPF_MAP_TYPE_BLOOM_FILTER) {
19815c14a3dSBrian Vazquez err = map->ops->map_push_elem(map, value, flags);
19915c14a3dSBrian Vazquez } else {
20015c14a3dSBrian Vazquez rcu_read_lock();
20115c14a3dSBrian Vazquez err = map->ops->map_update_elem(map, key, value, flags);
20215c14a3dSBrian Vazquez rcu_read_unlock();
20315c14a3dSBrian Vazquez }
204b6e5dae1SThomas Gleixner bpf_enable_instrumentation();
20515c14a3dSBrian Vazquez maybe_wait_bpf_programs(map);
20615c14a3dSBrian Vazquez
20715c14a3dSBrian Vazquez return err;
20815c14a3dSBrian Vazquez }
20915c14a3dSBrian Vazquez
bpf_map_copy_value(struct bpf_map * map,void * key,void * value,__u64 flags)21015c14a3dSBrian Vazquez static int bpf_map_copy_value(struct bpf_map *map, void *key, void *value,
21115c14a3dSBrian Vazquez __u64 flags)
21215c14a3dSBrian Vazquez {
21315c14a3dSBrian Vazquez void *ptr;
21415c14a3dSBrian Vazquez int err;
21515c14a3dSBrian Vazquez
2169d03ebc7SStanislav Fomichev if (bpf_map_is_offloaded(map))
217cb4d03abSBrian Vazquez return bpf_map_offload_lookup_elem(map, key, value);
21815c14a3dSBrian Vazquez
219b6e5dae1SThomas Gleixner bpf_disable_instrumentation();
22015c14a3dSBrian Vazquez if (map->map_type == BPF_MAP_TYPE_PERCPU_HASH ||
22115c14a3dSBrian Vazquez map->map_type == BPF_MAP_TYPE_LRU_PERCPU_HASH) {
22215c14a3dSBrian Vazquez err = bpf_percpu_hash_copy(map, key, value);
22315c14a3dSBrian Vazquez } else if (map->map_type == BPF_MAP_TYPE_PERCPU_ARRAY) {
22415c14a3dSBrian Vazquez err = bpf_percpu_array_copy(map, key, value);
22515c14a3dSBrian Vazquez } else if (map->map_type == BPF_MAP_TYPE_PERCPU_CGROUP_STORAGE) {
22615c14a3dSBrian Vazquez err = bpf_percpu_cgroup_storage_copy(map, key, value);
22715c14a3dSBrian Vazquez } else if (map->map_type == BPF_MAP_TYPE_STACK_TRACE) {
22815c14a3dSBrian Vazquez err = bpf_stackmap_copy(map, key, value);
22915c14a3dSBrian Vazquez } else if (IS_FD_ARRAY(map) || IS_FD_PROG_ARRAY(map)) {
23015c14a3dSBrian Vazquez err = bpf_fd_array_map_lookup_elem(map, key, value);
23115c14a3dSBrian Vazquez } else if (IS_FD_HASH(map)) {
23215c14a3dSBrian Vazquez err = bpf_fd_htab_map_lookup_elem(map, key, value);
23315c14a3dSBrian Vazquez } else if (map->map_type == BPF_MAP_TYPE_REUSEPORT_SOCKARRAY) {
23415c14a3dSBrian Vazquez err = bpf_fd_reuseport_array_lookup_elem(map, key, value);
23515c14a3dSBrian Vazquez } else if (map->map_type == BPF_MAP_TYPE_QUEUE ||
2369330986cSJoanne Koong map->map_type == BPF_MAP_TYPE_STACK ||
2379330986cSJoanne Koong map->map_type == BPF_MAP_TYPE_BLOOM_FILTER) {
23815c14a3dSBrian Vazquez err = map->ops->map_peek_elem(map, value);
23915c14a3dSBrian Vazquez } else if (map->map_type == BPF_MAP_TYPE_STRUCT_OPS) {
24015c14a3dSBrian Vazquez /* struct_ops map requires directly updating "value" */
24115c14a3dSBrian Vazquez err = bpf_struct_ops_map_sys_lookup_elem(map, key, value);
24215c14a3dSBrian Vazquez } else {
24315c14a3dSBrian Vazquez rcu_read_lock();
24415c14a3dSBrian Vazquez if (map->ops->map_lookup_elem_sys_only)
24515c14a3dSBrian Vazquez ptr = map->ops->map_lookup_elem_sys_only(map, key);
24615c14a3dSBrian Vazquez else
24715c14a3dSBrian Vazquez ptr = map->ops->map_lookup_elem(map, key);
24815c14a3dSBrian Vazquez if (IS_ERR(ptr)) {
24915c14a3dSBrian Vazquez err = PTR_ERR(ptr);
25015c14a3dSBrian Vazquez } else if (!ptr) {
25115c14a3dSBrian Vazquez err = -ENOENT;
25215c14a3dSBrian Vazquez } else {
25315c14a3dSBrian Vazquez err = 0;
25415c14a3dSBrian Vazquez if (flags & BPF_F_LOCK)
25515c14a3dSBrian Vazquez /* lock 'ptr' and copy everything but lock */
25615c14a3dSBrian Vazquez copy_map_value_locked(map, value, ptr, true);
25715c14a3dSBrian Vazquez else
25815c14a3dSBrian Vazquez copy_map_value(map, value, ptr);
25968134668SAlexei Starovoitov /* mask lock and timer, since value wasn't zero inited */
26068134668SAlexei Starovoitov check_and_init_map_value(map, value);
26115c14a3dSBrian Vazquez }
26215c14a3dSBrian Vazquez rcu_read_unlock();
26315c14a3dSBrian Vazquez }
26415c14a3dSBrian Vazquez
265b6e5dae1SThomas Gleixner bpf_enable_instrumentation();
26615c14a3dSBrian Vazquez maybe_wait_bpf_programs(map);
26715c14a3dSBrian Vazquez
26815c14a3dSBrian Vazquez return err;
26915c14a3dSBrian Vazquez }
27015c14a3dSBrian Vazquez
271d5299b67SRoman Gushchin /* Please, do not use this function outside from the map creation path
272d5299b67SRoman Gushchin * (e.g. in map update path) without taking care of setting the active
273d5299b67SRoman Gushchin * memory cgroup (see at bpf_map_kmalloc_node() for example).
274d5299b67SRoman Gushchin */
__bpf_map_area_alloc(u64 size,int numa_node,bool mmapable)275196e8ca7SDaniel Borkmann static void *__bpf_map_area_alloc(u64 size, int numa_node, bool mmapable)
276d407bd25SDaniel Borkmann {
277f01a7dbeSMartynas Pumputis /* We really just want to fail instead of triggering OOM killer
278f01a7dbeSMartynas Pumputis * under memory pressure, therefore we set __GFP_NORETRY to kmalloc,
279f01a7dbeSMartynas Pumputis * which is used for lower order allocation requests.
280f01a7dbeSMartynas Pumputis *
281f01a7dbeSMartynas Pumputis * It has been observed that higher order allocation requests done by
282f01a7dbeSMartynas Pumputis * vmalloc with __GFP_NORETRY being set might fail due to not trying
283f01a7dbeSMartynas Pumputis * to reclaim memory from the page cache, thus we set
284f01a7dbeSMartynas Pumputis * __GFP_RETRY_MAYFAIL to avoid such situations.
285d407bd25SDaniel Borkmann */
286f01a7dbeSMartynas Pumputis
287ee53cbfbSYafang Shao gfp_t gfp = bpf_memcg_flags(__GFP_NOWARN | __GFP_ZERO);
288041de93fSChristoph Hellwig unsigned int flags = 0;
289041de93fSChristoph Hellwig unsigned long align = 1;
290d407bd25SDaniel Borkmann void *area;
291d407bd25SDaniel Borkmann
292196e8ca7SDaniel Borkmann if (size >= SIZE_MAX)
293196e8ca7SDaniel Borkmann return NULL;
294196e8ca7SDaniel Borkmann
295fc970227SAndrii Nakryiko /* kmalloc()'ed memory can't be mmap()'ed */
296041de93fSChristoph Hellwig if (mmapable) {
297041de93fSChristoph Hellwig BUG_ON(!PAGE_ALIGNED(size));
298041de93fSChristoph Hellwig align = SHMLBA;
299041de93fSChristoph Hellwig flags = VM_USERMAP;
300041de93fSChristoph Hellwig } else if (size <= (PAGE_SIZE << PAGE_ALLOC_COSTLY_ORDER)) {
301041de93fSChristoph Hellwig area = kmalloc_node(size, gfp | GFP_USER | __GFP_NORETRY,
302f01a7dbeSMartynas Pumputis numa_node);
303d407bd25SDaniel Borkmann if (area != NULL)
304d407bd25SDaniel Borkmann return area;
305d407bd25SDaniel Borkmann }
306041de93fSChristoph Hellwig
307041de93fSChristoph Hellwig return __vmalloc_node_range(size, align, VMALLOC_START, VMALLOC_END,
308041de93fSChristoph Hellwig gfp | GFP_KERNEL | __GFP_RETRY_MAYFAIL, PAGE_KERNEL,
309041de93fSChristoph Hellwig flags, numa_node, __builtin_return_address(0));
310d407bd25SDaniel Borkmann }
311d407bd25SDaniel Borkmann
bpf_map_area_alloc(u64 size,int numa_node)312196e8ca7SDaniel Borkmann void *bpf_map_area_alloc(u64 size, int numa_node)
313fc970227SAndrii Nakryiko {
314fc970227SAndrii Nakryiko return __bpf_map_area_alloc(size, numa_node, false);
315fc970227SAndrii Nakryiko }
316fc970227SAndrii Nakryiko
bpf_map_area_mmapable_alloc(u64 size,int numa_node)317196e8ca7SDaniel Borkmann void *bpf_map_area_mmapable_alloc(u64 size, int numa_node)
318fc970227SAndrii Nakryiko {
319fc970227SAndrii Nakryiko return __bpf_map_area_alloc(size, numa_node, true);
320fc970227SAndrii Nakryiko }
321fc970227SAndrii Nakryiko
bpf_map_area_free(void * area)322d407bd25SDaniel Borkmann void bpf_map_area_free(void *area)
323d407bd25SDaniel Borkmann {
324d407bd25SDaniel Borkmann kvfree(area);
325d407bd25SDaniel Borkmann }
326d407bd25SDaniel Borkmann
bpf_map_flags_retain_permanent(u32 flags)327be70bcd5SDaniel Borkmann static u32 bpf_map_flags_retain_permanent(u32 flags)
328be70bcd5SDaniel Borkmann {
329be70bcd5SDaniel Borkmann /* Some map creation flags are not tied to the map object but
330be70bcd5SDaniel Borkmann * rather to the map fd instead, so they have no meaning upon
331be70bcd5SDaniel Borkmann * map object inspection since multiple file descriptors with
332be70bcd5SDaniel Borkmann * different (access) properties can exist here. Thus, given
333be70bcd5SDaniel Borkmann * this has zero meaning for the map itself, lets clear these
334be70bcd5SDaniel Borkmann * from here.
335be70bcd5SDaniel Borkmann */
336be70bcd5SDaniel Borkmann return flags & ~(BPF_F_RDONLY | BPF_F_WRONLY);
337be70bcd5SDaniel Borkmann }
338be70bcd5SDaniel Borkmann
bpf_map_init_from_attr(struct bpf_map * map,union bpf_attr * attr)339bd475643SJakub Kicinski void bpf_map_init_from_attr(struct bpf_map *map, union bpf_attr *attr)
340bd475643SJakub Kicinski {
341bd475643SJakub Kicinski map->map_type = attr->map_type;
342bd475643SJakub Kicinski map->key_size = attr->key_size;
343bd475643SJakub Kicinski map->value_size = attr->value_size;
344bd475643SJakub Kicinski map->max_entries = attr->max_entries;
345be70bcd5SDaniel Borkmann map->map_flags = bpf_map_flags_retain_permanent(attr->map_flags);
346bd475643SJakub Kicinski map->numa_node = bpf_map_attr_numa_node(attr);
3479330986cSJoanne Koong map->map_extra = attr->map_extra;
348bd475643SJakub Kicinski }
349bd475643SJakub Kicinski
bpf_map_alloc_id(struct bpf_map * map)350f3f1c054SMartin KaFai Lau static int bpf_map_alloc_id(struct bpf_map *map)
351f3f1c054SMartin KaFai Lau {
352f3f1c054SMartin KaFai Lau int id;
353f3f1c054SMartin KaFai Lau
354b76354cdSShaohua Li idr_preload(GFP_KERNEL);
355f3f1c054SMartin KaFai Lau spin_lock_bh(&map_idr_lock);
356f3f1c054SMartin KaFai Lau id = idr_alloc_cyclic(&map_idr, map, 1, INT_MAX, GFP_ATOMIC);
357f3f1c054SMartin KaFai Lau if (id > 0)
358f3f1c054SMartin KaFai Lau map->id = id;
359f3f1c054SMartin KaFai Lau spin_unlock_bh(&map_idr_lock);
360b76354cdSShaohua Li idr_preload_end();
361f3f1c054SMartin KaFai Lau
362f3f1c054SMartin KaFai Lau if (WARN_ON_ONCE(!id))
363f3f1c054SMartin KaFai Lau return -ENOSPC;
364f3f1c054SMartin KaFai Lau
365f3f1c054SMartin KaFai Lau return id > 0 ? 0 : id;
366f3f1c054SMartin KaFai Lau }
367f3f1c054SMartin KaFai Lau
bpf_map_free_id(struct bpf_map * map)368158e5e9eSTobias Klauser void bpf_map_free_id(struct bpf_map *map)
369f3f1c054SMartin KaFai Lau {
370930651a7SEric Dumazet unsigned long flags;
371930651a7SEric Dumazet
372a3884572SJakub Kicinski /* Offloaded maps are removed from the IDR store when their device
373a3884572SJakub Kicinski * disappears - even if someone holds an fd to them they are unusable,
374a3884572SJakub Kicinski * the memory is gone, all ops will fail; they are simply waiting for
375a3884572SJakub Kicinski * refcnt to drop to be freed.
376a3884572SJakub Kicinski */
377a3884572SJakub Kicinski if (!map->id)
378a3884572SJakub Kicinski return;
379a3884572SJakub Kicinski
380930651a7SEric Dumazet spin_lock_irqsave(&map_idr_lock, flags);
381bd5f5f4eSMartin KaFai Lau
382f3f1c054SMartin KaFai Lau idr_remove(&map_idr, map->id);
383a3884572SJakub Kicinski map->id = 0;
384bd5f5f4eSMartin KaFai Lau
385930651a7SEric Dumazet spin_unlock_irqrestore(&map_idr_lock, flags);
386f3f1c054SMartin KaFai Lau }
387f3f1c054SMartin KaFai Lau
38848edc1f7SRoman Gushchin #ifdef CONFIG_MEMCG_KMEM
bpf_map_save_memcg(struct bpf_map * map)38948edc1f7SRoman Gushchin static void bpf_map_save_memcg(struct bpf_map *map)
39048edc1f7SRoman Gushchin {
3914201d9abSRoman Gushchin /* Currently if a map is created by a process belonging to the root
3924201d9abSRoman Gushchin * memory cgroup, get_obj_cgroup_from_current() will return NULL.
3934201d9abSRoman Gushchin * So we have to check map->objcg for being NULL each time it's
3944201d9abSRoman Gushchin * being used.
3954201d9abSRoman Gushchin */
396ee53cbfbSYafang Shao if (memcg_bpf_enabled())
3974201d9abSRoman Gushchin map->objcg = get_obj_cgroup_from_current();
39848edc1f7SRoman Gushchin }
39948edc1f7SRoman Gushchin
bpf_map_release_memcg(struct bpf_map * map)40048edc1f7SRoman Gushchin static void bpf_map_release_memcg(struct bpf_map *map)
40148edc1f7SRoman Gushchin {
4024201d9abSRoman Gushchin if (map->objcg)
4034201d9abSRoman Gushchin obj_cgroup_put(map->objcg);
4044201d9abSRoman Gushchin }
4054201d9abSRoman Gushchin
bpf_map_get_memcg(const struct bpf_map * map)4064201d9abSRoman Gushchin static struct mem_cgroup *bpf_map_get_memcg(const struct bpf_map *map)
4074201d9abSRoman Gushchin {
4084201d9abSRoman Gushchin if (map->objcg)
4094201d9abSRoman Gushchin return get_mem_cgroup_from_objcg(map->objcg);
4104201d9abSRoman Gushchin
4114201d9abSRoman Gushchin return root_mem_cgroup;
41248edc1f7SRoman Gushchin }
41348edc1f7SRoman Gushchin
bpf_map_kmalloc_node(const struct bpf_map * map,size_t size,gfp_t flags,int node)41448edc1f7SRoman Gushchin void *bpf_map_kmalloc_node(const struct bpf_map *map, size_t size, gfp_t flags,
41548edc1f7SRoman Gushchin int node)
41648edc1f7SRoman Gushchin {
4174201d9abSRoman Gushchin struct mem_cgroup *memcg, *old_memcg;
41848edc1f7SRoman Gushchin void *ptr;
41948edc1f7SRoman Gushchin
4204201d9abSRoman Gushchin memcg = bpf_map_get_memcg(map);
4214201d9abSRoman Gushchin old_memcg = set_active_memcg(memcg);
42248edc1f7SRoman Gushchin ptr = kmalloc_node(size, flags | __GFP_ACCOUNT, node);
42348edc1f7SRoman Gushchin set_active_memcg(old_memcg);
4244201d9abSRoman Gushchin mem_cgroup_put(memcg);
42548edc1f7SRoman Gushchin
42648edc1f7SRoman Gushchin return ptr;
42748edc1f7SRoman Gushchin }
42848edc1f7SRoman Gushchin
bpf_map_kzalloc(const struct bpf_map * map,size_t size,gfp_t flags)42948edc1f7SRoman Gushchin void *bpf_map_kzalloc(const struct bpf_map *map, size_t size, gfp_t flags)
43048edc1f7SRoman Gushchin {
4314201d9abSRoman Gushchin struct mem_cgroup *memcg, *old_memcg;
43248edc1f7SRoman Gushchin void *ptr;
43348edc1f7SRoman Gushchin
4344201d9abSRoman Gushchin memcg = bpf_map_get_memcg(map);
4354201d9abSRoman Gushchin old_memcg = set_active_memcg(memcg);
43648edc1f7SRoman Gushchin ptr = kzalloc(size, flags | __GFP_ACCOUNT);
43748edc1f7SRoman Gushchin set_active_memcg(old_memcg);
4384201d9abSRoman Gushchin mem_cgroup_put(memcg);
43948edc1f7SRoman Gushchin
44048edc1f7SRoman Gushchin return ptr;
44148edc1f7SRoman Gushchin }
44248edc1f7SRoman Gushchin
bpf_map_kvcalloc(struct bpf_map * map,size_t n,size_t size,gfp_t flags)443ddef81b5SYafang Shao void *bpf_map_kvcalloc(struct bpf_map *map, size_t n, size_t size,
444ddef81b5SYafang Shao gfp_t flags)
445ddef81b5SYafang Shao {
446ddef81b5SYafang Shao struct mem_cgroup *memcg, *old_memcg;
447ddef81b5SYafang Shao void *ptr;
448ddef81b5SYafang Shao
449ddef81b5SYafang Shao memcg = bpf_map_get_memcg(map);
450ddef81b5SYafang Shao old_memcg = set_active_memcg(memcg);
451ddef81b5SYafang Shao ptr = kvcalloc(n, size, flags | __GFP_ACCOUNT);
452ddef81b5SYafang Shao set_active_memcg(old_memcg);
453ddef81b5SYafang Shao mem_cgroup_put(memcg);
454ddef81b5SYafang Shao
455ddef81b5SYafang Shao return ptr;
456ddef81b5SYafang Shao }
457ddef81b5SYafang Shao
bpf_map_alloc_percpu(const struct bpf_map * map,size_t size,size_t align,gfp_t flags)45848edc1f7SRoman Gushchin void __percpu *bpf_map_alloc_percpu(const struct bpf_map *map, size_t size,
45948edc1f7SRoman Gushchin size_t align, gfp_t flags)
46048edc1f7SRoman Gushchin {
4614201d9abSRoman Gushchin struct mem_cgroup *memcg, *old_memcg;
46248edc1f7SRoman Gushchin void __percpu *ptr;
46348edc1f7SRoman Gushchin
4644201d9abSRoman Gushchin memcg = bpf_map_get_memcg(map);
4654201d9abSRoman Gushchin old_memcg = set_active_memcg(memcg);
46648edc1f7SRoman Gushchin ptr = __alloc_percpu_gfp(size, align, flags | __GFP_ACCOUNT);
46748edc1f7SRoman Gushchin set_active_memcg(old_memcg);
4684201d9abSRoman Gushchin mem_cgroup_put(memcg);
46948edc1f7SRoman Gushchin
47048edc1f7SRoman Gushchin return ptr;
47148edc1f7SRoman Gushchin }
47248edc1f7SRoman Gushchin
47348edc1f7SRoman Gushchin #else
bpf_map_save_memcg(struct bpf_map * map)47448edc1f7SRoman Gushchin static void bpf_map_save_memcg(struct bpf_map *map)
47548edc1f7SRoman Gushchin {
47648edc1f7SRoman Gushchin }
47748edc1f7SRoman Gushchin
bpf_map_release_memcg(struct bpf_map * map)47848edc1f7SRoman Gushchin static void bpf_map_release_memcg(struct bpf_map *map)
47948edc1f7SRoman Gushchin {
48048edc1f7SRoman Gushchin }
48148edc1f7SRoman Gushchin #endif
48248edc1f7SRoman Gushchin
btf_field_cmp(const void * a,const void * b)483aa3496acSKumar Kartikeya Dwivedi static int btf_field_cmp(const void *a, const void *b)
48461df10c7SKumar Kartikeya Dwivedi {
485aa3496acSKumar Kartikeya Dwivedi const struct btf_field *f1 = a, *f2 = b;
48661df10c7SKumar Kartikeya Dwivedi
487aa3496acSKumar Kartikeya Dwivedi if (f1->offset < f2->offset)
48861df10c7SKumar Kartikeya Dwivedi return -1;
489aa3496acSKumar Kartikeya Dwivedi else if (f1->offset > f2->offset)
49061df10c7SKumar Kartikeya Dwivedi return 1;
49161df10c7SKumar Kartikeya Dwivedi return 0;
49261df10c7SKumar Kartikeya Dwivedi }
49361df10c7SKumar Kartikeya Dwivedi
btf_record_find(const struct btf_record * rec,u32 offset,u32 field_mask)494aa3496acSKumar Kartikeya Dwivedi struct btf_field *btf_record_find(const struct btf_record *rec, u32 offset,
49574843b57SDave Marchevsky u32 field_mask)
49661df10c7SKumar Kartikeya Dwivedi {
497aa3496acSKumar Kartikeya Dwivedi struct btf_field *field;
49861df10c7SKumar Kartikeya Dwivedi
49974843b57SDave Marchevsky if (IS_ERR_OR_NULL(rec) || !(rec->field_mask & field_mask))
50061df10c7SKumar Kartikeya Dwivedi return NULL;
501aa3496acSKumar Kartikeya Dwivedi field = bsearch(&offset, rec->fields, rec->cnt, sizeof(rec->fields[0]), btf_field_cmp);
50274843b57SDave Marchevsky if (!field || !(field->type & field_mask))
503aa3496acSKumar Kartikeya Dwivedi return NULL;
504aa3496acSKumar Kartikeya Dwivedi return field;
50561df10c7SKumar Kartikeya Dwivedi }
50661df10c7SKumar Kartikeya Dwivedi
btf_record_free(struct btf_record * rec)507aa3496acSKumar Kartikeya Dwivedi void btf_record_free(struct btf_record *rec)
50861df10c7SKumar Kartikeya Dwivedi {
50961df10c7SKumar Kartikeya Dwivedi int i;
51061df10c7SKumar Kartikeya Dwivedi
511aa3496acSKumar Kartikeya Dwivedi if (IS_ERR_OR_NULL(rec))
51261df10c7SKumar Kartikeya Dwivedi return;
513aa3496acSKumar Kartikeya Dwivedi for (i = 0; i < rec->cnt; i++) {
514aa3496acSKumar Kartikeya Dwivedi switch (rec->fields[i].type) {
515aa3496acSKumar Kartikeya Dwivedi case BPF_KPTR_UNREF:
516aa3496acSKumar Kartikeya Dwivedi case BPF_KPTR_REF:
517aa3496acSKumar Kartikeya Dwivedi if (rec->fields[i].kptr.module)
518aa3496acSKumar Kartikeya Dwivedi module_put(rec->fields[i].kptr.module);
519aa3496acSKumar Kartikeya Dwivedi btf_put(rec->fields[i].kptr.btf);
520aa3496acSKumar Kartikeya Dwivedi break;
521f0c5941fSKumar Kartikeya Dwivedi case BPF_LIST_HEAD:
5228ffa5cc1SKumar Kartikeya Dwivedi case BPF_LIST_NODE:
5239c395c1bSDave Marchevsky case BPF_RB_ROOT:
5249c395c1bSDave Marchevsky case BPF_RB_NODE:
5259c395c1bSDave Marchevsky case BPF_SPIN_LOCK:
5269c395c1bSDave Marchevsky case BPF_TIMER:
527d54730b5SDave Marchevsky case BPF_REFCOUNT:
5289c395c1bSDave Marchevsky /* Nothing to release */
529f0c5941fSKumar Kartikeya Dwivedi break;
530aa3496acSKumar Kartikeya Dwivedi default:
531aa3496acSKumar Kartikeya Dwivedi WARN_ON_ONCE(1);
53214a324f6SKumar Kartikeya Dwivedi continue;
53314a324f6SKumar Kartikeya Dwivedi }
534aa3496acSKumar Kartikeya Dwivedi }
535aa3496acSKumar Kartikeya Dwivedi kfree(rec);
536aa3496acSKumar Kartikeya Dwivedi }
537aa3496acSKumar Kartikeya Dwivedi
bpf_map_free_record(struct bpf_map * map)538aa3496acSKumar Kartikeya Dwivedi void bpf_map_free_record(struct bpf_map *map)
539aa3496acSKumar Kartikeya Dwivedi {
540aa3496acSKumar Kartikeya Dwivedi btf_record_free(map->record);
541aa3496acSKumar Kartikeya Dwivedi map->record = NULL;
542aa3496acSKumar Kartikeya Dwivedi }
543aa3496acSKumar Kartikeya Dwivedi
btf_record_dup(const struct btf_record * rec)544aa3496acSKumar Kartikeya Dwivedi struct btf_record *btf_record_dup(const struct btf_record *rec)
545aa3496acSKumar Kartikeya Dwivedi {
546aa3496acSKumar Kartikeya Dwivedi const struct btf_field *fields;
547aa3496acSKumar Kartikeya Dwivedi struct btf_record *new_rec;
548aa3496acSKumar Kartikeya Dwivedi int ret, size, i;
549aa3496acSKumar Kartikeya Dwivedi
550aa3496acSKumar Kartikeya Dwivedi if (IS_ERR_OR_NULL(rec))
551aa3496acSKumar Kartikeya Dwivedi return NULL;
552aa3496acSKumar Kartikeya Dwivedi size = offsetof(struct btf_record, fields[rec->cnt]);
553aa3496acSKumar Kartikeya Dwivedi new_rec = kmemdup(rec, size, GFP_KERNEL | __GFP_NOWARN);
554aa3496acSKumar Kartikeya Dwivedi if (!new_rec)
555aa3496acSKumar Kartikeya Dwivedi return ERR_PTR(-ENOMEM);
556aa3496acSKumar Kartikeya Dwivedi /* Do a deep copy of the btf_record */
557aa3496acSKumar Kartikeya Dwivedi fields = rec->fields;
558aa3496acSKumar Kartikeya Dwivedi new_rec->cnt = 0;
559aa3496acSKumar Kartikeya Dwivedi for (i = 0; i < rec->cnt; i++) {
560aa3496acSKumar Kartikeya Dwivedi switch (fields[i].type) {
561aa3496acSKumar Kartikeya Dwivedi case BPF_KPTR_UNREF:
562aa3496acSKumar Kartikeya Dwivedi case BPF_KPTR_REF:
563aa3496acSKumar Kartikeya Dwivedi btf_get(fields[i].kptr.btf);
564aa3496acSKumar Kartikeya Dwivedi if (fields[i].kptr.module && !try_module_get(fields[i].kptr.module)) {
565aa3496acSKumar Kartikeya Dwivedi ret = -ENXIO;
566aa3496acSKumar Kartikeya Dwivedi goto free;
567aa3496acSKumar Kartikeya Dwivedi }
568aa3496acSKumar Kartikeya Dwivedi break;
569f0c5941fSKumar Kartikeya Dwivedi case BPF_LIST_HEAD:
5708ffa5cc1SKumar Kartikeya Dwivedi case BPF_LIST_NODE:
5719c395c1bSDave Marchevsky case BPF_RB_ROOT:
5729c395c1bSDave Marchevsky case BPF_RB_NODE:
5739c395c1bSDave Marchevsky case BPF_SPIN_LOCK:
5749c395c1bSDave Marchevsky case BPF_TIMER:
575d54730b5SDave Marchevsky case BPF_REFCOUNT:
5769c395c1bSDave Marchevsky /* Nothing to acquire */
577f0c5941fSKumar Kartikeya Dwivedi break;
578aa3496acSKumar Kartikeya Dwivedi default:
579aa3496acSKumar Kartikeya Dwivedi ret = -EFAULT;
580aa3496acSKumar Kartikeya Dwivedi WARN_ON_ONCE(1);
581aa3496acSKumar Kartikeya Dwivedi goto free;
582aa3496acSKumar Kartikeya Dwivedi }
583aa3496acSKumar Kartikeya Dwivedi new_rec->cnt++;
584aa3496acSKumar Kartikeya Dwivedi }
585aa3496acSKumar Kartikeya Dwivedi return new_rec;
586aa3496acSKumar Kartikeya Dwivedi free:
587aa3496acSKumar Kartikeya Dwivedi btf_record_free(new_rec);
588aa3496acSKumar Kartikeya Dwivedi return ERR_PTR(ret);
589aa3496acSKumar Kartikeya Dwivedi }
590aa3496acSKumar Kartikeya Dwivedi
btf_record_equal(const struct btf_record * rec_a,const struct btf_record * rec_b)591aa3496acSKumar Kartikeya Dwivedi bool btf_record_equal(const struct btf_record *rec_a, const struct btf_record *rec_b)
592aa3496acSKumar Kartikeya Dwivedi {
593aa3496acSKumar Kartikeya Dwivedi bool a_has_fields = !IS_ERR_OR_NULL(rec_a), b_has_fields = !IS_ERR_OR_NULL(rec_b);
594aa3496acSKumar Kartikeya Dwivedi int size;
595aa3496acSKumar Kartikeya Dwivedi
596aa3496acSKumar Kartikeya Dwivedi if (!a_has_fields && !b_has_fields)
597aa3496acSKumar Kartikeya Dwivedi return true;
598aa3496acSKumar Kartikeya Dwivedi if (a_has_fields != b_has_fields)
599aa3496acSKumar Kartikeya Dwivedi return false;
600aa3496acSKumar Kartikeya Dwivedi if (rec_a->cnt != rec_b->cnt)
601aa3496acSKumar Kartikeya Dwivedi return false;
602aa3496acSKumar Kartikeya Dwivedi size = offsetof(struct btf_record, fields[rec_a->cnt]);
603c22dfdd2SKumar Kartikeya Dwivedi /* btf_parse_fields uses kzalloc to allocate a btf_record, so unused
604c22dfdd2SKumar Kartikeya Dwivedi * members are zeroed out. So memcmp is safe to do without worrying
605c22dfdd2SKumar Kartikeya Dwivedi * about padding/unused fields.
606c22dfdd2SKumar Kartikeya Dwivedi *
607c22dfdd2SKumar Kartikeya Dwivedi * While spin_lock, timer, and kptr have no relation to map BTF,
608c22dfdd2SKumar Kartikeya Dwivedi * list_head metadata is specific to map BTF, the btf and value_rec
609c22dfdd2SKumar Kartikeya Dwivedi * members in particular. btf is the map BTF, while value_rec points to
610c22dfdd2SKumar Kartikeya Dwivedi * btf_record in that map BTF.
611c22dfdd2SKumar Kartikeya Dwivedi *
612c22dfdd2SKumar Kartikeya Dwivedi * So while by default, we don't rely on the map BTF (which the records
613c22dfdd2SKumar Kartikeya Dwivedi * were parsed from) matching for both records, which is not backwards
614c22dfdd2SKumar Kartikeya Dwivedi * compatible, in case list_head is part of it, we implicitly rely on
615c22dfdd2SKumar Kartikeya Dwivedi * that by way of depending on memcmp succeeding for it.
616c22dfdd2SKumar Kartikeya Dwivedi */
617aa3496acSKumar Kartikeya Dwivedi return !memcmp(rec_a, rec_b, size);
618aa3496acSKumar Kartikeya Dwivedi }
619aa3496acSKumar Kartikeya Dwivedi
bpf_obj_free_timer(const struct btf_record * rec,void * obj)620db559117SKumar Kartikeya Dwivedi void bpf_obj_free_timer(const struct btf_record *rec, void *obj)
621db559117SKumar Kartikeya Dwivedi {
622db559117SKumar Kartikeya Dwivedi if (WARN_ON_ONCE(!btf_record_has_field(rec, BPF_TIMER)))
623db559117SKumar Kartikeya Dwivedi return;
624db559117SKumar Kartikeya Dwivedi bpf_timer_cancel_and_free(obj + rec->timer_off);
625db559117SKumar Kartikeya Dwivedi }
626db559117SKumar Kartikeya Dwivedi
6279e36a204SDave Marchevsky extern void __bpf_obj_drop_impl(void *p, const struct btf_record *rec);
6289e36a204SDave Marchevsky
bpf_obj_free_fields(const struct btf_record * rec,void * obj)629aa3496acSKumar Kartikeya Dwivedi void bpf_obj_free_fields(const struct btf_record *rec, void *obj)
630aa3496acSKumar Kartikeya Dwivedi {
631aa3496acSKumar Kartikeya Dwivedi const struct btf_field *fields;
632aa3496acSKumar Kartikeya Dwivedi int i;
633aa3496acSKumar Kartikeya Dwivedi
634aa3496acSKumar Kartikeya Dwivedi if (IS_ERR_OR_NULL(rec))
635aa3496acSKumar Kartikeya Dwivedi return;
636aa3496acSKumar Kartikeya Dwivedi fields = rec->fields;
637aa3496acSKumar Kartikeya Dwivedi for (i = 0; i < rec->cnt; i++) {
638c8e18754SDave Marchevsky struct btf_struct_meta *pointee_struct_meta;
639aa3496acSKumar Kartikeya Dwivedi const struct btf_field *field = &fields[i];
640aa3496acSKumar Kartikeya Dwivedi void *field_ptr = obj + field->offset;
641c8e18754SDave Marchevsky void *xchgd_field;
642aa3496acSKumar Kartikeya Dwivedi
643aa3496acSKumar Kartikeya Dwivedi switch (fields[i].type) {
644db559117SKumar Kartikeya Dwivedi case BPF_SPIN_LOCK:
645db559117SKumar Kartikeya Dwivedi break;
646db559117SKumar Kartikeya Dwivedi case BPF_TIMER:
647db559117SKumar Kartikeya Dwivedi bpf_timer_cancel_and_free(field_ptr);
648db559117SKumar Kartikeya Dwivedi break;
649aa3496acSKumar Kartikeya Dwivedi case BPF_KPTR_UNREF:
650aa3496acSKumar Kartikeya Dwivedi WRITE_ONCE(*(u64 *)field_ptr, 0);
651aa3496acSKumar Kartikeya Dwivedi break;
652aa3496acSKumar Kartikeya Dwivedi case BPF_KPTR_REF:
653c8e18754SDave Marchevsky xchgd_field = (void *)xchg((unsigned long *)field_ptr, 0);
6541431d0b5SDavid Vernet if (!xchgd_field)
6551431d0b5SDavid Vernet break;
6561431d0b5SDavid Vernet
657c8e18754SDave Marchevsky if (!btf_is_kernel(field->kptr.btf)) {
658c8e18754SDave Marchevsky pointee_struct_meta = btf_find_struct_meta(field->kptr.btf,
659c8e18754SDave Marchevsky field->kptr.btf_id);
6609e36a204SDave Marchevsky migrate_disable();
6619e36a204SDave Marchevsky __bpf_obj_drop_impl(xchgd_field, pointee_struct_meta ?
662c8e18754SDave Marchevsky pointee_struct_meta->record :
663c8e18754SDave Marchevsky NULL);
6649e36a204SDave Marchevsky migrate_enable();
665c8e18754SDave Marchevsky } else {
666c8e18754SDave Marchevsky field->kptr.dtor(xchgd_field);
667c8e18754SDave Marchevsky }
668aa3496acSKumar Kartikeya Dwivedi break;
669f0c5941fSKumar Kartikeya Dwivedi case BPF_LIST_HEAD:
670f0c5941fSKumar Kartikeya Dwivedi if (WARN_ON_ONCE(rec->spin_lock_off < 0))
671f0c5941fSKumar Kartikeya Dwivedi continue;
672f0c5941fSKumar Kartikeya Dwivedi bpf_list_head_free(field, field_ptr, obj + rec->spin_lock_off);
673f0c5941fSKumar Kartikeya Dwivedi break;
6749c395c1bSDave Marchevsky case BPF_RB_ROOT:
6759c395c1bSDave Marchevsky if (WARN_ON_ONCE(rec->spin_lock_off < 0))
6769c395c1bSDave Marchevsky continue;
6779c395c1bSDave Marchevsky bpf_rb_root_free(field, field_ptr, obj + rec->spin_lock_off);
6789c395c1bSDave Marchevsky break;
6798ffa5cc1SKumar Kartikeya Dwivedi case BPF_LIST_NODE:
6809c395c1bSDave Marchevsky case BPF_RB_NODE:
681d54730b5SDave Marchevsky case BPF_REFCOUNT:
6828ffa5cc1SKumar Kartikeya Dwivedi break;
683aa3496acSKumar Kartikeya Dwivedi default:
684aa3496acSKumar Kartikeya Dwivedi WARN_ON_ONCE(1);
685aa3496acSKumar Kartikeya Dwivedi continue;
686aa3496acSKumar Kartikeya Dwivedi }
68714a324f6SKumar Kartikeya Dwivedi }
68814a324f6SKumar Kartikeya Dwivedi }
68914a324f6SKumar Kartikeya Dwivedi
69099c55f7dSAlexei Starovoitov /* called from workqueue */
bpf_map_free_deferred(struct work_struct * work)69199c55f7dSAlexei Starovoitov static void bpf_map_free_deferred(struct work_struct *work)
69299c55f7dSAlexei Starovoitov {
69399c55f7dSAlexei Starovoitov struct bpf_map *map = container_of(work, struct bpf_map, work);
694d7f5ef65SKumar Kartikeya Dwivedi struct btf_record *rec = map->record;
695d048dcedSYonghong Song struct btf *btf = map->btf;
69699c55f7dSAlexei Starovoitov
697afdb09c7SChenbo Feng security_bpf_map_free(map);
69848edc1f7SRoman Gushchin bpf_map_release_memcg(map);
699d7f5ef65SKumar Kartikeya Dwivedi /* implementation dependent freeing */
70099c55f7dSAlexei Starovoitov map->ops->map_free(map);
701cd2a8079SDave Marchevsky /* Delay freeing of btf_record for maps, as map_free
702d7f5ef65SKumar Kartikeya Dwivedi * callback usually needs access to them. It is better to do it here
703d7f5ef65SKumar Kartikeya Dwivedi * than require each callback to do the free itself manually.
704d7f5ef65SKumar Kartikeya Dwivedi *
705d7f5ef65SKumar Kartikeya Dwivedi * Note that the btf_record stashed in map->inner_map_meta->record was
706d7f5ef65SKumar Kartikeya Dwivedi * already freed using the map_free callback for map in map case which
707d7f5ef65SKumar Kartikeya Dwivedi * eventually calls bpf_map_free_meta, since inner_map_meta is only a
708d7f5ef65SKumar Kartikeya Dwivedi * template bpf_map struct used during verification.
709d7f5ef65SKumar Kartikeya Dwivedi */
710d7f5ef65SKumar Kartikeya Dwivedi btf_record_free(rec);
711d048dcedSYonghong Song /* Delay freeing of btf for maps, as map_free callback may need
712d048dcedSYonghong Song * struct_meta info which will be freed with btf_put().
713d048dcedSYonghong Song */
714d048dcedSYonghong Song btf_put(btf);
71599c55f7dSAlexei Starovoitov }
71699c55f7dSAlexei Starovoitov
bpf_map_put_uref(struct bpf_map * map)717c9da161cSDaniel Borkmann static void bpf_map_put_uref(struct bpf_map *map)
718c9da161cSDaniel Borkmann {
7191e0bd5a0SAndrii Nakryiko if (atomic64_dec_and_test(&map->usercnt)) {
720ba6b8de4SJohn Fastabend if (map->ops->map_release_uref)
721ba6b8de4SJohn Fastabend map->ops->map_release_uref(map);
722c9da161cSDaniel Borkmann }
723c9da161cSDaniel Borkmann }
724c9da161cSDaniel Borkmann
bpf_map_free_in_work(struct bpf_map * map)725f91cd728SHou Tao static void bpf_map_free_in_work(struct bpf_map *map)
726f91cd728SHou Tao {
727f91cd728SHou Tao INIT_WORK(&map->work, bpf_map_free_deferred);
728f91cd728SHou Tao /* Avoid spawning kworkers, since they all might contend
729f91cd728SHou Tao * for the same mutex like slab_mutex.
730f91cd728SHou Tao */
731f91cd728SHou Tao queue_work(system_unbound_wq, &map->work);
732f91cd728SHou Tao }
733f91cd728SHou Tao
bpf_map_free_rcu_gp(struct rcu_head * rcu)734f91cd728SHou Tao static void bpf_map_free_rcu_gp(struct rcu_head *rcu)
735f91cd728SHou Tao {
736f91cd728SHou Tao bpf_map_free_in_work(container_of(rcu, struct bpf_map, rcu));
737f91cd728SHou Tao }
738f91cd728SHou Tao
bpf_map_free_mult_rcu_gp(struct rcu_head * rcu)739f91cd728SHou Tao static void bpf_map_free_mult_rcu_gp(struct rcu_head *rcu)
740f91cd728SHou Tao {
741f91cd728SHou Tao if (rcu_trace_implies_rcu_gp())
742f91cd728SHou Tao bpf_map_free_rcu_gp(rcu);
743f91cd728SHou Tao else
744f91cd728SHou Tao call_rcu(rcu, bpf_map_free_rcu_gp);
745f91cd728SHou Tao }
746f91cd728SHou Tao
74799c55f7dSAlexei Starovoitov /* decrement map refcnt and schedule it for freeing via workqueue
748158e5e9eSTobias Klauser * (underlying map implementation ops->map_free() might sleep)
74999c55f7dSAlexei Starovoitov */
bpf_map_put(struct bpf_map * map)750158e5e9eSTobias Klauser void bpf_map_put(struct bpf_map *map)
75199c55f7dSAlexei Starovoitov {
7521e0bd5a0SAndrii Nakryiko if (atomic64_dec_and_test(&map->refcnt)) {
75334ad5580SMartin KaFai Lau /* bpf_map_free_id() must be called first */
754158e5e9eSTobias Klauser bpf_map_free_id(map);
755f91cd728SHou Tao
756f91cd728SHou Tao if (READ_ONCE(map->free_after_mult_rcu_gp))
757f91cd728SHou Tao call_rcu_tasks_trace(&map->rcu, bpf_map_free_mult_rcu_gp);
758f91cd728SHou Tao else
759f91cd728SHou Tao bpf_map_free_in_work(map);
76099c55f7dSAlexei Starovoitov }
76199c55f7dSAlexei Starovoitov }
762630a4d38SJakub Kicinski EXPORT_SYMBOL_GPL(bpf_map_put);
763bd5f5f4eSMartin KaFai Lau
bpf_map_put_with_uref(struct bpf_map * map)764c9da161cSDaniel Borkmann void bpf_map_put_with_uref(struct bpf_map *map)
765c9da161cSDaniel Borkmann {
766c9da161cSDaniel Borkmann bpf_map_put_uref(map);
767c9da161cSDaniel Borkmann bpf_map_put(map);
768c9da161cSDaniel Borkmann }
769c9da161cSDaniel Borkmann
bpf_map_release(struct inode * inode,struct file * filp)77099c55f7dSAlexei Starovoitov static int bpf_map_release(struct inode *inode, struct file *filp)
77199c55f7dSAlexei Starovoitov {
77261d1b6a4SDaniel Borkmann struct bpf_map *map = filp->private_data;
77361d1b6a4SDaniel Borkmann
77461d1b6a4SDaniel Borkmann if (map->ops->map_release)
77561d1b6a4SDaniel Borkmann map->ops->map_release(map, filp);
77661d1b6a4SDaniel Borkmann
77761d1b6a4SDaniel Borkmann bpf_map_put_with_uref(map);
77899c55f7dSAlexei Starovoitov return 0;
77999c55f7dSAlexei Starovoitov }
78099c55f7dSAlexei Starovoitov
map_get_sys_perms(struct bpf_map * map,struct fd f)78187df15deSDaniel Borkmann static fmode_t map_get_sys_perms(struct bpf_map *map, struct fd f)
78287df15deSDaniel Borkmann {
78387df15deSDaniel Borkmann fmode_t mode = f.file->f_mode;
78487df15deSDaniel Borkmann
78587df15deSDaniel Borkmann /* Our file permissions may have been overridden by global
78687df15deSDaniel Borkmann * map permissions facing syscall side.
78787df15deSDaniel Borkmann */
78887df15deSDaniel Borkmann if (READ_ONCE(map->frozen))
78987df15deSDaniel Borkmann mode &= ~FMODE_CAN_WRITE;
79087df15deSDaniel Borkmann return mode;
79187df15deSDaniel Borkmann }
79287df15deSDaniel Borkmann
793f99bf205SDaniel Borkmann #ifdef CONFIG_PROC_FS
79490a5527dSYafang Shao /* Show the memory usage of a bpf map */
bpf_map_memory_usage(const struct bpf_map * map)79590a5527dSYafang Shao static u64 bpf_map_memory_usage(const struct bpf_map *map)
79680ee81e0SRoman Gushchin {
79790a5527dSYafang Shao return map->ops->map_mem_usage(map);
79880ee81e0SRoman Gushchin }
79980ee81e0SRoman Gushchin
bpf_map_show_fdinfo(struct seq_file * m,struct file * filp)800f99bf205SDaniel Borkmann static void bpf_map_show_fdinfo(struct seq_file *m, struct file *filp)
801f99bf205SDaniel Borkmann {
802f45d5b6cSToke Hoiland-Jorgensen struct bpf_map *map = filp->private_data;
8032beee5f5SDaniel Borkmann u32 type = 0, jited = 0;
80421116b70SDaniel Borkmann
805f45d5b6cSToke Hoiland-Jorgensen if (map_type_contains_progs(map)) {
806f45d5b6cSToke Hoiland-Jorgensen spin_lock(&map->owner.lock);
807f45d5b6cSToke Hoiland-Jorgensen type = map->owner.type;
808f45d5b6cSToke Hoiland-Jorgensen jited = map->owner.jited;
809f45d5b6cSToke Hoiland-Jorgensen spin_unlock(&map->owner.lock);
81021116b70SDaniel Borkmann }
811f99bf205SDaniel Borkmann
812f99bf205SDaniel Borkmann seq_printf(m,
813f99bf205SDaniel Borkmann "map_type:\t%u\n"
814f99bf205SDaniel Borkmann "key_size:\t%u\n"
815f99bf205SDaniel Borkmann "value_size:\t%u\n"
816322cea2fSDaniel Borkmann "max_entries:\t%u\n"
81721116b70SDaniel Borkmann "map_flags:\t%#x\n"
8189330986cSJoanne Koong "map_extra:\t%#llx\n"
81990a5527dSYafang Shao "memlock:\t%llu\n"
82087df15deSDaniel Borkmann "map_id:\t%u\n"
82187df15deSDaniel Borkmann "frozen:\t%u\n",
822f99bf205SDaniel Borkmann map->map_type,
823f99bf205SDaniel Borkmann map->key_size,
824f99bf205SDaniel Borkmann map->value_size,
825322cea2fSDaniel Borkmann map->max_entries,
82621116b70SDaniel Borkmann map->map_flags,
8279330986cSJoanne Koong (unsigned long long)map->map_extra,
82890a5527dSYafang Shao bpf_map_memory_usage(map),
82987df15deSDaniel Borkmann map->id,
83087df15deSDaniel Borkmann READ_ONCE(map->frozen));
8312beee5f5SDaniel Borkmann if (type) {
8322beee5f5SDaniel Borkmann seq_printf(m, "owner_prog_type:\t%u\n", type);
8332beee5f5SDaniel Borkmann seq_printf(m, "owner_jited:\t%u\n", jited);
8349780c0abSDaniel Borkmann }
835f99bf205SDaniel Borkmann }
836f99bf205SDaniel Borkmann #endif
837f99bf205SDaniel Borkmann
bpf_dummy_read(struct file * filp,char __user * buf,size_t siz,loff_t * ppos)8386e71b04aSChenbo Feng static ssize_t bpf_dummy_read(struct file *filp, char __user *buf, size_t siz,
8396e71b04aSChenbo Feng loff_t *ppos)
8406e71b04aSChenbo Feng {
8416e71b04aSChenbo Feng /* We need this handler such that alloc_file() enables
8426e71b04aSChenbo Feng * f_mode with FMODE_CAN_READ.
8436e71b04aSChenbo Feng */
8446e71b04aSChenbo Feng return -EINVAL;
8456e71b04aSChenbo Feng }
8466e71b04aSChenbo Feng
bpf_dummy_write(struct file * filp,const char __user * buf,size_t siz,loff_t * ppos)8476e71b04aSChenbo Feng static ssize_t bpf_dummy_write(struct file *filp, const char __user *buf,
8486e71b04aSChenbo Feng size_t siz, loff_t *ppos)
8496e71b04aSChenbo Feng {
8506e71b04aSChenbo Feng /* We need this handler such that alloc_file() enables
8516e71b04aSChenbo Feng * f_mode with FMODE_CAN_WRITE.
8526e71b04aSChenbo Feng */
8536e71b04aSChenbo Feng return -EINVAL;
8546e71b04aSChenbo Feng }
8556e71b04aSChenbo Feng
856fc970227SAndrii Nakryiko /* called for any extra memory-mapped regions (except initial) */
bpf_map_mmap_open(struct vm_area_struct * vma)857fc970227SAndrii Nakryiko static void bpf_map_mmap_open(struct vm_area_struct *vma)
858fc970227SAndrii Nakryiko {
859fc970227SAndrii Nakryiko struct bpf_map *map = vma->vm_file->private_data;
860fc970227SAndrii Nakryiko
861353050beSDaniel Borkmann if (vma->vm_flags & VM_MAYWRITE)
862353050beSDaniel Borkmann bpf_map_write_active_inc(map);
863fc970227SAndrii Nakryiko }
864fc970227SAndrii Nakryiko
865fc970227SAndrii Nakryiko /* called for all unmapped memory region (including initial) */
bpf_map_mmap_close(struct vm_area_struct * vma)866fc970227SAndrii Nakryiko static void bpf_map_mmap_close(struct vm_area_struct *vma)
867fc970227SAndrii Nakryiko {
868fc970227SAndrii Nakryiko struct bpf_map *map = vma->vm_file->private_data;
869fc970227SAndrii Nakryiko
870353050beSDaniel Borkmann if (vma->vm_flags & VM_MAYWRITE)
871353050beSDaniel Borkmann bpf_map_write_active_dec(map);
872fc970227SAndrii Nakryiko }
873fc970227SAndrii Nakryiko
874fc970227SAndrii Nakryiko static const struct vm_operations_struct bpf_map_default_vmops = {
875fc970227SAndrii Nakryiko .open = bpf_map_mmap_open,
876fc970227SAndrii Nakryiko .close = bpf_map_mmap_close,
877fc970227SAndrii Nakryiko };
878fc970227SAndrii Nakryiko
bpf_map_mmap(struct file * filp,struct vm_area_struct * vma)879fc970227SAndrii Nakryiko static int bpf_map_mmap(struct file *filp, struct vm_area_struct *vma)
880fc970227SAndrii Nakryiko {
881fc970227SAndrii Nakryiko struct bpf_map *map = filp->private_data;
882fc970227SAndrii Nakryiko int err;
883fc970227SAndrii Nakryiko
884db559117SKumar Kartikeya Dwivedi if (!map->ops->map_mmap || !IS_ERR_OR_NULL(map->record))
885fc970227SAndrii Nakryiko return -ENOTSUPP;
886fc970227SAndrii Nakryiko
887fc970227SAndrii Nakryiko if (!(vma->vm_flags & VM_SHARED))
888fc970227SAndrii Nakryiko return -EINVAL;
889fc970227SAndrii Nakryiko
890fc970227SAndrii Nakryiko mutex_lock(&map->freeze_mutex);
891fc970227SAndrii Nakryiko
892dfeb376dSAndrii Nakryiko if (vma->vm_flags & VM_WRITE) {
893dfeb376dSAndrii Nakryiko if (map->frozen) {
894fc970227SAndrii Nakryiko err = -EPERM;
895fc970227SAndrii Nakryiko goto out;
896fc970227SAndrii Nakryiko }
897dfeb376dSAndrii Nakryiko /* map is meant to be read-only, so do not allow mapping as
898dfeb376dSAndrii Nakryiko * writable, because it's possible to leak a writable page
899dfeb376dSAndrii Nakryiko * reference and allows user-space to still modify it after
900dfeb376dSAndrii Nakryiko * freezing, while verifier will assume contents do not change
901dfeb376dSAndrii Nakryiko */
902dfeb376dSAndrii Nakryiko if (map->map_flags & BPF_F_RDONLY_PROG) {
903dfeb376dSAndrii Nakryiko err = -EACCES;
904dfeb376dSAndrii Nakryiko goto out;
905dfeb376dSAndrii Nakryiko }
906dfeb376dSAndrii Nakryiko }
907fc970227SAndrii Nakryiko
908fc970227SAndrii Nakryiko /* set default open/close callbacks */
909fc970227SAndrii Nakryiko vma->vm_ops = &bpf_map_default_vmops;
910fc970227SAndrii Nakryiko vma->vm_private_data = map;
9111c71222eSSuren Baghdasaryan vm_flags_clear(vma, VM_MAYEXEC);
9121f6cb19bSAndrii Nakryiko if (!(vma->vm_flags & VM_WRITE))
9131f6cb19bSAndrii Nakryiko /* disallow re-mapping with PROT_WRITE */
9141c71222eSSuren Baghdasaryan vm_flags_clear(vma, VM_MAYWRITE);
915fc970227SAndrii Nakryiko
916fc970227SAndrii Nakryiko err = map->ops->map_mmap(map, vma);
917fc970227SAndrii Nakryiko if (err)
918fc970227SAndrii Nakryiko goto out;
919fc970227SAndrii Nakryiko
9201f6cb19bSAndrii Nakryiko if (vma->vm_flags & VM_MAYWRITE)
921353050beSDaniel Borkmann bpf_map_write_active_inc(map);
922fc970227SAndrii Nakryiko out:
923fc970227SAndrii Nakryiko mutex_unlock(&map->freeze_mutex);
924fc970227SAndrii Nakryiko return err;
925fc970227SAndrii Nakryiko }
926fc970227SAndrii Nakryiko
bpf_map_poll(struct file * filp,struct poll_table_struct * pts)927457f4436SAndrii Nakryiko static __poll_t bpf_map_poll(struct file *filp, struct poll_table_struct *pts)
928457f4436SAndrii Nakryiko {
929457f4436SAndrii Nakryiko struct bpf_map *map = filp->private_data;
930457f4436SAndrii Nakryiko
931457f4436SAndrii Nakryiko if (map->ops->map_poll)
932457f4436SAndrii Nakryiko return map->ops->map_poll(map, filp, pts);
933457f4436SAndrii Nakryiko
934457f4436SAndrii Nakryiko return EPOLLERR;
935457f4436SAndrii Nakryiko }
936457f4436SAndrii Nakryiko
937f66e448cSChenbo Feng const struct file_operations bpf_map_fops = {
938f99bf205SDaniel Borkmann #ifdef CONFIG_PROC_FS
939f99bf205SDaniel Borkmann .show_fdinfo = bpf_map_show_fdinfo,
940f99bf205SDaniel Borkmann #endif
94199c55f7dSAlexei Starovoitov .release = bpf_map_release,
9426e71b04aSChenbo Feng .read = bpf_dummy_read,
9436e71b04aSChenbo Feng .write = bpf_dummy_write,
944fc970227SAndrii Nakryiko .mmap = bpf_map_mmap,
945457f4436SAndrii Nakryiko .poll = bpf_map_poll,
94699c55f7dSAlexei Starovoitov };
94799c55f7dSAlexei Starovoitov
bpf_map_new_fd(struct bpf_map * map,int flags)9486e71b04aSChenbo Feng int bpf_map_new_fd(struct bpf_map *map, int flags)
949aa79781bSDaniel Borkmann {
950afdb09c7SChenbo Feng int ret;
951afdb09c7SChenbo Feng
952afdb09c7SChenbo Feng ret = security_bpf_map(map, OPEN_FMODE(flags));
953afdb09c7SChenbo Feng if (ret < 0)
954afdb09c7SChenbo Feng return ret;
955afdb09c7SChenbo Feng
956aa79781bSDaniel Borkmann return anon_inode_getfd("bpf-map", &bpf_map_fops, map,
9576e71b04aSChenbo Feng flags | O_CLOEXEC);
9586e71b04aSChenbo Feng }
9596e71b04aSChenbo Feng
bpf_get_file_flag(int flags)9606e71b04aSChenbo Feng int bpf_get_file_flag(int flags)
9616e71b04aSChenbo Feng {
9626e71b04aSChenbo Feng if ((flags & BPF_F_RDONLY) && (flags & BPF_F_WRONLY))
9636e71b04aSChenbo Feng return -EINVAL;
9646e71b04aSChenbo Feng if (flags & BPF_F_RDONLY)
9656e71b04aSChenbo Feng return O_RDONLY;
9666e71b04aSChenbo Feng if (flags & BPF_F_WRONLY)
9676e71b04aSChenbo Feng return O_WRONLY;
9686e71b04aSChenbo Feng return O_RDWR;
969aa79781bSDaniel Borkmann }
970aa79781bSDaniel Borkmann
97199c55f7dSAlexei Starovoitov /* helper macro to check that unused fields 'union bpf_attr' are zero */
97299c55f7dSAlexei Starovoitov #define CHECK_ATTR(CMD) \
97399c55f7dSAlexei Starovoitov memchr_inv((void *) &attr->CMD##_LAST_FIELD + \
97499c55f7dSAlexei Starovoitov sizeof(attr->CMD##_LAST_FIELD), 0, \
97599c55f7dSAlexei Starovoitov sizeof(*attr) - \
97699c55f7dSAlexei Starovoitov offsetof(union bpf_attr, CMD##_LAST_FIELD) - \
97799c55f7dSAlexei Starovoitov sizeof(attr->CMD##_LAST_FIELD)) != NULL
97899c55f7dSAlexei Starovoitov
9798e7ae251SMartin KaFai Lau /* dst and src must have at least "size" number of bytes.
9808e7ae251SMartin KaFai Lau * Return strlen on success and < 0 on error.
981cb4d2b3fSMartin KaFai Lau */
bpf_obj_name_cpy(char * dst,const char * src,unsigned int size)9828e7ae251SMartin KaFai Lau int bpf_obj_name_cpy(char *dst, const char *src, unsigned int size)
983cb4d2b3fSMartin KaFai Lau {
9848e7ae251SMartin KaFai Lau const char *end = src + size;
9858e7ae251SMartin KaFai Lau const char *orig_src = src;
986cb4d2b3fSMartin KaFai Lau
9878e7ae251SMartin KaFai Lau memset(dst, 0, size);
9883e0ddc4fSDaniel Borkmann /* Copy all isalnum(), '_' and '.' chars. */
989cb4d2b3fSMartin KaFai Lau while (src < end && *src) {
9903e0ddc4fSDaniel Borkmann if (!isalnum(*src) &&
9913e0ddc4fSDaniel Borkmann *src != '_' && *src != '.')
992cb4d2b3fSMartin KaFai Lau return -EINVAL;
993cb4d2b3fSMartin KaFai Lau *dst++ = *src++;
994cb4d2b3fSMartin KaFai Lau }
995cb4d2b3fSMartin KaFai Lau
9968e7ae251SMartin KaFai Lau /* No '\0' found in "size" number of bytes */
997cb4d2b3fSMartin KaFai Lau if (src == end)
998cb4d2b3fSMartin KaFai Lau return -EINVAL;
999cb4d2b3fSMartin KaFai Lau
10008e7ae251SMartin KaFai Lau return src - orig_src;
1001cb4d2b3fSMartin KaFai Lau }
1002cb4d2b3fSMartin KaFai Lau
map_check_no_btf(const struct bpf_map * map,const struct btf * btf,const struct btf_type * key_type,const struct btf_type * value_type)1003e8d2bec0SDaniel Borkmann int map_check_no_btf(const struct bpf_map *map,
10041b2b234bSRoman Gushchin const struct btf *btf,
1005e8d2bec0SDaniel Borkmann const struct btf_type *key_type,
1006e8d2bec0SDaniel Borkmann const struct btf_type *value_type)
1007e8d2bec0SDaniel Borkmann {
1008e8d2bec0SDaniel Borkmann return -ENOTSUPP;
1009e8d2bec0SDaniel Borkmann }
1010e8d2bec0SDaniel Borkmann
map_check_btf(struct bpf_map * map,const struct btf * btf,u32 btf_key_id,u32 btf_value_id)1011d83525caSAlexei Starovoitov static int map_check_btf(struct bpf_map *map, const struct btf *btf,
1012e8d2bec0SDaniel Borkmann u32 btf_key_id, u32 btf_value_id)
1013e8d2bec0SDaniel Borkmann {
1014e8d2bec0SDaniel Borkmann const struct btf_type *key_type, *value_type;
1015e8d2bec0SDaniel Borkmann u32 key_size, value_size;
1016e8d2bec0SDaniel Borkmann int ret = 0;
1017e8d2bec0SDaniel Borkmann
10182824ecb7SDaniel Borkmann /* Some maps allow key to be unspecified. */
10192824ecb7SDaniel Borkmann if (btf_key_id) {
1020e8d2bec0SDaniel Borkmann key_type = btf_type_id_size(btf, &btf_key_id, &key_size);
1021e8d2bec0SDaniel Borkmann if (!key_type || key_size != map->key_size)
1022e8d2bec0SDaniel Borkmann return -EINVAL;
10232824ecb7SDaniel Borkmann } else {
10242824ecb7SDaniel Borkmann key_type = btf_type_by_id(btf, 0);
10252824ecb7SDaniel Borkmann if (!map->ops->map_check_btf)
10262824ecb7SDaniel Borkmann return -EINVAL;
10272824ecb7SDaniel Borkmann }
1028e8d2bec0SDaniel Borkmann
1029e8d2bec0SDaniel Borkmann value_type = btf_type_id_size(btf, &btf_value_id, &value_size);
1030e8d2bec0SDaniel Borkmann if (!value_type || value_size != map->value_size)
1031e8d2bec0SDaniel Borkmann return -EINVAL;
1032e8d2bec0SDaniel Borkmann
1033f0c5941fSKumar Kartikeya Dwivedi map->record = btf_parse_fields(btf, value_type,
10349c395c1bSDave Marchevsky BPF_SPIN_LOCK | BPF_TIMER | BPF_KPTR | BPF_LIST_HEAD |
1035d54730b5SDave Marchevsky BPF_RB_ROOT | BPF_REFCOUNT,
1036db559117SKumar Kartikeya Dwivedi map->value_size);
1037aa3496acSKumar Kartikeya Dwivedi if (!IS_ERR_OR_NULL(map->record)) {
1038aa3496acSKumar Kartikeya Dwivedi int i;
1039aa3496acSKumar Kartikeya Dwivedi
104061df10c7SKumar Kartikeya Dwivedi if (!bpf_capable()) {
104161df10c7SKumar Kartikeya Dwivedi ret = -EPERM;
104261df10c7SKumar Kartikeya Dwivedi goto free_map_tab;
104361df10c7SKumar Kartikeya Dwivedi }
104461df10c7SKumar Kartikeya Dwivedi if (map->map_flags & (BPF_F_RDONLY_PROG | BPF_F_WRONLY_PROG)) {
104561df10c7SKumar Kartikeya Dwivedi ret = -EACCES;
104661df10c7SKumar Kartikeya Dwivedi goto free_map_tab;
104761df10c7SKumar Kartikeya Dwivedi }
1048aa3496acSKumar Kartikeya Dwivedi for (i = 0; i < sizeof(map->record->field_mask) * 8; i++) {
1049aa3496acSKumar Kartikeya Dwivedi switch (map->record->field_mask & (1 << i)) {
1050aa3496acSKumar Kartikeya Dwivedi case 0:
1051aa3496acSKumar Kartikeya Dwivedi continue;
1052db559117SKumar Kartikeya Dwivedi case BPF_SPIN_LOCK:
1053db559117SKumar Kartikeya Dwivedi if (map->map_type != BPF_MAP_TYPE_HASH &&
1054db559117SKumar Kartikeya Dwivedi map->map_type != BPF_MAP_TYPE_ARRAY &&
1055db559117SKumar Kartikeya Dwivedi map->map_type != BPF_MAP_TYPE_CGROUP_STORAGE &&
1056db559117SKumar Kartikeya Dwivedi map->map_type != BPF_MAP_TYPE_SK_STORAGE &&
1057db559117SKumar Kartikeya Dwivedi map->map_type != BPF_MAP_TYPE_INODE_STORAGE &&
1058db559117SKumar Kartikeya Dwivedi map->map_type != BPF_MAP_TYPE_TASK_STORAGE &&
1059db559117SKumar Kartikeya Dwivedi map->map_type != BPF_MAP_TYPE_CGRP_STORAGE) {
1060db559117SKumar Kartikeya Dwivedi ret = -EOPNOTSUPP;
1061db559117SKumar Kartikeya Dwivedi goto free_map_tab;
1062db559117SKumar Kartikeya Dwivedi }
1063db559117SKumar Kartikeya Dwivedi break;
1064db559117SKumar Kartikeya Dwivedi case BPF_TIMER:
1065db559117SKumar Kartikeya Dwivedi if (map->map_type != BPF_MAP_TYPE_HASH &&
1066db559117SKumar Kartikeya Dwivedi map->map_type != BPF_MAP_TYPE_LRU_HASH &&
1067db559117SKumar Kartikeya Dwivedi map->map_type != BPF_MAP_TYPE_ARRAY) {
1068c237bfa5SKumar Kartikeya Dwivedi ret = -EOPNOTSUPP;
1069db559117SKumar Kartikeya Dwivedi goto free_map_tab;
1070db559117SKumar Kartikeya Dwivedi }
1071db559117SKumar Kartikeya Dwivedi break;
1072aa3496acSKumar Kartikeya Dwivedi case BPF_KPTR_UNREF:
1073aa3496acSKumar Kartikeya Dwivedi case BPF_KPTR_REF:
1074d54730b5SDave Marchevsky case BPF_REFCOUNT:
107561df10c7SKumar Kartikeya Dwivedi if (map->map_type != BPF_MAP_TYPE_HASH &&
107665334e64SKumar Kartikeya Dwivedi map->map_type != BPF_MAP_TYPE_PERCPU_HASH &&
107761df10c7SKumar Kartikeya Dwivedi map->map_type != BPF_MAP_TYPE_LRU_HASH &&
107865334e64SKumar Kartikeya Dwivedi map->map_type != BPF_MAP_TYPE_LRU_PERCPU_HASH &&
10796df4ea1fSKumar Kartikeya Dwivedi map->map_type != BPF_MAP_TYPE_ARRAY &&
10809db44fddSKumar Kartikeya Dwivedi map->map_type != BPF_MAP_TYPE_PERCPU_ARRAY &&
10819db44fddSKumar Kartikeya Dwivedi map->map_type != BPF_MAP_TYPE_SK_STORAGE &&
10829db44fddSKumar Kartikeya Dwivedi map->map_type != BPF_MAP_TYPE_INODE_STORAGE &&
10839db44fddSKumar Kartikeya Dwivedi map->map_type != BPF_MAP_TYPE_TASK_STORAGE &&
10849db44fddSKumar Kartikeya Dwivedi map->map_type != BPF_MAP_TYPE_CGRP_STORAGE) {
108561df10c7SKumar Kartikeya Dwivedi ret = -EOPNOTSUPP;
108661df10c7SKumar Kartikeya Dwivedi goto free_map_tab;
108761df10c7SKumar Kartikeya Dwivedi }
1088aa3496acSKumar Kartikeya Dwivedi break;
1089f0c5941fSKumar Kartikeya Dwivedi case BPF_LIST_HEAD:
10909c395c1bSDave Marchevsky case BPF_RB_ROOT:
1091f0c5941fSKumar Kartikeya Dwivedi if (map->map_type != BPF_MAP_TYPE_HASH &&
1092f0c5941fSKumar Kartikeya Dwivedi map->map_type != BPF_MAP_TYPE_LRU_HASH &&
1093f0c5941fSKumar Kartikeya Dwivedi map->map_type != BPF_MAP_TYPE_ARRAY) {
1094f0c5941fSKumar Kartikeya Dwivedi ret = -EOPNOTSUPP;
1095f0c5941fSKumar Kartikeya Dwivedi goto free_map_tab;
1096f0c5941fSKumar Kartikeya Dwivedi }
1097f0c5941fSKumar Kartikeya Dwivedi break;
1098aa3496acSKumar Kartikeya Dwivedi default:
1099aa3496acSKumar Kartikeya Dwivedi /* Fail if map_type checks are missing for a field type */
1100aa3496acSKumar Kartikeya Dwivedi ret = -EOPNOTSUPP;
1101aa3496acSKumar Kartikeya Dwivedi goto free_map_tab;
1102aa3496acSKumar Kartikeya Dwivedi }
1103aa3496acSKumar Kartikeya Dwivedi }
110461df10c7SKumar Kartikeya Dwivedi }
1105e8d2bec0SDaniel Borkmann
1106865ce09aSKumar Kartikeya Dwivedi ret = btf_check_and_fixup_fields(btf, map->record);
1107865ce09aSKumar Kartikeya Dwivedi if (ret < 0)
1108865ce09aSKumar Kartikeya Dwivedi goto free_map_tab;
1109865ce09aSKumar Kartikeya Dwivedi
111061df10c7SKumar Kartikeya Dwivedi if (map->ops->map_check_btf) {
111161df10c7SKumar Kartikeya Dwivedi ret = map->ops->map_check_btf(map, btf, key_type, value_type);
111261df10c7SKumar Kartikeya Dwivedi if (ret < 0)
111361df10c7SKumar Kartikeya Dwivedi goto free_map_tab;
111461df10c7SKumar Kartikeya Dwivedi }
111561df10c7SKumar Kartikeya Dwivedi
111661df10c7SKumar Kartikeya Dwivedi return ret;
111761df10c7SKumar Kartikeya Dwivedi free_map_tab:
1118aa3496acSKumar Kartikeya Dwivedi bpf_map_free_record(map);
1119e8d2bec0SDaniel Borkmann return ret;
1120e8d2bec0SDaniel Borkmann }
1121e8d2bec0SDaniel Borkmann
11229330986cSJoanne Koong #define BPF_MAP_CREATE_LAST_FIELD map_extra
112399c55f7dSAlexei Starovoitov /* called via syscall */
map_create(union bpf_attr * attr)112499c55f7dSAlexei Starovoitov static int map_create(union bpf_attr *attr)
112599c55f7dSAlexei Starovoitov {
112622db4122SAndrii Nakryiko const struct bpf_map_ops *ops;
112796eabe7aSMartin KaFai Lau int numa_node = bpf_map_attr_numa_node(attr);
112822db4122SAndrii Nakryiko u32 map_type = attr->map_type;
112999c55f7dSAlexei Starovoitov struct bpf_map *map;
11306e71b04aSChenbo Feng int f_flags;
113199c55f7dSAlexei Starovoitov int err;
113299c55f7dSAlexei Starovoitov
113399c55f7dSAlexei Starovoitov err = CHECK_ATTR(BPF_MAP_CREATE);
113499c55f7dSAlexei Starovoitov if (err)
113599c55f7dSAlexei Starovoitov return -EINVAL;
113699c55f7dSAlexei Starovoitov
113785d33df3SMartin KaFai Lau if (attr->btf_vmlinux_value_type_id) {
113885d33df3SMartin KaFai Lau if (attr->map_type != BPF_MAP_TYPE_STRUCT_OPS ||
113985d33df3SMartin KaFai Lau attr->btf_key_type_id || attr->btf_value_type_id)
114085d33df3SMartin KaFai Lau return -EINVAL;
114185d33df3SMartin KaFai Lau } else if (attr->btf_key_type_id && !attr->btf_value_type_id) {
114285d33df3SMartin KaFai Lau return -EINVAL;
114385d33df3SMartin KaFai Lau }
114485d33df3SMartin KaFai Lau
11459330986cSJoanne Koong if (attr->map_type != BPF_MAP_TYPE_BLOOM_FILTER &&
11469330986cSJoanne Koong attr->map_extra != 0)
11479330986cSJoanne Koong return -EINVAL;
11489330986cSJoanne Koong
11496e71b04aSChenbo Feng f_flags = bpf_get_file_flag(attr->map_flags);
11506e71b04aSChenbo Feng if (f_flags < 0)
11516e71b04aSChenbo Feng return f_flags;
11526e71b04aSChenbo Feng
115396eabe7aSMartin KaFai Lau if (numa_node != NUMA_NO_NODE &&
115496e5ae4eSEric Dumazet ((unsigned int)numa_node >= nr_node_ids ||
115596e5ae4eSEric Dumazet !node_online(numa_node)))
115696eabe7aSMartin KaFai Lau return -EINVAL;
115796eabe7aSMartin KaFai Lau
115899c55f7dSAlexei Starovoitov /* find map type and init map: hashtable vs rbtree vs bloom vs ... */
115922db4122SAndrii Nakryiko map_type = attr->map_type;
116022db4122SAndrii Nakryiko if (map_type >= ARRAY_SIZE(bpf_map_types))
116122db4122SAndrii Nakryiko return -EINVAL;
116222db4122SAndrii Nakryiko map_type = array_index_nospec(map_type, ARRAY_SIZE(bpf_map_types));
116322db4122SAndrii Nakryiko ops = bpf_map_types[map_type];
116422db4122SAndrii Nakryiko if (!ops)
116522db4122SAndrii Nakryiko return -EINVAL;
116622db4122SAndrii Nakryiko
116722db4122SAndrii Nakryiko if (ops->map_alloc_check) {
116822db4122SAndrii Nakryiko err = ops->map_alloc_check(attr);
116922db4122SAndrii Nakryiko if (err)
117022db4122SAndrii Nakryiko return err;
117122db4122SAndrii Nakryiko }
117222db4122SAndrii Nakryiko if (attr->map_ifindex)
117322db4122SAndrii Nakryiko ops = &bpf_map_offload_ops;
117422db4122SAndrii Nakryiko if (!ops->map_mem_usage)
117522db4122SAndrii Nakryiko return -EINVAL;
117622db4122SAndrii Nakryiko
11771d28635aSAndrii Nakryiko /* Intent here is for unprivileged_bpf_disabled to block BPF map
11781d28635aSAndrii Nakryiko * creation for unprivileged users; other actions depend
11791d28635aSAndrii Nakryiko * on fd availability and access to bpffs, so are dependent on
11801d28635aSAndrii Nakryiko * object creation success. Even with unprivileged BPF disabled,
11811d28635aSAndrii Nakryiko * capability checks are still carried out.
11821d28635aSAndrii Nakryiko */
11831d28635aSAndrii Nakryiko if (sysctl_unprivileged_bpf_disabled && !bpf_capable())
11841d28635aSAndrii Nakryiko return -EPERM;
11851d28635aSAndrii Nakryiko
11866c3eba1cSAndrii Nakryiko /* check privileged map type permissions */
11876c3eba1cSAndrii Nakryiko switch (map_type) {
11886c3eba1cSAndrii Nakryiko case BPF_MAP_TYPE_ARRAY:
11896c3eba1cSAndrii Nakryiko case BPF_MAP_TYPE_PERCPU_ARRAY:
11906c3eba1cSAndrii Nakryiko case BPF_MAP_TYPE_PROG_ARRAY:
11916c3eba1cSAndrii Nakryiko case BPF_MAP_TYPE_PERF_EVENT_ARRAY:
11926c3eba1cSAndrii Nakryiko case BPF_MAP_TYPE_CGROUP_ARRAY:
11936c3eba1cSAndrii Nakryiko case BPF_MAP_TYPE_ARRAY_OF_MAPS:
11946c3eba1cSAndrii Nakryiko case BPF_MAP_TYPE_HASH:
11956c3eba1cSAndrii Nakryiko case BPF_MAP_TYPE_PERCPU_HASH:
11966c3eba1cSAndrii Nakryiko case BPF_MAP_TYPE_HASH_OF_MAPS:
11976c3eba1cSAndrii Nakryiko case BPF_MAP_TYPE_RINGBUF:
11986c3eba1cSAndrii Nakryiko case BPF_MAP_TYPE_USER_RINGBUF:
11996c3eba1cSAndrii Nakryiko case BPF_MAP_TYPE_CGROUP_STORAGE:
12006c3eba1cSAndrii Nakryiko case BPF_MAP_TYPE_PERCPU_CGROUP_STORAGE:
12016c3eba1cSAndrii Nakryiko /* unprivileged */
12026c3eba1cSAndrii Nakryiko break;
12036c3eba1cSAndrii Nakryiko case BPF_MAP_TYPE_SK_STORAGE:
12046c3eba1cSAndrii Nakryiko case BPF_MAP_TYPE_INODE_STORAGE:
12056c3eba1cSAndrii Nakryiko case BPF_MAP_TYPE_TASK_STORAGE:
12066c3eba1cSAndrii Nakryiko case BPF_MAP_TYPE_CGRP_STORAGE:
12076c3eba1cSAndrii Nakryiko case BPF_MAP_TYPE_BLOOM_FILTER:
12086c3eba1cSAndrii Nakryiko case BPF_MAP_TYPE_LPM_TRIE:
12096c3eba1cSAndrii Nakryiko case BPF_MAP_TYPE_REUSEPORT_SOCKARRAY:
12106c3eba1cSAndrii Nakryiko case BPF_MAP_TYPE_STACK_TRACE:
12116c3eba1cSAndrii Nakryiko case BPF_MAP_TYPE_QUEUE:
12126c3eba1cSAndrii Nakryiko case BPF_MAP_TYPE_STACK:
12136c3eba1cSAndrii Nakryiko case BPF_MAP_TYPE_LRU_HASH:
12146c3eba1cSAndrii Nakryiko case BPF_MAP_TYPE_LRU_PERCPU_HASH:
12156c3eba1cSAndrii Nakryiko case BPF_MAP_TYPE_STRUCT_OPS:
12166c3eba1cSAndrii Nakryiko case BPF_MAP_TYPE_CPUMAP:
12176c3eba1cSAndrii Nakryiko if (!bpf_capable())
12186c3eba1cSAndrii Nakryiko return -EPERM;
12196c3eba1cSAndrii Nakryiko break;
12206c3eba1cSAndrii Nakryiko case BPF_MAP_TYPE_SOCKMAP:
12216c3eba1cSAndrii Nakryiko case BPF_MAP_TYPE_SOCKHASH:
12226c3eba1cSAndrii Nakryiko case BPF_MAP_TYPE_DEVMAP:
12236c3eba1cSAndrii Nakryiko case BPF_MAP_TYPE_DEVMAP_HASH:
12246c3eba1cSAndrii Nakryiko case BPF_MAP_TYPE_XSKMAP:
12256c3eba1cSAndrii Nakryiko if (!capable(CAP_NET_ADMIN))
12266c3eba1cSAndrii Nakryiko return -EPERM;
12276c3eba1cSAndrii Nakryiko break;
12286c3eba1cSAndrii Nakryiko default:
12296c3eba1cSAndrii Nakryiko WARN(1, "unsupported map type %d", map_type);
12306c3eba1cSAndrii Nakryiko return -EPERM;
12316c3eba1cSAndrii Nakryiko }
12326c3eba1cSAndrii Nakryiko
123322db4122SAndrii Nakryiko map = ops->map_alloc(attr);
123499c55f7dSAlexei Starovoitov if (IS_ERR(map))
123599c55f7dSAlexei Starovoitov return PTR_ERR(map);
123622db4122SAndrii Nakryiko map->ops = ops;
123722db4122SAndrii Nakryiko map->map_type = map_type;
123899c55f7dSAlexei Starovoitov
12398e7ae251SMartin KaFai Lau err = bpf_obj_name_cpy(map->name, attr->map_name,
12408e7ae251SMartin KaFai Lau sizeof(attr->map_name));
12418e7ae251SMartin KaFai Lau if (err < 0)
1242b936ca64SRoman Gushchin goto free_map;
1243ad5b177bSMartin KaFai Lau
12441e0bd5a0SAndrii Nakryiko atomic64_set(&map->refcnt, 1);
12451e0bd5a0SAndrii Nakryiko atomic64_set(&map->usercnt, 1);
1246fc970227SAndrii Nakryiko mutex_init(&map->freeze_mutex);
1247f45d5b6cSToke Hoiland-Jorgensen spin_lock_init(&map->owner.lock);
124899c55f7dSAlexei Starovoitov
124985d33df3SMartin KaFai Lau if (attr->btf_key_type_id || attr->btf_value_type_id ||
125085d33df3SMartin KaFai Lau /* Even the map's value is a kernel's struct,
125185d33df3SMartin KaFai Lau * the bpf_prog.o must have BTF to begin with
125285d33df3SMartin KaFai Lau * to figure out the corresponding kernel's
125385d33df3SMartin KaFai Lau * counter part. Thus, attr->btf_fd has
125485d33df3SMartin KaFai Lau * to be valid also.
125585d33df3SMartin KaFai Lau */
125685d33df3SMartin KaFai Lau attr->btf_vmlinux_value_type_id) {
1257a26ca7c9SMartin KaFai Lau struct btf *btf;
1258a26ca7c9SMartin KaFai Lau
1259a26ca7c9SMartin KaFai Lau btf = btf_get_by_fd(attr->btf_fd);
1260a26ca7c9SMartin KaFai Lau if (IS_ERR(btf)) {
1261a26ca7c9SMartin KaFai Lau err = PTR_ERR(btf);
1262b936ca64SRoman Gushchin goto free_map;
1263a26ca7c9SMartin KaFai Lau }
1264350a5c4dSAlexei Starovoitov if (btf_is_kernel(btf)) {
1265350a5c4dSAlexei Starovoitov btf_put(btf);
1266350a5c4dSAlexei Starovoitov err = -EACCES;
1267350a5c4dSAlexei Starovoitov goto free_map;
1268350a5c4dSAlexei Starovoitov }
126985d33df3SMartin KaFai Lau map->btf = btf;
1270a26ca7c9SMartin KaFai Lau
127185d33df3SMartin KaFai Lau if (attr->btf_value_type_id) {
1272e8d2bec0SDaniel Borkmann err = map_check_btf(map, btf, attr->btf_key_type_id,
12739b2cf328SMartin KaFai Lau attr->btf_value_type_id);
127485d33df3SMartin KaFai Lau if (err)
1275b936ca64SRoman Gushchin goto free_map;
1276a26ca7c9SMartin KaFai Lau }
1277a26ca7c9SMartin KaFai Lau
12789b2cf328SMartin KaFai Lau map->btf_key_type_id = attr->btf_key_type_id;
12799b2cf328SMartin KaFai Lau map->btf_value_type_id = attr->btf_value_type_id;
128085d33df3SMartin KaFai Lau map->btf_vmlinux_value_type_id =
128185d33df3SMartin KaFai Lau attr->btf_vmlinux_value_type_id;
1282a26ca7c9SMartin KaFai Lau }
1283a26ca7c9SMartin KaFai Lau
12844d7d7f69SKumar Kartikeya Dwivedi err = security_bpf_map_alloc(map);
12854d7d7f69SKumar Kartikeya Dwivedi if (err)
1286cd2a8079SDave Marchevsky goto free_map;
12874d7d7f69SKumar Kartikeya Dwivedi
1288f3f1c054SMartin KaFai Lau err = bpf_map_alloc_id(map);
1289f3f1c054SMartin KaFai Lau if (err)
1290b936ca64SRoman Gushchin goto free_map_sec;
1291f3f1c054SMartin KaFai Lau
129248edc1f7SRoman Gushchin bpf_map_save_memcg(map);
129348edc1f7SRoman Gushchin
12946e71b04aSChenbo Feng err = bpf_map_new_fd(map, f_flags);
1295bd5f5f4eSMartin KaFai Lau if (err < 0) {
1296bd5f5f4eSMartin KaFai Lau /* failed to allocate fd.
1297352d20d6SPeng Sun * bpf_map_put_with_uref() is needed because the above
1298bd5f5f4eSMartin KaFai Lau * bpf_map_alloc_id() has published the map
1299bd5f5f4eSMartin KaFai Lau * to the userspace and the userspace may
1300bd5f5f4eSMartin KaFai Lau * have refcnt-ed it through BPF_MAP_GET_FD_BY_ID.
1301bd5f5f4eSMartin KaFai Lau */
1302352d20d6SPeng Sun bpf_map_put_with_uref(map);
1303bd5f5f4eSMartin KaFai Lau return err;
1304bd5f5f4eSMartin KaFai Lau }
130599c55f7dSAlexei Starovoitov
130699c55f7dSAlexei Starovoitov return err;
130799c55f7dSAlexei Starovoitov
1308afdb09c7SChenbo Feng free_map_sec:
1309afdb09c7SChenbo Feng security_bpf_map_free(map);
1310b936ca64SRoman Gushchin free_map:
1311a26ca7c9SMartin KaFai Lau btf_put(map->btf);
131299c55f7dSAlexei Starovoitov map->ops->map_free(map);
131399c55f7dSAlexei Starovoitov return err;
131499c55f7dSAlexei Starovoitov }
131599c55f7dSAlexei Starovoitov
1316db20fd2bSAlexei Starovoitov /* if error is returned, fd is released.
1317db20fd2bSAlexei Starovoitov * On success caller should complete fd access with matching fdput()
1318db20fd2bSAlexei Starovoitov */
__bpf_map_get(struct fd f)1319c2101297SDaniel Borkmann struct bpf_map *__bpf_map_get(struct fd f)
1320db20fd2bSAlexei Starovoitov {
1321db20fd2bSAlexei Starovoitov if (!f.file)
1322db20fd2bSAlexei Starovoitov return ERR_PTR(-EBADF);
1323db20fd2bSAlexei Starovoitov if (f.file->f_op != &bpf_map_fops) {
1324db20fd2bSAlexei Starovoitov fdput(f);
1325db20fd2bSAlexei Starovoitov return ERR_PTR(-EINVAL);
1326db20fd2bSAlexei Starovoitov }
1327db20fd2bSAlexei Starovoitov
1328c2101297SDaniel Borkmann return f.file->private_data;
1329c2101297SDaniel Borkmann }
1330c2101297SDaniel Borkmann
bpf_map_inc(struct bpf_map * map)13311e0bd5a0SAndrii Nakryiko void bpf_map_inc(struct bpf_map *map)
1332c9da161cSDaniel Borkmann {
13331e0bd5a0SAndrii Nakryiko atomic64_inc(&map->refcnt);
1334c9da161cSDaniel Borkmann }
1335630a4d38SJakub Kicinski EXPORT_SYMBOL_GPL(bpf_map_inc);
1336c9da161cSDaniel Borkmann
bpf_map_inc_with_uref(struct bpf_map * map)13371e0bd5a0SAndrii Nakryiko void bpf_map_inc_with_uref(struct bpf_map *map)
13381e0bd5a0SAndrii Nakryiko {
13391e0bd5a0SAndrii Nakryiko atomic64_inc(&map->refcnt);
13401e0bd5a0SAndrii Nakryiko atomic64_inc(&map->usercnt);
13411e0bd5a0SAndrii Nakryiko }
13421e0bd5a0SAndrii Nakryiko EXPORT_SYMBOL_GPL(bpf_map_inc_with_uref);
13431e0bd5a0SAndrii Nakryiko
bpf_map_get(u32 ufd)13441ed4d924SMartin KaFai Lau struct bpf_map *bpf_map_get(u32 ufd)
13451ed4d924SMartin KaFai Lau {
13461ed4d924SMartin KaFai Lau struct fd f = fdget(ufd);
13471ed4d924SMartin KaFai Lau struct bpf_map *map;
13481ed4d924SMartin KaFai Lau
13491ed4d924SMartin KaFai Lau map = __bpf_map_get(f);
13501ed4d924SMartin KaFai Lau if (IS_ERR(map))
13511ed4d924SMartin KaFai Lau return map;
13521ed4d924SMartin KaFai Lau
13531ed4d924SMartin KaFai Lau bpf_map_inc(map);
13541ed4d924SMartin KaFai Lau fdput(f);
13551ed4d924SMartin KaFai Lau
13561ed4d924SMartin KaFai Lau return map;
13571ed4d924SMartin KaFai Lau }
1358b1d18a75SAlexei Starovoitov EXPORT_SYMBOL(bpf_map_get);
13591ed4d924SMartin KaFai Lau
bpf_map_get_with_uref(u32 ufd)1360c9da161cSDaniel Borkmann struct bpf_map *bpf_map_get_with_uref(u32 ufd)
1361c2101297SDaniel Borkmann {
1362c2101297SDaniel Borkmann struct fd f = fdget(ufd);
1363c2101297SDaniel Borkmann struct bpf_map *map;
1364c2101297SDaniel Borkmann
1365c2101297SDaniel Borkmann map = __bpf_map_get(f);
1366c2101297SDaniel Borkmann if (IS_ERR(map))
1367c2101297SDaniel Borkmann return map;
1368c2101297SDaniel Borkmann
13691e0bd5a0SAndrii Nakryiko bpf_map_inc_with_uref(map);
1370c2101297SDaniel Borkmann fdput(f);
1371db20fd2bSAlexei Starovoitov
1372db20fd2bSAlexei Starovoitov return map;
1373db20fd2bSAlexei Starovoitov }
1374db20fd2bSAlexei Starovoitov
1375b671c206SKui-Feng Lee /* map_idr_lock should have been held or the map should have been
1376b671c206SKui-Feng Lee * protected by rcu read lock.
1377b671c206SKui-Feng Lee */
__bpf_map_inc_not_zero(struct bpf_map * map,bool uref)1378b671c206SKui-Feng Lee struct bpf_map *__bpf_map_inc_not_zero(struct bpf_map *map, bool uref)
1379bd5f5f4eSMartin KaFai Lau {
1380bd5f5f4eSMartin KaFai Lau int refold;
1381bd5f5f4eSMartin KaFai Lau
13821e0bd5a0SAndrii Nakryiko refold = atomic64_fetch_add_unless(&map->refcnt, 1, 0);
1383bd5f5f4eSMartin KaFai Lau if (!refold)
1384bd5f5f4eSMartin KaFai Lau return ERR_PTR(-ENOENT);
1385bd5f5f4eSMartin KaFai Lau if (uref)
13861e0bd5a0SAndrii Nakryiko atomic64_inc(&map->usercnt);
1387bd5f5f4eSMartin KaFai Lau
1388bd5f5f4eSMartin KaFai Lau return map;
1389bd5f5f4eSMartin KaFai Lau }
1390bd5f5f4eSMartin KaFai Lau
bpf_map_inc_not_zero(struct bpf_map * map)13911e0bd5a0SAndrii Nakryiko struct bpf_map *bpf_map_inc_not_zero(struct bpf_map *map)
1392b0e4701cSStanislav Fomichev {
1393b0e4701cSStanislav Fomichev spin_lock_bh(&map_idr_lock);
13941e0bd5a0SAndrii Nakryiko map = __bpf_map_inc_not_zero(map, false);
1395b0e4701cSStanislav Fomichev spin_unlock_bh(&map_idr_lock);
1396b0e4701cSStanislav Fomichev
1397b0e4701cSStanislav Fomichev return map;
1398b0e4701cSStanislav Fomichev }
1399b0e4701cSStanislav Fomichev EXPORT_SYMBOL_GPL(bpf_map_inc_not_zero);
1400b0e4701cSStanislav Fomichev
bpf_stackmap_copy(struct bpf_map * map,void * key,void * value)1401b8cdc051SAlexei Starovoitov int __weak bpf_stackmap_copy(struct bpf_map *map, void *key, void *value)
1402b8cdc051SAlexei Starovoitov {
1403b8cdc051SAlexei Starovoitov return -ENOTSUPP;
1404b8cdc051SAlexei Starovoitov }
1405b8cdc051SAlexei Starovoitov
__bpf_copy_key(void __user * ukey,u64 key_size)1406c9d29f46SMauricio Vasquez B static void *__bpf_copy_key(void __user *ukey, u64 key_size)
1407c9d29f46SMauricio Vasquez B {
1408c9d29f46SMauricio Vasquez B if (key_size)
140944779a4bSStanislav Fomichev return vmemdup_user(ukey, key_size);
1410c9d29f46SMauricio Vasquez B
1411c9d29f46SMauricio Vasquez B if (ukey)
1412c9d29f46SMauricio Vasquez B return ERR_PTR(-EINVAL);
1413c9d29f46SMauricio Vasquez B
1414c9d29f46SMauricio Vasquez B return NULL;
1415c9d29f46SMauricio Vasquez B }
1416c9d29f46SMauricio Vasquez B
___bpf_copy_key(bpfptr_t ukey,u64 key_size)1417af2ac3e1SAlexei Starovoitov static void *___bpf_copy_key(bpfptr_t ukey, u64 key_size)
1418af2ac3e1SAlexei Starovoitov {
1419af2ac3e1SAlexei Starovoitov if (key_size)
142044779a4bSStanislav Fomichev return kvmemdup_bpfptr(ukey, key_size);
1421af2ac3e1SAlexei Starovoitov
1422af2ac3e1SAlexei Starovoitov if (!bpfptr_is_null(ukey))
1423af2ac3e1SAlexei Starovoitov return ERR_PTR(-EINVAL);
1424af2ac3e1SAlexei Starovoitov
1425af2ac3e1SAlexei Starovoitov return NULL;
1426af2ac3e1SAlexei Starovoitov }
1427af2ac3e1SAlexei Starovoitov
1428db20fd2bSAlexei Starovoitov /* last field in 'union bpf_attr' used by this command */
142996049f3aSAlexei Starovoitov #define BPF_MAP_LOOKUP_ELEM_LAST_FIELD flags
1430db20fd2bSAlexei Starovoitov
map_lookup_elem(union bpf_attr * attr)1431db20fd2bSAlexei Starovoitov static int map_lookup_elem(union bpf_attr *attr)
1432db20fd2bSAlexei Starovoitov {
1433535e7b4bSMickaël Salaün void __user *ukey = u64_to_user_ptr(attr->key);
1434535e7b4bSMickaël Salaün void __user *uvalue = u64_to_user_ptr(attr->value);
1435db20fd2bSAlexei Starovoitov int ufd = attr->map_fd;
1436db20fd2bSAlexei Starovoitov struct bpf_map *map;
143715c14a3dSBrian Vazquez void *key, *value;
143815a07b33SAlexei Starovoitov u32 value_size;
1439592867bfSDaniel Borkmann struct fd f;
1440db20fd2bSAlexei Starovoitov int err;
1441db20fd2bSAlexei Starovoitov
1442db20fd2bSAlexei Starovoitov if (CHECK_ATTR(BPF_MAP_LOOKUP_ELEM))
1443db20fd2bSAlexei Starovoitov return -EINVAL;
1444db20fd2bSAlexei Starovoitov
144596049f3aSAlexei Starovoitov if (attr->flags & ~BPF_F_LOCK)
144696049f3aSAlexei Starovoitov return -EINVAL;
144796049f3aSAlexei Starovoitov
1448592867bfSDaniel Borkmann f = fdget(ufd);
1449c2101297SDaniel Borkmann map = __bpf_map_get(f);
1450db20fd2bSAlexei Starovoitov if (IS_ERR(map))
1451db20fd2bSAlexei Starovoitov return PTR_ERR(map);
145287df15deSDaniel Borkmann if (!(map_get_sys_perms(map, f) & FMODE_CAN_READ)) {
14536e71b04aSChenbo Feng err = -EPERM;
14546e71b04aSChenbo Feng goto err_put;
14556e71b04aSChenbo Feng }
14566e71b04aSChenbo Feng
145796049f3aSAlexei Starovoitov if ((attr->flags & BPF_F_LOCK) &&
1458db559117SKumar Kartikeya Dwivedi !btf_record_has_field(map->record, BPF_SPIN_LOCK)) {
145996049f3aSAlexei Starovoitov err = -EINVAL;
146096049f3aSAlexei Starovoitov goto err_put;
146196049f3aSAlexei Starovoitov }
146296049f3aSAlexei Starovoitov
1463c9d29f46SMauricio Vasquez B key = __bpf_copy_key(ukey, map->key_size);
1464e4448ed8SAl Viro if (IS_ERR(key)) {
1465e4448ed8SAl Viro err = PTR_ERR(key);
1466db20fd2bSAlexei Starovoitov goto err_put;
1467e4448ed8SAl Viro }
1468db20fd2bSAlexei Starovoitov
146915c14a3dSBrian Vazquez value_size = bpf_map_value_size(map);
147015a07b33SAlexei Starovoitov
14718ebe667cSAlexei Starovoitov err = -ENOMEM;
1472f0dce1d9SStanislav Fomichev value = kvmalloc(value_size, GFP_USER | __GFP_NOWARN);
1473db20fd2bSAlexei Starovoitov if (!value)
14748ebe667cSAlexei Starovoitov goto free_key;
14758ebe667cSAlexei Starovoitov
14769330986cSJoanne Koong if (map->map_type == BPF_MAP_TYPE_BLOOM_FILTER) {
14779330986cSJoanne Koong if (copy_from_user(value, uvalue, value_size))
14789330986cSJoanne Koong err = -EFAULT;
14799330986cSJoanne Koong else
14809330986cSJoanne Koong err = bpf_map_copy_value(map, key, value, attr->flags);
14819330986cSJoanne Koong goto free_value;
14829330986cSJoanne Koong }
14839330986cSJoanne Koong
148415c14a3dSBrian Vazquez err = bpf_map_copy_value(map, key, value, attr->flags);
148515a07b33SAlexei Starovoitov if (err)
14868ebe667cSAlexei Starovoitov goto free_value;
1487db20fd2bSAlexei Starovoitov
1488db20fd2bSAlexei Starovoitov err = -EFAULT;
148915a07b33SAlexei Starovoitov if (copy_to_user(uvalue, value, value_size) != 0)
14908ebe667cSAlexei Starovoitov goto free_value;
1491db20fd2bSAlexei Starovoitov
1492db20fd2bSAlexei Starovoitov err = 0;
1493db20fd2bSAlexei Starovoitov
14948ebe667cSAlexei Starovoitov free_value:
1495f0dce1d9SStanislav Fomichev kvfree(value);
1496db20fd2bSAlexei Starovoitov free_key:
149744779a4bSStanislav Fomichev kvfree(key);
1498db20fd2bSAlexei Starovoitov err_put:
1499db20fd2bSAlexei Starovoitov fdput(f);
1500db20fd2bSAlexei Starovoitov return err;
1501db20fd2bSAlexei Starovoitov }
1502db20fd2bSAlexei Starovoitov
15031ae80cf3SDaniel Colascione
15043274f520SAlexei Starovoitov #define BPF_MAP_UPDATE_ELEM_LAST_FIELD flags
1505db20fd2bSAlexei Starovoitov
map_update_elem(union bpf_attr * attr,bpfptr_t uattr)1506af2ac3e1SAlexei Starovoitov static int map_update_elem(union bpf_attr *attr, bpfptr_t uattr)
1507db20fd2bSAlexei Starovoitov {
1508af2ac3e1SAlexei Starovoitov bpfptr_t ukey = make_bpfptr(attr->key, uattr.is_kernel);
1509af2ac3e1SAlexei Starovoitov bpfptr_t uvalue = make_bpfptr(attr->value, uattr.is_kernel);
1510db20fd2bSAlexei Starovoitov int ufd = attr->map_fd;
1511db20fd2bSAlexei Starovoitov struct bpf_map *map;
1512db20fd2bSAlexei Starovoitov void *key, *value;
151315a07b33SAlexei Starovoitov u32 value_size;
1514592867bfSDaniel Borkmann struct fd f;
1515db20fd2bSAlexei Starovoitov int err;
1516db20fd2bSAlexei Starovoitov
1517db20fd2bSAlexei Starovoitov if (CHECK_ATTR(BPF_MAP_UPDATE_ELEM))
1518db20fd2bSAlexei Starovoitov return -EINVAL;
1519db20fd2bSAlexei Starovoitov
1520592867bfSDaniel Borkmann f = fdget(ufd);
1521c2101297SDaniel Borkmann map = __bpf_map_get(f);
1522db20fd2bSAlexei Starovoitov if (IS_ERR(map))
1523db20fd2bSAlexei Starovoitov return PTR_ERR(map);
1524353050beSDaniel Borkmann bpf_map_write_active_inc(map);
152587df15deSDaniel Borkmann if (!(map_get_sys_perms(map, f) & FMODE_CAN_WRITE)) {
15266e71b04aSChenbo Feng err = -EPERM;
15276e71b04aSChenbo Feng goto err_put;
15286e71b04aSChenbo Feng }
15296e71b04aSChenbo Feng
153096049f3aSAlexei Starovoitov if ((attr->flags & BPF_F_LOCK) &&
1531db559117SKumar Kartikeya Dwivedi !btf_record_has_field(map->record, BPF_SPIN_LOCK)) {
153296049f3aSAlexei Starovoitov err = -EINVAL;
153396049f3aSAlexei Starovoitov goto err_put;
153496049f3aSAlexei Starovoitov }
153596049f3aSAlexei Starovoitov
1536af2ac3e1SAlexei Starovoitov key = ___bpf_copy_key(ukey, map->key_size);
1537e4448ed8SAl Viro if (IS_ERR(key)) {
1538e4448ed8SAl Viro err = PTR_ERR(key);
1539db20fd2bSAlexei Starovoitov goto err_put;
1540e4448ed8SAl Viro }
1541db20fd2bSAlexei Starovoitov
1542f0dce1d9SStanislav Fomichev value_size = bpf_map_value_size(map);
1543a02c118eSWang Yufen value = kvmemdup_bpfptr(uvalue, value_size);
1544a02c118eSWang Yufen if (IS_ERR(value)) {
1545a02c118eSWang Yufen err = PTR_ERR(value);
1546db20fd2bSAlexei Starovoitov goto free_key;
1547a02c118eSWang Yufen }
1548db20fd2bSAlexei Starovoitov
15493af43ba4SHou Tao err = bpf_map_update_value(map, f.file, key, value, attr->flags);
15506710e112SJesper Dangaard Brouer
1551f0dce1d9SStanislav Fomichev kvfree(value);
1552db20fd2bSAlexei Starovoitov free_key:
155344779a4bSStanislav Fomichev kvfree(key);
1554db20fd2bSAlexei Starovoitov err_put:
1555353050beSDaniel Borkmann bpf_map_write_active_dec(map);
1556db20fd2bSAlexei Starovoitov fdput(f);
1557db20fd2bSAlexei Starovoitov return err;
1558db20fd2bSAlexei Starovoitov }
1559db20fd2bSAlexei Starovoitov
1560db20fd2bSAlexei Starovoitov #define BPF_MAP_DELETE_ELEM_LAST_FIELD key
1561db20fd2bSAlexei Starovoitov
map_delete_elem(union bpf_attr * attr,bpfptr_t uattr)1562b88df697SBenjamin Tissoires static int map_delete_elem(union bpf_attr *attr, bpfptr_t uattr)
1563db20fd2bSAlexei Starovoitov {
1564b88df697SBenjamin Tissoires bpfptr_t ukey = make_bpfptr(attr->key, uattr.is_kernel);
1565db20fd2bSAlexei Starovoitov int ufd = attr->map_fd;
1566db20fd2bSAlexei Starovoitov struct bpf_map *map;
1567592867bfSDaniel Borkmann struct fd f;
1568db20fd2bSAlexei Starovoitov void *key;
1569db20fd2bSAlexei Starovoitov int err;
1570db20fd2bSAlexei Starovoitov
1571db20fd2bSAlexei Starovoitov if (CHECK_ATTR(BPF_MAP_DELETE_ELEM))
1572db20fd2bSAlexei Starovoitov return -EINVAL;
1573db20fd2bSAlexei Starovoitov
1574592867bfSDaniel Borkmann f = fdget(ufd);
1575c2101297SDaniel Borkmann map = __bpf_map_get(f);
1576db20fd2bSAlexei Starovoitov if (IS_ERR(map))
1577db20fd2bSAlexei Starovoitov return PTR_ERR(map);
1578353050beSDaniel Borkmann bpf_map_write_active_inc(map);
157987df15deSDaniel Borkmann if (!(map_get_sys_perms(map, f) & FMODE_CAN_WRITE)) {
15806e71b04aSChenbo Feng err = -EPERM;
15816e71b04aSChenbo Feng goto err_put;
15826e71b04aSChenbo Feng }
15836e71b04aSChenbo Feng
1584b88df697SBenjamin Tissoires key = ___bpf_copy_key(ukey, map->key_size);
1585e4448ed8SAl Viro if (IS_ERR(key)) {
1586e4448ed8SAl Viro err = PTR_ERR(key);
1587db20fd2bSAlexei Starovoitov goto err_put;
1588e4448ed8SAl Viro }
1589db20fd2bSAlexei Starovoitov
15909d03ebc7SStanislav Fomichev if (bpf_map_is_offloaded(map)) {
1591a3884572SJakub Kicinski err = bpf_map_offload_delete_elem(map, key);
1592a3884572SJakub Kicinski goto out;
159385d33df3SMartin KaFai Lau } else if (IS_FD_PROG_ARRAY(map) ||
159485d33df3SMartin KaFai Lau map->map_type == BPF_MAP_TYPE_STRUCT_OPS) {
159585d33df3SMartin KaFai Lau /* These maps require sleepable context */
1596da765a2fSDaniel Borkmann err = map->ops->map_delete_elem(map, key);
1597da765a2fSDaniel Borkmann goto out;
1598a3884572SJakub Kicinski }
1599a3884572SJakub Kicinski
1600b6e5dae1SThomas Gleixner bpf_disable_instrumentation();
1601db20fd2bSAlexei Starovoitov rcu_read_lock();
1602db20fd2bSAlexei Starovoitov err = map->ops->map_delete_elem(map, key);
1603db20fd2bSAlexei Starovoitov rcu_read_unlock();
1604b6e5dae1SThomas Gleixner bpf_enable_instrumentation();
16051ae80cf3SDaniel Colascione maybe_wait_bpf_programs(map);
1606a3884572SJakub Kicinski out:
160744779a4bSStanislav Fomichev kvfree(key);
1608db20fd2bSAlexei Starovoitov err_put:
1609353050beSDaniel Borkmann bpf_map_write_active_dec(map);
1610db20fd2bSAlexei Starovoitov fdput(f);
1611db20fd2bSAlexei Starovoitov return err;
1612db20fd2bSAlexei Starovoitov }
1613db20fd2bSAlexei Starovoitov
1614db20fd2bSAlexei Starovoitov /* last field in 'union bpf_attr' used by this command */
1615db20fd2bSAlexei Starovoitov #define BPF_MAP_GET_NEXT_KEY_LAST_FIELD next_key
1616db20fd2bSAlexei Starovoitov
map_get_next_key(union bpf_attr * attr)1617db20fd2bSAlexei Starovoitov static int map_get_next_key(union bpf_attr *attr)
1618db20fd2bSAlexei Starovoitov {
1619535e7b4bSMickaël Salaün void __user *ukey = u64_to_user_ptr(attr->key);
1620535e7b4bSMickaël Salaün void __user *unext_key = u64_to_user_ptr(attr->next_key);
1621db20fd2bSAlexei Starovoitov int ufd = attr->map_fd;
1622db20fd2bSAlexei Starovoitov struct bpf_map *map;
1623db20fd2bSAlexei Starovoitov void *key, *next_key;
1624592867bfSDaniel Borkmann struct fd f;
1625db20fd2bSAlexei Starovoitov int err;
1626db20fd2bSAlexei Starovoitov
1627db20fd2bSAlexei Starovoitov if (CHECK_ATTR(BPF_MAP_GET_NEXT_KEY))
1628db20fd2bSAlexei Starovoitov return -EINVAL;
1629db20fd2bSAlexei Starovoitov
1630592867bfSDaniel Borkmann f = fdget(ufd);
1631c2101297SDaniel Borkmann map = __bpf_map_get(f);
1632db20fd2bSAlexei Starovoitov if (IS_ERR(map))
1633db20fd2bSAlexei Starovoitov return PTR_ERR(map);
163487df15deSDaniel Borkmann if (!(map_get_sys_perms(map, f) & FMODE_CAN_READ)) {
16356e71b04aSChenbo Feng err = -EPERM;
16366e71b04aSChenbo Feng goto err_put;
16376e71b04aSChenbo Feng }
16386e71b04aSChenbo Feng
16398fe45924STeng Qin if (ukey) {
1640c9d29f46SMauricio Vasquez B key = __bpf_copy_key(ukey, map->key_size);
1641e4448ed8SAl Viro if (IS_ERR(key)) {
1642e4448ed8SAl Viro err = PTR_ERR(key);
1643db20fd2bSAlexei Starovoitov goto err_put;
1644e4448ed8SAl Viro }
16458fe45924STeng Qin } else {
16468fe45924STeng Qin key = NULL;
16478fe45924STeng Qin }
1648db20fd2bSAlexei Starovoitov
1649db20fd2bSAlexei Starovoitov err = -ENOMEM;
165044779a4bSStanislav Fomichev next_key = kvmalloc(map->key_size, GFP_USER);
1651db20fd2bSAlexei Starovoitov if (!next_key)
1652db20fd2bSAlexei Starovoitov goto free_key;
1653db20fd2bSAlexei Starovoitov
16549d03ebc7SStanislav Fomichev if (bpf_map_is_offloaded(map)) {
1655a3884572SJakub Kicinski err = bpf_map_offload_get_next_key(map, key, next_key);
1656a3884572SJakub Kicinski goto out;
1657a3884572SJakub Kicinski }
1658a3884572SJakub Kicinski
1659db20fd2bSAlexei Starovoitov rcu_read_lock();
1660db20fd2bSAlexei Starovoitov err = map->ops->map_get_next_key(map, key, next_key);
1661db20fd2bSAlexei Starovoitov rcu_read_unlock();
1662a3884572SJakub Kicinski out:
1663db20fd2bSAlexei Starovoitov if (err)
1664db20fd2bSAlexei Starovoitov goto free_next_key;
1665db20fd2bSAlexei Starovoitov
1666db20fd2bSAlexei Starovoitov err = -EFAULT;
1667db20fd2bSAlexei Starovoitov if (copy_to_user(unext_key, next_key, map->key_size) != 0)
1668db20fd2bSAlexei Starovoitov goto free_next_key;
1669db20fd2bSAlexei Starovoitov
1670db20fd2bSAlexei Starovoitov err = 0;
1671db20fd2bSAlexei Starovoitov
1672db20fd2bSAlexei Starovoitov free_next_key:
167344779a4bSStanislav Fomichev kvfree(next_key);
1674db20fd2bSAlexei Starovoitov free_key:
167544779a4bSStanislav Fomichev kvfree(key);
1676db20fd2bSAlexei Starovoitov err_put:
1677db20fd2bSAlexei Starovoitov fdput(f);
1678db20fd2bSAlexei Starovoitov return err;
1679db20fd2bSAlexei Starovoitov }
1680db20fd2bSAlexei Starovoitov
generic_map_delete_batch(struct bpf_map * map,const union bpf_attr * attr,union bpf_attr __user * uattr)1681aa2e93b8SBrian Vazquez int generic_map_delete_batch(struct bpf_map *map,
1682aa2e93b8SBrian Vazquez const union bpf_attr *attr,
1683aa2e93b8SBrian Vazquez union bpf_attr __user *uattr)
1684aa2e93b8SBrian Vazquez {
1685aa2e93b8SBrian Vazquez void __user *keys = u64_to_user_ptr(attr->batch.keys);
1686aa2e93b8SBrian Vazquez u32 cp, max_count;
1687aa2e93b8SBrian Vazquez int err = 0;
1688aa2e93b8SBrian Vazquez void *key;
1689aa2e93b8SBrian Vazquez
1690aa2e93b8SBrian Vazquez if (attr->batch.elem_flags & ~BPF_F_LOCK)
1691aa2e93b8SBrian Vazquez return -EINVAL;
1692aa2e93b8SBrian Vazquez
1693aa2e93b8SBrian Vazquez if ((attr->batch.elem_flags & BPF_F_LOCK) &&
1694db559117SKumar Kartikeya Dwivedi !btf_record_has_field(map->record, BPF_SPIN_LOCK)) {
1695aa2e93b8SBrian Vazquez return -EINVAL;
1696aa2e93b8SBrian Vazquez }
1697aa2e93b8SBrian Vazquez
1698aa2e93b8SBrian Vazquez max_count = attr->batch.count;
1699aa2e93b8SBrian Vazquez if (!max_count)
1700aa2e93b8SBrian Vazquez return 0;
1701aa2e93b8SBrian Vazquez
1702702f1ed4SHou Tao if (put_user(0, &uattr->batch.count))
1703702f1ed4SHou Tao return -EFAULT;
1704702f1ed4SHou Tao
170544779a4bSStanislav Fomichev key = kvmalloc(map->key_size, GFP_USER | __GFP_NOWARN);
17062e3a94aaSBrian Vazquez if (!key)
17072e3a94aaSBrian Vazquez return -ENOMEM;
17082e3a94aaSBrian Vazquez
1709aa2e93b8SBrian Vazquez for (cp = 0; cp < max_count; cp++) {
17102e3a94aaSBrian Vazquez err = -EFAULT;
17112e3a94aaSBrian Vazquez if (copy_from_user(key, keys + cp * map->key_size,
17122e3a94aaSBrian Vazquez map->key_size))
1713aa2e93b8SBrian Vazquez break;
1714aa2e93b8SBrian Vazquez
17159d03ebc7SStanislav Fomichev if (bpf_map_is_offloaded(map)) {
1716aa2e93b8SBrian Vazquez err = bpf_map_offload_delete_elem(map, key);
1717aa2e93b8SBrian Vazquez break;
1718aa2e93b8SBrian Vazquez }
1719aa2e93b8SBrian Vazquez
1720b6e5dae1SThomas Gleixner bpf_disable_instrumentation();
1721aa2e93b8SBrian Vazquez rcu_read_lock();
1722aa2e93b8SBrian Vazquez err = map->ops->map_delete_elem(map, key);
1723aa2e93b8SBrian Vazquez rcu_read_unlock();
1724b6e5dae1SThomas Gleixner bpf_enable_instrumentation();
1725aa2e93b8SBrian Vazquez if (err)
1726aa2e93b8SBrian Vazquez break;
172775134f16SEric Dumazet cond_resched();
1728aa2e93b8SBrian Vazquez }
1729aa2e93b8SBrian Vazquez if (copy_to_user(&uattr->batch.count, &cp, sizeof(cp)))
1730aa2e93b8SBrian Vazquez err = -EFAULT;
17312e3a94aaSBrian Vazquez
173244779a4bSStanislav Fomichev kvfree(key);
17339087c6ffSEric Dumazet
17349087c6ffSEric Dumazet maybe_wait_bpf_programs(map);
1735aa2e93b8SBrian Vazquez return err;
1736aa2e93b8SBrian Vazquez }
1737aa2e93b8SBrian Vazquez
generic_map_update_batch(struct bpf_map * map,struct file * map_file,const union bpf_attr * attr,union bpf_attr __user * uattr)17383af43ba4SHou Tao int generic_map_update_batch(struct bpf_map *map, struct file *map_file,
1739aa2e93b8SBrian Vazquez const union bpf_attr *attr,
1740aa2e93b8SBrian Vazquez union bpf_attr __user *uattr)
1741aa2e93b8SBrian Vazquez {
1742aa2e93b8SBrian Vazquez void __user *values = u64_to_user_ptr(attr->batch.values);
1743aa2e93b8SBrian Vazquez void __user *keys = u64_to_user_ptr(attr->batch.keys);
1744aa2e93b8SBrian Vazquez u32 value_size, cp, max_count;
1745aa2e93b8SBrian Vazquez void *key, *value;
1746aa2e93b8SBrian Vazquez int err = 0;
1747aa2e93b8SBrian Vazquez
1748aa2e93b8SBrian Vazquez if (attr->batch.elem_flags & ~BPF_F_LOCK)
1749aa2e93b8SBrian Vazquez return -EINVAL;
1750aa2e93b8SBrian Vazquez
1751aa2e93b8SBrian Vazquez if ((attr->batch.elem_flags & BPF_F_LOCK) &&
1752db559117SKumar Kartikeya Dwivedi !btf_record_has_field(map->record, BPF_SPIN_LOCK)) {
1753aa2e93b8SBrian Vazquez return -EINVAL;
1754aa2e93b8SBrian Vazquez }
1755aa2e93b8SBrian Vazquez
1756aa2e93b8SBrian Vazquez value_size = bpf_map_value_size(map);
1757aa2e93b8SBrian Vazquez
1758aa2e93b8SBrian Vazquez max_count = attr->batch.count;
1759aa2e93b8SBrian Vazquez if (!max_count)
1760aa2e93b8SBrian Vazquez return 0;
1761aa2e93b8SBrian Vazquez
1762702f1ed4SHou Tao if (put_user(0, &uattr->batch.count))
1763702f1ed4SHou Tao return -EFAULT;
1764702f1ed4SHou Tao
176544779a4bSStanislav Fomichev key = kvmalloc(map->key_size, GFP_USER | __GFP_NOWARN);
17662e3a94aaSBrian Vazquez if (!key)
1767aa2e93b8SBrian Vazquez return -ENOMEM;
1768aa2e93b8SBrian Vazquez
1769f0dce1d9SStanislav Fomichev value = kvmalloc(value_size, GFP_USER | __GFP_NOWARN);
17702e3a94aaSBrian Vazquez if (!value) {
177144779a4bSStanislav Fomichev kvfree(key);
17722e3a94aaSBrian Vazquez return -ENOMEM;
1773aa2e93b8SBrian Vazquez }
17742e3a94aaSBrian Vazquez
17752e3a94aaSBrian Vazquez for (cp = 0; cp < max_count; cp++) {
1776aa2e93b8SBrian Vazquez err = -EFAULT;
17772e3a94aaSBrian Vazquez if (copy_from_user(key, keys + cp * map->key_size,
17782e3a94aaSBrian Vazquez map->key_size) ||
17792e3a94aaSBrian Vazquez copy_from_user(value, values + cp * value_size, value_size))
1780aa2e93b8SBrian Vazquez break;
1781aa2e93b8SBrian Vazquez
17823af43ba4SHou Tao err = bpf_map_update_value(map, map_file, key, value,
1783aa2e93b8SBrian Vazquez attr->batch.elem_flags);
1784aa2e93b8SBrian Vazquez
1785aa2e93b8SBrian Vazquez if (err)
1786aa2e93b8SBrian Vazquez break;
178775134f16SEric Dumazet cond_resched();
1788aa2e93b8SBrian Vazquez }
1789aa2e93b8SBrian Vazquez
1790aa2e93b8SBrian Vazquez if (copy_to_user(&uattr->batch.count, &cp, sizeof(cp)))
1791aa2e93b8SBrian Vazquez err = -EFAULT;
1792aa2e93b8SBrian Vazquez
1793f0dce1d9SStanislav Fomichev kvfree(value);
179444779a4bSStanislav Fomichev kvfree(key);
1795aa2e93b8SBrian Vazquez return err;
1796aa2e93b8SBrian Vazquez }
1797aa2e93b8SBrian Vazquez
1798cb4d03abSBrian Vazquez #define MAP_LOOKUP_RETRIES 3
1799cb4d03abSBrian Vazquez
generic_map_lookup_batch(struct bpf_map * map,const union bpf_attr * attr,union bpf_attr __user * uattr)1800cb4d03abSBrian Vazquez int generic_map_lookup_batch(struct bpf_map *map,
1801cb4d03abSBrian Vazquez const union bpf_attr *attr,
1802cb4d03abSBrian Vazquez union bpf_attr __user *uattr)
1803cb4d03abSBrian Vazquez {
1804cb4d03abSBrian Vazquez void __user *uobatch = u64_to_user_ptr(attr->batch.out_batch);
1805cb4d03abSBrian Vazquez void __user *ubatch = u64_to_user_ptr(attr->batch.in_batch);
1806cb4d03abSBrian Vazquez void __user *values = u64_to_user_ptr(attr->batch.values);
1807cb4d03abSBrian Vazquez void __user *keys = u64_to_user_ptr(attr->batch.keys);
1808cb4d03abSBrian Vazquez void *buf, *buf_prevkey, *prev_key, *key, *value;
1809cb4d03abSBrian Vazquez int err, retry = MAP_LOOKUP_RETRIES;
1810cb4d03abSBrian Vazquez u32 value_size, cp, max_count;
1811cb4d03abSBrian Vazquez
1812cb4d03abSBrian Vazquez if (attr->batch.elem_flags & ~BPF_F_LOCK)
1813cb4d03abSBrian Vazquez return -EINVAL;
1814cb4d03abSBrian Vazquez
1815cb4d03abSBrian Vazquez if ((attr->batch.elem_flags & BPF_F_LOCK) &&
1816db559117SKumar Kartikeya Dwivedi !btf_record_has_field(map->record, BPF_SPIN_LOCK))
1817cb4d03abSBrian Vazquez return -EINVAL;
1818cb4d03abSBrian Vazquez
1819cb4d03abSBrian Vazquez value_size = bpf_map_value_size(map);
1820cb4d03abSBrian Vazquez
1821cb4d03abSBrian Vazquez max_count = attr->batch.count;
1822cb4d03abSBrian Vazquez if (!max_count)
1823cb4d03abSBrian Vazquez return 0;
1824cb4d03abSBrian Vazquez
1825cb4d03abSBrian Vazquez if (put_user(0, &uattr->batch.count))
1826cb4d03abSBrian Vazquez return -EFAULT;
1827cb4d03abSBrian Vazquez
182844779a4bSStanislav Fomichev buf_prevkey = kvmalloc(map->key_size, GFP_USER | __GFP_NOWARN);
1829cb4d03abSBrian Vazquez if (!buf_prevkey)
1830cb4d03abSBrian Vazquez return -ENOMEM;
1831cb4d03abSBrian Vazquez
1832f0dce1d9SStanislav Fomichev buf = kvmalloc(map->key_size + value_size, GFP_USER | __GFP_NOWARN);
1833cb4d03abSBrian Vazquez if (!buf) {
183444779a4bSStanislav Fomichev kvfree(buf_prevkey);
1835cb4d03abSBrian Vazquez return -ENOMEM;
1836cb4d03abSBrian Vazquez }
1837cb4d03abSBrian Vazquez
1838cb4d03abSBrian Vazquez err = -EFAULT;
1839cb4d03abSBrian Vazquez prev_key = NULL;
1840cb4d03abSBrian Vazquez if (ubatch && copy_from_user(buf_prevkey, ubatch, map->key_size))
1841cb4d03abSBrian Vazquez goto free_buf;
1842cb4d03abSBrian Vazquez key = buf;
1843cb4d03abSBrian Vazquez value = key + map->key_size;
1844cb4d03abSBrian Vazquez if (ubatch)
1845cb4d03abSBrian Vazquez prev_key = buf_prevkey;
1846cb4d03abSBrian Vazquez
1847cb4d03abSBrian Vazquez for (cp = 0; cp < max_count;) {
1848cb4d03abSBrian Vazquez rcu_read_lock();
1849cb4d03abSBrian Vazquez err = map->ops->map_get_next_key(map, prev_key, key);
1850cb4d03abSBrian Vazquez rcu_read_unlock();
1851cb4d03abSBrian Vazquez if (err)
1852cb4d03abSBrian Vazquez break;
1853cb4d03abSBrian Vazquez err = bpf_map_copy_value(map, key, value,
1854cb4d03abSBrian Vazquez attr->batch.elem_flags);
1855cb4d03abSBrian Vazquez
1856cb4d03abSBrian Vazquez if (err == -ENOENT) {
1857cb4d03abSBrian Vazquez if (retry) {
1858cb4d03abSBrian Vazquez retry--;
1859cb4d03abSBrian Vazquez continue;
1860cb4d03abSBrian Vazquez }
1861cb4d03abSBrian Vazquez err = -EINTR;
1862cb4d03abSBrian Vazquez break;
1863cb4d03abSBrian Vazquez }
1864cb4d03abSBrian Vazquez
1865cb4d03abSBrian Vazquez if (err)
1866cb4d03abSBrian Vazquez goto free_buf;
1867cb4d03abSBrian Vazquez
1868cb4d03abSBrian Vazquez if (copy_to_user(keys + cp * map->key_size, key,
1869cb4d03abSBrian Vazquez map->key_size)) {
1870cb4d03abSBrian Vazquez err = -EFAULT;
1871cb4d03abSBrian Vazquez goto free_buf;
1872cb4d03abSBrian Vazquez }
1873cb4d03abSBrian Vazquez if (copy_to_user(values + cp * value_size, value, value_size)) {
1874cb4d03abSBrian Vazquez err = -EFAULT;
1875cb4d03abSBrian Vazquez goto free_buf;
1876cb4d03abSBrian Vazquez }
1877cb4d03abSBrian Vazquez
1878cb4d03abSBrian Vazquez if (!prev_key)
1879cb4d03abSBrian Vazquez prev_key = buf_prevkey;
1880cb4d03abSBrian Vazquez
1881cb4d03abSBrian Vazquez swap(prev_key, key);
1882cb4d03abSBrian Vazquez retry = MAP_LOOKUP_RETRIES;
1883cb4d03abSBrian Vazquez cp++;
188475134f16SEric Dumazet cond_resched();
1885cb4d03abSBrian Vazquez }
1886cb4d03abSBrian Vazquez
1887cb4d03abSBrian Vazquez if (err == -EFAULT)
1888cb4d03abSBrian Vazquez goto free_buf;
1889cb4d03abSBrian Vazquez
1890cb4d03abSBrian Vazquez if ((copy_to_user(&uattr->batch.count, &cp, sizeof(cp)) ||
1891cb4d03abSBrian Vazquez (cp && copy_to_user(uobatch, prev_key, map->key_size))))
1892cb4d03abSBrian Vazquez err = -EFAULT;
1893cb4d03abSBrian Vazquez
1894cb4d03abSBrian Vazquez free_buf:
189544779a4bSStanislav Fomichev kvfree(buf_prevkey);
1896f0dce1d9SStanislav Fomichev kvfree(buf);
1897cb4d03abSBrian Vazquez return err;
1898cb4d03abSBrian Vazquez }
1899cb4d03abSBrian Vazquez
19003e87f192SDenis Salopek #define BPF_MAP_LOOKUP_AND_DELETE_ELEM_LAST_FIELD flags
1901bd513cd0SMauricio Vasquez B
map_lookup_and_delete_elem(union bpf_attr * attr)1902bd513cd0SMauricio Vasquez B static int map_lookup_and_delete_elem(union bpf_attr *attr)
1903bd513cd0SMauricio Vasquez B {
1904bd513cd0SMauricio Vasquez B void __user *ukey = u64_to_user_ptr(attr->key);
1905bd513cd0SMauricio Vasquez B void __user *uvalue = u64_to_user_ptr(attr->value);
1906bd513cd0SMauricio Vasquez B int ufd = attr->map_fd;
1907bd513cd0SMauricio Vasquez B struct bpf_map *map;
1908540fefc0SAlexei Starovoitov void *key, *value;
1909bd513cd0SMauricio Vasquez B u32 value_size;
1910bd513cd0SMauricio Vasquez B struct fd f;
1911bd513cd0SMauricio Vasquez B int err;
1912bd513cd0SMauricio Vasquez B
1913bd513cd0SMauricio Vasquez B if (CHECK_ATTR(BPF_MAP_LOOKUP_AND_DELETE_ELEM))
1914bd513cd0SMauricio Vasquez B return -EINVAL;
1915bd513cd0SMauricio Vasquez B
19163e87f192SDenis Salopek if (attr->flags & ~BPF_F_LOCK)
19173e87f192SDenis Salopek return -EINVAL;
19183e87f192SDenis Salopek
1919bd513cd0SMauricio Vasquez B f = fdget(ufd);
1920bd513cd0SMauricio Vasquez B map = __bpf_map_get(f);
1921bd513cd0SMauricio Vasquez B if (IS_ERR(map))
1922bd513cd0SMauricio Vasquez B return PTR_ERR(map);
1923353050beSDaniel Borkmann bpf_map_write_active_inc(map);
19241ea0f912SAnton Protopopov if (!(map_get_sys_perms(map, f) & FMODE_CAN_READ) ||
19251ea0f912SAnton Protopopov !(map_get_sys_perms(map, f) & FMODE_CAN_WRITE)) {
1926bd513cd0SMauricio Vasquez B err = -EPERM;
1927bd513cd0SMauricio Vasquez B goto err_put;
1928bd513cd0SMauricio Vasquez B }
1929bd513cd0SMauricio Vasquez B
19303e87f192SDenis Salopek if (attr->flags &&
19313e87f192SDenis Salopek (map->map_type == BPF_MAP_TYPE_QUEUE ||
19323e87f192SDenis Salopek map->map_type == BPF_MAP_TYPE_STACK)) {
19333e87f192SDenis Salopek err = -EINVAL;
19343e87f192SDenis Salopek goto err_put;
19353e87f192SDenis Salopek }
19363e87f192SDenis Salopek
19373e87f192SDenis Salopek if ((attr->flags & BPF_F_LOCK) &&
1938db559117SKumar Kartikeya Dwivedi !btf_record_has_field(map->record, BPF_SPIN_LOCK)) {
19393e87f192SDenis Salopek err = -EINVAL;
19403e87f192SDenis Salopek goto err_put;
19413e87f192SDenis Salopek }
19423e87f192SDenis Salopek
1943bd513cd0SMauricio Vasquez B key = __bpf_copy_key(ukey, map->key_size);
1944bd513cd0SMauricio Vasquez B if (IS_ERR(key)) {
1945bd513cd0SMauricio Vasquez B err = PTR_ERR(key);
1946bd513cd0SMauricio Vasquez B goto err_put;
1947bd513cd0SMauricio Vasquez B }
1948bd513cd0SMauricio Vasquez B
19493e87f192SDenis Salopek value_size = bpf_map_value_size(map);
1950bd513cd0SMauricio Vasquez B
1951bd513cd0SMauricio Vasquez B err = -ENOMEM;
1952f0dce1d9SStanislav Fomichev value = kvmalloc(value_size, GFP_USER | __GFP_NOWARN);
1953bd513cd0SMauricio Vasquez B if (!value)
1954bd513cd0SMauricio Vasquez B goto free_key;
1955bd513cd0SMauricio Vasquez B
19563e87f192SDenis Salopek err = -ENOTSUPP;
1957bd513cd0SMauricio Vasquez B if (map->map_type == BPF_MAP_TYPE_QUEUE ||
1958bd513cd0SMauricio Vasquez B map->map_type == BPF_MAP_TYPE_STACK) {
1959bd513cd0SMauricio Vasquez B err = map->ops->map_pop_elem(map, value);
19603e87f192SDenis Salopek } else if (map->map_type == BPF_MAP_TYPE_HASH ||
19613e87f192SDenis Salopek map->map_type == BPF_MAP_TYPE_PERCPU_HASH ||
19623e87f192SDenis Salopek map->map_type == BPF_MAP_TYPE_LRU_HASH ||
19633e87f192SDenis Salopek map->map_type == BPF_MAP_TYPE_LRU_PERCPU_HASH) {
19649d03ebc7SStanislav Fomichev if (!bpf_map_is_offloaded(map)) {
19653e87f192SDenis Salopek bpf_disable_instrumentation();
19663e87f192SDenis Salopek rcu_read_lock();
19673e87f192SDenis Salopek err = map->ops->map_lookup_and_delete_elem(map, key, value, attr->flags);
19683e87f192SDenis Salopek rcu_read_unlock();
19693e87f192SDenis Salopek bpf_enable_instrumentation();
19703e87f192SDenis Salopek }
1971bd513cd0SMauricio Vasquez B }
1972bd513cd0SMauricio Vasquez B
1973bd513cd0SMauricio Vasquez B if (err)
1974bd513cd0SMauricio Vasquez B goto free_value;
1975bd513cd0SMauricio Vasquez B
19767f645462SWei Yongjun if (copy_to_user(uvalue, value, value_size) != 0) {
19777f645462SWei Yongjun err = -EFAULT;
1978bd513cd0SMauricio Vasquez B goto free_value;
19797f645462SWei Yongjun }
1980bd513cd0SMauricio Vasquez B
1981bd513cd0SMauricio Vasquez B err = 0;
1982bd513cd0SMauricio Vasquez B
1983bd513cd0SMauricio Vasquez B free_value:
1984f0dce1d9SStanislav Fomichev kvfree(value);
1985bd513cd0SMauricio Vasquez B free_key:
198644779a4bSStanislav Fomichev kvfree(key);
1987bd513cd0SMauricio Vasquez B err_put:
1988353050beSDaniel Borkmann bpf_map_write_active_dec(map);
1989bd513cd0SMauricio Vasquez B fdput(f);
1990bd513cd0SMauricio Vasquez B return err;
1991bd513cd0SMauricio Vasquez B }
1992bd513cd0SMauricio Vasquez B
199387df15deSDaniel Borkmann #define BPF_MAP_FREEZE_LAST_FIELD map_fd
199487df15deSDaniel Borkmann
map_freeze(const union bpf_attr * attr)199587df15deSDaniel Borkmann static int map_freeze(const union bpf_attr *attr)
199687df15deSDaniel Borkmann {
199787df15deSDaniel Borkmann int err = 0, ufd = attr->map_fd;
199887df15deSDaniel Borkmann struct bpf_map *map;
199987df15deSDaniel Borkmann struct fd f;
200087df15deSDaniel Borkmann
200187df15deSDaniel Borkmann if (CHECK_ATTR(BPF_MAP_FREEZE))
200287df15deSDaniel Borkmann return -EINVAL;
200387df15deSDaniel Borkmann
200487df15deSDaniel Borkmann f = fdget(ufd);
200587df15deSDaniel Borkmann map = __bpf_map_get(f);
200687df15deSDaniel Borkmann if (IS_ERR(map))
200787df15deSDaniel Borkmann return PTR_ERR(map);
2008fc970227SAndrii Nakryiko
2009db559117SKumar Kartikeya Dwivedi if (map->map_type == BPF_MAP_TYPE_STRUCT_OPS || !IS_ERR_OR_NULL(map->record)) {
2010849b4d94SMartin KaFai Lau fdput(f);
2011849b4d94SMartin KaFai Lau return -ENOTSUPP;
2012849b4d94SMartin KaFai Lau }
2013849b4d94SMartin KaFai Lau
2014c4c84f6fSAndrii Nakryiko if (!(map_get_sys_perms(map, f) & FMODE_CAN_WRITE)) {
20154266f41fSDaniel Borkmann fdput(f);
20164266f41fSDaniel Borkmann return -EPERM;
2017c4c84f6fSAndrii Nakryiko }
2018c4c84f6fSAndrii Nakryiko
2019fc970227SAndrii Nakryiko mutex_lock(&map->freeze_mutex);
2020353050beSDaniel Borkmann if (bpf_map_write_active(map)) {
2021fc970227SAndrii Nakryiko err = -EBUSY;
2022fc970227SAndrii Nakryiko goto err_put;
2023fc970227SAndrii Nakryiko }
202487df15deSDaniel Borkmann if (READ_ONCE(map->frozen)) {
202587df15deSDaniel Borkmann err = -EBUSY;
202687df15deSDaniel Borkmann goto err_put;
202787df15deSDaniel Borkmann }
202887df15deSDaniel Borkmann
202987df15deSDaniel Borkmann WRITE_ONCE(map->frozen, true);
203087df15deSDaniel Borkmann err_put:
2031fc970227SAndrii Nakryiko mutex_unlock(&map->freeze_mutex);
203287df15deSDaniel Borkmann fdput(f);
203387df15deSDaniel Borkmann return err;
203487df15deSDaniel Borkmann }
203587df15deSDaniel Borkmann
20367de16e3aSJakub Kicinski static const struct bpf_prog_ops * const bpf_prog_types[] = {
203791cc1a99SAlexei Starovoitov #define BPF_PROG_TYPE(_id, _name, prog_ctx_type, kern_ctx_type) \
20387de16e3aSJakub Kicinski [_id] = & _name ## _prog_ops,
20397de16e3aSJakub Kicinski #define BPF_MAP_TYPE(_id, _ops)
2040f2e10bffSAndrii Nakryiko #define BPF_LINK_TYPE(_id, _name)
20417de16e3aSJakub Kicinski #include <linux/bpf_types.h>
20427de16e3aSJakub Kicinski #undef BPF_PROG_TYPE
20437de16e3aSJakub Kicinski #undef BPF_MAP_TYPE
2044f2e10bffSAndrii Nakryiko #undef BPF_LINK_TYPE
20457de16e3aSJakub Kicinski };
20467de16e3aSJakub Kicinski
find_prog_type(enum bpf_prog_type type,struct bpf_prog * prog)204709756af4SAlexei Starovoitov static int find_prog_type(enum bpf_prog_type type, struct bpf_prog *prog)
204809756af4SAlexei Starovoitov {
2049d0f1a451SDaniel Borkmann const struct bpf_prog_ops *ops;
2050d0f1a451SDaniel Borkmann
2051d0f1a451SDaniel Borkmann if (type >= ARRAY_SIZE(bpf_prog_types))
2052d0f1a451SDaniel Borkmann return -EINVAL;
2053d0f1a451SDaniel Borkmann type = array_index_nospec(type, ARRAY_SIZE(bpf_prog_types));
2054d0f1a451SDaniel Borkmann ops = bpf_prog_types[type];
2055d0f1a451SDaniel Borkmann if (!ops)
2056be9370a7SJohannes Berg return -EINVAL;
205709756af4SAlexei Starovoitov
20589d03ebc7SStanislav Fomichev if (!bpf_prog_is_offloaded(prog->aux))
2059d0f1a451SDaniel Borkmann prog->aux->ops = ops;
2060ab3f0063SJakub Kicinski else
2061ab3f0063SJakub Kicinski prog->aux->ops = &bpf_offload_prog_ops;
206224701eceSDaniel Borkmann prog->type = type;
206309756af4SAlexei Starovoitov return 0;
206409756af4SAlexei Starovoitov }
206509756af4SAlexei Starovoitov
2066bae141f5SDaniel Borkmann enum bpf_audit {
2067bae141f5SDaniel Borkmann BPF_AUDIT_LOAD,
2068bae141f5SDaniel Borkmann BPF_AUDIT_UNLOAD,
2069bae141f5SDaniel Borkmann BPF_AUDIT_MAX,
2070bae141f5SDaniel Borkmann };
2071bae141f5SDaniel Borkmann
2072bae141f5SDaniel Borkmann static const char * const bpf_audit_str[BPF_AUDIT_MAX] = {
2073bae141f5SDaniel Borkmann [BPF_AUDIT_LOAD] = "LOAD",
2074bae141f5SDaniel Borkmann [BPF_AUDIT_UNLOAD] = "UNLOAD",
2075bae141f5SDaniel Borkmann };
2076bae141f5SDaniel Borkmann
bpf_audit_prog(const struct bpf_prog * prog,unsigned int op)2077bae141f5SDaniel Borkmann static void bpf_audit_prog(const struct bpf_prog *prog, unsigned int op)
2078bae141f5SDaniel Borkmann {
2079bae141f5SDaniel Borkmann struct audit_context *ctx = NULL;
2080bae141f5SDaniel Borkmann struct audit_buffer *ab;
2081bae141f5SDaniel Borkmann
2082bae141f5SDaniel Borkmann if (WARN_ON_ONCE(op >= BPF_AUDIT_MAX))
2083bae141f5SDaniel Borkmann return;
2084bae141f5SDaniel Borkmann if (audit_enabled == AUDIT_OFF)
2085bae141f5SDaniel Borkmann return;
2086ef01f4e2SPaul Moore if (!in_irq() && !irqs_disabled())
2087bae141f5SDaniel Borkmann ctx = audit_context();
2088bae141f5SDaniel Borkmann ab = audit_log_start(ctx, GFP_ATOMIC, AUDIT_BPF);
2089bae141f5SDaniel Borkmann if (unlikely(!ab))
2090bae141f5SDaniel Borkmann return;
2091bae141f5SDaniel Borkmann audit_log_format(ab, "prog-id=%u op=%s",
2092bae141f5SDaniel Borkmann prog->aux->id, bpf_audit_str[op]);
2093bae141f5SDaniel Borkmann audit_log_end(ab);
2094bae141f5SDaniel Borkmann }
2095bae141f5SDaniel Borkmann
bpf_prog_alloc_id(struct bpf_prog * prog)2096dc4bb0e2SMartin KaFai Lau static int bpf_prog_alloc_id(struct bpf_prog *prog)
2097dc4bb0e2SMartin KaFai Lau {
2098dc4bb0e2SMartin KaFai Lau int id;
2099dc4bb0e2SMartin KaFai Lau
2100b76354cdSShaohua Li idr_preload(GFP_KERNEL);
2101dc4bb0e2SMartin KaFai Lau spin_lock_bh(&prog_idr_lock);
2102dc4bb0e2SMartin KaFai Lau id = idr_alloc_cyclic(&prog_idr, prog, 1, INT_MAX, GFP_ATOMIC);
2103dc4bb0e2SMartin KaFai Lau if (id > 0)
2104dc4bb0e2SMartin KaFai Lau prog->aux->id = id;
2105dc4bb0e2SMartin KaFai Lau spin_unlock_bh(&prog_idr_lock);
2106b76354cdSShaohua Li idr_preload_end();
2107dc4bb0e2SMartin KaFai Lau
2108dc4bb0e2SMartin KaFai Lau /* id is in [1, INT_MAX) */
2109dc4bb0e2SMartin KaFai Lau if (WARN_ON_ONCE(!id))
2110dc4bb0e2SMartin KaFai Lau return -ENOSPC;
2111dc4bb0e2SMartin KaFai Lau
2112dc4bb0e2SMartin KaFai Lau return id > 0 ? 0 : id;
2113dc4bb0e2SMartin KaFai Lau }
2114dc4bb0e2SMartin KaFai Lau
bpf_prog_free_id(struct bpf_prog * prog)2115e7895f01SPaul Moore void bpf_prog_free_id(struct bpf_prog *prog)
2116dc4bb0e2SMartin KaFai Lau {
2117d809e134SAlexei Starovoitov unsigned long flags;
2118d809e134SAlexei Starovoitov
2119ad8ad79fSJakub Kicinski /* cBPF to eBPF migrations are currently not in the idr store.
2120ad8ad79fSJakub Kicinski * Offloaded programs are removed from the store when their device
2121ad8ad79fSJakub Kicinski * disappears - even if someone grabs an fd to them they are unusable,
2122ad8ad79fSJakub Kicinski * simply waiting for refcnt to drop to be freed.
2123ad8ad79fSJakub Kicinski */
2124dc4bb0e2SMartin KaFai Lau if (!prog->aux->id)
2125dc4bb0e2SMartin KaFai Lau return;
2126dc4bb0e2SMartin KaFai Lau
2127d809e134SAlexei Starovoitov spin_lock_irqsave(&prog_idr_lock, flags);
2128dc4bb0e2SMartin KaFai Lau idr_remove(&prog_idr, prog->aux->id);
2129ad8ad79fSJakub Kicinski prog->aux->id = 0;
2130d809e134SAlexei Starovoitov spin_unlock_irqrestore(&prog_idr_lock, flags);
2131dc4bb0e2SMartin KaFai Lau }
2132dc4bb0e2SMartin KaFai Lau
__bpf_prog_put_rcu(struct rcu_head * rcu)21331aacde3dSDaniel Borkmann static void __bpf_prog_put_rcu(struct rcu_head *rcu)
2134abf2e7d6SAlexei Starovoitov {
2135abf2e7d6SAlexei Starovoitov struct bpf_prog_aux *aux = container_of(rcu, struct bpf_prog_aux, rcu);
2136abf2e7d6SAlexei Starovoitov
21373b4d9eb2SDaniel Borkmann kvfree(aux->func_info);
21388c1b6e69SAlexei Starovoitov kfree(aux->func_info_aux);
21393ac1f01bSRoman Gushchin free_uid(aux->user);
2140afdb09c7SChenbo Feng security_bpf_prog_free(aux);
2141abf2e7d6SAlexei Starovoitov bpf_prog_free(aux->prog);
2142abf2e7d6SAlexei Starovoitov }
2143abf2e7d6SAlexei Starovoitov
__bpf_prog_put_noref(struct bpf_prog * prog,bool deferred)2144cd7455f1SDaniel Borkmann static void __bpf_prog_put_noref(struct bpf_prog *prog, bool deferred)
2145cd7455f1SDaniel Borkmann {
2146cd7455f1SDaniel Borkmann bpf_prog_kallsyms_del_all(prog);
2147cd7455f1SDaniel Borkmann btf_put(prog->aux->btf);
214831bf1dbcSViktor Malik module_put(prog->aux->mod);
2149e16301fbSMartin KaFai Lau kvfree(prog->aux->jited_linfo);
2150e16301fbSMartin KaFai Lau kvfree(prog->aux->linfo);
2151e6ac2450SMartin KaFai Lau kfree(prog->aux->kfunc_tab);
215222dc4a0fSAndrii Nakryiko if (prog->aux->attach_btf)
215322dc4a0fSAndrii Nakryiko btf_put(prog->aux->attach_btf);
2154cd7455f1SDaniel Borkmann
21551e6c62a8SAlexei Starovoitov if (deferred) {
21561e6c62a8SAlexei Starovoitov if (prog->aux->sleepable)
21571e6c62a8SAlexei Starovoitov call_rcu_tasks_trace(&prog->aux->rcu, __bpf_prog_put_rcu);
2158cd7455f1SDaniel Borkmann else
21591e6c62a8SAlexei Starovoitov call_rcu(&prog->aux->rcu, __bpf_prog_put_rcu);
21601e6c62a8SAlexei Starovoitov } else {
2161cd7455f1SDaniel Borkmann __bpf_prog_put_rcu(&prog->aux->rcu);
2162cd7455f1SDaniel Borkmann }
21631e6c62a8SAlexei Starovoitov }
2164cd7455f1SDaniel Borkmann
bpf_prog_put_deferred(struct work_struct * work)2165d809e134SAlexei Starovoitov static void bpf_prog_put_deferred(struct work_struct *work)
216609756af4SAlexei Starovoitov {
2167d809e134SAlexei Starovoitov struct bpf_prog_aux *aux;
2168d809e134SAlexei Starovoitov struct bpf_prog *prog;
2169d809e134SAlexei Starovoitov
2170d809e134SAlexei Starovoitov aux = container_of(work, struct bpf_prog_aux, work);
2171d809e134SAlexei Starovoitov prog = aux->prog;
21726ee52e2aSSong Liu perf_event_bpf_event(prog, PERF_BPF_EVENT_PROG_UNLOAD, 0);
2173bae141f5SDaniel Borkmann bpf_audit_prog(prog, BPF_AUDIT_UNLOAD);
2174e7895f01SPaul Moore bpf_prog_free_id(prog);
2175d809e134SAlexei Starovoitov __bpf_prog_put_noref(prog, true);
2176d809e134SAlexei Starovoitov }
2177d809e134SAlexei Starovoitov
__bpf_prog_put(struct bpf_prog * prog)2178e7895f01SPaul Moore static void __bpf_prog_put(struct bpf_prog *prog)
2179d809e134SAlexei Starovoitov {
2180d809e134SAlexei Starovoitov struct bpf_prog_aux *aux = prog->aux;
2181d809e134SAlexei Starovoitov
2182d809e134SAlexei Starovoitov if (atomic64_dec_and_test(&aux->refcnt)) {
2183d809e134SAlexei Starovoitov if (in_irq() || irqs_disabled()) {
2184d809e134SAlexei Starovoitov INIT_WORK(&aux->work, bpf_prog_put_deferred);
2185d809e134SAlexei Starovoitov schedule_work(&aux->work);
2186d809e134SAlexei Starovoitov } else {
2187d809e134SAlexei Starovoitov bpf_prog_put_deferred(&aux->work);
2188d809e134SAlexei Starovoitov }
218909756af4SAlexei Starovoitov }
2190a67edbf4SDaniel Borkmann }
2191b16d9aa4SMartin KaFai Lau
bpf_prog_put(struct bpf_prog * prog)2192b16d9aa4SMartin KaFai Lau void bpf_prog_put(struct bpf_prog *prog)
2193b16d9aa4SMartin KaFai Lau {
2194e7895f01SPaul Moore __bpf_prog_put(prog);
2195b16d9aa4SMartin KaFai Lau }
2196e2e9b654SDaniel Borkmann EXPORT_SYMBOL_GPL(bpf_prog_put);
219709756af4SAlexei Starovoitov
bpf_prog_release(struct inode * inode,struct file * filp)219809756af4SAlexei Starovoitov static int bpf_prog_release(struct inode *inode, struct file *filp)
219909756af4SAlexei Starovoitov {
220009756af4SAlexei Starovoitov struct bpf_prog *prog = filp->private_data;
220109756af4SAlexei Starovoitov
22021aacde3dSDaniel Borkmann bpf_prog_put(prog);
220309756af4SAlexei Starovoitov return 0;
220409756af4SAlexei Starovoitov }
220509756af4SAlexei Starovoitov
220661a0abaeSEric Dumazet struct bpf_prog_kstats {
220761a0abaeSEric Dumazet u64 nsecs;
220861a0abaeSEric Dumazet u64 cnt;
220961a0abaeSEric Dumazet u64 misses;
221061a0abaeSEric Dumazet };
221161a0abaeSEric Dumazet
bpf_prog_inc_misses_counter(struct bpf_prog * prog)221205b24ff9SJiri Olsa void notrace bpf_prog_inc_misses_counter(struct bpf_prog *prog)
221305b24ff9SJiri Olsa {
221405b24ff9SJiri Olsa struct bpf_prog_stats *stats;
221505b24ff9SJiri Olsa unsigned int flags;
221605b24ff9SJiri Olsa
221705b24ff9SJiri Olsa stats = this_cpu_ptr(prog->stats);
221805b24ff9SJiri Olsa flags = u64_stats_update_begin_irqsave(&stats->syncp);
221905b24ff9SJiri Olsa u64_stats_inc(&stats->misses);
222005b24ff9SJiri Olsa u64_stats_update_end_irqrestore(&stats->syncp, flags);
222105b24ff9SJiri Olsa }
222205b24ff9SJiri Olsa
bpf_prog_get_stats(const struct bpf_prog * prog,struct bpf_prog_kstats * stats)2223492ecee8SAlexei Starovoitov static void bpf_prog_get_stats(const struct bpf_prog *prog,
222461a0abaeSEric Dumazet struct bpf_prog_kstats *stats)
2225492ecee8SAlexei Starovoitov {
22269ed9e9baSAlexei Starovoitov u64 nsecs = 0, cnt = 0, misses = 0;
2227492ecee8SAlexei Starovoitov int cpu;
2228492ecee8SAlexei Starovoitov
2229492ecee8SAlexei Starovoitov for_each_possible_cpu(cpu) {
2230492ecee8SAlexei Starovoitov const struct bpf_prog_stats *st;
2231492ecee8SAlexei Starovoitov unsigned int start;
22329ed9e9baSAlexei Starovoitov u64 tnsecs, tcnt, tmisses;
2233492ecee8SAlexei Starovoitov
2234700d4796SAlexei Starovoitov st = per_cpu_ptr(prog->stats, cpu);
2235492ecee8SAlexei Starovoitov do {
223697c4090bSThomas Gleixner start = u64_stats_fetch_begin(&st->syncp);
223761a0abaeSEric Dumazet tnsecs = u64_stats_read(&st->nsecs);
223861a0abaeSEric Dumazet tcnt = u64_stats_read(&st->cnt);
223961a0abaeSEric Dumazet tmisses = u64_stats_read(&st->misses);
224097c4090bSThomas Gleixner } while (u64_stats_fetch_retry(&st->syncp, start));
2241492ecee8SAlexei Starovoitov nsecs += tnsecs;
2242492ecee8SAlexei Starovoitov cnt += tcnt;
22439ed9e9baSAlexei Starovoitov misses += tmisses;
2244492ecee8SAlexei Starovoitov }
2245492ecee8SAlexei Starovoitov stats->nsecs = nsecs;
2246492ecee8SAlexei Starovoitov stats->cnt = cnt;
22479ed9e9baSAlexei Starovoitov stats->misses = misses;
2248492ecee8SAlexei Starovoitov }
2249492ecee8SAlexei Starovoitov
22507bd509e3SDaniel Borkmann #ifdef CONFIG_PROC_FS
bpf_prog_show_fdinfo(struct seq_file * m,struct file * filp)22517bd509e3SDaniel Borkmann static void bpf_prog_show_fdinfo(struct seq_file *m, struct file *filp)
22527bd509e3SDaniel Borkmann {
22537bd509e3SDaniel Borkmann const struct bpf_prog *prog = filp->private_data;
2254f1f7714eSDaniel Borkmann char prog_tag[sizeof(prog->tag) * 2 + 1] = { };
225561a0abaeSEric Dumazet struct bpf_prog_kstats stats;
22567bd509e3SDaniel Borkmann
2257492ecee8SAlexei Starovoitov bpf_prog_get_stats(prog, &stats);
2258f1f7714eSDaniel Borkmann bin2hex(prog_tag, prog->tag, sizeof(prog->tag));
22597bd509e3SDaniel Borkmann seq_printf(m,
22607bd509e3SDaniel Borkmann "prog_type:\t%u\n"
22617bd509e3SDaniel Borkmann "prog_jited:\t%u\n"
2262f1f7714eSDaniel Borkmann "prog_tag:\t%s\n"
22634316b409SDaniel Borkmann "memlock:\t%llu\n"
2264492ecee8SAlexei Starovoitov "prog_id:\t%u\n"
2265492ecee8SAlexei Starovoitov "run_time_ns:\t%llu\n"
22669ed9e9baSAlexei Starovoitov "run_cnt:\t%llu\n"
2267aba64c7dSDave Marchevsky "recursion_misses:\t%llu\n"
2268aba64c7dSDave Marchevsky "verified_insns:\t%u\n",
22697bd509e3SDaniel Borkmann prog->type,
22707bd509e3SDaniel Borkmann prog->jited,
2271f1f7714eSDaniel Borkmann prog_tag,
22724316b409SDaniel Borkmann prog->pages * 1ULL << PAGE_SHIFT,
2273492ecee8SAlexei Starovoitov prog->aux->id,
2274492ecee8SAlexei Starovoitov stats.nsecs,
22759ed9e9baSAlexei Starovoitov stats.cnt,
2276aba64c7dSDave Marchevsky stats.misses,
2277aba64c7dSDave Marchevsky prog->aux->verified_insns);
22787bd509e3SDaniel Borkmann }
22797bd509e3SDaniel Borkmann #endif
22807bd509e3SDaniel Borkmann
2281f66e448cSChenbo Feng const struct file_operations bpf_prog_fops = {
22827bd509e3SDaniel Borkmann #ifdef CONFIG_PROC_FS
22837bd509e3SDaniel Borkmann .show_fdinfo = bpf_prog_show_fdinfo,
22847bd509e3SDaniel Borkmann #endif
228509756af4SAlexei Starovoitov .release = bpf_prog_release,
22866e71b04aSChenbo Feng .read = bpf_dummy_read,
22876e71b04aSChenbo Feng .write = bpf_dummy_write,
228809756af4SAlexei Starovoitov };
228909756af4SAlexei Starovoitov
bpf_prog_new_fd(struct bpf_prog * prog)2290b2197755SDaniel Borkmann int bpf_prog_new_fd(struct bpf_prog *prog)
2291aa79781bSDaniel Borkmann {
2292afdb09c7SChenbo Feng int ret;
2293afdb09c7SChenbo Feng
2294afdb09c7SChenbo Feng ret = security_bpf_prog(prog);
2295afdb09c7SChenbo Feng if (ret < 0)
2296afdb09c7SChenbo Feng return ret;
2297afdb09c7SChenbo Feng
2298aa79781bSDaniel Borkmann return anon_inode_getfd("bpf-prog", &bpf_prog_fops, prog,
2299aa79781bSDaniel Borkmann O_RDWR | O_CLOEXEC);
2300aa79781bSDaniel Borkmann }
2301aa79781bSDaniel Borkmann
____bpf_prog_get(struct fd f)2302113214beSDaniel Borkmann static struct bpf_prog *____bpf_prog_get(struct fd f)
230309756af4SAlexei Starovoitov {
230409756af4SAlexei Starovoitov if (!f.file)
230509756af4SAlexei Starovoitov return ERR_PTR(-EBADF);
230609756af4SAlexei Starovoitov if (f.file->f_op != &bpf_prog_fops) {
230709756af4SAlexei Starovoitov fdput(f);
230809756af4SAlexei Starovoitov return ERR_PTR(-EINVAL);
230909756af4SAlexei Starovoitov }
231009756af4SAlexei Starovoitov
2311c2101297SDaniel Borkmann return f.file->private_data;
231209756af4SAlexei Starovoitov }
231309756af4SAlexei Starovoitov
bpf_prog_add(struct bpf_prog * prog,int i)231485192dbfSAndrii Nakryiko void bpf_prog_add(struct bpf_prog *prog, int i)
231592117d84SAlexei Starovoitov {
231685192dbfSAndrii Nakryiko atomic64_add(i, &prog->aux->refcnt);
231792117d84SAlexei Starovoitov }
231859d3656dSBrenden Blanco EXPORT_SYMBOL_GPL(bpf_prog_add);
231959d3656dSBrenden Blanco
bpf_prog_sub(struct bpf_prog * prog,int i)2320c540594fSDaniel Borkmann void bpf_prog_sub(struct bpf_prog *prog, int i)
2321c540594fSDaniel Borkmann {
2322c540594fSDaniel Borkmann /* Only to be used for undoing previous bpf_prog_add() in some
2323c540594fSDaniel Borkmann * error path. We still know that another entity in our call
2324c540594fSDaniel Borkmann * path holds a reference to the program, thus atomic_sub() can
2325c540594fSDaniel Borkmann * be safely used in such cases!
2326c540594fSDaniel Borkmann */
232785192dbfSAndrii Nakryiko WARN_ON(atomic64_sub_return(i, &prog->aux->refcnt) == 0);
2328c540594fSDaniel Borkmann }
2329c540594fSDaniel Borkmann EXPORT_SYMBOL_GPL(bpf_prog_sub);
2330c540594fSDaniel Borkmann
bpf_prog_inc(struct bpf_prog * prog)233185192dbfSAndrii Nakryiko void bpf_prog_inc(struct bpf_prog *prog)
233259d3656dSBrenden Blanco {
233385192dbfSAndrii Nakryiko atomic64_inc(&prog->aux->refcnt);
233459d3656dSBrenden Blanco }
233597bc402dSDaniel Borkmann EXPORT_SYMBOL_GPL(bpf_prog_inc);
233692117d84SAlexei Starovoitov
2337b16d9aa4SMartin KaFai Lau /* prog_idr_lock should have been held */
bpf_prog_inc_not_zero(struct bpf_prog * prog)2338a6f6df69SJohn Fastabend struct bpf_prog *bpf_prog_inc_not_zero(struct bpf_prog *prog)
2339b16d9aa4SMartin KaFai Lau {
2340b16d9aa4SMartin KaFai Lau int refold;
2341b16d9aa4SMartin KaFai Lau
234285192dbfSAndrii Nakryiko refold = atomic64_fetch_add_unless(&prog->aux->refcnt, 1, 0);
2343b16d9aa4SMartin KaFai Lau
2344b16d9aa4SMartin KaFai Lau if (!refold)
2345b16d9aa4SMartin KaFai Lau return ERR_PTR(-ENOENT);
2346b16d9aa4SMartin KaFai Lau
2347b16d9aa4SMartin KaFai Lau return prog;
2348b16d9aa4SMartin KaFai Lau }
2349a6f6df69SJohn Fastabend EXPORT_SYMBOL_GPL(bpf_prog_inc_not_zero);
2350b16d9aa4SMartin KaFai Lau
bpf_prog_get_ok(struct bpf_prog * prog,enum bpf_prog_type * attach_type,bool attach_drv)2351040ee692SAl Viro bool bpf_prog_get_ok(struct bpf_prog *prog,
2352288b3de5SJakub Kicinski enum bpf_prog_type *attach_type, bool attach_drv)
2353248f346fSJakub Kicinski {
2354288b3de5SJakub Kicinski /* not an attachment, just a refcount inc, always allow */
2355288b3de5SJakub Kicinski if (!attach_type)
2356288b3de5SJakub Kicinski return true;
2357248f346fSJakub Kicinski
2358248f346fSJakub Kicinski if (prog->type != *attach_type)
2359248f346fSJakub Kicinski return false;
23609d03ebc7SStanislav Fomichev if (bpf_prog_is_offloaded(prog->aux) && !attach_drv)
2361248f346fSJakub Kicinski return false;
2362248f346fSJakub Kicinski
2363248f346fSJakub Kicinski return true;
2364248f346fSJakub Kicinski }
2365248f346fSJakub Kicinski
__bpf_prog_get(u32 ufd,enum bpf_prog_type * attach_type,bool attach_drv)2366248f346fSJakub Kicinski static struct bpf_prog *__bpf_prog_get(u32 ufd, enum bpf_prog_type *attach_type,
2367288b3de5SJakub Kicinski bool attach_drv)
236809756af4SAlexei Starovoitov {
236909756af4SAlexei Starovoitov struct fd f = fdget(ufd);
237009756af4SAlexei Starovoitov struct bpf_prog *prog;
237109756af4SAlexei Starovoitov
2372113214beSDaniel Borkmann prog = ____bpf_prog_get(f);
237309756af4SAlexei Starovoitov if (IS_ERR(prog))
237409756af4SAlexei Starovoitov return prog;
2375288b3de5SJakub Kicinski if (!bpf_prog_get_ok(prog, attach_type, attach_drv)) {
2376113214beSDaniel Borkmann prog = ERR_PTR(-EINVAL);
2377113214beSDaniel Borkmann goto out;
2378113214beSDaniel Borkmann }
237909756af4SAlexei Starovoitov
238085192dbfSAndrii Nakryiko bpf_prog_inc(prog);
2381113214beSDaniel Borkmann out:
238209756af4SAlexei Starovoitov fdput(f);
238309756af4SAlexei Starovoitov return prog;
238409756af4SAlexei Starovoitov }
2385113214beSDaniel Borkmann
bpf_prog_get(u32 ufd)2386113214beSDaniel Borkmann struct bpf_prog *bpf_prog_get(u32 ufd)
2387113214beSDaniel Borkmann {
2388288b3de5SJakub Kicinski return __bpf_prog_get(ufd, NULL, false);
2389113214beSDaniel Borkmann }
2390113214beSDaniel Borkmann
bpf_prog_get_type_dev(u32 ufd,enum bpf_prog_type type,bool attach_drv)2391248f346fSJakub Kicinski struct bpf_prog *bpf_prog_get_type_dev(u32 ufd, enum bpf_prog_type type,
2392288b3de5SJakub Kicinski bool attach_drv)
2393248f346fSJakub Kicinski {
23944d220ed0SAlexei Starovoitov return __bpf_prog_get(ufd, &type, attach_drv);
2395248f346fSJakub Kicinski }
23966c8dfe21SJakub Kicinski EXPORT_SYMBOL_GPL(bpf_prog_get_type_dev);
2397248f346fSJakub Kicinski
2398aac3fc32SAndrey Ignatov /* Initially all BPF programs could be loaded w/o specifying
2399aac3fc32SAndrey Ignatov * expected_attach_type. Later for some of them specifying expected_attach_type
2400aac3fc32SAndrey Ignatov * at load time became required so that program could be validated properly.
2401aac3fc32SAndrey Ignatov * Programs of types that are allowed to be loaded both w/ and w/o (for
2402aac3fc32SAndrey Ignatov * backward compatibility) expected_attach_type, should have the default attach
2403aac3fc32SAndrey Ignatov * type assigned to expected_attach_type for the latter case, so that it can be
2404aac3fc32SAndrey Ignatov * validated later at attach time.
2405aac3fc32SAndrey Ignatov *
2406aac3fc32SAndrey Ignatov * bpf_prog_load_fixup_attach_type() sets expected_attach_type in @attr if
2407aac3fc32SAndrey Ignatov * prog type requires it but has some attach types that have to be backward
2408aac3fc32SAndrey Ignatov * compatible.
2409aac3fc32SAndrey Ignatov */
bpf_prog_load_fixup_attach_type(union bpf_attr * attr)2410aac3fc32SAndrey Ignatov static void bpf_prog_load_fixup_attach_type(union bpf_attr *attr)
2411aac3fc32SAndrey Ignatov {
2412aac3fc32SAndrey Ignatov switch (attr->prog_type) {
2413aac3fc32SAndrey Ignatov case BPF_PROG_TYPE_CGROUP_SOCK:
2414aac3fc32SAndrey Ignatov /* Unfortunately BPF_ATTACH_TYPE_UNSPEC enumeration doesn't
2415aac3fc32SAndrey Ignatov * exist so checking for non-zero is the way to go here.
2416aac3fc32SAndrey Ignatov */
2417aac3fc32SAndrey Ignatov if (!attr->expected_attach_type)
2418aac3fc32SAndrey Ignatov attr->expected_attach_type =
2419aac3fc32SAndrey Ignatov BPF_CGROUP_INET_SOCK_CREATE;
2420aac3fc32SAndrey Ignatov break;
2421d5e4ddaeSKuniyuki Iwashima case BPF_PROG_TYPE_SK_REUSEPORT:
2422d5e4ddaeSKuniyuki Iwashima if (!attr->expected_attach_type)
2423d5e4ddaeSKuniyuki Iwashima attr->expected_attach_type =
2424d5e4ddaeSKuniyuki Iwashima BPF_SK_REUSEPORT_SELECT;
2425d5e4ddaeSKuniyuki Iwashima break;
2426aac3fc32SAndrey Ignatov }
2427aac3fc32SAndrey Ignatov }
2428aac3fc32SAndrey Ignatov
24295e43f899SAndrey Ignatov static int
bpf_prog_load_check_attach(enum bpf_prog_type prog_type,enum bpf_attach_type expected_attach_type,struct btf * attach_btf,u32 btf_id,struct bpf_prog * dst_prog)2430ccfe29ebSAlexei Starovoitov bpf_prog_load_check_attach(enum bpf_prog_type prog_type,
2431ccfe29ebSAlexei Starovoitov enum bpf_attach_type expected_attach_type,
2432290248a5SAndrii Nakryiko struct btf *attach_btf, u32 btf_id,
2433290248a5SAndrii Nakryiko struct bpf_prog *dst_prog)
24345e43f899SAndrey Ignatov {
243527ae7997SMartin KaFai Lau if (btf_id) {
2436c108e3c1SAlexei Starovoitov if (btf_id > BTF_MAX_TYPE)
2437c108e3c1SAlexei Starovoitov return -EINVAL;
243827ae7997SMartin KaFai Lau
2439290248a5SAndrii Nakryiko if (!attach_btf && !dst_prog)
2440290248a5SAndrii Nakryiko return -EINVAL;
2441290248a5SAndrii Nakryiko
244227ae7997SMartin KaFai Lau switch (prog_type) {
244327ae7997SMartin KaFai Lau case BPF_PROG_TYPE_TRACING:
24449e4e01dfSKP Singh case BPF_PROG_TYPE_LSM:
244527ae7997SMartin KaFai Lau case BPF_PROG_TYPE_STRUCT_OPS:
2446be8704ffSAlexei Starovoitov case BPF_PROG_TYPE_EXT:
2447c108e3c1SAlexei Starovoitov break;
2448c108e3c1SAlexei Starovoitov default:
2449c108e3c1SAlexei Starovoitov return -EINVAL;
2450c108e3c1SAlexei Starovoitov }
245127ae7997SMartin KaFai Lau }
245227ae7997SMartin KaFai Lau
2453290248a5SAndrii Nakryiko if (attach_btf && (!btf_id || dst_prog))
2454290248a5SAndrii Nakryiko return -EINVAL;
2455290248a5SAndrii Nakryiko
2456290248a5SAndrii Nakryiko if (dst_prog && prog_type != BPF_PROG_TYPE_TRACING &&
2457be8704ffSAlexei Starovoitov prog_type != BPF_PROG_TYPE_EXT)
245827ae7997SMartin KaFai Lau return -EINVAL;
2459c108e3c1SAlexei Starovoitov
2460c108e3c1SAlexei Starovoitov switch (prog_type) {
2461aac3fc32SAndrey Ignatov case BPF_PROG_TYPE_CGROUP_SOCK:
2462aac3fc32SAndrey Ignatov switch (expected_attach_type) {
2463aac3fc32SAndrey Ignatov case BPF_CGROUP_INET_SOCK_CREATE:
2464f5836749SStanislav Fomichev case BPF_CGROUP_INET_SOCK_RELEASE:
2465aac3fc32SAndrey Ignatov case BPF_CGROUP_INET4_POST_BIND:
2466aac3fc32SAndrey Ignatov case BPF_CGROUP_INET6_POST_BIND:
2467aac3fc32SAndrey Ignatov return 0;
2468aac3fc32SAndrey Ignatov default:
2469aac3fc32SAndrey Ignatov return -EINVAL;
2470aac3fc32SAndrey Ignatov }
24714fbac77dSAndrey Ignatov case BPF_PROG_TYPE_CGROUP_SOCK_ADDR:
24724fbac77dSAndrey Ignatov switch (expected_attach_type) {
24734fbac77dSAndrey Ignatov case BPF_CGROUP_INET4_BIND:
24744fbac77dSAndrey Ignatov case BPF_CGROUP_INET6_BIND:
2475d74bad4eSAndrey Ignatov case BPF_CGROUP_INET4_CONNECT:
2476d74bad4eSAndrey Ignatov case BPF_CGROUP_INET6_CONNECT:
24771b66d253SDaniel Borkmann case BPF_CGROUP_INET4_GETPEERNAME:
24781b66d253SDaniel Borkmann case BPF_CGROUP_INET6_GETPEERNAME:
24791b66d253SDaniel Borkmann case BPF_CGROUP_INET4_GETSOCKNAME:
24801b66d253SDaniel Borkmann case BPF_CGROUP_INET6_GETSOCKNAME:
24811cedee13SAndrey Ignatov case BPF_CGROUP_UDP4_SENDMSG:
24821cedee13SAndrey Ignatov case BPF_CGROUP_UDP6_SENDMSG:
2483983695faSDaniel Borkmann case BPF_CGROUP_UDP4_RECVMSG:
2484983695faSDaniel Borkmann case BPF_CGROUP_UDP6_RECVMSG:
24855e43f899SAndrey Ignatov return 0;
24864fbac77dSAndrey Ignatov default:
24874fbac77dSAndrey Ignatov return -EINVAL;
24884fbac77dSAndrey Ignatov }
24895cf1e914Sbrakmo case BPF_PROG_TYPE_CGROUP_SKB:
24905cf1e914Sbrakmo switch (expected_attach_type) {
24915cf1e914Sbrakmo case BPF_CGROUP_INET_INGRESS:
24925cf1e914Sbrakmo case BPF_CGROUP_INET_EGRESS:
24935cf1e914Sbrakmo return 0;
24945cf1e914Sbrakmo default:
24955cf1e914Sbrakmo return -EINVAL;
24965cf1e914Sbrakmo }
24970d01da6aSStanislav Fomichev case BPF_PROG_TYPE_CGROUP_SOCKOPT:
24980d01da6aSStanislav Fomichev switch (expected_attach_type) {
24990d01da6aSStanislav Fomichev case BPF_CGROUP_SETSOCKOPT:
25000d01da6aSStanislav Fomichev case BPF_CGROUP_GETSOCKOPT:
25010d01da6aSStanislav Fomichev return 0;
25020d01da6aSStanislav Fomichev default:
25030d01da6aSStanislav Fomichev return -EINVAL;
25040d01da6aSStanislav Fomichev }
2505e9ddbb77SJakub Sitnicki case BPF_PROG_TYPE_SK_LOOKUP:
2506e9ddbb77SJakub Sitnicki if (expected_attach_type == BPF_SK_LOOKUP)
2507e9ddbb77SJakub Sitnicki return 0;
2508e9ddbb77SJakub Sitnicki return -EINVAL;
2509d5e4ddaeSKuniyuki Iwashima case BPF_PROG_TYPE_SK_REUSEPORT:
2510d5e4ddaeSKuniyuki Iwashima switch (expected_attach_type) {
2511d5e4ddaeSKuniyuki Iwashima case BPF_SK_REUSEPORT_SELECT:
2512d5e4ddaeSKuniyuki Iwashima case BPF_SK_REUSEPORT_SELECT_OR_MIGRATE:
2513d5e4ddaeSKuniyuki Iwashima return 0;
2514d5e4ddaeSKuniyuki Iwashima default:
2515d5e4ddaeSKuniyuki Iwashima return -EINVAL;
2516d5e4ddaeSKuniyuki Iwashima }
2517132328e8SFlorian Westphal case BPF_PROG_TYPE_NETFILTER:
2518132328e8SFlorian Westphal if (expected_attach_type == BPF_NETFILTER)
2519132328e8SFlorian Westphal return 0;
2520132328e8SFlorian Westphal return -EINVAL;
252179a7f8bdSAlexei Starovoitov case BPF_PROG_TYPE_SYSCALL:
2522be8704ffSAlexei Starovoitov case BPF_PROG_TYPE_EXT:
2523be8704ffSAlexei Starovoitov if (expected_attach_type)
2524be8704ffSAlexei Starovoitov return -EINVAL;
2525df561f66SGustavo A. R. Silva fallthrough;
25264fbac77dSAndrey Ignatov default:
25274fbac77dSAndrey Ignatov return 0;
25284fbac77dSAndrey Ignatov }
25295e43f899SAndrey Ignatov }
25305e43f899SAndrey Ignatov
is_net_admin_prog_type(enum bpf_prog_type prog_type)25312c78ee89SAlexei Starovoitov static bool is_net_admin_prog_type(enum bpf_prog_type prog_type)
25322c78ee89SAlexei Starovoitov {
25332c78ee89SAlexei Starovoitov switch (prog_type) {
25342c78ee89SAlexei Starovoitov case BPF_PROG_TYPE_SCHED_CLS:
25352c78ee89SAlexei Starovoitov case BPF_PROG_TYPE_SCHED_ACT:
25362c78ee89SAlexei Starovoitov case BPF_PROG_TYPE_XDP:
25372c78ee89SAlexei Starovoitov case BPF_PROG_TYPE_LWT_IN:
25382c78ee89SAlexei Starovoitov case BPF_PROG_TYPE_LWT_OUT:
25392c78ee89SAlexei Starovoitov case BPF_PROG_TYPE_LWT_XMIT:
25402c78ee89SAlexei Starovoitov case BPF_PROG_TYPE_LWT_SEG6LOCAL:
25412c78ee89SAlexei Starovoitov case BPF_PROG_TYPE_SK_SKB:
25422c78ee89SAlexei Starovoitov case BPF_PROG_TYPE_SK_MSG:
25432c78ee89SAlexei Starovoitov case BPF_PROG_TYPE_FLOW_DISSECTOR:
25442c78ee89SAlexei Starovoitov case BPF_PROG_TYPE_CGROUP_DEVICE:
25452c78ee89SAlexei Starovoitov case BPF_PROG_TYPE_CGROUP_SOCK:
25462c78ee89SAlexei Starovoitov case BPF_PROG_TYPE_CGROUP_SOCK_ADDR:
25472c78ee89SAlexei Starovoitov case BPF_PROG_TYPE_CGROUP_SOCKOPT:
25482c78ee89SAlexei Starovoitov case BPF_PROG_TYPE_CGROUP_SYSCTL:
25492c78ee89SAlexei Starovoitov case BPF_PROG_TYPE_SOCK_OPS:
25502c78ee89SAlexei Starovoitov case BPF_PROG_TYPE_EXT: /* extends any prog */
255184601d6eSFlorian Westphal case BPF_PROG_TYPE_NETFILTER:
25522c78ee89SAlexei Starovoitov return true;
25532c78ee89SAlexei Starovoitov case BPF_PROG_TYPE_CGROUP_SKB:
25542c78ee89SAlexei Starovoitov /* always unpriv */
25552c78ee89SAlexei Starovoitov case BPF_PROG_TYPE_SK_REUSEPORT:
25562c78ee89SAlexei Starovoitov /* equivalent to SOCKET_FILTER. need CAP_BPF only */
25572c78ee89SAlexei Starovoitov default:
25582c78ee89SAlexei Starovoitov return false;
25592c78ee89SAlexei Starovoitov }
25602c78ee89SAlexei Starovoitov }
25612c78ee89SAlexei Starovoitov
is_perfmon_prog_type(enum bpf_prog_type prog_type)25622c78ee89SAlexei Starovoitov static bool is_perfmon_prog_type(enum bpf_prog_type prog_type)
25632c78ee89SAlexei Starovoitov {
25642c78ee89SAlexei Starovoitov switch (prog_type) {
25652c78ee89SAlexei Starovoitov case BPF_PROG_TYPE_KPROBE:
25662c78ee89SAlexei Starovoitov case BPF_PROG_TYPE_TRACEPOINT:
25672c78ee89SAlexei Starovoitov case BPF_PROG_TYPE_PERF_EVENT:
25682c78ee89SAlexei Starovoitov case BPF_PROG_TYPE_RAW_TRACEPOINT:
25692c78ee89SAlexei Starovoitov case BPF_PROG_TYPE_RAW_TRACEPOINT_WRITABLE:
25702c78ee89SAlexei Starovoitov case BPF_PROG_TYPE_TRACING:
25712c78ee89SAlexei Starovoitov case BPF_PROG_TYPE_LSM:
25722c78ee89SAlexei Starovoitov case BPF_PROG_TYPE_STRUCT_OPS: /* has access to struct sock */
25732c78ee89SAlexei Starovoitov case BPF_PROG_TYPE_EXT: /* extends any prog */
25742c78ee89SAlexei Starovoitov return true;
25752c78ee89SAlexei Starovoitov default:
25762c78ee89SAlexei Starovoitov return false;
25772c78ee89SAlexei Starovoitov }
25782c78ee89SAlexei Starovoitov }
25792c78ee89SAlexei Starovoitov
258009756af4SAlexei Starovoitov /* last field in 'union bpf_attr' used by this command */
258147a71c1fSAndrii Nakryiko #define BPF_PROG_LOAD_LAST_FIELD log_true_size
258209756af4SAlexei Starovoitov
bpf_prog_load(union bpf_attr * attr,bpfptr_t uattr,u32 uattr_size)258347a71c1fSAndrii Nakryiko static int bpf_prog_load(union bpf_attr *attr, bpfptr_t uattr, u32 uattr_size)
258409756af4SAlexei Starovoitov {
258509756af4SAlexei Starovoitov enum bpf_prog_type type = attr->prog_type;
2586290248a5SAndrii Nakryiko struct bpf_prog *prog, *dst_prog = NULL;
2587290248a5SAndrii Nakryiko struct btf *attach_btf = NULL;
258809756af4SAlexei Starovoitov int err;
258909756af4SAlexei Starovoitov char license[128];
259009756af4SAlexei Starovoitov
259109756af4SAlexei Starovoitov if (CHECK_ATTR(BPF_PROG_LOAD))
259209756af4SAlexei Starovoitov return -EINVAL;
259309756af4SAlexei Starovoitov
2594c240eff6SJiong Wang if (attr->prog_flags & ~(BPF_F_STRICT_ALIGNMENT |
2595c240eff6SJiong Wang BPF_F_ANY_ALIGNMENT |
259610d274e8SAlexei Starovoitov BPF_F_TEST_STATE_FREQ |
25971e6c62a8SAlexei Starovoitov BPF_F_SLEEPABLE |
2598c2f2cdbeSLorenzo Bianconi BPF_F_TEST_RND_HI32 |
25992b3486bcSStanislav Fomichev BPF_F_XDP_HAS_FRAGS |
26002b3486bcSStanislav Fomichev BPF_F_XDP_DEV_BOUND_ONLY))
2601e07b98d9SDavid S. Miller return -EINVAL;
2602e07b98d9SDavid S. Miller
2603e9ee9efcSDavid Miller if (!IS_ENABLED(CONFIG_HAVE_EFFICIENT_UNALIGNED_ACCESS) &&
2604e9ee9efcSDavid Miller (attr->prog_flags & BPF_F_ANY_ALIGNMENT) &&
26052c78ee89SAlexei Starovoitov !bpf_capable())
2606e9ee9efcSDavid Miller return -EPERM;
2607e9ee9efcSDavid Miller
26081d28635aSAndrii Nakryiko /* Intent here is for unprivileged_bpf_disabled to block BPF program
26091d28635aSAndrii Nakryiko * creation for unprivileged users; other actions depend
26101d28635aSAndrii Nakryiko * on fd availability and access to bpffs, so are dependent on
26111d28635aSAndrii Nakryiko * object creation success. Even with unprivileged BPF disabled,
26121d28635aSAndrii Nakryiko * capability checks are still carried out for these
26131d28635aSAndrii Nakryiko * and other operations.
26141d28635aSAndrii Nakryiko */
26151d28635aSAndrii Nakryiko if (sysctl_unprivileged_bpf_disabled && !bpf_capable())
26161d28635aSAndrii Nakryiko return -EPERM;
261709756af4SAlexei Starovoitov
2618c04c0d2bSAlexei Starovoitov if (attr->insn_cnt == 0 ||
26192c78ee89SAlexei Starovoitov attr->insn_cnt > (bpf_capable() ? BPF_COMPLEXITY_LIMIT_INSNS : BPF_MAXINSNS))
2620ef0915caSDaniel Borkmann return -E2BIG;
262180b7d819SChenbo Feng if (type != BPF_PROG_TYPE_SOCKET_FILTER &&
262280b7d819SChenbo Feng type != BPF_PROG_TYPE_CGROUP_SKB &&
26232c78ee89SAlexei Starovoitov !bpf_capable())
26242c78ee89SAlexei Starovoitov return -EPERM;
26252c78ee89SAlexei Starovoitov
2626b338cb92SMaciej Żenczykowski if (is_net_admin_prog_type(type) && !capable(CAP_NET_ADMIN) && !capable(CAP_SYS_ADMIN))
26272c78ee89SAlexei Starovoitov return -EPERM;
26282c78ee89SAlexei Starovoitov if (is_perfmon_prog_type(type) && !perfmon_capable())
26291be7f75dSAlexei Starovoitov return -EPERM;
26301be7f75dSAlexei Starovoitov
2631290248a5SAndrii Nakryiko /* attach_prog_fd/attach_btf_obj_fd can specify fd of either bpf_prog
2632290248a5SAndrii Nakryiko * or btf, we need to check which one it is
2633290248a5SAndrii Nakryiko */
2634290248a5SAndrii Nakryiko if (attr->attach_prog_fd) {
2635290248a5SAndrii Nakryiko dst_prog = bpf_prog_get(attr->attach_prog_fd);
2636290248a5SAndrii Nakryiko if (IS_ERR(dst_prog)) {
2637290248a5SAndrii Nakryiko dst_prog = NULL;
2638290248a5SAndrii Nakryiko attach_btf = btf_get_by_fd(attr->attach_btf_obj_fd);
2639290248a5SAndrii Nakryiko if (IS_ERR(attach_btf))
2640290248a5SAndrii Nakryiko return -EINVAL;
2641290248a5SAndrii Nakryiko if (!btf_is_kernel(attach_btf)) {
26428bdd8e27SAndrii Nakryiko /* attaching through specifying bpf_prog's BTF
26438bdd8e27SAndrii Nakryiko * objects directly might be supported eventually
26448bdd8e27SAndrii Nakryiko */
2645290248a5SAndrii Nakryiko btf_put(attach_btf);
26468bdd8e27SAndrii Nakryiko return -ENOTSUPP;
2647290248a5SAndrii Nakryiko }
2648290248a5SAndrii Nakryiko }
2649290248a5SAndrii Nakryiko } else if (attr->attach_btf_id) {
2650290248a5SAndrii Nakryiko /* fall back to vmlinux BTF, if BTF type ID is specified */
2651290248a5SAndrii Nakryiko attach_btf = bpf_get_btf_vmlinux();
2652290248a5SAndrii Nakryiko if (IS_ERR(attach_btf))
2653290248a5SAndrii Nakryiko return PTR_ERR(attach_btf);
2654290248a5SAndrii Nakryiko if (!attach_btf)
2655290248a5SAndrii Nakryiko return -EINVAL;
2656290248a5SAndrii Nakryiko btf_get(attach_btf);
2657290248a5SAndrii Nakryiko }
2658290248a5SAndrii Nakryiko
2659aac3fc32SAndrey Ignatov bpf_prog_load_fixup_attach_type(attr);
2660ccfe29ebSAlexei Starovoitov if (bpf_prog_load_check_attach(type, attr->expected_attach_type,
2661290248a5SAndrii Nakryiko attach_btf, attr->attach_btf_id,
2662290248a5SAndrii Nakryiko dst_prog)) {
2663290248a5SAndrii Nakryiko if (dst_prog)
2664290248a5SAndrii Nakryiko bpf_prog_put(dst_prog);
2665290248a5SAndrii Nakryiko if (attach_btf)
2666290248a5SAndrii Nakryiko btf_put(attach_btf);
26675e43f899SAndrey Ignatov return -EINVAL;
2668290248a5SAndrii Nakryiko }
26695e43f899SAndrey Ignatov
267009756af4SAlexei Starovoitov /* plain bpf_prog allocation */
267109756af4SAlexei Starovoitov prog = bpf_prog_alloc(bpf_prog_size(attr->insn_cnt), GFP_USER);
2672290248a5SAndrii Nakryiko if (!prog) {
2673290248a5SAndrii Nakryiko if (dst_prog)
2674290248a5SAndrii Nakryiko bpf_prog_put(dst_prog);
2675290248a5SAndrii Nakryiko if (attach_btf)
2676290248a5SAndrii Nakryiko btf_put(attach_btf);
267709756af4SAlexei Starovoitov return -ENOMEM;
2678290248a5SAndrii Nakryiko }
267909756af4SAlexei Starovoitov
26805e43f899SAndrey Ignatov prog->expected_attach_type = attr->expected_attach_type;
2681290248a5SAndrii Nakryiko prog->aux->attach_btf = attach_btf;
2682ccfe29ebSAlexei Starovoitov prog->aux->attach_btf_id = attr->attach_btf_id;
26833aac1eadSToke Høiland-Jørgensen prog->aux->dst_prog = dst_prog;
26842b3486bcSStanislav Fomichev prog->aux->dev_bound = !!attr->prog_ifindex;
26851e6c62a8SAlexei Starovoitov prog->aux->sleepable = attr->prog_flags & BPF_F_SLEEPABLE;
2686c2f2cdbeSLorenzo Bianconi prog->aux->xdp_has_frags = attr->prog_flags & BPF_F_XDP_HAS_FRAGS;
26879a18eedbSJakub Kicinski
2688afdb09c7SChenbo Feng err = security_bpf_prog_alloc(prog->aux);
2689aaac3ba9SAlexei Starovoitov if (err)
26903ac1f01bSRoman Gushchin goto free_prog;
2691aaac3ba9SAlexei Starovoitov
26923ac1f01bSRoman Gushchin prog->aux->user = get_current_user();
269309756af4SAlexei Starovoitov prog->len = attr->insn_cnt;
269409756af4SAlexei Starovoitov
269509756af4SAlexei Starovoitov err = -EFAULT;
2696af2ac3e1SAlexei Starovoitov if (copy_from_bpfptr(prog->insns,
2697af2ac3e1SAlexei Starovoitov make_bpfptr(attr->insns, uattr.is_kernel),
2698aafe6ae9SDaniel Borkmann bpf_prog_insn_size(prog)) != 0)
26993ac1f01bSRoman Gushchin goto free_prog_sec;
27007f6719f7SAndrii Nakryiko /* copy eBPF program license from user space */
27017f6719f7SAndrii Nakryiko if (strncpy_from_bpfptr(license,
27027f6719f7SAndrii Nakryiko make_bpfptr(attr->license, uattr.is_kernel),
27037f6719f7SAndrii Nakryiko sizeof(license) - 1) < 0)
27047f6719f7SAndrii Nakryiko goto free_prog_sec;
27057f6719f7SAndrii Nakryiko license[sizeof(license) - 1] = 0;
27067f6719f7SAndrii Nakryiko
27077f6719f7SAndrii Nakryiko /* eBPF programs must be GPL compatible to use GPL-ed functions */
27087f6719f7SAndrii Nakryiko prog->gpl_compatible = license_is_gpl_compatible(license) ? 1 : 0;
270909756af4SAlexei Starovoitov
271009756af4SAlexei Starovoitov prog->orig_prog = NULL;
2711a91263d5SDaniel Borkmann prog->jited = 0;
271209756af4SAlexei Starovoitov
271385192dbfSAndrii Nakryiko atomic64_set(&prog->aux->refcnt, 1);
271409756af4SAlexei Starovoitov
27159a18eedbSJakub Kicinski if (bpf_prog_is_dev_bound(prog->aux)) {
27162b3486bcSStanislav Fomichev err = bpf_prog_dev_bound_init(prog, attr);
2717ab3f0063SJakub Kicinski if (err)
27183ac1f01bSRoman Gushchin goto free_prog_sec;
2719ab3f0063SJakub Kicinski }
2720ab3f0063SJakub Kicinski
2721fd7c211dSToke Høiland-Jørgensen if (type == BPF_PROG_TYPE_EXT && dst_prog &&
2722fd7c211dSToke Høiland-Jørgensen bpf_prog_is_dev_bound(dst_prog->aux)) {
2723fd7c211dSToke Høiland-Jørgensen err = bpf_prog_dev_bound_inherit(prog, dst_prog);
2724cb4d2b3fSMartin KaFai Lau if (err)
2725cb4d2b3fSMartin KaFai Lau goto free_prog_sec;
2726cb4d2b3fSMartin KaFai Lau }
2727cb4d2b3fSMartin KaFai Lau
2728cb4d2b3fSMartin KaFai Lau /* find program type: socket_filter vs tracing_filter */
272909756af4SAlexei Starovoitov err = find_prog_type(type, prog);
27309bac3d6dSAlexei Starovoitov if (err < 0)
27313ac1f01bSRoman Gushchin goto free_prog_sec;
273209756af4SAlexei Starovoitov
27339285ec4cSJason A. Donenfeld prog->aux->load_time = ktime_get_boottime_ns();
27348e7ae251SMartin KaFai Lau err = bpf_obj_name_cpy(prog->aux->name, attr->prog_name,
27358e7ae251SMartin KaFai Lau sizeof(attr->prog_name));
27368e7ae251SMartin KaFai Lau if (err < 0)
27373ac1f01bSRoman Gushchin goto free_prog_sec;
273809756af4SAlexei Starovoitov
273951580e79SAlexei Starovoitov /* run eBPF verifier */
274047a71c1fSAndrii Nakryiko err = bpf_check(&prog, attr, uattr, uattr_size);
274109756af4SAlexei Starovoitov if (err < 0)
274209756af4SAlexei Starovoitov goto free_used_maps;
274309756af4SAlexei Starovoitov
2744d1c55ab5SDaniel Borkmann prog = bpf_prog_select_runtime(prog, &err);
274504fd61abSAlexei Starovoitov if (err < 0)
274604fd61abSAlexei Starovoitov goto free_used_maps;
274709756af4SAlexei Starovoitov
2748dc4bb0e2SMartin KaFai Lau err = bpf_prog_alloc_id(prog);
2749dc4bb0e2SMartin KaFai Lau if (err)
2750dc4bb0e2SMartin KaFai Lau goto free_used_maps;
2751dc4bb0e2SMartin KaFai Lau
2752c751798aSDaniel Borkmann /* Upon success of bpf_prog_alloc_id(), the BPF prog is
2753c751798aSDaniel Borkmann * effectively publicly exposed. However, retrieving via
2754c751798aSDaniel Borkmann * bpf_prog_get_fd_by_id() will take another reference,
2755c751798aSDaniel Borkmann * therefore it cannot be gone underneath us.
2756c751798aSDaniel Borkmann *
2757c751798aSDaniel Borkmann * Only for the time /after/ successful bpf_prog_new_fd()
2758c751798aSDaniel Borkmann * and before returning to userspace, we might just hold
2759c751798aSDaniel Borkmann * one reference and any parallel close on that fd could
2760c751798aSDaniel Borkmann * rip everything out. Hence, below notifications must
2761c751798aSDaniel Borkmann * happen before bpf_prog_new_fd().
2762c751798aSDaniel Borkmann *
2763c751798aSDaniel Borkmann * Also, any failure handling from this point onwards must
2764c751798aSDaniel Borkmann * be using bpf_prog_put() given the program is exposed.
2765b16d9aa4SMartin KaFai Lau */
276674451e66SDaniel Borkmann bpf_prog_kallsyms_add(prog);
27676ee52e2aSSong Liu perf_event_bpf_event(prog, PERF_BPF_EVENT_PROG_LOAD, 0);
2768bae141f5SDaniel Borkmann bpf_audit_prog(prog, BPF_AUDIT_LOAD);
2769c751798aSDaniel Borkmann
2770c751798aSDaniel Borkmann err = bpf_prog_new_fd(prog);
2771c751798aSDaniel Borkmann if (err < 0)
2772c751798aSDaniel Borkmann bpf_prog_put(prog);
277309756af4SAlexei Starovoitov return err;
277409756af4SAlexei Starovoitov
277509756af4SAlexei Starovoitov free_used_maps:
2776cd7455f1SDaniel Borkmann /* In case we have subprogs, we need to wait for a grace
2777cd7455f1SDaniel Borkmann * period before we can tear down JIT memory since symbols
2778cd7455f1SDaniel Borkmann * are already exposed under kallsyms.
2779cd7455f1SDaniel Borkmann */
2780cd7455f1SDaniel Borkmann __bpf_prog_put_noref(prog, prog->aux->func_cnt);
2781cd7455f1SDaniel Borkmann return err;
2782afdb09c7SChenbo Feng free_prog_sec:
27833ac1f01bSRoman Gushchin free_uid(prog->aux->user);
2784afdb09c7SChenbo Feng security_bpf_prog_free(prog->aux);
27853ac1f01bSRoman Gushchin free_prog:
278622dc4a0fSAndrii Nakryiko if (prog->aux->attach_btf)
278722dc4a0fSAndrii Nakryiko btf_put(prog->aux->attach_btf);
278809756af4SAlexei Starovoitov bpf_prog_free(prog);
278909756af4SAlexei Starovoitov return err;
279009756af4SAlexei Starovoitov }
279109756af4SAlexei Starovoitov
2792cb8edce2SAndrii Nakryiko #define BPF_OBJ_LAST_FIELD path_fd
2793b2197755SDaniel Borkmann
bpf_obj_pin(const union bpf_attr * attr)2794b2197755SDaniel Borkmann static int bpf_obj_pin(const union bpf_attr *attr)
2795b2197755SDaniel Borkmann {
2796cb8edce2SAndrii Nakryiko int path_fd;
2797cb8edce2SAndrii Nakryiko
2798cb8edce2SAndrii Nakryiko if (CHECK_ATTR(BPF_OBJ) || attr->file_flags & ~BPF_F_PATH_FD)
2799b2197755SDaniel Borkmann return -EINVAL;
2800b2197755SDaniel Borkmann
2801cb8edce2SAndrii Nakryiko /* path_fd has to be accompanied by BPF_F_PATH_FD flag */
2802cb8edce2SAndrii Nakryiko if (!(attr->file_flags & BPF_F_PATH_FD) && attr->path_fd)
2803cb8edce2SAndrii Nakryiko return -EINVAL;
2804cb8edce2SAndrii Nakryiko
2805cb8edce2SAndrii Nakryiko path_fd = attr->file_flags & BPF_F_PATH_FD ? attr->path_fd : AT_FDCWD;
2806cb8edce2SAndrii Nakryiko return bpf_obj_pin_user(attr->bpf_fd, path_fd,
2807cb8edce2SAndrii Nakryiko u64_to_user_ptr(attr->pathname));
2808b2197755SDaniel Borkmann }
2809b2197755SDaniel Borkmann
bpf_obj_get(const union bpf_attr * attr)2810b2197755SDaniel Borkmann static int bpf_obj_get(const union bpf_attr *attr)
2811b2197755SDaniel Borkmann {
2812cb8edce2SAndrii Nakryiko int path_fd;
2813cb8edce2SAndrii Nakryiko
28146e71b04aSChenbo Feng if (CHECK_ATTR(BPF_OBJ) || attr->bpf_fd != 0 ||
2815cb8edce2SAndrii Nakryiko attr->file_flags & ~(BPF_OBJ_FLAG_MASK | BPF_F_PATH_FD))
2816b2197755SDaniel Borkmann return -EINVAL;
2817b2197755SDaniel Borkmann
2818cb8edce2SAndrii Nakryiko /* path_fd has to be accompanied by BPF_F_PATH_FD flag */
2819cb8edce2SAndrii Nakryiko if (!(attr->file_flags & BPF_F_PATH_FD) && attr->path_fd)
2820cb8edce2SAndrii Nakryiko return -EINVAL;
2821cb8edce2SAndrii Nakryiko
2822cb8edce2SAndrii Nakryiko path_fd = attr->file_flags & BPF_F_PATH_FD ? attr->path_fd : AT_FDCWD;
2823cb8edce2SAndrii Nakryiko return bpf_obj_get_user(path_fd, u64_to_user_ptr(attr->pathname),
28246e71b04aSChenbo Feng attr->file_flags);
2825b2197755SDaniel Borkmann }
2826b2197755SDaniel Borkmann
bpf_link_init(struct bpf_link * link,enum bpf_link_type type,const struct bpf_link_ops * ops,struct bpf_prog * prog)2827f2e10bffSAndrii Nakryiko void bpf_link_init(struct bpf_link *link, enum bpf_link_type type,
2828a3b80e10SAndrii Nakryiko const struct bpf_link_ops *ops, struct bpf_prog *prog)
282970ed506cSAndrii Nakryiko {
283070ed506cSAndrii Nakryiko atomic64_set(&link->refcnt, 1);
2831f2e10bffSAndrii Nakryiko link->type = type;
2832a3b80e10SAndrii Nakryiko link->id = 0;
283370ed506cSAndrii Nakryiko link->ops = ops;
283470ed506cSAndrii Nakryiko link->prog = prog;
283570ed506cSAndrii Nakryiko }
283670ed506cSAndrii Nakryiko
bpf_link_free_id(int id)2837a3b80e10SAndrii Nakryiko static void bpf_link_free_id(int id)
2838a3b80e10SAndrii Nakryiko {
2839a3b80e10SAndrii Nakryiko if (!id)
2840a3b80e10SAndrii Nakryiko return;
2841a3b80e10SAndrii Nakryiko
2842a3b80e10SAndrii Nakryiko spin_lock_bh(&link_idr_lock);
2843a3b80e10SAndrii Nakryiko idr_remove(&link_idr, id);
2844a3b80e10SAndrii Nakryiko spin_unlock_bh(&link_idr_lock);
2845a3b80e10SAndrii Nakryiko }
2846a3b80e10SAndrii Nakryiko
284798868668SAndrii Nakryiko /* Clean up bpf_link and corresponding anon_inode file and FD. After
284898868668SAndrii Nakryiko * anon_inode is created, bpf_link can't be just kfree()'d due to deferred
2849a3b80e10SAndrii Nakryiko * anon_inode's release() call. This helper marks bpf_link as
2850a3b80e10SAndrii Nakryiko * defunct, releases anon_inode file and puts reserved FD. bpf_prog's refcnt
2851a3b80e10SAndrii Nakryiko * is not decremented, it's the responsibility of a calling code that failed
2852a3b80e10SAndrii Nakryiko * to complete bpf_link initialization.
285389ae89f5SJiri Olsa * This helper eventually calls link's dealloc callback, but does not call
285489ae89f5SJiri Olsa * link's release callback.
285598868668SAndrii Nakryiko */
bpf_link_cleanup(struct bpf_link_primer * primer)2856a3b80e10SAndrii Nakryiko void bpf_link_cleanup(struct bpf_link_primer *primer)
2857babf3164SAndrii Nakryiko {
2858a3b80e10SAndrii Nakryiko primer->link->prog = NULL;
2859a3b80e10SAndrii Nakryiko bpf_link_free_id(primer->id);
2860a3b80e10SAndrii Nakryiko fput(primer->file);
2861a3b80e10SAndrii Nakryiko put_unused_fd(primer->fd);
2862babf3164SAndrii Nakryiko }
2863babf3164SAndrii Nakryiko
bpf_link_inc(struct bpf_link * link)286470ed506cSAndrii Nakryiko void bpf_link_inc(struct bpf_link *link)
286570ed506cSAndrii Nakryiko {
286670ed506cSAndrii Nakryiko atomic64_inc(&link->refcnt);
286770ed506cSAndrii Nakryiko }
286870ed506cSAndrii Nakryiko
bpf_link_defer_dealloc_rcu_gp(struct rcu_head * rcu)2869*876941f5SAndrii Nakryiko static void bpf_link_defer_dealloc_rcu_gp(struct rcu_head *rcu)
2870*876941f5SAndrii Nakryiko {
2871*876941f5SAndrii Nakryiko struct bpf_link *link = container_of(rcu, struct bpf_link, rcu);
2872*876941f5SAndrii Nakryiko
2873*876941f5SAndrii Nakryiko /* free bpf_link and its containing memory */
2874*876941f5SAndrii Nakryiko link->ops->dealloc_deferred(link);
2875*876941f5SAndrii Nakryiko }
2876*876941f5SAndrii Nakryiko
bpf_link_defer_dealloc_mult_rcu_gp(struct rcu_head * rcu)2877*876941f5SAndrii Nakryiko static void bpf_link_defer_dealloc_mult_rcu_gp(struct rcu_head *rcu)
2878*876941f5SAndrii Nakryiko {
2879*876941f5SAndrii Nakryiko if (rcu_trace_implies_rcu_gp())
2880*876941f5SAndrii Nakryiko bpf_link_defer_dealloc_rcu_gp(rcu);
2881*876941f5SAndrii Nakryiko else
2882*876941f5SAndrii Nakryiko call_rcu(rcu, bpf_link_defer_dealloc_rcu_gp);
2883*876941f5SAndrii Nakryiko }
2884*876941f5SAndrii Nakryiko
288570ed506cSAndrii Nakryiko /* bpf_link_free is guaranteed to be called from process context */
bpf_link_free(struct bpf_link * link)288670ed506cSAndrii Nakryiko static void bpf_link_free(struct bpf_link *link)
288770ed506cSAndrii Nakryiko {
2888*876941f5SAndrii Nakryiko bool sleepable = false;
2889*876941f5SAndrii Nakryiko
2890a3b80e10SAndrii Nakryiko bpf_link_free_id(link->id);
2891babf3164SAndrii Nakryiko if (link->prog) {
2892*876941f5SAndrii Nakryiko sleepable = link->prog->aux->sleepable;
2893babf3164SAndrii Nakryiko /* detach BPF program, clean up used resources */
289470ed506cSAndrii Nakryiko link->ops->release(link);
2895babf3164SAndrii Nakryiko bpf_prog_put(link->prog);
2896babf3164SAndrii Nakryiko }
2897*876941f5SAndrii Nakryiko if (link->ops->dealloc_deferred) {
2898*876941f5SAndrii Nakryiko /* schedule BPF link deallocation; if underlying BPF program
2899*876941f5SAndrii Nakryiko * is sleepable, we need to first wait for RCU tasks trace
2900*876941f5SAndrii Nakryiko * sync, then go through "classic" RCU grace period
2901*876941f5SAndrii Nakryiko */
2902*876941f5SAndrii Nakryiko if (sleepable)
2903*876941f5SAndrii Nakryiko call_rcu_tasks_trace(&link->rcu, bpf_link_defer_dealloc_mult_rcu_gp);
2904*876941f5SAndrii Nakryiko else
2905*876941f5SAndrii Nakryiko call_rcu(&link->rcu, bpf_link_defer_dealloc_rcu_gp);
2906*876941f5SAndrii Nakryiko }
2907*876941f5SAndrii Nakryiko if (link->ops->dealloc)
2908babf3164SAndrii Nakryiko link->ops->dealloc(link);
290970ed506cSAndrii Nakryiko }
291070ed506cSAndrii Nakryiko
bpf_link_put_deferred(struct work_struct * work)291170ed506cSAndrii Nakryiko static void bpf_link_put_deferred(struct work_struct *work)
291270ed506cSAndrii Nakryiko {
291370ed506cSAndrii Nakryiko struct bpf_link *link = container_of(work, struct bpf_link, work);
291470ed506cSAndrii Nakryiko
291570ed506cSAndrii Nakryiko bpf_link_free(link);
291670ed506cSAndrii Nakryiko }
291770ed506cSAndrii Nakryiko
2918ab5d47bdSSebastian Andrzej Siewior /* bpf_link_put might be called from atomic context. It needs to be called
2919ab5d47bdSSebastian Andrzej Siewior * from sleepable context in order to acquire sleeping locks during the process.
292070ed506cSAndrii Nakryiko */
bpf_link_put(struct bpf_link * link)292170ed506cSAndrii Nakryiko void bpf_link_put(struct bpf_link *link)
292270ed506cSAndrii Nakryiko {
292370ed506cSAndrii Nakryiko if (!atomic64_dec_and_test(&link->refcnt))
292470ed506cSAndrii Nakryiko return;
292570ed506cSAndrii Nakryiko
292670ed506cSAndrii Nakryiko INIT_WORK(&link->work, bpf_link_put_deferred);
292770ed506cSAndrii Nakryiko schedule_work(&link->work);
292870ed506cSAndrii Nakryiko }
2929cb80ddc6SAlexei Starovoitov EXPORT_SYMBOL(bpf_link_put);
293070ed506cSAndrii Nakryiko
bpf_link_put_direct(struct bpf_link * link)2931ab5d47bdSSebastian Andrzej Siewior static void bpf_link_put_direct(struct bpf_link *link)
2932ab5d47bdSSebastian Andrzej Siewior {
2933ab5d47bdSSebastian Andrzej Siewior if (!atomic64_dec_and_test(&link->refcnt))
2934ab5d47bdSSebastian Andrzej Siewior return;
2935ab5d47bdSSebastian Andrzej Siewior bpf_link_free(link);
2936ab5d47bdSSebastian Andrzej Siewior }
2937ab5d47bdSSebastian Andrzej Siewior
bpf_link_release(struct inode * inode,struct file * filp)293870ed506cSAndrii Nakryiko static int bpf_link_release(struct inode *inode, struct file *filp)
293970ed506cSAndrii Nakryiko {
294070ed506cSAndrii Nakryiko struct bpf_link *link = filp->private_data;
294170ed506cSAndrii Nakryiko
2942ab5d47bdSSebastian Andrzej Siewior bpf_link_put_direct(link);
2943fec56f58SAlexei Starovoitov return 0;
2944fec56f58SAlexei Starovoitov }
2945fec56f58SAlexei Starovoitov
294670ed506cSAndrii Nakryiko #ifdef CONFIG_PROC_FS
2947f2e10bffSAndrii Nakryiko #define BPF_PROG_TYPE(_id, _name, prog_ctx_type, kern_ctx_type)
2948f2e10bffSAndrii Nakryiko #define BPF_MAP_TYPE(_id, _ops)
2949f2e10bffSAndrii Nakryiko #define BPF_LINK_TYPE(_id, _name) [_id] = #_name,
2950f2e10bffSAndrii Nakryiko static const char *bpf_link_type_strs[] = {
2951f2e10bffSAndrii Nakryiko [BPF_LINK_TYPE_UNSPEC] = "<invalid>",
2952f2e10bffSAndrii Nakryiko #include <linux/bpf_types.h>
2953f2e10bffSAndrii Nakryiko };
2954f2e10bffSAndrii Nakryiko #undef BPF_PROG_TYPE
2955f2e10bffSAndrii Nakryiko #undef BPF_MAP_TYPE
2956f2e10bffSAndrii Nakryiko #undef BPF_LINK_TYPE
295770ed506cSAndrii Nakryiko
bpf_link_show_fdinfo(struct seq_file * m,struct file * filp)295870ed506cSAndrii Nakryiko static void bpf_link_show_fdinfo(struct seq_file *m, struct file *filp)
295970ed506cSAndrii Nakryiko {
296070ed506cSAndrii Nakryiko const struct bpf_link *link = filp->private_data;
296170ed506cSAndrii Nakryiko const struct bpf_prog *prog = link->prog;
296270ed506cSAndrii Nakryiko char prog_tag[sizeof(prog->tag) * 2 + 1] = { };
296370ed506cSAndrii Nakryiko
296470ed506cSAndrii Nakryiko seq_printf(m,
296570ed506cSAndrii Nakryiko "link_type:\t%s\n"
296668b04864SKui-Feng Lee "link_id:\t%u\n",
296768b04864SKui-Feng Lee bpf_link_type_strs[link->type],
296868b04864SKui-Feng Lee link->id);
296968b04864SKui-Feng Lee if (prog) {
297068b04864SKui-Feng Lee bin2hex(prog_tag, prog->tag, sizeof(prog->tag));
297168b04864SKui-Feng Lee seq_printf(m,
297270ed506cSAndrii Nakryiko "prog_tag:\t%s\n"
297370ed506cSAndrii Nakryiko "prog_id:\t%u\n",
297470ed506cSAndrii Nakryiko prog_tag,
297570ed506cSAndrii Nakryiko prog->aux->id);
297668b04864SKui-Feng Lee }
2977f2e10bffSAndrii Nakryiko if (link->ops->show_fdinfo)
2978f2e10bffSAndrii Nakryiko link->ops->show_fdinfo(link, m);
297970ed506cSAndrii Nakryiko }
298070ed506cSAndrii Nakryiko #endif
298170ed506cSAndrii Nakryiko
29826f302bfbSZou Wei static const struct file_operations bpf_link_fops = {
298370ed506cSAndrii Nakryiko #ifdef CONFIG_PROC_FS
298470ed506cSAndrii Nakryiko .show_fdinfo = bpf_link_show_fdinfo,
298570ed506cSAndrii Nakryiko #endif
298670ed506cSAndrii Nakryiko .release = bpf_link_release,
2987fec56f58SAlexei Starovoitov .read = bpf_dummy_read,
2988fec56f58SAlexei Starovoitov .write = bpf_dummy_write,
2989fec56f58SAlexei Starovoitov };
2990fec56f58SAlexei Starovoitov
bpf_link_alloc_id(struct bpf_link * link)2991a3b80e10SAndrii Nakryiko static int bpf_link_alloc_id(struct bpf_link *link)
299270ed506cSAndrii Nakryiko {
2993a3b80e10SAndrii Nakryiko int id;
2994a3b80e10SAndrii Nakryiko
2995a3b80e10SAndrii Nakryiko idr_preload(GFP_KERNEL);
2996a3b80e10SAndrii Nakryiko spin_lock_bh(&link_idr_lock);
2997a3b80e10SAndrii Nakryiko id = idr_alloc_cyclic(&link_idr, link, 1, INT_MAX, GFP_ATOMIC);
2998a3b80e10SAndrii Nakryiko spin_unlock_bh(&link_idr_lock);
2999a3b80e10SAndrii Nakryiko idr_preload_end();
3000a3b80e10SAndrii Nakryiko
3001a3b80e10SAndrii Nakryiko return id;
300270ed506cSAndrii Nakryiko }
300370ed506cSAndrii Nakryiko
3004a3b80e10SAndrii Nakryiko /* Prepare bpf_link to be exposed to user-space by allocating anon_inode file,
3005a3b80e10SAndrii Nakryiko * reserving unused FD and allocating ID from link_idr. This is to be paired
3006a3b80e10SAndrii Nakryiko * with bpf_link_settle() to install FD and ID and expose bpf_link to
3007a3b80e10SAndrii Nakryiko * user-space, if bpf_link is successfully attached. If not, bpf_link and
3008a3b80e10SAndrii Nakryiko * pre-allocated resources are to be freed with bpf_cleanup() call. All the
3009a3b80e10SAndrii Nakryiko * transient state is passed around in struct bpf_link_primer.
3010a3b80e10SAndrii Nakryiko * This is preferred way to create and initialize bpf_link, especially when
3011a3b80e10SAndrii Nakryiko * there are complicated and expensive operations in between creating bpf_link
3012a3b80e10SAndrii Nakryiko * itself and attaching it to BPF hook. By using bpf_link_prime() and
3013a3b80e10SAndrii Nakryiko * bpf_link_settle() kernel code using bpf_link doesn't have to perform
3014a3b80e10SAndrii Nakryiko * expensive (and potentially failing) roll back operations in a rare case
3015a3b80e10SAndrii Nakryiko * that file, FD, or ID can't be allocated.
3016babf3164SAndrii Nakryiko */
bpf_link_prime(struct bpf_link * link,struct bpf_link_primer * primer)3017a3b80e10SAndrii Nakryiko int bpf_link_prime(struct bpf_link *link, struct bpf_link_primer *primer)
3018babf3164SAndrii Nakryiko {
3019babf3164SAndrii Nakryiko struct file *file;
3020a3b80e10SAndrii Nakryiko int fd, id;
3021babf3164SAndrii Nakryiko
3022babf3164SAndrii Nakryiko fd = get_unused_fd_flags(O_CLOEXEC);
3023babf3164SAndrii Nakryiko if (fd < 0)
3024a3b80e10SAndrii Nakryiko return fd;
3025babf3164SAndrii Nakryiko
3026babf3164SAndrii Nakryiko
3027a3b80e10SAndrii Nakryiko id = bpf_link_alloc_id(link);
3028a3b80e10SAndrii Nakryiko if (id < 0) {
3029a3b80e10SAndrii Nakryiko put_unused_fd(fd);
3030a3b80e10SAndrii Nakryiko return id;
3031a3b80e10SAndrii Nakryiko }
3032babf3164SAndrii Nakryiko
3033babf3164SAndrii Nakryiko file = anon_inode_getfile("bpf_link", &bpf_link_fops, link, O_CLOEXEC);
3034babf3164SAndrii Nakryiko if (IS_ERR(file)) {
3035138c6767SAndrii Nakryiko bpf_link_free_id(id);
3036babf3164SAndrii Nakryiko put_unused_fd(fd);
3037138c6767SAndrii Nakryiko return PTR_ERR(file);
3038babf3164SAndrii Nakryiko }
3039babf3164SAndrii Nakryiko
3040a3b80e10SAndrii Nakryiko primer->link = link;
3041a3b80e10SAndrii Nakryiko primer->file = file;
3042a3b80e10SAndrii Nakryiko primer->fd = fd;
3043a3b80e10SAndrii Nakryiko primer->id = id;
3044a3b80e10SAndrii Nakryiko return 0;
3045a3b80e10SAndrii Nakryiko }
3046a3b80e10SAndrii Nakryiko
bpf_link_settle(struct bpf_link_primer * primer)3047a3b80e10SAndrii Nakryiko int bpf_link_settle(struct bpf_link_primer *primer)
3048a3b80e10SAndrii Nakryiko {
3049a3b80e10SAndrii Nakryiko /* make bpf_link fetchable by ID */
3050a3b80e10SAndrii Nakryiko spin_lock_bh(&link_idr_lock);
3051a3b80e10SAndrii Nakryiko primer->link->id = primer->id;
3052a3b80e10SAndrii Nakryiko spin_unlock_bh(&link_idr_lock);
3053a3b80e10SAndrii Nakryiko /* make bpf_link fetchable by FD */
3054a3b80e10SAndrii Nakryiko fd_install(primer->fd, primer->file);
3055a3b80e10SAndrii Nakryiko /* pass through installed FD */
3056a3b80e10SAndrii Nakryiko return primer->fd;
3057a3b80e10SAndrii Nakryiko }
3058a3b80e10SAndrii Nakryiko
bpf_link_new_fd(struct bpf_link * link)3059a3b80e10SAndrii Nakryiko int bpf_link_new_fd(struct bpf_link *link)
3060a3b80e10SAndrii Nakryiko {
3061a3b80e10SAndrii Nakryiko return anon_inode_getfd("bpf-link", &bpf_link_fops, link, O_CLOEXEC);
3062babf3164SAndrii Nakryiko }
3063babf3164SAndrii Nakryiko
bpf_link_get_from_fd(u32 ufd)306470ed506cSAndrii Nakryiko struct bpf_link *bpf_link_get_from_fd(u32 ufd)
306570ed506cSAndrii Nakryiko {
306670ed506cSAndrii Nakryiko struct fd f = fdget(ufd);
306770ed506cSAndrii Nakryiko struct bpf_link *link;
306870ed506cSAndrii Nakryiko
306970ed506cSAndrii Nakryiko if (!f.file)
307070ed506cSAndrii Nakryiko return ERR_PTR(-EBADF);
307170ed506cSAndrii Nakryiko if (f.file->f_op != &bpf_link_fops) {
307270ed506cSAndrii Nakryiko fdput(f);
307370ed506cSAndrii Nakryiko return ERR_PTR(-EINVAL);
307470ed506cSAndrii Nakryiko }
307570ed506cSAndrii Nakryiko
307670ed506cSAndrii Nakryiko link = f.file->private_data;
307770ed506cSAndrii Nakryiko bpf_link_inc(link);
307870ed506cSAndrii Nakryiko fdput(f);
307970ed506cSAndrii Nakryiko
308070ed506cSAndrii Nakryiko return link;
308170ed506cSAndrii Nakryiko }
3082cb80ddc6SAlexei Starovoitov EXPORT_SYMBOL(bpf_link_get_from_fd);
308370ed506cSAndrii Nakryiko
bpf_tracing_link_release(struct bpf_link * link)308470ed506cSAndrii Nakryiko static void bpf_tracing_link_release(struct bpf_link *link)
308570ed506cSAndrii Nakryiko {
30863aac1eadSToke Høiland-Jørgensen struct bpf_tracing_link *tr_link =
3087f7e0beafSKui-Feng Lee container_of(link, struct bpf_tracing_link, link.link);
30883aac1eadSToke Høiland-Jørgensen
3089f7e0beafSKui-Feng Lee WARN_ON_ONCE(bpf_trampoline_unlink_prog(&tr_link->link,
30903aac1eadSToke Høiland-Jørgensen tr_link->trampoline));
30913aac1eadSToke Høiland-Jørgensen
30923aac1eadSToke Høiland-Jørgensen bpf_trampoline_put(tr_link->trampoline);
30933aac1eadSToke Høiland-Jørgensen
30943aac1eadSToke Høiland-Jørgensen /* tgt_prog is NULL if target is a kernel function */
30953aac1eadSToke Høiland-Jørgensen if (tr_link->tgt_prog)
30963aac1eadSToke Høiland-Jørgensen bpf_prog_put(tr_link->tgt_prog);
3097babf3164SAndrii Nakryiko }
3098babf3164SAndrii Nakryiko
bpf_tracing_link_dealloc(struct bpf_link * link)3099babf3164SAndrii Nakryiko static void bpf_tracing_link_dealloc(struct bpf_link *link)
3100babf3164SAndrii Nakryiko {
310170ed506cSAndrii Nakryiko struct bpf_tracing_link *tr_link =
3102f7e0beafSKui-Feng Lee container_of(link, struct bpf_tracing_link, link.link);
310370ed506cSAndrii Nakryiko
310470ed506cSAndrii Nakryiko kfree(tr_link);
310570ed506cSAndrii Nakryiko }
310670ed506cSAndrii Nakryiko
bpf_tracing_link_show_fdinfo(const struct bpf_link * link,struct seq_file * seq)3107f2e10bffSAndrii Nakryiko static void bpf_tracing_link_show_fdinfo(const struct bpf_link *link,
3108f2e10bffSAndrii Nakryiko struct seq_file *seq)
3109f2e10bffSAndrii Nakryiko {
3110f2e10bffSAndrii Nakryiko struct bpf_tracing_link *tr_link =
3111f7e0beafSKui-Feng Lee container_of(link, struct bpf_tracing_link, link.link);
3112e859e429SYafang Shao u32 target_btf_id, target_obj_id;
3113f2e10bffSAndrii Nakryiko
3114e859e429SYafang Shao bpf_trampoline_unpack_key(tr_link->trampoline->key,
3115e859e429SYafang Shao &target_obj_id, &target_btf_id);
3116f2e10bffSAndrii Nakryiko seq_printf(seq,
3117e859e429SYafang Shao "attach_type:\t%d\n"
3118e859e429SYafang Shao "target_obj_id:\t%u\n"
3119e859e429SYafang Shao "target_btf_id:\t%u\n",
3120e859e429SYafang Shao tr_link->attach_type,
3121e859e429SYafang Shao target_obj_id,
3122e859e429SYafang Shao target_btf_id);
3123f2e10bffSAndrii Nakryiko }
3124f2e10bffSAndrii Nakryiko
bpf_tracing_link_fill_link_info(const struct bpf_link * link,struct bpf_link_info * info)3125f2e10bffSAndrii Nakryiko static int bpf_tracing_link_fill_link_info(const struct bpf_link *link,
3126f2e10bffSAndrii Nakryiko struct bpf_link_info *info)
3127f2e10bffSAndrii Nakryiko {
3128f2e10bffSAndrii Nakryiko struct bpf_tracing_link *tr_link =
3129f7e0beafSKui-Feng Lee container_of(link, struct bpf_tracing_link, link.link);
3130f2e10bffSAndrii Nakryiko
3131f2e10bffSAndrii Nakryiko info->tracing.attach_type = tr_link->attach_type;
3132441e8c66SToke Høiland-Jørgensen bpf_trampoline_unpack_key(tr_link->trampoline->key,
3133441e8c66SToke Høiland-Jørgensen &info->tracing.target_obj_id,
3134441e8c66SToke Høiland-Jørgensen &info->tracing.target_btf_id);
3135f2e10bffSAndrii Nakryiko
3136f2e10bffSAndrii Nakryiko return 0;
3137f2e10bffSAndrii Nakryiko }
3138f2e10bffSAndrii Nakryiko
313970ed506cSAndrii Nakryiko static const struct bpf_link_ops bpf_tracing_link_lops = {
314070ed506cSAndrii Nakryiko .release = bpf_tracing_link_release,
3141babf3164SAndrii Nakryiko .dealloc = bpf_tracing_link_dealloc,
3142f2e10bffSAndrii Nakryiko .show_fdinfo = bpf_tracing_link_show_fdinfo,
3143f2e10bffSAndrii Nakryiko .fill_link_info = bpf_tracing_link_fill_link_info,
314470ed506cSAndrii Nakryiko };
314570ed506cSAndrii Nakryiko
bpf_tracing_prog_attach(struct bpf_prog * prog,int tgt_prog_fd,u32 btf_id,u64 bpf_cookie)31464a1e7c0cSToke Høiland-Jørgensen static int bpf_tracing_prog_attach(struct bpf_prog *prog,
31474a1e7c0cSToke Høiland-Jørgensen int tgt_prog_fd,
31482fcc8241SKui-Feng Lee u32 btf_id,
31492fcc8241SKui-Feng Lee u64 bpf_cookie)
3150fec56f58SAlexei Starovoitov {
3151a3b80e10SAndrii Nakryiko struct bpf_link_primer link_primer;
31523aac1eadSToke Høiland-Jørgensen struct bpf_prog *tgt_prog = NULL;
31534a1e7c0cSToke Høiland-Jørgensen struct bpf_trampoline *tr = NULL;
315470ed506cSAndrii Nakryiko struct bpf_tracing_link *link;
31554a1e7c0cSToke Høiland-Jørgensen u64 key = 0;
3156a3b80e10SAndrii Nakryiko int err;
3157fec56f58SAlexei Starovoitov
31589e4e01dfSKP Singh switch (prog->type) {
31599e4e01dfSKP Singh case BPF_PROG_TYPE_TRACING:
3160fec56f58SAlexei Starovoitov if (prog->expected_attach_type != BPF_TRACE_FENTRY &&
3161be8704ffSAlexei Starovoitov prog->expected_attach_type != BPF_TRACE_FEXIT &&
31629e4e01dfSKP Singh prog->expected_attach_type != BPF_MODIFY_RETURN) {
31639e4e01dfSKP Singh err = -EINVAL;
31649e4e01dfSKP Singh goto out_put_prog;
31659e4e01dfSKP Singh }
31669e4e01dfSKP Singh break;
31679e4e01dfSKP Singh case BPF_PROG_TYPE_EXT:
31689e4e01dfSKP Singh if (prog->expected_attach_type != 0) {
31699e4e01dfSKP Singh err = -EINVAL;
31709e4e01dfSKP Singh goto out_put_prog;
31719e4e01dfSKP Singh }
31729e4e01dfSKP Singh break;
31739e4e01dfSKP Singh case BPF_PROG_TYPE_LSM:
31749e4e01dfSKP Singh if (prog->expected_attach_type != BPF_LSM_MAC) {
31759e4e01dfSKP Singh err = -EINVAL;
31769e4e01dfSKP Singh goto out_put_prog;
31779e4e01dfSKP Singh }
31789e4e01dfSKP Singh break;
31799e4e01dfSKP Singh default:
3180fec56f58SAlexei Starovoitov err = -EINVAL;
3181fec56f58SAlexei Starovoitov goto out_put_prog;
3182fec56f58SAlexei Starovoitov }
3183fec56f58SAlexei Starovoitov
31844a1e7c0cSToke Høiland-Jørgensen if (!!tgt_prog_fd != !!btf_id) {
31854a1e7c0cSToke Høiland-Jørgensen err = -EINVAL;
31864a1e7c0cSToke Høiland-Jørgensen goto out_put_prog;
31874a1e7c0cSToke Høiland-Jørgensen }
31884a1e7c0cSToke Høiland-Jørgensen
31894a1e7c0cSToke Høiland-Jørgensen if (tgt_prog_fd) {
31904a1e7c0cSToke Høiland-Jørgensen /* For now we only allow new targets for BPF_PROG_TYPE_EXT */
31914a1e7c0cSToke Høiland-Jørgensen if (prog->type != BPF_PROG_TYPE_EXT) {
31924a1e7c0cSToke Høiland-Jørgensen err = -EINVAL;
31934a1e7c0cSToke Høiland-Jørgensen goto out_put_prog;
31944a1e7c0cSToke Høiland-Jørgensen }
31954a1e7c0cSToke Høiland-Jørgensen
31964a1e7c0cSToke Høiland-Jørgensen tgt_prog = bpf_prog_get(tgt_prog_fd);
31974a1e7c0cSToke Høiland-Jørgensen if (IS_ERR(tgt_prog)) {
31984a1e7c0cSToke Høiland-Jørgensen err = PTR_ERR(tgt_prog);
31994a1e7c0cSToke Høiland-Jørgensen tgt_prog = NULL;
32004a1e7c0cSToke Høiland-Jørgensen goto out_put_prog;
32014a1e7c0cSToke Høiland-Jørgensen }
32024a1e7c0cSToke Høiland-Jørgensen
320322dc4a0fSAndrii Nakryiko key = bpf_trampoline_compute_key(tgt_prog, NULL, btf_id);
32044a1e7c0cSToke Høiland-Jørgensen }
32054a1e7c0cSToke Høiland-Jørgensen
320670ed506cSAndrii Nakryiko link = kzalloc(sizeof(*link), GFP_USER);
320770ed506cSAndrii Nakryiko if (!link) {
320870ed506cSAndrii Nakryiko err = -ENOMEM;
3209fec56f58SAlexei Starovoitov goto out_put_prog;
3210fec56f58SAlexei Starovoitov }
3211f7e0beafSKui-Feng Lee bpf_link_init(&link->link.link, BPF_LINK_TYPE_TRACING,
3212f2e10bffSAndrii Nakryiko &bpf_tracing_link_lops, prog);
3213f2e10bffSAndrii Nakryiko link->attach_type = prog->expected_attach_type;
32142fcc8241SKui-Feng Lee link->link.cookie = bpf_cookie;
3215fec56f58SAlexei Starovoitov
32163aac1eadSToke Høiland-Jørgensen mutex_lock(&prog->aux->dst_mutex);
3217babf3164SAndrii Nakryiko
32184a1e7c0cSToke Høiland-Jørgensen /* There are a few possible cases here:
32194a1e7c0cSToke Høiland-Jørgensen *
32204a1e7c0cSToke Høiland-Jørgensen * - if prog->aux->dst_trampoline is set, the program was just loaded
32214a1e7c0cSToke Høiland-Jørgensen * and not yet attached to anything, so we can use the values stored
32224a1e7c0cSToke Høiland-Jørgensen * in prog->aux
32234a1e7c0cSToke Høiland-Jørgensen *
32244a1e7c0cSToke Høiland-Jørgensen * - if prog->aux->dst_trampoline is NULL, the program has already been
32254a1e7c0cSToke Høiland-Jørgensen * attached to a target and its initial target was cleared (below)
32264a1e7c0cSToke Høiland-Jørgensen *
32274a1e7c0cSToke Høiland-Jørgensen * - if tgt_prog != NULL, the caller specified tgt_prog_fd +
32284a1e7c0cSToke Høiland-Jørgensen * target_btf_id using the link_create API.
32294a1e7c0cSToke Høiland-Jørgensen *
32304a1e7c0cSToke Høiland-Jørgensen * - if tgt_prog == NULL when this function was called using the old
32314a1e7c0cSToke Høiland-Jørgensen * raw_tracepoint_open API, and we need a target from prog->aux
32324a1e7c0cSToke Høiland-Jørgensen *
3233f3a95075SJiri Olsa * - if prog->aux->dst_trampoline and tgt_prog is NULL, the program
3234f3a95075SJiri Olsa * was detached and is going for re-attachment.
32358c8bcd45SJiri Olsa *
32368c8bcd45SJiri Olsa * - if prog->aux->dst_trampoline is NULL and tgt_prog and prog->aux->attach_btf
32378c8bcd45SJiri Olsa * are NULL, then program was already attached and user did not provide
32388c8bcd45SJiri Olsa * tgt_prog_fd so we have no way to find out or create trampoline
32394a1e7c0cSToke Høiland-Jørgensen */
32404a1e7c0cSToke Høiland-Jørgensen if (!prog->aux->dst_trampoline && !tgt_prog) {
3241f3a95075SJiri Olsa /*
3242f3a95075SJiri Olsa * Allow re-attach for TRACING and LSM programs. If it's
3243f3a95075SJiri Olsa * currently linked, bpf_trampoline_link_prog will fail.
3244f3a95075SJiri Olsa * EXT programs need to specify tgt_prog_fd, so they
3245f3a95075SJiri Olsa * re-attach in separate code path.
3246f3a95075SJiri Olsa */
3247f3a95075SJiri Olsa if (prog->type != BPF_PROG_TYPE_TRACING &&
3248f3a95075SJiri Olsa prog->type != BPF_PROG_TYPE_LSM) {
3249f3a95075SJiri Olsa err = -EINVAL;
32503aac1eadSToke Høiland-Jørgensen goto out_unlock;
32513aac1eadSToke Høiland-Jørgensen }
32528c8bcd45SJiri Olsa /* We can allow re-attach only if we have valid attach_btf. */
32538c8bcd45SJiri Olsa if (!prog->aux->attach_btf) {
32548c8bcd45SJiri Olsa err = -EINVAL;
32558c8bcd45SJiri Olsa goto out_unlock;
32568c8bcd45SJiri Olsa }
3257f3a95075SJiri Olsa btf_id = prog->aux->attach_btf_id;
3258f3a95075SJiri Olsa key = bpf_trampoline_compute_key(NULL, prog->aux->attach_btf, btf_id);
3259f3a95075SJiri Olsa }
32604a1e7c0cSToke Høiland-Jørgensen
32614a1e7c0cSToke Høiland-Jørgensen if (!prog->aux->dst_trampoline ||
32624a1e7c0cSToke Høiland-Jørgensen (key && key != prog->aux->dst_trampoline->key)) {
32634a1e7c0cSToke Høiland-Jørgensen /* If there is no saved target, or the specified target is
32644a1e7c0cSToke Høiland-Jørgensen * different from the destination specified at load time, we
32654a1e7c0cSToke Høiland-Jørgensen * need a new trampoline and a check for compatibility
32664a1e7c0cSToke Høiland-Jørgensen */
32674a1e7c0cSToke Høiland-Jørgensen struct bpf_attach_target_info tgt_info = {};
32684a1e7c0cSToke Høiland-Jørgensen
32694a1e7c0cSToke Høiland-Jørgensen err = bpf_check_attach_target(NULL, prog, tgt_prog, btf_id,
32704a1e7c0cSToke Høiland-Jørgensen &tgt_info);
32714a1e7c0cSToke Høiland-Jørgensen if (err)
32724a1e7c0cSToke Høiland-Jørgensen goto out_unlock;
32734a1e7c0cSToke Høiland-Jørgensen
327431bf1dbcSViktor Malik if (tgt_info.tgt_mod) {
327531bf1dbcSViktor Malik module_put(prog->aux->mod);
327631bf1dbcSViktor Malik prog->aux->mod = tgt_info.tgt_mod;
327731bf1dbcSViktor Malik }
327831bf1dbcSViktor Malik
32794a1e7c0cSToke Høiland-Jørgensen tr = bpf_trampoline_get(key, &tgt_info);
32804a1e7c0cSToke Høiland-Jørgensen if (!tr) {
32814a1e7c0cSToke Høiland-Jørgensen err = -ENOMEM;
32824a1e7c0cSToke Høiland-Jørgensen goto out_unlock;
32834a1e7c0cSToke Høiland-Jørgensen }
32844a1e7c0cSToke Høiland-Jørgensen } else {
32854a1e7c0cSToke Høiland-Jørgensen /* The caller didn't specify a target, or the target was the
32864a1e7c0cSToke Høiland-Jørgensen * same as the destination supplied during program load. This
32874a1e7c0cSToke Høiland-Jørgensen * means we can reuse the trampoline and reference from program
32884a1e7c0cSToke Høiland-Jørgensen * load time, and there is no need to allocate a new one. This
32894a1e7c0cSToke Høiland-Jørgensen * can only happen once for any program, as the saved values in
32904a1e7c0cSToke Høiland-Jørgensen * prog->aux are cleared below.
32914a1e7c0cSToke Høiland-Jørgensen */
32923aac1eadSToke Høiland-Jørgensen tr = prog->aux->dst_trampoline;
32933aac1eadSToke Høiland-Jørgensen tgt_prog = prog->aux->dst_prog;
32944a1e7c0cSToke Høiland-Jørgensen }
32953aac1eadSToke Høiland-Jørgensen
3296f7e0beafSKui-Feng Lee err = bpf_link_prime(&link->link.link, &link_primer);
32973aac1eadSToke Høiland-Jørgensen if (err)
32983aac1eadSToke Høiland-Jørgensen goto out_unlock;
32993aac1eadSToke Høiland-Jørgensen
3300f7e0beafSKui-Feng Lee err = bpf_trampoline_link_prog(&link->link, tr);
3301babf3164SAndrii Nakryiko if (err) {
3302a3b80e10SAndrii Nakryiko bpf_link_cleanup(&link_primer);
33033aac1eadSToke Høiland-Jørgensen link = NULL;
33043aac1eadSToke Høiland-Jørgensen goto out_unlock;
3305babf3164SAndrii Nakryiko }
3306babf3164SAndrii Nakryiko
33073aac1eadSToke Høiland-Jørgensen link->tgt_prog = tgt_prog;
33083aac1eadSToke Høiland-Jørgensen link->trampoline = tr;
33093aac1eadSToke Høiland-Jørgensen
33104a1e7c0cSToke Høiland-Jørgensen /* Always clear the trampoline and target prog from prog->aux to make
33114a1e7c0cSToke Høiland-Jørgensen * sure the original attach destination is not kept alive after a
33124a1e7c0cSToke Høiland-Jørgensen * program is (re-)attached to another target.
33134a1e7c0cSToke Høiland-Jørgensen */
33144a1e7c0cSToke Høiland-Jørgensen if (prog->aux->dst_prog &&
33154a1e7c0cSToke Høiland-Jørgensen (tgt_prog_fd || tr != prog->aux->dst_trampoline))
33164a1e7c0cSToke Høiland-Jørgensen /* got extra prog ref from syscall, or attaching to different prog */
33174a1e7c0cSToke Høiland-Jørgensen bpf_prog_put(prog->aux->dst_prog);
33184a1e7c0cSToke Høiland-Jørgensen if (prog->aux->dst_trampoline && tr != prog->aux->dst_trampoline)
33194a1e7c0cSToke Høiland-Jørgensen /* we allocated a new trampoline, so free the old one */
33204a1e7c0cSToke Høiland-Jørgensen bpf_trampoline_put(prog->aux->dst_trampoline);
33214a1e7c0cSToke Høiland-Jørgensen
33223aac1eadSToke Høiland-Jørgensen prog->aux->dst_prog = NULL;
33233aac1eadSToke Høiland-Jørgensen prog->aux->dst_trampoline = NULL;
33243aac1eadSToke Høiland-Jørgensen mutex_unlock(&prog->aux->dst_mutex);
33253aac1eadSToke Høiland-Jørgensen
3326a3b80e10SAndrii Nakryiko return bpf_link_settle(&link_primer);
33273aac1eadSToke Høiland-Jørgensen out_unlock:
33284a1e7c0cSToke Høiland-Jørgensen if (tr && tr != prog->aux->dst_trampoline)
33294a1e7c0cSToke Høiland-Jørgensen bpf_trampoline_put(tr);
33303aac1eadSToke Høiland-Jørgensen mutex_unlock(&prog->aux->dst_mutex);
33313aac1eadSToke Høiland-Jørgensen kfree(link);
3332fec56f58SAlexei Starovoitov out_put_prog:
33334a1e7c0cSToke Høiland-Jørgensen if (tgt_prog_fd && tgt_prog)
33344a1e7c0cSToke Høiland-Jørgensen bpf_prog_put(tgt_prog);
3335fec56f58SAlexei Starovoitov return err;
3336fec56f58SAlexei Starovoitov }
3337fec56f58SAlexei Starovoitov
333870ed506cSAndrii Nakryiko struct bpf_raw_tp_link {
333970ed506cSAndrii Nakryiko struct bpf_link link;
3340c4f6699dSAlexei Starovoitov struct bpf_raw_event_map *btp;
3341c4f6699dSAlexei Starovoitov };
3342c4f6699dSAlexei Starovoitov
bpf_raw_tp_link_release(struct bpf_link * link)334370ed506cSAndrii Nakryiko static void bpf_raw_tp_link_release(struct bpf_link *link)
3344c4f6699dSAlexei Starovoitov {
334570ed506cSAndrii Nakryiko struct bpf_raw_tp_link *raw_tp =
334670ed506cSAndrii Nakryiko container_of(link, struct bpf_raw_tp_link, link);
3347c4f6699dSAlexei Starovoitov
334870ed506cSAndrii Nakryiko bpf_probe_unregister(raw_tp->btp, raw_tp->link.prog);
3349a38d1107SMatt Mullins bpf_put_raw_tracepoint(raw_tp->btp);
3350babf3164SAndrii Nakryiko }
3351babf3164SAndrii Nakryiko
bpf_raw_tp_link_dealloc(struct bpf_link * link)3352babf3164SAndrii Nakryiko static void bpf_raw_tp_link_dealloc(struct bpf_link *link)
3353babf3164SAndrii Nakryiko {
3354babf3164SAndrii Nakryiko struct bpf_raw_tp_link *raw_tp =
3355babf3164SAndrii Nakryiko container_of(link, struct bpf_raw_tp_link, link);
3356babf3164SAndrii Nakryiko
3357c4f6699dSAlexei Starovoitov kfree(raw_tp);
3358c4f6699dSAlexei Starovoitov }
3359c4f6699dSAlexei Starovoitov
bpf_raw_tp_link_show_fdinfo(const struct bpf_link * link,struct seq_file * seq)3360f2e10bffSAndrii Nakryiko static void bpf_raw_tp_link_show_fdinfo(const struct bpf_link *link,
3361f2e10bffSAndrii Nakryiko struct seq_file *seq)
3362f2e10bffSAndrii Nakryiko {
3363f2e10bffSAndrii Nakryiko struct bpf_raw_tp_link *raw_tp_link =
3364f2e10bffSAndrii Nakryiko container_of(link, struct bpf_raw_tp_link, link);
3365f2e10bffSAndrii Nakryiko
3366f2e10bffSAndrii Nakryiko seq_printf(seq,
3367f2e10bffSAndrii Nakryiko "tp_name:\t%s\n",
3368f2e10bffSAndrii Nakryiko raw_tp_link->btp->tp->name);
3369f2e10bffSAndrii Nakryiko }
3370f2e10bffSAndrii Nakryiko
bpf_copy_to_user(char __user * ubuf,const char * buf,u32 ulen,u32 len)337157d48537SYafang Shao static int bpf_copy_to_user(char __user *ubuf, const char *buf, u32 ulen,
337257d48537SYafang Shao u32 len)
337357d48537SYafang Shao {
337457d48537SYafang Shao if (ulen >= len + 1) {
337557d48537SYafang Shao if (copy_to_user(ubuf, buf, len + 1))
337657d48537SYafang Shao return -EFAULT;
337757d48537SYafang Shao } else {
337857d48537SYafang Shao char zero = '\0';
337957d48537SYafang Shao
338057d48537SYafang Shao if (copy_to_user(ubuf, buf, ulen - 1))
338157d48537SYafang Shao return -EFAULT;
338257d48537SYafang Shao if (put_user(zero, ubuf + ulen - 1))
338357d48537SYafang Shao return -EFAULT;
338457d48537SYafang Shao return -ENOSPC;
338557d48537SYafang Shao }
338657d48537SYafang Shao
338757d48537SYafang Shao return 0;
338857d48537SYafang Shao }
338957d48537SYafang Shao
bpf_raw_tp_link_fill_link_info(const struct bpf_link * link,struct bpf_link_info * info)3390f2e10bffSAndrii Nakryiko static int bpf_raw_tp_link_fill_link_info(const struct bpf_link *link,
3391f2e10bffSAndrii Nakryiko struct bpf_link_info *info)
3392f2e10bffSAndrii Nakryiko {
3393f2e10bffSAndrii Nakryiko struct bpf_raw_tp_link *raw_tp_link =
3394f2e10bffSAndrii Nakryiko container_of(link, struct bpf_raw_tp_link, link);
3395f2e10bffSAndrii Nakryiko char __user *ubuf = u64_to_user_ptr(info->raw_tracepoint.tp_name);
3396f2e10bffSAndrii Nakryiko const char *tp_name = raw_tp_link->btp->tp->name;
3397f2e10bffSAndrii Nakryiko u32 ulen = info->raw_tracepoint.tp_name_len;
3398f2e10bffSAndrii Nakryiko size_t tp_len = strlen(tp_name);
3399f2e10bffSAndrii Nakryiko
3400b474959dSYonghong Song if (!ulen ^ !ubuf)
3401f2e10bffSAndrii Nakryiko return -EINVAL;
3402f2e10bffSAndrii Nakryiko
3403f2e10bffSAndrii Nakryiko info->raw_tracepoint.tp_name_len = tp_len + 1;
3404f2e10bffSAndrii Nakryiko
3405f2e10bffSAndrii Nakryiko if (!ubuf)
3406f2e10bffSAndrii Nakryiko return 0;
3407f2e10bffSAndrii Nakryiko
340857d48537SYafang Shao return bpf_copy_to_user(ubuf, tp_name, ulen, tp_len);
3409f2e10bffSAndrii Nakryiko }
3410f2e10bffSAndrii Nakryiko
3411a3b80e10SAndrii Nakryiko static const struct bpf_link_ops bpf_raw_tp_link_lops = {
341270ed506cSAndrii Nakryiko .release = bpf_raw_tp_link_release,
3413*876941f5SAndrii Nakryiko .dealloc_deferred = bpf_raw_tp_link_dealloc,
3414f2e10bffSAndrii Nakryiko .show_fdinfo = bpf_raw_tp_link_show_fdinfo,
3415f2e10bffSAndrii Nakryiko .fill_link_info = bpf_raw_tp_link_fill_link_info,
3416c4f6699dSAlexei Starovoitov };
3417c4f6699dSAlexei Starovoitov
3418b89fbfbbSAndrii Nakryiko #ifdef CONFIG_PERF_EVENTS
3419b89fbfbbSAndrii Nakryiko struct bpf_perf_link {
3420b89fbfbbSAndrii Nakryiko struct bpf_link link;
3421b89fbfbbSAndrii Nakryiko struct file *perf_file;
3422b89fbfbbSAndrii Nakryiko };
3423b89fbfbbSAndrii Nakryiko
bpf_perf_link_release(struct bpf_link * link)3424b89fbfbbSAndrii Nakryiko static void bpf_perf_link_release(struct bpf_link *link)
3425b89fbfbbSAndrii Nakryiko {
3426b89fbfbbSAndrii Nakryiko struct bpf_perf_link *perf_link = container_of(link, struct bpf_perf_link, link);
3427b89fbfbbSAndrii Nakryiko struct perf_event *event = perf_link->perf_file->private_data;
3428b89fbfbbSAndrii Nakryiko
3429b89fbfbbSAndrii Nakryiko perf_event_free_bpf_prog(event);
3430b89fbfbbSAndrii Nakryiko fput(perf_link->perf_file);
3431b89fbfbbSAndrii Nakryiko }
3432b89fbfbbSAndrii Nakryiko
bpf_perf_link_dealloc(struct bpf_link * link)3433b89fbfbbSAndrii Nakryiko static void bpf_perf_link_dealloc(struct bpf_link *link)
3434b89fbfbbSAndrii Nakryiko {
3435b89fbfbbSAndrii Nakryiko struct bpf_perf_link *perf_link = container_of(link, struct bpf_perf_link, link);
3436b89fbfbbSAndrii Nakryiko
3437b89fbfbbSAndrii Nakryiko kfree(perf_link);
3438b89fbfbbSAndrii Nakryiko }
3439b89fbfbbSAndrii Nakryiko
bpf_perf_link_fill_common(const struct perf_event * event,char __user * uname,u32 ulen,u64 * probe_offset,u64 * probe_addr,u32 * fd_type)34401b715e1bSYafang Shao static int bpf_perf_link_fill_common(const struct perf_event *event,
34411b715e1bSYafang Shao char __user *uname, u32 ulen,
34421b715e1bSYafang Shao u64 *probe_offset, u64 *probe_addr,
34431b715e1bSYafang Shao u32 *fd_type)
34441b715e1bSYafang Shao {
34451b715e1bSYafang Shao const char *buf;
34461b715e1bSYafang Shao u32 prog_id;
34471b715e1bSYafang Shao size_t len;
34481b715e1bSYafang Shao int err;
34491b715e1bSYafang Shao
34501b715e1bSYafang Shao if (!ulen ^ !uname)
34511b715e1bSYafang Shao return -EINVAL;
34521b715e1bSYafang Shao
34531b715e1bSYafang Shao err = bpf_get_perf_event_info(event, &prog_id, fd_type, &buf,
34541b715e1bSYafang Shao probe_offset, probe_addr);
34551b715e1bSYafang Shao if (err)
34561b715e1bSYafang Shao return err;
34570aa35162SYafang Shao if (!uname)
34580aa35162SYafang Shao return 0;
34591b715e1bSYafang Shao if (buf) {
34601b715e1bSYafang Shao len = strlen(buf);
34611b715e1bSYafang Shao err = bpf_copy_to_user(uname, buf, ulen, len);
34621b715e1bSYafang Shao if (err)
34631b715e1bSYafang Shao return err;
34641b715e1bSYafang Shao } else {
34651b715e1bSYafang Shao char zero = '\0';
34661b715e1bSYafang Shao
34671b715e1bSYafang Shao if (put_user(zero, uname))
34681b715e1bSYafang Shao return -EFAULT;
34691b715e1bSYafang Shao }
34701b715e1bSYafang Shao return 0;
34711b715e1bSYafang Shao }
34721b715e1bSYafang Shao
34731b715e1bSYafang Shao #ifdef CONFIG_KPROBE_EVENTS
bpf_perf_link_fill_kprobe(const struct perf_event * event,struct bpf_link_info * info)34741b715e1bSYafang Shao static int bpf_perf_link_fill_kprobe(const struct perf_event *event,
34751b715e1bSYafang Shao struct bpf_link_info *info)
34761b715e1bSYafang Shao {
34771b715e1bSYafang Shao char __user *uname;
34781b715e1bSYafang Shao u64 addr, offset;
34791b715e1bSYafang Shao u32 ulen, type;
34801b715e1bSYafang Shao int err;
34811b715e1bSYafang Shao
34821b715e1bSYafang Shao uname = u64_to_user_ptr(info->perf_event.kprobe.func_name);
34831b715e1bSYafang Shao ulen = info->perf_event.kprobe.name_len;
34841b715e1bSYafang Shao err = bpf_perf_link_fill_common(event, uname, ulen, &offset, &addr,
34851b715e1bSYafang Shao &type);
34861b715e1bSYafang Shao if (err)
34871b715e1bSYafang Shao return err;
34881b715e1bSYafang Shao if (type == BPF_FD_TYPE_KRETPROBE)
34891b715e1bSYafang Shao info->perf_event.type = BPF_PERF_EVENT_KRETPROBE;
34901b715e1bSYafang Shao else
34911b715e1bSYafang Shao info->perf_event.type = BPF_PERF_EVENT_KPROBE;
34921b715e1bSYafang Shao
34931b715e1bSYafang Shao info->perf_event.kprobe.offset = offset;
34941b715e1bSYafang Shao if (!kallsyms_show_value(current_cred()))
34951b715e1bSYafang Shao addr = 0;
34961b715e1bSYafang Shao info->perf_event.kprobe.addr = addr;
34971b715e1bSYafang Shao return 0;
34981b715e1bSYafang Shao }
34991b715e1bSYafang Shao #endif
35001b715e1bSYafang Shao
35011b715e1bSYafang Shao #ifdef CONFIG_UPROBE_EVENTS
bpf_perf_link_fill_uprobe(const struct perf_event * event,struct bpf_link_info * info)35021b715e1bSYafang Shao static int bpf_perf_link_fill_uprobe(const struct perf_event *event,
35031b715e1bSYafang Shao struct bpf_link_info *info)
35041b715e1bSYafang Shao {
35051b715e1bSYafang Shao char __user *uname;
35061b715e1bSYafang Shao u64 addr, offset;
35071b715e1bSYafang Shao u32 ulen, type;
35081b715e1bSYafang Shao int err;
35091b715e1bSYafang Shao
35101b715e1bSYafang Shao uname = u64_to_user_ptr(info->perf_event.uprobe.file_name);
35111b715e1bSYafang Shao ulen = info->perf_event.uprobe.name_len;
35121b715e1bSYafang Shao err = bpf_perf_link_fill_common(event, uname, ulen, &offset, &addr,
35131b715e1bSYafang Shao &type);
35141b715e1bSYafang Shao if (err)
35151b715e1bSYafang Shao return err;
35161b715e1bSYafang Shao
35171b715e1bSYafang Shao if (type == BPF_FD_TYPE_URETPROBE)
35181b715e1bSYafang Shao info->perf_event.type = BPF_PERF_EVENT_URETPROBE;
35191b715e1bSYafang Shao else
35201b715e1bSYafang Shao info->perf_event.type = BPF_PERF_EVENT_UPROBE;
35211b715e1bSYafang Shao info->perf_event.uprobe.offset = offset;
35221b715e1bSYafang Shao return 0;
35231b715e1bSYafang Shao }
35241b715e1bSYafang Shao #endif
35251b715e1bSYafang Shao
bpf_perf_link_fill_probe(const struct perf_event * event,struct bpf_link_info * info)35261b715e1bSYafang Shao static int bpf_perf_link_fill_probe(const struct perf_event *event,
35271b715e1bSYafang Shao struct bpf_link_info *info)
35281b715e1bSYafang Shao {
35291b715e1bSYafang Shao #ifdef CONFIG_KPROBE_EVENTS
35301b715e1bSYafang Shao if (event->tp_event->flags & TRACE_EVENT_FL_KPROBE)
35311b715e1bSYafang Shao return bpf_perf_link_fill_kprobe(event, info);
35321b715e1bSYafang Shao #endif
35331b715e1bSYafang Shao #ifdef CONFIG_UPROBE_EVENTS
35341b715e1bSYafang Shao if (event->tp_event->flags & TRACE_EVENT_FL_UPROBE)
35351b715e1bSYafang Shao return bpf_perf_link_fill_uprobe(event, info);
35361b715e1bSYafang Shao #endif
35371b715e1bSYafang Shao return -EOPNOTSUPP;
35381b715e1bSYafang Shao }
35391b715e1bSYafang Shao
bpf_perf_link_fill_tracepoint(const struct perf_event * event,struct bpf_link_info * info)35401b715e1bSYafang Shao static int bpf_perf_link_fill_tracepoint(const struct perf_event *event,
35411b715e1bSYafang Shao struct bpf_link_info *info)
35421b715e1bSYafang Shao {
35431b715e1bSYafang Shao char __user *uname;
35441b715e1bSYafang Shao u32 ulen;
35451b715e1bSYafang Shao
35461b715e1bSYafang Shao uname = u64_to_user_ptr(info->perf_event.tracepoint.tp_name);
35471b715e1bSYafang Shao ulen = info->perf_event.tracepoint.name_len;
35481b715e1bSYafang Shao info->perf_event.type = BPF_PERF_EVENT_TRACEPOINT;
35491b715e1bSYafang Shao return bpf_perf_link_fill_common(event, uname, ulen, NULL, NULL, NULL);
35501b715e1bSYafang Shao }
35511b715e1bSYafang Shao
bpf_perf_link_fill_perf_event(const struct perf_event * event,struct bpf_link_info * info)35521b715e1bSYafang Shao static int bpf_perf_link_fill_perf_event(const struct perf_event *event,
35531b715e1bSYafang Shao struct bpf_link_info *info)
35541b715e1bSYafang Shao {
35551b715e1bSYafang Shao info->perf_event.event.type = event->attr.type;
35561b715e1bSYafang Shao info->perf_event.event.config = event->attr.config;
35571b715e1bSYafang Shao info->perf_event.type = BPF_PERF_EVENT_EVENT;
35581b715e1bSYafang Shao return 0;
35591b715e1bSYafang Shao }
35601b715e1bSYafang Shao
bpf_perf_link_fill_link_info(const struct bpf_link * link,struct bpf_link_info * info)35611b715e1bSYafang Shao static int bpf_perf_link_fill_link_info(const struct bpf_link *link,
35621b715e1bSYafang Shao struct bpf_link_info *info)
35631b715e1bSYafang Shao {
35641b715e1bSYafang Shao struct bpf_perf_link *perf_link;
35651b715e1bSYafang Shao const struct perf_event *event;
35661b715e1bSYafang Shao
35671b715e1bSYafang Shao perf_link = container_of(link, struct bpf_perf_link, link);
35681b715e1bSYafang Shao event = perf_get_event(perf_link->perf_file);
35691b715e1bSYafang Shao if (IS_ERR(event))
35701b715e1bSYafang Shao return PTR_ERR(event);
35711b715e1bSYafang Shao
35721b715e1bSYafang Shao switch (event->prog->type) {
35731b715e1bSYafang Shao case BPF_PROG_TYPE_PERF_EVENT:
35741b715e1bSYafang Shao return bpf_perf_link_fill_perf_event(event, info);
35751b715e1bSYafang Shao case BPF_PROG_TYPE_TRACEPOINT:
35761b715e1bSYafang Shao return bpf_perf_link_fill_tracepoint(event, info);
35771b715e1bSYafang Shao case BPF_PROG_TYPE_KPROBE:
35781b715e1bSYafang Shao return bpf_perf_link_fill_probe(event, info);
35791b715e1bSYafang Shao default:
35801b715e1bSYafang Shao return -EOPNOTSUPP;
35811b715e1bSYafang Shao }
35821b715e1bSYafang Shao }
35831b715e1bSYafang Shao
3584b89fbfbbSAndrii Nakryiko static const struct bpf_link_ops bpf_perf_link_lops = {
3585b89fbfbbSAndrii Nakryiko .release = bpf_perf_link_release,
3586b89fbfbbSAndrii Nakryiko .dealloc = bpf_perf_link_dealloc,
35871b715e1bSYafang Shao .fill_link_info = bpf_perf_link_fill_link_info,
3588b89fbfbbSAndrii Nakryiko };
3589b89fbfbbSAndrii Nakryiko
bpf_perf_link_attach(const union bpf_attr * attr,struct bpf_prog * prog)3590b89fbfbbSAndrii Nakryiko static int bpf_perf_link_attach(const union bpf_attr *attr, struct bpf_prog *prog)
3591b89fbfbbSAndrii Nakryiko {
3592b89fbfbbSAndrii Nakryiko struct bpf_link_primer link_primer;
3593b89fbfbbSAndrii Nakryiko struct bpf_perf_link *link;
3594b89fbfbbSAndrii Nakryiko struct perf_event *event;
3595b89fbfbbSAndrii Nakryiko struct file *perf_file;
3596b89fbfbbSAndrii Nakryiko int err;
3597b89fbfbbSAndrii Nakryiko
3598b89fbfbbSAndrii Nakryiko if (attr->link_create.flags)
3599b89fbfbbSAndrii Nakryiko return -EINVAL;
3600b89fbfbbSAndrii Nakryiko
3601b89fbfbbSAndrii Nakryiko perf_file = perf_event_get(attr->link_create.target_fd);
3602b89fbfbbSAndrii Nakryiko if (IS_ERR(perf_file))
3603b89fbfbbSAndrii Nakryiko return PTR_ERR(perf_file);
3604b89fbfbbSAndrii Nakryiko
3605b89fbfbbSAndrii Nakryiko link = kzalloc(sizeof(*link), GFP_USER);
3606b89fbfbbSAndrii Nakryiko if (!link) {
3607b89fbfbbSAndrii Nakryiko err = -ENOMEM;
3608b89fbfbbSAndrii Nakryiko goto out_put_file;
3609b89fbfbbSAndrii Nakryiko }
3610b89fbfbbSAndrii Nakryiko bpf_link_init(&link->link, BPF_LINK_TYPE_PERF_EVENT, &bpf_perf_link_lops, prog);
3611b89fbfbbSAndrii Nakryiko link->perf_file = perf_file;
3612b89fbfbbSAndrii Nakryiko
3613b89fbfbbSAndrii Nakryiko err = bpf_link_prime(&link->link, &link_primer);
3614b89fbfbbSAndrii Nakryiko if (err) {
3615b89fbfbbSAndrii Nakryiko kfree(link);
3616b89fbfbbSAndrii Nakryiko goto out_put_file;
3617b89fbfbbSAndrii Nakryiko }
3618b89fbfbbSAndrii Nakryiko
3619b89fbfbbSAndrii Nakryiko event = perf_file->private_data;
362082e6b1eeSAndrii Nakryiko err = perf_event_set_bpf_prog(event, prog, attr->link_create.perf_event.bpf_cookie);
3621b89fbfbbSAndrii Nakryiko if (err) {
3622b89fbfbbSAndrii Nakryiko bpf_link_cleanup(&link_primer);
3623b89fbfbbSAndrii Nakryiko goto out_put_file;
3624b89fbfbbSAndrii Nakryiko }
3625b89fbfbbSAndrii Nakryiko /* perf_event_set_bpf_prog() doesn't take its own refcnt on prog */
3626b89fbfbbSAndrii Nakryiko bpf_prog_inc(prog);
3627b89fbfbbSAndrii Nakryiko
3628b89fbfbbSAndrii Nakryiko return bpf_link_settle(&link_primer);
3629b89fbfbbSAndrii Nakryiko
3630b89fbfbbSAndrii Nakryiko out_put_file:
3631b89fbfbbSAndrii Nakryiko fput(perf_file);
3632b89fbfbbSAndrii Nakryiko return err;
3633b89fbfbbSAndrii Nakryiko }
36340dcac272SJiri Olsa #else
bpf_perf_link_attach(const union bpf_attr * attr,struct bpf_prog * prog)36350dcac272SJiri Olsa static int bpf_perf_link_attach(const union bpf_attr *attr, struct bpf_prog *prog)
36360dcac272SJiri Olsa {
36370dcac272SJiri Olsa return -EOPNOTSUPP;
36380dcac272SJiri Olsa }
3639b89fbfbbSAndrii Nakryiko #endif /* CONFIG_PERF_EVENTS */
3640b89fbfbbSAndrii Nakryiko
bpf_raw_tp_link_attach(struct bpf_prog * prog,const char __user * user_tp_name)3641df86ca0dSAndrii Nakryiko static int bpf_raw_tp_link_attach(struct bpf_prog *prog,
3642df86ca0dSAndrii Nakryiko const char __user *user_tp_name)
3643c4f6699dSAlexei Starovoitov {
3644a3b80e10SAndrii Nakryiko struct bpf_link_primer link_primer;
3645babf3164SAndrii Nakryiko struct bpf_raw_tp_link *link;
3646c4f6699dSAlexei Starovoitov struct bpf_raw_event_map *btp;
3647ac4414b5SAlexei Starovoitov const char *tp_name;
3648ac4414b5SAlexei Starovoitov char buf[128];
3649a3b80e10SAndrii Nakryiko int err;
3650c4f6699dSAlexei Starovoitov
36519e4e01dfSKP Singh switch (prog->type) {
36529e4e01dfSKP Singh case BPF_PROG_TYPE_TRACING:
36539e4e01dfSKP Singh case BPF_PROG_TYPE_EXT:
36549e4e01dfSKP Singh case BPF_PROG_TYPE_LSM:
3655df86ca0dSAndrii Nakryiko if (user_tp_name)
3656fec56f58SAlexei Starovoitov /* The attach point for this category of programs
3657fec56f58SAlexei Starovoitov * should be specified via btf_id during program load.
3658ac4414b5SAlexei Starovoitov */
3659df86ca0dSAndrii Nakryiko return -EINVAL;
36609e4e01dfSKP Singh if (prog->type == BPF_PROG_TYPE_TRACING &&
36619e4e01dfSKP Singh prog->expected_attach_type == BPF_TRACE_RAW_TP) {
366238207291SMartin KaFai Lau tp_name = prog->aux->attach_func_name;
36639e4e01dfSKP Singh break;
36649e4e01dfSKP Singh }
36652fcc8241SKui-Feng Lee return bpf_tracing_prog_attach(prog, 0, 0, 0);
36669e4e01dfSKP Singh case BPF_PROG_TYPE_RAW_TRACEPOINT:
36679e4e01dfSKP Singh case BPF_PROG_TYPE_RAW_TRACEPOINT_WRITABLE:
3668df86ca0dSAndrii Nakryiko if (strncpy_from_user(buf, user_tp_name, sizeof(buf) - 1) < 0)
3669df86ca0dSAndrii Nakryiko return -EFAULT;
3670ac4414b5SAlexei Starovoitov buf[sizeof(buf) - 1] = 0;
3671ac4414b5SAlexei Starovoitov tp_name = buf;
36729e4e01dfSKP Singh break;
36739e4e01dfSKP Singh default:
3674df86ca0dSAndrii Nakryiko return -EINVAL;
3675ac4414b5SAlexei Starovoitov }
3676c4f6699dSAlexei Starovoitov
3677a38d1107SMatt Mullins btp = bpf_get_raw_tracepoint(tp_name);
3678df86ca0dSAndrii Nakryiko if (!btp)
3679df86ca0dSAndrii Nakryiko return -ENOENT;
3680c4f6699dSAlexei Starovoitov
3681babf3164SAndrii Nakryiko link = kzalloc(sizeof(*link), GFP_USER);
3682babf3164SAndrii Nakryiko if (!link) {
3683a38d1107SMatt Mullins err = -ENOMEM;
3684a38d1107SMatt Mullins goto out_put_btp;
3685a38d1107SMatt Mullins }
3686f2e10bffSAndrii Nakryiko bpf_link_init(&link->link, BPF_LINK_TYPE_RAW_TRACEPOINT,
3687f2e10bffSAndrii Nakryiko &bpf_raw_tp_link_lops, prog);
3688babf3164SAndrii Nakryiko link->btp = btp;
3689c4f6699dSAlexei Starovoitov
3690a3b80e10SAndrii Nakryiko err = bpf_link_prime(&link->link, &link_primer);
3691a3b80e10SAndrii Nakryiko if (err) {
3692babf3164SAndrii Nakryiko kfree(link);
3693babf3164SAndrii Nakryiko goto out_put_btp;
3694c4f6699dSAlexei Starovoitov }
3695babf3164SAndrii Nakryiko
3696babf3164SAndrii Nakryiko err = bpf_probe_register(link->btp, prog);
3697babf3164SAndrii Nakryiko if (err) {
3698a3b80e10SAndrii Nakryiko bpf_link_cleanup(&link_primer);
3699babf3164SAndrii Nakryiko goto out_put_btp;
3700babf3164SAndrii Nakryiko }
3701babf3164SAndrii Nakryiko
3702a3b80e10SAndrii Nakryiko return bpf_link_settle(&link_primer);
3703c4f6699dSAlexei Starovoitov
3704a38d1107SMatt Mullins out_put_btp:
3705a38d1107SMatt Mullins bpf_put_raw_tracepoint(btp);
3706c4f6699dSAlexei Starovoitov return err;
3707c4f6699dSAlexei Starovoitov }
3708c4f6699dSAlexei Starovoitov
3709df86ca0dSAndrii Nakryiko #define BPF_RAW_TRACEPOINT_OPEN_LAST_FIELD raw_tracepoint.prog_fd
3710df86ca0dSAndrii Nakryiko
bpf_raw_tracepoint_open(const union bpf_attr * attr)3711df86ca0dSAndrii Nakryiko static int bpf_raw_tracepoint_open(const union bpf_attr *attr)
3712df86ca0dSAndrii Nakryiko {
3713df86ca0dSAndrii Nakryiko struct bpf_prog *prog;
3714df86ca0dSAndrii Nakryiko int fd;
3715df86ca0dSAndrii Nakryiko
3716df86ca0dSAndrii Nakryiko if (CHECK_ATTR(BPF_RAW_TRACEPOINT_OPEN))
3717df86ca0dSAndrii Nakryiko return -EINVAL;
3718df86ca0dSAndrii Nakryiko
3719df86ca0dSAndrii Nakryiko prog = bpf_prog_get(attr->raw_tracepoint.prog_fd);
3720df86ca0dSAndrii Nakryiko if (IS_ERR(prog))
3721df86ca0dSAndrii Nakryiko return PTR_ERR(prog);
3722df86ca0dSAndrii Nakryiko
3723df86ca0dSAndrii Nakryiko fd = bpf_raw_tp_link_attach(prog, u64_to_user_ptr(attr->raw_tracepoint.name));
3724df86ca0dSAndrii Nakryiko if (fd < 0)
3725df86ca0dSAndrii Nakryiko bpf_prog_put(prog);
3726df86ca0dSAndrii Nakryiko return fd;
3727df86ca0dSAndrii Nakryiko }
3728df86ca0dSAndrii Nakryiko
3729e28784e3SAndrii Nakryiko static enum bpf_prog_type
attach_type_to_prog_type(enum bpf_attach_type attach_type)3730e28784e3SAndrii Nakryiko attach_type_to_prog_type(enum bpf_attach_type attach_type)
3731e28784e3SAndrii Nakryiko {
3732e28784e3SAndrii Nakryiko switch (attach_type) {
3733e28784e3SAndrii Nakryiko case BPF_CGROUP_INET_INGRESS:
3734e28784e3SAndrii Nakryiko case BPF_CGROUP_INET_EGRESS:
3735e28784e3SAndrii Nakryiko return BPF_PROG_TYPE_CGROUP_SKB;
3736e28784e3SAndrii Nakryiko case BPF_CGROUP_INET_SOCK_CREATE:
3737f5836749SStanislav Fomichev case BPF_CGROUP_INET_SOCK_RELEASE:
3738e28784e3SAndrii Nakryiko case BPF_CGROUP_INET4_POST_BIND:
3739e28784e3SAndrii Nakryiko case BPF_CGROUP_INET6_POST_BIND:
3740e28784e3SAndrii Nakryiko return BPF_PROG_TYPE_CGROUP_SOCK;
3741e28784e3SAndrii Nakryiko case BPF_CGROUP_INET4_BIND:
3742e28784e3SAndrii Nakryiko case BPF_CGROUP_INET6_BIND:
3743e28784e3SAndrii Nakryiko case BPF_CGROUP_INET4_CONNECT:
3744e28784e3SAndrii Nakryiko case BPF_CGROUP_INET6_CONNECT:
37451b66d253SDaniel Borkmann case BPF_CGROUP_INET4_GETPEERNAME:
37461b66d253SDaniel Borkmann case BPF_CGROUP_INET6_GETPEERNAME:
37471b66d253SDaniel Borkmann case BPF_CGROUP_INET4_GETSOCKNAME:
37481b66d253SDaniel Borkmann case BPF_CGROUP_INET6_GETSOCKNAME:
3749e28784e3SAndrii Nakryiko case BPF_CGROUP_UDP4_SENDMSG:
3750e28784e3SAndrii Nakryiko case BPF_CGROUP_UDP6_SENDMSG:
3751e28784e3SAndrii Nakryiko case BPF_CGROUP_UDP4_RECVMSG:
3752e28784e3SAndrii Nakryiko case BPF_CGROUP_UDP6_RECVMSG:
3753e28784e3SAndrii Nakryiko return BPF_PROG_TYPE_CGROUP_SOCK_ADDR;
3754e28784e3SAndrii Nakryiko case BPF_CGROUP_SOCK_OPS:
3755e28784e3SAndrii Nakryiko return BPF_PROG_TYPE_SOCK_OPS;
3756e28784e3SAndrii Nakryiko case BPF_CGROUP_DEVICE:
3757e28784e3SAndrii Nakryiko return BPF_PROG_TYPE_CGROUP_DEVICE;
3758e28784e3SAndrii Nakryiko case BPF_SK_MSG_VERDICT:
3759e28784e3SAndrii Nakryiko return BPF_PROG_TYPE_SK_MSG;
3760e28784e3SAndrii Nakryiko case BPF_SK_SKB_STREAM_PARSER:
3761e28784e3SAndrii Nakryiko case BPF_SK_SKB_STREAM_VERDICT:
3762a7ba4558SCong Wang case BPF_SK_SKB_VERDICT:
3763e28784e3SAndrii Nakryiko return BPF_PROG_TYPE_SK_SKB;
3764e28784e3SAndrii Nakryiko case BPF_LIRC_MODE2:
3765e28784e3SAndrii Nakryiko return BPF_PROG_TYPE_LIRC_MODE2;
3766e28784e3SAndrii Nakryiko case BPF_FLOW_DISSECTOR:
3767e28784e3SAndrii Nakryiko return BPF_PROG_TYPE_FLOW_DISSECTOR;
3768e28784e3SAndrii Nakryiko case BPF_CGROUP_SYSCTL:
3769e28784e3SAndrii Nakryiko return BPF_PROG_TYPE_CGROUP_SYSCTL;
3770e28784e3SAndrii Nakryiko case BPF_CGROUP_GETSOCKOPT:
3771e28784e3SAndrii Nakryiko case BPF_CGROUP_SETSOCKOPT:
3772e28784e3SAndrii Nakryiko return BPF_PROG_TYPE_CGROUP_SOCKOPT;
3773de4e05caSYonghong Song case BPF_TRACE_ITER:
3774df86ca0dSAndrii Nakryiko case BPF_TRACE_RAW_TP:
3775df86ca0dSAndrii Nakryiko case BPF_TRACE_FENTRY:
3776df86ca0dSAndrii Nakryiko case BPF_TRACE_FEXIT:
3777df86ca0dSAndrii Nakryiko case BPF_MODIFY_RETURN:
3778de4e05caSYonghong Song return BPF_PROG_TYPE_TRACING;
3779df86ca0dSAndrii Nakryiko case BPF_LSM_MAC:
3780df86ca0dSAndrii Nakryiko return BPF_PROG_TYPE_LSM;
3781e9ddbb77SJakub Sitnicki case BPF_SK_LOOKUP:
3782e9ddbb77SJakub Sitnicki return BPF_PROG_TYPE_SK_LOOKUP;
3783aa8d3a71SAndrii Nakryiko case BPF_XDP:
3784aa8d3a71SAndrii Nakryiko return BPF_PROG_TYPE_XDP;
378569fd337aSStanislav Fomichev case BPF_LSM_CGROUP:
378669fd337aSStanislav Fomichev return BPF_PROG_TYPE_LSM;
3787e420bed0SDaniel Borkmann case BPF_TCX_INGRESS:
3788e420bed0SDaniel Borkmann case BPF_TCX_EGRESS:
3789e420bed0SDaniel Borkmann return BPF_PROG_TYPE_SCHED_CLS;
3790e28784e3SAndrii Nakryiko default:
3791e28784e3SAndrii Nakryiko return BPF_PROG_TYPE_UNSPEC;
3792e28784e3SAndrii Nakryiko }
3793e28784e3SAndrii Nakryiko }
3794e28784e3SAndrii Nakryiko
bpf_prog_attach_check_attach_type(const struct bpf_prog * prog,enum bpf_attach_type attach_type)37953505cb9fSJiri Olsa static int bpf_prog_attach_check_attach_type(const struct bpf_prog *prog,
37963505cb9fSJiri Olsa enum bpf_attach_type attach_type)
37973505cb9fSJiri Olsa {
37983505cb9fSJiri Olsa enum bpf_prog_type ptype;
37993505cb9fSJiri Olsa
38003505cb9fSJiri Olsa switch (prog->type) {
38013505cb9fSJiri Olsa case BPF_PROG_TYPE_CGROUP_SOCK:
38023505cb9fSJiri Olsa case BPF_PROG_TYPE_CGROUP_SOCK_ADDR:
38033505cb9fSJiri Olsa case BPF_PROG_TYPE_CGROUP_SOCKOPT:
38043505cb9fSJiri Olsa case BPF_PROG_TYPE_SK_LOOKUP:
38053505cb9fSJiri Olsa return attach_type == prog->expected_attach_type ? 0 : -EINVAL;
38063505cb9fSJiri Olsa case BPF_PROG_TYPE_CGROUP_SKB:
38073505cb9fSJiri Olsa if (!capable(CAP_NET_ADMIN))
38083505cb9fSJiri Olsa /* cg-skb progs can be loaded by unpriv user.
38093505cb9fSJiri Olsa * check permissions at attach time.
38103505cb9fSJiri Olsa */
38113505cb9fSJiri Olsa return -EPERM;
38123505cb9fSJiri Olsa return prog->enforce_expected_attach_type &&
38133505cb9fSJiri Olsa prog->expected_attach_type != attach_type ?
38143505cb9fSJiri Olsa -EINVAL : 0;
38153505cb9fSJiri Olsa case BPF_PROG_TYPE_EXT:
38163505cb9fSJiri Olsa return 0;
38173505cb9fSJiri Olsa case BPF_PROG_TYPE_NETFILTER:
38183505cb9fSJiri Olsa if (attach_type != BPF_NETFILTER)
38193505cb9fSJiri Olsa return -EINVAL;
38203505cb9fSJiri Olsa return 0;
38213505cb9fSJiri Olsa case BPF_PROG_TYPE_PERF_EVENT:
38223505cb9fSJiri Olsa case BPF_PROG_TYPE_TRACEPOINT:
38233505cb9fSJiri Olsa if (attach_type != BPF_PERF_EVENT)
38243505cb9fSJiri Olsa return -EINVAL;
38253505cb9fSJiri Olsa return 0;
38263505cb9fSJiri Olsa case BPF_PROG_TYPE_KPROBE:
38273505cb9fSJiri Olsa if (prog->expected_attach_type == BPF_TRACE_KPROBE_MULTI &&
38283505cb9fSJiri Olsa attach_type != BPF_TRACE_KPROBE_MULTI)
38293505cb9fSJiri Olsa return -EINVAL;
383089ae89f5SJiri Olsa if (prog->expected_attach_type == BPF_TRACE_UPROBE_MULTI &&
383189ae89f5SJiri Olsa attach_type != BPF_TRACE_UPROBE_MULTI)
383289ae89f5SJiri Olsa return -EINVAL;
38333505cb9fSJiri Olsa if (attach_type != BPF_PERF_EVENT &&
383489ae89f5SJiri Olsa attach_type != BPF_TRACE_KPROBE_MULTI &&
383589ae89f5SJiri Olsa attach_type != BPF_TRACE_UPROBE_MULTI)
38363505cb9fSJiri Olsa return -EINVAL;
38373505cb9fSJiri Olsa return 0;
38383505cb9fSJiri Olsa case BPF_PROG_TYPE_SCHED_CLS:
38393505cb9fSJiri Olsa if (attach_type != BPF_TCX_INGRESS &&
38403505cb9fSJiri Olsa attach_type != BPF_TCX_EGRESS)
38413505cb9fSJiri Olsa return -EINVAL;
38423505cb9fSJiri Olsa return 0;
38433505cb9fSJiri Olsa default:
38443505cb9fSJiri Olsa ptype = attach_type_to_prog_type(attach_type);
38453505cb9fSJiri Olsa if (ptype == BPF_PROG_TYPE_UNSPEC || ptype != prog->type)
38463505cb9fSJiri Olsa return -EINVAL;
38473505cb9fSJiri Olsa return 0;
38483505cb9fSJiri Olsa }
38493505cb9fSJiri Olsa }
38503505cb9fSJiri Olsa
3851e420bed0SDaniel Borkmann #define BPF_PROG_ATTACH_LAST_FIELD expected_revision
3852174a79ffSJohn Fastabend
3853e420bed0SDaniel Borkmann #define BPF_F_ATTACH_MASK_BASE \
3854e420bed0SDaniel Borkmann (BPF_F_ALLOW_OVERRIDE | \
3855e420bed0SDaniel Borkmann BPF_F_ALLOW_MULTI | \
3856e420bed0SDaniel Borkmann BPF_F_REPLACE)
3857e420bed0SDaniel Borkmann
3858e420bed0SDaniel Borkmann #define BPF_F_ATTACH_MASK_MPROG \
3859e420bed0SDaniel Borkmann (BPF_F_REPLACE | \
3860e420bed0SDaniel Borkmann BPF_F_BEFORE | \
3861e420bed0SDaniel Borkmann BPF_F_AFTER | \
3862e420bed0SDaniel Borkmann BPF_F_ID | \
3863e420bed0SDaniel Borkmann BPF_F_LINK)
3864324bda9eSAlexei Starovoitov
bpf_prog_attach(const union bpf_attr * attr)3865f4324551SDaniel Mack static int bpf_prog_attach(const union bpf_attr *attr)
3866f4324551SDaniel Mack {
38677f677633SAlexei Starovoitov enum bpf_prog_type ptype;
3868f4324551SDaniel Mack struct bpf_prog *prog;
38697f677633SAlexei Starovoitov int ret;
3870f4324551SDaniel Mack
3871f4324551SDaniel Mack if (CHECK_ATTR(BPF_PROG_ATTACH))
3872f4324551SDaniel Mack return -EINVAL;
3873f4324551SDaniel Mack
3874e28784e3SAndrii Nakryiko ptype = attach_type_to_prog_type(attr->attach_type);
3875e28784e3SAndrii Nakryiko if (ptype == BPF_PROG_TYPE_UNSPEC)
3876b2cd1257SDavid Ahern return -EINVAL;
3877ba62d611SLorenz Bauer if (bpf_mprog_supported(ptype)) {
3878ba62d611SLorenz Bauer if (attr->attach_flags & ~BPF_F_ATTACH_MASK_MPROG)
3879e420bed0SDaniel Borkmann return -EINVAL;
3880ba62d611SLorenz Bauer } else {
3881ba62d611SLorenz Bauer if (attr->attach_flags & ~BPF_F_ATTACH_MASK_BASE)
3882ba62d611SLorenz Bauer return -EINVAL;
3883ba62d611SLorenz Bauer if (attr->relative_fd ||
3884ba62d611SLorenz Bauer attr->expected_revision)
3885ba62d611SLorenz Bauer return -EINVAL;
3886ba62d611SLorenz Bauer }
3887b2cd1257SDavid Ahern
3888b2cd1257SDavid Ahern prog = bpf_prog_get_type(attr->attach_bpf_fd, ptype);
3889f4324551SDaniel Mack if (IS_ERR(prog))
3890f4324551SDaniel Mack return PTR_ERR(prog);
3891f4324551SDaniel Mack
38925e43f899SAndrey Ignatov if (bpf_prog_attach_check_attach_type(prog, attr->attach_type)) {
38935e43f899SAndrey Ignatov bpf_prog_put(prog);
38945e43f899SAndrey Ignatov return -EINVAL;
38955e43f899SAndrey Ignatov }
38965e43f899SAndrey Ignatov
3897fdb5c453SSean Young switch (ptype) {
3898fdb5c453SSean Young case BPF_PROG_TYPE_SK_SKB:
3899fdb5c453SSean Young case BPF_PROG_TYPE_SK_MSG:
3900604326b4SDaniel Borkmann ret = sock_map_get_from_fd(attr, prog);
3901fdb5c453SSean Young break;
3902fdb5c453SSean Young case BPF_PROG_TYPE_LIRC_MODE2:
3903fdb5c453SSean Young ret = lirc_prog_attach(attr, prog);
3904fdb5c453SSean Young break;
3905d58e468bSPetar Penkov case BPF_PROG_TYPE_FLOW_DISSECTOR:
3906a3fd7ceeSJakub Sitnicki ret = netns_bpf_prog_attach(attr, prog);
3907d58e468bSPetar Penkov break;
3908e28784e3SAndrii Nakryiko case BPF_PROG_TYPE_CGROUP_DEVICE:
3909e28784e3SAndrii Nakryiko case BPF_PROG_TYPE_CGROUP_SKB:
3910e28784e3SAndrii Nakryiko case BPF_PROG_TYPE_CGROUP_SOCK:
3911e28784e3SAndrii Nakryiko case BPF_PROG_TYPE_CGROUP_SOCK_ADDR:
3912e28784e3SAndrii Nakryiko case BPF_PROG_TYPE_CGROUP_SOCKOPT:
3913e28784e3SAndrii Nakryiko case BPF_PROG_TYPE_CGROUP_SYSCTL:
3914e28784e3SAndrii Nakryiko case BPF_PROG_TYPE_SOCK_OPS:
391569fd337aSStanislav Fomichev case BPF_PROG_TYPE_LSM:
391669fd337aSStanislav Fomichev if (ptype == BPF_PROG_TYPE_LSM &&
391769fd337aSStanislav Fomichev prog->expected_attach_type != BPF_LSM_CGROUP)
3918e89f3edfSMilan Landaverde ret = -EINVAL;
3919e89f3edfSMilan Landaverde else
3920fdb5c453SSean Young ret = cgroup_bpf_prog_attach(attr, ptype, prog);
3921e28784e3SAndrii Nakryiko break;
3922e420bed0SDaniel Borkmann case BPF_PROG_TYPE_SCHED_CLS:
3923e420bed0SDaniel Borkmann ret = tcx_prog_attach(attr, prog);
3924e420bed0SDaniel Borkmann break;
3925e28784e3SAndrii Nakryiko default:
3926e28784e3SAndrii Nakryiko ret = -EINVAL;
3927f4324551SDaniel Mack }
3928f4324551SDaniel Mack
39297f677633SAlexei Starovoitov if (ret)
39307f677633SAlexei Starovoitov bpf_prog_put(prog);
39317f677633SAlexei Starovoitov return ret;
3932f4324551SDaniel Mack }
3933f4324551SDaniel Mack
3934e420bed0SDaniel Borkmann #define BPF_PROG_DETACH_LAST_FIELD expected_revision
3935f4324551SDaniel Mack
bpf_prog_detach(const union bpf_attr * attr)3936f4324551SDaniel Mack static int bpf_prog_detach(const union bpf_attr *attr)
3937f4324551SDaniel Mack {
3938e420bed0SDaniel Borkmann struct bpf_prog *prog = NULL;
3939324bda9eSAlexei Starovoitov enum bpf_prog_type ptype;
3940e420bed0SDaniel Borkmann int ret;
3941f4324551SDaniel Mack
3942f4324551SDaniel Mack if (CHECK_ATTR(BPF_PROG_DETACH))
3943f4324551SDaniel Mack return -EINVAL;
3944f4324551SDaniel Mack
3945e28784e3SAndrii Nakryiko ptype = attach_type_to_prog_type(attr->attach_type);
3946e420bed0SDaniel Borkmann if (bpf_mprog_supported(ptype)) {
3947e420bed0SDaniel Borkmann if (ptype == BPF_PROG_TYPE_UNSPEC)
3948e420bed0SDaniel Borkmann return -EINVAL;
3949e420bed0SDaniel Borkmann if (attr->attach_flags & ~BPF_F_ATTACH_MASK_MPROG)
3950e420bed0SDaniel Borkmann return -EINVAL;
3951e420bed0SDaniel Borkmann if (attr->attach_bpf_fd) {
3952e420bed0SDaniel Borkmann prog = bpf_prog_get_type(attr->attach_bpf_fd, ptype);
3953e420bed0SDaniel Borkmann if (IS_ERR(prog))
3954e420bed0SDaniel Borkmann return PTR_ERR(prog);
3955e420bed0SDaniel Borkmann }
3956ba62d611SLorenz Bauer } else if (attr->attach_flags ||
3957ba62d611SLorenz Bauer attr->relative_fd ||
3958ba62d611SLorenz Bauer attr->expected_revision) {
3959ba62d611SLorenz Bauer return -EINVAL;
3960e420bed0SDaniel Borkmann }
3961e28784e3SAndrii Nakryiko
3962e28784e3SAndrii Nakryiko switch (ptype) {
3963e28784e3SAndrii Nakryiko case BPF_PROG_TYPE_SK_MSG:
3964e28784e3SAndrii Nakryiko case BPF_PROG_TYPE_SK_SKB:
3965e420bed0SDaniel Borkmann ret = sock_map_prog_detach(attr, ptype);
3966e420bed0SDaniel Borkmann break;
3967e28784e3SAndrii Nakryiko case BPF_PROG_TYPE_LIRC_MODE2:
3968e420bed0SDaniel Borkmann ret = lirc_prog_detach(attr);
3969e420bed0SDaniel Borkmann break;
3970e28784e3SAndrii Nakryiko case BPF_PROG_TYPE_FLOW_DISSECTOR:
3971e420bed0SDaniel Borkmann ret = netns_bpf_prog_detach(attr, ptype);
3972e420bed0SDaniel Borkmann break;
3973e28784e3SAndrii Nakryiko case BPF_PROG_TYPE_CGROUP_DEVICE:
3974e28784e3SAndrii Nakryiko case BPF_PROG_TYPE_CGROUP_SKB:
3975e28784e3SAndrii Nakryiko case BPF_PROG_TYPE_CGROUP_SOCK:
3976e28784e3SAndrii Nakryiko case BPF_PROG_TYPE_CGROUP_SOCK_ADDR:
3977e28784e3SAndrii Nakryiko case BPF_PROG_TYPE_CGROUP_SOCKOPT:
3978e28784e3SAndrii Nakryiko case BPF_PROG_TYPE_CGROUP_SYSCTL:
3979e28784e3SAndrii Nakryiko case BPF_PROG_TYPE_SOCK_OPS:
398069fd337aSStanislav Fomichev case BPF_PROG_TYPE_LSM:
3981e420bed0SDaniel Borkmann ret = cgroup_bpf_prog_detach(attr, ptype);
3982e420bed0SDaniel Borkmann break;
3983e420bed0SDaniel Borkmann case BPF_PROG_TYPE_SCHED_CLS:
3984e420bed0SDaniel Borkmann ret = tcx_prog_detach(attr, prog);
3985e420bed0SDaniel Borkmann break;
3986f4324551SDaniel Mack default:
3987e420bed0SDaniel Borkmann ret = -EINVAL;
3988f4324551SDaniel Mack }
398940304b2aSLawrence Brakmo
3990e420bed0SDaniel Borkmann if (prog)
3991e420bed0SDaniel Borkmann bpf_prog_put(prog);
3992e420bed0SDaniel Borkmann return ret;
3993e420bed0SDaniel Borkmann }
3994e420bed0SDaniel Borkmann
3995a4fe7838SDaniel Borkmann #define BPF_PROG_QUERY_LAST_FIELD query.revision
3996468e2f64SAlexei Starovoitov
bpf_prog_query(const union bpf_attr * attr,union bpf_attr __user * uattr)3997468e2f64SAlexei Starovoitov static int bpf_prog_query(const union bpf_attr *attr,
3998468e2f64SAlexei Starovoitov union bpf_attr __user *uattr)
3999468e2f64SAlexei Starovoitov {
4000468e2f64SAlexei Starovoitov if (!capable(CAP_NET_ADMIN))
4001468e2f64SAlexei Starovoitov return -EPERM;
4002468e2f64SAlexei Starovoitov if (CHECK_ATTR(BPF_PROG_QUERY))
4003468e2f64SAlexei Starovoitov return -EINVAL;
4004468e2f64SAlexei Starovoitov if (attr->query.query_flags & ~BPF_F_QUERY_EFFECTIVE)
4005468e2f64SAlexei Starovoitov return -EINVAL;
4006468e2f64SAlexei Starovoitov
4007468e2f64SAlexei Starovoitov switch (attr->query.attach_type) {
4008468e2f64SAlexei Starovoitov case BPF_CGROUP_INET_INGRESS:
4009468e2f64SAlexei Starovoitov case BPF_CGROUP_INET_EGRESS:
4010468e2f64SAlexei Starovoitov case BPF_CGROUP_INET_SOCK_CREATE:
4011f5836749SStanislav Fomichev case BPF_CGROUP_INET_SOCK_RELEASE:
40124fbac77dSAndrey Ignatov case BPF_CGROUP_INET4_BIND:
40134fbac77dSAndrey Ignatov case BPF_CGROUP_INET6_BIND:
4014aac3fc32SAndrey Ignatov case BPF_CGROUP_INET4_POST_BIND:
4015aac3fc32SAndrey Ignatov case BPF_CGROUP_INET6_POST_BIND:
4016d74bad4eSAndrey Ignatov case BPF_CGROUP_INET4_CONNECT:
4017d74bad4eSAndrey Ignatov case BPF_CGROUP_INET6_CONNECT:
40181b66d253SDaniel Borkmann case BPF_CGROUP_INET4_GETPEERNAME:
40191b66d253SDaniel Borkmann case BPF_CGROUP_INET6_GETPEERNAME:
40201b66d253SDaniel Borkmann case BPF_CGROUP_INET4_GETSOCKNAME:
40211b66d253SDaniel Borkmann case BPF_CGROUP_INET6_GETSOCKNAME:
40221cedee13SAndrey Ignatov case BPF_CGROUP_UDP4_SENDMSG:
40231cedee13SAndrey Ignatov case BPF_CGROUP_UDP6_SENDMSG:
4024983695faSDaniel Borkmann case BPF_CGROUP_UDP4_RECVMSG:
4025983695faSDaniel Borkmann case BPF_CGROUP_UDP6_RECVMSG:
4026468e2f64SAlexei Starovoitov case BPF_CGROUP_SOCK_OPS:
4027ebc614f6SRoman Gushchin case BPF_CGROUP_DEVICE:
40287b146cebSAndrey Ignatov case BPF_CGROUP_SYSCTL:
40290d01da6aSStanislav Fomichev case BPF_CGROUP_GETSOCKOPT:
40300d01da6aSStanislav Fomichev case BPF_CGROUP_SETSOCKOPT:
4031b79c9fc9SStanislav Fomichev case BPF_LSM_CGROUP:
4032e28784e3SAndrii Nakryiko return cgroup_bpf_prog_query(attr, uattr);
4033f4364dcfSSean Young case BPF_LIRC_MODE2:
4034f4364dcfSSean Young return lirc_prog_query(attr, uattr);
4035118c8e9aSStanislav Fomichev case BPF_FLOW_DISSECTOR:
4036e9ddbb77SJakub Sitnicki case BPF_SK_LOOKUP:
4037a3fd7ceeSJakub Sitnicki return netns_bpf_prog_query(attr, uattr);
4038748cd572SDi Zhu case BPF_SK_SKB_STREAM_PARSER:
4039748cd572SDi Zhu case BPF_SK_SKB_STREAM_VERDICT:
4040748cd572SDi Zhu case BPF_SK_MSG_VERDICT:
4041748cd572SDi Zhu case BPF_SK_SKB_VERDICT:
4042748cd572SDi Zhu return sock_map_bpf_prog_query(attr, uattr);
4043e420bed0SDaniel Borkmann case BPF_TCX_INGRESS:
4044e420bed0SDaniel Borkmann case BPF_TCX_EGRESS:
4045e420bed0SDaniel Borkmann return tcx_prog_query(attr, uattr);
4046468e2f64SAlexei Starovoitov default:
4047468e2f64SAlexei Starovoitov return -EINVAL;
4048468e2f64SAlexei Starovoitov }
4049468e2f64SAlexei Starovoitov }
4050f4324551SDaniel Mack
4051b530e9e1SToke Høiland-Jørgensen #define BPF_PROG_TEST_RUN_LAST_FIELD test.batch_size
40521cf1cae9SAlexei Starovoitov
bpf_prog_test_run(const union bpf_attr * attr,union bpf_attr __user * uattr)40531cf1cae9SAlexei Starovoitov static int bpf_prog_test_run(const union bpf_attr *attr,
40541cf1cae9SAlexei Starovoitov union bpf_attr __user *uattr)
40551cf1cae9SAlexei Starovoitov {
40561cf1cae9SAlexei Starovoitov struct bpf_prog *prog;
40571cf1cae9SAlexei Starovoitov int ret = -ENOTSUPP;
40581cf1cae9SAlexei Starovoitov
40591cf1cae9SAlexei Starovoitov if (CHECK_ATTR(BPF_PROG_TEST_RUN))
40601cf1cae9SAlexei Starovoitov return -EINVAL;
40611cf1cae9SAlexei Starovoitov
4062b0b9395dSStanislav Fomichev if ((attr->test.ctx_size_in && !attr->test.ctx_in) ||
4063b0b9395dSStanislav Fomichev (!attr->test.ctx_size_in && attr->test.ctx_in))
4064b0b9395dSStanislav Fomichev return -EINVAL;
4065b0b9395dSStanislav Fomichev
4066b0b9395dSStanislav Fomichev if ((attr->test.ctx_size_out && !attr->test.ctx_out) ||
4067b0b9395dSStanislav Fomichev (!attr->test.ctx_size_out && attr->test.ctx_out))
4068b0b9395dSStanislav Fomichev return -EINVAL;
4069b0b9395dSStanislav Fomichev
40701cf1cae9SAlexei Starovoitov prog = bpf_prog_get(attr->test.prog_fd);
40711cf1cae9SAlexei Starovoitov if (IS_ERR(prog))
40721cf1cae9SAlexei Starovoitov return PTR_ERR(prog);
40731cf1cae9SAlexei Starovoitov
40741cf1cae9SAlexei Starovoitov if (prog->aux->ops->test_run)
40751cf1cae9SAlexei Starovoitov ret = prog->aux->ops->test_run(prog, attr, uattr);
40761cf1cae9SAlexei Starovoitov
40771cf1cae9SAlexei Starovoitov bpf_prog_put(prog);
40781cf1cae9SAlexei Starovoitov return ret;
40791cf1cae9SAlexei Starovoitov }
40801cf1cae9SAlexei Starovoitov
408134ad5580SMartin KaFai Lau #define BPF_OBJ_GET_NEXT_ID_LAST_FIELD next_id
408234ad5580SMartin KaFai Lau
bpf_obj_get_next_id(const union bpf_attr * attr,union bpf_attr __user * uattr,struct idr * idr,spinlock_t * lock)408334ad5580SMartin KaFai Lau static int bpf_obj_get_next_id(const union bpf_attr *attr,
408434ad5580SMartin KaFai Lau union bpf_attr __user *uattr,
408534ad5580SMartin KaFai Lau struct idr *idr,
408634ad5580SMartin KaFai Lau spinlock_t *lock)
408734ad5580SMartin KaFai Lau {
408834ad5580SMartin KaFai Lau u32 next_id = attr->start_id;
408934ad5580SMartin KaFai Lau int err = 0;
409034ad5580SMartin KaFai Lau
409134ad5580SMartin KaFai Lau if (CHECK_ATTR(BPF_OBJ_GET_NEXT_ID) || next_id >= INT_MAX)
409234ad5580SMartin KaFai Lau return -EINVAL;
409334ad5580SMartin KaFai Lau
409434ad5580SMartin KaFai Lau if (!capable(CAP_SYS_ADMIN))
409534ad5580SMartin KaFai Lau return -EPERM;
409634ad5580SMartin KaFai Lau
409734ad5580SMartin KaFai Lau next_id++;
409834ad5580SMartin KaFai Lau spin_lock_bh(lock);
409934ad5580SMartin KaFai Lau if (!idr_get_next(idr, &next_id))
410034ad5580SMartin KaFai Lau err = -ENOENT;
410134ad5580SMartin KaFai Lau spin_unlock_bh(lock);
410234ad5580SMartin KaFai Lau
410334ad5580SMartin KaFai Lau if (!err)
410434ad5580SMartin KaFai Lau err = put_user(next_id, &uattr->next_id);
410534ad5580SMartin KaFai Lau
410634ad5580SMartin KaFai Lau return err;
410734ad5580SMartin KaFai Lau }
410834ad5580SMartin KaFai Lau
bpf_map_get_curr_or_next(u32 * id)41096086d29dSYonghong Song struct bpf_map *bpf_map_get_curr_or_next(u32 *id)
41106086d29dSYonghong Song {
41116086d29dSYonghong Song struct bpf_map *map;
41126086d29dSYonghong Song
41136086d29dSYonghong Song spin_lock_bh(&map_idr_lock);
41146086d29dSYonghong Song again:
41156086d29dSYonghong Song map = idr_get_next(&map_idr, id);
41166086d29dSYonghong Song if (map) {
41176086d29dSYonghong Song map = __bpf_map_inc_not_zero(map, false);
41186086d29dSYonghong Song if (IS_ERR(map)) {
41196086d29dSYonghong Song (*id)++;
41206086d29dSYonghong Song goto again;
41216086d29dSYonghong Song }
41226086d29dSYonghong Song }
41236086d29dSYonghong Song spin_unlock_bh(&map_idr_lock);
41246086d29dSYonghong Song
41256086d29dSYonghong Song return map;
41266086d29dSYonghong Song }
41276086d29dSYonghong Song
bpf_prog_get_curr_or_next(u32 * id)4128a228a64fSAlexei Starovoitov struct bpf_prog *bpf_prog_get_curr_or_next(u32 *id)
4129a228a64fSAlexei Starovoitov {
4130a228a64fSAlexei Starovoitov struct bpf_prog *prog;
4131a228a64fSAlexei Starovoitov
4132a228a64fSAlexei Starovoitov spin_lock_bh(&prog_idr_lock);
4133a228a64fSAlexei Starovoitov again:
4134a228a64fSAlexei Starovoitov prog = idr_get_next(&prog_idr, id);
4135a228a64fSAlexei Starovoitov if (prog) {
4136a228a64fSAlexei Starovoitov prog = bpf_prog_inc_not_zero(prog);
4137a228a64fSAlexei Starovoitov if (IS_ERR(prog)) {
4138a228a64fSAlexei Starovoitov (*id)++;
4139a228a64fSAlexei Starovoitov goto again;
4140a228a64fSAlexei Starovoitov }
4141a228a64fSAlexei Starovoitov }
4142a228a64fSAlexei Starovoitov spin_unlock_bh(&prog_idr_lock);
4143a228a64fSAlexei Starovoitov
4144a228a64fSAlexei Starovoitov return prog;
4145a228a64fSAlexei Starovoitov }
4146a228a64fSAlexei Starovoitov
4147b16d9aa4SMartin KaFai Lau #define BPF_PROG_GET_FD_BY_ID_LAST_FIELD prog_id
4148b16d9aa4SMartin KaFai Lau
bpf_prog_by_id(u32 id)41497e6897f9SBjörn Töpel struct bpf_prog *bpf_prog_by_id(u32 id)
41507e6897f9SBjörn Töpel {
41517e6897f9SBjörn Töpel struct bpf_prog *prog;
41527e6897f9SBjörn Töpel
41537e6897f9SBjörn Töpel if (!id)
41547e6897f9SBjörn Töpel return ERR_PTR(-ENOENT);
41557e6897f9SBjörn Töpel
41567e6897f9SBjörn Töpel spin_lock_bh(&prog_idr_lock);
41577e6897f9SBjörn Töpel prog = idr_find(&prog_idr, id);
41587e6897f9SBjörn Töpel if (prog)
41597e6897f9SBjörn Töpel prog = bpf_prog_inc_not_zero(prog);
41607e6897f9SBjörn Töpel else
41617e6897f9SBjörn Töpel prog = ERR_PTR(-ENOENT);
41627e6897f9SBjörn Töpel spin_unlock_bh(&prog_idr_lock);
41637e6897f9SBjörn Töpel return prog;
41647e6897f9SBjörn Töpel }
41657e6897f9SBjörn Töpel
bpf_prog_get_fd_by_id(const union bpf_attr * attr)4166b16d9aa4SMartin KaFai Lau static int bpf_prog_get_fd_by_id(const union bpf_attr *attr)
4167b16d9aa4SMartin KaFai Lau {
4168b16d9aa4SMartin KaFai Lau struct bpf_prog *prog;
4169b16d9aa4SMartin KaFai Lau u32 id = attr->prog_id;
4170b16d9aa4SMartin KaFai Lau int fd;
4171b16d9aa4SMartin KaFai Lau
4172b16d9aa4SMartin KaFai Lau if (CHECK_ATTR(BPF_PROG_GET_FD_BY_ID))
4173b16d9aa4SMartin KaFai Lau return -EINVAL;
4174b16d9aa4SMartin KaFai Lau
4175b16d9aa4SMartin KaFai Lau if (!capable(CAP_SYS_ADMIN))
4176b16d9aa4SMartin KaFai Lau return -EPERM;
4177b16d9aa4SMartin KaFai Lau
41787e6897f9SBjörn Töpel prog = bpf_prog_by_id(id);
4179b16d9aa4SMartin KaFai Lau if (IS_ERR(prog))
4180b16d9aa4SMartin KaFai Lau return PTR_ERR(prog);
4181b16d9aa4SMartin KaFai Lau
4182b16d9aa4SMartin KaFai Lau fd = bpf_prog_new_fd(prog);
4183b16d9aa4SMartin KaFai Lau if (fd < 0)
4184b16d9aa4SMartin KaFai Lau bpf_prog_put(prog);
4185b16d9aa4SMartin KaFai Lau
4186b16d9aa4SMartin KaFai Lau return fd;
4187b16d9aa4SMartin KaFai Lau }
4188b16d9aa4SMartin KaFai Lau
41896e71b04aSChenbo Feng #define BPF_MAP_GET_FD_BY_ID_LAST_FIELD open_flags
4190bd5f5f4eSMartin KaFai Lau
bpf_map_get_fd_by_id(const union bpf_attr * attr)4191bd5f5f4eSMartin KaFai Lau static int bpf_map_get_fd_by_id(const union bpf_attr *attr)
4192bd5f5f4eSMartin KaFai Lau {
4193bd5f5f4eSMartin KaFai Lau struct bpf_map *map;
4194bd5f5f4eSMartin KaFai Lau u32 id = attr->map_id;
41956e71b04aSChenbo Feng int f_flags;
4196bd5f5f4eSMartin KaFai Lau int fd;
4197bd5f5f4eSMartin KaFai Lau
41986e71b04aSChenbo Feng if (CHECK_ATTR(BPF_MAP_GET_FD_BY_ID) ||
41996e71b04aSChenbo Feng attr->open_flags & ~BPF_OBJ_FLAG_MASK)
4200bd5f5f4eSMartin KaFai Lau return -EINVAL;
4201bd5f5f4eSMartin KaFai Lau
4202bd5f5f4eSMartin KaFai Lau if (!capable(CAP_SYS_ADMIN))
4203bd5f5f4eSMartin KaFai Lau return -EPERM;
4204bd5f5f4eSMartin KaFai Lau
42056e71b04aSChenbo Feng f_flags = bpf_get_file_flag(attr->open_flags);
42066e71b04aSChenbo Feng if (f_flags < 0)
42076e71b04aSChenbo Feng return f_flags;
42086e71b04aSChenbo Feng
4209bd5f5f4eSMartin KaFai Lau spin_lock_bh(&map_idr_lock);
4210bd5f5f4eSMartin KaFai Lau map = idr_find(&map_idr, id);
4211bd5f5f4eSMartin KaFai Lau if (map)
4212b0e4701cSStanislav Fomichev map = __bpf_map_inc_not_zero(map, true);
4213bd5f5f4eSMartin KaFai Lau else
4214bd5f5f4eSMartin KaFai Lau map = ERR_PTR(-ENOENT);
4215bd5f5f4eSMartin KaFai Lau spin_unlock_bh(&map_idr_lock);
4216bd5f5f4eSMartin KaFai Lau
4217bd5f5f4eSMartin KaFai Lau if (IS_ERR(map))
4218bd5f5f4eSMartin KaFai Lau return PTR_ERR(map);
4219bd5f5f4eSMartin KaFai Lau
42206e71b04aSChenbo Feng fd = bpf_map_new_fd(map, f_flags);
4221bd5f5f4eSMartin KaFai Lau if (fd < 0)
4222781e6282SPeng Sun bpf_map_put_with_uref(map);
4223bd5f5f4eSMartin KaFai Lau
4224bd5f5f4eSMartin KaFai Lau return fd;
4225bd5f5f4eSMartin KaFai Lau }
4226bd5f5f4eSMartin KaFai Lau
bpf_map_from_imm(const struct bpf_prog * prog,unsigned long addr,u32 * off,u32 * type)42277105e828SDaniel Borkmann static const struct bpf_map *bpf_map_from_imm(const struct bpf_prog *prog,
4228d8eca5bbSDaniel Borkmann unsigned long addr, u32 *off,
4229d8eca5bbSDaniel Borkmann u32 *type)
42307105e828SDaniel Borkmann {
4231d8eca5bbSDaniel Borkmann const struct bpf_map *map;
42327105e828SDaniel Borkmann int i;
42337105e828SDaniel Borkmann
4234984fe94fSYiFei Zhu mutex_lock(&prog->aux->used_maps_mutex);
4235d8eca5bbSDaniel Borkmann for (i = 0, *off = 0; i < prog->aux->used_map_cnt; i++) {
4236d8eca5bbSDaniel Borkmann map = prog->aux->used_maps[i];
4237d8eca5bbSDaniel Borkmann if (map == (void *)addr) {
4238d8eca5bbSDaniel Borkmann *type = BPF_PSEUDO_MAP_FD;
4239984fe94fSYiFei Zhu goto out;
4240d8eca5bbSDaniel Borkmann }
4241d8eca5bbSDaniel Borkmann if (!map->ops->map_direct_value_meta)
4242d8eca5bbSDaniel Borkmann continue;
4243d8eca5bbSDaniel Borkmann if (!map->ops->map_direct_value_meta(map, addr, off)) {
4244d8eca5bbSDaniel Borkmann *type = BPF_PSEUDO_MAP_VALUE;
4245984fe94fSYiFei Zhu goto out;
4246d8eca5bbSDaniel Borkmann }
4247d8eca5bbSDaniel Borkmann }
4248984fe94fSYiFei Zhu map = NULL;
4249d8eca5bbSDaniel Borkmann
4250984fe94fSYiFei Zhu out:
4251984fe94fSYiFei Zhu mutex_unlock(&prog->aux->used_maps_mutex);
4252984fe94fSYiFei Zhu return map;
42537105e828SDaniel Borkmann }
42547105e828SDaniel Borkmann
bpf_insn_prepare_dump(const struct bpf_prog * prog,const struct cred * f_cred)425563960260SKees Cook static struct bpf_insn *bpf_insn_prepare_dump(const struct bpf_prog *prog,
425663960260SKees Cook const struct cred *f_cred)
42577105e828SDaniel Borkmann {
42587105e828SDaniel Borkmann const struct bpf_map *map;
42597105e828SDaniel Borkmann struct bpf_insn *insns;
4260d8eca5bbSDaniel Borkmann u32 off, type;
42617105e828SDaniel Borkmann u64 imm;
426229fcb05bSAndrii Nakryiko u8 code;
42637105e828SDaniel Borkmann int i;
42647105e828SDaniel Borkmann
42657105e828SDaniel Borkmann insns = kmemdup(prog->insnsi, bpf_prog_insn_size(prog),
42667105e828SDaniel Borkmann GFP_USER);
42677105e828SDaniel Borkmann if (!insns)
42687105e828SDaniel Borkmann return insns;
42697105e828SDaniel Borkmann
42707105e828SDaniel Borkmann for (i = 0; i < prog->len; i++) {
427129fcb05bSAndrii Nakryiko code = insns[i].code;
427229fcb05bSAndrii Nakryiko
427329fcb05bSAndrii Nakryiko if (code == (BPF_JMP | BPF_TAIL_CALL)) {
42747105e828SDaniel Borkmann insns[i].code = BPF_JMP | BPF_CALL;
42757105e828SDaniel Borkmann insns[i].imm = BPF_FUNC_tail_call;
42767105e828SDaniel Borkmann /* fall-through */
42777105e828SDaniel Borkmann }
427829fcb05bSAndrii Nakryiko if (code == (BPF_JMP | BPF_CALL) ||
427929fcb05bSAndrii Nakryiko code == (BPF_JMP | BPF_CALL_ARGS)) {
428029fcb05bSAndrii Nakryiko if (code == (BPF_JMP | BPF_CALL_ARGS))
42817105e828SDaniel Borkmann insns[i].code = BPF_JMP | BPF_CALL;
428263960260SKees Cook if (!bpf_dump_raw_ok(f_cred))
42837105e828SDaniel Borkmann insns[i].imm = 0;
42847105e828SDaniel Borkmann continue;
42857105e828SDaniel Borkmann }
428629fcb05bSAndrii Nakryiko if (BPF_CLASS(code) == BPF_LDX && BPF_MODE(code) == BPF_PROBE_MEM) {
428729fcb05bSAndrii Nakryiko insns[i].code = BPF_LDX | BPF_SIZE(code) | BPF_MEM;
428829fcb05bSAndrii Nakryiko continue;
428929fcb05bSAndrii Nakryiko }
42907105e828SDaniel Borkmann
429129fcb05bSAndrii Nakryiko if (code != (BPF_LD | BPF_IMM | BPF_DW))
42927105e828SDaniel Borkmann continue;
42937105e828SDaniel Borkmann
42947105e828SDaniel Borkmann imm = ((u64)insns[i + 1].imm << 32) | (u32)insns[i].imm;
4295d8eca5bbSDaniel Borkmann map = bpf_map_from_imm(prog, imm, &off, &type);
42967105e828SDaniel Borkmann if (map) {
4297d8eca5bbSDaniel Borkmann insns[i].src_reg = type;
42987105e828SDaniel Borkmann insns[i].imm = map->id;
4299d8eca5bbSDaniel Borkmann insns[i + 1].imm = off;
43007105e828SDaniel Borkmann continue;
43017105e828SDaniel Borkmann }
43027105e828SDaniel Borkmann }
43037105e828SDaniel Borkmann
43047105e828SDaniel Borkmann return insns;
43057105e828SDaniel Borkmann }
43067105e828SDaniel Borkmann
set_info_rec_size(struct bpf_prog_info * info)4307c454a46bSMartin KaFai Lau static int set_info_rec_size(struct bpf_prog_info *info)
4308c454a46bSMartin KaFai Lau {
4309c454a46bSMartin KaFai Lau /*
4310c454a46bSMartin KaFai Lau * Ensure info.*_rec_size is the same as kernel expected size
4311c454a46bSMartin KaFai Lau *
4312c454a46bSMartin KaFai Lau * or
4313c454a46bSMartin KaFai Lau *
4314c454a46bSMartin KaFai Lau * Only allow zero *_rec_size if both _rec_size and _cnt are
4315c454a46bSMartin KaFai Lau * zero. In this case, the kernel will set the expected
4316c454a46bSMartin KaFai Lau * _rec_size back to the info.
4317c454a46bSMartin KaFai Lau */
4318c454a46bSMartin KaFai Lau
431911d8b82dSYonghong Song if ((info->nr_func_info || info->func_info_rec_size) &&
4320c454a46bSMartin KaFai Lau info->func_info_rec_size != sizeof(struct bpf_func_info))
4321c454a46bSMartin KaFai Lau return -EINVAL;
4322c454a46bSMartin KaFai Lau
432311d8b82dSYonghong Song if ((info->nr_line_info || info->line_info_rec_size) &&
4324c454a46bSMartin KaFai Lau info->line_info_rec_size != sizeof(struct bpf_line_info))
4325c454a46bSMartin KaFai Lau return -EINVAL;
4326c454a46bSMartin KaFai Lau
432711d8b82dSYonghong Song if ((info->nr_jited_line_info || info->jited_line_info_rec_size) &&
4328c454a46bSMartin KaFai Lau info->jited_line_info_rec_size != sizeof(__u64))
4329c454a46bSMartin KaFai Lau return -EINVAL;
4330c454a46bSMartin KaFai Lau
4331c454a46bSMartin KaFai Lau info->func_info_rec_size = sizeof(struct bpf_func_info);
4332c454a46bSMartin KaFai Lau info->line_info_rec_size = sizeof(struct bpf_line_info);
4333c454a46bSMartin KaFai Lau info->jited_line_info_rec_size = sizeof(__u64);
4334c454a46bSMartin KaFai Lau
4335c454a46bSMartin KaFai Lau return 0;
4336c454a46bSMartin KaFai Lau }
4337c454a46bSMartin KaFai Lau
bpf_prog_get_info_by_fd(struct file * file,struct bpf_prog * prog,const union bpf_attr * attr,union bpf_attr __user * uattr)433863960260SKees Cook static int bpf_prog_get_info_by_fd(struct file *file,
433963960260SKees Cook struct bpf_prog *prog,
43401e270976SMartin KaFai Lau const union bpf_attr *attr,
43411e270976SMartin KaFai Lau union bpf_attr __user *uattr)
43421e270976SMartin KaFai Lau {
43431e270976SMartin KaFai Lau struct bpf_prog_info __user *uinfo = u64_to_user_ptr(attr->info.info);
43446644aabbSStanislav Fomichev struct btf *attach_btf = bpf_prog_get_target_btf(prog);
43455c6f2588SGreg Kroah-Hartman struct bpf_prog_info info;
43461e270976SMartin KaFai Lau u32 info_len = attr->info.info_len;
434761a0abaeSEric Dumazet struct bpf_prog_kstats stats;
43481e270976SMartin KaFai Lau char __user *uinsns;
43491e270976SMartin KaFai Lau u32 ulen;
43501e270976SMartin KaFai Lau int err;
43511e270976SMartin KaFai Lau
4352af2ac3e1SAlexei Starovoitov err = bpf_check_uarg_tail_zero(USER_BPFPTR(uinfo), sizeof(info), info_len);
43531e270976SMartin KaFai Lau if (err)
43541e270976SMartin KaFai Lau return err;
43551e270976SMartin KaFai Lau info_len = min_t(u32, sizeof(info), info_len);
43561e270976SMartin KaFai Lau
43575c6f2588SGreg Kroah-Hartman memset(&info, 0, sizeof(info));
43581e270976SMartin KaFai Lau if (copy_from_user(&info, uinfo, info_len))
435989b09689SDaniel Borkmann return -EFAULT;
43601e270976SMartin KaFai Lau
43611e270976SMartin KaFai Lau info.type = prog->type;
43621e270976SMartin KaFai Lau info.id = prog->aux->id;
4363cb4d2b3fSMartin KaFai Lau info.load_time = prog->aux->load_time;
4364cb4d2b3fSMartin KaFai Lau info.created_by_uid = from_kuid_munged(current_user_ns(),
4365cb4d2b3fSMartin KaFai Lau prog->aux->user->uid);
4366b85fab0eSJiri Olsa info.gpl_compatible = prog->gpl_compatible;
43671e270976SMartin KaFai Lau
43681e270976SMartin KaFai Lau memcpy(info.tag, prog->tag, sizeof(prog->tag));
4369cb4d2b3fSMartin KaFai Lau memcpy(info.name, prog->aux->name, sizeof(prog->aux->name));
4370cb4d2b3fSMartin KaFai Lau
4371984fe94fSYiFei Zhu mutex_lock(&prog->aux->used_maps_mutex);
4372cb4d2b3fSMartin KaFai Lau ulen = info.nr_map_ids;
4373cb4d2b3fSMartin KaFai Lau info.nr_map_ids = prog->aux->used_map_cnt;
4374cb4d2b3fSMartin KaFai Lau ulen = min_t(u32, info.nr_map_ids, ulen);
4375cb4d2b3fSMartin KaFai Lau if (ulen) {
4376721e08daSMartin KaFai Lau u32 __user *user_map_ids = u64_to_user_ptr(info.map_ids);
4377cb4d2b3fSMartin KaFai Lau u32 i;
4378cb4d2b3fSMartin KaFai Lau
4379cb4d2b3fSMartin KaFai Lau for (i = 0; i < ulen; i++)
4380cb4d2b3fSMartin KaFai Lau if (put_user(prog->aux->used_maps[i]->id,
4381984fe94fSYiFei Zhu &user_map_ids[i])) {
4382984fe94fSYiFei Zhu mutex_unlock(&prog->aux->used_maps_mutex);
4383cb4d2b3fSMartin KaFai Lau return -EFAULT;
4384cb4d2b3fSMartin KaFai Lau }
4385984fe94fSYiFei Zhu }
4386984fe94fSYiFei Zhu mutex_unlock(&prog->aux->used_maps_mutex);
43871e270976SMartin KaFai Lau
4388c454a46bSMartin KaFai Lau err = set_info_rec_size(&info);
4389c454a46bSMartin KaFai Lau if (err)
4390c454a46bSMartin KaFai Lau return err;
43917337224fSMartin KaFai Lau
43925f8f8b93SAlexei Starovoitov bpf_prog_get_stats(prog, &stats);
43935f8f8b93SAlexei Starovoitov info.run_time_ns = stats.nsecs;
43945f8f8b93SAlexei Starovoitov info.run_cnt = stats.cnt;
43959ed9e9baSAlexei Starovoitov info.recursion_misses = stats.misses;
43965f8f8b93SAlexei Starovoitov
4397aba64c7dSDave Marchevsky info.verified_insns = prog->aux->verified_insns;
4398aba64c7dSDave Marchevsky
43992c78ee89SAlexei Starovoitov if (!bpf_capable()) {
44001e270976SMartin KaFai Lau info.jited_prog_len = 0;
44011e270976SMartin KaFai Lau info.xlated_prog_len = 0;
4402dbecd738SSandipan Das info.nr_jited_ksyms = 0;
440328c2fae7SDaniel Borkmann info.nr_jited_func_lens = 0;
440411d8b82dSYonghong Song info.nr_func_info = 0;
440511d8b82dSYonghong Song info.nr_line_info = 0;
440611d8b82dSYonghong Song info.nr_jited_line_info = 0;
44071e270976SMartin KaFai Lau goto done;
44081e270976SMartin KaFai Lau }
44091e270976SMartin KaFai Lau
44101e270976SMartin KaFai Lau ulen = info.xlated_prog_len;
44119975a54bSDaniel Borkmann info.xlated_prog_len = bpf_prog_insn_size(prog);
44121e270976SMartin KaFai Lau if (info.xlated_prog_len && ulen) {
44137105e828SDaniel Borkmann struct bpf_insn *insns_sanitized;
44147105e828SDaniel Borkmann bool fault;
44157105e828SDaniel Borkmann
441663960260SKees Cook if (prog->blinded && !bpf_dump_raw_ok(file->f_cred)) {
44177105e828SDaniel Borkmann info.xlated_prog_insns = 0;
44187105e828SDaniel Borkmann goto done;
44197105e828SDaniel Borkmann }
442063960260SKees Cook insns_sanitized = bpf_insn_prepare_dump(prog, file->f_cred);
44217105e828SDaniel Borkmann if (!insns_sanitized)
44227105e828SDaniel Borkmann return -ENOMEM;
44231e270976SMartin KaFai Lau uinsns = u64_to_user_ptr(info.xlated_prog_insns);
44241e270976SMartin KaFai Lau ulen = min_t(u32, info.xlated_prog_len, ulen);
44257105e828SDaniel Borkmann fault = copy_to_user(uinsns, insns_sanitized, ulen);
44267105e828SDaniel Borkmann kfree(insns_sanitized);
44277105e828SDaniel Borkmann if (fault)
44281e270976SMartin KaFai Lau return -EFAULT;
44291e270976SMartin KaFai Lau }
44301e270976SMartin KaFai Lau
44319d03ebc7SStanislav Fomichev if (bpf_prog_is_offloaded(prog->aux)) {
4432675fc275SJakub Kicinski err = bpf_prog_offload_info_fill(&info, prog);
4433675fc275SJakub Kicinski if (err)
4434675fc275SJakub Kicinski return err;
4435fcfb126dSJiong Wang goto done;
4436fcfb126dSJiong Wang }
4437fcfb126dSJiong Wang
4438fcfb126dSJiong Wang /* NOTE: the following code is supposed to be skipped for offload.
4439fcfb126dSJiong Wang * bpf_prog_offload_info_fill() is the place to fill similar fields
4440fcfb126dSJiong Wang * for offload.
4441fcfb126dSJiong Wang */
4442fcfb126dSJiong Wang ulen = info.jited_prog_len;
44434d56a76eSSandipan Das if (prog->aux->func_cnt) {
44444d56a76eSSandipan Das u32 i;
44454d56a76eSSandipan Das
44464d56a76eSSandipan Das info.jited_prog_len = 0;
44474d56a76eSSandipan Das for (i = 0; i < prog->aux->func_cnt; i++)
44484d56a76eSSandipan Das info.jited_prog_len += prog->aux->func[i]->jited_len;
44494d56a76eSSandipan Das } else {
4450fcfb126dSJiong Wang info.jited_prog_len = prog->jited_len;
44514d56a76eSSandipan Das }
44524d56a76eSSandipan Das
4453fcfb126dSJiong Wang if (info.jited_prog_len && ulen) {
445463960260SKees Cook if (bpf_dump_raw_ok(file->f_cred)) {
4455fcfb126dSJiong Wang uinsns = u64_to_user_ptr(info.jited_prog_insns);
4456fcfb126dSJiong Wang ulen = min_t(u32, info.jited_prog_len, ulen);
44574d56a76eSSandipan Das
44584d56a76eSSandipan Das /* for multi-function programs, copy the JITed
44594d56a76eSSandipan Das * instructions for all the functions
44604d56a76eSSandipan Das */
44614d56a76eSSandipan Das if (prog->aux->func_cnt) {
44624d56a76eSSandipan Das u32 len, free, i;
44634d56a76eSSandipan Das u8 *img;
44644d56a76eSSandipan Das
44654d56a76eSSandipan Das free = ulen;
44664d56a76eSSandipan Das for (i = 0; i < prog->aux->func_cnt; i++) {
44674d56a76eSSandipan Das len = prog->aux->func[i]->jited_len;
44684d56a76eSSandipan Das len = min_t(u32, len, free);
44694d56a76eSSandipan Das img = (u8 *) prog->aux->func[i]->bpf_func;
44704d56a76eSSandipan Das if (copy_to_user(uinsns, img, len))
44714d56a76eSSandipan Das return -EFAULT;
44724d56a76eSSandipan Das uinsns += len;
44734d56a76eSSandipan Das free -= len;
44744d56a76eSSandipan Das if (!free)
44754d56a76eSSandipan Das break;
44764d56a76eSSandipan Das }
44774d56a76eSSandipan Das } else {
4478fcfb126dSJiong Wang if (copy_to_user(uinsns, prog->bpf_func, ulen))
4479fcfb126dSJiong Wang return -EFAULT;
44804d56a76eSSandipan Das }
4481fcfb126dSJiong Wang } else {
4482fcfb126dSJiong Wang info.jited_prog_insns = 0;
4483fcfb126dSJiong Wang }
4484675fc275SJakub Kicinski }
4485675fc275SJakub Kicinski
4486dbecd738SSandipan Das ulen = info.nr_jited_ksyms;
4487ff1889fcSSong Liu info.nr_jited_ksyms = prog->aux->func_cnt ? : 1;
44887a5725ddSSong Liu if (ulen) {
448963960260SKees Cook if (bpf_dump_raw_ok(file->f_cred)) {
4490ff1889fcSSong Liu unsigned long ksym_addr;
4491dbecd738SSandipan Das u64 __user *user_ksyms;
4492dbecd738SSandipan Das u32 i;
4493dbecd738SSandipan Das
4494dbecd738SSandipan Das /* copy the address of the kernel symbol
4495dbecd738SSandipan Das * corresponding to each function
4496dbecd738SSandipan Das */
4497dbecd738SSandipan Das ulen = min_t(u32, info.nr_jited_ksyms, ulen);
4498dbecd738SSandipan Das user_ksyms = u64_to_user_ptr(info.jited_ksyms);
4499ff1889fcSSong Liu if (prog->aux->func_cnt) {
4500dbecd738SSandipan Das for (i = 0; i < ulen; i++) {
4501ff1889fcSSong Liu ksym_addr = (unsigned long)
4502ff1889fcSSong Liu prog->aux->func[i]->bpf_func;
4503ff1889fcSSong Liu if (put_user((u64) ksym_addr,
4504ff1889fcSSong Liu &user_ksyms[i]))
4505ff1889fcSSong Liu return -EFAULT;
4506ff1889fcSSong Liu }
4507ff1889fcSSong Liu } else {
4508ff1889fcSSong Liu ksym_addr = (unsigned long) prog->bpf_func;
4509ff1889fcSSong Liu if (put_user((u64) ksym_addr, &user_ksyms[0]))
4510dbecd738SSandipan Das return -EFAULT;
4511dbecd738SSandipan Das }
4512dbecd738SSandipan Das } else {
4513dbecd738SSandipan Das info.jited_ksyms = 0;
4514dbecd738SSandipan Das }
4515dbecd738SSandipan Das }
4516dbecd738SSandipan Das
4517815581c1SSandipan Das ulen = info.nr_jited_func_lens;
4518ff1889fcSSong Liu info.nr_jited_func_lens = prog->aux->func_cnt ? : 1;
45197a5725ddSSong Liu if (ulen) {
452063960260SKees Cook if (bpf_dump_raw_ok(file->f_cred)) {
4521815581c1SSandipan Das u32 __user *user_lens;
4522815581c1SSandipan Das u32 func_len, i;
4523815581c1SSandipan Das
4524815581c1SSandipan Das /* copy the JITed image lengths for each function */
4525815581c1SSandipan Das ulen = min_t(u32, info.nr_jited_func_lens, ulen);
4526815581c1SSandipan Das user_lens = u64_to_user_ptr(info.jited_func_lens);
4527ff1889fcSSong Liu if (prog->aux->func_cnt) {
4528815581c1SSandipan Das for (i = 0; i < ulen; i++) {
4529ff1889fcSSong Liu func_len =
4530ff1889fcSSong Liu prog->aux->func[i]->jited_len;
4531815581c1SSandipan Das if (put_user(func_len, &user_lens[i]))
4532815581c1SSandipan Das return -EFAULT;
4533815581c1SSandipan Das }
4534815581c1SSandipan Das } else {
4535ff1889fcSSong Liu func_len = prog->jited_len;
4536ff1889fcSSong Liu if (put_user(func_len, &user_lens[0]))
4537ff1889fcSSong Liu return -EFAULT;
4538ff1889fcSSong Liu }
4539ff1889fcSSong Liu } else {
4540815581c1SSandipan Das info.jited_func_lens = 0;
4541815581c1SSandipan Das }
4542815581c1SSandipan Das }
4543815581c1SSandipan Das
45447337224fSMartin KaFai Lau if (prog->aux->btf)
454522dc4a0fSAndrii Nakryiko info.btf_id = btf_obj_id(prog->aux->btf);
4546b79c9fc9SStanislav Fomichev info.attach_btf_id = prog->aux->attach_btf_id;
45476644aabbSStanislav Fomichev if (attach_btf)
45486644aabbSStanislav Fomichev info.attach_btf_obj_id = btf_obj_id(attach_btf);
4549838e9690SYonghong Song
455011d8b82dSYonghong Song ulen = info.nr_func_info;
455111d8b82dSYonghong Song info.nr_func_info = prog->aux->func_info_cnt;
455211d8b82dSYonghong Song if (info.nr_func_info && ulen) {
4553838e9690SYonghong Song char __user *user_finfo;
4554838e9690SYonghong Song
4555838e9690SYonghong Song user_finfo = u64_to_user_ptr(info.func_info);
455611d8b82dSYonghong Song ulen = min_t(u32, info.nr_func_info, ulen);
4557ba64e7d8SYonghong Song if (copy_to_user(user_finfo, prog->aux->func_info,
45587337224fSMartin KaFai Lau info.func_info_rec_size * ulen))
4559838e9690SYonghong Song return -EFAULT;
4560838e9690SYonghong Song }
4561838e9690SYonghong Song
456211d8b82dSYonghong Song ulen = info.nr_line_info;
456311d8b82dSYonghong Song info.nr_line_info = prog->aux->nr_linfo;
456411d8b82dSYonghong Song if (info.nr_line_info && ulen) {
4565c454a46bSMartin KaFai Lau __u8 __user *user_linfo;
4566c454a46bSMartin KaFai Lau
4567c454a46bSMartin KaFai Lau user_linfo = u64_to_user_ptr(info.line_info);
456811d8b82dSYonghong Song ulen = min_t(u32, info.nr_line_info, ulen);
4569c454a46bSMartin KaFai Lau if (copy_to_user(user_linfo, prog->aux->linfo,
4570c454a46bSMartin KaFai Lau info.line_info_rec_size * ulen))
4571c454a46bSMartin KaFai Lau return -EFAULT;
4572c454a46bSMartin KaFai Lau }
4573c454a46bSMartin KaFai Lau
457411d8b82dSYonghong Song ulen = info.nr_jited_line_info;
4575c454a46bSMartin KaFai Lau if (prog->aux->jited_linfo)
457611d8b82dSYonghong Song info.nr_jited_line_info = prog->aux->nr_linfo;
4577c454a46bSMartin KaFai Lau else
457811d8b82dSYonghong Song info.nr_jited_line_info = 0;
457911d8b82dSYonghong Song if (info.nr_jited_line_info && ulen) {
458063960260SKees Cook if (bpf_dump_raw_ok(file->f_cred)) {
45812cd00852SPu Lehui unsigned long line_addr;
4582c454a46bSMartin KaFai Lau __u64 __user *user_linfo;
4583c454a46bSMartin KaFai Lau u32 i;
4584c454a46bSMartin KaFai Lau
4585c454a46bSMartin KaFai Lau user_linfo = u64_to_user_ptr(info.jited_line_info);
458611d8b82dSYonghong Song ulen = min_t(u32, info.nr_jited_line_info, ulen);
4587c454a46bSMartin KaFai Lau for (i = 0; i < ulen; i++) {
45882cd00852SPu Lehui line_addr = (unsigned long)prog->aux->jited_linfo[i];
45892cd00852SPu Lehui if (put_user((__u64)line_addr, &user_linfo[i]))
4590c454a46bSMartin KaFai Lau return -EFAULT;
4591c454a46bSMartin KaFai Lau }
4592c454a46bSMartin KaFai Lau } else {
4593c454a46bSMartin KaFai Lau info.jited_line_info = 0;
4594c454a46bSMartin KaFai Lau }
4595c454a46bSMartin KaFai Lau }
4596c454a46bSMartin KaFai Lau
4597c872bdb3SSong Liu ulen = info.nr_prog_tags;
4598c872bdb3SSong Liu info.nr_prog_tags = prog->aux->func_cnt ? : 1;
4599c872bdb3SSong Liu if (ulen) {
4600c872bdb3SSong Liu __u8 __user (*user_prog_tags)[BPF_TAG_SIZE];
4601c872bdb3SSong Liu u32 i;
4602c872bdb3SSong Liu
4603c872bdb3SSong Liu user_prog_tags = u64_to_user_ptr(info.prog_tags);
4604c872bdb3SSong Liu ulen = min_t(u32, info.nr_prog_tags, ulen);
4605c872bdb3SSong Liu if (prog->aux->func_cnt) {
4606c872bdb3SSong Liu for (i = 0; i < ulen; i++) {
4607c872bdb3SSong Liu if (copy_to_user(user_prog_tags[i],
4608c872bdb3SSong Liu prog->aux->func[i]->tag,
4609c872bdb3SSong Liu BPF_TAG_SIZE))
4610c872bdb3SSong Liu return -EFAULT;
4611c872bdb3SSong Liu }
4612c872bdb3SSong Liu } else {
4613c872bdb3SSong Liu if (copy_to_user(user_prog_tags[0],
4614c872bdb3SSong Liu prog->tag, BPF_TAG_SIZE))
4615c872bdb3SSong Liu return -EFAULT;
4616c872bdb3SSong Liu }
4617c872bdb3SSong Liu }
4618c872bdb3SSong Liu
46191e270976SMartin KaFai Lau done:
46201e270976SMartin KaFai Lau if (copy_to_user(uinfo, &info, info_len) ||
46211e270976SMartin KaFai Lau put_user(info_len, &uattr->info.info_len))
46221e270976SMartin KaFai Lau return -EFAULT;
46231e270976SMartin KaFai Lau
46241e270976SMartin KaFai Lau return 0;
46251e270976SMartin KaFai Lau }
46261e270976SMartin KaFai Lau
bpf_map_get_info_by_fd(struct file * file,struct bpf_map * map,const union bpf_attr * attr,union bpf_attr __user * uattr)462763960260SKees Cook static int bpf_map_get_info_by_fd(struct file *file,
462863960260SKees Cook struct bpf_map *map,
46291e270976SMartin KaFai Lau const union bpf_attr *attr,
46301e270976SMartin KaFai Lau union bpf_attr __user *uattr)
46311e270976SMartin KaFai Lau {
46321e270976SMartin KaFai Lau struct bpf_map_info __user *uinfo = u64_to_user_ptr(attr->info.info);
46335c6f2588SGreg Kroah-Hartman struct bpf_map_info info;
46341e270976SMartin KaFai Lau u32 info_len = attr->info.info_len;
46351e270976SMartin KaFai Lau int err;
46361e270976SMartin KaFai Lau
4637af2ac3e1SAlexei Starovoitov err = bpf_check_uarg_tail_zero(USER_BPFPTR(uinfo), sizeof(info), info_len);
46381e270976SMartin KaFai Lau if (err)
46391e270976SMartin KaFai Lau return err;
46401e270976SMartin KaFai Lau info_len = min_t(u32, sizeof(info), info_len);
46411e270976SMartin KaFai Lau
46425c6f2588SGreg Kroah-Hartman memset(&info, 0, sizeof(info));
46431e270976SMartin KaFai Lau info.type = map->map_type;
46441e270976SMartin KaFai Lau info.id = map->id;
46451e270976SMartin KaFai Lau info.key_size = map->key_size;
46461e270976SMartin KaFai Lau info.value_size = map->value_size;
46471e270976SMartin KaFai Lau info.max_entries = map->max_entries;
46481e270976SMartin KaFai Lau info.map_flags = map->map_flags;
46499330986cSJoanne Koong info.map_extra = map->map_extra;
4650ad5b177bSMartin KaFai Lau memcpy(info.name, map->name, sizeof(map->name));
46511e270976SMartin KaFai Lau
465278958fcaSMartin KaFai Lau if (map->btf) {
465322dc4a0fSAndrii Nakryiko info.btf_id = btf_obj_id(map->btf);
46549b2cf328SMartin KaFai Lau info.btf_key_type_id = map->btf_key_type_id;
46559b2cf328SMartin KaFai Lau info.btf_value_type_id = map->btf_value_type_id;
465678958fcaSMartin KaFai Lau }
465785d33df3SMartin KaFai Lau info.btf_vmlinux_value_type_id = map->btf_vmlinux_value_type_id;
465878958fcaSMartin KaFai Lau
46599d03ebc7SStanislav Fomichev if (bpf_map_is_offloaded(map)) {
466052775b33SJakub Kicinski err = bpf_map_offload_info_fill(&info, map);
466152775b33SJakub Kicinski if (err)
466252775b33SJakub Kicinski return err;
466352775b33SJakub Kicinski }
466452775b33SJakub Kicinski
46651e270976SMartin KaFai Lau if (copy_to_user(uinfo, &info, info_len) ||
46661e270976SMartin KaFai Lau put_user(info_len, &uattr->info.info_len))
46671e270976SMartin KaFai Lau return -EFAULT;
46681e270976SMartin KaFai Lau
46691e270976SMartin KaFai Lau return 0;
46701e270976SMartin KaFai Lau }
46711e270976SMartin KaFai Lau
bpf_btf_get_info_by_fd(struct file * file,struct btf * btf,const union bpf_attr * attr,union bpf_attr __user * uattr)467263960260SKees Cook static int bpf_btf_get_info_by_fd(struct file *file,
467363960260SKees Cook struct btf *btf,
467462dab84cSMartin KaFai Lau const union bpf_attr *attr,
467562dab84cSMartin KaFai Lau union bpf_attr __user *uattr)
467662dab84cSMartin KaFai Lau {
467762dab84cSMartin KaFai Lau struct bpf_btf_info __user *uinfo = u64_to_user_ptr(attr->info.info);
467862dab84cSMartin KaFai Lau u32 info_len = attr->info.info_len;
467962dab84cSMartin KaFai Lau int err;
468062dab84cSMartin KaFai Lau
4681af2ac3e1SAlexei Starovoitov err = bpf_check_uarg_tail_zero(USER_BPFPTR(uinfo), sizeof(*uinfo), info_len);
468262dab84cSMartin KaFai Lau if (err)
468362dab84cSMartin KaFai Lau return err;
468462dab84cSMartin KaFai Lau
468562dab84cSMartin KaFai Lau return btf_get_info_by_fd(btf, attr, uattr);
468662dab84cSMartin KaFai Lau }
468762dab84cSMartin KaFai Lau
bpf_link_get_info_by_fd(struct file * file,struct bpf_link * link,const union bpf_attr * attr,union bpf_attr __user * uattr)468863960260SKees Cook static int bpf_link_get_info_by_fd(struct file *file,
468963960260SKees Cook struct bpf_link *link,
4690f2e10bffSAndrii Nakryiko const union bpf_attr *attr,
4691f2e10bffSAndrii Nakryiko union bpf_attr __user *uattr)
4692f2e10bffSAndrii Nakryiko {
4693f2e10bffSAndrii Nakryiko struct bpf_link_info __user *uinfo = u64_to_user_ptr(attr->info.info);
4694f2e10bffSAndrii Nakryiko struct bpf_link_info info;
4695f2e10bffSAndrii Nakryiko u32 info_len = attr->info.info_len;
4696f2e10bffSAndrii Nakryiko int err;
4697f2e10bffSAndrii Nakryiko
4698af2ac3e1SAlexei Starovoitov err = bpf_check_uarg_tail_zero(USER_BPFPTR(uinfo), sizeof(info), info_len);
4699f2e10bffSAndrii Nakryiko if (err)
4700f2e10bffSAndrii Nakryiko return err;
4701f2e10bffSAndrii Nakryiko info_len = min_t(u32, sizeof(info), info_len);
4702f2e10bffSAndrii Nakryiko
4703f2e10bffSAndrii Nakryiko memset(&info, 0, sizeof(info));
4704f2e10bffSAndrii Nakryiko if (copy_from_user(&info, uinfo, info_len))
4705f2e10bffSAndrii Nakryiko return -EFAULT;
4706f2e10bffSAndrii Nakryiko
4707f2e10bffSAndrii Nakryiko info.type = link->type;
4708f2e10bffSAndrii Nakryiko info.id = link->id;
470968b04864SKui-Feng Lee if (link->prog)
4710f2e10bffSAndrii Nakryiko info.prog_id = link->prog->aux->id;
4711f2e10bffSAndrii Nakryiko
4712f2e10bffSAndrii Nakryiko if (link->ops->fill_link_info) {
4713f2e10bffSAndrii Nakryiko err = link->ops->fill_link_info(link, &info);
4714f2e10bffSAndrii Nakryiko if (err)
4715f2e10bffSAndrii Nakryiko return err;
4716f2e10bffSAndrii Nakryiko }
4717f2e10bffSAndrii Nakryiko
4718f2e10bffSAndrii Nakryiko if (copy_to_user(uinfo, &info, info_len) ||
4719f2e10bffSAndrii Nakryiko put_user(info_len, &uattr->info.info_len))
4720f2e10bffSAndrii Nakryiko return -EFAULT;
4721f2e10bffSAndrii Nakryiko
4722f2e10bffSAndrii Nakryiko return 0;
4723f2e10bffSAndrii Nakryiko }
4724f2e10bffSAndrii Nakryiko
4725f2e10bffSAndrii Nakryiko
47261e270976SMartin KaFai Lau #define BPF_OBJ_GET_INFO_BY_FD_LAST_FIELD info.info
47271e270976SMartin KaFai Lau
bpf_obj_get_info_by_fd(const union bpf_attr * attr,union bpf_attr __user * uattr)47281e270976SMartin KaFai Lau static int bpf_obj_get_info_by_fd(const union bpf_attr *attr,
47291e270976SMartin KaFai Lau union bpf_attr __user *uattr)
47301e270976SMartin KaFai Lau {
47311e270976SMartin KaFai Lau int ufd = attr->info.bpf_fd;
47321e270976SMartin KaFai Lau struct fd f;
47331e270976SMartin KaFai Lau int err;
47341e270976SMartin KaFai Lau
47351e270976SMartin KaFai Lau if (CHECK_ATTR(BPF_OBJ_GET_INFO_BY_FD))
47361e270976SMartin KaFai Lau return -EINVAL;
47371e270976SMartin KaFai Lau
47381e270976SMartin KaFai Lau f = fdget(ufd);
47391e270976SMartin KaFai Lau if (!f.file)
47401e270976SMartin KaFai Lau return -EBADFD;
47411e270976SMartin KaFai Lau
47421e270976SMartin KaFai Lau if (f.file->f_op == &bpf_prog_fops)
474363960260SKees Cook err = bpf_prog_get_info_by_fd(f.file, f.file->private_data, attr,
47441e270976SMartin KaFai Lau uattr);
47451e270976SMartin KaFai Lau else if (f.file->f_op == &bpf_map_fops)
474663960260SKees Cook err = bpf_map_get_info_by_fd(f.file, f.file->private_data, attr,
47471e270976SMartin KaFai Lau uattr);
474860197cfbSMartin KaFai Lau else if (f.file->f_op == &btf_fops)
474963960260SKees Cook err = bpf_btf_get_info_by_fd(f.file, f.file->private_data, attr, uattr);
4750f2e10bffSAndrii Nakryiko else if (f.file->f_op == &bpf_link_fops)
475163960260SKees Cook err = bpf_link_get_info_by_fd(f.file, f.file->private_data,
4752f2e10bffSAndrii Nakryiko attr, uattr);
47531e270976SMartin KaFai Lau else
47541e270976SMartin KaFai Lau err = -EINVAL;
47551e270976SMartin KaFai Lau
47561e270976SMartin KaFai Lau fdput(f);
47571e270976SMartin KaFai Lau return err;
47581e270976SMartin KaFai Lau }
47591e270976SMartin KaFai Lau
476047a71c1fSAndrii Nakryiko #define BPF_BTF_LOAD_LAST_FIELD btf_log_true_size
4761f56a653cSMartin KaFai Lau
bpf_btf_load(const union bpf_attr * attr,bpfptr_t uattr,__u32 uattr_size)476247a71c1fSAndrii Nakryiko static int bpf_btf_load(const union bpf_attr *attr, bpfptr_t uattr, __u32 uattr_size)
4763f56a653cSMartin KaFai Lau {
4764f56a653cSMartin KaFai Lau if (CHECK_ATTR(BPF_BTF_LOAD))
4765f56a653cSMartin KaFai Lau return -EINVAL;
4766f56a653cSMartin KaFai Lau
47672c78ee89SAlexei Starovoitov if (!bpf_capable())
4768f56a653cSMartin KaFai Lau return -EPERM;
4769f56a653cSMartin KaFai Lau
477047a71c1fSAndrii Nakryiko return btf_new_fd(attr, uattr, uattr_size);
4771f56a653cSMartin KaFai Lau }
4772f56a653cSMartin KaFai Lau
477378958fcaSMartin KaFai Lau #define BPF_BTF_GET_FD_BY_ID_LAST_FIELD btf_id
477478958fcaSMartin KaFai Lau
bpf_btf_get_fd_by_id(const union bpf_attr * attr)477578958fcaSMartin KaFai Lau static int bpf_btf_get_fd_by_id(const union bpf_attr *attr)
477678958fcaSMartin KaFai Lau {
477778958fcaSMartin KaFai Lau if (CHECK_ATTR(BPF_BTF_GET_FD_BY_ID))
477878958fcaSMartin KaFai Lau return -EINVAL;
477978958fcaSMartin KaFai Lau
478078958fcaSMartin KaFai Lau if (!capable(CAP_SYS_ADMIN))
478178958fcaSMartin KaFai Lau return -EPERM;
478278958fcaSMartin KaFai Lau
478378958fcaSMartin KaFai Lau return btf_get_fd_by_id(attr->btf_id);
478478958fcaSMartin KaFai Lau }
478578958fcaSMartin KaFai Lau
bpf_task_fd_query_copy(const union bpf_attr * attr,union bpf_attr __user * uattr,u32 prog_id,u32 fd_type,const char * buf,u64 probe_offset,u64 probe_addr)478641bdc4b4SYonghong Song static int bpf_task_fd_query_copy(const union bpf_attr *attr,
478741bdc4b4SYonghong Song union bpf_attr __user *uattr,
478841bdc4b4SYonghong Song u32 prog_id, u32 fd_type,
478941bdc4b4SYonghong Song const char *buf, u64 probe_offset,
479041bdc4b4SYonghong Song u64 probe_addr)
479141bdc4b4SYonghong Song {
479241bdc4b4SYonghong Song char __user *ubuf = u64_to_user_ptr(attr->task_fd_query.buf);
479341bdc4b4SYonghong Song u32 len = buf ? strlen(buf) : 0, input_len;
479441bdc4b4SYonghong Song int err = 0;
479541bdc4b4SYonghong Song
479641bdc4b4SYonghong Song if (put_user(len, &uattr->task_fd_query.buf_len))
479741bdc4b4SYonghong Song return -EFAULT;
479841bdc4b4SYonghong Song input_len = attr->task_fd_query.buf_len;
479941bdc4b4SYonghong Song if (input_len && ubuf) {
480041bdc4b4SYonghong Song if (!len) {
480141bdc4b4SYonghong Song /* nothing to copy, just make ubuf NULL terminated */
480241bdc4b4SYonghong Song char zero = '\0';
480341bdc4b4SYonghong Song
480441bdc4b4SYonghong Song if (put_user(zero, ubuf))
480541bdc4b4SYonghong Song return -EFAULT;
480641bdc4b4SYonghong Song } else if (input_len >= len + 1) {
480741bdc4b4SYonghong Song /* ubuf can hold the string with NULL terminator */
480841bdc4b4SYonghong Song if (copy_to_user(ubuf, buf, len + 1))
480941bdc4b4SYonghong Song return -EFAULT;
481041bdc4b4SYonghong Song } else {
481141bdc4b4SYonghong Song /* ubuf cannot hold the string with NULL terminator,
481241bdc4b4SYonghong Song * do a partial copy with NULL terminator.
481341bdc4b4SYonghong Song */
481441bdc4b4SYonghong Song char zero = '\0';
481541bdc4b4SYonghong Song
481641bdc4b4SYonghong Song err = -ENOSPC;
481741bdc4b4SYonghong Song if (copy_to_user(ubuf, buf, input_len - 1))
481841bdc4b4SYonghong Song return -EFAULT;
481941bdc4b4SYonghong Song if (put_user(zero, ubuf + input_len - 1))
482041bdc4b4SYonghong Song return -EFAULT;
482141bdc4b4SYonghong Song }
482241bdc4b4SYonghong Song }
482341bdc4b4SYonghong Song
482441bdc4b4SYonghong Song if (put_user(prog_id, &uattr->task_fd_query.prog_id) ||
482541bdc4b4SYonghong Song put_user(fd_type, &uattr->task_fd_query.fd_type) ||
482641bdc4b4SYonghong Song put_user(probe_offset, &uattr->task_fd_query.probe_offset) ||
482741bdc4b4SYonghong Song put_user(probe_addr, &uattr->task_fd_query.probe_addr))
482841bdc4b4SYonghong Song return -EFAULT;
482941bdc4b4SYonghong Song
483041bdc4b4SYonghong Song return err;
483141bdc4b4SYonghong Song }
483241bdc4b4SYonghong Song
483341bdc4b4SYonghong Song #define BPF_TASK_FD_QUERY_LAST_FIELD task_fd_query.probe_addr
483441bdc4b4SYonghong Song
bpf_task_fd_query(const union bpf_attr * attr,union bpf_attr __user * uattr)483541bdc4b4SYonghong Song static int bpf_task_fd_query(const union bpf_attr *attr,
483641bdc4b4SYonghong Song union bpf_attr __user *uattr)
483741bdc4b4SYonghong Song {
483841bdc4b4SYonghong Song pid_t pid = attr->task_fd_query.pid;
483941bdc4b4SYonghong Song u32 fd = attr->task_fd_query.fd;
484041bdc4b4SYonghong Song const struct perf_event *event;
484141bdc4b4SYonghong Song struct task_struct *task;
484241bdc4b4SYonghong Song struct file *file;
484341bdc4b4SYonghong Song int err;
484441bdc4b4SYonghong Song
484541bdc4b4SYonghong Song if (CHECK_ATTR(BPF_TASK_FD_QUERY))
484641bdc4b4SYonghong Song return -EINVAL;
484741bdc4b4SYonghong Song
484841bdc4b4SYonghong Song if (!capable(CAP_SYS_ADMIN))
484941bdc4b4SYonghong Song return -EPERM;
485041bdc4b4SYonghong Song
485141bdc4b4SYonghong Song if (attr->task_fd_query.flags != 0)
485241bdc4b4SYonghong Song return -EINVAL;
485341bdc4b4SYonghong Song
485483c10cc3SLee Jones rcu_read_lock();
485541bdc4b4SYonghong Song task = get_pid_task(find_vpid(pid), PIDTYPE_PID);
485683c10cc3SLee Jones rcu_read_unlock();
485741bdc4b4SYonghong Song if (!task)
485841bdc4b4SYonghong Song return -ENOENT;
485941bdc4b4SYonghong Song
486041bdc4b4SYonghong Song err = 0;
4861b48845afSEric W. Biederman file = fget_task(task, fd);
4862b48845afSEric W. Biederman put_task_struct(task);
486341bdc4b4SYonghong Song if (!file)
4864b48845afSEric W. Biederman return -EBADF;
486541bdc4b4SYonghong Song
486670ed506cSAndrii Nakryiko if (file->f_op == &bpf_link_fops) {
486770ed506cSAndrii Nakryiko struct bpf_link *link = file->private_data;
486870ed506cSAndrii Nakryiko
4869a3b80e10SAndrii Nakryiko if (link->ops == &bpf_raw_tp_link_lops) {
487070ed506cSAndrii Nakryiko struct bpf_raw_tp_link *raw_tp =
487170ed506cSAndrii Nakryiko container_of(link, struct bpf_raw_tp_link, link);
487241bdc4b4SYonghong Song struct bpf_raw_event_map *btp = raw_tp->btp;
487341bdc4b4SYonghong Song
487441bdc4b4SYonghong Song err = bpf_task_fd_query_copy(attr, uattr,
487570ed506cSAndrii Nakryiko raw_tp->link.prog->aux->id,
487641bdc4b4SYonghong Song BPF_FD_TYPE_RAW_TRACEPOINT,
487741bdc4b4SYonghong Song btp->tp->name, 0, 0);
487841bdc4b4SYonghong Song goto put_file;
487941bdc4b4SYonghong Song }
488070ed506cSAndrii Nakryiko goto out_not_supp;
488170ed506cSAndrii Nakryiko }
488241bdc4b4SYonghong Song
488341bdc4b4SYonghong Song event = perf_get_event(file);
488441bdc4b4SYonghong Song if (!IS_ERR(event)) {
488541bdc4b4SYonghong Song u64 probe_offset, probe_addr;
488641bdc4b4SYonghong Song u32 prog_id, fd_type;
488741bdc4b4SYonghong Song const char *buf;
488841bdc4b4SYonghong Song
488941bdc4b4SYonghong Song err = bpf_get_perf_event_info(event, &prog_id, &fd_type,
489041bdc4b4SYonghong Song &buf, &probe_offset,
489141bdc4b4SYonghong Song &probe_addr);
489241bdc4b4SYonghong Song if (!err)
489341bdc4b4SYonghong Song err = bpf_task_fd_query_copy(attr, uattr, prog_id,
489441bdc4b4SYonghong Song fd_type, buf,
489541bdc4b4SYonghong Song probe_offset,
489641bdc4b4SYonghong Song probe_addr);
489741bdc4b4SYonghong Song goto put_file;
489841bdc4b4SYonghong Song }
489941bdc4b4SYonghong Song
490070ed506cSAndrii Nakryiko out_not_supp:
490141bdc4b4SYonghong Song err = -ENOTSUPP;
490241bdc4b4SYonghong Song put_file:
490341bdc4b4SYonghong Song fput(file);
490441bdc4b4SYonghong Song return err;
490541bdc4b4SYonghong Song }
490641bdc4b4SYonghong Song
4907cb4d03abSBrian Vazquez #define BPF_MAP_BATCH_LAST_FIELD batch.flags
4908cb4d03abSBrian Vazquez
49093af43ba4SHou Tao #define BPF_DO_BATCH(fn, ...) \
4910cb4d03abSBrian Vazquez do { \
4911cb4d03abSBrian Vazquez if (!fn) { \
4912cb4d03abSBrian Vazquez err = -ENOTSUPP; \
4913cb4d03abSBrian Vazquez goto err_put; \
4914cb4d03abSBrian Vazquez } \
49153af43ba4SHou Tao err = fn(__VA_ARGS__); \
4916cb4d03abSBrian Vazquez } while (0)
4917cb4d03abSBrian Vazquez
bpf_map_do_batch(const union bpf_attr * attr,union bpf_attr __user * uattr,int cmd)4918cb4d03abSBrian Vazquez static int bpf_map_do_batch(const union bpf_attr *attr,
4919cb4d03abSBrian Vazquez union bpf_attr __user *uattr,
4920cb4d03abSBrian Vazquez int cmd)
4921cb4d03abSBrian Vazquez {
4922353050beSDaniel Borkmann bool has_read = cmd == BPF_MAP_LOOKUP_BATCH ||
4923353050beSDaniel Borkmann cmd == BPF_MAP_LOOKUP_AND_DELETE_BATCH;
4924353050beSDaniel Borkmann bool has_write = cmd != BPF_MAP_LOOKUP_BATCH;
4925cb4d03abSBrian Vazquez struct bpf_map *map;
4926cb4d03abSBrian Vazquez int err, ufd;
4927cb4d03abSBrian Vazquez struct fd f;
4928cb4d03abSBrian Vazquez
4929cb4d03abSBrian Vazquez if (CHECK_ATTR(BPF_MAP_BATCH))
4930cb4d03abSBrian Vazquez return -EINVAL;
4931cb4d03abSBrian Vazquez
4932cb4d03abSBrian Vazquez ufd = attr->batch.map_fd;
4933cb4d03abSBrian Vazquez f = fdget(ufd);
4934cb4d03abSBrian Vazquez map = __bpf_map_get(f);
4935cb4d03abSBrian Vazquez if (IS_ERR(map))
4936cb4d03abSBrian Vazquez return PTR_ERR(map);
4937353050beSDaniel Borkmann if (has_write)
4938353050beSDaniel Borkmann bpf_map_write_active_inc(map);
4939353050beSDaniel Borkmann if (has_read && !(map_get_sys_perms(map, f) & FMODE_CAN_READ)) {
4940cb4d03abSBrian Vazquez err = -EPERM;
4941cb4d03abSBrian Vazquez goto err_put;
4942cb4d03abSBrian Vazquez }
4943353050beSDaniel Borkmann if (has_write && !(map_get_sys_perms(map, f) & FMODE_CAN_WRITE)) {
4944cb4d03abSBrian Vazquez err = -EPERM;
4945cb4d03abSBrian Vazquez goto err_put;
4946cb4d03abSBrian Vazquez }
4947cb4d03abSBrian Vazquez
4948cb4d03abSBrian Vazquez if (cmd == BPF_MAP_LOOKUP_BATCH)
49493af43ba4SHou Tao BPF_DO_BATCH(map->ops->map_lookup_batch, map, attr, uattr);
495005799638SYonghong Song else if (cmd == BPF_MAP_LOOKUP_AND_DELETE_BATCH)
49513af43ba4SHou Tao BPF_DO_BATCH(map->ops->map_lookup_and_delete_batch, map, attr, uattr);
4952aa2e93b8SBrian Vazquez else if (cmd == BPF_MAP_UPDATE_BATCH)
49533af43ba4SHou Tao BPF_DO_BATCH(map->ops->map_update_batch, map, f.file, attr, uattr);
4954aa2e93b8SBrian Vazquez else
49553af43ba4SHou Tao BPF_DO_BATCH(map->ops->map_delete_batch, map, attr, uattr);
4956cb4d03abSBrian Vazquez err_put:
4957353050beSDaniel Borkmann if (has_write)
4958353050beSDaniel Borkmann bpf_map_write_active_dec(map);
4959cb4d03abSBrian Vazquez fdput(f);
4960cb4d03abSBrian Vazquez return err;
4961cb4d03abSBrian Vazquez }
4962cb4d03abSBrian Vazquez
4963b733eeadSJiri Olsa #define BPF_LINK_CREATE_LAST_FIELD link_create.uprobe_multi.pid
link_create(union bpf_attr * attr,bpfptr_t uattr)4964af2ac3e1SAlexei Starovoitov static int link_create(union bpf_attr *attr, bpfptr_t uattr)
4965af6eea57SAndrii Nakryiko {
4966af6eea57SAndrii Nakryiko struct bpf_prog *prog;
4967af6eea57SAndrii Nakryiko int ret;
4968af6eea57SAndrii Nakryiko
4969af6eea57SAndrii Nakryiko if (CHECK_ATTR(BPF_LINK_CREATE))
4970af6eea57SAndrii Nakryiko return -EINVAL;
4971af6eea57SAndrii Nakryiko
497268b04864SKui-Feng Lee if (attr->link_create.attach_type == BPF_STRUCT_OPS)
497368b04864SKui-Feng Lee return bpf_struct_ops_link_create(attr);
497468b04864SKui-Feng Lee
49754a1e7c0cSToke Høiland-Jørgensen prog = bpf_prog_get(attr->link_create.prog_fd);
4976af6eea57SAndrii Nakryiko if (IS_ERR(prog))
4977af6eea57SAndrii Nakryiko return PTR_ERR(prog);
4978af6eea57SAndrii Nakryiko
4979af6eea57SAndrii Nakryiko ret = bpf_prog_attach_check_attach_type(prog,
4980af6eea57SAndrii Nakryiko attr->link_create.attach_type);
4981af6eea57SAndrii Nakryiko if (ret)
49824a1e7c0cSToke Høiland-Jørgensen goto out;
49834a1e7c0cSToke Høiland-Jørgensen
4984b89fbfbbSAndrii Nakryiko switch (prog->type) {
4985af6eea57SAndrii Nakryiko case BPF_PROG_TYPE_CGROUP_SKB:
4986af6eea57SAndrii Nakryiko case BPF_PROG_TYPE_CGROUP_SOCK:
4987af6eea57SAndrii Nakryiko case BPF_PROG_TYPE_CGROUP_SOCK_ADDR:
4988af6eea57SAndrii Nakryiko case BPF_PROG_TYPE_SOCK_OPS:
4989af6eea57SAndrii Nakryiko case BPF_PROG_TYPE_CGROUP_DEVICE:
4990af6eea57SAndrii Nakryiko case BPF_PROG_TYPE_CGROUP_SYSCTL:
4991af6eea57SAndrii Nakryiko case BPF_PROG_TYPE_CGROUP_SOCKOPT:
4992af6eea57SAndrii Nakryiko ret = cgroup_bpf_link_attach(attr, prog);
4993af6eea57SAndrii Nakryiko break;
4994df86ca0dSAndrii Nakryiko case BPF_PROG_TYPE_EXT:
4995df86ca0dSAndrii Nakryiko ret = bpf_tracing_prog_attach(prog,
4996df86ca0dSAndrii Nakryiko attr->link_create.target_fd,
49972fcc8241SKui-Feng Lee attr->link_create.target_btf_id,
49982fcc8241SKui-Feng Lee attr->link_create.tracing.cookie);
4999df86ca0dSAndrii Nakryiko break;
5000df86ca0dSAndrii Nakryiko case BPF_PROG_TYPE_LSM:
5001de4e05caSYonghong Song case BPF_PROG_TYPE_TRACING:
5002df86ca0dSAndrii Nakryiko if (attr->link_create.attach_type != prog->expected_attach_type) {
5003df86ca0dSAndrii Nakryiko ret = -EINVAL;
5004df86ca0dSAndrii Nakryiko goto out;
5005df86ca0dSAndrii Nakryiko }
5006df86ca0dSAndrii Nakryiko if (prog->expected_attach_type == BPF_TRACE_RAW_TP)
5007df86ca0dSAndrii Nakryiko ret = bpf_raw_tp_link_attach(prog, NULL);
5008df86ca0dSAndrii Nakryiko else if (prog->expected_attach_type == BPF_TRACE_ITER)
5009df86ca0dSAndrii Nakryiko ret = bpf_iter_link_attach(attr, uattr, prog);
501069fd337aSStanislav Fomichev else if (prog->expected_attach_type == BPF_LSM_CGROUP)
501169fd337aSStanislav Fomichev ret = cgroup_bpf_link_attach(attr, prog);
5012df86ca0dSAndrii Nakryiko else
5013df86ca0dSAndrii Nakryiko ret = bpf_tracing_prog_attach(prog,
5014df86ca0dSAndrii Nakryiko attr->link_create.target_fd,
50152fcc8241SKui-Feng Lee attr->link_create.target_btf_id,
50162fcc8241SKui-Feng Lee attr->link_create.tracing.cookie);
5017de4e05caSYonghong Song break;
50187f045a49SJakub Sitnicki case BPF_PROG_TYPE_FLOW_DISSECTOR:
5019e9ddbb77SJakub Sitnicki case BPF_PROG_TYPE_SK_LOOKUP:
50207f045a49SJakub Sitnicki ret = netns_bpf_link_create(attr, prog);
50217f045a49SJakub Sitnicki break;
5022310ad797SAndrii Nakryiko #ifdef CONFIG_NET
5023aa8d3a71SAndrii Nakryiko case BPF_PROG_TYPE_XDP:
5024aa8d3a71SAndrii Nakryiko ret = bpf_xdp_link_attach(attr, prog);
5025aa8d3a71SAndrii Nakryiko break;
5026e420bed0SDaniel Borkmann case BPF_PROG_TYPE_SCHED_CLS:
5027e420bed0SDaniel Borkmann ret = tcx_link_attach(attr, prog);
5028e420bed0SDaniel Borkmann break;
502984601d6eSFlorian Westphal case BPF_PROG_TYPE_NETFILTER:
503084601d6eSFlorian Westphal ret = bpf_nf_link_attach(attr, prog);
503184601d6eSFlorian Westphal break;
5032310ad797SAndrii Nakryiko #endif
5033b89fbfbbSAndrii Nakryiko case BPF_PROG_TYPE_PERF_EVENT:
5034b89fbfbbSAndrii Nakryiko case BPF_PROG_TYPE_TRACEPOINT:
5035b89fbfbbSAndrii Nakryiko ret = bpf_perf_link_attach(attr, prog);
5036b89fbfbbSAndrii Nakryiko break;
50370dcac272SJiri Olsa case BPF_PROG_TYPE_KPROBE:
50380dcac272SJiri Olsa if (attr->link_create.attach_type == BPF_PERF_EVENT)
50390dcac272SJiri Olsa ret = bpf_perf_link_attach(attr, prog);
504089ae89f5SJiri Olsa else if (attr->link_create.attach_type == BPF_TRACE_KPROBE_MULTI)
50410dcac272SJiri Olsa ret = bpf_kprobe_multi_link_attach(attr, prog);
504289ae89f5SJiri Olsa else if (attr->link_create.attach_type == BPF_TRACE_UPROBE_MULTI)
504389ae89f5SJiri Olsa ret = bpf_uprobe_multi_link_attach(attr, prog);
50440dcac272SJiri Olsa break;
5045af6eea57SAndrii Nakryiko default:
5046af6eea57SAndrii Nakryiko ret = -EINVAL;
5047af6eea57SAndrii Nakryiko }
5048af6eea57SAndrii Nakryiko
50494a1e7c0cSToke Høiland-Jørgensen out:
5050af6eea57SAndrii Nakryiko if (ret < 0)
5051af6eea57SAndrii Nakryiko bpf_prog_put(prog);
5052af6eea57SAndrii Nakryiko return ret;
5053af6eea57SAndrii Nakryiko }
5054af6eea57SAndrii Nakryiko
link_update_map(struct bpf_link * link,union bpf_attr * attr)5055aef56f2eSKui-Feng Lee static int link_update_map(struct bpf_link *link, union bpf_attr *attr)
5056aef56f2eSKui-Feng Lee {
5057aef56f2eSKui-Feng Lee struct bpf_map *new_map, *old_map = NULL;
5058aef56f2eSKui-Feng Lee int ret;
5059aef56f2eSKui-Feng Lee
5060aef56f2eSKui-Feng Lee new_map = bpf_map_get(attr->link_update.new_map_fd);
5061aef56f2eSKui-Feng Lee if (IS_ERR(new_map))
506255fbae05SMartin KaFai Lau return PTR_ERR(new_map);
5063aef56f2eSKui-Feng Lee
5064aef56f2eSKui-Feng Lee if (attr->link_update.flags & BPF_F_REPLACE) {
5065aef56f2eSKui-Feng Lee old_map = bpf_map_get(attr->link_update.old_map_fd);
5066aef56f2eSKui-Feng Lee if (IS_ERR(old_map)) {
506755fbae05SMartin KaFai Lau ret = PTR_ERR(old_map);
5068aef56f2eSKui-Feng Lee goto out_put;
5069aef56f2eSKui-Feng Lee }
5070aef56f2eSKui-Feng Lee } else if (attr->link_update.old_map_fd) {
5071aef56f2eSKui-Feng Lee ret = -EINVAL;
5072aef56f2eSKui-Feng Lee goto out_put;
5073aef56f2eSKui-Feng Lee }
5074aef56f2eSKui-Feng Lee
5075aef56f2eSKui-Feng Lee ret = link->ops->update_map(link, new_map, old_map);
5076aef56f2eSKui-Feng Lee
5077aef56f2eSKui-Feng Lee if (old_map)
5078aef56f2eSKui-Feng Lee bpf_map_put(old_map);
5079aef56f2eSKui-Feng Lee out_put:
5080aef56f2eSKui-Feng Lee bpf_map_put(new_map);
5081aef56f2eSKui-Feng Lee return ret;
5082aef56f2eSKui-Feng Lee }
5083aef56f2eSKui-Feng Lee
50840c991ebcSAndrii Nakryiko #define BPF_LINK_UPDATE_LAST_FIELD link_update.old_prog_fd
50850c991ebcSAndrii Nakryiko
link_update(union bpf_attr * attr)50860c991ebcSAndrii Nakryiko static int link_update(union bpf_attr *attr)
50870c991ebcSAndrii Nakryiko {
50880c991ebcSAndrii Nakryiko struct bpf_prog *old_prog = NULL, *new_prog;
50890c991ebcSAndrii Nakryiko struct bpf_link *link;
50900c991ebcSAndrii Nakryiko u32 flags;
50910c991ebcSAndrii Nakryiko int ret;
50920c991ebcSAndrii Nakryiko
50930c991ebcSAndrii Nakryiko if (CHECK_ATTR(BPF_LINK_UPDATE))
50940c991ebcSAndrii Nakryiko return -EINVAL;
50950c991ebcSAndrii Nakryiko
50960c991ebcSAndrii Nakryiko flags = attr->link_update.flags;
50970c991ebcSAndrii Nakryiko if (flags & ~BPF_F_REPLACE)
50980c991ebcSAndrii Nakryiko return -EINVAL;
50990c991ebcSAndrii Nakryiko
51000c991ebcSAndrii Nakryiko link = bpf_link_get_from_fd(attr->link_update.link_fd);
51010c991ebcSAndrii Nakryiko if (IS_ERR(link))
51020c991ebcSAndrii Nakryiko return PTR_ERR(link);
51030c991ebcSAndrii Nakryiko
5104aef56f2eSKui-Feng Lee if (link->ops->update_map) {
5105aef56f2eSKui-Feng Lee ret = link_update_map(link, attr);
5106aef56f2eSKui-Feng Lee goto out_put_link;
5107aef56f2eSKui-Feng Lee }
5108aef56f2eSKui-Feng Lee
51090c991ebcSAndrii Nakryiko new_prog = bpf_prog_get(attr->link_update.new_prog_fd);
51104adb7a4aSAndrii Nakryiko if (IS_ERR(new_prog)) {
51114adb7a4aSAndrii Nakryiko ret = PTR_ERR(new_prog);
51124adb7a4aSAndrii Nakryiko goto out_put_link;
51134adb7a4aSAndrii Nakryiko }
51140c991ebcSAndrii Nakryiko
51150c991ebcSAndrii Nakryiko if (flags & BPF_F_REPLACE) {
51160c991ebcSAndrii Nakryiko old_prog = bpf_prog_get(attr->link_update.old_prog_fd);
51170c991ebcSAndrii Nakryiko if (IS_ERR(old_prog)) {
51180c991ebcSAndrii Nakryiko ret = PTR_ERR(old_prog);
51190c991ebcSAndrii Nakryiko old_prog = NULL;
51200c991ebcSAndrii Nakryiko goto out_put_progs;
51210c991ebcSAndrii Nakryiko }
51224adb7a4aSAndrii Nakryiko } else if (attr->link_update.old_prog_fd) {
51234adb7a4aSAndrii Nakryiko ret = -EINVAL;
51244adb7a4aSAndrii Nakryiko goto out_put_progs;
51250c991ebcSAndrii Nakryiko }
51260c991ebcSAndrii Nakryiko
5127f9d04127SAndrii Nakryiko if (link->ops->update_prog)
5128f9d04127SAndrii Nakryiko ret = link->ops->update_prog(link, new_prog, old_prog);
5129f9d04127SAndrii Nakryiko else
51300c991ebcSAndrii Nakryiko ret = -EINVAL;
51310c991ebcSAndrii Nakryiko
51320c991ebcSAndrii Nakryiko out_put_progs:
51330c991ebcSAndrii Nakryiko if (old_prog)
51340c991ebcSAndrii Nakryiko bpf_prog_put(old_prog);
51350c991ebcSAndrii Nakryiko if (ret)
51360c991ebcSAndrii Nakryiko bpf_prog_put(new_prog);
51374adb7a4aSAndrii Nakryiko out_put_link:
5138ab5d47bdSSebastian Andrzej Siewior bpf_link_put_direct(link);
51390c991ebcSAndrii Nakryiko return ret;
51400c991ebcSAndrii Nakryiko }
51410c991ebcSAndrii Nakryiko
514273b11c2aSAndrii Nakryiko #define BPF_LINK_DETACH_LAST_FIELD link_detach.link_fd
514373b11c2aSAndrii Nakryiko
link_detach(union bpf_attr * attr)514473b11c2aSAndrii Nakryiko static int link_detach(union bpf_attr *attr)
514573b11c2aSAndrii Nakryiko {
514673b11c2aSAndrii Nakryiko struct bpf_link *link;
514773b11c2aSAndrii Nakryiko int ret;
514873b11c2aSAndrii Nakryiko
514973b11c2aSAndrii Nakryiko if (CHECK_ATTR(BPF_LINK_DETACH))
515073b11c2aSAndrii Nakryiko return -EINVAL;
515173b11c2aSAndrii Nakryiko
515273b11c2aSAndrii Nakryiko link = bpf_link_get_from_fd(attr->link_detach.link_fd);
515373b11c2aSAndrii Nakryiko if (IS_ERR(link))
515473b11c2aSAndrii Nakryiko return PTR_ERR(link);
515573b11c2aSAndrii Nakryiko
515673b11c2aSAndrii Nakryiko if (link->ops->detach)
515773b11c2aSAndrii Nakryiko ret = link->ops->detach(link);
515873b11c2aSAndrii Nakryiko else
515973b11c2aSAndrii Nakryiko ret = -EOPNOTSUPP;
516073b11c2aSAndrii Nakryiko
5161ab5d47bdSSebastian Andrzej Siewior bpf_link_put_direct(link);
516273b11c2aSAndrii Nakryiko return ret;
516373b11c2aSAndrii Nakryiko }
516473b11c2aSAndrii Nakryiko
bpf_link_inc_not_zero(struct bpf_link * link)5165005142b8SAlexei Starovoitov static struct bpf_link *bpf_link_inc_not_zero(struct bpf_link *link)
51662d602c8cSAndrii Nakryiko {
5167005142b8SAlexei Starovoitov return atomic64_fetch_add_unless(&link->refcnt, 1, 0) ? link : ERR_PTR(-ENOENT);
5168005142b8SAlexei Starovoitov }
5169005142b8SAlexei Starovoitov
bpf_link_by_id(u32 id)5170005142b8SAlexei Starovoitov struct bpf_link *bpf_link_by_id(u32 id)
5171005142b8SAlexei Starovoitov {
5172005142b8SAlexei Starovoitov struct bpf_link *link;
5173005142b8SAlexei Starovoitov
5174005142b8SAlexei Starovoitov if (!id)
5175005142b8SAlexei Starovoitov return ERR_PTR(-ENOENT);
5176005142b8SAlexei Starovoitov
5177005142b8SAlexei Starovoitov spin_lock_bh(&link_idr_lock);
5178005142b8SAlexei Starovoitov /* before link is "settled", ID is 0, pretend it doesn't exist yet */
5179005142b8SAlexei Starovoitov link = idr_find(&link_idr, id);
5180005142b8SAlexei Starovoitov if (link) {
5181005142b8SAlexei Starovoitov if (link->id)
5182005142b8SAlexei Starovoitov link = bpf_link_inc_not_zero(link);
5183005142b8SAlexei Starovoitov else
5184005142b8SAlexei Starovoitov link = ERR_PTR(-EAGAIN);
5185005142b8SAlexei Starovoitov } else {
5186005142b8SAlexei Starovoitov link = ERR_PTR(-ENOENT);
5187005142b8SAlexei Starovoitov }
5188005142b8SAlexei Starovoitov spin_unlock_bh(&link_idr_lock);
5189005142b8SAlexei Starovoitov return link;
51902d602c8cSAndrii Nakryiko }
51912d602c8cSAndrii Nakryiko
bpf_link_get_curr_or_next(u32 * id)51929f883612SDmitrii Dolgov struct bpf_link *bpf_link_get_curr_or_next(u32 *id)
51939f883612SDmitrii Dolgov {
51949f883612SDmitrii Dolgov struct bpf_link *link;
51959f883612SDmitrii Dolgov
51969f883612SDmitrii Dolgov spin_lock_bh(&link_idr_lock);
51979f883612SDmitrii Dolgov again:
51989f883612SDmitrii Dolgov link = idr_get_next(&link_idr, id);
51999f883612SDmitrii Dolgov if (link) {
52009f883612SDmitrii Dolgov link = bpf_link_inc_not_zero(link);
52019f883612SDmitrii Dolgov if (IS_ERR(link)) {
52029f883612SDmitrii Dolgov (*id)++;
52039f883612SDmitrii Dolgov goto again;
52049f883612SDmitrii Dolgov }
52059f883612SDmitrii Dolgov }
52069f883612SDmitrii Dolgov spin_unlock_bh(&link_idr_lock);
52079f883612SDmitrii Dolgov
52089f883612SDmitrii Dolgov return link;
52099f883612SDmitrii Dolgov }
52109f883612SDmitrii Dolgov
52112d602c8cSAndrii Nakryiko #define BPF_LINK_GET_FD_BY_ID_LAST_FIELD link_id
52122d602c8cSAndrii Nakryiko
bpf_link_get_fd_by_id(const union bpf_attr * attr)52132d602c8cSAndrii Nakryiko static int bpf_link_get_fd_by_id(const union bpf_attr *attr)
52142d602c8cSAndrii Nakryiko {
52152d602c8cSAndrii Nakryiko struct bpf_link *link;
52162d602c8cSAndrii Nakryiko u32 id = attr->link_id;
5217005142b8SAlexei Starovoitov int fd;
52182d602c8cSAndrii Nakryiko
52192d602c8cSAndrii Nakryiko if (CHECK_ATTR(BPF_LINK_GET_FD_BY_ID))
52202d602c8cSAndrii Nakryiko return -EINVAL;
52212d602c8cSAndrii Nakryiko
52222d602c8cSAndrii Nakryiko if (!capable(CAP_SYS_ADMIN))
52232d602c8cSAndrii Nakryiko return -EPERM;
52242d602c8cSAndrii Nakryiko
5225005142b8SAlexei Starovoitov link = bpf_link_by_id(id);
5226005142b8SAlexei Starovoitov if (IS_ERR(link))
5227005142b8SAlexei Starovoitov return PTR_ERR(link);
52282d602c8cSAndrii Nakryiko
52292d602c8cSAndrii Nakryiko fd = bpf_link_new_fd(link);
52302d602c8cSAndrii Nakryiko if (fd < 0)
5231ab5d47bdSSebastian Andrzej Siewior bpf_link_put_direct(link);
52322d602c8cSAndrii Nakryiko
52332d602c8cSAndrii Nakryiko return fd;
52342d602c8cSAndrii Nakryiko }
52352d602c8cSAndrii Nakryiko
5236d46edd67SSong Liu DEFINE_MUTEX(bpf_stats_enabled_mutex);
5237d46edd67SSong Liu
bpf_stats_release(struct inode * inode,struct file * file)5238d46edd67SSong Liu static int bpf_stats_release(struct inode *inode, struct file *file)
5239d46edd67SSong Liu {
5240d46edd67SSong Liu mutex_lock(&bpf_stats_enabled_mutex);
5241d46edd67SSong Liu static_key_slow_dec(&bpf_stats_enabled_key.key);
5242d46edd67SSong Liu mutex_unlock(&bpf_stats_enabled_mutex);
5243d46edd67SSong Liu return 0;
5244d46edd67SSong Liu }
5245d46edd67SSong Liu
5246d46edd67SSong Liu static const struct file_operations bpf_stats_fops = {
5247d46edd67SSong Liu .release = bpf_stats_release,
5248d46edd67SSong Liu };
5249d46edd67SSong Liu
bpf_enable_runtime_stats(void)5250d46edd67SSong Liu static int bpf_enable_runtime_stats(void)
5251d46edd67SSong Liu {
5252d46edd67SSong Liu int fd;
5253d46edd67SSong Liu
5254d46edd67SSong Liu mutex_lock(&bpf_stats_enabled_mutex);
5255d46edd67SSong Liu
5256d46edd67SSong Liu /* Set a very high limit to avoid overflow */
5257d46edd67SSong Liu if (static_key_count(&bpf_stats_enabled_key.key) > INT_MAX / 2) {
5258d46edd67SSong Liu mutex_unlock(&bpf_stats_enabled_mutex);
5259d46edd67SSong Liu return -EBUSY;
5260d46edd67SSong Liu }
5261d46edd67SSong Liu
5262d46edd67SSong Liu fd = anon_inode_getfd("bpf-stats", &bpf_stats_fops, NULL, O_CLOEXEC);
5263d46edd67SSong Liu if (fd >= 0)
5264d46edd67SSong Liu static_key_slow_inc(&bpf_stats_enabled_key.key);
5265d46edd67SSong Liu
5266d46edd67SSong Liu mutex_unlock(&bpf_stats_enabled_mutex);
5267d46edd67SSong Liu return fd;
5268d46edd67SSong Liu }
5269d46edd67SSong Liu
5270d46edd67SSong Liu #define BPF_ENABLE_STATS_LAST_FIELD enable_stats.type
5271d46edd67SSong Liu
bpf_enable_stats(union bpf_attr * attr)5272d46edd67SSong Liu static int bpf_enable_stats(union bpf_attr *attr)
5273d46edd67SSong Liu {
5274d46edd67SSong Liu
5275d46edd67SSong Liu if (CHECK_ATTR(BPF_ENABLE_STATS))
5276d46edd67SSong Liu return -EINVAL;
5277d46edd67SSong Liu
5278d46edd67SSong Liu if (!capable(CAP_SYS_ADMIN))
5279d46edd67SSong Liu return -EPERM;
5280d46edd67SSong Liu
5281d46edd67SSong Liu switch (attr->enable_stats.type) {
5282d46edd67SSong Liu case BPF_STATS_RUN_TIME:
5283d46edd67SSong Liu return bpf_enable_runtime_stats();
5284d46edd67SSong Liu default:
5285d46edd67SSong Liu break;
5286d46edd67SSong Liu }
5287d46edd67SSong Liu return -EINVAL;
5288d46edd67SSong Liu }
5289d46edd67SSong Liu
5290ac51d99bSYonghong Song #define BPF_ITER_CREATE_LAST_FIELD iter_create.flags
5291ac51d99bSYonghong Song
bpf_iter_create(union bpf_attr * attr)5292ac51d99bSYonghong Song static int bpf_iter_create(union bpf_attr *attr)
5293ac51d99bSYonghong Song {
5294ac51d99bSYonghong Song struct bpf_link *link;
5295ac51d99bSYonghong Song int err;
5296ac51d99bSYonghong Song
5297ac51d99bSYonghong Song if (CHECK_ATTR(BPF_ITER_CREATE))
5298ac51d99bSYonghong Song return -EINVAL;
5299ac51d99bSYonghong Song
5300ac51d99bSYonghong Song if (attr->iter_create.flags)
5301ac51d99bSYonghong Song return -EINVAL;
5302ac51d99bSYonghong Song
5303ac51d99bSYonghong Song link = bpf_link_get_from_fd(attr->iter_create.link_fd);
5304ac51d99bSYonghong Song if (IS_ERR(link))
5305ac51d99bSYonghong Song return PTR_ERR(link);
5306ac51d99bSYonghong Song
5307ac51d99bSYonghong Song err = bpf_iter_new_fd(link);
5308ab5d47bdSSebastian Andrzej Siewior bpf_link_put_direct(link);
5309ac51d99bSYonghong Song
5310ac51d99bSYonghong Song return err;
5311ac51d99bSYonghong Song }
5312ac51d99bSYonghong Song
5313ef15314aSYiFei Zhu #define BPF_PROG_BIND_MAP_LAST_FIELD prog_bind_map.flags
5314ef15314aSYiFei Zhu
bpf_prog_bind_map(union bpf_attr * attr)5315ef15314aSYiFei Zhu static int bpf_prog_bind_map(union bpf_attr *attr)
5316ef15314aSYiFei Zhu {
5317ef15314aSYiFei Zhu struct bpf_prog *prog;
5318ef15314aSYiFei Zhu struct bpf_map *map;
5319ef15314aSYiFei Zhu struct bpf_map **used_maps_old, **used_maps_new;
5320ef15314aSYiFei Zhu int i, ret = 0;
5321ef15314aSYiFei Zhu
5322ef15314aSYiFei Zhu if (CHECK_ATTR(BPF_PROG_BIND_MAP))
5323ef15314aSYiFei Zhu return -EINVAL;
5324ef15314aSYiFei Zhu
5325ef15314aSYiFei Zhu if (attr->prog_bind_map.flags)
5326ef15314aSYiFei Zhu return -EINVAL;
5327ef15314aSYiFei Zhu
5328ef15314aSYiFei Zhu prog = bpf_prog_get(attr->prog_bind_map.prog_fd);
5329ef15314aSYiFei Zhu if (IS_ERR(prog))
5330ef15314aSYiFei Zhu return PTR_ERR(prog);
5331ef15314aSYiFei Zhu
5332ef15314aSYiFei Zhu map = bpf_map_get(attr->prog_bind_map.map_fd);
5333ef15314aSYiFei Zhu if (IS_ERR(map)) {
5334ef15314aSYiFei Zhu ret = PTR_ERR(map);
5335ef15314aSYiFei Zhu goto out_prog_put;
5336ef15314aSYiFei Zhu }
5337ef15314aSYiFei Zhu
5338ef15314aSYiFei Zhu mutex_lock(&prog->aux->used_maps_mutex);
5339ef15314aSYiFei Zhu
5340ef15314aSYiFei Zhu used_maps_old = prog->aux->used_maps;
5341ef15314aSYiFei Zhu
5342ef15314aSYiFei Zhu for (i = 0; i < prog->aux->used_map_cnt; i++)
53431028ae40SStanislav Fomichev if (used_maps_old[i] == map) {
53441028ae40SStanislav Fomichev bpf_map_put(map);
5345ef15314aSYiFei Zhu goto out_unlock;
53461028ae40SStanislav Fomichev }
5347ef15314aSYiFei Zhu
5348ef15314aSYiFei Zhu used_maps_new = kmalloc_array(prog->aux->used_map_cnt + 1,
5349ef15314aSYiFei Zhu sizeof(used_maps_new[0]),
5350ef15314aSYiFei Zhu GFP_KERNEL);
5351ef15314aSYiFei Zhu if (!used_maps_new) {
5352ef15314aSYiFei Zhu ret = -ENOMEM;
5353ef15314aSYiFei Zhu goto out_unlock;
5354ef15314aSYiFei Zhu }
5355ef15314aSYiFei Zhu
5356ef15314aSYiFei Zhu memcpy(used_maps_new, used_maps_old,
5357ef15314aSYiFei Zhu sizeof(used_maps_old[0]) * prog->aux->used_map_cnt);
5358ef15314aSYiFei Zhu used_maps_new[prog->aux->used_map_cnt] = map;
5359ef15314aSYiFei Zhu
5360ef15314aSYiFei Zhu prog->aux->used_map_cnt++;
5361ef15314aSYiFei Zhu prog->aux->used_maps = used_maps_new;
5362ef15314aSYiFei Zhu
5363ef15314aSYiFei Zhu kfree(used_maps_old);
5364ef15314aSYiFei Zhu
5365ef15314aSYiFei Zhu out_unlock:
5366ef15314aSYiFei Zhu mutex_unlock(&prog->aux->used_maps_mutex);
5367ef15314aSYiFei Zhu
5368ef15314aSYiFei Zhu if (ret)
5369ef15314aSYiFei Zhu bpf_map_put(map);
5370ef15314aSYiFei Zhu out_prog_put:
5371ef15314aSYiFei Zhu bpf_prog_put(prog);
5372ef15314aSYiFei Zhu return ret;
5373ef15314aSYiFei Zhu }
5374ef15314aSYiFei Zhu
__sys_bpf(int cmd,bpfptr_t uattr,unsigned int size)5375af2ac3e1SAlexei Starovoitov static int __sys_bpf(int cmd, bpfptr_t uattr, unsigned int size)
537699c55f7dSAlexei Starovoitov {
53778096f229SGreg Kroah-Hartman union bpf_attr attr;
537899c55f7dSAlexei Starovoitov int err;
537999c55f7dSAlexei Starovoitov
5380dcab51f1SMartin KaFai Lau err = bpf_check_uarg_tail_zero(uattr, sizeof(attr), size);
538199c55f7dSAlexei Starovoitov if (err)
538299c55f7dSAlexei Starovoitov return err;
53831e270976SMartin KaFai Lau size = min_t(u32, size, sizeof(attr));
538499c55f7dSAlexei Starovoitov
538599c55f7dSAlexei Starovoitov /* copy attributes from user space, may be less than sizeof(bpf_attr) */
53868096f229SGreg Kroah-Hartman memset(&attr, 0, sizeof(attr));
5387af2ac3e1SAlexei Starovoitov if (copy_from_bpfptr(&attr, uattr, size) != 0)
538899c55f7dSAlexei Starovoitov return -EFAULT;
538999c55f7dSAlexei Starovoitov
5390afdb09c7SChenbo Feng err = security_bpf(cmd, &attr, size);
5391afdb09c7SChenbo Feng if (err < 0)
5392afdb09c7SChenbo Feng return err;
5393afdb09c7SChenbo Feng
539499c55f7dSAlexei Starovoitov switch (cmd) {
539599c55f7dSAlexei Starovoitov case BPF_MAP_CREATE:
539699c55f7dSAlexei Starovoitov err = map_create(&attr);
539799c55f7dSAlexei Starovoitov break;
5398db20fd2bSAlexei Starovoitov case BPF_MAP_LOOKUP_ELEM:
5399db20fd2bSAlexei Starovoitov err = map_lookup_elem(&attr);
5400db20fd2bSAlexei Starovoitov break;
5401db20fd2bSAlexei Starovoitov case BPF_MAP_UPDATE_ELEM:
5402af2ac3e1SAlexei Starovoitov err = map_update_elem(&attr, uattr);
5403db20fd2bSAlexei Starovoitov break;
5404db20fd2bSAlexei Starovoitov case BPF_MAP_DELETE_ELEM:
5405b88df697SBenjamin Tissoires err = map_delete_elem(&attr, uattr);
5406db20fd2bSAlexei Starovoitov break;
5407db20fd2bSAlexei Starovoitov case BPF_MAP_GET_NEXT_KEY:
5408db20fd2bSAlexei Starovoitov err = map_get_next_key(&attr);
5409db20fd2bSAlexei Starovoitov break;
541087df15deSDaniel Borkmann case BPF_MAP_FREEZE:
541187df15deSDaniel Borkmann err = map_freeze(&attr);
541287df15deSDaniel Borkmann break;
541309756af4SAlexei Starovoitov case BPF_PROG_LOAD:
541447a71c1fSAndrii Nakryiko err = bpf_prog_load(&attr, uattr, size);
541509756af4SAlexei Starovoitov break;
5416b2197755SDaniel Borkmann case BPF_OBJ_PIN:
5417b2197755SDaniel Borkmann err = bpf_obj_pin(&attr);
5418b2197755SDaniel Borkmann break;
5419b2197755SDaniel Borkmann case BPF_OBJ_GET:
5420b2197755SDaniel Borkmann err = bpf_obj_get(&attr);
5421b2197755SDaniel Borkmann break;
5422f4324551SDaniel Mack case BPF_PROG_ATTACH:
5423f4324551SDaniel Mack err = bpf_prog_attach(&attr);
5424f4324551SDaniel Mack break;
5425f4324551SDaniel Mack case BPF_PROG_DETACH:
5426f4324551SDaniel Mack err = bpf_prog_detach(&attr);
5427f4324551SDaniel Mack break;
5428468e2f64SAlexei Starovoitov case BPF_PROG_QUERY:
5429af2ac3e1SAlexei Starovoitov err = bpf_prog_query(&attr, uattr.user);
5430468e2f64SAlexei Starovoitov break;
54311cf1cae9SAlexei Starovoitov case BPF_PROG_TEST_RUN:
5432af2ac3e1SAlexei Starovoitov err = bpf_prog_test_run(&attr, uattr.user);
54331cf1cae9SAlexei Starovoitov break;
543434ad5580SMartin KaFai Lau case BPF_PROG_GET_NEXT_ID:
5435af2ac3e1SAlexei Starovoitov err = bpf_obj_get_next_id(&attr, uattr.user,
543634ad5580SMartin KaFai Lau &prog_idr, &prog_idr_lock);
543734ad5580SMartin KaFai Lau break;
543834ad5580SMartin KaFai Lau case BPF_MAP_GET_NEXT_ID:
5439af2ac3e1SAlexei Starovoitov err = bpf_obj_get_next_id(&attr, uattr.user,
544034ad5580SMartin KaFai Lau &map_idr, &map_idr_lock);
544134ad5580SMartin KaFai Lau break;
54421b9ed84eSQuentin Monnet case BPF_BTF_GET_NEXT_ID:
5443af2ac3e1SAlexei Starovoitov err = bpf_obj_get_next_id(&attr, uattr.user,
54441b9ed84eSQuentin Monnet &btf_idr, &btf_idr_lock);
54451b9ed84eSQuentin Monnet break;
5446b16d9aa4SMartin KaFai Lau case BPF_PROG_GET_FD_BY_ID:
5447b16d9aa4SMartin KaFai Lau err = bpf_prog_get_fd_by_id(&attr);
5448b16d9aa4SMartin KaFai Lau break;
5449bd5f5f4eSMartin KaFai Lau case BPF_MAP_GET_FD_BY_ID:
5450bd5f5f4eSMartin KaFai Lau err = bpf_map_get_fd_by_id(&attr);
5451bd5f5f4eSMartin KaFai Lau break;
54521e270976SMartin KaFai Lau case BPF_OBJ_GET_INFO_BY_FD:
5453af2ac3e1SAlexei Starovoitov err = bpf_obj_get_info_by_fd(&attr, uattr.user);
54541e270976SMartin KaFai Lau break;
5455c4f6699dSAlexei Starovoitov case BPF_RAW_TRACEPOINT_OPEN:
5456c4f6699dSAlexei Starovoitov err = bpf_raw_tracepoint_open(&attr);
5457c4f6699dSAlexei Starovoitov break;
5458f56a653cSMartin KaFai Lau case BPF_BTF_LOAD:
545947a71c1fSAndrii Nakryiko err = bpf_btf_load(&attr, uattr, size);
5460f56a653cSMartin KaFai Lau break;
546178958fcaSMartin KaFai Lau case BPF_BTF_GET_FD_BY_ID:
546278958fcaSMartin KaFai Lau err = bpf_btf_get_fd_by_id(&attr);
546378958fcaSMartin KaFai Lau break;
546441bdc4b4SYonghong Song case BPF_TASK_FD_QUERY:
5465af2ac3e1SAlexei Starovoitov err = bpf_task_fd_query(&attr, uattr.user);
546641bdc4b4SYonghong Song break;
5467bd513cd0SMauricio Vasquez B case BPF_MAP_LOOKUP_AND_DELETE_ELEM:
5468bd513cd0SMauricio Vasquez B err = map_lookup_and_delete_elem(&attr);
5469bd513cd0SMauricio Vasquez B break;
5470cb4d03abSBrian Vazquez case BPF_MAP_LOOKUP_BATCH:
5471af2ac3e1SAlexei Starovoitov err = bpf_map_do_batch(&attr, uattr.user, BPF_MAP_LOOKUP_BATCH);
5472cb4d03abSBrian Vazquez break;
547305799638SYonghong Song case BPF_MAP_LOOKUP_AND_DELETE_BATCH:
5474af2ac3e1SAlexei Starovoitov err = bpf_map_do_batch(&attr, uattr.user,
547505799638SYonghong Song BPF_MAP_LOOKUP_AND_DELETE_BATCH);
547605799638SYonghong Song break;
5477aa2e93b8SBrian Vazquez case BPF_MAP_UPDATE_BATCH:
5478af2ac3e1SAlexei Starovoitov err = bpf_map_do_batch(&attr, uattr.user, BPF_MAP_UPDATE_BATCH);
5479aa2e93b8SBrian Vazquez break;
5480aa2e93b8SBrian Vazquez case BPF_MAP_DELETE_BATCH:
5481af2ac3e1SAlexei Starovoitov err = bpf_map_do_batch(&attr, uattr.user, BPF_MAP_DELETE_BATCH);
5482aa2e93b8SBrian Vazquez break;
5483af6eea57SAndrii Nakryiko case BPF_LINK_CREATE:
5484af2ac3e1SAlexei Starovoitov err = link_create(&attr, uattr);
5485af6eea57SAndrii Nakryiko break;
54860c991ebcSAndrii Nakryiko case BPF_LINK_UPDATE:
54870c991ebcSAndrii Nakryiko err = link_update(&attr);
54880c991ebcSAndrii Nakryiko break;
54892d602c8cSAndrii Nakryiko case BPF_LINK_GET_FD_BY_ID:
54902d602c8cSAndrii Nakryiko err = bpf_link_get_fd_by_id(&attr);
54912d602c8cSAndrii Nakryiko break;
54922d602c8cSAndrii Nakryiko case BPF_LINK_GET_NEXT_ID:
5493af2ac3e1SAlexei Starovoitov err = bpf_obj_get_next_id(&attr, uattr.user,
54942d602c8cSAndrii Nakryiko &link_idr, &link_idr_lock);
54952d602c8cSAndrii Nakryiko break;
5496d46edd67SSong Liu case BPF_ENABLE_STATS:
5497d46edd67SSong Liu err = bpf_enable_stats(&attr);
5498d46edd67SSong Liu break;
5499ac51d99bSYonghong Song case BPF_ITER_CREATE:
5500ac51d99bSYonghong Song err = bpf_iter_create(&attr);
5501ac51d99bSYonghong Song break;
550273b11c2aSAndrii Nakryiko case BPF_LINK_DETACH:
550373b11c2aSAndrii Nakryiko err = link_detach(&attr);
550473b11c2aSAndrii Nakryiko break;
5505ef15314aSYiFei Zhu case BPF_PROG_BIND_MAP:
5506ef15314aSYiFei Zhu err = bpf_prog_bind_map(&attr);
5507ef15314aSYiFei Zhu break;
550899c55f7dSAlexei Starovoitov default:
550999c55f7dSAlexei Starovoitov err = -EINVAL;
551099c55f7dSAlexei Starovoitov break;
551199c55f7dSAlexei Starovoitov }
551299c55f7dSAlexei Starovoitov
551399c55f7dSAlexei Starovoitov return err;
551499c55f7dSAlexei Starovoitov }
551579a7f8bdSAlexei Starovoitov
SYSCALL_DEFINE3(bpf,int,cmd,union bpf_attr __user *,uattr,unsigned int,size)5516af2ac3e1SAlexei Starovoitov SYSCALL_DEFINE3(bpf, int, cmd, union bpf_attr __user *, uattr, unsigned int, size)
5517af2ac3e1SAlexei Starovoitov {
5518af2ac3e1SAlexei Starovoitov return __sys_bpf(cmd, USER_BPFPTR(uattr), size);
5519af2ac3e1SAlexei Starovoitov }
5520af2ac3e1SAlexei Starovoitov
syscall_prog_is_valid_access(int off,int size,enum bpf_access_type type,const struct bpf_prog * prog,struct bpf_insn_access_aux * info)552179a7f8bdSAlexei Starovoitov static bool syscall_prog_is_valid_access(int off, int size,
552279a7f8bdSAlexei Starovoitov enum bpf_access_type type,
552379a7f8bdSAlexei Starovoitov const struct bpf_prog *prog,
552479a7f8bdSAlexei Starovoitov struct bpf_insn_access_aux *info)
552579a7f8bdSAlexei Starovoitov {
552679a7f8bdSAlexei Starovoitov if (off < 0 || off >= U16_MAX)
552779a7f8bdSAlexei Starovoitov return false;
552879a7f8bdSAlexei Starovoitov if (off % size != 0)
552979a7f8bdSAlexei Starovoitov return false;
553079a7f8bdSAlexei Starovoitov return true;
553179a7f8bdSAlexei Starovoitov }
553279a7f8bdSAlexei Starovoitov
BPF_CALL_3(bpf_sys_bpf,int,cmd,union bpf_attr *,attr,u32,attr_size)5533b1d18a75SAlexei Starovoitov BPF_CALL_3(bpf_sys_bpf, int, cmd, union bpf_attr *, attr, u32, attr_size)
553479a7f8bdSAlexei Starovoitov {
5535af2ac3e1SAlexei Starovoitov switch (cmd) {
5536af2ac3e1SAlexei Starovoitov case BPF_MAP_CREATE:
5537b88df697SBenjamin Tissoires case BPF_MAP_DELETE_ELEM:
5538af2ac3e1SAlexei Starovoitov case BPF_MAP_UPDATE_ELEM:
5539af2ac3e1SAlexei Starovoitov case BPF_MAP_FREEZE:
5540b88df697SBenjamin Tissoires case BPF_MAP_GET_FD_BY_ID:
5541af2ac3e1SAlexei Starovoitov case BPF_PROG_LOAD:
5542c571bd75SAlexei Starovoitov case BPF_BTF_LOAD:
5543b1d18a75SAlexei Starovoitov case BPF_LINK_CREATE:
5544b1d18a75SAlexei Starovoitov case BPF_RAW_TRACEPOINT_OPEN:
5545af2ac3e1SAlexei Starovoitov break;
554686f44fceSAlexei Starovoitov default:
554786f44fceSAlexei Starovoitov return -EINVAL;
554886f44fceSAlexei Starovoitov }
554986f44fceSAlexei Starovoitov return __sys_bpf(cmd, KERNEL_BPFPTR(attr), attr_size);
555086f44fceSAlexei Starovoitov }
555186f44fceSAlexei Starovoitov
55524e4588f1SAlexei Starovoitov
55534e4588f1SAlexei Starovoitov /* To shut up -Wmissing-prototypes.
55544e4588f1SAlexei Starovoitov * This function is used by the kernel light skeleton
55554e4588f1SAlexei Starovoitov * to load bpf programs when modules are loaded or during kernel boot.
55564e4588f1SAlexei Starovoitov * See tools/lib/bpf/skel_internal.h
55574e4588f1SAlexei Starovoitov */
55584e4588f1SAlexei Starovoitov int kern_sys_bpf(int cmd, union bpf_attr *attr, unsigned int size);
55594e4588f1SAlexei Starovoitov
kern_sys_bpf(int cmd,union bpf_attr * attr,unsigned int size)556086f44fceSAlexei Starovoitov int kern_sys_bpf(int cmd, union bpf_attr *attr, unsigned int size)
556186f44fceSAlexei Starovoitov {
556286f44fceSAlexei Starovoitov struct bpf_prog * __maybe_unused prog;
556386f44fceSAlexei Starovoitov struct bpf_tramp_run_ctx __maybe_unused run_ctx;
556486f44fceSAlexei Starovoitov
556586f44fceSAlexei Starovoitov switch (cmd) {
5566b1d18a75SAlexei Starovoitov #ifdef CONFIG_BPF_JIT /* __bpf_prog_enter_sleepable used by trampoline and JIT */
5567b1d18a75SAlexei Starovoitov case BPF_PROG_TEST_RUN:
5568b1d18a75SAlexei Starovoitov if (attr->test.data_in || attr->test.data_out ||
5569b1d18a75SAlexei Starovoitov attr->test.ctx_out || attr->test.duration ||
5570b1d18a75SAlexei Starovoitov attr->test.repeat || attr->test.flags)
5571b1d18a75SAlexei Starovoitov return -EINVAL;
5572b1d18a75SAlexei Starovoitov
5573b1d18a75SAlexei Starovoitov prog = bpf_prog_get_type(attr->test.prog_fd, BPF_PROG_TYPE_SYSCALL);
5574b1d18a75SAlexei Starovoitov if (IS_ERR(prog))
5575b1d18a75SAlexei Starovoitov return PTR_ERR(prog);
5576b1d18a75SAlexei Starovoitov
5577b1d18a75SAlexei Starovoitov if (attr->test.ctx_size_in < prog->aux->max_ctx_offset ||
5578b1d18a75SAlexei Starovoitov attr->test.ctx_size_in > U16_MAX) {
5579b1d18a75SAlexei Starovoitov bpf_prog_put(prog);
5580b1d18a75SAlexei Starovoitov return -EINVAL;
5581b1d18a75SAlexei Starovoitov }
5582b1d18a75SAlexei Starovoitov
5583e384c7b7SKui-Feng Lee run_ctx.bpf_cookie = 0;
5584271de525SMartin KaFai Lau if (!__bpf_prog_enter_sleepable_recur(prog, &run_ctx)) {
5585b1d18a75SAlexei Starovoitov /* recursion detected */
55867645629fSSebastian Andrzej Siewior __bpf_prog_exit_sleepable_recur(prog, 0, &run_ctx);
5587b1d18a75SAlexei Starovoitov bpf_prog_put(prog);
5588b1d18a75SAlexei Starovoitov return -EBUSY;
5589b1d18a75SAlexei Starovoitov }
5590b1d18a75SAlexei Starovoitov attr->test.retval = bpf_prog_run(prog, (void *) (long) attr->test.ctx_in);
5591271de525SMartin KaFai Lau __bpf_prog_exit_sleepable_recur(prog, 0 /* bpf_prog_run does runtime stats */,
5592271de525SMartin KaFai Lau &run_ctx);
5593b1d18a75SAlexei Starovoitov bpf_prog_put(prog);
5594b1d18a75SAlexei Starovoitov return 0;
5595b1d18a75SAlexei Starovoitov #endif
5596af2ac3e1SAlexei Starovoitov default:
559786f44fceSAlexei Starovoitov return ____bpf_sys_bpf(cmd, attr, size);
559879a7f8bdSAlexei Starovoitov }
5599af2ac3e1SAlexei Starovoitov }
560086f44fceSAlexei Starovoitov EXPORT_SYMBOL(kern_sys_bpf);
560179a7f8bdSAlexei Starovoitov
56023a2daa72SPu Lehui static const struct bpf_func_proto bpf_sys_bpf_proto = {
560379a7f8bdSAlexei Starovoitov .func = bpf_sys_bpf,
560479a7f8bdSAlexei Starovoitov .gpl_only = false,
560579a7f8bdSAlexei Starovoitov .ret_type = RET_INTEGER,
560679a7f8bdSAlexei Starovoitov .arg1_type = ARG_ANYTHING,
5607216e3cd2SHao Luo .arg2_type = ARG_PTR_TO_MEM | MEM_RDONLY,
560879a7f8bdSAlexei Starovoitov .arg3_type = ARG_CONST_SIZE,
560979a7f8bdSAlexei Starovoitov };
561079a7f8bdSAlexei Starovoitov
561179a7f8bdSAlexei Starovoitov const struct bpf_func_proto * __weak
tracing_prog_func_proto(enum bpf_func_id func_id,const struct bpf_prog * prog)561279a7f8bdSAlexei Starovoitov tracing_prog_func_proto(enum bpf_func_id func_id, const struct bpf_prog *prog)
561379a7f8bdSAlexei Starovoitov {
561479a7f8bdSAlexei Starovoitov return bpf_base_func_proto(func_id);
561579a7f8bdSAlexei Starovoitov }
561679a7f8bdSAlexei Starovoitov
BPF_CALL_1(bpf_sys_close,u32,fd)56173abea089SAlexei Starovoitov BPF_CALL_1(bpf_sys_close, u32, fd)
56183abea089SAlexei Starovoitov {
56193abea089SAlexei Starovoitov /* When bpf program calls this helper there should not be
56203abea089SAlexei Starovoitov * an fdget() without matching completed fdput().
56213abea089SAlexei Starovoitov * This helper is allowed in the following callchain only:
56223abea089SAlexei Starovoitov * sys_bpf->prog_test_run->bpf_prog->bpf_sys_close
56233abea089SAlexei Starovoitov */
56243abea089SAlexei Starovoitov return close_fd(fd);
56253abea089SAlexei Starovoitov }
56263abea089SAlexei Starovoitov
56273a2daa72SPu Lehui static const struct bpf_func_proto bpf_sys_close_proto = {
56283abea089SAlexei Starovoitov .func = bpf_sys_close,
56293abea089SAlexei Starovoitov .gpl_only = false,
56303abea089SAlexei Starovoitov .ret_type = RET_INTEGER,
56313abea089SAlexei Starovoitov .arg1_type = ARG_ANYTHING,
56323abea089SAlexei Starovoitov };
56333abea089SAlexei Starovoitov
BPF_CALL_4(bpf_kallsyms_lookup_name,const char *,name,int,name_sz,int,flags,u64 *,res)5634d6aef08aSKumar Kartikeya Dwivedi BPF_CALL_4(bpf_kallsyms_lookup_name, const char *, name, int, name_sz, int, flags, u64 *, res)
5635d6aef08aSKumar Kartikeya Dwivedi {
5636d6aef08aSKumar Kartikeya Dwivedi if (flags)
5637d6aef08aSKumar Kartikeya Dwivedi return -EINVAL;
5638d6aef08aSKumar Kartikeya Dwivedi
5639d6aef08aSKumar Kartikeya Dwivedi if (name_sz <= 1 || name[name_sz - 1])
5640d6aef08aSKumar Kartikeya Dwivedi return -EINVAL;
5641d6aef08aSKumar Kartikeya Dwivedi
5642d6aef08aSKumar Kartikeya Dwivedi if (!bpf_dump_raw_ok(current_cred()))
5643d6aef08aSKumar Kartikeya Dwivedi return -EPERM;
5644d6aef08aSKumar Kartikeya Dwivedi
5645d6aef08aSKumar Kartikeya Dwivedi *res = kallsyms_lookup_name(name);
5646d6aef08aSKumar Kartikeya Dwivedi return *res ? 0 : -ENOENT;
5647d6aef08aSKumar Kartikeya Dwivedi }
5648d6aef08aSKumar Kartikeya Dwivedi
5649dc368e1cSJoanne Koong static const struct bpf_func_proto bpf_kallsyms_lookup_name_proto = {
5650d6aef08aSKumar Kartikeya Dwivedi .func = bpf_kallsyms_lookup_name,
5651d6aef08aSKumar Kartikeya Dwivedi .gpl_only = false,
5652d6aef08aSKumar Kartikeya Dwivedi .ret_type = RET_INTEGER,
5653d6aef08aSKumar Kartikeya Dwivedi .arg1_type = ARG_PTR_TO_MEM,
5654d4efb170SKumar Kartikeya Dwivedi .arg2_type = ARG_CONST_SIZE_OR_ZERO,
5655d6aef08aSKumar Kartikeya Dwivedi .arg3_type = ARG_ANYTHING,
5656d6aef08aSKumar Kartikeya Dwivedi .arg4_type = ARG_PTR_TO_LONG,
5657d6aef08aSKumar Kartikeya Dwivedi };
5658d6aef08aSKumar Kartikeya Dwivedi
565979a7f8bdSAlexei Starovoitov static const struct bpf_func_proto *
syscall_prog_func_proto(enum bpf_func_id func_id,const struct bpf_prog * prog)566079a7f8bdSAlexei Starovoitov syscall_prog_func_proto(enum bpf_func_id func_id, const struct bpf_prog *prog)
566179a7f8bdSAlexei Starovoitov {
566279a7f8bdSAlexei Starovoitov switch (func_id) {
566379a7f8bdSAlexei Starovoitov case BPF_FUNC_sys_bpf:
566414b20b78SYiFei Zhu return !perfmon_capable() ? NULL : &bpf_sys_bpf_proto;
56653d78417bSAlexei Starovoitov case BPF_FUNC_btf_find_by_name_kind:
56663d78417bSAlexei Starovoitov return &bpf_btf_find_by_name_kind_proto;
56673abea089SAlexei Starovoitov case BPF_FUNC_sys_close:
56683abea089SAlexei Starovoitov return &bpf_sys_close_proto;
5669d6aef08aSKumar Kartikeya Dwivedi case BPF_FUNC_kallsyms_lookup_name:
5670d6aef08aSKumar Kartikeya Dwivedi return &bpf_kallsyms_lookup_name_proto;
567179a7f8bdSAlexei Starovoitov default:
567279a7f8bdSAlexei Starovoitov return tracing_prog_func_proto(func_id, prog);
567379a7f8bdSAlexei Starovoitov }
567479a7f8bdSAlexei Starovoitov }
567579a7f8bdSAlexei Starovoitov
567679a7f8bdSAlexei Starovoitov const struct bpf_verifier_ops bpf_syscall_verifier_ops = {
567779a7f8bdSAlexei Starovoitov .get_func_proto = syscall_prog_func_proto,
567879a7f8bdSAlexei Starovoitov .is_valid_access = syscall_prog_is_valid_access,
567979a7f8bdSAlexei Starovoitov };
568079a7f8bdSAlexei Starovoitov
568179a7f8bdSAlexei Starovoitov const struct bpf_prog_ops bpf_syscall_prog_ops = {
568279a7f8bdSAlexei Starovoitov .test_run = bpf_prog_test_run_syscall,
568379a7f8bdSAlexei Starovoitov };
56842900005eSYan Zhu
56852900005eSYan Zhu #ifdef CONFIG_SYSCTL
bpf_stats_handler(struct ctl_table * table,int write,void * buffer,size_t * lenp,loff_t * ppos)56862900005eSYan Zhu static int bpf_stats_handler(struct ctl_table *table, int write,
56872900005eSYan Zhu void *buffer, size_t *lenp, loff_t *ppos)
56882900005eSYan Zhu {
56892900005eSYan Zhu struct static_key *key = (struct static_key *)table->data;
56902900005eSYan Zhu static int saved_val;
56912900005eSYan Zhu int val, ret;
56922900005eSYan Zhu struct ctl_table tmp = {
56932900005eSYan Zhu .data = &val,
56942900005eSYan Zhu .maxlen = sizeof(val),
56952900005eSYan Zhu .mode = table->mode,
56962900005eSYan Zhu .extra1 = SYSCTL_ZERO,
56972900005eSYan Zhu .extra2 = SYSCTL_ONE,
56982900005eSYan Zhu };
56992900005eSYan Zhu
57002900005eSYan Zhu if (write && !capable(CAP_SYS_ADMIN))
57012900005eSYan Zhu return -EPERM;
57022900005eSYan Zhu
57032900005eSYan Zhu mutex_lock(&bpf_stats_enabled_mutex);
57042900005eSYan Zhu val = saved_val;
57052900005eSYan Zhu ret = proc_dointvec_minmax(&tmp, write, buffer, lenp, ppos);
57062900005eSYan Zhu if (write && !ret && val != saved_val) {
57072900005eSYan Zhu if (val)
57082900005eSYan Zhu static_key_slow_inc(key);
57092900005eSYan Zhu else
57102900005eSYan Zhu static_key_slow_dec(key);
57112900005eSYan Zhu saved_val = val;
57122900005eSYan Zhu }
57132900005eSYan Zhu mutex_unlock(&bpf_stats_enabled_mutex);
57142900005eSYan Zhu return ret;
57152900005eSYan Zhu }
57162900005eSYan Zhu
unpriv_ebpf_notify(int new_state)57172900005eSYan Zhu void __weak unpriv_ebpf_notify(int new_state)
57182900005eSYan Zhu {
57192900005eSYan Zhu }
57202900005eSYan Zhu
bpf_unpriv_handler(struct ctl_table * table,int write,void * buffer,size_t * lenp,loff_t * ppos)57212900005eSYan Zhu static int bpf_unpriv_handler(struct ctl_table *table, int write,
57222900005eSYan Zhu void *buffer, size_t *lenp, loff_t *ppos)
57232900005eSYan Zhu {
57242900005eSYan Zhu int ret, unpriv_enable = *(int *)table->data;
57252900005eSYan Zhu bool locked_state = unpriv_enable == 1;
57262900005eSYan Zhu struct ctl_table tmp = *table;
57272900005eSYan Zhu
57282900005eSYan Zhu if (write && !capable(CAP_SYS_ADMIN))
57292900005eSYan Zhu return -EPERM;
57302900005eSYan Zhu
57312900005eSYan Zhu tmp.data = &unpriv_enable;
57322900005eSYan Zhu ret = proc_dointvec_minmax(&tmp, write, buffer, lenp, ppos);
57332900005eSYan Zhu if (write && !ret) {
57342900005eSYan Zhu if (locked_state && unpriv_enable != 1)
57352900005eSYan Zhu return -EPERM;
57362900005eSYan Zhu *(int *)table->data = unpriv_enable;
57372900005eSYan Zhu }
57382900005eSYan Zhu
5739fedf9920SKui-Feng Lee if (write)
57402900005eSYan Zhu unpriv_ebpf_notify(unpriv_enable);
57412900005eSYan Zhu
57422900005eSYan Zhu return ret;
57432900005eSYan Zhu }
57442900005eSYan Zhu
57452900005eSYan Zhu static struct ctl_table bpf_syscall_table[] = {
57462900005eSYan Zhu {
57472900005eSYan Zhu .procname = "unprivileged_bpf_disabled",
57482900005eSYan Zhu .data = &sysctl_unprivileged_bpf_disabled,
57492900005eSYan Zhu .maxlen = sizeof(sysctl_unprivileged_bpf_disabled),
57502900005eSYan Zhu .mode = 0644,
57512900005eSYan Zhu .proc_handler = bpf_unpriv_handler,
57522900005eSYan Zhu .extra1 = SYSCTL_ZERO,
57532900005eSYan Zhu .extra2 = SYSCTL_TWO,
57542900005eSYan Zhu },
57552900005eSYan Zhu {
57562900005eSYan Zhu .procname = "bpf_stats_enabled",
57572900005eSYan Zhu .data = &bpf_stats_enabled_key.key,
57582900005eSYan Zhu .mode = 0644,
57592900005eSYan Zhu .proc_handler = bpf_stats_handler,
57602900005eSYan Zhu },
57612900005eSYan Zhu { }
57622900005eSYan Zhu };
57632900005eSYan Zhu
bpf_syscall_sysctl_init(void)57642900005eSYan Zhu static int __init bpf_syscall_sysctl_init(void)
57652900005eSYan Zhu {
57662900005eSYan Zhu register_sysctl_init("kernel", bpf_syscall_table);
57672900005eSYan Zhu return 0;
57682900005eSYan Zhu }
57692900005eSYan Zhu late_initcall(bpf_syscall_sysctl_init);
57702900005eSYan Zhu #endif /* CONFIG_SYSCTL */
5771