16086d29dSYonghong Song // SPDX-License-Identifier: GPL-2.0-only
26086d29dSYonghong Song /* Copyright (c) 2020 Facebook */
36086d29dSYonghong Song #include <linux/bpf.h>
46086d29dSYonghong Song #include <linux/fs.h>
56086d29dSYonghong Song #include <linux/filter.h>
66086d29dSYonghong Song #include <linux/kernel.h>
7951cf368SYonghong Song #include <linux/btf_ids.h>
86086d29dSYonghong Song
96086d29dSYonghong Song struct bpf_iter_seq_map_info {
103f9969f2SYonghong Song u32 map_id;
116086d29dSYonghong Song };
126086d29dSYonghong Song
bpf_map_seq_start(struct seq_file * seq,loff_t * pos)136086d29dSYonghong Song static void *bpf_map_seq_start(struct seq_file *seq, loff_t *pos)
146086d29dSYonghong Song {
156086d29dSYonghong Song struct bpf_iter_seq_map_info *info = seq->private;
166086d29dSYonghong Song struct bpf_map *map;
176086d29dSYonghong Song
183f9969f2SYonghong Song map = bpf_map_get_curr_or_next(&info->map_id);
196086d29dSYonghong Song if (!map)
206086d29dSYonghong Song return NULL;
216086d29dSYonghong Song
223f9969f2SYonghong Song if (*pos == 0)
236086d29dSYonghong Song ++*pos;
246086d29dSYonghong Song return map;
256086d29dSYonghong Song }
266086d29dSYonghong Song
bpf_map_seq_next(struct seq_file * seq,void * v,loff_t * pos)276086d29dSYonghong Song static void *bpf_map_seq_next(struct seq_file *seq, void *v, loff_t *pos)
286086d29dSYonghong Song {
296086d29dSYonghong Song struct bpf_iter_seq_map_info *info = seq->private;
306086d29dSYonghong Song
316086d29dSYonghong Song ++*pos;
323f9969f2SYonghong Song ++info->map_id;
336086d29dSYonghong Song bpf_map_put((struct bpf_map *)v);
343f9969f2SYonghong Song return bpf_map_get_curr_or_next(&info->map_id);
356086d29dSYonghong Song }
366086d29dSYonghong Song
376086d29dSYonghong Song struct bpf_iter__bpf_map {
386086d29dSYonghong Song __bpf_md_ptr(struct bpf_iter_meta *, meta);
396086d29dSYonghong Song __bpf_md_ptr(struct bpf_map *, map);
406086d29dSYonghong Song };
416086d29dSYonghong Song
DEFINE_BPF_ITER_FUNC(bpf_map,struct bpf_iter_meta * meta,struct bpf_map * map)426086d29dSYonghong Song DEFINE_BPF_ITER_FUNC(bpf_map, struct bpf_iter_meta *meta, struct bpf_map *map)
436086d29dSYonghong Song
446086d29dSYonghong Song static int __bpf_map_seq_show(struct seq_file *seq, void *v, bool in_stop)
456086d29dSYonghong Song {
466086d29dSYonghong Song struct bpf_iter__bpf_map ctx;
476086d29dSYonghong Song struct bpf_iter_meta meta;
486086d29dSYonghong Song struct bpf_prog *prog;
496086d29dSYonghong Song int ret = 0;
506086d29dSYonghong Song
516086d29dSYonghong Song ctx.meta = &meta;
526086d29dSYonghong Song ctx.map = v;
536086d29dSYonghong Song meta.seq = seq;
546086d29dSYonghong Song prog = bpf_iter_get_info(&meta, in_stop);
556086d29dSYonghong Song if (prog)
566086d29dSYonghong Song ret = bpf_iter_run_prog(prog, &ctx);
576086d29dSYonghong Song
586086d29dSYonghong Song return ret;
596086d29dSYonghong Song }
606086d29dSYonghong Song
bpf_map_seq_show(struct seq_file * seq,void * v)616086d29dSYonghong Song static int bpf_map_seq_show(struct seq_file *seq, void *v)
626086d29dSYonghong Song {
636086d29dSYonghong Song return __bpf_map_seq_show(seq, v, false);
646086d29dSYonghong Song }
656086d29dSYonghong Song
bpf_map_seq_stop(struct seq_file * seq,void * v)666086d29dSYonghong Song static void bpf_map_seq_stop(struct seq_file *seq, void *v)
676086d29dSYonghong Song {
686086d29dSYonghong Song if (!v)
696086d29dSYonghong Song (void)__bpf_map_seq_show(seq, v, true);
706086d29dSYonghong Song else
716086d29dSYonghong Song bpf_map_put((struct bpf_map *)v);
726086d29dSYonghong Song }
736086d29dSYonghong Song
746086d29dSYonghong Song static const struct seq_operations bpf_map_seq_ops = {
756086d29dSYonghong Song .start = bpf_map_seq_start,
766086d29dSYonghong Song .next = bpf_map_seq_next,
776086d29dSYonghong Song .stop = bpf_map_seq_stop,
786086d29dSYonghong Song .show = bpf_map_seq_show,
796086d29dSYonghong Song };
806086d29dSYonghong Song
815ba190c2SAnton Protopopov BTF_ID_LIST_GLOBAL_SINGLE(btf_bpf_map_id, struct, bpf_map)
82951cf368SYonghong Song
8314fc6bd6SYonghong Song static const struct bpf_iter_seq_info bpf_map_seq_info = {
846086d29dSYonghong Song .seq_ops = &bpf_map_seq_ops,
856086d29dSYonghong Song .init_seq_private = NULL,
866086d29dSYonghong Song .fini_seq_private = NULL,
876086d29dSYonghong Song .seq_priv_size = sizeof(struct bpf_iter_seq_map_info),
8814fc6bd6SYonghong Song };
8914fc6bd6SYonghong Song
9014fc6bd6SYonghong Song static struct bpf_iter_reg bpf_map_reg_info = {
9114fc6bd6SYonghong Song .target = "bpf_map",
923c32cc1bSYonghong Song .ctx_arg_info_size = 1,
933c32cc1bSYonghong Song .ctx_arg_info = {
943c32cc1bSYonghong Song { offsetof(struct bpf_iter__bpf_map, map),
95803370d3SAnton Protopopov PTR_TO_BTF_ID_OR_NULL | PTR_TRUSTED },
963c32cc1bSYonghong Song },
9714fc6bd6SYonghong Song .seq_info = &bpf_map_seq_info,
986086d29dSYonghong Song };
996086d29dSYonghong Song
bpf_iter_attach_map(struct bpf_prog * prog,union bpf_iter_link_info * linfo,struct bpf_iter_aux_info * aux)1005e7b3020SYonghong Song static int bpf_iter_attach_map(struct bpf_prog *prog,
1015e7b3020SYonghong Song union bpf_iter_link_info *linfo,
102a5cbe05aSYonghong Song struct bpf_iter_aux_info *aux)
103a5cbe05aSYonghong Song {
104d6c4503cSYonghong Song u32 key_acc_size, value_acc_size, key_size, value_size;
1055e7b3020SYonghong Song struct bpf_map *map;
106d6c4503cSYonghong Song bool is_percpu = false;
1075e7b3020SYonghong Song int err = -EINVAL;
1085e7b3020SYonghong Song
1095e7b3020SYonghong Song if (!linfo->map.map_fd)
1105e7b3020SYonghong Song return -EBADF;
1115e7b3020SYonghong Song
1125e7b3020SYonghong Song map = bpf_map_get_with_uref(linfo->map.map_fd);
1135e7b3020SYonghong Song if (IS_ERR(map))
1145e7b3020SYonghong Song return PTR_ERR(map);
115d6c4503cSYonghong Song
116d6c4503cSYonghong Song if (map->map_type == BPF_MAP_TYPE_PERCPU_HASH ||
117d3cc2ab5SYonghong Song map->map_type == BPF_MAP_TYPE_LRU_PERCPU_HASH ||
118d3cc2ab5SYonghong Song map->map_type == BPF_MAP_TYPE_PERCPU_ARRAY)
119d6c4503cSYonghong Song is_percpu = true;
120d6c4503cSYonghong Song else if (map->map_type != BPF_MAP_TYPE_HASH &&
121d3cc2ab5SYonghong Song map->map_type != BPF_MAP_TYPE_LRU_HASH &&
122d3cc2ab5SYonghong Song map->map_type != BPF_MAP_TYPE_ARRAY)
1235e7b3020SYonghong Song goto put_map;
124d6c4503cSYonghong Song
125d6c4503cSYonghong Song key_acc_size = prog->aux->max_rdonly_access;
126d6c4503cSYonghong Song value_acc_size = prog->aux->max_rdwr_access;
127d6c4503cSYonghong Song key_size = map->key_size;
128d6c4503cSYonghong Song if (!is_percpu)
129d6c4503cSYonghong Song value_size = map->value_size;
130d6c4503cSYonghong Song else
131d6c4503cSYonghong Song value_size = round_up(map->value_size, 8) * num_possible_cpus();
132d6c4503cSYonghong Song
1335e7b3020SYonghong Song if (key_acc_size > key_size || value_acc_size > value_size) {
1345e7b3020SYonghong Song err = -EACCES;
1355e7b3020SYonghong Song goto put_map;
1365e7b3020SYonghong Song }
137d6c4503cSYonghong Song
1385e7b3020SYonghong Song aux->map = map;
139d6c4503cSYonghong Song return 0;
1405e7b3020SYonghong Song
1415e7b3020SYonghong Song put_map:
1425e7b3020SYonghong Song bpf_map_put_with_uref(map);
1435e7b3020SYonghong Song return err;
1445e7b3020SYonghong Song }
1455e7b3020SYonghong Song
bpf_iter_detach_map(struct bpf_iter_aux_info * aux)1465e7b3020SYonghong Song static void bpf_iter_detach_map(struct bpf_iter_aux_info *aux)
1475e7b3020SYonghong Song {
1485e7b3020SYonghong Song bpf_map_put_with_uref(aux->map);
149a5cbe05aSYonghong Song }
150a5cbe05aSYonghong Song
bpf_iter_map_show_fdinfo(const struct bpf_iter_aux_info * aux,struct seq_file * seq)151b76f2226SYonghong Song void bpf_iter_map_show_fdinfo(const struct bpf_iter_aux_info *aux,
152b76f2226SYonghong Song struct seq_file *seq)
153b76f2226SYonghong Song {
154b76f2226SYonghong Song seq_printf(seq, "map_id:\t%u\n", aux->map->id);
155b76f2226SYonghong Song }
156b76f2226SYonghong Song
bpf_iter_map_fill_link_info(const struct bpf_iter_aux_info * aux,struct bpf_link_info * info)157b76f2226SYonghong Song int bpf_iter_map_fill_link_info(const struct bpf_iter_aux_info *aux,
158b76f2226SYonghong Song struct bpf_link_info *info)
159b76f2226SYonghong Song {
160b76f2226SYonghong Song info->iter.map.map_id = aux->map->id;
161b76f2226SYonghong Song return 0;
162b76f2226SYonghong Song }
163b76f2226SYonghong Song
164a5cbe05aSYonghong Song DEFINE_BPF_ITER_FUNC(bpf_map_elem, struct bpf_iter_meta *meta,
165a5cbe05aSYonghong Song struct bpf_map *map, void *key, void *value)
166a5cbe05aSYonghong Song
167a5cbe05aSYonghong Song static const struct bpf_iter_reg bpf_map_elem_reg_info = {
168a5cbe05aSYonghong Song .target = "bpf_map_elem",
1695e7b3020SYonghong Song .attach_target = bpf_iter_attach_map,
1705e7b3020SYonghong Song .detach_target = bpf_iter_detach_map,
171b76f2226SYonghong Song .show_fdinfo = bpf_iter_map_show_fdinfo,
172b76f2226SYonghong Song .fill_link_info = bpf_iter_map_fill_link_info,
173a5cbe05aSYonghong Song .ctx_arg_info_size = 2,
174a5cbe05aSYonghong Song .ctx_arg_info = {
175a5cbe05aSYonghong Song { offsetof(struct bpf_iter__bpf_map_elem, key),
17620b2aff4SHao Luo PTR_TO_BUF | PTR_MAYBE_NULL | MEM_RDONLY },
177a5cbe05aSYonghong Song { offsetof(struct bpf_iter__bpf_map_elem, value),
17820b2aff4SHao Luo PTR_TO_BUF | PTR_MAYBE_NULL },
179a5cbe05aSYonghong Song },
180a5cbe05aSYonghong Song };
181a5cbe05aSYonghong Song
bpf_map_iter_init(void)18215172a46SYonghong Song static int __init bpf_map_iter_init(void)
18315172a46SYonghong Song {
184a5cbe05aSYonghong Song int ret;
185a5cbe05aSYonghong Song
186951cf368SYonghong Song bpf_map_reg_info.ctx_arg_info[0].btf_id = *btf_bpf_map_id;
187a5cbe05aSYonghong Song ret = bpf_iter_reg_target(&bpf_map_reg_info);
188a5cbe05aSYonghong Song if (ret)
189a5cbe05aSYonghong Song return ret;
190a5cbe05aSYonghong Song
191a5cbe05aSYonghong Song return bpf_iter_reg_target(&bpf_map_elem_reg_info);
1926086d29dSYonghong Song }
1936086d29dSYonghong Song
1946086d29dSYonghong Song late_initcall(bpf_map_iter_init);
195803370d3SAnton Protopopov
196803370d3SAnton Protopopov __diag_push();
197803370d3SAnton Protopopov __diag_ignore_all("-Wmissing-prototypes",
198803370d3SAnton Protopopov "Global functions as their definitions will be in vmlinux BTF");
199803370d3SAnton Protopopov
bpf_map_sum_elem_count(const struct bpf_map * map)2009c298049SAnton Protopopov __bpf_kfunc s64 bpf_map_sum_elem_count(const struct bpf_map *map)
201803370d3SAnton Protopopov {
202803370d3SAnton Protopopov s64 *pcount;
203803370d3SAnton Protopopov s64 ret = 0;
204803370d3SAnton Protopopov int cpu;
205803370d3SAnton Protopopov
206803370d3SAnton Protopopov if (!map || !map->elem_count)
207803370d3SAnton Protopopov return 0;
208803370d3SAnton Protopopov
209803370d3SAnton Protopopov for_each_possible_cpu(cpu) {
210803370d3SAnton Protopopov pcount = per_cpu_ptr(map->elem_count, cpu);
211803370d3SAnton Protopopov ret += READ_ONCE(*pcount);
212803370d3SAnton Protopopov }
213803370d3SAnton Protopopov return ret;
214803370d3SAnton Protopopov }
215803370d3SAnton Protopopov
216803370d3SAnton Protopopov __diag_pop();
217803370d3SAnton Protopopov
218803370d3SAnton Protopopov BTF_SET8_START(bpf_map_iter_kfunc_ids)
219803370d3SAnton Protopopov BTF_ID_FLAGS(func, bpf_map_sum_elem_count, KF_TRUSTED_ARGS)
220803370d3SAnton Protopopov BTF_SET8_END(bpf_map_iter_kfunc_ids)
221803370d3SAnton Protopopov
222803370d3SAnton Protopopov static const struct btf_kfunc_id_set bpf_map_iter_kfunc_set = {
223803370d3SAnton Protopopov .owner = THIS_MODULE,
224803370d3SAnton Protopopov .set = &bpf_map_iter_kfunc_ids,
225803370d3SAnton Protopopov };
226803370d3SAnton Protopopov
init_subsystem(void)227803370d3SAnton Protopopov static int init_subsystem(void)
228803370d3SAnton Protopopov {
229*72829b1cSAnton Protopopov return register_btf_kfunc_id_set(BPF_PROG_TYPE_UNSPEC, &bpf_map_iter_kfunc_set);
230803370d3SAnton Protopopov }
231803370d3SAnton Protopopov late_initcall(init_subsystem);
232