Lines Matching full:map
26 for (i = 0; i < array->map.max_entries; i++) { in bpf_array_free_percpu()
37 for (i = 0; i < array->map.max_entries; i++) { in bpf_array_alloc_percpu()
38 ptr = bpf_map_alloc_percpu(&array->map, array->elem_size, 8, in bpf_array_alloc_percpu()
73 /* avoid overflow on round_up(map->value_size) */ in array_map_alloc_check()
76 /* percpu map value size is bound by PCPU_MIN_UNIT_SIZE */ in array_map_alloc_check()
130 /* allocate all map elements and zero-initialize them */ in array_map_alloc()
146 array->map.bypass_spec_v1 = bypass_spec_v1; in array_map_alloc()
148 /* copy mandatory map attributes */ in array_map_alloc()
149 bpf_map_init_from_attr(&array->map, attr); in array_map_alloc()
157 return &array->map; in array_map_alloc()
166 static void *array_map_lookup_elem(struct bpf_map *map, void *key) in array_map_lookup_elem() argument
168 struct bpf_array *array = container_of(map, struct bpf_array, map); in array_map_lookup_elem()
171 if (unlikely(index >= array->map.max_entries)) in array_map_lookup_elem()
177 static int array_map_direct_value_addr(const struct bpf_map *map, u64 *imm, in array_map_direct_value_addr() argument
180 struct bpf_array *array = container_of(map, struct bpf_array, map); in array_map_direct_value_addr()
182 if (map->max_entries != 1) in array_map_direct_value_addr()
184 if (off >= map->value_size) in array_map_direct_value_addr()
191 static int array_map_direct_value_meta(const struct bpf_map *map, u64 imm, in array_map_direct_value_meta() argument
194 struct bpf_array *array = container_of(map, struct bpf_array, map); in array_map_direct_value_meta()
198 if (map->max_entries != 1) in array_map_direct_value_meta()
208 static int array_map_gen_lookup(struct bpf_map *map, struct bpf_insn *insn_buf) in array_map_gen_lookup() argument
210 struct bpf_array *array = container_of(map, struct bpf_array, map); in array_map_gen_lookup()
217 if (map->map_flags & BPF_F_INNER_MAP) in array_map_gen_lookup()
222 if (!map->bypass_spec_v1) { in array_map_gen_lookup()
223 *insn++ = BPF_JMP_IMM(BPF_JGE, ret, map->max_entries, 4); in array_map_gen_lookup()
226 *insn++ = BPF_JMP_IMM(BPF_JGE, ret, map->max_entries, 3); in array_map_gen_lookup()
241 static void *percpu_array_map_lookup_elem(struct bpf_map *map, void *key) in percpu_array_map_lookup_elem() argument
243 struct bpf_array *array = container_of(map, struct bpf_array, map); in percpu_array_map_lookup_elem()
246 if (unlikely(index >= array->map.max_entries)) in percpu_array_map_lookup_elem()
252 static void *percpu_array_map_lookup_percpu_elem(struct bpf_map *map, void *key, u32 cpu) in percpu_array_map_lookup_percpu_elem() argument
254 struct bpf_array *array = container_of(map, struct bpf_array, map); in percpu_array_map_lookup_percpu_elem()
260 if (unlikely(index >= array->map.max_entries)) in percpu_array_map_lookup_percpu_elem()
266 int bpf_percpu_array_copy(struct bpf_map *map, void *key, void *value) in bpf_percpu_array_copy() argument
268 struct bpf_array *array = container_of(map, struct bpf_array, map); in bpf_percpu_array_copy()
274 if (unlikely(index >= array->map.max_entries)) in bpf_percpu_array_copy()
285 copy_map_value_long(map, value + off, per_cpu_ptr(pptr, cpu)); in bpf_percpu_array_copy()
286 check_and_init_map_value(map, value + off); in bpf_percpu_array_copy()
294 static int array_map_get_next_key(struct bpf_map *map, void *key, void *next_key) in array_map_get_next_key() argument
296 struct bpf_array *array = container_of(map, struct bpf_array, map); in array_map_get_next_key()
300 if (index >= array->map.max_entries) { in array_map_get_next_key()
305 if (index == array->map.max_entries - 1) in array_map_get_next_key()
313 static long array_map_update_elem(struct bpf_map *map, void *key, void *value, in array_map_update_elem() argument
316 struct bpf_array *array = container_of(map, struct bpf_array, map); in array_map_update_elem()
324 if (unlikely(index >= array->map.max_entries)) in array_map_update_elem()
333 !btf_record_has_field(map->record, BPF_SPIN_LOCK))) in array_map_update_elem()
336 if (array->map.map_type == BPF_MAP_TYPE_PERCPU_ARRAY) { in array_map_update_elem()
338 copy_map_value(map, val, value); in array_map_update_elem()
339 bpf_obj_free_fields(array->map.record, val); in array_map_update_elem()
344 copy_map_value_locked(map, val, value, false); in array_map_update_elem()
346 copy_map_value(map, val, value); in array_map_update_elem()
347 bpf_obj_free_fields(array->map.record, val); in array_map_update_elem()
352 int bpf_percpu_array_update(struct bpf_map *map, void *key, void *value, in bpf_percpu_array_update() argument
355 struct bpf_array *array = container_of(map, struct bpf_array, map); in bpf_percpu_array_update()
365 if (unlikely(index >= array->map.max_entries)) in bpf_percpu_array_update()
383 copy_map_value_long(map, per_cpu_ptr(pptr, cpu), value + off); in bpf_percpu_array_update()
384 bpf_obj_free_fields(array->map.record, per_cpu_ptr(pptr, cpu)); in bpf_percpu_array_update()
392 static long array_map_delete_elem(struct bpf_map *map, void *key) in array_map_delete_elem() argument
402 static void array_map_free_timers(struct bpf_map *map) in array_map_free_timers() argument
404 struct bpf_array *array = container_of(map, struct bpf_array, map); in array_map_free_timers()
408 if (!btf_record_has_field(map->record, BPF_TIMER)) in array_map_free_timers()
411 for (i = 0; i < array->map.max_entries; i++) in array_map_free_timers()
412 bpf_obj_free_timer(map->record, array_map_elem_ptr(array, i)); in array_map_free_timers()
415 /* Called when map->refcnt goes to zero, either from workqueue or from syscall */
416 static void array_map_free(struct bpf_map *map) in array_map_free() argument
418 struct bpf_array *array = container_of(map, struct bpf_array, map); in array_map_free()
421 if (!IS_ERR_OR_NULL(map->record)) { in array_map_free()
422 if (array->map.map_type == BPF_MAP_TYPE_PERCPU_ARRAY) { in array_map_free()
423 for (i = 0; i < array->map.max_entries; i++) { in array_map_free()
428 bpf_obj_free_fields(map->record, per_cpu_ptr(pptr, cpu)); in array_map_free()
433 for (i = 0; i < array->map.max_entries; i++) in array_map_free()
434 bpf_obj_free_fields(map->record, array_map_elem_ptr(array, i)); in array_map_free()
438 if (array->map.map_type == BPF_MAP_TYPE_PERCPU_ARRAY) in array_map_free()
441 if (array->map.map_flags & BPF_F_MMAPABLE) in array_map_free()
447 static void array_map_seq_show_elem(struct bpf_map *map, void *key, in array_map_seq_show_elem() argument
454 value = array_map_lookup_elem(map, key); in array_map_seq_show_elem()
460 if (map->btf_key_type_id) in array_map_seq_show_elem()
462 btf_type_seq_show(map->btf, map->btf_value_type_id, value, m); in array_map_seq_show_elem()
468 static void percpu_array_map_seq_show_elem(struct bpf_map *map, void *key, in percpu_array_map_seq_show_elem() argument
471 struct bpf_array *array = container_of(map, struct bpf_array, map); in percpu_array_map_seq_show_elem()
482 btf_type_seq_show(map->btf, map->btf_value_type_id, in percpu_array_map_seq_show_elem()
491 static int array_map_check_btf(const struct bpf_map *map, in array_map_check_btf() argument
498 /* One exception for keyless BTF: .bss/.data/.rodata map */ in array_map_check_btf()
500 if (map->map_type != BPF_MAP_TYPE_ARRAY || in array_map_check_btf()
501 map->max_entries != 1) in array_map_check_btf()
523 static int array_map_mmap(struct bpf_map *map, struct vm_area_struct *vma) in array_map_mmap() argument
525 struct bpf_array *array = container_of(map, struct bpf_array, map); in array_map_mmap()
528 if (!(map->map_flags & BPF_F_MMAPABLE)) in array_map_mmap()
532 PAGE_ALIGN((u64)array->map.max_entries * array->elem_size)) in array_map_mmap()
549 struct bpf_map *map; member
557 struct bpf_map *map = info->map; in bpf_array_map_seq_start() local
561 if (info->index >= map->max_entries) in bpf_array_map_seq_start()
566 array = container_of(map, struct bpf_array, map); in bpf_array_map_seq_start()
576 struct bpf_map *map = info->map; in bpf_array_map_seq_next() local
582 if (info->index >= map->max_entries) in bpf_array_map_seq_next()
585 array = container_of(map, struct bpf_array, map); in bpf_array_map_seq_next()
596 struct bpf_map *map = info->map; in __bpf_array_map_seq_show() local
597 struct bpf_array *array = container_of(map, struct bpf_array, map); in __bpf_array_map_seq_show()
610 ctx.map = info->map; in __bpf_array_map_seq_show()
620 copy_map_value_long(map, info->percpu_value_buf + off, in __bpf_array_map_seq_show()
622 check_and_init_map_value(map, info->percpu_value_buf + off); in __bpf_array_map_seq_show()
647 struct bpf_map *map = aux->map; in bpf_iter_init_array_map() local
648 struct bpf_array *array = container_of(map, struct bpf_array, map); in bpf_iter_init_array_map()
652 if (map->map_type == BPF_MAP_TYPE_PERCPU_ARRAY) { in bpf_iter_init_array_map()
661 /* bpf_iter_attach_map() acquires a map uref, and the uref may be in bpf_iter_init_array_map()
662 * released before or in the middle of iterating map elements, so in bpf_iter_init_array_map()
663 * acquire an extra map uref for iterator. in bpf_iter_init_array_map()
665 bpf_map_inc_with_uref(map); in bpf_iter_init_array_map()
666 seq_info->map = map; in bpf_iter_init_array_map()
674 bpf_map_put_with_uref(seq_info->map); in bpf_iter_fini_array_map()
692 static long bpf_for_each_array_elem(struct bpf_map *map, bpf_callback_t callback_fn, in bpf_for_each_array_elem() argument
704 is_percpu = map->map_type == BPF_MAP_TYPE_PERCPU_ARRAY; in bpf_for_each_array_elem()
705 array = container_of(map, struct bpf_array, map); in bpf_for_each_array_elem()
708 for (i = 0; i < map->max_entries; i++) { in bpf_for_each_array_elem()
715 ret = callback_fn((u64)(long)map, (u64)(long)&key, in bpf_for_each_array_elem()
727 static u64 array_map_mem_usage(const struct bpf_map *map) in array_map_mem_usage() argument
729 struct bpf_array *array = container_of(map, struct bpf_array, map); in array_map_mem_usage()
730 bool percpu = map->map_type == BPF_MAP_TYPE_PERCPU_ARRAY; in array_map_mem_usage()
732 u64 entries = map->max_entries; in array_map_mem_usage()
739 if (map->map_flags & BPF_F_MMAPABLE) { in array_map_mem_usage()
798 /* only file descriptors can be stored in this type of map */ in fd_array_map_alloc_check()
807 static void fd_array_map_free(struct bpf_map *map) in fd_array_map_free() argument
809 struct bpf_array *array = container_of(map, struct bpf_array, map); in fd_array_map_free()
813 for (i = 0; i < array->map.max_entries; i++) in fd_array_map_free()
819 static void *fd_array_map_lookup_elem(struct bpf_map *map, void *key) in fd_array_map_lookup_elem() argument
825 int bpf_fd_array_map_lookup_elem(struct bpf_map *map, void *key, u32 *value) in bpf_fd_array_map_lookup_elem() argument
830 if (!map->ops->map_fd_sys_lookup_elem) in bpf_fd_array_map_lookup_elem()
834 elem = array_map_lookup_elem(map, key); in bpf_fd_array_map_lookup_elem()
836 *value = map->ops->map_fd_sys_lookup_elem(ptr); in bpf_fd_array_map_lookup_elem()
845 int bpf_fd_array_map_update_elem(struct bpf_map *map, struct file *map_file, in bpf_fd_array_map_update_elem() argument
848 struct bpf_array *array = container_of(map, struct bpf_array, map); in bpf_fd_array_map_update_elem()
855 if (index >= array->map.max_entries) in bpf_fd_array_map_update_elem()
859 new_ptr = map->ops->map_fd_get_ptr(map, map_file, ufd); in bpf_fd_array_map_update_elem()
863 if (map->ops->map_poke_run) { in bpf_fd_array_map_update_elem()
866 map->ops->map_poke_run(map, index, old_ptr, new_ptr); in bpf_fd_array_map_update_elem()
873 map->ops->map_fd_put_ptr(map, old_ptr, true); in bpf_fd_array_map_update_elem()
877 static long __fd_array_map_delete_elem(struct bpf_map *map, void *key, bool need_defer) in __fd_array_map_delete_elem() argument
879 struct bpf_array *array = container_of(map, struct bpf_array, map); in __fd_array_map_delete_elem()
883 if (index >= array->map.max_entries) in __fd_array_map_delete_elem()
886 if (map->ops->map_poke_run) { in __fd_array_map_delete_elem()
889 map->ops->map_poke_run(map, index, old_ptr, NULL); in __fd_array_map_delete_elem()
896 map->ops->map_fd_put_ptr(map, old_ptr, need_defer); in __fd_array_map_delete_elem()
903 static long fd_array_map_delete_elem(struct bpf_map *map, void *key) in fd_array_map_delete_elem() argument
905 return __fd_array_map_delete_elem(map, key, true); in fd_array_map_delete_elem()
908 static void *prog_fd_array_get_ptr(struct bpf_map *map, in prog_fd_array_get_ptr() argument
916 if (!bpf_prog_map_compatible(map, prog)) { in prog_fd_array_get_ptr()
924 static void prog_fd_array_put_ptr(struct bpf_map *map, void *ptr, bool need_defer) in prog_fd_array_put_ptr() argument
935 /* decrement refcnt of all bpf_progs that are stored in this map */
936 static void bpf_fd_array_map_clear(struct bpf_map *map, bool need_defer) in bpf_fd_array_map_clear() argument
938 struct bpf_array *array = container_of(map, struct bpf_array, map); in bpf_fd_array_map_clear()
941 for (i = 0; i < array->map.max_entries; i++) in bpf_fd_array_map_clear()
942 __fd_array_map_delete_elem(map, &i, need_defer); in bpf_fd_array_map_clear()
945 static void prog_array_map_seq_show_elem(struct bpf_map *map, void *key, in prog_array_map_seq_show_elem() argument
953 elem = array_map_lookup_elem(map, key); in prog_array_map_seq_show_elem()
959 btf_type_seq_show(map->btf, map->btf_value_type_id, in prog_array_map_seq_show_elem()
973 static int prog_array_map_poke_track(struct bpf_map *map, in prog_array_map_poke_track() argument
980 aux = container_of(map, struct bpf_array, map)->aux; in prog_array_map_poke_track()
1006 static void prog_array_map_poke_untrack(struct bpf_map *map, in prog_array_map_poke_untrack() argument
1012 aux = container_of(map, struct bpf_array, map)->aux; in prog_array_map_poke_untrack()
1030 static void prog_array_map_poke_run(struct bpf_map *map, u32 key, in prog_array_map_poke_run() argument
1037 aux = container_of(map, struct bpf_array, map)->aux; in prog_array_map_poke_run()
1070 if (poke->tail_call.map != map || in prog_array_map_poke_run()
1081 struct bpf_map *map = container_of(work, struct bpf_array_aux, in prog_array_map_clear_deferred() local
1082 work)->map; in prog_array_map_clear_deferred()
1083 bpf_fd_array_map_clear(map, true); in prog_array_map_clear_deferred()
1084 bpf_map_put(map); in prog_array_map_clear_deferred()
1087 static void prog_array_map_clear(struct bpf_map *map) in prog_array_map_clear() argument
1089 struct bpf_array_aux *aux = container_of(map, struct bpf_array, in prog_array_map_clear()
1090 map)->aux; in prog_array_map_clear()
1091 bpf_map_inc(map); in prog_array_map_clear()
1098 struct bpf_map *map; in prog_array_map_alloc() local
1108 map = array_map_alloc(attr); in prog_array_map_alloc()
1109 if (IS_ERR(map)) { in prog_array_map_alloc()
1111 return map; in prog_array_map_alloc()
1114 container_of(map, struct bpf_array, map)->aux = aux; in prog_array_map_alloc()
1115 aux->map = map; in prog_array_map_alloc()
1117 return map; in prog_array_map_alloc()
1120 static void prog_array_map_free(struct bpf_map *map) in prog_array_map_free() argument
1125 aux = container_of(map, struct bpf_array, map)->aux; in prog_array_map_free()
1131 fd_array_map_free(map); in prog_array_map_free()
1187 static void *perf_event_fd_array_get_ptr(struct bpf_map *map, in perf_event_fd_array_get_ptr() argument
1213 static void perf_event_fd_array_put_ptr(struct bpf_map *map, void *ptr, bool need_defer) in perf_event_fd_array_put_ptr() argument
1219 static void perf_event_fd_array_release(struct bpf_map *map, in perf_event_fd_array_release() argument
1222 struct bpf_array *array = container_of(map, struct bpf_array, map); in perf_event_fd_array_release()
1226 if (map->map_flags & BPF_F_PRESERVE_ELEMS) in perf_event_fd_array_release()
1230 for (i = 0; i < array->map.max_entries; i++) { in perf_event_fd_array_release()
1233 __fd_array_map_delete_elem(map, &i, true); in perf_event_fd_array_release()
1238 static void perf_event_fd_array_map_free(struct bpf_map *map) in perf_event_fd_array_map_free() argument
1240 if (map->map_flags & BPF_F_PRESERVE_ELEMS) in perf_event_fd_array_map_free()
1241 bpf_fd_array_map_clear(map, false); in perf_event_fd_array_map_free()
1242 fd_array_map_free(map); in perf_event_fd_array_map_free()
1262 static void *cgroup_fd_array_get_ptr(struct bpf_map *map, in cgroup_fd_array_get_ptr() argument
1269 static void cgroup_fd_array_put_ptr(struct bpf_map *map, void *ptr, bool need_defer) in cgroup_fd_array_put_ptr() argument
1275 static void cgroup_fd_array_free(struct bpf_map *map) in cgroup_fd_array_free() argument
1277 bpf_fd_array_map_clear(map, false); in cgroup_fd_array_free()
1278 fd_array_map_free(map); in cgroup_fd_array_free()
1299 struct bpf_map *map, *inner_map_meta; in array_of_map_alloc() local
1305 map = array_map_alloc(attr); in array_of_map_alloc()
1306 if (IS_ERR(map)) { in array_of_map_alloc()
1308 return map; in array_of_map_alloc()
1311 map->inner_map_meta = inner_map_meta; in array_of_map_alloc()
1313 return map; in array_of_map_alloc()
1316 static void array_of_map_free(struct bpf_map *map) in array_of_map_free() argument
1318 /* map->inner_map_meta is only accessed by syscall which in array_of_map_free()
1321 bpf_map_meta_free(map->inner_map_meta); in array_of_map_free()
1322 bpf_fd_array_map_clear(map, false); in array_of_map_free()
1323 fd_array_map_free(map); in array_of_map_free()
1326 static void *array_of_map_lookup_elem(struct bpf_map *map, void *key) in array_of_map_lookup_elem() argument
1328 struct bpf_map **inner_map = array_map_lookup_elem(map, key); in array_of_map_lookup_elem()
1336 static int array_of_map_gen_lookup(struct bpf_map *map, in array_of_map_gen_lookup() argument
1339 struct bpf_array *array = container_of(map, struct bpf_array, map); in array_of_map_gen_lookup()
1348 if (!map->bypass_spec_v1) { in array_of_map_gen_lookup()
1349 *insn++ = BPF_JMP_IMM(BPF_JGE, ret, map->max_entries, 6); in array_of_map_gen_lookup()
1352 *insn++ = BPF_JMP_IMM(BPF_JGE, ret, map->max_entries, 5); in array_of_map_gen_lookup()