1450af8d0SKP Singh /* SPDX-License-Identifier: GPL-2.0 */ 2450af8d0SKP Singh /* 3450af8d0SKP Singh * Copyright (c) 2019 Facebook 4450af8d0SKP Singh * Copyright 2020 Google LLC. 5450af8d0SKP Singh */ 6450af8d0SKP Singh 7450af8d0SKP Singh #ifndef _BPF_LOCAL_STORAGE_H 8450af8d0SKP Singh #define _BPF_LOCAL_STORAGE_H 9450af8d0SKP Singh 10450af8d0SKP Singh #include <linux/bpf.h> 11b6459415SJakub Kicinski #include <linux/filter.h> 12450af8d0SKP Singh #include <linux/rculist.h> 13450af8d0SKP Singh #include <linux/list.h> 14450af8d0SKP Singh #include <linux/hash.h> 15450af8d0SKP Singh #include <linux/types.h> 1608a7ce38SMartin KaFai Lau #include <linux/bpf_mem_alloc.h> 17450af8d0SKP Singh #include <uapi/linux/btf.h> 18450af8d0SKP Singh 19450af8d0SKP Singh #define BPF_LOCAL_STORAGE_CACHE_SIZE 16 20450af8d0SKP Singh 210fe4b381SKP Singh #define bpf_rcu_lock_held() \ 220fe4b381SKP Singh (rcu_read_lock_held() || rcu_read_lock_trace_held() || \ 230fe4b381SKP Singh rcu_read_lock_bh_held()) 24450af8d0SKP Singh struct bpf_local_storage_map_bucket { 25450af8d0SKP Singh struct hlist_head list; 26450af8d0SKP Singh raw_spinlock_t lock; 27450af8d0SKP Singh }; 28450af8d0SKP Singh 29450af8d0SKP Singh /* Thp map is not the primary owner of a bpf_local_storage_elem. 30450af8d0SKP Singh * Instead, the container object (eg. sk->sk_bpf_storage) is. 31450af8d0SKP Singh * 32450af8d0SKP Singh * The map (bpf_local_storage_map) is for two purposes 33450af8d0SKP Singh * 1. Define the size of the "local storage". It is 34450af8d0SKP Singh * the map's value_size. 35450af8d0SKP Singh * 36450af8d0SKP Singh * 2. Maintain a list to keep track of all elems such 37450af8d0SKP Singh * that they can be cleaned up during the map destruction. 38450af8d0SKP Singh * 39450af8d0SKP Singh * When a bpf local storage is being looked up for a 40450af8d0SKP Singh * particular object, the "bpf_map" pointer is actually used 41450af8d0SKP Singh * as the "key" to search in the list of elem in 42450af8d0SKP Singh * the respective bpf_local_storage owned by the object. 43450af8d0SKP Singh * 44450af8d0SKP Singh * e.g. sk->sk_bpf_storage is the mini-map with the "bpf_map" pointer 45450af8d0SKP Singh * as the searching key. 46450af8d0SKP Singh */ 47450af8d0SKP Singh struct bpf_local_storage_map { 48450af8d0SKP Singh struct bpf_map map; 49450af8d0SKP Singh /* Lookup elem does not require accessing the map. 50450af8d0SKP Singh * 51450af8d0SKP Singh * Updating/Deleting requires a bucket lock to 52450af8d0SKP Singh * link/unlink the elem from the map. Having 53450af8d0SKP Singh * multiple buckets to improve contention. 54450af8d0SKP Singh */ 55450af8d0SKP Singh struct bpf_local_storage_map_bucket *buckets; 56450af8d0SKP Singh u32 bucket_log; 57450af8d0SKP Singh u16 elem_size; 58450af8d0SKP Singh u16 cache_idx; 5908a7ce38SMartin KaFai Lau struct bpf_mem_alloc selem_ma; 60*6ae9d5e9SMartin KaFai Lau struct bpf_mem_alloc storage_ma; 6108a7ce38SMartin KaFai Lau bool bpf_ma; 62450af8d0SKP Singh }; 63450af8d0SKP Singh 64450af8d0SKP Singh struct bpf_local_storage_data { 65450af8d0SKP Singh /* smap is used as the searching key when looking up 66450af8d0SKP Singh * from the object's bpf_local_storage. 67450af8d0SKP Singh * 68450af8d0SKP Singh * Put it in the same cacheline as the data to minimize 698fb33b60SZhen Lei * the number of cachelines accessed during the cache hit case. 70450af8d0SKP Singh */ 71450af8d0SKP Singh struct bpf_local_storage_map __rcu *smap; 72450af8d0SKP Singh u8 data[] __aligned(8); 73450af8d0SKP Singh }; 74450af8d0SKP Singh 75450af8d0SKP Singh /* Linked to bpf_local_storage and bpf_local_storage_map */ 76450af8d0SKP Singh struct bpf_local_storage_elem { 77450af8d0SKP Singh struct hlist_node map_node; /* Linked to bpf_local_storage_map */ 78450af8d0SKP Singh struct hlist_node snode; /* Linked to bpf_local_storage */ 79450af8d0SKP Singh struct bpf_local_storage __rcu *local_storage; 80450af8d0SKP Singh struct rcu_head rcu; 81450af8d0SKP Singh /* 8 bytes hole */ 828fb33b60SZhen Lei /* The data is stored in another cacheline to minimize 83450af8d0SKP Singh * the number of cachelines access during a cache hit. 84450af8d0SKP Singh */ 85450af8d0SKP Singh struct bpf_local_storage_data sdata ____cacheline_aligned; 86450af8d0SKP Singh }; 87450af8d0SKP Singh 88450af8d0SKP Singh struct bpf_local_storage { 89450af8d0SKP Singh struct bpf_local_storage_data __rcu *cache[BPF_LOCAL_STORAGE_CACHE_SIZE]; 90fc6652aaSMartin KaFai Lau struct bpf_local_storage_map __rcu *smap; 91450af8d0SKP Singh struct hlist_head list; /* List of bpf_local_storage_elem */ 92450af8d0SKP Singh void *owner; /* The object that owns the above "list" of 93450af8d0SKP Singh * bpf_local_storage_elem. 94450af8d0SKP Singh */ 95450af8d0SKP Singh struct rcu_head rcu; 96450af8d0SKP Singh raw_spinlock_t lock; /* Protect adding/removing from the "list" */ 97450af8d0SKP Singh }; 98450af8d0SKP Singh 99450af8d0SKP Singh /* U16_MAX is much more than enough for sk local storage 100450af8d0SKP Singh * considering a tcp_sock is ~2k. 101450af8d0SKP Singh */ 102450af8d0SKP Singh #define BPF_LOCAL_STORAGE_MAX_VALUE_SIZE \ 103450af8d0SKP Singh min_t(u32, \ 104450af8d0SKP Singh (KMALLOC_MAX_SIZE - MAX_BPF_STACK - \ 105450af8d0SKP Singh sizeof(struct bpf_local_storage_elem)), \ 106450af8d0SKP Singh (U16_MAX - sizeof(struct bpf_local_storage_elem))) 107450af8d0SKP Singh 108450af8d0SKP Singh #define SELEM(_SDATA) \ 109450af8d0SKP Singh container_of((_SDATA), struct bpf_local_storage_elem, sdata) 110450af8d0SKP Singh #define SDATA(_SELEM) (&(_SELEM)->sdata) 111450af8d0SKP Singh 112450af8d0SKP Singh #define BPF_LOCAL_STORAGE_CACHE_SIZE 16 113450af8d0SKP Singh 114450af8d0SKP Singh struct bpf_local_storage_cache { 115450af8d0SKP Singh spinlock_t idx_lock; 116450af8d0SKP Singh u64 idx_usage_counts[BPF_LOCAL_STORAGE_CACHE_SIZE]; 117450af8d0SKP Singh }; 118450af8d0SKP Singh 119450af8d0SKP Singh #define DEFINE_BPF_STORAGE_CACHE(name) \ 120450af8d0SKP Singh static struct bpf_local_storage_cache name = { \ 121450af8d0SKP Singh .idx_lock = __SPIN_LOCK_UNLOCKED(name.idx_lock), \ 122450af8d0SKP Singh } 123450af8d0SKP Singh 124450af8d0SKP Singh /* Helper functions for bpf_local_storage */ 125450af8d0SKP Singh int bpf_local_storage_map_alloc_check(union bpf_attr *attr); 126450af8d0SKP Singh 127c83597faSYonghong Song struct bpf_map * 128c83597faSYonghong Song bpf_local_storage_map_alloc(union bpf_attr *attr, 12908a7ce38SMartin KaFai Lau struct bpf_local_storage_cache *cache, 13008a7ce38SMartin KaFai Lau bool bpf_ma); 131450af8d0SKP Singh 132450af8d0SKP Singh struct bpf_local_storage_data * 133450af8d0SKP Singh bpf_local_storage_lookup(struct bpf_local_storage *local_storage, 134450af8d0SKP Singh struct bpf_local_storage_map *smap, 135450af8d0SKP Singh bool cacheit_lockit); 136450af8d0SKP Singh 1372ffcb6fcSMartin KaFai Lau void bpf_local_storage_destroy(struct bpf_local_storage *local_storage); 138c83597faSYonghong Song 139c83597faSYonghong Song void bpf_local_storage_map_free(struct bpf_map *map, 140c83597faSYonghong Song struct bpf_local_storage_cache *cache, 141bc235cdbSSong Liu int __percpu *busy_counter); 142450af8d0SKP Singh 143450af8d0SKP Singh int bpf_local_storage_map_check_btf(const struct bpf_map *map, 144450af8d0SKP Singh const struct btf *btf, 145450af8d0SKP Singh const struct btf_type *key_type, 146450af8d0SKP Singh const struct btf_type *value_type); 147450af8d0SKP Singh 148450af8d0SKP Singh void bpf_selem_link_storage_nolock(struct bpf_local_storage *local_storage, 149450af8d0SKP Singh struct bpf_local_storage_elem *selem); 150450af8d0SKP Singh 151a47eabf2SMartin KaFai Lau void bpf_selem_unlink(struct bpf_local_storage_elem *selem, bool reuse_now); 152450af8d0SKP Singh 153450af8d0SKP Singh void bpf_selem_link_map(struct bpf_local_storage_map *smap, 154450af8d0SKP Singh struct bpf_local_storage_elem *selem); 155450af8d0SKP Singh 156450af8d0SKP Singh struct bpf_local_storage_elem * 157450af8d0SKP Singh bpf_selem_alloc(struct bpf_local_storage_map *smap, void *owner, void *value, 158b00fa38aSJoanne Koong bool charge_mem, gfp_t gfp_flags); 159450af8d0SKP Singh 160c0d63f30SMartin KaFai Lau void bpf_selem_free(struct bpf_local_storage_elem *selem, 161c0d63f30SMartin KaFai Lau struct bpf_local_storage_map *smap, 162c0d63f30SMartin KaFai Lau bool reuse_now); 163c0d63f30SMartin KaFai Lau 164450af8d0SKP Singh int 165450af8d0SKP Singh bpf_local_storage_alloc(void *owner, 166450af8d0SKP Singh struct bpf_local_storage_map *smap, 167b00fa38aSJoanne Koong struct bpf_local_storage_elem *first_selem, 168b00fa38aSJoanne Koong gfp_t gfp_flags); 169450af8d0SKP Singh 170450af8d0SKP Singh struct bpf_local_storage_data * 171450af8d0SKP Singh bpf_local_storage_update(void *owner, struct bpf_local_storage_map *smap, 172b00fa38aSJoanne Koong void *value, u64 map_flags, gfp_t gfp_flags); 173450af8d0SKP Singh 1747490b7f1SYafang Shao u64 bpf_local_storage_map_mem_usage(const struct bpf_map *map); 1750fe4b381SKP Singh 176450af8d0SKP Singh #endif /* _BPF_LOCAL_STORAGE_H */ 177