1 // SPDX-License-Identifier: GPL-2.0 2 /* 3 * Copyright (c) 2019 Facebook 4 * Copyright 2020 Google LLC. 5 */ 6 7 #include <linux/rculist.h> 8 #include <linux/list.h> 9 #include <linux/hash.h> 10 #include <linux/types.h> 11 #include <linux/spinlock.h> 12 #include <linux/bpf.h> 13 #include <linux/bpf_local_storage.h> 14 #include <net/sock.h> 15 #include <uapi/linux/sock_diag.h> 16 #include <uapi/linux/btf.h> 17 #include <linux/bpf_lsm.h> 18 #include <linux/btf_ids.h> 19 #include <linux/fdtable.h> 20 #include <linux/rcupdate_trace.h> 21 22 DEFINE_BPF_STORAGE_CACHE(inode_cache); 23 24 static struct bpf_local_storage __rcu ** 25 inode_storage_ptr(void *owner) 26 { 27 struct inode *inode = owner; 28 struct bpf_storage_blob *bsb; 29 30 bsb = bpf_inode(inode); 31 if (!bsb) 32 return NULL; 33 return &bsb->storage; 34 } 35 36 static struct bpf_local_storage_data *inode_storage_lookup(struct inode *inode, 37 struct bpf_map *map, 38 bool cacheit_lockit) 39 { 40 struct bpf_local_storage *inode_storage; 41 struct bpf_local_storage_map *smap; 42 struct bpf_storage_blob *bsb; 43 44 bsb = bpf_inode(inode); 45 if (!bsb) 46 return NULL; 47 48 inode_storage = 49 rcu_dereference_check(bsb->storage, bpf_rcu_lock_held()); 50 if (!inode_storage) 51 return NULL; 52 53 smap = (struct bpf_local_storage_map *)map; 54 return bpf_local_storage_lookup(inode_storage, smap, cacheit_lockit); 55 } 56 57 void bpf_inode_storage_free(struct inode *inode) 58 { 59 struct bpf_local_storage_elem *selem; 60 struct bpf_local_storage *local_storage; 61 bool free_inode_storage = false; 62 struct bpf_storage_blob *bsb; 63 struct hlist_node *n; 64 65 bsb = bpf_inode(inode); 66 if (!bsb) 67 return; 68 69 rcu_read_lock(); 70 71 local_storage = rcu_dereference(bsb->storage); 72 if (!local_storage) { 73 rcu_read_unlock(); 74 return; 75 } 76 77 /* Neither the bpf_prog nor the bpf-map's syscall 78 * could be modifying the local_storage->list now. 79 * Thus, no elem can be added-to or deleted-from the 80 * local_storage->list by the bpf_prog or by the bpf-map's syscall. 81 * 82 * It is racing with bpf_local_storage_map_free() alone 83 * when unlinking elem from the local_storage->list and 84 * the map's bucket->list. 85 */ 86 raw_spin_lock_bh(&local_storage->lock); 87 hlist_for_each_entry_safe(selem, n, &local_storage->list, snode) { 88 /* Always unlink from map before unlinking from 89 * local_storage. 90 */ 91 bpf_selem_unlink_map(selem); 92 free_inode_storage = bpf_selem_unlink_storage_nolock( 93 local_storage, selem, false, false); 94 } 95 raw_spin_unlock_bh(&local_storage->lock); 96 rcu_read_unlock(); 97 98 /* free_inoode_storage should always be true as long as 99 * local_storage->list was non-empty. 100 */ 101 if (free_inode_storage) 102 kfree_rcu(local_storage, rcu); 103 } 104 105 static void *bpf_fd_inode_storage_lookup_elem(struct bpf_map *map, void *key) 106 { 107 struct bpf_local_storage_data *sdata; 108 struct file *f; 109 int fd; 110 111 fd = *(int *)key; 112 f = fget_raw(fd); 113 if (!f) 114 return ERR_PTR(-EBADF); 115 116 sdata = inode_storage_lookup(f->f_inode, map, true); 117 fput(f); 118 return sdata ? sdata->data : NULL; 119 } 120 121 static int bpf_fd_inode_storage_update_elem(struct bpf_map *map, void *key, 122 void *value, u64 map_flags) 123 { 124 struct bpf_local_storage_data *sdata; 125 struct file *f; 126 int fd; 127 128 fd = *(int *)key; 129 f = fget_raw(fd); 130 if (!f) 131 return -EBADF; 132 if (!inode_storage_ptr(f->f_inode)) { 133 fput(f); 134 return -EBADF; 135 } 136 137 sdata = bpf_local_storage_update(f->f_inode, 138 (struct bpf_local_storage_map *)map, 139 value, map_flags, GFP_ATOMIC); 140 fput(f); 141 return PTR_ERR_OR_ZERO(sdata); 142 } 143 144 static int inode_storage_delete(struct inode *inode, struct bpf_map *map) 145 { 146 struct bpf_local_storage_data *sdata; 147 148 sdata = inode_storage_lookup(inode, map, false); 149 if (!sdata) 150 return -ENOENT; 151 152 bpf_selem_unlink(SELEM(sdata), true); 153 154 return 0; 155 } 156 157 static int bpf_fd_inode_storage_delete_elem(struct bpf_map *map, void *key) 158 { 159 struct file *f; 160 int fd, err; 161 162 fd = *(int *)key; 163 f = fget_raw(fd); 164 if (!f) 165 return -EBADF; 166 167 err = inode_storage_delete(f->f_inode, map); 168 fput(f); 169 return err; 170 } 171 172 /* *gfp_flags* is a hidden argument provided by the verifier */ 173 BPF_CALL_5(bpf_inode_storage_get, struct bpf_map *, map, struct inode *, inode, 174 void *, value, u64, flags, gfp_t, gfp_flags) 175 { 176 struct bpf_local_storage_data *sdata; 177 178 WARN_ON_ONCE(!bpf_rcu_lock_held()); 179 if (flags & ~(BPF_LOCAL_STORAGE_GET_F_CREATE)) 180 return (unsigned long)NULL; 181 182 /* explicitly check that the inode_storage_ptr is not 183 * NULL as inode_storage_lookup returns NULL in this case and 184 * bpf_local_storage_update expects the owner to have a 185 * valid storage pointer. 186 */ 187 if (!inode || !inode_storage_ptr(inode)) 188 return (unsigned long)NULL; 189 190 sdata = inode_storage_lookup(inode, map, true); 191 if (sdata) 192 return (unsigned long)sdata->data; 193 194 /* This helper must only called from where the inode is guaranteed 195 * to have a refcount and cannot be freed. 196 */ 197 if (flags & BPF_LOCAL_STORAGE_GET_F_CREATE) { 198 sdata = bpf_local_storage_update( 199 inode, (struct bpf_local_storage_map *)map, value, 200 BPF_NOEXIST, gfp_flags); 201 return IS_ERR(sdata) ? (unsigned long)NULL : 202 (unsigned long)sdata->data; 203 } 204 205 return (unsigned long)NULL; 206 } 207 208 BPF_CALL_2(bpf_inode_storage_delete, 209 struct bpf_map *, map, struct inode *, inode) 210 { 211 WARN_ON_ONCE(!bpf_rcu_lock_held()); 212 if (!inode) 213 return -EINVAL; 214 215 /* This helper must only called from where the inode is guaranteed 216 * to have a refcount and cannot be freed. 217 */ 218 return inode_storage_delete(inode, map); 219 } 220 221 static int notsupp_get_next_key(struct bpf_map *map, void *key, 222 void *next_key) 223 { 224 return -ENOTSUPP; 225 } 226 227 static struct bpf_map *inode_storage_map_alloc(union bpf_attr *attr) 228 { 229 struct bpf_local_storage_map *smap; 230 231 smap = bpf_local_storage_map_alloc(attr); 232 if (IS_ERR(smap)) 233 return ERR_CAST(smap); 234 235 smap->cache_idx = bpf_local_storage_cache_idx_get(&inode_cache); 236 return &smap->map; 237 } 238 239 static void inode_storage_map_free(struct bpf_map *map) 240 { 241 struct bpf_local_storage_map *smap; 242 243 smap = (struct bpf_local_storage_map *)map; 244 bpf_local_storage_cache_idx_free(&inode_cache, smap->cache_idx); 245 bpf_local_storage_map_free(smap, NULL); 246 } 247 248 static int inode_storage_map_btf_id; 249 const struct bpf_map_ops inode_storage_map_ops = { 250 .map_meta_equal = bpf_map_meta_equal, 251 .map_alloc_check = bpf_local_storage_map_alloc_check, 252 .map_alloc = inode_storage_map_alloc, 253 .map_free = inode_storage_map_free, 254 .map_get_next_key = notsupp_get_next_key, 255 .map_lookup_elem = bpf_fd_inode_storage_lookup_elem, 256 .map_update_elem = bpf_fd_inode_storage_update_elem, 257 .map_delete_elem = bpf_fd_inode_storage_delete_elem, 258 .map_check_btf = bpf_local_storage_map_check_btf, 259 .map_btf_name = "bpf_local_storage_map", 260 .map_btf_id = &inode_storage_map_btf_id, 261 .map_owner_storage_ptr = inode_storage_ptr, 262 }; 263 264 BTF_ID_LIST_SINGLE(bpf_inode_storage_btf_ids, struct, inode) 265 266 const struct bpf_func_proto bpf_inode_storage_get_proto = { 267 .func = bpf_inode_storage_get, 268 .gpl_only = false, 269 .ret_type = RET_PTR_TO_MAP_VALUE_OR_NULL, 270 .arg1_type = ARG_CONST_MAP_PTR, 271 .arg2_type = ARG_PTR_TO_BTF_ID, 272 .arg2_btf_id = &bpf_inode_storage_btf_ids[0], 273 .arg3_type = ARG_PTR_TO_MAP_VALUE_OR_NULL, 274 .arg4_type = ARG_ANYTHING, 275 }; 276 277 const struct bpf_func_proto bpf_inode_storage_delete_proto = { 278 .func = bpf_inode_storage_delete, 279 .gpl_only = false, 280 .ret_type = RET_INTEGER, 281 .arg1_type = ARG_CONST_MAP_PTR, 282 .arg2_type = ARG_PTR_TO_BTF_ID, 283 .arg2_btf_id = &bpf_inode_storage_btf_ids[0], 284 }; 285