1 // SPDX-License-Identifier: GPL-2.0 2 /* 3 * Copyright (c) 2019 Facebook 4 * Copyright 2020 Google LLC. 5 */ 6 7 #include <linux/rculist.h> 8 #include <linux/list.h> 9 #include <linux/hash.h> 10 #include <linux/types.h> 11 #include <linux/spinlock.h> 12 #include <linux/bpf.h> 13 #include <linux/bpf_local_storage.h> 14 #include <net/sock.h> 15 #include <uapi/linux/sock_diag.h> 16 #include <uapi/linux/btf.h> 17 #include <linux/bpf_lsm.h> 18 #include <linux/btf_ids.h> 19 #include <linux/fdtable.h> 20 #include <linux/rcupdate_trace.h> 21 22 DEFINE_BPF_STORAGE_CACHE(inode_cache); 23 24 static struct bpf_local_storage __rcu ** 25 inode_storage_ptr(void *owner) 26 { 27 struct inode *inode = owner; 28 struct bpf_storage_blob *bsb; 29 30 bsb = bpf_inode(inode); 31 if (!bsb) 32 return NULL; 33 return &bsb->storage; 34 } 35 36 static struct bpf_local_storage_data *inode_storage_lookup(struct inode *inode, 37 struct bpf_map *map, 38 bool cacheit_lockit) 39 { 40 struct bpf_local_storage *inode_storage; 41 struct bpf_local_storage_map *smap; 42 struct bpf_storage_blob *bsb; 43 44 bsb = bpf_inode(inode); 45 if (!bsb) 46 return NULL; 47 48 inode_storage = 49 rcu_dereference_check(bsb->storage, bpf_rcu_lock_held()); 50 if (!inode_storage) 51 return NULL; 52 53 smap = (struct bpf_local_storage_map *)map; 54 return bpf_local_storage_lookup(inode_storage, smap, cacheit_lockit); 55 } 56 57 void bpf_inode_storage_free(struct inode *inode) 58 { 59 struct bpf_local_storage_elem *selem; 60 struct bpf_local_storage *local_storage; 61 bool free_inode_storage = false; 62 struct bpf_storage_blob *bsb; 63 struct hlist_node *n; 64 65 bsb = bpf_inode(inode); 66 if (!bsb) 67 return; 68 69 rcu_read_lock(); 70 71 local_storage = rcu_dereference(bsb->storage); 72 if (!local_storage) { 73 rcu_read_unlock(); 74 return; 75 } 76 77 /* Neither the bpf_prog nor the bpf-map's syscall 78 * could be modifying the local_storage->list now. 79 * Thus, no elem can be added-to or deleted-from the 80 * local_storage->list by the bpf_prog or by the bpf-map's syscall. 81 * 82 * It is racing with bpf_local_storage_map_free() alone 83 * when unlinking elem from the local_storage->list and 84 * the map's bucket->list. 85 */ 86 raw_spin_lock_bh(&local_storage->lock); 87 hlist_for_each_entry_safe(selem, n, &local_storage->list, snode) { 88 /* Always unlink from map before unlinking from 89 * local_storage. 90 */ 91 bpf_selem_unlink_map(selem); 92 free_inode_storage = bpf_selem_unlink_storage_nolock( 93 local_storage, selem, false); 94 } 95 raw_spin_unlock_bh(&local_storage->lock); 96 rcu_read_unlock(); 97 98 /* free_inoode_storage should always be true as long as 99 * local_storage->list was non-empty. 100 */ 101 if (free_inode_storage) 102 kfree_rcu(local_storage, rcu); 103 } 104 105 static void *bpf_fd_inode_storage_lookup_elem(struct bpf_map *map, void *key) 106 { 107 struct bpf_local_storage_data *sdata; 108 struct file *f; 109 int fd; 110 111 fd = *(int *)key; 112 f = fget_raw(fd); 113 if (!f) 114 return ERR_PTR(-EBADF); 115 116 sdata = inode_storage_lookup(f->f_inode, map, true); 117 fput(f); 118 return sdata ? sdata->data : NULL; 119 } 120 121 static int bpf_fd_inode_storage_update_elem(struct bpf_map *map, void *key, 122 void *value, u64 map_flags) 123 { 124 struct bpf_local_storage_data *sdata; 125 struct file *f; 126 int fd; 127 128 fd = *(int *)key; 129 f = fget_raw(fd); 130 if (!f) 131 return -EBADF; 132 if (!inode_storage_ptr(f->f_inode)) { 133 fput(f); 134 return -EBADF; 135 } 136 137 sdata = bpf_local_storage_update(f->f_inode, 138 (struct bpf_local_storage_map *)map, 139 value, map_flags); 140 fput(f); 141 return PTR_ERR_OR_ZERO(sdata); 142 } 143 144 static int inode_storage_delete(struct inode *inode, struct bpf_map *map) 145 { 146 struct bpf_local_storage_data *sdata; 147 148 sdata = inode_storage_lookup(inode, map, false); 149 if (!sdata) 150 return -ENOENT; 151 152 bpf_selem_unlink(SELEM(sdata)); 153 154 return 0; 155 } 156 157 static int bpf_fd_inode_storage_delete_elem(struct bpf_map *map, void *key) 158 { 159 struct file *f; 160 int fd, err; 161 162 fd = *(int *)key; 163 f = fget_raw(fd); 164 if (!f) 165 return -EBADF; 166 167 err = inode_storage_delete(f->f_inode, map); 168 fput(f); 169 return err; 170 } 171 172 BPF_CALL_4(bpf_inode_storage_get, struct bpf_map *, map, struct inode *, inode, 173 void *, value, u64, flags) 174 { 175 struct bpf_local_storage_data *sdata; 176 177 WARN_ON_ONCE(!bpf_rcu_lock_held()); 178 if (flags & ~(BPF_LOCAL_STORAGE_GET_F_CREATE)) 179 return (unsigned long)NULL; 180 181 /* explicitly check that the inode_storage_ptr is not 182 * NULL as inode_storage_lookup returns NULL in this case and 183 * bpf_local_storage_update expects the owner to have a 184 * valid storage pointer. 185 */ 186 if (!inode || !inode_storage_ptr(inode)) 187 return (unsigned long)NULL; 188 189 sdata = inode_storage_lookup(inode, map, true); 190 if (sdata) 191 return (unsigned long)sdata->data; 192 193 /* This helper must only called from where the inode is guaranteed 194 * to have a refcount and cannot be freed. 195 */ 196 if (flags & BPF_LOCAL_STORAGE_GET_F_CREATE) { 197 sdata = bpf_local_storage_update( 198 inode, (struct bpf_local_storage_map *)map, value, 199 BPF_NOEXIST); 200 return IS_ERR(sdata) ? (unsigned long)NULL : 201 (unsigned long)sdata->data; 202 } 203 204 return (unsigned long)NULL; 205 } 206 207 BPF_CALL_2(bpf_inode_storage_delete, 208 struct bpf_map *, map, struct inode *, inode) 209 { 210 WARN_ON_ONCE(!bpf_rcu_lock_held()); 211 if (!inode) 212 return -EINVAL; 213 214 /* This helper must only called from where the inode is guaranteed 215 * to have a refcount and cannot be freed. 216 */ 217 return inode_storage_delete(inode, map); 218 } 219 220 static int notsupp_get_next_key(struct bpf_map *map, void *key, 221 void *next_key) 222 { 223 return -ENOTSUPP; 224 } 225 226 static struct bpf_map *inode_storage_map_alloc(union bpf_attr *attr) 227 { 228 struct bpf_local_storage_map *smap; 229 230 smap = bpf_local_storage_map_alloc(attr); 231 if (IS_ERR(smap)) 232 return ERR_CAST(smap); 233 234 smap->cache_idx = bpf_local_storage_cache_idx_get(&inode_cache); 235 return &smap->map; 236 } 237 238 static void inode_storage_map_free(struct bpf_map *map) 239 { 240 struct bpf_local_storage_map *smap; 241 242 smap = (struct bpf_local_storage_map *)map; 243 bpf_local_storage_cache_idx_free(&inode_cache, smap->cache_idx); 244 bpf_local_storage_map_free(smap, NULL); 245 } 246 247 static int inode_storage_map_btf_id; 248 const struct bpf_map_ops inode_storage_map_ops = { 249 .map_meta_equal = bpf_map_meta_equal, 250 .map_alloc_check = bpf_local_storage_map_alloc_check, 251 .map_alloc = inode_storage_map_alloc, 252 .map_free = inode_storage_map_free, 253 .map_get_next_key = notsupp_get_next_key, 254 .map_lookup_elem = bpf_fd_inode_storage_lookup_elem, 255 .map_update_elem = bpf_fd_inode_storage_update_elem, 256 .map_delete_elem = bpf_fd_inode_storage_delete_elem, 257 .map_check_btf = bpf_local_storage_map_check_btf, 258 .map_btf_name = "bpf_local_storage_map", 259 .map_btf_id = &inode_storage_map_btf_id, 260 .map_owner_storage_ptr = inode_storage_ptr, 261 }; 262 263 BTF_ID_LIST_SINGLE(bpf_inode_storage_btf_ids, struct, inode) 264 265 const struct bpf_func_proto bpf_inode_storage_get_proto = { 266 .func = bpf_inode_storage_get, 267 .gpl_only = false, 268 .ret_type = RET_PTR_TO_MAP_VALUE_OR_NULL, 269 .arg1_type = ARG_CONST_MAP_PTR, 270 .arg2_type = ARG_PTR_TO_BTF_ID, 271 .arg2_btf_id = &bpf_inode_storage_btf_ids[0], 272 .arg3_type = ARG_PTR_TO_MAP_VALUE_OR_NULL, 273 .arg4_type = ARG_ANYTHING, 274 }; 275 276 const struct bpf_func_proto bpf_inode_storage_delete_proto = { 277 .func = bpf_inode_storage_delete, 278 .gpl_only = false, 279 .ret_type = RET_INTEGER, 280 .arg1_type = ARG_CONST_MAP_PTR, 281 .arg2_type = ARG_PTR_TO_BTF_ID, 282 .arg2_btf_id = &bpf_inode_storage_btf_ids[0], 283 }; 284