1 // SPDX-License-Identifier: GPL-2.0 2 /* 3 * Copyright (c) 2019 Facebook 4 * Copyright 2020 Google LLC. 5 */ 6 7 #include <linux/rculist.h> 8 #include <linux/list.h> 9 #include <linux/hash.h> 10 #include <linux/types.h> 11 #include <linux/spinlock.h> 12 #include <linux/bpf.h> 13 #include <linux/bpf_local_storage.h> 14 #include <net/sock.h> 15 #include <uapi/linux/sock_diag.h> 16 #include <uapi/linux/btf.h> 17 #include <linux/bpf_lsm.h> 18 #include <linux/btf_ids.h> 19 #include <linux/fdtable.h> 20 21 DEFINE_BPF_STORAGE_CACHE(inode_cache); 22 23 static struct bpf_local_storage __rcu ** 24 inode_storage_ptr(void *owner) 25 { 26 struct inode *inode = owner; 27 struct bpf_storage_blob *bsb; 28 29 bsb = bpf_inode(inode); 30 if (!bsb) 31 return NULL; 32 return &bsb->storage; 33 } 34 35 static struct bpf_local_storage_data *inode_storage_lookup(struct inode *inode, 36 struct bpf_map *map, 37 bool cacheit_lockit) 38 { 39 struct bpf_local_storage *inode_storage; 40 struct bpf_local_storage_map *smap; 41 struct bpf_storage_blob *bsb; 42 43 bsb = bpf_inode(inode); 44 if (!bsb) 45 return NULL; 46 47 inode_storage = rcu_dereference(bsb->storage); 48 if (!inode_storage) 49 return NULL; 50 51 smap = (struct bpf_local_storage_map *)map; 52 return bpf_local_storage_lookup(inode_storage, smap, cacheit_lockit); 53 } 54 55 void bpf_inode_storage_free(struct inode *inode) 56 { 57 struct bpf_local_storage_elem *selem; 58 struct bpf_local_storage *local_storage; 59 bool free_inode_storage = false; 60 struct bpf_storage_blob *bsb; 61 struct hlist_node *n; 62 63 bsb = bpf_inode(inode); 64 if (!bsb) 65 return; 66 67 rcu_read_lock(); 68 69 local_storage = rcu_dereference(bsb->storage); 70 if (!local_storage) { 71 rcu_read_unlock(); 72 return; 73 } 74 75 /* Neither the bpf_prog nor the bpf-map's syscall 76 * could be modifying the local_storage->list now. 77 * Thus, no elem can be added-to or deleted-from the 78 * local_storage->list by the bpf_prog or by the bpf-map's syscall. 79 * 80 * It is racing with bpf_local_storage_map_free() alone 81 * when unlinking elem from the local_storage->list and 82 * the map's bucket->list. 83 */ 84 raw_spin_lock_bh(&local_storage->lock); 85 hlist_for_each_entry_safe(selem, n, &local_storage->list, snode) { 86 /* Always unlink from map before unlinking from 87 * local_storage. 88 */ 89 bpf_selem_unlink_map(selem); 90 free_inode_storage = bpf_selem_unlink_storage_nolock( 91 local_storage, selem, false); 92 } 93 raw_spin_unlock_bh(&local_storage->lock); 94 rcu_read_unlock(); 95 96 /* free_inoode_storage should always be true as long as 97 * local_storage->list was non-empty. 98 */ 99 if (free_inode_storage) 100 kfree_rcu(local_storage, rcu); 101 } 102 103 static void *bpf_fd_inode_storage_lookup_elem(struct bpf_map *map, void *key) 104 { 105 struct bpf_local_storage_data *sdata; 106 struct file *f; 107 int fd; 108 109 fd = *(int *)key; 110 f = fget_raw(fd); 111 if (!f) 112 return ERR_PTR(-EBADF); 113 114 sdata = inode_storage_lookup(f->f_inode, map, true); 115 fput(f); 116 return sdata ? sdata->data : NULL; 117 } 118 119 static int bpf_fd_inode_storage_update_elem(struct bpf_map *map, void *key, 120 void *value, u64 map_flags) 121 { 122 struct bpf_local_storage_data *sdata; 123 struct file *f; 124 int fd; 125 126 fd = *(int *)key; 127 f = fget_raw(fd); 128 if (!f) 129 return -EBADF; 130 if (!inode_storage_ptr(f->f_inode)) { 131 fput(f); 132 return -EBADF; 133 } 134 135 sdata = bpf_local_storage_update(f->f_inode, 136 (struct bpf_local_storage_map *)map, 137 value, map_flags); 138 fput(f); 139 return PTR_ERR_OR_ZERO(sdata); 140 } 141 142 static int inode_storage_delete(struct inode *inode, struct bpf_map *map) 143 { 144 struct bpf_local_storage_data *sdata; 145 146 sdata = inode_storage_lookup(inode, map, false); 147 if (!sdata) 148 return -ENOENT; 149 150 bpf_selem_unlink(SELEM(sdata)); 151 152 return 0; 153 } 154 155 static int bpf_fd_inode_storage_delete_elem(struct bpf_map *map, void *key) 156 { 157 struct file *f; 158 int fd, err; 159 160 fd = *(int *)key; 161 f = fget_raw(fd); 162 if (!f) 163 return -EBADF; 164 165 err = inode_storage_delete(f->f_inode, map); 166 fput(f); 167 return err; 168 } 169 170 BPF_CALL_4(bpf_inode_storage_get, struct bpf_map *, map, struct inode *, inode, 171 void *, value, u64, flags) 172 { 173 struct bpf_local_storage_data *sdata; 174 175 if (flags & ~(BPF_LOCAL_STORAGE_GET_F_CREATE)) 176 return (unsigned long)NULL; 177 178 /* explicitly check that the inode_storage_ptr is not 179 * NULL as inode_storage_lookup returns NULL in this case and 180 * bpf_local_storage_update expects the owner to have a 181 * valid storage pointer. 182 */ 183 if (!inode || !inode_storage_ptr(inode)) 184 return (unsigned long)NULL; 185 186 sdata = inode_storage_lookup(inode, map, true); 187 if (sdata) 188 return (unsigned long)sdata->data; 189 190 /* This helper must only called from where the inode is guaranteed 191 * to have a refcount and cannot be freed. 192 */ 193 if (flags & BPF_LOCAL_STORAGE_GET_F_CREATE) { 194 sdata = bpf_local_storage_update( 195 inode, (struct bpf_local_storage_map *)map, value, 196 BPF_NOEXIST); 197 return IS_ERR(sdata) ? (unsigned long)NULL : 198 (unsigned long)sdata->data; 199 } 200 201 return (unsigned long)NULL; 202 } 203 204 BPF_CALL_2(bpf_inode_storage_delete, 205 struct bpf_map *, map, struct inode *, inode) 206 { 207 if (!inode) 208 return -EINVAL; 209 210 /* This helper must only called from where the inode is guaranteed 211 * to have a refcount and cannot be freed. 212 */ 213 return inode_storage_delete(inode, map); 214 } 215 216 static int notsupp_get_next_key(struct bpf_map *map, void *key, 217 void *next_key) 218 { 219 return -ENOTSUPP; 220 } 221 222 static struct bpf_map *inode_storage_map_alloc(union bpf_attr *attr) 223 { 224 struct bpf_local_storage_map *smap; 225 226 smap = bpf_local_storage_map_alloc(attr); 227 if (IS_ERR(smap)) 228 return ERR_CAST(smap); 229 230 smap->cache_idx = bpf_local_storage_cache_idx_get(&inode_cache); 231 return &smap->map; 232 } 233 234 static void inode_storage_map_free(struct bpf_map *map) 235 { 236 struct bpf_local_storage_map *smap; 237 238 smap = (struct bpf_local_storage_map *)map; 239 bpf_local_storage_cache_idx_free(&inode_cache, smap->cache_idx); 240 bpf_local_storage_map_free(smap, NULL); 241 } 242 243 static int inode_storage_map_btf_id; 244 const struct bpf_map_ops inode_storage_map_ops = { 245 .map_meta_equal = bpf_map_meta_equal, 246 .map_alloc_check = bpf_local_storage_map_alloc_check, 247 .map_alloc = inode_storage_map_alloc, 248 .map_free = inode_storage_map_free, 249 .map_get_next_key = notsupp_get_next_key, 250 .map_lookup_elem = bpf_fd_inode_storage_lookup_elem, 251 .map_update_elem = bpf_fd_inode_storage_update_elem, 252 .map_delete_elem = bpf_fd_inode_storage_delete_elem, 253 .map_check_btf = bpf_local_storage_map_check_btf, 254 .map_btf_name = "bpf_local_storage_map", 255 .map_btf_id = &inode_storage_map_btf_id, 256 .map_owner_storage_ptr = inode_storage_ptr, 257 }; 258 259 BTF_ID_LIST_SINGLE(bpf_inode_storage_btf_ids, struct, inode) 260 261 const struct bpf_func_proto bpf_inode_storage_get_proto = { 262 .func = bpf_inode_storage_get, 263 .gpl_only = false, 264 .ret_type = RET_PTR_TO_MAP_VALUE_OR_NULL, 265 .arg1_type = ARG_CONST_MAP_PTR, 266 .arg2_type = ARG_PTR_TO_BTF_ID, 267 .arg2_btf_id = &bpf_inode_storage_btf_ids[0], 268 .arg3_type = ARG_PTR_TO_MAP_VALUE_OR_NULL, 269 .arg4_type = ARG_ANYTHING, 270 }; 271 272 const struct bpf_func_proto bpf_inode_storage_delete_proto = { 273 .func = bpf_inode_storage_delete, 274 .gpl_only = false, 275 .ret_type = RET_INTEGER, 276 .arg1_type = ARG_CONST_MAP_PTR, 277 .arg2_type = ARG_PTR_TO_BTF_ID, 278 .arg2_btf_id = &bpf_inode_storage_btf_ids[0], 279 }; 280