1 // SPDX-License-Identifier: GPL-2.0
2 /*
3  * Copyright (c) 2019 Facebook
4  * Copyright 2020 Google LLC.
5  */
6 
7 #include <linux/rculist.h>
8 #include <linux/list.h>
9 #include <linux/hash.h>
10 #include <linux/types.h>
11 #include <linux/spinlock.h>
12 #include <linux/bpf.h>
13 #include <linux/bpf_local_storage.h>
14 #include <net/sock.h>
15 #include <uapi/linux/sock_diag.h>
16 #include <uapi/linux/btf.h>
17 #include <linux/bpf_lsm.h>
18 #include <linux/btf_ids.h>
19 #include <linux/fdtable.h>
20 #include <linux/rcupdate_trace.h>
21 
22 DEFINE_BPF_STORAGE_CACHE(inode_cache);
23 
24 static struct bpf_local_storage __rcu **
25 inode_storage_ptr(void *owner)
26 {
27 	struct inode *inode = owner;
28 	struct bpf_storage_blob *bsb;
29 
30 	bsb = bpf_inode(inode);
31 	if (!bsb)
32 		return NULL;
33 	return &bsb->storage;
34 }
35 
36 static struct bpf_local_storage_data *inode_storage_lookup(struct inode *inode,
37 							   struct bpf_map *map,
38 							   bool cacheit_lockit)
39 {
40 	struct bpf_local_storage *inode_storage;
41 	struct bpf_local_storage_map *smap;
42 	struct bpf_storage_blob *bsb;
43 
44 	bsb = bpf_inode(inode);
45 	if (!bsb)
46 		return NULL;
47 
48 	inode_storage =
49 		rcu_dereference_check(bsb->storage, bpf_rcu_lock_held());
50 	if (!inode_storage)
51 		return NULL;
52 
53 	smap = (struct bpf_local_storage_map *)map;
54 	return bpf_local_storage_lookup(inode_storage, smap, cacheit_lockit);
55 }
56 
57 void bpf_inode_storage_free(struct inode *inode)
58 {
59 	struct bpf_local_storage_elem *selem;
60 	struct bpf_local_storage *local_storage;
61 	bool free_inode_storage = false;
62 	struct bpf_storage_blob *bsb;
63 	struct hlist_node *n;
64 
65 	bsb = bpf_inode(inode);
66 	if (!bsb)
67 		return;
68 
69 	rcu_read_lock();
70 
71 	local_storage = rcu_dereference(bsb->storage);
72 	if (!local_storage) {
73 		rcu_read_unlock();
74 		return;
75 	}
76 
77 	/* Neither the bpf_prog nor the bpf-map's syscall
78 	 * could be modifying the local_storage->list now.
79 	 * Thus, no elem can be added-to or deleted-from the
80 	 * local_storage->list by the bpf_prog or by the bpf-map's syscall.
81 	 *
82 	 * It is racing with bpf_local_storage_map_free() alone
83 	 * when unlinking elem from the local_storage->list and
84 	 * the map's bucket->list.
85 	 */
86 	raw_spin_lock_bh(&local_storage->lock);
87 	hlist_for_each_entry_safe(selem, n, &local_storage->list, snode) {
88 		/* Always unlink from map before unlinking from
89 		 * local_storage.
90 		 */
91 		bpf_selem_unlink_map(selem);
92 		free_inode_storage = bpf_selem_unlink_storage_nolock(
93 			local_storage, selem, false, false);
94 	}
95 	raw_spin_unlock_bh(&local_storage->lock);
96 	rcu_read_unlock();
97 
98 	/* free_inoode_storage should always be true as long as
99 	 * local_storage->list was non-empty.
100 	 */
101 	if (free_inode_storage)
102 		kfree_rcu(local_storage, rcu);
103 }
104 
105 static void *bpf_fd_inode_storage_lookup_elem(struct bpf_map *map, void *key)
106 {
107 	struct bpf_local_storage_data *sdata;
108 	struct file *f;
109 	int fd;
110 
111 	fd = *(int *)key;
112 	f = fget_raw(fd);
113 	if (!f)
114 		return ERR_PTR(-EBADF);
115 
116 	sdata = inode_storage_lookup(f->f_inode, map, true);
117 	fput(f);
118 	return sdata ? sdata->data : NULL;
119 }
120 
121 static int bpf_fd_inode_storage_update_elem(struct bpf_map *map, void *key,
122 					 void *value, u64 map_flags)
123 {
124 	struct bpf_local_storage_data *sdata;
125 	struct file *f;
126 	int fd;
127 
128 	fd = *(int *)key;
129 	f = fget_raw(fd);
130 	if (!f)
131 		return -EBADF;
132 	if (!inode_storage_ptr(f->f_inode)) {
133 		fput(f);
134 		return -EBADF;
135 	}
136 
137 	sdata = bpf_local_storage_update(f->f_inode,
138 					 (struct bpf_local_storage_map *)map,
139 					 value, map_flags, GFP_ATOMIC);
140 	fput(f);
141 	return PTR_ERR_OR_ZERO(sdata);
142 }
143 
144 static int inode_storage_delete(struct inode *inode, struct bpf_map *map)
145 {
146 	struct bpf_local_storage_data *sdata;
147 
148 	sdata = inode_storage_lookup(inode, map, false);
149 	if (!sdata)
150 		return -ENOENT;
151 
152 	bpf_selem_unlink(SELEM(sdata), true);
153 
154 	return 0;
155 }
156 
157 static int bpf_fd_inode_storage_delete_elem(struct bpf_map *map, void *key)
158 {
159 	struct file *f;
160 	int fd, err;
161 
162 	fd = *(int *)key;
163 	f = fget_raw(fd);
164 	if (!f)
165 		return -EBADF;
166 
167 	err = inode_storage_delete(f->f_inode, map);
168 	fput(f);
169 	return err;
170 }
171 
172 /* *gfp_flags* is a hidden argument provided by the verifier */
173 BPF_CALL_5(bpf_inode_storage_get, struct bpf_map *, map, struct inode *, inode,
174 	   void *, value, u64, flags, gfp_t, gfp_flags)
175 {
176 	struct bpf_local_storage_data *sdata;
177 
178 	WARN_ON_ONCE(!bpf_rcu_lock_held());
179 	if (flags & ~(BPF_LOCAL_STORAGE_GET_F_CREATE))
180 		return (unsigned long)NULL;
181 
182 	/* explicitly check that the inode_storage_ptr is not
183 	 * NULL as inode_storage_lookup returns NULL in this case and
184 	 * bpf_local_storage_update expects the owner to have a
185 	 * valid storage pointer.
186 	 */
187 	if (!inode || !inode_storage_ptr(inode))
188 		return (unsigned long)NULL;
189 
190 	sdata = inode_storage_lookup(inode, map, true);
191 	if (sdata)
192 		return (unsigned long)sdata->data;
193 
194 	/* This helper must only called from where the inode is guaranteed
195 	 * to have a refcount and cannot be freed.
196 	 */
197 	if (flags & BPF_LOCAL_STORAGE_GET_F_CREATE) {
198 		sdata = bpf_local_storage_update(
199 			inode, (struct bpf_local_storage_map *)map, value,
200 			BPF_NOEXIST, gfp_flags);
201 		return IS_ERR(sdata) ? (unsigned long)NULL :
202 					     (unsigned long)sdata->data;
203 	}
204 
205 	return (unsigned long)NULL;
206 }
207 
208 BPF_CALL_2(bpf_inode_storage_delete,
209 	   struct bpf_map *, map, struct inode *, inode)
210 {
211 	WARN_ON_ONCE(!bpf_rcu_lock_held());
212 	if (!inode)
213 		return -EINVAL;
214 
215 	/* This helper must only called from where the inode is guaranteed
216 	 * to have a refcount and cannot be freed.
217 	 */
218 	return inode_storage_delete(inode, map);
219 }
220 
221 static int notsupp_get_next_key(struct bpf_map *map, void *key,
222 				void *next_key)
223 {
224 	return -ENOTSUPP;
225 }
226 
227 static struct bpf_map *inode_storage_map_alloc(union bpf_attr *attr)
228 {
229 	struct bpf_local_storage_map *smap;
230 
231 	smap = bpf_local_storage_map_alloc(attr);
232 	if (IS_ERR(smap))
233 		return ERR_CAST(smap);
234 
235 	smap->cache_idx = bpf_local_storage_cache_idx_get(&inode_cache);
236 	return &smap->map;
237 }
238 
239 static void inode_storage_map_free(struct bpf_map *map)
240 {
241 	struct bpf_local_storage_map *smap;
242 
243 	smap = (struct bpf_local_storage_map *)map;
244 	bpf_local_storage_cache_idx_free(&inode_cache, smap->cache_idx);
245 	bpf_local_storage_map_free(smap, NULL);
246 }
247 
248 BTF_ID_LIST_SINGLE(inode_storage_map_btf_ids, struct,
249 		   bpf_local_storage_map)
250 const struct bpf_map_ops inode_storage_map_ops = {
251 	.map_meta_equal = bpf_map_meta_equal,
252 	.map_alloc_check = bpf_local_storage_map_alloc_check,
253 	.map_alloc = inode_storage_map_alloc,
254 	.map_free = inode_storage_map_free,
255 	.map_get_next_key = notsupp_get_next_key,
256 	.map_lookup_elem = bpf_fd_inode_storage_lookup_elem,
257 	.map_update_elem = bpf_fd_inode_storage_update_elem,
258 	.map_delete_elem = bpf_fd_inode_storage_delete_elem,
259 	.map_check_btf = bpf_local_storage_map_check_btf,
260 	.map_btf_id = &inode_storage_map_btf_ids[0],
261 	.map_owner_storage_ptr = inode_storage_ptr,
262 };
263 
264 BTF_ID_LIST_SINGLE(bpf_inode_storage_btf_ids, struct, inode)
265 
266 const struct bpf_func_proto bpf_inode_storage_get_proto = {
267 	.func		= bpf_inode_storage_get,
268 	.gpl_only	= false,
269 	.ret_type	= RET_PTR_TO_MAP_VALUE_OR_NULL,
270 	.arg1_type	= ARG_CONST_MAP_PTR,
271 	.arg2_type	= ARG_PTR_TO_BTF_ID,
272 	.arg2_btf_id	= &bpf_inode_storage_btf_ids[0],
273 	.arg3_type	= ARG_PTR_TO_MAP_VALUE_OR_NULL,
274 	.arg4_type	= ARG_ANYTHING,
275 };
276 
277 const struct bpf_func_proto bpf_inode_storage_delete_proto = {
278 	.func		= bpf_inode_storage_delete,
279 	.gpl_only	= false,
280 	.ret_type	= RET_INTEGER,
281 	.arg1_type	= ARG_CONST_MAP_PTR,
282 	.arg2_type	= ARG_PTR_TO_BTF_ID,
283 	.arg2_btf_id	= &bpf_inode_storage_btf_ids[0],
284 };
285