1 // SPDX-License-Identifier: GPL-2.0
2 /*
3  * Copyright (c) 2019 Facebook
4  * Copyright 2020 Google LLC.
5  */
6 
7 #include <linux/rculist.h>
8 #include <linux/list.h>
9 #include <linux/hash.h>
10 #include <linux/types.h>
11 #include <linux/spinlock.h>
12 #include <linux/bpf.h>
13 #include <linux/bpf_local_storage.h>
14 #include <net/sock.h>
15 #include <uapi/linux/sock_diag.h>
16 #include <uapi/linux/btf.h>
17 #include <linux/bpf_lsm.h>
18 #include <linux/btf_ids.h>
19 #include <linux/fdtable.h>
20 #include <linux/rcupdate_trace.h>
21 
22 DEFINE_BPF_STORAGE_CACHE(inode_cache);
23 
24 static struct bpf_local_storage __rcu **
25 inode_storage_ptr(void *owner)
26 {
27 	struct inode *inode = owner;
28 	struct bpf_storage_blob *bsb;
29 
30 	bsb = bpf_inode(inode);
31 	if (!bsb)
32 		return NULL;
33 	return &bsb->storage;
34 }
35 
36 static struct bpf_local_storage_data *inode_storage_lookup(struct inode *inode,
37 							   struct bpf_map *map,
38 							   bool cacheit_lockit)
39 {
40 	struct bpf_local_storage *inode_storage;
41 	struct bpf_local_storage_map *smap;
42 	struct bpf_storage_blob *bsb;
43 
44 	bsb = bpf_inode(inode);
45 	if (!bsb)
46 		return NULL;
47 
48 	inode_storage =
49 		rcu_dereference_check(bsb->storage, bpf_rcu_lock_held());
50 	if (!inode_storage)
51 		return NULL;
52 
53 	smap = (struct bpf_local_storage_map *)map;
54 	return bpf_local_storage_lookup(inode_storage, smap, cacheit_lockit);
55 }
56 
57 void bpf_inode_storage_free(struct inode *inode)
58 {
59 	struct bpf_local_storage *local_storage;
60 	bool free_inode_storage = false;
61 	struct bpf_storage_blob *bsb;
62 
63 	bsb = bpf_inode(inode);
64 	if (!bsb)
65 		return;
66 
67 	rcu_read_lock();
68 
69 	local_storage = rcu_dereference(bsb->storage);
70 	if (!local_storage) {
71 		rcu_read_unlock();
72 		return;
73 	}
74 
75 	raw_spin_lock_bh(&local_storage->lock);
76 	free_inode_storage = bpf_local_storage_unlink_nolock(local_storage);
77 	raw_spin_unlock_bh(&local_storage->lock);
78 	rcu_read_unlock();
79 
80 	if (free_inode_storage)
81 		kfree_rcu(local_storage, rcu);
82 }
83 
84 static void *bpf_fd_inode_storage_lookup_elem(struct bpf_map *map, void *key)
85 {
86 	struct bpf_local_storage_data *sdata;
87 	struct file *f;
88 	int fd;
89 
90 	fd = *(int *)key;
91 	f = fget_raw(fd);
92 	if (!f)
93 		return ERR_PTR(-EBADF);
94 
95 	sdata = inode_storage_lookup(f->f_inode, map, true);
96 	fput(f);
97 	return sdata ? sdata->data : NULL;
98 }
99 
100 static int bpf_fd_inode_storage_update_elem(struct bpf_map *map, void *key,
101 					 void *value, u64 map_flags)
102 {
103 	struct bpf_local_storage_data *sdata;
104 	struct file *f;
105 	int fd;
106 
107 	fd = *(int *)key;
108 	f = fget_raw(fd);
109 	if (!f)
110 		return -EBADF;
111 	if (!inode_storage_ptr(f->f_inode)) {
112 		fput(f);
113 		return -EBADF;
114 	}
115 
116 	sdata = bpf_local_storage_update(f->f_inode,
117 					 (struct bpf_local_storage_map *)map,
118 					 value, map_flags, GFP_ATOMIC);
119 	fput(f);
120 	return PTR_ERR_OR_ZERO(sdata);
121 }
122 
123 static int inode_storage_delete(struct inode *inode, struct bpf_map *map)
124 {
125 	struct bpf_local_storage_data *sdata;
126 
127 	sdata = inode_storage_lookup(inode, map, false);
128 	if (!sdata)
129 		return -ENOENT;
130 
131 	bpf_selem_unlink(SELEM(sdata), true);
132 
133 	return 0;
134 }
135 
136 static int bpf_fd_inode_storage_delete_elem(struct bpf_map *map, void *key)
137 {
138 	struct file *f;
139 	int fd, err;
140 
141 	fd = *(int *)key;
142 	f = fget_raw(fd);
143 	if (!f)
144 		return -EBADF;
145 
146 	err = inode_storage_delete(f->f_inode, map);
147 	fput(f);
148 	return err;
149 }
150 
151 /* *gfp_flags* is a hidden argument provided by the verifier */
152 BPF_CALL_5(bpf_inode_storage_get, struct bpf_map *, map, struct inode *, inode,
153 	   void *, value, u64, flags, gfp_t, gfp_flags)
154 {
155 	struct bpf_local_storage_data *sdata;
156 
157 	WARN_ON_ONCE(!bpf_rcu_lock_held());
158 	if (flags & ~(BPF_LOCAL_STORAGE_GET_F_CREATE))
159 		return (unsigned long)NULL;
160 
161 	/* explicitly check that the inode_storage_ptr is not
162 	 * NULL as inode_storage_lookup returns NULL in this case and
163 	 * bpf_local_storage_update expects the owner to have a
164 	 * valid storage pointer.
165 	 */
166 	if (!inode || !inode_storage_ptr(inode))
167 		return (unsigned long)NULL;
168 
169 	sdata = inode_storage_lookup(inode, map, true);
170 	if (sdata)
171 		return (unsigned long)sdata->data;
172 
173 	/* This helper must only called from where the inode is guaranteed
174 	 * to have a refcount and cannot be freed.
175 	 */
176 	if (flags & BPF_LOCAL_STORAGE_GET_F_CREATE) {
177 		sdata = bpf_local_storage_update(
178 			inode, (struct bpf_local_storage_map *)map, value,
179 			BPF_NOEXIST, gfp_flags);
180 		return IS_ERR(sdata) ? (unsigned long)NULL :
181 					     (unsigned long)sdata->data;
182 	}
183 
184 	return (unsigned long)NULL;
185 }
186 
187 BPF_CALL_2(bpf_inode_storage_delete,
188 	   struct bpf_map *, map, struct inode *, inode)
189 {
190 	WARN_ON_ONCE(!bpf_rcu_lock_held());
191 	if (!inode)
192 		return -EINVAL;
193 
194 	/* This helper must only called from where the inode is guaranteed
195 	 * to have a refcount and cannot be freed.
196 	 */
197 	return inode_storage_delete(inode, map);
198 }
199 
200 static int notsupp_get_next_key(struct bpf_map *map, void *key,
201 				void *next_key)
202 {
203 	return -ENOTSUPP;
204 }
205 
206 static struct bpf_map *inode_storage_map_alloc(union bpf_attr *attr)
207 {
208 	return bpf_local_storage_map_alloc(attr, &inode_cache);
209 }
210 
211 static void inode_storage_map_free(struct bpf_map *map)
212 {
213 	bpf_local_storage_map_free(map, &inode_cache, NULL);
214 }
215 
216 const struct bpf_map_ops inode_storage_map_ops = {
217 	.map_meta_equal = bpf_map_meta_equal,
218 	.map_alloc_check = bpf_local_storage_map_alloc_check,
219 	.map_alloc = inode_storage_map_alloc,
220 	.map_free = inode_storage_map_free,
221 	.map_get_next_key = notsupp_get_next_key,
222 	.map_lookup_elem = bpf_fd_inode_storage_lookup_elem,
223 	.map_update_elem = bpf_fd_inode_storage_update_elem,
224 	.map_delete_elem = bpf_fd_inode_storage_delete_elem,
225 	.map_check_btf = bpf_local_storage_map_check_btf,
226 	.map_btf_id = &bpf_local_storage_map_btf_id[0],
227 	.map_owner_storage_ptr = inode_storage_ptr,
228 };
229 
230 BTF_ID_LIST_SINGLE(bpf_inode_storage_btf_ids, struct, inode)
231 
232 const struct bpf_func_proto bpf_inode_storage_get_proto = {
233 	.func		= bpf_inode_storage_get,
234 	.gpl_only	= false,
235 	.ret_type	= RET_PTR_TO_MAP_VALUE_OR_NULL,
236 	.arg1_type	= ARG_CONST_MAP_PTR,
237 	.arg2_type	= ARG_PTR_TO_BTF_ID,
238 	.arg2_btf_id	= &bpf_inode_storage_btf_ids[0],
239 	.arg3_type	= ARG_PTR_TO_MAP_VALUE_OR_NULL,
240 	.arg4_type	= ARG_ANYTHING,
241 };
242 
243 const struct bpf_func_proto bpf_inode_storage_delete_proto = {
244 	.func		= bpf_inode_storage_delete,
245 	.gpl_only	= false,
246 	.ret_type	= RET_INTEGER,
247 	.arg1_type	= ARG_CONST_MAP_PTR,
248 	.arg2_type	= ARG_PTR_TO_BTF_ID,
249 	.arg2_btf_id	= &bpf_inode_storage_btf_ids[0],
250 };
251