1 // SPDX-License-Identifier: GPL-2.0 2 /* 3 * Copyright (c) 2020 Facebook 4 * Copyright 2020 Google LLC. 5 */ 6 7 #include <linux/pid.h> 8 #include <linux/sched.h> 9 #include <linux/rculist.h> 10 #include <linux/list.h> 11 #include <linux/hash.h> 12 #include <linux/types.h> 13 #include <linux/spinlock.h> 14 #include <linux/bpf.h> 15 #include <linux/bpf_local_storage.h> 16 #include <linux/filter.h> 17 #include <uapi/linux/btf.h> 18 #include <linux/btf_ids.h> 19 #include <linux/fdtable.h> 20 #include <linux/rcupdate_trace.h> 21 22 DEFINE_BPF_STORAGE_CACHE(task_cache); 23 24 static DEFINE_PER_CPU(int, bpf_task_storage_busy); 25 26 static void bpf_task_storage_lock(void) 27 { 28 migrate_disable(); 29 this_cpu_inc(bpf_task_storage_busy); 30 } 31 32 static void bpf_task_storage_unlock(void) 33 { 34 this_cpu_dec(bpf_task_storage_busy); 35 migrate_enable(); 36 } 37 38 static bool bpf_task_storage_trylock(void) 39 { 40 migrate_disable(); 41 if (unlikely(this_cpu_inc_return(bpf_task_storage_busy) != 1)) { 42 this_cpu_dec(bpf_task_storage_busy); 43 migrate_enable(); 44 return false; 45 } 46 return true; 47 } 48 49 static struct bpf_local_storage __rcu **task_storage_ptr(void *owner) 50 { 51 struct task_struct *task = owner; 52 53 return &task->bpf_storage; 54 } 55 56 static struct bpf_local_storage_data * 57 task_storage_lookup(struct task_struct *task, struct bpf_map *map, 58 bool cacheit_lockit) 59 { 60 struct bpf_local_storage *task_storage; 61 struct bpf_local_storage_map *smap; 62 63 task_storage = 64 rcu_dereference_check(task->bpf_storage, bpf_rcu_lock_held()); 65 if (!task_storage) 66 return NULL; 67 68 smap = (struct bpf_local_storage_map *)map; 69 return bpf_local_storage_lookup(task_storage, smap, cacheit_lockit); 70 } 71 72 void bpf_task_storage_free(struct task_struct *task) 73 { 74 struct bpf_local_storage *local_storage; 75 bool free_task_storage = false; 76 unsigned long flags; 77 78 rcu_read_lock(); 79 80 local_storage = rcu_dereference(task->bpf_storage); 81 if (!local_storage) { 82 rcu_read_unlock(); 83 return; 84 } 85 86 bpf_task_storage_lock(); 87 raw_spin_lock_irqsave(&local_storage->lock, flags); 88 free_task_storage = bpf_local_storage_unlink_nolock(local_storage); 89 raw_spin_unlock_irqrestore(&local_storage->lock, flags); 90 bpf_task_storage_unlock(); 91 rcu_read_unlock(); 92 93 if (free_task_storage) 94 kfree_rcu(local_storage, rcu); 95 } 96 97 static void *bpf_pid_task_storage_lookup_elem(struct bpf_map *map, void *key) 98 { 99 struct bpf_local_storage_data *sdata; 100 struct task_struct *task; 101 unsigned int f_flags; 102 struct pid *pid; 103 int fd, err; 104 105 fd = *(int *)key; 106 pid = pidfd_get_pid(fd, &f_flags); 107 if (IS_ERR(pid)) 108 return ERR_CAST(pid); 109 110 /* We should be in an RCU read side critical section, it should be safe 111 * to call pid_task. 112 */ 113 WARN_ON_ONCE(!rcu_read_lock_held()); 114 task = pid_task(pid, PIDTYPE_PID); 115 if (!task) { 116 err = -ENOENT; 117 goto out; 118 } 119 120 bpf_task_storage_lock(); 121 sdata = task_storage_lookup(task, map, true); 122 bpf_task_storage_unlock(); 123 put_pid(pid); 124 return sdata ? sdata->data : NULL; 125 out: 126 put_pid(pid); 127 return ERR_PTR(err); 128 } 129 130 static int bpf_pid_task_storage_update_elem(struct bpf_map *map, void *key, 131 void *value, u64 map_flags) 132 { 133 struct bpf_local_storage_data *sdata; 134 struct task_struct *task; 135 unsigned int f_flags; 136 struct pid *pid; 137 int fd, err; 138 139 fd = *(int *)key; 140 pid = pidfd_get_pid(fd, &f_flags); 141 if (IS_ERR(pid)) 142 return PTR_ERR(pid); 143 144 /* We should be in an RCU read side critical section, it should be safe 145 * to call pid_task. 146 */ 147 WARN_ON_ONCE(!rcu_read_lock_held()); 148 task = pid_task(pid, PIDTYPE_PID); 149 if (!task) { 150 err = -ENOENT; 151 goto out; 152 } 153 154 bpf_task_storage_lock(); 155 sdata = bpf_local_storage_update( 156 task, (struct bpf_local_storage_map *)map, value, map_flags, 157 GFP_ATOMIC); 158 bpf_task_storage_unlock(); 159 160 err = PTR_ERR_OR_ZERO(sdata); 161 out: 162 put_pid(pid); 163 return err; 164 } 165 166 static int task_storage_delete(struct task_struct *task, struct bpf_map *map, 167 bool nobusy) 168 { 169 struct bpf_local_storage_data *sdata; 170 171 sdata = task_storage_lookup(task, map, false); 172 if (!sdata) 173 return -ENOENT; 174 175 if (!nobusy) 176 return -EBUSY; 177 178 bpf_selem_unlink(SELEM(sdata), true); 179 180 return 0; 181 } 182 183 static int bpf_pid_task_storage_delete_elem(struct bpf_map *map, void *key) 184 { 185 struct task_struct *task; 186 unsigned int f_flags; 187 struct pid *pid; 188 int fd, err; 189 190 fd = *(int *)key; 191 pid = pidfd_get_pid(fd, &f_flags); 192 if (IS_ERR(pid)) 193 return PTR_ERR(pid); 194 195 /* We should be in an RCU read side critical section, it should be safe 196 * to call pid_task. 197 */ 198 WARN_ON_ONCE(!rcu_read_lock_held()); 199 task = pid_task(pid, PIDTYPE_PID); 200 if (!task) { 201 err = -ENOENT; 202 goto out; 203 } 204 205 bpf_task_storage_lock(); 206 err = task_storage_delete(task, map, true); 207 bpf_task_storage_unlock(); 208 out: 209 put_pid(pid); 210 return err; 211 } 212 213 /* Called by bpf_task_storage_get*() helpers */ 214 static void *__bpf_task_storage_get(struct bpf_map *map, 215 struct task_struct *task, void *value, 216 u64 flags, gfp_t gfp_flags, bool nobusy) 217 { 218 struct bpf_local_storage_data *sdata; 219 220 sdata = task_storage_lookup(task, map, nobusy); 221 if (sdata) 222 return sdata->data; 223 224 /* only allocate new storage, when the task is refcounted */ 225 if (refcount_read(&task->usage) && 226 (flags & BPF_LOCAL_STORAGE_GET_F_CREATE) && nobusy) { 227 sdata = bpf_local_storage_update( 228 task, (struct bpf_local_storage_map *)map, value, 229 BPF_NOEXIST, gfp_flags); 230 return IS_ERR(sdata) ? NULL : sdata->data; 231 } 232 233 return NULL; 234 } 235 236 /* *gfp_flags* is a hidden argument provided by the verifier */ 237 BPF_CALL_5(bpf_task_storage_get_recur, struct bpf_map *, map, struct task_struct *, 238 task, void *, value, u64, flags, gfp_t, gfp_flags) 239 { 240 bool nobusy; 241 void *data; 242 243 WARN_ON_ONCE(!bpf_rcu_lock_held()); 244 if (flags & ~BPF_LOCAL_STORAGE_GET_F_CREATE || !task) 245 return (unsigned long)NULL; 246 247 nobusy = bpf_task_storage_trylock(); 248 data = __bpf_task_storage_get(map, task, value, flags, 249 gfp_flags, nobusy); 250 if (nobusy) 251 bpf_task_storage_unlock(); 252 return (unsigned long)data; 253 } 254 255 /* *gfp_flags* is a hidden argument provided by the verifier */ 256 BPF_CALL_5(bpf_task_storage_get, struct bpf_map *, map, struct task_struct *, 257 task, void *, value, u64, flags, gfp_t, gfp_flags) 258 { 259 void *data; 260 261 WARN_ON_ONCE(!bpf_rcu_lock_held()); 262 if (flags & ~BPF_LOCAL_STORAGE_GET_F_CREATE || !task) 263 return (unsigned long)NULL; 264 265 bpf_task_storage_lock(); 266 data = __bpf_task_storage_get(map, task, value, flags, 267 gfp_flags, true); 268 bpf_task_storage_unlock(); 269 return (unsigned long)data; 270 } 271 272 BPF_CALL_2(bpf_task_storage_delete_recur, struct bpf_map *, map, struct task_struct *, 273 task) 274 { 275 bool nobusy; 276 int ret; 277 278 WARN_ON_ONCE(!bpf_rcu_lock_held()); 279 if (!task) 280 return -EINVAL; 281 282 nobusy = bpf_task_storage_trylock(); 283 /* This helper must only be called from places where the lifetime of the task 284 * is guaranteed. Either by being refcounted or by being protected 285 * by an RCU read-side critical section. 286 */ 287 ret = task_storage_delete(task, map, nobusy); 288 if (nobusy) 289 bpf_task_storage_unlock(); 290 return ret; 291 } 292 293 BPF_CALL_2(bpf_task_storage_delete, struct bpf_map *, map, struct task_struct *, 294 task) 295 { 296 int ret; 297 298 WARN_ON_ONCE(!bpf_rcu_lock_held()); 299 if (!task) 300 return -EINVAL; 301 302 bpf_task_storage_lock(); 303 /* This helper must only be called from places where the lifetime of the task 304 * is guaranteed. Either by being refcounted or by being protected 305 * by an RCU read-side critical section. 306 */ 307 ret = task_storage_delete(task, map, true); 308 bpf_task_storage_unlock(); 309 return ret; 310 } 311 312 static int notsupp_get_next_key(struct bpf_map *map, void *key, void *next_key) 313 { 314 return -ENOTSUPP; 315 } 316 317 static struct bpf_map *task_storage_map_alloc(union bpf_attr *attr) 318 { 319 return bpf_local_storage_map_alloc(attr, &task_cache); 320 } 321 322 static void task_storage_map_free(struct bpf_map *map) 323 { 324 bpf_local_storage_map_free(map, &task_cache, &bpf_task_storage_busy); 325 } 326 327 BTF_ID_LIST_GLOBAL_SINGLE(bpf_local_storage_map_btf_id, struct, bpf_local_storage_map) 328 const struct bpf_map_ops task_storage_map_ops = { 329 .map_meta_equal = bpf_map_meta_equal, 330 .map_alloc_check = bpf_local_storage_map_alloc_check, 331 .map_alloc = task_storage_map_alloc, 332 .map_free = task_storage_map_free, 333 .map_get_next_key = notsupp_get_next_key, 334 .map_lookup_elem = bpf_pid_task_storage_lookup_elem, 335 .map_update_elem = bpf_pid_task_storage_update_elem, 336 .map_delete_elem = bpf_pid_task_storage_delete_elem, 337 .map_check_btf = bpf_local_storage_map_check_btf, 338 .map_btf_id = &bpf_local_storage_map_btf_id[0], 339 .map_owner_storage_ptr = task_storage_ptr, 340 }; 341 342 const struct bpf_func_proto bpf_task_storage_get_recur_proto = { 343 .func = bpf_task_storage_get_recur, 344 .gpl_only = false, 345 .ret_type = RET_PTR_TO_MAP_VALUE_OR_NULL, 346 .arg1_type = ARG_CONST_MAP_PTR, 347 .arg2_type = ARG_PTR_TO_BTF_ID, 348 .arg2_btf_id = &btf_tracing_ids[BTF_TRACING_TYPE_TASK], 349 .arg3_type = ARG_PTR_TO_MAP_VALUE_OR_NULL, 350 .arg4_type = ARG_ANYTHING, 351 }; 352 353 const struct bpf_func_proto bpf_task_storage_get_proto = { 354 .func = bpf_task_storage_get, 355 .gpl_only = false, 356 .ret_type = RET_PTR_TO_MAP_VALUE_OR_NULL, 357 .arg1_type = ARG_CONST_MAP_PTR, 358 .arg2_type = ARG_PTR_TO_BTF_ID, 359 .arg2_btf_id = &btf_tracing_ids[BTF_TRACING_TYPE_TASK], 360 .arg3_type = ARG_PTR_TO_MAP_VALUE_OR_NULL, 361 .arg4_type = ARG_ANYTHING, 362 }; 363 364 const struct bpf_func_proto bpf_task_storage_delete_recur_proto = { 365 .func = bpf_task_storage_delete_recur, 366 .gpl_only = false, 367 .ret_type = RET_INTEGER, 368 .arg1_type = ARG_CONST_MAP_PTR, 369 .arg2_type = ARG_PTR_TO_BTF_ID, 370 .arg2_btf_id = &btf_tracing_ids[BTF_TRACING_TYPE_TASK], 371 }; 372 373 const struct bpf_func_proto bpf_task_storage_delete_proto = { 374 .func = bpf_task_storage_delete, 375 .gpl_only = false, 376 .ret_type = RET_INTEGER, 377 .arg1_type = ARG_CONST_MAP_PTR, 378 .arg2_type = ARG_PTR_TO_BTF_ID, 379 .arg2_btf_id = &btf_tracing_ids[BTF_TRACING_TYPE_TASK], 380 }; 381