1 //SPDX-License-Identifier: GPL-2.0 2 #include <linux/bpf-cgroup.h> 3 #include <linux/bpf.h> 4 #include <linux/bug.h> 5 #include <linux/filter.h> 6 #include <linux/mm.h> 7 #include <linux/rbtree.h> 8 #include <linux/slab.h> 9 10 DEFINE_PER_CPU(void*, bpf_cgroup_storage); 11 12 #ifdef CONFIG_CGROUP_BPF 13 14 #define LOCAL_STORAGE_CREATE_FLAG_MASK \ 15 (BPF_F_NUMA_NODE | BPF_F_RDONLY | BPF_F_WRONLY) 16 17 struct bpf_cgroup_storage_map { 18 struct bpf_map map; 19 20 spinlock_t lock; 21 struct bpf_prog *prog; 22 struct rb_root root; 23 struct list_head list; 24 }; 25 26 static struct bpf_cgroup_storage_map *map_to_storage(struct bpf_map *map) 27 { 28 return container_of(map, struct bpf_cgroup_storage_map, map); 29 } 30 31 static int bpf_cgroup_storage_key_cmp( 32 const struct bpf_cgroup_storage_key *key1, 33 const struct bpf_cgroup_storage_key *key2) 34 { 35 if (key1->cgroup_inode_id < key2->cgroup_inode_id) 36 return -1; 37 else if (key1->cgroup_inode_id > key2->cgroup_inode_id) 38 return 1; 39 else if (key1->attach_type < key2->attach_type) 40 return -1; 41 else if (key1->attach_type > key2->attach_type) 42 return 1; 43 return 0; 44 } 45 46 static struct bpf_cgroup_storage *cgroup_storage_lookup( 47 struct bpf_cgroup_storage_map *map, struct bpf_cgroup_storage_key *key, 48 bool locked) 49 { 50 struct rb_root *root = &map->root; 51 struct rb_node *node; 52 53 if (!locked) 54 spin_lock_bh(&map->lock); 55 56 node = root->rb_node; 57 while (node) { 58 struct bpf_cgroup_storage *storage; 59 60 storage = container_of(node, struct bpf_cgroup_storage, node); 61 62 switch (bpf_cgroup_storage_key_cmp(key, &storage->key)) { 63 case -1: 64 node = node->rb_left; 65 break; 66 case 1: 67 node = node->rb_right; 68 break; 69 default: 70 if (!locked) 71 spin_unlock_bh(&map->lock); 72 return storage; 73 } 74 } 75 76 if (!locked) 77 spin_unlock_bh(&map->lock); 78 79 return NULL; 80 } 81 82 static int cgroup_storage_insert(struct bpf_cgroup_storage_map *map, 83 struct bpf_cgroup_storage *storage) 84 { 85 struct rb_root *root = &map->root; 86 struct rb_node **new = &(root->rb_node), *parent = NULL; 87 88 while (*new) { 89 struct bpf_cgroup_storage *this; 90 91 this = container_of(*new, struct bpf_cgroup_storage, node); 92 93 parent = *new; 94 switch (bpf_cgroup_storage_key_cmp(&storage->key, &this->key)) { 95 case -1: 96 new = &((*new)->rb_left); 97 break; 98 case 1: 99 new = &((*new)->rb_right); 100 break; 101 default: 102 return -EEXIST; 103 } 104 } 105 106 rb_link_node(&storage->node, parent, new); 107 rb_insert_color(&storage->node, root); 108 109 return 0; 110 } 111 112 static void *cgroup_storage_lookup_elem(struct bpf_map *_map, void *_key) 113 { 114 struct bpf_cgroup_storage_map *map = map_to_storage(_map); 115 struct bpf_cgroup_storage_key *key = _key; 116 struct bpf_cgroup_storage *storage; 117 118 storage = cgroup_storage_lookup(map, key, false); 119 if (!storage) 120 return NULL; 121 122 return &READ_ONCE(storage->buf)->data[0]; 123 } 124 125 static int cgroup_storage_update_elem(struct bpf_map *map, void *_key, 126 void *value, u64 flags) 127 { 128 struct bpf_cgroup_storage_key *key = _key; 129 struct bpf_cgroup_storage *storage; 130 struct bpf_storage_buffer *new; 131 132 if (flags != BPF_ANY && flags != BPF_EXIST) 133 return -EINVAL; 134 135 storage = cgroup_storage_lookup((struct bpf_cgroup_storage_map *)map, 136 key, false); 137 if (!storage) 138 return -ENOENT; 139 140 new = kmalloc_node(sizeof(struct bpf_storage_buffer) + 141 map->value_size, __GFP_ZERO | GFP_USER, 142 map->numa_node); 143 if (!new) 144 return -ENOMEM; 145 146 memcpy(&new->data[0], value, map->value_size); 147 148 new = xchg(&storage->buf, new); 149 kfree_rcu(new, rcu); 150 151 return 0; 152 } 153 154 static int cgroup_storage_get_next_key(struct bpf_map *_map, void *_key, 155 void *_next_key) 156 { 157 struct bpf_cgroup_storage_map *map = map_to_storage(_map); 158 struct bpf_cgroup_storage_key *key = _key; 159 struct bpf_cgroup_storage_key *next = _next_key; 160 struct bpf_cgroup_storage *storage; 161 162 spin_lock_bh(&map->lock); 163 164 if (list_empty(&map->list)) 165 goto enoent; 166 167 if (key) { 168 storage = cgroup_storage_lookup(map, key, true); 169 if (!storage) 170 goto enoent; 171 172 storage = list_next_entry(storage, list); 173 if (!storage) 174 goto enoent; 175 } else { 176 storage = list_first_entry(&map->list, 177 struct bpf_cgroup_storage, list); 178 } 179 180 spin_unlock_bh(&map->lock); 181 next->attach_type = storage->key.attach_type; 182 next->cgroup_inode_id = storage->key.cgroup_inode_id; 183 return 0; 184 185 enoent: 186 spin_unlock_bh(&map->lock); 187 return -ENOENT; 188 } 189 190 static struct bpf_map *cgroup_storage_map_alloc(union bpf_attr *attr) 191 { 192 int numa_node = bpf_map_attr_numa_node(attr); 193 struct bpf_cgroup_storage_map *map; 194 195 if (attr->key_size != sizeof(struct bpf_cgroup_storage_key)) 196 return ERR_PTR(-EINVAL); 197 198 if (attr->value_size == 0) 199 return ERR_PTR(-EINVAL); 200 201 if (attr->value_size > PAGE_SIZE) 202 return ERR_PTR(-E2BIG); 203 204 if (attr->map_flags & ~LOCAL_STORAGE_CREATE_FLAG_MASK) 205 /* reserved bits should not be used */ 206 return ERR_PTR(-EINVAL); 207 208 if (attr->max_entries) 209 /* max_entries is not used and enforced to be 0 */ 210 return ERR_PTR(-EINVAL); 211 212 map = kmalloc_node(sizeof(struct bpf_cgroup_storage_map), 213 __GFP_ZERO | GFP_USER, numa_node); 214 if (!map) 215 return ERR_PTR(-ENOMEM); 216 217 map->map.pages = round_up(sizeof(struct bpf_cgroup_storage_map), 218 PAGE_SIZE) >> PAGE_SHIFT; 219 220 /* copy mandatory map attributes */ 221 bpf_map_init_from_attr(&map->map, attr); 222 223 spin_lock_init(&map->lock); 224 map->root = RB_ROOT; 225 INIT_LIST_HEAD(&map->list); 226 227 return &map->map; 228 } 229 230 static void cgroup_storage_map_free(struct bpf_map *_map) 231 { 232 struct bpf_cgroup_storage_map *map = map_to_storage(_map); 233 234 WARN_ON(!RB_EMPTY_ROOT(&map->root)); 235 WARN_ON(!list_empty(&map->list)); 236 237 kfree(map); 238 } 239 240 static int cgroup_storage_delete_elem(struct bpf_map *map, void *key) 241 { 242 return -EINVAL; 243 } 244 245 const struct bpf_map_ops cgroup_storage_map_ops = { 246 .map_alloc = cgroup_storage_map_alloc, 247 .map_free = cgroup_storage_map_free, 248 .map_get_next_key = cgroup_storage_get_next_key, 249 .map_lookup_elem = cgroup_storage_lookup_elem, 250 .map_update_elem = cgroup_storage_update_elem, 251 .map_delete_elem = cgroup_storage_delete_elem, 252 .map_check_btf = map_check_no_btf, 253 }; 254 255 int bpf_cgroup_storage_assign(struct bpf_prog *prog, struct bpf_map *_map) 256 { 257 struct bpf_cgroup_storage_map *map = map_to_storage(_map); 258 int ret = -EBUSY; 259 260 spin_lock_bh(&map->lock); 261 262 if (map->prog && map->prog != prog) 263 goto unlock; 264 if (prog->aux->cgroup_storage && prog->aux->cgroup_storage != _map) 265 goto unlock; 266 267 map->prog = prog; 268 prog->aux->cgroup_storage = _map; 269 ret = 0; 270 unlock: 271 spin_unlock_bh(&map->lock); 272 273 return ret; 274 } 275 276 void bpf_cgroup_storage_release(struct bpf_prog *prog, struct bpf_map *_map) 277 { 278 struct bpf_cgroup_storage_map *map = map_to_storage(_map); 279 280 spin_lock_bh(&map->lock); 281 if (map->prog == prog) { 282 WARN_ON(prog->aux->cgroup_storage != _map); 283 map->prog = NULL; 284 prog->aux->cgroup_storage = NULL; 285 } 286 spin_unlock_bh(&map->lock); 287 } 288 289 struct bpf_cgroup_storage *bpf_cgroup_storage_alloc(struct bpf_prog *prog) 290 { 291 struct bpf_cgroup_storage *storage; 292 struct bpf_map *map; 293 u32 pages; 294 295 map = prog->aux->cgroup_storage; 296 if (!map) 297 return NULL; 298 299 pages = round_up(sizeof(struct bpf_cgroup_storage) + 300 sizeof(struct bpf_storage_buffer) + 301 map->value_size, PAGE_SIZE) >> PAGE_SHIFT; 302 if (bpf_map_charge_memlock(map, pages)) 303 return ERR_PTR(-EPERM); 304 305 storage = kmalloc_node(sizeof(struct bpf_cgroup_storage), 306 __GFP_ZERO | GFP_USER, map->numa_node); 307 if (!storage) { 308 bpf_map_uncharge_memlock(map, pages); 309 return ERR_PTR(-ENOMEM); 310 } 311 312 storage->buf = kmalloc_node(sizeof(struct bpf_storage_buffer) + 313 map->value_size, __GFP_ZERO | GFP_USER, 314 map->numa_node); 315 if (!storage->buf) { 316 bpf_map_uncharge_memlock(map, pages); 317 kfree(storage); 318 return ERR_PTR(-ENOMEM); 319 } 320 321 storage->map = (struct bpf_cgroup_storage_map *)map; 322 323 return storage; 324 } 325 326 void bpf_cgroup_storage_free(struct bpf_cgroup_storage *storage) 327 { 328 u32 pages; 329 struct bpf_map *map; 330 331 if (!storage) 332 return; 333 334 map = &storage->map->map; 335 pages = round_up(sizeof(struct bpf_cgroup_storage) + 336 sizeof(struct bpf_storage_buffer) + 337 map->value_size, PAGE_SIZE) >> PAGE_SHIFT; 338 bpf_map_uncharge_memlock(map, pages); 339 340 kfree_rcu(storage->buf, rcu); 341 kfree_rcu(storage, rcu); 342 } 343 344 void bpf_cgroup_storage_link(struct bpf_cgroup_storage *storage, 345 struct cgroup *cgroup, 346 enum bpf_attach_type type) 347 { 348 struct bpf_cgroup_storage_map *map; 349 350 if (!storage) 351 return; 352 353 storage->key.attach_type = type; 354 storage->key.cgroup_inode_id = cgroup->kn->id.id; 355 356 map = storage->map; 357 358 spin_lock_bh(&map->lock); 359 WARN_ON(cgroup_storage_insert(map, storage)); 360 list_add(&storage->list, &map->list); 361 spin_unlock_bh(&map->lock); 362 } 363 364 void bpf_cgroup_storage_unlink(struct bpf_cgroup_storage *storage) 365 { 366 struct bpf_cgroup_storage_map *map; 367 struct rb_root *root; 368 369 if (!storage) 370 return; 371 372 map = storage->map; 373 374 spin_lock_bh(&map->lock); 375 root = &map->root; 376 rb_erase(&storage->node, root); 377 378 list_del(&storage->list); 379 spin_unlock_bh(&map->lock); 380 } 381 382 #endif 383