cgroup.c (5bf7a60b8e70969f65c961d7e2c4eb40eb2c664d) cgroup.c (8bad74f9840f87661f20ced3dc80c84ab4fd55a1)
1/*
2 * Functions to manage eBPF programs attached to cgroups
3 *
4 * Copyright (c) 2016 Daniel Mack
5 *
6 * This file is subject to the terms and conditions of version 2 of the GNU
7 * General Public License. See the file COPYING in the main directory of the
8 * Linux distribution for more details.

--- 11 unchanged lines hidden (view full) ---

20EXPORT_SYMBOL(cgroup_bpf_enabled_key);
21
22/**
23 * cgroup_bpf_put() - put references of all bpf programs
24 * @cgrp: the cgroup to modify
25 */
26void cgroup_bpf_put(struct cgroup *cgrp)
27{
1/*
2 * Functions to manage eBPF programs attached to cgroups
3 *
4 * Copyright (c) 2016 Daniel Mack
5 *
6 * This file is subject to the terms and conditions of version 2 of the GNU
7 * General Public License. See the file COPYING in the main directory of the
8 * Linux distribution for more details.

--- 11 unchanged lines hidden (view full) ---

20EXPORT_SYMBOL(cgroup_bpf_enabled_key);
21
22/**
23 * cgroup_bpf_put() - put references of all bpf programs
24 * @cgrp: the cgroup to modify
25 */
26void cgroup_bpf_put(struct cgroup *cgrp)
27{
28 enum bpf_cgroup_storage_type stype;
28 unsigned int type;
29
30 for (type = 0; type < ARRAY_SIZE(cgrp->bpf.progs); type++) {
31 struct list_head *progs = &cgrp->bpf.progs[type];
32 struct bpf_prog_list *pl, *tmp;
33
34 list_for_each_entry_safe(pl, tmp, progs, node) {
35 list_del(&pl->node);
36 bpf_prog_put(pl->prog);
29 unsigned int type;
30
31 for (type = 0; type < ARRAY_SIZE(cgrp->bpf.progs); type++) {
32 struct list_head *progs = &cgrp->bpf.progs[type];
33 struct bpf_prog_list *pl, *tmp;
34
35 list_for_each_entry_safe(pl, tmp, progs, node) {
36 list_del(&pl->node);
37 bpf_prog_put(pl->prog);
37 bpf_cgroup_storage_unlink(pl->storage);
38 bpf_cgroup_storage_free(pl->storage);
38 for_each_cgroup_storage_type(stype) {
39 bpf_cgroup_storage_unlink(pl->storage[stype]);
40 bpf_cgroup_storage_free(pl->storage[stype]);
41 }
39 kfree(pl);
40 static_branch_dec(&cgroup_bpf_enabled_key);
41 }
42 bpf_prog_array_free(cgrp->bpf.effective[type]);
43 }
44}
45
46/* count number of elements in the list.

--- 45 unchanged lines hidden (view full) ---

92 * all parent programs.
93 * Note that parent's F_ALLOW_OVERRIDE-type program is yielding
94 * to programs in this cgroup
95 */
96static int compute_effective_progs(struct cgroup *cgrp,
97 enum bpf_attach_type type,
98 struct bpf_prog_array __rcu **array)
99{
42 kfree(pl);
43 static_branch_dec(&cgroup_bpf_enabled_key);
44 }
45 bpf_prog_array_free(cgrp->bpf.effective[type]);
46 }
47}
48
49/* count number of elements in the list.

--- 45 unchanged lines hidden (view full) ---

95 * all parent programs.
96 * Note that parent's F_ALLOW_OVERRIDE-type program is yielding
97 * to programs in this cgroup
98 */
99static int compute_effective_progs(struct cgroup *cgrp,
100 enum bpf_attach_type type,
101 struct bpf_prog_array __rcu **array)
102{
103 enum bpf_cgroup_storage_type stype;
100 struct bpf_prog_array *progs;
101 struct bpf_prog_list *pl;
102 struct cgroup *p = cgrp;
103 int cnt = 0;
104
105 /* count number of effective programs by walking parents */
106 do {
107 if (cnt == 0 || (p->bpf.flags[type] & BPF_F_ALLOW_MULTI))

--- 12 unchanged lines hidden (view full) ---

120 if (cnt > 0 && !(p->bpf.flags[type] & BPF_F_ALLOW_MULTI))
121 continue;
122
123 list_for_each_entry(pl, &p->bpf.progs[type], node) {
124 if (!pl->prog)
125 continue;
126
127 progs->items[cnt].prog = pl->prog;
104 struct bpf_prog_array *progs;
105 struct bpf_prog_list *pl;
106 struct cgroup *p = cgrp;
107 int cnt = 0;
108
109 /* count number of effective programs by walking parents */
110 do {
111 if (cnt == 0 || (p->bpf.flags[type] & BPF_F_ALLOW_MULTI))

--- 12 unchanged lines hidden (view full) ---

124 if (cnt > 0 && !(p->bpf.flags[type] & BPF_F_ALLOW_MULTI))
125 continue;
126
127 list_for_each_entry(pl, &p->bpf.progs[type], node) {
128 if (!pl->prog)
129 continue;
130
131 progs->items[cnt].prog = pl->prog;
128 progs->items[cnt].cgroup_storage = pl->storage;
132 for_each_cgroup_storage_type(stype)
133 progs->items[cnt].cgroup_storage[stype] =
134 pl->storage[stype];
129 cnt++;
130 }
131 } while ((p = cgroup_parent(p)));
132
133 rcu_assign_pointer(*array, progs);
134 return 0;
135}
136

--- 90 unchanged lines hidden (view full) ---

227 *
228 * Must be called with cgroup_mutex held.
229 */
230int __cgroup_bpf_attach(struct cgroup *cgrp, struct bpf_prog *prog,
231 enum bpf_attach_type type, u32 flags)
232{
233 struct list_head *progs = &cgrp->bpf.progs[type];
234 struct bpf_prog *old_prog = NULL;
135 cnt++;
136 }
137 } while ((p = cgroup_parent(p)));
138
139 rcu_assign_pointer(*array, progs);
140 return 0;
141}
142

--- 90 unchanged lines hidden (view full) ---

233 *
234 * Must be called with cgroup_mutex held.
235 */
236int __cgroup_bpf_attach(struct cgroup *cgrp, struct bpf_prog *prog,
237 enum bpf_attach_type type, u32 flags)
238{
239 struct list_head *progs = &cgrp->bpf.progs[type];
240 struct bpf_prog *old_prog = NULL;
235 struct bpf_cgroup_storage *storage, *old_storage = NULL;
241 struct bpf_cgroup_storage *storage[MAX_BPF_CGROUP_STORAGE_TYPE],
242 *old_storage[MAX_BPF_CGROUP_STORAGE_TYPE] = {NULL};
243 enum bpf_cgroup_storage_type stype;
236 struct bpf_prog_list *pl;
237 bool pl_was_allocated;
238 int err;
239
240 if ((flags & BPF_F_ALLOW_OVERRIDE) && (flags & BPF_F_ALLOW_MULTI))
241 /* invalid combination */
242 return -EINVAL;
243

--- 5 unchanged lines hidden (view full) ---

249 * of existing overridable in this cgroup.
250 * Disallow attaching multi-prog if overridable or none
251 */
252 return -EPERM;
253
254 if (prog_list_length(progs) >= BPF_CGROUP_MAX_PROGS)
255 return -E2BIG;
256
244 struct bpf_prog_list *pl;
245 bool pl_was_allocated;
246 int err;
247
248 if ((flags & BPF_F_ALLOW_OVERRIDE) && (flags & BPF_F_ALLOW_MULTI))
249 /* invalid combination */
250 return -EINVAL;
251

--- 5 unchanged lines hidden (view full) ---

257 * of existing overridable in this cgroup.
258 * Disallow attaching multi-prog if overridable or none
259 */
260 return -EPERM;
261
262 if (prog_list_length(progs) >= BPF_CGROUP_MAX_PROGS)
263 return -E2BIG;
264
257 storage = bpf_cgroup_storage_alloc(prog);
258 if (IS_ERR(storage))
259 return -ENOMEM;
265 for_each_cgroup_storage_type(stype) {
266 storage[stype] = bpf_cgroup_storage_alloc(prog, stype);
267 if (IS_ERR(storage[stype])) {
268 storage[stype] = NULL;
269 for_each_cgroup_storage_type(stype)
270 bpf_cgroup_storage_free(storage[stype]);
271 return -ENOMEM;
272 }
273 }
260
261 if (flags & BPF_F_ALLOW_MULTI) {
262 list_for_each_entry(pl, progs, node) {
263 if (pl->prog == prog) {
264 /* disallow attaching the same prog twice */
274
275 if (flags & BPF_F_ALLOW_MULTI) {
276 list_for_each_entry(pl, progs, node) {
277 if (pl->prog == prog) {
278 /* disallow attaching the same prog twice */
265 bpf_cgroup_storage_free(storage);
279 for_each_cgroup_storage_type(stype)
280 bpf_cgroup_storage_free(storage[stype]);
266 return -EINVAL;
267 }
268 }
269
270 pl = kmalloc(sizeof(*pl), GFP_KERNEL);
271 if (!pl) {
281 return -EINVAL;
282 }
283 }
284
285 pl = kmalloc(sizeof(*pl), GFP_KERNEL);
286 if (!pl) {
272 bpf_cgroup_storage_free(storage);
287 for_each_cgroup_storage_type(stype)
288 bpf_cgroup_storage_free(storage[stype]);
273 return -ENOMEM;
274 }
275
276 pl_was_allocated = true;
277 pl->prog = prog;
289 return -ENOMEM;
290 }
291
292 pl_was_allocated = true;
293 pl->prog = prog;
278 pl->storage = storage;
294 for_each_cgroup_storage_type(stype)
295 pl->storage[stype] = storage[stype];
279 list_add_tail(&pl->node, progs);
280 } else {
281 if (list_empty(progs)) {
282 pl = kmalloc(sizeof(*pl), GFP_KERNEL);
283 if (!pl) {
296 list_add_tail(&pl->node, progs);
297 } else {
298 if (list_empty(progs)) {
299 pl = kmalloc(sizeof(*pl), GFP_KERNEL);
300 if (!pl) {
284 bpf_cgroup_storage_free(storage);
301 for_each_cgroup_storage_type(stype)
302 bpf_cgroup_storage_free(storage[stype]);
285 return -ENOMEM;
286 }
287 pl_was_allocated = true;
288 list_add_tail(&pl->node, progs);
289 } else {
290 pl = list_first_entry(progs, typeof(*pl), node);
291 old_prog = pl->prog;
303 return -ENOMEM;
304 }
305 pl_was_allocated = true;
306 list_add_tail(&pl->node, progs);
307 } else {
308 pl = list_first_entry(progs, typeof(*pl), node);
309 old_prog = pl->prog;
292 old_storage = pl->storage;
293 bpf_cgroup_storage_unlink(old_storage);
310 for_each_cgroup_storage_type(stype) {
311 old_storage[stype] = pl->storage[stype];
312 bpf_cgroup_storage_unlink(old_storage[stype]);
313 }
294 pl_was_allocated = false;
295 }
296 pl->prog = prog;
314 pl_was_allocated = false;
315 }
316 pl->prog = prog;
297 pl->storage = storage;
317 for_each_cgroup_storage_type(stype)
318 pl->storage[stype] = storage[stype];
298 }
299
300 cgrp->bpf.flags[type] = flags;
301
302 err = update_effective_progs(cgrp, type);
303 if (err)
304 goto cleanup;
305
306 static_branch_inc(&cgroup_bpf_enabled_key);
319 }
320
321 cgrp->bpf.flags[type] = flags;
322
323 err = update_effective_progs(cgrp, type);
324 if (err)
325 goto cleanup;
326
327 static_branch_inc(&cgroup_bpf_enabled_key);
307 if (old_storage)
308 bpf_cgroup_storage_free(old_storage);
328 for_each_cgroup_storage_type(stype) {
329 if (!old_storage[stype])
330 continue;
331 bpf_cgroup_storage_free(old_storage[stype]);
332 }
309 if (old_prog) {
310 bpf_prog_put(old_prog);
311 static_branch_dec(&cgroup_bpf_enabled_key);
312 }
333 if (old_prog) {
334 bpf_prog_put(old_prog);
335 static_branch_dec(&cgroup_bpf_enabled_key);
336 }
313 bpf_cgroup_storage_link(storage, cgrp, type);
337 for_each_cgroup_storage_type(stype)
338 bpf_cgroup_storage_link(storage[stype], cgrp, type);
314 return 0;
315
316cleanup:
317 /* and cleanup the prog list */
318 pl->prog = old_prog;
339 return 0;
340
341cleanup:
342 /* and cleanup the prog list */
343 pl->prog = old_prog;
319 bpf_cgroup_storage_free(pl->storage);
320 pl->storage = old_storage;
321 bpf_cgroup_storage_link(old_storage, cgrp, type);
344 for_each_cgroup_storage_type(stype) {
345 bpf_cgroup_storage_free(pl->storage[stype]);
346 pl->storage[stype] = old_storage[stype];
347 bpf_cgroup_storage_link(old_storage[stype], cgrp, type);
348 }
322 if (pl_was_allocated) {
323 list_del(&pl->node);
324 kfree(pl);
325 }
326 return err;
327}
328
329/**

--- 4 unchanged lines hidden (view full) ---

334 * @type: Type of detach operation
335 *
336 * Must be called with cgroup_mutex held.
337 */
338int __cgroup_bpf_detach(struct cgroup *cgrp, struct bpf_prog *prog,
339 enum bpf_attach_type type, u32 unused_flags)
340{
341 struct list_head *progs = &cgrp->bpf.progs[type];
349 if (pl_was_allocated) {
350 list_del(&pl->node);
351 kfree(pl);
352 }
353 return err;
354}
355
356/**

--- 4 unchanged lines hidden (view full) ---

361 * @type: Type of detach operation
362 *
363 * Must be called with cgroup_mutex held.
364 */
365int __cgroup_bpf_detach(struct cgroup *cgrp, struct bpf_prog *prog,
366 enum bpf_attach_type type, u32 unused_flags)
367{
368 struct list_head *progs = &cgrp->bpf.progs[type];
369 enum bpf_cgroup_storage_type stype;
342 u32 flags = cgrp->bpf.flags[type];
343 struct bpf_prog *old_prog = NULL;
344 struct bpf_prog_list *pl;
345 int err;
346
347 if (flags & BPF_F_ALLOW_MULTI) {
348 if (!prog)
349 /* to detach MULTI prog the user has to specify valid FD

--- 30 unchanged lines hidden (view full) ---

380 }
381
382 err = update_effective_progs(cgrp, type);
383 if (err)
384 goto cleanup;
385
386 /* now can actually delete it from this cgroup list */
387 list_del(&pl->node);
370 u32 flags = cgrp->bpf.flags[type];
371 struct bpf_prog *old_prog = NULL;
372 struct bpf_prog_list *pl;
373 int err;
374
375 if (flags & BPF_F_ALLOW_MULTI) {
376 if (!prog)
377 /* to detach MULTI prog the user has to specify valid FD

--- 30 unchanged lines hidden (view full) ---

408 }
409
410 err = update_effective_progs(cgrp, type);
411 if (err)
412 goto cleanup;
413
414 /* now can actually delete it from this cgroup list */
415 list_del(&pl->node);
388 bpf_cgroup_storage_unlink(pl->storage);
389 bpf_cgroup_storage_free(pl->storage);
416 for_each_cgroup_storage_type(stype) {
417 bpf_cgroup_storage_unlink(pl->storage[stype]);
418 bpf_cgroup_storage_free(pl->storage[stype]);
419 }
390 kfree(pl);
391 if (list_empty(progs))
392 /* last program was detached, reset flags to zero */
393 cgrp->bpf.flags[type] = 0;
394
395 bpf_prog_put(old_prog);
396 static_branch_dec(&cgroup_bpf_enabled_key);
397 return 0;

--- 329 unchanged lines hidden --- 		420 kfree(pl);
421 if (list_empty(progs))
422 /* last program was detached, reset flags to zero */
423 cgrp->bpf.flags[type] = 0;
424
425 bpf_prog_put(old_prog);
426 static_branch_dec(&cgroup_bpf_enabled_key);
427 return 0;

--- 329 unchanged lines hidden ---