1 /* 2 * Generic process-grouping system. 3 * 4 * Based originally on the cpuset system, extracted by Paul Menage 5 * Copyright (C) 2006 Google, Inc 6 * 7 * Notifications support 8 * Copyright (C) 2009 Nokia Corporation 9 * Author: Kirill A. Shutemov 10 * 11 * Copyright notices from the original cpuset code: 12 * -------------------------------------------------- 13 * Copyright (C) 2003 BULL SA. 14 * Copyright (C) 2004-2006 Silicon Graphics, Inc. 15 * 16 * Portions derived from Patrick Mochel's sysfs code. 17 * sysfs is Copyright (c) 2001-3 Patrick Mochel 18 * 19 * 2003-10-10 Written by Simon Derr. 20 * 2003-10-22 Updates by Stephen Hemminger. 21 * 2004 May-July Rework by Paul Jackson. 22 * --------------------------------------------------- 23 * 24 * This file is subject to the terms and conditions of the GNU General Public 25 * License. See the file COPYING in the main directory of the Linux 26 * distribution for more details. 27 */ 28 29 #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt 30 31 #include "cgroup-internal.h" 32 33 #include <linux/cred.h> 34 #include <linux/errno.h> 35 #include <linux/init_task.h> 36 #include <linux/kernel.h> 37 #include <linux/magic.h> 38 #include <linux/mutex.h> 39 #include <linux/mount.h> 40 #include <linux/pagemap.h> 41 #include <linux/proc_fs.h> 42 #include <linux/rcupdate.h> 43 #include <linux/sched.h> 44 #include <linux/sched/task.h> 45 #include <linux/slab.h> 46 #include <linux/spinlock.h> 47 #include <linux/percpu-rwsem.h> 48 #include <linux/string.h> 49 #include <linux/hashtable.h> 50 #include <linux/idr.h> 51 #include <linux/kthread.h> 52 #include <linux/atomic.h> 53 #include <linux/cpuset.h> 54 #include <linux/proc_ns.h> 55 #include <linux/nsproxy.h> 56 #include <linux/file.h> 57 #include <linux/sched/cputime.h> 58 #include <linux/psi.h> 59 #include <net/sock.h> 60 61 #define CREATE_TRACE_POINTS 62 #include <trace/events/cgroup.h> 63 64 #define CGROUP_FILE_NAME_MAX (MAX_CGROUP_TYPE_NAMELEN + \ 65 MAX_CFTYPE_NAME + 2) 66 /* let's not notify more than 100 times per second */ 67 #define CGROUP_FILE_NOTIFY_MIN_INTV DIV_ROUND_UP(HZ, 100) 68 69 /* 70 * cgroup_mutex is the master lock. Any modification to cgroup or its 71 * hierarchy must be performed while holding it. 72 * 73 * css_set_lock protects task->cgroups pointer, the list of css_set 74 * objects, and the chain of tasks off each css_set. 75 * 76 * These locks are exported if CONFIG_PROVE_RCU so that accessors in 77 * cgroup.h can use them for lockdep annotations. 78 */ 79 DEFINE_MUTEX(cgroup_mutex); 80 DEFINE_SPINLOCK(css_set_lock); 81 82 #ifdef CONFIG_PROVE_RCU 83 EXPORT_SYMBOL_GPL(cgroup_mutex); 84 EXPORT_SYMBOL_GPL(css_set_lock); 85 #endif 86 87 DEFINE_SPINLOCK(trace_cgroup_path_lock); 88 char trace_cgroup_path[TRACE_CGROUP_PATH_LEN]; 89 90 /* 91 * Protects cgroup_idr and css_idr so that IDs can be released without 92 * grabbing cgroup_mutex. 93 */ 94 static DEFINE_SPINLOCK(cgroup_idr_lock); 95 96 /* 97 * Protects cgroup_file->kn for !self csses. It synchronizes notifications 98 * against file removal/re-creation across css hiding. 99 */ 100 static DEFINE_SPINLOCK(cgroup_file_kn_lock); 101 102 struct percpu_rw_semaphore cgroup_threadgroup_rwsem; 103 104 #define cgroup_assert_mutex_or_rcu_locked() \ 105 RCU_LOCKDEP_WARN(!rcu_read_lock_held() && \ 106 !lockdep_is_held(&cgroup_mutex), \ 107 "cgroup_mutex or RCU read lock required"); 108 109 /* 110 * cgroup destruction makes heavy use of work items and there can be a lot 111 * of concurrent destructions. Use a separate workqueue so that cgroup 112 * destruction work items don't end up filling up max_active of system_wq 113 * which may lead to deadlock. 114 */ 115 static struct workqueue_struct *cgroup_destroy_wq; 116 117 /* generate an array of cgroup subsystem pointers */ 118 #define SUBSYS(_x) [_x ## _cgrp_id] = &_x ## _cgrp_subsys, 119 struct cgroup_subsys *cgroup_subsys[] = { 120 #include <linux/cgroup_subsys.h> 121 }; 122 #undef SUBSYS 123 124 /* array of cgroup subsystem names */ 125 #define SUBSYS(_x) [_x ## _cgrp_id] = #_x, 126 static const char *cgroup_subsys_name[] = { 127 #include <linux/cgroup_subsys.h> 128 }; 129 #undef SUBSYS 130 131 /* array of static_keys for cgroup_subsys_enabled() and cgroup_subsys_on_dfl() */ 132 #define SUBSYS(_x) \ 133 DEFINE_STATIC_KEY_TRUE(_x ## _cgrp_subsys_enabled_key); \ 134 DEFINE_STATIC_KEY_TRUE(_x ## _cgrp_subsys_on_dfl_key); \ 135 EXPORT_SYMBOL_GPL(_x ## _cgrp_subsys_enabled_key); \ 136 EXPORT_SYMBOL_GPL(_x ## _cgrp_subsys_on_dfl_key); 137 #include <linux/cgroup_subsys.h> 138 #undef SUBSYS 139 140 #define SUBSYS(_x) [_x ## _cgrp_id] = &_x ## _cgrp_subsys_enabled_key, 141 static struct static_key_true *cgroup_subsys_enabled_key[] = { 142 #include <linux/cgroup_subsys.h> 143 }; 144 #undef SUBSYS 145 146 #define SUBSYS(_x) [_x ## _cgrp_id] = &_x ## _cgrp_subsys_on_dfl_key, 147 static struct static_key_true *cgroup_subsys_on_dfl_key[] = { 148 #include <linux/cgroup_subsys.h> 149 }; 150 #undef SUBSYS 151 152 static DEFINE_PER_CPU(struct cgroup_rstat_cpu, cgrp_dfl_root_rstat_cpu); 153 154 /* 155 * The default hierarchy, reserved for the subsystems that are otherwise 156 * unattached - it never has more than a single cgroup, and all tasks are 157 * part of that cgroup. 158 */ 159 struct cgroup_root cgrp_dfl_root = { .cgrp.rstat_cpu = &cgrp_dfl_root_rstat_cpu }; 160 EXPORT_SYMBOL_GPL(cgrp_dfl_root); 161 162 /* 163 * The default hierarchy always exists but is hidden until mounted for the 164 * first time. This is for backward compatibility. 165 */ 166 static bool cgrp_dfl_visible; 167 168 /* some controllers are not supported in the default hierarchy */ 169 static u16 cgrp_dfl_inhibit_ss_mask; 170 171 /* some controllers are implicitly enabled on the default hierarchy */ 172 static u16 cgrp_dfl_implicit_ss_mask; 173 174 /* some controllers can be threaded on the default hierarchy */ 175 static u16 cgrp_dfl_threaded_ss_mask; 176 177 /* The list of hierarchy roots */ 178 LIST_HEAD(cgroup_roots); 179 static int cgroup_root_count; 180 181 /* hierarchy ID allocation and mapping, protected by cgroup_mutex */ 182 static DEFINE_IDR(cgroup_hierarchy_idr); 183 184 /* 185 * Assign a monotonically increasing serial number to csses. It guarantees 186 * cgroups with bigger numbers are newer than those with smaller numbers. 187 * Also, as csses are always appended to the parent's ->children list, it 188 * guarantees that sibling csses are always sorted in the ascending serial 189 * number order on the list. Protected by cgroup_mutex. 190 */ 191 static u64 css_serial_nr_next = 1; 192 193 /* 194 * These bitmasks identify subsystems with specific features to avoid 195 * having to do iterative checks repeatedly. 196 */ 197 static u16 have_fork_callback __read_mostly; 198 static u16 have_exit_callback __read_mostly; 199 static u16 have_free_callback __read_mostly; 200 static u16 have_canfork_callback __read_mostly; 201 202 /* cgroup namespace for init task */ 203 struct cgroup_namespace init_cgroup_ns = { 204 .count = REFCOUNT_INIT(2), 205 .user_ns = &init_user_ns, 206 .ns.ops = &cgroupns_operations, 207 .ns.inum = PROC_CGROUP_INIT_INO, 208 .root_cset = &init_css_set, 209 }; 210 211 static struct file_system_type cgroup2_fs_type; 212 static struct cftype cgroup_base_files[]; 213 214 static int cgroup_apply_control(struct cgroup *cgrp); 215 static void cgroup_finalize_control(struct cgroup *cgrp, int ret); 216 static void css_task_iter_advance(struct css_task_iter *it); 217 static int cgroup_destroy_locked(struct cgroup *cgrp); 218 static struct cgroup_subsys_state *css_create(struct cgroup *cgrp, 219 struct cgroup_subsys *ss); 220 static void css_release(struct percpu_ref *ref); 221 static void kill_css(struct cgroup_subsys_state *css); 222 static int cgroup_addrm_files(struct cgroup_subsys_state *css, 223 struct cgroup *cgrp, struct cftype cfts[], 224 bool is_add); 225 226 /** 227 * cgroup_ssid_enabled - cgroup subsys enabled test by subsys ID 228 * @ssid: subsys ID of interest 229 * 230 * cgroup_subsys_enabled() can only be used with literal subsys names which 231 * is fine for individual subsystems but unsuitable for cgroup core. This 232 * is slower static_key_enabled() based test indexed by @ssid. 233 */ 234 bool cgroup_ssid_enabled(int ssid) 235 { 236 if (CGROUP_SUBSYS_COUNT == 0) 237 return false; 238 239 return static_key_enabled(cgroup_subsys_enabled_key[ssid]); 240 } 241 242 /** 243 * cgroup_on_dfl - test whether a cgroup is on the default hierarchy 244 * @cgrp: the cgroup of interest 245 * 246 * The default hierarchy is the v2 interface of cgroup and this function 247 * can be used to test whether a cgroup is on the default hierarchy for 248 * cases where a subsystem should behave differnetly depending on the 249 * interface version. 250 * 251 * The set of behaviors which change on the default hierarchy are still 252 * being determined and the mount option is prefixed with __DEVEL__. 253 * 254 * List of changed behaviors: 255 * 256 * - Mount options "noprefix", "xattr", "clone_children", "release_agent" 257 * and "name" are disallowed. 258 * 259 * - When mounting an existing superblock, mount options should match. 260 * 261 * - Remount is disallowed. 262 * 263 * - rename(2) is disallowed. 264 * 265 * - "tasks" is removed. Everything should be at process granularity. Use 266 * "cgroup.procs" instead. 267 * 268 * - "cgroup.procs" is not sorted. pids will be unique unless they got 269 * recycled inbetween reads. 270 * 271 * - "release_agent" and "notify_on_release" are removed. Replacement 272 * notification mechanism will be implemented. 273 * 274 * - "cgroup.clone_children" is removed. 275 * 276 * - "cgroup.subtree_populated" is available. Its value is 0 if the cgroup 277 * and its descendants contain no task; otherwise, 1. The file also 278 * generates kernfs notification which can be monitored through poll and 279 * [di]notify when the value of the file changes. 280 * 281 * - cpuset: tasks will be kept in empty cpusets when hotplug happens and 282 * take masks of ancestors with non-empty cpus/mems, instead of being 283 * moved to an ancestor. 284 * 285 * - cpuset: a task can be moved into an empty cpuset, and again it takes 286 * masks of ancestors. 287 * 288 * - memcg: use_hierarchy is on by default and the cgroup file for the flag 289 * is not created. 290 * 291 * - blkcg: blk-throttle becomes properly hierarchical. 292 * 293 * - debug: disallowed on the default hierarchy. 294 */ 295 bool cgroup_on_dfl(const struct cgroup *cgrp) 296 { 297 return cgrp->root == &cgrp_dfl_root; 298 } 299 300 /* IDR wrappers which synchronize using cgroup_idr_lock */ 301 static int cgroup_idr_alloc(struct idr *idr, void *ptr, int start, int end, 302 gfp_t gfp_mask) 303 { 304 int ret; 305 306 idr_preload(gfp_mask); 307 spin_lock_bh(&cgroup_idr_lock); 308 ret = idr_alloc(idr, ptr, start, end, gfp_mask & ~__GFP_DIRECT_RECLAIM); 309 spin_unlock_bh(&cgroup_idr_lock); 310 idr_preload_end(); 311 return ret; 312 } 313 314 static void *cgroup_idr_replace(struct idr *idr, void *ptr, int id) 315 { 316 void *ret; 317 318 spin_lock_bh(&cgroup_idr_lock); 319 ret = idr_replace(idr, ptr, id); 320 spin_unlock_bh(&cgroup_idr_lock); 321 return ret; 322 } 323 324 static void cgroup_idr_remove(struct idr *idr, int id) 325 { 326 spin_lock_bh(&cgroup_idr_lock); 327 idr_remove(idr, id); 328 spin_unlock_bh(&cgroup_idr_lock); 329 } 330 331 static bool cgroup_has_tasks(struct cgroup *cgrp) 332 { 333 return cgrp->nr_populated_csets; 334 } 335 336 bool cgroup_is_threaded(struct cgroup *cgrp) 337 { 338 return cgrp->dom_cgrp != cgrp; 339 } 340 341 /* can @cgrp host both domain and threaded children? */ 342 static bool cgroup_is_mixable(struct cgroup *cgrp) 343 { 344 /* 345 * Root isn't under domain level resource control exempting it from 346 * the no-internal-process constraint, so it can serve as a thread 347 * root and a parent of resource domains at the same time. 348 */ 349 return !cgroup_parent(cgrp); 350 } 351 352 /* can @cgrp become a thread root? should always be true for a thread root */ 353 static bool cgroup_can_be_thread_root(struct cgroup *cgrp) 354 { 355 /* mixables don't care */ 356 if (cgroup_is_mixable(cgrp)) 357 return true; 358 359 /* domain roots can't be nested under threaded */ 360 if (cgroup_is_threaded(cgrp)) 361 return false; 362 363 /* can only have either domain or threaded children */ 364 if (cgrp->nr_populated_domain_children) 365 return false; 366 367 /* and no domain controllers can be enabled */ 368 if (cgrp->subtree_control & ~cgrp_dfl_threaded_ss_mask) 369 return false; 370 371 return true; 372 } 373 374 /* is @cgrp root of a threaded subtree? */ 375 bool cgroup_is_thread_root(struct cgroup *cgrp) 376 { 377 /* thread root should be a domain */ 378 if (cgroup_is_threaded(cgrp)) 379 return false; 380 381 /* a domain w/ threaded children is a thread root */ 382 if (cgrp->nr_threaded_children) 383 return true; 384 385 /* 386 * A domain which has tasks and explicit threaded controllers 387 * enabled is a thread root. 388 */ 389 if (cgroup_has_tasks(cgrp) && 390 (cgrp->subtree_control & cgrp_dfl_threaded_ss_mask)) 391 return true; 392 393 return false; 394 } 395 396 /* a domain which isn't connected to the root w/o brekage can't be used */ 397 static bool cgroup_is_valid_domain(struct cgroup *cgrp) 398 { 399 /* the cgroup itself can be a thread root */ 400 if (cgroup_is_threaded(cgrp)) 401 return false; 402 403 /* but the ancestors can't be unless mixable */ 404 while ((cgrp = cgroup_parent(cgrp))) { 405 if (!cgroup_is_mixable(cgrp) && cgroup_is_thread_root(cgrp)) 406 return false; 407 if (cgroup_is_threaded(cgrp)) 408 return false; 409 } 410 411 return true; 412 } 413 414 /* subsystems visibly enabled on a cgroup */ 415 static u16 cgroup_control(struct cgroup *cgrp) 416 { 417 struct cgroup *parent = cgroup_parent(cgrp); 418 u16 root_ss_mask = cgrp->root->subsys_mask; 419 420 if (parent) { 421 u16 ss_mask = parent->subtree_control; 422 423 /* threaded cgroups can only have threaded controllers */ 424 if (cgroup_is_threaded(cgrp)) 425 ss_mask &= cgrp_dfl_threaded_ss_mask; 426 return ss_mask; 427 } 428 429 if (cgroup_on_dfl(cgrp)) 430 root_ss_mask &= ~(cgrp_dfl_inhibit_ss_mask | 431 cgrp_dfl_implicit_ss_mask); 432 return root_ss_mask; 433 } 434 435 /* subsystems enabled on a cgroup */ 436 static u16 cgroup_ss_mask(struct cgroup *cgrp) 437 { 438 struct cgroup *parent = cgroup_parent(cgrp); 439 440 if (parent) { 441 u16 ss_mask = parent->subtree_ss_mask; 442 443 /* threaded cgroups can only have threaded controllers */ 444 if (cgroup_is_threaded(cgrp)) 445 ss_mask &= cgrp_dfl_threaded_ss_mask; 446 return ss_mask; 447 } 448 449 return cgrp->root->subsys_mask; 450 } 451 452 /** 453 * cgroup_css - obtain a cgroup's css for the specified subsystem 454 * @cgrp: the cgroup of interest 455 * @ss: the subsystem of interest (%NULL returns @cgrp->self) 456 * 457 * Return @cgrp's css (cgroup_subsys_state) associated with @ss. This 458 * function must be called either under cgroup_mutex or rcu_read_lock() and 459 * the caller is responsible for pinning the returned css if it wants to 460 * keep accessing it outside the said locks. This function may return 461 * %NULL if @cgrp doesn't have @subsys_id enabled. 462 */ 463 static struct cgroup_subsys_state *cgroup_css(struct cgroup *cgrp, 464 struct cgroup_subsys *ss) 465 { 466 if (ss) 467 return rcu_dereference_check(cgrp->subsys[ss->id], 468 lockdep_is_held(&cgroup_mutex)); 469 else 470 return &cgrp->self; 471 } 472 473 /** 474 * cgroup_tryget_css - try to get a cgroup's css for the specified subsystem 475 * @cgrp: the cgroup of interest 476 * @ss: the subsystem of interest 477 * 478 * Find and get @cgrp's css assocaited with @ss. If the css doesn't exist 479 * or is offline, %NULL is returned. 480 */ 481 static struct cgroup_subsys_state *cgroup_tryget_css(struct cgroup *cgrp, 482 struct cgroup_subsys *ss) 483 { 484 struct cgroup_subsys_state *css; 485 486 rcu_read_lock(); 487 css = cgroup_css(cgrp, ss); 488 if (!css || !css_tryget_online(css)) 489 css = NULL; 490 rcu_read_unlock(); 491 492 return css; 493 } 494 495 /** 496 * cgroup_e_css - obtain a cgroup's effective css for the specified subsystem 497 * @cgrp: the cgroup of interest 498 * @ss: the subsystem of interest (%NULL returns @cgrp->self) 499 * 500 * Similar to cgroup_css() but returns the effective css, which is defined 501 * as the matching css of the nearest ancestor including self which has @ss 502 * enabled. If @ss is associated with the hierarchy @cgrp is on, this 503 * function is guaranteed to return non-NULL css. 504 */ 505 static struct cgroup_subsys_state *cgroup_e_css(struct cgroup *cgrp, 506 struct cgroup_subsys *ss) 507 { 508 lockdep_assert_held(&cgroup_mutex); 509 510 if (!ss) 511 return &cgrp->self; 512 513 /* 514 * This function is used while updating css associations and thus 515 * can't test the csses directly. Test ss_mask. 516 */ 517 while (!(cgroup_ss_mask(cgrp) & (1 << ss->id))) { 518 cgrp = cgroup_parent(cgrp); 519 if (!cgrp) 520 return NULL; 521 } 522 523 return cgroup_css(cgrp, ss); 524 } 525 526 /** 527 * cgroup_get_e_css - get a cgroup's effective css for the specified subsystem 528 * @cgrp: the cgroup of interest 529 * @ss: the subsystem of interest 530 * 531 * Find and get the effective css of @cgrp for @ss. The effective css is 532 * defined as the matching css of the nearest ancestor including self which 533 * has @ss enabled. If @ss is not mounted on the hierarchy @cgrp is on, 534 * the root css is returned, so this function always returns a valid css. 535 * The returned css must be put using css_put(). 536 */ 537 struct cgroup_subsys_state *cgroup_get_e_css(struct cgroup *cgrp, 538 struct cgroup_subsys *ss) 539 { 540 struct cgroup_subsys_state *css; 541 542 rcu_read_lock(); 543 544 do { 545 css = cgroup_css(cgrp, ss); 546 547 if (css && css_tryget_online(css)) 548 goto out_unlock; 549 cgrp = cgroup_parent(cgrp); 550 } while (cgrp); 551 552 css = init_css_set.subsys[ss->id]; 553 css_get(css); 554 out_unlock: 555 rcu_read_unlock(); 556 return css; 557 } 558 559 static void cgroup_get_live(struct cgroup *cgrp) 560 { 561 WARN_ON_ONCE(cgroup_is_dead(cgrp)); 562 css_get(&cgrp->self); 563 } 564 565 struct cgroup_subsys_state *of_css(struct kernfs_open_file *of) 566 { 567 struct cgroup *cgrp = of->kn->parent->priv; 568 struct cftype *cft = of_cft(of); 569 570 /* 571 * This is open and unprotected implementation of cgroup_css(). 572 * seq_css() is only called from a kernfs file operation which has 573 * an active reference on the file. Because all the subsystem 574 * files are drained before a css is disassociated with a cgroup, 575 * the matching css from the cgroup's subsys table is guaranteed to 576 * be and stay valid until the enclosing operation is complete. 577 */ 578 if (cft->ss) 579 return rcu_dereference_raw(cgrp->subsys[cft->ss->id]); 580 else 581 return &cgrp->self; 582 } 583 EXPORT_SYMBOL_GPL(of_css); 584 585 /** 586 * for_each_css - iterate all css's of a cgroup 587 * @css: the iteration cursor 588 * @ssid: the index of the subsystem, CGROUP_SUBSYS_COUNT after reaching the end 589 * @cgrp: the target cgroup to iterate css's of 590 * 591 * Should be called under cgroup_[tree_]mutex. 592 */ 593 #define for_each_css(css, ssid, cgrp) \ 594 for ((ssid) = 0; (ssid) < CGROUP_SUBSYS_COUNT; (ssid)++) \ 595 if (!((css) = rcu_dereference_check( \ 596 (cgrp)->subsys[(ssid)], \ 597 lockdep_is_held(&cgroup_mutex)))) { } \ 598 else 599 600 /** 601 * for_each_e_css - iterate all effective css's of a cgroup 602 * @css: the iteration cursor 603 * @ssid: the index of the subsystem, CGROUP_SUBSYS_COUNT after reaching the end 604 * @cgrp: the target cgroup to iterate css's of 605 * 606 * Should be called under cgroup_[tree_]mutex. 607 */ 608 #define for_each_e_css(css, ssid, cgrp) \ 609 for ((ssid) = 0; (ssid) < CGROUP_SUBSYS_COUNT; (ssid)++) \ 610 if (!((css) = cgroup_e_css(cgrp, cgroup_subsys[(ssid)]))) \ 611 ; \ 612 else 613 614 /** 615 * do_each_subsys_mask - filter for_each_subsys with a bitmask 616 * @ss: the iteration cursor 617 * @ssid: the index of @ss, CGROUP_SUBSYS_COUNT after reaching the end 618 * @ss_mask: the bitmask 619 * 620 * The block will only run for cases where the ssid-th bit (1 << ssid) of 621 * @ss_mask is set. 622 */ 623 #define do_each_subsys_mask(ss, ssid, ss_mask) do { \ 624 unsigned long __ss_mask = (ss_mask); \ 625 if (!CGROUP_SUBSYS_COUNT) { /* to avoid spurious gcc warning */ \ 626 (ssid) = 0; \ 627 break; \ 628 } \ 629 for_each_set_bit(ssid, &__ss_mask, CGROUP_SUBSYS_COUNT) { \ 630 (ss) = cgroup_subsys[ssid]; \ 631 { 632 633 #define while_each_subsys_mask() \ 634 } \ 635 } \ 636 } while (false) 637 638 /* iterate over child cgrps, lock should be held throughout iteration */ 639 #define cgroup_for_each_live_child(child, cgrp) \ 640 list_for_each_entry((child), &(cgrp)->self.children, self.sibling) \ 641 if (({ lockdep_assert_held(&cgroup_mutex); \ 642 cgroup_is_dead(child); })) \ 643 ; \ 644 else 645 646 /* walk live descendants in preorder */ 647 #define cgroup_for_each_live_descendant_pre(dsct, d_css, cgrp) \ 648 css_for_each_descendant_pre((d_css), cgroup_css((cgrp), NULL)) \ 649 if (({ lockdep_assert_held(&cgroup_mutex); \ 650 (dsct) = (d_css)->cgroup; \ 651 cgroup_is_dead(dsct); })) \ 652 ; \ 653 else 654 655 /* walk live descendants in postorder */ 656 #define cgroup_for_each_live_descendant_post(dsct, d_css, cgrp) \ 657 css_for_each_descendant_post((d_css), cgroup_css((cgrp), NULL)) \ 658 if (({ lockdep_assert_held(&cgroup_mutex); \ 659 (dsct) = (d_css)->cgroup; \ 660 cgroup_is_dead(dsct); })) \ 661 ; \ 662 else 663 664 /* 665 * The default css_set - used by init and its children prior to any 666 * hierarchies being mounted. It contains a pointer to the root state 667 * for each subsystem. Also used to anchor the list of css_sets. Not 668 * reference-counted, to improve performance when child cgroups 669 * haven't been created. 670 */ 671 struct css_set init_css_set = { 672 .refcount = REFCOUNT_INIT(1), 673 .dom_cset = &init_css_set, 674 .tasks = LIST_HEAD_INIT(init_css_set.tasks), 675 .mg_tasks = LIST_HEAD_INIT(init_css_set.mg_tasks), 676 .task_iters = LIST_HEAD_INIT(init_css_set.task_iters), 677 .threaded_csets = LIST_HEAD_INIT(init_css_set.threaded_csets), 678 .cgrp_links = LIST_HEAD_INIT(init_css_set.cgrp_links), 679 .mg_preload_node = LIST_HEAD_INIT(init_css_set.mg_preload_node), 680 .mg_node = LIST_HEAD_INIT(init_css_set.mg_node), 681 682 /* 683 * The following field is re-initialized when this cset gets linked 684 * in cgroup_init(). However, let's initialize the field 685 * statically too so that the default cgroup can be accessed safely 686 * early during boot. 687 */ 688 .dfl_cgrp = &cgrp_dfl_root.cgrp, 689 }; 690 691 static int css_set_count = 1; /* 1 for init_css_set */ 692 693 static bool css_set_threaded(struct css_set *cset) 694 { 695 return cset->dom_cset != cset; 696 } 697 698 /** 699 * css_set_populated - does a css_set contain any tasks? 700 * @cset: target css_set 701 * 702 * css_set_populated() should be the same as !!cset->nr_tasks at steady 703 * state. However, css_set_populated() can be called while a task is being 704 * added to or removed from the linked list before the nr_tasks is 705 * properly updated. Hence, we can't just look at ->nr_tasks here. 706 */ 707 static bool css_set_populated(struct css_set *cset) 708 { 709 lockdep_assert_held(&css_set_lock); 710 711 return !list_empty(&cset->tasks) || !list_empty(&cset->mg_tasks); 712 } 713 714 /** 715 * cgroup_update_populated - update the populated count of a cgroup 716 * @cgrp: the target cgroup 717 * @populated: inc or dec populated count 718 * 719 * One of the css_sets associated with @cgrp is either getting its first 720 * task or losing the last. Update @cgrp->nr_populated_* accordingly. The 721 * count is propagated towards root so that a given cgroup's 722 * nr_populated_children is zero iff none of its descendants contain any 723 * tasks. 724 * 725 * @cgrp's interface file "cgroup.populated" is zero if both 726 * @cgrp->nr_populated_csets and @cgrp->nr_populated_children are zero and 727 * 1 otherwise. When the sum changes from or to zero, userland is notified 728 * that the content of the interface file has changed. This can be used to 729 * detect when @cgrp and its descendants become populated or empty. 730 */ 731 static void cgroup_update_populated(struct cgroup *cgrp, bool populated) 732 { 733 struct cgroup *child = NULL; 734 int adj = populated ? 1 : -1; 735 736 lockdep_assert_held(&css_set_lock); 737 738 do { 739 bool was_populated = cgroup_is_populated(cgrp); 740 741 if (!child) { 742 cgrp->nr_populated_csets += adj; 743 } else { 744 if (cgroup_is_threaded(child)) 745 cgrp->nr_populated_threaded_children += adj; 746 else 747 cgrp->nr_populated_domain_children += adj; 748 } 749 750 if (was_populated == cgroup_is_populated(cgrp)) 751 break; 752 753 cgroup1_check_for_release(cgrp); 754 cgroup_file_notify(&cgrp->events_file); 755 756 child = cgrp; 757 cgrp = cgroup_parent(cgrp); 758 } while (cgrp); 759 } 760 761 /** 762 * css_set_update_populated - update populated state of a css_set 763 * @cset: target css_set 764 * @populated: whether @cset is populated or depopulated 765 * 766 * @cset is either getting the first task or losing the last. Update the 767 * populated counters of all associated cgroups accordingly. 768 */ 769 static void css_set_update_populated(struct css_set *cset, bool populated) 770 { 771 struct cgrp_cset_link *link; 772 773 lockdep_assert_held(&css_set_lock); 774 775 list_for_each_entry(link, &cset->cgrp_links, cgrp_link) 776 cgroup_update_populated(link->cgrp, populated); 777 } 778 779 /** 780 * css_set_move_task - move a task from one css_set to another 781 * @task: task being moved 782 * @from_cset: css_set @task currently belongs to (may be NULL) 783 * @to_cset: new css_set @task is being moved to (may be NULL) 784 * @use_mg_tasks: move to @to_cset->mg_tasks instead of ->tasks 785 * 786 * Move @task from @from_cset to @to_cset. If @task didn't belong to any 787 * css_set, @from_cset can be NULL. If @task is being disassociated 788 * instead of moved, @to_cset can be NULL. 789 * 790 * This function automatically handles populated counter updates and 791 * css_task_iter adjustments but the caller is responsible for managing 792 * @from_cset and @to_cset's reference counts. 793 */ 794 static void css_set_move_task(struct task_struct *task, 795 struct css_set *from_cset, struct css_set *to_cset, 796 bool use_mg_tasks) 797 { 798 lockdep_assert_held(&css_set_lock); 799 800 if (to_cset && !css_set_populated(to_cset)) 801 css_set_update_populated(to_cset, true); 802 803 if (from_cset) { 804 struct css_task_iter *it, *pos; 805 806 WARN_ON_ONCE(list_empty(&task->cg_list)); 807 808 /* 809 * @task is leaving, advance task iterators which are 810 * pointing to it so that they can resume at the next 811 * position. Advancing an iterator might remove it from 812 * the list, use safe walk. See css_task_iter_advance*() 813 * for details. 814 */ 815 list_for_each_entry_safe(it, pos, &from_cset->task_iters, 816 iters_node) 817 if (it->task_pos == &task->cg_list) 818 css_task_iter_advance(it); 819 820 list_del_init(&task->cg_list); 821 if (!css_set_populated(from_cset)) 822 css_set_update_populated(from_cset, false); 823 } else { 824 WARN_ON_ONCE(!list_empty(&task->cg_list)); 825 } 826 827 if (to_cset) { 828 /* 829 * We are synchronized through cgroup_threadgroup_rwsem 830 * against PF_EXITING setting such that we can't race 831 * against cgroup_exit() changing the css_set to 832 * init_css_set and dropping the old one. 833 */ 834 WARN_ON_ONCE(task->flags & PF_EXITING); 835 836 cgroup_move_task(task, to_cset); 837 list_add_tail(&task->cg_list, use_mg_tasks ? &to_cset->mg_tasks : 838 &to_cset->tasks); 839 } 840 } 841 842 /* 843 * hash table for cgroup groups. This improves the performance to find 844 * an existing css_set. This hash doesn't (currently) take into 845 * account cgroups in empty hierarchies. 846 */ 847 #define CSS_SET_HASH_BITS 7 848 static DEFINE_HASHTABLE(css_set_table, CSS_SET_HASH_BITS); 849 850 static unsigned long css_set_hash(struct cgroup_subsys_state *css[]) 851 { 852 unsigned long key = 0UL; 853 struct cgroup_subsys *ss; 854 int i; 855 856 for_each_subsys(ss, i) 857 key += (unsigned long)css[i]; 858 key = (key >> 16) ^ key; 859 860 return key; 861 } 862 863 void put_css_set_locked(struct css_set *cset) 864 { 865 struct cgrp_cset_link *link, *tmp_link; 866 struct cgroup_subsys *ss; 867 int ssid; 868 869 lockdep_assert_held(&css_set_lock); 870 871 if (!refcount_dec_and_test(&cset->refcount)) 872 return; 873 874 WARN_ON_ONCE(!list_empty(&cset->threaded_csets)); 875 876 /* This css_set is dead. unlink it and release cgroup and css refs */ 877 for_each_subsys(ss, ssid) { 878 list_del(&cset->e_cset_node[ssid]); 879 css_put(cset->subsys[ssid]); 880 } 881 hash_del(&cset->hlist); 882 css_set_count--; 883 884 list_for_each_entry_safe(link, tmp_link, &cset->cgrp_links, cgrp_link) { 885 list_del(&link->cset_link); 886 list_del(&link->cgrp_link); 887 if (cgroup_parent(link->cgrp)) 888 cgroup_put(link->cgrp); 889 kfree(link); 890 } 891 892 if (css_set_threaded(cset)) { 893 list_del(&cset->threaded_csets_node); 894 put_css_set_locked(cset->dom_cset); 895 } 896 897 kfree_rcu(cset, rcu_head); 898 } 899 900 /** 901 * compare_css_sets - helper function for find_existing_css_set(). 902 * @cset: candidate css_set being tested 903 * @old_cset: existing css_set for a task 904 * @new_cgrp: cgroup that's being entered by the task 905 * @template: desired set of css pointers in css_set (pre-calculated) 906 * 907 * Returns true if "cset" matches "old_cset" except for the hierarchy 908 * which "new_cgrp" belongs to, for which it should match "new_cgrp". 909 */ 910 static bool compare_css_sets(struct css_set *cset, 911 struct css_set *old_cset, 912 struct cgroup *new_cgrp, 913 struct cgroup_subsys_state *template[]) 914 { 915 struct cgroup *new_dfl_cgrp; 916 struct list_head *l1, *l2; 917 918 /* 919 * On the default hierarchy, there can be csets which are 920 * associated with the same set of cgroups but different csses. 921 * Let's first ensure that csses match. 922 */ 923 if (memcmp(template, cset->subsys, sizeof(cset->subsys))) 924 return false; 925 926 927 /* @cset's domain should match the default cgroup's */ 928 if (cgroup_on_dfl(new_cgrp)) 929 new_dfl_cgrp = new_cgrp; 930 else 931 new_dfl_cgrp = old_cset->dfl_cgrp; 932 933 if (new_dfl_cgrp->dom_cgrp != cset->dom_cset->dfl_cgrp) 934 return false; 935 936 /* 937 * Compare cgroup pointers in order to distinguish between 938 * different cgroups in hierarchies. As different cgroups may 939 * share the same effective css, this comparison is always 940 * necessary. 941 */ 942 l1 = &cset->cgrp_links; 943 l2 = &old_cset->cgrp_links; 944 while (1) { 945 struct cgrp_cset_link *link1, *link2; 946 struct cgroup *cgrp1, *cgrp2; 947 948 l1 = l1->next; 949 l2 = l2->next; 950 /* See if we reached the end - both lists are equal length. */ 951 if (l1 == &cset->cgrp_links) { 952 BUG_ON(l2 != &old_cset->cgrp_links); 953 break; 954 } else { 955 BUG_ON(l2 == &old_cset->cgrp_links); 956 } 957 /* Locate the cgroups associated with these links. */ 958 link1 = list_entry(l1, struct cgrp_cset_link, cgrp_link); 959 link2 = list_entry(l2, struct cgrp_cset_link, cgrp_link); 960 cgrp1 = link1->cgrp; 961 cgrp2 = link2->cgrp; 962 /* Hierarchies should be linked in the same order. */ 963 BUG_ON(cgrp1->root != cgrp2->root); 964 965 /* 966 * If this hierarchy is the hierarchy of the cgroup 967 * that's changing, then we need to check that this 968 * css_set points to the new cgroup; if it's any other 969 * hierarchy, then this css_set should point to the 970 * same cgroup as the old css_set. 971 */ 972 if (cgrp1->root == new_cgrp->root) { 973 if (cgrp1 != new_cgrp) 974 return false; 975 } else { 976 if (cgrp1 != cgrp2) 977 return false; 978 } 979 } 980 return true; 981 } 982 983 /** 984 * find_existing_css_set - init css array and find the matching css_set 985 * @old_cset: the css_set that we're using before the cgroup transition 986 * @cgrp: the cgroup that we're moving into 987 * @template: out param for the new set of csses, should be clear on entry 988 */ 989 static struct css_set *find_existing_css_set(struct css_set *old_cset, 990 struct cgroup *cgrp, 991 struct cgroup_subsys_state *template[]) 992 { 993 struct cgroup_root *root = cgrp->root; 994 struct cgroup_subsys *ss; 995 struct css_set *cset; 996 unsigned long key; 997 int i; 998 999 /* 1000 * Build the set of subsystem state objects that we want to see in the 1001 * new css_set. while subsystems can change globally, the entries here 1002 * won't change, so no need for locking. 1003 */ 1004 for_each_subsys(ss, i) { 1005 if (root->subsys_mask & (1UL << i)) { 1006 /* 1007 * @ss is in this hierarchy, so we want the 1008 * effective css from @cgrp. 1009 */ 1010 template[i] = cgroup_e_css(cgrp, ss); 1011 } else { 1012 /* 1013 * @ss is not in this hierarchy, so we don't want 1014 * to change the css. 1015 */ 1016 template[i] = old_cset->subsys[i]; 1017 } 1018 } 1019 1020 key = css_set_hash(template); 1021 hash_for_each_possible(css_set_table, cset, hlist, key) { 1022 if (!compare_css_sets(cset, old_cset, cgrp, template)) 1023 continue; 1024 1025 /* This css_set matches what we need */ 1026 return cset; 1027 } 1028 1029 /* No existing cgroup group matched */ 1030 return NULL; 1031 } 1032 1033 static void free_cgrp_cset_links(struct list_head *links_to_free) 1034 { 1035 struct cgrp_cset_link *link, *tmp_link; 1036 1037 list_for_each_entry_safe(link, tmp_link, links_to_free, cset_link) { 1038 list_del(&link->cset_link); 1039 kfree(link); 1040 } 1041 } 1042 1043 /** 1044 * allocate_cgrp_cset_links - allocate cgrp_cset_links 1045 * @count: the number of links to allocate 1046 * @tmp_links: list_head the allocated links are put on 1047 * 1048 * Allocate @count cgrp_cset_link structures and chain them on @tmp_links 1049 * through ->cset_link. Returns 0 on success or -errno. 1050 */ 1051 static int allocate_cgrp_cset_links(int count, struct list_head *tmp_links) 1052 { 1053 struct cgrp_cset_link *link; 1054 int i; 1055 1056 INIT_LIST_HEAD(tmp_links); 1057 1058 for (i = 0; i < count; i++) { 1059 link = kzalloc(sizeof(*link), GFP_KERNEL); 1060 if (!link) { 1061 free_cgrp_cset_links(tmp_links); 1062 return -ENOMEM; 1063 } 1064 list_add(&link->cset_link, tmp_links); 1065 } 1066 return 0; 1067 } 1068 1069 /** 1070 * link_css_set - a helper function to link a css_set to a cgroup 1071 * @tmp_links: cgrp_cset_link objects allocated by allocate_cgrp_cset_links() 1072 * @cset: the css_set to be linked 1073 * @cgrp: the destination cgroup 1074 */ 1075 static void link_css_set(struct list_head *tmp_links, struct css_set *cset, 1076 struct cgroup *cgrp) 1077 { 1078 struct cgrp_cset_link *link; 1079 1080 BUG_ON(list_empty(tmp_links)); 1081 1082 if (cgroup_on_dfl(cgrp)) 1083 cset->dfl_cgrp = cgrp; 1084 1085 link = list_first_entry(tmp_links, struct cgrp_cset_link, cset_link); 1086 link->cset = cset; 1087 link->cgrp = cgrp; 1088 1089 /* 1090 * Always add links to the tail of the lists so that the lists are 1091 * in choronological order. 1092 */ 1093 list_move_tail(&link->cset_link, &cgrp->cset_links); 1094 list_add_tail(&link->cgrp_link, &cset->cgrp_links); 1095 1096 if (cgroup_parent(cgrp)) 1097 cgroup_get_live(cgrp); 1098 } 1099 1100 /** 1101 * find_css_set - return a new css_set with one cgroup updated 1102 * @old_cset: the baseline css_set 1103 * @cgrp: the cgroup to be updated 1104 * 1105 * Return a new css_set that's equivalent to @old_cset, but with @cgrp 1106 * substituted into the appropriate hierarchy. 1107 */ 1108 static struct css_set *find_css_set(struct css_set *old_cset, 1109 struct cgroup *cgrp) 1110 { 1111 struct cgroup_subsys_state *template[CGROUP_SUBSYS_COUNT] = { }; 1112 struct css_set *cset; 1113 struct list_head tmp_links; 1114 struct cgrp_cset_link *link; 1115 struct cgroup_subsys *ss; 1116 unsigned long key; 1117 int ssid; 1118 1119 lockdep_assert_held(&cgroup_mutex); 1120 1121 /* First see if we already have a cgroup group that matches 1122 * the desired set */ 1123 spin_lock_irq(&css_set_lock); 1124 cset = find_existing_css_set(old_cset, cgrp, template); 1125 if (cset) 1126 get_css_set(cset); 1127 spin_unlock_irq(&css_set_lock); 1128 1129 if (cset) 1130 return cset; 1131 1132 cset = kzalloc(sizeof(*cset), GFP_KERNEL); 1133 if (!cset) 1134 return NULL; 1135 1136 /* Allocate all the cgrp_cset_link objects that we'll need */ 1137 if (allocate_cgrp_cset_links(cgroup_root_count, &tmp_links) < 0) { 1138 kfree(cset); 1139 return NULL; 1140 } 1141 1142 refcount_set(&cset->refcount, 1); 1143 cset->dom_cset = cset; 1144 INIT_LIST_HEAD(&cset->tasks); 1145 INIT_LIST_HEAD(&cset->mg_tasks); 1146 INIT_LIST_HEAD(&cset->task_iters); 1147 INIT_LIST_HEAD(&cset->threaded_csets); 1148 INIT_HLIST_NODE(&cset->hlist); 1149 INIT_LIST_HEAD(&cset->cgrp_links); 1150 INIT_LIST_HEAD(&cset->mg_preload_node); 1151 INIT_LIST_HEAD(&cset->mg_node); 1152 1153 /* Copy the set of subsystem state objects generated in 1154 * find_existing_css_set() */ 1155 memcpy(cset->subsys, template, sizeof(cset->subsys)); 1156 1157 spin_lock_irq(&css_set_lock); 1158 /* Add reference counts and links from the new css_set. */ 1159 list_for_each_entry(link, &old_cset->cgrp_links, cgrp_link) { 1160 struct cgroup *c = link->cgrp; 1161 1162 if (c->root == cgrp->root) 1163 c = cgrp; 1164 link_css_set(&tmp_links, cset, c); 1165 } 1166 1167 BUG_ON(!list_empty(&tmp_links)); 1168 1169 css_set_count++; 1170 1171 /* Add @cset to the hash table */ 1172 key = css_set_hash(cset->subsys); 1173 hash_add(css_set_table, &cset->hlist, key); 1174 1175 for_each_subsys(ss, ssid) { 1176 struct cgroup_subsys_state *css = cset->subsys[ssid]; 1177 1178 list_add_tail(&cset->e_cset_node[ssid], 1179 &css->cgroup->e_csets[ssid]); 1180 css_get(css); 1181 } 1182 1183 spin_unlock_irq(&css_set_lock); 1184 1185 /* 1186 * If @cset should be threaded, look up the matching dom_cset and 1187 * link them up. We first fully initialize @cset then look for the 1188 * dom_cset. It's simpler this way and safe as @cset is guaranteed 1189 * to stay empty until we return. 1190 */ 1191 if (cgroup_is_threaded(cset->dfl_cgrp)) { 1192 struct css_set *dcset; 1193 1194 dcset = find_css_set(cset, cset->dfl_cgrp->dom_cgrp); 1195 if (!dcset) { 1196 put_css_set(cset); 1197 return NULL; 1198 } 1199 1200 spin_lock_irq(&css_set_lock); 1201 cset->dom_cset = dcset; 1202 list_add_tail(&cset->threaded_csets_node, 1203 &dcset->threaded_csets); 1204 spin_unlock_irq(&css_set_lock); 1205 } 1206 1207 return cset; 1208 } 1209 1210 struct cgroup_root *cgroup_root_from_kf(struct kernfs_root *kf_root) 1211 { 1212 struct cgroup *root_cgrp = kf_root->kn->priv; 1213 1214 return root_cgrp->root; 1215 } 1216 1217 static int cgroup_init_root_id(struct cgroup_root *root) 1218 { 1219 int id; 1220 1221 lockdep_assert_held(&cgroup_mutex); 1222 1223 id = idr_alloc_cyclic(&cgroup_hierarchy_idr, root, 0, 0, GFP_KERNEL); 1224 if (id < 0) 1225 return id; 1226 1227 root->hierarchy_id = id; 1228 return 0; 1229 } 1230 1231 static void cgroup_exit_root_id(struct cgroup_root *root) 1232 { 1233 lockdep_assert_held(&cgroup_mutex); 1234 1235 idr_remove(&cgroup_hierarchy_idr, root->hierarchy_id); 1236 } 1237 1238 void cgroup_free_root(struct cgroup_root *root) 1239 { 1240 if (root) { 1241 idr_destroy(&root->cgroup_idr); 1242 kfree(root); 1243 } 1244 } 1245 1246 static void cgroup_destroy_root(struct cgroup_root *root) 1247 { 1248 struct cgroup *cgrp = &root->cgrp; 1249 struct cgrp_cset_link *link, *tmp_link; 1250 1251 trace_cgroup_destroy_root(root); 1252 1253 cgroup_lock_and_drain_offline(&cgrp_dfl_root.cgrp); 1254 1255 BUG_ON(atomic_read(&root->nr_cgrps)); 1256 BUG_ON(!list_empty(&cgrp->self.children)); 1257 1258 /* Rebind all subsystems back to the default hierarchy */ 1259 WARN_ON(rebind_subsystems(&cgrp_dfl_root, root->subsys_mask)); 1260 1261 /* 1262 * Release all the links from cset_links to this hierarchy's 1263 * root cgroup 1264 */ 1265 spin_lock_irq(&css_set_lock); 1266 1267 list_for_each_entry_safe(link, tmp_link, &cgrp->cset_links, cset_link) { 1268 list_del(&link->cset_link); 1269 list_del(&link->cgrp_link); 1270 kfree(link); 1271 } 1272 1273 spin_unlock_irq(&css_set_lock); 1274 1275 if (!list_empty(&root->root_list)) { 1276 list_del(&root->root_list); 1277 cgroup_root_count--; 1278 } 1279 1280 cgroup_exit_root_id(root); 1281 1282 mutex_unlock(&cgroup_mutex); 1283 1284 kernfs_destroy_root(root->kf_root); 1285 cgroup_free_root(root); 1286 } 1287 1288 /* 1289 * look up cgroup associated with current task's cgroup namespace on the 1290 * specified hierarchy 1291 */ 1292 static struct cgroup * 1293 current_cgns_cgroup_from_root(struct cgroup_root *root) 1294 { 1295 struct cgroup *res = NULL; 1296 struct css_set *cset; 1297 1298 lockdep_assert_held(&css_set_lock); 1299 1300 rcu_read_lock(); 1301 1302 cset = current->nsproxy->cgroup_ns->root_cset; 1303 if (cset == &init_css_set) { 1304 res = &root->cgrp; 1305 } else { 1306 struct cgrp_cset_link *link; 1307 1308 list_for_each_entry(link, &cset->cgrp_links, cgrp_link) { 1309 struct cgroup *c = link->cgrp; 1310 1311 if (c->root == root) { 1312 res = c; 1313 break; 1314 } 1315 } 1316 } 1317 rcu_read_unlock(); 1318 1319 BUG_ON(!res); 1320 return res; 1321 } 1322 1323 /* look up cgroup associated with given css_set on the specified hierarchy */ 1324 static struct cgroup *cset_cgroup_from_root(struct css_set *cset, 1325 struct cgroup_root *root) 1326 { 1327 struct cgroup *res = NULL; 1328 1329 lockdep_assert_held(&cgroup_mutex); 1330 lockdep_assert_held(&css_set_lock); 1331 1332 if (cset == &init_css_set) { 1333 res = &root->cgrp; 1334 } else if (root == &cgrp_dfl_root) { 1335 res = cset->dfl_cgrp; 1336 } else { 1337 struct cgrp_cset_link *link; 1338 1339 list_for_each_entry(link, &cset->cgrp_links, cgrp_link) { 1340 struct cgroup *c = link->cgrp; 1341 1342 if (c->root == root) { 1343 res = c; 1344 break; 1345 } 1346 } 1347 } 1348 1349 BUG_ON(!res); 1350 return res; 1351 } 1352 1353 /* 1354 * Return the cgroup for "task" from the given hierarchy. Must be 1355 * called with cgroup_mutex and css_set_lock held. 1356 */ 1357 struct cgroup *task_cgroup_from_root(struct task_struct *task, 1358 struct cgroup_root *root) 1359 { 1360 /* 1361 * No need to lock the task - since we hold cgroup_mutex the 1362 * task can't change groups, so the only thing that can happen 1363 * is that it exits and its css is set back to init_css_set. 1364 */ 1365 return cset_cgroup_from_root(task_css_set(task), root); 1366 } 1367 1368 /* 1369 * A task must hold cgroup_mutex to modify cgroups. 1370 * 1371 * Any task can increment and decrement the count field without lock. 1372 * So in general, code holding cgroup_mutex can't rely on the count 1373 * field not changing. However, if the count goes to zero, then only 1374 * cgroup_attach_task() can increment it again. Because a count of zero 1375 * means that no tasks are currently attached, therefore there is no 1376 * way a task attached to that cgroup can fork (the other way to 1377 * increment the count). So code holding cgroup_mutex can safely 1378 * assume that if the count is zero, it will stay zero. Similarly, if 1379 * a task holds cgroup_mutex on a cgroup with zero count, it 1380 * knows that the cgroup won't be removed, as cgroup_rmdir() 1381 * needs that mutex. 1382 * 1383 * A cgroup can only be deleted if both its 'count' of using tasks 1384 * is zero, and its list of 'children' cgroups is empty. Since all 1385 * tasks in the system use _some_ cgroup, and since there is always at 1386 * least one task in the system (init, pid == 1), therefore, root cgroup 1387 * always has either children cgroups and/or using tasks. So we don't 1388 * need a special hack to ensure that root cgroup cannot be deleted. 1389 * 1390 * P.S. One more locking exception. RCU is used to guard the 1391 * update of a tasks cgroup pointer by cgroup_attach_task() 1392 */ 1393 1394 static struct kernfs_syscall_ops cgroup_kf_syscall_ops; 1395 1396 static char *cgroup_file_name(struct cgroup *cgrp, const struct cftype *cft, 1397 char *buf) 1398 { 1399 struct cgroup_subsys *ss = cft->ss; 1400 1401 if (cft->ss && !(cft->flags & CFTYPE_NO_PREFIX) && 1402 !(cgrp->root->flags & CGRP_ROOT_NOPREFIX)) 1403 snprintf(buf, CGROUP_FILE_NAME_MAX, "%s.%s", 1404 cgroup_on_dfl(cgrp) ? ss->name : ss->legacy_name, 1405 cft->name); 1406 else 1407 strscpy(buf, cft->name, CGROUP_FILE_NAME_MAX); 1408 return buf; 1409 } 1410 1411 /** 1412 * cgroup_file_mode - deduce file mode of a control file 1413 * @cft: the control file in question 1414 * 1415 * S_IRUGO for read, S_IWUSR for write. 1416 */ 1417 static umode_t cgroup_file_mode(const struct cftype *cft) 1418 { 1419 umode_t mode = 0; 1420 1421 if (cft->read_u64 || cft->read_s64 || cft->seq_show) 1422 mode |= S_IRUGO; 1423 1424 if (cft->write_u64 || cft->write_s64 || cft->write) { 1425 if (cft->flags & CFTYPE_WORLD_WRITABLE) 1426 mode |= S_IWUGO; 1427 else 1428 mode |= S_IWUSR; 1429 } 1430 1431 return mode; 1432 } 1433 1434 /** 1435 * cgroup_calc_subtree_ss_mask - calculate subtree_ss_mask 1436 * @subtree_control: the new subtree_control mask to consider 1437 * @this_ss_mask: available subsystems 1438 * 1439 * On the default hierarchy, a subsystem may request other subsystems to be 1440 * enabled together through its ->depends_on mask. In such cases, more 1441 * subsystems than specified in "cgroup.subtree_control" may be enabled. 1442 * 1443 * This function calculates which subsystems need to be enabled if 1444 * @subtree_control is to be applied while restricted to @this_ss_mask. 1445 */ 1446 static u16 cgroup_calc_subtree_ss_mask(u16 subtree_control, u16 this_ss_mask) 1447 { 1448 u16 cur_ss_mask = subtree_control; 1449 struct cgroup_subsys *ss; 1450 int ssid; 1451 1452 lockdep_assert_held(&cgroup_mutex); 1453 1454 cur_ss_mask |= cgrp_dfl_implicit_ss_mask; 1455 1456 while (true) { 1457 u16 new_ss_mask = cur_ss_mask; 1458 1459 do_each_subsys_mask(ss, ssid, cur_ss_mask) { 1460 new_ss_mask |= ss->depends_on; 1461 } while_each_subsys_mask(); 1462 1463 /* 1464 * Mask out subsystems which aren't available. This can 1465 * happen only if some depended-upon subsystems were bound 1466 * to non-default hierarchies. 1467 */ 1468 new_ss_mask &= this_ss_mask; 1469 1470 if (new_ss_mask == cur_ss_mask) 1471 break; 1472 cur_ss_mask = new_ss_mask; 1473 } 1474 1475 return cur_ss_mask; 1476 } 1477 1478 /** 1479 * cgroup_kn_unlock - unlocking helper for cgroup kernfs methods 1480 * @kn: the kernfs_node being serviced 1481 * 1482 * This helper undoes cgroup_kn_lock_live() and should be invoked before 1483 * the method finishes if locking succeeded. Note that once this function 1484 * returns the cgroup returned by cgroup_kn_lock_live() may become 1485 * inaccessible any time. If the caller intends to continue to access the 1486 * cgroup, it should pin it before invoking this function. 1487 */ 1488 void cgroup_kn_unlock(struct kernfs_node *kn) 1489 { 1490 struct cgroup *cgrp; 1491 1492 if (kernfs_type(kn) == KERNFS_DIR) 1493 cgrp = kn->priv; 1494 else 1495 cgrp = kn->parent->priv; 1496 1497 mutex_unlock(&cgroup_mutex); 1498 1499 kernfs_unbreak_active_protection(kn); 1500 cgroup_put(cgrp); 1501 } 1502 1503 /** 1504 * cgroup_kn_lock_live - locking helper for cgroup kernfs methods 1505 * @kn: the kernfs_node being serviced 1506 * @drain_offline: perform offline draining on the cgroup 1507 * 1508 * This helper is to be used by a cgroup kernfs method currently servicing 1509 * @kn. It breaks the active protection, performs cgroup locking and 1510 * verifies that the associated cgroup is alive. Returns the cgroup if 1511 * alive; otherwise, %NULL. A successful return should be undone by a 1512 * matching cgroup_kn_unlock() invocation. If @drain_offline is %true, the 1513 * cgroup is drained of offlining csses before return. 1514 * 1515 * Any cgroup kernfs method implementation which requires locking the 1516 * associated cgroup should use this helper. It avoids nesting cgroup 1517 * locking under kernfs active protection and allows all kernfs operations 1518 * including self-removal. 1519 */ 1520 struct cgroup *cgroup_kn_lock_live(struct kernfs_node *kn, bool drain_offline) 1521 { 1522 struct cgroup *cgrp; 1523 1524 if (kernfs_type(kn) == KERNFS_DIR) 1525 cgrp = kn->priv; 1526 else 1527 cgrp = kn->parent->priv; 1528 1529 /* 1530 * We're gonna grab cgroup_mutex which nests outside kernfs 1531 * active_ref. cgroup liveliness check alone provides enough 1532 * protection against removal. Ensure @cgrp stays accessible and 1533 * break the active_ref protection. 1534 */ 1535 if (!cgroup_tryget(cgrp)) 1536 return NULL; 1537 kernfs_break_active_protection(kn); 1538 1539 if (drain_offline) 1540 cgroup_lock_and_drain_offline(cgrp); 1541 else 1542 mutex_lock(&cgroup_mutex); 1543 1544 if (!cgroup_is_dead(cgrp)) 1545 return cgrp; 1546 1547 cgroup_kn_unlock(kn); 1548 return NULL; 1549 } 1550 1551 static void cgroup_rm_file(struct cgroup *cgrp, const struct cftype *cft) 1552 { 1553 char name[CGROUP_FILE_NAME_MAX]; 1554 1555 lockdep_assert_held(&cgroup_mutex); 1556 1557 if (cft->file_offset) { 1558 struct cgroup_subsys_state *css = cgroup_css(cgrp, cft->ss); 1559 struct cgroup_file *cfile = (void *)css + cft->file_offset; 1560 1561 spin_lock_irq(&cgroup_file_kn_lock); 1562 cfile->kn = NULL; 1563 spin_unlock_irq(&cgroup_file_kn_lock); 1564 1565 del_timer_sync(&cfile->notify_timer); 1566 } 1567 1568 kernfs_remove_by_name(cgrp->kn, cgroup_file_name(cgrp, cft, name)); 1569 } 1570 1571 /** 1572 * css_clear_dir - remove subsys files in a cgroup directory 1573 * @css: taget css 1574 */ 1575 static void css_clear_dir(struct cgroup_subsys_state *css) 1576 { 1577 struct cgroup *cgrp = css->cgroup; 1578 struct cftype *cfts; 1579 1580 if (!(css->flags & CSS_VISIBLE)) 1581 return; 1582 1583 css->flags &= ~CSS_VISIBLE; 1584 1585 if (!css->ss) { 1586 if (cgroup_on_dfl(cgrp)) 1587 cfts = cgroup_base_files; 1588 else 1589 cfts = cgroup1_base_files; 1590 1591 cgroup_addrm_files(css, cgrp, cfts, false); 1592 } else { 1593 list_for_each_entry(cfts, &css->ss->cfts, node) 1594 cgroup_addrm_files(css, cgrp, cfts, false); 1595 } 1596 } 1597 1598 /** 1599 * css_populate_dir - create subsys files in a cgroup directory 1600 * @css: target css 1601 * 1602 * On failure, no file is added. 1603 */ 1604 static int css_populate_dir(struct cgroup_subsys_state *css) 1605 { 1606 struct cgroup *cgrp = css->cgroup; 1607 struct cftype *cfts, *failed_cfts; 1608 int ret; 1609 1610 if ((css->flags & CSS_VISIBLE) || !cgrp->kn) 1611 return 0; 1612 1613 if (!css->ss) { 1614 if (cgroup_on_dfl(cgrp)) 1615 cfts = cgroup_base_files; 1616 else 1617 cfts = cgroup1_base_files; 1618 1619 ret = cgroup_addrm_files(&cgrp->self, cgrp, cfts, true); 1620 if (ret < 0) 1621 return ret; 1622 } else { 1623 list_for_each_entry(cfts, &css->ss->cfts, node) { 1624 ret = cgroup_addrm_files(css, cgrp, cfts, true); 1625 if (ret < 0) { 1626 failed_cfts = cfts; 1627 goto err; 1628 } 1629 } 1630 } 1631 1632 css->flags |= CSS_VISIBLE; 1633 1634 return 0; 1635 err: 1636 list_for_each_entry(cfts, &css->ss->cfts, node) { 1637 if (cfts == failed_cfts) 1638 break; 1639 cgroup_addrm_files(css, cgrp, cfts, false); 1640 } 1641 return ret; 1642 } 1643 1644 int rebind_subsystems(struct cgroup_root *dst_root, u16 ss_mask) 1645 { 1646 struct cgroup *dcgrp = &dst_root->cgrp; 1647 struct cgroup_subsys *ss; 1648 int ssid, i, ret; 1649 1650 lockdep_assert_held(&cgroup_mutex); 1651 1652 do_each_subsys_mask(ss, ssid, ss_mask) { 1653 /* 1654 * If @ss has non-root csses attached to it, can't move. 1655 * If @ss is an implicit controller, it is exempt from this 1656 * rule and can be stolen. 1657 */ 1658 if (css_next_child(NULL, cgroup_css(&ss->root->cgrp, ss)) && 1659 !ss->implicit_on_dfl) 1660 return -EBUSY; 1661 1662 /* can't move between two non-dummy roots either */ 1663 if (ss->root != &cgrp_dfl_root && dst_root != &cgrp_dfl_root) 1664 return -EBUSY; 1665 } while_each_subsys_mask(); 1666 1667 do_each_subsys_mask(ss, ssid, ss_mask) { 1668 struct cgroup_root *src_root = ss->root; 1669 struct cgroup *scgrp = &src_root->cgrp; 1670 struct cgroup_subsys_state *css = cgroup_css(scgrp, ss); 1671 struct css_set *cset; 1672 1673 WARN_ON(!css || cgroup_css(dcgrp, ss)); 1674 1675 /* disable from the source */ 1676 src_root->subsys_mask &= ~(1 << ssid); 1677 WARN_ON(cgroup_apply_control(scgrp)); 1678 cgroup_finalize_control(scgrp, 0); 1679 1680 /* rebind */ 1681 RCU_INIT_POINTER(scgrp->subsys[ssid], NULL); 1682 rcu_assign_pointer(dcgrp->subsys[ssid], css); 1683 ss->root = dst_root; 1684 css->cgroup = dcgrp; 1685 1686 spin_lock_irq(&css_set_lock); 1687 hash_for_each(css_set_table, i, cset, hlist) 1688 list_move_tail(&cset->e_cset_node[ss->id], 1689 &dcgrp->e_csets[ss->id]); 1690 spin_unlock_irq(&css_set_lock); 1691 1692 /* default hierarchy doesn't enable controllers by default */ 1693 dst_root->subsys_mask |= 1 << ssid; 1694 if (dst_root == &cgrp_dfl_root) { 1695 static_branch_enable(cgroup_subsys_on_dfl_key[ssid]); 1696 } else { 1697 dcgrp->subtree_control |= 1 << ssid; 1698 static_branch_disable(cgroup_subsys_on_dfl_key[ssid]); 1699 } 1700 1701 ret = cgroup_apply_control(dcgrp); 1702 if (ret) 1703 pr_warn("partial failure to rebind %s controller (err=%d)\n", 1704 ss->name, ret); 1705 1706 if (ss->bind) 1707 ss->bind(css); 1708 } while_each_subsys_mask(); 1709 1710 kernfs_activate(dcgrp->kn); 1711 return 0; 1712 } 1713 1714 int cgroup_show_path(struct seq_file *sf, struct kernfs_node *kf_node, 1715 struct kernfs_root *kf_root) 1716 { 1717 int len = 0; 1718 char *buf = NULL; 1719 struct cgroup_root *kf_cgroot = cgroup_root_from_kf(kf_root); 1720 struct cgroup *ns_cgroup; 1721 1722 buf = kmalloc(PATH_MAX, GFP_KERNEL); 1723 if (!buf) 1724 return -ENOMEM; 1725 1726 spin_lock_irq(&css_set_lock); 1727 ns_cgroup = current_cgns_cgroup_from_root(kf_cgroot); 1728 len = kernfs_path_from_node(kf_node, ns_cgroup->kn, buf, PATH_MAX); 1729 spin_unlock_irq(&css_set_lock); 1730 1731 if (len >= PATH_MAX) 1732 len = -ERANGE; 1733 else if (len > 0) { 1734 seq_escape(sf, buf, " \t\n\\"); 1735 len = 0; 1736 } 1737 kfree(buf); 1738 return len; 1739 } 1740 1741 static int parse_cgroup_root_flags(char *data, unsigned int *root_flags) 1742 { 1743 char *token; 1744 1745 *root_flags = 0; 1746 1747 if (!data) 1748 return 0; 1749 1750 while ((token = strsep(&data, ",")) != NULL) { 1751 if (!strcmp(token, "nsdelegate")) { 1752 *root_flags |= CGRP_ROOT_NS_DELEGATE; 1753 continue; 1754 } 1755 1756 pr_err("cgroup2: unknown option \"%s\"\n", token); 1757 return -EINVAL; 1758 } 1759 1760 return 0; 1761 } 1762 1763 static void apply_cgroup_root_flags(unsigned int root_flags) 1764 { 1765 if (current->nsproxy->cgroup_ns == &init_cgroup_ns) { 1766 if (root_flags & CGRP_ROOT_NS_DELEGATE) 1767 cgrp_dfl_root.flags |= CGRP_ROOT_NS_DELEGATE; 1768 else 1769 cgrp_dfl_root.flags &= ~CGRP_ROOT_NS_DELEGATE; 1770 } 1771 } 1772 1773 static int cgroup_show_options(struct seq_file *seq, struct kernfs_root *kf_root) 1774 { 1775 if (cgrp_dfl_root.flags & CGRP_ROOT_NS_DELEGATE) 1776 seq_puts(seq, ",nsdelegate"); 1777 return 0; 1778 } 1779 1780 static int cgroup_remount(struct kernfs_root *kf_root, int *flags, char *data) 1781 { 1782 unsigned int root_flags; 1783 int ret; 1784 1785 ret = parse_cgroup_root_flags(data, &root_flags); 1786 if (ret) 1787 return ret; 1788 1789 apply_cgroup_root_flags(root_flags); 1790 return 0; 1791 } 1792 1793 /* 1794 * To reduce the fork() overhead for systems that are not actually using 1795 * their cgroups capability, we don't maintain the lists running through 1796 * each css_set to its tasks until we see the list actually used - in other 1797 * words after the first mount. 1798 */ 1799 static bool use_task_css_set_links __read_mostly; 1800 1801 static void cgroup_enable_task_cg_lists(void) 1802 { 1803 struct task_struct *p, *g; 1804 1805 /* 1806 * We need tasklist_lock because RCU is not safe against 1807 * while_each_thread(). Besides, a forking task that has passed 1808 * cgroup_post_fork() without seeing use_task_css_set_links = 1 1809 * is not guaranteed to have its child immediately visible in the 1810 * tasklist if we walk through it with RCU. 1811 */ 1812 read_lock(&tasklist_lock); 1813 spin_lock_irq(&css_set_lock); 1814 1815 if (use_task_css_set_links) 1816 goto out_unlock; 1817 1818 use_task_css_set_links = true; 1819 1820 do_each_thread(g, p) { 1821 WARN_ON_ONCE(!list_empty(&p->cg_list) || 1822 task_css_set(p) != &init_css_set); 1823 1824 /* 1825 * We should check if the process is exiting, otherwise 1826 * it will race with cgroup_exit() in that the list 1827 * entry won't be deleted though the process has exited. 1828 * Do it while holding siglock so that we don't end up 1829 * racing against cgroup_exit(). 1830 * 1831 * Interrupts were already disabled while acquiring 1832 * the css_set_lock, so we do not need to disable it 1833 * again when acquiring the sighand->siglock here. 1834 */ 1835 spin_lock(&p->sighand->siglock); 1836 if (!(p->flags & PF_EXITING)) { 1837 struct css_set *cset = task_css_set(p); 1838 1839 if (!css_set_populated(cset)) 1840 css_set_update_populated(cset, true); 1841 list_add_tail(&p->cg_list, &cset->tasks); 1842 get_css_set(cset); 1843 cset->nr_tasks++; 1844 } 1845 spin_unlock(&p->sighand->siglock); 1846 } while_each_thread(g, p); 1847 out_unlock: 1848 spin_unlock_irq(&css_set_lock); 1849 read_unlock(&tasklist_lock); 1850 } 1851 1852 static void init_cgroup_housekeeping(struct cgroup *cgrp) 1853 { 1854 struct cgroup_subsys *ss; 1855 int ssid; 1856 1857 INIT_LIST_HEAD(&cgrp->self.sibling); 1858 INIT_LIST_HEAD(&cgrp->self.children); 1859 INIT_LIST_HEAD(&cgrp->cset_links); 1860 INIT_LIST_HEAD(&cgrp->pidlists); 1861 mutex_init(&cgrp->pidlist_mutex); 1862 cgrp->self.cgroup = cgrp; 1863 cgrp->self.flags |= CSS_ONLINE; 1864 cgrp->dom_cgrp = cgrp; 1865 cgrp->max_descendants = INT_MAX; 1866 cgrp->max_depth = INT_MAX; 1867 INIT_LIST_HEAD(&cgrp->rstat_css_list); 1868 prev_cputime_init(&cgrp->prev_cputime); 1869 1870 for_each_subsys(ss, ssid) 1871 INIT_LIST_HEAD(&cgrp->e_csets[ssid]); 1872 1873 init_waitqueue_head(&cgrp->offline_waitq); 1874 INIT_WORK(&cgrp->release_agent_work, cgroup1_release_agent); 1875 } 1876 1877 void init_cgroup_root(struct cgroup_root *root, struct cgroup_sb_opts *opts) 1878 { 1879 struct cgroup *cgrp = &root->cgrp; 1880 1881 INIT_LIST_HEAD(&root->root_list); 1882 atomic_set(&root->nr_cgrps, 1); 1883 cgrp->root = root; 1884 init_cgroup_housekeeping(cgrp); 1885 idr_init(&root->cgroup_idr); 1886 1887 root->flags = opts->flags; 1888 if (opts->release_agent) 1889 strscpy(root->release_agent_path, opts->release_agent, PATH_MAX); 1890 if (opts->name) 1891 strscpy(root->name, opts->name, MAX_CGROUP_ROOT_NAMELEN); 1892 if (opts->cpuset_clone_children) 1893 set_bit(CGRP_CPUSET_CLONE_CHILDREN, &root->cgrp.flags); 1894 } 1895 1896 int cgroup_setup_root(struct cgroup_root *root, u16 ss_mask, int ref_flags) 1897 { 1898 LIST_HEAD(tmp_links); 1899 struct cgroup *root_cgrp = &root->cgrp; 1900 struct kernfs_syscall_ops *kf_sops; 1901 struct css_set *cset; 1902 int i, ret; 1903 1904 lockdep_assert_held(&cgroup_mutex); 1905 1906 ret = cgroup_idr_alloc(&root->cgroup_idr, root_cgrp, 1, 2, GFP_KERNEL); 1907 if (ret < 0) 1908 goto out; 1909 root_cgrp->id = ret; 1910 root_cgrp->ancestor_ids[0] = ret; 1911 1912 ret = percpu_ref_init(&root_cgrp->self.refcnt, css_release, 1913 ref_flags, GFP_KERNEL); 1914 if (ret) 1915 goto out; 1916 1917 /* 1918 * We're accessing css_set_count without locking css_set_lock here, 1919 * but that's OK - it can only be increased by someone holding 1920 * cgroup_lock, and that's us. Later rebinding may disable 1921 * controllers on the default hierarchy and thus create new csets, 1922 * which can't be more than the existing ones. Allocate 2x. 1923 */ 1924 ret = allocate_cgrp_cset_links(2 * css_set_count, &tmp_links); 1925 if (ret) 1926 goto cancel_ref; 1927 1928 ret = cgroup_init_root_id(root); 1929 if (ret) 1930 goto cancel_ref; 1931 1932 kf_sops = root == &cgrp_dfl_root ? 1933 &cgroup_kf_syscall_ops : &cgroup1_kf_syscall_ops; 1934 1935 root->kf_root = kernfs_create_root(kf_sops, 1936 KERNFS_ROOT_CREATE_DEACTIVATED | 1937 KERNFS_ROOT_SUPPORT_EXPORTOP, 1938 root_cgrp); 1939 if (IS_ERR(root->kf_root)) { 1940 ret = PTR_ERR(root->kf_root); 1941 goto exit_root_id; 1942 } 1943 root_cgrp->kn = root->kf_root->kn; 1944 1945 ret = css_populate_dir(&root_cgrp->self); 1946 if (ret) 1947 goto destroy_root; 1948 1949 ret = rebind_subsystems(root, ss_mask); 1950 if (ret) 1951 goto destroy_root; 1952 1953 ret = cgroup_bpf_inherit(root_cgrp); 1954 WARN_ON_ONCE(ret); 1955 1956 trace_cgroup_setup_root(root); 1957 1958 /* 1959 * There must be no failure case after here, since rebinding takes 1960 * care of subsystems' refcounts, which are explicitly dropped in 1961 * the failure exit path. 1962 */ 1963 list_add(&root->root_list, &cgroup_roots); 1964 cgroup_root_count++; 1965 1966 /* 1967 * Link the root cgroup in this hierarchy into all the css_set 1968 * objects. 1969 */ 1970 spin_lock_irq(&css_set_lock); 1971 hash_for_each(css_set_table, i, cset, hlist) { 1972 link_css_set(&tmp_links, cset, root_cgrp); 1973 if (css_set_populated(cset)) 1974 cgroup_update_populated(root_cgrp, true); 1975 } 1976 spin_unlock_irq(&css_set_lock); 1977 1978 BUG_ON(!list_empty(&root_cgrp->self.children)); 1979 BUG_ON(atomic_read(&root->nr_cgrps) != 1); 1980 1981 kernfs_activate(root_cgrp->kn); 1982 ret = 0; 1983 goto out; 1984 1985 destroy_root: 1986 kernfs_destroy_root(root->kf_root); 1987 root->kf_root = NULL; 1988 exit_root_id: 1989 cgroup_exit_root_id(root); 1990 cancel_ref: 1991 percpu_ref_exit(&root_cgrp->self.refcnt); 1992 out: 1993 free_cgrp_cset_links(&tmp_links); 1994 return ret; 1995 } 1996 1997 struct dentry *cgroup_do_mount(struct file_system_type *fs_type, int flags, 1998 struct cgroup_root *root, unsigned long magic, 1999 struct cgroup_namespace *ns) 2000 { 2001 struct dentry *dentry; 2002 bool new_sb; 2003 2004 dentry = kernfs_mount(fs_type, flags, root->kf_root, magic, &new_sb); 2005 2006 /* 2007 * In non-init cgroup namespace, instead of root cgroup's dentry, 2008 * we return the dentry corresponding to the cgroupns->root_cgrp. 2009 */ 2010 if (!IS_ERR(dentry) && ns != &init_cgroup_ns) { 2011 struct dentry *nsdentry; 2012 struct cgroup *cgrp; 2013 2014 mutex_lock(&cgroup_mutex); 2015 spin_lock_irq(&css_set_lock); 2016 2017 cgrp = cset_cgroup_from_root(ns->root_cset, root); 2018 2019 spin_unlock_irq(&css_set_lock); 2020 mutex_unlock(&cgroup_mutex); 2021 2022 nsdentry = kernfs_node_dentry(cgrp->kn, dentry->d_sb); 2023 dput(dentry); 2024 dentry = nsdentry; 2025 } 2026 2027 if (IS_ERR(dentry) || !new_sb) 2028 cgroup_put(&root->cgrp); 2029 2030 return dentry; 2031 } 2032 2033 static struct dentry *cgroup_mount(struct file_system_type *fs_type, 2034 int flags, const char *unused_dev_name, 2035 void *data) 2036 { 2037 struct cgroup_namespace *ns = current->nsproxy->cgroup_ns; 2038 struct dentry *dentry; 2039 int ret; 2040 2041 get_cgroup_ns(ns); 2042 2043 /* Check if the caller has permission to mount. */ 2044 if (!ns_capable(ns->user_ns, CAP_SYS_ADMIN)) { 2045 put_cgroup_ns(ns); 2046 return ERR_PTR(-EPERM); 2047 } 2048 2049 /* 2050 * The first time anyone tries to mount a cgroup, enable the list 2051 * linking each css_set to its tasks and fix up all existing tasks. 2052 */ 2053 if (!use_task_css_set_links) 2054 cgroup_enable_task_cg_lists(); 2055 2056 if (fs_type == &cgroup2_fs_type) { 2057 unsigned int root_flags; 2058 2059 ret = parse_cgroup_root_flags(data, &root_flags); 2060 if (ret) { 2061 put_cgroup_ns(ns); 2062 return ERR_PTR(ret); 2063 } 2064 2065 cgrp_dfl_visible = true; 2066 cgroup_get_live(&cgrp_dfl_root.cgrp); 2067 2068 dentry = cgroup_do_mount(&cgroup2_fs_type, flags, &cgrp_dfl_root, 2069 CGROUP2_SUPER_MAGIC, ns); 2070 if (!IS_ERR(dentry)) 2071 apply_cgroup_root_flags(root_flags); 2072 } else { 2073 dentry = cgroup1_mount(&cgroup_fs_type, flags, data, 2074 CGROUP_SUPER_MAGIC, ns); 2075 } 2076 2077 put_cgroup_ns(ns); 2078 return dentry; 2079 } 2080 2081 static void cgroup_kill_sb(struct super_block *sb) 2082 { 2083 struct kernfs_root *kf_root = kernfs_root_from_sb(sb); 2084 struct cgroup_root *root = cgroup_root_from_kf(kf_root); 2085 2086 /* 2087 * If @root doesn't have any mounts or children, start killing it. 2088 * This prevents new mounts by disabling percpu_ref_tryget_live(). 2089 * cgroup_mount() may wait for @root's release. 2090 * 2091 * And don't kill the default root. 2092 */ 2093 if (!list_empty(&root->cgrp.self.children) || 2094 root == &cgrp_dfl_root) 2095 cgroup_put(&root->cgrp); 2096 else 2097 percpu_ref_kill(&root->cgrp.self.refcnt); 2098 2099 kernfs_kill_sb(sb); 2100 } 2101 2102 struct file_system_type cgroup_fs_type = { 2103 .name = "cgroup", 2104 .mount = cgroup_mount, 2105 .kill_sb = cgroup_kill_sb, 2106 .fs_flags = FS_USERNS_MOUNT, 2107 }; 2108 2109 static struct file_system_type cgroup2_fs_type = { 2110 .name = "cgroup2", 2111 .mount = cgroup_mount, 2112 .kill_sb = cgroup_kill_sb, 2113 .fs_flags = FS_USERNS_MOUNT, 2114 }; 2115 2116 int cgroup_path_ns_locked(struct cgroup *cgrp, char *buf, size_t buflen, 2117 struct cgroup_namespace *ns) 2118 { 2119 struct cgroup *root = cset_cgroup_from_root(ns->root_cset, cgrp->root); 2120 2121 return kernfs_path_from_node(cgrp->kn, root->kn, buf, buflen); 2122 } 2123 2124 int cgroup_path_ns(struct cgroup *cgrp, char *buf, size_t buflen, 2125 struct cgroup_namespace *ns) 2126 { 2127 int ret; 2128 2129 mutex_lock(&cgroup_mutex); 2130 spin_lock_irq(&css_set_lock); 2131 2132 ret = cgroup_path_ns_locked(cgrp, buf, buflen, ns); 2133 2134 spin_unlock_irq(&css_set_lock); 2135 mutex_unlock(&cgroup_mutex); 2136 2137 return ret; 2138 } 2139 EXPORT_SYMBOL_GPL(cgroup_path_ns); 2140 2141 /** 2142 * task_cgroup_path - cgroup path of a task in the first cgroup hierarchy 2143 * @task: target task 2144 * @buf: the buffer to write the path into 2145 * @buflen: the length of the buffer 2146 * 2147 * Determine @task's cgroup on the first (the one with the lowest non-zero 2148 * hierarchy_id) cgroup hierarchy and copy its path into @buf. This 2149 * function grabs cgroup_mutex and shouldn't be used inside locks used by 2150 * cgroup controller callbacks. 2151 * 2152 * Return value is the same as kernfs_path(). 2153 */ 2154 int task_cgroup_path(struct task_struct *task, char *buf, size_t buflen) 2155 { 2156 struct cgroup_root *root; 2157 struct cgroup *cgrp; 2158 int hierarchy_id = 1; 2159 int ret; 2160 2161 mutex_lock(&cgroup_mutex); 2162 spin_lock_irq(&css_set_lock); 2163 2164 root = idr_get_next(&cgroup_hierarchy_idr, &hierarchy_id); 2165 2166 if (root) { 2167 cgrp = task_cgroup_from_root(task, root); 2168 ret = cgroup_path_ns_locked(cgrp, buf, buflen, &init_cgroup_ns); 2169 } else { 2170 /* if no hierarchy exists, everyone is in "/" */ 2171 ret = strlcpy(buf, "/", buflen); 2172 } 2173 2174 spin_unlock_irq(&css_set_lock); 2175 mutex_unlock(&cgroup_mutex); 2176 return ret; 2177 } 2178 EXPORT_SYMBOL_GPL(task_cgroup_path); 2179 2180 /** 2181 * cgroup_migrate_add_task - add a migration target task to a migration context 2182 * @task: target task 2183 * @mgctx: target migration context 2184 * 2185 * Add @task, which is a migration target, to @mgctx->tset. This function 2186 * becomes noop if @task doesn't need to be migrated. @task's css_set 2187 * should have been added as a migration source and @task->cg_list will be 2188 * moved from the css_set's tasks list to mg_tasks one. 2189 */ 2190 static void cgroup_migrate_add_task(struct task_struct *task, 2191 struct cgroup_mgctx *mgctx) 2192 { 2193 struct css_set *cset; 2194 2195 lockdep_assert_held(&css_set_lock); 2196 2197 /* @task either already exited or can't exit until the end */ 2198 if (task->flags & PF_EXITING) 2199 return; 2200 2201 /* leave @task alone if post_fork() hasn't linked it yet */ 2202 if (list_empty(&task->cg_list)) 2203 return; 2204 2205 cset = task_css_set(task); 2206 if (!cset->mg_src_cgrp) 2207 return; 2208 2209 mgctx->tset.nr_tasks++; 2210 2211 list_move_tail(&task->cg_list, &cset->mg_tasks); 2212 if (list_empty(&cset->mg_node)) 2213 list_add_tail(&cset->mg_node, 2214 &mgctx->tset.src_csets); 2215 if (list_empty(&cset->mg_dst_cset->mg_node)) 2216 list_add_tail(&cset->mg_dst_cset->mg_node, 2217 &mgctx->tset.dst_csets); 2218 } 2219 2220 /** 2221 * cgroup_taskset_first - reset taskset and return the first task 2222 * @tset: taskset of interest 2223 * @dst_cssp: output variable for the destination css 2224 * 2225 * @tset iteration is initialized and the first task is returned. 2226 */ 2227 struct task_struct *cgroup_taskset_first(struct cgroup_taskset *tset, 2228 struct cgroup_subsys_state **dst_cssp) 2229 { 2230 tset->cur_cset = list_first_entry(tset->csets, struct css_set, mg_node); 2231 tset->cur_task = NULL; 2232 2233 return cgroup_taskset_next(tset, dst_cssp); 2234 } 2235 2236 /** 2237 * cgroup_taskset_next - iterate to the next task in taskset 2238 * @tset: taskset of interest 2239 * @dst_cssp: output variable for the destination css 2240 * 2241 * Return the next task in @tset. Iteration must have been initialized 2242 * with cgroup_taskset_first(). 2243 */ 2244 struct task_struct *cgroup_taskset_next(struct cgroup_taskset *tset, 2245 struct cgroup_subsys_state **dst_cssp) 2246 { 2247 struct css_set *cset = tset->cur_cset; 2248 struct task_struct *task = tset->cur_task; 2249 2250 while (&cset->mg_node != tset->csets) { 2251 if (!task) 2252 task = list_first_entry(&cset->mg_tasks, 2253 struct task_struct, cg_list); 2254 else 2255 task = list_next_entry(task, cg_list); 2256 2257 if (&task->cg_list != &cset->mg_tasks) { 2258 tset->cur_cset = cset; 2259 tset->cur_task = task; 2260 2261 /* 2262 * This function may be called both before and 2263 * after cgroup_taskset_migrate(). The two cases 2264 * can be distinguished by looking at whether @cset 2265 * has its ->mg_dst_cset set. 2266 */ 2267 if (cset->mg_dst_cset) 2268 *dst_cssp = cset->mg_dst_cset->subsys[tset->ssid]; 2269 else 2270 *dst_cssp = cset->subsys[tset->ssid]; 2271 2272 return task; 2273 } 2274 2275 cset = list_next_entry(cset, mg_node); 2276 task = NULL; 2277 } 2278 2279 return NULL; 2280 } 2281 2282 /** 2283 * cgroup_taskset_migrate - migrate a taskset 2284 * @mgctx: migration context 2285 * 2286 * Migrate tasks in @mgctx as setup by migration preparation functions. 2287 * This function fails iff one of the ->can_attach callbacks fails and 2288 * guarantees that either all or none of the tasks in @mgctx are migrated. 2289 * @mgctx is consumed regardless of success. 2290 */ 2291 static int cgroup_migrate_execute(struct cgroup_mgctx *mgctx) 2292 { 2293 struct cgroup_taskset *tset = &mgctx->tset; 2294 struct cgroup_subsys *ss; 2295 struct task_struct *task, *tmp_task; 2296 struct css_set *cset, *tmp_cset; 2297 int ssid, failed_ssid, ret; 2298 2299 /* check that we can legitimately attach to the cgroup */ 2300 if (tset->nr_tasks) { 2301 do_each_subsys_mask(ss, ssid, mgctx->ss_mask) { 2302 if (ss->can_attach) { 2303 tset->ssid = ssid; 2304 ret = ss->can_attach(tset); 2305 if (ret) { 2306 failed_ssid = ssid; 2307 goto out_cancel_attach; 2308 } 2309 } 2310 } while_each_subsys_mask(); 2311 } 2312 2313 /* 2314 * Now that we're guaranteed success, proceed to move all tasks to 2315 * the new cgroup. There are no failure cases after here, so this 2316 * is the commit point. 2317 */ 2318 spin_lock_irq(&css_set_lock); 2319 list_for_each_entry(cset, &tset->src_csets, mg_node) { 2320 list_for_each_entry_safe(task, tmp_task, &cset->mg_tasks, cg_list) { 2321 struct css_set *from_cset = task_css_set(task); 2322 struct css_set *to_cset = cset->mg_dst_cset; 2323 2324 get_css_set(to_cset); 2325 to_cset->nr_tasks++; 2326 css_set_move_task(task, from_cset, to_cset, true); 2327 put_css_set_locked(from_cset); 2328 from_cset->nr_tasks--; 2329 } 2330 } 2331 spin_unlock_irq(&css_set_lock); 2332 2333 /* 2334 * Migration is committed, all target tasks are now on dst_csets. 2335 * Nothing is sensitive to fork() after this point. Notify 2336 * controllers that migration is complete. 2337 */ 2338 tset->csets = &tset->dst_csets; 2339 2340 if (tset->nr_tasks) { 2341 do_each_subsys_mask(ss, ssid, mgctx->ss_mask) { 2342 if (ss->attach) { 2343 tset->ssid = ssid; 2344 ss->attach(tset); 2345 } 2346 } while_each_subsys_mask(); 2347 } 2348 2349 ret = 0; 2350 goto out_release_tset; 2351 2352 out_cancel_attach: 2353 if (tset->nr_tasks) { 2354 do_each_subsys_mask(ss, ssid, mgctx->ss_mask) { 2355 if (ssid == failed_ssid) 2356 break; 2357 if (ss->cancel_attach) { 2358 tset->ssid = ssid; 2359 ss->cancel_attach(tset); 2360 } 2361 } while_each_subsys_mask(); 2362 } 2363 out_release_tset: 2364 spin_lock_irq(&css_set_lock); 2365 list_splice_init(&tset->dst_csets, &tset->src_csets); 2366 list_for_each_entry_safe(cset, tmp_cset, &tset->src_csets, mg_node) { 2367 list_splice_tail_init(&cset->mg_tasks, &cset->tasks); 2368 list_del_init(&cset->mg_node); 2369 } 2370 spin_unlock_irq(&css_set_lock); 2371 2372 /* 2373 * Re-initialize the cgroup_taskset structure in case it is reused 2374 * again in another cgroup_migrate_add_task()/cgroup_migrate_execute() 2375 * iteration. 2376 */ 2377 tset->nr_tasks = 0; 2378 tset->csets = &tset->src_csets; 2379 return ret; 2380 } 2381 2382 /** 2383 * cgroup_migrate_vet_dst - verify whether a cgroup can be migration destination 2384 * @dst_cgrp: destination cgroup to test 2385 * 2386 * On the default hierarchy, except for the mixable, (possible) thread root 2387 * and threaded cgroups, subtree_control must be zero for migration 2388 * destination cgroups with tasks so that child cgroups don't compete 2389 * against tasks. 2390 */ 2391 int cgroup_migrate_vet_dst(struct cgroup *dst_cgrp) 2392 { 2393 /* v1 doesn't have any restriction */ 2394 if (!cgroup_on_dfl(dst_cgrp)) 2395 return 0; 2396 2397 /* verify @dst_cgrp can host resources */ 2398 if (!cgroup_is_valid_domain(dst_cgrp->dom_cgrp)) 2399 return -EOPNOTSUPP; 2400 2401 /* mixables don't care */ 2402 if (cgroup_is_mixable(dst_cgrp)) 2403 return 0; 2404 2405 /* 2406 * If @dst_cgrp is already or can become a thread root or is 2407 * threaded, it doesn't matter. 2408 */ 2409 if (cgroup_can_be_thread_root(dst_cgrp) || cgroup_is_threaded(dst_cgrp)) 2410 return 0; 2411 2412 /* apply no-internal-process constraint */ 2413 if (dst_cgrp->subtree_control) 2414 return -EBUSY; 2415 2416 return 0; 2417 } 2418 2419 /** 2420 * cgroup_migrate_finish - cleanup after attach 2421 * @mgctx: migration context 2422 * 2423 * Undo cgroup_migrate_add_src() and cgroup_migrate_prepare_dst(). See 2424 * those functions for details. 2425 */ 2426 void cgroup_migrate_finish(struct cgroup_mgctx *mgctx) 2427 { 2428 LIST_HEAD(preloaded); 2429 struct css_set *cset, *tmp_cset; 2430 2431 lockdep_assert_held(&cgroup_mutex); 2432 2433 spin_lock_irq(&css_set_lock); 2434 2435 list_splice_tail_init(&mgctx->preloaded_src_csets, &preloaded); 2436 list_splice_tail_init(&mgctx->preloaded_dst_csets, &preloaded); 2437 2438 list_for_each_entry_safe(cset, tmp_cset, &preloaded, mg_preload_node) { 2439 cset->mg_src_cgrp = NULL; 2440 cset->mg_dst_cgrp = NULL; 2441 cset->mg_dst_cset = NULL; 2442 list_del_init(&cset->mg_preload_node); 2443 put_css_set_locked(cset); 2444 } 2445 2446 spin_unlock_irq(&css_set_lock); 2447 } 2448 2449 /** 2450 * cgroup_migrate_add_src - add a migration source css_set 2451 * @src_cset: the source css_set to add 2452 * @dst_cgrp: the destination cgroup 2453 * @mgctx: migration context 2454 * 2455 * Tasks belonging to @src_cset are about to be migrated to @dst_cgrp. Pin 2456 * @src_cset and add it to @mgctx->src_csets, which should later be cleaned 2457 * up by cgroup_migrate_finish(). 2458 * 2459 * This function may be called without holding cgroup_threadgroup_rwsem 2460 * even if the target is a process. Threads may be created and destroyed 2461 * but as long as cgroup_mutex is not dropped, no new css_set can be put 2462 * into play and the preloaded css_sets are guaranteed to cover all 2463 * migrations. 2464 */ 2465 void cgroup_migrate_add_src(struct css_set *src_cset, 2466 struct cgroup *dst_cgrp, 2467 struct cgroup_mgctx *mgctx) 2468 { 2469 struct cgroup *src_cgrp; 2470 2471 lockdep_assert_held(&cgroup_mutex); 2472 lockdep_assert_held(&css_set_lock); 2473 2474 /* 2475 * If ->dead, @src_set is associated with one or more dead cgroups 2476 * and doesn't contain any migratable tasks. Ignore it early so 2477 * that the rest of migration path doesn't get confused by it. 2478 */ 2479 if (src_cset->dead) 2480 return; 2481 2482 src_cgrp = cset_cgroup_from_root(src_cset, dst_cgrp->root); 2483 2484 if (!list_empty(&src_cset->mg_preload_node)) 2485 return; 2486 2487 WARN_ON(src_cset->mg_src_cgrp); 2488 WARN_ON(src_cset->mg_dst_cgrp); 2489 WARN_ON(!list_empty(&src_cset->mg_tasks)); 2490 WARN_ON(!list_empty(&src_cset->mg_node)); 2491 2492 src_cset->mg_src_cgrp = src_cgrp; 2493 src_cset->mg_dst_cgrp = dst_cgrp; 2494 get_css_set(src_cset); 2495 list_add_tail(&src_cset->mg_preload_node, &mgctx->preloaded_src_csets); 2496 } 2497 2498 /** 2499 * cgroup_migrate_prepare_dst - prepare destination css_sets for migration 2500 * @mgctx: migration context 2501 * 2502 * Tasks are about to be moved and all the source css_sets have been 2503 * preloaded to @mgctx->preloaded_src_csets. This function looks up and 2504 * pins all destination css_sets, links each to its source, and append them 2505 * to @mgctx->preloaded_dst_csets. 2506 * 2507 * This function must be called after cgroup_migrate_add_src() has been 2508 * called on each migration source css_set. After migration is performed 2509 * using cgroup_migrate(), cgroup_migrate_finish() must be called on 2510 * @mgctx. 2511 */ 2512 int cgroup_migrate_prepare_dst(struct cgroup_mgctx *mgctx) 2513 { 2514 struct css_set *src_cset, *tmp_cset; 2515 2516 lockdep_assert_held(&cgroup_mutex); 2517 2518 /* look up the dst cset for each src cset and link it to src */ 2519 list_for_each_entry_safe(src_cset, tmp_cset, &mgctx->preloaded_src_csets, 2520 mg_preload_node) { 2521 struct css_set *dst_cset; 2522 struct cgroup_subsys *ss; 2523 int ssid; 2524 2525 dst_cset = find_css_set(src_cset, src_cset->mg_dst_cgrp); 2526 if (!dst_cset) 2527 goto err; 2528 2529 WARN_ON_ONCE(src_cset->mg_dst_cset || dst_cset->mg_dst_cset); 2530 2531 /* 2532 * If src cset equals dst, it's noop. Drop the src. 2533 * cgroup_migrate() will skip the cset too. Note that we 2534 * can't handle src == dst as some nodes are used by both. 2535 */ 2536 if (src_cset == dst_cset) { 2537 src_cset->mg_src_cgrp = NULL; 2538 src_cset->mg_dst_cgrp = NULL; 2539 list_del_init(&src_cset->mg_preload_node); 2540 put_css_set(src_cset); 2541 put_css_set(dst_cset); 2542 continue; 2543 } 2544 2545 src_cset->mg_dst_cset = dst_cset; 2546 2547 if (list_empty(&dst_cset->mg_preload_node)) 2548 list_add_tail(&dst_cset->mg_preload_node, 2549 &mgctx->preloaded_dst_csets); 2550 else 2551 put_css_set(dst_cset); 2552 2553 for_each_subsys(ss, ssid) 2554 if (src_cset->subsys[ssid] != dst_cset->subsys[ssid]) 2555 mgctx->ss_mask |= 1 << ssid; 2556 } 2557 2558 return 0; 2559 err: 2560 cgroup_migrate_finish(mgctx); 2561 return -ENOMEM; 2562 } 2563 2564 /** 2565 * cgroup_migrate - migrate a process or task to a cgroup 2566 * @leader: the leader of the process or the task to migrate 2567 * @threadgroup: whether @leader points to the whole process or a single task 2568 * @mgctx: migration context 2569 * 2570 * Migrate a process or task denoted by @leader. If migrating a process, 2571 * the caller must be holding cgroup_threadgroup_rwsem. The caller is also 2572 * responsible for invoking cgroup_migrate_add_src() and 2573 * cgroup_migrate_prepare_dst() on the targets before invoking this 2574 * function and following up with cgroup_migrate_finish(). 2575 * 2576 * As long as a controller's ->can_attach() doesn't fail, this function is 2577 * guaranteed to succeed. This means that, excluding ->can_attach() 2578 * failure, when migrating multiple targets, the success or failure can be 2579 * decided for all targets by invoking group_migrate_prepare_dst() before 2580 * actually starting migrating. 2581 */ 2582 int cgroup_migrate(struct task_struct *leader, bool threadgroup, 2583 struct cgroup_mgctx *mgctx) 2584 { 2585 struct task_struct *task; 2586 2587 /* 2588 * Prevent freeing of tasks while we take a snapshot. Tasks that are 2589 * already PF_EXITING could be freed from underneath us unless we 2590 * take an rcu_read_lock. 2591 */ 2592 spin_lock_irq(&css_set_lock); 2593 rcu_read_lock(); 2594 task = leader; 2595 do { 2596 cgroup_migrate_add_task(task, mgctx); 2597 if (!threadgroup) 2598 break; 2599 } while_each_thread(leader, task); 2600 rcu_read_unlock(); 2601 spin_unlock_irq(&css_set_lock); 2602 2603 return cgroup_migrate_execute(mgctx); 2604 } 2605 2606 /** 2607 * cgroup_attach_task - attach a task or a whole threadgroup to a cgroup 2608 * @dst_cgrp: the cgroup to attach to 2609 * @leader: the task or the leader of the threadgroup to be attached 2610 * @threadgroup: attach the whole threadgroup? 2611 * 2612 * Call holding cgroup_mutex and cgroup_threadgroup_rwsem. 2613 */ 2614 int cgroup_attach_task(struct cgroup *dst_cgrp, struct task_struct *leader, 2615 bool threadgroup) 2616 { 2617 DEFINE_CGROUP_MGCTX(mgctx); 2618 struct task_struct *task; 2619 int ret; 2620 2621 ret = cgroup_migrate_vet_dst(dst_cgrp); 2622 if (ret) 2623 return ret; 2624 2625 /* look up all src csets */ 2626 spin_lock_irq(&css_set_lock); 2627 rcu_read_lock(); 2628 task = leader; 2629 do { 2630 cgroup_migrate_add_src(task_css_set(task), dst_cgrp, &mgctx); 2631 if (!threadgroup) 2632 break; 2633 } while_each_thread(leader, task); 2634 rcu_read_unlock(); 2635 spin_unlock_irq(&css_set_lock); 2636 2637 /* prepare dst csets and commit */ 2638 ret = cgroup_migrate_prepare_dst(&mgctx); 2639 if (!ret) 2640 ret = cgroup_migrate(leader, threadgroup, &mgctx); 2641 2642 cgroup_migrate_finish(&mgctx); 2643 2644 if (!ret) 2645 TRACE_CGROUP_PATH(attach_task, dst_cgrp, leader, threadgroup); 2646 2647 return ret; 2648 } 2649 2650 struct task_struct *cgroup_procs_write_start(char *buf, bool threadgroup) 2651 __acquires(&cgroup_threadgroup_rwsem) 2652 { 2653 struct task_struct *tsk; 2654 pid_t pid; 2655 2656 if (kstrtoint(strstrip(buf), 0, &pid) || pid < 0) 2657 return ERR_PTR(-EINVAL); 2658 2659 percpu_down_write(&cgroup_threadgroup_rwsem); 2660 2661 rcu_read_lock(); 2662 if (pid) { 2663 tsk = find_task_by_vpid(pid); 2664 if (!tsk) { 2665 tsk = ERR_PTR(-ESRCH); 2666 goto out_unlock_threadgroup; 2667 } 2668 } else { 2669 tsk = current; 2670 } 2671 2672 if (threadgroup) 2673 tsk = tsk->group_leader; 2674 2675 /* 2676 * kthreads may acquire PF_NO_SETAFFINITY during initialization. 2677 * If userland migrates such a kthread to a non-root cgroup, it can 2678 * become trapped in a cpuset, or RT kthread may be born in a 2679 * cgroup with no rt_runtime allocated. Just say no. 2680 */ 2681 if (tsk->no_cgroup_migration || (tsk->flags & PF_NO_SETAFFINITY)) { 2682 tsk = ERR_PTR(-EINVAL); 2683 goto out_unlock_threadgroup; 2684 } 2685 2686 get_task_struct(tsk); 2687 goto out_unlock_rcu; 2688 2689 out_unlock_threadgroup: 2690 percpu_up_write(&cgroup_threadgroup_rwsem); 2691 out_unlock_rcu: 2692 rcu_read_unlock(); 2693 return tsk; 2694 } 2695 2696 void cgroup_procs_write_finish(struct task_struct *task) 2697 __releases(&cgroup_threadgroup_rwsem) 2698 { 2699 struct cgroup_subsys *ss; 2700 int ssid; 2701 2702 /* release reference from cgroup_procs_write_start() */ 2703 put_task_struct(task); 2704 2705 percpu_up_write(&cgroup_threadgroup_rwsem); 2706 for_each_subsys(ss, ssid) 2707 if (ss->post_attach) 2708 ss->post_attach(); 2709 } 2710 2711 static void cgroup_print_ss_mask(struct seq_file *seq, u16 ss_mask) 2712 { 2713 struct cgroup_subsys *ss; 2714 bool printed = false; 2715 int ssid; 2716 2717 do_each_subsys_mask(ss, ssid, ss_mask) { 2718 if (printed) 2719 seq_putc(seq, ' '); 2720 seq_printf(seq, "%s", ss->name); 2721 printed = true; 2722 } while_each_subsys_mask(); 2723 if (printed) 2724 seq_putc(seq, '\n'); 2725 } 2726 2727 /* show controllers which are enabled from the parent */ 2728 static int cgroup_controllers_show(struct seq_file *seq, void *v) 2729 { 2730 struct cgroup *cgrp = seq_css(seq)->cgroup; 2731 2732 cgroup_print_ss_mask(seq, cgroup_control(cgrp)); 2733 return 0; 2734 } 2735 2736 /* show controllers which are enabled for a given cgroup's children */ 2737 static int cgroup_subtree_control_show(struct seq_file *seq, void *v) 2738 { 2739 struct cgroup *cgrp = seq_css(seq)->cgroup; 2740 2741 cgroup_print_ss_mask(seq, cgrp->subtree_control); 2742 return 0; 2743 } 2744 2745 /** 2746 * cgroup_update_dfl_csses - update css assoc of a subtree in default hierarchy 2747 * @cgrp: root of the subtree to update csses for 2748 * 2749 * @cgrp's control masks have changed and its subtree's css associations 2750 * need to be updated accordingly. This function looks up all css_sets 2751 * which are attached to the subtree, creates the matching updated css_sets 2752 * and migrates the tasks to the new ones. 2753 */ 2754 static int cgroup_update_dfl_csses(struct cgroup *cgrp) 2755 { 2756 DEFINE_CGROUP_MGCTX(mgctx); 2757 struct cgroup_subsys_state *d_css; 2758 struct cgroup *dsct; 2759 struct css_set *src_cset; 2760 int ret; 2761 2762 lockdep_assert_held(&cgroup_mutex); 2763 2764 percpu_down_write(&cgroup_threadgroup_rwsem); 2765 2766 /* look up all csses currently attached to @cgrp's subtree */ 2767 spin_lock_irq(&css_set_lock); 2768 cgroup_for_each_live_descendant_pre(dsct, d_css, cgrp) { 2769 struct cgrp_cset_link *link; 2770 2771 list_for_each_entry(link, &dsct->cset_links, cset_link) 2772 cgroup_migrate_add_src(link->cset, dsct, &mgctx); 2773 } 2774 spin_unlock_irq(&css_set_lock); 2775 2776 /* NULL dst indicates self on default hierarchy */ 2777 ret = cgroup_migrate_prepare_dst(&mgctx); 2778 if (ret) 2779 goto out_finish; 2780 2781 spin_lock_irq(&css_set_lock); 2782 list_for_each_entry(src_cset, &mgctx.preloaded_src_csets, mg_preload_node) { 2783 struct task_struct *task, *ntask; 2784 2785 /* all tasks in src_csets need to be migrated */ 2786 list_for_each_entry_safe(task, ntask, &src_cset->tasks, cg_list) 2787 cgroup_migrate_add_task(task, &mgctx); 2788 } 2789 spin_unlock_irq(&css_set_lock); 2790 2791 ret = cgroup_migrate_execute(&mgctx); 2792 out_finish: 2793 cgroup_migrate_finish(&mgctx); 2794 percpu_up_write(&cgroup_threadgroup_rwsem); 2795 return ret; 2796 } 2797 2798 /** 2799 * cgroup_lock_and_drain_offline - lock cgroup_mutex and drain offlined csses 2800 * @cgrp: root of the target subtree 2801 * 2802 * Because css offlining is asynchronous, userland may try to re-enable a 2803 * controller while the previous css is still around. This function grabs 2804 * cgroup_mutex and drains the previous css instances of @cgrp's subtree. 2805 */ 2806 void cgroup_lock_and_drain_offline(struct cgroup *cgrp) 2807 __acquires(&cgroup_mutex) 2808 { 2809 struct cgroup *dsct; 2810 struct cgroup_subsys_state *d_css; 2811 struct cgroup_subsys *ss; 2812 int ssid; 2813 2814 restart: 2815 mutex_lock(&cgroup_mutex); 2816 2817 cgroup_for_each_live_descendant_post(dsct, d_css, cgrp) { 2818 for_each_subsys(ss, ssid) { 2819 struct cgroup_subsys_state *css = cgroup_css(dsct, ss); 2820 DEFINE_WAIT(wait); 2821 2822 if (!css || !percpu_ref_is_dying(&css->refcnt)) 2823 continue; 2824 2825 cgroup_get_live(dsct); 2826 prepare_to_wait(&dsct->offline_waitq, &wait, 2827 TASK_UNINTERRUPTIBLE); 2828 2829 mutex_unlock(&cgroup_mutex); 2830 schedule(); 2831 finish_wait(&dsct->offline_waitq, &wait); 2832 2833 cgroup_put(dsct); 2834 goto restart; 2835 } 2836 } 2837 } 2838 2839 /** 2840 * cgroup_save_control - save control masks and dom_cgrp of a subtree 2841 * @cgrp: root of the target subtree 2842 * 2843 * Save ->subtree_control, ->subtree_ss_mask and ->dom_cgrp to the 2844 * respective old_ prefixed fields for @cgrp's subtree including @cgrp 2845 * itself. 2846 */ 2847 static void cgroup_save_control(struct cgroup *cgrp) 2848 { 2849 struct cgroup *dsct; 2850 struct cgroup_subsys_state *d_css; 2851 2852 cgroup_for_each_live_descendant_pre(dsct, d_css, cgrp) { 2853 dsct->old_subtree_control = dsct->subtree_control; 2854 dsct->old_subtree_ss_mask = dsct->subtree_ss_mask; 2855 dsct->old_dom_cgrp = dsct->dom_cgrp; 2856 } 2857 } 2858 2859 /** 2860 * cgroup_propagate_control - refresh control masks of a subtree 2861 * @cgrp: root of the target subtree 2862 * 2863 * For @cgrp and its subtree, ensure ->subtree_ss_mask matches 2864 * ->subtree_control and propagate controller availability through the 2865 * subtree so that descendants don't have unavailable controllers enabled. 2866 */ 2867 static void cgroup_propagate_control(struct cgroup *cgrp) 2868 { 2869 struct cgroup *dsct; 2870 struct cgroup_subsys_state *d_css; 2871 2872 cgroup_for_each_live_descendant_pre(dsct, d_css, cgrp) { 2873 dsct->subtree_control &= cgroup_control(dsct); 2874 dsct->subtree_ss_mask = 2875 cgroup_calc_subtree_ss_mask(dsct->subtree_control, 2876 cgroup_ss_mask(dsct)); 2877 } 2878 } 2879 2880 /** 2881 * cgroup_restore_control - restore control masks and dom_cgrp of a subtree 2882 * @cgrp: root of the target subtree 2883 * 2884 * Restore ->subtree_control, ->subtree_ss_mask and ->dom_cgrp from the 2885 * respective old_ prefixed fields for @cgrp's subtree including @cgrp 2886 * itself. 2887 */ 2888 static void cgroup_restore_control(struct cgroup *cgrp) 2889 { 2890 struct cgroup *dsct; 2891 struct cgroup_subsys_state *d_css; 2892 2893 cgroup_for_each_live_descendant_post(dsct, d_css, cgrp) { 2894 dsct->subtree_control = dsct->old_subtree_control; 2895 dsct->subtree_ss_mask = dsct->old_subtree_ss_mask; 2896 dsct->dom_cgrp = dsct->old_dom_cgrp; 2897 } 2898 } 2899 2900 static bool css_visible(struct cgroup_subsys_state *css) 2901 { 2902 struct cgroup_subsys *ss = css->ss; 2903 struct cgroup *cgrp = css->cgroup; 2904 2905 if (cgroup_control(cgrp) & (1 << ss->id)) 2906 return true; 2907 if (!(cgroup_ss_mask(cgrp) & (1 << ss->id))) 2908 return false; 2909 return cgroup_on_dfl(cgrp) && ss->implicit_on_dfl; 2910 } 2911 2912 /** 2913 * cgroup_apply_control_enable - enable or show csses according to control 2914 * @cgrp: root of the target subtree 2915 * 2916 * Walk @cgrp's subtree and create new csses or make the existing ones 2917 * visible. A css is created invisible if it's being implicitly enabled 2918 * through dependency. An invisible css is made visible when the userland 2919 * explicitly enables it. 2920 * 2921 * Returns 0 on success, -errno on failure. On failure, csses which have 2922 * been processed already aren't cleaned up. The caller is responsible for 2923 * cleaning up with cgroup_apply_control_disable(). 2924 */ 2925 static int cgroup_apply_control_enable(struct cgroup *cgrp) 2926 { 2927 struct cgroup *dsct; 2928 struct cgroup_subsys_state *d_css; 2929 struct cgroup_subsys *ss; 2930 int ssid, ret; 2931 2932 cgroup_for_each_live_descendant_pre(dsct, d_css, cgrp) { 2933 for_each_subsys(ss, ssid) { 2934 struct cgroup_subsys_state *css = cgroup_css(dsct, ss); 2935 2936 WARN_ON_ONCE(css && percpu_ref_is_dying(&css->refcnt)); 2937 2938 if (!(cgroup_ss_mask(dsct) & (1 << ss->id))) 2939 continue; 2940 2941 if (!css) { 2942 css = css_create(dsct, ss); 2943 if (IS_ERR(css)) 2944 return PTR_ERR(css); 2945 } 2946 2947 if (css_visible(css)) { 2948 ret = css_populate_dir(css); 2949 if (ret) 2950 return ret; 2951 } 2952 } 2953 } 2954 2955 return 0; 2956 } 2957 2958 /** 2959 * cgroup_apply_control_disable - kill or hide csses according to control 2960 * @cgrp: root of the target subtree 2961 * 2962 * Walk @cgrp's subtree and kill and hide csses so that they match 2963 * cgroup_ss_mask() and cgroup_visible_mask(). 2964 * 2965 * A css is hidden when the userland requests it to be disabled while other 2966 * subsystems are still depending on it. The css must not actively control 2967 * resources and be in the vanilla state if it's made visible again later. 2968 * Controllers which may be depended upon should provide ->css_reset() for 2969 * this purpose. 2970 */ 2971 static void cgroup_apply_control_disable(struct cgroup *cgrp) 2972 { 2973 struct cgroup *dsct; 2974 struct cgroup_subsys_state *d_css; 2975 struct cgroup_subsys *ss; 2976 int ssid; 2977 2978 cgroup_for_each_live_descendant_post(dsct, d_css, cgrp) { 2979 for_each_subsys(ss, ssid) { 2980 struct cgroup_subsys_state *css = cgroup_css(dsct, ss); 2981 2982 WARN_ON_ONCE(css && percpu_ref_is_dying(&css->refcnt)); 2983 2984 if (!css) 2985 continue; 2986 2987 if (css->parent && 2988 !(cgroup_ss_mask(dsct) & (1 << ss->id))) { 2989 kill_css(css); 2990 } else if (!css_visible(css)) { 2991 css_clear_dir(css); 2992 if (ss->css_reset) 2993 ss->css_reset(css); 2994 } 2995 } 2996 } 2997 } 2998 2999 /** 3000 * cgroup_apply_control - apply control mask updates to the subtree 3001 * @cgrp: root of the target subtree 3002 * 3003 * subsystems can be enabled and disabled in a subtree using the following 3004 * steps. 3005 * 3006 * 1. Call cgroup_save_control() to stash the current state. 3007 * 2. Update ->subtree_control masks in the subtree as desired. 3008 * 3. Call cgroup_apply_control() to apply the changes. 3009 * 4. Optionally perform other related operations. 3010 * 5. Call cgroup_finalize_control() to finish up. 3011 * 3012 * This function implements step 3 and propagates the mask changes 3013 * throughout @cgrp's subtree, updates csses accordingly and perform 3014 * process migrations. 3015 */ 3016 static int cgroup_apply_control(struct cgroup *cgrp) 3017 { 3018 int ret; 3019 3020 cgroup_propagate_control(cgrp); 3021 3022 ret = cgroup_apply_control_enable(cgrp); 3023 if (ret) 3024 return ret; 3025 3026 /* 3027 * At this point, cgroup_e_css() results reflect the new csses 3028 * making the following cgroup_update_dfl_csses() properly update 3029 * css associations of all tasks in the subtree. 3030 */ 3031 ret = cgroup_update_dfl_csses(cgrp); 3032 if (ret) 3033 return ret; 3034 3035 return 0; 3036 } 3037 3038 /** 3039 * cgroup_finalize_control - finalize control mask update 3040 * @cgrp: root of the target subtree 3041 * @ret: the result of the update 3042 * 3043 * Finalize control mask update. See cgroup_apply_control() for more info. 3044 */ 3045 static void cgroup_finalize_control(struct cgroup *cgrp, int ret) 3046 { 3047 if (ret) { 3048 cgroup_restore_control(cgrp); 3049 cgroup_propagate_control(cgrp); 3050 } 3051 3052 cgroup_apply_control_disable(cgrp); 3053 } 3054 3055 static int cgroup_vet_subtree_control_enable(struct cgroup *cgrp, u16 enable) 3056 { 3057 u16 domain_enable = enable & ~cgrp_dfl_threaded_ss_mask; 3058 3059 /* if nothing is getting enabled, nothing to worry about */ 3060 if (!enable) 3061 return 0; 3062 3063 /* can @cgrp host any resources? */ 3064 if (!cgroup_is_valid_domain(cgrp->dom_cgrp)) 3065 return -EOPNOTSUPP; 3066 3067 /* mixables don't care */ 3068 if (cgroup_is_mixable(cgrp)) 3069 return 0; 3070 3071 if (domain_enable) { 3072 /* can't enable domain controllers inside a thread subtree */ 3073 if (cgroup_is_thread_root(cgrp) || cgroup_is_threaded(cgrp)) 3074 return -EOPNOTSUPP; 3075 } else { 3076 /* 3077 * Threaded controllers can handle internal competitions 3078 * and are always allowed inside a (prospective) thread 3079 * subtree. 3080 */ 3081 if (cgroup_can_be_thread_root(cgrp) || cgroup_is_threaded(cgrp)) 3082 return 0; 3083 } 3084 3085 /* 3086 * Controllers can't be enabled for a cgroup with tasks to avoid 3087 * child cgroups competing against tasks. 3088 */ 3089 if (cgroup_has_tasks(cgrp)) 3090 return -EBUSY; 3091 3092 return 0; 3093 } 3094 3095 /* change the enabled child controllers for a cgroup in the default hierarchy */ 3096 static ssize_t cgroup_subtree_control_write(struct kernfs_open_file *of, 3097 char *buf, size_t nbytes, 3098 loff_t off) 3099 { 3100 u16 enable = 0, disable = 0; 3101 struct cgroup *cgrp, *child; 3102 struct cgroup_subsys *ss; 3103 char *tok; 3104 int ssid, ret; 3105 3106 /* 3107 * Parse input - space separated list of subsystem names prefixed 3108 * with either + or -. 3109 */ 3110 buf = strstrip(buf); 3111 while ((tok = strsep(&buf, " "))) { 3112 if (tok[0] == '\0') 3113 continue; 3114 do_each_subsys_mask(ss, ssid, ~cgrp_dfl_inhibit_ss_mask) { 3115 if (!cgroup_ssid_enabled(ssid) || 3116 strcmp(tok + 1, ss->name)) 3117 continue; 3118 3119 if (*tok == '+') { 3120 enable |= 1 << ssid; 3121 disable &= ~(1 << ssid); 3122 } else if (*tok == '-') { 3123 disable |= 1 << ssid; 3124 enable &= ~(1 << ssid); 3125 } else { 3126 return -EINVAL; 3127 } 3128 break; 3129 } while_each_subsys_mask(); 3130 if (ssid == CGROUP_SUBSYS_COUNT) 3131 return -EINVAL; 3132 } 3133 3134 cgrp = cgroup_kn_lock_live(of->kn, true); 3135 if (!cgrp) 3136 return -ENODEV; 3137 3138 for_each_subsys(ss, ssid) { 3139 if (enable & (1 << ssid)) { 3140 if (cgrp->subtree_control & (1 << ssid)) { 3141 enable &= ~(1 << ssid); 3142 continue; 3143 } 3144 3145 if (!(cgroup_control(cgrp) & (1 << ssid))) { 3146 ret = -ENOENT; 3147 goto out_unlock; 3148 } 3149 } else if (disable & (1 << ssid)) { 3150 if (!(cgrp->subtree_control & (1 << ssid))) { 3151 disable &= ~(1 << ssid); 3152 continue; 3153 } 3154 3155 /* a child has it enabled? */ 3156 cgroup_for_each_live_child(child, cgrp) { 3157 if (child->subtree_control & (1 << ssid)) { 3158 ret = -EBUSY; 3159 goto out_unlock; 3160 } 3161 } 3162 } 3163 } 3164 3165 if (!enable && !disable) { 3166 ret = 0; 3167 goto out_unlock; 3168 } 3169 3170 ret = cgroup_vet_subtree_control_enable(cgrp, enable); 3171 if (ret) 3172 goto out_unlock; 3173 3174 /* save and update control masks and prepare csses */ 3175 cgroup_save_control(cgrp); 3176 3177 cgrp->subtree_control |= enable; 3178 cgrp->subtree_control &= ~disable; 3179 3180 ret = cgroup_apply_control(cgrp); 3181 cgroup_finalize_control(cgrp, ret); 3182 if (ret) 3183 goto out_unlock; 3184 3185 kernfs_activate(cgrp->kn); 3186 out_unlock: 3187 cgroup_kn_unlock(of->kn); 3188 return ret ?: nbytes; 3189 } 3190 3191 /** 3192 * cgroup_enable_threaded - make @cgrp threaded 3193 * @cgrp: the target cgroup 3194 * 3195 * Called when "threaded" is written to the cgroup.type interface file and 3196 * tries to make @cgrp threaded and join the parent's resource domain. 3197 * This function is never called on the root cgroup as cgroup.type doesn't 3198 * exist on it. 3199 */ 3200 static int cgroup_enable_threaded(struct cgroup *cgrp) 3201 { 3202 struct cgroup *parent = cgroup_parent(cgrp); 3203 struct cgroup *dom_cgrp = parent->dom_cgrp; 3204 struct cgroup *dsct; 3205 struct cgroup_subsys_state *d_css; 3206 int ret; 3207 3208 lockdep_assert_held(&cgroup_mutex); 3209 3210 /* noop if already threaded */ 3211 if (cgroup_is_threaded(cgrp)) 3212 return 0; 3213 3214 /* 3215 * If @cgroup is populated or has domain controllers enabled, it 3216 * can't be switched. While the below cgroup_can_be_thread_root() 3217 * test can catch the same conditions, that's only when @parent is 3218 * not mixable, so let's check it explicitly. 3219 */ 3220 if (cgroup_is_populated(cgrp) || 3221 cgrp->subtree_control & ~cgrp_dfl_threaded_ss_mask) 3222 return -EOPNOTSUPP; 3223 3224 /* we're joining the parent's domain, ensure its validity */ 3225 if (!cgroup_is_valid_domain(dom_cgrp) || 3226 !cgroup_can_be_thread_root(dom_cgrp)) 3227 return -EOPNOTSUPP; 3228 3229 /* 3230 * The following shouldn't cause actual migrations and should 3231 * always succeed. 3232 */ 3233 cgroup_save_control(cgrp); 3234 3235 cgroup_for_each_live_descendant_pre(dsct, d_css, cgrp) 3236 if (dsct == cgrp || cgroup_is_threaded(dsct)) 3237 dsct->dom_cgrp = dom_cgrp; 3238 3239 ret = cgroup_apply_control(cgrp); 3240 if (!ret) 3241 parent->nr_threaded_children++; 3242 3243 cgroup_finalize_control(cgrp, ret); 3244 return ret; 3245 } 3246 3247 static int cgroup_type_show(struct seq_file *seq, void *v) 3248 { 3249 struct cgroup *cgrp = seq_css(seq)->cgroup; 3250 3251 if (cgroup_is_threaded(cgrp)) 3252 seq_puts(seq, "threaded\n"); 3253 else if (!cgroup_is_valid_domain(cgrp)) 3254 seq_puts(seq, "domain invalid\n"); 3255 else if (cgroup_is_thread_root(cgrp)) 3256 seq_puts(seq, "domain threaded\n"); 3257 else 3258 seq_puts(seq, "domain\n"); 3259 3260 return 0; 3261 } 3262 3263 static ssize_t cgroup_type_write(struct kernfs_open_file *of, char *buf, 3264 size_t nbytes, loff_t off) 3265 { 3266 struct cgroup *cgrp; 3267 int ret; 3268 3269 /* only switching to threaded mode is supported */ 3270 if (strcmp(strstrip(buf), "threaded")) 3271 return -EINVAL; 3272 3273 cgrp = cgroup_kn_lock_live(of->kn, false); 3274 if (!cgrp) 3275 return -ENOENT; 3276 3277 /* threaded can only be enabled */ 3278 ret = cgroup_enable_threaded(cgrp); 3279 3280 cgroup_kn_unlock(of->kn); 3281 return ret ?: nbytes; 3282 } 3283 3284 static int cgroup_max_descendants_show(struct seq_file *seq, void *v) 3285 { 3286 struct cgroup *cgrp = seq_css(seq)->cgroup; 3287 int descendants = READ_ONCE(cgrp->max_descendants); 3288 3289 if (descendants == INT_MAX) 3290 seq_puts(seq, "max\n"); 3291 else 3292 seq_printf(seq, "%d\n", descendants); 3293 3294 return 0; 3295 } 3296 3297 static ssize_t cgroup_max_descendants_write(struct kernfs_open_file *of, 3298 char *buf, size_t nbytes, loff_t off) 3299 { 3300 struct cgroup *cgrp; 3301 int descendants; 3302 ssize_t ret; 3303 3304 buf = strstrip(buf); 3305 if (!strcmp(buf, "max")) { 3306 descendants = INT_MAX; 3307 } else { 3308 ret = kstrtoint(buf, 0, &descendants); 3309 if (ret) 3310 return ret; 3311 } 3312 3313 if (descendants < 0) 3314 return -ERANGE; 3315 3316 cgrp = cgroup_kn_lock_live(of->kn, false); 3317 if (!cgrp) 3318 return -ENOENT; 3319 3320 cgrp->max_descendants = descendants; 3321 3322 cgroup_kn_unlock(of->kn); 3323 3324 return nbytes; 3325 } 3326 3327 static int cgroup_max_depth_show(struct seq_file *seq, void *v) 3328 { 3329 struct cgroup *cgrp = seq_css(seq)->cgroup; 3330 int depth = READ_ONCE(cgrp->max_depth); 3331 3332 if (depth == INT_MAX) 3333 seq_puts(seq, "max\n"); 3334 else 3335 seq_printf(seq, "%d\n", depth); 3336 3337 return 0; 3338 } 3339 3340 static ssize_t cgroup_max_depth_write(struct kernfs_open_file *of, 3341 char *buf, size_t nbytes, loff_t off) 3342 { 3343 struct cgroup *cgrp; 3344 ssize_t ret; 3345 int depth; 3346 3347 buf = strstrip(buf); 3348 if (!strcmp(buf, "max")) { 3349 depth = INT_MAX; 3350 } else { 3351 ret = kstrtoint(buf, 0, &depth); 3352 if (ret) 3353 return ret; 3354 } 3355 3356 if (depth < 0) 3357 return -ERANGE; 3358 3359 cgrp = cgroup_kn_lock_live(of->kn, false); 3360 if (!cgrp) 3361 return -ENOENT; 3362 3363 cgrp->max_depth = depth; 3364 3365 cgroup_kn_unlock(of->kn); 3366 3367 return nbytes; 3368 } 3369 3370 static int cgroup_events_show(struct seq_file *seq, void *v) 3371 { 3372 seq_printf(seq, "populated %d\n", 3373 cgroup_is_populated(seq_css(seq)->cgroup)); 3374 return 0; 3375 } 3376 3377 static int cgroup_stat_show(struct seq_file *seq, void *v) 3378 { 3379 struct cgroup *cgroup = seq_css(seq)->cgroup; 3380 3381 seq_printf(seq, "nr_descendants %d\n", 3382 cgroup->nr_descendants); 3383 seq_printf(seq, "nr_dying_descendants %d\n", 3384 cgroup->nr_dying_descendants); 3385 3386 return 0; 3387 } 3388 3389 static int __maybe_unused cgroup_extra_stat_show(struct seq_file *seq, 3390 struct cgroup *cgrp, int ssid) 3391 { 3392 struct cgroup_subsys *ss = cgroup_subsys[ssid]; 3393 struct cgroup_subsys_state *css; 3394 int ret; 3395 3396 if (!ss->css_extra_stat_show) 3397 return 0; 3398 3399 css = cgroup_tryget_css(cgrp, ss); 3400 if (!css) 3401 return 0; 3402 3403 ret = ss->css_extra_stat_show(seq, css); 3404 css_put(css); 3405 return ret; 3406 } 3407 3408 static int cpu_stat_show(struct seq_file *seq, void *v) 3409 { 3410 struct cgroup __maybe_unused *cgrp = seq_css(seq)->cgroup; 3411 int ret = 0; 3412 3413 cgroup_base_stat_cputime_show(seq); 3414 #ifdef CONFIG_CGROUP_SCHED 3415 ret = cgroup_extra_stat_show(seq, cgrp, cpu_cgrp_id); 3416 #endif 3417 return ret; 3418 } 3419 3420 #ifdef CONFIG_PSI 3421 static int cgroup_io_pressure_show(struct seq_file *seq, void *v) 3422 { 3423 return psi_show(seq, &seq_css(seq)->cgroup->psi, PSI_IO); 3424 } 3425 static int cgroup_memory_pressure_show(struct seq_file *seq, void *v) 3426 { 3427 return psi_show(seq, &seq_css(seq)->cgroup->psi, PSI_MEM); 3428 } 3429 static int cgroup_cpu_pressure_show(struct seq_file *seq, void *v) 3430 { 3431 return psi_show(seq, &seq_css(seq)->cgroup->psi, PSI_CPU); 3432 } 3433 #endif 3434 3435 static int cgroup_file_open(struct kernfs_open_file *of) 3436 { 3437 struct cftype *cft = of->kn->priv; 3438 3439 if (cft->open) 3440 return cft->open(of); 3441 return 0; 3442 } 3443 3444 static void cgroup_file_release(struct kernfs_open_file *of) 3445 { 3446 struct cftype *cft = of->kn->priv; 3447 3448 if (cft->release) 3449 cft->release(of); 3450 } 3451 3452 static ssize_t cgroup_file_write(struct kernfs_open_file *of, char *buf, 3453 size_t nbytes, loff_t off) 3454 { 3455 struct cgroup_namespace *ns = current->nsproxy->cgroup_ns; 3456 struct cgroup *cgrp = of->kn->parent->priv; 3457 struct cftype *cft = of->kn->priv; 3458 struct cgroup_subsys_state *css; 3459 int ret; 3460 3461 /* 3462 * If namespaces are delegation boundaries, disallow writes to 3463 * files in an non-init namespace root from inside the namespace 3464 * except for the files explicitly marked delegatable - 3465 * cgroup.procs and cgroup.subtree_control. 3466 */ 3467 if ((cgrp->root->flags & CGRP_ROOT_NS_DELEGATE) && 3468 !(cft->flags & CFTYPE_NS_DELEGATABLE) && 3469 ns != &init_cgroup_ns && ns->root_cset->dfl_cgrp == cgrp) 3470 return -EPERM; 3471 3472 if (cft->write) 3473 return cft->write(of, buf, nbytes, off); 3474 3475 /* 3476 * kernfs guarantees that a file isn't deleted with operations in 3477 * flight, which means that the matching css is and stays alive and 3478 * doesn't need to be pinned. The RCU locking is not necessary 3479 * either. It's just for the convenience of using cgroup_css(). 3480 */ 3481 rcu_read_lock(); 3482 css = cgroup_css(cgrp, cft->ss); 3483 rcu_read_unlock(); 3484 3485 if (cft->write_u64) { 3486 unsigned long long v; 3487 ret = kstrtoull(buf, 0, &v); 3488 if (!ret) 3489 ret = cft->write_u64(css, cft, v); 3490 } else if (cft->write_s64) { 3491 long long v; 3492 ret = kstrtoll(buf, 0, &v); 3493 if (!ret) 3494 ret = cft->write_s64(css, cft, v); 3495 } else { 3496 ret = -EINVAL; 3497 } 3498 3499 return ret ?: nbytes; 3500 } 3501 3502 static void *cgroup_seqfile_start(struct seq_file *seq, loff_t *ppos) 3503 { 3504 return seq_cft(seq)->seq_start(seq, ppos); 3505 } 3506 3507 static void *cgroup_seqfile_next(struct seq_file *seq, void *v, loff_t *ppos) 3508 { 3509 return seq_cft(seq)->seq_next(seq, v, ppos); 3510 } 3511 3512 static void cgroup_seqfile_stop(struct seq_file *seq, void *v) 3513 { 3514 if (seq_cft(seq)->seq_stop) 3515 seq_cft(seq)->seq_stop(seq, v); 3516 } 3517 3518 static int cgroup_seqfile_show(struct seq_file *m, void *arg) 3519 { 3520 struct cftype *cft = seq_cft(m); 3521 struct cgroup_subsys_state *css = seq_css(m); 3522 3523 if (cft->seq_show) 3524 return cft->seq_show(m, arg); 3525 3526 if (cft->read_u64) 3527 seq_printf(m, "%llu\n", cft->read_u64(css, cft)); 3528 else if (cft->read_s64) 3529 seq_printf(m, "%lld\n", cft->read_s64(css, cft)); 3530 else 3531 return -EINVAL; 3532 return 0; 3533 } 3534 3535 static struct kernfs_ops cgroup_kf_single_ops = { 3536 .atomic_write_len = PAGE_SIZE, 3537 .open = cgroup_file_open, 3538 .release = cgroup_file_release, 3539 .write = cgroup_file_write, 3540 .seq_show = cgroup_seqfile_show, 3541 }; 3542 3543 static struct kernfs_ops cgroup_kf_ops = { 3544 .atomic_write_len = PAGE_SIZE, 3545 .open = cgroup_file_open, 3546 .release = cgroup_file_release, 3547 .write = cgroup_file_write, 3548 .seq_start = cgroup_seqfile_start, 3549 .seq_next = cgroup_seqfile_next, 3550 .seq_stop = cgroup_seqfile_stop, 3551 .seq_show = cgroup_seqfile_show, 3552 }; 3553 3554 /* set uid and gid of cgroup dirs and files to that of the creator */ 3555 static int cgroup_kn_set_ugid(struct kernfs_node *kn) 3556 { 3557 struct iattr iattr = { .ia_valid = ATTR_UID | ATTR_GID, 3558 .ia_uid = current_fsuid(), 3559 .ia_gid = current_fsgid(), }; 3560 3561 if (uid_eq(iattr.ia_uid, GLOBAL_ROOT_UID) && 3562 gid_eq(iattr.ia_gid, GLOBAL_ROOT_GID)) 3563 return 0; 3564 3565 return kernfs_setattr(kn, &iattr); 3566 } 3567 3568 static void cgroup_file_notify_timer(struct timer_list *timer) 3569 { 3570 cgroup_file_notify(container_of(timer, struct cgroup_file, 3571 notify_timer)); 3572 } 3573 3574 static int cgroup_add_file(struct cgroup_subsys_state *css, struct cgroup *cgrp, 3575 struct cftype *cft) 3576 { 3577 char name[CGROUP_FILE_NAME_MAX]; 3578 struct kernfs_node *kn; 3579 struct lock_class_key *key = NULL; 3580 int ret; 3581 3582 #ifdef CONFIG_DEBUG_LOCK_ALLOC 3583 key = &cft->lockdep_key; 3584 #endif 3585 kn = __kernfs_create_file(cgrp->kn, cgroup_file_name(cgrp, cft, name), 3586 cgroup_file_mode(cft), 3587 GLOBAL_ROOT_UID, GLOBAL_ROOT_GID, 3588 0, cft->kf_ops, cft, 3589 NULL, key); 3590 if (IS_ERR(kn)) 3591 return PTR_ERR(kn); 3592 3593 ret = cgroup_kn_set_ugid(kn); 3594 if (ret) { 3595 kernfs_remove(kn); 3596 return ret; 3597 } 3598 3599 if (cft->file_offset) { 3600 struct cgroup_file *cfile = (void *)css + cft->file_offset; 3601 3602 timer_setup(&cfile->notify_timer, cgroup_file_notify_timer, 0); 3603 3604 spin_lock_irq(&cgroup_file_kn_lock); 3605 cfile->kn = kn; 3606 spin_unlock_irq(&cgroup_file_kn_lock); 3607 } 3608 3609 return 0; 3610 } 3611 3612 /** 3613 * cgroup_addrm_files - add or remove files to a cgroup directory 3614 * @css: the target css 3615 * @cgrp: the target cgroup (usually css->cgroup) 3616 * @cfts: array of cftypes to be added 3617 * @is_add: whether to add or remove 3618 * 3619 * Depending on @is_add, add or remove files defined by @cfts on @cgrp. 3620 * For removals, this function never fails. 3621 */ 3622 static int cgroup_addrm_files(struct cgroup_subsys_state *css, 3623 struct cgroup *cgrp, struct cftype cfts[], 3624 bool is_add) 3625 { 3626 struct cftype *cft, *cft_end = NULL; 3627 int ret = 0; 3628 3629 lockdep_assert_held(&cgroup_mutex); 3630 3631 restart: 3632 for (cft = cfts; cft != cft_end && cft->name[0] != '\0'; cft++) { 3633 /* does cft->flags tell us to skip this file on @cgrp? */ 3634 if ((cft->flags & __CFTYPE_ONLY_ON_DFL) && !cgroup_on_dfl(cgrp)) 3635 continue; 3636 if ((cft->flags & __CFTYPE_NOT_ON_DFL) && cgroup_on_dfl(cgrp)) 3637 continue; 3638 if ((cft->flags & CFTYPE_NOT_ON_ROOT) && !cgroup_parent(cgrp)) 3639 continue; 3640 if ((cft->flags & CFTYPE_ONLY_ON_ROOT) && cgroup_parent(cgrp)) 3641 continue; 3642 3643 if (is_add) { 3644 ret = cgroup_add_file(css, cgrp, cft); 3645 if (ret) { 3646 pr_warn("%s: failed to add %s, err=%d\n", 3647 __func__, cft->name, ret); 3648 cft_end = cft; 3649 is_add = false; 3650 goto restart; 3651 } 3652 } else { 3653 cgroup_rm_file(cgrp, cft); 3654 } 3655 } 3656 return ret; 3657 } 3658 3659 static int cgroup_apply_cftypes(struct cftype *cfts, bool is_add) 3660 { 3661 struct cgroup_subsys *ss = cfts[0].ss; 3662 struct cgroup *root = &ss->root->cgrp; 3663 struct cgroup_subsys_state *css; 3664 int ret = 0; 3665 3666 lockdep_assert_held(&cgroup_mutex); 3667 3668 /* add/rm files for all cgroups created before */ 3669 css_for_each_descendant_pre(css, cgroup_css(root, ss)) { 3670 struct cgroup *cgrp = css->cgroup; 3671 3672 if (!(css->flags & CSS_VISIBLE)) 3673 continue; 3674 3675 ret = cgroup_addrm_files(css, cgrp, cfts, is_add); 3676 if (ret) 3677 break; 3678 } 3679 3680 if (is_add && !ret) 3681 kernfs_activate(root->kn); 3682 return ret; 3683 } 3684 3685 static void cgroup_exit_cftypes(struct cftype *cfts) 3686 { 3687 struct cftype *cft; 3688 3689 for (cft = cfts; cft->name[0] != '\0'; cft++) { 3690 /* free copy for custom atomic_write_len, see init_cftypes() */ 3691 if (cft->max_write_len && cft->max_write_len != PAGE_SIZE) 3692 kfree(cft->kf_ops); 3693 cft->kf_ops = NULL; 3694 cft->ss = NULL; 3695 3696 /* revert flags set by cgroup core while adding @cfts */ 3697 cft->flags &= ~(__CFTYPE_ONLY_ON_DFL | __CFTYPE_NOT_ON_DFL); 3698 } 3699 } 3700 3701 static int cgroup_init_cftypes(struct cgroup_subsys *ss, struct cftype *cfts) 3702 { 3703 struct cftype *cft; 3704 3705 for (cft = cfts; cft->name[0] != '\0'; cft++) { 3706 struct kernfs_ops *kf_ops; 3707 3708 WARN_ON(cft->ss || cft->kf_ops); 3709 3710 if (cft->seq_start) 3711 kf_ops = &cgroup_kf_ops; 3712 else 3713 kf_ops = &cgroup_kf_single_ops; 3714 3715 /* 3716 * Ugh... if @cft wants a custom max_write_len, we need to 3717 * make a copy of kf_ops to set its atomic_write_len. 3718 */ 3719 if (cft->max_write_len && cft->max_write_len != PAGE_SIZE) { 3720 kf_ops = kmemdup(kf_ops, sizeof(*kf_ops), GFP_KERNEL); 3721 if (!kf_ops) { 3722 cgroup_exit_cftypes(cfts); 3723 return -ENOMEM; 3724 } 3725 kf_ops->atomic_write_len = cft->max_write_len; 3726 } 3727 3728 cft->kf_ops = kf_ops; 3729 cft->ss = ss; 3730 } 3731 3732 return 0; 3733 } 3734 3735 static int cgroup_rm_cftypes_locked(struct cftype *cfts) 3736 { 3737 lockdep_assert_held(&cgroup_mutex); 3738 3739 if (!cfts || !cfts[0].ss) 3740 return -ENOENT; 3741 3742 list_del(&cfts->node); 3743 cgroup_apply_cftypes(cfts, false); 3744 cgroup_exit_cftypes(cfts); 3745 return 0; 3746 } 3747 3748 /** 3749 * cgroup_rm_cftypes - remove an array of cftypes from a subsystem 3750 * @cfts: zero-length name terminated array of cftypes 3751 * 3752 * Unregister @cfts. Files described by @cfts are removed from all 3753 * existing cgroups and all future cgroups won't have them either. This 3754 * function can be called anytime whether @cfts' subsys is attached or not. 3755 * 3756 * Returns 0 on successful unregistration, -ENOENT if @cfts is not 3757 * registered. 3758 */ 3759 int cgroup_rm_cftypes(struct cftype *cfts) 3760 { 3761 int ret; 3762 3763 mutex_lock(&cgroup_mutex); 3764 ret = cgroup_rm_cftypes_locked(cfts); 3765 mutex_unlock(&cgroup_mutex); 3766 return ret; 3767 } 3768 3769 /** 3770 * cgroup_add_cftypes - add an array of cftypes to a subsystem 3771 * @ss: target cgroup subsystem 3772 * @cfts: zero-length name terminated array of cftypes 3773 * 3774 * Register @cfts to @ss. Files described by @cfts are created for all 3775 * existing cgroups to which @ss is attached and all future cgroups will 3776 * have them too. This function can be called anytime whether @ss is 3777 * attached or not. 3778 * 3779 * Returns 0 on successful registration, -errno on failure. Note that this 3780 * function currently returns 0 as long as @cfts registration is successful 3781 * even if some file creation attempts on existing cgroups fail. 3782 */ 3783 static int cgroup_add_cftypes(struct cgroup_subsys *ss, struct cftype *cfts) 3784 { 3785 int ret; 3786 3787 if (!cgroup_ssid_enabled(ss->id)) 3788 return 0; 3789 3790 if (!cfts || cfts[0].name[0] == '\0') 3791 return 0; 3792 3793 ret = cgroup_init_cftypes(ss, cfts); 3794 if (ret) 3795 return ret; 3796 3797 mutex_lock(&cgroup_mutex); 3798 3799 list_add_tail(&cfts->node, &ss->cfts); 3800 ret = cgroup_apply_cftypes(cfts, true); 3801 if (ret) 3802 cgroup_rm_cftypes_locked(cfts); 3803 3804 mutex_unlock(&cgroup_mutex); 3805 return ret; 3806 } 3807 3808 /** 3809 * cgroup_add_dfl_cftypes - add an array of cftypes for default hierarchy 3810 * @ss: target cgroup subsystem 3811 * @cfts: zero-length name terminated array of cftypes 3812 * 3813 * Similar to cgroup_add_cftypes() but the added files are only used for 3814 * the default hierarchy. 3815 */ 3816 int cgroup_add_dfl_cftypes(struct cgroup_subsys *ss, struct cftype *cfts) 3817 { 3818 struct cftype *cft; 3819 3820 for (cft = cfts; cft && cft->name[0] != '\0'; cft++) 3821 cft->flags |= __CFTYPE_ONLY_ON_DFL; 3822 return cgroup_add_cftypes(ss, cfts); 3823 } 3824 3825 /** 3826 * cgroup_add_legacy_cftypes - add an array of cftypes for legacy hierarchies 3827 * @ss: target cgroup subsystem 3828 * @cfts: zero-length name terminated array of cftypes 3829 * 3830 * Similar to cgroup_add_cftypes() but the added files are only used for 3831 * the legacy hierarchies. 3832 */ 3833 int cgroup_add_legacy_cftypes(struct cgroup_subsys *ss, struct cftype *cfts) 3834 { 3835 struct cftype *cft; 3836 3837 for (cft = cfts; cft && cft->name[0] != '\0'; cft++) 3838 cft->flags |= __CFTYPE_NOT_ON_DFL; 3839 return cgroup_add_cftypes(ss, cfts); 3840 } 3841 3842 /** 3843 * cgroup_file_notify - generate a file modified event for a cgroup_file 3844 * @cfile: target cgroup_file 3845 * 3846 * @cfile must have been obtained by setting cftype->file_offset. 3847 */ 3848 void cgroup_file_notify(struct cgroup_file *cfile) 3849 { 3850 unsigned long flags; 3851 3852 spin_lock_irqsave(&cgroup_file_kn_lock, flags); 3853 if (cfile->kn) { 3854 unsigned long last = cfile->notified_at; 3855 unsigned long next = last + CGROUP_FILE_NOTIFY_MIN_INTV; 3856 3857 if (time_in_range(jiffies, last, next)) { 3858 timer_reduce(&cfile->notify_timer, next); 3859 } else { 3860 kernfs_notify(cfile->kn); 3861 cfile->notified_at = jiffies; 3862 } 3863 } 3864 spin_unlock_irqrestore(&cgroup_file_kn_lock, flags); 3865 } 3866 3867 /** 3868 * css_next_child - find the next child of a given css 3869 * @pos: the current position (%NULL to initiate traversal) 3870 * @parent: css whose children to walk 3871 * 3872 * This function returns the next child of @parent and should be called 3873 * under either cgroup_mutex or RCU read lock. The only requirement is 3874 * that @parent and @pos are accessible. The next sibling is guaranteed to 3875 * be returned regardless of their states. 3876 * 3877 * If a subsystem synchronizes ->css_online() and the start of iteration, a 3878 * css which finished ->css_online() is guaranteed to be visible in the 3879 * future iterations and will stay visible until the last reference is put. 3880 * A css which hasn't finished ->css_online() or already finished 3881 * ->css_offline() may show up during traversal. It's each subsystem's 3882 * responsibility to synchronize against on/offlining. 3883 */ 3884 struct cgroup_subsys_state *css_next_child(struct cgroup_subsys_state *pos, 3885 struct cgroup_subsys_state *parent) 3886 { 3887 struct cgroup_subsys_state *next; 3888 3889 cgroup_assert_mutex_or_rcu_locked(); 3890 3891 /* 3892 * @pos could already have been unlinked from the sibling list. 3893 * Once a cgroup is removed, its ->sibling.next is no longer 3894 * updated when its next sibling changes. CSS_RELEASED is set when 3895 * @pos is taken off list, at which time its next pointer is valid, 3896 * and, as releases are serialized, the one pointed to by the next 3897 * pointer is guaranteed to not have started release yet. This 3898 * implies that if we observe !CSS_RELEASED on @pos in this RCU 3899 * critical section, the one pointed to by its next pointer is 3900 * guaranteed to not have finished its RCU grace period even if we 3901 * have dropped rcu_read_lock() inbetween iterations. 3902 * 3903 * If @pos has CSS_RELEASED set, its next pointer can't be 3904 * dereferenced; however, as each css is given a monotonically 3905 * increasing unique serial number and always appended to the 3906 * sibling list, the next one can be found by walking the parent's 3907 * children until the first css with higher serial number than 3908 * @pos's. While this path can be slower, it happens iff iteration 3909 * races against release and the race window is very small. 3910 */ 3911 if (!pos) { 3912 next = list_entry_rcu(parent->children.next, struct cgroup_subsys_state, sibling); 3913 } else if (likely(!(pos->flags & CSS_RELEASED))) { 3914 next = list_entry_rcu(pos->sibling.next, struct cgroup_subsys_state, sibling); 3915 } else { 3916 list_for_each_entry_rcu(next, &parent->children, sibling) 3917 if (next->serial_nr > pos->serial_nr) 3918 break; 3919 } 3920 3921 /* 3922 * @next, if not pointing to the head, can be dereferenced and is 3923 * the next sibling. 3924 */ 3925 if (&next->sibling != &parent->children) 3926 return next; 3927 return NULL; 3928 } 3929 3930 /** 3931 * css_next_descendant_pre - find the next descendant for pre-order walk 3932 * @pos: the current position (%NULL to initiate traversal) 3933 * @root: css whose descendants to walk 3934 * 3935 * To be used by css_for_each_descendant_pre(). Find the next descendant 3936 * to visit for pre-order traversal of @root's descendants. @root is 3937 * included in the iteration and the first node to be visited. 3938 * 3939 * While this function requires cgroup_mutex or RCU read locking, it 3940 * doesn't require the whole traversal to be contained in a single critical 3941 * section. This function will return the correct next descendant as long 3942 * as both @pos and @root are accessible and @pos is a descendant of @root. 3943 * 3944 * If a subsystem synchronizes ->css_online() and the start of iteration, a 3945 * css which finished ->css_online() is guaranteed to be visible in the 3946 * future iterations and will stay visible until the last reference is put. 3947 * A css which hasn't finished ->css_online() or already finished 3948 * ->css_offline() may show up during traversal. It's each subsystem's 3949 * responsibility to synchronize against on/offlining. 3950 */ 3951 struct cgroup_subsys_state * 3952 css_next_descendant_pre(struct cgroup_subsys_state *pos, 3953 struct cgroup_subsys_state *root) 3954 { 3955 struct cgroup_subsys_state *next; 3956 3957 cgroup_assert_mutex_or_rcu_locked(); 3958 3959 /* if first iteration, visit @root */ 3960 if (!pos) 3961 return root; 3962 3963 /* visit the first child if exists */ 3964 next = css_next_child(NULL, pos); 3965 if (next) 3966 return next; 3967 3968 /* no child, visit my or the closest ancestor's next sibling */ 3969 while (pos != root) { 3970 next = css_next_child(pos, pos->parent); 3971 if (next) 3972 return next; 3973 pos = pos->parent; 3974 } 3975 3976 return NULL; 3977 } 3978 3979 /** 3980 * css_rightmost_descendant - return the rightmost descendant of a css 3981 * @pos: css of interest 3982 * 3983 * Return the rightmost descendant of @pos. If there's no descendant, @pos 3984 * is returned. This can be used during pre-order traversal to skip 3985 * subtree of @pos. 3986 * 3987 * While this function requires cgroup_mutex or RCU read locking, it 3988 * doesn't require the whole traversal to be contained in a single critical 3989 * section. This function will return the correct rightmost descendant as 3990 * long as @pos is accessible. 3991 */ 3992 struct cgroup_subsys_state * 3993 css_rightmost_descendant(struct cgroup_subsys_state *pos) 3994 { 3995 struct cgroup_subsys_state *last, *tmp; 3996 3997 cgroup_assert_mutex_or_rcu_locked(); 3998 3999 do { 4000 last = pos; 4001 /* ->prev isn't RCU safe, walk ->next till the end */ 4002 pos = NULL; 4003 css_for_each_child(tmp, last) 4004 pos = tmp; 4005 } while (pos); 4006 4007 return last; 4008 } 4009 4010 static struct cgroup_subsys_state * 4011 css_leftmost_descendant(struct cgroup_subsys_state *pos) 4012 { 4013 struct cgroup_subsys_state *last; 4014 4015 do { 4016 last = pos; 4017 pos = css_next_child(NULL, pos); 4018 } while (pos); 4019 4020 return last; 4021 } 4022 4023 /** 4024 * css_next_descendant_post - find the next descendant for post-order walk 4025 * @pos: the current position (%NULL to initiate traversal) 4026 * @root: css whose descendants to walk 4027 * 4028 * To be used by css_for_each_descendant_post(). Find the next descendant 4029 * to visit for post-order traversal of @root's descendants. @root is 4030 * included in the iteration and the last node to be visited. 4031 * 4032 * While this function requires cgroup_mutex or RCU read locking, it 4033 * doesn't require the whole traversal to be contained in a single critical 4034 * section. This function will return the correct next descendant as long 4035 * as both @pos and @cgroup are accessible and @pos is a descendant of 4036 * @cgroup. 4037 * 4038 * If a subsystem synchronizes ->css_online() and the start of iteration, a 4039 * css which finished ->css_online() is guaranteed to be visible in the 4040 * future iterations and will stay visible until the last reference is put. 4041 * A css which hasn't finished ->css_online() or already finished 4042 * ->css_offline() may show up during traversal. It's each subsystem's 4043 * responsibility to synchronize against on/offlining. 4044 */ 4045 struct cgroup_subsys_state * 4046 css_next_descendant_post(struct cgroup_subsys_state *pos, 4047 struct cgroup_subsys_state *root) 4048 { 4049 struct cgroup_subsys_state *next; 4050 4051 cgroup_assert_mutex_or_rcu_locked(); 4052 4053 /* if first iteration, visit leftmost descendant which may be @root */ 4054 if (!pos) 4055 return css_leftmost_descendant(root); 4056 4057 /* if we visited @root, we're done */ 4058 if (pos == root) 4059 return NULL; 4060 4061 /* if there's an unvisited sibling, visit its leftmost descendant */ 4062 next = css_next_child(pos, pos->parent); 4063 if (next) 4064 return css_leftmost_descendant(next); 4065 4066 /* no sibling left, visit parent */ 4067 return pos->parent; 4068 } 4069 4070 /** 4071 * css_has_online_children - does a css have online children 4072 * @css: the target css 4073 * 4074 * Returns %true if @css has any online children; otherwise, %false. This 4075 * function can be called from any context but the caller is responsible 4076 * for synchronizing against on/offlining as necessary. 4077 */ 4078 bool css_has_online_children(struct cgroup_subsys_state *css) 4079 { 4080 struct cgroup_subsys_state *child; 4081 bool ret = false; 4082 4083 rcu_read_lock(); 4084 css_for_each_child(child, css) { 4085 if (child->flags & CSS_ONLINE) { 4086 ret = true; 4087 break; 4088 } 4089 } 4090 rcu_read_unlock(); 4091 return ret; 4092 } 4093 4094 static struct css_set *css_task_iter_next_css_set(struct css_task_iter *it) 4095 { 4096 struct list_head *l; 4097 struct cgrp_cset_link *link; 4098 struct css_set *cset; 4099 4100 lockdep_assert_held(&css_set_lock); 4101 4102 /* find the next threaded cset */ 4103 if (it->tcset_pos) { 4104 l = it->tcset_pos->next; 4105 4106 if (l != it->tcset_head) { 4107 it->tcset_pos = l; 4108 return container_of(l, struct css_set, 4109 threaded_csets_node); 4110 } 4111 4112 it->tcset_pos = NULL; 4113 } 4114 4115 /* find the next cset */ 4116 l = it->cset_pos; 4117 l = l->next; 4118 if (l == it->cset_head) { 4119 it->cset_pos = NULL; 4120 return NULL; 4121 } 4122 4123 if (it->ss) { 4124 cset = container_of(l, struct css_set, e_cset_node[it->ss->id]); 4125 } else { 4126 link = list_entry(l, struct cgrp_cset_link, cset_link); 4127 cset = link->cset; 4128 } 4129 4130 it->cset_pos = l; 4131 4132 /* initialize threaded css_set walking */ 4133 if (it->flags & CSS_TASK_ITER_THREADED) { 4134 if (it->cur_dcset) 4135 put_css_set_locked(it->cur_dcset); 4136 it->cur_dcset = cset; 4137 get_css_set(cset); 4138 4139 it->tcset_head = &cset->threaded_csets; 4140 it->tcset_pos = &cset->threaded_csets; 4141 } 4142 4143 return cset; 4144 } 4145 4146 /** 4147 * css_task_iter_advance_css_set - advance a task itererator to the next css_set 4148 * @it: the iterator to advance 4149 * 4150 * Advance @it to the next css_set to walk. 4151 */ 4152 static void css_task_iter_advance_css_set(struct css_task_iter *it) 4153 { 4154 struct css_set *cset; 4155 4156 lockdep_assert_held(&css_set_lock); 4157 4158 /* Advance to the next non-empty css_set */ 4159 do { 4160 cset = css_task_iter_next_css_set(it); 4161 if (!cset) { 4162 it->task_pos = NULL; 4163 return; 4164 } 4165 } while (!css_set_populated(cset)); 4166 4167 if (!list_empty(&cset->tasks)) 4168 it->task_pos = cset->tasks.next; 4169 else 4170 it->task_pos = cset->mg_tasks.next; 4171 4172 it->tasks_head = &cset->tasks; 4173 it->mg_tasks_head = &cset->mg_tasks; 4174 4175 /* 4176 * We don't keep css_sets locked across iteration steps and thus 4177 * need to take steps to ensure that iteration can be resumed after 4178 * the lock is re-acquired. Iteration is performed at two levels - 4179 * css_sets and tasks in them. 4180 * 4181 * Once created, a css_set never leaves its cgroup lists, so a 4182 * pinned css_set is guaranteed to stay put and we can resume 4183 * iteration afterwards. 4184 * 4185 * Tasks may leave @cset across iteration steps. This is resolved 4186 * by registering each iterator with the css_set currently being 4187 * walked and making css_set_move_task() advance iterators whose 4188 * next task is leaving. 4189 */ 4190 if (it->cur_cset) { 4191 list_del(&it->iters_node); 4192 put_css_set_locked(it->cur_cset); 4193 } 4194 get_css_set(cset); 4195 it->cur_cset = cset; 4196 list_add(&it->iters_node, &cset->task_iters); 4197 } 4198 4199 static void css_task_iter_advance(struct css_task_iter *it) 4200 { 4201 struct list_head *next; 4202 4203 lockdep_assert_held(&css_set_lock); 4204 repeat: 4205 /* 4206 * Advance iterator to find next entry. cset->tasks is consumed 4207 * first and then ->mg_tasks. After ->mg_tasks, we move onto the 4208 * next cset. 4209 */ 4210 next = it->task_pos->next; 4211 4212 if (next == it->tasks_head) 4213 next = it->mg_tasks_head->next; 4214 4215 if (next == it->mg_tasks_head) 4216 css_task_iter_advance_css_set(it); 4217 else 4218 it->task_pos = next; 4219 4220 /* if PROCS, skip over tasks which aren't group leaders */ 4221 if ((it->flags & CSS_TASK_ITER_PROCS) && it->task_pos && 4222 !thread_group_leader(list_entry(it->task_pos, struct task_struct, 4223 cg_list))) 4224 goto repeat; 4225 } 4226 4227 /** 4228 * css_task_iter_start - initiate task iteration 4229 * @css: the css to walk tasks of 4230 * @flags: CSS_TASK_ITER_* flags 4231 * @it: the task iterator to use 4232 * 4233 * Initiate iteration through the tasks of @css. The caller can call 4234 * css_task_iter_next() to walk through the tasks until the function 4235 * returns NULL. On completion of iteration, css_task_iter_end() must be 4236 * called. 4237 */ 4238 void css_task_iter_start(struct cgroup_subsys_state *css, unsigned int flags, 4239 struct css_task_iter *it) 4240 { 4241 /* no one should try to iterate before mounting cgroups */ 4242 WARN_ON_ONCE(!use_task_css_set_links); 4243 4244 memset(it, 0, sizeof(*it)); 4245 4246 spin_lock_irq(&css_set_lock); 4247 4248 it->ss = css->ss; 4249 it->flags = flags; 4250 4251 if (it->ss) 4252 it->cset_pos = &css->cgroup->e_csets[css->ss->id]; 4253 else 4254 it->cset_pos = &css->cgroup->cset_links; 4255 4256 it->cset_head = it->cset_pos; 4257 4258 css_task_iter_advance_css_set(it); 4259 4260 spin_unlock_irq(&css_set_lock); 4261 } 4262 4263 /** 4264 * css_task_iter_next - return the next task for the iterator 4265 * @it: the task iterator being iterated 4266 * 4267 * The "next" function for task iteration. @it should have been 4268 * initialized via css_task_iter_start(). Returns NULL when the iteration 4269 * reaches the end. 4270 */ 4271 struct task_struct *css_task_iter_next(struct css_task_iter *it) 4272 { 4273 if (it->cur_task) { 4274 put_task_struct(it->cur_task); 4275 it->cur_task = NULL; 4276 } 4277 4278 spin_lock_irq(&css_set_lock); 4279 4280 if (it->task_pos) { 4281 it->cur_task = list_entry(it->task_pos, struct task_struct, 4282 cg_list); 4283 get_task_struct(it->cur_task); 4284 css_task_iter_advance(it); 4285 } 4286 4287 spin_unlock_irq(&css_set_lock); 4288 4289 return it->cur_task; 4290 } 4291 4292 /** 4293 * css_task_iter_end - finish task iteration 4294 * @it: the task iterator to finish 4295 * 4296 * Finish task iteration started by css_task_iter_start(). 4297 */ 4298 void css_task_iter_end(struct css_task_iter *it) 4299 { 4300 if (it->cur_cset) { 4301 spin_lock_irq(&css_set_lock); 4302 list_del(&it->iters_node); 4303 put_css_set_locked(it->cur_cset); 4304 spin_unlock_irq(&css_set_lock); 4305 } 4306 4307 if (it->cur_dcset) 4308 put_css_set(it->cur_dcset); 4309 4310 if (it->cur_task) 4311 put_task_struct(it->cur_task); 4312 } 4313 4314 static void cgroup_procs_release(struct kernfs_open_file *of) 4315 { 4316 if (of->priv) { 4317 css_task_iter_end(of->priv); 4318 kfree(of->priv); 4319 } 4320 } 4321 4322 static void *cgroup_procs_next(struct seq_file *s, void *v, loff_t *pos) 4323 { 4324 struct kernfs_open_file *of = s->private; 4325 struct css_task_iter *it = of->priv; 4326 4327 return css_task_iter_next(it); 4328 } 4329 4330 static void *__cgroup_procs_start(struct seq_file *s, loff_t *pos, 4331 unsigned int iter_flags) 4332 { 4333 struct kernfs_open_file *of = s->private; 4334 struct cgroup *cgrp = seq_css(s)->cgroup; 4335 struct css_task_iter *it = of->priv; 4336 4337 /* 4338 * When a seq_file is seeked, it's always traversed sequentially 4339 * from position 0, so we can simply keep iterating on !0 *pos. 4340 */ 4341 if (!it) { 4342 if (WARN_ON_ONCE((*pos)++)) 4343 return ERR_PTR(-EINVAL); 4344 4345 it = kzalloc(sizeof(*it), GFP_KERNEL); 4346 if (!it) 4347 return ERR_PTR(-ENOMEM); 4348 of->priv = it; 4349 css_task_iter_start(&cgrp->self, iter_flags, it); 4350 } else if (!(*pos)++) { 4351 css_task_iter_end(it); 4352 css_task_iter_start(&cgrp->self, iter_flags, it); 4353 } 4354 4355 return cgroup_procs_next(s, NULL, NULL); 4356 } 4357 4358 static void *cgroup_procs_start(struct seq_file *s, loff_t *pos) 4359 { 4360 struct cgroup *cgrp = seq_css(s)->cgroup; 4361 4362 /* 4363 * All processes of a threaded subtree belong to the domain cgroup 4364 * of the subtree. Only threads can be distributed across the 4365 * subtree. Reject reads on cgroup.procs in the subtree proper. 4366 * They're always empty anyway. 4367 */ 4368 if (cgroup_is_threaded(cgrp)) 4369 return ERR_PTR(-EOPNOTSUPP); 4370 4371 return __cgroup_procs_start(s, pos, CSS_TASK_ITER_PROCS | 4372 CSS_TASK_ITER_THREADED); 4373 } 4374 4375 static int cgroup_procs_show(struct seq_file *s, void *v) 4376 { 4377 seq_printf(s, "%d\n", task_pid_vnr(v)); 4378 return 0; 4379 } 4380 4381 static int cgroup_procs_write_permission(struct cgroup *src_cgrp, 4382 struct cgroup *dst_cgrp, 4383 struct super_block *sb) 4384 { 4385 struct cgroup_namespace *ns = current->nsproxy->cgroup_ns; 4386 struct cgroup *com_cgrp = src_cgrp; 4387 struct inode *inode; 4388 int ret; 4389 4390 lockdep_assert_held(&cgroup_mutex); 4391 4392 /* find the common ancestor */ 4393 while (!cgroup_is_descendant(dst_cgrp, com_cgrp)) 4394 com_cgrp = cgroup_parent(com_cgrp); 4395 4396 /* %current should be authorized to migrate to the common ancestor */ 4397 inode = kernfs_get_inode(sb, com_cgrp->procs_file.kn); 4398 if (!inode) 4399 return -ENOMEM; 4400 4401 ret = inode_permission(inode, MAY_WRITE); 4402 iput(inode); 4403 if (ret) 4404 return ret; 4405 4406 /* 4407 * If namespaces are delegation boundaries, %current must be able 4408 * to see both source and destination cgroups from its namespace. 4409 */ 4410 if ((cgrp_dfl_root.flags & CGRP_ROOT_NS_DELEGATE) && 4411 (!cgroup_is_descendant(src_cgrp, ns->root_cset->dfl_cgrp) || 4412 !cgroup_is_descendant(dst_cgrp, ns->root_cset->dfl_cgrp))) 4413 return -ENOENT; 4414 4415 return 0; 4416 } 4417 4418 static ssize_t cgroup_procs_write(struct kernfs_open_file *of, 4419 char *buf, size_t nbytes, loff_t off) 4420 { 4421 struct cgroup *src_cgrp, *dst_cgrp; 4422 struct task_struct *task; 4423 ssize_t ret; 4424 4425 dst_cgrp = cgroup_kn_lock_live(of->kn, false); 4426 if (!dst_cgrp) 4427 return -ENODEV; 4428 4429 task = cgroup_procs_write_start(buf, true); 4430 ret = PTR_ERR_OR_ZERO(task); 4431 if (ret) 4432 goto out_unlock; 4433 4434 /* find the source cgroup */ 4435 spin_lock_irq(&css_set_lock); 4436 src_cgrp = task_cgroup_from_root(task, &cgrp_dfl_root); 4437 spin_unlock_irq(&css_set_lock); 4438 4439 ret = cgroup_procs_write_permission(src_cgrp, dst_cgrp, 4440 of->file->f_path.dentry->d_sb); 4441 if (ret) 4442 goto out_finish; 4443 4444 ret = cgroup_attach_task(dst_cgrp, task, true); 4445 4446 out_finish: 4447 cgroup_procs_write_finish(task); 4448 out_unlock: 4449 cgroup_kn_unlock(of->kn); 4450 4451 return ret ?: nbytes; 4452 } 4453 4454 static void *cgroup_threads_start(struct seq_file *s, loff_t *pos) 4455 { 4456 return __cgroup_procs_start(s, pos, 0); 4457 } 4458 4459 static ssize_t cgroup_threads_write(struct kernfs_open_file *of, 4460 char *buf, size_t nbytes, loff_t off) 4461 { 4462 struct cgroup *src_cgrp, *dst_cgrp; 4463 struct task_struct *task; 4464 ssize_t ret; 4465 4466 buf = strstrip(buf); 4467 4468 dst_cgrp = cgroup_kn_lock_live(of->kn, false); 4469 if (!dst_cgrp) 4470 return -ENODEV; 4471 4472 task = cgroup_procs_write_start(buf, false); 4473 ret = PTR_ERR_OR_ZERO(task); 4474 if (ret) 4475 goto out_unlock; 4476 4477 /* find the source cgroup */ 4478 spin_lock_irq(&css_set_lock); 4479 src_cgrp = task_cgroup_from_root(task, &cgrp_dfl_root); 4480 spin_unlock_irq(&css_set_lock); 4481 4482 /* thread migrations follow the cgroup.procs delegation rule */ 4483 ret = cgroup_procs_write_permission(src_cgrp, dst_cgrp, 4484 of->file->f_path.dentry->d_sb); 4485 if (ret) 4486 goto out_finish; 4487 4488 /* and must be contained in the same domain */ 4489 ret = -EOPNOTSUPP; 4490 if (src_cgrp->dom_cgrp != dst_cgrp->dom_cgrp) 4491 goto out_finish; 4492 4493 ret = cgroup_attach_task(dst_cgrp, task, false); 4494 4495 out_finish: 4496 cgroup_procs_write_finish(task); 4497 out_unlock: 4498 cgroup_kn_unlock(of->kn); 4499 4500 return ret ?: nbytes; 4501 } 4502 4503 /* cgroup core interface files for the default hierarchy */ 4504 static struct cftype cgroup_base_files[] = { 4505 { 4506 .name = "cgroup.type", 4507 .flags = CFTYPE_NOT_ON_ROOT, 4508 .seq_show = cgroup_type_show, 4509 .write = cgroup_type_write, 4510 }, 4511 { 4512 .name = "cgroup.procs", 4513 .flags = CFTYPE_NS_DELEGATABLE, 4514 .file_offset = offsetof(struct cgroup, procs_file), 4515 .release = cgroup_procs_release, 4516 .seq_start = cgroup_procs_start, 4517 .seq_next = cgroup_procs_next, 4518 .seq_show = cgroup_procs_show, 4519 .write = cgroup_procs_write, 4520 }, 4521 { 4522 .name = "cgroup.threads", 4523 .flags = CFTYPE_NS_DELEGATABLE, 4524 .release = cgroup_procs_release, 4525 .seq_start = cgroup_threads_start, 4526 .seq_next = cgroup_procs_next, 4527 .seq_show = cgroup_procs_show, 4528 .write = cgroup_threads_write, 4529 }, 4530 { 4531 .name = "cgroup.controllers", 4532 .seq_show = cgroup_controllers_show, 4533 }, 4534 { 4535 .name = "cgroup.subtree_control", 4536 .flags = CFTYPE_NS_DELEGATABLE, 4537 .seq_show = cgroup_subtree_control_show, 4538 .write = cgroup_subtree_control_write, 4539 }, 4540 { 4541 .name = "cgroup.events", 4542 .flags = CFTYPE_NOT_ON_ROOT, 4543 .file_offset = offsetof(struct cgroup, events_file), 4544 .seq_show = cgroup_events_show, 4545 }, 4546 { 4547 .name = "cgroup.max.descendants", 4548 .seq_show = cgroup_max_descendants_show, 4549 .write = cgroup_max_descendants_write, 4550 }, 4551 { 4552 .name = "cgroup.max.depth", 4553 .seq_show = cgroup_max_depth_show, 4554 .write = cgroup_max_depth_write, 4555 }, 4556 { 4557 .name = "cgroup.stat", 4558 .seq_show = cgroup_stat_show, 4559 }, 4560 { 4561 .name = "cpu.stat", 4562 .flags = CFTYPE_NOT_ON_ROOT, 4563 .seq_show = cpu_stat_show, 4564 }, 4565 #ifdef CONFIG_PSI 4566 { 4567 .name = "io.pressure", 4568 .flags = CFTYPE_NOT_ON_ROOT, 4569 .seq_show = cgroup_io_pressure_show, 4570 }, 4571 { 4572 .name = "memory.pressure", 4573 .flags = CFTYPE_NOT_ON_ROOT, 4574 .seq_show = cgroup_memory_pressure_show, 4575 }, 4576 { 4577 .name = "cpu.pressure", 4578 .flags = CFTYPE_NOT_ON_ROOT, 4579 .seq_show = cgroup_cpu_pressure_show, 4580 }, 4581 #endif 4582 { } /* terminate */ 4583 }; 4584 4585 /* 4586 * css destruction is four-stage process. 4587 * 4588 * 1. Destruction starts. Killing of the percpu_ref is initiated. 4589 * Implemented in kill_css(). 4590 * 4591 * 2. When the percpu_ref is confirmed to be visible as killed on all CPUs 4592 * and thus css_tryget_online() is guaranteed to fail, the css can be 4593 * offlined by invoking offline_css(). After offlining, the base ref is 4594 * put. Implemented in css_killed_work_fn(). 4595 * 4596 * 3. When the percpu_ref reaches zero, the only possible remaining 4597 * accessors are inside RCU read sections. css_release() schedules the 4598 * RCU callback. 4599 * 4600 * 4. After the grace period, the css can be freed. Implemented in 4601 * css_free_work_fn(). 4602 * 4603 * It is actually hairier because both step 2 and 4 require process context 4604 * and thus involve punting to css->destroy_work adding two additional 4605 * steps to the already complex sequence. 4606 */ 4607 static void css_free_rwork_fn(struct work_struct *work) 4608 { 4609 struct cgroup_subsys_state *css = container_of(to_rcu_work(work), 4610 struct cgroup_subsys_state, destroy_rwork); 4611 struct cgroup_subsys *ss = css->ss; 4612 struct cgroup *cgrp = css->cgroup; 4613 4614 percpu_ref_exit(&css->refcnt); 4615 4616 if (ss) { 4617 /* css free path */ 4618 struct cgroup_subsys_state *parent = css->parent; 4619 int id = css->id; 4620 4621 ss->css_free(css); 4622 cgroup_idr_remove(&ss->css_idr, id); 4623 cgroup_put(cgrp); 4624 4625 if (parent) 4626 css_put(parent); 4627 } else { 4628 /* cgroup free path */ 4629 atomic_dec(&cgrp->root->nr_cgrps); 4630 cgroup1_pidlist_destroy_all(cgrp); 4631 cancel_work_sync(&cgrp->release_agent_work); 4632 4633 if (cgroup_parent(cgrp)) { 4634 /* 4635 * We get a ref to the parent, and put the ref when 4636 * this cgroup is being freed, so it's guaranteed 4637 * that the parent won't be destroyed before its 4638 * children. 4639 */ 4640 cgroup_put(cgroup_parent(cgrp)); 4641 kernfs_put(cgrp->kn); 4642 psi_cgroup_free(cgrp); 4643 if (cgroup_on_dfl(cgrp)) 4644 cgroup_rstat_exit(cgrp); 4645 kfree(cgrp); 4646 } else { 4647 /* 4648 * This is root cgroup's refcnt reaching zero, 4649 * which indicates that the root should be 4650 * released. 4651 */ 4652 cgroup_destroy_root(cgrp->root); 4653 } 4654 } 4655 } 4656 4657 static void css_release_work_fn(struct work_struct *work) 4658 { 4659 struct cgroup_subsys_state *css = 4660 container_of(work, struct cgroup_subsys_state, destroy_work); 4661 struct cgroup_subsys *ss = css->ss; 4662 struct cgroup *cgrp = css->cgroup; 4663 4664 mutex_lock(&cgroup_mutex); 4665 4666 css->flags |= CSS_RELEASED; 4667 list_del_rcu(&css->sibling); 4668 4669 if (ss) { 4670 /* css release path */ 4671 if (!list_empty(&css->rstat_css_node)) { 4672 cgroup_rstat_flush(cgrp); 4673 list_del_rcu(&css->rstat_css_node); 4674 } 4675 4676 cgroup_idr_replace(&ss->css_idr, NULL, css->id); 4677 if (ss->css_released) 4678 ss->css_released(css); 4679 } else { 4680 struct cgroup *tcgrp; 4681 4682 /* cgroup release path */ 4683 TRACE_CGROUP_PATH(release, cgrp); 4684 4685 if (cgroup_on_dfl(cgrp)) 4686 cgroup_rstat_flush(cgrp); 4687 4688 for (tcgrp = cgroup_parent(cgrp); tcgrp; 4689 tcgrp = cgroup_parent(tcgrp)) 4690 tcgrp->nr_dying_descendants--; 4691 4692 cgroup_idr_remove(&cgrp->root->cgroup_idr, cgrp->id); 4693 cgrp->id = -1; 4694 4695 /* 4696 * There are two control paths which try to determine 4697 * cgroup from dentry without going through kernfs - 4698 * cgroupstats_build() and css_tryget_online_from_dir(). 4699 * Those are supported by RCU protecting clearing of 4700 * cgrp->kn->priv backpointer. 4701 */ 4702 if (cgrp->kn) 4703 RCU_INIT_POINTER(*(void __rcu __force **)&cgrp->kn->priv, 4704 NULL); 4705 4706 cgroup_bpf_put(cgrp); 4707 } 4708 4709 mutex_unlock(&cgroup_mutex); 4710 4711 INIT_RCU_WORK(&css->destroy_rwork, css_free_rwork_fn); 4712 queue_rcu_work(cgroup_destroy_wq, &css->destroy_rwork); 4713 } 4714 4715 static void css_release(struct percpu_ref *ref) 4716 { 4717 struct cgroup_subsys_state *css = 4718 container_of(ref, struct cgroup_subsys_state, refcnt); 4719 4720 INIT_WORK(&css->destroy_work, css_release_work_fn); 4721 queue_work(cgroup_destroy_wq, &css->destroy_work); 4722 } 4723 4724 static void init_and_link_css(struct cgroup_subsys_state *css, 4725 struct cgroup_subsys *ss, struct cgroup *cgrp) 4726 { 4727 lockdep_assert_held(&cgroup_mutex); 4728 4729 cgroup_get_live(cgrp); 4730 4731 memset(css, 0, sizeof(*css)); 4732 css->cgroup = cgrp; 4733 css->ss = ss; 4734 css->id = -1; 4735 INIT_LIST_HEAD(&css->sibling); 4736 INIT_LIST_HEAD(&css->children); 4737 INIT_LIST_HEAD(&css->rstat_css_node); 4738 css->serial_nr = css_serial_nr_next++; 4739 atomic_set(&css->online_cnt, 0); 4740 4741 if (cgroup_parent(cgrp)) { 4742 css->parent = cgroup_css(cgroup_parent(cgrp), ss); 4743 css_get(css->parent); 4744 } 4745 4746 if (cgroup_on_dfl(cgrp) && ss->css_rstat_flush) 4747 list_add_rcu(&css->rstat_css_node, &cgrp->rstat_css_list); 4748 4749 BUG_ON(cgroup_css(cgrp, ss)); 4750 } 4751 4752 /* invoke ->css_online() on a new CSS and mark it online if successful */ 4753 static int online_css(struct cgroup_subsys_state *css) 4754 { 4755 struct cgroup_subsys *ss = css->ss; 4756 int ret = 0; 4757 4758 lockdep_assert_held(&cgroup_mutex); 4759 4760 if (ss->css_online) 4761 ret = ss->css_online(css); 4762 if (!ret) { 4763 css->flags |= CSS_ONLINE; 4764 rcu_assign_pointer(css->cgroup->subsys[ss->id], css); 4765 4766 atomic_inc(&css->online_cnt); 4767 if (css->parent) 4768 atomic_inc(&css->parent->online_cnt); 4769 } 4770 return ret; 4771 } 4772 4773 /* if the CSS is online, invoke ->css_offline() on it and mark it offline */ 4774 static void offline_css(struct cgroup_subsys_state *css) 4775 { 4776 struct cgroup_subsys *ss = css->ss; 4777 4778 lockdep_assert_held(&cgroup_mutex); 4779 4780 if (!(css->flags & CSS_ONLINE)) 4781 return; 4782 4783 if (ss->css_offline) 4784 ss->css_offline(css); 4785 4786 css->flags &= ~CSS_ONLINE; 4787 RCU_INIT_POINTER(css->cgroup->subsys[ss->id], NULL); 4788 4789 wake_up_all(&css->cgroup->offline_waitq); 4790 } 4791 4792 /** 4793 * css_create - create a cgroup_subsys_state 4794 * @cgrp: the cgroup new css will be associated with 4795 * @ss: the subsys of new css 4796 * 4797 * Create a new css associated with @cgrp - @ss pair. On success, the new 4798 * css is online and installed in @cgrp. This function doesn't create the 4799 * interface files. Returns 0 on success, -errno on failure. 4800 */ 4801 static struct cgroup_subsys_state *css_create(struct cgroup *cgrp, 4802 struct cgroup_subsys *ss) 4803 { 4804 struct cgroup *parent = cgroup_parent(cgrp); 4805 struct cgroup_subsys_state *parent_css = cgroup_css(parent, ss); 4806 struct cgroup_subsys_state *css; 4807 int err; 4808 4809 lockdep_assert_held(&cgroup_mutex); 4810 4811 css = ss->css_alloc(parent_css); 4812 if (!css) 4813 css = ERR_PTR(-ENOMEM); 4814 if (IS_ERR(css)) 4815 return css; 4816 4817 init_and_link_css(css, ss, cgrp); 4818 4819 err = percpu_ref_init(&css->refcnt, css_release, 0, GFP_KERNEL); 4820 if (err) 4821 goto err_free_css; 4822 4823 err = cgroup_idr_alloc(&ss->css_idr, NULL, 2, 0, GFP_KERNEL); 4824 if (err < 0) 4825 goto err_free_css; 4826 css->id = err; 4827 4828 /* @css is ready to be brought online now, make it visible */ 4829 list_add_tail_rcu(&css->sibling, &parent_css->children); 4830 cgroup_idr_replace(&ss->css_idr, css, css->id); 4831 4832 err = online_css(css); 4833 if (err) 4834 goto err_list_del; 4835 4836 if (ss->broken_hierarchy && !ss->warned_broken_hierarchy && 4837 cgroup_parent(parent)) { 4838 pr_warn("%s (%d) created nested cgroup for controller \"%s\" which has incomplete hierarchy support. Nested cgroups may change behavior in the future.\n", 4839 current->comm, current->pid, ss->name); 4840 if (!strcmp(ss->name, "memory")) 4841 pr_warn("\"memory\" requires setting use_hierarchy to 1 on the root\n"); 4842 ss->warned_broken_hierarchy = true; 4843 } 4844 4845 return css; 4846 4847 err_list_del: 4848 list_del_rcu(&css->sibling); 4849 err_free_css: 4850 list_del_rcu(&css->rstat_css_node); 4851 INIT_RCU_WORK(&css->destroy_rwork, css_free_rwork_fn); 4852 queue_rcu_work(cgroup_destroy_wq, &css->destroy_rwork); 4853 return ERR_PTR(err); 4854 } 4855 4856 /* 4857 * The returned cgroup is fully initialized including its control mask, but 4858 * it isn't associated with its kernfs_node and doesn't have the control 4859 * mask applied. 4860 */ 4861 static struct cgroup *cgroup_create(struct cgroup *parent) 4862 { 4863 struct cgroup_root *root = parent->root; 4864 struct cgroup *cgrp, *tcgrp; 4865 int level = parent->level + 1; 4866 int ret; 4867 4868 /* allocate the cgroup and its ID, 0 is reserved for the root */ 4869 cgrp = kzalloc(struct_size(cgrp, ancestor_ids, (level + 1)), 4870 GFP_KERNEL); 4871 if (!cgrp) 4872 return ERR_PTR(-ENOMEM); 4873 4874 ret = percpu_ref_init(&cgrp->self.refcnt, css_release, 0, GFP_KERNEL); 4875 if (ret) 4876 goto out_free_cgrp; 4877 4878 if (cgroup_on_dfl(parent)) { 4879 ret = cgroup_rstat_init(cgrp); 4880 if (ret) 4881 goto out_cancel_ref; 4882 } 4883 4884 /* 4885 * Temporarily set the pointer to NULL, so idr_find() won't return 4886 * a half-baked cgroup. 4887 */ 4888 cgrp->id = cgroup_idr_alloc(&root->cgroup_idr, NULL, 2, 0, GFP_KERNEL); 4889 if (cgrp->id < 0) { 4890 ret = -ENOMEM; 4891 goto out_stat_exit; 4892 } 4893 4894 init_cgroup_housekeeping(cgrp); 4895 4896 cgrp->self.parent = &parent->self; 4897 cgrp->root = root; 4898 cgrp->level = level; 4899 4900 ret = psi_cgroup_alloc(cgrp); 4901 if (ret) 4902 goto out_idr_free; 4903 4904 ret = cgroup_bpf_inherit(cgrp); 4905 if (ret) 4906 goto out_psi_free; 4907 4908 for (tcgrp = cgrp; tcgrp; tcgrp = cgroup_parent(tcgrp)) { 4909 cgrp->ancestor_ids[tcgrp->level] = tcgrp->id; 4910 4911 if (tcgrp != cgrp) 4912 tcgrp->nr_descendants++; 4913 } 4914 4915 if (notify_on_release(parent)) 4916 set_bit(CGRP_NOTIFY_ON_RELEASE, &cgrp->flags); 4917 4918 if (test_bit(CGRP_CPUSET_CLONE_CHILDREN, &parent->flags)) 4919 set_bit(CGRP_CPUSET_CLONE_CHILDREN, &cgrp->flags); 4920 4921 cgrp->self.serial_nr = css_serial_nr_next++; 4922 4923 /* allocation complete, commit to creation */ 4924 list_add_tail_rcu(&cgrp->self.sibling, &cgroup_parent(cgrp)->self.children); 4925 atomic_inc(&root->nr_cgrps); 4926 cgroup_get_live(parent); 4927 4928 /* 4929 * @cgrp is now fully operational. If something fails after this 4930 * point, it'll be released via the normal destruction path. 4931 */ 4932 cgroup_idr_replace(&root->cgroup_idr, cgrp, cgrp->id); 4933 4934 /* 4935 * On the default hierarchy, a child doesn't automatically inherit 4936 * subtree_control from the parent. Each is configured manually. 4937 */ 4938 if (!cgroup_on_dfl(cgrp)) 4939 cgrp->subtree_control = cgroup_control(cgrp); 4940 4941 cgroup_propagate_control(cgrp); 4942 4943 return cgrp; 4944 4945 out_psi_free: 4946 psi_cgroup_free(cgrp); 4947 out_idr_free: 4948 cgroup_idr_remove(&root->cgroup_idr, cgrp->id); 4949 out_stat_exit: 4950 if (cgroup_on_dfl(parent)) 4951 cgroup_rstat_exit(cgrp); 4952 out_cancel_ref: 4953 percpu_ref_exit(&cgrp->self.refcnt); 4954 out_free_cgrp: 4955 kfree(cgrp); 4956 return ERR_PTR(ret); 4957 } 4958 4959 static bool cgroup_check_hierarchy_limits(struct cgroup *parent) 4960 { 4961 struct cgroup *cgroup; 4962 int ret = false; 4963 int level = 1; 4964 4965 lockdep_assert_held(&cgroup_mutex); 4966 4967 for (cgroup = parent; cgroup; cgroup = cgroup_parent(cgroup)) { 4968 if (cgroup->nr_descendants >= cgroup->max_descendants) 4969 goto fail; 4970 4971 if (level > cgroup->max_depth) 4972 goto fail; 4973 4974 level++; 4975 } 4976 4977 ret = true; 4978 fail: 4979 return ret; 4980 } 4981 4982 int cgroup_mkdir(struct kernfs_node *parent_kn, const char *name, umode_t mode) 4983 { 4984 struct cgroup *parent, *cgrp; 4985 struct kernfs_node *kn; 4986 int ret; 4987 4988 /* do not accept '\n' to prevent making /proc/<pid>/cgroup unparsable */ 4989 if (strchr(name, '\n')) 4990 return -EINVAL; 4991 4992 parent = cgroup_kn_lock_live(parent_kn, false); 4993 if (!parent) 4994 return -ENODEV; 4995 4996 if (!cgroup_check_hierarchy_limits(parent)) { 4997 ret = -EAGAIN; 4998 goto out_unlock; 4999 } 5000 5001 cgrp = cgroup_create(parent); 5002 if (IS_ERR(cgrp)) { 5003 ret = PTR_ERR(cgrp); 5004 goto out_unlock; 5005 } 5006 5007 /* create the directory */ 5008 kn = kernfs_create_dir(parent->kn, name, mode, cgrp); 5009 if (IS_ERR(kn)) { 5010 ret = PTR_ERR(kn); 5011 goto out_destroy; 5012 } 5013 cgrp->kn = kn; 5014 5015 /* 5016 * This extra ref will be put in cgroup_free_fn() and guarantees 5017 * that @cgrp->kn is always accessible. 5018 */ 5019 kernfs_get(kn); 5020 5021 ret = cgroup_kn_set_ugid(kn); 5022 if (ret) 5023 goto out_destroy; 5024 5025 ret = css_populate_dir(&cgrp->self); 5026 if (ret) 5027 goto out_destroy; 5028 5029 ret = cgroup_apply_control_enable(cgrp); 5030 if (ret) 5031 goto out_destroy; 5032 5033 TRACE_CGROUP_PATH(mkdir, cgrp); 5034 5035 /* let's create and online css's */ 5036 kernfs_activate(kn); 5037 5038 ret = 0; 5039 goto out_unlock; 5040 5041 out_destroy: 5042 cgroup_destroy_locked(cgrp); 5043 out_unlock: 5044 cgroup_kn_unlock(parent_kn); 5045 return ret; 5046 } 5047 5048 /* 5049 * This is called when the refcnt of a css is confirmed to be killed. 5050 * css_tryget_online() is now guaranteed to fail. Tell the subsystem to 5051 * initate destruction and put the css ref from kill_css(). 5052 */ 5053 static void css_killed_work_fn(struct work_struct *work) 5054 { 5055 struct cgroup_subsys_state *css = 5056 container_of(work, struct cgroup_subsys_state, destroy_work); 5057 5058 mutex_lock(&cgroup_mutex); 5059 5060 do { 5061 offline_css(css); 5062 css_put(css); 5063 /* @css can't go away while we're holding cgroup_mutex */ 5064 css = css->parent; 5065 } while (css && atomic_dec_and_test(&css->online_cnt)); 5066 5067 mutex_unlock(&cgroup_mutex); 5068 } 5069 5070 /* css kill confirmation processing requires process context, bounce */ 5071 static void css_killed_ref_fn(struct percpu_ref *ref) 5072 { 5073 struct cgroup_subsys_state *css = 5074 container_of(ref, struct cgroup_subsys_state, refcnt); 5075 5076 if (atomic_dec_and_test(&css->online_cnt)) { 5077 INIT_WORK(&css->destroy_work, css_killed_work_fn); 5078 queue_work(cgroup_destroy_wq, &css->destroy_work); 5079 } 5080 } 5081 5082 /** 5083 * kill_css - destroy a css 5084 * @css: css to destroy 5085 * 5086 * This function initiates destruction of @css by removing cgroup interface 5087 * files and putting its base reference. ->css_offline() will be invoked 5088 * asynchronously once css_tryget_online() is guaranteed to fail and when 5089 * the reference count reaches zero, @css will be released. 5090 */ 5091 static void kill_css(struct cgroup_subsys_state *css) 5092 { 5093 lockdep_assert_held(&cgroup_mutex); 5094 5095 if (css->flags & CSS_DYING) 5096 return; 5097 5098 css->flags |= CSS_DYING; 5099 5100 /* 5101 * This must happen before css is disassociated with its cgroup. 5102 * See seq_css() for details. 5103 */ 5104 css_clear_dir(css); 5105 5106 /* 5107 * Killing would put the base ref, but we need to keep it alive 5108 * until after ->css_offline(). 5109 */ 5110 css_get(css); 5111 5112 /* 5113 * cgroup core guarantees that, by the time ->css_offline() is 5114 * invoked, no new css reference will be given out via 5115 * css_tryget_online(). We can't simply call percpu_ref_kill() and 5116 * proceed to offlining css's because percpu_ref_kill() doesn't 5117 * guarantee that the ref is seen as killed on all CPUs on return. 5118 * 5119 * Use percpu_ref_kill_and_confirm() to get notifications as each 5120 * css is confirmed to be seen as killed on all CPUs. 5121 */ 5122 percpu_ref_kill_and_confirm(&css->refcnt, css_killed_ref_fn); 5123 } 5124 5125 /** 5126 * cgroup_destroy_locked - the first stage of cgroup destruction 5127 * @cgrp: cgroup to be destroyed 5128 * 5129 * css's make use of percpu refcnts whose killing latency shouldn't be 5130 * exposed to userland and are RCU protected. Also, cgroup core needs to 5131 * guarantee that css_tryget_online() won't succeed by the time 5132 * ->css_offline() is invoked. To satisfy all the requirements, 5133 * destruction is implemented in the following two steps. 5134 * 5135 * s1. Verify @cgrp can be destroyed and mark it dying. Remove all 5136 * userland visible parts and start killing the percpu refcnts of 5137 * css's. Set up so that the next stage will be kicked off once all 5138 * the percpu refcnts are confirmed to be killed. 5139 * 5140 * s2. Invoke ->css_offline(), mark the cgroup dead and proceed with the 5141 * rest of destruction. Once all cgroup references are gone, the 5142 * cgroup is RCU-freed. 5143 * 5144 * This function implements s1. After this step, @cgrp is gone as far as 5145 * the userland is concerned and a new cgroup with the same name may be 5146 * created. As cgroup doesn't care about the names internally, this 5147 * doesn't cause any problem. 5148 */ 5149 static int cgroup_destroy_locked(struct cgroup *cgrp) 5150 __releases(&cgroup_mutex) __acquires(&cgroup_mutex) 5151 { 5152 struct cgroup *tcgrp, *parent = cgroup_parent(cgrp); 5153 struct cgroup_subsys_state *css; 5154 struct cgrp_cset_link *link; 5155 int ssid; 5156 5157 lockdep_assert_held(&cgroup_mutex); 5158 5159 /* 5160 * Only migration can raise populated from zero and we're already 5161 * holding cgroup_mutex. 5162 */ 5163 if (cgroup_is_populated(cgrp)) 5164 return -EBUSY; 5165 5166 /* 5167 * Make sure there's no live children. We can't test emptiness of 5168 * ->self.children as dead children linger on it while being 5169 * drained; otherwise, "rmdir parent/child parent" may fail. 5170 */ 5171 if (css_has_online_children(&cgrp->self)) 5172 return -EBUSY; 5173 5174 /* 5175 * Mark @cgrp and the associated csets dead. The former prevents 5176 * further task migration and child creation by disabling 5177 * cgroup_lock_live_group(). The latter makes the csets ignored by 5178 * the migration path. 5179 */ 5180 cgrp->self.flags &= ~CSS_ONLINE; 5181 5182 spin_lock_irq(&css_set_lock); 5183 list_for_each_entry(link, &cgrp->cset_links, cset_link) 5184 link->cset->dead = true; 5185 spin_unlock_irq(&css_set_lock); 5186 5187 /* initiate massacre of all css's */ 5188 for_each_css(css, ssid, cgrp) 5189 kill_css(css); 5190 5191 /* clear and remove @cgrp dir, @cgrp has an extra ref on its kn */ 5192 css_clear_dir(&cgrp->self); 5193 kernfs_remove(cgrp->kn); 5194 5195 if (parent && cgroup_is_threaded(cgrp)) 5196 parent->nr_threaded_children--; 5197 5198 for (tcgrp = cgroup_parent(cgrp); tcgrp; tcgrp = cgroup_parent(tcgrp)) { 5199 tcgrp->nr_descendants--; 5200 tcgrp->nr_dying_descendants++; 5201 } 5202 5203 cgroup1_check_for_release(parent); 5204 5205 /* put the base reference */ 5206 percpu_ref_kill(&cgrp->self.refcnt); 5207 5208 return 0; 5209 }; 5210 5211 int cgroup_rmdir(struct kernfs_node *kn) 5212 { 5213 struct cgroup *cgrp; 5214 int ret = 0; 5215 5216 cgrp = cgroup_kn_lock_live(kn, false); 5217 if (!cgrp) 5218 return 0; 5219 5220 ret = cgroup_destroy_locked(cgrp); 5221 if (!ret) 5222 TRACE_CGROUP_PATH(rmdir, cgrp); 5223 5224 cgroup_kn_unlock(kn); 5225 return ret; 5226 } 5227 5228 static struct kernfs_syscall_ops cgroup_kf_syscall_ops = { 5229 .show_options = cgroup_show_options, 5230 .remount_fs = cgroup_remount, 5231 .mkdir = cgroup_mkdir, 5232 .rmdir = cgroup_rmdir, 5233 .show_path = cgroup_show_path, 5234 }; 5235 5236 static void __init cgroup_init_subsys(struct cgroup_subsys *ss, bool early) 5237 { 5238 struct cgroup_subsys_state *css; 5239 5240 pr_debug("Initializing cgroup subsys %s\n", ss->name); 5241 5242 mutex_lock(&cgroup_mutex); 5243 5244 idr_init(&ss->css_idr); 5245 INIT_LIST_HEAD(&ss->cfts); 5246 5247 /* Create the root cgroup state for this subsystem */ 5248 ss->root = &cgrp_dfl_root; 5249 css = ss->css_alloc(cgroup_css(&cgrp_dfl_root.cgrp, ss)); 5250 /* We don't handle early failures gracefully */ 5251 BUG_ON(IS_ERR(css)); 5252 init_and_link_css(css, ss, &cgrp_dfl_root.cgrp); 5253 5254 /* 5255 * Root csses are never destroyed and we can't initialize 5256 * percpu_ref during early init. Disable refcnting. 5257 */ 5258 css->flags |= CSS_NO_REF; 5259 5260 if (early) { 5261 /* allocation can't be done safely during early init */ 5262 css->id = 1; 5263 } else { 5264 css->id = cgroup_idr_alloc(&ss->css_idr, css, 1, 2, GFP_KERNEL); 5265 BUG_ON(css->id < 0); 5266 } 5267 5268 /* Update the init_css_set to contain a subsys 5269 * pointer to this state - since the subsystem is 5270 * newly registered, all tasks and hence the 5271 * init_css_set is in the subsystem's root cgroup. */ 5272 init_css_set.subsys[ss->id] = css; 5273 5274 have_fork_callback |= (bool)ss->fork << ss->id; 5275 have_exit_callback |= (bool)ss->exit << ss->id; 5276 have_free_callback |= (bool)ss->free << ss->id; 5277 have_canfork_callback |= (bool)ss->can_fork << ss->id; 5278 5279 /* At system boot, before all subsystems have been 5280 * registered, no tasks have been forked, so we don't 5281 * need to invoke fork callbacks here. */ 5282 BUG_ON(!list_empty(&init_task.tasks)); 5283 5284 BUG_ON(online_css(css)); 5285 5286 mutex_unlock(&cgroup_mutex); 5287 } 5288 5289 /** 5290 * cgroup_init_early - cgroup initialization at system boot 5291 * 5292 * Initialize cgroups at system boot, and initialize any 5293 * subsystems that request early init. 5294 */ 5295 int __init cgroup_init_early(void) 5296 { 5297 static struct cgroup_sb_opts __initdata opts; 5298 struct cgroup_subsys *ss; 5299 int i; 5300 5301 init_cgroup_root(&cgrp_dfl_root, &opts); 5302 cgrp_dfl_root.cgrp.self.flags |= CSS_NO_REF; 5303 5304 RCU_INIT_POINTER(init_task.cgroups, &init_css_set); 5305 5306 for_each_subsys(ss, i) { 5307 WARN(!ss->css_alloc || !ss->css_free || ss->name || ss->id, 5308 "invalid cgroup_subsys %d:%s css_alloc=%p css_free=%p id:name=%d:%s\n", 5309 i, cgroup_subsys_name[i], ss->css_alloc, ss->css_free, 5310 ss->id, ss->name); 5311 WARN(strlen(cgroup_subsys_name[i]) > MAX_CGROUP_TYPE_NAMELEN, 5312 "cgroup_subsys_name %s too long\n", cgroup_subsys_name[i]); 5313 5314 ss->id = i; 5315 ss->name = cgroup_subsys_name[i]; 5316 if (!ss->legacy_name) 5317 ss->legacy_name = cgroup_subsys_name[i]; 5318 5319 if (ss->early_init) 5320 cgroup_init_subsys(ss, true); 5321 } 5322 return 0; 5323 } 5324 5325 static u16 cgroup_disable_mask __initdata; 5326 5327 /** 5328 * cgroup_init - cgroup initialization 5329 * 5330 * Register cgroup filesystem and /proc file, and initialize 5331 * any subsystems that didn't request early init. 5332 */ 5333 int __init cgroup_init(void) 5334 { 5335 struct cgroup_subsys *ss; 5336 int ssid; 5337 5338 BUILD_BUG_ON(CGROUP_SUBSYS_COUNT > 16); 5339 BUG_ON(percpu_init_rwsem(&cgroup_threadgroup_rwsem)); 5340 BUG_ON(cgroup_init_cftypes(NULL, cgroup_base_files)); 5341 BUG_ON(cgroup_init_cftypes(NULL, cgroup1_base_files)); 5342 5343 cgroup_rstat_boot(); 5344 5345 /* 5346 * The latency of the synchronize_sched() is too high for cgroups, 5347 * avoid it at the cost of forcing all readers into the slow path. 5348 */ 5349 rcu_sync_enter_start(&cgroup_threadgroup_rwsem.rss); 5350 5351 get_user_ns(init_cgroup_ns.user_ns); 5352 5353 mutex_lock(&cgroup_mutex); 5354 5355 /* 5356 * Add init_css_set to the hash table so that dfl_root can link to 5357 * it during init. 5358 */ 5359 hash_add(css_set_table, &init_css_set.hlist, 5360 css_set_hash(init_css_set.subsys)); 5361 5362 BUG_ON(cgroup_setup_root(&cgrp_dfl_root, 0, 0)); 5363 5364 mutex_unlock(&cgroup_mutex); 5365 5366 for_each_subsys(ss, ssid) { 5367 if (ss->early_init) { 5368 struct cgroup_subsys_state *css = 5369 init_css_set.subsys[ss->id]; 5370 5371 css->id = cgroup_idr_alloc(&ss->css_idr, css, 1, 2, 5372 GFP_KERNEL); 5373 BUG_ON(css->id < 0); 5374 } else { 5375 cgroup_init_subsys(ss, false); 5376 } 5377 5378 list_add_tail(&init_css_set.e_cset_node[ssid], 5379 &cgrp_dfl_root.cgrp.e_csets[ssid]); 5380 5381 /* 5382 * Setting dfl_root subsys_mask needs to consider the 5383 * disabled flag and cftype registration needs kmalloc, 5384 * both of which aren't available during early_init. 5385 */ 5386 if (cgroup_disable_mask & (1 << ssid)) { 5387 static_branch_disable(cgroup_subsys_enabled_key[ssid]); 5388 printk(KERN_INFO "Disabling %s control group subsystem\n", 5389 ss->name); 5390 continue; 5391 } 5392 5393 if (cgroup1_ssid_disabled(ssid)) 5394 printk(KERN_INFO "Disabling %s control group subsystem in v1 mounts\n", 5395 ss->name); 5396 5397 cgrp_dfl_root.subsys_mask |= 1 << ss->id; 5398 5399 /* implicit controllers must be threaded too */ 5400 WARN_ON(ss->implicit_on_dfl && !ss->threaded); 5401 5402 if (ss->implicit_on_dfl) 5403 cgrp_dfl_implicit_ss_mask |= 1 << ss->id; 5404 else if (!ss->dfl_cftypes) 5405 cgrp_dfl_inhibit_ss_mask |= 1 << ss->id; 5406 5407 if (ss->threaded) 5408 cgrp_dfl_threaded_ss_mask |= 1 << ss->id; 5409 5410 if (ss->dfl_cftypes == ss->legacy_cftypes) { 5411 WARN_ON(cgroup_add_cftypes(ss, ss->dfl_cftypes)); 5412 } else { 5413 WARN_ON(cgroup_add_dfl_cftypes(ss, ss->dfl_cftypes)); 5414 WARN_ON(cgroup_add_legacy_cftypes(ss, ss->legacy_cftypes)); 5415 } 5416 5417 if (ss->bind) 5418 ss->bind(init_css_set.subsys[ssid]); 5419 5420 mutex_lock(&cgroup_mutex); 5421 css_populate_dir(init_css_set.subsys[ssid]); 5422 mutex_unlock(&cgroup_mutex); 5423 } 5424 5425 /* init_css_set.subsys[] has been updated, re-hash */ 5426 hash_del(&init_css_set.hlist); 5427 hash_add(css_set_table, &init_css_set.hlist, 5428 css_set_hash(init_css_set.subsys)); 5429 5430 WARN_ON(sysfs_create_mount_point(fs_kobj, "cgroup")); 5431 WARN_ON(register_filesystem(&cgroup_fs_type)); 5432 WARN_ON(register_filesystem(&cgroup2_fs_type)); 5433 WARN_ON(!proc_create_single("cgroups", 0, NULL, proc_cgroupstats_show)); 5434 5435 return 0; 5436 } 5437 5438 static int __init cgroup_wq_init(void) 5439 { 5440 /* 5441 * There isn't much point in executing destruction path in 5442 * parallel. Good chunk is serialized with cgroup_mutex anyway. 5443 * Use 1 for @max_active. 5444 * 5445 * We would prefer to do this in cgroup_init() above, but that 5446 * is called before init_workqueues(): so leave this until after. 5447 */ 5448 cgroup_destroy_wq = alloc_workqueue("cgroup_destroy", 0, 1); 5449 BUG_ON(!cgroup_destroy_wq); 5450 return 0; 5451 } 5452 core_initcall(cgroup_wq_init); 5453 5454 void cgroup_path_from_kernfs_id(const union kernfs_node_id *id, 5455 char *buf, size_t buflen) 5456 { 5457 struct kernfs_node *kn; 5458 5459 kn = kernfs_get_node_by_id(cgrp_dfl_root.kf_root, id); 5460 if (!kn) 5461 return; 5462 kernfs_path(kn, buf, buflen); 5463 kernfs_put(kn); 5464 } 5465 5466 /* 5467 * proc_cgroup_show() 5468 * - Print task's cgroup paths into seq_file, one line for each hierarchy 5469 * - Used for /proc/<pid>/cgroup. 5470 */ 5471 int proc_cgroup_show(struct seq_file *m, struct pid_namespace *ns, 5472 struct pid *pid, struct task_struct *tsk) 5473 { 5474 char *buf; 5475 int retval; 5476 struct cgroup_root *root; 5477 5478 retval = -ENOMEM; 5479 buf = kmalloc(PATH_MAX, GFP_KERNEL); 5480 if (!buf) 5481 goto out; 5482 5483 mutex_lock(&cgroup_mutex); 5484 spin_lock_irq(&css_set_lock); 5485 5486 for_each_root(root) { 5487 struct cgroup_subsys *ss; 5488 struct cgroup *cgrp; 5489 int ssid, count = 0; 5490 5491 if (root == &cgrp_dfl_root && !cgrp_dfl_visible) 5492 continue; 5493 5494 seq_printf(m, "%d:", root->hierarchy_id); 5495 if (root != &cgrp_dfl_root) 5496 for_each_subsys(ss, ssid) 5497 if (root->subsys_mask & (1 << ssid)) 5498 seq_printf(m, "%s%s", count++ ? "," : "", 5499 ss->legacy_name); 5500 if (strlen(root->name)) 5501 seq_printf(m, "%sname=%s", count ? "," : "", 5502 root->name); 5503 seq_putc(m, ':'); 5504 5505 cgrp = task_cgroup_from_root(tsk, root); 5506 5507 /* 5508 * On traditional hierarchies, all zombie tasks show up as 5509 * belonging to the root cgroup. On the default hierarchy, 5510 * while a zombie doesn't show up in "cgroup.procs" and 5511 * thus can't be migrated, its /proc/PID/cgroup keeps 5512 * reporting the cgroup it belonged to before exiting. If 5513 * the cgroup is removed before the zombie is reaped, 5514 * " (deleted)" is appended to the cgroup path. 5515 */ 5516 if (cgroup_on_dfl(cgrp) || !(tsk->flags & PF_EXITING)) { 5517 retval = cgroup_path_ns_locked(cgrp, buf, PATH_MAX, 5518 current->nsproxy->cgroup_ns); 5519 if (retval >= PATH_MAX) 5520 retval = -ENAMETOOLONG; 5521 if (retval < 0) 5522 goto out_unlock; 5523 5524 seq_puts(m, buf); 5525 } else { 5526 seq_puts(m, "/"); 5527 } 5528 5529 if (cgroup_on_dfl(cgrp) && cgroup_is_dead(cgrp)) 5530 seq_puts(m, " (deleted)\n"); 5531 else 5532 seq_putc(m, '\n'); 5533 } 5534 5535 retval = 0; 5536 out_unlock: 5537 spin_unlock_irq(&css_set_lock); 5538 mutex_unlock(&cgroup_mutex); 5539 kfree(buf); 5540 out: 5541 return retval; 5542 } 5543 5544 /** 5545 * cgroup_fork - initialize cgroup related fields during copy_process() 5546 * @child: pointer to task_struct of forking parent process. 5547 * 5548 * A task is associated with the init_css_set until cgroup_post_fork() 5549 * attaches it to the parent's css_set. Empty cg_list indicates that 5550 * @child isn't holding reference to its css_set. 5551 */ 5552 void cgroup_fork(struct task_struct *child) 5553 { 5554 RCU_INIT_POINTER(child->cgroups, &init_css_set); 5555 INIT_LIST_HEAD(&child->cg_list); 5556 } 5557 5558 /** 5559 * cgroup_can_fork - called on a new task before the process is exposed 5560 * @child: the task in question. 5561 * 5562 * This calls the subsystem can_fork() callbacks. If the can_fork() callback 5563 * returns an error, the fork aborts with that error code. This allows for 5564 * a cgroup subsystem to conditionally allow or deny new forks. 5565 */ 5566 int cgroup_can_fork(struct task_struct *child) 5567 { 5568 struct cgroup_subsys *ss; 5569 int i, j, ret; 5570 5571 do_each_subsys_mask(ss, i, have_canfork_callback) { 5572 ret = ss->can_fork(child); 5573 if (ret) 5574 goto out_revert; 5575 } while_each_subsys_mask(); 5576 5577 return 0; 5578 5579 out_revert: 5580 for_each_subsys(ss, j) { 5581 if (j >= i) 5582 break; 5583 if (ss->cancel_fork) 5584 ss->cancel_fork(child); 5585 } 5586 5587 return ret; 5588 } 5589 5590 /** 5591 * cgroup_cancel_fork - called if a fork failed after cgroup_can_fork() 5592 * @child: the task in question 5593 * 5594 * This calls the cancel_fork() callbacks if a fork failed *after* 5595 * cgroup_can_fork() succeded. 5596 */ 5597 void cgroup_cancel_fork(struct task_struct *child) 5598 { 5599 struct cgroup_subsys *ss; 5600 int i; 5601 5602 for_each_subsys(ss, i) 5603 if (ss->cancel_fork) 5604 ss->cancel_fork(child); 5605 } 5606 5607 /** 5608 * cgroup_post_fork - called on a new task after adding it to the task list 5609 * @child: the task in question 5610 * 5611 * Adds the task to the list running through its css_set if necessary and 5612 * call the subsystem fork() callbacks. Has to be after the task is 5613 * visible on the task list in case we race with the first call to 5614 * cgroup_task_iter_start() - to guarantee that the new task ends up on its 5615 * list. 5616 */ 5617 void cgroup_post_fork(struct task_struct *child) 5618 { 5619 struct cgroup_subsys *ss; 5620 int i; 5621 5622 /* 5623 * This may race against cgroup_enable_task_cg_lists(). As that 5624 * function sets use_task_css_set_links before grabbing 5625 * tasklist_lock and we just went through tasklist_lock to add 5626 * @child, it's guaranteed that either we see the set 5627 * use_task_css_set_links or cgroup_enable_task_cg_lists() sees 5628 * @child during its iteration. 5629 * 5630 * If we won the race, @child is associated with %current's 5631 * css_set. Grabbing css_set_lock guarantees both that the 5632 * association is stable, and, on completion of the parent's 5633 * migration, @child is visible in the source of migration or 5634 * already in the destination cgroup. This guarantee is necessary 5635 * when implementing operations which need to migrate all tasks of 5636 * a cgroup to another. 5637 * 5638 * Note that if we lose to cgroup_enable_task_cg_lists(), @child 5639 * will remain in init_css_set. This is safe because all tasks are 5640 * in the init_css_set before cg_links is enabled and there's no 5641 * operation which transfers all tasks out of init_css_set. 5642 */ 5643 if (use_task_css_set_links) { 5644 struct css_set *cset; 5645 5646 spin_lock_irq(&css_set_lock); 5647 cset = task_css_set(current); 5648 if (list_empty(&child->cg_list)) { 5649 get_css_set(cset); 5650 cset->nr_tasks++; 5651 css_set_move_task(child, NULL, cset, false); 5652 } 5653 spin_unlock_irq(&css_set_lock); 5654 } 5655 5656 /* 5657 * Call ss->fork(). This must happen after @child is linked on 5658 * css_set; otherwise, @child might change state between ->fork() 5659 * and addition to css_set. 5660 */ 5661 do_each_subsys_mask(ss, i, have_fork_callback) { 5662 ss->fork(child); 5663 } while_each_subsys_mask(); 5664 } 5665 5666 /** 5667 * cgroup_exit - detach cgroup from exiting task 5668 * @tsk: pointer to task_struct of exiting process 5669 * 5670 * Description: Detach cgroup from @tsk and release it. 5671 * 5672 * Note that cgroups marked notify_on_release force every task in 5673 * them to take the global cgroup_mutex mutex when exiting. 5674 * This could impact scaling on very large systems. Be reluctant to 5675 * use notify_on_release cgroups where very high task exit scaling 5676 * is required on large systems. 5677 * 5678 * We set the exiting tasks cgroup to the root cgroup (top_cgroup). We 5679 * call cgroup_exit() while the task is still competent to handle 5680 * notify_on_release(), then leave the task attached to the root cgroup in 5681 * each hierarchy for the remainder of its exit. No need to bother with 5682 * init_css_set refcnting. init_css_set never goes away and we can't race 5683 * with migration path - PF_EXITING is visible to migration path. 5684 */ 5685 void cgroup_exit(struct task_struct *tsk) 5686 { 5687 struct cgroup_subsys *ss; 5688 struct css_set *cset; 5689 int i; 5690 5691 /* 5692 * Unlink from @tsk from its css_set. As migration path can't race 5693 * with us, we can check css_set and cg_list without synchronization. 5694 */ 5695 cset = task_css_set(tsk); 5696 5697 if (!list_empty(&tsk->cg_list)) { 5698 spin_lock_irq(&css_set_lock); 5699 css_set_move_task(tsk, cset, NULL, false); 5700 cset->nr_tasks--; 5701 spin_unlock_irq(&css_set_lock); 5702 } else { 5703 get_css_set(cset); 5704 } 5705 5706 /* see cgroup_post_fork() for details */ 5707 do_each_subsys_mask(ss, i, have_exit_callback) { 5708 ss->exit(tsk); 5709 } while_each_subsys_mask(); 5710 } 5711 5712 void cgroup_free(struct task_struct *task) 5713 { 5714 struct css_set *cset = task_css_set(task); 5715 struct cgroup_subsys *ss; 5716 int ssid; 5717 5718 do_each_subsys_mask(ss, ssid, have_free_callback) { 5719 ss->free(task); 5720 } while_each_subsys_mask(); 5721 5722 put_css_set(cset); 5723 } 5724 5725 static int __init cgroup_disable(char *str) 5726 { 5727 struct cgroup_subsys *ss; 5728 char *token; 5729 int i; 5730 5731 while ((token = strsep(&str, ",")) != NULL) { 5732 if (!*token) 5733 continue; 5734 5735 for_each_subsys(ss, i) { 5736 if (strcmp(token, ss->name) && 5737 strcmp(token, ss->legacy_name)) 5738 continue; 5739 cgroup_disable_mask |= 1 << i; 5740 } 5741 } 5742 return 1; 5743 } 5744 __setup("cgroup_disable=", cgroup_disable); 5745 5746 /** 5747 * css_tryget_online_from_dir - get corresponding css from a cgroup dentry 5748 * @dentry: directory dentry of interest 5749 * @ss: subsystem of interest 5750 * 5751 * If @dentry is a directory for a cgroup which has @ss enabled on it, try 5752 * to get the corresponding css and return it. If such css doesn't exist 5753 * or can't be pinned, an ERR_PTR value is returned. 5754 */ 5755 struct cgroup_subsys_state *css_tryget_online_from_dir(struct dentry *dentry, 5756 struct cgroup_subsys *ss) 5757 { 5758 struct kernfs_node *kn = kernfs_node_from_dentry(dentry); 5759 struct file_system_type *s_type = dentry->d_sb->s_type; 5760 struct cgroup_subsys_state *css = NULL; 5761 struct cgroup *cgrp; 5762 5763 /* is @dentry a cgroup dir? */ 5764 if ((s_type != &cgroup_fs_type && s_type != &cgroup2_fs_type) || 5765 !kn || kernfs_type(kn) != KERNFS_DIR) 5766 return ERR_PTR(-EBADF); 5767 5768 rcu_read_lock(); 5769 5770 /* 5771 * This path doesn't originate from kernfs and @kn could already 5772 * have been or be removed at any point. @kn->priv is RCU 5773 * protected for this access. See css_release_work_fn() for details. 5774 */ 5775 cgrp = rcu_dereference(*(void __rcu __force **)&kn->priv); 5776 if (cgrp) 5777 css = cgroup_css(cgrp, ss); 5778 5779 if (!css || !css_tryget_online(css)) 5780 css = ERR_PTR(-ENOENT); 5781 5782 rcu_read_unlock(); 5783 return css; 5784 } 5785 5786 /** 5787 * css_from_id - lookup css by id 5788 * @id: the cgroup id 5789 * @ss: cgroup subsys to be looked into 5790 * 5791 * Returns the css if there's valid one with @id, otherwise returns NULL. 5792 * Should be called under rcu_read_lock(). 5793 */ 5794 struct cgroup_subsys_state *css_from_id(int id, struct cgroup_subsys *ss) 5795 { 5796 WARN_ON_ONCE(!rcu_read_lock_held()); 5797 return idr_find(&ss->css_idr, id); 5798 } 5799 5800 /** 5801 * cgroup_get_from_path - lookup and get a cgroup from its default hierarchy path 5802 * @path: path on the default hierarchy 5803 * 5804 * Find the cgroup at @path on the default hierarchy, increment its 5805 * reference count and return it. Returns pointer to the found cgroup on 5806 * success, ERR_PTR(-ENOENT) if @path doens't exist and ERR_PTR(-ENOTDIR) 5807 * if @path points to a non-directory. 5808 */ 5809 struct cgroup *cgroup_get_from_path(const char *path) 5810 { 5811 struct kernfs_node *kn; 5812 struct cgroup *cgrp; 5813 5814 mutex_lock(&cgroup_mutex); 5815 5816 kn = kernfs_walk_and_get(cgrp_dfl_root.cgrp.kn, path); 5817 if (kn) { 5818 if (kernfs_type(kn) == KERNFS_DIR) { 5819 cgrp = kn->priv; 5820 cgroup_get_live(cgrp); 5821 } else { 5822 cgrp = ERR_PTR(-ENOTDIR); 5823 } 5824 kernfs_put(kn); 5825 } else { 5826 cgrp = ERR_PTR(-ENOENT); 5827 } 5828 5829 mutex_unlock(&cgroup_mutex); 5830 return cgrp; 5831 } 5832 EXPORT_SYMBOL_GPL(cgroup_get_from_path); 5833 5834 /** 5835 * cgroup_get_from_fd - get a cgroup pointer from a fd 5836 * @fd: fd obtained by open(cgroup2_dir) 5837 * 5838 * Find the cgroup from a fd which should be obtained 5839 * by opening a cgroup directory. Returns a pointer to the 5840 * cgroup on success. ERR_PTR is returned if the cgroup 5841 * cannot be found. 5842 */ 5843 struct cgroup *cgroup_get_from_fd(int fd) 5844 { 5845 struct cgroup_subsys_state *css; 5846 struct cgroup *cgrp; 5847 struct file *f; 5848 5849 f = fget_raw(fd); 5850 if (!f) 5851 return ERR_PTR(-EBADF); 5852 5853 css = css_tryget_online_from_dir(f->f_path.dentry, NULL); 5854 fput(f); 5855 if (IS_ERR(css)) 5856 return ERR_CAST(css); 5857 5858 cgrp = css->cgroup; 5859 if (!cgroup_on_dfl(cgrp)) { 5860 cgroup_put(cgrp); 5861 return ERR_PTR(-EBADF); 5862 } 5863 5864 return cgrp; 5865 } 5866 EXPORT_SYMBOL_GPL(cgroup_get_from_fd); 5867 5868 /* 5869 * sock->sk_cgrp_data handling. For more info, see sock_cgroup_data 5870 * definition in cgroup-defs.h. 5871 */ 5872 #ifdef CONFIG_SOCK_CGROUP_DATA 5873 5874 #if defined(CONFIG_CGROUP_NET_PRIO) || defined(CONFIG_CGROUP_NET_CLASSID) 5875 5876 DEFINE_SPINLOCK(cgroup_sk_update_lock); 5877 static bool cgroup_sk_alloc_disabled __read_mostly; 5878 5879 void cgroup_sk_alloc_disable(void) 5880 { 5881 if (cgroup_sk_alloc_disabled) 5882 return; 5883 pr_info("cgroup: disabling cgroup2 socket matching due to net_prio or net_cls activation\n"); 5884 cgroup_sk_alloc_disabled = true; 5885 } 5886 5887 #else 5888 5889 #define cgroup_sk_alloc_disabled false 5890 5891 #endif 5892 5893 void cgroup_sk_alloc(struct sock_cgroup_data *skcd) 5894 { 5895 if (cgroup_sk_alloc_disabled) 5896 return; 5897 5898 /* Socket clone path */ 5899 if (skcd->val) { 5900 /* 5901 * We might be cloning a socket which is left in an empty 5902 * cgroup and the cgroup might have already been rmdir'd. 5903 * Don't use cgroup_get_live(). 5904 */ 5905 cgroup_get(sock_cgroup_ptr(skcd)); 5906 return; 5907 } 5908 5909 rcu_read_lock(); 5910 5911 while (true) { 5912 struct css_set *cset; 5913 5914 cset = task_css_set(current); 5915 if (likely(cgroup_tryget(cset->dfl_cgrp))) { 5916 skcd->val = (unsigned long)cset->dfl_cgrp; 5917 break; 5918 } 5919 cpu_relax(); 5920 } 5921 5922 rcu_read_unlock(); 5923 } 5924 5925 void cgroup_sk_free(struct sock_cgroup_data *skcd) 5926 { 5927 cgroup_put(sock_cgroup_ptr(skcd)); 5928 } 5929 5930 #endif /* CONFIG_SOCK_CGROUP_DATA */ 5931 5932 #ifdef CONFIG_CGROUP_BPF 5933 int cgroup_bpf_attach(struct cgroup *cgrp, struct bpf_prog *prog, 5934 enum bpf_attach_type type, u32 flags) 5935 { 5936 int ret; 5937 5938 mutex_lock(&cgroup_mutex); 5939 ret = __cgroup_bpf_attach(cgrp, prog, type, flags); 5940 mutex_unlock(&cgroup_mutex); 5941 return ret; 5942 } 5943 int cgroup_bpf_detach(struct cgroup *cgrp, struct bpf_prog *prog, 5944 enum bpf_attach_type type, u32 flags) 5945 { 5946 int ret; 5947 5948 mutex_lock(&cgroup_mutex); 5949 ret = __cgroup_bpf_detach(cgrp, prog, type, flags); 5950 mutex_unlock(&cgroup_mutex); 5951 return ret; 5952 } 5953 int cgroup_bpf_query(struct cgroup *cgrp, const union bpf_attr *attr, 5954 union bpf_attr __user *uattr) 5955 { 5956 int ret; 5957 5958 mutex_lock(&cgroup_mutex); 5959 ret = __cgroup_bpf_query(cgrp, attr, uattr); 5960 mutex_unlock(&cgroup_mutex); 5961 return ret; 5962 } 5963 #endif /* CONFIG_CGROUP_BPF */ 5964 5965 #ifdef CONFIG_SYSFS 5966 static ssize_t show_delegatable_files(struct cftype *files, char *buf, 5967 ssize_t size, const char *prefix) 5968 { 5969 struct cftype *cft; 5970 ssize_t ret = 0; 5971 5972 for (cft = files; cft && cft->name[0] != '\0'; cft++) { 5973 if (!(cft->flags & CFTYPE_NS_DELEGATABLE)) 5974 continue; 5975 5976 if (prefix) 5977 ret += snprintf(buf + ret, size - ret, "%s.", prefix); 5978 5979 ret += snprintf(buf + ret, size - ret, "%s\n", cft->name); 5980 5981 if (unlikely(ret >= size)) { 5982 WARN_ON(1); 5983 break; 5984 } 5985 } 5986 5987 return ret; 5988 } 5989 5990 static ssize_t delegate_show(struct kobject *kobj, struct kobj_attribute *attr, 5991 char *buf) 5992 { 5993 struct cgroup_subsys *ss; 5994 int ssid; 5995 ssize_t ret = 0; 5996 5997 ret = show_delegatable_files(cgroup_base_files, buf, PAGE_SIZE - ret, 5998 NULL); 5999 6000 for_each_subsys(ss, ssid) 6001 ret += show_delegatable_files(ss->dfl_cftypes, buf + ret, 6002 PAGE_SIZE - ret, 6003 cgroup_subsys_name[ssid]); 6004 6005 return ret; 6006 } 6007 static struct kobj_attribute cgroup_delegate_attr = __ATTR_RO(delegate); 6008 6009 static ssize_t features_show(struct kobject *kobj, struct kobj_attribute *attr, 6010 char *buf) 6011 { 6012 return snprintf(buf, PAGE_SIZE, "nsdelegate\n"); 6013 } 6014 static struct kobj_attribute cgroup_features_attr = __ATTR_RO(features); 6015 6016 static struct attribute *cgroup_sysfs_attrs[] = { 6017 &cgroup_delegate_attr.attr, 6018 &cgroup_features_attr.attr, 6019 NULL, 6020 }; 6021 6022 static const struct attribute_group cgroup_sysfs_attr_group = { 6023 .attrs = cgroup_sysfs_attrs, 6024 .name = "cgroup", 6025 }; 6026 6027 static int __init cgroup_sysfs_init(void) 6028 { 6029 return sysfs_create_group(kernel_kobj, &cgroup_sysfs_attr_group); 6030 } 6031 subsys_initcall(cgroup_sysfs_init); 6032 #endif /* CONFIG_SYSFS */ 6033