Lines Matching +full:psi +full:- +full:l
2 * Generic process-grouping system.
12 * --------------------------------------------------
14 * Copyright (C) 2004-2006 Silicon Graphics, Inc.
17 * sysfs is Copyright (c) 2001-3 Patrick Mochel
19 * 2003-10-10 Written by Simon Derr.
20 * 2003-10-22 Updates by Stephen Hemminger.
21 * 2004 May-July Rework by Paul Jackson.
22 * ---------------------------------------------------
31 #include "cgroup-internal.h"
33 #include <linux/bpf-cgroup.h>
48 #include <linux/percpu-rwsem.h>
61 #include <linux/psi.h>
74 * that attempts to access what would be a 0-element array (i.e. sized
84 * css_set_lock protects task->cgroups pointer, the list of css_set
109 * Protects cgroup_file->kn for !self csses. It synchronizes notifications
110 * against file removal/re-creation across css hiding.
195 * Also, as csses are always appended to the parent's ->children list, it
259 * cgroup_ssid_enabled - cgroup subsys enabled test by subsys ID
275 * cgroup_on_dfl - test whether a cgroup is on the default hierarchy
285 * - Mount options "noprefix", "xattr", "clone_children", "release_agent"
288 * - When mounting an existing superblock, mount options should match.
290 * - rename(2) is disallowed.
292 * - "tasks" is removed. Everything should be at process granularity. Use
295 * - "cgroup.procs" is not sorted. pids will be unique unless they got
296 * recycled in-between reads.
298 * - "release_agent" and "notify_on_release" are removed. Replacement
301 * - "cgroup.clone_children" is removed.
303 * - "cgroup.subtree_populated" is available. Its value is 0 if the cgroup
308 * - cpuset: tasks will be kept in empty cpusets when hotplug happens and
309 * take masks of ancestors with non-empty cpus/mems, instead of being
312 * - cpuset: a task can be moved into an empty cpuset, and again it takes
315 * - blkcg: blk-throttle becomes properly hierarchical.
319 return cgrp->root == &cgrp_dfl_root; in cgroup_on_dfl()
355 return cgrp->nr_populated_csets; in cgroup_has_tasks()
360 return cgrp->dom_cgrp != cgrp; in cgroup_is_threaded()
368 * the no-internal-process constraint, so it can serve as a thread in cgroup_is_mixable()
386 if (cgrp->nr_populated_domain_children) in cgroup_can_be_thread_root()
390 if (cgrp->subtree_control & ~cgrp_dfl_threaded_ss_mask) in cgroup_can_be_thread_root()
404 if (cgrp->nr_threaded_children) in cgroup_is_thread_root()
412 (cgrp->subtree_control & cgrp_dfl_threaded_ss_mask)) in cgroup_is_thread_root()
440 u16 root_ss_mask = cgrp->root->subsys_mask; in cgroup_control()
443 u16 ss_mask = parent->subtree_control; in cgroup_control()
463 u16 ss_mask = parent->subtree_ss_mask; in cgroup_ss_mask()
471 return cgrp->root->subsys_mask; in cgroup_ss_mask()
475 * cgroup_css - obtain a cgroup's css for the specified subsystem
477 * @ss: the subsystem of interest (%NULL returns @cgrp->self)
489 return rcu_dereference_check(cgrp->subsys[ss->id], in cgroup_css()
492 return &cgrp->self; in cgroup_css()
496 * cgroup_e_css_by_mask - obtain a cgroup's effective css for the specified ss
498 * @ss: the subsystem of interest (%NULL returns @cgrp->self)
503 * function is guaranteed to return non-NULL css.
511 return &cgrp->self; in cgroup_e_css_by_mask()
517 while (!(cgroup_ss_mask(cgrp) & (1 << ss->id))) { in cgroup_e_css_by_mask()
527 * cgroup_e_css - obtain a cgroup's effective css for the specified subsystem
555 return init_css_set.subsys[ss->id]; in cgroup_e_css()
559 * cgroup_get_e_css - get a cgroup's effective css for the specified subsystem
587 css = init_css_set.subsys[ss->id]; in cgroup_get_e_css()
602 * __cgroup_task_count - count the number of tasks in a cgroup. The caller
613 list_for_each_entry(link, &cgrp->cset_links, cset_link) in __cgroup_task_count()
614 count += link->cset->nr_tasks; in __cgroup_task_count()
620 * cgroup_task_count - count the number of tasks in a cgroup.
636 struct cgroup *cgrp = of->kn->parent->priv; in of_css()
647 if (CGROUP_HAS_SUBSYS_CONFIG && cft->ss) in of_css()
648 return rcu_dereference_raw(cgrp->subsys[cft->ss->id]); in of_css()
650 return &cgrp->self; in of_css()
655 * for_each_css - iterate all css's of a cgroup
665 (cgrp)->subsys[(ssid)], \
670 * do_each_subsys_mask - filter for_each_subsys with a bitmask
675 * The block will only run for cases where the ssid-th bit (1 << ssid) of
695 list_for_each_entry((child), &(cgrp)->self.children, self.sibling) \
705 (dsct) = (d_css)->cgroup; \
714 (dsct) = (d_css)->cgroup; \
720 * The default css_set - used by init and its children prior to any
723 * reference-counted, to improve performance when child cgroups
740 * The following field is re-initialized when this cset gets linked
752 return cset->dom_cset != cset; in css_set_threaded()
756 * css_set_populated - does a css_set contain any tasks?
759 * css_set_populated() should be the same as !!cset->nr_tasks at steady
762 * properly updated. Hence, we can't just look at ->nr_tasks here.
768 return !list_empty(&cset->tasks) || !list_empty(&cset->mg_tasks); in css_set_populated()
772 * cgroup_update_populated - update the populated count of a cgroup
777 * task or losing the last. Update @cgrp->nr_populated_* accordingly. The
783 * @cgrp->nr_populated_csets and @cgrp->nr_populated_children are zero and
791 int adj = populated ? 1 : -1; in cgroup_update_populated()
799 cgrp->nr_populated_csets += adj; in cgroup_update_populated()
802 cgrp->nr_populated_threaded_children += adj; in cgroup_update_populated()
804 cgrp->nr_populated_domain_children += adj; in cgroup_update_populated()
813 cgroup_file_notify(&cgrp->events_file); in cgroup_update_populated()
821 * css_set_update_populated - update populated state of a css_set
834 list_for_each_entry(link, &cset->cgrp_links, cgrp_link) in css_set_update_populated()
835 cgroup_update_populated(link->cgrp, populated); in css_set_update_populated()
849 list_for_each_entry_safe(it, pos, &cset->task_iters, iters_node) in css_set_skip_task_iters()
854 * css_set_move_task - move a task from one css_set to another
858 * @use_mg_tasks: move to @to_cset->mg_tasks instead of ->tasks
878 WARN_ON_ONCE(list_empty(&task->cg_list)); in css_set_move_task()
881 list_del_init(&task->cg_list); in css_set_move_task()
885 WARN_ON_ONCE(!list_empty(&task->cg_list)); in css_set_move_task()
894 WARN_ON_ONCE(task->flags & PF_EXITING); in css_set_move_task()
897 list_add_tail(&task->cg_list, use_mg_tasks ? &to_cset->mg_tasks : in css_set_move_task()
898 &to_cset->tasks); in css_set_move_task()
931 if (!refcount_dec_and_test(&cset->refcount)) in put_css_set_locked()
934 WARN_ON_ONCE(!list_empty(&cset->threaded_csets)); in put_css_set_locked()
938 list_del(&cset->e_cset_node[ssid]); in put_css_set_locked()
939 css_put(cset->subsys[ssid]); in put_css_set_locked()
941 hash_del(&cset->hlist); in put_css_set_locked()
942 css_set_count--; in put_css_set_locked()
944 list_for_each_entry_safe(link, tmp_link, &cset->cgrp_links, cgrp_link) { in put_css_set_locked()
945 list_del(&link->cset_link); in put_css_set_locked()
946 list_del(&link->cgrp_link); in put_css_set_locked()
947 if (cgroup_parent(link->cgrp)) in put_css_set_locked()
948 cgroup_put(link->cgrp); in put_css_set_locked()
953 list_del(&cset->threaded_csets_node); in put_css_set_locked()
954 put_css_set_locked(cset->dom_cset); in put_css_set_locked()
961 * compare_css_sets - helper function for find_existing_css_set().
965 * @template: desired set of css pointers in css_set (pre-calculated)
983 if (memcmp(template, cset->subsys, sizeof(cset->subsys))) in compare_css_sets()
991 new_dfl_cgrp = old_cset->dfl_cgrp; in compare_css_sets()
993 if (new_dfl_cgrp->dom_cgrp != cset->dom_cset->dfl_cgrp) in compare_css_sets()
1002 l1 = &cset->cgrp_links; in compare_css_sets()
1003 l2 = &old_cset->cgrp_links; in compare_css_sets()
1008 l1 = l1->next; in compare_css_sets()
1009 l2 = l2->next; in compare_css_sets()
1010 /* See if we reached the end - both lists are equal length. */ in compare_css_sets()
1011 if (l1 == &cset->cgrp_links) { in compare_css_sets()
1012 BUG_ON(l2 != &old_cset->cgrp_links); in compare_css_sets()
1015 BUG_ON(l2 == &old_cset->cgrp_links); in compare_css_sets()
1020 cgrp1 = link1->cgrp; in compare_css_sets()
1021 cgrp2 = link2->cgrp; in compare_css_sets()
1023 BUG_ON(cgrp1->root != cgrp2->root); in compare_css_sets()
1032 if (cgrp1->root == new_cgrp->root) { in compare_css_sets()
1044 * find_existing_css_set - init css array and find the matching css_set
1053 struct cgroup_root *root = cgrp->root; in find_existing_css_set()
1065 if (root->subsys_mask & (1UL << i)) { in find_existing_css_set()
1076 template[i] = old_cset->subsys[i]; in find_existing_css_set()
1098 list_del(&link->cset_link); in free_cgrp_cset_links()
1104 * allocate_cgrp_cset_links - allocate cgrp_cset_links
1109 * through ->cset_link. Returns 0 on success or -errno.
1122 return -ENOMEM; in allocate_cgrp_cset_links()
1124 list_add(&link->cset_link, tmp_links); in allocate_cgrp_cset_links()
1130 * link_css_set - a helper function to link a css_set to a cgroup
1143 cset->dfl_cgrp = cgrp; in link_css_set()
1146 link->cset = cset; in link_css_set()
1147 link->cgrp = cgrp; in link_css_set()
1153 list_move_tail(&link->cset_link, &cgrp->cset_links); in link_css_set()
1154 list_add_tail(&link->cgrp_link, &cset->cgrp_links); in link_css_set()
1161 * find_css_set - return a new css_set with one cgroup updated
1202 refcount_set(&cset->refcount, 1); in find_css_set()
1203 cset->dom_cset = cset; in find_css_set()
1204 INIT_LIST_HEAD(&cset->tasks); in find_css_set()
1205 INIT_LIST_HEAD(&cset->mg_tasks); in find_css_set()
1206 INIT_LIST_HEAD(&cset->dying_tasks); in find_css_set()
1207 INIT_LIST_HEAD(&cset->task_iters); in find_css_set()
1208 INIT_LIST_HEAD(&cset->threaded_csets); in find_css_set()
1209 INIT_HLIST_NODE(&cset->hlist); in find_css_set()
1210 INIT_LIST_HEAD(&cset->cgrp_links); in find_css_set()
1211 INIT_LIST_HEAD(&cset->mg_src_preload_node); in find_css_set()
1212 INIT_LIST_HEAD(&cset->mg_dst_preload_node); in find_css_set()
1213 INIT_LIST_HEAD(&cset->mg_node); in find_css_set()
1217 memcpy(cset->subsys, template, sizeof(cset->subsys)); in find_css_set()
1221 list_for_each_entry(link, &old_cset->cgrp_links, cgrp_link) { in find_css_set()
1222 struct cgroup *c = link->cgrp; in find_css_set()
1224 if (c->root == cgrp->root) in find_css_set()
1234 key = css_set_hash(cset->subsys); in find_css_set()
1235 hash_add(css_set_table, &cset->hlist, key); in find_css_set()
1238 struct cgroup_subsys_state *css = cset->subsys[ssid]; in find_css_set()
1240 list_add_tail(&cset->e_cset_node[ssid], in find_css_set()
1241 &css->cgroup->e_csets[ssid]); in find_css_set()
1253 if (cgroup_is_threaded(cset->dfl_cgrp)) { in find_css_set()
1256 dcset = find_css_set(cset, cset->dfl_cgrp->dom_cgrp); in find_css_set()
1263 cset->dom_cset = dcset; in find_css_set()
1264 list_add_tail(&cset->threaded_csets_node, in find_css_set()
1265 &dcset->threaded_csets); in find_css_set()
1274 struct cgroup *root_cgrp = kernfs_root_to_node(kf_root)->priv; in cgroup_root_from_kf()
1276 return root_cgrp->root; in cgroup_root_from_kf()
1281 bool favoring = root->flags & CGRP_ROOT_FAVOR_DYNMODS; in cgroup_favor_dynmods()
1286 root->flags |= CGRP_ROOT_FAVOR_DYNMODS; in cgroup_favor_dynmods()
1289 root->flags &= ~CGRP_ROOT_FAVOR_DYNMODS; in cgroup_favor_dynmods()
1303 root->hierarchy_id = id; in cgroup_init_root_id()
1311 idr_remove(&cgroup_hierarchy_idr, root->hierarchy_id); in cgroup_exit_root_id()
1321 struct cgroup *cgrp = &root->cgrp; in cgroup_destroy_root()
1328 BUG_ON(atomic_read(&root->nr_cgrps)); in cgroup_destroy_root()
1329 BUG_ON(!list_empty(&cgrp->self.children)); in cgroup_destroy_root()
1332 WARN_ON(rebind_subsystems(&cgrp_dfl_root, root->subsys_mask)); in cgroup_destroy_root()
1340 list_for_each_entry_safe(link, tmp_link, &cgrp->cset_links, cset_link) { in cgroup_destroy_root()
1341 list_del(&link->cset_link); in cgroup_destroy_root()
1342 list_del(&link->cgrp_link); in cgroup_destroy_root()
1348 if (!list_empty(&root->root_list)) { in cgroup_destroy_root()
1349 list_del_rcu(&root->root_list); in cgroup_destroy_root()
1350 cgroup_root_count--; in cgroup_destroy_root()
1359 kernfs_destroy_root(root->kf_root); in cgroup_destroy_root()
1372 res_cgroup = &root->cgrp; in __cset_cgroup_from_root()
1374 res_cgroup = cset->dfl_cgrp; in __cset_cgroup_from_root()
1379 list_for_each_entry(link, &cset->cgrp_links, cgrp_link) { in __cset_cgroup_from_root()
1380 struct cgroup *c = link->cgrp; in __cset_cgroup_from_root()
1382 if (c->root == root) { in __cset_cgroup_from_root()
1415 cset = current->nsproxy->cgroup_ns->root_cset; in current_cgns_cgroup_from_root()
1428 * - Internal rcu_read_lock is unnecessary because we don't dereference any rcu
1430 * - css_set_lock is not needed because we just read cset->dfl_cgrp.
1431 * - As a bonus returned cgrp is pinned with the current because it cannot
1438 if (current->nsproxy) { in current_cgns_cgroup_dfl()
1439 cset = current->nsproxy->cgroup_ns->root_cset; in current_cgns_cgroup_dfl()
1471 * No need to lock the task - since we hold css_set_lock the in task_cgroup_from_root()
1508 struct cgroup_subsys *ss = cft->ss; in cgroup_file_name()
1510 if (cft->ss && !(cft->flags & CFTYPE_NO_PREFIX) && in cgroup_file_name()
1511 !(cgrp->root->flags & CGRP_ROOT_NOPREFIX)) { in cgroup_file_name()
1512 const char *dbg = (cft->flags & CFTYPE_DEBUG) ? ".__DEBUG__." : ""; in cgroup_file_name()
1515 dbg, cgroup_on_dfl(cgrp) ? ss->name : ss->legacy_name, in cgroup_file_name()
1516 cft->name); in cgroup_file_name()
1518 strscpy(buf, cft->name, CGROUP_FILE_NAME_MAX); in cgroup_file_name()
1524 * cgroup_file_mode - deduce file mode of a control file
1533 if (cft->read_u64 || cft->read_s64 || cft->seq_show) in cgroup_file_mode()
1536 if (cft->write_u64 || cft->write_s64 || cft->write) { in cgroup_file_mode()
1537 if (cft->flags & CFTYPE_WORLD_WRITABLE) in cgroup_file_mode()
1547 * cgroup_calc_subtree_ss_mask - calculate subtree_ss_mask
1552 * enabled together through its ->depends_on mask. In such cases, more
1572 new_ss_mask |= ss->depends_on; in cgroup_calc_subtree_ss_mask()
1577 * happen only if some depended-upon subsystems were bound in cgroup_calc_subtree_ss_mask()
1578 * to non-default hierarchies. in cgroup_calc_subtree_ss_mask()
1591 * cgroup_kn_unlock - unlocking helper for cgroup kernfs methods
1605 cgrp = kn->priv; in cgroup_kn_unlock()
1607 cgrp = kn->parent->priv; in cgroup_kn_unlock()
1616 * cgroup_kn_lock_live - locking helper for cgroup kernfs methods
1630 * including self-removal.
1637 cgrp = kn->priv; in cgroup_kn_lock_live()
1639 cgrp = kn->parent->priv; in cgroup_kn_lock_live()
1669 if (cft->file_offset) { in cgroup_rm_file()
1670 struct cgroup_subsys_state *css = cgroup_css(cgrp, cft->ss); in cgroup_rm_file()
1671 struct cgroup_file *cfile = (void *)css + cft->file_offset; in cgroup_rm_file()
1674 cfile->kn = NULL; in cgroup_rm_file()
1677 del_timer_sync(&cfile->notify_timer); in cgroup_rm_file()
1680 kernfs_remove_by_name(cgrp->kn, cgroup_file_name(cgrp, cft, name)); in cgroup_rm_file()
1684 * css_clear_dir - remove subsys files in a cgroup directory
1689 struct cgroup *cgrp = css->cgroup; in css_clear_dir()
1692 if (!(css->flags & CSS_VISIBLE)) in css_clear_dir()
1695 css->flags &= ~CSS_VISIBLE; in css_clear_dir()
1697 if (!css->ss) { in css_clear_dir()
1709 list_for_each_entry(cfts, &css->ss->cfts, node) in css_clear_dir()
1715 * css_populate_dir - create subsys files in a cgroup directory
1722 struct cgroup *cgrp = css->cgroup; in css_populate_dir()
1726 if (css->flags & CSS_VISIBLE) in css_populate_dir()
1729 if (!css->ss) { in css_populate_dir()
1747 list_for_each_entry(cfts, &css->ss->cfts, node) { in css_populate_dir()
1756 css->flags |= CSS_VISIBLE; in css_populate_dir()
1760 list_for_each_entry(cfts, &css->ss->cfts, node) { in css_populate_dir()
1770 struct cgroup *dcgrp = &dst_root->cgrp; in rebind_subsystems()
1779 * If @ss has non-root csses attached to it, can't move. in rebind_subsystems()
1783 if (css_next_child(NULL, cgroup_css(&ss->root->cgrp, ss)) && in rebind_subsystems()
1784 !ss->implicit_on_dfl) in rebind_subsystems()
1785 return -EBUSY; in rebind_subsystems()
1787 /* can't move between two non-dummy roots either */ in rebind_subsystems()
1788 if (ss->root != &cgrp_dfl_root && dst_root != &cgrp_dfl_root) in rebind_subsystems()
1789 return -EBUSY; in rebind_subsystems()
1795 if (ss->root == &cgrp_dfl_root) in rebind_subsystems()
1813 struct cgroup_root *src_root = ss->root; in rebind_subsystems()
1814 struct cgroup *scgrp = &src_root->cgrp; in rebind_subsystems()
1823 src_root->subsys_mask &= ~(1 << ssid); in rebind_subsystems()
1829 RCU_INIT_POINTER(scgrp->subsys[ssid], NULL); in rebind_subsystems()
1830 rcu_assign_pointer(dcgrp->subsys[ssid], css); in rebind_subsystems()
1831 ss->root = dst_root; in rebind_subsystems()
1834 css->cgroup = dcgrp; in rebind_subsystems()
1835 WARN_ON(!list_empty(&dcgrp->e_csets[ss->id])); in rebind_subsystems()
1836 list_for_each_entry_safe(cset, cset_pos, &scgrp->e_csets[ss->id], in rebind_subsystems()
1837 e_cset_node[ss->id]) { in rebind_subsystems()
1838 list_move_tail(&cset->e_cset_node[ss->id], in rebind_subsystems()
1839 &dcgrp->e_csets[ss->id]); in rebind_subsystems()
1842 * patch in-flight iterators to preserve correct iteration. in rebind_subsystems()
1844 * finished when it->cset_pos meets it->cset_head, so only in rebind_subsystems()
1845 * update it->cset_head is enough here. in rebind_subsystems()
1847 list_for_each_entry(it, &cset->task_iters, iters_node) in rebind_subsystems()
1848 if (it->cset_head == &scgrp->e_csets[ss->id]) in rebind_subsystems()
1849 it->cset_head = &dcgrp->e_csets[ss->id]; in rebind_subsystems()
1853 if (ss->css_rstat_flush) { in rebind_subsystems()
1854 list_del_rcu(&css->rstat_css_node); in rebind_subsystems()
1856 list_add_rcu(&css->rstat_css_node, in rebind_subsystems()
1857 &dcgrp->rstat_css_list); in rebind_subsystems()
1861 dst_root->subsys_mask |= 1 << ssid; in rebind_subsystems()
1865 dcgrp->subtree_control |= 1 << ssid; in rebind_subsystems()
1872 ss->name, ret); in rebind_subsystems()
1874 if (ss->bind) in rebind_subsystems()
1875 ss->bind(css); in rebind_subsystems()
1878 kernfs_activate(dcgrp->kn); in rebind_subsystems()
1892 return -ENOMEM; in cgroup_show_path()
1896 len = kernfs_path_from_node(kf_node, ns_cgroup->kn, buf, PATH_MAX); in cgroup_show_path()
1899 if (len == -E2BIG) in cgroup_show_path()
1900 len = -ERANGE; in cgroup_show_path()
1937 ctx->flags |= CGRP_ROOT_NS_DELEGATE; in cgroup2_parse_param()
1940 ctx->flags |= CGRP_ROOT_FAVOR_DYNMODS; in cgroup2_parse_param()
1943 ctx->flags |= CGRP_ROOT_MEMORY_LOCAL_EVENTS; in cgroup2_parse_param()
1946 ctx->flags |= CGRP_ROOT_MEMORY_RECURSIVE_PROT; in cgroup2_parse_param()
1949 return -EINVAL; in cgroup2_parse_param()
1954 if (current->nsproxy->cgroup_ns == &init_cgroup_ns) { in apply_cgroup_root_flags()
1992 apply_cgroup_root_flags(ctx->flags); in cgroup_reconfigure()
2001 INIT_LIST_HEAD(&cgrp->self.sibling); in init_cgroup_housekeeping()
2002 INIT_LIST_HEAD(&cgrp->self.children); in init_cgroup_housekeeping()
2003 INIT_LIST_HEAD(&cgrp->cset_links); in init_cgroup_housekeeping()
2004 INIT_LIST_HEAD(&cgrp->pidlists); in init_cgroup_housekeeping()
2005 mutex_init(&cgrp->pidlist_mutex); in init_cgroup_housekeeping()
2006 cgrp->self.cgroup = cgrp; in init_cgroup_housekeeping()
2007 cgrp->self.flags |= CSS_ONLINE; in init_cgroup_housekeeping()
2008 cgrp->dom_cgrp = cgrp; in init_cgroup_housekeeping()
2009 cgrp->max_descendants = INT_MAX; in init_cgroup_housekeeping()
2010 cgrp->max_depth = INT_MAX; in init_cgroup_housekeeping()
2011 INIT_LIST_HEAD(&cgrp->rstat_css_list); in init_cgroup_housekeeping()
2012 prev_cputime_init(&cgrp->prev_cputime); in init_cgroup_housekeeping()
2015 INIT_LIST_HEAD(&cgrp->e_csets[ssid]); in init_cgroup_housekeeping()
2017 init_waitqueue_head(&cgrp->offline_waitq); in init_cgroup_housekeeping()
2018 INIT_WORK(&cgrp->release_agent_work, cgroup1_release_agent); in init_cgroup_housekeeping()
2023 struct cgroup_root *root = ctx->root; in init_cgroup_root()
2024 struct cgroup *cgrp = &root->cgrp; in init_cgroup_root()
2026 INIT_LIST_HEAD_RCU(&root->root_list); in init_cgroup_root()
2027 atomic_set(&root->nr_cgrps, 1); in init_cgroup_root()
2028 cgrp->root = root; in init_cgroup_root()
2032 root->flags = ctx->flags & ~CGRP_ROOT_FAVOR_DYNMODS; in init_cgroup_root()
2033 if (ctx->release_agent) in init_cgroup_root()
2034 strscpy(root->release_agent_path, ctx->release_agent, PATH_MAX); in init_cgroup_root()
2035 if (ctx->name) in init_cgroup_root()
2036 strscpy(root->name, ctx->name, MAX_CGROUP_ROOT_NAMELEN); in init_cgroup_root()
2037 if (ctx->cpuset_clone_children) in init_cgroup_root()
2038 set_bit(CGRP_CPUSET_CLONE_CHILDREN, &root->cgrp.flags); in init_cgroup_root()
2044 struct cgroup *root_cgrp = &root->cgrp; in cgroup_setup_root()
2051 ret = percpu_ref_init(&root_cgrp->self.refcnt, css_release, in cgroup_setup_root()
2058 * but that's OK - it can only be increased by someone holding in cgroup_setup_root()
2074 root->kf_root = kernfs_create_root(kf_sops, in cgroup_setup_root()
2079 if (IS_ERR(root->kf_root)) { in cgroup_setup_root()
2080 ret = PTR_ERR(root->kf_root); in cgroup_setup_root()
2083 root_cgrp->kn = kernfs_root_to_node(root->kf_root); in cgroup_setup_root()
2085 root_cgrp->ancestors[0] = root_cgrp; in cgroup_setup_root()
2087 ret = css_populate_dir(&root_cgrp->self); in cgroup_setup_root()
2111 list_add_rcu(&root->root_list, &cgroup_roots); in cgroup_setup_root()
2126 BUG_ON(!list_empty(&root_cgrp->self.children)); in cgroup_setup_root()
2127 BUG_ON(atomic_read(&root->nr_cgrps) != 1); in cgroup_setup_root()
2135 kernfs_destroy_root(root->kf_root); in cgroup_setup_root()
2136 root->kf_root = NULL; in cgroup_setup_root()
2140 percpu_ref_exit(&root_cgrp->self.refcnt); in cgroup_setup_root()
2151 ctx->kfc.root = ctx->root->kf_root; in cgroup_do_get_tree()
2152 if (fc->fs_type == &cgroup2_fs_type) in cgroup_do_get_tree()
2153 ctx->kfc.magic = CGROUP2_SUPER_MAGIC; in cgroup_do_get_tree()
2155 ctx->kfc.magic = CGROUP_SUPER_MAGIC; in cgroup_do_get_tree()
2159 * In non-init cgroup namespace, instead of root cgroup's dentry, in cgroup_do_get_tree()
2160 * we return the dentry corresponding to the cgroupns->root_cgrp. in cgroup_do_get_tree()
2162 if (!ret && ctx->ns != &init_cgroup_ns) { in cgroup_do_get_tree()
2164 struct super_block *sb = fc->root->d_sb; in cgroup_do_get_tree()
2170 cgrp = cset_cgroup_from_root(ctx->ns->root_cset, ctx->root); in cgroup_do_get_tree()
2175 nsdentry = kernfs_node_dentry(cgrp->kn, sb); in cgroup_do_get_tree()
2176 dput(fc->root); in cgroup_do_get_tree()
2182 fc->root = nsdentry; in cgroup_do_get_tree()
2185 if (!ctx->kfc.new_sb_created) in cgroup_do_get_tree()
2186 cgroup_put(&ctx->root->cgrp); in cgroup_do_get_tree()
2198 kfree(ctx->name); in cgroup_fs_context_free()
2199 kfree(ctx->release_agent); in cgroup_fs_context_free()
2200 put_cgroup_ns(ctx->ns); in cgroup_fs_context_free()
2212 ctx->root = &cgrp_dfl_root; in cgroup_get_tree()
2216 apply_cgroup_root_flags(ctx->flags); in cgroup_get_tree()
2244 return -ENOMEM; in cgroup_init_fs_context()
2246 ctx->ns = current->nsproxy->cgroup_ns; in cgroup_init_fs_context()
2247 get_cgroup_ns(ctx->ns); in cgroup_init_fs_context()
2248 fc->fs_private = &ctx->kfc; in cgroup_init_fs_context()
2249 if (fc->fs_type == &cgroup2_fs_type) in cgroup_init_fs_context()
2250 fc->ops = &cgroup_fs_context_ops; in cgroup_init_fs_context()
2252 fc->ops = &cgroup1_fs_context_ops; in cgroup_init_fs_context()
2253 put_user_ns(fc->user_ns); in cgroup_init_fs_context()
2254 fc->user_ns = get_user_ns(ctx->ns->user_ns); in cgroup_init_fs_context()
2255 fc->global = true; in cgroup_init_fs_context()
2258 ctx->flags |= CGRP_ROOT_FAVOR_DYNMODS; in cgroup_init_fs_context()
2274 if (list_empty(&root->cgrp.self.children) && root != &cgrp_dfl_root && in cgroup_kill_sb()
2275 !percpu_ref_is_dying(&root->cgrp.self.refcnt)) in cgroup_kill_sb()
2276 percpu_ref_kill(&root->cgrp.self.refcnt); in cgroup_kill_sb()
2277 cgroup_put(&root->cgrp); in cgroup_kill_sb()
2320 fc->ops = &cpuset_fs_context_ops; in cpuset_init_fs_context()
2323 ctx->subsys_mask = 1 << cpuset_cgrp_id; in cpuset_init_fs_context()
2324 ctx->flags |= CGRP_ROOT_NOPREFIX; in cpuset_init_fs_context()
2325 ctx->release_agent = agent; in cpuset_init_fs_context()
2328 put_filesystem(fc->fs_type); in cpuset_init_fs_context()
2329 fc->fs_type = &cgroup_fs_type; in cpuset_init_fs_context()
2344 struct cgroup *root = cset_cgroup_from_root(ns->root_cset, cgrp->root); in cgroup_path_ns_locked()
2346 return kernfs_path_from_node(cgrp->kn, root->kn, buf, buflen); in cgroup_path_ns_locked()
2367 * cgroup_attach_lock - Lock for ->attach()
2371 * exits by write-locking cgroup_threadgroup_rwsem. However, some ->attach()
2373 * Unfortunately, letting ->attach() operations acquire cpus_read_lock() can
2377 * read-locking threadgroup_rwsem, so threadgroup_rwsem nests inside
2378 * cpus_read_lock(). If we call an ->attach() which acquires the cpus lock while
2379 * write-locking threadgroup_rwsem, the locking order is reversed and we end up
2380 * waiting for an on-going CPU hotplug operation which in turn is waiting for
2386 * write-locking cgroup_threadgroup_rwsem. This allows ->attach() to assume that
2397 * cgroup_attach_unlock - Undo cgroup_attach_lock()
2408 * cgroup_migrate_add_task - add a migration target task to a migration context
2412 * Add @task, which is a migration target, to @mgctx->tset. This function
2414 * should have been added as a migration source and @task->cg_list will be
2425 if (task->flags & PF_EXITING) in cgroup_migrate_add_task()
2429 WARN_ON_ONCE(list_empty(&task->cg_list)); in cgroup_migrate_add_task()
2432 if (!cset->mg_src_cgrp) in cgroup_migrate_add_task()
2435 mgctx->tset.nr_tasks++; in cgroup_migrate_add_task()
2437 list_move_tail(&task->cg_list, &cset->mg_tasks); in cgroup_migrate_add_task()
2438 if (list_empty(&cset->mg_node)) in cgroup_migrate_add_task()
2439 list_add_tail(&cset->mg_node, in cgroup_migrate_add_task()
2440 &mgctx->tset.src_csets); in cgroup_migrate_add_task()
2441 if (list_empty(&cset->mg_dst_cset->mg_node)) in cgroup_migrate_add_task()
2442 list_add_tail(&cset->mg_dst_cset->mg_node, in cgroup_migrate_add_task()
2443 &mgctx->tset.dst_csets); in cgroup_migrate_add_task()
2447 * cgroup_taskset_first - reset taskset and return the first task
2456 tset->cur_cset = list_first_entry(tset->csets, struct css_set, mg_node); in cgroup_taskset_first()
2457 tset->cur_task = NULL; in cgroup_taskset_first()
2463 * cgroup_taskset_next - iterate to the next task in taskset
2473 struct css_set *cset = tset->cur_cset; in cgroup_taskset_next()
2474 struct task_struct *task = tset->cur_task; in cgroup_taskset_next()
2476 while (CGROUP_HAS_SUBSYS_CONFIG && &cset->mg_node != tset->csets) { in cgroup_taskset_next()
2478 task = list_first_entry(&cset->mg_tasks, in cgroup_taskset_next()
2483 if (&task->cg_list != &cset->mg_tasks) { in cgroup_taskset_next()
2484 tset->cur_cset = cset; in cgroup_taskset_next()
2485 tset->cur_task = task; in cgroup_taskset_next()
2491 * has its ->mg_dst_cset set. in cgroup_taskset_next()
2493 if (cset->mg_dst_cset) in cgroup_taskset_next()
2494 *dst_cssp = cset->mg_dst_cset->subsys[tset->ssid]; in cgroup_taskset_next()
2496 *dst_cssp = cset->subsys[tset->ssid]; in cgroup_taskset_next()
2509 * cgroup_migrate_execute - migrate a taskset
2513 * This function fails iff one of the ->can_attach callbacks fails and
2519 struct cgroup_taskset *tset = &mgctx->tset; in cgroup_migrate_execute()
2526 if (tset->nr_tasks) { in cgroup_migrate_execute()
2527 do_each_subsys_mask(ss, ssid, mgctx->ss_mask) { in cgroup_migrate_execute()
2528 if (ss->can_attach) { in cgroup_migrate_execute()
2529 tset->ssid = ssid; in cgroup_migrate_execute()
2530 ret = ss->can_attach(tset); in cgroup_migrate_execute()
2545 list_for_each_entry(cset, &tset->src_csets, mg_node) { in cgroup_migrate_execute()
2546 list_for_each_entry_safe(task, tmp_task, &cset->mg_tasks, cg_list) { in cgroup_migrate_execute()
2548 struct css_set *to_cset = cset->mg_dst_cset; in cgroup_migrate_execute()
2551 to_cset->nr_tasks++; in cgroup_migrate_execute()
2553 from_cset->nr_tasks--; in cgroup_migrate_execute()
2558 cgroup_freezer_migrate_task(task, from_cset->dfl_cgrp, in cgroup_migrate_execute()
2559 to_cset->dfl_cgrp); in cgroup_migrate_execute()
2571 tset->csets = &tset->dst_csets; in cgroup_migrate_execute()
2573 if (tset->nr_tasks) { in cgroup_migrate_execute()
2574 do_each_subsys_mask(ss, ssid, mgctx->ss_mask) { in cgroup_migrate_execute()
2575 if (ss->attach) { in cgroup_migrate_execute()
2576 tset->ssid = ssid; in cgroup_migrate_execute()
2577 ss->attach(tset); in cgroup_migrate_execute()
2586 if (tset->nr_tasks) { in cgroup_migrate_execute()
2587 do_each_subsys_mask(ss, ssid, mgctx->ss_mask) { in cgroup_migrate_execute()
2590 if (ss->cancel_attach) { in cgroup_migrate_execute()
2591 tset->ssid = ssid; in cgroup_migrate_execute()
2592 ss->cancel_attach(tset); in cgroup_migrate_execute()
2598 list_splice_init(&tset->dst_csets, &tset->src_csets); in cgroup_migrate_execute()
2599 list_for_each_entry_safe(cset, tmp_cset, &tset->src_csets, mg_node) { in cgroup_migrate_execute()
2600 list_splice_tail_init(&cset->mg_tasks, &cset->tasks); in cgroup_migrate_execute()
2601 list_del_init(&cset->mg_node); in cgroup_migrate_execute()
2606 * Re-initialize the cgroup_taskset structure in case it is reused in cgroup_migrate_execute()
2610 tset->nr_tasks = 0; in cgroup_migrate_execute()
2611 tset->csets = &tset->src_csets; in cgroup_migrate_execute()
2616 * cgroup_migrate_vet_dst - verify whether a cgroup can be migration destination
2631 if (!cgroup_is_valid_domain(dst_cgrp->dom_cgrp)) in cgroup_migrate_vet_dst()
2632 return -EOPNOTSUPP; in cgroup_migrate_vet_dst()
2641 /* apply no-internal-process constraint */ in cgroup_migrate_vet_dst()
2642 if (dst_cgrp->subtree_control) in cgroup_migrate_vet_dst()
2643 return -EBUSY; in cgroup_migrate_vet_dst()
2649 * cgroup_migrate_finish - cleanup after attach
2663 list_for_each_entry_safe(cset, tmp_cset, &mgctx->preloaded_src_csets, in cgroup_migrate_finish()
2665 cset->mg_src_cgrp = NULL; in cgroup_migrate_finish()
2666 cset->mg_dst_cgrp = NULL; in cgroup_migrate_finish()
2667 cset->mg_dst_cset = NULL; in cgroup_migrate_finish()
2668 list_del_init(&cset->mg_src_preload_node); in cgroup_migrate_finish()
2672 list_for_each_entry_safe(cset, tmp_cset, &mgctx->preloaded_dst_csets, in cgroup_migrate_finish()
2674 cset->mg_src_cgrp = NULL; in cgroup_migrate_finish()
2675 cset->mg_dst_cgrp = NULL; in cgroup_migrate_finish()
2676 cset->mg_dst_cset = NULL; in cgroup_migrate_finish()
2677 list_del_init(&cset->mg_dst_preload_node); in cgroup_migrate_finish()
2685 * cgroup_migrate_add_src - add a migration source css_set
2691 * @src_cset and add it to @mgctx->src_csets, which should later be cleaned
2710 * If ->dead, @src_set is associated with one or more dead cgroups in cgroup_migrate_add_src()
2714 if (src_cset->dead) in cgroup_migrate_add_src()
2717 if (!list_empty(&src_cset->mg_src_preload_node)) in cgroup_migrate_add_src()
2720 src_cgrp = cset_cgroup_from_root(src_cset, dst_cgrp->root); in cgroup_migrate_add_src()
2722 WARN_ON(src_cset->mg_src_cgrp); in cgroup_migrate_add_src()
2723 WARN_ON(src_cset->mg_dst_cgrp); in cgroup_migrate_add_src()
2724 WARN_ON(!list_empty(&src_cset->mg_tasks)); in cgroup_migrate_add_src()
2725 WARN_ON(!list_empty(&src_cset->mg_node)); in cgroup_migrate_add_src()
2727 src_cset->mg_src_cgrp = src_cgrp; in cgroup_migrate_add_src()
2728 src_cset->mg_dst_cgrp = dst_cgrp; in cgroup_migrate_add_src()
2730 list_add_tail(&src_cset->mg_src_preload_node, &mgctx->preloaded_src_csets); in cgroup_migrate_add_src()
2734 * cgroup_migrate_prepare_dst - prepare destination css_sets for migration
2738 * preloaded to @mgctx->preloaded_src_csets. This function looks up and
2740 * to @mgctx->preloaded_dst_csets.
2754 list_for_each_entry_safe(src_cset, tmp_cset, &mgctx->preloaded_src_csets, in cgroup_migrate_prepare_dst()
2760 dst_cset = find_css_set(src_cset, src_cset->mg_dst_cgrp); in cgroup_migrate_prepare_dst()
2762 return -ENOMEM; in cgroup_migrate_prepare_dst()
2764 WARN_ON_ONCE(src_cset->mg_dst_cset || dst_cset->mg_dst_cset); in cgroup_migrate_prepare_dst()
2772 src_cset->mg_src_cgrp = NULL; in cgroup_migrate_prepare_dst()
2773 src_cset->mg_dst_cgrp = NULL; in cgroup_migrate_prepare_dst()
2774 list_del_init(&src_cset->mg_src_preload_node); in cgroup_migrate_prepare_dst()
2780 src_cset->mg_dst_cset = dst_cset; in cgroup_migrate_prepare_dst()
2782 if (list_empty(&dst_cset->mg_dst_preload_node)) in cgroup_migrate_prepare_dst()
2783 list_add_tail(&dst_cset->mg_dst_preload_node, in cgroup_migrate_prepare_dst()
2784 &mgctx->preloaded_dst_csets); in cgroup_migrate_prepare_dst()
2789 if (src_cset->subsys[ssid] != dst_cset->subsys[ssid]) in cgroup_migrate_prepare_dst()
2790 mgctx->ss_mask |= 1 << ssid; in cgroup_migrate_prepare_dst()
2797 * cgroup_migrate - migrate a process or task to a cgroup
2808 * As long as a controller's ->can_attach() doesn't fail, this function is
2809 * guaranteed to succeed. This means that, excluding ->can_attach()
2837 * cgroup_attach_task - attach a task or a whole threadgroup to a cgroup
2883 return ERR_PTR(-EINVAL); in cgroup_procs_write_start()
2901 tsk = ERR_PTR(-ESRCH); in cgroup_procs_write_start()
2909 tsk = tsk->group_leader; in cgroup_procs_write_start()
2913 * If userland migrates such a kthread to a non-root cgroup, it can in cgroup_procs_write_start()
2917 if (tsk->no_cgroup_migration || (tsk->flags & PF_NO_SETAFFINITY)) { in cgroup_procs_write_start()
2918 tsk = ERR_PTR(-EINVAL); in cgroup_procs_write_start()
2944 if (ss->post_attach) in cgroup_procs_write_finish()
2945 ss->post_attach(); in cgroup_procs_write_finish()
2957 seq_puts(seq, ss->name); in cgroup_print_ss_mask()
2967 struct cgroup *cgrp = seq_css(seq)->cgroup; in cgroup_controllers_show()
2976 struct cgroup *cgrp = seq_css(seq)->cgroup; in cgroup_subtree_control_show()
2978 cgroup_print_ss_mask(seq, cgrp->subtree_control); in cgroup_subtree_control_show()
2983 * cgroup_update_dfl_csses - update css assoc of a subtree in default hierarchy
3016 list_for_each_entry(link, &dsct->cset_links, cset_link) in cgroup_update_dfl_csses()
3017 cgroup_migrate_add_src(link->cset, dsct, &mgctx); in cgroup_update_dfl_csses()
3022 * We need to write-lock threadgroup_rwsem while migrating tasks. in cgroup_update_dfl_csses()
3025 * write-locking can be skipped safely. in cgroup_update_dfl_csses()
3041 list_for_each_entry_safe(task, ntask, &src_cset->tasks, cg_list) in cgroup_update_dfl_csses()
3054 * cgroup_lock_and_drain_offline - lock cgroup_mutex and drain offlined csses
3057 * Because css offlining is asynchronous, userland may try to re-enable a
3077 if (!css || !percpu_ref_is_dying(&css->refcnt)) in cgroup_lock_and_drain_offline()
3081 prepare_to_wait(&dsct->offline_waitq, &wait, in cgroup_lock_and_drain_offline()
3086 finish_wait(&dsct->offline_waitq, &wait); in cgroup_lock_and_drain_offline()
3095 * cgroup_save_control - save control masks and dom_cgrp of a subtree
3098 * Save ->subtree_control, ->subtree_ss_mask and ->dom_cgrp to the
3108 dsct->old_subtree_control = dsct->subtree_control; in cgroup_save_control()
3109 dsct->old_subtree_ss_mask = dsct->subtree_ss_mask; in cgroup_save_control()
3110 dsct->old_dom_cgrp = dsct->dom_cgrp; in cgroup_save_control()
3115 * cgroup_propagate_control - refresh control masks of a subtree
3118 * For @cgrp and its subtree, ensure ->subtree_ss_mask matches
3119 * ->subtree_control and propagate controller availability through the
3128 dsct->subtree_control &= cgroup_control(dsct); in cgroup_propagate_control()
3129 dsct->subtree_ss_mask = in cgroup_propagate_control()
3130 cgroup_calc_subtree_ss_mask(dsct->subtree_control, in cgroup_propagate_control()
3136 * cgroup_restore_control - restore control masks and dom_cgrp of a subtree
3139 * Restore ->subtree_control, ->subtree_ss_mask and ->dom_cgrp from the
3149 dsct->subtree_control = dsct->old_subtree_control; in cgroup_restore_control()
3150 dsct->subtree_ss_mask = dsct->old_subtree_ss_mask; in cgroup_restore_control()
3151 dsct->dom_cgrp = dsct->old_dom_cgrp; in cgroup_restore_control()
3157 struct cgroup_subsys *ss = css->ss; in css_visible()
3158 struct cgroup *cgrp = css->cgroup; in css_visible()
3160 if (cgroup_control(cgrp) & (1 << ss->id)) in css_visible()
3162 if (!(cgroup_ss_mask(cgrp) & (1 << ss->id))) in css_visible()
3164 return cgroup_on_dfl(cgrp) && ss->implicit_on_dfl; in css_visible()
3168 * cgroup_apply_control_enable - enable or show csses according to control
3176 * Returns 0 on success, -errno on failure. On failure, csses which have
3191 if (!(cgroup_ss_mask(dsct) & (1 << ss->id))) in cgroup_apply_control_enable()
3200 WARN_ON_ONCE(percpu_ref_is_dying(&css->refcnt)); in cgroup_apply_control_enable()
3214 * cgroup_apply_control_disable - kill or hide csses according to control
3223 * Controllers which may be depended upon should provide ->css_reset() for
3240 WARN_ON_ONCE(percpu_ref_is_dying(&css->refcnt)); in cgroup_apply_control_disable()
3242 if (css->parent && in cgroup_apply_control_disable()
3243 !(cgroup_ss_mask(dsct) & (1 << ss->id))) { in cgroup_apply_control_disable()
3247 if (ss->css_reset) in cgroup_apply_control_disable()
3248 ss->css_reset(css); in cgroup_apply_control_disable()
3255 * cgroup_apply_control - apply control mask updates to the subtree
3262 * 2. Update ->subtree_control masks in the subtree as desired.
3290 * cgroup_finalize_control - finalize control mask update
3315 if (!cgroup_is_valid_domain(cgrp->dom_cgrp)) in cgroup_vet_subtree_control_enable()
3316 return -EOPNOTSUPP; in cgroup_vet_subtree_control_enable()
3325 return -EOPNOTSUPP; in cgroup_vet_subtree_control_enable()
3341 return -EBUSY; in cgroup_vet_subtree_control_enable()
3358 * Parse input - space separated list of subsystem names prefixed in cgroup_subtree_control_write()
3359 * with either + or -. in cgroup_subtree_control_write()
3367 strcmp(tok + 1, ss->name)) in cgroup_subtree_control_write()
3373 } else if (*tok == '-') { in cgroup_subtree_control_write()
3377 return -EINVAL; in cgroup_subtree_control_write()
3382 return -EINVAL; in cgroup_subtree_control_write()
3385 cgrp = cgroup_kn_lock_live(of->kn, true); in cgroup_subtree_control_write()
3387 return -ENODEV; in cgroup_subtree_control_write()
3391 if (cgrp->subtree_control & (1 << ssid)) { in cgroup_subtree_control_write()
3397 ret = -ENOENT; in cgroup_subtree_control_write()
3401 if (!(cgrp->subtree_control & (1 << ssid))) { in cgroup_subtree_control_write()
3408 if (child->subtree_control & (1 << ssid)) { in cgroup_subtree_control_write()
3409 ret = -EBUSY; in cgroup_subtree_control_write()
3428 cgrp->subtree_control |= enable; in cgroup_subtree_control_write()
3429 cgrp->subtree_control &= ~disable; in cgroup_subtree_control_write()
3436 kernfs_activate(cgrp->kn); in cgroup_subtree_control_write()
3438 cgroup_kn_unlock(of->kn); in cgroup_subtree_control_write()
3443 * cgroup_enable_threaded - make @cgrp threaded
3454 struct cgroup *dom_cgrp = parent->dom_cgrp; in cgroup_enable_threaded()
3472 cgrp->subtree_control & ~cgrp_dfl_threaded_ss_mask) in cgroup_enable_threaded()
3473 return -EOPNOTSUPP; in cgroup_enable_threaded()
3478 return -EOPNOTSUPP; in cgroup_enable_threaded()
3488 dsct->dom_cgrp = dom_cgrp; in cgroup_enable_threaded()
3492 parent->nr_threaded_children++; in cgroup_enable_threaded()
3500 struct cgroup *cgrp = seq_css(seq)->cgroup; in cgroup_type_show()
3522 return -EINVAL; in cgroup_type_write()
3524 /* drain dying csses before we re-apply (threaded) subtree control */ in cgroup_type_write()
3525 cgrp = cgroup_kn_lock_live(of->kn, true); in cgroup_type_write()
3527 return -ENOENT; in cgroup_type_write()
3532 cgroup_kn_unlock(of->kn); in cgroup_type_write()
3538 struct cgroup *cgrp = seq_css(seq)->cgroup; in cgroup_max_descendants_show()
3539 int descendants = READ_ONCE(cgrp->max_descendants); in cgroup_max_descendants_show()
3566 return -ERANGE; in cgroup_max_descendants_write()
3568 cgrp = cgroup_kn_lock_live(of->kn, false); in cgroup_max_descendants_write()
3570 return -ENOENT; in cgroup_max_descendants_write()
3572 cgrp->max_descendants = descendants; in cgroup_max_descendants_write()
3574 cgroup_kn_unlock(of->kn); in cgroup_max_descendants_write()
3581 struct cgroup *cgrp = seq_css(seq)->cgroup; in cgroup_max_depth_show()
3582 int depth = READ_ONCE(cgrp->max_depth); in cgroup_max_depth_show()
3609 return -ERANGE; in cgroup_max_depth_write()
3611 cgrp = cgroup_kn_lock_live(of->kn, false); in cgroup_max_depth_write()
3613 return -ENOENT; in cgroup_max_depth_write()
3615 cgrp->max_depth = depth; in cgroup_max_depth_write()
3617 cgroup_kn_unlock(of->kn); in cgroup_max_depth_write()
3624 struct cgroup *cgrp = seq_css(seq)->cgroup; in cgroup_events_show()
3627 seq_printf(seq, "frozen %d\n", test_bit(CGRP_FROZEN, &cgrp->flags)); in cgroup_events_show()
3634 struct cgroup *cgroup = seq_css(seq)->cgroup; in cgroup_stat_show()
3637 cgroup->nr_descendants); in cgroup_stat_show()
3639 cgroup->nr_dying_descendants); in cgroup_stat_show()
3646 * cgroup_tryget_css - try to get a cgroup's css for the specified subsystem
3669 struct cgroup *cgrp = seq_css(seq)->cgroup; in cgroup_extra_stat_show()
3674 if (!ss->css_extra_stat_show) in cgroup_extra_stat_show()
3681 ret = ss->css_extra_stat_show(seq, css); in cgroup_extra_stat_show()
3693 if (!ss->css_local_stat_show) in cgroup_local_stat_show()
3700 ret = ss->css_local_stat_show(seq, css); in cgroup_local_stat_show()
3719 struct cgroup __maybe_unused *cgrp = seq_css(seq)->cgroup; in cpu_local_stat_show()
3731 struct cgroup *cgrp = seq_css(seq)->cgroup; in cgroup_io_pressure_show()
3732 struct psi_group *psi = cgroup_psi(cgrp); in cgroup_io_pressure_show() local
3734 return psi_show(seq, psi, PSI_IO); in cgroup_io_pressure_show()
3738 struct cgroup *cgrp = seq_css(seq)->cgroup; in cgroup_memory_pressure_show()
3739 struct psi_group *psi = cgroup_psi(cgrp); in cgroup_memory_pressure_show() local
3741 return psi_show(seq, psi, PSI_MEM); in cgroup_memory_pressure_show()
3745 struct cgroup *cgrp = seq_css(seq)->cgroup; in cgroup_cpu_pressure_show()
3746 struct psi_group *psi = cgroup_psi(cgrp); in cgroup_cpu_pressure_show() local
3748 return psi_show(seq, psi, PSI_CPU); in cgroup_cpu_pressure_show()
3754 struct cgroup_file_ctx *ctx = of->priv; in pressure_write()
3757 struct psi_group *psi; in pressure_write() local
3759 cgrp = cgroup_kn_lock_live(of->kn, false); in pressure_write()
3761 return -ENODEV; in pressure_write()
3764 cgroup_kn_unlock(of->kn); in pressure_write()
3767 if (ctx->psi.trigger) { in pressure_write()
3769 return -EBUSY; in pressure_write()
3772 psi = cgroup_psi(cgrp); in pressure_write()
3773 new = psi_trigger_create(psi, buf, res, of->file, of); in pressure_write()
3779 smp_store_release(&ctx->psi.trigger, new); in pressure_write()
3809 struct cgroup *cgrp = seq_css(seq)->cgroup; in cgroup_irq_pressure_show()
3810 struct psi_group *psi = cgroup_psi(cgrp); in cgroup_irq_pressure_show() local
3812 return psi_show(seq, psi, PSI_IRQ); in cgroup_irq_pressure_show()
3825 struct cgroup *cgrp = seq_css(seq)->cgroup; in cgroup_pressure_show()
3826 struct psi_group *psi = cgroup_psi(cgrp); in cgroup_pressure_show() local
3828 seq_printf(seq, "%d\n", psi->enabled); in cgroup_pressure_show()
3840 struct psi_group *psi; in cgroup_pressure_write() local
3847 return -ERANGE; in cgroup_pressure_write()
3849 cgrp = cgroup_kn_lock_live(of->kn, false); in cgroup_pressure_write()
3851 return -ENOENT; in cgroup_pressure_write()
3853 psi = cgroup_psi(cgrp); in cgroup_pressure_write()
3854 if (psi->enabled != enable) { in cgroup_pressure_write()
3859 cgroup_file_show(&cgrp->psi_files[i], enable); in cgroup_pressure_write()
3861 psi->enabled = enable; in cgroup_pressure_write()
3863 psi_cgroup_restart(psi); in cgroup_pressure_write()
3866 cgroup_kn_unlock(of->kn); in cgroup_pressure_write()
3874 struct cgroup_file_ctx *ctx = of->priv; in cgroup_pressure_poll()
3876 return psi_trigger_poll(&ctx->psi.trigger, of->file, pt); in cgroup_pressure_poll()
3881 struct cgroup_file_ctx *ctx = of->priv; in cgroup_pressure_release()
3883 psi_trigger_destroy(ctx->psi.trigger); in cgroup_pressure_release()
3904 struct cgroup *cgrp = seq_css(seq)->cgroup; in cgroup_freeze_show()
3906 seq_printf(seq, "%d\n", cgrp->freezer.freeze); in cgroup_freeze_show()
3923 return -ERANGE; in cgroup_freeze_write()
3925 cgrp = cgroup_kn_lock_live(of->kn, false); in cgroup_freeze_write()
3927 return -ENOENT; in cgroup_freeze_write()
3931 cgroup_kn_unlock(of->kn); in cgroup_freeze_write()
3944 set_bit(CGRP_KILL, &cgrp->flags); in __cgroup_kill()
3947 css_task_iter_start(&cgrp->self, CSS_TASK_ITER_PROCS | CSS_TASK_ITER_THREADED, &it); in __cgroup_kill()
3950 if (task->flags & PF_KTHREAD) in __cgroup_kill()
3962 clear_bit(CGRP_KILL, &cgrp->flags); in __cgroup_kill()
3989 return -ERANGE; in cgroup_kill_write()
3991 cgrp = cgroup_kn_lock_live(of->kn, false); in cgroup_kill_write()
3993 return -ENOENT; in cgroup_kill_write()
3996 * Killing is a process directed operation, i.e. the whole thread-group in cgroup_kill_write()
3998 * writable in non-threaded cgroups. in cgroup_kill_write()
4001 ret = -EOPNOTSUPP; in cgroup_kill_write()
4005 cgroup_kn_unlock(of->kn); in cgroup_kill_write()
4018 return -ENOMEM; in cgroup_file_open()
4020 ctx->ns = current->nsproxy->cgroup_ns; in cgroup_file_open()
4021 get_cgroup_ns(ctx->ns); in cgroup_file_open()
4022 of->priv = ctx; in cgroup_file_open()
4024 if (!cft->open) in cgroup_file_open()
4027 ret = cft->open(of); in cgroup_file_open()
4029 put_cgroup_ns(ctx->ns); in cgroup_file_open()
4038 struct cgroup_file_ctx *ctx = of->priv; in cgroup_file_release()
4040 if (cft->release) in cgroup_file_release()
4041 cft->release(of); in cgroup_file_release()
4042 put_cgroup_ns(ctx->ns); in cgroup_file_release()
4049 struct cgroup_file_ctx *ctx = of->priv; in cgroup_file_write()
4050 struct cgroup *cgrp = of->kn->parent->priv; in cgroup_file_write()
4060 * files in an non-init namespace root from inside the namespace in cgroup_file_write()
4061 * except for the files explicitly marked delegatable - in cgroup_file_write()
4064 if ((cgrp->root->flags & CGRP_ROOT_NS_DELEGATE) && in cgroup_file_write()
4065 !(cft->flags & CFTYPE_NS_DELEGATABLE) && in cgroup_file_write()
4066 ctx->ns != &init_cgroup_ns && ctx->ns->root_cset->dfl_cgrp == cgrp) in cgroup_file_write()
4067 return -EPERM; in cgroup_file_write()
4069 if (cft->write) in cgroup_file_write()
4070 return cft->write(of, buf, nbytes, off); in cgroup_file_write()
4079 css = cgroup_css(cgrp, cft->ss); in cgroup_file_write()
4082 if (cft->write_u64) { in cgroup_file_write()
4086 ret = cft->write_u64(css, cft, v); in cgroup_file_write()
4087 } else if (cft->write_s64) { in cgroup_file_write()
4091 ret = cft->write_s64(css, cft, v); in cgroup_file_write()
4093 ret = -EINVAL; in cgroup_file_write()
4103 if (cft->poll) in cgroup_file_poll()
4104 return cft->poll(of, pt); in cgroup_file_poll()
4111 return seq_cft(seq)->seq_start(seq, ppos); in cgroup_seqfile_start()
4116 return seq_cft(seq)->seq_next(seq, v, ppos); in cgroup_seqfile_next()
4121 if (seq_cft(seq)->seq_stop) in cgroup_seqfile_stop()
4122 seq_cft(seq)->seq_stop(seq, v); in cgroup_seqfile_stop()
4130 if (cft->seq_show) in cgroup_seqfile_show()
4131 return cft->seq_show(m, arg); in cgroup_seqfile_show()
4133 if (cft->read_u64) in cgroup_seqfile_show()
4134 seq_printf(m, "%llu\n", cft->read_u64(css, cft)); in cgroup_seqfile_show()
4135 else if (cft->read_s64) in cgroup_seqfile_show()
4136 seq_printf(m, "%lld\n", cft->read_s64(css, cft)); in cgroup_seqfile_show()
4138 return -EINVAL; in cgroup_seqfile_show()
4192 key = &cft->lockdep_key; in cgroup_add_file()
4194 kn = __kernfs_create_file(cgrp->kn, cgroup_file_name(cgrp, cft, name), in cgroup_add_file()
4197 0, cft->kf_ops, cft, in cgroup_add_file()
4208 if (cft->file_offset) { in cgroup_add_file()
4209 struct cgroup_file *cfile = (void *)css + cft->file_offset; in cgroup_add_file()
4211 timer_setup(&cfile->notify_timer, cgroup_file_notify_timer, 0); in cgroup_add_file()
4214 cfile->kn = kn; in cgroup_add_file()
4222 * cgroup_addrm_files - add or remove files to a cgroup directory
4224 * @cgrp: the target cgroup (usually css->cgroup)
4241 for (cft = cfts; cft != cft_end && cft->name[0] != '\0'; cft++) { in cgroup_addrm_files()
4242 /* does cft->flags tell us to skip this file on @cgrp? */ in cgroup_addrm_files()
4243 if ((cft->flags & __CFTYPE_ONLY_ON_DFL) && !cgroup_on_dfl(cgrp)) in cgroup_addrm_files()
4245 if ((cft->flags & __CFTYPE_NOT_ON_DFL) && cgroup_on_dfl(cgrp)) in cgroup_addrm_files()
4247 if ((cft->flags & CFTYPE_NOT_ON_ROOT) && !cgroup_parent(cgrp)) in cgroup_addrm_files()
4249 if ((cft->flags & CFTYPE_ONLY_ON_ROOT) && cgroup_parent(cgrp)) in cgroup_addrm_files()
4251 if ((cft->flags & CFTYPE_DEBUG) && !cgroup_debug) in cgroup_addrm_files()
4257 __func__, cft->name, ret); in cgroup_addrm_files()
4272 struct cgroup *root = &ss->root->cgrp; in cgroup_apply_cftypes()
4280 struct cgroup *cgrp = css->cgroup; in cgroup_apply_cftypes()
4282 if (!(css->flags & CSS_VISIBLE)) in cgroup_apply_cftypes()
4291 kernfs_activate(root->kn); in cgroup_apply_cftypes()
4299 for (cft = cfts; cft->name[0] != '\0'; cft++) { in cgroup_exit_cftypes()
4301 if (cft->max_write_len && cft->max_write_len != PAGE_SIZE) in cgroup_exit_cftypes()
4302 kfree(cft->kf_ops); in cgroup_exit_cftypes()
4303 cft->kf_ops = NULL; in cgroup_exit_cftypes()
4304 cft->ss = NULL; in cgroup_exit_cftypes()
4307 cft->flags &= ~(__CFTYPE_ONLY_ON_DFL | __CFTYPE_NOT_ON_DFL | in cgroup_exit_cftypes()
4317 for (cft = cfts; cft->name[0] != '\0'; cft++) { in cgroup_init_cftypes()
4320 WARN_ON(cft->ss || cft->kf_ops); in cgroup_init_cftypes()
4322 if (cft->flags & __CFTYPE_ADDED) { in cgroup_init_cftypes()
4323 ret = -EBUSY; in cgroup_init_cftypes()
4327 if (cft->seq_start) in cgroup_init_cftypes()
4336 if (cft->max_write_len && cft->max_write_len != PAGE_SIZE) { in cgroup_init_cftypes()
4339 ret = -ENOMEM; in cgroup_init_cftypes()
4342 kf_ops->atomic_write_len = cft->max_write_len; in cgroup_init_cftypes()
4345 cft->kf_ops = kf_ops; in cgroup_init_cftypes()
4346 cft->ss = ss; in cgroup_init_cftypes()
4347 cft->flags |= __CFTYPE_ADDED; in cgroup_init_cftypes()
4359 list_del(&cfts->node); in cgroup_rm_cftypes_locked()
4365 * cgroup_rm_cftypes - remove an array of cftypes from a subsystem
4366 * @cfts: zero-length name terminated array of cftypes
4372 * Returns 0 on successful unregistration, -ENOENT if @cfts is not
4381 return -ENOENT; in cgroup_rm_cftypes()
4390 * cgroup_add_cftypes - add an array of cftypes to a subsystem
4392 * @cfts: zero-length name terminated array of cftypes
4399 * Returns 0 on successful registration, -errno on failure. Note that this
4407 if (!cgroup_ssid_enabled(ss->id)) in cgroup_add_cftypes()
4419 list_add_tail(&cfts->node, &ss->cfts); in cgroup_add_cftypes()
4429 * cgroup_add_dfl_cftypes - add an array of cftypes for default hierarchy
4431 * @cfts: zero-length name terminated array of cftypes
4440 for (cft = cfts; cft && cft->name[0] != '\0'; cft++) in cgroup_add_dfl_cftypes()
4441 cft->flags |= __CFTYPE_ONLY_ON_DFL; in cgroup_add_dfl_cftypes()
4446 * cgroup_add_legacy_cftypes - add an array of cftypes for legacy hierarchies
4448 * @cfts: zero-length name terminated array of cftypes
4457 for (cft = cfts; cft && cft->name[0] != '\0'; cft++) in cgroup_add_legacy_cftypes()
4458 cft->flags |= __CFTYPE_NOT_ON_DFL; in cgroup_add_legacy_cftypes()
4463 * cgroup_file_notify - generate a file modified event for a cgroup_file
4466 * @cfile must have been obtained by setting cftype->file_offset.
4473 if (cfile->kn) { in cgroup_file_notify()
4474 unsigned long last = cfile->notified_at; in cgroup_file_notify()
4478 timer_reduce(&cfile->notify_timer, next); in cgroup_file_notify()
4480 kernfs_notify(cfile->kn); in cgroup_file_notify()
4481 cfile->notified_at = jiffies; in cgroup_file_notify()
4488 * cgroup_file_show - show or hide a hidden cgroup file
4489 * @cfile: target cgroup_file obtained by setting cftype->file_offset
4497 kn = cfile->kn; in cgroup_file_show()
4508 * css_next_child - find the next child of a given css
4517 * If a subsystem synchronizes ->css_online() and the start of iteration, a
4518 * css which finished ->css_online() is guaranteed to be visible in the
4520 * A css which hasn't finished ->css_online() or already finished
4521 * ->css_offline() may show up during traversal. It's each subsystem's
4533 * Once a cgroup is removed, its ->sibling.next is no longer in css_next_child()
4541 * have dropped rcu_read_lock() in-between iterations. in css_next_child()
4552 next = list_entry_rcu(parent->children.next, struct cgroup_subsys_state, sibling); in css_next_child()
4553 } else if (likely(!(pos->flags & CSS_RELEASED))) { in css_next_child()
4554 next = list_entry_rcu(pos->sibling.next, struct cgroup_subsys_state, sibling); in css_next_child()
4556 list_for_each_entry_rcu(next, &parent->children, sibling, in css_next_child()
4558 if (next->serial_nr > pos->serial_nr) in css_next_child()
4566 if (&next->sibling != &parent->children) in css_next_child()
4572 * css_next_descendant_pre - find the next descendant for pre-order walk
4577 * to visit for pre-order traversal of @root's descendants. @root is
4585 * If a subsystem synchronizes ->css_online() and the start of iteration, a
4586 * css which finished ->css_online() is guaranteed to be visible in the
4588 * A css which hasn't finished ->css_online() or already finished
4589 * ->css_offline() may show up during traversal. It's each subsystem's
4611 next = css_next_child(pos, pos->parent); in css_next_descendant_pre()
4614 pos = pos->parent; in css_next_descendant_pre()
4622 * css_rightmost_descendant - return the rightmost descendant of a css
4626 * is returned. This can be used during pre-order traversal to skip
4643 /* ->prev isn't RCU safe, walk ->next till the end */ in css_rightmost_descendant()
4666 * css_next_descendant_post - find the next descendant for post-order walk
4671 * to visit for post-order traversal of @root's descendants. @root is
4680 * If a subsystem synchronizes ->css_online() and the start of iteration, a
4681 * css which finished ->css_online() is guaranteed to be visible in the
4683 * A css which hasn't finished ->css_online() or already finished
4684 * ->css_offline() may show up during traversal. It's each subsystem's
4704 next = css_next_child(pos, pos->parent); in css_next_descendant_post()
4709 return pos->parent; in css_next_descendant_post()
4713 * css_has_online_children - does a css have online children
4727 if (child->flags & CSS_ONLINE) { in css_has_online_children()
4738 struct list_head *l; in css_task_iter_next_css_set() local
4745 if (it->tcset_pos) { in css_task_iter_next_css_set()
4746 l = it->tcset_pos->next; in css_task_iter_next_css_set()
4748 if (l != it->tcset_head) { in css_task_iter_next_css_set()
4749 it->tcset_pos = l; in css_task_iter_next_css_set()
4750 return container_of(l, struct css_set, in css_task_iter_next_css_set()
4754 it->tcset_pos = NULL; in css_task_iter_next_css_set()
4758 l = it->cset_pos; in css_task_iter_next_css_set()
4759 l = l->next; in css_task_iter_next_css_set()
4760 if (l == it->cset_head) { in css_task_iter_next_css_set()
4761 it->cset_pos = NULL; in css_task_iter_next_css_set()
4765 if (it->ss) { in css_task_iter_next_css_set()
4766 cset = container_of(l, struct css_set, e_cset_node[it->ss->id]); in css_task_iter_next_css_set()
4768 link = list_entry(l, struct cgrp_cset_link, cset_link); in css_task_iter_next_css_set()
4769 cset = link->cset; in css_task_iter_next_css_set()
4772 it->cset_pos = l; in css_task_iter_next_css_set()
4775 if (it->flags & CSS_TASK_ITER_THREADED) { in css_task_iter_next_css_set()
4776 if (it->cur_dcset) in css_task_iter_next_css_set()
4777 put_css_set_locked(it->cur_dcset); in css_task_iter_next_css_set()
4778 it->cur_dcset = cset; in css_task_iter_next_css_set()
4781 it->tcset_head = &cset->threaded_csets; in css_task_iter_next_css_set()
4782 it->tcset_pos = &cset->threaded_csets; in css_task_iter_next_css_set()
4789 * css_task_iter_advance_css_set - advance a task iterator to the next css_set
4800 /* Advance to the next non-empty css_set and find first non-empty tasks list*/ in css_task_iter_advance_css_set()
4802 if (!list_empty(&cset->tasks)) { in css_task_iter_advance_css_set()
4803 it->cur_tasks_head = &cset->tasks; in css_task_iter_advance_css_set()
4805 } else if (!list_empty(&cset->mg_tasks)) { in css_task_iter_advance_css_set()
4806 it->cur_tasks_head = &cset->mg_tasks; in css_task_iter_advance_css_set()
4808 } else if (!list_empty(&cset->dying_tasks)) { in css_task_iter_advance_css_set()
4809 it->cur_tasks_head = &cset->dying_tasks; in css_task_iter_advance_css_set()
4814 it->task_pos = NULL; in css_task_iter_advance_css_set()
4817 it->task_pos = it->cur_tasks_head->next; in css_task_iter_advance_css_set()
4822 * the lock is re-acquired. Iteration is performed at two levels - in css_task_iter_advance_css_set()
4834 if (it->cur_cset) { in css_task_iter_advance_css_set()
4835 list_del(&it->iters_node); in css_task_iter_advance_css_set()
4836 put_css_set_locked(it->cur_cset); in css_task_iter_advance_css_set()
4839 it->cur_cset = cset; in css_task_iter_advance_css_set()
4840 list_add(&it->iters_node, &cset->task_iters); in css_task_iter_advance_css_set()
4848 if (it->task_pos == &task->cg_list) { in css_task_iter_skip()
4849 it->task_pos = it->task_pos->next; in css_task_iter_skip()
4850 it->flags |= CSS_TASK_ITER_SKIPPED; in css_task_iter_skip()
4860 if (it->task_pos) { in css_task_iter_advance()
4866 if (it->flags & CSS_TASK_ITER_SKIPPED) in css_task_iter_advance()
4867 it->flags &= ~CSS_TASK_ITER_SKIPPED; in css_task_iter_advance()
4869 it->task_pos = it->task_pos->next; in css_task_iter_advance()
4871 if (it->task_pos == &it->cur_cset->tasks) { in css_task_iter_advance()
4872 it->cur_tasks_head = &it->cur_cset->mg_tasks; in css_task_iter_advance()
4873 it->task_pos = it->cur_tasks_head->next; in css_task_iter_advance()
4875 if (it->task_pos == &it->cur_cset->mg_tasks) { in css_task_iter_advance()
4876 it->cur_tasks_head = &it->cur_cset->dying_tasks; in css_task_iter_advance()
4877 it->task_pos = it->cur_tasks_head->next; in css_task_iter_advance()
4879 if (it->task_pos == &it->cur_cset->dying_tasks) in css_task_iter_advance()
4886 if (!it->task_pos) in css_task_iter_advance()
4889 task = list_entry(it->task_pos, struct task_struct, cg_list); in css_task_iter_advance()
4891 if (it->flags & CSS_TASK_ITER_PROCS) { in css_task_iter_advance()
4897 if (it->cur_tasks_head == &it->cur_cset->dying_tasks && in css_task_iter_advance()
4898 !atomic_read(&task->signal->live)) in css_task_iter_advance()
4902 if (it->cur_tasks_head == &it->cur_cset->dying_tasks) in css_task_iter_advance()
4908 * css_task_iter_start - initiate task iteration
4925 it->ss = css->ss; in css_task_iter_start()
4926 it->flags = flags; in css_task_iter_start()
4928 if (CGROUP_HAS_SUBSYS_CONFIG && it->ss) in css_task_iter_start()
4929 it->cset_pos = &css->cgroup->e_csets[css->ss->id]; in css_task_iter_start()
4931 it->cset_pos = &css->cgroup->cset_links; in css_task_iter_start()
4933 it->cset_head = it->cset_pos; in css_task_iter_start()
4941 * css_task_iter_next - return the next task for the iterator
4950 if (it->cur_task) { in css_task_iter_next()
4951 put_task_struct(it->cur_task); in css_task_iter_next()
4952 it->cur_task = NULL; in css_task_iter_next()
4957 /* @it may be half-advanced by skips, finish advancing */ in css_task_iter_next()
4958 if (it->flags & CSS_TASK_ITER_SKIPPED) in css_task_iter_next()
4961 if (it->task_pos) { in css_task_iter_next()
4962 it->cur_task = list_entry(it->task_pos, struct task_struct, in css_task_iter_next()
4964 get_task_struct(it->cur_task); in css_task_iter_next()
4970 return it->cur_task; in css_task_iter_next()
4974 * css_task_iter_end - finish task iteration
4981 if (it->cur_cset) { in css_task_iter_end()
4983 list_del(&it->iters_node); in css_task_iter_end()
4984 put_css_set_locked(it->cur_cset); in css_task_iter_end()
4988 if (it->cur_dcset) in css_task_iter_end()
4989 put_css_set(it->cur_dcset); in css_task_iter_end()
4991 if (it->cur_task) in css_task_iter_end()
4992 put_task_struct(it->cur_task); in css_task_iter_end()
4997 struct cgroup_file_ctx *ctx = of->priv; in cgroup_procs_release()
4999 if (ctx->procs.started) in cgroup_procs_release()
5000 css_task_iter_end(&ctx->procs.iter); in cgroup_procs_release()
5005 struct kernfs_open_file *of = s->private; in cgroup_procs_next()
5006 struct cgroup_file_ctx *ctx = of->priv; in cgroup_procs_next()
5011 return css_task_iter_next(&ctx->procs.iter); in cgroup_procs_next()
5017 struct kernfs_open_file *of = s->private; in __cgroup_procs_start()
5018 struct cgroup *cgrp = seq_css(s)->cgroup; in __cgroup_procs_start()
5019 struct cgroup_file_ctx *ctx = of->priv; in __cgroup_procs_start()
5020 struct css_task_iter *it = &ctx->procs.iter; in __cgroup_procs_start()
5026 if (!ctx->procs.started) { in __cgroup_procs_start()
5028 return ERR_PTR(-EINVAL); in __cgroup_procs_start()
5029 css_task_iter_start(&cgrp->self, iter_flags, it); in __cgroup_procs_start()
5030 ctx->procs.started = true; in __cgroup_procs_start()
5033 css_task_iter_start(&cgrp->self, iter_flags, it); in __cgroup_procs_start()
5035 return it->cur_task; in __cgroup_procs_start()
5042 struct cgroup *cgrp = seq_css(s)->cgroup; in cgroup_procs_start()
5051 return ERR_PTR(-EOPNOTSUPP); in cgroup_procs_start()
5070 inode = kernfs_get_inode(sb, cgrp->procs_file.kn); in cgroup_may_write()
5072 return -ENOMEM; in cgroup_may_write()
5103 (!cgroup_is_descendant(src_cgrp, ns->root_cset->dfl_cgrp) || in cgroup_procs_write_permission()
5104 !cgroup_is_descendant(dst_cgrp, ns->root_cset->dfl_cgrp))) in cgroup_procs_write_permission()
5105 return -ENOENT; in cgroup_procs_write_permission()
5125 if (!threadgroup && (src_cgrp->dom_cgrp != dst_cgrp->dom_cgrp)) in cgroup_attach_permissions()
5126 ret = -EOPNOTSUPP; in cgroup_attach_permissions()
5134 struct cgroup_file_ctx *ctx = of->priv; in __cgroup_procs_write()
5141 dst_cgrp = cgroup_kn_lock_live(of->kn, false); in __cgroup_procs_write()
5143 return -ENODEV; in __cgroup_procs_write()
5160 saved_cred = override_creds(of->file->f_cred); in __cgroup_procs_write()
5162 of->file->f_path.dentry->d_sb, in __cgroup_procs_write()
5163 threadgroup, ctx->ns); in __cgroup_procs_write()
5173 cgroup_kn_unlock(of->kn); in __cgroup_procs_write()
5320 * css destruction is four-stage process.
5338 * and thus involve punting to css->destroy_work adding two additional
5345 struct cgroup_subsys *ss = css->ss; in css_free_rwork_fn()
5346 struct cgroup *cgrp = css->cgroup; in css_free_rwork_fn()
5348 percpu_ref_exit(&css->refcnt); in css_free_rwork_fn()
5352 struct cgroup_subsys_state *parent = css->parent; in css_free_rwork_fn()
5353 int id = css->id; in css_free_rwork_fn()
5355 ss->css_free(css); in css_free_rwork_fn()
5356 cgroup_idr_remove(&ss->css_idr, id); in css_free_rwork_fn()
5363 atomic_dec(&cgrp->root->nr_cgrps); in css_free_rwork_fn()
5365 cancel_work_sync(&cgrp->release_agent_work); in css_free_rwork_fn()
5376 kernfs_put(cgrp->kn); in css_free_rwork_fn()
5386 cgroup_destroy_root(cgrp->root); in css_free_rwork_fn()
5395 struct cgroup_subsys *ss = css->ss; in css_release_work_fn()
5396 struct cgroup *cgrp = css->cgroup; in css_release_work_fn()
5400 css->flags |= CSS_RELEASED; in css_release_work_fn()
5401 list_del_rcu(&css->sibling); in css_release_work_fn()
5405 if (!list_empty(&css->rstat_css_node)) { in css_release_work_fn()
5407 list_del_rcu(&css->rstat_css_node); in css_release_work_fn()
5410 cgroup_idr_replace(&ss->css_idr, NULL, css->id); in css_release_work_fn()
5411 if (ss->css_released) in css_release_work_fn()
5412 ss->css_released(css); in css_release_work_fn()
5424 tcgrp->nr_dying_descendants--; in css_release_work_fn()
5429 * cgroup from dentry without going through kernfs - in css_release_work_fn()
5432 * cgrp->kn->priv backpointer. in css_release_work_fn()
5434 if (cgrp->kn) in css_release_work_fn()
5435 RCU_INIT_POINTER(*(void __rcu __force **)&cgrp->kn->priv, in css_release_work_fn()
5441 INIT_RCU_WORK(&css->destroy_rwork, css_free_rwork_fn); in css_release_work_fn()
5442 queue_rcu_work(cgroup_destroy_wq, &css->destroy_rwork); in css_release_work_fn()
5450 INIT_WORK(&css->destroy_work, css_release_work_fn); in css_release()
5451 queue_work(cgroup_destroy_wq, &css->destroy_work); in css_release()
5462 css->cgroup = cgrp; in init_and_link_css()
5463 css->ss = ss; in init_and_link_css()
5464 css->id = -1; in init_and_link_css()
5465 INIT_LIST_HEAD(&css->sibling); in init_and_link_css()
5466 INIT_LIST_HEAD(&css->children); in init_and_link_css()
5467 INIT_LIST_HEAD(&css->rstat_css_node); in init_and_link_css()
5468 css->serial_nr = css_serial_nr_next++; in init_and_link_css()
5469 atomic_set(&css->online_cnt, 0); in init_and_link_css()
5472 css->parent = cgroup_css(cgroup_parent(cgrp), ss); in init_and_link_css()
5473 css_get(css->parent); in init_and_link_css()
5476 if (ss->css_rstat_flush) in init_and_link_css()
5477 list_add_rcu(&css->rstat_css_node, &cgrp->rstat_css_list); in init_and_link_css()
5482 /* invoke ->css_online() on a new CSS and mark it online if successful */
5485 struct cgroup_subsys *ss = css->ss; in online_css()
5490 if (ss->css_online) in online_css()
5491 ret = ss->css_online(css); in online_css()
5493 css->flags |= CSS_ONLINE; in online_css()
5494 rcu_assign_pointer(css->cgroup->subsys[ss->id], css); in online_css()
5496 atomic_inc(&css->online_cnt); in online_css()
5497 if (css->parent) in online_css()
5498 atomic_inc(&css->parent->online_cnt); in online_css()
5503 /* if the CSS is online, invoke ->css_offline() on it and mark it offline */
5506 struct cgroup_subsys *ss = css->ss; in offline_css()
5510 if (!(css->flags & CSS_ONLINE)) in offline_css()
5513 if (ss->css_offline) in offline_css()
5514 ss->css_offline(css); in offline_css()
5516 css->flags &= ~CSS_ONLINE; in offline_css()
5517 RCU_INIT_POINTER(css->cgroup->subsys[ss->id], NULL); in offline_css()
5519 wake_up_all(&css->cgroup->offline_waitq); in offline_css()
5523 * css_create - create a cgroup_subsys_state
5527 * Create a new css associated with @cgrp - @ss pair. On success, the new
5529 * interface files. Returns 0 on success, -errno on failure.
5541 css = ss->css_alloc(parent_css); in css_create()
5543 css = ERR_PTR(-ENOMEM); in css_create()
5549 err = percpu_ref_init(&css->refcnt, css_release, 0, GFP_KERNEL); in css_create()
5553 err = cgroup_idr_alloc(&ss->css_idr, NULL, 2, 0, GFP_KERNEL); in css_create()
5556 css->id = err; in css_create()
5559 list_add_tail_rcu(&css->sibling, &parent_css->children); in css_create()
5560 cgroup_idr_replace(&ss->css_idr, css, css->id); in css_create()
5569 list_del_rcu(&css->sibling); in css_create()
5571 list_del_rcu(&css->rstat_css_node); in css_create()
5572 INIT_RCU_WORK(&css->destroy_rwork, css_free_rwork_fn); in css_create()
5573 queue_rcu_work(cgroup_destroy_wq, &css->destroy_rwork); in css_create()
5584 struct cgroup_root *root = parent->root; in cgroup_create()
5587 int level = parent->level + 1; in cgroup_create()
5593 return ERR_PTR(-ENOMEM); in cgroup_create()
5595 ret = percpu_ref_init(&cgrp->self.refcnt, css_release, 0, GFP_KERNEL); in cgroup_create()
5604 kn = kernfs_create_dir(parent->kn, name, mode, cgrp); in cgroup_create()
5609 cgrp->kn = kn; in cgroup_create()
5613 cgrp->self.parent = &parent->self; in cgroup_create()
5614 cgrp->root = root; in cgroup_create()
5615 cgrp->level = level; in cgroup_create()
5621 if (cgrp->root == &cgrp_dfl_root) { in cgroup_create()
5631 cgrp->freezer.e_freeze = parent->freezer.e_freeze; in cgroup_create()
5632 if (cgrp->freezer.e_freeze) { in cgroup_create()
5639 set_bit(CGRP_FREEZE, &cgrp->flags); in cgroup_create()
5640 set_bit(CGRP_FROZEN, &cgrp->flags); in cgroup_create()
5645 cgrp->ancestors[tcgrp->level] = tcgrp; in cgroup_create()
5648 tcgrp->nr_descendants++; in cgroup_create()
5655 if (cgrp->freezer.e_freeze) in cgroup_create()
5656 tcgrp->freezer.nr_frozen_descendants++; in cgroup_create()
5662 set_bit(CGRP_NOTIFY_ON_RELEASE, &cgrp->flags); in cgroup_create()
5664 if (test_bit(CGRP_CPUSET_CLONE_CHILDREN, &parent->flags)) in cgroup_create()
5665 set_bit(CGRP_CPUSET_CLONE_CHILDREN, &cgrp->flags); in cgroup_create()
5667 cgrp->self.serial_nr = css_serial_nr_next++; in cgroup_create()
5670 list_add_tail_rcu(&cgrp->self.sibling, &cgroup_parent(cgrp)->self.children); in cgroup_create()
5671 atomic_inc(&root->nr_cgrps); in cgroup_create()
5679 cgrp->subtree_control = cgroup_control(cgrp); in cgroup_create()
5688 kernfs_remove(cgrp->kn); in cgroup_create()
5692 percpu_ref_exit(&cgrp->self.refcnt); in cgroup_create()
5707 if (cgroup->nr_descendants >= cgroup->max_descendants) in cgroup_check_hierarchy_limits()
5710 if (level >= cgroup->max_depth) in cgroup_check_hierarchy_limits()
5728 return -EINVAL; in cgroup_mkdir()
5732 return -ENODEV; in cgroup_mkdir()
5735 ret = -EAGAIN; in cgroup_mkdir()
5747 * that @cgrp->kn is always accessible. in cgroup_mkdir()
5749 kernfs_get(cgrp->kn); in cgroup_mkdir()
5751 ret = cgroup_kn_set_ugid(cgrp->kn); in cgroup_mkdir()
5755 ret = css_populate_dir(&cgrp->self); in cgroup_mkdir()
5766 kernfs_activate(cgrp->kn); in cgroup_mkdir()
5794 css = css->parent; in css_killed_work_fn()
5795 } while (css && atomic_dec_and_test(&css->online_cnt)); in css_killed_work_fn()
5806 if (atomic_dec_and_test(&css->online_cnt)) { in css_killed_ref_fn()
5807 INIT_WORK(&css->destroy_work, css_killed_work_fn); in css_killed_ref_fn()
5808 queue_work(cgroup_destroy_wq, &css->destroy_work); in css_killed_ref_fn()
5813 * kill_css - destroy a css
5817 * files and putting its base reference. ->css_offline() will be invoked
5825 if (css->flags & CSS_DYING) in kill_css()
5828 css->flags |= CSS_DYING; in kill_css()
5838 * until after ->css_offline(). in kill_css()
5843 * cgroup core guarantees that, by the time ->css_offline() is in kill_css()
5852 percpu_ref_kill_and_confirm(&css->refcnt, css_killed_ref_fn); in kill_css()
5856 * cgroup_destroy_locked - the first stage of cgroup destruction
5862 * ->css_offline() is invoked. To satisfy all the requirements,
5870 * s2. Invoke ->css_offline(), mark the cgroup dead and proceed with the
5872 * cgroup is RCU-freed.
5894 return -EBUSY; in cgroup_destroy_locked()
5898 * ->self.children as dead children linger on it while being in cgroup_destroy_locked()
5901 if (css_has_online_children(&cgrp->self)) in cgroup_destroy_locked()
5902 return -EBUSY; in cgroup_destroy_locked()
5910 cgrp->self.flags &= ~CSS_ONLINE; in cgroup_destroy_locked()
5913 list_for_each_entry(link, &cgrp->cset_links, cset_link) in cgroup_destroy_locked()
5914 link->cset->dead = true; in cgroup_destroy_locked()
5922 css_clear_dir(&cgrp->self); in cgroup_destroy_locked()
5923 kernfs_remove(cgrp->kn); in cgroup_destroy_locked()
5926 parent->nr_threaded_children--; in cgroup_destroy_locked()
5930 tcgrp->nr_descendants--; in cgroup_destroy_locked()
5931 tcgrp->nr_dying_descendants++; in cgroup_destroy_locked()
5936 if (test_bit(CGRP_FROZEN, &cgrp->flags)) in cgroup_destroy_locked()
5937 tcgrp->freezer.nr_frozen_descendants--; in cgroup_destroy_locked()
5943 if (cgrp->root == &cgrp_dfl_root) in cgroup_destroy_locked()
5947 percpu_ref_kill(&cgrp->self.refcnt); in cgroup_destroy_locked()
5980 pr_debug("Initializing cgroup subsys %s\n", ss->name); in cgroup_init_subsys()
5984 idr_init(&ss->css_idr); in cgroup_init_subsys()
5985 INIT_LIST_HEAD(&ss->cfts); in cgroup_init_subsys()
5988 ss->root = &cgrp_dfl_root; in cgroup_init_subsys()
5989 css = ss->css_alloc(NULL); in cgroup_init_subsys()
5998 css->flags |= CSS_NO_REF; in cgroup_init_subsys()
6002 css->id = 1; in cgroup_init_subsys()
6004 css->id = cgroup_idr_alloc(&ss->css_idr, css, 1, 2, GFP_KERNEL); in cgroup_init_subsys()
6005 BUG_ON(css->id < 0); in cgroup_init_subsys()
6009 * pointer to this state - since the subsystem is in cgroup_init_subsys()
6012 init_css_set.subsys[ss->id] = css; in cgroup_init_subsys()
6014 have_fork_callback |= (bool)ss->fork << ss->id; in cgroup_init_subsys()
6015 have_exit_callback |= (bool)ss->exit << ss->id; in cgroup_init_subsys()
6016 have_release_callback |= (bool)ss->release << ss->id; in cgroup_init_subsys()
6017 have_canfork_callback |= (bool)ss->can_fork << ss->id; in cgroup_init_subsys()
6030 * cgroup_init_early - cgroup initialization at system boot
6048 WARN(!ss->css_alloc || !ss->css_free || ss->name || ss->id, in cgroup_init_early()
6050 i, cgroup_subsys_name[i], ss->css_alloc, ss->css_free, in cgroup_init_early()
6051 ss->id, ss->name); in cgroup_init_early()
6055 ss->id = i; in cgroup_init_early()
6056 ss->name = cgroup_subsys_name[i]; in cgroup_init_early()
6057 if (!ss->legacy_name) in cgroup_init_early()
6058 ss->legacy_name = cgroup_subsys_name[i]; in cgroup_init_early()
6060 if (ss->early_init) in cgroup_init_early()
6067 * cgroup_init - cgroup initialization
6100 if (ss->early_init) { in cgroup_init()
6102 init_css_set.subsys[ss->id]; in cgroup_init()
6104 css->id = cgroup_idr_alloc(&ss->css_idr, css, 1, 2, in cgroup_init()
6106 BUG_ON(css->id < 0); in cgroup_init()
6124 ss->name); in cgroup_init()
6126 cgrp_dfl_root.subsys_mask |= 1 << ss->id; in cgroup_init()
6129 WARN_ON(ss->implicit_on_dfl && !ss->threaded); in cgroup_init()
6131 if (ss->implicit_on_dfl) in cgroup_init()
6132 cgrp_dfl_implicit_ss_mask |= 1 << ss->id; in cgroup_init()
6133 else if (!ss->dfl_cftypes) in cgroup_init()
6134 cgrp_dfl_inhibit_ss_mask |= 1 << ss->id; in cgroup_init()
6136 if (ss->threaded) in cgroup_init()
6137 cgrp_dfl_threaded_ss_mask |= 1 << ss->id; in cgroup_init()
6139 if (ss->dfl_cftypes == ss->legacy_cftypes) { in cgroup_init()
6140 WARN_ON(cgroup_add_cftypes(ss, ss->dfl_cftypes)); in cgroup_init()
6142 WARN_ON(cgroup_add_dfl_cftypes(ss, ss->dfl_cftypes)); in cgroup_init()
6143 WARN_ON(cgroup_add_legacy_cftypes(ss, ss->legacy_cftypes)); in cgroup_init()
6146 if (ss->bind) in cgroup_init()
6147 ss->bind(init_css_set.subsys[ssid]); in cgroup_init()
6154 /* init_css_set.subsys[] has been updated, re-hash */ in cgroup_init()
6210 return ERR_PTR(-ENOENT); in cgroup_get_from_id()
6214 return ERR_PTR(-ENOENT); in cgroup_get_from_id()
6219 cgrp = rcu_dereference(*(void __rcu __force **)&kn->priv); in cgroup_get_from_id()
6227 return ERR_PTR(-ENOENT); in cgroup_get_from_id()
6232 return ERR_PTR(-ENOENT); in cgroup_get_from_id()
6241 * - Print task's cgroup paths into seq_file, one line for each hierarchy
6242 * - Used for /proc/<pid>/cgroup.
6251 retval = -ENOMEM; in proc_cgroup_show()
6267 seq_printf(m, "%d:", root->hierarchy_id); in proc_cgroup_show()
6270 if (root->subsys_mask & (1 << ssid)) in proc_cgroup_show()
6272 ss->legacy_name); in proc_cgroup_show()
6273 if (strlen(root->name)) in proc_cgroup_show()
6275 root->name); in proc_cgroup_show()
6289 if (cgroup_on_dfl(cgrp) || !(tsk->flags & PF_EXITING)) { in proc_cgroup_show()
6291 current->nsproxy->cgroup_ns); in proc_cgroup_show()
6292 if (retval == -E2BIG) in proc_cgroup_show()
6293 retval = -ENAMETOOLONG; in proc_cgroup_show()
6318 * cgroup_fork - initialize cgroup related fields during copy_process()
6326 RCU_INIT_POINTER(child->cgroups, &init_css_set); in cgroup_fork()
6327 INIT_LIST_HEAD(&child->cg_list); in cgroup_fork()
6331 * cgroup_v1v2_get_from_file - get a cgroup pointer from a file pointer
6342 css = css_tryget_online_from_dir(f->f_path.dentry, NULL); in cgroup_v1v2_get_from_file()
6346 return css->cgroup; in cgroup_v1v2_get_from_file()
6350 * cgroup_get_from_file - same as cgroup_v1v2_get_from_file, but only supports
6363 return ERR_PTR(-EBADF); in cgroup_get_from_file()
6370 * cgroup_css_set_fork - find or create a css_set for a child process
6394 if (kargs->flags & CLONE_INTO_CGROUP) in cgroup_css_set_fork()
6404 if (!(kargs->flags & CLONE_INTO_CGROUP)) { in cgroup_css_set_fork()
6405 kargs->cset = cset; in cgroup_css_set_fork()
6409 f = fget_raw(kargs->cgroup); in cgroup_css_set_fork()
6411 ret = -EBADF; in cgroup_css_set_fork()
6414 sb = f->f_path.dentry->d_sb; in cgroup_css_set_fork()
6424 ret = -ENODEV; in cgroup_css_set_fork()
6445 * write(fd, <child-pid>, ...); in cgroup_css_set_fork()
6451 ret = cgroup_attach_permissions(cset->dfl_cgrp, dst_cgrp, sb, in cgroup_css_set_fork()
6452 !(kargs->flags & CLONE_THREAD), in cgroup_css_set_fork()
6453 current->nsproxy->cgroup_ns); in cgroup_css_set_fork()
6457 kargs->cset = find_css_set(cset, dst_cgrp); in cgroup_css_set_fork()
6458 if (!kargs->cset) { in cgroup_css_set_fork()
6459 ret = -ENOMEM; in cgroup_css_set_fork()
6465 kargs->cgrp = dst_cgrp; in cgroup_css_set_fork()
6476 if (kargs->cset) in cgroup_css_set_fork()
6477 put_css_set(kargs->cset); in cgroup_css_set_fork()
6482 * cgroup_css_set_put_fork - drop references we took during fork
6491 struct cgroup *cgrp = kargs->cgrp; in cgroup_css_set_put_fork()
6492 struct css_set *cset = kargs->cset; in cgroup_css_set_put_fork()
6498 kargs->cset = NULL; in cgroup_css_set_put_fork()
6501 if (kargs->flags & CLONE_INTO_CGROUP) { in cgroup_css_set_put_fork()
6505 kargs->cgrp = NULL; in cgroup_css_set_put_fork()
6511 * cgroup_can_fork - called on a new task before the process is exposed
6531 ret = ss->can_fork(child, kargs->cset); in cgroup_can_fork()
6542 if (ss->cancel_fork) in cgroup_can_fork()
6543 ss->cancel_fork(child, kargs->cset); in cgroup_can_fork()
6552 * cgroup_cancel_fork - called if a fork failed after cgroup_can_fork()
6567 if (ss->cancel_fork) in cgroup_cancel_fork()
6568 ss->cancel_fork(child, kargs->cset); in cgroup_cancel_fork()
6574 * cgroup_post_fork - finalize cgroup setup for the child process
6591 cset = kargs->cset; in cgroup_post_fork()
6592 kargs->cset = NULL; in cgroup_post_fork()
6597 if (likely(child->pid)) { in cgroup_post_fork()
6598 if (kargs->cgrp) in cgroup_post_fork()
6599 cgrp_flags = kargs->cgrp->flags; in cgroup_post_fork()
6601 cgrp_flags = cset->dfl_cgrp->flags; in cgroup_post_fork()
6603 WARN_ON_ONCE(!list_empty(&child->cg_list)); in cgroup_post_fork()
6604 cset->nr_tasks++; in cgroup_post_fork()
6611 if (!(child->flags & PF_KTHREAD)) { in cgroup_post_fork()
6618 spin_lock(&child->sighand->siglock); in cgroup_post_fork()
6619 WARN_ON_ONCE(child->frozen); in cgroup_post_fork()
6620 child->jobctl |= JOBCTL_TRAP_FREEZE; in cgroup_post_fork()
6621 spin_unlock(&child->sighand->siglock); in cgroup_post_fork()
6642 * Call ss->fork(). This must happen after @child is linked on in cgroup_post_fork()
6643 * css_set; otherwise, @child might change state between ->fork() in cgroup_post_fork()
6647 ss->fork(child); in cgroup_post_fork()
6651 if (kargs->flags & CLONE_NEWCGROUP) { in cgroup_post_fork()
6652 struct css_set *rcset = child->nsproxy->cgroup_ns->root_cset; in cgroup_post_fork()
6655 child->nsproxy->cgroup_ns->root_cset = cset; in cgroup_post_fork()
6667 * cgroup_exit - detach cgroup from exiting task
6681 WARN_ON_ONCE(list_empty(&tsk->cg_list)); in cgroup_exit()
6684 list_add_tail(&tsk->cg_list, &cset->dying_tasks); in cgroup_exit()
6685 cset->nr_tasks--; in cgroup_exit()
6691 if (unlikely(!(tsk->flags & PF_KTHREAD) && in cgroup_exit()
6692 test_bit(CGRP_FREEZE, &task_dfl_cgroup(tsk)->flags))) in cgroup_exit()
6699 ss->exit(tsk); in cgroup_exit()
6709 ss->release(task); in cgroup_release()
6714 list_del_init(&task->cg_list); in cgroup_release()
6735 if (strcmp(token, ss->name) && in cgroup_disable()
6736 strcmp(token, ss->legacy_name)) in cgroup_disable()
6741 ss->name); in cgroup_disable()
6768 * css_tryget_online_from_dir - get corresponding css from a cgroup dentry
6780 struct file_system_type *s_type = dentry->d_sb->s_type; in css_tryget_online_from_dir()
6787 return ERR_PTR(-EBADF); in css_tryget_online_from_dir()
6793 * have been or be removed at any point. @kn->priv is RCU in css_tryget_online_from_dir()
6796 cgrp = rcu_dereference(*(void __rcu __force **)&kn->priv); in css_tryget_online_from_dir()
6801 css = ERR_PTR(-ENOENT); in css_tryget_online_from_dir()
6808 * css_from_id - lookup css by id
6818 return idr_find(&ss->css_idr, id); in css_from_id()
6822 * cgroup_get_from_path - lookup and get a cgroup from its default hierarchy path
6827 * success, ERR_PTR(-ENOENT) if @path doesn't exist or if the cgroup has already
6828 * been released and ERR_PTR(-ENOTDIR) if @path points to a non-directory.
6833 struct cgroup *cgrp = ERR_PTR(-ENOENT); in cgroup_get_from_path()
6837 kn = kernfs_walk_and_get(root_cgrp->kn, path); in cgroup_get_from_path()
6842 cgrp = ERR_PTR(-ENOTDIR); in cgroup_get_from_path()
6848 cgrp = rcu_dereference(*(void __rcu __force **)&kn->priv); in cgroup_get_from_path()
6850 cgrp = ERR_PTR(-ENOENT); in cgroup_get_from_path()
6862 * cgroup_v1v2_get_from_fd - get a cgroup pointer from a fd
6875 return ERR_PTR(-EBADF); in cgroup_v1v2_get_from_fd()
6883 * cgroup_get_from_fd - same as cgroup_v1v2_get_from_fd, but only supports
6896 return ERR_PTR(-EBADF); in cgroup_get_from_fd()
6905 while (power--) in power_of_ten()
6911 * cgroup_parse_float - parse a floating number
6919 * Returns 0 on success, -errno otherwise.
6930 return -EINVAL; in cgroup_parse_float()
6932 return -EINVAL; in cgroup_parse_float()
6934 flen = fend > fstart ? fend - fstart : 0; in cgroup_parse_float()
6936 frac *= power_of_ten(dec_shift - flen); in cgroup_parse_float()
6938 frac = DIV_ROUND_CLOSEST_ULL(frac, power_of_ten(flen - dec_shift)); in cgroup_parse_float()
6945 * sock->sk_cgrp_data handling. For more info, see sock_cgroup_data
6946 * definition in cgroup-defs.h.
6966 if (likely(cgroup_tryget(cset->dfl_cgrp))) { in cgroup_sk_alloc()
6967 cgroup = cset->dfl_cgrp; in cgroup_sk_alloc()
6973 skcd->cgroup = cgroup; in cgroup_sk_alloc()
7008 for (cft = files; cft && cft->name[0] != '\0'; cft++) { in show_delegatable_files()
7009 if (!(cft->flags & CFTYPE_NS_DELEGATABLE)) in show_delegatable_files()
7013 ret += snprintf(buf + ret, size - ret, "%s.", prefix); in show_delegatable_files()
7015 ret += snprintf(buf + ret, size - ret, "%s\n", cft->name); in show_delegatable_files()
7032 PAGE_SIZE - ret, NULL); in delegate_show()
7035 PAGE_SIZE - ret, NULL); in delegate_show()
7038 ret += show_delegatable_files(ss->dfl_cftypes, buf + ret, in delegate_show()
7039 PAGE_SIZE - ret, in delegate_show()