614 		count += link->cset->nr_tasks;  in __cgroup_task_count()
750 static bool css_set_threaded(struct css_set *cset)  in css_set_threaded()  argument
752 	return cset->dom_cset != cset;  in css_set_threaded()
764 static bool css_set_populated(struct css_set *cset)  in css_set_populated()  argument
768 	return !list_empty(&cset->tasks) || !list_empty(&cset->mg_tasks);  in css_set_populated()
828 static void css_set_update_populated(struct css_set *cset, bool populated)  in css_set_update_populated()  argument
834 	list_for_each_entry(link, &cset->cgrp_links, cgrp_link)  in css_set_update_populated()
844 static void css_set_skip_task_iters(struct css_set *cset,  in css_set_skip_task_iters()  argument
849 	list_for_each_entry_safe(it, pos, &cset->task_iters, iters_node)  in css_set_skip_task_iters()
923 void put_css_set_locked(struct css_set *cset)  in put_css_set_locked()  argument
931 	if (!refcount_dec_and_test(&cset->refcount))  in put_css_set_locked()
934 	WARN_ON_ONCE(!list_empty(&cset->threaded_csets));  in put_css_set_locked()
938 		list_del(&cset->e_cset_node[ssid]);  in put_css_set_locked()
939 		css_put(cset->subsys[ssid]);  in put_css_set_locked()
941 	hash_del(&cset->hlist);  in put_css_set_locked()
944 	list_for_each_entry_safe(link, tmp_link, &cset->cgrp_links, cgrp_link) {  in put_css_set_locked()
952 	if (css_set_threaded(cset)) {  in put_css_set_locked()
953 		list_del(&cset->threaded_csets_node);  in put_css_set_locked()
954 		put_css_set_locked(cset->dom_cset);  in put_css_set_locked()
957 	kfree_rcu(cset, rcu_head);  in put_css_set_locked()
970 static bool compare_css_sets(struct css_set *cset,  in compare_css_sets()  argument
983 	if (memcmp(template, cset->subsys, sizeof(cset->subsys)))  in compare_css_sets()
993 	if (new_dfl_cgrp->dom_cgrp != cset->dom_cset->dfl_cgrp)  in compare_css_sets()
1002 	l1 = &cset->cgrp_links;  in compare_css_sets()
1011 		if (l1 == &cset->cgrp_links) {  in compare_css_sets()
1055 	struct css_set *cset;  in find_existing_css_set()  local
1081 	hash_for_each_possible(css_set_table, cset, hlist, key) {  in find_existing_css_set()
1082 		if (!compare_css_sets(cset, old_cset, cgrp, template))  in find_existing_css_set()
1086 		return cset;  in find_existing_css_set()
1135 static void link_css_set(struct list_head *tmp_links, struct css_set *cset,  in link_css_set()  argument
1143 		cset->dfl_cgrp = cgrp;  in link_css_set()
1146 	link->cset = cset;  in link_css_set()
1154 	list_add_tail(&link->cgrp_link, &cset->cgrp_links);  in link_css_set()
1172 	struct css_set *cset;  in find_css_set()  local
1184 	cset = find_existing_css_set(old_cset, cgrp, template);  in find_css_set()
1185 	if (cset)  in find_css_set()
1186 		get_css_set(cset);  in find_css_set()
1189 	if (cset)  in find_css_set()
1190 		return cset;  in find_css_set()
1192 	cset = kzalloc(sizeof(*cset), GFP_KERNEL);  in find_css_set()
1193 	if (!cset)  in find_css_set()
1198 		kfree(cset);  in find_css_set()
1202 	refcount_set(&cset->refcount, 1);  in find_css_set()
1203 	cset->dom_cset = cset;  in find_css_set()
1204 	INIT_LIST_HEAD(&cset->tasks);  in find_css_set()
1205 	INIT_LIST_HEAD(&cset->mg_tasks);  in find_css_set()
1206 	INIT_LIST_HEAD(&cset->dying_tasks);  in find_css_set()
1207 	INIT_LIST_HEAD(&cset->task_iters);  in find_css_set()
1208 	INIT_LIST_HEAD(&cset->threaded_csets);  in find_css_set()
1209 	INIT_HLIST_NODE(&cset->hlist);  in find_css_set()
1210 	INIT_LIST_HEAD(&cset->cgrp_links);  in find_css_set()
1211 	INIT_LIST_HEAD(&cset->mg_src_preload_node);  in find_css_set()
1212 	INIT_LIST_HEAD(&cset->mg_dst_preload_node);  in find_css_set()
1213 	INIT_LIST_HEAD(&cset->mg_node);  in find_css_set()
1217 	memcpy(cset->subsys, template, sizeof(cset->subsys));  in find_css_set()
1226 		link_css_set(&tmp_links, cset, c);  in find_css_set()
1234 	key = css_set_hash(cset->subsys);  in find_css_set()
1235 	hash_add(css_set_table, &cset->hlist, key);  in find_css_set()
1238 		struct cgroup_subsys_state *css = cset->subsys[ssid];  in find_css_set()
1240 		list_add_tail(&cset->e_cset_node[ssid],  in find_css_set()
1253 	if (cgroup_is_threaded(cset->dfl_cgrp)) {  in find_css_set()
1256 		dcset = find_css_set(cset, cset->dfl_cgrp->dom_cgrp);  in find_css_set()
1258 			put_css_set(cset);  in find_css_set()
1263 		cset->dom_cset = dcset;  in find_css_set()
1264 		list_add_tail(&cset->threaded_csets_node,  in find_css_set()
1269 	return cset;  in find_css_set()
1366 static inline struct cgroup *__cset_cgroup_from_root(struct css_set *cset,  in __cset_cgroup_from_root()  argument
1371 	if (cset == &init_css_set) {  in __cset_cgroup_from_root()
1374 		res_cgroup = cset->dfl_cgrp;  in __cset_cgroup_from_root()
1379 		list_for_each_entry(link, &cset->cgrp_links, cgrp_link) {  in __cset_cgroup_from_root()
1401 	struct css_set *cset;  in current_cgns_cgroup_from_root()  local
1407 	cset = current->nsproxy->cgroup_ns->root_cset;  in current_cgns_cgroup_from_root()
1408 	res = __cset_cgroup_from_root(cset, root);  in current_cgns_cgroup_from_root()
1428 	struct css_set *cset;  in current_cgns_cgroup_dfl()  local
1431 		cset = current->nsproxy->cgroup_ns->root_cset;  in current_cgns_cgroup_dfl()
1432 		return __cset_cgroup_from_root(cset, &cgrp_dfl_root);  in current_cgns_cgroup_dfl()
1445 static struct cgroup *cset_cgroup_from_root(struct css_set *cset,  in cset_cgroup_from_root()  argument
1451 	return __cset_cgroup_from_root(cset, root);  in cset_cgroup_from_root()
1807 		struct css_set *cset, *cset_pos;  in rebind_subsystems()  local
1827 		list_for_each_entry_safe(cset, cset_pos, &scgrp->e_csets[ss->id],  in rebind_subsystems()
1829 			list_move_tail(&cset->e_cset_node[ss->id],  in rebind_subsystems()
1838 			list_for_each_entry(it, &cset->task_iters, iters_node)  in rebind_subsystems()
2037 	struct css_set *cset;  in cgroup_setup_root()  local
2108 	hash_for_each(css_set_table, i, cset, hlist) {  in cgroup_setup_root()
2109 		link_css_set(&tmp_links, cset, root_cgrp);  in cgroup_setup_root()
2110 		if (css_set_populated(cset))  in cgroup_setup_root()
2411 	struct css_set *cset;  in cgroup_migrate_add_task()  local
2422 	cset = task_css_set(task);  in cgroup_migrate_add_task()
2423 	if (!cset->mg_src_cgrp)  in cgroup_migrate_add_task()
2428 	list_move_tail(&task->cg_list, &cset->mg_tasks);  in cgroup_migrate_add_task()
2429 	if (list_empty(&cset->mg_node))  in cgroup_migrate_add_task()
2430 		list_add_tail(&cset->mg_node,  in cgroup_migrate_add_task()
2432 	if (list_empty(&cset->mg_dst_cset->mg_node))  in cgroup_migrate_add_task()
2433 		list_add_tail(&cset->mg_dst_cset->mg_node,  in cgroup_migrate_add_task()
2464 	struct css_set *cset = tset->cur_cset;  in cgroup_taskset_next()  local
2467 	while (CGROUP_HAS_SUBSYS_CONFIG && &cset->mg_node != tset->csets) {  in cgroup_taskset_next()
2469 			task = list_first_entry(&cset->mg_tasks,  in cgroup_taskset_next()
2474 		if (&task->cg_list != &cset->mg_tasks) {  in cgroup_taskset_next()
2475 			tset->cur_cset = cset;  in cgroup_taskset_next()
2484 			if (cset->mg_dst_cset)  in cgroup_taskset_next()
2485 				*dst_cssp = cset->mg_dst_cset->subsys[tset->ssid];  in cgroup_taskset_next()
2487 				*dst_cssp = cset->subsys[tset->ssid];  in cgroup_taskset_next()
2492 		cset = list_next_entry(cset, mg_node);  in cgroup_taskset_next()
2513 	struct css_set *cset, *tmp_cset;  in cgroup_migrate_execute()  local
2536 	list_for_each_entry(cset, &tset->src_csets, mg_node) {  in cgroup_migrate_execute()
2537 		list_for_each_entry_safe(task, tmp_task, &cset->mg_tasks, cg_list) {  in cgroup_migrate_execute()
2539 			struct css_set *to_cset = cset->mg_dst_cset;  in cgroup_migrate_execute()
2590 	list_for_each_entry_safe(cset, tmp_cset, &tset->src_csets, mg_node) {  in cgroup_migrate_execute()
2591 		list_splice_tail_init(&cset->mg_tasks, &cset->tasks);  in cgroup_migrate_execute()
2592 		list_del_init(&cset->mg_node);  in cgroup_migrate_execute()
2648 	struct css_set *cset, *tmp_cset;  in cgroup_migrate_finish()  local
2654 	list_for_each_entry_safe(cset, tmp_cset, &mgctx->preloaded_src_csets,  in cgroup_migrate_finish()
2656 		cset->mg_src_cgrp = NULL;  in cgroup_migrate_finish()
2657 		cset->mg_dst_cgrp = NULL;  in cgroup_migrate_finish()
2658 		cset->mg_dst_cset = NULL;  in cgroup_migrate_finish()
2659 		list_del_init(&cset->mg_src_preload_node);  in cgroup_migrate_finish()
2660 		put_css_set_locked(cset);  in cgroup_migrate_finish()
2663 	list_for_each_entry_safe(cset, tmp_cset, &mgctx->preloaded_dst_csets,  in cgroup_migrate_finish()
2665 		cset->mg_src_cgrp = NULL;  in cgroup_migrate_finish()
2666 		cset->mg_dst_cgrp = NULL;  in cgroup_migrate_finish()
2667 		cset->mg_dst_cset = NULL;  in cgroup_migrate_finish()
2668 		list_del_init(&cset->mg_dst_preload_node);  in cgroup_migrate_finish()
2669 		put_css_set_locked(cset);  in cgroup_migrate_finish()
3008 			cgroup_migrate_add_src(link->cset, dsct, &mgctx);  in cgroup_update_dfl_csses()
4731 	struct css_set *cset;  in css_task_iter_next_css_set()  local
4757 		cset = container_of(l, struct css_set, e_cset_node[it->ss->id]);  in css_task_iter_next_css_set()
4760 		cset = link->cset;  in css_task_iter_next_css_set()
4769 		it->cur_dcset = cset;  in css_task_iter_next_css_set()
4770 		get_css_set(cset);  in css_task_iter_next_css_set()
4772 		it->tcset_head = &cset->threaded_csets;  in css_task_iter_next_css_set()
4773 		it->tcset_pos = &cset->threaded_csets;  in css_task_iter_next_css_set()
4776 	return cset;  in css_task_iter_next_css_set()
4787 	struct css_set *cset;  in css_task_iter_advance_css_set()  local
4792 	while ((cset = css_task_iter_next_css_set(it))) {  in css_task_iter_advance_css_set()
4793 		if (!list_empty(&cset->tasks)) {  in css_task_iter_advance_css_set()
4794 			it->cur_tasks_head = &cset->tasks;  in css_task_iter_advance_css_set()
4796 		} else if (!list_empty(&cset->mg_tasks)) {  in css_task_iter_advance_css_set()
4797 			it->cur_tasks_head = &cset->mg_tasks;  in css_task_iter_advance_css_set()
4799 		} else if (!list_empty(&cset->dying_tasks)) {  in css_task_iter_advance_css_set()
4800 			it->cur_tasks_head = &cset->dying_tasks;  in css_task_iter_advance_css_set()
4804 	if (!cset) {  in css_task_iter_advance_css_set()
4829 	get_css_set(cset);  in css_task_iter_advance_css_set()
4830 	it->cur_cset = cset;  in css_task_iter_advance_css_set()
4831 	list_add(&it->iters_node, &cset->task_iters);  in css_task_iter_advance_css_set()
5903 		link->cset->dead = true;  in cgroup_destroy_locked()
6378 	struct css_set *cset;  in cgroup_css_set_fork()  local
6388 	cset = task_css_set(current);  in cgroup_css_set_fork()
6389 	get_css_set(cset);  in cgroup_css_set_fork()
6393 		kargs->cset = cset;  in cgroup_css_set_fork()
6439 	ret = cgroup_attach_permissions(cset->dfl_cgrp, dst_cgrp, sb,  in cgroup_css_set_fork()
6445 	kargs->cset = find_css_set(cset, dst_cgrp);  in cgroup_css_set_fork()
6446 	if (!kargs->cset) {  in cgroup_css_set_fork()
6451 	put_css_set(cset);  in cgroup_css_set_fork()
6463 	put_css_set(cset);  in cgroup_css_set_fork()
6464 	if (kargs->cset)  in cgroup_css_set_fork()
6465 		put_css_set(kargs->cset);  in cgroup_css_set_fork()
6480 	struct css_set *cset = kargs->cset;  in cgroup_css_set_put_fork()  local
6484 	if (cset) {  in cgroup_css_set_put_fork()
6485 		put_css_set(cset);  in cgroup_css_set_put_fork()
6486 		kargs->cset = NULL;  in cgroup_css_set_put_fork()
6519 		ret = ss->can_fork(child, kargs->cset);  in cgroup_can_fork()
6531 			ss->cancel_fork(child, kargs->cset);  in cgroup_can_fork()
6556 			ss->cancel_fork(child, kargs->cset);  in cgroup_cancel_fork()
6576 	struct css_set *cset;  in cgroup_post_fork()  local
6579 	cset = kargs->cset;  in cgroup_post_fork()
6580 	kargs->cset = NULL;  in cgroup_post_fork()
6589 			cgrp_flags = cset->dfl_cgrp->flags;  in cgroup_post_fork()
6592 		cset->nr_tasks++;  in cgroup_post_fork()
6593 		css_set_move_task(child, NULL, cset, false);  in cgroup_post_fork()
6595 		put_css_set(cset);  in cgroup_post_fork()
6596 		cset = NULL;  in cgroup_post_fork()
6642 		get_css_set(cset);  in cgroup_post_fork()
6643 		child->nsproxy->cgroup_ns->root_cset = cset;  in cgroup_post_fork()
6664 	struct css_set *cset;  in cgroup_exit()  local
6670 	cset = task_css_set(tsk);  in cgroup_exit()
6671 	css_set_move_task(tsk, cset, NULL, false);  in cgroup_exit()
6672 	list_add_tail(&tsk->cg_list, &cset->dying_tasks);  in cgroup_exit()
6673 	cset->nr_tasks--;  in cgroup_exit()
6708 	struct css_set *cset = task_css_set(task);  in cgroup_free()  local
6709 	put_css_set(cset);  in cgroup_free()
6951 		struct css_set *cset;  in cgroup_sk_alloc()  local
6953 		cset = task_css_set(current);  in cgroup_sk_alloc()
6954 		if (likely(cgroup_tryget(cset->dfl_cgrp))) {  in cgroup_sk_alloc()
6955 			cgroup = cset->dfl_cgrp;  in cgroup_sk_alloc()