36 #include <linux/percpu-rwsem.h>
48  * struct cpuhp_cpu_state - Per cpu hotplug state storage
58  * @node:	Remote CPU node; for multi-instance, do a
60  * @last:	For multi-instance rollback, remember how far we got
64  * @done_up:	Signal completion to the issuer of the task for cpu-up
65  * @done_down:	Signal completion to the issuer of the task for cpu-down
97 	STATIC_LOCKDEP_MAP_INIT("cpuhp_state-up", &cpuhp_state_up_map);
99 	STATIC_LOCKDEP_MAP_INIT("cpuhp_state-down", &cpuhp_state_down_map);
119  * struct cpuhp_step - Hotplug state machine step
130 		int		(*multi)(unsigned int cpu,  member
135 		int		(*multi)(unsigned int cpu,  member
155 	return bringup ? !step->startup.single : !step->teardown.single;  in cpuhp_step_empty()
159  * cpuhp_invoke_callback - Invoke the callbacks for a given state
163  * @node:	For multi-instance, do a single entry callback for install/remove
164  * @lastp:	For multi-instance rollback, remember how far we got
180 	if (st->fail == state) {  in cpuhp_invoke_callback()
181 		st->fail = CPUHP_INVALID;  in cpuhp_invoke_callback()
182 		return -EAGAIN;  in cpuhp_invoke_callback()
190 	if (!step->multi_instance) {  in cpuhp_invoke_callback()
192 		cb = bringup ? step->startup.single : step->teardown.single;  in cpuhp_invoke_callback()
194 		trace_cpuhp_enter(cpu, st->target, state, cb);  in cpuhp_invoke_callback()
196 		trace_cpuhp_exit(cpu, st->state, state, ret);  in cpuhp_invoke_callback()
199 	cbm = bringup ? step->startup.multi : step->teardown.multi;  in cpuhp_invoke_callback()
204 		trace_cpuhp_multi_enter(cpu, st->target, state, cbm, node);  in cpuhp_invoke_callback()
206 		trace_cpuhp_exit(cpu, st->state, state, ret);  in cpuhp_invoke_callback()
212 	hlist_for_each(node, &step->list) {  in cpuhp_invoke_callback()
216 		trace_cpuhp_multi_enter(cpu, st->target, state, cbm, node);  in cpuhp_invoke_callback()
218 		trace_cpuhp_exit(cpu, st->state, state, ret);  in cpuhp_invoke_callback()
233 	cbm = !bringup ? step->startup.multi : step->teardown.multi;  in cpuhp_invoke_callback()
237 	hlist_for_each(node, &step->list) {  in cpuhp_invoke_callback()
238 		if (!cnt--)  in cpuhp_invoke_callback()
241 		trace_cpuhp_multi_enter(cpu, st->target, state, cbm, node);  in cpuhp_invoke_callback()
243 		trace_cpuhp_exit(cpu, st->state, state, ret);  in cpuhp_invoke_callback()
264 	struct completion *done = bringup ? &st->done_up : &st->done_down;  in wait_for_ap_thread()
270 	struct completion *done = bringup ? &st->done_up : &st->done_down;  in complete_ap_thread()
294  * cpuhp_ap_update_sync_state - Update synchronization state during bringup/teardown
330 		} else if (now - start < NSEC_PER_MSEC) {  in cpuhp_wait_for_sync_state()
346  * cpuhp_ap_report_dead - Update synchronization state to DEAD
387  * cpuhp_ap_sync_alive - Synchronize AP with the control CPU once it is alive
446 		ret = -EIO;  in cpuhp_bp_sync_alive()
478  * If set, cpu_up and cpu_down will return -EBUSY and do nothing.
565 	cpu_hotplug_disabled--;  in __cpu_hotplug_enable()
589  * Architectures that need SMT-specific errata handling during SMT hotplug
702 	enum cpuhp_state prev_state = st->state;  in cpuhp_set_state()
703 	bool bringup = st->state < target;  in cpuhp_set_state()
705 	st->rollback = false;  in cpuhp_set_state()
706 	st->last = NULL;  in cpuhp_set_state()
708 	st->target = target;  in cpuhp_set_state()
709 	st->single = false;  in cpuhp_set_state()
710 	st->bringup = bringup;  in cpuhp_set_state()
721 	bool bringup = !st->bringup;  in cpuhp_reset_state()
723 	st->target = prev_state;  in cpuhp_reset_state()
729 	if (st->rollback)  in cpuhp_reset_state()
732 	st->rollback = true;  in cpuhp_reset_state()
735 	 * If we have st->last we need to undo partial multi_instance of this  in cpuhp_reset_state()
738 	if (!st->last) {  in cpuhp_reset_state()
739 		if (st->bringup)  in cpuhp_reset_state()
740 			st->state--;  in cpuhp_reset_state()
742 			st->state++;  in cpuhp_reset_state()
745 	st->bringup = bringup;  in cpuhp_reset_state()
753 	if (!st->single && st->state == st->target)  in __cpuhp_kick_ap()
756 	st->result = 0;  in __cpuhp_kick_ap()
762 	st->should_run = true;  in __cpuhp_kick_ap()
763 	wake_up_process(st->thread);  in __cpuhp_kick_ap()
764 	wait_for_ap_thread(st, st->bringup);  in __cpuhp_kick_ap()
775 	if ((ret = st->result)) {  in cpuhp_kick_ap()
790 		return -ECANCELED;  in bringup_wait_for_ap_online()
793 	kthread_unpark(st->thread);  in bringup_wait_for_ap_online()
803 		return -ECANCELED;  in bringup_wait_for_ap_online()
811 		return -EAGAIN;  in cpuhp_kick_ap_alive()
838 	if (st->target <= CPUHP_AP_ONLINE_IDLE)  in cpuhp_bringup_ap()
841 	return cpuhp_kick_ap(cpu, st, st->target);  in cpuhp_bringup_ap()
855 		return -EAGAIN;  in bringup_cpu()
882 	if (st->target <= CPUHP_AP_ONLINE_IDLE)  in bringup_cpu()
885 	return cpuhp_kick_ap(cpu, st, st->target);  in bringup_cpu()
896 	struct mm_struct *mm = idle->active_mm;  in finish_cpu()
903 		idle->active_mm = &init_mm;  in finish_cpu()
916  * st->state will be modified ahead of time, to match state_to_run, as if it
926 			if (st->state >= target)  in cpuhp_next_state()
929 			*state_to_run = ++st->state;  in cpuhp_next_state()
931 			if (st->state <= target)  in cpuhp_next_state()
934 			*state_to_run = st->state--;  in cpuhp_next_state()
963 				cpuhp_get_step(st->state)->name,  in __cpuhp_invoke_callback_range()
964 				st->state, err);  in __cpuhp_invoke_callback_range()
965 			ret = -1;  in __cpuhp_invoke_callback_range()
1002 	return st->state <= CPUHP_BRINGUP_CPU;  in can_rollback_cpu()
1008 	enum cpuhp_state prev_state = st->state;  in cpuhp_up_callbacks()
1014 			 ret, cpu, cpuhp_get_step(st->state)->name,  in cpuhp_up_callbacks()
1015 			 st->state);  in cpuhp_up_callbacks()
1032 	return st->should_run;  in cpuhp_should_run()
1043  *  - single: runs st->cb_state
1044  *  - up:     runs ++st->state, while st->state < st->target
1045  *  - down:   runs st->state--, while st->state > st->target
1052 	bool bringup = st->bringup;  in cpuhp_thread_fun()
1055 	if (WARN_ON_ONCE(!st->should_run))  in cpuhp_thread_fun()
1059 	 * ACQUIRE for the cpuhp_should_run() load of ->should_run. Ensures  in cpuhp_thread_fun()
1060 	 * that if we see ->should_run we also see the rest of the state.  in cpuhp_thread_fun()
1072 	if (st->single) {  in cpuhp_thread_fun()
1073 		state = st->cb_state;  in cpuhp_thread_fun()
1074 		st->should_run = false;  in cpuhp_thread_fun()
1076 		st->should_run = cpuhp_next_state(bringup, &state, st, st->target);  in cpuhp_thread_fun()
1077 		if (!st->should_run)  in cpuhp_thread_fun()
1085 		st->result = cpuhp_invoke_callback(cpu, state, bringup, st->node, &st->last);  in cpuhp_thread_fun()
1091 		WARN_ON_ONCE(st->result);  in cpuhp_thread_fun()
1093 		st->result = cpuhp_invoke_callback(cpu, state, bringup, st->node, &st->last);  in cpuhp_thread_fun()
1096 	if (st->result) {  in cpuhp_thread_fun()
1102 		WARN_ON_ONCE(st->rollback);  in cpuhp_thread_fun()
1103 		st->should_run = false;  in cpuhp_thread_fun()
1110 	if (!st->should_run)  in cpuhp_thread_fun()
1135 	if (!st->thread)  in cpuhp_invoke_ap_callback()
1138 	st->rollback = false;  in cpuhp_invoke_ap_callback()
1139 	st->last = NULL;  in cpuhp_invoke_ap_callback()
1141 	st->node = node;  in cpuhp_invoke_ap_callback()
1142 	st->bringup = bringup;  in cpuhp_invoke_ap_callback()
1143 	st->cb_state = state;  in cpuhp_invoke_ap_callback()
1144 	st->single = true;  in cpuhp_invoke_ap_callback()
1151 	if ((ret = st->result) && st->last) {  in cpuhp_invoke_ap_callback()
1152 		st->rollback = true;  in cpuhp_invoke_ap_callback()
1153 		st->bringup = !bringup;  in cpuhp_invoke_ap_callback()
1162 	st->node = st->last = NULL;  in cpuhp_invoke_ap_callback()
1169 	enum cpuhp_state prev_state = st->state;  in cpuhp_kick_ap_work()
1178 	trace_cpuhp_enter(cpu, st->target, prev_state, cpuhp_kick_ap_work);  in cpuhp_kick_ap_work()
1179 	ret = cpuhp_kick_ap(cpu, st, st->target);  in cpuhp_kick_ap_work()
1180 	trace_cpuhp_exit(cpu, st->state, prev_state, ret);  in cpuhp_kick_ap_work()
1200 		init_completion(&st->done_up);  in cpuhp_init_state()
1201 		init_completion(&st->done_down);  in cpuhp_init_state()
1264  * clear_tasks_mm_cpumask - Safely clear tasks' mm_cpumask for a CPU
1269  * trivial, there are various non-obvious corner cases, which this function
1282 	 * their mm mask. -- Peter Zijlstra  in clear_tasks_mm_cpumask()
1284 	 * full-fledged tasklist_lock.  in clear_tasks_mm_cpumask()
1298 		arch_clear_mm_cpumask_cpu(cpu, t->mm);  in clear_tasks_mm_cpumask()
1308 	enum cpuhp_state target = max((int)st->target, CPUHP_AP_OFFLINE);  in take_cpu_down()
1318 	 * down, that the current state is CPUHP_TEARDOWN_CPU - 1.  in take_cpu_down()
1320 	WARN_ON(st->state != (CPUHP_TEARDOWN_CPU - 1));  in take_cpu_down()
1342 	kthread_park(st->thread);  in takedown_cpu()
1358 		kthread_unpark(st->thread);  in takedown_cpu()
1371 	BUG_ON(st->state != CPUHP_AP_IDLE_DEAD);  in takedown_cpu()
1398 	BUG_ON(st->state != CPUHP_AP_OFFLINE);  in cpuhp_report_idle_dead()
1400 	st->state = CPUHP_AP_IDLE_DEAD;  in cpuhp_report_idle_dead()
1412 	enum cpuhp_state prev_state = st->state;  in cpuhp_down_callbacks()
1418 			 ret, cpu, cpuhp_get_step(st->state)->name,  in cpuhp_down_callbacks()
1419 			 st->state);  in cpuhp_down_callbacks()
1423 		if (st->state < prev_state)  in cpuhp_down_callbacks()
1439 		return -EBUSY;  in _cpu_down()
1442 		return -EINVAL;  in _cpu_down()
1453 	if (st->state > CPUHP_TEARDOWN_CPU) {  in _cpu_down()
1454 		st->target = max((int)target, CPUHP_TEARDOWN_CPU);  in _cpu_down()
1467 		if (st->state > CPUHP_TEARDOWN_CPU)  in _cpu_down()
1470 		st->target = target;  in _cpu_down()
1477 	if (ret && st->state < prev_state) {  in _cpu_down()
1478 		if (st->state == CPUHP_TEARDOWN_CPU) {  in _cpu_down()
1507 	return _cpu_down(work->cpu, 0, work->target);  in __cpu_down_maps_locked()
1519 		return -EOPNOTSUPP;  in cpu_down_maps_locked()
1521 		return -EBUSY;  in cpu_down_maps_locked()
1525 	 * CPU to prevent a deadlock against cfs_b->period_timer.  in cpu_down_maps_locked()
1533 	return -EBUSY;  in cpu_down_maps_locked()
1547  * cpu_device_down - Bring down a cpu device
1558 	return cpu_down(dev->id, CPUHP_OFFLINE);  in cpu_device_down()
1594 			pr_err("Failed to offline CPU%d - error=%d",  in smp_shutdown_nonboot_cpus()
1620  * notify_cpu_starting(cpu) - Invoke the callbacks on the starting CPU
1629 	enum cpuhp_state target = min((int)st->target, CPUHP_AP_ONLINE);  in notify_cpu_starting()
1661 	st->state = CPUHP_AP_ONLINE_IDLE;  in cpuhp_online_idle()
1675 		ret = -EINVAL;  in _cpu_up()
1683 	if (st->state >= target)  in _cpu_up()
1686 	if (st->state == CPUHP_OFFLINE) {  in _cpu_up()
1708 	if (st->state > CPUHP_BRINGUP_CPU) {  in _cpu_up()
1737 		pr_err("can't online cpu %d because it is not configured as may-hotadd at boot time\n",  in cpu_up()
1742 		return -EINVAL;  in cpu_up()
1752 		err = -EBUSY;  in cpu_up()
1756 		err = -EPERM;  in cpu_up()
1767  * cpu_device_up - Bring up a cpu device
1778 	return cpu_up(dev->id, CPUHP_ONLINE);  in cpu_device_up()
1794  * bringup_hibernate_cpu - Bring up the CPU that we hibernated on
1811 			pr_err("Failed to bring hibernate-CPU up!\n");  in bringup_hibernate_cpu()
1835 		if (!--ncpus)  in cpuhp_bringup_mask()
1891 		ncpus -= num_online_cpus();  in cpuhp_bringup_cpus_parallel()
1899 	/* Bring the not-yet started CPUs up */  in cpuhp_bringup_cpus_parallel()
1929 	if (primary == -1) {  in freeze_secondary_cpus()
1939 	 * We take down all of the non-boot CPUs in one shot to avoid races  in freeze_secondary_cpus()
1944 	pr_info("Disabling non-boot CPUs ...\n");  in freeze_secondary_cpus()
1951 			error = -EBUSY;  in freeze_secondary_cpus()
1969 		pr_err("Non-boot CPUs are not disabled\n");  in freeze_secondary_cpus()
2000 	pr_info("Enabling non-boot CPUs ...\n");  in thaw_secondary_cpus()
2025 		return -ENOMEM;  in alloc_frozen_cpus()
2138 	 * On the tear-down path, timers_dead_cpu() must be invoked
2170 	 * All-in-one CPU bringup state which includes the kick alive.
2297 		return -EINVAL;  in cpuhp_cb_check()
2321 		return -EINVAL;  in cpuhp_reserve_state()
2325 		if (!step->name)  in cpuhp_reserve_state()
2329 	return -ENOSPC;  in cpuhp_reserve_state()
2358 	if (name && sp->name)  in cpuhp_store_callbacks()
2359 		return -EBUSY;  in cpuhp_store_callbacks()
2361 	sp->startup.single = startup;  in cpuhp_store_callbacks()
2362 	sp->teardown.single = teardown;  in cpuhp_store_callbacks()
2363 	sp->name = name;  in cpuhp_store_callbacks()
2364 	sp->multi_instance = multi_instance;  in cpuhp_store_callbacks()
2365 	INIT_HLIST_HEAD(&sp->list);  in cpuhp_store_callbacks()
2371 	return cpuhp_get_step(state)->teardown.single;  in cpuhp_get_teardown_cb()
2419 		int cpustate = st->state;  in cpuhp_rollback_install()
2441 	if (sp->multi_instance == false)  in __cpuhp_state_add_instance_cpuslocked()
2442 		return -EINVAL;  in __cpuhp_state_add_instance_cpuslocked()
2446 	if (!invoke || !sp->startup.multi)  in __cpuhp_state_add_instance_cpuslocked()
2455 		int cpustate = st->state;  in __cpuhp_state_add_instance_cpuslocked()
2462 			if (sp->teardown.multi)  in __cpuhp_state_add_instance_cpuslocked()
2469 	hlist_add_head(node, &sp->list);  in __cpuhp_state_add_instance_cpuslocked()
2488  * __cpuhp_setup_state_cpuslocked - Setup the callbacks for an hotplug machine state
2517 		return -EINVAL;  in __cpuhp_setup_state_cpuslocked()
2539 		int cpustate = st->state;  in __cpuhp_setup_state_cpuslocked()
2588 	if (!sp->multi_instance)  in __cpuhp_state_remove_instance()
2589 		return -EINVAL;  in __cpuhp_state_remove_instance()
2603 		int cpustate = st->state;  in __cpuhp_state_remove_instance()
2619  * __cpuhp_remove_state_cpuslocked - Remove the callbacks for an hotplug machine state
2638 	if (sp->multi_instance) {  in __cpuhp_remove_state_cpuslocked()
2639 		WARN(!hlist_empty(&sp->list),  in __cpuhp_remove_state_cpuslocked()
2655 		int cpustate = st->state;  in __cpuhp_remove_state_cpuslocked()
2679 	dev->offline = true;  in cpuhp_offline_cpu_device()
2681 	kobject_uevent(&dev->kobj, KOBJ_OFFLINE);  in cpuhp_offline_cpu_device()
2688 	dev->offline = false;  in cpuhp_online_cpu_device()
2690 	kobject_uevent(&dev->kobj, KOBJ_ONLINE);  in cpuhp_online_cpu_device()
2766 			  struct device_attribute *attr, char *buf)  in state_show()  argument
2768 	struct cpuhp_cpu_state *st = per_cpu_ptr(&cpuhp_state, dev->id);  in state_show()
2770 	return sprintf(buf, "%d\n", st->state);  in state_show()
2774 static ssize_t target_store(struct device *dev, struct device_attribute *attr,  in target_store()  argument
2777 	struct cpuhp_cpu_state *st = per_cpu_ptr(&cpuhp_state, dev->id);  in target_store()
2787 		return -EINVAL;  in target_store()
2790 		return -EINVAL;  in target_store()
2799 	ret = !sp->name || sp->cant_stop ? -EINVAL : 0;  in target_store()
2804 	if (st->state < target)  in target_store()
2805 		ret = cpu_up(dev->id, target);  in target_store()
2806 	else if (st->state > target)  in target_store()
2807 		ret = cpu_down(dev->id, target);  in target_store()
2808 	else if (WARN_ON(st->target != target))  in target_store()
2809 		st->target = target;  in target_store()
2816 			   struct device_attribute *attr, char *buf)  in target_show()  argument
2818 	struct cpuhp_cpu_state *st = per_cpu_ptr(&cpuhp_state, dev->id);  in target_show()
2820 	return sprintf(buf, "%d\n", st->target);  in target_show()
2824 static ssize_t fail_store(struct device *dev, struct device_attribute *attr,  in fail_store()  argument
2827 	struct cpuhp_cpu_state *st = per_cpu_ptr(&cpuhp_state, dev->id);  in fail_store()
2836 		st->fail = fail;  in fail_store()
2841 		return -EINVAL;  in fail_store()
2847 		return -EINVAL;  in fail_store()
2855 	if (fail <= CPUHP_BRINGUP_CPU && st->state > CPUHP_BRINGUP_CPU)  in fail_store()
2856 		return -EINVAL;  in fail_store()
2863 	if (!sp->startup.single && !sp->teardown.single)  in fail_store()
2864 		ret = -EINVAL;  in fail_store()
2869 	st->fail = fail;  in fail_store()
2875 			 struct device_attribute *attr, char *buf)  in fail_show()  argument
2877 	struct cpuhp_cpu_state *st = per_cpu_ptr(&cpuhp_state, dev->id);  in fail_show()
2879 	return sprintf(buf, "%d\n", st->fail);  in fail_show()
2885 	&dev_attr_state.attr,
2886 	&dev_attr_target.attr,
2887 	&dev_attr_fail.attr,
2898 				 struct device_attribute *attr, char *buf)  in states_show()  argument
2907 		if (sp->name) {  in states_show()
2908 			cur = sprintf(buf, "%3d: %s\n", i, sp->name);  in states_show()
2919 	&dev_attr_states.attr,
2939 __store_smt_control(struct device *dev, struct device_attribute *attr,  in __store_smt_control()  argument
2946 		return -EPERM;  in __store_smt_control()
2949 		return -ENODEV;  in __store_smt_control()
2966 			return -EINVAL;  in __store_smt_control()
2968 		return -EINVAL;  in __store_smt_control()
2991 __store_smt_control(struct device *dev, struct device_attribute *attr,  in __store_smt_control()  argument
2994 	return -ENODEV;  in __store_smt_control()
3007 			    struct device_attribute *attr, char *buf)  in control_show()  argument
3022 	return snprintf(buf, PAGE_SIZE - 2, "%s\n", state);  in control_show()
3025 static ssize_t control_store(struct device *dev, struct device_attribute *attr,  in control_store()  argument
3028 	return __store_smt_control(dev, attr, buf, count);  in control_store()
3033 			   struct device_attribute *attr, char *buf)  in active_show()  argument
3035 	return snprintf(buf, PAGE_SIZE - 2, "%d\n", sched_smt_active());  in active_show()
3040 	&dev_attr_control.attr,
3041 	&dev_attr_active.attr,
3054 	int ret = -ENODEV;  in cpu_smt_sysfs_init()
3058 		ret = sysfs_create_group(&dev_root->kobj, &cpuhp_smt_attr_group);  in cpu_smt_sysfs_init()
3075 		ret = sysfs_create_group(&dev_root->kobj, &cpuhp_cpu_root_attr_group);  in cpuhp_sysfs_init()
3086 		ret = sysfs_create_group(&dev->kobj, &cpuhp_cpu_attr_group);  in cpuhp_sysfs_init()
3103 /* cpu_bit_bitmap[0] is empty - so we can back into it */