xref: /openbmc/linux/kernel/cpu.c (revision ec5c05e5ac8bcb4a6bcd92970e15494a85400d34)
1  /* CPU control.
2   * (C) 2001, 2002, 2003, 2004 Rusty Russell
3   *
4   * This code is licenced under the GPL.
5   */
6  #include <linux/sched/mm.h>
7  #include <linux/proc_fs.h>
8  #include <linux/smp.h>
9  #include <linux/init.h>
10  #include <linux/notifier.h>
11  #include <linux/sched/signal.h>
12  #include <linux/sched/hotplug.h>
13  #include <linux/sched/isolation.h>
14  #include <linux/sched/task.h>
15  #include <linux/sched/smt.h>
16  #include <linux/unistd.h>
17  #include <linux/cpu.h>
18  #include <linux/oom.h>
19  #include <linux/rcupdate.h>
20  #include <linux/export.h>
21  #include <linux/bug.h>
22  #include <linux/kthread.h>
23  #include <linux/stop_machine.h>
24  #include <linux/mutex.h>
25  #include <linux/gfp.h>
26  #include <linux/suspend.h>
27  #include <linux/lockdep.h>
28  #include <linux/tick.h>
29  #include <linux/irq.h>
30  #include <linux/nmi.h>
31  #include <linux/smpboot.h>
32  #include <linux/relay.h>
33  #include <linux/slab.h>
34  #include <linux/scs.h>
35  #include <linux/percpu-rwsem.h>
36  #include <linux/cpuset.h>
37  #include <linux/random.h>
38  #include <linux/cc_platform.h>
39  
40  #include <trace/events/power.h>
41  #define CREATE_TRACE_POINTS
42  #include <trace/events/cpuhp.h>
43  
44  #include "smpboot.h"
45  
46  /**
47   * struct cpuhp_cpu_state - Per cpu hotplug state storage
48   * @state:	The current cpu state
49   * @target:	The target state
50   * @fail:	Current CPU hotplug callback state
51   * @thread:	Pointer to the hotplug thread
52   * @should_run:	Thread should execute
53   * @rollback:	Perform a rollback
54   * @single:	Single callback invocation
55   * @bringup:	Single callback bringup or teardown selector
56   * @cpu:	CPU number
57   * @node:	Remote CPU node; for multi-instance, do a
58   *		single entry callback for install/remove
59   * @last:	For multi-instance rollback, remember how far we got
60   * @cb_state:	The state for a single callback (install/uninstall)
61   * @result:	Result of the operation
62   * @done_up:	Signal completion to the issuer of the task for cpu-up
63   * @done_down:	Signal completion to the issuer of the task for cpu-down
64   */
65  struct cpuhp_cpu_state {
66  	enum cpuhp_state	state;
67  	enum cpuhp_state	target;
68  	enum cpuhp_state	fail;
69  #ifdef CONFIG_SMP
70  	struct task_struct	*thread;
71  	bool			should_run;
72  	bool			rollback;
73  	bool			single;
74  	bool			bringup;
75  	struct hlist_node	*node;
76  	struct hlist_node	*last;
77  	enum cpuhp_state	cb_state;
78  	int			result;
79  	struct completion	done_up;
80  	struct completion	done_down;
81  #endif
82  };
83  
84  static DEFINE_PER_CPU(struct cpuhp_cpu_state, cpuhp_state) = {
85  	.fail = CPUHP_INVALID,
86  };
87  
88  #ifdef CONFIG_SMP
89  cpumask_t cpus_booted_once_mask;
90  #endif
91  
92  #if defined(CONFIG_LOCKDEP) && defined(CONFIG_SMP)
93  static struct lockdep_map cpuhp_state_up_map =
94  	STATIC_LOCKDEP_MAP_INIT("cpuhp_state-up", &cpuhp_state_up_map);
95  static struct lockdep_map cpuhp_state_down_map =
96  	STATIC_LOCKDEP_MAP_INIT("cpuhp_state-down", &cpuhp_state_down_map);
97  
98  
99  static inline void cpuhp_lock_acquire(bool bringup)
100  {
101  	lock_map_acquire(bringup ? &cpuhp_state_up_map : &cpuhp_state_down_map);
102  }
103  
104  static inline void cpuhp_lock_release(bool bringup)
105  {
106  	lock_map_release(bringup ? &cpuhp_state_up_map : &cpuhp_state_down_map);
107  }
108  #else
109  
110  static inline void cpuhp_lock_acquire(bool bringup) { }
111  static inline void cpuhp_lock_release(bool bringup) { }
112  
113  #endif
114  
115  /**
116   * struct cpuhp_step - Hotplug state machine step
117   * @name:	Name of the step
118   * @startup:	Startup function of the step
119   * @teardown:	Teardown function of the step
120   * @cant_stop:	Bringup/teardown can't be stopped at this step
121   * @multi_instance:	State has multiple instances which get added afterwards
122   */
123  struct cpuhp_step {
124  	const char		*name;
125  	union {
126  		int		(*single)(unsigned int cpu);
127  		int		(*multi)(unsigned int cpu,
128  					 struct hlist_node *node);
129  	} startup;
130  	union {
131  		int		(*single)(unsigned int cpu);
132  		int		(*multi)(unsigned int cpu,
133  					 struct hlist_node *node);
134  	} teardown;
135  	/* private: */
136  	struct hlist_head	list;
137  	/* public: */
138  	bool			cant_stop;
139  	bool			multi_instance;
140  };
141  
142  static DEFINE_MUTEX(cpuhp_state_mutex);
143  static struct cpuhp_step cpuhp_hp_states[];
144  
145  static struct cpuhp_step *cpuhp_get_step(enum cpuhp_state state)
146  {
147  	return cpuhp_hp_states + state;
148  }
149  
150  static bool cpuhp_step_empty(bool bringup, struct cpuhp_step *step)
151  {
152  	return bringup ? !step->startup.single : !step->teardown.single;
153  }
154  
155  /**
156   * cpuhp_invoke_callback - Invoke the callbacks for a given state
157   * @cpu:	The cpu for which the callback should be invoked
158   * @state:	The state to do callbacks for
159   * @bringup:	True if the bringup callback should be invoked
160   * @node:	For multi-instance, do a single entry callback for install/remove
161   * @lastp:	For multi-instance rollback, remember how far we got
162   *
163   * Called from cpu hotplug and from the state register machinery.
164   *
165   * Return: %0 on success or a negative errno code
166   */
167  static int cpuhp_invoke_callback(unsigned int cpu, enum cpuhp_state state,
168  				 bool bringup, struct hlist_node *node,
169  				 struct hlist_node **lastp)
170  {
171  	struct cpuhp_cpu_state *st = per_cpu_ptr(&cpuhp_state, cpu);
172  	struct cpuhp_step *step = cpuhp_get_step(state);
173  	int (*cbm)(unsigned int cpu, struct hlist_node *node);
174  	int (*cb)(unsigned int cpu);
175  	int ret, cnt;
176  
177  	if (st->fail == state) {
178  		st->fail = CPUHP_INVALID;
179  		return -EAGAIN;
180  	}
181  
182  	if (cpuhp_step_empty(bringup, step)) {
183  		WARN_ON_ONCE(1);
184  		return 0;
185  	}
186  
187  	if (!step->multi_instance) {
188  		WARN_ON_ONCE(lastp && *lastp);
189  		cb = bringup ? step->startup.single : step->teardown.single;
190  
191  		trace_cpuhp_enter(cpu, st->target, state, cb);
192  		ret = cb(cpu);
193  		trace_cpuhp_exit(cpu, st->state, state, ret);
194  		return ret;
195  	}
196  	cbm = bringup ? step->startup.multi : step->teardown.multi;
197  
198  	/* Single invocation for instance add/remove */
199  	if (node) {
200  		WARN_ON_ONCE(lastp && *lastp);
201  		trace_cpuhp_multi_enter(cpu, st->target, state, cbm, node);
202  		ret = cbm(cpu, node);
203  		trace_cpuhp_exit(cpu, st->state, state, ret);
204  		return ret;
205  	}
206  
207  	/* State transition. Invoke on all instances */
208  	cnt = 0;
209  	hlist_for_each(node, &step->list) {
210  		if (lastp && node == *lastp)
211  			break;
212  
213  		trace_cpuhp_multi_enter(cpu, st->target, state, cbm, node);
214  		ret = cbm(cpu, node);
215  		trace_cpuhp_exit(cpu, st->state, state, ret);
216  		if (ret) {
217  			if (!lastp)
218  				goto err;
219  
220  			*lastp = node;
221  			return ret;
222  		}
223  		cnt++;
224  	}
225  	if (lastp)
226  		*lastp = NULL;
227  	return 0;
228  err:
229  	/* Rollback the instances if one failed */
230  	cbm = !bringup ? step->startup.multi : step->teardown.multi;
231  	if (!cbm)
232  		return ret;
233  
234  	hlist_for_each(node, &step->list) {
235  		if (!cnt--)
236  			break;
237  
238  		trace_cpuhp_multi_enter(cpu, st->target, state, cbm, node);
239  		ret = cbm(cpu, node);
240  		trace_cpuhp_exit(cpu, st->state, state, ret);
241  		/*
242  		 * Rollback must not fail,
243  		 */
244  		WARN_ON_ONCE(ret);
245  	}
246  	return ret;
247  }
248  
249  #ifdef CONFIG_SMP
250  static bool cpuhp_is_ap_state(enum cpuhp_state state)
251  {
252  	/*
253  	 * The extra check for CPUHP_TEARDOWN_CPU is only for documentation
254  	 * purposes as that state is handled explicitly in cpu_down.
255  	 */
256  	return state > CPUHP_BRINGUP_CPU && state != CPUHP_TEARDOWN_CPU;
257  }
258  
259  static inline void wait_for_ap_thread(struct cpuhp_cpu_state *st, bool bringup)
260  {
261  	struct completion *done = bringup ? &st->done_up : &st->done_down;
262  	wait_for_completion(done);
263  }
264  
265  static inline void complete_ap_thread(struct cpuhp_cpu_state *st, bool bringup)
266  {
267  	struct completion *done = bringup ? &st->done_up : &st->done_down;
268  	complete(done);
269  }
270  
271  /*
272   * The former STARTING/DYING states, ran with IRQs disabled and must not fail.
273   */
274  static bool cpuhp_is_atomic_state(enum cpuhp_state state)
275  {
276  	return CPUHP_AP_IDLE_DEAD <= state && state < CPUHP_AP_ONLINE;
277  }
278  
279  /* Serializes the updates to cpu_online_mask, cpu_present_mask */
280  static DEFINE_MUTEX(cpu_add_remove_lock);
281  bool cpuhp_tasks_frozen;
282  EXPORT_SYMBOL_GPL(cpuhp_tasks_frozen);
283  
284  /*
285   * The following two APIs (cpu_maps_update_begin/done) must be used when
286   * attempting to serialize the updates to cpu_online_mask & cpu_present_mask.
287   */
288  void cpu_maps_update_begin(void)
289  {
290  	mutex_lock(&cpu_add_remove_lock);
291  }
292  
293  void cpu_maps_update_done(void)
294  {
295  	mutex_unlock(&cpu_add_remove_lock);
296  }
297  
298  /*
299   * If set, cpu_up and cpu_down will return -EBUSY and do nothing.
300   * Should always be manipulated under cpu_add_remove_lock
301   */
302  static int cpu_hotplug_disabled;
303  
304  #ifdef CONFIG_HOTPLUG_CPU
305  
306  DEFINE_STATIC_PERCPU_RWSEM(cpu_hotplug_lock);
307  
308  void cpus_read_lock(void)
309  {
310  	percpu_down_read(&cpu_hotplug_lock);
311  }
312  EXPORT_SYMBOL_GPL(cpus_read_lock);
313  
314  int cpus_read_trylock(void)
315  {
316  	return percpu_down_read_trylock(&cpu_hotplug_lock);
317  }
318  EXPORT_SYMBOL_GPL(cpus_read_trylock);
319  
320  void cpus_read_unlock(void)
321  {
322  	percpu_up_read(&cpu_hotplug_lock);
323  }
324  EXPORT_SYMBOL_GPL(cpus_read_unlock);
325  
326  void cpus_write_lock(void)
327  {
328  	percpu_down_write(&cpu_hotplug_lock);
329  }
330  
331  void cpus_write_unlock(void)
332  {
333  	percpu_up_write(&cpu_hotplug_lock);
334  }
335  
336  void lockdep_assert_cpus_held(void)
337  {
338  	/*
339  	 * We can't have hotplug operations before userspace starts running,
340  	 * and some init codepaths will knowingly not take the hotplug lock.
341  	 * This is all valid, so mute lockdep until it makes sense to report
342  	 * unheld locks.
343  	 */
344  	if (system_state < SYSTEM_RUNNING)
345  		return;
346  
347  	percpu_rwsem_assert_held(&cpu_hotplug_lock);
348  }
349  
350  #ifdef CONFIG_LOCKDEP
351  int lockdep_is_cpus_held(void)
352  {
353  	return percpu_rwsem_is_held(&cpu_hotplug_lock);
354  }
355  #endif
356  
357  static void lockdep_acquire_cpus_lock(void)
358  {
359  	rwsem_acquire(&cpu_hotplug_lock.dep_map, 0, 0, _THIS_IP_);
360  }
361  
362  static void lockdep_release_cpus_lock(void)
363  {
364  	rwsem_release(&cpu_hotplug_lock.dep_map, _THIS_IP_);
365  }
366  
367  /*
368   * Wait for currently running CPU hotplug operations to complete (if any) and
369   * disable future CPU hotplug (from sysfs). The 'cpu_add_remove_lock' protects
370   * the 'cpu_hotplug_disabled' flag. The same lock is also acquired by the
371   * hotplug path before performing hotplug operations. So acquiring that lock
372   * guarantees mutual exclusion from any currently running hotplug operations.
373   */
374  void cpu_hotplug_disable(void)
375  {
376  	cpu_maps_update_begin();
377  	cpu_hotplug_disabled++;
378  	cpu_maps_update_done();
379  }
380  EXPORT_SYMBOL_GPL(cpu_hotplug_disable);
381  
382  static void __cpu_hotplug_enable(void)
383  {
384  	if (WARN_ONCE(!cpu_hotplug_disabled, "Unbalanced cpu hotplug enable\n"))
385  		return;
386  	cpu_hotplug_disabled--;
387  }
388  
389  void cpu_hotplug_enable(void)
390  {
391  	cpu_maps_update_begin();
392  	__cpu_hotplug_enable();
393  	cpu_maps_update_done();
394  }
395  EXPORT_SYMBOL_GPL(cpu_hotplug_enable);
396  
397  #else
398  
399  static void lockdep_acquire_cpus_lock(void)
400  {
401  }
402  
403  static void lockdep_release_cpus_lock(void)
404  {
405  }
406  
407  #endif	/* CONFIG_HOTPLUG_CPU */
408  
409  /*
410   * Architectures that need SMT-specific errata handling during SMT hotplug
411   * should override this.
412   */
413  void __weak arch_smt_update(void) { }
414  
415  #ifdef CONFIG_HOTPLUG_SMT
416  enum cpuhp_smt_control cpu_smt_control __read_mostly = CPU_SMT_ENABLED;
417  
418  void __init cpu_smt_disable(bool force)
419  {
420  	if (!cpu_smt_possible())
421  		return;
422  
423  	if (force) {
424  		pr_info("SMT: Force disabled\n");
425  		cpu_smt_control = CPU_SMT_FORCE_DISABLED;
426  	} else {
427  		pr_info("SMT: disabled\n");
428  		cpu_smt_control = CPU_SMT_DISABLED;
429  	}
430  }
431  
432  /*
433   * The decision whether SMT is supported can only be done after the full
434   * CPU identification. Called from architecture code.
435   */
436  void __init cpu_smt_check_topology(void)
437  {
438  	if (!topology_smt_supported())
439  		cpu_smt_control = CPU_SMT_NOT_SUPPORTED;
440  }
441  
442  static int __init smt_cmdline_disable(char *str)
443  {
444  	cpu_smt_disable(str && !strcmp(str, "force"));
445  	return 0;
446  }
447  early_param("nosmt", smt_cmdline_disable);
448  
449  static inline bool cpu_smt_allowed(unsigned int cpu)
450  {
451  	if (cpu_smt_control == CPU_SMT_ENABLED)
452  		return true;
453  
454  	if (topology_is_primary_thread(cpu))
455  		return true;
456  
457  	/*
458  	 * On x86 it's required to boot all logical CPUs at least once so
459  	 * that the init code can get a chance to set CR4.MCE on each
460  	 * CPU. Otherwise, a broadcasted MCE observing CR4.MCE=0b on any
461  	 * core will shutdown the machine.
462  	 */
463  	return !cpumask_test_cpu(cpu, &cpus_booted_once_mask);
464  }
465  
466  /* Returns true if SMT is not supported of forcefully (irreversibly) disabled */
467  bool cpu_smt_possible(void)
468  {
469  	return cpu_smt_control != CPU_SMT_FORCE_DISABLED &&
470  		cpu_smt_control != CPU_SMT_NOT_SUPPORTED;
471  }
472  EXPORT_SYMBOL_GPL(cpu_smt_possible);
473  #else
474  static inline bool cpu_smt_allowed(unsigned int cpu) { return true; }
475  #endif
476  
477  static inline enum cpuhp_state
478  cpuhp_set_state(int cpu, struct cpuhp_cpu_state *st, enum cpuhp_state target)
479  {
480  	enum cpuhp_state prev_state = st->state;
481  	bool bringup = st->state < target;
482  
483  	st->rollback = false;
484  	st->last = NULL;
485  
486  	st->target = target;
487  	st->single = false;
488  	st->bringup = bringup;
489  	if (cpu_dying(cpu) != !bringup)
490  		set_cpu_dying(cpu, !bringup);
491  
492  	return prev_state;
493  }
494  
495  static inline void
496  cpuhp_reset_state(int cpu, struct cpuhp_cpu_state *st,
497  		  enum cpuhp_state prev_state)
498  {
499  	bool bringup = !st->bringup;
500  
501  	st->target = prev_state;
502  
503  	/*
504  	 * Already rolling back. No need invert the bringup value or to change
505  	 * the current state.
506  	 */
507  	if (st->rollback)
508  		return;
509  
510  	st->rollback = true;
511  
512  	/*
513  	 * If we have st->last we need to undo partial multi_instance of this
514  	 * state first. Otherwise start undo at the previous state.
515  	 */
516  	if (!st->last) {
517  		if (st->bringup)
518  			st->state--;
519  		else
520  			st->state++;
521  	}
522  
523  	st->bringup = bringup;
524  	if (cpu_dying(cpu) != !bringup)
525  		set_cpu_dying(cpu, !bringup);
526  }
527  
528  /* Regular hotplug invocation of the AP hotplug thread */
529  static void __cpuhp_kick_ap(struct cpuhp_cpu_state *st)
530  {
531  	if (!st->single && st->state == st->target)
532  		return;
533  
534  	st->result = 0;
535  	/*
536  	 * Make sure the above stores are visible before should_run becomes
537  	 * true. Paired with the mb() above in cpuhp_thread_fun()
538  	 */
539  	smp_mb();
540  	st->should_run = true;
541  	wake_up_process(st->thread);
542  	wait_for_ap_thread(st, st->bringup);
543  }
544  
545  static int cpuhp_kick_ap(int cpu, struct cpuhp_cpu_state *st,
546  			 enum cpuhp_state target)
547  {
548  	enum cpuhp_state prev_state;
549  	int ret;
550  
551  	prev_state = cpuhp_set_state(cpu, st, target);
552  	__cpuhp_kick_ap(st);
553  	if ((ret = st->result)) {
554  		cpuhp_reset_state(cpu, st, prev_state);
555  		__cpuhp_kick_ap(st);
556  	}
557  
558  	return ret;
559  }
560  
561  static int bringup_wait_for_ap(unsigned int cpu)
562  {
563  	struct cpuhp_cpu_state *st = per_cpu_ptr(&cpuhp_state, cpu);
564  
565  	/* Wait for the CPU to reach CPUHP_AP_ONLINE_IDLE */
566  	wait_for_ap_thread(st, true);
567  	if (WARN_ON_ONCE((!cpu_online(cpu))))
568  		return -ECANCELED;
569  
570  	/* Unpark the hotplug thread of the target cpu */
571  	kthread_unpark(st->thread);
572  
573  	/*
574  	 * SMT soft disabling on X86 requires to bring the CPU out of the
575  	 * BIOS 'wait for SIPI' state in order to set the CR4.MCE bit.  The
576  	 * CPU marked itself as booted_once in notify_cpu_starting() so the
577  	 * cpu_smt_allowed() check will now return false if this is not the
578  	 * primary sibling.
579  	 */
580  	if (!cpu_smt_allowed(cpu))
581  		return -ECANCELED;
582  
583  	if (st->target <= CPUHP_AP_ONLINE_IDLE)
584  		return 0;
585  
586  	return cpuhp_kick_ap(cpu, st, st->target);
587  }
588  
589  static int bringup_cpu(unsigned int cpu)
590  {
591  	struct task_struct *idle = idle_thread_get(cpu);
592  	int ret;
593  
594  	/*
595  	 * Reset stale stack state from the last time this CPU was online.
596  	 */
597  	scs_task_reset(idle);
598  	kasan_unpoison_task_stack(idle);
599  
600  	/*
601  	 * Some architectures have to walk the irq descriptors to
602  	 * setup the vector space for the cpu which comes online.
603  	 * Prevent irq alloc/free across the bringup.
604  	 */
605  	irq_lock_sparse();
606  
607  	/* Arch-specific enabling code. */
608  	ret = __cpu_up(cpu, idle);
609  	irq_unlock_sparse();
610  	if (ret)
611  		return ret;
612  	return bringup_wait_for_ap(cpu);
613  }
614  
615  static int finish_cpu(unsigned int cpu)
616  {
617  	struct task_struct *idle = idle_thread_get(cpu);
618  	struct mm_struct *mm = idle->active_mm;
619  
620  	/*
621  	 * idle_task_exit() will have switched to &init_mm, now
622  	 * clean up any remaining active_mm state.
623  	 */
624  	if (mm != &init_mm)
625  		idle->active_mm = &init_mm;
626  	mmdrop(mm);
627  	return 0;
628  }
629  
630  /*
631   * Hotplug state machine related functions
632   */
633  
634  /*
635   * Get the next state to run. Empty ones will be skipped. Returns true if a
636   * state must be run.
637   *
638   * st->state will be modified ahead of time, to match state_to_run, as if it
639   * has already ran.
640   */
641  static bool cpuhp_next_state(bool bringup,
642  			     enum cpuhp_state *state_to_run,
643  			     struct cpuhp_cpu_state *st,
644  			     enum cpuhp_state target)
645  {
646  	do {
647  		if (bringup) {
648  			if (st->state >= target)
649  				return false;
650  
651  			*state_to_run = ++st->state;
652  		} else {
653  			if (st->state <= target)
654  				return false;
655  
656  			*state_to_run = st->state--;
657  		}
658  
659  		if (!cpuhp_step_empty(bringup, cpuhp_get_step(*state_to_run)))
660  			break;
661  	} while (true);
662  
663  	return true;
664  }
665  
666  static int __cpuhp_invoke_callback_range(bool bringup,
667  					 unsigned int cpu,
668  					 struct cpuhp_cpu_state *st,
669  					 enum cpuhp_state target,
670  					 bool nofail)
671  {
672  	enum cpuhp_state state;
673  	int ret = 0;
674  
675  	while (cpuhp_next_state(bringup, &state, st, target)) {
676  		int err;
677  
678  		err = cpuhp_invoke_callback(cpu, state, bringup, NULL, NULL);
679  		if (!err)
680  			continue;
681  
682  		if (nofail) {
683  			pr_warn("CPU %u %s state %s (%d) failed (%d)\n",
684  				cpu, bringup ? "UP" : "DOWN",
685  				cpuhp_get_step(st->state)->name,
686  				st->state, err);
687  			ret = -1;
688  		} else {
689  			ret = err;
690  			break;
691  		}
692  	}
693  
694  	return ret;
695  }
696  
697  static inline int cpuhp_invoke_callback_range(bool bringup,
698  					      unsigned int cpu,
699  					      struct cpuhp_cpu_state *st,
700  					      enum cpuhp_state target)
701  {
702  	return __cpuhp_invoke_callback_range(bringup, cpu, st, target, false);
703  }
704  
705  static inline void cpuhp_invoke_callback_range_nofail(bool bringup,
706  						      unsigned int cpu,
707  						      struct cpuhp_cpu_state *st,
708  						      enum cpuhp_state target)
709  {
710  	__cpuhp_invoke_callback_range(bringup, cpu, st, target, true);
711  }
712  
713  static inline bool can_rollback_cpu(struct cpuhp_cpu_state *st)
714  {
715  	if (IS_ENABLED(CONFIG_HOTPLUG_CPU))
716  		return true;
717  	/*
718  	 * When CPU hotplug is disabled, then taking the CPU down is not
719  	 * possible because takedown_cpu() and the architecture and
720  	 * subsystem specific mechanisms are not available. So the CPU
721  	 * which would be completely unplugged again needs to stay around
722  	 * in the current state.
723  	 */
724  	return st->state <= CPUHP_BRINGUP_CPU;
725  }
726  
727  static int cpuhp_up_callbacks(unsigned int cpu, struct cpuhp_cpu_state *st,
728  			      enum cpuhp_state target)
729  {
730  	enum cpuhp_state prev_state = st->state;
731  	int ret = 0;
732  
733  	ret = cpuhp_invoke_callback_range(true, cpu, st, target);
734  	if (ret) {
735  		pr_debug("CPU UP failed (%d) CPU %u state %s (%d)\n",
736  			 ret, cpu, cpuhp_get_step(st->state)->name,
737  			 st->state);
738  
739  		cpuhp_reset_state(cpu, st, prev_state);
740  		if (can_rollback_cpu(st))
741  			WARN_ON(cpuhp_invoke_callback_range(false, cpu, st,
742  							    prev_state));
743  	}
744  	return ret;
745  }
746  
747  /*
748   * The cpu hotplug threads manage the bringup and teardown of the cpus
749   */
750  static int cpuhp_should_run(unsigned int cpu)
751  {
752  	struct cpuhp_cpu_state *st = this_cpu_ptr(&cpuhp_state);
753  
754  	return st->should_run;
755  }
756  
757  /*
758   * Execute teardown/startup callbacks on the plugged cpu. Also used to invoke
759   * callbacks when a state gets [un]installed at runtime.
760   *
761   * Each invocation of this function by the smpboot thread does a single AP
762   * state callback.
763   *
764   * It has 3 modes of operation:
765   *  - single: runs st->cb_state
766   *  - up:     runs ++st->state, while st->state < st->target
767   *  - down:   runs st->state--, while st->state > st->target
768   *
769   * When complete or on error, should_run is cleared and the completion is fired.
770   */
771  static void cpuhp_thread_fun(unsigned int cpu)
772  {
773  	struct cpuhp_cpu_state *st = this_cpu_ptr(&cpuhp_state);
774  	bool bringup = st->bringup;
775  	enum cpuhp_state state;
776  
777  	if (WARN_ON_ONCE(!st->should_run))
778  		return;
779  
780  	/*
781  	 * ACQUIRE for the cpuhp_should_run() load of ->should_run. Ensures
782  	 * that if we see ->should_run we also see the rest of the state.
783  	 */
784  	smp_mb();
785  
786  	/*
787  	 * The BP holds the hotplug lock, but we're now running on the AP,
788  	 * ensure that anybody asserting the lock is held, will actually find
789  	 * it so.
790  	 */
791  	lockdep_acquire_cpus_lock();
792  	cpuhp_lock_acquire(bringup);
793  
794  	if (st->single) {
795  		state = st->cb_state;
796  		st->should_run = false;
797  	} else {
798  		st->should_run = cpuhp_next_state(bringup, &state, st, st->target);
799  		if (!st->should_run)
800  			goto end;
801  	}
802  
803  	WARN_ON_ONCE(!cpuhp_is_ap_state(state));
804  
805  	if (cpuhp_is_atomic_state(state)) {
806  		local_irq_disable();
807  		st->result = cpuhp_invoke_callback(cpu, state, bringup, st->node, &st->last);
808  		local_irq_enable();
809  
810  		/*
811  		 * STARTING/DYING must not fail!
812  		 */
813  		WARN_ON_ONCE(st->result);
814  	} else {
815  		st->result = cpuhp_invoke_callback(cpu, state, bringup, st->node, &st->last);
816  	}
817  
818  	if (st->result) {
819  		/*
820  		 * If we fail on a rollback, we're up a creek without no
821  		 * paddle, no way forward, no way back. We loose, thanks for
822  		 * playing.
823  		 */
824  		WARN_ON_ONCE(st->rollback);
825  		st->should_run = false;
826  	}
827  
828  end:
829  	cpuhp_lock_release(bringup);
830  	lockdep_release_cpus_lock();
831  
832  	if (!st->should_run)
833  		complete_ap_thread(st, bringup);
834  }
835  
836  /* Invoke a single callback on a remote cpu */
837  static int
838  cpuhp_invoke_ap_callback(int cpu, enum cpuhp_state state, bool bringup,
839  			 struct hlist_node *node)
840  {
841  	struct cpuhp_cpu_state *st = per_cpu_ptr(&cpuhp_state, cpu);
842  	int ret;
843  
844  	if (!cpu_online(cpu))
845  		return 0;
846  
847  	cpuhp_lock_acquire(false);
848  	cpuhp_lock_release(false);
849  
850  	cpuhp_lock_acquire(true);
851  	cpuhp_lock_release(true);
852  
853  	/*
854  	 * If we are up and running, use the hotplug thread. For early calls
855  	 * we invoke the thread function directly.
856  	 */
857  	if (!st->thread)
858  		return cpuhp_invoke_callback(cpu, state, bringup, node, NULL);
859  
860  	st->rollback = false;
861  	st->last = NULL;
862  
863  	st->node = node;
864  	st->bringup = bringup;
865  	st->cb_state = state;
866  	st->single = true;
867  
868  	__cpuhp_kick_ap(st);
869  
870  	/*
871  	 * If we failed and did a partial, do a rollback.
872  	 */
873  	if ((ret = st->result) && st->last) {
874  		st->rollback = true;
875  		st->bringup = !bringup;
876  
877  		__cpuhp_kick_ap(st);
878  	}
879  
880  	/*
881  	 * Clean up the leftovers so the next hotplug operation wont use stale
882  	 * data.
883  	 */
884  	st->node = st->last = NULL;
885  	return ret;
886  }
887  
888  static int cpuhp_kick_ap_work(unsigned int cpu)
889  {
890  	struct cpuhp_cpu_state *st = per_cpu_ptr(&cpuhp_state, cpu);
891  	enum cpuhp_state prev_state = st->state;
892  	int ret;
893  
894  	cpuhp_lock_acquire(false);
895  	cpuhp_lock_release(false);
896  
897  	cpuhp_lock_acquire(true);
898  	cpuhp_lock_release(true);
899  
900  	trace_cpuhp_enter(cpu, st->target, prev_state, cpuhp_kick_ap_work);
901  	ret = cpuhp_kick_ap(cpu, st, st->target);
902  	trace_cpuhp_exit(cpu, st->state, prev_state, ret);
903  
904  	return ret;
905  }
906  
907  static struct smp_hotplug_thread cpuhp_threads = {
908  	.store			= &cpuhp_state.thread,
909  	.thread_should_run	= cpuhp_should_run,
910  	.thread_fn		= cpuhp_thread_fun,
911  	.thread_comm		= "cpuhp/%u",
912  	.selfparking		= true,
913  };
914  
915  static __init void cpuhp_init_state(void)
916  {
917  	struct cpuhp_cpu_state *st;
918  	int cpu;
919  
920  	for_each_possible_cpu(cpu) {
921  		st = per_cpu_ptr(&cpuhp_state, cpu);
922  		init_completion(&st->done_up);
923  		init_completion(&st->done_down);
924  	}
925  }
926  
927  void __init cpuhp_threads_init(void)
928  {
929  	cpuhp_init_state();
930  	BUG_ON(smpboot_register_percpu_thread(&cpuhp_threads));
931  	kthread_unpark(this_cpu_read(cpuhp_state.thread));
932  }
933  
934  /*
935   *
936   * Serialize hotplug trainwrecks outside of the cpu_hotplug_lock
937   * protected region.
938   *
939   * The operation is still serialized against concurrent CPU hotplug via
940   * cpu_add_remove_lock, i.e. CPU map protection.  But it is _not_
941   * serialized against other hotplug related activity like adding or
942   * removing of state callbacks and state instances, which invoke either the
943   * startup or the teardown callback of the affected state.
944   *
945   * This is required for subsystems which are unfixable vs. CPU hotplug and
946   * evade lock inversion problems by scheduling work which has to be
947   * completed _before_ cpu_up()/_cpu_down() returns.
948   *
949   * Don't even think about adding anything to this for any new code or even
950   * drivers. It's only purpose is to keep existing lock order trainwrecks
951   * working.
952   *
953   * For cpu_down() there might be valid reasons to finish cleanups which are
954   * not required to be done under cpu_hotplug_lock, but that's a different
955   * story and would be not invoked via this.
956   */
957  static void cpu_up_down_serialize_trainwrecks(bool tasks_frozen)
958  {
959  	/*
960  	 * cpusets delegate hotplug operations to a worker to "solve" the
961  	 * lock order problems. Wait for the worker, but only if tasks are
962  	 * _not_ frozen (suspend, hibernate) as that would wait forever.
963  	 *
964  	 * The wait is required because otherwise the hotplug operation
965  	 * returns with inconsistent state, which could even be observed in
966  	 * user space when a new CPU is brought up. The CPU plug uevent
967  	 * would be delivered and user space reacting on it would fail to
968  	 * move tasks to the newly plugged CPU up to the point where the
969  	 * work has finished because up to that point the newly plugged CPU
970  	 * is not assignable in cpusets/cgroups. On unplug that's not
971  	 * necessarily a visible issue, but it is still inconsistent state,
972  	 * which is the real problem which needs to be "fixed". This can't
973  	 * prevent the transient state between scheduling the work and
974  	 * returning from waiting for it.
975  	 */
976  	if (!tasks_frozen)
977  		cpuset_wait_for_hotplug();
978  }
979  
980  #ifdef CONFIG_HOTPLUG_CPU
981  #ifndef arch_clear_mm_cpumask_cpu
982  #define arch_clear_mm_cpumask_cpu(cpu, mm) cpumask_clear_cpu(cpu, mm_cpumask(mm))
983  #endif
984  
985  /**
986   * clear_tasks_mm_cpumask - Safely clear tasks' mm_cpumask for a CPU
987   * @cpu: a CPU id
988   *
989   * This function walks all processes, finds a valid mm struct for each one and
990   * then clears a corresponding bit in mm's cpumask.  While this all sounds
991   * trivial, there are various non-obvious corner cases, which this function
992   * tries to solve in a safe manner.
993   *
994   * Also note that the function uses a somewhat relaxed locking scheme, so it may
995   * be called only for an already offlined CPU.
996   */
997  void clear_tasks_mm_cpumask(int cpu)
998  {
999  	struct task_struct *p;
1000  
1001  	/*
1002  	 * This function is called after the cpu is taken down and marked
1003  	 * offline, so its not like new tasks will ever get this cpu set in
1004  	 * their mm mask. -- Peter Zijlstra
1005  	 * Thus, we may use rcu_read_lock() here, instead of grabbing
1006  	 * full-fledged tasklist_lock.
1007  	 */
1008  	WARN_ON(cpu_online(cpu));
1009  	rcu_read_lock();
1010  	for_each_process(p) {
1011  		struct task_struct *t;
1012  
1013  		/*
1014  		 * Main thread might exit, but other threads may still have
1015  		 * a valid mm. Find one.
1016  		 */
1017  		t = find_lock_task_mm(p);
1018  		if (!t)
1019  			continue;
1020  		arch_clear_mm_cpumask_cpu(cpu, t->mm);
1021  		task_unlock(t);
1022  	}
1023  	rcu_read_unlock();
1024  }
1025  
1026  /* Take this CPU down. */
1027  static int take_cpu_down(void *_param)
1028  {
1029  	struct cpuhp_cpu_state *st = this_cpu_ptr(&cpuhp_state);
1030  	enum cpuhp_state target = max((int)st->target, CPUHP_AP_OFFLINE);
1031  	int err, cpu = smp_processor_id();
1032  
1033  	/* Ensure this CPU doesn't handle any more interrupts. */
1034  	err = __cpu_disable();
1035  	if (err < 0)
1036  		return err;
1037  
1038  	/*
1039  	 * Must be called from CPUHP_TEARDOWN_CPU, which means, as we are going
1040  	 * down, that the current state is CPUHP_TEARDOWN_CPU - 1.
1041  	 */
1042  	WARN_ON(st->state != (CPUHP_TEARDOWN_CPU - 1));
1043  
1044  	/*
1045  	 * Invoke the former CPU_DYING callbacks. DYING must not fail!
1046  	 */
1047  	cpuhp_invoke_callback_range_nofail(false, cpu, st, target);
1048  
1049  	/* Give up timekeeping duties */
1050  	tick_handover_do_timer();
1051  	/* Remove CPU from timer broadcasting */
1052  	tick_offline_cpu(cpu);
1053  	/* Park the stopper thread */
1054  	stop_machine_park(cpu);
1055  	return 0;
1056  }
1057  
1058  static int takedown_cpu(unsigned int cpu)
1059  {
1060  	struct cpuhp_cpu_state *st = per_cpu_ptr(&cpuhp_state, cpu);
1061  	int err;
1062  
1063  	/* Park the smpboot threads */
1064  	kthread_park(st->thread);
1065  
1066  	/*
1067  	 * Prevent irq alloc/free while the dying cpu reorganizes the
1068  	 * interrupt affinities.
1069  	 */
1070  	irq_lock_sparse();
1071  
1072  	/*
1073  	 * So now all preempt/rcu users must observe !cpu_active().
1074  	 */
1075  	err = stop_machine_cpuslocked(take_cpu_down, NULL, cpumask_of(cpu));
1076  	if (err) {
1077  		/* CPU refused to die */
1078  		irq_unlock_sparse();
1079  		/* Unpark the hotplug thread so we can rollback there */
1080  		kthread_unpark(st->thread);
1081  		return err;
1082  	}
1083  	BUG_ON(cpu_online(cpu));
1084  
1085  	/*
1086  	 * The teardown callback for CPUHP_AP_SCHED_STARTING will have removed
1087  	 * all runnable tasks from the CPU, there's only the idle task left now
1088  	 * that the migration thread is done doing the stop_machine thing.
1089  	 *
1090  	 * Wait for the stop thread to go away.
1091  	 */
1092  	wait_for_ap_thread(st, false);
1093  	BUG_ON(st->state != CPUHP_AP_IDLE_DEAD);
1094  
1095  	/* Interrupts are moved away from the dying cpu, reenable alloc/free */
1096  	irq_unlock_sparse();
1097  
1098  	hotplug_cpu__broadcast_tick_pull(cpu);
1099  	/* This actually kills the CPU. */
1100  	__cpu_die(cpu);
1101  
1102  	tick_cleanup_dead_cpu(cpu);
1103  	rcutree_migrate_callbacks(cpu);
1104  	return 0;
1105  }
1106  
1107  static void cpuhp_complete_idle_dead(void *arg)
1108  {
1109  	struct cpuhp_cpu_state *st = arg;
1110  
1111  	complete_ap_thread(st, false);
1112  }
1113  
1114  void cpuhp_report_idle_dead(void)
1115  {
1116  	struct cpuhp_cpu_state *st = this_cpu_ptr(&cpuhp_state);
1117  
1118  	BUG_ON(st->state != CPUHP_AP_OFFLINE);
1119  	rcu_report_dead(smp_processor_id());
1120  	st->state = CPUHP_AP_IDLE_DEAD;
1121  	/*
1122  	 * We cannot call complete after rcu_report_dead() so we delegate it
1123  	 * to an online cpu.
1124  	 */
1125  	smp_call_function_single(cpumask_first(cpu_online_mask),
1126  				 cpuhp_complete_idle_dead, st, 0);
1127  }
1128  
1129  static int cpuhp_down_callbacks(unsigned int cpu, struct cpuhp_cpu_state *st,
1130  				enum cpuhp_state target)
1131  {
1132  	enum cpuhp_state prev_state = st->state;
1133  	int ret = 0;
1134  
1135  	ret = cpuhp_invoke_callback_range(false, cpu, st, target);
1136  	if (ret) {
1137  		pr_debug("CPU DOWN failed (%d) CPU %u state %s (%d)\n",
1138  			 ret, cpu, cpuhp_get_step(st->state)->name,
1139  			 st->state);
1140  
1141  		cpuhp_reset_state(cpu, st, prev_state);
1142  
1143  		if (st->state < prev_state)
1144  			WARN_ON(cpuhp_invoke_callback_range(true, cpu, st,
1145  							    prev_state));
1146  	}
1147  
1148  	return ret;
1149  }
1150  
1151  /* Requires cpu_add_remove_lock to be held */
1152  static int __ref _cpu_down(unsigned int cpu, int tasks_frozen,
1153  			   enum cpuhp_state target)
1154  {
1155  	struct cpuhp_cpu_state *st = per_cpu_ptr(&cpuhp_state, cpu);
1156  	int prev_state, ret = 0;
1157  
1158  	if (num_online_cpus() == 1)
1159  		return -EBUSY;
1160  
1161  	if (!cpu_present(cpu))
1162  		return -EINVAL;
1163  
1164  	cpus_write_lock();
1165  
1166  	cpuhp_tasks_frozen = tasks_frozen;
1167  
1168  	prev_state = cpuhp_set_state(cpu, st, target);
1169  	/*
1170  	 * If the current CPU state is in the range of the AP hotplug thread,
1171  	 * then we need to kick the thread.
1172  	 */
1173  	if (st->state > CPUHP_TEARDOWN_CPU) {
1174  		st->target = max((int)target, CPUHP_TEARDOWN_CPU);
1175  		ret = cpuhp_kick_ap_work(cpu);
1176  		/*
1177  		 * The AP side has done the error rollback already. Just
1178  		 * return the error code..
1179  		 */
1180  		if (ret)
1181  			goto out;
1182  
1183  		/*
1184  		 * We might have stopped still in the range of the AP hotplug
1185  		 * thread. Nothing to do anymore.
1186  		 */
1187  		if (st->state > CPUHP_TEARDOWN_CPU)
1188  			goto out;
1189  
1190  		st->target = target;
1191  	}
1192  	/*
1193  	 * The AP brought itself down to CPUHP_TEARDOWN_CPU. So we need
1194  	 * to do the further cleanups.
1195  	 */
1196  	ret = cpuhp_down_callbacks(cpu, st, target);
1197  	if (ret && st->state < prev_state) {
1198  		if (st->state == CPUHP_TEARDOWN_CPU) {
1199  			cpuhp_reset_state(cpu, st, prev_state);
1200  			__cpuhp_kick_ap(st);
1201  		} else {
1202  			WARN(1, "DEAD callback error for CPU%d", cpu);
1203  		}
1204  	}
1205  
1206  out:
1207  	cpus_write_unlock();
1208  	/*
1209  	 * Do post unplug cleanup. This is still protected against
1210  	 * concurrent CPU hotplug via cpu_add_remove_lock.
1211  	 */
1212  	lockup_detector_cleanup();
1213  	arch_smt_update();
1214  	cpu_up_down_serialize_trainwrecks(tasks_frozen);
1215  	return ret;
1216  }
1217  
1218  static int cpu_down_maps_locked(unsigned int cpu, enum cpuhp_state target)
1219  {
1220  	/*
1221  	 * If the platform does not support hotplug, report it explicitly to
1222  	 * differentiate it from a transient offlining failure.
1223  	 */
1224  	if (cc_platform_has(CC_ATTR_HOTPLUG_DISABLED))
1225  		return -EOPNOTSUPP;
1226  	if (cpu_hotplug_disabled)
1227  		return -EBUSY;
1228  	return _cpu_down(cpu, 0, target);
1229  }
1230  
1231  static int cpu_down(unsigned int cpu, enum cpuhp_state target)
1232  {
1233  	int err;
1234  
1235  	cpu_maps_update_begin();
1236  	err = cpu_down_maps_locked(cpu, target);
1237  	cpu_maps_update_done();
1238  	return err;
1239  }
1240  
1241  /**
1242   * cpu_device_down - Bring down a cpu device
1243   * @dev: Pointer to the cpu device to offline
1244   *
1245   * This function is meant to be used by device core cpu subsystem only.
1246   *
1247   * Other subsystems should use remove_cpu() instead.
1248   *
1249   * Return: %0 on success or a negative errno code
1250   */
1251  int cpu_device_down(struct device *dev)
1252  {
1253  	return cpu_down(dev->id, CPUHP_OFFLINE);
1254  }
1255  
1256  int remove_cpu(unsigned int cpu)
1257  {
1258  	int ret;
1259  
1260  	lock_device_hotplug();
1261  	ret = device_offline(get_cpu_device(cpu));
1262  	unlock_device_hotplug();
1263  
1264  	return ret;
1265  }
1266  EXPORT_SYMBOL_GPL(remove_cpu);
1267  
1268  void smp_shutdown_nonboot_cpus(unsigned int primary_cpu)
1269  {
1270  	unsigned int cpu;
1271  	int error;
1272  
1273  	cpu_maps_update_begin();
1274  
1275  	/*
1276  	 * Make certain the cpu I'm about to reboot on is online.
1277  	 *
1278  	 * This is inline to what migrate_to_reboot_cpu() already do.
1279  	 */
1280  	if (!cpu_online(primary_cpu))
1281  		primary_cpu = cpumask_first(cpu_online_mask);
1282  
1283  	for_each_online_cpu(cpu) {
1284  		if (cpu == primary_cpu)
1285  			continue;
1286  
1287  		error = cpu_down_maps_locked(cpu, CPUHP_OFFLINE);
1288  		if (error) {
1289  			pr_err("Failed to offline CPU%d - error=%d",
1290  				cpu, error);
1291  			break;
1292  		}
1293  	}
1294  
1295  	/*
1296  	 * Ensure all but the reboot CPU are offline.
1297  	 */
1298  	BUG_ON(num_online_cpus() > 1);
1299  
1300  	/*
1301  	 * Make sure the CPUs won't be enabled by someone else after this
1302  	 * point. Kexec will reboot to a new kernel shortly resetting
1303  	 * everything along the way.
1304  	 */
1305  	cpu_hotplug_disabled++;
1306  
1307  	cpu_maps_update_done();
1308  }
1309  
1310  #else
1311  #define takedown_cpu		NULL
1312  #endif /*CONFIG_HOTPLUG_CPU*/
1313  
1314  /**
1315   * notify_cpu_starting(cpu) - Invoke the callbacks on the starting CPU
1316   * @cpu: cpu that just started
1317   *
1318   * It must be called by the arch code on the new cpu, before the new cpu
1319   * enables interrupts and before the "boot" cpu returns from __cpu_up().
1320   */
1321  void notify_cpu_starting(unsigned int cpu)
1322  {
1323  	struct cpuhp_cpu_state *st = per_cpu_ptr(&cpuhp_state, cpu);
1324  	enum cpuhp_state target = min((int)st->target, CPUHP_AP_ONLINE);
1325  
1326  	rcu_cpu_starting(cpu);	/* Enables RCU usage on this CPU. */
1327  	cpumask_set_cpu(cpu, &cpus_booted_once_mask);
1328  
1329  	/*
1330  	 * STARTING must not fail!
1331  	 */
1332  	cpuhp_invoke_callback_range_nofail(true, cpu, st, target);
1333  }
1334  
1335  /*
1336   * Called from the idle task. Wake up the controlling task which brings the
1337   * hotplug thread of the upcoming CPU up and then delegates the rest of the
1338   * online bringup to the hotplug thread.
1339   */
1340  void cpuhp_online_idle(enum cpuhp_state state)
1341  {
1342  	struct cpuhp_cpu_state *st = this_cpu_ptr(&cpuhp_state);
1343  
1344  	/* Happens for the boot cpu */
1345  	if (state != CPUHP_AP_ONLINE_IDLE)
1346  		return;
1347  
1348  	/*
1349  	 * Unpart the stopper thread before we start the idle loop (and start
1350  	 * scheduling); this ensures the stopper task is always available.
1351  	 */
1352  	stop_machine_unpark(smp_processor_id());
1353  
1354  	st->state = CPUHP_AP_ONLINE_IDLE;
1355  	complete_ap_thread(st, true);
1356  }
1357  
1358  /* Requires cpu_add_remove_lock to be held */
1359  static int _cpu_up(unsigned int cpu, int tasks_frozen, enum cpuhp_state target)
1360  {
1361  	struct cpuhp_cpu_state *st = per_cpu_ptr(&cpuhp_state, cpu);
1362  	struct task_struct *idle;
1363  	int ret = 0;
1364  
1365  	cpus_write_lock();
1366  
1367  	if (!cpu_present(cpu)) {
1368  		ret = -EINVAL;
1369  		goto out;
1370  	}
1371  
1372  	/*
1373  	 * The caller of cpu_up() might have raced with another
1374  	 * caller. Nothing to do.
1375  	 */
1376  	if (st->state >= target)
1377  		goto out;
1378  
1379  	if (st->state == CPUHP_OFFLINE) {
1380  		/* Let it fail before we try to bring the cpu up */
1381  		idle = idle_thread_get(cpu);
1382  		if (IS_ERR(idle)) {
1383  			ret = PTR_ERR(idle);
1384  			goto out;
1385  		}
1386  	}
1387  
1388  	cpuhp_tasks_frozen = tasks_frozen;
1389  
1390  	cpuhp_set_state(cpu, st, target);
1391  	/*
1392  	 * If the current CPU state is in the range of the AP hotplug thread,
1393  	 * then we need to kick the thread once more.
1394  	 */
1395  	if (st->state > CPUHP_BRINGUP_CPU) {
1396  		ret = cpuhp_kick_ap_work(cpu);
1397  		/*
1398  		 * The AP side has done the error rollback already. Just
1399  		 * return the error code..
1400  		 */
1401  		if (ret)
1402  			goto out;
1403  	}
1404  
1405  	/*
1406  	 * Try to reach the target state. We max out on the BP at
1407  	 * CPUHP_BRINGUP_CPU. After that the AP hotplug thread is
1408  	 * responsible for bringing it up to the target state.
1409  	 */
1410  	target = min((int)target, CPUHP_BRINGUP_CPU);
1411  	ret = cpuhp_up_callbacks(cpu, st, target);
1412  out:
1413  	cpus_write_unlock();
1414  	arch_smt_update();
1415  	cpu_up_down_serialize_trainwrecks(tasks_frozen);
1416  	return ret;
1417  }
1418  
1419  static int cpu_up(unsigned int cpu, enum cpuhp_state target)
1420  {
1421  	int err = 0;
1422  
1423  	if (!cpu_possible(cpu)) {
1424  		pr_err("can't online cpu %d because it is not configured as may-hotadd at boot time\n",
1425  		       cpu);
1426  #if defined(CONFIG_IA64)
1427  		pr_err("please check additional_cpus= boot parameter\n");
1428  #endif
1429  		return -EINVAL;
1430  	}
1431  
1432  	err = try_online_node(cpu_to_node(cpu));
1433  	if (err)
1434  		return err;
1435  
1436  	cpu_maps_update_begin();
1437  
1438  	if (cpu_hotplug_disabled) {
1439  		err = -EBUSY;
1440  		goto out;
1441  	}
1442  	if (!cpu_smt_allowed(cpu)) {
1443  		err = -EPERM;
1444  		goto out;
1445  	}
1446  
1447  	err = _cpu_up(cpu, 0, target);
1448  out:
1449  	cpu_maps_update_done();
1450  	return err;
1451  }
1452  
1453  /**
1454   * cpu_device_up - Bring up a cpu device
1455   * @dev: Pointer to the cpu device to online
1456   *
1457   * This function is meant to be used by device core cpu subsystem only.
1458   *
1459   * Other subsystems should use add_cpu() instead.
1460   *
1461   * Return: %0 on success or a negative errno code
1462   */
1463  int cpu_device_up(struct device *dev)
1464  {
1465  	return cpu_up(dev->id, CPUHP_ONLINE);
1466  }
1467  
1468  int add_cpu(unsigned int cpu)
1469  {
1470  	int ret;
1471  
1472  	lock_device_hotplug();
1473  	ret = device_online(get_cpu_device(cpu));
1474  	unlock_device_hotplug();
1475  
1476  	return ret;
1477  }
1478  EXPORT_SYMBOL_GPL(add_cpu);
1479  
1480  /**
1481   * bringup_hibernate_cpu - Bring up the CPU that we hibernated on
1482   * @sleep_cpu: The cpu we hibernated on and should be brought up.
1483   *
1484   * On some architectures like arm64, we can hibernate on any CPU, but on
1485   * wake up the CPU we hibernated on might be offline as a side effect of
1486   * using maxcpus= for example.
1487   *
1488   * Return: %0 on success or a negative errno code
1489   */
1490  int bringup_hibernate_cpu(unsigned int sleep_cpu)
1491  {
1492  	int ret;
1493  
1494  	if (!cpu_online(sleep_cpu)) {
1495  		pr_info("Hibernated on a CPU that is offline! Bringing CPU up.\n");
1496  		ret = cpu_up(sleep_cpu, CPUHP_ONLINE);
1497  		if (ret) {
1498  			pr_err("Failed to bring hibernate-CPU up!\n");
1499  			return ret;
1500  		}
1501  	}
1502  	return 0;
1503  }
1504  
1505  void bringup_nonboot_cpus(unsigned int setup_max_cpus)
1506  {
1507  	unsigned int cpu;
1508  
1509  	for_each_present_cpu(cpu) {
1510  		if (num_online_cpus() >= setup_max_cpus)
1511  			break;
1512  		if (!cpu_online(cpu))
1513  			cpu_up(cpu, CPUHP_ONLINE);
1514  	}
1515  }
1516  
1517  #ifdef CONFIG_PM_SLEEP_SMP
1518  static cpumask_var_t frozen_cpus;
1519  
1520  int freeze_secondary_cpus(int primary)
1521  {
1522  	int cpu, error = 0;
1523  
1524  	cpu_maps_update_begin();
1525  	if (primary == -1) {
1526  		primary = cpumask_first(cpu_online_mask);
1527  		if (!housekeeping_cpu(primary, HK_TYPE_TIMER))
1528  			primary = housekeeping_any_cpu(HK_TYPE_TIMER);
1529  	} else {
1530  		if (!cpu_online(primary))
1531  			primary = cpumask_first(cpu_online_mask);
1532  	}
1533  
1534  	/*
1535  	 * We take down all of the non-boot CPUs in one shot to avoid races
1536  	 * with the userspace trying to use the CPU hotplug at the same time
1537  	 */
1538  	cpumask_clear(frozen_cpus);
1539  
1540  	pr_info("Disabling non-boot CPUs ...\n");
1541  	for_each_online_cpu(cpu) {
1542  		if (cpu == primary)
1543  			continue;
1544  
1545  		if (pm_wakeup_pending()) {
1546  			pr_info("Wakeup pending. Abort CPU freeze\n");
1547  			error = -EBUSY;
1548  			break;
1549  		}
1550  
1551  		trace_suspend_resume(TPS("CPU_OFF"), cpu, true);
1552  		error = _cpu_down(cpu, 1, CPUHP_OFFLINE);
1553  		trace_suspend_resume(TPS("CPU_OFF"), cpu, false);
1554  		if (!error)
1555  			cpumask_set_cpu(cpu, frozen_cpus);
1556  		else {
1557  			pr_err("Error taking CPU%d down: %d\n", cpu, error);
1558  			break;
1559  		}
1560  	}
1561  
1562  	if (!error)
1563  		BUG_ON(num_online_cpus() > 1);
1564  	else
1565  		pr_err("Non-boot CPUs are not disabled\n");
1566  
1567  	/*
1568  	 * Make sure the CPUs won't be enabled by someone else. We need to do
1569  	 * this even in case of failure as all freeze_secondary_cpus() users are
1570  	 * supposed to do thaw_secondary_cpus() on the failure path.
1571  	 */
1572  	cpu_hotplug_disabled++;
1573  
1574  	cpu_maps_update_done();
1575  	return error;
1576  }
1577  
1578  void __weak arch_thaw_secondary_cpus_begin(void)
1579  {
1580  }
1581  
1582  void __weak arch_thaw_secondary_cpus_end(void)
1583  {
1584  }
1585  
1586  void thaw_secondary_cpus(void)
1587  {
1588  	int cpu, error;
1589  
1590  	/* Allow everyone to use the CPU hotplug again */
1591  	cpu_maps_update_begin();
1592  	__cpu_hotplug_enable();
1593  	if (cpumask_empty(frozen_cpus))
1594  		goto out;
1595  
1596  	pr_info("Enabling non-boot CPUs ...\n");
1597  
1598  	arch_thaw_secondary_cpus_begin();
1599  
1600  	for_each_cpu(cpu, frozen_cpus) {
1601  		trace_suspend_resume(TPS("CPU_ON"), cpu, true);
1602  		error = _cpu_up(cpu, 1, CPUHP_ONLINE);
1603  		trace_suspend_resume(TPS("CPU_ON"), cpu, false);
1604  		if (!error) {
1605  			pr_info("CPU%d is up\n", cpu);
1606  			continue;
1607  		}
1608  		pr_warn("Error taking CPU%d up: %d\n", cpu, error);
1609  	}
1610  
1611  	arch_thaw_secondary_cpus_end();
1612  
1613  	cpumask_clear(frozen_cpus);
1614  out:
1615  	cpu_maps_update_done();
1616  }
1617  
1618  static int __init alloc_frozen_cpus(void)
1619  {
1620  	if (!alloc_cpumask_var(&frozen_cpus, GFP_KERNEL|__GFP_ZERO))
1621  		return -ENOMEM;
1622  	return 0;
1623  }
1624  core_initcall(alloc_frozen_cpus);
1625  
1626  /*
1627   * When callbacks for CPU hotplug notifications are being executed, we must
1628   * ensure that the state of the system with respect to the tasks being frozen
1629   * or not, as reported by the notification, remains unchanged *throughout the
1630   * duration* of the execution of the callbacks.
1631   * Hence we need to prevent the freezer from racing with regular CPU hotplug.
1632   *
1633   * This synchronization is implemented by mutually excluding regular CPU
1634   * hotplug and Suspend/Hibernate call paths by hooking onto the Suspend/
1635   * Hibernate notifications.
1636   */
1637  static int
1638  cpu_hotplug_pm_callback(struct notifier_block *nb,
1639  			unsigned long action, void *ptr)
1640  {
1641  	switch (action) {
1642  
1643  	case PM_SUSPEND_PREPARE:
1644  	case PM_HIBERNATION_PREPARE:
1645  		cpu_hotplug_disable();
1646  		break;
1647  
1648  	case PM_POST_SUSPEND:
1649  	case PM_POST_HIBERNATION:
1650  		cpu_hotplug_enable();
1651  		break;
1652  
1653  	default:
1654  		return NOTIFY_DONE;
1655  	}
1656  
1657  	return NOTIFY_OK;
1658  }
1659  
1660  
1661  static int __init cpu_hotplug_pm_sync_init(void)
1662  {
1663  	/*
1664  	 * cpu_hotplug_pm_callback has higher priority than x86
1665  	 * bsp_pm_callback which depends on cpu_hotplug_pm_callback
1666  	 * to disable cpu hotplug to avoid cpu hotplug race.
1667  	 */
1668  	pm_notifier(cpu_hotplug_pm_callback, 0);
1669  	return 0;
1670  }
1671  core_initcall(cpu_hotplug_pm_sync_init);
1672  
1673  #endif /* CONFIG_PM_SLEEP_SMP */
1674  
1675  int __boot_cpu_id;
1676  
1677  #endif /* CONFIG_SMP */
1678  
1679  /* Boot processor state steps */
1680  static struct cpuhp_step cpuhp_hp_states[] = {
1681  	[CPUHP_OFFLINE] = {
1682  		.name			= "offline",
1683  		.startup.single		= NULL,
1684  		.teardown.single	= NULL,
1685  	},
1686  #ifdef CONFIG_SMP
1687  	[CPUHP_CREATE_THREADS]= {
1688  		.name			= "threads:prepare",
1689  		.startup.single		= smpboot_create_threads,
1690  		.teardown.single	= NULL,
1691  		.cant_stop		= true,
1692  	},
1693  	[CPUHP_PERF_PREPARE] = {
1694  		.name			= "perf:prepare",
1695  		.startup.single		= perf_event_init_cpu,
1696  		.teardown.single	= perf_event_exit_cpu,
1697  	},
1698  	[CPUHP_RANDOM_PREPARE] = {
1699  		.name			= "random:prepare",
1700  		.startup.single		= random_prepare_cpu,
1701  		.teardown.single	= NULL,
1702  	},
1703  	[CPUHP_WORKQUEUE_PREP] = {
1704  		.name			= "workqueue:prepare",
1705  		.startup.single		= workqueue_prepare_cpu,
1706  		.teardown.single	= NULL,
1707  	},
1708  	[CPUHP_HRTIMERS_PREPARE] = {
1709  		.name			= "hrtimers:prepare",
1710  		.startup.single		= hrtimers_prepare_cpu,
1711  		.teardown.single	= hrtimers_dead_cpu,
1712  	},
1713  	[CPUHP_SMPCFD_PREPARE] = {
1714  		.name			= "smpcfd:prepare",
1715  		.startup.single		= smpcfd_prepare_cpu,
1716  		.teardown.single	= smpcfd_dead_cpu,
1717  	},
1718  	[CPUHP_RELAY_PREPARE] = {
1719  		.name			= "relay:prepare",
1720  		.startup.single		= relay_prepare_cpu,
1721  		.teardown.single	= NULL,
1722  	},
1723  	[CPUHP_SLAB_PREPARE] = {
1724  		.name			= "slab:prepare",
1725  		.startup.single		= slab_prepare_cpu,
1726  		.teardown.single	= slab_dead_cpu,
1727  	},
1728  	[CPUHP_RCUTREE_PREP] = {
1729  		.name			= "RCU/tree:prepare",
1730  		.startup.single		= rcutree_prepare_cpu,
1731  		.teardown.single	= rcutree_dead_cpu,
1732  	},
1733  	/*
1734  	 * On the tear-down path, timers_dead_cpu() must be invoked
1735  	 * before blk_mq_queue_reinit_notify() from notify_dead(),
1736  	 * otherwise a RCU stall occurs.
1737  	 */
1738  	[CPUHP_TIMERS_PREPARE] = {
1739  		.name			= "timers:prepare",
1740  		.startup.single		= timers_prepare_cpu,
1741  		.teardown.single	= timers_dead_cpu,
1742  	},
1743  	/* Kicks the plugged cpu into life */
1744  	[CPUHP_BRINGUP_CPU] = {
1745  		.name			= "cpu:bringup",
1746  		.startup.single		= bringup_cpu,
1747  		.teardown.single	= finish_cpu,
1748  		.cant_stop		= true,
1749  	},
1750  	/* Final state before CPU kills itself */
1751  	[CPUHP_AP_IDLE_DEAD] = {
1752  		.name			= "idle:dead",
1753  	},
1754  	/*
1755  	 * Last state before CPU enters the idle loop to die. Transient state
1756  	 * for synchronization.
1757  	 */
1758  	[CPUHP_AP_OFFLINE] = {
1759  		.name			= "ap:offline",
1760  		.cant_stop		= true,
1761  	},
1762  	/* First state is scheduler control. Interrupts are disabled */
1763  	[CPUHP_AP_SCHED_STARTING] = {
1764  		.name			= "sched:starting",
1765  		.startup.single		= sched_cpu_starting,
1766  		.teardown.single	= sched_cpu_dying,
1767  	},
1768  	[CPUHP_AP_RCUTREE_DYING] = {
1769  		.name			= "RCU/tree:dying",
1770  		.startup.single		= NULL,
1771  		.teardown.single	= rcutree_dying_cpu,
1772  	},
1773  	[CPUHP_AP_SMPCFD_DYING] = {
1774  		.name			= "smpcfd:dying",
1775  		.startup.single		= NULL,
1776  		.teardown.single	= smpcfd_dying_cpu,
1777  	},
1778  	/* Entry state on starting. Interrupts enabled from here on. Transient
1779  	 * state for synchronsization */
1780  	[CPUHP_AP_ONLINE] = {
1781  		.name			= "ap:online",
1782  	},
1783  	/*
1784  	 * Handled on control processor until the plugged processor manages
1785  	 * this itself.
1786  	 */
1787  	[CPUHP_TEARDOWN_CPU] = {
1788  		.name			= "cpu:teardown",
1789  		.startup.single		= NULL,
1790  		.teardown.single	= takedown_cpu,
1791  		.cant_stop		= true,
1792  	},
1793  
1794  	[CPUHP_AP_SCHED_WAIT_EMPTY] = {
1795  		.name			= "sched:waitempty",
1796  		.startup.single		= NULL,
1797  		.teardown.single	= sched_cpu_wait_empty,
1798  	},
1799  
1800  	/* Handle smpboot threads park/unpark */
1801  	[CPUHP_AP_SMPBOOT_THREADS] = {
1802  		.name			= "smpboot/threads:online",
1803  		.startup.single		= smpboot_unpark_threads,
1804  		.teardown.single	= smpboot_park_threads,
1805  	},
1806  	[CPUHP_AP_IRQ_AFFINITY_ONLINE] = {
1807  		.name			= "irq/affinity:online",
1808  		.startup.single		= irq_affinity_online_cpu,
1809  		.teardown.single	= NULL,
1810  	},
1811  	[CPUHP_AP_PERF_ONLINE] = {
1812  		.name			= "perf:online",
1813  		.startup.single		= perf_event_init_cpu,
1814  		.teardown.single	= perf_event_exit_cpu,
1815  	},
1816  	[CPUHP_AP_WATCHDOG_ONLINE] = {
1817  		.name			= "lockup_detector:online",
1818  		.startup.single		= lockup_detector_online_cpu,
1819  		.teardown.single	= lockup_detector_offline_cpu,
1820  	},
1821  	[CPUHP_AP_WORKQUEUE_ONLINE] = {
1822  		.name			= "workqueue:online",
1823  		.startup.single		= workqueue_online_cpu,
1824  		.teardown.single	= workqueue_offline_cpu,
1825  	},
1826  	[CPUHP_AP_RANDOM_ONLINE] = {
1827  		.name			= "random:online",
1828  		.startup.single		= random_online_cpu,
1829  		.teardown.single	= NULL,
1830  	},
1831  	[CPUHP_AP_RCUTREE_ONLINE] = {
1832  		.name			= "RCU/tree:online",
1833  		.startup.single		= rcutree_online_cpu,
1834  		.teardown.single	= rcutree_offline_cpu,
1835  	},
1836  #endif
1837  	/*
1838  	 * The dynamically registered state space is here
1839  	 */
1840  
1841  #ifdef CONFIG_SMP
1842  	/* Last state is scheduler control setting the cpu active */
1843  	[CPUHP_AP_ACTIVE] = {
1844  		.name			= "sched:active",
1845  		.startup.single		= sched_cpu_activate,
1846  		.teardown.single	= sched_cpu_deactivate,
1847  	},
1848  #endif
1849  
1850  	/* CPU is fully up and running. */
1851  	[CPUHP_ONLINE] = {
1852  		.name			= "online",
1853  		.startup.single		= NULL,
1854  		.teardown.single	= NULL,
1855  	},
1856  };
1857  
1858  /* Sanity check for callbacks */
1859  static int cpuhp_cb_check(enum cpuhp_state state)
1860  {
1861  	if (state <= CPUHP_OFFLINE || state >= CPUHP_ONLINE)
1862  		return -EINVAL;
1863  	return 0;
1864  }
1865  
1866  /*
1867   * Returns a free for dynamic slot assignment of the Online state. The states
1868   * are protected by the cpuhp_slot_states mutex and an empty slot is identified
1869   * by having no name assigned.
1870   */
1871  static int cpuhp_reserve_state(enum cpuhp_state state)
1872  {
1873  	enum cpuhp_state i, end;
1874  	struct cpuhp_step *step;
1875  
1876  	switch (state) {
1877  	case CPUHP_AP_ONLINE_DYN:
1878  		step = cpuhp_hp_states + CPUHP_AP_ONLINE_DYN;
1879  		end = CPUHP_AP_ONLINE_DYN_END;
1880  		break;
1881  	case CPUHP_BP_PREPARE_DYN:
1882  		step = cpuhp_hp_states + CPUHP_BP_PREPARE_DYN;
1883  		end = CPUHP_BP_PREPARE_DYN_END;
1884  		break;
1885  	default:
1886  		return -EINVAL;
1887  	}
1888  
1889  	for (i = state; i <= end; i++, step++) {
1890  		if (!step->name)
1891  			return i;
1892  	}
1893  	WARN(1, "No more dynamic states available for CPU hotplug\n");
1894  	return -ENOSPC;
1895  }
1896  
1897  static int cpuhp_store_callbacks(enum cpuhp_state state, const char *name,
1898  				 int (*startup)(unsigned int cpu),
1899  				 int (*teardown)(unsigned int cpu),
1900  				 bool multi_instance)
1901  {
1902  	/* (Un)Install the callbacks for further cpu hotplug operations */
1903  	struct cpuhp_step *sp;
1904  	int ret = 0;
1905  
1906  	/*
1907  	 * If name is NULL, then the state gets removed.
1908  	 *
1909  	 * CPUHP_AP_ONLINE_DYN and CPUHP_BP_PREPARE_DYN are handed out on
1910  	 * the first allocation from these dynamic ranges, so the removal
1911  	 * would trigger a new allocation and clear the wrong (already
1912  	 * empty) state, leaving the callbacks of the to be cleared state
1913  	 * dangling, which causes wreckage on the next hotplug operation.
1914  	 */
1915  	if (name && (state == CPUHP_AP_ONLINE_DYN ||
1916  		     state == CPUHP_BP_PREPARE_DYN)) {
1917  		ret = cpuhp_reserve_state(state);
1918  		if (ret < 0)
1919  			return ret;
1920  		state = ret;
1921  	}
1922  	sp = cpuhp_get_step(state);
1923  	if (name && sp->name)
1924  		return -EBUSY;
1925  
1926  	sp->startup.single = startup;
1927  	sp->teardown.single = teardown;
1928  	sp->name = name;
1929  	sp->multi_instance = multi_instance;
1930  	INIT_HLIST_HEAD(&sp->list);
1931  	return ret;
1932  }
1933  
1934  static void *cpuhp_get_teardown_cb(enum cpuhp_state state)
1935  {
1936  	return cpuhp_get_step(state)->teardown.single;
1937  }
1938  
1939  /*
1940   * Call the startup/teardown function for a step either on the AP or
1941   * on the current CPU.
1942   */
1943  static int cpuhp_issue_call(int cpu, enum cpuhp_state state, bool bringup,
1944  			    struct hlist_node *node)
1945  {
1946  	struct cpuhp_step *sp = cpuhp_get_step(state);
1947  	int ret;
1948  
1949  	/*
1950  	 * If there's nothing to do, we done.
1951  	 * Relies on the union for multi_instance.
1952  	 */
1953  	if (cpuhp_step_empty(bringup, sp))
1954  		return 0;
1955  	/*
1956  	 * The non AP bound callbacks can fail on bringup. On teardown
1957  	 * e.g. module removal we crash for now.
1958  	 */
1959  #ifdef CONFIG_SMP
1960  	if (cpuhp_is_ap_state(state))
1961  		ret = cpuhp_invoke_ap_callback(cpu, state, bringup, node);
1962  	else
1963  		ret = cpuhp_invoke_callback(cpu, state, bringup, node, NULL);
1964  #else
1965  	ret = cpuhp_invoke_callback(cpu, state, bringup, node, NULL);
1966  #endif
1967  	BUG_ON(ret && !bringup);
1968  	return ret;
1969  }
1970  
1971  /*
1972   * Called from __cpuhp_setup_state on a recoverable failure.
1973   *
1974   * Note: The teardown callbacks for rollback are not allowed to fail!
1975   */
1976  static void cpuhp_rollback_install(int failedcpu, enum cpuhp_state state,
1977  				   struct hlist_node *node)
1978  {
1979  	int cpu;
1980  
1981  	/* Roll back the already executed steps on the other cpus */
1982  	for_each_present_cpu(cpu) {
1983  		struct cpuhp_cpu_state *st = per_cpu_ptr(&cpuhp_state, cpu);
1984  		int cpustate = st->state;
1985  
1986  		if (cpu >= failedcpu)
1987  			break;
1988  
1989  		/* Did we invoke the startup call on that cpu ? */
1990  		if (cpustate >= state)
1991  			cpuhp_issue_call(cpu, state, false, node);
1992  	}
1993  }
1994  
1995  int __cpuhp_state_add_instance_cpuslocked(enum cpuhp_state state,
1996  					  struct hlist_node *node,
1997  					  bool invoke)
1998  {
1999  	struct cpuhp_step *sp;
2000  	int cpu;
2001  	int ret;
2002  
2003  	lockdep_assert_cpus_held();
2004  
2005  	sp = cpuhp_get_step(state);
2006  	if (sp->multi_instance == false)
2007  		return -EINVAL;
2008  
2009  	mutex_lock(&cpuhp_state_mutex);
2010  
2011  	if (!invoke || !sp->startup.multi)
2012  		goto add_node;
2013  
2014  	/*
2015  	 * Try to call the startup callback for each present cpu
2016  	 * depending on the hotplug state of the cpu.
2017  	 */
2018  	for_each_present_cpu(cpu) {
2019  		struct cpuhp_cpu_state *st = per_cpu_ptr(&cpuhp_state, cpu);
2020  		int cpustate = st->state;
2021  
2022  		if (cpustate < state)
2023  			continue;
2024  
2025  		ret = cpuhp_issue_call(cpu, state, true, node);
2026  		if (ret) {
2027  			if (sp->teardown.multi)
2028  				cpuhp_rollback_install(cpu, state, node);
2029  			goto unlock;
2030  		}
2031  	}
2032  add_node:
2033  	ret = 0;
2034  	hlist_add_head(node, &sp->list);
2035  unlock:
2036  	mutex_unlock(&cpuhp_state_mutex);
2037  	return ret;
2038  }
2039  
2040  int __cpuhp_state_add_instance(enum cpuhp_state state, struct hlist_node *node,
2041  			       bool invoke)
2042  {
2043  	int ret;
2044  
2045  	cpus_read_lock();
2046  	ret = __cpuhp_state_add_instance_cpuslocked(state, node, invoke);
2047  	cpus_read_unlock();
2048  	return ret;
2049  }
2050  EXPORT_SYMBOL_GPL(__cpuhp_state_add_instance);
2051  
2052  /**
2053   * __cpuhp_setup_state_cpuslocked - Setup the callbacks for an hotplug machine state
2054   * @state:		The state to setup
2055   * @name:		Name of the step
2056   * @invoke:		If true, the startup function is invoked for cpus where
2057   *			cpu state >= @state
2058   * @startup:		startup callback function
2059   * @teardown:		teardown callback function
2060   * @multi_instance:	State is set up for multiple instances which get
2061   *			added afterwards.
2062   *
2063   * The caller needs to hold cpus read locked while calling this function.
2064   * Return:
2065   *   On success:
2066   *      Positive state number if @state is CPUHP_AP_ONLINE_DYN;
2067   *      0 for all other states
2068   *   On failure: proper (negative) error code
2069   */
2070  int __cpuhp_setup_state_cpuslocked(enum cpuhp_state state,
2071  				   const char *name, bool invoke,
2072  				   int (*startup)(unsigned int cpu),
2073  				   int (*teardown)(unsigned int cpu),
2074  				   bool multi_instance)
2075  {
2076  	int cpu, ret = 0;
2077  	bool dynstate;
2078  
2079  	lockdep_assert_cpus_held();
2080  
2081  	if (cpuhp_cb_check(state) || !name)
2082  		return -EINVAL;
2083  
2084  	mutex_lock(&cpuhp_state_mutex);
2085  
2086  	ret = cpuhp_store_callbacks(state, name, startup, teardown,
2087  				    multi_instance);
2088  
2089  	dynstate = state == CPUHP_AP_ONLINE_DYN;
2090  	if (ret > 0 && dynstate) {
2091  		state = ret;
2092  		ret = 0;
2093  	}
2094  
2095  	if (ret || !invoke || !startup)
2096  		goto out;
2097  
2098  	/*
2099  	 * Try to call the startup callback for each present cpu
2100  	 * depending on the hotplug state of the cpu.
2101  	 */
2102  	for_each_present_cpu(cpu) {
2103  		struct cpuhp_cpu_state *st = per_cpu_ptr(&cpuhp_state, cpu);
2104  		int cpustate = st->state;
2105  
2106  		if (cpustate < state)
2107  			continue;
2108  
2109  		ret = cpuhp_issue_call(cpu, state, true, NULL);
2110  		if (ret) {
2111  			if (teardown)
2112  				cpuhp_rollback_install(cpu, state, NULL);
2113  			cpuhp_store_callbacks(state, NULL, NULL, NULL, false);
2114  			goto out;
2115  		}
2116  	}
2117  out:
2118  	mutex_unlock(&cpuhp_state_mutex);
2119  	/*
2120  	 * If the requested state is CPUHP_AP_ONLINE_DYN, return the
2121  	 * dynamically allocated state in case of success.
2122  	 */
2123  	if (!ret && dynstate)
2124  		return state;
2125  	return ret;
2126  }
2127  EXPORT_SYMBOL(__cpuhp_setup_state_cpuslocked);
2128  
2129  int __cpuhp_setup_state(enum cpuhp_state state,
2130  			const char *name, bool invoke,
2131  			int (*startup)(unsigned int cpu),
2132  			int (*teardown)(unsigned int cpu),
2133  			bool multi_instance)
2134  {
2135  	int ret;
2136  
2137  	cpus_read_lock();
2138  	ret = __cpuhp_setup_state_cpuslocked(state, name, invoke, startup,
2139  					     teardown, multi_instance);
2140  	cpus_read_unlock();
2141  	return ret;
2142  }
2143  EXPORT_SYMBOL(__cpuhp_setup_state);
2144  
2145  int __cpuhp_state_remove_instance(enum cpuhp_state state,
2146  				  struct hlist_node *node, bool invoke)
2147  {
2148  	struct cpuhp_step *sp = cpuhp_get_step(state);
2149  	int cpu;
2150  
2151  	BUG_ON(cpuhp_cb_check(state));
2152  
2153  	if (!sp->multi_instance)
2154  		return -EINVAL;
2155  
2156  	cpus_read_lock();
2157  	mutex_lock(&cpuhp_state_mutex);
2158  
2159  	if (!invoke || !cpuhp_get_teardown_cb(state))
2160  		goto remove;
2161  	/*
2162  	 * Call the teardown callback for each present cpu depending
2163  	 * on the hotplug state of the cpu. This function is not
2164  	 * allowed to fail currently!
2165  	 */
2166  	for_each_present_cpu(cpu) {
2167  		struct cpuhp_cpu_state *st = per_cpu_ptr(&cpuhp_state, cpu);
2168  		int cpustate = st->state;
2169  
2170  		if (cpustate >= state)
2171  			cpuhp_issue_call(cpu, state, false, node);
2172  	}
2173  
2174  remove:
2175  	hlist_del(node);
2176  	mutex_unlock(&cpuhp_state_mutex);
2177  	cpus_read_unlock();
2178  
2179  	return 0;
2180  }
2181  EXPORT_SYMBOL_GPL(__cpuhp_state_remove_instance);
2182  
2183  /**
2184   * __cpuhp_remove_state_cpuslocked - Remove the callbacks for an hotplug machine state
2185   * @state:	The state to remove
2186   * @invoke:	If true, the teardown function is invoked for cpus where
2187   *		cpu state >= @state
2188   *
2189   * The caller needs to hold cpus read locked while calling this function.
2190   * The teardown callback is currently not allowed to fail. Think
2191   * about module removal!
2192   */
2193  void __cpuhp_remove_state_cpuslocked(enum cpuhp_state state, bool invoke)
2194  {
2195  	struct cpuhp_step *sp = cpuhp_get_step(state);
2196  	int cpu;
2197  
2198  	BUG_ON(cpuhp_cb_check(state));
2199  
2200  	lockdep_assert_cpus_held();
2201  
2202  	mutex_lock(&cpuhp_state_mutex);
2203  	if (sp->multi_instance) {
2204  		WARN(!hlist_empty(&sp->list),
2205  		     "Error: Removing state %d which has instances left.\n",
2206  		     state);
2207  		goto remove;
2208  	}
2209  
2210  	if (!invoke || !cpuhp_get_teardown_cb(state))
2211  		goto remove;
2212  
2213  	/*
2214  	 * Call the teardown callback for each present cpu depending
2215  	 * on the hotplug state of the cpu. This function is not
2216  	 * allowed to fail currently!
2217  	 */
2218  	for_each_present_cpu(cpu) {
2219  		struct cpuhp_cpu_state *st = per_cpu_ptr(&cpuhp_state, cpu);
2220  		int cpustate = st->state;
2221  
2222  		if (cpustate >= state)
2223  			cpuhp_issue_call(cpu, state, false, NULL);
2224  	}
2225  remove:
2226  	cpuhp_store_callbacks(state, NULL, NULL, NULL, false);
2227  	mutex_unlock(&cpuhp_state_mutex);
2228  }
2229  EXPORT_SYMBOL(__cpuhp_remove_state_cpuslocked);
2230  
2231  void __cpuhp_remove_state(enum cpuhp_state state, bool invoke)
2232  {
2233  	cpus_read_lock();
2234  	__cpuhp_remove_state_cpuslocked(state, invoke);
2235  	cpus_read_unlock();
2236  }
2237  EXPORT_SYMBOL(__cpuhp_remove_state);
2238  
2239  #ifdef CONFIG_HOTPLUG_SMT
2240  static void cpuhp_offline_cpu_device(unsigned int cpu)
2241  {
2242  	struct device *dev = get_cpu_device(cpu);
2243  
2244  	dev->offline = true;
2245  	/* Tell user space about the state change */
2246  	kobject_uevent(&dev->kobj, KOBJ_OFFLINE);
2247  }
2248  
2249  static void cpuhp_online_cpu_device(unsigned int cpu)
2250  {
2251  	struct device *dev = get_cpu_device(cpu);
2252  
2253  	dev->offline = false;
2254  	/* Tell user space about the state change */
2255  	kobject_uevent(&dev->kobj, KOBJ_ONLINE);
2256  }
2257  
2258  int cpuhp_smt_disable(enum cpuhp_smt_control ctrlval)
2259  {
2260  	int cpu, ret = 0;
2261  
2262  	cpu_maps_update_begin();
2263  	for_each_online_cpu(cpu) {
2264  		if (topology_is_primary_thread(cpu))
2265  			continue;
2266  		ret = cpu_down_maps_locked(cpu, CPUHP_OFFLINE);
2267  		if (ret)
2268  			break;
2269  		/*
2270  		 * As this needs to hold the cpu maps lock it's impossible
2271  		 * to call device_offline() because that ends up calling
2272  		 * cpu_down() which takes cpu maps lock. cpu maps lock
2273  		 * needs to be held as this might race against in kernel
2274  		 * abusers of the hotplug machinery (thermal management).
2275  		 *
2276  		 * So nothing would update device:offline state. That would
2277  		 * leave the sysfs entry stale and prevent onlining after
2278  		 * smt control has been changed to 'off' again. This is
2279  		 * called under the sysfs hotplug lock, so it is properly
2280  		 * serialized against the regular offline usage.
2281  		 */
2282  		cpuhp_offline_cpu_device(cpu);
2283  	}
2284  	if (!ret)
2285  		cpu_smt_control = ctrlval;
2286  	cpu_maps_update_done();
2287  	return ret;
2288  }
2289  
2290  int cpuhp_smt_enable(void)
2291  {
2292  	int cpu, ret = 0;
2293  
2294  	cpu_maps_update_begin();
2295  	cpu_smt_control = CPU_SMT_ENABLED;
2296  	for_each_present_cpu(cpu) {
2297  		/* Skip online CPUs and CPUs on offline nodes */
2298  		if (cpu_online(cpu) || !node_online(cpu_to_node(cpu)))
2299  			continue;
2300  		ret = _cpu_up(cpu, 0, CPUHP_ONLINE);
2301  		if (ret)
2302  			break;
2303  		/* See comment in cpuhp_smt_disable() */
2304  		cpuhp_online_cpu_device(cpu);
2305  	}
2306  	cpu_maps_update_done();
2307  	return ret;
2308  }
2309  #endif
2310  
2311  #if defined(CONFIG_SYSFS) && defined(CONFIG_HOTPLUG_CPU)
2312  static ssize_t state_show(struct device *dev,
2313  			  struct device_attribute *attr, char *buf)
2314  {
2315  	struct cpuhp_cpu_state *st = per_cpu_ptr(&cpuhp_state, dev->id);
2316  
2317  	return sprintf(buf, "%d\n", st->state);
2318  }
2319  static DEVICE_ATTR_RO(state);
2320  
2321  static ssize_t target_store(struct device *dev, struct device_attribute *attr,
2322  			    const char *buf, size_t count)
2323  {
2324  	struct cpuhp_cpu_state *st = per_cpu_ptr(&cpuhp_state, dev->id);
2325  	struct cpuhp_step *sp;
2326  	int target, ret;
2327  
2328  	ret = kstrtoint(buf, 10, &target);
2329  	if (ret)
2330  		return ret;
2331  
2332  #ifdef CONFIG_CPU_HOTPLUG_STATE_CONTROL
2333  	if (target < CPUHP_OFFLINE || target > CPUHP_ONLINE)
2334  		return -EINVAL;
2335  #else
2336  	if (target != CPUHP_OFFLINE && target != CPUHP_ONLINE)
2337  		return -EINVAL;
2338  #endif
2339  
2340  	ret = lock_device_hotplug_sysfs();
2341  	if (ret)
2342  		return ret;
2343  
2344  	mutex_lock(&cpuhp_state_mutex);
2345  	sp = cpuhp_get_step(target);
2346  	ret = !sp->name || sp->cant_stop ? -EINVAL : 0;
2347  	mutex_unlock(&cpuhp_state_mutex);
2348  	if (ret)
2349  		goto out;
2350  
2351  	if (st->state < target)
2352  		ret = cpu_up(dev->id, target);
2353  	else if (st->state > target)
2354  		ret = cpu_down(dev->id, target);
2355  	else if (WARN_ON(st->target != target))
2356  		st->target = target;
2357  out:
2358  	unlock_device_hotplug();
2359  	return ret ? ret : count;
2360  }
2361  
2362  static ssize_t target_show(struct device *dev,
2363  			   struct device_attribute *attr, char *buf)
2364  {
2365  	struct cpuhp_cpu_state *st = per_cpu_ptr(&cpuhp_state, dev->id);
2366  
2367  	return sprintf(buf, "%d\n", st->target);
2368  }
2369  static DEVICE_ATTR_RW(target);
2370  
2371  static ssize_t fail_store(struct device *dev, struct device_attribute *attr,
2372  			  const char *buf, size_t count)
2373  {
2374  	struct cpuhp_cpu_state *st = per_cpu_ptr(&cpuhp_state, dev->id);
2375  	struct cpuhp_step *sp;
2376  	int fail, ret;
2377  
2378  	ret = kstrtoint(buf, 10, &fail);
2379  	if (ret)
2380  		return ret;
2381  
2382  	if (fail == CPUHP_INVALID) {
2383  		st->fail = fail;
2384  		return count;
2385  	}
2386  
2387  	if (fail < CPUHP_OFFLINE || fail > CPUHP_ONLINE)
2388  		return -EINVAL;
2389  
2390  	/*
2391  	 * Cannot fail STARTING/DYING callbacks.
2392  	 */
2393  	if (cpuhp_is_atomic_state(fail))
2394  		return -EINVAL;
2395  
2396  	/*
2397  	 * DEAD callbacks cannot fail...
2398  	 * ... neither can CPUHP_BRINGUP_CPU during hotunplug. The latter
2399  	 * triggering STARTING callbacks, a failure in this state would
2400  	 * hinder rollback.
2401  	 */
2402  	if (fail <= CPUHP_BRINGUP_CPU && st->state > CPUHP_BRINGUP_CPU)
2403  		return -EINVAL;
2404  
2405  	/*
2406  	 * Cannot fail anything that doesn't have callbacks.
2407  	 */
2408  	mutex_lock(&cpuhp_state_mutex);
2409  	sp = cpuhp_get_step(fail);
2410  	if (!sp->startup.single && !sp->teardown.single)
2411  		ret = -EINVAL;
2412  	mutex_unlock(&cpuhp_state_mutex);
2413  	if (ret)
2414  		return ret;
2415  
2416  	st->fail = fail;
2417  
2418  	return count;
2419  }
2420  
2421  static ssize_t fail_show(struct device *dev,
2422  			 struct device_attribute *attr, char *buf)
2423  {
2424  	struct cpuhp_cpu_state *st = per_cpu_ptr(&cpuhp_state, dev->id);
2425  
2426  	return sprintf(buf, "%d\n", st->fail);
2427  }
2428  
2429  static DEVICE_ATTR_RW(fail);
2430  
2431  static struct attribute *cpuhp_cpu_attrs[] = {
2432  	&dev_attr_state.attr,
2433  	&dev_attr_target.attr,
2434  	&dev_attr_fail.attr,
2435  	NULL
2436  };
2437  
2438  static const struct attribute_group cpuhp_cpu_attr_group = {
2439  	.attrs = cpuhp_cpu_attrs,
2440  	.name = "hotplug",
2441  	NULL
2442  };
2443  
2444  static ssize_t states_show(struct device *dev,
2445  				 struct device_attribute *attr, char *buf)
2446  {
2447  	ssize_t cur, res = 0;
2448  	int i;
2449  
2450  	mutex_lock(&cpuhp_state_mutex);
2451  	for (i = CPUHP_OFFLINE; i <= CPUHP_ONLINE; i++) {
2452  		struct cpuhp_step *sp = cpuhp_get_step(i);
2453  
2454  		if (sp->name) {
2455  			cur = sprintf(buf, "%3d: %s\n", i, sp->name);
2456  			buf += cur;
2457  			res += cur;
2458  		}
2459  	}
2460  	mutex_unlock(&cpuhp_state_mutex);
2461  	return res;
2462  }
2463  static DEVICE_ATTR_RO(states);
2464  
2465  static struct attribute *cpuhp_cpu_root_attrs[] = {
2466  	&dev_attr_states.attr,
2467  	NULL
2468  };
2469  
2470  static const struct attribute_group cpuhp_cpu_root_attr_group = {
2471  	.attrs = cpuhp_cpu_root_attrs,
2472  	.name = "hotplug",
2473  	NULL
2474  };
2475  
2476  #ifdef CONFIG_HOTPLUG_SMT
2477  
2478  static ssize_t
2479  __store_smt_control(struct device *dev, struct device_attribute *attr,
2480  		    const char *buf, size_t count)
2481  {
2482  	int ctrlval, ret;
2483  
2484  	if (sysfs_streq(buf, "on"))
2485  		ctrlval = CPU_SMT_ENABLED;
2486  	else if (sysfs_streq(buf, "off"))
2487  		ctrlval = CPU_SMT_DISABLED;
2488  	else if (sysfs_streq(buf, "forceoff"))
2489  		ctrlval = CPU_SMT_FORCE_DISABLED;
2490  	else
2491  		return -EINVAL;
2492  
2493  	if (cpu_smt_control == CPU_SMT_FORCE_DISABLED)
2494  		return -EPERM;
2495  
2496  	if (cpu_smt_control == CPU_SMT_NOT_SUPPORTED)
2497  		return -ENODEV;
2498  
2499  	ret = lock_device_hotplug_sysfs();
2500  	if (ret)
2501  		return ret;
2502  
2503  	if (ctrlval != cpu_smt_control) {
2504  		switch (ctrlval) {
2505  		case CPU_SMT_ENABLED:
2506  			ret = cpuhp_smt_enable();
2507  			break;
2508  		case CPU_SMT_DISABLED:
2509  		case CPU_SMT_FORCE_DISABLED:
2510  			ret = cpuhp_smt_disable(ctrlval);
2511  			break;
2512  		}
2513  	}
2514  
2515  	unlock_device_hotplug();
2516  	return ret ? ret : count;
2517  }
2518  
2519  #else /* !CONFIG_HOTPLUG_SMT */
2520  static ssize_t
2521  __store_smt_control(struct device *dev, struct device_attribute *attr,
2522  		    const char *buf, size_t count)
2523  {
2524  	return -ENODEV;
2525  }
2526  #endif /* CONFIG_HOTPLUG_SMT */
2527  
2528  static const char *smt_states[] = {
2529  	[CPU_SMT_ENABLED]		= "on",
2530  	[CPU_SMT_DISABLED]		= "off",
2531  	[CPU_SMT_FORCE_DISABLED]	= "forceoff",
2532  	[CPU_SMT_NOT_SUPPORTED]		= "notsupported",
2533  	[CPU_SMT_NOT_IMPLEMENTED]	= "notimplemented",
2534  };
2535  
2536  static ssize_t control_show(struct device *dev,
2537  			    struct device_attribute *attr, char *buf)
2538  {
2539  	const char *state = smt_states[cpu_smt_control];
2540  
2541  	return snprintf(buf, PAGE_SIZE - 2, "%s\n", state);
2542  }
2543  
2544  static ssize_t control_store(struct device *dev, struct device_attribute *attr,
2545  			     const char *buf, size_t count)
2546  {
2547  	return __store_smt_control(dev, attr, buf, count);
2548  }
2549  static DEVICE_ATTR_RW(control);
2550  
2551  static ssize_t active_show(struct device *dev,
2552  			   struct device_attribute *attr, char *buf)
2553  {
2554  	return snprintf(buf, PAGE_SIZE - 2, "%d\n", sched_smt_active());
2555  }
2556  static DEVICE_ATTR_RO(active);
2557  
2558  static struct attribute *cpuhp_smt_attrs[] = {
2559  	&dev_attr_control.attr,
2560  	&dev_attr_active.attr,
2561  	NULL
2562  };
2563  
2564  static const struct attribute_group cpuhp_smt_attr_group = {
2565  	.attrs = cpuhp_smt_attrs,
2566  	.name = "smt",
2567  	NULL
2568  };
2569  
2570  static int __init cpu_smt_sysfs_init(void)
2571  {
2572  	return sysfs_create_group(&cpu_subsys.dev_root->kobj,
2573  				  &cpuhp_smt_attr_group);
2574  }
2575  
2576  static int __init cpuhp_sysfs_init(void)
2577  {
2578  	int cpu, ret;
2579  
2580  	ret = cpu_smt_sysfs_init();
2581  	if (ret)
2582  		return ret;
2583  
2584  	ret = sysfs_create_group(&cpu_subsys.dev_root->kobj,
2585  				 &cpuhp_cpu_root_attr_group);
2586  	if (ret)
2587  		return ret;
2588  
2589  	for_each_possible_cpu(cpu) {
2590  		struct device *dev = get_cpu_device(cpu);
2591  
2592  		if (!dev)
2593  			continue;
2594  		ret = sysfs_create_group(&dev->kobj, &cpuhp_cpu_attr_group);
2595  		if (ret)
2596  			return ret;
2597  	}
2598  	return 0;
2599  }
2600  device_initcall(cpuhp_sysfs_init);
2601  #endif /* CONFIG_SYSFS && CONFIG_HOTPLUG_CPU */
2602  
2603  /*
2604   * cpu_bit_bitmap[] is a special, "compressed" data structure that
2605   * represents all NR_CPUS bits binary values of 1<<nr.
2606   *
2607   * It is used by cpumask_of() to get a constant address to a CPU
2608   * mask value that has a single bit set only.
2609   */
2610  
2611  /* cpu_bit_bitmap[0] is empty - so we can back into it */
2612  #define MASK_DECLARE_1(x)	[x+1][0] = (1UL << (x))
2613  #define MASK_DECLARE_2(x)	MASK_DECLARE_1(x), MASK_DECLARE_1(x+1)
2614  #define MASK_DECLARE_4(x)	MASK_DECLARE_2(x), MASK_DECLARE_2(x+2)
2615  #define MASK_DECLARE_8(x)	MASK_DECLARE_4(x), MASK_DECLARE_4(x+4)
2616  
2617  const unsigned long cpu_bit_bitmap[BITS_PER_LONG+1][BITS_TO_LONGS(NR_CPUS)] = {
2618  
2619  	MASK_DECLARE_8(0),	MASK_DECLARE_8(8),
2620  	MASK_DECLARE_8(16),	MASK_DECLARE_8(24),
2621  #if BITS_PER_LONG > 32
2622  	MASK_DECLARE_8(32),	MASK_DECLARE_8(40),
2623  	MASK_DECLARE_8(48),	MASK_DECLARE_8(56),
2624  #endif
2625  };
2626  EXPORT_SYMBOL_GPL(cpu_bit_bitmap);
2627  
2628  const DECLARE_BITMAP(cpu_all_bits, NR_CPUS) = CPU_BITS_ALL;
2629  EXPORT_SYMBOL(cpu_all_bits);
2630  
2631  #ifdef CONFIG_INIT_ALL_POSSIBLE
2632  struct cpumask __cpu_possible_mask __read_mostly
2633  	= {CPU_BITS_ALL};
2634  #else
2635  struct cpumask __cpu_possible_mask __read_mostly;
2636  #endif
2637  EXPORT_SYMBOL(__cpu_possible_mask);
2638  
2639  struct cpumask __cpu_online_mask __read_mostly;
2640  EXPORT_SYMBOL(__cpu_online_mask);
2641  
2642  struct cpumask __cpu_present_mask __read_mostly;
2643  EXPORT_SYMBOL(__cpu_present_mask);
2644  
2645  struct cpumask __cpu_active_mask __read_mostly;
2646  EXPORT_SYMBOL(__cpu_active_mask);
2647  
2648  struct cpumask __cpu_dying_mask __read_mostly;
2649  EXPORT_SYMBOL(__cpu_dying_mask);
2650  
2651  atomic_t __num_online_cpus __read_mostly;
2652  EXPORT_SYMBOL(__num_online_cpus);
2653  
2654  void init_cpu_present(const struct cpumask *src)
2655  {
2656  	cpumask_copy(&__cpu_present_mask, src);
2657  }
2658  
2659  void init_cpu_possible(const struct cpumask *src)
2660  {
2661  	cpumask_copy(&__cpu_possible_mask, src);
2662  }
2663  
2664  void init_cpu_online(const struct cpumask *src)
2665  {
2666  	cpumask_copy(&__cpu_online_mask, src);
2667  }
2668  
2669  void set_cpu_online(unsigned int cpu, bool online)
2670  {
2671  	/*
2672  	 * atomic_inc/dec() is required to handle the horrid abuse of this
2673  	 * function by the reboot and kexec code which invoke it from
2674  	 * IPI/NMI broadcasts when shutting down CPUs. Invocation from
2675  	 * regular CPU hotplug is properly serialized.
2676  	 *
2677  	 * Note, that the fact that __num_online_cpus is of type atomic_t
2678  	 * does not protect readers which are not serialized against
2679  	 * concurrent hotplug operations.
2680  	 */
2681  	if (online) {
2682  		if (!cpumask_test_and_set_cpu(cpu, &__cpu_online_mask))
2683  			atomic_inc(&__num_online_cpus);
2684  	} else {
2685  		if (cpumask_test_and_clear_cpu(cpu, &__cpu_online_mask))
2686  			atomic_dec(&__num_online_cpus);
2687  	}
2688  }
2689  
2690  /*
2691   * Activate the first processor.
2692   */
2693  void __init boot_cpu_init(void)
2694  {
2695  	int cpu = smp_processor_id();
2696  
2697  	/* Mark the boot cpu "present", "online" etc for SMP and UP case */
2698  	set_cpu_online(cpu, true);
2699  	set_cpu_active(cpu, true);
2700  	set_cpu_present(cpu, true);
2701  	set_cpu_possible(cpu, true);
2702  
2703  #ifdef CONFIG_SMP
2704  	__boot_cpu_id = cpu;
2705  #endif
2706  }
2707  
2708  /*
2709   * Must be called _AFTER_ setting up the per_cpu areas
2710   */
2711  void __init boot_cpu_hotplug_init(void)
2712  {
2713  #ifdef CONFIG_SMP
2714  	cpumask_set_cpu(smp_processor_id(), &cpus_booted_once_mask);
2715  #endif
2716  	this_cpu_write(cpuhp_state.state, CPUHP_ONLINE);
2717  	this_cpu_write(cpuhp_state.target, CPUHP_ONLINE);
2718  }
2719  
2720  /*
2721   * These are used for a global "mitigations=" cmdline option for toggling
2722   * optional CPU mitigations.
2723   */
2724  enum cpu_mitigations {
2725  	CPU_MITIGATIONS_OFF,
2726  	CPU_MITIGATIONS_AUTO,
2727  	CPU_MITIGATIONS_AUTO_NOSMT,
2728  };
2729  
2730  static enum cpu_mitigations cpu_mitigations __ro_after_init =
2731  	CPU_MITIGATIONS_AUTO;
2732  
2733  static int __init mitigations_parse_cmdline(char *arg)
2734  {
2735  	if (!strcmp(arg, "off"))
2736  		cpu_mitigations = CPU_MITIGATIONS_OFF;
2737  	else if (!strcmp(arg, "auto"))
2738  		cpu_mitigations = CPU_MITIGATIONS_AUTO;
2739  	else if (!strcmp(arg, "auto,nosmt"))
2740  		cpu_mitigations = CPU_MITIGATIONS_AUTO_NOSMT;
2741  	else
2742  		pr_crit("Unsupported mitigations=%s, system may still be vulnerable\n",
2743  			arg);
2744  
2745  	return 0;
2746  }
2747  early_param("mitigations", mitigations_parse_cmdline);
2748  
2749  /* mitigations=off */
2750  bool cpu_mitigations_off(void)
2751  {
2752  	return cpu_mitigations == CPU_MITIGATIONS_OFF;
2753  }
2754  EXPORT_SYMBOL_GPL(cpu_mitigations_off);
2755  
2756  /* mitigations=auto,nosmt */
2757  bool cpu_mitigations_auto_nosmt(void)
2758  {
2759  	return cpu_mitigations == CPU_MITIGATIONS_AUTO_NOSMT;
2760  }
2761  EXPORT_SYMBOL_GPL(cpu_mitigations_auto_nosmt);
2762