xref: /openbmc/linux/kernel/cpu.c (revision 2f0754f2)
1 /* CPU control.
2  * (C) 2001, 2002, 2003, 2004 Rusty Russell
3  *
4  * This code is licenced under the GPL.
5  */
6 #include <linux/sched/mm.h>
7 #include <linux/proc_fs.h>
8 #include <linux/smp.h>
9 #include <linux/init.h>
10 #include <linux/notifier.h>
11 #include <linux/sched/signal.h>
12 #include <linux/sched/hotplug.h>
13 #include <linux/sched/isolation.h>
14 #include <linux/sched/task.h>
15 #include <linux/sched/smt.h>
16 #include <linux/unistd.h>
17 #include <linux/cpu.h>
18 #include <linux/oom.h>
19 #include <linux/rcupdate.h>
20 #include <linux/export.h>
21 #include <linux/bug.h>
22 #include <linux/kthread.h>
23 #include <linux/stop_machine.h>
24 #include <linux/mutex.h>
25 #include <linux/gfp.h>
26 #include <linux/suspend.h>
27 #include <linux/lockdep.h>
28 #include <linux/tick.h>
29 #include <linux/irq.h>
30 #include <linux/nmi.h>
31 #include <linux/smpboot.h>
32 #include <linux/relay.h>
33 #include <linux/slab.h>
34 #include <linux/scs.h>
35 #include <linux/percpu-rwsem.h>
36 #include <linux/cpuset.h>
37 
38 #include <trace/events/power.h>
39 #define CREATE_TRACE_POINTS
40 #include <trace/events/cpuhp.h>
41 
42 #include "smpboot.h"
43 
44 /**
45  * struct cpuhp_cpu_state - Per cpu hotplug state storage
46  * @state:	The current cpu state
47  * @target:	The target state
48  * @fail:	Current CPU hotplug callback state
49  * @thread:	Pointer to the hotplug thread
50  * @should_run:	Thread should execute
51  * @rollback:	Perform a rollback
52  * @single:	Single callback invocation
53  * @bringup:	Single callback bringup or teardown selector
54  * @cpu:	CPU number
55  * @node:	Remote CPU node; for multi-instance, do a
56  *		single entry callback for install/remove
57  * @last:	For multi-instance rollback, remember how far we got
58  * @cb_state:	The state for a single callback (install/uninstall)
59  * @result:	Result of the operation
60  * @done_up:	Signal completion to the issuer of the task for cpu-up
61  * @done_down:	Signal completion to the issuer of the task for cpu-down
62  */
63 struct cpuhp_cpu_state {
64 	enum cpuhp_state	state;
65 	enum cpuhp_state	target;
66 	enum cpuhp_state	fail;
67 #ifdef CONFIG_SMP
68 	struct task_struct	*thread;
69 	bool			should_run;
70 	bool			rollback;
71 	bool			single;
72 	bool			bringup;
73 	int			cpu;
74 	struct hlist_node	*node;
75 	struct hlist_node	*last;
76 	enum cpuhp_state	cb_state;
77 	int			result;
78 	struct completion	done_up;
79 	struct completion	done_down;
80 #endif
81 };
82 
83 static DEFINE_PER_CPU(struct cpuhp_cpu_state, cpuhp_state) = {
84 	.fail = CPUHP_INVALID,
85 };
86 
87 #ifdef CONFIG_SMP
88 cpumask_t cpus_booted_once_mask;
89 #endif
90 
91 #if defined(CONFIG_LOCKDEP) && defined(CONFIG_SMP)
92 static struct lockdep_map cpuhp_state_up_map =
93 	STATIC_LOCKDEP_MAP_INIT("cpuhp_state-up", &cpuhp_state_up_map);
94 static struct lockdep_map cpuhp_state_down_map =
95 	STATIC_LOCKDEP_MAP_INIT("cpuhp_state-down", &cpuhp_state_down_map);
96 
97 
98 static inline void cpuhp_lock_acquire(bool bringup)
99 {
100 	lock_map_acquire(bringup ? &cpuhp_state_up_map : &cpuhp_state_down_map);
101 }
102 
103 static inline void cpuhp_lock_release(bool bringup)
104 {
105 	lock_map_release(bringup ? &cpuhp_state_up_map : &cpuhp_state_down_map);
106 }
107 #else
108 
109 static inline void cpuhp_lock_acquire(bool bringup) { }
110 static inline void cpuhp_lock_release(bool bringup) { }
111 
112 #endif
113 
114 /**
115  * struct cpuhp_step - Hotplug state machine step
116  * @name:	Name of the step
117  * @startup:	Startup function of the step
118  * @teardown:	Teardown function of the step
119  * @cant_stop:	Bringup/teardown can't be stopped at this step
120  * @multi_instance:	State has multiple instances which get added afterwards
121  */
122 struct cpuhp_step {
123 	const char		*name;
124 	union {
125 		int		(*single)(unsigned int cpu);
126 		int		(*multi)(unsigned int cpu,
127 					 struct hlist_node *node);
128 	} startup;
129 	union {
130 		int		(*single)(unsigned int cpu);
131 		int		(*multi)(unsigned int cpu,
132 					 struct hlist_node *node);
133 	} teardown;
134 	/* private: */
135 	struct hlist_head	list;
136 	/* public: */
137 	bool			cant_stop;
138 	bool			multi_instance;
139 };
140 
141 static DEFINE_MUTEX(cpuhp_state_mutex);
142 static struct cpuhp_step cpuhp_hp_states[];
143 
144 static struct cpuhp_step *cpuhp_get_step(enum cpuhp_state state)
145 {
146 	return cpuhp_hp_states + state;
147 }
148 
149 static bool cpuhp_step_empty(bool bringup, struct cpuhp_step *step)
150 {
151 	return bringup ? !step->startup.single : !step->teardown.single;
152 }
153 
154 /**
155  * cpuhp_invoke_callback - Invoke the callbacks for a given state
156  * @cpu:	The cpu for which the callback should be invoked
157  * @state:	The state to do callbacks for
158  * @bringup:	True if the bringup callback should be invoked
159  * @node:	For multi-instance, do a single entry callback for install/remove
160  * @lastp:	For multi-instance rollback, remember how far we got
161  *
162  * Called from cpu hotplug and from the state register machinery.
163  *
164  * Return: %0 on success or a negative errno code
165  */
166 static int cpuhp_invoke_callback(unsigned int cpu, enum cpuhp_state state,
167 				 bool bringup, struct hlist_node *node,
168 				 struct hlist_node **lastp)
169 {
170 	struct cpuhp_cpu_state *st = per_cpu_ptr(&cpuhp_state, cpu);
171 	struct cpuhp_step *step = cpuhp_get_step(state);
172 	int (*cbm)(unsigned int cpu, struct hlist_node *node);
173 	int (*cb)(unsigned int cpu);
174 	int ret, cnt;
175 
176 	if (st->fail == state) {
177 		st->fail = CPUHP_INVALID;
178 		return -EAGAIN;
179 	}
180 
181 	if (cpuhp_step_empty(bringup, step)) {
182 		WARN_ON_ONCE(1);
183 		return 0;
184 	}
185 
186 	if (!step->multi_instance) {
187 		WARN_ON_ONCE(lastp && *lastp);
188 		cb = bringup ? step->startup.single : step->teardown.single;
189 
190 		trace_cpuhp_enter(cpu, st->target, state, cb);
191 		ret = cb(cpu);
192 		trace_cpuhp_exit(cpu, st->state, state, ret);
193 		return ret;
194 	}
195 	cbm = bringup ? step->startup.multi : step->teardown.multi;
196 
197 	/* Single invocation for instance add/remove */
198 	if (node) {
199 		WARN_ON_ONCE(lastp && *lastp);
200 		trace_cpuhp_multi_enter(cpu, st->target, state, cbm, node);
201 		ret = cbm(cpu, node);
202 		trace_cpuhp_exit(cpu, st->state, state, ret);
203 		return ret;
204 	}
205 
206 	/* State transition. Invoke on all instances */
207 	cnt = 0;
208 	hlist_for_each(node, &step->list) {
209 		if (lastp && node == *lastp)
210 			break;
211 
212 		trace_cpuhp_multi_enter(cpu, st->target, state, cbm, node);
213 		ret = cbm(cpu, node);
214 		trace_cpuhp_exit(cpu, st->state, state, ret);
215 		if (ret) {
216 			if (!lastp)
217 				goto err;
218 
219 			*lastp = node;
220 			return ret;
221 		}
222 		cnt++;
223 	}
224 	if (lastp)
225 		*lastp = NULL;
226 	return 0;
227 err:
228 	/* Rollback the instances if one failed */
229 	cbm = !bringup ? step->startup.multi : step->teardown.multi;
230 	if (!cbm)
231 		return ret;
232 
233 	hlist_for_each(node, &step->list) {
234 		if (!cnt--)
235 			break;
236 
237 		trace_cpuhp_multi_enter(cpu, st->target, state, cbm, node);
238 		ret = cbm(cpu, node);
239 		trace_cpuhp_exit(cpu, st->state, state, ret);
240 		/*
241 		 * Rollback must not fail,
242 		 */
243 		WARN_ON_ONCE(ret);
244 	}
245 	return ret;
246 }
247 
248 #ifdef CONFIG_SMP
249 static bool cpuhp_is_ap_state(enum cpuhp_state state)
250 {
251 	/*
252 	 * The extra check for CPUHP_TEARDOWN_CPU is only for documentation
253 	 * purposes as that state is handled explicitly in cpu_down.
254 	 */
255 	return state > CPUHP_BRINGUP_CPU && state != CPUHP_TEARDOWN_CPU;
256 }
257 
258 static inline void wait_for_ap_thread(struct cpuhp_cpu_state *st, bool bringup)
259 {
260 	struct completion *done = bringup ? &st->done_up : &st->done_down;
261 	wait_for_completion(done);
262 }
263 
264 static inline void complete_ap_thread(struct cpuhp_cpu_state *st, bool bringup)
265 {
266 	struct completion *done = bringup ? &st->done_up : &st->done_down;
267 	complete(done);
268 }
269 
270 /*
271  * The former STARTING/DYING states, ran with IRQs disabled and must not fail.
272  */
273 static bool cpuhp_is_atomic_state(enum cpuhp_state state)
274 {
275 	return CPUHP_AP_IDLE_DEAD <= state && state < CPUHP_AP_ONLINE;
276 }
277 
278 /* Serializes the updates to cpu_online_mask, cpu_present_mask */
279 static DEFINE_MUTEX(cpu_add_remove_lock);
280 bool cpuhp_tasks_frozen;
281 EXPORT_SYMBOL_GPL(cpuhp_tasks_frozen);
282 
283 /*
284  * The following two APIs (cpu_maps_update_begin/done) must be used when
285  * attempting to serialize the updates to cpu_online_mask & cpu_present_mask.
286  */
287 void cpu_maps_update_begin(void)
288 {
289 	mutex_lock(&cpu_add_remove_lock);
290 }
291 
292 void cpu_maps_update_done(void)
293 {
294 	mutex_unlock(&cpu_add_remove_lock);
295 }
296 
297 /*
298  * If set, cpu_up and cpu_down will return -EBUSY and do nothing.
299  * Should always be manipulated under cpu_add_remove_lock
300  */
301 static int cpu_hotplug_disabled;
302 
303 #ifdef CONFIG_HOTPLUG_CPU
304 
305 DEFINE_STATIC_PERCPU_RWSEM(cpu_hotplug_lock);
306 
307 void cpus_read_lock(void)
308 {
309 	percpu_down_read(&cpu_hotplug_lock);
310 }
311 EXPORT_SYMBOL_GPL(cpus_read_lock);
312 
313 int cpus_read_trylock(void)
314 {
315 	return percpu_down_read_trylock(&cpu_hotplug_lock);
316 }
317 EXPORT_SYMBOL_GPL(cpus_read_trylock);
318 
319 void cpus_read_unlock(void)
320 {
321 	percpu_up_read(&cpu_hotplug_lock);
322 }
323 EXPORT_SYMBOL_GPL(cpus_read_unlock);
324 
325 void cpus_write_lock(void)
326 {
327 	percpu_down_write(&cpu_hotplug_lock);
328 }
329 
330 void cpus_write_unlock(void)
331 {
332 	percpu_up_write(&cpu_hotplug_lock);
333 }
334 
335 void lockdep_assert_cpus_held(void)
336 {
337 	/*
338 	 * We can't have hotplug operations before userspace starts running,
339 	 * and some init codepaths will knowingly not take the hotplug lock.
340 	 * This is all valid, so mute lockdep until it makes sense to report
341 	 * unheld locks.
342 	 */
343 	if (system_state < SYSTEM_RUNNING)
344 		return;
345 
346 	percpu_rwsem_assert_held(&cpu_hotplug_lock);
347 }
348 
349 #ifdef CONFIG_LOCKDEP
350 int lockdep_is_cpus_held(void)
351 {
352 	return percpu_rwsem_is_held(&cpu_hotplug_lock);
353 }
354 #endif
355 
356 static void lockdep_acquire_cpus_lock(void)
357 {
358 	rwsem_acquire(&cpu_hotplug_lock.dep_map, 0, 0, _THIS_IP_);
359 }
360 
361 static void lockdep_release_cpus_lock(void)
362 {
363 	rwsem_release(&cpu_hotplug_lock.dep_map, _THIS_IP_);
364 }
365 
366 /*
367  * Wait for currently running CPU hotplug operations to complete (if any) and
368  * disable future CPU hotplug (from sysfs). The 'cpu_add_remove_lock' protects
369  * the 'cpu_hotplug_disabled' flag. The same lock is also acquired by the
370  * hotplug path before performing hotplug operations. So acquiring that lock
371  * guarantees mutual exclusion from any currently running hotplug operations.
372  */
373 void cpu_hotplug_disable(void)
374 {
375 	cpu_maps_update_begin();
376 	cpu_hotplug_disabled++;
377 	cpu_maps_update_done();
378 }
379 EXPORT_SYMBOL_GPL(cpu_hotplug_disable);
380 
381 static void __cpu_hotplug_enable(void)
382 {
383 	if (WARN_ONCE(!cpu_hotplug_disabled, "Unbalanced cpu hotplug enable\n"))
384 		return;
385 	cpu_hotplug_disabled--;
386 }
387 
388 void cpu_hotplug_enable(void)
389 {
390 	cpu_maps_update_begin();
391 	__cpu_hotplug_enable();
392 	cpu_maps_update_done();
393 }
394 EXPORT_SYMBOL_GPL(cpu_hotplug_enable);
395 
396 #else
397 
398 static void lockdep_acquire_cpus_lock(void)
399 {
400 }
401 
402 static void lockdep_release_cpus_lock(void)
403 {
404 }
405 
406 #endif	/* CONFIG_HOTPLUG_CPU */
407 
408 /*
409  * Architectures that need SMT-specific errata handling during SMT hotplug
410  * should override this.
411  */
412 void __weak arch_smt_update(void) { }
413 
414 #ifdef CONFIG_HOTPLUG_SMT
415 enum cpuhp_smt_control cpu_smt_control __read_mostly = CPU_SMT_ENABLED;
416 
417 void __init cpu_smt_disable(bool force)
418 {
419 	if (!cpu_smt_possible())
420 		return;
421 
422 	if (force) {
423 		pr_info("SMT: Force disabled\n");
424 		cpu_smt_control = CPU_SMT_FORCE_DISABLED;
425 	} else {
426 		pr_info("SMT: disabled\n");
427 		cpu_smt_control = CPU_SMT_DISABLED;
428 	}
429 }
430 
431 /*
432  * The decision whether SMT is supported can only be done after the full
433  * CPU identification. Called from architecture code.
434  */
435 void __init cpu_smt_check_topology(void)
436 {
437 	if (!topology_smt_supported())
438 		cpu_smt_control = CPU_SMT_NOT_SUPPORTED;
439 }
440 
441 static int __init smt_cmdline_disable(char *str)
442 {
443 	cpu_smt_disable(str && !strcmp(str, "force"));
444 	return 0;
445 }
446 early_param("nosmt", smt_cmdline_disable);
447 
448 static inline bool cpu_smt_allowed(unsigned int cpu)
449 {
450 	if (cpu_smt_control == CPU_SMT_ENABLED)
451 		return true;
452 
453 	if (topology_is_primary_thread(cpu))
454 		return true;
455 
456 	/*
457 	 * On x86 it's required to boot all logical CPUs at least once so
458 	 * that the init code can get a chance to set CR4.MCE on each
459 	 * CPU. Otherwise, a broadcasted MCE observing CR4.MCE=0b on any
460 	 * core will shutdown the machine.
461 	 */
462 	return !cpumask_test_cpu(cpu, &cpus_booted_once_mask);
463 }
464 
465 /* Returns true if SMT is not supported of forcefully (irreversibly) disabled */
466 bool cpu_smt_possible(void)
467 {
468 	return cpu_smt_control != CPU_SMT_FORCE_DISABLED &&
469 		cpu_smt_control != CPU_SMT_NOT_SUPPORTED;
470 }
471 EXPORT_SYMBOL_GPL(cpu_smt_possible);
472 #else
473 static inline bool cpu_smt_allowed(unsigned int cpu) { return true; }
474 #endif
475 
476 static inline enum cpuhp_state
477 cpuhp_set_state(struct cpuhp_cpu_state *st, enum cpuhp_state target)
478 {
479 	enum cpuhp_state prev_state = st->state;
480 	bool bringup = st->state < target;
481 
482 	st->rollback = false;
483 	st->last = NULL;
484 
485 	st->target = target;
486 	st->single = false;
487 	st->bringup = bringup;
488 	if (cpu_dying(st->cpu) != !bringup)
489 		set_cpu_dying(st->cpu, !bringup);
490 
491 	return prev_state;
492 }
493 
494 static inline void
495 cpuhp_reset_state(struct cpuhp_cpu_state *st, enum cpuhp_state prev_state)
496 {
497 	bool bringup = !st->bringup;
498 
499 	st->target = prev_state;
500 
501 	/*
502 	 * Already rolling back. No need invert the bringup value or to change
503 	 * the current state.
504 	 */
505 	if (st->rollback)
506 		return;
507 
508 	st->rollback = true;
509 
510 	/*
511 	 * If we have st->last we need to undo partial multi_instance of this
512 	 * state first. Otherwise start undo at the previous state.
513 	 */
514 	if (!st->last) {
515 		if (st->bringup)
516 			st->state--;
517 		else
518 			st->state++;
519 	}
520 
521 	st->bringup = bringup;
522 	if (cpu_dying(st->cpu) != !bringup)
523 		set_cpu_dying(st->cpu, !bringup);
524 }
525 
526 /* Regular hotplug invocation of the AP hotplug thread */
527 static void __cpuhp_kick_ap(struct cpuhp_cpu_state *st)
528 {
529 	if (!st->single && st->state == st->target)
530 		return;
531 
532 	st->result = 0;
533 	/*
534 	 * Make sure the above stores are visible before should_run becomes
535 	 * true. Paired with the mb() above in cpuhp_thread_fun()
536 	 */
537 	smp_mb();
538 	st->should_run = true;
539 	wake_up_process(st->thread);
540 	wait_for_ap_thread(st, st->bringup);
541 }
542 
543 static int cpuhp_kick_ap(struct cpuhp_cpu_state *st, enum cpuhp_state target)
544 {
545 	enum cpuhp_state prev_state;
546 	int ret;
547 
548 	prev_state = cpuhp_set_state(st, target);
549 	__cpuhp_kick_ap(st);
550 	if ((ret = st->result)) {
551 		cpuhp_reset_state(st, prev_state);
552 		__cpuhp_kick_ap(st);
553 	}
554 
555 	return ret;
556 }
557 
558 static int bringup_wait_for_ap(unsigned int cpu)
559 {
560 	struct cpuhp_cpu_state *st = per_cpu_ptr(&cpuhp_state, cpu);
561 
562 	/* Wait for the CPU to reach CPUHP_AP_ONLINE_IDLE */
563 	wait_for_ap_thread(st, true);
564 	if (WARN_ON_ONCE((!cpu_online(cpu))))
565 		return -ECANCELED;
566 
567 	/* Unpark the hotplug thread of the target cpu */
568 	kthread_unpark(st->thread);
569 
570 	/*
571 	 * SMT soft disabling on X86 requires to bring the CPU out of the
572 	 * BIOS 'wait for SIPI' state in order to set the CR4.MCE bit.  The
573 	 * CPU marked itself as booted_once in notify_cpu_starting() so the
574 	 * cpu_smt_allowed() check will now return false if this is not the
575 	 * primary sibling.
576 	 */
577 	if (!cpu_smt_allowed(cpu))
578 		return -ECANCELED;
579 
580 	if (st->target <= CPUHP_AP_ONLINE_IDLE)
581 		return 0;
582 
583 	return cpuhp_kick_ap(st, st->target);
584 }
585 
586 static int bringup_cpu(unsigned int cpu)
587 {
588 	struct task_struct *idle = idle_thread_get(cpu);
589 	int ret;
590 
591 	/*
592 	 * Reset stale stack state from the last time this CPU was online.
593 	 */
594 	scs_task_reset(idle);
595 	kasan_unpoison_task_stack(idle);
596 
597 	/*
598 	 * Some architectures have to walk the irq descriptors to
599 	 * setup the vector space for the cpu which comes online.
600 	 * Prevent irq alloc/free across the bringup.
601 	 */
602 	irq_lock_sparse();
603 
604 	/* Arch-specific enabling code. */
605 	ret = __cpu_up(cpu, idle);
606 	irq_unlock_sparse();
607 	if (ret)
608 		return ret;
609 	return bringup_wait_for_ap(cpu);
610 }
611 
612 static int finish_cpu(unsigned int cpu)
613 {
614 	struct task_struct *idle = idle_thread_get(cpu);
615 	struct mm_struct *mm = idle->active_mm;
616 
617 	/*
618 	 * idle_task_exit() will have switched to &init_mm, now
619 	 * clean up any remaining active_mm state.
620 	 */
621 	if (mm != &init_mm)
622 		idle->active_mm = &init_mm;
623 	mmdrop(mm);
624 	return 0;
625 }
626 
627 /*
628  * Hotplug state machine related functions
629  */
630 
631 /*
632  * Get the next state to run. Empty ones will be skipped. Returns true if a
633  * state must be run.
634  *
635  * st->state will be modified ahead of time, to match state_to_run, as if it
636  * has already ran.
637  */
638 static bool cpuhp_next_state(bool bringup,
639 			     enum cpuhp_state *state_to_run,
640 			     struct cpuhp_cpu_state *st,
641 			     enum cpuhp_state target)
642 {
643 	do {
644 		if (bringup) {
645 			if (st->state >= target)
646 				return false;
647 
648 			*state_to_run = ++st->state;
649 		} else {
650 			if (st->state <= target)
651 				return false;
652 
653 			*state_to_run = st->state--;
654 		}
655 
656 		if (!cpuhp_step_empty(bringup, cpuhp_get_step(*state_to_run)))
657 			break;
658 	} while (true);
659 
660 	return true;
661 }
662 
663 static int cpuhp_invoke_callback_range(bool bringup,
664 				       unsigned int cpu,
665 				       struct cpuhp_cpu_state *st,
666 				       enum cpuhp_state target)
667 {
668 	enum cpuhp_state state;
669 	int err = 0;
670 
671 	while (cpuhp_next_state(bringup, &state, st, target)) {
672 		err = cpuhp_invoke_callback(cpu, state, bringup, NULL, NULL);
673 		if (err)
674 			break;
675 	}
676 
677 	return err;
678 }
679 
680 static inline bool can_rollback_cpu(struct cpuhp_cpu_state *st)
681 {
682 	if (IS_ENABLED(CONFIG_HOTPLUG_CPU))
683 		return true;
684 	/*
685 	 * When CPU hotplug is disabled, then taking the CPU down is not
686 	 * possible because takedown_cpu() and the architecture and
687 	 * subsystem specific mechanisms are not available. So the CPU
688 	 * which would be completely unplugged again needs to stay around
689 	 * in the current state.
690 	 */
691 	return st->state <= CPUHP_BRINGUP_CPU;
692 }
693 
694 static int cpuhp_up_callbacks(unsigned int cpu, struct cpuhp_cpu_state *st,
695 			      enum cpuhp_state target)
696 {
697 	enum cpuhp_state prev_state = st->state;
698 	int ret = 0;
699 
700 	ret = cpuhp_invoke_callback_range(true, cpu, st, target);
701 	if (ret) {
702 		pr_debug("CPU UP failed (%d) CPU %u state %s (%d)\n",
703 			 ret, cpu, cpuhp_get_step(st->state)->name,
704 			 st->state);
705 
706 		cpuhp_reset_state(st, prev_state);
707 		if (can_rollback_cpu(st))
708 			WARN_ON(cpuhp_invoke_callback_range(false, cpu, st,
709 							    prev_state));
710 	}
711 	return ret;
712 }
713 
714 /*
715  * The cpu hotplug threads manage the bringup and teardown of the cpus
716  */
717 static void cpuhp_create(unsigned int cpu)
718 {
719 	struct cpuhp_cpu_state *st = per_cpu_ptr(&cpuhp_state, cpu);
720 
721 	init_completion(&st->done_up);
722 	init_completion(&st->done_down);
723 	st->cpu = cpu;
724 }
725 
726 static int cpuhp_should_run(unsigned int cpu)
727 {
728 	struct cpuhp_cpu_state *st = this_cpu_ptr(&cpuhp_state);
729 
730 	return st->should_run;
731 }
732 
733 /*
734  * Execute teardown/startup callbacks on the plugged cpu. Also used to invoke
735  * callbacks when a state gets [un]installed at runtime.
736  *
737  * Each invocation of this function by the smpboot thread does a single AP
738  * state callback.
739  *
740  * It has 3 modes of operation:
741  *  - single: runs st->cb_state
742  *  - up:     runs ++st->state, while st->state < st->target
743  *  - down:   runs st->state--, while st->state > st->target
744  *
745  * When complete or on error, should_run is cleared and the completion is fired.
746  */
747 static void cpuhp_thread_fun(unsigned int cpu)
748 {
749 	struct cpuhp_cpu_state *st = this_cpu_ptr(&cpuhp_state);
750 	bool bringup = st->bringup;
751 	enum cpuhp_state state;
752 
753 	if (WARN_ON_ONCE(!st->should_run))
754 		return;
755 
756 	/*
757 	 * ACQUIRE for the cpuhp_should_run() load of ->should_run. Ensures
758 	 * that if we see ->should_run we also see the rest of the state.
759 	 */
760 	smp_mb();
761 
762 	/*
763 	 * The BP holds the hotplug lock, but we're now running on the AP,
764 	 * ensure that anybody asserting the lock is held, will actually find
765 	 * it so.
766 	 */
767 	lockdep_acquire_cpus_lock();
768 	cpuhp_lock_acquire(bringup);
769 
770 	if (st->single) {
771 		state = st->cb_state;
772 		st->should_run = false;
773 	} else {
774 		st->should_run = cpuhp_next_state(bringup, &state, st, st->target);
775 		if (!st->should_run)
776 			goto end;
777 	}
778 
779 	WARN_ON_ONCE(!cpuhp_is_ap_state(state));
780 
781 	if (cpuhp_is_atomic_state(state)) {
782 		local_irq_disable();
783 		st->result = cpuhp_invoke_callback(cpu, state, bringup, st->node, &st->last);
784 		local_irq_enable();
785 
786 		/*
787 		 * STARTING/DYING must not fail!
788 		 */
789 		WARN_ON_ONCE(st->result);
790 	} else {
791 		st->result = cpuhp_invoke_callback(cpu, state, bringup, st->node, &st->last);
792 	}
793 
794 	if (st->result) {
795 		/*
796 		 * If we fail on a rollback, we're up a creek without no
797 		 * paddle, no way forward, no way back. We loose, thanks for
798 		 * playing.
799 		 */
800 		WARN_ON_ONCE(st->rollback);
801 		st->should_run = false;
802 	}
803 
804 end:
805 	cpuhp_lock_release(bringup);
806 	lockdep_release_cpus_lock();
807 
808 	if (!st->should_run)
809 		complete_ap_thread(st, bringup);
810 }
811 
812 /* Invoke a single callback on a remote cpu */
813 static int
814 cpuhp_invoke_ap_callback(int cpu, enum cpuhp_state state, bool bringup,
815 			 struct hlist_node *node)
816 {
817 	struct cpuhp_cpu_state *st = per_cpu_ptr(&cpuhp_state, cpu);
818 	int ret;
819 
820 	if (!cpu_online(cpu))
821 		return 0;
822 
823 	cpuhp_lock_acquire(false);
824 	cpuhp_lock_release(false);
825 
826 	cpuhp_lock_acquire(true);
827 	cpuhp_lock_release(true);
828 
829 	/*
830 	 * If we are up and running, use the hotplug thread. For early calls
831 	 * we invoke the thread function directly.
832 	 */
833 	if (!st->thread)
834 		return cpuhp_invoke_callback(cpu, state, bringup, node, NULL);
835 
836 	st->rollback = false;
837 	st->last = NULL;
838 
839 	st->node = node;
840 	st->bringup = bringup;
841 	st->cb_state = state;
842 	st->single = true;
843 
844 	__cpuhp_kick_ap(st);
845 
846 	/*
847 	 * If we failed and did a partial, do a rollback.
848 	 */
849 	if ((ret = st->result) && st->last) {
850 		st->rollback = true;
851 		st->bringup = !bringup;
852 
853 		__cpuhp_kick_ap(st);
854 	}
855 
856 	/*
857 	 * Clean up the leftovers so the next hotplug operation wont use stale
858 	 * data.
859 	 */
860 	st->node = st->last = NULL;
861 	return ret;
862 }
863 
864 static int cpuhp_kick_ap_work(unsigned int cpu)
865 {
866 	struct cpuhp_cpu_state *st = per_cpu_ptr(&cpuhp_state, cpu);
867 	enum cpuhp_state prev_state = st->state;
868 	int ret;
869 
870 	cpuhp_lock_acquire(false);
871 	cpuhp_lock_release(false);
872 
873 	cpuhp_lock_acquire(true);
874 	cpuhp_lock_release(true);
875 
876 	trace_cpuhp_enter(cpu, st->target, prev_state, cpuhp_kick_ap_work);
877 	ret = cpuhp_kick_ap(st, st->target);
878 	trace_cpuhp_exit(cpu, st->state, prev_state, ret);
879 
880 	return ret;
881 }
882 
883 static struct smp_hotplug_thread cpuhp_threads = {
884 	.store			= &cpuhp_state.thread,
885 	.create			= &cpuhp_create,
886 	.thread_should_run	= cpuhp_should_run,
887 	.thread_fn		= cpuhp_thread_fun,
888 	.thread_comm		= "cpuhp/%u",
889 	.selfparking		= true,
890 };
891 
892 void __init cpuhp_threads_init(void)
893 {
894 	BUG_ON(smpboot_register_percpu_thread(&cpuhp_threads));
895 	kthread_unpark(this_cpu_read(cpuhp_state.thread));
896 }
897 
898 /*
899  *
900  * Serialize hotplug trainwrecks outside of the cpu_hotplug_lock
901  * protected region.
902  *
903  * The operation is still serialized against concurrent CPU hotplug via
904  * cpu_add_remove_lock, i.e. CPU map protection.  But it is _not_
905  * serialized against other hotplug related activity like adding or
906  * removing of state callbacks and state instances, which invoke either the
907  * startup or the teardown callback of the affected state.
908  *
909  * This is required for subsystems which are unfixable vs. CPU hotplug and
910  * evade lock inversion problems by scheduling work which has to be
911  * completed _before_ cpu_up()/_cpu_down() returns.
912  *
913  * Don't even think about adding anything to this for any new code or even
914  * drivers. It's only purpose is to keep existing lock order trainwrecks
915  * working.
916  *
917  * For cpu_down() there might be valid reasons to finish cleanups which are
918  * not required to be done under cpu_hotplug_lock, but that's a different
919  * story and would be not invoked via this.
920  */
921 static void cpu_up_down_serialize_trainwrecks(bool tasks_frozen)
922 {
923 	/*
924 	 * cpusets delegate hotplug operations to a worker to "solve" the
925 	 * lock order problems. Wait for the worker, but only if tasks are
926 	 * _not_ frozen (suspend, hibernate) as that would wait forever.
927 	 *
928 	 * The wait is required because otherwise the hotplug operation
929 	 * returns with inconsistent state, which could even be observed in
930 	 * user space when a new CPU is brought up. The CPU plug uevent
931 	 * would be delivered and user space reacting on it would fail to
932 	 * move tasks to the newly plugged CPU up to the point where the
933 	 * work has finished because up to that point the newly plugged CPU
934 	 * is not assignable in cpusets/cgroups. On unplug that's not
935 	 * necessarily a visible issue, but it is still inconsistent state,
936 	 * which is the real problem which needs to be "fixed". This can't
937 	 * prevent the transient state between scheduling the work and
938 	 * returning from waiting for it.
939 	 */
940 	if (!tasks_frozen)
941 		cpuset_wait_for_hotplug();
942 }
943 
944 #ifdef CONFIG_HOTPLUG_CPU
945 #ifndef arch_clear_mm_cpumask_cpu
946 #define arch_clear_mm_cpumask_cpu(cpu, mm) cpumask_clear_cpu(cpu, mm_cpumask(mm))
947 #endif
948 
949 /**
950  * clear_tasks_mm_cpumask - Safely clear tasks' mm_cpumask for a CPU
951  * @cpu: a CPU id
952  *
953  * This function walks all processes, finds a valid mm struct for each one and
954  * then clears a corresponding bit in mm's cpumask.  While this all sounds
955  * trivial, there are various non-obvious corner cases, which this function
956  * tries to solve in a safe manner.
957  *
958  * Also note that the function uses a somewhat relaxed locking scheme, so it may
959  * be called only for an already offlined CPU.
960  */
961 void clear_tasks_mm_cpumask(int cpu)
962 {
963 	struct task_struct *p;
964 
965 	/*
966 	 * This function is called after the cpu is taken down and marked
967 	 * offline, so its not like new tasks will ever get this cpu set in
968 	 * their mm mask. -- Peter Zijlstra
969 	 * Thus, we may use rcu_read_lock() here, instead of grabbing
970 	 * full-fledged tasklist_lock.
971 	 */
972 	WARN_ON(cpu_online(cpu));
973 	rcu_read_lock();
974 	for_each_process(p) {
975 		struct task_struct *t;
976 
977 		/*
978 		 * Main thread might exit, but other threads may still have
979 		 * a valid mm. Find one.
980 		 */
981 		t = find_lock_task_mm(p);
982 		if (!t)
983 			continue;
984 		arch_clear_mm_cpumask_cpu(cpu, t->mm);
985 		task_unlock(t);
986 	}
987 	rcu_read_unlock();
988 }
989 
990 /* Take this CPU down. */
991 static int take_cpu_down(void *_param)
992 {
993 	struct cpuhp_cpu_state *st = this_cpu_ptr(&cpuhp_state);
994 	enum cpuhp_state target = max((int)st->target, CPUHP_AP_OFFLINE);
995 	int err, cpu = smp_processor_id();
996 	int ret;
997 
998 	/* Ensure this CPU doesn't handle any more interrupts. */
999 	err = __cpu_disable();
1000 	if (err < 0)
1001 		return err;
1002 
1003 	/*
1004 	 * Must be called from CPUHP_TEARDOWN_CPU, which means, as we are going
1005 	 * down, that the current state is CPUHP_TEARDOWN_CPU - 1.
1006 	 */
1007 	WARN_ON(st->state != (CPUHP_TEARDOWN_CPU - 1));
1008 
1009 	/* Invoke the former CPU_DYING callbacks */
1010 	ret = cpuhp_invoke_callback_range(false, cpu, st, target);
1011 
1012 	/*
1013 	 * DYING must not fail!
1014 	 */
1015 	WARN_ON_ONCE(ret);
1016 
1017 	/* Give up timekeeping duties */
1018 	tick_handover_do_timer();
1019 	/* Remove CPU from timer broadcasting */
1020 	tick_offline_cpu(cpu);
1021 	/* Park the stopper thread */
1022 	stop_machine_park(cpu);
1023 	return 0;
1024 }
1025 
1026 static int takedown_cpu(unsigned int cpu)
1027 {
1028 	struct cpuhp_cpu_state *st = per_cpu_ptr(&cpuhp_state, cpu);
1029 	int err;
1030 
1031 	/* Park the smpboot threads */
1032 	kthread_park(st->thread);
1033 
1034 	/*
1035 	 * Prevent irq alloc/free while the dying cpu reorganizes the
1036 	 * interrupt affinities.
1037 	 */
1038 	irq_lock_sparse();
1039 
1040 	/*
1041 	 * So now all preempt/rcu users must observe !cpu_active().
1042 	 */
1043 	err = stop_machine_cpuslocked(take_cpu_down, NULL, cpumask_of(cpu));
1044 	if (err) {
1045 		/* CPU refused to die */
1046 		irq_unlock_sparse();
1047 		/* Unpark the hotplug thread so we can rollback there */
1048 		kthread_unpark(st->thread);
1049 		return err;
1050 	}
1051 	BUG_ON(cpu_online(cpu));
1052 
1053 	/*
1054 	 * The teardown callback for CPUHP_AP_SCHED_STARTING will have removed
1055 	 * all runnable tasks from the CPU, there's only the idle task left now
1056 	 * that the migration thread is done doing the stop_machine thing.
1057 	 *
1058 	 * Wait for the stop thread to go away.
1059 	 */
1060 	wait_for_ap_thread(st, false);
1061 	BUG_ON(st->state != CPUHP_AP_IDLE_DEAD);
1062 
1063 	/* Interrupts are moved away from the dying cpu, reenable alloc/free */
1064 	irq_unlock_sparse();
1065 
1066 	hotplug_cpu__broadcast_tick_pull(cpu);
1067 	/* This actually kills the CPU. */
1068 	__cpu_die(cpu);
1069 
1070 	tick_cleanup_dead_cpu(cpu);
1071 	rcutree_migrate_callbacks(cpu);
1072 	return 0;
1073 }
1074 
1075 static void cpuhp_complete_idle_dead(void *arg)
1076 {
1077 	struct cpuhp_cpu_state *st = arg;
1078 
1079 	complete_ap_thread(st, false);
1080 }
1081 
1082 void cpuhp_report_idle_dead(void)
1083 {
1084 	struct cpuhp_cpu_state *st = this_cpu_ptr(&cpuhp_state);
1085 
1086 	BUG_ON(st->state != CPUHP_AP_OFFLINE);
1087 	rcu_report_dead(smp_processor_id());
1088 	st->state = CPUHP_AP_IDLE_DEAD;
1089 	/*
1090 	 * We cannot call complete after rcu_report_dead() so we delegate it
1091 	 * to an online cpu.
1092 	 */
1093 	smp_call_function_single(cpumask_first(cpu_online_mask),
1094 				 cpuhp_complete_idle_dead, st, 0);
1095 }
1096 
1097 static int cpuhp_down_callbacks(unsigned int cpu, struct cpuhp_cpu_state *st,
1098 				enum cpuhp_state target)
1099 {
1100 	enum cpuhp_state prev_state = st->state;
1101 	int ret = 0;
1102 
1103 	ret = cpuhp_invoke_callback_range(false, cpu, st, target);
1104 	if (ret) {
1105 		pr_debug("CPU DOWN failed (%d) CPU %u state %s (%d)\n",
1106 			 ret, cpu, cpuhp_get_step(st->state)->name,
1107 			 st->state);
1108 
1109 		cpuhp_reset_state(st, prev_state);
1110 
1111 		if (st->state < prev_state)
1112 			WARN_ON(cpuhp_invoke_callback_range(true, cpu, st,
1113 							    prev_state));
1114 	}
1115 
1116 	return ret;
1117 }
1118 
1119 /* Requires cpu_add_remove_lock to be held */
1120 static int __ref _cpu_down(unsigned int cpu, int tasks_frozen,
1121 			   enum cpuhp_state target)
1122 {
1123 	struct cpuhp_cpu_state *st = per_cpu_ptr(&cpuhp_state, cpu);
1124 	int prev_state, ret = 0;
1125 
1126 	if (num_online_cpus() == 1)
1127 		return -EBUSY;
1128 
1129 	if (!cpu_present(cpu))
1130 		return -EINVAL;
1131 
1132 	cpus_write_lock();
1133 
1134 	cpuhp_tasks_frozen = tasks_frozen;
1135 
1136 	prev_state = cpuhp_set_state(st, target);
1137 	/*
1138 	 * If the current CPU state is in the range of the AP hotplug thread,
1139 	 * then we need to kick the thread.
1140 	 */
1141 	if (st->state > CPUHP_TEARDOWN_CPU) {
1142 		st->target = max((int)target, CPUHP_TEARDOWN_CPU);
1143 		ret = cpuhp_kick_ap_work(cpu);
1144 		/*
1145 		 * The AP side has done the error rollback already. Just
1146 		 * return the error code..
1147 		 */
1148 		if (ret)
1149 			goto out;
1150 
1151 		/*
1152 		 * We might have stopped still in the range of the AP hotplug
1153 		 * thread. Nothing to do anymore.
1154 		 */
1155 		if (st->state > CPUHP_TEARDOWN_CPU)
1156 			goto out;
1157 
1158 		st->target = target;
1159 	}
1160 	/*
1161 	 * The AP brought itself down to CPUHP_TEARDOWN_CPU. So we need
1162 	 * to do the further cleanups.
1163 	 */
1164 	ret = cpuhp_down_callbacks(cpu, st, target);
1165 	if (ret && st->state < prev_state) {
1166 		if (st->state == CPUHP_TEARDOWN_CPU) {
1167 			cpuhp_reset_state(st, prev_state);
1168 			__cpuhp_kick_ap(st);
1169 		} else {
1170 			WARN(1, "DEAD callback error for CPU%d", cpu);
1171 		}
1172 	}
1173 
1174 out:
1175 	cpus_write_unlock();
1176 	/*
1177 	 * Do post unplug cleanup. This is still protected against
1178 	 * concurrent CPU hotplug via cpu_add_remove_lock.
1179 	 */
1180 	lockup_detector_cleanup();
1181 	arch_smt_update();
1182 	cpu_up_down_serialize_trainwrecks(tasks_frozen);
1183 	return ret;
1184 }
1185 
1186 static int cpu_down_maps_locked(unsigned int cpu, enum cpuhp_state target)
1187 {
1188 	if (cpu_hotplug_disabled)
1189 		return -EBUSY;
1190 	return _cpu_down(cpu, 0, target);
1191 }
1192 
1193 static int cpu_down(unsigned int cpu, enum cpuhp_state target)
1194 {
1195 	int err;
1196 
1197 	cpu_maps_update_begin();
1198 	err = cpu_down_maps_locked(cpu, target);
1199 	cpu_maps_update_done();
1200 	return err;
1201 }
1202 
1203 /**
1204  * cpu_device_down - Bring down a cpu device
1205  * @dev: Pointer to the cpu device to offline
1206  *
1207  * This function is meant to be used by device core cpu subsystem only.
1208  *
1209  * Other subsystems should use remove_cpu() instead.
1210  *
1211  * Return: %0 on success or a negative errno code
1212  */
1213 int cpu_device_down(struct device *dev)
1214 {
1215 	return cpu_down(dev->id, CPUHP_OFFLINE);
1216 }
1217 
1218 int remove_cpu(unsigned int cpu)
1219 {
1220 	int ret;
1221 
1222 	lock_device_hotplug();
1223 	ret = device_offline(get_cpu_device(cpu));
1224 	unlock_device_hotplug();
1225 
1226 	return ret;
1227 }
1228 EXPORT_SYMBOL_GPL(remove_cpu);
1229 
1230 void smp_shutdown_nonboot_cpus(unsigned int primary_cpu)
1231 {
1232 	unsigned int cpu;
1233 	int error;
1234 
1235 	cpu_maps_update_begin();
1236 
1237 	/*
1238 	 * Make certain the cpu I'm about to reboot on is online.
1239 	 *
1240 	 * This is inline to what migrate_to_reboot_cpu() already do.
1241 	 */
1242 	if (!cpu_online(primary_cpu))
1243 		primary_cpu = cpumask_first(cpu_online_mask);
1244 
1245 	for_each_online_cpu(cpu) {
1246 		if (cpu == primary_cpu)
1247 			continue;
1248 
1249 		error = cpu_down_maps_locked(cpu, CPUHP_OFFLINE);
1250 		if (error) {
1251 			pr_err("Failed to offline CPU%d - error=%d",
1252 				cpu, error);
1253 			break;
1254 		}
1255 	}
1256 
1257 	/*
1258 	 * Ensure all but the reboot CPU are offline.
1259 	 */
1260 	BUG_ON(num_online_cpus() > 1);
1261 
1262 	/*
1263 	 * Make sure the CPUs won't be enabled by someone else after this
1264 	 * point. Kexec will reboot to a new kernel shortly resetting
1265 	 * everything along the way.
1266 	 */
1267 	cpu_hotplug_disabled++;
1268 
1269 	cpu_maps_update_done();
1270 }
1271 
1272 #else
1273 #define takedown_cpu		NULL
1274 #endif /*CONFIG_HOTPLUG_CPU*/
1275 
1276 /**
1277  * notify_cpu_starting(cpu) - Invoke the callbacks on the starting CPU
1278  * @cpu: cpu that just started
1279  *
1280  * It must be called by the arch code on the new cpu, before the new cpu
1281  * enables interrupts and before the "boot" cpu returns from __cpu_up().
1282  */
1283 void notify_cpu_starting(unsigned int cpu)
1284 {
1285 	struct cpuhp_cpu_state *st = per_cpu_ptr(&cpuhp_state, cpu);
1286 	enum cpuhp_state target = min((int)st->target, CPUHP_AP_ONLINE);
1287 	int ret;
1288 
1289 	rcu_cpu_starting(cpu);	/* Enables RCU usage on this CPU. */
1290 	cpumask_set_cpu(cpu, &cpus_booted_once_mask);
1291 	ret = cpuhp_invoke_callback_range(true, cpu, st, target);
1292 
1293 	/*
1294 	 * STARTING must not fail!
1295 	 */
1296 	WARN_ON_ONCE(ret);
1297 }
1298 
1299 /*
1300  * Called from the idle task. Wake up the controlling task which brings the
1301  * hotplug thread of the upcoming CPU up and then delegates the rest of the
1302  * online bringup to the hotplug thread.
1303  */
1304 void cpuhp_online_idle(enum cpuhp_state state)
1305 {
1306 	struct cpuhp_cpu_state *st = this_cpu_ptr(&cpuhp_state);
1307 
1308 	/* Happens for the boot cpu */
1309 	if (state != CPUHP_AP_ONLINE_IDLE)
1310 		return;
1311 
1312 	/*
1313 	 * Unpart the stopper thread before we start the idle loop (and start
1314 	 * scheduling); this ensures the stopper task is always available.
1315 	 */
1316 	stop_machine_unpark(smp_processor_id());
1317 
1318 	st->state = CPUHP_AP_ONLINE_IDLE;
1319 	complete_ap_thread(st, true);
1320 }
1321 
1322 /* Requires cpu_add_remove_lock to be held */
1323 static int _cpu_up(unsigned int cpu, int tasks_frozen, enum cpuhp_state target)
1324 {
1325 	struct cpuhp_cpu_state *st = per_cpu_ptr(&cpuhp_state, cpu);
1326 	struct task_struct *idle;
1327 	int ret = 0;
1328 
1329 	cpus_write_lock();
1330 
1331 	if (!cpu_present(cpu)) {
1332 		ret = -EINVAL;
1333 		goto out;
1334 	}
1335 
1336 	/*
1337 	 * The caller of cpu_up() might have raced with another
1338 	 * caller. Nothing to do.
1339 	 */
1340 	if (st->state >= target)
1341 		goto out;
1342 
1343 	if (st->state == CPUHP_OFFLINE) {
1344 		/* Let it fail before we try to bring the cpu up */
1345 		idle = idle_thread_get(cpu);
1346 		if (IS_ERR(idle)) {
1347 			ret = PTR_ERR(idle);
1348 			goto out;
1349 		}
1350 	}
1351 
1352 	cpuhp_tasks_frozen = tasks_frozen;
1353 
1354 	cpuhp_set_state(st, target);
1355 	/*
1356 	 * If the current CPU state is in the range of the AP hotplug thread,
1357 	 * then we need to kick the thread once more.
1358 	 */
1359 	if (st->state > CPUHP_BRINGUP_CPU) {
1360 		ret = cpuhp_kick_ap_work(cpu);
1361 		/*
1362 		 * The AP side has done the error rollback already. Just
1363 		 * return the error code..
1364 		 */
1365 		if (ret)
1366 			goto out;
1367 	}
1368 
1369 	/*
1370 	 * Try to reach the target state. We max out on the BP at
1371 	 * CPUHP_BRINGUP_CPU. After that the AP hotplug thread is
1372 	 * responsible for bringing it up to the target state.
1373 	 */
1374 	target = min((int)target, CPUHP_BRINGUP_CPU);
1375 	ret = cpuhp_up_callbacks(cpu, st, target);
1376 out:
1377 	cpus_write_unlock();
1378 	arch_smt_update();
1379 	cpu_up_down_serialize_trainwrecks(tasks_frozen);
1380 	return ret;
1381 }
1382 
1383 static int cpu_up(unsigned int cpu, enum cpuhp_state target)
1384 {
1385 	int err = 0;
1386 
1387 	if (!cpu_possible(cpu)) {
1388 		pr_err("can't online cpu %d because it is not configured as may-hotadd at boot time\n",
1389 		       cpu);
1390 #if defined(CONFIG_IA64)
1391 		pr_err("please check additional_cpus= boot parameter\n");
1392 #endif
1393 		return -EINVAL;
1394 	}
1395 
1396 	err = try_online_node(cpu_to_node(cpu));
1397 	if (err)
1398 		return err;
1399 
1400 	cpu_maps_update_begin();
1401 
1402 	if (cpu_hotplug_disabled) {
1403 		err = -EBUSY;
1404 		goto out;
1405 	}
1406 	if (!cpu_smt_allowed(cpu)) {
1407 		err = -EPERM;
1408 		goto out;
1409 	}
1410 
1411 	err = _cpu_up(cpu, 0, target);
1412 out:
1413 	cpu_maps_update_done();
1414 	return err;
1415 }
1416 
1417 /**
1418  * cpu_device_up - Bring up a cpu device
1419  * @dev: Pointer to the cpu device to online
1420  *
1421  * This function is meant to be used by device core cpu subsystem only.
1422  *
1423  * Other subsystems should use add_cpu() instead.
1424  *
1425  * Return: %0 on success or a negative errno code
1426  */
1427 int cpu_device_up(struct device *dev)
1428 {
1429 	return cpu_up(dev->id, CPUHP_ONLINE);
1430 }
1431 
1432 int add_cpu(unsigned int cpu)
1433 {
1434 	int ret;
1435 
1436 	lock_device_hotplug();
1437 	ret = device_online(get_cpu_device(cpu));
1438 	unlock_device_hotplug();
1439 
1440 	return ret;
1441 }
1442 EXPORT_SYMBOL_GPL(add_cpu);
1443 
1444 /**
1445  * bringup_hibernate_cpu - Bring up the CPU that we hibernated on
1446  * @sleep_cpu: The cpu we hibernated on and should be brought up.
1447  *
1448  * On some architectures like arm64, we can hibernate on any CPU, but on
1449  * wake up the CPU we hibernated on might be offline as a side effect of
1450  * using maxcpus= for example.
1451  *
1452  * Return: %0 on success or a negative errno code
1453  */
1454 int bringup_hibernate_cpu(unsigned int sleep_cpu)
1455 {
1456 	int ret;
1457 
1458 	if (!cpu_online(sleep_cpu)) {
1459 		pr_info("Hibernated on a CPU that is offline! Bringing CPU up.\n");
1460 		ret = cpu_up(sleep_cpu, CPUHP_ONLINE);
1461 		if (ret) {
1462 			pr_err("Failed to bring hibernate-CPU up!\n");
1463 			return ret;
1464 		}
1465 	}
1466 	return 0;
1467 }
1468 
1469 void bringup_nonboot_cpus(unsigned int setup_max_cpus)
1470 {
1471 	unsigned int cpu;
1472 
1473 	for_each_present_cpu(cpu) {
1474 		if (num_online_cpus() >= setup_max_cpus)
1475 			break;
1476 		if (!cpu_online(cpu))
1477 			cpu_up(cpu, CPUHP_ONLINE);
1478 	}
1479 }
1480 
1481 #ifdef CONFIG_PM_SLEEP_SMP
1482 static cpumask_var_t frozen_cpus;
1483 
1484 int freeze_secondary_cpus(int primary)
1485 {
1486 	int cpu, error = 0;
1487 
1488 	cpu_maps_update_begin();
1489 	if (primary == -1) {
1490 		primary = cpumask_first(cpu_online_mask);
1491 		if (!housekeeping_cpu(primary, HK_FLAG_TIMER))
1492 			primary = housekeeping_any_cpu(HK_FLAG_TIMER);
1493 	} else {
1494 		if (!cpu_online(primary))
1495 			primary = cpumask_first(cpu_online_mask);
1496 	}
1497 
1498 	/*
1499 	 * We take down all of the non-boot CPUs in one shot to avoid races
1500 	 * with the userspace trying to use the CPU hotplug at the same time
1501 	 */
1502 	cpumask_clear(frozen_cpus);
1503 
1504 	pr_info("Disabling non-boot CPUs ...\n");
1505 	for_each_online_cpu(cpu) {
1506 		if (cpu == primary)
1507 			continue;
1508 
1509 		if (pm_wakeup_pending()) {
1510 			pr_info("Wakeup pending. Abort CPU freeze\n");
1511 			error = -EBUSY;
1512 			break;
1513 		}
1514 
1515 		trace_suspend_resume(TPS("CPU_OFF"), cpu, true);
1516 		error = _cpu_down(cpu, 1, CPUHP_OFFLINE);
1517 		trace_suspend_resume(TPS("CPU_OFF"), cpu, false);
1518 		if (!error)
1519 			cpumask_set_cpu(cpu, frozen_cpus);
1520 		else {
1521 			pr_err("Error taking CPU%d down: %d\n", cpu, error);
1522 			break;
1523 		}
1524 	}
1525 
1526 	if (!error)
1527 		BUG_ON(num_online_cpus() > 1);
1528 	else
1529 		pr_err("Non-boot CPUs are not disabled\n");
1530 
1531 	/*
1532 	 * Make sure the CPUs won't be enabled by someone else. We need to do
1533 	 * this even in case of failure as all freeze_secondary_cpus() users are
1534 	 * supposed to do thaw_secondary_cpus() on the failure path.
1535 	 */
1536 	cpu_hotplug_disabled++;
1537 
1538 	cpu_maps_update_done();
1539 	return error;
1540 }
1541 
1542 void __weak arch_thaw_secondary_cpus_begin(void)
1543 {
1544 }
1545 
1546 void __weak arch_thaw_secondary_cpus_end(void)
1547 {
1548 }
1549 
1550 void thaw_secondary_cpus(void)
1551 {
1552 	int cpu, error;
1553 
1554 	/* Allow everyone to use the CPU hotplug again */
1555 	cpu_maps_update_begin();
1556 	__cpu_hotplug_enable();
1557 	if (cpumask_empty(frozen_cpus))
1558 		goto out;
1559 
1560 	pr_info("Enabling non-boot CPUs ...\n");
1561 
1562 	arch_thaw_secondary_cpus_begin();
1563 
1564 	for_each_cpu(cpu, frozen_cpus) {
1565 		trace_suspend_resume(TPS("CPU_ON"), cpu, true);
1566 		error = _cpu_up(cpu, 1, CPUHP_ONLINE);
1567 		trace_suspend_resume(TPS("CPU_ON"), cpu, false);
1568 		if (!error) {
1569 			pr_info("CPU%d is up\n", cpu);
1570 			continue;
1571 		}
1572 		pr_warn("Error taking CPU%d up: %d\n", cpu, error);
1573 	}
1574 
1575 	arch_thaw_secondary_cpus_end();
1576 
1577 	cpumask_clear(frozen_cpus);
1578 out:
1579 	cpu_maps_update_done();
1580 }
1581 
1582 static int __init alloc_frozen_cpus(void)
1583 {
1584 	if (!alloc_cpumask_var(&frozen_cpus, GFP_KERNEL|__GFP_ZERO))
1585 		return -ENOMEM;
1586 	return 0;
1587 }
1588 core_initcall(alloc_frozen_cpus);
1589 
1590 /*
1591  * When callbacks for CPU hotplug notifications are being executed, we must
1592  * ensure that the state of the system with respect to the tasks being frozen
1593  * or not, as reported by the notification, remains unchanged *throughout the
1594  * duration* of the execution of the callbacks.
1595  * Hence we need to prevent the freezer from racing with regular CPU hotplug.
1596  *
1597  * This synchronization is implemented by mutually excluding regular CPU
1598  * hotplug and Suspend/Hibernate call paths by hooking onto the Suspend/
1599  * Hibernate notifications.
1600  */
1601 static int
1602 cpu_hotplug_pm_callback(struct notifier_block *nb,
1603 			unsigned long action, void *ptr)
1604 {
1605 	switch (action) {
1606 
1607 	case PM_SUSPEND_PREPARE:
1608 	case PM_HIBERNATION_PREPARE:
1609 		cpu_hotplug_disable();
1610 		break;
1611 
1612 	case PM_POST_SUSPEND:
1613 	case PM_POST_HIBERNATION:
1614 		cpu_hotplug_enable();
1615 		break;
1616 
1617 	default:
1618 		return NOTIFY_DONE;
1619 	}
1620 
1621 	return NOTIFY_OK;
1622 }
1623 
1624 
1625 static int __init cpu_hotplug_pm_sync_init(void)
1626 {
1627 	/*
1628 	 * cpu_hotplug_pm_callback has higher priority than x86
1629 	 * bsp_pm_callback which depends on cpu_hotplug_pm_callback
1630 	 * to disable cpu hotplug to avoid cpu hotplug race.
1631 	 */
1632 	pm_notifier(cpu_hotplug_pm_callback, 0);
1633 	return 0;
1634 }
1635 core_initcall(cpu_hotplug_pm_sync_init);
1636 
1637 #endif /* CONFIG_PM_SLEEP_SMP */
1638 
1639 int __boot_cpu_id;
1640 
1641 #endif /* CONFIG_SMP */
1642 
1643 /* Boot processor state steps */
1644 static struct cpuhp_step cpuhp_hp_states[] = {
1645 	[CPUHP_OFFLINE] = {
1646 		.name			= "offline",
1647 		.startup.single		= NULL,
1648 		.teardown.single	= NULL,
1649 	},
1650 #ifdef CONFIG_SMP
1651 	[CPUHP_CREATE_THREADS]= {
1652 		.name			= "threads:prepare",
1653 		.startup.single		= smpboot_create_threads,
1654 		.teardown.single	= NULL,
1655 		.cant_stop		= true,
1656 	},
1657 	[CPUHP_PERF_PREPARE] = {
1658 		.name			= "perf:prepare",
1659 		.startup.single		= perf_event_init_cpu,
1660 		.teardown.single	= perf_event_exit_cpu,
1661 	},
1662 	[CPUHP_WORKQUEUE_PREP] = {
1663 		.name			= "workqueue:prepare",
1664 		.startup.single		= workqueue_prepare_cpu,
1665 		.teardown.single	= NULL,
1666 	},
1667 	[CPUHP_HRTIMERS_PREPARE] = {
1668 		.name			= "hrtimers:prepare",
1669 		.startup.single		= hrtimers_prepare_cpu,
1670 		.teardown.single	= hrtimers_dead_cpu,
1671 	},
1672 	[CPUHP_SMPCFD_PREPARE] = {
1673 		.name			= "smpcfd:prepare",
1674 		.startup.single		= smpcfd_prepare_cpu,
1675 		.teardown.single	= smpcfd_dead_cpu,
1676 	},
1677 	[CPUHP_RELAY_PREPARE] = {
1678 		.name			= "relay:prepare",
1679 		.startup.single		= relay_prepare_cpu,
1680 		.teardown.single	= NULL,
1681 	},
1682 	[CPUHP_SLAB_PREPARE] = {
1683 		.name			= "slab:prepare",
1684 		.startup.single		= slab_prepare_cpu,
1685 		.teardown.single	= slab_dead_cpu,
1686 	},
1687 	[CPUHP_RCUTREE_PREP] = {
1688 		.name			= "RCU/tree:prepare",
1689 		.startup.single		= rcutree_prepare_cpu,
1690 		.teardown.single	= rcutree_dead_cpu,
1691 	},
1692 	/*
1693 	 * On the tear-down path, timers_dead_cpu() must be invoked
1694 	 * before blk_mq_queue_reinit_notify() from notify_dead(),
1695 	 * otherwise a RCU stall occurs.
1696 	 */
1697 	[CPUHP_TIMERS_PREPARE] = {
1698 		.name			= "timers:prepare",
1699 		.startup.single		= timers_prepare_cpu,
1700 		.teardown.single	= timers_dead_cpu,
1701 	},
1702 	/* Kicks the plugged cpu into life */
1703 	[CPUHP_BRINGUP_CPU] = {
1704 		.name			= "cpu:bringup",
1705 		.startup.single		= bringup_cpu,
1706 		.teardown.single	= finish_cpu,
1707 		.cant_stop		= true,
1708 	},
1709 	/* Final state before CPU kills itself */
1710 	[CPUHP_AP_IDLE_DEAD] = {
1711 		.name			= "idle:dead",
1712 	},
1713 	/*
1714 	 * Last state before CPU enters the idle loop to die. Transient state
1715 	 * for synchronization.
1716 	 */
1717 	[CPUHP_AP_OFFLINE] = {
1718 		.name			= "ap:offline",
1719 		.cant_stop		= true,
1720 	},
1721 	/* First state is scheduler control. Interrupts are disabled */
1722 	[CPUHP_AP_SCHED_STARTING] = {
1723 		.name			= "sched:starting",
1724 		.startup.single		= sched_cpu_starting,
1725 		.teardown.single	= sched_cpu_dying,
1726 	},
1727 	[CPUHP_AP_RCUTREE_DYING] = {
1728 		.name			= "RCU/tree:dying",
1729 		.startup.single		= NULL,
1730 		.teardown.single	= rcutree_dying_cpu,
1731 	},
1732 	[CPUHP_AP_SMPCFD_DYING] = {
1733 		.name			= "smpcfd:dying",
1734 		.startup.single		= NULL,
1735 		.teardown.single	= smpcfd_dying_cpu,
1736 	},
1737 	/* Entry state on starting. Interrupts enabled from here on. Transient
1738 	 * state for synchronsization */
1739 	[CPUHP_AP_ONLINE] = {
1740 		.name			= "ap:online",
1741 	},
1742 	/*
1743 	 * Handled on control processor until the plugged processor manages
1744 	 * this itself.
1745 	 */
1746 	[CPUHP_TEARDOWN_CPU] = {
1747 		.name			= "cpu:teardown",
1748 		.startup.single		= NULL,
1749 		.teardown.single	= takedown_cpu,
1750 		.cant_stop		= true,
1751 	},
1752 
1753 	[CPUHP_AP_SCHED_WAIT_EMPTY] = {
1754 		.name			= "sched:waitempty",
1755 		.startup.single		= NULL,
1756 		.teardown.single	= sched_cpu_wait_empty,
1757 	},
1758 
1759 	/* Handle smpboot threads park/unpark */
1760 	[CPUHP_AP_SMPBOOT_THREADS] = {
1761 		.name			= "smpboot/threads:online",
1762 		.startup.single		= smpboot_unpark_threads,
1763 		.teardown.single	= smpboot_park_threads,
1764 	},
1765 	[CPUHP_AP_IRQ_AFFINITY_ONLINE] = {
1766 		.name			= "irq/affinity:online",
1767 		.startup.single		= irq_affinity_online_cpu,
1768 		.teardown.single	= NULL,
1769 	},
1770 	[CPUHP_AP_PERF_ONLINE] = {
1771 		.name			= "perf:online",
1772 		.startup.single		= perf_event_init_cpu,
1773 		.teardown.single	= perf_event_exit_cpu,
1774 	},
1775 	[CPUHP_AP_WATCHDOG_ONLINE] = {
1776 		.name			= "lockup_detector:online",
1777 		.startup.single		= lockup_detector_online_cpu,
1778 		.teardown.single	= lockup_detector_offline_cpu,
1779 	},
1780 	[CPUHP_AP_WORKQUEUE_ONLINE] = {
1781 		.name			= "workqueue:online",
1782 		.startup.single		= workqueue_online_cpu,
1783 		.teardown.single	= workqueue_offline_cpu,
1784 	},
1785 	[CPUHP_AP_RCUTREE_ONLINE] = {
1786 		.name			= "RCU/tree:online",
1787 		.startup.single		= rcutree_online_cpu,
1788 		.teardown.single	= rcutree_offline_cpu,
1789 	},
1790 #endif
1791 	/*
1792 	 * The dynamically registered state space is here
1793 	 */
1794 
1795 #ifdef CONFIG_SMP
1796 	/* Last state is scheduler control setting the cpu active */
1797 	[CPUHP_AP_ACTIVE] = {
1798 		.name			= "sched:active",
1799 		.startup.single		= sched_cpu_activate,
1800 		.teardown.single	= sched_cpu_deactivate,
1801 	},
1802 #endif
1803 
1804 	/* CPU is fully up and running. */
1805 	[CPUHP_ONLINE] = {
1806 		.name			= "online",
1807 		.startup.single		= NULL,
1808 		.teardown.single	= NULL,
1809 	},
1810 };
1811 
1812 /* Sanity check for callbacks */
1813 static int cpuhp_cb_check(enum cpuhp_state state)
1814 {
1815 	if (state <= CPUHP_OFFLINE || state >= CPUHP_ONLINE)
1816 		return -EINVAL;
1817 	return 0;
1818 }
1819 
1820 /*
1821  * Returns a free for dynamic slot assignment of the Online state. The states
1822  * are protected by the cpuhp_slot_states mutex and an empty slot is identified
1823  * by having no name assigned.
1824  */
1825 static int cpuhp_reserve_state(enum cpuhp_state state)
1826 {
1827 	enum cpuhp_state i, end;
1828 	struct cpuhp_step *step;
1829 
1830 	switch (state) {
1831 	case CPUHP_AP_ONLINE_DYN:
1832 		step = cpuhp_hp_states + CPUHP_AP_ONLINE_DYN;
1833 		end = CPUHP_AP_ONLINE_DYN_END;
1834 		break;
1835 	case CPUHP_BP_PREPARE_DYN:
1836 		step = cpuhp_hp_states + CPUHP_BP_PREPARE_DYN;
1837 		end = CPUHP_BP_PREPARE_DYN_END;
1838 		break;
1839 	default:
1840 		return -EINVAL;
1841 	}
1842 
1843 	for (i = state; i <= end; i++, step++) {
1844 		if (!step->name)
1845 			return i;
1846 	}
1847 	WARN(1, "No more dynamic states available for CPU hotplug\n");
1848 	return -ENOSPC;
1849 }
1850 
1851 static int cpuhp_store_callbacks(enum cpuhp_state state, const char *name,
1852 				 int (*startup)(unsigned int cpu),
1853 				 int (*teardown)(unsigned int cpu),
1854 				 bool multi_instance)
1855 {
1856 	/* (Un)Install the callbacks for further cpu hotplug operations */
1857 	struct cpuhp_step *sp;
1858 	int ret = 0;
1859 
1860 	/*
1861 	 * If name is NULL, then the state gets removed.
1862 	 *
1863 	 * CPUHP_AP_ONLINE_DYN and CPUHP_BP_PREPARE_DYN are handed out on
1864 	 * the first allocation from these dynamic ranges, so the removal
1865 	 * would trigger a new allocation and clear the wrong (already
1866 	 * empty) state, leaving the callbacks of the to be cleared state
1867 	 * dangling, which causes wreckage on the next hotplug operation.
1868 	 */
1869 	if (name && (state == CPUHP_AP_ONLINE_DYN ||
1870 		     state == CPUHP_BP_PREPARE_DYN)) {
1871 		ret = cpuhp_reserve_state(state);
1872 		if (ret < 0)
1873 			return ret;
1874 		state = ret;
1875 	}
1876 	sp = cpuhp_get_step(state);
1877 	if (name && sp->name)
1878 		return -EBUSY;
1879 
1880 	sp->startup.single = startup;
1881 	sp->teardown.single = teardown;
1882 	sp->name = name;
1883 	sp->multi_instance = multi_instance;
1884 	INIT_HLIST_HEAD(&sp->list);
1885 	return ret;
1886 }
1887 
1888 static void *cpuhp_get_teardown_cb(enum cpuhp_state state)
1889 {
1890 	return cpuhp_get_step(state)->teardown.single;
1891 }
1892 
1893 /*
1894  * Call the startup/teardown function for a step either on the AP or
1895  * on the current CPU.
1896  */
1897 static int cpuhp_issue_call(int cpu, enum cpuhp_state state, bool bringup,
1898 			    struct hlist_node *node)
1899 {
1900 	struct cpuhp_step *sp = cpuhp_get_step(state);
1901 	int ret;
1902 
1903 	/*
1904 	 * If there's nothing to do, we done.
1905 	 * Relies on the union for multi_instance.
1906 	 */
1907 	if (cpuhp_step_empty(bringup, sp))
1908 		return 0;
1909 	/*
1910 	 * The non AP bound callbacks can fail on bringup. On teardown
1911 	 * e.g. module removal we crash for now.
1912 	 */
1913 #ifdef CONFIG_SMP
1914 	if (cpuhp_is_ap_state(state))
1915 		ret = cpuhp_invoke_ap_callback(cpu, state, bringup, node);
1916 	else
1917 		ret = cpuhp_invoke_callback(cpu, state, bringup, node, NULL);
1918 #else
1919 	ret = cpuhp_invoke_callback(cpu, state, bringup, node, NULL);
1920 #endif
1921 	BUG_ON(ret && !bringup);
1922 	return ret;
1923 }
1924 
1925 /*
1926  * Called from __cpuhp_setup_state on a recoverable failure.
1927  *
1928  * Note: The teardown callbacks for rollback are not allowed to fail!
1929  */
1930 static void cpuhp_rollback_install(int failedcpu, enum cpuhp_state state,
1931 				   struct hlist_node *node)
1932 {
1933 	int cpu;
1934 
1935 	/* Roll back the already executed steps on the other cpus */
1936 	for_each_present_cpu(cpu) {
1937 		struct cpuhp_cpu_state *st = per_cpu_ptr(&cpuhp_state, cpu);
1938 		int cpustate = st->state;
1939 
1940 		if (cpu >= failedcpu)
1941 			break;
1942 
1943 		/* Did we invoke the startup call on that cpu ? */
1944 		if (cpustate >= state)
1945 			cpuhp_issue_call(cpu, state, false, node);
1946 	}
1947 }
1948 
1949 int __cpuhp_state_add_instance_cpuslocked(enum cpuhp_state state,
1950 					  struct hlist_node *node,
1951 					  bool invoke)
1952 {
1953 	struct cpuhp_step *sp;
1954 	int cpu;
1955 	int ret;
1956 
1957 	lockdep_assert_cpus_held();
1958 
1959 	sp = cpuhp_get_step(state);
1960 	if (sp->multi_instance == false)
1961 		return -EINVAL;
1962 
1963 	mutex_lock(&cpuhp_state_mutex);
1964 
1965 	if (!invoke || !sp->startup.multi)
1966 		goto add_node;
1967 
1968 	/*
1969 	 * Try to call the startup callback for each present cpu
1970 	 * depending on the hotplug state of the cpu.
1971 	 */
1972 	for_each_present_cpu(cpu) {
1973 		struct cpuhp_cpu_state *st = per_cpu_ptr(&cpuhp_state, cpu);
1974 		int cpustate = st->state;
1975 
1976 		if (cpustate < state)
1977 			continue;
1978 
1979 		ret = cpuhp_issue_call(cpu, state, true, node);
1980 		if (ret) {
1981 			if (sp->teardown.multi)
1982 				cpuhp_rollback_install(cpu, state, node);
1983 			goto unlock;
1984 		}
1985 	}
1986 add_node:
1987 	ret = 0;
1988 	hlist_add_head(node, &sp->list);
1989 unlock:
1990 	mutex_unlock(&cpuhp_state_mutex);
1991 	return ret;
1992 }
1993 
1994 int __cpuhp_state_add_instance(enum cpuhp_state state, struct hlist_node *node,
1995 			       bool invoke)
1996 {
1997 	int ret;
1998 
1999 	cpus_read_lock();
2000 	ret = __cpuhp_state_add_instance_cpuslocked(state, node, invoke);
2001 	cpus_read_unlock();
2002 	return ret;
2003 }
2004 EXPORT_SYMBOL_GPL(__cpuhp_state_add_instance);
2005 
2006 /**
2007  * __cpuhp_setup_state_cpuslocked - Setup the callbacks for an hotplug machine state
2008  * @state:		The state to setup
2009  * @name:		Name of the step
2010  * @invoke:		If true, the startup function is invoked for cpus where
2011  *			cpu state >= @state
2012  * @startup:		startup callback function
2013  * @teardown:		teardown callback function
2014  * @multi_instance:	State is set up for multiple instances which get
2015  *			added afterwards.
2016  *
2017  * The caller needs to hold cpus read locked while calling this function.
2018  * Return:
2019  *   On success:
2020  *      Positive state number if @state is CPUHP_AP_ONLINE_DYN;
2021  *      0 for all other states
2022  *   On failure: proper (negative) error code
2023  */
2024 int __cpuhp_setup_state_cpuslocked(enum cpuhp_state state,
2025 				   const char *name, bool invoke,
2026 				   int (*startup)(unsigned int cpu),
2027 				   int (*teardown)(unsigned int cpu),
2028 				   bool multi_instance)
2029 {
2030 	int cpu, ret = 0;
2031 	bool dynstate;
2032 
2033 	lockdep_assert_cpus_held();
2034 
2035 	if (cpuhp_cb_check(state) || !name)
2036 		return -EINVAL;
2037 
2038 	mutex_lock(&cpuhp_state_mutex);
2039 
2040 	ret = cpuhp_store_callbacks(state, name, startup, teardown,
2041 				    multi_instance);
2042 
2043 	dynstate = state == CPUHP_AP_ONLINE_DYN;
2044 	if (ret > 0 && dynstate) {
2045 		state = ret;
2046 		ret = 0;
2047 	}
2048 
2049 	if (ret || !invoke || !startup)
2050 		goto out;
2051 
2052 	/*
2053 	 * Try to call the startup callback for each present cpu
2054 	 * depending on the hotplug state of the cpu.
2055 	 */
2056 	for_each_present_cpu(cpu) {
2057 		struct cpuhp_cpu_state *st = per_cpu_ptr(&cpuhp_state, cpu);
2058 		int cpustate = st->state;
2059 
2060 		if (cpustate < state)
2061 			continue;
2062 
2063 		ret = cpuhp_issue_call(cpu, state, true, NULL);
2064 		if (ret) {
2065 			if (teardown)
2066 				cpuhp_rollback_install(cpu, state, NULL);
2067 			cpuhp_store_callbacks(state, NULL, NULL, NULL, false);
2068 			goto out;
2069 		}
2070 	}
2071 out:
2072 	mutex_unlock(&cpuhp_state_mutex);
2073 	/*
2074 	 * If the requested state is CPUHP_AP_ONLINE_DYN, return the
2075 	 * dynamically allocated state in case of success.
2076 	 */
2077 	if (!ret && dynstate)
2078 		return state;
2079 	return ret;
2080 }
2081 EXPORT_SYMBOL(__cpuhp_setup_state_cpuslocked);
2082 
2083 int __cpuhp_setup_state(enum cpuhp_state state,
2084 			const char *name, bool invoke,
2085 			int (*startup)(unsigned int cpu),
2086 			int (*teardown)(unsigned int cpu),
2087 			bool multi_instance)
2088 {
2089 	int ret;
2090 
2091 	cpus_read_lock();
2092 	ret = __cpuhp_setup_state_cpuslocked(state, name, invoke, startup,
2093 					     teardown, multi_instance);
2094 	cpus_read_unlock();
2095 	return ret;
2096 }
2097 EXPORT_SYMBOL(__cpuhp_setup_state);
2098 
2099 int __cpuhp_state_remove_instance(enum cpuhp_state state,
2100 				  struct hlist_node *node, bool invoke)
2101 {
2102 	struct cpuhp_step *sp = cpuhp_get_step(state);
2103 	int cpu;
2104 
2105 	BUG_ON(cpuhp_cb_check(state));
2106 
2107 	if (!sp->multi_instance)
2108 		return -EINVAL;
2109 
2110 	cpus_read_lock();
2111 	mutex_lock(&cpuhp_state_mutex);
2112 
2113 	if (!invoke || !cpuhp_get_teardown_cb(state))
2114 		goto remove;
2115 	/*
2116 	 * Call the teardown callback for each present cpu depending
2117 	 * on the hotplug state of the cpu. This function is not
2118 	 * allowed to fail currently!
2119 	 */
2120 	for_each_present_cpu(cpu) {
2121 		struct cpuhp_cpu_state *st = per_cpu_ptr(&cpuhp_state, cpu);
2122 		int cpustate = st->state;
2123 
2124 		if (cpustate >= state)
2125 			cpuhp_issue_call(cpu, state, false, node);
2126 	}
2127 
2128 remove:
2129 	hlist_del(node);
2130 	mutex_unlock(&cpuhp_state_mutex);
2131 	cpus_read_unlock();
2132 
2133 	return 0;
2134 }
2135 EXPORT_SYMBOL_GPL(__cpuhp_state_remove_instance);
2136 
2137 /**
2138  * __cpuhp_remove_state_cpuslocked - Remove the callbacks for an hotplug machine state
2139  * @state:	The state to remove
2140  * @invoke:	If true, the teardown function is invoked for cpus where
2141  *		cpu state >= @state
2142  *
2143  * The caller needs to hold cpus read locked while calling this function.
2144  * The teardown callback is currently not allowed to fail. Think
2145  * about module removal!
2146  */
2147 void __cpuhp_remove_state_cpuslocked(enum cpuhp_state state, bool invoke)
2148 {
2149 	struct cpuhp_step *sp = cpuhp_get_step(state);
2150 	int cpu;
2151 
2152 	BUG_ON(cpuhp_cb_check(state));
2153 
2154 	lockdep_assert_cpus_held();
2155 
2156 	mutex_lock(&cpuhp_state_mutex);
2157 	if (sp->multi_instance) {
2158 		WARN(!hlist_empty(&sp->list),
2159 		     "Error: Removing state %d which has instances left.\n",
2160 		     state);
2161 		goto remove;
2162 	}
2163 
2164 	if (!invoke || !cpuhp_get_teardown_cb(state))
2165 		goto remove;
2166 
2167 	/*
2168 	 * Call the teardown callback for each present cpu depending
2169 	 * on the hotplug state of the cpu. This function is not
2170 	 * allowed to fail currently!
2171 	 */
2172 	for_each_present_cpu(cpu) {
2173 		struct cpuhp_cpu_state *st = per_cpu_ptr(&cpuhp_state, cpu);
2174 		int cpustate = st->state;
2175 
2176 		if (cpustate >= state)
2177 			cpuhp_issue_call(cpu, state, false, NULL);
2178 	}
2179 remove:
2180 	cpuhp_store_callbacks(state, NULL, NULL, NULL, false);
2181 	mutex_unlock(&cpuhp_state_mutex);
2182 }
2183 EXPORT_SYMBOL(__cpuhp_remove_state_cpuslocked);
2184 
2185 void __cpuhp_remove_state(enum cpuhp_state state, bool invoke)
2186 {
2187 	cpus_read_lock();
2188 	__cpuhp_remove_state_cpuslocked(state, invoke);
2189 	cpus_read_unlock();
2190 }
2191 EXPORT_SYMBOL(__cpuhp_remove_state);
2192 
2193 #ifdef CONFIG_HOTPLUG_SMT
2194 static void cpuhp_offline_cpu_device(unsigned int cpu)
2195 {
2196 	struct device *dev = get_cpu_device(cpu);
2197 
2198 	dev->offline = true;
2199 	/* Tell user space about the state change */
2200 	kobject_uevent(&dev->kobj, KOBJ_OFFLINE);
2201 }
2202 
2203 static void cpuhp_online_cpu_device(unsigned int cpu)
2204 {
2205 	struct device *dev = get_cpu_device(cpu);
2206 
2207 	dev->offline = false;
2208 	/* Tell user space about the state change */
2209 	kobject_uevent(&dev->kobj, KOBJ_ONLINE);
2210 }
2211 
2212 int cpuhp_smt_disable(enum cpuhp_smt_control ctrlval)
2213 {
2214 	int cpu, ret = 0;
2215 
2216 	cpu_maps_update_begin();
2217 	for_each_online_cpu(cpu) {
2218 		if (topology_is_primary_thread(cpu))
2219 			continue;
2220 		ret = cpu_down_maps_locked(cpu, CPUHP_OFFLINE);
2221 		if (ret)
2222 			break;
2223 		/*
2224 		 * As this needs to hold the cpu maps lock it's impossible
2225 		 * to call device_offline() because that ends up calling
2226 		 * cpu_down() which takes cpu maps lock. cpu maps lock
2227 		 * needs to be held as this might race against in kernel
2228 		 * abusers of the hotplug machinery (thermal management).
2229 		 *
2230 		 * So nothing would update device:offline state. That would
2231 		 * leave the sysfs entry stale and prevent onlining after
2232 		 * smt control has been changed to 'off' again. This is
2233 		 * called under the sysfs hotplug lock, so it is properly
2234 		 * serialized against the regular offline usage.
2235 		 */
2236 		cpuhp_offline_cpu_device(cpu);
2237 	}
2238 	if (!ret)
2239 		cpu_smt_control = ctrlval;
2240 	cpu_maps_update_done();
2241 	return ret;
2242 }
2243 
2244 int cpuhp_smt_enable(void)
2245 {
2246 	int cpu, ret = 0;
2247 
2248 	cpu_maps_update_begin();
2249 	cpu_smt_control = CPU_SMT_ENABLED;
2250 	for_each_present_cpu(cpu) {
2251 		/* Skip online CPUs and CPUs on offline nodes */
2252 		if (cpu_online(cpu) || !node_online(cpu_to_node(cpu)))
2253 			continue;
2254 		ret = _cpu_up(cpu, 0, CPUHP_ONLINE);
2255 		if (ret)
2256 			break;
2257 		/* See comment in cpuhp_smt_disable() */
2258 		cpuhp_online_cpu_device(cpu);
2259 	}
2260 	cpu_maps_update_done();
2261 	return ret;
2262 }
2263 #endif
2264 
2265 #if defined(CONFIG_SYSFS) && defined(CONFIG_HOTPLUG_CPU)
2266 static ssize_t state_show(struct device *dev,
2267 			  struct device_attribute *attr, char *buf)
2268 {
2269 	struct cpuhp_cpu_state *st = per_cpu_ptr(&cpuhp_state, dev->id);
2270 
2271 	return sprintf(buf, "%d\n", st->state);
2272 }
2273 static DEVICE_ATTR_RO(state);
2274 
2275 static ssize_t target_store(struct device *dev, struct device_attribute *attr,
2276 			    const char *buf, size_t count)
2277 {
2278 	struct cpuhp_cpu_state *st = per_cpu_ptr(&cpuhp_state, dev->id);
2279 	struct cpuhp_step *sp;
2280 	int target, ret;
2281 
2282 	ret = kstrtoint(buf, 10, &target);
2283 	if (ret)
2284 		return ret;
2285 
2286 #ifdef CONFIG_CPU_HOTPLUG_STATE_CONTROL
2287 	if (target < CPUHP_OFFLINE || target > CPUHP_ONLINE)
2288 		return -EINVAL;
2289 #else
2290 	if (target != CPUHP_OFFLINE && target != CPUHP_ONLINE)
2291 		return -EINVAL;
2292 #endif
2293 
2294 	ret = lock_device_hotplug_sysfs();
2295 	if (ret)
2296 		return ret;
2297 
2298 	mutex_lock(&cpuhp_state_mutex);
2299 	sp = cpuhp_get_step(target);
2300 	ret = !sp->name || sp->cant_stop ? -EINVAL : 0;
2301 	mutex_unlock(&cpuhp_state_mutex);
2302 	if (ret)
2303 		goto out;
2304 
2305 	if (st->state < target)
2306 		ret = cpu_up(dev->id, target);
2307 	else
2308 		ret = cpu_down(dev->id, target);
2309 out:
2310 	unlock_device_hotplug();
2311 	return ret ? ret : count;
2312 }
2313 
2314 static ssize_t target_show(struct device *dev,
2315 			   struct device_attribute *attr, char *buf)
2316 {
2317 	struct cpuhp_cpu_state *st = per_cpu_ptr(&cpuhp_state, dev->id);
2318 
2319 	return sprintf(buf, "%d\n", st->target);
2320 }
2321 static DEVICE_ATTR_RW(target);
2322 
2323 static ssize_t fail_store(struct device *dev, struct device_attribute *attr,
2324 			  const char *buf, size_t count)
2325 {
2326 	struct cpuhp_cpu_state *st = per_cpu_ptr(&cpuhp_state, dev->id);
2327 	struct cpuhp_step *sp;
2328 	int fail, ret;
2329 
2330 	ret = kstrtoint(buf, 10, &fail);
2331 	if (ret)
2332 		return ret;
2333 
2334 	if (fail == CPUHP_INVALID) {
2335 		st->fail = fail;
2336 		return count;
2337 	}
2338 
2339 	if (fail < CPUHP_OFFLINE || fail > CPUHP_ONLINE)
2340 		return -EINVAL;
2341 
2342 	/*
2343 	 * Cannot fail STARTING/DYING callbacks.
2344 	 */
2345 	if (cpuhp_is_atomic_state(fail))
2346 		return -EINVAL;
2347 
2348 	/*
2349 	 * DEAD callbacks cannot fail...
2350 	 * ... neither can CPUHP_BRINGUP_CPU during hotunplug. The latter
2351 	 * triggering STARTING callbacks, a failure in this state would
2352 	 * hinder rollback.
2353 	 */
2354 	if (fail <= CPUHP_BRINGUP_CPU && st->state > CPUHP_BRINGUP_CPU)
2355 		return -EINVAL;
2356 
2357 	/*
2358 	 * Cannot fail anything that doesn't have callbacks.
2359 	 */
2360 	mutex_lock(&cpuhp_state_mutex);
2361 	sp = cpuhp_get_step(fail);
2362 	if (!sp->startup.single && !sp->teardown.single)
2363 		ret = -EINVAL;
2364 	mutex_unlock(&cpuhp_state_mutex);
2365 	if (ret)
2366 		return ret;
2367 
2368 	st->fail = fail;
2369 
2370 	return count;
2371 }
2372 
2373 static ssize_t fail_show(struct device *dev,
2374 			 struct device_attribute *attr, char *buf)
2375 {
2376 	struct cpuhp_cpu_state *st = per_cpu_ptr(&cpuhp_state, dev->id);
2377 
2378 	return sprintf(buf, "%d\n", st->fail);
2379 }
2380 
2381 static DEVICE_ATTR_RW(fail);
2382 
2383 static struct attribute *cpuhp_cpu_attrs[] = {
2384 	&dev_attr_state.attr,
2385 	&dev_attr_target.attr,
2386 	&dev_attr_fail.attr,
2387 	NULL
2388 };
2389 
2390 static const struct attribute_group cpuhp_cpu_attr_group = {
2391 	.attrs = cpuhp_cpu_attrs,
2392 	.name = "hotplug",
2393 	NULL
2394 };
2395 
2396 static ssize_t states_show(struct device *dev,
2397 				 struct device_attribute *attr, char *buf)
2398 {
2399 	ssize_t cur, res = 0;
2400 	int i;
2401 
2402 	mutex_lock(&cpuhp_state_mutex);
2403 	for (i = CPUHP_OFFLINE; i <= CPUHP_ONLINE; i++) {
2404 		struct cpuhp_step *sp = cpuhp_get_step(i);
2405 
2406 		if (sp->name) {
2407 			cur = sprintf(buf, "%3d: %s\n", i, sp->name);
2408 			buf += cur;
2409 			res += cur;
2410 		}
2411 	}
2412 	mutex_unlock(&cpuhp_state_mutex);
2413 	return res;
2414 }
2415 static DEVICE_ATTR_RO(states);
2416 
2417 static struct attribute *cpuhp_cpu_root_attrs[] = {
2418 	&dev_attr_states.attr,
2419 	NULL
2420 };
2421 
2422 static const struct attribute_group cpuhp_cpu_root_attr_group = {
2423 	.attrs = cpuhp_cpu_root_attrs,
2424 	.name = "hotplug",
2425 	NULL
2426 };
2427 
2428 #ifdef CONFIG_HOTPLUG_SMT
2429 
2430 static ssize_t
2431 __store_smt_control(struct device *dev, struct device_attribute *attr,
2432 		    const char *buf, size_t count)
2433 {
2434 	int ctrlval, ret;
2435 
2436 	if (sysfs_streq(buf, "on"))
2437 		ctrlval = CPU_SMT_ENABLED;
2438 	else if (sysfs_streq(buf, "off"))
2439 		ctrlval = CPU_SMT_DISABLED;
2440 	else if (sysfs_streq(buf, "forceoff"))
2441 		ctrlval = CPU_SMT_FORCE_DISABLED;
2442 	else
2443 		return -EINVAL;
2444 
2445 	if (cpu_smt_control == CPU_SMT_FORCE_DISABLED)
2446 		return -EPERM;
2447 
2448 	if (cpu_smt_control == CPU_SMT_NOT_SUPPORTED)
2449 		return -ENODEV;
2450 
2451 	ret = lock_device_hotplug_sysfs();
2452 	if (ret)
2453 		return ret;
2454 
2455 	if (ctrlval != cpu_smt_control) {
2456 		switch (ctrlval) {
2457 		case CPU_SMT_ENABLED:
2458 			ret = cpuhp_smt_enable();
2459 			break;
2460 		case CPU_SMT_DISABLED:
2461 		case CPU_SMT_FORCE_DISABLED:
2462 			ret = cpuhp_smt_disable(ctrlval);
2463 			break;
2464 		}
2465 	}
2466 
2467 	unlock_device_hotplug();
2468 	return ret ? ret : count;
2469 }
2470 
2471 #else /* !CONFIG_HOTPLUG_SMT */
2472 static ssize_t
2473 __store_smt_control(struct device *dev, struct device_attribute *attr,
2474 		    const char *buf, size_t count)
2475 {
2476 	return -ENODEV;
2477 }
2478 #endif /* CONFIG_HOTPLUG_SMT */
2479 
2480 static const char *smt_states[] = {
2481 	[CPU_SMT_ENABLED]		= "on",
2482 	[CPU_SMT_DISABLED]		= "off",
2483 	[CPU_SMT_FORCE_DISABLED]	= "forceoff",
2484 	[CPU_SMT_NOT_SUPPORTED]		= "notsupported",
2485 	[CPU_SMT_NOT_IMPLEMENTED]	= "notimplemented",
2486 };
2487 
2488 static ssize_t control_show(struct device *dev,
2489 			    struct device_attribute *attr, char *buf)
2490 {
2491 	const char *state = smt_states[cpu_smt_control];
2492 
2493 	return snprintf(buf, PAGE_SIZE - 2, "%s\n", state);
2494 }
2495 
2496 static ssize_t control_store(struct device *dev, struct device_attribute *attr,
2497 			     const char *buf, size_t count)
2498 {
2499 	return __store_smt_control(dev, attr, buf, count);
2500 }
2501 static DEVICE_ATTR_RW(control);
2502 
2503 static ssize_t active_show(struct device *dev,
2504 			   struct device_attribute *attr, char *buf)
2505 {
2506 	return snprintf(buf, PAGE_SIZE - 2, "%d\n", sched_smt_active());
2507 }
2508 static DEVICE_ATTR_RO(active);
2509 
2510 static struct attribute *cpuhp_smt_attrs[] = {
2511 	&dev_attr_control.attr,
2512 	&dev_attr_active.attr,
2513 	NULL
2514 };
2515 
2516 static const struct attribute_group cpuhp_smt_attr_group = {
2517 	.attrs = cpuhp_smt_attrs,
2518 	.name = "smt",
2519 	NULL
2520 };
2521 
2522 static int __init cpu_smt_sysfs_init(void)
2523 {
2524 	return sysfs_create_group(&cpu_subsys.dev_root->kobj,
2525 				  &cpuhp_smt_attr_group);
2526 }
2527 
2528 static int __init cpuhp_sysfs_init(void)
2529 {
2530 	int cpu, ret;
2531 
2532 	ret = cpu_smt_sysfs_init();
2533 	if (ret)
2534 		return ret;
2535 
2536 	ret = sysfs_create_group(&cpu_subsys.dev_root->kobj,
2537 				 &cpuhp_cpu_root_attr_group);
2538 	if (ret)
2539 		return ret;
2540 
2541 	for_each_possible_cpu(cpu) {
2542 		struct device *dev = get_cpu_device(cpu);
2543 
2544 		if (!dev)
2545 			continue;
2546 		ret = sysfs_create_group(&dev->kobj, &cpuhp_cpu_attr_group);
2547 		if (ret)
2548 			return ret;
2549 	}
2550 	return 0;
2551 }
2552 device_initcall(cpuhp_sysfs_init);
2553 #endif /* CONFIG_SYSFS && CONFIG_HOTPLUG_CPU */
2554 
2555 /*
2556  * cpu_bit_bitmap[] is a special, "compressed" data structure that
2557  * represents all NR_CPUS bits binary values of 1<<nr.
2558  *
2559  * It is used by cpumask_of() to get a constant address to a CPU
2560  * mask value that has a single bit set only.
2561  */
2562 
2563 /* cpu_bit_bitmap[0] is empty - so we can back into it */
2564 #define MASK_DECLARE_1(x)	[x+1][0] = (1UL << (x))
2565 #define MASK_DECLARE_2(x)	MASK_DECLARE_1(x), MASK_DECLARE_1(x+1)
2566 #define MASK_DECLARE_4(x)	MASK_DECLARE_2(x), MASK_DECLARE_2(x+2)
2567 #define MASK_DECLARE_8(x)	MASK_DECLARE_4(x), MASK_DECLARE_4(x+4)
2568 
2569 const unsigned long cpu_bit_bitmap[BITS_PER_LONG+1][BITS_TO_LONGS(NR_CPUS)] = {
2570 
2571 	MASK_DECLARE_8(0),	MASK_DECLARE_8(8),
2572 	MASK_DECLARE_8(16),	MASK_DECLARE_8(24),
2573 #if BITS_PER_LONG > 32
2574 	MASK_DECLARE_8(32),	MASK_DECLARE_8(40),
2575 	MASK_DECLARE_8(48),	MASK_DECLARE_8(56),
2576 #endif
2577 };
2578 EXPORT_SYMBOL_GPL(cpu_bit_bitmap);
2579 
2580 const DECLARE_BITMAP(cpu_all_bits, NR_CPUS) = CPU_BITS_ALL;
2581 EXPORT_SYMBOL(cpu_all_bits);
2582 
2583 #ifdef CONFIG_INIT_ALL_POSSIBLE
2584 struct cpumask __cpu_possible_mask __read_mostly
2585 	= {CPU_BITS_ALL};
2586 #else
2587 struct cpumask __cpu_possible_mask __read_mostly;
2588 #endif
2589 EXPORT_SYMBOL(__cpu_possible_mask);
2590 
2591 struct cpumask __cpu_online_mask __read_mostly;
2592 EXPORT_SYMBOL(__cpu_online_mask);
2593 
2594 struct cpumask __cpu_present_mask __read_mostly;
2595 EXPORT_SYMBOL(__cpu_present_mask);
2596 
2597 struct cpumask __cpu_active_mask __read_mostly;
2598 EXPORT_SYMBOL(__cpu_active_mask);
2599 
2600 struct cpumask __cpu_dying_mask __read_mostly;
2601 EXPORT_SYMBOL(__cpu_dying_mask);
2602 
2603 atomic_t __num_online_cpus __read_mostly;
2604 EXPORT_SYMBOL(__num_online_cpus);
2605 
2606 void init_cpu_present(const struct cpumask *src)
2607 {
2608 	cpumask_copy(&__cpu_present_mask, src);
2609 }
2610 
2611 void init_cpu_possible(const struct cpumask *src)
2612 {
2613 	cpumask_copy(&__cpu_possible_mask, src);
2614 }
2615 
2616 void init_cpu_online(const struct cpumask *src)
2617 {
2618 	cpumask_copy(&__cpu_online_mask, src);
2619 }
2620 
2621 void set_cpu_online(unsigned int cpu, bool online)
2622 {
2623 	/*
2624 	 * atomic_inc/dec() is required to handle the horrid abuse of this
2625 	 * function by the reboot and kexec code which invoke it from
2626 	 * IPI/NMI broadcasts when shutting down CPUs. Invocation from
2627 	 * regular CPU hotplug is properly serialized.
2628 	 *
2629 	 * Note, that the fact that __num_online_cpus is of type atomic_t
2630 	 * does not protect readers which are not serialized against
2631 	 * concurrent hotplug operations.
2632 	 */
2633 	if (online) {
2634 		if (!cpumask_test_and_set_cpu(cpu, &__cpu_online_mask))
2635 			atomic_inc(&__num_online_cpus);
2636 	} else {
2637 		if (cpumask_test_and_clear_cpu(cpu, &__cpu_online_mask))
2638 			atomic_dec(&__num_online_cpus);
2639 	}
2640 }
2641 
2642 /*
2643  * Activate the first processor.
2644  */
2645 void __init boot_cpu_init(void)
2646 {
2647 	int cpu = smp_processor_id();
2648 
2649 	/* Mark the boot cpu "present", "online" etc for SMP and UP case */
2650 	set_cpu_online(cpu, true);
2651 	set_cpu_active(cpu, true);
2652 	set_cpu_present(cpu, true);
2653 	set_cpu_possible(cpu, true);
2654 
2655 #ifdef CONFIG_SMP
2656 	__boot_cpu_id = cpu;
2657 #endif
2658 }
2659 
2660 /*
2661  * Must be called _AFTER_ setting up the per_cpu areas
2662  */
2663 void __init boot_cpu_hotplug_init(void)
2664 {
2665 #ifdef CONFIG_SMP
2666 	cpumask_set_cpu(smp_processor_id(), &cpus_booted_once_mask);
2667 #endif
2668 	this_cpu_write(cpuhp_state.state, CPUHP_ONLINE);
2669 }
2670 
2671 /*
2672  * These are used for a global "mitigations=" cmdline option for toggling
2673  * optional CPU mitigations.
2674  */
2675 enum cpu_mitigations {
2676 	CPU_MITIGATIONS_OFF,
2677 	CPU_MITIGATIONS_AUTO,
2678 	CPU_MITIGATIONS_AUTO_NOSMT,
2679 };
2680 
2681 static enum cpu_mitigations cpu_mitigations __ro_after_init =
2682 	CPU_MITIGATIONS_AUTO;
2683 
2684 static int __init mitigations_parse_cmdline(char *arg)
2685 {
2686 	if (!strcmp(arg, "off"))
2687 		cpu_mitigations = CPU_MITIGATIONS_OFF;
2688 	else if (!strcmp(arg, "auto"))
2689 		cpu_mitigations = CPU_MITIGATIONS_AUTO;
2690 	else if (!strcmp(arg, "auto,nosmt"))
2691 		cpu_mitigations = CPU_MITIGATIONS_AUTO_NOSMT;
2692 	else
2693 		pr_crit("Unsupported mitigations=%s, system may still be vulnerable\n",
2694 			arg);
2695 
2696 	return 0;
2697 }
2698 early_param("mitigations", mitigations_parse_cmdline);
2699 
2700 /* mitigations=off */
2701 bool cpu_mitigations_off(void)
2702 {
2703 	return cpu_mitigations == CPU_MITIGATIONS_OFF;
2704 }
2705 EXPORT_SYMBOL_GPL(cpu_mitigations_off);
2706 
2707 /* mitigations=auto,nosmt */
2708 bool cpu_mitigations_auto_nosmt(void)
2709 {
2710 	return cpu_mitigations == CPU_MITIGATIONS_AUTO_NOSMT;
2711 }
2712 EXPORT_SYMBOL_GPL(cpu_mitigations_auto_nosmt);
2713