16ff3f917SThomas Gleixner // SPDX-License-Identifier: GPL-2.0-or-later
21142d810STejun Heo /*
31142d810STejun Heo * kernel/stop_machine.c
41142d810STejun Heo *
51142d810STejun Heo * Copyright (C) 2008, 2005 IBM Corporation.
61142d810STejun Heo * Copyright (C) 2008, 2005 Rusty Russell rusty@rustcorp.com.au
71142d810STejun Heo * Copyright (C) 2010 SUSE Linux Products GmbH
81142d810STejun Heo * Copyright (C) 2010 Tejun Heo <tj@kernel.org>
9e5582ca2SRusty Russell */
10b1fc5833SMark Rutland #include <linux/compiler.h>
111142d810STejun Heo #include <linux/completion.h>
121da177e4SLinus Torvalds #include <linux/cpu.h>
131142d810STejun Heo #include <linux/init.h>
14ee527cd3SPrarit Bhargava #include <linux/kthread.h>
159984de1aSPaul Gortmaker #include <linux/export.h>
161142d810STejun Heo #include <linux/percpu.h>
17ee527cd3SPrarit Bhargava #include <linux/sched.h>
18ee527cd3SPrarit Bhargava #include <linux/stop_machine.h>
19a12bb444SBenjamin Herrenschmidt #include <linux/interrupt.h>
201142d810STejun Heo #include <linux/kallsyms.h>
2114e568e7SThomas Gleixner #include <linux/smpboot.h>
2260063497SArun Sharma #include <linux/atomic.h>
23ce4f06dcSOleg Nesterov #include <linux/nmi.h>
240b26351bSPeter Zijlstra #include <linux/sched/wake_q.h>
251142d810STejun Heo
261142d810STejun Heo /*
271142d810STejun Heo * Structure to determine completion condition and record errors. May
281142d810STejun Heo * be shared by works on different cpus.
291142d810STejun Heo */
301142d810STejun Heo struct cpu_stop_done {
311142d810STejun Heo atomic_t nr_todo; /* nr left to execute */
321142d810STejun Heo int ret; /* collected return value */
331142d810STejun Heo struct completion completion; /* fired if nr_todo reaches 0 */
341142d810STejun Heo };
351142d810STejun Heo
361142d810STejun Heo /* the actual stopper, one per every possible cpu, enabled on online cpus */
371142d810STejun Heo struct cpu_stopper {
3802cb7aa9SOleg Nesterov struct task_struct *thread;
3902cb7aa9SOleg Nesterov
40de5b55c1SThomas Gleixner raw_spinlock_t lock;
41878ae127SRichard Kennedy bool enabled; /* is this stopper enabled? */
421142d810STejun Heo struct list_head works; /* list of pending works */
4302cb7aa9SOleg Nesterov
4402cb7aa9SOleg Nesterov struct cpu_stop_work stop_work; /* for stop_cpus */
45a8b62fd0SPeter Zijlstra unsigned long caller;
46a8b62fd0SPeter Zijlstra cpu_stop_fn_t fn;
471142d810STejun Heo };
481142d810STejun Heo
491142d810STejun Heo static DEFINE_PER_CPU(struct cpu_stopper, cpu_stopper);
50f445027eSJeremy Fitzhardinge static bool stop_machine_initialized = false;
511142d810STejun Heo
print_stop_info(const char * log_lvl,struct task_struct * task)52a8b62fd0SPeter Zijlstra void print_stop_info(const char *log_lvl, struct task_struct *task)
53a8b62fd0SPeter Zijlstra {
54a8b62fd0SPeter Zijlstra /*
55a8b62fd0SPeter Zijlstra * If @task is a stopper task, it cannot migrate and task_cpu() is
56a8b62fd0SPeter Zijlstra * stable.
57a8b62fd0SPeter Zijlstra */
58a8b62fd0SPeter Zijlstra struct cpu_stopper *stopper = per_cpu_ptr(&cpu_stopper, task_cpu(task));
59a8b62fd0SPeter Zijlstra
60a8b62fd0SPeter Zijlstra if (task != stopper->thread)
61a8b62fd0SPeter Zijlstra return;
62a8b62fd0SPeter Zijlstra
63a8b62fd0SPeter Zijlstra printk("%sStopper: %pS <- %pS\n", log_lvl, stopper->fn, (void *)stopper->caller);
64a8b62fd0SPeter Zijlstra }
65a8b62fd0SPeter Zijlstra
66e6253970SOleg Nesterov /* static data for stop_cpus */
67e6253970SOleg Nesterov static DEFINE_MUTEX(stop_cpus_mutex);
68e6253970SOleg Nesterov static bool stop_cpus_in_progress;
697053ea1aSRik van Riel
cpu_stop_init_done(struct cpu_stop_done * done,unsigned int nr_todo)701142d810STejun Heo static void cpu_stop_init_done(struct cpu_stop_done *done, unsigned int nr_todo)
711142d810STejun Heo {
721142d810STejun Heo memset(done, 0, sizeof(*done));
731142d810STejun Heo atomic_set(&done->nr_todo, nr_todo);
741142d810STejun Heo init_completion(&done->completion);
751142d810STejun Heo }
761142d810STejun Heo
771142d810STejun Heo /* signal completion unless @done is NULL */
cpu_stop_signal_done(struct cpu_stop_done * done)786fa3b826SOleg Nesterov static void cpu_stop_signal_done(struct cpu_stop_done *done)
791142d810STejun Heo {
801142d810STejun Heo if (atomic_dec_and_test(&done->nr_todo))
811142d810STejun Heo complete(&done->completion);
821142d810STejun Heo }
831142d810STejun Heo
__cpu_stop_queue_work(struct cpu_stopper * stopper,struct cpu_stop_work * work,struct wake_q_head * wakeq)845caa1c08SOleg Nesterov static void __cpu_stop_queue_work(struct cpu_stopper *stopper,
850b26351bSPeter Zijlstra struct cpu_stop_work *work,
860b26351bSPeter Zijlstra struct wake_q_head *wakeq)
875caa1c08SOleg Nesterov {
885caa1c08SOleg Nesterov list_add_tail(&work->list, &stopper->works);
890b26351bSPeter Zijlstra wake_q_add(wakeq, stopper->thread);
905caa1c08SOleg Nesterov }
915caa1c08SOleg Nesterov
921142d810STejun Heo /* queue @work to @stopper. if offline, @work is completed immediately */
cpu_stop_queue_work(unsigned int cpu,struct cpu_stop_work * work)931b034bd9SOleg Nesterov static bool cpu_stop_queue_work(unsigned int cpu, struct cpu_stop_work *work)
941142d810STejun Heo {
95860a0ffaSThomas Gleixner struct cpu_stopper *stopper = &per_cpu(cpu_stopper, cpu);
960b26351bSPeter Zijlstra DEFINE_WAKE_Q(wakeq);
971142d810STejun Heo unsigned long flags;
981b034bd9SOleg Nesterov bool enabled;
991142d810STejun Heo
100cfd35514SPrasad Sodagudi preempt_disable();
101de5b55c1SThomas Gleixner raw_spin_lock_irqsave(&stopper->lock, flags);
1021b034bd9SOleg Nesterov enabled = stopper->enabled;
1031b034bd9SOleg Nesterov if (enabled)
1040b26351bSPeter Zijlstra __cpu_stop_queue_work(stopper, work, &wakeq);
105dd2e3121SOleg Nesterov else if (work->done)
1066fa3b826SOleg Nesterov cpu_stop_signal_done(work->done);
107de5b55c1SThomas Gleixner raw_spin_unlock_irqrestore(&stopper->lock, flags);
1081b034bd9SOleg Nesterov
1090b26351bSPeter Zijlstra wake_up_q(&wakeq);
110cfd35514SPrasad Sodagudi preempt_enable();
1110b26351bSPeter Zijlstra
1121b034bd9SOleg Nesterov return enabled;
1131142d810STejun Heo }
1141142d810STejun Heo
1151142d810STejun Heo /**
1161142d810STejun Heo * stop_one_cpu - stop a cpu
1171142d810STejun Heo * @cpu: cpu to stop
1181142d810STejun Heo * @fn: function to execute
1191142d810STejun Heo * @arg: argument to @fn
1201142d810STejun Heo *
1211142d810STejun Heo * Execute @fn(@arg) on @cpu. @fn is run in a process context with
1221142d810STejun Heo * the highest priority preempting any task on the cpu and
1231142d810STejun Heo * monopolizing it. This function returns after the execution is
1241142d810STejun Heo * complete.
1251142d810STejun Heo *
1261142d810STejun Heo * This function doesn't guarantee @cpu stays online till @fn
1271142d810STejun Heo * completes. If @cpu goes down in the middle, execution may happen
1281142d810STejun Heo * partially or fully on different cpus. @fn should either be ready
1291142d810STejun Heo * for that or the caller should ensure that @cpu stays online until
1301142d810STejun Heo * this function completes.
1311142d810STejun Heo *
1321142d810STejun Heo * CONTEXT:
1331142d810STejun Heo * Might sleep.
1341142d810STejun Heo *
1351142d810STejun Heo * RETURNS:
1361142d810STejun Heo * -ENOENT if @fn(@arg) was not executed because @cpu was offline;
1371142d810STejun Heo * otherwise, the return value of @fn.
1381142d810STejun Heo */
stop_one_cpu(unsigned int cpu,cpu_stop_fn_t fn,void * arg)1391142d810STejun Heo int stop_one_cpu(unsigned int cpu, cpu_stop_fn_t fn, void *arg)
1401142d810STejun Heo {
1411142d810STejun Heo struct cpu_stop_done done;
142a8b62fd0SPeter Zijlstra struct cpu_stop_work work = { .fn = fn, .arg = arg, .done = &done, .caller = _RET_IP_ };
1431142d810STejun Heo
1441142d810STejun Heo cpu_stop_init_done(&done, 1);
145958c5f84SOleg Nesterov if (!cpu_stop_queue_work(cpu, &work))
146958c5f84SOleg Nesterov return -ENOENT;
147bf89a304SCheng Chao /*
148bf89a304SCheng Chao * In case @cpu == smp_proccessor_id() we can avoid a sleep+wakeup
149bf89a304SCheng Chao * cycle by doing a preemption:
150bf89a304SCheng Chao */
151bf89a304SCheng Chao cond_resched();
1521142d810STejun Heo wait_for_completion(&done.completion);
153958c5f84SOleg Nesterov return done.ret;
1541142d810STejun Heo }
1551142d810STejun Heo
1561be0bd77SPeter Zijlstra /* This controls the threads on each CPU. */
1571be0bd77SPeter Zijlstra enum multi_stop_state {
1581be0bd77SPeter Zijlstra /* Dummy starting state for thread. */
1591be0bd77SPeter Zijlstra MULTI_STOP_NONE,
1601be0bd77SPeter Zijlstra /* Awaiting everyone to be scheduled. */
1611be0bd77SPeter Zijlstra MULTI_STOP_PREPARE,
1621be0bd77SPeter Zijlstra /* Disable interrupts. */
1631be0bd77SPeter Zijlstra MULTI_STOP_DISABLE_IRQ,
1641be0bd77SPeter Zijlstra /* Run the function */
1651be0bd77SPeter Zijlstra MULTI_STOP_RUN,
1661be0bd77SPeter Zijlstra /* Exit */
1671be0bd77SPeter Zijlstra MULTI_STOP_EXIT,
1681be0bd77SPeter Zijlstra };
1691be0bd77SPeter Zijlstra
1701be0bd77SPeter Zijlstra struct multi_stop_data {
1719a301f22SOleg Nesterov cpu_stop_fn_t fn;
1721be0bd77SPeter Zijlstra void *data;
1731be0bd77SPeter Zijlstra /* Like num_online_cpus(), but hotplug cpu uses us, so we need this. */
1741be0bd77SPeter Zijlstra unsigned int num_threads;
1751be0bd77SPeter Zijlstra const struct cpumask *active_cpus;
1761be0bd77SPeter Zijlstra
1771be0bd77SPeter Zijlstra enum multi_stop_state state;
1781be0bd77SPeter Zijlstra atomic_t thread_ack;
1791be0bd77SPeter Zijlstra };
1801be0bd77SPeter Zijlstra
set_state(struct multi_stop_data * msdata,enum multi_stop_state newstate)1811be0bd77SPeter Zijlstra static void set_state(struct multi_stop_data *msdata,
1821be0bd77SPeter Zijlstra enum multi_stop_state newstate)
1831be0bd77SPeter Zijlstra {
1841be0bd77SPeter Zijlstra /* Reset ack counter. */
1851be0bd77SPeter Zijlstra atomic_set(&msdata->thread_ack, msdata->num_threads);
1861be0bd77SPeter Zijlstra smp_wmb();
187b1fc5833SMark Rutland WRITE_ONCE(msdata->state, newstate);
1881be0bd77SPeter Zijlstra }
1891be0bd77SPeter Zijlstra
1901be0bd77SPeter Zijlstra /* Last one to ack a state moves to the next state. */
ack_state(struct multi_stop_data * msdata)1911be0bd77SPeter Zijlstra static void ack_state(struct multi_stop_data *msdata)
1921be0bd77SPeter Zijlstra {
1931be0bd77SPeter Zijlstra if (atomic_dec_and_test(&msdata->thread_ack))
1941be0bd77SPeter Zijlstra set_state(msdata, msdata->state + 1);
1951be0bd77SPeter Zijlstra }
1961be0bd77SPeter Zijlstra
stop_machine_yield(const struct cpumask * cpumask)1974230e2deSZong Li notrace void __weak stop_machine_yield(const struct cpumask *cpumask)
1984ecf0a43SHeiko Carstens {
1994ecf0a43SHeiko Carstens cpu_relax();
2004ecf0a43SHeiko Carstens }
2014ecf0a43SHeiko Carstens
2021be0bd77SPeter Zijlstra /* This is the cpu_stop function which stops the CPU. */
multi_cpu_stop(void * data)2031be0bd77SPeter Zijlstra static int multi_cpu_stop(void *data)
2041be0bd77SPeter Zijlstra {
2051be0bd77SPeter Zijlstra struct multi_stop_data *msdata = data;
206b1fc5833SMark Rutland enum multi_stop_state newstate, curstate = MULTI_STOP_NONE;
2071be0bd77SPeter Zijlstra int cpu = smp_processor_id(), err = 0;
20838f2c691SMartin Schwidefsky const struct cpumask *cpumask;
2091be0bd77SPeter Zijlstra unsigned long flags;
2101be0bd77SPeter Zijlstra bool is_active;
2111be0bd77SPeter Zijlstra
2121be0bd77SPeter Zijlstra /*
2131be0bd77SPeter Zijlstra * When called from stop_machine_from_inactive_cpu(), irq might
2141be0bd77SPeter Zijlstra * already be disabled. Save the state and restore it on exit.
2151be0bd77SPeter Zijlstra */
2161be0bd77SPeter Zijlstra local_save_flags(flags);
2171be0bd77SPeter Zijlstra
21838f2c691SMartin Schwidefsky if (!msdata->active_cpus) {
21938f2c691SMartin Schwidefsky cpumask = cpu_online_mask;
22038f2c691SMartin Schwidefsky is_active = cpu == cpumask_first(cpumask);
22138f2c691SMartin Schwidefsky } else {
22238f2c691SMartin Schwidefsky cpumask = msdata->active_cpus;
22338f2c691SMartin Schwidefsky is_active = cpumask_test_cpu(cpu, cpumask);
22438f2c691SMartin Schwidefsky }
2251be0bd77SPeter Zijlstra
2261be0bd77SPeter Zijlstra /* Simple state machine */
2271be0bd77SPeter Zijlstra do {
2281be0bd77SPeter Zijlstra /* Chill out and ensure we re-read multi_stop_state. */
2294ecf0a43SHeiko Carstens stop_machine_yield(cpumask);
230b1fc5833SMark Rutland newstate = READ_ONCE(msdata->state);
231b1fc5833SMark Rutland if (newstate != curstate) {
232b1fc5833SMark Rutland curstate = newstate;
2331be0bd77SPeter Zijlstra switch (curstate) {
2341be0bd77SPeter Zijlstra case MULTI_STOP_DISABLE_IRQ:
2351be0bd77SPeter Zijlstra local_irq_disable();
2361be0bd77SPeter Zijlstra hard_irq_disable();
2371be0bd77SPeter Zijlstra break;
2381be0bd77SPeter Zijlstra case MULTI_STOP_RUN:
2391be0bd77SPeter Zijlstra if (is_active)
2401be0bd77SPeter Zijlstra err = msdata->fn(msdata->data);
2411be0bd77SPeter Zijlstra break;
2421be0bd77SPeter Zijlstra default:
2431be0bd77SPeter Zijlstra break;
2441be0bd77SPeter Zijlstra }
2451be0bd77SPeter Zijlstra ack_state(msdata);
246ce4f06dcSOleg Nesterov } else if (curstate > MULTI_STOP_PREPARE) {
247ce4f06dcSOleg Nesterov /*
248ce4f06dcSOleg Nesterov * At this stage all other CPUs we depend on must spin
249ce4f06dcSOleg Nesterov * in the same loop. Any reason for hard-lockup should
250ce4f06dcSOleg Nesterov * be detected and reported on their side.
251ce4f06dcSOleg Nesterov */
252ce4f06dcSOleg Nesterov touch_nmi_watchdog();
2531be0bd77SPeter Zijlstra }
254366237e7SPaul E. McKenney rcu_momentary_dyntick_idle();
2551be0bd77SPeter Zijlstra } while (curstate != MULTI_STOP_EXIT);
2561be0bd77SPeter Zijlstra
2571be0bd77SPeter Zijlstra local_irq_restore(flags);
2581be0bd77SPeter Zijlstra return err;
2591be0bd77SPeter Zijlstra }
2601be0bd77SPeter Zijlstra
cpu_stop_queue_two_works(int cpu1,struct cpu_stop_work * work1,int cpu2,struct cpu_stop_work * work2)2615caa1c08SOleg Nesterov static int cpu_stop_queue_two_works(int cpu1, struct cpu_stop_work *work1,
2625caa1c08SOleg Nesterov int cpu2, struct cpu_stop_work *work2)
2635caa1c08SOleg Nesterov {
264d8bc8535SOleg Nesterov struct cpu_stopper *stopper1 = per_cpu_ptr(&cpu_stopper, cpu1);
265d8bc8535SOleg Nesterov struct cpu_stopper *stopper2 = per_cpu_ptr(&cpu_stopper, cpu2);
2660b26351bSPeter Zijlstra DEFINE_WAKE_Q(wakeq);
267d8bc8535SOleg Nesterov int err;
268b80a2bfcSPeter Zijlstra
269e6253970SOleg Nesterov retry:
270b80a2bfcSPeter Zijlstra /*
271b80a2bfcSPeter Zijlstra * The waking up of stopper threads has to happen in the same
272b80a2bfcSPeter Zijlstra * scheduling context as the queueing. Otherwise, there is a
273b80a2bfcSPeter Zijlstra * possibility of one of the above stoppers being woken up by another
274b80a2bfcSPeter Zijlstra * CPU, and preempting us. This will cause us to not wake up the other
275b80a2bfcSPeter Zijlstra * stopper forever.
276b80a2bfcSPeter Zijlstra */
277b80a2bfcSPeter Zijlstra preempt_disable();
278de5b55c1SThomas Gleixner raw_spin_lock_irq(&stopper1->lock);
279de5b55c1SThomas Gleixner raw_spin_lock_nested(&stopper2->lock, SINGLE_DEPTH_NESTING);
280d8bc8535SOleg Nesterov
281b80a2bfcSPeter Zijlstra if (!stopper1->enabled || !stopper2->enabled) {
282d8bc8535SOleg Nesterov err = -ENOENT;
283d8bc8535SOleg Nesterov goto unlock;
284b80a2bfcSPeter Zijlstra }
285b80a2bfcSPeter Zijlstra
286e6253970SOleg Nesterov /*
287e6253970SOleg Nesterov * Ensure that if we race with __stop_cpus() the stoppers won't get
288e6253970SOleg Nesterov * queued up in reverse order leading to system deadlock.
289e6253970SOleg Nesterov *
290e6253970SOleg Nesterov * We can't miss stop_cpus_in_progress if queue_stop_cpus_work() has
291e6253970SOleg Nesterov * queued a work on cpu1 but not on cpu2, we hold both locks.
292e6253970SOleg Nesterov *
293e6253970SOleg Nesterov * It can be falsely true but it is safe to spin until it is cleared,
294e6253970SOleg Nesterov * queue_stop_cpus_work() does everything under preempt_disable().
295e6253970SOleg Nesterov */
296b80a2bfcSPeter Zijlstra if (unlikely(stop_cpus_in_progress)) {
297e6253970SOleg Nesterov err = -EDEADLK;
298e6253970SOleg Nesterov goto unlock;
299b80a2bfcSPeter Zijlstra }
300d8bc8535SOleg Nesterov
301d8bc8535SOleg Nesterov err = 0;
3020b26351bSPeter Zijlstra __cpu_stop_queue_work(stopper1, work1, &wakeq);
3030b26351bSPeter Zijlstra __cpu_stop_queue_work(stopper2, work2, &wakeq);
304b80a2bfcSPeter Zijlstra
305d8bc8535SOleg Nesterov unlock:
306de5b55c1SThomas Gleixner raw_spin_unlock(&stopper2->lock);
307de5b55c1SThomas Gleixner raw_spin_unlock_irq(&stopper1->lock);
3085caa1c08SOleg Nesterov
309e6253970SOleg Nesterov if (unlikely(err == -EDEADLK)) {
310b80a2bfcSPeter Zijlstra preempt_enable();
311b80a2bfcSPeter Zijlstra
312e6253970SOleg Nesterov while (stop_cpus_in_progress)
313e6253970SOleg Nesterov cpu_relax();
314b80a2bfcSPeter Zijlstra
315e6253970SOleg Nesterov goto retry;
316e6253970SOleg Nesterov }
3170b26351bSPeter Zijlstra
3180b26351bSPeter Zijlstra wake_up_q(&wakeq);
3199fb8d5dcSIsaac J. Manjarres preempt_enable();
3200b26351bSPeter Zijlstra
321d8bc8535SOleg Nesterov return err;
3225caa1c08SOleg Nesterov }
3231be0bd77SPeter Zijlstra /**
3241be0bd77SPeter Zijlstra * stop_two_cpus - stops two cpus
3251be0bd77SPeter Zijlstra * @cpu1: the cpu to stop
3261be0bd77SPeter Zijlstra * @cpu2: the other cpu to stop
3271be0bd77SPeter Zijlstra * @fn: function to execute
3281be0bd77SPeter Zijlstra * @arg: argument to @fn
3291be0bd77SPeter Zijlstra *
3301be0bd77SPeter Zijlstra * Stops both the current and specified CPU and runs @fn on one of them.
3311be0bd77SPeter Zijlstra *
3321be0bd77SPeter Zijlstra * returns when both are completed.
3331be0bd77SPeter Zijlstra */
stop_two_cpus(unsigned int cpu1,unsigned int cpu2,cpu_stop_fn_t fn,void * arg)3341be0bd77SPeter Zijlstra int stop_two_cpus(unsigned int cpu1, unsigned int cpu2, cpu_stop_fn_t fn, void *arg)
3351be0bd77SPeter Zijlstra {
3361be0bd77SPeter Zijlstra struct cpu_stop_done done;
3371be0bd77SPeter Zijlstra struct cpu_stop_work work1, work2;
3386acce3efSPeter Zijlstra struct multi_stop_data msdata;
3396acce3efSPeter Zijlstra
3406acce3efSPeter Zijlstra msdata = (struct multi_stop_data){
3411be0bd77SPeter Zijlstra .fn = fn,
3421be0bd77SPeter Zijlstra .data = arg,
3431be0bd77SPeter Zijlstra .num_threads = 2,
3441be0bd77SPeter Zijlstra .active_cpus = cpumask_of(cpu1),
3451be0bd77SPeter Zijlstra };
3461be0bd77SPeter Zijlstra
3471be0bd77SPeter Zijlstra work1 = work2 = (struct cpu_stop_work){
3481be0bd77SPeter Zijlstra .fn = multi_cpu_stop,
3491be0bd77SPeter Zijlstra .arg = &msdata,
350a8b62fd0SPeter Zijlstra .done = &done,
351a8b62fd0SPeter Zijlstra .caller = _RET_IP_,
3521be0bd77SPeter Zijlstra };
3531be0bd77SPeter Zijlstra
3541be0bd77SPeter Zijlstra cpu_stop_init_done(&done, 2);
3551be0bd77SPeter Zijlstra set_state(&msdata, MULTI_STOP_PREPARE);
3561be0bd77SPeter Zijlstra
3575caa1c08SOleg Nesterov if (cpu1 > cpu2)
3585caa1c08SOleg Nesterov swap(cpu1, cpu2);
3596a190051SOleg Nesterov if (cpu_stop_queue_two_works(cpu1, &work1, cpu2, &work2))
3605caa1c08SOleg Nesterov return -ENOENT;
3611be0bd77SPeter Zijlstra
3621be0bd77SPeter Zijlstra wait_for_completion(&done.completion);
3636a190051SOleg Nesterov return done.ret;
3641be0bd77SPeter Zijlstra }
3651be0bd77SPeter Zijlstra
3661142d810STejun Heo /**
3671142d810STejun Heo * stop_one_cpu_nowait - stop a cpu but don't wait for completion
3681142d810STejun Heo * @cpu: cpu to stop
3691142d810STejun Heo * @fn: function to execute
3701142d810STejun Heo * @arg: argument to @fn
371cf250040SFabian Frederick * @work_buf: pointer to cpu_stop_work structure
3721142d810STejun Heo *
3731142d810STejun Heo * Similar to stop_one_cpu() but doesn't wait for completion. The
3741142d810STejun Heo * caller is responsible for ensuring @work_buf is currently unused
3751142d810STejun Heo * and will remain untouched until stopper starts executing @fn.
3761142d810STejun Heo *
3771142d810STejun Heo * CONTEXT:
3781142d810STejun Heo * Don't care.
3791b034bd9SOleg Nesterov *
3801b034bd9SOleg Nesterov * RETURNS:
3811b034bd9SOleg Nesterov * true if cpu_stop_work was queued successfully and @fn will be called,
3821b034bd9SOleg Nesterov * false otherwise.
3831142d810STejun Heo */
stop_one_cpu_nowait(unsigned int cpu,cpu_stop_fn_t fn,void * arg,struct cpu_stop_work * work_buf)3841b034bd9SOleg Nesterov bool stop_one_cpu_nowait(unsigned int cpu, cpu_stop_fn_t fn, void *arg,
3851142d810STejun Heo struct cpu_stop_work *work_buf)
3861142d810STejun Heo {
387a8b62fd0SPeter Zijlstra *work_buf = (struct cpu_stop_work){ .fn = fn, .arg = arg, .caller = _RET_IP_, };
3881b034bd9SOleg Nesterov return cpu_stop_queue_work(cpu, work_buf);
3891142d810STejun Heo }
3901142d810STejun Heo
queue_stop_cpus_work(const struct cpumask * cpumask,cpu_stop_fn_t fn,void * arg,struct cpu_stop_done * done)3914aff1ca6SOleg Nesterov static bool queue_stop_cpus_work(const struct cpumask *cpumask,
392fd7355baSTejun Heo cpu_stop_fn_t fn, void *arg,
393fd7355baSTejun Heo struct cpu_stop_done *done)
3941142d810STejun Heo {
3951142d810STejun Heo struct cpu_stop_work *work;
3961142d810STejun Heo unsigned int cpu;
3974aff1ca6SOleg Nesterov bool queued = false;
3981142d810STejun Heo
3991142d810STejun Heo /*
4001142d810STejun Heo * Disable preemption while queueing to avoid getting
4011142d810STejun Heo * preempted by a stopper which might wait for other stoppers
4021142d810STejun Heo * to enter @fn which can lead to deadlock.
4031142d810STejun Heo */
404e6253970SOleg Nesterov preempt_disable();
405e6253970SOleg Nesterov stop_cpus_in_progress = true;
40699d84bf8SPeter Zijlstra barrier();
407b377c2a0SOleg Nesterov for_each_cpu(cpu, cpumask) {
408b377c2a0SOleg Nesterov work = &per_cpu(cpu_stopper.stop_work, cpu);
409b377c2a0SOleg Nesterov work->fn = fn;
410b377c2a0SOleg Nesterov work->arg = arg;
411b377c2a0SOleg Nesterov work->done = done;
4122a2f80ffSValentin Schneider work->caller = _RET_IP_;
4134aff1ca6SOleg Nesterov if (cpu_stop_queue_work(cpu, work))
4144aff1ca6SOleg Nesterov queued = true;
415b377c2a0SOleg Nesterov }
41699d84bf8SPeter Zijlstra barrier();
417e6253970SOleg Nesterov stop_cpus_in_progress = false;
418e6253970SOleg Nesterov preempt_enable();
4194aff1ca6SOleg Nesterov
4204aff1ca6SOleg Nesterov return queued;
421fd7355baSTejun Heo }
4221142d810STejun Heo
__stop_cpus(const struct cpumask * cpumask,cpu_stop_fn_t fn,void * arg)423fd7355baSTejun Heo static int __stop_cpus(const struct cpumask *cpumask,
424fd7355baSTejun Heo cpu_stop_fn_t fn, void *arg)
425fd7355baSTejun Heo {
426fd7355baSTejun Heo struct cpu_stop_done done;
427fd7355baSTejun Heo
428fd7355baSTejun Heo cpu_stop_init_done(&done, cpumask_weight(cpumask));
4294aff1ca6SOleg Nesterov if (!queue_stop_cpus_work(cpumask, fn, arg, &done))
4304aff1ca6SOleg Nesterov return -ENOENT;
4311142d810STejun Heo wait_for_completion(&done.completion);
4324aff1ca6SOleg Nesterov return done.ret;
4331142d810STejun Heo }
4341142d810STejun Heo
4351142d810STejun Heo /**
4361142d810STejun Heo * stop_cpus - stop multiple cpus
4371142d810STejun Heo * @cpumask: cpus to stop
4381142d810STejun Heo * @fn: function to execute
4391142d810STejun Heo * @arg: argument to @fn
4401142d810STejun Heo *
4411142d810STejun Heo * Execute @fn(@arg) on online cpus in @cpumask. On each target cpu,
4421142d810STejun Heo * @fn is run in a process context with the highest priority
4431142d810STejun Heo * preempting any task on the cpu and monopolizing it. This function
4441142d810STejun Heo * returns after all executions are complete.
4451142d810STejun Heo *
4461142d810STejun Heo * This function doesn't guarantee the cpus in @cpumask stay online
4471142d810STejun Heo * till @fn completes. If some cpus go down in the middle, execution
4481142d810STejun Heo * on the cpu may happen partially or fully on different cpus. @fn
4491142d810STejun Heo * should either be ready for that or the caller should ensure that
4501142d810STejun Heo * the cpus stay online until this function completes.
4511142d810STejun Heo *
4521142d810STejun Heo * All stop_cpus() calls are serialized making it safe for @fn to wait
4531142d810STejun Heo * for all cpus to start executing it.
4541142d810STejun Heo *
4551142d810STejun Heo * CONTEXT:
4561142d810STejun Heo * Might sleep.
4571142d810STejun Heo *
4581142d810STejun Heo * RETURNS:
4591142d810STejun Heo * -ENOENT if @fn(@arg) was not executed at all because all cpus in
4601142d810STejun Heo * @cpumask were offline; otherwise, 0 if all executions of @fn
4611142d810STejun Heo * returned 0, any non zero return value if any returned non zero.
4621142d810STejun Heo */
stop_cpus(const struct cpumask * cpumask,cpu_stop_fn_t fn,void * arg)46335f4cd96SYangtao Li static int stop_cpus(const struct cpumask *cpumask, cpu_stop_fn_t fn, void *arg)
4641142d810STejun Heo {
4651142d810STejun Heo int ret;
4661142d810STejun Heo
4671142d810STejun Heo /* static works are used, process one request at a time */
4681142d810STejun Heo mutex_lock(&stop_cpus_mutex);
4691142d810STejun Heo ret = __stop_cpus(cpumask, fn, arg);
4701142d810STejun Heo mutex_unlock(&stop_cpus_mutex);
4711142d810STejun Heo return ret;
4721142d810STejun Heo }
4731142d810STejun Heo
cpu_stop_should_run(unsigned int cpu)47414e568e7SThomas Gleixner static int cpu_stop_should_run(unsigned int cpu)
4751142d810STejun Heo {
47614e568e7SThomas Gleixner struct cpu_stopper *stopper = &per_cpu(cpu_stopper, cpu);
47714e568e7SThomas Gleixner unsigned long flags;
47814e568e7SThomas Gleixner int run;
47914e568e7SThomas Gleixner
480de5b55c1SThomas Gleixner raw_spin_lock_irqsave(&stopper->lock, flags);
48114e568e7SThomas Gleixner run = !list_empty(&stopper->works);
482de5b55c1SThomas Gleixner raw_spin_unlock_irqrestore(&stopper->lock, flags);
48314e568e7SThomas Gleixner return run;
48414e568e7SThomas Gleixner }
48514e568e7SThomas Gleixner
cpu_stopper_thread(unsigned int cpu)48614e568e7SThomas Gleixner static void cpu_stopper_thread(unsigned int cpu)
48714e568e7SThomas Gleixner {
48814e568e7SThomas Gleixner struct cpu_stopper *stopper = &per_cpu(cpu_stopper, cpu);
4891142d810STejun Heo struct cpu_stop_work *work;
4901142d810STejun Heo
4911142d810STejun Heo repeat:
4921142d810STejun Heo work = NULL;
493de5b55c1SThomas Gleixner raw_spin_lock_irq(&stopper->lock);
4941142d810STejun Heo if (!list_empty(&stopper->works)) {
4951142d810STejun Heo work = list_first_entry(&stopper->works,
4961142d810STejun Heo struct cpu_stop_work, list);
4971142d810STejun Heo list_del_init(&work->list);
4981142d810STejun Heo }
499de5b55c1SThomas Gleixner raw_spin_unlock_irq(&stopper->lock);
5001142d810STejun Heo
5011142d810STejun Heo if (work) {
5021142d810STejun Heo cpu_stop_fn_t fn = work->fn;
5031142d810STejun Heo void *arg = work->arg;
5041142d810STejun Heo struct cpu_stop_done *done = work->done;
505accaf6eaSOleg Nesterov int ret;
5061142d810STejun Heo
507accaf6eaSOleg Nesterov /* cpu stop callbacks must not sleep, make in_atomic() == T */
508a8b62fd0SPeter Zijlstra stopper->caller = work->caller;
509a8b62fd0SPeter Zijlstra stopper->fn = fn;
510accaf6eaSOleg Nesterov preempt_count_inc();
5111142d810STejun Heo ret = fn(arg);
512dd2e3121SOleg Nesterov if (done) {
513dd2e3121SOleg Nesterov if (ret)
5141142d810STejun Heo done->ret = ret;
5156fa3b826SOleg Nesterov cpu_stop_signal_done(done);
516dd2e3121SOleg Nesterov }
517accaf6eaSOleg Nesterov preempt_count_dec();
518a8b62fd0SPeter Zijlstra stopper->fn = NULL;
519a8b62fd0SPeter Zijlstra stopper->caller = 0;
5201142d810STejun Heo WARN_ONCE(preempt_count(),
521d75f773cSSakari Ailus "cpu_stop: %ps(%p) leaked preempt count\n", fn, arg);
5221142d810STejun Heo goto repeat;
5231142d810STejun Heo }
52414e568e7SThomas Gleixner }
5251142d810STejun Heo
stop_machine_park(int cpu)526233e7f26SOleg Nesterov void stop_machine_park(int cpu)
527233e7f26SOleg Nesterov {
528233e7f26SOleg Nesterov struct cpu_stopper *stopper = &per_cpu(cpu_stopper, cpu);
529233e7f26SOleg Nesterov /*
530233e7f26SOleg Nesterov * Lockless. cpu_stopper_thread() will take stopper->lock and flush
531233e7f26SOleg Nesterov * the pending works before it parks, until then it is fine to queue
532233e7f26SOleg Nesterov * the new works.
533233e7f26SOleg Nesterov */
534233e7f26SOleg Nesterov stopper->enabled = false;
535233e7f26SOleg Nesterov kthread_park(stopper->thread);
536233e7f26SOleg Nesterov }
537233e7f26SOleg Nesterov
cpu_stop_create(unsigned int cpu)53834f971f6SPeter Zijlstra static void cpu_stop_create(unsigned int cpu)
53934f971f6SPeter Zijlstra {
54014e568e7SThomas Gleixner sched_set_stop_task(cpu, per_cpu(cpu_stopper.thread, cpu));
5411142d810STejun Heo }
54202cb7aa9SOleg Nesterov
cpu_stop_park(unsigned int cpu)54314e568e7SThomas Gleixner static void cpu_stop_park(unsigned int cpu)
54414e568e7SThomas Gleixner {
54514e568e7SThomas Gleixner struct cpu_stopper *stopper = &per_cpu(cpu_stopper, cpu);
54614e568e7SThomas Gleixner
5471142d810STejun Heo WARN_ON(!list_empty(&stopper->works));
5489c6f7e43SIngo Molnar }
549233e7f26SOleg Nesterov
stop_machine_unpark(int cpu)55014e568e7SThomas Gleixner void stop_machine_unpark(int cpu)
55114e568e7SThomas Gleixner {
552c00166d8SOleg Nesterov struct cpu_stopper *stopper = &per_cpu(cpu_stopper, cpu);
553c00166d8SOleg Nesterov
554c00166d8SOleg Nesterov stopper->enabled = true;
555c00166d8SOleg Nesterov kthread_unpark(stopper->thread);
556f0cf16cbSOleg Nesterov }
557c00166d8SOleg Nesterov
558c00166d8SOleg Nesterov static struct smp_hotplug_thread cpu_stop_threads = {
559c00166d8SOleg Nesterov .store = &cpu_stopper.thread,
56014e568e7SThomas Gleixner .thread_should_run = cpu_stop_should_run,
56102cb7aa9SOleg Nesterov .thread_fn = cpu_stopper_thread,
56214e568e7SThomas Gleixner .thread_comm = "migration/%u",
56314e568e7SThomas Gleixner .create = cpu_stop_create,
56414e568e7SThomas Gleixner .park = cpu_stop_park,
56514e568e7SThomas Gleixner .selfparking = true,
56614e568e7SThomas Gleixner };
56714e568e7SThomas Gleixner
cpu_stop_init(void)5681142d810STejun Heo static int __init cpu_stop_init(void)
5691142d810STejun Heo {
5701142d810STejun Heo unsigned int cpu;
5711142d810STejun Heo
5721142d810STejun Heo for_each_possible_cpu(cpu) {
5731142d810STejun Heo struct cpu_stopper *stopper = &per_cpu(cpu_stopper, cpu);
5741142d810STejun Heo
5751142d810STejun Heo raw_spin_lock_init(&stopper->lock);
5761142d810STejun Heo INIT_LIST_HEAD(&stopper->works);
577de5b55c1SThomas Gleixner }
5781142d810STejun Heo
5791142d810STejun Heo BUG_ON(smpboot_register_percpu_thread(&cpu_stop_threads));
5801142d810STejun Heo stop_machine_unpark(raw_smp_processor_id());
58114e568e7SThomas Gleixner stop_machine_initialized = true;
582c00166d8SOleg Nesterov return 0;
583f445027eSJeremy Fitzhardinge }
5841142d810STejun Heo early_initcall(cpu_stop_init);
5851142d810STejun Heo
stop_machine_cpuslocked(cpu_stop_fn_t fn,void * data,const struct cpumask * cpus)5861142d810STejun Heo int stop_machine_cpuslocked(cpu_stop_fn_t fn, void *data,
5871da177e4SLinus Torvalds const struct cpumask *cpus)
588fe5595c0SSebastian Andrzej Siewior {
589fe5595c0SSebastian Andrzej Siewior struct multi_stop_data msdata = {
5901da177e4SLinus Torvalds .fn = fn,
5911be0bd77SPeter Zijlstra .data = data,
5921be0bd77SPeter Zijlstra .num_threads = num_online_cpus(),
5931be0bd77SPeter Zijlstra .active_cpus = cpus,
5943fc1f1e2STejun Heo };
5951be0bd77SPeter Zijlstra
5961be0bd77SPeter Zijlstra lockdep_assert_cpus_held();
5971da177e4SLinus Torvalds
598fe5595c0SSebastian Andrzej Siewior if (!stop_machine_initialized) {
599fe5595c0SSebastian Andrzej Siewior /*
600f445027eSJeremy Fitzhardinge * Handle the case where stop_machine() is called
601f445027eSJeremy Fitzhardinge * early in boot before stop_machine() has been
602f445027eSJeremy Fitzhardinge * initialized.
603f445027eSJeremy Fitzhardinge */
604f445027eSJeremy Fitzhardinge unsigned long flags;
605f445027eSJeremy Fitzhardinge int ret;
606f445027eSJeremy Fitzhardinge
607f445027eSJeremy Fitzhardinge WARN_ON_ONCE(msdata.num_threads != 1);
608f445027eSJeremy Fitzhardinge
6091be0bd77SPeter Zijlstra local_irq_save(flags);
610f445027eSJeremy Fitzhardinge hard_irq_disable();
611f445027eSJeremy Fitzhardinge ret = (*fn)(data);
612f445027eSJeremy Fitzhardinge local_irq_restore(flags);
613f445027eSJeremy Fitzhardinge
614f445027eSJeremy Fitzhardinge return ret;
615f445027eSJeremy Fitzhardinge }
616f445027eSJeremy Fitzhardinge
617f445027eSJeremy Fitzhardinge /* Set the initial state and stop all online cpus. */
618f445027eSJeremy Fitzhardinge set_state(&msdata, MULTI_STOP_PREPARE);
6193fc1f1e2STejun Heo return stop_cpus(cpu_online_mask, multi_cpu_stop, &msdata);
6201be0bd77SPeter Zijlstra }
6211be0bd77SPeter Zijlstra
stop_machine(cpu_stop_fn_t fn,void * data,const struct cpumask * cpus)6221da177e4SLinus Torvalds int stop_machine(cpu_stop_fn_t fn, void *data, const struct cpumask *cpus)
6231da177e4SLinus Torvalds {
6249a301f22SOleg Nesterov int ret;
6251da177e4SLinus Torvalds
6261da177e4SLinus Torvalds /* No CPUs can come up or down during this. */
6271da177e4SLinus Torvalds cpus_read_lock();
6281da177e4SLinus Torvalds ret = stop_machine_cpuslocked(fn, data, cpus);
629fe5595c0SSebastian Andrzej Siewior cpus_read_unlock();
630fe5595c0SSebastian Andrzej Siewior return ret;
631fe5595c0SSebastian Andrzej Siewior }
6321da177e4SLinus Torvalds EXPORT_SYMBOL_GPL(stop_machine);
6331da177e4SLinus Torvalds
634eeec4fadSRusty Russell #ifdef CONFIG_SCHED_SMT
stop_core_cpuslocked(unsigned int cpu,cpu_stop_fn_t fn,void * data)635bbf1bb3eSTejun Heo int stop_core_cpuslocked(unsigned int cpu, cpu_stop_fn_t fn, void *data)
636*2760f5a4SPeter Zijlstra {
637*2760f5a4SPeter Zijlstra const struct cpumask *smt_mask = cpu_smt_mask(cpu);
638*2760f5a4SPeter Zijlstra
639*2760f5a4SPeter Zijlstra struct multi_stop_data msdata = {
640*2760f5a4SPeter Zijlstra .fn = fn,
641*2760f5a4SPeter Zijlstra .data = data,
642*2760f5a4SPeter Zijlstra .num_threads = cpumask_weight(smt_mask),
643*2760f5a4SPeter Zijlstra .active_cpus = smt_mask,
644*2760f5a4SPeter Zijlstra };
645*2760f5a4SPeter Zijlstra
646*2760f5a4SPeter Zijlstra lockdep_assert_cpus_held();
647*2760f5a4SPeter Zijlstra
648*2760f5a4SPeter Zijlstra /* Set the initial state and stop all online cpus. */
649*2760f5a4SPeter Zijlstra set_state(&msdata, MULTI_STOP_PREPARE);
650*2760f5a4SPeter Zijlstra return stop_cpus(smt_mask, multi_cpu_stop, &msdata);
651*2760f5a4SPeter Zijlstra }
652*2760f5a4SPeter Zijlstra EXPORT_SYMBOL_GPL(stop_core_cpuslocked);
653*2760f5a4SPeter Zijlstra #endif
654*2760f5a4SPeter Zijlstra
655*2760f5a4SPeter Zijlstra /**
656*2760f5a4SPeter Zijlstra * stop_machine_from_inactive_cpu - stop_machine() from inactive CPU
657f740e6cdSTejun Heo * @fn: the function to run
658f740e6cdSTejun Heo * @data: the data ptr for the @fn()
659f740e6cdSTejun Heo * @cpus: the cpus to run the @fn() on (NULL = any online cpu)
660f740e6cdSTejun Heo *
661f740e6cdSTejun Heo * This is identical to stop_machine() but can be called from a CPU which
662f740e6cdSTejun Heo * is not active. The local CPU is in the process of hotplug (so no other
663f740e6cdSTejun Heo * CPU hotplug can start) and not marked active and doesn't have enough
664f740e6cdSTejun Heo * context to sleep.
665f740e6cdSTejun Heo *
666f740e6cdSTejun Heo * This function provides stop_machine() functionality for such state by
667f740e6cdSTejun Heo * using busy-wait for synchronization and executing @fn directly for local
668f740e6cdSTejun Heo * CPU.
669f740e6cdSTejun Heo *
670f740e6cdSTejun Heo * CONTEXT:
671f740e6cdSTejun Heo * Local CPU is inactive. Temporarily stops all active CPUs.
672f740e6cdSTejun Heo *
673f740e6cdSTejun Heo * RETURNS:
674f740e6cdSTejun Heo * 0 if all executions of @fn returned 0, any non zero return value if any
675f740e6cdSTejun Heo * returned non zero.
676f740e6cdSTejun Heo */
stop_machine_from_inactive_cpu(cpu_stop_fn_t fn,void * data,const struct cpumask * cpus)677f740e6cdSTejun Heo int stop_machine_from_inactive_cpu(cpu_stop_fn_t fn, void *data,
678f740e6cdSTejun Heo const struct cpumask *cpus)
6799a301f22SOleg Nesterov {
680f740e6cdSTejun Heo struct multi_stop_data msdata = { .fn = fn, .data = data,
681f740e6cdSTejun Heo .active_cpus = cpus };
6821be0bd77SPeter Zijlstra struct cpu_stop_done done;
683f740e6cdSTejun Heo int ret;
684f740e6cdSTejun Heo
685f740e6cdSTejun Heo /* Local CPU must be inactive and CPU hotplug in progress. */
686f740e6cdSTejun Heo BUG_ON(cpu_active(raw_smp_processor_id()));
687f740e6cdSTejun Heo msdata.num_threads = num_active_cpus() + 1; /* +1 for local */
688f740e6cdSTejun Heo
6891be0bd77SPeter Zijlstra /* No proper task established and can't sleep - busy wait for lock. */
690f740e6cdSTejun Heo while (!mutex_trylock(&stop_cpus_mutex))
691f740e6cdSTejun Heo cpu_relax();
692f740e6cdSTejun Heo
693f740e6cdSTejun Heo /* Schedule work on other CPUs and execute directly for local CPU */
694f740e6cdSTejun Heo set_state(&msdata, MULTI_STOP_PREPARE);
695f740e6cdSTejun Heo cpu_stop_init_done(&done, num_active_cpus());
6961be0bd77SPeter Zijlstra queue_stop_cpus_work(cpu_active_mask, multi_cpu_stop, &msdata,
697f740e6cdSTejun Heo &done);
6981be0bd77SPeter Zijlstra ret = multi_cpu_stop(&msdata);
699f740e6cdSTejun Heo
7001be0bd77SPeter Zijlstra /* Busy wait for completion. */
701f740e6cdSTejun Heo while (!completion_done(&done.completion))
702f740e6cdSTejun Heo cpu_relax();
703f740e6cdSTejun Heo
704f740e6cdSTejun Heo mutex_unlock(&stop_cpus_mutex);
705f740e6cdSTejun Heo return ret ?: done.ret;
706f740e6cdSTejun Heo }
707f740e6cdSTejun Heo