1 /* Copyright 2008, 2005 Rusty Russell rusty@rustcorp.com.au IBM Corporation. 2 * GPL v2 and any later version. 3 */ 4 #include <linux/cpu.h> 5 #include <linux/err.h> 6 #include <linux/kthread.h> 7 #include <linux/module.h> 8 #include <linux/sched.h> 9 #include <linux/stop_machine.h> 10 #include <linux/syscalls.h> 11 #include <linux/interrupt.h> 12 13 #include <asm/atomic.h> 14 #include <asm/uaccess.h> 15 16 /* This controls the threads on each CPU. */ 17 enum stopmachine_state { 18 /* Dummy starting state for thread. */ 19 STOPMACHINE_NONE, 20 /* Awaiting everyone to be scheduled. */ 21 STOPMACHINE_PREPARE, 22 /* Disable interrupts. */ 23 STOPMACHINE_DISABLE_IRQ, 24 /* Run the function */ 25 STOPMACHINE_RUN, 26 /* Exit */ 27 STOPMACHINE_EXIT, 28 }; 29 static enum stopmachine_state state; 30 31 struct stop_machine_data { 32 int (*fn)(void *); 33 void *data; 34 int fnret; 35 }; 36 37 /* Like num_online_cpus(), but hotplug cpu uses us, so we need this. */ 38 static unsigned int num_threads; 39 static atomic_t thread_ack; 40 static DEFINE_MUTEX(lock); 41 42 static struct workqueue_struct *stop_machine_wq; 43 static struct stop_machine_data active, idle; 44 static const cpumask_t *active_cpus; 45 static void *stop_machine_work; 46 47 static void set_state(enum stopmachine_state newstate) 48 { 49 /* Reset ack counter. */ 50 atomic_set(&thread_ack, num_threads); 51 smp_wmb(); 52 state = newstate; 53 } 54 55 /* Last one to ack a state moves to the next state. */ 56 static void ack_state(void) 57 { 58 if (atomic_dec_and_test(&thread_ack)) 59 set_state(state + 1); 60 } 61 62 /* This is the actual function which stops the CPU. It runs 63 * in the context of a dedicated stopmachine workqueue. */ 64 static void stop_cpu(struct work_struct *unused) 65 { 66 enum stopmachine_state curstate = STOPMACHINE_NONE; 67 struct stop_machine_data *smdata = &idle; 68 int cpu = smp_processor_id(); 69 int err; 70 71 if (!active_cpus) { 72 if (cpu == first_cpu(cpu_online_map)) 73 smdata = &active; 74 } else { 75 if (cpu_isset(cpu, *active_cpus)) 76 smdata = &active; 77 } 78 /* Simple state machine */ 79 do { 80 /* Chill out and ensure we re-read stopmachine_state. */ 81 cpu_relax(); 82 if (state != curstate) { 83 curstate = state; 84 switch (curstate) { 85 case STOPMACHINE_DISABLE_IRQ: 86 local_irq_disable(); 87 hard_irq_disable(); 88 break; 89 case STOPMACHINE_RUN: 90 /* On multiple CPUs only a single error code 91 * is needed to tell that something failed. */ 92 err = smdata->fn(smdata->data); 93 if (err) 94 smdata->fnret = err; 95 break; 96 default: 97 break; 98 } 99 ack_state(); 100 } 101 } while (curstate != STOPMACHINE_EXIT); 102 103 local_irq_enable(); 104 } 105 106 /* Callback for CPUs which aren't supposed to do anything. */ 107 static int chill(void *unused) 108 { 109 return 0; 110 } 111 112 int __stop_machine(int (*fn)(void *), void *data, const cpumask_t *cpus) 113 { 114 struct work_struct *sm_work; 115 int i; 116 117 /* Set up initial state. */ 118 mutex_lock(&lock); 119 num_threads = num_online_cpus(); 120 active_cpus = cpus; 121 active.fn = fn; 122 active.data = data; 123 active.fnret = 0; 124 idle.fn = chill; 125 idle.data = NULL; 126 127 set_state(STOPMACHINE_PREPARE); 128 129 /* Schedule the stop_cpu work on all cpus: hold this CPU so one 130 * doesn't hit this CPU until we're ready. */ 131 get_cpu(); 132 for_each_online_cpu(i) { 133 sm_work = percpu_ptr(stop_machine_work, i); 134 INIT_WORK(sm_work, stop_cpu); 135 queue_work_on(i, stop_machine_wq, sm_work); 136 } 137 /* This will release the thread on our CPU. */ 138 put_cpu(); 139 flush_workqueue(stop_machine_wq); 140 mutex_unlock(&lock); 141 return active.fnret; 142 } 143 144 int stop_machine(int (*fn)(void *), void *data, const cpumask_t *cpus) 145 { 146 int ret; 147 148 /* No CPUs can come up or down during this. */ 149 get_online_cpus(); 150 ret = __stop_machine(fn, data, cpus); 151 put_online_cpus(); 152 153 return ret; 154 } 155 EXPORT_SYMBOL_GPL(stop_machine); 156 157 static int __init stop_machine_init(void) 158 { 159 stop_machine_wq = create_rt_workqueue("kstop"); 160 stop_machine_work = alloc_percpu(struct work_struct); 161 return 0; 162 } 163 core_initcall(stop_machine_init); 164