1 /* 2 * Common SMP CPU bringup/teardown functions 3 */ 4 #include <linux/cpu.h> 5 #include <linux/err.h> 6 #include <linux/smp.h> 7 #include <linux/init.h> 8 #include <linux/list.h> 9 #include <linux/slab.h> 10 #include <linux/sched.h> 11 #include <linux/export.h> 12 #include <linux/percpu.h> 13 #include <linux/kthread.h> 14 #include <linux/smpboot.h> 15 16 #include "smpboot.h" 17 18 #ifdef CONFIG_SMP 19 20 #ifdef CONFIG_GENERIC_SMP_IDLE_THREAD 21 /* 22 * For the hotplug case we keep the task structs around and reuse 23 * them. 24 */ 25 static DEFINE_PER_CPU(struct task_struct *, idle_threads); 26 27 struct task_struct * __cpuinit idle_thread_get(unsigned int cpu) 28 { 29 struct task_struct *tsk = per_cpu(idle_threads, cpu); 30 31 if (!tsk) 32 return ERR_PTR(-ENOMEM); 33 init_idle(tsk, cpu); 34 return tsk; 35 } 36 37 void __init idle_thread_set_boot_cpu(void) 38 { 39 per_cpu(idle_threads, smp_processor_id()) = current; 40 } 41 42 /** 43 * idle_init - Initialize the idle thread for a cpu 44 * @cpu: The cpu for which the idle thread should be initialized 45 * 46 * Creates the thread if it does not exist. 47 */ 48 static inline void idle_init(unsigned int cpu) 49 { 50 struct task_struct *tsk = per_cpu(idle_threads, cpu); 51 52 if (!tsk) { 53 tsk = fork_idle(cpu); 54 if (IS_ERR(tsk)) 55 pr_err("SMP: fork_idle() failed for CPU %u\n", cpu); 56 else 57 per_cpu(idle_threads, cpu) = tsk; 58 } 59 } 60 61 /** 62 * idle_threads_init - Initialize idle threads for all cpus 63 */ 64 void __init idle_threads_init(void) 65 { 66 unsigned int cpu, boot_cpu; 67 68 boot_cpu = smp_processor_id(); 69 70 for_each_possible_cpu(cpu) { 71 if (cpu != boot_cpu) 72 idle_init(cpu); 73 } 74 } 75 #endif 76 77 #endif /* #ifdef CONFIG_SMP */ 78 79 static LIST_HEAD(hotplug_threads); 80 static DEFINE_MUTEX(smpboot_threads_lock); 81 82 struct smpboot_thread_data { 83 unsigned int cpu; 84 unsigned int status; 85 struct smp_hotplug_thread *ht; 86 }; 87 88 enum { 89 HP_THREAD_NONE = 0, 90 HP_THREAD_ACTIVE, 91 HP_THREAD_PARKED, 92 }; 93 94 /** 95 * smpboot_thread_fn - percpu hotplug thread loop function 96 * @data: thread data pointer 97 * 98 * Checks for thread stop and park conditions. Calls the necessary 99 * setup, cleanup, park and unpark functions for the registered 100 * thread. 101 * 102 * Returns 1 when the thread should exit, 0 otherwise. 103 */ 104 static int smpboot_thread_fn(void *data) 105 { 106 struct smpboot_thread_data *td = data; 107 struct smp_hotplug_thread *ht = td->ht; 108 109 while (1) { 110 set_current_state(TASK_INTERRUPTIBLE); 111 preempt_disable(); 112 if (kthread_should_stop()) { 113 set_current_state(TASK_RUNNING); 114 preempt_enable(); 115 if (ht->cleanup) 116 ht->cleanup(td->cpu, cpu_online(td->cpu)); 117 kfree(td); 118 return 0; 119 } 120 121 if (kthread_should_park()) { 122 __set_current_state(TASK_RUNNING); 123 preempt_enable(); 124 if (ht->park && td->status == HP_THREAD_ACTIVE) { 125 BUG_ON(td->cpu != smp_processor_id()); 126 ht->park(td->cpu); 127 td->status = HP_THREAD_PARKED; 128 } 129 kthread_parkme(); 130 /* We might have been woken for stop */ 131 continue; 132 } 133 134 BUG_ON(td->cpu != smp_processor_id()); 135 136 /* Check for state change setup */ 137 switch (td->status) { 138 case HP_THREAD_NONE: 139 preempt_enable(); 140 if (ht->setup) 141 ht->setup(td->cpu); 142 td->status = HP_THREAD_ACTIVE; 143 preempt_disable(); 144 break; 145 case HP_THREAD_PARKED: 146 preempt_enable(); 147 if (ht->unpark) 148 ht->unpark(td->cpu); 149 td->status = HP_THREAD_ACTIVE; 150 preempt_disable(); 151 break; 152 } 153 154 if (!ht->thread_should_run(td->cpu)) { 155 preempt_enable(); 156 schedule(); 157 } else { 158 set_current_state(TASK_RUNNING); 159 preempt_enable(); 160 ht->thread_fn(td->cpu); 161 } 162 } 163 } 164 165 static int 166 __smpboot_create_thread(struct smp_hotplug_thread *ht, unsigned int cpu) 167 { 168 struct task_struct *tsk = *per_cpu_ptr(ht->store, cpu); 169 struct smpboot_thread_data *td; 170 171 if (tsk) 172 return 0; 173 174 td = kzalloc_node(sizeof(*td), GFP_KERNEL, cpu_to_node(cpu)); 175 if (!td) 176 return -ENOMEM; 177 td->cpu = cpu; 178 td->ht = ht; 179 180 tsk = kthread_create_on_cpu(smpboot_thread_fn, td, cpu, 181 ht->thread_comm); 182 if (IS_ERR(tsk)) { 183 kfree(td); 184 return PTR_ERR(tsk); 185 } 186 get_task_struct(tsk); 187 *per_cpu_ptr(ht->store, cpu) = tsk; 188 if (ht->create) 189 ht->create(cpu); 190 return 0; 191 } 192 193 int smpboot_create_threads(unsigned int cpu) 194 { 195 struct smp_hotplug_thread *cur; 196 int ret = 0; 197 198 mutex_lock(&smpboot_threads_lock); 199 list_for_each_entry(cur, &hotplug_threads, list) { 200 ret = __smpboot_create_thread(cur, cpu); 201 if (ret) 202 break; 203 } 204 mutex_unlock(&smpboot_threads_lock); 205 return ret; 206 } 207 208 static void smpboot_unpark_thread(struct smp_hotplug_thread *ht, unsigned int cpu) 209 { 210 struct task_struct *tsk = *per_cpu_ptr(ht->store, cpu); 211 212 kthread_unpark(tsk); 213 } 214 215 void smpboot_unpark_threads(unsigned int cpu) 216 { 217 struct smp_hotplug_thread *cur; 218 219 mutex_lock(&smpboot_threads_lock); 220 list_for_each_entry(cur, &hotplug_threads, list) 221 smpboot_unpark_thread(cur, cpu); 222 mutex_unlock(&smpboot_threads_lock); 223 } 224 225 static void smpboot_park_thread(struct smp_hotplug_thread *ht, unsigned int cpu) 226 { 227 struct task_struct *tsk = *per_cpu_ptr(ht->store, cpu); 228 229 if (tsk && !ht->selfparking) 230 kthread_park(tsk); 231 } 232 233 void smpboot_park_threads(unsigned int cpu) 234 { 235 struct smp_hotplug_thread *cur; 236 237 mutex_lock(&smpboot_threads_lock); 238 list_for_each_entry_reverse(cur, &hotplug_threads, list) 239 smpboot_park_thread(cur, cpu); 240 mutex_unlock(&smpboot_threads_lock); 241 } 242 243 static void smpboot_destroy_threads(struct smp_hotplug_thread *ht) 244 { 245 unsigned int cpu; 246 247 /* We need to destroy also the parked threads of offline cpus */ 248 for_each_possible_cpu(cpu) { 249 struct task_struct *tsk = *per_cpu_ptr(ht->store, cpu); 250 251 if (tsk) { 252 kthread_stop(tsk); 253 put_task_struct(tsk); 254 *per_cpu_ptr(ht->store, cpu) = NULL; 255 } 256 } 257 } 258 259 /** 260 * smpboot_register_percpu_thread - Register a per_cpu thread related to hotplug 261 * @plug_thread: Hotplug thread descriptor 262 * 263 * Creates and starts the threads on all online cpus. 264 */ 265 int smpboot_register_percpu_thread(struct smp_hotplug_thread *plug_thread) 266 { 267 unsigned int cpu; 268 int ret = 0; 269 270 mutex_lock(&smpboot_threads_lock); 271 for_each_online_cpu(cpu) { 272 ret = __smpboot_create_thread(plug_thread, cpu); 273 if (ret) { 274 smpboot_destroy_threads(plug_thread); 275 goto out; 276 } 277 smpboot_unpark_thread(plug_thread, cpu); 278 } 279 list_add(&plug_thread->list, &hotplug_threads); 280 out: 281 mutex_unlock(&smpboot_threads_lock); 282 return ret; 283 } 284 EXPORT_SYMBOL_GPL(smpboot_register_percpu_thread); 285 286 /** 287 * smpboot_unregister_percpu_thread - Unregister a per_cpu thread related to hotplug 288 * @plug_thread: Hotplug thread descriptor 289 * 290 * Stops all threads on all possible cpus. 291 */ 292 void smpboot_unregister_percpu_thread(struct smp_hotplug_thread *plug_thread) 293 { 294 get_online_cpus(); 295 mutex_lock(&smpboot_threads_lock); 296 list_del(&plug_thread->list); 297 smpboot_destroy_threads(plug_thread); 298 mutex_unlock(&smpboot_threads_lock); 299 put_online_cpus(); 300 } 301 EXPORT_SYMBOL_GPL(smpboot_unregister_percpu_thread); 302