1 /* 2 * Common SMP CPU bringup/teardown functions 3 */ 4 #include <linux/cpu.h> 5 #include <linux/err.h> 6 #include <linux/smp.h> 7 #include <linux/init.h> 8 #include <linux/list.h> 9 #include <linux/slab.h> 10 #include <linux/sched.h> 11 #include <linux/export.h> 12 #include <linux/percpu.h> 13 #include <linux/kthread.h> 14 #include <linux/smpboot.h> 15 16 #include "smpboot.h" 17 18 #ifdef CONFIG_SMP 19 20 #ifdef CONFIG_GENERIC_SMP_IDLE_THREAD 21 /* 22 * For the hotplug case we keep the task structs around and reuse 23 * them. 24 */ 25 static DEFINE_PER_CPU(struct task_struct *, idle_threads); 26 27 struct task_struct *idle_thread_get(unsigned int cpu) 28 { 29 struct task_struct *tsk = per_cpu(idle_threads, cpu); 30 31 if (!tsk) 32 return ERR_PTR(-ENOMEM); 33 init_idle(tsk, cpu); 34 return tsk; 35 } 36 37 void __init idle_thread_set_boot_cpu(void) 38 { 39 per_cpu(idle_threads, smp_processor_id()) = current; 40 } 41 42 /** 43 * idle_init - Initialize the idle thread for a cpu 44 * @cpu: The cpu for which the idle thread should be initialized 45 * 46 * Creates the thread if it does not exist. 47 */ 48 static inline void idle_init(unsigned int cpu) 49 { 50 struct task_struct *tsk = per_cpu(idle_threads, cpu); 51 52 if (!tsk) { 53 tsk = fork_idle(cpu); 54 if (IS_ERR(tsk)) 55 pr_err("SMP: fork_idle() failed for CPU %u\n", cpu); 56 else 57 per_cpu(idle_threads, cpu) = tsk; 58 } 59 } 60 61 /** 62 * idle_threads_init - Initialize idle threads for all cpus 63 */ 64 void __init idle_threads_init(void) 65 { 66 unsigned int cpu, boot_cpu; 67 68 boot_cpu = smp_processor_id(); 69 70 for_each_possible_cpu(cpu) { 71 if (cpu != boot_cpu) 72 idle_init(cpu); 73 } 74 } 75 #endif 76 77 #endif /* #ifdef CONFIG_SMP */ 78 79 static LIST_HEAD(hotplug_threads); 80 static DEFINE_MUTEX(smpboot_threads_lock); 81 82 struct smpboot_thread_data { 83 unsigned int cpu; 84 unsigned int status; 85 struct smp_hotplug_thread *ht; 86 }; 87 88 enum { 89 HP_THREAD_NONE = 0, 90 HP_THREAD_ACTIVE, 91 HP_THREAD_PARKED, 92 }; 93 94 /** 95 * smpboot_thread_fn - percpu hotplug thread loop function 96 * @data: thread data pointer 97 * 98 * Checks for thread stop and park conditions. Calls the necessary 99 * setup, cleanup, park and unpark functions for the registered 100 * thread. 101 * 102 * Returns 1 when the thread should exit, 0 otherwise. 103 */ 104 static int smpboot_thread_fn(void *data) 105 { 106 struct smpboot_thread_data *td = data; 107 struct smp_hotplug_thread *ht = td->ht; 108 109 while (1) { 110 set_current_state(TASK_INTERRUPTIBLE); 111 preempt_disable(); 112 if (kthread_should_stop()) { 113 __set_current_state(TASK_RUNNING); 114 preempt_enable(); 115 if (ht->cleanup) 116 ht->cleanup(td->cpu, cpu_online(td->cpu)); 117 kfree(td); 118 return 0; 119 } 120 121 if (kthread_should_park()) { 122 __set_current_state(TASK_RUNNING); 123 preempt_enable(); 124 if (ht->park && td->status == HP_THREAD_ACTIVE) { 125 BUG_ON(td->cpu != smp_processor_id()); 126 ht->park(td->cpu); 127 td->status = HP_THREAD_PARKED; 128 } 129 kthread_parkme(); 130 /* We might have been woken for stop */ 131 continue; 132 } 133 134 BUG_ON(td->cpu != smp_processor_id()); 135 136 /* Check for state change setup */ 137 switch (td->status) { 138 case HP_THREAD_NONE: 139 __set_current_state(TASK_RUNNING); 140 preempt_enable(); 141 if (ht->setup) 142 ht->setup(td->cpu); 143 td->status = HP_THREAD_ACTIVE; 144 continue; 145 146 case HP_THREAD_PARKED: 147 __set_current_state(TASK_RUNNING); 148 preempt_enable(); 149 if (ht->unpark) 150 ht->unpark(td->cpu); 151 td->status = HP_THREAD_ACTIVE; 152 continue; 153 } 154 155 if (!ht->thread_should_run(td->cpu)) { 156 preempt_enable_no_resched(); 157 schedule(); 158 } else { 159 __set_current_state(TASK_RUNNING); 160 preempt_enable(); 161 ht->thread_fn(td->cpu); 162 } 163 } 164 } 165 166 static int 167 __smpboot_create_thread(struct smp_hotplug_thread *ht, unsigned int cpu) 168 { 169 struct task_struct *tsk = *per_cpu_ptr(ht->store, cpu); 170 struct smpboot_thread_data *td; 171 172 if (tsk) 173 return 0; 174 175 td = kzalloc_node(sizeof(*td), GFP_KERNEL, cpu_to_node(cpu)); 176 if (!td) 177 return -ENOMEM; 178 td->cpu = cpu; 179 td->ht = ht; 180 181 tsk = kthread_create_on_cpu(smpboot_thread_fn, td, cpu, 182 ht->thread_comm); 183 if (IS_ERR(tsk)) { 184 kfree(td); 185 return PTR_ERR(tsk); 186 } 187 get_task_struct(tsk); 188 *per_cpu_ptr(ht->store, cpu) = tsk; 189 if (ht->create) { 190 /* 191 * Make sure that the task has actually scheduled out 192 * into park position, before calling the create 193 * callback. At least the migration thread callback 194 * requires that the task is off the runqueue. 195 */ 196 if (!wait_task_inactive(tsk, TASK_PARKED)) 197 WARN_ON(1); 198 else 199 ht->create(cpu); 200 } 201 return 0; 202 } 203 204 int smpboot_create_threads(unsigned int cpu) 205 { 206 struct smp_hotplug_thread *cur; 207 int ret = 0; 208 209 mutex_lock(&smpboot_threads_lock); 210 list_for_each_entry(cur, &hotplug_threads, list) { 211 ret = __smpboot_create_thread(cur, cpu); 212 if (ret) 213 break; 214 } 215 mutex_unlock(&smpboot_threads_lock); 216 return ret; 217 } 218 219 static void smpboot_unpark_thread(struct smp_hotplug_thread *ht, unsigned int cpu) 220 { 221 struct task_struct *tsk = *per_cpu_ptr(ht->store, cpu); 222 223 if (ht->pre_unpark) 224 ht->pre_unpark(cpu); 225 kthread_unpark(tsk); 226 } 227 228 void smpboot_unpark_threads(unsigned int cpu) 229 { 230 struct smp_hotplug_thread *cur; 231 232 mutex_lock(&smpboot_threads_lock); 233 list_for_each_entry(cur, &hotplug_threads, list) 234 smpboot_unpark_thread(cur, cpu); 235 mutex_unlock(&smpboot_threads_lock); 236 } 237 238 static void smpboot_park_thread(struct smp_hotplug_thread *ht, unsigned int cpu) 239 { 240 struct task_struct *tsk = *per_cpu_ptr(ht->store, cpu); 241 242 if (tsk && !ht->selfparking) 243 kthread_park(tsk); 244 } 245 246 void smpboot_park_threads(unsigned int cpu) 247 { 248 struct smp_hotplug_thread *cur; 249 250 mutex_lock(&smpboot_threads_lock); 251 list_for_each_entry_reverse(cur, &hotplug_threads, list) 252 smpboot_park_thread(cur, cpu); 253 mutex_unlock(&smpboot_threads_lock); 254 } 255 256 static void smpboot_destroy_threads(struct smp_hotplug_thread *ht) 257 { 258 unsigned int cpu; 259 260 /* We need to destroy also the parked threads of offline cpus */ 261 for_each_possible_cpu(cpu) { 262 struct task_struct *tsk = *per_cpu_ptr(ht->store, cpu); 263 264 if (tsk) { 265 kthread_stop(tsk); 266 put_task_struct(tsk); 267 *per_cpu_ptr(ht->store, cpu) = NULL; 268 } 269 } 270 } 271 272 /** 273 * smpboot_register_percpu_thread - Register a per_cpu thread related to hotplug 274 * @plug_thread: Hotplug thread descriptor 275 * 276 * Creates and starts the threads on all online cpus. 277 */ 278 int smpboot_register_percpu_thread(struct smp_hotplug_thread *plug_thread) 279 { 280 unsigned int cpu; 281 int ret = 0; 282 283 mutex_lock(&smpboot_threads_lock); 284 for_each_online_cpu(cpu) { 285 ret = __smpboot_create_thread(plug_thread, cpu); 286 if (ret) { 287 smpboot_destroy_threads(plug_thread); 288 goto out; 289 } 290 smpboot_unpark_thread(plug_thread, cpu); 291 } 292 list_add(&plug_thread->list, &hotplug_threads); 293 out: 294 mutex_unlock(&smpboot_threads_lock); 295 return ret; 296 } 297 EXPORT_SYMBOL_GPL(smpboot_register_percpu_thread); 298 299 /** 300 * smpboot_unregister_percpu_thread - Unregister a per_cpu thread related to hotplug 301 * @plug_thread: Hotplug thread descriptor 302 * 303 * Stops all threads on all possible cpus. 304 */ 305 void smpboot_unregister_percpu_thread(struct smp_hotplug_thread *plug_thread) 306 { 307 get_online_cpus(); 308 mutex_lock(&smpboot_threads_lock); 309 list_del(&plug_thread->list); 310 smpboot_destroy_threads(plug_thread); 311 mutex_unlock(&smpboot_threads_lock); 312 put_online_cpus(); 313 } 314 EXPORT_SYMBOL_GPL(smpboot_unregister_percpu_thread); 315