1 /* CPU control. 2 * (C) 2001, 2002, 2003, 2004 Rusty Russell 3 * 4 * This code is licenced under the GPL. 5 */ 6 #include <linux/proc_fs.h> 7 #include <linux/smp.h> 8 #include <linux/init.h> 9 #include <linux/notifier.h> 10 #include <linux/sched.h> 11 #include <linux/unistd.h> 12 #include <linux/cpu.h> 13 #include <linux/oom.h> 14 #include <linux/rcupdate.h> 15 #include <linux/export.h> 16 #include <linux/bug.h> 17 #include <linux/kthread.h> 18 #include <linux/stop_machine.h> 19 #include <linux/mutex.h> 20 #include <linux/gfp.h> 21 #include <linux/suspend.h> 22 #include <linux/lockdep.h> 23 #include <linux/tick.h> 24 #include <linux/irq.h> 25 #include <linux/smpboot.h> 26 27 #include <trace/events/power.h> 28 #define CREATE_TRACE_POINTS 29 #include <trace/events/cpuhp.h> 30 31 #include "smpboot.h" 32 33 /** 34 * cpuhp_cpu_state - Per cpu hotplug state storage 35 * @state: The current cpu state 36 * @target: The target state 37 * @thread: Pointer to the hotplug thread 38 * @should_run: Thread should execute 39 * @rollback: Perform a rollback 40 * @cb_stat: The state for a single callback (install/uninstall) 41 * @cb: Single callback function (install/uninstall) 42 * @result: Result of the operation 43 * @done: Signal completion to the issuer of the task 44 */ 45 struct cpuhp_cpu_state { 46 enum cpuhp_state state; 47 enum cpuhp_state target; 48 #ifdef CONFIG_SMP 49 struct task_struct *thread; 50 bool should_run; 51 bool rollback; 52 enum cpuhp_state cb_state; 53 int (*cb)(unsigned int cpu); 54 int result; 55 struct completion done; 56 #endif 57 }; 58 59 static DEFINE_PER_CPU(struct cpuhp_cpu_state, cpuhp_state); 60 61 /** 62 * cpuhp_step - Hotplug state machine step 63 * @name: Name of the step 64 * @startup: Startup function of the step 65 * @teardown: Teardown function of the step 66 * @skip_onerr: Do not invoke the functions on error rollback 67 * Will go away once the notifiers are gone 68 * @cant_stop: Bringup/teardown can't be stopped at this step 69 */ 70 struct cpuhp_step { 71 const char *name; 72 int (*startup)(unsigned int cpu); 73 int (*teardown)(unsigned int cpu); 74 bool skip_onerr; 75 bool cant_stop; 76 }; 77 78 static DEFINE_MUTEX(cpuhp_state_mutex); 79 static struct cpuhp_step cpuhp_bp_states[]; 80 static struct cpuhp_step cpuhp_ap_states[]; 81 82 /** 83 * cpuhp_invoke_callback _ Invoke the callbacks for a given state 84 * @cpu: The cpu for which the callback should be invoked 85 * @step: The step in the state machine 86 * @cb: The callback function to invoke 87 * 88 * Called from cpu hotplug and from the state register machinery 89 */ 90 static int cpuhp_invoke_callback(unsigned int cpu, enum cpuhp_state step, 91 int (*cb)(unsigned int)) 92 { 93 struct cpuhp_cpu_state *st = per_cpu_ptr(&cpuhp_state, cpu); 94 int ret = 0; 95 96 if (cb) { 97 trace_cpuhp_enter(cpu, st->target, step, cb); 98 ret = cb(cpu); 99 trace_cpuhp_exit(cpu, st->state, step, ret); 100 } 101 return ret; 102 } 103 104 #ifdef CONFIG_SMP 105 /* Serializes the updates to cpu_online_mask, cpu_present_mask */ 106 static DEFINE_MUTEX(cpu_add_remove_lock); 107 bool cpuhp_tasks_frozen; 108 EXPORT_SYMBOL_GPL(cpuhp_tasks_frozen); 109 110 /* 111 * The following two APIs (cpu_maps_update_begin/done) must be used when 112 * attempting to serialize the updates to cpu_online_mask & cpu_present_mask. 113 * The APIs cpu_notifier_register_begin/done() must be used to protect CPU 114 * hotplug callback (un)registration performed using __register_cpu_notifier() 115 * or __unregister_cpu_notifier(). 116 */ 117 void cpu_maps_update_begin(void) 118 { 119 mutex_lock(&cpu_add_remove_lock); 120 } 121 EXPORT_SYMBOL(cpu_notifier_register_begin); 122 123 void cpu_maps_update_done(void) 124 { 125 mutex_unlock(&cpu_add_remove_lock); 126 } 127 EXPORT_SYMBOL(cpu_notifier_register_done); 128 129 static RAW_NOTIFIER_HEAD(cpu_chain); 130 131 /* If set, cpu_up and cpu_down will return -EBUSY and do nothing. 132 * Should always be manipulated under cpu_add_remove_lock 133 */ 134 static int cpu_hotplug_disabled; 135 136 #ifdef CONFIG_HOTPLUG_CPU 137 138 static struct { 139 struct task_struct *active_writer; 140 /* wait queue to wake up the active_writer */ 141 wait_queue_head_t wq; 142 /* verifies that no writer will get active while readers are active */ 143 struct mutex lock; 144 /* 145 * Also blocks the new readers during 146 * an ongoing cpu hotplug operation. 147 */ 148 atomic_t refcount; 149 150 #ifdef CONFIG_DEBUG_LOCK_ALLOC 151 struct lockdep_map dep_map; 152 #endif 153 } cpu_hotplug = { 154 .active_writer = NULL, 155 .wq = __WAIT_QUEUE_HEAD_INITIALIZER(cpu_hotplug.wq), 156 .lock = __MUTEX_INITIALIZER(cpu_hotplug.lock), 157 #ifdef CONFIG_DEBUG_LOCK_ALLOC 158 .dep_map = {.name = "cpu_hotplug.lock" }, 159 #endif 160 }; 161 162 /* Lockdep annotations for get/put_online_cpus() and cpu_hotplug_begin/end() */ 163 #define cpuhp_lock_acquire_read() lock_map_acquire_read(&cpu_hotplug.dep_map) 164 #define cpuhp_lock_acquire_tryread() \ 165 lock_map_acquire_tryread(&cpu_hotplug.dep_map) 166 #define cpuhp_lock_acquire() lock_map_acquire(&cpu_hotplug.dep_map) 167 #define cpuhp_lock_release() lock_map_release(&cpu_hotplug.dep_map) 168 169 170 void get_online_cpus(void) 171 { 172 might_sleep(); 173 if (cpu_hotplug.active_writer == current) 174 return; 175 cpuhp_lock_acquire_read(); 176 mutex_lock(&cpu_hotplug.lock); 177 atomic_inc(&cpu_hotplug.refcount); 178 mutex_unlock(&cpu_hotplug.lock); 179 } 180 EXPORT_SYMBOL_GPL(get_online_cpus); 181 182 void put_online_cpus(void) 183 { 184 int refcount; 185 186 if (cpu_hotplug.active_writer == current) 187 return; 188 189 refcount = atomic_dec_return(&cpu_hotplug.refcount); 190 if (WARN_ON(refcount < 0)) /* try to fix things up */ 191 atomic_inc(&cpu_hotplug.refcount); 192 193 if (refcount <= 0 && waitqueue_active(&cpu_hotplug.wq)) 194 wake_up(&cpu_hotplug.wq); 195 196 cpuhp_lock_release(); 197 198 } 199 EXPORT_SYMBOL_GPL(put_online_cpus); 200 201 /* 202 * This ensures that the hotplug operation can begin only when the 203 * refcount goes to zero. 204 * 205 * Note that during a cpu-hotplug operation, the new readers, if any, 206 * will be blocked by the cpu_hotplug.lock 207 * 208 * Since cpu_hotplug_begin() is always called after invoking 209 * cpu_maps_update_begin(), we can be sure that only one writer is active. 210 * 211 * Note that theoretically, there is a possibility of a livelock: 212 * - Refcount goes to zero, last reader wakes up the sleeping 213 * writer. 214 * - Last reader unlocks the cpu_hotplug.lock. 215 * - A new reader arrives at this moment, bumps up the refcount. 216 * - The writer acquires the cpu_hotplug.lock finds the refcount 217 * non zero and goes to sleep again. 218 * 219 * However, this is very difficult to achieve in practice since 220 * get_online_cpus() not an api which is called all that often. 221 * 222 */ 223 void cpu_hotplug_begin(void) 224 { 225 DEFINE_WAIT(wait); 226 227 cpu_hotplug.active_writer = current; 228 cpuhp_lock_acquire(); 229 230 for (;;) { 231 mutex_lock(&cpu_hotplug.lock); 232 prepare_to_wait(&cpu_hotplug.wq, &wait, TASK_UNINTERRUPTIBLE); 233 if (likely(!atomic_read(&cpu_hotplug.refcount))) 234 break; 235 mutex_unlock(&cpu_hotplug.lock); 236 schedule(); 237 } 238 finish_wait(&cpu_hotplug.wq, &wait); 239 } 240 241 void cpu_hotplug_done(void) 242 { 243 cpu_hotplug.active_writer = NULL; 244 mutex_unlock(&cpu_hotplug.lock); 245 cpuhp_lock_release(); 246 } 247 248 /* 249 * Wait for currently running CPU hotplug operations to complete (if any) and 250 * disable future CPU hotplug (from sysfs). The 'cpu_add_remove_lock' protects 251 * the 'cpu_hotplug_disabled' flag. The same lock is also acquired by the 252 * hotplug path before performing hotplug operations. So acquiring that lock 253 * guarantees mutual exclusion from any currently running hotplug operations. 254 */ 255 void cpu_hotplug_disable(void) 256 { 257 cpu_maps_update_begin(); 258 cpu_hotplug_disabled++; 259 cpu_maps_update_done(); 260 } 261 EXPORT_SYMBOL_GPL(cpu_hotplug_disable); 262 263 void cpu_hotplug_enable(void) 264 { 265 cpu_maps_update_begin(); 266 WARN_ON(--cpu_hotplug_disabled < 0); 267 cpu_maps_update_done(); 268 } 269 EXPORT_SYMBOL_GPL(cpu_hotplug_enable); 270 #endif /* CONFIG_HOTPLUG_CPU */ 271 272 /* Need to know about CPUs going up/down? */ 273 int register_cpu_notifier(struct notifier_block *nb) 274 { 275 int ret; 276 cpu_maps_update_begin(); 277 ret = raw_notifier_chain_register(&cpu_chain, nb); 278 cpu_maps_update_done(); 279 return ret; 280 } 281 282 int __register_cpu_notifier(struct notifier_block *nb) 283 { 284 return raw_notifier_chain_register(&cpu_chain, nb); 285 } 286 287 static int __cpu_notify(unsigned long val, unsigned int cpu, int nr_to_call, 288 int *nr_calls) 289 { 290 unsigned long mod = cpuhp_tasks_frozen ? CPU_TASKS_FROZEN : 0; 291 void *hcpu = (void *)(long)cpu; 292 293 int ret; 294 295 ret = __raw_notifier_call_chain(&cpu_chain, val | mod, hcpu, nr_to_call, 296 nr_calls); 297 298 return notifier_to_errno(ret); 299 } 300 301 static int cpu_notify(unsigned long val, unsigned int cpu) 302 { 303 return __cpu_notify(val, cpu, -1, NULL); 304 } 305 306 static void cpu_notify_nofail(unsigned long val, unsigned int cpu) 307 { 308 BUG_ON(cpu_notify(val, cpu)); 309 } 310 311 /* Notifier wrappers for transitioning to state machine */ 312 static int notify_prepare(unsigned int cpu) 313 { 314 int nr_calls = 0; 315 int ret; 316 317 ret = __cpu_notify(CPU_UP_PREPARE, cpu, -1, &nr_calls); 318 if (ret) { 319 nr_calls--; 320 printk(KERN_WARNING "%s: attempt to bring up CPU %u failed\n", 321 __func__, cpu); 322 __cpu_notify(CPU_UP_CANCELED, cpu, nr_calls, NULL); 323 } 324 return ret; 325 } 326 327 static int notify_online(unsigned int cpu) 328 { 329 cpu_notify(CPU_ONLINE, cpu); 330 return 0; 331 } 332 333 static int notify_starting(unsigned int cpu) 334 { 335 cpu_notify(CPU_STARTING, cpu); 336 return 0; 337 } 338 339 static int bringup_wait_for_ap(unsigned int cpu) 340 { 341 struct cpuhp_cpu_state *st = per_cpu_ptr(&cpuhp_state, cpu); 342 343 wait_for_completion(&st->done); 344 return st->result; 345 } 346 347 static int bringup_cpu(unsigned int cpu) 348 { 349 struct task_struct *idle = idle_thread_get(cpu); 350 int ret; 351 352 /* Arch-specific enabling code. */ 353 ret = __cpu_up(cpu, idle); 354 if (ret) { 355 cpu_notify(CPU_UP_CANCELED, cpu); 356 return ret; 357 } 358 ret = bringup_wait_for_ap(cpu); 359 BUG_ON(!cpu_online(cpu)); 360 return ret; 361 } 362 363 /* 364 * Hotplug state machine related functions 365 */ 366 static void undo_cpu_down(unsigned int cpu, struct cpuhp_cpu_state *st, 367 struct cpuhp_step *steps) 368 { 369 for (st->state++; st->state < st->target; st->state++) { 370 struct cpuhp_step *step = steps + st->state; 371 372 if (!step->skip_onerr) 373 cpuhp_invoke_callback(cpu, st->state, step->startup); 374 } 375 } 376 377 static int cpuhp_down_callbacks(unsigned int cpu, struct cpuhp_cpu_state *st, 378 struct cpuhp_step *steps, enum cpuhp_state target) 379 { 380 enum cpuhp_state prev_state = st->state; 381 int ret = 0; 382 383 for (; st->state > target; st->state--) { 384 struct cpuhp_step *step = steps + st->state; 385 386 ret = cpuhp_invoke_callback(cpu, st->state, step->teardown); 387 if (ret) { 388 st->target = prev_state; 389 undo_cpu_down(cpu, st, steps); 390 break; 391 } 392 } 393 return ret; 394 } 395 396 static void undo_cpu_up(unsigned int cpu, struct cpuhp_cpu_state *st, 397 struct cpuhp_step *steps) 398 { 399 for (st->state--; st->state > st->target; st->state--) { 400 struct cpuhp_step *step = steps + st->state; 401 402 if (!step->skip_onerr) 403 cpuhp_invoke_callback(cpu, st->state, step->teardown); 404 } 405 } 406 407 static int cpuhp_up_callbacks(unsigned int cpu, struct cpuhp_cpu_state *st, 408 struct cpuhp_step *steps, enum cpuhp_state target) 409 { 410 enum cpuhp_state prev_state = st->state; 411 int ret = 0; 412 413 while (st->state < target) { 414 struct cpuhp_step *step; 415 416 st->state++; 417 step = steps + st->state; 418 ret = cpuhp_invoke_callback(cpu, st->state, step->startup); 419 if (ret) { 420 st->target = prev_state; 421 undo_cpu_up(cpu, st, steps); 422 break; 423 } 424 } 425 return ret; 426 } 427 428 /* 429 * The cpu hotplug threads manage the bringup and teardown of the cpus 430 */ 431 static void cpuhp_create(unsigned int cpu) 432 { 433 struct cpuhp_cpu_state *st = per_cpu_ptr(&cpuhp_state, cpu); 434 435 init_completion(&st->done); 436 } 437 438 static int cpuhp_should_run(unsigned int cpu) 439 { 440 struct cpuhp_cpu_state *st = this_cpu_ptr(&cpuhp_state); 441 442 return st->should_run; 443 } 444 445 /* Execute the teardown callbacks. Used to be CPU_DOWN_PREPARE */ 446 static int cpuhp_ap_offline(unsigned int cpu, struct cpuhp_cpu_state *st) 447 { 448 enum cpuhp_state target = max((int)st->target, CPUHP_TEARDOWN_CPU); 449 450 return cpuhp_down_callbacks(cpu, st, cpuhp_ap_states, target); 451 } 452 453 /* Execute the online startup callbacks. Used to be CPU_ONLINE */ 454 static int cpuhp_ap_online(unsigned int cpu, struct cpuhp_cpu_state *st) 455 { 456 return cpuhp_up_callbacks(cpu, st, cpuhp_ap_states, st->target); 457 } 458 459 /* 460 * Execute teardown/startup callbacks on the plugged cpu. Also used to invoke 461 * callbacks when a state gets [un]installed at runtime. 462 */ 463 static void cpuhp_thread_fun(unsigned int cpu) 464 { 465 struct cpuhp_cpu_state *st = this_cpu_ptr(&cpuhp_state); 466 int ret = 0; 467 468 /* 469 * Paired with the mb() in cpuhp_kick_ap_work and 470 * cpuhp_invoke_ap_callback, so the work set is consistent visible. 471 */ 472 smp_mb(); 473 if (!st->should_run) 474 return; 475 476 st->should_run = false; 477 478 /* Single callback invocation for [un]install ? */ 479 if (st->cb) { 480 if (st->cb_state < CPUHP_AP_ONLINE) { 481 local_irq_disable(); 482 ret = cpuhp_invoke_callback(cpu, st->cb_state, st->cb); 483 local_irq_enable(); 484 } else { 485 ret = cpuhp_invoke_callback(cpu, st->cb_state, st->cb); 486 } 487 } else if (st->rollback) { 488 BUG_ON(st->state < CPUHP_AP_ONLINE_IDLE); 489 490 undo_cpu_down(cpu, st, cpuhp_ap_states); 491 /* 492 * This is a momentary workaround to keep the notifier users 493 * happy. Will go away once we got rid of the notifiers. 494 */ 495 cpu_notify_nofail(CPU_DOWN_FAILED, cpu); 496 st->rollback = false; 497 } else { 498 /* Cannot happen .... */ 499 BUG_ON(st->state < CPUHP_AP_ONLINE_IDLE); 500 501 /* Regular hotplug work */ 502 if (st->state < st->target) 503 ret = cpuhp_ap_online(cpu, st); 504 else if (st->state > st->target) 505 ret = cpuhp_ap_offline(cpu, st); 506 } 507 st->result = ret; 508 complete(&st->done); 509 } 510 511 /* Invoke a single callback on a remote cpu */ 512 static int cpuhp_invoke_ap_callback(int cpu, enum cpuhp_state state, 513 int (*cb)(unsigned int)) 514 { 515 struct cpuhp_cpu_state *st = per_cpu_ptr(&cpuhp_state, cpu); 516 517 if (!cpu_online(cpu)) 518 return 0; 519 520 st->cb_state = state; 521 st->cb = cb; 522 /* 523 * Make sure the above stores are visible before should_run becomes 524 * true. Paired with the mb() above in cpuhp_thread_fun() 525 */ 526 smp_mb(); 527 st->should_run = true; 528 wake_up_process(st->thread); 529 wait_for_completion(&st->done); 530 return st->result; 531 } 532 533 /* Regular hotplug invocation of the AP hotplug thread */ 534 static void __cpuhp_kick_ap_work(struct cpuhp_cpu_state *st) 535 { 536 st->result = 0; 537 st->cb = NULL; 538 /* 539 * Make sure the above stores are visible before should_run becomes 540 * true. Paired with the mb() above in cpuhp_thread_fun() 541 */ 542 smp_mb(); 543 st->should_run = true; 544 wake_up_process(st->thread); 545 } 546 547 static int cpuhp_kick_ap_work(unsigned int cpu) 548 { 549 struct cpuhp_cpu_state *st = per_cpu_ptr(&cpuhp_state, cpu); 550 enum cpuhp_state state = st->state; 551 552 trace_cpuhp_enter(cpu, st->target, state, cpuhp_kick_ap_work); 553 __cpuhp_kick_ap_work(st); 554 wait_for_completion(&st->done); 555 trace_cpuhp_exit(cpu, st->state, state, st->result); 556 return st->result; 557 } 558 559 static struct smp_hotplug_thread cpuhp_threads = { 560 .store = &cpuhp_state.thread, 561 .create = &cpuhp_create, 562 .thread_should_run = cpuhp_should_run, 563 .thread_fn = cpuhp_thread_fun, 564 .thread_comm = "cpuhp/%u", 565 .selfparking = true, 566 }; 567 568 void __init cpuhp_threads_init(void) 569 { 570 BUG_ON(smpboot_register_percpu_thread(&cpuhp_threads)); 571 kthread_unpark(this_cpu_read(cpuhp_state.thread)); 572 } 573 574 #ifdef CONFIG_HOTPLUG_CPU 575 EXPORT_SYMBOL(register_cpu_notifier); 576 EXPORT_SYMBOL(__register_cpu_notifier); 577 void unregister_cpu_notifier(struct notifier_block *nb) 578 { 579 cpu_maps_update_begin(); 580 raw_notifier_chain_unregister(&cpu_chain, nb); 581 cpu_maps_update_done(); 582 } 583 EXPORT_SYMBOL(unregister_cpu_notifier); 584 585 void __unregister_cpu_notifier(struct notifier_block *nb) 586 { 587 raw_notifier_chain_unregister(&cpu_chain, nb); 588 } 589 EXPORT_SYMBOL(__unregister_cpu_notifier); 590 591 /** 592 * clear_tasks_mm_cpumask - Safely clear tasks' mm_cpumask for a CPU 593 * @cpu: a CPU id 594 * 595 * This function walks all processes, finds a valid mm struct for each one and 596 * then clears a corresponding bit in mm's cpumask. While this all sounds 597 * trivial, there are various non-obvious corner cases, which this function 598 * tries to solve in a safe manner. 599 * 600 * Also note that the function uses a somewhat relaxed locking scheme, so it may 601 * be called only for an already offlined CPU. 602 */ 603 void clear_tasks_mm_cpumask(int cpu) 604 { 605 struct task_struct *p; 606 607 /* 608 * This function is called after the cpu is taken down and marked 609 * offline, so its not like new tasks will ever get this cpu set in 610 * their mm mask. -- Peter Zijlstra 611 * Thus, we may use rcu_read_lock() here, instead of grabbing 612 * full-fledged tasklist_lock. 613 */ 614 WARN_ON(cpu_online(cpu)); 615 rcu_read_lock(); 616 for_each_process(p) { 617 struct task_struct *t; 618 619 /* 620 * Main thread might exit, but other threads may still have 621 * a valid mm. Find one. 622 */ 623 t = find_lock_task_mm(p); 624 if (!t) 625 continue; 626 cpumask_clear_cpu(cpu, mm_cpumask(t->mm)); 627 task_unlock(t); 628 } 629 rcu_read_unlock(); 630 } 631 632 static inline void check_for_tasks(int dead_cpu) 633 { 634 struct task_struct *g, *p; 635 636 read_lock(&tasklist_lock); 637 for_each_process_thread(g, p) { 638 if (!p->on_rq) 639 continue; 640 /* 641 * We do the check with unlocked task_rq(p)->lock. 642 * Order the reading to do not warn about a task, 643 * which was running on this cpu in the past, and 644 * it's just been woken on another cpu. 645 */ 646 rmb(); 647 if (task_cpu(p) != dead_cpu) 648 continue; 649 650 pr_warn("Task %s (pid=%d) is on cpu %d (state=%ld, flags=%x)\n", 651 p->comm, task_pid_nr(p), dead_cpu, p->state, p->flags); 652 } 653 read_unlock(&tasklist_lock); 654 } 655 656 static int notify_down_prepare(unsigned int cpu) 657 { 658 int err, nr_calls = 0; 659 660 err = __cpu_notify(CPU_DOWN_PREPARE, cpu, -1, &nr_calls); 661 if (err) { 662 nr_calls--; 663 __cpu_notify(CPU_DOWN_FAILED, cpu, nr_calls, NULL); 664 pr_warn("%s: attempt to take down CPU %u failed\n", 665 __func__, cpu); 666 } 667 return err; 668 } 669 670 static int notify_dying(unsigned int cpu) 671 { 672 cpu_notify(CPU_DYING, cpu); 673 return 0; 674 } 675 676 /* Take this CPU down. */ 677 static int take_cpu_down(void *_param) 678 { 679 struct cpuhp_cpu_state *st = this_cpu_ptr(&cpuhp_state); 680 enum cpuhp_state target = max((int)st->target, CPUHP_AP_OFFLINE); 681 int err, cpu = smp_processor_id(); 682 683 /* Ensure this CPU doesn't handle any more interrupts. */ 684 err = __cpu_disable(); 685 if (err < 0) 686 return err; 687 688 /* Invoke the former CPU_DYING callbacks */ 689 for (; st->state > target; st->state--) { 690 struct cpuhp_step *step = cpuhp_ap_states + st->state; 691 692 cpuhp_invoke_callback(cpu, st->state, step->teardown); 693 } 694 /* Give up timekeeping duties */ 695 tick_handover_do_timer(); 696 /* Park the stopper thread */ 697 stop_machine_park(cpu); 698 return 0; 699 } 700 701 static int takedown_cpu(unsigned int cpu) 702 { 703 struct cpuhp_cpu_state *st = per_cpu_ptr(&cpuhp_state, cpu); 704 int err; 705 706 /* Park the smpboot threads */ 707 kthread_park(per_cpu_ptr(&cpuhp_state, cpu)->thread); 708 smpboot_park_threads(cpu); 709 710 /* 711 * Prevent irq alloc/free while the dying cpu reorganizes the 712 * interrupt affinities. 713 */ 714 irq_lock_sparse(); 715 716 /* 717 * So now all preempt/rcu users must observe !cpu_active(). 718 */ 719 err = stop_machine(take_cpu_down, NULL, cpumask_of(cpu)); 720 if (err) { 721 /* CPU refused to die */ 722 irq_unlock_sparse(); 723 /* Unpark the hotplug thread so we can rollback there */ 724 kthread_unpark(per_cpu_ptr(&cpuhp_state, cpu)->thread); 725 return err; 726 } 727 BUG_ON(cpu_online(cpu)); 728 729 /* 730 * The migration_call() CPU_DYING callback will have removed all 731 * runnable tasks from the cpu, there's only the idle task left now 732 * that the migration thread is done doing the stop_machine thing. 733 * 734 * Wait for the stop thread to go away. 735 */ 736 wait_for_completion(&st->done); 737 BUG_ON(st->state != CPUHP_AP_IDLE_DEAD); 738 739 /* Interrupts are moved away from the dying cpu, reenable alloc/free */ 740 irq_unlock_sparse(); 741 742 hotplug_cpu__broadcast_tick_pull(cpu); 743 /* This actually kills the CPU. */ 744 __cpu_die(cpu); 745 746 tick_cleanup_dead_cpu(cpu); 747 return 0; 748 } 749 750 static int notify_dead(unsigned int cpu) 751 { 752 cpu_notify_nofail(CPU_DEAD, cpu); 753 check_for_tasks(cpu); 754 return 0; 755 } 756 757 static void cpuhp_complete_idle_dead(void *arg) 758 { 759 struct cpuhp_cpu_state *st = arg; 760 761 complete(&st->done); 762 } 763 764 void cpuhp_report_idle_dead(void) 765 { 766 struct cpuhp_cpu_state *st = this_cpu_ptr(&cpuhp_state); 767 768 BUG_ON(st->state != CPUHP_AP_OFFLINE); 769 rcu_report_dead(smp_processor_id()); 770 st->state = CPUHP_AP_IDLE_DEAD; 771 /* 772 * We cannot call complete after rcu_report_dead() so we delegate it 773 * to an online cpu. 774 */ 775 smp_call_function_single(cpumask_first(cpu_online_mask), 776 cpuhp_complete_idle_dead, st, 0); 777 } 778 779 #else 780 #define notify_down_prepare NULL 781 #define takedown_cpu NULL 782 #define notify_dead NULL 783 #define notify_dying NULL 784 #endif 785 786 #ifdef CONFIG_HOTPLUG_CPU 787 788 /* Requires cpu_add_remove_lock to be held */ 789 static int __ref _cpu_down(unsigned int cpu, int tasks_frozen, 790 enum cpuhp_state target) 791 { 792 struct cpuhp_cpu_state *st = per_cpu_ptr(&cpuhp_state, cpu); 793 int prev_state, ret = 0; 794 bool hasdied = false; 795 796 if (num_online_cpus() == 1) 797 return -EBUSY; 798 799 if (!cpu_present(cpu)) 800 return -EINVAL; 801 802 cpu_hotplug_begin(); 803 804 cpuhp_tasks_frozen = tasks_frozen; 805 806 prev_state = st->state; 807 st->target = target; 808 /* 809 * If the current CPU state is in the range of the AP hotplug thread, 810 * then we need to kick the thread. 811 */ 812 if (st->state > CPUHP_TEARDOWN_CPU) { 813 ret = cpuhp_kick_ap_work(cpu); 814 /* 815 * The AP side has done the error rollback already. Just 816 * return the error code.. 817 */ 818 if (ret) 819 goto out; 820 821 /* 822 * We might have stopped still in the range of the AP hotplug 823 * thread. Nothing to do anymore. 824 */ 825 if (st->state > CPUHP_TEARDOWN_CPU) 826 goto out; 827 } 828 /* 829 * The AP brought itself down to CPUHP_TEARDOWN_CPU. So we need 830 * to do the further cleanups. 831 */ 832 ret = cpuhp_down_callbacks(cpu, st, cpuhp_bp_states, target); 833 if (ret && st->state > CPUHP_TEARDOWN_CPU && st->state < prev_state) { 834 st->target = prev_state; 835 st->rollback = true; 836 cpuhp_kick_ap_work(cpu); 837 } 838 839 hasdied = prev_state != st->state && st->state == CPUHP_OFFLINE; 840 out: 841 cpu_hotplug_done(); 842 /* This post dead nonsense must die */ 843 if (!ret && hasdied) 844 cpu_notify_nofail(CPU_POST_DEAD, cpu); 845 return ret; 846 } 847 848 static int do_cpu_down(unsigned int cpu, enum cpuhp_state target) 849 { 850 int err; 851 852 cpu_maps_update_begin(); 853 854 if (cpu_hotplug_disabled) { 855 err = -EBUSY; 856 goto out; 857 } 858 859 err = _cpu_down(cpu, 0, target); 860 861 out: 862 cpu_maps_update_done(); 863 return err; 864 } 865 int cpu_down(unsigned int cpu) 866 { 867 return do_cpu_down(cpu, CPUHP_OFFLINE); 868 } 869 EXPORT_SYMBOL(cpu_down); 870 #endif /*CONFIG_HOTPLUG_CPU*/ 871 872 /** 873 * notify_cpu_starting(cpu) - call the CPU_STARTING notifiers 874 * @cpu: cpu that just started 875 * 876 * This function calls the cpu_chain notifiers with CPU_STARTING. 877 * It must be called by the arch code on the new cpu, before the new cpu 878 * enables interrupts and before the "boot" cpu returns from __cpu_up(). 879 */ 880 void notify_cpu_starting(unsigned int cpu) 881 { 882 struct cpuhp_cpu_state *st = per_cpu_ptr(&cpuhp_state, cpu); 883 enum cpuhp_state target = min((int)st->target, CPUHP_AP_ONLINE); 884 885 while (st->state < target) { 886 struct cpuhp_step *step; 887 888 st->state++; 889 step = cpuhp_ap_states + st->state; 890 cpuhp_invoke_callback(cpu, st->state, step->startup); 891 } 892 } 893 894 /* 895 * Called from the idle task. We need to set active here, so we can kick off 896 * the stopper thread and unpark the smpboot threads. If the target state is 897 * beyond CPUHP_AP_ONLINE_IDLE we kick cpuhp thread and let it bring up the 898 * cpu further. 899 */ 900 void cpuhp_online_idle(enum cpuhp_state state) 901 { 902 struct cpuhp_cpu_state *st = this_cpu_ptr(&cpuhp_state); 903 unsigned int cpu = smp_processor_id(); 904 905 /* Happens for the boot cpu */ 906 if (state != CPUHP_AP_ONLINE_IDLE) 907 return; 908 909 st->state = CPUHP_AP_ONLINE_IDLE; 910 911 /* Unpark the stopper thread and the hotplug thread of this cpu */ 912 stop_machine_unpark(cpu); 913 kthread_unpark(st->thread); 914 915 /* Should we go further up ? */ 916 if (st->target > CPUHP_AP_ONLINE_IDLE) 917 __cpuhp_kick_ap_work(st); 918 else 919 complete(&st->done); 920 } 921 922 /* Requires cpu_add_remove_lock to be held */ 923 static int _cpu_up(unsigned int cpu, int tasks_frozen, enum cpuhp_state target) 924 { 925 struct cpuhp_cpu_state *st = per_cpu_ptr(&cpuhp_state, cpu); 926 struct task_struct *idle; 927 int ret = 0; 928 929 cpu_hotplug_begin(); 930 931 if (!cpu_present(cpu)) { 932 ret = -EINVAL; 933 goto out; 934 } 935 936 /* 937 * The caller of do_cpu_up might have raced with another 938 * caller. Ignore it for now. 939 */ 940 if (st->state >= target) 941 goto out; 942 943 if (st->state == CPUHP_OFFLINE) { 944 /* Let it fail before we try to bring the cpu up */ 945 idle = idle_thread_get(cpu); 946 if (IS_ERR(idle)) { 947 ret = PTR_ERR(idle); 948 goto out; 949 } 950 } 951 952 cpuhp_tasks_frozen = tasks_frozen; 953 954 st->target = target; 955 /* 956 * If the current CPU state is in the range of the AP hotplug thread, 957 * then we need to kick the thread once more. 958 */ 959 if (st->state > CPUHP_BRINGUP_CPU) { 960 ret = cpuhp_kick_ap_work(cpu); 961 /* 962 * The AP side has done the error rollback already. Just 963 * return the error code.. 964 */ 965 if (ret) 966 goto out; 967 } 968 969 /* 970 * Try to reach the target state. We max out on the BP at 971 * CPUHP_BRINGUP_CPU. After that the AP hotplug thread is 972 * responsible for bringing it up to the target state. 973 */ 974 target = min((int)target, CPUHP_BRINGUP_CPU); 975 ret = cpuhp_up_callbacks(cpu, st, cpuhp_bp_states, target); 976 out: 977 cpu_hotplug_done(); 978 return ret; 979 } 980 981 static int do_cpu_up(unsigned int cpu, enum cpuhp_state target) 982 { 983 int err = 0; 984 985 if (!cpu_possible(cpu)) { 986 pr_err("can't online cpu %d because it is not configured as may-hotadd at boot time\n", 987 cpu); 988 #if defined(CONFIG_IA64) 989 pr_err("please check additional_cpus= boot parameter\n"); 990 #endif 991 return -EINVAL; 992 } 993 994 err = try_online_node(cpu_to_node(cpu)); 995 if (err) 996 return err; 997 998 cpu_maps_update_begin(); 999 1000 if (cpu_hotplug_disabled) { 1001 err = -EBUSY; 1002 goto out; 1003 } 1004 1005 err = _cpu_up(cpu, 0, target); 1006 out: 1007 cpu_maps_update_done(); 1008 return err; 1009 } 1010 1011 int cpu_up(unsigned int cpu) 1012 { 1013 return do_cpu_up(cpu, CPUHP_ONLINE); 1014 } 1015 EXPORT_SYMBOL_GPL(cpu_up); 1016 1017 #ifdef CONFIG_PM_SLEEP_SMP 1018 static cpumask_var_t frozen_cpus; 1019 1020 int disable_nonboot_cpus(void) 1021 { 1022 int cpu, first_cpu, error = 0; 1023 1024 cpu_maps_update_begin(); 1025 first_cpu = cpumask_first(cpu_online_mask); 1026 /* 1027 * We take down all of the non-boot CPUs in one shot to avoid races 1028 * with the userspace trying to use the CPU hotplug at the same time 1029 */ 1030 cpumask_clear(frozen_cpus); 1031 1032 pr_info("Disabling non-boot CPUs ...\n"); 1033 for_each_online_cpu(cpu) { 1034 if (cpu == first_cpu) 1035 continue; 1036 trace_suspend_resume(TPS("CPU_OFF"), cpu, true); 1037 error = _cpu_down(cpu, 1, CPUHP_OFFLINE); 1038 trace_suspend_resume(TPS("CPU_OFF"), cpu, false); 1039 if (!error) 1040 cpumask_set_cpu(cpu, frozen_cpus); 1041 else { 1042 pr_err("Error taking CPU%d down: %d\n", cpu, error); 1043 break; 1044 } 1045 } 1046 1047 if (!error) 1048 BUG_ON(num_online_cpus() > 1); 1049 else 1050 pr_err("Non-boot CPUs are not disabled\n"); 1051 1052 /* 1053 * Make sure the CPUs won't be enabled by someone else. We need to do 1054 * this even in case of failure as all disable_nonboot_cpus() users are 1055 * supposed to do enable_nonboot_cpus() on the failure path. 1056 */ 1057 cpu_hotplug_disabled++; 1058 1059 cpu_maps_update_done(); 1060 return error; 1061 } 1062 1063 void __weak arch_enable_nonboot_cpus_begin(void) 1064 { 1065 } 1066 1067 void __weak arch_enable_nonboot_cpus_end(void) 1068 { 1069 } 1070 1071 void enable_nonboot_cpus(void) 1072 { 1073 int cpu, error; 1074 1075 /* Allow everyone to use the CPU hotplug again */ 1076 cpu_maps_update_begin(); 1077 WARN_ON(--cpu_hotplug_disabled < 0); 1078 if (cpumask_empty(frozen_cpus)) 1079 goto out; 1080 1081 pr_info("Enabling non-boot CPUs ...\n"); 1082 1083 arch_enable_nonboot_cpus_begin(); 1084 1085 for_each_cpu(cpu, frozen_cpus) { 1086 trace_suspend_resume(TPS("CPU_ON"), cpu, true); 1087 error = _cpu_up(cpu, 1, CPUHP_ONLINE); 1088 trace_suspend_resume(TPS("CPU_ON"), cpu, false); 1089 if (!error) { 1090 pr_info("CPU%d is up\n", cpu); 1091 continue; 1092 } 1093 pr_warn("Error taking CPU%d up: %d\n", cpu, error); 1094 } 1095 1096 arch_enable_nonboot_cpus_end(); 1097 1098 cpumask_clear(frozen_cpus); 1099 out: 1100 cpu_maps_update_done(); 1101 } 1102 1103 static int __init alloc_frozen_cpus(void) 1104 { 1105 if (!alloc_cpumask_var(&frozen_cpus, GFP_KERNEL|__GFP_ZERO)) 1106 return -ENOMEM; 1107 return 0; 1108 } 1109 core_initcall(alloc_frozen_cpus); 1110 1111 /* 1112 * When callbacks for CPU hotplug notifications are being executed, we must 1113 * ensure that the state of the system with respect to the tasks being frozen 1114 * or not, as reported by the notification, remains unchanged *throughout the 1115 * duration* of the execution of the callbacks. 1116 * Hence we need to prevent the freezer from racing with regular CPU hotplug. 1117 * 1118 * This synchronization is implemented by mutually excluding regular CPU 1119 * hotplug and Suspend/Hibernate call paths by hooking onto the Suspend/ 1120 * Hibernate notifications. 1121 */ 1122 static int 1123 cpu_hotplug_pm_callback(struct notifier_block *nb, 1124 unsigned long action, void *ptr) 1125 { 1126 switch (action) { 1127 1128 case PM_SUSPEND_PREPARE: 1129 case PM_HIBERNATION_PREPARE: 1130 cpu_hotplug_disable(); 1131 break; 1132 1133 case PM_POST_SUSPEND: 1134 case PM_POST_HIBERNATION: 1135 cpu_hotplug_enable(); 1136 break; 1137 1138 default: 1139 return NOTIFY_DONE; 1140 } 1141 1142 return NOTIFY_OK; 1143 } 1144 1145 1146 static int __init cpu_hotplug_pm_sync_init(void) 1147 { 1148 /* 1149 * cpu_hotplug_pm_callback has higher priority than x86 1150 * bsp_pm_callback which depends on cpu_hotplug_pm_callback 1151 * to disable cpu hotplug to avoid cpu hotplug race. 1152 */ 1153 pm_notifier(cpu_hotplug_pm_callback, 0); 1154 return 0; 1155 } 1156 core_initcall(cpu_hotplug_pm_sync_init); 1157 1158 #endif /* CONFIG_PM_SLEEP_SMP */ 1159 1160 #endif /* CONFIG_SMP */ 1161 1162 /* Boot processor state steps */ 1163 static struct cpuhp_step cpuhp_bp_states[] = { 1164 [CPUHP_OFFLINE] = { 1165 .name = "offline", 1166 .startup = NULL, 1167 .teardown = NULL, 1168 }, 1169 #ifdef CONFIG_SMP 1170 [CPUHP_CREATE_THREADS]= { 1171 .name = "threads:create", 1172 .startup = smpboot_create_threads, 1173 .teardown = NULL, 1174 .cant_stop = true, 1175 }, 1176 /* 1177 * Preparatory and dead notifiers. Will be replaced once the notifiers 1178 * are converted to states. 1179 */ 1180 [CPUHP_NOTIFY_PREPARE] = { 1181 .name = "notify:prepare", 1182 .startup = notify_prepare, 1183 .teardown = notify_dead, 1184 .skip_onerr = true, 1185 .cant_stop = true, 1186 }, 1187 /* Kicks the plugged cpu into life */ 1188 [CPUHP_BRINGUP_CPU] = { 1189 .name = "cpu:bringup", 1190 .startup = bringup_cpu, 1191 .teardown = NULL, 1192 .cant_stop = true, 1193 }, 1194 /* 1195 * Handled on controll processor until the plugged processor manages 1196 * this itself. 1197 */ 1198 [CPUHP_TEARDOWN_CPU] = { 1199 .name = "cpu:teardown", 1200 .startup = NULL, 1201 .teardown = takedown_cpu, 1202 .cant_stop = true, 1203 }, 1204 #endif 1205 }; 1206 1207 /* Application processor state steps */ 1208 static struct cpuhp_step cpuhp_ap_states[] = { 1209 #ifdef CONFIG_SMP 1210 /* Final state before CPU kills itself */ 1211 [CPUHP_AP_IDLE_DEAD] = { 1212 .name = "idle:dead", 1213 }, 1214 /* 1215 * Last state before CPU enters the idle loop to die. Transient state 1216 * for synchronization. 1217 */ 1218 [CPUHP_AP_OFFLINE] = { 1219 .name = "ap:offline", 1220 .cant_stop = true, 1221 }, 1222 /* First state is scheduler control. Interrupts are disabled */ 1223 [CPUHP_AP_SCHED_STARTING] = { 1224 .name = "sched:starting", 1225 .startup = sched_cpu_starting, 1226 .teardown = sched_cpu_dying, 1227 }, 1228 /* 1229 * Low level startup/teardown notifiers. Run with interrupts 1230 * disabled. Will be removed once the notifiers are converted to 1231 * states. 1232 */ 1233 [CPUHP_AP_NOTIFY_STARTING] = { 1234 .name = "notify:starting", 1235 .startup = notify_starting, 1236 .teardown = notify_dying, 1237 .skip_onerr = true, 1238 .cant_stop = true, 1239 }, 1240 /* Entry state on starting. Interrupts enabled from here on. Transient 1241 * state for synchronsization */ 1242 [CPUHP_AP_ONLINE] = { 1243 .name = "ap:online", 1244 }, 1245 /* Handle smpboot threads park/unpark */ 1246 [CPUHP_AP_SMPBOOT_THREADS] = { 1247 .name = "smpboot:threads", 1248 .startup = smpboot_unpark_threads, 1249 .teardown = NULL, 1250 }, 1251 /* 1252 * Online/down_prepare notifiers. Will be removed once the notifiers 1253 * are converted to states. 1254 */ 1255 [CPUHP_AP_NOTIFY_ONLINE] = { 1256 .name = "notify:online", 1257 .startup = notify_online, 1258 .teardown = notify_down_prepare, 1259 .skip_onerr = true, 1260 }, 1261 #endif 1262 /* 1263 * The dynamically registered state space is here 1264 */ 1265 1266 #ifdef CONFIG_SMP 1267 /* Last state is scheduler control setting the cpu active */ 1268 [CPUHP_AP_ACTIVE] = { 1269 .name = "sched:active", 1270 .startup = sched_cpu_activate, 1271 .teardown = sched_cpu_deactivate, 1272 }, 1273 #endif 1274 1275 /* CPU is fully up and running. */ 1276 [CPUHP_ONLINE] = { 1277 .name = "online", 1278 .startup = NULL, 1279 .teardown = NULL, 1280 }, 1281 }; 1282 1283 /* Sanity check for callbacks */ 1284 static int cpuhp_cb_check(enum cpuhp_state state) 1285 { 1286 if (state <= CPUHP_OFFLINE || state >= CPUHP_ONLINE) 1287 return -EINVAL; 1288 return 0; 1289 } 1290 1291 static bool cpuhp_is_ap_state(enum cpuhp_state state) 1292 { 1293 /* 1294 * The extra check for CPUHP_TEARDOWN_CPU is only for documentation 1295 * purposes as that state is handled explicitely in cpu_down. 1296 */ 1297 return state > CPUHP_BRINGUP_CPU && state != CPUHP_TEARDOWN_CPU; 1298 } 1299 1300 static struct cpuhp_step *cpuhp_get_step(enum cpuhp_state state) 1301 { 1302 struct cpuhp_step *sp; 1303 1304 sp = cpuhp_is_ap_state(state) ? cpuhp_ap_states : cpuhp_bp_states; 1305 return sp + state; 1306 } 1307 1308 static void cpuhp_store_callbacks(enum cpuhp_state state, 1309 const char *name, 1310 int (*startup)(unsigned int cpu), 1311 int (*teardown)(unsigned int cpu)) 1312 { 1313 /* (Un)Install the callbacks for further cpu hotplug operations */ 1314 struct cpuhp_step *sp; 1315 1316 mutex_lock(&cpuhp_state_mutex); 1317 sp = cpuhp_get_step(state); 1318 sp->startup = startup; 1319 sp->teardown = teardown; 1320 sp->name = name; 1321 mutex_unlock(&cpuhp_state_mutex); 1322 } 1323 1324 static void *cpuhp_get_teardown_cb(enum cpuhp_state state) 1325 { 1326 return cpuhp_get_step(state)->teardown; 1327 } 1328 1329 /* 1330 * Call the startup/teardown function for a step either on the AP or 1331 * on the current CPU. 1332 */ 1333 static int cpuhp_issue_call(int cpu, enum cpuhp_state state, 1334 int (*cb)(unsigned int), bool bringup) 1335 { 1336 int ret; 1337 1338 if (!cb) 1339 return 0; 1340 /* 1341 * The non AP bound callbacks can fail on bringup. On teardown 1342 * e.g. module removal we crash for now. 1343 */ 1344 #ifdef CONFIG_SMP 1345 if (cpuhp_is_ap_state(state)) 1346 ret = cpuhp_invoke_ap_callback(cpu, state, cb); 1347 else 1348 ret = cpuhp_invoke_callback(cpu, state, cb); 1349 #else 1350 ret = cpuhp_invoke_callback(cpu, state, cb); 1351 #endif 1352 BUG_ON(ret && !bringup); 1353 return ret; 1354 } 1355 1356 /* 1357 * Called from __cpuhp_setup_state on a recoverable failure. 1358 * 1359 * Note: The teardown callbacks for rollback are not allowed to fail! 1360 */ 1361 static void cpuhp_rollback_install(int failedcpu, enum cpuhp_state state, 1362 int (*teardown)(unsigned int cpu)) 1363 { 1364 int cpu; 1365 1366 if (!teardown) 1367 return; 1368 1369 /* Roll back the already executed steps on the other cpus */ 1370 for_each_present_cpu(cpu) { 1371 struct cpuhp_cpu_state *st = per_cpu_ptr(&cpuhp_state, cpu); 1372 int cpustate = st->state; 1373 1374 if (cpu >= failedcpu) 1375 break; 1376 1377 /* Did we invoke the startup call on that cpu ? */ 1378 if (cpustate >= state) 1379 cpuhp_issue_call(cpu, state, teardown, false); 1380 } 1381 } 1382 1383 /* 1384 * Returns a free for dynamic slot assignment of the Online state. The states 1385 * are protected by the cpuhp_slot_states mutex and an empty slot is identified 1386 * by having no name assigned. 1387 */ 1388 static int cpuhp_reserve_state(enum cpuhp_state state) 1389 { 1390 enum cpuhp_state i; 1391 1392 mutex_lock(&cpuhp_state_mutex); 1393 for (i = CPUHP_AP_ONLINE_DYN; i <= CPUHP_AP_ONLINE_DYN_END; i++) { 1394 if (cpuhp_ap_states[i].name) 1395 continue; 1396 1397 cpuhp_ap_states[i].name = "Reserved"; 1398 mutex_unlock(&cpuhp_state_mutex); 1399 return i; 1400 } 1401 mutex_unlock(&cpuhp_state_mutex); 1402 WARN(1, "No more dynamic states available for CPU hotplug\n"); 1403 return -ENOSPC; 1404 } 1405 1406 /** 1407 * __cpuhp_setup_state - Setup the callbacks for an hotplug machine state 1408 * @state: The state to setup 1409 * @invoke: If true, the startup function is invoked for cpus where 1410 * cpu state >= @state 1411 * @startup: startup callback function 1412 * @teardown: teardown callback function 1413 * 1414 * Returns 0 if successful, otherwise a proper error code 1415 */ 1416 int __cpuhp_setup_state(enum cpuhp_state state, 1417 const char *name, bool invoke, 1418 int (*startup)(unsigned int cpu), 1419 int (*teardown)(unsigned int cpu)) 1420 { 1421 int cpu, ret = 0; 1422 int dyn_state = 0; 1423 1424 if (cpuhp_cb_check(state) || !name) 1425 return -EINVAL; 1426 1427 get_online_cpus(); 1428 1429 /* currently assignments for the ONLINE state are possible */ 1430 if (state == CPUHP_AP_ONLINE_DYN) { 1431 dyn_state = 1; 1432 ret = cpuhp_reserve_state(state); 1433 if (ret < 0) 1434 goto out; 1435 state = ret; 1436 } 1437 1438 cpuhp_store_callbacks(state, name, startup, teardown); 1439 1440 if (!invoke || !startup) 1441 goto out; 1442 1443 /* 1444 * Try to call the startup callback for each present cpu 1445 * depending on the hotplug state of the cpu. 1446 */ 1447 for_each_present_cpu(cpu) { 1448 struct cpuhp_cpu_state *st = per_cpu_ptr(&cpuhp_state, cpu); 1449 int cpustate = st->state; 1450 1451 if (cpustate < state) 1452 continue; 1453 1454 ret = cpuhp_issue_call(cpu, state, startup, true); 1455 if (ret) { 1456 cpuhp_rollback_install(cpu, state, teardown); 1457 cpuhp_store_callbacks(state, NULL, NULL, NULL); 1458 goto out; 1459 } 1460 } 1461 out: 1462 put_online_cpus(); 1463 if (!ret && dyn_state) 1464 return state; 1465 return ret; 1466 } 1467 EXPORT_SYMBOL(__cpuhp_setup_state); 1468 1469 /** 1470 * __cpuhp_remove_state - Remove the callbacks for an hotplug machine state 1471 * @state: The state to remove 1472 * @invoke: If true, the teardown function is invoked for cpus where 1473 * cpu state >= @state 1474 * 1475 * The teardown callback is currently not allowed to fail. Think 1476 * about module removal! 1477 */ 1478 void __cpuhp_remove_state(enum cpuhp_state state, bool invoke) 1479 { 1480 int (*teardown)(unsigned int cpu) = cpuhp_get_teardown_cb(state); 1481 int cpu; 1482 1483 BUG_ON(cpuhp_cb_check(state)); 1484 1485 get_online_cpus(); 1486 1487 if (!invoke || !teardown) 1488 goto remove; 1489 1490 /* 1491 * Call the teardown callback for each present cpu depending 1492 * on the hotplug state of the cpu. This function is not 1493 * allowed to fail currently! 1494 */ 1495 for_each_present_cpu(cpu) { 1496 struct cpuhp_cpu_state *st = per_cpu_ptr(&cpuhp_state, cpu); 1497 int cpustate = st->state; 1498 1499 if (cpustate >= state) 1500 cpuhp_issue_call(cpu, state, teardown, false); 1501 } 1502 remove: 1503 cpuhp_store_callbacks(state, NULL, NULL, NULL); 1504 put_online_cpus(); 1505 } 1506 EXPORT_SYMBOL(__cpuhp_remove_state); 1507 1508 #if defined(CONFIG_SYSFS) && defined(CONFIG_HOTPLUG_CPU) 1509 static ssize_t show_cpuhp_state(struct device *dev, 1510 struct device_attribute *attr, char *buf) 1511 { 1512 struct cpuhp_cpu_state *st = per_cpu_ptr(&cpuhp_state, dev->id); 1513 1514 return sprintf(buf, "%d\n", st->state); 1515 } 1516 static DEVICE_ATTR(state, 0444, show_cpuhp_state, NULL); 1517 1518 static ssize_t write_cpuhp_target(struct device *dev, 1519 struct device_attribute *attr, 1520 const char *buf, size_t count) 1521 { 1522 struct cpuhp_cpu_state *st = per_cpu_ptr(&cpuhp_state, dev->id); 1523 struct cpuhp_step *sp; 1524 int target, ret; 1525 1526 ret = kstrtoint(buf, 10, &target); 1527 if (ret) 1528 return ret; 1529 1530 #ifdef CONFIG_CPU_HOTPLUG_STATE_CONTROL 1531 if (target < CPUHP_OFFLINE || target > CPUHP_ONLINE) 1532 return -EINVAL; 1533 #else 1534 if (target != CPUHP_OFFLINE && target != CPUHP_ONLINE) 1535 return -EINVAL; 1536 #endif 1537 1538 ret = lock_device_hotplug_sysfs(); 1539 if (ret) 1540 return ret; 1541 1542 mutex_lock(&cpuhp_state_mutex); 1543 sp = cpuhp_get_step(target); 1544 ret = !sp->name || sp->cant_stop ? -EINVAL : 0; 1545 mutex_unlock(&cpuhp_state_mutex); 1546 if (ret) 1547 return ret; 1548 1549 if (st->state < target) 1550 ret = do_cpu_up(dev->id, target); 1551 else 1552 ret = do_cpu_down(dev->id, target); 1553 1554 unlock_device_hotplug(); 1555 return ret ? ret : count; 1556 } 1557 1558 static ssize_t show_cpuhp_target(struct device *dev, 1559 struct device_attribute *attr, char *buf) 1560 { 1561 struct cpuhp_cpu_state *st = per_cpu_ptr(&cpuhp_state, dev->id); 1562 1563 return sprintf(buf, "%d\n", st->target); 1564 } 1565 static DEVICE_ATTR(target, 0644, show_cpuhp_target, write_cpuhp_target); 1566 1567 static struct attribute *cpuhp_cpu_attrs[] = { 1568 &dev_attr_state.attr, 1569 &dev_attr_target.attr, 1570 NULL 1571 }; 1572 1573 static struct attribute_group cpuhp_cpu_attr_group = { 1574 .attrs = cpuhp_cpu_attrs, 1575 .name = "hotplug", 1576 NULL 1577 }; 1578 1579 static ssize_t show_cpuhp_states(struct device *dev, 1580 struct device_attribute *attr, char *buf) 1581 { 1582 ssize_t cur, res = 0; 1583 int i; 1584 1585 mutex_lock(&cpuhp_state_mutex); 1586 for (i = CPUHP_OFFLINE; i <= CPUHP_ONLINE; i++) { 1587 struct cpuhp_step *sp = cpuhp_get_step(i); 1588 1589 if (sp->name) { 1590 cur = sprintf(buf, "%3d: %s\n", i, sp->name); 1591 buf += cur; 1592 res += cur; 1593 } 1594 } 1595 mutex_unlock(&cpuhp_state_mutex); 1596 return res; 1597 } 1598 static DEVICE_ATTR(states, 0444, show_cpuhp_states, NULL); 1599 1600 static struct attribute *cpuhp_cpu_root_attrs[] = { 1601 &dev_attr_states.attr, 1602 NULL 1603 }; 1604 1605 static struct attribute_group cpuhp_cpu_root_attr_group = { 1606 .attrs = cpuhp_cpu_root_attrs, 1607 .name = "hotplug", 1608 NULL 1609 }; 1610 1611 static int __init cpuhp_sysfs_init(void) 1612 { 1613 int cpu, ret; 1614 1615 ret = sysfs_create_group(&cpu_subsys.dev_root->kobj, 1616 &cpuhp_cpu_root_attr_group); 1617 if (ret) 1618 return ret; 1619 1620 for_each_possible_cpu(cpu) { 1621 struct device *dev = get_cpu_device(cpu); 1622 1623 if (!dev) 1624 continue; 1625 ret = sysfs_create_group(&dev->kobj, &cpuhp_cpu_attr_group); 1626 if (ret) 1627 return ret; 1628 } 1629 return 0; 1630 } 1631 device_initcall(cpuhp_sysfs_init); 1632 #endif 1633 1634 /* 1635 * cpu_bit_bitmap[] is a special, "compressed" data structure that 1636 * represents all NR_CPUS bits binary values of 1<<nr. 1637 * 1638 * It is used by cpumask_of() to get a constant address to a CPU 1639 * mask value that has a single bit set only. 1640 */ 1641 1642 /* cpu_bit_bitmap[0] is empty - so we can back into it */ 1643 #define MASK_DECLARE_1(x) [x+1][0] = (1UL << (x)) 1644 #define MASK_DECLARE_2(x) MASK_DECLARE_1(x), MASK_DECLARE_1(x+1) 1645 #define MASK_DECLARE_4(x) MASK_DECLARE_2(x), MASK_DECLARE_2(x+2) 1646 #define MASK_DECLARE_8(x) MASK_DECLARE_4(x), MASK_DECLARE_4(x+4) 1647 1648 const unsigned long cpu_bit_bitmap[BITS_PER_LONG+1][BITS_TO_LONGS(NR_CPUS)] = { 1649 1650 MASK_DECLARE_8(0), MASK_DECLARE_8(8), 1651 MASK_DECLARE_8(16), MASK_DECLARE_8(24), 1652 #if BITS_PER_LONG > 32 1653 MASK_DECLARE_8(32), MASK_DECLARE_8(40), 1654 MASK_DECLARE_8(48), MASK_DECLARE_8(56), 1655 #endif 1656 }; 1657 EXPORT_SYMBOL_GPL(cpu_bit_bitmap); 1658 1659 const DECLARE_BITMAP(cpu_all_bits, NR_CPUS) = CPU_BITS_ALL; 1660 EXPORT_SYMBOL(cpu_all_bits); 1661 1662 #ifdef CONFIG_INIT_ALL_POSSIBLE 1663 struct cpumask __cpu_possible_mask __read_mostly 1664 = {CPU_BITS_ALL}; 1665 #else 1666 struct cpumask __cpu_possible_mask __read_mostly; 1667 #endif 1668 EXPORT_SYMBOL(__cpu_possible_mask); 1669 1670 struct cpumask __cpu_online_mask __read_mostly; 1671 EXPORT_SYMBOL(__cpu_online_mask); 1672 1673 struct cpumask __cpu_present_mask __read_mostly; 1674 EXPORT_SYMBOL(__cpu_present_mask); 1675 1676 struct cpumask __cpu_active_mask __read_mostly; 1677 EXPORT_SYMBOL(__cpu_active_mask); 1678 1679 void init_cpu_present(const struct cpumask *src) 1680 { 1681 cpumask_copy(&__cpu_present_mask, src); 1682 } 1683 1684 void init_cpu_possible(const struct cpumask *src) 1685 { 1686 cpumask_copy(&__cpu_possible_mask, src); 1687 } 1688 1689 void init_cpu_online(const struct cpumask *src) 1690 { 1691 cpumask_copy(&__cpu_online_mask, src); 1692 } 1693 1694 /* 1695 * Activate the first processor. 1696 */ 1697 void __init boot_cpu_init(void) 1698 { 1699 int cpu = smp_processor_id(); 1700 1701 /* Mark the boot cpu "present", "online" etc for SMP and UP case */ 1702 set_cpu_online(cpu, true); 1703 set_cpu_active(cpu, true); 1704 set_cpu_present(cpu, true); 1705 set_cpu_possible(cpu, true); 1706 } 1707 1708 /* 1709 * Must be called _AFTER_ setting up the per_cpu areas 1710 */ 1711 void __init boot_cpu_state_init(void) 1712 { 1713 per_cpu_ptr(&cpuhp_state, smp_processor_id())->state = CPUHP_ONLINE; 1714 } 1715