1 /* CPU control. 2 * (C) 2001, 2002, 2003, 2004 Rusty Russell 3 * 4 * This code is licenced under the GPL. 5 */ 6 #include <linux/proc_fs.h> 7 #include <linux/smp.h> 8 #include <linux/init.h> 9 #include <linux/notifier.h> 10 #include <linux/sched.h> 11 #include <linux/unistd.h> 12 #include <linux/cpu.h> 13 #include <linux/oom.h> 14 #include <linux/rcupdate.h> 15 #include <linux/export.h> 16 #include <linux/bug.h> 17 #include <linux/kthread.h> 18 #include <linux/stop_machine.h> 19 #include <linux/mutex.h> 20 #include <linux/gfp.h> 21 #include <linux/suspend.h> 22 #include <linux/lockdep.h> 23 #include <linux/tick.h> 24 #include <linux/irq.h> 25 #include <linux/smpboot.h> 26 27 #include <trace/events/power.h> 28 #define CREATE_TRACE_POINTS 29 #include <trace/events/cpuhp.h> 30 31 #include "smpboot.h" 32 33 /** 34 * cpuhp_cpu_state - Per cpu hotplug state storage 35 * @state: The current cpu state 36 * @target: The target state 37 * @thread: Pointer to the hotplug thread 38 * @should_run: Thread should execute 39 * @cb_stat: The state for a single callback (install/uninstall) 40 * @cb: Single callback function (install/uninstall) 41 * @result: Result of the operation 42 * @done: Signal completion to the issuer of the task 43 */ 44 struct cpuhp_cpu_state { 45 enum cpuhp_state state; 46 enum cpuhp_state target; 47 #ifdef CONFIG_SMP 48 struct task_struct *thread; 49 bool should_run; 50 enum cpuhp_state cb_state; 51 int (*cb)(unsigned int cpu); 52 int result; 53 struct completion done; 54 #endif 55 }; 56 57 static DEFINE_PER_CPU(struct cpuhp_cpu_state, cpuhp_state); 58 59 /** 60 * cpuhp_step - Hotplug state machine step 61 * @name: Name of the step 62 * @startup: Startup function of the step 63 * @teardown: Teardown function of the step 64 * @skip_onerr: Do not invoke the functions on error rollback 65 * Will go away once the notifiers are gone 66 * @cant_stop: Bringup/teardown can't be stopped at this step 67 */ 68 struct cpuhp_step { 69 const char *name; 70 int (*startup)(unsigned int cpu); 71 int (*teardown)(unsigned int cpu); 72 bool skip_onerr; 73 bool cant_stop; 74 }; 75 76 static DEFINE_MUTEX(cpuhp_state_mutex); 77 static struct cpuhp_step cpuhp_bp_states[]; 78 static struct cpuhp_step cpuhp_ap_states[]; 79 80 /** 81 * cpuhp_invoke_callback _ Invoke the callbacks for a given state 82 * @cpu: The cpu for which the callback should be invoked 83 * @step: The step in the state machine 84 * @cb: The callback function to invoke 85 * 86 * Called from cpu hotplug and from the state register machinery 87 */ 88 static int cpuhp_invoke_callback(unsigned int cpu, enum cpuhp_state step, 89 int (*cb)(unsigned int)) 90 { 91 struct cpuhp_cpu_state *st = per_cpu_ptr(&cpuhp_state, cpu); 92 int ret = 0; 93 94 if (cb) { 95 trace_cpuhp_enter(cpu, st->target, step, cb); 96 ret = cb(cpu); 97 trace_cpuhp_exit(cpu, st->state, step, ret); 98 } 99 return ret; 100 } 101 102 #ifdef CONFIG_SMP 103 /* Serializes the updates to cpu_online_mask, cpu_present_mask */ 104 static DEFINE_MUTEX(cpu_add_remove_lock); 105 bool cpuhp_tasks_frozen; 106 EXPORT_SYMBOL_GPL(cpuhp_tasks_frozen); 107 108 /* 109 * The following two APIs (cpu_maps_update_begin/done) must be used when 110 * attempting to serialize the updates to cpu_online_mask & cpu_present_mask. 111 * The APIs cpu_notifier_register_begin/done() must be used to protect CPU 112 * hotplug callback (un)registration performed using __register_cpu_notifier() 113 * or __unregister_cpu_notifier(). 114 */ 115 void cpu_maps_update_begin(void) 116 { 117 mutex_lock(&cpu_add_remove_lock); 118 } 119 EXPORT_SYMBOL(cpu_notifier_register_begin); 120 121 void cpu_maps_update_done(void) 122 { 123 mutex_unlock(&cpu_add_remove_lock); 124 } 125 EXPORT_SYMBOL(cpu_notifier_register_done); 126 127 static RAW_NOTIFIER_HEAD(cpu_chain); 128 129 /* If set, cpu_up and cpu_down will return -EBUSY and do nothing. 130 * Should always be manipulated under cpu_add_remove_lock 131 */ 132 static int cpu_hotplug_disabled; 133 134 #ifdef CONFIG_HOTPLUG_CPU 135 136 static struct { 137 struct task_struct *active_writer; 138 /* wait queue to wake up the active_writer */ 139 wait_queue_head_t wq; 140 /* verifies that no writer will get active while readers are active */ 141 struct mutex lock; 142 /* 143 * Also blocks the new readers during 144 * an ongoing cpu hotplug operation. 145 */ 146 atomic_t refcount; 147 148 #ifdef CONFIG_DEBUG_LOCK_ALLOC 149 struct lockdep_map dep_map; 150 #endif 151 } cpu_hotplug = { 152 .active_writer = NULL, 153 .wq = __WAIT_QUEUE_HEAD_INITIALIZER(cpu_hotplug.wq), 154 .lock = __MUTEX_INITIALIZER(cpu_hotplug.lock), 155 #ifdef CONFIG_DEBUG_LOCK_ALLOC 156 .dep_map = {.name = "cpu_hotplug.lock" }, 157 #endif 158 }; 159 160 /* Lockdep annotations for get/put_online_cpus() and cpu_hotplug_begin/end() */ 161 #define cpuhp_lock_acquire_read() lock_map_acquire_read(&cpu_hotplug.dep_map) 162 #define cpuhp_lock_acquire_tryread() \ 163 lock_map_acquire_tryread(&cpu_hotplug.dep_map) 164 #define cpuhp_lock_acquire() lock_map_acquire(&cpu_hotplug.dep_map) 165 #define cpuhp_lock_release() lock_map_release(&cpu_hotplug.dep_map) 166 167 168 void get_online_cpus(void) 169 { 170 might_sleep(); 171 if (cpu_hotplug.active_writer == current) 172 return; 173 cpuhp_lock_acquire_read(); 174 mutex_lock(&cpu_hotplug.lock); 175 atomic_inc(&cpu_hotplug.refcount); 176 mutex_unlock(&cpu_hotplug.lock); 177 } 178 EXPORT_SYMBOL_GPL(get_online_cpus); 179 180 void put_online_cpus(void) 181 { 182 int refcount; 183 184 if (cpu_hotplug.active_writer == current) 185 return; 186 187 refcount = atomic_dec_return(&cpu_hotplug.refcount); 188 if (WARN_ON(refcount < 0)) /* try to fix things up */ 189 atomic_inc(&cpu_hotplug.refcount); 190 191 if (refcount <= 0 && waitqueue_active(&cpu_hotplug.wq)) 192 wake_up(&cpu_hotplug.wq); 193 194 cpuhp_lock_release(); 195 196 } 197 EXPORT_SYMBOL_GPL(put_online_cpus); 198 199 /* 200 * This ensures that the hotplug operation can begin only when the 201 * refcount goes to zero. 202 * 203 * Note that during a cpu-hotplug operation, the new readers, if any, 204 * will be blocked by the cpu_hotplug.lock 205 * 206 * Since cpu_hotplug_begin() is always called after invoking 207 * cpu_maps_update_begin(), we can be sure that only one writer is active. 208 * 209 * Note that theoretically, there is a possibility of a livelock: 210 * - Refcount goes to zero, last reader wakes up the sleeping 211 * writer. 212 * - Last reader unlocks the cpu_hotplug.lock. 213 * - A new reader arrives at this moment, bumps up the refcount. 214 * - The writer acquires the cpu_hotplug.lock finds the refcount 215 * non zero and goes to sleep again. 216 * 217 * However, this is very difficult to achieve in practice since 218 * get_online_cpus() not an api which is called all that often. 219 * 220 */ 221 void cpu_hotplug_begin(void) 222 { 223 DEFINE_WAIT(wait); 224 225 cpu_hotplug.active_writer = current; 226 cpuhp_lock_acquire(); 227 228 for (;;) { 229 mutex_lock(&cpu_hotplug.lock); 230 prepare_to_wait(&cpu_hotplug.wq, &wait, TASK_UNINTERRUPTIBLE); 231 if (likely(!atomic_read(&cpu_hotplug.refcount))) 232 break; 233 mutex_unlock(&cpu_hotplug.lock); 234 schedule(); 235 } 236 finish_wait(&cpu_hotplug.wq, &wait); 237 } 238 239 void cpu_hotplug_done(void) 240 { 241 cpu_hotplug.active_writer = NULL; 242 mutex_unlock(&cpu_hotplug.lock); 243 cpuhp_lock_release(); 244 } 245 246 /* 247 * Wait for currently running CPU hotplug operations to complete (if any) and 248 * disable future CPU hotplug (from sysfs). The 'cpu_add_remove_lock' protects 249 * the 'cpu_hotplug_disabled' flag. The same lock is also acquired by the 250 * hotplug path before performing hotplug operations. So acquiring that lock 251 * guarantees mutual exclusion from any currently running hotplug operations. 252 */ 253 void cpu_hotplug_disable(void) 254 { 255 cpu_maps_update_begin(); 256 cpu_hotplug_disabled++; 257 cpu_maps_update_done(); 258 } 259 EXPORT_SYMBOL_GPL(cpu_hotplug_disable); 260 261 void cpu_hotplug_enable(void) 262 { 263 cpu_maps_update_begin(); 264 WARN_ON(--cpu_hotplug_disabled < 0); 265 cpu_maps_update_done(); 266 } 267 EXPORT_SYMBOL_GPL(cpu_hotplug_enable); 268 #endif /* CONFIG_HOTPLUG_CPU */ 269 270 /* Need to know about CPUs going up/down? */ 271 int register_cpu_notifier(struct notifier_block *nb) 272 { 273 int ret; 274 cpu_maps_update_begin(); 275 ret = raw_notifier_chain_register(&cpu_chain, nb); 276 cpu_maps_update_done(); 277 return ret; 278 } 279 280 int __register_cpu_notifier(struct notifier_block *nb) 281 { 282 return raw_notifier_chain_register(&cpu_chain, nb); 283 } 284 285 static int __cpu_notify(unsigned long val, unsigned int cpu, int nr_to_call, 286 int *nr_calls) 287 { 288 unsigned long mod = cpuhp_tasks_frozen ? CPU_TASKS_FROZEN : 0; 289 void *hcpu = (void *)(long)cpu; 290 291 int ret; 292 293 ret = __raw_notifier_call_chain(&cpu_chain, val | mod, hcpu, nr_to_call, 294 nr_calls); 295 296 return notifier_to_errno(ret); 297 } 298 299 static int cpu_notify(unsigned long val, unsigned int cpu) 300 { 301 return __cpu_notify(val, cpu, -1, NULL); 302 } 303 304 /* Notifier wrappers for transitioning to state machine */ 305 static int notify_prepare(unsigned int cpu) 306 { 307 int nr_calls = 0; 308 int ret; 309 310 ret = __cpu_notify(CPU_UP_PREPARE, cpu, -1, &nr_calls); 311 if (ret) { 312 nr_calls--; 313 printk(KERN_WARNING "%s: attempt to bring up CPU %u failed\n", 314 __func__, cpu); 315 __cpu_notify(CPU_UP_CANCELED, cpu, nr_calls, NULL); 316 } 317 return ret; 318 } 319 320 static int notify_online(unsigned int cpu) 321 { 322 cpu_notify(CPU_ONLINE, cpu); 323 return 0; 324 } 325 326 static int notify_starting(unsigned int cpu) 327 { 328 cpu_notify(CPU_STARTING, cpu); 329 return 0; 330 } 331 332 static int bringup_wait_for_ap(unsigned int cpu) 333 { 334 struct cpuhp_cpu_state *st = per_cpu_ptr(&cpuhp_state, cpu); 335 336 wait_for_completion(&st->done); 337 return st->result; 338 } 339 340 static int bringup_cpu(unsigned int cpu) 341 { 342 struct task_struct *idle = idle_thread_get(cpu); 343 int ret; 344 345 /* Arch-specific enabling code. */ 346 ret = __cpu_up(cpu, idle); 347 if (ret) { 348 cpu_notify(CPU_UP_CANCELED, cpu); 349 return ret; 350 } 351 ret = bringup_wait_for_ap(cpu); 352 BUG_ON(!cpu_online(cpu)); 353 return ret; 354 } 355 356 /* 357 * Hotplug state machine related functions 358 */ 359 static void undo_cpu_down(unsigned int cpu, struct cpuhp_cpu_state *st, 360 struct cpuhp_step *steps) 361 { 362 for (st->state++; st->state < st->target; st->state++) { 363 struct cpuhp_step *step = steps + st->state; 364 365 if (!step->skip_onerr) 366 cpuhp_invoke_callback(cpu, st->state, step->startup); 367 } 368 } 369 370 static int cpuhp_down_callbacks(unsigned int cpu, struct cpuhp_cpu_state *st, 371 struct cpuhp_step *steps, enum cpuhp_state target) 372 { 373 enum cpuhp_state prev_state = st->state; 374 int ret = 0; 375 376 for (; st->state > target; st->state--) { 377 struct cpuhp_step *step = steps + st->state; 378 379 ret = cpuhp_invoke_callback(cpu, st->state, step->teardown); 380 if (ret) { 381 st->target = prev_state; 382 undo_cpu_down(cpu, st, steps); 383 break; 384 } 385 } 386 return ret; 387 } 388 389 static void undo_cpu_up(unsigned int cpu, struct cpuhp_cpu_state *st, 390 struct cpuhp_step *steps) 391 { 392 for (st->state--; st->state > st->target; st->state--) { 393 struct cpuhp_step *step = steps + st->state; 394 395 if (!step->skip_onerr) 396 cpuhp_invoke_callback(cpu, st->state, step->teardown); 397 } 398 } 399 400 static int cpuhp_up_callbacks(unsigned int cpu, struct cpuhp_cpu_state *st, 401 struct cpuhp_step *steps, enum cpuhp_state target) 402 { 403 enum cpuhp_state prev_state = st->state; 404 int ret = 0; 405 406 while (st->state < target) { 407 struct cpuhp_step *step; 408 409 st->state++; 410 step = steps + st->state; 411 ret = cpuhp_invoke_callback(cpu, st->state, step->startup); 412 if (ret) { 413 st->target = prev_state; 414 undo_cpu_up(cpu, st, steps); 415 break; 416 } 417 } 418 return ret; 419 } 420 421 /* 422 * The cpu hotplug threads manage the bringup and teardown of the cpus 423 */ 424 static void cpuhp_create(unsigned int cpu) 425 { 426 struct cpuhp_cpu_state *st = per_cpu_ptr(&cpuhp_state, cpu); 427 428 init_completion(&st->done); 429 } 430 431 static int cpuhp_should_run(unsigned int cpu) 432 { 433 struct cpuhp_cpu_state *st = this_cpu_ptr(&cpuhp_state); 434 435 return st->should_run; 436 } 437 438 /* Execute the teardown callbacks. Used to be CPU_DOWN_PREPARE */ 439 static int cpuhp_ap_offline(unsigned int cpu, struct cpuhp_cpu_state *st) 440 { 441 enum cpuhp_state target = max((int)st->target, CPUHP_TEARDOWN_CPU); 442 443 return cpuhp_down_callbacks(cpu, st, cpuhp_ap_states, target); 444 } 445 446 /* Execute the online startup callbacks. Used to be CPU_ONLINE */ 447 static int cpuhp_ap_online(unsigned int cpu, struct cpuhp_cpu_state *st) 448 { 449 return cpuhp_up_callbacks(cpu, st, cpuhp_ap_states, st->target); 450 } 451 452 /* 453 * Execute teardown/startup callbacks on the plugged cpu. Also used to invoke 454 * callbacks when a state gets [un]installed at runtime. 455 */ 456 static void cpuhp_thread_fun(unsigned int cpu) 457 { 458 struct cpuhp_cpu_state *st = this_cpu_ptr(&cpuhp_state); 459 int ret = 0; 460 461 /* 462 * Paired with the mb() in cpuhp_kick_ap_work and 463 * cpuhp_invoke_ap_callback, so the work set is consistent visible. 464 */ 465 smp_mb(); 466 if (!st->should_run) 467 return; 468 469 st->should_run = false; 470 471 /* Single callback invocation for [un]install ? */ 472 if (st->cb) { 473 if (st->cb_state < CPUHP_AP_ONLINE) { 474 local_irq_disable(); 475 ret = cpuhp_invoke_callback(cpu, st->cb_state, st->cb); 476 local_irq_enable(); 477 } else { 478 ret = cpuhp_invoke_callback(cpu, st->cb_state, st->cb); 479 } 480 } else { 481 /* Cannot happen .... */ 482 BUG_ON(st->state < CPUHP_AP_ONLINE_IDLE); 483 484 /* Regular hotplug work */ 485 if (st->state < st->target) 486 ret = cpuhp_ap_online(cpu, st); 487 else if (st->state > st->target) 488 ret = cpuhp_ap_offline(cpu, st); 489 } 490 st->result = ret; 491 complete(&st->done); 492 } 493 494 /* Invoke a single callback on a remote cpu */ 495 static int cpuhp_invoke_ap_callback(int cpu, enum cpuhp_state state, 496 int (*cb)(unsigned int)) 497 { 498 struct cpuhp_cpu_state *st = per_cpu_ptr(&cpuhp_state, cpu); 499 500 if (!cpu_online(cpu)) 501 return 0; 502 503 st->cb_state = state; 504 st->cb = cb; 505 /* 506 * Make sure the above stores are visible before should_run becomes 507 * true. Paired with the mb() above in cpuhp_thread_fun() 508 */ 509 smp_mb(); 510 st->should_run = true; 511 wake_up_process(st->thread); 512 wait_for_completion(&st->done); 513 return st->result; 514 } 515 516 /* Regular hotplug invocation of the AP hotplug thread */ 517 static void __cpuhp_kick_ap_work(struct cpuhp_cpu_state *st) 518 { 519 st->result = 0; 520 st->cb = NULL; 521 /* 522 * Make sure the above stores are visible before should_run becomes 523 * true. Paired with the mb() above in cpuhp_thread_fun() 524 */ 525 smp_mb(); 526 st->should_run = true; 527 wake_up_process(st->thread); 528 } 529 530 static int cpuhp_kick_ap_work(unsigned int cpu) 531 { 532 struct cpuhp_cpu_state *st = per_cpu_ptr(&cpuhp_state, cpu); 533 enum cpuhp_state state = st->state; 534 535 trace_cpuhp_enter(cpu, st->target, state, cpuhp_kick_ap_work); 536 __cpuhp_kick_ap_work(st); 537 wait_for_completion(&st->done); 538 trace_cpuhp_exit(cpu, st->state, state, st->result); 539 return st->result; 540 } 541 542 static struct smp_hotplug_thread cpuhp_threads = { 543 .store = &cpuhp_state.thread, 544 .create = &cpuhp_create, 545 .thread_should_run = cpuhp_should_run, 546 .thread_fn = cpuhp_thread_fun, 547 .thread_comm = "cpuhp/%u", 548 .selfparking = true, 549 }; 550 551 void __init cpuhp_threads_init(void) 552 { 553 BUG_ON(smpboot_register_percpu_thread(&cpuhp_threads)); 554 kthread_unpark(this_cpu_read(cpuhp_state.thread)); 555 } 556 557 #ifdef CONFIG_HOTPLUG_CPU 558 EXPORT_SYMBOL(register_cpu_notifier); 559 EXPORT_SYMBOL(__register_cpu_notifier); 560 void unregister_cpu_notifier(struct notifier_block *nb) 561 { 562 cpu_maps_update_begin(); 563 raw_notifier_chain_unregister(&cpu_chain, nb); 564 cpu_maps_update_done(); 565 } 566 EXPORT_SYMBOL(unregister_cpu_notifier); 567 568 void __unregister_cpu_notifier(struct notifier_block *nb) 569 { 570 raw_notifier_chain_unregister(&cpu_chain, nb); 571 } 572 EXPORT_SYMBOL(__unregister_cpu_notifier); 573 574 /** 575 * clear_tasks_mm_cpumask - Safely clear tasks' mm_cpumask for a CPU 576 * @cpu: a CPU id 577 * 578 * This function walks all processes, finds a valid mm struct for each one and 579 * then clears a corresponding bit in mm's cpumask. While this all sounds 580 * trivial, there are various non-obvious corner cases, which this function 581 * tries to solve in a safe manner. 582 * 583 * Also note that the function uses a somewhat relaxed locking scheme, so it may 584 * be called only for an already offlined CPU. 585 */ 586 void clear_tasks_mm_cpumask(int cpu) 587 { 588 struct task_struct *p; 589 590 /* 591 * This function is called after the cpu is taken down and marked 592 * offline, so its not like new tasks will ever get this cpu set in 593 * their mm mask. -- Peter Zijlstra 594 * Thus, we may use rcu_read_lock() here, instead of grabbing 595 * full-fledged tasklist_lock. 596 */ 597 WARN_ON(cpu_online(cpu)); 598 rcu_read_lock(); 599 for_each_process(p) { 600 struct task_struct *t; 601 602 /* 603 * Main thread might exit, but other threads may still have 604 * a valid mm. Find one. 605 */ 606 t = find_lock_task_mm(p); 607 if (!t) 608 continue; 609 cpumask_clear_cpu(cpu, mm_cpumask(t->mm)); 610 task_unlock(t); 611 } 612 rcu_read_unlock(); 613 } 614 615 static inline void check_for_tasks(int dead_cpu) 616 { 617 struct task_struct *g, *p; 618 619 read_lock(&tasklist_lock); 620 for_each_process_thread(g, p) { 621 if (!p->on_rq) 622 continue; 623 /* 624 * We do the check with unlocked task_rq(p)->lock. 625 * Order the reading to do not warn about a task, 626 * which was running on this cpu in the past, and 627 * it's just been woken on another cpu. 628 */ 629 rmb(); 630 if (task_cpu(p) != dead_cpu) 631 continue; 632 633 pr_warn("Task %s (pid=%d) is on cpu %d (state=%ld, flags=%x)\n", 634 p->comm, task_pid_nr(p), dead_cpu, p->state, p->flags); 635 } 636 read_unlock(&tasklist_lock); 637 } 638 639 static void cpu_notify_nofail(unsigned long val, unsigned int cpu) 640 { 641 BUG_ON(cpu_notify(val, cpu)); 642 } 643 644 static int notify_down_prepare(unsigned int cpu) 645 { 646 int err, nr_calls = 0; 647 648 err = __cpu_notify(CPU_DOWN_PREPARE, cpu, -1, &nr_calls); 649 if (err) { 650 nr_calls--; 651 __cpu_notify(CPU_DOWN_FAILED, cpu, nr_calls, NULL); 652 pr_warn("%s: attempt to take down CPU %u failed\n", 653 __func__, cpu); 654 } 655 return err; 656 } 657 658 static int notify_dying(unsigned int cpu) 659 { 660 cpu_notify(CPU_DYING, cpu); 661 return 0; 662 } 663 664 /* Take this CPU down. */ 665 static int take_cpu_down(void *_param) 666 { 667 struct cpuhp_cpu_state *st = this_cpu_ptr(&cpuhp_state); 668 enum cpuhp_state target = max((int)st->target, CPUHP_AP_OFFLINE); 669 int err, cpu = smp_processor_id(); 670 671 /* Ensure this CPU doesn't handle any more interrupts. */ 672 err = __cpu_disable(); 673 if (err < 0) 674 return err; 675 676 /* Invoke the former CPU_DYING callbacks */ 677 for (; st->state > target; st->state--) { 678 struct cpuhp_step *step = cpuhp_ap_states + st->state; 679 680 cpuhp_invoke_callback(cpu, st->state, step->teardown); 681 } 682 /* Give up timekeeping duties */ 683 tick_handover_do_timer(); 684 /* Park the stopper thread */ 685 stop_machine_park(cpu); 686 return 0; 687 } 688 689 static int takedown_cpu(unsigned int cpu) 690 { 691 struct cpuhp_cpu_state *st = per_cpu_ptr(&cpuhp_state, cpu); 692 int err; 693 694 /* 695 * By now we've cleared cpu_active_mask, wait for all preempt-disabled 696 * and RCU users of this state to go away such that all new such users 697 * will observe it. 698 * 699 * For CONFIG_PREEMPT we have preemptible RCU and its sync_rcu() might 700 * not imply sync_sched(), so wait for both. 701 * 702 * Do sync before park smpboot threads to take care the rcu boost case. 703 */ 704 if (IS_ENABLED(CONFIG_PREEMPT)) 705 synchronize_rcu_mult(call_rcu, call_rcu_sched); 706 else 707 synchronize_rcu(); 708 709 /* Park the smpboot threads */ 710 kthread_park(per_cpu_ptr(&cpuhp_state, cpu)->thread); 711 smpboot_park_threads(cpu); 712 713 /* 714 * Prevent irq alloc/free while the dying cpu reorganizes the 715 * interrupt affinities. 716 */ 717 irq_lock_sparse(); 718 719 /* 720 * So now all preempt/rcu users must observe !cpu_active(). 721 */ 722 err = stop_machine(take_cpu_down, NULL, cpumask_of(cpu)); 723 if (err) { 724 /* CPU didn't die: tell everyone. Can't complain. */ 725 cpu_notify_nofail(CPU_DOWN_FAILED, cpu); 726 irq_unlock_sparse(); 727 return err; 728 } 729 BUG_ON(cpu_online(cpu)); 730 731 /* 732 * The migration_call() CPU_DYING callback will have removed all 733 * runnable tasks from the cpu, there's only the idle task left now 734 * that the migration thread is done doing the stop_machine thing. 735 * 736 * Wait for the stop thread to go away. 737 */ 738 wait_for_completion(&st->done); 739 BUG_ON(st->state != CPUHP_AP_IDLE_DEAD); 740 741 /* Interrupts are moved away from the dying cpu, reenable alloc/free */ 742 irq_unlock_sparse(); 743 744 hotplug_cpu__broadcast_tick_pull(cpu); 745 /* This actually kills the CPU. */ 746 __cpu_die(cpu); 747 748 tick_cleanup_dead_cpu(cpu); 749 return 0; 750 } 751 752 static int notify_dead(unsigned int cpu) 753 { 754 cpu_notify_nofail(CPU_DEAD, cpu); 755 check_for_tasks(cpu); 756 return 0; 757 } 758 759 static void cpuhp_complete_idle_dead(void *arg) 760 { 761 struct cpuhp_cpu_state *st = arg; 762 763 complete(&st->done); 764 } 765 766 void cpuhp_report_idle_dead(void) 767 { 768 struct cpuhp_cpu_state *st = this_cpu_ptr(&cpuhp_state); 769 770 BUG_ON(st->state != CPUHP_AP_OFFLINE); 771 rcu_report_dead(smp_processor_id()); 772 st->state = CPUHP_AP_IDLE_DEAD; 773 /* 774 * We cannot call complete after rcu_report_dead() so we delegate it 775 * to an online cpu. 776 */ 777 smp_call_function_single(cpumask_first(cpu_online_mask), 778 cpuhp_complete_idle_dead, st, 0); 779 } 780 781 #else 782 #define notify_down_prepare NULL 783 #define takedown_cpu NULL 784 #define notify_dead NULL 785 #define notify_dying NULL 786 #endif 787 788 #ifdef CONFIG_HOTPLUG_CPU 789 790 /* Requires cpu_add_remove_lock to be held */ 791 static int __ref _cpu_down(unsigned int cpu, int tasks_frozen, 792 enum cpuhp_state target) 793 { 794 struct cpuhp_cpu_state *st = per_cpu_ptr(&cpuhp_state, cpu); 795 int prev_state, ret = 0; 796 bool hasdied = false; 797 798 if (num_online_cpus() == 1) 799 return -EBUSY; 800 801 if (!cpu_present(cpu)) 802 return -EINVAL; 803 804 cpu_hotplug_begin(); 805 806 cpuhp_tasks_frozen = tasks_frozen; 807 808 prev_state = st->state; 809 st->target = target; 810 /* 811 * If the current CPU state is in the range of the AP hotplug thread, 812 * then we need to kick the thread. 813 */ 814 if (st->state > CPUHP_TEARDOWN_CPU) { 815 ret = cpuhp_kick_ap_work(cpu); 816 /* 817 * The AP side has done the error rollback already. Just 818 * return the error code.. 819 */ 820 if (ret) 821 goto out; 822 823 /* 824 * We might have stopped still in the range of the AP hotplug 825 * thread. Nothing to do anymore. 826 */ 827 if (st->state > CPUHP_TEARDOWN_CPU) 828 goto out; 829 } 830 /* 831 * The AP brought itself down to CPUHP_TEARDOWN_CPU. So we need 832 * to do the further cleanups. 833 */ 834 ret = cpuhp_down_callbacks(cpu, st, cpuhp_bp_states, target); 835 836 hasdied = prev_state != st->state && st->state == CPUHP_OFFLINE; 837 out: 838 cpu_hotplug_done(); 839 /* This post dead nonsense must die */ 840 if (!ret && hasdied) 841 cpu_notify_nofail(CPU_POST_DEAD, cpu); 842 return ret; 843 } 844 845 static int do_cpu_down(unsigned int cpu, enum cpuhp_state target) 846 { 847 int err; 848 849 cpu_maps_update_begin(); 850 851 if (cpu_hotplug_disabled) { 852 err = -EBUSY; 853 goto out; 854 } 855 856 err = _cpu_down(cpu, 0, target); 857 858 out: 859 cpu_maps_update_done(); 860 return err; 861 } 862 int cpu_down(unsigned int cpu) 863 { 864 return do_cpu_down(cpu, CPUHP_OFFLINE); 865 } 866 EXPORT_SYMBOL(cpu_down); 867 #endif /*CONFIG_HOTPLUG_CPU*/ 868 869 /** 870 * notify_cpu_starting(cpu) - call the CPU_STARTING notifiers 871 * @cpu: cpu that just started 872 * 873 * This function calls the cpu_chain notifiers with CPU_STARTING. 874 * It must be called by the arch code on the new cpu, before the new cpu 875 * enables interrupts and before the "boot" cpu returns from __cpu_up(). 876 */ 877 void notify_cpu_starting(unsigned int cpu) 878 { 879 struct cpuhp_cpu_state *st = per_cpu_ptr(&cpuhp_state, cpu); 880 enum cpuhp_state target = min((int)st->target, CPUHP_AP_ONLINE); 881 882 while (st->state < target) { 883 struct cpuhp_step *step; 884 885 st->state++; 886 step = cpuhp_ap_states + st->state; 887 cpuhp_invoke_callback(cpu, st->state, step->startup); 888 } 889 } 890 891 /* 892 * Called from the idle task. We need to set active here, so we can kick off 893 * the stopper thread and unpark the smpboot threads. If the target state is 894 * beyond CPUHP_AP_ONLINE_IDLE we kick cpuhp thread and let it bring up the 895 * cpu further. 896 */ 897 void cpuhp_online_idle(enum cpuhp_state state) 898 { 899 struct cpuhp_cpu_state *st = this_cpu_ptr(&cpuhp_state); 900 unsigned int cpu = smp_processor_id(); 901 902 /* Happens for the boot cpu */ 903 if (state != CPUHP_AP_ONLINE_IDLE) 904 return; 905 906 st->state = CPUHP_AP_ONLINE_IDLE; 907 908 /* The cpu is marked online, set it active now */ 909 set_cpu_active(cpu, true); 910 /* Unpark the stopper thread and the hotplug thread of this cpu */ 911 stop_machine_unpark(cpu); 912 kthread_unpark(st->thread); 913 914 /* Should we go further up ? */ 915 if (st->target > CPUHP_AP_ONLINE_IDLE) 916 __cpuhp_kick_ap_work(st); 917 else 918 complete(&st->done); 919 } 920 921 /* Requires cpu_add_remove_lock to be held */ 922 static int _cpu_up(unsigned int cpu, int tasks_frozen, enum cpuhp_state target) 923 { 924 struct cpuhp_cpu_state *st = per_cpu_ptr(&cpuhp_state, cpu); 925 struct task_struct *idle; 926 int ret = 0; 927 928 cpu_hotplug_begin(); 929 930 if (!cpu_present(cpu)) { 931 ret = -EINVAL; 932 goto out; 933 } 934 935 /* 936 * The caller of do_cpu_up might have raced with another 937 * caller. Ignore it for now. 938 */ 939 if (st->state >= target) 940 goto out; 941 942 if (st->state == CPUHP_OFFLINE) { 943 /* Let it fail before we try to bring the cpu up */ 944 idle = idle_thread_get(cpu); 945 if (IS_ERR(idle)) { 946 ret = PTR_ERR(idle); 947 goto out; 948 } 949 } 950 951 cpuhp_tasks_frozen = tasks_frozen; 952 953 st->target = target; 954 /* 955 * If the current CPU state is in the range of the AP hotplug thread, 956 * then we need to kick the thread once more. 957 */ 958 if (st->state > CPUHP_BRINGUP_CPU) { 959 ret = cpuhp_kick_ap_work(cpu); 960 /* 961 * The AP side has done the error rollback already. Just 962 * return the error code.. 963 */ 964 if (ret) 965 goto out; 966 } 967 968 /* 969 * Try to reach the target state. We max out on the BP at 970 * CPUHP_BRINGUP_CPU. After that the AP hotplug thread is 971 * responsible for bringing it up to the target state. 972 */ 973 target = min((int)target, CPUHP_BRINGUP_CPU); 974 ret = cpuhp_up_callbacks(cpu, st, cpuhp_bp_states, target); 975 out: 976 cpu_hotplug_done(); 977 return ret; 978 } 979 980 static int do_cpu_up(unsigned int cpu, enum cpuhp_state target) 981 { 982 int err = 0; 983 984 if (!cpu_possible(cpu)) { 985 pr_err("can't online cpu %d because it is not configured as may-hotadd at boot time\n", 986 cpu); 987 #if defined(CONFIG_IA64) 988 pr_err("please check additional_cpus= boot parameter\n"); 989 #endif 990 return -EINVAL; 991 } 992 993 err = try_online_node(cpu_to_node(cpu)); 994 if (err) 995 return err; 996 997 cpu_maps_update_begin(); 998 999 if (cpu_hotplug_disabled) { 1000 err = -EBUSY; 1001 goto out; 1002 } 1003 1004 err = _cpu_up(cpu, 0, target); 1005 out: 1006 cpu_maps_update_done(); 1007 return err; 1008 } 1009 1010 int cpu_up(unsigned int cpu) 1011 { 1012 return do_cpu_up(cpu, CPUHP_ONLINE); 1013 } 1014 EXPORT_SYMBOL_GPL(cpu_up); 1015 1016 #ifdef CONFIG_PM_SLEEP_SMP 1017 static cpumask_var_t frozen_cpus; 1018 1019 int disable_nonboot_cpus(void) 1020 { 1021 int cpu, first_cpu, error = 0; 1022 1023 cpu_maps_update_begin(); 1024 first_cpu = cpumask_first(cpu_online_mask); 1025 /* 1026 * We take down all of the non-boot CPUs in one shot to avoid races 1027 * with the userspace trying to use the CPU hotplug at the same time 1028 */ 1029 cpumask_clear(frozen_cpus); 1030 1031 pr_info("Disabling non-boot CPUs ...\n"); 1032 for_each_online_cpu(cpu) { 1033 if (cpu == first_cpu) 1034 continue; 1035 trace_suspend_resume(TPS("CPU_OFF"), cpu, true); 1036 error = _cpu_down(cpu, 1, CPUHP_OFFLINE); 1037 trace_suspend_resume(TPS("CPU_OFF"), cpu, false); 1038 if (!error) 1039 cpumask_set_cpu(cpu, frozen_cpus); 1040 else { 1041 pr_err("Error taking CPU%d down: %d\n", cpu, error); 1042 break; 1043 } 1044 } 1045 1046 if (!error) 1047 BUG_ON(num_online_cpus() > 1); 1048 else 1049 pr_err("Non-boot CPUs are not disabled\n"); 1050 1051 /* 1052 * Make sure the CPUs won't be enabled by someone else. We need to do 1053 * this even in case of failure as all disable_nonboot_cpus() users are 1054 * supposed to do enable_nonboot_cpus() on the failure path. 1055 */ 1056 cpu_hotplug_disabled++; 1057 1058 cpu_maps_update_done(); 1059 return error; 1060 } 1061 1062 void __weak arch_enable_nonboot_cpus_begin(void) 1063 { 1064 } 1065 1066 void __weak arch_enable_nonboot_cpus_end(void) 1067 { 1068 } 1069 1070 void enable_nonboot_cpus(void) 1071 { 1072 int cpu, error; 1073 1074 /* Allow everyone to use the CPU hotplug again */ 1075 cpu_maps_update_begin(); 1076 WARN_ON(--cpu_hotplug_disabled < 0); 1077 if (cpumask_empty(frozen_cpus)) 1078 goto out; 1079 1080 pr_info("Enabling non-boot CPUs ...\n"); 1081 1082 arch_enable_nonboot_cpus_begin(); 1083 1084 for_each_cpu(cpu, frozen_cpus) { 1085 trace_suspend_resume(TPS("CPU_ON"), cpu, true); 1086 error = _cpu_up(cpu, 1, CPUHP_ONLINE); 1087 trace_suspend_resume(TPS("CPU_ON"), cpu, false); 1088 if (!error) { 1089 pr_info("CPU%d is up\n", cpu); 1090 continue; 1091 } 1092 pr_warn("Error taking CPU%d up: %d\n", cpu, error); 1093 } 1094 1095 arch_enable_nonboot_cpus_end(); 1096 1097 cpumask_clear(frozen_cpus); 1098 out: 1099 cpu_maps_update_done(); 1100 } 1101 1102 static int __init alloc_frozen_cpus(void) 1103 { 1104 if (!alloc_cpumask_var(&frozen_cpus, GFP_KERNEL|__GFP_ZERO)) 1105 return -ENOMEM; 1106 return 0; 1107 } 1108 core_initcall(alloc_frozen_cpus); 1109 1110 /* 1111 * When callbacks for CPU hotplug notifications are being executed, we must 1112 * ensure that the state of the system with respect to the tasks being frozen 1113 * or not, as reported by the notification, remains unchanged *throughout the 1114 * duration* of the execution of the callbacks. 1115 * Hence we need to prevent the freezer from racing with regular CPU hotplug. 1116 * 1117 * This synchronization is implemented by mutually excluding regular CPU 1118 * hotplug and Suspend/Hibernate call paths by hooking onto the Suspend/ 1119 * Hibernate notifications. 1120 */ 1121 static int 1122 cpu_hotplug_pm_callback(struct notifier_block *nb, 1123 unsigned long action, void *ptr) 1124 { 1125 switch (action) { 1126 1127 case PM_SUSPEND_PREPARE: 1128 case PM_HIBERNATION_PREPARE: 1129 cpu_hotplug_disable(); 1130 break; 1131 1132 case PM_POST_SUSPEND: 1133 case PM_POST_HIBERNATION: 1134 cpu_hotplug_enable(); 1135 break; 1136 1137 default: 1138 return NOTIFY_DONE; 1139 } 1140 1141 return NOTIFY_OK; 1142 } 1143 1144 1145 static int __init cpu_hotplug_pm_sync_init(void) 1146 { 1147 /* 1148 * cpu_hotplug_pm_callback has higher priority than x86 1149 * bsp_pm_callback which depends on cpu_hotplug_pm_callback 1150 * to disable cpu hotplug to avoid cpu hotplug race. 1151 */ 1152 pm_notifier(cpu_hotplug_pm_callback, 0); 1153 return 0; 1154 } 1155 core_initcall(cpu_hotplug_pm_sync_init); 1156 1157 #endif /* CONFIG_PM_SLEEP_SMP */ 1158 1159 #endif /* CONFIG_SMP */ 1160 1161 /* Boot processor state steps */ 1162 static struct cpuhp_step cpuhp_bp_states[] = { 1163 [CPUHP_OFFLINE] = { 1164 .name = "offline", 1165 .startup = NULL, 1166 .teardown = NULL, 1167 }, 1168 #ifdef CONFIG_SMP 1169 [CPUHP_CREATE_THREADS]= { 1170 .name = "threads:create", 1171 .startup = smpboot_create_threads, 1172 .teardown = NULL, 1173 .cant_stop = true, 1174 }, 1175 /* 1176 * Preparatory and dead notifiers. Will be replaced once the notifiers 1177 * are converted to states. 1178 */ 1179 [CPUHP_NOTIFY_PREPARE] = { 1180 .name = "notify:prepare", 1181 .startup = notify_prepare, 1182 .teardown = notify_dead, 1183 .skip_onerr = true, 1184 .cant_stop = true, 1185 }, 1186 /* Kicks the plugged cpu into life */ 1187 [CPUHP_BRINGUP_CPU] = { 1188 .name = "cpu:bringup", 1189 .startup = bringup_cpu, 1190 .teardown = NULL, 1191 .cant_stop = true, 1192 }, 1193 /* 1194 * Handled on controll processor until the plugged processor manages 1195 * this itself. 1196 */ 1197 [CPUHP_TEARDOWN_CPU] = { 1198 .name = "cpu:teardown", 1199 .startup = NULL, 1200 .teardown = takedown_cpu, 1201 .cant_stop = true, 1202 }, 1203 #endif 1204 }; 1205 1206 /* Application processor state steps */ 1207 static struct cpuhp_step cpuhp_ap_states[] = { 1208 #ifdef CONFIG_SMP 1209 /* Final state before CPU kills itself */ 1210 [CPUHP_AP_IDLE_DEAD] = { 1211 .name = "idle:dead", 1212 }, 1213 /* 1214 * Last state before CPU enters the idle loop to die. Transient state 1215 * for synchronization. 1216 */ 1217 [CPUHP_AP_OFFLINE] = { 1218 .name = "ap:offline", 1219 .cant_stop = true, 1220 }, 1221 /* 1222 * Low level startup/teardown notifiers. Run with interrupts 1223 * disabled. Will be removed once the notifiers are converted to 1224 * states. 1225 */ 1226 [CPUHP_AP_NOTIFY_STARTING] = { 1227 .name = "notify:starting", 1228 .startup = notify_starting, 1229 .teardown = notify_dying, 1230 .skip_onerr = true, 1231 .cant_stop = true, 1232 }, 1233 /* Entry state on starting. Interrupts enabled from here on. Transient 1234 * state for synchronsization */ 1235 [CPUHP_AP_ONLINE] = { 1236 .name = "ap:online", 1237 }, 1238 /* Handle smpboot threads park/unpark */ 1239 [CPUHP_AP_SMPBOOT_THREADS] = { 1240 .name = "smpboot:threads", 1241 .startup = smpboot_unpark_threads, 1242 .teardown = NULL, 1243 }, 1244 /* 1245 * Online/down_prepare notifiers. Will be removed once the notifiers 1246 * are converted to states. 1247 */ 1248 [CPUHP_AP_NOTIFY_ONLINE] = { 1249 .name = "notify:online", 1250 .startup = notify_online, 1251 .teardown = notify_down_prepare, 1252 }, 1253 #endif 1254 /* 1255 * The dynamically registered state space is here 1256 */ 1257 1258 /* CPU is fully up and running. */ 1259 [CPUHP_ONLINE] = { 1260 .name = "online", 1261 .startup = NULL, 1262 .teardown = NULL, 1263 }, 1264 }; 1265 1266 /* Sanity check for callbacks */ 1267 static int cpuhp_cb_check(enum cpuhp_state state) 1268 { 1269 if (state <= CPUHP_OFFLINE || state >= CPUHP_ONLINE) 1270 return -EINVAL; 1271 return 0; 1272 } 1273 1274 static bool cpuhp_is_ap_state(enum cpuhp_state state) 1275 { 1276 /* 1277 * The extra check for CPUHP_TEARDOWN_CPU is only for documentation 1278 * purposes as that state is handled explicitely in cpu_down. 1279 */ 1280 return state > CPUHP_BRINGUP_CPU && state != CPUHP_TEARDOWN_CPU; 1281 } 1282 1283 static struct cpuhp_step *cpuhp_get_step(enum cpuhp_state state) 1284 { 1285 struct cpuhp_step *sp; 1286 1287 sp = cpuhp_is_ap_state(state) ? cpuhp_ap_states : cpuhp_bp_states; 1288 return sp + state; 1289 } 1290 1291 static void cpuhp_store_callbacks(enum cpuhp_state state, 1292 const char *name, 1293 int (*startup)(unsigned int cpu), 1294 int (*teardown)(unsigned int cpu)) 1295 { 1296 /* (Un)Install the callbacks for further cpu hotplug operations */ 1297 struct cpuhp_step *sp; 1298 1299 mutex_lock(&cpuhp_state_mutex); 1300 sp = cpuhp_get_step(state); 1301 sp->startup = startup; 1302 sp->teardown = teardown; 1303 sp->name = name; 1304 mutex_unlock(&cpuhp_state_mutex); 1305 } 1306 1307 static void *cpuhp_get_teardown_cb(enum cpuhp_state state) 1308 { 1309 return cpuhp_get_step(state)->teardown; 1310 } 1311 1312 /* 1313 * Call the startup/teardown function for a step either on the AP or 1314 * on the current CPU. 1315 */ 1316 static int cpuhp_issue_call(int cpu, enum cpuhp_state state, 1317 int (*cb)(unsigned int), bool bringup) 1318 { 1319 int ret; 1320 1321 if (!cb) 1322 return 0; 1323 /* 1324 * The non AP bound callbacks can fail on bringup. On teardown 1325 * e.g. module removal we crash for now. 1326 */ 1327 #ifdef CONFIG_SMP 1328 if (cpuhp_is_ap_state(state)) 1329 ret = cpuhp_invoke_ap_callback(cpu, state, cb); 1330 else 1331 ret = cpuhp_invoke_callback(cpu, state, cb); 1332 #else 1333 ret = cpuhp_invoke_callback(cpu, state, cb); 1334 #endif 1335 BUG_ON(ret && !bringup); 1336 return ret; 1337 } 1338 1339 /* 1340 * Called from __cpuhp_setup_state on a recoverable failure. 1341 * 1342 * Note: The teardown callbacks for rollback are not allowed to fail! 1343 */ 1344 static void cpuhp_rollback_install(int failedcpu, enum cpuhp_state state, 1345 int (*teardown)(unsigned int cpu)) 1346 { 1347 int cpu; 1348 1349 if (!teardown) 1350 return; 1351 1352 /* Roll back the already executed steps on the other cpus */ 1353 for_each_present_cpu(cpu) { 1354 struct cpuhp_cpu_state *st = per_cpu_ptr(&cpuhp_state, cpu); 1355 int cpustate = st->state; 1356 1357 if (cpu >= failedcpu) 1358 break; 1359 1360 /* Did we invoke the startup call on that cpu ? */ 1361 if (cpustate >= state) 1362 cpuhp_issue_call(cpu, state, teardown, false); 1363 } 1364 } 1365 1366 /* 1367 * Returns a free for dynamic slot assignment of the Online state. The states 1368 * are protected by the cpuhp_slot_states mutex and an empty slot is identified 1369 * by having no name assigned. 1370 */ 1371 static int cpuhp_reserve_state(enum cpuhp_state state) 1372 { 1373 enum cpuhp_state i; 1374 1375 mutex_lock(&cpuhp_state_mutex); 1376 for (i = CPUHP_AP_ONLINE_DYN; i <= CPUHP_AP_ONLINE_DYN_END; i++) { 1377 if (cpuhp_ap_states[i].name) 1378 continue; 1379 1380 cpuhp_ap_states[i].name = "Reserved"; 1381 mutex_unlock(&cpuhp_state_mutex); 1382 return i; 1383 } 1384 mutex_unlock(&cpuhp_state_mutex); 1385 WARN(1, "No more dynamic states available for CPU hotplug\n"); 1386 return -ENOSPC; 1387 } 1388 1389 /** 1390 * __cpuhp_setup_state - Setup the callbacks for an hotplug machine state 1391 * @state: The state to setup 1392 * @invoke: If true, the startup function is invoked for cpus where 1393 * cpu state >= @state 1394 * @startup: startup callback function 1395 * @teardown: teardown callback function 1396 * 1397 * Returns 0 if successful, otherwise a proper error code 1398 */ 1399 int __cpuhp_setup_state(enum cpuhp_state state, 1400 const char *name, bool invoke, 1401 int (*startup)(unsigned int cpu), 1402 int (*teardown)(unsigned int cpu)) 1403 { 1404 int cpu, ret = 0; 1405 int dyn_state = 0; 1406 1407 if (cpuhp_cb_check(state) || !name) 1408 return -EINVAL; 1409 1410 get_online_cpus(); 1411 1412 /* currently assignments for the ONLINE state are possible */ 1413 if (state == CPUHP_AP_ONLINE_DYN) { 1414 dyn_state = 1; 1415 ret = cpuhp_reserve_state(state); 1416 if (ret < 0) 1417 goto out; 1418 state = ret; 1419 } 1420 1421 cpuhp_store_callbacks(state, name, startup, teardown); 1422 1423 if (!invoke || !startup) 1424 goto out; 1425 1426 /* 1427 * Try to call the startup callback for each present cpu 1428 * depending on the hotplug state of the cpu. 1429 */ 1430 for_each_present_cpu(cpu) { 1431 struct cpuhp_cpu_state *st = per_cpu_ptr(&cpuhp_state, cpu); 1432 int cpustate = st->state; 1433 1434 if (cpustate < state) 1435 continue; 1436 1437 ret = cpuhp_issue_call(cpu, state, startup, true); 1438 if (ret) { 1439 cpuhp_rollback_install(cpu, state, teardown); 1440 cpuhp_store_callbacks(state, NULL, NULL, NULL); 1441 goto out; 1442 } 1443 } 1444 out: 1445 put_online_cpus(); 1446 if (!ret && dyn_state) 1447 return state; 1448 return ret; 1449 } 1450 EXPORT_SYMBOL(__cpuhp_setup_state); 1451 1452 /** 1453 * __cpuhp_remove_state - Remove the callbacks for an hotplug machine state 1454 * @state: The state to remove 1455 * @invoke: If true, the teardown function is invoked for cpus where 1456 * cpu state >= @state 1457 * 1458 * The teardown callback is currently not allowed to fail. Think 1459 * about module removal! 1460 */ 1461 void __cpuhp_remove_state(enum cpuhp_state state, bool invoke) 1462 { 1463 int (*teardown)(unsigned int cpu) = cpuhp_get_teardown_cb(state); 1464 int cpu; 1465 1466 BUG_ON(cpuhp_cb_check(state)); 1467 1468 get_online_cpus(); 1469 1470 if (!invoke || !teardown) 1471 goto remove; 1472 1473 /* 1474 * Call the teardown callback for each present cpu depending 1475 * on the hotplug state of the cpu. This function is not 1476 * allowed to fail currently! 1477 */ 1478 for_each_present_cpu(cpu) { 1479 struct cpuhp_cpu_state *st = per_cpu_ptr(&cpuhp_state, cpu); 1480 int cpustate = st->state; 1481 1482 if (cpustate >= state) 1483 cpuhp_issue_call(cpu, state, teardown, false); 1484 } 1485 remove: 1486 cpuhp_store_callbacks(state, NULL, NULL, NULL); 1487 put_online_cpus(); 1488 } 1489 EXPORT_SYMBOL(__cpuhp_remove_state); 1490 1491 #if defined(CONFIG_SYSFS) && defined(CONFIG_HOTPLUG_CPU) 1492 static ssize_t show_cpuhp_state(struct device *dev, 1493 struct device_attribute *attr, char *buf) 1494 { 1495 struct cpuhp_cpu_state *st = per_cpu_ptr(&cpuhp_state, dev->id); 1496 1497 return sprintf(buf, "%d\n", st->state); 1498 } 1499 static DEVICE_ATTR(state, 0444, show_cpuhp_state, NULL); 1500 1501 static ssize_t write_cpuhp_target(struct device *dev, 1502 struct device_attribute *attr, 1503 const char *buf, size_t count) 1504 { 1505 struct cpuhp_cpu_state *st = per_cpu_ptr(&cpuhp_state, dev->id); 1506 struct cpuhp_step *sp; 1507 int target, ret; 1508 1509 ret = kstrtoint(buf, 10, &target); 1510 if (ret) 1511 return ret; 1512 1513 #ifdef CONFIG_CPU_HOTPLUG_STATE_CONTROL 1514 if (target < CPUHP_OFFLINE || target > CPUHP_ONLINE) 1515 return -EINVAL; 1516 #else 1517 if (target != CPUHP_OFFLINE && target != CPUHP_ONLINE) 1518 return -EINVAL; 1519 #endif 1520 1521 ret = lock_device_hotplug_sysfs(); 1522 if (ret) 1523 return ret; 1524 1525 mutex_lock(&cpuhp_state_mutex); 1526 sp = cpuhp_get_step(target); 1527 ret = !sp->name || sp->cant_stop ? -EINVAL : 0; 1528 mutex_unlock(&cpuhp_state_mutex); 1529 if (ret) 1530 return ret; 1531 1532 if (st->state < target) 1533 ret = do_cpu_up(dev->id, target); 1534 else 1535 ret = do_cpu_down(dev->id, target); 1536 1537 unlock_device_hotplug(); 1538 return ret ? ret : count; 1539 } 1540 1541 static ssize_t show_cpuhp_target(struct device *dev, 1542 struct device_attribute *attr, char *buf) 1543 { 1544 struct cpuhp_cpu_state *st = per_cpu_ptr(&cpuhp_state, dev->id); 1545 1546 return sprintf(buf, "%d\n", st->target); 1547 } 1548 static DEVICE_ATTR(target, 0644, show_cpuhp_target, write_cpuhp_target); 1549 1550 static struct attribute *cpuhp_cpu_attrs[] = { 1551 &dev_attr_state.attr, 1552 &dev_attr_target.attr, 1553 NULL 1554 }; 1555 1556 static struct attribute_group cpuhp_cpu_attr_group = { 1557 .attrs = cpuhp_cpu_attrs, 1558 .name = "hotplug", 1559 NULL 1560 }; 1561 1562 static ssize_t show_cpuhp_states(struct device *dev, 1563 struct device_attribute *attr, char *buf) 1564 { 1565 ssize_t cur, res = 0; 1566 int i; 1567 1568 mutex_lock(&cpuhp_state_mutex); 1569 for (i = CPUHP_OFFLINE; i <= CPUHP_ONLINE; i++) { 1570 struct cpuhp_step *sp = cpuhp_get_step(i); 1571 1572 if (sp->name) { 1573 cur = sprintf(buf, "%3d: %s\n", i, sp->name); 1574 buf += cur; 1575 res += cur; 1576 } 1577 } 1578 mutex_unlock(&cpuhp_state_mutex); 1579 return res; 1580 } 1581 static DEVICE_ATTR(states, 0444, show_cpuhp_states, NULL); 1582 1583 static struct attribute *cpuhp_cpu_root_attrs[] = { 1584 &dev_attr_states.attr, 1585 NULL 1586 }; 1587 1588 static struct attribute_group cpuhp_cpu_root_attr_group = { 1589 .attrs = cpuhp_cpu_root_attrs, 1590 .name = "hotplug", 1591 NULL 1592 }; 1593 1594 static int __init cpuhp_sysfs_init(void) 1595 { 1596 int cpu, ret; 1597 1598 ret = sysfs_create_group(&cpu_subsys.dev_root->kobj, 1599 &cpuhp_cpu_root_attr_group); 1600 if (ret) 1601 return ret; 1602 1603 for_each_possible_cpu(cpu) { 1604 struct device *dev = get_cpu_device(cpu); 1605 1606 if (!dev) 1607 continue; 1608 ret = sysfs_create_group(&dev->kobj, &cpuhp_cpu_attr_group); 1609 if (ret) 1610 return ret; 1611 } 1612 return 0; 1613 } 1614 device_initcall(cpuhp_sysfs_init); 1615 #endif 1616 1617 /* 1618 * cpu_bit_bitmap[] is a special, "compressed" data structure that 1619 * represents all NR_CPUS bits binary values of 1<<nr. 1620 * 1621 * It is used by cpumask_of() to get a constant address to a CPU 1622 * mask value that has a single bit set only. 1623 */ 1624 1625 /* cpu_bit_bitmap[0] is empty - so we can back into it */ 1626 #define MASK_DECLARE_1(x) [x+1][0] = (1UL << (x)) 1627 #define MASK_DECLARE_2(x) MASK_DECLARE_1(x), MASK_DECLARE_1(x+1) 1628 #define MASK_DECLARE_4(x) MASK_DECLARE_2(x), MASK_DECLARE_2(x+2) 1629 #define MASK_DECLARE_8(x) MASK_DECLARE_4(x), MASK_DECLARE_4(x+4) 1630 1631 const unsigned long cpu_bit_bitmap[BITS_PER_LONG+1][BITS_TO_LONGS(NR_CPUS)] = { 1632 1633 MASK_DECLARE_8(0), MASK_DECLARE_8(8), 1634 MASK_DECLARE_8(16), MASK_DECLARE_8(24), 1635 #if BITS_PER_LONG > 32 1636 MASK_DECLARE_8(32), MASK_DECLARE_8(40), 1637 MASK_DECLARE_8(48), MASK_DECLARE_8(56), 1638 #endif 1639 }; 1640 EXPORT_SYMBOL_GPL(cpu_bit_bitmap); 1641 1642 const DECLARE_BITMAP(cpu_all_bits, NR_CPUS) = CPU_BITS_ALL; 1643 EXPORT_SYMBOL(cpu_all_bits); 1644 1645 #ifdef CONFIG_INIT_ALL_POSSIBLE 1646 struct cpumask __cpu_possible_mask __read_mostly 1647 = {CPU_BITS_ALL}; 1648 #else 1649 struct cpumask __cpu_possible_mask __read_mostly; 1650 #endif 1651 EXPORT_SYMBOL(__cpu_possible_mask); 1652 1653 struct cpumask __cpu_online_mask __read_mostly; 1654 EXPORT_SYMBOL(__cpu_online_mask); 1655 1656 struct cpumask __cpu_present_mask __read_mostly; 1657 EXPORT_SYMBOL(__cpu_present_mask); 1658 1659 struct cpumask __cpu_active_mask __read_mostly; 1660 EXPORT_SYMBOL(__cpu_active_mask); 1661 1662 void init_cpu_present(const struct cpumask *src) 1663 { 1664 cpumask_copy(&__cpu_present_mask, src); 1665 } 1666 1667 void init_cpu_possible(const struct cpumask *src) 1668 { 1669 cpumask_copy(&__cpu_possible_mask, src); 1670 } 1671 1672 void init_cpu_online(const struct cpumask *src) 1673 { 1674 cpumask_copy(&__cpu_online_mask, src); 1675 } 1676 1677 /* 1678 * Activate the first processor. 1679 */ 1680 void __init boot_cpu_init(void) 1681 { 1682 int cpu = smp_processor_id(); 1683 1684 /* Mark the boot cpu "present", "online" etc for SMP and UP case */ 1685 set_cpu_online(cpu, true); 1686 set_cpu_active(cpu, true); 1687 set_cpu_present(cpu, true); 1688 set_cpu_possible(cpu, true); 1689 } 1690 1691 /* 1692 * Must be called _AFTER_ setting up the per_cpu areas 1693 */ 1694 void __init boot_cpu_state_init(void) 1695 { 1696 per_cpu_ptr(&cpuhp_state, smp_processor_id())->state = CPUHP_ONLINE; 1697 } 1698