1 /* CPU control. 2 * (C) 2001, 2002, 2003, 2004 Rusty Russell 3 * 4 * This code is licenced under the GPL. 5 */ 6 #include <linux/proc_fs.h> 7 #include <linux/smp.h> 8 #include <linux/init.h> 9 #include <linux/notifier.h> 10 #include <linux/sched/signal.h> 11 #include <linux/sched/hotplug.h> 12 #include <linux/sched/task.h> 13 #include <linux/unistd.h> 14 #include <linux/cpu.h> 15 #include <linux/oom.h> 16 #include <linux/rcupdate.h> 17 #include <linux/export.h> 18 #include <linux/bug.h> 19 #include <linux/kthread.h> 20 #include <linux/stop_machine.h> 21 #include <linux/mutex.h> 22 #include <linux/gfp.h> 23 #include <linux/suspend.h> 24 #include <linux/lockdep.h> 25 #include <linux/tick.h> 26 #include <linux/irq.h> 27 #include <linux/nmi.h> 28 #include <linux/smpboot.h> 29 #include <linux/relay.h> 30 #include <linux/slab.h> 31 #include <linux/percpu-rwsem.h> 32 33 #include <trace/events/power.h> 34 #define CREATE_TRACE_POINTS 35 #include <trace/events/cpuhp.h> 36 37 #include "smpboot.h" 38 39 /** 40 * cpuhp_cpu_state - Per cpu hotplug state storage 41 * @state: The current cpu state 42 * @target: The target state 43 * @thread: Pointer to the hotplug thread 44 * @should_run: Thread should execute 45 * @rollback: Perform a rollback 46 * @single: Single callback invocation 47 * @bringup: Single callback bringup or teardown selector 48 * @cb_state: The state for a single callback (install/uninstall) 49 * @result: Result of the operation 50 * @done_up: Signal completion to the issuer of the task for cpu-up 51 * @done_down: Signal completion to the issuer of the task for cpu-down 52 */ 53 struct cpuhp_cpu_state { 54 enum cpuhp_state state; 55 enum cpuhp_state target; 56 enum cpuhp_state fail; 57 #ifdef CONFIG_SMP 58 struct task_struct *thread; 59 bool should_run; 60 bool rollback; 61 bool single; 62 bool bringup; 63 struct hlist_node *node; 64 struct hlist_node *last; 65 enum cpuhp_state cb_state; 66 int result; 67 struct completion done_up; 68 struct completion done_down; 69 #endif 70 }; 71 72 static DEFINE_PER_CPU(struct cpuhp_cpu_state, cpuhp_state) = { 73 .fail = CPUHP_INVALID, 74 }; 75 76 #if defined(CONFIG_LOCKDEP) && defined(CONFIG_SMP) 77 static struct lockdep_map cpuhp_state_up_map = 78 STATIC_LOCKDEP_MAP_INIT("cpuhp_state-up", &cpuhp_state_up_map); 79 static struct lockdep_map cpuhp_state_down_map = 80 STATIC_LOCKDEP_MAP_INIT("cpuhp_state-down", &cpuhp_state_down_map); 81 82 83 static void inline cpuhp_lock_acquire(bool bringup) 84 { 85 lock_map_acquire(bringup ? &cpuhp_state_up_map : &cpuhp_state_down_map); 86 } 87 88 static void inline cpuhp_lock_release(bool bringup) 89 { 90 lock_map_release(bringup ? &cpuhp_state_up_map : &cpuhp_state_down_map); 91 } 92 #else 93 94 static void inline cpuhp_lock_acquire(bool bringup) { } 95 static void inline cpuhp_lock_release(bool bringup) { } 96 97 #endif 98 99 /** 100 * cpuhp_step - Hotplug state machine step 101 * @name: Name of the step 102 * @startup: Startup function of the step 103 * @teardown: Teardown function of the step 104 * @skip_onerr: Do not invoke the functions on error rollback 105 * Will go away once the notifiers are gone 106 * @cant_stop: Bringup/teardown can't be stopped at this step 107 */ 108 struct cpuhp_step { 109 const char *name; 110 union { 111 int (*single)(unsigned int cpu); 112 int (*multi)(unsigned int cpu, 113 struct hlist_node *node); 114 } startup; 115 union { 116 int (*single)(unsigned int cpu); 117 int (*multi)(unsigned int cpu, 118 struct hlist_node *node); 119 } teardown; 120 struct hlist_head list; 121 bool skip_onerr; 122 bool cant_stop; 123 bool multi_instance; 124 }; 125 126 static DEFINE_MUTEX(cpuhp_state_mutex); 127 static struct cpuhp_step cpuhp_bp_states[]; 128 static struct cpuhp_step cpuhp_ap_states[]; 129 130 static bool cpuhp_is_ap_state(enum cpuhp_state state) 131 { 132 /* 133 * The extra check for CPUHP_TEARDOWN_CPU is only for documentation 134 * purposes as that state is handled explicitly in cpu_down. 135 */ 136 return state > CPUHP_BRINGUP_CPU && state != CPUHP_TEARDOWN_CPU; 137 } 138 139 static struct cpuhp_step *cpuhp_get_step(enum cpuhp_state state) 140 { 141 struct cpuhp_step *sp; 142 143 sp = cpuhp_is_ap_state(state) ? cpuhp_ap_states : cpuhp_bp_states; 144 return sp + state; 145 } 146 147 /** 148 * cpuhp_invoke_callback _ Invoke the callbacks for a given state 149 * @cpu: The cpu for which the callback should be invoked 150 * @state: The state to do callbacks for 151 * @bringup: True if the bringup callback should be invoked 152 * @node: For multi-instance, do a single entry callback for install/remove 153 * @lastp: For multi-instance rollback, remember how far we got 154 * 155 * Called from cpu hotplug and from the state register machinery. 156 */ 157 static int cpuhp_invoke_callback(unsigned int cpu, enum cpuhp_state state, 158 bool bringup, struct hlist_node *node, 159 struct hlist_node **lastp) 160 { 161 struct cpuhp_cpu_state *st = per_cpu_ptr(&cpuhp_state, cpu); 162 struct cpuhp_step *step = cpuhp_get_step(state); 163 int (*cbm)(unsigned int cpu, struct hlist_node *node); 164 int (*cb)(unsigned int cpu); 165 int ret, cnt; 166 167 if (st->fail == state) { 168 st->fail = CPUHP_INVALID; 169 170 if (!(bringup ? step->startup.single : step->teardown.single)) 171 return 0; 172 173 return -EAGAIN; 174 } 175 176 if (!step->multi_instance) { 177 WARN_ON_ONCE(lastp && *lastp); 178 cb = bringup ? step->startup.single : step->teardown.single; 179 if (!cb) 180 return 0; 181 trace_cpuhp_enter(cpu, st->target, state, cb); 182 ret = cb(cpu); 183 trace_cpuhp_exit(cpu, st->state, state, ret); 184 return ret; 185 } 186 cbm = bringup ? step->startup.multi : step->teardown.multi; 187 if (!cbm) 188 return 0; 189 190 /* Single invocation for instance add/remove */ 191 if (node) { 192 WARN_ON_ONCE(lastp && *lastp); 193 trace_cpuhp_multi_enter(cpu, st->target, state, cbm, node); 194 ret = cbm(cpu, node); 195 trace_cpuhp_exit(cpu, st->state, state, ret); 196 return ret; 197 } 198 199 /* State transition. Invoke on all instances */ 200 cnt = 0; 201 hlist_for_each(node, &step->list) { 202 if (lastp && node == *lastp) 203 break; 204 205 trace_cpuhp_multi_enter(cpu, st->target, state, cbm, node); 206 ret = cbm(cpu, node); 207 trace_cpuhp_exit(cpu, st->state, state, ret); 208 if (ret) { 209 if (!lastp) 210 goto err; 211 212 *lastp = node; 213 return ret; 214 } 215 cnt++; 216 } 217 if (lastp) 218 *lastp = NULL; 219 return 0; 220 err: 221 /* Rollback the instances if one failed */ 222 cbm = !bringup ? step->startup.multi : step->teardown.multi; 223 if (!cbm) 224 return ret; 225 226 hlist_for_each(node, &step->list) { 227 if (!cnt--) 228 break; 229 230 trace_cpuhp_multi_enter(cpu, st->target, state, cbm, node); 231 ret = cbm(cpu, node); 232 trace_cpuhp_exit(cpu, st->state, state, ret); 233 /* 234 * Rollback must not fail, 235 */ 236 WARN_ON_ONCE(ret); 237 } 238 return ret; 239 } 240 241 #ifdef CONFIG_SMP 242 static inline void wait_for_ap_thread(struct cpuhp_cpu_state *st, bool bringup) 243 { 244 struct completion *done = bringup ? &st->done_up : &st->done_down; 245 wait_for_completion(done); 246 } 247 248 static inline void complete_ap_thread(struct cpuhp_cpu_state *st, bool bringup) 249 { 250 struct completion *done = bringup ? &st->done_up : &st->done_down; 251 complete(done); 252 } 253 254 /* 255 * The former STARTING/DYING states, ran with IRQs disabled and must not fail. 256 */ 257 static bool cpuhp_is_atomic_state(enum cpuhp_state state) 258 { 259 return CPUHP_AP_IDLE_DEAD <= state && state < CPUHP_AP_ONLINE; 260 } 261 262 /* Serializes the updates to cpu_online_mask, cpu_present_mask */ 263 static DEFINE_MUTEX(cpu_add_remove_lock); 264 bool cpuhp_tasks_frozen; 265 EXPORT_SYMBOL_GPL(cpuhp_tasks_frozen); 266 267 /* 268 * The following two APIs (cpu_maps_update_begin/done) must be used when 269 * attempting to serialize the updates to cpu_online_mask & cpu_present_mask. 270 */ 271 void cpu_maps_update_begin(void) 272 { 273 mutex_lock(&cpu_add_remove_lock); 274 } 275 276 void cpu_maps_update_done(void) 277 { 278 mutex_unlock(&cpu_add_remove_lock); 279 } 280 281 /* 282 * If set, cpu_up and cpu_down will return -EBUSY and do nothing. 283 * Should always be manipulated under cpu_add_remove_lock 284 */ 285 static int cpu_hotplug_disabled; 286 287 #ifdef CONFIG_HOTPLUG_CPU 288 289 DEFINE_STATIC_PERCPU_RWSEM(cpu_hotplug_lock); 290 291 void cpus_read_lock(void) 292 { 293 percpu_down_read(&cpu_hotplug_lock); 294 } 295 EXPORT_SYMBOL_GPL(cpus_read_lock); 296 297 void cpus_read_unlock(void) 298 { 299 percpu_up_read(&cpu_hotplug_lock); 300 } 301 EXPORT_SYMBOL_GPL(cpus_read_unlock); 302 303 void cpus_write_lock(void) 304 { 305 percpu_down_write(&cpu_hotplug_lock); 306 } 307 308 void cpus_write_unlock(void) 309 { 310 percpu_up_write(&cpu_hotplug_lock); 311 } 312 313 void lockdep_assert_cpus_held(void) 314 { 315 percpu_rwsem_assert_held(&cpu_hotplug_lock); 316 } 317 318 /* 319 * Wait for currently running CPU hotplug operations to complete (if any) and 320 * disable future CPU hotplug (from sysfs). The 'cpu_add_remove_lock' protects 321 * the 'cpu_hotplug_disabled' flag. The same lock is also acquired by the 322 * hotplug path before performing hotplug operations. So acquiring that lock 323 * guarantees mutual exclusion from any currently running hotplug operations. 324 */ 325 void cpu_hotplug_disable(void) 326 { 327 cpu_maps_update_begin(); 328 cpu_hotplug_disabled++; 329 cpu_maps_update_done(); 330 } 331 EXPORT_SYMBOL_GPL(cpu_hotplug_disable); 332 333 static void __cpu_hotplug_enable(void) 334 { 335 if (WARN_ONCE(!cpu_hotplug_disabled, "Unbalanced cpu hotplug enable\n")) 336 return; 337 cpu_hotplug_disabled--; 338 } 339 340 void cpu_hotplug_enable(void) 341 { 342 cpu_maps_update_begin(); 343 __cpu_hotplug_enable(); 344 cpu_maps_update_done(); 345 } 346 EXPORT_SYMBOL_GPL(cpu_hotplug_enable); 347 #endif /* CONFIG_HOTPLUG_CPU */ 348 349 static inline enum cpuhp_state 350 cpuhp_set_state(struct cpuhp_cpu_state *st, enum cpuhp_state target) 351 { 352 enum cpuhp_state prev_state = st->state; 353 354 st->rollback = false; 355 st->last = NULL; 356 357 st->target = target; 358 st->single = false; 359 st->bringup = st->state < target; 360 361 return prev_state; 362 } 363 364 static inline void 365 cpuhp_reset_state(struct cpuhp_cpu_state *st, enum cpuhp_state prev_state) 366 { 367 st->rollback = true; 368 369 /* 370 * If we have st->last we need to undo partial multi_instance of this 371 * state first. Otherwise start undo at the previous state. 372 */ 373 if (!st->last) { 374 if (st->bringup) 375 st->state--; 376 else 377 st->state++; 378 } 379 380 st->target = prev_state; 381 st->bringup = !st->bringup; 382 } 383 384 /* Regular hotplug invocation of the AP hotplug thread */ 385 static void __cpuhp_kick_ap(struct cpuhp_cpu_state *st) 386 { 387 if (!st->single && st->state == st->target) 388 return; 389 390 st->result = 0; 391 /* 392 * Make sure the above stores are visible before should_run becomes 393 * true. Paired with the mb() above in cpuhp_thread_fun() 394 */ 395 smp_mb(); 396 st->should_run = true; 397 wake_up_process(st->thread); 398 wait_for_ap_thread(st, st->bringup); 399 } 400 401 static int cpuhp_kick_ap(struct cpuhp_cpu_state *st, enum cpuhp_state target) 402 { 403 enum cpuhp_state prev_state; 404 int ret; 405 406 prev_state = cpuhp_set_state(st, target); 407 __cpuhp_kick_ap(st); 408 if ((ret = st->result)) { 409 cpuhp_reset_state(st, prev_state); 410 __cpuhp_kick_ap(st); 411 } 412 413 return ret; 414 } 415 416 static int bringup_wait_for_ap(unsigned int cpu) 417 { 418 struct cpuhp_cpu_state *st = per_cpu_ptr(&cpuhp_state, cpu); 419 420 /* Wait for the CPU to reach CPUHP_AP_ONLINE_IDLE */ 421 wait_for_ap_thread(st, true); 422 if (WARN_ON_ONCE((!cpu_online(cpu)))) 423 return -ECANCELED; 424 425 /* Unpark the stopper thread and the hotplug thread of the target cpu */ 426 stop_machine_unpark(cpu); 427 kthread_unpark(st->thread); 428 429 if (st->target <= CPUHP_AP_ONLINE_IDLE) 430 return 0; 431 432 return cpuhp_kick_ap(st, st->target); 433 } 434 435 static int bringup_cpu(unsigned int cpu) 436 { 437 struct task_struct *idle = idle_thread_get(cpu); 438 int ret; 439 440 /* 441 * Some architectures have to walk the irq descriptors to 442 * setup the vector space for the cpu which comes online. 443 * Prevent irq alloc/free across the bringup. 444 */ 445 irq_lock_sparse(); 446 447 /* Arch-specific enabling code. */ 448 ret = __cpu_up(cpu, idle); 449 irq_unlock_sparse(); 450 if (ret) 451 return ret; 452 return bringup_wait_for_ap(cpu); 453 } 454 455 /* 456 * Hotplug state machine related functions 457 */ 458 459 static void undo_cpu_up(unsigned int cpu, struct cpuhp_cpu_state *st) 460 { 461 for (st->state--; st->state > st->target; st->state--) { 462 struct cpuhp_step *step = cpuhp_get_step(st->state); 463 464 if (!step->skip_onerr) 465 cpuhp_invoke_callback(cpu, st->state, false, NULL, NULL); 466 } 467 } 468 469 static int cpuhp_up_callbacks(unsigned int cpu, struct cpuhp_cpu_state *st, 470 enum cpuhp_state target) 471 { 472 enum cpuhp_state prev_state = st->state; 473 int ret = 0; 474 475 while (st->state < target) { 476 st->state++; 477 ret = cpuhp_invoke_callback(cpu, st->state, true, NULL, NULL); 478 if (ret) { 479 st->target = prev_state; 480 undo_cpu_up(cpu, st); 481 break; 482 } 483 } 484 return ret; 485 } 486 487 /* 488 * The cpu hotplug threads manage the bringup and teardown of the cpus 489 */ 490 static void cpuhp_create(unsigned int cpu) 491 { 492 struct cpuhp_cpu_state *st = per_cpu_ptr(&cpuhp_state, cpu); 493 494 init_completion(&st->done_up); 495 init_completion(&st->done_down); 496 } 497 498 static int cpuhp_should_run(unsigned int cpu) 499 { 500 struct cpuhp_cpu_state *st = this_cpu_ptr(&cpuhp_state); 501 502 return st->should_run; 503 } 504 505 /* 506 * Execute teardown/startup callbacks on the plugged cpu. Also used to invoke 507 * callbacks when a state gets [un]installed at runtime. 508 * 509 * Each invocation of this function by the smpboot thread does a single AP 510 * state callback. 511 * 512 * It has 3 modes of operation: 513 * - single: runs st->cb_state 514 * - up: runs ++st->state, while st->state < st->target 515 * - down: runs st->state--, while st->state > st->target 516 * 517 * When complete or on error, should_run is cleared and the completion is fired. 518 */ 519 static void cpuhp_thread_fun(unsigned int cpu) 520 { 521 struct cpuhp_cpu_state *st = this_cpu_ptr(&cpuhp_state); 522 bool bringup = st->bringup; 523 enum cpuhp_state state; 524 525 /* 526 * ACQUIRE for the cpuhp_should_run() load of ->should_run. Ensures 527 * that if we see ->should_run we also see the rest of the state. 528 */ 529 smp_mb(); 530 531 if (WARN_ON_ONCE(!st->should_run)) 532 return; 533 534 cpuhp_lock_acquire(bringup); 535 536 if (st->single) { 537 state = st->cb_state; 538 st->should_run = false; 539 } else { 540 if (bringup) { 541 st->state++; 542 state = st->state; 543 st->should_run = (st->state < st->target); 544 WARN_ON_ONCE(st->state > st->target); 545 } else { 546 state = st->state; 547 st->state--; 548 st->should_run = (st->state > st->target); 549 WARN_ON_ONCE(st->state < st->target); 550 } 551 } 552 553 WARN_ON_ONCE(!cpuhp_is_ap_state(state)); 554 555 if (st->rollback) { 556 struct cpuhp_step *step = cpuhp_get_step(state); 557 if (step->skip_onerr) 558 goto next; 559 } 560 561 if (cpuhp_is_atomic_state(state)) { 562 local_irq_disable(); 563 st->result = cpuhp_invoke_callback(cpu, state, bringup, st->node, &st->last); 564 local_irq_enable(); 565 566 /* 567 * STARTING/DYING must not fail! 568 */ 569 WARN_ON_ONCE(st->result); 570 } else { 571 st->result = cpuhp_invoke_callback(cpu, state, bringup, st->node, &st->last); 572 } 573 574 if (st->result) { 575 /* 576 * If we fail on a rollback, we're up a creek without no 577 * paddle, no way forward, no way back. We loose, thanks for 578 * playing. 579 */ 580 WARN_ON_ONCE(st->rollback); 581 st->should_run = false; 582 } 583 584 next: 585 cpuhp_lock_release(bringup); 586 587 if (!st->should_run) 588 complete_ap_thread(st, bringup); 589 } 590 591 /* Invoke a single callback on a remote cpu */ 592 static int 593 cpuhp_invoke_ap_callback(int cpu, enum cpuhp_state state, bool bringup, 594 struct hlist_node *node) 595 { 596 struct cpuhp_cpu_state *st = per_cpu_ptr(&cpuhp_state, cpu); 597 int ret; 598 599 if (!cpu_online(cpu)) 600 return 0; 601 602 cpuhp_lock_acquire(false); 603 cpuhp_lock_release(false); 604 605 cpuhp_lock_acquire(true); 606 cpuhp_lock_release(true); 607 608 /* 609 * If we are up and running, use the hotplug thread. For early calls 610 * we invoke the thread function directly. 611 */ 612 if (!st->thread) 613 return cpuhp_invoke_callback(cpu, state, bringup, node, NULL); 614 615 st->rollback = false; 616 st->last = NULL; 617 618 st->node = node; 619 st->bringup = bringup; 620 st->cb_state = state; 621 st->single = true; 622 623 __cpuhp_kick_ap(st); 624 625 /* 626 * If we failed and did a partial, do a rollback. 627 */ 628 if ((ret = st->result) && st->last) { 629 st->rollback = true; 630 st->bringup = !bringup; 631 632 __cpuhp_kick_ap(st); 633 } 634 635 /* 636 * Clean up the leftovers so the next hotplug operation wont use stale 637 * data. 638 */ 639 st->node = st->last = NULL; 640 return ret; 641 } 642 643 static int cpuhp_kick_ap_work(unsigned int cpu) 644 { 645 struct cpuhp_cpu_state *st = per_cpu_ptr(&cpuhp_state, cpu); 646 enum cpuhp_state prev_state = st->state; 647 int ret; 648 649 cpuhp_lock_acquire(false); 650 cpuhp_lock_release(false); 651 652 cpuhp_lock_acquire(true); 653 cpuhp_lock_release(true); 654 655 trace_cpuhp_enter(cpu, st->target, prev_state, cpuhp_kick_ap_work); 656 ret = cpuhp_kick_ap(st, st->target); 657 trace_cpuhp_exit(cpu, st->state, prev_state, ret); 658 659 return ret; 660 } 661 662 static struct smp_hotplug_thread cpuhp_threads = { 663 .store = &cpuhp_state.thread, 664 .create = &cpuhp_create, 665 .thread_should_run = cpuhp_should_run, 666 .thread_fn = cpuhp_thread_fun, 667 .thread_comm = "cpuhp/%u", 668 .selfparking = true, 669 }; 670 671 void __init cpuhp_threads_init(void) 672 { 673 BUG_ON(smpboot_register_percpu_thread(&cpuhp_threads)); 674 kthread_unpark(this_cpu_read(cpuhp_state.thread)); 675 } 676 677 #ifdef CONFIG_HOTPLUG_CPU 678 /** 679 * clear_tasks_mm_cpumask - Safely clear tasks' mm_cpumask for a CPU 680 * @cpu: a CPU id 681 * 682 * This function walks all processes, finds a valid mm struct for each one and 683 * then clears a corresponding bit in mm's cpumask. While this all sounds 684 * trivial, there are various non-obvious corner cases, which this function 685 * tries to solve in a safe manner. 686 * 687 * Also note that the function uses a somewhat relaxed locking scheme, so it may 688 * be called only for an already offlined CPU. 689 */ 690 void clear_tasks_mm_cpumask(int cpu) 691 { 692 struct task_struct *p; 693 694 /* 695 * This function is called after the cpu is taken down and marked 696 * offline, so its not like new tasks will ever get this cpu set in 697 * their mm mask. -- Peter Zijlstra 698 * Thus, we may use rcu_read_lock() here, instead of grabbing 699 * full-fledged tasklist_lock. 700 */ 701 WARN_ON(cpu_online(cpu)); 702 rcu_read_lock(); 703 for_each_process(p) { 704 struct task_struct *t; 705 706 /* 707 * Main thread might exit, but other threads may still have 708 * a valid mm. Find one. 709 */ 710 t = find_lock_task_mm(p); 711 if (!t) 712 continue; 713 cpumask_clear_cpu(cpu, mm_cpumask(t->mm)); 714 task_unlock(t); 715 } 716 rcu_read_unlock(); 717 } 718 719 /* Take this CPU down. */ 720 static int take_cpu_down(void *_param) 721 { 722 struct cpuhp_cpu_state *st = this_cpu_ptr(&cpuhp_state); 723 enum cpuhp_state target = max((int)st->target, CPUHP_AP_OFFLINE); 724 int err, cpu = smp_processor_id(); 725 int ret; 726 727 /* Ensure this CPU doesn't handle any more interrupts. */ 728 err = __cpu_disable(); 729 if (err < 0) 730 return err; 731 732 /* 733 * We get here while we are in CPUHP_TEARDOWN_CPU state and we must not 734 * do this step again. 735 */ 736 WARN_ON(st->state != CPUHP_TEARDOWN_CPU); 737 st->state--; 738 /* Invoke the former CPU_DYING callbacks */ 739 for (; st->state > target; st->state--) { 740 ret = cpuhp_invoke_callback(cpu, st->state, false, NULL, NULL); 741 /* 742 * DYING must not fail! 743 */ 744 WARN_ON_ONCE(ret); 745 } 746 747 /* Give up timekeeping duties */ 748 tick_handover_do_timer(); 749 /* Park the stopper thread */ 750 stop_machine_park(cpu); 751 return 0; 752 } 753 754 static int takedown_cpu(unsigned int cpu) 755 { 756 struct cpuhp_cpu_state *st = per_cpu_ptr(&cpuhp_state, cpu); 757 int err; 758 759 /* Park the smpboot threads */ 760 kthread_park(per_cpu_ptr(&cpuhp_state, cpu)->thread); 761 smpboot_park_threads(cpu); 762 763 /* 764 * Prevent irq alloc/free while the dying cpu reorganizes the 765 * interrupt affinities. 766 */ 767 irq_lock_sparse(); 768 769 /* 770 * So now all preempt/rcu users must observe !cpu_active(). 771 */ 772 err = stop_machine_cpuslocked(take_cpu_down, NULL, cpumask_of(cpu)); 773 if (err) { 774 /* CPU refused to die */ 775 irq_unlock_sparse(); 776 /* Unpark the hotplug thread so we can rollback there */ 777 kthread_unpark(per_cpu_ptr(&cpuhp_state, cpu)->thread); 778 return err; 779 } 780 BUG_ON(cpu_online(cpu)); 781 782 /* 783 * The CPUHP_AP_SCHED_MIGRATE_DYING callback will have removed all 784 * runnable tasks from the cpu, there's only the idle task left now 785 * that the migration thread is done doing the stop_machine thing. 786 * 787 * Wait for the stop thread to go away. 788 */ 789 wait_for_ap_thread(st, false); 790 BUG_ON(st->state != CPUHP_AP_IDLE_DEAD); 791 792 /* Interrupts are moved away from the dying cpu, reenable alloc/free */ 793 irq_unlock_sparse(); 794 795 hotplug_cpu__broadcast_tick_pull(cpu); 796 /* This actually kills the CPU. */ 797 __cpu_die(cpu); 798 799 tick_cleanup_dead_cpu(cpu); 800 rcutree_migrate_callbacks(cpu); 801 return 0; 802 } 803 804 static void cpuhp_complete_idle_dead(void *arg) 805 { 806 struct cpuhp_cpu_state *st = arg; 807 808 complete_ap_thread(st, false); 809 } 810 811 void cpuhp_report_idle_dead(void) 812 { 813 struct cpuhp_cpu_state *st = this_cpu_ptr(&cpuhp_state); 814 815 BUG_ON(st->state != CPUHP_AP_OFFLINE); 816 rcu_report_dead(smp_processor_id()); 817 st->state = CPUHP_AP_IDLE_DEAD; 818 /* 819 * We cannot call complete after rcu_report_dead() so we delegate it 820 * to an online cpu. 821 */ 822 smp_call_function_single(cpumask_first(cpu_online_mask), 823 cpuhp_complete_idle_dead, st, 0); 824 } 825 826 static void undo_cpu_down(unsigned int cpu, struct cpuhp_cpu_state *st) 827 { 828 for (st->state++; st->state < st->target; st->state++) { 829 struct cpuhp_step *step = cpuhp_get_step(st->state); 830 831 if (!step->skip_onerr) 832 cpuhp_invoke_callback(cpu, st->state, true, NULL, NULL); 833 } 834 } 835 836 static int cpuhp_down_callbacks(unsigned int cpu, struct cpuhp_cpu_state *st, 837 enum cpuhp_state target) 838 { 839 enum cpuhp_state prev_state = st->state; 840 int ret = 0; 841 842 for (; st->state > target; st->state--) { 843 ret = cpuhp_invoke_callback(cpu, st->state, false, NULL, NULL); 844 if (ret) { 845 st->target = prev_state; 846 undo_cpu_down(cpu, st); 847 break; 848 } 849 } 850 return ret; 851 } 852 853 /* Requires cpu_add_remove_lock to be held */ 854 static int __ref _cpu_down(unsigned int cpu, int tasks_frozen, 855 enum cpuhp_state target) 856 { 857 struct cpuhp_cpu_state *st = per_cpu_ptr(&cpuhp_state, cpu); 858 int prev_state, ret = 0; 859 860 if (num_online_cpus() == 1) 861 return -EBUSY; 862 863 if (!cpu_present(cpu)) 864 return -EINVAL; 865 866 cpus_write_lock(); 867 868 cpuhp_tasks_frozen = tasks_frozen; 869 870 prev_state = cpuhp_set_state(st, target); 871 /* 872 * If the current CPU state is in the range of the AP hotplug thread, 873 * then we need to kick the thread. 874 */ 875 if (st->state > CPUHP_TEARDOWN_CPU) { 876 st->target = max((int)target, CPUHP_TEARDOWN_CPU); 877 ret = cpuhp_kick_ap_work(cpu); 878 /* 879 * The AP side has done the error rollback already. Just 880 * return the error code.. 881 */ 882 if (ret) 883 goto out; 884 885 /* 886 * We might have stopped still in the range of the AP hotplug 887 * thread. Nothing to do anymore. 888 */ 889 if (st->state > CPUHP_TEARDOWN_CPU) 890 goto out; 891 892 st->target = target; 893 } 894 /* 895 * The AP brought itself down to CPUHP_TEARDOWN_CPU. So we need 896 * to do the further cleanups. 897 */ 898 ret = cpuhp_down_callbacks(cpu, st, target); 899 if (ret && st->state > CPUHP_TEARDOWN_CPU && st->state < prev_state) { 900 cpuhp_reset_state(st, prev_state); 901 __cpuhp_kick_ap(st); 902 } 903 904 out: 905 cpus_write_unlock(); 906 /* 907 * Do post unplug cleanup. This is still protected against 908 * concurrent CPU hotplug via cpu_add_remove_lock. 909 */ 910 lockup_detector_cleanup(); 911 return ret; 912 } 913 914 static int do_cpu_down(unsigned int cpu, enum cpuhp_state target) 915 { 916 int err; 917 918 cpu_maps_update_begin(); 919 920 if (cpu_hotplug_disabled) { 921 err = -EBUSY; 922 goto out; 923 } 924 925 err = _cpu_down(cpu, 0, target); 926 927 out: 928 cpu_maps_update_done(); 929 return err; 930 } 931 932 int cpu_down(unsigned int cpu) 933 { 934 return do_cpu_down(cpu, CPUHP_OFFLINE); 935 } 936 EXPORT_SYMBOL(cpu_down); 937 938 #else 939 #define takedown_cpu NULL 940 #endif /*CONFIG_HOTPLUG_CPU*/ 941 942 /** 943 * notify_cpu_starting(cpu) - Invoke the callbacks on the starting CPU 944 * @cpu: cpu that just started 945 * 946 * It must be called by the arch code on the new cpu, before the new cpu 947 * enables interrupts and before the "boot" cpu returns from __cpu_up(). 948 */ 949 void notify_cpu_starting(unsigned int cpu) 950 { 951 struct cpuhp_cpu_state *st = per_cpu_ptr(&cpuhp_state, cpu); 952 enum cpuhp_state target = min((int)st->target, CPUHP_AP_ONLINE); 953 int ret; 954 955 rcu_cpu_starting(cpu); /* Enables RCU usage on this CPU. */ 956 while (st->state < target) { 957 st->state++; 958 ret = cpuhp_invoke_callback(cpu, st->state, true, NULL, NULL); 959 /* 960 * STARTING must not fail! 961 */ 962 WARN_ON_ONCE(ret); 963 } 964 } 965 966 /* 967 * Called from the idle task. Wake up the controlling task which brings the 968 * stopper and the hotplug thread of the upcoming CPU up and then delegates 969 * the rest of the online bringup to the hotplug thread. 970 */ 971 void cpuhp_online_idle(enum cpuhp_state state) 972 { 973 struct cpuhp_cpu_state *st = this_cpu_ptr(&cpuhp_state); 974 975 /* Happens for the boot cpu */ 976 if (state != CPUHP_AP_ONLINE_IDLE) 977 return; 978 979 st->state = CPUHP_AP_ONLINE_IDLE; 980 complete_ap_thread(st, true); 981 } 982 983 /* Requires cpu_add_remove_lock to be held */ 984 static int _cpu_up(unsigned int cpu, int tasks_frozen, enum cpuhp_state target) 985 { 986 struct cpuhp_cpu_state *st = per_cpu_ptr(&cpuhp_state, cpu); 987 struct task_struct *idle; 988 int ret = 0; 989 990 cpus_write_lock(); 991 992 if (!cpu_present(cpu)) { 993 ret = -EINVAL; 994 goto out; 995 } 996 997 /* 998 * The caller of do_cpu_up might have raced with another 999 * caller. Ignore it for now. 1000 */ 1001 if (st->state >= target) 1002 goto out; 1003 1004 if (st->state == CPUHP_OFFLINE) { 1005 /* Let it fail before we try to bring the cpu up */ 1006 idle = idle_thread_get(cpu); 1007 if (IS_ERR(idle)) { 1008 ret = PTR_ERR(idle); 1009 goto out; 1010 } 1011 } 1012 1013 cpuhp_tasks_frozen = tasks_frozen; 1014 1015 cpuhp_set_state(st, target); 1016 /* 1017 * If the current CPU state is in the range of the AP hotplug thread, 1018 * then we need to kick the thread once more. 1019 */ 1020 if (st->state > CPUHP_BRINGUP_CPU) { 1021 ret = cpuhp_kick_ap_work(cpu); 1022 /* 1023 * The AP side has done the error rollback already. Just 1024 * return the error code.. 1025 */ 1026 if (ret) 1027 goto out; 1028 } 1029 1030 /* 1031 * Try to reach the target state. We max out on the BP at 1032 * CPUHP_BRINGUP_CPU. After that the AP hotplug thread is 1033 * responsible for bringing it up to the target state. 1034 */ 1035 target = min((int)target, CPUHP_BRINGUP_CPU); 1036 ret = cpuhp_up_callbacks(cpu, st, target); 1037 out: 1038 cpus_write_unlock(); 1039 return ret; 1040 } 1041 1042 static int do_cpu_up(unsigned int cpu, enum cpuhp_state target) 1043 { 1044 int err = 0; 1045 1046 if (!cpu_possible(cpu)) { 1047 pr_err("can't online cpu %d because it is not configured as may-hotadd at boot time\n", 1048 cpu); 1049 #if defined(CONFIG_IA64) 1050 pr_err("please check additional_cpus= boot parameter\n"); 1051 #endif 1052 return -EINVAL; 1053 } 1054 1055 err = try_online_node(cpu_to_node(cpu)); 1056 if (err) 1057 return err; 1058 1059 cpu_maps_update_begin(); 1060 1061 if (cpu_hotplug_disabled) { 1062 err = -EBUSY; 1063 goto out; 1064 } 1065 1066 err = _cpu_up(cpu, 0, target); 1067 out: 1068 cpu_maps_update_done(); 1069 return err; 1070 } 1071 1072 int cpu_up(unsigned int cpu) 1073 { 1074 return do_cpu_up(cpu, CPUHP_ONLINE); 1075 } 1076 EXPORT_SYMBOL_GPL(cpu_up); 1077 1078 #ifdef CONFIG_PM_SLEEP_SMP 1079 static cpumask_var_t frozen_cpus; 1080 1081 int freeze_secondary_cpus(int primary) 1082 { 1083 int cpu, error = 0; 1084 1085 cpu_maps_update_begin(); 1086 if (!cpu_online(primary)) 1087 primary = cpumask_first(cpu_online_mask); 1088 /* 1089 * We take down all of the non-boot CPUs in one shot to avoid races 1090 * with the userspace trying to use the CPU hotplug at the same time 1091 */ 1092 cpumask_clear(frozen_cpus); 1093 1094 pr_info("Disabling non-boot CPUs ...\n"); 1095 for_each_online_cpu(cpu) { 1096 if (cpu == primary) 1097 continue; 1098 trace_suspend_resume(TPS("CPU_OFF"), cpu, true); 1099 error = _cpu_down(cpu, 1, CPUHP_OFFLINE); 1100 trace_suspend_resume(TPS("CPU_OFF"), cpu, false); 1101 if (!error) 1102 cpumask_set_cpu(cpu, frozen_cpus); 1103 else { 1104 pr_err("Error taking CPU%d down: %d\n", cpu, error); 1105 break; 1106 } 1107 } 1108 1109 if (!error) 1110 BUG_ON(num_online_cpus() > 1); 1111 else 1112 pr_err("Non-boot CPUs are not disabled\n"); 1113 1114 /* 1115 * Make sure the CPUs won't be enabled by someone else. We need to do 1116 * this even in case of failure as all disable_nonboot_cpus() users are 1117 * supposed to do enable_nonboot_cpus() on the failure path. 1118 */ 1119 cpu_hotplug_disabled++; 1120 1121 cpu_maps_update_done(); 1122 return error; 1123 } 1124 1125 void __weak arch_enable_nonboot_cpus_begin(void) 1126 { 1127 } 1128 1129 void __weak arch_enable_nonboot_cpus_end(void) 1130 { 1131 } 1132 1133 void enable_nonboot_cpus(void) 1134 { 1135 int cpu, error; 1136 1137 /* Allow everyone to use the CPU hotplug again */ 1138 cpu_maps_update_begin(); 1139 __cpu_hotplug_enable(); 1140 if (cpumask_empty(frozen_cpus)) 1141 goto out; 1142 1143 pr_info("Enabling non-boot CPUs ...\n"); 1144 1145 arch_enable_nonboot_cpus_begin(); 1146 1147 for_each_cpu(cpu, frozen_cpus) { 1148 trace_suspend_resume(TPS("CPU_ON"), cpu, true); 1149 error = _cpu_up(cpu, 1, CPUHP_ONLINE); 1150 trace_suspend_resume(TPS("CPU_ON"), cpu, false); 1151 if (!error) { 1152 pr_info("CPU%d is up\n", cpu); 1153 continue; 1154 } 1155 pr_warn("Error taking CPU%d up: %d\n", cpu, error); 1156 } 1157 1158 arch_enable_nonboot_cpus_end(); 1159 1160 cpumask_clear(frozen_cpus); 1161 out: 1162 cpu_maps_update_done(); 1163 } 1164 1165 static int __init alloc_frozen_cpus(void) 1166 { 1167 if (!alloc_cpumask_var(&frozen_cpus, GFP_KERNEL|__GFP_ZERO)) 1168 return -ENOMEM; 1169 return 0; 1170 } 1171 core_initcall(alloc_frozen_cpus); 1172 1173 /* 1174 * When callbacks for CPU hotplug notifications are being executed, we must 1175 * ensure that the state of the system with respect to the tasks being frozen 1176 * or not, as reported by the notification, remains unchanged *throughout the 1177 * duration* of the execution of the callbacks. 1178 * Hence we need to prevent the freezer from racing with regular CPU hotplug. 1179 * 1180 * This synchronization is implemented by mutually excluding regular CPU 1181 * hotplug and Suspend/Hibernate call paths by hooking onto the Suspend/ 1182 * Hibernate notifications. 1183 */ 1184 static int 1185 cpu_hotplug_pm_callback(struct notifier_block *nb, 1186 unsigned long action, void *ptr) 1187 { 1188 switch (action) { 1189 1190 case PM_SUSPEND_PREPARE: 1191 case PM_HIBERNATION_PREPARE: 1192 cpu_hotplug_disable(); 1193 break; 1194 1195 case PM_POST_SUSPEND: 1196 case PM_POST_HIBERNATION: 1197 cpu_hotplug_enable(); 1198 break; 1199 1200 default: 1201 return NOTIFY_DONE; 1202 } 1203 1204 return NOTIFY_OK; 1205 } 1206 1207 1208 static int __init cpu_hotplug_pm_sync_init(void) 1209 { 1210 /* 1211 * cpu_hotplug_pm_callback has higher priority than x86 1212 * bsp_pm_callback which depends on cpu_hotplug_pm_callback 1213 * to disable cpu hotplug to avoid cpu hotplug race. 1214 */ 1215 pm_notifier(cpu_hotplug_pm_callback, 0); 1216 return 0; 1217 } 1218 core_initcall(cpu_hotplug_pm_sync_init); 1219 1220 #endif /* CONFIG_PM_SLEEP_SMP */ 1221 1222 int __boot_cpu_id; 1223 1224 #endif /* CONFIG_SMP */ 1225 1226 /* Boot processor state steps */ 1227 static struct cpuhp_step cpuhp_bp_states[] = { 1228 [CPUHP_OFFLINE] = { 1229 .name = "offline", 1230 .startup.single = NULL, 1231 .teardown.single = NULL, 1232 }, 1233 #ifdef CONFIG_SMP 1234 [CPUHP_CREATE_THREADS]= { 1235 .name = "threads:prepare", 1236 .startup.single = smpboot_create_threads, 1237 .teardown.single = NULL, 1238 .cant_stop = true, 1239 }, 1240 [CPUHP_PERF_PREPARE] = { 1241 .name = "perf:prepare", 1242 .startup.single = perf_event_init_cpu, 1243 .teardown.single = perf_event_exit_cpu, 1244 }, 1245 [CPUHP_WORKQUEUE_PREP] = { 1246 .name = "workqueue:prepare", 1247 .startup.single = workqueue_prepare_cpu, 1248 .teardown.single = NULL, 1249 }, 1250 [CPUHP_HRTIMERS_PREPARE] = { 1251 .name = "hrtimers:prepare", 1252 .startup.single = hrtimers_prepare_cpu, 1253 .teardown.single = hrtimers_dead_cpu, 1254 }, 1255 [CPUHP_SMPCFD_PREPARE] = { 1256 .name = "smpcfd:prepare", 1257 .startup.single = smpcfd_prepare_cpu, 1258 .teardown.single = smpcfd_dead_cpu, 1259 }, 1260 [CPUHP_RELAY_PREPARE] = { 1261 .name = "relay:prepare", 1262 .startup.single = relay_prepare_cpu, 1263 .teardown.single = NULL, 1264 }, 1265 [CPUHP_SLAB_PREPARE] = { 1266 .name = "slab:prepare", 1267 .startup.single = slab_prepare_cpu, 1268 .teardown.single = slab_dead_cpu, 1269 }, 1270 [CPUHP_RCUTREE_PREP] = { 1271 .name = "RCU/tree:prepare", 1272 .startup.single = rcutree_prepare_cpu, 1273 .teardown.single = rcutree_dead_cpu, 1274 }, 1275 /* 1276 * On the tear-down path, timers_dead_cpu() must be invoked 1277 * before blk_mq_queue_reinit_notify() from notify_dead(), 1278 * otherwise a RCU stall occurs. 1279 */ 1280 [CPUHP_TIMERS_DEAD] = { 1281 .name = "timers:dead", 1282 .startup.single = NULL, 1283 .teardown.single = timers_dead_cpu, 1284 }, 1285 /* Kicks the plugged cpu into life */ 1286 [CPUHP_BRINGUP_CPU] = { 1287 .name = "cpu:bringup", 1288 .startup.single = bringup_cpu, 1289 .teardown.single = NULL, 1290 .cant_stop = true, 1291 }, 1292 [CPUHP_AP_SMPCFD_DYING] = { 1293 .name = "smpcfd:dying", 1294 .startup.single = NULL, 1295 .teardown.single = smpcfd_dying_cpu, 1296 }, 1297 /* 1298 * Handled on controll processor until the plugged processor manages 1299 * this itself. 1300 */ 1301 [CPUHP_TEARDOWN_CPU] = { 1302 .name = "cpu:teardown", 1303 .startup.single = NULL, 1304 .teardown.single = takedown_cpu, 1305 .cant_stop = true, 1306 }, 1307 #else 1308 [CPUHP_BRINGUP_CPU] = { }, 1309 #endif 1310 }; 1311 1312 /* Application processor state steps */ 1313 static struct cpuhp_step cpuhp_ap_states[] = { 1314 #ifdef CONFIG_SMP 1315 /* Final state before CPU kills itself */ 1316 [CPUHP_AP_IDLE_DEAD] = { 1317 .name = "idle:dead", 1318 }, 1319 /* 1320 * Last state before CPU enters the idle loop to die. Transient state 1321 * for synchronization. 1322 */ 1323 [CPUHP_AP_OFFLINE] = { 1324 .name = "ap:offline", 1325 .cant_stop = true, 1326 }, 1327 /* First state is scheduler control. Interrupts are disabled */ 1328 [CPUHP_AP_SCHED_STARTING] = { 1329 .name = "sched:starting", 1330 .startup.single = sched_cpu_starting, 1331 .teardown.single = sched_cpu_dying, 1332 }, 1333 [CPUHP_AP_RCUTREE_DYING] = { 1334 .name = "RCU/tree:dying", 1335 .startup.single = NULL, 1336 .teardown.single = rcutree_dying_cpu, 1337 }, 1338 /* Entry state on starting. Interrupts enabled from here on. Transient 1339 * state for synchronsization */ 1340 [CPUHP_AP_ONLINE] = { 1341 .name = "ap:online", 1342 }, 1343 /* Handle smpboot threads park/unpark */ 1344 [CPUHP_AP_SMPBOOT_THREADS] = { 1345 .name = "smpboot/threads:online", 1346 .startup.single = smpboot_unpark_threads, 1347 .teardown.single = NULL, 1348 }, 1349 [CPUHP_AP_IRQ_AFFINITY_ONLINE] = { 1350 .name = "irq/affinity:online", 1351 .startup.single = irq_affinity_online_cpu, 1352 .teardown.single = NULL, 1353 }, 1354 [CPUHP_AP_PERF_ONLINE] = { 1355 .name = "perf:online", 1356 .startup.single = perf_event_init_cpu, 1357 .teardown.single = perf_event_exit_cpu, 1358 }, 1359 [CPUHP_AP_WORKQUEUE_ONLINE] = { 1360 .name = "workqueue:online", 1361 .startup.single = workqueue_online_cpu, 1362 .teardown.single = workqueue_offline_cpu, 1363 }, 1364 [CPUHP_AP_RCUTREE_ONLINE] = { 1365 .name = "RCU/tree:online", 1366 .startup.single = rcutree_online_cpu, 1367 .teardown.single = rcutree_offline_cpu, 1368 }, 1369 #endif 1370 /* 1371 * The dynamically registered state space is here 1372 */ 1373 1374 #ifdef CONFIG_SMP 1375 /* Last state is scheduler control setting the cpu active */ 1376 [CPUHP_AP_ACTIVE] = { 1377 .name = "sched:active", 1378 .startup.single = sched_cpu_activate, 1379 .teardown.single = sched_cpu_deactivate, 1380 }, 1381 #endif 1382 1383 /* CPU is fully up and running. */ 1384 [CPUHP_ONLINE] = { 1385 .name = "online", 1386 .startup.single = NULL, 1387 .teardown.single = NULL, 1388 }, 1389 }; 1390 1391 /* Sanity check for callbacks */ 1392 static int cpuhp_cb_check(enum cpuhp_state state) 1393 { 1394 if (state <= CPUHP_OFFLINE || state >= CPUHP_ONLINE) 1395 return -EINVAL; 1396 return 0; 1397 } 1398 1399 /* 1400 * Returns a free for dynamic slot assignment of the Online state. The states 1401 * are protected by the cpuhp_slot_states mutex and an empty slot is identified 1402 * by having no name assigned. 1403 */ 1404 static int cpuhp_reserve_state(enum cpuhp_state state) 1405 { 1406 enum cpuhp_state i, end; 1407 struct cpuhp_step *step; 1408 1409 switch (state) { 1410 case CPUHP_AP_ONLINE_DYN: 1411 step = cpuhp_ap_states + CPUHP_AP_ONLINE_DYN; 1412 end = CPUHP_AP_ONLINE_DYN_END; 1413 break; 1414 case CPUHP_BP_PREPARE_DYN: 1415 step = cpuhp_bp_states + CPUHP_BP_PREPARE_DYN; 1416 end = CPUHP_BP_PREPARE_DYN_END; 1417 break; 1418 default: 1419 return -EINVAL; 1420 } 1421 1422 for (i = state; i <= end; i++, step++) { 1423 if (!step->name) 1424 return i; 1425 } 1426 WARN(1, "No more dynamic states available for CPU hotplug\n"); 1427 return -ENOSPC; 1428 } 1429 1430 static int cpuhp_store_callbacks(enum cpuhp_state state, const char *name, 1431 int (*startup)(unsigned int cpu), 1432 int (*teardown)(unsigned int cpu), 1433 bool multi_instance) 1434 { 1435 /* (Un)Install the callbacks for further cpu hotplug operations */ 1436 struct cpuhp_step *sp; 1437 int ret = 0; 1438 1439 /* 1440 * If name is NULL, then the state gets removed. 1441 * 1442 * CPUHP_AP_ONLINE_DYN and CPUHP_BP_PREPARE_DYN are handed out on 1443 * the first allocation from these dynamic ranges, so the removal 1444 * would trigger a new allocation and clear the wrong (already 1445 * empty) state, leaving the callbacks of the to be cleared state 1446 * dangling, which causes wreckage on the next hotplug operation. 1447 */ 1448 if (name && (state == CPUHP_AP_ONLINE_DYN || 1449 state == CPUHP_BP_PREPARE_DYN)) { 1450 ret = cpuhp_reserve_state(state); 1451 if (ret < 0) 1452 return ret; 1453 state = ret; 1454 } 1455 sp = cpuhp_get_step(state); 1456 if (name && sp->name) 1457 return -EBUSY; 1458 1459 sp->startup.single = startup; 1460 sp->teardown.single = teardown; 1461 sp->name = name; 1462 sp->multi_instance = multi_instance; 1463 INIT_HLIST_HEAD(&sp->list); 1464 return ret; 1465 } 1466 1467 static void *cpuhp_get_teardown_cb(enum cpuhp_state state) 1468 { 1469 return cpuhp_get_step(state)->teardown.single; 1470 } 1471 1472 /* 1473 * Call the startup/teardown function for a step either on the AP or 1474 * on the current CPU. 1475 */ 1476 static int cpuhp_issue_call(int cpu, enum cpuhp_state state, bool bringup, 1477 struct hlist_node *node) 1478 { 1479 struct cpuhp_step *sp = cpuhp_get_step(state); 1480 int ret; 1481 1482 /* 1483 * If there's nothing to do, we done. 1484 * Relies on the union for multi_instance. 1485 */ 1486 if ((bringup && !sp->startup.single) || 1487 (!bringup && !sp->teardown.single)) 1488 return 0; 1489 /* 1490 * The non AP bound callbacks can fail on bringup. On teardown 1491 * e.g. module removal we crash for now. 1492 */ 1493 #ifdef CONFIG_SMP 1494 if (cpuhp_is_ap_state(state)) 1495 ret = cpuhp_invoke_ap_callback(cpu, state, bringup, node); 1496 else 1497 ret = cpuhp_invoke_callback(cpu, state, bringup, node, NULL); 1498 #else 1499 ret = cpuhp_invoke_callback(cpu, state, bringup, node, NULL); 1500 #endif 1501 BUG_ON(ret && !bringup); 1502 return ret; 1503 } 1504 1505 /* 1506 * Called from __cpuhp_setup_state on a recoverable failure. 1507 * 1508 * Note: The teardown callbacks for rollback are not allowed to fail! 1509 */ 1510 static void cpuhp_rollback_install(int failedcpu, enum cpuhp_state state, 1511 struct hlist_node *node) 1512 { 1513 int cpu; 1514 1515 /* Roll back the already executed steps on the other cpus */ 1516 for_each_present_cpu(cpu) { 1517 struct cpuhp_cpu_state *st = per_cpu_ptr(&cpuhp_state, cpu); 1518 int cpustate = st->state; 1519 1520 if (cpu >= failedcpu) 1521 break; 1522 1523 /* Did we invoke the startup call on that cpu ? */ 1524 if (cpustate >= state) 1525 cpuhp_issue_call(cpu, state, false, node); 1526 } 1527 } 1528 1529 int __cpuhp_state_add_instance_cpuslocked(enum cpuhp_state state, 1530 struct hlist_node *node, 1531 bool invoke) 1532 { 1533 struct cpuhp_step *sp; 1534 int cpu; 1535 int ret; 1536 1537 lockdep_assert_cpus_held(); 1538 1539 sp = cpuhp_get_step(state); 1540 if (sp->multi_instance == false) 1541 return -EINVAL; 1542 1543 mutex_lock(&cpuhp_state_mutex); 1544 1545 if (!invoke || !sp->startup.multi) 1546 goto add_node; 1547 1548 /* 1549 * Try to call the startup callback for each present cpu 1550 * depending on the hotplug state of the cpu. 1551 */ 1552 for_each_present_cpu(cpu) { 1553 struct cpuhp_cpu_state *st = per_cpu_ptr(&cpuhp_state, cpu); 1554 int cpustate = st->state; 1555 1556 if (cpustate < state) 1557 continue; 1558 1559 ret = cpuhp_issue_call(cpu, state, true, node); 1560 if (ret) { 1561 if (sp->teardown.multi) 1562 cpuhp_rollback_install(cpu, state, node); 1563 goto unlock; 1564 } 1565 } 1566 add_node: 1567 ret = 0; 1568 hlist_add_head(node, &sp->list); 1569 unlock: 1570 mutex_unlock(&cpuhp_state_mutex); 1571 return ret; 1572 } 1573 1574 int __cpuhp_state_add_instance(enum cpuhp_state state, struct hlist_node *node, 1575 bool invoke) 1576 { 1577 int ret; 1578 1579 cpus_read_lock(); 1580 ret = __cpuhp_state_add_instance_cpuslocked(state, node, invoke); 1581 cpus_read_unlock(); 1582 return ret; 1583 } 1584 EXPORT_SYMBOL_GPL(__cpuhp_state_add_instance); 1585 1586 /** 1587 * __cpuhp_setup_state_cpuslocked - Setup the callbacks for an hotplug machine state 1588 * @state: The state to setup 1589 * @invoke: If true, the startup function is invoked for cpus where 1590 * cpu state >= @state 1591 * @startup: startup callback function 1592 * @teardown: teardown callback function 1593 * @multi_instance: State is set up for multiple instances which get 1594 * added afterwards. 1595 * 1596 * The caller needs to hold cpus read locked while calling this function. 1597 * Returns: 1598 * On success: 1599 * Positive state number if @state is CPUHP_AP_ONLINE_DYN 1600 * 0 for all other states 1601 * On failure: proper (negative) error code 1602 */ 1603 int __cpuhp_setup_state_cpuslocked(enum cpuhp_state state, 1604 const char *name, bool invoke, 1605 int (*startup)(unsigned int cpu), 1606 int (*teardown)(unsigned int cpu), 1607 bool multi_instance) 1608 { 1609 int cpu, ret = 0; 1610 bool dynstate; 1611 1612 lockdep_assert_cpus_held(); 1613 1614 if (cpuhp_cb_check(state) || !name) 1615 return -EINVAL; 1616 1617 mutex_lock(&cpuhp_state_mutex); 1618 1619 ret = cpuhp_store_callbacks(state, name, startup, teardown, 1620 multi_instance); 1621 1622 dynstate = state == CPUHP_AP_ONLINE_DYN; 1623 if (ret > 0 && dynstate) { 1624 state = ret; 1625 ret = 0; 1626 } 1627 1628 if (ret || !invoke || !startup) 1629 goto out; 1630 1631 /* 1632 * Try to call the startup callback for each present cpu 1633 * depending on the hotplug state of the cpu. 1634 */ 1635 for_each_present_cpu(cpu) { 1636 struct cpuhp_cpu_state *st = per_cpu_ptr(&cpuhp_state, cpu); 1637 int cpustate = st->state; 1638 1639 if (cpustate < state) 1640 continue; 1641 1642 ret = cpuhp_issue_call(cpu, state, true, NULL); 1643 if (ret) { 1644 if (teardown) 1645 cpuhp_rollback_install(cpu, state, NULL); 1646 cpuhp_store_callbacks(state, NULL, NULL, NULL, false); 1647 goto out; 1648 } 1649 } 1650 out: 1651 mutex_unlock(&cpuhp_state_mutex); 1652 /* 1653 * If the requested state is CPUHP_AP_ONLINE_DYN, return the 1654 * dynamically allocated state in case of success. 1655 */ 1656 if (!ret && dynstate) 1657 return state; 1658 return ret; 1659 } 1660 EXPORT_SYMBOL(__cpuhp_setup_state_cpuslocked); 1661 1662 int __cpuhp_setup_state(enum cpuhp_state state, 1663 const char *name, bool invoke, 1664 int (*startup)(unsigned int cpu), 1665 int (*teardown)(unsigned int cpu), 1666 bool multi_instance) 1667 { 1668 int ret; 1669 1670 cpus_read_lock(); 1671 ret = __cpuhp_setup_state_cpuslocked(state, name, invoke, startup, 1672 teardown, multi_instance); 1673 cpus_read_unlock(); 1674 return ret; 1675 } 1676 EXPORT_SYMBOL(__cpuhp_setup_state); 1677 1678 int __cpuhp_state_remove_instance(enum cpuhp_state state, 1679 struct hlist_node *node, bool invoke) 1680 { 1681 struct cpuhp_step *sp = cpuhp_get_step(state); 1682 int cpu; 1683 1684 BUG_ON(cpuhp_cb_check(state)); 1685 1686 if (!sp->multi_instance) 1687 return -EINVAL; 1688 1689 cpus_read_lock(); 1690 mutex_lock(&cpuhp_state_mutex); 1691 1692 if (!invoke || !cpuhp_get_teardown_cb(state)) 1693 goto remove; 1694 /* 1695 * Call the teardown callback for each present cpu depending 1696 * on the hotplug state of the cpu. This function is not 1697 * allowed to fail currently! 1698 */ 1699 for_each_present_cpu(cpu) { 1700 struct cpuhp_cpu_state *st = per_cpu_ptr(&cpuhp_state, cpu); 1701 int cpustate = st->state; 1702 1703 if (cpustate >= state) 1704 cpuhp_issue_call(cpu, state, false, node); 1705 } 1706 1707 remove: 1708 hlist_del(node); 1709 mutex_unlock(&cpuhp_state_mutex); 1710 cpus_read_unlock(); 1711 1712 return 0; 1713 } 1714 EXPORT_SYMBOL_GPL(__cpuhp_state_remove_instance); 1715 1716 /** 1717 * __cpuhp_remove_state_cpuslocked - Remove the callbacks for an hotplug machine state 1718 * @state: The state to remove 1719 * @invoke: If true, the teardown function is invoked for cpus where 1720 * cpu state >= @state 1721 * 1722 * The caller needs to hold cpus read locked while calling this function. 1723 * The teardown callback is currently not allowed to fail. Think 1724 * about module removal! 1725 */ 1726 void __cpuhp_remove_state_cpuslocked(enum cpuhp_state state, bool invoke) 1727 { 1728 struct cpuhp_step *sp = cpuhp_get_step(state); 1729 int cpu; 1730 1731 BUG_ON(cpuhp_cb_check(state)); 1732 1733 lockdep_assert_cpus_held(); 1734 1735 mutex_lock(&cpuhp_state_mutex); 1736 if (sp->multi_instance) { 1737 WARN(!hlist_empty(&sp->list), 1738 "Error: Removing state %d which has instances left.\n", 1739 state); 1740 goto remove; 1741 } 1742 1743 if (!invoke || !cpuhp_get_teardown_cb(state)) 1744 goto remove; 1745 1746 /* 1747 * Call the teardown callback for each present cpu depending 1748 * on the hotplug state of the cpu. This function is not 1749 * allowed to fail currently! 1750 */ 1751 for_each_present_cpu(cpu) { 1752 struct cpuhp_cpu_state *st = per_cpu_ptr(&cpuhp_state, cpu); 1753 int cpustate = st->state; 1754 1755 if (cpustate >= state) 1756 cpuhp_issue_call(cpu, state, false, NULL); 1757 } 1758 remove: 1759 cpuhp_store_callbacks(state, NULL, NULL, NULL, false); 1760 mutex_unlock(&cpuhp_state_mutex); 1761 } 1762 EXPORT_SYMBOL(__cpuhp_remove_state_cpuslocked); 1763 1764 void __cpuhp_remove_state(enum cpuhp_state state, bool invoke) 1765 { 1766 cpus_read_lock(); 1767 __cpuhp_remove_state_cpuslocked(state, invoke); 1768 cpus_read_unlock(); 1769 } 1770 EXPORT_SYMBOL(__cpuhp_remove_state); 1771 1772 #if defined(CONFIG_SYSFS) && defined(CONFIG_HOTPLUG_CPU) 1773 static ssize_t show_cpuhp_state(struct device *dev, 1774 struct device_attribute *attr, char *buf) 1775 { 1776 struct cpuhp_cpu_state *st = per_cpu_ptr(&cpuhp_state, dev->id); 1777 1778 return sprintf(buf, "%d\n", st->state); 1779 } 1780 static DEVICE_ATTR(state, 0444, show_cpuhp_state, NULL); 1781 1782 static ssize_t write_cpuhp_target(struct device *dev, 1783 struct device_attribute *attr, 1784 const char *buf, size_t count) 1785 { 1786 struct cpuhp_cpu_state *st = per_cpu_ptr(&cpuhp_state, dev->id); 1787 struct cpuhp_step *sp; 1788 int target, ret; 1789 1790 ret = kstrtoint(buf, 10, &target); 1791 if (ret) 1792 return ret; 1793 1794 #ifdef CONFIG_CPU_HOTPLUG_STATE_CONTROL 1795 if (target < CPUHP_OFFLINE || target > CPUHP_ONLINE) 1796 return -EINVAL; 1797 #else 1798 if (target != CPUHP_OFFLINE && target != CPUHP_ONLINE) 1799 return -EINVAL; 1800 #endif 1801 1802 ret = lock_device_hotplug_sysfs(); 1803 if (ret) 1804 return ret; 1805 1806 mutex_lock(&cpuhp_state_mutex); 1807 sp = cpuhp_get_step(target); 1808 ret = !sp->name || sp->cant_stop ? -EINVAL : 0; 1809 mutex_unlock(&cpuhp_state_mutex); 1810 if (ret) 1811 goto out; 1812 1813 if (st->state < target) 1814 ret = do_cpu_up(dev->id, target); 1815 else 1816 ret = do_cpu_down(dev->id, target); 1817 out: 1818 unlock_device_hotplug(); 1819 return ret ? ret : count; 1820 } 1821 1822 static ssize_t show_cpuhp_target(struct device *dev, 1823 struct device_attribute *attr, char *buf) 1824 { 1825 struct cpuhp_cpu_state *st = per_cpu_ptr(&cpuhp_state, dev->id); 1826 1827 return sprintf(buf, "%d\n", st->target); 1828 } 1829 static DEVICE_ATTR(target, 0644, show_cpuhp_target, write_cpuhp_target); 1830 1831 1832 static ssize_t write_cpuhp_fail(struct device *dev, 1833 struct device_attribute *attr, 1834 const char *buf, size_t count) 1835 { 1836 struct cpuhp_cpu_state *st = per_cpu_ptr(&cpuhp_state, dev->id); 1837 struct cpuhp_step *sp; 1838 int fail, ret; 1839 1840 ret = kstrtoint(buf, 10, &fail); 1841 if (ret) 1842 return ret; 1843 1844 /* 1845 * Cannot fail STARTING/DYING callbacks. 1846 */ 1847 if (cpuhp_is_atomic_state(fail)) 1848 return -EINVAL; 1849 1850 /* 1851 * Cannot fail anything that doesn't have callbacks. 1852 */ 1853 mutex_lock(&cpuhp_state_mutex); 1854 sp = cpuhp_get_step(fail); 1855 if (!sp->startup.single && !sp->teardown.single) 1856 ret = -EINVAL; 1857 mutex_unlock(&cpuhp_state_mutex); 1858 if (ret) 1859 return ret; 1860 1861 st->fail = fail; 1862 1863 return count; 1864 } 1865 1866 static ssize_t show_cpuhp_fail(struct device *dev, 1867 struct device_attribute *attr, char *buf) 1868 { 1869 struct cpuhp_cpu_state *st = per_cpu_ptr(&cpuhp_state, dev->id); 1870 1871 return sprintf(buf, "%d\n", st->fail); 1872 } 1873 1874 static DEVICE_ATTR(fail, 0644, show_cpuhp_fail, write_cpuhp_fail); 1875 1876 static struct attribute *cpuhp_cpu_attrs[] = { 1877 &dev_attr_state.attr, 1878 &dev_attr_target.attr, 1879 &dev_attr_fail.attr, 1880 NULL 1881 }; 1882 1883 static const struct attribute_group cpuhp_cpu_attr_group = { 1884 .attrs = cpuhp_cpu_attrs, 1885 .name = "hotplug", 1886 NULL 1887 }; 1888 1889 static ssize_t show_cpuhp_states(struct device *dev, 1890 struct device_attribute *attr, char *buf) 1891 { 1892 ssize_t cur, res = 0; 1893 int i; 1894 1895 mutex_lock(&cpuhp_state_mutex); 1896 for (i = CPUHP_OFFLINE; i <= CPUHP_ONLINE; i++) { 1897 struct cpuhp_step *sp = cpuhp_get_step(i); 1898 1899 if (sp->name) { 1900 cur = sprintf(buf, "%3d: %s\n", i, sp->name); 1901 buf += cur; 1902 res += cur; 1903 } 1904 } 1905 mutex_unlock(&cpuhp_state_mutex); 1906 return res; 1907 } 1908 static DEVICE_ATTR(states, 0444, show_cpuhp_states, NULL); 1909 1910 static struct attribute *cpuhp_cpu_root_attrs[] = { 1911 &dev_attr_states.attr, 1912 NULL 1913 }; 1914 1915 static const struct attribute_group cpuhp_cpu_root_attr_group = { 1916 .attrs = cpuhp_cpu_root_attrs, 1917 .name = "hotplug", 1918 NULL 1919 }; 1920 1921 static int __init cpuhp_sysfs_init(void) 1922 { 1923 int cpu, ret; 1924 1925 ret = sysfs_create_group(&cpu_subsys.dev_root->kobj, 1926 &cpuhp_cpu_root_attr_group); 1927 if (ret) 1928 return ret; 1929 1930 for_each_possible_cpu(cpu) { 1931 struct device *dev = get_cpu_device(cpu); 1932 1933 if (!dev) 1934 continue; 1935 ret = sysfs_create_group(&dev->kobj, &cpuhp_cpu_attr_group); 1936 if (ret) 1937 return ret; 1938 } 1939 return 0; 1940 } 1941 device_initcall(cpuhp_sysfs_init); 1942 #endif 1943 1944 /* 1945 * cpu_bit_bitmap[] is a special, "compressed" data structure that 1946 * represents all NR_CPUS bits binary values of 1<<nr. 1947 * 1948 * It is used by cpumask_of() to get a constant address to a CPU 1949 * mask value that has a single bit set only. 1950 */ 1951 1952 /* cpu_bit_bitmap[0] is empty - so we can back into it */ 1953 #define MASK_DECLARE_1(x) [x+1][0] = (1UL << (x)) 1954 #define MASK_DECLARE_2(x) MASK_DECLARE_1(x), MASK_DECLARE_1(x+1) 1955 #define MASK_DECLARE_4(x) MASK_DECLARE_2(x), MASK_DECLARE_2(x+2) 1956 #define MASK_DECLARE_8(x) MASK_DECLARE_4(x), MASK_DECLARE_4(x+4) 1957 1958 const unsigned long cpu_bit_bitmap[BITS_PER_LONG+1][BITS_TO_LONGS(NR_CPUS)] = { 1959 1960 MASK_DECLARE_8(0), MASK_DECLARE_8(8), 1961 MASK_DECLARE_8(16), MASK_DECLARE_8(24), 1962 #if BITS_PER_LONG > 32 1963 MASK_DECLARE_8(32), MASK_DECLARE_8(40), 1964 MASK_DECLARE_8(48), MASK_DECLARE_8(56), 1965 #endif 1966 }; 1967 EXPORT_SYMBOL_GPL(cpu_bit_bitmap); 1968 1969 const DECLARE_BITMAP(cpu_all_bits, NR_CPUS) = CPU_BITS_ALL; 1970 EXPORT_SYMBOL(cpu_all_bits); 1971 1972 #ifdef CONFIG_INIT_ALL_POSSIBLE 1973 struct cpumask __cpu_possible_mask __read_mostly 1974 = {CPU_BITS_ALL}; 1975 #else 1976 struct cpumask __cpu_possible_mask __read_mostly; 1977 #endif 1978 EXPORT_SYMBOL(__cpu_possible_mask); 1979 1980 struct cpumask __cpu_online_mask __read_mostly; 1981 EXPORT_SYMBOL(__cpu_online_mask); 1982 1983 struct cpumask __cpu_present_mask __read_mostly; 1984 EXPORT_SYMBOL(__cpu_present_mask); 1985 1986 struct cpumask __cpu_active_mask __read_mostly; 1987 EXPORT_SYMBOL(__cpu_active_mask); 1988 1989 void init_cpu_present(const struct cpumask *src) 1990 { 1991 cpumask_copy(&__cpu_present_mask, src); 1992 } 1993 1994 void init_cpu_possible(const struct cpumask *src) 1995 { 1996 cpumask_copy(&__cpu_possible_mask, src); 1997 } 1998 1999 void init_cpu_online(const struct cpumask *src) 2000 { 2001 cpumask_copy(&__cpu_online_mask, src); 2002 } 2003 2004 /* 2005 * Activate the first processor. 2006 */ 2007 void __init boot_cpu_init(void) 2008 { 2009 int cpu = smp_processor_id(); 2010 2011 /* Mark the boot cpu "present", "online" etc for SMP and UP case */ 2012 set_cpu_online(cpu, true); 2013 set_cpu_active(cpu, true); 2014 set_cpu_present(cpu, true); 2015 set_cpu_possible(cpu, true); 2016 2017 #ifdef CONFIG_SMP 2018 __boot_cpu_id = cpu; 2019 #endif 2020 } 2021 2022 /* 2023 * Must be called _AFTER_ setting up the per_cpu areas 2024 */ 2025 void __init boot_cpu_state_init(void) 2026 { 2027 per_cpu_ptr(&cpuhp_state, smp_processor_id())->state = CPUHP_ONLINE; 2028 } 2029